diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 64c85c1101e..fb11c82bb96 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,13 +1,10 @@ { - "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", + "image": "ghcr.io/python/devcontainer:latest", "onCreateCommand": [ // Install common tooling. "dnf", "install", "-y", - "which", - "zsh", - "fish", // For umask fix below. "/usr/bin/setfacl" ], diff --git a/.devcontainer/wasi/devcontainer.json b/.devcontainer/wasi/devcontainer.json new file mode 100644 index 00000000000..4266144ce47 --- /dev/null +++ b/.devcontainer/wasi/devcontainer.json @@ -0,0 +1,73 @@ +{ + "image": "ghcr.io/python/wasicontainer:latest", + "onCreateCommand": [ + // Install common tooling. + "dnf", + "install", + "-y", + // For umask fix below. + "/usr/bin/setfacl" + ], + "updateContentCommand": { + // Using the shell for `nproc` usage. + "python": "python3 Tools/wasm/wasi build --quiet -- --with-pydebug -C" + }, + "postCreateCommand": { + // https://github.com/orgs/community/discussions/26026 + "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."], + "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"] + }, + "customizations": { + "vscode": { + "extensions": [ + // Highlighting for Parser/Python.asdl. + "brettcannon.zephyr-asdl", + // Highlighting for configure.ac. + "maelvalais.autoconf", + // C auto-complete. + "ms-vscode.cpptools", + // Python auto-complete. + "ms-python.python" + ], + "settings": { + "C_Cpp.default.compilerPath": "/usr/bin/clang", + "C_Cpp.default.cStandard": "c11", + "C_Cpp.default.defines": [ + "CONFIG_64", + "Py_BUILD_CORE" + ], + "C_Cpp.default.includePath": [ + "${workspaceFolder}/*", + "${workspaceFolder}/Include/**" + ], + // https://github.com/microsoft/vscode-cpptools/issues/10732 + "C_Cpp.errorSquiggles": "disabled", + "editor.insertSpaces": true, + "editor.rulers": [ + 80 + ], + "editor.tabSize": 4, + "editor.trimAutoWhitespace": true, + "files.associations": { + "*.h": "c" + }, + "files.encoding": "utf8", + "files.eol": "\n", + "files.insertFinalNewline": true, + "files.trimTrailingWhitespace": true, + "python.analysis.diagnosticSeverityOverrides": { + // Complains about shadowing the stdlib w/ the stdlib. + "reportShadowedImports": "none", + // Doesn't like _frozen_importlib. + "reportMissingImports": "none" + }, + "python.analysis.extraPaths": [ + "Lib" + ], + "[restructuredtext]": { + "editor.tabSize": 3 + } + } + } + } +} diff --git a/.editorconfig b/.editorconfig index 5b04b32a89e..25bc5935258 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,11 +1,11 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml,yaml}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h}] +[*.{py,c,cpp,h,gram}] indent_size = 4 [*.rst] diff --git a/.gitattributes b/.gitattributes index 2f5a030981f..d6547212393 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,7 @@ *.ico binary *.jpg binary *.pck binary +*.pdf binary *.png binary *.psd binary *.tar binary @@ -67,6 +68,7 @@ PCbuild/readme.txt dos **/clinic/*.cpp.h generated **/clinic/*.h.h generated *_db.h generated +Doc/c-api/lifecycle.dot.svg generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated @@ -81,6 +83,7 @@ Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated +Lib/idlelib/help.html generated Lib/test/certdata/*.pem generated Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated @@ -101,3 +104,4 @@ Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated configure generated +*.min.js generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 16331b65e52..6acc156ebff 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,142 +1,261 @@ # See https://help.github.com/articles/about-codeowners/ -# for more info about CODEOWNERS file +# for further details about the .github/CODEOWNERS file. # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format +# Notably, a later match overrides earlier matches, so order matters. +# If using a wildcard pattern, try to be as specific as possible to avoid +# matching unintended files or overriding previous entries. +# To exclude a file from ownership, add a line with only the file. +# See the exclusions section at the end of the file for examples. -# GitHub -.github/** @ezio-melotti @hugovk @AA-Turner +# ======= +# Purpose +# ======= +# +# An entry in this file does not imply 'ownership', despite the name of the +# file, but instead that those listed take an interest in that part of the +# project and will automatically be added as reviewers to PRs that affect +# the matching files. +# See also the Experts Index in the Python Developer's Guide: +# https://devguide.python.org/core-developers/experts/ +# +# ========= +# Structure +# ========= +# +# The CODEOWNERS file is organised by topic area. +# Please add new entries in alphabetical order within the relevant section. +# Where possible, keep related files together. For example, documentation, +# code, and tests for a given item should all be listed in the same place. +# +# GitHub usernames should be aligned to column 31, or the next multiple +# of three if the relevant paths are too long to fit. +# +# Top-level sections are: +# +# * Buildbots, Continuous Integration, and Testing +# project-wide configuration files, internal tools for use in CI, +# linting. +# * Build System +# the Makefile, autoconf, and other autotools files. +# * Documentation +# broader sections of documentation, documentation tools +# * Internal Tools & Data +# internal tools, integration with external systems, +# entries that don't fit elsewhere +# * Platform Support +# relating to support for specific platforms +# * Interpreter Core +# the grammar, parser, compiler, interpreter, etc. +# * Standard Library +# standard library modules (from both Lib and Modules) +# and related files (such as their tests and docs) +# * Exclusions +# exclusions from .github/CODEOWNERS should go at the very end +# because the final matching pattern will take precedence. -# pre-commit -.pre-commit-config.yaml @hugovk @AlexWaygood +# ---------------------------------------------------------------------------- +# Buildbots, Continuous Integration, and Testing +# ---------------------------------------------------------------------------- + +# Azure Pipelines +.azure-pipelines/ @AA-Turner + +# GitHub & related scripts +.github/ @ezio-melotti @hugovk @AA-Turner +Tools/build/compute-changes.py @AA-Turner +Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg + +# Pre-commit +.pre-commit-config.yaml @hugovk .ruff.toml @hugovk @AlexWaygood @AA-Turner -# Build system -configure* @erlend-aasland @corona10 -Makefile.pre.in @erlend-aasland -Modules/Setup* @erlend-aasland +# Patchcheck +Tools/patchcheck/ @AA-Turner -# argparse -**/*argparse* @savannahostrowski -# asyncio -**/*asyncio* @1st1 @asvetlov @kumaraditya303 @willingc +# ---------------------------------------------------------------------------- +# Build System +# ---------------------------------------------------------------------------- -# Core -**/*context* @1st1 -**/*genobject* @markshannon -**/*hamt* @1st1 -**/*jit* @brandtbucher @savannahostrowski -Objects/set* @rhettinger -Objects/dict* @methane @markshannon -Objects/typevarobject.c @JelleZijlstra -Objects/unionobject.c @JelleZijlstra -Objects/type* @markshannon -Objects/codeobject.c @markshannon -Objects/frameobject.c @markshannon -Objects/call.c @markshannon -Python/ceval*.c @markshannon -Python/ceval*.h @markshannon -Python/codegen.c @markshannon @iritkatriel -Python/compile.c @markshannon @iritkatriel -Python/assemble.c @markshannon @iritkatriel -Python/flowgraph.c @markshannon @iritkatriel -Python/instruction_sequence.c @iritkatriel -Python/bytecodes.c @markshannon -Python/optimizer*.c @markshannon -Python/optimizer_analysis.c @Fidget-Spinner -Python/optimizer_bytecodes.c @Fidget-Spinner -Python/symtable.c @JelleZijlstra @carljm -Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv -Lib/test/test_patma.py @brandtbucher -Lib/test/test_type_*.py @JelleZijlstra -Lib/test/test_capi/test_misc.py @markshannon -Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv -Tools/c-analyzer/ @ericsnowcurrently +# Autotools +configure* @erlend-aasland @corona10 @AA-Turner @emmatyping +Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping +Modules/makesetup @erlend-aasland @AA-Turner @emmatyping +Modules/Setup* @erlend-aasland @AA-Turner @emmatyping +Tools/build/regen-configure.sh @AA-Turner -# dbm -**/*dbm* @corona10 @erlend-aasland @serhiy-storchaka +# generate-build-details +Tools/build/generate-build-details.py @FFY00 +Lib/test/test_build_details.py @FFY00 -# Doc/ tools -Doc/conf.py @AA-Turner @hugovk + +# ---------------------------------------------------------------------------- +# Documentation +# ---------------------------------------------------------------------------- + +# Internal Docs +InternalDocs/ @AA-Turner + +# Tools, Configuration, etc Doc/Makefile @AA-Turner @hugovk +Doc/_static/ @AA-Turner @hugovk +Doc/conf.py @AA-Turner @hugovk Doc/make.bat @AA-Turner @hugovk Doc/requirements.txt @AA-Turner @hugovk -Doc/_static/** @AA-Turner @hugovk -Doc/tools/** @AA-Turner @hugovk +Doc/tools/ @AA-Turner @hugovk -# runtime state/lifecycle -**/*pylifecycle* @ericsnowcurrently -**/*pystate* @ericsnowcurrently -**/*preconfig* @ericsnowcurrently -**/*initconfig* @ericsnowcurrently -**/*pathconfig* @ericsnowcurrently -**/*sysmodule* @ericsnowcurrently +# PR Previews +.readthedocs.yml @AA-Turner + +# Sections +Doc/reference/ @willingc @AA-Turner +Doc/whatsnew/ @AA-Turner + + +# ---------------------------------------------------------------------------- +# Internal Tools and Data +# ---------------------------------------------------------------------------- + +# Argument Clinic +Tools/clinic/ @erlend-aasland @AA-Turner +Lib/test/test_clinic.py @erlend-aasland @AA-Turner +Doc/howto/clinic.rst @erlend-aasland @AA-Turner + +# C Analyser +Tools/c-analyzer/ @ericsnowcurrently + +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity + +# Fuzzing +Modules/_xxtestfuzz/ @ammaraskar + +# Limited C API & Stable ABI +Doc/c-api/stable.rst @encukou +Doc/data/*.abi @encukou +Misc/stable_abi.toml @encukou +Tools/build/stable_abi.py @encukou + +# SBOM +Misc/externals.spdx.json @sethmlarson +Misc/sbom.spdx.json @sethmlarson +Tools/build/generate_sbom.py @sethmlarson + + +# ---------------------------------------------------------------------------- +# Platform Support +# ---------------------------------------------------------------------------- + +# Android +Android/ @mhsmith @freakboy3742 +Doc/using/android.rst @mhsmith @freakboy3742 +Lib/_android_support.py @mhsmith @freakboy3742 +Lib/test/test_android.py @mhsmith @freakboy3742 + +# iOS +Doc/using/ios.rst @freakboy3742 +Lib/_ios_support.py @freakboy3742 +Apple/ @freakboy3742 +iOS/ @freakboy3742 + +# macOS +Mac/ @python/macos-team +Lib/_osx_support.py @python/macos-team +Lib/test/test__osx_support.py @python/macos-team + +# WebAssembly +Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping + +# WebAssembly (Emscripten) +Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping + +# WebAssembly (WASI) +Tools/wasm/wasi-env @brettcannon @emmatyping +Tools/wasm/wasi.py @brettcannon @emmatyping +Tools/wasm/wasi @brettcannon @emmatyping + +# Windows +PC/ @python/windows-team +PCbuild/ @python/windows-team + +# Windows installer packages +Tools/msi/ @python/windows-team +Tools/nuget/ @python/windows-team + +# Windows Launcher +PC/launcher.c @python/windows-team @vsajip + + +# ---------------------------------------------------------------------------- +# Interpreter Core +# ---------------------------------------------------------------------------- + +# AST +Lib/_ast_unparse.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Lib/ast.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Lib/test/test_ast/ @eclips4 @tomasr8 +Parser/asdl.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Python/ast.c @isidentical @JelleZijlstra @eclips4 @tomasr8 +Python/ast_preprocess.c @isidentical @eclips4 @tomasr8 + +# Built-in types +Objects/call.c @markshannon +Objects/codeobject.c @markshannon +Objects/dict* @methane @markshannon +Objects/frameobject.c @markshannon +**/*genobject* @markshannon +Objects/object.c @ZeroIntensity +Objects/set* @rhettinger +Objects/type* @markshannon +Objects/typevarobject.c @JelleZijlstra +Objects/unionobject.c @JelleZijlstra + +# Byte code interpreter ('the eval loop') +Python/bytecodes.c @markshannon +Python/ceval* @markshannon +Tools/cases_generator/ @markshannon + +# Compiler (AST to byte code) +Python/assemble.c @markshannon @iritkatriel +Python/codegen.c @markshannon @iritkatriel +Python/compile.c @markshannon @iritkatriel +Python/flowgraph.c @markshannon @iritkatriel +Python/instruction_sequence.c @iritkatriel +Python/symtable.c @JelleZijlstra @carljm + +# Context variables & HAMT +**/contextvars* @1st1 +**/*hamt* @1st1 +Include/cpython/context.h @1st1 +Include/internal/pycore_context.h @1st1 +Lib/test/test_context.py @1st1 +Python/context.c @1st1 + +# Core Modules **/*bltinmodule* @ericsnowcurrently -**/*gil* @ericsnowcurrently -Include/internal/pycore_runtime.h @ericsnowcurrently -Include/internal/pycore_interp.h @ericsnowcurrently -Include/internal/pycore_tstate.h @ericsnowcurrently -Include/internal/pycore_*_state.h @ericsnowcurrently -Include/internal/pycore_*_init.h @ericsnowcurrently -Include/internal/pycore_atexit.h @ericsnowcurrently -Include/internal/pycore_freelist.h @ericsnowcurrently -Include/internal/pycore_global_objects.h @ericsnowcurrently -Include/internal/pycore_obmalloc.h @ericsnowcurrently -Include/internal/pycore_pymem.h @ericsnowcurrently -Include/internal/pycore_stackref.h @Fidget-Spinner -Modules/main.c @ericsnowcurrently -Programs/_bootstrap_python.c @ericsnowcurrently -Programs/python.c @ericsnowcurrently -Tools/build/generate_global_objects.py @ericsnowcurrently - -# Initialization -Doc/library/sys_path_init.rst @FFY00 -Doc/c-api/init_config.rst @FFY00 - -# getpath -**/*getpath* @FFY00 - -# site -**/*site.py @FFY00 -Doc/library/site.rst @FFY00 +**/*sysmodule* @ericsnowcurrently # Exceptions Lib/test/test_except*.py @iritkatriel Objects/exceptions.c @iritkatriel -# Hashing & cryptographic primitives -**/*hashlib* @gpshead @tiran @picnixz -**/*hashopenssl* @gpshead @tiran @picnixz -**/*pyhash* @gpshead @tiran @picnixz -Modules/*blake* @gpshead @tiran @picnixz -Modules/*md5* @gpshead @tiran @picnixz -Modules/*sha* @gpshead @tiran @picnixz -Modules/_hacl/** @gpshead @picnixz -**/*hmac* @gpshead @picnixz +# Getpath +Lib/test/test_getpath.py @FFY00 +Modules/getpath* @FFY00 -# libssl -**/*ssl* @gpshead @picnixz +# Hashing / ``hash()`` and related +Include/cpython/pyhash.h @gpshead @picnixz +Include/internal/pycore_pyhash.h @gpshead @picnixz +Include/pyhash.h @gpshead @picnixz +Python/pyhash.c @gpshead @picnixz -# logging -**/*logging* @vsajip - -# venv -**/*venv* @vsajip @FFY00 - -# Launcher -/PC/launcher.c @vsajip - -# HTML -/Lib/html/ @ezio-melotti -/Lib/_markupbase.py @ezio-melotti -/Lib/test/test_html*.py @ezio-melotti -/Tools/build/parse_html5_entities.py @ezio-melotti - -# Import (including importlib). +# The import system (including importlib) **/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw -/Python/import.c @kumaraditya303 -Python/dynload_*.c @ericsnowcurrently +Python/import.c @brettcannon @ericsnowcurrently @ncoghlan @warsaw @kumaraditya303 **/*freeze* @ericsnowcurrently **/*frozen* @ericsnowcurrently **/*modsupport* @ericsnowcurrently @@ -147,19 +266,171 @@ Python/dynload_*.c @ericsnowcurrently **/*pythonrun* @ericsnowcurrently **/*runpy* @ericsnowcurrently **/*singlephase* @ericsnowcurrently -Lib/test/test_module/ @ericsnowcurrently Doc/c-api/module.rst @ericsnowcurrently -**/*importlib/resources/* @jaraco @warsaw @FFY00 -**/*importlib/metadata/* @jaraco @warsaw +Lib/test/test_module/ @ericsnowcurrently +Python/dynload_*.c @ericsnowcurrently + +# Initialisation +**/*initconfig* @ericsnowcurrently +**/*pathconfig* @ericsnowcurrently +**/*preconfig* @ericsnowcurrently +Doc/library/sys_path_init.rst @FFY00 +Doc/c-api/init_config.rst @FFY00 + +# Interpreter main program +Modules/main.c @ericsnowcurrently +Programs/_bootstrap_python.c @ericsnowcurrently +Programs/python.c @ericsnowcurrently + +# JIT +Include/internal/pycore_jit.h @brandtbucher @savannahostrowski @diegorusso +Python/jit.c @brandtbucher @savannahostrowski @diegorusso +Tools/jit/ @brandtbucher @savannahostrowski @diegorusso +InternalDocs/jit.md @brandtbucher @savannahostrowski @diegorusso @AA-Turner + +# Micro-op / μop / Tier 2 Optimiser +Python/optimizer.c @markshannon +Python/optimizer_analysis.c @markshannon @tomasr8 @Fidget-Spinner +Python/optimizer_bytecodes.c @markshannon @tomasr8 @Fidget-Spinner +Python/optimizer_symbols.c @markshannon @tomasr8 + +# Parser, Lexer, and Grammar +Grammar/python.gram @pablogsal @lysnikolaou +Lib/test/test_peg_generator/ @pablogsal @lysnikolaou +Lib/test/test_tokenize.py @pablogsal @lysnikolaou +Lib/tokenize.py @pablogsal @lysnikolaou +Parser/ @pablogsal @lysnikolaou +Tools/peg_generator/ @pablogsal @lysnikolaou + +# Runtime state/lifecycle +**/*gil* @ericsnowcurrently +**/*pylifecycle* @ericsnowcurrently @ZeroIntensity +**/*pystate* @ericsnowcurrently @ZeroIntensity +Include/internal/pycore_*_init.h @ericsnowcurrently +Include/internal/pycore_*_state.h @ericsnowcurrently +Include/internal/pycore_atexit.h @ericsnowcurrently +Include/internal/pycore_freelist.h @ericsnowcurrently +Include/internal/pycore_global_objects.h @ericsnowcurrently +Include/internal/pycore_interp.h @ericsnowcurrently +Include/internal/pycore_obmalloc.h @ericsnowcurrently +Include/internal/pycore_pymem.h @ericsnowcurrently +Include/internal/pycore_runtime.h @ericsnowcurrently +Include/internal/pycore_stackref.h @Fidget-Spinner +Include/internal/pycore_tstate.h @ericsnowcurrently +Tools/build/generate_global_objects.py @ericsnowcurrently + +# Remote Debugging +Python/remote_debug.h @pablogsal +Python/remote_debugging.c @pablogsal +Modules/_remote_debugging_module.c @pablogsal @ambv @1st1 + +# Sub-Interpreters +**/*crossinterp* @ericsnowcurrently +**/*interpreteridobject.* @ericsnowcurrently +Doc/library/concurrent.interpreters.rst @ericsnowcurrently +Lib/concurrent/futures/interpreter.py @ericsnowcurrently +Lib/concurrent/interpreters/ @ericsnowcurrently +Lib/test/support/channels.py @ericsnowcurrently +Lib/test/test__interp*.py @ericsnowcurrently +Lib/test/test_interpreters/ @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently + +# Template string literals (t-strings) +Lib/test/test_tstring.py @lysnikolaou +Objects/interpolationobject.c @lysnikolaou +Objects/templateobject.c @lysnikolaou + +# Tests +Lib/test/test_patma.py @brandtbucher +Lib/test/test_type_*.py @JelleZijlstra +Lib/test/test_capi/test_misc.py @markshannon + + +# ---------------------------------------------------------------------------- +# Standard Library +# ---------------------------------------------------------------------------- + +# Annotationlib +Doc/library/annotationlib.rst @JelleZijlstra +Lib/annotationlib.py @JelleZijlstra +Lib/test/test_annotationlib.py @JelleZijlstra + +# Argparse +Doc/**/argparse*.rst @savannahostrowski +Lib/argparse.py @savannahostrowski +Lib/test/test_argparse.py @savannahostrowski + +# Asyncio +Doc/library/asyncio*.rst @1st1 @asvetlov @kumaraditya303 @willingc +InternalDocs/asyncio.md @1st1 @asvetlov @kumaraditya303 @willingc @AA-Turner +Lib/asyncio/ @1st1 @asvetlov @kumaraditya303 @willingc +Lib/test/test_asyncio/ @1st1 @asvetlov @kumaraditya303 @willingc +Modules/_asynciomodule.c @1st1 @asvetlov @kumaraditya303 @willingc + +# Bisect +Doc/library/bisect.rst @rhettinger +Lib/bisect.py @rhettinger +Lib/test/test_bisect.py @rhettinger +Modules/_bisectmodule.c @rhettinger + +# Calendar +Lib/calendar.py @AA-Turner +Lib/test/test_calendar.py @AA-Turner + +# Cryptographic Primitives and Applications +**/*hashlib* @gpshead @picnixz +**/*hashopenssl* @gpshead @picnixz +**/*hmac* @gpshead @picnixz +**/*ssl* @gpshead @picnixz +Modules/_hacl/ @gpshead @picnixz +Modules/*blake* @gpshead @picnixz +Modules/*md5* @gpshead @picnixz +Modules/*sha* @gpshead @picnixz + +# Codecs +Modules/cjkcodecs/ @corona10 +Tools/unicode/gencjkcodecs.py @corona10 + +# Collections +Doc/library/collections.abc.rst @rhettinger +Doc/library/collections.rst @rhettinger +Lib/_collections_abc.py @rhettinger +Lib/collections/ @rhettinger +Lib/test/test_collections.py @rhettinger +Modules/_collectionsmodule.c @rhettinger + +# Colorize +Lib/_colorize.py @hugovk +Lib/test/test__colorize.py @hugovk + +# Config Parser +Lib/configparser.py @jaraco +Lib/test/test_configparser.py @jaraco + +# Dataclasses +Doc/library/dataclasses.rst @ericvsmith +Lib/dataclasses.py @ericvsmith +Lib/test/test_dataclasses/ @ericvsmith # Dates and times -**/*datetime* @pganssle @abalkin -**/*str*time* @pganssle @abalkin -Doc/library/time.rst @pganssle @abalkin -Lib/test/test_time.py @pganssle @abalkin -Modules/timemodule.c @pganssle @abalkin -Python/pytime.c @pganssle @abalkin -Include/internal/pycore_time.h @pganssle @abalkin +Doc/**/*time.rst @pganssle @abalkin +Doc/library/zoneinfo.rst @pganssle +Include/datetime.h @pganssle @abalkin +Include/internal/pycore_time.h @pganssle @abalkin +Lib/test/test_zoneinfo/ @pganssle +Lib/zoneinfo/ @pganssle +Lib/*time.py @pganssle @abalkin +Lib/test/datetimetester.py @pganssle @abalkin +Lib/test/test_*time.py @pganssle @abalkin +Modules/*zoneinfo* @pganssle +Modules/*time* @pganssle @abalkin +Python/pytime.c @pganssle @abalkin + +# Dbm +Doc/library/dbm.rst @corona10 @erlend-aasland @serhiy-storchaka +Lib/dbm/ @corona10 @erlend-aasland @serhiy-storchaka +Lib/test/test_dbm*.py @corona10 @erlend-aasland @serhiy-storchaka +Modules/*dbm* @corona10 @erlend-aasland @serhiy-storchaka # Email and related **/*mail* @python/email-team @@ -168,161 +439,197 @@ Include/internal/pycore_time.h @pganssle @abalkin **/*imap* @python/email-team **/*poplib* @python/email-team -# Exclude .mailmap from being owned by @python/email-team -/.mailmap +# Ensurepip +Doc/library/ensurepip.rst @pfmoore @pradyunsg +Lib/ensurepip/ @pfmoore @pradyunsg +Lib/test/test_ensurepip.py @pfmoore @pradyunsg + +# Enum +Doc/howto/enum.rst @ethanfurman +Doc/library/enum.rst @ethanfurman +Lib/enum.py @ethanfurman +Lib/test/test_enum.py @ethanfurman +Lib/test/test_json/test_enum.py @ethanfurman + +# FTP +Doc/library/ftplib.rst @giampaolo +Lib/ftplib.py @giampaolo +Lib/test/test_ftplib.py @giampaolo + +# Functools +Doc/library/functools.rst @rhettinger +Lib/functools.py @rhettinger +Lib/test/test_functools.py @rhettinger +Modules/_functoolsmodule.c @rhettinger # Garbage collector -/Modules/gcmodule.c @pablogsal -/Doc/library/gc.rst @pablogsal +Modules/gcmodule.c @pablogsal +Doc/library/gc.rst @pablogsal -# Parser -/Parser/ @pablogsal @lysnikolaou -/Tools/peg_generator/ @pablogsal @lysnikolaou -/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou -/Grammar/python.gram @pablogsal @lysnikolaou -/Lib/tokenize.py @pablogsal @lysnikolaou -/Lib/test/test_tokenize.py @pablogsal @lysnikolaou +# Gettext +Doc/library/gettext.rst @tomasr8 +Lib/gettext.py @tomasr8 +Lib/test/test_gettext.py @tomasr8 +Tools/i18n/pygettext.py @tomasr8 -# Code generator -/Tools/cases_generator/ @markshannon +# Heapq +Doc/library/heapq* @rhettinger +Lib/heapq.py @rhettinger +Lib/test/test_heapq.py @rhettinger +Modules/_heapqmodule.c @rhettinger -# AST -Python/ast.c @isidentical @JelleZijlstra @eclips4 -Python/ast_preprocess.c @isidentical @eclips4 -Parser/asdl.py @isidentical @JelleZijlstra @eclips4 -Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 -Lib/ast.py @isidentical @JelleZijlstra @eclips4 -Lib/_ast_unparse.py @isidentical @JelleZijlstra @eclips4 -Lib/test/test_ast/ @eclips4 +# HTML +Doc/library/html* @ezio-melotti +Lib/html/ @ezio-melotti +Lib/_markupbase.py @ezio-melotti +Lib/test/test_html*.py @ezio-melotti +Tools/build/parse_html5_entities.py @ezio-melotti -# Mock -/Lib/unittest/mock.py @cjw296 -/Lib/test/test_unittest/testmock/* @cjw296 +# IDLE +Doc/library/idle.rst @terryjreedy +Lib/idlelib/ @terryjreedy +Lib/turtledemo/ @terryjreedy -# multiprocessing -**/*multiprocessing* @gpshead +# importlib.metadata +Doc/library/importlib.metadata.rst @jaraco @warsaw +Lib/importlib/metadata/ @jaraco @warsaw +Lib/test/test_importlib/metadata/ @jaraco @warsaw + +# importlib.resources +Doc/library/importlib.resources.abc.rst @jaraco @warsaw +Doc/library/importlib.resources.rst @jaraco @warsaw +Lib/importlib/resources/ @jaraco @warsaw @FFY00 +Lib/test/test_importlib/resources/ @jaraco @warsaw @FFY00 + +# Itertools +Doc/library/itertools.rst @rhettinger +Lib/test/test_itertools.py @rhettinger +Modules/itertoolsmodule.c @rhettinger + +# Logging +Doc/**/logging* @vsajip +Lib/logging/ @vsajip +Lib/test/test_logging.py @vsajip + +# Multiprocessing +Doc/library/multiprocessing*.rst @gpshead +Lib/multiprocessing/ @gpshead +Lib/test/*multiprocessing.py @gpshead +Lib/test/test_multiprocessing*/ @gpshead +Modules/_multiprocessing/ @gpshead + +# Pathlib +Doc/library/pathlib.rst @barneygale +Lib/pathlib/ @barneygale +Lib/test/test_pathlib/ @barneygale + +# Pdb & Bdb +Doc/library/bdb.rst @gaogaotiantian +Doc/library/pdb.rst @gaogaotiantian +Lib/bdb.py @gaogaotiantian +Lib/pdb.py @gaogaotiantian +Lib/test/test_bdb.py @gaogaotiantian +Lib/test/test_pdb.py @gaogaotiantian +Lib/test/test_remote_pdb.py @gaogaotiantian + +# Pydoc +Lib/pydoc.py @AA-Turner +Lib/pydoc_data/ @AA-Turner +Lib/test/test_pydoc/ @AA-Turner + +# PyREPL +Lib/_pyrepl/ @pablogsal @lysnikolaou @ambv +Lib/test/test_pyrepl/ @pablogsal @lysnikolaou @ambv + +# Random +Doc/library/random.rst @rhettinger +Lib/random.py @rhettinger +Lib/test/test_random.py @rhettinger +Modules/_randommodule.c @rhettinger + +# Shutil +Doc/library/shutil.rst @giampaolo +Lib/shutil.py @giampaolo +Lib/test/test_shutil.py @giampaolo + +# Site +Lib/site.py @FFY00 +Lib/test/test_site.py @FFY00 +Doc/library/site.rst @FFY00 + +# string.templatelib +Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner +Lib/string/templatelib.py @lysnikolaou @AA-Turner +Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner + +# Sysconfig +**/*sysconfig* @FFY00 # SQLite 3 -**/*sqlite* @berkerpeksag @erlend-aasland +Doc/library/sqlite3.rst @berkerpeksag @erlend-aasland +Lib/sqlite3/ @berkerpeksag @erlend-aasland +Lib/test/test_sqlite3/ @berkerpeksag @erlend-aasland +Modules/_sqlite/ @berkerpeksag @erlend-aasland -# subprocess -/Lib/subprocess.py @gpshead -/Lib/test/test_subprocess.py @gpshead -/Modules/*subprocess* @gpshead +# Subprocess +Lib/subprocess.py @gpshead +Lib/test/test_subprocess.py @gpshead +Modules/*subprocess* @gpshead -# debugger -**/*pdb* @gaogaotiantian -**/*bdb* @gaogaotiantian +# Tarfile +Doc/library/tarfile.rst @ethanfurman +Lib/tarfile.py @ethanfurman +Lib/test/test_tarfile.py @ethanfurman -# Limited C API & stable ABI -Tools/build/stable_abi.py @encukou -Misc/stable_abi.toml @encukou -Doc/data/*.abi @encukou -Doc/c-api/stable.rst @encukou +# TOML +Doc/library/tomllib.rst @encukou @hauntsaninja +Lib/test/test_tomllib/ @encukou @hauntsaninja +Lib/tomllib/ @encukou @hauntsaninja -# Windows -/PC/ @python/windows-team -/PCbuild/ @python/windows-team +# Typing +Doc/library/typing.rst @JelleZijlstra @AlexWaygood +Lib/test/test_typing.py @JelleZijlstra @AlexWaygood +Lib/test/typinganndata/ @JelleZijlstra @AlexWaygood +Lib/typing.py @JelleZijlstra @AlexWaygood +Modules/_typingmodule.c @JelleZijlstra @AlexWaygood + +# Types +Lib/test/test_types.py @AA-Turner +Lib/types.py @AA-Turner +Modules/_typesmodule.c @AA-Turner + +# Unittest +Lib/unittest/mock.py @cjw296 +Lib/test/test_unittest/testmock/ @cjw296 # Urllib **/*robotparser* @berkerpeksag -# Windows installer packages -/Tools/msi/ @python/windows-team -/Tools/nuget/ @python/windows-team - -# Misc -**/*itertools* @rhettinger -**/*collections* @rhettinger -**/*random* @rhettinger -**/*bisect* @rhettinger -**/*heapq* @rhettinger -**/*functools* @rhettinger - -**/*dataclasses* @ericvsmith - -**/*ensurepip* @pfmoore @pradyunsg - -/Doc/library/idle.rst @terryjreedy -**/*idlelib* @terryjreedy -**/*turtledemo* @terryjreedy - -**/*annotationlib* @JelleZijlstra -**/*typing* @JelleZijlstra @AlexWaygood - -**/*ftplib @giampaolo -**/*shutil @giampaolo - -**/*enum* @ethanfurman -**/*cgi* @ethanfurman -**/*tarfile* @ethanfurman - -**/*tomllib* @encukou @hauntsaninja - -**/*sysconfig* @FFY00 - -**/*cjkcodecs* @corona10 - -# macOS -/Mac/ @python/macos-team -**/*osx_support* @python/macos-team - -# pathlib -**/*pathlib* @barneygale - -# zipfile.Path -**/*zipfile/_path/* @jaraco - -# Argument Clinic -/Tools/clinic/** @erlend-aasland -/Lib/test/test_clinic.py @erlend-aasland -Doc/howto/clinic.rst @erlend-aasland - -# Subinterpreters -**/*interpreteridobject.* @ericsnowcurrently -**/*crossinterp* @ericsnowcurrently -Lib/test/support/interpreters/ @ericsnowcurrently -Modules/_interp*module.c @ericsnowcurrently -Lib/test/test_interpreters/ @ericsnowcurrently - -# Android -**/*Android* @mhsmith @freakboy3742 -**/*android* @mhsmith @freakboy3742 - -# iOS (but not termios) -**/iOS* @freakboy3742 -**/ios* @freakboy3742 -**/*_iOS* @freakboy3742 -**/*_ios* @freakboy3742 -**/*-iOS* @freakboy3742 -**/*-ios* @freakboy3742 - -# WebAssembly -/Tools/wasm/ @brettcannon @freakboy3742 - -# SBOM -/Misc/externals.spdx.json @sethmlarson -/Misc/sbom.spdx.json @sethmlarson -/Tools/build/generate_sbom.py @sethmlarson - -# Config Parser -Lib/configparser.py @jaraco -Lib/test/test_configparser.py @jaraco - -# Doc sections -Doc/reference/ @willingc @AA-Turner +# Venv +**/*venv* @vsajip @FFY00 +# Weakref **/*weakref* @kumaraditya303 -# Colorize -Lib/_colorize.py @hugovk -Lib/test/test__colorize.py @hugovk +# Zipfile.Path +Lib/test/test_zipfile/_path/ @jaraco +Lib/zipfile/_path/ @jaraco -# Fuzzing -Modules/_xxtestfuzz/ @ammaraskar +# Zstandard +Lib/compression/zstd/ @AA-Turner @emmatyping +Lib/test/test_zstd.py @AA-Turner @emmatyping +Modules/_zstd/ @AA-Turner @emmatyping -# t-strings -**/*interpolationobject* @lysnikolaou -**/*templateobject* @lysnikolaou -**/*templatelib* @lysnikolaou -**/*tstring* @lysnikolaou +# ---------------------------------------------------------------------------- + +# Exclusions from .github/CODEOWNERS should go at the very end +# because the final matching pattern will take precedence. + +# Exclude .mailmap from being owned by @python/email-team +.mailmap + +# Exclude Argument Clinic directories +Modules/**/clinic/ +Objects/**/clinic/ +PC/**/clinic/ +Python/**/clinic/ diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 2ef9cdc1914..94b67ce3dbe 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -4,7 +4,7 @@ Contributing to Python Build Status ------------ -- `Buildbot status overview `_ +- `Buildbot status overview `_ - `GitHub Actions status `_ @@ -28,23 +28,23 @@ Please be aware that our workflow does deviate slightly from the typical GitHub project. Details on how to properly submit a pull request are covered in `Lifecycle of a Pull Request `_. We utilize various bots and status checks to help with this, so do follow the -comments they leave and their "Details" links, respectively. The key points of -our workflow that are not covered by a bot or status check are: +comments they leave and their "Details" links, respectively. -- All discussions that are not directly related to the code in the pull request - should happen on `GitHub Issues `_. -- Upon your first non-trivial pull request (which includes documentation changes), - feel free to add yourself to ``Misc/ACKS`` +The final key part of our workflow is that all discussions that are not +directly related to the code in the pull request should happen on +`GitHub Issues `__, generally in the +pull request's parent issue. Setting Expectations -------------------- -Due to the fact that this project is entirely volunteer-run (i.e. no one is paid -to work on Python full-time), we unfortunately can make no guarantees as to if +Due to the fact that this project is run by volunteers, +unfortunately we cannot make any guarantees as to if or when a core developer will get around to reviewing your pull request. If no core developer has done a review or responded to changes made because of a -"changes requested" review, please feel free to email python-dev to ask if -someone could take a look at your pull request. +"changes requested" review within a month, you can ask for someone to +review your pull request via a post in the `Core Development Discourse +category `__. Code of Conduct diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 7b7810cf696..da70710b7ec 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -40,6 +40,7 @@ body: - "3.12" - "3.13" - "3.14" + - "3.15" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 75d174307ce..de6e8756b03 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,3 +5,6 @@ contact_links: - name: "Proposing new features" about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first." url: "https://discuss.python.org/c/ideas/6" + - name: "Python Install Manager issues" + about: "Report issues with the Python Install Manager (for Windows)" + url: "https://github.com/python/pymanager/issues" diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index 58da2dfe0c7..470ad581367 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -33,6 +33,7 @@ body: - "3.12" - "3.13" - "3.14" + - "3.15" - "CPython main branch" validations: required: true diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 68aae196357..267ff6b42a8 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,6 +1,7 @@ self-hosted-runner: # Pending https://github.com/rhysd/actionlint/issues/533 - labels: ["windows-11-arm"] + # and https://github.com/rhysd/actionlint/issues/571 + labels: ["windows-11-arm", "macos-15-intel"] config-variables: null diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8a3165d690..7f3376f8ddb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,11 @@ updates: update-types: - "version-update:semver-minor" - "version-update:semver-patch" + cooldown: + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # Cooldowns protect against supply chain attacks by avoiding the + # highest-risk window immediately after new releases. + default-days: 14 - package-ecosystem: "pip" directory: "/Tools/" schedule: @@ -19,3 +24,5 @@ updates: labels: - "skip issue" - "skip news" + cooldown: + default-days: 14 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b192508c786..3d889fa128e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,13 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency + # 'group' must be a key uniquely representing a PR or push event. + # github.workflow is the workflow name + # github.actor is the user invoking the workflow + # github.head_ref is the source branch of the PR or otherwise blank + # github.run_id is a unique number for the current run + group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true env: @@ -103,20 +109,10 @@ jobs: python-version: '3.x' - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -124,7 +120,7 @@ jobs: - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names regen-sbom regen-unicodedata + make regen-stdlib-module-names regen-sbom - name: Check for changes run: | git add -u @@ -146,6 +142,9 @@ jobs: - name: Check for unsupported C global variables if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + - name: Check for undocumented C APIs + run: make check-c-api-docs + build-windows: name: >- @@ -172,8 +171,8 @@ jobs: free-threading: ${{ matrix.free-threading }} build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-msi) strategy: @@ -196,28 +195,19 @@ jobs: strategy: fail-fast: false matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. + # macos-14 is M1, macos-15-intel is Intel. + # macos-15-intel only runs tests against the GIL-enabled CPython. os: - - ghcr.io/cirruslabs/macos-runner:sonoma - macos-14 - - macos-13 - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} + - macos-15-intel free-threading: - false - true exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - - os: macos-13 + - os: macos-15-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -249,12 +239,11 @@ jobs: bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} - build-ubuntu-ssltests: + build-ubuntu-ssltests-openssl: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ${{ matrix.os }} timeout-minutes: 60 @@ -264,7 +253,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] + # Keep 1.1.1w in our list despite it being upstream EOL and otherwise + # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs + # supported by important vendors such as AWS-LC. + openssl_ver: [1.1.1w, 3.0.18, 3.2.6, 3.3.5, 3.4.3, 3.5.4] # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -277,11 +269,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -303,10 +290,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython @@ -316,13 +299,122 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build-ubuntu-ssltests-awslc: + name: 'Ubuntu SSL tests with AWS-LC' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + awslc_ver: [1.55.0] + env: + AWSLC_VER: ${{ matrix.awslc_ver}} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure SSL lib env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore AWS-LC build' + id: cache-aws-lc + uses: actions/cache@v4 + with: + path: ./multissl/aws-lc/${{ matrix.awslc_ver }} + key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} + - name: Install AWS-LC + if: steps.cache-aws-lc.outputs.cache-hit != 'true' + run: | + python3 Tools/ssl/multissltests.py \ + --steps=library \ + --base-directory "$MULTISSL_DIR" \ + --awslc ${{ matrix.awslc_ver }} \ + --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure CPython + run: | + ./configure CFLAGS="-fdiagnostics-format=json" \ + --config-cache \ + --enable-slower-safety \ + --with-pydebug \ + --with-openssl="$OPENSSL_DIR" \ + --with-builtin-hashlib-hashes=blake2 \ + --with-ssl-default-suites=openssl + - name: Build CPython + run: make -j + - name: Display build info + run: make pythoninfo + - name: Verify python is linked to AWS-LC + run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: SSL tests + run: ./python Lib/test/ssltests.py + + build-android: + name: Android (${{ matrix.arch }}) + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - arch: aarch64 + runs-on: macos-14 + - arch: x86_64 + runs-on: ubuntu-24.04 + + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build and test + run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android + + build-ios: + name: iOS + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + timeout-minutes: 60 + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + # GitHub recommends explicitly selecting the desired Xcode version: + # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140 + # This became a necessity as a result of + # https://github.com/actions/runner-images/issues/12541 and + # https://github.com/actions/runner-images/issues/12751. + - name: Select Xcode version + run: | + sudo xcode-select --switch /Applications/Xcode_16.4.app + + - name: Build and test + run: python3 Apple ci iOS --fast-ci --simulator 'iPhone 16e,OS=18.5' + build-wasi: name: 'WASI' needs: build-context if: needs.build-context.outputs.run-tests == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} test-hypothesis: name: "Hypothesis tests on Ubuntu" @@ -331,7 +423,7 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 @@ -358,10 +450,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -372,11 +460,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -451,7 +534,7 @@ jobs: matrix: os: [ubuntu-24.04] env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: @@ -460,11 +543,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -490,11 +568,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -504,21 +577,28 @@ jobs: - name: Tests run: xvfb-run make ci - build-tsan: - name: >- - Thread sanitizer - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + build-san: + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: fail-fast: false matrix: + check-name: + - Thread free-threading: - false - true - uses: ./.github/workflows/reusable-tsan.yml + sanitizer: + - TSan + include: + - check-name: Undefined behavior + sanitizer: UBSan + free-threading: false + uses: ./.github/workflows/reusable-san.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} + sanitizer: ${{ matrix.sanitizer }} free-threading: ${{ matrix.free-threading }} cross-build-linux: @@ -533,11 +613,6 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir @@ -614,11 +689,14 @@ jobs: - build-windows-msi - build-macos - build-ubuntu - - build-ubuntu-ssltests + - build-ubuntu-ssltests-awslc + - build-ubuntu-ssltests-openssl + - build-android + - build-ios - build-wasi - test-hypothesis - build-asan - - build-tsan + - build-san - cross-build-linux - cifuzz if: always() @@ -629,7 +707,8 @@ jobs: with: allowed-failures: >- build-windows-msi, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, test-hypothesis, cifuzz, allowed-skips: >- @@ -647,11 +726,14 @@ jobs: check-generated-files, build-macos, build-ubuntu, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, + build-android, + build-ios, build-wasi, test-hypothesis, build-asan, - build-tsan, + build-san, cross-build-linux, ' || '' diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 116e0c1e945..62325250bd3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -5,6 +5,8 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' + - 'Python/executor_cases.c.h' + - 'Python/optimizer_cases.c.h' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' @@ -13,6 +15,8 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' + - 'Python/executor_cases.c.h' + - 'Python/optimizer_cases.c.h' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' @@ -64,20 +68,20 @@ jobs: - true - false llvm: - - 19 + - 21 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 - runner: windows-latest + runner: windows-2022 - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-latest + runner: windows-2022 - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 - runner: macos-13 + runner: macos-15-intel - target: aarch64-apple-darwin/clang architecture: aarch64 runner: macos-14 @@ -102,17 +106,16 @@ jobs: ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - name: macOS if: runner.os == 'macOS' run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" + # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to + # make sure we don't break downstream distributors (like uv): + export CFLAGS_JIT='-Werror=unguarded-availability' + export MACOSX_DEPLOYMENT_TARGET=10.15 ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -126,30 +129,81 @@ jobs: make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # XXX: GH-133171 - # jit-with-disabled-gil: - # name: Free-Threaded (Debug) - # needs: interpreter - # runs-on: ubuntu-24.04 - # timeout-minutes: 90 - # strategy: - # fail-fast: false - # matrix: - # llvm: - # - 19 - # steps: - # - uses: actions/checkout@v4 - # with: - # persist-credentials: false - # - uses: actions/setup-python@v5 - # with: - # python-version: '3.11' - # - name: Build with JIT enabled and GIL disabled - # run: | - # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - # ./configure --enable-experimental-jit --with-pydebug --disable-gil - # make all --jobs 4 - # - name: Run tests - # run: | - # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + jit-with-disabled-gil: + name: Free-Threaded (Debug) + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT enabled and GIL disabled + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit --with-pydebug --disable-gil + make all --jobs 4 + - name: Run tests + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + continue-on-error: true + + no-opt-jit: + name: JIT without optimizations (Debug) + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit --with-pydebug + make all --jobs 4 + - name: Run tests without optimizations + run: | + PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + tail-call-jit: + name: JIT with tail calling interpreter + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build with JIT and tailcall + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug + make all --jobs 4 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 908daaf3a60..fac0fa8aba3 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,13 +13,23 @@ on: - "Lib/test/libregrtest/**" - "Lib/tomllib/**" - "Misc/mypy/**" + - "Tools/build/check_extension_modules.py" + - "Tools/build/check_warnings.py" + - "Tools/build/compute-changes.py" + - "Tools/build/consts_getter.py" + - "Tools/build/deepfreeze.py" + - "Tools/build/generate-build-details.py" - "Tools/build/generate_sbom.py" + - "Tools/build/generate_stdlib_module_names.py" + - "Tools/build/mypy.ini" + - "Tools/build/umarshal.py" + - "Tools/build/update_file.py" + - "Tools/build/verify_ensurepip_wheels.py" - "Tools/cases_generator/**" - "Tools/clinic/**" - "Tools/jit/**" - "Tools/peg_generator/**" - "Tools/requirements-dev.txt" - - "Tools/wasm/**" workflow_dispatch: permissions: @@ -51,7 +61,6 @@ jobs: "Tools/clinic", "Tools/jit", "Tools/peg_generator", - "Tools/wasm", ] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index d5538cd9367..0b64367e6c4 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -5,6 +5,7 @@ apt-get -yq install \ build-essential \ pkg-config \ ccache \ + cmake \ gdb \ lcov \ libb2-dev \ @@ -17,6 +18,7 @@ apt-get -yq install \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ + libzstd-dev \ lzma \ lzma-dev \ strace \ @@ -24,3 +26,10 @@ apt-get -yq install \ uuid-dev \ xvfb \ zlib1g-dev + +# Workaround missing libmpdec-dev on ubuntu 24.04: +# https://launchpad.net/~ondrej/+archive/ubuntu/php +# https://deb.sury.org/ +sudo add-apt-repository ppa:ondrej/php +apt-get update +apt-get -yq install libmpdec-dev diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml deleted file mode 100644 index 1d9d637ec84..00000000000 --- a/.github/workflows/project-updater.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Update GH projects - -on: - issues: - types: - - opened - - labeled - -permissions: - contents: read - -jobs: - add-to-project: - name: Add issues to projects - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - include: - # if an issue has any of these labels, it will be added - # to the corresponding project - - { project: 2, label: "release-blocker, deferred-blocker" } - - { project: 32, label: sprint } - - steps: - - uses: actions/add-to-project@v1.0.0 - with: - project-url: https://github.com/orgs/python/projects/${{ matrix.project }} - github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} - labeled: ${{ matrix.label }} diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index 73dc254edc5..66c7cc47de0 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -17,9 +17,6 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config-hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool @@ -42,7 +39,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} run-docs: ${{ steps.changes.outputs.run-docs }} run-tests: ${{ steps.changes.outputs.run-tests }} @@ -97,10 +93,6 @@ jobs: run: python Tools/build/compute-changes.py env: GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 657e0a6bf66..65154aae4c4 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -66,7 +66,7 @@ jobs: run: | set -Eeuo pipefail # Build docs with the nit-picky option; write warnings to file - make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --warning-file sphinx-warnings.txt" html + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --warning-file sphinx-warnings.txt" html - name: 'Check warnings' if: github.event_name == 'pull_request' run: | @@ -102,3 +102,30 @@ jobs: # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning" doctest + + check-epub: + name: 'Check EPUB' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: 'Set up Python' + uses: actions/setup-python@v5 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: | + make -C Doc/ venv + python -m pip install epubcheck + - name: 'Build EPUB documentation' + run: make -C Doc/ PYTHON=../python epub + - name: 'Run epubcheck' + continue-on-error: true + run: epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt + - run: cat Doc/epubcheck.txt + - name: 'Check for fatal errors in EPUB' + run: python Doc/tools/check-epub.py diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index de0c4022136..98d557ba1ea 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean @@ -36,16 +33,11 @@ jobs: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make + brew install pkg-config openssl@3.0 xz gdbm tcl-tk@9 make # Because alternate versions are not symlinked into place by default: - brew link --overwrite tcl-tk@8 + brew link --overwrite tcl-tk@9 - name: Configure CPython run: | MACOSX_DEPLOYMENT_TARGET=10.15 \ @@ -60,15 +52,15 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-15-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml new file mode 100644 index 00000000000..c601d0b7338 --- /dev/null +++ b/.github/workflows/reusable-san.yml @@ -0,0 +1,111 @@ +name: Reusable Sanitizer + +on: + workflow_call: + inputs: + sanitizer: + required: true + type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + +env: + FORCE_COLOR: 1 + +jobs: + build-san-reusable: + name: >- + ${{ inputs.sanitizer }}${{ + inputs.free-threading + && ' (free-threading)' + || '' + }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + + if [ "${SANITIZER}" = "TSan" ]; then + sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 + sudo update-alternatives --set clang /usr/bin/clang-17 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-17 + # Reduce ASLR to avoid TSan crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + else + sudo ./llvm.sh 20 + fi + + - name: Sanitizer option setup + run: | + if [ "${SANITIZER}" = "TSan" ]; then + echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ + fromJSON(inputs.free-threading) + && '_free_threading' + || '' + }}.txt handle_segv=0" >> "$GITHUB_ENV" + else + echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" + fi + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + env: + SANITIZER: ${{ inputs.sanitizer }} + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure CPython + run: >- + ./configure + --config-cache + ${{ + inputs.sanitizer == 'TSan' + && '--with-thread-sanitizer' + || '--with-undefined-behavior-sanitizer' + }} + --with-pydebug + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: >- + ./python -m test + ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} + -j4 + - name: Parallel tests + if: >- + inputs.sanitizer == 'TSan' + && fromJSON(inputs.free-threading) + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + - name: Display logs + if: always() + run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 + - name: Archive logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: >- + ${{ inputs.sanitizer }}-logs-${{ + fromJSON(inputs.free-threading) + && 'free-threading' + || 'default' + }} + path: san_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml deleted file mode 100644 index 6a58e5305f8..00000000000 --- a/.github/workflows/reusable-tsan.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Reusable Thread Sanitizer - -on: - workflow_call: - inputs: - config_hash: - required: true - type: string - free-threading: - description: Whether to use free-threaded mode - required: false - type: boolean - default: false - -env: - FORCE_COLOR: 1 - -jobs: - build-tsan-reusable: - name: 'Thread sanitizer' - runs-on: ubuntu-24.04 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - - name: Install dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - # Install clang-18 - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 - # Reduce ASLR to avoid TSAN crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - - name: TSAN option setup - run: | - echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ - fromJSON(inputs.free-threading) - && '_free_threading' - || '' - }}.txt handle_segv=0" >> "$GITHUB_ENV" - echo "CC=clang" >> "$GITHUB_ENV" - echo "CXX=clang++" >> "$GITHUB_ENV" - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: >- - ./configure - --config-cache - --with-thread-sanitizer - --with-pydebug - ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: ./python -m test --tsan -j4 - - name: Parallel tests - if: fromJSON(inputs.free-threading) - run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 - - name: Display TSAN logs - if: always() - run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 - - name: Archive TSAN logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: >- - tsan-logs-${{ - fromJSON(inputs.free-threading) - && 'free-threading' - || 'default' - }} - path: tsan_log.* - if-no-files-found: ignore diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 76b19fd5d1a..0c1ebe29ae3 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,9 +3,6 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string bolt-optimizations: description: Whether to enable BOLT optimizations required: false @@ -30,7 +27,7 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.0.15 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: @@ -64,11 +61,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -79,11 +71,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} # `test_unpickle_module_race` writes to the source directory, which is diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 6beb91e66d4..91d76fd1b5f 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,10 +2,6 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string env: FORCE_COLOR: 1 @@ -13,11 +9,11 @@ env: jobs: build-wasi-reusable: name: 'build and test' - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-arm timeout-minutes: 60 env: - WASMTIME_VERSION: 22.0.0 - WASI_SDK_VERSION: 24 + WASMTIME_VERSION: 38.0.3 + WASI_SDK_VERSION: 29 WASI_SDK_PATH: /opt/wasi-sdk CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 @@ -40,13 +36,8 @@ jobs: if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' run: | mkdir "${WASI_SDK_PATH}" && \ - curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \ + curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-arm64-linux.tar.gz" | \ tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - - name: "Configure ccache action" - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: "Add ccache to PATH" run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: "Install Python" @@ -55,29 +46,15 @@ jobs: python-version: '3.x' - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: "Restore Python build config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. - # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" - run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python - - name: "Restore host config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + run: python3 Tools/wasm/wasi make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + run: python3 Tools/wasm/wasi configure-host -- --config-cache - name: "Make host" - run: python3 Tools/wasm/wasi.py make-host + run: python3 Tools/wasm/wasi make-host - name: "Display build info" run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index a50de344bba..c95e40a3809 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -17,7 +17,7 @@ env: jobs: build: name: installer for ${{ inputs.arch }} - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 37c802095b0..0648b770753 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -21,7 +21,7 @@ env: jobs: build: name: Build and test (${{ inputs.arch }}) - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 4636372e26c..e99e317182e 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -49,16 +49,16 @@ jobs: include: # - target: i686-pc-windows-msvc/msvc # architecture: Win32 -# runner: windows-latest +# runner: windows-2022 - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-latest + runner: windows-2022 # - target: aarch64-pc-windows-msvc/msvc # architecture: ARM64 -# runner: windows-latest +# runner: windows-2022 - target: x86_64-apple-darwin/clang architecture: x86_64 - runner: macos-13 + runner: macos-15-intel - target: aarch64-apple-darwin/clang architecture: aarch64 runner: macos-14 @@ -101,21 +101,14 @@ jobs: set LLVMInstallDir=C:\Program Files\LLVM ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - # Note: when a new LLVM is released, the homebrew installation directory changes, so the builds will fail. - # We either need to upgrade LLVM or change the directory being pointed to. - name: Native macOS (release) if: runner.os == 'macOS' run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - export PATH="/usr/local/opt/llvm/bin:$PATH" - export PATH="/opt/homebrew/opt/llvm/bin:$PATH" + export PATH="/usr/local/opt/llvm@${{ matrix.llvm }}/bin:$PATH" + export PATH="/opt/homebrew/opt/llvm@${{ matrix.llvm }}/bin:$PATH" CC=clang-20 ./configure --with-tail-call-interp make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -137,4 +130,3 @@ jobs: CC=clang-20 ./configure --with-tail-call-interp --disable-gil make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - diff --git a/.gitignore b/.gitignore index 2a6f249275c..4ea2fd96554 100644 --- a/.gitignore +++ b/.gitignore @@ -71,16 +71,15 @@ Lib/test/data/* /Makefile /Makefile.pre /iOSTestbed.* -iOS/Frameworks/ -iOS/Resources/Info.plist -iOS/testbed/build -iOS/testbed/Python.xcframework/ios-*/bin -iOS/testbed/Python.xcframework/ios-*/include -iOS/testbed/Python.xcframework/ios-*/lib -iOS/testbed/Python.xcframework/ios-*/Python.framework -iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace -iOS/testbed/iOSTestbed.xcodeproj/xcuserdata -iOS/testbed/iOSTestbed.xcodeproj/xcshareddata +Apple/iOS/Frameworks/ +Apple/iOS/Resources/Info.plist +Apple/testbed/build +Apple/testbed/Python.xcframework/*/bin +Apple/testbed/Python.xcframework/*/include +Apple/testbed/Python.xcframework/*/lib +Apple/testbed/Python.xcframework/*/Python.framework +Apple/testbed/*Testbed.xcodeproj/project.xcworkspace +Apple/testbed/*Testbed.xcodeproj/xcuserdata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -131,11 +130,11 @@ Tools/unicode/data/ /autom4te.cache /build/ /builddir/ +/compile_commands.json /config.cache /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ /jit_stencils*.h @@ -171,5 +170,10 @@ Python/frozen_modules/MANIFEST /python !/Python/ +# People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. +/.claude/ +CLAUDE.local.md + +#### main branch only stuff below this line, things to backport go above. #### # main branch only: ABI files are not checked/maintained. Doc/data/python*.abi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7ad829c94d5..c5767ee841e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,43 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: v0.13.2 hooks: - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple/ + - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/ - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Tools/i18n/ + args: [--exit-non-zero-on-fix, --config=Tools/i18n/.ruff.toml] + files: ^Tools/i18n/ + - id: ruff-check name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - id: ruff-check + name: Run Ruff (lint) on Tools/peg_generator/ + args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml] + files: ^Tools/peg_generator/ + - id: ruff-check + name: Run Ruff (lint) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple - id: ruff-format name: Run Ruff (format) on Doc/ args: [--check] @@ -26,16 +46,26 @@ repos: name: Run Ruff (format) on Tools/build/check_warnings.py args: [--check, --config=Tools/build/.ruff.toml] files: ^Tools/build/check_warnings.py + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--check, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 + rev: 25.9.0 hooks: - id: black name: Run Black on Tools/jit/ files: ^Tools/jit/ + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.5 + hooks: + - id: remove-tabs + types: [python] + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -43,13 +73,17 @@ repos: exclude: ^Lib/test/test_tomllib/ - id: check-yaml - id: end-of-file-fixer - types: [python] + types_or: [python, yaml] exclude: Lib/test/tokenizedata/coding20731.py + - id: end-of-file-fixer + files: '^\.github/CODEOWNERS$' - id: trailing-whitespace - types_or: [c, inc, python, rst] + types_or: [c, inc, python, rst, yaml] + - id: trailing-whitespace + files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.33.0 + rev: 0.34.0 hooks: - id: check-dependabot - id: check-github-workflows @@ -61,7 +95,7 @@ repos: - id: actionlint - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.6.0 + rev: v1.14.1 hooks: - id: zizmor diff --git a/.readthedocs.yml b/.readthedocs.yml index a57de00544e..0a2c3f83453 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -32,4 +32,3 @@ build: - make -C Doc venv html - mkdir _readthedocs - mv Doc/build/html _readthedocs/html - diff --git a/.well-known/funding-manifest-urls b/.well-known/funding-manifest-urls new file mode 100644 index 00000000000..d56ca9eb52f --- /dev/null +++ b/.well-known/funding-manifest-urls @@ -0,0 +1 @@ +https://www.python.org/funding.json diff --git a/Android/README.md b/Android/README.md index 6cabd6ba5d6..9f71aeb934f 100644 --- a/Android/README.md +++ b/Android/README.md @@ -96,10 +96,12 @@ ## Packaging ## Testing -The Python test suite can be run on Linux, macOS, or Windows: +The Python test suite can be run on Linux, macOS, or Windows. -* On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. +On Linux, the emulator needs access to the KVM virtualization interface. This may +require adding your user to a group, or changing your udev rules. On GitHub +Actions, the test script will do this automatically using the commands shown +[here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). The test suite can usually be run on a device with 2 GB of RAM, but this is borderline, so you may need to increase it to 4 GB. As of Android @@ -156,6 +158,10 @@ ## Testing and architecture-specific files such as sysconfigdata, will not take effect until you re-run `android.py make-host` or `build`. +The testbed app can also be used to test third-party packages. For more details, +run `android.py test --help`, paying attention to the options `--site-packages`, +`--cwd`, `-c` and `-m`. + ## Using in your own app diff --git a/Android/android-env.sh b/Android/android-env.sh index bab4130c9e9..5859c0eac4a 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -3,7 +3,7 @@ : "${HOST:?}" # GNU target triplet # You may also override the following: -: "${api_level:=24}" # Minimum Android API level the build will run on +: "${ANDROID_API_LEVEL:=24}" # Minimum Android API level the build will run on : "${PREFIX:-}" # Path in which to find required libraries @@ -24,7 +24,7 @@ fail() { # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md -ndk_version=27.1.12297006 +ndk_version=27.3.13750724 ndk=$ANDROID_HOME/ndk/$ndk_version if ! [ -e "$ndk" ]; then @@ -43,7 +43,7 @@ fi toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) export AR="$toolchain/bin/llvm-ar" export AS="$toolchain/bin/llvm-as" -export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang" export CXX="${CC}++" export LD="$toolchain/bin/ld" export NM="$toolchain/bin/llvm-nm" diff --git a/Android/android.py b/Android/android.py index 3f48b42aa17..d1a10be776e 100755 --- a/Android/android.py +++ b/Android/android.py @@ -2,7 +2,9 @@ import asyncio import argparse +import json import os +import platform import re import shlex import shutil @@ -14,7 +16,7 @@ from contextlib import asynccontextmanager from datetime import datetime, timezone from glob import glob -from os.path import basename, relpath +from os.path import abspath, basename, relpath from pathlib import Path from subprocess import CalledProcessError from tempfile import TemporaryDirectory @@ -22,9 +24,14 @@ SCRIPT_NAME = Path(__file__).name ANDROID_DIR = Path(__file__).resolve().parent -CHECKOUT = ANDROID_DIR.parent +PYTHON_DIR = ANDROID_DIR.parent +in_source_tree = ( + ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() +) + +ENV_SCRIPT = ANDROID_DIR / "android-env.sh" TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = CHECKOUT / "cross-build" +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] APP_ID = "org.python.testbed" @@ -46,7 +53,19 @@ + (".bat" if os.name == "nt" else "") ) -logcat_started = False +# Whether we've seen any output from Python yet. +python_started = False + +# Buffer for verbose output which will be displayed only if a test fails and +# there has been no output from Python. +hidden_output = [] + + +def log_verbose(context, line, stream=sys.stdout): + if context.verbose: + stream.write(line) + else: + hidden_output.append((stream, line)) def delete_glob(pattern): @@ -76,39 +95,67 @@ def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) if env is None: env = os.environ.copy() - original_env = env.copy() if host: - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={subdir(host)}/prefix; " - f". {env_script}; " - f"export", - check=True, shell=True, text=True, stdout=subprocess.PIPE - ).stdout - - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if env.get(key) != value: - print(line) - env[key] = value - - if env == original_env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) + host_env = android_env(host) + print_env(host_env) + env.update(host_env) if log: - print(">", " ".join(map(str, command))) + print(">", join_command(command)) return subprocess.run(command, env=env, **kwargs) +# Format a command so it can be copied into a shell. Like shlex.join, but also +# accepts arguments which are Paths, or a single string/Path outside of a list. +def join_command(args): + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + +# Format the environment so it can be pasted into a shell. +def print_env(env): + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + + +def android_env(host): + if host: + prefix = subdir(host) / "prefix" + else: + prefix = ANDROID_DIR / "prefix" + sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + sysconfig_filename = next(sysconfig_files).name + host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] + + env_output = subprocess.run( + f"set -eu; " + f"HOST={host}; " + f"PREFIX={prefix}; " + f". {ENV_SCRIPT}; " + f"export", + check=True, shell=True, capture_output=True, encoding='utf-8', + ).stdout + + env = {} + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if os.environ.get(key) != value: + env[key] = value + + if not env: + raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n" + + env_output) + return env + + def build_python_path(): """The path to the build Python binary.""" build_dir = subdir("build") @@ -127,7 +174,7 @@ def configure_build_python(context): clean("build") os.chdir(subdir("build", create=True)) - command = [relpath(CHECKOUT / "configure")] + command = [relpath(PYTHON_DIR / "configure")] if context.args: command.extend(context.args) run(command) @@ -138,13 +185,20 @@ def make_build_python(context): run(["make", "-j", str(os.cpu_count())]) +# To create new builds of these dependencies, usually all that's necessary is to +# push a tag to the cpython-android-source-deps repository, and GitHub Actions +# will do the rest. +# +# If you're a member of the Python core team, and you'd like to be able to push +# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee. def unpack_deps(host, prefix_dir): + os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", - "sqlite-3.49.1-0", "xz-5.4.6-1"]: + for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.18-0", + "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") - shutil.unpack_archive(filename, prefix_dir) + shutil.unpack_archive(filename) os.remove(filename) @@ -167,7 +221,7 @@ def configure_host_python(context): os.chdir(host_dir) command = [ # Basic cross-compiling configuration - relpath(CHECKOUT / "configure"), + relpath(PYTHON_DIR / "configure"), f"--host={context.host}", f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", f"--with-build-python={build_python_path()}", @@ -196,9 +250,18 @@ def make_host_python(context): for pattern in ("include/python*", "lib/libpython*", "lib/python*"): delete_glob(f"{prefix_dir}/{pattern}") + # The Android environment variables were already captured in the Makefile by + # `configure`, and passing them again when running `make` may cause some + # flags to be duplicated. So we don't use the `host` argument here. os.chdir(host_dir) - run(["make", "-j", str(os.cpu_count())], host=context.host) - run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + run(["make", "-j", str(os.cpu_count())]) + + # The `make install` output is very verbose and rarely useful, so + # suppress it by default. + run( + ["make", "install", f"prefix={prefix_dir}"], + capture_output=not context.verbose, + ) def build_all(context): @@ -217,6 +280,33 @@ def clean_all(context): clean(host) +def setup_ci(): + if "GITHUB_ACTIONS" in os.environ: + # Enable emulator hardware acceleration + # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). + if platform.system() == "Linux": + run( + ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], + input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', + text=True, + ) + run(["sudo", "udevadm", "control", "--reload-rules"]) + run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + + # Free up disk space by deleting unused versions of the NDK + # (https://github.com/freakboy3742/pyspamsum/pull/108). + for line in ENV_SCRIPT.read_text().splitlines(): + if match := re.fullmatch(r"ndk_version=(.+)", line): + ndk_version = match[1] + break + else: + raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}") + + for item in (android_home / "ndk").iterdir(): + if item.name[0].isdigit() and item.name != ndk_version: + delete_glob(item) + + def setup_sdk(): sdkmanager = android_home / ( "cmdline-tools/latest/bin/sdkmanager" @@ -228,7 +318,12 @@ def setup_sdk(): if not all((android_home / "licenses" / path).exists() for path in [ "android-sdk-arm-dbt-license", "android-sdk-license" ]): - run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + run( + [sdkmanager, "--licenses"], + text=True, + capture_output=True, + input="y\n" * 100, + ) # Gradle may install this automatically, but we can't rely on that because # we need to run adb within the logcat task. @@ -411,17 +506,19 @@ async def logcat_task(context, initial_devices): # `--pid` requires API level 24 or higher. args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] - hidden_output = [] + logcat_started = False async with async_process( *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + logcat_started = True level, message = match.groups() else: - # If the regex doesn't match, this is probably the second or - # subsequent line of a multi-line message. Python won't produce - # such messages, but other components might. + # If the regex doesn't match, this is either a logcat startup + # error, or the second or subsequent line of a multi-line + # message. Python won't produce multi-line messages, but other + # components might. level, message = None, line # Exclude high-volume messages which are rarely useful. @@ -441,25 +538,22 @@ async def logcat_task(context, initial_devices): # tag indicators from Python's stdout and stderr. for prefix in ["python.stdout: ", "python.stderr: "]: if message.startswith(prefix): - global logcat_started - logcat_started = True + global python_started + python_started = True stream.write(message.removeprefix(prefix)) break else: - if context.verbose: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - stream.write(line) - else: - hidden_output.append(line) + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + log_verbose(context, line, stream) # If the device disconnects while logcat is running, which always # happens in --managed mode, some versions of adb return non-zero. # Distinguish this from a logcat startup error by checking whether we've - # received a message from Python yet. + # received any logcat messages yet. status = await wait_for(process.wait(), timeout=1) if status != 0 and not logcat_started: - raise CalledProcessError(status, args, "".join(hidden_output)) + raise CalledProcessError(status, args) def stop_app(serial): @@ -474,12 +568,38 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected + if context.ci_mode: + context.args[0:0] = [ + # See _add_ci_python_opts in libregrtest/main.py. + "-W", "error", "-bb", "-E", + + # Randomization is disabled because order-dependent failures are + # much less likely to pass on a rerun in single-process mode. + "-m", "test", + f"--{context.ci_mode}-ci", "--single-process", "--no-randomize" + ] + + if not any(arg in context.args for arg in ["-c", "-m"]): + context.args[0:0] = ["-m", "test"] + args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" - + shlex.join(context.args), + ] + [ + f"-P{name}={value}" + for name, value in [ + ("python.sitePackages", context.site_packages), + ("python.cwd", context.cwd), + ( + "android.testInstrumentationRunnerArguments.pythonArgs", + json.dumps(context.args), + ), + ] + if value ] - hidden_output = [] + if context.verbose >= 2: + args.append("--info") + log_verbose(context, f"> {join_command(args)}\n") + try: async with async_process( *args, cwd=TESTBED_DIR, env=env, @@ -488,10 +608,10 @@ async def gradle_task(context): while line := (await process.stdout.readline()).decode(*DECODE_ARGS): # Gradle may take several minutes to install SDK packages, so # it's worth showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): + if line.startswith('Preparing "Install'): sys.stdout.write(line) else: - hidden_output.append(line) + log_verbose(context, line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -499,17 +619,13 @@ async def gradle_task(context): else: raise CalledProcessError(status, args) finally: - # If logcat never started, then something has gone badly wrong, so the - # user probably wants to see the Gradle output even in non-verbose mode. - if hidden_output and not logcat_started: - sys.stdout.write("".join(hidden_output)) - # Gradle does not stop the tests when interrupted. if context.connected: stop_app(context.connected) async def run_testbed(context): + setup_ci() setup_sdk() setup_testbed() @@ -533,6 +649,12 @@ async def run_testbed(context): except* MySystemExit as e: raise SystemExit(*e.exceptions[0].args) from None except* CalledProcessError as e: + # If Python produced no output, then the user probably wants to see the + # verbose output to explain why the test failed. + if not python_started: + for stream, line in hidden_output: + stream.write(line) + # Extract it from the ExceptionGroup so it can be handled by `main`. raise e.exceptions[0] @@ -597,11 +719,64 @@ def package(context): else: shutil.copy2(src, dst, follow_symlinks=False) + # Strip debug information. + if not context.debug: + so_files = glob(f"{temp_dir}/**/*.so", recursive=True) + run([android_env(context.host)["STRIP"], *so_files], log=False) + dist_dir = subdir(context.host, "dist", create=True) package_path = shutil.make_archive( f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir ) print(f"Wrote {package_path}") + return package_path + + +def ci(context): + for step in [ + configure_build_python, + make_build_python, + configure_host_python, + make_host_python, + package, + ]: + caption = ( + step.__name__.replace("_", " ") + .capitalize() + .replace("python", "Python") + ) + print(f"::group::{caption}") + result = step(context) + if step is package: + package_path = result + print("::endgroup::") + + if ( + "GITHUB_ACTIONS" in os.environ + and (platform.system(), platform.machine()) != ("Linux", "x86_64") + ): + print( + "Skipping tests: GitHub Actions does not support the Android " + "emulator on this platform." + ) + else: + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + print("::group::Tests") + + # Prove the package is self-contained by using it to run the tests. + shutil.unpack_archive(package_path, temp_dir) + launcher_args = [ + "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci" + ] + run( + ["./android.py", "test", *launcher_args], + cwd=temp_dir + ) + print("::endgroup::") + + +def env(context): + print_env(android_env(getattr(context, "host", None))) # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated @@ -615,45 +790,54 @@ def signal_handler(*args): def parse_args(): parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") + subcommands = parser.add_subparsers(dest="subcommand", required=True) + + def add_parser(*args, **kwargs): + parser = subcommands.add_parser(*args, **kwargs) + parser.add_argument( + "-v", "--verbose", action="count", default=0, + help="Show verbose output. Use twice to be even more verbose.") + return parser # Subcommands - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for Android") - make_host = subcommands.add_parser("make-host", - help="Run `make` for Android") - subcommands.add_parser( - "clean", help="Delete all build and prefix directories") - subcommands.add_parser( - "build-testbed", help="Build the testbed app") - test = subcommands.add_parser( - "test", help="Run the test suite") - package = subcommands.add_parser("package", help="Make a release package") + build = add_parser( + "build", help="Run configure-build, make-build, configure-host and " + "make-host") + configure_build = add_parser( + "configure-build", help="Run `configure` for the build Python") + add_parser( + "make-build", help="Run `make` for the build Python") + configure_host = add_parser( + "configure-host", help="Run `configure` for Android") + make_host = add_parser( + "make-host", help="Run `make` for Android") + + add_parser("clean", help="Delete all build directories") + add_parser("build-testbed", help="Build the testbed app") + test = add_parser("test", help="Run the testbed app") + package = add_parser("package", help="Make a release package") + ci = add_parser("ci", help="Run build, package and test") + env = add_parser("env", help="Print environment variables") # Common arguments - for subcommand in build, configure_build, configure_host: + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", - help="Delete the relevant build and prefix directories first") - for subcommand in [build, configure_host, make_host, package]: + help="Delete the relevant build directories first") + + host_commands = [build, configure_host, make_host, package, ci] + if in_source_tree: + host_commands.append(env) + for subcommand in host_commands: subcommand.add_argument( "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") - for subcommand in build, configure_build, configure_host: + + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") # Test arguments - test.add_argument( - "-v", "--verbose", action="count", default=0, - help="Show Gradle output, and non-Python logcat messages. " - "Use twice to include high-volume messages which are rarely useful.") device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -661,9 +845,34 @@ def parse_args(): device_group.add_argument( "--managed", metavar="NAME", help="Run on a Gradle-managed device. " "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + test.add_argument( - "args", nargs="*", help=f"Arguments for `python -m test`. " - f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + "--site-packages", metavar="DIR", type=abspath, + help="Directory to copy as the app's site-packages.") + test.add_argument( + "--cwd", metavar="DIR", type=abspath, + help="Directory to copy as the app's working directory.") + test.add_argument( + "args", nargs="*", help=f"Python command-line arguments. " + f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. " + f"If neither -c nor -m are included, `-m test` will be prepended, " + f"which will run Python's own test suite.") + + # Package arguments. + for subcommand in [package, ci]: + subcommand.add_argument( + "-g", action="store_true", default=False, dest="debug", + help="Include debug information in package") + + # CI arguments + for subcommand in [test, ci]: + group = subcommand.add_mutually_exclusive_group(required=subcommand is ci) + group.add_argument( + "--fast-ci", action="store_const", dest="ci_mode", const="fast", + help="Add test arguments for GitHub Actions") + group.add_argument( + "--slow-ci", action="store_const", dest="ci_mode", const="slow", + help="Add test arguments for buildbots") return parser.parse_args() @@ -688,6 +897,8 @@ def main(): "build-testbed": build_testbed, "test": run_testbed, "package": package, + "ci": ci, + "env": env, } try: @@ -702,20 +913,17 @@ def main(): def print_called_process_error(e): for stream_name in ["stdout", "stderr"]: content = getattr(e, stream_name) + if isinstance(content, bytes): + content = content.decode(*DECODE_ARGS) stream = getattr(sys, stream_name) if content: stream.write(content) if not content.endswith("\n"): stream.write("\n") - # Format the command so it can be copied into a shell. shlex uses single - # quotes, so we surround the whole command with double quotes. - args_joined = ( - e.cmd if isinstance(e.cmd, str) - else " ".join(shlex.quote(str(arg)) for arg in e.cmd) - ) + # shlex uses single quotes, so we surround the command with double quotes. print( - f'Command "{args_joined}" returned exit status {e.returncode}' + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' ) diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index c627cb1b0e0..14d43d8c4d5 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -47,7 +47,7 @@ for ((i, prefix) in prefixes.withIndex()) { val libDir = file("$prefix/lib") val version = run { for (filename in libDir.list()!!) { - """python(\d+\.\d+)""".toRegex().matchEntire(filename)?.let { + """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let { return@run it.groupValues[1] } } @@ -64,9 +64,10 @@ for ((i, prefix) in prefixes.withIndex()) { val libPythonDir = file("$libDir/python$pythonVersion") val triplet = run { for (filename in libPythonDir.list()!!) { - """_sysconfigdata__android_(.+).py""".toRegex().matchEntire(filename)?.let { - return@run it.groupValues[1] - } + """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex() + .matchEntire(filename)?.let { + return@run it.groupValues[1] + } } throw GradleException("Failed to find Python triplet in $libPythonDir") } @@ -78,20 +79,20 @@ android { val androidEnvFile = file("../../android-env.sh").absoluteFile namespace = "org.python.testbed" - compileSdk = 34 + compileSdk = 35 defaultConfig { applicationId = "org.python.testbed" minSdk = androidEnvFile.useLines { for (line in it) { - """api_level:=(\d+)""".toRegex().find(line)?.let { + """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { return@useLines it.groupValues[1].toInt() } } throw GradleException("Failed to find API level in $androidEnvFile") } - targetSdk = 34 + targetSdk = 35 versionCode = 1 versionName = "1.0" @@ -205,11 +206,29 @@ androidComponents.onVariants { variant -> into("site-packages") { from("$projectDir/src/main/python") + + val sitePackages = findProperty("python.sitePackages") as String? + if (!sitePackages.isNullOrEmpty()) { + if (!file(sitePackages).exists()) { + throw GradleException("$sitePackages does not exist") + } + from(sitePackages) + } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE exclude("**/__pycache__") } + + into("cwd") { + val cwd = findProperty("python.cwd") as String? + if (!cwd.isNullOrEmpty()) { + if (!file(cwd).exists()) { + throw GradleException("$cwd does not exist") + } + from(cwd) + } + } } } diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt index 0e888ab71d8..e57243566f9 100644 --- a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -17,11 +17,11 @@ class PythonSuite { fun testPython() { val start = System.currentTimeMillis() try { - val context = + val status = PythonTestRunner( InstrumentationRegistry.getInstrumentation().targetContext - val args = - InstrumentationRegistry.getArguments().getString("pythonArgs", "") - val status = PythonTestRunner(context).run(args) + ).run( + InstrumentationRegistry.getArguments().getString("pythonArgs")!!, + ) assertEquals(0, status) } finally { // Make sure the process lives long enough for the test script to diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c index ec7f93a3e5e..7f024f0a348 100644 --- a/Android/testbed/app/src/main/c/main_activity.c +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,13 @@ static void throw_runtime_exception(JNIEnv *env, const char *message) { message); } +static void throw_errno(JNIEnv *env, const char *error_prefix) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); +} + // --- Stdio redirection ------------------------------------------------------ @@ -95,10 +103,7 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL for (StreamInfo *si = STREAMS; si->file; si++) { char *error_prefix; if ((error_prefix = redirect_stream(si))) { - char error_message[1024]; - snprintf(error_message, sizeof(error_message), - "%s: %s", error_prefix, strerror(errno)); - throw_runtime_exception(env, error_message); + throw_errno(env, error_prefix); return; } } @@ -107,13 +112,38 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL // --- Python initialization --------------------------------------------------- -static PyStatus set_config_string( - JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value -) { - const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); - PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); - (*env)->ReleaseStringUTFChars(env, value, value_utf8); - return status; +static char *init_signals() { + // Some tests use SIGUSR1, but that's blocked by default in an Android app in + // order to make it available to `sigwait` in the Signal Catcher thread. + // (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). + // That thread's functionality is only useful for debugging the JVM, so disabling + // it should not weaken the tests. + // + // There's no safe way of stopping the thread completely (#123982), but simply + // unblocking SIGUSR1 is enough to fix most tests. + // + // However, in tests that generate multiple different signals in quick + // succession, it's possible for SIGUSR1 to arrive while the main thread is busy + // running the C-level handler for a different signal. In that case, the SIGUSR1 + // may be sent to the Signal Catcher thread instead, which will generate a log + // message containing the text "reacting to signal". + // + // Such tests may need to be changed in one of the following ways: + // * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in + // test_signal.py). + // * Send the signal to a specific thread rather than the whole process (e.g. + // test_signals in test_threadsignals.py. + sigset_t set; + if (sigemptyset(&set)) { + return "sigemptyset"; + } + if (sigaddset(&set, SIGUSR1)) { + return "sigaddset"; + } + if ((errno = pthread_sigmask(SIG_UNBLOCK, &set, NULL))) { + return "pthread_sigmask"; + } + return NULL; } static void throw_status(JNIEnv *env, PyStatus status) { @@ -121,27 +151,47 @@ static void throw_status(JNIEnv *env, PyStatus status) { } JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( - JNIEnv *env, jobject obj, jstring home, jstring runModule + JNIEnv *env, jobject obj, jstring home, jarray args ) { + const char *home_utf8 = (*env)->GetStringUTFChars(env, home, NULL); + char cwd[PATH_MAX]; + snprintf(cwd, sizeof(cwd), "%s/%s", home_utf8, "cwd"); + if (chdir(cwd)) { + throw_errno(env, "chdir"); + return 1; + } + + char *error_prefix; + if ((error_prefix = init_signals())) { + throw_errno(env, error_prefix); + return 1; + } + PyConfig config; PyStatus status; - PyConfig_InitIsolatedConfig(&config); + PyConfig_InitPythonConfig(&config); - status = set_config_string(env, &config, &config.home, home); - if (PyStatus_Exception(status)) { + jsize argc = (*env)->GetArrayLength(env, args); + const char *argv[argc + 1]; + for (int i = 0; i < argc; i++) { + jobject arg = (*env)->GetObjectArrayElement(env, args, i); + argv[i] = (*env)->GetStringUTFChars(env, arg, NULL); + } + argv[argc] = NULL; + + // PyConfig_SetBytesArgv "must be called before other methods, since the + // preinitialization configuration depends on command line arguments" + if (PyStatus_Exception(status = PyConfig_SetBytesArgv(&config, argc, (char**)argv))) { throw_status(env, status); return 1; } - status = set_config_string(env, &config, &config.run_module, runModule); + status = PyConfig_SetBytesString(&config, &config.home, home_utf8); if (PyStatus_Exception(status)) { throw_status(env, status); return 1; } - // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. - config.install_signal_handlers = 1; - status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { throw_status(env, status); diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index c4bf6cbe83d..5727b0fe6c3 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -5,6 +5,7 @@ import android.os.* import android.system.Os import android.widget.TextView import androidx.appcompat.app.* +import org.json.JSONArray import java.io.* @@ -15,18 +16,25 @@ class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - val status = PythonTestRunner(this).run("-W -uall") + val status = PythonTestRunner(this).run("""["-m", "test", "-W", "-uall"]""") findViewById(R.id.tvHello).text = "Exit status $status" } } class PythonTestRunner(val context: Context) { - /** @param args Extra arguments for `python -m test`. - * @return The Python exit status: zero if the tests passed, nonzero if - * they failed. */ - fun run(args: String = "") : Int { - Os.setenv("PYTHON_ARGS", args, true) + /** Run Python. + * + * @param args Python command-line, encoded as JSON. + * @return The Python exit status: zero on success, nonzero on failure. */ + fun run(args: String) : Int { + // We leave argument 0 as an empty string, which is a placeholder for the + // executable name in embedded mode. + val argsJsonArray = JSONArray(args) + val argsStringArray = Array(argsJsonArray.length() + 1) { it -> ""} + for (i in 0..) : Int } diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py deleted file mode 100644 index d6941b14412..00000000000 --- a/Android/testbed/app/src/main/python/main.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import runpy -import shlex -import signal -import sys - -# Some tests use SIGUSR1, but that's blocked by default in an Android app in -# order to make it available to `sigwait` in the Signal Catcher thread. -# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). -# That thread's functionality is only useful for debugging the JVM, so disabling -# it should not weaken the tests. -# -# There's no safe way of stopping the thread completely (#123982), but simply -# unblocking SIGUSR1 is enough to fix most tests. -# -# However, in tests that generate multiple different signals in quick -# succession, it's possible for SIGUSR1 to arrive while the main thread is busy -# running the C-level handler for a different signal. In that case, the SIGUSR1 -# may be sent to the Signal Catcher thread instead, which will generate a log -# message containing the text "reacting to signal". -# -# Such tests may need to be changed in one of the following ways: -# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in -# test_signal.py). -# * Send the signal to a specific thread rather than the whole process (e.g. -# test_signals in test_threadsignals.py. -signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) - -sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) - -# The test module will call sys.exit to indicate whether the tests passed. -runpy.run_module("test") diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts index 4d1d6f87594..451517b3f1a 100644 --- a/Android/testbed/build.gradle.kts +++ b/Android/testbed/build.gradle.kts @@ -1,5 +1,5 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { - id("com.android.application") version "8.6.1" apply false + id("com.android.application") version "8.10.0" apply false id("org.jetbrains.kotlin.android") version "1.9.22" apply false } diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties index 36529c89642..5d42fbae084 100644 --- a/Android/testbed/gradle/wrapper/gradle-wrapper.properties +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon Feb 19 20:29:06 GMT 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/Apple/.ruff.toml b/Apple/.ruff.toml new file mode 100644 index 00000000000..4cdc39ebee4 --- /dev/null +++ b/Apple/.ruff.toml @@ -0,0 +1,22 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] diff --git a/Apple/__main__.py b/Apple/__main__.py new file mode 100644 index 00000000000..256966e76c2 --- /dev/null +++ b/Apple/__main__.py @@ -0,0 +1,1065 @@ +#!/usr/bin/env python3 +########################################################################## +# Apple XCframework build script +# +# This script simplifies the process of configuring, compiling and packaging an +# XCframework for an Apple platform. +# +# At present, it only supports iOS, but it has been constructed so that it +# could be used on any Apple platform. +# +# The simplest entry point is: +# +# $ python Apple ci iOS +# +# which will: +# * Clean any pre-existing build artefacts +# * Configure and make a Python that can be used for the build +# * Configure and make a Python for each supported iOS architecture and ABI +# * Combine the outputs of the builds from the previous step into a single +# XCframework, merging binaries into a "fat" binary if necessary +# * Clone a copy of the testbed, configured to use the XCframework +# * Construct a tarball containing the release artefacts +# * Run the test suite using the generated XCframework. +# +# This is the complete sequence that would be needed in CI to build and test +# a candidate release artefact. +# +# Each individual step can be invoked individually - there are commands to +# clean, configure-build, make-build, configure-host, make-host, package, and +# test. +# +# There is also a build command that can be used to combine the configure and +# make steps for the build Python, an individual host, all hosts, or all +# builds. +########################################################################## +from __future__ import annotations + +import argparse +import os +import platform +import re +import shlex +import shutil +import signal +import subprocess +import sys +import sysconfig +import time +from collections.abc import Callable, Sequence +from contextlib import contextmanager +from datetime import datetime, timezone +from os.path import basename, relpath +from pathlib import Path +from subprocess import CalledProcessError + +EnvironmentT = dict[str, str] +ArgsT = Sequence[str | Path] + +SCRIPT_NAME = Path(__file__).name +PYTHON_DIR = Path(__file__).resolve().parent.parent + +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" + +HOSTS: dict[str, dict[str, dict[str, str]]] = { + # Structure of this data: + # * Platform identifier + # * an XCframework slice that must exist for that platform + # * a host triple: the multiarch spec for that host + "iOS": { + "ios-arm64": { + "arm64-apple-ios": "arm64-iphoneos", + }, + "ios-arm64_x86_64-simulator": { + "arm64-apple-ios-simulator": "arm64-iphonesimulator", + "x86_64-apple-ios-simulator": "x86_64-iphonesimulator", + }, + }, +} + + +def subdir(name: str, create: bool = False) -> Path: + """Ensure that a cross-build directory for the given name exists.""" + path = CROSS_BUILD_DIR / name + if not path.exists(): + if not create: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}." + ) + else: + path.mkdir(parents=True) + return path + + +def run( + command: ArgsT, + *, + host: str | None = None, + env: EnvironmentT | None = None, + log: bool | None = True, + **kwargs, +) -> subprocess.CompletedProcess: + """Run a command in an Apple development environment. + + Optionally logs the executed command to the console. + """ + kwargs.setdefault("check", True) + if env is None: + env = os.environ.copy() + + if host: + host_env = apple_env(host) + print_env(host_env) + env.update(host_env) + + if log: + print(">", join_command(command)) + return subprocess.run(command, env=env, **kwargs) + + +def join_command(args: str | Path | ArgsT) -> str: + """Format a command so it can be copied into a shell. + + Similar to `shlex.join`, but also accepts arguments which are Paths, or a + single string/Path outside of a list. + """ + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + +def print_env(env: EnvironmentT) -> None: + """Format the environment so it can be pasted into a shell.""" + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + + +def apple_env(host: str) -> EnvironmentT: + """Construct an Apple development environment for the given host.""" + env = { + "PATH": ":".join([ + str(PYTHON_DIR / "Apple/iOS/Resources/bin"), + str(subdir(host) / "prefix"), + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/Library/Apple/usr/bin", + ]), + } + + return env + + +def delete_path(name: str) -> None: + """Delete the named cross-build directory, if it exists.""" + path = CROSS_BUILD_DIR / name + if path.exists(): + print(f"Deleting {path} ...") + shutil.rmtree(path) + + +def all_host_triples(platform: str) -> list[str]: + """Return all host triples for the given platform. + + The host triples are the platform definitions used as input to configure + (e.g., "arm64-apple-ios-simulator"). + """ + triples = [] + for slice_name, slice_parts in HOSTS[platform].items(): + triples.extend(list(slice_parts)) + return triples + + +def clean(context: argparse.Namespace, target: str = "all") -> None: + """The implementation of the "clean" command.""" + # If we're explicitly targeting the build, there's no platform or + # distribution artefacts. If we're cleaning tests, we keep all built + # artefacts. Otherwise, the built artefacts must be dirty, so we remove + # them. + if target not in {"build", "test"}: + paths = ["dist", context.platform] + list(HOSTS[context.platform]) + else: + paths = [] + + if target in {"all", "build"}: + paths.append("build") + + if target in {"all", "hosts"}: + paths.extend(all_host_triples(context.platform)) + elif target not in {"build", "test", "package"}: + paths.append(target) + + if target in {"all", "hosts", "test"}: + paths.extend([ + path.name + for path in CROSS_BUILD_DIR.glob(f"{context.platform}-testbed.*") + ]) + + for path in paths: + delete_path(path) + + +def build_python_path() -> Path: + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {build_dir}" + ) + + return binary + + +@contextmanager +def group(text: str): + """A context manager that outputs a log marker around a section of a build. + + If running in a GitHub Actions environment, the GitHub syntax for + collapsible log sections is used. + """ + if "GITHUB_ACTIONS" in os.environ: + print(f"::group::{text}") + else: + print(f"===== {text} " + "=" * (70 - len(text))) + + yield + + if "GITHUB_ACTIONS" in os.environ: + print("::endgroup::") + else: + print() + + +@contextmanager +def cwd(subdir: Path): + """A context manager that sets the current working directory.""" + orig = os.getcwd() + os.chdir(subdir) + yield + os.chdir(orig) + + +def configure_build_python(context: argparse.Namespace) -> None: + """The implementation of the "configure-build" command.""" + if context.clean: + clean(context, "build") + + with ( + group("Configuring build Python"), + cwd(subdir("build", create=True)), + ): + command = [relpath(PYTHON_DIR / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context: argparse.Namespace) -> None: + """The implementation of the "make-build" command.""" + with ( + group("Compiling build Python"), + cwd(subdir("build")), + ): + run(["make", "-j", str(os.cpu_count())]) + + +def apple_target(host: str) -> str: + """Return the Apple platform identifier for a given host triple.""" + for _, platform_slices in HOSTS.items(): + for slice_name, slice_parts in platform_slices.items(): + for host_triple, multiarch in slice_parts.items(): + if host == host_triple: + return ".".join(multiarch.split("-")[::-1]) + + raise KeyError(host) + + +def apple_multiarch(host: str) -> str: + """Return the multiarch descriptor for a given host triple.""" + for _, platform_slices in HOSTS.items(): + for slice_name, slice_parts in platform_slices.items(): + for host_triple, multiarch in slice_parts.items(): + if host == host_triple: + return multiarch + + raise KeyError(host) + + +def unpack_deps( + platform: str, + host: str, + prefix_dir: Path, + cache_dir: Path, +) -> None: + """Unpack binary dependencies into a provided directory. + + Downloads binaries if they aren't already present. Downloads will be stored + in provided cache directory. + + On iOS, as a safety mechanism, any dynamic libraries will be purged from + the unpacked dependencies. + """ + # To create new builds of these dependencies, usually all that's necessary + # is to push a tag to the cpython-apple-source-deps repository, and GitHub + # Actions will do the rest. + # + # If you're a member of the Python core team, and you'd like to be able to + # push these tags yourself, please contact Malcolm Smith or Russell + # Keith-Magee. + deps_url = "https://github.com/beeware/cpython-apple-source-deps/releases/download" + for name_ver in [ + "BZip2-1.0.8-2", + "libFFI-3.4.7-2", + "OpenSSL-3.0.18-1", + "XZ-5.6.4-2", + "mpdecimal-4.0.0-2", + "zstd-1.5.7-1", + ]: + filename = f"{name_ver.lower()}-{apple_target(host)}.tar.gz" + archive_path = download( + f"{deps_url}/{name_ver}/{filename}", + target_dir=cache_dir, + ) + shutil.unpack_archive(archive_path, prefix_dir) + + # Dynamic libraries will be preferentially linked over static; + # On iOS, ensure that no dylibs are available in the prefix folder. + if platform == "iOS": + for dylib in prefix_dir.glob("**/*.dylib"): + dylib.unlink() + + +def download(url: str, target_dir: Path) -> Path: + """Download the specified URL into the given directory. + + :return: The path to the downloaded archive. + """ + target_path = Path(target_dir).resolve() + target_path.mkdir(exist_ok=True, parents=True) + + out_path = target_path / basename(url) + if not Path(out_path).is_file(): + run([ + "curl", + "-Lf", + "--retry", + "5", + "--retry-all-errors", + "-o", + out_path, + url, + ]) + else: + print(f"Using cached version of {basename(url)}") + return out_path + + +def configure_host_python( + context: argparse.Namespace, + host: str | None = None, +) -> None: + """The implementation of the "configure-host" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, host) + + host_dir = subdir(host, create=True) + prefix_dir = host_dir / "prefix" + + with group(f"Downloading dependencies ({host})"): + if not prefix_dir.exists(): + prefix_dir.mkdir() + unpack_deps(context.platform, host, prefix_dir, context.cache_dir) + else: + print("Dependencies already installed") + + with ( + group(f"Configuring host Python ({host})"), + cwd(host_dir), + ): + command = [ + # Basic cross-compiling configuration + relpath(PYTHON_DIR / "configure"), + f"--host={host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--with-system-libmpdec", + "--enable-framework", + # Dependent libraries. + f"--with-openssl={prefix_dir}", + f"LIBLZMA_CFLAGS=-I{prefix_dir}/include", + f"LIBLZMA_LIBS=-L{prefix_dir}/lib -llzma", + f"LIBFFI_CFLAGS=-I{prefix_dir}/include", + f"LIBFFI_LIBS=-L{prefix_dir}/lib -lffi", + f"LIBMPDEC_CFLAGS=-I{prefix_dir}/include", + f"LIBMPDEC_LIBS=-L{prefix_dir}/lib -lmpdec", + f"LIBZSTD_CFLAGS=-I{prefix_dir}/include", + f"LIBZSTD_LIBS=-L{prefix_dir}/lib -lzstd", + ] + + if context.args: + command.extend(context.args) + run(command, host=host) + + +def make_host_python( + context: argparse.Namespace, + host: str | None = None, +) -> None: + """The implementation of the "make-host" command.""" + if host is None: + host = context.host + + with ( + group(f"Compiling host Python ({host})"), + cwd(subdir(host)), + ): + run(["make", "-j", str(os.cpu_count())], host=host) + run(["make", "install"], host=host) + + +def framework_path(host_triple: str, multiarch: str) -> Path: + """The path to a built single-architecture framework product. + + :param host_triple: The host triple (e.g., arm64-apple-ios-simulator) + :param multiarch: The multiarch identifier (e.g., arm64-simulator) + """ + return CROSS_BUILD_DIR / f"{host_triple}/Apple/iOS/Frameworks/{multiarch}" + + +def package_version(prefix_path: Path) -> str: + """Extract the Python version being built from patchlevel.h.""" + for path in prefix_path.glob("**/patchlevel.h"): + text = path.read_text(encoding="utf-8") + if match := re.search( + r'\n\s*#define\s+PY_VERSION\s+"(.+)"\s*\n', text + ): + version = match[1] + # If not building against a tagged commit, add a timestamp to the + # version. Follow the PyPA version number rules, as this will make + # it easier to process with other tools. The version will have a + # `+` suffix once any official release has been made; a freshly + # forked main branch will have a version of 3.X.0a0. + if version.endswith("a0"): + version += "+" + if version.endswith("+"): + version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") + + return version + + sys.exit("Unable to determine Python version being packaged.") + + +def lib_platform_files(dirname, names): + """A file filter that ignores platform-specific files in lib.""" + path = Path(dirname) + if ( + path.parts[-3] == "lib" + and path.parts[-2].startswith("python") + and path.parts[-1] == "lib-dynload" + ): + return names + elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"): + ignored_names = { + name + for name in names + if ( + name.startswith("_sysconfigdata_") + or name.startswith("_sysconfig_vars_") + or name == "build-details.json" + ) + } + elif path.parts[-1] == "lib": + ignored_names = { + name + for name in names + if name.startswith("libpython") and name.endswith(".dylib") + } + else: + ignored_names = set() + + return ignored_names + + +def lib_non_platform_files(dirname, names): + """A file filter that ignores anything *except* platform-specific files + in the lib directory. + """ + path = Path(dirname) + if path.parts[-2] == "lib" and path.parts[-1].startswith("python"): + return ( + set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + ) + else: + return set() + + +def create_xcframework(platform: str) -> str: + """Build an XCframework from the component parts for the platform. + + :return: The version number of the Python version that was packaged. + """ + package_path = CROSS_BUILD_DIR / platform + try: + package_path.mkdir() + except FileExistsError: + raise RuntimeError( + f"{platform} XCframework already exists; do you need to run " + "with --clean?" + ) from None + + frameworks = [] + # Merge Frameworks for each component SDK. If there's only one architecture + # for the SDK, we can use the compiled Python.framework as-is. However, if + # there's more than architecture, we need to merge the individual built + # frameworks into a merged "fat" framework. + for slice_name, slice_parts in HOSTS[platform].items(): + # Some parts are the same across all slices, so we use can any of the + # host frameworks as the source for the merged version. Use the first + # one on the list, as it's as representative as any other. + first_host_triple, first_multiarch = next(iter(slice_parts.items())) + first_framework = ( + framework_path(first_host_triple, first_multiarch) + / "Python.framework" + ) + + if len(slice_parts) == 1: + # The first framework is the only framework, so copy it. + print(f"Copying framework for {slice_name}...") + frameworks.append(first_framework) + else: + print(f"Merging framework for {slice_name}...") + slice_path = CROSS_BUILD_DIR / slice_name + slice_framework = slice_path / "Python.framework" + slice_framework.mkdir(exist_ok=True, parents=True) + + # Copy the Info.plist + shutil.copy( + first_framework / "Info.plist", + slice_framework / "Info.plist", + ) + + # Copy the headers + shutil.copytree( + first_framework / "Headers", + slice_framework / "Headers", + ) + + # Create the "fat" library binary for the slice + run( + ["lipo", "-create", "-output", slice_framework / "Python"] + + [ + ( + framework_path(host_triple, multiarch) + / "Python.framework/Python" + ) + for host_triple, multiarch in slice_parts.items() + ] + ) + + # Add this merged slice to the list to be added to the XCframework + frameworks.append(slice_framework) + + print() + print("Build XCframework...") + cmd = [ + "xcodebuild", + "-create-xcframework", + "-output", + package_path / "Python.xcframework", + ] + for framework in frameworks: + cmd.extend(["-framework", framework]) + + run(cmd) + + # Extract the package version from the merged framework + version = package_version(package_path / "Python.xcframework") + version_tag = ".".join(version.split(".")[:2]) + + # On non-macOS platforms, each framework in XCframework only contains the + # headers, libPython, plus an Info.plist. Other resources like the standard + # library and binary shims aren't allowed to live in framework; they need + # to be copied in separately. + print() + print("Copy additional resources...") + has_common_stdlib = False + for slice_name, slice_parts in HOSTS[platform].items(): + # Some parts are the same across all slices, so we can any of the + # host frameworks as the source for the merged version. + first_host_triple, first_multiarch = next(iter(slice_parts.items())) + first_path = framework_path(first_host_triple, first_multiarch) + first_framework = first_path / "Python.framework" + + slice_path = package_path / f"Python.xcframework/{slice_name}" + slice_framework = slice_path / "Python.framework" + + # Copy the binary helpers + print(f" - {slice_name} binaries") + shutil.copytree(first_path / "bin", slice_path / "bin") + + # Copy the include path (a symlink to the framework headers) + print(f" - {slice_name} include files") + shutil.copytree( + first_path / "include", + slice_path / "include", + symlinks=True, + ) + + # Copy in the cross-architecture pyconfig.h + shutil.copy( + PYTHON_DIR / f"Apple/{platform}/Resources/pyconfig.h", + slice_framework / "Headers/pyconfig.h", + ) + + print(f" - {slice_name} shared library") + # Create a simlink for the fat library + shared_lib = slice_path / f"lib/libpython{version_tag}.dylib" + shared_lib.parent.mkdir() + shared_lib.symlink_to("../Python.framework/Python") + + print(f" - {slice_name} architecture-specific files") + for host_triple, multiarch in slice_parts.items(): + print(f" - {multiarch} standard library") + arch, _ = multiarch.split("-", 1) + + if not has_common_stdlib: + print(" - using this architecture as the common stdlib") + shutil.copytree( + framework_path(host_triple, multiarch) / "lib", + package_path / "Python.xcframework/lib", + ignore=lib_platform_files, + symlinks=True, + ) + has_common_stdlib = True + + shutil.copytree( + framework_path(host_triple, multiarch) / "lib", + slice_path / f"lib-{arch}", + ignore=lib_non_platform_files, + symlinks=True, + ) + + # Copy the host's pyconfig.h to an architecture-specific name. + arch = multiarch.split("-")[0] + host_path = ( + CROSS_BUILD_DIR + / host_triple + / "Apple/iOS/Frameworks" + / multiarch + ) + host_framework = host_path / "Python.framework" + shutil.copy( + host_framework / "Headers/pyconfig.h", + slice_framework / f"Headers/pyconfig-{arch}.h", + ) + + # Apple identifies certain libraries as "security risks"; if you + # statically link those libraries into a Framework, you become + # responsible for providing a privacy manifest for that framework. + xcprivacy_file = { + "OpenSSL": subdir(host_triple) + / "prefix/share/OpenSSL.xcprivacy" + } + print(f" - {multiarch} xcprivacy files") + for module, lib in [ + ("_hashlib", "OpenSSL"), + ("_ssl", "OpenSSL"), + ]: + shutil.copy( + xcprivacy_file[lib], + slice_path + / f"lib-{arch}/python{version_tag}" + / f"lib-dynload/{module}.xcprivacy", + ) + + print(" - build tools") + shutil.copytree( + PYTHON_DIR / "Apple/testbed/Python.xcframework/build", + package_path / "Python.xcframework/build", + ) + + return version + + +def package(context: argparse.Namespace) -> None: + """The implementation of the "package" command.""" + if context.clean: + clean(context, "package") + + with group("Building package"): + # Create an XCframework + version = create_xcframework(context.platform) + + # Clone testbed + print() + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + CROSS_BUILD_DIR / context.platform / "Python.xcframework", + CROSS_BUILD_DIR / context.platform / "testbed", + ]) + + # Build the final archive + archive_name = ( + CROSS_BUILD_DIR + / "dist" + / f"python-{version}-{context.platform}-XCframework" + ) + + print() + print("Create package archive...") + shutil.make_archive( + str(CROSS_BUILD_DIR / archive_name), + format="gztar", + root_dir=CROSS_BUILD_DIR / context.platform, + base_dir=".", + ) + print() + print(f"{archive_name.relative_to(PYTHON_DIR)}.tar.gz created.") + + +def build(context: argparse.Namespace, host: str | None = None) -> None: + """The implementation of the "build" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, host) + + if host in {"all", "build"}: + for step in [ + configure_build_python, + make_build_python, + ]: + step(context) + + if host == "build": + hosts = [] + elif host in {"all", "hosts"}: + hosts = all_host_triples(context.platform) + else: + hosts = [host] + + for step_host in hosts: + for step in [ + configure_host_python, + make_host_python, + ]: + step(context, host=step_host) + + if host in {"all", "hosts"}: + package(context) + + +def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: PT028 + """The implementation of the "test" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, "test") + + with group(f"Test {'XCframework' if host in {'all', 'hosts'} else host}"): + timestamp = str(time.time_ns())[:-6] + testbed_dir = ( + CROSS_BUILD_DIR / f"{context.platform}-testbed.{timestamp}" + ) + if host in {"all", "hosts"}: + framework_path = ( + CROSS_BUILD_DIR / context.platform / "Python.xcframework" + ) + else: + build_arch = platform.machine() + host_arch = host.split("-")[0] + + if not host.endswith("-simulator"): + print("Skipping test suite non-simulator build.") + return + elif build_arch != host_arch: + print( + f"Skipping test suite for an {host_arch} build " + f"on an {build_arch} machine." + ) + return + else: + framework_path = ( + CROSS_BUILD_DIR + / host + / f"Apple/{context.platform}" + / f"Frameworks/{apple_multiarch(host)}" + ) + + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + framework_path, + testbed_dir, + ]) + + run( + [ + sys.executable, + testbed_dir, + "run", + "--verbose", + ] + + ( + ["--simulator", str(context.simulator)] + if context.simulator + else [] + ) + + [ + "--", + "test", + f"--{context.ci_mode}-ci", + "--single-process", + "--no-randomize", + # Timeout handling requires subprocesses; explicitly setting + # the timeout to -1 disables the faulthandler. + "--timeout=-1", + # Adding Python options requires the use of a subprocess to + # start a new Python interpreter. + "--dont-add-python-opts", + ] + ) + + +def apple_sim_host(platform_name: str) -> str: + """Determine the native simulator target for this platform.""" + for _, slice_parts in HOSTS[platform_name].items(): + for host_triple in slice_parts: + parts = host_triple.split("-") + if parts[0] == platform.machine() and parts[-1] == "simulator": + return host_triple + + raise KeyError(platform_name) + + +def ci(context: argparse.Namespace) -> None: + """The implementation of the "ci" command. + + In "Fast" mode, this compiles the build python, and the simulator for the + build machine's architecture; and runs the test suite with `--fast-ci` + configuration. + + In "Slow" mode, it compiles the build python, plus all candidate + architectures (both device and simulator); then runs the test suite with + `--slow-ci` configuration. + """ + clean(context, "all") + if context.ci_mode == "slow": + # In slow mode, build and test the full XCframework + build(context, host="all") + test(context, host="all") + else: + # In fast mode, just build the simulator platform. + sim_host = apple_sim_host(context.platform) + build(context, host="build") + build(context, host=sim_host) + test(context, host=sim_host) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "A tool for managing the build, package and test process of " + "CPython on Apple platforms." + ), + ) + parser.suggest_on_error = True + subcommands = parser.add_subparsers(dest="subcommand", required=True) + + clean = subcommands.add_parser( + "clean", + help="Delete all build directories", + ) + + configure_build = subcommands.add_parser( + "configure-build", help="Run `configure` for the build Python" + ) + subcommands.add_parser( + "make-build", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for a specific platform and target", + ) + make_host = subcommands.add_parser( + "make-host", + help="Run `make` for a specific platform and target", + ) + package = subcommands.add_parser( + "package", + help="Create a release package for the platform", + ) + build = subcommands.add_parser( + "build", + help="Build all platform targets and create the XCframework", + ) + test = subcommands.add_parser( + "test", + help="Run the testbed for a specific platform", + ) + ci = subcommands.add_parser( + "ci", + help="Run build, package, and test", + ) + + # platform argument + for cmd in [clean, configure_host, make_host, package, build, test, ci]: + cmd.add_argument( + "platform", + choices=HOSTS.keys(), + help="The target platform to build", + ) + + # host triple argument + for cmd in [configure_host, make_host]: + cmd.add_argument( + "host", + help="The host triple to build (e.g., arm64-apple-ios-simulator)", + ) + # optional host triple argument + for cmd in [clean, build, test]: + cmd.add_argument( + "host", + nargs="?", + default="all", + help=( + "The host triple to build (e.g., arm64-apple-ios-simulator), " + "or 'build' for just the build platform, or 'hosts' for all " + "host platforms, or 'all' for the build platform and all " + "hosts. Defaults to 'all'" + ), + ) + + # --clean option + for cmd in [configure_build, configure_host, build, package, test, ci]: + cmd.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete the relevant build directories first", + ) + + # --cache-dir option + for cmd in [configure_host, build, ci]: + cmd.add_argument( + "--cache-dir", + default="./cross-build/downloads", + help="The directory to store cached downloads.", + ) + + # --simulator option + for cmd in [test, ci]: + cmd.add_argument( + "--simulator", + help=( + "The name of the simulator to use (eg: 'iPhone 16e'). " + "Defaults to the most recently released 'entry level' " + "iPhone device. Device architecture and OS version can also " + "be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would " + "run on an ARM64 iPhone 16 Pro simulator running iOS 26.0." + ), + ) + group = cmd.add_mutually_exclusive_group() + group.add_argument( + "--fast-ci", + action="store_const", + dest="ci_mode", + const="fast", + help="Add test arguments for GitHub Actions", + ) + group.add_argument( + "--slow-ci", + action="store_const", + dest="ci_mode", + const="slow", + help="Add test arguments for buildbots", + ) + + for subcommand in [configure_build, configure_host, build, ci]: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + + return parser.parse_args() + + +def print_called_process_error(e: subprocess.CalledProcessError) -> None: + for stream_name in ["stdout", "stderr"]: + content = getattr(e, stream_name) + stream = getattr(sys, stream_name) + if content: + stream.write(content) + if not content.endswith("\n"): + stream.write("\n") + + # shlex uses single quotes, so we surround the command with double quotes. + print( + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' + ) + + +def main() -> None: + # Handle SIGTERM the same way as SIGINT. This ensures that if we're + # terminated by the buildbot worker, we'll make an attempt to clean up our + # subprocesses. + def signal_handler(*args): + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, signal_handler) + + # Process command line arguments + context = parse_args() + dispatch: dict[str, Callable] = { + "clean": clean, + "configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "package": package, + "build": build, + "test": test, + "ci": ci, + } + + try: + dispatch[context.subcommand](context) + except CalledProcessError as e: + print() + print_called_process_error(e) + sys.exit(1) + except RuntimeError as e: + print() + print(e) + sys.exit(2) + + +if __name__ == "__main__": + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + + main() diff --git a/Apple/iOS/README.md b/Apple/iOS/README.md new file mode 100644 index 00000000000..7ee257b5d64 --- /dev/null +++ b/Apple/iOS/README.md @@ -0,0 +1,339 @@ +# Python on iOS README + +**iOS support is [tier 3](https://peps.python.org/pep-0011/#tier-3).** + +This document provides a quick overview of some iOS specific features in the +Python distribution. + +These instructions are only needed if you're planning to compile Python for iOS +yourself. Most users should *not* need to do this. If you're looking to +experiment with writing an iOS app in Python, tools such as [BeeWare's +Briefcase](https://briefcase.readthedocs.io) and [Kivy's +Buildozer](https://buildozer.readthedocs.io) will provide a much more +approachable user experience. + +## Compilers for building on iOS + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting an open the Platforms tab of the Xcode Settings panel. + +## Building Python on iOS + +### ABIs and Architectures + +iOS apps can be deployed on physical devices, and on the iOS simulator. Although +the API used on these devices is identical, the ABI is different - you need to +link against different libraries for an iOS device build (`iphoneos`) or an +iOS simulator build (`iphonesimulator`). + +Apple uses the `XCframework` format to allow specifying a single dependency +that supports multiple ABIs. An `XCframework` is a wrapper around multiple +ABI-specific frameworks that share a common API. + +iOS can also support different CPU architectures within each ABI. At present, +there is only a single supported architecture on physical devices - ARM64. +However, the *simulator* supports 2 architectures - ARM64 (for running on Apple +Silicon machines), and x86_64 (for running on older Intel-based machines). + +To support multiple CPU architectures on a single platform, Apple uses a "fat +binary" format - a single physical file that contains support for multiple +architectures. It is possible to compile and use a "thin" single architecture +version of a binary for testing purposes; however, the "thin" binary will not be +portable to machines using other architectures. + +### Building a multi-architecture iOS XCframework + +The `Apple` subfolder of the Python repository acts as a build script that +can be used to coordinate the compilation of a complete iOS XCframework. To use +it, run:: + + python Apple build iOS + +This will: + +* Configure and compile a version of Python to run on the build machine +* Download pre-compiled binary dependencies for each platform +* Configure and build a `Python.framework` for each required architecture and + iOS SDK +* Merge the multiple `Python.framework` folders into a single `Python.xcframework` +* Produce a `.tar.gz` archive in the `cross-build/dist` folder containing + the `Python.xcframework`, plus a copy of the Testbed app pre-configured to + use the XCframework. + +The `Apple` build script has other entry points that will perform the +individual parts of the overall `build` target, plus targets to test the +build, clean the `cross-build` folder of iOS build products, and perform a +complete "build and test" CI run. The `--clean` flag can also be used on +individual commands to ensure that a stale build product are removed before +building. + +### Building a single-architecture framework + +If you're using the `Apple` build script, you won't need to build +individual frameworks. However, if you do need to manually configure an iOS +Python build for a single framework, the following options are available. + +#### iOS specific arguments to configure + +* `--enable-framework[=DIR]` + + This argument specifies the location where the Python.framework will be + installed. If `DIR` is not specified, the framework will be installed into + a subdirectory of the `iOS/Frameworks` folder. + + This argument *must* be provided when configuring iOS builds. iOS does not + support non-framework builds. + +* `--with-framework-name=NAME` + + Specify the name for the Python framework; defaults to `Python`. + + > [!NOTE] + > Unless you know what you're doing, changing the name of the Python + > framework on iOS is not advised. If you use this option, you won't be able + > to run the `Apple` build script without making significant manual + > alterations, and you won't be able to use any binary packages unless you + > compile them yourself using your own framework name. + +#### Building Python for iOS + +The Python build system will create a `Python.framework` that supports a +*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a +framework to contain non-library content, so the iOS build will produce a +`bin` and `lib` folder in the same output folder as `Python.framework`. +The `lib` folder will be needed at runtime to support the Python library. + +If you want to use Python in a real iOS project, you need to produce multiple +`Python.framework` builds, one for each ABI and architecture. iOS builds of +Python *must* be constructed as framework builds. To support this, you must +provide the `--enable-framework` flag when configuring the build. The build +also requires the use of cross-compilation. The minimal commands for building +Python for the ARM64 iOS simulator will look something like: +``` +export PATH="$(pwd)/Apple/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" +./configure \ + --enable-framework \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=/path/to/python.exe +make +make install +``` + +In this invocation: + +* `Apple/iOS/Resources/bin` has been added to the path, providing some shims for the + compilers and linkers needed by the build. Xcode requires the use of `xcrun` + to invoke compiler tooling. However, if `xcrun` is pre-evaluated and the + result passed to `configure`, these results can embed user- and + version-specific paths into the sysconfig data, which limits the portability + of the compiled Python. Alternatively, if `xcrun` is used *as* the compiler, + it requires that compiler variables like `CC` include spaces, which can + cause significant problems with many C configuration systems which assume that + `CC` will be a single executable. + + To work around this problem, the `Apple/iOS/Resources/bin` folder contains some + wrapper scripts that present as simple compilers and linkers, but wrap + underlying calls to `xcrun`. This allows configure to use a `CC` + definition without spaces, and without user- or version-specific paths, while + retaining the ability to adapt to the local Xcode install. These scripts are + included in the `bin` directory of an iOS install. + + These scripts will, by default, use the currently active Xcode installation. + If you want to use a different Xcode installation, you can use + `xcode-select` to set a new default Xcode globally, or you can use the + `DEVELOPER_DIR` environment variable to specify an Xcode install. The + scripts will use the default `iphoneos`/`iphonesimulator` SDK version for + the select Xcode install; if you want to use a different SDK, you can set the + `IOS_SDK_VERSION` environment variable. (e.g, setting + `IOS_SDK_VERSION=17.1` would cause the scripts to use the `iphoneos17.1` + and `iphonesimulator17.1` SDKs, regardless of the Xcode default.) + + The path has also been cleared of any user customizations. A common source of + bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS + build. Resetting the path to a known "bare bones" value is the easiest way to + avoid these problems. + +* `--host` is the architecture and ABI that you want to build, in GNU compiler + triple format. This will be one of: + + - `arm64-apple-ios` for ARM64 iOS devices. + - `arm64-apple-ios-simulator` for the iOS simulator running on Apple + Silicon devices. + - `x86_64-apple-ios-simulator` for the iOS simulator running on Intel + devices. + +* `--build` is the GNU compiler triple for the machine that will be running + the compiler. This is one of: + + - `arm64-apple-darwin` for Apple Silicon devices. + - `x86_64-apple-darwin` for Intel devices. + +* `/path/to/python.exe` is the path to a Python binary on the machine that + will be running the compiler. This is needed because the Python compilation + process involves running some Python code. On a normal desktop build of + Python, you can compile a python interpreter and then use that interpreter to + run Python code. However, the binaries produced for iOS won't run on macOS, so + you need to provide an external Python interpreter. This interpreter must be + the same version as the Python that is being compiled. To be completely safe, + this should be the *exact* same commit hash. However, the longer a Python + release has been stable, the more likely it is that this constraint can be + relaxed - the same micro version will often be sufficient. + +* The `install` target for iOS builds is slightly different to other + platforms. On most platforms, `make install` will install the build into + the final runtime location. This won't be the case for iOS, as the final + runtime location will be on a physical device. + + However, you still need to run the `install` target for iOS builds, as it + performs some final framework assembly steps. The location specified with + `--enable-framework` will be the location where `make install` will + assemble the complete iOS framework. This completed framework can then + be copied and relocated as required. + +For a full CPython build, you also need to specify the paths to iOS builds of +the binary libraries that CPython depends on (such as XZ, LibFFI and OpenSSL). +This can be done by defining library specific environment variables (such as +`LIBLZMA_CFLAGS`, `LIBLZMA_LIBS`), and the `--with-openssl` configure +option. Versions of these libraries pre-compiled for iOS can be found in [this +repository](https://github.com/beeware/cpython-apple-source-deps/releases). +LibFFI is especially important, as many parts of the standard library +(including the `platform`, `sysconfig` and `webbrowser` modules) require +the use of the `ctypes` module at runtime. + +By default, Python will be compiled with an iOS deployment target (i.e., the +minimum supported iOS version) of 13.0. To specify a different deployment +target, provide the version number as part of the `--host` argument - for +example, `--host=arm64-apple-ios15.4-simulator` would compile an ARM64 +simulator build with a deployment target of 15.4. + +## Testing Python on iOS + +### Testing a multi-architecture framework + +Once you have a built an XCframework, you can test that framework by running: + + $ python Apple test iOS + +This test will attempt to find an "SE-class" simulator (i.e., an iPhone SE, or +iPhone 16e, or similar), and run the test suite on the most recent version of +iOS that is available. You can specify a simulator using the `--simulator` +command line argument, providing the name of the simulator (e.g., `--simulator +'iPhone 16 Pro'`). You can also use this argument to control the OS version used +for testing; `--simulator 'iPhone 16 Pro,OS=18.2'` would attempt to run the +tests on an iPhone 16 Pro running iOS 18.2. + +If the test runner is executed on GitHub Actions, the `GITHUB_ACTIONS` +environment variable will be exposed to the iOS process at runtime. + +### Testing a single-architecture framework + +The `Apple/testbed` folder that contains an Xcode project that is able to run +the Python test suite on Apple platforms. This project converts the Python test +suite into a single test case in Xcode's XCTest framework. The single XCTest +passes if the test suite passes. + +To run the test suite, configure a Python build for an iOS simulator (i.e., +`--host=arm64-apple-ios-simulator` or `--host=x86_64-apple-ios-simulator` +), specifying a framework build (i.e. `--enable-framework`). Ensure that your +`PATH` has been configured to include the `Apple/iOS/Resources/bin` folder and +exclude any non-iOS tools, then run: +``` +make all +make install +make testios +``` + +This will: + +* Build an iOS framework for your chosen architecture; +* Finalize the single-platform framework; +* Make a clean copy of the testbed project; +* Install the Python iOS framework into the copy of the testbed project; and +* Run the test suite on an "entry-level device" simulator (i.e., an iPhone SE, + iPhone 16e, or a similar). + +On success, the test suite will exit and report successful completion of the +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 +minutes to run; a couple of extra minutes is required to compile the testbed +project, and then boot and prepare the iOS simulator. + +### Debugging test failures + +Running `python Apple test iOS` generates a standalone version of the +`Apple/testbed` project, and runs the full test suite. It does this using +`Apple/testbed` itself - the folder is an executable module that can be used +to create and run a clone of the testbed project. The standalone version of the +testbed will be created in a directory named +`cross-build/iOS-testbed.`. + +You can generate your own standalone testbed instance by running: +``` +python cross-build/iOS/testbed clone my-testbed +``` + +In this invocation, `my-testbed` is the name of the folder for the new +testbed clone. + +If you've built your own XCframework, or you only want to test a single architecture, +you can construct a standalone testbed instance by running: +``` +python Apple/testbed clone --platform iOS --framework my-testbed +``` + +The framework path can be the path path to a `Python.xcframework`, or the +path to a folder that contains a single-platform `Python.framework`. + +You can then use the `my-testbed` folder to run the Python test suite, +passing in any command line arguments you may require. For example, if you're +trying to diagnose a failure in the `os` module, you might run: +``` +python my-testbed run -- test -W test_os +``` + +This is the equivalent of running `python -m test -W test_os` on a desktop +Python build. Any arguments after the `--` will be passed to testbed as if +they were arguments to `python -m` on a desktop machine. + +### Testing in Xcode + +You can also open the testbed project in Xcode by running: +``` +open my-testbed/iOSTestbed.xcodeproj +``` + +This will allow you to use the full Xcode suite of tools for debugging. + +The arguments used to run the test suite are defined as part of the test plan. +To modify the test plan, select the test plan node of the project tree (it +should be the first child of the root node), and select the "Configurations" +tab. Modify the "Arguments Passed On Launch" value to change the testing +arguments. + +The test plan also disables parallel testing, and specifies the use of the +`Testbed.lldbinit` file for providing configuration of the debugger. The +default debugger configuration disables automatic breakpoints on the +`SIGINT`, `SIGUSR1`, `SIGUSR2`, and `SIGXFSZ` signals. + +### Testing on an iOS device + +To test on an iOS device, the app needs to be signed with known developer +credentials. To obtain these credentials, you must have an iOS Developer +account, and your Xcode install will need to be logged into your account (see +the Accounts tab of the Preferences dialog). + +Once the project is open, and you're signed into your Apple Developer account, +select the root node of the project tree (labeled "iOSTestbed"), then the +"Signing & Capabilities" tab in the details page. Select a development team +(this will likely be your own name), and plug in a physical device to your +macOS machine with a USB cable. You should then be able to select your physical +device from the list of targets in the pulldown in the Xcode titlebar. diff --git a/iOS/Resources/Info.plist.in b/Apple/iOS/Resources/Info.plist.in similarity index 100% rename from iOS/Resources/Info.plist.in rename to Apple/iOS/Resources/Info.plist.in diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-ar b/Apple/iOS/Resources/bin/arm64-apple-ios-ar new file mode 100755 index 00000000000..3cf3eb21874 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang b/Apple/iOS/Resources/bin/arm64-apple-ios-clang new file mode 100755 index 00000000000..f50d5b5142f --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ new file mode 100755 index 00000000000..0794731d7dc --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-cpp b/Apple/iOS/Resources/bin/arm64-apple-ios-cpp new file mode 100755 index 00000000000..24fa1506bab --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} -E "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar new file mode 100755 index 00000000000..b836b6db902 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang new file mode 100755 index 00000000000..4891a00876e --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000..58b2a5f6f18 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp new file mode 100755 index 00000000000..c9df94e8b7c --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip new file mode 100755 index 00000000000..fd59d309b73 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-strip b/Apple/iOS/Resources/bin/arm64-apple-ios-strip new file mode 100755 index 00000000000..75e823a3d02 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar new file mode 100755 index 00000000000..b836b6db902 --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang new file mode 100755 index 00000000000..f4739a7b945 --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000..c348ae4c103 --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp new file mode 100755 index 00000000000..6d7f8084c9f --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip new file mode 100755 index 00000000000..c5cfb289291 --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch x86_64 "$@" diff --git a/iOS/Resources/pyconfig.h b/Apple/iOS/Resources/pyconfig.h similarity index 100% rename from iOS/Resources/pyconfig.h rename to Apple/iOS/Resources/pyconfig.h diff --git a/iOS/testbed/Python.xcframework/Info.plist b/Apple/testbed/Python.xcframework/Info.plist similarity index 100% rename from iOS/testbed/Python.xcframework/Info.plist rename to Apple/testbed/Python.xcframework/Info.plist diff --git a/iOS/Resources/dylib-Info-template.plist b/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist similarity index 96% rename from iOS/Resources/dylib-Info-template.plist rename to Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist index f652e272f71..d6caa01c1e4 100644 --- a/iOS/Resources/dylib-Info-template.plist +++ b/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist @@ -19,7 +19,7 @@ iPhoneOS MinimumOSVersion - 12.0 + 13.0 CFBundleVersion 1 diff --git a/Apple/testbed/Python.xcframework/build/utils.sh b/Apple/testbed/Python.xcframework/build/utils.sh new file mode 100755 index 00000000000..e7155d8b30e --- /dev/null +++ b/Apple/testbed/Python.xcframework/build/utils.sh @@ -0,0 +1,151 @@ +# Utility methods for use in an Xcode project. +# +# An iOS XCframework cannot include any content other than the library binary +# and relevant metadata. However, Python requires a standard library at runtime. +# Therefore, it is necessary to add a build step to an Xcode app target that +# processes the standard library and puts the content into the final app. +# +# In general, these tools will be invoked after bundle resources have been +# copied into the app, but before framework embedding (and signing). +# +# The following is an example script, assuming that: +# * Python.xcframework is in the root of the project +# * There is an `app` folder that contains the app code +# * There is an `app_packages` folder that contains installed Python packages. +# ----- +# set -e +# source $PROJECT_DIR/Python.xcframework/build/build_utils.sh +# install_python Python.xcframework app app_packages +# ----- + +# Copy the standard library from the XCframework into the app bundle. +# +# Accepts one argument: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +install_stdlib() { + PYTHON_XCFRAMEWORK_PATH=$1 + + mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" + if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then + echo "Installing Python modules for iOS Simulator" + if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/ios-arm64-simulator" ]; then + SLICE_FOLDER="ios-arm64-simulator" + else + SLICE_FOLDER="ios-arm64_x86_64-simulator" + fi + else + echo "Installing Python modules for iOS Device" + SLICE_FOLDER="ios-arm64" + fi + + # If the XCframework has a shared lib folder, then it's a full framework. + # Copy both the common and slice-specific part of the lib directory. + # Otherwise, it's a single-arch framework; use the "full" lib folder. + if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib" ]; then + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" + else + # A single-arch framework will have a libpython symlink; that can't be included at runtime + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' + fi +} + +# Convert a single .so library into a framework that iOS can load. +# +# Accepts three arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +# 2. The base path, relative to the installed location in the app bundle, that +# needs to be processed. Any .so file found in this path (or a subdirectory +# of it) will be processed. +# 2. The full path to a single .so file to process. This path should include +# the base path. +install_dylib () { + PYTHON_XCFRAMEWORK_PATH=$1 + INSTALL_BASE=$2 + FULL_EXT=$3 + + # The name of the extension file + EXT=$(basename "$FULL_EXT") + # The name and location of the module + MODULE_PATH=$(dirname "$FULL_EXT") + MODULE_NAME=$(echo $EXT | cut -d "." -f 1) + # The location of the extension file, relative to the bundle + RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} + # The path to the extension file, relative to the install base + PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} + # The full dotted name of the extension module, constructed from the file path. + FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); + # A bundle identifier; not actually used, but required by Xcode framework packaging + FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") + # The name of the framework folder. + FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" + + # If the framework folder doesn't exist, create it. + if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then + echo "Creating framework for $RELATIVE_EXT" + mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" + cp "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/build/$PLATFORM_FAMILY_NAME-dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + fi + + echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + # Create a placeholder .fwork file where the .so was + echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork + # Create a back reference to the .so file location in the framework + echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" + + # If the framework provides an xcprivacy file, install it. + if [ -e "$MODULE_PATH/$MODULE_NAME.xcprivacy" ]; then + echo "Installing XCPrivacy file for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + XCPRIVACY_FILE="$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/PrivacyInfo.xcprivacy" + if [ -e "$XCPRIVACY_FILE" ]; then + rm -rf "$XCPRIVACY_FILE" + fi + mv "$MODULE_PATH/$MODULE_NAME.xcprivacy" "$XCPRIVACY_FILE" + fi + + echo "Signing framework as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." + /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" +} + +# Process all the dynamic libraries in a path into Framework format. +# +# Accepts two arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +# 2. The base path, relative to the installed location in the app bundle, that +# needs to be processed. Any .so file found in this path (or a subdirectory +# of it) will be processed. +process_dylibs () { + PYTHON_XCFRAMEWORK_PATH=$1 + LIB_PATH=$2 + find "$CODESIGNING_FOLDER_PATH/$LIB_PATH" -name "*.so" | while read FULL_EXT; do + install_dylib $PYTHON_XCFRAMEWORK_PATH "$LIB_PATH/" "$FULL_EXT" + done +} + +# The entry point for post-processing a Python XCframework. +# +# Accepts 1 or more arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. If the XCframework is in the root of the project, +# 2+. The path of a package, relative to the root of the packaged app, that contains +# library content that should be processed for binary libraries. +install_python() { + PYTHON_XCFRAMEWORK_PATH=$1 + shift + + install_stdlib $PYTHON_XCFRAMEWORK_PATH + PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") + echo "Install Python $PYTHON_VER standard library extension modules..." + process_dylibs $PYTHON_XCFRAMEWORK_PATH python/lib/$PYTHON_VER/lib-dynload + + for package_path in $@; do + echo "Installing $package_path extension modules ..." + process_dylibs $PYTHON_XCFRAMEWORK_PATH $package_path + done +} diff --git a/iOS/testbed/Python.xcframework/ios-arm64/README b/Apple/testbed/Python.xcframework/ios-arm64/README similarity index 100% rename from iOS/testbed/Python.xcframework/ios-arm64/README rename to Apple/testbed/Python.xcframework/ios-arm64/README diff --git a/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README b/Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README similarity index 100% rename from iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README rename to Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README diff --git a/Apple/testbed/Testbed.lldbinit b/Apple/testbed/Testbed.lldbinit new file mode 100644 index 00000000000..4cf00dd0f9d --- /dev/null +++ b/Apple/testbed/Testbed.lldbinit @@ -0,0 +1,4 @@ +process handle SIGINT -n true -p true -s false +process handle SIGUSR1 -n true -p true -s false +process handle SIGUSR2 -n true -p true -s false +process handle SIGXFSZ -n true -p true -s false diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/Apple/testbed/TestbedTests/TestbedTests.m similarity index 69% rename from iOS/testbed/iOSTestbedTests/iOSTestbedTests.m rename to Apple/testbed/TestbedTests/TestbedTests.m index dd6e76f9496..f7788c47f2c 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/Apple/testbed/TestbedTests/TestbedTests.m @@ -1,11 +1,11 @@ #import #import -@interface iOSTestbedTests : XCTestCase +@interface TestbedTests : XCTestCase @end -@implementation iOSTestbedTests +@implementation TestbedTests - (void)testPython { @@ -15,6 +15,11 @@ PyStatus status; PyPreConfig preconfig; PyConfig config; + PyObject *app_packages_path; + PyObject *method_args; + PyObject *result; + PyObject *site_module; + PyObject *site_addsitedir_attr; PyObject *sys_module; PyObject *sys_path_attr; NSArray *test_args; @@ -30,19 +35,26 @@ setenv("NO_COLOR", "1", true); setenv("PYTHON_COLORS", "0", true); + if (getenv("GITHUB_ACTIONS")) { + NSLog(@"Running in a GitHub Actions environment"); + } // Arguments to pass into the test suite runner. // argv[0] must identify the process; any subsequent arg // will be handled as if it were an argument to `python -m test` - test_args = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"TestArgs"]; + // The processInfo arguments contain the binary that is running, + // followed by the arguments defined in the test plan. This means: + // run_module = test_args[1] + // argv = ["Testbed"] + test_args[2:] + test_args = [[NSProcessInfo processInfo] arguments]; if (test_args == NULL) { NSLog(@"Unable to identify test arguments."); } - argv = malloc(sizeof(char *) * ([test_args count] + 1)); - argv[0] = "iOSTestbed"; - for (int i = 1; i < [test_args count]; i++) { - argv[i] = [[test_args objectAtIndex:i] UTF8String]; + NSLog(@"Test arguments: %@", test_args); + argv = malloc(sizeof(char *) * ([test_args count] - 1)); + argv[0] = "Testbed"; + for (int i = 1; i < [test_args count] - 1; i++) { + argv[i] = [[test_args objectAtIndex:i+1] UTF8String]; } - NSLog(@"Test command: %@", test_args); // Generate an isolated Python configuration. NSLog(@"Configuring isolated Python..."); @@ -63,7 +75,7 @@ // Ensure that signal handlers are installed config.install_signal_handlers = 1; // Run the test module. - config.run_module = Py_DecodeLocale([[test_args objectAtIndex:0] UTF8String], NULL); + config.run_module = Py_DecodeLocale([[test_args objectAtIndex:1] UTF8String], NULL); // For debugging - enable verbose mode. // config.verbose = 1; @@ -96,7 +108,7 @@ } NSLog(@"Configure argc/argv..."); - status = PyConfig_SetBytesArgv(&config, [test_args count], (char**) argv); + status = PyConfig_SetBytesArgv(&config, [test_args count] - 1, (char**) argv); if (PyStatus_Exception(status)) { XCTFail(@"Unable to configure argc/argv: %s", status.err_msg); PyConfig_Clear(&config); @@ -111,6 +123,43 @@ return; } + // Add app_packages as a site directory. This both adds to sys.path, + // and ensures that any .pth files in that directory will be executed. + site_module = PyImport_ImportModule("site"); + if (site_module == NULL) { + XCTFail(@"Could not import site module"); + return; + } + + site_addsitedir_attr = PyObject_GetAttrString(site_module, "addsitedir"); + if (site_addsitedir_attr == NULL || !PyCallable_Check(site_addsitedir_attr)) { + XCTFail(@"Could not access site.addsitedir"); + return; + } + + path = [NSString stringWithFormat:@"%@/app_packages", resourcePath, nil]; + NSLog(@"App packages path: %@", path); + wtmp_str = Py_DecodeLocale([path UTF8String], NULL); + app_packages_path = PyUnicode_FromWideChar(wtmp_str, wcslen(wtmp_str)); + if (app_packages_path == NULL) { + XCTFail(@"Could not convert app_packages path to unicode"); + return; + } + PyMem_RawFree(wtmp_str); + + method_args = Py_BuildValue("(O)", app_packages_path); + if (method_args == NULL) { + XCTFail(@"Could not create arguments for site.addsitedir"); + return; + } + + result = PyObject_CallObject(site_addsitedir_attr, method_args); + if (result == NULL) { + XCTFail(@"Could not add app_packages directory using site.addsitedir"); + return; + } + + // Add test code to sys.path sys_module = PyImport_ImportModule("sys"); if (sys_module == NULL) { XCTFail(@"Could not import sys module"); @@ -123,17 +172,6 @@ return; } - // Add the app packages path - path = [NSString stringWithFormat:@"%@/app_packages", resourcePath, nil]; - NSLog(@"App packages path: %@", path); - wtmp_str = Py_DecodeLocale([path UTF8String], NULL); - failed = PyList_Insert(sys_path_attr, 0, PyUnicode_FromString([path UTF8String])); - if (failed) { - XCTFail(@"Unable to add app packages to sys.path"); - return; - } - PyMem_RawFree(wtmp_str); - path = [NSString stringWithFormat:@"%@/app", resourcePath, nil]; NSLog(@"App path: %@", path); wtmp_str = Py_DecodeLocale([path UTF8String], NULL); diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py new file mode 100644 index 00000000000..0dd77ab8b82 --- /dev/null +++ b/Apple/testbed/__main__.py @@ -0,0 +1,435 @@ +import argparse +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +TEST_SLICES = { + "iOS": "ios-arm64_x86_64-simulator", +} + +DECODE_ARGS = ("UTF-8", "backslashreplace") + +# The system log prefixes each line: +# 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ... +# 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ... + +LOG_PREFIX_REGEX = re.compile( + r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD + r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}" # HH:MM:SS.ssssss+ZZZZ + r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID +) + + +# Select a simulator device to use. +def select_simulator_device(platform): + # List the testing simulators, in JSON format + raw_json = subprocess.check_output(["xcrun", "simctl", "list", "-j"]) + json_data = json.loads(raw_json) + + if platform == "iOS": + # Any iOS device will do; we'll look for "SE" devices - but the name + # isn't consistent over time. Older Xcode versions will use "iPhone SE + # (Nth generation)"; As of 2025, they've started using "iPhone 16e". + # + # When Xcode is updated after a new release, new devices will be + # available and old ones will be dropped from the set available on the + # latest iOS version. Select the one with the highest minimum runtime + # version - this is an indicator of the "newest" released device, which + # should always be supported on the "most recent" iOS version. + se_simulators = sorted( + (devicetype["minRuntimeVersion"], devicetype["name"]) + for devicetype in json_data["devicetypes"] + if devicetype["productFamily"] == "iPhone" + and ( + ( + "iPhone " in devicetype["name"] + and devicetype["name"].endswith("e") + ) + or "iPhone SE " in devicetype["name"] + ) + ) + simulator = se_simulators[-1][1] + else: + raise ValueError(f"Unknown platform {platform}") + + return simulator + + +def xcode_test(location: Path, platform: str, simulator: str, verbose: bool): + # Build and run the test suite on the named simulator. + args = [ + "-project", + str(location / f"{platform}Testbed.xcodeproj"), + "-scheme", + f"{platform}Testbed", + "-destination", + f"platform={platform} Simulator,name={simulator}", + "-derivedDataPath", + str(location / "DerivedData"), + ] + verbosity_args = [] if verbose else ["-quiet"] + + print("Building test project...") + subprocess.run( + ["xcodebuild", "build-for-testing"] + args + verbosity_args, + check=True, + ) + + # Any environment variable prefixed with TEST_RUNNER_ is exposed into the + # test runner environment. There are some variables (like those identifying + # CI platforms) that can be useful to have access to. + test_env = os.environ.copy() + if "GITHUB_ACTIONS" in os.environ: + test_env["TEST_RUNNER_GITHUB_ACTIONS"] = os.environ["GITHUB_ACTIONS"] + + print("Running test project...") + # Test execution *can't* be run -quiet; verbose mode + # is how we see the output of the test output. + process = subprocess.Popen( + ["xcodebuild", "test-without-building"] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=test_env, + ) + while line := (process.stdout.readline()).decode(*DECODE_ARGS): + # Strip the timestamp/process prefix from each log line + line = LOG_PREFIX_REGEX.sub("", line) + sys.stdout.write(line) + sys.stdout.flush() + + status = process.wait(timeout=5) + exit(status) + + +def copy(src, tgt): + """An all-purpose copy. + + If src is a file, it is copied. If src is a symlink, it is copied *as a + symlink*. If src is a directory, the full tree is duplicated, with symlinks + being preserved. + """ + if src.is_file() or src.is_symlink(): + shutil.copyfile(src, tgt, follow_symlinks=False) + else: + shutil.copytree(src, tgt, symlinks=True) + + +def clone_testbed( + source: Path, + target: Path, + framework: Path, + platform: str, + apps: list[Path], +) -> None: + if target.exists(): + print(f"{target} already exists; aborting without creating project.") + sys.exit(10) + + if framework is None: + if not ( + source / "Python.xcframework" / TEST_SLICES[platform] / "bin" + ).is_dir(): + print( + f"The testbed being cloned ({source}) does not contain " + "a framework with slices. Re-run with --framework" + ) + sys.exit(11) + else: + if not framework.is_dir(): + print(f"{framework} does not exist.") + sys.exit(12) + elif not ( + framework.suffix == ".xcframework" + or (framework / "Python.framework").is_dir() + ): + print( + f"{framework} is not an XCframework, " + f"or a simulator slice of a framework build." + ) + sys.exit(13) + + print("Cloning testbed project:") + print(f" Cloning {source}...", end="") + # Only copy the files for the platform being cloned plus the files common + # to all platforms. The XCframework will be copied later, if needed. + target.mkdir(parents=True) + + for name in [ + "__main__.py", + "TestbedTests", + "Testbed.lldbinit", + f"{platform}Testbed", + f"{platform}Testbed.xcodeproj", + f"{platform}Testbed.xctestplan", + ]: + copy(source / name, target / name) + + print(" done") + + orig_xc_framework_path = source / "Python.xcframework" + xc_framework_path = target / "Python.xcframework" + test_framework_path = xc_framework_path / TEST_SLICES[platform] + if framework is not None: + if framework.suffix == ".xcframework": + print(" Installing XCFramework...", end="") + xc_framework_path.symlink_to( + framework.relative_to(xc_framework_path.parent, walk_up=True) + ) + print(" done") + else: + print(" Installing simulator framework...", end="") + # We're only installing a slice of a framework; we need + # to do a full tree copy to make sure we don't damage + # symlinked content. + shutil.copytree(orig_xc_framework_path, xc_framework_path) + if test_framework_path.is_dir(): + shutil.rmtree(test_framework_path) + else: + test_framework_path.unlink(missing_ok=True) + test_framework_path.symlink_to( + framework.relative_to(test_framework_path.parent, walk_up=True) + ) + print(" done") + else: + copy(orig_xc_framework_path, xc_framework_path) + + if ( + xc_framework_path.is_symlink() + and not xc_framework_path.readlink().is_absolute() + ): + # XCFramework is a relative symlink. Rewrite the symlink relative + # to the new location. + print(" Rewriting symlink to XCframework...", end="") + resolved_xc_framework_path = ( + source / xc_framework_path.readlink() + ).resolve() + xc_framework_path.unlink() + xc_framework_path.symlink_to( + resolved_xc_framework_path.relative_to( + xc_framework_path.parent, walk_up=True + ) + ) + print(" done") + elif ( + test_framework_path.is_symlink() + and not test_framework_path.readlink().is_absolute() + ): + print(" Rewriting symlink to simulator framework...", end="") + # Simulator framework is a relative symlink. Rewrite the symlink + # relative to the new location. + orig_test_framework_path = ( + source / "Python.XCframework" / test_framework_path.readlink() + ).resolve() + test_framework_path.unlink() + test_framework_path.symlink_to( + orig_test_framework_path.relative_to( + test_framework_path.parent, walk_up=True + ) + ) + print(" done") + else: + print(" Using pre-existing Python framework.") + + for app_src in apps: + print(f" Installing app {app_src.name!r}...", end="") + app_target = target / f"Testbed/app/{app_src.name}" + if app_target.is_dir(): + shutil.rmtree(app_target) + shutil.copytree(app_src, app_target) + print(" done") + + print(f"Successfully cloned testbed: {target.resolve()}") + + +def update_test_plan(testbed_path, platform, args): + # Modify the test plan to use the requested test arguments. + test_plan_path = testbed_path / f"{platform}Testbed.xctestplan" + with test_plan_path.open("r", encoding="utf-8") as f: + test_plan = json.load(f) + + test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ + {"argument": shlex.quote(arg)} for arg in args + ] + + with test_plan_path.open("w", encoding="utf-8") as f: + json.dump(test_plan, f, indent=2) + + +def run_testbed( + platform: str, + simulator: str | None, + args: list[str], + verbose: bool = False, +): + location = Path(__file__).parent + print("Updating test plan...", end="") + update_test_plan(location, platform, args) + print(" done.") + + if simulator is None: + simulator = select_simulator_device(platform) + print(f"Running test on {simulator}") + + xcode_test( + location, + platform=platform, + simulator=simulator, + verbose=verbose, + ) + + +def main(): + # Look for directories like `iOSTestbed` as an indicator of the platforms + # that the testbed folder supports. The original source testbed can support + # many platforms, but when cloned, only one platform is preserved. + available_platforms = [ + platform + for platform in ["iOS"] + if (Path(__file__).parent / f"{platform}Testbed").is_dir() + ] + + parser = argparse.ArgumentParser( + description=( + "Manages the process of testing an Apple Python project " + "through Xcode." + ), + ) + + subcommands = parser.add_subparsers(dest="subcommand") + clone = subcommands.add_parser( + "clone", + description=( + "Clone the testbed project, copying in a Python framework and" + "any specified application code." + ), + help="Clone a testbed project to a new location.", + ) + clone.add_argument( + "--framework", + help=( + "The location of the XCFramework (or simulator-only slice of an " + "XCFramework) to use when running the testbed" + ), + ) + clone.add_argument( + "--platform", + dest="platform", + choices=available_platforms, + default=available_platforms[0], + help=f"The platform to target (default: {available_platforms[0]})", + ) + clone.add_argument( + "--app", + dest="apps", + action="append", + default=[], + help="The location of any code to include in the testbed project", + ) + clone.add_argument( + "location", + help="The path where the testbed will be cloned.", + ) + + run = subcommands.add_parser( + "run", + usage=( + "%(prog)s [-h] [--simulator SIMULATOR] -- " + " [ ...]" + ), + description=( + "Run a testbed project. The arguments provided after `--` will be " + "passed to the running iOS process as if they were arguments to " + "`python -m`." + ), + help="Run a testbed project", + ) + run.add_argument( + "--platform", + dest="platform", + choices=available_platforms, + default=available_platforms[0], + help=f"The platform to target (default: {available_platforms[0]})", + ) + run.add_argument( + "--simulator", + help=( + "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " + "the most recently released 'entry level' iPhone device. Device " + "architecture and OS version can also be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " + "an ARM64 iPhone 16 Pro simulator running iOS 26.0." + ), + ) + run.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable verbose output", + ) + + try: + pos = sys.argv.index("--") + testbed_args = sys.argv[1:pos] + test_args = sys.argv[pos + 1 :] + except ValueError: + testbed_args = sys.argv[1:] + test_args = [] + + context = parser.parse_args(testbed_args) + + if context.subcommand == "clone": + clone_testbed( + source=Path(__file__).parent.resolve(), + target=Path(context.location).resolve(), + framework=Path(context.framework).resolve() + if context.framework + else None, + platform=context.platform, + apps=[Path(app) for app in context.apps], + ) + elif context.subcommand == "run": + if test_args: + if not ( + Path(__file__).parent + / "Python.xcframework" + / TEST_SLICES[context.platform] + / "bin" + ).is_dir(): + print( + "Testbed does not contain a compiled Python framework. " + f"Use `python {sys.argv[0]} clone ...` to create a " + "runnable clone of this testbed." + ) + sys.exit(20) + + run_testbed( + platform=context.platform, + simulator=context.simulator, + verbose=context.verbose, + args=test_args, + ) + else: + print( + "Must specify test arguments " + f"(e.g., {sys.argv[0]} run -- test)" + ) + print() + parser.print_help(sys.stderr) + sys.exit(21) + else: + parser.print_help(sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + main() diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj similarity index 78% rename from iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj rename to Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj index c7d63909ee2..f8835a3bc58 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -11,12 +11,11 @@ 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 607A66212B0EFA390010BFC8 /* Assets.xcassets */; }; 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */; }; 607A66282B0EFA390010BFC8 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66272B0EFA390010BFC8 /* main.m */; }; - 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */; }; + 607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */; }; 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; - 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; }; 608619542CB77BA900F46182 /* app_packages in Resources */ = {isa = PBXBuildFile; fileRef = 608619532CB77BA900F46182 /* app_packages */; }; 608619562CB7819B00F46182 /* app in Resources */ = {isa = PBXBuildFile; fileRef = 608619552CB7819B00F46182 /* app */; }; /* End PBXBuildFile section */ @@ -64,12 +63,12 @@ 607A66242B0EFA390010BFC8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 607A66272B0EFA390010BFC8 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = iOSTestbedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; - 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iOSTestbedTests.m; sourceTree = ""; }; + 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = TestbedTests.m; sourceTree = ""; }; 607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = ""; }; - 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = ""; }; 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = ""; }; 608619532CB77BA900F46182 /* app_packages */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app_packages; sourceTree = ""; }; 608619552CB7819B00F46182 /* app */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app; sourceTree = ""; }; + 60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = iOSTestbed.xctestplan; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -95,9 +94,10 @@ 607A66092B0EFA380010BFC8 = { isa = PBXGroup; children = ( + 60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */, 607A664A2B0EFB310010BFC8 /* Python.xcframework */, 607A66142B0EFA380010BFC8 /* iOSTestbed */, - 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */, + 607A66302B0EFA3A0010BFC8 /* TestbedTests */, 607A66132B0EFA380010BFC8 /* Products */, 607A664F2B0EFFE00010BFC8 /* Frameworks */, ); @@ -118,7 +118,6 @@ 608619552CB7819B00F46182 /* app */, 608619532CB77BA900F46182 /* app_packages */, 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */, - 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */, 607A66152B0EFA380010BFC8 /* AppDelegate.h */, 607A66162B0EFA380010BFC8 /* AppDelegate.m */, 607A66212B0EFA390010BFC8 /* Assets.xcassets */, @@ -128,12 +127,12 @@ path = iOSTestbed; sourceTree = ""; }; - 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + 607A66302B0EFA3A0010BFC8 /* TestbedTests */ = { isa = PBXGroup; children = ( - 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */, + 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */, ); - path = iOSTestbedTests; + path = TestbedTests; sourceTree = ""; }; 607A664F2B0EFFE00010BFC8 /* Frameworks */ = { @@ -153,8 +152,7 @@ 607A660E2B0EFA380010BFC8 /* Sources */, 607A660F2B0EFA380010BFC8 /* Frameworks */, 607A66102B0EFA380010BFC8 /* Resources */, - 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */, - 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */, + 607A66552B0F061D0010BFC8 /* Process Python libraries */, 607A664E2B0EFC080010BFC8 /* Embed Frameworks */, ); buildRules = ( @@ -228,7 +226,6 @@ buildActionMask = 2147483647; files = ( 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */, - 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */, 608619562CB7819B00F46182 /* app in Resources */, 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */, 608619542CB77BA900F46182 /* app_packages in Resources */, @@ -245,7 +242,7 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ - 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */ = { + 607A66552B0F061D0010BFC8 /* Process Python libraries */ = { isa = PBXShellScriptBuildPhase; alwaysOutOfDate = 1; buildActionMask = 2147483647; @@ -255,34 +252,14 @@ ); inputPaths = ( ); - name = "Install Target Specific Python Standard Library"; + name = "Process Python libraries"; outputFileListPaths = ( ); outputPaths = ( ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; - showEnvVarsInLog = 0; - }; - 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { - isa = PBXShellScriptBuildPhase; - alwaysOutOfDate = 1; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - ); - inputPaths = ( - ); - name = "Prepare Python Binary Modules"; - outputFileListPaths = ( - ); - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\nsource $PROJECT_DIR/Python.xcframework/build/utils.sh\ninstall_python Python.xcframework app app_packages\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ @@ -301,7 +278,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */, + 607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -379,7 +356,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; @@ -434,7 +411,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; @@ -460,7 +437,7 @@ INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -491,7 +468,7 @@ INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -514,7 +491,7 @@ DEVELOPMENT_TEAM = 3HEZE76D99; GENERATE_INFOPLIST_FILE = YES; HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -534,7 +511,7 @@ DEVELOPMENT_TEAM = 3HEZE76D99; GENERATE_INFOPLIST_FILE = YES; HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme b/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme new file mode 100644 index 00000000000..3c330a4152b --- /dev/null +++ b/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Apple/testbed/iOSTestbed.xctestplan b/Apple/testbed/iOSTestbed.xctestplan new file mode 100644 index 00000000000..0c4ab9eb2ba --- /dev/null +++ b/Apple/testbed/iOSTestbed.xctestplan @@ -0,0 +1,46 @@ +{ + "configurations" : [ + { + "id" : "F5A95CE4-1ADE-4A6E-A0E1-CDBAE26DF0C5", + "name" : "Test Scheme Action", + "options" : { + + } + } + ], + "defaultOptions" : { + "commandLineArgumentEntries" : [ + { + "argument" : "test" + }, + { + "argument" : "-uall" + }, + { + "argument" : "--single-process" + }, + { + "argument" : "--rerun" + }, + { + "argument" : "-W" + } + ], + "targetForVariableExpansion" : { + "containerPath" : "container:iOSTestbed.xcodeproj", + "identifier" : "607A66112B0EFA380010BFC8", + "name" : "iOSTestbed" + } + }, + "testTargets" : [ + { + "parallelizable" : false, + "target" : { + "containerPath" : "container:iOSTestbed.xcodeproj", + "identifier" : "607A662C2B0EFA3A0010BFC8", + "name" : "iOSTestbedTests" + } + } + ], + "version" : 1 +} diff --git a/iOS/testbed/iOSTestbed/AppDelegate.h b/Apple/testbed/iOSTestbed/AppDelegate.h similarity index 100% rename from iOS/testbed/iOSTestbed/AppDelegate.h rename to Apple/testbed/iOSTestbed/AppDelegate.h diff --git a/iOS/testbed/iOSTestbed/AppDelegate.m b/Apple/testbed/iOSTestbed/AppDelegate.m similarity index 100% rename from iOS/testbed/iOSTestbed/AppDelegate.m rename to Apple/testbed/iOSTestbed/AppDelegate.m diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json diff --git a/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard b/Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard similarity index 100% rename from iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard rename to Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard diff --git a/iOS/testbed/iOSTestbed/app/README b/Apple/testbed/iOSTestbed/app/README similarity index 92% rename from iOS/testbed/iOSTestbed/app/README rename to Apple/testbed/iOSTestbed/app/README index af22c685f87..46c0e8e2a29 100644 --- a/iOS/testbed/iOSTestbed/app/README +++ b/Apple/testbed/iOSTestbed/app/README @@ -1,7 +1,7 @@ This folder can contain any Python application code. During the build, any binary modules found in this folder will be processed into -iOS Framework form. +Framework form. When the test suite runs, this folder will be on the PYTHONPATH, and will be the working directory for the test suite. diff --git a/iOS/testbed/iOSTestbed/app_packages/README b/Apple/testbed/iOSTestbed/app_packages/README similarity index 92% rename from iOS/testbed/iOSTestbed/app_packages/README rename to Apple/testbed/iOSTestbed/app_packages/README index 42d7fdeb813..02c2beccfbd 100644 --- a/iOS/testbed/iOSTestbed/app_packages/README +++ b/Apple/testbed/iOSTestbed/app_packages/README @@ -2,6 +2,6 @@ This folder can be a target for installing any Python dependencies needed by the test suite. During the build, any binary modules found in this folder will be processed into -iOS Framework form. +Framework form. When the test suite runs, this folder will be on the PYTHONPATH. diff --git a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist b/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist similarity index 74% rename from iOS/testbed/iOSTestbed/iOSTestbed-Info.plist rename to Apple/testbed/iOSTestbed/iOSTestbed-Info.plist index a582f42a212..fea45e1fad6 100644 --- a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist +++ b/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist @@ -41,18 +41,6 @@ UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight - TestArgs - - test - -uall - --single-process - --rerun - -W - - UIApplicationSceneManifest UIApplicationSupportsMultipleScenes diff --git a/iOS/testbed/iOSTestbed/main.m b/Apple/testbed/iOSTestbed/main.m similarity index 100% rename from iOS/testbed/iOSTestbed/main.m rename to Apple/testbed/iOSTestbed/main.m diff --git a/Doc/Makefile b/Doc/Makefile index c8a749a02a8..f16d9cacb1b 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -170,6 +170,7 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ + set -e; \ echo "Creating venv in $(VENVDIR)"; \ if $(UV) --version >/dev/null 2>&1; then \ $(UV) venv --python=$(PYTHON) $(VENVDIR); \ @@ -183,7 +184,7 @@ venv: fi .PHONY: dist-no-html -dist-no-html: dist-text dist-pdf dist-epub dist-texinfo +dist-no-html: dist-text dist-epub dist-texinfo .PHONY: dist dist: @@ -240,7 +241,8 @@ dist-pdf: # as otherwise the full latexmk process is run twice. # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile - (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) + if [ -n "$(filter output-sync,$(value .FEATURES))" ]; then OUTPUTSYNC=--output-sync; else OUTPUTSYNC=; fi && \ + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`getconf _NPROCESSORS_ONLN`+1)) $$OUTPUTSYNC LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" diff --git a/Doc/about.rst b/Doc/about.rst index 8f635d7f743..5c1b497ca6b 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -32,8 +32,9 @@ Contributors to the Python documentation ---------------------------------------- Many people have contributed to the Python language, the Python standard -library, and the Python documentation. See :source:`Misc/ACKS` in the Python -source distribution for a partial list of contributors. +library, and the Python documentation. See the `CPython +GitHub repository `__ +for a partial list of contributors. It is only with the input and contributions of the Python community that Python has such wonderful documentation -- Thank You! diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 5d0f68ca696..0683eebbaf6 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -19,6 +19,12 @@ If you find a bug in this documentation or would like to propose an improvement, please submit a bug report on the :ref:`issue tracker `. If you have a suggestion on how to fix it, include that as well. +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, submit the report to the + `translation’s repository `_ instead. + You can also open a discussion item on our `Documentation Discourse forum `_. @@ -37,8 +43,8 @@ tracker `_. `Helping with Documentation `_ Comprehensive guide for individuals that are interested in contributing to Python documentation. - `Documentation Translations `_ - A list of GitHub pages for documentation translation and their primary contacts. + `Documentation Translations `_ + A list of GitHub pages for documentation translation and their coordination teams. .. _using-the-tracker: diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 7cbc99ad145..59044d2d88c 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -16,7 +16,20 @@ Allocating Objects on the Heap Initialize a newly allocated object *op* with its type and initial reference. Returns the initialized object. Other fields of the object are - not affected. + not initialized. Despite its name, this function is unrelated to the + object's :meth:`~object.__init__` method (:c:member:`~PyTypeObject.tp_init` + slot). Specifically, this function does **not** call the object's + :meth:`!__init__` method. + + In general, consider this function to be a low-level routine. Use + :c:member:`~PyTypeObject.tp_alloc` where possible. + For implementing :c:member:`!tp_alloc` for your type, prefer + :c:func:`PyType_GenericAlloc` or :c:func:`PyObject_New`. + + .. note:: + + This function only initializes the object's memory corresponding to the + initial :c:type:`PyObject` structure. It does not zero the rest. .. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) @@ -24,43 +37,108 @@ Allocating Objects on the Heap This does everything :c:func:`PyObject_Init` does, and also initializes the length information for a variable-size object. + .. note:: + + This function only initializes some of the object's memory. It does not + zero the rest. + .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* - and the Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header are not initialized. - The caller will own the only reference to the object - (i.e. its reference count will be one). - The size of the memory allocation is determined from the - :c:member:`~PyTypeObject.tp_basicsize` field of the type object. + Allocates a new Python object using the C structure type *TYPE* and the + Python type object *typeobj* (``PyTypeObject*``) by calling + :c:func:`PyObject_Malloc` to allocate memory and initializing it like + :c:func:`PyObject_Init`. The caller will own the only reference to the + object (i.e. its reference count will be one). - Note that this function is unsuitable if *typeobj* has - :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects, - use :c:func:`PyObject_GC_New` instead. + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This macro does not call :c:member:`~PyTypeObject.tp_alloc`, + :c:member:`~PyTypeObject.tp_new` (:meth:`~object.__new__`), or + :c:member:`~PyTypeObject.tp_init` (:meth:`~object.__init__`). + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_New` instead. + + Memory allocated by this macro must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *foo = PyObject_CallNoArgs((PyObject *)&PyFoo_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* (``Py_ssize_t``) fields of the size - given by the :c:member:`~PyTypeObject.tp_itemsize` field of - *typeobj*. This is useful for implementing objects like tuples, which are - able to determine their size at construction time. Embedding the array of - fields into the same allocation decreases the number of allocations, - improving the memory management efficiency. + Like :c:macro:`PyObject_New` except: - Note that this function is unsuitable if *typeobj* has - :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects, - use :c:func:`PyObject_GC_NewVar` instead. + * It allocates enough memory for the *TYPE* structure plus *size* + (``Py_ssize_t``) fields of the size given by the + :c:member:`~PyTypeObject.tp_itemsize` field of *typeobj*. + * The memory is initialized like :c:func:`PyObject_InitVar`. + This is useful for implementing objects like tuples, which are able to + determine their size at construction time. Embedding the array of fields + into the same allocation decreases the number of allocations, improving the + memory management efficiency. -.. c:function:: void PyObject_Del(void *op) + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_NewVar` + instead. + + Memory allocated by this function must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *list_instance = PyObject_CallNoArgs((PyObject *)&PyList_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` - Same as :c:func:`PyObject_Free`. .. c:var:: PyObject _Py_NoneStruct @@ -71,6 +149,38 @@ Allocating Objects on the Heap .. seealso:: - :c:func:`PyModule_Create` + :ref:`moduleobjects` To allocate and create extension modules. + +Deprecated aliases +^^^^^^^^^^^^^^^^^^ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Function + * * .. c:macro:: PyObject_NEW(type, typeobj) + * :c:macro:`PyObject_New` + * * .. c:macro:: PyObject_NEW_VAR(type, typeobj, n) + * :c:macro:`PyObject_NewVar` + * * .. c:macro:: PyObject_INIT(op, typeobj) + * :c:func:`PyObject_Init` + * * .. c:macro:: PyObject_INIT_VAR(op, typeobj, n) + * :c:func:`PyObject_InitVar` + * * .. c:macro:: PyObject_MALLOC(n) + * :c:func:`PyObject_Malloc` + * * .. c:macro:: PyObject_REALLOC(p, n) + * :c:func:`PyObject_Realloc` + * * .. c:macro:: PyObject_FREE(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_DEL(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_Del(p) + * :c:func:`PyObject_Free` diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 3bbc990b632..fd6be6a9b67 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -113,18 +113,14 @@ There are three ways strings and buffers can be converted to C: ``z`` (:class:`str` or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s?`` with the C pointer was initialized to ``NULL``. ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. - It is the same as ``s*?`` with the ``buf`` member of the :c:type:`Py_buffer` - structure was initialized to ``NULL``. ``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Like ``s#``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s#?`` with the C pointer was initialized to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] This format converts a bytes-like object to a C pointer to a @@ -164,7 +160,7 @@ There are three ways strings and buffers can be converted to C: ``w*`` (read-write :term:`bytes-like object`) [Py_buffer] This format accepts any object which implements the read-write buffer interface. It fills a :c:type:`Py_buffer` structure provided by the caller. - The buffer may contain embedded null bytes. The caller have to call + The buffer may contain embedded null bytes. The caller has to call :c:func:`PyBuffer_Release` when it is done with the buffer. ``es`` (:class:`str`) [const char \*encoding, char \*\*buffer] @@ -241,9 +237,11 @@ the Python object to the required type. For signed integer formats, :exc:`OverflowError` is raised if the value is out of range for the C type. -For unsigned integer formats, no range checking is done --- the +For unsigned integer formats, the most significant bits are silently truncated when the receiving field is too -small to receive the value. +small to receive the value, and :exc:`DeprecationWarning` is emitted when +the value is larger than the maximal value for the C type or less than +the minimal value for the corresponding signed integer type of the same size. ``b`` (:class:`int`) [unsigned char] Convert a nonnegative Python integer to an unsigned tiny integer, stored in a C @@ -252,27 +250,25 @@ small to receive the value. ``B`` (:class:`int`) [unsigned char] Convert a Python integer to a tiny integer without overflow checking, stored in a C :c:expr:`unsigned char`. + Convert a Python integer to a C :c:expr:`unsigned char`. ``h`` (:class:`int`) [short int] Convert a Python integer to a C :c:expr:`short int`. ``H`` (:class:`int`) [unsigned short int] - Convert a Python integer to a C :c:expr:`unsigned short int`, without overflow - checking. + Convert a Python integer to a C :c:expr:`unsigned short int`. ``i`` (:class:`int`) [int] Convert a Python integer to a plain C :c:expr:`int`. ``I`` (:class:`int`) [unsigned int] - Convert a Python integer to a C :c:expr:`unsigned int`, without overflow - checking. + Convert a Python integer to a C :c:expr:`unsigned int`. ``l`` (:class:`int`) [long int] Convert a Python integer to a C :c:expr:`long int`. ``k`` (:class:`int`) [unsigned long] - Convert a Python integer to a C :c:expr:`unsigned long` without - overflow checking. + Convert a Python integer to a C :c:expr:`unsigned long`. .. versionchanged:: 3.14 Use :meth:`~object.__index__` if available. @@ -281,8 +277,7 @@ small to receive the value. Convert a Python integer to a C :c:expr:`long long`. ``K`` (:class:`int`) [unsigned long long] - Convert a Python integer to a C :c:expr:`unsigned long long` - without overflow checking. + Convert a Python integer to a C :c:expr:`unsigned long long`. .. versionchanged:: 3.14 Use :meth:`~object.__index__` if available. @@ -310,6 +305,14 @@ small to receive the value. ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. +.. deprecated:: 3.15 + + For unsigned integer formats ``B``, ``H``, ``I``, ``k`` and ``K``, + :exc:`DeprecationWarning` is emitted when the value is larger than + the maximal value for the C type or less than the minimal value for + the corresponding signed integer type of the same size. + + Other objects ------------- @@ -387,17 +390,6 @@ Other objects Non-tuple sequences are deprecated if *items* contains format units which store a borrowed buffer or a borrowed reference. -``unit?`` (anything or ``None``) [*matching-variable(s)*] - ``?`` modifies the behavior of the preceding format unit. - The C variable(s) corresponding to that parameter should be initialized - to their default value --- when the argument is ``None``, - :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding - C variable(s). - If the argument is not ``None``, it is parsed according to the specified - format unit. - - .. versionadded:: 3.14 - A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: @@ -685,6 +677,13 @@ Building values ``p`` (:class:`bool`) [int] Convert a C :c:expr:`int` to a Python :class:`bool` object. + + Be aware that this format requires an ``int`` argument. + Unlike most other contexts in C, variadic arguments are not coerced to + a suitable type automatically. + You can convert another type (for example, a pointer or a float) to a + suitable ``int`` value using ``(x) ? 1 : 0`` or ``!!x``. + .. versionadded:: 3.14 ``c`` (:class:`bytes` of length 1) [char] diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index d3081894ead..6bb72a2312b 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -261,6 +261,10 @@ readonly, format MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` can be used to request a simple writable buffer. + .. c:macro:: PyBUF_WRITEABLE + + This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d47beee68ea..82c25573683 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,6 +47,10 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. + .. deprecated:: 3.15 + ``PyBytes_FromStringAndSize(NULL, len)`` is :term:`soft deprecated`, + use the :c:type:`PyBytesWriter` API instead. + .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -219,3 +223,209 @@ called with a non-bytes parameter. reallocation fails, the original bytes object at *\*bytes* is deallocated, *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. + + .. deprecated:: 3.15 + The function is :term:`soft deprecated`, + use the :c:type:`PyBytesWriter` API instead. + + +.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) + + Get the string representation of *bytes*. This function is currently used to + implement :meth:`!bytes.__repr__` in Python. + + This function does not do type checking; it is undefined behavior to pass + *bytes* as a non-bytes object or ``NULL``. + + If *smartquotes* is true, the representation will use a double-quoted string + instead of single-quoted string when single-quotes are present in *bytes*. + For example, the byte string ``'Python'`` would be represented as + ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is + false. + + On success, this function returns a :term:`strong reference` to a + :class:`str` object containing the representation. On failure, this + returns ``NULL`` with an exception set. + + +.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding) + + Unescape a backslash-escaped string *s*. *s* must not be ``NULL``. + *len* must be the size of *s*. + + *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If + *errors* is ``NULL``, then ``"strict"`` is used by default. + + On success, this function returns a :term:`strong reference` to a Python + :class:`bytes` object containing the unescaped string. On failure, this + function returns ``NULL`` with an exception set. + + .. versionchanged:: 3.9 + *unicode* and *recode_encoding* are now unused. + + +.. _pybyteswriter: + +PyBytesWriter +------------- + +The :c:type:`PyBytesWriter` API can be used to create a Python :class:`bytes` +object. + +.. versionadded:: 3.15 + +.. c:type:: PyBytesWriter + + A bytes writer instance. + + The API is **not thread safe**: a writer should only be used by a single + thread at the same time. + + The instance must be destroyed by :c:func:`PyBytesWriter_Finish` on + success, or :c:func:`PyBytesWriter_Discard` on error. + + +Create, Finish, Discard +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyBytesWriter* PyBytesWriter_Create(Py_ssize_t size) + + Create a :c:type:`PyBytesWriter` to write *size* bytes. + + If *size* is greater than zero, allocate *size* bytes, and set the + writer size to *size*. The caller is responsible to write *size* + bytes using :c:func:`PyBytesWriter_GetData`. + This function does not overallocate. + + On error, set an exception and return ``NULL``. + + *size* must be positive or zero. + +.. c:function:: PyObject* PyBytesWriter_Finish(PyBytesWriter *writer) + + Finish a :c:type:`PyBytesWriter` created by + :c:func:`PyBytesWriter_Create`. + + On success, return a Python :class:`bytes` object. + On error, set an exception and return ``NULL``. + + The writer instance is invalid after the call in any case. + No API can be called on the writer after :c:func:`PyBytesWriter_Finish`. + +.. c:function:: PyObject* PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + to *size* bytes before creating the :class:`bytes` object. + +.. c:function:: PyObject* PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + using *buf* pointer before creating the :class:`bytes` object. + + Set an exception and return ``NULL`` if *buf* pointer is outside the + internal buffer bounds. + + Function pseudo-code:: + + Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer); + return PyBytesWriter_FinishWithSize(writer, size); + +.. c:function:: void PyBytesWriter_Discard(PyBytesWriter *writer) + + Discard a :c:type:`PyBytesWriter` created by :c:func:`PyBytesWriter_Create`. + + Do nothing if *writer* is ``NULL``. + + The writer instance is invalid after the call. + No API can be called on the writer after :c:func:`PyBytesWriter_Discard`. + +High-level API +^^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_WriteBytes(PyBytesWriter *writer, const void *bytes, Py_ssize_t size) + + Grow the *writer* internal buffer by *size* bytes, + write *size* bytes of *bytes* at the *writer* end, + and add *size* to the *writer* size. + + If *size* is equal to ``-1``, call ``strlen(bytes)`` to get the + string length. + + On success, return ``0``. + On error, set an exception and return ``-1``. + +.. c:function:: int PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) + + Similar to :c:func:`PyBytes_FromFormat`, but write the output directly at + the writer end. Grow the writer internal buffer on demand. Then add the + written size to the writer size. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + +Getters +^^^^^^^ + +.. c:function:: Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer) + + Get the writer size. + +.. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) + + Get the writer data: start of the internal buffer. + + The pointer is valid until :c:func:`PyBytesWriter_Finish` or + :c:func:`PyBytesWriter_Discard` is called on *writer*. + + +Low-level API +^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) + + Resize the writer to *size* bytes. It can be used to enlarge or to + shrink the writer. + This function typically overallocates to achieve amortized performance when + resizing multiple times. + + Newly allocated bytes are left uninitialized. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* must be positive or zero. + +.. c:function:: int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) + + Resize the writer by adding *grow* bytes to the current writer size. + This function typically overallocates to achieve amortized performance when + resizing multiple times. + + Newly allocated bytes are left uninitialized. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* can be negative to shrink the writer. + +.. c:function:: void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf) + + Similar to :c:func:`PyBytesWriter_Grow`, but update also the *buf* + pointer. + + The *buf* pointer is moved if the internal buffer is moved in memory. + The *buf* relative position within the internal buffer is left + unchanged. + + On error, set an exception and return ``NULL``. + + *buf* must not be ``NULL``. + + Function pseudo-code:: + + Py_ssize_t pos = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (PyBytesWriter_Grow(writer, size) < 0) { + return NULL; + } + return (char*)PyBytesWriter_GetData(writer) + pos; diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index cdb8aa33e9f..03a848d68ed 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -15,13 +15,19 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:type:: PyCapsule This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:expr:`void*` + extension modules which need to pass an opaque value (as a :c:expr:`void*` pointer) through Python code to other C code. It is often used to make a C function pointer defined in one module available to other modules, so the regular import mechanism can be used to access C APIs defined in dynamically loaded modules. +.. c:var:: PyTypeObject PyCapsule_Type + + The type object corresponding to capsule objects. This is the same object + as :class:`types.CapsuleType` in the Python layer. + + .. c:type:: PyCapsule_Destructor The type of a destructor callback for a capsule. Defined as:: @@ -105,9 +111,19 @@ Refer to :ref:`using-capsules` for more information on using these objects. ``module.attribute``. The *name* stored in the capsule must match this string exactly. + This function splits *name* on the ``.`` character, and imports the first + element. It then processes further elements using attribute lookups. + Return the capsule's internal *pointer* on success. On failure, set an exception and return ``NULL``. + .. note:: + + If *name* points to an attribute of some submodule or subpackage, this + submodule or subpackage must be previously imported using other means + (for example, by using :c:func:`PyImport_ImportModule`) for the + attribute lookups to succeed. + .. versionchanged:: 3.3 *no_block* has no effect anymore. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 61eb994c370..2501ed9580d 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -7,7 +7,7 @@ Cell Objects "Cell" objects are used to implement variables referenced by multiple scopes. For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to +variables of each stack frame that references the value contain a reference to the cells from outer scopes which also use that variable. When the value is accessed, the value contained in the cell is used instead of the cell object itself. This de-referencing of the cell object requires support from the diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 6eae24b38fa..45f5e83adc4 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -182,7 +182,7 @@ bound into a function. Type of a code object watcher callback function. If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked - after `co` has been fully initialized. Otherwise, the callback is invoked + after *co* has been fully initialized. Otherwise, the callback is invoked before the destruction of *co* takes place, so the prior state of *co* can be inspected. @@ -211,6 +211,82 @@ bound into a function. .. versionadded:: 3.12 +.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) + + This is a :term:`soft deprecated` function that does nothing. + + Prior to Python 3.10, this function would perform basic optimizations to a + code object. + + .. versionchanged:: 3.10 + This function now does nothing. + + +.. _c_codeobject_flags: + +Code Object Flags +----------------- + +Code objects contain a bit-field of flags, which can be retrieved as the +:attr:`~codeobject.co_flags` Python attribute (for example using +:c:func:`PyObject_GetAttrString`), and set using a *flags* argument to +:c:func:`PyUnstable_Code_New` and similar functions. + +Flags whose names start with ``CO_FUTURE_`` correspond to features normally +selectable by :ref:`future statements `. These flags can be used in +:c:member:`PyCompilerFlags.cf_flags`. +Note that many ``CO_FUTURE_`` flags are mandatory in current versions of +Python, and setting them has no effect. + +The following flags are available. +For their meaning, see the linked documentation of their Python equivalents. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Flag + * Meaning + * * .. c:macro:: CO_OPTIMIZED + * :py:data:`inspect.CO_OPTIMIZED` + * * .. c:macro:: CO_NEWLOCALS + * :py:data:`inspect.CO_NEWLOCALS` + * * .. c:macro:: CO_VARARGS + * :py:data:`inspect.CO_VARARGS` + * * .. c:macro:: CO_VARKEYWORDS + * :py:data:`inspect.CO_VARKEYWORDS` + * * .. c:macro:: CO_NESTED + * :py:data:`inspect.CO_NESTED` + * * .. c:macro:: CO_GENERATOR + * :py:data:`inspect.CO_GENERATOR` + * * .. c:macro:: CO_COROUTINE + * :py:data:`inspect.CO_COROUTINE` + * * .. c:macro:: CO_ITERABLE_COROUTINE + * :py:data:`inspect.CO_ITERABLE_COROUTINE` + * * .. c:macro:: CO_ASYNC_GENERATOR + * :py:data:`inspect.CO_ASYNC_GENERATOR` + * * .. c:macro:: CO_HAS_DOCSTRING + * :py:data:`inspect.CO_HAS_DOCSTRING` + * * .. c:macro:: CO_METHOD + * :py:data:`inspect.CO_METHOD` + + * * .. c:macro:: CO_FUTURE_DIVISION + * no effect (:py:data:`__future__.division`) + * * .. c:macro:: CO_FUTURE_ABSOLUTE_IMPORT + * no effect (:py:data:`__future__.absolute_import`) + * * .. c:macro:: CO_FUTURE_WITH_STATEMENT + * no effect (:py:data:`__future__.with_statement`) + * * .. c:macro:: CO_FUTURE_PRINT_FUNCTION + * no effect (:py:data:`__future__.print_function`) + * * .. c:macro:: CO_FUTURE_UNICODE_LITERALS + * no effect (:py:data:`__future__.unicode_literals`) + * * .. c:macro:: CO_FUTURE_GENERATOR_STOP + * no effect (:py:data:`__future__.generator_stop`) + * * .. c:macro:: CO_FUTURE_ANNOTATIONS + * :py:data:`__future__.annotations` + + Extra information ----------------- @@ -224,7 +300,7 @@ may change without deprecation warnings. .. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) - Return a new an opaque index value used to adding data to code objects. + Return a new opaque index value used to adding data to code objects. You generally call this function once (per interpreter) and use the result with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 8ae5c4fecd6..35ee048bd5f 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -7,7 +7,7 @@ Codec registry and support functions Register a new codec search function. - As side effect, this tries to load the :mod:`!encodings` package, if not yet + As a side effect, this tries to load the :mod:`!encodings` package, if not yet done, to make sure that it is always first in the list of search functions. .. c:function:: int PyCodec_Unregister(PyObject *search_function) @@ -39,7 +39,7 @@ Codec registry and support functions *object* is passed through the decoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may be ``NULL`` to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. + :exc:`LookupError` if no decoder can be found. Codec lookup API @@ -129,3 +129,13 @@ Registry API for Unicode encoding error handlers Replace the unicode encode error with ``\N{...}`` escapes. .. versionadded:: 3.5 + + +Codec utility variables +----------------------- + +.. c:var:: const char *Py_hexdigits + + A string constant containing the lowercase hexadecimal digits: ``"0123456789abcdef"``. + + .. versionadded:: 3.3 diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 16bd79475dc..629312bd771 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -3,93 +3,25 @@ .. _complexobjects: Complex Number Objects ----------------------- +====================== .. index:: pair: object; complex number -Python's complex number objects are implemented as two distinct types when -viewed from the C API: one is the Python object exposed to Python programs, and -the other is a C structure which represents the actual complex number value. -The API provides functions for working with both. - - -Complex Numbers as C Structures -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that the functions which accept these structures as parameters and return -them as results do so *by value* rather than dereferencing them through -pointers. This is consistent throughout the API. - - -.. c:type:: Py_complex - - The C structure which corresponds to the value portion of a Python complex - number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. - - .. c:member:: double real - double imag - - The structure is defined as:: - - typedef struct { - double real; - double imag; - } Py_complex; - - -.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) - - Return the sum of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) - - Return the difference between two complex numbers, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_neg(Py_complex num) - - Return the negation of the complex number *num*, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) - - Return the product of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) - - Return the quotient of two complex numbers, using the C :c:type:`Py_complex` - representation. - - If *divisor* is null, this method returns zero and sets - :c:data:`errno` to :c:macro:`!EDOM`. - - -.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) - - Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` - representation. - - If *num* is null and *exp* is not a positive real number, - this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. - - Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. - - -Complex Numbers as Python Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. c:type:: PyComplexObject This subtype of :c:type:`PyObject` represents a Python complex number object. + .. c:member:: Py_complex cval + + The complex number value, using the C :c:type:`Py_complex` representation. + + .. deprecated-removed:: 3.15 3.20 + Use :c:func:`PyComplex_AsCComplex` and + :c:func:`PyComplex_FromCComplex` to convert a + Python complex number to/from the C :c:type:`Py_complex` + representation. + .. c:var:: PyTypeObject PyComplex_Type @@ -109,12 +41,6 @@ Complex Numbers as Python Objects :c:type:`PyComplexObject`. This function always succeeds. -.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) - - Create a new Python complex number object from a C :c:type:`Py_complex` value. - Return ``NULL`` with an exception set on error. - - .. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) Return a new :c:type:`PyComplexObject` object from *real* and *imag*. @@ -153,6 +79,29 @@ Complex Numbers as Python Objects .. versionchanged:: 3.13 Use :meth:`~object.__complex__` if available. + +.. c:type:: Py_complex + + This C structure defines an export format for a Python complex + number object. + + .. c:member:: double real + double imag + + The structure is defined as:: + + typedef struct { + double real; + double imag; + } Py_complex; + + +.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) + + Create a new Python complex number object from a C :c:type:`Py_complex` value. + Return ``NULL`` with an exception set on error. + + .. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) Return the :c:type:`Py_complex` value of the complex number *op*. @@ -169,3 +118,82 @@ Complex Numbers as Python Objects .. versionchanged:: 3.8 Use :meth:`~object.__index__` if available. + + +Complex Numbers as C Structures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The API also provides functions for working with complex numbers, using the +:c:type:`Py_complex` representation. Note that the functions which accept +these structures as parameters and return them as results do so *by value* +rather than dereferencing them through pointers. + +Please note, that these functions are :term:`soft deprecated` since Python +3.15. Avoid using this API in a new code to do complex arithmetic: either use +the `Number Protocol `_ API or use native complex types, like +:c:expr:`double complex`. + + +.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) + + Return the sum of two complex numbers, using the C :c:type:`Py_complex` + representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) + + Return the difference between two complex numbers, using the C + :c:type:`Py_complex` representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_neg(Py_complex num) + + Return the negation of the complex number *num*, using the C + :c:type:`Py_complex` representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) + + Return the product of two complex numbers, using the C :c:type:`Py_complex` + representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) + + Return the quotient of two complex numbers, using the C :c:type:`Py_complex` + representation. + + If *divisor* is null, this method returns zero and sets + :c:data:`errno` to :c:macro:`!EDOM`. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) + + Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` + representation. + + If *num* is null and *exp* is not a positive real number, + this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 + + +.. c:function:: double _Py_c_abs(Py_complex num) + + Return the absolute value of the complex number *num*. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 880f7b15ce6..1746fe95eaa 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,11 +109,20 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst frame.rst gen.rst coro.rst contextvars.rst - datetime.rst typehints.rst + + +C API for extension modules +=========================== + +.. toctree:: + + curses.rst + datetime.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c92ef4c653a..96078d22710 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -41,7 +41,7 @@ The return value (*rv*) for these functions should be interpreted as follows: ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` in this case. -* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in +* When ``rv < 0``, the output conversion failed and ``str[size-1]`` is ``'\0'`` in this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with + if ``overflow_exception`` is ``NULL`` return :c:macro:`!INFINITY` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set @@ -128,18 +128,28 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values ``Py_DTSF_SIGN``, - ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: + *flags* can be zero or more of the following values or-ed together: - * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign - character, even if *val* is non-negative. + .. c:macro:: Py_DTSF_SIGN - * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look - like an integer. + Always precede the returned string with a sign + character, even if *val* is non-negative. - * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. + .. c:macro:: Py_DTSF_ADD_DOT_0 + + Ensure that the returned string will not look like an integer. + + .. c:macro:: Py_DTSF_ALT + + Apply "alternate" formatting rules. + See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for + details. + + .. c:macro:: Py_DTSF_NO_NEG_0 + + Negative zero is converted to positive zero. + + .. versionadded:: 3.11 If *ptype* is non-``NULL``, then the value it points to will be set to one of ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that @@ -152,13 +162,85 @@ The following functions provide locale-independent string to number conversions. .. versionadded:: 3.1 -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) +.. c:function:: int PyOS_mystricmp(const char *str1, const char *str2) + int PyOS_mystrnicmp(const char *str1, const char *str2, Py_ssize_t size) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strcmp` except that it ignores the case. + Case insensitive comparison of strings. These functions work almost + identically to :c:func:`!strcmp` and :c:func:`!strncmp` (respectively), + except that they ignore the case of ASCII characters. + + Return ``0`` if the strings are equal, a negative value if *str1* sorts + lexicographically before *str2*, or a positive value if it sorts after. + + In the *str1* or *str2* arguments, a NUL byte marks the end of the string. + For :c:func:`!PyOS_mystrnicmp`, the *size* argument gives the maximum size + of the string, as if NUL was present at the index given by *size*. + + These functions do not use the locale. -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) +.. c:function:: int PyOS_stricmp(const char *str1, const char *str2) + int PyOS_strnicmp(const char *str1, const char *str2, Py_ssize_t size) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strncmp` except that it ignores the case. + Case insensitive comparison of strings. + + On Windows, these are aliases of :c:func:`!stricmp` and :c:func:`!strnicmp`, + respectively. + + On other platforms, they are aliases of :c:func:`PyOS_mystricmp` and + :c:func:`PyOS_mystrnicmp`, respectively. + + +Character classification and conversion +======================================= + +The following macros provide locale-independent (unlike the C standard library +``ctype.h``) character classification and conversion. +The argument must be a signed or unsigned :c:expr:`char`. + + +.. c:macro:: Py_ISALNUM(c) + + Return true if the character *c* is an alphanumeric character. + + +.. c:macro:: Py_ISALPHA(c) + + Return true if the character *c* is an alphabetic character (``a-z`` and ``A-Z``). + + +.. c:macro:: Py_ISDIGIT(c) + + Return true if the character *c* is a decimal digit (``0-9``). + + +.. c:macro:: Py_ISLOWER(c) + + Return true if the character *c* is a lowercase ASCII letter (``a-z``). + + +.. c:macro:: Py_ISUPPER(c) + + Return true if the character *c* is an uppercase ASCII letter (``A-Z``). + + +.. c:macro:: Py_ISSPACE(c) + + Return true if the character *c* is a whitespace character (space, tab, + carriage return, newline, vertical tab, or form feed). + + +.. c:macro:: Py_ISXDIGIT(c) + + Return true if the character *c* is a hexadecimal digit (``0-9``, ``a-f``, and + ``A-F``). + + +.. c:macro:: Py_TOLOWER(c) + + Return the lowercase equivalent of the character *c*. + + +.. c:macro:: Py_TOUPPER(c) + + Return the uppercase equivalent of the character *c*. diff --git a/Doc/c-api/curses.rst b/Doc/c-api/curses.rst new file mode 100644 index 00000000000..5a1697c43cc --- /dev/null +++ b/Doc/c-api/curses.rst @@ -0,0 +1,138 @@ +.. highlight:: c + +Curses C API +------------ + +:mod:`curses` exposes a small C interface for extension modules. +Consumers must include the header file :file:`py_curses.h` (which is not +included by default by :file:`Python.h`) and :c:func:`import_curses` must +be invoked, usually as part of the module initialisation function, to populate +:c:var:`PyCurses_API`. + +.. warning:: + + Neither the C API nor the pure Python :mod:`curses` module are compatible + with subinterpreters. + +.. c:macro:: import_curses() + + Import the curses C API. The macro does not need a semi-colon to be called. + + On success, populate the :c:var:`PyCurses_API` pointer. + + On failure, set :c:var:`PyCurses_API` to NULL and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + import_curses(); // semi-colon is optional but recommended + if (PyErr_Occurred()) { /* cleanup */ } + + +.. c:var:: void **PyCurses_API + + Dynamically allocated object containing the curses C API. + This variable is only available once :c:macro:`import_curses` succeeds. + + ``PyCurses_API[0]`` corresponds to :c:data:`PyCursesWindow_Type`. + + ``PyCurses_API[1]``, ``PyCurses_API[2]``, and ``PyCurses_API[3]`` + are pointers to predicate functions of type ``int (*)(void)``. + + When called, these predicates return whether :func:`curses.setupterm`, + :func:`curses.initscr`, and :func:`curses.start_color` have been called + respectively. + + See also the convenience macros :c:macro:`PyCursesSetupTermCalled`, + :c:macro:`PyCursesInitialised`, and :c:macro:`PyCursesInitialisedColor`. + + .. note:: + + The number of entries in this structure is subject to changes. + Consider using :c:macro:`PyCurses_API_pointers` to check if + new fields are available or not. + + +.. c:macro:: PyCurses_API_pointers + + The number of accessible fields (``4``) in :c:var:`PyCurses_API`. + This number is incremented whenever new fields are added. + + +.. c:var:: PyTypeObject PyCursesWindow_Type + + The :ref:`heap type ` corresponding to :class:`curses.window`. + + +.. c:function:: int PyCursesWindow_Check(PyObject *op) + + Return true if *op* is a :class:`curses.window` instance, false otherwise. + + +The following macros are convenience macros expanding into C statements. +In particular, they can only be used as ``macro;`` or ``macro``, but not +``macro()`` or ``macro();``. + +.. c:macro:: PyCursesSetupTermCalled + + Macro checking if :func:`curses.setupterm` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_setupterm_called = (predicate_t)PyCurses_API[1]; + if (!was_setupterm_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialised + + Macro checking if :func:`curses.initscr` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_initscr_called = (predicate_t)PyCurses_API[2]; + if (!was_initscr_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialisedColor + + Macro checking if :func:`curses.start_color` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_start_color_called = (predicate_t)PyCurses_API[3]; + if (!was_start_color_called()) { + return NULL; + } + } + + +Internal data +------------- + +The following objects are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyCurses_CAPSULE_NAME + + Name of the curses capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`import_curses` instead. + diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index d2d4d5309c7..127d7c9c91a 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,42 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following macros. +.. c:macro:: PyDateTime_IMPORT() + + Import the datetime C API. + + On success, populate the :c:var:`PyDateTimeAPI` pointer. + On failure, set :c:var:`PyDateTimeAPI` to ``NULL`` and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + PyDateTime_IMPORT; + if (PyErr_Occurred()) { /* cleanup */ } + + .. warning:: + + This is not compatible with subinterpreters. + +.. c:type:: PyDateTime_CAPI + + Structure containing the fields for the datetime C API. + + The fields of this structure are private and subject to change. + + Do not use this directly; prefer ``PyDateTime_*`` APIs instead. + +.. c:var:: PyDateTime_CAPI *PyDateTimeAPI + + Dynamically allocated object containing the datetime C API. + + This variable is only available once :c:macro:`PyDateTime_IMPORT` succeeds. + .. c:type:: PyDateTime_Date This subtype of :c:type:`PyObject` represents a Python date object. @@ -46,7 +77,7 @@ macros. .. c:var:: PyTypeObject PyDateTime_DeltaType - This instance of :c:type:`PyTypeObject` represents Python type for + This instance of :c:type:`PyTypeObject` represents the Python type for the difference between two datetime values; it is the same object as :class:`datetime.timedelta` in the Python layer. @@ -325,3 +356,16 @@ Macros for the convenience of modules implementing the DB API: Create and return a new :class:`datetime.date` object given an argument tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. + + +Internal data +------------- + +The following symbols are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyDateTime_CAPSULE_NAME + + Name of the datetime capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`PyDateTime_IMPORT` instead. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b32c113e5f0..313c534545a 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -21,20 +21,104 @@ found in the dictionary of type objects. .. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) +.. c:var:: PyTypeObject PyMemberDescr_Type + + The type object for member descriptor objects created from + :c:type:`PyMemberDef` structures. These descriptors expose fields of a + C struct as attributes on a type, and correspond + to :class:`types.MemberDescriptorType` objects in Python. + + + +.. c:var:: PyTypeObject PyGetSetDescr_Type + + The type object for get/set descriptor objects created from + :c:type:`PyGetSetDef` structures. These descriptors implement attributes + whose value is computed by C getter and setter functions, and are used + for many built-in type attributes. + + .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) +.. c:var:: PyTypeObject PyMethodDescr_Type + + The type object for method descriptor objects created from + :c:type:`PyMethodDef` structures. These descriptors expose C functions as + methods on a type, and correspond to :class:`types.MemberDescriptorType` + objects in Python. + + .. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) +.. c:var:: PyTypeObject PyWrapperDescr_Type + + The type object for wrapper descriptor objects created by + :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper + descriptors are used internally to expose special methods implemented + via wrapper structures, and appear in Python as + :class:`types.WrapperDescriptorType` objects. + + .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) .. c:function:: int PyDescr_IsData(PyObject *descr) - Return non-zero if the descriptor objects *descr* describes a data attribute, or + Return non-zero if the descriptor object *descr* describes a data attribute, or ``0`` if it describes a method. *descr* must be a descriptor object; there is no error checking. .. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) + + +Built-in descriptors +^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PySuper_Type + + The type object for super objects. This is the same object as + :class:`super` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethod_Type + + The type of class method objects. This is the same object as + :class:`classmethod` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethodDescr_Type + + The type object for C-level class method descriptor objects. + This is the type of the descriptors created for :func:`classmethod` defined in + C extension types, and is the same object as :class:`classmethod` + in Python. + + +.. c:function:: PyObject *PyClassMethod_New(PyObject *callable) + + Create a new :class:`classmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new class + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + + +.. c:var:: PyTypeObject PyStaticMethod_Type + + The type of static method objects. This is the same object as + :class:`staticmethod` in the Python layer. + + +.. c:function:: PyObject *PyStaticMethod_New(PyObject *callable) + + Create a new :class:`staticmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new static + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e55c5c80cb8..9c4428ced41 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -43,6 +43,17 @@ Dictionary Objects prevent modification of the dictionary for non-dynamic class types. +.. c:var:: PyTypeObject PyDictProxy_Type + + The type object for mapping proxy objects created by + :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute + of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a + dynamic, read-only view of an underlying dictionary: changes to the + underlying dictionary are reflected in the proxy, but the proxy itself + does not support mutation operations. This corresponds to + :class:`types.MappingProxyType` in Python. + + .. c:function:: void PyDict_Clear(PyObject *p) Empty an existing dictionary of all key-value pairs. @@ -50,7 +61,7 @@ Dictionary Objects .. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) - Determine if dictionary *p* contains *key*. If an item in *p* is matches + Determine if dictionary *p* contains *key*. If an item in *p* matches *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. This is equivalent to the Python expression ``key in p``. @@ -198,7 +209,7 @@ Dictionary Objects .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) Remove *key* from dictionary *p* and optionally return the removed value. - Do not raise :exc:`KeyError` if the key missing. + Do not raise :exc:`KeyError` if the key is missing. - If the key is present, set *\*result* to a new reference to the removed value if *result* is not ``NULL``, and return ``1``. @@ -207,7 +218,7 @@ Dictionary Objects - On error, raise an exception and return ``-1``. Similar to :meth:`dict.pop`, but without the default value and - not raising :exc:`KeyError` if the key missing. + not raising :exc:`KeyError` if the key is missing. .. versionadded:: 3.13 @@ -245,6 +256,11 @@ Dictionary Objects ``len(p)`` on a dictionary. +.. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p) + + Similar to :c:func:`PyDict_Size`, but without error checking. + + .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) Iterate over all key-value pairs in the dictionary *p*. The @@ -301,6 +317,15 @@ Dictionary Objects } Py_END_CRITICAL_SECTION(); + .. note:: + + On the free-threaded build, this function can be used safely inside a + critical section. However, the references returned for *pkey* and *pvalue* + are :term:`borrowed ` and are only valid while the + critical section is held. If you need to use these objects outside the + critical section or when the critical section can be suspended, create a + :term:`strong reference ` (for example, using + :c:func:`Py_NewRef`). .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) @@ -417,3 +442,138 @@ Dictionary Objects it before returning. .. versionadded:: 3.12 + + +Dictionary View Objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDictViewSet_Check(PyObject *op) + + Return true if *op* is a view of a set inside a dictionary. This is currently + equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This + function always succeeds. + + +.. c:var:: PyTypeObject PyDictKeys_Type + + Type object for a view of dictionary keys. In Python, this is the type of + the object returned by :meth:`dict.keys`. + + +.. c:function:: int PyDictKeys_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary keys view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictValues_Type + + Type object for a view of dictionary values. In Python, this is the type of + the object returned by :meth:`dict.values`. + + +.. c:function:: int PyDictValues_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary values view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictItems_Type + + Type object for a view of dictionary items. In Python, this is the type of + the object returned by :meth:`dict.items`. + + +.. c:function:: int PyDictItems_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary items view. This function + always succeeds. + + +Ordered Dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c8e1b5c2461..d7fe9e2c9ec 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -309,6 +309,14 @@ For convenience, some of these functions will always return a .. versionadded:: 3.4 +.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) + + Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the + *end_lineno* and *end_col_offset* information for the current exception. + + .. versionadded:: 3.10 + + .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string @@ -331,6 +339,23 @@ For convenience, some of these functions will always return a use. +.. c:function:: PyObject *PyErr_ProgramTextObject(PyObject *filename, int lineno) + + Get the source line in *filename* at line *lineno*. *filename* should be a + Python :class:`str` object. + + On success, this function returns a Python string object with the found line. + On failure, this function returns ``NULL`` without an exception set. + + +.. c:function:: PyObject *PyErr_ProgramText(const char *filename, int lineno) + + Similar to :c:func:`PyErr_ProgramTextObject`, but *filename* is a + :c:expr:`const char *`, which is decoded with the + :term:`filesystem encoding and error handler`, instead of a + Python object reference. + + Issuing warnings ================ @@ -394,6 +419,15 @@ an error value). .. versionadded:: 3.2 +.. c:function:: int PyErr_WarnExplicitFormat(PyObject *category, const char *filename, int lineno, const char *module, PyObject *registry, const char *format, ...) + + Similar to :c:func:`PyErr_WarnExplicit`, but uses + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is @@ -749,9 +783,30 @@ Exception Classes .. versionadded:: 3.2 +.. c:function:: int PyExceptionClass_Check(PyObject *ob) + + Return non-zero if *ob* is an exception class, zero otherwise. This function always succeeds. + + +.. c:function:: const char *PyExceptionClass_Name(PyObject *ob) + + Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*. + + Exception Objects ================= +.. c:function:: int PyExceptionInstance_Check(PyObject *op) + + Return true if *op* is an instance of :class:`BaseException`, false + otherwise. This function always succeeds. + + +.. c:macro:: PyExceptionInstance_Class(op) + + Equivalent to :c:func:`Py_TYPE(op) `. + + .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as @@ -929,6 +984,9 @@ because the :ref:`call protocol ` takes care of recursion handling. be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. + .. seealso:: + The :c:func:`PyUnstable_ThreadState_SetStackProtection` function. + .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. @@ -969,184 +1027,159 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. Ends a :c:func:`Py_ReprEnter`. Must be called once for each invocation of :c:func:`Py_ReprEnter` that returns zero. +.. c:function:: int Py_GetRecursionLimit(void) + + Get the recursion limit for the current interpreter. It can be set with + :c:func:`Py_SetRecursionLimit`. The recursion limit prevents the + Python interpreter stack from growing infinitely. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.getrecursionlimit` + +.. c:function:: void Py_SetRecursionLimit(int new_limit) + + Set the recursion limit for the current interpreter. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.setrecursionlimit` .. _standardexceptions: -Standard Exceptions -=================== +Exception and warning types +=========================== -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: +All standard Python exceptions and warning categories are available as global +variables whose names are ``PyExc_`` followed by the Python exception name. +These have the type :c:expr:`PyObject*`; they are all class objects. -.. index:: - single: PyExc_BaseException (C var) - single: PyExc_Exception (C var) - single: PyExc_ArithmeticError (C var) - single: PyExc_AssertionError (C var) - single: PyExc_AttributeError (C var) - single: PyExc_BlockingIOError (C var) - single: PyExc_BrokenPipeError (C var) - single: PyExc_BufferError (C var) - single: PyExc_ChildProcessError (C var) - single: PyExc_ConnectionAbortedError (C var) - single: PyExc_ConnectionError (C var) - single: PyExc_ConnectionRefusedError (C var) - single: PyExc_ConnectionResetError (C var) - single: PyExc_EOFError (C var) - single: PyExc_FileExistsError (C var) - single: PyExc_FileNotFoundError (C var) - single: PyExc_FloatingPointError (C var) - single: PyExc_GeneratorExit (C var) - single: PyExc_ImportError (C var) - single: PyExc_IndentationError (C var) - single: PyExc_IndexError (C var) - single: PyExc_InterruptedError (C var) - single: PyExc_IsADirectoryError (C var) - single: PyExc_KeyError (C var) - single: PyExc_KeyboardInterrupt (C var) - single: PyExc_LookupError (C var) - single: PyExc_MemoryError (C var) - single: PyExc_ModuleNotFoundError (C var) - single: PyExc_NameError (C var) - single: PyExc_NotADirectoryError (C var) - single: PyExc_NotImplementedError (C var) - single: PyExc_OSError (C var) - single: PyExc_OverflowError (C var) - single: PyExc_PermissionError (C var) - single: PyExc_ProcessLookupError (C var) - single: PyExc_PythonFinalizationError (C var) - single: PyExc_RecursionError (C var) - single: PyExc_ReferenceError (C var) - single: PyExc_RuntimeError (C var) - single: PyExc_StopAsyncIteration (C var) - single: PyExc_StopIteration (C var) - single: PyExc_SyntaxError (C var) - single: PyExc_SystemError (C var) - single: PyExc_SystemExit (C var) - single: PyExc_TabError (C var) - single: PyExc_TimeoutError (C var) - single: PyExc_TypeError (C var) - single: PyExc_UnboundLocalError (C var) - single: PyExc_UnicodeDecodeError (C var) - single: PyExc_UnicodeEncodeError (C var) - single: PyExc_UnicodeError (C var) - single: PyExc_UnicodeTranslateError (C var) - single: PyExc_ValueError (C var) - single: PyExc_ZeroDivisionError (C var) +For completeness, here are all the variables: -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ +Exception types +--------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_BaseException + * :exc:`BaseException` + * * .. c:var:: PyObject *PyExc_BaseExceptionGroup + * :exc:`BaseExceptionGroup` + * * .. c:var:: PyObject *PyExc_Exception + * :exc:`Exception` + * * .. c:var:: PyObject *PyExc_ArithmeticError + * :exc:`ArithmeticError` + * * .. c:var:: PyObject *PyExc_AssertionError + * :exc:`AssertionError` + * * .. c:var:: PyObject *PyExc_AttributeError + * :exc:`AttributeError` + * * .. c:var:: PyObject *PyExc_BlockingIOError + * :exc:`BlockingIOError` + * * .. c:var:: PyObject *PyExc_BrokenPipeError + * :exc:`BrokenPipeError` + * * .. c:var:: PyObject *PyExc_BufferError + * :exc:`BufferError` + * * .. c:var:: PyObject *PyExc_ChildProcessError + * :exc:`ChildProcessError` + * * .. c:var:: PyObject *PyExc_ConnectionAbortedError + * :exc:`ConnectionAbortedError` + * * .. c:var:: PyObject *PyExc_ConnectionError + * :exc:`ConnectionError` + * * .. c:var:: PyObject *PyExc_ConnectionRefusedError + * :exc:`ConnectionRefusedError` + * * .. c:var:: PyObject *PyExc_ConnectionResetError + * :exc:`ConnectionResetError` + * * .. c:var:: PyObject *PyExc_EOFError + * :exc:`EOFError` + * * .. c:var:: PyObject *PyExc_FileExistsError + * :exc:`FileExistsError` + * * .. c:var:: PyObject *PyExc_FileNotFoundError + * :exc:`FileNotFoundError` + * * .. c:var:: PyObject *PyExc_FloatingPointError + * :exc:`FloatingPointError` + * * .. c:var:: PyObject *PyExc_GeneratorExit + * :exc:`GeneratorExit` + * * .. c:var:: PyObject *PyExc_ImportError + * :exc:`ImportError` + * * .. c:var:: PyObject *PyExc_IndentationError + * :exc:`IndentationError` + * * .. c:var:: PyObject *PyExc_IndexError + * :exc:`IndexError` + * * .. c:var:: PyObject *PyExc_InterruptedError + * :exc:`InterruptedError` + * * .. c:var:: PyObject *PyExc_IsADirectoryError + * :exc:`IsADirectoryError` + * * .. c:var:: PyObject *PyExc_KeyError + * :exc:`KeyError` + * * .. c:var:: PyObject *PyExc_KeyboardInterrupt + * :exc:`KeyboardInterrupt` + * * .. c:var:: PyObject *PyExc_LookupError + * :exc:`LookupError` + * * .. c:var:: PyObject *PyExc_MemoryError + * :exc:`MemoryError` + * * .. c:var:: PyObject *PyExc_ModuleNotFoundError + * :exc:`ModuleNotFoundError` + * * .. c:var:: PyObject *PyExc_NameError + * :exc:`NameError` + * * .. c:var:: PyObject *PyExc_NotADirectoryError + * :exc:`NotADirectoryError` + * * .. c:var:: PyObject *PyExc_NotImplementedError + * :exc:`NotImplementedError` + * * .. c:var:: PyObject *PyExc_OSError + * :exc:`OSError` + * * .. c:var:: PyObject *PyExc_OverflowError + * :exc:`OverflowError` + * * .. c:var:: PyObject *PyExc_PermissionError + * :exc:`PermissionError` + * * .. c:var:: PyObject *PyExc_ProcessLookupError + * :exc:`ProcessLookupError` + * * .. c:var:: PyObject *PyExc_PythonFinalizationError + * :exc:`PythonFinalizationError` + * * .. c:var:: PyObject *PyExc_RecursionError + * :exc:`RecursionError` + * * .. c:var:: PyObject *PyExc_ReferenceError + * :exc:`ReferenceError` + * * .. c:var:: PyObject *PyExc_RuntimeError + * :exc:`RuntimeError` + * * .. c:var:: PyObject *PyExc_StopAsyncIteration + * :exc:`StopAsyncIteration` + * * .. c:var:: PyObject *PyExc_StopIteration + * :exc:`StopIteration` + * * .. c:var:: PyObject *PyExc_SyntaxError + * :exc:`SyntaxError` + * * .. c:var:: PyObject *PyExc_SystemError + * :exc:`SystemError` + * * .. c:var:: PyObject *PyExc_SystemExit + * :exc:`SystemExit` + * * .. c:var:: PyObject *PyExc_TabError + * :exc:`TabError` + * * .. c:var:: PyObject *PyExc_TimeoutError + * :exc:`TimeoutError` + * * .. c:var:: PyObject *PyExc_TypeError + * :exc:`TypeError` + * * .. c:var:: PyObject *PyExc_UnboundLocalError + * :exc:`UnboundLocalError` + * * .. c:var:: PyObject *PyExc_UnicodeDecodeError + * :exc:`UnicodeDecodeError` + * * .. c:var:: PyObject *PyExc_UnicodeEncodeError + * :exc:`UnicodeEncodeError` + * * .. c:var:: PyObject *PyExc_UnicodeError + * :exc:`UnicodeError` + * * .. c:var:: PyObject *PyExc_UnicodeTranslateError + * :exc:`UnicodeTranslateError` + * * .. c:var:: PyObject *PyExc_ValueError + * :exc:`ValueError` + * * .. c:var:: PyObject *PyExc_ZeroDivisionError + * :exc:`ZeroDivisionError` .. versionadded:: 3.3 :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, @@ -1164,88 +1197,116 @@ the variables: .. versionadded:: 3.6 :c:data:`PyExc_ModuleNotFoundError`. -These are compatibility aliases to :c:data:`PyExc_OSError`: +.. versionadded:: 3.11 + :c:data:`PyExc_BaseExceptionGroup`. -.. index:: - single: PyExc_EnvironmentError (C var) - single: PyExc_IOError (C var) - single: PyExc_WindowsError (C var) -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`!PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_WindowsError` | [2]_ | -+-------------------------------------+----------+ +OSError aliases +--------------- + +The following are a compatibility aliases to :c:data:`PyExc_OSError`. .. versionchanged:: 3.3 These aliases used to be separate exception types. +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * Notes + * * .. c:var:: PyObject *PyExc_EnvironmentError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_IOError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_WindowsError + * :exc:`OSError` + * [win]_ + Notes: -.. [1] - This is a base class for other standard exceptions. +.. [win] + :c:var:`!PyExc_WindowsError` is only defined on Windows; protect code that + uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. -.. [2] - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. .. _standardwarningcategories: -Standard Warning Categories -=========================== +Warning types +------------- -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 -.. index:: - single: PyExc_Warning (C var) - single: PyExc_BytesWarning (C var) - single: PyExc_DeprecationWarning (C var) - single: PyExc_FutureWarning (C var) - single: PyExc_ImportWarning (C var) - single: PyExc_PendingDeprecationWarning (C var) - single: PyExc_ResourceWarning (C var) - single: PyExc_RuntimeWarning (C var) - single: PyExc_SyntaxWarning (C var) - single: PyExc_UnicodeWarning (C var) - single: PyExc_UserWarning (C var) - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | [3]_ | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_Warning + * :exc:`Warning` + * * .. c:var:: PyObject *PyExc_BytesWarning + * :exc:`BytesWarning` + * * .. c:var:: PyObject *PyExc_DeprecationWarning + * :exc:`DeprecationWarning` + * * .. c:var:: PyObject *PyExc_EncodingWarning + * :exc:`EncodingWarning` + * * .. c:var:: PyObject *PyExc_FutureWarning + * :exc:`FutureWarning` + * * .. c:var:: PyObject *PyExc_ImportWarning + * :exc:`ImportWarning` + * * .. c:var:: PyObject *PyExc_PendingDeprecationWarning + * :exc:`PendingDeprecationWarning` + * * .. c:var:: PyObject *PyExc_ResourceWarning + * :exc:`ResourceWarning` + * * .. c:var:: PyObject *PyExc_RuntimeWarning + * :exc:`RuntimeWarning` + * * .. c:var:: PyObject *PyExc_SyntaxWarning + * :exc:`SyntaxWarning` + * * .. c:var:: PyObject *PyExc_UnicodeWarning + * :exc:`UnicodeWarning` + * * .. c:var:: PyObject *PyExc_UserWarning + * :exc:`UserWarning` .. versionadded:: 3.2 :c:data:`PyExc_ResourceWarning`. -Notes: +.. versionadded:: 3.10 + :c:data:`PyExc_EncodingWarning`. -.. [3] - This is a base class for other standard warning categories. + +Tracebacks +========== + +.. c:var:: PyTypeObject PyTraceBack_Type + + Type object for traceback objects. This is available as + :class:`types.TracebackType` in the Python layer. + + +.. c:function:: int PyTraceBack_Check(PyObject *op) + + Return true if *op* is a traceback object, false otherwise. This function + does not account for subtypes. + + +.. c:function:: int PyTraceBack_Here(PyFrameObject *f) + + Replace the :attr:`~BaseException.__traceback__` attribute on the current + exception with a new traceback prepending *f* to the existing chain. + + Calling this function without an exception set is undefined behavior. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. + + +.. c:function:: int PyTraceBack_Print(PyObject *tb, PyObject *f) + + Write the traceback *tb* into the file *f*. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst new file mode 100644 index 00000000000..0ce173b4bfe --- /dev/null +++ b/Doc/c-api/extension-modules.rst @@ -0,0 +1,342 @@ +.. highlight:: c + +.. _extension-modules: + +Defining extension modules +-------------------------- + +A C extension for CPython is a shared library (for example, a ``.so`` file +on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process +(for example, it is compiled with compatible compiler settings), and which +exports an :dfn:`export hook` function (or an +old-style :ref:`initialization function `). + +To be importable by default (that is, by +:py:class:`importlib.machinery.ExtensionFileLoader`), +the shared library must be available on :py:attr:`sys.path`, +and must be named after the module name plus an extension listed in +:py:attr:`importlib.machinery.EXTENSION_SUFFIXES`. + +.. note:: + + Building, packaging and distributing extension modules is best done with + third-party tools, and is out of scope of this document. + One suitable tool is Setuptools, whose documentation can be found at + https://setuptools.pypa.io/en/latest/setuptools.html. + +.. _extension-export-hook: + +Extension export hook +..................... + +.. versionadded:: next + + Support for the :samp:`PyModExport_{}` export hook was added in Python + 3.15. The older way of defining modules is still available: consult either + the :ref:`extension-pyinit` section or earlier versions of this + documentation if you plan to support earlier Python versions. + +The export hook must be an exported function with the following signature: + +.. c:function:: PyModuleDef_Slot *PyModExport_modulename(void) + +For modules with ASCII-only names, the :ref:`export hook ` +must be named :samp:`PyModExport_{}`, +with ```` replaced by the module's name. + +For non-ASCII module names, the export hook must instead be named +:samp:`PyModExportU_{}` (note the ``U``), with ```` encoded using +Python's *punycode* encoding with hyphens replaced by underscores. In Python: + +.. code-block:: python + + def hook_name(name): + try: + suffix = b'_' + name.encode('ascii') + except UnicodeEncodeError: + suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') + return b'PyModExport' + suffix + +The export hook returns an array of :c:type:`PyModuleDef_Slot` entries, +terminated by an entry with a slot ID of ``0``. +These slots describe how the module should be created and initialized. + +This array must remain valid and constant until interpreter shutdown. +Typically, it should use ``static`` storage. +Prefer using the :c:macro:`Py_mod_create` and :c:macro:`Py_mod_exec` slots +for any dynamic behavior. + +The export hook may return ``NULL`` with an exception set to signal failure. + +It is recommended to define the export hook function using a helper macro: + +.. c:macro:: PyMODEXPORT_FUNC + + Declare an extension module export hook. + This macro: + + * specifies the :c:expr:`PyModuleDef_Slot*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. + +For example, a module called ``spam`` would be defined like this:: + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot spam_slots[] = { + {Py_mod_abi, &abi_info}, + {Py_mod_name, "spam"}, + {Py_mod_init, spam_init_function}, + ... + {0, NULL}, + }; + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +The export hook is typically the only non-\ ``static`` +item defined in the module's C source. + +The hook should be kept short -- ideally, one line as above. +If you do need to use Python C API in this function, it is recommended to call +``PyABIInfo_Check(&abi_info, "modulename")`` first to raise an exception, +rather than crash, in common cases of ABI mismatch. + + +.. note:: + + It is possible to export multiple modules from a single shared library by + defining multiple export hooks. + However, importing them requires a custom importer or suitably named + copies/links of the extension file, because Python's import machinery only + finds the function corresponding to the filename. + See the `Multiple modules in one library `__ + section in :pep:`489` for details. + + +.. _multi-phase-initialization: + +Multi-phase initialization +.......................... + +The process of creating an extension module follows several phases: + +- Python finds and calls the export hook to get information on how to + create the module. +- Before any substantial code is executed, Python can determine which + capabilities the module supports, and it can adjust the environment or + refuse loading an incompatible extension. + Slots like :c:data:`Py_mod_abi`, :c:data:`Py_mod_gil` and + :c:data:`Py_mod_multiple_interpreters` influence this step. +- By default, Python itself then creates the module object -- that is, it does + the equivalent of calling :py:meth:`~object.__new__` when creating an object. + This step can be overridden using the :c:data:`Py_mod_create` slot. +- Python sets initial module attributes like :attr:`~module.__package__` and + :attr:`~module.__loader__`, and inserts the module object into + :py:attr:`sys.modules`. +- Afterwards, the module object is initialized in an extension-specific way + -- the equivalent of :py:meth:`~object.__init__` when creating an object, + or of executing top-level code in a Python-language module. + The behavior is specified using the :c:data:`Py_mod_exec` slot. + +This is called *multi-phase initialization* to distinguish it from the legacy +(but still supported) :ref:`single-phase initialization `, +where an initialization function returns a fully constructed module. + +.. versionchanged:: 3.5 + + Added support for multi-phase initialization (:pep:`489`). + + +Multiple module instances +......................... + +By default, extension modules are not singletons. +For example, if the :py:attr:`sys.modules` entry is removed and the module +is re-imported, a new module object is created and, typically, populated with +fresh method and type objects. +The old module is subject to normal garbage collection. +This mirrors the behavior of pure-Python modules. + +Additional module instances may be created in +:ref:`sub-interpreters ` +or after Python runtime reinitialization +(:c:func:`Py_Finalize` and :c:func:`Py_Initialize`). +In these cases, sharing Python objects between module instances would likely +cause crashes or undefined behavior. + +To avoid such issues, each instance of an extension module should +be *isolated*: changes to one instance should not implicitly affect the others, +and all state owned by the module, including references to Python objects, +should be specific to a particular module instance. +See :ref:`isolating-extensions-howto` for more details and a practical guide. + +A simpler way to avoid these issues is +:ref:`raising an error on repeated initialization `. + +All modules are expected to support +:ref:`sub-interpreters `, or otherwise explicitly +signal a lack of support. +This is usually achieved by isolation or blocking repeated initialization, +as above. +A module may also be limited to the main interpreter using +the :c:data:`Py_mod_multiple_interpreters` slot. + + +.. _extension-pyinit: + +``PyInit`` function +................... + +.. deprecated:: next + + This functionality is :term:`soft deprecated`. + It will not get new features, but there are no plans to remove it. + +Instead of :c:func:`PyModExport_modulename`, an extension module can define +an older-style :dfn:`initialization function` with the signature: + +.. c:function:: PyObject* PyInit_modulename(void) + +Its name should be :samp:`PyInit_{}`, with ```` replaced by the +name of the module. +For non-ASCII module names, use :samp:`PyInitU_{}` instead, with +```` encoded in the same way as for the +:ref:`export hook ` (that is, using Punycode +with underscores). + +If a module exports both :samp:`PyInit_{}` and +:samp:`PyModExport_{}`, the :samp:`PyInit_{}` function +is ignored. + +Like with :c:macro:`PyMODEXPORT_FUNC`, it is recommended to define the +initialization function using a helper macro: + +.. c:macro:: PyMODINIT_FUNC + + Declare an extension module initialization function. + This macro: + + * specifies the :c:expr:`PyObject*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. + + +Normally, the initialization function (``PyInit_modulename``) returns +a :c:type:`PyModuleDef` instance with non-``NULL`` +:c:member:`~PyModuleDef.m_slots`. This allows Python to use +:ref:`multi-phase initialization `. + +Before it is returned, the ``PyModuleDef`` instance must be initialized +using the following function: + +.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) + + Ensure a module definition is a properly initialized Python object that + correctly reports its type and a reference count. + + Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. + + Calling this function is required before returning a :c:type:`PyModuleDef` + from a module initialization function. + It should not be used in other contexts. + + Note that Python assumes that ``PyModuleDef`` structures are statically + allocated. + This function may return either a new reference or a borrowed one; + this reference must not be released. + + .. versionadded:: 3.5 + + +For example, a module called ``spam`` would be defined like this:: + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); + } + + +.. _single-phase-initialization: + +Legacy single-phase initialization +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. deprecated:: next + + Single-phase initialization is :term:`soft deprecated`. + It is a legacy mechanism to initialize extension + modules, with known drawbacks and design flaws. Extension module authors + are encouraged to use multi-phase initialization instead. + + However, there are no plans to remove support for it. + +In single-phase initialization, the old-style +:ref:`initializaton function ` (``PyInit_modulename``) +should create, populate and return a module object. +This is typically done using :c:func:`PyModule_Create` and functions like +:c:func:`PyModule_AddObjectRef`. + +Single-phase initialization differs from the :ref:`default ` +in the following ways: + +* Single-phase modules are, or rather *contain*, “singletons”. + + When the module is first initialized, Python saves the contents of + the module's ``__dict__`` (that is, typically, the module's functions and + types). + + For subsequent imports, Python does not call the initialization function + again. + Instead, it creates a new module object with a new ``__dict__``, and copies + the saved contents to it. + For example, given a single-phase module ``_testsinglephase`` + [#testsinglephase]_ that defines a function ``sum`` and an exception class + ``error``: + + .. code-block:: python + + >>> import sys + >>> import _testsinglephase as one + >>> del sys.modules['_testsinglephase'] + >>> import _testsinglephase as two + >>> one is two + False + >>> one.__dict__ is two.__dict__ + False + >>> one.sum is two.sum + True + >>> one.error is two.error + True + + The exact behavior should be considered a CPython implementation detail. + +* To work around the fact that ``PyInit_modulename`` does not take a *spec* + argument, some state of the import machinery is saved and applied to the + first suitable module created during the ``PyInit_modulename`` call. + Specifically, when a sub-module is imported, this mechanism prepends the + parent package name to the name of the module. + + A single-phase ``PyInit_modulename`` function should create “its” module + object as soon as possible, before any other module objects can be created. + +* Non-ASCII module names (``PyInitU_modulename``) are not supported. + +* Single-phase modules support module lookup functions like + :c:func:`PyState_FindModule`. + +* The module's :c:member:`PyModuleDef.m_slots` must be NULL. + +.. [#testsinglephase] ``_testsinglephase`` is an internal module used + in CPython's self-test suite; your installation may or may not + include it. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index e9019a0d500..9d01254ddb2 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -93,6 +93,29 @@ the :mod:`io` APIs instead. .. versionadded:: 3.8 +.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path) + + Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str` + object. The behavior of this function may be overridden by + :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the + text. + + This is analogous to :func:`io.open_code` in Python. + + On success, this function returns a :term:`strong reference` to a Python + file object. On failure, this function returns ``NULL`` with an exception + set. + + .. versionadded:: 3.8 + + +.. c:function:: PyObject *PyFile_OpenCode(const char *path) + + Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a + UTF-8 encoded :c:expr:`const char*`. + + .. versionadded:: 3.8 + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index c5a7653efca..b0d440580b9 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,111 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_INFINITY + + This macro expands a to constant expression of type :c:expr:`double`, that + represents the positive infinity. + + It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard + ```` header. + + .. deprecated:: 3.15 + The macro is :term:`soft deprecated`. + + +.. c:macro:: Py_NAN + + This macro expands a to constant expression of type :c:expr:`double`, that + represents a quiet not-a-number (qNaN) value. + + On most platforms, this is equivalent to the :c:macro:`!NAN` macro from + the C11 standard ```` header. + + +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + + +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + +.. c:macro:: Py_MATH_El + + High precision (long double) definition of :data:`~math.e` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + +.. c:macro:: Py_MATH_PIl + + High precision (long double) definition of :data:`~math.pi` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + +.. c:macro:: Py_RETURN_NAN + + Return :data:`math.nan` from a function. + + On most platforms, this is equivalent to ``return PyFloat_FromDouble(NAN)``. + + +.. c:macro:: Py_RETURN_INF(sign) + + Return :data:`math.inf` or :data:`-math.inf ` from a function, + depending on the sign of *sign*. + + On most platforms, this is equivalent to the following:: + + return PyFloat_FromDouble(copysign(INFINITY, sign)); + + +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- @@ -96,8 +201,8 @@ NaNs (if such things exist on the platform) isn't handled correctly, and attempting to unpack a bytes string containing an IEEE INF or NaN will raise an exception. -Note that NaNs type may not be preserved on IEEE platforms (silent NaN become -quiet), for example on x86 systems in 32-bit mode. +Note that NaNs type may not be preserved on IEEE platforms (signaling NaN become +quiet NaN), for example on x86 systems in 32-bit mode. On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less @@ -124,15 +229,15 @@ There are two problems on non-IEEE platforms: * What this does is undefined if *x* is a NaN or infinity. * ``-0.0`` and ``+0.0`` produce the same bytes string. -.. c:function:: int PyFloat_Pack2(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack2(double x, char *p, int le) Pack a C double as the IEEE 754 binary16 half-precision format. -.. c:function:: int PyFloat_Pack4(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack4(double x, char *p, int le) Pack a C double as the IEEE 754 binary32 single precision format. -.. c:function:: int PyFloat_Pack8(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack8(double x, char *p, int le) Pack a C double as the IEEE 754 binary64 double precision format. @@ -154,14 +259,14 @@ Return value: The unpacked double. On error, this is ``-1.0`` and Note that on a non-IEEE platform this will refuse to unpack a bytes string that represents a NaN or infinity. -.. c:function:: double PyFloat_Unpack2(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack2(const char *p, int le) Unpack the IEEE 754 binary16 half-precision format as a C double. -.. c:function:: double PyFloat_Unpack4(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack4(const char *p, int le) Unpack the IEEE 754 binary32 single precision format as a C double. -.. c:function:: double PyFloat_Unpack8(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack8(const char *p, int le) Unpack the IEEE 754 binary64 double precision format as a C double. diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1a52e146a69..fb17cf7f1da 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -29,6 +29,12 @@ See also :ref:`Reflection `. Previously, this type was only available after including ````. +.. c:function:: PyFrameObject *PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) + + Create a new frame object. This function returns a :term:`strong reference` + to the new frame object on success, and returns ``NULL`` with an exception + set on failure. + .. c:function:: int PyFrame_Check(PyObject *obj) Return non-zero if *obj* is a frame object. @@ -161,6 +167,57 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. + +Legacy Local Variable APIs +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. +They exist solely for backwards compatibility. + + +.. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` + attribute of *f* to the internal "fast" array of local variables, allowing + changes in frame objects to be visible to the interpreter. If *clear* was + true, this function would process variables that were unset in the locals + dictionary. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function would copy the internal "fast" array + of local variables (which is used by the interpreter) to the + :attr:`~frame.f_locals` attribute of *f*, allowing changes in local + variables to be visible to frame objects. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function was similar to + :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and + ``-1`` with an exception set on failure. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. seealso:: + :pep:`667` + + Internal Frames ^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 58792edeed2..609b5e885b6 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -95,6 +95,22 @@ There are a few functions specific to Python functions. .. versionadded:: 3.12 + +.. c:function:: PyObject* PyFunction_GetKwDefaults(PyObject *op) + + Return the keyword-only argument default values of the function object *op*. This can be a + dictionary of arguments or ``NULL``. + + +.. c:function:: int PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) + + Set the keyword-only argument default values of the function object *op*. + *defaults* must be a dictionary of keyword-only arguments or ``Py_None``. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) Return the closure associated with the function object *op*. This can be ``NULL`` @@ -123,6 +139,19 @@ There are a few functions specific to Python functions. Raises :exc:`SystemError` and returns ``-1`` on failure. +.. c:function:: PyObject *PyFunction_GET_CODE(PyObject *op) + PyObject *PyFunction_GET_GLOBALS(PyObject *op) + PyObject *PyFunction_GET_MODULE(PyObject *op) + PyObject *PyFunction_GET_DEFAULTS(PyObject *op) + PyObject *PyFunction_GET_KW_DEFAULTS(PyObject *op) + PyObject *PyFunction_GET_CLOSURE(PyObject *op) + PyObject *PyFunction_GET_ANNOTATIONS(PyObject *op) + + These functions are similar to their ``PyFunction_Get*`` counterparts, but + do not do type checking. Passing anything other than an instance of + :c:data:`PyFunction_Type` is undefined behavior. + + .. c:function:: int PyFunction_AddWatcher(PyFunction_WatchCallback callback) Register *callback* as a function watcher for the current interpreter. @@ -155,6 +184,9 @@ There are a few functions specific to Python functions. .. versionadded:: 3.12 + - ``PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME`` + + .. versionadded:: 3.15 .. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) @@ -169,7 +201,7 @@ There are a few functions specific to Python functions. unpredictable effects, including infinite recursion. If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked - after `func` has been fully initialized. Otherwise, the callback is invoked + after *func* has been fully initialized. Otherwise, the callback is invoked before the modification to *func* takes place, so the prior state of *func* can be inspected. The runtime is permitted to optimize away the creation of function objects when possible. In such cases no event will be emitted. @@ -177,7 +209,7 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + If *event* is ``PyFunction_EVENT_DESTROY``, taking a reference in the callback to the about-to-be-destroyed function will resurrect it, preventing it from being freed at this time. When the resurrected object is destroyed later, any watcher callbacks active at that time will be called again. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index d1f0982b818..f6fa52b36c5 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -57,11 +57,49 @@ rules: Analogous to :c:macro:`PyObject_New` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size) Analogous to :c:macro:`PyObject_NewVar` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) Analogous to :c:macro:`PyObject_GC_New` but allocates *extra_size* @@ -73,6 +111,10 @@ rules: The extra data will be deallocated with the object, but otherwise it is not managed by Python. + Memory allocated by this function must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + .. warning:: The function is marked as unstable because the final mechanism for reserving extra data after an instance is not yet decided. @@ -136,6 +178,21 @@ rules: Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_New`, + :c:macro:`PyObject_NewVar`, or related allocation functions; use + :c:func:`PyObject_Free` instead. + + .. seealso:: + + * :c:func:`PyObject_Free` is the non-GC equivalent of this function. + * :c:macro:`PyObject_GC_New` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. c:function:: void PyObject_GC_UnTrack(void *op) @@ -180,9 +237,9 @@ provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` must name its arguments exactly *visit* and *arg*: -.. c:function:: void Py_VISIT(PyObject *o) +.. c:macro:: Py_VISIT(o) - If *o* is not ``NULL``, call the *visit* callback, with arguments *o* + If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o* and *arg*. If *visit* returns a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 0eb5922f6da..44f3bdbf959 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -44,3 +44,41 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + +.. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen) + + Return a new :term:`strong reference` to the code object wrapped by *gen*. + This function always succeeds. + + +Asynchronous Generator Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. seealso:: + :pep:`525` + +.. c:var:: PyTypeObject PyAsyncGen_Type + + The type object corresponding to asynchronous generator objects. This is + available as :class:`types.AsyncGeneratorType` in the Python layer. + + .. versionadded:: 3.6 + +.. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) + + Create a new asynchronous generator wrapping *frame*, with ``__name__`` and + ``__qualname__`` set to *name* and *qualname*. *frame* is stolen by this + function and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to the + new asynchronous generator. On failure, this function returns ``NULL`` + with an exception set. + + .. versionadded:: 3.6 + +.. c:function:: int PyAsyncGen_CheckExact(PyObject *op) + + Return true if *op* is an asynchronous generator object, false otherwise. + This function always succeeds. + + .. versionadded:: 3.6 diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 00f8cb887dc..1ad712b0ce4 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -11,47 +11,103 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.2 + .. c:type:: Py_uhash_t Hash value type: unsigned integer. .. versionadded:: 3.2 + +.. c:macro:: Py_HASH_ALGORITHM + + A numerical value indicating the algorithm for hashing of :class:`str`, + :class:`bytes`, and :class:`memoryview`. + + The algorithm name is exposed by :data:`sys.hash_info.algorithm`. + + .. versionadded:: 3.4 + + +.. c:macro:: Py_HASH_FNV + Py_HASH_SIPHASH24 + Py_HASH_SIPHASH13 + + Numerical values to compare to :c:macro:`Py_HASH_ALGORITHM` to determine + which algorithm is used for hashing. The hash algorithm can be configured + via the configure :option:`--with-hash-algorithm` option. + + .. versionadded:: 3.4 + Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. + + .. versionadded:: 3.11 + Add :c:macro:`!Py_HASH_SIPHASH13`. + + +.. c:macro:: Py_HASH_CUTOFF + + Buffers of length in range ``[1, Py_HASH_CUTOFF)`` are hashed using DJBX33A + instead of the algorithm described by :c:macro:`Py_HASH_ALGORITHM`. + + - A :c:macro:`!Py_HASH_CUTOFF` of 0 disables the optimization. + - :c:macro:`!Py_HASH_CUTOFF` must be non-negative and less or equal than 7. + + 32-bit platforms should use a cutoff smaller than 64-bit platforms because + it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms + and 5 on 32-bit platforms should provide a decent safety margin. + + This corresponds to the :data:`sys.hash_info.cutoff` constant. + + .. versionadded:: 3.4 + + .. c:macro:: PyHASH_MODULUS - The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + The `Mersenne prime `_ ``P = 2**n -1``, + used for numeric hash scheme. + + This corresponds to the :data:`sys.hash_info.modulus` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. .. versionadded:: 3.13 + .. c:macro:: PyHASH_MULTIPLIER Prime multiplier used in string and various other hashes. .. versionadded:: 3.13 + .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. + .. versionadded:: 3.13 + .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. + .. versionadded:: 3.13 + .. c:type:: PyHash_FuncDef Hash function definition used by :c:func:`PyHash_GetFuncDef`. - .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + .. c:member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) Hash function. @@ -59,14 +115,20 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. Hash function name (UTF-8 encoded string). + This corresponds to the :data:`sys.hash_info.algorithm` constant. + .. c:member:: const int hash_bits Internal size of the hash value in bits. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. + .. c:member:: const int seed_bits Size of seed input in bits. + This corresponds to the :data:`sys.hash_info.seed_bits` constant. + .. versionadded:: 3.4 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 1cab3ce3061..1786ac6b503 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -16,19 +16,6 @@ Importing Modules This is a wrapper around :c:func:`PyImport_Import()` which takes a :c:expr:`const char *` as an argument instead of a :c:expr:`PyObject *`. -.. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) - - This function is a deprecated alias of :c:func:`PyImport_ImportModule`. - - .. versionchanged:: 3.3 - This function used to fail immediately when the import lock was held - by another thread. In Python 3.3 though, the locking scheme switched - to per-module locks for most purposes, so this function's special - behaviour isn't needed anymore. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyImport_ImportModule` instead. - .. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) @@ -327,6 +314,13 @@ Importing Modules initialization. +.. c:var:: struct _inittab *PyImport_Inittab + + The table of built-in modules used by Python initialization. Do not use this directly; + use :c:func:`PyImport_AppendInittab` and :c:func:`PyImport_ExtendInittab` + instead. + + .. c:function:: PyObject* PyImport_ImportModuleAttr(PyObject *mod_name, PyObject *attr_name) Import the module *mod_name* and get its attribute *attr_name*. @@ -346,3 +340,24 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index ba56b03c6ac..e9df2a304d9 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -17,6 +17,7 @@ document the API functions in detail. veryhigh.rst refcounting.rst exceptions.rst + extension-modules.rst utilities.rst abstract.rst concrete.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 52f64a61006..7411644f9e1 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -41,7 +41,6 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyObject_SetArenaAllocator` * :c:func:`Py_SetProgramName` * :c:func:`Py_SetPythonHome` - * :c:func:`PySys_ResetWarnOptions` * the configuration functions covered in :ref:`init-config` * Informative functions: @@ -77,10 +76,7 @@ The following functions can be safely called before Python is initialized: Despite their apparent similarity to some of the functions listed above, the following functions **should not be called** before the interpreter has - been initialized: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, - :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - :c:func:`Py_GetProgramName`, :c:func:`PyEval_InitThreads`, and + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and :c:func:`Py_RunMain`. @@ -145,9 +141,6 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. :c:member:`PyConfig.pathconfig_warnings` should be used instead, see :ref:`Python Initialization Configuration `. - Suppress error messages when calculating the module search path in - :c:func:`Py_GetPath`. - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. .. deprecated-removed:: 3.12 3.15 @@ -203,7 +196,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_IsolatedFlag @@ -498,17 +491,8 @@ Initializing and finalizing the interpreter strings other than those passed in (however, the contents of the strings pointed to by the argument list are not modified). - The return value will be ``0`` if the interpreter exits normally (i.e., - without an exception), ``1`` if the interpreter exits due to an exception, - or ``2`` if the argument list does not represent a valid Python command - line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will - drop into the interactive Python prompt, at which point a second otherwise - unhandled :exc:`SystemExit` will still exit the process, while any other - means of exiting will set the return value as described above. + The return value is ``2`` if the argument list does not represent a valid + Python command line, and otherwise the same as :c:func:`Py_RunMain`. In terms of the CPython runtime configuration APIs documented in the :ref:`runtime configuration ` section (and without accounting @@ -545,23 +529,18 @@ Initializing and finalizing the interpreter If :c:member:`PyConfig.inspect` is not set (the default), the return value will be ``0`` if the interpreter exits normally (that is, without raising - an exception), or ``1`` if the interpreter exits due to an exception. If an - otherwise unhandled :exc:`SystemExit` is raised, the function will immediately - exit the process instead of returning ``1``. + an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` + for any other unhandled exception. If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option is used), rather than returning when the interpreter exits, execution will instead resume in an interactive Python prompt (REPL) using the ``__main__`` module's global namespace. If the interpreter exited with an exception, it is immediately raised in the REPL session. The function return value is - then determined by the way the *REPL session* terminates: returning ``0`` - if the session terminates without raising an unhandled exception, exiting - immediately for an unhandled :exc:`SystemExit`, and returning ``1`` for - any other unhandled exception. + then determined by the way the *REPL session* terminates: ``0``, ``1``, or + the status of a :exc:`SystemExit`, as specified above. - This function always finalizes the Python interpreter regardless of whether - it returns a value or immediately exits the process due to an unhandled - :exc:`SystemExit` exception. + This function always finalizes the Python interpreter before it returns. See :ref:`Python Configuration ` for an example of a customized Python that always runs in isolated mode using @@ -586,7 +565,6 @@ Process-wide parameters .. index:: single: Py_Initialize() single: main() - single: Py_GetPath() This API is kept for backward compatibility: setting :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python @@ -596,7 +574,7 @@ Process-wide parameters the first time, if it is called at all. It tells the interpreter the value of the ``argv[0]`` argument to the :c:func:`main` function of the program (converted to wide characters). - This is used by :c:func:`Py_GetPath` and some other functions below to find + This is used by some other functions below to find the Python run-time libraries relative to the interpreter executable. The default value is ``'python'``. The argument should point to a zero-terminated wide character string in static storage whose contents will not @@ -609,146 +587,6 @@ Process-wide parameters .. deprecated-removed:: 3.11 3.15 -.. c:function:: wchar_t* Py_GetProgramName() - - Return the program name set with :c:member:`PyConfig.program_name`, or the default. - The returned string points into static storage; the caller should not modify its - value. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("executable") ` - (:data:`sys.executable`) instead. - - -.. c:function:: wchar_t* Py_GetPrefix() - - Return the *prefix* for installed platform-independent files. This is derived - through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The - returned string points into static storage; the caller should not modify its - value. This corresponds to the :makevar:`prefix` variable in the top-level - :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.base_prefix``. - It is only useful on Unix. See also the next function. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("base_prefix") ` - (:data:`sys.base_prefix`) instead. Use :c:func:`PyConfig_Get("prefix") - ` (:data:`sys.prefix`) if :ref:`virtual environments - ` need to be handled. - - -.. c:function:: wchar_t* Py_GetExecPrefix() - - Return the *exec-prefix* for installed platform-*dependent* files. This is - derived through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the exec-prefix is - ``'/usr/local'``. The returned string points into static storage; the caller - should not modify its value. This corresponds to the :makevar:`exec_prefix` - variable in the top-level :file:`Makefile` and the ``--exec-prefix`` - argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.base_exec_prefix``. It is only useful on - Unix. - - Background: The exec-prefix differs from the prefix when platform dependent - files (such as executables and shared libraries) are installed in a different - directory tree. In a typical installation, platform dependent files may be - installed in the :file:`/usr/local/plat` subtree while platform independent may - be installed in :file:`/usr/local`. - - Generally speaking, a platform is a combination of hardware and software - families, e.g. Sparc machines running the Solaris 2.x operating system are - considered the same platform, but Intel machines running Solaris 2.x are another - platform, and Intel machines running Linux are yet another platform. Different - major revisions of the same operating system generally also form different - platforms. Non-Unix operating systems are a different story; the installation - strategies on those systems are so different that the prefix and exec-prefix are - meaningless, and set to the empty string. Note that compiled Python bytecode - files are platform independent (but not independent from the Python version by - which they were compiled!). - - System administrators will know how to configure the :program:`mount` or - :program:`automount` programs to share :file:`/usr/local` between platforms - while having :file:`/usr/local/plat` be a different filesystem for each - platform. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("base_exec_prefix") ` - (:data:`sys.base_exec_prefix`) instead. Use - :c:func:`PyConfig_Get("exec_prefix") ` - (:data:`sys.exec_prefix`) if :ref:`virtual environments ` need - to be handled. - -.. c:function:: wchar_t* Py_GetProgramFullPath() - - .. index:: - single: executable (in module sys) - - Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:member:`PyConfig.program_name`). The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.executable``. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("executable") ` - (:data:`sys.executable`) instead. - - -.. c:function:: wchar_t* Py_GetPath() - - .. index:: - triple: module; search; path - single: path (in module sys) - - Return the default module search path; this is computed from the program name - (set by :c:member:`PyConfig.program_name`) and some environment variables. - The returned string consists of a series of directory names separated by a - platform dependent delimiter character. The delimiter character is ``':'`` - on Unix and macOS, ``';'`` on Windows. The returned string points into - static storage; the caller should not modify its value. The list - :data:`sys.path` is initialized with this value on interpreter startup; it - can be (and usually is) modified later to change the search path for loading - modules. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. XXX should give the exact rules - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("module_search_paths") ` - (:data:`sys.path`) instead. - .. c:function:: const char* Py_GetVersion() Return the version of this Python interpreter. This is a string that looks @@ -919,23 +757,6 @@ Process-wide parameters .. deprecated-removed:: 3.11 3.15 -.. c:function:: wchar_t* Py_GetPythonHome() - - Return the default "home", that is, the value set by - :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME` - environment variable if it is set. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("home") ` or the - :envvar:`PYTHONHOME` environment variable instead. - - .. _threads: Thread State and the Global Interpreter Lock @@ -1083,8 +904,36 @@ Note that the ``PyGILState_*`` functions assume there is only one global interpreter (created automatically by :c:func:`Py_Initialize`). Python supports the creation of additional interpreters (using :c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. +``PyGILState_*`` API is unsupported. This is because :c:func:`PyGILState_Ensure` +and similar functions default to :term:`attaching ` a +:term:`thread state` for the main interpreter, meaning that the thread can't safely +interact with the calling subinterpreter. +Supporting subinterpreters in non-Python threads +------------------------------------------------ + +If you would like to support subinterpreters with non-Python created threads, you +must use the ``PyThreadState_*`` API instead of the traditional ``PyGILState_*`` +API. + +In particular, you must store the interpreter state from the calling +function and pass it to :c:func:`PyThreadState_New`, which will ensure that +the :term:`thread state` is targeting the correct interpreter:: + + /* The return value of PyInterpreterState_Get() from the + function that created this thread. */ + PyInterpreterState *interp = ThreadData->interp; + PyThreadState *tstate = PyThreadState_New(interp); + PyThreadState_Swap(tstate); + + /* GIL of the subinterpreter is now held. + Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + + /* Destroy the thread state. No Python API allowed beyond this point. */ + PyThreadState_Clear(tstate); + PyThreadState_DeleteCurrent(); .. _fork-and-threads: @@ -1170,6 +1019,12 @@ code, or when embedding the Python interpreter: interpreter lock is also shared by all threads, regardless of to which interpreter they belong. + .. versionchanged:: 3.12 + + :pep:`684` introduced the possibility + of a :ref:`per-interpreter GIL `. + See :c:func:`Py_NewInterpreterFromConfig`. + .. c:type:: PyThreadState @@ -1258,13 +1113,30 @@ code, or when embedding the Python interpreter: This function is safe to call without an :term:`attached thread state`; it will simply return ``NULL`` indicating that there was no prior thread state. - .. seealso: + .. seealso:: :c:func:`PyEval_ReleaseThread` + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + The following functions use thread-local storage, and are not compatible with sub-interpreters: +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + .. c:function:: PyGILState_STATE PyGILState_Ensure() Ensure that the current thread is ready to call the Python C API regardless @@ -1287,10 +1159,10 @@ with sub-interpreters: When the function returns, there will be an :term:`attached thread state` and the thread will be able to call arbitrary Python code. Failure is a fatal error. - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to + .. warning:: + Calling this function when the runtime is finalizing is unsafe. Doing + so will either hang the thread until the program ends, or fully crash + the interpreter in rare cases. Refer to :ref:`cautions-regarding-runtime-finalization` for more details. .. versionchanged:: 3.14 @@ -1307,7 +1179,6 @@ with sub-interpreters: Every call to :c:func:`PyGILState_Ensure` must be matched by a call to :c:func:`PyGILState_Release` on the same thread. - .. c:function:: PyThreadState* PyGILState_GetThisThreadState() Get the :term:`attached thread state` for this thread. May return ``NULL`` if no @@ -1315,20 +1186,30 @@ with sub-interpreters: always has such a thread-state, even if no auto-thread-state call has been made on the main thread. This is mainly a helper/diagnostic function. - .. seealso: :c:func:`PyThreadState_Get`` + .. note:: + This function may return non-``NULL`` even when the :term:`thread state` + is detached. + Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + for most cases. + .. seealso:: :c:func:`PyThreadState_Get` .. c:function:: int PyGILState_Check() Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. This function can be called from any thread at any time. - Only if it has had its Python thread state initialized and currently is - holding the :term:`GIL` will it return ``1``. + Only if it has had its :term:`thread state ` initialized + via :c:func:`PyGILState_Ensure` will it return ``1``. This is mainly a helper/diagnostic function. It can be useful for example in callback contexts or memory allocation functions when knowing that the :term:`GIL` is locked can allow the caller to perform sensitive actions or otherwise behave differently. + .. note:: + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` + for most cases. + .. versionadded:: 3.4 @@ -1387,7 +1268,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) Reset all information in an interpreter state object. There must be - an :term:`attached thread state` for the the interpreter. + an :term:`attached thread state` for the interpreter. .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear @@ -1410,11 +1291,11 @@ All of the following functions must be called after :c:func:`Py_Initialize`. must be :term:`attached ` .. versionchanged:: 3.9 - This function now calls the :c:member:`PyThreadState.on_delete` callback. + This function now calls the :c:member:`!PyThreadState.on_delete` callback. Previously, that happened in :c:func:`PyThreadState_Delete`. .. versionchanged:: 3.13 - The :c:member:`PyThreadState.on_delete` callback was removed. + The :c:member:`!PyThreadState.on_delete` callback was removed. .. c:function:: void PyThreadState_Delete(PyThreadState *tstate) @@ -1485,6 +1366,43 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.11 +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) Get the current interpreter. @@ -1514,6 +1432,9 @@ All of the following functions must be called after :c:func:`Py_Initialize`. This is not a replacement for :c:func:`PyModule_GetState()`, which extensions should use to store interpreter-specific state information. + The returned dictionary is borrowed from the interpreter and is valid until + interpreter shutdown. + .. versionadded:: 3.8 @@ -1796,7 +1717,8 @@ function. You can create and destroy them using the following functions: Only C-level static and global variables are shared between these module objects. - * For modules using single-phase initialization, + * For modules using legacy + :ref:`single-phase initialization `, e.g. :c:func:`PyModule_Create`, the first time a particular extension is imported, it is initialized normally, and a (shallow) copy of its module's dictionary is squirreled away. @@ -1848,6 +1770,8 @@ function. You can create and destroy them using the following functions: haven't been explicitly destroyed at that point. +.. _per-interpreter-gil: + A Per-Interpreter GIL --------------------- @@ -1859,7 +1783,7 @@ being blocked by other interpreters or blocking any others. Thus a single Python process can truly take advantage of multiple CPU cores when running Python code. The isolation also encourages a different approach to concurrency than that of just using threads. -(See :pep:`554`.) +(See :pep:`554` and :pep:`684`.) Using an isolated interpreter requires vigilance in preserving that isolation. That especially means not sharing any objects or mutable @@ -1964,6 +1888,29 @@ pointer and a void pointer argument. called from the main interpreter. Each subinterpreter now has its own list of scheduled calls. + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + .. _profiling: Profiling and Tracing @@ -2140,6 +2087,11 @@ Reference tracing is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer that was provided when :c:func:`PyRefTracer_SetTracer` was called. + If a new tracing function is registered replacing the current a call to the + trace function will be made with the object set to **NULL** and **event** set to + :c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new + function is registered. + .. versionadded:: 3.13 .. c:var:: int PyRefTracer_CREATE @@ -2152,6 +2104,13 @@ Reference tracing The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python object has been destroyed. +.. c:var:: int PyRefTracer_TRACKER_REMOVED + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when the + current tracer is about to be replaced by a new one. + + .. versionadded:: 3.14 + .. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) Register a reference tracer function. The function will be called when a new @@ -2167,6 +2126,10 @@ Reference tracing There must be an :term:`attached thread state` when calling this function. + If another tracer function was already registered, the old function will be + called with **event** set to :c:data:`PyRefTracer_TRACKER_REMOVED` just before + the new function is registered. + .. versionadded:: 3.13 .. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) @@ -2234,7 +2197,7 @@ use a thread key and functions to associate a :c:expr:`void*` value per thread. A :term:`thread state` does *not* need to be :term:`attached ` -when calling these functions; they suppl their own locking. +when calling these functions; they supply their own locking. Note that :file:`Python.h` does not include the declaration of the TLS APIs, you need to include :file:`pythread.h` to use thread-local storage. @@ -2414,6 +2377,18 @@ The C-API provides a basic mutual exclusion lock. .. versionadded:: 3.13 +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + .. _python-critical-section-api: Python Critical Section API @@ -2424,12 +2399,26 @@ per-object locks for :term:`free-threaded ` CPython. They are intended to replace reliance on the :term:`global interpreter lock`, and are no-ops in versions of Python with the global interpreter lock. +Critical sections are intended to be used for custom types implemented +in C-API extensions. They should generally not be used with built-in types like +:class:`list` and :class:`dict` because their public C-APIs +already use critical sections internally, with the notable +exception of :c:func:`PyDict_Next`, which requires critical section +to be acquired externally. + Critical sections avoid deadlocks by implicitly suspending active critical -sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`. -When :c:func:`PyEval_RestoreThread` is called, the most recent critical section -is resumed, and its locks reacquired. This means the critical section API -provides weaker guarantees than traditional locks -- they are useful because -their behavior is similar to the :term:`GIL`. +sections, hence, they do not provide exclusive access such as provided by +traditional locks like :c:type:`PyMutex`. When a critical section is started, +the per-object lock for the object is acquired. If the code executed inside the +critical section calls C-API functions then it can suspend the critical section thereby +releasing the per-object lock, so other threads can acquire the per-object lock +for the same object. + +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. The functions and structs used by the macros are exposed for cases where C macros are not available. They should only be used as in the @@ -2476,6 +2465,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + .. c:macro:: Py_END_CRITICAL_SECTION() Ends the critical section and releases the per-object lock. @@ -2505,6 +2511,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + .. c:macro:: Py_END_CRITICAL_SECTION2() Ends the critical section and releases the per-object locks. @@ -2517,3 +2540,220 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. In the default build, this macro expands to ``}``. .. versionadded:: 3.13 + + +Legacy Locking APIs +------------------- + +These APIs are obsolete since Python 3.13 with the introduction of +:c:type:`PyMutex`. + +.. versionchanged:: 3.15 + These APIs are now a simple wrapper around ``PyMutex``. + + +.. c:type:: PyThread_type_lock + + A pointer to a mutual exclusion lock. + + +.. c:type:: PyLockStatus + + The result of acquiring a lock with a timeout. + + .. c:namespace:: NULL + + .. c:enumerator:: PY_LOCK_FAILURE + + Failed to acquire the lock. + + .. c:enumerator:: PY_LOCK_ACQUIRED + + The lock was successfully acquired. + + .. c:enumerator:: PY_LOCK_INTR + + The lock was interrupted by a signal. + + +.. c:function:: PyThread_type_lock PyThread_allocate_lock(void) + + Allocate a new lock. + + On success, this function returns a lock; on failure, this + function returns ``0`` without an exception set. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionchanged:: 3.15 + This function now always uses :c:type:`PyMutex`. In prior versions, this + would use a lock provided by the operating system. + + +.. c:function:: void PyThread_free_lock(PyThread_type_lock lock) + + Destroy *lock*. The lock should not be held by any thread when calling + this. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, long long microseconds, int intr_flag) + + Acquire *lock* with a timeout. + + This will wait for *microseconds* microseconds to acquire the lock. If the + timeout expires, this function returns :c:enumerator:`PY_LOCK_FAILURE`. + If *microseconds* is ``-1``, this will wait indefinitely until the lock has + been released. + + If *intr_flag* is ``1``, acquiring the lock may be interrupted by a signal, + in which case this function returns :c:enumerator:`PY_LOCK_INTR`. Upon + interruption, it's generally expected that the caller makes a call to + :c:func:`Py_MakePendingCalls` to propagate an exception to Python code. + + If the lock is successfully acquired, this function returns + :c:enumerator:`PY_LOCK_ACQUIRED`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) + + Acquire *lock*. + + If *waitflag* is ``1`` and another thread currently holds the lock, this + function will wait until the lock can be acquired and will always return + ``1``. + + If *waitflag* is ``0`` and another thread holds the lock, this function will + not wait and instead return ``0``. If the lock is not held by any other + thread, then this function will acquire it and return ``1``. + + Unlike :c:func:`PyThread_acquire_lock_timed`, acquiring the lock cannot be + interrupted by a signal. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_release_lock(PyThread_type_lock lock) + + Release *lock*. If *lock* is not held, then this function issues a + fatal error. + + The caller does not need to hold an :term:`attached thread state`. + + +Operating System Thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index e1931655618..c345029e4ac 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -102,7 +102,7 @@ Error Handling * Set *\*err_msg* and return ``1`` if an error is set. * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. - An error message is an UTF-8 encoded string. + An error message is a UTF-8 encoded string. If *config* has an exit code, format the exit code as an error message. @@ -975,9 +975,7 @@ PyPreConfig Set to ``0`` or ``1`` by the :option:`-X utf8 <-X>` command line option and the :envvar:`PYTHONUTF8` environment variable. - Also set to ``1`` if the ``LC_CTYPE`` locale is ``C`` or ``POSIX``. - - Default: ``-1`` in Python config and ``0`` in isolated config. + Default: ``1``. .. _c-preinit: @@ -1280,6 +1278,11 @@ PyConfig Default: ``0``. + .. deprecated-removed:: 3.15 3.17 + + The :option:`-b` and :option:`!-bb` options will become no-op in 3.17. + :c:member:`~PyConfig.bytes_warning` member will be removed in 3.17. + .. c:member:: int warn_default_encoding If non-zero, emit a :exc:`EncodingWarning` warning when :class:`io.TextIOWrapper` @@ -2111,7 +2114,7 @@ initialization:: /* Specify sys.path explicitly */ /* If you want to modify the default set of paths, finish - initialization first and then use PySys_GetObject("path") */ + initialization first and then use PySys_GetAttrString("path") */ config.module_search_paths_set = 1; status = PyWideStringList_Append(&config.module_search_paths, L"/path/to/stdlib"); @@ -2260,6 +2263,7 @@ If a ``._pth`` file is present: * Set :c:member:`~PyConfig.isolated` to ``1``. * Set :c:member:`~PyConfig.use_environment` to ``0``. * Set :c:member:`~PyConfig.site_import` to ``0``. +* Set :c:member:`~PyConfig.user_site_directory` to ``0`` (since 3.15). * Set :c:member:`~PyConfig.safe_path` to ``1``. If :c:member:`~PyConfig.home` is not set and a ``pyvenv.cfg`` file is present in @@ -2280,6 +2284,12 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set therefore affected by :option:`-S`. +.. versionchanged:: 3.15 + + :c:member:`~PyConfig.user_site_directory` is now set to ``0`` when a + ``._pth`` file is present. + + Py_GetArgcArgv() ================ diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c8c60eb9f48..bb94bcb86a7 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -111,38 +111,20 @@ Useful macros ============= Several useful macros are defined in the Python header files. Many are -defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`). +defined closer to where they are useful (for example, :c:macro:`Py_RETURN_NONE`, +:c:macro:`PyMODINIT_FUNC`). Others of a more general utility are defined here. This is not necessarily a complete listing. -.. c:macro:: PyMODINIT_FUNC - - Declare an extension module ``PyInit`` initialization function. The function - return type is :c:expr:`PyObject*`. The macro declares any special linkage - declarations required by the platform, and for C++ declares the function as - ``extern "C"``. - - The initialization function must be named :samp:`PyInit_{name}`, where - *name* is the name of the module, and should be the only non-\ ``static`` - item defined in the module file. Example:: - - static struct PyModuleDef spam_module = { - PyModuleDef_HEAD_INIT, - .m_name = "spam", - ... - }; - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModule_Create(&spam_module); - } - .. c:macro:: Py_ABS(x) Return the absolute value of ``x``. + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + .. versionadded:: 3.3 .. c:macro:: Py_ALWAYS_INLINE @@ -189,6 +171,17 @@ complete listing. Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the command line (see :c:member:`PyConfig.use_environment`). +.. c:macro:: Py_LOCAL(type) + + Declare a function returning the specified *type* using a fast-calling + qualifier for functions that are local to the current file. + Semantically, this is equivalent to ``static type``. + +.. c:macro:: Py_LOCAL_INLINE(type) + + Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function + be inlined. + .. c:macro:: Py_MAX(x, y) Return the maximum value between ``x`` and ``y``. @@ -201,6 +194,14 @@ complete listing. .. versionadded:: 3.6 +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. + Use :c:func:`!memcpy` directly instead. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + .. c:macro:: Py_MIN(x, y) Return the minimum value between ``x`` and ``y``. @@ -255,9 +256,32 @@ complete listing. .. versionadded:: 3.4 +.. c:macro:: Py_BUILD_ASSERT(cond) + + Asserts a compile-time condition *cond*, as a statement. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); + + .. versionadded:: 3.3 + +.. c:macro:: Py_BUILD_ASSERT_EXPR(cond) + + Asserts a compile-time condition *cond*, as an expression that evaluates to ``0``. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + #define foo_to_char(foo) \ + ((char *)(foo) + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + .. versionadded:: 3.3 + .. c:macro:: PyDoc_STRVAR(name, str) - Creates a variable with name ``name`` that can be used in docstrings. + Creates a variable with name *name* that can be used in docstrings. If Python is built without docstrings, the value will be empty. Use :c:macro:`PyDoc_STRVAR` for docstrings to support building @@ -289,6 +313,28 @@ complete listing. {NULL, NULL} }; +.. c:macro:: PyDoc_VAR(name) + + Declares a static character array variable with the given name *name*. + + For example:: + + PyDoc_VAR(python_doc) = PyDoc_STR("A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); + +.. c:macro:: Py_ARRAY_LENGTH(array) + + Compute the length of a statically allocated C array at compile time. + + The *array* argument must be a C array with a size known at compile time. + Passing an array with an unknown size, such as a heap-allocated array, + will result in a compilation error on some compilers, or otherwise produce + incorrect results. + + This is roughly equivalent to:: + + sizeof(array) / sizeof((array)[0]) + .. _api-objects: @@ -779,20 +825,11 @@ found along :envvar:`PATH`.) The user can override this behavior by setting the environment variable :envvar:`PYTHONHOME`, or insert additional directories in front of the standard path by setting :envvar:`PYTHONPATH`. -.. index:: - single: Py_GetPath (C function) - single: Py_GetPrefix (C function) - single: Py_GetExecPrefix (C function) - single: Py_GetProgramFullPath (C function) - The embedding application can steer the search by setting :c:member:`PyConfig.program_name` *before* calling :c:func:`Py_InitializeFromConfig`. Note that :envvar:`PYTHONHOME` still overrides this and :envvar:`PYTHONPATH` is still -inserted in front of the standard path. An application that requires total -control has to provide its own implementation of :c:func:`Py_GetPath`, -:c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and -:c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). +inserted in front of the standard path. .. index:: single: Py_IsInitialized (C function) @@ -826,14 +863,17 @@ frequently used builds will be described in the remainder of this section. Compiling the interpreter with the :c:macro:`!Py_DEBUG` macro defined produces what is generally meant by :ref:`a debug build of Python `. -:c:macro:`!Py_DEBUG` is enabled in the Unix build by adding -:option:`--with-pydebug` to the :file:`./configure` command. -It is also implied by the presence of the -not-Python-specific :c:macro:`!_DEBUG` macro. When :c:macro:`!Py_DEBUG` is enabled -in the Unix build, compiler optimization is disabled. + +On Unix, :c:macro:`!Py_DEBUG` can be enabled by adding :option:`--with-pydebug` +to the :file:`./configure` command. This will also disable compiler optimization. + +On Windows, selecting a debug build (e.g., by passing the :option:`-d` option to +:file:`PCbuild/build.bat`) automatically enables :c:macro:`!Py_DEBUG`. +Additionally, the presence of the not-Python-specific :c:macro:`!_DEBUG` macro, +when defined by the compiler, will also implicitly enable :c:macro:`!Py_DEBUG`. In addition to the reference count debugging described below, extra checks are -performed, see :ref:`Python Debug Build `. +performed. See :ref:`Python Debug Build ` for more details. Defining :c:macro:`Py_TRACE_REFS` enables reference tracing (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). @@ -844,3 +884,41 @@ after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. + + +.. _c-api-tools: + +Recommended third party tools +============================= + +The following third party tools offer both simpler and more sophisticated +approaches to creating C, C++ and Rust extensions for Python: + +* `Cython `_ +* `cffi `_ +* `HPy `_ +* `nanobind `_ (C++) +* `Numba `_ +* `pybind11 `_ (C++) +* `PyO3 `_ (Rust) +* `SWIG `_ + +Using tools such as these can help avoid writing code that is tightly bound to +a particular version of CPython, avoid reference counting errors, and focus +more on your own code than on using the CPython API. In general, new versions +of Python can be supported by updating the tool, and your code will often use +newer and more efficient APIs automatically. Some tools also support compiling +for other implementations of Python from a single set of sources. + +These projects are not supported by the same people who maintain Python, and +issues need to be raised with the projects directly. Remember to check that the +project is still maintained and supported, as the list above may become +outdated. + +.. seealso:: + + `Python Packaging User Guide: Binary Extensions `_ + The Python Packaging User Guide not only covers several available + tools that simplify the creation of binary extensions, but also + discusses the various reasons why creating an extension module may be + desirable in the first place. diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index bf9df62c6f1..6cfd24c5ae6 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -54,6 +54,6 @@ There are two functions specifically for working with iterators. - ``PYGEN_RETURN`` if iterator returns. Return value is returned via *presult*. - ``PYGEN_NEXT`` if iterator yields. Yielded value is returned via *presult*. - - ``PYGEN_ERROR`` if iterator has raised and exception. *presult* is set to ``NULL``. + - ``PYGEN_ERROR`` if iterator has raised an exception. *presult* is set to ``NULL``. .. versionadded:: 3.10 diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 6b7ba8c9979..bfbfe3c9279 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -50,3 +50,72 @@ sentinel value is returned. callable object that can be called with no parameters; each call to it should return the next item in the iteration. When *callable* returns a value equal to *sentinel*, the iteration will be terminated. + + +Range Objects +^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyRange_Type + + The type object for :class:`range` objects. + + +.. c:function:: int PyRange_Check(PyObject *o) + + Return true if the object *o* is an instance of a :class:`range` object. + This function always succeeds. + + +Builtin Iterator Types +^^^^^^^^^^^^^^^^^^^^^^ + +These are built-in iteration types that are included in Python's C API, but +provide no additional functions. They are here for completeness. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * C type + * Python type + * * .. c:var:: PyTypeObject PyEnum_Type + * :py:class:`enumerate` + * * .. c:var:: PyTypeObject PyFilter_Type + * :py:class:`filter` + * * .. c:var:: PyTypeObject PyMap_Type + * :py:class:`map` + * * .. c:var:: PyTypeObject PyReversed_Type + * :py:class:`reversed` + * * .. c:var:: PyTypeObject PyZip_Type + * :py:class:`zip` + + +Other Iterator Objects +^^^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyByteArrayIter_Type +.. c:var:: PyTypeObject PyBytesIter_Type +.. c:var:: PyTypeObject PyListIter_Type +.. c:var:: PyTypeObject PyListRevIter_Type +.. c:var:: PyTypeObject PySetIter_Type +.. c:var:: PyTypeObject PyTupleIter_Type +.. c:var:: PyTypeObject PyRangeIter_Type +.. c:var:: PyTypeObject PyLongRangeIter_Type +.. c:var:: PyTypeObject PyDictIterKey_Type +.. c:var:: PyTypeObject PyDictRevIterKey_Type +.. c:var:: PyTypeObject PyDictIterValue_Type +.. c:var:: PyTypeObject PyDictRevIterValue_Type +.. c:var:: PyTypeObject PyDictIterItem_Type +.. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type + + Type objects for iterators of various built-in objects. + + Do not create instances of these directly; prefer calling + :c:func:`PyObject_GetIter` instead. + + Note that there is no guarantee that a given built-in type uses a given iterator + type. For example, iterating over :class:`range` will use one of two iterator + types depending on the size of the range. Other types may start using a + similar scheme in the future, without warning. diff --git a/Doc/c-api/lifecycle.dot b/Doc/c-api/lifecycle.dot new file mode 100644 index 00000000000..dca9f87e9e0 --- /dev/null +++ b/Doc/c-api/lifecycle.dot @@ -0,0 +1,156 @@ +digraph "Life Events" { + graph [ + fontnames="svg" + fontsize=12.0 + id="life_events_graph" + layout="dot" + margin="0,0" + ranksep=0.25 + stylesheet="lifecycle.dot.css" + ] + node [ + fontname="Courier" + fontsize=12.0 + ] + edge [ + fontname="Times-Italic" + fontsize=12.0 + ] + + "start" [fontname="Times-Italic" shape=plain label=< start > style=invis] + { + rank="same" + "tp_new" [href="typeobj.html#c.PyTypeObject.tp_new" target="_top"] + "tp_alloc" [href="typeobj.html#c.PyTypeObject.tp_alloc" target="_top"] + } + "tp_init" [href="typeobj.html#c.PyTypeObject.tp_init" target="_top"] + "reachable" [fontname="Times-Italic" shape=box] + "tp_traverse" [ + href="typeobj.html#c.PyTypeObject.tp_traverse" + ordering="in" + target="_top" + ] + "finalized?" [ + fontname="Times-Italic" + label=finalized?> + ordering="in" + shape=diamond + tooltip="marked as finalized?" + ] + "tp_finalize" [ + href="typeobj.html#c.PyTypeObject.tp_finalize" + ordering="in" + target="_top" + ] + "tp_clear" [href="typeobj.html#c.PyTypeObject.tp_clear" target="_top"] + "uncollectable" [ + fontname="Times-Italic" + label=(leaked)> + shape=box + tooltip="uncollectable (leaked)" + ] + "tp_dealloc" [ + href="typeobj.html#c.PyTypeObject.tp_dealloc" + ordering="in" + target="_top" + ] + "tp_free" [href="typeobj.html#c.PyTypeObject.tp_free" target="_top"] + + "start" -> "tp_new" [ + label=< type call > + ] + "tp_new" -> "tp_alloc" [ + label=< direct call > arrowhead=empty + labeltooltip="tp_new to tp_alloc: direct call" + tooltip="tp_new to tp_alloc: direct call" + ] + "tp_new" -> "tp_init" [tooltip="tp_new to tp_init"] + "tp_init" -> "reachable" [tooltip="tp_init to reachable"] + "reachable" -> "tp_traverse" [ + dir="back" + label=< not in a
cyclic
isolate > + labeltooltip="tp_traverse to reachable: not in a cyclic isolate" + tooltip="tp_traverse to reachable: not in a cyclic isolate" + ] + "reachable" -> "tp_traverse" [ + label=< periodic
cyclic isolate
detection > + labeltooltip="reachable to tp_traverse: periodic cyclic isolate detection" + tooltip="reachable to tp_traverse: periodic cyclic isolate detection" + ] + "reachable" -> "tp_init" [tooltip="reachable to tp_init"] + "reachable" -> "tp_finalize" [ + dir="back" + label=< resurrected
(maybe remove
finalized mark) > + labeltooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + tooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + ] + "tp_traverse" -> "finalized?" [ + label=< cyclic
isolate > + labeltooltip="tp_traverse to finalized?: cyclic isolate" + tooltip="tp_traverse to finalized?: cyclic isolate" + ] + "reachable" -> "finalized?" [ + label=< no refs > + labeltooltip="reachable to finalized?: no refs" + tooltip="reachable to finalized?: no refs" + ] + "finalized?" -> "tp_finalize" [ + label=< no (mark
as finalized) > + labeltooltip="finalized? to tp_finalize: no (mark as finalized)" + tooltip="finalized? to tp_finalize: no (mark as finalized)" + ] + "finalized?" -> "tp_clear" [ + label=< yes > + labeltooltip="finalized? to tp_clear: yes" + tooltip="finalized? to tp_clear: yes" + ] + "tp_finalize" -> "tp_clear" [ + label=< no refs or
cyclic isolate > + labeltooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + tooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + ] + "tp_finalize" -> "tp_dealloc" [ + arrowtail=empty + dir="back" + href="lifecycle.html#c.PyObject_CallFinalizerFromDealloc" + style=dashed + label=< recommended
call (see
explanation)> + labeltooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + target="_top" + tooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + ] + "tp_finalize" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_finalize to tp_dealloc: no refs" + tooltip="tp_finalize to tp_dealloc: no refs" + ] + "tp_clear" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_clear to tp_dealloc: no refs" + tooltip="tp_clear to tp_dealloc: no refs" + ] + "tp_clear" -> "uncollectable" [ + label=< cyclic
isolate > + labeltooltip="tp_clear to uncollectable: cyclic isolate" + tooltip="tp_clear to uncollectable: cyclic isolate" + ] + "uncollectable" -> "tp_dealloc" [ + style=invis + tooltip="uncollectable to tp_dealloc" + ] + "reachable" -> "uncollectable" [ + label=< cyclic
isolate
(no GC
support) > + labeltooltip="reachable to uncollectable: cyclic isolate (no GC support)" + tooltip="reachable to uncollectable: cyclic isolate (no GC support)" + ] + "reachable" -> "tp_dealloc" [ + label=< no refs> + labeltooltip="reachable to tp_dealloc: no refs" + ] + "tp_dealloc" -> "tp_free" [ + arrowhead=empty + label=< direct call > + labeltooltip="tp_dealloc to tp_free: direct call" + tooltip="tp_dealloc to tp_free: direct call" + ] +} diff --git a/Doc/c-api/lifecycle.dot.css b/Doc/c-api/lifecycle.dot.css new file mode 100644 index 00000000000..3abf95b74da --- /dev/null +++ b/Doc/c-api/lifecycle.dot.css @@ -0,0 +1,21 @@ +#life_events_graph { + --svg-fgcolor: currentcolor; + --svg-bgcolor: transparent; +} +#life_events_graph a { + color: inherit; +} +#life_events_graph [stroke="black"] { + stroke: var(--svg-fgcolor); +} +#life_events_graph text, +#life_events_graph [fill="black"] { + fill: var(--svg-fgcolor); +} +#life_events_graph [fill="white"] { + fill: var(--svg-bgcolor); +} +#life_events_graph [fill="none"] { + /* On links, setting fill will make the entire shape clickable */ + fill: var(--svg-bgcolor); +} diff --git a/Doc/c-api/lifecycle.dot.pdf b/Doc/c-api/lifecycle.dot.pdf new file mode 100644 index 00000000000..ed5b5039c83 Binary files /dev/null and b/Doc/c-api/lifecycle.dot.pdf differ diff --git a/Doc/c-api/lifecycle.dot.svg b/Doc/c-api/lifecycle.dot.svg new file mode 100644 index 00000000000..7ace27dfcba --- /dev/null +++ b/Doc/c-api/lifecycle.dot.svg @@ -0,0 +1,374 @@ + + + + + + + +Life Events + + + + +tp_new + + +tp_new + + + + + +start->tp_new + + + + + + +    type call   + + + + + +tp_alloc + + +tp_alloc + + + + + +tp_new->tp_alloc + + + + + + +  direct call   + + + + + +tp_init + + +tp_init + + + + + +tp_new->tp_init + + + + + + + + +reachable + +reachable + + + +tp_init->reachable + + + + + + + + +reachable->tp_init + + + + + + + + +tp_traverse + + +tp_traverse + + + + + +reachable->tp_traverse + + + + + + +  not in a   +  cyclic   +  isolate   + + + + + +reachable->tp_traverse + + + + + + +  periodic   +  cyclic isolate    +  detection   + + + + + +finalized? + + +marked as +finalized? + + + + + +reachable->finalized? + + + + + + +  no refs   + + + + + +tp_finalize + + +tp_finalize + + + + + +reachable->tp_finalize + + + + + + +  resurrected   +  (maybe remove   +  finalized mark)   + + + + + +uncollectable + + +uncollectable +(leaked) + + + + + +reachable->uncollectable + + + + + + +  cyclic   +  isolate   +  (no GC   +  support)   + + + + + +tp_dealloc + + +tp_dealloc + + + + + +reachable->tp_dealloc + + + +  no refs + + + + + +tp_traverse->finalized? + + + + + + +  cyclic   +  isolate   + + + + + +finalized?->tp_finalize + + + + + + +  no (mark   +  as finalized)   + + + + + +tp_clear + + +tp_clear + + + + + +finalized?->tp_clear + + + + + + +  yes   + + + + + +tp_finalize->tp_clear + + + + + + +  no refs or    +  cyclic isolate   + + + + + +tp_finalize->tp_dealloc + + + + + + +  recommended +  call (see +  explanation) + + + + + +tp_finalize->tp_dealloc + + + + + + +   no refs   + + + + + +tp_clear->uncollectable + + + + + + +  cyclic   +  isolate   + + + + + +tp_clear->tp_dealloc + + + + + + +  no refs   + + + + + + +tp_free + + +tp_free + + + + + +tp_dealloc->tp_free + + + + + + +    direct call   + + + + + diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst new file mode 100644 index 00000000000..5a170862a26 --- /dev/null +++ b/Doc/c-api/lifecycle.rst @@ -0,0 +1,271 @@ +.. highlight:: c + +.. _life-cycle: + +Object Life Cycle +================= + +This section explains how a type's slots relate to each other throughout the +life of an object. It is not intended to be a complete canonical reference for +the slots; instead, refer to the slot-specific documentation in +:ref:`type-structs` for details about a particular slot. + + +Life Events +----------- + +The figure below illustrates the order of events that can occur throughout an +object's life. An arrow from *A* to *B* indicates that event *B* can occur +after event *A* has occurred, with the arrow's label indicating the condition +that must be true for *B* to occur after *A*. + +.. only:: html and not epub + + .. raw:: html + + + + .. raw:: html + :file: lifecycle.dot.svg + + .. raw:: html + + + +.. only:: epub or not (html or latex) + + .. image:: lifecycle.dot.svg + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail below. + +.. only:: latex + + .. image:: lifecycle.dot.pdf + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail below. + +.. container:: + :name: life-events-graph-description + + Explanation: + + * When a new object is constructed by calling its type: + + #. :c:member:`~PyTypeObject.tp_new` is called to create a new object. + #. :c:member:`~PyTypeObject.tp_alloc` is directly called by + :c:member:`~PyTypeObject.tp_new` to allocate the memory for the new + object. + #. :c:member:`~PyTypeObject.tp_init` initializes the newly created object. + :c:member:`!tp_init` can be called again to re-initialize an object, if + desired. The :c:member:`!tp_init` call can also be skipped entirely, + for example by Python code calling :py:meth:`~object.__new__`. + + * After :c:member:`!tp_init` completes, the object is ready to use. + * Some time after the last reference to an object is removed: + + #. If an object is not marked as *finalized*, it might be finalized by + marking it as *finalized* and calling its + :c:member:`~PyTypeObject.tp_finalize` function. Python does + *not* finalize an object when the last reference to it is deleted; use + :c:func:`PyObject_CallFinalizerFromDealloc` to ensure that + :c:member:`~PyTypeObject.tp_finalize` is always called. + #. If the object is marked as finalized, + :c:member:`~PyTypeObject.tp_clear` might be called by the garbage collector + to clear references held by the object. It is *not* called when the + object's reference count reaches zero. + #. :c:member:`~PyTypeObject.tp_dealloc` is called to destroy the object. + To avoid code duplication, :c:member:`~PyTypeObject.tp_dealloc` typically + calls into :c:member:`~PyTypeObject.tp_clear` to free up the object's + references. + #. When :c:member:`~PyTypeObject.tp_dealloc` finishes object destruction, + it directly calls :c:member:`~PyTypeObject.tp_free` (usually set to + :c:func:`PyObject_Free` or :c:func:`PyObject_GC_Del` automatically as + appropriate for the type) to deallocate the memory. + + * The :c:member:`~PyTypeObject.tp_finalize` function is permitted to add a + reference to the object if desired. If it does, the object is + *resurrected*, preventing its pending destruction. (Only + :c:member:`!tp_finalize` is allowed to resurrect an object; + :c:member:`~PyTypeObject.tp_clear` and + :c:member:`~PyTypeObject.tp_dealloc` cannot without calling into + :c:member:`!tp_finalize`.) Resurrecting an object may + or may not cause the object's *finalized* mark to be removed. Currently, + Python does not remove the *finalized* mark from a resurrected object if + it supports garbage collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set) but does remove the mark if the object does not support + garbage collection; either or both of these behaviors may change in the + future. + * :c:member:`~PyTypeObject.tp_dealloc` can optionally call + :c:member:`~PyTypeObject.tp_finalize` via + :c:func:`PyObject_CallFinalizerFromDealloc` if it wishes to reuse that + code to help with object destruction. This is recommended because it + guarantees that :c:member:`!tp_finalize` is always called before + destruction. See the :c:member:`~PyTypeObject.tp_dealloc` documentation + for example code. + * If the object is a member of a :term:`cyclic isolate` and either + :c:member:`~PyTypeObject.tp_clear` fails to break the reference cycle or + the cyclic isolate is not detected (perhaps :func:`gc.disable` was called, + or the :c:macro:`Py_TPFLAGS_HAVE_GC` flag was erroneously omitted in one + of the involved types), the objects remain indefinitely uncollectable + (they "leak"). See :data:`gc.garbage`. + + If the object is marked as supporting garbage collection (the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`), the following events are also possible: + + * The garbage collector occasionally calls + :c:member:`~PyTypeObject.tp_traverse` to identify :term:`cyclic isolates + `. + * When the garbage collector discovers a :term:`cyclic isolate`, it + finalizes one of the objects in the group by marking it as *finalized* and + calling its :c:member:`~PyTypeObject.tp_finalize` function, if it has one. + This repeats until the cyclic isolate doesn't exist or all of the objects + have been finalized. + * :c:member:`~PyTypeObject.tp_finalize` is permitted to resurrect the object + by adding a reference from outside the :term:`cyclic isolate`. The new + reference causes the group of objects to no longer form a cyclic isolate + (the reference cycle may still exist, but if it does the objects are no + longer isolated). + * When the garbage collector discovers a :term:`cyclic isolate` and all of + the objects in the group have already been marked as *finalized*, the + garbage collector clears one or more of the uncleared objects in the group + (possibly concurrently) by calling each's + :c:member:`~PyTypeObject.tp_clear` function. This repeats as long as the + cyclic isolate still exists and not all of the objects have been cleared. + + +Cyclic Isolate Destruction +-------------------------- + +Listed below are the stages of life of a hypothetical :term:`cyclic isolate` +that continues to exist after each member object is finalized or cleared. It +is a memory leak if a cyclic isolate progresses through all of these stages; it should +vanish once all objects are cleared, if not sooner. A cyclic isolate can +vanish either because the reference cycle is broken or because the objects are +no longer isolated due to finalizer resurrection (see +:c:member:`~PyTypeObject.tp_finalize`). + +0. **Reachable** (not yet a cyclic isolate): All objects are in their normal, + reachable state. A reference cycle could exist, but an external reference + means the objects are not yet isolated. +#. **Unreachable but consistent:** The final reference from outside the cyclic + group of objects has been removed, causing the objects to become isolated + (thus a cyclic isolate is born). None of the group's objects have been + finalized or cleared yet. The cyclic isolate remains at this stage until + some future run of the garbage collector (not necessarily the next run + because the next run might not scan every object). +#. **Mix of finalized and not finalized:** Objects in a cyclic isolate are + finalized one at a time, which means that there is a period of time when the + cyclic isolate is composed of a mix of finalized and non-finalized objects. + Finalization order is unspecified, so it can appear random. A finalized + object must behave in a sane manner when non-finalized objects interact with + it, and a non-finalized object must be able to tolerate the finalization of + an arbitrary subset of its referents. +#. **All finalized:** All objects in a cyclic isolate are finalized before any + of them are cleared. +#. **Mix of finalized and cleared:** The objects can be cleared serially or + concurrently (but with the :term:`GIL` held); either way, some will finish + before others. A finalized object must be able to tolerate the clearing of + a subset of its referents. :pep:`442` calls this stage "cyclic trash". +#. **Leaked:** If a cyclic isolate still exists after all objects in the group + have been finalized and cleared, then the objects remain indefinitely + uncollectable (see :data:`gc.garbage`). It is a bug if a cyclic isolate + reaches this stage---it means the :c:member:`~PyTypeObject.tp_clear` methods + of the participating objects have failed to break the reference cycle as + required. + +If :c:member:`~PyTypeObject.tp_clear` did not exist, then Python would have no +way to safely break a reference cycle. Simply destroying an object in a cyclic +isolate would result in a dangling pointer, triggering undefined behavior when +an object referencing the destroyed object is itself destroyed. The clearing +step makes object destruction a two-phase process: first +:c:member:`~PyTypeObject.tp_clear` is called to partially destroy the objects +enough to detangle them from each other, then +:c:member:`~PyTypeObject.tp_dealloc` is called to complete the destruction. + +Unlike clearing, finalization is not a phase of destruction. A finalized +object must still behave properly by continuing to fulfill its design +contracts. An object's finalizer is allowed to execute arbitrary Python code, +and is even allowed to prevent the impending destruction by adding a reference. +The finalizer is only related to destruction by call order---if it runs, it runs +before destruction, which starts with :c:member:`~PyTypeObject.tp_clear` (if +called) and concludes with :c:member:`~PyTypeObject.tp_dealloc`. + +The finalization step is not necessary to safely reclaim the objects in a +cyclic isolate, but its existence makes it easier to design types that behave +in a sane manner when objects are cleared. Clearing an object might +necessarily leave it in a broken, partially destroyed state---it might be +unsafe to call any of the cleared object's methods or access any of its +attributes. With finalization, only finalized objects can possibly interact +with cleared objects; non-finalized objects are guaranteed to interact with +only non-cleared (but potentially finalized) objects. + +To summarize the possible interactions: + +* A non-finalized object might have references to or from non-finalized and + finalized objects, but not to or from cleared objects. +* A finalized object might have references to or from non-finalized, finalized, + and cleared objects. +* A cleared object might have references to or from finalized and cleared + objects, but not to or from non-finalized objects. + +Without any reference cycles, an object can be simply destroyed once its last +reference is deleted; the finalization and clearing steps are not necessary to +safely reclaim unused objects. However, it can be useful to automatically call +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` +before destruction anyway because type design is simplified when all objects +always experience the same series of events regardless of whether they +participated in a cyclic isolate. Python currently only calls +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` as +needed to destroy a cyclic isolate; this may change in a future version. + + +Functions +--------- + +To allocate and free memory, see :ref:`allocating-objects`. + + +.. c:function:: void PyObject_CallFinalizer(PyObject *op) + + Finalizes the object as described in :c:member:`~PyTypeObject.tp_finalize`. + Call this function (or :c:func:`PyObject_CallFinalizerFromDealloc`) instead + of calling :c:member:`~PyTypeObject.tp_finalize` directly because this + function may deduplicate multiple calls to :c:member:`!tp_finalize`. + Currently, calls are only deduplicated if the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may + change in the future. + + +.. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op) + + Same as :c:func:`PyObject_CallFinalizer` but meant to be called at the + beginning of the object's destructor (:c:member:`~PyTypeObject.tp_dealloc`). + There must not be any references to the object. If the object's finalizer + resurrects the object, this function returns -1; no further destruction + should happen. Otherwise, this function returns 0 and destruction can + continue normally. + + .. seealso:: + + :c:member:`~PyTypeObject.tp_dealloc` for example code. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 25d9e62e387..ed34efe716d 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -40,9 +40,11 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Return a new :c:type:`PyLongObject` object from *v*, or ``NULL`` on failure. - The current implementation keeps an array of integer objects for all integers - between ``-5`` and ``256``. When you create an int in that range you actually - just get back a reference to the existing object. + .. impl-detail:: + + CPython keeps an array of integer objects for all integers + between ``-5`` and ``1024``. When you create an int in that range + you actually just get back a reference to the existing object. .. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) @@ -159,6 +161,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_FromPid(pid) + + Macro for creating a Python integer from a process identifier. + + This can be defined as an alias to :c:func:`PyLong_FromLong` or + :c:func:`PyLong_FromLongLong`, depending on the size of the system's + PID type. + + .. versionadded:: 3.2 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -372,6 +385,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` representation of *obj*. + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + If the *obj* value is out of range, raise an :exc:`OverflowError`. Set *\*value* and return ``0`` on success. @@ -439,7 +456,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. All *n_bytes* of the buffer are written: large buffers are padded with zeroes. - If the returned value is greater than than *n_bytes*, the value was + If the returned value is greater than *n_bytes*, the value was truncated: as many of the lowest bits of the value as could fit are written, and the higher bits are ignored. This matches the typical behavior of a C-style downcast. @@ -569,6 +586,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_AsPid(pid) + + Macro for converting a Python integer into a process identifier. + + This can be defined as an alias to :c:func:`PyLong_AsLong`, + :c:func:`PyLong_FromLongLong`, or :c:func:`PyLong_AsInt`, depending on the + size of the system's PID type. + + .. versionadded:: 3.2 + + .. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) Get the sign of the integer object *obj*. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 1f55c0aa955..2476ebb9b69 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -102,7 +102,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions which occur when this calls :meth:`~object.__getitem__` + Exceptions which occur when this calls the :meth:`~object.__getitem__` method are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. @@ -116,7 +116,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions that occur when this calls :meth:`~object.__getitem__` + Exceptions that occur when this calls the :meth:`~object.__getitem__` method or while creating the temporary :class:`str` object are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 61218a1bf6f..668a163b2df 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -82,7 +82,7 @@ The following functions allow marshalled values to be read back in. assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading + file. Only use this variant if you are certain that you won't be reading anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 64ae35daa70..23958980102 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -102,7 +102,7 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. +table can be found at :ref:`default-memory-allocators`. The APIs used to allocate and free a block of memory must be from the same domain. For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. @@ -376,6 +376,24 @@ The :ref:`default object allocator ` uses the If *p* is ``NULL``, no operation is performed. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_GC_New` or + :c:macro:`PyObject_GC_NewVar`; use :c:func:`PyObject_GC_Del` instead. + + .. seealso:: + + * :c:func:`PyObject_GC_Del` is the equivalent of this function for memory + allocated by types that support garbage collection. + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:macro:`PyObject_New` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. _default-memory-allocators: @@ -654,6 +672,10 @@ This allocator is disabled if Python is configured with the :option:`--without-pymalloc` option. It can also be disabled at runtime using the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). +Typically, it makes sense to disable the pymalloc allocator when building +Python with AddressSanitizer (:option:`--with-address-sanitizer`) which helps +uncover low level bugs within the C code. + Customize pymalloc Arena Allocator ---------------------------------- diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index f6038032805..e4ac8b57673 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -13,6 +13,12 @@ A :class:`memoryview` object exposes the C level :ref:`buffer interface any other object. +.. c:var:: PyTypeObject PyMemoryView_Type + + This instance of :c:type:`PyTypeObject` represents the Python memoryview + type. This is the same object as :class:`memoryview` in the Python layer. + + .. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) Create a memoryview object from an object that provides the buffer interface. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index f7f4d37d4c7..a12f6331c85 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -3,17 +3,16 @@ .. _moduleobjects: Module Objects --------------- +============== .. index:: pair: object; module - .. c:var:: PyTypeObject PyModule_Type .. index:: single: ModuleType (in module types) This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. + is exposed to Python programs as :py:class:`types.ModuleType`. .. c:function:: int PyModule_Check(PyObject *p) @@ -71,6 +70,9 @@ Module Objects ``PyObject_*`` functions rather than directly manipulate a module's :attr:`~object.__dict__`. + The returned reference is borrowed from the module; it is valid until + the module is destroyed. + .. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) @@ -90,18 +92,19 @@ Module Objects Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to ``'utf-8'``. -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or ``NULL``. See - :c:member:`PyModuleDef.m_size`. - + The returned buffer is only valid until the module is renamed or destroyed. + Note that Python code may rename a module by setting its :py:attr:`~module.__name__` + attribute. .. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) Return a pointer to the :c:type:`PyModuleDef` struct from which the module was created, or ``NULL`` if the module wasn't created from a definition. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from a missing + :c:type:`!PyModuleDef`. + .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) @@ -122,273 +125,116 @@ Module Objects Similar to :c:func:`PyModule_GetFilenameObject` but return the filename encoded to 'utf-8'. + The returned buffer is only valid until the module's :py:attr:`~module.__file__` attribute + is reassigned or the module is destroyed. + .. deprecated:: 3.2 :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _initializing-modules: +.. _pymoduledef_slot: -Initializing C modules -^^^^^^^^^^^^^^^^^^^^^^ +Module definition +----------------- -Modules objects are usually created from extension modules (shared libraries -which export an initialization function), or compiled-in modules -(where the initialization function is added using :c:func:`PyImport_AppendInittab`). -See :ref:`building` or :ref:`extending-with-embedding` for details. +Modules created using the C API are typically defined using an +array of :dfn:`slots`. +The slots provide a "description" of how a module should be created. -The initialization function can either pass a module definition instance -to :c:func:`PyModule_Create`, and return the resulting module object, -or request "multi-phase initialization" by returning the definition struct itself. +.. versionchanged:: next -.. c:type:: PyModuleDef + Previously, a :c:type:`PyModuleDef` struct was necessary to define modules. + The older way of defining modules is still available: consult either the + :ref:`pymoduledef` section or earlier versions of this documentation + if you plan to support earlier Python versions. - The module definition struct, which holds all information needed to create - a module object. There is usually only one statically initialized variable - of this type for each module. +The slots array is usually used to define an extension module's “main” +module object (see :ref:`extension-modules` for details). +It can also be used to +:ref:`create extension modules dynamically `. - .. c:member:: PyModuleDef_Base m_base +Unless specified otherwise, the same slot ID may not be repeated +in an array of slots. - Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. - - .. c:member:: const char *m_name - - Name for the new module. - - .. c:member:: const char *m_doc - - Docstring for the module; usually a docstring variable created with - :c:macro:`PyDoc_STRVAR` is used. - - .. c:member:: Py_ssize_t m_size - - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. - - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`~PyModuleDef.m_free` function has been called, if present. - - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. Non-negative ``m_size`` is required for multi-phase - initialization. - - See :PEP:`3121` for more details. - - .. c:member:: PyMethodDef* m_methods - - A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. - - .. c:member:: PyModuleDef_Slot* m_slots - - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be ``NULL``. - - .. versionchanged:: 3.5 - - Prior to version 3.5, this member was always set to ``NULL``, - and was defined as: - - .. c:member:: inquiry m_reload - - .. c:member:: traverseproc m_traverse - - A traversal function to call during GC traversal of the module object, or - ``NULL`` if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - - .. c:member:: inquiry m_clear - - A clear function to call during GC clearing of the module object, or - ``NULL`` if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* - called before a module is deallocated. For example, when reference - counting is enough to determine that an object is no longer used, - the cyclic garbage collector is not involved and - :c:member:`~PyModuleDef.m_free` is called directly. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - - .. c:member:: freefunc m_free - - A function to call during deallocation of the module object, or ``NULL`` - if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - -Single-phase initialization -........................... - -The module initialization function may create and return the module object -directly. This is referred to as "single-phase initialization", and uses one -of the following two module creation functions: - -.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) - - Create a new module object, given the definition in *def*. This behaves - like :c:func:`PyModule_Create2` with *module_api_version* set to - :c:macro:`PYTHON_API_VERSION`. - - -.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) - - Create a new module object, given the definition in *def*, assuming the - API version *module_api_version*. If that version does not match the version - of the running interpreter, a :exc:`RuntimeWarning` is emitted. - - Return ``NULL`` with an exception set on error. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_Create` - instead; only use this if you are sure you need it. - -Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObjectRef`. - -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -An alternate way to specify extensions is to request "multi-phase initialization". -Extension modules created this way behave more like Python modules: the -initialization is split between the *creation phase*, when the module object -is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`!__new__` and :py:meth:`!__init__` methods -of classes. - -Unlike modules created using single-phase initialization, these modules are not -singletons: if the *sys.modules* entry is removed and the module is re-imported, -a new module object is created, and the old module is subject to normal garbage -collection -- as with Python modules. -By default, multiple modules created from the same definition should be -independent: changes to one should not affect the others. -This means that all state should be specific to the module object (using e.g. -using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`~object.__dict__` or individual classes created with :c:func:`PyType_FromSpec`). - -All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters `. Making sure multiple modules -are independent is typically enough to achieve this. - -To request multi-phase initialization, the initialization function -(PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty -:c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` -instance must be initialized with the following function: - -.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) - - Ensures a module definition is a properly initialized Python object that - correctly reports its type and reference count. - - Returns *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. - - .. versionadded:: 3.5 - -The *m_slots* member of the module definition must point to an array of -``PyModuleDef_Slot`` structures: .. c:type:: PyModuleDef_Slot .. c:member:: int slot - A slot ID, chosen from the available values explained below. + A slot ID, chosen from the available ``Py_mod_*`` values explained below. + + An ID of 0 marks the end of a :c:type:`!PyModuleDef_Slot` array. .. c:member:: void* value Value of the slot, whose meaning depends on the slot ID. + The value may not be NULL. + To leave a slot out, omit the :c:type:`PyModuleDef_Slot` entry entirely. + .. versionadded:: 3.5 -The *m_slots* array must be terminated by a slot with id 0. -The available slot types are: +Metadata slots +.............. -.. c:macro:: Py_mod_create +.. c:macro:: Py_mod_name - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: + :c:type:`Slot ID ` for the name of the new module, + as a NUL-terminated UTF8-encoded ``const char *``. - .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) - :no-index-entry: - :no-contents-entry: + Note that modules are typically created using a + :py:class:`~importlib.machinery.ModuleSpec`, and when they are, the + name from the spec will be used instead of :c:data:`!Py_mod_name`. + However, it is still recommended to include this slot for introspection + and debugging purposes. - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error - and return ``NULL``. + .. versionadded:: next - This function should be kept minimal. In particular, it should not - call arbitrary Python code, as trying to import the same module again may - result in an infinite loop. + Use :c:member:`PyModuleDef.m_name` instead to support previous versions. - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. +.. c:macro:: Py_mod_doc - If ``Py_mod_create`` is not specified, the import machinery will create - a normal module object using :c:func:`PyModule_New`. The name is taken from - *spec*, not the definition, to allow extension modules to dynamically adjust - to their place in the module hierarchy and be imported under different - names through symlinks, all while sharing a single module definition. + :c:type:`Slot ID ` for the docstring of the new + module, as a NUL-terminated UTF8-encoded ``const char *``. - There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. + Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`. -.. c:macro:: Py_mod_exec + .. versionadded:: next - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. - The signature of the function is: + Use :c:member:`PyModuleDef.m_doc` instead to support previous versions. - .. c:function:: int exec_module(PyObject* module) - :no-index-entry: - :no-contents-entry: - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. +Feature slots +............. + +.. c:macro:: Py_mod_abi + + :c:type:`Slot ID ` whose value points to + a :c:struct:`PyABIInfo` structure describing the ABI that + the extension is using. + + A suitable :c:struct:`!PyABIInfo` variable can be defined using the + :c:macro:`PyABIInfo_VAR` macro, as in: + + .. code-block:: c + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot mymodule_slots[] = { + {Py_mod_abi, &abi_info}, + ... + }; + + When creating a module, Python checks the value of this slot + using :c:func:`PyABIInfo_Check`. + + .. versionadded:: 3.15 .. c:macro:: Py_mod_multiple_interpreters - Specifies one of the following values: + :c:type:`Slot ID ` whose value is one of: .. c:namespace:: NULL @@ -411,9 +257,6 @@ The available slot types are: This slot determines whether or not importing this module in a subinterpreter will fail. - Multiple ``Py_mod_multiple_interpreters`` slots may not be specified - in one module definition. - If ``Py_mod_multiple_interpreters`` is not specified, the import machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. @@ -421,7 +264,7 @@ The available slot types are: .. c:macro:: Py_mod_gil - Specifies one of the following values: + :c:type:`Slot ID ` whose value is one of: .. c:namespace:: NULL @@ -439,28 +282,514 @@ The available slot types are: this module will cause the GIL to be automatically enabled. See :ref:`whatsnew313-free-threaded-cpython` for more detail. - Multiple ``Py_mod_gil`` slots may not be specified in one module definition. - If ``Py_mod_gil`` is not specified, the import machinery defaults to ``Py_MOD_GIL_USED``. .. versionadded:: 3.13 -See :PEP:`489` for more details on multi-phase initialization. -Low-level module creation functions -................................... +Creation and initialization slots +................................. -The following functions are called under the hood when using multi-phase -initialization. They can be used directly, for example when creating module -objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and -``PyModule_ExecDef`` must be called to fully initialize a module. +.. c:macro:: Py_mod_create + + :c:type:`Slot ID ` for a function that creates + the module object itself. + The function must have the signature: + + .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) + :no-index-entry: + :no-contents-entry: + + The function will be called with: + + - *spec*: a ``ModuleSpec``-like object, meaning that any attributes defined + for :py:class:`importlib.machinery.ModuleSpec` have matching semantics. + However, any of the attributes may be missing. + - *def*: ``NULL``, or the module definition if the module is created from one. + + The function should return a new module object, or set an error + and return ``NULL``. + + This function should be kept minimal. In particular, it should not + call arbitrary Python code, as trying to import the same module again may + result in an infinite loop. + + If ``Py_mod_create`` is not specified, the import machinery will create + a normal module object using :c:func:`PyModule_New`. The name is taken from + *spec*, not the definition, to allow extension modules to dynamically adjust + to their place in the module hierarchy and be imported under different + names through symlinks, all while sharing a single module definition. + + There is no requirement for the returned object to be an instance of + :c:type:`PyModule_Type`. + However, some slots may only be used with + :c:type:`!PyModule_Type` instances; in particular: + + - :c:macro:`Py_mod_exec`, + - :ref:`module state slots ` (``Py_mod_state_*``), + - :c:macro:`Py_mod_token`. + + .. versionadded:: 3.5 + + .. versionchanged:: next + + The *slots* argument may be a ``ModuleSpec``-like object, rather than + a true :py:class:`~importlib.machinery.ModuleSpec` instance. + Note that previous versions of CPython did not enforce this. + + The *def* argument may now be ``NULL``, since modules are not necessarily + made from definitions. + +.. c:macro:: Py_mod_exec + + :c:type:`Slot ID ` for a function that will + :dfn:`execute`, or initialize, the module. + This function does the equivalent to executing the code of a Python module: + typically, it adds classes and constants to the module. + The signature of the function is: + + .. c:function:: int exec_module(PyObject* module) + :no-index-entry: + :no-contents-entry: + + See the :ref:`capi-module-support-functions` section for some useful + functions to call. + + For backwards compatibility, the :c:type:`PyModuleDef.m_slots` array may + contain multiple :c:macro:`!Py_mod_exec` slots; these are processed in the + order they appear in the array. + Elsewhere (that is, in arguments to :c:func:`PyModule_FromSlotsAndSpec` + and in return values of :samp:`PyModExport_{}`), repeating the slot + is not allowed. + + .. versionadded:: 3.5 + + .. versionchanged:: next + + Repeated ``Py_mod_exec`` slots are disallowed, except in + :c:type:`PyModuleDef.m_slots`. + +.. c:macro:: Py_mod_methods + + :c:type:`Slot ID ` for a table of module-level + functions, as an array of :c:type:`PyMethodDef` values suitable as the + *functions* argument to :c:func:`PyModule_AddFunctions`. + + Like other slot IDs, a slots array may only contain one + :c:macro:`!Py_mod_methods` entry. + To add functions from multiple :c:type:`PyMethodDef` arrays, call + :c:func:`PyModule_AddFunctions` in the :c:macro:`Py_mod_exec` function. + + The table must be statically allocated (or otherwise guaranteed to outlive + the module object). + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_methods` instead to support previous versions. + +.. _ext-module-state: + +Module state +------------ + +Extension modules can have *module state* -- a +piece of memory that is allocated on module creation, +and freed when the module object is deallocated. +The module state is specified using :ref:`dedicated slots `. + +A typical use of module state is storing an exception type -- or indeed *any* +type object defined by the module -- + +Unlike the module's Python attributes, Python code cannot replace or delete +data stored in module state. + +Keeping per-module information in attributes and module state, rather than in +static globals, makes module objects *isolated* and safer for use in +multiple sub-interpreters. +It also helps Python do an orderly clean-up when it shuts down. + +Extensions that keep references to Python objects as part of module state must +implement :c:macro:`Py_mod_state_traverse` and :c:macro:`Py_mod_state_clear` +functions to avoid reference leaks. + +To retrieve the state from a given module, use the following functions: + +.. c:function:: void* PyModule_GetState(PyObject *module) + + Return the "state" of the module, that is, a pointer to the block of memory + allocated at module creation time, or ``NULL``. See + :c:macro:`Py_mod_state_size`. + + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from missing + module state. + + +.. c:function:: int PyModule_GetStateSize(PyObject *, Py_ssize_t *result) + + Set *\*result* to the size of the module's state, as specified using + :c:macro:`Py_mod_state_size` (or :c:member:`PyModuleDef.m_size`), + and return 0. + + On error, set *\*result* to -1, and return -1 with an exception set. + + .. versionadded:: next + + + +.. _ext-module-state-slots: + +Slots for defining module state +............................... + +The following :c:member:`PyModuleDef_Slot.slot` IDs are available for +defining the module state. + +.. c:macro:: Py_mod_state_size + + :c:type:`Slot ID ` for the size of the module state, + in bytes. + + Setting the value to a non-negative value means that the module can be + re-initialized and specifies the additional amount of memory it requires + for its state. + + See :PEP:`3121` for more details. + + Use :c:func:`PyModule_GetStateSize` to retrieve the size of a given module. + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_traverse + + :c:type:`Slot ID ` for a traversal function to call + during GC traversal of the module object. + + The signature of the function, and meanings of the arguments, + is similar as for :c:member:`PyTypeObject.tp_traverse`: + + .. c:function:: int traverse_module_state(PyObject *module, visitproc visit, void *arg) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_clear + + :c:type:`Slot ID ` for a clear function to call + during GC clearing of the module object. + + The signature of the function is: + + .. c:function:: int clear_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* + called before a module is deallocated. For example, when reference + counting is enough to determine that an object is no longer used, + the cyclic garbage collector is not involved and + the :c:macro:`Py_mod_state_free` function is called directly. + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_clear` instead to support previous versions. + +.. c:macro:: Py_mod_state_free + + :c:type:`Slot ID ` for a function to call during + deallocation of the module object. + + The signature of the function is: + + .. c:function:: int free_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_free` instead to support previous versions. + + +.. _ext-module-token: + +Module token +............ + +Each module may have an associated *token*: a pointer-sized value intended to +identify of the module state's memory layout. +This means that if you have a module object, but you are not sure if it +“belongs” to your extension, you can check using code like this: + +.. code-block:: c + + PyObject *module = + + void *module_token; + if (PyModule_GetToken(module, &module_token) < 0) { + return NULL; + } + if (module_token != your_token) { + PyErr_SetString(PyExc_ValueError, "unexpected module") + return NULL; + } + + // This module's state has the expected memory layout; it's safe to cast + struct my_state state = (struct my_state*)PyModule_GetState(module) + +A module's token -- and the *your_token* value to use in the above code -- is: + +- For modules created with :c:type:`PyModuleDef`: the address of that + :c:type:`PyModuleDef`; +- For modules defined with the :c:macro:`Py_mod_token` slot: the value + of that slot; +- For modules created from an ``PyModExport_*`` + :ref:`export hook `: the slots array that the export + hook returned (unless overriden with :c:macro:`Py_mod_token`). + +.. c:macro:: Py_mod_token + + :c:type:`Slot ID ` for the module token. + + If you use this slot to set the module token (rather than rely on the + default), you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + * If the token points to a :c:type:`PyModuleDef` struct, the module should + behave as if it was created from that :c:type:`PyModuleDef`. + In particular, the module state must have matching layout and semantics. + + Modules created from :c:type:`PyModuleDef` allways use the address of + the :c:type:`PyModuleDef` as the token. + This means that :c:macro:`!Py_mod_token` cannot be used in + :c:member:`PyModuleDef.m_slots`. + + .. versionadded:: next + +.. c:function:: int PyModule_GetToken(PyObject *module, void** result) + + Set *\*result* to the module's token and return 0. + + On error, set *\*result* to NULL, and return -1 with an exception set. + + .. versionadded:: next + +See also :c:func:`PyType_GetModuleByToken`. + + +.. _module-from-slots: + +Creating extension modules dynamically +-------------------------------------- + +The following functions may be used to create an extension module dynamically, +rather than from an extension's :ref:`export hook `. + +.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) + + Create a new module object, given an array of :ref:`slots ` + and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. + + The *slots* argument must point to an array of :c:type:`PyModuleDef_Slot` + structures, terminated by an entry slot with slot ID of 0 + (typically written as ``{0}`` or ``{0, NULL}`` in C). + The *slots* argument may not be ``NULL``. + + The *spec* argument may be any ``ModuleSpec``-like object, as described + in :c:macro:`Py_mod_create` documentation. + Currently, the *spec* must have a ``name`` attribute. + + On success, return the new module. + On error, return ``NULL`` with an exception set. + + Note that this does not process the module's execution slot + (:c:data:`Py_mod_exec`). + Both :c:func:`!PyModule_FromSlotsAndSpec` and :c:func:`PyModule_Exec` + must be called to fully initialize a module. + (See also :ref:`multi-phase-initialization`.) + + The *slots* array only needs to be valid for the duration of the + :c:func:`!PyModule_FromSlotsAndSpec` call. + In particular, it may be heap-allocated. + + .. versionadded:: next + +.. c:function:: int PyModule_Exec(PyObject *module) + + Execute the :c:data:`Py_mod_exec` slot(s) of the given *module*. + + On success, return 0. + On error, return -1 with an exception set. + + For clarity: If *module* has no slots, for example if it uses + :ref:`legacy single-phase initialization `, + this function does nothing and returns 0. + + .. versionadded:: next + + + +.. _pymoduledef: + +Module definition struct +------------------------ + +Traditionally, extension modules were defined using a *module definition* +as the “description" of how a module should be created. +Rather than using an array of :ref:`slots ` directly, +the definition has dedicated members for most common functionality, +and allows additional slots as an extension mechanism. + +This way of defining modules is still available and there are no plans to +remove it. + +.. c:type:: PyModuleDef + + The module definition struct, which holds information needed to create + a module object. + + This structure must be statically allocated (or be otherwise guaranteed + to be valid while any modules created from it exist). + Usually, there is only one variable of this type for each extension module + defined this way. + + .. c:member:: PyModuleDef_Base m_base + + Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`: + + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Base + + The type of :c:member:`!PyModuleDef.m_base`. + + .. c:macro:: PyModuleDef_HEAD_INIT + + The required initial value for :c:member:`!PyModuleDef.m_base`. + + .. c:member:: const char *m_name + + Corresponds to the :c:macro:`Py_mod_name` slot. + + .. c:member:: const char *m_doc + + These members correspond to the :c:macro:`Py_mod_doc` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: Py_ssize_t m_size + + Corresponds to the :c:macro:`Py_mod_state_size` slot. + Setting this to zero is equivalent to omitting the slot. + + When using :ref:`legacy single-phase initialization ` + or when creating modules dynamically using :c:func:`PyModule_Create` + or :c:func:`PyModule_Create2`, :c:member:`!m_size` may be set to -1. + This indicates that the module does not support sub-interpreters, + because it has global state. + + .. c:member:: PyMethodDef *m_methods + + Corresponds to the :c:macro:`Py_mod_methods` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: PyModuleDef_Slot* m_slots + + An array of additional slots, terminated by a ``{0, NULL}`` entry. + + This array may not contain slots corresponding to :c:type:`PyModuleDef` + members. + For example, you cannot use :c:macro:`Py_mod_name` in :c:member:`!m_slots`; + the module name must be given as :c:member:`PyModuleDef.m_name`. + + .. versionchanged:: 3.5 + + Prior to version 3.5, this member was always set to ``NULL``, + and was defined as: + + .. c:member:: inquiry m_reload + + .. c:member:: traverseproc m_traverse + inquiry m_clear + freefunc m_free + + These members correspond to the :c:macro:`Py_mod_state_traverse`, + :c:macro:`Py_mod_state_clear`, and :c:macro:`Py_mod_state_free` slots, + respectively. + + Setting these members to NULL is equivalent to omitting the + corresponding slots. + + .. versionchanged:: 3.9 + + :c:member:`m_traverse`, :c:member:`m_clear` and :c:member:`m_free` + functions are longer called before the module state is allocated. + + +.. _moduledef-dynamic: + +The following API can be used to create modules from a :c:type:`!PyModuleDef` +struct: + +.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) + + Create a new module object, given the definition in *def*. + This is a macro that calls :c:func:`PyModule_Create2` with + *module_api_version* set to :c:macro:`PYTHON_API_VERSION`, or + to :c:macro:`PYTHON_ABI_VERSION` if using the + :ref:`limited API `. + +.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) + + Create a new module object, given the definition in *def*, assuming the + API version *module_api_version*. If that version does not match the version + of the running interpreter, a :exc:`RuntimeWarning` is emitted. + + Return ``NULL`` with an exception set on error. + + This function does not support slots. + The :c:member:`~PyModuleDef.m_slots` member of *def* must be ``NULL``. + + + .. note:: + + Most uses of this function should be using :c:func:`PyModule_Create` + instead; only use this if you are sure you need it. .. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - Create a new module object, given the definition in *def* and the - ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` - with *module_api_version* set to :c:macro:`PYTHON_API_VERSION`. + This macro calls :c:func:`PyModule_FromDefAndSpec2` with + *module_api_version* set to :c:macro:`PYTHON_API_VERSION`, or + to :c:macro:`PYTHON_ABI_VERSION` if using the + :ref:`limited API `. .. versionadded:: 3.5 @@ -473,6 +802,10 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and Return ``NULL`` with an exception set on error. + Note that this does not process execution slots (:c:data:`Py_mod_exec`). + Both ``PyModule_FromDefAndSpec`` and ``PyModule_ExecDef`` must be called + to fully initialize a module. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -486,35 +819,30 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. versionadded:: 3.5 -.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) +.. c:macro:: PYTHON_API_VERSION - Set the docstring for *module* to *docstring*. - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. + The C API version. Defined for backwards compatibility. - .. versionadded:: 3.5 + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. -.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) +.. c:macro:: PYTHON_ABI_VERSION - Add the functions from the ``NULL`` terminated *functions* array to *module*. - Refer to the :c:type:`PyMethodDef` documentation for details on individual - entries (due to the lack of a shared module namespace, module level - "functions" implemented in C typically receive the module as their first - parameter, making them similar to instance methods on Python classes). - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. + Defined as ``3`` for backwards compatibility. - .. versionadded:: 3.5 + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. + + +.. _capi-module-support-functions: Support functions -................. +----------------- -The module initialization function (if using single phase initialization) or -a function called from a module execution slot (if using multi-phase -initialization), can use the following functions to help initialize the module -state: +The following functions are provided to help initialize a module object. +They are intended for a module's execution slot (:c:data:`Py_mod_exec`), +the initialization function for legacy :ref:`single-phase initialization `, +or code that creates modules dynamically. .. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) @@ -663,12 +991,42 @@ state: .. versionadded:: 3.9 +.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) + + Add the functions from the ``NULL`` terminated *functions* array to *module*. + Refer to the :c:type:`PyMethodDef` documentation for details on individual + entries (due to the lack of a shared module namespace, module level + "functions" implemented in C typically receive the module as their first + parameter, making them similar to instance methods on Python classes). + + This function is called automatically when creating a module from + ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, + ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + Some module authors may prefer defining functions in multiple + :c:type:`PyMethodDef` arrays; in that case they should call this function + directly. + + The *functions* array must be statically allocated (or otherwise guaranteed + to outlive the module object). + + .. versionadded:: 3.5 + +.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) + + Set the docstring for *module* to *docstring*. + This function is called automatically when creating a module from + ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, + ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + + .. versionadded:: 3.5 + .. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) Indicate that *module* does or does not support running without the global interpreter lock (GIL), using one of the values from :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization - function. If this function is not called during module initialization, the + function when using :ref:`single-phase-initialization`. + If this function is not called during module initialization, the import machinery assumes the module does not support running without the GIL. This function is only available in Python builds configured with :option:`--disable-gil`. @@ -677,10 +1035,11 @@ state: .. versionadded:: 3.13 -Module lookup -^^^^^^^^^^^^^ +Module lookup (single-phase initialization) +........................................... -Single-phase initialization creates singleton modules that can be looked up +The legacy :ref:`single-phase initialization ` +initialization scheme creates singleton modules that can be looked up in the context of the current interpreter. This allows the module object to be retrieved later with only a reference to the module definition. @@ -701,7 +1060,8 @@ since multiple such modules can be created from a single definition. Only effective on modules created using single-phase initialization. - Python calls ``PyState_AddModule`` automatically after importing a module, + Python calls ``PyState_AddModule`` automatically after importing a module + that uses :ref:`single-phase initialization `, so it is unnecessary (but harmless) to call it from module initialization code. An explicit call is needed only if the module's own init code subsequently calls ``PyState_FindModule``. @@ -709,6 +1069,9 @@ since multiple such modules can be created from a single definition. mechanisms (either by calling it directly, or by referring to its implementation for details of the required state updates). + If a module was attached previously using the same *def*, it is replaced + by the new *module*. + The caller must have an :term:`attached thread state`. Return ``-1`` with an exception set on error, ``0`` on success. diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 7926148302a..b0227c2f4fa 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -136,7 +136,7 @@ Managing the Monitoring State ----------------------------- Monitoring states can be managed with the help of monitoring scopes. A scope -would typically correspond to a python function. +would typically correspond to a Python function. .. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 0fd159f1eb8..76971c46c16 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -73,7 +73,7 @@ Object Protocol Flag to be used with multiple functions that print the object (like :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). - If passed, these function would use the :func:`str` of the object + If passed, these functions use the :func:`str` of the object instead of the :func:`repr`. @@ -85,6 +85,35 @@ Object Protocol instead of the :func:`repr`. +.. c:function:: void PyUnstable_Object_Dump(PyObject *op) + + Dump an object *op* to ``stderr``. This should only be used for debugging. + + The output is intended to try dumping objects even after memory corruption: + + * Information is written starting with fields that are the least likely to + crash when accessed. + * This function can be called without an :term:`attached thread state`, but + it's not recommended to do so: it can cause deadlocks. + * An object that does not belong to the current interpreter may be dumped, + but this may also cause crashes or unintended behavior. + * Implement a heuristic to detect if the object memory has been freed. Don't + display the object contents in this case, only its memory address. + * The output format may change at any time. + + Example of output: + + .. code-block:: output + + object address : 0x7f80124702c0 + object refcount : 2 + object type : 0x9902e0 + object type name: str + object repr : 'abcdef' + + .. versionadded:: next + + .. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. @@ -197,6 +226,13 @@ Object Protocol in favour of using :c:func:`PyObject_DelAttr`, but there are currently no plans to remove it. + The function must not be called with a ``NULL`` *v* and an exception set. + This case can arise from forgetting ``NULL`` checks and would delete the + attribute. + + .. versionchanged:: 3.15 + Must not be called with NULL value if an exception is set. + .. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) @@ -207,6 +243,10 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. + The function must not be called with a ``NULL`` *v* and an exception set. + This case can arise from forgetting ``NULL`` checks and would delete the + attribute. + The number of different attribute names passed to this function should be kept small, usually by using a statically allocated string as *attr_name*. @@ -215,6 +255,10 @@ Object Protocol For more details, see :c:func:`PyUnicode_InternFromString`, which may be used internally to create a key object. + .. versionchanged:: 3.15 + Must not be called with NULL value if an exception is set. + + .. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) Generic attribute setter and deleter function that is meant @@ -585,7 +629,7 @@ Object Protocol Clear the managed dictionary of *obj*. - This function must only be called in a traverse function of the type which + This function must only be called in a clear function of the type which has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. .. versionadded:: 3.13 @@ -596,12 +640,13 @@ Object Protocol if supported by the runtime. In the :term:`free-threaded ` build, this allows the interpreter to avoid reference count adjustments to *obj*, which may improve multi-threaded performance. The tradeoff is - that *obj* will only be deallocated by the tracing garbage collector. + that *obj* will only be deallocated by the tracing garbage collector, and + not when the interpreter no longer has any references to it. - This function returns ``1`` if deferred reference counting is enabled on *obj* - (including when it was enabled before the call), + This function returns ``1`` if deferred reference counting is enabled on *obj*, and ``0`` if deferred reference counting is not supported or if the hint was - ignored by the runtime. This function is thread-safe, and cannot fail. + ignored by the interpreter, such as when deferred reference counting is already + enabled on *obj*. This function is thread-safe, and cannot fail. This function does nothing on builds with the :term:`GIL` enabled, which do not support deferred reference counting. This also does nothing if *obj* is not @@ -609,7 +654,8 @@ Object Protocol :c:func:`PyObject_GC_IsTracked`). This function is intended to be used soon after *obj* is created, - by the code that creates it. + by the code that creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` + slot. .. versionadded:: 3.14 diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst index 8bd8c107c98..83de4248039 100644 --- a/Doc/c-api/objimpl.rst +++ b/Doc/c-api/objimpl.rst @@ -12,6 +12,7 @@ object types. .. toctree:: allocation.rst + lifecycle.rst structures.rst typeobj.rst gcsupport.rst diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 77b5e3c0876..76a1e9f528d 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -5,11 +5,12 @@ Support for Perf Maps ---------------------- -On supported platforms (as of this writing, only Linux), the runtime can take +On supported platforms (Linux and macOS), the runtime can take advantage of *perf map files* to make Python functions visible to an external -profiling tool (such as `perf `_). -A running process may create a file in the ``/tmp`` directory, which contains entries -that can map a section of executable code to a name. This interface is described in the +profiling tool (such as `perf `_ or +`samply `_). A running process may create a +file in the ``/tmp`` directory, which contains entries that can map a section +of executable code to a name. This interface is described in the `documentation of the Linux Perf tool `_. diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 00000000000..9e2d92341b0 --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index b23f016f9b0..57a0728d4e9 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -210,7 +210,7 @@ of Python objects. Py_SETREF(dst, src); - That arranges to set *dst* to *src* _before_ releasing the reference + That arranges to set *dst* to *src* *before* releasing the reference to the old value of *dst*, so that any code triggered as a side-effect of *dst* getting torn down no longer believes *dst* points to a valid object. diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index cba823aa027..09c0fb6b9c5 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -147,7 +147,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 124e58cf950..f5e6b7ad157 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -2,9 +2,9 @@ .. _stable: -*************** -C API Stability -*************** +*********************** +C API and ABI Stability +*********************** Unless documented otherwise, Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`. @@ -51,6 +51,7 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-application-binary-interface: Stable Application Binary Interface =================================== @@ -198,6 +199,162 @@ This is the case with Windows and macOS releases from ``python.org`` and many third-party distributors. +ABI Checking +============ + +.. versionadded:: 3.15 + +Python includes a rudimentary check for ABI compatibility. + +This check is not comprehensive. +It only guards against common cases of incompatible modules being +installed for the wrong interpreter. +It also does not take :ref:`platform incompatibilities ` +into account. +It can only be done after an extension is successfully loaded. + +Despite these limitations, it is recommended that extension modules use this +mechanism, so that detectable incompatibilities raise exceptions rather than +crash. + +Most modules can use this check via the :c:data:`Py_mod_abi` +slot and the :c:macro:`PyABIInfo_VAR` macro, for example like this: + +.. code-block:: c + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot mymodule_slots[] = { + {Py_mod_abi, &abi_info}, + ... + }; + + +The full API is described below for advanced use cases. + +.. c:function:: int PyABIInfo_Check(PyABIInfo *info, const char *module_name) + + Verify that the given *info* is compatible with the currently running + interpreter. + + Return 0 on success. On failure, raise an exception and return -1. + + If the ABI is incompatible, the raised exception will be :py:exc:`ImportError`. + + The *module_name* argument can be ``NULL``, or point to a NUL-terminated + UTF-8-encoded string used for error messages. + + Note that if *info* describes the ABI that the current code uses (as defined + by :c:macro:`PyABIInfo_VAR`, for example), using any other Python C API + may lead to crashes. + In particular, it is not safe to examine the raised exception. + + .. versionadded:: 3.15 + +.. c:macro:: PyABIInfo_VAR(NAME) + + Define a static :c:struct:`PyABIInfo` variable with the given *NAME* that + describes the ABI that the current code will use. + This macro expands to: + + .. code-block:: c + + static PyABIInfo NAME = { + 1, 0, + PyABIInfo_DEFAULT_FLAGS, + PY_VERSION_HEX, + PyABIInfo_DEFAULT_ABI_VERSION + } + + .. versionadded:: 3.15 + +.. c:type:: PyABIInfo + + .. c:member:: uint8_t abiinfo_major_version + + The major version of :c:struct:`PyABIInfo`. Can be set to: + + * ``0`` to skip all checking, or + * ``1`` to specify this version of :c:struct:`!PyABIInfo`. + + .. c:member:: uint8_t abiinfo_minor_version + + The minor version of :c:struct:`PyABIInfo`. + Must be set to ``0``; larger values are reserved for backwards-compatible + future versions of :c:struct:`!PyABIInfo`. + + .. c:member:: uint16_t flags + + .. c:namespace:: NULL + + This field is usually set to the following macro: + + .. c:macro:: PyABIInfo_DEFAULT_FLAGS + + Default flags, based on current values of macros such as + :c:macro:`Py_LIMITED_API` and :c:macro:`Py_GIL_DISABLED`. + + Alternately, the field can be set to the following flags, combined + by bitwise OR. + Unused bits must be set to zero. + + ABI variant -- one of: + + .. c:macro:: PyABIInfo_STABLE + + Specifies that the stable ABI is used. + + .. c:macro:: PyABIInfo_INTERNAL + + Specifies ABI specific to a particular build of CPython. + Internal use only. + + Free-threading compatibility -- one of: + + .. c:macro:: PyABIInfo_FREETHREADED + + Specifies ABI compatible with free-threading builds of CPython. + (That is, ones compiled with :option:`--disable-gil`; with ``t`` + in :py:data:`sys.abiflags`) + + .. c:macro:: PyABIInfo_GIL + + Specifies ABI compatible with non-free-threading builds of CPython + (ones compiled *without* :option:`--disable-gil`). + + .. c:member:: uint32_t build_version + + The version of the Python headers used to build the code, in the format + used by :c:macro:`PY_VERSION_HEX`. + + This can be set to ``0`` to skip any checks related to this field. + This option is meant mainly for projects that do not use the CPython + headers directly, and do not emulate a specific version of them. + + .. c:member:: uint32_t abi_version + + The ABI version. + + For the Stable ABI, this field should be the value of + :c:macro:`Py_LIMITED_API` + (except if :c:macro:`Py_LIMITED_API` is ``3``; use + :c:expr:`Py_PACK_VERSION(3, 2)` in that case). + + Otherwise, it should be set to :c:macro:`PY_VERSION_HEX`. + + It can also be set to ``0`` to skip any checks related to this field. + + .. c:namespace:: NULL + + .. c:macro:: PyABIInfo_DEFAULT_ABI_VERSION + + The value that should be used for this field, based on current + values of macros such as :c:macro:`Py_LIMITED_API`, + :c:macro:`PY_VERSION_HEX` and :c:macro:`Py_GIL_DISABLED`. + + .. versionadded:: 3.15 + + .. _limited-api-list: Contents of Limited API diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 987bc167c68..62f45def04f 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -28,18 +28,52 @@ under :ref:`reference counting `. object. In a normal "release" build, it contains only the object's reference count and a pointer to the corresponding type object. Nothing is actually declared to be a :c:type:`PyObject`, but every pointer - to a Python object can be cast to a :c:expr:`PyObject*`. Access to the - members must be done by using the macros :c:macro:`Py_REFCNT` and - :c:macro:`Py_TYPE`. + to a Python object can be cast to a :c:expr:`PyObject*`. + + The members must not be accessed directly; instead use macros such as + :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_refcnt + + The object's reference count, as returned by :c:macro:`Py_REFCNT`. + Do not use this field directly; instead use functions and macros such as + :c:macro:`!Py_REFCNT`, :c:func:`Py_INCREF` and :c:func:`Py_DecRef`. + + The field type may be different from ``Py_ssize_t``, depending on + build configuration and platform. + + .. c:member:: PyTypeObject* ob_type + + The object's type. + Do not use this field directly; use :c:macro:`Py_TYPE` and + :c:func:`Py_SET_TYPE` instead. .. c:type:: PyVarObject - This is an extension of :c:type:`PyObject` that adds the :c:member:`~PyVarObject.ob_size` - field. This is only used for objects that have some notion of *length*. - This type does not often appear in the Python/C API. - Access to the members must be done by using the macros - :c:macro:`Py_REFCNT`, :c:macro:`Py_TYPE`, and :c:macro:`Py_SIZE`. + An extension of :c:type:`PyObject` that adds the + :c:member:`~PyVarObject.ob_size` field. + This is intended for objects that have some notion of *length*. + + As with :c:type:`!PyObject`, the members must not be accessed directly; + instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and + :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_size + + A size field, whose contents should be considered an object's internal + implementation detail. + + Do not use this field directly; use :c:macro:`Py_SIZE` instead. + + Object creation functions such as :c:func:`PyObject_NewVar` will + generally set this field to the requested size (number of items). + After creation, arbitrary values can be stored in :c:member:`!ob_size` + using :c:macro:`Py_SET_SIZE`. + + To get an object's publicly exposed length, as returned by + the Python function :py:func:`len`, use :c:func:`PyObject_Length` + instead. .. c:macro:: PyObject_HEAD @@ -103,9 +137,8 @@ under :ref:`reference counting `. Get the type of the Python object *o*. - Return a :term:`borrowed reference`. - - Use the :c:func:`Py_SET_TYPE` function to set an object type. + The returned reference is :term:`borrowed ` from *o*. + Do not release it with :c:func:`Py_DECREF` or similar. .. versionchanged:: 3.11 :c:func:`Py_TYPE()` is changed to an inline static function. @@ -122,16 +155,26 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_TYPE(PyObject *o, PyTypeObject *type) - Set the object *o* type to *type*. + Set the type of object *o* to *type*, without any checking or reference + counting. + + This is a very low-level operation. + Consider instead setting the Python attribute :attr:`~object.__class__` + using :c:func:`PyObject_SetAttrString` or similar. + + Note that assigning an incompatible type can lead to undefined behavior. + + If *type* is a :ref:`heap type `, the caller must create a + new reference to it. + Similarly, if the old type of *o* is a heap type, the caller must release + a reference to that type. .. versionadded:: 3.9 .. c:function:: Py_ssize_t Py_SIZE(PyVarObject *o) - Get the size of the Python object *o*. - - Use the :c:func:`Py_SET_SIZE` function to set an object size. + Get the :c:member:`~PyVarObject.ob_size` field of *o*. .. versionchanged:: 3.11 :c:func:`Py_SIZE()` is changed to an inline static function. @@ -140,7 +183,7 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) - Set the object *o* size to *size*. + Set the :c:member:`~PyVarObject.ob_size` field of *o* to *size*. .. versionadded:: 3.9 @@ -237,6 +280,8 @@ Implementing functions and methods Name of the method. + A ``NULL`` *ml_name* marks the end of a :c:type:`!PyMethodDef` array. + .. c:member:: PyCFunction ml_meth Pointer to the C implementation. @@ -404,6 +449,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -429,6 +493,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -439,6 +521,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- @@ -562,14 +700,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index b3c89800e38..ee73c1c8ada 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: @@ -258,10 +276,57 @@ These are utility functions that make functionality from the :mod:`sys` module accessible to C code. They all work with the current interpreter thread's :mod:`sys` module's dict, which is contained in the internal thread state structure. +.. c:function:: PyObject *PySys_GetAttr(PyObject *name) + + Get the attribute *name* of the :mod:`sys` module. + Return a :term:`strong reference`. + Raise :exc:`RuntimeError` and return ``NULL`` if it does not exist or + if the :mod:`sys` module cannot be found. + + If the non-existing object should not be treated as a failure, you can use + :c:func:`PySys_GetOptionalAttr` instead. + + .. versionadded:: 3.15 + +.. c:function:: PyObject *PySys_GetAttrString(const char *name) + + This is the same as :c:func:`PySys_GetAttr`, but *name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + If the non-existing object should not be treated as a failure, you can use + :c:func:`PySys_GetOptionalAttrString` instead. + + .. versionadded:: 3.15 + +.. c:function:: int PySys_GetOptionalAttr(PyObject *name, PyObject **result) + + Variant of :c:func:`PySys_GetAttr` which doesn't raise + exception if the object does not exist. + + * Set *\*result* to a new :term:`strong reference` to the object and + return ``1`` if the object exists. + * Set *\*result* to ``NULL`` and return ``0`` without setting an exception + if the object does not exist. + * Set an exception, set *\*result* to ``NULL``, and return ``-1``, + if an error occurred. + + .. versionadded:: 3.15 + +.. c:function:: int PySys_GetOptionalAttrString(const char *name, PyObject **result) + + This is the same as :c:func:`PySys_GetOptionalAttr`, but *name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: 3.15 + .. c:function:: PyObject *PySys_GetObject(const char *name) - Return the object *name* from the :mod:`sys` module or ``NULL`` if it does - not exist, without setting an exception. + Similar to :c:func:`PySys_GetAttrString`, but return a :term:`borrowed + reference` and return ``NULL`` *without* setting exception on failure. + + Preserves exception that was set before the call. .. c:function:: int PySys_SetObject(const char *name, PyObject *v) @@ -269,14 +334,6 @@ accessible to C code. They all work with the current interpreter thread's case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` on error. -.. c:function:: void PySys_ResetWarnOptions() - - Reset :data:`sys.warnoptions` to an empty list. This function may be - called prior to :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - .. c:function:: void PySys_WriteStdout(const char *format, ...) Write the output string described by *format* to :data:`sys.stdout`. No diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 815afddad19..d0add48d7e8 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -37,6 +37,19 @@ Tuple Objects or ``NULL`` with an exception set on failure. +.. c:function:: PyObject* PyTuple_FromArray(PyObject *const *array, Py_ssize_t size) + + Create a tuple of *size* items and copy references from *array* to the new + tuple. + + *array* can be NULL if *size* is ``0``. + + On success, return a new reference. + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.15 + + .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) Return a new tuple object of size *n*, @@ -48,7 +61,7 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. - On error, return ``-1`` and with an exception set. + On error, return ``-1`` with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index ec2867b0ce0..1f57cc04f5d 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -116,6 +116,20 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) + + Mark *type* as not watched. This undoes a previous call to + :c:func:`PyType_Watch`. *type* must not be ``NULL``. + + An extension should never call this function with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + + On success, this function returns ``0``. On failure, this function returns + ``-1`` with an exception set. + + .. versionadded:: 3.12 + + .. c:type:: int (*PyType_WatchCallback)(PyObject *type) Type of a type-watcher callback function. @@ -133,6 +147,18 @@ Type Objects Type features are denoted by single bit flags. +.. c:function:: int PyType_FastSubclass(PyTypeObject *type, int flag) + + Return non-zero if the type object *type* sets the subclass flag *flag*. + Subclass flags are denoted by + :c:macro:`Py_TPFLAGS_*_SUBCLASS `. + This function is used by many ``_Check`` functions for common types. + + .. seealso:: + :c:func:`PyObject_TypeCheck`, which is used as a slower alternative in + ``_Check`` functions for types that don't come with subclass flags. + + .. c:function:: int PyType_IS_GC(PyTypeObject *o) Return true if the type object includes support for the cycle detector; this @@ -151,14 +177,31 @@ Type Objects .. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use - Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to ``NULL``. + Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type + object. Uses Python's default memory allocation mechanism to allocate memory + for a new instance, zeros the memory, then initializes the memory as if by + calling :c:func:`PyObject_Init` or :c:func:`PyObject_InitVar`. + + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + For types that support garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set), this function behaves like + :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar` (except the + memory is guaranteed to be zeroed before initialization), and should be + paired with :c:func:`PyObject_GC_Del` in :c:member:`~PyTypeObject.tp_free`. + Otherwise, it behaves like :c:macro:`PyObject_New` or + :c:macro:`PyObject_NewVar` (except the memory is guaranteed to be zeroed + before initialization) and should be paired with :c:func:`PyObject_Free` in + :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) - Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Create a - new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot. + Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type + object. Creates a new instance using the type's + :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) @@ -176,6 +219,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -183,6 +227,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -198,6 +243,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -205,6 +251,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -221,6 +268,7 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was @@ -235,11 +283,12 @@ Type Objects ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses are not necessarily defined in the same module as their superclass. See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot - be used. + See :c:func:`PyType_GetModuleByToken` for cases when :c:type:`!PyCMethod` + cannot be used. .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -254,10 +303,11 @@ Type Objects .. versionadded:: 3.9 -.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) - Find the first superclass whose module was created from - the given :c:type:`PyModuleDef` *def*, and return that module. +.. c:function:: PyObject* PyType_GetModuleByToken(PyTypeObject *type, const void *mod_token) + + Find the first superclass whose module has the given + :ref:`module token `, and return that module. If no module is found, raises a :py:class:`TypeError` and returns ``NULL``. @@ -267,12 +317,34 @@ Type Objects and other places where a method's defining class cannot be passed using the :c:type:`PyCMethod` calling convention. + .. versionadded:: next + + +.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) + + Find the first superclass whose module was created from the given + :c:type:`PyModuleDef` *def*, or whose :ref:`module token ` + is equal to *def*, and return that module. + + Note that modules created from a :c:type:`PyModuleDef` always have their + token set to the :c:type:`PyModuleDef`'s address. + In other words, this function is equivalent to + :c:func:`PyType_GetModuleByToken`, except that it: + + - returns a borrowed reference, and + - has a non-``void*`` argument type (which is a cosmetic difference in C). + + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + .. versionadded:: 3.11 -.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) + +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *tp_token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose - :c:macro:`Py_tp_token` token is equal to the given one. + :c:macro:`Py_tp_token` token is equal to *tp_token*. * If found, set *\*result* to a new :term:`strong reference` to it and return ``1``. @@ -283,10 +355,11 @@ Type Objects The *result* argument may be ``NULL``, in which case *\*result* is not set. Use this if you need only the return value. - The *token* argument may not be ``NULL``. + The *tp_token* argument may not be ``NULL``. .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -297,6 +370,16 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) + + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. + + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` + + Creating Heap-Allocated Types ............................. @@ -317,8 +400,8 @@ The following functions and structs are used to create The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. + If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. + If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. The *module* argument can be used to record the module in which the new @@ -345,6 +428,7 @@ The following functions and structs are used to create .. versionadded:: 3.12 + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -371,6 +455,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -392,6 +477,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -412,6 +498,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: int PyType_Freeze(PyTypeObject *type) Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. @@ -520,9 +607,9 @@ The following functions and structs are used to create :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. For example, use: - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` + * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` + * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` An additional slot is supported that does not correspond to a :c:type:`!PyTypeObject` struct field: @@ -541,7 +628,7 @@ The following functions and structs are used to create If it is not possible to switch to a ``MANAGED`` flag (for example, for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. + offset in :c:data:`Py_tp_members`. See :ref:`PyMemberDef documentation ` for details. @@ -568,8 +655,8 @@ The following functions and structs are used to create under the :ref:`limited API `. .. versionchanged:: 3.14 - The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation + The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set + using :c:data:`Py_tp_vectorcall`. See the field's documentation for details. .. c:member:: void *pfunc @@ -579,10 +666,11 @@ The following functions and structs are used to create *pfunc* values may not be ``NULL``, except for the following slots: - * ``Py_tp_doc`` + * :c:data:`Py_tp_doc` * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) + .. c:macro:: Py_tp_token A :c:member:`~PyType_Slot.slot` that records a static memory layout ID diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 3b9f07778d5..49fe02d919d 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -79,7 +79,7 @@ Quick Reference | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | | | | __delattr__ | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | :ref:`sub-slots` | | | | % | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -325,9 +325,10 @@ sub-slots +---------------------------------------------------------+-----------------------------------+---------------+ | | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | | + | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | __buffer__ | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | | + | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | __release_\ | + | | | buffer\__ | +---------------------------------------------------------+-----------------------------------+---------------+ .. _slot-typedefs-table: @@ -491,9 +492,9 @@ metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that it type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. -.. c:member:: Py_ssize_t PyObject.ob_refcnt +:c:member:`PyObject.ob_refcnt` - This is the type object's reference count, initialized to ``1`` by the + The type object's reference count is initialized to ``1`` by the ``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type objects `, the type's instances (objects whose :c:member:`~PyObject.ob_type` points back to the type) do *not* count as references. But for @@ -505,7 +506,7 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. This field is not inherited by subtypes. -.. c:member:: PyTypeObject* PyObject.ob_type +:c:member:`PyObject.ob_type` This is the type's type, in other words its metatype. It is initialized by the argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be @@ -531,14 +532,13 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. PyVarObject Slots ----------------- -.. c:member:: Py_ssize_t PyVarObject.ob_size +:c:member:`PyVarObject.ob_size` For :ref:`statically allocated type objects `, this should be initialized to zero. For :ref:`dynamically allocated type objects `, this field has a special internal meaning. - This field should be accessed using the :c:func:`Py_SIZE()` and - :c:func:`Py_SET_SIZE()` macros. + This field should be accessed using the :c:func:`Py_SIZE()` macro. **Inheritance:** @@ -676,77 +676,144 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_dealloc - A pointer to the instance destructor function. This function must be defined - unless the type guarantees that its instances will never be deallocated (as is - the case for the singletons ``None`` and ``Ellipsis``). The function signature is:: + .. corresponding-type-slot:: Py_tp_dealloc + + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); - The destructor function is called by the :c:func:`Py_DECREF` and - :c:func:`Py_XDECREF` macros when the new reference count is zero. At this point, - the instance is still in existence, but there are no references to it. The - destructor function should free all references which the instance owns, free all - memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and call the type's - :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable - (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is - permissible to call the object deallocator directly instead of via - :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Free` if the instance was allocated - using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or - :c:func:`PyObject_GC_Del` if the instance was allocated using - :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. + The destructor function should remove all references which the instance owns + (e.g., call :c:func:`Py_CLEAR`), free all memory buffers owned by the + instance, and call the type's :c:member:`~PyTypeObject.tp_free` function to + free the object itself. - If the type supports garbage collection (has the :c:macro:`Py_TPFLAGS_HAVE_GC` - flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack` - before clearing any member fields. + If you may call functions that may set the error indicator, you must use + :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` + to ensure you don't clobber a preexisting error indicator (the deallocation + could have occurred while processing a different error): .. code-block:: c static void - foo_dealloc(PyObject *op) + foo_dealloc(foo_object *self) { + PyObject *et, *ev, *etb; + PyObject *exc = PyErr_GetRaisedException(); + ... + PyErr_SetRaisedException(exc); + } + + The dealloc handler itself must not raise an exception; if it hits an error + case it should call :c:func:`PyErr_FormatUnraisable` to log (and clear) an + unraisable exception. + + No guarantees are made about when an object is destroyed, except: + + * Python will destroy an object immediately or some time after the final + reference to the object is deleted, unless its finalizer + (:c:member:`~PyTypeObject.tp_finalize`) subsequently resurrects the + object. + * An object will not be destroyed while it is being automatically finalized + (:c:member:`~PyTypeObject.tp_finalize`) or automatically cleared + (:c:member:`~PyTypeObject.tp_clear`). + + CPython currently destroys an object immediately from :c:func:`Py_DECREF` + when the new reference count is zero, but this may change in a future + version. + + It is recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is always + finalized before destruction. + + If the type supports garbage collection (the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set), the destructor should call :c:func:`PyObject_GC_UnTrack` + before clearing any member fields. + + It is permissible to call :c:member:`~PyTypeObject.tp_clear` from + :c:member:`!tp_dealloc` to reduce code duplication and to guarantee that the + object is always cleared before destruction. Beware that + :c:member:`!tp_clear` might have already been called. + + If the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should release the owned reference to its type object (via + :c:func:`Py_DECREF`) after calling the type deallocator. See the example + code below.:: + + static void + foo_dealloc(PyObject *op) + { foo_object *self = (foo_object *) op; PyObject_GC_UnTrack(self); Py_CLEAR(self->ref); Py_TYPE(self)->tp_free(self); - } + } - Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should release the owned reference to its type object - (via :c:func:`Py_DECREF`) after - calling the type deallocator. In order to avoid dangling pointers, the - recommended way to achieve this is: + :c:member:`!tp_dealloc` must leave the exception status unchanged. If it + needs to call something that might raise an exception, the exception state + must be backed up first and restored later (after logging any exceptions + with :c:func:`PyErr_WriteUnraisable`). - .. code-block:: c + Example:: - static void - foo_dealloc(PyObject *op) - { - PyTypeObject *tp = Py_TYPE(op); - // free references and buffers here - tp->tp_free(op); - Py_DECREF(tp); - } + static void + foo_dealloc(PyObject *self) + { + PyObject *exc = PyErr_GetRaisedException(); - .. warning:: + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + // self was resurrected. + goto done; + } - In a garbage collected Python, :c:member:`!tp_dealloc` may be called from - any Python thread, not just the thread which created the object (if the - object becomes part of a refcount cycle, that cycle might be collected by - a garbage collection on any thread). This is not a problem for Python - API calls, since the thread on which :c:member:`!tp_dealloc` is called - with an :term:`attached thread state`. However, if the object being - destroyed in turn destroys objects from some other C or C++ library, care - should be taken to ensure that destroying those objects on the thread - which called :c:member:`!tp_dealloc` will not violate any assumptions of - the library. + PyTypeObject *tp = Py_TYPE(self); + + if (tp->tp_flags & Py_TPFLAGS_HAVE_GC) { + PyObject_GC_UnTrack(self); + } + + // Optional, but convenient to avoid code duplication. + if (tp->tp_clear && tp->tp_clear(self) < 0) { + PyErr_WriteUnraisable(self); + } + + // Any additional destruction goes here. + + tp->tp_free(self); + self = NULL; // In case PyErr_WriteUnraisable() is called below. + + if (tp->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_CLEAR(tp); + } + + done: + // Optional, if something was called that might have raised an + // exception. + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + PyErr_SetRaisedException(exc); + } + + :c:member:`!tp_dealloc` may be called from + any Python thread, not just the thread which created the object (if the + object becomes part of a refcount cycle, that cycle might be collected by + a garbage collection on any thread). This is not a problem for Python + API calls, since the thread on which :c:member:`!tp_dealloc` is called + with an :term:`attached thread state`. However, if the object being + destroyed in turn destroys objects from some other C library, care + should be taken to ensure that destroying those objects on the thread + which called :c:member:`!tp_dealloc` will not violate any assumptions of + the library. **Inheritance:** This field is inherited by subtypes. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset @@ -795,6 +862,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrfunc PyTypeObject.tp_getattr + .. corresponding-type-slot:: Py_tp_getattr + An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function @@ -812,6 +881,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrfunc PyTypeObject.tp_setattr + .. corresponding-type-slot:: Py_tp_setattr + An optional pointer to the function for setting and deleting attributes. This field is deprecated. When it is defined, it should point to a function @@ -844,6 +915,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr + .. corresponding-type-slot:: Py_tp_repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function @@ -909,6 +982,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash + .. corresponding-type-slot:: Py_tp_hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function @@ -950,6 +1025,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: ternaryfunc PyTypeObject.tp_call + .. corresponding-type-slot:: Py_tp_call + An optional pointer to a function that implements calling the object. This should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: @@ -963,6 +1040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_str + .. corresponding-type-slot:: Py_tp_str + An optional pointer to a function that implements the built-in operation :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the constructor for that type. This constructor calls :c:func:`PyObject_Str` to do @@ -988,6 +1067,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrofunc PyTypeObject.tp_getattro + .. corresponding-type-slot:: Py_tp_getattro + An optional pointer to the get-attribute function. The signature is the same as for :c:func:`PyObject_GetAttr`:: @@ -1012,6 +1093,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrofunc PyTypeObject.tp_setattro + .. corresponding-type-slot:: Py_tp_setattro + An optional pointer to the function for setting and deleting attributes. The signature is the same as for :c:func:`PyObject_SetAttr`:: @@ -1137,11 +1220,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_GC This bit is set when the object supports garbage collection. If this bit - is set, instances must be created using :c:macro:`PyObject_GC_New` and - destroyed using :c:func:`PyObject_GC_Del`. More information in section - :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in - the type object. + is set, memory for new instances (see :c:member:`~PyTypeObject.tp_alloc`) + must be allocated using :c:macro:`PyObject_GC_New` or + :c:func:`PyType_GenericAlloc` and deallocated (see + :c:member:`~PyTypeObject.tp_free`) using :c:func:`PyObject_GC_Del`. More + information in section :ref:`supporting-cycle-detection`. **Inheritance:** @@ -1192,10 +1275,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a `~object.__dict__` + This bit indicates that instances of the class have a :attr:`~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. - If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` must also be set. The type traverse function must call :c:func:`PyObject_VisitManagedDict` and its clear function must call :c:func:`PyObject_ClearManagedDict`. @@ -1213,6 +1296,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit indicates that instances of the class should be weakly referenceable. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` must also be set. + .. versionadded:: 3.12 **Inheritance:** @@ -1221,6 +1306,19 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass. + .. c:macro:: Py_TPFLAGS_PREHEADER + + These bits indicate that the VM will manage some fields by storing them + before the object. Currently, this macro is equivalent to + :c:expr:`Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_MANAGED_WEAKREF`. + + This macro value relies on the implementation of the VM, so its value is not + stable and may change in a future version. Prefer using individual + flags instead. + + .. versionadded:: 3.12 + + .. c:macro:: Py_TPFLAGS_ITEMS_AT_END Only usable with variable-size types, i.e. ones with non-zero @@ -1253,8 +1351,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass + Functions such as :c:func:`PyLong_Check` will call :c:func:`PyType_FastSubclass` + with one of these flags to quickly determine if a type is a subclass of a built-in type; such specific checks are faster than a generic check, like :c:func:`PyObject_IsInstance`. Custom types that inherit from built-ins should have their :c:member:`~PyTypeObject.tp_flags` @@ -1395,6 +1493,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_doc + .. corresponding-type-slot:: Py_tp_doc + An optional pointer to a NUL-terminated C string giving the docstring for this type object. This is exposed as the :attr:`~type.__doc__` attribute on the type and instances of the type. @@ -1406,6 +1506,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: traverseproc PyTypeObject.tp_traverse + .. corresponding-type-slot:: Py_tp_traverse + An optional pointer to a traversal function for the garbage collector. This is only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: @@ -1478,6 +1580,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) heap-allocated superclass). If they do not, the type object may not be garbage-collected. + .. note:: + + The :c:member:`~PyTypeObject.tp_traverse` function can be called from any + thread. + .. versionchanged:: 3.9 Heap-allocated types are expected to visit ``Py_TYPE(self)`` in @@ -1497,20 +1604,103 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear - An optional pointer to a clear function for the garbage collector. This is only - used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + .. corresponding-type-slot:: Py_tp_clear + + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); - The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` - functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, - the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's - possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to - break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. + The purpose of this function is to break reference cycles that are causing a + :term:`cyclic isolate` so that the objects can be safely destroyed. A + cleared object is a partially destroyed object; the object is not obligated + to satisfy design invariants held during normal use. + + :c:member:`!tp_clear` does not need to delete references to objects that + can't participate in reference cycles, such as Python strings or Python + integers. However, it may be convenient to clear all references, and write + the type's :c:member:`~PyTypeObject.tp_dealloc` function to invoke + :c:member:`!tp_clear` to avoid code duplication. (Beware that + :c:member:`!tp_clear` might have already been called. Prefer calling + idempotent functions like :c:func:`Py_CLEAR`.) + + Any non-trivial cleanup should be performed in + :c:member:`~PyTypeObject.tp_finalize` instead of :c:member:`!tp_clear`. + + .. note:: + + If :c:member:`!tp_clear` fails to break a reference cycle then the + objects in the :term:`cyclic isolate` may remain indefinitely + uncollectable ("leak"). See :data:`gc.garbage`. + + .. note:: + + Referents (direct and indirect) might have already been cleared; they are + not guaranteed to be in a consistent state. + + .. note:: + + The :c:member:`~PyTypeObject.tp_clear` function can be called from any + thread. + + .. note:: + + An object is not guaranteed to be automatically cleared before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. + + This function differs from the destructor + (:c:member:`~PyTypeObject.tp_dealloc`) in the following ways: + + * The purpose of clearing an object is to remove references to other objects + that might participate in a reference cycle. The purpose of the + destructor, on the other hand, is a superset: it must release *all* + resources it owns, including references to objects that cannot participate + in a reference cycle (e.g., integers) as well as the object's own memory + (by calling :c:member:`~PyTypeObject.tp_free`). + * When :c:member:`!tp_clear` is called, other objects might still hold + references to the object being cleared. Because of this, + :c:member:`!tp_clear` must not deallocate the object's own memory + (:c:member:`~PyTypeObject.tp_free`). The destructor, on the other hand, + is only called when no (strong) references exist, and as such, must + safely destroy the object itself by deallocating it. + * :c:member:`!tp_clear` might never be automatically called. An object's + destructor, on the other hand, will be automatically called some time + after the object becomes unreachable (i.e., either there are no references + to the object or the object is a member of a :term:`cyclic isolate`). + + No guarantees are made about when, if, or how often Python automatically + clears an object, except: + + * Python will not automatically clear an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically clear an object if it has not been + automatically finalized (see :c:member:`~PyTypeObject.tp_finalize`). (If + the finalizer resurrected the object, the object may or may not be + automatically finalized again before it is cleared.) + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically clear it if any member of the cyclic isolate has not yet + been automatically finalized (:c:member:`~PyTypeObject.tp_finalize`). + * Python will not destroy an object until after any automatic calls to its + :c:member:`!tp_clear` function have returned. This ensures that the act + of breaking a reference cycle does not invalidate the ``self`` pointer + while :c:member:`!tp_clear` is still executing. + * Python will not automatically call :c:member:`!tp_clear` multiple times + concurrently. + + CPython currently only automatically clears objects as needed to break + reference cycles in a :term:`cyclic isolate`, but future versions might + clear objects regularly before their destruction. + + Taken together, all :c:member:`~PyTypeObject.tp_clear` functions in the + system must combine to break all reference cycles. This is subtle, and if + in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For + example, the tuple type does not implement a + :c:member:`~PyTypeObject.tp_clear` function, because it's possible to prove + that no reference cycle can be composed entirely of tuples. Therefore the + :c:member:`~PyTypeObject.tp_clear` functions of other types are responsible + for breaking any cycle containing a tuple. This isn't immediately obvious, + and there's rarely a good reason to avoid implementing + :c:member:`~PyTypeObject.tp_clear`. Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those @@ -1540,23 +1730,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:func:`Py_CLEAR` macro performs the operations in a safe order. If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:member:`~PyTypeObject.tp_flags` field, the clear function must call :c:func:`PyObject_ClearManagedDict` like this:: PyObject_ClearManagedDict((PyObject*)self); - Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called - before an instance is deallocated. For example, when reference counting - is enough to determine that an object is no longer used, the cyclic garbage - collector is not involved and :c:member:`~PyTypeObject.tp_dealloc` is - called directly. - - Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, - it's not necessary to clear contained objects like Python strings or Python - integers, which can't participate in reference cycles. On the other hand, it may - be convenient to clear all contained Python objects, and write the type's - :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. - More information about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. @@ -1569,9 +1747,15 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: richcmpfunc PyTypeObject.tp_richcompare + .. corresponding-type-slot:: Py_tp_richcompare + An optional pointer to the rich comparison function, whose signature is:: PyObject *tp_richcompare(PyObject *self, PyObject *other, int op); @@ -1674,6 +1858,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getiterfunc PyTypeObject.tp_iter + .. corresponding-type-slot:: Py_tp_iter + An optional pointer to a function that returns an :term:`iterator` for the object. Its presence normally signals that the instances of this type are :term:`iterable` (although sequences may be iterable without this function). @@ -1689,6 +1875,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: iternextfunc PyTypeObject.tp_iternext + .. corresponding-type-slot:: Py_tp_iternext + An optional pointer to a function that returns the next item in an :term:`iterator`. The signature is:: @@ -1712,6 +1900,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods + .. corresponding-type-slot:: Py_tp_methods + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. @@ -1726,6 +1916,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members + .. corresponding-type-slot:: Py_tp_members + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1741,6 +1933,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset + .. corresponding-type-slot:: Py_tp_getset + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. @@ -1755,6 +1949,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base + .. corresponding-type-slot:: Py_tp_base + An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. @@ -1827,6 +2023,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrgetfunc PyTypeObject.tp_descr_get + .. corresponding-type-slot:: Py_tp_descr_get + An optional pointer to a "descriptor get" function. The function signature is:: @@ -1842,6 +2040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrsetfunc PyTypeObject.tp_descr_set + .. corresponding-type-slot:: Py_tp_descr_set + An optional pointer to a function for setting and deleting a descriptor's value. @@ -1902,6 +2102,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: initproc PyTypeObject.tp_init + .. corresponding-type-slot:: Py_tp_init + An optional pointer to an instance initialization function. This function corresponds to the :meth:`~object.__init__` method of classes. Like @@ -1937,6 +2139,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: allocfunc PyTypeObject.tp_alloc + .. corresponding-type-slot:: Py_tp_alloc + An optional pointer to an instance allocation function. The function signature is:: @@ -1945,22 +2149,23 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement). + Static subtypes inherit this slot, which will be + :c:func:`PyType_GenericAlloc` if inherited from :class:`object`. + + :ref:`Heap subtypes ` do not inherit this slot. **Default:** - For dynamic subtypes, this field is always set to - :c:func:`PyType_GenericAlloc`, to force a standard heap - allocation strategy. + For heap subtypes, this field is always set to + :c:func:`PyType_GenericAlloc`. - For static subtypes, :c:data:`PyBaseObject_Type` uses - :c:func:`PyType_GenericAlloc`. That is the recommended value - for all statically defined types. + For static subtypes, this slot is inherited (see above). .. c:member:: newfunc PyTypeObject.tp_new + .. corresponding-type-slot:: Py_tp_new + An optional pointer to an instance creation function. The function signature is:: @@ -2000,28 +2205,39 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: freefunc PyTypeObject.tp_free + .. corresponding-type-slot:: Py_tp_free + An optional pointer to an instance deallocation function. Its signature is:: void tp_free(void *self); - An initializer that is compatible with this signature is :c:func:`PyObject_Free`. + This function must free the memory allocated by + :c:member:`~PyTypeObject.tp_alloc`. **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement) + Static subtypes inherit this slot, which will be :c:func:`PyObject_Free` if + inherited from :class:`object`. Exception: If the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and it would inherit + :c:func:`PyObject_Free`, then this slot is not inherited but instead defaults + to :c:func:`PyObject_GC_Del`. + + :ref:`Heap subtypes ` do not inherit this slot. **Default:** - In dynamic subtypes, this field is set to a deallocator suitable to - match :c:func:`PyType_GenericAlloc` and the value of the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. + For :ref:`heap subtypes `, this slot defaults to a deallocator suitable to match + :c:func:`PyType_GenericAlloc` and the value of the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Free`. + For static subtypes, this slot is inherited (see above). .. c:member:: inquiry PyTypeObject.tp_is_gc + .. corresponding-type-slot:: Py_tp_is_gc + An optional pointer to a function called by the garbage collector. The garbage collector needs to know whether a particular object is collectible @@ -2050,12 +2266,14 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases + .. corresponding-type-slot:: Py_tp_bases + Tuple of base types. This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` + For dynamically created classes, the :c:data:`Py_tp_bases` :c:type:`slot ` can be used instead of the *bases* argument of :c:func:`PyType_FromSpecWithBases`. The argument form is preferred. @@ -2130,6 +2348,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_del + .. corresponding-type-slot:: Py_tp_del + This field is deprecated. Use :c:member:`~PyTypeObject.tp_finalize` instead. @@ -2144,29 +2364,140 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize - An optional pointer to an instance finalization function. Its signature is:: + .. corresponding-type-slot:: Py_tp_finalize + + An optional pointer to an instance finalization function. This is the C + implementation of the :meth:`~object.__del__` special method. Its signature + is:: void tp_finalize(PyObject *self); - If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when - finalizing an instance. It is called either from the garbage - collector (if the instance is part of an isolated reference cycle) or - just before the object is deallocated. Either way, it is guaranteed - to be called before attempting to break reference cycles, ensuring - that it finds the object in a sane state. + The primary purpose of finalization is to perform any non-trivial cleanup + that must be performed before the object is destroyed, while the object and + any other objects it directly or indirectly references are still in a + consistent state. The finalizer is allowed to execute + arbitrary Python code. - :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; - therefore, a recommended way to write a non-trivial finalizer is:: + Before Python automatically finalizes an object, some of the object's direct + or indirect referents might have themselves been automatically finalized. + However, none of the referents will have been automatically cleared + (:c:member:`~PyTypeObject.tp_clear`) yet. + + Other non-finalized objects might still be using a finalized object, so the + finalizer must leave the object in a sane state (e.g., invariants are still + met). + + .. note:: + + After Python automatically finalizes an object, Python might start + automatically clearing (:c:member:`~PyTypeObject.tp_clear`) the object + and its referents (direct and indirect). Cleared objects are not + guaranteed to be in a consistent state; a finalized object must be able + to tolerate cleared referents. + + .. note:: + + An object is not guaranteed to be automatically finalized before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. It is + recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is + always finalized before destruction. + + .. note:: + + The :c:member:`~PyTypeObject.tp_finalize` function can be called from any + thread, although the :term:`GIL` will be held. + + .. note:: + + The :c:member:`!tp_finalize` function can be called during shutdown, + after some global variables have been deleted. See the documentation of + the :meth:`~object.__del__` method for details. + + When Python finalizes an object, it behaves like the following algorithm: + + #. Python might mark the object as *finalized*. Currently, Python always + marks objects whose type supports garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and never marks other types of + objects; this might change in a future version. + #. If the object is not marked as *finalized* and its + :c:member:`!tp_finalize` finalizer function is non-``NULL``, the + finalizer function is called. + #. If the finalizer function was called and the finalizer made the object + reachable (i.e., there is a reference to the object and it is not a + member of a :term:`cyclic isolate`), then the finalizer is said to have + *resurrected* the object. It is unspecified whether the finalizer can + also resurrect the object by adding a new reference to the object that + does not make it reachable, i.e., the object is (still) a member of a + cyclic isolate. + #. If the finalizer resurrected the object, the object's pending destruction + is canceled and the object's *finalized* mark might be removed if + present. Currently, Python never removes the *finalized* mark; this + might change in a future version. + + *Automatic finalization* refers to any finalization performed by Python + except via calls to :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. No guarantees are made about + when, if, or how often an object is automatically finalized, except: + + * Python will not automatically finalize an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically finalize an object if finalizing it would + not mark the object as *finalized*. Currently, this applies to objects + whose type does not support garbage collection, i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is not set. Such objects can still be + manually finalized by calling :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. + * Python will not automatically finalize any two members of a :term:`cyclic + isolate` concurrently. + * Python will not automatically finalize an object after it has + automatically cleared (:c:member:`~PyTypeObject.tp_clear`) the object. + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically finalize it after automatically clearing (see + :c:member:`~PyTypeObject.tp_clear`) any other member. + * Python will automatically finalize every member of a :term:`cyclic + isolate` before it automatically clears (see + :c:member:`~PyTypeObject.tp_clear`) any of them. + * If Python is going to automatically clear an object + (:c:member:`~PyTypeObject.tp_clear`), it will automatically finalize the + object first. + + Python currently only automatically finalizes objects that are members of a + :term:`cyclic isolate`, but future versions might finalize objects regularly + before their destruction. + + To manually finalize an object, do not call this function directly; call + :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc` instead. + + :c:member:`~PyTypeObject.tp_finalize` should leave the current exception + status unchanged. The recommended way to write a non-trivial finalizer is + to back up the exception at the beginning by calling + :c:func:`PyErr_GetRaisedException` and restore the exception at the end by + calling :c:func:`PyErr_SetRaisedException`. If an exception is encountered + in the middle of the finalizer, log and clear it with + :c:func:`PyErr_WriteUnraisable` or :c:func:`PyErr_FormatUnraisable`. For + example:: static void - local_finalize(PyObject *self) + foo_finalize(PyObject *self) { - /* Save the current exception, if any. */ + // Save the current exception, if any. PyObject *exc = PyErr_GetRaisedException(); - /* ... */ + // ... - /* Restore the saved exception. */ + if (do_something_that_might_raise() != success_indicator) { + PyErr_WriteUnraisable(self); + goto done; + } + + done: + // Restore the saved exception. This silently discards any exception + // raised above, so be sure to call PyErr_WriteUnraisable first if + // necessary. PyErr_SetRaisedException(exc); } @@ -2182,11 +2513,19 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be used. This is no longer required. - .. seealso:: "Safe object finalization" (:pep:`442`) + .. seealso:: + + * :pep:`442`: "Safe object finalization" + * :ref:`life-cycle` for details about how this slot relates to other + slots. + * :c:func:`PyObject_CallFinalizer` + * :c:func:`PyObject_CallFinalizerFromDealloc` .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall + .. corresponding-type-slot:: Py_tp_vectorcall + A :ref:`vectorcall function ` to use for calls of this type object (rather than instances). In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, @@ -2352,42 +2691,148 @@ Number Object Structures Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add + + .. corresponding-type-slot:: Py_nb_add + .. c:member:: binaryfunc PyNumberMethods.nb_subtract + + .. corresponding-type-slot:: Py_nb_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_multiply + + .. corresponding-type-slot:: Py_nb_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_remainder + + .. corresponding-type-slot:: Py_nb_remainder + .. c:member:: binaryfunc PyNumberMethods.nb_divmod + + .. corresponding-type-slot:: Py_nb_divmod + .. c:member:: ternaryfunc PyNumberMethods.nb_power + + .. corresponding-type-slot:: Py_nb_power + .. c:member:: unaryfunc PyNumberMethods.nb_negative + + .. corresponding-type-slot:: Py_nb_negative + .. c:member:: unaryfunc PyNumberMethods.nb_positive + + .. corresponding-type-slot:: Py_nb_positive + .. c:member:: unaryfunc PyNumberMethods.nb_absolute + + .. corresponding-type-slot:: Py_nb_absolute + .. c:member:: inquiry PyNumberMethods.nb_bool + + .. corresponding-type-slot:: Py_nb_bool + .. c:member:: unaryfunc PyNumberMethods.nb_invert + + .. corresponding-type-slot:: Py_nb_invert + .. c:member:: binaryfunc PyNumberMethods.nb_lshift + + .. corresponding-type-slot:: Py_nb_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_rshift + + .. corresponding-type-slot:: Py_nb_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_and + + .. corresponding-type-slot:: Py_nb_and + .. c:member:: binaryfunc PyNumberMethods.nb_xor + + .. corresponding-type-slot:: Py_nb_xor + .. c:member:: binaryfunc PyNumberMethods.nb_or + + .. corresponding-type-slot:: Py_nb_or + .. c:member:: unaryfunc PyNumberMethods.nb_int + + .. corresponding-type-slot:: Py_nb_int + .. c:member:: void *PyNumberMethods.nb_reserved + .. c:member:: unaryfunc PyNumberMethods.nb_float + + .. corresponding-type-slot:: Py_nb_float + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_add + + .. corresponding-type-slot:: Py_nb_inplace_add + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_subtract + + .. corresponding-type-slot:: Py_nb_inplace_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_multiply + + .. corresponding-type-slot:: Py_nb_inplace_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_remainder + + .. corresponding-type-slot:: Py_nb_inplace_remainder + .. c:member:: ternaryfunc PyNumberMethods.nb_inplace_power + + .. corresponding-type-slot:: Py_nb_inplace_power + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_lshift + + .. corresponding-type-slot:: Py_nb_inplace_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_rshift + + .. corresponding-type-slot:: Py_nb_inplace_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_and + + .. corresponding-type-slot:: Py_nb_inplace_and + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_xor + + .. corresponding-type-slot:: Py_nb_inplace_xor + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_or + + .. corresponding-type-slot:: Py_nb_inplace_or + .. c:member:: binaryfunc PyNumberMethods.nb_floor_divide + + .. corresponding-type-slot:: Py_nb_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_true_divide + + .. corresponding-type-slot:: Py_nb_true_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_floor_divide + + .. corresponding-type-slot:: Py_nb_inplace_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_true_divide + + .. corresponding-type-slot:: Py_nb_inplace_true_divide + .. c:member:: unaryfunc PyNumberMethods.nb_index + + .. corresponding-type-slot:: Py_nb_index + .. c:member:: binaryfunc PyNumberMethods.nb_matrix_multiply + + .. corresponding-type-slot:: Py_nb_matrix_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_matrix_multiply + .. corresponding-type-slot:: Py_nb_inplace_matrix_multiply + + .. _mapping-structs: @@ -2404,12 +2849,16 @@ Mapping Object Structures .. c:member:: lenfunc PyMappingMethods.mp_length + .. corresponding-type-slot:: Py_mp_length + This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript + .. corresponding-type-slot:: Py_mp_subscript + This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the @@ -2418,6 +2867,8 @@ Mapping Object Structures .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript + .. corresponding-type-slot:: Py_mp_ass_subscript + This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and :c:func:`PySequence_DelSlice`. It has the same signature as @@ -2441,6 +2892,8 @@ Sequence Object Structures .. c:member:: lenfunc PySequenceMethods.sq_length + .. corresponding-type-slot:: Py_sq_length + This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`, and has the same signature. It is also used for handling negative indices via the :c:member:`~PySequenceMethods.sq_item` @@ -2448,18 +2901,24 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_concat + .. corresponding-type-slot:: Py_sq_concat + This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric addition via the :c:member:`~PyNumberMethods.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat + .. corresponding-type-slot:: Py_sq_repeat + This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item + .. corresponding-type-slot:: Py_sq_item + This function is used by :c:func:`PySequence_GetItem` and has the same signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. @@ -2473,6 +2932,8 @@ Sequence Object Structures .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item + .. corresponding-type-slot:: Py_sq_ass_item + This function is used by :c:func:`PySequence_SetItem` and has the same signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion @@ -2482,6 +2943,8 @@ Sequence Object Structures .. c:member:: objobjproc PySequenceMethods.sq_contains + .. corresponding-type-slot:: Py_sq_contains + This function may be used by :c:func:`PySequence_Contains` and has the same signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it @@ -2489,6 +2952,8 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat + .. corresponding-type-slot:: Py_sq_inplace_concat + This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` @@ -2498,6 +2963,8 @@ Sequence Object Structures .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat + .. corresponding-type-slot:: Py_sq_inplace_repeat + This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` @@ -2523,6 +2990,8 @@ Buffer Object Structures .. c:member:: getbufferproc PyBufferProcs.bf_getbuffer + .. corresponding-type-slot:: Py_bf_getbuffer + The signature of this function is:: int (PyObject *exporter, Py_buffer *view, int flags); @@ -2572,6 +3041,8 @@ Buffer Object Structures .. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer + .. corresponding-type-slot:: Py_bf_releasebuffer + The signature of this function is:: void (PyObject *exporter, Py_buffer *view); @@ -2626,6 +3097,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_await + .. corresponding-type-slot:: Py_am_await + The signature of this function is:: PyObject *am_await(PyObject *self); @@ -2637,6 +3110,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_aiter + .. corresponding-type-slot:: Py_am_aiter + The signature of this function is:: PyObject *am_aiter(PyObject *self); @@ -2649,6 +3124,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_anext + .. corresponding-type-slot:: Py_am_anext + The signature of this function is:: PyObject *am_anext(PyObject *self); @@ -2659,6 +3136,8 @@ Async Object Structures .. c:member:: sendfunc PyAsyncMethods.am_send + .. corresponding-type-slot:: Py_am_send + The signature of this function is:: PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 95987e872ce..ca7c8bb11a5 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -191,6 +191,22 @@ access to internal read-only data of Unicode objects: .. versionadded:: 3.2 +.. c:function:: Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *str) + + If the hash of *str*, as returned by :c:func:`PyObject_Hash`, has been + cached and is immediately available, return it. + Otherwise, return ``-1`` *without* setting an exception. + + If *str* is not a string (that is, if ``PyUnicode_Check(obj)`` + is false), the behavior is undefined. + + This function never fails with an exception. + + Note that there are no guarantees on when an object's hash is cached, + and the (non-)existence of a cached hash does not imply that the string has + any other properties. + + Unicode Character Properties """""""""""""""""""""""""""" @@ -305,12 +321,22 @@ These APIs can be used to work with surrogates: Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). +.. c:function:: Py_UCS4 Py_UNICODE_HIGH_SURROGATE(Py_UCS4 ch) + + Return the high UTF-16 surrogate (``0xD800`` to ``0xDBFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + +.. c:function:: Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) + + Return the low UTF-16 surrogate (``0xDC00`` to ``0xDFFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must - be in the range [0xDC00; 0xDFFF]. + surrogate pair. *high* must be in the range ``[0xD800; 0xDBFF]`` and *low* must + be in the range ``[0xDC00; 0xDFFF]``. Creating and accessing Unicode strings @@ -645,6 +671,17 @@ APIs: difference being that it decrements the reference count of *right* by one. +.. c:function:: PyObject* PyUnicode_BuildEncodingMap(PyObject* string) + + Return a mapping suitable for decoding a custom single-byte encoding. + Given a Unicode string *string* of up to 256 characters representing an encoding + table, returns either a compact internal mapping object or a dictionary + mapping character ordinals to byte values. Raises a :exc:`TypeError` and + return ``NULL`` on invalid input. + + .. versionadded:: 3.2 + + .. c:function:: const char* PyUnicode_GetDefaultEncoding(void) Return the name of the default string encoding, ``"utf-8"``. @@ -720,7 +757,7 @@ APIs: Return ``0`` on success, ``-1`` on error with an exception set. This function checks that *unicode* is a Unicode object, that the index is - not out of bounds, and that the object's reference count is one). + not out of bounds, and that the object's reference count is one. See :c:func:`PyUnicode_WRITE` for a version that skips these checks, making them your responsibility. @@ -1450,10 +1487,6 @@ the user settings on the machine running the codec. .. versionadded:: 3.3 -Methods & Slots -""""""""""""""" - - .. _unicodemethodsandslots: Methods and Slot Functions @@ -1715,10 +1748,6 @@ They all return ``NULL`` or ``-1`` if an exception occurs. from user input, prefer calling :c:func:`PyUnicode_FromString` and :c:func:`PyUnicode_InternInPlace` directly. - .. impl-detail:: - - Strings interned this way are made :term:`immortal`. - .. c:function:: unsigned int PyUnicode_CHECK_INTERNED(PyObject *str) @@ -1795,9 +1824,24 @@ object. See also :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. +.. c:function:: int PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Write the ASCII string *str* into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *str* must only contain ASCII characters. The behavior is undefined if + *str* contains non-ASCII characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + .. versionadded:: 3.14 + .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) - Writer the wide string *str* into *writer*. + Write the wide string *str* into *writer*. *size* is a number of wide characters. If *size* is equal to ``-1``, call ``wcslen(str)`` to get the string length. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 1ef4181d52e..7eb9f0b54ab 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -13,8 +13,9 @@ the interpreter. Several of these functions accept a start symbol from the grammar as a parameter. The available start symbols are :c:data:`Py_eval_input`, -:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described -following the functions which accept them as parameters. +:c:data:`Py_file_input`, :c:data:`Py_single_input`, and +:c:data:`Py_func_type_input`. These are described following the functions +which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` @@ -99,6 +100,20 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) + + Read and execute a single statement from a file associated with an + interactive device according to the *flags* argument. The user will be + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. + + Returns ``0`` when the input was + executed successfully, ``-1`` if there was an exception, or an error code + from the :file:`errcode.h` include file distributed as part of Python if + there was a parse error. (Note that :file:`errcode.h` is not included by + :file:`Python.h`, so must be included specifically if needed.) + + .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, @@ -107,17 +122,10 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - Read and execute a single statement from a file associated with an - interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the :term:`filesystem encoding and error handler`. - Returns ``0`` when the input was - executed successfully, ``-1`` if there was an exception, or an error code - from the :file:`errcode.h` include file distributed as part of Python if - there was a parse error. (Note that :file:`errcode.h` is not included by - :file:`Python.h`, so must be included specifically if needed.) - .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) @@ -140,7 +148,7 @@ the same library that the Python runtime is using. interpreter prompt is about to become idle and wait for user input from the terminal. The return value is ignored. Overriding this hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the + event loops, as done in :file:`Modules/_tkinter.c` in the Python source code. .. versionchanged:: 3.12 @@ -183,7 +191,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start token that should be used to parse the source code. + the start symbol and must one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. @@ -231,9 +239,9 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) Parse and compile the Python source code in *str*, returning the resulting code - object. The start token is given by *start*; this can be used to constrain the - code which can be compiled and should be :c:data:`Py_eval_input`, - :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by + object. The start symbol is given by *start*; this can be used to constrain the + code which can be compiled and should be :ref:`available start symbols + `. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. @@ -296,32 +304,6 @@ the same library that the Python runtime is using. true on success, false on failure. -.. c:var:: int Py_eval_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for isolated expressions; for use with - :c:func:`Py_CompileString`. - - -.. c:var:: int Py_file_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for sequences of statements as read - from a file or other source; for use with :c:func:`Py_CompileString`. This is - the symbol to use when compiling arbitrarily long Python source code. - - -.. c:var:: int Py_single_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for a single statement; for use with - :c:func:`Py_CompileString`. This is the symbol used for the interactive - interpreter loop. - - .. c:struct:: PyCompilerFlags This is the structure used to hold compiler flags. In cases where code is only @@ -361,7 +343,96 @@ the same library that the Python runtime is using. :py:mod:`!ast` Python module, which exports these constants under the same names. - .. c:var:: int CO_FUTURE_DIVISION + The "``PyCF``" flags above can be combined with "``CO_FUTURE``" flags such + as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally + selectable using :ref:`future statements `. + See :ref:`c_codeobject_flags` for a complete list. - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. + +.. _start-symbols: + +Available start symbols +^^^^^^^^^^^^^^^^^^^^^^^ + + +.. c:var:: int Py_eval_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for isolated expressions; for use with + :c:func:`Py_CompileString`. + + +.. c:var:: int Py_file_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for sequences of statements as read + from a file or other source; for use with :c:func:`Py_CompileString`. This is + the symbol to use when compiling arbitrarily long Python source code. + + +.. c:var:: int Py_single_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a single statement; for use with + :c:func:`Py_CompileString`. This is the symbol used for the interactive + interpreter loop. + + +.. c:var:: int Py_func_type_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a function type; for use with + :c:func:`Py_CompileString`. This is used to parse "signature type comments" + from :pep:`484`. + + This requires the :c:macro:`PyCF_ONLY_AST` flag to be set. + + .. seealso:: + * :py:class:`ast.FunctionType` + * :pep:`484` + + .. versionadded:: 3.8 + + +Stack Effects +^^^^^^^^^^^^^ + +.. seealso:: + :py:func:`dis.stack_effect` + + +.. c:macro:: PY_INVALID_STACK_EFFECT + + Sentinel value representing an invalid stack effect. + + This is currently equivalent to ``INT_MAX``. + + .. versionadded:: 3.8 + + +.. c:function:: int PyCompile_OpcodeStackEffect(int opcode, int oparg) + + Compute the stack effect of *opcode* with argument *oparg*. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.4 + + +.. c:function:: int PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) + + Similar to :c:func:`PyCompile_OpcodeStackEffect`, but don't include the + stack effect of jumping if *jump* is zero. + + If *jump* is ``0``, this will not include the stack effect of jumping, but + if *jump* is ``1`` or ``-1``, this will include it. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.8 diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index c3c6cf413dc..db6ae0a9d4e 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -19,7 +19,14 @@ as much as it can. .. c:function:: int PyWeakref_CheckRef(PyObject *ob) - Return non-zero if *ob* is a reference object. This function always succeeds. + Return non-zero if *ob* is a reference object or a subclass of the reference + type. This function always succeeds. + + +.. c:function:: int PyWeakref_CheckRefExact(PyObject *ob) + + Return non-zero if *ob* is a reference object, but not a subclass of the + reference type. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(PyObject *ob) @@ -38,6 +45,10 @@ as much as it can. weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -50,6 +61,10 @@ as much as it can. is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) @@ -64,30 +79,6 @@ as much as it can. .. versionadded:: 3.13 -.. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) - - Return a :term:`borrowed reference` to the referenced object from a weak - reference, *ref*. If the referent is no longer live, returns ``Py_None``. - - .. note:: - - This function returns a :term:`borrowed reference` to the referenced object. - This means that you should always call :c:func:`Py_INCREF` on the object - except when it cannot be destroyed before the last usage of the borrowed - reference. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyWeakref_GetRef` instead. - - -.. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) - - Similar to :c:func:`PyWeakref_GetObject`, but does no error checking. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyWeakref_GetRef` instead. - - .. c:function:: int PyWeakref_IsDead(PyObject *ref) Test if the weak reference *ref* is dead. Returns 1 if the reference is diff --git a/Doc/conf.py b/Doc/conf.py index 467961dd5e2..a4275835059 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -79,6 +79,10 @@ rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` .. |python_x_dot_y_literal| replace:: ``python{version}`` +.. |python_x_dot_y_t_literal| replace:: ``python{version}t`` +.. |python_x_dot_y_t_literal_config| replace:: ``python{version}t-config`` +.. |x_dot_y_b2_literal| replace:: ``{version}.0b2`` +.. |applications_python_version_literal| replace:: ``/Applications/Python {version}/`` .. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` .. Apparently this how you hack together a formatted link: @@ -217,99 +221,18 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), - # Deprecated function that was never documented: - ('py:func', 'getargspec'), - ('py:func', 'inspect.getargspec'), - # Undocumented modules that users shouldn't have to worry about - # (implementation details of `os.path`): - ('py:mod', 'ntpath'), - ('py:mod', 'posixpath'), ] # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # C API: Standard Python exception classes - ('c:data', 'PyExc_ArithmeticError'), - ('c:data', 'PyExc_AssertionError'), - ('c:data', 'PyExc_AttributeError'), - ('c:data', 'PyExc_BaseException'), - ('c:data', 'PyExc_BlockingIOError'), - ('c:data', 'PyExc_BrokenPipeError'), - ('c:data', 'PyExc_BufferError'), - ('c:data', 'PyExc_ChildProcessError'), - ('c:data', 'PyExc_ConnectionAbortedError'), - ('c:data', 'PyExc_ConnectionError'), - ('c:data', 'PyExc_ConnectionRefusedError'), - ('c:data', 'PyExc_ConnectionResetError'), - ('c:data', 'PyExc_EOFError'), - ('c:data', 'PyExc_Exception'), - ('c:data', 'PyExc_FileExistsError'), - ('c:data', 'PyExc_FileNotFoundError'), - ('c:data', 'PyExc_FloatingPointError'), - ('c:data', 'PyExc_GeneratorExit'), - ('c:data', 'PyExc_ImportError'), - ('c:data', 'PyExc_IndentationError'), - ('c:data', 'PyExc_IndexError'), - ('c:data', 'PyExc_InterruptedError'), - ('c:data', 'PyExc_IsADirectoryError'), - ('c:data', 'PyExc_KeyboardInterrupt'), - ('c:data', 'PyExc_KeyError'), - ('c:data', 'PyExc_LookupError'), - ('c:data', 'PyExc_MemoryError'), - ('c:data', 'PyExc_ModuleNotFoundError'), - ('c:data', 'PyExc_NameError'), - ('c:data', 'PyExc_NotADirectoryError'), - ('c:data', 'PyExc_NotImplementedError'), - ('c:data', 'PyExc_OSError'), - ('c:data', 'PyExc_OverflowError'), - ('c:data', 'PyExc_PermissionError'), - ('c:data', 'PyExc_ProcessLookupError'), - ('c:data', 'PyExc_PythonFinalizationError'), - ('c:data', 'PyExc_RecursionError'), - ('c:data', 'PyExc_ReferenceError'), - ('c:data', 'PyExc_RuntimeError'), - ('c:data', 'PyExc_StopAsyncIteration'), - ('c:data', 'PyExc_StopIteration'), - ('c:data', 'PyExc_SyntaxError'), - ('c:data', 'PyExc_SystemError'), - ('c:data', 'PyExc_SystemExit'), - ('c:data', 'PyExc_TabError'), - ('c:data', 'PyExc_TimeoutError'), - ('c:data', 'PyExc_TypeError'), - ('c:data', 'PyExc_UnboundLocalError'), - ('c:data', 'PyExc_UnicodeDecodeError'), - ('c:data', 'PyExc_UnicodeEncodeError'), - ('c:data', 'PyExc_UnicodeError'), - ('c:data', 'PyExc_UnicodeTranslateError'), - ('c:data', 'PyExc_ValueError'), - ('c:data', 'PyExc_ZeroDivisionError'), - # C API: Standard Python warning classes - ('c:data', 'PyExc_BytesWarning'), - ('c:data', 'PyExc_DeprecationWarning'), - ('c:data', 'PyExc_FutureWarning'), - ('c:data', 'PyExc_ImportWarning'), - ('c:data', 'PyExc_PendingDeprecationWarning'), - ('c:data', 'PyExc_ResourceWarning'), - ('c:data', 'PyExc_RuntimeWarning'), - ('c:data', 'PyExc_SyntaxWarning'), - ('c:data', 'PyExc_UnicodeWarning'), - ('c:data', 'PyExc_UserWarning'), - ('c:data', 'PyExc_Warning'), - # Undocumented public C macros - ('c:macro', 'Py_BUILD_ASSERT'), - ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), # Attributes/methods/etc. that definitely should be documented better, # but are deferred for now: - ('py:attr', '__annotations__'), - ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), - ('py:attr', 'decimal.Context.clamp'), - ('py:meth', 'index'), # list.index, tuple.index, etc. ] # gh-106948: Copy standard C types declared in the "c:type" domain and C @@ -433,11 +356,12 @@ 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', + 'maxlistdepth': '8', # See https://github.com/python/cpython/issues/139588 } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = 'Guido van Rossum and the Python development team' +_stdauthor = 'The Python development team' latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), ( @@ -512,11 +436,31 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +epub_exclude_files = ('index.xhtml', 'download.xhtml') # index pages are not valid xhtml # https://github.com/sphinx-doc/sphinx/issues/12359 epub_use_index = False +# translation tag +# --------------- + +language_code = None +for arg in sys.argv: + if arg.startswith('language='): + language_code = arg.split('=', 1)[1] + +if language_code: + tags.add('translation') # noqa: F821 + + rst_epilog += f"""\ +.. _TRANSLATION_REPO: https://github.com/python/python-docs-{language_code.replace("_", "-").lower()} +""" # noqa: F821 +else: + rst_epilog += """\ +.. _TRANSLATION_REPO: https://github.com/python +""" + # Options for the coverage checker # -------------------------------- @@ -630,13 +574,11 @@ 'image': '_static/og-image.png', 'line_color': '#3776ab', } -ogp_custom_meta_tags = [ - '', -] +ogp_custom_meta_tags = ('',) if 'create-social-cards' not in tags: # noqa: F821 # Define a static preview image when not creating social cards ogp_image = '_static/og-image.png' - ogp_custom_meta_tags += [ + ogp_custom_meta_tags += ( '', '', - ] + ) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ca99b9e6d37..64399f6ab1f 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -963,21 +963,45 @@ PyFunction_Check:PyObject*:o:0: PyFunction_GetAnnotations:PyObject*::0: PyFunction_GetAnnotations:PyObject*:op:0: +PyFunction_GET_ANNOTATIONS:PyObject*::0: +PyFunction_GET_ANNOTATIONS:PyObject*:op:0: + PyFunction_GetClosure:PyObject*::0: PyFunction_GetClosure:PyObject*:op:0: +PyFunction_GET_CLOSURE:PyObject*::0: +PyFunction_GET_CLOSURE:PyObject*:op:0: + PyFunction_GetCode:PyObject*::0: PyFunction_GetCode:PyObject*:op:0: +PyFunction_GET_CODE:PyObject*::0: +PyFunction_GET_CODE:PyObject*:op:0: + PyFunction_GetDefaults:PyObject*::0: PyFunction_GetDefaults:PyObject*:op:0: +PyFunction_GET_DEFAULTS:PyObject*::0: +PyFunction_GET_DEFAULTS:PyObject*:op:0: + +PyFunction_GetKwDefaults:PyObject*::0: +PyFunction_GetKwDefaults:PyObject*:op:0: + +PyFunction_GET_KW_DEFAULTS:PyObject*::0: +PyFunction_GET_KW_DEFAULTS:PyObject*:op:0: + PyFunction_GetGlobals:PyObject*::0: PyFunction_GetGlobals:PyObject*:op:0: +PyFunction_GET_GLOBALS:PyObject*::0: +PyFunction_GET_GLOBALS:PyObject*:op:0: + PyFunction_GetModule:PyObject*::0: PyFunction_GetModule:PyObject*:op:0: +PyFunction_GET_MODULE:PyObject*::0: +PyFunction_GET_MODULE:PyObject*:op:0: + PyFunction_New:PyObject*::+1: PyFunction_New:PyObject*:code:+1: PyFunction_New:PyObject*:globals:+1: @@ -1093,9 +1117,6 @@ PyImport_ImportModuleLevelObject:PyObject*:locals:0:??? PyImport_ImportModuleLevelObject:PyObject*:fromlist:0:??? PyImport_ImportModuleLevelObject:int:level:: -PyImport_ImportModuleNoBlock:PyObject*::+1: -PyImport_ImportModuleNoBlock:const char*:name:: - PyImport_ReloadModule:PyObject*::+1: PyImport_ReloadModule:PyObject*:m:0: @@ -1120,6 +1141,9 @@ PyInterpreterState_Clear:PyInterpreterState*:interp:: PyInterpreterState_Delete:void::: PyInterpreterState_Delete:PyInterpreterState*:interp:: +PyInterpreterState_GetDict:PyObject*::0: +PyInterpreterState_GetDict:PyInterpreterState*:interp:: + PyInterpreterState_GetID:int64_t::: PyInterpreterState_GetID:PyInterpreterState*:interp:: @@ -1448,6 +1472,9 @@ PyModule_Create2:PyObject*::+1: PyModule_Create2:PyModuleDef*:def:: PyModule_Create2:int:module_api_version:: +PyModule_Exec:int::: +PyModule_ExecDef:PyObject*:module:0: + PyModule_ExecDef:int::: PyModule_ExecDef:PyObject*:module:0: PyModule_ExecDef:PyModuleDef*:def:: @@ -1461,6 +1488,10 @@ PyModule_FromDefAndSpec2:PyModuleDef*:def:: PyModule_FromDefAndSpec2:PyObject*:spec:0: PyModule_FromDefAndSpec2:int:module_api_version:: +PyModule_FromSlotsAndSpec:PyObject*::+1: +PyModule_FromSlotsAndSpec:const PyModuleDef_Slot *:slots:: +PyModule_FromSlotsAndSpec:PyObject*:spec:0: + PyModule_GetDef:PyModuleDef*::0: PyModule_GetDef:PyObject*:module:0: @@ -1482,6 +1513,14 @@ PyModule_GetNameObject:PyObject*:module:0: PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: +PyModule_GetStateSize:int::: +PyModule_GetStateSize:PyObject*:module:0: +PyModule_GetToken:Py_ssize_t**:result:: + +PyModule_GetToken:int::: +PyModule_GetToken:PyObject*:module:0: +PyModule_GetToken:void**:result:: + PyModule_New:PyObject*::+1: PyModule_New:char*:name:: @@ -1492,9 +1531,6 @@ PyModule_SetDocString:int::: PyModule_SetDocString:PyObject*:module:0: PyModule_SetDocString:const char*:docstring:: -PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:: - PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: @@ -2391,6 +2427,14 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyObject*::+1: +PyType_GetModuleByToken:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyModuleDef*:def:: + +PyType_GetModuleByDef:PyObject*::0: +PyType_GetModuleByDef:PyTypeObject*:type:0: +PyType_GetModuleByDef:PyModuleDef*:def:: + PyType_GetQualName:PyObject*::+1: PyType_GetQualName:PyTypeObject*:type:0: @@ -2781,6 +2825,9 @@ PyUnicode_AppendAndDel:void::: PyUnicode_AppendAndDel:PyObject**:p_left:0: PyUnicode_AppendAndDel:PyObject*:right:-1: +PyUnicode_BuildEncodingMap:PyObject*::+1: +PyUnicode_BuildEncodingMap:PyObject*:string::: + PyUnicode_GetDefaultEncoding:const char*::: PyUnicode_GetDefaultEncoding::void:: @@ -2922,12 +2969,6 @@ PyWeakref_CheckProxy:PyObject*:ob:0: PyWeakref_CheckRef:int::: PyWeakref_CheckRef:PyObject*:ob:0: -PyWeakref_GET_OBJECT:PyObject*::0: -PyWeakref_GET_OBJECT:PyObject*:ref:0: - -PyWeakref_GetObject:PyObject*::0: -PyWeakref_GetObject:PyObject*:ref:0: - PyWeakref_GetRef:int::: PyWeakref_GetRef:PyObject*:ref:0: PyWeakref_GetRef:PyObject**:pobj:+1: @@ -3007,18 +3048,8 @@ Py_GetCompiler:const char*::: Py_GetCopyright:const char*::: -Py_GetExecPrefix:wchar_t*::: - -Py_GetPath:wchar_t*::: - Py_GetPlatform:const char*::: -Py_GetPrefix:wchar_t*::: - -Py_GetProgramFullPath:wchar_t*::: - -Py_GetProgramName:wchar_t*::: - Py_GetVersion:const char*::: Py_INCREF:void::: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 3d68487d07b..9c5fdcefaf8 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,5 +1,22 @@ role,name,added,ifdef_note,struct_abi_kind +macro,METH_CLASS,3.2,, +macro,METH_COEXIST,3.2,, +macro,METH_FASTCALL,3.7,, +macro,METH_METHOD,3.7,, +macro,METH_NOARGS,3.2,, +macro,METH_O,3.2,, +macro,METH_STATIC,3.2,, +macro,METH_VARARGS,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, +type,PyABIInfo,3.15,,full-abi +func,PyABIInfo_Check,3.15,, +macro,PyABIInfo_DEFAULT_ABI_VERSION,3.15,, +macro,PyABIInfo_DEFAULT_FLAGS,3.15,, +macro,PyABIInfo_FREETHREADED,3.15,, +macro,PyABIInfo_FREETHREADING_AGNOSTIC,3.15,, +macro,PyABIInfo_GIL,3.15,, +macro,PyABIInfo_STABLE,3.15,, +macro,PyABIInfo_VAR,3.15,, func,PyAIter_Check,3.10,, func,PyArg_Parse,3.2,, func,PyArg_ParseTuple,3.2,, @@ -8,6 +25,26 @@ func,PyArg_UnpackTuple,3.2,, func,PyArg_VaParse,3.2,, func,PyArg_VaParseTupleAndKeywords,3.2,, func,PyArg_ValidateKeywordArguments,3.2,, +macro,PyBUF_ANY_CONTIGUOUS,3.11,, +macro,PyBUF_CONTIG,3.11,, +macro,PyBUF_CONTIG_RO,3.11,, +macro,PyBUF_C_CONTIGUOUS,3.11,, +macro,PyBUF_FORMAT,3.11,, +macro,PyBUF_FULL,3.11,, +macro,PyBUF_FULL_RO,3.11,, +macro,PyBUF_F_CONTIGUOUS,3.11,, +macro,PyBUF_INDIRECT,3.11,, +macro,PyBUF_MAX_NDIM,3.11,, +macro,PyBUF_ND,3.11,, +macro,PyBUF_READ,3.11,, +macro,PyBUF_RECORDS,3.11,, +macro,PyBUF_RECORDS_RO,3.11,, +macro,PyBUF_SIMPLE,3.11,, +macro,PyBUF_STRIDED,3.11,, +macro,PyBUF_STRIDED_RO,3.11,, +macro,PyBUF_STRIDES,3.11,, +macro,PyBUF_WRITABLE,3.11,, +macro,PyBUF_WRITE,3.11,, data,PyBaseObject_Type,3.2,, func,PyBool_FromLong,3.2,, data,PyBool_Type,3.2,, @@ -123,6 +160,7 @@ func,PyDict_Merge,3.2,, func,PyDict_MergeFromSeq2,3.2,, func,PyDict_New,3.2,, func,PyDict_Next,3.2,, +func,PyDict_SetDefaultRef,3.15,, func,PyDict_SetItem,3.2,, func,PyDict_SetItemString,3.2,, func,PyDict_Size,3.2,, @@ -323,7 +361,6 @@ func,PyImport_ImportFrozenModuleObject,3.7,, func,PyImport_ImportModule,3.2,, func,PyImport_ImportModuleLevel,3.2,, func,PyImport_ImportModuleLevelObject,3.7,, -func,PyImport_ImportModuleNoBlock,3.2,, func,PyImport_ReloadModule,3.2,, func,PyIndex_Check,3.8,, type,PyInterpreterState,3.2,,opaque @@ -389,6 +426,7 @@ func,PyLong_FromUnsignedNativeBytes,3.14,, func,PyLong_FromVoidPtr,3.2,, func,PyLong_GetInfo,3.2,, data,PyLong_Type,3.2,, +macro,PyMODEXPORT_FUNC,3.15,, data,PyMap_Type,3.2,, func,PyMapping_Check,3.2,, func,PyMapping_GetItemString,3.2,, @@ -426,6 +464,7 @@ data,PyMethodDescr_Type,3.2,, type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi func,PyModuleDef_Init,3.5,, +type,PyModuleDef_Slot,3.5,,full-abi data,PyModuleDef_Type,3.5,, func,PyModule_Add,3.13,, func,PyModule_AddFunctions,3.7,, @@ -435,8 +474,10 @@ func,PyModule_AddObjectRef,3.10,, func,PyModule_AddStringConstant,3.2,, func,PyModule_AddType,3.10,, func,PyModule_Create2,3.2,, +func,PyModule_Exec,3.15,, func,PyModule_ExecDef,3.7,, func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_FromSlotsAndSpec,3.15,, func,PyModule_GetDef,3.2,, func,PyModule_GetDict,3.2,, func,PyModule_GetFilename,3.2,, @@ -444,6 +485,8 @@ func,PyModule_GetFilenameObject,3.2,, func,PyModule_GetName,3.2,, func,PyModule_GetNameObject,3.7,, func,PyModule_GetState,3.2,, +func,PyModule_GetStateSize,3.15,, +func,PyModule_GetToken,3.15,, func,PyModule_New,3.2,, func,PyModule_NewObject,3.7,, func,PyModule_SetDocString,3.7,, @@ -629,9 +672,12 @@ func,PySys_Audit,3.13,, func,PySys_AuditTuple,3.13,, func,PySys_FormatStderr,3.2,, func,PySys_FormatStdout,3.2,, +func,PySys_GetAttr,3.15,, +func,PySys_GetAttrString,3.15,, func,PySys_GetObject,3.2,, +func,PySys_GetOptionalAttr,3.15,, +func,PySys_GetOptionalAttrString,3.15,, func,PySys_GetXOptions,3.7,, -func,PySys_ResetWarnOptions,3.2,, func,PySys_SetArgv,3.2,, func,PySys_SetArgvEx,3.2,, func,PySys_SetObject,3.2,, @@ -699,6 +745,7 @@ func,PyType_GetFlags,3.2,, func,PyType_GetFullyQualifiedName,3.13,, func,PyType_GetModule,3.10,, func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleByToken,3.15,, func,PyType_GetModuleName,3.13,, func,PyType_GetModuleState,3.10,, func,PyType_GetName,3.11,, @@ -740,11 +787,7 @@ func,PyUnicode_Append,3.2,, func,PyUnicode_AppendAndDel,3.2,, func,PyUnicode_AsASCIIString,3.2,, func,PyUnicode_AsCharmapString,3.2,, -func,PyUnicode_AsDecodedObject,3.2,, -func,PyUnicode_AsDecodedUnicode,3.2,, -func,PyUnicode_AsEncodedObject,3.2,, func,PyUnicode_AsEncodedString,3.2,, -func,PyUnicode_AsEncodedUnicode,3.2,, func,PyUnicode_AsLatin1String,3.2,, func,PyUnicode_AsMBCSString,3.7,on Windows, func,PyUnicode_AsRawUnicodeEscapeString,3.2,, @@ -829,13 +872,20 @@ member,PyVarObject.ob_size,3.2,, func,PyVectorcall_Call,3.12,, func,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque -func,PyWeakref_GetObject,3.2,, func,PyWeakref_GetRef,3.13,, func,PyWeakref_NewProxy,3.2,, func,PyWeakref_NewRef,3.2,, data,PyWrapperDescr_Type,3.2,, func,PyWrapper_New,3.2,, data,PyZip_Type,3.2,, +macro,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, +macro,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_DEFAULTS,3.14,, +macro,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, +macro,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, +macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, @@ -862,16 +912,11 @@ func,Py_GetCompiler,3.2,, func,Py_GetConstant,3.13,, func,Py_GetConstantBorrowed,3.13,, func,Py_GetCopyright,3.2,, -func,Py_GetExecPrefix,3.2,, -func,Py_GetPath,3.2,, func,Py_GetPlatform,3.2,, -func,Py_GetPrefix,3.2,, -func,Py_GetProgramFullPath,3.2,, -func,Py_GetProgramName,3.2,, -func,Py_GetPythonHome,3.2,, func,Py_GetRecursionLimit,3.2,, func,Py_GetVersion,3.2,, data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IS_TYPE,3.15,, func,Py_IncRef,3.2,, func,Py_Initialize,3.2,, func,Py_InitializeEx,3.2,, @@ -888,22 +933,147 @@ func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, func,Py_PACK_FULL_VERSION,3.14,, func,Py_PACK_VERSION,3.14,, +macro,Py_READONLY,3.12,, func,Py_REFCNT,3.14,, +macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, +func,Py_SET_SIZE,3.15,, +func,Py_SIZE,3.15,, func,Py_SetProgramName,3.2,, func,Py_SetPythonHome,3.2,, func,Py_SetRecursionLimit,3.2,, +macro,Py_TPFLAGS_BASETYPE,3.2,, +macro,Py_TPFLAGS_DEFAULT,3.2,, +macro,Py_TPFLAGS_HAVE_GC,3.2,, +macro,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, +macro,Py_TPFLAGS_ITEMS_AT_END,3.12,, +macro,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, +macro,Py_TP_USE_SPEC,3.14,, func,Py_TYPE,3.14,, +macro,Py_T_BOOL,3.12,, +macro,Py_T_BYTE,3.12,, +macro,Py_T_CHAR,3.12,, +macro,Py_T_DOUBLE,3.12,, +macro,Py_T_FLOAT,3.12,, +macro,Py_T_INT,3.12,, +macro,Py_T_LONG,3.12,, +macro,Py_T_LONGLONG,3.12,, +macro,Py_T_OBJECT_EX,3.12,, +macro,Py_T_PYSSIZET,3.12,, +macro,Py_T_SHORT,3.12,, +macro,Py_T_STRING,3.12,, +macro,Py_T_STRING_INPLACE,3.12,, +macro,Py_T_UBYTE,3.12,, +macro,Py_T_UINT,3.12,, +macro,Py_T_ULONG,3.12,, +macro,Py_T_ULONGLONG,3.12,, +macro,Py_T_USHORT,3.12,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, data,Py_UTF8Mode,3.8,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, +macro,Py_am_aiter,3.5,, +macro,Py_am_anext,3.5,, +macro,Py_am_await,3.5,, +macro,Py_am_send,3.10,, +macro,Py_bf_getbuffer,3.11,, +macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, +macro,Py_mod_abi,3.15,, +macro,Py_mod_create,3.5,, +macro,Py_mod_doc,3.15,, +macro,Py_mod_exec,3.5,, +macro,Py_mod_gil,3.13,, +macro,Py_mod_methods,3.15,, +macro,Py_mod_multiple_interpreters,3.12,, +macro,Py_mod_name,3.15,, +macro,Py_mod_state_clear,3.15,, +macro,Py_mod_state_free,3.15,, +macro,Py_mod_state_size,3.15,, +macro,Py_mod_state_traverse,3.15,, +macro,Py_mod_token,3.15,, +macro,Py_mp_ass_subscript,3.2,, +macro,Py_mp_length,3.2,, +macro,Py_mp_subscript,3.2,, +macro,Py_nb_absolute,3.2,, +macro,Py_nb_add,3.2,, +macro,Py_nb_and,3.2,, +macro,Py_nb_bool,3.2,, +macro,Py_nb_divmod,3.2,, +macro,Py_nb_float,3.2,, +macro,Py_nb_floor_divide,3.2,, +macro,Py_nb_index,3.2,, +macro,Py_nb_inplace_add,3.2,, +macro,Py_nb_inplace_and,3.2,, +macro,Py_nb_inplace_floor_divide,3.2,, +macro,Py_nb_inplace_lshift,3.2,, +macro,Py_nb_inplace_matrix_multiply,3.5,, +macro,Py_nb_inplace_multiply,3.2,, +macro,Py_nb_inplace_or,3.2,, +macro,Py_nb_inplace_power,3.2,, +macro,Py_nb_inplace_remainder,3.2,, +macro,Py_nb_inplace_rshift,3.2,, +macro,Py_nb_inplace_subtract,3.2,, +macro,Py_nb_inplace_true_divide,3.2,, +macro,Py_nb_inplace_xor,3.2,, +macro,Py_nb_int,3.2,, +macro,Py_nb_invert,3.2,, +macro,Py_nb_lshift,3.2,, +macro,Py_nb_matrix_multiply,3.5,, +macro,Py_nb_multiply,3.2,, +macro,Py_nb_negative,3.2,, +macro,Py_nb_or,3.2,, +macro,Py_nb_positive,3.2,, +macro,Py_nb_power,3.2,, +macro,Py_nb_remainder,3.2,, +macro,Py_nb_rshift,3.2,, +macro,Py_nb_subtract,3.2,, +macro,Py_nb_true_divide,3.2,, +macro,Py_nb_xor,3.2,, +macro,Py_sq_ass_item,3.2,, +macro,Py_sq_concat,3.2,, +macro,Py_sq_contains,3.2,, +macro,Py_sq_inplace_concat,3.2,, +macro,Py_sq_inplace_repeat,3.2,, +macro,Py_sq_item,3.2,, +macro,Py_sq_length,3.2,, +macro,Py_sq_repeat,3.2,, type,Py_ssize_t,3.2,, +macro,Py_tp_alloc,3.2,, +macro,Py_tp_base,3.2,, +macro,Py_tp_bases,3.2,, +macro,Py_tp_call,3.2,, +macro,Py_tp_clear,3.2,, +macro,Py_tp_dealloc,3.2,, +macro,Py_tp_del,3.2,, +macro,Py_tp_descr_get,3.2,, +macro,Py_tp_descr_set,3.2,, +macro,Py_tp_doc,3.2,, +macro,Py_tp_finalize,3.5,, +macro,Py_tp_free,3.2,, +macro,Py_tp_getattr,3.2,, +macro,Py_tp_getattro,3.2,, +macro,Py_tp_getset,3.2,, +macro,Py_tp_hash,3.2,, +macro,Py_tp_init,3.2,, +macro,Py_tp_is_gc,3.2,, +macro,Py_tp_iter,3.2,, +macro,Py_tp_iternext,3.2,, +macro,Py_tp_members,3.2,, +macro,Py_tp_methods,3.2,, +macro,Py_tp_new,3.2,, +macro,Py_tp_repr,3.2,, +macro,Py_tp_richcompare,3.2,, +macro,Py_tp_setattr,3.2,, +macro,Py_tp_setattro,3.2,, +macro,Py_tp_str,3.2,, +macro,Py_tp_token,3.14,, +macro,Py_tp_traverse,3.2,, +macro,Py_tp_vectorcall,3.14,, type,Py_uintptr_t,3.2,, type,allocfunc,3.2,, type,binaryfunc,3.2,, diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index a5cc8f1d5b3..9927b876760 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -1,10 +1,9 @@ Pending removal in Python 3.15 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The bundled copy of ``libmpdecimal``. -* The :c:func:`PyImport_ImportModuleNoBlock`: +* The :c:func:`!PyImport_ImportModuleNoBlock`: Use :c:func:`PyImport_ImportModule` instead. -* :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: +* :c:func:`!PyWeakref_GetObject` and :c:func:`!PyWeakref_GET_OBJECT`: Use :c:func:`PyWeakref_GetRef` instead. The `pythoncapi-compat project `__ can be used to get :c:func:`PyWeakref_GetRef` on Python 3.12 and older. @@ -22,27 +21,27 @@ Pending removal in Python 3.15 may return a type other than :class:`bytes`, such as :class:`str`. * Python initialization functions, deprecated in Python 3.13: - * :c:func:`Py_GetPath`: + * :c:func:`!Py_GetPath`: Use :c:func:`PyConfig_Get("module_search_paths") ` (:data:`sys.path`) instead. - * :c:func:`Py_GetPrefix`: + * :c:func:`!Py_GetPrefix`: Use :c:func:`PyConfig_Get("base_prefix") ` (:data:`sys.base_prefix`) instead. Use :c:func:`PyConfig_Get("prefix") ` (:data:`sys.prefix`) if :ref:`virtual environments ` need to be handled. - * :c:func:`Py_GetExecPrefix`: + * :c:func:`!Py_GetExecPrefix`: Use :c:func:`PyConfig_Get("base_exec_prefix") ` (:data:`sys.base_exec_prefix`) instead. Use :c:func:`PyConfig_Get("exec_prefix") ` (:data:`sys.exec_prefix`) if :ref:`virtual environments ` need to be handled. - * :c:func:`Py_GetProgramFullPath`: + * :c:func:`!Py_GetProgramFullPath`: Use :c:func:`PyConfig_Get("executable") ` (:data:`sys.executable`) instead. - * :c:func:`Py_GetProgramName`: + * :c:func:`!Py_GetProgramName`: Use :c:func:`PyConfig_Get("executable") ` (:data:`sys.executable`) instead. - * :c:func:`Py_GetPythonHome`: + * :c:func:`!Py_GetPythonHome`: Use :c:func:`PyConfig_Get("home") ` or the :envvar:`PYTHONHOME` environment variable instead. @@ -60,7 +59,7 @@ Pending removal in Python 3.15 Set :c:member:`PyConfig.program_name` instead. * :c:func:`!Py_SetPythonHome()`: Set :c:member:`PyConfig.home` instead. - * :c:func:`PySys_ResetWarnOptions`: + * :c:func:`!PySys_ResetWarnOptions`: Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. The :c:func:`Py_InitializeFromConfig` API should be used with diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst new file mode 100644 index 00000000000..9453f83799c --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -0,0 +1,4 @@ +Pending removal in Python 3.16 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The bundled copy of ``libmpdec``. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 564cfb79a0b..022aee93aa7 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -1,11 +1,12 @@ Pending removal in Python 3.18 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Deprecated private functions (:gh:`128863`): +* The following private functions are deprecated + and planned for removal in Python 3.18: * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. - * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`. + * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. @@ -31,7 +32,7 @@ Pending removal in Python 3.18 :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. + :c:func:`PyUnicodeWriter_WriteASCII(writer, str) `. * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. @@ -41,5 +42,6 @@ Pending removal in Python 3.18 * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. The `pythoncapi-compat project - `__ can be used to get these - new public functions on Python 3.13 and older. + `__ can be used to get + these new public functions on Python 3.13 and older. + (Contributed by Victor Stinner in :gh:`128863`.) diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst new file mode 100644 index 00000000000..18623b19a2a --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -0,0 +1,9 @@ +Pending removal in Python 3.20 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The ``cval`` field in :c:type:`PyComplexObject` (:gh:`128813`). + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. + +* Macros :c:macro:`!Py_MATH_PIl` and :c:macro:`!Py_MATH_El`. diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index d064f2bec42..f3cecb321d6 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -9,6 +9,8 @@ Deprecations .. include:: pending-removal-in-3.19.rst +.. include:: pending-removal-in-3.20.rst + .. include:: pending-removal-in-future.rst C API deprecations @@ -18,4 +20,6 @@ C API deprecations .. include:: c-api-pending-removal-in-3.18.rst +.. include:: c-api-pending-removal-in-3.20.rst + .. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a..d5b8c80e8f9 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index 6159fa48848..171758156ab 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -38,12 +38,6 @@ Pending removal in Python 3.14 is no current event loop set and it decides to create one. (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, - or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) - * :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. (Contributed by Alan Williams in :gh:`72346`.) @@ -78,7 +72,7 @@ Pending removal in Python 3.14 :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:!pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -96,9 +90,6 @@ Pending removal in Python 3.14 if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. -* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, - now causes a :exc:`DeprecationWarning` to be emitted when it is used. - * :mod:`urllib`: :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a public API. diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index 7b32275ad86..09cbd6f01a0 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -20,7 +20,7 @@ Pending removal in Python 3.15 * :mod:`http.server`: - * The obsolete and rarely used :class:`~http.server.CGIHTTPRequestHandler` + * The obsolete and rarely used :class:`!CGIHTTPRequestHandler` has been deprecated since Python 3.13. No direct replacement exists. *Anything* is better than CGI to interface @@ -45,13 +45,13 @@ Pending removal in Python 3.15 * :mod:`pathlib`: - * :meth:`.PurePath.is_reserved` + * :meth:`!.PurePath.is_reserved` has been deprecated since Python 3.13. Use :func:`os.path.isreserved` to detect reserved paths on Windows. * :mod:`platform`: - * :func:`~platform.java_ver` has been deprecated since Python 3.13. + * :func:`!platform.java_ver` has been deprecated since Python 3.13. This function is only useful for Jython support, has a confusing API, and is largely untested. @@ -85,20 +85,28 @@ Pending removal in Python 3.15 has been deprecated since Python 3.13. Use the class-based syntax or the functional syntax instead. - * The :func:`typing.no_type_check_decorator` decorator function + * When using the functional syntax of :class:`~typing.TypedDict`\s, failing + to pass a value to the *fields* parameter (``TD = TypedDict("TD")``) or + passing ``None`` (``TD = TypedDict("TD", None)``) has been deprecated + since Python 3.13. + Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` + to create a TypedDict with zero field. + + * The :func:`!typing.no_type_check_decorator` decorator function has been deprecated since Python 3.13. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + * :mod:`wave`: - * The :meth:`~wave.Wave_read.getmark`, :meth:`!setmark`, - and :meth:`~wave.Wave_read.getmarkers` methods of + * The ``getmark()``, ``setmark()`` and ``getmarkers()`` methods of the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes have been deprecated since Python 3.13. * :mod:`zipimport`: - * :meth:`~zipimport.zipimporter.load_module` has been deprecated since + * :meth:`!zipimport.zipimporter.load_module` has been deprecated since Python 3.10. Use :meth:`~zipimport.zipimporter.exec_module` instead. - (Contributed by Jiahao Li in :gh:`125746`.) + (:gh:`125746`.) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index cdd76ee693f..b00c7002b03 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -63,9 +63,9 @@ Pending removal in Python 3.16 * :mod:`logging`: - Support for custom logging handlers with the *strm* argument is deprecated - and scheduled for removal in Python 3.16. Define handlers with the *stream* - argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) + * Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) * :mod:`mimetypes`: diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 370b98307e5..e769c9d371e 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -1,6 +1,34 @@ Pending removal in Python 3.17 ------------------------------ +* :mod:`collections.abc`: + + - :class:`collections.abc.ByteString` is scheduled for removal in Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) + + +* :mod:`encodings`: + + - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` + is deprecated and scheduled for removal in Python 3.17. + (Contributed by Stan Ulbrych in :gh:`136702`) + * :mod:`typing`: - Before Python 3.14, old-style unions were implemented using the private class @@ -8,3 +36,22 @@ Pending removal in Python 3.17 but it has been retained for backward compatibility, with removal scheduled for Python 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` and :func:`typing.get_args` instead of relying on private implementation details. + - :class:`typing.ByteString`, deprecated since Python 3.9, is scheduled for removal in + Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 3936f63ca5b..25f9cba390d 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -6,3 +6,19 @@ Pending removal in Python 3.19 * Implicitly switching to the MSVC-compatible struct layout by setting :attr:`~ctypes.Structure._pack_` but not :attr:`~ctypes.Structure._layout_` on non-Windows platforms. + +* :mod:`hashlib`: + + - In hash function constructors such as :func:`~hashlib.new` or the + direct hash-named constructors such as :func:`~hashlib.md5` and + :func:`~hashlib.sha256`, their optional initial data parameter could + also be passed a keyword argument named ``data=`` or ``string=`` in + various :mod:`!hashlib` implementations. + + Support for the ``string`` keyword argument name is now deprecated + and slated for removal in Python 3.19. + + Before Python 3.13, the ``string`` keyword parameter was not correctly + supported depending on the backend implementation of hash functions. + Prefer passing the initial data as a positional argument for maximum + backwards compatibility. diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst new file mode 100644 index 00000000000..c0feda19682 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -0,0 +1,26 @@ +Pending removal in Python 3.20 +------------------------------ + +* The ``__version__`` attribute has been deprecated in these standard library + modules and will be removed in Python 3.20. + Use :py:data:`sys.version_info` instead. + + - :mod:`argparse` + - :mod:`csv` + - :mod:`!ctypes.macholib` + - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`imaplib` + - :mod:`ipaddress` + - :mod:`json` + - :mod:`logging` (``__date__`` also deprecated) + - :mod:`optparse` + - :mod:`pickle` + - :mod:`platform` + - :mod:`re` + - :mod:`socketserver` + - :mod:`tabnanny` + - :mod:`tkinter.font` + - :mod:`tkinter.ttk` + - :mod:`zlib` + + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 4c4a368baca..30186741670 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -15,7 +15,6 @@ although there is currently no date scheduled for their removal. * :mod:`builtins`: - * ``bool(NotImplemented)``. * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, the single argument signature. @@ -77,7 +76,7 @@ although there is currently no date scheduled for their removal. * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. +* :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process. * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is deprecated, use an exception instance. @@ -89,8 +88,6 @@ although there is currently no date scheduled for their removal. underscore. (Contributed by Serhiy Storchaka in :gh:`91760`.) -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - * :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in Python 3.12; use the *onexc* parameter instead. diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index ddde567f6f3..098dde39ea5 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -6,41 +6,10 @@ Building C and C++ Extensions ***************************** -A C extension for CPython is a shared library (e.g. a ``.so`` file on Linux, -``.pyd`` on Windows), which exports an *initialization function*. +A C extension for CPython is a shared library (for example, a ``.so`` file on +Linux, ``.pyd`` on Windows), which exports an *initialization function*. -To be importable, the shared library must be available on :envvar:`PYTHONPATH`, -and must be named after the module name, with an appropriate extension. -When using setuptools, the correct filename is generated automatically. - -The initialization function has the signature: - -.. c:function:: PyObject* PyInit_modulename(void) - -It returns either a fully initialized module, or a :c:type:`PyModuleDef` -instance. See :ref:`initializing-modules` for details. - -.. highlight:: python - -For modules with ASCII-only names, the function must be named -``PyInit_``, with ```` replaced by the name of the -module. When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -``PyInitU_``, with ```` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python:: - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the *"Multiple modules in one library"* section in :pep:`489` for details. +See :ref:`extension-modules` for details. .. highlight:: c @@ -51,7 +20,11 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. Building C and C++ Extensions with setuptools ============================================= -Python 3.12 and newer no longer come with distutils. Please refer to the -``setuptools`` documentation at -https://setuptools.readthedocs.io/en/latest/setuptools.html -to learn more about how build and distribute C/C++ extensions with setuptools. + +Building, packaging and distributing extension modules is best done with +third-party tools, and is out of scope of this document. +One suitable tool is Setuptools, whose documentation can be found at +https://setuptools.pypa.io/en/latest/setuptools.html. + +The :mod:`distutils` module, which was included in the standard library +until Python 3.12, is now maintained as part of Setuptools. diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index b777862da79..cb41889437c 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -245,21 +245,23 @@ Python extension. For example:: return PyLong_FromLong(numargs); } - static PyMethodDef EmbMethods[] = { + static PyMethodDef emb_module_methods[] = { {"numargs", emb_numargs, METH_VARARGS, "Return the number of arguments received by the process."}, {NULL, NULL, 0, NULL} }; - static PyModuleDef EmbModule = { - PyModuleDef_HEAD_INIT, "emb", NULL, -1, EmbMethods, - NULL, NULL, NULL, NULL + static struct PyModuleDef emb_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "emb", + .m_size = 0, + .m_methods = emb_module_methods, }; static PyObject* PyInit_emb(void) { - return PyModule_Create(&EmbModule); + return PyModuleDef_Init(&emb_module); } Insert the above code just above the :c:func:`main` function. Also, insert the diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index b0493bed75b..f9b65643dfe 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -75,12 +75,37 @@ the module and a copyright notice if you like). See :ref:`arg-parsing-string-and-buffers` for a description of this macro. All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. For convenience, and -since they are used extensively by the Python interpreter, ``"Python.h"`` -includes a few standard header files: ````, ````, -````, and ````. If the latter header file does not exist on -your system, it declares the functions :c:func:`malloc`, :c:func:`free` and -:c:func:`realloc` directly. +``PY``, except those defined in standard header files. + +.. tip:: + + For backward compatibility, :file:`Python.h` includes several standard header files. + C extensions should include the standard headers that they use, + and should not rely on these implicit includes. + If using the limited C API version 3.13 or newer, the implicit includes are: + + * ```` + * ```` (on Windows) + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` (if present) + + If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.12 or older, + the headers below are also included: + + * ```` + * ```` (on POSIX) + + If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.10 or older, + the headers below are also included: + + * ```` + * ```` + * ```` + * ```` The next thing we add to our module file is the C function that will be called when the Python expression ``spam.system(string)`` is evaluated (we'll see @@ -203,31 +228,57 @@ function usually raises :c:data:`PyExc_TypeError`. If you have an argument whos value must be in a particular range or must satisfy other conditions, :c:data:`PyExc_ValueError` is appropriate. -You can also define a new exception that is unique to your module. For this, you -usually declare a static object variable at the beginning of your file:: +You can also define a new exception that is unique to your module. +The simplest way to do this is to declare a static global object variable at +the beginning of the file:: - static PyObject *SpamError; + static PyObject *SpamError = NULL; -and initialize it in your module's initialization function (:c:func:`!PyInit_spam`) -with an exception object:: +and initialize it by calling :c:func:`PyErr_NewException` in the module's +:c:data:`Py_mod_exec` function (:c:func:`!spam_module_exec`):: + + SpamError = PyErr_NewException("spam.error", NULL, NULL); + +Since :c:data:`!SpamError` is a global variable, it will be overwritten every time +the module is reinitialized, when the :c:data:`Py_mod_exec` function is called. + +For now, let's avoid the issue: we will block repeated initialization by raising an +:py:exc:`ImportError`:: + + static PyObject *SpamError = NULL; + + static int + spam_module_exec(PyObject *m) + { + if (SpamError != NULL) { + PyErr_SetString(PyExc_ImportError, + "cannot initialize spam module more than once"); + return -1; + } + SpamError = PyErr_NewException("spam.error", NULL, NULL); + if (PyModule_AddObjectRef(m, "SpamError", SpamError) < 0) { + return -1; + } + + return 0; + } + + static PyModuleDef_Slot spam_module_slots[] = { + {Py_mod_exec, spam_module_exec}, + {0, NULL} + }; + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + .m_size = 0, // non-negative + .m_slots = spam_module_slots, + }; PyMODINIT_FUNC PyInit_spam(void) { - PyObject *m; - - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - - SpamError = PyErr_NewException("spam.error", NULL, NULL); - if (PyModule_AddObjectRef(m, "error", SpamError) < 0) { - Py_CLEAR(SpamError); - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&spam_module); } Note that the Python name for the exception object is :exc:`!spam.error`. The @@ -242,6 +293,11 @@ needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. +For now, the :c:func:`Py_DECREF` call to remove this reference is missing. +Even when the Python interpreter shuts down, the global :c:data:`!SpamError` +variable will not be garbage-collected. It will "leak". +We did, however, ensure that this will happen at most once per process. + We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. @@ -318,7 +374,7 @@ The Module's Method Table and Initialization Function I promised to show how :c:func:`!spam_system` is called from Python programs. First, we need to list its name and address in a "method table":: - static PyMethodDef SpamMethods[] = { + static PyMethodDef spam_methods[] = { ... {"system", spam_system, METH_VARARGS, "Execute a shell command."}, @@ -343,13 +399,10 @@ function. The method table must be referenced in the module definition structure:: - static struct PyModuleDef spammodule = { - PyModuleDef_HEAD_INIT, - "spam", /* name of module */ - spam_doc, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - SpamMethods + static struct PyModuleDef spam_module = { + ... + .m_methods = spam_methods, + ... }; This structure, in turn, must be passed to the interpreter in the module's @@ -360,23 +413,17 @@ only non-\ ``static`` item defined in the module file:: PyMODINIT_FUNC PyInit_spam(void) { - return PyModule_Create(&spammodule); + return PyModuleDef_Init(&spam_module); } Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, declares any special linkage declarations required by the platform, and for C++ declares the function as ``extern "C"``. -When the Python program imports module :mod:`!spam` for the first time, -:c:func:`!PyInit_spam` is called. (See below for comments about embedding Python.) -It calls :c:func:`PyModule_Create`, which returns a module object, and -inserts built-in function objects into the newly created module based upon the -table (an array of :c:type:`PyMethodDef` structures) found in the module definition. -:c:func:`PyModule_Create` returns a pointer to the module object -that it creates. It may abort with a fatal error for -certain errors, or return ``NULL`` if the module could not be initialized -satisfactorily. The init function must return the module object to its caller, -so that it then gets inserted into ``sys.modules``. +:c:func:`!PyInit_spam` is called when each interpreter imports its module +:mod:`!spam` for the first time. (See below for comments about embedding Python.) +A pointer to the module definition must be returned via :c:func:`PyModuleDef_Init`, +so that the import machinery can create the module and store it in ``sys.modules``. When embedding Python, the :c:func:`!PyInit_spam` function is not called automatically unless there's an entry in the :c:data:`PyImport_Inittab` table. @@ -433,23 +480,19 @@ optionally followed by an import of the module:: .. note:: - Removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process (or following a :c:func:`fork` without an - intervening :c:func:`exec`) can create problems for some extension modules. - Extension module authors should exercise caution when initializing internal data - structures. + If you declare a global variable or a local static one, the module may + experience unintended side-effects on re-initialisation, for example when + removing entries from ``sys.modules`` or importing compiled modules into + multiple interpreters within a process + (or following a :c:func:`fork` without an intervening :c:func:`exec`). + If module state is not yet fully :ref:`isolated `, + authors should consider marking the module as having no support for subinterpreters + (via :c:macro:`Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED`). A more substantial example module is included in the Python source distribution -as :file:`Modules/xxmodule.c`. This file may be used as a template or simply +as :file:`Modules/xxlimited.c`. This file may be used as a template or simply read as an example. -.. note:: - - Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization* - (new in Python 3.5), where a PyModuleDef structure is returned from - ``PyInit_spam``, and creation of the module is left to the import machinery. - For details on multi-phase initialization, see :PEP:`489`. - .. _compilation: @@ -790,18 +833,17 @@ Philbrick (philbrick@hks.com):: {NULL, NULL, 0, NULL} /* sentinel */ }; - static struct PyModuleDef keywdargmodule = { - PyModuleDef_HEAD_INIT, - "keywdarg", - NULL, - -1, - keywdarg_methods + static struct PyModuleDef keywdarg_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "keywdarg", + .m_size = 0, + .m_methods = keywdarg_methods, }; PyMODINIT_FUNC PyInit_keywdarg(void) { - return PyModule_Create(&keywdargmodule); + return PyModuleDef_Init(&keywdarg_module); } @@ -1042,7 +1084,14 @@ references to all its items, so when item 1 is replaced, it has to dispose of the original item 1. Now let's suppose the original item 1 was an instance of a user-defined class, and let's further suppose that the class defined a :meth:`!__del__` method. If this class instance has a reference count of 1, -disposing of it will call its :meth:`!__del__` method. +disposing of it will call its :meth:`!__del__` method. Internally, +:c:func:`PyList_SetItem` calls :c:func:`Py_DECREF` on the replaced item, +which invokes replaced item's corresponding +:c:member:`~PyTypeObject.tp_dealloc` function. During +deallocation, :c:member:`~PyTypeObject.tp_dealloc` calls +:c:member:`~PyTypeObject.tp_finalize`, which is mapped to the +:meth:`!__del__` method for class instances (see :pep:`442`). This entire +sequence happens synchronously within the :c:func:`PyList_SetItem` call. Since it is written in Python, the :meth:`!__del__` method can execute arbitrary Python code. Could it perhaps do something to invalidate the reference to @@ -1072,8 +1121,9 @@ why his :meth:`!__del__` methods would fail... The second case of problems with a borrowed reference is a variant involving threads. Normally, multiple threads in the Python interpreter can't get in each -other's way, because there is a global lock protecting Python's entire object -space. However, it is possible to temporarily release this lock using the macro +other's way, because there is a :term:`global lock ` +protecting Python's entire object space. +However, it is possible to temporarily release this lock using the macro :c:macro:`Py_BEGIN_ALLOW_THREADS`, and to re-acquire it using :c:macro:`Py_END_ALLOW_THREADS`. This is common around blocking I/O calls, to let other threads use the processor while waiting for the I/O to complete. @@ -1259,20 +1309,15 @@ two more lines must be added:: #include "spammodule.h" The ``#define`` is used to tell the header file that it is being included in the -exporting module, not a client module. Finally, the module's initialization -function must take care of initializing the C API pointer array:: +exporting module, not a client module. Finally, the module's :c:data:`mod_exec +` function must take care of initializing the C API pointer array:: - PyMODINIT_FUNC - PyInit_spam(void) + static int + spam_module_exec(PyObject *m) { - PyObject *m; static void *PySpam_API[PySpam_API_pointers]; PyObject *c_api_object; - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - /* Initialize the C API pointer array */ PySpam_API[PySpam_System_NUM] = (void *)PySpam_System; @@ -1280,11 +1325,10 @@ function must take care of initializing the C API pointer array:: c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { - Py_DECREF(m); - return NULL; + return -1; } - return m; + return 0; } Note that ``PySpam_API`` is declared ``static``; otherwise the pointer @@ -1343,20 +1387,16 @@ like this:: All that a client module must do in order to have access to the function :c:func:`!PySpam_System` is to call the function (or rather macro) -:c:func:`!import_spam` in its initialization function:: +:c:func:`!import_spam` in its :c:data:`mod_exec ` function:: - PyMODINIT_FUNC - PyInit_client(void) + static int + client_module_exec(PyObject *m) { - PyObject *m; - - m = PyModule_Create(&clientmodule); - if (m == NULL) - return NULL; - if (import_spam() < 0) - return NULL; + if (import_spam() < 0) { + return -1; + } /* additional initialization can happen here */ - return m; + return 0; } The main disadvantage of this approach is that the file :file:`spammodule.h` is diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 01b4df6d44a..4cc2c96d8d5 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -26,19 +26,9 @@ Recommended third party tools ============================= This guide only covers the basic tools for creating extensions provided -as part of this version of CPython. Third party tools like -`Cython `_, `cffi `_, -`SWIG `_ and `Numba `_ -offer both simpler and more sophisticated approaches to creating C and C++ -extensions for Python. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions `_ - The Python Packaging User Guide not only covers several available - tools that simplify the creation of binary extensions, but also - discusses the various reasons why creating an extension module may be - desirable in the first place. +as part of this version of CPython. Some :ref:`third party tools +` offer both simpler and more sophisticated approaches to creating +C and C++ extensions for Python. Creating extensions without third party tools @@ -49,6 +39,10 @@ assistance from third party tools. It is intended primarily for creators of those tools, rather than being a recommended way to create your own C extensions. +.. seealso:: + + :pep:`489` -- Multi-phase extension module initialization + .. toctree:: :maxdepth: 2 :numbered: diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index e3612f3a187..26085b5cebd 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -560,6 +560,8 @@ For an object to be weakly referenceable, the extension type must set the field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. +If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + Concretely, here is how the statically declared type object would look:: static PyTypeObject TrivialType = { diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 3fc91841416..3bbee33bd50 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -55,8 +55,10 @@ from the previous chapter. This file defines three things: #. How the :class:`!Custom` **type** behaves: this is the ``CustomType`` struct, which defines a set of flags and function pointers that the interpreter inspects when specific operations are requested. -#. How to initialize the :mod:`!custom` module: this is the ``PyInit_custom`` - function and the associated ``custommodule`` struct. +#. How to define and execute the :mod:`!custom` module: this is the + ``PyInit_custom`` function and the associated ``custom_module`` struct for + defining the module, and the ``custom_module_exec`` function to set up + a fresh module object. The first bit is:: @@ -171,18 +173,18 @@ implementation provided by the API function :c:func:`PyType_GenericNew`. :: .tp_new = PyType_GenericNew, Everything else in the file should be familiar, except for some code in -:c:func:`!PyInit_custom`:: +:c:func:`!custom_module_exec`:: - if (PyType_Ready(&CustomType) < 0) - return; + if (PyType_Ready(&CustomType) < 0) { + return -1; + } This initializes the :class:`!Custom` type, filling in a number of members to the appropriate default values, including :c:member:`~PyObject.ob_type` that we initially set to ``NULL``. :: if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; + return -1; } This adds the type to the module dictionary. This allows us to create @@ -275,7 +277,7 @@ be an instance of a subclass. The explicit cast to ``CustomObject *`` above is needed because we defined ``Custom_dealloc`` to take a ``PyObject *`` argument, as the ``tp_dealloc`` function pointer expects to receive a ``PyObject *`` argument. - By assigning to the the ``tp_dealloc`` slot of a type, we declare + By assigning to the ``tp_dealloc`` slot of a type, we declare that it can only be called with instances of our ``CustomObject`` class, so the cast to ``(CustomObject *)`` is safe. This is object-oriented polymorphism, in C! @@ -875,27 +877,22 @@ but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_ The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base` specifying the type's concrete base class. Due to cross-platform compiler issues, you can't fill that field directly with a reference to -:c:type:`PyList_Type`; it should be done later in the module initialization +:c:type:`PyList_Type`; it should be done in the :c:data:`Py_mod_exec` function:: - PyMODINIT_FUNC - PyInit_sublist(void) + static int + sublist_module_exec(PyObject *m) { - PyObject* m; SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(m); - return NULL; + if (PyType_Ready(&SubListType) < 0) { + return -1; } - return m; + if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { + return -1; + } + + return 0; } Before calling :c:func:`PyType_Ready`, the type structure must have the diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index 56aa44e4e58..a97c6182553 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -121,7 +121,7 @@ When creating DLLs in Windows, you can use the CPython library in two ways: :file:`Python.h` triggers an implicit, configure-aware link with the library. The header file chooses :file:`pythonXY_d.lib` for Debug, :file:`pythonXY.lib` for Release, and :file:`pythonX.lib` for Release with - the `Limited API `_ enabled. + the :ref:`Limited API ` enabled. To build two DLLs, spam and ni (which uses C functions found in spam), you could use these commands:: diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index e2710fab9cf..ac0aa81e56b 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -420,10 +420,12 @@ strings representing the files in the current directory. Functions which operate on this output would generally not break if you added another file or two to the directory. -Tuples are immutable, meaning that once a tuple has been created, you can't -replace any of its elements with a new value. Lists are mutable, meaning that -you can always change a list's elements. Only immutable elements can be used as -dictionary keys, and hence only tuples and not lists can be used as keys. +Tuples are :term:`immutable`, meaning that once a tuple has been created, you can't +replace any of its elements with a new value. Lists are :term:`mutable`, meaning that +you can always change a list's elements. Only :term:`hashable` objects can +be used as dictionary keys. Most immutable types are hashable, which is why +tuples, but not lists, can be used as keys. Note, however, that a tuple is +only hashable if all of its elements are hashable. How are lists implemented in CPython? @@ -589,9 +591,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`!list.append` method is expected to add new elements +program. For example, the :meth:`list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`!list.append` implementation will actually do this correctly, but it's +your :meth:`list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 3147fda7c37..1d5abed2317 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -37,24 +37,9 @@ Writing C is hard; are there any alternatives? ---------------------------------------------- There are a number of alternatives to writing your own C extensions, depending -on what you're trying to do. - -.. XXX make sure these all work - -`Cython `_ and its relative `Pyrex -`_ are compilers -that accept a slightly modified form of Python and generate the corresponding -C code. Cython and Pyrex make it possible to write an extension without having -to learn Python's C API. - -If you need to interface to some C or C++ library for which no Python extension -currently exists, you can try wrapping the library's data types and functions -with a tool such as `SWIG `_. `SIP -`__, `CXX -`_ `Boost -`_, or `Weave -`_ are also -alternatives for wrapping C++ libraries. +on what you're trying to do. :ref:`Recommended third party tools ` +offer both simpler and more sophisticated approaches to creating C and C++ +extensions for Python. How can I execute arbitrary Python statements from C? diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 578777d7f23..698f2117ff5 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -186,7 +186,7 @@ How do I get documentation on Python? ------------------------------------- The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are +at https://docs.python.org/3/. EPUB, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. The documentation is written in reStructuredText and processed by `the Sphinx diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 9f9e4fab685..6f9dfa8616e 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -454,7 +454,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`!append`, the content of the mutable object has +After the call to :meth:`~sequence.append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -1397,9 +1397,9 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list -and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for :meth:`!list.extend`:: +and (b) for lists, :meth:`!__iadd__` is equivalent to calling +:meth:`~sequence.extend` on the list and returning the list. +That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`list.extend`:: >>> a_list = [] >>> a_list += [1] diff --git a/Doc/faq/python-video-icon.png b/Doc/faq/python-video-icon.png deleted file mode 100644 index 265da50c7b3..00000000000 Binary files a/Doc/faq/python-video-icon.png and /dev/null differ diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 0b26e18efd7..a4066d42927 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -21,7 +21,9 @@ Glossary right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. - * The :const:`Ellipsis` built-in constant. + .. index:: single: ...; ellipsis literal + + * The three dots form of the :ref:`Ellipsis ` object. abstract base class Abstract base classes complement :term:`duck-typing` by @@ -107,7 +109,7 @@ Glossary statements. asynchronous generator iterator - An object created by a :term:`asynchronous generator` function. + An object created by an :term:`asynchronous generator` function. This is an :term:`asynchronous iterator` which when called using the :meth:`~object.__anext__` method returns an awaitable object which will execute @@ -355,6 +357,12 @@ Glossary tasks (see :mod:`asyncio`) associate each task with a context which becomes the current context whenever the task starts or resumes execution. + cyclic isolate + A subgroup of one or more objects that reference each other in a reference + cycle, but are not referenced by objects outside the group. The goal of + the :term:`cyclic garbage collector ` is to identify these groups and break the reference + cycles so that the memory can be reclaimed. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for @@ -429,6 +437,11 @@ Glossary with :term:`abstract base classes `.) Instead, it typically employs :func:`hasattr` tests or :term:`EAFP` programming. + dunder + An informal short-hand for "double underscore", used when talking about a + :term:`special method`. For example, ``__init__`` is often pronounced + "dunder init". + EAFP Easier to ask for forgiveness than permission. This common Python coding style assumes the existence of valid keys or attributes and catches @@ -456,7 +469,8 @@ Glossary core and with user code. f-string - String literals prefixed with ``'f'`` or ``'F'`` are commonly called + f-strings + String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -1011,6 +1025,15 @@ Glossary applied to all scopes, only those relying on a known set of local and nonlocal variable names are restricted to optimized scopes. + optional module + An :term:`extension module` that is part of the :term:`standard library`, + but may be absent in some builds of :term:`CPython`, + usually due to missing third-party libraries or because the module + is not available for a given platform. + + See :ref:`optional-module-requirements` for a list of optional modules + that require third-party libraries. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -1202,6 +1225,11 @@ Glossary :func:`sys.getrefcount` function to return the reference count for a particular object. + In :term:`CPython`, reference counts are not considered to be stable + or well-defined values; the number of references to an object, and how + that number is affected by Python code, may be different between + versions. + regular package A traditional :term:`package`, such as a directory containing an ``__init__.py`` file. @@ -1232,8 +1260,9 @@ Glossary The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding - :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and - :meth:`~object.__reversed__`. Types that implement this expanded + :meth:`~sequence.count`, :meth:`~sequence.index`, + :meth:`~object.__contains__`, and :meth:`~object.__reversed__`. + Types that implement this expanded interface can be registered explicitly using :func:`~abc.ABCMeta.register`. For more documentation on sequence methods generally, see @@ -1274,6 +1303,16 @@ Glossary and ending with double underscores. Special methods are documented in :ref:`specialnames`. + standard library + The collection of :term:`packages `, :term:`modules ` + and :term:`extension modules ` distributed as a part + of the official Python interpreter package. The exact membership of the + collection may vary based on platform, available system libraries, or + other criteria. Documentation can be found at :ref:`library-index`. + + See also :data:`sys.stdlib_module_names` for a list of all possible + standard library module names. + statement A statement is part of a suite (a "block" of code). A statement is either an :term:`expression` or one of several constructs with a keyword, such @@ -1284,6 +1323,9 @@ Glossary issues such as incorrect types. See also :term:`type hints ` and the :mod:`typing` module. + stdlib + An abbreviation of :term:`standard library`. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong @@ -1298,6 +1340,12 @@ Glossary See also :term:`borrowed reference`. + t-string + t-strings + String literals prefixed with ``t`` or ``T`` are commonly called + "t-strings" which is short for + :ref:`template string literals `. + text encoding A string in Python is a sequence of Unicode code points (in range ``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be @@ -1443,6 +1491,11 @@ Glossary A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. + walrus operator + A light-hearted way to refer to the :ref:`assignment expression + ` operator ``:=`` because it looks a bit like a + walrus if you turn your head. + Zen of Python Listing of Python design principles and philosophies that are helpful in understanding and using the language. The listing can be found by typing diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst new file mode 100644 index 00000000000..3adfedbf410 --- /dev/null +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -0,0 +1,619 @@ +.. _a-conceptual-overview-of-asyncio: + +**************************************** +A conceptual overview of :mod:`!asyncio` +**************************************** + +This :ref:`HOWTO ` article seeks to help you build a sturdy mental +model of how :mod:`asyncio` fundamentally works, helping you understand the +how and why behind the recommended patterns. + +You might be curious about some key :mod:`!asyncio` concepts. +By the end of this article, you'll be able to comfortably answer these questions: + +- What's happening behind the scenes when an object is awaited? +- How does :mod:`!asyncio` differentiate between a task which doesn't need + CPU time (such as a network request or file read) as opposed to a task that + does (such as computing n-factorial)? +- How to write an asynchronous variant of an operation, such as + an async sleep or database request. + +.. seealso:: + + * The `guide `_ that inspired this HOWTO article, by Alexander Nordin. + * This in-depth `YouTube tutorial series `_ on + ``asyncio`` created by Python core team member, Łukasz Langa. + * `500 Lines or Less: A Web Crawler With asyncio Coroutines `_ by A. + Jesse Jiryu Davis and Guido van Rossum. + +-------------------------------------------- +A conceptual overview part 1: the high-level +-------------------------------------------- + +In part 1, we'll cover the main, high-level building blocks of :mod:`!asyncio`: +the event loop, coroutine functions, coroutine objects, tasks, and ``await``. + +========== +Event loop +========== + +Everything in :mod:`!asyncio` happens relative to the event loop. +It's the star of the show, but prefers to work behind the scenes, managing +and coordinating resources. +It's like an orchestra conductor. +Some power is explicitly granted to it, but a lot of its ability to get things +done comes from the respect and cooperation of its band members. + +In more technical terms, the event loop contains a collection of jobs to be run. +Some jobs are added directly by you, and some indirectly by :mod:`!asyncio`. +The event loop takes a job from its backlog of work and invokes it (or "gives +it control"), similar to calling a function, and then that job runs. +Once it pauses or completes, it returns control to the event loop. +The event loop will then select another job from its pool and invoke it. +You can *roughly* think of the collection of jobs as a queue: jobs are added and +then processed one at a time, generally (but not always) in order. +This process repeats indefinitely, with the event loop cycling endlessly +onwards. +If there are no more jobs pending execution, the event loop is smart enough to +rest and avoid needlessly wasting CPU cycles, and will come back when there's +more work to be done, such as when I/O operations complete or timers expire. + +Effective execution relies on jobs sharing well and cooperating; a greedy job +could hog control and leave the other jobs to starve, rendering the overall +event loop approach rather useless. + +:: + + import asyncio + + # This creates an event loop and indefinitely cycles through + # its collection of jobs. + event_loop = asyncio.new_event_loop() + event_loop.run_forever() + +===================================== +Asynchronous functions and coroutines +===================================== + +This is a basic, boring Python function:: + + def hello_printer(): + print( + "Hi, I am a lowly, simple printer, though I have all I " + "need in life -- \nfresh paper and my dearly beloved octopus " + "partner in crime." + ) + +Calling a regular function invokes its logic or body:: + + >>> hello_printer() + Hi, I am a lowly, simple printer, though I have all I need in life -- + fresh paper and my dearly beloved octopus partner in crime. + +The :ref:`async def `, as opposed to just a plain ``def``, makes +this an asynchronous function (or "coroutine function"). +Calling it creates and returns a :ref:`coroutine ` object. + +:: + + async def loudmouth_penguin(magic_number: int): + print( + "I am a super special talking penguin. Far cooler than that printer. " + f"By the way, my lucky number is: {magic_number}." + ) + +Calling the async function, ``loudmouth_penguin``, does not execute the print statement; +instead, it creates a coroutine object:: + + >>> loudmouth_penguin(magic_number=3) + + +The terms "coroutine function" and "coroutine object" are often conflated +as coroutine. +That can be confusing! +In this article, coroutine specifically refers to a coroutine object, or more +precisely, an instance of :data:`types.CoroutineType` (native coroutine). +Note that coroutines can also exist as instances of +:class:`collections.abc.Coroutine` -- a distinction that matters for type +checking. + +A coroutine represents the function's body or logic. +A coroutine has to be explicitly started; again, merely creating the coroutine +does not start it. +Notably, the coroutine can be paused and resumed at various points within the +function's body. +That pausing and resuming ability is what allows for asynchronous behavior! + +Coroutines and coroutine functions were built by leveraging the functionality +of :term:`generators ` and +:term:`generator functions `. +Recall, a generator function is a function that :keyword:`yield`\s, like this +one:: + + def get_random_number(): + # This would be a bad random number generator! + print("Hi") + yield 1 + print("Hello") + yield 7 + print("Howdy") + yield 4 + ... + +Similar to a coroutine function, calling a generator function does not run it. +Instead, it creates a generator object:: + + >>> get_random_number() + + +You can proceed to the next ``yield`` of a generator by using the +built-in function :func:`next`. +In other words, the generator runs, then pauses. +For example:: + + >>> generator = get_random_number() + >>> next(generator) + Hi + 1 + >>> next(generator) + Hello + 7 + +===== +Tasks +===== + +Roughly speaking, :ref:`tasks ` are coroutines (not coroutine +functions) tied to an event loop. +A task also maintains a list of callback functions whose importance will become +clear in a moment when we discuss :keyword:`await`. + +Creating a task automatically schedules it for execution (by adding a +callback to run it in the event loop's to-do list, that is, collection of jobs). +The recommended way to create tasks is via :func:`asyncio.create_task`. + +:mod:`!asyncio` automatically associates tasks with the event loop for you. +This automatic association was purposely designed into :mod:`!asyncio` for +the sake of simplicity. +Without it, you'd have to keep track of the event loop object and pass it to +any coroutine function that wants to create tasks, adding redundant clutter +to your code. + +:: + + coroutine = loudmouth_penguin(magic_number=5) + # This creates a Task object and schedules its execution via the event loop. + task = asyncio.create_task(coroutine) + +Earlier, we manually created the event loop and set it to run forever. +In practice, it's recommended to use (and common to see) :func:`asyncio.run`, +which takes care of managing the event loop and ensuring the provided +coroutine finishes before advancing. +For example, many async programs follow this setup:: + + import asyncio + + async def main(): + # Perform all sorts of wacky, wild asynchronous things... + ... + + if __name__ == "__main__": + asyncio.run(main()) + # The program will not reach the following print statement until the + # coroutine main() finishes. + print("coroutine main() is done!") + +It's important to be aware that the task itself is not added to the event loop, +only a callback to the task is. +This matters if the task object you created is garbage collected before it's +called by the event loop. +For example, consider this program: + +.. code-block:: + :linenos: + + async def hello(): + print("hello!") + + async def main(): + asyncio.create_task(hello()) + # Other asynchronous instructions which run for a while + # and cede control to the event loop... + ... + + asyncio.run(main()) + +Because there's no reference to the task object created on line 5, it *might* +be garbage collected before the event loop invokes it. +Later instructions in the coroutine ``main()`` hand control back to the event +loop so it can invoke other jobs. +When the event loop eventually tries to run the task, it might fail and +discover the task object does not exist! +This can also happen even if a coroutine keeps a reference to a task but +completes before that task finishes. +When the coroutine exits, local variables go out of scope and may be subject +to garbage collection. +In practice, ``asyncio`` and Python's garbage collector work pretty hard to +ensure this sort of thing doesn't happen. +But that's no reason to be reckless! + +===== +await +===== + +:keyword:`await` is a Python keyword that's commonly used in one of two +different ways:: + + await task + await coroutine + +In a crucial way, the behavior of ``await`` depends on the type of object +being awaited. + +^^^^^^^^^^^^^^ +Awaiting tasks +^^^^^^^^^^^^^^ + +Awaiting a task will cede control from the current task or coroutine to +the event loop. +In the process of relinquishing control, a few important things happen. +We'll use the following code example to illustrate:: + + async def plant_a_tree(): + dig_the_hole_task = asyncio.create_task(dig_the_hole()) + await dig_the_hole_task + + # Other instructions associated with planting a tree. + ... + +In this example, imagine the event loop has passed control to the start of the +coroutine ``plant_a_tree()``. +As seen above, the coroutine creates a task and then awaits it. +The ``await dig_the_hole_task`` instruction adds a callback (which will resume +``plant_a_tree()``) to the ``dig_the_hole_task`` object's list of callbacks. +And then, the instruction cedes control to the event loop. +Some time later, the event loop will pass control to ``dig_the_hole_task`` +and the task will finish whatever it needs to do. +Once the task finishes, it will add its various callbacks to the event loop, +in this case, a call to resume ``plant_a_tree()``. + +Generally speaking, when the awaited task finishes (``dig_the_hole_task``), +the original task or coroutine (``plant_a_tree()``) is added back to the event +loop's to-do list to be resumed. + +This is a basic, yet reliable mental model. +In practice, the control handoffs are slightly more complex, but not by much. +In part 2, we'll walk through the details that make this possible. + +^^^^^^^^^^^^^^^^^^^ +Awaiting coroutines +^^^^^^^^^^^^^^^^^^^ + +**Unlike tasks, awaiting a coroutine does not hand control back to the event +loop!** +Wrapping a coroutine in a task first, then awaiting that would cede +control. +The behavior of ``await coroutine`` is effectively the same as invoking a +regular, synchronous Python function. +Consider this program:: + + import asyncio + + async def coro_a(): + print("I am coro_a(). Hi!") + + async def coro_b(): + print("I am coro_b(). I sure hope no one hogs the event loop...") + + async def main(): + task_b = asyncio.create_task(coro_b()) + num_repeats = 3 + for _ in range(num_repeats): + await coro_a() + await task_b + + asyncio.run(main()) + +The first statement in the coroutine ``main()`` creates ``task_b`` and schedules +it for execution via the event loop. +Then, ``coro_a()`` is repeatedly awaited. Control never cedes to the +event loop, which is why we see the output of all three ``coro_a()`` +invocations before ``coro_b()``'s output: + +.. code-block:: none + + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_b(). I sure hope no one hogs the event loop... + +If we change ``await coro_a()`` to ``await asyncio.create_task(coro_a())``, the +behavior changes. +The coroutine ``main()`` cedes control to the event loop with that statement. +The event loop then proceeds through its backlog of work, calling ``task_b`` +and then the task which wraps ``coro_a()`` before resuming the coroutine +``main()``. + +.. code-block:: none + + I am coro_b(). I sure hope no one hogs the event loop... + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + +This behavior of ``await coroutine`` can trip a lot of people up! +That example highlights how using only ``await coroutine`` could +unintentionally hog control from other tasks and effectively stall the event +loop. +:func:`asyncio.run` can help you detect such occurrences via the +``debug=True`` flag, which enables +:ref:`debug mode `. +Among other things, it will log any coroutines that monopolize execution for +100ms or longer. + +The design intentionally trades off some conceptual clarity around usage of +``await`` for improved performance. +Each time a task is awaited, control needs to be passed all the way up the +call stack to the event loop. +Then, the event loop needs to manage its internal state and work through +its processing logic to resume the next job. +That might sound minor, but in a large program with many ``await``\ s, that +overhead can add up to a non-negligible performance drag. + +------------------------------------------------ +A conceptual overview part 2: the nuts and bolts +------------------------------------------------ + +Part 2 goes into detail on the mechanisms :mod:`!asyncio` uses to manage +control flow. +This is where the magic happens. +You'll come away from this section knowing what ``await`` does behind the scenes +and how to make your own asynchronous operators. + +================================ +The inner workings of coroutines +================================ + +:mod:`!asyncio` leverages four components of Python to pass +around control. + +:meth:`coroutine.send(arg) ` is the method used to start or +resume a coroutine. +If the coroutine was paused and is now being resumed, the argument ``arg`` +will be sent in as the return value of the ``yield`` statement which originally +paused it. +If the coroutine is being used for the first time (as opposed to being resumed), +``arg`` must be ``None``. + +.. code-block:: + :linenos: + + class Rock: + def __await__(self): + value_sent_in = yield 7 + print(f"Rock.__await__ resuming with value: {value_sent_in}.") + return value_sent_in + + async def main(): + print("Beginning coroutine main().") + rock = Rock() + print("Awaiting rock...") + value_from_rock = await rock + print(f"Coroutine received value: {value_from_rock} from rock.") + return 23 + + coroutine = main() + intermediate_result = coroutine.send(None) + print(f"Coroutine paused and returned intermediate value: {intermediate_result}.") + + print(f"Resuming coroutine and sending in value: 42.") + try: + coroutine.send(42) + except StopIteration as e: + returned_value = e.value + print(f"Coroutine main() finished and provided value: {returned_value}.") + +:ref:`yield `, as usual, pauses execution and returns control +to the caller. +In the example above, the ``yield``, on line 3, is called by +``... = await rock`` on line 11. +More broadly speaking, ``await`` calls the :meth:`~object.__await__` method of +the given object. +``await`` also does one more very special thing: it propagates (or "passes +along") any ``yield``\ s it receives up the call chain. +In this case, that's back to ``... = coroutine.send(None)`` on line 16. + +The coroutine is resumed via the ``coroutine.send(42)`` call on line 21. +The coroutine picks back up from where it ``yield``\ ed (or paused) on line 3 +and executes the remaining statements in its body. +When a coroutine finishes, it raises a :exc:`StopIteration` exception with the +return value attached in the :attr:`~StopIteration.value` attribute. + +That snippet produces this output: + +.. code-block:: none + + Beginning coroutine main(). + Awaiting rock... + Coroutine paused and returned intermediate value: 7. + Resuming coroutine and sending in value: 42. + Rock.__await__ resuming with value: 42. + Coroutine received value: 42 from rock. + Coroutine main() finished and provided value: 23. + +It's worth pausing for a moment here and making sure you followed the various +ways that control flow and values were passed. A lot of important ideas were +covered and it's worth ensuring your understanding is firm. + +The only way to yield (or effectively cede control) from a coroutine is to +``await`` an object that ``yield``\ s in its ``__await__`` method. +That might sound odd to you. You might be thinking: + + 1. What about a ``yield`` directly within the coroutine function? The + coroutine function becomes an + :ref:`async generator function `, a + different beast entirely. + + 2. What about a :ref:`yield from ` within the coroutine function to a (plain) + generator? + That causes the error: ``SyntaxError: yield from not allowed in a coroutine.`` + This was intentionally designed for the sake of simplicity -- mandating only + one way of using coroutines. + Despite that, ``yield from`` and ``await`` effectively do the same thing. + Initially ``yield`` was barred as well, but was re-accepted to allow for + async generators. + +======= +Futures +======= + +A :ref:`future ` is an object meant to represent a +computation's status and result. +The term is a nod to the idea of something still to come or not yet happened, +and the object is a way to keep an eye on that something. + +A future has a few important attributes. One is its state, which can be either +"pending", "cancelled", or "done". +Another is its result, which is set when the state transitions to done. +Unlike a coroutine, a future does not represent the actual computation to be +done; instead, it represents the status and result of that computation, kind of +like a status light (red, yellow, or green) or indicator. + +:class:`asyncio.Task` subclasses :class:`asyncio.Future` in order to gain +these various capabilities. +The prior section said tasks store a list of callbacks, which wasn't entirely +accurate. +It's actually the ``Future`` class that implements this logic, which ``Task`` +inherits. + +Futures may also be used directly (not via tasks). +Tasks mark themselves as done when their coroutine is complete. +Futures are much more versatile and will be marked as done when you say so. +In this way, they're the flexible interface for you to make your own conditions +for waiting and resuming. + +======================== +A homemade asyncio.sleep +======================== + +We'll go through an example of how you could leverage a future to create your +own variant of asynchronous sleep (``async_sleep``) which mimics +:func:`asyncio.sleep`. + +This snippet registers a few tasks with the event loop and then awaits the task +created by ``asyncio.create_task``, which wraps the ``async_sleep(3)`` coroutine. +We want that task to finish only after three seconds have elapsed, but without +preventing other tasks from running. + +:: + + async def other_work(): + print("I like work. Work work.") + + async def main(): + # Add a few other tasks to the event loop, so there's something + # to do while asynchronously sleeping. + work_tasks = [ + asyncio.create_task(other_work()), + asyncio.create_task(other_work()), + asyncio.create_task(other_work()) + ] + print( + "Beginning asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + await asyncio.create_task(async_sleep(3)) + print( + "Done asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + # asyncio.gather effectively awaits each task in the collection. + await asyncio.gather(*work_tasks) + + +Below, we use a future to enable custom control over when that task will be +marked as done. +If :meth:`future.set_result() ` (the method +responsible for marking that future as done) is never called, then this task +will never finish. +We've also enlisted the help of another task, which we'll see in a moment, that +will monitor how much time has elapsed and, accordingly, call +``future.set_result()``. + +:: + + async def async_sleep(seconds: float): + future = asyncio.Future() + time_to_wake = time.time() + seconds + # Add the watcher-task to the event loop. + watcher_task = asyncio.create_task(_sleep_watcher(future, time_to_wake)) + # Block until the future is marked as done. + await future + +Below, we use a rather bare ``YieldToEventLoop()`` object to ``yield`` +from its ``__await__`` method, ceding control to the event loop. +This is effectively the same as calling ``asyncio.sleep(0)``, but this approach +offers more clarity, not to mention it's somewhat cheating to use +``asyncio.sleep`` when showcasing how to implement it! + +As usual, the event loop cycles through its tasks, giving them control +and receiving control back when they pause or finish. +The ``watcher_task``, which runs the coroutine ``_sleep_watcher(...)``, will +be invoked once per full cycle of the event loop. +On each resumption, it'll check the time and if not enough has elapsed, then +it'll pause once again and hand control back to the event loop. +Once enough time has elapsed, ``_sleep_watcher(...)`` +marks the future as done and completes by exiting its +infinite ``while`` loop. +Given this helper task is only invoked once per cycle of the event loop, +you'd be correct to note that this asynchronous sleep will sleep *at least* +three seconds, rather than exactly three seconds. +Note this is also true of ``asyncio.sleep``. + +:: + + class YieldToEventLoop: + def __await__(self): + yield + + async def _sleep_watcher(future, time_to_wake): + while True: + if time.time() >= time_to_wake: + # This marks the future as done. + future.set_result(None) + break + else: + await YieldToEventLoop() + +Here is the full program's output: + +.. code-block:: none + + $ python custom-async-sleep.py + Beginning asynchronous sleep at time: 14:52:22. + I like work. Work work. + I like work. Work work. + I like work. Work work. + Done asynchronous sleep at time: 14:52:25. + +You might feel this implementation of asynchronous sleep was unnecessarily +convoluted. +And, well, it was. +The example was meant to showcase the versatility of futures with a simple +example that could be mimicked for more complex needs. +For reference, you could implement it without futures, like so:: + + async def simpler_async_sleep(seconds): + time_to_wake = time.time() + seconds + while True: + if time.time() >= time_to_wake: + return + else: + await YieldToEventLoop() + +But that's all for now. Hopefully you're ready to more confidently dive into +some async programming or check out advanced topics in the +:mod:`rest of the documentation `. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 78f3704ba5d..d7deb6c6bc1 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -248,4 +248,9 @@ quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or :func:`inspect.get_annotations` on Python 3.10+. On earlier versions of Python, you can avoid these bugs by accessing the annotations from the class's :attr:`~type.__dict__` -(e.g., ``cls.__dict__.get('__annotations__', None)``). +(for example, ``cls.__dict__.get('__annotations__', None)``). + +In some versions of Python, instances of classes may have an ``__annotations__`` +attribute. However, this is not supported functionality. If you need the +annotations of an instance, you can use :func:`type` to access its class +(for example, ``annotationlib.get_annotations(type(myinstance))`` on Python 3.14+). diff --git a/Doc/howto/cporting.rst b/Doc/howto/cporting.rst index 7773620b40b..cf857aed042 100644 --- a/Doc/howto/cporting.rst +++ b/Doc/howto/cporting.rst @@ -14,13 +14,11 @@ We recommend the following resources for porting extension modules to Python 3: module. * The `Porting guide`_ from the *py3c* project provides opinionated suggestions with supporting code. -* The `Cython`_ and `CFFI`_ libraries offer abstractions over - Python's C API. +* :ref:`Recommended third party tools ` offer abstractions over + the Python's C API. Extensions generally need to be re-written to use one of them, but the library then handles differences between various Python versions and implementations. .. _Migrating C extensions: http://python3porting.com/cextensions.html .. _Porting guide: https://py3c.readthedocs.io/en/latest/guide.html -.. _Cython: https://cython.org/ -.. _CFFI: https://cffi.readthedocs.io/en/latest/ diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 6994a5328e8..816639552d7 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -161,6 +161,8 @@ your terminal won't be left in a funny state on exception and you'll be able to read the exception's message and traceback. +.. _windows-and-pads: + Windows and Pads ================ diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index f6c3e473f1c..9d5a9ac8b71 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -420,7 +420,7 @@ Here are three practical data validation utilities: def validate(self, value): if not isinstance(value, str): - raise TypeError(f'Expected {value!r} to be an str') + raise TypeError(f'Expected {value!r} to be a str') if self.minsize is not None and len(value) < self.minsize: raise ValueError( f'Expected {value!r} to be no smaller than {self.minsize!r}' diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 6441b7aed1e..7713aede6d5 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -990,9 +990,9 @@ Supported ``_sunder_`` names from the final class - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden -- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing member. -- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an existing member. See `MultiValueEnum`_ for an example. .. note:: diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 3f6ee517050..83eba8cfea3 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -6,8 +6,8 @@ C API Extension Support for Free Threading ****************************************** -Starting with the 3.13 release, CPython has experimental support for running -with the :term:`global interpreter lock` (GIL) disabled in a configuration +Starting with the 3.13 release, CPython has support for running with +the :term:`global interpreter lock` (GIL) disabled in a configuration called :term:`free threading`. This document describes how to adapt C API extensions to support free threading. @@ -23,6 +23,14 @@ You can use it to enable code that only runs under the free-threaded build:: /* code that only runs in the free-threaded build */ #endif +.. note:: + + On Windows, this macro is not defined automatically, but must be specified + to the compiler when building. The :func:`sysconfig.get_config_var` function + can be used to determine whether the current running interpreter had the + macro defined. + + Module Initialization ===================== @@ -37,9 +45,12 @@ single-phase initialization. Multi-Phase Initialization .......................... -Extensions that use multi-phase initialization (i.e., -:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the -module definition. If your extension supports older versions of CPython, +Extensions that use :ref:`multi-phase initialization ` +(functions like :c:func:`PyModuleDef_Init`, +:c:func:`PyModExport_* ` export hook, +:c:func:`PyModule_FromSlotsAndSpec`) should add a +:c:data:`Py_mod_gil` slot in the module definition. +If your extension supports older versions of CPython, you should guard the slot with a :c:data:`PY_VERSION_HEX` check. :: @@ -52,18 +63,12 @@ you should guard the slot with a :c:data:`PY_VERSION_HEX` check. {0, NULL} }; - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - .m_slots = module_slots, - ... - }; - Single-Phase Initialization ........................... -Extensions that use single-phase initialization (i.e., -:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +Extensions that use legacy :ref:`single-phase initialization ` +(that is, :c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to indicate that they support running with the GIL disabled. The function is only defined in the free-threaded build, so you should guard the call with ``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. @@ -153,6 +158,8 @@ that return :term:`strong references `. +===================================+===================================+ | :c:func:`PyList_GetItem` | :c:func:`PyList_GetItemRef` | +-----------------------------------+-----------------------------------+ +| :c:func:`PyList_GET_ITEM` | :c:func:`PyList_GetItemRef` | ++-----------------------------------+-----------------------------------+ | :c:func:`PyDict_GetItem` | :c:func:`PyDict_GetItemRef` | +-----------------------------------+-----------------------------------+ | :c:func:`PyDict_GetItemWithError` | :c:func:`PyDict_GetItemRef` | @@ -163,9 +170,9 @@ that return :term:`strong references `. +-----------------------------------+-----------------------------------+ | :c:func:`PyDict_Next` | none (see :ref:`PyDict_Next`) | +-----------------------------------+-----------------------------------+ -| :c:func:`PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | +| :c:func:`!PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | +-----------------------------------+-----------------------------------+ -| :c:func:`PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | +| :c:func:`!PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | +-----------------------------------+-----------------------------------+ | :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | +-----------------------------------+-----------------------------------+ @@ -193,7 +200,7 @@ Memory Allocation APIs Python's memory management C API provides functions in three different :ref:`allocation domains `: "raw", "mem", and "object". For thread-safety, the free-threaded build requires that only Python objects -are allocated using the object domain, and that all Python object are +are allocated using the object domain, and that all Python objects are allocated using that domain. This differs from the prior Python versions, where this was only a best practice and not a hard requirement. @@ -334,12 +341,12 @@ This means you cannot rely on nested critical sections to lock multiple objects at once, as the inner critical section may suspend the outer ones. Instead, use :c:macro:`Py_BEGIN_CRITICAL_SECTION2` to lock two objects simultaneously. -Note that the locks described above are only :c:type:`!PyMutex` based locks. +Note that the locks described above are only :c:type:`PyMutex` based locks. The critical section implementation does not know about or affect other locking mechanisms that might be in use, like POSIX mutexes. Also note that while -blocking on any :c:type:`!PyMutex` causes the critical sections to be +blocking on any :c:type:`PyMutex` causes the critical sections to be suspended, only the mutexes that are part of the critical sections are -released. If :c:type:`!PyMutex` is used without a critical section, it will +released. If :c:type:`PyMutex` is used without a critical section, it will not be released and therefore does not get the same deadlock avoidance. Important Considerations @@ -387,8 +394,9 @@ The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the free-threaded build, with the ``t`` suffix, such as ``python3.13t``. * `pypa/cibuildwheel `_ supports the - free-threaded build if you set - `CIBW_FREE_THREADED_SUPPORT `_. + free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded + wheels will be built by default. On Python 3.13, you will need to set + `CIBW_ENABLE to cpython-freethreading `_. Limited C API and Stable ABI ............................ diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index f7a894ac2cd..380c2be0495 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -1,18 +1,19 @@ .. _freethreading-python-howto: -********************************************** -Python experimental support for free threading -********************************************** +********************************* +Python support for free threading +********************************* -Starting with the 3.13 release, CPython has experimental support for a build of +Starting with the 3.13 release, CPython has support for a build of Python called :term:`free threading` where the :term:`global interpreter lock` (GIL) is disabled. Free-threaded execution allows for full utilization of the available processing power by running threads in parallel on available CPU cores. While not all software will benefit from this automatically, programs designed with threading in mind will run faster on multi-core hardware. -**The free-threaded mode is experimental** and work is ongoing to improve it: -expect some bugs and a substantial single-threaded performance hit. +Some third-party packages, in particular ones +with an :term:`extension module`, may not be ready for use in a +free-threaded build, and will re-enable the :term:`GIL`. This document describes the implications of free threading for Python code. See :ref:`freethreading-extensions-howto` for information on @@ -32,7 +33,7 @@ optionally support installing free-threaded Python binaries. The installers are available at https://www.python.org/downloads/. For information on other platforms, see the `Installing a Free-Threaded Python -`_, a +`_, a community-maintained installation guide for installing free-threaded Python. When building CPython from source, the :option:`--disable-gil` configure option @@ -43,7 +44,7 @@ Identifying free-threaded Python ================================ To check if the current interpreter supports free-threading, :option:`python -VV <-V>` -and :data:`sys.version` contain "experimental free-threading build". +and :data:`sys.version` contain "free-threading build". The new :func:`sys._is_gil_enabled` function can be used to check whether the GIL is actually disabled in the running process. @@ -98,60 +99,42 @@ This section describes known limitations of the free-threaded CPython build. Immortalization --------------- -The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +In the free-threaded build, some objects are :term:`immortal`. Immortal objects are not deallocated and have reference counts that are never modified. This is done to avoid reference count contention that would prevent efficient multi-threaded scaling. -An object will be made immortal when a new thread is started for the first time -after the main thread is running. The following objects are immortalized: +As of the 3.14 release, immortalization is limited to: -* :ref:`function ` objects declared at the module level -* :ref:`method ` descriptors -* :ref:`code ` objects -* :term:`module` objects and their dictionaries -* :ref:`classes ` (type objects) - -Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage. This is expected to be -addressed in the 3.14 release. - -Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized. This behavior is -expected to remain in the 3.14 free-threaded build. +* Code constants: numeric literals, string literals, and tuple literals + composed of other constants. +* Strings interned by :func:`sys.intern`. Frame objects ------------- -It is not safe to access :ref:`frame ` objects from other -threads and doing so may cause your program to crash . This means that -:func:`sys._current_frames` is generally not safe to use in a free-threaded -build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` -are generally safe as long as the resulting frame object is not passed to -another thread. +It is not safe to access :attr:`frame.f_locals` from a :ref:`frame ` +object if that frame is currently executing in another thread, and doing so may +crash the interpreter. + Iterators --------- -Sharing the same iterator object between multiple threads is generally not -safe and threads may see duplicate or missing elements when iterating or crash -the interpreter. +It is generally not thread-safe to access the same iterator object from +multiple threads concurrently, and threads may see duplicate or missing +elements. Single-threaded performance --------------------------- The free-threaded build has additional overhead when executing Python code -compared to the default GIL-enabled build. In 3.13, this overhead is about -40% on the `pyperformance `_ suite. -Programs that spend most of their time in C extensions or I/O will see -less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. We expect -to re-enable it in a thread-safe way in the 3.14 release. This overhead is -expected to be reduced in upcoming Python release. We are aiming for an -overhead of 10% or less on the pyperformance suite compared to the default -GIL-enabled build. +compared to the default GIL-enabled build. The amount of overhead depends +on the workload and hardware. On the pyperformance benchmark suite, the +average overhead ranges from about 1% on macOS aarch64 to 8% on x86-64 Linux +systems. Behavioral changes diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 1f0608fb0fc..552514063c9 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -4,7 +4,7 @@ Functional Programming HOWTO ******************************** -:Author: A. M. Kuchling +:Author: \A. M. Kuchling :Release: 0.32 In this document, we'll take a tour of Python's features suitable for @@ -372,8 +372,8 @@ have the form:: for expr2 in sequence2 if condition2 for expr3 in sequence3 - ... if condition3 + ... for exprN in sequenceN if conditionN ) @@ -602,7 +602,7 @@ generators: raise an exception inside the generator; the exception is raised by the ``yield`` expression where the generator's execution is paused. -* :meth:`~generator.close` raises a :exc:`GeneratorExit` exception inside the +* :meth:`~generator.close` sends a :exc:`GeneratorExit` exception to the generator to terminate the iteration. On receiving this exception, the generator's code must either raise :exc:`GeneratorExit` or :exc:`StopIteration`; catching the exception and doing anything else is @@ -1217,7 +1217,7 @@ flow inside a program. The book uses Scheme for its examples, but many of the design approaches described in these chapters are applicable to functional-style Python code. -https://www.defmacro.org/ramblings/fp.html: A general introduction to functional +https://defmacro.org/2006/06/19/fp.html: A general introduction to functional programming that uses Java examples and has a lengthy historical introduction. https://en.wikipedia.org/wiki/Functional_programming: General Wikipedia entry diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index f350141004c..81fc7e63f35 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -1,3 +1,5 @@ +.. _how-tos: + *************** Python HOWTOs *************** @@ -11,6 +13,7 @@ Python Library Reference. :maxdepth: 1 :hidden: + a-conceptual-overview-of-asyncio.rst cporting.rst curses.rst descriptor.rst @@ -38,6 +41,7 @@ Python Library Reference. General: +* :ref:`a-conceptual-overview-of-asyncio` * :ref:`annotations-howto` * :ref:`argparse-tutorial` * :ref:`descriptorhowto` diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 6e03ef20a21..b3db1189e5d 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -269,6 +269,8 @@ should instead read: (assuming a :ref:`debug build ` of CPython 3.6) +.. _static-markers: + Available static markers ------------------------ diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index a636e06bda8..6092c75f48f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -168,7 +168,7 @@ possible, consider explicit locking. If it is necessary to use process-global state, the simplest way to avoid issues with multiple interpreters is to explicitly prevent a module from being loaded more than once per process—see -`Opt-Out: Limiting to One Module Object per Process`_. +:ref:`isolating-extensions-optout`. Managing Per-Module State @@ -207,6 +207,8 @@ An example of a module with per-module state is currently available as example module initialization shown at the bottom of the file. +.. _isolating-extensions-optout: + Opt-Out: Limiting to One Module Object per Process -------------------------------------------------- @@ -215,21 +217,36 @@ multiple interpreters correctly. If this is not yet the case for your module, you can explicitly make your module loadable only once per process. For example:: + // A process-wide flag static int loaded = 0; + // Mutex to provide thread safety (only needed for free-threaded Python) + static PyMutex modinit_mutex = {0}; + static int exec_module(PyObject* module) { + PyMutex_Lock(&modinit_mutex); if (loaded) { + PyMutex_Unlock(&modinit_mutex); PyErr_SetString(PyExc_ImportError, "cannot load module more than once per process"); return -1; } loaded = 1; + PyMutex_Unlock(&modinit_mutex); // ... rest of initialization } +If your module's :c:member:`PyModuleDef.m_clear` function is able to prepare +for future re-initialization, it should clear the ``loaded`` flag. +In this case, your module won't support multiple instances existing +*concurrently*, but it will, for example, support being loaded after +Python runtime shutdown (:c:func:`Py_FinalizeEx`) and re-initialization +(:c:func:`Py_Initialize`). + + Module State Access from Functions ---------------------------------- @@ -336,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of @@ -436,7 +453,7 @@ Avoiding ``PyObject_New`` GC-tracked objects need to be allocated using GC-aware functions. -If you use use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: +If you use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: - Get and call type's :c:member:`~PyTypeObject.tp_alloc` slot, if possible. That is, replace ``TYPE *o = PyObject_New(TYPE, typeobj)`` with:: @@ -609,8 +626,7 @@ Open Issues Several issues around per-module state and heap types are still open. -Discussions about improving the situation are best held on the `capi-sig -mailing list `__. +Discussions about improving the situation are best held on the `discuss forum under c-api tag `__. Per-Class Scope diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 7d64a02358a..52537a91df5 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4078,6 +4078,104 @@ lines. With this approach, you get better output: WARNING:demo: 1/0 WARNING:demo:ZeroDivisionError: division by zero +How to uniformly handle newlines in logging output +-------------------------------------------------- + +Usually, messages that are logged (say to console or file) consist of a single +line of text. However, sometimes there is a need to handle messages with +multiple lines - whether because a logging format string contains newlines, or +logged data contains newlines. If you want to handle such messages uniformly, so +that each line in the logged message appears uniformly formatted as if it was +logged separately, you can do this using a handler mixin, as in the following +snippet: + +.. code-block:: python + + # Assume this is in a module mymixins.py + import copy + + class MultilineMixin: + def emit(self, record): + s = record.getMessage() + if '\n' not in s: + super().emit(record) + else: + lines = s.splitlines() + rec = copy.copy(record) + rec.args = None + for line in lines: + rec.msg = line + super().emit(rec) + +You can use the mixin as in the following script: + +.. code-block:: python + + import logging + + from mymixins import MultilineMixin + + logger = logging.getLogger(__name__) + + class StreamHandler(MultilineMixin, logging.StreamHandler): + pass + + if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-9s %(message)s', + handlers = [StreamHandler()]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +The script, when run, prints something like: + +.. code-block:: text + + 2025-07-02 13:54:47,234 DEBUG Single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me once ... + 2025-07-02 13:54:47,234 DEBUG Another single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me ... + 2025-07-02 13:54:47,234 DEBUG can't get fooled again + +If, on the other hand, you are concerned about `log injection +`_, you can use a +formatter which escapes newlines, as per the following example: + +.. code-block:: python + + import logging + + logger = logging.getLogger(__name__) + + class EscapingFormatter(logging.Formatter): + def format(self, record): + s = super().format(record) + return s.replace('\n', r'\n') + + if __name__ == '__main__': + h = logging.StreamHandler() + h.setFormatter(EscapingFormatter('%(asctime)s %(levelname)-9s %(message)s')) + logging.basicConfig(level=logging.DEBUG, handlers = [h]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +You can, of course, use whatever escaping scheme makes the most sense for you. +The script, when run, should produce output like this: + +.. code-block:: text + + 2025-07-09 06:47:33,783 DEBUG Single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me once ... + 2025-07-09 06:47:33,783 DEBUG Another single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me ...\ncan't get fooled again + +Escaping behaviour can't be the stdlib default , as it would break backwards +compatibility. .. patterns-to-avoid: diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 2982cf88bf9..b7225ff1c2c 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -302,10 +302,10 @@ reading the following sections. If you're ready for that, grab some of your favourite beverage and carry on. If your logging needs are simple, then use the above examples to incorporate -logging into your own scripts, and if you run into problems or don't -understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/g/comp.lang.python) and you -should receive help before too long. +logging into your own scripts, and if you run into problems or don't understand +something, please post a question in the Help category of the `Python +discussion forum `_ and you should receive +help before too long. Still here? You can carry on reading the next few sections, which provide a slightly more advanced/in-depth tutorial than the basic one above. After that, diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index b579d776576..fc4772bbcca 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -2,34 +2,35 @@ .. _perf_profiling: -============================================== -Python support for the Linux ``perf`` profiler -============================================== +======================================================== +Python support for the ``perf map`` compatible profilers +======================================================== :author: Pablo Galindo -`The Linux perf profiler `_ -is a very powerful tool that allows you to profile and obtain -information about the performance of your application. -``perf`` also has a very vibrant ecosystem of tools -that aid with the analysis of the data that it produces. +`The Linux perf profiler `_ and +`samply `_ are powerful tools that allow you to +profile and obtain information about the performance of your application. +Both tools have vibrant ecosystems that aid with the analysis of the data they produce. -The main problem with using the ``perf`` profiler with Python applications is that -``perf`` only gets information about native symbols, that is, the names of +The main problem with using these profilers with Python applications is that +they only get information about native symbols, that is, the names of functions and procedures written in C. This means that the names and file names -of Python functions in your code will not appear in the output of ``perf``. +of Python functions in your code will not appear in the profiler output. Since Python 3.12, the interpreter can run in a special mode that allows Python -functions to appear in the output of the ``perf`` profiler. When this mode is +functions to appear in the output of compatible profilers. When this mode is enabled, the interpreter will interpose a small piece of code compiled on the -fly before the execution of every Python function and it will teach ``perf`` the +fly before the execution of every Python function and it will teach the profiler the relationship between this piece of code and the associated Python function using :doc:`perf map files <../c-api/perfmaps>`. .. note:: - Support for the ``perf`` profiler is currently only available for Linux on - select architectures. Check the output of the ``configure`` build step or + Support for profiling is available on Linux and macOS on select architectures. + Perf is available on Linux, while samply can be used on both Linux and macOS. + samply support on macOS is available starting from Python 3.15. + Check the output of the ``configure`` build step or check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE`` to see if your system is supported. @@ -92,7 +93,7 @@ Then we can use ``perf report`` to analyze the data: | | | | | | | |--51.67%--_PyEval_EvalFrameDefault | | | | | - | | | | |--11.52%--_PyLong_Add + | | | | |--11.52%--_PyCompactLong_Add | | | | | | | | | | | |--2.97%--_PyObject_Malloc ... @@ -142,12 +143,37 @@ Instead, if we run the same experiment with ``perf`` support enabled we get: | | | | | | | |--51.81%--_PyEval_EvalFrameDefault | | | | | - | | | | |--13.77%--_PyLong_Add + | | | | |--13.77%--_PyCompactLong_Add | | | | | | | | | | | |--3.26%--_PyObject_Malloc +Using the samply profiler +------------------------- + +samply is a modern profiler that can be used as an alternative to perf. +It uses the same perf map files that Python generates, making it compatible +with Python's profiling support. samply is particularly useful on macOS +where perf is not available. + +To use samply with Python, first install it following the instructions at +https://github.com/mstange/samply, then run:: + + $ samply record PYTHONPERFSUPPORT=1 python my_script.py + +This will open a web interface where you can analyze the profiling data +interactively. The advantage of samply is that it provides a modern +web-based interface for analyzing profiling data and works on both Linux +and macOS. + +On macOS, samply support requires Python 3.15 or later. Also on macOS, samply +can't profile signed Python executables due to restrictions by macOS. You can +profile with Python binaries that you've compiled yourself, or which are +unsigned or locally-signed (such as anything installed by Homebrew). In +order to attach to running processes on macOS, run ``samply setup`` once (and +every time samply is updated) to self-sign the samply binary. + How to enable ``perf`` profiling support ---------------------------------------- diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index e543f6d5657..7486a378dbb 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -1016,7 +1016,9 @@ extension. This regular expression matches ``foo.bar`` and Now, consider complicating the problem a bit; what if you want to match filenames where the extension is not ``bat``? Some incorrect attempts: -``.*[.][^b].*$`` The first attempt above tries to exclude ``bat`` by requiring +``.*[.][^b].*$`` + +The first attempt above tries to exclude ``bat`` by requiring that the first character of the extension is not a ``b``. This is wrong, because the pattern also doesn't match ``foo.bar``. @@ -1043,7 +1045,9 @@ confusing. A negative lookahead cuts through all this confusion: -``.*[.](?!bat$)[^.]*$`` The negative lookahead means: if the expression ``bat`` +``.*[.](?!bat$)[^.]*$`` + +The negative lookahead means: if the expression ``bat`` doesn't match at this point, try the rest of the pattern; if ``bat$`` does match, the whole pattern will fail. The trailing ``$`` is required to ensure that something like ``sample.batch``, where the extension only starts with diff --git a/Doc/howto/remote_debugging.rst b/Doc/howto/remote_debugging.rst index 3adb6ad03e5..78b40bcdf71 100644 --- a/Doc/howto/remote_debugging.rst +++ b/Doc/howto/remote_debugging.rst @@ -3,6 +3,78 @@ Remote debugging attachment protocol ==================================== +This protocol enables external tools to attach to a running CPython process and +execute Python code remotely. + +Most platforms require elevated privileges to attach to another Python process. + +.. _permission-requirements: + +Permission requirements +======================= + +Attaching to a running Python process for remote debugging requires elevated +privileges on most platforms. The specific requirements and troubleshooting +steps depend on your operating system: + +.. rubric:: Linux + +The tracer process must have the ``CAP_SYS_PTRACE`` capability or equivalent +privileges. You can only trace processes you own and can signal. Tracing may +fail if the process is already being traced, or if it is running with +set-user-ID or set-group-ID. Security modules like Yama may further restrict +tracing. + +To temporarily relax ptrace restrictions (until reboot), run: + + ``echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope`` + +.. note:: + + Disabling ``ptrace_scope`` reduces system hardening and should only be done + in trusted environments. + +If running inside a container, use ``--cap-add=SYS_PTRACE`` or +``--privileged``, and run as root if needed. + +Try re-running the command with elevated privileges: + + ``sudo -E !!`` + + +.. rubric:: macOS + +To attach to another process, you typically need to run your debugging tool +with elevated privileges. This can be done by using ``sudo`` or running as +root. + +Even when attaching to processes you own, macOS may block debugging unless +the debugger is run with root privileges due to system security restrictions. + + +.. rubric:: Windows + +To attach to another process, you usually need to run your debugging tool +with administrative privileges. Start the command prompt or terminal as +Administrator. + +Some processes may still be inaccessible even with Administrator rights, +unless you have the ``SeDebugPrivilege`` privilege enabled. + +To resolve file or folder access issues, adjust the security permissions: + + 1. Right-click the file or folder and select **Properties**. + 2. Go to the **Security** tab to view users and groups with access. + 3. Click **Edit** to modify permissions. + 4. Select your user account. + 5. In **Permissions**, check **Read** or **Full control** as needed. + 6. Click **Apply**, then **OK** to confirm. + + +.. note:: + + Ensure you've satisfied all :ref:`permission-requirements` before proceeding. + This section describes the low-level protocol that enables external tools to inject and execute a Python script within a running CPython process. @@ -374,13 +446,13 @@ To locate a thread: reliable thread to target. 3. Optionally, use the offset ``interpreter_state.threads_head`` to iterate -through the linked list of all thread states. Each ``PyThreadState`` structure -contains a ``native_thread_id`` field, which may be compared to a target thread -ID to find a specific thread. + through the linked list of all thread states. Each ``PyThreadState`` + structure contains a ``native_thread_id`` field, which may be compared to + a target thread ID to find a specific thread. -1. Once a valid ``PyThreadState`` has been found, its address can be used in -later steps of the protocol, such as writing debugger control fields and -scheduling execution. +4. Once a valid ``PyThreadState`` has been found, its address can be used in + later steps of the protocol, such as writing debugger control fields and + scheduling execution. The following is an example implementation that locates the main thread state:: @@ -454,15 +526,15 @@ its fields are defined by the ``_Py_DebugOffsets`` structure and include the following: - ``debugger_script_path``: A fixed-size buffer that holds the full path to a - Python source file (``.py``). This file must be accessible and readable by - the target process when execution is triggered. + Python source file (``.py``). This file must be accessible and readable by + the target process when execution is triggered. - ``debugger_pending_call``: An integer flag. Setting this to ``1`` tells the - interpreter that a script is ready to be executed. + interpreter that a script is ready to be executed. - ``eval_breaker``: A field checked by the interpreter during execution. - Setting bit 5 (``_PY_EVAL_PLEASE_STOP_BIT``, value ``1U << 5``) in this - field causes the interpreter to pause and check for debugger activity. + Setting bit 5 (``_PY_EVAL_PLEASE_STOP_BIT``, value ``1U << 5``) in this + field causes the interpreter to pause and check for debugger activity. To complete the injection, the debugger must perform the following steps: diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 254fe729355..243cc27bac7 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -352,6 +352,8 @@ If you don't include such a comment, the default encoding used will be UTF-8 as already mentioned. See also :pep:`263` for more information. +.. _unicode-properties: + Unicode Properties ------------------ diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 33a2a7ea89e..4e77d2cb407 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -15,7 +15,7 @@ Introduction You may also find useful the following article on fetching web resources with Python: - * `Basic Authentication `_ + * `Basic Authentication `__ A tutorial on *Basic Authentication*, with examples in Python. @@ -245,75 +245,27 @@ codes in the 100--299 range indicate success, you will usually only see error codes in the 400--599 range. :attr:`http.server.BaseHTTPRequestHandler.responses` is a useful dictionary of -response codes in that shows all the response codes used by :rfc:`2616`. The -dictionary is reproduced here for convenience :: +response codes that shows all the response codes used by :rfc:`2616`. +An excerpt from the dictionary is shown below :: - # Table mapping response codes to messages; entries have the - # form {code: (shortmessage, longmessage)}. responses = { - 100: ('Continue', 'Request received, please continue'), - 101: ('Switching Protocols', - 'Switching to new protocol; obey Upgrade header'), - - 200: ('OK', 'Request fulfilled, document follows'), - 201: ('Created', 'Document created, URL follows'), - 202: ('Accepted', - 'Request accepted, processing continues off-line'), - 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), - 204: ('No Content', 'Request fulfilled, nothing follows'), - 205: ('Reset Content', 'Clear input form for further input.'), - 206: ('Partial Content', 'Partial content follows.'), - - 300: ('Multiple Choices', - 'Object has several resources -- see URI list'), - 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), - 302: ('Found', 'Object moved temporarily -- see URI list'), - 303: ('See Other', 'Object moved -- see Method and URL list'), - 304: ('Not Modified', - 'Document has not changed since given time'), - 305: ('Use Proxy', - 'You must use proxy specified in Location to access this ' - 'resource.'), - 307: ('Temporary Redirect', - 'Object moved temporarily -- see URI list'), - - 400: ('Bad Request', - 'Bad request syntax or unsupported method'), - 401: ('Unauthorized', - 'No permission -- see authorization schemes'), - 402: ('Payment Required', - 'No payment -- see charging schemes'), - 403: ('Forbidden', - 'Request forbidden -- authorization will not help'), - 404: ('Not Found', 'Nothing matches the given URI'), - 405: ('Method Not Allowed', - 'Specified method is invalid for this server.'), - 406: ('Not Acceptable', 'URI not available in preferred format.'), - 407: ('Proxy Authentication Required', 'You must authenticate with ' - 'this proxy before proceeding.'), - 408: ('Request Timeout', 'Request timed out; try again later.'), - 409: ('Conflict', 'Request conflict.'), - 410: ('Gone', - 'URI no longer exists and has been permanently removed.'), - 411: ('Length Required', 'Client must specify Content-Length.'), - 412: ('Precondition Failed', 'Precondition in headers is false.'), - 413: ('Request Entity Too Large', 'Entity is too large.'), - 414: ('Request-URI Too Long', 'URI is too long.'), - 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), - 416: ('Requested Range Not Satisfiable', - 'Cannot satisfy request range.'), - 417: ('Expectation Failed', - 'Expect condition could not be satisfied.'), - - 500: ('Internal Server Error', 'Server got itself in trouble'), - 501: ('Not Implemented', - 'Server does not support this operation'), - 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), - 503: ('Service Unavailable', - 'The server cannot process the request due to a high load'), - 504: ('Gateway Timeout', - 'The gateway server did not receive a timely response'), - 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), + ... + : ('OK', 'Request fulfilled, document follows'), + ... + : ('Forbidden', + 'Request forbidden -- authorization will ' + 'not help'), + : ('Not Found', + 'Nothing matches the given URI'), + ... + : ("I'm a Teapot", + 'Server refuses to brew coffee because ' + 'it is a teapot'), + ... + : ('Service Unavailable', + 'The server cannot process the ' + 'request due to a high load'), + ... } When an error is raised the server responds by returning an HTTP error code diff --git a/Doc/includes/newtypes/custom.c b/Doc/includes/newtypes/custom.c index 5253f879360..039a1a72193 100644 --- a/Doc/includes/newtypes/custom.c +++ b/Doc/includes/newtypes/custom.c @@ -16,28 +16,37 @@ static PyTypeObject CustomType = { .tp_new = PyType_GenericNew, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + // Just use this while using static types + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom2.c b/Doc/includes/newtypes/custom2.c index a87917583ca..1ff8e707d1b 100644 --- a/Doc/includes/newtypes/custom2.c +++ b/Doc/includes/newtypes/custom2.c @@ -106,28 +106,36 @@ static PyTypeObject CustomType = { .tp_methods = Custom_methods, }; -static PyModuleDef custommodule = { - .m_base =PyModuleDef_HEAD_INIT, +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom2", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom2(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom3.c b/Doc/includes/newtypes/custom3.c index 854034d4066..22f50eb0e1d 100644 --- a/Doc/includes/newtypes/custom3.c +++ b/Doc/includes/newtypes/custom3.c @@ -151,28 +151,36 @@ static PyTypeObject CustomType = { .tp_getset = Custom_getsetters, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom3", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom3(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom4.c b/Doc/includes/newtypes/custom4.c index a0a1eeb2891..07585aff598 100644 --- a/Doc/includes/newtypes/custom4.c +++ b/Doc/includes/newtypes/custom4.c @@ -170,28 +170,36 @@ static PyTypeObject CustomType = { .tp_getset = Custom_getsetters, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom4", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom4(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/sublist.c b/Doc/includes/newtypes/sublist.c index 00664f34541..b784456a4ef 100644 --- a/Doc/includes/newtypes/sublist.c +++ b/Doc/includes/newtypes/sublist.c @@ -31,7 +31,7 @@ SubList_init(PyObject *op, PyObject *args, PyObject *kwds) } static PyTypeObject SubListType = { - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "sublist.SubList", .tp_doc = PyDoc_STR("SubList objects"), .tp_basicsize = sizeof(SubListObject), @@ -41,29 +41,37 @@ static PyTypeObject SubListType = { .tp_methods = SubList_methods, }; -static PyModuleDef sublistmodule = { - PyModuleDef_HEAD_INIT, +static int +sublist_module_exec(PyObject *m) +{ + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot sublist_module_slots[] = { + {Py_mod_exec, sublist_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef sublist_module = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "sublist", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = sublist_module_slots, }; PyMODINIT_FUNC PyInit_sublist(void) { - PyObject *m; - SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&sublist_module); } diff --git a/Doc/includes/optional-module.rst b/Doc/includes/optional-module.rst new file mode 100644 index 00000000000..262e73f2eaa --- /dev/null +++ b/Doc/includes/optional-module.rst @@ -0,0 +1,9 @@ +This is an :term:`optional module`. +If it is missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + +.. Similar notes appear in the docs of the modules: + - zipfile + - tarfile diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index a46c1caefe4..3a485a43a5a 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -188,7 +188,7 @@ switch:: Once the Development & Deployment part of PPUG is fleshed out, some of those sections should be linked from new questions here (most notably, we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + https://packaging.python.org/en/latest/guides/index-mirrors-and-caches/) Common installation issues diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 4f3b663006f..5d916b30112 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -37,38 +37,52 @@ No feature description will ever be deleted from :mod:`__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations*, | -| | | | :pep:`649`: *Deferred evaluation of | -| | | | annotations using descriptors* | -+------------------+-------------+--------------+---------------------------------------------+ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * feature + * optional in + * mandatory in + * effect + * * .. data:: nested_scopes + * 2.1.0b1 + * 2.2 + * :pep:`227`: *Statically Nested Scopes* + * * .. data:: generators + * 2.2.0a1 + * 2.3 + * :pep:`255`: *Simple Generators* + * * .. data:: division + * 2.2.0a2 + * 3.0 + * :pep:`238`: *Changing the Division Operator* + * * .. data:: absolute_import + * 2.5.0a1 + * 3.0 + * :pep:`328`: *Imports: Multi-Line and Absolute/Relative* + * * .. data:: with_statement + * 2.5.0a1 + * 2.6 + * :pep:`343`: *The “with” Statement* + * * .. data:: print_function + * 2.6.0a2 + * 3.0 + * :pep:`3105`: *Make print a function* + * * .. data:: unicode_literals + * 2.6.0a2 + * 3.0 + * :pep:`3112`: *Bytes literals in Python 3000* + * * .. data:: generator_stop + * 3.5.0b1 + * 3.7 + * :pep:`479`: *StopIteration handling inside generators* + * * .. data:: annotations + * 3.7.0b1 + * Never [1]_ + * :pep:`563`: *Postponed evaluation of annotations*, + :pep:`649`: *Deferred evaluation of annotations using descriptors* .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index ed29ac70035..47f5eabb6f2 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -120,13 +120,16 @@ This module defines the following constants and functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD, Solaris. .. versionadded:: 3.8 .. versionchanged:: 3.13 Added support for GNU/kFreeBSD. + .. versionchanged:: 3.15 + Added support for Solaris. + .. function:: stack_size([size]) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index ff9578b6088..40f2a6dc304 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -4,6 +4,7 @@ .. module:: annotationlib :synopsis: Functionality for introspecting annotations +.. versionadded:: 3.14 **Source code:** :source:`Lib/annotationlib.py` @@ -45,6 +46,10 @@ and :func:`call_annotate_function`, as well as the :func:`call_evaluate_function` function for working with :term:`evaluate functions `. +.. caution:: + + Most functionality in this module can execute arbitrary code; see + :ref:`the security section ` for more information. .. seealso:: @@ -127,25 +132,8 @@ Classes Values are the result of evaluating the annotation expressions. - .. attribute:: FORWARDREF - :value: 2 - - Values are real annotation values (as per :attr:`Format.VALUE` format) - for defined values, and :class:`ForwardRef` proxies for undefined - values. Real objects may contain references to :class:`ForwardRef` - proxy objects. - - .. attribute:: STRING - :value: 3 - - Values are the text string of the annotation as it appears in the - source code, up to modifications including, but not restricted to, - whitespace normalizations and constant values optimizations. - - The exact values of these strings may change in future versions of Python. - .. attribute:: VALUE_WITH_FAKE_GLOBALS - :value: 4 + :value: 2 Special value used to signal that an annotate function is being evaluated in a special environment with fake globals. When passed this @@ -155,6 +143,23 @@ Classes This format is only used internally and should not be passed to the functions in this module. + .. attribute:: FORWARDREF + :value: 3 + + Values are real annotation values (as per :attr:`Format.VALUE` format) + for defined values, and :class:`ForwardRef` proxies for undefined + values. Real objects may contain references to :class:`ForwardRef` + proxy objects. + + .. attribute:: STRING + :value: 4 + + Values are the text string of the annotation as it appears in the + source code, up to modifications including, but not restricted to, + whitespace normalizations and constant values optimizations. + + The exact values of these strings may change in future versions of Python. + .. versionadded:: 3.14 .. class:: ForwardRef @@ -211,6 +216,10 @@ Classes means may not have any information about their scope, so passing arguments to this method may be necessary to evaluate them successfully. + If no *owner*, *globals*, *locals*, or *type_params* are provided and the + :class:`~ForwardRef` does not contain information about its origin, + empty globals and locals dictionaries are used. + .. versionadded:: 3.14 @@ -331,14 +340,29 @@ Functions * VALUE: :attr:`!object.__annotations__` is tried first; if that does not exist, the :attr:`!object.__annotate__` function is called if it exists. + * FORWARDREF: If :attr:`!object.__annotations__` exists and can be evaluated successfully, it is used; otherwise, the :attr:`!object.__annotate__` function is called. If it does not exist either, :attr:`!object.__annotations__` is tried again and any error from accessing it is re-raised. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.FORWARDREF`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + * STRING: If :attr:`!object.__annotate__` exists, it is called first; otherwise, :attr:`!object.__annotations__` is used and stringified using :func:`annotations_to_string`. + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.STRING`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE` + with the result converted using :func:`annotations_to_string`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + Returns a dict. :func:`!get_annotations` returns a new dict every time it's called; calling it twice on the same object will return two different but equivalent dicts. @@ -485,3 +509,137 @@ annotations from the class and puts them in a separate attribute: typ.classvars = classvars # Store the ClassVars in a separate attribute return typ + +Limitations of the ``STRING`` format +------------------------------------ + +The :attr:`~Format.STRING` format is meant to approximate the source code +of the annotation, but the implementation strategy used means that it is not +always possible to recover the exact source code. + +First, the stringifier of course cannot recover any information that is not present in +the compiled code, including comments, whitespace, parenthesization, and operations that +get simplified by the compiler. + +Second, the stringifier can intercept almost all operations that involve names looked +up in some scope, but it cannot intercept operations that operate fully on constants. +As a corollary, this also means it is not safe to request the ``STRING`` format on +untrusted code: Python is powerful enough that it is possible to achieve arbitrary +code execution even with no access to any globals or builtins. For example: + +.. code-block:: pycon + + >>> def f(x: (1).__class__.__base__.__subclasses__()[-1].__init__.__builtins__["print"]("Hello world")): pass + ... + >>> annotationlib.get_annotations(f, format=annotationlib.Format.STRING) + Hello world + {'x': 'None'} + +.. note:: + This particular example works as of the time of writing, but it relies on + implementation details and is not guaranteed to work in the future. + +Among the different kinds of expressions that exist in Python, +as represented by the :mod:`ast` module, some expressions are supported, +meaning that the ``STRING`` format can generally recover the original source code; +others are unsupported, meaning that they may result in incorrect output or an error. + +The following are supported (sometimes with caveats): + +* :class:`ast.BinOp` +* :class:`ast.UnaryOp` + + * :class:`ast.Invert` (``~``), :class:`ast.UAdd` (``+``), and :class:`ast.USub` (``-``) are supported + * :class:`ast.Not` (``not``) is not supported + +* :class:`ast.Dict` (except when using ``**`` unpacking) +* :class:`ast.Set` +* :class:`ast.Compare` + + * :class:`ast.Eq` and :class:`ast.NotEq` are supported + * :class:`ast.Lt`, :class:`ast.LtE`, :class:`ast.Gt`, and :class:`ast.GtE` are supported, but the operand may be flipped + * :class:`ast.Is`, :class:`ast.IsNot`, :class:`ast.In`, and :class:`ast.NotIn` are not supported + +* :class:`ast.Call` (except when using ``**`` unpacking) +* :class:`ast.Constant` (though not the exact representation of the constant; for example, escape + sequences in strings are lost; hexadecimal numbers are converted to decimal) +* :class:`ast.Attribute` (assuming the value is not a constant) +* :class:`ast.Subscript` (assuming the value is not a constant) +* :class:`ast.Starred` (``*`` unpacking) +* :class:`ast.Name` +* :class:`ast.List` +* :class:`ast.Tuple` +* :class:`ast.Slice` + +The following are unsupported, but throw an informative error when encountered by the +stringifier: + +* :class:`ast.FormattedValue` (f-strings; error is not detected if conversion specifiers like ``!r`` + are used) +* :class:`ast.JoinedStr` (f-strings) + +The following are unsupported and result in incorrect output: + +* :class:`ast.BoolOp` (``and`` and ``or``) +* :class:`ast.IfExp` +* :class:`ast.Lambda` +* :class:`ast.ListComp` +* :class:`ast.SetComp` +* :class:`ast.DictComp` +* :class:`ast.GeneratorExp` + +The following are disallowed in annotation scopes and therefore not relevant: + +* :class:`ast.NamedExpr` (``:=``) +* :class:`ast.Await` +* :class:`ast.Yield` +* :class:`ast.YieldFrom` + + +Limitations of the ``FORWARDREF`` format +---------------------------------------- + +The :attr:`~Format.FORWARDREF` format aims to produce real values as much +as possible, with anything that cannot be resolved replaced with +:class:`ForwardRef` objects. It is affected by broadly the same Limitations +as the :attr:`~Format.STRING` format: annotations that perform operations on +literals or that use unsupported expression types may raise exceptions when +evaluated using the :attr:`~Format.FORWARDREF` format. + +Below are a few examples of the behavior with unsupported expressions: + +.. code-block:: pycon + + >>> from annotationlib import get_annotations, Format + >>> def zerodiv(x: 1 / 0): ... + >>> get_annotations(zerodiv, format=Format.STRING) + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> get_annotations(zerodiv, format=Format.FORWARDREF) + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def ifexp(x: 1 if y else 0): ... + >>> get_annotations(ifexp, format=Format.STRING) + {'x': '1'} + +.. _annotationlib-security: + +Security implications of introspecting annotations +-------------------------------------------------- + +Much of the functionality in this module involves executing code related to annotations, +which can then do arbitrary things. For example, +:func:`get_annotations` may call an arbitrary :term:`annotate function`, and +:meth:`ForwardRef.evaluate` may call :func:`eval` on an arbitrary string. Code contained +in an annotation might make arbitrary system calls, enter an infinite loop, or perform any +other operation. This is also true for any access of the :attr:`~object.__annotations__` attribute, +and for various functions in the :mod:`typing` module that work with annotations, such as +:func:`typing.get_type_hints`. + +Any security issue arising from this also applies immediately after importing +code that may contain untrusted annotations: importing code can always cause arbitrary operations +to be performed. However, it is unsafe to accept strings or other input from an untrusted source and +pass them to any of the APIs for introspecting annotations, for example by editing an +``__annotations__`` dictionary or directly creating a :class:`ForwardRef` object. diff --git a/Doc/library/archiving.rst b/Doc/library/archiving.rst index c9284949af4..da0b3f8c3e7 100644 --- a/Doc/library/archiving.rst +++ b/Doc/library/archiving.rst @@ -5,13 +5,15 @@ Data Compression and Archiving ****************************** The modules described in this chapter support data compression with the zlib, -gzip, bzip2 and lzma algorithms, and the creation of ZIP- and tar-format +gzip, bzip2, lzma, and zstd algorithms, and the creation of ZIP- and tar-format archives. See also :ref:`archiving-operations` provided by the :mod:`shutil` module. .. toctree:: + compression.rst + compression.zstd.rst zlib.rst gzip.rst bz2.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 29396c7a036..71c4f094886 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -74,7 +74,7 @@ ArgumentParser objects prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ add_help=True, allow_abbrev=True, exit_on_error=True, \ - *, suggest_on_error=False, color=False) + *, suggest_on_error=True, color=True) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description @@ -117,9 +117,9 @@ ArgumentParser objects error info when an error occurs. (default: ``True``) * suggest_on_error_ - Enables suggestions for mistyped argument choices - and subparser names (default: ``False``) + and subparser names (default: ``True``) - * color_ - Allow color output (default: ``False``) + * color_ - Allow color output (default: ``True``) .. versionchanged:: 3.5 *allow_abbrev* parameter was added. @@ -134,6 +134,9 @@ ArgumentParser objects .. versionchanged:: 3.14 *suggest_on_error* and *color* parameters were added. + .. versionchanged:: 3.15 + *suggest_on_error* default changed to ``True``. + The following sections describe how each of these are used. @@ -434,12 +437,18 @@ arguments they contain. For example:: >>> parser.parse_args(['-f', 'foo', '@args.txt']) Namespace(f='bar') -Arguments read from a file must by default be one per line (but see also +Arguments read from a file must be one per line by default (but see also :meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they were in the same place as the original file referencing argument on the command line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. +.. note:: + + Empty lines are treated as empty strings (``''``), which are allowed as values but + not as arguments. Empty lines that are read as arguments will result in an + "unrecognized arguments" error. + :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -590,13 +599,11 @@ suggest_on_error ^^^^^^^^^^^^^^^^ By default, when a user passes an invalid argument choice or subparser name, -:class:`ArgumentParser` will exit with error info and list the permissible -argument choices (if specified) or subparser names as part of the error message. - -If the user would like to enable suggestions for mistyped argument choices and -subparser names, the feature can be enabled by setting ``suggest_on_error`` to -``True``. Note that this only applies for arguments when the choices specified -are strings:: +:class:`ArgumentParser` will exit with error info and provide suggestions for +mistyped arguments. The error message will list the permissible argument +choices (if specified) or subparser names, along with a "maybe you meant" +suggestion if a close match is found. Note that this only applies for arguments +when the choices specified are strings:: >>> parser = argparse.ArgumentParser(description='Process some integers.', suggest_on_error=True) @@ -606,40 +613,35 @@ are strings:: >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') -If you're writing code that needs to be compatible with older Python versions -and want to opportunistically use ``suggest_on_error`` when it's available, you -can set it as an attribute after initializing the parser instead of using the -keyword argument:: +You can disable suggestions by setting ``suggest_on_error`` to ``False``:: - >>> parser = argparse.ArgumentParser(description='Process some integers.') - >>> parser.suggest_on_error = True + >>> parser = argparse.ArgumentParser(description='Process some integers.', + suggest_on_error=False) .. versionadded:: 3.14 - +.. versionchanged:: 3.15 + Changed default value of ``suggest_on_error`` from ``False`` to ``True``. color ^^^^^ -By default, the help message is printed in plain text. If you want to allow -color in help messages, you can enable it by setting ``color`` to ``True``:: +By default, the help message is printed in color using `ANSI escape sequences +`__. +If you want plain text help messages, you can disable this :ref:`in your local +environment `, or in the argument parser itself +by setting ``color`` to ``False``:: >>> parser = argparse.ArgumentParser(description='Process some integers.', - ... color=True) + ... color=False) >>> parser.add_argument('--action', choices=['sum', 'max']) >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', ... help='an integer for the accumulator') >>> parser.parse_args(['--help']) -Even if a CLI author has enabled color, it can be -:ref:`controlled using environment variables `. - -If you're writing code that needs to be compatible with older Python versions -and want to opportunistically use ``color`` when it's available, you -can set it as an attribute after initializing the parser instead of using the -keyword argument:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - >>> parser.color = True +Note that when ``color=True``, colored output depends on both environment +variables and terminal capabilities. However, if ``color=False``, colored +output is always disabled, even if environment variables like ``FORCE_COLOR`` +are set. .. versionadded:: 3.14 @@ -765,9 +767,9 @@ how the command-line arguments should be handled. The supplied actions are: Namespace(foo=42) * ``'store_true'`` and ``'store_false'`` - These are special cases of - ``'store_const'`` used for storing the values ``True`` and ``False`` - respectively. In addition, they create default values of ``False`` and - ``True`` respectively:: + ``'store_const'`` that respectively store the values ``True`` and ``False`` + with default values of ``False`` and + ``True``:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -776,19 +778,19 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--foo --bar'.split()) Namespace(foo=True, bar=False, baz=True) -* ``'append'`` - This stores a list, and appends each argument value to the - list. It is useful to allow an option to be specified multiple times. - If the default value is non-empty, the default elements will be present - in the parsed value for the option, with any values from the - command line appended after those default values. Example usage:: +* ``'append'`` - This appends each argument value to a list. + It is useful for allowing an option to be specified multiple times. + If the default value is a non-empty list, the parsed value will start + with the default list's elements and any values from the command line + will be appended after those default values. Example usage:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='append') + >>> parser.add_argument('--foo', action='append', default=['0']) >>> parser.parse_args('--foo 1 --foo 2'.split()) - Namespace(foo=['1', '2']) + Namespace(foo=['0', '1', '2']) -* ``'append_const'`` - This stores a list, and appends the value specified by - the const_ keyword argument to the list; note that the const_ keyword +* ``'append_const'`` - This appends the value specified by + the const_ keyword argument to a list; note that the const_ keyword argument defaults to ``None``. The ``'append_const'`` action is typically useful when multiple arguments need to store constants to the same list. For example:: @@ -799,8 +801,8 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--str --int'.split()) Namespace(types=[, ]) -* ``'extend'`` - This stores a list and appends each item from the multi-value - argument list to it. +* ``'extend'`` - This appends each item from a multi-value + argument to a list. The ``'extend'`` action is typically used with the nargs_ keyword argument value ``'+'`` or ``'*'``. Note that when nargs_ is ``None`` (the default) or ``'?'``, each @@ -814,7 +816,7 @@ how the command-line arguments should be handled. The supplied actions are: .. versionadded:: 3.8 -* ``'count'`` - This counts the number of times a keyword argument occurs. For +* ``'count'`` - This counts the number of times an argument occurs. For example, this is useful for increasing verbosity levels:: >>> parser = argparse.ArgumentParser() @@ -839,23 +841,11 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 -Only actions that consume command-line arguments (e.g. ``'store'``, -``'append'`` or ``'extend'``) can be used with positional arguments. - -.. class:: BooleanOptionalAction - - You may also specify an arbitrary action by passing an :class:`Action` subclass or - other object that implements the same interface. The :class:`!BooleanOptionalAction` - is available in :mod:`!argparse` and adds support for boolean actions such as - ``--foo`` and ``--no-foo``:: - - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) - >>> parser.parse_args(['--no-foo']) - Namespace(foo=False) - - .. versionadded:: 3.9 +You may also specify an arbitrary action by passing an :class:`Action` subclass +(e.g. :class:`BooleanOptionalAction`) or other object that implements the same +interface. Only actions that consume command-line arguments (e.g. ``'store'``, +``'append'``, ``'extend'``, or custom actions with non-zero ``nargs``) can be used +with positional arguments. The recommended way to create a custom action is to extend :class:`Action`, overriding the :meth:`!__call__` method and optionally the :meth:`!__init__` and @@ -955,7 +945,7 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: .. index:: single: + (plus); in argparse module -* ``'+'``. Just like ``'*'``, all command-line args present are gathered into a +* ``'+'``. Just like ``'*'``, all command-line arguments present are gathered into a list. Additionally, an error message will be generated if there wasn't at least one command-line argument present. For example:: @@ -995,8 +985,8 @@ the various :class:`ArgumentParser` actions. The two most common uses of it are (like ``-f`` or ``--foo``) and ``nargs='?'``. This creates an optional argument that can be followed by zero or one command-line arguments. When parsing the command line, if the option string is encountered with no - command-line argument following it, the value of ``const`` will be assumed to - be ``None`` instead. See the nargs_ description for examples. + command-line argument following it, the value from ``const`` will be used. + See the nargs_ description for examples. .. versionchanged:: 3.11 ``const=None`` by default, including when ``action='append_const'`` or @@ -1156,16 +1146,21 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* sequence is checked after any type_ -conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified. - Any sequence can be passed as the *choices* value, so :class:`list` objects, :class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. +Note that *choices* are checked after any type_ +conversions have been performed, so objects in *choices* +should match the type_ specified. This can make *choices* +appear unfamiliar in usage, help, or error messages. + +To keep *choices* user-friendly, consider a custom type wrapper that +converts and formats values, or omit type_ and handle conversion in +your application code. + Formatted choices override the default *metavar* which is normally derived from *dest*. This is usually what you want because the user never sees the *dest* parameter. If this display isn't desirable (perhaps because there are @@ -1327,8 +1322,12 @@ attribute is determined by the ``dest`` keyword argument of For optional argument actions, the value of ``dest`` is normally inferred from the option strings. :class:`ArgumentParser` generates the value of ``dest`` by -taking the first long option string and stripping away the initial ``--`` -string. If no long option strings were supplied, ``dest`` will be derived from +taking the first double-dash long option string and stripping away the initial +``-`` characters. +If no double-dash long option strings were supplied, ``dest`` will be derived +from the first single-dash long option string by stripping the initial ``-`` +character. +If no long option strings were supplied, ``dest`` will be derived from the first short option string by stripping the initial ``-`` character. Any internal ``-`` characters will be converted to ``_`` characters to make sure the string is a valid attribute name. The examples below illustrate this @@ -1336,11 +1335,12 @@ behavior:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('-f', '--foo-bar', '--foo') + >>> parser.add_argument('-q', '-quz') >>> parser.add_argument('-x', '-y') - >>> parser.parse_args('-f 1 -x 2'.split()) - Namespace(foo_bar='1', x='2') - >>> parser.parse_args('--foo 1 -y 2'.split()) - Namespace(foo_bar='1', x='2') + >>> parser.parse_args('-f 1 -q 2 -x 3'.split()) + Namespace(foo_bar='1', quz='2', x='3') + >>> parser.parse_args('--foo 1 -quz 2 -y 3'.split()) + Namespace(foo_bar='1', quz='2', x='2') ``dest`` allows a custom attribute name to be provided:: @@ -1349,6 +1349,9 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +.. versionchanged:: next + Single-dash long option now takes precedence over short options. + .. _deprecated: @@ -1429,6 +1432,31 @@ this API may be passed as the ``action`` parameter to and return a string which will be used when printing the usage of the program. If such method is not provided, a sensible default will be used. +.. class:: BooleanOptionalAction + + A subclass of :class:`Action` for handling boolean flags with positive + and negative options. Adding a single argument such as ``--foo`` automatically + creates both ``--foo`` and ``--no-foo`` options, storing ``True`` and ``False`` + respectively:: + + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + >>> parser.parse_args(['--no-foo']) + Namespace(foo=False) + + Single-dash long options are also supported. + For example, negative option ``-nofoo`` is automatically added for + positive option ``-foo``. + But no additional options are added for short options such as ``-f``. + + .. versionadded:: 3.9 + + .. versionchanged:: next + Added support for single-dash options. + + Added support for alternate prefix_chars_. + The parse_args() method ----------------------- @@ -2060,7 +2088,9 @@ Parser defaults >>> parser.parse_args(['736']) Namespace(bar=42, baz='badger', foo=736) - Note that parser-level defaults always override argument-level defaults:: + Note that defaults can be set at both the parser level using :meth:`set_defaults` + and at the argument level using :meth:`add_argument`. If both are called for the + same argument, the last default set for an argument is used:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default='bar') @@ -2122,12 +2152,15 @@ Partial parsing .. method:: ArgumentParser.parse_known_args(args=None, namespace=None) - Sometimes a script may only parse a few of the command-line arguments, passing - the remaining arguments on to another script or program. In these cases, the - :meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like - :meth:`~ArgumentParser.parse_args` except that it does not produce an error when - extra arguments are present. Instead, it returns a two item tuple containing - the populated namespace and the list of remaining argument strings. + Sometimes a script only needs to handle a specific set of command-line + arguments, leaving any unrecognized arguments for another script or program. + In these cases, the :meth:`~ArgumentParser.parse_known_args` method can be + useful. + + This method works similarly to :meth:`~ArgumentParser.parse_args`, but it does + not raise an error for extra, unrecognized arguments. Instead, it parses the + known arguments and returns a two item tuple that contains the populated + namespace and the list of any unrecognized arguments. :: diff --git a/Doc/library/array.rst b/Doc/library/array.rst index e0b1eb89cf6..1f04f697c75 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -24,7 +24,7 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'u'`` | wchar_t | Unicode character | 2 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'w'`` | Py_UCS4 | Unicode character | 4 | | +| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(2) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -60,6 +60,9 @@ Notes: .. deprecated-removed:: 3.3 3.16 Please migrate to ``'w'`` typecode. +(2) + .. versionadded:: 3.13 + The actual representation of values is determined by the machine architecture (strictly speaking, by the C implementation). The actual size can be accessed diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ca9a6b0712c..2e7d0dbc26e 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -252,12 +252,11 @@ Root nodes >>> print(ast.dump(ast.parse('(int, str) -> List[int]', mode='func_type'), indent=4)) FunctionType( argtypes=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], + Name(id='int'), + Name(id='str')], returns=Subscript( - value=Name(id='List', ctx=Load()), - slice=Name(id='int', ctx=Load()), - ctx=Load())) + value=Name(id='List'), + slice=Name(id='int'))) .. versionadded:: 3.8 @@ -268,9 +267,9 @@ Literals .. class:: Constant(value) A constant value. The ``value`` attribute of the ``Constant`` literal contains the - Python object it represents. The values represented can be simple types - such as a number, string or ``None``, but also immutable container types - (tuples and frozensets) if all of their elements are constant. + Python object it represents. The values represented can be instances of :class:`str`, + :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, + and the constants :data:`None` and :data:`Ellipsis`. .. doctest:: @@ -290,9 +289,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` formatting + * 114 (``ord('r')``): ``!r`` :func:`repr` formatting + * 115 (``ord('s')``): ``!s`` :func:`string ` formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -312,20 +311,77 @@ Literals values=[ Constant(value='sin('), FormattedValue( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), conversion=-1), Constant(value=') is '), FormattedValue( value=Call( - func=Name(id='sin', ctx=Load()), + func=Name(id='sin'), args=[ - Name(id='a', ctx=Load())]), + Name(id='a')]), conversion=-1, format_spec=JoinedStr( values=[ Constant(value='.3')]))])) +.. class:: TemplateStr(values, /) + + .. versionadded:: 3.14 + + Node representing a template string literal, comprising a series of + :class:`Interpolation` and :class:`Constant` nodes. + These nodes may be any order, and do not need to be interleaved. + + .. doctest:: + + >>> expr = ast.parse('t"{name} finished {place:ordinal}"', mode='eval') + >>> print(ast.dump(expr, indent=4)) + Expression( + body=TemplateStr( + values=[ + Interpolation( + value=Name(id='name'), + str='name', + conversion=-1), + Constant(value=' finished '), + Interpolation( + value=Name(id='place'), + str='place', + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='ordinal')]))])) + +.. class:: Interpolation(value, str, conversion, format_spec=None) + + .. versionadded:: 3.14 + + Node representing a single interpolation field in a template string literal. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + This has the same meaning as ``FormattedValue.value``. + * ``str`` is a constant containing the text of the interpolation expression. + + If ``str`` is set to ``None``, then ``value`` is used to generate code + when calling :func:`ast.unparse`. This no longer guarantees that the + generated code is identical to the original and is intended for code + generation. + * ``conversion`` is an integer: + + * -1: no conversion + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` conversion + * 114 (``ord('r')``): ``!r`` :func:`repr` conversion + * 115 (``ord('s')``): ``!s`` :func:`string ` conversion + + This has the same meaning as ``FormattedValue.conversion``. + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + This has the same meaning as ``FormattedValue.format_spec``. + + .. class:: List(elts, ctx) Tuple(elts, ctx) @@ -341,16 +397,14 @@ Literals elts=[ Constant(value=1), Constant(value=2), - Constant(value=3)], - ctx=Load())) + Constant(value=3)])) >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) Expression( body=Tuple( elts=[ Constant(value=1), Constant(value=2), - Constant(value=3)], - ctx=Load())) + Constant(value=3)])) .. class:: Set(elts) @@ -388,7 +442,7 @@ Literals None], values=[ Constant(value=1), - Name(id='d', ctx=Load())])) + Name(id='d')])) Variables @@ -414,7 +468,7 @@ Variables Module( body=[ Expr( - value=Name(id='a', ctx=Load()))]) + value=Name(id='a'))]) >>> print(ast.dump(ast.parse('a = 1'), indent=4)) Module( @@ -452,7 +506,7 @@ Variables value=Name(id='b', ctx=Store()), ctx=Store())], ctx=Store())], - value=Name(id='it', ctx=Load()))]) + value=Name(id='it'))]) .. _ast-expressions: @@ -475,7 +529,7 @@ Expressions Expr( value=UnaryOp( op=USub(), - operand=Name(id='a', ctx=Load())))]) + operand=Name(id='a')))]) .. class:: UnaryOp(op, operand) @@ -498,7 +552,7 @@ Expressions Expression( body=UnaryOp( op=Not(), - operand=Name(id='x', ctx=Load()))) + operand=Name(id='x'))) .. class:: BinOp(left, op, right) @@ -511,9 +565,9 @@ Expressions >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) Expression( body=BinOp( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), op=Add(), - right=Name(id='y', ctx=Load()))) + right=Name(id='y'))) .. class:: Add @@ -549,8 +603,8 @@ Expressions body=BoolOp( op=Or(), values=[ - Name(id='x', ctx=Load()), - Name(id='y', ctx=Load())])) + Name(id='x'), + Name(id='y')])) .. class:: And @@ -575,7 +629,7 @@ Expressions LtE(), Lt()], comparators=[ - Name(id='a', ctx=Load()), + Name(id='a'), Constant(value=10)])) @@ -609,18 +663,17 @@ Expressions >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) Expression( body=Call( - func=Name(id='func', ctx=Load()), + func=Name(id='func'), args=[ - Name(id='a', ctx=Load()), + Name(id='a'), Starred( - value=Name(id='d', ctx=Load()), - ctx=Load())], + value=Name(id='d'))], keywords=[ keyword( arg='b', - value=Name(id='c', ctx=Load())), + value=Name(id='c')), keyword( - value=Name(id='e', ctx=Load()))])) + value=Name(id='e'))])) .. class:: keyword(arg, value) @@ -639,9 +692,9 @@ Expressions >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) Expression( body=IfExp( - test=Name(id='b', ctx=Load()), - body=Name(id='a', ctx=Load()), - orelse=Name(id='c', ctx=Load()))) + test=Name(id='b'), + body=Name(id='a'), + orelse=Name(id='c'))) .. class:: Attribute(value, attr, ctx) @@ -656,9 +709,8 @@ Expressions >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) Expression( body=Attribute( - value=Name(id='snake', ctx=Load()), - attr='colour', - ctx=Load())) + value=Name(id='snake'), + attr='colour')) .. class:: NamedExpr(target, value) @@ -694,15 +746,13 @@ Subscripting >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) Expression( body=Subscript( - value=Name(id='l', ctx=Load()), + value=Name(id='l'), slice=Tuple( elts=[ Slice( lower=Constant(value=1), upper=Constant(value=2)), - Constant(value=3)], - ctx=Load()), - ctx=Load())) + Constant(value=3)]))) .. class:: Slice(lower, upper, step) @@ -716,11 +766,10 @@ Subscripting >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) Expression( body=Subscript( - value=Name(id='l', ctx=Load()), + value=Name(id='l'), slice=Slice( lower=Constant(value=1), - upper=Constant(value=2)), - ctx=Load())) + upper=Constant(value=2)))) Comprehensions @@ -745,11 +794,11 @@ Comprehensions ... )) Expression( body=ListComp( - elt=Name(id='x', ctx=Load()), + elt=Name(id='x'), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) >>> print(ast.dump( ... ast.parse('{x: x**2 for x in numbers}', mode='eval'), @@ -757,15 +806,15 @@ Comprehensions ... )) Expression( body=DictComp( - key=Name(id='x', ctx=Load()), + key=Name(id='x'), value=BinOp( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), op=Pow(), right=Constant(value=2)), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) >>> print(ast.dump( ... ast.parse('{x for x in numbers}', mode='eval'), @@ -773,11 +822,11 @@ Comprehensions ... )) Expression( body=SetComp( - elt=Name(id='x', ctx=Load()), + elt=Name(id='x'), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) @@ -798,17 +847,17 @@ Comprehensions Expression( body=ListComp( elt=Call( - func=Name(id='ord', ctx=Load()), + func=Name(id='ord'), args=[ - Name(id='c', ctx=Load())]), + Name(id='c')]), generators=[ comprehension( target=Name(id='line', ctx=Store()), - iter=Name(id='file', ctx=Load()), + iter=Name(id='file'), is_async=0), comprehension( target=Name(id='c', ctx=Store()), - iter=Name(id='line', ctx=Load()), + iter=Name(id='line'), is_async=0)])) >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), @@ -816,22 +865,22 @@ Comprehensions Expression( body=GeneratorExp( elt=BinOp( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), op=Pow(), right=Constant(value=2)), generators=[ comprehension( target=Name(id='n', ctx=Store()), - iter=Name(id='it', ctx=Load()), + iter=Name(id='it'), ifs=[ Compare( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), ops=[ Gt()], comparators=[ Constant(value=5)]), Compare( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), ops=[ Lt()], comparators=[ @@ -842,11 +891,11 @@ Comprehensions ... indent=4)) # Async comprehension Expression( body=ListComp( - elt=Name(id='i', ctx=Load()), + elt=Name(id='i'), generators=[ comprehension( target=Name(id='i', ctx=Store()), - iter=Name(id='soc', ctx=Load()), + iter=Name(id='soc'), is_async=1)])) @@ -888,7 +937,7 @@ Statements Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], ctx=Store())], - value=Name(id='c', ctx=Load()))]) + value=Name(id='c'))]) .. class:: AnnAssign(target, annotation, value, simple) @@ -911,7 +960,7 @@ Statements body=[ AnnAssign( target=Name(id='c', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=1)]) >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis @@ -919,7 +968,7 @@ Statements body=[ AnnAssign( target=Name(id='a', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), value=Constant(value=1), simple=0)]) @@ -928,10 +977,10 @@ Statements body=[ AnnAssign( target=Attribute( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), attr='b', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=0)]) >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation @@ -939,10 +988,10 @@ Statements body=[ AnnAssign( target=Subscript( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), slice=Constant(value=1), ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=0)]) @@ -979,8 +1028,8 @@ Statements Module( body=[ Raise( - exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))]) + exc=Name(id='x'), + cause=Name(id='y'))]) .. class:: Assert(test, msg) @@ -994,8 +1043,8 @@ Statements Module( body=[ Assert( - test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))]) + test=Name(id='x'), + msg=Name(id='y'))]) .. class:: Delete(targets) @@ -1041,7 +1090,7 @@ Statements body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), - value=Name(id='int', ctx=Load()))]) + value=Name(id='int'))]) .. versionadded:: 3.12 @@ -1134,13 +1183,13 @@ Control flow Module( body=[ If( - test=Name(id='x', ctx=Load()), + test=Name(id='x'), body=[ Expr( value=Constant(value=Ellipsis))], orelse=[ If( - test=Name(id='y', ctx=Load()), + test=Name(id='y'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1174,7 +1223,7 @@ Control flow body=[ For( target=Name(id='x', ctx=Store()), - iter=Name(id='y', ctx=Load()), + iter=Name(id='y'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1199,7 +1248,7 @@ Control flow Module( body=[ While( - test=Name(id='x', ctx=Load()), + test=Name(id='x'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1227,11 +1276,11 @@ Control flow body=[ For( target=Name(id='a', ctx=Store()), - iter=Name(id='b', ctx=Load()), + iter=Name(id='b'), body=[ If( test=Compare( - left=Name(id='a', ctx=Load()), + left=Name(id='a'), ops=[ Gt()], comparators=[ @@ -1269,12 +1318,12 @@ Control flow value=Constant(value=Ellipsis))], handlers=[ ExceptHandler( - type=Name(id='Exception', ctx=Load()), + type=Name(id='Exception'), body=[ Expr( value=Constant(value=Ellipsis))]), ExceptHandler( - type=Name(id='OtherException', ctx=Load()), + type=Name(id='OtherException'), name='e', body=[ Expr( @@ -1309,7 +1358,7 @@ Control flow value=Constant(value=Ellipsis))], handlers=[ ExceptHandler( - type=Name(id='Exception', ctx=Load()), + type=Name(id='Exception'), body=[ Expr( value=Constant(value=Ellipsis))])])]) @@ -1337,12 +1386,12 @@ Control flow body=[ Expr( value=BinOp( - left=Name(id='a', ctx=Load()), + left=Name(id='a'), op=Add(), right=Constant(value=1)))], handlers=[ ExceptHandler( - type=Name(id='TypeError', ctx=Load()), + type=Name(id='TypeError'), body=[ Pass()])])]) @@ -1375,18 +1424,18 @@ Control flow With( items=[ withitem( - context_expr=Name(id='a', ctx=Load()), + context_expr=Name(id='a'), optional_vars=Name(id='b', ctx=Store())), withitem( - context_expr=Name(id='c', ctx=Load()), + context_expr=Name(id='c'), optional_vars=Name(id='d', ctx=Store()))], body=[ Expr( value=Call( - func=Name(id='something', ctx=Load()), + func=Name(id='something'), args=[ - Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())]))])]) + Name(id='b'), + Name(id='d')]))])]) Pattern matching @@ -1426,14 +1475,14 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( patterns=[ MatchAs(name='x')]), guard=Compare( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), ops=[ Gt()], comparators=[ @@ -1443,7 +1492,7 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='tuple', ctx=Load())), + cls=Name(id='tuple')), body=[ Expr( value=Constant(value=Ellipsis))])])]) @@ -1467,7 +1516,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchValue( @@ -1494,7 +1543,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSingleton(value=None), @@ -1521,7 +1570,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( @@ -1554,7 +1603,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( @@ -1603,7 +1652,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchMapping( @@ -1653,11 +1702,11 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchClass( - cls=Name(id='Point2D', ctx=Load()), + cls=Name(id='Point2D'), patterns=[ MatchValue( value=Constant(value=0)), @@ -1668,7 +1717,7 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='Point3D', ctx=Load()), + cls=Name(id='Point3D'), kwd_attrs=[ 'x', 'y', @@ -1709,7 +1758,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchAs( @@ -1746,7 +1795,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchOr( @@ -1786,7 +1835,7 @@ Type annotations body=[ AnnAssign( target=Name(id='x', ctx=Store()), - annotation=Name(id='bool', ctx=Load()), + annotation=Name(id='bool'), value=Constant(value=1), simple=1)], type_ignores=[ @@ -1824,12 +1873,11 @@ aliases. type_params=[ TypeVar( name='T', - bound=Name(id='int', ctx=Load()), - default_value=Name(id='bool', ctx=Load()))], + bound=Name(id='int'), + default_value=Name(id='bool'))], value=Subscript( - value=Name(id='list', ctx=Load()), - slice=Name(id='T', ctx=Load()), - ctx=Load()))]) + value=Name(id='list'), + slice=Name(id='T')))]) .. versionadded:: 3.12 @@ -1854,17 +1902,14 @@ aliases. name='P', default_value=List( elts=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], - ctx=Load()))], + Name(id='int'), + Name(id='str')]))], value=Subscript( - value=Name(id='Callable', ctx=Load()), + value=Name(id='Callable'), slice=Tuple( elts=[ - Name(id='P', ctx=Load()), - Name(id='int', ctx=Load())], - ctx=Load()), - ctx=Load()))]) + Name(id='P'), + Name(id='int')])))]) .. versionadded:: 3.12 @@ -1885,18 +1930,13 @@ aliases. TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - TypeVarTuple( - name='Ts', - default_value=Tuple(ctx=Load()))], + TypeVarTuple(name='Ts', default_value=Tuple())], value=Subscript( - value=Name(id='tuple', ctx=Load()), + value=Name(id='tuple'), slice=Tuple( elts=[ Starred( - value=Name(id='Ts', ctx=Load()), - ctx=Load())], - ctx=Load()), - ctx=Load()))]) + value=Name(id='Ts'))])))]) .. versionadded:: 3.12 @@ -2001,8 +2041,8 @@ Function and class definitions body=[ Pass()], decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], + Name(id='decorator1'), + Name(id='decorator2')], returns=Constant(value='return annotation'))]) @@ -2032,14 +2072,14 @@ Function and class definitions body=[ Expr( value=Yield( - value=Name(id='x', ctx=Load())))]) + value=Name(id='x')))]) >>> print(ast.dump(ast.parse('yield from x'), indent=4)) Module( body=[ Expr( value=YieldFrom( - value=Name(id='x', ctx=Load())))]) + value=Name(id='x')))]) .. class:: Global(names) @@ -2094,17 +2134,17 @@ Function and class definitions ClassDef( name='Foo', bases=[ - Name(id='base1', ctx=Load()), - Name(id='base2', ctx=Load())], + Name(id='base1'), + Name(id='base2')], keywords=[ keyword( arg='metaclass', - value=Name(id='meta', ctx=Load()))], + value=Name(id='meta'))], body=[ Pass()], decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())])]) + Name(id='decorator1'), + Name(id='decorator2')])]) .. versionchanged:: 3.12 Added ``type_params``. @@ -2141,7 +2181,7 @@ Async and await Expr( value=Await( value=Call( - func=Name(id='other_func', ctx=Load()))))])]) + func=Name(id='other_func'))))])]) .. class:: AsyncFor(target, iter, body, orelse, type_comment) @@ -2165,10 +2205,10 @@ Async and await Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``, where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` and ``ast.PyCF_OPTIMIZED_AST`` otherwise. @@ -2221,6 +2261,9 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionadded:: 3.15 + Added the *module* parameter. + .. function:: unparse(ast_obj) @@ -2402,7 +2445,7 @@ and classes for traversing abstract syntax trees: def visit_Name(self, node): return Subscript( - value=Name(id='data', ctx=Load()), + value=Name(id='data'), slice=Constant(value=node.id), ctx=node.ctx ) @@ -2445,8 +2488,26 @@ and classes for traversing abstract syntax trees: indents that many spaces per level. If *indent* is a string (such as ``"\t"``), that string is used to indent each level. - If *show_empty* is ``False`` (the default), empty lists and fields that are ``None`` - will be omitted from the output. + If *show_empty* is false (the default), optional empty lists and + ``Load()`` values will be omitted from the output. + Optional ``None`` values are always omitted. + + .. doctest:: + + >>> tree = ast.parse('print(None)', '?', 'eval') + >>> print(ast.dump(tree, indent=4)) + Expression( + body=Call( + func=Name(id='print'), + args=[ + Constant(value=None)])) + >>> print(ast.dump(tree, indent=4, show_empty=True)) + Expression( + body=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=None)], + keywords=[])) .. versionchanged:: 3.9 Added the *indent* option. @@ -2454,32 +2515,8 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.13 Added the *show_empty* option. - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... async def f(): - ... await other_func() - ... """), indent=4, show_empty=True)) - Module( - body=[ - AsyncFunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=[ - Expr( - value=Await( - value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[], - type_params=[])], - type_ignores=[]) + .. versionchanged:: 3.15 + Omit optional ``Load()`` values by default. .. _ast-compiler-flags: diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 44b507a9811..7831b613bd4 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -46,10 +46,6 @@ In addition to enabling the debug mode, consider also: When the debug mode is enabled: -* asyncio checks for :ref:`coroutines that were not awaited - ` and logs them; this mitigates - the "forgotten await" pitfall. - * Many non-threadsafe asyncio APIs (such as :meth:`loop.call_soon` and :meth:`loop.call_at` methods) raise an exception if they are called from a wrong thread. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 21f7d0547af..72f484fd1cb 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -304,6 +304,12 @@ clocks to track time. custom :class:`contextvars.Context` for the *callback* to run in. The current context is used when no *context* is provided. + .. note:: + + For performance, callbacks scheduled with :meth:`loop.call_later` + may run up to one clock-resolution early (see + ``time.get_clock_info('monotonic').resolution``). + .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. @@ -324,6 +330,12 @@ clocks to track time. An instance of :class:`asyncio.TimerHandle` is returned which can be used to cancel the callback. + .. note:: + + For performance, callbacks scheduled with :meth:`loop.call_at` + may run up to one clock-resolution early (see + ``time.get_clock_info('monotonic').resolution``). + .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. @@ -361,7 +373,7 @@ Creating Futures and Tasks .. versionadded:: 3.5.2 -.. method:: loop.create_task(coro, *, name=None, context=None, eager_start=None) +.. method:: loop.create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Schedule the execution of :ref:`coroutine ` *coro*. Return a :class:`Task` object. @@ -370,6 +382,10 @@ Creating Futures and Tasks for interoperability. In this case, the result type is a subclass of :class:`Task`. + The full function signature is largely the same as that of the + :class:`Task` constructor (or factory) - all of the keyword arguments to + this function are passed through to that interface. + If the *name* argument is provided and not ``None``, it is set as the name of the task using :meth:`Task.set_name`. @@ -388,8 +404,15 @@ Creating Futures and Tasks .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.13.3 + Added ``kwargs`` which passes on arbitrary extra parameters, including ``name`` and ``context``. + + .. versionchanged:: 3.13.4 + Rolled back the change that passes on *name* and *context* (if it is None), + while still passing on other arbitrary keyword arguments (to avoid breaking backwards compatibility with 3.13.3). + .. versionchanged:: 3.14 - Added the *eager_start* parameter. + All *kwargs* are now passed on. The *eager_start* parameter works with eager task factories. .. method:: loop.set_task_factory(factory) @@ -402,6 +425,16 @@ Creating Futures and Tasks event loop, and *coro* is a coroutine object. The callable must pass on all *kwargs*, and return a :class:`asyncio.Task`-compatible object. + .. versionchanged:: 3.13.3 + Required that all *kwargs* are passed on to :class:`asyncio.Task`. + + .. versionchanged:: 3.13.4 + *name* is no longer passed to task factories. *context* is no longer passed + to task factories if it is ``None``. + + .. versionchanged:: 3.14 + *name* and *context* are now unconditionally passed on to task factories again. + .. method:: loop.get_task_factory() Return a task factory or ``None`` if the default one is in use. @@ -590,6 +623,12 @@ Opening network connections to bind the socket locally. The *local_host* and *local_port* are looked up using :meth:`getaddrinfo`. + .. note:: + + On Windows, when using the proactor event loop with ``local_addr=None``, + an :exc:`OSError` with :attr:`!errno.WSAEINVAL` will be raised + when running it. + * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used to connect the socket to a remote address. The *remote_host* and *remote_port* are looked up using :meth:`getaddrinfo`. @@ -1592,6 +1631,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. method:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) @@ -1615,6 +1657,9 @@ async/await code consider using the high-level conforms to the :class:`SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. note:: It is the application's responsibility to ensure that all whitespace and special characters are quoted appropriately to avoid `shell injection diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 32771ba72e0..4b69e569523 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -75,6 +75,7 @@ Future Functions Deprecation warning is emitted if *future* is not a Future-like object and *loop* is not specified and there is no running event loop. +.. _asyncio-future-obj: Future Object ============= diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 7c08d65f26b..5208f14c94a 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -390,11 +390,11 @@ Subprocess Transports Return the transport for the communication pipe corresponding to the integer file descriptor *fd*: - * ``0``: readable streaming transport of the standard input (*stdin*), + * ``0``: writable streaming transport of the standard input (*stdin*), or :const:`None` if the subprocess was not created with ``stdin=PIPE`` - * ``1``: writable streaming transport of the standard output (*stdout*), + * ``1``: readable streaming transport of the standard output (*stdout*), or :const:`None` if the subprocess was not created with ``stdout=PIPE`` - * ``2``: writable streaming transport of the standard error (*stderr*), + * ``2``: readable streaming transport of the standard error (*stderr*), or :const:`None` if the subprocess was not created with ``stderr=PIPE`` * other *fd*: :const:`None` diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d99213aa81d..d481a1921d5 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -102,17 +102,34 @@ Queue .. method:: shutdown(immediate=False) - Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` + Put a :class:`Queue` instance into a shutdown mode. + + The queue can no longer grow. + Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`. + Currently blocked callers of :meth:`~Queue.put` will be unblocked + and will raise :exc:`QueueShutDown` in the formerly blocked thread. + + If *immediate* is false (the default), the queue can be wound + down normally with :meth:`~Queue.get` calls to extract tasks + that have already been loaded. + + And if :meth:`~Queue.task_done` is called for each remaining task, a + pending :meth:`~Queue.join` will be unblocked normally. + + Once the queue is empty, future calls to :meth:`~Queue.get` will raise :exc:`QueueShutDown`. - By default, :meth:`~Queue.get` on a shut down queue will only - raise once the queue is empty. Set *immediate* to true to make - :meth:`~Queue.get` raise immediately instead. + If *immediate* is true, the queue is terminated immediately. + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` + are unblocked and will raise :exc:`QueueShutDown` because the + queue is empty. - All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` - will be unblocked. If *immediate* is true, a task will be marked - as done for each remaining item in the queue, which may unblock - callers of :meth:`~Queue.join`. + Use caution when using :meth:`~Queue.join` with *immediate* set + to true. This unblocks the join even when no work has been done + on the tasks, violating the usual invariant for joining a queue. .. versionadded:: 3.13 @@ -129,9 +146,6 @@ Queue call was received for every item that had been :meth:`~Queue.put` into the queue). - ``shutdown(immediate=True)`` calls :meth:`task_done` for each - remaining item in the queue. - Raises :exc:`ValueError` if called more times than there were items placed in the queue. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index c56166cabb9..05445219510 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -171,13 +171,17 @@ and work with streams: .. function:: start_unix_server(client_connected_cb, path=None, \ *, limit=None, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, start_serving=True) + ssl_shutdown_timeout=None, start_serving=True, cleanup_socket=True) :async: Start a Unix socket server. Similar to :func:`start_server` but works with Unix sockets. + If *cleanup_socket* is true then the Unix socket will automatically + be removed from the filesystem when the server is closed, unless the + socket has been replaced after the server has been created. + See also the documentation of :meth:`loop.create_unix_server`. .. note:: @@ -198,6 +202,9 @@ and work with streams: .. versionchanged:: 3.11 Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.13 + Added the *cleanup_socket* parameter. + StreamReader ============ @@ -309,11 +316,15 @@ StreamWriter If that fails, the data is queued in an internal write buffer until it can be sent. + The *data* buffer should be a bytes, bytearray, or C-contiguous one-dimensional + memoryview object. + The method should be used along with the ``drain()`` method:: stream.write(data) await stream.drain() + .. method:: writelines(data) The method writes a list (or any iterable) of bytes to the underlying socket diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 03e76bc8689..9416c758e51 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -76,6 +76,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -95,6 +98,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. important:: It is the application's responsibility to ensure that all whitespace and diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 968c812ee3c..f9e98e05cab 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -157,7 +157,7 @@ Event Clear (unset) the event. - Tasks awaiting on :meth:`~Event.wait` will now block until the + Subsequent tasks awaiting on :meth:`~Event.wait` will now block until the :meth:`~Event.set` method is called again. .. method:: is_set() diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 59acce1990a..863b3e33657 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -238,18 +238,24 @@ Creating Tasks ----------------------------------------------- -.. function:: create_task(coro, *, name=None, context=None) +.. function:: create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Wrap the *coro* :ref:`coroutine ` into a :class:`Task` and schedule its execution. Return the Task object. - If *name* is not ``None``, it is set as the name of the task using - :meth:`Task.set_name`. + The full function signature is largely the same as that of the + :class:`Task` constructor (or factory) - all of the keyword arguments to + this function are passed through to that interface. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *coro* to run in. The current context copy is created when no *context* is provided. + An optional keyword-only *eager_start* argument allows specifying + if the task should execute eagerly during the call to create_task, + or be scheduled later. If *eager_start* is not passed the mode set + by :meth:`loop.set_task_factory` will be used. + The task is executed in the loop returned by :func:`get_running_loop`, :exc:`RuntimeError` is raised if there is no running loop in current thread. @@ -290,6 +296,9 @@ Creating Tasks .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.14 + Added the *eager_start* parameter by passing on all *kwargs*. + Task Cancellation ================= @@ -330,7 +339,7 @@ and reliable way to wait for all tasks in the group to finish. .. versionadded:: 3.11 - .. method:: create_task(coro, *, name=None, context=None) + .. method:: create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Create a task in this task group. The signature matches that of :func:`asyncio.create_task`. @@ -342,6 +351,10 @@ and reliable way to wait for all tasks in the group to finish. Close the given coroutine if the task group is not active. + .. versionchanged:: 3.14 + + Passes on all *kwargs* to :meth:`loop.create_task` + Example:: async def main(): @@ -1180,6 +1193,7 @@ Introspection .. versionadded:: 3.4 +.. _asyncio-task-obj: Task Object =========== @@ -1207,8 +1221,8 @@ Task Object To cancel a running Task use the :meth:`cancel` method. Calling it will cause the Task to throw a :exc:`CancelledError` exception into - the wrapped coroutine. If a coroutine is awaiting on a Future - object during cancellation, the Future object will be cancelled. + the wrapped coroutine. If a coroutine is awaiting on a future-like + object during cancellation, the awaited object will be cancelled. :meth:`cancelled` can be used to check if the Task was cancelled. The method returns ``True`` if the wrapped coroutine did not @@ -1397,6 +1411,10 @@ Task Object the cancellation, it needs to call :meth:`Task.uncancel` in addition to catching the exception. + If the Task being cancelled is currently awaiting on a future-like + object, that awaited object will also be cancelled. This cancellation + propagates down the entire chain of awaited objects. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 7d368dae49d..0f72e31dee5 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -29,6 +29,11 @@ database connection libraries, distributed task queues, etc. asyncio is often a perfect fit for IO-bound and high-level **structured** network code. +.. seealso:: + + :ref:`a-conceptual-overview-of-asyncio` + Explanation of the fundamentals of asyncio. + asyncio provides a set of **high-level** APIs to: * :ref:`run Python coroutines ` concurrently and @@ -74,6 +79,10 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: >>> await asyncio.sleep(10, result='hello') 'hello' +This REPL provides limited compatibility with :envvar:`PYTHON_BASIC_REPL`. +It is recommended that the default REPL is used +for full functionality and the latest features. + .. audit-event:: cpython.run_stdin "" "" .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 834ab2536e6..529a7242443 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -15,14 +15,9 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. -It provides encoding and decoding functions for the encodings specified in -:rfc:`4648`, which defines the Base16, Base32, and Base64 algorithms, -and for the de-facto standard Ascii85 and Base85 encodings. - -The :rfc:`4648` encodings are suitable for encoding binary data so that it can be -safely sent by email, used as parts of URLs, or included as part of an HTTP -POST request. The encoding algorithm is not the same as the -:program:`uuencode` program. +This includes the :ref:`encodings specified in ` +:rfc:`4648` (Base64, Base32 and Base16) +and the non-standard :ref:`Base85 encodings `. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -30,7 +25,7 @@ supports encoding :term:`bytes-like objects ` to ASCII strings containing ASCII to :class:`bytes`. Both base-64 alphabets defined in :rfc:`4648` (normal, and URL- and filesystem-safe) are supported. -The legacy interface does not support decoding from strings, but it does +The :ref:`legacy interface ` does not support decoding from strings, but it does provide functions for encoding and decoding to and from :term:`file objects `. It only supports the Base64 standard alphabet, and it adds newlines every 76 characters as per :rfc:`2045`. Note that if you are looking @@ -46,7 +41,15 @@ package instead. Any :term:`bytes-like objects ` are now accepted by all encoding and decoding functions in this module. Ascii85/Base85 support added. -The modern interface provides: + +.. _base64-rfc-4648: + +RFC 4648 Encodings +------------------ + +The :rfc:`4648` encodings are suitable for encoding binary data so that it can be +safely sent by email, used as parts of URLs, or included as part of an HTTP +POST request. .. function:: b64encode(s, altchars=None) @@ -181,6 +184,26 @@ The modern interface provides: incorrectly padded or if there are non-alphabet characters present in the input. +.. _base64-base-85: + +Base85 Encodings +----------------- + +Base85 encoding is not formally specified but rather a de facto standard, +thus different systems perform the encoding differently. + +The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of +the de facto standard. You should call the function with the Base85 +implementation used by the software you intend to work with. + +The two functions present in this module differ in how they handle the following: + +* Whether to include enclosing ``<~`` and ``~>`` markers +* Whether to include newline characters +* The set of ASCII characters used for encoding +* Handling of null bytes + +Refer to the documentation of the individual functions for more information. .. function:: a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) @@ -262,7 +285,10 @@ The modern interface provides: .. versionadded:: 3.13 -The legacy interface: +.. _base64-legacy: + +Legacy Interface +---------------- .. function:: decode(input, output) diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 90f042aa377..a3c6da7a6d6 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -192,12 +192,8 @@ The :mod:`bdb` module also defines two classes: entered. * ``"return"``: A function or other code block is about to return. * ``"exception"``: An exception has occurred. - * ``"c_call"``: A C function is about to be called. - * ``"c_return"``: A C function has returned. - * ``"c_exception"``: A C function has raised an exception. - For the Python events, specialized functions (see below) are called. For - the C events, no action is taken. + For all the events, specialized functions (see below) are called. The *arg* parameter depends on the previous event. @@ -240,7 +236,7 @@ The :mod:`bdb` module also defines two classes: Normally derived classes don't override the following methods, but they may if they want to redefine the definition of stopping and breakpoints. - .. method:: is_skipped_line(module_name) + .. method:: is_skipped_module(module_name) Return ``True`` if *module_name* matches any skip pattern. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 78da563397b..d5ec4212c1f 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -24,6 +24,16 @@ method to determine whether a value has been found. Instead, the functions only call the :meth:`~object.__lt__` method and will return an insertion point between values in an array. +.. note:: + + The functions in this module are not thread-safe. If multiple threads + concurrently use :mod:`bisect` functions on the same sequence, this + may result in undefined behaviour. Likewise, if the provided sequence + is mutated by a different thread while a :mod:`bisect` function + is operating on it, the result is undefined. For example, using + :py:func:`~bisect.insort_left` on the same list from multiple threads + may result in the list becoming unsorted. + .. _bisect functions: The following functions are provided: @@ -73,7 +83,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. - Next, it runs the :meth:`!insert` method on *a* to insert *x* at the + Next, it runs the :meth:`~sequence.insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is @@ -93,7 +103,7 @@ The following functions are provided: entries of *x*. This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. - Next, it runs the :meth:`!insert` method on *a* to insert *x* at the + Next, it runs the :meth:`~sequence.insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ebe2e43feba..12650861c0f 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -25,6 +25,8 @@ The :mod:`bz2` module contains: * The :func:`compress` and :func:`decompress` functions for one-shot (de)compression. +.. include:: ../includes/optional-module.rst + (De)compression of files ------------------------ diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 39090e36ed9..f76b1013dfb 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -251,7 +251,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is 3) specifies the number of months per row. *css* is the name for the cascading style sheet to be used. :const:`None` can be passed if no style sheet should be used. *encoding* specifies the encoding to be used for the - output (defaulting to the system default encoding). + output (defaulting to ``'utf-8'``). .. method:: formatmonthname(theyear, themonth, withyear=True) @@ -501,6 +501,14 @@ The :mod:`calendar` module exports the following data attributes: >>> list(calendar.month_name) ['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + .. caution:: + + In locales with alternative month names forms, the :data:`!month_name` sequence + may not be suitable when a month name stands by itself and not as part of a date. + For instance, in Greek and in many Slavic and Baltic languages, :data:`!month_name` + will produce the month in genitive case. Use :data:`standalone_month_name` for a form + suitable for standalone use. + .. data:: month_abbr @@ -512,6 +520,31 @@ The :mod:`calendar` module exports the following data attributes: >>> list(calendar.month_abbr) ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + .. caution:: + + In locales with alternative month names forms, the :data:`!month_abbr` sequence + may not be suitable when a month name stands by itself and not as part of a date. + Use :data:`standalone_month_abbr` for a form suitable for standalone use. + + +.. data:: standalone_month_name + + A sequence that represents the months of the year in the current locale + in the standalone form if the locale provides one. Else it is equivalent + to :data:`month_name`. + + .. versionadded:: 3.15 + + +.. data:: standalone_month_abbr + + A sequence that represents the abbreviated months of the year in the current + locale in the standalone form if the locale provides one. Else it is + equivalent to :data:`month_abbr`. + + .. versionadded:: 3.15 + + .. data:: JANUARY FEBRUARY MARCH @@ -677,8 +710,7 @@ The following options are accepted: .. option:: month The month of the specified :option:`year` to print the calendar for. - Must be a number between 1 and 12, - and may only be used in text mode. + Must be a number between 1 and 12. Defaults to printing a calendar for the full year. diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 26518a0458f..b6d5dbee21d 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -338,7 +338,7 @@ Constants .. data:: nan A floating-point "not a number" (NaN) value. Equivalent to - ``float('nan')``. + ``float('nan')``. See also :data:`math.nan`. .. versionadded:: 3.6 diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 16c67ddbf7c..c43b10157f9 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -16,17 +16,17 @@ The following modules have a command-line interface. * :ref:`dis ` * :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` * :ref:`mimetypes ` -* :mod:`pdb` +* :ref:`pdb ` * :ref:`pickle ` * :ref:`pickletools ` * :ref:`platform ` @@ -52,8 +52,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/cmdlinelibs.rst b/Doc/library/cmdlinelibs.rst index 085d31af7bc..32f8c2c9f4a 100644 --- a/Doc/library/cmdlinelibs.rst +++ b/Doc/library/cmdlinelibs.rst @@ -1,7 +1,7 @@ .. _cmdlinelibs: ******************************** -Command Line Interface Libraries +Command-line interface libraries ******************************** The modules described in this chapter assist with implementing @@ -19,3 +19,4 @@ Here's an overview: curses.rst curses.ascii.rst curses.panel.rst + cmd.rst diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 8f7692df9fb..59c016d2150 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -22,8 +22,14 @@ build applications which provide an interactive interpreter prompt. it defaults to a newly created dictionary with key ``'__name__'`` set to ``'__console__'`` and key ``'__doc__'`` set to ``None``. + Note that functions and classes objects created under an + :class:`!InteractiveInterpreter` instance will belong to the namespace + specified by *locals*. + They are only pickleable if *locals* is the namespace of an existing + module. -.. class:: InteractiveConsole(locals=None, filename="", local_exit=False) + +.. class:: InteractiveConsole(locals=None, filename="", *, local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 86511602fa5..305e5d07a35 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -53,6 +53,14 @@ any codec: :exc:`UnicodeDecodeError`). Refer to :ref:`codec-base-classes` for more information on codec error handling. +.. function:: charmap_build(string) + + Return a mapping suitable for encoding with a custom single-byte encoding. + Given a :class:`str` *string* of up to 256 characters representing a + decoding table, returns either a compact internal mapping object + ``EncodingMap`` or a :class:`dictionary ` mapping character ordinals + to byte values. Raises a :exc:`TypeError` on invalid input. + The full details for each codec can also be looked up directly: .. function:: lookup(encoding, /) @@ -60,11 +68,21 @@ The full details for each codec can also be looked up directly: Looks up the codec info in the Python codec registry and returns a :class:`CodecInfo` object as defined below. - Encodings are first looked up in the registry's cache. If not found, the list of + This function first normalizes the *encoding*: all ASCII letters are + converted to lower case, spaces are replaced with hyphens. + Then encoding is looked up in the registry's cache. If not found, the list of registered search functions is scanned. If no :class:`CodecInfo` object is found, a :exc:`LookupError` is raised. Otherwise, the :class:`CodecInfo` object is stored in the cache and returned to the caller. + .. versionchanged:: 3.9 + Any characters except ASCII letters and digits and a dot are converted to underscore. + + .. versionchanged:: 3.15 + No characters are converted to underscore anymore. + Spaces are converted to hyphens. + + .. class:: CodecInfo(encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None) Codec details when looking up the codec registry. The constructor @@ -159,14 +177,11 @@ function: .. function:: register(search_function, /) Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters with hyphens - and spaces converted to underscores, and return a :class:`CodecInfo` object. + argument, being the encoding name in all lower case letters with spaces + converted to hyphens, and return a :class:`CodecInfo` object. In case a search function cannot find a given encoding, it should return ``None``. - .. versionchanged:: 3.9 - Hyphens and spaces are converted to underscore. - .. function:: unregister(search_function, /) @@ -235,8 +250,8 @@ wider range of codecs when working with binary files: .. function:: iterencode(iterator, encoding, errors='strict', **kwargs) Uses an incremental encoder to iteratively encode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`str` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental encoder. This function requires that the codec accept text :class:`str` objects @@ -247,8 +262,8 @@ wider range of codecs when working with binary files: .. function:: iterdecode(iterator, encoding, errors='strict', **kwargs) Uses an incremental decoder to iteratively decode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`bytes` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental decoder. This function requires that the codec accept :class:`bytes` objects @@ -257,6 +272,20 @@ wider range of codecs when working with binary files: :func:`iterencode`. +.. function:: readbuffer_encode(buffer, errors=None, /) + + Return a :class:`tuple` containing the raw bytes of *buffer*, a + :ref:`buffer-compatible object ` or :class:`str` + (encoded to UTF-8 before processing), and their length in bytes. + + The *errors* argument is ignored. + + .. code-block:: pycon + + >>> codecs.readbuffer_encode(b"Zito") + (b'Zito', 4) + + The module also provides the following constants which are useful for reading and writing to platform dependent files: @@ -960,17 +989,22 @@ defined in Unicode. A simple and straightforward way that can store each Unicode code point, is to store each code point as four consecutive bytes. There are two possibilities: store the bytes in big endian or in little endian order. These two encodings are called ``UTF-32-BE`` and ``UTF-32-LE`` respectively. Their -disadvantage is that if e.g. you use ``UTF-32-BE`` on a little endian machine you -will always have to swap bytes on encoding and decoding. ``UTF-32`` avoids this -problem: bytes will always be in natural endianness. When these bytes are read -by a CPU with a different endianness, then bytes have to be swapped though. To -be able to detect the endianness of a ``UTF-16`` or ``UTF-32`` byte sequence, -there's the so called BOM ("Byte Order Mark"). This is the Unicode character -``U+FEFF``. This character can be prepended to every ``UTF-16`` or ``UTF-32`` -byte sequence. The byte swapped version of this character (``0xFFFE``) is an -illegal character that may not appear in a Unicode text. So when the -first character in a ``UTF-16`` or ``UTF-32`` byte sequence -appears to be a ``U+FFFE`` the bytes have to be swapped on decoding. +disadvantage is that if, for example, you use ``UTF-32-BE`` on a little endian +machine you will always have to swap bytes on encoding and decoding. +Python's ``UTF-16`` and ``UTF-32`` codecs avoid this problem by using the +platform's native byte order when no BOM is present. +Python follows prevailing platform +practice, so native-endian data round-trips without redundant byte swapping, +even though the Unicode Standard defaults to big-endian when the byte order is +unspecified. When these bytes are read by a CPU with a different endianness, +the bytes have to be swapped. To be able to detect the endianness of a +``UTF-16`` or ``UTF-32`` byte sequence, a BOM ("Byte Order Mark") is used. +This is the Unicode character ``U+FEFF``. This character can be prepended to every +``UTF-16`` or ``UTF-32`` byte sequence. The byte swapped version of this character +(``0xFFFE``) is an illegal character that may not appear in a Unicode text. +When the first character of a ``UTF-16`` or ``UTF-32`` byte sequence is +``U+FFFE``, the bytes have to be swapped on decoding. + Unfortunately the character ``U+FEFF`` had a second purpose as a ``ZERO WIDTH NO-BREAK SPACE``: a character that has no width and doesn't allow a word to be split. It can e.g. be used to give hints to a ligature algorithm. @@ -1043,11 +1077,18 @@ or with dictionaries as mapping tables. The following table lists the codecs by name, together with a few common aliases, and the languages for which the encoding is likely used. Neither the list of aliases nor the list of languages is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases; therefore, -e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. +case or use a space or a hyphen instead of an underscore are also valid aliases +because they are equivalent when normalized by +:func:`~encodings.normalize_encoding`. For example, ``'utf-8'`` is a valid +alias for the ``'utf_8'`` codec. + +.. note:: + + The below table lists the most common aliases, for a complete list + refer to the source :source:`aliases.py ` file. On Windows, ``cpXXX`` codecs are available for all code pages. -But only codecs listed in the following table are guarantead to exist on +But only codecs listed in the following table are guaranteed to exist on other platforms. .. impl-detail:: @@ -1222,7 +1263,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | iso8859_3 | iso-8859-3, latin3, L3 | Esperanto, Maltese | +-----------------+--------------------------------+--------------------------------+ -| iso8859_4 | iso-8859-4, latin4, L4 | Baltic languages | +| iso8859_4 | iso-8859-4, latin4, L4 | Northern Europe | +-----------------+--------------------------------+--------------------------------+ | iso8859_5 | iso-8859-5, cyrillic | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | @@ -1303,7 +1344,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | utf_8 | U8, UTF, utf8, cp65001 | all languages | +-----------------+--------------------------------+--------------------------------+ -| utf_8_sig | | all languages | +| utf_8_sig | utf8-sig | all languages | +-----------------+--------------------------------+--------------------------------+ .. versionchanged:: 3.4 @@ -1373,7 +1414,11 @@ encodings. | | | It is used in the Python | | | | pickle protocol. | +--------------------+---------+---------------------------+ -| undefined | | Raise an exception for | +| undefined | | This Codec should only | +| | | be used for testing | +| | | purposes. | +| | | | +| | | Raise an exception for | | | | all conversions, even | | | | empty strings. The error | | | | handler is ignored. | @@ -1450,6 +1495,36 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` Restoration of the aliases for the binary transforms. +.. _standalone-codec-functions: + +Standalone Codec Functions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions provide encoding and decoding functionality similar to codecs, +but are not available as named codecs through :func:`codecs.encode` or :func:`codecs.decode`. +They are used internally (for example, by :mod:`pickle`) and behave similarly to the +``string_escape`` codec that was removed in Python 3. + +.. function:: codecs.escape_encode(input, errors=None) + + Encode *input* using escape sequences. Similar to how :func:`repr` on bytes + produces escaped byte values. + + *input* must be a :class:`bytes` object. + + Returns a tuple ``(output, length)`` where *output* is a :class:`bytes` + object and *length* is the number of bytes consumed. + +.. function:: codecs.escape_decode(input, errors=None) + + Decode *input* from escape sequences back to the original bytes. + + *input* must be a :term:`bytes-like object`. + + Returns a tuple ``(output, length)`` where *output* is a :class:`bytes` + object and *length* is the number of bytes consumed. + + .. _text-transforms: Text Transforms @@ -1476,6 +1551,66 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. +:mod:`encodings` --- Encodings package +-------------------------------------- + +.. module:: encodings + :synopsis: Encodings package + +This module implements the following functions: + +.. function:: normalize_encoding(encoding) + + Normalize encoding name *encoding*. + + Normalization works as follows: all non-alphanumeric characters except the + dot used for Python package names are collapsed and replaced with a single + underscore, leading and trailing underscores are removed. + For example, ``' -;#'`` becomes ``'_'``. + + Note that *encoding* should be ASCII only. + + +.. note:: + The following functions should not be used directly, except for testing + purposes; :func:`codecs.lookup` should be used instead. + + +.. function:: search_function(encoding) + + Search for the codec module corresponding to the given encoding name + *encoding*. + + This function first normalizes the *encoding* using + :func:`normalize_encoding`, then looks for a corresponding alias. + It attempts to import a codec module from the encodings package using either + the alias or the normalized name. If the module is found and defines a valid + ``getregentry()`` function that returns a :class:`codecs.CodecInfo` object, + the codec is cached and returned. + + If the codec module defines a ``getaliases()`` function any returned aliases + are registered for future use. + + +.. function:: win32_code_page_search_function(encoding) + + Search for a Windows code page encoding *encoding* of the form ``cpXXXX``. + + If the code page is valid and supported, return a :class:`codecs.CodecInfo` + object for it. + + .. availability:: Windows. + + .. versionadded:: 3.14 + + +This module implements the following exception: + +.. exception:: CodecRegistryError + + Raised when a codec is invalid or incompatible. + + :mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------ diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index daa9af6d1dd..e6daccb91f2 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -140,6 +140,9 @@ ABC Inherits from Abstract Methods Mi ``__len__``, ``insert`` +:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods + ``__len__`` + :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, ``__len__`` ``__sub__``, ``__rsub__``, ``__xor__``, ``__rxor__`` @@ -260,21 +263,50 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Sequence MutableSequence + ByteString ABCs for read-only and mutable :term:`sequences `. Implementation note: Some of the mixin methods, such as - :meth:`~container.__iter__`, :meth:`~object.__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`~object.__getitem__` method. + :meth:`~container.__iter__`, :meth:`~object.__reversed__`, + and :meth:`~sequence.index` make repeated calls to the underlying + :meth:`~object.__getitem__` method. Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; however, if the underlying method is linear (as it would be with a linked list), the mixins will have quadratic performance and will likely need to be overridden. - .. versionchanged:: 3.5 - The index() method added support for *stop* and *start* - arguments. + .. method:: index(value, start=0, stop=None) + + Return first index of *value*. + + Raises :exc:`ValueError` if the value is not present. + + Supporting the *start* and *stop* arguments is optional, but recommended. + + .. versionchanged:: 3.5 + The :meth:`~sequence.index` method gained support for + the *stop* and *start* arguments. + + .. deprecated-removed:: 3.12 3.17 + The :class:`ByteString` ABC has been deprecated. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`Buffer` or a union that + explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was + an instance of :class:`!ByteString` never actually told you anything + useful about the object. Other common buffer types such as + :class:`memoryview` were also never understood as subtypes of + :class:`!ByteString` (either at runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. .. class:: Set MutableSet @@ -304,7 +336,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -322,7 +354,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 5fbdb12f40c..4e0db485e06 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -367,9 +367,11 @@ Several mathematical operations are provided for combining :class:`Counter` objects to produce multisets (counters that have counts greater than zero). Addition and subtraction combine counters by adding or subtracting the counts of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Equality and inclusion compare -corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. +maximum of corresponding counts. Symmetric difference returns the difference +between the maximum and minimum of the corresponding counts. Equality and +inclusion compare corresponding counts. Each operation can accept inputs +with signed counts, but the output will exclude results with counts of zero +or below. .. doctest:: @@ -383,6 +385,8 @@ counts, but the output will exclude results with counts of zero or less. Counter({'a': 1, 'b': 1}) >>> c | d # union: max(c[x], d[x]) Counter({'a': 3, 'b': 2}) + >>> c ^ d # max(c[x], d[x]) - min(c[x], d[x]) + Counter({'a': 2, 'b': 1}) >>> c == d # equality: c[x] == d[x] False >>> c <= d # inclusion: c[x] <= d[x] @@ -400,6 +404,9 @@ or subtracting from an empty counter. .. versionadded:: 3.3 Added support for unary plus, unary minus, and in-place multiset operations. +.. versionadded:: 3.15 + Added support for the symmetric difference multiset operation, ``c ^ d``. + .. note:: Counters were primarily designed to work with positive integers to represent @@ -758,9 +765,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, .. attribute:: default_factory - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. + This attribute is used by the :meth:`~defaultdict.__missing__` method; + it is initialized from the first argument to the constructor, if present, + or to ``None``, if absent. .. versionchanged:: 3.9 Added merge (``|``) and update (``|=``) operators, specified in @@ -783,10 +790,10 @@ sequence of key-value pairs into a dictionary of lists: When each key is encountered for the first time, it is not already in the mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`!list.append` +function which returns an empty :class:`list`. The :meth:`list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the -:meth:`!list.append` operation adds another value to the list. This technique is +:meth:`list.append` operation adds another value to the list. This technique is simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> d = {} @@ -1202,7 +1209,7 @@ If a new entry overwrites an existing entry, the original insertion position is changed and moved to the end:: class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' + 'Store items in the order that the keys were last updated.' def __setitem__(self, key, value): super().__setitem__(key, value) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index c42288419c4..ebbbf857e71 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -56,11 +56,18 @@ compile Python sources. executed. .. option:: -s strip_prefix + + Remove the given prefix from paths recorded in the ``.pyc`` files. + Paths are made relative to the prefix. + + This option can be used with ``-p`` but not with ``-d``. + .. option:: -p prepend_prefix - Remove (``-s``) or append (``-p``) the given prefix of paths - recorded in the ``.pyc`` files. - Cannot be combined with ``-d``. + Prepend the given prefix to paths recorded in the ``.pyc`` files. + Use ``-p /`` to make the paths absolute. + + This option can be used with ``-s`` but not with ``-d``. .. option:: -x regex diff --git a/Doc/library/compression.rst b/Doc/library/compression.rst new file mode 100644 index 00000000000..98719be9992 --- /dev/null +++ b/Doc/library/compression.rst @@ -0,0 +1,20 @@ +The :mod:`!compression` package +=============================== + +.. module:: compression + +.. versionadded:: 3.14 + +The :mod:`!compression` package contains the canonical compression modules +containing interfaces to several different compression algorithms. Some of +these modules have historically been available as separate modules; those will +continue to be available under their original names for compatibility reasons, +and will not be removed without a deprecation cycle. The use of modules in +:mod:`!compression` is encouraged where practical. + +* :mod:`!compression.bz2` -- Re-exports :mod:`bz2` +* :mod:`!compression.gzip` -- Re-exports :mod:`gzip` +* :mod:`!compression.lzma` -- Re-exports :mod:`lzma` +* :mod:`!compression.zlib` -- Re-exports :mod:`zlib` +* :mod:`compression.zstd` -- Wrapper for the Zstandard compression library + diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst new file mode 100644 index 00000000000..89b6fe540f5 --- /dev/null +++ b/Doc/library/compression.zstd.rst @@ -0,0 +1,899 @@ +:mod:`!compression.zstd` --- Compression compatible with the Zstandard format +============================================================================= + +.. module:: compression.zstd + :synopsis: Low-level interface to compression and decompression routines in + the zstd library. + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/compression/zstd/__init__.py` + +-------------- + +This module provides classes and functions for compressing and decompressing +data using the Zstandard (or *zstd*) compression algorithm. The +`zstd manual `__ +describes Zstandard as "a fast lossless compression algorithm, targeting +real-time compression scenarios at zlib-level and better compression ratios." +Also included is a file interface that supports reading and writing the +contents of ``.zst`` files created by the :program:`zstd` utility, as well as +raw zstd compressed streams. + +The :mod:`!compression.zstd` module contains: + +* The :func:`.open` function and :class:`ZstdFile` class for reading and + writing compressed files. +* The :class:`ZstdCompressor` and :class:`ZstdDecompressor` classes for + incremental (de)compression. +* The :func:`compress` and :func:`decompress` functions for one-shot + (de)compression. +* The :func:`train_dict` and :func:`finalize_dict` functions and the + :class:`ZstdDict` class to train and manage Zstandard dictionaries. +* The :class:`CompressionParameter`, :class:`DecompressionParameter`, and + :class:`Strategy` classes for setting advanced (de)compression parameters. + +.. include:: ../includes/optional-module.rst + + +Exceptions +---------- + +.. exception:: ZstdError + + This exception is raised when an error occurs during compression or + decompression, or while initializing the (de)compressor state. + + +Reading and writing compressed files +------------------------------------ + +.. function:: open(file, /, mode='rb', *, level=None, options=None, \ + zstd_dict=None, encoding=None, errors=None, newline=None) + + Open a Zstandard-compressed file in binary or text mode, returning a + :term:`file object`. + + The *file* argument can be either a file name (given as a + :class:`str`, :class:`bytes` or :term:`path-like ` + object), in which case the named file is opened, or it can be an existing + file object to read from or write to. + + The mode argument can be either ``'rb'`` for reading (default), ``'wb'`` for + overwriting, ``'ab'`` for appending, or ``'xb'`` for exclusive creation. + These can equivalently be given as ``'r'``, ``'w'``, ``'a'``, and ``'x'`` + respectively. You may also open in text mode with ``'rt'``, ``'wt'``, + ``'at'``, and ``'xt'`` respectively. + + When reading, the *options* argument can be a dictionary providing advanced + decompression parameters; see :class:`DecompressionParameter` for detailed + information about supported + parameters. The *zstd_dict* argument is a :class:`ZstdDict` instance to be + used during decompression. When reading, if the *level* + argument is not None, a :exc:`!TypeError` will be raised. + + When writing, the *options* argument can be a dictionary + providing advanced decompression parameters; see + :class:`CompressionParameter` for detailed information about supported + parameters. The *level* argument is the compression level to use when + writing compressed data. Only one of *level* or *options* may be non-None. + The *zstd_dict* argument is a :class:`ZstdDict` instance to be used during + compression. + + In binary mode, this function is equivalent to the :class:`ZstdFile` + constructor: ``ZstdFile(file, mode, ...)``. In this case, the + *encoding*, *errors*, and *newline* parameters must not be provided. + + In text mode, a :class:`ZstdFile` object is created, and wrapped in an + :class:`io.TextIOWrapper` instance with the specified encoding, error + handling behavior, and line endings. + + +.. class:: ZstdFile(file, /, mode='rb', *, level=None, options=None, \ + zstd_dict=None) + + Open a Zstandard-compressed file in binary mode. + + A :class:`ZstdFile` can wrap an already-open :term:`file object`, or operate + directly on a named file. The *file* argument specifies either the file + object to wrap, or the name of the file to open (as a :class:`str`, + :class:`bytes` or :term:`path-like ` object). If + wrapping an existing file object, the wrapped file will not be closed when + the :class:`ZstdFile` is closed. + + The *mode* argument can be either ``'rb'`` for reading (default), ``'wb'`` + for overwriting, ``'xb'`` for exclusive creation, or ``'ab'`` for appending. + These can equivalently be given as ``'r'``, ``'w'``, ``'x'`` and ``'a'`` + respectively. + + If *file* is a file object (rather than an actual file name), a mode of + ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. + + When reading, the *options* argument can be a dictionary + providing advanced decompression parameters; see + :class:`DecompressionParameter` for detailed information about supported + parameters. The *zstd_dict* argument is a :class:`ZstdDict` instance to be + used during decompression. When reading, if the *level* + argument is not None, a :exc:`!TypeError` will be raised. + + When writing, the *options* argument can be a dictionary + providing advanced decompression parameters; see + :class:`CompressionParameter` for detailed information about supported + parameters. The *level* argument is the compression level to use when + writing compressed data. Only one of *level* or *options* may be passed. The + *zstd_dict* argument is a :class:`ZstdDict` instance to be used during + compression. + + :class:`!ZstdFile` supports all the members specified by + :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach` + and :meth:`~io.IOBase.truncate`. + Iteration and the :keyword:`with` statement are supported. + + The following method and attributes are also provided: + + .. method:: peek(size=-1) + + Return buffered data without advancing the file position. At least one + byte of data will be returned, unless EOF has been reached. The exact + number of bytes returned is unspecified (the *size* argument is ignored). + + .. note:: While calling :meth:`peek` does not change the file position of + the :class:`ZstdFile`, it may change the position of the underlying + file object (for example, if the :class:`ZstdFile` was constructed by + passing a file object for *file*). + + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. attribute:: name + + The name of the Zstandard file. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + +Compressing and decompressing data in memory +-------------------------------------------- + +.. function:: compress(data, level=None, options=None, zstd_dict=None) + + Compress *data* (a :term:`bytes-like object`), returning the compressed + data as a :class:`bytes` object. + + The *level* argument is an integer controlling the level of + compression. *level* is an alternative to setting + :attr:`CompressionParameter.compression_level` in *options*. Use + :meth:`~CompressionParameter.bounds` on + :attr:`~CompressionParameter.compression_level` to get the values that can + be passed for *level*. If advanced compression options are needed, the + *level* argument must be omitted and in the *options* dictionary the + :attr:`!CompressionParameter.compression_level` parameter should be set. + + The *options* argument is a Python dictionary containing advanced + compression parameters. The valid keys and values for compression parameters + are documented as part of the :class:`CompressionParameter` documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data to improve compression efficiency. The + function :func:`train_dict` can be used to generate a Zstandard dictionary. + + +.. function:: decompress(data, zstd_dict=None, options=None) + + Decompress *data* (a :term:`bytes-like object`), returning the uncompressed + data as a :class:`bytes` object. + + The *options* argument is a Python dictionary containing advanced + decompression parameters. The valid keys and values for compression + parameters are documented as part of the :class:`DecompressionParameter` + documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data used during compression. This must be + the same Zstandard dictionary used during compression. + + If *data* is the concatenation of multiple distinct compressed frames, + decompress all of these frames, and return the concatenation of the results. + + +.. class:: ZstdCompressor(level=None, options=None, zstd_dict=None) + + Create a compressor object, which can be used to compress data + incrementally. + + For a more convenient way of compressing a single chunk of data, see the + module-level function :func:`compress`. + + The *level* argument is an integer controlling the level of + compression. *level* is an alternative to setting + :attr:`CompressionParameter.compression_level` in *options*. Use + :meth:`~CompressionParameter.bounds` on + :attr:`~CompressionParameter.compression_level` to get the values that can + be passed for *level*. If advanced compression options are needed, the + *level* argument must be omitted and in the *options* dictionary the + :attr:`!CompressionParameter.compression_level` parameter should be set. + + The *options* argument is a Python dictionary containing advanced + compression parameters. The valid keys and values for compression parameters + are documented as part of the :class:`CompressionParameter` documentation. + + The *zstd_dict* argument is an optional instance of :class:`ZstdDict` + containing trained data to improve compression efficiency. The + function :func:`train_dict` can be used to generate a Zstandard dictionary. + + + .. method:: compress(data, mode=ZstdCompressor.CONTINUE) + + Compress *data* (a :term:`bytes-like object`), returning a :class:`bytes` + object with compressed data if possible, or otherwise an empty + :class:`!bytes` object. Some of *data* may be buffered internally, for + use in later calls to :meth:`!compress` and :meth:`~.flush`. The returned + data should be concatenated with the output of any previous calls to + :meth:`~.compress`. + + The *mode* argument is a :class:`ZstdCompressor` attribute, either + :attr:`~.CONTINUE`, :attr:`~.FLUSH_BLOCK`, + or :attr:`~.FLUSH_FRAME`. + + When all data has been provided to the compressor, call the + :meth:`~.flush` method to finish the compression process. If + :meth:`~.compress` is called with *mode* set to :attr:`~.FLUSH_FRAME`, + :meth:`~.flush` should not be called, as it would write out a new empty + frame. + + .. method:: flush(mode=ZstdCompressor.FLUSH_FRAME) + + Finish the compression process, returning a :class:`bytes` object + containing any data stored in the compressor's internal buffers. + + The *mode* argument is a :class:`ZstdCompressor` attribute, either + :attr:`~.FLUSH_BLOCK`, or :attr:`~.FLUSH_FRAME`. + + .. method:: set_pledged_input_size(size) + + Specify the amount of uncompressed data *size* that will be provided for + the next frame. *size* will be written into the frame header of the next + frame unless :attr:`CompressionParameter.content_size_flag` is ``False`` + or ``0``. A size of ``0`` means that the frame is empty. If *size* is + ``None``, the frame header will omit the frame size. Frames that include + the uncompressed data size require less memory to decompress, especially + at higher compression levels. + + If :attr:`last_mode` is not :attr:`FLUSH_FRAME`, a + :exc:`ValueError` is raised as the compressor is not at the start of + a frame. If the pledged size does not match the actual size of data + provided to :meth:`.compress`, future calls to :meth:`!compress` or + :meth:`flush` may raise :exc:`ZstdError` and the last chunk of data may + be lost. + + After :meth:`flush` or :meth:`.compress` are called with mode + :attr:`FLUSH_FRAME`, the next frame will not include the frame size into + the header unless :meth:`!set_pledged_input_size` is called again. + + .. attribute:: CONTINUE + + Collect more data for compression, which may or may not generate output + immediately. This mode optimizes the compression ratio by maximizing the + amount of data per block and frame. + + .. attribute:: FLUSH_BLOCK + + Complete and write a block to the data stream. The data returned so far + can be immediately decompressed. Past data can still be referenced in + future blocks generated by calls to :meth:`~.compress`, + improving compression. + + .. attribute:: FLUSH_FRAME + + Complete and write out a frame. Future data provided to + :meth:`~.compress` will be written into a new frame and + *cannot* reference past data. + + .. attribute:: last_mode + + The last mode passed to either :meth:`~.compress` or :meth:`~.flush`. + The value can be one of :attr:`~.CONTINUE`, :attr:`~.FLUSH_BLOCK`, or + :attr:`~.FLUSH_FRAME`. The initial value is :attr:`~.FLUSH_FRAME`, + signifying that the compressor is at the start of a new frame. + + +.. class:: ZstdDecompressor(zstd_dict=None, options=None) + + Create a decompressor object, which can be used to decompress data + incrementally. + + For a more convenient way of decompressing an entire compressed stream at + once, see the module-level function :func:`decompress`. + + The *options* argument is a Python dictionary containing advanced + decompression parameters. The valid keys and values for compression + parameters are documented as part of the :class:`DecompressionParameter` + documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data used during compression. This must be + the same Zstandard dictionary used during compression. + + .. note:: + This class does not transparently handle inputs containing multiple + compressed frames, unlike the :func:`decompress` function and + :class:`ZstdFile` class. To decompress a multi-frame input, you should + use :func:`decompress`, :class:`ZstdFile` if working with a + :term:`file object`, or multiple :class:`!ZstdDecompressor` instances. + + .. method:: decompress(data, max_length=-1) + + Decompress *data* (a :term:`bytes-like object`), returning + uncompressed data as bytes. Some of *data* may be buffered + internally, for use in later calls to :meth:`!decompress`. + The returned data should be concatenated with the output of any previous + calls to :meth:`!decompress`. + + If *max_length* is non-negative, the method returns at most *max_length* + bytes of decompressed data. If this limit is reached and further + output can be produced, the :attr:`~.needs_input` attribute will + be set to ``False``. In this case, the next call to + :meth:`~.decompress` may provide *data* as ``b''`` to obtain + more of the output. + + If all of the input data was decompressed and returned (either + because this was less than *max_length* bytes, or because + *max_length* was negative), the :attr:`~.needs_input` attribute + will be set to ``True``. + + Attempting to decompress data after the end of a frame will raise a + :exc:`ZstdError`. Any data found after the end of the frame is ignored + and saved in the :attr:`~.unused_data` attribute. + + .. attribute:: eof + + ``True`` if the end-of-stream marker has been reached. + + .. attribute:: unused_data + + Data found after the end of the compressed stream. + + Before the end of the stream is reached, this will be ``b''``. + + .. attribute:: needs_input + + ``False`` if the :meth:`.decompress` method can provide more + decompressed data before requiring new compressed input. + + +Zstandard dictionaries +---------------------- + + +.. function:: train_dict(samples, dict_size) + + Train a Zstandard dictionary, returning a :class:`ZstdDict` instance. + Zstandard dictionaries enable more efficient compression of smaller sizes + of data, which is traditionally difficult to compress due to less + repetition. If you are compressing multiple similar groups of data (such as + similar files), Zstandard dictionaries can improve compression ratios and + speed significantly. + + The *samples* argument (an iterable of :class:`bytes` objects), is the + population of samples used to train the Zstandard dictionary. + + The *dict_size* argument, an integer, is the maximum size (in bytes) the + Zstandard dictionary should be. The Zstandard documentation suggests an + absolute maximum of no more than 100 KB, but the maximum can often be smaller + depending on the data. Larger dictionaries generally slow down compression, + but improve compression ratios. Smaller dictionaries lead to faster + compression, but reduce the compression ratio. + + +.. function:: finalize_dict(zstd_dict, /, samples, dict_size, level) + + An advanced function for converting a "raw content" Zstandard dictionary into + a regular Zstandard dictionary. "Raw content" dictionaries are a sequence of + bytes that do not need to follow the structure of a normal Zstandard + dictionary. + + The *zstd_dict* argument is a :class:`ZstdDict` instance with + the :attr:`~ZstdDict.dict_content` containing the raw dictionary contents. + + The *samples* argument (an iterable of :class:`bytes` objects), contains + sample data for generating the Zstandard dictionary. + + The *dict_size* argument, an integer, is the maximum size (in bytes) the + Zstandard dictionary should be. See :func:`train_dict` for + suggestions on the maximum dictionary size. + + The *level* argument (an integer) is the compression level expected to be + passed to the compressors using this dictionary. The dictionary information + varies for each compression level, so tuning for the proper compression + level can make compression more efficient. + + +.. class:: ZstdDict(dict_content, /, *, is_raw=False) + + A wrapper around Zstandard dictionaries. Dictionaries can be used to improve + the compression of many small chunks of data. Use :func:`train_dict` if you + need to train a new dictionary from sample data. + + The *dict_content* argument (a :term:`bytes-like object`), is the already + trained dictionary information. + + The *is_raw* argument, a boolean, is an advanced parameter controlling the + meaning of *dict_content*. ``True`` means *dict_content* is a "raw content" + dictionary, without any format restrictions. ``False`` means *dict_content* + is an ordinary Zstandard dictionary, created from Zstandard functions, + for example, :func:`train_dict` or the external :program:`zstd` CLI. + + When passing a :class:`!ZstdDict` to a function, the + :attr:`!as_digested_dict` and :attr:`!as_undigested_dict` attributes can + control how the dictionary is loaded by passing them as the ``zstd_dict`` + argument, for example, ``compress(data, zstd_dict=zd.as_digested_dict)``. + Digesting a dictionary is a costly operation that occurs when loading a + Zstandard dictionary. When making multiple calls to compression or + decompression, passing a digested dictionary will reduce the overhead of + loading the dictionary. + + .. list-table:: Difference for compression + :widths: 10 14 10 + :header-rows: 1 + + * - + - Digested dictionary + - Undigested dictionary + * - Advanced parameters of the compressor which may be overridden by + the dictionary's parameters + - ``window_log``, ``hash_log``, ``chain_log``, ``search_log``, + ``min_match``, ``target_length``, ``strategy``, + ``enable_long_distance_matching``, ``ldm_hash_log``, + ``ldm_min_match``, ``ldm_bucket_size_log``, ``ldm_hash_rate_log``, + and some non-public parameters. + - None + * - :class:`!ZstdDict` internally caches the dictionary + - Yes. It's faster when loading a digested dictionary again with the + same compression level. + - No. If you wish to load an undigested dictionary multiple times, + consider reusing a compressor object. + + If passing a :class:`!ZstdDict` without any attribute, an undigested + dictionary is passed by default when compressing and a digested dictionary + is generated if necessary and passed by default when decompressing. + + .. attribute:: dict_content + + The content of the Zstandard dictionary, a ``bytes`` object. It's the + same as the *dict_content* argument in the ``__init__`` method. It can + be used with other programs, such as the ``zstd`` CLI program. + + .. attribute:: dict_id + + Identifier of the Zstandard dictionary, a non-negative int value. + + Non-zero means the dictionary is ordinary, created by Zstandard + functions and following the Zstandard format. + + ``0`` means a "raw content" dictionary, free of any format restriction, + used for advanced users. + + .. note:: + + The meaning of ``0`` for :attr:`!ZstdDict.dict_id` is different + from the ``dictionary_id`` attribute to the :func:`get_frame_info` + function. + + .. attribute:: as_digested_dict + + Load as a digested dictionary. + + .. attribute:: as_undigested_dict + + Load as an undigested dictionary. + + +Advanced parameter control +-------------------------- + +.. class:: CompressionParameter() + + An :class:`~enum.IntEnum` containing the advanced compression parameter + keys that can be used when compressing data. + + The :meth:`~.bounds` method can be used on any attribute to get the valid + values for that parameter. + + Parameters are optional; any omitted parameter will have it's value selected + automatically. + + Example getting the lower and upper bound of :attr:`~.compression_level`:: + + lower, upper = CompressionParameter.compression_level.bounds() + + Example setting the :attr:`~.window_log` to the maximum size:: + + _lower, upper = CompressionParameter.window_log.bounds() + options = {CompressionParameter.window_log: upper} + compress(b'venezuelan beaver cheese', options=options) + + .. method:: bounds() + + Return the tuple of int bounds, ``(lower, upper)``, of a compression + parameter. This method should be called on the attribute you wish to + retrieve the bounds of. For example, to get the valid values for + :attr:`~.compression_level`, one may check the result of + ``CompressionParameter.compression_level.bounds()``. + + Both the lower and upper bounds are inclusive. + + .. attribute:: compression_level + + A high-level means of setting other compression parameters that affect + the speed and ratio of compressing data. + + Regular compression levels are greater than ``0``. Values greater than + ``20`` are considered "ultra" compression and require more memory than + other levels. Negative values can be used to trade off faster compression + for worse compression ratios. + + Setting the level to zero uses :attr:`COMPRESSION_LEVEL_DEFAULT`. + + .. attribute:: window_log + + Maximum allowed back-reference distance the compressor can use when + compressing data, expressed as power of two, ``1 << window_log`` bytes. + This parameter greatly influences the memory usage of compression. Higher + values require more memory but gain better compression values. + + A value of zero causes the value to be selected automatically. + + .. attribute:: hash_log + + Size of the initial probe table, as a power of two. The resulting memory + usage is ``1 << (hash_log+2)`` bytes. Larger tables improve compression + ratio of strategies <= :attr:`~Strategy.dfast`, and improve compression + speed of strategies > :attr:`~Strategy.dfast`. + + A value of zero causes the value to be selected automatically. + + .. attribute:: chain_log + + Size of the multi-probe search table, as a power of two. The resulting + memory usage is ``1 << (chain_log+2)`` bytes. Larger tables result in + better and slower compression. This parameter has no effect for the + :attr:`~Strategy.fast` strategy. It's still useful when using + :attr:`~Strategy.dfast` strategy, in which case it defines a secondary + probe table. + + A value of zero causes the value to be selected automatically. + + .. attribute:: search_log + + Number of search attempts, as a power of two. More attempts result in + better and slower compression. This parameter is useless for + :attr:`~Strategy.fast` and :attr:`~Strategy.dfast` strategies. + + A value of zero causes the value to be selected automatically. + + .. attribute:: min_match + + Minimum size of searched matches. Larger values increase compression and + decompression speed, but decrease ratio. Note that Zstandard can still + find matches of smaller size, it just tweaks its search algorithm to look + for this size and larger. For all strategies < :attr:`~Strategy.btopt`, + the effective minimum is ``4``; for all strategies + > :attr:`~Strategy.fast`, the effective maximum is ``6``. + + A value of zero causes the value to be selected automatically. + + .. attribute:: target_length + + The impact of this field depends on the selected :class:`Strategy`. + + For strategies :attr:`~Strategy.btopt`, :attr:`~Strategy.btultra` and + :attr:`~Strategy.btultra2`, the value is the length of a match + considered "good enough" to stop searching. Larger values make + compression ratios better, but compresses slower. + + For strategy :attr:`~Strategy.fast`, it is the distance between match + sampling. Larger values make compression faster, but with a worse + compression ratio. + + A value of zero causes the value to be selected automatically. + + .. attribute:: strategy + + The higher the value of selected strategy, the more complex the + compression technique used by zstd, resulting in higher compression + ratios but slower compression. + + .. seealso:: :class:`Strategy` + + .. attribute:: enable_long_distance_matching + + Long distance matching can be used to improve compression for large + inputs by finding large matches at greater distances. It increases memory + usage and window size. + + ``True`` or ``1`` enable long distance matching while ``False`` or ``0`` + disable it. + + Enabling this parameter increases default + :attr:`~CompressionParameter.window_log` to 128 MiB except when expressly + set to a different value. This setting is enabled by default if + :attr:`!window_log` >= 128 MiB and the compression + strategy >= :attr:`~Strategy.btopt` (compression level 16+). + + .. attribute:: ldm_hash_log + + Size of the table for long distance matching, as a power of two. Larger + values increase memory usage and compression ratio, but decrease + compression speed. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_min_match + + Minimum match size for long distance matcher. Larger or too small values + can often decrease the compression ratio. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_bucket_size_log + + Log size of each bucket in the long distance matcher hash table for + collision resolution. Larger values improve collision resolution but + decrease compression speed. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_hash_rate_log + + Frequency of inserting/looking up entries into the long distance matcher + hash table. Larger values improve compression speed. Deviating far from + the default value will likely result in a compression ratio decrease. + + A value of zero causes the value to be selected automatically. + + .. attribute:: content_size_flag + + Write the size of the data to be compressed into the Zstandard frame + header when known prior to compressing. + + This flag only takes effect under the following scenarios: + + * Calling :func:`compress` for one-shot compression + * Providing all of the data to be compressed in the frame in a single + :meth:`ZstdCompressor.compress` call, with the + :attr:`ZstdCompressor.FLUSH_FRAME` mode. + * Calling :meth:`ZstdCompressor.set_pledged_input_size` with the exact + amount of data that will be provided to the compressor prior to any + calls to :meth:`ZstdCompressor.compress` for the current frame. + :meth:`!ZstdCompressor.set_pledged_input_size` must be called for each + new frame. + + All other compression calls may not write the size information into the + frame header. + + ``True`` or ``1`` enable the content size flag while ``False`` or ``0`` + disable it. + + .. attribute:: checksum_flag + + A four-byte checksum using XXHash64 of the uncompressed content is + written at the end of each frame. Zstandard's decompression code verifies + the checksum. If there is a mismatch a :class:`ZstdError` exception is + raised. + + ``True`` or ``1`` enable checksum generation while ``False`` or ``0`` + disable it. + + .. attribute:: dict_id_flag + + When compressing with a :class:`ZstdDict`, the dictionary's ID is written + into the frame header. + + ``True`` or ``1`` enable storing the dictionary ID while ``False`` or + ``0`` disable it. + + .. attribute:: nb_workers + + Select how many threads will be spawned to compress in parallel. When + :attr:`!nb_workers` > 0, enables multi-threaded compression, a value of + ``1`` means "one-thread multi-threaded mode". More workers improve speed, + but also increase memory usage and slightly reduce compression ratio. + + A value of zero disables multi-threading. + + .. attribute:: job_size + + Size of a compression job, in bytes. This value is enforced only when + :attr:`~CompressionParameter.nb_workers` >= 1. Each compression job is + completed in parallel, so this value can indirectly impact the number of + active threads. + + A value of zero causes the value to be selected automatically. + + .. attribute:: overlap_log + + Sets how much data is reloaded from previous jobs (threads) for new jobs + to be used by the look behind window during compression. This value is + only used when :attr:`~CompressionParameter.nb_workers` >= 1. Acceptable + values vary from 0 to 9. + + * 0 means dynamically set the overlap amount + * 1 means no overlap + * 9 means use a full window size from the previous job + + Each increment halves/doubles the overlap size. "8" means an overlap of + ``window_size/2``, "7" means an overlap of ``window_size/4``, etc. + +.. class:: DecompressionParameter() + + An :class:`~enum.IntEnum` containing the advanced decompression parameter + keys that can be used when decompressing data. Parameters are optional; any + omitted parameter will have it's value selected automatically. + + The :meth:`~.bounds` method can be used on any attribute to get the valid + values for that parameter. + + Example setting the :attr:`~.window_log_max` to the maximum size:: + + data = compress(b'Some very long buffer of bytes...') + + _lower, upper = DecompressionParameter.window_log_max.bounds() + + options = {DecompressionParameter.window_log_max: upper} + decompress(data, options=options) + + .. method:: bounds() + + Return the tuple of int bounds, ``(lower, upper)``, of a decompression + parameter. This method should be called on the attribute you wish to + retrieve the bounds of. + + Both the lower and upper bounds are inclusive. + + .. attribute:: window_log_max + + The base-two logarithm of the maximum size of the window used during + decompression. This can be useful to limit the amount of memory used when + decompressing data. A larger maximum window size leads to faster + decompression. + + A value of zero causes the value to be selected automatically. + + +.. class:: Strategy() + + An :class:`~enum.IntEnum` containing strategies for compression. + Higher-numbered strategies correspond to more complex and slower + compression. + + .. note:: + + The values of attributes of :class:`!Strategy` are not necessarily stable + across zstd versions. Only the ordering of the attributes may be relied + upon. The attributes are listed below in order. + + The following strategies are available: + + .. attribute:: fast + + .. attribute:: dfast + + .. attribute:: greedy + + .. attribute:: lazy + + .. attribute:: lazy2 + + .. attribute:: btlazy2 + + .. attribute:: btopt + + .. attribute:: btultra + + .. attribute:: btultra2 + + +Miscellaneous +------------- + +.. function:: get_frame_info(frame_buffer) + + Retrieve a :class:`FrameInfo` object containing metadata about a Zstandard + frame. Frames contain metadata related to the compressed data they hold. + + +.. class:: FrameInfo + + Metadata related to a Zstandard frame. + + .. attribute:: decompressed_size + + The size of the decompressed contents of the frame. + + .. attribute:: dictionary_id + + An integer representing the Zstandard dictionary ID needed for + decompressing the frame. ``0`` means the dictionary ID was not + recorded in the frame header. This may mean that a Zstandard dictionary + is not needed, or that the ID of a required dictionary was not recorded. + + +.. attribute:: COMPRESSION_LEVEL_DEFAULT + + The default compression level for Zstandard: ``3``. + + +.. attribute:: zstd_version_info + + Version number of the runtime zstd library as a tuple of integers + (major, minor, release). + + +Examples +-------- + +Reading in a compressed file: + +.. code-block:: python + + from compression import zstd + + with zstd.open("file.zst") as f: + file_content = f.read() + +Creating a compressed file: + +.. code-block:: python + + from compression import zstd + + data = b"Insert Data Here" + with zstd.open("file.zst", "w") as f: + f.write(data) + +Compressing data in memory: + +.. code-block:: python + + from compression import zstd + + data_in = b"Insert Data Here" + data_out = zstd.compress(data_in) + +Incremental compression: + +.. code-block:: python + + from compression import zstd + + comp = zstd.ZstdCompressor() + out1 = comp.compress(b"Some data\n") + out2 = comp.compress(b"Another piece of data\n") + out3 = comp.compress(b"Even more data\n") + out4 = comp.flush() + # Concatenate all the partial results: + result = b"".join([out1, out2, out3, out4]) + +Writing compressed data to an already-open file: + +.. code-block:: python + + from compression import zstd + + with open("myfile", "wb") as f: + f.write(b"This data will not be compressed\n") + with zstd.open(f, "w") as zstf: + zstf.write(b"This *will* be compressed\n") + f.write(b"Not compressed\n") + +Creating a compressed file using compression parameters: + +.. code-block:: python + + from compression import zstd + + options = { + zstd.CompressionParameter.checksum_flag: 1 + } + with zstd.open("file.zst", "w", options=options) as f: + f.write(b"Mind if I squeeze in?") diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 5be1a1106b0..18f9443cbfe 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -18,6 +18,7 @@ multitasking). Here's an overview: multiprocessing.shared_memory.rst concurrent.rst concurrent.futures.rst + concurrent.interpreters.rst subprocess.rst sched.rst queue.rst diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 7efae9e628b..c2e2f7f820f 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -6,8 +6,9 @@ .. versionadded:: 3.2 -**Source code:** :source:`Lib/concurrent/futures/thread.py` -and :source:`Lib/concurrent/futures/process.py` +**Source code:** :source:`Lib/concurrent/futures/thread.py`, +:source:`Lib/concurrent/futures/process.py`, +and :source:`Lib/concurrent/futures/interpreter.py` -------------- @@ -100,10 +101,10 @@ Executor Objects executor has started running will be completed prior to this method returning. The remaining futures are cancelled. - You can avoid having to call this method explicitly if you use the - :keyword:`with` statement, which will shutdown the :class:`Executor` - (waiting as if :meth:`Executor.shutdown` were called with *wait* set to - ``True``):: + You can avoid having to call this method explicitly if you use the executor + as a :term:`context manager` via the :keyword:`with` statement, which + will shutdown the :class:`Executor` (waiting as if :meth:`Executor.shutdown` + were called with *wait* set to ``True``):: import shutil with ThreadPoolExecutor(max_workers=4) as e: @@ -238,6 +239,8 @@ ThreadPoolExecutor Example InterpreterPoolExecutor ----------------------- +.. versionadded:: 3.14 + The :class:`InterpreterPoolExecutor` class uses a pool of interpreters to execute calls asynchronously. It is a :class:`ThreadPoolExecutor` subclass, which means each worker is running in its own thread. @@ -264,7 +267,7 @@ Each worker's interpreter is isolated from all the other interpreters. "Isolated" means each interpreter has its own runtime state and operates completely independently. For example, if you redirect :data:`sys.stdout` in one interpreter, it will not be automatically -redirected any other interpreter. If you import a module in one +redirected to any other interpreter. If you import a module in one interpreter, it is not automatically imported in any other. You would need to import the module separately in interpreter where you need it. In fact, each module imported in an interpreter is @@ -286,7 +289,7 @@ efficient alternative is to serialize with :mod:`pickle` and then send the bytes over a shared :mod:`socket ` or :func:`pipe `. -.. class:: InterpreterPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=(), shared=None) +.. class:: InterpreterPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) A :class:`ThreadPoolExecutor` subclass that executes calls asynchronously using a pool of at most *max_workers* threads. Each thread runs @@ -303,21 +306,10 @@ the bytes over a shared :mod:`socket ` or and *initargs* using :mod:`pickle` when sending them to the worker's interpreter. - .. note:: - Functions defined in the ``__main__`` module cannot be pickled - and thus cannot be used. - .. note:: The executor may replace uncaught exceptions from *initializer* with :class:`~concurrent.futures.interpreter.ExecutionFailed`. - The optional *shared* argument is a :class:`dict` of objects that all - interpreters in the pool share. The *shared* items are added to each - interpreter's ``__main__`` module. Not all objects are shareable. - Shareable objects include the builtin singletons, :class:`str` - and :class:`bytes`, and :class:`memoryview`. See :pep:`734` - for more info. - Other caveats from parent :class:`ThreadPoolExecutor` apply here. :meth:`~Executor.submit` and :meth:`~Executor.map` work like normal, @@ -325,10 +317,6 @@ except the worker serializes the callable and arguments using :mod:`pickle` when sending them to its interpreter. The worker likewise serializes the return value when sending it back. -.. note:: - Functions defined in the ``__main__`` module cannot be pickled - and thus cannot be used. - When a worker's current task raises an uncaught exception, the worker always tries to preserve the exception as-is. If that is successful then it also sets the ``__cause__`` to a corresponding @@ -356,6 +344,11 @@ that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. Calling :class:`Executor` or :class:`Future` methods from a callable submitted to a :class:`ProcessPoolExecutor` will result in deadlock. +Note that the restrictions on functions and arguments needing to picklable as +per :class:`multiprocessing.Process` apply when using :meth:`~Executor.submit` +and :meth:`~Executor.map` on a :class:`ProcessPoolExecutor`. A function defined +in a REPL or a lambda should not be expected to work. + .. class:: ProcessPoolExecutor(max_workers=None, mp_context=None, initializer=None, initargs=(), max_tasks_per_child=None) An :class:`Executor` subclass that executes calls asynchronously using a pool diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst new file mode 100644 index 00000000000..55036090e8d --- /dev/null +++ b/Doc/library/concurrent.interpreters.rst @@ -0,0 +1,387 @@ +:mod:`!concurrent.interpreters` --- Multiple interpreters in the same process +============================================================================= + +.. module:: concurrent.interpreters + :synopsis: Multiple interpreters in the same process + +.. moduleauthor:: Eric Snow +.. sectionauthor:: Eric Snow + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/concurrent/interpreters` + +-------------- + +The :mod:`!concurrent.interpreters` module constructs higher-level +interfaces on top of the lower level :mod:`!_interpreters` module. + +The module is primarily meant to provide a basic API for managing +interpreters (AKA "subinterpreters") and running things in them. +Running mostly involves switching to an interpreter (in the current +thread) and calling a function in that execution context. + +For concurrency, interpreters themselves (and this module) don't +provide much more than isolation, which on its own isn't useful. +Actual concurrency is available separately through +:mod:`threads ` See `below `_ + +.. seealso:: + + :class:`~concurrent.futures.InterpreterPoolExecutor` + Combines threads with interpreters in a familiar interface. + + .. XXX Add references to the upcoming HOWTO docs in the seealso block. + + :ref:`isolating-extensions-howto` + How to update an extension module to support multiple interpreters. + + :pep:`554` + + :pep:`734` + + :pep:`684` + +.. XXX Why do we disallow multiple interpreters on WASM? + +.. include:: ../includes/wasm-notavail.rst + + +Key details +----------- + +Before we dive in further, there are a small number of details +to keep in mind about using multiple interpreters: + +* `isolated `_, by default +* no implicit threads +* not all PyPI packages support use in multiple interpreters yet + +.. XXX Are there other relevant details to list? + + +.. _interpreters-intro: + +Introduction +------------ + +An "interpreter" is effectively the execution context of the Python +runtime. It contains all of the state the runtime needs to execute +a program. This includes things like the import state and builtins. +(Each thread, even if there's only the main thread, has some extra +runtime state, in addition to the current interpreter, related to +the current exception and the bytecode eval loop.) + +The concept and functionality of the interpreter have been a part of +Python since version 2.2, but the feature was only available through +the C-API and not well known, and the `isolation `_ +was relatively incomplete until version 3.12. + +.. _interp-isolation: + +Multiple Interpreters and Isolation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A Python implementation may support using multiple interpreters in the +same process. CPython has this support. Each interpreter is +effectively isolated from the others (with a limited number of +carefully managed process-global exceptions to the rule). + +That isolation is primarily useful as a strong separation between +distinct logical components of a program, where you want to have +careful control of how those components interact. + +.. note:: + + Interpreters in the same process can technically never be strictly + isolated from one another since there are few restrictions on memory + access within the same process. The Python runtime makes a best + effort at isolation but extension modules may easily violate that. + Therefore, do not use multiple interpreters in security-sensitive + situations, where they shouldn't have access to each other's data. + +Running in an Interpreter +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Running in a different interpreter involves switching to it in the +current thread and then calling some function. The runtime will +execute the function using the current interpreter's state. The +:mod:`!concurrent.interpreters` module provides a basic API for +creating and managing interpreters, as well as the switch-and-call +operation. + +No other threads are automatically started for the operation. +There is `a helper `_ for that though. +There is another dedicated helper for calling the builtin +:func:`exec` in an interpreter. + +When :func:`exec` (or :func:`eval`) are called in an interpreter, +they run using the interpreter's :mod:`!__main__` module as the +"globals" namespace. The same is true for functions that aren't +associated with any module. This is the same as how scripts invoked +from the command-line run in the :mod:`!__main__` module. + + +.. _interp-concurrency: + +Concurrency and Parallelism +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As noted earlier, interpreters do not provide any concurrency +on their own. They strictly represent the isolated execution +context the runtime will use *in the current thread*. That isolation +makes them similar to processes, but they still enjoy in-process +efficiency, like threads. + +All that said, interpreters do naturally support certain flavors of +concurrency. +There's a powerful side effect of that isolation. It enables a +different approach to concurrency than you can take with async or +threads. It's a similar concurrency model to CSP or the actor model, +a model which is relatively easy to reason about. + +You can take advantage of that concurrency model in a single thread, +switching back and forth between interpreters, Stackless-style. +However, this model is more useful when you combine interpreters +with multiple threads. This mostly involves starting a new thread, +where you switch to another interpreter and run what you want there. + +Each actual thread in Python, even if you're only running in the main +thread, has its own *current* execution context. Multiple threads can +use the same interpreter or different ones. + +At a high level, you can think of the combination of threads and +interpreters as threads with opt-in sharing. + +As a significant bonus, interpreters are sufficiently isolated that +they do not share the :term:`GIL`, which means combining threads with +multiple interpreters enables full multi-core parallelism. +(This has been the case since Python 3.12.) + +Communication Between Interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In practice, multiple interpreters are useful only if we have a way +to communicate between them. This usually involves some form of +message passing, but can even mean sharing data in some carefully +managed way. + +With this in mind, the :mod:`!concurrent.interpreters` module provides +a :class:`queue.Queue` implementation, available through +:func:`create_queue`. + +.. _interp-object-sharing: + +"Sharing" Objects +^^^^^^^^^^^^^^^^^ + +Any data actually shared between interpreters loses the thread-safety +provided by the :term:`GIL`. There are various options for dealing with +this in extension modules. However, from Python code the lack of +thread-safety means objects can't actually be shared, with a few +exceptions. Instead, a copy must be created, which means mutable +objects won't stay in sync. + +By default, most objects are copied with :mod:`pickle` when they are +passed to another interpreter. Nearly all of the immutable builtin +objects are either directly shared or copied efficiently. For example: + +* :const:`None` +* :class:`bool` (:const:`True` and :const:`False`) +* :class:`bytes` +* :class:`str` +* :class:`int` +* :class:`float` +* :class:`tuple` (of similarly supported objects) + +There is a small number of Python types that actually share mutable +data between interpreters: + +* :class:`memoryview` +* :class:`Queue` + + +Reference +--------- + +This module defines the following functions: + +.. function:: list_all() + + Return a :class:`list` of :class:`Interpreter` objects, + one for each existing interpreter. + +.. function:: get_current() + + Return an :class:`Interpreter` object for the currently running + interpreter. + +.. function:: get_main() + + Return an :class:`Interpreter` object for the main interpreter. + This is the interpreter the runtime created to run the :term:`REPL` + or the script given at the command-line. It is usually the only one. + +.. function:: create() + + Initialize a new (idle) Python interpreter + and return a :class:`Interpreter` object for it. + +.. function:: create_queue() + + Initialize a new cross-interpreter queue and return a :class:`Queue` + object for it. + + +Interpreter objects +^^^^^^^^^^^^^^^^^^^ + +.. class:: Interpreter(id) + + A single interpreter in the current process. + + Generally, :class:`Interpreter` shouldn't be called directly. + Instead, use :func:`create` or one of the other module functions. + + .. attribute:: id + + (read-only) + + The underlying interpreter's ID. + + .. attribute:: whence + + (read-only) + + A string describing where the interpreter came from. + + .. method:: is_running() + + Return ``True`` if the interpreter is currently executing code + in its :mod:`!__main__` module and ``False`` otherwise. + + .. method:: close() + + Finalize and destroy the interpreter. + + .. method:: prepare_main(ns=None, **kwargs) + + Bind objects in the interpreter's :mod:`!__main__` module. + + Some objects are actually shared and some are copied efficiently, + but most are copied via :mod:`pickle`. See :ref:`interp-object-sharing`. + + .. method:: exec(code, /, dedent=True) + + Run the given source code in the interpreter (in the current thread). + + .. method:: call(callable, /, *args, **kwargs) + + Return the result of calling running the given function in the + interpreter (in the current thread). + + .. _interp-call-in-thread: + + .. method:: call_in_thread(callable, /, *args, **kwargs) + + Run the given function in the interpreter (in a new thread). + +Exceptions +^^^^^^^^^^ + +.. exception:: InterpreterError + + This exception, a subclass of :exc:`Exception`, is raised when + an interpreter-related error happens. + +.. exception:: InterpreterNotFoundError + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the targeted interpreter no longer exists. + +.. exception:: ExecutionFailed + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the running code raised an uncaught exception. + + .. attribute:: excinfo + + A basic snapshot of the exception raised in the other interpreter. + +.. XXX Document the excinfoattrs? + +.. exception:: NotShareableError + + This exception, a subclass of :exc:`TypeError`, is raised when + an object cannot be sent to another interpreter. + + +Communicating Between Interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: Queue(id) + + A wrapper around a low-level, cross-interpreter queue, which + implements the :class:`queue.Queue` interface. The underlying queue + can only be created through :func:`create_queue`. + + Some objects are actually shared and some are copied efficiently, + but most are copied via :mod:`pickle`. See :ref:`interp-object-sharing`. + + .. attribute:: id + + (read-only) + + The queue's ID. + + +.. exception:: QueueEmptyError + + This exception, a subclass of :exc:`queue.Empty`, is raised from + :meth:`!Queue.get` and :meth:`!Queue.get_nowait` when the queue + is empty. + +.. exception:: QueueFullError + + This exception, a subclass of :exc:`queue.Full`, is raised from + :meth:`!Queue.put` and :meth:`!Queue.put_nowait` when the queue + is full. + + +Basic usage +----------- + +Creating an interpreter and running code in it:: + + from concurrent import interpreters + + interp = interpreters.create() + + # Run in the current OS thread. + + interp.exec('print("spam!")') + + interp.exec("""if True: + print('spam!') + """) + + from textwrap import dedent + interp.exec(dedent(""" + print('spam!') + """)) + + def run(arg): + return arg + + res = interp.call(run, 'spam!') + print(res) + + def run(): + print('spam!') + + interp.call(run) + + # Run in new OS thread. + + t = interp.call_in_thread(run) + t.join() diff --git a/Doc/library/concurrent.rst b/Doc/library/concurrent.rst index 8caea78bbb5..748c72c733b 100644 --- a/Doc/library/concurrent.rst +++ b/Doc/library/concurrent.rst @@ -1,6 +1,7 @@ The :mod:`!concurrent` package ============================== -Currently, there is only one module in this package: +This package contains the following modules: * :mod:`concurrent.futures` -- Launching parallel tasks +* :mod:`concurrent.interpreters` -- Multiple interpreters in the same process diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index c0ac4ea8412..d058ba206c6 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -65,8 +65,9 @@ A small number of constants live in the built-in namespace. They are: .. index:: single: ...; ellipsis literal .. data:: Ellipsis - The same as the ellipsis literal "``...``". Special value used mostly in conjunction - with extended slicing syntax for user-defined container data types. + The same as the ellipsis literal "``...``", an object frequently used to + indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but + assignment to ``...`` raises a :exc:`SyntaxError`. ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. @@ -97,15 +98,17 @@ should not be used in programs. exit(code=None) Objects that when printed, print a message like "Use quit() or Ctrl-D - (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the + (i.e. EOF) to exit", and when accessed directly in the interactive + interpreter or called as functions, raise :exc:`SystemExit` with the specified exit code. .. data:: help :noindex: Object that when printed, prints the message "Type help() for interactive - help, or help(object) for help about object.", and when called, - acts as described :func:`elsewhere `. + help, or help(object) for help about object.", and when accessed directly + in the interactive interpreter, invokes the built-in help system + (see :func:`help`). .. data:: copyright credits diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 176be4ff333..d0fa645093a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -327,10 +327,10 @@ Functions and classes provided: .. function:: redirect_stdout(new_target) Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. + another :term:`file object`. This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. + is hardwired to :data:`sys.stdout`. For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an @@ -366,8 +366,8 @@ Functions and classes provided: .. function:: redirect_stderr(new_target) - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. + Similar to :func:`~contextlib.redirect_stdout` but redirecting the global + :data:`sys.stderr` to another :term:`file object`. This context manager is :ref:`reentrant `. diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 57580ce026e..b218468a084 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -313,7 +313,7 @@ client:: addr = writer.transport.get_extra_info('socket').getpeername() client_addr_var.set(addr) - # In any code that we call is now possible to get + # In any code that we call, it is now possible to get the # client's address by calling 'client_addr_var.get()'. while True: diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 95b41f988a0..210ad718800 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -122,6 +122,8 @@ and only supports named tuples created by :func:`~collections.namedtuple`, This method should create a new object of the same type, replacing fields with values from *changes*. + .. versionadded:: 3.13 + .. seealso:: diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 9ff37196ccf..647cb4925f3 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -13,7 +13,7 @@ being deprecated in Python 3.11. The removal was decided in :pep:`594`. Applications can use the :mod:`hashlib` module from the standard library. Other possible replacements are third-party libraries from PyPI: -:pypi:`legacycrypt`, :pypi:`bcrypt`, :pypi:`argon2-cffi`, or :pypi:`passlib`. +:pypi:`legacycrypt`, :pypi:`bcrypt`, or :pypi:`argon2-cffi`. These are not supported or maintained by the Python core team. The last version of Python that provided the :mod:`!crypt` module was diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 533cdf13974..4a033d823e6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -53,7 +53,7 @@ The :mod:`csv` module defines the following functions: .. index:: single: universal newlines; csv.reader function -.. function:: reader(csvfile, dialect='excel', **fmtparams) +.. function:: reader(csvfile, /, dialect='excel', **fmtparams) Return a :ref:`reader object ` that will process lines from the given *csvfile*. A csvfile must be an iterable of @@ -70,7 +70,7 @@ The :mod:`csv` module defines the following functions: section :ref:`csv-fmt-params`. Each row read from the csv file is returned as a list of strings. No - automatic data type conversion is performed unless the ``QUOTE_NONNUMERIC`` format + automatic data type conversion is performed unless the :data:`QUOTE_NONNUMERIC` format option is specified (in which case unquoted fields are transformed into floats). A short usage example:: @@ -84,7 +84,7 @@ The :mod:`csv` module defines the following functions: Spam, Lovely Spam, Wonderful Spam -.. function:: writer(csvfile, dialect='excel', **fmtparams) +.. function:: writer(csvfile, /, dialect='excel', **fmtparams) Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a @@ -113,7 +113,7 @@ The :mod:`csv` module defines the following functions: spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) -.. function:: register_dialect(name[, dialect[, **fmtparams]]) +.. function:: register_dialect(name, /, dialect='excel', **fmtparams) Associate *dialect* with *name*. *name* must be a string. The dialect can be specified either by passing a sub-class of :class:`Dialect`, or @@ -139,7 +139,8 @@ The :mod:`csv` module defines the following functions: Return the names of all registered dialects. -.. function:: field_size_limit([new_limit]) +.. function:: field_size_limit() + field_size_limit(new_limit) Returns the current maximum field size allowed by the parser. If *new_limit* is given, this becomes the new limit. @@ -294,8 +295,8 @@ The :mod:`csv` module defines the following classes: - the second through n-th rows contain strings where at least one value's length differs from that of the putative header of that column. - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. + Twenty-one rows after the header are sampled; if more than half of the + columns + rows meet the criteria, :const:`True` is returned. .. note:: @@ -323,23 +324,32 @@ The :mod:`csv` module defines the following constants: .. data:: QUOTE_MINIMAL Instructs :class:`writer` objects to only quote those fields which contain - special characters such as *delimiter*, *quotechar* or any of the characters in - *lineterminator*. + special characters such as *delimiter*, *quotechar*, ``'\r'``, ``'\n'`` + or any of the characters in *lineterminator*. .. data:: QUOTE_NONNUMERIC Instructs :class:`writer` objects to quote all non-numeric fields. - Instructs :class:`reader` objects to convert all non-quoted fields to type *float*. + Instructs :class:`reader` objects to convert all non-quoted fields to type :class:`float`. + .. note:: + Some numeric types, such as :class:`bool`, :class:`~fractions.Fraction`, + or :class:`~enum.IntEnum`, have a string representation that cannot be + converted to :class:`float`. + They cannot be read in the :data:`QUOTE_NONNUMERIC` and + :data:`QUOTE_STRINGS` modes. .. data:: QUOTE_NONE - Instructs :class:`writer` objects to never quote fields. When the current - *delimiter* occurs in output data it is preceded by the current *escapechar* - character. If *escapechar* is not set, the writer will raise :exc:`Error` if + Instructs :class:`writer` objects to never quote fields. + When the current *delimiter*, *quotechar*, *escapechar*, ``'\r'``, ``'\n'`` + or any of the characters in *lineterminator* occurs in output data + it is preceded by the current *escapechar* character. + If *escapechar* is not set, the writer will raise :exc:`Error` if any characters that require escaping are encountered. + Set *quotechar* to ``None`` to prevent its escaping. Instructs :class:`reader` objects to perform no special processing of quote characters. @@ -408,9 +418,16 @@ Dialects support the following attributes: .. attribute:: Dialect.escapechar - A one-character string used by the writer to escape the *delimiter* if *quoting* - is set to :const:`QUOTE_NONE` and the *quotechar* if *doublequote* is - :const:`False`. On reading, the *escapechar* removes any special meaning from + A one-character string used by the writer to escape characters that + require escaping: + + * the *delimiter*, the *quotechar*, ``'\r'``, ``'\n'`` and any of the + characters in *lineterminator* are escaped if *quoting* is set to + :const:`QUOTE_NONE`; + * the *quotechar* is escaped if *doublequote* is :const:`False`; + * the *escapechar* itself. + + On reading, the *escapechar* removes any special meaning from the following character. It defaults to :const:`None`, which disables escaping. .. versionchanged:: 3.11 @@ -430,9 +447,12 @@ Dialects support the following attributes: .. attribute:: Dialect.quotechar - A one-character string used to quote fields containing special characters, such - as the *delimiter* or *quotechar*, or which contain new-line characters. It - defaults to ``'"'``. + A one-character string used to quote fields containing special characters, + such as the *delimiter* or the *quotechar*, or which contain new-line + characters (``'\r'``, ``'\n'`` or any of the characters in *lineterminator*). + It defaults to ``'"'``. + Can be set to ``None`` to prevent escaping ``'"'`` if *quoting* is set + to :const:`QUOTE_NONE`. .. versionchanged:: 3.11 An empty *quotechar* is not allowed. @@ -441,13 +461,15 @@ Dialects support the following attributes: Controls when quotes should be generated by the writer and recognised by the reader. It can take on any of the :ref:`QUOTE_\* constants ` - and defaults to :const:`QUOTE_MINIMAL`. + and defaults to :const:`QUOTE_MINIMAL` if *quotechar* is not ``None``, + and :const:`QUOTE_NONE` otherwise. .. attribute:: Dialect.skipinitialspace When :const:`True`, spaces immediately following the *delimiter* are ignored. - The default is :const:`False`. + The default is :const:`False`. When combining ``delimiter=' '`` with + ``skipinitialspace=True``, unquoted empty fields are not allowed. .. attribute:: Dialect.strict @@ -506,7 +528,7 @@ out surrounded by parens. This may cause some problems for other programs which read CSV files (assuming they support complex numbers at all). -.. method:: csvwriter.writerow(row) +.. method:: csvwriter.writerow(row, /) Write the *row* parameter to the writer's file object, formatted according to the current :class:`Dialect`. Return the return value of the call to the @@ -515,7 +537,7 @@ read CSV files (assuming they support complex numbers at all). .. versionchanged:: 3.5 Added support of arbitrary iterables. -.. method:: csvwriter.writerows(rows) +.. method:: csvwriter.writerows(rows, /) Write all elements in *rows* (an iterable of *row* objects as described above) to the writer's file object, formatted according to the current @@ -603,7 +625,7 @@ A slightly more advanced use of the reader --- catching and reporting errors:: for row in reader: print(row) except csv.Error as e: - sys.exit('file {}, line {}: {}'.format(filename, reader.line_num, e)) + sys.exit(f'file {filename}, line {reader.line_num}: {e}') And while the module doesn't directly support parsing strings, it can easily be done:: @@ -616,7 +638,7 @@ done:: .. rubric:: Footnotes .. [1] If ``newline=''`` is not specified, newlines embedded inside quoted fields - will not be interpreted correctly, and on platforms that use ``\r\n`` linendings + will not be interpreted correctly, and on platforms that use ``\r\n`` line endings on write an extra ``\r`` will be added. It should always be safe to specify ``newline=''``, since the csv module does its own (:term:`universal `) newline handling. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 5b733d5321e..9c0b246c095 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,8 @@ data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. +.. include:: ../includes/optional-module.rst + .. _ctypes-ctypes-tutorial: @@ -232,8 +234,24 @@ Fundamental data types +----------------------+------------------------------------------+----------------------------+ | :class:`c_int` | :c:expr:`int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_int8` | :c:type:`int8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int16` | :c:type:`int16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int32` | :c:type:`int32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int64` | :c:type:`int64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_uint` | :c:expr:`unsigned int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint8` | :c:type:`uint8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint16` | :c:type:`uint16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint32` | :c:type:`uint32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint64` | :c:type:`uint64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_long` | :c:expr:`long` | int | +----------------------+------------------------------------------+----------------------------+ | :class:`c_ulong` | :c:expr:`unsigned long` | int | @@ -684,14 +702,10 @@ compiler does it. It is possible to override this behavior entirely by specifyi :attr:`~Structure._layout_` class attribute in the subclass definition; see the attribute documentation for details. -It is possible to specify the maximum alignment for the fields by setting -the :attr:`~Structure._pack_` class attribute to a positive integer. -This matches what ``#pragma pack(n)`` does in MSVC. - -It is also possible to set a minimum alignment for how the subclass itself is packed in the -same way ``#pragma align(n)`` works in MSVC. -This can be achieved by specifying a :attr:`~Structure._align_` class attribute -in the subclass definition. +It is possible to specify the maximum alignment for the fields and/or for the +structure itself by setting the class attributes :attr:`~Structure._pack_` +and/or :attr:`~Structure._align_`, respectively. +See the attribute documentation for details. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -714,10 +728,16 @@ item in the :attr:`~Structure._fields_` tuples:: ... ("second_16", c_int, 16)] ... >>> print(Int.first_16) - + >>> print(Int.second_16) - - >>> + + +It is important to note that bit field allocation and layout in memory are not +defined as a C standard; their implementation is compiler-specific. +By default, Python will attempt to match the behavior of a "native" compiler +for the current platform. +See the :attr:`~Structure._layout_` attribute for details on the default +behavior and how to change it. .. _ctypes-arrays: @@ -876,7 +896,7 @@ invalid non-\ ``NULL`` pointers would crash Python):: Thread safety without the GIL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In Python 3.13, the :term:`GIL` may be disabled on :term:`experimental free threaded ` builds. +From Python 3.13 onward, the :term:`GIL` can be disabled on :term:`free threaded ` builds. In ctypes, reads and writes to a single object concurrently is safe, but not across multiple objects: .. code-block:: pycon @@ -2031,35 +2051,55 @@ Utility functions pointer. -.. function:: create_string_buffer(init_or_size, size=None) +.. function:: create_string_buffer(init, size=None) + create_string_buffer(size) This function creates a mutable character buffer. The returned object is a ctypes array of :class:`c_char`. - *init_or_size* must be an integer which specifies the size of the array, or a - bytes object which will be used to initialize the array items. + If *size* is given (and not ``None``), it must be an :class:`int`. + It specifies the size of the returned array. - If a bytes object is specified as first argument, the buffer is made one item - larger than its length so that the last element in the array is a NUL - termination character. An integer can be passed as second argument which allows - specifying the size of the array if the length of the bytes should not be used. + If the *init* argument is given, it must be :class:`bytes`. It is used + to initialize the array items. Bytes not initialized this way are + set to zero (NUL). + + If *size* is not given (or if it is ``None``), the buffer is made one element + larger than *init*, effectively adding a NUL terminator. + + If both arguments are given, *size* must not be less than ``len(init)``. + + .. warning:: + + If *size* is equal to ``len(init)``, a NUL terminator is + not added. Do not treat such a buffer as a C string. + + For example:: + + >>> bytes(create_string_buffer(2)) + b'\x00\x00' + >>> bytes(create_string_buffer(b'ab')) + b'ab\x00' + >>> bytes(create_string_buffer(b'ab', 2)) + b'ab' + >>> bytes(create_string_buffer(b'ab', 4)) + b'ab\x00\x00' + >>> bytes(create_string_buffer(b'abcdef', 2)) + Traceback (most recent call last): + ... + ValueError: byte string too long .. audit-event:: ctypes.create_string_buffer init,size ctypes.create_string_buffer -.. function:: create_unicode_buffer(init_or_size, size=None) +.. function:: create_unicode_buffer(init, size=None) + create_unicode_buffer(size) This function creates a mutable unicode character buffer. The returned object is a ctypes array of :class:`c_wchar`. - *init_or_size* must be an integer which specifies the size of the array, or a - string which will be used to initialize the array items. - - If a string is specified as first argument, the buffer is made one item - larger than the length of the string so that the last element in the array is a - NUL termination character. An integer can be passed as second argument which - allows specifying the size of the array if the length of the string should not - be used. + The function takes the same arguments as :func:`~create_string_buffer` except + *init* must be a string and *size* counts :class:`c_wchar`. .. audit-event:: ctypes.create_unicode_buffer init,size ctypes.create_unicode_buffer @@ -2498,7 +2538,7 @@ These are the fundamental ctypes data types: .. class:: c_int8 - Represents the C 8-bit :c:expr:`signed int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`signed int` datatype. It is an alias for :class:`c_byte`. @@ -2573,7 +2613,7 @@ These are the fundamental ctypes data types: .. class:: c_uint8 - Represents the C 8-bit :c:expr:`unsigned int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`unsigned int` datatype. It is an alias for :class:`c_ubyte`. @@ -2750,11 +2790,18 @@ fields, or any other data types containing pointer type fields. .. attribute:: _pack_ An optional small integer that allows overriding the alignment of - structure fields in the instance. :attr:`_pack_` must already be defined - when :attr:`_fields_` is assigned, otherwise it will have no effect. - Setting this attribute to 0 is the same as not setting it at all. + structure fields in the instance. - This is only implemented for the MSVC-compatible memory layout. + This is only implemented for the MSVC-compatible memory layout + (see :attr:`_layout_`). + + Setting :attr:`!_pack_` to 0 is the same as not setting it at all. + Otherwise, the value must be a positive power of two. + The effect is equivalent to ``#pragma pack(N)`` in C, except + :mod:`ctypes` may allow larger *n* than what the compiler accepts. + + :attr:`!_pack_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. deprecated-removed:: 3.14 3.19 @@ -2767,9 +2814,22 @@ fields, or any other data types containing pointer type fields. .. attribute:: _align_ - An optional small integer that allows overriding the alignment of + An optional small integer that allows increasing the alignment of the structure when being packed or unpacked to/from memory. - Setting this attribute to 0 is the same as not setting it at all. + + The value must not be negative. + The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC + or ``#pragma align(N)`` on MSVC, except :mod:`ctypes` may allow + values that the compiler would reject. + + :attr:`!_align_` can only *increase* a structure's alignment + requirements. Setting it to 0 or 1 has no effect. + + Using values that are not powers of two is discouraged and may lead to + surprising behavior. + + :attr:`!_align_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. versionadded:: 3.13 @@ -2939,7 +2999,7 @@ fields, or any other data types containing pointer type fields. .. attribute:: is_anonymous True if this field is anonymous, that is, it contains nested sub-fields - that should be be merged into a containing structure or union. + that should be merged into a containing structure or union. .. _ctypes-arrays-pointers: diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 137504c51b4..057d338edda 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -23,6 +23,8 @@ Linux and the BSD variants of Unix. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -68,7 +70,7 @@ The module :mod:`curses` defines the following exception: The module :mod:`curses` defines the following functions: -.. function:: assume_default_colors(fg, bg) +.. function:: assume_default_colors(fg, bg, /) Allow use of default values for colors on terminals supporting this feature. Use this to support transparency in your application. @@ -716,8 +718,10 @@ The module :mod:`curses` defines the following functions: Window Objects -------------- -Window objects, as returned by :func:`initscr` and :func:`newwin` above, have -the following methods and attributes: +.. class:: window + + Window objects, as returned by :func:`initscr` and :func:`newwin` above, have + the following methods and attributes: .. method:: window.addch(ch[, attr]) @@ -770,7 +774,7 @@ the following methods and attributes: .. method:: window.attron(attr) - Add attribute *attr* from the "background" set applied to all writes to the + Add attribute *attr* to the "background" set applied to all writes to the current window. @@ -988,6 +992,10 @@ the following methods and attributes: window.getstr(y, x, n) Read a bytes object from the user, with primitive line editing capacity. + The maximum value for *n* is 2047. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. .. method:: window.getyx() @@ -1079,6 +1087,10 @@ the following methods and attributes: current cursor position, or at *y*, *x* if specified. Attributes are stripped from the characters. If *n* is specified, :meth:`instr` returns a string at most *n* characters long (exclusive of the trailing NUL). + The maximum value for *n* is 2047. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. .. method:: window.is_linetouched(line) @@ -1339,7 +1351,6 @@ The :mod:`curses` module defines the following data members: .. data:: version -.. data:: __version__ A bytes object representing the current version of the module. diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 72612211b43..cff36e25822 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -121,8 +121,11 @@ Module contents :meth:`!__le__`, :meth:`!__gt__`, or :meth:`!__ge__`, then :exc:`TypeError` is raised. - - *unsafe_hash*: If ``False`` (the default), a :meth:`~object.__hash__` method - is generated according to how *eq* and *frozen* are set. + - *unsafe_hash*: If true, force ``dataclasses`` to create a + :meth:`~object.__hash__` method, even though it may not be safe to do so. + Otherwise, generate a :meth:`~object.__hash__` method according to how + *eq* and *frozen* are set. + The default value is ``False``. :meth:`!__hash__` is used by built-in :meth:`hash`, and when objects are added to hashed collections such as dictionaries and sets. Having a @@ -158,9 +161,11 @@ Module contents :class:`object`, this means it will fall back to id-based hashing). - *frozen*: If true (the default is ``False``), assigning to fields will - generate an exception. This emulates read-only frozen instances. If - :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then - :exc:`TypeError` is raised. See the discussion below. + generate an exception. This emulates read-only frozen instances. + See the :ref:`discussion ` below. + + If :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class + and *frozen* is true, then :exc:`TypeError` is raised. - *match_args*: If true (the default is ``True``), the :attr:`~object.__match_args__` tuple will be created from the list of @@ -304,15 +309,15 @@ Module contents .. versionadded:: 3.10 - - ``doc``: optional docstring for this field. + - *doc*: optional docstring for this field. - .. versionadded:: 3.13 + .. versionadded:: 3.14 If the default value of a field is specified by a call to :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`@dataclass ` decorator runs, the class + that after the :deco:`dataclass` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -422,7 +427,7 @@ Module contents :data:`typing.Any` is used for ``type``. The values of *init*, *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, *match_args*, *kw_only*, *slots*, and *weakref_slot* have - the same meaning as they do in :func:`@dataclass `. + the same meaning as they do in :deco:`dataclass`. If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. @@ -430,12 +435,12 @@ Module contents The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments - as :func:`@dataclass `. By default, the :func:`@dataclass ` + as :deco:`dataclass`. By default, the :deco:`dataclass` function is used. This function is not strictly required, because any Python - mechanism for creating a new class with :attr:`!__annotations__` can - then apply the :func:`@dataclass ` function to convert that class to + mechanism for creating a new class with :attr:`~object.__annotations__` can + then apply the :deco:`dataclass` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -564,7 +569,7 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +The :meth:`~object.__init__` method generated by :deco:`dataclass` does not call base class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -594,7 +599,7 @@ parameters to :meth:`!__post_init__`. Also see the warning about how Class variables --------------- -One of the few places where :func:`@dataclass ` actually inspects the type +One of the few places where :deco:`dataclass` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is :data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded @@ -607,7 +612,7 @@ module-level :func:`fields` function. Init-only variables ------------------- -Another place where :func:`@dataclass ` inspects a type annotation is to +Another place where :deco:`dataclass` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type :class:`InitVar`. If a field is an :class:`InitVar`, it is considered a pseudo-field called an init-only @@ -641,7 +646,7 @@ Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can +by passing ``frozen=True`` to the :deco:`dataclass` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -657,7 +662,7 @@ must use :meth:`!object.__setattr__`. Inheritance ----------- -When the dataclass is being created by the :func:`@dataclass ` decorator, +When the dataclass is being created by the :deco:`dataclass` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -781,7 +786,7 @@ for :attr:`!x` when creating a class instance will share the same copy of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it +:deco:`dataclass` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -815,7 +820,7 @@ default value have the following special behaviors: :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, :func:`@dataclass ` +* To determine whether a field contains a default value, :deco:`dataclass` will call the descriptor's :meth:`!__get__` method using its class access form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1ce2013f05d..8ae1c1fb9e4 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -261,6 +261,22 @@ A :class:`timedelta` object represents a duration, the difference between two >>> (d.days, d.seconds, d.microseconds) (-1, 86399, 999999) + Since the string representation of :class:`!timedelta` objects can be confusing, + use the following recipe to produce a more readable format: + + .. code-block:: pycon + + >>> def pretty_timedelta(td): + ... if td.days >= 0: + ... return str(td) + ... return f'-({-td!s})' + ... + >>> d = timedelta(hours=-1) + >>> str(d) # not human-friendly + '-1 day, 23:00:00' + >>> pretty_timedelta(d) + '-(1:00:00)' + Class attributes: @@ -519,6 +535,9 @@ Other constructors, all class methods: :c:func:`localtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`localtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. classmethod:: date.fromordinal(ordinal) @@ -1004,6 +1023,10 @@ Other constructors, all class methods: .. versionchanged:: 3.6 :meth:`fromtimestamp` may return instances with :attr:`.fold` set to 1. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + + .. classmethod:: datetime.utcfromtimestamp(timestamp) Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with @@ -1044,6 +1067,9 @@ Other constructors, all class methods: Use :meth:`datetime.fromtimestamp` with :const:`UTC` instead. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. classmethod:: datetime.fromordinal(ordinal) @@ -1486,11 +1512,11 @@ Instance methods: returned by :func:`time.time`. Naive :class:`.datetime` instances are assumed to represent local - time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`.datetime` - supports wider range of values than :c:func:`mktime` on many - platforms, this method may raise :exc:`OverflowError` or :exc:`OSError` - for times far in the past or far in the future. + time and this method relies on platform C functions to perform + the conversion. Since :class:`.datetime` supports a wider range of + values than the platform C functions on many platforms, this + method may raise :exc:`OverflowError` or :exc:`OSError` for times + far in the past or far in the future. For aware :class:`.datetime` instances, the return value is computed as:: @@ -1503,6 +1529,10 @@ Instance methods: The :meth:`timestamp` method uses the :attr:`.fold` attribute to disambiguate the times during a repeated interval. + .. versionchanged:: 3.6 + This method no longer relies on the platform C :c:func:`mktime` + function to perform conversions. + .. note:: There is no method to obtain the POSIX timestamp directly from a @@ -2363,7 +2393,7 @@ locations where different offsets are used in different days of the year or where historical changes have been made to civil time. -.. class:: timezone(offset, name=None) +.. class:: timezone(offset[, name]) The *offset* argument must be specified as a :class:`timedelta` object representing the difference between the local time and UTC. It must @@ -2609,7 +2639,10 @@ differences between platforms in handling of unsupported format specifiers. ``%G``, ``%u`` and ``%V`` were added. .. versionadded:: 3.12 - ``%:z`` was added. + ``%:z`` was added for :meth:`~.datetime.strftime` + +.. versionadded:: 3.15 + ``%:z`` was added for :meth:`~.datetime.strptime` Technical Detail ^^^^^^^^^^^^^^^^ @@ -2704,12 +2737,18 @@ Notes: When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, the UTC offsets can have a colon as a separator between hours, minutes and seconds. - For example, ``'+01:00:00'`` will be parsed as an offset of one hour. - In addition, providing ``'Z'`` is identical to ``'+00:00'``. + For example, both ``'+010000'`` and ``'+01:00:00'`` will be parsed as an offset + of one hour. In addition, providing ``'Z'`` is identical to ``'+00:00'``. ``%:z`` - Behaves exactly as ``%z``, but has a colon separator added between - hours, minutes and seconds. + When used with :meth:`~.datetime.strftime`, behaves exactly as ``%z``, + except that a colon separator is added between hours, minutes and seconds. + + When used with :meth:`~.datetime.strptime`, the UTC offset is *required* + to have a colon as a separator between hours, minutes and seconds. + For example, ``'+01:00:00'`` (but *not* ``'+010000'``) will be parsed as + an offset of one hour. In addition, providing ``'Z'`` is identical to + ``'+00:00'``. ``%Z`` In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 36221c026d6..02eb68d7b49 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -15,10 +15,16 @@ * :mod:`dbm.ndbm` If none of these modules are installed, the -slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There +slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There is a `third party interface `_ to the Oracle Berkeley DB. +.. note:: + None of the underlying modules will automatically shrink the disk space used by + the database file. However, :mod:`dbm.sqlite3`, :mod:`dbm.gnu` and :mod:`dbm.dumb` + provide a :meth:`!reorganize` method that can be used for this purpose. + + .. exception:: error A tuple containing the exceptions that can be raised by each of the supported @@ -84,10 +90,13 @@ the Oracle Berkeley DB. .. versionchanged:: 3.11 *file* accepts a :term:`path-like object`. -The object returned by :func:`~dbm.open` supports the same basic functionality as a -:class:`dict`; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`!keys` method are -available, as well as :meth:`!get` and :meth:`!setdefault` methods. +The object returned by :func:`~dbm.open` supports the basic +functionality of mutable :term:`mappings `; +keys and their corresponding values can be stored, retrieved, and +deleted, and iteration, the :keyword:`in` operator and methods :meth:`!keys`, +:meth:`!get`, :meth:`!setdefault` and :meth:`!clear` are available. +The :meth:`!keys` method returns a list instead of a view object. +The :meth:`!setdefault` method requires two arguments. Key and values are always stored as :class:`bytes`. This means that when strings are used they are implicitly converted to the default encoding before @@ -108,6 +117,10 @@ will automatically close them when done. Deleting a key from a read-only database raises a database module specific exception instead of :exc:`KeyError`. +.. versionchanged:: 3.13 + :meth:`!clear` methods are now available for all :mod:`dbm` backends. + + The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: @@ -167,9 +180,6 @@ or any other SQLite browser, including the SQLite CLI. .. function:: open(filename, /, flag="r", mode=0o666) Open an SQLite database. - The returned object behaves like a :term:`mapping`, - implements a :meth:`!close` method, - and supports a "closing" context manager via the :keyword:`with` keyword. :param filename: The path to the database to be opened. @@ -186,6 +196,29 @@ or any other SQLite browser, including the SQLite CLI. The Unix file access mode of the file (default: octal ``0o666``), used only when the database has to be created. + The returned database object behaves similar to a mutable :term:`mapping`, + but the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. + + The following methods are also provided: + + .. method:: sqlite3.close() + + Close the SQLite database. + + .. method:: sqlite3.reorganize() + + If you have carried out a lot of deletions and would like to shrink the space + used on disk, this method will reorganize the database; otherwise, deleted file + space will be kept and reused as new (key, value) pairs are added. + + .. note:: + While reorganizing, as much as two times the size of the original database is required + in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + + .. versionadded:: 3.15 + :mod:`dbm.gnu` --- GNU database manager --------------------------------------- @@ -215,6 +248,11 @@ functionality like crash tolerance. raised for general mapping errors like specifying an incorrect key. +.. data:: open_flags + + A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + + .. function:: open(filename, flag="r", mode=0o666, /) Open a GDBM database and return a :class:`!gdbm` object. @@ -250,14 +288,25 @@ functionality like crash tolerance. .. versionchanged:: 3.11 *filename* accepts a :term:`path-like object`. - .. data:: open_flags + :class:`!gdbm` objects behave similar to mutable :term:`mappings `, + but methods :meth:`!items`, :meth:`!values`, :meth:`!pop`, :meth:`!popitem`, + and :meth:`!update` are not supported, + the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + .. versionchanged:: 3.2 + Added the :meth:`!get` and :meth:`!setdefault` methods. + + .. versionchanged:: 3.13 + Added the :meth:`!clear` method. - :class:`!gdbm` objects behave similar to :term:`mappings `, - but :meth:`!items` and :meth:`!values` methods are not supported. The following methods are also provided: + .. method:: gdbm.close() + + Close the GDBM database. + .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the @@ -284,21 +333,15 @@ functionality like crash tolerance. reorganization; otherwise, deleted file space will be kept and reused as new (key, value) pairs are added. + .. note:: + While reorganizing, as much as one time the size of the original database is required + in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + .. method:: gdbm.sync() When the database has been opened in fast mode, this method forces any unwritten data to be written to the disk. - .. method:: gdbm.close() - - Close the GDBM database. - - .. method:: gdbm.clear() - - Remove all items from the GDBM database. - - .. versionadded:: 3.13 - :mod:`dbm.ndbm` --- New Database Manager ---------------------------------------- @@ -359,23 +402,28 @@ This module can be used with the "classic" NDBM interface or the :param int mode: |mode_param_doc| - :class:`!ndbm` objects behave similar to :term:`mappings `, - but :meth:`!items` and :meth:`!values` methods are not supported. - The following methods are also provided: - .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. + :class:`!ndbm` objects behave similar to mutable :term:`mappings `, + but methods :meth:`!items`, :meth:`!values`, :meth:`!pop`, :meth:`!popitem`, + and :meth:`!update` are not supported, + the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. + + .. versionchanged:: 3.2 + Added the :meth:`!get` and :meth:`!setdefault` methods. + + .. versionchanged:: 3.13 + Added the :meth:`!clear` method. + + The following method is also provided: + .. method:: ndbm.close() Close the NDBM database. - .. method:: ndbm.clear() - - Remove all items from the NDBM database. - - .. versionadded:: 3.13 - :mod:`dbm.dumb` --- Portable DBM implementation ----------------------------------------------- @@ -412,9 +460,6 @@ The :mod:`!dbm.dumb` module defines the following: .. function:: open(filename, flag="c", mode=0o666) Open a :mod:`!dbm.dumb` database. - The returned database object behaves similar to a :term:`mapping`, - in addition to providing :meth:`~dumbdbm.sync` and :meth:`~dumbdbm.close` - methods. :param filename: The basename of the database file (without extensions). @@ -438,6 +483,11 @@ The :mod:`!dbm.dumb` module defines the following: with a sufficiently large/complex entry due to stack depth limitations in Python's AST compiler. + .. warning:: + :mod:`dbm.dumb` does not support concurrent read/write access. (Multiple + simultaneous read accesses are safe.) When a program has the database open + for writing, no other program should have it open for reading or writing. + .. versionchanged:: 3.5 :func:`~dbm.dumb.open` always creates a new database when *flag* is ``'n'``. @@ -448,15 +498,30 @@ The :mod:`!dbm.dumb` module defines the following: .. versionchanged:: 3.11 *filename* accepts a :term:`path-like object`. - In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, - the following methods are provided: + The returned database object behaves similar to a mutable :term:`mapping`, + but the :meth:`!keys` and :meth:`!items` methods return lists, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. + + The following methods are also provided: + + .. method:: dumbdbm.close() + + Close the database. + + .. method:: dumbdbm.reorganize() + + If you have carried out a lot of deletions and would like to shrink the space + used on disk, this method will reorganize the database; otherwise, deleted file + space will not be reused. + + .. note:: + While reorganizing, no additional free disk space is required. However, be aware + that this factor changes for each :mod:`dbm` submodule. + + .. versionadded:: 3.15 .. method:: dumbdbm.sync() Synchronize the on-disk directory and data files. This method is called by the :meth:`shelve.Shelf.sync` method. - - .. method:: dumbdbm.close() - - Close the database. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index f53c1f34670..621aa23ecc8 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -2,7 +2,7 @@ ===================================================================== .. module:: decimal - :synopsis: Implementation of the General Decimal Arithmetic Specification. + :synopsis: Implementation of the General Decimal Arithmetic Specification. .. moduleauthor:: Eric Price .. moduleauthor:: Facundo Batista @@ -121,7 +121,7 @@ reset them before monitoring a calculation. .. _decimal-tutorial: -Quick-start Tutorial +Quick-start tutorial -------------------- The usual start to using decimals is importing the module, viewing the current @@ -264,7 +264,7 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. In accordance with the standard, the :mod:`decimal` module provides two ready to @@ -573,7 +573,7 @@ Decimal objects >>> Decimal(321).exp() Decimal('2.561702493119680037517373933E+139') - .. classmethod:: from_float(f) + .. classmethod:: from_float(f, /) Alternative constructor that only accepts instances of :class:`float` or :class:`int`. @@ -600,7 +600,7 @@ Decimal objects .. versionadded:: 3.1 - .. classmethod:: from_number(number) + .. classmethod:: from_number(number, /) Alternative constructor that only accepts instances of :class:`float`, :class:`int` or :class:`Decimal`, but not strings @@ -991,7 +991,7 @@ Each thread has its own current context which is accessed or changed using the Return the current context for the active thread. -.. function:: setcontext(c) +.. function:: setcontext(c, /) Set the current context for the active thread to *c*. @@ -1096,40 +1096,52 @@ In addition to the three supplied contexts, new contexts can be created with the default values are copied from the :const:`DefaultContext`. If the *flags* field is not specified or is :const:`None`, all flags are cleared. - *prec* is an integer in the range [``1``, :const:`MAX_PREC`] that sets - the precision for arithmetic operations in the context. + .. attribute:: prec - The *rounding* option is one of the constants listed in the section - `Rounding Modes`_. + An integer in the range [``1``, :const:`MAX_PREC`] that sets + the precision for arithmetic operations in the context. - The *traps* and *flags* fields list any signals to be set. Generally, new - contexts should only set traps and leave the flags clear. + .. attribute:: rounding - The *Emin* and *Emax* fields are integers specifying the outer limits allowable - for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, ``0``], - *Emax* in the range [``0``, :const:`MAX_EMAX`]. + One of the constants listed in the section `Rounding Modes`_. - The *capitals* field is either ``0`` or ``1`` (the default). If set to - ``1``, exponents are printed with a capital ``E``; otherwise, a - lowercase ``e`` is used: ``Decimal('6.02e+23')``. + .. attribute:: traps + flags - The *clamp* field is either ``0`` (the default) or ``1``. - If set to ``1``, the exponent ``e`` of a :class:`Decimal` - instance representable in this context is strictly limited to the - range ``Emin - prec + 1 <= e <= Emax - prec + 1``. If *clamp* is - ``0`` then a weaker condition holds: the adjusted exponent of - the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is - ``1``, a large normal number will, where possible, have its - exponent reduced and a corresponding number of zeros added to its - coefficient, in order to fit the exponent constraints; this - preserves the value of the number but loses information about - significant trailing zeros. For example:: + Lists of any signals to be set. Generally, new contexts should only set + traps and leave the flags clear. - >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') - Decimal('1.23000E+999') + .. attribute:: Emin + Emax - A *clamp* value of ``1`` allows compatibility with the - fixed-width decimal interchange formats specified in IEEE 754. + Integers specifying the outer limits allowable for exponents. *Emin* must + be in the range [:const:`MIN_EMIN`, ``0``], *Emax* in the range + [``0``, :const:`MAX_EMAX`]. + + .. attribute:: capitals + + Either ``0`` or ``1`` (the default). If set to + ``1``, exponents are printed with a capital ``E``; otherwise, a + lowercase ``e`` is used: ``Decimal('6.02e+23')``. + + .. attribute:: clamp + + Either ``0`` (the default) or ``1``. If set to ``1``, the exponent ``e`` + of a :class:`Decimal` instance representable in this context is strictly + limited to the range ``Emin - prec + 1 <= e <= Emax - prec + 1``. + If *clamp* is ``0`` then a weaker condition holds: the adjusted exponent of + the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is + ``1``, a large normal number will, where possible, have its + exponent reduced and a corresponding number of zeros added to its + coefficient, in order to fit the exponent constraints; this + preserves the value of the number but loses information about + significant trailing zeros. For example:: + + >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') + Decimal('1.23000E+999') + + A *clamp* value of ``1`` allows compatibility with the + fixed-width decimal interchange formats specified in IEEE 754. The :class:`Context` class defines several general purpose methods as well as a large number of methods for doing arithmetic directly in a given context. @@ -1156,11 +1168,11 @@ In addition to the three supplied contexts, new contexts can be created with the Return a duplicate of the context. - .. method:: copy_decimal(num) + .. method:: copy_decimal(num, /) Return a copy of the Decimal instance num. - .. method:: create_decimal(num) + .. method:: create_decimal(num='0', /) Creates a new Decimal instance from *num* but using *self* as context. Unlike the :class:`Decimal` constructor, the context precision, @@ -1184,7 +1196,7 @@ In addition to the three supplied contexts, new contexts can be created with the If the argument is a string, no leading or trailing whitespace or underscores are permitted. - .. method:: create_decimal_from_float(f) + .. method:: create_decimal_from_float(f, /) Creates a new Decimal instance from a float *f* but rounding using *self* as the context. Unlike the :meth:`Decimal.from_float` class method, @@ -1222,222 +1234,222 @@ In addition to the three supplied contexts, new contexts can be created with the recounted here. - .. method:: abs(x) + .. method:: abs(x, /) Returns the absolute value of *x*. - .. method:: add(x, y) + .. method:: add(x, y, /) Return the sum of *x* and *y*. - .. method:: canonical(x) + .. method:: canonical(x, /) Returns the same Decimal object *x*. - .. method:: compare(x, y) + .. method:: compare(x, y, /) Compares *x* and *y* numerically. - .. method:: compare_signal(x, y) + .. method:: compare_signal(x, y, /) Compares the values of the two operands numerically. - .. method:: compare_total(x, y) + .. method:: compare_total(x, y, /) Compares two operands using their abstract representation. - .. method:: compare_total_mag(x, y) + .. method:: compare_total_mag(x, y, /) Compares two operands using their abstract representation, ignoring sign. - .. method:: copy_abs(x) + .. method:: copy_abs(x, /) Returns a copy of *x* with the sign set to 0. - .. method:: copy_negate(x) + .. method:: copy_negate(x, /) Returns a copy of *x* with the sign inverted. - .. method:: copy_sign(x, y) + .. method:: copy_sign(x, y, /) Copies the sign from *y* to *x*. - .. method:: divide(x, y) + .. method:: divide(x, y, /) Return *x* divided by *y*. - .. method:: divide_int(x, y) + .. method:: divide_int(x, y, /) Return *x* divided by *y*, truncated to an integer. - .. method:: divmod(x, y) + .. method:: divmod(x, y, /) Divides two numbers and returns the integer part of the result. - .. method:: exp(x) + .. method:: exp(x, /) Returns ``e ** x``. - .. method:: fma(x, y, z) + .. method:: fma(x, y, z, /) Returns *x* multiplied by *y*, plus *z*. - .. method:: is_canonical(x) + .. method:: is_canonical(x, /) Returns ``True`` if *x* is canonical; otherwise returns ``False``. - .. method:: is_finite(x) + .. method:: is_finite(x, /) Returns ``True`` if *x* is finite; otherwise returns ``False``. - .. method:: is_infinite(x) + .. method:: is_infinite(x, /) Returns ``True`` if *x* is infinite; otherwise returns ``False``. - .. method:: is_nan(x) + .. method:: is_nan(x, /) Returns ``True`` if *x* is a qNaN or sNaN; otherwise returns ``False``. - .. method:: is_normal(x) + .. method:: is_normal(x, /) Returns ``True`` if *x* is a normal number; otherwise returns ``False``. - .. method:: is_qnan(x) + .. method:: is_qnan(x, /) Returns ``True`` if *x* is a quiet NaN; otherwise returns ``False``. - .. method:: is_signed(x) + .. method:: is_signed(x, /) Returns ``True`` if *x* is negative; otherwise returns ``False``. - .. method:: is_snan(x) + .. method:: is_snan(x, /) Returns ``True`` if *x* is a signaling NaN; otherwise returns ``False``. - .. method:: is_subnormal(x) + .. method:: is_subnormal(x, /) Returns ``True`` if *x* is subnormal; otherwise returns ``False``. - .. method:: is_zero(x) + .. method:: is_zero(x, /) Returns ``True`` if *x* is a zero; otherwise returns ``False``. - .. method:: ln(x) + .. method:: ln(x, /) Returns the natural (base e) logarithm of *x*. - .. method:: log10(x) + .. method:: log10(x, /) Returns the base 10 logarithm of *x*. - .. method:: logb(x) + .. method:: logb(x, /) Returns the exponent of the magnitude of the operand's MSD. - .. method:: logical_and(x, y) + .. method:: logical_and(x, y, /) Applies the logical operation *and* between each operand's digits. - .. method:: logical_invert(x) + .. method:: logical_invert(x, /) Invert all the digits in *x*. - .. method:: logical_or(x, y) + .. method:: logical_or(x, y, /) Applies the logical operation *or* between each operand's digits. - .. method:: logical_xor(x, y) + .. method:: logical_xor(x, y, /) Applies the logical operation *xor* between each operand's digits. - .. method:: max(x, y) + .. method:: max(x, y, /) Compares two values numerically and returns the maximum. - .. method:: max_mag(x, y) + .. method:: max_mag(x, y, /) Compares the values numerically with their sign ignored. - .. method:: min(x, y) + .. method:: min(x, y, /) Compares two values numerically and returns the minimum. - .. method:: min_mag(x, y) + .. method:: min_mag(x, y, /) Compares the values numerically with their sign ignored. - .. method:: minus(x) + .. method:: minus(x, /) Minus corresponds to the unary prefix minus operator in Python. - .. method:: multiply(x, y) + .. method:: multiply(x, y, /) Return the product of *x* and *y*. - .. method:: next_minus(x) + .. method:: next_minus(x, /) Returns the largest representable number smaller than *x*. - .. method:: next_plus(x) + .. method:: next_plus(x, /) Returns the smallest representable number larger than *x*. - .. method:: next_toward(x, y) + .. method:: next_toward(x, y, /) Returns the number closest to *x*, in direction towards *y*. - .. method:: normalize(x) + .. method:: normalize(x, /) Reduces *x* to its simplest form. - .. method:: number_class(x) + .. method:: number_class(x, /) Returns an indication of the class of *x*. - .. method:: plus(x) + .. method:: plus(x, /) Plus corresponds to the unary prefix plus operator in Python. This operation applies the context precision and rounding, so it is *not* an @@ -1478,7 +1490,7 @@ In addition to the three supplied contexts, new contexts can be created with the always exact. - .. method:: quantize(x, y) + .. method:: quantize(x, y, /) Returns a value equal to *x* (rounded), having the exponent of *y*. @@ -1488,7 +1500,7 @@ In addition to the three supplied contexts, new contexts can be created with the Just returns 10, as this is Decimal, :) - .. method:: remainder(x, y) + .. method:: remainder(x, y, /) Returns the remainder from integer division. @@ -1496,43 +1508,43 @@ In addition to the three supplied contexts, new contexts can be created with the dividend. - .. method:: remainder_near(x, y) + .. method:: remainder_near(x, y, /) Returns ``x - y * n``, where *n* is the integer nearest the exact value of ``x / y`` (if the result is 0 then its sign will be the sign of *x*). - .. method:: rotate(x, y) + .. method:: rotate(x, y, /) Returns a rotated copy of *x*, *y* times. - .. method:: same_quantum(x, y) + .. method:: same_quantum(x, y, /) Returns ``True`` if the two operands have the same exponent. - .. method:: scaleb (x, y) + .. method:: scaleb (x, y, /) Returns the first operand after adding the second value its exp. - .. method:: shift(x, y) + .. method:: shift(x, y, /) Returns a shifted copy of *x*, *y* times. - .. method:: sqrt(x) + .. method:: sqrt(x, /) Square root of a non-negative number to context precision. - .. method:: subtract(x, y) + .. method:: subtract(x, y, /) Return the difference between *x* and *y*. - .. method:: to_eng_string(x) + .. method:: to_eng_string(x, /) Convert to a string, using engineering notation if an exponent is needed. @@ -1541,12 +1553,12 @@ In addition to the three supplied contexts, new contexts can be created with the require the addition of either one or two trailing zeros. - .. method:: to_integral_exact(x) + .. method:: to_integral_exact(x, /) Rounds to an integer. - .. method:: to_sci_string(x) + .. method:: to_sci_string(x, /) Converts a number to a string using scientific notation. @@ -1557,7 +1569,16 @@ In addition to the three supplied contexts, new contexts can be created with the Constants --------- -The constants in this section are only relevant for the C module. They +.. data:: SPEC_VERSION + + The highest version of the General Decimal Arithmetic + Specification that this implementation complies with. + See https://speleotrove.com/decimal/decarith.html for the specification. + + .. versionadded:: 3.15 + + +The following constants are only relevant for the C module. They are also included in the pure Python version for compatibility. +---------------------------------+---------------------+-------------------------------+ @@ -1769,7 +1790,7 @@ The following table summarizes the hierarchy of signals:: .. _decimal-notes: -Floating-Point Notes +Floating-point notes -------------------- @@ -2088,20 +2109,20 @@ to work with the :class:`Decimal` class:: Decimal FAQ ----------- -Q. It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to +Q: It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to minimize typing when using the interactive interpreter? -A. Some users abbreviate the constructor to just a single letter: +A: Some users abbreviate the constructor to just a single letter: >>> D = decimal.Decimal >>> D('1.23') + D('3.45') Decimal('4.68') -Q. In a fixed-point application with two decimal places, some inputs have many +Q: In a fixed-point application with two decimal places, some inputs have many places and need to be rounded. Others are not supposed to have excess digits and need to be validated. What methods should be used? -A. The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If +A: The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If the :const:`Inexact` trap is set, it is also useful for validation: >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') @@ -2119,10 +2140,10 @@ the :const:`Inexact` trap is set, it is also useful for validation: ... Inexact: None -Q. Once I have valid two place inputs, how do I maintain that invariant +Q: Once I have valid two place inputs, how do I maintain that invariant throughout an application? -A. Some operations like addition, subtraction, and multiplication by an integer +A: Some operations like addition, subtraction, and multiplication by an integer will automatically preserve fixed point. Others operations, like division and non-integer multiplication, will change the number of decimal places and need to be followed-up with a :meth:`~Decimal.quantize` step: @@ -2154,21 +2175,21 @@ to handle the :meth:`~Decimal.quantize` step: >>> div(b, a) Decimal('0.03') -Q. There are many ways to express the same value. The numbers ``200``, +Q: There are many ways to express the same value. The numbers ``200``, ``200.000``, ``2E2``, and ``.02E+4`` all have the same value at various precisions. Is there a way to transform them to a single recognizable canonical value? -A. The :meth:`~Decimal.normalize` method maps all equivalent values to a single +A: The :meth:`~Decimal.normalize` method maps all equivalent values to a single representative: >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) >>> [v.normalize() for v in values] [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] -Q. When does rounding occur in a computation? +Q: When does rounding occur in a computation? -A. It occurs *after* the computation. The philosophy of the decimal +A: It occurs *after* the computation. The philosophy of the decimal specification is that numbers are considered exact and are created independent of the current context. They can even have greater precision than current context. Computations process with those @@ -2186,10 +2207,10 @@ applied to the *result* of the computation:: >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded Decimal('3.1416') -Q. Some decimal values always print with exponential notation. Is there a way +Q: Some decimal values always print with exponential notation. Is there a way to get a non-exponential representation? -A. For some values, exponential notation is the only way to express the number +A: For some values, exponential notation is the only way to express the number of significant places in the coefficient. For example, expressing ``5.0E+3`` as ``5000`` keeps the value constant but cannot show the original's two-place significance. @@ -2204,9 +2225,9 @@ value unchanged: >>> remove_exponent(Decimal('5E+3')) Decimal('5000') -Q. Is there a way to convert a regular float to a :class:`Decimal`? +Q: Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating-point number can be exactly expressed as a +A: Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2215,19 +2236,19 @@ suggest: >>> Decimal(math.pi) Decimal('3.141592653589793115997963468544185161590576171875') -Q. Within a complex calculation, how can I make sure that I haven't gotten a +Q: Within a complex calculation, how can I make sure that I haven't gotten a spurious result because of insufficient precision or rounding anomalies. -A. The decimal module makes it easy to test results. A best practice is to +A: The decimal module makes it easy to test results. A best practice is to re-run calculations using greater precision and with various rounding modes. Widely differing results indicate insufficient precision, rounding mode issues, ill-conditioned inputs, or a numerically unstable algorithm. -Q. I noticed that context precision is applied to the results of operations but +Q: I noticed that context precision is applied to the results of operations but not to the inputs. Is there anything to watch out for when mixing values of different precisions? -A. Yes. The principle is that all values are considered to be exact and so is +A: Yes. The principle is that all values are considered to be exact and so is the arithmetic on those values. Only the results are rounded. The advantage for inputs is that "what you type is what you get". A disadvantage is that the results can look odd if you forget that the inputs haven't been rounded: @@ -2255,9 +2276,9 @@ Alternatively, inputs can be rounded upon creation using the >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') Decimal('1.2345') -Q. Is the CPython implementation fast for large numbers? +Q: Is the CPython implementation fast for large numbers? -A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of +A: Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index 191e0da1210..e0693e8eb6e 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -220,7 +220,7 @@ is the base class for dialogs defined in other supporting modules. .. class:: Dialog(master=None, **options) - .. method:: show(color=None, **options) + .. method:: show(**options) Render the Dialog window. diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index ce948a6860f..9e5a62d8fe5 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -231,7 +231,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. *linejunk*: A function that accepts a single string argument, and returns true if the string is junk, or false if not. The default is ``None``. There is also a module-level function :func:`IS_LINE_JUNK`, which filters out lines - without visible characters, except for at most one pound character (``'#'``) + without visible characters, except for at most one hash character (``'#'``) -- however the underlying :class:`SequenceMatcher` class does a dynamic analysis of which lines are so frequent as to constitute noise, and this usually works better than using this function. @@ -278,7 +278,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. emu -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, color=False) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in unified diff format. @@ -297,6 +297,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. For inputs that do not have trailing newlines, set the *lineterm* argument to ``""`` so that the output will be uniformly newline free. + Set *color* to ``True`` to enable output in color, similar to + :program:`git diff --color`. Even if enabled, it can be + :ref:`controlled using environment variables `. + The unified diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for *fromfile*, *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally @@ -319,6 +323,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. See :ref:`difflib-interface` for a more detailed example. + .. versionchanged:: 3.15 + Added the *color* parameter. + + .. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a @@ -351,9 +359,9 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. seealso:: - `Pattern Matching: The Gestalt Approach `_ + `Pattern Matching: The Gestalt Approach `_ Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. + was published in Dr. Dobb's Journal in July, 1988. .. _sequence-matcher: diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 44767b5dd2d..a24589fd0a5 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -585,6 +585,22 @@ operations on it as if it was a Python list. The top of the stack corresponds to generate line tracing events. +.. opcode:: NOT_TAKEN + + Do nothing code. + Used by the interpreter to record :monitoring-event:`BRANCH_LEFT` + and :monitoring-event:`BRANCH_RIGHT` events for :mod:`sys.monitoring`. + + .. versionadded:: 3.14 + + +.. opcode:: POP_ITER + + Removes the iterator from the top of the stack. + + .. versionadded:: 3.14 + + .. opcode:: POP_TOP Removes the top-of-stack item:: @@ -1094,14 +1110,6 @@ iterations of the loop. .. versionadded:: 3.14 -.. opcode:: LOAD_CONST_IMMORTAL (consti) - - Pushes ``co_consts[consti]`` onto the stack. - Can be used when the constant value is known to be immortal. - - .. versionadded:: 3.14 - - .. opcode:: LOAD_NAME (namei) Pushes the value associated with ``co_names[namei]`` onto the stack. @@ -1128,6 +1136,48 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: BUILD_TEMPLATE + + Constructs a new :class:`~string.templatelib.Template` instance from a tuple + of strings and a tuple of interpolations and pushes the resulting object + onto the stack:: + + interpolations = STACK.pop() + strings = STACK.pop() + STACK.append(_build_template(strings, interpolations)) + + .. versionadded:: 3.14 + + +.. opcode:: BUILD_INTERPOLATION (format) + + Constructs a new :class:`~string.templatelib.Interpolation` instance from a + value and its source expression and pushes the resulting object onto the + stack. + + If no conversion or format specification is present, ``format`` is set to + ``2``. + + If the low bit of ``format`` is set, it indicates that the interpolation + contains a format specification. + + If ``format >> 2`` is non-zero, it indicates that the interpolation + contains a conversion. The value of ``format >> 2`` is the conversion type + (``0`` for no conversion, ``1`` for ``!s``, ``2`` for ``!r``, and + ``3`` for ``!a``):: + + conversion = format >> 2 + if format & 1: + format_spec = STACK.pop() + else: + format_spec = None + expression = STACK.pop() + value = STACK.pop() + STACK.append(_build_interpolation(value, expression, conversion, format_spec)) + + .. versionadded:: 3.14 + + .. opcode:: BUILD_TUPLE (count) Creates a tuple consuming *count* items from the stack, and pushes the @@ -1592,7 +1642,7 @@ iterations of the loop. Pushes a ``NULL`` to the stack. Used in the call sequence to match the ``NULL`` pushed by - :opcode:`LOAD_METHOD` for non-method calls. + :opcode:`!LOAD_METHOD` for non-method calls. .. versionadded:: 3.11 @@ -1623,9 +1673,13 @@ iterations of the loop. * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure + * ``0x10`` the :term:`annotate function` for the function object .. versionadded:: 3.13 + .. versionchanged:: 3.14 + Added ``0x10`` to indicate the annotate function for the function object. + .. opcode:: BUILD_SLICE (argc) @@ -1913,14 +1967,15 @@ but are replaced by real opcodes or removed before bytecode is generated. Marks the end of the code block associated with the last ``SETUP_FINALLY``, ``SETUP_CLEANUP`` or ``SETUP_WITH``. + .. opcode:: JUMP -.. opcode:: JUMP_NO_INTERRUPT + JUMP_NO_INTERRUPT Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. .. opcode:: JUMP_IF_TRUE -.. opcode:: JUMP_IF_FALSE + JUMP_IF_FALSE Conditional jumps which do not impact the stack. Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. @@ -1936,12 +1991,6 @@ but are replaced by real opcodes or removed before bytecode is generated. This opcode is now a pseudo-instruction. -.. opcode:: LOAD_METHOD - - Optimized unbound method lookup. Emitted as a ``LOAD_ATTR`` opcode - with a flag set in the arg. - - .. _opcode_collections: Opcode collections diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index b86fef9fd6f..df3de8f622a 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -174,7 +174,7 @@ with assorted summaries at the end. You can force verbose mode by passing ``verbose=True`` to :func:`testmod`, or prohibit it by passing ``verbose=False``. In either of those cases, -``sys.argv`` is not examined by :func:`testmod` (so passing ``-v`` or not +:data:`sys.argv` is not examined by :func:`testmod` (so passing ``-v`` or not has no effect). There is also a command line shortcut for running :func:`testmod`, see section @@ -231,7 +231,7 @@ documentation:: As with :func:`testmod`, :func:`testfile` won't display anything unless an example fails. If an example does fail, then the failing example(s) and the cause(s) of the failure(s) are printed to stdout, using the same format as -:func:`testmod`. +:func:`!testmod`. By default, :func:`testfile` looks for files in the calling module's directory. See section :ref:`doctest-basic-api` for a description of the optional arguments @@ -311,6 +311,13 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. +.. currentmodule:: None + +.. attribute:: module.__test__ + :no-typesetting: + +.. currentmodule:: doctest + In addition, there are cases when you want tests to be part of a module but not part of the help text, which requires that the tests not be included in the docstring. Doctest looks for a module-level variable called ``__test__`` and uses it to locate other @@ -343,6 +350,13 @@ searches them recursively for docstrings, which are then scanned for tests. Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. +.. note:: + + ``doctest`` can only automatically discover classes and functions that are + defined at the module level or inside other classes. + + Since nested classes and functions only exist when an outer function + is called, they cannot be discovered. Define them outside to make them visible. .. _doctest-finding-examples: @@ -533,7 +547,7 @@ Some details you should read once, but won't need to remember: * The interactive shell omits the traceback header line for some :exc:`SyntaxError`\ s. But doctest uses the traceback header line to distinguish exceptions from non-exceptions. So in the rare case where you need - to test a :exc:`SyntaxError` that omits the traceback header, you will need to + to test a :exc:`!SyntaxError` that omits the traceback header, you will need to manually add the traceback header line to your test example. .. index:: single: ^ (caret); marker @@ -860,15 +874,15 @@ The :const:`ELLIPSIS` directive gives a nice approach for the last example: Floating-point numbers are also subject to small output variations across -platforms, because Python defers to the platform C library for float formatting, -and C libraries vary widely in quality here. :: +platforms, because Python defers to the platform C library for some +floating-point calculations, and C libraries vary widely in quality here. :: - >>> 1./7 # risky - 0.14285714285714285 - >>> print(1./7) # safer - 0.142857142857 - >>> print(round(1./7, 6)) # much safer - 0.142857 + >>> 1000**0.1 # risky + 1.9952623149688797 + >>> round(1000**0.1, 9) # safer + 1.995262315 + >>> print(f'{1000**0.1:.4f}') # much safer + 1.9953 Numbers of the form ``I/2.**J`` are safe across all platforms, and I often contrive doctest examples to produce numbers of that form:: @@ -938,13 +952,13 @@ and :ref:`doctest-simple-testfile`. Optional argument *verbose* prints lots of stuff if true, and prints only failures if false; by default, or if ``None``, it's true if and only if ``'-v'`` - is in ``sys.argv``. + is in :data:`sys.argv`. Optional argument *report* prints a summary at the end when true, else prints nothing at the end. In verbose mode, the summary is detailed, else the summary is very brief (in fact, empty if all tests passed). - Optional argument *optionflags* (default value 0) takes the + Optional argument *optionflags* (default value ``0``) takes the :ref:`bitwise OR ` of option flags. See section :ref:`doctest-options`. @@ -1043,12 +1057,15 @@ from text files and modules with doctests: Convert doctest tests from one or more text files to a :class:`unittest.TestSuite`. - The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs the interactive examples in each file. If an example in any file - fails, then the synthesized unit test fails, and a :exc:`failureException` - exception is raised showing the name of the file containing the test and a - (sometimes approximate) line number. If all the examples in a file are - skipped, then the synthesized unit test is also marked as skipped. + The returned :class:`unittest.TestSuite` is to be run by the unittest + framework and runs the interactive examples in each file. + Each file is run as a separate unit test, and each example in a file + is run as a :ref:`subtest `. + If any example in a file fails, then the synthesized unit test fails. + The traceback for failure or error contains the name of the file + containing the test and a (sometimes approximate) line number. + If all the examples in a file are skipped, then the synthesized unit + test is also marked as skipped. Pass one or more paths (as strings) to text files to be examined. @@ -1078,13 +1095,14 @@ from text files and modules with doctests: Optional argument *setUp* specifies a set-up function for the test suite. This is called before running the tests in each file. The *setUp* function - will be passed a :class:`DocTest` object. The setUp function can access the - test globals as the *globs* attribute of the test passed. + will be passed a :class:`DocTest` object. The *setUp* function can access the + test globals as the :attr:`~DocTest.globs` attribute of the test passed. Optional argument *tearDown* specifies a tear-down function for the test suite. This is called after running the tests in each file. The *tearDown* - function will be passed a :class:`DocTest` object. The setUp function can - access the test globals as the *globs* attribute of the test passed. + function will be passed a :class:`DocTest` object. The *tearDown* function can + access the test globals as the :attr:`~DocTest.globs` attribute of the test + passed. Optional argument *globs* is a dictionary containing the initial global variables for the tests. A new copy of this dictionary is created for each @@ -1105,16 +1123,22 @@ from text files and modules with doctests: The global ``__file__`` is added to the globals provided to doctests loaded from a text file using :func:`DocFileSuite`. + .. versionchanged:: 3.15 + Run each example as a :ref:`subtest `. + .. function:: DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, setUp=None, tearDown=None, optionflags=0, checker=None) Convert doctest tests for a module to a :class:`unittest.TestSuite`. - The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs each doctest in the module. If any of the doctests fail, then the - synthesized unit test fails, and a :exc:`failureException` exception is raised - showing the name of the file containing the test and a (sometimes approximate) - line number. If all the examples in a docstring are skipped, then the + The returned :class:`unittest.TestSuite` is to be run by the unittest + framework and runs each doctest in the module. + Each docstring is run as a separate unit test, and each example in + a docstring is run as a :ref:`subtest `. + If any of the doctests fail, then the synthesized unit test fails. + The traceback for failure or error contains the name of the file + containing the test and a (sometimes approximate) line number. + If all the examples in a docstring are skipped, then the synthesized unit test is also marked as skipped. Optional argument *module* provides the module to be tested. It can be a module @@ -1123,7 +1147,7 @@ from text files and modules with doctests: Optional argument *globs* is a dictionary containing the initial global variables for the tests. A new copy of this dictionary is created for each - test. By default, *globs* is a new empty dictionary. + test. By default, *globs* is the module's :attr:`~module.__dict__`. Optional argument *extraglobs* specifies an extra set of global variables, which is merged into *globs*. By default, no extra globals are used. @@ -1132,7 +1156,7 @@ from text files and modules with doctests: drop-in replacement) that is used to extract doctests from the module. Optional arguments *setUp*, *tearDown*, and *optionflags* are the same as for - function :func:`DocFileSuite` above. + function :func:`DocFileSuite` above, but they are called for each docstring. This function uses the same search technique as :func:`testmod`. @@ -1140,11 +1164,8 @@ from text files and modules with doctests: :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* contains no docstrings instead of raising :exc:`ValueError`. -.. exception:: failureException - - When doctests which have been converted to unit tests by :func:`DocFileSuite` - or :func:`DocTestSuite` fail, this exception is raised showing the name of - the file containing the test and a (sometimes approximate) line number. + .. versionchanged:: 3.15 + Run each example as a :ref:`subtest `. Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a @@ -1158,15 +1179,15 @@ of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of :class:`!DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`doctest` options in -use directly, by passing option flags to :mod:`doctest` functions. However, if -you're writing a :mod:`unittest` framework, :mod:`unittest` ultimately controls +:mod:`doctest` functions yourself, you can control the :mod:`!doctest` options in +use directly, by passing option flags to :mod:`!doctest` functions. However, if +you're writing a :mod:`unittest` framework, :mod:`!unittest` ultimately controls when and how tests get run. The framework author typically wants to control -:mod:`doctest` reporting options (perhaps, e.g., specified by command line -options), but there's no way to pass options through :mod:`unittest` to -:mod:`doctest` test runners. +:mod:`!doctest` reporting options (perhaps, e.g., specified by command line +options), but there's no way to pass options through :mod:`!unittest` to +:mod:`!doctest` test runners. -For this reason, :mod:`doctest` also supports a notion of :mod:`doctest` +For this reason, :mod:`doctest` also supports a notion of :mod:`!doctest` reporting flags specific to :mod:`unittest` support, via this function: @@ -1181,12 +1202,12 @@ reporting flags specific to :mod:`unittest` support, via this function: :mod:`unittest`: the :meth:`!runTest` method of :class:`!DocTestCase` looks at the option flags specified for the test case when the :class:`!DocTestCase` instance was constructed. If no reporting flags were specified (which is the - typical and expected case), :mod:`!doctest`'s :mod:`unittest` reporting flags are + typical and expected case), :mod:`!doctest`'s :mod:`!unittest` reporting flags are :ref:`bitwise ORed ` into the option flags, and the option flags so augmented are passed to the :class:`DocTestRunner` instance created to run the doctest. If any reporting flags were specified when the :class:`!DocTestCase` instance was constructed, :mod:`!doctest`'s - :mod:`unittest` reporting flags are ignored. + :mod:`!unittest` reporting flags are ignored. The value of the :mod:`unittest` reporting flags in effect before the function was called is returned by the function. @@ -1279,7 +1300,7 @@ DocTest Objects .. attribute:: filename The name of the file that this :class:`DocTest` was extracted from; or - ``None`` if the filename is unknown, or if the :class:`DocTest` was not + ``None`` if the filename is unknown, or if the :class:`!DocTest` was not extracted from a file. @@ -1419,10 +1440,10 @@ DocTestFinder objects The globals for each :class:`DocTest` is formed by combining *globs* and *extraglobs* (bindings in *extraglobs* override bindings in *globs*). A new - shallow copy of the globals dictionary is created for each :class:`DocTest`. - If *globs* is not specified, then it defaults to the module's *__dict__*, if - specified, or ``{}`` otherwise. If *extraglobs* is not specified, then it - defaults to ``{}``. + shallow copy of the globals dictionary is created for each :class:`!DocTest`. + If *globs* is not specified, then it defaults to the module's + :attr:`~module.__dict__`, if specified, or ``{}`` otherwise. + If *extraglobs* is not specified, then it defaults to ``{}``. .. _doctest-doctestparser: @@ -1446,7 +1467,7 @@ DocTestParser objects :class:`DocTest` object. *globs*, *name*, *filename*, and *lineno* are attributes for the new - :class:`DocTest` object. See the documentation for :class:`DocTest` for more + :class:`!DocTest` object. See the documentation for :class:`DocTest` for more information. @@ -1461,7 +1482,7 @@ DocTestParser objects Divide the given string into examples and intervening text, and return them as a list of alternating :class:`Example`\ s and strings. Line numbers for the - :class:`Example`\ s are 0-based. The optional argument *name* is a name + :class:`!Example`\ s are 0-based. The optional argument *name* is a name identifying this string, and is only used for error messages. @@ -1501,14 +1522,14 @@ DocTestRunner objects :class:`OutputChecker`. This comparison may be customized with a number of option flags; see section :ref:`doctest-options` for more information. If the option flags are insufficient, then the comparison may also be customized by - passing a subclass of :class:`OutputChecker` to the constructor. + passing a subclass of :class:`!OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. First, an output function can be passed to :meth:`run`; this function will be called with strings that should be displayed. It defaults to ``sys.stdout.write``. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods - :meth:`report_start`, :meth:`report_success`, + :meth:`report_skip`, :meth:`report_start`, :meth:`report_success`, :meth:`report_unexpected_exception`, and :meth:`report_failure`. The optional keyword argument *checker* specifies the :class:`OutputChecker` @@ -1533,6 +1554,19 @@ DocTestRunner objects :class:`DocTestRunner` defines the following methods: + .. method:: report_skip(out, test, example) + + Report that the given example was skipped. This method is provided to + allow subclasses of :class:`DocTestRunner` to customize their output; it + should not be called directly. + + *example* is the example about to be processed. *test* is the test + containing *example*. *out* is the output function that was passed to + :meth:`DocTestRunner.run`. + + .. versionadded:: 3.15 + + .. method:: report_start(out, test, example) Report that the test runner is about to process the given example. This method @@ -1540,7 +1574,7 @@ DocTestRunner objects output; it should not be called directly. *example* is the example about to be processed. *test* is the test - *containing example*. *out* is the output function that was passed to + containing *example*. *out* is the output function that was passed to :meth:`DocTestRunner.run`. @@ -1940,7 +1974,7 @@ several options for organizing tests: containing test cases for the named topics. These functions can be included in the same file as the module, or separated out into a separate test file. -* Define a ``__test__`` dictionary mapping from regression test topics to +* Define a :attr:`~module.__test__` dictionary mapping from regression test topics to docstrings containing test cases. When you have placed your tests in a module, the module can itself be the test diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 4285c436e8d..5754c2b65b2 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: :meth:`set_payload` instead. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the related ``make`` and ``add`` methods. @@ -224,7 +224,7 @@ Here are the methods of the :class:`Message` class: ASCII charset. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.get_content` and :meth:`~email.message.EmailMessage.iter_parts`. @@ -236,7 +236,7 @@ Here are the methods of the :class:`Message` class: the message's default character set; see :meth:`set_charset` for details. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content`. @@ -265,9 +265,9 @@ Here are the methods of the :class:`Message` class: using that :mailheader:`Content-Transfer-Encoding` and is not modified. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *charset* parameter of the - :meth:`email.emailmessage.EmailMessage.set_content` method. + :meth:`email.message.EmailMessage.set_content` method. .. method:: get_charset() @@ -276,7 +276,7 @@ Here are the methods of the :class:`Message` class: message's payload. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class it always returns + :class:`~email.message.EmailMessage` class it always returns ``None``. @@ -486,7 +486,7 @@ Here are the methods of the :class:`Message` class: search instead of :mailheader:`Content-Type`. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *params* property of the individual header objects returned by the header access methods. @@ -524,7 +524,7 @@ Here are the methods of the :class:`Message` class: to ``False``. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *params* property of the individual header objects returned by the header access methods. @@ -579,7 +579,7 @@ Here are the methods of the :class:`Message` class: header is also added. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the ``make_`` and ``add_`` methods. diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index 219fad0d2f6..f49885b8785 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -178,16 +178,36 @@ The :mod:`email.header` module also provides the following convenient functions. Decode a message header value without converting the character set. The header value is in *header*. - This function returns a list of ``(decoded_string, charset)`` pairs containing - each of the decoded parts of the header. *charset* is ``None`` for non-encoded - parts of the header, otherwise a lower case string containing the name of the - character set specified in the encoded string. + For historical reasons, this function may return either: - Here's an example:: + 1. A list of pairs containing each of the decoded parts of the header, + ``(decoded_bytes, charset)``, where *decoded_bytes* is always an instance of + :class:`bytes`, and *charset* is either: + + - A lower case string containing the name of the character set specified. + + - ``None`` for non-encoded parts of the header. + + 2. A list of length 1 containing a pair ``(string, None)``, where + *string* is always an instance of :class:`str`. + + An :exc:`email.errors.HeaderParseError` may be raised when certain decoding + errors occur (e.g. a base64 decoding exception). + + Here are examples: >>> from email.header import decode_header >>> decode_header('=?iso-8859-1?q?p=F6stal?=') [(b'p\xf6stal', 'iso-8859-1')] + >>> decode_header('unencoded_string') + [('unencoded_string', None)] + >>> decode_header('bar =?utf-8?B?ZsOzbw==?=') + [(b'bar ', None), (b'f\xc3\xb3o', 'utf-8')] + + .. note:: + + This function exists for backwards compatibility only. For + new code, we recommend using :class:`email.headerregistry.HeaderRegistry`. .. function:: make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' ') @@ -203,3 +223,7 @@ The :mod:`email.header` module also provides the following convenient functions. :class:`Header` instance. Optional *maxlinelen*, *header_name*, and *continuation_ws* are as in the :class:`Header` constructor. + .. note:: + + This function exists for backwards compatibility only, and is + not recommended for use in new code. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 7f8044932fa..ff8b601fe3d 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -294,7 +294,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. ``inline`` and ``attachment`` are the only valid values in common use. -.. class:: ContentTransferEncoding +.. class:: ContentTransferEncodingHeader Handles the :mailheader:`Content-Transfer-Encoding` header. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 439b5c8f34b..6a70714dc3e 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -116,7 +116,7 @@ Here is the API for the :class:`BytesFeedParser`: Works like :class:`BytesFeedParser` except that the input to the :meth:`~BytesFeedParser.feed` method must be a string. This is of limited utility, since the only way for such a message to be valid is for it to - contain only ASCII text or, if :attr:`~email.policy.Policy.utf8` is + contain only ASCII text or, if :attr:`~email.policy.EmailPolicy.utf8` is ``True``, no binary attachments. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -159,7 +159,7 @@ message body, instead setting the payload to the raw body. methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` - (or, if :attr:`~email.policy.Policy.utf8` is ``True``, :rfc:`6532`) + (or, if :attr:`~email.policy.EmailPolicy.utf8` is ``True``, :rfc:`6532`) style headers and header continuation lines, optionally preceded by an envelope header. The header block is terminated either by the end of the data or by a blank line. Following the header block is the body of the diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index fa102c4a080..32b92c01570 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -30,6 +30,8 @@ when creating a virtual environment) or after explicitly uninstalling needed to bootstrap ``pip`` are included as internal parts of the package. +.. include:: ../includes/optional-module.rst + .. seealso:: :ref:`installing-index` @@ -40,7 +42,9 @@ when creating a virtual environment) or after explicitly uninstalling .. include:: ../includes/wasm-mobile-notavail.rst -Command line interface +.. _ensurepip-cli: + +Command-line interface ---------------------- .. program:: ensurepip diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index c9b2c7d76b6..a8a7e671aad 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -175,6 +175,10 @@ Data Types final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. + .. versionadded:: 3.11 + + Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. + .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -206,7 +210,7 @@ Data Types >>> Color.RED.value in Color True - .. versionchanged:: 3.12 + .. versionchanged:: 3.12 Before Python 3.12, a ``TypeError`` is raised if a non-Enum-member is used in a containment check. @@ -251,20 +255,6 @@ Data Types >>> list(reversed(Color)) [, , ] - .. method:: EnumType._add_alias_ - - Adds a new name as an alias to an existing member. Raises a - :exc:`NameError` if the name is already assigned to a different member. - - .. method:: EnumType._add_value_alias_ - - Adds a new value as an alias to an existing member. Raises a - :exc:`ValueError` if the value is already linked with a different member. - - .. versionadded:: 3.11 - - Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. - .. class:: Enum @@ -325,6 +315,7 @@ Data Types Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and any public methods defined on *self.__class__*:: + >>> from enum import Enum >>> from datetime import date >>> class Weekday(Enum): ... MONDAY = 1 @@ -351,7 +342,7 @@ Data Types A *staticmethod* that is used to determine the next value returned by :class:`auto`:: - >>> from enum import auto + >>> from enum import auto, Enum >>> class PowersOfThree(Enum): ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): @@ -383,7 +374,7 @@ Data Types A *classmethod* for looking up values not found in *cls*. By default it does nothing, but can be overridden to implement custom search behavior:: - >>> from enum import StrEnum + >>> from enum import auto, StrEnum >>> class Build(StrEnum): ... DEBUG = auto() ... OPTIMIZED = auto() @@ -422,6 +413,7 @@ Data Types Returns the string used for *repr()* calls. By default, returns the *Enum* name, member name, and value, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -438,6 +430,7 @@ Data Types Returns the string used for *str()* calls. By default, returns the *Enum* name and member name, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -453,6 +446,7 @@ Data Types Returns the string used for *format()* and *f-string* calls. By default, returns :meth:`__str__` return value, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -470,6 +464,30 @@ Data Types .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. method:: Enum._add_alias_ + + Adds a new name as an alias to an existing member:: + + >>> Color.RED._add_alias_("ERROR") + >>> Color.ERROR + + + Raises a :exc:`NameError` if the name is already assigned to a different member. + + .. versionadded:: 3.13 + + .. method:: Enum._add_value_alias_ + + Adds a new value as an alias to an existing member:: + + >>> Color.RED._add_value_alias_(42) + >>> Color(42) + + + Raises a :exc:`ValueError` if the value is already linked with a different member. + + .. versionadded:: 3.13 + .. class:: IntEnum @@ -504,16 +522,31 @@ Data Types .. class:: StrEnum - ``StrEnum`` is the same as :class:`Enum`, but its members are also strings and can be used - in most of the same places that a string can be used. The result of any string - operation performed on or with a *StrEnum* member is not part of the enumeration. + *StrEnum* is the same as :class:`Enum`, but its members are also strings and + can be used in most of the same places that a string can be used. The result + of any string operation performed on or with a *StrEnum* member is not part + of the enumeration. + + >>> from enum import StrEnum, auto + >>> class Color(StrEnum): + ... RED = 'r' + ... GREEN = 'g' + ... BLUE = 'b' + ... UNKNOWN = auto() + ... + >>> Color.RED + + >>> Color.UNKNOWN + + >>> str(Color.UNKNOWN) + 'unknown' .. note:: There are places in the stdlib that check for an exact :class:`str` instead of a :class:`str` subclass (i.e. ``type(unknown) == str`` instead of ``isinstance(unknown, str)``), and in those locations you - will need to use ``str(StrEnum.member)``. + will need to use ``str(MyStrEnum.MY_MEMBER)``. .. note:: @@ -864,10 +897,6 @@ Once all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing - member. -- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an - existing member. - :attr:`~Enum._name_` -- name of the member - :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` - :meth:`~Enum._missing_` -- a lookup function used when a value is not found; @@ -888,6 +917,11 @@ Supported ``_sunder_`` names For :class:`Flag` classes the next value chosen will be the next highest power-of-two. +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an + existing member. + - While ``_sunder_`` names are generally reserved for the further development of the :class:`Enum` class and can not be used, some are explicitly allowed: diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index bb72032891e..b5e3a84b455 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -204,10 +204,16 @@ The following exceptions are the exceptions that are usually raised. assignment fails. (When an object does not support attribute references or attribute assignments at all, :exc:`TypeError` is raised.) - The :attr:`name` and :attr:`obj` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the attribute - that was attempted to be accessed and the object that was accessed for said - attribute, respectively. + The optional *name* and *obj* keyword-only arguments + set the corresponding attributes: + + .. attribute:: name + + The name of the attribute that was attempted to be accessed. + + .. attribute:: obj + + The object that was accessed for the named attribute. .. versionchanged:: 3.10 Added the :attr:`name` and :attr:`obj` attributes. @@ -215,7 +221,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: EOFError Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (N.B.: the :meth:`io.IOBase.read` and + without reading any data. (Note: the :meth:`!io.IOBase.read` and :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) @@ -312,9 +318,11 @@ The following exceptions are the exceptions that are usually raised. unqualified names. The associated value is an error message that includes the name that could not be found. - The :attr:`name` attribute can be set using a keyword-only argument to the - constructor. When set it represent the name of the variable that was attempted - to be accessed. + The optional *name* keyword-only argument sets the attribute: + + .. attribute:: name + + The name of the variable that was attempted to be accessed. .. versionchanged:: 3.10 Added the :attr:`name` attribute. @@ -382,7 +390,7 @@ The following exceptions are the exceptions that are usually raised. The corresponding error message, as provided by the operating system. It is formatted by the C - functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage` + functions :c:func:`!perror` under POSIX, and :c:func:`!FormatMessage` under Windows. .. attribute:: filename @@ -398,7 +406,7 @@ The following exceptions are the exceptions that are usually raised. .. versionchanged:: 3.3 :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`, :exc:`socket.error`, :exc:`select.error` and - :exc:`mmap.error` have been merged into :exc:`OSError`, and the + :exc:`!mmap.error` have been merged into :exc:`OSError`, and the constructor may return a subclass. .. versionchanged:: 3.4 @@ -429,7 +437,9 @@ The following exceptions are the exceptions that are usually raised. * Creating a new Python thread. * :meth:`Joining ` a running daemon thread. - * :func:`os.fork`. + * :func:`os.fork`, + * acquiring a lock such as :class:`threading.Lock`, when it is known that + the operation would otherwise deadlock. See also the :func:`sys.is_finalizing` function. @@ -440,6 +450,11 @@ The following exceptions are the exceptions that are usually raised. :meth:`threading.Thread.join` can now raise this exception. + .. versionchanged:: 3.15 + + This exception may be raised when acquiring :meth:`threading.Lock` + or :meth:`threading.RLock`. + .. exception:: RecursionError This exception is derived from :exc:`RuntimeError`. It is raised when the @@ -590,7 +605,7 @@ The following exceptions are the exceptions that are usually raised. handled, the Python interpreter exits; no stack traceback is printed. The constructor accepts the same optional argument passed to :func:`sys.exit`. If the value is an integer, it specifies the system exit status (passed to - C's :c:func:`exit` function); if it is ``None``, the exit status is zero; if + C's :c:func:`!exit` function); if it is ``None``, the exit status is zero; if it has another type (such as a string), the object's value is printed and the exit status is one. @@ -882,6 +897,9 @@ The following exceptions are used as warning categories; see the Base class for warnings about dubious syntax. + This warning is typically emitted when compiling Python source code, and usually won't be reported + when running already compiled code. + .. exception:: RuntimeWarning @@ -960,6 +978,12 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + .. impl-detail:: + + The ``excs`` parameter may be any sequence, but lists and tuples are + specifically processed more efficiently here. For optimal performance, + pass a tuple as ``excs``. + .. attribute:: message The ``msg`` argument to the constructor. This is a read-only attribute. @@ -1048,7 +1072,7 @@ their subgroups based on the types of the contained exceptions. subclasses that need a different constructor signature need to override that rather than :meth:`~object.__init__`. For example, the following defines an exception group subclass which accepts an exit_code and - and constructs the group's message from it. :: + constructs the group's message from it. :: class Errors(ExceptionGroup): def __new__(cls, errors, exit_code): diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 5058b85bffb..677966a8b2e 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -90,7 +90,7 @@ An error will be printed instead of the stack. Additionally, some compilers do not support :term:`CPython's ` implementation of C stack dumps. As a result, a different error may be printed -instead of the stack, even if the the operating system supports dumping stacks. +instead of the stack, even if the operating system supports dumping stacks. .. note:: @@ -228,6 +228,41 @@ handler: Fatal Python error: Segmentation fault Current thread 0x00007fb899f39700 (most recent call first): - File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at + File "/opt/python/Lib/ctypes/__init__.py", line 486 in string_at File "", line 1 in + + Current thread's C stack trace (most recent call first): + Binary file "/opt/python/python", at _Py_DumpStack+0x42 [0x5b27f7d7147e] + Binary file "/opt/python/python", at +0x32dcbd [0x5b27f7d85cbd] + Binary file "/opt/python/python", at +0x32df8a [0x5b27f7d85f8a] + Binary file "/usr/lib/libc.so.6", at +0x3def0 [0x77b73226bef0] + Binary file "/usr/lib/libc.so.6", at +0x17ef9c [0x77b7323acf9c] + Binary file "/opt/python/build/lib.linux-x86_64-3.15/_ctypes.cpython-315d-x86_64-linux-gnu.so", at +0xcdf6 [0x77b7315dddf6] + Binary file "/usr/lib/libffi.so.8", at +0x7976 [0x77b73158f976] + Binary file "/usr/lib/libffi.so.8", at +0x413c [0x77b73158c13c] + Binary file "/usr/lib/libffi.so.8", at ffi_call+0x12e [0x77b73158ef0e] + Binary file "/opt/python/build/lib.linux-x86_64-3.15/_ctypes.cpython-315d-x86_64-linux-gnu.so", at +0x15a33 [0x77b7315e6a33] + Binary file "/opt/python/build/lib.linux-x86_64-3.15/_ctypes.cpython-315d-x86_64-linux-gnu.so", at +0x164fa [0x77b7315e74fa] + Binary file "/opt/python/build/lib.linux-x86_64-3.15/_ctypes.cpython-315d-x86_64-linux-gnu.so", at +0xc624 [0x77b7315dd624] + Binary file "/opt/python/python", at _PyObject_MakeTpCall+0xce [0x5b27f7b73883] + Binary file "/opt/python/python", at +0x11bab6 [0x5b27f7b73ab6] + Binary file "/opt/python/python", at PyObject_Vectorcall+0x23 [0x5b27f7b73b04] + Binary file "/opt/python/python", at _PyEval_EvalFrameDefault+0x490c [0x5b27f7cbb302] + Binary file "/opt/python/python", at +0x2818e6 [0x5b27f7cd98e6] + Binary file "/opt/python/python", at +0x281aab [0x5b27f7cd9aab] + Binary file "/opt/python/python", at PyEval_EvalCode+0xc5 [0x5b27f7cd9ba3] + Binary file "/opt/python/python", at +0x255957 [0x5b27f7cad957] + Binary file "/opt/python/python", at +0x255ab4 [0x5b27f7cadab4] + Binary file "/opt/python/python", at _PyEval_EvalFrameDefault+0x6c3e [0x5b27f7cbd634] + Binary file "/opt/python/python", at +0x2818e6 [0x5b27f7cd98e6] + Binary file "/opt/python/python", at +0x281aab [0x5b27f7cd9aab] + Binary file "/opt/python/python", at +0x11b6e1 [0x5b27f7b736e1] + Binary file "/opt/python/python", at +0x11d348 [0x5b27f7b75348] + Binary file "/opt/python/python", at +0x11d626 [0x5b27f7b75626] + Binary file "/opt/python/python", at PyObject_Call+0x20 [0x5b27f7b7565e] + Binary file "/opt/python/python", at +0x32a67a [0x5b27f7d8267a] + Binary file "/opt/python/python", at +0x32a7f8 [0x5b27f7d827f8] + Binary file "/opt/python/python", at +0x32ac1b [0x5b27f7d82c1b] + Binary file "/opt/python/python", at Py_RunMain+0x31 [0x5b27f7d82ebe] + Segmentation fault diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index c8ce86cc7af..f57fcdf0bcf 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -107,15 +107,15 @@ The module defines the following functions: passed to the C :c:func:`fcntl` call. The return value after a successful call is the contents of the buffer, converted to a :class:`bytes` object. The length of the returned object will be the same as the length of the - *arg* argument. This is limited to 1024 bytes. + *arg* argument. If the :c:func:`fcntl` call fails, an :exc:`OSError` is raised. .. note:: - If the type or the size of *arg* does not match the type or size - of the argument of the operation (for example, if an integer is + If the type or size of *arg* does not match the type or size + of the operation's argument (for example, if an integer is passed when a pointer is expected, or the information returned in - the buffer by the operating system is larger than 1024 bytes), + the buffer by the operating system is larger than the size of *arg*), this is most likely to result in a segmentation violation or a more subtle data corruption. @@ -125,6 +125,9 @@ The module defines the following functions: Add support of arbitrary :term:`bytes-like objects `, not only :class:`bytes`. + .. versionchanged:: 3.15 + The size of bytes-like objects is no longer limited to 1024 bytes. + .. function:: ioctl(fd, request, arg=0, mutate_flag=True, /) @@ -161,8 +164,7 @@ The module defines the following functions: If the type or size of *arg* does not match the type or size of the operation's argument (for example, if an integer is passed when a pointer is expected, or the information returned in - the buffer by the operating system is larger than 1024 bytes, - or the size of the mutable bytes-like object is too small), + the buffer by the operating system is larger than the size of *arg*), this is most likely to result in a segmentation violation or a more subtle data corruption. @@ -185,6 +187,10 @@ The module defines the following functions: The GIL is always released during a system call. System calls failing with EINTR are automatically retried. + .. versionchanged:: 3.15 + The size of not mutated bytes-like objects is no longer + limited to 1024 bytes. + .. function:: flock(fd, operation, /) Perform the lock operation *operation* on file descriptor *fd* (file objects providing diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 12e61bc36f5..ee654b7a83e 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -53,7 +53,7 @@ a :class:`!str` filename, and vice-versa. Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following -functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. +functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. .. function:: fnmatch(name, pat) diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index fc7f9a6301a..d6d1c7a461c 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -14,8 +14,8 @@ The :mod:`fractions` module provides support for rational number arithmetic. -A Fraction instance can be constructed from a pair of integers, from -another rational number, or from a string. +A Fraction instance can be constructed from a pair of rational numbers, from +a single number, or from a string. .. index:: single: as_integer_ratio() @@ -25,8 +25,8 @@ another rational number, or from a string. The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value ``numerator/denominator``. If *denominator* is ``0``, it - raises a :exc:`ZeroDivisionError`. + with a value equal to ``numerator/denominator``. + If *denominator* is zero, it raises a :exc:`ZeroDivisionError`. The second version requires that *number* is an instance of :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method @@ -125,7 +125,8 @@ another rational number, or from a string. .. attribute:: denominator - Denominator of the Fraction in lowest term. + Denominator of the Fraction in lowest terms. + Guaranteed to be positive. .. method:: as_integer_ratio() @@ -142,7 +143,7 @@ another rational number, or from a string. .. versionadded:: 3.12 - .. classmethod:: from_float(flt) + .. classmethod:: from_float(f) Alternative constructor which only accepts instances of :class:`float` or :class:`numbers.Integral`. Beware that diff --git a/Doc/library/frameworks.rst b/Doc/library/frameworks.rst index 15ceeec9c25..f8e2f6bb18c 100644 --- a/Doc/library/frameworks.rst +++ b/Doc/library/frameworks.rst @@ -1,18 +1,13 @@ +:orphan: + .. _frameworks: ****************** -Program Frameworks +Program frameworks ****************** -The modules described in this chapter are frameworks that will largely dictate -the structure of your program. Currently the modules described here are all -oriented toward writing command-line interfaces. +This chapter is no longer maintained, and the modules it contained have been moved to their respective topical documentation. -The full list of modules described in this chapter is: - - -.. toctree:: - - turtle.rst - cmd.rst - shlex.rst +* :mod:`cmd` — :doc:`Command Line Interface Libraries <./cmdlinelibs>` +* :mod:`shlex` — :doc:`Unix Specific Services <./unix>` +* :mod:`turtle` — :doc:`Graphical User Interfaces with Tk <./tk>` diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7e367a0f2b6..8314fed80fa 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -54,7 +54,7 @@ are always available. They are listed here in alphabetical order. .. |func-bytearray| replace:: ``bytearray()`` .. |func-bytes| replace:: ``bytes()`` -.. function:: abs(x) +.. function:: abs(number, /) Return the absolute value of a number. The argument may be an integer, a floating-point number, or an object implementing @@ -62,7 +62,7 @@ are always available. They are listed here in alphabetical order. If the argument is a complex number, its magnitude is returned. -.. function:: aiter(async_iterable) +.. function:: aiter(async_iterable, /) Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. @@ -71,7 +71,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: all(iterable) +.. function:: all(iterable, /) Return ``True`` if all elements of the *iterable* are true (or if the iterable is empty). Equivalent to:: @@ -83,8 +83,8 @@ are always available. They are listed here in alphabetical order. return True -.. awaitablefunction:: anext(async_iterator) - anext(async_iterator, default) +.. awaitablefunction:: anext(async_iterator, /) + anext(async_iterator, default, /) When awaited, return the next item from the given :term:`asynchronous iterator`, or *default* if given and the iterator is exhausted. @@ -99,7 +99,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: any(iterable) +.. function:: any(iterable, /) Return ``True`` if any element of the *iterable* is true. If the iterable is empty, return ``False``. Equivalent to:: @@ -111,7 +111,7 @@ are always available. They are listed here in alphabetical order. return False -.. function:: ascii(object) +.. function:: ascii(object, /) As :func:`repr`, return a string containing a printable representation of an object, but escape the non-ASCII characters in the string returned by @@ -119,10 +119,10 @@ are always available. They are listed here in alphabetical order. similar to that returned by :func:`repr` in Python 2. -.. function:: bin(x) +.. function:: bin(integer, /) Convert an integer number to a binary string prefixed with "0b". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it + is a valid Python expression. If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. Some examples: @@ -183,8 +183,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytearray: .. class:: bytearray(source=b'') - bytearray(source, encoding) - bytearray(source, encoding, errors) + bytearray(source, encoding, errors='strict') :noindex: Return a new array of bytes. The :class:`bytearray` class is a mutable @@ -215,8 +214,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytes: .. class:: bytes(source=b'') - bytes(source, encoding) - bytes(source, encoding, errors) + bytes(source, encoding, errors='strict') :noindex: Return a new "bytes" object which is an immutable sequence of integers in @@ -231,7 +229,7 @@ are always available. They are listed here in alphabetical order. See also :ref:`binaryseq`, :ref:`typebytes`, and :ref:`bytes-methods`. -.. function:: callable(object) +.. function:: callable(object, /) Return :const:`True` if the *object* argument appears callable, :const:`False` if not. If this returns ``True``, it is still possible that a @@ -244,14 +242,14 @@ are always available. They are listed here in alphabetical order. in Python 3.2. -.. function:: chr(i) +.. function:: chr(codepoint, /) - Return the string representing a character whose Unicode code point is the - integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while + Return the string representing a character with the specified Unicode code point. + For example, ``chr(97)`` returns the string ``'a'``, while ``chr(8364)`` returns the string ``'€'``. This is the inverse of :func:`ord`. The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in - base 16). :exc:`ValueError` will be raised if *i* is outside that range. + base 16). :exc:`ValueError` will be raised if it is outside that range. .. decorator:: classmethod @@ -294,7 +292,9 @@ are always available. They are listed here in alphabetical order. :func:`property`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) +.. function:: compile(source, filename, mode, flags=0, \ + dont_inherit=False, optimize=-1, \ + *, module=None) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a normal string, a @@ -336,6 +336,10 @@ are always available. They are listed here in alphabetical order. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`ValueError` if the source contains null bytes. @@ -373,6 +377,9 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. + .. versionadded:: 3.15 + Added the *module* parameter. + .. class:: complex(number=0, /) complex(string, /) @@ -458,7 +465,7 @@ are always available. They are listed here in alphabetical order. deprecated; it should only be passed as a single positional argument. -.. function:: delattr(object, name) +.. function:: delattr(object, name, /) This is a relative of :func:`setattr`. The arguments are an object and a string. The string must be the name of one of the object's attributes. The @@ -468,9 +475,9 @@ are always available. They are listed here in alphabetical order. .. _func-dict: -.. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) +.. class:: dict(**kwargs) + dict(mapping, /, **kwargs) + dict(iterable, /, **kwargs) :noindex: Create a new dictionary. The :class:`dict` object is the dictionary class. @@ -481,7 +488,7 @@ are always available. They are listed here in alphabetical order. .. function:: dir() - dir(object) + dir(object, /) Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object. @@ -541,7 +548,7 @@ are always available. They are listed here in alphabetical order. class. -.. function:: divmod(a, b) +.. function:: divmod(a, b, /) Take two (non-complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using integer division. With @@ -729,7 +736,7 @@ are always available. They are listed here in alphabetical order. described for the :func:`locals` builtin. -.. function:: filter(function, iterable) +.. function:: filter(function, iterable, /) Construct an iterator from those elements of *iterable* for which *function* is true. *iterable* may be either a sequence, a container which @@ -823,7 +830,7 @@ are always available. They are listed here in alphabetical order. single: __format__ single: string; format() (built-in function) -.. function:: format(value, format_spec="") +.. function:: format(value, format_spec="", /) Convert a *value* to a "formatted" representation, as controlled by *format_spec*. The interpretation of *format_spec* will depend on the type @@ -846,7 +853,7 @@ are always available. They are listed here in alphabetical order. .. _func-frozenset: -.. class:: frozenset(iterable=set()) +.. class:: frozenset(iterable=(), /) :noindex: Return a new :class:`frozenset` object, optionally with elements taken from @@ -858,8 +865,8 @@ are always available. They are listed here in alphabetical order. module. -.. function:: getattr(object, name) - getattr(object, name, default) +.. function:: getattr(object, name, /) + getattr(object, name, default, /) Return the value of the named attribute of *object*. *name* must be a string. If the string is the name of one of the object's attributes, the result is the @@ -883,7 +890,7 @@ are always available. They are listed here in alphabetical order. regardless of where the function is called. -.. function:: hasattr(object, name) +.. function:: hasattr(object, name, /) The arguments are an object and a string. The result is ``True`` if the string is the name of one of the object's attributes, ``False`` if not. (This @@ -891,7 +898,7 @@ are always available. They are listed here in alphabetical order. raises an :exc:`AttributeError` or not.) -.. function:: hash(object) +.. function:: hash(object, /) Return the hash value of the object (if it has one). Hash values are integers. They are used to quickly compare dictionary keys during a @@ -926,10 +933,10 @@ are always available. They are listed here in alphabetical order. signatures for callables are now more comprehensive and consistent. -.. function:: hex(x) +.. function:: hex(integer, /) Convert an integer number to a lowercase hexadecimal string prefixed with - "0x". If *x* is not a Python :class:`int` object, it has to define an + "0x". If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. Some examples: >>> hex(255) @@ -958,7 +965,7 @@ are always available. They are listed here in alphabetical order. :meth:`float.hex` method. -.. function:: id(object) +.. function:: id(object, /) Return the "identity" of an object. This is an integer which is guaranteed to be unique and constant for this object during its lifetime. @@ -971,7 +978,7 @@ are always available. They are listed here in alphabetical order. .. function:: input() - input(prompt) + input(prompt, /) If the *prompt* argument is present, it is written to standard output without a trailing newline. The function then reads a line from input, converts it @@ -1071,7 +1078,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.14 :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. -.. function:: isinstance(object, classinfo) +.. function:: isinstance(object, classinfo, /) Return ``True`` if the *object* argument is an instance of the *classinfo* argument, or of a (direct, indirect, or :term:`virtual `) of *classinfo*. A @@ -1102,19 +1109,19 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: iter(object) - iter(object, sentinel) +.. function:: iter(iterable, /) + iter(callable, sentinel, /) Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a - second argument, *object* must be a collection object which supports the + second argument, the single argument must be a collection object which supports the :term:`iterable` protocol (the :meth:`~object.__iter__` method), or it must support the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, - then *object* must be a callable object. The iterator created in this case - will call *object* with no arguments for each call to its + then the first argument must be a callable object. The iterator created in this case + will call *callable* with no arguments for each call to its :meth:`~iterator.__next__` method; if the value returned is equal to *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will be returned. @@ -1131,7 +1138,7 @@ are always available. They are listed here in alphabetical order. process_block(block) -.. function:: len(s) +.. function:: len(object, /) Return the length (the number of items) of an object. The argument may be a sequence (such as a string, bytes, tuple, list, or range) or a collection @@ -1144,8 +1151,7 @@ are always available. They are listed here in alphabetical order. .. _func-list: -.. class:: list() - list(iterable) +.. class:: list(iterable=(), /) :noindex: Rather than being a function, :class:`list` is actually a mutable @@ -1154,44 +1160,44 @@ are always available. They are listed here in alphabetical order. .. function:: locals() - Return a mapping object representing the current local symbol table, with - variable names as the keys, and their currently bound references as the - values. + Return a mapping object representing the current local symbol table, with + variable names as the keys, and their currently bound references as the + values. - At module scope, as well as when using :func:`exec` or :func:`eval` with - a single namespace, this function returns the same namespace as - :func:`globals`. + At module scope, as well as when using :func:`exec` or :func:`eval` with + a single namespace, this function returns the same namespace as + :func:`globals`. - At class scope, it returns the namespace that will be passed to the - metaclass constructor. + At class scope, it returns the namespace that will be passed to the + metaclass constructor. - When using ``exec()`` or ``eval()`` with separate local and global - arguments, it returns the local namespace passed in to the function call. + When using ``exec()`` or ``eval()`` with separate local and global + arguments, it returns the local namespace passed in to the function call. - In all of the above cases, each call to ``locals()`` in a given frame of - execution will return the *same* mapping object. Changes made through - the mapping object returned from ``locals()`` will be visible as assigned, - reassigned, or deleted local variables, and assigning, reassigning, or - deleting local variables will immediately affect the contents of the - returned mapping object. + In all of the above cases, each call to ``locals()`` in a given frame of + execution will return the *same* mapping object. Changes made through + the mapping object returned from ``locals()`` will be visible as assigned, + reassigned, or deleted local variables, and assigning, reassigning, or + deleting local variables will immediately affect the contents of the + returned mapping object. - In an :term:`optimized scope` (including functions, generators, and - coroutines), each call to ``locals()`` instead returns a fresh dictionary - containing the current bindings of the function's local variables and any - nonlocal cell references. In this case, name binding changes made via the - returned dict are *not* written back to the corresponding local variables - or nonlocal cell references, and assigning, reassigning, or deleting local - variables and nonlocal cell references does *not* affect the contents - of previously returned dictionaries. + In an :term:`optimized scope` (including functions, generators, and + coroutines), each call to ``locals()`` instead returns a fresh dictionary + containing the current bindings of the function's local variables and any + nonlocal cell references. In this case, name binding changes made via the + returned dict are *not* written back to the corresponding local variables + or nonlocal cell references, and assigning, reassigning, or deleting local + variables and nonlocal cell references does *not* affect the contents + of previously returned dictionaries. - Calling ``locals()`` as part of a comprehension in a function, generator, or - coroutine is equivalent to calling it in the containing scope, except that - the comprehension's initialised iteration variables will be included. In - other scopes, it behaves as if the comprehension were running as a nested - function. + Calling ``locals()`` as part of a comprehension in a function, generator, or + coroutine is equivalent to calling it in the containing scope, except that + the comprehension's initialised iteration variables will be included. In + other scopes, it behaves as if the comprehension were running as a nested + function. - Calling ``locals()`` as part of a generator expression is equivalent to - calling it in a nested generator function. + Calling ``locals()`` as part of a generator expression is equivalent to + calling it in a nested generator function. .. versionchanged:: 3.12 The behaviour of ``locals()`` in a comprehension has been updated as @@ -1220,9 +1226,9 @@ are always available. They are listed here in alphabetical order. Added the *strict* parameter. -.. function:: max(iterable, *, key=None) - max(iterable, *, default, key=None) - max(arg1, arg2, *args, key=None) +.. function:: max(iterable, /, *, key=None) + max(iterable, /, *, default, key=None) + max(arg1, arg2, /, *args, key=None) Return the largest item in an iterable or the largest of two or more arguments. @@ -1258,9 +1264,9 @@ are always available. They are listed here in alphabetical order. :ref:`typememoryview` for more information. -.. function:: min(iterable, *, key=None) - min(iterable, *, default, key=None) - min(arg1, arg2, *args, key=None) +.. function:: min(iterable, /, *, key=None) + min(iterable, /, *, default, key=None) + min(arg1, arg2, /, *args, key=None) Return the smallest item in an iterable or the smallest of two or more arguments. @@ -1288,8 +1294,8 @@ are always available. They are listed here in alphabetical order. The *key* can be ``None``. -.. function:: next(iterator) - next(iterator, default) +.. function:: next(iterator, /) + next(iterator, default, /) Retrieve the next item from the :term:`iterator` by calling its :meth:`~iterator.__next__` method. If *default* is given, it is returned @@ -1310,10 +1316,10 @@ are always available. They are listed here in alphabetical order. :class:`object`. -.. function:: oct(x) +.. function:: oct(integer, /) Convert an integer number to an octal string prefixed with "0o". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it + is a valid Python expression. If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. For example: @@ -1422,38 +1428,10 @@ are always available. They are listed here in alphabetical order. *errors* is an optional string that specifies how encoding and decoding errors are to be handled—this cannot be used in binary mode. - A variety of standard error handlers are available - (listed under :ref:`error-handlers`), though any - error handling name that has been registered with + A variety of standard error handlers are available, + though any error handling name that has been registered with :func:`codecs.register_error` is also valid. The standard names - include: - - * ``'strict'`` to raise a :exc:`ValueError` exception if there is - an encoding error. The default value of ``None`` has the same - effect. - - * ``'ignore'`` ignores errors. Note that ignoring encoding errors - can lead to data loss. - - * ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted - where there is malformed data. - - * ``'surrogateescape'`` will represent any incorrect bytes as low - surrogate code units ranging from U+DC80 to U+DCFF. - These surrogate code units will then be turned back into - the same bytes when the ``surrogateescape`` error handler is used - when writing data. This is useful for processing files in an - unknown encoding. - - * ``'xmlcharrefreplace'`` is only supported when writing to a file. - Characters not supported by the encoding are replaced with the - appropriate XML character reference :samp:`&#{nnn};`. - - * ``'backslashreplace'`` replaces malformed data by Python's backslashed - escape sequences. - - * ``'namereplace'`` (also only supported when writing) - replaces unsupported characters with ``\N{...}`` escape sequences. + can be found in :ref:`error-handlers`. .. index:: single: universal newlines; open() built-in function @@ -1562,13 +1540,19 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 The ``'U'`` mode has been removed. -.. function:: ord(c) +.. function:: ord(character, /) - Given a string representing one Unicode character, return an integer - representing the Unicode code point of that character. For example, + Return the ordinal value of a character. + + If the argument is a one-character string, return the Unicode code point + of that character. For example, ``ord('a')`` returns the integer ``97`` and ``ord('€')`` (Euro sign) returns ``8364``. This is the inverse of :func:`chr`. + If the argument is a :class:`bytes` or :class:`bytearray` object of + length 1, return its single byte value. + For example, ``ord(b'a')`` returns the integer ``97``. + .. function:: pow(base, exp, mod=None) @@ -1577,7 +1561,7 @@ are always available. They are listed here in alphabetical order. ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is equivalent to using the power operator: ``base**exp``. - The arguments must have numeric types. With mixed operand types, the + When arguments are builtin numeric types with mixed operand types, the coercion rules for binary arithmetic operators apply. For :class:`int` operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are @@ -1729,15 +1713,15 @@ are always available. They are listed here in alphabetical order. .. _func-range: -.. class:: range(stop) - range(start, stop, step=1) +.. class:: range(stop, /) + range(start, stop, step=1, /) :noindex: Rather than being a function, :class:`range` is actually an immutable sequence type, as documented in :ref:`typesseq-range` and :ref:`typesseq`. -.. function:: repr(object) +.. function:: repr(object, /) Return a string containing a printable representation of an object. For many types, this function makes an attempt to return a string that would yield an @@ -1761,9 +1745,9 @@ are always available. They are listed here in alphabetical order. return f"Person('{self.name}', {self.age})" -.. function:: reversed(seq) +.. function:: reversed(object, /) - Return a reverse :term:`iterator`. *seq* must be an object which has + Return a reverse :term:`iterator`. The argument must be an object which has a :meth:`~object.__reversed__` method or supports the sequence protocol (the :meth:`~object.__len__` method and the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). @@ -1797,8 +1781,7 @@ are always available. They are listed here in alphabetical order. .. _func-set: -.. class:: set() - set(iterable) +.. class:: set(iterable=(), /) :noindex: Return a new :class:`set` object, optionally with elements taken from @@ -1810,7 +1793,7 @@ are always available. They are listed here in alphabetical order. module. -.. function:: setattr(object, name, value) +.. function:: setattr(object, name, value, /) This is the counterpart of :func:`getattr`. The arguments are an object, a string, and an arbitrary value. The string may name an existing attribute or a @@ -1832,22 +1815,22 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. -.. class:: slice(stop) - slice(start, stop, step=None) +.. class:: slice(stop, /) + slice(start, stop, step=None, /) Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to ``None``. + Slice objects have read-only data attributes :attr:`!start`, + :attr:`!stop`, and :attr:`!step` which merely return the argument + values (or their default). They have no other explicit functionality; + however, they are used by NumPy and other third-party packages. + .. attribute:: slice.start .. attribute:: slice.stop .. attribute:: slice.step - Slice objects have read-only data attributes :attr:`!start`, - :attr:`!stop`, and :attr:`!step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. - Slice objects are also generated when extended indexing syntax is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See :func:`itertools.islice` for an alternate version that returns an @@ -1885,7 +1868,7 @@ are always available. They are listed here in alphabetical order. the same data with other ordering tools such as :func:`max` that rely on a different underlying method. Implementing all six comparisons also helps avoid confusion for mixed type comparisons which can call - reflected the :meth:`~object.__gt__` method. + the reflected :meth:`~object.__gt__` method. For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. @@ -1938,8 +1921,10 @@ are always available. They are listed here in alphabetical order. single: string; str() (built-in function) .. _func-str: -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') +.. class:: str(*, encoding='utf-8', errors='strict') + str(object) + str(object, encoding, errors='strict') + str(object, *, errors) :noindex: Return a :class:`str` version of *object*. See :func:`str` for details. @@ -1972,7 +1957,7 @@ are always available. They are listed here in alphabetical order. .. class:: super() - super(type, object_or_type=None) + super(type, object_or_type=None, /) Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have @@ -2054,16 +2039,15 @@ are always available. They are listed here in alphabetical order. .. _func-tuple: -.. class:: tuple() - tuple(iterable) +.. class:: tuple(iterable=(), /) :noindex: Rather than being a function, :class:`tuple` is actually an immutable sequence type, as documented in :ref:`typesseq-tuple` and :ref:`typesseq`. -.. class:: type(object) - type(name, bases, dict, **kwds) +.. class:: type(object, /) + type(name, bases, dict, /, **kwargs) .. index:: pair: object; type @@ -2106,7 +2090,7 @@ are always available. They are listed here in alphabetical order. longer use the one-argument form to get the type of an object. .. function:: vars() - vars(object) + vars(object, /) Return the :attr:`~object.__dict__` attribute for a module, class, instance, or any other object with a :attr:`!__dict__` attribute. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3a933dff057..221c0712c7c 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -42,11 +42,11 @@ The :mod:`functools` module defines the following functions: def factorial(n): return n * factorial(n-1) if n else 1 - >>> factorial(10) # no previously cached result, makes 11 recursive calls + >>> factorial(10) # no previously cached result, makes 11 recursive calls 3628800 - >>> factorial(5) # just looks up cached value result + >>> factorial(5) # no new calls, just returns the cached result 120 - >>> factorial(12) # makes two new recursive calls, the other 10 are cached + >>> factorial(12) # two new recursive calls, factorial(10) is cached 479001600 The cache is threadsafe so that the wrapped function can be used in @@ -57,6 +57,10 @@ The :mod:`functools` module defines the following functions: another thread makes an additional call before the initial call has been completed and cached. + Call-once behavior is not guaranteed because locks are not held during the + function call. Potentially another call with the same arguments could + occur while the first call is still running. + .. versionadded:: 3.9 @@ -190,7 +194,7 @@ The :mod:`functools` module defines the following functions: Note, type specificity applies only to the function's immediate arguments rather than their contents. The scalar arguments, ``Decimal(42)`` and - ``Fraction(42)`` are be treated as distinct calls with distinct results. + ``Fraction(42)`` are treated as distinct calls with distinct results. In contrast, the tuple arguments ``('answer', Decimal(42))`` and ``('answer', Fraction(42))`` are treated as equivalent. @@ -199,12 +203,18 @@ The :mod:`functools` module defines the following functions: and *typed*. This is for information purposes only. Mutating the values has no effect. + .. method:: lru_cache.cache_info() + :no-typesetting: + To help measure the effectiveness of the cache and tune the *maxsize* - parameter, the wrapped function is instrumented with a :func:`cache_info` + parameter, the wrapped function is instrumented with a :func:`!cache_info` function that returns a :term:`named tuple` showing *hits*, *misses*, *maxsize* and *currsize*. - The decorator also provides a :func:`cache_clear` function for clearing or + .. method:: lru_cache.cache_clear() + :no-typesetting: + + The decorator also provides a :func:`!cache_clear` function for clearing or invalidating the cache. The original underlying function is accessible through the @@ -284,9 +294,9 @@ The :mod:`functools` module defines the following functions: class decorator supplies the rest. This simplifies the effort involved in specifying all of the possible rich comparison operations: - The class must define one of :meth:`__lt__`, :meth:`__le__`, - :meth:`__gt__`, or :meth:`__ge__`. - In addition, the class should supply an :meth:`__eq__` method. + The class must define one of :meth:`~object.__lt__`, :meth:`~object.__le__`, + :meth:`~object.__gt__`, or :meth:`~object.__ge__`. + In addition, the class should supply an :meth:`~object.__eq__` method. For example:: @@ -403,8 +413,7 @@ The :mod:`functools` module defines the following functions: >>> remove_first_dear(message) 'Hello, dear world!' - :data:`!Placeholder` has no special treatment when used in a keyword - argument to :func:`!partial`. + :data:`!Placeholder` cannot be passed to :func:`!partial` as a keyword argument. .. versionchanged:: 3.14 Added support for :data:`Placeholder` in positional arguments. @@ -419,7 +428,7 @@ The :mod:`functools` module defines the following functions: like normal functions, are handled as descriptors). When *func* is a descriptor (such as a normal Python function, - :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or + :func:`classmethod`, :func:`staticmethod`, :func:`~abc.abstractmethod` or another instance of :class:`partialmethod`), calls to ``__get__`` are delegated to the underlying descriptor, and an appropriate :ref:`partial object` returned as the result. @@ -500,7 +509,10 @@ The :mod:`functools` module defines the following functions: ... print("Let me just say,", end=" ") ... print(arg) - To add overloaded implementations to the function, use the :func:`register` + .. method:: singledispatch.register() + :no-typesetting: + + To add overloaded implementations to the function, use the :func:`!register` attribute of the generic function, which can be used as a decorator. For functions annotated with types, the decorator will infer the type of the first argument automatically:: @@ -566,14 +578,14 @@ The :mod:`functools` module defines the following functions: runtime impact. To enable registering :term:`lambdas` and pre-existing functions, - the :func:`register` attribute can also be used in a functional form:: + the :func:`~singledispatch.register` attribute can also be used in a functional form:: >>> def nothing(arg, verbose=False): ... print("Nothing.") ... >>> fun.register(type(None), nothing) - The :func:`register` attribute returns the undecorated function. This + The :func:`~singledispatch.register` attribute returns the undecorated function. This enables decorator stacking, :mod:`pickling`, and the creation of unit tests for each variant independently:: @@ -651,10 +663,10 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 .. versionchanged:: 3.7 - The :func:`register` attribute now supports using type annotations. + The :func:`~singledispatch.register` attribute now supports using type annotations. .. versionchanged:: 3.11 - The :func:`register` attribute now supports + The :func:`~singledispatch.register` attribute now supports :class:`typing.Union` as a type annotation. @@ -664,7 +676,7 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. When defining a function using ``@singledispatchmethod``, note + decorator. When defining a method using ``@singledispatchmethod``, note that the dispatch happens on the type of the first non-*self* or non-*cls* argument:: @@ -682,7 +694,7 @@ The :mod:`functools` module defines the following functions: return not arg ``@singledispatchmethod`` supports nesting with other decorators such as - :func:`@classmethod`. Note that to allow for + :deco:`classmethod`. Note that to allow for ``dispatcher.register``, ``singledispatchmethod`` must be the *outer most* decorator. Here is the ``Negator`` class with the ``neg`` methods bound to the class, rather than an instance of the class:: @@ -704,11 +716,13 @@ The :mod:`functools` module defines the following functions: return not arg The same pattern can be used for other similar decorators: - :func:`@staticmethod`, - :func:`@abstractmethod`, and others. + :deco:`staticmethod`, :deco:`~abc.abstractmethod`, and others. .. versionadded:: 3.8 + .. versionchanged:: 3.15 + Added support of non-:term:`descriptor` callables. + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) @@ -784,7 +798,7 @@ The :mod:`functools` module defines the following functions: 'Docstring' Without the use of this decorator factory, the name of the example function - would have been ``'wrapper'``, and the docstring of the original :func:`example` + would have been ``'wrapper'``, and the docstring of the original :func:`!example` would have been lost. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 7ccb0e6bdf9..79a8c38626f 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -60,7 +60,7 @@ The :mod:`gc` module provides the following functions: The effect of calling ``gc.collect()`` while the interpreter is already performing a collection is undefined. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. @@ -83,13 +83,13 @@ The :mod:`gc` module provides the following functions: returned. If *generation* is not ``None``, return only the objects as follows: * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.13) + * 1: No objects, as there is no generation 1 (as of Python 3.14) * 2: All objects in the old generation .. versionchanged:: 3.8 New *generation* parameter. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Generation 1 is removed .. audit-event:: gc.get_objects generation gc.get_objects @@ -108,10 +108,19 @@ The :mod:`gc` module provides the following functions: * ``uncollectable`` is the total number of objects which were found to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. + list) inside this generation; + + * ``candidates`` is the total number of objects in this generation which were + considered for collection and traversed; + + * ``duration`` is the total time in seconds spent in collections for this + generation. .. versionadded:: 3.4 + .. versionchanged:: next + Add ``duration`` and ``candidates``. + .. function:: set_threshold(threshold0, [threshold1, [threshold2]]) @@ -142,7 +151,7 @@ The :mod:`gc` module provides the following functions: See `Garbage collector design `_ for more information. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 *threshold2* is ignored @@ -313,6 +322,12 @@ values but should not rebind them): "uncollectable": When *phase* is "stop", the number of objects that could not be collected and were put in :data:`garbage`. + "candidates": When *phase* is "stop", the total number of objects in this + generation which were considered for collection and traversed. + + "duration": When *phase* is "stop", the time in seconds spent in the + collection. + Applications can add their own callbacks to this list. The primary use cases are: @@ -325,6 +340,9 @@ values but should not rebind them): .. versionadded:: 3.3 + .. versionchanged:: next + Add "duration" and "candidates". + The following constants are provided for use with :func:`set_debug`: diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 0fb0fc88683..a0c0c6dee2d 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -27,9 +27,9 @@ The :mod:`getpass` module provides two functions: The *echo_char* argument controls how user input is displayed while typing. If *echo_char* is ``None`` (default), input remains hidden. Otherwise, - *echo_char* must be a printable ASCII string and each typed character - is replaced by it. For example, ``echo_char='*'`` will display - asterisks instead of the actual input. + *echo_char* must be a single printable ASCII character and each + typed character is replaced by it. For example, ``echo_char='*'`` will + display asterisks instead of the actual input. If echo free input is unavailable getpass() falls back to printing a warning message to *stream* and reading from ``sys.stdin`` and @@ -39,6 +39,14 @@ The :mod:`getpass` module provides two functions: If you call getpass from within IDLE, the input may be done in the terminal you launched IDLE from rather than the idle window itself. + .. note:: + On Unix systems, when *echo_char* is set, the terminal will be + configured to operate in + :manpage:`noncanonical mode `. + In particular, this means that line editing shortcuts such as + :kbd:`Ctrl+U` will not work and may insert unexpected characters into + the input. + .. versionchanged:: 3.14 Added the *echo_char* parameter for keyboard feedback. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 59ad1b07f27..52c44928153 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -18,23 +18,27 @@ single: - (minus); in glob-style wildcards single: . (dot); in glob-style wildcards -The :mod:`glob` module finds all the pathnames matching a specified pattern -according to the rules used by the Unix shell, although results are returned in -arbitrary order. No tilde expansion is done, but ``*``, ``?``, and character +The :mod:`!glob` module finds pathnames +using pattern matching rules similar to the Unix shell. +No tilde expansion is done, but ``*``, ``?``, and character ranges expressed with ``[]`` will be correctly matched. This is done by using the :func:`os.scandir` and :func:`fnmatch.fnmatch` functions in concert, and not by actually invoking a subshell. -Note that files beginning with a dot (``.``) can only be matched by +.. note:: + The pathnames are returned in no particular order. If you need a specific + order, sort the results. + +Files beginning with a dot (``.``) can only be matched by patterns that also start with a dot, unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. -(For tilde and shell variable expansion, use :func:`os.path.expanduser` and -:func:`os.path.expandvars`.) +For tilde and shell variable expansion, use :func:`os.path.expanduser` and +:func:`os.path.expandvars`. For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. -The :mod:`glob` module defines the following functions: +The :mod:`!glob` module defines the following functions: .. function:: glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ @@ -51,7 +55,7 @@ The :mod:`glob` module defines the following functions: If *root_dir* is not ``None``, it should be a :term:`path-like object` specifying the root directory for searching. It has the same effect on - :func:`glob` as changing the current directory before calling it. If + :func:`!glob` as changing the current directory before calling it. If *pathname* is relative, the result will contain paths relative to *root_dir*. diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index c9d96085ef7..d23c0741ddb 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -11,6 +11,8 @@ This module provides a simple interface to compress and decompress files just like the GNU programs :program:`gzip` and :program:`gunzip` would. +.. include:: ../includes/optional-module.rst + The data compression is provided by the :mod:`zlib` module. The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the @@ -26,7 +28,7 @@ Note that additional file formats which can be decompressed by the The module defines the following items: -.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None) Open a gzip-compressed file in binary or text mode, returning a :term:`file object`. @@ -59,6 +61,11 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + The default compression level was reduced to 6 (down from 9). + It is the default level used by most compression tools and a better + tradeoff between speed and performance. + .. exception:: BadGzipFile An exception raised for invalid gzip files. It inherits from :exc:`OSError`. @@ -67,7 +74,7 @@ The module defines the following items: .. versionadded:: 3.8 -.. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) +.. class:: GzipFile(filename=None, mode=None, compresslevel=6, fileobj=None, mtime=None) Constructor for the :class:`GzipFile` class, which simulates most of the methods of a :term:`file object`, with the exception of the :meth:`~io.IOBase.truncate` @@ -181,8 +188,13 @@ The module defines the following items: Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` attribute instead. + .. versionchanged:: 3.15 + The default compression level was reduced to 6 (down from 9). + It is the default level used by most compression tools and a better + tradeoff between speed and performance. -.. function:: compress(data, compresslevel=9, *, mtime=0) + +.. function:: compress(data, compresslevel=6, *, mtime=0) Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in @@ -206,6 +218,10 @@ The module defines the following items: The *mtime* parameter now defaults to 0 for reproducible output. For the previous behaviour of using the current time, pass ``None`` to *mtime*. + .. versionchanged:: 3.15 + The default compression level was reduced to 6 (down from 9). + It is the default level used by most compression tools and a better + tradeoff between speed and performance. .. function:: decompress(data) @@ -267,7 +283,7 @@ Example of how to GZIP compress a binary string:: .. _gzip-cli: -Command Line Interface +Command-line interface ---------------------- The :mod:`gzip` module provides a simple command line interface to compress or @@ -280,7 +296,7 @@ Once executed the :mod:`gzip` module keeps the input file(s). Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: file diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index bb2d2fad23b..b21ecdaede6 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -94,6 +94,13 @@ accessible by name via :func:`new`. See :data:`algorithms_available`. OpenSSL does not provide we fall back to a verified implementation from the `HACL\* project`_. +.. deprecated-removed:: 3.15 3.19 + The undocumented ``string`` keyword parameter in :func:`!_hashlib.new` + and hash-named constructors such as :func:`!_md5.md5` is deprecated. + Prefer passing the initial data as a positional argument for maximum + backwards compatibility. + + Usage ----- @@ -284,7 +291,7 @@ a file or file-like object. Example: >>> import io, hashlib, hmac - >>> with open(hashlib.__file__, "rb") as f: + >>> with open("library/hashlib.rst", "rb") as f: ... digest = hashlib.file_digest(f, "sha256") ... >>> digest.hexdigest() # doctest: +ELLIPSIS @@ -303,7 +310,7 @@ a file or file-like object. .. versionadded:: 3.11 .. versionchanged:: 3.14 - Now raises a :exc:`BlockingIOError` if the file is opened in blocking + Now raises a :exc:`BlockingIOError` if the file is opened in non-blocking mode. Previously, spurious null bytes were added to the digest. diff --git a/Doc/library/heapq-binary-tree.svg b/Doc/library/heapq-binary-tree.svg new file mode 100644 index 00000000000..074a9a44275 --- /dev/null +++ b/Doc/library/heapq-binary-tree.svg @@ -0,0 +1,211 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 922ba0c8aa4..5049262306a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -58,6 +58,11 @@ functions, respectively. The following functions are provided for min-heaps: +.. function:: heapify(x) + + Transform list *x* into a min-heap, in-place, in linear time. + + .. function:: heappush(heap, item) Push the value *item* onto the *heap*, maintaining the min-heap invariant. @@ -77,11 +82,6 @@ The following functions are provided for min-heaps: followed by a separate call to :func:`heappop`. -.. function:: heapify(x) - - Transform list *x* into a min-heap, in-place, in linear time. - - .. function:: heapreplace(heap, item) Pop and return the smallest item from the *heap*, and also push the new *item*. @@ -231,6 +231,42 @@ Heap elements can be tuples. This is useful for assigning comparison values (1, 'write spec') +Other Applications +------------------ + +`Medians `_ are a measure of +central tendency for a set of numbers. In distributions skewed by +outliers, the median provides a more stable estimate than an average +(arithmetic mean). A running median is an `online algorithm +`_ that updates +continuously as new data arrives. + +A running median can be efficiently implemented by balancing two heaps, +a max-heap for values at or below the midpoint and a min-heap for values +above the midpoint. When the two heaps have the same size, the new +median is the average of the tops of the two heaps; otherwise, the +median is at the top of the larger heap:: + + def running_median(iterable): + "Yields the cumulative median of values seen so far." + + lo = [] # max-heap + hi = [] # min-heap (same size as or one smaller than lo) + + for x in iterable: + if len(lo) == len(hi): + heappush_max(lo, heappushpop(hi, x)) + yield lo[0] + else: + heappush(hi, heappushpop_max(lo, x)) + yield (lo[0] + hi[0]) / 2 + +For example:: + + >>> list(running_median([5.0, 9.0, 4.0, 12.0, 8.0, 9.0])) + [5.0, 7.0, 5.0, 7.0, 8.0, 8.5] + + Priority Queue Implementation Notes ----------------------------------- @@ -312,17 +348,12 @@ elements are considered to be infinite. The interesting property of a heap is that ``a[0]`` is always its smallest element. The strange invariant above is meant to be an efficient memory representation -for a tournament. The numbers below are *k*, not ``a[k]``:: +for a tournament. The numbers below are *k*, not ``a[k]``: - 0 - - 1 2 - - 3 4 5 6 - - 7 8 9 10 11 12 13 14 - - 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 +.. figure:: heapq-binary-tree.svg + :class: invert-in-dark-mode + :align: center + :alt: Example (min-heap) binary tree. In the tree above, each cell *k* is topping ``2*k+1`` and ``2*k+2``. In a usual binary tournament we see in sports, each cell is the winner over the two cells diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index d6692033b2d..d5608bd7543 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -12,6 +12,9 @@ -------------- This module implements the HMAC algorithm as described by :rfc:`2104`. +The interface allows to use any hash function with a *fixed* digest size. +In particular, extendable output functions such as SHAKE-128 or SHAKE-256 +cannot be used with HMAC. .. function:: new(key, msg=None, digestmod) @@ -47,7 +50,9 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. .. versionadded:: 3.7 -An HMAC object has the following methods: +.. class:: HMAC + + An HMAC object has the following methods: .. method:: HMAC.update(msg) diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 6d433b5a04f..341a8337ba2 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -15,14 +15,18 @@ This module defines a class :class:`HTMLParser` which serves as the basis for parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. -.. class:: HTMLParser(*, convert_charrefs=True) +.. class:: HTMLParser(*, convert_charrefs=True, scripting=False) Create a parser instance able to parse invalid markup. - If *convert_charrefs* is ``True`` (the default), all character - references (except the ones in ``script``/``style`` elements) are + If *convert_charrefs* is true (the default), all character + references (except the ones in elements like ``script`` and ``style``) are automatically converted to the corresponding Unicode characters. + If *scripting* is false (the default), the content of the ``noscript`` + element is parsed normally; if it's true, it's returned as is without + being parsed. + An :class:`.HTMLParser` instance is fed HTML data and calls handler methods when start tags, end tags, text, comments, and other markup elements are encountered. The user should subclass :class:`.HTMLParser` and override its @@ -37,13 +41,18 @@ parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. .. versionchanged:: 3.5 The default value for argument *convert_charrefs* is now ``True``. + .. versionchanged:: 3.14.1 + Added the *scripting* parameter. + Example HTML Parser Application ------------------------------- As a basic example, below is a simple HTML parser that uses the :class:`HTMLParser` class to print out start tags, end tags, and data -as they are encountered:: +as they are encountered: + +.. testcode:: from html.parser import HTMLParser @@ -63,7 +72,7 @@ as they are encountered:: The output will then be: -.. code-block:: none +.. testoutput:: Encountered a start tag: html Encountered a start tag: head @@ -159,15 +168,15 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): .. method:: HTMLParser.handle_data(data) This method is called to process arbitrary data (e.g. text nodes and the - content of ```` and ````). + content of elements like ``script`` and ``style``). .. method:: HTMLParser.handle_entityref(name) This method is called to process a named character reference of the form ``&name;`` (e.g. ``>``), where *name* is a general entity reference - (e.g. ``'gt'``). This method is never called if *convert_charrefs* is - ``True``. + (e.g. ``'gt'``). + This method is only called if *convert_charrefs* is false. .. method:: HTMLParser.handle_charref(name) @@ -175,8 +184,8 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): This method is called to process decimal and hexadecimal numeric character references of the form :samp:`&#{NNN};` and :samp:`&#x{NNN};`. For example, the decimal equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; - in this case the method will receive ``'62'`` or ``'x3E'``. This method - is never called if *convert_charrefs* is ``True``. + in this case the method will receive ``'62'`` or ``'x3E'``. + This method is only called if *convert_charrefs* is false. .. method:: HTMLParser.handle_comment(data) @@ -230,7 +239,9 @@ Examples -------- The following class implements a parser that will be used to illustrate more -examples:: +examples: + +.. testcode:: from html.parser import HTMLParser from html.entities import name2codepoint @@ -266,13 +277,17 @@ examples:: parser = MyHTMLParser() -Parsing a doctype:: +Parsing a doctype: + +.. doctest:: >>> parser.feed('') Decl : DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd" -Parsing an element with a few attributes and a title:: +Parsing an element with a few attributes and a title: + +.. doctest:: >>> parser.feed('The Python logo') Start tag: img @@ -284,8 +299,10 @@ Parsing an element with a few attributes and a title:: Data : Python End tag : h1 -The content of ``script`` and ``style`` elements is returned as is, without -further parsing:: +The content of elements like ``script`` and ``style`` is returned as is, +without further parsing: + +.. doctest:: >>> parser.feed('') Start tag: style @@ -294,22 +311,31 @@ further parsing:: End tag : style >>> parser.feed('') + ... 'alert("hello! ☺");') Start tag: script attr: ('type', 'text/javascript') - Data : alert("hello!"); + Data : alert("hello! ☺"); End tag : script -Parsing comments:: +Parsing comments: - >>> parser.feed('' +.. doctest:: + + >>> parser.feed('' ... '') - Comment : a comment + Comment : a comment Comment : [if IE 9]>IE-specific content'``):: +correct char (note: these 3 references are all equivalent to ``'>'``): +.. doctest:: + + >>> parser = MyHTMLParser() + >>> parser.feed('>>>') + Data : >>> + + >>> parser = MyHTMLParser(convert_charrefs=False) >>> parser.feed('>>>') Named ent: > Num ent : > @@ -317,18 +343,22 @@ correct char (note: these 3 references are all equivalent to ``'>'``):: Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but :meth:`~HTMLParser.handle_data` might be called more than once -(unless *convert_charrefs* is set to ``True``):: +if *convert_charrefs* is false: - >>> for chunk in ['buff', 'ered ', 'text']: +.. doctest:: + + >>> for chunk in ['buff', 'ered', ' text']: ... parser.feed(chunk) ... Start tag: span Data : buff Data : ered - Data : text + Data : text End tag : span -Parsing invalid HTML (e.g. unquoted attributes) also works:: +Parsing invalid HTML (e.g. unquoted attributes) also works: + +.. doctest:: >>> parser.feed('

tag soup

') Start tag: p diff --git a/Doc/library/html.rst b/Doc/library/html.rst index 9aa39ba9a42..65c49a4107a 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -14,9 +14,12 @@ This module defines utilities to manipulate HTML. Convert the characters ``&``, ``<`` and ``>`` in string *s* to HTML-safe sequences. Use this if you need to display text that might contain such - characters in HTML. If the optional flag *quote* is true, the characters - (``"``) and (``'``) are also translated; this helps for inclusion in an HTML - attribute value delimited by quotes, as in ````. + characters in HTML. If the optional flag *quote* is true (the default), the + characters (``"``) and (``'``) are also translated; this helps for inclusion + in an HTML attribute value delimited by quotes, as in ````. + If *quote* is set to false, the characters (``"``) and (``'``) are not + translated. + .. versionadded:: 3.2 diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 2835c8d0eb7..7c258b324d9 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -34,7 +34,7 @@ The module provides the following classes: .. class:: HTTPConnection(host, port=None[, timeout], source_address=None, \ - blocksize=8192) + blocksize=8192, max_response_headers=None) An :class:`HTTPConnection` instance represents one transaction with an HTTP server. It should be instantiated by passing it a host and optional port @@ -46,7 +46,9 @@ The module provides the following classes: The optional *source_address* parameter may be a tuple of a (host, port) to use as the source address the HTTP connection is made from. The optional *blocksize* parameter sets the buffer size in bytes for - sending a file-like message body. + sending a file-like message body. The optional *max_response_headers* + parameter sets the maximum number of allowed response headers to help + prevent denial-of-service attacks, otherwise the default value (100) is used. For example, the following calls all create instances that connect to the server at the same host and port:: @@ -66,10 +68,13 @@ The module provides the following classes: .. versionchanged:: 3.7 *blocksize* parameter was added. + .. versionchanged:: 3.15 + *max_response_headers* parameter was added. + .. class:: HTTPSConnection(host, port=None, *[, timeout], \ source_address=None, context=None, \ - blocksize=8192) + blocksize=8192, max_response_headers=None) A subclass of :class:`HTTPConnection` that uses SSL for communication with secure servers. Default port is ``443``. If *context* is specified, it @@ -109,6 +114,9 @@ The module provides the following classes: The deprecated *key_file*, *cert_file* and *check_hostname* parameters have been removed. + .. versionchanged:: 3.15 + *max_response_headers* parameter was added. + .. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) @@ -125,7 +133,7 @@ This module provides the following function: Parse the headers from a file pointer *fp* representing a HTTP request/response. The file has to be a :class:`~io.BufferedIOBase` reader - (i.e. not text) and must provide a valid :rfc:`2822` style header. + (i.e. not text) and must provide a valid :rfc:`5322` style header. This function returns an instance of :class:`http.client.HTTPMessage` that holds the header fields, but no payload @@ -416,6 +424,14 @@ HTTPConnection Objects .. versionadded:: 3.7 +.. attribute:: HTTPConnection.max_response_headers + + The maximum number of allowed response headers to help prevent denial-of-service + attacks. By default, the maximum number of allowed headers is set to 100. + + .. versionadded:: 3.15 + + As an alternative to using the :meth:`~HTTPConnection.request` method described above, you can also send your request step by step, by using the four functions below. diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 23ddecf8738..fcb0069b760 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -12,7 +12,7 @@ -------------- The :mod:`http.cookiejar` module defines classes for automatic handling of HTTP -cookies. It is useful for accessing web sites that require small pieces of data +cookies. It is useful for accessing websites that require small pieces of data -- :dfn:`cookies` -- to be set on the client machine by an HTTP response from a web server, and then returned to the server in later HTTP requests. @@ -570,7 +570,7 @@ Netscape protocol strictness switches: Don't allow setting cookies whose path doesn't path-match request URI. -:attr:`strict_ns_domain` is a collection of flags. Its value is constructed by +:attr:`~DefaultCookiePolicy.strict_ns_domain` is a collection of flags. Its value is constructed by or-ing together (for example, ``DomainStrictNoDots|DomainStrictNonDomain`` means both flags are set). diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index eb196320721..88e978d7f5e 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -28,8 +28,10 @@ The character set, :data:`string.ascii_letters`, :data:`string.digits` and in a cookie name (as :attr:`~Morsel.key`). .. versionchanged:: 3.3 - Allowed ':' as a valid cookie name character. + Allowed '``:``' as a valid cookie name character. +.. versionchanged:: 3.15 + Allowed '``"``' as a valid cookie value character. .. note:: @@ -148,9 +150,12 @@ Morsel Objects in HTTP requests, and is not accessible through JavaScript. This is intended to mitigate some forms of cross-site scripting. - The attribute :attr:`samesite` specifies that the browser is not allowed to - send the cookie along with cross-site requests. This helps to mitigate CSRF - attacks. Valid values for this attribute are "Strict" and "Lax". + The attribute :attr:`samesite` controls when the browser sends the cookie with + cross-site requests. This helps to mitigate CSRF attacks. Valid values are + "Strict" (only sent with same-site requests), "Lax" (sent with same-site + requests and top-level navigations), and "None" (sent with same-site and + cross-site requests). When using "None", the "secure" attribute must also + be set, as required by modern browsers. The attribute :attr:`partitioned` indicates to user agents that these cross-site cookies *should* only be available in the same top-level context @@ -311,3 +316,10 @@ The following example demonstrates how to use the :mod:`http.cookies` module. >>> print(C) Set-Cookie: number=7 Set-Cookie: string=seven + >>> import json + >>> C = cookies.SimpleCookie() + >>> C.load(f'cookies=7; mixins="{json.dumps({"chips": "dark chocolate"})}"; state=gooey') + >>> print(C) + Set-Cookie: cookies=7 + Set-Cookie: mixins="{"chips": "dark chocolate"}" + Set-Cookie: state=gooey diff --git a/Doc/library/http.rst b/Doc/library/http.rst index ce3fb9f8120..b0bdfc65e45 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -139,7 +139,8 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as .. versionchanged:: 3.13 Implemented RFC9110 naming for status constants. Old constant names are preserved for - backwards compatibility. + backwards compatibility: ``413 REQUEST_ENTITY_TOO_LARGE``, ``414 REQUEST_URI_TOO_LONG``, + ``416 REQUESTED_RANGE_NOT_SATISFIABLE`` and ``422 UNPROCESSABLE_ENTITY``. HTTP status category -------------------- diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 54df4a7e804..58f09634f95 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -154,7 +154,7 @@ instantiation, of which this module provides three different variants: variable. This instance parses and manages the headers in the HTTP request. The :func:`~http.client.parse_headers` function from :mod:`http.client` is used to parse the headers and it requires that the - HTTP request provide a valid :rfc:`2822` style header. + HTTP request provide a valid :rfc:`5322` style header. .. attribute:: rfile @@ -429,8 +429,7 @@ instantiation, of which this module provides three different variants: ``'Last-Modified:'`` header with the file's modification time. Then follows a blank line signifying the end of the headers, and then the - contents of the file are output. If the file's MIME type starts with - ``text/`` the file is opened in text mode; otherwise binary mode is used. + contents of the file are output. For example usage, see the implementation of the ``test`` function in :source:`Lib/http/server.py`. @@ -459,55 +458,6 @@ such as using different index file names by overriding the class attribute :attr:`index_pages`. -.. class:: CGIHTTPRequestHandler(request, client_address, server) - - This class is used to serve either files or output of CGI scripts from the - current directory and below. Note that mapping HTTP hierarchic structure to - local directory structure is exactly as in :class:`SimpleHTTPRequestHandler`. - - .. note:: - - CGI scripts run by the :class:`CGIHTTPRequestHandler` class cannot execute - redirects (HTTP code 302), because code 200 (script output follows) is - sent prior to execution of the CGI script. This pre-empts the status - code. - - The class will however, run the CGI script, instead of serving it as a file, - if it guesses it to be a CGI script. Only directory-based CGI are used --- - the other common server configuration is to treat special extensions as - denoting CGI scripts. - - The :func:`do_GET` and :func:`do_HEAD` functions are modified to run CGI scripts - and serve the output, instead of serving files, if the request leads to - somewhere below the ``cgi_directories`` path. - - The :class:`CGIHTTPRequestHandler` defines the following data member: - - .. attribute:: cgi_directories - - This defaults to ``['/cgi-bin', '/htbin']`` and describes directories to - treat as containing CGI scripts. - - The :class:`CGIHTTPRequestHandler` defines the following method: - - .. method:: do_POST() - - This method serves the ``'POST'`` request type, only allowed for CGI - scripts. Error 501, "Can only POST to CGI scripts", is output when trying - to POST to a non-CGI url. - - Note that CGI scripts will be run with UID of user nobody, for security - reasons. Problems with the CGI script will be translated to error 403. - - .. deprecated-removed:: 3.13 3.15 - - :class:`CGIHTTPRequestHandler` is being removed in 3.15. CGI has not - been considered a good way to do things for well over a decade. This code - has been unmaintained for a while now and sees very little practical use. - Retaining it could lead to further :ref:`security considerations - `. - - .. _http-server-cli: Command-line interface @@ -564,24 +514,6 @@ The following options are accepted: .. versionadded:: 3.11 -.. option:: --cgi - - :class:`CGIHTTPRequestHandler` can be enabled in the command line by passing - the ``--cgi`` option:: - - python -m http.server --cgi - - .. deprecated-removed:: 3.13 3.15 - - :mod:`http.server` command line ``--cgi`` support is being removed - because :class:`CGIHTTPRequestHandler` is being removed. - -.. warning:: - - :class:`CGIHTTPRequestHandler` and the ``--cgi`` command-line option - are not intended for use by untrusted clients and may be vulnerable - to exploitation. Always use within a secure environment. - .. option:: --tls-cert Specifies a TLS certificate chain for HTTPS connections:: diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index fabea611e0e..a16f46ef812 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -13,7 +13,7 @@ IDLE --- Python editor and shell single: Integrated Development Environment .. - Remember to update Lib/idlelib/help.html with idlelib.help.copy_source() when modifying this file. + Remember to update Lib/idlelib/help.html with idlelib.help.copy_strip() when modifying this file. -------------- @@ -37,6 +37,10 @@ IDLE has the following features: * configuration, browsers, and other dialogs +The IDLE application is implemented in the :mod:`idlelib` package. + +.. include:: ../includes/optional-module.rst + Menus ----- @@ -88,7 +92,7 @@ Save Save As... Save the current window with a Save As dialog. The file saved becomes the - new associated file for the window. (If your file namager is set to hide + new associated file for the window. (If your file manager is set to hide extensions, the current extension will be omitted in the file name box. If the new filename has no '.', '.py' and '.txt' will be added for Python and text files, except that on macOS Aqua,'.py' is added for all files.) @@ -204,9 +208,9 @@ New Indent Width Open a dialog to change indent width. The accepted default by the Python community is 4 spaces. -Strip Trailing Chitespace +Strip Trailing Whitespace Remove trailing space and other whitespace characters after the last - non-whitespace character of a line by applying str.rstrip to each line, + non-whitespace character of a line by applying :meth:`str.rstrip` to each line, including lines within multiline strings. Except for Shell windows, remove extra newlines at the end of the file. @@ -657,7 +661,9 @@ looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE's Python shell. -Command line usage +.. _idlelib-cli: + +Command-line usage ^^^^^^^^^^^^^^^^^^ .. program:: idle diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 9f198aebcb6..0b0537d3bbd 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -413,6 +413,9 @@ An :class:`IMAP4` instance has the following methods: the password. Will only work if the server ``CAPABILITY`` response includes the phrase ``AUTH=CRAM-MD5``. + .. versionchanged:: 3.15 + An :exc:`IMAP4.error` is raised if MD5 support is not available. + .. method:: IMAP4.logout() diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 7a77466bcba..8253a33f591 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -49,44 +49,44 @@ .. method:: open_resource(resource) :abstractmethod: - Returns an opened, :term:`file-like object` for binary reading - of the *resource*. + Returns an opened, :term:`file-like object` for binary reading + of the *resource*. - If the resource cannot be found, :exc:`FileNotFoundError` is - raised. + If the resource cannot be found, :exc:`FileNotFoundError` is + raised. .. method:: resource_path(resource) :abstractmethod: - Returns the file system path to the *resource*. + Returns the file system path to the *resource*. - If the resource does not concretely exist on the file system, - raise :exc:`FileNotFoundError`. + If the resource does not concretely exist on the file system, + raise :exc:`FileNotFoundError`. .. method:: is_resource(name) :abstractmethod: - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. + Returns ``True`` if the named *name* is considered a resource. + :exc:`FileNotFoundError` is raised if *name* does not exist. .. method:: contents() :abstractmethod: - Returns an :term:`iterable` of strings over the contents of - the package. Do note that it is not required that all names - returned by the iterator be actual resources, e.g. it is - acceptable to return names for which :meth:`is_resource` would - be false. + Returns an :term:`iterable` of strings over the contents of + the package. Do note that it is not required that all names + returned by the iterator be actual resources, e.g. it is + acceptable to return names for which :meth:`is_resource` would + be false. - Allowing non-resource names to be returned is to allow for - situations where how a package and its resources are stored - are known a priori and the non-resource names would be useful. - For instance, returning subdirectory names is allowed so that - when it is known that the package and resources are stored on - the file system then those subdirectory names can be used - directly. + Allowing non-resource names to be returned is to allow for + situations where how a package and its resources are stored + are known a priori and the non-resource names would be useful. + For instance, returning subdirectory names is allowed so that + when it is known that the package and resources are stored on + the file system then those subdirectory names can be used + directly. - The abstract method returns an iterable of no items. + The abstract method returns an iterable of no items. .. class:: Traversable diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index e002198899c..20297f9fe30 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -16,11 +16,12 @@ within *packages*. "Resources" are file-like resources associated with a module or package in Python. The resources may be contained directly in a package, within a subdirectory contained in that package, or adjacent to modules outside a -package. Resources may be text or binary. As a result, Python module sources -(.py) of a package and compilation artifacts (pycache) are technically -de-facto resources of that package. In practice, however, resources are -primarily those non-Python artifacts exposed specifically by the package -author. +package. Resources may be text or binary. As a result, a package's Python +module sources (.py), compilation artifacts (pycache), and installation +artifacts (like :func:`reserved filenames ` +in directories) are technically de-facto resources of that package. +In practice, however, resources are primarily those non-Python artifacts +exposed specifically by the package author. Resources can be opened or read in either binary or text mode. @@ -72,12 +73,15 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 .. versionchanged:: 3.12 - *package* parameter was renamed to *anchor*. *anchor* can now - be a non-package module and if omitted will default to the caller's - module. *package* is still accepted for compatibility but will raise - a :exc:`DeprecationWarning`. Consider passing the anchor positionally or - using ``importlib_resources >= 5.10`` for a compatible interface - on older Pythons. + *package* parameter was renamed to *anchor*. + *package* was still accepted, but deprecated. + + .. versionchanged:: 3.15 + *package* parameter was fully removed. *anchor* can now be a + non-package module and if omitted will default to the caller's module. + *package* is no longer accepted since Python 3.15. Consider passing the + anchor positionally or using ``importlib_resources >= 5.10`` for a + compatible interface on older Pythons. .. function:: as_file(traversable) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index ea5a7702868..3f0a54ac535 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -124,6 +124,10 @@ Functions need to call :func:`invalidate_caches` in order for the new module to be noticed by the import system. + If the module cannot be imported, :func:`import_module` will raise + :exc:`ImportError` or an appropriate subclass like + :exc:`ModuleNotFoundError`. + .. versionchanged:: 3.3 Parent packages are automatically imported. @@ -206,6 +210,10 @@ Functions :exc:`ModuleNotFoundError` is raised when the module being reloaded lacks a :class:`~importlib.machinery.ModuleSpec`. + .. warning:: + This function is not thread-safe. Calling it from multiple threads can result + in unexpected behavior. It's recommended to use the :class:`threading.Lock` + or other synchronization primitives for thread-safe module reloading. :mod:`importlib.abc` -- Abstract base classes related to import --------------------------------------------------------------- @@ -389,6 +397,8 @@ ABC hierarchy:: .. deprecated:: 3.7 This ABC is deprecated in favour of supporting resource loading through :class:`importlib.resources.abc.TraversableResources`. + This class exists for backwards compatibility only with other ABCs in + this module. .. method:: get_data(path) :abstractmethod: @@ -449,7 +459,7 @@ ABC hierarchy:: .. versionchanged:: 3.4 Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. - .. staticmethod:: source_to_code(data, path='') + .. staticmethod:: source_to_code(data, path='', fullname=None) Create a code object from Python source. @@ -461,11 +471,19 @@ ABC hierarchy:: With the subsequent code object one can execute it in a module by running ``exec(code, module.__dict__)``. + The optional argument *fullname* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax + warnings by module name. + .. versionadded:: 3.4 .. versionchanged:: 3.5 Made the method static. + .. versionadded:: 3.15 + Added the *fullname* parameter. + + .. method:: exec_module(module) Implementation of :meth:`Loader.exec_module`. @@ -1003,6 +1021,36 @@ find and load modules. :exc:`ImportError` is raised. +.. class:: NamespacePath(name, path, path_finder) + + Represents a :term:`namespace package`'s path (:attr:`module.__path__`). + + When its ``__path__`` value is accessed it will be recomputed if necessary. + This keeps it in-sync with the global state (:attr:`sys.modules`). + + The *name* argument is the name of the namespace module. + + The *path* argument is the initial path value. + + The *path_finder* argument is the callable used to recompute the path value. + The callable has the same signature as :meth:`importlib.abc.MetaPathFinder.find_spec`. + + When the parent's :attr:`module.__path__` attribute is updated, the path + value is recomputed. + + If the parent module is missing from :data:`sys.modules`, then + :exc:`ModuleNotFoundError` will be raised. + + For top-level modules, the parent module's path is :data:`sys.path`. + + .. note:: + + :meth:`PathFinder.invalidate_caches` invalidates :class:`NamespacePath`, + forcing the path value to be recomputed next time it is accessed. + + .. versionadded:: 3.15 + + .. class:: SourceFileLoader(fullname, path) A concrete implementation of :class:`importlib.abc.SourceLoader` by @@ -1247,7 +1295,7 @@ find and load modules. To accommodate this requirement, when running on iOS, extension module binaries are *not* packaged as ``.so`` files on ``sys.path``, but as individual standalone frameworks. To discover those frameworks, this loader - is be registered against the ``.fwork`` file extension, with a ``.fwork`` + is registered against the ``.fwork`` file extension, with a ``.fwork`` file acting as a placeholder in the original location of the binary on ``sys.path``. The ``.fwork`` file contains the path of the actual binary in the ``Frameworks`` folder, relative to the app bundle. To allow for diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 44b218948d0..163e1679c65 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -63,7 +63,6 @@ the `Python Package Index `_. internet.rst mm.rst i18n.rst - frameworks.rst tk.rst development.rst debug.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index e8d1176f477..5220c559d3d 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -253,6 +253,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | gi_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | gi_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | gi_code | code | +-----------------+-------------------+---------------------------+ | | gi_yieldfrom | object being iterated by | @@ -616,17 +619,29 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Retrieving source code ---------------------- -.. function:: getdoc(object) +.. function:: getdoc(object, *, inherit_class_doc=True, fallback_to_class_doc=True) Get the documentation string for an object, cleaned up with :func:`cleandoc`. - If the documentation string for an object is not provided and the object is - a class, a method, a property or a descriptor, retrieve the documentation - string from the inheritance hierarchy. + If the documentation string for an object is not provided: + + * if the object is a class and *inherit_class_doc* is true (by default), + retrieve the documentation string from the inheritance hierarchy; + * if the object is a method, a property or a descriptor, retrieve + the documentation string from the inheritance hierarchy; + * otherwise, if *fallback_to_class_doc* is true (by default), retrieve + the documentation string from the class of the object. + Return ``None`` if the documentation string is invalid or missing. .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. + .. versionchanged:: 3.15 + Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + + Documentation strings on :class:`~functools.cached_property` + objects are now inherited if not overriden. + .. function:: getcomments(object) @@ -1179,7 +1194,7 @@ Classes and functions :func:`signature` in Python 3.5, but that decision has been reversed in order to restore a clearly supported standard interface for single-source Python 2/3 code migrating away from the legacy - :func:`getargspec` API. + :func:`!getargspec` API. .. versionchanged:: 3.7 Python only explicitly guaranteed that it preserved the declaration @@ -1289,6 +1304,11 @@ Classes and functions This is an alias for :func:`annotationlib.get_annotations`; see the documentation of that function for more information. + .. caution:: + + This function may execute arbitrary code contained in annotations. + See :ref:`annotationlib-security` for more information. + .. versionadded:: 3.10 .. versionchanged:: 3.14 @@ -1768,7 +1788,7 @@ Buffer flags .. _inspect-module-cli: -Command Line Interface +Command-line interface ---------------------- The :mod:`inspect` module also provides a basic introspection capability diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 3aa2f35f05e..dfebccb5a9c 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -528,14 +528,13 @@ I/O Base Classes It inherits from :class:`IOBase`. The main difference with :class:`RawIOBase` is that methods :meth:`read`, - :meth:`readinto` and :meth:`write` will try (respectively) to read as much - input as requested or to consume all given output, at the expense of - making perhaps more than one system call. + :meth:`readinto` and :meth:`write` will try (respectively) to read + as much input as requested or to emit all provided data. - In addition, those methods can raise :exc:`BlockingIOError` if the - underlying raw stream is in non-blocking mode and cannot take or give - enough data; unlike their :class:`RawIOBase` counterparts, they will - never return ``None``. + In addition, if the underlying raw stream is in non-blocking mode, when the + system returns would block :meth:`write` will raise :exc:`BlockingIOError` + with :attr:`BlockingIOError.characters_written` and :meth:`read` will return + data read so far or ``None`` if no data is available. Besides, the :meth:`read` method does not have a default implementation that defers to :meth:`readinto`. @@ -568,29 +567,40 @@ I/O Base Classes .. method:: read(size=-1, /) - Read and return up to *size* bytes. If the argument is omitted, ``None``, - or negative, data is read and returned until EOF is reached. An empty - :class:`bytes` object is returned if the stream is already at EOF. + Read and return up to *size* bytes. If the argument is omitted, ``None``, + or negative read as much as possible. - If the argument is positive, and the underlying raw stream is not - interactive, multiple raw reads may be issued to satisfy the byte count - (unless EOF is reached first). But for interactive raw streams, at most - one raw read will be issued, and a short result does not imply that EOF is - imminent. + Fewer bytes may be returned than requested. An empty :class:`bytes` object + is returned if the stream is already at EOF. More than one read may be + made and calls may be retried if specific errors are encountered, see + :meth:`os.read` and :pep:`475` for more details. Less than size bytes + being returned does not imply that EOF is imminent. - A :exc:`BlockingIOError` is raised if the underlying raw stream is in - non blocking-mode, and has no data available at the moment. + When reading as much as possible the default implementation will use + ``raw.readall`` if available (which should implement + :meth:`RawIOBase.readall`), otherwise will read in a loop until read + returns ``None``, an empty :class:`bytes`, or a non-retryable error. For + most streams this is to EOF, but for non-blocking streams more data may + become available. + + .. note:: + + When the underlying raw stream is non-blocking, implementations may + either raise :exc:`BlockingIOError` or return ``None`` if no data is + available. :mod:`io` implementations return ``None``. .. method:: read1(size=-1, /) - Read and return up to *size* bytes, with at most one call to the - underlying raw stream's :meth:`~RawIOBase.read` (or - :meth:`~RawIOBase.readinto`) method. This can be useful if you are - implementing your own buffering on top of a :class:`BufferedIOBase` - object. + Read and return up to *size* bytes, calling :meth:`~RawIOBase.readinto` + which may retry if :py:const:`~errno.EINTR` is encountered per + :pep:`475`. If *size* is ``-1`` or not provided, the implementation will + choose an arbitrary value for *size*. - If *size* is ``-1`` (the default), an arbitrary number of bytes are - returned (more than zero unless EOF is reached). + .. note:: + + When the underlying raw stream is non-blocking, implementations may + either raise :exc:`BlockingIOError` or return ``None`` if no data is + available. :mod:`io` implementations return ``None``. .. method:: readinto(b, /) @@ -709,6 +719,9 @@ than raw I/O does. The optional argument *initial_bytes* is a :term:`bytes-like object` that contains initial data. + Methods may be used from multiple threads without external locking in + :term:`free threading` builds. + :class:`BytesIO` provides or overrides these methods in addition to those from :class:`BufferedIOBase` and :class:`IOBase`: @@ -767,34 +780,21 @@ than raw I/O does. .. method:: peek(size=0, /) - Return bytes from the stream without advancing the position. At most one - single read on the raw stream is done to satisfy the call. The number of - bytes returned may be less or more than requested. + Return bytes from the stream without advancing the position. The number of + bytes returned may be less or more than requested. If the underlying raw + stream is non-blocking and the operation would block, returns empty bytes. .. method:: read(size=-1, /) - Read and return *size* bytes, or if *size* is not given or negative, until - EOF or if the read call would block in non-blocking mode. - - .. note:: - - When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` - may be raised if a read operation cannot be completed immediately. + In :class:`BufferedReader` this is the same as :meth:`io.BufferedIOBase.read` .. method:: read1(size=-1, /) - Read and return up to *size* bytes with only one call on the raw stream. - If at least one byte is buffered, only buffered bytes are returned. - Otherwise, one raw stream read call is made. + In :class:`BufferedReader` this is the same as :meth:`io.BufferedIOBase.read1` .. versionchanged:: 3.7 The *size* argument is now optional. - .. note:: - - When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` - may be raised if a read operation cannot be completed immediately. - .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a writeable, non @@ -826,8 +826,8 @@ than raw I/O does. Write the :term:`bytes-like object`, *b*, and return the number of bytes written. When in non-blocking mode, a - :exc:`BlockingIOError` is raised if the buffer needs to be written out but - the raw stream blocks. + :exc:`BlockingIOError` with :attr:`BlockingIOError.characters_written` set + is raised if the buffer needs to be written out but the raw stream blocks. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) @@ -894,9 +894,10 @@ Text I/O .. attribute:: buffer - The underlying binary buffer (a :class:`BufferedIOBase` instance) that - :class:`TextIOBase` deals with. This is not part of the - :class:`TextIOBase` API and may not exist in some implementations. + The underlying binary buffer (a :class:`BufferedIOBase` + or :class:`RawIOBase` instance) that :class:`TextIOBase` deals with. + This is not part of the :class:`TextIOBase` API and may not exist + in some implementations. .. method:: detach() diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index e5bdfbb144b..9e887d8e657 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -240,7 +240,16 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_reserved - ``True`` if the address is otherwise IETF reserved. + ``True`` if the address is noted as reserved by the IETF. + For IPv4, this is only ``240.0.0.0/4``, the ``Reserved`` address block. + For IPv6, this is all addresses `allocated `__ as + ``Reserved by IETF`` for future use. + + .. note:: For IPv4, ``is_reserved`` is not related to the address block value of the + ``Reserved-by-Protocol`` column in iana-ipv4-special-registry_. + + .. caution:: For IPv6, ``fec0::/10`` a former Site-Local scoped address prefix is + currently excluded from that list (see :attr:`~IPv6Address.is_site_local` & :rfc:`3879`). .. attribute:: is_loopback @@ -261,6 +270,7 @@ write code that handles both IP versions correctly. Address objects are .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +.. _iana-ipv6-address-space: https://www.iana.org/assignments/ipv6-address-space/ipv6-address-space.xhtml .. method:: IPv4Address.__format__(fmt) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 00925ae920a..aa46920d352 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -47,7 +47,7 @@ Iterator Arguments Results Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=2) → AB CD EF G`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` @@ -181,7 +181,7 @@ loops that truncate the stream. Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 3) → ABC DEF G + # batched('ABCDEFG', 2) → AB CD EF G if n < 1: raise ValueError('n must be at least one') iterator = iter(iterable) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 26579ec6328..8b4217c210d 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -18,12 +18,17 @@ is a lightweight data interchange format inspired by `JavaScript `_ object literal syntax (although it is not a strict subset of JavaScript [#rfc-errata]_ ). +.. note:: + The term "object" in the context of JSON processing in Python can be + ambiguous. All values in Python are objects. In JSON, an object refers to + any data wrapped in curly braces, similar to a Python dictionary. + .. warning:: Be cautious when parsing JSON data from untrusted sources. A malicious JSON string may cause the decoder to consume considerable CPU and memory resources. Limiting the size of data to be parsed is recommended. -:mod:`json` exposes an API familiar to users of the standard library +This module exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. Encoding basic Python object hierarchies:: @@ -60,7 +65,7 @@ Pretty printing:: "6": 7 } -Specializing JSON object encoding:: +Customizing JSON object encoding:: >>> import json >>> def custom_json(obj): @@ -83,7 +88,7 @@ Decoding JSON:: >>> json.load(io) ['streaming API'] -Specializing JSON object decoding:: +Customizing JSON object decoding:: >>> import json >>> def as_complex(dct): @@ -178,8 +183,10 @@ Basic Usage :param bool ensure_ascii: If ``True`` (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. - If ``False``, these characters will be outputted as-is. + have all incoming non-ASCII and non-printable characters escaped. + If ``False``, all characters will be outputted as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. :param bool check_circular: If ``False``, the circular reference check for container types is skipped @@ -279,7 +286,7 @@ Basic Usage :param object_hook: If set, a function that is called with the result of - any object literal decoded (a :class:`dict`). + any JSON object literal decoded (a :class:`dict`). The return value of this function will be used instead of the :class:`dict`. This feature can be used to implement custom decoders, @@ -289,7 +296,7 @@ Basic Usage :param object_pairs_hook: If set, a function that is called with the result of - any object literal decoded with an ordered list of pairs. + any JSON object literal decoded with an ordered list of pairs. The return value of this function will be used instead of the :class:`dict`. This feature can be used to implement custom decoders. @@ -490,8 +497,10 @@ Encoders and Decoders :class:`bool` or ``None``. If *skipkeys* is true, such items are simply skipped. If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. + have all incoming non-ASCII and non-printable characters escaped. + If *ensure_ascii* is false, all characters will be output as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. If *check_circular* is true (the default), then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -631,7 +640,7 @@ UTF-32, with UTF-8 being the recommended default for maximum interoperability. As permitted, though not required, by the RFC, this module's serializer sets *ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. +strings only contain printable ASCII characters. Other than the *ensure_ascii* parameter, this module is defined strictly in terms of conversion between Python objects and diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 426e3a06e1e..94fc046d3f3 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -34,12 +34,17 @@ The :mod:`locale` module defines the following exception and functions: If *locale* is given and not ``None``, :func:`setlocale` modifies the locale setting for the *category*. The available categories are listed in the data - description below. *locale* may be a string, or an iterable of two strings - (language code and encoding). If it's an iterable, it's converted to a locale - name using the locale aliasing engine. An empty string specifies the user's + description below. *locale* may be a :ref:`string `, or a pair, + language code and encoding. An empty string specifies the user's default settings. If the modification of the locale fails, the exception :exc:`Error` is raised. If successful, the new locale setting is returned. + If *locale* is a pair, it is converted to a locale name using + the locale aliasing engine. + The language code has the same format as a :ref:`locale name `, + but without encoding. + The language code and encoding can be ``None``. + If *locale* is omitted or ``None``, the current setting for *category* is returned. @@ -53,6 +58,9 @@ The :mod:`locale` module defines the following exception and functions: specified in the :envvar:`LANG` environment variable). If the locale is not changed thereafter, using multithreading should not cause problems. + .. versionchanged:: 3.15 + Support language codes with ``@``-modifiers. + .. function:: localeconv() @@ -345,22 +353,30 @@ The :mod:`locale` module defines the following exception and functions: ``'LANG'``. The GNU gettext search path contains ``'LC_ALL'``, ``'LC_CTYPE'``, ``'LANG'`` and ``'LANGUAGE'``, in that order. - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be + The language code has the same format as a :ref:`locale name `, + but without encoding and ``@``-modifier. + The language code and encoding may be ``None`` if their values cannot be determined. + The "C" locale is represented as ``(None, None)``. .. deprecated-removed:: 3.11 3.15 .. function:: getlocale(category=LC_CTYPE) - Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`!LC_\*` values - except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. + Returns the current setting for the given locale category as a tuple containing + the language code and encoding. *category* may be one of the :const:`!LC_\*` + values except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be + The language code has the same format as a :ref:`locale name `, + but without encoding. + The language code and encoding may be ``None`` if their values cannot be determined. + The "C" locale is represented as ``(None, None)``. + + .. versionchanged:: 3.15 + ``@``-modifier are no longer silently removed, but included in + the language code. .. function:: getpreferredencoding(do_setlocale=True) @@ -508,8 +524,8 @@ The :mod:`locale` module defines the following exception and functions: SSH connections. Python doesn't internally use locale-dependent character transformation functions - from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent - equivalents like :c:macro:`!Py_TOLOWER`. + from ``ctype.h``. Instead, ``pyctype.h`` provides locale-independent + equivalents like :c:macro:`Py_TOLOWER`. .. data:: LC_COLLATE @@ -615,6 +631,61 @@ whose high bit is set (i.e., non-ASCII bytes) are never converted or considered part of a character class such as letter or whitespace. +.. _locale_name: + +Locale names +------------ + +The format of the locale name is platform dependent, and the set of supported +locales can depend on the system configuration. + +On Posix platforms, it usually has the format [1]_: + +.. productionlist:: locale_name + : language ["_" territory] ["." charset] ["@" modifier] + +where *language* is a two- or three-letter language code from `ISO 639`_, +*territory* is a two-letter country or region code from `ISO 3166`_, +*charset* is a locale encoding, and *modifier* is a script name, +a language subtag, a sort order identifier, or other locale modifier +(for example, "latin", "valencia", "stroke" and "euro"). + +On Windows, several formats are supported. [2]_ [3]_ +A subset of `IETF BCP 47`_ tags: + +.. productionlist:: locale_name + : language ["-" script] ["-" territory] ["." charset] + : language ["-" script] "-" territory "-" modifier + +where *language* and *territory* have the same meaning as in Posix, +*script* is a four-letter script code from `ISO 15924`_, +and *modifier* is a language subtag, a sort order identifier +or custom modifier (for example, "valencia", "stroke" or "x-python"). +Both hyphen (``'-'``) and underscore (``'_'``) separators are supported. +Only UTF-8 encoding is allowed for BCP 47 tags. + +Windows also supports locale names in the format: + +.. productionlist:: locale_name + : language ["_" territory] ["." charset] + +where *language* and *territory* are full names, such as "English" and +"United States", and *charset* is either a code page number (for example, "1252") +or UTF-8. +Only the underscore separator is supported in this format. + +The "C" locale is supported on all platforms. + +.. _ISO 639: https://www.iso.org/iso-639-language-code +.. _ISO 3166: https://www.iso.org/iso-3166-country-codes.html +.. _IETF BCP 47: https://www.rfc-editor.org/info/bcp47 +.. _ISO 15924: https://www.unicode.org/iso15924/ + +.. [1] `IEEE Std 1003.1-2024; 8.2 Internationalization Variables `_ +.. [2] `UCRT Locale names, Languages, and Country/Region strings `_ +.. [3] `Locale Names `_ + + .. _embedding-locale: For extension writers and programs that embed Python diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 0e9dc33ae21..96cca3073fe 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -548,7 +548,7 @@ mnemonic that the corresponding value is a callable. The ``filters`` member of ``handlers`` and ``loggers`` can take filter instances in addition to ids. -You can also specify a special key ``'.'`` whose value is a dictionary is a +You can also specify a special key ``'.'`` whose value is a mapping of attribute names to values. If found, the specified attributes will be set on the user-defined object before it is returned. Thus, with the following configuration:: @@ -586,7 +586,7 @@ configuration dictionary for the handler named ``foo``, and later (once that handler has been configured) it points to the configured handler instance. Thus, ``cfg://handlers.foo`` could resolve to either a dictionary or a handler instance. In general, it is wise to name handlers in a way such that dependent -handlers are configured _after_ any handlers they depend on; that allows +handlers are configured *after* any handlers they depend on; that allows something like ``cfg://handlers.foo`` to be used in configuring a handler that depends on handler ``foo``. If that dependent handler were named ``bar``, problems would result, because the configuration of ``bar`` would be attempted diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 63ef533e82c..c9cfbdb4126 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -352,6 +352,10 @@ module, supports rotation of disk log files. Outputs the record to the file, catering for rollover as described previously. + .. method:: shouldRollover(record) + + See if the supplied record would cause the file to exceed the configured size limit. + .. _timed-rotating-file-handler: TimedRotatingFileHandler @@ -461,6 +465,11 @@ timed intervals. Returns a list of filenames which should be deleted as part of rollover. These are the absolute paths of the oldest backup log files written by the handler. + .. method:: shouldRollover(record) + + See if enough time has passed for a rollover to occur and if it has, compute + the next rollover time. + .. _socket-handler: SocketHandler @@ -1051,6 +1060,15 @@ possible, while any potentially slow operations (such as sending an email via .. note:: If you are using :mod:`multiprocessing`, you should avoid using :class:`~queue.SimpleQueue` and instead use :class:`multiprocessing.Queue`. + .. warning:: + + The :mod:`multiprocessing` module uses an internal logger created and + accessed via :meth:`~multiprocessing.get_logger`. + :class:`multiprocessing.Queue` will log ``DEBUG`` level messages upon + items being queued. If those log messages are processed by a + :class:`QueueHandler` using the same :class:`multiprocessing.Queue` instance, + it will cause a deadlock or infinite recursion. + .. method:: emit(record) Enqueues the result of preparing the LogRecord. Should an exception diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 72190e97240..0cf5b1c0d9b 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -671,8 +671,7 @@ Formatter Objects which is just the logged message. :type fmt: str - :param datefmt: A format string in the given *style* for - the date/time portion of the logged output. + :param datefmt: A format string for the date/time portion of the logged output. If not specified, the default described in :meth:`formatTime` is used. :type datefmt: str @@ -680,7 +679,7 @@ Formatter Objects how the format string will be merged with its data: using one of :ref:`old-string-formatting` (``%``), :meth:`str.format` (``{``) or :class:`string.Template` (``$``). This only applies to - *fmt* and *datefmt* (e.g. ``'%(message)s'`` versus ``'{message}'``), + *fmt* (e.g. ``'%(message)s'`` versus ``'{message}'``), not to the actual log messages passed to the logging methods. However, there are :ref:`other ways ` to use ``{``- and ``$``-formatting for log messages. @@ -1083,12 +1082,13 @@ LoggerAdapter Objects information into logging calls. For a usage example, see the section on :ref:`adding contextual information to your logging output `. -.. class:: LoggerAdapter(logger, extra, merge_extra=False) +.. class:: LoggerAdapter(logger, extra=None, merge_extra=False) Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance, a dict-like object (*extra*), and a - boolean (*merge_extra*) indicating whether or not the *extra* argument of - individual log calls should be merged with the :class:`LoggerAdapter` extra. + underlying :class:`Logger` instance, an optional dict-like object (*extra*), + and an optional boolean (*merge_extra*) indicating whether or not + the *extra* argument of individual log calls should be merged with + the :class:`LoggerAdapter` extra. The default behavior is to ignore the *extra* argument of individual log calls and only use the one of the :class:`LoggerAdapter` instance @@ -1128,16 +1128,20 @@ information into logging calls. For a usage example, see the section on Attribute :attr:`!manager` and method :meth:`!_log` were added, which delegate to the underlying logger and allow adapters to be nested. + .. versionchanged:: 3.10 + + The *extra* argument is now optional. + .. versionchanged:: 3.13 - The *merge_extra* argument was added. + The *merge_extra* parameter was added. Thread Safety ------------- The logging module is intended to be thread-safe without any special work -needing to be done by its clients. It achieves this though using threading +needing to be done by its clients. It achieves this through using threading locks; there is one lock to serialize access to the module's shared data, and each handler also creates a lock to serialize access to its underlying I/O. @@ -1317,7 +1321,7 @@ functions. In Python versions earlier than 3.4, this function could also be passed a text level, and would return the corresponding numeric value of the level. This undocumented behaviour was considered a mistake, and was removed in - Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. + Python 3.4, but reinstated in 3.4.2 in order to retain backward compatibility. .. function:: getHandlerByName(name) @@ -1342,8 +1346,9 @@ functions. .. function:: basicConfig(**kwargs) - Does basic configuration for the logging system by creating a - :class:`StreamHandler` with a default :class:`Formatter` and adding it to the + Does basic configuration for the logging system by either creating a + :class:`StreamHandler` with a default :class:`Formatter` + or using the given *formatter* instance, and adding it to the root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, :func:`error` and :func:`critical` will call :func:`basicConfig` automatically if no handlers are defined for the root logger. @@ -1428,6 +1433,19 @@ functions. | | which means that it will be treated the | | | same as passing 'errors'. | +--------------+---------------------------------------------+ + | *formatter* | If specified, set this formatter instance | + | | (see :ref:`formatter-objects`) | + | | for all involved handlers. | + | | If not specified, the default is to create | + | | and use an instance of | + | | :class:`logging.Formatter` based on | + | | arguments *format*, *datefmt* and *style*. | + | | When *formatter* is specified together with | + | | any of the three arguments *format*, | + | | *datefmt* and *style*, a ``ValueError`` is | + | | raised to signal that these arguments would | + | | lose meaning otherwise. | + +--------------+---------------------------------------------+ .. versionchanged:: 3.2 The *style* argument was added. @@ -1444,6 +1462,9 @@ functions. .. versionchanged:: 3.9 The *encoding* and *errors* arguments were added. + .. versionchanged:: 3.15 + The *formatter* argument was added. + .. function:: shutdown() Informs the logging system to perform an orderly shutdown by flushing and diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 69f7cb8d48d..8a4f68f3502 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -23,6 +23,8 @@ module. Note that :class:`LZMAFile` and :class:`bz2.BZ2File` are *not* thread-safe, so if you need to use a single :class:`LZMAFile` instance from multiple threads, it is necessary to protect it with a lock. +.. include:: ../includes/optional-module.rst + .. exception:: LZMAError diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index e8a96f29ea1..62e289573c0 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -917,7 +917,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. copied; furthermore, any format-specific information is converted insofar as possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, - or a file, it should contain an :rfc:`2822`\ -compliant message, which is read + or a file, it should contain an :rfc:`5322`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files are accepted for backward compatibility. diff --git a/Doc/library/math.integer.rst b/Doc/library/math.integer.rst new file mode 100644 index 00000000000..0068ae2bdd5 --- /dev/null +++ b/Doc/library/math.integer.rst @@ -0,0 +1,85 @@ +:mod:`math.integer` --- integer-specific mathematics functions +============================================================== + +.. module:: math.integer + :synopsis: Integer-specific mathematics functions. + +.. versionadded:: 3.15 + +-------------- + +This module provides access to the mathematical functions defined for integer arguments. +These functions accept integers and objects that implement the +:meth:`~object.__index__` method which is used to convert the object to an integer +number. + +The following functions are provided by this module. All return values are +computed exactly and are integers. + + +.. function:: comb(n, k, /) + + Return the number of ways to choose *k* items from *n* items without repetition + and without order. + + Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + Also called the binomial coefficient because it is equivalent + to the coefficient of k-th term in polynomial expansion of + ``(1 + x)ⁿ``. + + Raises :exc:`ValueError` if either of the arguments are negative. + + +.. function:: factorial(n, /) + + Return factorial of the nonnegative integer *n*. + + +.. function:: gcd(*integers) + + Return the greatest common divisor of the specified integer arguments. + If any of the arguments is nonzero, then the returned value is the largest + positive integer that is a divisor of all arguments. If all arguments + are zero, then the returned value is ``0``. ``gcd()`` without arguments + returns ``0``. + + +.. function:: isqrt(n, /) + + Return the integer square root of the nonnegative integer *n*. This is the + floor of the exact square root of *n*, or equivalently the greatest integer + *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. + + For some applications, it may be more convenient to have the least integer + *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of + the exact square root of *n*. For positive *n*, this can be computed using + ``a = 1 + isqrt(n - 1)``. + + + .. |nbsp| unicode:: 0xA0 + :trim: + + +.. function:: lcm(*integers) + + Return the least common multiple of the specified integer arguments. + If all arguments are nonzero, then the returned value is the smallest + positive integer that is a multiple of all arguments. If any of the arguments + is zero, then the returned value is ``0``. ``lcm()`` without arguments + returns ``1``. + + +.. function:: perm(n, k=None, /) + + Return the number of ways to choose *k* items from *n* items + without repetition and with order. + + Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates + to zero when ``k > n``. + + If *k* is not specified or is ``None``, then *k* defaults to *n* + and the function returns ``n!``. + + Raises :exc:`ValueError` if either of the arguments are negative. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 0749367045d..d2ff74822f9 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -10,8 +10,8 @@ -------------- -This module provides access to the mathematical functions defined by the C -standard. +This module provides access to common mathematical functions and constants, +including those defined by the C standard. These functions cannot be used with complex numbers; use the functions of the same name from the :mod:`cmath` module if you require support for complex @@ -27,21 +27,14 @@ noted otherwise, all return values are floats. ==================================================== ============================================ -**Number-theoretic functions** --------------------------------------------------------------------------------------------------- -:func:`comb(n, k) ` Number of ways to choose *k* items from *n* items without repetition and without order -:func:`factorial(n) ` *n* factorial -:func:`gcd(*integers) ` Greatest common divisor of the integer arguments -:func:`isqrt(n) ` Integer square root of a nonnegative integer *n* -:func:`lcm(*integers) ` Least common multiple of the integer arguments -:func:`perm(n, k) ` Number of ways to choose *k* items from *n* items without repetition and with order - **Floating point arithmetic** -------------------------------------------------------------------------------------------------- :func:`ceil(x) ` Ceiling of *x*, the smallest integer greater than or equal to *x* :func:`fabs(x) ` Absolute value of *x* :func:`floor(x) ` Floor of *x*, the largest integer less than or equal to *x* :func:`fma(x, y, z) ` Fused multiply-add operation: ``(x * y) + z`` +:func:`fmax(x, y) ` Maximum of two floating-point values +:func:`fmin(x, y) ` Minimum of two floating-point values :func:`fmod(x, y) ` Remainder of division ``x / y`` :func:`modf(x) ` Fractional and integer parts of *x* :func:`remainder(x, y) ` Remainder of *x* with respect to *y* @@ -53,10 +46,13 @@ noted otherwise, all return values are floats. :func:`frexp(x) ` Mantissa and exponent of *x* :func:`isclose(a, b, rel_tol, abs_tol) ` Check if the values *a* and *b* are close to each other :func:`isfinite(x) ` Check if *x* is neither an infinity nor a NaN +:func:`isnormal(x) ` Check if *x* is a normal number +:func:`issubnormal(x) ` Check if *x* is a subnormal number :func:`isinf(x) ` Check if *x* is a positive or negative infinity :func:`isnan(x) ` Check if *x* is a NaN (not a number) :func:`ldexp(x, i) ` ``x * (2**i)``, inverse of function :func:`frexp` :func:`nextafter(x, y, steps) ` Floating-point value *steps* steps after *x* towards *y* +:func:`signbit(x) ` Check if *x* is a negative number :func:`ulp(x) ` Value of the least significant bit of *x* **Power, exponential and logarithmic functions** @@ -121,93 +117,6 @@ noted otherwise, all return values are floats. ==================================================== ============================================ -Number-theoretic functions --------------------------- - -.. function:: comb(n, k) - - Return the number of ways to choose *k* items from *n* items without repetition - and without order. - - Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - Also called the binomial coefficient because it is equivalent - to the coefficient of k-th term in polynomial expansion of - ``(1 + x)ⁿ``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - -.. function:: factorial(n) - - Return *n* factorial as an integer. Raises :exc:`ValueError` if *n* is not integral or - is negative. - - .. versionchanged:: 3.10 - Floats with integral values (like ``5.0``) are no longer accepted. - - -.. function:: gcd(*integers) - - Return the greatest common divisor of the specified integer arguments. - If any of the arguments is nonzero, then the returned value is the largest - positive integer that is a divisor of all arguments. If all arguments - are zero, then the returned value is ``0``. ``gcd()`` without arguments - returns ``0``. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.9 - Added support for an arbitrary number of arguments. Formerly, only two - arguments were supported. - - -.. function:: isqrt(n) - - Return the integer square root of the nonnegative integer *n*. This is the - floor of the exact square root of *n*, or equivalently the greatest integer - *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. - - For some applications, it may be more convenient to have the least integer - *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of - the exact square root of *n*. For positive *n*, this can be computed using - ``a = 1 + isqrt(n - 1)``. - - .. versionadded:: 3.8 - - -.. function:: lcm(*integers) - - Return the least common multiple of the specified integer arguments. - If all arguments are nonzero, then the returned value is the smallest - positive integer that is a multiple of all arguments. If any of the arguments - is zero, then the returned value is ``0``. ``lcm()`` without arguments - returns ``1``. - - .. versionadded:: 3.9 - - -.. function:: perm(n, k=None) - - Return the number of ways to choose *k* items from *n* items - without repetition and with order. - - Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - If *k* is not specified or is ``None``, then *k* defaults to *n* - and the function returns ``n!``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - Floating point arithmetic ------------------------- @@ -246,6 +155,30 @@ Floating point arithmetic .. versionadded:: 3.13 +.. function:: fmax(x, y) + + Get the larger of two floating-point values, treating NaNs as missing data. + + When both operands are (signed) NaNs or zeroes, return ``nan`` and ``0`` + respectively and the sign of the result is implementation-defined, that + is, :func:`!fmax` is not required to be sensitive to the sign of such + operands (see Annex F of the C11 standard, §F.10.0.3 and §F.10.9.2). + + .. versionadded:: 3.15 + + +.. function:: fmin(x, y) + + Get the smaller of two floating-point values, treating NaNs as missing data. + + When both operands are (signed) NaNs or zeroes, return ``nan`` and ``0`` + respectively and the sign of the result is implementation-defined, that + is, :func:`!fmin` is not required to be sensitive to the sign of such + operands (see Annex F of the C11 standard, §F.10.0.3 and §F.10.9.3). + + .. versionadded:: 3.15 + + .. function:: fmod(x, y) Return the floating-point remainder of ``x / y``, @@ -374,6 +307,24 @@ Floating point manipulation functions .. versionadded:: 3.2 +.. function:: isnormal(x) + + Return ``True`` if *x* is a normal number, that is a finite + nonzero number that is not a subnormal (see :func:`issubnormal`). + Return ``False`` otherwise. + + .. versionadded:: 3.15 + + +.. function:: issubnormal(x) + + Return ``True`` if *x* is a subnormal number, that is a finite + nonzero number with a magnitude smaller than :data:`sys.float_info.min`. + Return ``False`` otherwise. + + .. versionadded:: 3.15 + + .. function:: isinf(x) Return ``True`` if *x* is a positive or negative infinity, and @@ -412,6 +363,15 @@ Floating point manipulation functions Added the *steps* argument. +.. function:: signbit(x) + + Return ``True`` if the sign of *x* is negative and ``False`` otherwise. + + This is useful to detect the sign bit of zeroes, infinities and NaNs. + + .. versionadded:: 3.15 + + .. function:: ulp(x) Return the value of the least significant bit of the float *x*: @@ -546,7 +506,7 @@ Summation and product functions Roughly equivalent to:: - sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q))) + sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q, strict=True))) .. versionadded:: 3.8 @@ -757,6 +717,75 @@ Special functions .. versionadded:: 3.2 +Number-theoretic functions +-------------------------- + +For backward compatibility, the :mod:`math` module provides also aliases of +the following functions from the :mod:`math.integer` module: + +.. list-table:: + + * - .. function:: comb(n, k) + :no-typesetting: + + :func:`comb(n, k) ` + - Number of ways to choose *k* items from *n* items without repetition + and without order + + * - .. function:: factorial(n) + :no-typesetting: + + :func:`factorial(n) ` + - *n* factorial + + * - .. function:: gcd(*integers) + :no-typesetting: + + :func:`gcd(*integers) ` + - Greatest common divisor of the integer arguments + + * - .. function:: isqrt(n) + :no-typesetting: + + :func:`isqrt(n) ` + - Integer square root of a nonnegative integer *n* + + * - .. function:: lcm(*integers) + :no-typesetting: + + :func:`lcm(*integers) ` + - Least common multiple of the integer arguments + + * - .. function:: perm(n, k) + :no-typesetting: + + :func:`perm(n, k) ` + - Number of ways to choose *k* items from *n* items without repetition + and with order + +.. versionadded:: 3.5 + The :func:`gcd` function. + +.. versionadded:: 3.8 + The :func:`comb`, :func:`perm` and :func:`isqrt` functions. + +.. versionadded:: 3.9 + The :func:`lcm` function. + +.. versionchanged:: 3.9 + Added support for an arbitrary number of arguments in the :func:`gcd` + function. + Formerly, only two arguments were supported. + +.. versionchanged:: 3.10 + Floats with integral values (like ``5.0``) are no longer accepted in the + :func:`factorial` function. + +.. deprecated:: 3.15 + These aliases are :term:`soft deprecated` in favor of the + :mod:`math.integer` functions. + + Constants --------- @@ -775,7 +804,7 @@ Constants The mathematical constant *τ* = 6.283185..., to available precision. Tau is a circle constant equal to 2\ *π*, the ratio of a circle's circumference to its radius. To learn more about Tau, check out Vi Hart's video `Pi is (still) - Wrong `_, and start celebrating + Wrong `_, and start celebrating `Tau day `_ by eating twice as much pie! .. versionadded:: 3.6 @@ -839,5 +868,5 @@ Constants Module :mod:`cmath` Complex number versions of many of these functions. -.. |nbsp| unicode:: 0xA0 - :trim: + Module :mod:`math.integer` + Integer-specific mathematics functions. diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 4e20c07331a..f32aa322c40 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -48,10 +48,11 @@ update the underlying file. To map anonymous memory, -1 should be passed as the fileno along with the length. -.. class:: mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT, offset=0) +.. class:: mmap(fileno, length, tagname=None, \ + access=ACCESS_DEFAULT, offset=0, *, trackfd=True) **(Windows version)** Maps *length* bytes from the file specified by the - file handle *fileno*, and creates a mmap object. If *length* is larger + file descriptor *fileno*, and creates a mmap object. If *length* is larger than the current size of the file, the file is extended to contain *length* bytes. If *length* is ``0``, the maximum length of the map is the current size of the file, except that if the file is empty Windows raises an @@ -69,6 +70,17 @@ To map anonymous memory, -1 should be passed as the fileno along with the length will be relative to the offset from the beginning of the file. *offset* defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. + If *trackfd* is ``False``, the file handle corresponding to *fileno* will + not be duplicated, and the resulting :class:`!mmap` object will not + be associated with the map's underlying file. + This means that the :meth:`~mmap.mmap.size` and :meth:`~mmap.mmap.resize` + methods will fail. + This mode is useful to limit the number of open file handles. + The original file can be renamed (but not deleted) after closing *fileno*. + + .. versionchanged:: 3.15 + The *trackfd* parameter was added. + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap .. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, \ @@ -217,6 +229,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length on error under Windows. A zero value was returned on success; an exception was raised on error under Unix. + .. versionchanged:: 3.15 + Allow specifying *offset* without *size*. Previously, both *offset* + and *size* parameters were required together. Now *offset* can be + specified alone, and the flush operation will extend from *offset* + to the end of the mmap. + .. method:: madvise(option[, start[, length]]) @@ -269,7 +287,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Resizing a map created with *access* of :const:`ACCESS_READ` or :const:`ACCESS_COPY`, will raise a :exc:`TypeError` exception. - Resizing a map created with with *trackfd* set to ``False``, + Resizing a map created with *trackfd* set to ``False``, will raise a :exc:`ValueError` exception. **On Windows**: Resizing the map will raise an :exc:`OSError` if there are other @@ -277,6 +295,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length pagefile) will silently create a new map with the original data copied over up to the length of the new size. + Availability: Windows and systems with the ``mremap()`` system call. + .. versionchanged:: 3.11 Correctly fails if attempting to resize when another map is held Allows resize against an anonymous map on Windows @@ -312,6 +332,10 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Return the length of the file, which can be larger than the size of the memory-mapped area. + For an anonymous mapping, return its size. + + .. versionchanged:: 3.15 + Anonymous mappings are now supported on Unix. .. method:: tell() diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 327cc3602b1..a2c5e375d2c 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -22,6 +22,8 @@ api. The normal API deals only with ASCII characters and is of limited use for internationalized applications. The wide char API should be used where ever possible. +.. availability:: Windows. + .. versionchanged:: 3.3 Operations in this module now raise :exc:`OSError` where :exc:`IOError` was raised. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 6c43d5fe166..92605c57527 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -22,8 +22,7 @@ to this, the :mod:`multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both POSIX and Windows. -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the +The :mod:`multiprocessing` module also introduces the :class:`~multiprocessing.pool.Pool` object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following @@ -44,6 +43,10 @@ will print to standard output :: [1, 4, 9] +The :mod:`multiprocessing` module also introduces APIs which do not have +analogs in the :mod:`threading` module, like the ability to :meth:`terminate +`, :meth:`interrupt ` or :meth:`kill +` a running process. .. seealso:: @@ -97,6 +100,10 @@ To show the individual process IDs involved, here is an expanded example:: For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. +The arguments to :class:`Process` usually need to be unpickleable from within +the child process. If you tried typing the above example directly into a REPL it +could lead to an :exc:`AttributeError` in the child process trying to locate the +*f* function in the ``__main__`` module. .. _multiprocessing-start-methods: @@ -233,9 +240,12 @@ processes for a different context. In particular, locks created using the *fork* context cannot be passed to processes started using the *spawn* or *forkserver* start methods. -A library which wants to use a particular start method should probably -use :func:`get_context` to avoid interfering with the choice of the -library user. +Libraries using :mod:`multiprocessing` or +:class:`~concurrent.futures.ProcessPoolExecutor` should be designed to allow +their users to provide their own multiprocessing context. Using a specific +context of your own within a library can lead to incompatibilities with the +rest of the library user's application. Always document if your library +requires a specific start method. .. warning:: @@ -538,9 +548,42 @@ The :mod:`multiprocessing` package mostly replicates the API of the to pass to *target*. If a subclass overrides the constructor, it must make sure it invokes the - base class constructor (:meth:`Process.__init__`) before doing anything else + base class constructor (``super().__init__()``) before doing anything else to the process. + .. note:: + + In general, all arguments to :class:`Process` must be picklable. This is + frequently observed when trying to create a :class:`Process` or use a + :class:`concurrent.futures.ProcessPoolExecutor` from a REPL with a + locally defined *target* function. + + Passing a callable object defined in the current REPL session causes the + child process to die via an uncaught :exc:`AttributeError` exception when + starting as *target* must have been defined within an importable module + in order to be loaded during unpickling. + + Example of this uncatchable error from the child:: + + >>> import multiprocessing as mp + >>> def knigit(): + ... print("Ni!") + ... + >>> process = mp.Process(target=knigit) + >>> process.start() + >>> Traceback (most recent call last): + File ".../multiprocessing/spawn.py", line ..., in spawn_main + File ".../multiprocessing/spawn.py", line ..., in _main + AttributeError: module '__main__' has no attribute 'knigit' + >>> process + + + See :ref:`multiprocessing-programming-spawn`. While this restriction is + not true if using the ``"fork"`` start method, as of Python ``3.14`` that + is no longer the default on any platform. See + :ref:`multiprocessing-start-methods`. + See also :gh:`132898`. + .. versionchanged:: 3.3 Added the *daemon* parameter. @@ -789,8 +832,8 @@ raising an exception. One difference from other Python queue implementations, is that :mod:`multiprocessing` queues serializes all objects that are put into them using :mod:`pickle`. -The object return by the get method is a re-created object that does not share memory -with the original object. +The object returned by the get method is a re-created object that does not share +memory with the original object. Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. @@ -847,7 +890,7 @@ For an example of the usage of queues for interprocess communication see :ref:`multiprocessing-examples`. -.. function:: Pipe([duplex]) +.. function:: Pipe(duplex=True) Returns a pair ``(conn1, conn2)`` of :class:`~multiprocessing.connection.Connection` objects representing the @@ -936,8 +979,13 @@ For an example of the usage of queues for interprocess communication see .. method:: close() - Indicate that no more data will be put on this queue by the current - process. The background thread will quit once it has flushed all buffered + Close the queue: release internal resources. + + A queue must not be used anymore after it is closed. For example, + :meth:`~Queue.get`, :meth:`~Queue.put` and :meth:`~Queue.empty` + methods must no longer be called. + + The background thread will quit once it has flushed all buffered data to the pipe. This is called automatically when the queue is garbage collected. @@ -1081,7 +1129,7 @@ Miscellaneous .. function:: freeze_support() Add support for when a program which uses :mod:`multiprocessing` has been - frozen to produce a Windows executable. (Has been tested with **py2exe**, + frozen to produce an executable. (Has been tested with **py2exe**, **PyInstaller** and **cx_Freeze**.) One needs to call this function straight after the ``if __name__ == @@ -1099,10 +1147,10 @@ Miscellaneous If the ``freeze_support()`` line is omitted then trying to run the frozen executable will raise :exc:`RuntimeError`. - Calling ``freeze_support()`` has no effect when invoked on any operating - system other than Windows. In addition, if the module is being run - normally by the Python interpreter on Windows (the program has not been - frozen), then ``freeze_support()`` has no effect. + Calling ``freeze_support()`` has no effect when the start method is not + *spawn*. In addition, if the module is being run normally by the Python + interpreter (the program has not been frozen), then ``freeze_support()`` + has no effect. .. function:: get_all_start_methods() @@ -1118,7 +1166,9 @@ Miscellaneous Return a context object which has the same attributes as the :mod:`multiprocessing` module. - If *method* is ``None`` then the default context is returned. + If *method* is ``None`` then the default context is returned. Note that if + the global start method has not been set, this will set it to the + default method. Otherwise *method* should be ``'fork'``, ``'spawn'``, ``'forkserver'``. :exc:`ValueError` is raised if the specified start method is not available. See :ref:`multiprocessing-start-methods`. @@ -1129,10 +1179,10 @@ Miscellaneous Return the name of start method used for starting processes. - If the start method has not been fixed and *allow_none* is false, - then the start method is fixed to the default and the name is - returned. If the start method has not been fixed and *allow_none* - is true then ``None`` is returned. + If the global start method has not been set and *allow_none* is + ``False``, then the start method is set to the default and the name + is returned. If the start method has not been set and *allow_none* is + ``True`` then ``None`` is returned. The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. @@ -1369,6 +1419,12 @@ object -- see :ref:`multiprocessing-managers`. A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. + .. method:: locked() + + Return a boolean indicating whether this object is locked right now. + + .. versionadded:: 3.14 + .. note:: On macOS, this is indistinguishable from :class:`Semaphore` because ``sem_getvalue()`` is not implemented on that platform. @@ -1521,6 +1577,22 @@ object -- see :ref:`multiprocessing-managers`. A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. + + .. method:: get_value() + + Return the current value of semaphore. + + Note that this may raise :exc:`NotImplementedError` on platforms like + macOS where ``sem_getvalue()`` is not implemented. + + + .. method:: locked() + + Return a boolean indicating whether this object is locked right now. + + .. versionadded:: 3.14 + + .. note:: On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with @@ -3051,10 +3123,10 @@ start method. More picklability - Ensure that all arguments to :meth:`Process.__init__` are picklable. - Also, if you subclass :class:`~multiprocessing.Process` then make sure that - instances will be picklable when the :meth:`Process.start - ` method is called. + Ensure that all arguments to :class:`~multiprocessing.Process` are + picklable. Also, if you subclass ``Process.__init__``, you must make sure + that instances will be picklable when the + :meth:`Process.start ` method is called. Global variables diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index f6260383b2b..74c97e8c9a9 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -24,12 +24,14 @@ the Unix :program:`ftp` program and other FTP clients. a :exc:`FileNotFoundError` exception will be raised. Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. + If no argument is specified on a POSIX system, the presence of passwords in the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file ownership or permissions are insecure (owned by a user other than the user running the process, or accessible for read or write by any other user). This implements security behavior equivalent to that of ftp and other - programs that use :file:`.netrc`. + programs that use :file:`.netrc`. Such security checks are not available + on platforms that do not support :func:`os.getuid`. .. versionchanged:: 3.4 Added the POSIX permission check. diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 681d0b76f2a..57b35017072 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -69,11 +69,11 @@ The numeric tower .. attribute:: numerator - Abstract. + Abstract. The numerator of this rational number. .. attribute:: denominator - Abstract. + Abstract. The denominator of this rational number. .. class:: Integral diff --git a/Doc/library/numeric.rst b/Doc/library/numeric.rst index 7c76a479d73..a2109a29bfb 100644 --- a/Doc/library/numeric.rst +++ b/Doc/library/numeric.rst @@ -19,6 +19,7 @@ The following modules are documented in this chapter: numbers.rst math.rst + math.integer.rst cmath.rst decimal.rst fractions.rst diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index ecbbc1d7605..c90e30ccf4f 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -42,8 +42,8 @@ the :mod:`glob` module.) a path that is *always* in one of the different formats. They all have the same interface: - * :mod:`posixpath` for UNIX-style paths - * :mod:`ntpath` for Windows paths + * :mod:`!posixpath` for UNIX-style paths + * :mod:`!ntpath` for Windows paths .. versionchanged:: 3.8 @@ -64,7 +64,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: basename(path) +.. function:: basename(path, /) Return the base name of pathname *path*. This is the second element of the pair returned by passing *path* to the function :func:`split`. Note that @@ -94,7 +94,7 @@ the :mod:`glob` module.) Any iterable can now be passed, rather than just sequences. -.. function:: commonprefix(list) +.. function:: commonprefix(list, /) Return the longest path prefix (taken character-by-character) that is a prefix of all paths in *list*. If *list* is empty, return the empty string @@ -118,7 +118,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: dirname(path) +.. function:: dirname(path, /) Return the directory name of pathname *path*. This is the first element of the pair returned by passing *path* to the function :func:`split`. @@ -199,14 +199,14 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getatime(path) +.. function:: getatime(path, /) Return the time of last access of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. -.. function:: getmtime(path) +.. function:: getmtime(path, /) Return the time of last modification of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). @@ -216,7 +216,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getctime(path) +.. function:: getctime(path, /) Return the system's ctime which, on some systems (like Unix) is the time of the last metadata change, and, on others (like Windows), is the creation time for *path*. @@ -228,7 +228,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getsize(path) +.. function:: getsize(path, /) Return the size, in bytes, of *path*. Raise :exc:`OSError` if the file does not exist or is inaccessible. @@ -237,7 +237,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: isabs(path) +.. function:: isabs(path, /) Return ``True`` if *path* is an absolute pathname. On Unix, that means it begins with a slash, on Windows that it begins with two (back)slashes, or a @@ -261,7 +261,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: isdir(path) +.. function:: isdir(path, /) Return ``True`` if *path* is an :func:`existing ` directory. This follows symbolic links, so both :func:`islink` and :func:`isdir` can be true @@ -298,9 +298,10 @@ the :mod:`glob` module.) device than *path*, or whether :file:`{path}/..` and *path* point to the same i-node on the same device --- this should detect mount points for all Unix and POSIX variants. It is not able to reliably detect bind mounts on the - same filesystem. On Windows, a drive letter root and a share UNC are - always mount points, and for any other path ``GetVolumePathName`` is called - to see if it is different from the input path. + same filesystem. On Linux systems, it will always return ``True`` for btrfs + subvolumes, even if they aren't mount points. On Windows, a drive letter root + and a share UNC are always mount points, and for any other path + ``GetVolumePathName`` is called to see if it is different from the input path. .. versionchanged:: 3.4 Added support for detecting non-root mount points on Windows. @@ -350,7 +351,7 @@ the :mod:`glob` module.) .. versionadded:: 3.13 -.. function:: join(path, *paths) +.. function:: join(path, /, *paths) Join one or more path segments intelligently. The return value is the concatenation of *path* and all members of *\*paths*, with exactly one @@ -371,7 +372,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object` for *path* and *paths*. -.. function:: normcase(path) +.. function:: normcase(path, /) Normalize the case of a pathname. On Windows, convert all characters in the pathname to lowercase, and also convert forward slashes to backward slashes. @@ -401,16 +402,35 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: realpath(path, *, strict=False) +.. function:: realpath(path, /, *, strict=False) Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path (if they are supported by the operating system). On Windows, this function will also resolve MS-DOS (also called 8.3) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. - If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False`` these errors - are ignored, and so the result might be missing or otherwise inaccessible. + By default, the path is evaluated up to the first component that does not + exist, is a symlink loop, or whose evaluation raises :exc:`OSError`. + All such components are appended unchanged to the existing part of the path. + + Some errors that are handled this way include "access denied", "not a + directory", or "bad argument to internal function". Thus, the + resulting path may be missing or inaccessible, may still contain + links or loops, and may traverse non-directories. + + This behavior can be modified by keyword arguments: + + If *strict* is ``True``, the first error encountered when evaluating the path is + re-raised. + In particular, :exc:`FileNotFoundError` is raised if *path* does not exist, + or another :exc:`OSError` if it is otherwise inaccessible. + If *strict* is :data:`ALL_BUT_LAST`, the last component of the path + is allowed to be missing, but all other errors are raised. + + If *strict* is :py:data:`os.path.ALLOW_MISSING`, errors other than + :exc:`FileNotFoundError` are re-raised (as with ``strict=True``). + Thus, the returned path will not contain any symbolic links, but the named + file and some of its parent directories may be missing. .. note:: This function emulates the operating system's procedure for making a path @@ -429,6 +449,22 @@ the :mod:`glob` module.) .. versionchanged:: 3.10 The *strict* parameter was added. + .. versionchanged:: 3.15 + The :data:`ALL_BUT_LAST` and :data:`ALLOW_MISSING` values for + the *strict* parameter was added. + +.. data:: ALL_BUT_LAST + + Special value used for the *strict* argument in :func:`realpath`. + + .. versionadded:: 3.15 + +.. data:: ALLOW_MISSING + + Special value used for the *strict* argument in :func:`realpath`. + + .. versionadded:: 3.15 + .. function:: relpath(path, start=os.curdir) @@ -444,7 +480,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: samefile(path1, path2) +.. function:: samefile(path1, path2, /) Return ``True`` if both pathname arguments refer to the same file or directory. This is determined by the device number and i-node number and raises an @@ -471,7 +507,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: samestat(stat1, stat2) +.. function:: samestat(stat1, stat2, /) Return ``True`` if the stat tuples *stat1* and *stat2* refer to the same file. These structures may have been returned by :func:`os.fstat`, @@ -481,11 +517,8 @@ the :mod:`glob` module.) .. versionchanged:: 3.4 Added Windows support. - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. function:: split(path) +.. function:: split(path, /) Split the pathname *path* into a pair, ``(head, tail)`` where *tail* is the last pathname component and *head* is everything leading up to that. The @@ -501,7 +534,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: splitdrive(path) +.. function:: splitdrive(path, /) Split the pathname *path* into a pair ``(drive, tail)`` where *drive* is either a mount point or the empty string. On systems which do not use drive @@ -526,7 +559,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: splitroot(path) +.. function:: splitroot(path, /) Split the pathname *path* into a 3-item tuple ``(drive, root, tail)`` where *drive* is a device name or mount point, *root* is a string of separators @@ -559,7 +592,7 @@ the :mod:`glob` module.) .. versionadded:: 3.12 -.. function:: splitext(path) +.. function:: splitext(path, /) Split the pathname *path* into a pair ``(root, ext)`` such that ``root + ext == path``, and the extension, *ext*, is empty or begins with a period and contains at diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 1e54cfec609..671270d6112 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -108,6 +108,12 @@ Python UTF-8 Mode .. versionadded:: 3.7 See :pep:`540` for more details. +.. versionchanged:: 3.15 + + Python UTF-8 mode is now enabled by default (:pep:`686`). + It may be disabled with by setting :envvar:`PYTHONUTF8=0 ` as + an environment variable or by using the :option:`-X utf8=0 <-X>` command line option. + The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage of the UTF-8 encoding: @@ -139,31 +145,22 @@ level APIs also exhibit different default behaviours: default so that attempting to open a binary file in text mode is likely to raise an exception rather than producing nonsense data. -The :ref:`Python UTF-8 Mode ` is enabled if the LC_CTYPE locale is -``C`` or ``POSIX`` at Python startup (see the :c:func:`PyConfig_Read` -function). - -It can be enabled or disabled using the :option:`-X utf8 <-X>` command line -option and the :envvar:`PYTHONUTF8` environment variable. - -If the :envvar:`PYTHONUTF8` environment variable is not set at all, then the -interpreter defaults to using the current locale settings, *unless* the current -locale is identified as a legacy ASCII-based locale (as described for -:envvar:`PYTHONCOERCECLOCALE`), and locale coercion is either disabled or -fails. In such legacy locales, the interpreter will default to enabling UTF-8 -mode unless explicitly instructed not to do so. - -The Python UTF-8 Mode can only be enabled at the Python startup. Its value +The :ref:`Python UTF-8 Mode ` is enabled by default. +It can be disabled using the :option:`-X utf8=0 <-X>` command line +option or the :envvar:`PYTHONUTF8=0 ` environment variable. +The Python UTF-8 Mode can only be disabled at Python startup. Its value can be read from :data:`sys.flags.utf8_mode `. +If the UTF-8 mode is disabled, the interpreter defaults to using +the current locale settings, *unless* the current locale is identified +as a legacy ASCII-based locale (as described for :envvar:`PYTHONCOERCECLOCALE`), +and locale coercion is either disabled or fails. +In such legacy locales, the interpreter will default to enabling UTF-8 mode +unless explicitly instructed not to do so. + See also the :ref:`UTF-8 mode on Windows ` and the :term:`filesystem encoding and error handler`. -.. seealso:: - - :pep:`686` - Python 3.15 will make :ref:`utf8-mode` default. - .. _os-procinfo: @@ -219,8 +216,8 @@ process and user. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from - :data:`os.environ`, and when one of the :meth:`pop` or :meth:`clear` methods is - called. + :data:`os.environ`, and when one of the :meth:`~dict.pop` or + :meth:`~dict.clear` methods is called. .. seealso:: @@ -433,8 +430,8 @@ process and user. associated with the effective user id of the process; the group access list may change over the lifetime of the process, it is not affected by calls to :func:`setgroups`, and its length is not limited to 16. The - deployment target value, :const:`MACOSX_DEPLOYMENT_TARGET`, can be - obtained with :func:`sysconfig.get_config_var`. + deployment target value can be obtained with + :func:`sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') `. .. function:: getlogin() @@ -561,7 +558,7 @@ process and user. .. function:: initgroups(username, gid, /) - Call the system initgroups() to initialize the group access list with all of + Call the system ``initgroups()`` to initialize the group access list with all of the groups of which the specified username is a member, plus the specified group id. @@ -1504,6 +1501,7 @@ or `the MSDN `_ on Windo - :data:`RWF_HIPRI` - :data:`RWF_NOWAIT` + - :data:`RWF_DONTCACHE` Return the total number of bytes actually read which can be less than the total capacity of all the objects. @@ -1549,6 +1547,15 @@ or `the MSDN `_ on Windo .. versionadded:: 3.7 +.. data:: RWF_DONTCACHE + + Use uncached buffered IO. + + .. availability:: Linux >= 6.14 + + .. versionadded:: 3.15 + + .. function:: ptsname(fd, /) Return the name of the slave pseudo-terminal device associated with the @@ -1590,6 +1597,7 @@ or `the MSDN `_ on Windo - :data:`RWF_DSYNC` - :data:`RWF_SYNC` - :data:`RWF_APPEND` + - :data:`RWF_DONTCACHE` Return the total number of bytes actually written. @@ -2009,8 +2017,8 @@ features: must be a string specifying a file path. However, some functions now alternatively accept an open file descriptor for their *path* argument. The function will then operate on the file referred to by the descriptor. - (For POSIX systems, Python will call the variant of the function prefixed - with ``f`` (e.g. call ``fchdir`` instead of ``chdir``).) + For POSIX systems, Python will call the variant of the function prefixed + with ``f`` (e.g. call ``fchdir`` instead of ``chdir``). You can check whether or not *path* can be specified as a file descriptor for a particular function on your platform using :data:`os.supports_fd`. @@ -2025,7 +2033,7 @@ features: * **paths relative to directory descriptors:** If *dir_fd* is not ``None``, it should be a file descriptor referring to a directory, and the path to operate on should be relative; path will then be relative to that directory. If the - path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call + path is absolute, *dir_fd* is ignored. For POSIX systems, Python will call the variant of the function with an ``at`` suffix and possibly prefixed with ``f`` (e.g. call ``faccessat`` instead of ``access``). @@ -2038,8 +2046,8 @@ features: * **not following symlinks:** If *follow_symlinks* is ``False``, and the last element of the path to operate on is a symbolic link, the function will operate on the symbolic link itself rather than the file - pointed to by the link. (For POSIX systems, Python will call the ``l...`` - variant of the function.) + pointed to by the link. For POSIX systems, Python will call the ``l...`` + variant of the function. You can check whether or not *follow_symlinks* is supported for a particular function on your platform using :data:`os.supports_follow_symlinks`. @@ -2598,10 +2606,10 @@ features: Create a filesystem node (file, device special file or named pipe) named *path*. *mode* specifies both the permissions to use and the type of node - to be created, being combined (bitwise OR) with one of ``stat.S_IFREG``, - ``stat.S_IFCHR``, ``stat.S_IFBLK``, and ``stat.S_IFIFO`` (those constants are - available in :mod:`stat`). For ``stat.S_IFCHR`` and ``stat.S_IFBLK``, - *device* defines the newly created device special file (probably using + to be created, being combined (bitwise OR) with one of :const:`stat.S_IFREG`, + :const:`stat.S_IFCHR`, :const:`stat.S_IFBLK`, and :const:`stat.S_IFIFO`. + For :const:`stat.S_IFCHR` and :const:`stat.S_IFBLK`, *device* defines the + newly created device special file (probably using :func:`os.makedev`), otherwise it is ignored. This function can also support :ref:`paths relative to directory descriptors @@ -2619,13 +2627,13 @@ features: .. function:: major(device, /) Extract the device major number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). + :attr:`~stat_result.st_dev` or :attr:`~stat_result.st_rdev` field from :c:struct:`stat`). .. function:: minor(device, /) Extract the device minor number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). + :attr:`~stat_result.st_dev` or :attr:`~stat_result.st_rdev` field from :c:struct:`stat`). .. function:: makedev(major, minor, /) @@ -2633,6 +2641,13 @@ features: Compose a raw device number from the major and minor device numbers. +.. data:: NODEV + + Non-existent device. + + .. versionadded:: 3.15 + + .. function:: pathconf(path, name) Return system configuration information relevant to a named file. *name* @@ -3349,8 +3364,8 @@ features: .. versionchanged:: 3.8 On Windows, the :attr:`st_mode` member now identifies special - files as :const:`S_IFCHR`, :const:`S_IFIFO` or :const:`S_IFBLK` - as appropriate. + files as :const:`~stat.S_IFCHR`, :const:`~stat.S_IFIFO` or + :const:`~stat.S_IFBLK` as appropriate. .. versionchanged:: 3.12 On Windows, :attr:`st_ctime` is deprecated. Eventually, it will @@ -3368,6 +3383,359 @@ features: Added the :attr:`st_birthtime` member on Windows. +.. function:: statx(path, mask, *, flags=0, dir_fd=None, follow_symlinks=True) + + Get the status of a file or file descriptor by performing a :c:func:`!statx` + system call on the given path. + + *path* is a :term:`path-like object` or an open file descriptor. *mask* is a + combination of the module-level :const:`STATX_* ` constants + specifying the information to retrieve. *flags* is a combination of the + module-level :const:`AT_STATX_* ` constants and/or + :const:`AT_NO_AUTOMOUNT`. Returns a :class:`statx_result` object whose + :attr:`~os.statx_result.stx_mask` attribute specifies the information + actually retrieved (which may differ from *mask*). + + This function supports :ref:`specifying a file descriptor `, + :ref:`paths relative to directory descriptors `, and + :ref:`not following symlinks `. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. class:: statx_result + + Information about a file returned by :func:`os.statx`. + + :class:`!statx_result` has the following attributes: + + .. attribute:: stx_atime + + Time of most recent access expressed in seconds. + + Equal to ``None`` if :data:`STATX_ATIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_atime_ns + + Time of most recent access expressed in nanoseconds as an integer. + + Equal to ``None`` if :data:`STATX_ATIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_atomic_write_segments_max + + Maximum iovecs for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_unit_max + + Maximum size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_atomic_write_unit_max_opt + + Maximum optimized size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.16. + + .. attribute:: stx_atomic_write_unit_min + + Minimum size for direct I/O with torn-write protection. + + Equal to ``None`` if :data:`STATX_WRITE_ATOMIC` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.11. + + .. attribute:: stx_attributes + + Bitmask of :const:`STATX_ATTR_* ` constants + specifying the attributes of this file. + + .. attribute:: stx_attributes_mask + + A mask indicating which bits in :attr:`stx_attributes` are supported by + the VFS and the filesystem. + + .. attribute:: stx_blksize + + "Preferred" blocksize for efficient file system I/O. Writing to a file in + smaller chunks may cause an inefficient read-modify-rewrite. + + .. attribute:: stx_blocks + + Number of 512-byte blocks allocated for file. + This may be smaller than :attr:`stx_size`/512 when the file has holes. + + Equal to ``None`` if :data:`STATX_BLOCKS` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_btime + + Time of file creation expressed in seconds. + + Equal to ``None`` if :data:`STATX_BTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_btime_ns + + Time of file creation expressed in nanoseconds as an integer. + + Equal to ``None`` if :data:`STATX_BTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ctime + + Time of most recent metadata change expressed in seconds. + + Equal to ``None`` if :data:`STATX_CTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ctime_ns + + Time of most recent metadata change expressed in nanoseconds as an + integer. + + Equal to ``None`` if :data:`STATX_CTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_dev + + Identifier of the device on which this file resides. + + .. attribute:: stx_dev_major + + Major number of the device on which this file resides. + + .. attribute:: stx_dev_minor + + Minor number of the device on which this file resides. + + .. attribute:: stx_dio_mem_align + + Direct I/O memory buffer alignment requirement. + + Equal to ``None`` if :data:`STATX_DIOALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.1. + + .. attribute:: stx_dio_offset_align + + Direct I/O file offset alignment requirement. + + Equal to ``None`` if :data:`STATX_DIOALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.1. + + .. attribute:: stx_dio_read_offset_align + + Direct I/O file offset alignment requirement for reads. + + Equal to ``None`` if :data:`STATX_DIO_READ_ALIGN` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.14. + + .. attribute:: stx_gid + + Group identifier of the file owner. + + Equal to ``None`` if :data:`STATX_GID` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_ino + + Inode number. + + Equal to ``None`` if :data:`STATX_INO` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mask + + Bitmask of :const:`STATX_* ` constants specifying the + information retrieved, which may differ from what was requested. + + .. attribute:: stx_mnt_id + + Mount identifier. + + Equal to ``None`` if :data:`STATX_MNT_ID` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 5.8. + + .. attribute:: stx_mode + + File mode: file type and file mode bits (permissions). + + Equal to ``None`` if :data:`STATX_TYPE | STATX_MODE ` + is missing from :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mtime + + Time of most recent content modification expressed in seconds. + + Equal to ``None`` if :data:`STATX_MTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_mtime_ns + + Time of most recent content modification expressed in nanoseconds as an + integer. + + Equal to ``None`` if :data:`STATX_MTIME` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_nlink + + Number of hard links. + + Equal to ``None`` if :data:`STATX_NLINK` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_rdev + + Type of device if an inode device. + + .. attribute:: stx_rdev_major + + Major number of the device this file represents. + + .. attribute:: stx_rdev_minor + + Minor number of the device this file represents. + + .. attribute:: stx_size + + Size of the file in bytes, if it is a regular file or a symbolic link. + The size of a symbolic link is the length of the pathname it contains, + without a terminating null byte. + + Equal to ``None`` if :data:`STATX_SIZE` is missing from + :attr:`~statx_result.stx_mask`. + + .. attribute:: stx_subvol + + Subvolume identifier. + + Equal to ``None`` if :data:`STATX_SUBVOL` is missing from + :attr:`~statx_result.stx_mask`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28 and build-time kernel + userspace API headers >= 6.10. + + .. attribute:: stx_uid + + User identifier of the file owner. + + Equal to ``None`` if :data:`STATX_UID` is missing from + :attr:`~statx_result.stx_mask`. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. data:: STATX_TYPE + STATX_MODE + STATX_NLINK + STATX_UID + STATX_GID + STATX_ATIME + STATX_MTIME + STATX_CTIME + STATX_INO + STATX_SIZE + STATX_BLOCKS + STATX_BASIC_STATS + STATX_BTIME + STATX_MNT_ID + STATX_DIOALIGN + STATX_MNT_ID_UNIQUE + STATX_SUBVOL + STATX_WRITE_ATOMIC + STATX_DIO_READ_ALIGN + + Bitflags for use in the *mask* parameter to :func:`os.statx`. Some of these + flags may be available even when their corresponding members in + :class:`statx_result` are not available. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_FORCE_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + up-to-date information even when doing so is expensive (for example, + requiring a round trip to the server for a file on a network filesystem). + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_DONT_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + cached information if possible. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + +.. data:: AT_STATX_SYNC_AS_STAT + + A flag for the :func:`os.statx` function. This flag is defined as ``0``, so + it has no effect, but it can be used to explicitly indicate neither + :data:`AT_STATX_FORCE_SYNC` nor :data:`AT_STATX_DONT_SYNC` is being passed. + In the absence of the other two flags, the kernel will generally return + information as fresh as :func:`os.stat` would return. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: 3.15 + + +.. data:: AT_NO_AUTOMOUNT + + If the final component of a path is an automount point, operate on the + automount point instead of performing the automount. On Linux, + :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` always behave this + way. + + .. availability:: Linux. + + .. versionadded:: 3.15 + + .. function:: statvfs(path) Perform a :c:func:`!statvfs` system call on the given path. The return value is @@ -3505,6 +3873,9 @@ features: Create a symbolic link pointing to *src* named *dst*. + The *src* parameter refers to the target of the link (the file or directory being linked to), + and *dst* is the name of the link being created. + On Windows, a symlink represents either a file or a directory, and does not morph to the target dynamically. If the target is present, the type of the symlink will be created to match. Otherwise, the symlink will be created @@ -3603,7 +3974,8 @@ features: where each member is an int expressing nanoseconds. - If *times* is not ``None``, it must be a 2-tuple of the form ``(atime, mtime)`` - where each member is an int or float expressing seconds. + where each member is a real number expressing seconds, + rounded down to nanoseconds. - If *times* is ``None`` and *ns* is unspecified, this is equivalent to specifying ``ns=(atime_ns, mtime_ns)`` where both times are the current time. @@ -3630,6 +4002,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + Accepts any real numbers as *times*, not only integers or floats. + .. function:: walk(top, topdown=True, onerror=None, followlinks=False) @@ -3760,9 +4135,9 @@ features: import os for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'): - print(root, "consumes", end="") + print(root, "consumes", end=" ") print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]), - end="") + end=" ") print("bytes in", len(files), "non-directory files") if '__pycache__' in dirs: dirs.remove('__pycache__') # don't visit __pycache__ directories @@ -4035,7 +4410,7 @@ Naturally, they are all only available on Linux. the timer will fire when the timer's clock (set by *clockid* in :func:`timerfd_create`) reaches *initial* seconds. - The timer's interval is set by the *interval* :py:class:`float`. + The timer's interval is set by the *interval* real number. If *interval* is zero, the timer only fires once, on the initial expiration. If *interval* is greater than zero, the timer fires every time *interval* seconds have elapsed since the previous expiration. @@ -4266,10 +4641,10 @@ to be ignored. .. function:: abort() - Generate a :const:`SIGABRT` signal to the current process. On Unix, the default + Generate a :const:`~signal.SIGABRT` signal to the current process. On Unix, the default behavior is to produce a core dump; on Windows, the process immediately returns an exit code of ``3``. Be aware that calling this function will not call the - Python signal handler registered for :const:`SIGABRT` with + Python signal handler registered for :const:`~signal.SIGABRT` with :func:`signal.signal`. @@ -4573,6 +4948,8 @@ written in Python, such as a mail server's external command delivery program. master end of the pseudo-terminal. For a more portable approach, use the :mod:`pty` module. If an error occurs :exc:`OSError` is raised. + The returned file descriptor *fd* is :ref:`non-inheritable `. + .. audit-event:: os.forkpty "" os.forkpty .. warning:: @@ -4589,6 +4966,9 @@ written in Python, such as a mail server's external command delivery program. threads, this now raises a :exc:`DeprecationWarning`. See the longer explanation on :func:`os.fork`. + .. versionchanged:: 3.15 + The returned file descriptor is now made non-inheritable. + .. availability:: Unix, not WASI, not Android, not iOS. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 7d7692dea5c..79e0b7f09ea 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -311,7 +311,7 @@ Pure paths provide the following methods and properties: .. attribute:: PurePath.parser The implementation of the :mod:`os.path` module used for low-level path - parsing and joining: either :mod:`posixpath` or :mod:`ntpath`. + parsing and joining: either :mod:`!posixpath` or :mod:`!ntpath`. .. versionadded:: 3.13 @@ -542,20 +542,6 @@ Pure paths provide the following methods and properties: Passing additional arguments is deprecated; if supplied, they are joined with *other*. -.. method:: PurePath.is_reserved() - - With :class:`PureWindowsPath`, return ``True`` if the path is considered - reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, - ``False`` is always returned. - - .. versionchanged:: 3.13 - Windows path names that contain a colon, or end with a dot or a space, - are considered reserved. UNC paths may be reserved. - - .. deprecated-removed:: 3.13 3.15 - This method is deprecated; use :func:`os.path.isreserved` to detect - reserved paths on Windows. - .. method:: PurePath.joinpath(*pathsegments) Calling this method is equivalent to combining the path with each of @@ -1781,9 +1767,12 @@ The following wildcards are supported in patterns for ``?`` Matches one non-separator character. ``[seq]`` - Matches one character in *seq*. + Matches one character in *seq*, where *seq* is a sequence of characters. + Range expressions are supported; for example, ``[a-z]`` matches any lowercase ASCII letter. + Multiple ranges can be combined: ``[a-zA-Z0-9_]`` matches any ASCII letter, digit, or underscore. + ``[!seq]`` - Matches one character not in *seq*. + Matches one character not in *seq*, where *seq* follows the same rules as above. For a literal match, wrap the meta-characters in brackets. For example, ``"[?]"`` matches the character ``"?"``. @@ -1982,7 +1971,7 @@ The :mod:`pathlib.types` module provides types for static type checking. If *follow_symlinks* is ``False``, return ``True`` only if the path is a file (without following symlinks); return ``False`` if the path - is a directory or other other non-file, or if it doesn't exist. + is a directory or other non-file, or if it doesn't exist. .. method:: is_symlink() diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index a0304edddf6..0bbdc425352 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -75,12 +75,17 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug arguments of the ``p`` command. +.. _pdb-cli: + +Command-line interface +---------------------- + .. program:: pdb You can also invoke :mod:`pdb` from the command line to debug other scripts. For example:: - python -m pdb [-c command] (-m module | pyfile) [args ...] + python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] When invoked as a module, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post-mortem debugging (or @@ -104,6 +109,24 @@ useful than quitting the debugger upon program's exit. .. versionchanged:: 3.7 Added the ``-m`` option. +.. option:: -p, --pid + + Attach to the process with the specified PID. + + .. versionadded:: 3.14 + + +To attach to a running Python process for remote debugging, use the ``-p`` or +``--pid`` option with the target process's PID:: + + python -m pdb -p 1234 + +.. note:: + + Attaching to a process that is blocked in a system call or waiting for I/O + will only work once the next bytecode instruction is executed or when the + process receives a signal. + Typical usage to execute a statement under control of the debugger is:: >>> import pdb @@ -315,7 +338,7 @@ access further features, you have to do this yourself: .. _debugger-commands: -Debugger Commands +Debugger commands ----------------- The commands recognized by the debugger are listed below. Most commands can be diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 007c9fe1b95..3a9b66ec7e7 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -732,8 +732,8 @@ or both. These items will be appended to the object either using ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. This is primarily used for list subclasses, but may be used by other - classes as long as they have - :ref:`append and extend methods ` with + classes as long as they have :meth:`~sequence.append` + and :meth:`~sequence.extend` methods with the appropriate signature. (Whether :meth:`!append` or :meth:`!extend` is used depends on which pickle protocol version is used as well as the number of items to append, so both must be supported.) diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 20b8f6bcf19..47d24b6f7d0 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -69,8 +69,8 @@ support. Yield :term:`finder` objects for the given module name. - If fullname contains a ``'.'``, the finders will be for the package - containing fullname, otherwise they will be all registered top level + If *fullname* contains a ``'.'``, the finders will be for the package + containing *fullname*, otherwise they will be all registered top level finders (i.e. those on both :data:`sys.meta_path` and :data:`sys.path_hooks`). If the named module is in a package, that package is imported as a side diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 5c999054323..d05c6e5a2aa 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -56,6 +56,8 @@ Cross platform Returns the machine type, e.g. ``'AMD64'``. An empty string is returned if the value cannot be determined. + The output is platform-dependent and may differ in casing and naming conventions. + .. function:: node() @@ -176,8 +178,8 @@ Cross platform :attr:`processor` is resolved late, on demand. Note: the first two attribute names differ from the names presented by - :func:`os.uname`, where they are named :attr:`sysname` and - :attr:`nodename`. + :func:`os.uname`, where they are named :attr:`!sysname` and + :attr:`!nodename`. Entries which cannot be determined are set to ``''``. @@ -187,23 +189,13 @@ Cross platform .. versionchanged:: 3.9 :attr:`processor` is resolved late instead of immediately. +.. function:: invalidate_caches() -Java platform -------------- + Clear out the internal cache of information, such as the :func:`uname`. + This is typically useful when the platform's :func:`node` is changed + by an external process and one needs to retrieve the updated value. - -.. function:: java_ver(release='', vendor='', vminfo=('','',''), osinfo=('','','')) - - Version interface for Jython. - - Returns a tuple ``(release, vendor, vminfo, osinfo)`` with *vminfo* being a - tuple ``(vm_name, vm_release, vm_vendor)`` and *osinfo* being a tuple - ``(os_name, os_version, os_arch)``. Values which cannot be determined are set to - the defaults given as parameters (which all default to ``''``). - - .. deprecated-removed:: 3.13 3.15 - It was largely untested, had a confusing API, - and was only useful for Jython support. + .. versionadded:: 3.14 Windows platform @@ -388,14 +380,3 @@ The following options are accepted: You can also pass one or more positional arguments (``terse``, ``nonaliased``) to explicitly control the output format. These behave similarly to their corresponding options. - -Miscellaneous -------------- - -.. function:: invalidate_caches() - - Clear out the internal cache of information, such as the :func:`uname`. - This is typically useful when the platform's :func:`node` is changed - by an external process and one needs to retrieve the updated value. - - .. versionadded:: 3.14 diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 2985f31bacb..f5189245079 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -22,8 +22,6 @@ The formatted representation keeps objects on a single line if it can, and breaks them onto multiple lines if they don't fit within the allowed width, adjustable by the *width* parameter defaulting to 80 characters. -Dictionaries are sorted by key before the display is computed. - .. versionchanged:: 3.9 Added support for pretty-printing :class:`types.SimpleNamespace`. diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index b6e51dffc40..03ad50b2c5e 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -4,7 +4,7 @@ The Python Profilers ******************** -**Source code:** :source:`Lib/profile.py` and :source:`Lib/pstats.py` +**Source code:** :source:`Lib/profile.py`, :source:`Lib/pstats.py`, and :source:`Lib/profile/sample.py` -------------- @@ -14,23 +14,32 @@ Introduction to the profilers ============================= .. index:: + single: statistical profiling + single: profiling, statistical single: deterministic profiling single: profiling, deterministic -:mod:`cProfile` and :mod:`profile` provide :dfn:`deterministic profiling` of +Python provides both :dfn:`statistical profiling` and :dfn:`deterministic profiling` of Python programs. A :dfn:`profile` is a set of statistics that describes how often and for how long various parts of the program executed. These statistics can be formatted into reports via the :mod:`pstats` module. -The Python standard library provides two different implementations of the same -profiling interface: +The Python standard library provides three different profiling implementations: -1. :mod:`cProfile` is recommended for most users; it's a C extension with +**Statistical Profiler:** + +1. :mod:`!profiling.sampling` provides statistical profiling of running Python processes + using periodic stack sampling. It can attach to any running Python process without + requiring code modification or restart, making it ideal for production debugging. + +**Deterministic Profilers:** + +2. :mod:`cProfile` is recommended for development and testing; it's a C extension with reasonable overhead that makes it suitable for profiling long-running programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted Czotter. -2. :mod:`profile`, a pure Python module whose interface is imitated by +3. :mod:`profile`, a pure Python module whose interface is imitated by :mod:`cProfile`, but which adds significant overhead to profiled programs. If you're trying to extend the profiler in some way, the task might be easier with this module. Originally designed and written by Jim Roskind. @@ -44,6 +53,77 @@ profiling interface: but not for C-level functions, and so the C code would seem faster than any Python one. +**Profiler Comparison:** + ++-------------------+--------------------------+----------------------+----------------------+ +| Feature | Statistical | Deterministic | Deterministic | +| | (``profiling.sampling``) | (``cProfile``) | (``profile``) | ++===================+==========================+======================+======================+ +| **Target** | Running process | Code you run | Code you run | ++-------------------+--------------------------+----------------------+----------------------+ +| **Overhead** | Virtually none | Moderate | High | ++-------------------+--------------------------+----------------------+----------------------+ +| **Accuracy** | Statistical approx. | Exact call counts | Exact call counts | ++-------------------+--------------------------+----------------------+----------------------+ +| **Setup** | Attach to any PID | Instrument code | Instrument code | ++-------------------+--------------------------+----------------------+----------------------+ +| **Use Case** | Production debugging | Development/testing | Profiler extension | ++-------------------+--------------------------+----------------------+----------------------+ +| **Implementation**| C extension | C extension | Pure Python | ++-------------------+--------------------------+----------------------+----------------------+ + +.. note:: + + The statistical profiler (:mod:`!profiling.sampling`) is recommended for most production + use cases due to its extremely low overhead and ability to profile running processes + without modification. It can attach to any Python process and collect performance + data with minimal impact on execution speed, making it ideal for debugging + performance issues in live applications. + + +.. _statistical-profiling: + +What Is Statistical Profiling? +============================== + +:dfn:`Statistical profiling` works by periodically interrupting a running +program to capture its current call stack. Rather than monitoring every +function entry and exit like deterministic profilers, it takes snapshots at +regular intervals to build a statistical picture of where the program spends +its time. + +The sampling profiler uses process memory reading (via system calls like +``process_vm_readv`` on Linux, ``vm_read`` on macOS, and ``ReadProcessMemory`` on +Windows) to attach to a running Python process and extract stack trace +information without requiring any code modification or restart of the target +process. This approach provides several key advantages over traditional +profiling methods. + +The fundamental principle is that if a function appears frequently in the +collected stack samples, it is likely consuming significant CPU time. By +analyzing thousands of samples, the profiler can accurately estimate the +relative time spent in different parts of the program. The statistical nature +means that while individual measurements may vary, the aggregate results +converge to represent the true performance characteristics of the application. + +Since statistical profiling operates externally to the target process, it +introduces virtually no overhead to the running program. The profiler process +runs separately and reads the target process memory without interrupting its +execution. This makes it suitable for profiling production systems where +performance impact must be minimized. + +The accuracy of statistical profiling improves with the number of samples +collected. Short-lived functions may be missed or underrepresented, while +long-running functions will be captured proportionally to their execution time. +This characteristic makes statistical profiling particularly effective for +identifying the most significant performance bottlenecks rather than providing +exhaustive coverage of all function calls. + +Statistical profiling excels at answering questions like "which functions +consume the most CPU time?" and "where should I focus optimization efforts?" +rather than "exactly how many times was this function called?" The trade-off +between precision and practicality makes it an invaluable tool for performance +analysis in real-world applications. .. _profile-instant: @@ -54,6 +134,18 @@ This section is provided for users that "don't want to read the manual." It provides a very brief overview, and allows a user to rapidly perform profiling on an existing application. +**Statistical Profiling (Recommended for Production):** + +To profile an existing running process:: + + python -m profiling.sampling 1234 + +To profile with custom settings:: + + python -m profiling.sampling -i 50 -d 30 1234 + +**Deterministic Profiling (Development/Testing):** + To profile a function that takes a single argument, you can do:: import cProfile @@ -121,8 +213,143 @@ results to a file by specifying a filename to the :func:`run` function:: The :class:`pstats.Stats` class reads profile results from a file and formats them in various ways. +.. _sampling-profiler-cli: + +Statistical Profiler Command Line Interface +=========================================== + +.. program:: profiling.sampling + +The :mod:`!profiling.sampling` module can be invoked as a script to profile running processes:: + + python -m profiling.sampling [options] PID + +**Basic Usage Examples:** + +Profile process 1234 for 10 seconds with default settings:: + + python -m profiling.sampling 1234 + +Profile with custom interval and duration, save to file:: + + python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234 + +Generate collapsed stacks to use with tools like `flamegraph.pl +`_:: + + python -m profiling.sampling --collapsed 1234 + +Profile all threads, sort by total time:: + + python -m profiling.sampling -a --sort-tottime 1234 + +Profile with real-time sampling statistics:: + + python -m profiling.sampling --realtime-stats 1234 + +**Command Line Options:** + +.. option:: PID + + Process ID of the Python process to profile (required) + +.. option:: -i, --interval INTERVAL + + Sampling interval in microseconds (default: 100) + +.. option:: -d, --duration DURATION + + Sampling duration in seconds (default: 10) + +.. option:: -a, --all-threads + + Sample all threads in the process instead of just the main thread + +.. option:: --native + + Include artificial ```` frames to denote calls to non-Python code. + +.. option:: --no-gc + + Don't include artificial ```` frames to denote active garbage collection. + +.. option:: --realtime-stats + + Print real-time sampling statistics during profiling + +.. option:: --pstats + + Generate pstats output (default) + +.. option:: --collapsed + + Generate collapsed stack traces for flamegraphs + +.. option:: -o, --outfile OUTFILE + + Save output to a file + +**Sorting Options (pstats format only):** + +.. option:: --sort-nsamples + + Sort by number of direct samples + +.. option:: --sort-tottime + + Sort by total time + +.. option:: --sort-cumtime + + Sort by cumulative time (default) + +.. option:: --sort-sample-pct + + Sort by sample percentage + +.. option:: --sort-cumul-pct + + Sort by cumulative sample percentage + +.. option:: --sort-nsamples-cumul + + Sort by cumulative samples + +.. option:: --sort-name + + Sort by function name + +.. option:: -l, --limit LIMIT + + Limit the number of rows in the output (default: 15) + +.. option:: --no-summary + + Disable the summary section in the output + +**Understanding Statistical Profile Output:** + +The statistical profiler produces output similar to deterministic profilers but with different column meanings:: + + Profile Stats: + nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function) + 45/67 12.5 23.450 18.6 56.780 mymodule.py:42(process_data) + 23/23 6.4 15.230 6.4 15.230 :0(len) + +**Column Meanings:** + +- **nsamples**: ``direct/cumulative`` - Times function was directly executing / on call stack +- **sample%**: Percentage of total samples where function was directly executing +- **tottime**: Estimated time spent directly in this function +- **cumul%**: Percentage of samples where function was anywhere on call stack +- **cumtime**: Estimated cumulative time including called functions +- **filename:lineno(function)**: Location and name of the function + .. _profile-cli: +Deterministic Profiler Command Line Interface +============================================= + .. program:: cProfile The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to @@ -499,7 +726,7 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. significant entries. Initially, the list is taken to be the complete set of profiled functions. Each restriction is either an integer (to select a count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a string that will interpreted as a + select a percentage of lines), or a string that will be interpreted as a regular expression (to pattern match the standard name that is printed). If several restrictions are provided, then they are applied sequentially. For example:: @@ -564,7 +791,7 @@ What Is Deterministic Profiling? call*, *function return*, and *exception* events are monitored, and precise timings are made for the intervals between these events (during which time the user's code is executing). In contrast, :dfn:`statistical profiling` (which is -not done by this module) randomly samples the effective instruction pointer, and +provided by the :mod:`!profiling.sampling` module) periodically samples the effective instruction pointer, and deduces where time is being spent. The latter technique traditionally involves less overhead (as the code does not need to be instrumented), but provides only relative indications of where time is being spent. diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 1a44bb13a84..2912c9e16c6 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -33,9 +33,14 @@ The :mod:`pty` module defines the following functions: file descriptor connected to the child's controlling terminal (and also to the child's standard input and output). + The returned file descriptor *fd* is :ref:`non-inheritable `. + .. warning:: On macOS the use of this function is unsafe when mixed with using higher-level system APIs, and that includes using :mod:`urllib.request`. + .. versionchanged:: 3.15 + The returned file descriptor is now made non-inheritable. + .. function:: openpty() diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 2d57cff10a9..2f5db81955c 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -16,11 +16,10 @@ references to these attributes should be marked using the :member: role. -.. warning:: +.. note:: - The :mod:`pyexpat` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. index:: single: Expat @@ -73,6 +72,13 @@ The :mod:`xml.parsers.expat` module contains two functions: *encoding* [1]_ is given it will override the implicit or explicit encoding of the document. + .. _xmlparser-non-root: + + Parsers created through :func:`!ParserCreate` are called "root" parsers, + in the sense that they do not have any parent parser attached. Non-root + parsers are created by :meth:`parser.ExternalEntityParserCreate + `. + Expat can optionally do XML namespace processing for you, enabled by providing a value for *namespace_separator*. The value must be a one-character string; a :exc:`ValueError` will be raised if the string has an illegal length (``None`` @@ -217,10 +223,10 @@ XMLParser Objects Calling ``SetReparseDeferralEnabled(True)`` allows re-enabling reparse deferral. - Note that :meth:`SetReparseDeferralEnabled` has been backported to some - prior releases of CPython as a security fix. Check for availability of - :meth:`SetReparseDeferralEnabled` using :func:`hasattr` if used in code - running across a variety of Python versions. + :meth:`!SetReparseDeferralEnabled` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. .. versionadded:: 3.13 @@ -232,6 +238,131 @@ XMLParser Objects .. versionadded:: 3.13 +:class:`!xmlparser` objects have the following methods to tune protections +against some common XML vulnerabilities. + +.. method:: xmlparser.SetBillionLaughsAttackProtectionActivationThreshold(threshold, /) + + Sets the number of output bytes needed to activate protection against + `billion laughs`_ attacks. + + The number of output bytes includes amplification from entity expansion + and reading DTD files. + + Parser objects usually have a protection activation threshold of 8 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetBillionLaughsAttackProtectionActivationThreshold` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + Activation thresholds below 4 MiB are known to break support for DITA 1.3 + payload and are hence not recommended. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor, /) + + Sets the maximum tolerated amplification factor for protection against + `billion laughs`_ attacks. + + The amplification factor is calculated as ``(direct + indirect) / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``indirect`` is the number of + bytes added by expanding entities and reading of external DTD files. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Peak amplifications of factor 15,000 for the entire payload + and of factor 30,000 in the middle of parsing have been observed with small + benign files in practice. In particular, the activation threshold should be + carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetBillionLaughsAttackProtectionMaximumAmplification` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetBillionLaughsAttackProtectionActivationThreshold` + is exceeded. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetAllocTrackerActivationThreshold(threshold, /) + + Sets the number of allocated bytes of dynamic memory needed to activate + protection against disproportionate use of RAM. + + Parser objects usually have an allocation activation threshold of 64 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetAllocTrackerActivationThreshold` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. versionadded:: 3.15 + +.. method:: xmlparser.SetAllocTrackerMaximumAmplification(max_factor, /) + + Sets the maximum amplification factor between direct input and bytes + of dynamic memory allocated. + + The amplification factor is calculated as ``allocated / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``allocated`` is the number + of bytes of dynamic memory allocated in the parser hierarchy. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Amplification factors greater than 100.0 can be observed + near the start of parsing even with benign files in practice. In particular, + the activation threshold should be carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + :meth:`!SetAllocTrackerMaximumAmplification` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetAllocTrackerActivationThreshold` + is exceeded. + + .. versionadded:: 3.15 + + :class:`xmlparser` objects have the following attributes: @@ -503,6 +634,15 @@ otherwise stated. .. method:: xmlparser.ExternalEntityRefHandler(context, base, systemId, publicId) + .. warning:: + + Implementing a handler that accesses local files and/or the network + may create a vulnerability to + `external entity attacks `_ + if :class:`xmlparser` is used with user-provided XML content. + Please reflect on your `threat model `_ + before implementing this handler. + Called for references to external entities. *base* is the current base, as set by a previous call to :meth:`SetBase`. The public and system identifiers, *systemId* and *publicId*, are strings if given; if the public identifier is not @@ -955,3 +1095,6 @@ The ``errors`` module has the following attributes: not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. + +.. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack +.. |xml-non-root-parser| replace:: :ref:`non-root ` diff --git a/Doc/library/python.rst b/Doc/library/python.rst index c2c231af7c3..c5c762e11b9 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -27,3 +27,8 @@ overview: inspect.rst annotationlib.rst site.rst + +.. seealso:: + + * See the :mod:`concurrent.interpreters` module, which similarly + exposes core runtime functionality. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index fbbebcf4ed8..1b75582f0cf 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -187,9 +187,6 @@ fully processed by daemon consumer threads. processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put` into the queue). - ``shutdown(immediate=True)`` calls :meth:`task_done` for each remaining item - in the queue. - Raises a :exc:`ValueError` if called more times than there were items placed in the queue. @@ -204,6 +201,9 @@ fully processed by daemon consumer threads. count of unfinished tasks drops to zero, :meth:`join` unblocks. +Waiting for task completion +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Example of how to wait for enqueued tasks to be completed:: import threading @@ -233,22 +233,39 @@ Example of how to wait for enqueued tasks to be completed:: Terminating queues ^^^^^^^^^^^^^^^^^^ -:class:`Queue` objects can be made to prevent further interaction by shutting -them down. +When no longer needed, :class:`Queue` objects can be wound down +until empty or terminated immediately with a hard shutdown. .. method:: Queue.shutdown(immediate=False) - Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` raise - :exc:`ShutDown`. + Put a :class:`Queue` instance into a shutdown mode. - By default, :meth:`~Queue.get` on a shut down queue will only raise once the - queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise - immediately instead. + The queue can no longer grow. + Future calls to :meth:`~Queue.put` raise :exc:`ShutDown`. + Currently blocked callers of :meth:`~Queue.put` will be unblocked + and will raise :exc:`ShutDown` in the formerly blocked thread. - All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` will be - unblocked. If *immediate* is true, a task will be marked as done for each - remaining item in the queue, which may unblock callers of - :meth:`~Queue.join`. + If *immediate* is false (the default), the queue can be wound + down normally with :meth:`~Queue.get` calls to extract tasks + that have already been loaded. + + And if :meth:`~Queue.task_done` is called for each remaining task, a + pending :meth:`~Queue.join` will be unblocked normally. + + Once the queue is empty, future calls to :meth:`~Queue.get` will + raise :exc:`ShutDown`. + + If *immediate* is true, the queue is terminated immediately. + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` + are unblocked and will raise :exc:`ShutDown` because the + queue is empty. + + Use caution when using :meth:`~Queue.join` with *immediate* set + to true. This unblocks the join even when no work has been done + on the tasks, violating the usual invariant for joining a queue. .. versionadded:: 3.13 diff --git a/Doc/library/random.rst b/Doc/library/random.rst index ef0cfb0e76c..4e55e301b89 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -447,6 +447,11 @@ Alternative Generator Override this method in subclasses to customise the :meth:`~random.getrandbits` behaviour of :class:`!Random` instances. + .. method:: Random.randbytes(n) + + Override this method in subclasses to customise the + :meth:`~random.randbytes` behaviour of :class:`!Random` instances. + .. class:: SystemRandom([seed]) @@ -625,7 +630,8 @@ Recipes ------- These recipes show how to efficiently make random selections -from the combinatoric iterators in the :mod:`itertools` module: +from the combinatoric iterators in the :mod:`itertools` module +or the :pypi:`more-itertools` project: .. testcode:: import random @@ -656,6 +662,17 @@ from the combinatoric iterators in the :mod:`itertools` module: indices = sorted(random.choices(range(n), k=r)) return tuple(pool[i] for i in indices) + def random_derangement(iterable): + "Choose a permutation where no element is in its original position." + seq = tuple(iterable) + if len(seq) < 2: + raise ValueError('derangements require at least two values') + perm = list(seq) + while True: + random.shuffle(perm) + if all(p != q for p, q in zip(seq, perm)): + return tuple(perm) + The default :func:`.random` returns multiples of 2⁻⁵³ in the range *0.0 ≤ x < 1.0*. All such numbers are evenly spaced and are exactly representable as Python floats. However, many other representable diff --git a/Doc/library/re.rst b/Doc/library/re.rst index eb3b1e5549c..75ebbf11c8e 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -991,8 +991,8 @@ Functions That way, separator components are always found at the same relative indices within the result list. - Empty matches for the pattern split the string only when not adjacent - to a previous empty match. + Adjacent empty matches are not possible, but an empty match can occur + immediately after a non-empty match. .. code:: pycon @@ -1095,9 +1095,12 @@ Functions The optional argument *count* is the maximum number of pattern occurrences to be replaced; *count* must be a non-negative integer. If omitted or zero, all - occurrences will be replaced. Empty matches for the pattern are replaced only - when not adjacent to a previous empty match, so ``sub('x*', '-', 'abxd')`` returns - ``'-a-b--d-'``. + occurrences will be replaced. + + Adjacent empty matches are not possible, but an empty match can occur + immediately after a non-empty match. + As a result, ``sub('x*', '-', 'abxd')`` returns ``'-a-b--d-'`` + instead of ``'-a-b-d-'``. .. index:: single: \g; in regular expressions @@ -1128,8 +1131,7 @@ Functions .. versionchanged:: 3.7 Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now are errors. - Empty matches for the pattern are replaced when adjacent to a previous - non-empty match. + An empty match can occur immediately after a non-empty match. .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index f649fce5efc..780cc773403 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -26,6 +26,8 @@ Readline library in general. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + .. note:: The underlying Readline library API may be implemented by @@ -244,6 +246,15 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. function:: get_pre_input_hook() + + Get the current pre-input hook function, or ``None`` if no pre-input hook + function has been set. This function only exists if Python was compiled + for a version of the library that supports it. + + .. versionadded:: next + + .. _readline-completion: Completion diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 0515d205bbc..c58dc4243ec 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -51,6 +51,20 @@ this module for those platforms. Constant used to represent the limit for an unlimited resource. + .. versionchanged:: 3.15 + It is now always positive. + Previously, it could be negative, such as -1 or -3. + + +.. data:: RLIM_SAVED_CUR +.. data:: RLIM_SAVED_MAX + + Constants used to represent the soft and hard limit values if they + cannot be represented in the ``rlim_t`` value in C. + Can be equal to :data:`RLIM_INFINITY`. + + .. versionadded:: 3.15 + .. function:: getrlimit(resource) @@ -63,12 +77,12 @@ this module for those platforms. Sets new limits of consumption of *resource*. The *limits* argument must be a tuple ``(soft, hard)`` of two integers describing the new limits. A value of - :data:`~resource.RLIM_INFINITY` can be used to request a limit that is + :const:`~resource.RLIM_INFINITY` can be used to request a limit that is unlimited. Raises :exc:`ValueError` if an invalid resource is specified, if the new soft limit exceeds the hard limit, or if a process tries to raise its hard limit. - Specifying a limit of :data:`~resource.RLIM_INFINITY` when the hard or + Specifying a limit of :const:`~resource.RLIM_INFINITY` when the hard or system limit for that resource is not unlimited will result in a :exc:`ValueError`. A process with the effective UID of super-user can request any valid limit value, including unlimited, but :exc:`ValueError` @@ -78,7 +92,7 @@ this module for those platforms. ``setrlimit`` may also raise :exc:`error` if the underlying system call fails. - VxWorks only supports setting :data:`RLIMIT_NOFILE`. + VxWorks only supports setting :const:`RLIMIT_NOFILE`. .. audit-event:: resource.setrlimit resource,limits resource.setrlimit @@ -127,7 +141,7 @@ platform. .. data:: RLIMIT_CPU The maximum amount of processor time (in seconds) that a process can use. If - this limit is exceeded, a :const:`SIGXCPU` signal is sent to the process. (See + this limit is exceeded, a :const:`~signal.SIGXCPU` signal is sent to the process. (See the :mod:`signal` module documentation for information about how to catch this signal and do something useful, e.g. flush open files to disk.) @@ -176,8 +190,9 @@ platform. .. data:: RLIMIT_VMEM The largest area of mapped memory which the process may occupy. + Usually an alias of :const:`RLIMIT_AS`. - .. availability:: FreeBSD >= 11. + .. availability:: Solaris, FreeBSD, NetBSD. .. data:: RLIMIT_AS @@ -230,16 +245,18 @@ platform. .. versionadded:: 3.4 + .. data:: RLIMIT_SBSIZE The maximum size (in bytes) of socket buffer usage for this user. This limits the amount of network memory, and hence the amount of mbufs, that this user may hold at any time. - .. availability:: FreeBSD. + .. availability:: FreeBSD, NetBSD. .. versionadded:: 3.4 + .. data:: RLIMIT_SWAP The maximum size (in bytes) of the swap space that may be reserved or @@ -249,18 +266,20 @@ platform. `tuning(7) `__ for a complete description of this sysctl. - .. availability:: FreeBSD. + .. availability:: FreeBSD >= 8. .. versionadded:: 3.4 + .. data:: RLIMIT_NPTS The maximum number of pseudo-terminals created by this user id. - .. availability:: FreeBSD. + .. availability:: FreeBSD >= 8. .. versionadded:: 3.4 + .. data:: RLIMIT_KQUEUES The maximum number of kqueues this user id is allowed to create. @@ -269,6 +288,46 @@ platform. .. versionadded:: 3.10 + +.. data:: RLIMIT_NTHR + + The maximum number of threads for this user id, not counting the main + and kernel threads. + + .. availability:: NetBSD >= 7.0. + + .. versionadded:: 3.15 + + +.. data:: RLIMIT_PIPEBUF + + The maximum total size of in-kernel buffers for bi-directional pipes/fifos + that this user id is allowed to consume. + + .. availability:: FreeBSD >= 14.2. + + .. versionadded:: 3.15 + + +.. data:: RLIMIT_THREADS + + The maximum number of threads each process can create. + + .. availability:: AIX. + + .. versionadded:: 3.15 + + +.. data:: RLIMIT_UMTXP + + The limit of the number of process-shared Posix thread library objects + allocated by user id. + + .. availability:: FreeBSD >= 11. + + .. versionadded:: 3.15 + + Resource Usage -------------- @@ -304,47 +363,47 @@ These functions are used to retrieve resource usage information: For backward compatibility, the return value is also accessible as a tuple of 16 elements. - The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are + The fields :attr:`!ru_utime` and :attr:`!ru_stime` of the return value are floating-point values representing the amount of time spent executing in user mode and the amount of time spent executing in system mode, respectively. The remaining values are integers. Consult the :manpage:`getrusage(2)` man page for detailed information about these values. A brief summary is presented here: - +--------+---------------------+---------------------------------------+ - | Index | Field | Resource | - +========+=====================+=======================================+ - | ``0`` | :attr:`ru_utime` | time in user mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``1`` | :attr:`ru_stime` | time in system mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``2`` | :attr:`ru_maxrss` | maximum resident set size | - +--------+---------------------+---------------------------------------+ - | ``3`` | :attr:`ru_ixrss` | shared memory size | - +--------+---------------------+---------------------------------------+ - | ``4`` | :attr:`ru_idrss` | unshared memory size | - +--------+---------------------+---------------------------------------+ - | ``5`` | :attr:`ru_isrss` | unshared stack size | - +--------+---------------------+---------------------------------------+ - | ``6`` | :attr:`ru_minflt` | page faults not requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``7`` | :attr:`ru_majflt` | page faults requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``8`` | :attr:`ru_nswap` | number of swap outs | - +--------+---------------------+---------------------------------------+ - | ``9`` | :attr:`ru_inblock` | block input operations | - +--------+---------------------+---------------------------------------+ - | ``10`` | :attr:`ru_oublock` | block output operations | - +--------+---------------------+---------------------------------------+ - | ``11`` | :attr:`ru_msgsnd` | messages sent | - +--------+---------------------+---------------------------------------+ - | ``12`` | :attr:`ru_msgrcv` | messages received | - +--------+---------------------+---------------------------------------+ - | ``13`` | :attr:`ru_nsignals` | signals received | - +--------+---------------------+---------------------------------------+ - | ``14`` | :attr:`ru_nvcsw` | voluntary context switches | - +--------+---------------------+---------------------------------------+ - | ``15`` | :attr:`ru_nivcsw` | involuntary context switches | - +--------+---------------------+---------------------------------------+ + +--------+----------------------+---------------------------------------+ + | Index | Field | Resource | + +========+======================+=======================================+ + | ``0`` | :attr:`!ru_utime` | time in user mode (float seconds) | + +--------+----------------------+---------------------------------------+ + | ``1`` | :attr:`!ru_stime` | time in system mode (float seconds) | + +--------+----------------------+---------------------------------------+ + | ``2`` | :attr:`!ru_maxrss` | maximum resident set size | + +--------+----------------------+---------------------------------------+ + | ``3`` | :attr:`!ru_ixrss` | shared memory size | + +--------+----------------------+---------------------------------------+ + | ``4`` | :attr:`!ru_idrss` | unshared memory size | + +--------+----------------------+---------------------------------------+ + | ``5`` | :attr:`!ru_isrss` | unshared stack size | + +--------+----------------------+---------------------------------------+ + | ``6`` | :attr:`!ru_minflt` | page faults not requiring I/O | + +--------+----------------------+---------------------------------------+ + | ``7`` | :attr:`!ru_majflt` | page faults requiring I/O | + +--------+----------------------+---------------------------------------+ + | ``8`` | :attr:`!ru_nswap` | number of swap outs | + +--------+----------------------+---------------------------------------+ + | ``9`` | :attr:`!ru_inblock` | block input operations | + +--------+----------------------+---------------------------------------+ + | ``10`` | :attr:`!ru_oublock` | block output operations | + +--------+----------------------+---------------------------------------+ + | ``11`` | :attr:`!ru_msgsnd` | messages sent | + +--------+----------------------+---------------------------------------+ + | ``12`` | :attr:`!ru_msgrcv` | messages received | + +--------+----------------------+---------------------------------------+ + | ``13`` | :attr:`!ru_nsignals` | signals received | + +--------+----------------------+---------------------------------------+ + | ``14`` | :attr:`!ru_nvcsw` | voluntary context switches | + +--------+----------------------+---------------------------------------+ + | ``15`` | :attr:`!ru_nivcsw` | involuntary context switches | + +--------+----------------------+---------------------------------------+ This function will raise a :exc:`ValueError` if an invalid *who* parameter is specified. It may also raise :exc:`error` exception in unusual circumstances. diff --git a/Doc/library/security_warnings.rst b/Doc/library/security_warnings.rst index a573c98f73e..70c359cc1c0 100644 --- a/Doc/library/security_warnings.rst +++ b/Doc/library/security_warnings.rst @@ -28,7 +28,7 @@ The following modules have specific security considerations: ` * :mod:`tempfile`: :ref:`mktemp is deprecated due to vulnerability to race conditions ` -* :mod:`xml`: :ref:`XML vulnerabilities ` +* :mod:`xml`: :ref:`XML security ` * :mod:`zipfile`: :ref:`maliciously prepared .zip files can cause disk volume exhaustion ` diff --git a/Doc/library/select.rst b/Doc/library/select.rst index d2094283d54..62b5161fb80 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -115,7 +115,7 @@ The module defines the following: :ref:`kevent-objects` below for the methods supported by kevent objects. -.. function:: select(rlist, wlist, xlist[, timeout]) +.. function:: select(rlist, wlist, xlist, timeout=None) This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either @@ -129,8 +129,9 @@ The module defines the following: Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The - optional *timeout* argument specifies a time-out as a floating-point number - in seconds. When the *timeout* argument is omitted the function blocks until + optional *timeout* argument specifies a time-out in seconds; it may be + a non-integer to specify fractions of seconds. + When the *timeout* argument is omitted or ``None``, the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. @@ -164,6 +165,9 @@ The module defines the following: :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. data:: PIPE_BUF @@ -270,6 +274,9 @@ object. :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _epoll-objects: @@ -368,7 +375,9 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.poll(timeout=None, maxevents=-1) - Wait for events. timeout in seconds (float) + Wait for events. + If *timeout* is given, it specifies the length of time in seconds + (may be non-integer) which the system will wait for events before returning. .. versionchanged:: 3.5 The function is now retried with a recomputed timeout when interrupted by @@ -376,6 +385,9 @@ Edge and Level Trigger Polling (epoll) Objects :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _poll-objects: @@ -464,6 +476,9 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _kqueue-objects: @@ -496,7 +511,7 @@ Kqueue Objects - changelist must be an iterable of kevent objects or ``None`` - max_events must be 0 or a positive integer - - timeout in seconds (floats possible); the default is ``None``, + - timeout in seconds (non-integers are possible); the default is ``None``, to wait forever .. versionchanged:: 3.5 @@ -505,6 +520,9 @@ Kqueue Objects :pep:`475` for the rationale), instead of raising :exc:`InterruptedError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _kevent-objects: diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 6e74a59b82b..51bae2fce30 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -17,7 +17,8 @@ This includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are ordinary strings. -.. function:: open(filename, flag='c', protocol=None, writeback=False) +.. function:: open(filename, flag='c', protocol=None, writeback=False, *, \ + serializer=None, deserializer=None) Open a persistent dictionary. The filename specified is the base filename for the underlying database. As a side-effect, an extension may be added to the @@ -41,6 +42,21 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). + By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` + for serializing and deserializing. This can be changed by supplying + *serializer* and *deserializer*, respectively. + + The *serializer* argument must be a callable which takes an object ``obj`` + and the *protocol* as inputs and returns the representation ``obj`` as a + :term:`bytes-like object`; the *protocol* value may be ignored by the + serializer. + + The *deserializer* argument must be callable which takes a serialized object + given as a :class:`bytes` object and returns the corresponding object. + + A :exc:`ShelveError` is raised if *serializer* is given but *deserializer* + is not, or vice-versa. + .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. @@ -48,6 +64,10 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. + .. versionchanged:: 3.15 + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. + .. note:: Do not rely on the shelf being closed automatically; always call @@ -75,8 +95,15 @@ Two additional methods are supported: Write back all entries in the cache if the shelf was opened with *writeback* set to :const:`True`. Also empty the cache and synchronize the persistent - dictionary on disk, if feasible. This is called automatically when the shelf - is closed with :meth:`close`. + dictionary on disk, if feasible. This is called automatically when + :meth:`reorganize` is called or the shelf is closed with :meth:`close`. + +.. method:: Shelf.reorganize() + + Calls :meth:`sync` and attempts to shrink space used on disk by removing empty + space resulting from deletions. + + .. versionadded:: 3.15 .. method:: Shelf.close() @@ -116,8 +143,14 @@ Restrictions * On macOS :mod:`dbm.ndbm` can silently corrupt the database file on updates, which can cause hard crashes when trying to read from the database. +* :meth:`Shelf.reorganize` may not be available for all database packages and + may temporarily increase resource usage (especially disk space) when called. + Additionally, it will never run automatically and instead needs to be called + explicitly. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') + +.. class:: Shelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', *, serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -135,6 +168,9 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. + The *serializer* and *deserializer* parameters have the same interpretation + as in :func:`~shelve.open`. + A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. @@ -149,8 +185,13 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. + .. versionchanged:: 3.15 + Added the *serializer* and *deserializer* parameters. -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') + +.. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', *, \ + serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -160,18 +201,27 @@ Restrictions modules. The *dict* object passed to the constructor must support those methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The - optional *protocol*, *writeback*, and *keyencoding* parameters have the same - interpretation as for the :class:`Shelf` class. + optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* + parameters have the same interpretation as in :func:`~shelve.open`. + + .. versionchanged:: 3.15 + Added the *serializer* and *deserializer* parameters. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ + writeback=False, *, serializer=None, \ + deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By default, the file will be created and opened for both read and write. The - optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol* and *writeback* parameters have the same - interpretation as for the :class:`Shelf` class. + optional *flag* parameter has the same interpretation as for the + :func:`.open` function. The optional *protocol*, *writeback*, *serializer* + and *deserializer* parameters have the same interpretation as in + :func:`~shelve.open`. + + .. versionchanged:: 3.15 + Added the *serializer* and *deserializer* parameters. .. _shelve-example: @@ -213,6 +263,20 @@ object):: d.close() # close it +Exceptions +---------- + +.. exception:: ShelveError + + Exception raised when one of the arguments *deserializer* and *serializer* + is missing in the :func:`~shelve.open`, :class:`Shelf`, :class:`BsdDbShelf` + and :class:`DbfilenameShelf`. + + The *deserializer* and *serializer* arguments must be given together. + + .. versionadded:: 3.15 + + .. seealso:: Module :mod:`dbm` diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 2cbf95bcf53..3a4631e7c65 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -47,6 +47,13 @@ Directory and files operations 0, only the contents from the current file position to the end of the file will be copied. + :func:`copyfileobj` will *not* guarantee that the destination stream has + been flushed on completion of the copy. If you want to read from the + destination at the completion of the copy operation (for example, reading + the contents of a temporary file that has been copied from a HTTP stream), + you must ensure that you have called :func:`~io.IOBase.flush` or + :func:`~io.IOBase.close` on the file-like object before attempting to read + the destination file. .. function:: copyfile(src, dst, *, follow_symlinks=True) @@ -83,6 +90,13 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. +.. exception:: SpecialFileError + + This exception is raised when :func:`copyfile` or :func:`copytree` attempt + to copy a named pipe. + + .. versionadded:: 2.7 + .. exception:: SameFileError This exception is raised if source and destination in :func:`copyfile` @@ -327,6 +341,10 @@ Directory and files operations The deprecated *onerror* is similar to *onexc*, except that the third parameter it receives is the tuple returned from :func:`sys.exc_info`. + .. seealso:: + :ref:`shutil-rmtree-example` for an example of handling the removal + of a directory tree that contains read-only files. + .. audit-event:: shutil.rmtree path,dir_fd shutil.rmtree .. versionchanged:: 3.3 @@ -454,6 +472,10 @@ Directory and files operations :envvar:`PATH` environment variable is read from :data:`os.environ`, falling back to :data:`os.defpath` if it is not set. + If *cmd* contains a directory component, :func:`!which` only checks the + specified path directly and does not search the directories listed in + *path* or in the system's :envvar:`PATH` environment variable. + On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the Windows API ``NeedCurrentDirectoryForExePathW`` will be consulted to @@ -603,7 +625,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is - available), or "xztar" (if the :mod:`lzma` module is available). + available), "xztar" (if the :mod:`lzma` module is available), or "zstdtar" + (if the :mod:`compression.zstd` module is available). *root_dir* is a directory that will be the root directory of the archive, all paths in the archive will be relative to it; for example, @@ -658,6 +681,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). + - *zstdtar*: Zstandard compressed tar-file (if the :mod:`compression.zstd` + module is available). You can register new formats or provide your own archiver for any existing formats, by using :func:`register_archive_format`. @@ -701,8 +726,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. *extract_dir* is the name of the target directory where the archive is unpacked. If not provided, the current working directory is used. - *format* is the archive format: one of "zip", "tar", "gztar", "bztar", or - "xztar". Or any other format registered with + *format* is the archive format: one of "zip", "tar", "gztar", "bztar", + "xztar", or "zstdtar". Or any other format registered with :func:`register_unpack_format`. If not provided, :func:`unpack_archive` will use the archive file name extension and see if an unpacker was registered for that extension. In case none is found, @@ -774,6 +799,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). + - *zstdtar*: Zstandard compressed tar-file (if the :mod:`compression.zstd` + module is available). You can register new formats or provide your own unpacker for any existing formats, by using :func:`register_unpack_format`. @@ -840,7 +867,7 @@ In the final archive, :file:`please_add.txt` should be included, but ... root_dir='tmp/root', ... base_dir='structure/content', ... ) - '/Users/tarek/my_archive.tar' + '/Users/tarek/myarchive.tar' Listing the files in the resulting archive gives us: diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index c28841dbb8c..cdefcd29ef7 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -205,14 +205,30 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGPROF + + Profiling timer expired. + + .. availability:: Unix. + +.. data:: SIGQUIT + + Terminal quit signal. + + .. availability:: Unix. + .. data:: SIGSEGV Segmentation fault: invalid memory reference. +.. data:: SIGSTOP + + Stop executing (cannot be caught or ignored). + .. data:: SIGSTKFLT - Stack fault on coprocessor. The Linux kernel does not raise this signal: it - can only be raised in user space. + Stack fault on coprocessor. The Linux kernel does not raise this signal: it + can only be raised in user space. .. availability:: Linux. @@ -237,18 +253,30 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGVTALRM + + Virtual timer expired. + + .. availability:: Unix. + .. data:: SIGWINCH Window resize signal. .. availability:: Unix. +.. data:: SIGXCPU + + CPU time limit exceeded. + + .. availability:: Unix. + .. data:: SIG* All the signal numbers are defined symbolically. For example, the hangup signal is defined as :const:`signal.SIGHUP`; the variable names are identical to the names used in C programs, as found in ````. The Unix man page for - ':c:func:`signal`' lists the existing signals (on some systems this is + '``signal``' lists the existing signals (on some systems this is :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that not all systems define the same set of signal names; only those names defined by the system are defined by this module. @@ -478,11 +506,11 @@ The :mod:`signal` module defines the following functions: .. versionadded:: 3.3 -.. function:: setitimer(which, seconds, interval=0.0) +.. function:: setitimer(which, seconds, interval=0) Sets given interval timer (one of :const:`signal.ITIMER_REAL`, :const:`signal.ITIMER_VIRTUAL` or :const:`signal.ITIMER_PROF`) specified - by *which* to fire after *seconds* (float is accepted, different from + by *which* to fire after *seconds* (rounded up to microseconds, different from :func:`alarm`) and after that every *interval* seconds (if *interval* is non-zero). The interval timer specified by *which* can be cleared by setting *seconds* to zero. @@ -493,13 +521,18 @@ The :mod:`signal` module defines the following functions: :const:`signal.ITIMER_VIRTUAL` sends :const:`SIGVTALRM`, and :const:`signal.ITIMER_PROF` will deliver :const:`SIGPROF`. - The old values are returned as a tuple: (delay, interval). + The old values are returned as a two-tuple of floats: + (``delay``, ``interval``). Attempting to pass an invalid interval timer will cause an :exc:`ItimerError`. .. availability:: Unix. + .. versionchanged:: 3.15 + Accepts any real numbers as *seconds* and *interval*, not only integers + or floats. + .. function:: getitimer(which) @@ -639,9 +672,8 @@ The :mod:`signal` module defines the following functions: *sigset*. The return value is an object representing the data contained in the - :c:type:`siginfo_t` structure, namely: :attr:`si_signo`, :attr:`si_code`, - :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, - :attr:`si_band`. + ``siginfo_t`` structure, namely: ``si_signo``, ``si_code``, + ``si_errno``, ``si_pid``, ``si_uid``, ``si_status``, ``si_band``. .. availability:: Unix. @@ -676,6 +708,9 @@ The :mod:`signal` module defines the following functions: by a signal not in *sigset* and the signal handler does not raise an exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. _signal-example: diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e98dd83b60e..d93e4dc7c75 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -270,7 +270,7 @@ Module contents .. _site-commandline: -Command Line Interface +Command-line interface ---------------------- .. program:: site diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index c5f8516f768..3ee8b82a188 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -24,10 +24,13 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) An :class:`SMTP` instance encapsulates an SMTP connection. It has methods - that support a full repertoire of SMTP and ESMTP operations. If the optional - *host* and *port* parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO + that support a full repertoire of SMTP and ESMTP operations. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with + host and port automatically when the object is created; otherwise, :meth:`!connect` must + be called manually. + + If specified, *local_hostname* is used as the FQDN of the local host in the HELO/EHLO command. Otherwise, the local hostname is found using :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other than a success code, an :exc:`SMTPConnectError` is raised. The optional @@ -62,6 +65,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, where ``data`` is the bytes about to be sent to the remote host. + .. attribute:: SMTP.default_port + + The default port used for SMTP connections (25). + .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -80,15 +87,23 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`starttls` is - not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same + required from the beginning of the connection and using :meth:`SMTP.starttls` is + not appropriate. + + If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with host + and port automatically when the object is created; otherwise, :meth:`!SMTP.connect` must + be called manually. + + The optional arguments *local_hostname*, *timeout* and *source_address* have the same meaning as they do in the :class:`SMTP` class. *context*, also optional, can contain a :class:`~ssl.SSLContext` and allows configuring various aspects of the secure connection. Please read :ref:`ssl-security` for best practices. + .. attribute:: SMTP_SSL.default_port + + The default port used for SMTP-over-SSL connections (465). + .. versionchanged:: 3.3 *context* was added. @@ -112,7 +127,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our - :meth:`connect` method must support that as well as a regular host:port + :meth:`~SMTP.connect` method must support that as well as a regular host:port server. The optional arguments *local_hostname* and *source_address* have the same meaning as they do in the :class:`SMTP` class. To specify a Unix socket, you must use an absolute path for *host*, starting with a '/'. @@ -147,9 +162,15 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPResponseException Base class for all exceptions that include an SMTP error code. These exceptions - are generated in some instances when the SMTP server returns an error code. The - error code is stored in the :attr:`smtp_code` attribute of the error, and the - :attr:`smtp_error` attribute is set to the error message. + are generated in some instances when the SMTP server returns an error code. + + .. attribute:: smtp_code + + The error code. + + .. attribute:: smtp_error + + The error message. .. exception:: SMTPSenderRefused @@ -161,9 +182,13 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPRecipientsRefused - All recipient addresses refused. The errors for each recipient are accessible - through the attribute :attr:`recipients`, which is a dictionary of exactly the - same sort as :meth:`SMTP.sendmail` returns. + All recipient addresses refused. + + .. attribute:: recipients + + A dictionary of exactly the same sort as returned + by :meth:`SMTP.sendmail` containing the errors for + each recipient. .. exception:: SMTPDataError @@ -213,7 +238,6 @@ SMTP Objects An :class:`SMTP` instance has the following methods: - .. method:: SMTP.set_debuglevel(level) Set the debug output level. A value of 1 or ``True`` for *level* results in @@ -250,6 +274,9 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. + If port is not changed from its default value of 0, the value of the :attr:`default_port` + attribute is used. + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect @@ -417,7 +444,7 @@ An :class:`SMTP` instance has the following methods: .. versionchanged:: 3.4 The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indicator* (see :const:`~ssl.HAS_SNI`). .. versionchanged:: 3.5 @@ -435,7 +462,7 @@ An :class:`SMTP` instance has the following methods: ESMTP options (such as ``DSN`` commands) that should be used with all ``RCPT`` commands can be passed as *rcpt_options*. (If you need to use different ESMTP options to different recipients you have to use the low-level methods such as - :meth:`mail`, :meth:`rcpt` and :meth:`data` to send the message.) + :meth:`!mail`, :meth:`!rcpt` and :meth:`!data` to send the message.) .. note:: @@ -467,10 +494,7 @@ An :class:`SMTP` instance has the following methods: This method may raise the following exceptions: :exc:`SMTPRecipientsRefused` - All recipients were refused. Nobody got the mail. The :attr:`recipients` - attribute of the exception object is a dictionary with information about the - refused recipients (like the one returned when at least one recipient was - accepted). + All recipients were refused. Nobody got the mail. :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. @@ -546,6 +570,30 @@ Low-level methods corresponding to the standard SMTP/ESMTP commands ``HELP``, Normally these do not need to be called directly, so they are not documented here. For details, consult the module code. +Additionally, an SMTP instance has the following attributes: + + +.. attribute:: SMTP.helo_resp + + The response to the ``HELO`` command, see :meth:`helo`. + + +.. attribute:: SMTP.ehlo_resp + + The response to the ``EHLO`` command, see :meth:`ehlo`. + + +.. attribute:: SMTP.does_esmtp + + A boolean value indicating whether the server supports ESMTP, see + :meth:`ehlo`. + + +.. attribute:: SMTP.esmtp_features + + A dictionary of the names of SMTP service extensions supported by the server, + see :meth:`ehlo`. + .. _smtp-example: diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 8fd5187e3a4..b16cde34ec1 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -362,10 +362,10 @@ Exceptions Constants ^^^^^^^^^ - The AF_* and SOCK_* constants are now :class:`AddressFamily` and - :class:`SocketKind` :class:`.IntEnum` collections. +The AF_* and SOCK_* constants are now :class:`AddressFamily` and +:class:`SocketKind` :class:`.IntEnum` collections. - .. versionadded:: 3.4 +.. versionadded:: 3.4 .. data:: AF_UNIX AF_INET @@ -482,6 +482,9 @@ Constants .. versionchanged:: 3.14 Added support for ``TCP_QUICKACK`` on Windows platforms when available. + .. versionchanged:: next + ``IPV6_HDRINCL`` was added. + .. data:: AF_CAN PF_CAN @@ -773,9 +776,9 @@ Constants Constant to optimize CPU locality, to be used in conjunction with :data:`SO_REUSEPORT`. - .. versionadded:: 3.11 + .. versionadded:: 3.11 - .. availability:: Linux >= 3.9 + .. availability:: Linux >= 3.9 .. data:: SO_REUSEPORT_LB @@ -1407,11 +1410,14 @@ The :mod:`socket` module also offers various network-related services: .. function:: setdefaulttimeout(timeout) - Set the default timeout in seconds (float) for new socket objects. When + Set the default timeout in seconds (real number) for new socket objects. When the socket module is first imported, the default is ``None``. See :meth:`~socket.settimeout` for possible values and their respective meanings. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: sethostname(name) @@ -1492,7 +1498,7 @@ The :mod:`socket` module also offers various network-related services: The *fds* parameter is a sequence of file descriptors. Consult :meth:`~socket.sendmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not WASI. + .. availability:: Unix, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1506,9 +1512,9 @@ The :mod:`socket` module also offers various network-related services: Return ``(msg, list(fds), flags, addr)``. Consult :meth:`~socket.recvmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not WASI. + .. availability:: Unix, not WASI. - Unix platforms supporting :meth:`~socket.sendmsg` + Unix platforms supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism. .. versionadded:: 3.9 @@ -2073,7 +2079,7 @@ to sockets. .. method:: socket.settimeout(value) Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating-point number expressing seconds, or ``None``. + nonnegative real number expressing seconds, or ``None``. If a non-zero value is given, subsequent socket operations will raise a :exc:`timeout` exception if the timeout period *value* has elapsed before the operation has completed. If zero is given, the socket is put in @@ -2085,12 +2091,12 @@ to sockets. The method no longer toggles :const:`SOCK_NONBLOCK` flag on :attr:`socket.type`. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. -.. method:: socket.setsockopt(level, optname, value: int) -.. method:: socket.setsockopt(level, optname, value: buffer) - :noindex: -.. method:: socket.setsockopt(level, optname, None, optlen: int) - :noindex: + +.. method:: socket.setsockopt(level, optname, value: int | Buffer) + socket.setsockopt(level, optname, None, optlen: int) .. index:: pair: module; struct diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 59cfa136a3b..491b8769f44 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -24,6 +24,8 @@ There are four basic concrete server classes: :meth:`~BaseServer.server_activate`. The other parameters are passed to the :class:`BaseServer` base class. + .. versionchanged:: 3.15 + The default queue size is now ``socket.SOMAXCONN`` for :class:`socketserver.TCPServer`. .. class:: UDPServer(server_address, RequestHandlerClass, bind_and_activate=True) @@ -541,10 +543,10 @@ objects that simplify communication by providing the standard file interface):: The difference is that the ``readline()`` call in the second handler will call ``recv()`` multiple times until it encounters a newline character, while the -the first handler had to use a ``recv()`` loop to accumulate data until a +first handler had to use a ``recv()`` loop to accumulate data until a newline itself. If it had just used a single ``recv()`` without the loop it would just have returned what has been received so far from the client. -TCP is stream based: data arrives in the order it was sent, but there no +TCP is stream based: data arrives in the order it was sent, but there is no correlation between client ``send()`` or ``sendall()`` calls and the number of ``recv()`` calls on the server required to receive it. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index c615650b622..3b1a9c2f6ee 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -31,7 +31,9 @@ PostgreSQL or Oracle. The :mod:`!sqlite3` module was written by Gerhard Häring. It provides an SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.15.2 or newer. +requires the third-party `SQLite `_ library. + +.. include:: ../includes/optional-module.rst This document includes four main sections: @@ -259,10 +261,10 @@ Reference Module functions ^^^^^^^^^^^^^^^^ -.. function:: connect(database, timeout=5.0, detect_types=0, \ +.. function:: connect(database, *, timeout=5.0, detect_types=0, \ isolation_level="DEFERRED", check_same_thread=True, \ factory=sqlite3.Connection, cached_statements=128, \ - uri=False, *, \ + uri=False, \ autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL) Open a connection to an SQLite database. @@ -355,11 +357,8 @@ Module functions .. versionchanged:: 3.12 Added the *autocommit* parameter. - .. versionchanged:: 3.13 - Positional use of the parameters *timeout*, *detect_types*, - *isolation_level*, *check_same_thread*, *factory*, *cached_statements*, - and *uri* is deprecated. - They will become keyword-only parameters in Python 3.15. + .. versionchanged:: 3.15 + All parameters except *database* are now keyword-only. .. function:: complete_statement(statement) @@ -510,6 +509,15 @@ Module constants Version number of the runtime SQLite library as a :class:`tuple` of :class:`integers `. +.. data:: SQLITE_KEYWORDS + + A :class:`tuple` containing all SQLite keywords. + + This constant is only available if Python was compiled with SQLite + 3.24.0 or greater. + + .. versionadded:: 3.15 + .. data:: threadsafety Integer constant required by the DB-API 2.0, stating the level of thread @@ -614,7 +622,7 @@ Connection objects supplied, this must be a :term:`callable` returning an instance of :class:`Cursor` or its subclasses. - .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") + .. method:: blobopen(table, column, rowid, /, *, readonly=False, name="main") Open a :class:`Blob` handle to an existing :abbr:`BLOB (Binary Large OBject)`. @@ -625,8 +633,8 @@ Connection objects :param str column: The name of the column where the blob is located. - :param str row: - The name of the row where the blob is located. + :param int rowid: + The row id where the blob is located. :param bool readonly: Set to ``True`` if the blob should be opened without write @@ -693,7 +701,7 @@ Connection objects :meth:`~Cursor.executescript` on it with the given *sql_script*. Return the new cursor object. - .. method:: create_function(name, narg, func, *, deterministic=False) + .. method:: create_function(name, narg, func, /, *, deterministic=False) Create or remove a user-defined SQL function. @@ -719,6 +727,9 @@ Connection objects .. versionchanged:: 3.8 Added the *deterministic* parameter. + .. versionchanged:: 3.15 + The first three parameters are now positional-only. + Example: .. doctest:: @@ -733,13 +744,8 @@ Connection objects ('acbd18db4cc2f85cedef654fccc4a4d8',) >>> con.close() - .. versionchanged:: 3.13 - Passing *name*, *narg*, and *func* as keyword arguments is deprecated. - These parameters will become positional-only in Python 3.15. - - - .. method:: create_aggregate(name, n_arg, aggregate_class) + .. method:: create_aggregate(name, n_arg, aggregate_class, /) Create or remove a user-defined SQL aggregate function. @@ -763,6 +769,9 @@ Connection objects Set to ``None`` to remove an existing SQL aggregate function. :type aggregate_class: :term:`class` | None + .. versionchanged:: 3.15 + All three parameters are now positional-only. + Example: .. testcode:: @@ -792,11 +801,6 @@ Connection objects 3 - .. versionchanged:: 3.13 - - Passing *name*, *n_arg*, and *aggregate_class* as keyword arguments is deprecated. - These parameters will become positional-only in Python 3.15. - .. method:: create_window_function(name, num_params, aggregate_class, /) @@ -937,7 +941,7 @@ Connection objects Aborted queries will raise an :exc:`OperationalError`. - .. method:: set_authorizer(authorizer_callback) + .. method:: set_authorizer(authorizer_callback, /) Register :term:`callable` *authorizer_callback* to be invoked for each attempt to access a column of a table in the database. @@ -962,12 +966,11 @@ Connection objects .. versionchanged:: 3.11 Added support for disabling the authorizer using ``None``. - .. versionchanged:: 3.13 - Passing *authorizer_callback* as a keyword argument is deprecated. - The parameter will become positional-only in Python 3.15. + .. versionchanged:: 3.15 + The only parameter is now positional-only. - .. method:: set_progress_handler(progress_handler, n) + .. method:: set_progress_handler(progress_handler, /, n) Register :term:`callable` *progress_handler* to be invoked for every *n* instructions of the SQLite virtual machine. This is useful if you want to @@ -981,12 +984,11 @@ Connection objects currently executing query and cause it to raise a :exc:`DatabaseError` exception. - .. versionchanged:: 3.13 - Passing *progress_handler* as a keyword argument is deprecated. - The parameter will become positional-only in Python 3.15. + .. versionchanged:: 3.15 + The first parameter is now positional-only. - .. method:: set_trace_callback(trace_callback) + .. method:: set_trace_callback(trace_callback, /) Register :term:`callable` *trace_callback* to be invoked for each SQL statement that is actually executed by the SQLite backend. @@ -1009,9 +1011,8 @@ Connection objects .. versionadded:: 3.3 - .. versionchanged:: 3.13 - Passing *trace_callback* as a keyword argument is deprecated. - The parameter will become positional-only in Python 3.15. + .. versionchanged:: 3.15 + The first parameter is now positional-only. .. method:: enable_load_extension(enabled, /) @@ -1492,7 +1493,9 @@ Cursor objects :type parameters: :class:`dict` | :term:`sequence` :raises ProgrammingError: - If *sql* contains more than one SQL statement. + When *sql* contains more than one SQL statement. + When :ref:`named placeholders ` are used + and *parameters* is a sequence instead of a :class:`dict`. If :attr:`~Connection.autocommit` is :data:`LEGACY_TRANSACTION_CONTROL`, @@ -1501,13 +1504,11 @@ Cursor objects and there is no open transaction, a transaction is implicitly opened before executing *sql*. - .. deprecated-removed:: 3.12 3.14 + .. versionchanged:: 3.14 - :exc:`DeprecationWarning` is emitted if + :exc:`ProgrammingError` is emitted if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. - Starting with Python 3.14, :exc:`ProgrammingError` will - be raised instead. Use :meth:`executescript` to execute multiple SQL statements. @@ -1529,8 +1530,10 @@ Cursor objects :type parameters: :term:`iterable` :raises ProgrammingError: - If *sql* contains more than one SQL statement, - or is not a DML statement. + When *sql* contains more than one SQL statement + or is not a DML statement, + When :ref:`named placeholders ` are used + and the items in *parameters* are sequences instead of :class:`dict`\s. Example: @@ -1554,14 +1557,12 @@ Cursor objects .. _RETURNING clauses: https://www.sqlite.org/lang_returning.html - .. deprecated-removed:: 3.12 3.14 + .. versionchanged:: 3.14 - :exc:`DeprecationWarning` is emitted if + :exc:`ProgrammingError` is emitted if :ref:`named placeholders ` are used and the items in *parameters* are sequences instead of :class:`dict`\s. - Starting with Python 3.14, :exc:`ProgrammingError` will - be raised instead. .. method:: executescript(sql_script, /) @@ -1612,6 +1613,9 @@ Cursor objects If the *size* parameter is used, then it is best for it to retain the same value from one :meth:`fetchmany` call to the next. + .. versionchanged:: 3.15 + Negative *size* values are rejected by raising :exc:`ValueError`. + .. method:: fetchall() Return all (remaining) rows of a query result as a :class:`list`. @@ -1639,6 +1643,9 @@ Cursor objects Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. The default value is 1 which means a single row would be fetched per call. + .. versionchanged:: 3.15 + Negative values are rejected by raising :exc:`ValueError`. + .. attribute:: connection Read-only attribute that provides the SQLite database :class:`Connection` @@ -2289,7 +2296,7 @@ This section shows recipes for common adapters and converters. def adapt_datetime_iso(val): """Adapt datetime.datetime to timezone-naive ISO 8601 date.""" - return val.isoformat() + return val.replace(tzinfo=None).isoformat() def adapt_datetime_epoch(val): """Adapt datetime.datetime to Unix timestamp.""" diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index c0dcecf737e..7d30094963d 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -18,8 +18,9 @@ This module provides access to Transport Layer Security (often known as "Secure Sockets Layer") encryption and peer authentication facilities for network sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, macOS, and -probably additional platforms, as long as OpenSSL is installed on that platform. +library. + +.. include:: ../includes/optional-module.rst .. note:: @@ -125,7 +126,8 @@ Context creation A convenience function helps create :class:`SSLContext` objects for common purposes. -.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) +.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, *,\ + cafile=None, capath=None, cadata=None) Return a new :class:`SSLContext` object with default settings for the given *purpose*. The settings are chosen by the :mod:`ssl` module, @@ -215,6 +217,25 @@ purposes. :data:`VERIFY_X509_STRICT` in its default verify flags. +Signature algorithms +^^^^^^^^^^^^^^^^^^^^ + +.. function:: get_sigalgs() + + Return a list of available TLS signature algorithm names used + by servers to complete the TLS handshake or clients requesting + certificate-based authentication. For example:: + + >>> ssl.get_sigalgs() # doctest: +SKIP + ['ecdsa_secp256r1_sha256', 'ecdsa_secp384r1_sha384', ...] + + These names can be used when building string values to pass to the + :meth:`SSLContext.set_client_sigalgs` and + :meth:`SSLContext.set_server_sigalgs` methods. + + .. versionadded:: 3.15 + + Exceptions ^^^^^^^^^^ @@ -314,7 +335,7 @@ Exceptions Random generation ^^^^^^^^^^^^^^^^^ -.. function:: RAND_bytes(num) +.. function:: RAND_bytes(num, /) Return *num* cryptographically strong pseudo-random bytes. Raises an :class:`SSLError` if the PRNG has not been seeded with enough data or if the @@ -334,11 +355,10 @@ Random generation .. function:: RAND_status() Return ``True`` if the SSL pseudo-random number generator has been seeded - with 'enough' randomness, and ``False`` otherwise. You can use - :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of - the pseudo-random number generator. + with 'enough' randomness, and ``False`` otherwise. Use :func:`ssl.RAND_add` + to increase the randomness of the pseudo-random number generator. -.. function:: RAND_add(bytes, entropy) +.. function:: RAND_add(bytes, entropy, /) Mix the given *bytes* into the SSL pseudo-random number generator. The parameter *entropy* (a float) is a lower bound on the entropy contained in @@ -406,12 +426,12 @@ Certificate handling .. versionchanged:: 3.10 The *timeout* parameter was added. -.. function:: DER_cert_to_PEM_cert(DER_cert_bytes) +.. function:: DER_cert_to_PEM_cert(der_cert_bytes) Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded string version of the same certificate. -.. function:: PEM_cert_to_DER_cert(PEM_cert_string) +.. function:: PEM_cert_to_DER_cert(pem_cert_string) Given a certificate as an ASCII PEM string, returns a DER-encoded sequence of bytes for that same certificate. @@ -934,6 +954,13 @@ Constants .. versionadded:: 3.13 +.. data:: HAS_PSK_TLS13 + + Whether the OpenSSL library has built-in support for External PSKs in TLS + 1.3 as described in :rfc:`9258`. + + .. versionadded:: 3.15 + .. data:: HAS_PHA Whether the OpenSSL library has built-in support for TLS-PHA. @@ -1071,8 +1098,9 @@ SSL Sockets (but passing a non-zero ``flags`` argument is not allowed) - :meth:`~socket.socket.send`, :meth:`~socket.socket.sendall` (with the same limitation) - - :meth:`~socket.socket.sendfile` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send` will be used) + - :meth:`~socket.socket.sendfile` (it may be high-performant only when + the kernel TLS is enabled by setting :data:`~ssl.OP_ENABLE_KTLS` or when a + socket is plain-text, else :meth:`~socket.socket.send` will be used) - :meth:`~socket.socket.shutdown` However, since the SSL (and TLS) protocol has its own framing atop @@ -1106,6 +1134,11 @@ SSL Sockets functions support reading and writing of data larger than 2 GB. Writing zero-length data no longer fails with a protocol violation error. + .. versionchanged:: 3.15 + Python now uses ``SSL_sendfile`` internally when possible. The + function sends a file more efficiently because it performs TLS encryption + in the kernel to avoid additional context switches. + SSL sockets also have the following additional methods and attributes: .. method:: SSLSocket.read(len=1024, buffer=None) @@ -1128,10 +1161,10 @@ SSL sockets also have the following additional methods and attributes: .. deprecated:: 3.6 Use :meth:`~SSLSocket.recv` instead of :meth:`~SSLSocket.read`. -.. method:: SSLSocket.write(buf) +.. method:: SSLSocket.write(data) - Write *buf* to the SSL socket and return the number of bytes written. The - *buf* argument must be an object supporting the buffer interface. + Write *data* to the SSL socket and return the number of bytes written. The + *data* argument must be an object supporting the buffer interface. Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is :ref:`non-blocking ` and the write would block. @@ -1141,7 +1174,7 @@ SSL sockets also have the following additional methods and attributes: .. versionchanged:: 3.5 The socket timeout is no longer reset each time bytes are received or sent. - The socket timeout is now the maximum total duration to write *buf*. + The socket timeout is now the maximum total duration to write *data*. .. deprecated:: 3.6 Use :meth:`~SSLSocket.send` instead of :meth:`~SSLSocket.write`. @@ -1158,12 +1191,15 @@ SSL sockets also have the following additional methods and attributes: :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` instead of these methods. -.. method:: SSLSocket.do_handshake() +.. method:: SSLSocket.do_handshake(block=False) Perform the SSL setup handshake. + If *block* is true and the timeout obtained by :meth:`~socket.socket.gettimeout` + is zero, the socket is set in blocking mode until the handshake is performed. + .. versionchanged:: 3.4 - The handshake method also performs :func:`match_hostname` when the + The handshake method also performs :func:`!match_hostname` when the :attr:`~SSLContext.check_hostname` attribute of the socket's :attr:`~SSLSocket.context` is true. @@ -1173,7 +1209,7 @@ SSL sockets also have the following additional methods and attributes: .. versionchanged:: 3.7 Hostname or IP address is matched by OpenSSL during handshake. The - function :func:`match_hostname` is no longer used. In case OpenSSL + function :func:`!match_hostname` is no longer used. In case OpenSSL refuses a hostname or IP address, the handshake is aborted early and a TLS alert message is sent to the peer. @@ -1277,6 +1313,29 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.5 +.. method:: SSLSocket.group() + + Return the group used for doing key agreement on this connection. If no + connection has been established, returns ``None``. + + .. versionadded:: 3.15 + +.. method:: SSLSocket.client_sigalg() + + Return the signature algorithm used for performing certificate-based client + authentication on this connection, or ``None`` if no connection has been + established or client authentication didn't occur. + + .. versionadded:: 3.15 + +.. method:: SSLSocket.server_sigalg() + + Return the signature algorithm used by the server to complete the TLS + handshake on this connection, or ``None`` if no connection has been + established or the cipher suite has no signature. + + .. versionadded:: 3.15 + .. method:: SSLSocket.compression() Return the compression algorithm being used as a string, or ``None`` @@ -1634,6 +1693,25 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.6 +.. method:: SSLContext.get_groups(*, include_aliases=False) + + Get a list of groups implemented for key agreement, taking into + account the current TLS :attr:`~SSLContext.minimum_version` and + :attr:`~SSLContext.maximum_version` values. For example:: + + >>> ctx = ssl.create_default_context() + >>> ctx.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> ctx.maximum_version = ssl.TLSVersion.TLSv1_3 + >>> ctx.get_groups() # doctest: +SKIP + ['secp256r1', 'secp384r1', 'secp521r1', 'x25519', 'x448', ...] + + By default, this method returns only the preferred IANA names for the + available groups. However, if the ``include_aliases`` parameter is set to + :const:`True` this method will also return any associated aliases such as + the ECDH curve names supported in older versions of OpenSSL. + + .. versionadded:: 3.15 + .. method:: SSLContext.set_default_verify_paths() Load a set of default "certification authority" (CA) certificates from @@ -1643,23 +1721,79 @@ to speed up repeated connections from the same clients. provided as part of the operating system, though, it is likely to be configured properly. -.. method:: SSLContext.set_ciphers(ciphers) +.. method:: SSLContext.set_ciphers(ciphers, /) - Set the available ciphers for sockets created with this context. - It should be a string in the `OpenSSL cipher list format + Set the allowed ciphers for sockets created with this context when + connecting using TLS 1.2 and earlier. The *ciphers* argument should + be a string in the `OpenSSL cipher list format `_. + To set allowed TLS 1.3 ciphers, use :meth:`SSLContext.set_ciphersuites`. + If no cipher can be selected (because compile-time options or other configuration forbids use of all the specified ciphers), an :class:`SSLError` will be raised. .. note:: - when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will - give the currently selected cipher. + When connected, the :meth:`SSLSocket.cipher` method of SSL sockets will + return details about the negotiated cipher. - TLS 1.3 cipher suites cannot be disabled with - :meth:`~SSLContext.set_ciphers`. +.. method:: SSLContext.set_ciphersuites(ciphersuites, /) -.. method:: SSLContext.set_alpn_protocols(protocols) + Set the allowed ciphers for sockets created with this context when + connecting using TLS 1.3. The *ciphersuites* argument should be a + colon-separate string of TLS 1.3 cipher names. If no cipher can be + selected (because compile-time options or other configuration forbids + use of all the specified ciphers), an :class:`SSLError` will be raised. + + .. note:: + When connected, the :meth:`SSLSocket.cipher` method of SSL sockets will + return details about the negotiated cipher. + + .. versionadded:: 3.15 + +.. method:: SSLContext.set_groups(groups, /) + + Set the groups allowed for key agreement for sockets created with this + context. It should be a string in the `OpenSSL group list format + `_. + + .. note:: + + When connected, the :meth:`SSLSocket.group` method of SSL sockets will + return the group used for key agreement on that connection. + + .. versionadded:: 3.15 + +.. method:: SSLContext.set_client_sigalgs(sigalgs, /) + + Set the signature algorithms allowed for certificate-based client + authentication. It should be a string in the `OpenSSL client sigalgs + list format + `_. + + .. note:: + + When connected, the :meth:`SSLSocket.client_sigalg` method of SSL + sockets will return the signature algorithm used for performing + certificate-based client authentication on that connection. + + .. versionadded:: 3.15 + +.. method:: SSLContext.set_server_sigalgs(sigalgs, /) + + Set the signature algorithms allowed for the server to complete the TLS + handshake. It should be a string in the `OpenSSL sigalgs list format + `_. + + .. note:: + + When connected, the :meth:`SSLSocket.server_sigalg` method of SSL + sockets will return the signature algorithm used by the server to + complete the TLS handshake on that connection. + + .. versionadded:: 3.15 + +.. method:: SSLContext.set_alpn_protocols(alpn_protocols) Specify which protocols the socket should advertise during the SSL/TLS handshake. It should be a list of ASCII strings, like ``['http/1.1', @@ -1673,7 +1807,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.5 -.. method:: SSLContext.set_npn_protocols(protocols) +.. method:: SSLContext.set_npn_protocols(npn_protocols) Specify which protocols the socket should advertise during the SSL/TLS handshake. It should be a list of strings, like ``['http/1.1', 'spdy/2']``, @@ -1740,7 +1874,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 -.. attribute:: SSLContext.set_servername_callback(server_name_callback) +.. method:: SSLContext.set_servername_callback(server_name_callback) This is a legacy API retained for backwards compatibility. When possible, you should use :attr:`sni_callback` instead. The given *server_name_callback* @@ -1754,7 +1888,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.4 -.. method:: SSLContext.load_dh_params(dhfile) +.. method:: SSLContext.load_dh_params(dhfile, /) Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of @@ -1767,7 +1901,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 -.. method:: SSLContext.set_ecdh_curve(curve_name) +.. method:: SSLContext.set_ecdh_curve(curve_name, /) Set the curve name for Elliptic Curve-based Diffie-Hellman (ECDH) key exchange. ECDH is significantly faster than regular DH while arguably @@ -1846,8 +1980,9 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.sslsocket_class The return type of :meth:`SSLContext.wrap_socket`, defaults to - :class:`SSLSocket`. The attribute can be overridden on instance of class - in order to return a custom subclass of :class:`SSLSocket`. + :class:`SSLSocket`. The attribute can be assigned to on instances of + :class:`SSLContext` in order to return a custom subclass of + :class:`SSLSocket`. .. versionadded:: 3.7 @@ -2640,12 +2775,12 @@ purpose. It wraps an OpenSSL memory BIO (Basic IO) object: A boolean indicating whether the memory BIO is current at the end-of-file position. - .. method:: MemoryBIO.read(n=-1) + .. method:: MemoryBIO.read(n=-1, /) Read up to *n* bytes from the memory buffer. If *n* is not specified or negative, all bytes are returned. - .. method:: MemoryBIO.write(buf) + .. method:: MemoryBIO.write(buf, /) Write the bytes from *buf* to the memory BIO. The *buf* argument must be an object supporting the buffer protocol. @@ -2728,7 +2863,7 @@ This common check is automatically performed when .. versionchanged:: 3.7 Hostname matchings is now performed by OpenSSL. Python no longer uses - :func:`match_hostname`. + :func:`!match_hostname`. In server mode, if you want to authenticate your clients using the SSL layer (rather than using a higher-level authentication mechanism), you'll also have @@ -2792,10 +2927,15 @@ TLS 1.3 The TLS 1.3 protocol behaves slightly differently than previous version of TLS/SSL. Some new TLS 1.3 features are not yet available. -- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and - ChaCha20 cipher suites are enabled by default. The method - :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3 - ciphers yet, but :meth:`SSLContext.get_ciphers` returns them. +- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and ChaCha20 + cipher suites are enabled by default. To restrict which TLS 1.3 ciphers + are allowed, the :meth:`SSLContext.set_ciphersuites` method should be + called instead of :meth:`SSLContext.set_ciphers`, which only affects + ciphers in older TLS versions. The :meth:`SSLContext.get_ciphers` method + returns information about ciphers for both TLS 1.3 and earlier versions + and the method :meth:`SSLSocket.cipher` returns information about the + negotiated cipher for both TLS 1.3 and earlier versions once a connection + is established. - Session tickets are no longer sent as part of the initial handshake and are handled differently. :attr:`SSLSocket.session` and :class:`SSLSession` are not compatible with TLS 1.3. @@ -2804,7 +2944,7 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. process certificate requests while they send or receive application data from the server. - TLS 1.3 features like early data, deferred TLS client cert request, - signature algorithm configuration, and rekeying are not supported yet. + and rekeying are not supported yet. .. seealso:: @@ -2819,16 +2959,16 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. Steve Kent :rfc:`RFC 4086: Randomness Requirements for Security <4086>` - Donald E., Jeffrey I. Schiller + Donald E. Eastlake, Jeffrey I. Schiller, Steve Crocker :rfc:`RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile <5280>` - D. Cooper + David Cooper et al. :rfc:`RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <5246>` - T. Dierks et. al. + Tim Dierks and Eric Rescorla. :rfc:`RFC 6066: Transport Layer Security (TLS) Extensions <6066>` - D. Eastlake + Donald E. Eastlake `IANA TLS: Transport Layer Security (TLS) Parameters `_ IANA diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 8434b2e8c75..82012b31a00 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -493,3 +493,22 @@ constants, but are not an exhaustive list. IO_REPARSE_TAG_APPEXECLINK .. versionadded:: 3.8 + +On Linux, the following file attribute constants are available for use when +testing bits in the :attr:`~os.statx_result.stx_attributes` and +:attr:`~os.statx_result.stx_attributes_mask` members returned by +:func:`os.statx`. See the :manpage:`statx(2)` man page for more detail on the +meaning of these constants. + +.. data:: STATX_ATTR_COMPRESSED + STATX_ATTR_IMMUTABLE + STATX_ATTR_APPEND + STATX_ATTR_NODUMP + STATX_ATTR_ENCRYPTED + STATX_ATTR_AUTOMOUNT + STATX_ATTR_MOUNT_ROOT + STATX_ATTR_VERITY + STATX_ATTR_DAX + STATX_ATTR_WRITE_ATOMIC + + .. versionadded:: 3.15 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 39aaa5da078..3899e5b59d8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1000,8 +1000,6 @@ operations have the same priority as the corresponding numeric operations. [3]_ pair: slice; operation pair: operator; in pair: operator; not in - single: count() (sequence method) - single: index() (sequence method) +--------------------------+--------------------------------+----------+ | Operation | Result | Notes | @@ -1018,7 +1016,7 @@ operations have the same priority as the corresponding numeric operations. [3]_ | ``s * n`` or | equivalent to adding *s* to | (2)(7) | | ``n * s`` | itself *n* times | | +--------------------------+--------------------------------+----------+ -| ``s[i]`` | *i*\ th item of *s*, origin 0 | \(3) | +| ``s[i]`` | *i*\ th item of *s*, origin 0 | (3)(8) | +--------------------------+--------------------------------+----------+ | ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | +--------------------------+--------------------------------+----------+ @@ -1031,13 +1029,6 @@ operations have the same priority as the corresponding numeric operations. [3]_ +--------------------------+--------------------------------+----------+ | ``max(s)`` | largest item of *s* | | +--------------------------+--------------------------------+----------+ -| ``s.index(x[, i[, j]])`` | index of the first occurrence | \(8) | -| | of *x* in *s* (at or after | | -| | index *i* and before index *j*)| | -+--------------------------+--------------------------------+----------+ -| ``s.count(x)`` | total number of occurrences of | | -| | *x* in *s* | | -+--------------------------+--------------------------------+----------+ Sequences of the same type also support comparisons. In particular, tuples and lists are compared lexicographically by comparing corresponding elements. @@ -1143,12 +1134,41 @@ Notes: concatenation or repetition. (8) - ``index`` raises :exc:`ValueError` when *x* is not found in *s*. - Not all implementations support passing the additional arguments *i* and *j*. - These arguments allow efficient searching of subsections of the sequence. Passing - the extra arguments is roughly equivalent to using ``s[i:j].index(x)``, only - without copying any data and with the returned index being relative to - the start of the sequence rather than the start of the slice. + An :exc:`IndexError` is raised if *i* is outside the sequence range. + +.. rubric:: Sequence Methods + +Sequence types also support the following methods: + +.. method:: list.count(value, /) + range.count(value, /) + tuple.count(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.count(value, /) + + Return the total number of occurrences of *value* in *sequence*. + +.. method:: list.index(value[, start[, stop]) + range.index(value[, start[, stop]) + tuple.index(value[, start[, stop]) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.index(value[, start[, stop]) + + Return the index of the first occurrence of *value* in *sequence*. + + Raises :exc:`ValueError` if *value* is not found in *sequence*. + + The *start* or *stop* arguments allow for efficient searching + of subsections of the sequence, beginning at *start* and ending at *stop*. + This is roughly equivalent to ``start + sequence[start:stop].index(value)``, + only without copying any data. + + .. caution:: + Not all sequence types support passing the *start* and *stop* arguments. .. _typesseq-immutable: @@ -1199,14 +1219,6 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). pair: subscript; assignment pair: slice; assignment pair: statement; del - single: append() (sequence method) - single: clear() (sequence method) - single: copy() (sequence method) - single: extend() (sequence method) - single: insert() (sequence method) - single: pop() (sequence method) - single: remove() (sequence method) - single: reverse() (sequence method) +------------------------------+--------------------------------+---------------------+ | Operation | Result | Notes | @@ -1214,11 +1226,15 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``s[i] = x`` | item *i* of *s* is replaced by | | | | *x* | | +------------------------------+--------------------------------+---------------------+ +| ``del s[i]`` | removes item *i* of *s* | | ++------------------------------+--------------------------------+---------------------+ | ``s[i:j] = t`` | slice of *s* from *i* to *j* | | | | is replaced by the contents of | | | | the iterable *t* | | +------------------------------+--------------------------------+---------------------+ -| ``del s[i:j]`` | same as ``s[i:j] = []`` | | +| ``del s[i:j]`` | removes the elements of | | +| | ``s[i:j]`` from the list | | +| | (same as ``s[i:j] = []``) | | +------------------------------+--------------------------------+---------------------+ | ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` | \(1) | | | are replaced by those of *t* | | @@ -1226,39 +1242,14 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``del s[i:j:k]`` | removes the elements of | | | | ``s[i:j:k]`` from the list | | +------------------------------+--------------------------------+---------------------+ -| ``s.append(x)`` | appends *x* to the end of the | | -| | sequence (same as | | -| | ``s[len(s):len(s)] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.clear()`` | removes all items from *s* | \(5) | -| | (same as ``del s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.copy()`` | creates a shallow copy of *s* | \(5) | -| | (same as ``s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.extend(t)`` or | extends *s* with the | | -| ``s += t`` | contents of *t* (for the | | +| ``s += t`` | extends *s* with the | | +| | contents of *t* (for the | | | | most part the same as | | | | ``s[len(s):len(s)] = t``) | | +------------------------------+--------------------------------+---------------------+ -| ``s *= n`` | updates *s* with its contents | \(6) | +| ``s *= n`` | updates *s* with its contents | \(2) | | | repeated *n* times | | +------------------------------+--------------------------------+---------------------+ -| ``s.insert(i, x)`` | inserts *x* into *s* at the | | -| | index given by *i* | | -| | (same as ``s[i:i] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.pop()`` or ``s.pop(i)`` | retrieves the item at *i* and | \(2) | -| | also removes it from *s* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | removes the first item from | \(3) | -| | *s* where ``s[i]`` is equal to | | -| | *x* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.reverse()`` | reverses the items of *s* in | \(4) | -| | place | | -+------------------------------+--------------------------------+---------------------+ - Notes: @@ -1266,33 +1257,106 @@ Notes: If *k* is not equal to ``1``, *t* must have the same length as the slice it is replacing. (2) - The optional argument *i* defaults to ``-1``, so that by default the last - item is removed and returned. - -(3) - :meth:`remove` raises :exc:`ValueError` when *x* is not found in *s*. - -(4) - The :meth:`reverse` method modifies the sequence in place for economy of - space when reversing a large sequence. To remind users that it operates by - side effect, it does not return the reversed sequence. - -(5) - :meth:`clear` and :meth:`!copy` are included for consistency with the - interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`). :meth:`!copy` is not part of the - :class:`collections.abc.MutableSequence` ABC, but most concrete - mutable sequence classes provide it. - - .. versionadded:: 3.3 - :meth:`clear` and :meth:`!copy` methods. - -(6) The value *n* is an integer, or an object implementing :meth:`~object.__index__`. Zero and negative values of *n* clear the sequence. Items in the sequence are not copied; they are referenced multiple times, as explained for ``s * n`` under :ref:`typesseq-common`. +.. rubric:: Mutable Sequence Methods + +Mutable sequence types also support the following methods: + +.. method:: bytearray.append(value, /) + list.append(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.append(value, /) + + Append *value* to the end of the sequence + This is equivalent to writing ``seq[len(seq):len(seq)] = [value]``. + +.. method:: bytearray.clear() + list.clear() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.clear() + + .. versionadded:: 3.3 + + Remove all items from *sequence*. + This is equivalent to writing ``del sequence[:]``. + +.. method:: bytearray.copy() + list.copy() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.copy() + + .. versionadded:: 3.3 + + Create a shallow copy of *sequence*. + This is equivalent to writing ``sequence[:]``. + + .. hint:: The :meth:`!copy` method is not part of the + :class:`~collections.abc.MutableSequence` :class:`~abc.ABC`, + but most concrete mutable sequence types provide it. + +.. method:: bytearray.extend(iterable, /) + list.extend(iterable, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. + For the most part, this is the same as writing + ``seq[len(seq):len(seq)] = iterable``. + +.. method:: bytearray.insert(index, value, /) + list.insert(index, value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. + This is equivalent to writing ``sequence[index:index] = [value]``. + +.. method:: bytearray.pop(index=-1, /) + list.pop(index=-1, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +.. method:: bytearray.remove(value, /) + list.remove(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.remove(value, /) + + Remove the first item from *sequence* where ``sequence[i] == value``. + + Raises :exc:`ValueError` if *value* is not found in *sequence*. + +.. method:: bytearray.reverse() + list.reverse() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.reverse() + + Reverse the items of *sequence* in place. + This method maintains economy of space when reversing a large sequence. + To remind users that it operates by side-effect, it returns ``None``. + .. _typesseq-list: @@ -1305,7 +1369,7 @@ Lists are mutable sequences, typically used to store collections of homogeneous items (where the precise degree of similarity will vary by application). -.. class:: list([iterable]) +.. class:: list(iterable=(), /) Lists may be constructed in several ways: @@ -1386,7 +1450,7 @@ built-in). Tuples are also used for cases where an immutable sequence of homogeneous data is needed (such as allowing storage in a :class:`set` or :class:`dict` instance). -.. class:: tuple([iterable]) +.. class:: tuple(iterable=(), /) Tuples may be constructed in a number of ways: @@ -1428,8 +1492,8 @@ The :class:`range` type represents an immutable sequence of numbers and is commonly used for looping a specific number of times in :keyword:`for` loops. -.. class:: range(stop) - range(start, stop[, step]) +.. class:: range(stop, /) + range(start, stop, step=1, /) The arguments to the range constructor must be integers (either built-in :class:`int` or any object that implements the :meth:`~object.__index__` special @@ -1689,8 +1753,10 @@ multiple fragments. .. index:: single: string; str (built-in class) -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') +.. class:: str(*, encoding='utf-8', errors='strict') + str(object) + str(object, encoding, errors='strict') + str(object, *, errors) Return a :ref:`string ` version of *object*. If *object* is not provided, returns the empty string. Otherwise, the behavior of ``str()`` @@ -1777,19 +1843,25 @@ expression support in the :mod:`re` module). lowercase, :meth:`lower` would do nothing to ``'ß'``; :meth:`casefold` converts it to ``"ss"``. - The casefolding algorithm is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + The casefolding algorithm is `described in section 3.13.3 'Default Case + Folding' of the Unicode Standard + `__. .. versionadded:: 3.3 -.. method:: str.center(width[, fillchar]) +.. method:: str.center(width, fillchar=' ', /) Return centered in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is - returned if *width* is less than or equal to ``len(s)``. + returned if *width* is less than or equal to ``len(s)``. For example:: + >>> 'Python'.center(10) + ' Python ' + >>> 'Python'.center(10, '-') + '--Python--' + >>> 'Python'.center(4) + 'Python' .. method:: str.count(sub[, start[, end]]) @@ -1799,8 +1871,18 @@ expression support in the :mod:`re` module). interpreted as in slice notation. If *sub* is empty, returns the number of empty strings between characters - which is the length of the string plus one. + which is the length of the string plus one. For example:: + >>> 'spam, spam, spam'.count('spam') + 3 + >>> 'spam, spam, spam'.count('spam', 5) + 2 + >>> 'spam, spam, spam'.count('spam', 5, 10) + 1 + >>> 'spam, spam, spam'.count('eggs') + 0 + >>> 'spam, spam, spam'.count('') + 17 .. method:: str.encode(encoding="utf-8", errors="strict") @@ -1820,6 +1902,14 @@ expression support in the :mod:`re` module). unless an encoding error actually occurs, :ref:`devmode` is enabled or a :ref:`debug build ` is used. + For example:: + + >>> encoded_str_to_bytes = 'Python'.encode() + >>> type(encoded_str_to_bytes) + + >>> encoded_str_to_bytes + b'Python' + .. versionchanged:: 3.1 Added support for keyword arguments. @@ -1834,7 +1924,19 @@ expression support in the :mod:`re` module). Return ``True`` if the string ends with the specified *suffix*, otherwise return ``False``. *suffix* can also be a tuple of suffixes to look for. With optional *start*, test beginning at that position. With optional *end*, stop comparing - at that position. + at that position. Using *start* and *end* is equivalent to + ``str[start:end].endswith(suffix)``. For example:: + + >>> 'Python'.endswith('on') + True + >>> 'a tuple of suffixes'.endswith(('at', 'in')) + False + >>> 'a tuple of suffixes'.endswith(('at', 'es')) + True + >>> 'Python is amazing'.endswith('is', 0, 9) + True + + See also :meth:`startswith` and :meth:`removesuffix`. .. method:: str.expandtabs(tabsize=8) @@ -1850,12 +1952,15 @@ expression support in the :mod:`re` module). (``\n``) or return (``\r``), it is copied and the current column is reset to zero. Any other character is copied unchanged and the current column is incremented by one regardless of how the character is represented when - printed. + printed. For example:: >>> '01\t012\t0123\t01234'.expandtabs() '01 012 0123 01234' >>> '01\t012\t0123\t01234'.expandtabs(4) '01 012 0123 01234' + >>> print('01\t012\n0123\t01234'.expandtabs(4)) + 01 012 + 0123 01234 .. method:: str.find(sub[, start[, end]]) @@ -1863,6 +1968,14 @@ expression support in the :mod:`re` module). Return the lowest index in the string where substring *sub* is found within the slice ``s[start:end]``. Optional arguments *start* and *end* are interpreted as in slice notation. Return ``-1`` if *sub* is not found. + For example:: + + >>> 'spam, spam, spam'.find('sp') + 0 + >>> 'spam, spam, spam'.find('sp', 5) + 6 + + See also :meth:`rfind` and :meth:`index`. .. note:: @@ -1881,10 +1994,16 @@ expression support in the :mod:`re` module). ``{}``. Each replacement field contains either the numeric index of a positional argument, or the name of a keyword argument. Returns a copy of the string where each replacement field is replaced with the string value of - the corresponding argument. + the corresponding argument. For example: + + .. doctest:: >>> "The sum of 1 + 2 is {0}".format(1+2) 'The sum of 1 + 2 is 3' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' + >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") + 'Nobody expects the Spanish Inquisition!' See :ref:`formatstrings` for a description of the various formatting options that can be specified in format strings. @@ -1943,14 +2062,33 @@ expression support in the :mod:`re` module). property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard - `_. + `__. + For example: + + .. doctest:: + + >>> 'Letters and spaces'.isalpha() + False + >>> 'LettersOnly'.isalpha() + True + >>> 'µ'.isalpha() # non-ASCII characters can be considered alphabetical too + True + + See :ref:`unicode-properties`. .. method:: str.isascii() Return ``True`` if the string is empty or all characters in the string are ASCII, ``False`` otherwise. - ASCII characters have code points in the range U+0000-U+007F. + ASCII characters have code points in the range U+0000-U+007F. For example: + + .. doctest:: + + >>> 'ASCII characters'.isascii() + True + >>> 'µ'.isascii() + False .. versionadded:: 3.7 @@ -1960,9 +2098,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are decimal characters and there is at least one character, ``False`` otherwise. Decimal characters are those that can be used to form - numbers in base 10, e.g. U+0660, ARABIC-INDIC DIGIT + numbers in base 10, such as U+0660, ARABIC-INDIC DIGIT ZERO. Formally a decimal character is a character in the Unicode - General Category "Nd". + General Category "Nd". For example: + + .. doctest:: + + >>> '0123456789'.isdecimal() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdecimal() # Arabic-Indic digits zero to nine + True + >>> 'alphabetic'.isdecimal() + False .. method:: str.isdigit() @@ -2012,7 +2159,7 @@ expression support in the :mod:`re` module). .. method:: str.isprintable() - Return true if all characters in the string are printable, false if it + Return ``True`` if all characters in the string are printable, ``False`` if it contains at least one non-printable character. Here "printable" means the character is suitable for :func:`repr` to use in @@ -2044,6 +2191,19 @@ expression support in the :mod:`re` module). character, for example uppercase characters may only follow uncased characters and lowercase characters only cased ones. Return ``False`` otherwise. + For example: + + .. doctest:: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also :meth:`title`. + .. method:: str.isupper() @@ -2063,15 +2223,24 @@ expression support in the :mod:`re` module). .. _meth-str-join: -.. method:: str.join(iterable) +.. method:: str.join(iterable, /) Return a string which is the concatenation of the strings in *iterable*. A :exc:`TypeError` will be raised if there are any non-string values in *iterable*, including :class:`bytes` objects. The separator between - elements is the string providing this method. + elements is the string providing this method. For example: + + .. doctest:: + + >>> ', '.join(['spam', 'spam', 'spam']) + 'spam, spam, spam' + >>> '-'.join('Python') + 'P-y-t-h-o-n' + + See also :meth:`split`. -.. method:: str.ljust(width[, fillchar]) +.. method:: str.ljust(width, fillchar=' ', /) Return the string left justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The @@ -2083,12 +2252,12 @@ expression support in the :mod:`re` module). Return a copy of the string with all the cased characters [4]_ converted to lowercase. - The lowercasing algorithm used is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + The lowercasing algorithm used is `described in section 3.13.2 'Default Case + Conversion' of the Unicode Standard + `__. -.. method:: str.lstrip([chars]) +.. method:: str.lstrip(chars=None, /) Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted @@ -2109,7 +2278,8 @@ expression support in the :mod:`re` module). 'three!' -.. staticmethod:: str.maketrans(x[, y[, z]]) +.. staticmethod:: str.maketrans(dict, /) + str.maketrans(from, to, remove='', /) This static method returns a translation table usable for :meth:`str.translate`. @@ -2119,12 +2289,12 @@ expression support in the :mod:`re` module). converted to ordinals. If there are two arguments, they must be strings of equal length, and in the - resulting dictionary, each character in x will be mapped to the character at - the same position in y. If there is a third argument, it must be a string, + resulting dictionary, each character in *from* will be mapped to the character at + the same position in *to*. If there is a third argument, it must be a string, whose characters will be mapped to ``None`` in the result. -.. method:: str.partition(sep) +.. method:: str.partition(sep, /) Split the string at the first occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself, and the part @@ -2160,7 +2330,7 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 -.. method:: str.replace(old, new, count=-1) +.. method:: str.replace(old, new, /, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by *new*. If *count* is given, only the first *count* occurrences are replaced. @@ -2183,14 +2353,14 @@ expression support in the :mod:`re` module). found. -.. method:: str.rjust(width[, fillchar]) +.. method:: str.rjust(width, fillchar=' ', /) Return the string right justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. -.. method:: str.rpartition(sep) +.. method:: str.rpartition(sep, /) Split the string at the last occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself, and the part @@ -2207,7 +2377,7 @@ expression support in the :mod:`re` module). :meth:`split` which is described in detail below. -.. method:: str.rstrip([chars]) +.. method:: str.rstrip(chars=None, /) Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted @@ -2269,6 +2439,20 @@ expression support in the :mod:`re` module). >>> ' 1 2 3 '.split() ['1', '2', '3'] + If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only + leading runs of consecutive whitespace are considered. + + For example:: + + >>> "".split(None, 0) + [] + >>> " ".split(None, 0) + [] + >>> " foo ".split(maxsplit=0) + ['foo '] + + See also :meth:`join`. + .. index:: single: universal newlines; str.splitlines method @@ -2344,7 +2528,7 @@ expression support in the :mod:`re` module). string at that position. -.. method:: str.strip([chars]) +.. method:: str.strip(chars=None, /) Return a copy of the string with the leading and trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. @@ -2408,8 +2592,10 @@ expression support in the :mod:`re` module). >>> titlecase("they're bill's friends.") "They're Bill's Friends." + See also :meth:`istitle`. -.. method:: str.translate(table) + +.. method:: str.translate(table, /) Return a copy of the string in which each character has been mapped through the given translation table. The table must be an object that implements @@ -2435,12 +2621,12 @@ expression support in the :mod:`re` module). character(s) is not "Lu" (Letter, uppercase), but e.g. "Lt" (Letter, titlecase). - The uppercasing algorithm used is - `described in section 3.13 'Default Case Folding' of the Unicode Standard - `__. + The uppercasing algorithm used is `described in section 3.13.2 'Default Case + Conversion' of the Unicode Standard + `__. -.. method:: str.zfill(width) +.. method:: str.zfill(width, /) Return a copy of the string left filled with ASCII ``'0'`` digits to make a string of length *width*. A leading sign prefix (``'+'``/``'-'``) @@ -2470,6 +2656,8 @@ expression support in the :mod:`re` module). single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals +.. _stdtypes-fstrings: + Formatted String Literals (f-strings) ------------------------------------- @@ -2478,123 +2666,147 @@ Formatted String Literals (f-strings) The :keyword:`await` and :keyword:`async for` can be used in expressions within f-strings. .. versionchanged:: 3.8 - Added the debugging operator (``=``) + Added the debug specifier (``=``) .. versionchanged:: 3.12 Many restrictions on expressions within f-strings have been removed. Notably, nested strings, comments, and backslashes are now permitted. An :dfn:`f-string` (formally a :dfn:`formatted string literal`) is a string literal that is prefixed with ``f`` or ``F``. -This type of string literal allows embedding arbitrary Python expressions -within *replacement fields*, which are delimited by curly brackets (``{}``). -These expressions are evaluated at runtime, similarly to :meth:`str.format`, -and are converted into regular :class:`str` objects. -For example: +This type of string literal allows embedding the results of arbitrary Python +expressions within *replacement fields*, which are delimited by curly +brackets (``{}``). +Each replacement field must contain an expression, optionally followed by: -.. doctest:: +* a *debug specifier* -- an equal sign (``=``); +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). - >>> who = 'nobody' - >>> nationality = 'Spanish' - >>> f'{who.title()} expects the {nationality} Inquisition!' - 'Nobody expects the Spanish Inquisition!' +See the :ref:`Lexical Analysis section on f-strings ` for details +on the syntax of these fields. -It is also possible to use a multi line f-string: +Debug specifier +^^^^^^^^^^^^^^^ -.. doctest:: +.. versionadded:: 3.8 - >>> f'''This is a string - ... on two lines''' - 'This is a string\non two lines' +If a debug specifier -- an equal sign (``=``) -- appears after the replacement +field expression, the resulting f-string will contain the expression's source, +the equal sign, and the value of the expression. +This is often useful for debugging:: -A single opening curly bracket, ``'{'``, marks a *replacement field* that -can contain any Python expression: + >>> number = 14.3 + >>> f'{number=}' + 'number=14.3' -.. doctest:: +Whitespace before, inside and after the expression, as well as whitespace +after the equal sign, is significant --- it is retained in the result:: - >>> nationality = 'Spanish' - >>> f'The {nationality} Inquisition!' - 'The Spanish Inquisition!' + >>> f'{ number - 4 = }' + ' number - 4 = 10.3' -To include a literal ``{`` or ``}``, use a double bracket: -.. doctest:: +Conversion specifier +^^^^^^^^^^^^^^^^^^^^ - >>> x = 42 - >>> f'{{x}} is {x}' - '{x} is 42' - -Functions can also be used, and :ref:`format specifiers `: - -.. doctest:: - - >>> from math import sqrt - >>> f'√2 \N{ALMOST EQUAL TO} {sqrt(2):.5f}' - '√2 ≈ 1.41421' - -Any non-string expression is converted using :func:`str`, by default: - -.. doctest:: +By default, the value of a replacement field expression is converted to +a string using :func:`str`:: >>> from fractions import Fraction - >>> f'{Fraction(1, 3)}' + >>> one_third = Fraction(1, 3) + >>> f'{one_third}' '1/3' -To use an explicit conversion, use the ``!`` (exclamation mark) operator, -followed by any of the valid formats, which are: +When a debug specifier but no format specifier is used, the default conversion +instead uses :func:`repr`:: -========== ============== -Conversion Meaning -========== ============== -``!a`` :func:`ascii` -``!r`` :func:`repr` -``!s`` :func:`str` -========== ============== + >>> f'{one_third = }' + 'one_third = Fraction(1, 3)' -For example: +The conversion can be specified explicitly using one of these specifiers: -.. doctest:: +* ``!s`` for :func:`str` +* ``!r`` for :func:`repr` +* ``!a`` for :func:`ascii` - >>> from fractions import Fraction - >>> f'{Fraction(1, 3)!s}' +For example:: + + >>> str(one_third) '1/3' - >>> f'{Fraction(1, 3)!r}' + >>> repr(one_third) 'Fraction(1, 3)' - >>> question = '¿Dónde está el Presidente?' - >>> print(f'{question!a}') - '\xbfD\xf3nde est\xe1 el Presidente?' -While debugging it may be helpful to see both the expression and its value, -by using the equals sign (``=``) after the expression. -This preserves spaces within the brackets, and can be used with a converter. -By default, the debugging operator uses the :func:`repr` (``!r``) conversion. -For example: + >>> f'{one_third!s} is {one_third!r}' + '1/3 is Fraction(1, 3)' -.. doctest:: + >>> string = "¡kočka 😸!" + >>> ascii(string) + "'\\xa1ko\\u010dka \\U0001f638!'" + + >>> f'{string = !a}' + "string = '\\xa1ko\\u010dka \\U0001f638!'" + + +Format specifier +^^^^^^^^^^^^^^^^ + +After the expression has been evaluated, and possibly converted using an +explicit conversion specifier, it is formatted using the :func:`format` function. +If the replacement field includes a *format specifier* introduced by a colon +(``:``), the specifier is passed to :func:`!format` as the second argument. +The result of :func:`!format` is then used as the final value for the +replacement field. For example:: >>> from fractions import Fraction - >>> calculation = Fraction(1, 3) - >>> f'{calculation=}' - 'calculation=Fraction(1, 3)' - >>> f'{calculation = }' - 'calculation = Fraction(1, 3)' - >>> f'{calculation = !s}' - 'calculation = 1/3' + >>> one_third = Fraction(1, 3) + >>> f'{one_third:.6f}' + '0.333333' + >>> f'{one_third:_^+10}' + '___+1/3___' + >>> >>> f'{one_third!r:_^20}' + '___Fraction(1, 3)___' + >>> f'{one_third = :~>10}~' + 'one_third = ~~~~~~~1/3~' -Once the output has been evaluated, it can be formatted using a -:ref:`format specifier ` following a colon (``':'``). -After the expression has been evaluated, and possibly converted to a string, -the :meth:`!__format__` method of the result is called with the format specifier, -or the empty string if no format specifier is given. -The formatted result is then used as the final value for the replacement field. -For example: +.. _stdtypes-tstrings: -.. doctest:: +Template String Literals (t-strings) +------------------------------------ - >>> from fractions import Fraction - >>> f'{Fraction(1, 7):.6f}' - '0.142857' - >>> f'{Fraction(1, 7):_^+10}' - '___+1/7___' +An :dfn:`t-string` (formally a :dfn:`template string literal`) is +a string literal that is prefixed with ``t`` or ``T``. + +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, +with for the following differences: + +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. + +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. + +* Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _old-string-formatting: @@ -2615,11 +2827,12 @@ For example: The formatting operations described here exhibit a variety of quirks that lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :ref:`formatted string literals - `, the :meth:`str.format` interface, or :ref:`template strings - ` may help avoid these errors. Each of these - alternatives provides their own trade-offs and benefits of simplicity, - flexibility, and/or extensibility. + dictionaries correctly). + + Using :ref:`formatted string literals `, the :meth:`str.format` + interface, or :class:`string.Template` may help avoid these errors. + Each of these alternatives provides their own trade-offs and benefits of + simplicity, flexibility, and/or extensibility. String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. @@ -2831,7 +3044,8 @@ binary protocols are based on the ASCII text encoding, bytes objects offer several methods that are only valid when working with ASCII compatible data and are closely related to string objects in a variety of other ways. -.. class:: bytes([source[, encoding[, errors]]]) +.. class:: bytes(source=b'') + bytes(source, encoding, errors='strict') Firstly, the syntax for bytes literals is largely the same as that for string literals, except that a ``b`` prefix is added: @@ -2871,7 +3085,7 @@ data and are closely related to string objects in a variety of other ways. numbers are a commonly used format for describing binary data. Accordingly, the bytes type has an additional class method to read data in that format: - .. classmethod:: fromhex(string) + .. classmethod:: fromhex(string, /) This :class:`bytes` class method returns a bytes object, decoding the given string object. The string must contain two hexadecimal digits per @@ -2891,7 +3105,8 @@ data and are closely related to string objects in a variety of other ways. A reverse conversion function exists to transform a bytes object into its hexadecimal representation. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the instance. @@ -2940,7 +3155,8 @@ Bytearray Objects :class:`bytearray` objects are a mutable counterpart to :class:`bytes` objects. -.. class:: bytearray([source[, encoding[, errors]]]) +.. class:: bytearray(source=b'') + bytearray(source, encoding, errors='strict') There is no dedicated literal syntax for bytearray objects, instead they are always created by calling the constructor: @@ -2960,7 +3176,7 @@ objects. numbers are a commonly used format for describing binary data. Accordingly, the bytearray type has an additional class method to read data in that format: - .. classmethod:: fromhex(string) + .. classmethod:: fromhex(string, /) This :class:`bytearray` class method returns bytearray object, decoding the given string object. The string must contain two hexadecimal digits @@ -2980,7 +3196,8 @@ objects. A reverse conversion function exists to transform a bytearray object into its hexadecimal representation. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the instance. @@ -2995,7 +3212,7 @@ objects. optional *sep* and *bytes_per_sep* parameters to insert separators between bytes in the hex output. - .. method:: resize(size) + .. method:: resize(size, /) Resize the :class:`bytearray` to contain *size* bytes. *size* must be greater than or equal to 0. @@ -3027,6 +3244,30 @@ objects. .. versionadded:: 3.14 + .. method:: take_bytes(n=None, /) + + Remove the first *n* bytes from the bytearray and return them as an immutable + :class:`bytes`. + By default (if *n* is ``None``), return all bytes and clear the bytearray. + + If *n* is negative, index from the end and take the first :func:`len` + plus *n* bytes. If *n* is out of bounds, raise :exc:`IndexError`. + + Taking less than the full length will leave remaining bytes in the + :class:`bytearray`, which requires a copy. If the remaining bytes should be + discarded, use :func:`~bytearray.resize` or :keyword:`del` to truncate + then :func:`~bytearray.take_bytes` without a size. + + .. impl-detail:: + + Taking all bytes is a zero-copy operation. + + .. versionadded:: 3.15 + + See the :ref:`What's New ` entry for + common code patterns which can be optimized with + :meth:`bytearray.take_bytes`. + Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytearray object of length 1. (This contrasts with text strings, where @@ -3218,8 +3459,8 @@ arbitrary binary data. Also accept an integer in the range 0 to 255 as the subsequence. -.. method:: bytes.join(iterable) - bytearray.join(iterable) +.. method:: bytes.join(iterable, /) + bytearray.join(iterable, /) Return a bytes or bytearray object which is the concatenation of the binary data sequences in *iterable*. A :exc:`TypeError` will be raised @@ -3229,8 +3470,8 @@ arbitrary binary data. bytearray object providing this method. -.. staticmethod:: bytes.maketrans(from, to) - bytearray.maketrans(from, to) +.. staticmethod:: bytes.maketrans(from, to, /) + bytearray.maketrans(from, to, /) This static method returns a translation table usable for :meth:`bytes.translate` that will map each character in *from* into the @@ -3240,8 +3481,8 @@ arbitrary binary data. .. versionadded:: 3.1 -.. method:: bytes.partition(sep) - bytearray.partition(sep) +.. method:: bytes.partition(sep, /) + bytearray.partition(sep, /) Split the sequence at the first occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself or its @@ -3253,8 +3494,8 @@ arbitrary binary data. The separator to search for may be any :term:`bytes-like object`. -.. method:: bytes.replace(old, new[, count]) - bytearray.replace(old, new[, count]) +.. method:: bytes.replace(old, new, count=-1, /) + bytearray.replace(old, new, count=-1, /) Return a copy of the sequence with all occurrences of subsequence *old* replaced by *new*. If the optional argument *count* is given, only the @@ -3297,8 +3538,8 @@ arbitrary binary data. Also accept an integer in the range 0 to 255 as the subsequence. -.. method:: bytes.rpartition(sep) - bytearray.rpartition(sep) +.. method:: bytes.rpartition(sep, /) + bytearray.rpartition(sep, /) Split the sequence at the last occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself or its @@ -3348,8 +3589,8 @@ with arbitrary binary data by passing appropriate arguments. Note that all of the bytearray methods in this section do *not* operate in place, and instead produce new objects. -.. method:: bytes.center(width[, fillbyte]) - bytearray.center(width[, fillbyte]) +.. method:: bytes.center(width, fillbyte=b' ', /) + bytearray.center(width, fillbyte=b' ', /) Return a copy of the object centered in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3362,8 +3603,8 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.ljust(width[, fillbyte]) - bytearray.ljust(width[, fillbyte]) +.. method:: bytes.ljust(width, fillbyte=b' ', /) + bytearray.ljust(width, fillbyte=b' ', /) Return a copy of the object left justified in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3376,14 +3617,13 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.lstrip([chars]) - bytearray.lstrip([chars]) +.. method:: bytes.lstrip(bytes=None, /) + bytearray.lstrip(bytes=None, /) Return a copy of the sequence with specified leading bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults - to removing ASCII whitespace. The *chars* argument is not a prefix; + *bytes* argument is a binary sequence specifying the set of byte values to + be removed. If omitted or ``None``, the *bytes* argument defaults + to removing ASCII whitespace. The *bytes* argument is not a prefix; rather, all combinations of its values are stripped:: >>> b' spacious '.lstrip() @@ -3407,8 +3647,8 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.rjust(width[, fillbyte]) - bytearray.rjust(width[, fillbyte]) +.. method:: bytes.rjust(width, fillbyte=b' ', /) + bytearray.rjust(width, fillbyte=b' ', /) Return a copy of the object right justified in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3432,14 +3672,13 @@ produce new objects. :meth:`split` which is described in detail below. -.. method:: bytes.rstrip([chars]) - bytearray.rstrip([chars]) +.. method:: bytes.rstrip(bytes=None, /) + bytearray.rstrip(bytes=None, /) Return a copy of the sequence with specified trailing bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults to - removing ASCII whitespace. The *chars* argument is not a suffix; rather, + *bytes* argument is a binary sequence specifying the set of byte values to + be removed. If omitted or ``None``, the *bytes* argument defaults to + removing ASCII whitespace. The *bytes* argument is not a suffix; rather, all combinations of its values are stripped:: >>> b' spacious '.rstrip() @@ -3509,14 +3748,13 @@ produce new objects. [b'1', b'2', b'3'] -.. method:: bytes.strip([chars]) - bytearray.strip([chars]) +.. method:: bytes.strip(bytes=None, /) + bytearray.strip(bytes=None, /) Return a copy of the sequence with specified leading and trailing bytes - removed. The *chars* argument is a binary sequence specifying the set of - byte values to be removed - the name refers to the fact this method is - usually used with ASCII characters. If omitted or ``None``, the *chars* - argument defaults to removing ASCII whitespace. The *chars* argument is + removed. The *bytes* argument is a binary sequence specifying the set of + byte values to be removed. If omitted or ``None``, the *bytes* + argument defaults to removing ASCII whitespace. The *bytes* argument is not a prefix or suffix; rather, all combinations of its values are stripped:: @@ -3838,8 +4076,8 @@ place, and instead produce new objects. always produces a new object, even if no changes were made. -.. method:: bytes.zfill(width) - bytearray.zfill(width) +.. method:: bytes.zfill(width, /) + bytearray.zfill(width, /) Return a copy of the sequence left filled with ASCII ``b'0'`` digits to make a sequence of length *width*. A leading sign prefix (``b'+'``/ @@ -4251,7 +4489,8 @@ copying. in-memory Fortran order is preserved. For non-contiguous views, the data is converted to C first. *order=None* is the same as *order='C'*. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the buffer. :: @@ -4336,7 +4575,8 @@ copying. .. versionadded:: 3.2 - .. method:: cast(format[, shape]) + .. method:: cast(format, /) + cast(format, shape, /) Cast a memoryview to a new format or shape. *shape* defaults to ``[byte_length//new_itemsize]``, which means that the result view @@ -4435,7 +4675,7 @@ copying. .. versionadded:: 3.14 - .. method:: index(value, start=0, stop=sys.maxsize, /) + .. method:: index(value, start=0, stop=sys.maxsize, /) Return the index of the first occurrence of *value* (at or after index *start* and before index *stop*). @@ -4586,11 +4826,12 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no -hash value and cannot be used as either a dictionary key or as an element of -another set. The :class:`frozenset` type is immutable and :term:`hashable` --- -its contents cannot be altered after it is created; it can therefore be used as -a dictionary key or as an element of another set. +like :meth:`~set.add` and :meth:`~set.remove`. +Since it is mutable, it has no hash value and cannot be used as +either a dictionary key or as an element of another set. +The :class:`frozenset` type is immutable and :term:`hashable` --- +its contents cannot be altered after it is created; +it can therefore be used as a dictionary key or as an element of another set. Non-empty sets (not frozensets) can be created by placing a comma-separated list of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the @@ -4598,8 +4839,8 @@ of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to t The constructors for both classes work the same: -.. class:: set([iterable]) - frozenset([iterable]) +.. class:: set(iterable=(), /) + frozenset(iterable=(), /) Return a new set or frozenset object whose elements are taken from *iterable*. The elements of a set must be :term:`hashable`. To @@ -4607,164 +4848,172 @@ The constructors for both classes work the same: objects. If *iterable* is not specified, a new empty set is returned. - Sets can be created by several means: +Sets can be created by several means: - * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` - * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` - * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` +* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` +* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` +* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` - Instances of :class:`set` and :class:`frozenset` provide the following - operations: +Instances of :class:`set` and :class:`frozenset` provide the following +operations: - .. describe:: len(s) +.. describe:: len(s) - Return the number of elements in set *s* (cardinality of *s*). + Return the number of elements in set *s* (cardinality of *s*). - .. describe:: x in s +.. describe:: x in s - Test *x* for membership in *s*. + Test *x* for membership in *s*. - .. describe:: x not in s +.. describe:: x not in s - Test *x* for non-membership in *s*. + Test *x* for non-membership in *s*. - .. method:: isdisjoint(other) +.. method:: frozenset.isdisjoint(other, /) + set.isdisjoint(other, /) - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. + Return ``True`` if the set has no elements in common with *other*. Sets are + disjoint if and only if their intersection is the empty set. - .. method:: issubset(other) - set <= other +.. method:: frozenset.issubset(other, /) + set.issubset(other, /) +.. describe:: set <= other - Test whether every element in the set is in *other*. + Test whether every element in the set is in *other*. - .. method:: set < other +.. describe:: set < other - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. + Test whether the set is a proper subset of *other*, that is, + ``set <= other and set != other``. - .. method:: issuperset(other) - set >= other +.. method:: frozenset.issuperset(other, /) + set.issuperset(other, /) +.. describe:: set >= other - Test whether every element in *other* is in the set. + Test whether every element in *other* is in the set. - .. method:: set > other +.. describe:: set > other - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. + Test whether the set is a proper superset of *other*, that is, ``set >= + other and set != other``. - .. method:: union(*others) - set | other | ... +.. method:: frozenset.union(*others) + set.union(*others) +.. describe:: set | other | ... - Return a new set with elements from the set and all others. + Return a new set with elements from the set and all others. - .. method:: intersection(*others) - set & other & ... +.. method:: frozenset.intersection(*others) + set.intersection(*others) +.. describe:: set & other & ... - Return a new set with elements common to the set and all others. + Return a new set with elements common to the set and all others. - .. method:: difference(*others) - set - other - ... +.. method:: frozenset.difference(*others) + set.difference(*others) +.. describe:: set - other - ... - Return a new set with elements in the set that are not in the others. + Return a new set with elements in the set that are not in the others. - .. method:: symmetric_difference(other) - set ^ other +.. method:: frozenset.symmetric_difference(other, /) + set.symmetric_difference(other, /) +.. describe:: set ^ other - Return a new set with elements in either the set or *other* but not both. + Return a new set with elements in either the set or *other* but not both. - .. method:: copy() +.. method:: frozenset.copy() + set.copy() - Return a shallow copy of the set. + Return a shallow copy of the set. - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. +Note, the non-operator versions of :meth:`~frozenset.union`, +:meth:`~frozenset.intersection`, :meth:`~frozenset.difference`, :meth:`~frozenset.symmetric_difference`, :meth:`~frozenset.issubset`, and +:meth:`~frozenset.issuperset` methods will accept any iterable as an argument. In +contrast, their operator based counterparts require their arguments to be +sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` +in favor of the more readable ``set('abc').intersection('cbs')``. - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). +Both :class:`set` and :class:`frozenset` support set to set comparisons. Two +sets are equal if and only if every element of each set is contained in the +other (each is a subset of the other). A set is less than another set if and +only if the first set is a proper subset of the second set (is a subset, but +is not equal). A set is greater than another set if and only if the first set +is a proper superset of the second set (is a superset, but is not equal). - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. +Instances of :class:`set` are compared to instances of :class:`frozenset` +based on their members. For example, ``set('abc') == frozenset('abc')`` +returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. +The subset and equality comparisons do not generalize to a total ordering +function. For example, any two nonempty disjoint sets are not equal and are not +subsets of each other, so *all* of the following return ``False``: ``ab``. - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. +Since sets only define partial ordering (subset relationships), the output of +the :meth:`list.sort` method is undefined for lists of sets. - Set elements, like dictionary keys, must be :term:`hashable`. +Set elements, like dictionary keys, must be :term:`hashable`. - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. +Binary operations that mix :class:`set` instances with :class:`frozenset` +return the type of the first operand. For example: ``frozenset('ab') | +set('bc')`` returns an instance of :class:`frozenset`. - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: +The following table lists operations available for :class:`set` that do not +apply to immutable instances of :class:`frozenset`: - .. method:: update(*others) - set |= other | ... +.. method:: set.update(*others) +.. describe:: set |= other | ... - Update the set, adding elements from all others. + Update the set, adding elements from all others. - .. method:: intersection_update(*others) - set &= other & ... +.. method:: set.intersection_update(*others) +.. describe:: set &= other & ... - Update the set, keeping only elements found in it and all others. + Update the set, keeping only elements found in it and all others. - .. method:: difference_update(*others) - set -= other | ... +.. method:: set.difference_update(*others) +.. describe:: set -= other | ... - Update the set, removing elements found in others. + Update the set, removing elements found in others. - .. method:: symmetric_difference_update(other) - set ^= other +.. method:: set.symmetric_difference_update(other, /) +.. describe:: set ^= other - Update the set, keeping only elements found in either set, but not in both. + Update the set, keeping only elements found in either set, but not in both. - .. method:: add(elem) +.. method:: set.add(elem, /) - Add element *elem* to the set. + Add element *elem* to the set. - .. method:: remove(elem) +.. method:: set.remove(elem, /) - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. + Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is + not contained in the set. - .. method:: discard(elem) +.. method:: set.discard(elem, /) - Remove element *elem* from the set if it is present. + Remove element *elem* from the set if it is present. - .. method:: pop() +.. method:: set.pop() - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. + Remove and return an arbitrary element from the set. Raises + :exc:`KeyError` if the set is empty. - .. method:: clear() +.. method:: set.clear() - Remove all elements from the set. + Remove all elements from the set. - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. +Note, the non-operator versions of the :meth:`~set.update`, +:meth:`~set.intersection_update`, :meth:`~set.difference_update`, and +:meth:`~set.symmetric_difference_update` methods will accept any iterable as an +argument. - Note, the *elem* argument to the :meth:`~object.__contains__`, - :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. +Note, the *elem* argument to the :meth:`~object.__contains__`, +:meth:`~set.remove`, and +:meth:`~set.discard` methods may be a set. To support searching for an equivalent +frozenset, a temporary one is created from *elem*. .. _typesmapping: @@ -4794,8 +5043,8 @@ Values that compare equal (such as ``1``, ``1.0``, and ``True``) can be used interchangeably to index the same dictionary entry. .. class:: dict(**kwargs) - dict(mapping, **kwargs) - dict(iterable, **kwargs) + dict(mapping, /, **kwargs) + dict(iterable, /, **kwargs) Return a new dictionary initialized from an optional positional argument and a possibly empty set of keyword arguments. @@ -4823,7 +5072,13 @@ can be used interchangeably to index the same dictionary entry. being added is already present, the value from the keyword argument replaces the value from the positional argument. - To illustrate, the following examples all return a dictionary equal to + Providing keyword arguments as in the first example only works for keys that + are valid Python identifiers. Otherwise, any valid keys can be used. + + Dictionaries compare equal if and only if they have the same ``(key, + value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise + :exc:`TypeError`. To illustrate dictionary creation and equality, + the following examples all return a dictionary equal to ``{"one": 1, "two": 2, "three": 3}``:: >>> a = dict(one=1, two=2, three=3) @@ -4838,6 +5093,27 @@ can be used interchangeably to index the same dictionary entry. Providing keyword arguments as in the first example only works for keys that are valid Python identifiers. Otherwise, any valid keys can be used. + Dictionaries preserve insertion order. Note that updating a key does not + affect the order. Keys added after deletion are inserted at the end. :: + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + .. versionchanged:: 3.7 + Dictionary order is guaranteed to be insertion order. This behavior was + an implementation detail of CPython from 3.6. These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -4857,13 +5133,13 @@ can be used interchangeably to index the same dictionary entry. .. index:: __missing__() - If a subclass of dict defines a method :meth:`__missing__` and *key* + If a subclass of dict defines a method :meth:`~object.__missing__` and *key* is not present, the ``d[key]`` operation calls that method with the key *key* as argument. The ``d[key]`` operation then returns or raises whatever is returned or raised by the ``__missing__(key)`` call. - No other operations or methods invoke :meth:`__missing__`. If - :meth:`__missing__` is not defined, :exc:`KeyError` is raised. - :meth:`__missing__` must be a method; it cannot be an instance variable:: + No other operations or methods invoke :meth:`~object.__missing__`. If + :meth:`~object.__missing__` is not defined, :exc:`KeyError` is raised. + :meth:`~object.__missing__` must be a method; it cannot be an instance variable:: >>> class Counter(dict): ... def __missing__(self, key): @@ -4877,7 +5153,8 @@ can be used interchangeably to index the same dictionary entry. 1 The example above shows part of the implementation of - :class:`collections.Counter`. A different ``__missing__`` method is used + :class:`collections.Counter`. + A different :meth:`!__missing__` method is used by :class:`collections.defaultdict`. .. describe:: d[key] = value @@ -4936,7 +5213,8 @@ can be used interchangeably to index the same dictionary entry. Return a new view of the dictionary's keys. See the :ref:`documentation of view objects `. - .. method:: pop(key[, default]) + .. method:: pop(key, /) + pop(key, default, /) If *key* is in the dictionary, remove it and return its value, else return *default*. If *default* is not given and *key* is not in the dictionary, @@ -4968,9 +5246,11 @@ can be used interchangeably to index the same dictionary entry. with a value of *default* and return *default*. *default* defaults to ``None``. - .. method:: update([other]) + .. method:: update(**kwargs) + update(mapping, /, **kwargs) + update(iterable, /, **kwargs) - Update the dictionary with the key/value pairs from *other*, overwriting + Update the dictionary with the key/value pairs from *mapping* or *iterable* and *kwargs*, overwriting existing keys. Return ``None``. :meth:`update` accepts either another object with a ``keys()`` method (in @@ -5008,32 +5288,6 @@ can be used interchangeably to index the same dictionary entry. .. versionadded:: 3.9 - Dictionaries compare equal if and only if they have the same ``(key, - value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise - :exc:`TypeError`. - - Dictionaries preserve insertion order. Note that updating a key does not - affect the order. Keys added after deletion are inserted at the end. :: - - >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} - >>> d - {'one': 1, 'two': 2, 'three': 3, 'four': 4} - >>> list(d) - ['one', 'two', 'three', 'four'] - >>> list(d.values()) - [1, 2, 3, 4] - >>> d["one"] = 42 - >>> d - {'one': 42, 'two': 2, 'three': 3, 'four': 4} - >>> del d["two"] - >>> d["two"] = None - >>> d - {'one': 42, 'three': 3, 'four': 4, 'two': None} - - .. versionchanged:: 3.7 - Dictionary order is guaranteed to be insertion order. This behavior was - an implementation detail of CPython from 3.6. - Dictionaries and dictionary views are reversible. :: >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} @@ -5409,6 +5663,7 @@ list is non-exhaustive. * :class:`collections.abc.MutableMapping` * :class:`collections.abc.Sequence` * :class:`collections.abc.MutableSequence` +* :class:`collections.abc.ByteString` * :class:`collections.abc.MappingView` * :class:`collections.abc.KeysView` * :class:`collections.abc.ItemsView` @@ -5684,9 +5939,10 @@ Methods .. index:: pair: object; method -Methods are functions that are called using the attribute notation. There are -two flavors: :ref:`built-in methods ` (such as :meth:`append` -on lists) and :ref:`class instance method `. +Methods are functions that are called using the attribute notation. +There are two flavors: :ref:`built-in methods ` +(such as :meth:`~list.append` on lists) +and :ref:`class instance method `. Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) through an @@ -5792,13 +6048,34 @@ It is written as ``None``. The Ellipsis Object ------------------- -This object is commonly used by slicing (see :ref:`slicings`). It supports no -special operations. There is exactly one ellipsis object, named +This object is commonly used to indicate that something is omitted. +It supports no special operations. There is exactly one ellipsis object, named :const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` produces the :const:`Ellipsis` singleton. It is written as ``Ellipsis`` or ``...``. +In typical use, ``...`` as the ``Ellipsis`` object appears in a few different +places, for instance: + +- In type annotations, such as :ref:`callable arguments ` + or :ref:`tuple elements `. + +- As the body of a function instead of a :ref:`pass statement `. + +- In third-party libraries, such as `Numpy's slicing and striding + `_. + +Python also uses three dots in ways that are not ``Ellipsis`` objects, for instance: + +- Doctest's :const:`ELLIPSIS `, as a pattern for missing content. + +- The default Python prompt of the :term:`interactive` shell when partial input is incomplete. + +Lastly, the Python documentation often uses three dots in conventional English +usage to mean omitted content, even in code examples that also use them as the +``Ellipsis``. + .. _bltin-notimplemented-object: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index b44d98819b6..58c836c7382 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -4,7 +4,7 @@ .. module:: string :synopsis: Common string operations. -**Source code:** :source:`Lib/string.py` +**Source code:** :source:`Lib/string/__init__.py` -------------- @@ -198,8 +198,9 @@ Format String Syntax The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but it is -less sophisticated and, in particular, does not support arbitrary expressions. +related to that of :ref:`formatted string literals ` and +:ref:`template string literals `, but it is less sophisticated +and, in particular, does not support arbitrary expressions in interpolations. .. index:: single: {} (curly brackets); in string formatting @@ -264,6 +265,8 @@ Some simple format string examples:: "Weight in tons {0.weight}" # 'weight' attribute of first positional arg "Units destroyed: {players[0]}" # First element of keyword argument 'players'. +.. _formatstrings-conversion: + The *conversion* field causes a type coercion before formatting. Normally, the job of formatting a value is done by the :meth:`~object.__format__` method of the value itself. However, in some cases it is desirable to force a type to be formatted @@ -306,7 +309,7 @@ Format Specification Mini-Language "Format specifications" are used within replacement fields contained within a format string to define how individual values are presented (see -:ref:`formatstrings` and :ref:`f-strings`). +:ref:`formatstrings`, :ref:`f-strings`, and :ref:`t-strings`). They can also be passed directly to the built-in :func:`format` function. Each formattable type may define how the format specification is to be interpreted. @@ -328,7 +331,7 @@ The general form of a *standard format specifier* is: sign: "+" | "-" | " " width_and_precision: [`width_with_grouping`][`precision_with_grouping`] width_with_grouping: [`width`][`grouping`] - precision_with_grouping: "." [`precision`][`grouping`] + precision_with_grouping: "." [`precision`][`grouping`] | "." `grouping` width: `~python-grammar:digit`+ precision: `~python-grammar:digit`+ grouping: "," | "_" @@ -789,10 +792,22 @@ Nesting arguments and more complex examples:: -.. _template-strings: +.. _template-strings-pep292: -Template strings ----------------- +Template strings ($-strings) +---------------------------- + +.. note:: + + The feature described here was introduced in Python 2.4; + a simple templating method based upon regular expressions. + It predates :meth:`str.format`, :ref:`formatted string literals `, + and :ref:`template string literals `. + + It is unrelated to template string literals (t-strings), + which were introduced in Python 3.14. + These evaluate to :class:`string.templatelib.Template` objects, + found in the :mod:`string.templatelib` module. Template strings provide simpler string substitutions as described in :pep:`292`. A primary use case for template strings is for @@ -858,7 +873,7 @@ these rules. The methods of :class:`Template` are: .. method:: is_valid() - Returns false if the template has invalid placeholders that will cause + Returns ``False`` if the template has invalid placeholders that will cause :meth:`substitute` to raise :exc:`ValueError`. .. versionadded:: 3.11 diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst new file mode 100644 index 00000000000..a5b2d796aaf --- /dev/null +++ b/Doc/library/string.templatelib.rst @@ -0,0 +1,349 @@ +:mod:`!string.templatelib` --- Support for template string literals +=================================================================== + +.. module:: string.templatelib + :synopsis: Support for template string literals. + +**Source code:** :source:`Lib/string/templatelib.py` + +-------------- + +.. seealso:: + + * :ref:`Format strings ` + * :ref:`Template string literal (t-string) syntax ` + * :pep:`750` + +.. _template-strings: + +Template strings +---------------- + +.. versionadded:: 3.14 + +Template strings are a mechanism for custom string processing. +They have the full flexibility of Python's :ref:`f-strings`, +but return a :class:`Template` instance that gives access +to the static and interpolated (in curly brackets) parts of a string +*before* they are combined. + +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``, like so: + +.. code-block:: pycon + + >>> pi = 3.14 + >>> t't-strings are new in Python {pi!s}!' + Template( + strings=('t-strings are new in Python ', '!'), + interpolations=(Interpolation(3.14, 'pi', 's', ''),) + ) + +Types +----- + +.. class:: Template + + The :class:`!Template` class describes the contents of a template string. + It is immutable, meaning that attributes of a template cannot be reassigned. + + The most common way to create a :class:`!Template` instance is to use the + :ref:`template string literal syntax `. + This syntax is identical to that of :ref:`f-strings `, + except that it uses a ``t`` prefix in place of an ``f``: + + >>> cheese = 'Red Leicester' + >>> template = t"We're fresh out of {cheese}, sir." + >>> type(template) + + + Templates are stored as sequences of literal :attr:`~Template.strings` + and dynamic :attr:`~Template.interpolations`. + A :attr:`~Template.values` attribute holds the values of the interpolations: + + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.strings + ('Ah! We do have ', '.') + >>> template.interpolations + (Interpolation('Camembert', ...),) + >>> template.values + ('Camembert',) + + The :attr:`!strings` tuple has one more element than :attr:`!interpolations` + and :attr:`!values`; the interpolations “belong” between the strings. + This may be easier to understand when tuples are aligned + + .. code-block:: python + + template.strings: ('Ah! We do have ', '.') + template.values: ( 'Camembert', ) + + .. rubric:: Attributes + + .. attribute:: strings + :type: tuple[str, ...] + + A :class:`tuple` of the static strings in the template. + + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.strings + ('Ah! We do have ', '.') + + Empty strings *are* included in the tuple: + + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> template = t'Ah! {response}{cheese}.' + >>> template.strings + ('Ah! ', '', '.') + + The ``strings`` tuple is never empty, and always contains one more + string than the ``interpolations`` and ``values`` tuples: + + >>> t''.strings + ('',) + >>> t''.values + () + >>> t'{'cheese'}'.strings + ('', '') + >>> t'{'cheese'}'.values + ('cheese',) + + .. attribute:: interpolations + :type: tuple[Interpolation, ...] + + A :class:`tuple` of the interpolations in the template. + + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.interpolations + (Interpolation('Camembert', 'cheese', None, ''),) + + The ``interpolations`` tuple may be empty and always contains one fewer + values than the ``strings`` tuple: + + >>> t'Red Leicester'.interpolations + () + + .. attribute:: values + :type: tuple[object, ...] + + A tuple of all interpolated values in the template. + + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.values + ('Camembert',) + + The ``values`` tuple always has the same length as the + ``interpolations`` tuple. It is always equivalent to + ``tuple(i.value for i in template.interpolations)``. + + .. rubric:: Methods + + .. method:: __new__(*args: str | Interpolation) + + While literal syntax is the most common way to create a :class:`!Template`, + it is also possible to create them directly using the constructor: + + >>> from string.templatelib import Interpolation, Template + >>> cheese = 'Camembert' + >>> template = Template( + ... 'Ah! We do have ', Interpolation(cheese, 'cheese'), '.' + ... ) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] + + If multiple strings are passed consecutively, they will be concatenated + into a single value in the :attr:`~Template.strings` attribute. For example, + the following code creates a :class:`Template` with a single final string: + + >>> from string.templatelib import Template + >>> template = Template('Ah! We do have ', 'Camembert', '.') + >>> template.strings + ('Ah! We do have Camembert.',) + + If multiple interpolations are passed consecutively, they will be treated + as separate interpolations and an empty string will be inserted between them. + For example, the following code creates a template with empty placeholders + in the :attr:`~Template.strings` attribute: + + >>> from string.templatelib import Interpolation, Template + >>> template = Template( + ... Interpolation('Camembert', 'cheese'), + ... Interpolation('.', 'punctuation'), + ... ) + >>> template.strings + ('', '', '') + + .. describe:: iter(template) + + Iterate over the template, yielding each non-empty string and + :class:`Interpolation` in the correct order: + + >>> cheese = 'Camembert' + >>> list(t'Ah! We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] + + .. caution:: + + Empty strings are **not** included in the iteration: + + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> list(t'Ah! {response}{cheese}.') # doctest: +NORMALIZE_WHITESPACE + ['Ah! ', + Interpolation('We do have ', 'response', None, ''), + Interpolation('Camembert', 'cheese', None, ''), + '.'] + + .. describe:: template + other + template += other + + Concatenate this template with another, returning a new + :class:`!Template` instance: + + >>> cheese = 'Camembert' + >>> list(t'Ah! ' + t'We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] + + Concatenating a :class:`!Template` and a ``str`` is **not** supported. + This is because it is unclear whether the string should be treated as + a static string or an interpolation. + If you want to concatenate a :class:`!Template` with a string, + you should either wrap the string directly in a :class:`!Template` + (to treat it as a static string) + or use an :class:`!Interpolation` (to treat it as dynamic): + + >>> from string.templatelib import Interpolation, Template + >>> template = t'Ah! ' + >>> # Treat 'We do have ' as a static string + >>> template += Template('We do have ') + >>> # Treat cheese as an interpolation + >>> cheese = 'Camembert' + >>> template += Template(Interpolation(cheese, 'cheese')) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, '')] + + +.. class:: Interpolation + + The :class:`!Interpolation` type represents an expression inside a template string. + It is immutable, meaning that attributes of an interpolation cannot be reassigned. + + Interpolations support pattern matching, allowing you to match against + their attributes with the :ref:`match statement `: + + >>> from string.templatelib import Interpolation + >>> interpolation = t'{1. + 2.:.2f}'.interpolations[0] + >>> interpolation + Interpolation(3.0, '1. + 2.', None, '.2f') + >>> match interpolation: + ... case Interpolation(value, expression, conversion, format_spec): + ... print(value, expression, conversion, format_spec, sep=' | ') + ... + 3.0 | 1. + 2. | None | .2f + + .. rubric:: Attributes + + .. attribute:: value + :type: object + + The evaluated value of the interpolation. + + >>> t'{1 + 2}'.interpolations[0].value + 3 + + .. attribute:: expression + :type: str + + For interpolations created by t-string literals, :attr:`!expression` + is the expression text found inside the curly brackets (``{`` & ``}``), + including any whitespace, excluding the curly brackets themselves, + and ending before the first ``!``, ``:``, or ``=`` if any is present. + For manually created interpolations, :attr:`!expression` is the arbitrary + string provided when constructing the interpolation instance. + + We recommend using valid Python expressions or the empty string for the + ``expression`` field of manually created :class:`!Interpolation` + instances, although this is not enforced at runtime. + + >>> t'{1 + 2}'.interpolations[0].expression + '1 + 2' + + .. attribute:: conversion + :type: typing.Literal['a', 'r', 's'] | None + + The conversion to apply to the value, or ``None``. + + The :attr:`!conversion` is the optional conversion to apply + to the value: + + >>> t'{1 + 2!a}'.interpolations[0].conversion + 'a' + + .. note:: + + Unlike f-strings, where conversions are applied automatically, + the expected behavior with t-strings is that code that *processes* the + :class:`!Template` will decide how to interpret and whether to apply + the :attr:`!conversion`. + For convenience, the :func:`convert` function can be used to mimic + f-string conversion semantics. + + .. attribute:: format_spec + :type: str + + The format specification to apply to the value. + + The :attr:`!format_spec` is an optional, arbitrary string + used as the format specification to present the value: + + >>> t'{1 + 2:.2f}'.interpolations[0].format_spec + '.2f' + + .. note:: + + Unlike f-strings, where format specifications are applied automatically + via the :func:`format` protocol, the expected behavior with + t-strings is that code that *processes* the interpolation will + decide how to interpret and whether to apply the format specification. + As a result, :attr:`!format_spec` values in interpolations + can be arbitrary strings, + including those that do not conform to the :func:`format` protocol. + + .. rubric:: Methods + + .. method:: __new__(value: object, \ + expression: str, \ + conversion: typing.Literal['a', 'r', 's'] | None = None, \ + format_spec: str = '') + + Create a new :class:`!Interpolation` object from component parts. + + :param value: The evaluated, in-scope result of the interpolation. + :param expression: The text of a valid Python expression, + or an empty string. + :param conversion: The :ref:`conversion ` to be used, + one of ``None``, ``'a'``, ``'r'``, or ``'s'``. + :param format_spec: An optional, arbitrary string used as the + :ref:`format specification ` to present the value. + + +Helper functions +---------------- + +.. function:: convert(obj, /, conversion) + + Applies formatted string literal :ref:`conversion ` + semantics to the given object *obj*. + This is frequently useful for custom template string processing logic. + + Three conversion flags are currently supported: + + * ``'s'`` which calls :func:`str` on the value (like ``!s``), + * ``'r'`` which calls :func:`repr` (like ``!r``), and + * ``'a'`` which calls :func:`ascii` (like ``!a``). + + If the conversion flag is ``None``, *obj* is returned unchanged. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 028a7861f36..b8dfcc31077 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -649,7 +649,7 @@ functions. If specified, *env* must provide any variables required for the program to execute. On Windows, in order to run a `side-by-side assembly`_ the - specified *env* **must** include a valid :envvar:`SystemRoot`. + specified *env* **must** include a valid ``%SystemRoot%``. .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly @@ -831,7 +831,9 @@ Instances of the :class:`Popen` class have the following methods: If the process does not terminate after *timeout* seconds, a :exc:`TimeoutExpired` exception will be raised. Catching this exception and - retrying communication will not lose any output. + retrying communication will not lose any output. Supplying *input* to a + subsequent post-timeout :meth:`communicate` call is in undefined behavior + and may become an error in the future. The child process is not killed if the timeout expires, so in order to cleanup properly a well-behaved application should kill the child process and @@ -844,6 +846,11 @@ Instances of the :class:`Popen` class have the following methods: proc.kill() outs, errs = proc.communicate() + After a call to :meth:`~Popen.communicate` raises :exc:`TimeoutExpired`, do + not call :meth:`~Popen.wait`. Use an additional :meth:`~Popen.communicate` + call to finish handling pipes and populate the :attr:`~Popen.returncode` + attribute. + .. note:: The data read is buffered in memory, so do not use this method if the data @@ -1473,7 +1480,7 @@ handling consistency are valid for these functions. Return ``(exitcode, output)`` of executing *cmd* in a shell. - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and + Execute the string *cmd* in a shell with :func:`check_output` and return a 2-tuple ``(exitcode, output)``. *encoding* and *errors* are used to decode output; see the notes on :ref:`frequently-used-arguments` for more details. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd6..f5e6f9f8acf 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -21,11 +21,17 @@ tables. Generating Symbol Tables ------------------------ -.. function:: symtable(code, filename, compile_type) +.. function:: symtable(code, filename, compile_type, *, module=None) Return the toplevel :class:`SymbolTable` for the Python source *code*. *filename* is the name of the file containing the code. *compile_type* is like the *mode* argument to :func:`compile`. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + .. versionadded:: 3.15 + Added the *module* parameter. Examining Symbol Tables diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0674074b8c0..303655fb128 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -137,7 +137,8 @@ The following events are supported: .. monitoring-event:: PY_UNWIND - Exit from a Python function during exception unwinding. + Exit from a Python function during exception unwinding. This includes exceptions raised directly within the + function and that are allowed to continue to propagate. .. monitoring-event:: PY_YIELD @@ -171,7 +172,7 @@ events, use the expression ``PY_RETURN | PY_START``. if get_events(DEBUGGER_ID) == NO_EVENTS: ... -Events are divided into three groups: + Setting this event deactivates all events. .. _monitoring-event-local: @@ -215,14 +216,17 @@ by another event: The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events are controlled by the :monitoring-event:`CALL` event. -:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the -corresponding :monitoring-event:`CALL` event is being monitored. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be +seen if the corresponding :monitoring-event:`CALL` event is being monitored. + + +.. _monitoring-event-global: Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled. +program and cannot be individually disabled via :data:`DISABLE`. The other events that can be monitored are: @@ -243,20 +247,23 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike +:monitoring-event:`RAISE`. -Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` -event for a :exc:`StopIteration` exception are equivalent, and are treated as interchangeable -when generating events. Implementations will favor :monitoring-event:`STOP_ITERATION` for -performance reasons, but may generate a :monitoring-event:`RAISE` event with a :exc:`StopIteration`. +Note that the :monitoring-event:`STOP_ITERATION` event and the +:monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are +equivalent, and are treated as interchangeable when generating events. +Implementations will favor :monitoring-event:`STOP_ITERATION` for performance +reasons, but may generate a :monitoring-event:`RAISE` event with a +:exc:`StopIteration`. Turning events on and off ------------------------- In order to monitor an event, it must be turned on and a corresponding callback -must be registered. -Events can be turned on or off by setting the events either globally or -for a particular code object. +must be registered. Events can be turned on or off by setting the events either +globally and/or for a particular code object. An event will trigger only once, +even if it is turned on both globally and locally. Setting events globally @@ -285,16 +292,13 @@ in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int - Returns all the local events for *code* + Returns all the :ref:`local events ` for *code* .. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None - Activates all the local events for *code* which are set in *event_set*. - Raises a :exc:`ValueError` if *tool_id* is not in use. - -Local events add to global events, but do not mask them. -In other words, all global events will trigger for a code object, -regardless of the local events. + Activates all the :ref:`local events ` for *code* + which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not + in use. Disabling events @@ -305,15 +309,21 @@ Disabling events A special value that can be returned from a callback function to disable events for the current code location. -Local events can be disabled for a specific code location by returning -:data:`sys.monitoring.DISABLE` from a callback function. This does not change -which events are set, or any other code locations for the same event. +:ref:`Local events ` can be disabled for a specific code +location by returning :data:`sys.monitoring.DISABLE` from a callback function. +This does not change which events are set, or any other code locations for the +same event. Disabling events for specific locations is very important for high performance monitoring. For example, a program can be run under a debugger with no overhead if the debugger disables all monitoring except for a few breakpoints. +If :data:`DISABLE` is returned by a callback for a +:ref:`global event `, :exc:`ValueError` will be raised +by the interpreter in a non-specific location (that is, no traceback will be +provided). + .. function:: restart_events() -> None Enable all the events that were disabled by :data:`sys.monitoring.DISABLE` @@ -325,8 +335,6 @@ except for a few breakpoints. Registering callback functions ------------------------------ -To register a callable for events call - .. function:: register_callback(tool_id: int, event: int, func: Callable | None, /) -> Callable | None Registers the callable *func* for the *event* with the given *tool_id* @@ -335,13 +343,17 @@ To register a callable for events call it is unregistered and returned. Otherwise :func:`register_callback` returns ``None``. + .. audit-event:: sys.monitoring.register_callback func sys.monitoring.register_callback Functions can be unregistered by calling ``sys.monitoring.register_callback(tool_id, event, None)``. Callback functions can be registered and unregistered at any time. -Registering or unregistering a callback function will generate a :func:`sys.audit` event. +Callbacks are called only once regardless if the event is turned on both +globally and locally. As such, if an event could be turned on for both global +and local events by your code then the callback needs to be written to handle +either trigger. Callback function arguments @@ -353,37 +365,46 @@ Callback function arguments that there are no arguments to the call. When an active event occurs, the registered callback function is called. +Callback functions returning an object other than :data:`DISABLE` will have no effect. Different events will provide the callback function with different arguments, as follows: * :monitoring-event:`PY_START` and :monitoring-event:`PY_RESUME`:: - func(code: CodeType, instruction_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int) -> object * :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_YIELD`:: - func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, retval: object) -> object -* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN`:: +* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN` + (*arg0* can be :data:`MISSING` specifically):: - func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, callable: object, arg0: object) -> object + *code* represents the code object where the call is being made, while + *callable* is the object that is about to be called (and thus + triggered the event). If there are no arguments, *arg0* is set to :data:`sys.monitoring.MISSING`. + For instance methods, *callable* will be the function object as found on the + class with *arg0* set to the instance (i.e. the ``self`` argument to the + method). + * :monitoring-event:`RAISE`, :monitoring-event:`RERAISE`, :monitoring-event:`EXCEPTION_HANDLED`, :monitoring-event:`PY_UNWIND`, :monitoring-event:`PY_THROW` and :monitoring-event:`STOP_ITERATION`:: - func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, exception: BaseException) -> object * :monitoring-event:`LINE`:: - func(code: CodeType, line_number: int) -> DISABLE | Any + func(code: CodeType, line_number: int) -> object * :monitoring-event:`BRANCH_LEFT`, :monitoring-event:`BRANCH_RIGHT` and :monitoring-event:`JUMP`:: - func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, destination_offset: int) -> object Note that the *destination_offset* is where the code will next execute. * :monitoring-event:`INSTRUCTION`:: - func(code: CodeType, instruction_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int) -> object diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 55e442b20ff..a0621d4b0db 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -11,6 +11,51 @@ interpreter and to functions that interact strongly with the interpreter. It is always available. Unless explicitly noted otherwise, all variables are read-only. +.. data:: abi_info + + .. versionadded:: 3.15 + + An object containing information about the ABI of the currently running + Python interpreter. + It should include information that affect the CPython ABI in ways that + require a specific build of the interpreter chosen from variants that can + co-exist on a single machine. + For example, it does not encode the base OS (Linux or Windows), but does + include pointer size since some systems support both 32- and 64-bit builds. + The available entries are the same on all platforms; + e.g. *pointer_size* is available even on 64-bit-only architectures. + + The following attributes are available: + + .. attribute:: abi_info.pointer_bits + + The width of pointers in bits, as an integer, + equivalent to ``8 * sizeof(void *)``. + Usually, this is ``32`` or ``64``. + + .. attribute:: abi_info.free_threaded + + A Boolean indicating whether the interpreter was built with + :term:`free threading` support. + This reflects either the presence of the :option:`--disable-gil` + :file:`configure` option (on Unix) + or setting the ``DisableGil`` property (on Windows). + + .. attribute:: abi_info.debug + + A Boolean indicating whether the interpreter was built in + :ref:`debug mode `. + This reflects either the presence of the :option:`--with-pydebug` + :file:`configure` option (on Unix) + or the ``Debug`` configuration (on Windows). + + .. attribute:: abi_info.byteorder + + A string indicating the native byte order, + either ``'big'`` or ``'little'``. + This is the same as the :data:`byteorder` attribute. + + .. data:: abiflags On POSIX systems where Python was built with the standard ``configure`` @@ -523,8 +568,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only Since :func:`exit` ultimately "only" raises an exception, it will only exit the process when called from the main thread, and the exception is not - intercepted. Cleanup actions specified by finally clauses of :keyword:`try` statements - are honored, and it is possible to intercept the exit attempt at an outer level. + intercepted. Cleanup actions specified by :keyword:`finally` clauses of + :keyword:`try` statements are honored, and it is possible to intercept the + exit attempt at an outer level. .. versionchanged:: 3.6 If an error occurs in the cleanup after the Python interpreter @@ -953,6 +999,8 @@ always available. Unless explicitly noted otherwise, all variables are read-only This function should be used for internal and specialized purposes only. It is not guaranteed to exist in all implementations of Python. + .. versionadded:: 3.12 + .. function:: getobjects(limit[, type]) @@ -1128,10 +1176,14 @@ always available. Unless explicitly noted otherwise, all variables are read-only The size of the seed key of the hash algorithm + .. attribute:: hash_info.cutoff + + Cutoff for small string DJBX33A optimization in range ``[1, cutoff)``. + .. versionadded:: 3.2 .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* + Added *algorithm*, *hash_bits*, *seed_bits*, and *cutoff*. .. data:: hexversion @@ -1185,6 +1237,15 @@ always available. Unless explicitly noted otherwise, all variables are read-only ``cache_tag`` is set to ``None``, it indicates that module caching should be disabled. + *supports_isolated_interpreters* is a boolean value, whether + this implementation supports multiple isolated interpreters. + It is ``True`` for CPython on most platforms. Platforms with + this support implement the low-level :mod:`!_interpreters` module. + + .. seealso:: + + :pep:`684`, :pep:`734`, and :mod:`concurrent.interpreters`. + :data:`sys.implementation` may contain additional attributes specific to the Python implementation. These non-standard attributes must start with an underscore, and are not described here. Regardless of its contents, @@ -1194,6 +1255,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.3 + .. versionchanged:: 3.14 + Added ``supports_isolated_interpreters`` field. + .. note:: The addition of new required attributes must go through the normal PEP @@ -1750,7 +1814,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only :func:`settrace` for each thread being debugged or use :func:`threading.settrace`. Trace functions should have three arguments: *frame*, *event*, and - *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, + *arg*. *frame* is the :ref:`current stack frame `. *event* is a string: ``'call'``, ``'line'``, ``'return'``, ``'exception'`` or ``'opcode'``. *arg* depends on the event type. @@ -1933,6 +1997,22 @@ always available. Unless explicitly noted otherwise, all variables are read-only interpreter is pre-release (alpha, beta, or release candidate) then the local and remote interpreters must be the same exact version. + .. audit-event:: sys.remote_exec pid script_path + + When the code is executed in the remote process, an + :ref:`auditing event ` ``sys.remote_exec`` is raised with + the *pid* and the path to the script file. + This event is raised in the process that called :func:`sys.remote_exec`. + + .. audit-event:: cpython.remote_debugger_script script_path + + When the script is executed in the remote process, an + :ref:`auditing event ` + ``cpython.remote_debugger_script`` is raised + with the path in the remote process. + This event is raised in the remote process, not the one + that called :func:`sys.remote_exec`. + .. availability:: Unix, Windows. .. versionadded:: 3.14 @@ -2122,11 +2202,16 @@ always available. Unless explicitly noted otherwise, all variables are read-only The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if :attr:`!err_msg` is ``None``. + if :attr:`!err_msg` is ``None``. Similar to the :mod:`traceback` module, + this adds color to exceptions by default. This can be disabled using + :ref:`environment variables `. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. + .. versionchanged:: 3.15 + Exceptions are now printed with colorful text. + .. seealso:: :func:`excepthook` which handles uncaught exceptions. @@ -2161,8 +2246,11 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. data:: api_version - The C API version for this interpreter. Programmers may find this useful when - debugging version conflicts between Python and extension modules. + The C API version, equivalent to the C macro :c:macro:`PYTHON_API_VERSION`. + Defined for backwards compatibility. + + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. .. data:: version_info diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 684d14a74c4..3b0bfb85da7 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -382,22 +382,22 @@ Other functions Examples of returned values: - - linux-i586 - - linux-alpha (?) + - linux-x86_64 + - linux-aarch64 - solaris-2.6-sun4u - Windows will return one of: + Windows: - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) - win-arm64 (64-bit Windows on ARM64, aka AArch64) - win32 (all others - specifically, sys.platform is returned) - macOS can return: + POSIX based OS: - - macosx-10.6-ppc - - macosx-10.4-ppc64 - - macosx-10.3-i386 - - macosx-10.4-fat + - linux-x86_64 + - macosx-15.5-arm64 + - macosx-26.0-universal2 (macOS on Apple Silicon or Intel) + - android-24-arm64_v8a For other non-POSIX platforms, currently just returns :data:`sys.platform`. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f9cb5495e60..5ff8502bbe2 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -18,8 +18,16 @@ higher-level functions in :ref:`shutil `. Some facts and figures: -* reads and writes :mod:`gzip`, :mod:`bz2` and :mod:`lzma` compressed archives - if the respective modules are available. +* reads and writes :mod:`gzip`, :mod:`bz2`, :mod:`compression.zstd`, and + :mod:`lzma` compressed archives if the respective modules are available. + + .. + The following paragraph should be similar to ../includes/optional-module.rst + + If any of these :term:`optional modules ` are missing from + your copy of CPython, look for documentation from your distributor (that is, + whoever provided Python to you). + If you are the distributor, see :ref:`optional-module-requirements`. * read/write support for the POSIX.1-1988 (ustar) format. @@ -47,6 +55,10 @@ Some facts and figures: or paths outside of the destination. Previously, the filter strategy was equivalent to :func:`fully_trusted `. +.. versionchanged:: 3.14 + + Added support for Zstandard compression using :mod:`compression.zstd`. + .. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) Return a :class:`TarFile` object for the pathname *name*. For detailed @@ -59,8 +71,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | mode | action | +==================+=============================================+ - | ``'r' or 'r:*'`` | Open for reading with transparent | - | | compression (recommended). | + | ``'r'`` or | Open for reading with transparent | + | ``'r:*'`` | compression (recommended). | +------------------+---------------------------------------------+ | ``'r:'`` | Open for reading exclusively without | | | compression. | @@ -71,6 +83,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | ``'r:xz'`` | Open for reading with lzma compression. | +------------------+---------------------------------------------+ + | ``'r:zst'`` | Open for reading with Zstandard compression.| + +------------------+---------------------------------------------+ | ``'x'`` or | Create a tarfile exclusively without | | ``'x:'`` | compression. | | | Raise a :exc:`FileExistsError` exception | @@ -88,10 +102,15 @@ Some facts and figures: | | Raise a :exc:`FileExistsError` exception | | | if it already exists. | +------------------+---------------------------------------------+ - | ``'a' or 'a:'`` | Open for appending with no compression. The | - | | file is created if it does not exist. | + | ``'x:zst'`` | Create a tarfile with Zstandard compression.| + | | Raise a :exc:`FileExistsError` exception | + | | if it already exists. | +------------------+---------------------------------------------+ - | ``'w' or 'w:'`` | Open for uncompressed writing. | + | ``'a'`` or | Open for appending with no compression. The | + | ``'a:'`` | file is created if it does not exist. | + +------------------+---------------------------------------------+ + | ``'w'`` or | Open for uncompressed writing. | + | ``'w:'`` | | +------------------+---------------------------------------------+ | ``'w:gz'`` | Open for gzip compressed writing. | +------------------+---------------------------------------------+ @@ -99,6 +118,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | ``'w:xz'`` | Open for lzma compressed writing. | +------------------+---------------------------------------------+ + | ``'w:zst'`` | Open for Zstandard compressed writing. | + +------------------+---------------------------------------------+ Note that ``'a:gz'``, ``'a:bz2'`` or ``'a:xz'`` is not possible. If *mode* is not suitable to open a certain (compressed) file for reading, @@ -110,11 +131,20 @@ Some facts and figures: For modes ``'w:gz'``, ``'x:gz'``, ``'w|gz'``, ``'w:bz2'``, ``'x:bz2'``, ``'w|bz2'``, :func:`tarfile.open` accepts the keyword argument - *compresslevel* (default ``9``) to specify the compression level of the file. + *compresslevel* (default ``6``) to specify the compression level of the file. For modes ``'w:xz'``, ``'x:xz'`` and ``'w|xz'``, :func:`tarfile.open` accepts the keyword argument *preset* to specify the compression level of the file. + For modes ``'w:zst'``, ``'x:zst'`` and ``'w|zst'``, :func:`tarfile.open` + accepts the keyword argument *level* to specify the compression level of + the file. The keyword argument *options* may also be passed, providing + advanced Zstandard compression parameters described by + :class:`~compression.zstd.CompressionParameter`. The keyword argument + *zstd_dict* can be passed to provide a :class:`~compression.zstd.ZstdDict`, + a Zstandard dictionary used to improve compression of smaller amounts of + data. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -146,6 +176,9 @@ Some facts and figures: | ``'r|xz'`` | Open an lzma compressed *stream* for | | | reading. | +-------------+--------------------------------------------+ + | ``'r|zst'`` | Open a Zstandard compressed *stream* for | + | | reading. | + +-------------+--------------------------------------------+ | ``'w|'`` | Open an uncompressed *stream* for writing. | +-------------+--------------------------------------------+ | ``'w|gz'`` | Open a gzip compressed *stream* for | @@ -157,6 +190,9 @@ Some facts and figures: | ``'w|xz'`` | Open an lzma compressed *stream* for | | | writing. | +-------------+--------------------------------------------+ + | ``'w|zst'`` | Open a Zstandard compressed *stream* for | + | | writing. | + +-------------+--------------------------------------------+ .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. @@ -170,6 +206,10 @@ Some facts and figures: .. versionchanged:: 3.14 The *preset* keyword argument also works for streams. + .. versionchanged:: 3.15 + The default compression level was reduced to 6 (down from 9). + It is the default level used by most compression tools and a better + tradeoff between speed and performance. .. class:: TarFile :noindex: @@ -255,6 +295,15 @@ The :mod:`tarfile` module defines the following exceptions: Raised to refuse extracting a symbolic link pointing outside the destination directory. +.. exception:: LinkFallbackError + + Raised to refuse emulating a link (hard or symbolic) by extracting another + archive member, when that member would be rejected by the filter location. + The exception that was raised to reject the replacement member is available + as :attr:`!BaseException.__context__`. + + .. versionadded:: 3.15 + The following constants are available at the module level: @@ -1068,6 +1117,12 @@ reused in custom filters: Implements the ``'data'`` filter. In addition to what ``tar_filter`` does: + - Normalize link targets (:attr:`TarInfo.linkname`) using + :func:`os.path.normpath`. + Note that this removes internal ``..`` components, which may change the + meaning of the link if the path in :attr:`!TarInfo.linkname` traverses + symbolic links. + - :ref:`Refuse ` to extract links (hard or soft) that link to absolute paths, or ones that link outside the destination. @@ -1099,6 +1154,10 @@ reused in custom filters: Note that this filter does not block *all* dangerous archive features. See :ref:`tarfile-further-verification` for details. + .. versionchanged:: 3.15 + + Link targets are now normalized. + .. _tarfile-extraction-refuse: @@ -1127,6 +1186,7 @@ Here is an incomplete list of things to consider: * Extract to a :func:`new temporary directory ` to prevent e.g. exploiting pre-existing links, and to make it easier to clean up after a failed extraction. +* Disallow symbolic links if you do not need the functionality. * When working with untrusted data, use external (e.g. OS-level) limits on disk, memory and CPU usage. * Check filenames against an allow-list of characters @@ -1305,6 +1365,9 @@ Command-line options Examples -------- +Reading examples +~~~~~~~~~~~~~~~~~~~ + How to extract an entire tar archive to the current working directory:: import tarfile @@ -1327,6 +1390,23 @@ a generator function instead of a list:: tar.extractall(members=py_files(tar)) tar.close() +How to read a gzip compressed tar archive and display some member information:: + + import tarfile + tar = tarfile.open("sample.tar.gz", "r:gz") + for tarinfo in tar: + print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") + if tarinfo.isreg(): + print("a regular file.") + elif tarinfo.isdir(): + print("a directory.") + else: + print("something else.") + tar.close() + +Writing examples +~~~~~~~~~~~~~~~~ + How to create an uncompressed tar archive from a list of filenames:: import tarfile @@ -1342,19 +1422,15 @@ The same example using the :keyword:`with` statement:: for name in ["foo", "bar", "quux"]: tar.add(name) -How to read a gzip compressed tar archive and display some member information:: +How to create and write an archive to stdout using +:data:`sys.stdout.buffer ` in the *fileobj* parameter +in :meth:`TarFile.add`:: - import tarfile - tar = tarfile.open("sample.tar.gz", "r:gz") - for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") - if tarinfo.isreg(): - print("a regular file.") - elif tarinfo.isdir(): - print("a directory.") - else: - print("something else.") - tar.close() + import sys + import tarfile + with tarfile.open("sample.tar.gz", "w|gz", fileobj=sys.stdout.buffer) as tar: + for name in ["foo", "bar", "quux"]: + tar.add(name) How to create an archive and reset the user information using the *filter* parameter in :meth:`TarFile.add`:: diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 0aae14c15a6..395cde21ccf 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -851,7 +851,7 @@ The :mod:`test.support` module defines the following functions: Decorator for tests that fill the address space. -.. function:: linked_with_musl() +.. function:: linked_to_musl() Return ``False`` if there is no evidence the interpreter was compiled with ``musl``, otherwise return a version triple, either ``(0, 0, 0)`` if the @@ -1384,6 +1384,13 @@ The :mod:`test.support.threading_helper` module provides support for threading t .. versionadded:: 3.8 +.. function:: run_concurrently(worker_func, nthreads, args=(), kwargs={}) + + Run the worker function concurrently in multiple threads. + Re-raises an exception if any thread raises one, after all threads have + finished. + + :mod:`test.support.os_helper` --- Utilities for os tests ======================================================================== diff --git a/Doc/library/text.rst b/Doc/library/text.rst index 47b678434fc..92e7dd9a53b 100644 --- a/Doc/library/text.rst +++ b/Doc/library/text.rst @@ -16,6 +16,7 @@ Python's built-in string type in :ref:`textseq`. .. toctree:: string.rst + string.templatelib.rst re.rst difflib.rst textwrap.rst diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 249c0a5cb03..19cc4f191df 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -11,6 +11,52 @@ This module constructs higher-level threading interfaces on top of the lower level :mod:`_thread` module. +.. include:: ../includes/wasm-notavail.rst + +Introduction +------------ + +The :mod:`!threading` module provides a way to run multiple `threads +`_ (smaller +units of a process) concurrently within a single process. It allows for the +creation and management of threads, making it possible to execute tasks in +parallel, sharing memory space. Threads are particularly useful when tasks are +I/O bound, such as file operations or making network requests, +where much of the time is spent waiting for external resources. + +A typical use case for :mod:`!threading` includes managing a pool of worker +threads that can process multiple tasks concurrently. Here's a basic example of +creating and starting threads using :class:`~threading.Thread`:: + + import threading + import time + + def crawl(link, delay=3): + print(f"crawl started for {link}") + time.sleep(delay) # Blocking I/O (simulating a network request) + print(f"crawl ended for {link}") + + links = [ + "https://python.org", + "https://docs.python.org", + "https://peps.python.org", + ] + + # Start threads for each link + threads = [] + for link in links: + # Using `args` to pass positional arguments and `kwargs` for keyword arguments + t = threading.Thread(target=crawl, args=(link,), kwargs={"delay": 2}) + threads.append(t) + + # Start each thread + for t in threads: + t.start() + + # Wait for all threads to finish + for t in threads: + t.join() + .. versionchanged:: 3.7 This module used to be optional, it is now always available. @@ -45,7 +91,25 @@ level :mod:`_thread` module. However, threading is still an appropriate model if you want to run multiple I/O-bound tasks simultaneously. -.. include:: ../includes/wasm-notavail.rst +GIL and performance considerations +---------------------------------- + +Unlike the :mod:`multiprocessing` module, which uses separate processes to +bypass the :term:`global interpreter lock` (GIL), the threading module operates +within a single process, meaning that all threads share the same memory space. +However, the GIL limits the performance gains of threading when it comes to +CPU-bound tasks, as only one thread can execute Python bytecode at a time. +Despite this, threads remain a useful tool for achieving concurrency in many +scenarios. + +As of Python 3.13, :term:`free-threaded ` builds +can disable the GIL, enabling true parallel execution of threads, but this +feature is not available by default (see :pep:`703`). + +.. TODO: At some point this feature will become available by default. + +Reference +--------- This module defines the following functions: @@ -62,7 +126,7 @@ This module defines the following functions: Return the current :class:`Thread` object, corresponding to the caller's thread of control. If the caller's thread of control was not created through the - :mod:`threading` module, a dummy thread object with limited functionality is + :mod:`!threading` module, a dummy thread object with limited functionality is returned. The function ``currentThread`` is a deprecated alias for this function. @@ -127,13 +191,16 @@ This module defines the following functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD, Solaris. .. versionadded:: 3.8 .. versionchanged:: 3.13 Added support for GNU/kFreeBSD. + .. versionchanged:: 3.15 + Added support for Solaris. + .. function:: enumerate() @@ -157,13 +224,13 @@ This module defines the following functions: .. index:: single: trace function - Set a trace function for all threads started from the :mod:`threading` module. + Set a trace function for all threads started from the :mod:`!threading` module. The *func* will be passed to :func:`sys.settrace` for each thread, before its :meth:`~Thread.run` method is called. .. function:: settrace_all_threads(func) - Set a trace function for all threads started from the :mod:`threading` module + Set a trace function for all threads started from the :mod:`!threading` module and all Python threads that are currently executing. The *func* will be passed to :func:`sys.settrace` for each thread, before its @@ -186,13 +253,13 @@ This module defines the following functions: .. index:: single: profile function - Set a profile function for all threads started from the :mod:`threading` module. + Set a profile function for all threads started from the :mod:`!threading` module. The *func* will be passed to :func:`sys.setprofile` for each thread, before its :meth:`~Thread.run` method is called. .. function:: setprofile_all_threads(func) - Set a profile function for all threads started from the :mod:`threading` module + Set a profile function for all threads started from the :mod:`!threading` module and all Python threads that are currently executing. The *func* will be passed to :func:`sys.setprofile` for each thread, before its @@ -257,8 +324,8 @@ when implemented, are mapped to module-level functions. All of the methods described below are executed atomically. -Thread-Local Data ------------------ +Thread-local data +^^^^^^^^^^^^^^^^^ Thread-local data is data whose values are thread specific. If you have data that you want to be local to a thread, create a @@ -389,8 +456,8 @@ affects what we see:: .. _thread-objects: -Thread Objects --------------- +Thread objects +^^^^^^^^^^^^^^ The :class:`Thread` class represents an activity that is run in a separate thread of control. There are two ways to specify the activity: by passing a @@ -541,7 +608,7 @@ since it is impossible to detect the termination of alien threads. timeout occurs. When the *timeout* argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds + real number specifying a timeout for the operation in seconds (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to decide whether a timeout happened -- if the thread is still alive, the @@ -557,7 +624,7 @@ since it is impossible to detect the termination of alien threads. an error to :meth:`~Thread.join` a thread before it has been started and attempts to do so raise the same exception. - If an attempt is made to join a running daemonic thread in in late stages + If an attempt is made to join a running daemonic thread in late stages of :term:`Python finalization ` :meth:`!join` raises a :exc:`PythonFinalizationError`. @@ -565,6 +632,9 @@ since it is impossible to detect the termination of alien threads. May raise :exc:`PythonFinalizationError`. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. attribute:: name A string used for identification purposes only. It has no semantics. @@ -645,8 +715,8 @@ since it is impossible to detect the termination of alien threads. .. _lock-objects: -Lock Objects ------------- +Lock objects +^^^^^^^^^^^^ A primitive lock is a synchronization primitive that is not owned by a particular thread when locked. In Python, it is currently the lowest level @@ -697,7 +767,7 @@ All methods are executed atomically. If a call with *blocking* set to ``True`` would block, return ``False`` immediately; otherwise, set the lock to locked and return ``True``. - When invoked with the floating-point *timeout* argument set to a positive + When invoked with the *timeout* argument set to a positive value, block for at most the number of seconds specified by *timeout* and as long as the lock cannot be acquired. A *timeout* argument of ``-1`` specifies an unbounded wait. It is forbidden to specify a *timeout* @@ -716,6 +786,9 @@ All methods are executed atomically. .. versionchanged:: 3.14 Lock acquisition can now be interrupted by signals on Windows. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release() @@ -738,8 +811,8 @@ All methods are executed atomically. .. _rlock-objects: -RLock Objects -------------- +RLock objects +^^^^^^^^^^^^^ A reentrant lock is a synchronization primitive that may be acquired multiple times by the same thread. Internally, it uses the concepts of "owning thread" @@ -796,7 +869,7 @@ call release as many times the lock has been acquired can lead to deadlock. * If no thread owns the lock, acquire the lock and return immediately. * If another thread owns the lock, block until we are able to acquire - lock, or *timeout*, if set to a positive float value. + lock, or *timeout*, if set to a positive value. * If the same thread owns the lock, acquire the lock again, and return immediately. This is the difference between :class:`Lock` and @@ -823,6 +896,9 @@ call release as many times the lock has been acquired can lead to deadlock. .. versionchanged:: 3.2 The *timeout* parameter is new. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release() @@ -848,8 +924,8 @@ call release as many times the lock has been acquired can lead to deadlock. .. _condition-objects: -Condition Objects ------------------ +Condition objects +^^^^^^^^^^^^^^^^^ A condition variable is always associated with some kind of lock; this can be passed in or one will be created by default. Passing one in is useful when @@ -956,7 +1032,7 @@ item to the buffer only needs to wake up one consumer thread. occurs. Once awakened or timed out, it re-acquires the lock and returns. When the *timeout* argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds + real number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an :class:`RLock`, it is not released using @@ -1026,8 +1102,8 @@ item to the buffer only needs to wake up one consumer thread. .. _semaphore-objects: -Semaphore Objects ------------------ +Semaphore objects +^^^^^^^^^^^^^^^^^ This is one of the oldest synchronization primitives in the history of computer science, invented by the early Dutch computer scientist Edsger W. Dijkstra (he @@ -1083,6 +1159,9 @@ Semaphores also support the :ref:`context management protocol `. .. versionchanged:: 3.2 The *timeout* parameter is new. + .. versionchanged:: 3.15 + Accepts any real number as *timeout*, not only integer or float. + .. method:: release(n=1) Release a semaphore, incrementing the internal counter by *n*. When it @@ -1107,7 +1186,7 @@ Semaphores also support the :ref:`context management protocol `. .. _semaphore-examples: -:class:`Semaphore` Example +:class:`Semaphore` example ^^^^^^^^^^^^^^^^^^^^^^^^^^ Semaphores are often used to guard resources with limited capacity, for example, @@ -1135,8 +1214,8 @@ causes the semaphore to be released more than it's acquired will go undetected. .. _event-objects: -Event Objects -------------- +Event objects +^^^^^^^^^^^^^ This is one of the simplest mechanisms for communication between threads: one thread signals an event and other threads wait for it. @@ -1183,7 +1262,7 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. the internal flag did not become true within the given wait time. When the timeout argument is present and not ``None``, it should be a - floating-point number specifying a timeout for the operation in seconds, + real number specifying a timeout for the operation in seconds, or fractions thereof. .. versionchanged:: 3.1 @@ -1192,8 +1271,8 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. .. _timer-objects: -Timer Objects -------------- +Timer objects +^^^^^^^^^^^^^ This class represents an action that should be run only after a certain amount of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` @@ -1230,8 +1309,8 @@ For example:: only work if the timer is still in its waiting stage. -Barrier Objects ---------------- +Barrier objects +^^^^^^^^^^^^^^^ .. versionadded:: 3.2 diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 542493a82af..a931134331f 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -189,7 +189,7 @@ Functions .. versionadded:: 3.7 -.. function:: clock_settime(clk_id, time: float) +.. function:: clock_settime(clk_id, time) Set the time of the specified clock *clk_id*. Currently, :data:`CLOCK_REALTIME` is the only accepted value for *clk_id*. @@ -201,6 +201,9 @@ Functions .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Accepts any real number as *time*, not only integer or float. + .. function:: clock_settime_ns(clk_id, time: int) @@ -223,6 +226,9 @@ Functions ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: get_clock_info(name) @@ -238,8 +244,8 @@ Functions The result has the following attributes: - - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by - a NTP daemon) or manually by the system administrator, ``False`` otherwise + - *adjustable*: ``True`` if the clock can be set to jump forward or backward + in time, ``False`` otherwise. Does not refer to gradual NTP rate adjustments. - *implementation*: The name of the underlying C function used to get the clock value. Refer to :ref:`time-clock-id-constants` for possible values. - *monotonic*: ``True`` if the clock cannot go backward, @@ -258,6 +264,9 @@ Functions :class:`struct_time` object. See :func:`calendar.timegm` for the inverse of this function. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: localtime([secs]) @@ -271,6 +280,9 @@ Functions :c:func:`gmtime` failure. It's common for this to be restricted to years between 1970 and 2038. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. function:: mktime(t) @@ -306,10 +318,11 @@ Functions .. versionadded:: 3.3 .. versionchanged:: 3.5 - The function is now always available and always system-wide. + The function is now always available and the clock is now the same for + all processes. .. versionchanged:: 3.10 - On macOS, the function is now system-wide. + On macOS, the clock is now the same for all processes. .. function:: monotonic_ns() -> int @@ -325,7 +338,8 @@ Functions Return the value (in fractional seconds) of a performance counter, i.e. a clock with the highest available resolution to measure a short duration. It - does include time elapsed during sleep and is system-wide. The reference + does include time elapsed during sleep. The clock is the same for all + processes. The reference point of the returned value is undefined, so that only the difference between the results of two calls is valid. @@ -340,7 +354,7 @@ Functions .. versionadded:: 3.3 .. versionchanged:: 3.10 - On Windows, the function is now system-wide. + On Windows, the clock is now the same for all processes. .. versionchanged:: 3.13 Use the same clock as :func:`time.monotonic`. @@ -380,8 +394,7 @@ Functions .. function:: sleep(secs) Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating-point number to indicate a more precise sleep - time. + The argument may be a non-integer to indicate a more precise sleep time. If the sleep is interrupted by a signal and no exception is raised by the signal handler, the sleep is restarted with a recomputed timeout. @@ -394,9 +407,9 @@ Functions On Windows, if *secs* is zero, the thread relinquishes the remainder of its time slice to any other thread that is ready to run. If there are no other threads ready to run, the function returns immediately, and the thread - continues execution. On Windows 8.1 and newer the implementation uses + continues execution. On Windows 10 and newer the implementation uses a `high-resolution timer - `_ + `_ which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. .. rubric:: Unix implementation @@ -426,6 +439,9 @@ Functions .. versionchanged:: 3.13 Raises an auditing event. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. + .. index:: single: % (percent); datetime format @@ -568,7 +584,7 @@ Functions calculations when the day of the week and the year are specified. Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [1]_ :: + :rfc:`5322` Internet email standard. [1]_ :: >>> from time import gmtime, strftime >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) @@ -712,13 +728,18 @@ Functions Clock: - * On Windows, call ``GetSystemTimeAsFileTime()``. + * On Windows, call ``GetSystemTimePreciseAsFileTime()``. * Call ``clock_gettime(CLOCK_REALTIME)`` if available. * Otherwise, call ``gettimeofday()``. Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` type. +.. versionchanged:: 3.13 + + On Windows, calls ``GetSystemTimePreciseAsFileTime()`` instead of + ``GetSystemTimeAsFileTime()``. + .. function:: time_ns() -> int @@ -928,7 +949,7 @@ These constants are used as parameters for :func:`clock_getres` and .. data:: CLOCK_TAI - `International Atomic Time `_ + `International Atomic Time `_ The system must have a current leap second table in order for this to give the correct answer. PTP or NTP software can maintain a leap second table. @@ -982,8 +1003,8 @@ The following constant is the only parameter that can be sent to .. data:: CLOCK_REALTIME - System-wide real-time clock. Setting this clock requires appropriate - privileges. + Real-time clock. Setting this clock requires appropriate privileges. + The clock is the same for all processes. .. availability:: Unix. @@ -1045,4 +1066,5 @@ Timezone Constants strict reading of the original 1982 :rfc:`822` standard calls for a two-digit year (``%y`` rather than ``%Y``), but practice moved to 4-digit years long before the year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has - been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. + been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`, + with :rfc:`5322` continuing this requirement. diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst index 0593f8b73ea..fa3c7e910ce 100644 --- a/Doc/library/tk.rst +++ b/Doc/library/tk.rst @@ -1,7 +1,7 @@ .. _tkinter: ********************************* -Graphical User Interfaces with Tk +Graphical user interfaces with Tk ********************************* .. index:: @@ -39,6 +39,7 @@ alternative `GUI frameworks and tools `_ @@ -392,7 +394,7 @@ by spaces. Without getting into too many details, notice the following: * Operations which are implemented as separate *commands* in Tcl (like ``grid`` or ``destroy``) are represented as *methods* on Tkinter widget objects. As you'll see shortly, at other times Tcl uses what appear to be - method calls on widget objects, which more closely mirror what would is + method calls on widget objects, which more closely mirror what is used in Tkinter. @@ -839,8 +841,7 @@ geometry For example: ``fred["geometry"] = "200x100"``. justify - Legal values are the strings: ``"left"``, ``"center"``, ``"right"``, and - ``"fill"``. + Legal values are the strings: ``"left"``, ``"center"``, and ``"right"``. region This is a string with four space-delimited elements, each of which is a legal diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 1f92b5df430..c228006d4c1 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -51,7 +51,7 @@ The token constants are: .. data:: NAME Token value that indicates an :ref:`identifier `. - Note that keywords are also initially tokenized an ``NAME`` tokens. + Note that keywords are also initially tokenized as ``NAME`` tokens. .. data:: NUMBER diff --git a/Doc/library/tulip_coro.dia b/Doc/library/tulip_coro.dia deleted file mode 100644 index 70a33e3c00c..00000000000 Binary files a/Doc/library/tulip_coro.dia and /dev/null differ diff --git a/Doc/library/tulip_coro.png b/Doc/library/tulip_coro.png deleted file mode 100644 index aad41c93015..00000000000 Binary files a/Doc/library/tulip_coro.png and /dev/null differ diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index fea6b57edf0..95a57c57e71 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -29,6 +29,8 @@ introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon in 1967. +.. include:: ../includes/optional-module.rst + Get started =========== @@ -777,13 +779,17 @@ Turtle motion 180.0 -.. function:: dot(size=None, *color) +.. function:: dot() + dot(size) + dot(color, /) + dot(size, color, /) + dot(size, r, g, b, /) :param size: an integer >= 1 (if given) :param color: a colorstring or a numeric color tuple Draw a circular dot with diameter *size*, using *color*. If *size* is - not given, the maximum of pensize+4 and 2*pensize is used. + not given, the maximum of ``pensize+4`` and ``2*pensize`` is used. .. doctest:: @@ -1152,7 +1158,9 @@ Drawing state Color control ~~~~~~~~~~~~~ -.. function:: pencolor(*args) +.. function:: pencolor() + pencolor(color, /) + pencolor(r, g, b, /) Return or set the pencolor. @@ -1161,7 +1169,7 @@ Color control ``pencolor()`` Return the current pencolor as color specification string or as a tuple (see example). May be used as input to another - color/pencolor/fillcolor call. + color/pencolor/fillcolor/bgcolor call. ``pencolor(colorstring)`` Set pencolor to *colorstring*, which is a Tk color specification string, @@ -1201,7 +1209,9 @@ Color control (50.0, 193.0, 143.0) -.. function:: fillcolor(*args) +.. function:: fillcolor() + fillcolor(color, /) + fillcolor(r, g, b, /) Return or set the fillcolor. @@ -1210,7 +1220,7 @@ Color control ``fillcolor()`` Return the current fillcolor as color specification string, possibly in tuple format (see example). May be used as input to another - color/pencolor/fillcolor call. + color/pencolor/fillcolor/bgcolor call. ``fillcolor(colorstring)`` Set fillcolor to *colorstring*, which is a Tk color specification string, @@ -1244,7 +1254,10 @@ Color control (255.0, 255.0, 255.0) -.. function:: color(*args) +.. function:: color() + color(color, /) + color(r, g, b, /) + color(pencolor, fillcolor, /) Return or set pencolor and fillcolor. @@ -1870,13 +1883,32 @@ Most of the examples in this section refer to a TurtleScreen instance called Window control -------------- -.. function:: bgcolor(*args) +.. function:: bgcolor() + bgcolor(color, /) + bgcolor(r, g, b, /) - :param args: a color string or three numbers in the range 0..colormode or a - 3-tuple of such numbers + Return or set the background color of the TurtleScreen. + Four input formats are allowed: - Set or return background color of the TurtleScreen. + ``bgcolor()`` + Return the current background color as color specification string or + as a tuple (see example). May be used as input to another + color/pencolor/fillcolor/bgcolor call. + + ``bgcolor(colorstring)`` + Set the background color to *colorstring*, which is a Tk color + specification string, such as ``"red"``, ``"yellow"``, or ``"#33cc8c"``. + + ``bgcolor((r, g, b))`` + Set the background color to the RGB color represented by the tuple of + *r*, *g*, and *b*. + Each of *r*, *g*, and *b* must be in the range 0..colormode, where + colormode is either 1.0 or 255 (see :func:`colormode`). + + ``bgcolor(r, g, b)`` + Set the background color to the RGB color represented by *r*, *g*, and *b*. Each of + *r*, *g*, and *b* must be in the range 0..colormode. .. doctest:: :skipif: _tkinter is None @@ -2769,68 +2801,68 @@ The demo scripts are: .. tabularcolumns:: |l|L|L| -+----------------+------------------------------+-----------------------+ -| Name | Description | Features | -+================+==============================+=======================+ -| bytedesign | complex classical | :func:`tracer`, delay,| -| | turtle graphics pattern | :func:`update` | -+----------------+------------------------------+-----------------------+ -| chaos | graphs Verhulst dynamics, | world coordinates | -| | shows that computer's | | -| | computations can generate | | -| | results sometimes against the| | -| | common sense expectations | | -+----------------+------------------------------+-----------------------+ -| clock | analog clock showing time | turtles as clock's | -| | of your computer | hands, ontimer | -+----------------+------------------------------+-----------------------+ -| colormixer | experiment with r, g, b | :func:`ondrag` | -+----------------+------------------------------+-----------------------+ -| forest | 3 breadth-first trees | randomization | -+----------------+------------------------------+-----------------------+ -| fractalcurves | Hilbert & Koch curves | recursion | -+----------------+------------------------------+-----------------------+ -| lindenmayer | ethnomathematics | L-System | -| | (indian kolams) | | -+----------------+------------------------------+-----------------------+ -| minimal_hanoi | Towers of Hanoi | Rectangular Turtles | -| | | as Hanoi discs | -| | | (shape, shapesize) | -+----------------+------------------------------+-----------------------+ -| nim | play the classical nim game | turtles as nimsticks, | -| | with three heaps of sticks | event driven (mouse, | -| | against the computer. | keyboard) | -+----------------+------------------------------+-----------------------+ -| paint | super minimalistic | :func:`onclick` | -| | drawing program | | -+----------------+------------------------------+-----------------------+ -| peace | elementary | turtle: appearance | -| | | and animation | -+----------------+------------------------------+-----------------------+ -| penrose | aperiodic tiling with | :func:`stamp` | -| | kites and darts | | -+----------------+------------------------------+-----------------------+ -| planet_and_moon| simulation of | compound shapes, | -| | gravitational system | :class:`Vec2D` | -+----------------+------------------------------+-----------------------+ -| rosette | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ -| round_dance | dancing turtles rotating | compound shapes, clone| -| | pairwise in opposite | shapesize, tilt, | -| | direction | get_shapepoly, update | -+----------------+------------------------------+-----------------------+ -| sorting_animate| visual demonstration of | simple alignment, | -| | different sorting methods | randomization | -+----------------+------------------------------+-----------------------+ -| tree | a (graphical) breadth | :func:`clone` | -| | first tree (using generators)| | -+----------------+------------------------------+-----------------------+ -| two_canvases | simple design | turtles on two | -| | | canvases | -+----------------+------------------------------+-----------------------+ -| yinyang | another elementary example | :func:`circle` | -+----------------+------------------------------+-----------------------+ ++------------------------+------------------------------+--------------------------------------+ +| Name | Description | Features | ++========================+==============================+======================================+ +| ``bytedesign`` | complex classical | :func:`tracer`, :func:`delay`, | +| | turtle graphics pattern | :func:`update` | ++------------------------+------------------------------+--------------------------------------+ +| ``chaos`` | graphs Verhulst dynamics, | world coordinates | +| | shows that computer's | | +| | computations can generate | | +| | results sometimes against the| | +| | common sense expectations | | ++------------------------+------------------------------+--------------------------------------+ +| ``clock`` | analog clock showing time | turtles as clock's | +| | of your computer | hands, :func:`ontimer` | ++------------------------+------------------------------+--------------------------------------+ +| ``colormixer`` | experiment with r, g, b | :func:`ondrag` | ++------------------------+------------------------------+--------------------------------------+ +| ``forest`` | 3 breadth-first trees | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``fractalcurves`` | Hilbert & Koch curves | recursion | ++------------------------+------------------------------+--------------------------------------+ +| ``lindenmayer`` | ethnomathematics | L-System | +| | (indian kolams) | | ++------------------------+------------------------------+--------------------------------------+ +| ``minimal_hanoi`` | Towers of Hanoi | Rectangular Turtles | +| | | as Hanoi discs | +| | | (:func:`shape`, :func:`shapesize`) | ++------------------------+------------------------------+--------------------------------------+ +| ``nim`` | play the classical nim game | turtles as nimsticks, | +| | with three heaps of sticks | event driven (mouse, | +| | against the computer. | keyboard) | ++------------------------+------------------------------+--------------------------------------+ +| ``paint`` | super minimalistic | :func:`onclick` | +| | drawing program | | ++------------------------+------------------------------+--------------------------------------+ +| ``peace`` | elementary | turtle: appearance | +| | | and animation | ++------------------------+------------------------------+--------------------------------------+ +| ``penrose`` | aperiodic tiling with | :func:`stamp` | +| | kites and darts | | ++------------------------+------------------------------+--------------------------------------+ +| ``planet_and_moon`` | simulation of | compound shapes, | +| | gravitational system | :class:`Vec2D` | ++------------------------+------------------------------+--------------------------------------+ +| ``rosette`` | a pattern from the wikipedia | :func:`clone`, | +| | article on turtle graphics | :func:`undo` | ++------------------------+------------------------------+--------------------------------------+ +| ``round_dance`` | dancing turtles rotating | compound shapes, :func:`clone` | +| | pairwise in opposite | :func:`shapesize`, :func:`tilt`, | +| | direction | :func:`get_shapepoly`, :func:`update`| ++------------------------+------------------------------+--------------------------------------+ +| ``sorting_animate`` | visual demonstration of | simple alignment, | +| | different sorting methods | randomization | ++------------------------+------------------------------+--------------------------------------+ +| ``tree`` | a (graphical) breadth | :func:`clone` | +| | first tree (using generators)| | ++------------------------+------------------------------+--------------------------------------+ +| ``two_canvases`` | simple design | turtles on two | +| | | canvases | ++------------------------+------------------------------+--------------------------------------+ +| ``yinyang`` | another elementary example | :func:`circle` | ++------------------------+------------------------------+--------------------------------------+ Have fun! diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 2bedd7fdd3c..40b5f3db13d 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -333,6 +333,16 @@ Standard names are defined for the following types: :attr:`tb.tb_frame ` if ``tb`` is a traceback object. +.. data:: FrameLocalsProxyType + + The type of frame locals proxy objects, as found on the + :attr:`frame.f_locals` attribute. + + .. versionadded:: 3.15 + + .. seealso:: :pep:`667` + + .. data:: GetSetDescriptorType The type of objects defined in extension modules with ``PyGetSetDef``, such diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 54cc3ea3311..58f3a8ecd7c 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -45,15 +45,15 @@ provides backports of these new features to older versions of Python. .. seealso:: - `"Typing cheat sheet" `_ + `Typing cheat sheet `_ A quick overview of type hints (hosted at the mypy docs) - "Type System Reference" section of `the mypy docs `_ + Type System Reference section of `the mypy docs `_ The Python typing system is standardised via PEPs, so this reference should broadly apply to most Python type checkers. (Some parts may still be specific to mypy.) - `"Static Typing with Python" `_ + `Static Typing with Python `_ Type-checker-agnostic documentation written by the community detailing type system features, useful typing related tools and typing best practices. @@ -64,7 +64,7 @@ Specification for the Python Type System ======================================== The canonical, up-to-date specification of the Python type system can be -found at `"Specification for the Python type system" `_. +found at `Specification for the Python type system `_. .. _type-aliases: @@ -230,9 +230,11 @@ For example: callback: Callable[[str], Awaitable[None]] = on_update +.. index:: single: ...; ellipsis literal + The subscription syntax must always be used with exactly two values: the argument list and the return type. The argument list must be a list of types, -a :class:`ParamSpec`, :data:`Concatenate`, or an ellipsis. The return type must +a :class:`ParamSpec`, :data:`Concatenate`, or an ellipsis (``...``). The return type must be a single type. If a literal ellipsis ``...`` is given as the argument list, it indicates that @@ -375,8 +377,11 @@ accepts *any number* of type arguments:: # but ``z`` has been assigned to a tuple of length 3 z: tuple[int] = (1, 2, 3) +.. index:: single: ...; ellipsis literal + To denote a tuple which could be of *any* length, and in which all elements are -of the same type ``T``, use ``tuple[T, ...]``. To denote an empty tuple, use +of the same type ``T``, use the literal ellipsis ``...``: ``tuple[T, ...]``. +To denote an empty tuple, use ``tuple[()]``. Using plain ``tuple`` as an annotation is equivalent to using ``tuple[Any, ...]``:: @@ -1162,6 +1167,8 @@ These can be used as types in annotations. They all support subscription using Special form for annotating higher-order functions. + .. index:: single: ...; ellipsis literal + ``Concatenate`` can be used in conjunction with :ref:`Callable ` and :class:`ParamSpec` to annotate a higher-order callable which adds, removes, or transforms parameters of another @@ -1383,7 +1390,7 @@ These can be used as types in annotations. They all support subscription using Using ``Annotated[T, x]`` as an annotation still allows for static typechecking of ``T``, as type checkers will simply ignore the metadata ``x``. In this way, ``Annotated`` differs from the - :func:`@no_type_check ` decorator, which can also be used for + :deco:`no_type_check` decorator, which can also be used for adding annotations outside the scope of the typing system, but completely disables typechecking for a function or class. @@ -2236,7 +2243,7 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.10 -.. class:: TypeAliasType(name, value, *, type_params=()) +.. class:: TypeAliasType(name, value, *, type_params=(), qualname=None) The type of type aliases created through the :keyword:`type` statement. @@ -2260,9 +2267,23 @@ without the dedicated syntax, as documented below. >>> Alias.__name__ 'Alias' + .. attribute:: __qualname__ + + The :term:`qualified name` of the type alias: + + .. doctest:: + + >>> class Class: + ... type Alias = int + ... + >>> Class.Alias.__qualname__ + 'Class.Alias' + + .. versionadded:: 3.15 + .. attribute:: __module__ - The module in which the type alias was defined:: + The name of the module in which the type alias was defined:: >>> type Alias = int >>> Alias.__module__ @@ -2428,19 +2449,6 @@ types. Using :func:`super` (and the ``__class__`` :term:`closure variable`) in methods of ``NamedTuple`` subclasses is unsupported and causes a :class:`TypeError`. - .. deprecated-removed:: 3.13 3.15 - The undocumented keyword argument syntax for creating NamedTuple classes - (``NT = NamedTuple("NT", x=int)``) is deprecated, and will be disallowed - in 3.15. Use the class-based syntax or the functional syntax instead. - - .. deprecated-removed:: 3.13 3.15 - When using the functional syntax to create a NamedTuple class, failing to - pass a value to the 'fields' parameter (``NT = NamedTuple("NT")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``NT = NamedTuple("NT", None)``) is also deprecated. Both will be - disallowed in Python 3.15. To create a NamedTuple class with 0 fields, - use ``class NT(NamedTuple): pass`` or ``NT = NamedTuple("NT", [])``. - .. class:: NewType(name, tp) Helper class to create low-overhead :ref:`distinct types `. @@ -2455,7 +2463,7 @@ types. .. attribute:: __module__ - The module in which the new type is defined. + The name of the module in which the new type is defined. .. attribute:: __name__ @@ -2573,7 +2581,7 @@ types. at runtime as soon as the class has been created. Monkey-patching attributes onto a runtime-checkable protocol will still work, but will have no impact on :func:`isinstance` checks comparing objects to the - protocol. See :ref:`"What's new in Python 3.12" ` + protocol. See :ref:`What's new in Python 3.12 ` for more details. @@ -2816,19 +2824,12 @@ types. .. versionchanged:: 3.13 Support for the :data:`ReadOnly` qualifier was added. - .. deprecated-removed:: 3.13 3.15 - When using the functional syntax to create a TypedDict class, failing to - pass a value to the 'fields' parameter (``TD = TypedDict("TD")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``TD = TypedDict("TD", None)``) is also deprecated. Both will be - disallowed in Python 3.15. To create a TypedDict class with 0 fields, - use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. Protocols --------- The following protocols are provided by the :mod:`!typing` module. All are decorated -with :func:`@runtime_checkable `. +with :deco:`runtime_checkable`. .. class:: SupportsAbs @@ -3009,7 +3010,7 @@ Functions and decorators The presence of ``@dataclass_transform()`` tells a static type checker that the decorated object performs runtime "magic" that transforms a class in a similar way to - :func:`@dataclasses.dataclass `. + :deco:`dataclasses.dataclass`. Example usage with a decorator function: @@ -3047,14 +3048,14 @@ Functions and decorators The ``CustomerModel`` classes defined above will be treated by type checkers similarly to classes created with - :func:`@dataclasses.dataclass `. + :deco:`dataclasses.dataclass`. For example, type checkers will assume these classes have ``__init__`` methods that accept ``id`` and ``name``. The decorated class, metaclass, or function may accept the following bool arguments which type checkers will assume have the same effect as they would have on the - :func:`@dataclasses.dataclass` decorator: ``init``, + :deco:`dataclasses.dataclass` decorator: ``init``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, ``kw_only``, and ``slots``. It must be possible for the value of these arguments (``True`` or ``False``) to be statically evaluated. @@ -3182,12 +3183,12 @@ Functions and decorators .. function:: get_overloads(func) - Return a sequence of :func:`@overload `-decorated definitions for + Return a sequence of :deco:`overload`-decorated definitions for *func*. *func* is the function object for the implementation of the overloaded function. For example, given the definition of ``process`` in - the documentation for :func:`@overload `, + the documentation for :deco:`overload`, ``get_overloads(process)`` will return a sequence of three function objects for the three defined overloads. If called on a function with no overloads, ``get_overloads()`` returns an empty sequence. @@ -3257,17 +3258,6 @@ Functions and decorators ``@no_type_check`` mutates the decorated object in place. -.. decorator:: no_type_check_decorator - - Decorator to give another decorator the :func:`no_type_check` effect. - - This wraps the decorator with something that wraps the decorated - function in :func:`no_type_check`. - - .. deprecated-removed:: 3.13 3.15 - No type checker ever added support for ``@no_type_check_decorator``. It - is therefore deprecated, and will be removed in Python 3.15. - .. decorator:: override Decorator to indicate that a method in a subclass is intended to override a @@ -3344,7 +3334,7 @@ Introspection helpers If *globalns* or *localns* is not given, appropriate namespace dictionaries are inferred from *obj*. * ``None`` is replaced with :class:`types.NoneType`. - * If :func:`@no_type_check ` has been applied to *obj*, an + * If :deco:`no_type_check` has been applied to *obj*, an empty dictionary is returned. * If *obj* is a class ``C``, the function returns a dictionary that merges annotations from ``C``'s base classes with those on ``C`` directly. This @@ -3353,13 +3343,19 @@ Introspection helpers ``__annotations__`` dictionaries. Annotations on classes appearing earlier in the :term:`method resolution order` always take precedence over annotations on classes appearing later in the method resolution order. - * The function recursively replaces all occurrences of ``Annotated[T, ...]`` + * The function recursively replaces all occurrences of + ``Annotated[T, ...]``, ``Required[T]``, ``NotRequired[T]``, and ``ReadOnly[T]`` with ``T``, unless *include_extras* is set to ``True`` (see :class:`Annotated` for more information). - See also :func:`inspect.get_annotations`, a lower-level function that + See also :func:`annotationlib.get_annotations`, a lower-level function that returns annotations more directly. + .. caution:: + + This function may execute arbitrary code contained in annotations. + See :ref:`annotationlib-security` for more information. + .. note:: If any forward references in the annotations of *obj* are not resolvable @@ -3500,20 +3496,16 @@ Introspection helpers Evaluate an :class:`annotationlib.ForwardRef` as a :term:`type hint`. This is similar to calling :meth:`annotationlib.ForwardRef.evaluate`, - but unlike that method, :func:`!evaluate_forward_ref` also: - - * Recursively evaluates forward references nested within the type hint. - * Raises :exc:`TypeError` when it encounters certain objects that are - not valid type hints. - * Replaces type hints that evaluate to :const:`!None` with - :class:`types.NoneType`. - * Supports the :attr:`~annotationlib.Format.FORWARDREF` and - :attr:`~annotationlib.Format.STRING` formats. + but unlike that method, :func:`!evaluate_forward_ref` also + recursively evaluates forward references nested within the type hint. See the documentation for :meth:`annotationlib.ForwardRef.evaluate` for - the meaning of the *owner*, *globals*, *locals*, and *type_params* parameters. - *format* specifies the format of the annotation and is a member of - the :class:`annotationlib.Format` enum. + the meaning of the *owner*, *globals*, *locals*, *type_params*, and *format* parameters. + + .. caution:: + + This function may execute arbitrary code contained in annotations. + See :ref:`annotationlib-security` for more information. .. versionadded:: 3.14 @@ -3539,28 +3531,32 @@ Constant .. data:: TYPE_CHECKING A special constant that is assumed to be ``True`` by 3rd party static - type checkers. It is ``False`` at runtime. + type checkers. It's ``False`` at runtime. + + A module which is expensive to import, and which only contain types + used for typing annotations, can be safely imported inside an + ``if TYPE_CHECKING:`` block. This prevents the module from actually + being imported at runtime; annotations aren't eagerly evaluated + (see :pep:`649`) so using undefined symbols in annotations is + harmless--as long as you don't later examine them. + Your static type analysis tool will set ``TYPE_CHECKING`` to + ``True`` during static type analysis, which means the module will + be imported and the types will be checked properly during such analysis. Usage:: if TYPE_CHECKING: import expensive_mod - def fun(arg: 'expensive_mod.SomeType') -> None: + def fun(arg: expensive_mod.SomeType) -> None: local_var: expensive_mod.AnotherType = other_fun() - The first type annotation must be enclosed in quotes, making it a - "forward reference", to hide the ``expensive_mod`` reference from the - interpreter runtime. Type annotations for local variables are not - evaluated, so the second annotation does not need to be enclosed in quotes. - - .. note:: - - If ``from __future__ import annotations`` is used, - annotations are not evaluated at function definition time. - Instead, they are stored as strings in ``__annotations__``. - This makes it unnecessary to use quotes around the annotation - (see :pep:`563`). + If you occasionally need to examine type annotations at runtime + which may contain undefined symbols, use + :meth:`annotationlib.get_annotations` with a ``format`` parameter + of :attr:`annotationlib.Format.STRING` or + :attr:`annotationlib.Format.FORWARDREF` to safely retrieve the + annotations without raising :exc:`NameError`. .. versionadded:: 3.5.2 @@ -3776,6 +3772,28 @@ Aliases to container ABCs in :mod:`collections.abc` :class:`collections.abc.Set` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. +.. class:: ByteString(Sequence[int]) + + Deprecated alias to :class:`collections.abc.ByteString`. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use in + type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` were + also never understood as subtypes of :class:`!ByteString` (either at runtime + or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + + .. deprecated-removed:: 3.9 3.17 + .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) Deprecated alias to :class:`collections.abc.Collection`. @@ -4069,6 +4087,10 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.9 - Undecided (see :ref:`deprecated-aliases` for more information) - :pep:`585` + * - :class:`typing.ByteString` + - 3.9 + - 3.17 + - :gh:`91896` * - :data:`typing.Text` - 3.11 - Undecided @@ -4081,10 +4103,6 @@ convenience. This is subject to change, and not all deprecations are listed. - 3.12 - Undecided - :pep:`695` - * - :func:`@typing.no_type_check_decorator ` - - 3.13 - - 3.15 - - :gh:`106309` * - :data:`typing.AnyStr` - 3.13 - 3.18 diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index 0aef597d064..34f21f49b4b 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -17,91 +17,174 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 16.0.0 -`_. +this database is compiled from the `UCD version 17.0.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" `_. It defines the following functions: +.. seealso:: -.. function:: lookup(name) + The :ref:`unicode-howto` for more information about Unicode and how to use + this module. + + +.. function:: lookup(name, /) Look up character by name. If a character with the given name is found, return the corresponding character. If not found, :exc:`KeyError` is raised. + For example:: + + >>> unicodedata.lookup('LEFT CURLY BRACKET') + '{' + + The characters returned by this function are the same as those produced by + ``\N`` escape sequence in string literals. For example:: + + >>> unicodedata.lookup('MIDDLE DOT') == '\N{MIDDLE DOT}' + True .. versionchanged:: 3.3 Support for name aliases [#]_ and named sequences [#]_ has been added. -.. function:: name(chr[, default]) +.. function:: name(chr, default=None, /) Returns the name assigned to the character *chr* as a string. If no name is defined, *default* is returned, or, if not given, :exc:`ValueError` is - raised. + raised. For example:: + + >>> unicodedata.name('½') + 'VULGAR FRACTION ONE HALF' + >>> unicodedata.name('\uFFFF', 'fallback') + 'fallback' -.. function:: decimal(chr[, default]) +.. function:: decimal(chr, default=None, /) Returns the decimal value assigned to the character *chr* as integer. If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. + :exc:`ValueError` is raised. For example:: + + >>> unicodedata.decimal('\N{ARABIC-INDIC DIGIT NINE}') + 9 + >>> unicodedata.decimal('\N{SUPERSCRIPT NINE}', -1) + -1 -.. function:: digit(chr[, default]) +.. function:: digit(chr, default=None, /) Returns the digit value assigned to the character *chr* as integer. If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. + :exc:`ValueError` is raised:: + + >>> unicodedata.digit('\N{SUPERSCRIPT NINE}') + 9 -.. function:: numeric(chr[, default]) +.. function:: numeric(chr, default=None, /) Returns the numeric value assigned to the character *chr* as float. If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. + :exc:`ValueError` is raised:: + + >>> unicodedata.numeric('½') + 0.5 -.. function:: category(chr) +.. function:: category(chr, /) Returns the general category assigned to the character *chr* as - string. + string. General category names consist of two letters. + See the `General Category Values section of the Unicode Character + Database documentation `_ + for a list of category codes. For example:: + + >>> unicodedata.category('A') # 'L'etter, 'u'ppercase + 'Lu' -.. function:: bidirectional(chr) +.. function:: bidirectional(chr, /) Returns the bidirectional class assigned to the character *chr* as string. If no such value is defined, an empty string is returned. + See the `Bidirectional Class Values section of the Unicode Character + Database `_ + documentation for a list of bidirectional codes. For example:: + + >>> unicodedata.bidirectional('\N{ARABIC-INDIC DIGIT SEVEN}') # 'A'rabic, 'N'umber + 'AN' -.. function:: combining(chr) +.. function:: combining(chr, /) Returns the canonical combining class assigned to the character *chr* as integer. Returns ``0`` if no combining class is defined. + See the `Canonical Combining Class Values section of the Unicode Character + Database `_ + for more information. -.. function:: east_asian_width(chr) +.. function:: east_asian_width(chr, /) Returns the east asian width assigned to the character *chr* as - string. + string. For a list of widths and or more information, see the + `Unicode Standard Annex #11 `_. -.. function:: mirrored(chr) +.. function:: mirrored(chr, /) Returns the mirrored property assigned to the character *chr* as integer. Returns ``1`` if the character has been identified as a "mirrored" - character in bidirectional text, ``0`` otherwise. + character in bidirectional text, ``0`` otherwise. For example:: + + >>> unicodedata.mirrored('>') + 1 -.. function:: decomposition(chr) +.. function:: isxidstart(chr, /) + + Return ``True`` if *chr* is a valid identifier start per the + `Unicode Standard Annex #31 `_, + that is, it has the ``XID_Start`` property. Return ``False`` otherwise. + For example:: + + >>> unicodedata.isxidstart('S') + True + >>> unicodedata.isxidstart('0') + False + + .. versionadded:: 3.15 + + +.. function:: isxidcontinue(chr, /) + + Return ``True`` if *chr* is a valid identifier character per the + `Unicode Standard Annex #31 `_, + that is, it has the ``XID_Continue`` property. Return ``False`` otherwise. + For example:: + + >>> unicodedata.isxidcontinue('S') + True + >>> unicodedata.isxidcontinue(' ') + False + + .. versionadded:: 3.15 + + +.. function:: decomposition(chr, /) Returns the character decomposition mapping assigned to the character *chr* as string. An empty string is returned in case no such mapping is - defined. + defined. For example:: + + >>> unicodedata.decomposition('Ã') + '0041 0303' -.. function:: normalize(form, unistr) +.. function:: normalize(form, unistr, /) Return the normal form *form* for the Unicode string *unistr*. Valid values for *form* are 'NFC', 'NFKC', 'NFD', and 'NFKD'. @@ -122,9 +205,9 @@ following functions: normally would be unified with other characters. For example, U+2160 (ROMAN NUMERAL ONE) is really the same thing as U+0049 (LATIN CAPITAL LETTER I). However, it is supported in Unicode for compatibility with existing character - sets (e.g. gb2312). + sets (for example, gb2312). - The normal form KD (NFKD) will apply the compatibility decomposition, i.e. + The normal form KD (NFKD) will apply the compatibility decomposition, that is, replace all compatibility characters with their equivalents. The normal form KC (NFKC) first applies the compatibility decomposition, followed by the canonical composition. @@ -133,7 +216,8 @@ following functions: a human reader, if one has combining characters and the other doesn't, they may not compare equal. -.. function:: is_normalized(form, unistr) + +.. function:: is_normalized(form, unistr, /) Return whether the Unicode string *unistr* is in the normal form *form*. Valid values for *form* are 'NFC', 'NFKC', 'NFD', and 'NFKD'. @@ -154,27 +238,9 @@ In addition, the module exposes the following constant: Unicode database version 3.2 instead, for applications that require this specific version of the Unicode database (such as IDNA). -Examples: - - >>> import unicodedata - >>> unicodedata.lookup('LEFT CURLY BRACKET') - '{' - >>> unicodedata.name('/') - 'SOLIDUS' - >>> unicodedata.decimal('9') - 9 - >>> unicodedata.decimal('a') - Traceback (most recent call last): - File "", line 1, in - ValueError: not a decimal - >>> unicodedata.category('A') # 'L'etter, 'u'ppercase - 'Lu' - >>> unicodedata.bidirectional('\u0660') # 'A'rabic, 'N'umber - 'AN' - .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/16.0.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/17.0.0/ucd/NameAliases.txt -.. [#] https://www.unicode.org/Public/16.0.0/ucd/NamedSequences.txt +.. [#] https://www.unicode.org/Public/17.0.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 00cc9bfc0a5..61c75b5a03b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -600,13 +600,13 @@ this list of calls for us:: Partial mocking ~~~~~~~~~~~~~~~ -In some tests I wanted to mock out a call to :meth:`datetime.date.today` -to return a known date, but I didn't want to prevent the code under test from -creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`datetime.date.today` method. +For some tests, you may want to mock out a call to :meth:`datetime.date.today` +to return a known date, but don't want to prevent the code under test from +creating new date objects. Unfortunately :class:`datetime.date` is written in C, +so you cannot just monkey-patch out the static :meth:`datetime.date.today` method. -I found a simple way of doing this that involved effectively wrapping the date -class with a mock, but passing through calls to the constructor to the real +Instead, you can effectively wrap the date +class with a mock, while passing through calls to the constructor to the real class (and returning real instances). The :func:`patch decorator ` is used here to @@ -743,16 +743,15 @@ exception is raised in the setUp then tearDown is not called. Mocking Unbound Methods ~~~~~~~~~~~~~~~~~~~~~~~ -Whilst writing tests today I needed to patch an *unbound method* (patching the -method on the class rather than on the instance). I needed self to be passed -in as the first argument because I want to make asserts about which objects -were calling this particular method. The issue is that you can't patch with a -mock for this, because if you replace an unbound method with a mock it doesn't -become a bound method when fetched from the instance, and so it doesn't get -self passed in. The workaround is to patch the unbound method with a real -function instead. The :func:`patch` decorator makes it so simple to -patch out methods with a mock that having to create a real function becomes a -nuisance. +Sometimes a test needs to patch an *unbound method*, which means patching the +method on the class rather than on the instance. In order to make assertions +about which objects were calling this particular method, you need to pass +``self`` as the first argument. The issue is that you can't patch with a mock for +this, because if you replace an unbound method with a mock it doesn't become +a bound method when fetched from the instance, and so it doesn't get ``self`` +passed in. The workaround is to patch the unbound method with a real function +instead. The :func:`patch` decorator makes it so simple to patch out methods +with a mock that having to create a real function becomes a nuisance. If you pass ``autospec=True`` to patch then it does the patching with a *real* function object. This function object has the same signature as the one @@ -760,8 +759,8 @@ it is replacing, but delegates to a mock under the hood. You still get your mock auto-created in exactly the same way as before. What it means though, is that if you use it to patch out an unbound method on a class the mocked function will be turned into a bound method if it is fetched from an instance. -It will have ``self`` passed in as the first argument, which is exactly what I -wanted: +It will have ``self`` passed in as the first argument, which is exactly what +was needed: >>> class Foo: ... def foo(self): @@ -864,9 +863,9 @@ Here's one solution that uses the :attr:`~Mock.side_effect` functionality. If you provide a ``side_effect`` function for a mock then ``side_effect`` will be called with the same args as the mock. This gives us an opportunity to copy the arguments and store them for later assertions. In this -example I'm using *another* mock to store the arguments so that I can use the +example we're using *another* mock to store the arguments so that we can use the mock methods for doing the assertion. Again a helper function sets this up for -me. :: +us. :: >>> from copy import deepcopy >>> from unittest.mock import Mock, patch, DEFAULT diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 27c169dde72..91f90a0726a 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -639,7 +639,7 @@ the *new_callable* argument to :func:`patch`. This is either ``None`` (if the mock hasn't been called), or the arguments that the mock was last called with. This will be in the form of a tuple: the first member, which can also be accessed through - the ``args`` property, is any ordered arguments the mock was + the ``args`` property, is any positional arguments the mock was called with (or an empty tuple) and the second member, which can also be accessed through the ``kwargs`` property, is any keyword arguments (or an empty dictionary). @@ -2654,9 +2654,9 @@ with any methods on the mock: .. code-block:: pycon - >>> mock.has_data() + >>> mock.header_items() - >>> mock.has_data.assret_called_with() # Intentional typo! + >>> mock.header_items.assret_called_with() # Intentional typo! Auto-speccing solves this problem. You can either pass ``autospec=True`` to :func:`patch` / :func:`patch.object` or use the :func:`create_autospec` function to create a diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 61022fe052c..0bc0a953fd9 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -109,7 +109,7 @@ Here is a short script to test three string methods:: unittest.main() -A testcase is created by subclassing :class:`unittest.TestCase`. The three +A test case is created by subclassing :class:`unittest.TestCase`. The three individual tests are defined with methods whose names start with the letters ``test``. This naming convention informs the test runner about which methods represent tests. @@ -438,7 +438,7 @@ run whether the test method succeeded or not. Such a working environment for the testing code is called a :dfn:`test fixture`. A new TestCase instance is created as a unique test fixture used to execute each individual test method. Thus -:meth:`~TestCase.setUp`, :meth:`~TestCase.tearDown`, and :meth:`~TestCase.__init__` +:meth:`~TestCase.setUp`, :meth:`~TestCase.tearDown`, and :meth:`!TestCase.__init__` will be called once per test. It is recommended that you use TestCase implementations to group tests together @@ -518,7 +518,7 @@ set-up and tear-down methods:: subclasses will make future test refactorings infinitely easier. In some cases, the existing tests may have been written using the :mod:`doctest` -module. If so, :mod:`doctest` provides a :class:`DocTestSuite` class that can +module. If so, :mod:`doctest` provides a :class:`~doctest.DocTestSuite` class that can automatically build :class:`unittest.TestSuite` instances from the existing :mod:`doctest`\ -based tests. @@ -1023,7 +1023,7 @@ Test cases additional keyword argument *msg*. The context manager will store the caught exception object in its - :attr:`exception` attribute. This can be useful if the intention + :attr:`!exception` attribute. This can be useful if the intention is to perform additional checks on the exception raised:: with self.assertRaises(SomeException) as cm: @@ -1036,7 +1036,7 @@ Test cases Added the ability to use :meth:`assertRaises` as a context manager. .. versionchanged:: 3.2 - Added the :attr:`exception` attribute. + Added the :attr:`!exception` attribute. .. versionchanged:: 3.3 Added the *msg* keyword argument when used as a context manager. @@ -1089,8 +1089,8 @@ Test cases additional keyword argument *msg*. The context manager will store the caught warning object in its - :attr:`warning` attribute, and the source line which triggered the - warnings in the :attr:`filename` and :attr:`lineno` attributes. + :attr:`!warning` attribute, and the source line which triggered the + warnings in the :attr:`!filename` and :attr:`!lineno` attributes. This can be useful if the intention is to perform additional checks on the warning caught:: @@ -1131,7 +1131,7 @@ Test cases .. versionchanged:: 3.3 Added the *msg* keyword argument when used as a context manager. - .. method:: assertLogs(logger=None, level=None) + .. method:: assertLogs(logger=None, level=None, formatter=None) A context manager to test that at least one message is logged on the *logger* or one of its children, with at least the given @@ -1146,6 +1146,10 @@ Test cases its string equivalent (for example either ``"ERROR"`` or :const:`logging.ERROR`). The default is :const:`logging.INFO`. + If given, *formatter* should be a :class:`logging.Formatter` object. + The default is a formatter with format string + ``"%(levelname)s:%(name)s:%(message)s"`` + The test passes if at least one message emitted inside the ``with`` block matches the *logger* and *level* conditions, otherwise it fails. @@ -1173,6 +1177,9 @@ Test cases .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Now accepts a *formatter* to control how messages are formatted. + .. method:: assertNoLogs(logger=None, level=None) A context manager to test that no messages are logged on @@ -1430,7 +1437,7 @@ Test cases that lists the differences between the sets. This method is used by default when comparing sets or frozensets with :meth:`assertEqual`. - Fails if either of *first* or *second* does not have a :meth:`set.difference` + Fails if either of *first* or *second* does not have a :meth:`~frozenset.difference` method. .. versionadded:: 3.1 @@ -1638,7 +1645,7 @@ Test cases .. method:: asyncSetUp() :async: - Method called to prepare the test fixture. This is called after :meth:`setUp`. + Method called to prepare the test fixture. This is called after :meth:`TestCase.setUp`. This is called immediately before calling the test method; other than :exc:`AssertionError` or :exc:`SkipTest`, any exception raised by this method will be considered an error rather than a test failure. The default implementation @@ -1648,7 +1655,7 @@ Test cases :async: Method called immediately after the test method has been called and the - result recorded. This is called before :meth:`tearDown`. This is called even if + result recorded. This is called before :meth:`~TestCase.tearDown`. This is called even if the test method raised an exception, so the implementation in subclasses may need to be particularly careful about checking internal state. Any exception, other than :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be @@ -1677,7 +1684,7 @@ Test cases Sets up a new event loop to run the test, collecting the result into the :class:`TestResult` object passed as *result*. If *result* is omitted or ``None``, a temporary result object is created (by calling - the :meth:`defaultTestResult` method) and used. The result object is + the :meth:`~TestCase.defaultTestResult` method) and used. The result object is returned to :meth:`run`'s caller. At the end of the test all the tasks in the event loop are cancelled. @@ -1798,7 +1805,7 @@ Grouping tests returned by repeated iterations before :meth:`TestSuite.run` must be the same for each call iteration. After :meth:`TestSuite.run`, callers should not rely on the tests returned by this method unless the caller uses a - subclass that overrides :meth:`TestSuite._removeTestAtIndex` to preserve + subclass that overrides :meth:`!TestSuite._removeTestAtIndex` to preserve test references. .. versionchanged:: 3.2 @@ -1809,10 +1816,10 @@ Grouping tests .. versionchanged:: 3.4 In earlier versions the :class:`TestSuite` held references to each :class:`TestCase` after :meth:`TestSuite.run`. Subclasses can restore - that behavior by overriding :meth:`TestSuite._removeTestAtIndex`. + that behavior by overriding :meth:`!TestSuite._removeTestAtIndex`. In the typical usage of a :class:`TestSuite` object, the :meth:`run` method - is invoked by a :class:`TestRunner` rather than by the end-user test harness. + is invoked by a :class:`!TestRunner` rather than by the end-user test harness. Loading and running tests @@ -1846,12 +1853,12 @@ Loading and running tests .. method:: loadTestsFromTestCase(testCaseClass) Return a suite of all test cases contained in the :class:`TestCase`\ -derived - :class:`testCaseClass`. + :class:`!testCaseClass`. A test case instance is created for each method named by :meth:`getTestCaseNames`. By default these are the method names beginning with ``test``. If :meth:`getTestCaseNames` returns no - methods, but the :meth:`runTest` method is implemented, a single test + methods, but the :meth:`!runTest` method is implemented, a single test case is created for that method instead. @@ -1898,13 +1905,13 @@ Loading and running tests case class will be picked up as "a test method within a test case class", rather than "a callable object". - For example, if you have a module :mod:`SampleTests` containing a - :class:`TestCase`\ -derived class :class:`SampleTestCase` with three test - methods (:meth:`test_one`, :meth:`test_two`, and :meth:`test_three`), the + For example, if you have a module :mod:`!SampleTests` containing a + :class:`TestCase`\ -derived class :class:`!SampleTestCase` with three test + methods (:meth:`!test_one`, :meth:`!test_two`, and :meth:`!test_three`), the specifier ``'SampleTests.SampleTestCase'`` would cause this method to return a suite which will run all three test methods. Using the specifier ``'SampleTests.SampleTestCase.test_two'`` would cause it to return a test - suite which will run only the :meth:`test_two` test method. The specifier + suite which will run only the :meth:`!test_two` test method. The specifier can refer to modules and packages which have not been imported; they will be imported as a side-effect. @@ -2051,7 +2058,7 @@ Loading and running tests Testing frameworks built on top of :mod:`unittest` may want access to the :class:`TestResult` object generated by running a set of tests for reporting purposes; a :class:`TestResult` instance is returned by the - :meth:`TestRunner.run` method for this purpose. + :meth:`!TestRunner.run` method for this purpose. :class:`TestResult` instances have the following attributes that will be of interest when inspecting the results of running a set of tests: @@ -2137,12 +2144,12 @@ Loading and running tests This method can be called to signal that the set of tests being run should be aborted by setting the :attr:`shouldStop` attribute to ``True``. - :class:`TestRunner` objects should respect this flag and return without + :class:`!TestRunner` objects should respect this flag and return without running any additional tests. For example, this feature is used by the :class:`TextTestRunner` class to stop the test framework when the user signals an interrupt from the - keyboard. Interactive tools which provide :class:`TestRunner` + keyboard. Interactive tools which provide :class:`!TestRunner` implementations can use this in a similar manner. The following methods of the :class:`TestResult` class are used to maintain @@ -2462,9 +2469,9 @@ Class and Module Fixtures ------------------------- Class and module level fixtures are implemented in :class:`TestSuite`. When -the test suite encounters a test from a new class then :meth:`tearDownClass` -from the previous class (if there is one) is called, followed by -:meth:`setUpClass` from the new class. +the test suite encounters a test from a new class then +:meth:`~TestCase.tearDownClass` from the previous class (if there is one) +is called, followed by :meth:`~TestCase.setUpClass` from the new class. Similarly if a test is from a different module from the previous test then ``tearDownModule`` from the previous module is run, followed by @@ -2525,6 +2532,10 @@ instead of as an error. setUpModule and tearDownModule ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. function:: setUpModule + tearDownModule + :no-typesetting: + These should be implemented as functions:: def setUpModule(): @@ -2556,7 +2567,7 @@ To add cleanup code that must be run even in the case of an exception, use .. versionadded:: 3.8 -.. classmethod:: enterModuleContext(cm) +.. function:: enterModuleContext(cm) Enter the supplied :term:`context manager`. If successful, also add its :meth:`~object.__exit__` method as a cleanup function by diff --git a/Doc/library/unix.rst b/Doc/library/unix.rst index 4553a104d15..bae32b36280 100644 --- a/Doc/library/unix.rst +++ b/Doc/library/unix.rst @@ -1,7 +1,7 @@ .. _unix: ********************** -Unix Specific Services +Unix-specific services ********************** The modules described in this chapter provide interfaces to features that are @@ -11,6 +11,7 @@ of it. Here's an overview: .. toctree:: + shlex.rst posix.rst pwd.rst grp.rst diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index b0f26724d0c..5f796578eaa 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -210,6 +210,9 @@ The :mod:`urllib.request` module defines the following functions: Windows a UNC path is returned (as before), and on other platforms a :exc:`~urllib.error.URLError` is raised. + .. versionchanged:: 3.14 + The URL query and fragment components are discarded if present. + .. versionchanged:: 3.14 The *require_scheme* and *resolve_host* parameters were added. @@ -829,10 +832,13 @@ The following attribute and methods should only be used by classes derived from errors. It will be called automatically by the :class:`OpenerDirector` getting the error, and should not normally be called in other circumstances. - *req* will be a :class:`Request` object, *fp* will be a file-like object with - the HTTP error body, *code* will be the three-digit code of the error, *msg* - will be the user-visible explanation of the code and *hdrs* will be a mapping - object with the headers of the error. + :class:`OpenerDirector` will call this method with five positional arguments: + + 1. a :class:`Request` object, + #. a file-like object with the HTTP error body, + #. the three-digit code of the error, as a string, + #. the user-visible explanation of the code, as a string, and + #. the headers of the error, as a mapping object. Return values and exceptions raised should be the same as those of :func:`urlopen`. @@ -1121,7 +1127,7 @@ HTTPHandler Objects .. method:: HTTPHandler.http_open(req) Send an HTTP request, which can be either GET or POST, depending on - ``req.has_data()``. + ``req.data``. .. _https-handler-objects: @@ -1133,7 +1139,7 @@ HTTPSHandler Objects .. method:: HTTPSHandler.https_open(req) Send an HTTPS request, which can be either GET or POST, depending on - ``req.has_data()``. + ``req.data``. .. _file-handler-objects: diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 016fcdc75da..674f646c633 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -19,7 +19,7 @@ This module provides a single class, :class:`RobotFileParser`, which answers questions about whether or not a particular user agent can fetch a URL on the -web site that published the :file:`robots.txt` file. For more details on the +website that published the :file:`robots.txt` file. For more details on the structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index 8cce6b98cbc..aa4f1bf940b 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -193,43 +193,52 @@ The :mod:`uuid` module defines the following functions: .. function:: uuid1(node=None, clock_seq=None) - Generate a UUID from a host ID, sequence number, and the current time. If *node* - is not given, :func:`getnode` is used to obtain the hardware address. If - *clock_seq* is given, it is used as the sequence number; otherwise a random - 14-bit sequence number is chosen. + Generate a UUID from a host ID, sequence number, and the current time + according to :rfc:`RFC 9562, §5.1 <9562#section-5.1>`. + + When *node* is not specified, :func:`getnode` is used to obtain the hardware + address as a 48-bit positive integer. When a sequence number *clock_seq* is + not specified, a pseudo-random 14-bit positive integer is generated. + + If *node* or *clock_seq* exceed their expected bit count, + only their least significant bits are kept. .. function:: uuid3(namespace, name) Generate a UUID based on the MD5 hash of a namespace identifier (which is a UUID) and a name (which is a :class:`bytes` object or a string - that will be encoded using UTF-8). + that will be encoded using UTF-8) + according to :rfc:`RFC 9562, §5.3 <9562#section-5.3>`. .. function:: uuid4() - Generate a random UUID. + Generate a random UUID in a cryptographically-secure method + according to :rfc:`RFC 9562, §5.4 <9562#section-5.4>`. .. function:: uuid5(namespace, name) Generate a UUID based on the SHA-1 hash of a namespace identifier (which is a UUID) and a name (which is a :class:`bytes` object or a string - that will be encoded using UTF-8). + that will be encoded using UTF-8) + according to :rfc:`RFC 9562, §5.5 <9562#section-5.5>`. .. function:: uuid6(node=None, clock_seq=None) Generate a UUID from a sequence number and the current time according to - :rfc:`9562`. + :rfc:`RFC 9562, §5.6 <9562#section-5.6>`. + This is an alternative to :func:`uuid1` to improve database locality. When *node* is not specified, :func:`getnode` is used to obtain the hardware address as a 48-bit positive integer. When a sequence number *clock_seq* is not specified, a pseudo-random 14-bit positive integer is generated. - If *node* or *clock_seq* exceed their expected bit count, only their least - significant bits are kept. + If *node* or *clock_seq* exceed their expected bit count, + only their least significant bits are kept. .. versionadded:: 3.14 @@ -257,6 +266,10 @@ The :mod:`uuid` module defines the following functions: non-specified arguments are substituted for a pseudo-random integer of appropriate size. + By default, *a*, *b* and *c* are not generated by a cryptographically + secure pseudo-random number generator (CSPRNG). Use :func:`uuid4` when + a UUID needs to be used in a security-sensitive context. + .. versionadded:: 3.14 @@ -398,7 +411,7 @@ Here are some examples of typical usage of the :mod:`uuid` module:: >>> import uuid >>> # make a UUID based on the host ID and current time - >>> uuid.uuid1() + >>> uuid.uuid1() # doctest: +SKIP UUID('a8098c1a-f86e-11da-bd1a-00112444be1e') >>> # make a UUID using an MD5 hash of a namespace UUID and a name @@ -436,15 +449,24 @@ Here are some examples of typical usage of the :mod:`uuid` module:: >>> uuid.MAX UUID('ffffffff-ffff-ffff-ffff-ffffffffffff') + >>> # same as UUIDv1 but with fields reordered to improve DB locality + >>> uuid.uuid6() # doctest: +SKIP + UUID('1f0799c0-98b9-62db-92c6-a0d365b91053') + >>> # get UUIDv7 creation (local) time as a timestamp in milliseconds >>> u = uuid.uuid7() >>> u.time # doctest: +SKIP 1743936859822 + >>> # get UUIDv7 creation (local) time as a datetime object >>> import datetime as dt >>> dt.datetime.fromtimestamp(u.time / 1000) # doctest: +SKIP datetime.datetime(...) + >>> # make a UUID with custom blocks + >>> uuid.uuid8(0x12345678, 0x9abcdef0, 0x11223344) + UUID('00001234-5678-8ef0-8000-000011223344') + .. _uuid-cli-example: diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index bed799aedfd..4c000a92e87 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -22,10 +22,10 @@ The :mod:`!venv` module supports creating lightweight "virtual environments", each with their own independent set of Python packages installed in their :mod:`site` directories. A virtual environment is created on top of an existing -Python installation, known as the virtual environment's "base" Python, and may -optionally be isolated from the packages in the base environment, -so only those explicitly installed in the virtual environment are available. -See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s +Python installation, known as the virtual environment's "base" Python, and by +default is isolated from the packages in the base environment, +so that only those explicitly installed in the virtual environment are +available. See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s :ref:`virtual environments documentation ` for more information. @@ -78,7 +78,7 @@ It also creates a :file:`bin` (or :file:`Scripts` on Windows) subdirectory containing a copy or symlink of the Python executable (as appropriate for the platform or arguments used at environment creation time). It also creates a :file:`lib/pythonX.Y/site-packages` subdirectory -(on Windows, this is :file:`Lib\site-packages`). +(on Windows, this is :file:`Lib\\site-packages`). If an existing directory is specified, it will be re-used. .. versionchanged:: 3.5 @@ -105,36 +105,52 @@ The command, if run with ``-h``, will show the available options:: Creates virtual Python environments in one or more target directories. - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory - if it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been - upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-files - Skips adding SCM ignore files to the environment - directory (Git is supported by default). - Once an environment has been created, you may wish to activate it, e.g. by sourcing an activate script in its bin directory. +.. _venv-cli: +.. program:: venv + +.. option:: ENV_DIR + + A required argument specifying the directory to create the environment in. + +.. option:: --system-site-packages + + Give the virtual environment access to the system site-packages directory. + +.. option:: --symlinks + + Try to use symlinks rather than copies, when symlinks are not the default for the platform. + +.. option:: --copies + + Try to use copies rather than symlinks, even when symlinks are the default for the platform. + +.. option:: --clear + + Delete the contents of the environment directory if it already exists, before environment creation. + +.. option:: --upgrade + + Upgrade the environment directory to use this version of Python, assuming Python has been upgraded in-place. + +.. option:: --without-pip + + Skips installing or upgrading pip in the virtual environment (pip is bootstrapped by default). + +.. option:: --prompt + + Provides an alternative prompt prefix for this environment. + +.. option:: --upgrade-deps + + Upgrade core dependencies (pip) to the latest version in PyPI. + +.. option:: --without-scm-ignore-files + + Skips adding SCM ignore files to the environment directory (Git is supported by default). + .. versionchanged:: 3.4 Installs pip by default, added the ``--without-pip`` and ``--copies`` @@ -391,6 +407,8 @@ creation according to their needs, the :class:`EnvBuilder` class. * ``lib_path`` - The purelib path for the virtual environment. + * ``platlib_path`` - The platlib path for the virtual environment. + * ``bin_path`` - The script path for the virtual environment. * ``bin_name`` - The name of the script path relative to the virtual @@ -415,6 +433,9 @@ creation according to their needs, the :class:`EnvBuilder` class. The attribute ``lib_path`` was added to the context, and the context object was documented. + .. versionchanged:: 3.15 + The attribute ``platlib_path`` was added to the context. + .. method:: create_configuration(context) Creates the ``pyvenv.cfg`` configuration file in the environment. diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 00bafd1be4b..0de7a90bfcb 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -80,7 +80,9 @@ The following warnings category classes are currently defined: | | unless triggered by code in ``__main__``). | +----------------------------------+-----------------------------------------------+ | :exc:`SyntaxWarning` | Base category for warnings about dubious | -| | syntactic features. | +| | syntactic features (typically emitted when | +| | compiling Python source code, and hence | +| | may not be suppressed by runtime filters) | +----------------------------------+-----------------------------------------------+ | :exc:`RuntimeWarning` | Base category for warnings about dubious | | | runtime features. | @@ -157,8 +159,10 @@ the disposition of the match. Each entry is a tuple of the form (*action*, * *message* is a string containing a regular expression that the start of the warning message must match, case-insensitively. In :option:`-W` and - :envvar:`PYTHONWARNINGS`, *message* is a literal string that the start of the - warning message must contain (case-insensitively), ignoring any whitespace at + :envvar:`PYTHONWARNINGS`, if *message* starts and ends with a forward slash + (``/``), it specifies a regular expression as above; + otherwise it is a literal string that the start of the + warning message must match (case-insensitively), ignoring any whitespace at the start or end of *message*. * *category* is a class (a subclass of :exc:`Warning`) of which the warning @@ -166,7 +170,9 @@ the disposition of the match. Each entry is a tuple of the form (*action*, * *module* is a string containing a regular expression that the start of the fully qualified module name must match, case-sensitively. In :option:`-W` and - :envvar:`PYTHONWARNINGS`, *module* is a literal string that the + :envvar:`PYTHONWARNINGS`, if *module* starts and ends with a forward slash + (``/``), it specifies a regular expression as above; + otherwise it is a literal string that the fully qualified module name must be equal to (case-sensitively), ignoring any whitespace at the start or end of *module*. @@ -458,7 +464,7 @@ Available Functions lower.one_way(**kw) This makes the warning refer to both the ``example.lower.one_way()`` and - ``package.higher.another_way()`` call sites only from calling code living + ``example.higher.another_way()`` call sites only from calling code living outside of ``example`` package. *source*, if supplied, is the destroyed object which emitted a @@ -474,14 +480,28 @@ Available Functions .. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) This is a low-level interface to the functionality of :func:`warn`, passing in - explicitly the message, category, filename and line number, and optionally the - module name and the registry (which should be the ``__warningregistry__`` - dictionary of the module). The module name defaults to the filename with - ``.py`` stripped; if no registry is passed, the warning is never suppressed. + explicitly the message, category, filename and line number, and optionally + other arguments. *message* must be a string and *category* a subclass of :exc:`Warning` or *message* may be a :exc:`Warning` instance, in which case *category* will be ignored. + *module*, if supplied, should be the module name. + If no module is passed, the module regular expression in + :ref:`warnings filter ` will be tested against the module + names constructed from the path components starting from all parent + directories (with ``/__init__.py``, ``.py`` and, on Windows, ``.pyw`` + stripped) and against the filename with ``.py`` stripped. + For example, when the filename is ``'/path/to/package/module.py'``, it will + be tested against ``'path.to.package.module'``, ``'to.package.module'`` + ``'package.module'``, ``'module'``, and ``'/path/to/package/module'``. + + *registry*, if supplied, should be the ``__warningregistry__`` dictionary + of the module. + If no registry is passed, each warning is treated as the first occurrence, + that is, filter actions ``"default"``, ``"module"`` and ``"once"`` are + handled as ``"always"``. + *module_globals*, if supplied, should be the global namespace in use by the code for which the warning is issued. (This argument is used to support displaying source for modules found in zipfiles or other non-filesystem import @@ -493,6 +513,10 @@ Available Functions .. versionchanged:: 3.6 Add the *source* parameter. + .. versionchanged:: 3.15 + If no module is passed, test the filter regular expression against + module names created from the path, not only the path itself. + .. function:: showwarning(message, category, filename, lineno, file=None, line=None) @@ -584,7 +608,7 @@ Available Functions The deprecation message passed to the decorator is saved in the ``__deprecated__`` attribute on the decorated object. If applied to an overload, the decorator - must be after the :func:`@overload ` decorator + must be after the :deco:`~typing.overload` decorator for the attribute to exist on the overload as returned by :func:`typing.get_overloads`. diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 36c2bde87fb..7ff2c97992c 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -25,8 +25,9 @@ The :mod:`wave` module defines the following function and exception: .. function:: open(file, mode=None) - If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: + If *file* is a string, a :term:`path-like object` or a + :term:`bytes-like object` open the file by that name, otherwise treat it as + a file-like object. *mode* can be: ``'rb'`` Read only mode. @@ -52,6 +53,10 @@ The :mod:`wave` module defines the following function and exception: .. versionchanged:: 3.4 Added support for unseekable files. + .. versionchanged:: 3.15 + Added support for :term:`path-like objects ` + and :term:`bytes-like objects `. + .. exception:: Error An error raised when something is impossible because it violates the WAV @@ -123,26 +128,6 @@ Wave_read Objects Rewind the file pointer to the beginning of the audio stream. - The following two methods are defined for compatibility with the old :mod:`!aifc` - module, and don't do anything interesting. - - - .. method:: getmarkers() - - Returns ``None``. - - .. deprecated-removed:: 3.13 3.15 - The method only existed for compatibility with the :mod:`!aifc` module - which has been removed in Python 3.13. - - - .. method:: getmark(id) - - Raise an error. - - .. deprecated-removed:: 3.13 3.15 - The method only existed for compatibility with the :mod:`!aifc` module - which has been removed in Python 3.13. The following two methods define a term "position" which is compatible between them, and is otherwise implementation dependent. diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index fd6abc70261..a2103d8fdd8 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -49,6 +49,11 @@ a new tab, with the browser being brought to the foreground. The use of the :mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If :mod:`ctypes` isn't available, calls to :func:`.open` will fail. +.. _webbrowser-cli: + +Command-line interface +---------------------- + .. program:: webbrowser The script :program:`webbrowser` can be used as a command-line interface for the @@ -232,7 +237,7 @@ Here are some simple examples:: .. _browser-controllers: -Browser Controller Objects +Browser controller objects -------------------------- Browser controllers provide the :attr:`~controller.name` attribute, diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index b3a824fb69a..89def6e2afe 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -14,6 +14,8 @@ integer as the registry handle, a :ref:`handle object ` is used to ensure that the handles are closed correctly, even if the programmer neglects to explicitly close them. +.. availability:: Windows. + .. _exception-changed: .. versionchanged:: 3.3 @@ -173,6 +175,24 @@ This module offers the following functions: See :ref:`above `. +.. function:: DeleteTree(key, sub_key=None) + + Deletes the specified key and all its subkeys and values recursively. + + *key* is an already open key, or one of the predefined + :ref:`HKEY_* constants `. + + *sub_key* is a string that names the subkey to delete. If ``None``, + deletes all subkeys and values of the specified key. + + This function deletes a key and all its descendants. If *sub_key* is + ``None``, all subkeys and values of the specified key are deleted. + + .. audit-event:: winreg.DeleteTree key,sub_key winreg.DeleteTree + + .. versionadded:: 3.15 + + .. function:: DeleteValue(key, value) Removes a named value from a registry key. @@ -753,8 +773,9 @@ Handle objects provide semantics for :meth:`~object.__bool__` -- thus :: will print ``Yes`` if the handle is currently valid (has not been closed or detached). -The object also support comparison semantics, so handle objects will compare -true if they both reference the same underlying Windows handle value. +The object also support equality comparison semantics, so handle objects will +compare equal if they both reference the same underlying Windows handle value. +Closed handle objects (those with a handle value of zero) always compare equal. Handle objects can be converted to an integer (e.g., using the built-in :func:`int` function), in which case the underlying Windows handle value is @@ -797,3 +818,6 @@ integer handle, and also disconnect the Windows handle from the handle object. will automatically close *key* when control leaves the :keyword:`with` block. +.. versionchanged:: 3.15 + Handle objects are now compared by their underlying Windows handle value + instead of object identity for equality comparisons. diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 925984c3cdb..93c0c025982 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -13,6 +13,8 @@ The :mod:`winsound` module provides access to the basic sound-playing machinery provided by Windows platforms. It includes functions and several constants. +.. availability:: Windows. + .. function:: Beep(frequency, duration) diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 00a18751207..9ffedf7366a 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -19,11 +19,10 @@ not already proficient with the DOM should consider using the :mod:`xml.etree.ElementTree` module for their XML processing instead. -.. warning:: +.. note:: - The :mod:`xml.dom.minidom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. DOM applications typically start by parsing some XML into a DOM. With diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index fd96765cbe3..a21cfaa4645 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -19,11 +19,10 @@ responsible for explicitly pulling events from the stream, looping over those events until either processing is finished or an error condition occurs. -.. warning:: +.. note:: - The :mod:`xml.dom.pulldom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. versionchanged:: 3.7.1 @@ -75,7 +74,7 @@ given point) or to make use of the :func:`DOMEventStream.expandNode` method and switch to DOM-related processing. -.. class:: PullDom(documentFactory=None) +.. class:: PullDOM(documentFactory=None) Subclass of :class:`xml.sax.handler.ContentHandler`. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 1daf6628013..e59759683a6 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -20,11 +20,10 @@ for parsing and creating XML data. The :mod:`!xml.etree.cElementTree` module is deprecated. -.. warning:: +.. note:: - The :mod:`xml.etree.ElementTree` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. Tutorial -------- @@ -657,6 +656,10 @@ Functions .. versionchanged:: 3.13 Added the :meth:`!close` method. + .. versionchanged:: 3.15 + A :exc:`ResourceWarning` is now emitted if the iterator opened a file + and is not explicitly closed. + .. function:: parse(source, parser=None) @@ -1399,10 +1402,10 @@ XMLParser Objects Disabling reparse deferral has security consequences; please see :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. - Note that :meth:`flush` has been backported to some prior releases of - CPython as a security fix. Check for availability of :meth:`flush` - using :func:`hasattr` if used in code running across a variety of Python - versions. + :meth:`!flush` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. .. versionadded:: 3.13 @@ -1477,10 +1480,10 @@ XMLPullParser Objects Disabling reparse deferral has security consequences; please see :meth:`xml.parsers.expat.xmlparser.SetReparseDeferralEnabled` for details. - Note that :meth:`flush` has been backported to some prior releases of - CPython as a security fix. Check for availability of :meth:`flush` - using :func:`hasattr` if used in code running across a variety of Python - versions. + :meth:`!flush` + has been backported to some prior releases of CPython as a security fix. + Check for availability using :func:`hasattr` if used in code running + across a variety of Python versions. .. versionadded:: 3.13 diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index d4959953989..acd8d399fe3 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -15,12 +15,10 @@ XML Processing Modules Python's interfaces for processing XML are grouped in the ``xml`` package. -.. warning:: +.. note:: - The XML modules are not secure against erroneous or maliciously - constructed data. If you need to parse untrusted or - unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defusedxml-package` sections. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. It is important to note that modules in the :mod:`xml` package require that there be at least one SAX-compliant XML parser available. The Expat parser is @@ -47,46 +45,33 @@ The XML handling submodules are: * :mod:`xml.parsers.expat`: the Expat parser binding +.. _xml-security: .. _xml-vulnerabilities: -XML vulnerabilities -------------------- +XML security +------------ -The XML processing modules are not secure against maliciously constructed data. An attacker can abuse XML features to carry out denial of service attacks, access local files, generate network connections to other machines, or -circumvent firewalls. +circumvent firewalls when attacker-controlled XML is being parsed, +in Python or elsewhere. -The following table gives an overview of the known attacks and whether -the various modules are vulnerable to them. +The built-in XML parsers of Python rely on the library `libexpat`_, commonly +called Expat, for parsing XML. -========================= ================== ================== ================== ================== ================== -kind sax etree minidom pulldom xmlrpc -========================= ================== ================== ================== ================== ================== -billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) -quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) -external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) -`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -large tokens **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) -========================= ================== ================== ================== ================== ================== +By default, Expat itself does not access local files or create network +connections. -1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and - "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to - potential reliance on system-provided libraries. Check - :const:`!pyexpat.EXPAT_VERSION`. -2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a - :exc:`~xml.etree.ElementTree.ParseError` when an entity occurs. -3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns - the unexpanded entity verbatim. -4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. -5. Since Python 3.7.1, external general entities are no longer processed by - default. -6. Expat 2.6.0 and newer is not vulnerable to denial of service - through quadratic runtime caused by parsing large tokens. - Items still listed as vulnerable due to - potential reliance on system-provided libraries. Check - :const:`!pyexpat.EXPAT_VERSION`. +Expat versions lower than 2.7.2 may be vulnerable to the "billion laughs", +"quadratic blowup" and "large tokens" vulnerabilities, or to disproportional +use of dynamic memory. +Python bundles a copy of Expat, and whether Python uses the bundled or a +system-wide Expat, depends on how the Python interpreter +:option:`has been configured <--with-system-expat>` in your environment. +Python may be vulnerable if it uses such older versions of Expat. +Check :const:`!pyexpat.EXPAT_VERSION`. + +:mod:`xmlrpc` is **vulnerable** to the "decompression bomb" attack. billion laughs / exponential entity expansion @@ -103,16 +88,6 @@ quadratic blowup entity expansion efficient as the exponential case but it avoids triggering parser countermeasures that forbid deeply nested entities. -external entity expansion - Entity declarations can contain more than just text for replacement. They can - also point to external resources or local files. The XML - parser accesses the resource and embeds the content into the XML document. - -`DTD`_ retrieval - Some XML libraries like Python's :mod:`xml.dom.pulldom` retrieve document type - definitions from remote or local locations. The feature has similar - implications as the external entity expansion issue. - decompression bomb Decompression bombs (aka `ZIP bomb`_) apply to all XML libraries that can parse compressed XML streams such as gzipped HTTP streams or @@ -126,21 +101,6 @@ large tokens be used to cause denial of service in the application parsing XML. The issue is known as :cve:`2023-52425`. -The documentation for :pypi:`defusedxml` on PyPI has further information about -all known attack vectors with examples and references. - -.. _defusedxml-package: - -The :mod:`!defusedxml` Package ------------------------------- - -:pypi:`defusedxml` is a pure Python package with modified subclasses of all stdlib -XML parsers that prevent any potentially malicious operation. Use of this -package is recommended for any server code that parses untrusted XML data. The -package also ships with example exploits and extended documentation on more -XML exploits such as XPath injection. - - +.. _libexpat: https://github.com/libexpat/libexpat .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb -.. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index c2c9d6424b5..f1af7253e43 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -96,6 +96,14 @@ for the feature and property names. .. data:: feature_external_ges + .. warning:: + + Enabling opens a vulnerability to + `external entity attacks `_ + if the parser is used with user-provided XML content. + Please reflect on your `threat model `_ + before enabling this feature. + | value: ``"http://xml.org/sax/features/external-general-entities"`` | true: Include all external general (text) entities. | false: Do not include external general entities. @@ -248,8 +256,7 @@ events in the input document: The *name* parameter contains the raw XML 1.0 name of the element type as a string and the *attrs* parameter holds an object of the - :class:`~xml.sax.xmlreader.Attributes` - interface (see :ref:`attributes-objects`) containing the attributes of + :ref:`Attributes ` interface containing the attributes of the element. The object passed as *attrs* may be re-used by the parser; holding on to a reference to it is not a reliable way to keep a copy of the attributes. To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* @@ -271,8 +278,7 @@ events in the input document: The *name* parameter contains the name of the element type as a ``(uri, localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in the source document, and the *attrs* parameter holds an instance of the - :class:`~xml.sax.xmlreader.AttributesNS` interface (see - :ref:`attributes-ns-objects`) + :ref:`AttributesNS ` interface containing the attributes of the element. If no namespace is associated with the element, the *uri* component of *name* will be ``None``. The object passed as *attrs* may be re-used by the parser; holding on to a reference to it is not diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index c60e9e505f7..5fa92645a44 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -18,11 +18,10 @@ SAX exceptions and the convenience functions which will be most used by users of the SAX API. -.. warning:: +.. note:: - The :mod:`xml.sax` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. versionchanged:: 3.7.1 diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 5ee11d58c3d..7731f03d875 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -37,7 +37,7 @@ or as base classes. You can unescape other strings of data by passing a dictionary as the optional *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` + replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` are always unescaped, even if *entities* is provided. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 654154cb43d..7e511237a6a 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -24,8 +24,8 @@ between conformable Python objects and XML on the wire. .. warning:: The :mod:`xmlrpc.client` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + constructed data. If you need to parse untrusted or unauthenticated data, + see :ref:`xml-security`. .. versionchanged:: 3.5 @@ -179,9 +179,9 @@ ServerProxy Objects A :class:`ServerProxy` instance has a method corresponding to each remote procedure call accepted by the XML-RPC server. Calling the method performs an RPC, dispatched by both name and argument signature (e.g. the same method name -can be overloaded with multiple argument signatures). The RPC finishes by -returning a value, which may be either returned data in a conformant type or a -:class:`Fault` or :class:`ProtocolError` object indicating an error. +can be overloaded with multiple argument signatures). The RPC finishes either +by returning data in a conformant type or by raising a :class:`Fault` or +:class:`ProtocolError` exception indicating an error. Servers that support the XML introspection API support some common methods grouped under the reserved :attr:`~ServerProxy.system` attribute: @@ -472,7 +472,7 @@ remote server into a single request [#]_. Create an object used to boxcar method calls. *server* is the eventual target of the call. Calls can be made to the result object, but they will immediately - return ``None``, and only store the call name and parameters in the + return ``None``, and only store the call name and arguments in the :class:`MultiCall` object. Calling the object itself causes all stored calls to be transmitted as a single ``system.multicall`` request. The result of this call is a :term:`generator`; iterating over this generator yields the individual @@ -524,7 +524,7 @@ Convenience Functions .. function:: dumps(params, methodname=None, methodresponse=None, encoding=None, allow_none=False) - Convert *params* into an XML-RPC request. or into a response if *methodresponse* + Convert *params* into an XML-RPC request, or into a response if *methodresponse* is true. *params* can be either a tuple of arguments or an instance of the :exc:`Fault` exception class. If *methodresponse* is true, only a single value can be returned, meaning that *params* must be of length 1. *encoding*, if diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 06169c7eca8..2a8f6f8d5fc 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -20,8 +20,8 @@ servers written in Python. Servers can either be free standing, using .. warning:: The :mod:`xmlrpc.server` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + constructed data. If you need to parse untrusted or unauthenticated data, + see :ref:`xml-security`. .. include:: ../includes/wasm-notavail.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 6a4fa67332e..ae4e25b13b9 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -16,13 +16,23 @@ provides tools to create, read, write, append, and list a ZIP file. Any advanced use of this module will require an understanding of the format, as defined in `PKZIP Application Note`_. -This module does not currently handle multi-disk ZIP files. +This module does not handle multipart ZIP files. It can handle ZIP files that use the ZIP64 extensions (that is ZIP files that are more than 4 GiB in size). It supports -decryption of encrypted files in ZIP archives, but it currently cannot +decryption of encrypted files in ZIP archives, but it cannot create an encrypted file. Decryption is extremely slow as it is implemented in native Python rather than C. +.. + The following paragraph should be similar to ../includes/optional-module.rst + +Handling compressed archives requires :term:`optional modules ` +such as :mod:`zlib`, :mod:`bz2`, :mod:`lzma`, and :mod:`compression.zstd`. +If any of them are missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + The module defines the following items: .. exception:: BadZipFile @@ -129,14 +139,28 @@ The module defines the following items: .. versionadded:: 3.3 +.. data:: ZIP_ZSTANDARD + + The numeric constant for Zstandard compression. This requires the + :mod:`compression.zstd` module. + .. note:: - The ZIP file format specification has included support for bzip2 compression - since 2001, and for LZMA compression since 2006. However, some tools - (including older Python releases) do not support these compression - methods, and may either refuse to process the ZIP file altogether, - or fail to extract individual files. + In APPNOTE 6.3.7, the method ID ``20`` was assigned to Zstandard + compression. This was changed in APPNOTE 6.3.8 to method ID ``93`` to + avoid conflicts, with method ID ``20`` being deprecated. For + compatibility, the :mod:`!zipfile` module reads both method IDs but will + only write data with method ID ``93``. + .. versionadded:: 3.14 + +.. note:: + + The ZIP file format specification has included support for bzip2 compression + since 2001, for LZMA compression since 2006, and Zstandard compression since + 2020. However, some tools (including older Python releases) do not support + these compression methods, and may either refuse to process the ZIP file + altogether, or fail to extract individual files. .. seealso:: @@ -151,7 +175,7 @@ The module defines the following items: .. _zipfile-objects: -ZipFile Objects +ZipFile objects --------------- @@ -176,10 +200,11 @@ ZipFile Objects *compression* is the ZIP compression method to use when writing the archive, and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized - values will cause :exc:`NotImplementedError` to be raised. If - :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified - but the corresponding module (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not + :const:`ZIP_BZIP2`, :const:`ZIP_LZMA`, or :const:`ZIP_ZSTANDARD`; + unrecognized values will cause :exc:`NotImplementedError` to be raised. If + :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2`, :const:`ZIP_LZMA`, or + :const:`ZIP_ZSTANDARD` is specified but the corresponding module + (:mod:`zlib`, :mod:`bz2`, :mod:`lzma`, or :mod:`compression.zstd`) is not available, :exc:`RuntimeError` is raised. The default is :const:`ZIP_STORED`. If *allowZip64* is ``True`` (the default) zipfile will create ZIP files that @@ -194,6 +219,10 @@ ZipFile Objects (see :class:`zlib ` for more information). When using :const:`ZIP_BZIP2` integers ``1`` through ``9`` are accepted (see :class:`bz2 ` for more information). + When using :const:`ZIP_ZSTANDARD` integers ``-131072`` through ``22`` are + commonly accepted (see + :attr:`CompressionParameter.compression_level ` + for more on retrieving valid values and their meaning). The *strict_timestamps* argument, when set to ``False``, allows to zip files older than 1980-01-01 at the cost of setting the @@ -219,7 +248,7 @@ ZipFile Objects .. note:: *metadata_encoding* is an instance-wide setting for the ZipFile. - It is not currently possible to set this on a per-member basis. + It is not possible to set this on a per-member basis. This attribute is a workaround for legacy implementations which produce archives with names in the current locale encoding or code page (mostly @@ -415,9 +444,10 @@ ZipFile Objects read or append. *pwd* is the password used for encrypted files as a :class:`bytes` object and, if specified, overrides the default password set with :meth:`setpassword`. Calling :meth:`read` on a ZipFile that uses a compression method other than - :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or - :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also - be raised if the corresponding compression module is not available. + :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2`, + :const:`ZIP_LZMA`, or :const:`ZIP_ZSTANDARD` will raise a + :exc:`NotImplementedError`. An error will also be raised if the + corresponding compression module is not available. .. versionchanged:: 3.6 Calling :meth:`read` on a closed ZipFile will raise a :exc:`ValueError`. @@ -538,18 +568,10 @@ The following data attributes are also available: it should be no longer than 65535 bytes. Comments longer than this will be truncated. -.. attribute:: ZipFile.data_offset - - The offset to the start of ZIP data from the beginning of the file. When the - :class:`ZipFile` is opened in either mode ``'w'`` or ``'x'`` and the - underlying file does not support ``tell()``, the value will be ``None`` - instead. - - .. versionadded:: 3.14 .. _path-objects: -Path Objects +Path objects ------------ .. class:: Path(root, at='') @@ -685,7 +707,7 @@ changes. .. _pyzipfile-objects: -PyZipFile Objects +PyZipFile objects ----------------- The :class:`PyZipFile` constructor takes the same parameters as the @@ -762,7 +784,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. _zipinfo-objects: -ZipInfo Objects +ZipInfo objects --------------- Instances of the :class:`ZipInfo` class are returned by the :meth:`.getinfo` and @@ -818,7 +840,10 @@ Instances have the following methods and attributes: .. attribute:: ZipInfo.date_time The time and date of the last modification to the archive member. This is a - tuple of six values: + tuple of six values representing the "last [modified] file time" and "last [modified] file date" + fields from the ZIP file's central directory. + + The tuple contains: +-------+--------------------------+ | Index | Value | @@ -838,7 +863,15 @@ Instances have the following methods and attributes: .. note:: - The ZIP file format does not support timestamps before 1980. + The ZIP format supports multiple timestamp fields in different locations + (central directory, extra fields for NTFS/UNIX systems, etc.). This attribute + specifically returns the timestamp from the central directory. The central + directory timestamp format in ZIP files does not support timestamps before + 1980. While some extra field formats (such as UNIX timestamps) can represent + earlier dates, this attribute only returns the central directory timestamp. + + The central directory timestamp is interpreted as representing local + time, rather than UTC time, to match the behavior of other zip tools. .. attribute:: ZipInfo.compress_type @@ -921,7 +954,7 @@ Instances have the following methods and attributes: .. _zipfile-commandline: .. program:: zipfile -Command-Line Interface +Command-line interface ---------------------- The :mod:`zipfile` module provides a simple command-line interface to interact @@ -996,7 +1029,7 @@ From file itself Decompression may fail due to incorrect password / CRC checksum / ZIP format or unsupported compression method / decryption. -File System limitations +File system limitations ~~~~~~~~~~~~~~~~~~~~~~~ Exceeding limitations on different file systems can cause decompression failed. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index cd76f29a556..444c3d631a4 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -30,6 +30,9 @@ Any files may be present in the ZIP archive, but importers are only invoked for corresponding :file:`.pyc` file, meaning that if a ZIP archive doesn't contain :file:`.pyc` files, importing may be rather slow. +.. versionchanged:: 3.15 + Zstandard (*zstd*) compressed zip file entries are supported. + .. versionchanged:: 3.13 ZIP64 is supported @@ -142,17 +145,6 @@ zipimporter Objects :exc:`ZipImportError` if the module couldn't be found. - .. method:: load_module(fullname) - - Load the module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. Returns the imported module on success, - raises :exc:`ZipImportError` on failure. - - .. deprecated-removed:: 3.10 3.15 - - Use :meth:`exec_module` instead. - - .. method:: invalidate_caches() Clear out the internal cache of information about files found within @@ -188,17 +180,20 @@ Here is an example that imports a module from a ZIP archive - note that the .. code-block:: shell-session - $ unzip -l example.zip - Archive: example.zip + $ unzip -l example_archive.zip + Archive: example_archive.zip Length Date Time Name -------- ---- ---- ---- - 8467 11-26-02 22:30 jwzthreading.py + 8467 01-01-00 12:30 example.py -------- ------- 8467 1 file - $ ./python - Python 2.3 (#1, Aug 1 2003, 19:54:32) + +.. code-block:: pycon + >>> import sys - >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path - >>> import jwzthreading - >>> jwzthreading.__file__ - 'example.zip/jwzthreading.py' + >>> # Add the archive to the front of the module search path + >>> sys.path.insert(0, 'example_archive.zip') + >>> import example + >>> example.__file__ + 'example_archive.zip/example.py' + diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 75ead3c4cb1..ce0a22b9456 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -8,15 +8,13 @@ -------------- For applications that require data compression, the functions in this module -allow compression and decompression, using the zlib library. The zlib library -has its own home page at https://www.zlib.net. There are known -incompatibilities between the Python module and versions of the zlib library -earlier than 1.1.3; 1.1.3 has a `security vulnerability `_, so we recommend using -1.1.4 or later. +allow compression and decompression, using the `zlib library `_. + +.. include:: ../includes/optional-module.rst zlib's functions have many options and often need to be used in a particular order. This documentation doesn't attempt to cover all of the permutations; -consult the zlib manual at http://www.zlib.net/manual.html for authoritative +consult the `zlib manual `_ for authoritative information. For reading and writing ``.gz`` files see the :mod:`gzip` module. @@ -44,21 +42,34 @@ The available exception and functions in this module are: .. versionchanged:: 3.0 The result is always unsigned. -.. function:: compress(data, /, level=-1, wbits=MAX_WBITS) +.. function:: adler32_combine(adler1, adler2, len2, /) + + Combine two Adler-32 checksums into one. + + Given the Adler-32 checksum *adler1* of a sequence ``A`` and the + Adler-32 checksum *adler2* of a sequence ``B`` of length *len2*, + return the Adler-32 checksum of ``A`` and ``B`` concatenated. + + This function is typically useful to combine Adler-32 checksums + that were concurrently computed. To compute checksums sequentially, use + :func:`adler32` with the running checksum as the ``value`` argument. + + .. versionadded:: 3.15 + +.. function:: compress(data, /, level=Z_DEFAULT_COMPRESSION, wbits=MAX_WBITS) Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) - is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. - The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + See :const:`Z_BEST_SPEED` (``1``), :const:`Z_BEST_COMPRESSION` (``9``), + :const:`Z_NO_COMPRESSION` (``0``), and the default, + :const:`Z_DEFAULT_COMPRESSION` (``-1``) for more information about these values. .. _compress-wbits: The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and trailer is included in the output. It can take several ranges of values, - defaulting to ``15`` (MAX_WBITS): + defaulting to ``15`` (:const:`MAX_WBITS`): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -82,17 +93,15 @@ The available exception and functions in this module are: The *wbits* parameter is now available to set window bits and compression type. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=Z_DEFAULT_COMPRESSION, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, - while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. - ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). - Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression - (currently equivalent to level 6). + See :const:`Z_BEST_SPEED` (``1``), :const:`Z_BEST_COMPRESSION` (``9``), + :const:`Z_NO_COMPRESSION` (``0``), and the default, + :const:`Z_DEFAULT_COMPRESSION` (``-1``) for more information about these values. *method* is the compression algorithm. Currently, the only supported value is :const:`DEFLATED`. @@ -107,7 +116,7 @@ The available exception and functions in this module are: *strategy* is used to tune the compression algorithm. Possible values are :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, - :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). + :const:`Z_RLE` and :const:`Z_FIXED`. *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -136,6 +145,20 @@ The available exception and functions in this module are: .. versionchanged:: 3.0 The result is always unsigned. +.. function:: crc32_combine(crc1, crc2, len2, /) + + Combine two CRC-32 checksums into one. + + Given the CRC-32 checksum *crc1* of a sequence ``A`` and the + CRC-32 checksum *crc2* of a sequence ``B`` of length *len2*, + return the CRC-32 checksum of ``A`` and ``B`` concatenated. + + This function is typically useful to combine CRC-32 checksums + that were concurrently computed. To compute checksums sequentially, use + :func:`crc32` with the running checksum as the ``value`` argument. + + .. versionadded:: 3.15 + .. function:: decompress(data, /, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) Decompresses the bytes in *data*, returning a bytes object containing the @@ -221,7 +244,7 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, - :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK`, or :const:`Z_FINISH`, defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the compressed stream and prevents compressing any more data. After calling :meth:`flush` @@ -312,6 +335,137 @@ Decompression objects support the following methods and attributes: objects. +The following constants are available to configure compression and decompression +behavior: + +.. data:: DEFLATED + + The deflate compression method. + + +.. data:: MAX_WBITS + + The maximum window size, expressed as a power of 2. + For example, if :const:`!MAX_WBITS` is ``15`` it results in a window size + of ``32 KiB``. + + +.. data:: DEF_MEM_LEVEL + + The default memory level for compression objects. + + +.. data:: DEF_BUF_SIZE + + The default buffer size for decompression operations. + + +.. data:: Z_NO_COMPRESSION + + Compression level ``0``; no compression. + + .. versionadded:: 3.6 + + +.. data:: Z_BEST_SPEED + + Compression level ``1``; fastest and produces the least compression. + + +.. data:: Z_BEST_COMPRESSION + + Compression level ``9``; slowest and produces the most compression. + + +.. data:: Z_DEFAULT_COMPRESSION + + Default compression level (``-1``); a compromise between speed and + compression. Currently equivalent to compression level ``6``. + + +.. data:: Z_DEFAULT_STRATEGY + + Default compression strategy, for normal data. + + +.. data:: Z_FILTERED + + Compression strategy for data produced by a filter (or predictor). + + +.. data:: Z_HUFFMAN_ONLY + + Compression strategy that forces Huffman coding only. + + +.. data:: Z_RLE + + Compression strategy that limits match distances to one (run-length encoding). + + This constant is only available if Python was compiled with zlib + 1.2.0.1 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_FIXED + + Compression strategy that prevents the use of dynamic Huffman codes. + + This constant is only available if Python was compiled with zlib + 1.2.2.2 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_NO_FLUSH + + Flush mode ``0``. No special flushing behavior. + + .. versionadded:: 3.6 + + +.. data:: Z_PARTIAL_FLUSH + + Flush mode ``1``. Flush as much output as possible. + + +.. data:: Z_SYNC_FLUSH + + Flush mode ``2``. All output is flushed and the output is aligned to a byte boundary. + + +.. data:: Z_FULL_FLUSH + + Flush mode ``3``. All output is flushed and the compression state is reset. + + +.. data:: Z_FINISH + + Flush mode ``4``. All pending input is processed, no more input is expected. + + +.. data:: Z_BLOCK + + Flush mode ``5``. A deflate block is completed and emitted. + + This constant is only available if Python was compiled with zlib + 1.2.2.2 or greater. + + .. versionadded:: 3.6 + + +.. data:: Z_TREES + + Flush mode ``6``, for inflate operations. Instructs inflate to return when + it gets to the next deflate block boundary. + + This constant is only available if Python was compiled with zlib + 1.2.3.4 or greater. + + .. versionadded:: 3.6 + + Information about the version of the zlib library in use is available through the following constants: @@ -347,10 +501,10 @@ the following constants: Module :mod:`gzip` Reading and writing :program:`gzip`\ -format files. - http://www.zlib.net + https://www.zlib.net The zlib library home page. - http://www.zlib.net/manual.html + https://www.zlib.net/manual.html The zlib manual explains the semantics and usage of the library's many functions. diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index a57f3b8b3e8..8147e58d322 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -195,7 +195,7 @@ The ``ZoneInfo`` class The ``ZoneInfo`` class has two alternate constructors: -.. classmethod:: ZoneInfo.from_file(fobj, /, key=None) +.. classmethod:: ZoneInfo.from_file(file_obj, /, key=None) Constructs a ``ZoneInfo`` object from a file-like object returning bytes (e.g. a file opened in binary mode or an :class:`io.BytesIO` object). @@ -299,7 +299,7 @@ The behavior of a ``ZoneInfo`` file depends on how it was constructed: 1. ``ZoneInfo(key)``: When constructed with the primary constructor, a ``ZoneInfo`` object is serialized by key, and when deserialized, the deserializing process uses the primary and thus it is expected that these - are expected to be the same object as other references to the same time + are the same object as other references to the same time zone. For example, if ``europe_berlin_pkl`` is a string containing a pickle constructed from ``ZoneInfo("Europe/Berlin")``, one would expect the following behavior: @@ -325,7 +325,7 @@ The behavior of a ``ZoneInfo`` file depends on how it was constructed: >>> a is b False -3. ``ZoneInfo.from_file(fobj, /, key=None)``: When constructed from a file, the +3. ``ZoneInfo.from_file(file_obj, /, key=None)``: When constructed from a file, the ``ZoneInfo`` object raises an exception on pickling. If an end user wants to pickle a ``ZoneInfo`` constructed from a file, it is recommended that they use a wrapper type or a custom serialization function: either serializing by @@ -349,7 +349,7 @@ Functions This function only includes canonical zone names and does not include "special" zones such as those under the ``posix/`` and ``right/`` - directories, or the ``posixrules`` zone. + directories, the ``posixrules`` or the ``localtime`` zone. .. caution:: diff --git a/Doc/license.rst b/Doc/license.rst index 480414bb84c..269d14cefbc 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1169,3 +1169,33 @@ contributors. The pyzstd code is distributed under the 3-Clause BSD License:: CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Profiling module +---------------- + +The :mod:`!profiling` module includes vendored third-party libraries in +:file:`Lib/profiling/sampling/_vendor/` with the following licenses: + +**d3-flamegraph** + +The d3-flamegraph library is distributed under the Apache License, Version 2.0. +See the OpenSSL section above for the full text of the Apache License Version 2.0. + +**d3.js** + +The d3.js library contains the following notice:: + + Copyright 2010-2021 Mike Bostock + + Permission to use, copy, modify, and/or distribute this software for any purpose + with or without fee is hereby granted, provided that the above copyright notice + and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER + TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF + THIS SOFTWARE. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index f36ed3e122f..6521b4bee50 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -154,15 +154,15 @@ The :keyword:`for` statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: .. productionlist:: python-grammar - for_stmt: "for" `target_list` "in" `starred_list` ":" `suite` + for_stmt: "for" `target_list` "in" `starred_expression_list` ":" `suite` : ["else" ":" `suite`] -The ``starred_list`` expression is evaluated once; it should yield an -:term:`iterable` object. An :term:`iterator` is created for that iterable. -The first item provided -by the iterator is then assigned to the target list using the standard -rules for assignments (see :ref:`assignment`), and the suite is executed. This -repeats for each item provided by the iterator. When the iterator is exhausted, +The :token:`~python-grammar:starred_expression_list` expression is evaluated +once; it should yield an :term:`iterable` object. An :term:`iterator` is +created for that iterable. The first item provided by the iterator is then +assigned to the target list using the standard rules for assignments +(see :ref:`assignment`), and the suite is executed. This repeats for each +item provided by the iterator. When the iterator is exhausted, the suite in the :keyword:`!else` clause, if present, is executed, and the loop terminates. @@ -335,15 +335,29 @@ stored in the :mod:`sys` module is reset to its previous value:: :keyword:`!except*` clause -------------------------- -The :keyword:`!except*` clause(s) are used for handling -:exc:`ExceptionGroup`\s. The exception type for matching is interpreted as in -the case of :keyword:`except`, but in the case of exception groups we can have -partial matches when the type matches some of the exceptions in the group. -This means that multiple :keyword:`!except*` clauses can execute, -each handling part of the exception group. -Each clause executes at most once and handles an exception group -of all matching exceptions. Each exception in the group is handled by at most -one :keyword:`!except*` clause, the first that matches it. :: +The :keyword:`!except*` clause(s) specify one or more handlers for groups of +exceptions (:exc:`BaseExceptionGroup` instances). A :keyword:`try` statement +can have either :keyword:`except` or :keyword:`!except*` clauses, but not both. +The exception type for matching is mandatory in the case of :keyword:`!except*`, +so ``except*:`` is a syntax error. The type is interpreted as in the case of +:keyword:`!except`, but matching is performed on the exceptions contained in the +group that is being handled. An :exc:`TypeError` is raised if a matching +type is a subclass of :exc:`!BaseExceptionGroup`, because that would have +ambiguous semantics. + +When an exception group is raised in the try block, each :keyword:`!except*` +clause splits (see :meth:`~BaseExceptionGroup.split`) it into the subgroups +of matching and non-matching exceptions. If the matching subgroup is not empty, +it becomes the handled exception (the value returned from :func:`sys.exception`) +and assigned to the target of the :keyword:`!except*` clause (if there is one). +Then, the body of the :keyword:`!except*` clause executes. If the non-matching +subgroup is not empty, it is processed by the next :keyword:`!except*` in the +same manner. This continues until all exceptions in the group have been matched, +or the last :keyword:`!except*` clause has run. + +After all :keyword:`!except*` clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from within +:keyword:`!except*` clauses. This merged exception group propagates on.:: >>> try: ... raise ExceptionGroup("eg", @@ -356,22 +370,18 @@ one :keyword:`!except*` clause, the first that matches it. :: caught with nested (TypeError(2),) caught with nested (OSError(3), OSError(4)) + Exception Group Traceback (most recent call last): - | File "", line 2, in - | ExceptionGroup: eg + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: 1 +------------------------------------ - -Any remaining exceptions that were not handled by any :keyword:`!except*` -clause are re-raised at the end, along with all exceptions that were -raised from within the :keyword:`!except*` clauses. If this list contains -more than one exception to reraise, they are combined into an exception -group. - -If the raised exception is not an exception group and its type matches -one of the :keyword:`!except*` clauses, it is caught and wrapped by an -exception group with an empty message string. :: +If the exception raised from the :keyword:`try` block is not an exception group +and its type matches one of the :keyword:`!except*` clauses, it is caught and +wrapped by an exception group with an empty message string. This ensures that the +type of the target ``e`` is consistently :exc:`BaseExceptionGroup`:: >>> try: ... raise BlockingIOError @@ -380,12 +390,6 @@ exception group with an empty message string. :: ... ExceptionGroup('', (BlockingIOError())) -An :keyword:`!except*` clause must have a matching expression; it cannot be ``except*:``. -Furthermore, this expression cannot contain exception group types, because that would -have ambiguous semantics. - -It is not possible to mix :keyword:`except` and :keyword:`!except*` -in the same :keyword:`try`. :keyword:`break`, :keyword:`continue` and :keyword:`return` cannot appear in an :keyword:`!except*` clause. @@ -416,12 +420,14 @@ clauses. -------------------------- If :keyword:`!finally` is present, it specifies a 'cleanup' handler. The -:keyword:`try` clause is executed, including any :keyword:`except` and -:keyword:`else` clauses. If an exception occurs in any of the clauses and is -not handled, the exception is temporarily saved. The :keyword:`!finally` clause -is executed. If there is a saved exception it is re-raised at the end of the -:keyword:`!finally` clause. If the :keyword:`!finally` clause raises another -exception, the saved exception is set as the context of the new exception. +:keyword:`try` clause is executed, including any :keyword:`except` +and :keyword:`else ` clauses. +If an exception occurs in any of the clauses and is not handled, +the exception is temporarily saved. +The :keyword:`!finally` clause is executed. If there is a saved exception +it is re-raised at the end of the :keyword:`!finally` clause. +If the :keyword:`!finally` clause raises another exception, the saved exception +is set as the context of the new exception. If the :keyword:`!finally` clause executes a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement, the saved exception is discarded. For example, this function returns 42. @@ -588,6 +594,7 @@ the items are surrounded by parentheses. For example:: statement. .. _match: +.. _case: The :keyword:`!match` statement =============================== @@ -608,9 +615,9 @@ The match statement is used for pattern matching. Syntax: .. productionlist:: python-grammar match_stmt: 'match' `subject_expr` ":" NEWLINE INDENT `case_block`+ DEDENT - subject_expr: `star_named_expression` "," `star_named_expressions`? - : | `named_expression` - case_block: 'case' `patterns` [`guard`] ":" `block` + subject_expr: `!star_named_expression` "," `!star_named_expressions`? + : | `!named_expression` + case_block: 'case' `patterns` [`guard`] ":" `!block` .. note:: This section uses single quotes to denote @@ -699,7 +706,7 @@ Guards .. index:: ! guard .. productionlist:: python-grammar - guard: "if" `named_expression` + guard: "if" `!named_expression` A ``guard`` (which is part of the ``case``) must succeed for code inside the ``case`` block to execute. It takes the form: :keyword:`if` followed by an @@ -852,8 +859,8 @@ A literal pattern corresponds to most The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are -supported. Raw strings and byte strings are supported. :ref:`f-strings` are -not supported. +supported. Raw strings and byte strings are supported. :ref:`f-strings` +and :ref:`t-strings` are not supported. The forms ``signed_number '+' NUMBER`` and ``signed_number '-' NUMBER`` are for expressing :ref:`complex numbers `; they require a real number @@ -1013,8 +1020,8 @@ subject value: items, as for a fixed-length sequence. .. note:: The length of the subject sequence is obtained via - :func:`len` (i.e. via the :meth:`__len__` protocol). This length may be - cached by the interpreter in a similar manner as + :func:`len` (i.e. via the :meth:`~object.__len__` protocol). + This length may be cached by the interpreter in a similar manner as :ref:`value patterns `. @@ -1065,8 +1072,8 @@ subject value: .. note:: Key-value pairs are matched using the two-argument form of the mapping subject's ``get()`` method. Matched key-value pairs must already be present - in the mapping, and not created on-the-fly via :meth:`__missing__` or - :meth:`~object.__getitem__`. + in the mapping, and not created on-the-fly via :meth:`~object.__missing__` + or :meth:`~object.__getitem__`. In simple terms ``{KEY1: P1, KEY2: P2, ... }`` matches only if all the following happens: @@ -1421,6 +1428,9 @@ is equivalent to :: class Foo(object): pass +There may be one or more base classes; see :ref:`multiple-inheritance` below for more +information. + The class's suite is then executed in a new execution frame (see :ref:`naming`), using a newly created local namespace and the original global namespace. (Usually, the suite contains mostly function definitions.) When the class's @@ -1490,6 +1500,119 @@ can be used to create instance variables with different implementation details. were introduced in :pep:`318`. +.. _multiple-inheritance: + +Multiple inheritance +-------------------- + +Python classes may have multiple base classes, a technique known as +*multiple inheritance*. The base classes are specified in the class definition +by listing them in parentheses after the class name, separated by commas. +For example, the following class definition: + +.. doctest:: + + >>> class A: pass + >>> class B: pass + >>> class C(A, B): pass + +defines a class ``C`` that inherits from classes ``A`` and ``B``. + +The :term:`method resolution order` (MRO) is the order in which base classes are +searched when looking up an attribute on a class. See :ref:`python_2.3_mro` for a +description of how Python determines the MRO for a class. + +Multiple inheritance is not always allowed. Attempting to define a class with multiple +inheritance will raise an error if one of the bases does not allow subclassing, if a consistent MRO +cannot be created, if no valid metaclass can be determined, or if there is an instance +layout conflict. We'll discuss each of these in turn. + +First, all base classes must allow subclassing. While most classes allow subclassing, +some built-in classes do not, such as :class:`bool`: + +.. doctest:: + + >>> class SubBool(bool): # TypeError + ... pass + Traceback (most recent call last): + ... + TypeError: type 'bool' is not an acceptable base type + +In the resolved MRO of a class, the class's bases appear in the order they were +specified in the class's bases list. Additionally, the MRO always lists a child +class before any of its bases. A class definition will fail if it is impossible to +resolve a consistent MRO that satisfies these rules from the list of bases provided: + +.. doctest:: + + >>> class Base: pass + >>> class Child(Base): pass + >>> class Grandchild(Base, Child): pass # TypeError + Traceback (most recent call last): + ... + TypeError: Cannot create a consistent method resolution order (MRO) for bases Base, Child + +In the MRO of ``Grandchild``, ``Base`` must appear before ``Child`` because it is first +in the base class list, but it must also appear after ``Child`` because it is a parent of +``Child``. This is a contradiction, so the class cannot be defined. + +If some of the bases have a custom :term:`metaclass`, the metaclass of the resulting class +is chosen among the metaclasses of the bases and the explicitly specified metaclass of the +child class. It must be a metaclass that is a subclass of +all other candidate metaclasses. If no such metaclass exists among the candidates, +the class cannot be created, as explained in :ref:`metaclass-determination`. + +Finally, the instance layouts of the bases must be compatible. This means that it must be +possible to compute a *solid base* for the class. Exactly which classes are solid bases +depends on the Python implementation. + +.. impl-detail:: + + In CPython, a class is a solid base if it has a + nonempty :attr:`~object.__slots__` definition. + Many but not all classes defined in C are also solid bases, including most + builtins (such as :class:`int` or :class:`BaseException`) + but excluding most concrete :class:`Exception` classes. Generally, a C class + is a solid base if its underlying struct is different in size from its base class. + +Every class has a solid base. :class:`object`, the base class, has itself as its solid base. +If there is a single base, the child class's solid base is that class if it is a solid base, +or else the base class's solid base. If there are multiple bases, we first find the solid base +for each base class to produce a list of candidate solid bases. If there is a unique solid base +that is a subclass of all others, then that class is the solid base. Otherwise, class creation +fails. + +Example: + +.. doctest:: + + >>> class Solid1: + ... __slots__ = ("solid1",) + >>> + >>> class Solid2: + ... __slots__ = ("solid2",) + >>> + >>> class SolidChild(Solid1): + ... __slots__ = ("solid_child",) + >>> + >>> class C1: # solid base is `object` + ... pass + >>> + >>> # OK: solid bases are `Solid1` and `object`, and `Solid1` is a subclass of `object`. + >>> class C2(Solid1, C1): # solid base is `Solid1` + ... pass + >>> + >>> # OK: solid bases are `SolidChild` and `Solid1`, and `SolidChild` is a subclass of `Solid1`. + >>> class C3(SolidChild, Solid1): # solid base is `SolidChild` + ... pass + >>> + >>> # Error: solid bases are `Solid1` and `Solid2`, but neither is a subclass of the other. + >>> class C4(Solid1, Solid2): # error: no single solid base + ... pass + Traceback (most recent call last): + ... + TypeError: multiple bases have instance lay-out conflict + .. _async: Coroutines @@ -1885,7 +2008,7 @@ expressions. The presence of annotations does not change the runtime semantics o the code, except if some mechanism is used that introspects and uses the annotations (such as :mod:`dataclasses` or :func:`functools.singledispatch`). -By default, annotations are lazily evaluated in a :ref:`annotation scope `. +By default, annotations are lazily evaluated in an :ref:`annotation scope `. This means that they are not evaluated when the code containing the annotation is evaluated. Instead, the interpreter saves information that can be used to evaluate the annotation later if requested. The :mod:`annotationlib` module provides tools for evaluating annotations. @@ -1898,6 +2021,12 @@ all annotations are instead stored as strings:: >>> f.__annotations__ {'param': 'annotation'} +This future statement will be deprecated and removed in a future version of Python, +but not before Python 3.13 reaches its end of life (see :pep:`749`). +When it is used, introspection tools like +:func:`annotationlib.get_annotations` and :func:`typing.get_type_hints` are +less likely to be able to resolve annotations at runtime. + .. rubric:: Footnotes diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 005a768f684..5f79c6fe8f5 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -16,9 +16,8 @@ Objects, values and types single: data :dfn:`Objects` are Python's abstraction for data. All data in a Python program -is represented by objects or by relations between objects. (In a sense, and in -conformance to Von Neumann's model of a "stored program computer", code is also -represented by objects.) +is represented by objects or by relations between objects. Even code is +represented by objects. .. index:: pair: built-in function; id @@ -29,9 +28,6 @@ represented by objects.) single: mutable object single: immutable object -.. XXX it *is* now possible in some cases to change an object's - type, under certain controlled conditions - Every object has an identity, a type and a value. An object's *identity* never changes once it has been created; you may think of it as the object's address in memory. The :keyword:`is` operator compares the identity of two objects; the @@ -262,6 +258,8 @@ Booleans (:class:`bool`) a string, the strings ``"False"`` or ``"True"`` are returned, respectively. +.. _datamodel-float: + :class:`numbers.Real` (:class:`float`) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1183,6 +1181,7 @@ Special attributes single: __module__ (class attribute) single: __dict__ (class attribute) single: __bases__ (class attribute) + single: __base__ (class attribute) single: __doc__ (class attribute) single: __annotations__ (class attribute) single: __annotate__ (class attribute) @@ -1217,6 +1216,13 @@ Special attributes In most cases, for a class defined as ``class X(A, B, C)``, ``X.__bases__`` will be exactly equal to ``(A, B, C)``. + * - .. attribute:: type.__base__ + - .. impl-detail:: + + The single base class in the inheritance chain that is responsible + for the memory layout of instances. This attribute corresponds to + :c:member:`~PyTypeObject.tp_base` at the C level. + * - .. attribute:: type.__doc__ - The class's documentation string, or ``None`` if undefined. Not inherited by subclasses. @@ -1228,10 +1234,22 @@ Special attributes :attr:`__annotations__ attributes `. For best practices on working with :attr:`~object.__annotations__`, - please see :mod:`annotationlib`. Where possible, use + please see :mod:`annotationlib`. Use :func:`annotationlib.get_annotations` instead of accessing this attribute directly. + .. warning:: + + Accessing the :attr:`!__annotations__` attribute directly + on a class object may return annotations for the wrong class, specifically + in certain cases where the class, its base class, or a metaclass + is defined under ``from __future__ import annotations``. + See :pep:`749 <749#pep749-metaclasses>` for details. + + This attribute does not exist on certain builtin classes. On + user-defined classes without ``__annotations__``, it is an + empty dictionary. + .. versionchanged:: 3.14 Annotations are now :ref:`lazily evaluated `. See :pep:`649`. @@ -1258,7 +1276,7 @@ Special attributes * - .. attribute:: type.__firstlineno__ - The line number of the first line of the class definition, including decorators. - Setting the :attr:`__module__` attribute removes the + Setting the :attr:`~type.__module__` attribute removes the :attr:`!__firstlineno__` item from the type's dictionary. .. versionadded:: 3.13 @@ -1624,6 +1642,7 @@ and are also passed to registered trace functions. single: f_locals (frame attribute) single: f_lasti (frame attribute) single: f_builtins (frame attribute) + single: f_generator (frame attribute) Special read-only attributes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1661,6 +1680,12 @@ Special read-only attributes (this is an index into the :term:`bytecode` string of the :ref:`code object `) + * - .. attribute:: frame.f_generator + - The :term:`generator` or :term:`coroutine` object that owns this frame, + or ``None`` if the frame is a normal function. + + .. versionadded:: 3.14 + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) @@ -1882,9 +1907,9 @@ falling back to :meth:`~object.__getitem__`). [#]_ When implementing a class that emulates any built-in type, it is important that the emulation only be implemented to the degree that it makes sense for the object being modelled. For example, some sequences may work well with retrieval -of individual elements, but extracting a slice may not make sense. (One example -of this is the :class:`~xml.dom.NodeList` interface in the W3C's Document -Object Model.) +of individual elements, but extracting a slice may not make sense. +(One example of this is the :ref:`NodeList ` interface +in the W3C's Document Object Model.) .. _customization: @@ -2337,6 +2362,9 @@ Customizing module attribute access single: __dir__ (module attribute) single: __class__ (module attribute) +.. method:: module.__getattr__ + module.__dir__ + Special names ``__getattr__`` and ``__dir__`` can be also used to customize access to module attributes. The ``__getattr__`` function at the module level should accept one argument which is the name of an attribute and return the @@ -2350,6 +2378,8 @@ The ``__dir__`` function should accept no arguments, and return an iterable of strings that represents the names accessible on module. If present, this function overrides the standard :func:`dir` search on a module. +.. attribute:: module.__class__ + For a more fine grained customization of the module behavior (setting attributes, properties, etc.), one can set the ``__class__`` attribute of a module object to a subclass of :class:`types.ModuleType`. For example:: @@ -2532,7 +2562,7 @@ instance dictionary. In contrast, non-data descriptors can be overridden by instances. Python methods (including those decorated with -:func:`@staticmethod ` and :func:`@classmethod `) are +:deco:`staticmethod` and :deco:`classmethod`) are implemented as non-data descriptors. Accordingly, instances can redefine and override methods. This allows individual instances to acquire behaviors that differ from other instances of the same class. @@ -2600,8 +2630,8 @@ Notes on using *__slots__*: descriptor directly from the base class). This renders the meaning of the program undefined. In the future, a check may be added to prevent this. -* :exc:`TypeError` will be raised if nonempty *__slots__* are defined for a - class derived from a +* :exc:`TypeError` will be raised if *__slots__* other than *__dict__* and + *__weakref__* are defined for a class derived from a :c:member:`"variable-length" built-in type ` such as :class:`int`, :class:`bytes`, and :class:`tuple`. @@ -2615,7 +2645,7 @@ Notes on using *__slots__*: * :attr:`~object.__class__` assignment works only if both classes have the same *__slots__*. -* :ref:`Multiple inheritance ` with multiple slotted parent +* :ref:`Multiple inheritance ` with multiple slotted parent classes can be used, but only one parent is allowed to have attributes created by slots (the other bases must have empty slot layouts) - violations raise @@ -2626,6 +2656,10 @@ Notes on using *__slots__*: of the iterator's values. However, the *__slots__* attribute will be an empty iterator. +.. versionchanged:: next + Allowed defining the *__dict__* and *__weakref__* *__slots__* for any class. + + .. _class-customization: Customizing class creation @@ -2671,7 +2705,7 @@ class defining the method. .. versionadded:: 3.6 -When a class is created, :meth:`type.__new__` scans the class variables +When a class is created, :meth:`!type.__new__` scans the class variables and makes callbacks to those with a :meth:`~object.__set_name__` hook. .. method:: object.__set_name__(self, owner, name) @@ -2765,6 +2799,8 @@ Resolving MRO entries Core support for typing module and generic types. +.. _metaclass-determination: + Determining the appropriate metaclass ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. index:: @@ -2965,7 +3001,7 @@ class method ``__class_getitem__()``. When defined on a class, ``__class_getitem__()`` is automatically a class method. As such, there is no need for it to be decorated with - :func:`@classmethod` when it is defined. + :deco:`classmethod` when it is defined. The purpose of *__class_getitem__* @@ -3114,16 +3150,20 @@ objects. The :mod:`collections.abc` module provides a :term:`abstract base class` to help create those methods from a base set of :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__delitem__`, and :meth:`!keys`. -Mutable sequences should provide methods :meth:`!append`, :meth:`!count`, -:meth:`!index`, :meth:`!extend`, :meth:`!insert`, :meth:`!pop`, :meth:`!remove`, -:meth:`!reverse` and :meth:`!sort`, like Python standard :class:`list` -objects. Finally, -sequence types should implement addition (meaning concatenation) and + +Mutable sequences should provide methods +:meth:`~sequence.append`, :meth:`~sequence.clear`, :meth:`~sequence.count`, +:meth:`~sequence.extend`, :meth:`~sequence.index`, :meth:`~sequence.insert`, +:meth:`~sequence.pop`, :meth:`~sequence.remove`, and :meth:`~sequence.reverse`, +like Python standard :class:`list` objects. +Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods :meth:`~object.__add__`, :meth:`~object.__radd__`, :meth:`~object.__iadd__`, :meth:`~object.__mul__`, :meth:`~object.__rmul__` and :meth:`~object.__imul__` described below; they should not define other numerical -operators. It is recommended that both mappings and sequences implement the +operators. + +It is recommended that both mappings and sequences implement the :meth:`~object.__contains__` method to allow efficient use of the ``in`` operator; for mappings, ``in`` should search the mapping's keys; for sequences, it should diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index cb6c524dd97..639c232571e 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -398,6 +398,192 @@ See also the description of the :keyword:`try` statement in section :ref:`try` and :keyword:`raise` statement in section :ref:`raise`. +.. _execcomponents: + +Runtime Components +================== + +General Computing Model +----------------------- + +Python's execution model does not operate in a vacuum. It runs on +a host machine and through that host's runtime environment, including +its operating system (OS), if there is one. When a program runs, +the conceptual layers of how it runs on the host look something +like this: + + | **host machine** + | **process** (global resources) + | **thread** (runs machine code) + +Each process represents a program running on the host. Think of each +process itself as the data part of its program. Think of the process' +threads as the execution part of the program. This distinction will +be important to understand the conceptual Python runtime. + +The process, as the data part, is the execution context in which the +program runs. It mostly consists of the set of resources assigned to +the program by the host, including memory, signals, file handles, +sockets, and environment variables. + +Processes are isolated and independent from one another. (The same +is true for hosts.) The host manages the process' access to its +assigned resources, in addition to coordinating between processes. + +Each thread represents the actual execution of the program's machine +code, running relative to the resources assigned to the program's +process. It's strictly up to the host how and when that execution +takes place. + +From the point of view of Python, a program always starts with exactly +one thread. However, the program may grow to run in multiple +simultaneous threads. Not all hosts support multiple threads per +process, but most do. Unlike processes, threads in a process are not +isolated and independent from one another. Specifically, all threads +in a process share all of the process' resources. + +The fundamental point of threads is that each one does *run* +independently, at the same time as the others. That may be only +conceptually at the same time ("concurrently") or physically +("in parallel"). Either way, the threads effectively run +at a non-synchronized rate. + +.. note:: + + That non-synchronized rate means none of the process' memory is + guaranteed to stay consistent for the code running in any given + thread. Thus multi-threaded programs must take care to coordinate + access to intentionally shared resources. Likewise, they must take + care to be absolutely diligent about not accessing any *other* + resources in multiple threads; otherwise two threads running at the + same time might accidentally interfere with each other's use of some + shared data. All this is true for both Python programs and the + Python runtime. + + The cost of this broad, unstructured requirement is the tradeoff for + the kind of raw concurrency that threads provide. The alternative + to the required discipline generally means dealing with + non-deterministic bugs and data corruption. + +Python Runtime Model +-------------------- + +The same conceptual layers apply to each Python program, with some +extra data layers specific to Python: + + | **host machine** + | **process** (global resources) + | Python global runtime (*state*) + | Python interpreter (*state*) + | **thread** (runs Python bytecode and "C-API") + | Python thread *state* + +At the conceptual level: when a Python program starts, it looks exactly +like that diagram, with one of each. The runtime may grow to include +multiple interpreters, and each interpreter may grow to include +multiple thread states. + +.. note:: + + A Python implementation won't necessarily implement the runtime + layers distinctly or even concretely. The only exception is places + where distinct layers are directly specified or exposed to users, + like through the :mod:`threading` module. + +.. note:: + + The initial interpreter is typically called the "main" interpreter. + Some Python implementations, like CPython, assign special roles + to the main interpreter. + + Likewise, the host thread where the runtime was initialized is known + as the "main" thread. It may be different from the process' initial + thread, though they are often the same. In some cases "main thread" + may be even more specific and refer to the initial thread state. + A Python runtime might assign specific responsibilities + to the main thread, such as handling signals. + +As a whole, the Python runtime consists of the global runtime state, +interpreters, and thread states. The runtime ensures all that state +stays consistent over its lifetime, particularly when used with +multiple host threads. + +The global runtime, at the conceptual level, is just a set of +interpreters. While those interpreters are otherwise isolated and +independent from one another, they may share some data or other +resources. The runtime is responsible for managing these global +resources safely. The actual nature and management of these resources +is implementation-specific. Ultimately, the external utility of the +global runtime is limited to managing interpreters. + +In contrast, an "interpreter" is conceptually what we would normally +think of as the (full-featured) "Python runtime". When machine code +executing in a host thread interacts with the Python runtime, it calls +into Python in the context of a specific interpreter. + +.. note:: + + The term "interpreter" here is not the same as the "bytecode + interpreter", which is what regularly runs in threads, executing + compiled Python code. + + In an ideal world, "Python runtime" would refer to what we currently + call "interpreter". However, it's been called "interpreter" at least + since introduced in 1997 (`CPython:a027efa5b`_). + + .. _CPython:a027efa5b: https://github.com/python/cpython/commit/a027efa5b + +Each interpreter completely encapsulates all of the non-process-global, +non-thread-specific state needed for the Python runtime to work. +Notably, the interpreter's state persists between uses. It includes +fundamental data like :data:`sys.modules`. The runtime ensures +multiple threads using the same interpreter will safely +share it between them. + +A Python implementation may support using multiple interpreters at the +same time in the same process. They are independent and isolated from +one another. For example, each interpreter has its own +:data:`sys.modules`. + +For thread-specific runtime state, each interpreter has a set of thread +states, which it manages, in the same way the global runtime contains +a set of interpreters. It can have thread states for as many host +threads as it needs. It may even have multiple thread states for +the same host thread, though that isn't as common. + +Each thread state, conceptually, has all the thread-specific runtime +data an interpreter needs to operate in one host thread. The thread +state includes the current raised exception and the thread's Python +call stack. It may include other thread-specific resources. + +.. note:: + + The term "Python thread" can sometimes refer to a thread state, but + normally it means a thread created using the :mod:`threading` module. + +Each thread state, over its lifetime, is always tied to exactly one +interpreter and exactly one host thread. It will only ever be used in +that thread and with that interpreter. + +Multiple thread states may be tied to the same host thread, whether for +different interpreters or even the same interpreter. However, for any +given host thread, only one of the thread states tied to it can be used +by the thread at a time. + +Thread states are isolated and independent from one another and don't +share any data, except for possibly sharing an interpreter and objects +or other resources belonging to that interpreter. + +Once a program is running, new Python threads can be created using the +:mod:`threading` module (on platforms and Python implementations that +support threads). Additional processes can be created using the +:mod:`os`, :mod:`subprocess`, and :mod:`multiprocessing` modules. +Interpreters can be created and used with the +:mod:`~concurrent.interpreters` module. Coroutines (async) can +be run using :mod:`asyncio` in each interpreter, typically only +in a single thread (often the main thread). + + .. rubric:: Footnotes .. [#] This limitation occurs because the code that is executed by these operations diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 8837344e5dd..165dfa69f88 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -133,14 +133,18 @@ Literals Python supports string and bytes literals and various numeric literals: -.. productionlist:: python-grammar - literal: `stringliteral` | `bytesliteral` - : | `integer` | `floatnumber` | `imagnumber` +.. grammar-snippet:: + :group: python-grammar + + literal: `strings` | `NUMBER` Evaluation of a literal yields an object of the given type (string, bytes, integer, floating-point number, complex number) with the given value. The value may be approximated in the case of floating-point and imaginary (complex) -literals. See section :ref:`literals` for details. +literals. +See section :ref:`literals` for details. +See section :ref:`string-concatenation` for details on ``strings``. + .. index:: triple: immutable; data; type @@ -153,6 +157,58 @@ occurrence) may obtain the same object or a different object with the same value. +.. _string-concatenation: + +String literal concatenation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Multiple adjacent string or bytes literals (delimited by whitespace), possibly +using different quoting conventions, are allowed, and their meaning is the same +as their concatenation:: + + >>> "hello" 'world' + "helloworld" + +Formally: + +.. grammar-snippet:: + :group: python-grammar + + strings: ( `STRING` | `fstring`)+ | `tstring`+ + +This feature is defined at the syntactical level, so it only works with literals. +To concatenate string expressions at run time, the '+' operator may be used:: + + >>> greeting = "Hello" + >>> space = " " + >>> name = "Blaise" + >>> print(greeting + space + name) # not: print(greeting space name) + Hello Blaise + +Literal concatenation can freely mix raw strings, triple-quoted strings, +and formatted string literals. +For example:: + + >>> "Hello" r', ' f"{name}!" + "Hello, Blaise!" + +This feature can be used to reduce the number of backslashes +needed, to split long strings conveniently across long lines, or even to add +comments to parts of strings. For example:: + + re.compile("[A-Za-z_]" # letter or underscore + "[A-Za-z0-9_]*" # letter, digit or underscore + ) + +However, bytes literals may only be combined with other byte literals; +not with string literals of any kind. +Also, template string literals may only be combined with other template +string literals:: + + >>> t"Hello" t"{name}!" + Template(strings=('Hello', '!'), interpolations=(...)) + + .. _parenthesized: Parenthesized forms @@ -406,8 +462,9 @@ brackets or curly braces. Variables used in the generator expression are evaluated lazily when the :meth:`~generator.__next__` method is called for the generator object (in the same fashion as normal generators). However, the iterable expression in the -leftmost :keyword:`!for` clause is immediately evaluated, so that an error -produced by it will be emitted at the point where the generator expression +leftmost :keyword:`!for` clause is immediately evaluated, and the +:term:`iterator` is immediately created for that iterable, so that an error +produced while creating the iterator will be emitted at the point where the generator expression is defined, rather than at the point where the first value is retrieved. Subsequent :keyword:`!for` clauses and any filter condition in the leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as they may @@ -625,8 +682,10 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.close() - Raises a :exc:`GeneratorExit` at the point where the generator function was - paused. If the generator function catches the exception and returns a + Raises a :exc:`GeneratorExit` exception at the point where the generator + function was paused (equivalent to calling ``throw(GeneratorExit)``). + The exception is raised by the yield expression where the generator was paused. + If the generator function catches the exception and returns a value, this value is returned from :meth:`close`. If the generator function is already closed, or raises :exc:`GeneratorExit` (by not catching the exception), :meth:`close` returns :const:`None`. If the generator yields a @@ -1023,7 +1082,7 @@ series of :term:`arguments `: : ["," `keywords_arguments`] : | `starred_and_keywords` ["," `keywords_arguments`] : | `keywords_arguments` - positional_arguments: positional_item ("," positional_item)* + positional_arguments: `positional_item` ("," `positional_item`)* positional_item: `assignment_expression` | "*" `expression` starred_and_keywords: ("*" `expression` | `keyword_item`) : ("," "*" `expression` | "," `keyword_item`)* @@ -1879,8 +1938,9 @@ Conditional expressions conditional_expression: `or_test` ["if" `or_test` "else" `expression`] expression: `conditional_expression` | `lambda_expr` -Conditional expressions (sometimes called a "ternary operator") have the lowest -priority of all Python operations. +A conditional expression (sometimes called a "ternary operator") is an +alternative to the if-else statement. As it is an expression, it returns a value +and can appear as a sub-expression. The expression ``x if C else y`` first evaluates the condition, *C* rather than *x*. If *C* is true, *x* is evaluated and its value is returned; otherwise, *y* is @@ -1928,7 +1988,7 @@ Expression lists single: , (comma); expression list .. productionlist:: python-grammar - starred_expression: ["*"] `or_expr` + starred_expression: "*" `or_expr` | `expression` flexible_expression: `assignment_expression` | `starred_expression` flexible_expression_list: `flexible_expression` ("," `flexible_expression`)* [","] starred_expression_list: `starred_expression` ("," `starred_expression`)* [","] diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index b9cca4444c9..1037feb691f 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -8,15 +8,12 @@ used to generate the CPython parser (see :source:`Grammar/python.gram`). The version here omits details related to code generation and error recovery. -The notation is a mixture of `EBNF -`_ -and `PEG `_. -In particular, ``&`` followed by a symbol, token or parenthesized -group indicates a positive lookahead (i.e., is required to match but -not consumed), while ``!`` indicates a negative lookahead (i.e., is -required *not* to match). We use the ``|`` separator to mean PEG's -"ordered choice" (written as ``/`` in traditional PEG grammars). See -:pep:`617` for more details on the grammar's syntax. +The notation used here is the same as in the preceding docs, +and is described in the :ref:`notation ` section, +except for an extra complication: + +* ``~`` ("cut"): commit to the current alternative and fail the rule + even if this fails to parse .. literalinclude:: ../../Grammar/python.gram :language: peg diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index b7b70e6be5a..c62240b18cf 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -90,44 +90,130 @@ Notation .. index:: BNF, grammar, syntax, notation -The descriptions of lexical analysis and syntax use a modified -`Backus–Naur form (BNF) `_ grammar -notation. This uses the following style of definition: +The descriptions of lexical analysis and syntax use a grammar notation that +is a mixture of +`EBNF `_ +and `PEG `_. +For example: -.. productionlist:: notation - name: `lc_letter` (`lc_letter` | "_")* - lc_letter: "a"..."z" +.. grammar-snippet:: + :group: notation -The first line says that a ``name`` is an ``lc_letter`` followed by a sequence -of zero or more ``lc_letter``\ s and underscores. An ``lc_letter`` in turn is -any of the single characters ``'a'`` through ``'z'``. (This rule is actually -adhered to for the names defined in lexical and grammar rules in this document.) + name: `letter` (`letter` | `digit` | "_")* + letter: "a"..."z" | "A"..."Z" + digit: "0"..."9" -Each rule begins with a name (which is the name defined by the rule) and -``::=``. A vertical bar (``|``) is used to separate alternatives; it is the -least binding operator in this notation. A star (``*``) means zero or more -repetitions of the preceding item; likewise, a plus (``+``) means one or more -repetitions, and a phrase enclosed in square brackets (``[ ]``) means zero or -one occurrences (in other words, the enclosed phrase is optional). The ``*`` -and ``+`` operators bind as tightly as possible; parentheses are used for -grouping. Literal strings are enclosed in quotes. White space is only -meaningful to separate tokens. Rules are normally contained on a single line; -rules with many alternatives may be formatted alternatively with each line after -the first beginning with a vertical bar. +In this example, the first line says that a ``name`` is a ``letter`` followed +by a sequence of zero or more ``letter``\ s, ``digit``\ s, and underscores. +A ``letter`` in turn is any of the single characters ``'a'`` through +``'z'`` and ``A`` through ``Z``; a ``digit`` is a single character from ``0`` +to ``9``. -.. index:: lexical definitions, ASCII +Each rule begins with a name (which identifies the rule that's being defined) +followed by a colon, ``:``. +The definition to the right of the colon uses the following syntax elements: -In lexical definitions (as the example above), two more conventions are used: -Two literal characters separated by three dots mean a choice of any single -character in the given (inclusive) range of ASCII characters. A phrase between -angular brackets (``<...>``) gives an informal description of the symbol -defined; e.g., this could be used to describe the notion of 'control character' -if needed. +* ``name``: A name refers to another rule. + Where possible, it is a link to the rule's definition. -Even though the notation used is almost the same, there is a big difference -between the meaning of lexical and syntactic definitions: a lexical definition -operates on the individual characters of the input source, while a syntax -definition operates on the stream of tokens generated by the lexical analysis. -All uses of BNF in the next chapter ("Lexical Analysis") are lexical -definitions; uses in subsequent chapters are syntactic definitions. + * ``TOKEN``: An uppercase name refers to a :term:`token`. + For the purposes of grammar definitions, tokens are the same as rules. +* ``"text"``, ``'text'``: Text in single or double quotes must match literally + (without the quotes). The type of quote is chosen according to the meaning + of ``text``: + + * ``'if'``: A name in single quotes denotes a :ref:`keyword `. + * ``"case"``: A name in double quotes denotes a + :ref:`soft-keyword `. + * ``'@'``: A non-letter symbol in single quotes denotes an + :py:data:`~token.OP` token, that is, a :ref:`delimiter ` or + :ref:`operator `. + +* ``e1 e2``: Items separated only by whitespace denote a sequence. + Here, ``e1`` must be followed by ``e2``. +* ``e1 | e2``: A vertical bar is used to separate alternatives. + It denotes PEG's "ordered choice": if ``e1`` matches, ``e2`` is + not considered. + In traditional PEG grammars, this is written as a slash, ``/``, rather than + a vertical bar. + See :pep:`617` for more background and details. +* ``e*``: A star means zero or more repetitions of the preceding item. +* ``e+``: Likewise, a plus means one or more repetitions. +* ``[e]``: A phrase enclosed in square brackets means zero or + one occurrences. In other words, the enclosed phrase is optional. +* ``e?``: A question mark has exactly the same meaning as square brackets: + the preceding item is optional. +* ``(e)``: Parentheses are used for grouping. + +The following notation is only used in +:ref:`lexical definitions `. + +* ``"a"..."z"``: Two literal characters separated by three dots mean a choice + of any single character in the given (inclusive) range of ASCII characters. +* ``<...>``: A phrase between angular brackets gives an informal description + of the matched symbol (for example, ````), + or an abbreviation that is defined in nearby text (for example, ````). + +.. _lexical-lookaheads: + +Some definitions also use *lookaheads*, which indicate that an element +must (or must not) match at a given position, but without consuming any input: + +* ``&e``: a positive lookahead (that is, ``e`` is required to match) +* ``!e``: a negative lookahead (that is, ``e`` is required *not* to match) + +The unary operators (``*``, ``+``, ``?``) bind as tightly as possible; +the vertical bar (``|``) binds most loosely. + +White space is only meaningful to separate tokens. + +Rules are normally contained on a single line, but rules that are too long +may be wrapped: + +.. grammar-snippet:: + :group: notation + + literal: stringliteral | bytesliteral + | integer | floatnumber | imagnumber + +Alternatively, rules may be formatted with the first line ending at the colon, +and each alternative beginning with a vertical bar on a new line. +For example: + + +.. grammar-snippet:: + :group: notation-alt + + literal: + | stringliteral + | bytesliteral + | integer + | floatnumber + | imagnumber + +This does *not* mean that there is an empty first alternative. + +.. index:: lexical definitions + +.. _notation-lexical-vs-syntactic: + +Lexical and Syntactic definitions +--------------------------------- + +There is some difference between *lexical* and *syntactic* analysis: +the :term:`lexical analyzer` operates on the individual characters of the +input source, while the *parser* (syntactic analyzer) operates on the stream +of :term:`tokens ` generated by the lexical analysis. +However, in some cases the exact boundary between the two phases is a +CPython implementation detail. + +The practical difference between the two is that in *lexical* definitions, +all whitespace is significant. +The lexical analyzer :ref:`discards ` all whitespace that is not +converted to tokens like :data:`token.INDENT` or :data:`~token.NEWLINE`. +*Syntactic* definitions then use these tokens, rather than source characters. + +This documentation uses the same BNF grammar for both styles of definitions. +All uses of BNF in the next chapter (:ref:`lexical`) are lexical definitions; +uses in subsequent chapters are syntactic definitions. diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index ff801a7d4fc..9322d8571f7 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -10,12 +10,76 @@ Lexical analysis A Python program is read by a *parser*. Input to the parser is a stream of :term:`tokens `, generated by the *lexical analyzer* (also known as the *tokenizer*). -This chapter describes how the lexical analyzer breaks a file into tokens. +This chapter describes how the lexical analyzer produces these tokens. -Python reads program text as Unicode code points; the encoding of a source file -can be given by an encoding declaration and defaults to UTF-8, see :pep:`3120` -for details. If the source file cannot be decoded, a :exc:`SyntaxError` is -raised. +The lexical analyzer determines the program text's :ref:`encoding ` +(UTF-8 by default), and decodes the text into +:ref:`source characters `. +If the text cannot be decoded, a :exc:`SyntaxError` is raised. + +Next, the lexical analyzer uses the source characters to generate a stream of tokens. +The type of a generated token generally depends on the next source character to +be processed. Similarly, other special behavior of the analyzer depends on +the first source character that hasn't yet been processed. +The following table gives a quick summary of these source characters, +with links to sections that contain more information. + +.. list-table:: + :header-rows: 1 + + * - Character + - Next token (or other relevant documentation) + + * - * space + * tab + * formfeed + - * :ref:`Whitespace ` + + * - * CR, LF + - * :ref:`New line ` + * :ref:`Indentation ` + + * - * backslash (``\``) + - * :ref:`Explicit line joining ` + * (Also significant in :ref:`string escape sequences `) + + * - * hash (``#``) + - * :ref:`Comment ` + + * - * quote (``'``, ``"``) + - * :ref:`String literal ` + + * - * ASCII letter (``a``-``z``, ``A``-``Z``) + * non-ASCII character + - * :ref:`Name ` + * Prefixed :ref:`string or bytes literal ` + + * - * underscore (``_``) + - * :ref:`Name ` + * (Can also be part of :ref:`numeric literals `) + + * - * number (``0``-``9``) + - * :ref:`Numeric literal ` + + * - * dot (``.``) + - * :ref:`Numeric literal ` + * :ref:`Operator ` + + * - * question mark (``?``) + * dollar (``$``) + * + .. (the following uses zero-width space characters to render + .. a literal backquote) + + backquote (``​`​``) + * control character + - * Error (outside string literals and comments) + + * - * other printing character + - * :ref:`Operator or delimiter ` + + * - * end of file + - * :ref:`End marker ` .. _line-structure: @@ -35,11 +99,12 @@ Logical lines .. index:: logical line, physical line, line joining, NEWLINE token -The end of a logical line is represented by the token NEWLINE. Statements -cannot cross logical line boundaries except where NEWLINE is allowed by the -syntax (e.g., between statements in compound statements). A logical line is -constructed from one or more *physical lines* by following the explicit or -implicit *line joining* rules. +The end of a logical line is represented by the token :data:`~token.NEWLINE`. +Statements cannot cross logical line boundaries except where :data:`!NEWLINE` +is allowed by the syntax (e.g., between statements in compound statements). +A logical line is constructed from one or more *physical lines* by following +the :ref:`explicit ` or :ref:`implicit ` +*line joining* rules. .. _physical-lines: @@ -47,17 +112,30 @@ implicit *line joining* rules. Physical lines -------------- -A physical line is a sequence of characters terminated by an end-of-line -sequence. In source files and strings, any of the standard platform line -termination sequences can be used - the Unix form using ASCII LF (linefeed), -the Windows form using the ASCII sequence CR LF (return followed by linefeed), -or the old Macintosh form using the ASCII CR (return) character. All of these -forms can be used equally, regardless of platform. The end of input also serves -as an implicit terminator for the final physical line. +A physical line is a sequence of characters terminated by one the following +end-of-line sequences: -When embedding Python, source code strings should be passed to Python APIs using -the standard C conventions for newline characters (the ``\n`` character, -representing ASCII LF, is the line terminator). +* the Unix form using ASCII LF (linefeed), +* the Windows form using the ASCII sequence CR LF (return followed by linefeed), +* the '`Classic Mac OS`__' form using the ASCII CR (return) character. + + __ https://en.wikipedia.org/wiki/Classic_Mac_OS + +Regardless of platform, each of these sequences is replaced by a single +ASCII LF (linefeed) character. +(This is done even inside :ref:`string literals `.) +Each line can use any of the sequences; they do not need to be consistent +within a file. + +The end of input also serves as an implicit terminator for the final +physical line. + +Formally: + +.. grammar-snippet:: + :group: python-grammar + + newline: | | .. _comments: @@ -99,13 +177,25 @@ which is recognized by Bram Moolenaar's VIM. If no encoding declaration is found, the default encoding is UTF-8. If the implicit or explicit encoding of a file is UTF-8, an initial UTF-8 byte-order -mark (b'\xef\xbb\xbf') is ignored rather than being a syntax error. +mark (``b'\xef\xbb\xbf'``) is ignored rather than being a syntax error. If an encoding is declared, the encoding name must be recognized by Python (see :ref:`standard-encodings`). The encoding is used for all lexical analysis, including string literals, comments and identifiers. +.. _lexical-source-character: + +All lexical analysis, including string literals, comments +and identifiers, works on Unicode text decoded using the source encoding. +Any Unicode code point, except the NUL control character, can appear in +Python source. + +.. grammar-snippet:: + :group: python-grammar + + source_character: + .. _explicit-joining: @@ -160,11 +250,12 @@ Blank lines .. index:: single: blank line A logical line that contains only spaces, tabs, formfeeds and possibly a -comment, is ignored (i.e., no NEWLINE token is generated). During interactive -input of statements, handling of a blank line may differ depending on the -implementation of the read-eval-print loop. In the standard interactive -interpreter, an entirely blank logical line (i.e. one containing not even -whitespace or a comment) terminates a multi-line statement. +comment, is ignored (i.e., no :data:`~token.NEWLINE` token is generated). +During interactive input of statements, handling of a blank line may differ +depending on the implementation of the read-eval-print loop. +In the standard interactive interpreter, an entirely blank logical line (that +is, one containing not even whitespace or a comment) terminates a multi-line +statement. .. _indentation: @@ -202,19 +293,20 @@ the space count to zero). .. index:: INDENT token, DEDENT token -The indentation levels of consecutive lines are used to generate INDENT and -DEDENT tokens, using a stack, as follows. +The indentation levels of consecutive lines are used to generate +:data:`~token.INDENT` and :data:`~token.DEDENT` tokens, using a stack, +as follows. Before the first line of the file is read, a single zero is pushed on the stack; this will never be popped off again. The numbers pushed on the stack will always be strictly increasing from bottom to top. At the beginning of each logical line, the line's indentation level is compared to the top of the stack. If it is equal, nothing happens. If it is larger, it is pushed on the stack, and -one INDENT token is generated. If it is smaller, it *must* be one of the +one :data:`!INDENT` token is generated. If it is smaller, it *must* be one of the numbers occurring on the stack; all numbers on the stack that are larger are -popped off, and for each number popped off a DEDENT token is generated. At the -end of the file, a DEDENT token is generated for each number remaining on the -stack that is larger than zero. +popped off, and for each number popped off a :data:`!DEDENT` token is generated. +At the end of the file, a :data:`!DEDENT` token is generated for each number +remaining on the stack that is larger than zero. Here is an example of a correctly (though confusingly) indented piece of Python code:: @@ -253,9 +345,27 @@ Whitespace between tokens Except at the beginning of a logical line or in string literals, the whitespace characters space, tab and formfeed can be used interchangeably to separate -tokens. Whitespace is needed between two tokens only if their concatenation -could otherwise be interpreted as a different token (e.g., ab is one token, but -a b is two tokens). +tokens: + +.. grammar-snippet:: + :group: python-grammar + + whitespace: ' ' | tab | formfeed + + +Whitespace is needed between two tokens only if their concatenation +could otherwise be interpreted as a different token. For example, ``ab`` is one +token, but ``a b`` is two tokens. However, ``+a`` and ``+ a`` both produce +two tokens, ``+`` and ``a``, as ``+a`` is not a valid token. + + +.. _endmarker-token: + +End marker +---------- + +At the end of non-interactive input, the lexical analyzer generates an +:data:`~token.ENDMARKER` token. .. _other-tokens: @@ -263,67 +373,50 @@ a b is two tokens). Other tokens ============ -Besides NEWLINE, INDENT and DEDENT, the following categories of tokens exist: -*identifiers*, *keywords*, *literals*, *operators*, and *delimiters*. Whitespace -characters (other than line terminators, discussed earlier) are not tokens, but -serve to delimit tokens. Where ambiguity exists, a token comprises the longest -possible string that forms a legal token, when read from left to right. +Besides :data:`~token.NEWLINE`, :data:`~token.INDENT` and :data:`~token.DEDENT`, +the following categories of tokens exist: +*identifiers* and *keywords* (:data:`~token.NAME`), *literals* (such as +:data:`~token.NUMBER` and :data:`~token.STRING`), and other symbols +(*operators* and *delimiters*, :data:`~token.OP`). +Whitespace characters (other than logical line terminators, discussed earlier) +are not tokens, but serve to delimit tokens. +Where ambiguity exists, a token comprises the longest possible string that +forms a legal token, when read from left to right. .. _identifiers: -Identifiers and keywords -======================== +Names (identifiers and keywords) +================================ .. index:: identifier, name -Identifiers (also referred to as *names*) are described by the following lexical -definitions. +:data:`~token.NAME` tokens represent *identifiers*, *keywords*, and +*soft keywords*. -The syntax of identifiers in Python is based on the Unicode standard annex -UAX-31, with elaboration and changes as defined below; see also :pep:`3131` for -further details. +Names are composed of the following characters: -Within the ASCII range (U+0001..U+007F), the valid characters for identifiers -include the uppercase and lowercase letters ``A`` through -``Z``, the underscore ``_`` and, except for the first character, the digits -``0`` through ``9``. -Python 3.0 introduced additional characters from outside the ASCII range (see -:pep:`3131`). For these characters, the classification uses the version of the -Unicode Character Database as included in the :mod:`unicodedata` module. +* uppercase and lowercase letters (``A-Z`` and ``a-z``), +* the underscore (``_``), +* digits (``0`` through ``9``), which cannot appear as the first character, and +* non-ASCII characters. Valid names may only contain "letter-like" and + "digit-like" characters; see :ref:`lexical-names-nonascii` for details. -Identifiers are unlimited in length. Case is significant. +Names must contain at least one character, but have no upper length limit. +Case is significant. -.. productionlist:: python-grammar - identifier: `xid_start` `xid_continue`* - id_start: - id_continue: - xid_start: - xid_continue: +Formally, names are described by the following lexical definitions: -The Unicode category codes mentioned above stand for: +.. grammar-snippet:: + :group: python-grammar -* *Lu* - uppercase letters -* *Ll* - lowercase letters -* *Lt* - titlecase letters -* *Lm* - modifier letters -* *Lo* - other letters -* *Nl* - letter numbers -* *Mn* - nonspacing marks -* *Mc* - spacing combining marks -* *Nd* - decimal numbers -* *Pc* - connector punctuations -* *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards - compatibility -* *Other_ID_Continue* - likewise + NAME: `name_start` `name_continue`* + name_start: "a"..."z" | "A"..."Z" | "_" | + name_continue: name_start | "0"..."9" + identifier: <`NAME`, except keywords> -All identifiers are converted into the normal form NFKC while parsing; comparison -of identifiers is based on NFKC. - -A non-normative HTML file listing all valid identifier characters for Unicode -16.0.0 can be found at -https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt +Note that not all names matched by this grammar are valid; see +:ref:`lexical-names-nonascii` for details. .. _keywords: @@ -335,7 +428,7 @@ Keywords single: keyword single: reserved word -The following identifiers are used as reserved words, or *keywords* of the +The following names are used as reserved words, or *keywords* of the language, and cannot be used as ordinary identifiers. They must be spelled exactly as written here: @@ -359,18 +452,19 @@ Soft Keywords .. versionadded:: 3.10 -Some identifiers are only reserved under specific contexts. These are known as -*soft keywords*. The identifiers ``match``, ``case``, ``type`` and ``_`` can -syntactically act as keywords in certain contexts, +Some names are only reserved under specific contexts. These are known as +*soft keywords*: + +- ``match``, ``case``, and ``_``, when used in the :keyword:`match` statement. +- ``type``, when used in the :keyword:`type` statement. + +These syntactically act as keywords in their specific contexts, but this distinction is done at the parser level, not when tokenizing. As soft keywords, their use in the grammar is possible while still preserving compatibility with existing code that uses these names as identifier names. -``match``, ``case``, and ``_`` are used in the :keyword:`match` statement. -``type`` is used in the :keyword:`type` statement. - .. versionchanged:: 3.12 ``type`` is now a soft keyword. @@ -425,6 +519,95 @@ characters: :ref:`atom-identifiers`. +.. _lexical-names-nonascii: + +Non-ASCII characters in names +----------------------------- + +Names that contain non-ASCII characters need additional normalization +and validation beyond the rules and grammar explained +:ref:`above `. +For example, ``ř_1``, ``蛇``, or ``साँप`` are valid names, but ``r〰2``, +``€``, or ``🐍`` are not. + +This section explains the exact rules. + +All names are converted into the `normalization form`_ NFKC while parsing. +This means that, for example, some typographic variants of characters are +converted to their "basic" form. For example, ``fiⁿₐˡᵢᶻₐᵗᵢᵒₙ`` normalizes to +``finalization``, so Python treats them as the same name:: + + >>> fiⁿₐˡᵢᶻₐᵗᵢᵒₙ = 3 + >>> finalization + 3 + +.. note:: + + Normalization is done at the lexical level only. + Run-time functions that take names as *strings* generally do not normalize + their arguments. + For example, the variable defined above is accessible at run time in the + :func:`globals` dictionary as ``globals()["finalization"]`` but not + ``globals()["fiⁿₐˡᵢᶻₐᵗᵢᵒₙ"]``. + +Similarly to how ASCII-only names must contain only letters, digits and +the underscore, and cannot start with a digit, a valid name must +start with a character in the "letter-like" set ``xid_start``, +and the remaining characters must be in the "letter- and digit-like" set +``xid_continue``. + +These sets based on the *XID_Start* and *XID_Continue* sets as defined by the +Unicode standard annex `UAX-31`_. +Python's ``xid_start`` additionally includes the underscore (``_``). +Note that Python does not necessarily conform to `UAX-31`_. + +A non-normative listing of characters in the *XID_Start* and *XID_Continue* +sets as defined by Unicode is available in the `DerivedCoreProperties.txt`_ +file in the Unicode Character Database. +For reference, the construction rules for the ``xid_*`` sets are given below. + +The set ``id_start`` is defined as the union of: + +* Unicode category ```` - uppercase letters (includes ``A`` to ``Z``) +* Unicode category ```` - lowercase letters (includes ``a`` to ``z``) +* Unicode category ```` - titlecase letters +* Unicode category ```` - modifier letters +* Unicode category ```` - other letters +* Unicode category ```` - letter numbers +* {``"_"``} - the underscore +* ```` - an explicit set of characters in `PropList.txt`_ + to support backwards compatibility + +The set ``xid_start`` then closes this set under NFKC normalization, by +removing all characters whose normalization is not of the form +``id_start id_continue*``. + +The set ``id_continue`` is defined as the union of: + +* ``id_start`` (see above) +* Unicode category ```` - decimal numbers (includes ``0`` to ``9``) +* Unicode category ```` - connector punctuations +* Unicode category ```` - nonspacing marks +* Unicode category ```` - spacing combining marks +* ```` - another explicit set of characters in + `PropList.txt`_ to support backwards compatibility + +Again, ``xid_continue`` closes this set under NFKC normalization. + +Unicode categories use the version of the Unicode Character Database as +included in the :mod:`unicodedata` module. + +.. _UAX-31: https://www.unicode.org/reports/tr31/ +.. _PropList.txt: https://www.unicode.org/Public/17.0.0/ucd/PropList.txt +.. _DerivedCoreProperties.txt: https://www.unicode.org/Public/17.0.0/ucd/DerivedCoreProperties.txt +.. _normalization form: https://www.unicode.org/reports/tr15/#Norm_Forms + +.. seealso:: + + * :pep:`3131` -- Supporting Non-ASCII Identifiers + * :pep:`672` -- Unicode-related Security Considerations for Python + + .. _literals: Literals @@ -434,79 +617,110 @@ Literals Literals are notations for constant values of some built-in types. +In terms of lexical analysis, Python has :ref:`string, bytes ` +and :ref:`numeric ` literals. + +Other "literals" are lexically denoted using :ref:`keywords ` +(``None``, ``True``, ``False``) and the special +:ref:`ellipsis token ` (``...``). + .. index:: string literal, bytes literal, ASCII single: ' (single quote); string literal single: " (double quote); string literal - single: u'; string literal - single: u"; string literal .. _strings: String and Bytes literals -------------------------- +========================= -String literals are described by the following lexical definitions: +String literals are text enclosed in single quotes (``'``) or double +quotes (``"``). For example: -.. productionlist:: python-grammar - stringliteral: [`stringprefix`](`shortstring` | `longstring`) - stringprefix: "r" | "u" | "R" | "U" | "f" | "F" - : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" - shortstring: "'" `shortstringitem`* "'" | '"' `shortstringitem`* '"' - longstring: "'''" `longstringitem`* "'''" | '"""' `longstringitem`* '"""' - shortstringitem: `shortstringchar` | `stringescapeseq` - longstringitem: `longstringchar` | `stringescapeseq` - shortstringchar: - longstringchar: - stringescapeseq: "\" +.. code-block:: python -.. productionlist:: python-grammar - bytesliteral: `bytesprefix`(`shortbytes` | `longbytes`) - bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" - shortbytes: "'" `shortbytesitem`* "'" | '"' `shortbytesitem`* '"' - longbytes: "'''" `longbytesitem`* "'''" | '"""' `longbytesitem`* '"""' - shortbytesitem: `shortbyteschar` | `bytesescapeseq` - longbytesitem: `longbyteschar` | `bytesescapeseq` - shortbyteschar: - longbyteschar: - bytesescapeseq: "\" + "spam" + 'eggs' -One syntactic restriction not indicated by these productions is that whitespace -is not allowed between the :token:`~python-grammar:stringprefix` or -:token:`~python-grammar:bytesprefix` and the rest of the literal. The source -character set is defined by the encoding declaration; it is UTF-8 if no encoding -declaration is given in the source file; see section :ref:`encodings`. +The quote used to start the literal also terminates it, so a string literal +can only contain the other quote (except with escape sequences, see below). +For example: -.. index:: triple-quoted string, Unicode Consortium, raw string +.. code-block:: python + + 'Say "Hello", please.' + "Don't do that!" + +Except for this limitation, the choice of quote character (``'`` or ``"``) +does not affect how the literal is parsed. + +Inside a string literal, the backslash (``\``) character introduces an +:dfn:`escape sequence`, which has special meaning depending on the character +after the backslash. +For example, ``\"`` denotes the double quote character, and does *not* end +the string: + +.. code-block:: pycon + + >>> print("Say \"Hello\" to everyone!") + Say "Hello" to everyone! + +See :ref:`escape sequences ` below for a full list of such +sequences, and more details. + + +.. index:: triple-quoted string single: """; string literal single: '''; string literal -In plain English: Both types of literals can be enclosed in matching single quotes -(``'``) or double quotes (``"``). They can also be enclosed in matching groups -of three single or double quotes (these are generally referred to as -*triple-quoted strings*). The backslash (``\``) character is used to give special -meaning to otherwise ordinary characters like ``n``, which means 'newline' when -escaped (``\n``). It can also be used to escape characters that otherwise have a -special meaning, such as newline, backslash itself, or the quote character. -See :ref:`escape sequences ` below for examples. +Triple-quoted strings +--------------------- + +Strings can also be enclosed in matching groups of three single or double +quotes. +These are generally referred to as :dfn:`triple-quoted strings`:: + + """This is a triple-quoted string.""" + +In triple-quoted literals, unescaped quotes are allowed (and are +retained), except that three unescaped quotes in a row terminate the literal, +if they are of the same kind (``'`` or ``"``) used at the start:: + + """This string has "quotes" inside.""" + +Unescaped newlines are also allowed and retained:: + + '''This triple-quoted string + continues on the next line.''' + .. index:: - single: b'; bytes literal - single: b"; bytes literal + single: u'; string literal + single: u"; string literal -Bytes literals are always prefixed with ``'b'`` or ``'B'``; they produce an -instance of the :class:`bytes` type instead of the :class:`str` type. They -may only contain ASCII characters; bytes with a numeric value of 128 or greater -must be expressed with escapes. +String prefixes +--------------- -.. index:: - single: r'; raw string literal - single: r"; raw string literal +String literals can have an optional :dfn:`prefix` that influences how the +content of the literal is parsed, for example: -Both string and bytes literals may optionally be prefixed with a letter ``'r'`` -or ``'R'``; such constructs are called :dfn:`raw string literals` -and :dfn:`raw bytes literals` respectively and treat backslashes as -literal characters. As a result, in raw string literals, ``'\U'`` and ``'\u'`` -escapes are not treated specially. +.. code-block:: python + + b"data" + f'{result=}' + +The allowed prefixes are: + +* ``b``: :ref:`Bytes literal ` +* ``r``: :ref:`Raw string ` +* ``f``: :ref:`Formatted string literal ` ("f-string") +* ``t``: :ref:`Template string literal ` ("t-string") +* ``u``: No effect (allowed for backwards compatibility) + +See the linked sections for details on each type. + +Prefixes are case-insensitive (for example, '``B``' works the same as '``b``'). +The '``r``' prefix can be combined with '``f``', '``t``' or '``b``', so '``fr``', +'``rf``', '``tr``', '``rt``', '``br``', and '``rb``' are also valid prefixes. .. versionadded:: 3.3 The ``'rb'`` prefix of raw bytes literals has been added as a synonym @@ -516,18 +730,35 @@ escapes are not treated specially. to simplify the maintenance of dual Python 2.x and 3.x codebases. See :pep:`414` for more information. -.. index:: - single: f'; formatted string literal - single: f"; formatted string literal -A string literal with ``'f'`` or ``'F'`` in its prefix is a -:dfn:`formatted string literal`; see :ref:`f-strings`. The ``'f'`` may be -combined with ``'r'``, but not with ``'b'`` or ``'u'``, therefore raw -formatted strings are possible, but formatted bytes literals are not. +Formal grammar +-------------- -In triple-quoted literals, unescaped newlines and quotes are allowed (and are -retained), except that three unescaped quotes in a row terminate the literal. (A -"quote" is the character used to open the literal, i.e. either ``'`` or ``"``.) +String literals, except :ref:`"f-strings" ` and +:ref:`"t-strings" `, are described by the +following lexical definitions. + +These definitions use :ref:`negative lookaheads ` (``!``) +to indicate that an ending quote ends the literal. + +.. grammar-snippet:: + :group: python-grammar + + STRING: [`stringprefix`] (`stringcontent`) + stringprefix: <("r" | "u" | "b" | "br" | "rb"), case-insensitive> + stringcontent: + | "'''" ( !"'''" `longstringitem`)* "'''" + | '"""' ( !'"""' `longstringitem`)* '"""' + | "'" ( !"'" `stringitem`)* "'" + | '"' ( !'"' `stringitem`)* '"' + stringitem: `stringchar` | `stringescapeseq` + stringchar: + longstringitem: `stringitem` | newline + stringescapeseq: "\" + +Note that as in all lexical definitions, whitespace is significant. +In particular, the prefix (if any) must be immediately followed by the starting +quote. .. index:: physical line, escape sequence, Standard C, C single: \ (backslash); escape sequence @@ -546,120 +777,237 @@ retained), except that three unescaped quotes in a row terminate the literal. ( .. _escape-sequences: - Escape sequences -^^^^^^^^^^^^^^^^ +---------------- -Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and +Unless an '``r``' or '``R``' prefix is present, escape sequences in string and bytes literals are interpreted according to rules similar to those used by Standard C. The recognized escape sequences are: -+-------------------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=========================+=================================+=======+ -| ``\``\ | Backslash and newline ignored | \(1) | -+-------------------------+---------------------------------+-------+ -| ``\\`` | Backslash (``\``) | | -+-------------------------+---------------------------------+-------+ -| ``\'`` | Single quote (``'``) | | -+-------------------------+---------------------------------+-------+ -| ``\"`` | Double quote (``"``) | | -+-------------------------+---------------------------------+-------+ -| ``\a`` | ASCII Bell (BEL) | | -+-------------------------+---------------------------------+-------+ -| ``\b`` | ASCII Backspace (BS) | | -+-------------------------+---------------------------------+-------+ -| ``\f`` | ASCII Formfeed (FF) | | -+-------------------------+---------------------------------+-------+ -| ``\n`` | ASCII Linefeed (LF) | | -+-------------------------+---------------------------------+-------+ -| ``\r`` | ASCII Carriage Return (CR) | | -+-------------------------+---------------------------------+-------+ -| ``\t`` | ASCII Horizontal Tab (TAB) | | -+-------------------------+---------------------------------+-------+ -| ``\v`` | ASCII Vertical Tab (VT) | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\\\{ooo}` | Character with octal value | (2,4) | -| | *ooo* | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\x{hh}` | Character with hex value *hh* | (3,4) | -+-------------------------+---------------------------------+-------+ +.. list-table:: + :widths: auto + :header-rows: 1 -Escape sequences only recognized in string literals are: + * * Escape Sequence + * Meaning + * * ``\``\ + * :ref:`string-escape-ignore` + * * ``\\`` + * :ref:`Backslash ` + * * ``\'`` + * :ref:`Single quote ` + * * ``\"`` + * :ref:`Double quote ` + * * ``\a`` + * ASCII Bell (BEL) + * * ``\b`` + * ASCII Backspace (BS) + * * ``\f`` + * ASCII Formfeed (FF) + * * ``\n`` + * ASCII Linefeed (LF) + * * ``\r`` + * ASCII Carriage Return (CR) + * * ``\t`` + * ASCII Horizontal Tab (TAB) + * * ``\v`` + * ASCII Vertical Tab (VT) + * * :samp:`\\\\{ooo}` + * :ref:`string-escape-oct` + * * :samp:`\\x{hh}` + * :ref:`string-escape-hex` + * * :samp:`\\N\\{{name}\\}` + * :ref:`string-escape-named` + * * :samp:`\\u{xxxx}` + * :ref:`Hexadecimal Unicode character ` + * * :samp:`\\U{xxxxxxxx}` + * :ref:`Hexadecimal Unicode character ` -+-------------------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=========================+=================================+=======+ -| :samp:`\\N\\{{name}\\}` | Character named *name* in the | \(5) | -| | Unicode database | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\u{xxxx}` | Character with 16-bit hex value | \(6) | -| | *xxxx* | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\U{xxxxxxxx}` | Character with 32-bit hex value | \(7) | -| | *xxxxxxxx* | | -+-------------------------+---------------------------------+-------+ +.. _string-escape-ignore: -Notes: +Ignored end of line +^^^^^^^^^^^^^^^^^^^ -(1) - A backslash can be added at the end of a line to ignore the newline:: +A backslash can be added at the end of a line to ignore the newline:: - >>> 'This string will not include \ - ... backslashes or newline characters.' - 'This string will not include backslashes or newline characters.' + >>> 'This string will not include \ + ... backslashes or newline characters.' + 'This string will not include backslashes or newline characters.' - The same result can be achieved using :ref:`triple-quoted strings `, - or parentheses and :ref:`string literal concatenation `. +The same result can be achieved using :ref:`triple-quoted strings `, +or parentheses and :ref:`string literal concatenation `. +.. _string-escape-escaped-char: -(2) - As in Standard C, up to three octal digits are accepted. +Escaped characters +^^^^^^^^^^^^^^^^^^ - .. versionchanged:: 3.11 - Octal escapes with value larger than ``0o377`` produce a - :exc:`DeprecationWarning`. +To include a backslash in a non-:ref:`raw ` Python string +literal, it must be doubled. The ``\\`` escape sequence denotes a single +backslash character:: - .. versionchanged:: 3.12 - Octal escapes with value larger than ``0o377`` produce a - :exc:`SyntaxWarning`. In a future Python version they will be eventually - a :exc:`SyntaxError`. + >>> print('C:\\Program Files') + C:\Program Files -(3) - Unlike in Standard C, exactly two hex digits are required. +Similarly, the ``\'`` and ``\"`` sequences denote the single and double +quote character, respectively:: -(4) - In a bytes literal, hexadecimal and octal escapes denote the byte with the - given value. In a string literal, these escapes denote a Unicode character - with the given value. + >>> print('\' and \"') + ' and " -(5) - .. versionchanged:: 3.3 - Support for name aliases [#]_ has been added. +.. _string-escape-oct: -(6) - Exactly four hex digits are required. +Octal character +^^^^^^^^^^^^^^^ -(7) - Any Unicode character can be encoded this way. Exactly eight hex digits - are required. +The sequence :samp:`\\\\{ooo}` denotes a *character* with the octal (base 8) +value *ooo*:: + + >>> '\120' + 'P' + +Up to three octal digits (0 through 7) are accepted. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given value. + +.. versionchanged:: 3.11 + Octal escapes with value larger than ``0o377`` (255) produce a + :exc:`DeprecationWarning`. + +.. versionchanged:: 3.12 + Octal escapes with value larger than ``0o377`` (255) produce a + :exc:`SyntaxWarning`. + In a future Python version they will raise a :exc:`SyntaxError`. + +.. _string-escape-hex: + +Hexadecimal character +^^^^^^^^^^^^^^^^^^^^^ + +The sequence :samp:`\\x{hh}` denotes a *character* with the hex (base 16) +value *hh*:: + + >>> '\x50' + 'P' + +Unlike in Standard C, exactly two hex digits are required. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given value. + +.. _string-escape-named: + +Named Unicode character +^^^^^^^^^^^^^^^^^^^^^^^ + +The sequence :samp:`\\N\\{{name}\\}` denotes a Unicode character +with the given *name*:: + + >>> '\N{LATIN CAPITAL LETTER P}' + 'P' + >>> '\N{SNAKE}' + '🐍' + +This sequence cannot appear in :ref:`bytes literals `. + +.. versionchanged:: 3.3 + Support for `name aliases `__ + has been added. + +.. _string-escape-long-hex: + +Hexadecimal Unicode characters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These sequences :samp:`\\u{xxxx}` and :samp:`\\U{xxxxxxxx}` denote the +Unicode character with the given hex (base 16) value. +Exactly four digits are required for ``\u``; exactly eight digits are +required for ``\U``. +The latter can encode any Unicode character. + +.. code-block:: pycon + + >>> '\u1234' + 'ሴ' + >>> '\U0001f40d' + '🐍' + +These sequences cannot appear in :ref:`bytes literals `. .. index:: unrecognized escape sequence -Unlike Standard C, all unrecognized escape sequences are left in the string -unchanged, i.e., *the backslash is left in the result*. (This behavior is -useful when debugging: if an escape sequence is mistyped, the resulting output -is more easily recognized as broken.) It is also important to note that the -escape sequences only recognized in string literals fall into the category of -unrecognized escapes for bytes literals. +Unrecognized escape sequences +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Unlike in Standard C, all unrecognized escape sequences are left in the string +unchanged, that is, *the backslash is left in the result*:: + + >>> print('\q') + \q + >>> list('\q') + ['\\', 'q'] + +Note that for bytes literals, the escape sequences only recognized in string +literals (``\N...``, ``\u...``, ``\U...``) fall into the category of +unrecognized escapes. .. versionchanged:: 3.6 Unrecognized escape sequences produce a :exc:`DeprecationWarning`. .. versionchanged:: 3.12 - Unrecognized escape sequences produce a :exc:`SyntaxWarning`. In a future - Python version they will be eventually a :exc:`SyntaxError`. + Unrecognized escape sequences produce a :exc:`SyntaxWarning`. + In a future Python version they will raise a :exc:`SyntaxError`. + + +.. index:: + single: b'; bytes literal + single: b"; bytes literal + + +.. _bytes-literal: + +Bytes literals +-------------- + +:dfn:`Bytes literals` are always prefixed with '``b``' or '``B``'; they produce an +instance of the :class:`bytes` type instead of the :class:`str` type. +They may only contain ASCII characters; bytes with a numeric value of 128 +or greater must be expressed with escape sequences (typically +:ref:`string-escape-hex` or :ref:`string-escape-oct`): + +.. code-block:: pycon + + >>> b'\x89PNG\r\n\x1a\n' + b'\x89PNG\r\n\x1a\n' + >>> list(b'\x89PNG\r\n\x1a\n') + [137, 80, 78, 71, 13, 10, 26, 10] + +Similarly, a zero byte must be expressed using an escape sequence (typically +``\0`` or ``\x00``). + + +.. index:: + single: r'; raw string literal + single: r"; raw string literal + +.. _raw-strings: + +Raw string literals +------------------- + +Both string and bytes literals may optionally be prefixed with a letter '``r``' +or '``R``'; such constructs are called :dfn:`raw string literals` +and :dfn:`raw bytes literals` respectively and treat backslashes as +literal characters. +As a result, in raw string literals, :ref:`escape sequences ` +are not treated specially: + +.. code-block:: pycon + + >>> r'\d{4}-\d{2}-\d{2}' + '\\d{4}-\\d{2}-\\d{2}' Even in a raw literal, quotes can be escaped with a backslash, but the backslash remains in the result; for example, ``r"\""`` is a valid string @@ -671,29 +1019,6 @@ that a single backslash followed by a newline is interpreted as those two characters as part of the literal, *not* as a line continuation. -.. _string-concatenation: - -String literal concatenation ----------------------------- - -Multiple adjacent string or bytes literals (delimited by whitespace), possibly -using different quoting conventions, are allowed, and their meaning is the same -as their concatenation. Thus, ``"hello" 'world'`` is equivalent to -``"helloworld"``. This feature can be used to reduce the number of backslashes -needed, to split long strings conveniently across long lines, or even to add -comments to parts of strings, for example:: - - re.compile("[A-Za-z_]" # letter or underscore - "[A-Za-z0-9_]*" # letter, digit or underscore - ) - -Note that this feature is defined at the syntactical level, but implemented at -compile time. The '+' operator must be used to concatenate string expressions -at run time. Also note that literal concatenation can use different quoting -styles for each component (even mixing raw strings and triple quoted strings), -and formatted string literals may be concatenated with plain string literals. - - .. index:: single: formatted string literal single: interpolated string literal @@ -701,6 +1026,8 @@ and formatted string literals may be concatenated with plain string literals. single: string; interpolated literal single: f-string single: fstring + single: f'; formatted string literal + single: f"; formatted string literal single: {} (curly brackets); in formatted string literal single: ! (exclamation); in formatted string literal single: : (colon); in formatted string literal @@ -713,124 +1040,59 @@ f-strings --------- .. versionadded:: 3.6 +.. versionchanged:: 3.7 + The :keyword:`await` and :keyword:`async for` can be used in expressions + within f-strings. +.. versionchanged:: 3.8 + Added the debug specifier (``=``) +.. versionchanged:: 3.12 + Many restrictions on expressions within f-strings have been removed. + Notably, nested strings, comments, and backslashes are now permitted. A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with ``'f'`` or ``'F'``. These strings may contain -replacement fields, which are expressions delimited by curly braces ``{}``. -While other string literals always have a constant value, formatted strings -are really expressions evaluated at run time. +that is prefixed with '``f``' or '``F``'. +Unlike other string literals, f-strings do not have a constant value. +They may contain *replacement fields* delimited by curly braces ``{}``. +Replacement fields contain expressions which are evaluated at run time. +For example:: -Escape sequences are decoded like in ordinary string literals (except when -a literal is also marked as a raw string). After decoding, the grammar -for the contents of the string is: + >>> who = 'nobody' + >>> nationality = 'Spanish' + >>> f'{who.title()} expects the {nationality} Inquisition!' + 'Nobody expects the Spanish Inquisition!' -.. productionlist:: python-grammar - f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}" - f_expression: (`conditional_expression` | "*" `or_expr`) - : ("," `conditional_expression` | "," "*" `or_expr`)* [","] - : | `yield_expression` - conversion: "s" | "r" | "a" - format_spec: (`literal_char` | `replacement_field`)* - literal_char: +Any doubled curly braces (``{{`` or ``}}``) outside replacement fields +are replaced with the corresponding single curly brace:: -The parts of the string outside curly braces are treated literally, -except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced -with the corresponding single curly brace. A single opening curly -bracket ``'{'`` marks a replacement field, which starts with a -Python expression. To display both the expression text and its value after -evaluation, (useful in debugging), an equal sign ``'='`` may be added after the -expression. A conversion field, introduced by an exclamation point ``'!'`` may -follow. A format specifier may also be appended, introduced by a colon ``':'``. -A replacement field ends with a closing curly bracket ``'}'``. + >>> print(f'{{...}}') + {...} + +Other characters outside replacement fields are treated like in ordinary +string literals. +This means that escape sequences are decoded (except when a literal is +also marked as a raw string), and newlines are possible in triple-quoted +f-strings:: + + >>> name = 'Galahad' + >>> favorite_color = 'blue' + >>> print(f'{name}:\t{favorite_color}') + Galahad: blue + >>> print(rf"C:\Users\{name}") + C:\Users\Galahad + >>> print(f'''Three shall be the number of the counting + ... and the number of the counting shall be three.''') + Three shall be the number of the counting + and the number of the counting shall be three. Expressions in formatted string literals are treated like regular -Python expressions surrounded by parentheses, with a few exceptions. -An empty expression is not allowed, and both :keyword:`lambda` and -assignment expressions ``:=`` must be surrounded by explicit parentheses. +Python expressions. Each expression is evaluated in the context where the formatted string literal -appears, in order from left to right. Replacement expressions can contain -newlines in both single-quoted and triple-quoted f-strings and they can contain -comments. Everything that comes after a ``#`` inside a replacement field -is a comment (even closing braces and quotes). In that case, replacement fields -must be closed in a different line. - -.. code-block:: text - - >>> f"abc{a # This is a comment }" - ... + 3}" - 'abc5' - -.. versionchanged:: 3.7 - Prior to Python 3.7, an :keyword:`await` expression and comprehensions - containing an :keyword:`async for` clause were illegal in the expressions - in formatted string literals due to a problem with the implementation. - -.. versionchanged:: 3.12 - Prior to Python 3.12, comments were not allowed inside f-string replacement - fields. - -When the equal sign ``'='`` is provided, the output will have the expression -text, the ``'='`` and the evaluated value. Spaces after the opening brace -``'{'``, within the expression and after the ``'='`` are all retained in the -output. By default, the ``'='`` causes the :func:`repr` of the expression to be -provided, unless there is a format specified. When a format is specified it -defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is -declared. - -.. versionadded:: 3.8 - The equal sign ``'='``. - -If a conversion is specified, the result of evaluating the expression -is converted before formatting. Conversion ``'!s'`` calls :func:`str` on -the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. - -The result is then formatted using the :func:`format` protocol. The -format specifier is passed to the :meth:`~object.__format__` method of the -expression or conversion result. An empty string is passed when the -format specifier is omitted. The formatted result is then included in -the final value of the whole string. - -Top-level format specifiers may include nested replacement fields. These nested -fields may include their own conversion fields and :ref:`format specifiers -`, but may not include more deeply nested replacement fields. The -:ref:`format specifier mini-language ` is the same as that used by -the :meth:`str.format` method. - -Formatted string literals may be concatenated, but replacement fields -cannot be split across literals. - -Some examples of formatted string literals:: - - >>> name = "Fred" - >>> f"He said his name is {name!r}." - "He said his name is 'Fred'." - >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r - "He said his name is 'Fred'." - >>> width = 10 - >>> precision = 4 - >>> value = decimal.Decimal("12.34567") - >>> f"result: {value:{width}.{precision}}" # nested fields - 'result: 12.35' - >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%B %d, %Y}" # using date format specifier - 'January 27, 2017' - >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging - 'today=January 27, 2017' - >>> number = 1024 - >>> f"{number:#0x}" # using integer format specifier - '0x400' - >>> foo = "bar" - >>> f"{ foo = }" # preserves whitespace - " foo = 'bar'" - >>> line = "The mill's closed" - >>> f"{line = }" - 'line = "The mill\'s closed"' - >>> f"{line = :20}" - "line = The mill's closed " - >>> f"{line = !r:20}" - 'line = "The mill\'s closed" ' +appears, in order from left to right. +An empty expression is not allowed, and both :keyword:`lambda` and +assignment expressions ``:=`` must be surrounded by explicit parentheses:: + >>> f'{(half := 1/2)}, {half * 42}' + '0.5, 21.0' Reusing the outer f-string quoting type inside a replacement field is permitted:: @@ -839,10 +1101,6 @@ permitted:: >>> f"abc {a["x"]} def" 'abc 2 def' -.. versionchanged:: 3.12 - Prior to Python 3.12, reuse of the same quoting type of the outer f-string - inside a replacement field was not possible. - Backslashes are also allowed in replacement fields and are evaluated the same way as in any other context:: @@ -853,39 +1111,214 @@ way as in any other context:: b c -.. versionchanged:: 3.12 - Prior to Python 3.12, backslashes were not permitted inside an f-string - replacement field. +It is possible to nest f-strings:: -Formatted string literals cannot be used as docstrings, even if they do not -include expressions. + >>> name = 'world' + >>> f'Repeated:{f' hello {name}' * 3}' + 'Repeated: hello world hello world hello world' -:: +Portable Python programs should not use more than 5 levels of nesting. + +.. impl-detail:: + + CPython does not limit nesting of f-strings. + +Replacement expressions can contain newlines in both single-quoted and +triple-quoted f-strings and they can contain comments. +Everything that comes after a ``#`` inside a replacement field +is a comment (even closing braces and quotes). +This means that replacement fields with comments must be closed in a +different line: + +.. code-block:: text + + >>> a = 2 + >>> f"abc{a # This comment }" continues until the end of the line + ... + 3}" + 'abc5' + +After the expression, replacement fields may optionally contain: + +* a *debug specifier* -- an equal sign (``=``), optionally surrounded by + whitespace on one or both sides; +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). + +See the :ref:`Standard Library section on f-strings ` +for details on how these fields are evaluated. + +As that section explains, *format specifiers* are passed as the second argument +to the :func:`format` function to format a replacement field value. +For example, they can be used to specify a field width and padding characters +using the :ref:`Format Specification Mini-Language `:: + + >>> number = 14.3 + >>> f'{number:20.7f}' + ' 14.3000000' + +Top-level format specifiers may include nested replacement fields:: + + >>> field_size = 20 + >>> precision = 7 + >>> f'{number:{field_size}.{precision}f}' + ' 14.3000000' + +These nested fields may include their own conversion fields and +:ref:`format specifiers `:: + + >>> number = 3 + >>> f'{number:{field_size}}' + ' 3' + >>> f'{number:{field_size:05}}' + '00000000000000000003' + +However, these nested fields may not include more deeply nested replacement +fields. + +Formatted string literals cannot be used as :term:`docstrings `, +even if they do not include expressions:: >>> def foo(): ... f"Not a docstring" ... - >>> foo.__doc__ is None - True + >>> print(foo.__doc__) + None -See also :pep:`498` for the proposal that added formatted string literals, -and :meth:`str.format`, which uses a related format string mechanism. +.. seealso:: + + * :pep:`498` -- Literal String Interpolation + * :pep:`701` -- Syntactic formalization of f-strings + * :meth:`str.format`, which uses a related format string mechanism. + + +.. _t-strings: +.. _template-string-literals: + +t-strings +--------- + +.. versionadded:: 3.14 + +A :dfn:`template string literal` or :dfn:`t-string` is a string literal +that is prefixed with '``t``' or '``T``'. +These strings follow the same syntax rules as +:ref:`formatted string literals `. +For differences in evaluation rules, see the +:ref:`Standard Library section on t-strings ` + + +Formal grammar for f-strings +---------------------------- + +F-strings are handled partly by the :term:`lexical analyzer`, which produces the +tokens :py:data:`~token.FSTRING_START`, :py:data:`~token.FSTRING_MIDDLE` +and :py:data:`~token.FSTRING_END`, and partly by the parser, which handles +expressions in the replacement field. +The exact way the work is split is a CPython implementation detail. + +Correspondingly, the f-string grammar is a mix of +:ref:`lexical and syntactic definitions `. + +Whitespace is significant in these situations: + +* There may be no whitespace in :py:data:`~token.FSTRING_START` (between + the prefix and quote). +* Whitespace in :py:data:`~token.FSTRING_MIDDLE` is part of the literal + string contents. +* In ``fstring_replacement_field``, if ``f_debug_specifier`` is present, + all whitespace after the opening brace until the ``f_debug_specifier``, + as well as whitespace immediatelly following ``f_debug_specifier``, + is retained as part of the expression. + + .. impl-detail:: + + The expression is not handled in the tokenization phase; it is + retrieved from the source code using locations of the ``{`` token + and the token after ``=``. + + +The ``FSTRING_MIDDLE`` definition uses +:ref:`negative lookaheads ` (``!``) +to indicate special characters (backslash, newline, ``{``, ``}``) and +sequences (``f_quote``). + +.. grammar-snippet:: + :group: python-grammar + + fstring: `FSTRING_START` `fstring_middle`* `FSTRING_END` + + FSTRING_START: `fstringprefix` ("'" | '"' | "'''" | '"""') + FSTRING_END: `f_quote` + fstringprefix: <("f" | "fr" | "rf"), case-insensitive> + f_debug_specifier: '=' + f_quote: + + fstring_middle: + | `fstring_replacement_field` + | `FSTRING_MIDDLE` + FSTRING_MIDDLE: + | (!"\" !`newline` !'{' !'}' !`f_quote`) `source_character` + | `stringescapeseq` + | "{{" + | "}}" + | + fstring_replacement_field: + | '{' `f_expression` [`f_debug_specifier`] [`fstring_conversion`] + [`fstring_full_format_spec`] '}' + fstring_conversion: + | "!" ("s" | "r" | "a") + fstring_full_format_spec: + | ':' `fstring_format_spec`* + fstring_format_spec: + | `FSTRING_MIDDLE` + | `fstring_replacement_field` + f_expression: + | ','.(`conditional_expression` | "*" `or_expr`)+ [","] + | `yield_expression` + +.. note:: + + In the above grammar snippet, the ``f_quote`` and ``FSTRING_MIDDLE`` rules + are context-sensitive -- they depend on the contents of ``FSTRING_START`` + of the nearest enclosing ``fstring``. + + Constructing a more traditional formal grammar from this template is left + as an exercise for the reader. + +The grammar for t-strings is identical to the one for f-strings, with *t* +instead of *f* at the beginning of rule and token names and in the prefix. + +.. grammar-snippet:: + :group: python-grammar + + tstring: TSTRING_START tstring_middle* TSTRING_END + + .. _numbers: Numeric literals ----------------- +================ .. index:: number, numeric literal, integer literal floating-point literal, hexadecimal literal octal literal, binary literal, decimal literal, imaginary literal, complex literal -There are three types of numeric literals: integers, floating-point numbers, and -imaginary numbers. There are no complex literals (complex numbers can be formed -by adding a real number and an imaginary number). +:data:`~token.NUMBER` tokens represent numeric literals, of which there are +three types: integers, floating-point numbers, and imaginary numbers. -Note that numeric literals do not include a sign; a phrase like ``-1`` is +.. grammar-snippet:: + :group: python-grammar + + NUMBER: `integer` | `floatnumber` | `imagnumber` + +The numeric value of a numeric literal is the same as if it were passed as a +string to the :class:`int`, :class:`float` or :class:`complex` class +constructor, respectively. +Note that not all valid inputs for those constructors are also valid literals. + +Numeric literals do not include a sign; a phrase like ``-1`` is actually an expression composed of the unary operator '``-``' and the literal ``1``. @@ -901,36 +1334,65 @@ actually an expression composed of the unary operator '``-``' and the literal Integer literals ---------------- -Integer literals are described by the following lexical definitions: +Integer literals denote whole numbers. For example:: -.. productionlist:: python-grammar - integer: `decinteger` | `bininteger` | `octinteger` | `hexinteger` - decinteger: `nonzerodigit` (["_"] `digit`)* | "0"+ (["_"] "0")* - bininteger: "0" ("b" | "B") (["_"] `bindigit`)+ - octinteger: "0" ("o" | "O") (["_"] `octdigit`)+ - hexinteger: "0" ("x" | "X") (["_"] `hexdigit`)+ - nonzerodigit: "1"..."9" - digit: "0"..."9" - bindigit: "0" | "1" - octdigit: "0"..."7" - hexdigit: `digit` | "a"..."f" | "A"..."F" + 7 + 3 + 2147483647 There is no limit for the length of integer literals apart from what can be -stored in available memory. +stored in available memory:: -Underscores are ignored for determining the numeric value of the literal. They -can be used to group digits for enhanced readability. One underscore can occur -between digits, and after base specifiers like ``0x``. + 7922816251426433759354395033679228162514264337593543950336 -Note that leading zeros in a non-zero decimal number are not allowed. This is -for disambiguation with C-style octal literals, which Python used before version -3.0. +Underscores can be used to group digits for enhanced readability, +and are ignored for determining the numeric value of the literal. +For example, the following literals are equivalent:: -Some examples of integer literals:: + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 - 7 2147483647 0o177 0b100110111 - 3 79228162514264337593543950336 0o377 0xdeadbeef - 100_000_000_000 0b_1110_0101 +Underscores can only occur between digits. +For example, ``_123``, ``321_``, and ``123__321`` are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or hexadecimal +(base 16) using the prefixes ``0b``, ``0o`` and ``0x``, respectively. +Hexadecimal digits 10 through 15 are represented by letters ``A``-``F``, +case-insensitive. For example:: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. +For example, ``0x_1f`` is a valid literal, but ``0_x1f`` and ``0x__1f`` are +not. + +Leading zeros in a non-zero decimal number are not allowed. +For example, ``0123`` is not a valid literal. +This is for disambiguation with C-style octal literals, which Python used +before version 3.0. + +Formally, integer literals are described by the following lexical definitions: + +.. grammar-snippet:: + :group: python-grammar + + integer: `decinteger` | `bininteger` | `octinteger` | `hexinteger` | `zerointeger` + decinteger: `nonzerodigit` (["_"] `digit`)* + bininteger: "0" ("b" | "B") (["_"] `bindigit`)+ + octinteger: "0" ("o" | "O") (["_"] `octdigit`)+ + hexinteger: "0" ("x" | "X") (["_"] `hexdigit`)+ + zerointeger: "0"+ (["_"] "0")* + nonzerodigit: "1"..."9" + digit: "0"..."9" + bindigit: "0" | "1" + octdigit: "0"..."7" + hexdigit: `digit` | "a"..."f" | "A"..."F" .. versionchanged:: 3.6 Underscores are now allowed for grouping purposes in literals. @@ -945,24 +1407,56 @@ Some examples of integer literals:: Floating-point literals ----------------------- -Floating-point literals are described by the following lexical definitions: +Floating-point (float) literals, such as ``3.14`` or ``1.5``, denote +:ref:`approximations of real numbers `. -.. productionlist:: python-grammar - floatnumber: `pointfloat` | `exponentfloat` - pointfloat: [`digitpart`] `fraction` | `digitpart` "." - exponentfloat: (`digitpart` | `pointfloat`) `exponent` +They consist of *integer* and *fraction* parts, each composed of decimal digits. +The parts are separated by a decimal point, ``.``:: + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed. +For example, ``077.010`` is legal, and denotes the same number as ``77.01``. + +As in integer literals, single underscores may occur between digits to help +readability:: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example:: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter ``e`` or ``E``, followed by an optional sign, ``+`` or ``-``, +and a number in the same format as the integer and fraction parts. +The ``e`` or ``E`` represents "times ten raised to the power of":: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may be +omitted:: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + +.. grammar-snippet:: + :group: python-grammar + + floatnumber: + | `digitpart` "." [`digitpart`] [`exponent`] + | "." `digitpart` [`exponent`] + | `digitpart` `exponent` digitpart: `digit` (["_"] `digit`)* - fraction: "." `digitpart` - exponent: ("e" | "E") ["+" | "-"] `digitpart` - -Note that the integer and exponent parts are always interpreted using radix 10. -For example, ``077e010`` is legal, and denotes the same number as ``77e10``. The -allowed range of floating-point literals is implementation-dependent. As in -integer literals, underscores are supported for digit grouping. - -Some examples of floating-point literals:: - - 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 + exponent: ("e" | "E") ["+" | "-"] `digitpart` .. versionchanged:: 3.6 Underscores are now allowed for grouping purposes in literals. @@ -975,73 +1469,118 @@ Some examples of floating-point literals:: Imaginary literals ------------------ -Imaginary literals are described by the following lexical definitions: +Python has :ref:`complex number ` objects, but no complex +literals. +Instead, *imaginary literals* denote complex numbers with a zero +real part. + +For example, in math, the complex number 3+4.2\ *i* is written +as the real number 3 added to the imaginary number 4.2\ *i*. +Python uses a similar syntax, except the imaginary unit is written as ``j`` +rather than *i*:: + + 3+4.2j + +This is an expression composed +of the :ref:`integer literal ` ``3``, +the :ref:`operator ` '``+``', +and the :ref:`imaginary literal ` ``4.2j``. +Since these are three separate tokens, whitespace is allowed between them:: + + 3 + 4.2j + +No whitespace is allowed *within* each token. +In particular, the ``j`` suffix, may not be separated from the number +before it. + +The number before the ``j`` has the same syntax as a floating-point literal. +Thus, the following are valid imaginary literals:: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if the +imaginary number only has an integer part. +The number is still evaluated as a floating-point number, not an integer:: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The ``j`` suffix is case-insensitive. +That means you can use ``J`` instead:: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical definition: + +.. grammar-snippet:: + :group: python-grammar -.. productionlist:: python-grammar imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") -An imaginary literal yields a complex number with a real part of 0.0. Complex -numbers are represented as a pair of floating-point numbers and have the same -restrictions on their range. To create a complex number with a nonzero real -part, add a floating-point number to it, e.g., ``(3+4j)``. Some examples of -imaginary literals:: - - 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j - - -.. _operators: - -Operators -========= - -.. index:: single: operators - -The following tokens are operators: - -.. code-block:: none - - - + - * ** / // % @ - << >> & | ^ ~ := - < > <= >= == != - .. _delimiters: +.. _operators: +.. _lexical-ellipsis: -Delimiters -========== +Operators and delimiters +======================== -.. index:: single: delimiters +.. index:: + single: operators + single: delimiters -The following tokens serve as delimiters in the grammar: +The following grammar defines :dfn:`operator` and :dfn:`delimiter` tokens, +that is, the generic :data:`~token.OP` token type. +A :ref:`list of these tokens and their names ` +is also available in the :mod:`!token` module documentation. -.. code-block:: none +.. grammar-snippet:: + :group: python-grammar - ( ) [ ] { } - , : ! . ; @ = - -> += -= *= /= //= %= - @= &= |= ^= >>= <<= **= + OP: + | assignment_operator + | bitwise_operator + | comparison_operator + | enclosing_delimiter + | other_delimiter + | arithmetic_operator + | "..." + | other_op -The period can also occur in floating-point and imaginary literals. A sequence -of three periods has a special meaning as an ellipsis literal. The second half -of the list, the augmented assignment operators, serve lexically as delimiters, -but also perform an operation. + assignment_operator: "+=" | "-=" | "*=" | "**=" | "/=" | "//=" | "%=" | + "&=" | "|=" | "^=" | "<<=" | ">>=" | "@=" | ":=" + bitwise_operator: "&" | "|" | "^" | "~" | "<<" | ">>" + comparison_operator: "<=" | ">=" | "<" | ">" | "==" | "!=" + enclosing_delimiter: "(" | ")" | "[" | "]" | "{" | "}" + other_delimiter: "," | ":" | "!" | ";" | "=" | "->" + arithmetic_operator: "+" | "-" | "**" | "*" | "//" | "/" | "%" + other_op: "." | "@" -The following printing ASCII characters have special meaning as part of other -tokens or are otherwise significant to the lexical analyzer: +.. note:: -.. code-block:: none + Generally, *operators* are used to combine :ref:`expressions `, + while *delimiters* serve other purposes. + However, there is no clear, formal distinction between the two categories. - ' " # \ + Some tokens can serve as either operators or delimiters, depending on usage. + For example, ``*`` is both the multiplication operator and a delimiter used + for sequence unpacking, and ``@`` is both the matrix multiplication and + a delimiter that introduces decorators. -The following printing ASCII characters are not used in Python. Their -occurrence outside string literals and comments is an unconditional error: + For some tokens, the distinction is unclear. + For example, some people consider ``.``, ``(``, and ``)`` to be delimiters, while others + see the :py:func:`getattr` operator and the function call operator(s). -.. code-block:: none + Some of Python's operators, like ``and``, ``or``, and ``not in``, use + :ref:`keyword ` tokens rather than "symbols" (operator tokens). - $ ? ` +A sequence of three consecutive periods (``...``) has a special +meaning as an :py:data:`Ellipsis` literal. - -.. rubric:: Footnotes - -.. [#] https://www.unicode.org/Public/16.0.0/ucd/NameAliases.txt diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 2a72af4e9a3..9c022570e7e 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -465,8 +465,8 @@ Deletion of a target list recursively deletes each target, from left to right. Deletion of a name removes the binding of that name from the local or global namespace, depending on whether the name occurs in a :keyword:`global` statement -in the same code block. If the name is unbound, a :exc:`NameError` exception -will be raised. +in the same code block. Trying to delete an unbound name raises a +:exc:`NameError` exception. .. index:: pair: attribute; deletion @@ -831,6 +831,9 @@ where the :keyword:`import` statement occurs. .. index:: single: __all__ (optional module attribute) +.. attribute:: module.__all__ + :no-typesetting: + The *public names* defined by a module are determined by checking the module's namespace for a variable named ``__all__``; if defined, it must be a sequence of strings which are names defined or imported by that module. The names @@ -971,10 +974,17 @@ as globals. It would be impossible to assign to a global variable without :keyword:`!global`, although free variables may refer to globals without being declared global. -The :keyword:`global` statement applies to the entire scope of a function or -class body. A :exc:`SyntaxError` is raised if a variable is used or +The :keyword:`!global` statement applies to the entire current scope +(module, function body or class definition). +A :exc:`SyntaxError` is raised if a variable is used or assigned to prior to its global declaration in the scope. +At the module level, all variables are global, so a :keyword:`!global` +statement has no effect. +However, variables must still not be used or +assigned to prior to their :keyword:`!global` declaration. +This requirement is relaxed in the interactive prompt (:term:`REPL`). + .. index:: pair: built-in function; exec pair: built-in function; eval diff --git a/Doc/requirements.txt b/Doc/requirements.txt index a2960ea9aa0..716772b7f28 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,11 +7,11 @@ # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. # Keep this version in sync with ``Doc/conf.py``. -sphinx~=8.2.0 +sphinx~=9.0.0 blurb -sphinxext-opengraph~=0.10.0 +sphinxext-opengraph~=0.13.0 sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 926d2cd42bd..c41c70f0ed3 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -4,22 +4,14 @@ Doc/c-api/descriptor.rst Doc/c-api/float.rst -Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst -Doc/c-api/module.rst Doc/c-api/stable.rst -Doc/c-api/type.rst Doc/c-api/typeobj.rst -Doc/extending/extending.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst -Doc/library/decimal.rst Doc/library/email.charset.rst -Doc/library/email.compat32-message.rst Doc/library/email.parser.rst -Doc/library/exceptions.rst -Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst Doc/library/importlib.rst @@ -31,49 +23,27 @@ Doc/library/multiprocessing.rst Doc/library/optparse.rst Doc/library/os.rst Doc/library/pickletools.rst -Doc/library/platform.rst Doc/library/profile.rst Doc/library/pyexpat.rst -Doc/library/resource.rst Doc/library/select.rst -Doc/library/signal.rst -Doc/library/smtplib.rst Doc/library/socket.rst Doc/library/ssl.rst -Doc/library/stdtypes.rst -Doc/library/subprocess.rst Doc/library/termios.rst Doc/library/test.rst Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst Doc/library/tkinter.ttk.rst Doc/library/unittest.mock.rst -Doc/library/unittest.rst Doc/library/urllib.parse.rst Doc/library/urllib.request.rst Doc/library/wsgiref.rst Doc/library/xml.dom.minidom.rst Doc/library/xml.dom.pulldom.rst Doc/library/xml.dom.rst -Doc/library/xml.sax.handler.rst Doc/library/xml.sax.reader.rst Doc/library/xml.sax.rst Doc/library/xmlrpc.client.rst Doc/library/xmlrpc.server.rst -Doc/library/zlib.rst -Doc/reference/compound_stmts.rst -Doc/reference/datamodel.rst -Doc/using/windows.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst -Doc/whatsnew/2.7.rst -Doc/whatsnew/3.3.rst -Doc/whatsnew/3.4.rst -Doc/whatsnew/3.5.rst -Doc/whatsnew/3.6.rst -Doc/whatsnew/3.7.rst -Doc/whatsnew/3.8.rst -Doc/whatsnew/3.9.rst -Doc/whatsnew/3.10.rst -Doc/whatsnew/3.11.rst diff --git a/Doc/tools/check-epub.py b/Doc/tools/check-epub.py new file mode 100644 index 00000000000..6a10096c117 --- /dev/null +++ b/Doc/tools/check-epub.py @@ -0,0 +1,30 @@ +from pathlib import Path + +CPYTHON_ROOT = Path( + __file__, # cpython/Doc/tools/check-epub.py + '..', # cpython/Doc/tools + '..', # cpython/Doc + '..', # cpython +).resolve() +EPUBCHECK_PATH = CPYTHON_ROOT / 'Doc' / 'epubcheck.txt' + + +def main() -> int: + lines = EPUBCHECK_PATH.read_text(encoding='utf-8').splitlines() + fatal_errors = [line for line in lines if line.startswith('FATAL')] + + if fatal_errors: + err_count = len(fatal_errors) + s = 's' * (err_count != 1) + print() + print(f'Error: epubcheck reported {err_count} fatal error{s}:') + print() + print('\n'.join(fatal_errors)) + return 1 + + print('Success: no fatal errors found.') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index c686eecf8d9..2f2bb9e2dcb 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -15,7 +15,7 @@ from typing import TextIO # Fail if NEWS nit found before this line number -NEWS_NIT_THRESHOLD = 1700 +NEWS_NIT_THRESHOLD = 8550 # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. diff --git a/Doc/tools/extensions/audit_events.py b/Doc/tools/extensions/audit_events.py index 23d82c0f441..385a58b2145 100644 --- a/Doc/tools/extensions/audit_events.py +++ b/Doc/tools/extensions/audit_events.py @@ -13,7 +13,7 @@ from sphinx.util.docutils import SphinxDirective if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Iterator, Set from sphinx.application import Sphinx from sphinx.builders import Builder @@ -33,7 +33,7 @@ class AuditEvents: def __init__(self) -> None: self.events: dict[str, list[str]] = {} - self.sources: dict[str, list[tuple[str, str]]] = {} + self.sources: dict[str, set[tuple[str, str]]] = {} def __iter__(self) -> Iterator[tuple[str, list[str], tuple[str, str]]]: for name, args in self.events.items(): @@ -47,7 +47,7 @@ def add_event( self._check_args_match(name, args) else: self.events[name] = args - self.sources.setdefault(name, []).append(source) + self.sources.setdefault(name, set()).add(source) def _check_args_match(self, name: str, args: list[str]) -> None: current_args = self.events[name] @@ -69,11 +69,11 @@ def _check_args_match(self, name: str, args: list[str]) -> None: return def id_for(self, name) -> str: - source_count = len(self.sources.get(name, ())) + source_count = len(self.sources.get(name, set())) name_clean = re.sub(r"\W", "_", name) return f"audit_event_{name_clean}_{source_count}" - def rows(self) -> Iterator[tuple[str, list[str], list[tuple[str, str]]]]: + def rows(self) -> Iterator[tuple[str, list[str], Set[tuple[str, str]]]]: for name in sorted(self.events.keys()): yield name, self.events[name], self.sources[name] @@ -218,7 +218,7 @@ def _make_row( docname: str, name: str, args: list[str], - sources: list[tuple[str, str]], + sources: Set[tuple[str, str]], ) -> nodes.row: row = nodes.row() name_node = nodes.paragraph("", nodes.Text(name)) @@ -233,7 +233,7 @@ def _make_row( row += nodes.entry("", args_node) backlinks_node = nodes.paragraph() - backlinks = enumerate(sorted(set(sources)), start=1) + backlinks = enumerate(sorted(sources), start=1) for i, (doc, label) in backlinks: if isinstance(label, str): ref = nodes.reference("", f"[{i}]", internal=True) @@ -258,7 +258,7 @@ def setup(app: Sphinx): app.connect("env-purge-doc", audit_events_purge) app.connect("env-merge-info", audit_events_merge) return { - "version": "1.0", + "version": "2.0", "parallel_read_safe": True, "parallel_write_safe": True, } diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 089614a1f6c..e04a5f144c4 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -154,7 +154,10 @@ def add_annotations(app: Sphinx, doctree: nodes.document) -> None: node.insert(0, annotation) -def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: +def _stable_abi_annotation( + record: StableABIEntry, + is_corresponding_slot: bool = False, +) -> nodes.emphasis: """Create the Stable ABI annotation. These have two forms: @@ -168,9 +171,28 @@ def _stable_abi_annotation(record: StableABIEntry) -> nodes.emphasis: ... all of which can have "since version X.Y" appended. """ stable_added = record.added - message = sphinx_gettext("Part of the") - message = message.center(len(message) + 2) - emph_node = nodes.emphasis(message, message, classes=["stableabi"]) + emph_node = nodes.emphasis('', '', classes=["stableabi"]) + if is_corresponding_slot: + # See "Type slot annotations" in add_annotations + ref_node = addnodes.pending_xref( + "slot ID", + refdomain="c", + reftarget="PyType_Slot", + reftype="type", + refexplicit="True", + ) + ref_node += nodes.Text(sphinx_gettext("slot ID")) + + message = sphinx_gettext("The corresponding") + emph_node += nodes.Text(" " + message + " ") + emph_node += ref_node + emph_node += nodes.Text(" ") + emph_node += nodes.literal(record.name, record.name) + message = sphinx_gettext("is part of the") + emph_node += nodes.Text(" " + message + " ") + else: + message = sphinx_gettext("Part of the") + emph_node += nodes.Text(" " + message + " ") ref_node = addnodes.pending_xref( "Stable ABI", refdomain="std", @@ -265,6 +287,51 @@ def run(self) -> list[nodes.Node]: return [node] +class CorrespondingTypeSlot(SphinxDirective): + """Type slot annotations + + Docs for these are with the corresponding field, for example, + "Py_tp_repr" is documented under "PyTypeObject.tp_repr", with + only a stable ABI note mentioning "Py_tp_repr" (and linking to + docs on how this works). + + If there is no corresponding field, these should be documented as normal + macros. + """ + + has_content = False + + required_arguments = 1 + optional_arguments = 0 + + def run(self) -> list[nodes.Node]: + name = self.arguments[0] + state = self.env.domaindata["c_annotations"] + stable_abi_data = state["stable_abi_data"] + + try: + record = stable_abi_data[name] + except LookupError as err: + raise LookupError( + f"{name} is not part of stable ABI. " + + "Document it as `c:macro::` rather than " + + "`corresponding-type-slot::`." + ) from err + + annotation = _stable_abi_annotation(record, is_corresponding_slot=True) + + node = nodes.paragraph() + content = [ + ".. c:namespace:: NULL", + "", + ".. c:macro:: " + name, + " :no-typesetting:", + ] + self.state.nested_parse(StringList(content), 0, node) + node.insert(0, annotation) + return [node] + + def init_annotations(app: Sphinx) -> None: # Using domaindata is a bit hack-ish, # but allows storing state without a global variable or closure. @@ -281,6 +348,7 @@ def setup(app: Sphinx) -> ExtensionMetadata: app.add_config_value("refcount_file", "", "env", types={str}) app.add_config_value("stable_abi_file", "", "env", types={str}) app.add_directive("limited-api-list", LimitedAPIList) + app.add_directive("corresponding-type-slot", CorrespondingTypeSlot) app.connect("builder-inited", init_annotations) app.connect("doctree-read", add_annotations) diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 502b6cd95bc..bcda4b7b435 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -38,7 +38,7 @@ def process_glossary_nodes( rendered = app.builder.render_partial(definition) terms[term.lower()] = { 'title': term, - 'body': rendered['html_body'], + 'body': rendered['fragment'], } diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index fd6b8f95f73..f9bf273e762 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -22,15 +22,7 @@ from sphinx.util.docutils import SphinxDirective # Used in conf.py and updated here by python/release-tools/run_release.py -SOURCE_URI = 'https://github.com/python/cpython/tree/3.14/%s' - -# monkey-patch reST parser to disable alphabetic and roman enumerated lists -from docutils.parsers.rst.states import Body -Body.enum.converters['loweralpha'] = \ - Body.enum.converters['upperalpha'] = \ - Body.enum.converters['lowerroman'] = \ - Body.enum.converters['upperroman'] = lambda x: None - +SOURCE_URI = 'https://github.com/python/cpython/tree/main/%s' class PyAwaitableMixin(object): def handle_signature(self, sig, signode): diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index eb9db9e341b..8e271bca1e0 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -1,11 +1,11 @@ {%- if show_source and has_source and sourcename %}
-

{{ _('This Page') }}

+

{{ _('This page') }}

 diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 4645f7d394e..c78c650b1cb 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -27,12 +27,11 @@ {%- endblock -%} {% block body %} -

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

+

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} -

{% trans %}To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table.{% endtrans %}

+

{% trans %}Download an archive containing all the documentation for this version of Python:{% endtrans %}

@@ -40,50 +39,41 @@

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

- - - - - - - + + - - + + - - + + - +
{% trans %}Packed as .zip{% endtrans %} {% trans %}Packed as .tar.bz2{% endtrans %}
{% trans %}PDF{% endtrans %}{% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %}
{% trans %}HTML{% endtrans %}{% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans %}Download{% endtrans %}{% trans %}Download{% endtrans %}
{% trans %}Plain text{% endtrans %}{% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans %}Download{% endtrans %}{% trans %}Download{% endtrans %}
{% trans %}Texinfo{% endtrans %}{% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans %}Download{% endtrans %}{% trans %}Download{% endtrans %}
{% trans %}EPUB{% endtrans %}{% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %}{% trans %}Download{% endtrans %}
-

{% trans %}These archives contain all the content in the documentation.{% endtrans %}

- - -

{% trans %}Unpacking{% endtrans %}

- -

{% trans %}Unix users should download the .tar.bz2 archives; these are bzipped tar -archives and can be handled in the usual way using tar and the bzip2 -program. The Info-ZIP unzip program can be -used to handle the ZIP archives if desired. The .tar.bz2 archives provide the -best compression and fastest download times.{% endtrans %}

- -

{% trans %}Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the Info-ZIP zip program.{% endtrans %}

+

{% trans %} +We no longer provide pre-built PDFs of the documentation. +To build a PDF archive, follow the instructions in the +Developer's Guide +and run make dist-pdf in the Doc/ directory of a copy of the CPython repository. +{% endtrans %}

+

{% trans %} +See the directory listing +for file sizes.{% endtrans %}

{% trans %}Problems{% endtrans %}

- -

{% trans %}If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.{% endtrans %}

+{% set bugs = pathto('bugs') %} +

{% trans bugs = bugs %}Open an issue +if you have comments or suggestions for the Python documentation.{% endtrans %}

{% endblock %} diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 0fdbe2a5801..75f6607d8f3 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -27,8 +27,8 @@ In extensions/changes.py: -{% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} -{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} +{% trans %}Deprecated since version %s, will be removed in version %s{% endtrans %} +{% trans %}Deprecated since version %s, removed in version %s{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 06a4223643a..544cc4234f4 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -72,7 +72,7 @@

{{ docstitle|e }}

' % self.cssclass_noday + return f'' else: - return '' % (self.cssclasses[weekday], day) + return f'' def formatweek(self, theweek): """ Return a complete week as a table row. """ s = ''.join(self.formatday(d, wd) for (d, wd) in theweek) - return '%s' % s + return f'{s}' def formatweekday(self, day): """ Return a weekday name as a table header. """ - return '' % ( - self.cssclasses_weekday_head[day], day_abbr[day]) + return f'' def formatweekheader(self): """ Return a header for a week as a table row. """ s = ''.join(self.formatweekday(i) for i in self.iterweekdays()) - return '%s' % s + return f'{s}' def formatmonthname(self, theyear, themonth, withyear=True): """ @@ -510,11 +528,10 @@ def formatmonthname(self, theyear, themonth, withyear=True): """ _validate_month(themonth) if withyear: - s = '%s %s' % (month_name[themonth], theyear) + s = f'{standalone_month_name[themonth]} {theyear}' else: - s = '%s' % month_name[themonth] - return '' % ( - self.cssclass_month_head, s) + s = standalone_month_name[themonth] + return f'' def formatmonth(self, theyear, themonth, withyear=True): """ @@ -522,8 +539,7 @@ def formatmonth(self, theyear, themonth, withyear=True): """ v = [] a = v.append - a('

{% trans %}Reporting issues{% endtrans %}

-

{% trans %}Contributing to Docs{% endtrans %}

+

{% trans %}Contributing to docs{% endtrans %}

{% trans %}Download the documentation{% endtrans %}

{% trans %}History and license of Python{% endtrans %}

diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index eea29e2449a..086f15662cf 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -9,9 +9,9 @@

{% trans %}Docs by version{% endtrans %}

{% trans %}Other resources{% endtrans %}

diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 56023ebf962..54c6eb9b5ef 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,11 +26,11 @@ {% endblock %} {% block extrahead %} - {% if builder == "html" and enable_analytics %} - - {% endif %} - - {% if builder != "htmlhelp" %} + {% if builder == "html" %} + {% if enable_analytics %} + + {% endif %} + {% if pagename == 'whatsnew/changelog' and not embedded %} {% endif %} {% endif %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 6a1611afadb..5020c428741 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -15,7 +15,7 @@ basic interpreter is supported on all platforms with minimal line control capabilities. On Windows, or Unix-like systems with :mod:`curses` support, -a new interactive shell is used by default. +a new interactive shell is used by default since Python 3.13. This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 9d0fab8861d..9ab003d5cd3 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -359,7 +359,7 @@ Usually, a method is called right after it is bound:: x.f() -In the :class:`!MyClass` example, this will return the string ``'hello world'``. +If ``x = MyClass()``, as above, this will return the string ``'hello world'``. However, it is not necessary to call a method right away: ``x.f`` is a method object, and can be stored away and called at a later time. For example:: @@ -663,6 +663,9 @@ Taken together, these properties make it possible to design reliable and extensible classes with multiple inheritance. For more detail, see :ref:`python_2.3_mro`. +In some cases multiple inheritance is not allowed; see :ref:`multiple-inheritance` +for details. + .. _tut-private: diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 95939242fb7..5ec8789f98c 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -251,6 +251,7 @@ statements: a ``try`` statement's ``else`` clause runs when no exception occurs, and a loop's ``else`` clause runs when no ``break`` occurs. For more on the ``try`` statement and exceptions, see :ref:`tut-handling`. +.. index:: single: ...; ellipsis literal .. _tut-pass: :keyword:`!pass` Statements @@ -277,6 +278,12 @@ at a more abstract level. The :keyword:`!pass` is silently ignored:: ... pass # Remember to implement this! ... +For this last case, many people use the ellipsis literal :code:`...` instead of +:code:`pass`. This use has no special meaning to Python, and is not part of +the language definition (you could use any constant expression here), but +:code:`...` is used conventionally as a placeholder body as well. +See :ref:`bltin-ellipsis-object`. + .. _tut-match: @@ -561,7 +568,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`!append` shown in the example is defined for list objects; it + The method :meth:`~list.append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. @@ -999,7 +1006,8 @@ scope:: 43 The above example uses a lambda expression to return a function. Another use -is to pass a small function as an argument:: +is to pass a small function as an argument. For instance, :meth:`list.sort` +takes a sorting key function *key* which can be a lambda function:: >>> pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')] >>> pairs.sort(key=lambda pair: pair[1]) @@ -1055,7 +1063,7 @@ Here is an example of a multi-line docstring:: >>> print(my_function.__doc__) Do nothing, but document it. - No, really, it doesn't do anything. + No, really, it doesn't do anything. .. _tut-annotations: @@ -1073,7 +1081,7 @@ Function Annotations information about the types used by user-defined functions (see :pep:`3107` and :pep:`484` for more information). -:term:`Annotations ` are stored in the :attr:`!__annotations__` +:term:`Annotations ` are stored in the :attr:`~object.__annotations__` attribute of the function as a dictionary and have no effect on any other part of the function. Parameter annotations are defined by a colon after the parameter name, followed by an expression evaluating to the value of the annotation. Return annotations are diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index cbe780e075b..7e02e74177c 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -12,9 +12,8 @@ and adds some new things as well. More on Lists ============= -The list data type has some more methods. Here are all of the methods of list -objects: - +The :ref:`list ` data type has some more methods. Here are all +of the methods of list objects: .. method:: list.append(x) :noindex: @@ -62,7 +61,7 @@ objects: .. method:: list.index(x[, start[, end]]) :noindex: - Return zero-based index in the list of the first item whose value is equal to *x*. + Return zero-based index of the first occurrence of *x* in the list. Raises a :exc:`ValueError` if there is no such item. The optional arguments *start* and *end* are interpreted as in the slice @@ -142,8 +141,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`!append`. To retrieve an item from the -top of the stack, use :meth:`!pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the +top of the stack, use :meth:`~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -340,7 +339,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`!pop` method +value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -445,10 +444,11 @@ packing and sequence unpacking. Sets ==== -Python also includes a data type for *sets*. A set is an unordered collection -with no duplicate elements. Basic uses include membership testing and -eliminating duplicate entries. Set objects also support mathematical operations -like union, intersection, difference, and symmetric difference. +Python also includes a data type for :ref:`sets `. A set is +an unordered collection with no duplicate elements. Basic uses include +membership testing and eliminating duplicate entries. Set objects also +support mathematical operations like union, intersection, difference, and +symmetric difference. Curly braces or the :func:`set` function can be used to create sets. Note: to create an empty set you have to use ``set()``, not ``{}``; the latter creates an @@ -500,8 +500,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`!append` and -:meth:`!extend`. +assignments, slice assignments, or methods like :meth:`~list.append` and +:meth:`~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of @@ -512,8 +512,12 @@ dictionary; this is also the way dictionaries are written on output. The main operations on a dictionary are storing a value with some key and extracting the value given the key. It is also possible to delete a key:value pair with ``del``. If you store using a key that is already in use, the old -value associated with that key is forgotten. It is an error to extract a value -using a non-existent key. +value associated with that key is forgotten. + +Extracting a value for a non-existent key by subscripting (``d[key]``) raises a +:exc:`KeyError`. To avoid getting this error when trying to access a possibly +non-existent key, use the :meth:`~dict.get` method instead, which returns +``None`` (or a specified default value) if the key is not in the dictionary. Performing ``list(d)`` on a dictionary returns a list of all the keys used in the dictionary, in insertion order (if you want it sorted, just use @@ -528,6 +532,12 @@ Here is a small example using a dictionary:: {'jack': 4098, 'sape': 4139, 'guido': 4127} >>> tel['jack'] 4098 + >>> tel['irv'] + Traceback (most recent call last): + File "", line 1, in + KeyError: 'irv' + >>> print(tel.get('irv')) + None >>> del tel['sape'] >>> tel['irv'] = 4127 >>> tel diff --git a/Doc/tutorial/index.rst b/Doc/tutorial/index.rst index 96791f88c86..20fe161be4a 100644 --- a/Doc/tutorial/index.rst +++ b/Doc/tutorial/index.rst @@ -4,6 +4,10 @@ The Python Tutorial ###################### +.. Tip:: This tutorial is designed for + *programmers* that are new to the Python language, + **not** *beginners* who are new to programming. + Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python's elegant syntax and dynamic typing, @@ -11,7 +15,7 @@ together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. The Python interpreter and the extensive standard library are freely available -in source or binary form for all major platforms from the Python web site, +in source or binary form for all major platforms from the Python website, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation. @@ -21,7 +25,8 @@ implemented in C or C++ (or other languages callable from C). Python is also suitable as an extension language for customizable applications. This tutorial introduces the reader informally to the basic concepts and -features of the Python language and system. It helps to have a Python +features of the Python language and system. Be aware that it expects you to +have a basic understanding of programming in general. It helps to have a Python interpreter handy for hands-on experience, but all examples are self-contained, so the tutorial can be read off-line as well. diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 35b8c7cd8eb..a00f06cf46c 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -95,10 +95,11 @@ Some examples:: >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -The :mod:`string` module contains a :class:`~string.Template` class that offers -yet another way to substitute values into strings, using placeholders like -``$x`` and replacing them with values from a dictionary, but offers much less -control of the formatting. +The :mod:`string` module contains support for a simple templating approach +based upon regular expressions, via :class:`string.Template`. +This offers yet another way to substitute values into strings, +using placeholders like ``$x`` and replacing them with values from a dictionary. +This syntax is easy to use, although it offers much less control for formatting. .. index:: single: formatted string literal diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 02e7de77322..cd526071424 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -16,7 +16,7 @@ Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.14 + python3.15 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.14 - Python 3.14 (default, April 4 2024, 09:25:04) + $ python3.15 + Python 3.15 (default, May 7 2025, 15:46:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index bec5da8fd75..deabac52530 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -13,10 +13,9 @@ end a multi-line command. .. only:: html - You can toggle the display of prompts and output by clicking on ``>>>`` - in the upper-right corner of an example box. If you hide the prompts - and output for an example, then you can easily copy and paste the input - lines into your interpreter. + You can use the "Copy" button (it appears in the upper-right corner + when hovering over or tapping a code example), which strips prompts + and omits output, to copy and paste the input lines into your interpreter. .. index:: single: # (hash); comment @@ -50,7 +49,7 @@ primary prompt, ``>>>``. (It shouldn't take long.) Numbers ------- -The interpreter acts as a simple calculator: you can type an expression at it +The interpreter acts as a simple calculator: you can type an expression into it and it will write the value. Expression syntax is straightforward: the operators ``+``, ``-``, ``*`` and ``/`` can be used to perform arithmetic; parentheses (``()``) can be used for grouping. @@ -147,6 +146,8 @@ Python can manipulate text (represented by type :class:`str`, so-called "``Yay! :)``". They can be enclosed in single quotes (``'...'``) or double quotes (``"..."``) with the same result [#]_. +.. code-block:: pycon + >>> 'spam eggs' # single quotes 'spam eggs' >>> "Paris rabbit got your back :)! Yay!" # double quotes @@ -419,7 +420,7 @@ type, i.e. it is possible to change their content:: [1, 8, 27, 64, 125] You can also add new items at the end of the list, by using -the :meth:`!list.append` *method* (we will see more about methods later):: +the :meth:`list.append` *method* (we will see more about methods later):: >>> cubes.append(216) # add the cube of 6 >>> cubes.append(7 ** 3) # and the cube of 7 diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index de7aa0e2342..f8105cd5441 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -27,14 +27,16 @@ called :file:`fibo.py` in the current directory with the following contents:: # Fibonacci numbers module - def fib(n): # write Fibonacci series up to n + def fib(n): + """Write Fibonacci series up to n.""" a, b = 0, 1 while a < n: print(a, end=' ') a, b = b, a+b print() - def fib2(n): # return Fibonacci series up to n + def fib2(n): + """Return Fibonacci series up to n.""" result = [] a, b = 0, 1 while a < n: @@ -577,8 +579,8 @@ module for example, you might use:: from .. import formats from ..filters import equalizer -Note that relative imports are based on the name of the current module. Since -the name of the main module is always ``"__main__"``, modules intended for use +Note that relative imports are based on the name of the current module's package. +Since the main module does not have a package, modules intended for use as the main module of a Python application must always use absolute imports. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 4b3eef313e7..342c1a00193 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python314' + 'C:\\Python315' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') @@ -335,7 +335,7 @@ sophisticated and robust capabilities of its larger packages. For example: names, no direct knowledge or handling of XML is needed. * The :mod:`email` package is a library for managing email messages, including - MIME and other :rfc:`2822`-based message documents. Unlike :mod:`smtplib` and + MIME and other :rfc:`5322`-based message documents. Unlike :mod:`smtplib` and :mod:`poplib` which actually send and receive messages, the email package has a complete toolset for building or decoding complex message structures (including attachments) and for implementing internet encoding and header diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index a2f96b34b2d..678b71c9274 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -279,7 +279,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python314/lib/weakref.py", line 46, in __getitem__ + File "C:/python315/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index dbe2d7fc099..359cf80a7b2 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -30,7 +30,7 @@ the set are: More Python resources: -* https://www.python.org: The major Python web site. It contains code, +* https://www.python.org: The major Python website. It contains code, documentation, and pointers to Python-related pages around the web. * https://docs.python.org: Fast access to Python's documentation. diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 65bf23dc994..45345d045dd 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -40,8 +40,15 @@ If you're sure you want to do all of this manually, read on. You can use the :source:`testbed app ` as a guide; each step below contains a link to the relevant file. -* Build Python by following the instructions in :source:`Android/README.md`. - This will create the directory ``cross-build/HOST/prefix``. +* First, acquire a build of Python for Android: + + * The easiest way is to download an Android release from `python.org + `__. The ``prefix`` directory + mentioned below is at the top level of the package. + + * Or if you want to build it yourself, follow the instructions in + :source:`Android/README.md`. The ``prefix`` directory will be created under + :samp:`cross-build/{HOST}`. * Add code to your :source:`build.gradle ` file to copy the following items into your project. All except your own Python @@ -63,3 +70,12 @@ link to the relevant file. * Add code to your app to :source:`start Python in embedded mode `. This will need to be C code called via JNI. + +Building a Python package for Android +------------------------------------- + +Python packages can be built for Android as wheels and released on PyPI. The +recommended tool for doing this is `cibuildwheel +`__, which automates +all the details of setting up a cross-compilation environment, building the +wheel, and testing it on an emulator. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 40a46a62031..aff165191b7 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -254,6 +254,15 @@ Miscellaneous options .. versionchanged:: 3.5 Affects also comparisons of :class:`bytes` with :class:`int`. + .. deprecated:: 3.15 + + Deprecate :option:`-b` and :option:`!-bb` command line options + and schedule them to become no-op in Python 3.17. + These were primarily helpers for the Python 2 -> 3 transition. + Starting with Python 3.17, no :exc:`BytesWarning` will be raised + for these cases; use a type checker instead. + + .. option:: -B If given, Python won't try to write ``.pyc`` files on the @@ -369,8 +378,8 @@ Miscellaneous options .. option:: -R Turn on hash randomization. This option only has an effect if the - :envvar:`PYTHONHASHSEED` environment variable is set to ``0``, since hash - randomization is enabled by default. + :envvar:`PYTHONHASHSEED` environment variable is set to anything other + than ``random``, since hash randomization is enabled by default. On previous versions of Python, this option turns on hash randomization, so that the :meth:`~object.__hash__` values of str and bytes objects @@ -470,8 +479,10 @@ Miscellaneous options The *action* field is as explained above but only applies to warnings that match the remaining fields. - The *message* field must match the whole warning message; this match is - case-insensitive. + The *message* field must match the start of the warning message; + this match is case-insensitive. + If it starts and ends with a forward slash (``/``), it specifies + a regular expression, otherwise it specifies a literal string. The *category* field matches the warning category (ex: ``DeprecationWarning``). This must be a class name; the match test @@ -480,6 +491,10 @@ Miscellaneous options The *module* field matches the (fully qualified) module name; this match is case-sensitive. + If it starts and ends with a forward slash (``/``), it specifies + a regular expression that the start of the fully qualified module name + must match, otherwise it specifies a literal string that the fully + qualified module name must be equal to. The *lineno* field matches the line number, where zero matches all line numbers and is thus equivalent to an omitted line number. @@ -497,6 +512,9 @@ Miscellaneous options See :ref:`warning-filter` and :ref:`describing-warning-filters` for more details. + .. versionchanged:: 3.15 + Added regular expression support for *message* and *module*. + .. option:: -x @@ -653,7 +671,7 @@ Miscellaneous options .. versionadded:: 3.13 * :samp:`-X thread_inherit_context={0,1}` causes :class:`~threading.Thread` - to, by default, use a copy of context of of the caller of + to, by default, use a copy of context of the caller of ``Thread.start()`` when starting. Otherwise, threads will start with an empty context. If unset, the value of this option defaults to ``1`` on free-threaded builds and to ``0`` otherwise. See also @@ -669,6 +687,13 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X tlbc={0,1}` enables (1, the default) or disables (0) thread-local + bytecode in builds configured with :option:`--disable-gil`. When disabled, + this also disables the specializing interpreter. See also + :envvar:`PYTHON_TLBC`. + + .. versionadded:: 3.14 + It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -964,6 +989,9 @@ conflict. See :ref:`warning-filter` and :ref:`describing-warning-filters` for more details. + .. versionchanged:: 3.15 + Added regular expression support for *message* and *module*. + .. envvar:: PYTHONFAULTHANDLER @@ -1249,9 +1277,8 @@ conflict. .. envvar:: PYTHON_BASIC_REPL If this variable is set to any value, the interpreter will not attempt to - load the Python-based :term:`REPL` that requires :mod:`curses` and - :mod:`readline`, and will instead use the traditional parser-based - :term:`REPL`. + load the Python-based :term:`REPL` that requires :mod:`readline`, and will + instead use the traditional parser-based :term:`REPL`. .. versionadded:: 3.13 @@ -1277,7 +1304,7 @@ conflict. .. envvar:: PYTHON_THREAD_INHERIT_CONTEXT If this variable is set to ``1`` then :class:`~threading.Thread` will, - by default, use a copy of context of of the caller of ``Thread.start()`` + by default, use a copy of context of the caller of ``Thread.start()`` when starting. Otherwise, new threads will start with an empty context. If unset, this variable defaults to ``1`` on free-threaded builds and to ``0`` otherwise. See also :option:`-X thread_inherit_context<-X>`. @@ -1302,6 +1329,16 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_TLBC + + If set to ``1`` enables thread-local bytecode. If set to ``0`` thread-local + bytecode and the specializing interpreter are disabled. Only applies to + builds configured with :option:`--disable-gil`. + + See also the :option:`-X tlbc <-X>` command-line option. + + .. versionadded:: 3.14 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index b914d3397b6..dff0fe036ea 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -4,10 +4,13 @@ Configure Python .. highlight:: sh + +.. _build-requirements: + Build Requirements ================== -Features and minimum versions required to build CPython: +To build CPython, you will need: * A `C11 `_ compiler. `Optional C11 features @@ -22,49 +25,138 @@ Features and minimum versions required to build CPython: * Support for threads. -* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.16 is the recommended - minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules. - -* SQLite 3.15.2 for the :mod:`sqlite3` extension module. - -* Tcl/Tk 8.5.12 for the :mod:`tkinter` module. - -* Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the - :file:`configure` script. - -.. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. - .. versionchanged:: 3.5 On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. .. versionchanged:: 3.6 - Selected C99 features are now required, like ```` and ``static - inline`` functions. + Selected C99 features, like ```` and ``static inline`` functions, + are now required. .. versionchanged:: 3.7 - Thread support and OpenSSL 1.0.2 are now required. - -.. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. + Thread support is now required. .. versionchanged:: 3.11 C11 compiler, IEEE 754 and NaN support are now required. On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. - -.. versionchanged:: 3.13 - Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. - -.. versionchanged:: 3.14 - Autoconf 2.72 is now required. See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". +.. _optional-module-requirements: + +Requirements for optional modules +--------------------------------- + +Some :term:`optional modules ` of the standard library +require third-party libraries installed for development +(for example, header files must be available). + +Missing requirements are reported in the ``configure`` output. +Modules that are missing due to missing dependencies are listed near the end +of the ``make`` output, +sometimes using an internal name, for example, ``_ctypes`` for :mod:`ctypes` +module. + +If you distribute a CPython interpreter without optional modules, +it's best practice to advise users, who generally expect that +standard library modules are available. + +Dependencies to build optional modules are: + +.. list-table:: + :header-rows: 1 + :align: left + + * - Dependency + - Minimum version + - Python module + * - `libbz2 `_ + - + - :mod:`bz2` + * - `libffi `_ + - 3.3.0 recommended + - :mod:`ctypes` + * - `liblzma `_ + - + - :mod:`lzma` + * - `libmpdec `_ + - 2.5.0 + - :mod:`decimal` [1]_ + * - `libreadline `_ or + `libedit `_ [2]_ + - + - :mod:`readline` + * - `libuuid `_ + - + - ``_uuid`` [3]_ + * - `ncurses `_ [4]_ + - + - :mod:`curses` + * - `OpenSSL `_ + - | 3.0.18 recommended + | (1.1.1 minimum) + - :mod:`ssl`, :mod:`hashlib` [5]_ + * - `SQLite `_ + - 3.15.2 + - :mod:`sqlite3` + * - `Tcl/Tk `_ + - 8.5.12 + - :mod:`tkinter`, :ref:`IDLE `, :mod:`turtle` + * - `zlib `_ + - 1.2.2.1 + - :mod:`zlib`, :mod:`gzip`, :mod:`ensurepip` + * - `zstd `_ + - 1.4.5 + - :mod:`compression.zstd` + +.. [1] If *libmpdec* is not available, the :mod:`decimal` module will use + a pure-Python implementation. + See :option:`--with-system-libmpdec` for details. +.. [2] See :option:`--with-readline` for choosing the backend for the + :mod:`readline` module. +.. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. + See the module documentation for details. +.. [4] The :mod:`curses` module requires the ``libncurses`` or ``libncursesw`` + library. + The :mod:`curses.panel` module additionally requires the ``libpanel`` or + ``libpanelw`` library. +.. [5] If OpenSSL is not available, the :mod:`hashlib` module will use + bundled implementations of several hash functions. + See :option:`--with-builtin-hashlib-hashes` for *forcing* usage of OpenSSL. + +Note that the table does not include all optional modules; in particular, +platform-specific modules like :mod:`winreg` are not listed here. + +.. seealso:: + + * The `devguide `_ + includes a full list of dependencies required to build all modules and + instructions on how to install them on common platforms. + * :option:`--with-system-expat` allows building with an external + `libexpat `_ library. + * :ref:`configure-options-for-dependencies` + +.. versionchanged:: 3.1 + Tcl/Tk version 8.3.1 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.5 + Tcl/Tk version 8.4 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.7 + OpenSSL 1.0.2 is now required for :mod:`hashlib` and :mod:`ssl`. + +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required for :mod:`hashlib` and :mod:`ssl`. + SQLite 3.7.15 is now required for :mod:`sqlite3`. + +.. versionchanged:: 3.11 + Tcl/Tk version 8.5.12 is now required for :mod:`tkinter`. + +.. versionchanged:: 3.13 + SQLite 3.15.2 is now required for :mod:`sqlite3`. + + Generated files =============== @@ -91,8 +183,19 @@ The container is optional, the following command can be run locally:: autoreconf -ivf -Werror -The generated files can change depending on the exact ``autoconf-archive``, -``aclocal`` and ``pkg-config`` versions. +The generated files can change depending on the exact versions of the +tools used. +The container that CPython uses has +`Autoconf `_ 2.72, +``aclocal`` from `Automake `_ 1.16.5, +and `pkg-config `_ 1.8.1. + +.. versionchanged:: 3.13 + Autoconf 2.71 and aclocal 1.16.5 and are now used to regenerate + :file:`configure`. + +.. versionchanged:: 3.14 + Autoconf 2.72 is now used to regenerate :file:`configure`. .. _configure-options: @@ -219,6 +322,30 @@ General Options .. versionadded:: 3.11 +.. option:: --with-missing-stdlib-config=FILE + + Path to a `JSON `_ configuration file + containing custom error messages for missing :term:`standard library` modules. + + This option is intended for Python distributors who wish to provide + distribution-specific guidance when users encounter standard library + modules that are missing or packaged separately. + + The JSON file should map missing module names to custom error message strings. + For example, if your distribution packages :mod:`tkinter` and + :mod:`_tkinter` separately and excludes :mod:`!_gdbm` for legal reasons, + the configuration could contain: + + .. code-block:: json + + { + "_gdbm": "The '_gdbm' module is not available in this distribution", + "tkinter": "Install the python-tk package to use tkinter", + "_tkinter": "Install the python-tk package to use tkinter", + } + + .. versionadded:: next + .. option:: --enable-pystats Turn on internal Python performance statistics gathering. @@ -290,8 +417,11 @@ General Options .. option:: --disable-gil - Enables **experimental** support for running Python without the - :term:`global interpreter lock` (GIL): free threading build. + .. c:macro:: Py_GIL_DISABLED + :no-typesetting: + + Enables support for running Python without the :term:`global interpreter + lock` (GIL): free threading build. Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to :data:`sys.abiflags`. @@ -370,6 +500,8 @@ Linker options Name for machine-dependent library files. +.. _configure-options-for-dependencies: + Options for third-party dependencies ------------------------------------ @@ -392,12 +524,6 @@ Options for third-party dependencies C compiler and linker flags for ``gdbm``. -.. option:: LIBB2_CFLAGS -.. option:: LIBB2_LIBS - - C compiler and linker flags for ``libb2`` (:ref:`BLAKE2 `), - used by :mod:`hashlib` module, overriding ``pkg-config``. - .. option:: LIBEDIT_CFLAGS .. option:: LIBEDIT_LIBS @@ -445,6 +571,14 @@ Options for third-party dependencies C compiler and linker flags for ``libuuid``, used by :mod:`uuid` module, overriding ``pkg-config``. +.. option:: LIBZSTD_CFLAGS +.. option:: LIBZSTD_LIBS + + C compiler and linker flags for ``libzstd``, used by :mod:`compression.zstd` module, + overriding ``pkg-config``. + + .. versionadded:: 3.14 + .. option:: PANEL_CFLAGS .. option:: PANEL_LIBS @@ -675,6 +809,13 @@ also be used to improve performance. not compiled. This includes both the functionality to schedule code to be executed and the functionality to receive code to be executed. + .. c:macro:: Py_REMOTE_DEBUG + + This macro is defined by default, unless Python is configured with + :option:`--without-remote-debug`. + + Note that even if the macro is defined, remote debugging may not be + available (for example, on an incompatible platform). .. versionadded:: 3.14 @@ -784,6 +925,9 @@ Debug options .. option:: --with-address-sanitizer Enable AddressSanitizer memory error detector, ``asan`` (default is no). + To improve ASan detection capabilities you may also want to combine this + with :option:`--without-pymalloc` to disable the specialized small-object + allocator whose allocations are not tracked by ASan. .. versionadded:: 3.6 @@ -845,9 +989,16 @@ Libraries options .. versionchanged:: 3.13 Default to using the installed ``mpdecimal`` library. - .. deprecated-removed:: 3.13 3.15 + .. versionchanged:: 3.15 + + A bundled copy of the library will no longer be selected + implicitly if an installed ``mpdecimal`` library is not found. + In Python 3.15 only, it can still be selected explicitly using + ``--with-system-libmpdec=no`` or ``--without-system-libmpdec``. + + .. deprecated-removed:: 3.13 3.16 A copy of the ``mpdecimal`` library sources will no longer be distributed - with Python 3.15. + with Python 3.16. .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 7d5c6331bef..5e4033fb6ce 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -170,7 +170,7 @@ helpful. To add Python to an iOS Xcode project: 1. Build or obtain a Python ``XCFramework``. See the instructions in - :source:`iOS/README.rst` (in the CPython source distribution) for details on + :source:`Apple/iOS/README.md` (in the CPython source distribution) for details on how to build a Python ``XCFramework``. At a minimum, you will need a build that supports ``arm64-apple-ios``, plus one of either ``arm64-apple-ios-simulator`` or ``x86_64-apple-ios-simulator``. @@ -180,22 +180,19 @@ To add Python to an iOS Xcode project: of your project; however, you can use any other location that you want by adjusting paths as needed. -3. Drag the ``iOS/Resources/dylib-Info-template.plist`` file into your project, - and ensure it is associated with the app target. - -4. Add your application code as a folder in your Xcode project. In the +3. Add your application code as a folder in your Xcode project. In the following instructions, we'll assume that your user code is in a folder named ``app`` in the root of your project; you can use any other location by adjusting paths as needed. Ensure that this folder is associated with your app target. -5. Select the app target by selecting the root node of your Xcode project, then +4. Select the app target by selecting the root node of your Xcode project, then the target name in the sidebar that appears. -6. In the "General" settings, under "Frameworks, Libraries and Embedded +5. In the "General" settings, under "Frameworks, Libraries and Embedded Content", add ``Python.xcframework``, with "Embed & Sign" selected. -7. In the "Build Settings" tab, modify the following: +6. In the "Build Settings" tab, modify the following: - Build Options @@ -211,86 +208,24 @@ To add Python to an iOS Xcode project: * Quoted Include In Framework Header: No -8. Add a build step that copies the Python standard library into your app. In - the "Build Phases" tab, add a new "Run Script" build step *before* the - "Embed Frameworks" step, but *after* the "Copy Bundle Resources" step. Name - the step "Install Target Specific Python Standard Library", disable the - "Based on dependency analysis" checkbox, and set the script content to: +7. Add a build step that processes the Python standard library, and your own + Python binary dependencies. In the "Build Phases" tab, add a new "Run + Script" build step *before* the "Embed Frameworks" step, but *after* the + "Copy Bundle Resources" step. Name the step "Process Python libraries", + disable the "Based on dependency analysis" checkbox, and set the script + content to: .. code-block:: bash - set -e + set -e + source $PROJECT_DIR/Python.xcframework/build/build_utils.sh + install_python Python.xcframework app - mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" - if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then - echo "Installing Python modules for iOS Simulator" - rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - else - echo "Installing Python modules for iOS Device" - rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - fi + If you have placed your XCframework somewhere other than the root of your + project, modify the path to the first argument. - Note that the name of the simulator "slice" in the XCframework may be - different, depending the CPU architectures your ``XCFramework`` supports. - -9. Add a second build step that processes the binary extension modules in the - standard library into "Framework" format. Add a "Run Script" build step - *directly after* the one you added in step 8, named "Prepare Python Binary - Modules". It should also have "Based on dependency analysis" unchecked, with - the following script content: - - .. code-block:: bash - - set -e - - install_dylib () { - INSTALL_BASE=$1 - FULL_EXT=$2 - - # The name of the extension file - EXT=$(basename "$FULL_EXT") - # The location of the extension file, relative to the bundle - RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} - # The path to the extension file, relative to the install base - PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} - # The full dotted name of the extension module, constructed from the file path. - FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); - # A bundle identifier; not actually used, but required by Xcode framework packaging - FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") - # The name of the framework folder. - FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" - - # If the framework folder doesn't exist, create it. - if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then - echo "Creating framework for $RELATIVE_EXT" - mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" - cp "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - fi - - echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" - mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" - # Create a placeholder .fwork file where the .so was - echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork - # Create a back reference to the .so file location in the framework - echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" - } - - PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") - echo "Install Python $PYTHON_VER standard library extension modules..." - find "$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload" -name "*.so" | while read FULL_EXT; do - install_dylib python/lib/$PYTHON_VER/lib-dynload/ "$FULL_EXT" - done - - # Clean up dylib template - rm -f "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" - - echo "Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." - find "$CODESIGNING_FOLDER_PATH/Frameworks" -name "*.framework" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "{}" \; - -10. Add Objective C code to initialize and use a Python interpreter in embedded - mode. You should ensure that: +8. Add Objective C code to initialize and use a Python interpreter in embedded + mode. You should ensure that: * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; @@ -298,9 +233,9 @@ To add Python to an iOS Xcode project: * Signal handlers (:c:member:`PyConfig.install_signal_handlers`) are *enabled*; * System logging (:c:member:`PyConfig.use_system_logger`) is *enabled* (optional, but strongly recommended; this is enabled by default); - * ``PYTHONHOME`` for the interpreter is configured to point at the + * :envvar:`PYTHONHOME` for the interpreter is configured to point at the ``python`` subfolder of your app's bundle; and - * The ``PYTHONPATH`` for the interpreter includes: + * The :envvar:`PYTHONPATH` for the interpreter includes: - the ``python/lib/python3.X`` subfolder of your app's bundle, - the ``python/lib/python3.X/lib-dynload`` subfolder of your app's bundle, and @@ -309,45 +244,52 @@ To add Python to an iOS Xcode project: Your app's bundle location can be determined using ``[[NSBundle mainBundle] resourcePath]``. -Steps 8, 9 and 10 of these instructions assume that you have a single folder of +Steps 7 and 8 of these instructions assume that you have a single folder of pure Python application code, named ``app``. If you have third-party binary modules in your app, some additional steps will be required: * You need to ensure that any folders containing third-party binaries are - either associated with the app target, or copied in as part of step 8. Step 8 - should also purge any binaries that are not appropriate for the platform a - specific build is targeting (i.e., delete any device binaries if you're - building an app targeting the simulator). + either associated with the app target, or are explicitly copied as part of + step 7. Step 7 should also purge any binaries that are not appropriate for + the platform a specific build is targeting (i.e., delete any device binaries + if you're building an app targeting the simulator). -* Any folders that contain third-party binaries must be processed into - framework form by step 9. The invocation of ``install_dylib`` that processes - the ``lib-dynload`` folder can be copied and adapted for this purpose. +* If you're using a separate folder for third-party packages, ensure that + folder is added to the end of the call to ``install_python`` in step 7, and + as part of the :envvar:`PYTHONPATH` configuration in step 8. -* If you're using a separate folder for third-party packages, ensure that folder - is included as part of the ``PYTHONPATH`` configuration in step 10. +* If any of the folders that contain third-party packages will contain ``.pth`` + files, you should add that folder as a *site directory* (using + :meth:`site.addsitedir`), rather than adding to :envvar:`PYTHONPATH` or + :attr:`sys.path` directly. Testing a Python package ------------------------ -The CPython source tree contains :source:`a testbed project ` that +The CPython source tree contains :source:`a testbed project ` that is used to run the CPython test suite on the iOS simulator. This testbed can also be used as a testbed project for running your Python library's test suite on iOS. -After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` -for details), create a clone of the Python iOS testbed project by running: +After building or obtaining an iOS XCFramework (see :source:`Apple/iOS/README.md` +for details), create a clone of the Python iOS testbed project. If you used the +``Apple`` build script to build the XCframework, you can run: .. code-block:: bash - $ python iOS/testbed clone --framework --app --app app-testbed + $ python cross-build/iOS/testbed clone --app --app app-testbed -You will need to modify the ``iOS/testbed`` reference to point to that -directory in the CPython source tree; any folders specified with the ``--app`` -flag will be copied into the cloned testbed project. The resulting testbed will -be created in the ``app-testbed`` folder. In this example, the ``module1`` and -``module2`` would be importable modules at runtime. If your project has -additional dependencies, they can be installed into the -``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target -app-testbed/iOSTestbed/app_packages`` or similar). +Or, if you've sourced your own XCframework, by running: + +.. code-block:: bash + + $ python Apple/testbed clone --platform iOS --framework --app --app app-testbed + +Any folders specified with the ``--app`` flag will be copied into the cloned +testbed project. The resulting testbed will be created in the ``app-testbed`` +folder. In this example, the ``module1`` and ``module2`` would be importable +modules at runtime. If your project has additional dependencies, they can be +installed into the ``app-testbed/Testbed/app_packages`` folder (using ``pip +install --target app-testbed/Testbed/app_packages`` or similar). You can then use the ``app-testbed`` folder to run the test suite for your app, For example, if ``module1.tests`` was the entry point to your test suite, you @@ -369,13 +311,29 @@ You can also open the testbed project in Xcode by running: This will allow you to use the full Xcode suite of tools for debugging. +The arguments used to run the test suite are defined as part of the test plan. +To modify the test plan, select the test plan node of the project tree (it +should be the first child of the root node), and select the "Configurations" +tab. Modify the "Arguments Passed On Launch" value to change the testing +arguments. + +The test plan also disables parallel testing, and specifies the use of the +``Testbed.lldbinit`` file for providing configuration of the debugger. The +default debugger configuration disables automatic breakpoints on the +``SIGINT``, ``SIGUSR1``, ``SIGUSR2``, and ``SIGXFSZ`` signals. + App Store Compliance ==================== The only mechanism for distributing apps to third-party iOS devices is to submit the app to the iOS App Store; apps submitted for distribution must pass Apple's app review process. This process includes a set of automated validation -rules that inspect the submitted application bundle for problematic code. +rules that inspect the submitted application bundle for problematic code. There +are some steps that must be taken to ensure that your app will be able to pass +these validation steps. + +Incompatible code in the standard library +----------------------------------------- The Python standard library contains some code that is known to violate these automated rules. While these violations appear to be false positives, Apple's @@ -386,3 +344,18 @@ The Python source tree contains :source:`a patch file ` that will remove all code that is known to cause issues with the App Store review process. This patch is applied automatically when building for iOS. + +Privacy manifests +----------------- + +In April 2025, Apple introduced a requirement for `certain third-party +libraries to provide a Privacy Manifest +`__. +As a result, if you have a binary module that uses one of the affected +libraries, you must provide an ``.xcprivacy`` file for that library. +OpenSSL is one library affected by this requirement, but there are others. + +If you produce a binary module named ``mymodule.so``, and use you the Xcode +build script described in step 7 above, you can place a ``mymodule.xcprivacy`` +file next to ``mymodule.so``, and the privacy manifest will be installed into +the required location when the binary module is converted into a framework. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 4b6c884f3d4..2fd6eace2b5 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -20,13 +20,6 @@ the Pythons provided by the CPython release team for download from the `python.org website `_. See :ref:`alternative_bundles` for some other options. -.. |usemac_x_dot_y| replace:: 3.13 -.. |usemac_python_x_dot_y_literal| replace:: ``python3.13`` -.. |usemac_python_x_dot_y_t_literal| replace:: ``python3.13t`` -.. |usemac_python_x_dot_y_t_literal_config| replace:: ``python3.13t-config`` -.. |usemac_applications_folder_name| replace:: ``Python 3.13`` -.. |usemac_applications_folder_version| replace:: ``/Applications/Python 3.13/`` - .. _getting-osx: .. _getting-and-installing-macpython: @@ -46,7 +39,7 @@ Current installers provide a `universal2 binary `_ build of Python which runs natively on all Macs (Apple Silicon and Intel) that are supported by a wide range of macOS versions, -currently typically from at least **macOS 10.13 High Sierra** on. +currently typically from at least **macOS 10.15 Catalina** on. The downloaded file is a standard macOS installer package file (``.pkg``). File integrity information (checksum, size, sigstore signature, etc) for each file is included @@ -64,7 +57,7 @@ Clicking on the **Continue** button brings up the **Read Me** for this installer Besides other important information, the **Read Me** documents which Python version is going to be installed and on what versions of macOS it is supported. You may need to scroll through to read the whole file. By default, this **Read Me** will also be -installed in |usemac_applications_folder_version| and available to read anytime. +installed in |applications_python_version_literal| and available to read anytime. .. image:: mac_installer_02_readme.png @@ -83,7 +76,7 @@ display. For most uses, the standard set of installation operations is appropria By pressing the **Customize** button, you can choose to omit or select certain package components of the installer. Click on each package name to see a description of what it installs. -To also install support for the optional experimental free-threaded feature, +To also install support for the optional free-threaded feature, see :ref:`install-freethreaded-macos`. .. image:: mac_installer_05_custom_install.png @@ -97,7 +90,7 @@ When the installation is complete, the **Summary** window will appear. .. image:: mac_installer_06_summary.png Double-click on the :command:`Install Certificates.command` -icon or file in the |usemac_applications_folder_version| window to complete the +icon or file in the |applications_python_version_literal| window to complete the installation. .. image:: mac_installer_07_applications.png @@ -114,7 +107,7 @@ Close this terminal window and the installer window. A default install will include: -* A |usemac_applications_folder_name| folder in your :file:`Applications` folder. In here +* A |python_version_literal| folder in your :file:`Applications` folder. In here you find :program:`IDLE`, the development environment that is a standard part of official Python distributions; and :program:`Python Launcher`, which handles double-clicking Python scripts from the macOS `Finder `_. @@ -141,7 +134,7 @@ How to run a Python script There are two ways to invoke the Python interpreter. If you are familiar with using a Unix shell in a terminal -window, you can invoke |usemac_python_x_dot_y_literal| or ``python3`` optionally +window, you can invoke |python_x_dot_y_literal| or ``python3`` optionally followed by one or more command line options (described in :ref:`using-on-general`). The Python tutorial also has a useful section on :ref:`using Python interactively from a shell `. @@ -160,7 +153,7 @@ for more information. To run a Python script file from the terminal window, you can invoke the interpreter with the name of the script file: - |usemac_python_x_dot_y_literal| ``myscript.py`` + |python_x_dot_y_literal| ``myscript.py`` To run your script from the Finder, you can either: @@ -259,20 +252,20 @@ Advanced Topics Installing Free-threaded Binaries --------------------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 The ``python.org`` :ref:`Python for macOS ` installer package can optionally install an additional build of -Python |usemac_x_dot_y| that supports :pep:`703`, the experimental free-threading feature +Python |version| that supports :pep:`703`, the free-threading feature (running with the :term:`global interpreter lock` disabled). Check the release page on ``python.org`` for possible updated information. -Because this feature is still considered experimental, the support for it +The free-threaded mode is working and continues to be improved, but +there is some additional overhead in single-threaded workloads compared +to the regular build. Additionally, third-party packages, in particular ones +with an :term:`extension module`, may not be ready for use in a +free-threaded build, and will re-enable the :term:`GIL`. +Therefore, the support for free-threading is not installed by default. It is packaged as a separate install option, available by clicking the **Customize** button on the **Installation Type** step of the installer as described above. @@ -282,46 +275,54 @@ step of the installer as described above. If the box next to the **Free-threaded Python** package name is checked, a separate :file:`PythonT.framework` will also be installed alongside the normal :file:`Python.framework` in :file:`/Library/Frameworks`. -This configuration allows a free-threaded Python |usemac_x_dot_y| build to co-exist -on your system with a traditional (GIL only) Python |usemac_x_dot_y| build with -minimal risk while installing or testing. This installation layout is itself -experimental and is subject to change in future releases. +This configuration allows a free-threaded Python |version| build to co-exist +on your system with a traditional (GIL only) Python |version| build with +minimal risk while installing or testing. This installation layout may +change in future releases. Known cautions and limitations: - The **UNIX command-line tools** package, which is selected by default, - will install links in :file:`/usr/local/bin` for |usemac_python_x_dot_y_t_literal|, - the free-threaded interpreter, and |usemac_python_x_dot_y_t_literal_config|, + will install links in :file:`/usr/local/bin` for |python_x_dot_y_t_literal|, + the free-threaded interpreter, and |python_x_dot_y_t_literal_config|, a configuration utility which may be useful for package builders. Since :file:`/usr/local/bin` is typically included in your shell ``PATH``, in most cases no changes to your ``PATH`` environment variables should - be needed to use |usemac_python_x_dot_y_t_literal|. + be needed to use |python_x_dot_y_t_literal|. - For this release, the **Shell profile updater** package and the - :file:`Update Shell Profile.command` in |usemac_applications_folder_version| + :file:`Update Shell Profile.command` in |applications_python_version_literal| do not support the free-threaded package. - The free-threaded build and the traditional build have separate search paths and separate :file:`site-packages` directories so, by default, if you need a package available in both builds, it may need to be installed in both. The free-threaded package will install a separate instance of :program:`pip` for use - with |usemac_python_x_dot_y_t_literal|. + with |python_x_dot_y_t_literal|. - To install a package using :command:`pip` without a :command:`venv`: - |usemac_python_x_dot_y_t_literal| ``-m pip install `` + .. parsed-literal:: + + python\ |version|\ t -m pip install - When working with multiple Python environments, it is usually safest and easiest to :ref:`create and use virtual environments `. This can avoid possible command name conflicts and confusion about which Python is in use: - |usemac_python_x_dot_y_t_literal| ``-m venv `` + .. parsed-literal:: + + python\ |version|\ t -m venv + then :command:`activate`. - To run a free-threaded version of IDLE: - |usemac_python_x_dot_y_t_literal| ``-m idlelib`` + .. parsed-literal:: + + python\ |version|\ t -m idlelib + - The interpreters in both builds respond to the same :ref:`PYTHON environment variables ` @@ -337,28 +338,28 @@ Known cautions and limitations: thus it only needs to be run once. - If you cannot depend on the link in ``/usr/local/bin`` pointing to the - ``python.org`` free-threaded |usemac_python_x_dot_y_t_literal| (for example, if you want + ``python.org`` free-threaded |python_x_dot_y_t_literal| (for example, if you want to install your own version there or some other distribution does), you can explicitly set your shell ``PATH`` environment variable to include the ``PythonT`` framework ``bin`` directory: - .. code-block:: sh + .. parsed-literal:: - export PATH="/Library/Frameworks/PythonT.framework/Versions/3.13/bin":"$PATH" + export PATH="/Library/Frameworks/PythonT.framework/Versions/\ |version|\ /bin":"$PATH" The traditional framework installation by default does something similar, except for :file:`Python.framework`. Be aware that having both framework ``bin`` directories in ``PATH`` can lead to confusion if there are duplicate names - like ``python3.13`` in both; which one is actually used depends on the order + like |python_x_dot_y_literal| in both; which one is actually used depends on the order they appear in ``PATH``. The ``which python3.x`` or ``which python3.xt`` commands can show which path is being used. Using virtual environments can help avoid such ambiguities. Another option might be to create a shell :command:`alias` to the desired interpreter, like: - .. code-block:: sh + .. parsed-literal:: - alias py3.13="/Library/Frameworks/Python.framework/Versions/3.13/bin/python3.13" - alias py3.13t="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t" + alias py\ |version|\ ="/Library/Frameworks/Python.framework/Versions/\ |version|\ /bin/python\ |version|\ " + alias py\ |version|\ t="/Library/Frameworks/PythonT.framework/Versions/\ |version|\ /bin/python\ |version|\ t" Installing using the command line --------------------------------- @@ -369,22 +370,22 @@ the macOS command line :command:`installer` utility lets you select non-default options, too. If you are not familiar with :command:`installer`, it can be somewhat cryptic (see :command:`man installer` for more information). As an example, the following shell snippet shows one way to do it, -using the ``3.13.0b2`` release and selecting the free-threaded interpreter +using the |x_dot_y_b2_literal| release and selecting the free-threaded interpreter option: -.. code-block:: sh +.. parsed-literal:: - RELEASE="python-3.13.0b2-macos11.pkg" + RELEASE="python-\ |version|\ 0b2-macos11.pkg" # download installer pkg - curl -O https://www.python.org/ftp/python/3.13.0/${RELEASE} + curl -O \https://www.python.org/ftp/python/\ |version|\ .0/${RELEASE} # create installer choicechanges to customize the install: - # enable the PythonTFramework-3.13 package + # enable the PythonTFramework-\ |version|\ package # while accepting the other defaults (install all other packages) cat > ./choicechanges.plist < - + @@ -393,7 +394,7 @@ option: choiceAttribute selected choiceIdentifier - org.python.Python.PythonTFramework-3.13 + org.python.Python.PythonTFramework-\ |version|\ @@ -404,19 +405,19 @@ option: You can then test that both installer builds are now available with something like: -.. code-block:: console +.. parsed-literal:: $ # test that the free-threaded interpreter was installed if the Unix Command Tools package was enabled - $ /usr/local/bin/python3.13t -VV - Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ /usr/local/bin/python\ |version|\ t -VV + Python \ |version|\ .0b2 free-threading build (v\ |version|\ .0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] $ # and the traditional interpreter - $ /usr/local/bin/python3.13 -VV - Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + $ /usr/local/bin/python\ |version|\ -VV + Python \ |version|\ .0b2 (v\ |version|\ .0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] $ # test that they are also available without the prefix if /usr/local/bin is on $PATH - $ python3.13t -VV - Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] - $ python3.13 -VV - Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + $ python\ |version|\ t -VV + Python \ |version|\ .0b2 free-threading build (v\ |version|\ .0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ python\ |version|\ -VV + Python \ |version|\ .0b2 (v\ |version|\ .0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] .. note:: diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png index 940219cad6f..c8b3aa1099a 100644 Binary files a/Doc/using/mac_installer_07_applications.png and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 9ec4e341932..a9950ef7525 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -84,11 +84,17 @@ On FreeBSD and OpenBSD Building Python =============== +.. seealso:: + + If you want to contribute to CPython, refer to the + `devguide `_, + which includes build instructions and other tips on setting up environment. + If you want to compile CPython yourself, first thing you should do is get the `source `_. You can download either the -latest release's source or just grab a fresh `clone -`_. (If you want -to contribute patches, you will need a clone.) +latest release's source or grab a fresh `clone +`_. +You will also need to install the :ref:`build requirements `. The build process consists of the usual commands:: diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png index 0aa01c1df6e..12b89c0165d 100644 Binary files a/Doc/using/win_install_freethreaded.png and b/Doc/using/win_install_freethreaded.png differ diff --git a/Doc/using/win_installer.png b/Doc/using/win_installer.png index 03bf2d7b16c..fc9605a79cf 100644 Binary files a/Doc/using/win_installer.png and b/Doc/using/win_installer.png differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 74d6db5d7d1..ee182519199 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -4,6 +4,8 @@ .. _Microsoft Store app: https://apps.microsoft.com/detail/9NQ7512CXL7T +.. _legacy launcher: https://www.python.org/ftp/python/3.14.0/win32/launcher.msi + .. _using-on-windows: ************************* @@ -60,7 +62,7 @@ packages. .. _windows-path-mod: .. _launcher: -Python Install Manager +Python install manager ====================== Installation @@ -77,34 +79,33 @@ To install the file downloaded from python.org, either double-click and select "Install", or run ``Add-AppxPackage `` in Windows Powershell. After installation, the ``python``, ``py``, and ``pymanager`` commands should be -available. If they are not, click Start and search for "Manage app execution -aliases". This settings page will let you enable the relevant commands. They -will be labelled "Python (default)", "Python (default windowed)", and "Python -install manager". - -If you have existing installations of Python, or you have modified your -:envvar:`PATH` variable, you may need to remove them or undo the modifications -in order for the commands to work. Old versions of Python can be reinstalled -using the Python install manager. +available. If you have existing installations of Python, or you have modified +your :envvar:`PATH` variable, you may need to remove them or undo the +modifications. See :ref:`pymanager-troubleshoot` for more help with fixing +non-working commands. When you first install a runtime, you will likely be prompted to add a directory to your :envvar:`PATH`. This is optional, if you prefer to use the ``py`` command, but is offered for those who prefer the full range of aliases (such as ``python3.14.exe``) to be available. The directory will be -:file:`%LocalAppData%\Python\bin` by default, but may be customized by an +:file:`%LocalAppData%\\Python\\bin` by default, but may be customized by an administrator. Click Start and search for "Edit environment variables for your account" for the system settings page to add the path. +Each Python runtime you install will have its own directory for scripts. These +also need to be added to :envvar:`PATH` if you want to use them. + The Python install manager will be automatically updated to new releases. This does not affect any installs of Python runtimes. Uninstalling the Python install manager does not uninstall any Python runtimes. If you are not able to install an MSIX in your context, for example, you are -using automated deployment software that does not support it, please see -:ref:`pymanager-advancedinstall` below for more information. +using automated deployment software that does not support it, or are targeting +Windows Server 2019, please see :ref:`pymanager-advancedinstall` below for more +information. -Basic Use +Basic use --------- The recommended command for launching Python is ``python``, which will either @@ -147,6 +148,10 @@ want to be passed to the runtime (such as script files or the module to launch): $> py -m this ... +The default runtime can be overridden with the :envvar:`PYTHON_MANAGER_DEFAULT` +environment variable, or a configuration file. See :ref:`pymanager-config` for +information about configuration settings. + To launch a specific runtime, the ``py`` command accepts a ``-V:`` option. This option must be specified before any others. The tag is part or all of the identifier for the runtime; for those from the CPython team, it looks like the @@ -190,7 +195,7 @@ installed if automatic installation is configured (most likely by setting ``pymanager exec`` forms of the command were used. -Command Help +Command help ------------ The ``py help`` command will display the full list of supported commands, along @@ -215,7 +220,7 @@ override multiple settings at once. See :ref:`pymanager-config` below for more information about these files. -Listing Runtimes +Listing runtimes ---------------- .. code:: @@ -256,7 +261,7 @@ For compatibility with the old launcher, the ``--list``, ``--list-paths``, additional options, and will produce legacy formatted output. -Installing Runtimes +Installing runtimes ------------------- .. code:: @@ -295,7 +300,7 @@ useful for embedding runtimes into larger applications. .. _pymanager-offline: -Offline Installs +Offline installs ---------------- To perform offline installs of Python, you will need to first create an offline @@ -327,7 +332,7 @@ In this way, Python runtimes can be installed and managed on a machine without access to the internet. -Uninstalling Runtimes +Uninstalling runtimes --------------------- .. code:: @@ -382,50 +387,96 @@ The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative customization. -.. csv-table:: Standard configuration options - :header: "Config Key", "Environment Variable", "Description" - :widths: 2, 2, 4 +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 2, 2, 4 - ``default_tag``,``PYTHON_MANAGER_DEFAULT``,"The preferred default - version to launch or install. By default, this is interpreted as the most - recent non-prerelease version from the CPython team. - " - ``default_platform``,``PYTHON_MANAGER_DEFAULT_PLATFORM``,"The preferred - default platform to launch or install. This is treated as a suffix to the - specified tag, such that ``py -V:3.14`` would prefer an install for - ``3.14-64`` if it exists (and ``default_platform`` is ``-64``), but will use - ``3.14`` if no tagged install exists. - " - ``logs_dir``,``PYTHON_MANAGER_LOGS``,"The location where log files are - written. By default, :file:`%TEMP%`. - " - ``automatic_install``,``PYTHON_MANAGER_AUTOMATIC_INSTALL``,"True to - allow automatic installs when specifying a particular runtime to launch. - By default, true. - " - ``include_unmanaged``,``PYTHON_MANAGER_INCLUDE_UNMANAGED``,"True to - allow listing and launching runtimes that were not installed by the Python - install manager, or false to exclude them. By default, true. - " - ``shebang_can_run_anything``,"``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` - ","True to allow shebangs in ``.py`` files to launch applications other than - Python runtimes, or false to prevent it. By default, true. - " - ``log_level``,"``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG``","Set - the default level of output (0-50) By default, 20. Lower values produce more - output. The environment variables are boolean, and may produce additional - output during startup that is later suppressed by other configuration. - " - ``confirm``,``PYTHON_MANAGER_CONFIRM``,"True to confirm certain actions - before taking them (such as uninstall), or false to skip the confirmation. By - default, true. - " - ``install.source``,``PYTHON_MANAGER_SOURCE_URL``,"Override the index - feed to obtain new installs from. - " - ``list.format``,``PYTHON_MANAGER_LIST_FORMAT``,"Specify the default - format used by the ``py list`` command. By default, ``table``. - " +.. rubric:: Standard configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Environment Variable + - Description + + * - ``default_tag`` + - .. envvar:: PYTHON_MANAGER_DEFAULT + - The preferred default version to launch or install. + By default, this is interpreted as the most recent non-prerelease version + from the CPython team. + + * - ``default_platform`` + - ``PYTHON_MANAGER_DEFAULT_PLATFORM`` + - The preferred default platform to launch or install. + This is treated as a suffix to the specified tag, such that ``py -V:3.14`` + would prefer an install for ``3.14-64`` if it exists + (and ``default_platform`` is ``-64``), + but will use ``3.14`` if no tagged install exists. + + * - ``logs_dir`` + - ``PYTHON_MANAGER_LOGS`` + - The location where log files are written. + By default, :file:`%TEMP%`. + + * - ``automatic_install`` + - ``PYTHON_MANAGER_AUTOMATIC_INSTALL`` + - True to allow automatic installs when specifying a particular runtime + to launch. + By default, true. + + * - ``include_unmanaged`` + - ``PYTHON_MANAGER_INCLUDE_UNMANAGED`` + - True to allow listing and launching runtimes that were not installed + by the Python install manager, or false to exclude them. + By default, true. + + * - ``shebang_can_run_anything`` + - ``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` + - True to allow shebangs in ``.py`` files to launch applications other than + Python runtimes, or false to prevent it. + By default, true. + + * - ``log_level`` + - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` + - Set the default level of output (0-50). + By default, 20. + Lower values produce more output. + The environment variables are boolean, and may produce additional + output during startup that is later suppressed by other configuration. + + * - ``confirm`` + - ``PYTHON_MANAGER_CONFIRM`` + - True to confirm certain actions before taking them (such as uninstall), + or false to skip the confirmation. + By default, true. + + * - ``install.source`` + - ``PYTHON_MANAGER_SOURCE_URL`` + - Override the index feed to obtain new installs from. + + * - ``list.format`` + - ``PYTHON_MANAGER_LIST_FORMAT`` + - Specify the default format used by the ``py list`` command. + By default, ``table``. + + * - ``install_dir`` + - (none) + - Specify the root directory that runtimes will be installed into. + If you change this setting, previously installed runtimes will not be + usable unless you move them to the new location. + + * - ``global_dir`` + - (none) + - Specify the directory where global commands (such as ``python3.14.exe``) + are stored. + This directory should be added to your :envvar:`PATH` to make the + commands available from your terminal. + + * - ``download_dir`` + - (none) + - Specify the directory where downloaded files are stored. + This directory is a temporary cache, and can be cleaned up from time to + time. Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -469,6 +520,10 @@ directory (which you may have added to your :envvar:`PATH` environment variable) can be used in a shebang, even if it is not on your :envvar:`PATH`. This allows the use of shebangs like ``/usr/bin/python3.12`` to select a particular runtime. +If no runtimes are installed, or if automatic installation is enabled, the +requested runtime will be installed if necessary. See :ref:`pymanager-config` +for information about configuration settings. + The ``/usr/bin/env`` form of shebang line will also search the :envvar:`PATH` environment variable for unrecognized commands. This corresponds to the behaviour of the Unix ``env`` program, which performs the same search, but @@ -485,21 +540,26 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. -.. note: +.. note:: The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no longer apply. If you are specifically reliant on the old behaviour or - configuration, we recommend keeping the legacy launcher. It may be - `downloaded independently `_ - and installed on its own. The legacy launcher's ``py`` command will override - PyManager's one, and you will need to use ``pymanager`` commands for - installing and uninstalling. + configuration, we recommend installing the `legacy launcher`_. The legacy + launcher's ``py`` command will override PyManager's one by default, and you + will need to use ``pymanager`` commands for installing and uninstalling. +.. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage + +.. _Remove-AppxPackage: https://learn.microsoft.com/powershell/module/appx/remove-appxpackage + +.. _Add-AppxProvisionedPackage: https://learn.microsoft.com/powershell/module/dism/add-appxprovisionedpackage + +.. _PackageManager: https://learn.microsoft.com/uwp/api/windows.management.deployment.packagemanager .. _pymanager-advancedinstall: -Advanced Installation +Advanced installation --------------------- For situations where an MSIX cannot be installed, such as some older @@ -509,6 +569,11 @@ per-machine installs to its default location in Program Files. It will attempt to modify the system :envvar:`PATH` environment variable to include this install location, but be sure to validate this on your configuration. +.. note:: + + Windows Server 2019 is the only version of Windows that CPython supports that + does not support MSIX. For Windows Server 2019, you should use the MSI. + Be aware that the MSI package does not bundle any runtimes, and so is not suitable for installs into offline environments without also creating an offline install index. See :ref:`pymanager-offline` and :ref:`pymanager-admin-config` @@ -529,29 +594,66 @@ depending on whether it was installed from python.org or through the Windows Store. Attempting to run the executable directly from Program Files is not recommended. -To programmatically install or uninstall the MSIX without using your -distribution platform's native support, the `Add-AppxPackage -`_ and -`Remove-AppxPackage `_ -PowerShell cmdlets are simplest to use: +To programmatically install the Python install manager, it is easiest to use +WinGet, which is included with all supported versions of Windows: -.. code:: +.. code-block:: powershell + + $> winget install 9NQ7512CXL7T -e --accept-package-agreements --disable-interactivity + + # Optionally run the configuration checker and accept all changes + $> py install --configure -y + +To download the Python install manager and install on another machine, the +following WinGet command will download the required files from the Store to your +Downloads directory (add ``-d `` to customize the output location). +This also generates a YAML file that appears to be unnecessary, as the +downloaded MSIX can be installed by launching or using the commands below. + +.. code-block:: powershell + + $> winget download 9NQ7512CXL7T -e --skip-license --accept-package-agreements --accept-source-agreements + +To programmatically install or uninstall an MSIX using only PowerShell, the +`Add-AppxPackage`_ and `Remove-AppxPackage`_ PowerShell cmdlets are recommended: + +.. code-block:: powershell $> Add-AppxPackage C:\Downloads\python-manager-25.0.msix ... $> Get-AppxPackage PythonSoftwareFoundation.PythonManager | Remove-AppxPackage -The native APIs for package management may be found on the Windows -`PackageManager `_ -class. The :func:`!AddPackageAsync` method installs for the current user, or use -:func:`!StagePackageAsync` followed by :func:`!ProvisionPackageForAllUsersAsync` -to install the Python install manager for all users from the MSIX package. Users -will still need to install their own copies of Python itself, as there is no way -to trigger those installs without being a logged in user. +The latest release can be downloaded and installed by Windows by passing the +AppInstaller file to the Add-AppxPackage command. This installs using the MSIX +on python.org, and is only recommended for cases where installing via the Store +(interactively or using WinGet) is not possible. + +.. code-block:: powershell + + $> Add-AppxPackage -AppInstallerFile https://www.python.org/ftp/python/pymanager/pymanager.appinstaller + +Other tools and APIs may also be used to provision an MSIX package for all users +on a machine, but Python does not consider this a supported scenario. We suggest +looking into the PowerShell `Add-AppxProvisionedPackage`_ cmdlet, the native +Windows `PackageManager`_ class, or the documentation and support for your +deployment tool. + +Regardless of the install method, users will still need to install their own +copies of Python itself, as there is no way to trigger those installs without +being a logged in user. When using the MSIX, the latest version of Python will +be available for all users to install without network access. + +Note that the MSIX downloadable from the Store and from the Python website are +subtly different and cannot be installed at the same time. Wherever possible, +we suggest using the above WinGet commands to download the package from the +Store to reduce the risk of setting up conflicting installs. There are no +licensing restrictions on the Python install manager that would prevent using +the Store package in this way. + .. _pymanager-admin-config: -Administrative Configuration +Administrative configuration ---------------------------- There are a number of options that may be useful for administrators to override @@ -581,60 +683,73 @@ variable will be used instead. Configuration settings that are paths are interpreted as relative to the directory containing the configuration file that specified them. -.. csv-table:: Administrative configuration options - :header: "Config Key", "Description" - :widths: 1, 4 +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 4 - ``base_config``,"The highest priority configuration file to read. Note that - only the built-in configuration file and the registry can modify this - setting. - " - ``user_config``,"The second configuration file to read. - " - ``additional_config``,"The third configuration file to read. - " - ``registry_override_key``,"Registry location to check for overrides. Note - that only the built-in configuration file can modify this setting. - " - ``bundled_dir``,"Read-only directory containing locally cached files. - " - ``install.fallback_source``,"Path or URL to an index to consult when the - main index cannot be accessed. - " - ``install.enable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to allow (e.g. ``""pep514,start""``). Enabled shortcuts may still be disabled - by ``disable_shortcut_kinds``. - " - ``install.disable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to exclude (e.g. ``""pep514,start""``). Disabled shortcuts are not - reactivated by ``enable_shortcut_kinds``. - " - ``pep514_root``,"Registry location to read and write PEP 514 entries into. - By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. - " - ``start_folder``,"Start menu folder to write shortcuts into. By default, - ``Python``. This path is relative to the user's Programs folder. - " - ``virtual_env``,"Path to the active virtual environment. By default, this - is ``%VIRTUAL_ENV%``, but may be set empty to disable venv detection. - " - ``shebang_can_run_anything_silently``,"True to suppress visible warnings - when a shebang launches an application other than a Python runtime. - " +.. rubric:: Administrative configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Description + + * - ``base_config`` + - The highest priority configuration file to read. + Note that only the built-in configuration file and the registry can + modify this setting. + + * - ``user_config`` + - The second configuration file to read. + + * - ``additional_config`` + - The third configuration file to read. + + * - ``registry_override_key`` + - Registry location to check for overrides. + Note that only the built-in configuration file can modify this setting. + + * - ``bundled_dir`` + - Read-only directory containing locally cached files. + + * - ``install.fallback_source`` + - Path or URL to an index to consult when the main index cannot be accessed. + + * - ``install.enable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to allow (e.g. ``"pep514,start"``). + Enabled shortcuts may still be disabled by ``disable_shortcut_kinds``. + + * - ``install.disable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to exclude + (e.g. ``"pep514,start"``). + Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + + * - ``pep514_root`` + - Registry location to read and write PEP 514 entries into. + By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. + + * - ``start_folder`` + - Start menu folder to write shortcuts into. + By default, ``Python``. + This path is relative to the user's Programs folder. + + * - ``virtual_env`` + - Path to the active virtual environment. + By default, this is ``%VIRTUAL_ENV%``, but may be set empty + to disable venv detection. + + * - ``shebang_can_run_anything_silently`` + - True to suppress visible warnings when a shebang launches an application + other than a Python runtime. .. _install-freethreaded-windows: -Installing Free-threaded Binaries +Installing free-threaded binaries --------------------------------- -.. versionadded:: 3.13 (Experimental) +.. versionadded:: 3.13 -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. - -Pre-built distributions of the experimental free-threaded build are available +Pre-built distributions of the free-threaded build are available by installing tags with the ``t`` suffix. .. code:: @@ -659,60 +774,100 @@ issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). -.. csv-table:: Troubleshooting - :header: "Symptom", "Things to try" - :widths: 1, 1 +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 1 - "``python`` gives me a ""command not found"" error or opens the Store app - when I type it in my terminal.", "Did you :ref:`install the Python install - manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python (default)"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "", "Check that the ``py`` and ``pymanager`` commands work. - " - "``py`` gives me a ""command not found"" error when I type it in my - terminal.","Did you :ref:`install the Python install manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python install manager"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "``py`` gives me a ""can't open file"" error when I type commands in my - terminal.", "This usually means you have the legacy launcher installed and it - has priority over the Python install manager. To remove, click Start, open - ""Installed apps"", search for ""Python launcher"" and uninstall it. - " - "``python`` doesn't launch the same runtime as ``py``", "Click Start, open - ""Installed apps"", look for any existing Python runtimes, and either remove - them or Modify and disable the :envvar:`PATH` options. - " - "", "Click Start, open ""Manage app execution aliases"", and check that your - ``python.exe`` alias is set to ""Python (default)"" - " - "``python`` and ``py`` don't launch the runtime I expect", "Check your - ``PYTHON_MANAGER_DEFAULT`` environment variable or ``default_tag`` - configuration. The ``py list`` command will show your default based on these - settings. - " - "", "Installs that are managed by the Python install manager will be chosen - ahead of unmanaged installs. Use ``py install`` to install the runtime you - expect, or configure your default tag. - " - "", "Prerelease and experimental installs that are not managed by the Python - install manager may be chosen ahead of stable releases. Configure your - default tag or uninstall the prerelease runtime and reinstall using ``py - install``. - " - "``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or - ``py``","Click Start, open ""Manage app execution aliases"", and check that - your ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your - others. - " +.. rubric:: Troubleshooting + +.. list-table:: + :header-rows: 1 + + * - Symptom + - Things to try + + * - ``python`` gives me a "command not found" error or opens the Store app + when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - + - Check that the ``py`` and ``pymanager`` commands work. + + * - ``py`` gives me a "command not found" error when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - ``py`` gives me a "can't open file" error when I type commands in my + terminal. + - This usually means you have the legacy launcher installed and + it has priority over the Python install manager. + To remove, click Start, open "Installed apps", + search for "Python launcher" and uninstall it. + + * - ``python`` doesn't launch the same runtime as ``py`` + - Click Start, open "Installed apps", look for any existing Python runtimes, + and either remove them or Modify and disable the :envvar:`PATH` options. + + * - + - Click Start, open "Manage app execution aliases", and check that your + ``python.exe`` alias is set to "Python (default)" + + * - ``python`` and ``py`` don't launch the runtime I expect + - Check your :envvar:`PYTHON_MANAGER_DEFAULT` environment variable + or ``default_tag`` configuration. + The ``py list`` command will show your default based on these settings. + + * - + - Installs that are managed by the Python install manager will be chosen + ahead of unmanaged installs. + Use ``py install`` to install the runtime you expect, + or configure your default tag. + + * - + - Prerelease and experimental installs that are not managed by the Python + install manager may be chosen ahead of stable releases. + Configure your default tag or uninstall the prerelease runtime + and reinstall using ``py install``. + + * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` + - Click Start, open "Manage app execution aliases", and check that your + ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your others. + + * - ``pip`` gives me a "command not found" error when I type it in my terminal. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - The package may be available but missing the generated executable. + We recommend using the ``python -m pip`` command instead, + or alternatively the ``python -m pip install --force pip`` command + will recreate the executables and show you the path to + add to :envvar:`PATH`. + These scripts are separated for each runtime, and so you may need to + add multiple paths. + + * - Typing ``script-name.py`` in the terminal opens in a new window. + - This is a known limitation of the operating system. Either specify ``py`` + before the script name, create a batch file containing ``@py "%~dpn0.py" %*`` + with the same name as the script, or install the `legacy launcher`_ + and select it as the association for scripts. + + * - Drag-dropping files onto a script doesn't work + - This is a known limitation of the operating system. It is supported with + the `legacy launcher`_, or with the Python install manager when installed + from the MSI. .. _windows-embeddable: @@ -754,7 +909,7 @@ versions before providing updates to users. The two recommended use cases for this distribution are described below. -Python Application +Python application ------------------ An application written in Python does not necessarily require users to be aware @@ -858,12 +1013,7 @@ for the 64-bit version, `www.nuget.org/packages/pythonx86 Free-threaded packages ---------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 Packages containing free-threaded binaries are named `python-freethreaded `_ @@ -915,7 +1065,7 @@ please install Python 3.12. .. _max-path: -Removing the MAX_PATH Limitation +Removing the MAX_PATH limitation ================================ Windows historically has limited path lengths to 260 characters. This meant that @@ -939,6 +1089,9 @@ UTF-8 mode ========== .. versionadded:: 3.7 +.. versionchanged:: 3.15 + + Python UTF-8 mode is now enabled by default (:pep:`686`). Windows still uses legacy encodings for the system encoding (the ANSI Code Page). Python uses it for the default encoding of text files (e.g. @@ -947,20 +1100,22 @@ Page). Python uses it for the default encoding of text files (e.g. This may cause issues because UTF-8 is widely used on the internet and most Unix systems, including WSL (Windows Subsystem for Linux). -You can use the :ref:`Python UTF-8 Mode ` to change the default text -encoding to UTF-8. You can enable the :ref:`Python UTF-8 Mode ` via -the ``-X utf8`` command line option, or the ``PYTHONUTF8=1`` environment -variable. See :envvar:`PYTHONUTF8` for enabling UTF-8 mode, and -:ref:`setting-envvars` for how to modify environment variables. - -When the :ref:`Python UTF-8 Mode ` is enabled, you can still use the +The :ref:`Python UTF-8 Mode `, enabled by default, can help by +changing the default text encoding to UTF-8. +When the :ref:`UTF-8 mode ` is enabled, you can still use the system encoding (the ANSI Code Page) via the "mbcs" codec. -Note that adding ``PYTHONUTF8=1`` to the default environment variables -will affect all Python 3.7+ applications on your system. -If you have any Python 3.7+ applications which rely on the legacy -system encoding, it is recommended to set the environment variable -temporarily or use the ``-X utf8`` command line option. +You can disable the :ref:`Python UTF-8 Mode ` via +the ``-X utf8=0`` command line option, or the ``PYTHONUTF8=0`` environment +variable. See :envvar:`PYTHONUTF8` for disabling UTF-8 mode, and +:ref:`setting-envvars` for how to modify environment variables. + +.. hint:: + Adding ``PYTHONUTF8={0,1}`` to the default environment variables + will affect all Python 3.7+ applications on your system. + If you have any Python 3.7+ applications which rely on the legacy + system encoding, it is recommended to set the environment variable + temporarily or use the ``-X utf8`` command line option. .. note:: Even when UTF-8 mode is disabled, Python uses UTF-8 by default @@ -1196,7 +1351,7 @@ installation". In this case: * Shortcuts are available for all users -Removing the MAX_PATH Limitation +Removing the MAX_PATH limitation -------------------------------- Windows historically has limited path lengths to 260 characters. This meant that @@ -1219,7 +1374,7 @@ After changing the above option, no further configuration is required. .. _install-quiet-option: -Installing Without UI +Installing without UI --------------------- All of the options available in the installer UI can also be specified from the @@ -1368,7 +1523,7 @@ example file sets the same options as the previous example: .. _install-layout-option: -Installing Without Downloading +Installing without downloading ------------------------------ As some features of Python are not included in the initial installer download, @@ -1409,15 +1564,10 @@ settings and replace any that have been removed or modified. :ref:`launcher`, which has its own entry in Programs and Features. -Installing Free-threaded Binaries +Installing free-threaded binaries --------------------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 To install pre-built binaries with free-threading enabled (see :pep:`703`), you should select "Customize installation". The second page of options includes the @@ -1449,7 +1599,7 @@ builds. Free-threaded binaries are also available :ref:`on nuget.org `. -Python Launcher for Windows (Deprecated) +Python launcher for Windows (deprecated) ======================================== .. deprecated:: 3.14 @@ -1601,7 +1751,7 @@ have the script specify the version which should be used. The key benefit of this is that a single launcher can support multiple Python versions at the same time depending on the contents of the first line. -Shebang Lines +Shebang lines ------------- If the first line of a script file starts with ``#!``, it is known as a @@ -1666,7 +1816,7 @@ program, which performs a :envvar:`PATH` search. If an executable matching the first argument after the ``env`` command cannot be found, but the argument starts with ``python``, it will be handled as described for the other virtual commands. -The environment variable :envvar:`PYLAUNCHER_NO_SEARCH_PATH` may be set +The environment variable :envvar:`!PYLAUNCHER_NO_SEARCH_PATH` may be set (to any value) to skip this search of :envvar:`PATH`. Shebang lines that do not match any of these patterns are looked up in the @@ -1733,7 +1883,7 @@ For example, a shebang line of ``#!python`` has no version qualifier, while ``#!python3`` has a version qualifier which specifies only a major version. If no version qualifiers are found in a command, the environment -variable :envvar:`PY_PYTHON` can be set to specify the default version +variable :envvar:`!PY_PYTHON` can be set to specify the default version qualifier. If it is not set, the default is "3". The variable can specify any value that may be passed on the command line, such as "3", "3.7", "3.7-32" or "3.7-64". (Note that the "-64" option is only @@ -1806,17 +1956,17 @@ For example: Diagnostics ----------- -If an environment variable :envvar:`PYLAUNCHER_DEBUG` is set (to any value), the +If an environment variable :envvar:`!PYLAUNCHER_DEBUG` is set (to any value), the launcher will print diagnostic information to stderr (i.e. to the console). While this information manages to be simultaneously verbose *and* terse, it should allow you to see what versions of Python were located, why a particular version was chosen and the exact command-line used to execute the target Python. It is primarily intended for testing and debugging. -Dry Run +Dry run ------- -If an environment variable :envvar:`PYLAUNCHER_DRYRUN` is set (to any value), +If an environment variable :envvar:`!PYLAUNCHER_DRYRUN` is set (to any value), the launcher will output the command it would have run, but will not actually launch Python. This may be useful for tools that want to use the launcher to detect and then launch Python directly. Note that the command written to @@ -1826,14 +1976,14 @@ the console. Install on demand ----------------- -If an environment variable :envvar:`PYLAUNCHER_ALLOW_INSTALL` is set (to any +If an environment variable :envvar:`!PYLAUNCHER_ALLOW_INSTALL` is set (to any value), and the requested Python version is not installed but is available on the Microsoft Store, the launcher will attempt to install it. This may require user interaction to complete, and you may need to run the command again. -An additional :envvar:`PYLAUNCHER_ALWAYS_INSTALL` variable causes the launcher +An additional :envvar:`!PYLAUNCHER_ALWAYS_INSTALL` variable causes the launcher to always try to install Python, even if it is detected. This is mainly intended -for testing (and should be used with :envvar:`PYLAUNCHER_DRYRUN`). +for testing (and should be used with :envvar:`!PYLAUNCHER_DRYRUN`). Return codes ------------ diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 1a949ec4035..c157c0337a1 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -656,7 +656,8 @@ break. The change which will probably break the most code is tightening up the arguments accepted by some methods. Some methods would take multiple arguments and treat them as a tuple, particularly various list methods such as -:meth:`!append` and :meth:`!insert`. In earlier versions of Python, if ``L`` is +:meth:`~list.append` and :meth:`~list.insert`. +In earlier versions of Python, if ``L`` is a list, ``L.append( 1,2 )`` appends the tuple ``(1,2)`` to the list. In Python 2.0 this causes a :exc:`TypeError` exception to be raised, with the message: 'append requires exactly 1 argument; 2 given'. The fix is to simply add an diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index b7e4e73f4ce..f43692b3dce 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -66,7 +66,7 @@ Here's a simple example:: The union and intersection of sets can be computed with the :meth:`~frozenset.union` and :meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: +:meth:`!union_update` and :meth:`~set.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~set.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 3430ac8668e..e195d9d462d 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -2169,9 +2169,9 @@ Changes to Python's build process and to the C API include: * Two new macros can be used to indicate C functions that are local to the current file so that a faster calling convention can be used. - ``Py_LOCAL(type)`` declares the function as returning a value of the + :c:macro:`Py_LOCAL` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. - ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the + :c:macro:`Py_LOCAL_INLINE` does the same thing and also requests the function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 9dbc07a34e2..f5e3a47037c 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -56,7 +56,7 @@ Python 2.6 incorporates new features and syntax from 3.0 while remaining compatible with existing code by not removing older features or syntax. When it's not possible to do that, Python 2.6 tries to do what it can, adding compatibility functions in a -:mod:`future_builtins` module and a :option:`!-3` switch to warn about +:mod:`!future_builtins` module and a :option:`!-3` switch to warn about usages that will become unsupported in 3.0. Some significant new packages have been added to the standard library, @@ -109,7 +109,7 @@ are: Python 3.0 adds several new built-in functions and changes the semantics of some existing builtins. Functions that are new in 3.0 such as :func:`bin` have simply been added to Python 2.6, but existing -builtins haven't been changed; instead, the :mod:`future_builtins` +builtins haven't been changed; instead, the :mod:`!future_builtins` module has versions with the new 3.0 semantics. Code written to be compatible with 3.0 can do ``from future_builtins import hex, map`` as necessary. @@ -118,7 +118,7 @@ A new command-line switch, :option:`!-3`, enables warnings about features that will be removed in Python 3.0. You can run code with this switch to see how much work will be necessary to port code to 3.0. The value of this switch is available -to Python code as the boolean variable :data:`sys.py3kwarning`, +to Python code as the boolean variable :data:`!sys.py3kwarning`, and to C extension code as :c:data:`!Py_Py3kWarningFlag`. .. seealso:: @@ -307,9 +307,9 @@ The :mod:`threading` module's locks and condition variables also support the The lock is acquired before the block is executed and always released once the block is complete. -The :func:`localcontext` function in the :mod:`decimal` module makes it easy -to save and restore the current decimal context, which encapsulates the desired -precision and rounding characteristics for computations:: +The :func:`~decimal.localcontext` function in the :mod:`decimal` module makes +it easy to save and restore the current decimal context, which encapsulates +the desired precision and rounding characteristics for computations:: from decimal import Decimal, Context, localcontext @@ -337,12 +337,12 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` - methods. + manager". The context manager must have :meth:`~object.__enter__` and + :meth:`~object.__exit__` methods. -* The context manager's :meth:`~object.__enter__` method is called. The value returned - is assigned to *VAR*. If no ``as VAR`` clause is present, the value is simply - discarded. +* The context manager's :meth:`~object.__enter__` method is called. The value + returned is assigned to *VAR*. If no ``as VAR`` clause is present, the + value is simply discarded. * The code in *BLOCK* is executed. @@ -378,7 +378,7 @@ be to let the user write code like this:: The transaction should be committed if the code in the block runs flawlessly or rolled back if there's an exception. Here's the basic interface for -:class:`DatabaseConnection` that I'll assume:: +:class:`!DatabaseConnection` that I'll assume:: class DatabaseConnection: # Database interface @@ -431,14 +431,15 @@ The contextlib module The :mod:`contextlib` module provides some functions and a decorator that are useful when writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single -generator function instead of defining a new class. The generator should yield -exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`~object.__enter__` method, and the value yielded will be the method's return -value that will get bound to the variable in the ':keyword:`with`' statement's -:keyword:`!as` clause, if any. The code after the :keyword:`!yield` will be -executed in the :meth:`~object.__exit__` method. Any exception raised in the block will -be raised by the :keyword:`!yield` statement. +The decorator is called :func:`~contextlib.contextmanager`, and lets you write +a single generator function instead of defining a new class. The generator +should yield exactly one value. The code up to the :keyword:`yield` will be +executed as the :meth:`~object.__enter__` method, and the value yielded will +be the method's return value that will get bound to the variable in the +':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after +the :keyword:`!yield` will be executed in the :meth:`~object.__exit__` method. +Any exception raised in the block will be raised by the :keyword:`!yield` +statement. Using this decorator, our database example from the previous section could be written as:: @@ -469,7 +470,7 @@ statement both starts a database transaction and acquires a thread lock:: with nested (db_transaction(db), lock) as (cursor, locked): ... -Finally, the :func:`closing` function returns its argument so that it can be +Finally, the :func:`~contextlib.closing` function returns its argument so that it can be bound to a variable, and calls the argument's ``.close()`` method at the end of the block. :: @@ -538,7 +539,7 @@ If you don't like the default directory, it can be overridden by an environment variable. :envvar:`PYTHONUSERBASE` sets the root directory used for all Python versions supporting this feature. On Windows, the directory for application-specific data can be changed by -setting the :envvar:`APPDATA` environment variable. You can also +setting the :envvar:`!APPDATA` environment variable. You can also modify the :file:`site.py` file for your Python installation. The feature can be disabled entirely by running Python with the @@ -568,11 +569,12 @@ The :mod:`multiprocessing` module started out as an exact emulation of the :mod:`threading` module using processes instead of threads. That goal was discarded along the path to Python 2.6, but the general approach of the module is still similar. The fundamental class -is the :class:`Process`, which is passed a callable object and -a collection of arguments. The :meth:`start` method +is the :class:`~multiprocessing.Process`, which is passed a callable object and +a collection of arguments. The :meth:`~multiprocessing.Process.start` method sets the callable running in a subprocess, after which you can call -the :meth:`is_alive` method to check whether the subprocess is still running -and the :meth:`join` method to wait for the process to exit. +the :meth:`~multiprocessing.Process.is_alive` method to check whether the +subprocess is still running and the :meth:`~multiprocessing.Process.join` +method to wait for the process to exit. Here's a simple example where the subprocess will calculate a factorial. The function doing the calculation is written strangely so @@ -619,13 +621,16 @@ the object to communicate. (If the parent were to change the value of the global variable, the child's value would be unaffected, and vice versa.) -Two other classes, :class:`Pool` and :class:`Manager`, provide -higher-level interfaces. :class:`Pool` will create a fixed number of -worker processes, and requests can then be distributed to the workers -by calling :meth:`apply` or :meth:`apply_async` to add a single request, -and :meth:`map` or :meth:`map_async` to add a number of -requests. The following code uses a :class:`Pool` to spread requests -across 5 worker processes and retrieve a list of results:: +Two other classes, :class:`~multiprocessing.pool.Pool` and +:class:`~multiprocessing.Manager`, provide higher-level interfaces. +:class:`~multiprocessing.pool.Pool` will create a fixed number of worker +processes, and requests can then be distributed to the workers by calling +:meth:`~multiprocessing.pool.Pool.apply` or +:meth:`~multiprocessing.pool.Pool.apply_async` to add a single request, and +:meth:`~multiprocessing.pool.Pool.map` or +:meth:`~multiprocessing.pool.Pool.map_async` to add a number of +requests. The following code uses a :class:`~multiprocessing.pool.Pool` to +spread requests across 5 worker processes and retrieve a list of results:: from multiprocessing import Pool @@ -646,15 +651,18 @@ This produces the following output:: 33452526613163807108170062053440751665152000000000 ... -The other high-level interface, the :class:`Manager` class, creates a -separate server process that can hold master copies of Python data +The other high-level interface, the :class:`~multiprocessing.Manager` class, +creates a separate server process that can hold master copies of Python data structures. Other processes can then access and modify these data structures using proxy objects. The following example creates a shared dictionary by calling the :meth:`dict` method; the worker processes then insert values into the dictionary. (Locking is not done for you automatically, which doesn't matter in this example. -:class:`Manager`'s methods also include :meth:`Lock`, :meth:`RLock`, -and :meth:`Semaphore` to create shared locks.) +:class:`~multiprocessing.Manager`'s methods also include +:meth:`~multiprocessing.managers.SyncManager.Lock`, +:meth:`~multiprocessing.managers.SyncManager.RLock`, +and :meth:`~multiprocessing.managers.SyncManager.Semaphore` to create +shared locks.) :: @@ -824,7 +832,7 @@ documentation for a :ref:`complete list `; here's a sample: format, followed by a percent sign. ===== ======================================================================== -Classes and types can define a :meth:`__format__` method to control how they're +Classes and types can define a :meth:`~object.__format__` method to control how they're formatted. It receives a single argument, the format specifier:: def __format__(self, format_spec): @@ -834,7 +842,7 @@ formatted. It receives a single argument, the format specifier:: return str(self) There's also a :func:`format` builtin that will format a single -value. It calls the type's :meth:`__format__` method with the +value. It calls the type's :meth:`~object.__format__` method with the provided specifier:: >>> format(75.6564, '.2f') @@ -997,9 +1005,10 @@ sequence of bytes:: u'\u31ef \u3244' Byte arrays support most of the methods of string types, such as -:meth:`startswith`/:meth:`endswith`, :meth:`find`/:meth:`rfind`, -and some of the methods of lists, such as :meth:`append`, -:meth:`pop`, and :meth:`reverse`. +:meth:`~bytearray.startswith`/:meth:`~bytearray.endswith`, +:meth:`~bytearray.find`/:meth:`~bytearray.rfind`, +and some of the methods of lists, such as :meth:`~bytearray.append`, +:meth:`~bytearray.pop`, and :meth:`~bytearray.reverse`. :: @@ -1028,56 +1037,58 @@ PEP 3116: New I/O Library Python's built-in file objects support a number of methods, but file-like objects don't necessarily support all of them. Objects that -imitate files usually support :meth:`read` and :meth:`write`, but they -may not support :meth:`readline`, for example. Python 3.0 introduces -a layered I/O library in the :mod:`io` module that separates buffering -and text-handling features from the fundamental read and write -operations. +imitate files usually support :meth:`!read` and +:meth:`!write`, but they may not support :meth:`!readline`, +for example. Python 3.0 introduces a layered I/O library in the :mod:`io` +module that separates buffering and text-handling features from the +fundamental read and write operations. There are three levels of abstract base classes provided by the :mod:`io` module: -* :class:`RawIOBase` defines raw I/O operations: :meth:`read`, - :meth:`readinto`, - :meth:`write`, :meth:`seek`, :meth:`tell`, :meth:`truncate`, - and :meth:`close`. +* :class:`~io.RawIOBase` defines raw I/O operations: :meth:`~io.RawIOBase.read`, + :meth:`~io.RawIOBase.readinto`, :meth:`~io.RawIOBase.write`, + :meth:`~io.IOBase.seek`, :meth:`~io.IOBase.tell`, :meth:`~io.IOBase.truncate`, + and :meth:`~io.IOBase.close`. Most of the methods of this class will often map to a single system call. - There are also :meth:`readable`, :meth:`writable`, and :meth:`seekable` - methods for determining what operations a given object will allow. + There are also :meth:`~io.IOBase.readable`, :meth:`~io.IOBase.writable`, + and :meth:`~io.IOBase.seekable` methods for determining what operations a + given object will allow. Python 3.0 has concrete implementations of this class for files and sockets, but Python 2.6 hasn't restructured its file and socket objects in this way. -* :class:`BufferedIOBase` is an abstract base class that +* :class:`~io.BufferedIOBase` is an abstract base class that buffers data in memory to reduce the number of system calls used, making I/O processing more efficient. - It supports all of the methods of :class:`RawIOBase`, - and adds a :attr:`raw` attribute holding the underlying raw object. + It supports all of the methods of :class:`~io.RawIOBase`, + and adds a :attr:`~io.BufferedIOBase.raw` attribute holding the underlying + raw object. There are five concrete classes implementing this ABC. - :class:`BufferedWriter` and :class:`BufferedReader` are for objects - that support write-only or read-only usage that have a :meth:`seek` - method for random access. :class:`BufferedRandom` objects support + :class:`~io.BufferedWriter` and :class:`~io.BufferedReader` are for objects + that support write-only or read-only usage that have a :meth:`~io.IOBase.seek` + method for random access. :class:`~io.BufferedRandom` objects support read and write access upon the same underlying stream, and - :class:`BufferedRWPair` is for objects such as TTYs that have both + :class:`~io.BufferedRWPair` is for objects such as TTYs that have both read and write operations acting upon unconnected streams of data. - The :class:`BytesIO` class supports reading, writing, and seeking + The :class:`~io.BytesIO` class supports reading, writing, and seeking over an in-memory buffer. .. index:: single: universal newlines; What's new -* :class:`TextIOBase`: Provides functions for reading and writing +* :class:`~io.TextIOBase`: Provides functions for reading and writing strings (remember, strings will be Unicode in Python 3.0), - and supporting :term:`universal newlines`. :class:`TextIOBase` defines + and supporting :term:`universal newlines`. :class:`~io.TextIOBase` defines the :meth:`readline` method and supports iteration upon objects. - There are two concrete implementations. :class:`TextIOWrapper` + There are two concrete implementations. :class:`~io.TextIOWrapper` wraps a buffered I/O object, supporting all of the methods for - text I/O and adding a :attr:`buffer` attribute for access - to the underlying object. :class:`StringIO` simply buffers + text I/O and adding a :attr:`~io.TextIOBase.buffer` attribute for access + to the underlying object. :class:`~io.StringIO` simply buffers everything in memory without ever writing anything to disk. (In Python 2.6, :class:`io.StringIO` is implemented in @@ -1161,7 +1172,7 @@ Some object-oriented languages such as Java support interfaces, declaring that a class has a given set of methods or supports a given access protocol. Abstract Base Classes (or ABCs) are an equivalent feature for Python. The ABC support consists of an :mod:`abc` module -containing a metaclass called :class:`ABCMeta`, special handling of +containing a metaclass called :class:`~abc.ABCMeta`, special handling of this metaclass by the :func:`isinstance` and :func:`issubclass` builtins, and a collection of basic ABCs that the Python developers think will be widely useful. Future versions of Python will probably @@ -1171,17 +1182,17 @@ Let's say you have a particular class and wish to know whether it supports dictionary-style access. The phrase "dictionary-style" is vague, however. It probably means that accessing items with ``obj[1]`` works. Does it imply that setting items with ``obj[2] = value`` works? -Or that the object will have :meth:`keys`, :meth:`values`, and :meth:`items` -methods? What about the iterative variants such as :meth:`iterkeys`? :meth:`copy` -and :meth:`update`? Iterating over the object with :func:`iter`? +Or that the object will have :meth:`!keys`, :meth:`!values`, and :meth:`!items` +methods? What about the iterative variants such as :meth:`!iterkeys`? +:meth:`!copy`and :meth:`!update`? Iterating over the object with :func:`!iter`? The Python 2.6 :mod:`collections` module includes a number of different ABCs that represent these distinctions. :class:`Iterable` -indicates that a class defines :meth:`__iter__`, and -:class:`Container` means the class defines a :meth:`__contains__` +indicates that a class defines :meth:`~object.__iter__`, and +:class:`Container` means the class defines a :meth:`~object.__contains__` method and therefore supports ``x in y`` expressions. The basic dictionary interface of getting items, setting items, and -:meth:`keys`, :meth:`values`, and :meth:`items`, is defined by the +:meth:`!keys`, :meth:`!values`, and :meth:`!items`, is defined by the :class:`MutableMapping` ABC. You can derive your own classes from a particular ABC @@ -1195,7 +1206,7 @@ to indicate they support that ABC's interface:: Alternatively, you could write the class without deriving from the desired ABC and instead register the class by -calling the ABC's :meth:`register` method:: +calling the ABC's :meth:`~abc.ABCMeta.register` method:: import collections @@ -1205,10 +1216,10 @@ calling the ABC's :meth:`register` method:: collections.MutableMapping.register(Storage) For classes that you write, deriving from the ABC is probably clearer. -The :meth:`register` method is useful when you've written a new +The :meth:`~abc.ABCMeta.register` method is useful when you've written a new ABC that can describe an existing type or class, or if you want to declare that some third-party class implements an ABC. -For example, if you defined a :class:`PrintableType` ABC, +For example, if you defined a :class:`!PrintableType` ABC, it's legal to do:: # Register Python's types @@ -1255,16 +1266,16 @@ metaclass in a class definition:: ... -In the :class:`Drawable` ABC above, the :meth:`draw_doubled` method +In the :class:`!Drawable` ABC above, the :meth:`!draw_doubled` method renders the object at twice its size and can be implemented in terms -of other methods described in :class:`Drawable`. Classes implementing +of other methods described in :class:`!Drawable`. Classes implementing this ABC therefore don't need to provide their own implementation -of :meth:`draw_doubled`, though they can do so. An implementation -of :meth:`draw` is necessary, though; the ABC can't provide +of :meth:`!draw_doubled`, though they can do so. An implementation +of :meth:`!draw` is necessary, though; the ABC can't provide a useful generic implementation. -You can apply the ``@abstractmethod`` decorator to methods such as -:meth:`draw` that must be implemented; Python will then raise an +You can apply the :deco:`~abc.abstractmethod` decorator to methods such as +:meth:`!draw` that must be implemented; Python will then raise an exception for classes that don't define the method. Note that the exception is only raised when you actually try to create an instance of a subclass lacking the method:: @@ -1288,7 +1299,7 @@ Abstract data attributes can be declared using the def readonly(self): return self._x -Subclasses must then define a :meth:`readonly` property. +Subclasses must then define a ``readonly`` property. .. seealso:: @@ -1528,8 +1539,8 @@ Some smaller changes made to the core Python language are: the :exc:`StopIteration` exception will be raised. (Backported in :issue:`2719`.) -* Tuples now have :meth:`index` and :meth:`count` methods matching the - list type's :meth:`index` and :meth:`count` methods:: +* Tuples now have :meth:`~tuple.index` and :meth:`~tuple.count` methods + matching the list type's :meth:`~list.index` and :meth:`~list.count` methods:: >>> t = (0,1,2,3,4,0,1,2) >>> t.index(3) @@ -2738,13 +2749,13 @@ numbers. .. ====================================================================== -The :mod:`future_builtins` module +The :mod:`!future_builtins` module -------------------------------------- Python 3.0 makes many changes to the repertoire of built-in functions, and most of the changes can't be introduced in the Python 2.x series because they would break compatibility. -The :mod:`future_builtins` module provides versions +The :mod:`!future_builtins` module provides versions of these built-in functions that can be imported when writing 3.0-compatible code. @@ -3043,7 +3054,7 @@ Changes to Python's build process and to the C API include: * Importing modules simultaneously in two different threads no longer deadlocks; it will now raise an :exc:`ImportError`. A new API - function, :c:func:`PyImport_ImportModuleNoBlock`, will look for a + function, :c:func:`!PyImport_ImportModuleNoBlock`, will look for a module in ``sys.modules`` first, then try to import it after acquiring an import lock. If the import lock is held by another thread, an :exc:`ImportError` is raised. diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index caed3192be8..7296296d144 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -858,8 +858,8 @@ Some smaller changes made to the core Python language are: .. XXX bytearray doesn't seem to be documented -* When using :class:`@classmethod ` and - :class:`@staticmethod ` to wrap +* When using :deco:`classmethod` and + :deco:`staticmethod` to wrap methods as class or static methods, the wrapper object now exposes the wrapped function as their :attr:`~method.__func__` attribute. @@ -1541,7 +1541,7 @@ changes, or look through the Subversion logs for all the details. buffer API, which fixed a test suite failure (fix by Antoine Pitrou; :issue:`7133`) and automatically set OpenSSL's :c:macro:`!SSL_MODE_AUTO_RETRY`, which will prevent an error - code being returned from :meth:`recv` operations that trigger an SSL + code being returned from :meth:`!recv` operations that trigger an SSL renegotiation (fix by Antoine Pitrou; :issue:`8222`). The :func:`~ssl.SSLContext.wrap_socket` constructor function now takes a @@ -2031,7 +2031,7 @@ version 1.3. Some of the new features are: * ElementTree's code for converting trees to a string has been significantly reworked, making it roughly twice as fast in many cases. The :meth:`ElementTree.write() ` - and :meth:`Element.write` methods now have a *method* parameter that can be + and :meth:`!Element.write` methods now have a *method* parameter that can be "xml" (the default), "html", or "text". HTML mode will output empty elements as ```` instead of ````, and text mode will skip over elements and only output the text chunks. If @@ -2044,7 +2044,7 @@ version 1.3. Some of the new features are: Namespace handling has also been improved. All ``xmlns:`` declarations are now output on the root element, not scattered throughout the resulting XML. You can set the default namespace for a tree - by setting the :attr:`default_namespace` attribute and can + by setting the :attr:`!default_namespace` attribute and can register new prefixes with :meth:`~xml.etree.ElementTree.register_namespace`. In XML mode, you can use the true/false *xml_declaration* parameter to suppress the XML declaration. @@ -2196,8 +2196,6 @@ Changes to Python's build process and to the C API include: locale-independent way. (Added by Eric Smith; :issue:`5793`.) - .. XXX these macros don't seem to be described in the c-api docs. - * Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be @@ -2234,7 +2232,7 @@ Changes to Python's build process and to the C API include: * When using the :c:type:`PyMemberDef` structure to define attributes of a type, Python will no longer let you try to delete or set a - :c:macro:`T_STRING_INPLACE` attribute. + :c:macro:`!T_STRING_INPLACE` attribute. .. rev 79644 @@ -2259,12 +2257,12 @@ Changes to Python's build process and to the C API include: :issue:`6491`.) * The :program:`configure` script now checks for floating-point rounding bugs - on certain 32-bit Intel chips and defines a :c:macro:`X87_DOUBLE_ROUNDING` + on certain 32-bit Intel chips and defines a :c:macro:`!X87_DOUBLE_ROUNDING` preprocessor definition. No code currently uses this definition, but it's available if anyone wishes to use it. (Added by Mark Dickinson; :issue:`2937`.) - :program:`configure` also now sets a :envvar:`LDCXXSHARED` Makefile + :program:`configure` also now sets a :envvar:`!LDCXXSHARED` Makefile variable for supporting C++ linking. (Contributed by Arfrever Frehtes Taifersar Arahesis; :issue:`1222585`.) diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 6e1fda22ed2..d858586138e 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -870,7 +870,7 @@ to the C API. * :c:func:`!PyNumber_Coerce`, :c:func:`!PyNumber_CoerceEx`, :c:func:`!PyMember_Get`, and :c:func:`!PyMember_Set` C APIs are removed. -* New C API :c:func:`PyImport_ImportModuleNoBlock`, works like +* New C API :c:func:`!PyImport_ImportModuleNoBlock`, works like :c:func:`PyImport_ImportModule` but won't block on the import lock (returning an error instead). diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 3c815721a92..d8251185fa7 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -551,11 +551,12 @@ Patterns and classes If you are using classes to structure your data, you can use as a pattern the class name followed by an argument list resembling a constructor. This -pattern has the ability to capture class attributes into variables:: +pattern has the ability to capture instance attributes into variables:: class Point: - x: int - y: int + def __init__(self, x, y): + self.x = x + self.y = y def location(point): match point: @@ -846,8 +847,8 @@ Other Language Changes respectively. (Contributed by Joshua Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) -* Static methods (:func:`@staticmethod `) and class methods - (:func:`@classmethod `) now inherit the method attributes +* Static methods (:deco:`staticmethod`) and class methods + (:deco:`classmethod`) now inherit the method attributes (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, ``__annotations__``) and have a new ``__wrapped__`` attribute. Moreover, static methods are now callable as regular functions. @@ -900,7 +901,7 @@ Improved Modules asyncio ------- -Add missing :meth:`~asyncio.events.AbstractEventLoop.connect_accepted_socket` +Add missing :meth:`~asyncio.loop.connect_accepted_socket` method. (Contributed by Alex Grönholm in :issue:`41332`.) @@ -932,7 +933,7 @@ Base32 Encoding with Extended Hex Alphabet. bdb --- -Add :meth:`~bdb.Breakpoint.clearBreakpoints` to reset all set breakpoints. +Add :meth:`!clearBreakpoints` to reset all set breakpoints. (Contributed by Irit Katriel in :issue:`24160`.) bisect @@ -1397,7 +1398,7 @@ A new verify flag :const:`~ssl.VERIFY_X509_PARTIAL_CHAIN` has been added. sqlite3 ------- -Add audit events for :func:`~sqlite3.connect/handle`, +Add audit events for :func:`~sqlite3.connect`, :meth:`~sqlite3.Connection.enable_load_extension`, and :meth:`~sqlite3.Connection.load_extension`. (Contributed by Erlend E. Aasland in :issue:`43762`.) @@ -2176,9 +2177,9 @@ Porting to Python 3.10 ``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. (Contributed by Victor Stinner in :issue:`42157`.) -* :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and - :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before +* :c:func:`!Py_GetPath`, :c:func:`!Py_GetPrefix`, :c:func:`!Py_GetExecPrefix`, + :c:func:`!Py_GetProgramFullPath`, :c:func:`!Py_GetPythonHome` and + :c:func:`!Py_GetProgramName` functions now return ``NULL`` if called before :c:func:`Py_Initialize` (before Python is initialized). Use the new :ref:`init-config` API to get the :ref:`init-path-config`. (Contributed by Victor Stinner in :issue:`42260`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 2dd205dd2b8..a095d887352 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2673,7 +2673,7 @@ Removed (Contributed by Victor Stinner in :issue:`45474`.) -* Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never +* Exclude :c:func:`!PyWeakref_GET_OBJECT` from the limited C API. It never worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. (Contributed by Victor Stinner in :issue:`35134`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a65f59c0a72..8badfe9a6b4 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1191,9 +1191,23 @@ Deprecated replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. +* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` implements + the :ref:`buffer protocol ` at runtime. For use in type + annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` were + also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. (Contributed by Shantanu Jain in :gh:`91896`.) * :mod:`datetime`: :class:`datetime.datetime`'s :meth:`~datetime.datetime.utcnow` and @@ -1301,7 +1315,7 @@ Deprecated :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are deprecated. (:gh:`94309`.) - * :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a + * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. (Contributed by Alex Waygood in :gh:`91896`.) @@ -1349,6 +1363,10 @@ Deprecated .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + .. include:: ../deprecations/pending-removal-in-future.rst .. _whatsnew312-removed: @@ -2233,6 +2251,8 @@ Deprecated .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + .. include:: ../deprecations/c-api-pending-removal-in-future.rst Removed diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e20e49325c0..dc105156f33 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -60,7 +60,7 @@ Summary -- Release Highlights .. This section singles out the most important changes in Python 3.13. Brevity is key. -Python 3.13 is the latest stable release of the Python programming +Python 3.13 is a stable release of the Python programming language, with a mix of changes to the language, the implementation and the standard library. The biggest changes include a new `interactive interpreter @@ -730,6 +730,22 @@ asyncio never awaited). (Contributed by Arthur Tacca and Jason Zhang in :gh:`115957`.) +* The function and methods named ``create_task`` have received a new + ``**kwargs`` argument that is passed through to the task constructor. + This change was accidentally added in 3.13.3, + and broke the API contract for custom task factories. + Several third-party task factories implemented workarounds for this. + In 3.13.4 and later releases the old factory contract is honored + once again (until 3.14). + To keep the workarounds working, the extra ``**kwargs`` argument still + allows passing additional keyword arguments to :class:`~asyncio.Task` + and to custom task factories. + + This affects the following function and methods: + :meth:`asyncio.create_task`, + :meth:`asyncio.loop.create_task`, + :meth:`asyncio.TaskGroup.create_task`. + (Contributed by Thomas Grainger in :gh:`128307`.) base64 ------ @@ -818,7 +834,7 @@ dbm (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) * Allow removing all items from the database through - the new :meth:`.gdbm.clear` and :meth:`.ndbm.clear` methods. + the new :meth:`!clear` methods of the GDBM and NDBM database objects. (Contributed by Donghee Na in :gh:`107122`.) @@ -1553,8 +1569,6 @@ and are now removed: * :pypi:`bcrypt`: Modern password hashing for your software and your servers. - * :pypi:`passlib`: - Comprehensive password hashing framework supporting over 30 schemes. * :pypi:`argon2-cffi`: The secure Argon2 password hashing algorithm. * :pypi:`legacycrypt`: @@ -1871,7 +1885,7 @@ New Deprecations * :mod:`http.server`: - * Deprecate :class:`~http.server.CGIHTTPRequestHandler`, + * Deprecate :class:`!CGIHTTPRequestHandler`, to be removed in Python 3.15. Process-based CGI HTTP servers have been out of favor for a very long time. This code was outdated, unmaintained, and rarely used. @@ -1901,14 +1915,14 @@ New Deprecations * :mod:`pathlib`: - * Deprecate :meth:`.PurePath.is_reserved`, + * Deprecate :meth:`!.PurePath.is_reserved`, to be removed in Python 3.15. Use :func:`os.path.isreserved` to detect reserved paths on Windows. (Contributed by Barney Gale in :gh:`88569`.) * :mod:`platform`: - * Deprecate :func:`~platform.java_ver`, + * Deprecate :func:`!platform.java_ver`, to be removed in Python 3.15. This function is only useful for Jython support, has a confusing API, and is largely untested. @@ -1979,8 +1993,8 @@ New Deprecations use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) - * Deprecate the :func:`typing.no_type_check_decorator` decorator function, - to be removed in in Python 3.15. + * Deprecate the :func:`!typing.no_type_check_decorator` decorator function, + to be removed in Python 3.15. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. (Contributed by Alex Waygood in :gh:`106309`.) @@ -1995,8 +2009,7 @@ New Deprecations * :mod:`wave`: - * Deprecate the :meth:`~wave.Wave_read.getmark`, :meth:`!setmark`, - and :meth:`~wave.Wave_read.getmarkers` methods of + * Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` methods of the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes, to be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) @@ -2011,6 +2024,10 @@ New Deprecations .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + .. include:: ../deprecations/pending-removal-in-future.rst CPython Bytecode Changes @@ -2161,7 +2178,7 @@ New Features (Contributed by Sam Gross in :gh:`114329`.) * Add the :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions, - mirroring the Python :meth:`!list.extend` and :meth:`!list.clear` methods. + mirroring the Python :meth:`list.extend` and :meth:`list.clear` methods. (Contributed by Victor Stinner in :gh:`111138`.) * Add the :c:func:`PyLong_AsInt` function. @@ -2231,7 +2248,7 @@ New Features (Contributed by Serhiy Storchaka in :gh:`110289`.) * Add the :c:func:`PyWeakref_GetRef` function - as an alternative to :c:func:`PyWeakref_GetObject` + as an alternative to :c:func:`!PyWeakref_GetObject` that returns a :term:`strong reference` or ``NULL`` if the referent is no longer live. (Contributed by Victor Stinner in :gh:`105927`.) @@ -2475,19 +2492,19 @@ Deprecated C APIs * Deprecate old Python initialization functions: - * :c:func:`PySys_ResetWarnOptions`: + * :c:func:`!PySys_ResetWarnOptions`: Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: + * :c:func:`!Py_GetExecPrefix`: Get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: + * :c:func:`!Py_GetPath`: Get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: + * :c:func:`!Py_GetPrefix`: Get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: + * :c:func:`!Py_GetProgramFullPath`: Get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: + * :c:func:`!Py_GetProgramName`: Get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: + * :c:func:`!Py_GetPythonHome`: Get :c:member:`PyConfig.home` or the :envvar:`PYTHONHOME` environment variable instead. @@ -2499,7 +2516,7 @@ Deprecated C APIs which return a :term:`borrowed reference`. (Soft deprecated as part of :pep:`667`.) -* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function, +* Deprecate the :c:func:`!PyImport_ImportModuleNoBlock` function, which is just an alias to :c:func:`PyImport_ImportModule` since Python 3.3. (Contributed by Victor Stinner in :gh:`105396`.) @@ -2516,8 +2533,8 @@ Deprecated C APIs are just aliases to :c:type:`!wchar_t`. (Contributed by Victor Stinner in :gh:`105156`.) -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, +* Deprecate the :c:func:`!PyWeakref_GetObject` and + :c:func:`!PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed reference`. Replace them with the new :c:func:`PyWeakref_GetRef` function, which returns a :term:`strong reference`. @@ -2531,6 +2548,8 @@ Deprecated C APIs .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + .. include:: ../deprecations/c-api-pending-removal-in-3.18.rst .. include:: ../deprecations/c-api-pending-removal-in-future.rst @@ -2577,7 +2596,7 @@ Build Changes * The :file:`configure` option :option:`--with-system-libmpdec` now defaults to ``yes``. - The bundled copy of ``libmpdecimal`` will be removed in Python 3.15. + The bundled copy of ``libmpdec`` will be removed in Python 3.16. * Python built with :file:`configure` :option:`--with-trace-refs` (tracing references) is now ABI compatible with the Python release build diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 894f011ec86..9459b73bcb5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1,8 +1,9 @@ + **************************** What's new in Python 3.14 **************************** -:Editor: Hugo van Kemenade +:Editors: Adam Turner and Hugo van Kemenade .. Rules for maintenance: @@ -45,334 +46,117 @@ when researching a change. This article explains the new features in Python 3.14, compared to 3.13. - +Python 3.14 was released on 7 October 2025. For full details, see the :ref:`changelog `. .. seealso:: :pep:`745` -- Python 3.14 release schedule -.. note:: - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.14 moves towards release, - so it's worth checking back even after reading earlier versions. - - -Summary -- release highlights +Summary -- Release highlights ============================= .. This section singles out the most important changes in Python 3.14. Brevity is key. -Python 3.14 beta is the pre-release of the next version of the Python -programming language, with a mix of changes to the language, the -implementation and the standard library. +Python 3.14 is the latest stable release of the Python programming +language, with a mix of changes to the language, the implementation, +and the standard library. +The biggest changes include :ref:`template string literals +`, +:ref:`deferred evaluation of annotations `, +and support for :ref:`subinterpreters ` in +the standard library. -The biggest changes to the implementation include template strings (:pep:`750`), -deferred evaluation of annotations (:pep:`649`), -and a new type of interpreter that uses tail calls. - -The library changes include the addition of a new :mod:`!annotationlib` module -for introspecting and wrapping annotations (:pep:`649`), -a new :mod:`!compression.zstd` module for Zstandard support (:pep:`784`), -plus syntax highlighting in the REPL, +The library changes include significantly improved capabilities for +:ref:`introspection in asyncio `, +:ref:`support for Zstandard ` via a new +:mod:`compression.zstd` module, syntax highlighting in the REPL, as well as the usual deprecations and removals, and improvements in user-friendliness and correctness. +This article doesn't attempt to provide a complete specification +of all new features, but instead gives a convenient overview. +For full details refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +To understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. +See `Porting to Python 3.14`_ for guidance on upgrading from +earlier versions of Python. + +-------------- + .. PEP-sized items next. -* :ref:`PEP 649: deferred evaluation of annotations ` -* :ref:`PEP 741: Python Configuration C API ` -* :ref:`PEP 750: Template strings ` -* :ref:`PEP 758: Allow except and except* expressions without parentheses ` -* :ref:`PEP 761: Discontinuation of PGP signatures ` -* :ref:`PEP 765: Disallow return/break/continue that exit a finally block ` -* :ref:`PEP 768: Safe external debugger interface for CPython ` -* :ref:`PEP 784: Adding Zstandard to the standard library ` -* :ref:`A new type of interpreter ` -* :ref:`Syntax highlighting in PyREPL `, - and color output in :ref:`unittest `, - :ref:`argparse `, - :ref:`json ` and - :ref:`calendar ` CLIs -* :ref:`Binary releases for the experimental just-in-time compiler ` +Interpreter improvements: +* :pep:`649` and :pep:`749`: :ref:`Deferred evaluation of annotations + ` +* :pep:`734`: :ref:`Multiple interpreters in the standard library + ` +* :pep:`750`: :ref:`Template strings ` +* :pep:`758`: :ref:`Allow except and except* expressions without brackets + ` +* :pep:`765`: :ref:`Control flow in finally blocks + ` +* :pep:`768`: :ref:`Safe external debugger interface for CPython + ` +* :ref:`A new type of interpreter ` +* :ref:`Free-threaded mode improvements ` +* :ref:`Improved error messages ` +* :ref:`Incremental garbage collection ` -Incompatible changes -==================== +Significant improvements in the standard library: -On platforms other than macOS and Windows, the default :ref:`start -method ` for :mod:`multiprocessing` -and :class:`~concurrent.futures.ProcessPoolExecutor` switches from -*fork* to *forkserver*. +* :pep:`784`: :ref:`Zstandard support in the standard library + ` +* :ref:`whatsnew314-asyncio-introspection` +* :ref:`whatsnew314-concurrent-warnings-control` +* :ref:`Syntax highlighting in the default interactive shell + `, and color output in several + standard library CLIs -See :ref:`(1) ` and -:ref:`(2) ` for details. +C API improvements: -If you encounter :exc:`NameError`\s or pickling errors coming out of -:mod:`multiprocessing` or :mod:`concurrent.futures`, see the -:ref:`forkserver restrictions `. +* :pep:`741`: :ref:`Python configuration C API ` + +Platform support: + +* :pep:`776`: Emscripten is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. + +Release changes: + +* :pep:`779`: :ref:`Free-threaded Python is officially supported + ` +* :pep:`761`: :ref:`PGP signatures have been discontinued for official releases + ` +* :ref:`Windows and macOS binary releases now support the experimental + just-in-time compiler ` +* :ref:`Binary releases for Android are now provided ` -The interpreter avoids some reference count modifications internally when -it's safe to do so. This can lead to different values returned from -:func:`sys.getrefcount` and :c:func:`Py_REFCNT` compared to previous versions -of Python. See :ref:`below ` for details. New features ============ -.. _whatsnew314-pep750: +.. _whatsnew314-deferred-annotations: -PEP 750: Template strings -------------------------- - -Template string literals (t-strings) are a generalization of f-strings, -using a ``t`` in place of the ``f`` prefix. Instead of evaluating -to :class:`str`, t-strings evaluate to a new :class:`!string.templatelib.Template` type: - -.. code-block:: python - - from string.templatelib import Template - - name = "World" - template: Template = t"Hello {name}" - -The template can then be combined with functions that operate on the template's -structure to produce a :class:`str` or a string-like result. -For example, sanitizing input: - -.. code-block:: python - - evil = "" - template = t"

{evil}

" - assert html(template) == "

<script>alert('evil')</script>

" - -As another example, generating HTML attributes from data: - -.. code-block:: python - - attributes = {"src": "shrubbery.jpg", "alt": "looks nice"} - template = t"" - assert html(template) == 'looks nice' - -Unlike f-strings, the ``html`` function has access to template attributes -containing the original information: static strings, interpolations, and values -from the original scope. Unlike existing templating approaches, t-strings build -from the well-known f-string syntax and rules. Template systems thus benefit -from Python tooling as they are much closer to the Python language, syntax, -scoping, and more. - -Writing template handlers is straightforward: - -.. code-block:: python - - from string.templatelib import Template, Interpolation - - def lower_upper(template: Template) -> str: - """Render static parts lowercased and interpolations uppercased.""" - parts: list[str] = [] - for item in template: - if isinstance(item, Interpolation): - parts.append(str(item.value).upper()) - else: - parts.append(item.lower()) - return "".join(parts) - - name = "world" - assert lower_upper(t"HELLO {name}") == "hello WORLD" - -With this in place, developers can write template systems to sanitize SQL, make -safe shell operations, improve logging, tackle modern ideas in web development -(HTML, CSS, and so on), and implement lightweight, custom business DSLs. - -(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, -Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, -and Pablo Galindo Salgado in :gh:`132661`.) - -.. seealso:: - :pep:`750`. - - -.. _whatsnew314-pep768: - -PEP 768: Safe external debugger interface for CPython ------------------------------------------------------ - -:pep:`768` introduces a zero-overhead debugging interface that allows debuggers and profilers -to safely attach to running Python processes. This is a significant enhancement to Python's -debugging capabilities allowing debuggers to forego unsafe alternatives. See -:ref:`below ` for how this feature is leveraged to -implement the new :mod:`pdb` module's remote attaching capabilities. - -The new interface provides safe execution points for attaching debugger code without modifying -the interpreter's normal execution path or adding runtime overhead. This enables tools to -inspect and interact with Python applications in real-time without stopping or restarting -them — a crucial capability for high-availability systems and production environments. - -For convenience, CPython implements this interface through the :mod:`sys` module with a -:func:`sys.remote_exec` function:: - - sys.remote_exec(pid, script_path) - -This function allows sending Python code to be executed in a target process at the next safe -execution point. However, tool authors can also implement the protocol directly as described -in the PEP, which details the underlying mechanisms used to safely attach to running processes. - -Here's a simple example that inspects object types in a running Python process: - - .. code-block:: python - - import os - import sys - import tempfile - - # Create a temporary script - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: - script_path = f.name - f.write(f"import my_debugger; my_debugger.connect({os.getpid()})") - try: - # Execute in process with PID 1234 - print("Behold! An offering:") - sys.remote_exec(1234, script_path) - finally: - os.unlink(script_path) - -The debugging interface has been carefully designed with security in mind and includes several -mechanisms to control access: - -* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. -* A :option:`-X disable-remote-debug` command-line option. -* A :option:`--without-remote-debug` configure flag to completely disable the feature at build time. - -A key implementation detail is that the interface piggybacks on the interpreter's existing evaluation -loop and safe points, ensuring zero overhead during normal execution while providing a reliable way -for external processes to coordinate debugging operations. - -(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic in :gh:`131591`.) - -.. seealso:: - :pep:`768`. - - -.. _whatsnew314-pep784: - -PEP 784: Adding Zstandard to the standard library -------------------------------------------------- - -The new ``compression`` package contains modules :mod:`!compression.lzma`, -:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` -which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` -modules respectively. The new import names under ``compression`` are the -canonical names for importing these compression modules going forward. However, -the existing modules names have not been deprecated. Any deprecation or removal -of the existing compression modules will occur no sooner than five years after -the release of 3.14. - -The new :mod:`!compression.zstd` module provides compression and decompression -APIs for the Zstandard format via bindings to `Meta's zstd library -`__. Zstandard is a widely adopted, highly -efficient, and fast compression format. In addition to the APIs introduced in -:mod:`!compression.zstd`, support for reading and writing Zstandard compressed -archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and -:mod:`shutil` modules. - -Here's an example of using the new module to compress some data: - -.. code-block:: python - - from compression import zstd - import math - - data = str(math.pi).encode() * 20 - - compressed = zstd.compress(data) - - ratio = len(compressed) / len(data) - print(f"Achieved compression ratio of {ratio}") - -As can be seen, the API is similar to the APIs of the :mod:`!lzma` and -:mod:`!bz2` modules. - -(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, -Victor Stinner, and Rogdham in :gh:`132983`) - -.. seealso:: - :pep:`784`. - - -.. _whatsnew314-remote-pdb: - -Remote attaching to a running Python process with PDB ------------------------------------------------------ - -The :mod:`pdb` module now supports remote attaching to a running Python process -using a new ``-p PID`` command-line option: - -.. code-block:: sh - - python -m pdb -p 1234 - -This will connect to the Python process with the given PID and allow you to -debug it interactively. Notice that due to how the Python interpreter works -attaching to a remote process that is blocked in a system call or waiting for -I/O will only work once the next bytecode instruction is executed or when the -process receives a signal. - -This feature uses :pep:`768` and the :func:`sys.remote_exec` function -to attach to the remote process and send the PDB commands to it. - - -(Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) - -.. seealso:: - :pep:`768`. - - -.. _whatsnew314-pep758: - -PEP 758 – Allow except and except* expressions without parentheses ------------------------------------------------------------------- - -The :keyword:`except` and :keyword:`except* ` expressions now allow -parentheses to be omitted when there are multiple exception types and the ``as`` clause is not used. -For example the following expressions are now valid: - -.. code-block:: python - - try: - release_new_sleep_token_album() - except AlbumNotFound, SongsTooGoodToBeReleased: - print("Sorry, no new album this year.") - - # The same applies to except* (for exception groups): - try: - release_new_sleep_token_album() - except* AlbumNotFound, SongsTooGoodToBeReleased: - print("Sorry, no new album this year.") - -Check :pep:`758` for more details. - -(Contributed by Pablo Galindo and Brett Cannon in :gh:`131831`.) - -.. seealso:: - :pep:`758`. - - -.. _whatsnew314-pep649: - -PEP 649: deferred evaluation of annotations -------------------------------------------- +:pep:`649` & :pep:`749`: Deferred evaluation of annotations +------------------------------------------------------------ The :term:`annotations ` on functions, classes, and modules are no longer evaluated eagerly. Instead, annotations are stored in special-purpose :term:`annotate functions ` and evaluated only when -necessary. This is specified in :pep:`649` and :pep:`749`. +necessary (except if ``from __future__ import annotations`` is used). -This change is designed to make annotations in Python more performant and more -usable in most circumstances. The runtime cost for defining annotations is +This change is designed to improve performance and usability of annotations +in Python in most circumstances. The runtime cost for defining annotations is minimized, but it remains possible to introspect annotations at runtime. -It is usually no longer necessary to enclose annotations in strings if they +It is no longer necessary to enclose annotations in strings if they contain forward references. The new :mod:`annotationlib` module provides tools for inspecting deferred @@ -398,51 +182,344 @@ This example shows how these formats behave: >>> get_annotations(func, format=Format.STRING) {'arg': 'Undefined'} -Implications for annotated code -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :ref:`porting ` section contains guidance +on changes that may be needed due to these changes, though in the majority of +cases, code will continue working as-is. -If you define annotations in your code (for example, for use with a static type -checker), then this change probably does not affect you: you can keep -writing annotations the same way you did with previous versions of Python. +(Contributed by Jelle Zijlstra in :pep:`749` and :gh:`119180`; +:pep:`649` was written by Larry Hastings.) -You will likely be able to remove quoted strings in annotations, which are frequently -used for forward references. Similarly, if you use ``from __future__ import annotations`` -to avoid having to write strings in annotations, you may well be able to -remove that import. However, if you rely on third-party libraries that read annotations, -those libraries may need changes to support unquoted annotations before they -work as expected. +.. seealso:: -Implications for readers of ``__annotations__`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + :pep:`649` + Deferred Evaluation Of Annotations Using Descriptors + :pep:`749` + Implementing PEP 649 -If your code reads the ``__annotations__`` attribute on objects, you may want -to make changes in order to support code that relies on deferred evaluation of -annotations. For example, you may want to use :func:`annotationlib.get_annotations` -with the :attr:`~annotationlib.Format.FORWARDREF` format, as the :mod:`dataclasses` -module now does. -Related changes -^^^^^^^^^^^^^^^ +.. _whatsnew314-multiple-interpreters: -The changes in Python 3.14 are designed to rework how ``__annotations__`` -works at runtime while minimizing breakage to code that contains -annotations in source code and to code that reads ``__annotations__``. However, -if you rely on undocumented details of the annotation behavior or on private -functions in the standard library, there are many ways in which your code may -not work in Python 3.14. To safeguard your code against future changes, -use only the documented functionality of the :mod:`annotationlib` module. +:pep:`734`: Multiple interpreters in the standard library +--------------------------------------------------------- -``from __future__ import annotations`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The CPython runtime supports running multiple copies of Python in the +same process simultaneously and has done so for over 20 years. +Each of these separate copies is called an 'interpreter'. +However, the feature had been available only through +the :ref:`C-API `. -In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` -directive, which turns all annotations into strings. This directive is now -considered deprecated and it is expected to be removed in a future version of Python. -However, this removal will not happen until after Python 3.13, the last version of -Python without deferred evaluation of annotations, reaches its end of life in 2029. -In Python 3.14, the behavior of code using ``from __future__ import annotations`` -is unchanged. +That limitation is removed in Python 3.14, +with the new :mod:`concurrent.interpreters` module. +There are at least two notable reasons why using multiple interpreters +has significant benefits: + +* they support a new (to Python), human-friendly concurrency model +* true multi-core parallelism + +For some use cases, concurrency in software improves efficiency and +can simplify design, at a high level. +At the same time, implementing and maintaining all but the simplest concurrency +is often a struggle for the human brain. +That especially applies to plain threads (for example, :mod:`threading`), +where all memory is shared between all threads. + +With multiple isolated interpreters, you can take advantage of a class +of concurrency models, like Communicating Sequential Processes (CSP) +or the actor model, that have found +success in other programming languages, like Smalltalk, Erlang, +Haskell, and Go. Think of multiple interpreters as threads +but with opt-in sharing. + +Regarding multi-core parallelism: as of Python 3.12, interpreters +are now sufficiently isolated from one another to be used in parallel +(see :pep:`684`). This unlocks a variety of CPU-intensive use cases +for Python that were limited by the :term:`GIL`. + +Using multiple interpreters is similar in many ways to +:mod:`multiprocessing`, in that they both provide isolated logical +"processes" that can run in parallel, with no sharing by default. +However, when using multiple interpreters, an application will use +fewer system resources and will operate more efficiently (since it +stays within the same process). Think of multiple interpreters as +having the isolation of processes with the efficiency of threads. + +.. XXX Add an example or two. +.. XXX Link to the not-yet-added HOWTO doc. + +While the feature has been around for decades, multiple interpreters +have not been used widely, due to low awareness and the lack of a +standard library module. Consequently, they currently have several +notable limitations, which are expected to improve significantly now +that the feature is going mainstream. + +Current limitations: + +* starting each interpreter has not been optimized yet +* each interpreter uses more memory than necessary + (work continues on extensive internal sharing between interpreters) +* there aren't many options *yet* for truly sharing objects or other + data between interpreters (other than :type:`memoryview`) +* many third-party extension modules on PyPI are not yet compatible + with multiple interpreters + (all standard library extension modules *are* compatible) +* the approach to writing applications that use multiple isolated + interpreters is mostly unfamiliar to Python users, for now + +The impact of these limitations will depend on future CPython +improvements, how interpreters are used, and what the community solves +through PyPI packages. Depending on the use case, the limitations may +not have much impact, so try it out! + +Furthermore, future CPython releases will reduce or eliminate overhead +and provide utilities that are less appropriate on PyPI. In the +meantime, most of the limitations can also be addressed through +extension modules, meaning PyPI packages can fill any gap for 3.14, and +even back to 3.12 where interpreters were finally properly isolated and +stopped sharing the :term:`GIL`. Likewise, libraries on PyPI are expected +to emerge for high-level abstractions on top of interpreters. + +Regarding extension modules, work is in progress to update some PyPI +projects, as well as tools like Cython, pybind11, nanobind, and PyO3. +The steps for isolating an extension module are found at +:ref:`isolating-extensions-howto`. +Isolating a module has a lot of overlap with what is required to support +:ref:`free-threading `, so the ongoing +work in the community in that area will help accelerate support +for multiple interpreters. + +Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor +`. + +(Contributed by Eric Snow in :gh:`134939`.) + +.. seealso:: :pep:`734` + + +.. _whatsnew314-template-string-literals: + +:pep:`750`: Template string literals +------------------------------------ + +Template strings are a new mechanism for custom string processing. +They share the familiar syntax of f-strings but, unlike f-strings, +return an object representing the static and interpolated parts of +the string, instead of a simple :class:`str`. + +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``: + +.. doctest:: + + >>> variety = 'Stilton' + >>> template = t'Try some {variety} cheese!' + >>> type(template) + + +:class:`~string.templatelib.Template` objects provide access to the static +and interpolated (in curly braces) parts of a string *before* they are combined. +Iterate over :class:`!Template` instances to access their parts in order: + +.. testsetup:: + + variety = 'Stilton' + template = t'Try some {variety} cheese!' + +.. doctest:: + + >>> list(template) + ['Try some ', Interpolation('Stilton', 'variety', None, ''), ' cheese!'] + +It's easy to write (or call) code to process :class:`!Template` instances. +For example, here's a function that renders static parts lowercase and +:class:`~string.templatelib.Interpolation` instances uppercase: + +.. code-block:: python + + from string.templatelib import Interpolation + + def lower_upper(template): + """Render static parts lowercase and interpolations uppercase.""" + parts = [] + for part in template: + if isinstance(part, Interpolation): + parts.append(str(part.value).upper()) + else: + parts.append(part.lower()) + return ''.join(parts) + + name = 'Wenslydale' + template = t'Mister {name}' + assert lower_upper(template) == 'mister WENSLYDALE' + +Because :class:`!Template` instances distinguish between static strings and +interpolations at runtime, they can be useful for sanitising user input. +Writing a :func:`!html` function that escapes user input in HTML is an exercise +left to the reader! +Template processing code can provide improved flexibility. +For instance, a more advanced :func:`!html` function could accept +a :class:`!dict` of HTML attributes directly in the template: + +.. code-block:: python + + attributes = {'src': 'limburger.jpg', 'alt': 'lovely cheese'} + template = t'' + assert html(template) == 'lovely cheese' + +Of course, template processing code does not need to return a string-like result. +An even *more* advanced :func:`!html` could return a custom type representing +a DOM-like structure. + +With t-strings in place, developers can write systems that sanitise SQL, +make safe shell operations, improve logging, tackle modern ideas in web +development (HTML, CSS, and so on), and implement lightweight custom business DSLs. + +(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, +Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, +and Pablo Galindo Salgado in :gh:`132661`.) + +.. seealso:: :pep:`750`. + + +.. _whatsnew314-remote-debugging: + +:pep:`768`: Safe external debugger interface +-------------------------------------------- + +Python 3.14 introduces a zero-overhead debugging interface that allows +debuggers and profilers to safely attach to running Python processes +without stopping or restarting them. +This is a significant enhancement to Python's debugging capabilities, +meaning that unsafe alternatives are no longer required. + +The new interface provides safe execution points for attaching debugger code +without modifying the interpreter's normal execution path +or adding any overhead at runtime. +Due to this, tools can now inspect and interact with Python applications +in real-time, which is a crucial capability for high-availability systems +and production environments. + +For convenience, this interface is implemented in the :func:`sys.remote_exec` +function. For example: + +.. code-block:: python + + import sys + from tempfile import NamedTemporaryFile + + with NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + script_path = f.name + f.write(f'import my_debugger; my_debugger.connect({os.getpid()})') + + # Execute in process with PID 1234 + print('Behold! An offering:') + sys.remote_exec(1234, script_path) + + +This function allows sending Python code to be executed in a target process +at the next safe execution point. +However, tool authors can also implement the protocol directly as described +in the PEP, which details the underlying mechanisms used to safely attach to +running processes. + +The debugging interface has been carefully designed with security in mind +and includes several mechanisms to control access: + +* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. +* A :option:`-X disable-remote-debug` command-line option. +* A :option:`--without-remote-debug` configure flag to completely disable + the feature at build time. + +(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic +in :gh:`131591`.) + +.. seealso:: :pep:`768`. + + +.. _whatsnew314-tail-call-interpreter: + +A new type of interpreter +------------------------- + +A new type of interpreter has been added to CPython. +It uses tail calls between small C functions that implement individual +Python opcodes, rather than one large C ``case`` statement. +For certain newer compilers, this interpreter provides +significantly better performance. Preliminary benchmarks suggest a geometric +mean of 3-5% faster on the standard ``pyperformance`` benchmark suite, +depending on platform and architecture. +The baseline is Python 3.14 built with Clang 19, without this new interpreter. + +This interpreter currently only works with Clang 19 and newer +on x86-64 and AArch64 architectures. +However, a future release of GCC is expected to support this as well. + +This feature is opt-in for now. Enabling profile-guided optimization is highly +recommendeded when using the new interpreter as it is the only configuration +that has been tested and validated for improved performance. +For further information, see :option:`--with-tail-call-interp`. + +.. note:: + + This is not to be confused with `tail call optimization`__ of Python + functions, which is currently not implemented in CPython. + + This new interpreter type is an internal implementation detail of the CPython + interpreter. It doesn't change the visible behavior of Python programs at + all. It can improve their performance, but doesn't change anything else. + + __ https://en.wikipedia.org/wiki/Tail_call + +(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this +in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) + + +.. _whatsnew314-free-threaded-cpython: + +Free-threaded mode improvements +------------------------------- + +CPython's free-threaded mode (:pep:`703`), initially added in 3.13, +has been significantly improved in Python 3.14. +The implementation described in PEP 703 has been finished, including C API +changes, and temporary workarounds in the interpreter were replaced with +more permanent solutions. +The specializing adaptive interpreter (:pep:`659`) is now enabled +in free-threaded mode, which along with many other optimizations +greatly improves its performance. +The performance penalty on single-threaded code in free-threaded mode +is now roughly 5-10%, depending on the platform and C compiler used. + +From Python 3.14, when compiling extension modules for the free-threaded build of +CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to +be specified by the build backend, as it will no longer be determined +automatically by the C compiler. For a running interpreter, the setting that +was used at compile time can be found using :func:`sysconfig.get_config_var`. + +The new :option:`-X context_aware_warnings <-X>` flag controls if +:ref:`concurrent safe warnings control ` +is enabled. The flag defaults to true for the free-threaded build +and false for the GIL-enabled build. + +A new :data:`~sys.flags.thread_inherit_context` flag has been added, +which if enabled means that threads created with :class:`threading.Thread` +start with a copy of the :class:`~contextvars.Context()` of the caller of +:meth:`~threading.Thread.start`. Most significantly, this makes the warning +filtering context established by :class:`~warnings.catch_warnings` be +"inherited" by threads (or asyncio tasks) started within that context. It also +affects other modules that use context variables, such as the :mod:`decimal` +context manager. +This flag defaults to true for the free-threaded build and false for +the GIL-enabled build. + +(Contributed by Sam Gross, Matt Page, Neil Schemenauer, Thomas Wouters, +Donghee Na, Kirill Podoprigora, Ken Jin, Itamar Oren, Brett Simmers, +Dino Viehland, Nathan Goldbaum, Ralf Gommers, Lysandros Nikolaou, Kumar Aditya, +Edgar Margffoy, and many others. +Some of these contributors are employed by Meta, which has continued to provide +significant engineering resources to support this project.) + + +.. _whatsnew314-improved-error-messages: Improved error messages ----------------------- @@ -463,47 +540,12 @@ Improved error messages ^^^^^^ SyntaxError: invalid syntax. Did you mean 'while'? - >>> asynch def fetch_data(): - ... pass - Traceback (most recent call last): - File "", line 1 - asynch def fetch_data(): - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'async'? - - >>> async def foo(): - ... awaid fetch_data() - Traceback (most recent call last): - File "", line 2 - awaid fetch_data() - ^^^^^ - SyntaxError: invalid syntax. Did you mean 'await'? - - >>> raisee ValueError("Error") - Traceback (most recent call last): - File "", line 1 - raisee ValueError("Error") - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'raise'? - While the feature focuses on the most common cases, some variations of misspellings may still result in regular syntax errors. (Contributed by Pablo Galindo in :gh:`132449`.) -* When unpacking assignment fails due to incorrect number of variables, the - error message prints the received number of values in more cases than before. - (Contributed by Tushar Sadhwani in :gh:`122239`.) - - .. code-block:: pycon - - >>> x, y, z = 1, 2, 3, 4 - Traceback (most recent call last): - File "", line 1, in - x, y, z = 1, 2, 3, 4 - ^^^^^^^ - ValueError: too many values to unpack (expected 3, got 4) - -* :keyword:`elif` statements that follow an :keyword:`else` block now have a specific error message. +* :keyword:`elif` statements that follow an :keyword:`else` block now have + a specific error message. (Contributed by Steele Farnsworth in :gh:`129902`.) .. code-block:: pycon @@ -519,11 +561,9 @@ Improved error messages ^^^^ SyntaxError: 'elif' block follows an 'else' block -* If a statement (:keyword:`pass`, :keyword:`del`, :keyword:`return`, - :keyword:`yield`, :keyword:`raise`, :keyword:`break`, :keyword:`continue`, - :keyword:`assert`, :keyword:`import`, :keyword:`from`) is passed to the - :ref:`if_expr` after :keyword:`else`, or one of :keyword:`pass`, - :keyword:`break`, or :keyword:`continue` is passed before :keyword:`if`, then the +* If a statement is passed to the :ref:`if_expr` after :keyword:`else`, + or one of :keyword:`pass`, :keyword:`break`, or :keyword:`continue` + is passed before :keyword:`if`, then the error message highlights where the :token:`~python-grammar:expression` is required. (Contributed by Sergey Miryanov in :gh:`129515`.) @@ -543,10 +583,9 @@ Improved error messages ^^^^^^^^ SyntaxError: expected expression before 'if', but statement is given - * When incorrectly closed strings are detected, the error message suggests - that the string may be intended to be part of the string. (Contributed by - Pablo Galindo in :gh:`88535`.) + that the string may be intended to be part of the string. + (Contributed by Pablo Galindo in :gh:`88535`.) .. code-block:: pycon @@ -555,8 +594,8 @@ Improved error messages SyntaxError: invalid syntax. Is this intended to be part of the string? * When strings have incompatible prefixes, the error now shows - which prefixes are incompatible. (Contributed by - Nikita Sobolev in :gh:`133197`.) + which prefixes are incompatible. + (Contributed by Nikita Sobolev in :gh:`133197`.) .. code-block:: pycon @@ -573,80 +612,101 @@ Improved error messages - Except handlers: ``except ... as ...`` - Pattern-match cases: ``case ... as ...`` - (Contributed by Nikita Sobolev in :gh:`123539`, - :gh:`123562`, and :gh:`123440`.) + (Contributed by Nikita Sobolev in :gh:`123539`, :gh:`123562`, and :gh:`123440`.) + +* Improved error message when trying to add an instance of an unhashable type to + a :class:`dict` or :class:`set`. + (Contributed by CF Bolz-Tereick and Victor Stinner in :gh:`132828`.) .. code-block:: pycon - >>> import ast as arr[0] - File "", line 1 - import ast as arr[0] - ^^^^^^ - SyntaxError: cannot use subscript as import target + >>> s = set() + >>> s.add({'pages': 12, 'grade': 'A'}) + Traceback (most recent call last): + File "", line 1, in + s.add({'pages': 12, 'grade': 'A'}) + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + TypeError: cannot use 'dict' as a set element (unhashable type: 'dict') + >>> d = {} + >>> l = [1, 2, 3] + >>> d[l] = 12 + Traceback (most recent call last): + File "", line 1, in + d[l] = 12 + ~^^^ + TypeError: cannot use 'list' as a dict key (unhashable type: 'list') -.. seealso:: - :pep:`649`. +* Improved error message when an object supporting the synchronous + context manager protocol is entered using :keyword:`async with` + instead of :keyword:`with`, + and vice versa for the asynchronous context manager protocol. + (Contributed by Bénédikt Tran in :gh:`128398`.) -.. _whatsnew314-pep741: +.. _whatsnew314-zstandard: -PEP 741: Python Configuration C API ------------------------------------ +:pep:`784`: Zstandard support in the standard library +----------------------------------------------------- -Add a :ref:`PyInitConfig C API ` to configure the Python -initialization without relying on C structures and the ability to make -ABI-compatible changes in the future. +The new :mod:`!compression` package contains modules :mod:`!compression.lzma`, +:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` +which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` +modules respectively. The new import names under :mod:`!compression` are the +preferred names for importing these compression modules from Python 3.14. However, +the existing modules names have not been deprecated. Any deprecation or removal +of the existing compression modules will occur no sooner than five years after +the release of 3.14. -Complete the :pep:`587` :ref:`PyConfig C API ` by adding -:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension -module; feature previously referred to as the “inittab”. +The new :mod:`!compression.zstd` module provides compression and decompression +APIs for the Zstandard format via bindings to `Meta's zstd library +`__. Zstandard is a widely adopted, highly +efficient, and fast compression format. In addition to the APIs introduced in +:mod:`!compression.zstd`, support for reading and writing Zstandard compressed +archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and +:mod:`shutil` modules. -Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set -the current runtime configuration. +Here's an example of using the new module to compress some data: -PEP 587 “Python Initialization Configuration” unified all the ways to configure -the Python initialization. This PEP unifies also the configuration of the -Python preinitialization and the Python initialization in a single API. -Moreover, this PEP only provides a single choice to embed Python, instead of -having two “Python” and “Isolated” choices (PEP 587), to simplify the API -further. +.. code-block:: python -The lower level PEP 587 PyConfig API remains available for use cases with an -intentionally higher level of coupling to CPython implementation details (such -as emulating the full functionality of CPython’s CLI, including its -configuration mechanisms). + from compression import zstd + import math -(Contributed by Victor Stinner in :gh:`107954`.) + data = str(math.pi).encode() * 20 + compressed = zstd.compress(data) + ratio = len(compressed) / len(data) + print(f"Achieved compression ratio of {ratio}") + +As can be seen, the API is similar to the APIs of the :mod:`!lzma` and +:mod:`!bz2` modules. + +(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, +Victor Stinner, and Rogdham in :gh:`132983`.) + +.. seealso:: :pep:`784`. -.. seealso:: - :pep:`741`. .. _whatsnew314-asyncio-introspection: Asyncio introspection capabilities ---------------------------------- -Added a new command-line interface to inspect running Python processes using -asynchronous tasks, available via: +Added a new command-line interface to inspect running Python processes +using asynchronous tasks, available via ``python -m asyncio ps PID`` +or ``python -m asyncio pstree PID``. -.. code-block:: bash +The ``ps`` subcommand inspects the given process ID (PID) and displays +information about currently running asyncio tasks. +It outputs a task table: a flat listing of all tasks, their names, +their coroutine stacks, and which tasks are awaiting them. - python -m asyncio ps PID -This tool inspects the given process ID (PID) and displays information about -currently running asyncio tasks. It outputs a task table: a flat -listing of all tasks, their names, their coroutine stacks, and which tasks are -awaiting them. - -.. code-block:: bash - - python -m asyncio pstree PID - -This tool fetches the same information, but renders a visual async call tree, -showing coroutine relationships in a hierarchical format. This command is -particularly useful for debugging long-running or stuck asynchronous programs. -It can help developers quickly identify where a program is blocked, what tasks -are pending, and how coroutines are chained together. +The ``pstree`` subcommand fetches the same information, but instead renders a +visual async call tree, showing coroutine relationships in a hierarchical format. +This command is particularly useful for debugging long-running or stuck +asynchronous programs. +It can help developers quickly identify where a program is blocked, +what tasks are pending, and how coroutines are chained together. For example given this code: @@ -654,23 +714,25 @@ For example given this code: import asyncio - async def play(track): + async def play_track(track): await asyncio.sleep(5) - print(f"🎵 Finished: {track}") + print(f'🎵 Finished: {track}') - async def album(name, tracks): + async def play_album(name, tracks): async with asyncio.TaskGroup() as tg: for track in tracks: - tg.create_task(play(track), name=track) + tg.create_task(play_track(track), name=track) async def main(): async with asyncio.TaskGroup() as tg: tg.create_task( - album("Sundowning", ["TNDNBTG", "Levitate"]), name="Sundowning") + play_album('Sundowning', ['TNDNBTG', 'Levitate']), + name='Sundowning') tg.create_task( - album("TMBTE", ["DYWTYLM", "Aqua Regia"]), name="TMBTE") + play_album('TMBTE', ['DYWTYLM', 'Aqua Regia']), + name='TMBTE') - if __name__ == "__main__": + if __name__ == '__main__': asyncio.run(main()) Executing the new tool on the running process will yield a table like this: @@ -679,277 +741,298 @@ Executing the new tool on the running process will yield a table like this: python -m asyncio ps 12345 - tid task id task name coroutine chain awaiter name awaiter id - --------------------------------------------------------------------------------------------------------------------------------------- - 8138752 0x564bd3d0210 Task-1 0x0 - 8138752 0x564bd3d0410 Sundowning _aexit -> __aexit__ -> main Task-1 0x564bd3d0210 - 8138752 0x564bd3d0610 TMBTE _aexit -> __aexit__ -> main Task-1 0x564bd3d0210 - 8138752 0x564bd3d0810 TNDNBTG _aexit -> __aexit__ -> album Sundowning 0x564bd3d0410 - 8138752 0x564bd3d0a10 Levitate _aexit -> __aexit__ -> album Sundowning 0x564bd3d0410 - 8138752 0x564bd3e0550 DYWTYLM _aexit -> __aexit__ -> album TMBTE 0x564bd3d0610 - 8138752 0x564bd3e0710 Aqua Regia _aexit -> __aexit__ -> album TMBTE 0x564bd3d0610 + tid task id task name coroutine stack awaiter chain awaiter name awaiter id + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 1935500 0x7fc930c18050 Task-1 TaskGroup._aexit -> TaskGroup.__aexit__ -> main 0x0 + 1935500 0x7fc930c18230 Sundowning TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x7fc930c18050 + 1935500 0x7fc93173fa50 TMBTE TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x7fc930c18050 + 1935500 0x7fc93173fdf0 TNDNBTG sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x7fc930c18230 + 1935500 0x7fc930d32510 Levitate sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x7fc930c18230 + 1935500 0x7fc930d32890 DYWTYLM sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x7fc93173fa50 + 1935500 0x7fc93161ec30 Aqua Regia sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x7fc93173fa50 - -or: +or a tree like this: .. code-block:: bash python -m asyncio pstree 12345 └── (T) Task-1 - └── main - └── __aexit__ - └── _aexit + └── main example.py:13 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:72 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:121 ├── (T) Sundowning - │ └── album - │ └── __aexit__ - │ └── _aexit + │ └── album example.py:8 + │ └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:72 + │ └── TaskGroup._aexit Lib/asyncio/taskgroups.py:121 │ ├── (T) TNDNBTG + │ │ └── play example.py:4 + │ │ └── sleep Lib/asyncio/tasks.py:702 │ └── (T) Levitate + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 └── (T) TMBTE - └── album - └── __aexit__ - └── _aexit + └── album example.py:8 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:72 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:121 ├── (T) DYWTYLM + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 └── (T) Aqua Regia + └── play example.py:4 + └── sleep Lib/asyncio/tasks.py:702 If a cycle is detected in the async await graph (which could indicate a programming issue), the tool raises an error and lists the cycle paths that -prevent tree construction. +prevent tree construction: + +.. code-block:: bash + + python -m asyncio pstree 12345 + + ERROR: await-graph contains cycles - cannot print a tree! + + cycle: Task-2 → Task-3 → Task-2 (Contributed by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta Gomez Macias in :gh:`91048`.) -.. _whatsnew314-tail-call: -A new type of interpreter -------------------------- +.. _whatsnew314-concurrent-warnings-control: -A new type of interpreter has been added to CPython. -It uses tail calls between small C functions that implement individual -Python opcodes, rather than one large C case statement. -For certain newer compilers, this interpreter provides -significantly better performance. Preliminary numbers on our machines suggest -anywhere up to 30% faster Python code, and a geometric mean of 3-5% -faster on ``pyperformance`` depending on platform and architecture. The -baseline is Python 3.14 built with Clang 19 without this new interpreter. +Concurrent safe warnings control +-------------------------------- -This interpreter currently only works with Clang 19 and newer -on x86-64 and AArch64 architectures. However, we expect -that a future release of GCC will support this as well. +The :class:`warnings.catch_warnings` context manager will now optionally +use a context variable for warning filters. This is enabled by setting +the :data:`~sys.flags.context_aware_warnings` flag, either with the ``-X`` +command-line option or an environment variable. This gives predictable +warnings control when using :class:`~warnings.catch_warnings` combined with +multiple threads or asynchronous tasks. The flag defaults to true for the +free-threaded build and false for the GIL-enabled build. -This feature is opt-in for now. We highly recommend enabling profile-guided -optimization with the new interpreter as it is the only configuration we have -tested and can validate its improved performance. -For further information on how to build Python, see -:option:`--with-tail-call-interp`. - -.. note:: - - This is not to be confused with `tail call optimization`__ of Python - functions, which is currently not implemented in CPython. - - This new interpreter type is an internal implementation detail of the CPython - interpreter. It doesn't change the visible behavior of Python programs at - all. It can improve their performance, but doesn't change anything else. - - __ https://en.wikipedia.org/wiki/Tail_call - -.. attention:: - - This section previously reported a 9-15% geometric mean speedup. This number has since been - cautiously revised down to 3-5%. While we expect performance results to be better - than what we report, our estimates are more conservative due to a - `compiler bug `_ found in - Clang/LLVM 19, which causes the normal interpreter to be slower. We were unaware of this bug, - resulting in inaccurate results. We sincerely apologize for - communicating results that were only accurate for LLVM v19.1.x and v20.1.0. In the meantime, - the bug has been fixed in LLVM v20.1.1 and for the upcoming v21.1, but it will remain - unfixed for LLVM v19.1.x and v20.1.0. Thus - any benchmarks with those versions of LLVM may produce inaccurate numbers. - (Thanks to Nelson Elhage for bringing this to light.) - -(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this -in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) - - -.. _whatsnew314-pyrepl-highlighting: - -Syntax highlighting in PyREPL ------------------------------ - -The default :term:`interactive` shell now highlights Python syntax as you -type. The feature is enabled by default unless the -:envvar:`PYTHON_BASIC_REPL` environment is set or any color-disabling -environment variables are used. See :ref:`using-on-controlling-color` for -details. - -The default color theme for syntax highlighting strives for good contrast -and uses exclusively the 4-bit VGA standard ANSI color codes for maximum -compatibility. The theme can be customized using an experimental API -``_colorize.set_theme()``. This can be called interactively, as well as -in the :envvar:`PYTHONSTARTUP` script. - -(Contributed by Łukasz Langa in :gh:`131507`.) - - -.. _whatsnew314-jit-compiler: - -Binary releases for the experimental just-in-time compiler ----------------------------------------------------------- - -The official macOS and Windows release binaries now include an *experimental* -just-in-time (JIT) compiler. Although it is **not** recommended for production -use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an -environment variable. Downstream source builds and redistributors can use the -:option:`--enable-experimental-jit=yes-off` configuration option for similar -behavior. - -The JIT is at an early stage and still in active development. As such, the -typical performance impact of enabling it can range from 10% slower to 20% -faster, depending on workload. To aid in testing and evaluation, a set of -introspection functions has been provided in the :data:`sys._jit` namespace. -:func:`sys._jit.is_available` can be used to determine if the current executable -supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell -if JIT compilation has been enabled for the current process. - -Currently, the most significant missing functionality is that native debuggers -and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames -(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to -work without modification). Free-threaded builds do not support JIT compilation. - -Please report any bugs or major performance regressions that you encounter! - -.. seealso:: :pep:`744` +(Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) Other language changes ====================== -* The default :term:`interactive` shell now supports import autocompletion. - This means that typing ``import foo`` and pressing ```` will suggest - modules starting with ``foo``. Similarly, typing ``from foo import b`` will - suggest submodules of ``foo`` starting with ``b``. Note that autocompletion - of module attributes is not currently supported. - (Contributed by Tomas Roun in :gh:`69605`.) +* All Windows code pages are now supported as 'cpXXX' codecs on Windows. + (Contributed by Serhiy Storchaka in :gh:`123803`.) -* The :func:`map` built-in now has an optional keyword-only *strict* flag - like :func:`zip` to check that all the iterables are of equal length. - (Contributed by Wannes Boeykens in :gh:`119793`.) +* Implement mixed-mode arithmetic rules combining real and complex numbers + as specified by the C standard since C99. + (Contributed by Sergey B Kirpichev in :gh:`69639`.) -* Incorrect usage of :keyword:`await` and asynchronous comprehensions - is now detected even if the code is optimized away by the :option:`-O` - command-line option. For example, ``python -O -c 'assert await 1'`` - now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.) +* More syntax errors are now detected regardless of optimisation and + the :option:`-O` command-line option. + This includes writes to ``__debug__``, incorrect use of :keyword:`await`, + and asynchronous comprehensions outside asynchronous functions. + For example, ``python -O -c 'assert (__debug__ := 1)'`` + or ``python -O -c 'assert await 1'`` now produce :exc:`SyntaxError`\ s. + (Contributed by Irit Katriel and Jelle Zijlstra in :gh:`122245` & :gh:`121637`.) -* Writes to ``__debug__`` are now detected even if the code is optimized - away by the :option:`-O` command-line option. For example, - ``python -O -c 'assert (__debug__ := 1)'`` now produces a - :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) +* When subclassing a pure C type, the C slots for the new type + are no longer replaced with a wrapped version on class creation + if they are not explicitly overridden in the subclass. + (Contributed by Tomasz Pytel in :gh:`132284`.) + + +Built-ins +--------- + +* The :meth:`bytes.fromhex` and :meth:`bytearray.fromhex` methods now accept + ASCII :class:`bytes` and :term:`bytes-like objects `. + (Contributed by Daniel Pope in :gh:`129349`.) * Add class methods :meth:`float.from_number` and :meth:`complex.from_number` to convert a number to :class:`float` or :class:`complex` type correspondingly. - They raise an error if the argument is a string. + They raise a :exc:`TypeError` if the argument is not a real number. (Contributed by Serhiy Storchaka in :gh:`84978`.) -* Implement mixed-mode arithmetic rules combining real and complex numbers as - specified by C standards since C99. - (Contributed by Sergey B Kirpichev in :gh:`69639`.) - -* All Windows code pages are now supported as "cpXXX" codecs on Windows. - (Contributed by Serhiy Storchaka in :gh:`123803`.) - -* :class:`super` objects are now :mod:`pickleable ` and - :mod:`copyable `. - (Contributed by Serhiy Storchaka in :gh:`125767`.) - -* The :class:`memoryview` type now supports subscription, - making it a :term:`generic type`. - (Contributed by Brian Schubert in :gh:`126012`.) - * Support underscore and comma as thousands separators in the fractional part for floating-point presentation types of the new-style string formatting (with :func:`format` or :ref:`f-strings`). (Contributed by Sergey B Kirpichev in :gh:`87790`.) -* The :func:`bytes.fromhex` and :func:`bytearray.fromhex` methods now accept - ASCII :class:`bytes` and :term:`bytes-like objects `. - (Contributed by Daniel Pope in :gh:`129349`.) +* The :func:`int` function no longer delegates to :meth:`~object.__trunc__`. + Classes that want to support conversion to :func:`!int` must implement + either :meth:`~object.__int__` or :meth:`~object.__index__`. + (Contributed by Mark Dickinson in :gh:`119743`.) -* Support ``\z`` as a synonym for ``\Z`` in :mod:`regular expressions `. - It is interpreted unambiguously in many other regular expression engines, - unlike ``\Z``, which has subtly different behavior. - (Contributed by Serhiy Storchaka in :gh:`133306`.) +* The :func:`map` function now has an optional keyword-only *strict* flag + like :func:`zip` to check that all the iterables are of equal length. + (Contributed by Wannes Boeykens in :gh:`119793`.) -* ``\B`` in :mod:`regular expression ` now matches empty input string. - Now it is always the opposite of ``\b``. - (Contributed by Serhiy Storchaka in :gh:`124130`.) +* The :class:`memoryview` type now supports subscription, + making it a :term:`generic type`. + (Contributed by Brian Schubert in :gh:`126012`.) -* iOS and macOS apps can now be configured to redirect ``stdout`` and - ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in - :gh:`127592`.) +* Using :data:`NotImplemented` in a boolean context + will now raise a :exc:`TypeError`. + This has raised a :exc:`DeprecationWarning` since Python 3.9. + (Contributed by Jelle Zijlstra in :gh:`118767`.) -* The iOS testbed is now able to stream test output while the test is running. - The testbed can also be used to run the test suite of projects other than - CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) - -* Three-argument :func:`pow` now tries calling :meth:`~object.__rpow__` if - necessary. Previously it was only called in two-argument :func:`!pow` and the - binary power operator. +* Three-argument :func:`pow` now tries calling :meth:`~object.__rpow__` + if necessary. + Previously it was only called in two-argument :func:`!pow` + and the binary power operator. (Contributed by Serhiy Storchaka in :gh:`130104`.) -* Add a built-in implementation for HMAC (:rfc:`2104`) using formally verified - code from the `HACL* `__ project. - This implementation is used as a fallback when the OpenSSL implementation - of HMAC is not available. - (Contributed by Bénédikt Tran in :gh:`99108`.) +* :class:`super` objects are now :mod:`copyable ` and :mod:`pickleable + `. + (Contributed by Serhiy Storchaka in :gh:`125767`.) + + +Command line and environment +---------------------------- * The import time flag can now track modules that are already loaded ('cached'), via the new :option:`-X importtime=2 <-X>`. When such a module is imported, the ``self`` and ``cumulative`` times are replaced by the string ``cached``. - Values above ``2`` for ``-X importtime`` are now reserved for future use. - (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) -* When subclassing from a pure C type, the C slots for the new type are no - longer replaced with a wrapped version on class creation if they are not - explicitly overridden in the subclass. - (Contributed by Tomasz Pytel in :gh:`132329`.) + Values above ``2`` for ``-X importtime`` are now reserved for future use. + + (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) * The command-line option :option:`-c` now automatically dedents its code argument before execution. The auto-dedentation behavior mirrors :func:`textwrap.dedent`. (Contributed by Jon Crall and Steven Sun in :gh:`103998`.) -* Improve error message when an object supporting the synchronous - context manager protocol is entered using :keyword:`async - with` instead of :keyword:`with`. - And vice versa with the asynchronous context manager protocol. - (Contributed by Bénédikt Tran in :gh:`128398`.) - * :option:`!-J` is no longer a reserved flag for Jython_, and now has no special meaning. (Contributed by Adam Turner in :gh:`133336`.) .. _Jython: https://www.jython.org/ -.. _whatsnew314-pep765: -PEP 765: Disallow ``return``/``break``/``continue`` that exit a ``finally`` block ---------------------------------------------------------------------------------- +.. _whatsnew314-bracketless-except: -The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or -:keyword:`continue` statements appears where it exits a :keyword:`finally` block. +PEP 758: Allow ``except`` and ``except*`` expressions without brackets +---------------------------------------------------------------------- + +The :keyword:`except` and :keyword:`except* ` expressions +now allow brackets to be omitted when there are multiple exception types +and the ``as`` clause is not used. +For example: + +.. code-block:: python + + try: + connect_to_server() + except TimeoutError, ConnectionRefusedError: + print('The network has ceased to be!') + +(Contributed by Pablo Galindo and Brett Cannon in :pep:`758` and :gh:`131831`.) + + +.. _whatsnew314-finally-syntaxwarning: + +PEP 765: Control flow in :keyword:`finally` blocks +-------------------------------------------------- + +The compiler now emits a :exc:`SyntaxWarning` when a :keyword:`return`, +:keyword:`break`, or :keyword:`continue` statement have the effect of +leaving a :keyword:`finally` block. This change is specified in :pep:`765`. +In situations where this change is inconvenient (such as those where the +warnings are redundant due to code linting), the :ref:`warning filter +` can be used to turn off all syntax warnings by adding +``ignore::SyntaxWarning`` as a filter. This can be specified in combination +with a filter that converts other warnings to errors (for example, passing +``-Werror -Wignore::SyntaxWarning`` as CLI options, or setting +``PYTHONWARNINGS=error,ignore::SyntaxWarning``). + +Note that applying such a filter at runtime using the :mod:`warnings` module +will only suppress the warning in code that is compiled *after* the filter is +adjusted. Code that is compiled prior to the filter adjustment (for example, +when a module is imported) will still emit the syntax warning. + +(Contributed by Irit Katriel in :gh:`130080`.) + + +.. _whatsnew314-incremental-gc: + +Incremental garbage collection +------------------------------ + +The cycle garbage collector is now incremental. +This means that maximum pause times are reduced +by an order of magnitude or more for larger heaps. + +There are now only two generations: young and old. +When :func:`gc.collect` is not called directly, the +GC is invoked a little less frequently. When invoked, it +collects the young generation and an increment of the +old generation, instead of collecting one or more generations. + +The behavior of :func:`!gc.collect` changes slightly: + +* ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. +* Other calls to :func:`!gc.collect` are unchanged. + +(Contributed by Mark Shannon in :gh:`108362`.) + + +Default interactive shell +------------------------- + +.. _whatsnew314-pyrepl-highlighting: + +* The default :term:`interactive` shell now highlights Python syntax. + The feature is enabled by default, save if :envvar:`PYTHON_BASIC_REPL` + or any other environment variable that disables colour is set. + See :ref:`using-on-controlling-color` for details. + + The default color theme for syntax highlighting strives for good contrast + and exclusively uses the 4-bit VGA standard ANSI color codes for maximum + compatibility. The theme can be customized using an experimental API + :func:`!_colorize.set_theme`. + This can be called interactively or in the :envvar:`PYTHONSTARTUP` script. + Note that this function has no stability guarantees, + and may change or be removed. + + (Contributed by Łukasz Langa in :gh:`131507`.) + +* The default :term:`interactive` shell now supports import auto-completion. + This means that typing ``import co`` and pressing :kbd:`` will suggest + modules starting with ``co``. Similarly, typing ``from concurrent import i`` + will suggest submodules of ``concurrent`` starting with ``i``. + Note that autocompletion of module attributes is not currently supported. + (Contributed by Tomas Roun in :gh:`69605`.) + New modules =========== -* :mod:`annotationlib`: For introspecting :term:`annotations `. - See :pep:`749` for more details. +* :mod:`annotationlib`: + For introspecting :term:`annotations `. + See :ref:`PEP 749 ` for more details. (Contributed by Jelle Zijlstra in :gh:`119180`.) +* :mod:`compression` (including :mod:`compression.zstd`): + A package for compression-related modules, + including a new module to support the Zstandard compression format. + See :ref:`PEP 784 ` for more details. + (Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, + Victor Stinner, and Rogdham in :gh:`132983`.) + +* :mod:`concurrent.interpreters`: + Support for multiple interpreters in the standard library. + See :ref:`PEP 734 ` for more details. + (Contributed by Eric Snow in :gh:`134939`.) + +* :mod:`string.templatelib`: + Support for template string literals (t-strings). + See :ref:`PEP 750 ` for more details. + (Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, + Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, + and Pablo Galindo Salgado in :gh:`132661`.) + Improved modules ================ @@ -967,20 +1050,17 @@ argparse and subparser names if mistyped by the user. (Contributed by Savannah Ostrowski in :gh:`124456`.) - .. _whatsnew314-color-argparse: - -* Introduced the optional *color* parameter to - :class:`argparse.ArgumentParser`, enabling color for help text. - This can be controlled by :ref:`environment variables - `. Color has also been enabled for help in the - :ref:`stdlib CLIs ` which use :mod:`!argparse`. +* Enable color for help text, which can be disabled with the optional *color* + parameter to :class:`argparse.ArgumentParser`. + This can also be controlled by :ref:`environment variables + `. (Contributed by Hugo van Kemenade in :gh:`130645`.) ast --- -* Add :func:`ast.compare` for comparing two ASTs. +* Add :func:`~ast.compare`, a function for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :gh:`60191`.) * Add support for :func:`copy.replace` for AST nodes. @@ -989,30 +1069,52 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) -* The ``repr()`` output for AST nodes now includes more information. +* The :func:`repr` output for AST nodes now includes more information. (Contributed by Tomas Roun in :gh:`116022`.) -* :func:`ast.parse`, when called with an AST as input, now always verifies - that the root node type is appropriate. +* When called with an AST as input, the :func:`~ast.parse` function + now always verifies that the root node type is appropriate. (Contributed by Irit Katriel in :gh:`130139`.) -* Add new ``--feature-version``, ``--optimize``, ``--show-empty`` options to - command-line interface. +* Add new options to the command-line interface: + :option:`--feature-version `, + :option:`--optimize `, and + :option:`--show-empty `. (Contributed by Semyon Moroz in :gh:`133367`.) -bdb ---- +asyncio +------- -* The :mod:`bdb` module now supports the :mod:`sys.monitoring` backend. - (Contributed by Tian Gao in :gh:`124533`.) +* The function and methods named :func:`!create_task` now take an arbitrary + list of keyword arguments. All keyword arguments are passed to the + :class:`~asyncio.Task` constructor or the custom task factory. + (See :meth:`~asyncio.loop.set_task_factory` for details.) + The ``name`` and ``context`` keyword arguments are no longer special; + the name should now be set using the ``name`` keyword argument of the factory, + and ``context`` may be ``None``. + This affects the following function and methods: + :meth:`asyncio.create_task`, + :meth:`asyncio.loop.create_task`, + :meth:`asyncio.TaskGroup.create_task`. + + (Contributed by Thomas Grainger in :gh:`128307`.) + +* There are two new utility functions for + introspecting and printing a program's call graph: + :func:`~asyncio.capture_call_graph` and :func:`~asyncio.print_call_graph`. + See :ref:`Asyncio introspection capabilities + ` for more details. + (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa + in :gh:`91048`.) - .. _whatsnew314-color-calendar: calendar -------- +.. _whatsnew314-color-calendar: + * By default, today's date is highlighted in color in :mod:`calendar`'s :ref:`command-line ` text output. This can be controlled by :ref:`environment variables @@ -1023,19 +1125,27 @@ calendar concurrent.futures ------------------ -* Add :class:`~concurrent.futures.InterpreterPoolExecutor`, - which exposes "subinterpreters" (multiple Python interpreters in the - same process) to Python code. This is separate from the proposed API - in :pep:`734`. +.. _whatsnew314-concurrent-futures-interp-pool: + +* Add a new executor class, :class:`~concurrent.futures.InterpreterPoolExecutor`, + which exposes multiple Python interpreters in the same process + ('subinterpreters') to Python code. + This uses a pool of independent Python interpreters to execute calls + asynchronously. + + This is separate from the new :mod:`~concurrent.interpreters` module + introduced by :ref:`PEP 734 `. (Contributed by Eric Snow in :gh:`124548`.) .. _whatsnew314-concurrent-futures-start-method: -* The default :class:`~concurrent.futures.ProcessPoolExecutor` - :ref:`start method ` changed - from :ref:`fork ` to :ref:`forkserver - ` on platforms other than macOS and - Windows where it was already :ref:`spawn `. +* On Unix platforms other than macOS, :ref:`'forkserver' + ` is now the default :ref:`start + method ` for + :class:`~concurrent.futures.ProcessPoolExecutor` + (replacing :ref:`'fork' `). + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly request it by supplying a multiprocessing context *mp_context* to @@ -1048,23 +1158,36 @@ concurrent.futures (Contributed by Gregory P. Smith in :gh:`84559`.) -* Add :meth:`concurrent.futures.ProcessPoolExecutor.terminate_workers` and - :meth:`concurrent.futures.ProcessPoolExecutor.kill_workers` as - ways to terminate or kill all living worker processes in the given pool. +* Add two new methods to :class:`~concurrent.futures.ProcessPoolExecutor`, + :meth:`~concurrent.futures.ProcessPoolExecutor.terminate_workers` + and :meth:`~concurrent.futures.ProcessPoolExecutor.kill_workers`, + as ways to terminate or kill all living worker processes in the given pool. (Contributed by Charles Machalow in :gh:`130849`.) -* Add the optional ``buffersize`` parameter to - :meth:`concurrent.futures.Executor.map` to limit the number of submitted +* Add the optional *buffersize* parameter to :meth:`Executor.map + ` to limit the number of submitted tasks whose results have not yet been yielded. If the buffer is full, iteration over the *iterables* pauses until a result is yielded from the buffer. (Contributed by Enzo Bonnal and Josh Rosenberg in :gh:`74028`.) +configparser +------------ + +* :mod:`!configparser` will no longer write config files it cannot read, + to improve security. + Attempting to :meth:`~configparser.ConfigParser.write` keys containing + delimiters or beginning with the section header pattern will raise an + :class:`~configparser.InvalidWriteError`. + (Contributed by Jacob Lincoln in :gh:`129270`.) + + contextvars ----------- -* Support context manager protocol by :class:`contextvars.Token`. +* Support the :term:`context manager` protocol + for :class:`~contextvars.Token` objects. (Contributed by Andrew Svetlov in :gh:`129889`.) @@ -1072,8 +1195,8 @@ ctypes ------ * The layout of :ref:`bit fields ` - in :class:`~ctypes.Structure` and :class:`~ctypes.Union` - now matches platform defaults (GCC/Clang or MSVC) more closely. + in :class:`~ctypes.Structure` and :class:`~ctypes.Union` objects + is now a closer match to platform defaults (GCC/Clang or MSVC). In particular, fields no longer overlap. (Contributed by Matthias Görgens in :gh:`97702`.) @@ -1092,7 +1215,7 @@ ctypes * On Windows, the :func:`~ctypes.CopyComPointer` function is now public. (Contributed by Jun Komoda in :gh:`127275`.) -* :func:`ctypes.memoryview_at` now exists to create a +* Add :func:`~ctypes.memoryview_at`, a function to create a :class:`memoryview` object that refers to the supplied pointer and length. This works like :func:`ctypes.string_at` except it avoids a buffer copy, and is typically useful when implementing pure Python @@ -1100,7 +1223,7 @@ ctypes (Contributed by Rian Hunter in :gh:`112018`.) * Complex types, :class:`~ctypes.c_float_complex`, - :class:`~ctypes.c_double_complex` and :class:`~ctypes.c_longdouble_complex`, + :class:`~ctypes.c_double_complex`, and :class:`~ctypes.c_longdouble_complex`, are now available if both the compiler and the ``libffi`` library support complex C types. (Contributed by Sergey B Kirpichev in :gh:`61103`.) @@ -1110,47 +1233,57 @@ ctypes (Contributed by Brian Ward in :gh:`119349`.) * Move :func:`ctypes.POINTER` types cache from a global internal cache - (``_pointer_type_cache``) to the :attr:`ctypes._CData.__pointer_type__` - attribute of the corresponding :mod:`ctypes` types. + (``_pointer_type_cache``) to the :attr:`_CData.__pointer_type__ + ` attribute of the corresponding + :mod:`!ctypes` types. This will stop the cache from growing without limits in some situations. (Contributed by Sergey Miryanov in :gh:`100926`.) -* The :class:`ctypes.py_object` type now supports subscription, +* The :class:`~ctypes.py_object` type now supports subscription, making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`132168`.) +* :mod:`!ctypes` now supports :term:`free-threading builds `. + (Contributed by Kumar Aditya and Peter Bierma in :gh:`127945`.) + + curses ------ * Add the :func:`~curses.assume_default_colors` function, a refinement of the :func:`~curses.use_default_colors` function which - allows to change the color pair ``0``. + allows changing the color pair ``0``. (Contributed by Serhiy Storchaka in :gh:`133139`.) + datetime -------- -* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. +* Add the :meth:`~datetime.date.strptime` method to the + :class:`datetime.date` and :class:`datetime.time` classes. (Contributed by Wannes Boeykens in :gh:`41431`.) + decimal ------- -* Add alternative :class:`~decimal.Decimal` constructor - :meth:`Decimal.from_number() `. +* Add :meth:`.Decimal.from_number` as an alternative constructor for + :class:`~decimal.Decimal`. (Contributed by Serhiy Storchaka in :gh:`121798`.) -* Expose :func:`decimal.IEEEContext` to support creation of contexts +* Expose :func:`~decimal.IEEEContext` to support creation of contexts corresponding to the IEEE 754 (2008) decimal interchange formats. (Contributed by Sergey B Kirpichev in :gh:`53032`.) + difflib ------- * Comparison pages with highlighted changes generated by the - :class:`difflib.HtmlDiff` class now support dark mode. + :class:`~difflib.HtmlDiff` class now support 'dark mode'. (Contributed by Jiahao Li in :gh:`129939`.) + dis --- @@ -1175,7 +1308,7 @@ dis errno ----- -* Add :data:`errno.EHWPOISON` error code. +* Add the :data:`~errno.EHWPOISON` error code constant. (Contributed by James Roy in :gh:`126585`.) @@ -1183,39 +1316,43 @@ faulthandler ------------ * Add support for printing the C stack trace on systems that - :ref:`support it ` via :func:`faulthandler.dump_c_stack` - or via the *c_stack* argument in :func:`faulthandler.enable`. + :ref:`support it ` via the new + :func:`~faulthandler.dump_c_stack` function or via the *c_stack* argument + in :func:`faulthandler.enable`. (Contributed by Peter Bierma in :gh:`127604`.) fnmatch ------- -* Added :func:`fnmatch.filterfalse` for excluding names matching a pattern. +* Add :func:`~fnmatch.filterfalse`, a function to reject names + matching a given pattern. (Contributed by Bénédikt Tran in :gh:`74598`.) fractions --------- -* Add support for converting any objects that have the - :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. +* A :class:`~fractions.Fraction` object may now be constructed from any + object with the :meth:`!as_integer_ratio` method. (Contributed by Serhiy Storchaka in :gh:`82017`.) -* Add alternative :class:`~fractions.Fraction` constructor - :meth:`Fraction.from_number() `. +* Add :meth:`.Fraction.from_number` as an alternative constructor for + :class:`~fractions.Fraction`. (Contributed by Serhiy Storchaka in :gh:`121797`.) functools --------- -* Add support to :func:`functools.partial` and - :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels - to reserve a place for positional arguments. +* Add the :data:`~functools.Placeholder` sentinel. + This may be used with the :func:`~functools.partial` + or :func:`~functools.partialmethod` functions to reserve a place + for positional arguments in the returned :ref:`partial object + `. (Contributed by Dominykas Grigonis in :gh:`119127`.) -* Allow the *initial* parameter of :func:`functools.reduce` to be passed +* Allow the *initial* parameter of :func:`~functools.reduce` to be passed as a keyword argument. (Contributed by Sayandip Dutta in :gh:`125916`.) @@ -1233,16 +1370,17 @@ getopt getpass ------- -* Support keyboard feedback by :func:`getpass.getpass` via the keyword-only - optional argument ``echo_char``. Placeholder characters are rendered whenever - a character is entered, and removed when a character is deleted. +* Support keyboard feedback in the :func:`~getpass.getpass` function via + the keyword-only optional argument *echo_char*. + Placeholder characters are rendered whenever a character is entered, + and removed when a character is deleted. (Contributed by Semyon Moroz in :gh:`77065`.) graphlib -------- -* Allow :meth:`graphlib.TopologicalSorter.prepare` to be called more than once +* Allow :meth:`.TopologicalSorter.prepare` to be called more than once as long as sorting has not started. (Contributed by Daniel Pope in :gh:`130914`.) @@ -1250,13 +1388,14 @@ graphlib heapq ----- -* Add functions for working with max-heaps: +* The :mod:`!heapq` module has improved support for working with max-heaps, + via the following new functions: - * :func:`heapq.heapify_max`, - * :func:`heapq.heappush_max`, - * :func:`heapq.heappop_max`, - * :func:`heapq.heapreplace_max` - * :func:`heapq.heappushpop_max` + * :func:`~heapq.heapify_max` + * :func:`~heapq.heappush_max` + * :func:`~heapq.heappop_max` + * :func:`~heapq.heapreplace_max` + * :func:`~heapq.heappushpop_max` hmac @@ -1264,6 +1403,8 @@ hmac * Add a built-in implementation for HMAC (:rfc:`2104`) using formally verified code from the `HACL* `__ project. + This implementation is used as a fallback when the OpenSSL implementation + of HMAC is not available. (Contributed by Bénédikt Tran in :gh:`99108`.) @@ -1279,9 +1420,12 @@ http the command-line interface (``python -m http.server``) through the following options: - * ``--tls-cert ``: Path to the TLS certificate file. - * ``--tls-key ``: Optional path to the private key file. - * ``--tls-password-file ``: Optional path to the password file for the private key. + * :option:`--tls-cert \ `: + Path to the TLS certificate file. + * :option:`--tls-key \ `: + Optional path to the private key file. + * :option:`--tls-password-file \ `: + Optional path to the password file for the private key. (Contributed by Semyon Moroz in :gh:`85162`.) @@ -1289,7 +1433,7 @@ http imaplib ------- -* Add :meth:`IMAP4.idle() `, implementing the IMAP4 +* Add :meth:`.IMAP4.idle`, implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. (Contributed by Forest in :gh:`55454`.) @@ -1297,15 +1441,16 @@ imaplib inspect ------- -* :func:`inspect.signature` takes a new argument *annotation_format* to control +* :func:`~inspect.signature` takes a new argument *annotation_format* to control the :class:`annotationlib.Format` used for representing annotations. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* :meth:`inspect.Signature.format` takes a new argument *unquote_annotations*. - If true, string :term:`annotations ` are displayed without surrounding quotes. +* :meth:`.Signature.format` takes a new argument *unquote_annotations*. + If true, string :term:`annotations ` are displayed without + surrounding quotes. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* Add function :func:`inspect.ispackage` to determine whether an object is a +* Add function :func:`~inspect.ispackage` to determine whether an object is a :term:`package` or not. (Contributed by Zhikang Yan in :gh:`125634`.) @@ -1317,7 +1462,7 @@ io :exc:`BlockingIOError` if the operation cannot immediately return bytes. (Contributed by Giovanni Siragusa in :gh:`109523`.) -* Add protocols :class:`io.Reader` and :class:`io.Writer` as a simpler +* Add the :class:`~io.Reader` and :class:`~io.Writer` protocols as simpler alternatives to the pseudo-protocols :class:`typing.IO`, :class:`typing.TextIO`, and :class:`typing.BinaryIO`. (Contributed by Sebastian Rittau in :gh:`127648`.) @@ -1326,35 +1471,36 @@ io json ---- -* Add notes for JSON serialization errors that allow to identify the source - of the error. +* Add exception notes for JSON serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122163`.) -* Enable the :mod:`json` module to work as a script using the :option:`-m` - switch: :program:`python -m json`. +* Allow using the :mod:`json` module as a script using the :option:`-m` switch: + :program:`python -m json`. + This is now preferred to :program:`python -m json.tool`, + which is :term:`soft deprecated`. See the :ref:`JSON command-line interface ` documentation. (Contributed by Trey Hunner in :gh:`122873`.) - .. _whatsnew314-color-json: - * By default, the output of the :ref:`JSON command-line interface ` is highlighted in color. This can be controlled by :ref:`environment variables `. (Contributed by Tomas Roun in :gh:`131952`.) + linecache --------- -* :func:`linecache.getline` can retrieve source code for frozen modules. +* :func:`~linecache.getline` can now retrieve source code for frozen modules. (Contributed by Tian Gao in :gh:`131638`.) logging.handlers ---------------- -* :class:`logging.handlers.QueueListener` now implements the context - manager protocol, allowing it to be used in a :keyword:`with` statement. +* :class:`~logging.handlers.QueueListener` objects now support the + :term:`context manager` protocol. (Contributed by Charles Machalow in :gh:`132106`.) * :meth:`QueueListener.start ` now @@ -1366,20 +1512,19 @@ math ---- * Added more detailed error messages for domain errors in the module. - (Contributed by by Charlie Zhao and Sergey B Kirpichev in :gh:`101410`.) + (Contributed by Charlie Zhao and Sergey B Kirpichev in :gh:`101410`.) mimetypes --------- -* Document the command-line for :mod:`mimetypes`. - It now exits with ``1`` on failure instead of ``0`` - and ``2`` on incorrect command-line parameters instead of ``1``. - Also, errors are printed to stderr instead of stdout and their text is made - tighter. +* Add a public :ref:`command-line ` for the module, + invoked via :program:`python -m mimetypes`. (Contributed by Oleg Iarygin and Hugo van Kemenade in :gh:`93096`.) -* Add MS and :rfc:`8081` MIME types for fonts: +* Add several new MIME types based on RFCs and common usage: + + .. rubric:: Microsoft and :rfc:`8081` MIME types for fonts * Embedded OpenType: ``application/vnd.ms-fontobject`` * OpenType Layout (OTF) ``font/otf`` @@ -1387,18 +1532,14 @@ mimetypes * WOFF 1.0 ``font/woff`` * WOFF 2.0 ``font/woff2`` - (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`.) - -* Add :rfc:`9559` MIME types for Matroska audiovisual data container - structures, containing: + .. rubric:: :rfc:`9559` MIME types for Matroska audiovisual + data container structures * audio with no video: ``audio/matroska`` (``.mka``) * video: ``video/matroska`` (``.mkv``) * stereoscopic video: ``video/matroska-3d`` (``.mk3d``) - (Contributed by Hugo van Kemenade in :gh:`89416`.) - -* Add MIME types for images with RFCs: + .. rubric:: Images with RFCs * :rfc:`1494`: CCITT Group 3 (``.g3``) * :rfc:`3362`: Real-time Facsimile, T.38 (``.t38``) @@ -1407,9 +1548,7 @@ mimetypes * :rfc:`4047`: Flexible Image Transport System (``.fits``) * :rfc:`7903`: Enhanced Metafile (``.emf``) and Windows Metafile (``.wmf``) - (Contributed by Hugo van Kemenade in :gh:`85957`.) - -* More MIME type changes: + .. rubric:: Other MIME type additions and changes * :rfc:`2361`: Change type for ``.avi`` to ``video/vnd.avi`` and for ``.wav`` to ``audio/vnd.wave`` @@ -1417,6 +1556,8 @@ mimetypes * :rfc:`5334`: Add Ogg media (``.oga``, ``.ogg`` and ``.ogx``) * :rfc:`6713`: Add gzip ``application/gzip`` (``.gz``) * :rfc:`9639`: Add FLAC ``audio/flac`` (``.flac``) + * :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` + and ``.yml``) * Add 7z ``application/x-7z-compressed`` (``.7z``) * Add Android Package ``application/vnd.android.package-archive`` (``.apk``) when not strict @@ -1439,11 +1580,9 @@ mimetypes * `W3C `__: Add EPUB ``application/epub+zip`` (``.epub``) - (Contributed by Hugo van Kemenade in :gh:`129965`.) - -* Add :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` - and ``.yml``). (Contributed by Sasha "Nelie" Chernykh and Hugo van Kemenade - in :gh:`132056`.) + (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`, + by Sasha "Nelie" Chernykh and Hugo van Kemenade in :gh:`132056`, + and by Hugo van Kemenade in :gh:`89416`, :gh:`85957`, and :gh:`129965`.) multiprocessing @@ -1451,14 +1590,16 @@ multiprocessing .. _whatsnew314-multiprocessing-start-method: -* The default :ref:`start method ` changed - from :ref:`fork ` to :ref:`forkserver - ` on platforms other than macOS and - Windows where it was already :ref:`spawn `. +* On Unix platforms other than macOS, :ref:`'forkserver' + ` is now the default :ref:`start + method ` + (replacing :ref:`'fork' `). + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly - request it via a context from :func:`multiprocessing.get_context` (preferred) - or change the default via :func:`multiprocessing.set_start_method`. + request it via a context from :func:`~multiprocessing.get_context` (preferred) + or change the default via :func:`~multiprocessing.set_start_method`. See :ref:`forkserver restrictions ` for information and differences with the *fork* method and how this change @@ -1467,7 +1608,7 @@ multiprocessing (Contributed by Gregory P. Smith in :gh:`84559`.) -* :mod:`multiprocessing`'s ``"forkserver"`` start method now authenticates +* :mod:`multiprocessing`'s ``'forkserver'`` start method now authenticates its control socket to avoid solely relying on filesystem permissions to restrict what other processes could cause the forkserver to spawn workers and run code. @@ -1483,20 +1624,22 @@ multiprocessing (Contributed by Roy Hyunjin Han for :gh:`103134`.) * Add support for shared :class:`set` objects via - :meth:`SyncManager.set() `. - The :func:`set` in :func:`multiprocessing.Manager` method is now available. + :meth:`.SyncManager.set`. + The :func:`set` in :func:`~multiprocessing.Manager` method is now available. (Contributed by Mingyu Park in :gh:`129949`.) -* Add :func:`multiprocessing.Process.interrupt` which terminates the child +* Add the :meth:`~multiprocessing.Process.interrupt` + to :class:`multiprocessing.Process` objects, which terminates the child process by sending :py:const:`~signal.SIGINT`. This enables :keyword:`finally` clauses to print a stack trace for the terminated process. (Contributed by Artem Pulkin in :gh:`131913`.) + operator -------- -* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` - have been added, such that ``operator.is_none(obj)`` is equivalent +* Add :func:`~operator.is_none` and :func:`~operator.is_not_none` as a pair + of functions, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. (Contributed by Raymond Hettinger and Nico Mexis in :gh:`115808`.) @@ -1505,21 +1648,31 @@ operator os -- -* Add the :func:`os.reload_environ` function to update :data:`os.environ` and +* Add the :func:`~os.reload_environ` function to update :data:`os.environ` and :data:`os.environb` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) * Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants - to the :mod:`os` module. + to the :mod:`!os` module. (Contributed by James Roy in :gh:`127688`.) -* Add the :func:`os.readinto` function to read into a +* Add the :func:`~os.readinto` function to read into a :ref:`buffer object ` from a file descriptor. (Contributed by Cody Maloney in :gh:`129205`.) +os.path +------- + +* The *strict* parameter to :func:`~os.path.realpath` accepts a new value, + :data:`~os.path.ALLOW_MISSING`. + If used, errors other than :exc:`FileNotFoundError` will be re-raised; + the resulting path can be missing but it will be free of symlinks. + (Contributed by Petr Viktorin for :cve:`2025-4517`.) + + pathlib ------- @@ -1533,8 +1686,8 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) -* Add :attr:`pathlib.Path.info` attribute, which stores an object - implementing the :class:`pathlib.types.PathInfo` protocol (also new). The +* Add the :attr:`~pathlib.Path.info` attribute, which stores an object + implementing the new :class:`pathlib.types.PathInfo` protocol. The object supports querying the file type and internally caching :func:`~os.stat` results. Path objects generated by :meth:`~pathlib.Path.iterdir` are initialized with file type information @@ -1545,7 +1698,26 @@ pathlib pdb --- -* Hardcoded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now +* The :mod:`pdb` module now supports remote attaching to a running Python process + using a new :option:`-p PID ` command-line option: + + .. code-block:: sh + + python -m pdb -p 1234 + + This will connect to the Python process with the given PID and allow you to + debug it interactively. Notice that due to how the Python interpreter works + attaching to a remote process that is blocked in a system call or waiting for + I/O will only work once the next bytecode instruction is executed or when the + process receives a signal. + + This feature uses :ref:`PEP 768 ` + and the new :func:`sys.remote_exec` function to attach to the remote process + and send the PDB commands to it. + + (Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) + +* Hardcoded breakpoints (:func:`breakpoint` and :func:`~pdb.set_trace`) now reuse the most recent :class:`~pdb.Pdb` instance that calls :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. As a result, all the instance specific data like :pdbcmd:`display` and @@ -1578,21 +1750,14 @@ pdb * ``$_asynctask`` is added to access the current asyncio task if applicable. (Contributed by Tian Gao in :gh:`124367`.) -* :mod:`pdb` now supports two backends: :func:`sys.settrace` and - :mod:`sys.monitoring`. Using :mod:`pdb` CLI or :func:`breakpoint` will - always use the :mod:`sys.monitoring` backend. Explicitly instantiating - :class:`pdb.Pdb` and its derived classes will use the :func:`sys.settrace` - backend by default, which is configurable. - (Contributed by Tian Gao in :gh:`124533`.) - * :func:`pdb.set_trace_async` is added to support debugging asyncio coroutines. :keyword:`await` statements are supported with this function. (Contributed by Tian Gao in :gh:`132576`.) * Source code displayed in :mod:`pdb` will be syntax-highlighted. This feature - can be controlled using the same methods as PyREPL, in addition to the newly - added ``colorize`` argument of :class:`pdb.Pdb`. + can be controlled using the same methods as the default :term:`interactive` + shell, in addition to the newly added ``colorize`` argument of :class:`pdb.Pdb`. (Contributed by Tian Gao and Łukasz Langa in :gh:`133355`.) @@ -1602,15 +1767,16 @@ pickle * Set the default protocol version on the :mod:`pickle` module to 5. For more details, see :ref:`pickle protocols `. -* Add notes for pickle serialization errors that allow to identify the source - of the error. +* Add exception notes for pickle serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122213`.) platform -------- -* Add :func:`platform.invalidate_caches` to invalidate the cached results. +* Add :func:`~platform.invalidate_caches`, a function to invalidate + cached results in the :mod:`!platform` module. (Contributed by Bénédikt Tran in :gh:`122549`.) @@ -1622,6 +1788,19 @@ pydoc (Contributed by Jelle Zijlstra in :gh:`101552`.) +re +-- + +* Support ``\z`` as a synonym for ``\Z`` in :mod:`regular expressions `. + It is interpreted unambiguously in many other regular expression engines, + unlike ``\Z``, which has subtly different behavior. + (Contributed by Serhiy Storchaka in :gh:`133306`.) + +* ``\B`` in :mod:`regular expression ` now matches the empty input string, + meaning that it is now always the opposite of ``\b``. + (Contributed by Serhiy Storchaka in :gh:`124130`.) + + socket ------ @@ -1648,11 +1827,12 @@ socket * Add many new constants. (Contributed by Serhiy Storchaka in :gh:`132734`.) + ssl --- -* Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports - TLSv1.3 post-handshake client authentication (PHA). +* Indicate through the :data:`~ssl.HAS_PHA` Boolean whether the :mod:`!ssl` + module supports TLSv1.3 post-handshake client authentication (PHA). (Contributed by Will Childs-Klein in :gh:`128036`.) @@ -1668,7 +1848,7 @@ struct symtable -------- -* Expose the following :class:`symtable.Symbol` methods: +* Expose the following :class:`~symtable.Symbol` methods: * :meth:`~symtable.Symbol.is_comp_cell` * :meth:`~symtable.Symbol.is_comp_iter` @@ -1683,31 +1863,69 @@ sys * The previously undocumented special function :func:`sys.getobjects`, which only exists in specialized builds of Python, may now return objects from other interpreters than the one it's called in. + (Contributed by Eric Snow in :gh:`125286`.) * Add :func:`sys._is_immortal` for determining if an object is :term:`immortal`. (Contributed by Peter Bierma in :gh:`128509`.) -* On FreeBSD, :data:`sys.platform` doesn't contain the major version anymore. +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. It is always ``'freebsd'``, instead of ``'freebsd13'`` or ``'freebsd14'``. + (Contributed by Michael Osipov in :gh:`129393`.) * Raise :exc:`DeprecationWarning` for :func:`sys._clear_type_cache`. This function was deprecated in Python 3.13 but it didn't raise a runtime warning. +* Add :func:`sys.remote_exec` to implement the new external debugger interface. + See :ref:`PEP 768 ` for details. + (Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic + in :gh:`131591`.) + +* Add the :data:`sys._jit` namespace, containing utilities for introspecting + just-in-time compilation. + (Contributed by Brandt Bucher in :gh:`133231`.) + sys.monitoring -------------- -* Two new events are added: :monitoring-event:`BRANCH_LEFT` and - :monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. +* Add two new monitoring events, :monitoring-event:`BRANCH_LEFT` and + :monitoring-event:`BRANCH_RIGHT`. + These replace and deprecate the :monitoring-event:`!BRANCH` event. + (Contributed by Mark Shannon in :gh:`122548`.) sysconfig --------- -* Add ``ABIFLAGS`` key to :func:`sysconfig.get_config_vars` on Windows. +* Add ``ABIFLAGS`` key to :func:`~sysconfig.get_config_vars` on Windows. (Contributed by Xuehai Pan in :gh:`131799`.) +tarfile +------- + +* :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to + avoid path traversal attacks. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) + +* :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes + when a directory was removed or replaced by another kind of file. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) + +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + now (re-)apply the extraction filter when substituting a link (hard or + symbolic) with a copy of another archive member, and when fixing up + directory attributes. + The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. + (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) + +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + no longer extract rejected members when + :func:`~tarfile.TarFile.errorlevel` is zero. + (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` + and :cve:`2025-4435`.) + + threading --------- @@ -1720,17 +1938,18 @@ tkinter ------- * Make :mod:`tkinter` widget methods :meth:`!after` and :meth:`!after_idle` - accept arguments passed by keyword. + accept keyword arguments. (Contributed by Zhikang Yan in :gh:`126899`.) -* Add ability to specify name for :class:`!tkinter.OptionMenu` and +* Add ability to specify a name for :class:`!tkinter.OptionMenu` and :class:`!tkinter.ttk.OptionMenu`. (Contributed by Zhikang Yan in :gh:`130482`.) + turtle ------ -* Add context managers for :func:`turtle.fill`, :func:`turtle.poly` +* Add context managers for :func:`turtle.fill`, :func:`turtle.poly`, and :func:`turtle.no_animation`. (Contributed by Marie Roald and Yngve Mardal Moe in :gh:`126350`.) @@ -1748,43 +1967,60 @@ typing .. _whatsnew314-typing-union: -* :class:`types.UnionType` and :class:`typing.Union` are now aliases for each other, - meaning that both old-style unions (created with ``Union[int, str]``) and new-style - unions (``int | str``) now create instances of the same runtime type. This unifies - the behavior between the two syntaxes, but leads to some differences in behavior that +* The :class:`types.UnionType` and :class:`typing.Union` types are now + aliases for each other, meaning that both old-style unions + (created with ``Union[int, str]``) and new-style unions (``int | str``) + now create instances of the same runtime type. This unifies the behavior + between the two syntaxes, but leads to some differences in behavior that may affect users who introspect types at runtime: - - Both syntaxes for creating a union now produce the same string representation in - ``repr()``. For example, ``repr(Union[int, str])`` - is now ``"int | str"`` instead of ``"typing.Union[int, str]"``. - - Unions created using the old syntax are no longer cached. Previously, running - ``Union[int, str]`` multiple times would return the same object - (``Union[int, str] is Union[int, str]`` would be ``True``), but now it will - return two different objects. Users should use ``==`` to compare unions for equality, not - ``is``. New-style unions have never been cached this way. - This change could increase memory usage for some programs that use a large number of - unions created by subscripting ``typing.Union``. However, several factors offset this cost: - unions used in annotations are no longer evaluated by default in Python 3.14 - because of :pep:`649`; an instance of :class:`types.UnionType` is - itself much smaller than the object returned by ``Union[]`` was on prior Python - versions; and removing the cache also saves some space. It is therefore - unlikely that this change will cause a significant increase in memory usage for most - users. + - Both syntaxes for creating a union now produce the same string + representation in :func:`repr`. + For example, ``repr(Union[int, str])`` is now ``"int | str"`` instead of + ``"typing.Union[int, str]"``. + + - Unions created using the old syntax are no longer cached. + Previously, running ``Union[int, str]`` multiple times would return + the same object (``Union[int, str] is Union[int, str]`` would be ``True``), + but now it will return two different objects. + Use ``==`` to compare unions for equality, not ``is``. + New-style unions have never been cached this way. + This change could increase memory usage for some programs that use + a large number of unions created by subscripting ``typing.Union``. + However, several factors offset this cost: + unions used in annotations are no longer evaluated by default in Python + 3.14 because of :pep:`649`; an instance of :class:`types.UnionType` is + itself much smaller than the object returned by ``Union[]`` was on prior + Python versions; and removing the cache also saves some space. + It is therefore unlikely that this change will cause a significant increase + in memory usage for most users. + - Previously, old-style unions were implemented using the private class - ``typing._UnionGenericAlias``. This class is no longer needed for the implementation, - but it has been retained for backward compatibility, with removal scheduled for Python - 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` - and :func:`typing.get_args` instead of relying on private implementation details. - - It is now possible to use :class:`typing.Union` itself in :func:`isinstance` checks. - For example, ``isinstance(int | str, typing.Union)`` will return ``True``; previously - this raised :exc:`TypeError`. - - The ``__args__`` attribute of :class:`typing.Union` objects is no longer writable. - - It is no longer possible to set any attributes on :class:`typing.Union` objects. + ``typing._UnionGenericAlias``. + This class is no longer needed for the implementation, + but it has been retained for backward compatibility, + with removal scheduled for Python 3.17. + Users should use documented introspection helpers like + :func:`~typing.get_origin` and :func:`typing.get_args` instead of + relying on private implementation details. + + - It is now possible to use :class:`typing.Union` itself in + :func:`isinstance` checks. + For example, ``isinstance(int | str, typing.Union)`` will return ``True``; + previously this raised :exc:`TypeError`. + + - The :attr:`!__args__` attribute of :class:`typing.Union` objects is + no longer writable. + + - It is no longer possible to set any attributes on :class:`~typing.Union` + objects. This only ever worked for dunder attributes on previous versions, was never documented to work, and was subtly broken in many cases. (Contributed by Jelle Zijlstra in :gh:`105499`.) +* :class:`~typing.TypeAliasType` now supports star unpacking. + unicodedata ----------- @@ -1792,11 +2028,11 @@ unicodedata * The Unicode database has been updated to Unicode 16.0.0. -.. _whatsnew314-color-unittest: - unittest -------- +.. _whatsnew314-color-unittest: + * :mod:`unittest` output is now colored by default. This can be controlled by :ref:`environment variables `. @@ -1819,7 +2055,7 @@ unittest :meth:`~unittest.TestCase.assertNotStartsWith`, :meth:`~unittest.TestCase.assertEndsWith` and :meth:`~unittest.TestCase.assertNotEndsWith` check whether the Unicode - or byte string starts or ends with particular string(s). + or byte string starts or ends with particular strings. (Contributed by Serhiy Storchaka in :gh:`71339`.) @@ -1834,17 +2070,18 @@ urllib * Improve ergonomics and standards compliance when parsing and emitting ``file:`` URLs. - In :func:`urllib.request.url2pathname`: + In :func:`~urllib.request.url2pathname`: - Accept a complete URL when the new *require_scheme* argument is set to true. - Discard URL authority if it matches the local hostname. - Discard URL authority if it resolves to a local IP address when the new *resolve_host* argument is set to true. + - Discard URL query and fragment components. - Raise :exc:`~urllib.error.URLError` if a URL authority isn't local, except on Windows where we return a UNC path as before. - In :func:`urllib.request.pathname2url`: + In :func:`~urllib.request.pathname2url`: - Return a complete URL when the new *add_scheme* argument is set to true. - Include an empty URL authority when a path begins with a slash. For @@ -1860,16 +2097,17 @@ urllib uuid ---- -* Add support for UUID versions 6, 7, and 8 via :func:`uuid.uuid6`, - :func:`uuid.uuid7`, and :func:`uuid.uuid8` respectively, as specified +* Add support for UUID versions 6, 7, and 8 via :func:`~uuid.uuid6`, + :func:`~uuid.uuid7`, and :func:`~uuid.uuid8` respectively, as specified in :rfc:`9562`. (Contributed by Bénédikt Tran in :gh:`89083`.) -* :const:`uuid.NIL` and :const:`uuid.MAX` are now available to represent the +* :const:`~uuid.NIL` and :const:`~uuid.MAX` are now available to represent the Nil and Max UUID formats as defined by :rfc:`9562`. (Contributed by Nick Pope in :gh:`128427`.) -* Allow to generate multiple UUIDs at once via :option:`python -m uuid --count `. +* Allow generating multiple UUIDs simultaneously on the command-line via + :option:`python -m uuid --count `. (Contributed by Simon Legner in :gh:`131236`.) @@ -1884,116 +2122,477 @@ webbrowser supported browsers on macOS. -zipinfo +zipfile ------- -* Added :func:`ZipInfo._for_archive ` +* Added :meth:`ZipInfo._for_archive `, a method to resolve suitable defaults for a :class:`~zipfile.ZipInfo` object as used by :func:`ZipFile.writestr `. (Contributed by Bénédikt Tran in :gh:`123424`.) -* :meth:`zipfile.ZipFile.writestr` now respect ``SOURCE_DATE_EPOCH`` that - distributions can set centrally and have build tools consume this in order - to produce reproducible output. +* :meth:`.ZipFile.writestr` now respects the :envvar:`SOURCE_DATE_EPOCH` + environment variable in order to better support reproducible builds. (Contributed by Jiahao Li in :gh:`91279`.) .. Add improved modules above alphabetically, not here at the end. + Optimizations ============= * The import time for several standard library modules has been improved, - including :mod:`ast`, :mod:`asyncio`, :mod:`base64`, :mod:`cmd`, :mod:`csv`, - :mod:`gettext`, :mod:`importlib.util`, :mod:`locale`, :mod:`mimetypes`, - :mod:`optparse`, :mod:`pickle`, :mod:`pprint`, :mod:`pstats`, :mod:`socket`, - :mod:`subprocess`, :mod:`threading`, :mod:`tomllib`, and :mod:`zipfile`. + including :mod:`annotationlib`, :mod:`ast`, :mod:`asyncio`, :mod:`base64`, + :mod:`cmd`, :mod:`csv`, :mod:`gettext`, :mod:`importlib.util`, :mod:`locale`, + :mod:`mimetypes`, :mod:`optparse`, :mod:`pickle`, :mod:`pprint`, + :mod:`pstats`, :mod:`shlex`, :mod:`socket`, :mod:`string`, :mod:`subprocess`, + :mod:`threading`, :mod:`tomllib`, :mod:`types`, and :mod:`zipfile`. (Contributed by Adam Turner, Bénédikt Tran, Chris Markiewicz, Eli Schwartz, Hugo van Kemenade, Jelle Zijlstra, and others in :gh:`118761`.) +* The interpreter now avoids some reference count modifications internally + when it's safe to do so. + This can lead to different values being returned from :func:`sys.getrefcount` + and :c:func:`Py_REFCNT` compared to previous versions of Python. + See :ref:`below ` for details. + asyncio ------- -* :mod:`asyncio` now uses double linked list implementation for native tasks - which speeds up execution by 10% on standard pyperformance benchmarks and - reduces memory usage. +* Standard benchmark results have improved by 10-20% following the + implementation of a new per-thread doubly linked list + for :class:`native tasks `, + also reducing memory usage. + This enables external introspection tools such as + :ref:`python -m asyncio pstree ` + to introspect the call graph of asyncio tasks running in all threads. (Contributed by Kumar Aditya in :gh:`107803`.) -* :mod:`asyncio` has new utility functions for introspecting and printing - the program's call graph: :func:`asyncio.capture_call_graph` and - :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa - in :gh:`91048`.) +* The module now has first class support for + :term:`free-threading builds `. + This enables parallel execution of multiple event loops across + different threads, scaling linearly with the number of threads. + (Contributed by Kumar Aditya in :gh:`128002`.) + base64 ------ -* Improve the performance of :func:`base64.b16decode` by up to ten times, - and reduce the import time of :mod:`base64` by up to six times. - (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) +* :func:`~base64.b16decode` is now up to six times faster. + (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner + in :gh:`118761`.) + + +bdb +--- + +* The basic debugger now has a :mod:`sys.monitoring`-based backend, + which can be selected via the passing ``'monitoring'`` + to the :class:`~bdb.Bdb` class's new *backend* parameter. + (Contributed by Tian Gao in :gh:`124533`.) + + +difflib +------- + +* The :func:`~difflib.IS_LINE_JUNK` function is now up to twice as fast. + (Contributed by Adam Turner and Semyon Moroz in :gh:`130167`.) + + +gc +-- + +* The new :ref:`incremental garbage collector ` + means that maximum pause times are reduced + by an order of magnitude or more for larger heaps. + + Because of this optimization, the meaning of the results of + :meth:`~gc.get_threshold` and :meth:`~gc.set_threshold` have changed, + along with :meth:`~gc.get_count` and :meth:`~gc.get_stats`. + + - For backwards compatibility, :meth:`~gc.get_threshold` continues to return + a three-item tuple. + The first value is the threshold for young collections, as before; + the second value determines the rate at which the old collection is scanned + (the default is 10, and higher values mean that the old collection + is scanned more slowly). + The third value is now meaningless and is always zero. + + - :meth:`~gc.set_threshold` now ignores any items after the second. + + - :meth:`~gc.get_count` and :meth:`~gc.get_stats` continue to return + the same format of results. + The only difference is that instead of the results referring to + the young, aging and old generations, + the results refer to the young generation + and the aging and collecting spaces of the old generation. + + In summary, code that attempted to manipulate the behavior of the cycle GC + may not work exactly as intended, but it is very unlikely to be harmful. + All other code will work just fine. + + (Contributed by Mark Shannon in :gh:`108362`.) io --- -* :mod:`io` which provides the built-in :func:`open` makes less system calls - when opening regular files as well as reading whole files. Reading a small - operating system cached file in full is up to 15% faster. - :func:`pathlib.Path.read_bytes` has the most optimizations for reading a - file's bytes in full. (Contributed by Cody Maloney and Victor Stinner in - :gh:`120754` and :gh:`90102`.) + +* Opening and reading files now executes fewer system calls. + Reading a small operating system cached file in full is up to 15% faster. + (Contributed by Cody Maloney and Victor Stinner + in :gh:`120754` and :gh:`90102`.) + + +pathlib +------- + +* :func:`Path.read_bytes ` now uses unbuffered mode + to open files, which is between 9% and 17% faster to read in full. + (Contributed by Cody Maloney in :gh:`120754`.) + + +pdb +--- + +* :mod:`pdb` now supports two backends, based on either + :func:`sys.settrace` or :mod:`sys.monitoring`. + Using the :ref:`pdb CLI ` or :func:`breakpoint` + will always use the :mod:`sys.monitoring` backend. + Explicitly instantiating :class:`pdb.Pdb` and its derived classes + will use the :func:`sys.settrace` backend by default, which is configurable. + (Contributed by Tian Gao in :gh:`124533`.) uuid ---- -* Improve generation of :class:`~uuid.UUID` objects via their dedicated - functions: - - * :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are both roughly 40% faster - for 16-byte names and 20% faster for 1024-byte names. Performance for - longer names remains unchanged. - * :func:`~uuid.uuid4` and :func:`~uuid.uuid8` are 30% and 40% faster - respectively. +* :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are now both roughly 40% faster + for 16-byte names and 20% faster for 1024-byte names. + Performance for longer names remains unchanged. + (Contributed by Bénédikt Tran in :gh:`128150`.) +* :func:`~uuid.uuid4` is now c. 30% faster. (Contributed by Bénédikt Tran in :gh:`128150`.) zlib ---- -* On Windows, ``zlib-ng`` is now used as the implementation of the - :mod:`zlib` module. This should produce compatible and comparable - results with better performance, though it is worth noting that - ``zlib.Z_BEST_SPEED`` (1) may result in significantly less - compression than the previous implementation (while also significantly - reducing the time taken to compress). +* On Windows, `zlib-ng `__ + is now used as the implementation of the :mod:`zlib` module + in the default binaries. + There are no known incompatibilities between ``zlib-ng`` + and the previously-used ``zlib`` implementation. + This should result in better performance at all compression levels. + + It is worth noting that ``zlib.Z_BEST_SPEED`` (``1``) may result in + significantly less compression than the previous implementation, + whilst also significantly reducing the time taken to compress. + (Contributed by Steve Dower in :gh:`91349`.) +Removed +======= + +argparse +-------- + +* Remove the *type*, *choices*, and *metavar* parameters + of :class:`!BooleanOptionalAction`. + These have been deprecated since Python 3.12. + (Contributed by Nikita Sobolev in :gh:`118805`.) + +* Calling :meth:`~argparse.ArgumentParser.add_argument_group` + on an argument group now raises a :exc:`ValueError`. + Similarly, :meth:`~argparse.ArgumentParser.add_argument_group` + or :meth:`~argparse.ArgumentParser.add_mutually_exclusive_group` + on a mutually exclusive group now both raise :exc:`ValueError`\ s. + This 'nesting' was never supported, often failed to work correctly, + and was unintentionally exposed through inheritance. + This functionality has been deprecated since Python 3.11. + (Contributed by Savannah Ostrowski in :gh:`127186`.) + + +ast +--- + +* Remove the following classes, which have been deprecated aliases of + :class:`~ast.Constant` since Python 3.8 and have emitted + deprecation warnings since Python 3.12: + + * :class:`!Bytes` + * :class:`!Ellipsis` + * :class:`!NameConstant` + * :class:`!Num` + * :class:`!Str` + + As a consequence of these removals, user-defined ``visit_Num``, ``visit_Str``, + ``visit_Bytes``, ``visit_NameConstant`` and ``visit_Ellipsis`` methods + on custom :class:`~ast.NodeVisitor` subclasses will no longer be called + when the :class:`!NodeVisitor` subclass is visiting an AST. + Define a ``visit_Constant`` method instead. + + (Contributed by Alex Waygood in :gh:`119562`.) + +* Remove the following deprecated properties on :class:`ast.Constant`, + which were present for compatibility with the now-removed AST classes: + + * :attr:`!Constant.n` + * :attr:`!Constant.s` + + Use :attr:`!Constant.value` instead. + (Contributed by Alex Waygood in :gh:`119562`.) + + +asyncio +------- + +* Remove the following classes, methods, and functions, + which have been deprecated since Python 3.12: + + * :class:`!AbstractChildWatcher` + * :class:`!FastChildWatcher` + * :class:`!MultiLoopChildWatcher` + * :class:`!PidfdChildWatcher` + * :class:`!SafeChildWatcher` + * :class:`!ThreadedChildWatcher` + * :meth:`!AbstractEventLoopPolicy.get_child_watcher` + * :meth:`!AbstractEventLoopPolicy.set_child_watcher` + * :func:`!get_child_watcher` + * :func:`!set_child_watcher` + + (Contributed by Kumar Aditya in :gh:`120804`.) + +* :func:`asyncio.get_event_loop` now raises a :exc:`RuntimeError` + if there is no current event loop, + and no longer implicitly creates an event loop. + + (Contributed by Kumar Aditya in :gh:`126353`.) + + .. TODO: move these patterns to the asyncio docs? + quite long for What's New + + There's a few patterns that use :func:`asyncio.get_event_loop`, most + of them can be replaced with :func:`asyncio.run`. + + If you're running an async function, simply use :func:`asyncio.run`. + + Before: + + .. code:: python + + async def main(): + ... + + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(main()) + finally: + loop.close() + + After: + + .. code:: python + + async def main(): + ... + + asyncio.run(main()) + + If you need to start something, for example, a server listening on a socket + and then run forever, use :func:`asyncio.run` and an + :class:`asyncio.Event`. + + Before: + + .. code:: python + + def start_server(loop): ... + + loop = asyncio.get_event_loop() + try: + start_server(loop) + loop.run_forever() + finally: + loop.close() + + After: + + .. code:: python + + def start_server(loop): ... + + async def main(): + start_server(asyncio.get_running_loop()) + await asyncio.Event().wait() + + asyncio.run(main()) + + If you need to run something in an event loop, then run some blocking + code around it, use :class:`asyncio.Runner`. + + Before: + + .. code:: python + + async def operation_one(): ... + def blocking_code(): ... + async def operation_two(): ... + + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(operation_one()) + blocking_code() + loop.run_until_complete(operation_two()) + finally: + loop.close() + + After: + + .. code:: python + + async def operation_one(): ... + def blocking_code(): ... + async def operation_two(): ... + + with asyncio.Runner() as runner: + runner.run(operation_one()) + blocking_code() + runner.run(operation_two()) + + +email +----- + +* Remove :func:`email.utils.localtime`'s *isdst* parameter, + which was deprecated in and has been ignored since Python 3.12. + (Contributed by Hugo van Kemenade in :gh:`118798`.) + + +importlib.abc +------------- + +* Remove deprecated :mod:`importlib.abc` classes: + + * :class:`!ResourceReader` + (use :class:`~importlib.resources.abc.TraversableResources`) + * :class:`!Traversable` + (use :class:`~importlib.resources.abc.Traversable`) + * :class:`!TraversableResources` + (use :class:`~importlib.resources.abc.TraversableResources`) + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + + +itertools +--------- + +* Remove support for copy, deepcopy, and pickle operations + from :mod:`itertools` iterators. + These have emitted a :exc:`DeprecationWarning` since Python 3.12. + (Contributed by Raymond Hettinger in :gh:`101588`.) + + +pathlib +------- + +* Remove support for passing additional keyword arguments + to :class:`~pathlib.Path`. + In previous versions, any such arguments are ignored. + (Contributed by Barney Gale in :gh:`74033`.) + +* Remove support for passing additional positional arguments to + :meth:`.PurePath.relative_to` and :meth:`~pathlib.PurePath.is_relative_to`. + In previous versions, any such arguments are joined onto *other*. + (Contributed by Barney Gale in :gh:`78707`.) + + +pkgutil +------- + +* Remove the :func:`!get_loader` and :func:`!find_loader` functions, + which have been deprecated since Python 3.12. + (Contributed by Bénédikt Tran in :gh:`97850`.) + + +pty +--- + +* Remove the :func:`!master_open` and :func:`!slave_open` functions, + which have been deprecated since Python 3.12. + Use :func:`pty.openpty` instead. + (Contributed by Nikita Sobolev in :gh:`118824`.) + + +sqlite3 +------- + +* Remove :data:`!version` and :data:`!version_info` from + the :mod:`sqlite3` module; + use :data:`~sqlite3.sqlite_version` and :data:`~sqlite3.sqlite_version_info` + for the actual version number of the runtime SQLite library. + (Contributed by Hugo van Kemenade in :gh:`118924`.) + +* Using a sequence of parameters with named placeholders now + raises a :exc:`~sqlite3.ProgrammingError`, + having been deprecated since Python 3.12. + (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) + + +urllib +------ + +* Remove the :class:`!Quoter` class from :mod:`urllib.parse`, + which has been deprecated since Python 3.11. + (Contributed by Nikita Sobolev in :gh:`118827`.) + +* Remove the :class:`!URLopener` and :class:`!FancyURLopener` classes + from :mod:`urllib.request`, + which have been deprecated since Python 3.3. + + ``myopener.open()`` can be replaced with :func:`~urllib.request.urlopen`. + ``myopener.retrieve()`` can be replaced with + :func:`~urllib.request.urlretrieve`. + Customisations to the opener classes can be replaced by passing + customized handlers to :func:`~urllib.request.build_opener`. + (Contributed by Barney Gale in :gh:`84850`.) + + Deprecated ========== +New deprecations +---------------- + +* Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; + complex numbers should only be passed as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + * :mod:`argparse`: - * Passing the undocumented keyword argument *prefix_chars* to - :meth:`~argparse.ArgumentParser.add_argument_group` is now - deprecated. + * Passing the undocumented keyword argument *prefix_chars* to the + :meth:`~argparse.ArgumentParser.add_argument_group` method is now deprecated. (Contributed by Savannah Ostrowski in :gh:`125563`.) + * Deprecated the :class:`argparse.FileType` type converter. - Anything with resource management should be done downstream after the - arguments are parsed. + Anything relating to resource management should be handled + downstream, after the arguments have been parsed. (Contributed by Serhiy Storchaka in :gh:`58032`.) * :mod:`asyncio`: - * :func:`!asyncio.iscoroutinefunction` is deprecated + * The :func:`!asyncio.iscoroutinefunction` is now deprecated and will be removed in Python 3.16; use :func:`inspect.iscoroutinefunction` instead. (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) - * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + * The :mod:`asyncio` policy system is deprecated + and will be removed in Python 3.16. In particular, the following classes and functions are deprecated: * :class:`asyncio.AbstractEventLoopPolicy` @@ -2002,102 +2601,104 @@ Deprecated * :class:`asyncio.WindowsProactorEventLoopPolicy` * :func:`asyncio.get_event_loop_policy` * :func:`asyncio.set_event_loop_policy` - * :func:`asyncio.set_event_loop` Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with - *loop_factory* to use the desired event loop implementation. + the *loop_factory* argument to use the desired event loop implementation. - For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + For example, to use :class:`asyncio.SelectorEventLoop` on Windows: - import asyncio + .. code-block:: python - async def main(): - ... + import asyncio - asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) (Contributed by Kumar Aditya in :gh:`127949`.) -* :mod:`builtins`: - Passing a complex number as the *real* or *imag* argument in the - :func:`complex` constructor is now deprecated; it should only be passed - as a single positional argument. - (Contributed by Serhiy Storchaka in :gh:`109218`.) - * :mod:`codecs`: - :func:`codecs.open` is now deprecated. Use :func:`open` instead. + The :func:`codecs.open` function is now deprecated, + and will be removed in a future version of Python. + Use :func:`open` instead. (Contributed by Inada Naoki in :gh:`133036`.) * :mod:`ctypes`: * On non-Windows platforms, setting :attr:`.Structure._pack_` to use a - MSVC-compatible default memory layout is deprecated in favor of setting - :attr:`.Structure._layout_` to ``'ms'``. + MSVC-compatible default memory layout is now deprecated in favor of setting + :attr:`.Structure._layout_` to ``'ms'``, and will be removed in Python 3.19. (Contributed by Petr Viktorin in :gh:`131747`.) - * Calling :func:`ctypes.POINTER` on a string is deprecated. - Use :ref:`ctypes-incomplete-types` for self-referential structures. + * Calling :func:`ctypes.POINTER` on a string is now deprecated. + Use :ref:`incomplete types ` + for self-referential structures. Also, the internal ``ctypes._pointer_type_cache`` is deprecated. See :func:`ctypes.POINTER` for updated implementation details. (Contributed by Sergey Myrianov in :gh:`100926`.) * :mod:`functools`: Calling the Python implementation of :func:`functools.reduce` with *function* - or *sequence* as keyword arguments is now deprecated. + or *sequence* as keyword arguments is now deprecated; + the parameters will be made positional-only in Python 3.16. (Contributed by Kirill Podoprigora in :gh:`121676`.) * :mod:`logging`: - Support for custom logging handlers with the *strm* argument is deprecated - and scheduled for removal in Python 3.16. Define handlers with the *stream* - argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) + Support for custom logging handlers with the *strm* argument + is now deprecated and scheduled for removal in Python 3.16. + Define handlers with the *stream* argument instead. + (Contributed by Mariusz Felisiak in :gh:`115032`.) * :mod:`mimetypes`: - Valid extensions start with a '.' or are empty for + Valid extensions are either empty or must start with '.' for :meth:`mimetypes.MimeTypes.add_type`. Undotted extensions are deprecated and will raise a :exc:`ValueError` in Python 3.16. (Contributed by Hugo van Kemenade in :gh:`75223`.) -* :mod:`!nturl2path`: This module is now deprecated. Call - :func:`urllib.request.url2pathname` and :func:`~urllib.request.pathname2url` - instead. +* :mod:`!nturl2path`: + This module is now deprecated. Call :func:`urllib.request.url2pathname` + and :func:`~urllib.request.pathname2url` instead. (Contributed by Barney Gale in :gh:`125866`.) * :mod:`os`: - :term:`Soft deprecate ` :func:`os.popen` and - :func:`os.spawn* ` functions. They should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + The :func:`os.popen` and :func:`os.spawn* ` functions + are now :term:`soft deprecated`. + They should no longer be used to write new code. + The :mod:`subprocess` module is recommended instead. (Contributed by Victor Stinner in :gh:`120743`.) * :mod:`pathlib`: - :meth:`!pathlib.PurePath.as_uri` is deprecated and will be removed in Python - 3.19. Use :meth:`pathlib.Path.as_uri` instead. + :meth:`!pathlib.PurePath.as_uri` is now deprecated + and scheduled for removal in Python 3.19. + Use :meth:`pathlib.Path.as_uri` instead. (Contributed by Barney Gale in :gh:`123599`.) * :mod:`pdb`: The undocumented ``pdb.Pdb.curframe_locals`` attribute is now a deprecated - read-only property. The low overhead dynamic frame locals access added in - Python 3.13 by PEP 667 means the frame locals cache reference previously - stored in this attribute is no longer needed. Derived debuggers should access + read-only property, which will be removed in a future version of Python. + The low overhead dynamic frame locals access added in Python 3.13 by :pep:`667` + means the frame locals cache reference previously stored in this attribute + is no longer needed. Derived debuggers should access ``pdb.Pdb.curframe.f_locals`` directly in Python 3.13 and later versions. (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) * :mod:`symtable`: - Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. + Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest, + scheduled for removal in Python 3.16. (Contributed by Bénédikt Tran in :gh:`119698`.) * :mod:`tkinter`: The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, :meth:`!trace_vdelete` and :meth:`!trace_vinfo` are now deprecated. - Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` - instead. + Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` instead. (Contributed by Serhiy Storchaka in :gh:`120220`.) * :mod:`urllib.parse`: Accepting objects with false values (like ``0`` and ``[]``) except empty - strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions - :func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs` is now - deprecated. + strings, bytes-like objects and ``None`` in :func:`~urllib.parse.parse_qsl` + and :func:`~urllib.parse.parse_qs` is now deprecated. (Contributed by Serhiy Storchaka in :gh:`116897`.) .. Add deprecations above alphabetically, not here at the end. @@ -2110,389 +2711,141 @@ Deprecated .. include:: ../deprecations/pending-removal-in-3.19.rst +.. include:: ../deprecations/pending-removal-in-3.20.rst + .. include:: ../deprecations/pending-removal-in-future.rst -Removed -======= - -argparse --------- - -* Remove the *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction`. - They were deprecated since 3.12. - -* Calling :meth:`~argparse.ArgumentParser.add_argument_group` on an argument - group, and calling :meth:`~argparse.ArgumentParser.add_argument_group` or - :meth:`~argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually - exclusive group now raise exceptions. This nesting was never supported, - often failed to work correctly, and was unintentionally exposed through - inheritance. This functionality has been deprecated since Python 3.11. - (Contributed by Savannah Ostrowski in :gh:`127186`.) - -ast ---- - -* Remove the following classes. They were all deprecated since Python 3.8, - and have emitted deprecation warnings since Python 3.12: - - * :class:`!ast.Bytes` - * :class:`!ast.Ellipsis` - * :class:`!ast.NameConstant` - * :class:`!ast.Num` - * :class:`!ast.Str` - - Use :class:`ast.Constant` instead. As a consequence of these removals, - user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, - ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom - :class:`ast.NodeVisitor` subclasses will no longer be called when the - :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` - method instead. - - Also, remove the following deprecated properties on :class:`ast.Constant`, - which were present for compatibility with the now-removed AST classes: - - * :attr:`!ast.Constant.n` - * :attr:`!ast.Constant.s` - - Use :attr:`!ast.Constant.value` instead. - (Contributed by Alex Waygood in :gh:`119562`.) - -asyncio -------- - -* Remove the following classes and functions. They were all deprecated and - emitted deprecation warnings since Python 3.12: - - * :func:`!asyncio.get_child_watcher` - * :func:`!asyncio.set_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` - * :class:`!asyncio.AbstractChildWatcher` - * :class:`!asyncio.FastChildWatcher` - * :class:`!asyncio.MultiLoopChildWatcher` - * :class:`!asyncio.PidfdChildWatcher` - * :class:`!asyncio.SafeChildWatcher` - * :class:`!asyncio.ThreadedChildWatcher` - - (Contributed by Kumar Aditya in :gh:`120804`.) - -* Removed implicit creation of event loop by :func:`asyncio.get_event_loop`. - It now raises a :exc:`RuntimeError` if there is no current event loop. - (Contributed by Kumar Aditya in :gh:`126353`.) - - There's a few patterns that use :func:`asyncio.get_event_loop`, most - of them can be replaced with :func:`asyncio.run`. - - If you're running an async function, simply use :func:`asyncio.run`. - - Before:: - - async def main(): - ... - - - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(main()) - finally: - loop.close() - - After:: - - async def main(): - ... - - asyncio.run(main()) - - If you need to start something, for example, a server listening on a socket - and then run forever, use :func:`asyncio.run` and an - :class:`asyncio.Event`. - - Before:: - - def start_server(loop): - ... - - loop = asyncio.get_event_loop() - try: - start_server(loop) - loop.run_forever() - finally: - loop.close() - - After:: - - def start_server(loop): - ... - - async def main(): - start_server(asyncio.get_running_loop()) - await asyncio.Event().wait() - - asyncio.run(main()) - - If you need to run something in an event loop, then run some blocking - code around it, use :class:`asyncio.Runner`. - - Before:: - - async def operation_one(): - ... - - def blocking_code(): - ... - - async def operation_two(): - ... - - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(operation_one()) - blocking_code() - loop.run_until_complete(operation_two()) - finally: - loop.close() - - After:: - - async def operation_one(): - ... - - def blocking_code(): - ... - - async def operation_two(): - ... - - with asyncio.Runner() as runner: - runner.run(operation_one()) - blocking_code() - runner.run(operation_two()) - - - -collections.abc ---------------- - -* Remove :class:`!collections.abc.ByteString`. It had previously raised a - :exc:`DeprecationWarning` since Python 3.12. - -email ------ - -* Remove the *isdst* parameter from :func:`email.utils.localtime`. - (Contributed by Hugo van Kemenade in :gh:`118798`.) - -importlib ---------- - -* Remove deprecated :mod:`importlib.abc` classes: - - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` - - Use :mod:`importlib.resources.abc` classes instead: - - * :class:`importlib.resources.abc.Traversable` - * :class:`importlib.resources.abc.TraversableResources` - - (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) - -itertools ---------- - -* Remove :mod:`itertools` support for copy, deepcopy, and pickle operations. - These had previously raised a :exc:`DeprecationWarning` since Python 3.12. - (Contributed by Raymond Hettinger in :gh:`101588`.) - -pathlib -------- - -* Remove support for passing additional keyword arguments to - :class:`pathlib.Path`. In previous versions, any such arguments are ignored. -* Remove support for passing additional positional arguments to - :meth:`pathlib.PurePath.relative_to` and - :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such - arguments are joined onto *other*. - -pkgutil -------- - -* Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. - These had previously raised a :exc:`DeprecationWarning` since Python 3.12. - (Contributed by Bénédikt Tran in :gh:`97850`.) - -pty ---- - -* Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. - They had previously raised a :exc:`DeprecationWarning` since Python 3.12. - Use :func:`pty.openpty` instead. - (Contributed by Nikita Sobolev in :gh:`118824`.) - -sqlite3 -------- - -* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. - (Contributed by Hugo van Kemenade in :gh:`118924`.) - -* Disallow using a sequence of parameters with named placeholders. - This had previously raised a :exc:`DeprecationWarning` since Python 3.12; - it will now raise a :exc:`sqlite3.ProgrammingError`. - (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) - -typing ------- - -* Remove :class:`!typing.ByteString`. It had previously raised a - :exc:`DeprecationWarning` since Python 3.12. - -* :class:`typing.TypeAliasType` now supports star unpacking. - -urllib ------- - -* Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. - It had previously raised a :exc:`DeprecationWarning` since Python 3.11. - (Contributed by Nikita Sobolev in :gh:`118827`.) -* Remove deprecated :class:`!URLopener` and :class:`!FancyURLopener` classes - from :mod:`urllib.request`. They had previously raised a - :exc:`DeprecationWarning` since Python 3.3. - - ``myopener.open()`` can be replaced with :func:`~urllib.request.urlopen`, - and ``myopener.retrieve()`` can be replaced with - :func:`~urllib.request.urlretrieve`. Customizations to the opener - classes can be replaced by passing customized handlers to - :func:`~urllib.request.build_opener`. - (Contributed by Barney Gale in :gh:`84850`.) - -Others ------- - -* Using :data:`NotImplemented` in a boolean context will now raise a :exc:`TypeError`. - It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed - by Jelle Zijlstra in :gh:`118767`.) - -* The :func:`int` built-in no longer delegates to - :meth:`~object.__trunc__`. Classes that want to support conversion to - integer must implement either :meth:`~object.__int__` or - :meth:`~object.__index__`. (Contributed by Mark Dickinson in :gh:`119743`.) - CPython bytecode changes ======================== -* Replaced the opcode ``BINARY_SUBSCR`` by :opcode:`BINARY_OP` with oparg ``NB_SUBSCR``. +* Replaced the opcode :opcode:`!BINARY_SUBSCR` by the :opcode:`BINARY_OP` + opcode with the ``NB_SUBSCR`` oparg. (Contributed by Irit Katriel in :gh:`100239`.) -Porting to Python 3.14 -====================== +* Add the :opcode:`BUILD_INTERPOLATION` and :opcode:`BUILD_TEMPLATE` + opcodes to construct new :class:`~string.templatelib.Interpolation` + and :class:`~string.templatelib.Template` instances, respectively. + (Contributed by Lysandros Nikolaou and others in :gh:`132661`; + see also :ref:`PEP 750: Template strings `). -This section lists previously described changes and other bugfixes -that may require changes to your code. +* Remove the :opcode:`!BUILD_CONST_KEY_MAP` opcode. + Use :opcode:`BUILD_MAP` instead. + (Contributed by Mark Shannon in :gh:`122160`.) -Changes in the Python API -------------------------- +* Replace the :opcode:`!LOAD_ASSERTION_ERROR` opcode with + :opcode:`LOAD_COMMON_CONSTANT` and add support for loading + :exc:`NotImplementedError`. -* :class:`functools.partial` is now a method descriptor. - Wrap it in :func:`staticmethod` if you want to preserve the old behavior. - (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) +* Add the :opcode:`LOAD_FAST_BORROW` and :opcode:`LOAD_FAST_BORROW_LOAD_FAST_BORROW` + opcodes to reduce reference counting overhead when the interpreter can prove + that the reference in the frame outlives the reference loaded onto the stack. + (Contributed by Matt Page in :gh:`130704`.) -* The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` - locale in some cases. - This temporary change affects other threads. - (Contributed by Serhiy Storchaka in :gh:`69998`.) +* Add the :opcode:`LOAD_SMALL_INT` opcode, which pushes a small integer + equal to the ``oparg`` to the stack. + The :opcode:`!RETURN_CONST` opcode is removed as it is no longer used. + (Contributed by Mark Shannon in :gh:`125837`.) -* :class:`types.UnionType` is now an alias for :class:`typing.Union`, - causing changes in some behaviors. - See :ref:`above ` for more details. - (Contributed by Jelle Zijlstra in :gh:`105499`.) +* Add the new :opcode:`LOAD_SPECIAL` instruction. + Generate code for :keyword:`with` and :keyword:`async with` statements + using the new instruction. + Removed the :opcode:`!BEFORE_WITH` and :opcode:`!BEFORE_ASYNC_WITH` instructions. + (Contributed by Mark Shannon in :gh:`120507`.) + +* Add the :opcode:`POP_ITER` opcode to support 'virtual' iterators. + (Contributed by Mark Shannon in :gh:`132554`.) -Build changes -============= +Pseudo-instructions +------------------- -* GNU Autoconf 2.72 is now required to generate :file:`configure`. - (Contributed by Erlend Aasland in :gh:`115765`.) +* Add the :opcode:`!ANNOTATIONS_PLACEHOLDER` pseudo instruction + to support partially executed module-level annotations with + :ref:`deferred evaluation of annotations `. + (Contributed by Jelle Zijlstra in :gh:`130907`.) -* ``#pragma``-based linking with ``python3*.lib`` can now be switched off - with :c:expr:`Py_NO_LINK_LIB`. (Contributed by Jean-Christophe - Fillion-Robin in :gh:`82909`.) +* Add the :opcode:`!BINARY_OP_EXTEND` pseudo instruction, + which executes a pair of functions (guard and specialization functions) + accessed from the inline cache. + (Contributed by Irit Katriel in :gh:`100239`.) -.. _whatsnew314-pep761: +* Add three specializations for :opcode:`CALL_KW`; + :opcode:`!CALL_KW_PY` for calls to Python functions, + :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods, and + :opcode:`!CALL_KW_NON_PY` for all other calls. + (Contributed by Mark Shannon in :gh:`118093`.) -PEP 761: Discontinuation of PGP signatures ------------------------------------------- +* Add the :opcode:`JUMP_IF_TRUE` and :opcode:`JUMP_IF_FALSE` pseudo instructions, + conditional jumps which do not impact the stack. + Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. + (Contributed by Irit Katriel in :gh:`124285`.) -PGP signatures will not be available for CPython 3.14 and onwards. -Users verifying artifacts must use `Sigstore verification materials`_ for -verifying CPython artifacts. This change in release process is specified -in :pep:`761`. +* Add the :opcode:`!LOAD_CONST_MORTAL` pseudo instruction. + (Contributed by Mark Shannon in :gh:`128685`.) -.. _Sigstore verification materials: https://www.python.org/downloads/metadata/sigstore/ +* Add the :opcode:`!LOAD_CONST_IMMORTAL` pseudo instruction, + which does the same as :opcode:`!LOAD_CONST`, but is more efficient + for immortal objects. + (Contributed by Mark Shannon in :gh:`125837`.) + +* Add the :opcode:`NOT_TAKEN` pseudo instruction, used by :mod:`sys.monitoring` + to record branch events (such as :monitoring-event:`BRANCH_LEFT`). + (Contributed by Mark Shannon in :gh:`122548`.) C API changes ============= -New features ------------- +.. _whatsnew314-capi-config: -* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. - (Contributed by Sergey B Kirpichev in :gh:`116560`.) +Python configuration C API +-------------------------- -* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` - object: +Add a :ref:`PyInitConfig C API ` to configure the Python +initialization without relying on C structures and the ability to make +ABI-compatible changes in the future. - * :c:func:`PyUnicodeWriter_Create` - * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` - * :c:func:`PyUnicodeWriter_Discard` - * :c:func:`PyUnicodeWriter_Finish` - * :c:func:`PyUnicodeWriter_Format` - * :c:func:`PyUnicodeWriter_WriteChar` - * :c:func:`PyUnicodeWriter_WriteRepr` - * :c:func:`PyUnicodeWriter_WriteStr` - * :c:func:`PyUnicodeWriter_WriteSubstring` - * :c:func:`PyUnicodeWriter_WriteUCS4` - * :c:func:`PyUnicodeWriter_WriteUTF8` - * :c:func:`PyUnicodeWriter_WriteWideChar` +Complete the :pep:`587` :ref:`PyConfig C API ` by adding +:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension +module; a feature previously referred to as the "inittab". - (Contributed by Victor Stinner in :gh:`119182`.) +Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set +the current runtime configuration. -* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, - which has an ambiguous return value. - (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) +:pep:`587` 'Python Initialization Configuration' unified all the ways +to configure Python's initialization. This PEP also unifies the configuration +of Python's preinitialization and initialization in a single API. +Moreover, this PEP only provides a single choice to embed Python, +instead of having two 'Python' and 'Isolated' choices (PEP 587), +to further simplify the API. -* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` - and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` - is positive, negative, or zero, respectively. - (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) +The lower level PEP 587 PyConfig API remains available for use cases +with an intentionally higher level of coupling to CPython implementation details +(such as emulating the full functionality of CPython's CLI, including its +configuration mechanisms). -* Add new functions to convert C ```` numbers from/to Python - :class:`int`: +(Contributed by Victor Stinner in :gh:`107954`.) - * :c:func:`PyLong_AsInt32` - * :c:func:`PyLong_AsInt64` - * :c:func:`PyLong_AsUInt32` - * :c:func:`PyLong_AsUInt64` - * :c:func:`PyLong_FromInt32` - * :c:func:`PyLong_FromInt64` - * :c:func:`PyLong_FromUInt32` - * :c:func:`PyLong_FromUInt64` +.. seealso:: :pep:`741` and :pep:`587` - (Contributed by Victor Stinner in :gh:`120389`.) + +New features in the C API +------------------------- + +* Add :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION`, + two new macros for bit-packing Python version numbers. + This is useful for comparisons with :c:var:`Py_Version` + or :c:macro:`PY_VERSION_HEX`. + (Contributed by Petr Viktorin in :gh:`128629`.) * Add :c:func:`PyBytes_Join(sep, iterable) ` function, similar to ``sep.join(iterable)`` in Python. (Contributed by Victor Stinner in :gh:`121645`.) -* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. - (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) - -* Add functions to get and set the current runtime Python configuration - (:pep:`741`): +* Add functions to manipulate the configuration of the current + runtime Python interpreter + (:ref:`PEP 741: Python configuration C API `): * :c:func:`PyConfig_Get` * :c:func:`PyConfig_GetInt` @@ -2501,7 +2854,8 @@ New features (Contributed by Victor Stinner in :gh:`107954`.) -* Add functions to configure the Python initialization (:pep:`741`): +* Add functions to configure Python initialization + (:ref:`PEP 741: Python configuration C API `): * :c:func:`Py_InitializeFromInitConfig` * :c:func:`PyInitConfig_AddModule` @@ -2520,7 +2874,50 @@ New features (Contributed by Victor Stinner in :gh:`107954`.) -* Add a new import and export API for Python :class:`int` objects (:pep:`757`): +* Add :c:func:`Py_fopen` function to open a file. + This works similarly to the standard C :c:func:`!fopen` function, + instead accepting a Python object for the *path* parameter + and setting an exception on error. + The corresponding new :c:func:`Py_fclose` function should be used + to close a file. + (Contributed by Victor Stinner in :gh:`127350`.) + +* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. + (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) + +* Add :c:func:`PyImport_ImportModuleAttr` and + :c:func:`PyImport_ImportModuleAttrString` helper functions to import a module + and get an attribute of the module. + (Contributed by Victor Stinner in :gh:`128911`.) + +* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, + which has an ambiguous return value. + (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) + +* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. + (Contributed by Sergey B Kirpichev in :gh:`116560`.) + +* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` + and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` + is positive, negative, or zero, respectively. + (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) + +* Add new functions to convert C ```` numbers to/from + Python :class:`int` objects: + + * :c:func:`PyLong_AsInt32` + * :c:func:`PyLong_AsInt64` + * :c:func:`PyLong_AsUInt32` + * :c:func:`PyLong_AsUInt64` + * :c:func:`PyLong_FromInt32` + * :c:func:`PyLong_FromInt64` + * :c:func:`PyLong_FromUInt32` + * :c:func:`PyLong_FromUInt64` + + (Contributed by Victor Stinner in :gh:`120389`.) + +* Add a new import and export API for Python :class:`int` objects + (:pep:`757`): * :c:func:`PyLong_GetNativeLayout` * :c:func:`PyLong_Export` @@ -2531,88 +2928,465 @@ New features (Contributed by Sergey B Kirpichev and Victor Stinner in :gh:`102471`.) -* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier - superclass identification, which attempts to resolve the `type checking issue - `__ mentioned in :pep:`630` - (:gh:`124153`). - -* Add :c:func:`PyUnicode_Equal` function to the limited C API: - test if two strings are equal. - (Contributed by Victor Stinner in :gh:`124502`.) - -* Add :c:func:`PyType_Freeze` function to make a type immutable. - (Contributed by Victor Stinner in :gh:`121654`.) - -* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling - deferred reference counting, as outlined in :pep:`703`. - * Add :c:func:`PyMonitoring_FireBranchLeftEvent` and :c:func:`PyMonitoring_FireBranchRightEvent` for generating :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events, respectively. + (Contributed by Mark Shannon in :gh:`122548`.) -* Add :c:func:`Py_fopen` function to open a file. Similar to the - :c:func:`!fopen` function, but the *path* parameter is a Python object and an - exception is set on error. Add also :c:func:`Py_fclose` function to close a - file. - (Contributed by Victor Stinner in :gh:`127350`.) +* Add :c:func:`PyType_Freeze` function to make a type immutable. + (Contributed by Victor Stinner in :gh:`121654`.) -* Add support of nullable arguments in :c:func:`PyArg_ParseTuple` and - similar functions. - Adding ``?`` after any format unit makes ``None`` be accepted as a value. - (Contributed by Serhiy Storchaka in :gh:`112068`.) +* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot + for easier superclass identification, which attempts to resolve the + type checking issue mentioned in :pep:`PEP 630 <630#type-checking>`. + (Contributed in :gh:`124153`.) + +* Add a new :c:func:`PyUnicode_Equal` function to test if two + strings are equal. + The function is also added to the Limited C API. + (Contributed by Victor Stinner in :gh:`124502`.) + +* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` + object, with the following functions: + + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_WriteASCII` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteWideChar` + + (Contributed by Victor Stinner in :gh:`119182`.) * The ``k`` and ``K`` formats in :c:func:`PyArg_ParseTuple` and similar functions now use :meth:`~object.__index__` if available, like all other integer formats. (Contributed by Serhiy Storchaka in :gh:`112068`.) -* Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for - bit-packing Python version numbers. - (Contributed by Petr Viktorin in :gh:`128629`.) +* Add support for a new ``p`` format unit in :c:func:`Py_BuildValue` + that produces a Python :class:`bool` object from a C integer. + (Contributed by Pablo Galindo in :issue:`45325`.) -* Add :c:func:`PyUnstable_IsImmortal` for determining whether an object is :term:`immortal`, - for debugging purposes. +* Add :c:func:`PyUnstable_IsImmortal` for determining if + an object is :term:`immortal`, for debugging purposes. + (Contributed by Peter Bierma in :gh:`128509`.) -* Add :c:func:`PyImport_ImportModuleAttr` and - :c:func:`PyImport_ImportModuleAttrString` helper functions to import a module - and get an attribute of the module. - (Contributed by Victor Stinner in :gh:`128911`.) +* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling + deferred reference counting, as outlined in :pep:`703`. -* Add support for a new ``p`` format unit in :c:func:`Py_BuildValue` that allows to - take a C integer and produce a Python :class:`bool` object. (Contributed by - Pablo Galindo in :issue:`45325`.) - -* Add :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary` to determine if an object - is a unique temporary object on the interpreter's operand stack. This can - be used in some cases as a replacement for checking if :c:func:`Py_REFCNT` - is ``1`` for Python objects passed as arguments to C API functions. - -* Add :c:func:`PyUnstable_Object_IsUniquelyReferenced` as a replacement for - ``Py_REFCNT(op) == 1`` on :term:`free threaded ` builds. +* Add :c:func:`PyUnstable_Object_IsUniquelyReferenced` as + a replacement for ``Py_REFCNT(op) == 1`` on :term:`free threaded + ` builds. (Contributed by Peter Bierma in :gh:`133140`.) +* Add :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary` to + determine if an object is a unique temporary object on the + interpreter's operand stack. + This can be used in some cases as a replacement for checking + if :c:func:`Py_REFCNT` is ``1`` for Python objects passed + as arguments to C API functions. + (Contributed by Sam Gross in :gh:`133164`.) + Limited C API changes --------------------- -* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` and - :c:func:`Py_REFCNT` are now implemented as an opaque function call to hide - implementation details. +* In the limited C API version 3.14 and newer, :c:func:`Py_TYPE` and + :c:func:`Py_REFCNT` are now implemented as an opaque function call + to hide implementation details. (Contributed by Victor Stinner in :gh:`120600` and :gh:`124127`.) * Remove the :c:macro:`PySequence_Fast_GET_SIZE`, - :c:macro:`PySequence_Fast_GET_ITEM` and :c:macro:`PySequence_Fast_ITEMS` - macros from the limited C API, since these macros never worked in the limited - C API. Keep :c:func:`PySequence_Fast` in the limited C API. + :c:macro:`PySequence_Fast_GET_ITEM`, + and :c:macro:`PySequence_Fast_ITEMS` + macros from the limited C API, since they have always been broken + in the limited C API. (Contributed by Victor Stinner in :gh:`91417`.) +.. _whatsnew314-c-api-removed: + +Removed C APIs +-------------- + +* Creating :c:data:`immutable types ` with + mutable bases was deprecated in Python 3.12, + and now raises a :exc:`TypeError`. + (Contributed by Nikita Sobolev in :gh:`119775`.) + +* Remove ``PyDictObject.ma_version_tag`` member, which was deprecated + in Python 3.12. + Use the :c:func:`PyDict_AddWatcher` API instead. + (Contributed by Sam Gross in :gh:`124296`.) + +* Remove the private ``_Py_InitializeMain()`` function. + It was a :term:`provisional API` added to Python 3.8 by :pep:`587`. + (Contributed by Victor Stinner in :gh:`129033`.) + +* Remove the undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` + and :c:member:`!PyThreadState.c_recursion_remaining`. + These were added in 3.13 and have been removed without deprecation. + Use :c:func:`Py_EnterRecursiveCall` to guard against runaway + recursion in C code. + (Removed by Petr Viktorin in :gh:`133079`, see also :gh:`130396`.) + + +.. _whatsnew314-c-api-deprecated: + +Deprecated C APIs +----------------- + +* The :c:macro:`!Py_HUGE_VAL` macro is now :term:`soft deprecated`. + Use :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`120026`.) + +* The :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY`, + and :c:macro:`!Py_IS_FINITE` macros are now :term:`soft deprecated`. + Use :c:macro:`!isnan`, :c:macro:`!isinf` and :c:macro:`!isfinite` + instead, available from :file:`math.h` since C99. + (Contributed by Sergey B Kirpichev in :gh:`119613`.) + +* Non-tuple sequences are now deprecated as argument for the ``(items)`` + format unit in :c:func:`PyArg_ParseTuple` and other :ref:`argument + parsing ` functions if *items* contains format units + which store a :ref:`borrowed buffer ` or a + :term:`borrowed reference`. + (Contributed by Serhiy Storchaka in :gh:`50333`.) + +* The ``_PyMonitoring_FireBranchEvent`` function is now deprecated + and should be replaced with calls to + :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent`. + +* The previously undocumented function :c:func:`PySequence_In` is + now :term:`soft deprecated`. + Use :c:func:`PySequence_Contains` instead. + (Contributed by Yuki Kobayashi in :gh:`127896`.) + +.. Add C API deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + + +.. _whatsnew314-build-changes: + +Build changes +============= + +* :pep:`776`: Emscripten is now an officially supported platform at + :pep:`tier 3 <11#tier-3>`. As a part of this effort, more than 25 bugs in + `Emscripten libc`__ were fixed. Emscripten now includes support + for :mod:`ctypes`, :mod:`termios`, and :mod:`fcntl`, as well as + experimental support for the new :ref:`default interactive shell + `. + (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) + + __ https://emscripten.org/docs/porting/emscripten-runtime-environment.html + +* Official Android binary releases are now provided on python.org__. + + __ https://www.python.org/downloads/android/ + +* GNU Autoconf 2.72 is now required to generate :file:`configure`. + (Contributed by Erlend Aasland in :gh:`115765`.) + +* ``wasm32-unknown-emscripten`` is now a :pep:`11` tier 3 platform. + (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) + +* ``#pragma``-based linking with ``python3*.lib`` can now be switched off + with :c:expr:`Py_NO_LINK_LIB`. + (Contributed by Jean-Christophe Fillion-Robin in :gh:`82909`.) + +* CPython now enables a set of recommended compiler options by default + for improved security. + Use the :option:`--disable-safety` :file:`configure` option to disable them, + or the :option:`--enable-slower-safety` option for a larger set + of compiler options, albeit with a performance cost. + +* The ``WITH_FREELISTS`` macro and ``--without-freelists`` :file:`configure` + option have been removed. + +* The new :file:`configure` option :option:`--with-tail-call-interp` + may be used to enable the experimental tail call interpreter. + See :ref:`whatsnew314-tail-call-interpreter` for further details. + +* To disable the new remote debugging support, use the + :option:`--without-remote-debug` :file:`configure` option. + This may be useful for security reasons. + +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + + +.. _whatsnew314-build_details: + +:file:`build-details.json` +-------------------------- + +Installations of Python now contain a new file, :file:`build-details.json`. +This is a static JSON document containing build details for CPython, +to allow for introspection without needing to run code. +This is helpful for use-cases such as Python launchers, cross-compilation, +and so on. + +:file:`build-details.json` must be installed in the platform-independent +standard library directory. This corresponds to the :ref:`'stdlib' +` :mod:`sysconfig` installation path, +which can be found by running ``sysconfig.get_path('stdlib')``. + +.. seealso:: + :pep:`739` -- ``build-details.json`` 1.0 -- a static description file + for Python build details + + +.. _whatsnew314-no-more-pgp: + +Discontinuation of PGP signatures +--------------------------------- + +PGP (Pretty Good Privacy) signatures will not be provided +for releases of Python 3.14 or future versions. +To verify CPython artifacts, users must use `Sigstore verification materials +`__. +Releases have been signed using Sigstore_ since Python 3.11. + +This change in release process was specified in :pep:`761`. + +.. _Sigstore: https://www.sigstore.dev/ + + +.. _whatsnew314-free-threaded-now-supported: + +Free-threaded Python is officially supported +-------------------------------------------- + +The free-threaded build of Python is now supported and no longer experimental. +This is the start of `phase II `__ where +free-threaded Python is officially supported but still optional. + +The free-threading team are confident that the project is on the right path, +and appreciate the continued dedication from everyone working to make +free-threading ready for broader adoption across the Python community. + +With these recommendations and the acceptance of this PEP, the Python developer +community should broadly advertise that free-threading is a supported +Python build option now and into the future, and that it will not be removed +without a proper deprecation schedule. + +Any decision to transition to `phase III `__, +with free-threading as the default or sole build of Python is still undecided, +and dependent on many factors both within CPython itself and the community. +This decision is for the future. + +.. seealso:: + + :pep:`779` + + `PEP 779's acceptance `__ + + +.. _whatsnew314-jit-compiler: + +Binary releases for the experimental just-in-time compiler +---------------------------------------------------------- + +The official macOS and Windows release binaries now include an *experimental* +just-in-time (JIT) compiler. Although it is **not** recommended for production +use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an +environment variable. Downstream source builds and redistributors can use the +:option:`--enable-experimental-jit=yes-off` configuration option for similar +behavior. + +The JIT is at an early stage and still in active development. As such, the +typical performance impact of enabling it can range from 10% slower to 20% +faster, depending on workload. To aid in testing and evaluation, a set of +introspection functions has been provided in the :data:`sys._jit` namespace. +:func:`sys._jit.is_available` can be used to determine if the current executable +supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell +if JIT compilation has been enabled for the current process. + +Currently, the most significant missing functionality is that native debuggers +and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames +(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to +work without modification). Free-threaded builds do not support JIT compilation. + +Please report any bugs or major performance regressions that you encounter! + +.. seealso:: :pep:`744` + + Porting to Python 3.14 ----------------------- +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Changes in the Python API +------------------------- + +* On Unix platforms other than macOS, *forkserver* is now the default + :ref:`start method ` for :mod:`multiprocessing` + and :class:`~concurrent.futures.ProcessPoolExecutor`, instead of *fork*. + + See :ref:`(1) ` and + :ref:`(2) ` for details. + + If you encounter :exc:`NameError`\s or pickling errors coming out of + :mod:`multiprocessing` or :mod:`concurrent.futures`, see the + :ref:`forkserver restrictions `. + + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. + +* :class:`functools.partial` is now a method descriptor. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) + +* The :ref:`garbage collector is now incremental `, + which means that the behavior of :func:`gc.collect` changes slightly: + + * ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. + * Other calls to :func:`!gc.collect` are unchanged. + +* The :func:`locale.nl_langinfo` function now temporarily sets the ``LC_CTYPE`` + locale in some cases. + This temporary change affects other threads. + (Contributed by Serhiy Storchaka in :gh:`69998`.) + +* :class:`types.UnionType` is now an alias for :class:`typing.Union`, + causing changes in some behaviors. + See :ref:`above ` for more details. + (Contributed by Jelle Zijlstra in :gh:`105499`.) + +* The runtime behavior of annotations has changed in various ways; see + :ref:`above ` for details. While most code that interacts + with annotations should continue to work, some undocumented details may behave + differently. + +* As part of making the :mod:`mimetypes` CLI public, + it now exits with ``1`` on failure instead of ``0`` + and ``2`` on incorrect command-line parameters instead of ``1``. + Error messages are now printed to stderr. + +* The ``\B`` pattern in regular expression now matches the empty string + when given as the entire pattern, which may cause behavioural changes. + +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. + + +.. _whatsnew314-porting-annotations: + +Changes in annotations (:pep:`649` and :pep:`749`) +-------------------------------------------------- + +This section contains guidance on changes that may be needed to annotations +or Python code that interacts with or introspects annotations, +due to the changes related to :ref:`deferred evaluation of annotations +`. + +In the majority of cases, working code from older versions of Python +will not require any changes. + + +Implications for annotated code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you define annotations in your code (for example, for use with a static type +checker), then this change probably does not affect you: you can keep +writing annotations the same way you did with previous versions of Python. + +You will likely be able to remove quoted strings in annotations, which are frequently +used for forward references. Similarly, if you use ``from __future__ import annotations`` +to avoid having to write strings in annotations, you may well be able to +remove that import once you support only Python 3.14 and newer. +However, if you rely on third-party libraries that read annotations, +those libraries may need changes to support unquoted annotations before they +work as expected. + + +Implications for readers of ``__annotations__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your code reads the :attr:`~object.__annotations__` attribute on objects, +you may want to make changes in order to support code that relies on +deferred evaluation of annotations. +For example, you may want to use :func:`annotationlib.get_annotations` with +the :attr:`~annotationlib.Format.FORWARDREF` format, +as the :mod:`dataclasses` module now does. + +The external :pypi:`typing_extensions` package provides partial backports +of some of the functionality of the :mod:`annotationlib` module, +such as the :class:`~annotationlib.Format` enum and +the :func:`~annotationlib.get_annotations` function. +These can be used to write cross-version code that takes advantage of +the new behavior in Python 3.14. + + +Related changes +^^^^^^^^^^^^^^^ + +The changes in Python 3.14 are designed to rework how :attr:`!__annotations__` +works at runtime while minimizing breakage to code that contains +annotations in source code and to code that reads :attr:`!__annotations__`. +However, if you rely on undocumented details of the annotation behavior +or on private functions in the standard library, there are many ways in which +your code may not work in Python 3.14. +To safeguard your code against future changes, only use the documented +functionality of the :mod:`annotationlib` module. + +In particular, do not read annotations directly from the namespace dictionary +attribute of type objects. +Use :func:`annotationlib.get_annotate_from_class_namespace` during class +construction and :func:`annotationlib.get_annotations` afterwards. + +In previous releases, it was sometimes possible to access class annotations +from an instance of an annotated class. This behavior was undocumented +and accidental, and will no longer work in Python 3.14. + + +``from __future__ import annotations`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` +:ref:`future statement `, which turns all annotations into strings. + +However, this statement is now deprecated and it is expected to be removed +in a future version of Python. +This removal will not happen until after Python 3.13 reaches its end of life +in 2029, being the last version of Python without support for deferred +evaluation of annotations. + +In Python 3.14, the behavior of code using ``from __future__ import annotations`` +is unchanged. + + +Changes in the C API +-------------------- * :c:func:`Py_Finalize` now deletes all interned strings. This - is backwards incompatible to any C-Extension that holds onto an interned + is backwards incompatible to any C extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will normally result in crashes during the execution of the subsequent call to @@ -2657,113 +3431,9 @@ Porting to Python 3.14 * ``_Py_GetConfig()``: :c:func:`PyConfig_Get` and :c:func:`PyConfig_GetInt` * ``_Py_HashBytes()``: :c:func:`Py_HashBuffer` * ``_Py_fopen_obj()``: :c:func:`Py_fopen` + * ``PyMutex_IsLocked()`` : :c:func:`PyMutex_IsLocked` The `pythoncapi-compat project`_ can be used to get most of these new functions on Python 3.13 and older. .. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ - - -.. _whatsnew314-c-api-deprecated: - -Deprecated ----------- - -* The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`, - use :c:macro:`!Py_INFINITY` instead. - (Contributed by Sergey B Kirpichev in :gh:`120026`.) - -* Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` - and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, - use instead :c:macro:`!isnan`, :c:macro:`!isinf` and - :c:macro:`!isfinite` available from :file:`math.h` - since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) - -* Non-tuple sequences are deprecated as argument for the ``(items)`` - format unit in :c:func:`PyArg_ParseTuple` and other - :ref:`argument parsing ` functions if *items* contains - format units which store a :ref:`borrowed buffer ` - or a :term:`borrowed reference`. - (Contributed by Serhiy Storchaka in :gh:`50333`.) - -* The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. - Use :c:func:`PySequence_Contains` instead. - (Contributed by Yuki Kobayashi in :gh:`127896`.) - -.. Add C API deprecations above alphabetically, not here at the end. - -* The ``PyMonitoring_FireBranchEvent`` function is deprecated and should - be replaced with calls to :c:func:`PyMonitoring_FireBranchLeftEvent` - and :c:func:`PyMonitoring_FireBranchRightEvent`. - -* The following private functions are deprecated and planned for removal in - Python 3.18: - - * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. - * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. - * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. - * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. - * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: - use :c:func:`PyLongWriter_Create`. - * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. - * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. - * :c:func:`!_PyUnicodeWriter_Init`: - replace ``_PyUnicodeWriter_Init(&writer)`` with - :c:func:`writer = PyUnicodeWriter_Create(0) `. - * :c:func:`!_PyUnicodeWriter_Finish`: - replace ``_PyUnicodeWriter_Finish(&writer)`` with - :c:func:`PyUnicodeWriter_Finish(writer) `. - * :c:func:`!_PyUnicodeWriter_Dealloc`: - replace ``_PyUnicodeWriter_Dealloc(&writer)`` with - :c:func:`PyUnicodeWriter_Discard(writer) `. - * :c:func:`!_PyUnicodeWriter_WriteChar`: - replace ``_PyUnicodeWriter_WriteChar(&writer, ch)`` with - :c:func:`PyUnicodeWriter_WriteChar(writer, ch) `. - * :c:func:`!_PyUnicodeWriter_WriteStr`: - replace ``_PyUnicodeWriter_WriteStr(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteStr(writer, str) `. - * :c:func:`!_PyUnicodeWriter_WriteSubstring`: - replace ``_PyUnicodeWriter_WriteSubstring(&writer, str, start, end)`` with - :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. - * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: - replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. - * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: - replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. - * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. - * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. - - The `pythoncapi-compat project`_ can be used to get these new public - functions on Python 3.13 and older. - (Contributed by Victor Stinner in :gh:`128863`.) - -.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst - -.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst - -.. include:: ../deprecations/c-api-pending-removal-in-future.rst - - -.. _whatsnew314-c-api-removed: - -Removed -------- - -* Creating :c:data:`immutable types ` with mutable - bases was deprecated since 3.12 and now raises a :exc:`TypeError`. - -* Remove ``PyDictObject.ma_version_tag`` member which was deprecated since - Python 3.12. Use the :c:func:`PyDict_AddWatcher` API instead. - (Contributed by Sam Gross in :gh:`124296`.) - -* Remove the private ``_Py_InitializeMain()`` function. It was a - :term:`provisional API` added to Python 3.8 by :pep:`587`. - (Contributed by Victor Stinner in :gh:`129033`.) - -* The undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` and - :c:member:`!PyThreadState.c_recursion_remaining`, added in 3.13, are removed - without a deprecation period. - Please use :c:func:`Py_EnterRecursiveCall` to guard against runaway recursion - in C code. - (Removed in :gh:`133079`, see also :gh:`130396`.) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst new file mode 100644 index 00000000000..1bd82545e58 --- /dev/null +++ b/Doc/whatsnew/3.15.rst @@ -0,0 +1,1277 @@ + +**************************** + What's new in Python 3.15 +**************************** + +:Editor: Hugo van Kemenade + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.15, compared to 3.14. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.15 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.15. + Brevity is key. + + +.. PEP-sized items next. + +* :pep:`799`: :ref:`A dedicated profiling package for organizing Python + profiling tools ` +* :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding + ` +* :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object + ` +* :ref:`Improved error messages ` + + +New features +============ + +.. _whatsnew315-profiling-package: + +:pep:`799`: A dedicated profiling package +----------------------------------------- + +A new :mod:`!profiling` module has been added to organize Python's built-in +profiling tools under a single, coherent namespace. This module contains: + +* :mod:`!profiling.tracing`: deterministic function-call tracing (relocated from + :mod:`cProfile`). +* :mod:`!profiling.sampling`: a new statistical sampling profiler (named Tachyon). + +The :mod:`cProfile` module remains as an alias for backwards compatibility. +The :mod:`profile` module is deprecated and will be removed in Python 3.17. + +.. seealso:: :pep:`799` for further details. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.) + + +.. _whatsnew315-sampling-profiler: + +Tachyon: High frequency statistical sampling profiler +----------------------------------------------------- + +A new statistical sampling profiler (Tachyon) has been added as +:mod:`!profiling.sampling`. This profiler enables low-overhead performance analysis of +running Python processes without requiring code modification or process restart. + +Unlike deterministic profilers (:mod:`cProfile` and :mod:`profile`) that instrument +every function call, the sampling profiler periodically captures stack traces from +running processes. This approach provides virtually zero overhead while achieving +sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler +available for Python (at the time of its contribution) and ideal for debugging +performance issues in production environments. This capability is particularly +valuable for debugging performance issues in production systems where traditional +profiling approaches would be too intrusive. + +Key features include: + +* **Zero-overhead profiling**: Attach to any running Python process without + affecting its performance. Ideal for production debugging where you can't afford + to restart or slow down your application. + +* **No code modification required**: Profile existing applications without restart. + Simply point the profiler at a running process by PID and start collecting data. + +* **Flexible target modes**: + + * Profile running processes by PID (``attach``) - attach to already-running applications + * Run and profile scripts directly (``run``) - profile from the very start of execution + * Execute and profile modules (``run -m``) - profile packages run as ``python -m module`` + +* **Multiple profiling modes**: Choose what to measure based on your performance investigation: + + * **Wall-clock time** (``--mode wall``, default): Measures real elapsed time including I/O, + network waits, and blocking operations. Use this to understand where your program spends + calendar time, including when waiting for external resources. + * **CPU time** (``--mode cpu``): Measures only active CPU execution time, excluding I/O waits + and blocking. Use this to identify CPU-bound bottlenecks and optimize computational work. + * **GIL-holding time** (``--mode gil``): Measures time spent holding Python's Global Interpreter + Lock. Use this to identify which threads dominate GIL usage in multi-threaded applications. + +* **Thread-aware profiling**: Option to profile all threads (``-a``) or just the main thread, + essential for understanding multi-threaded application behavior. + +* **Multiple output formats**: Choose the visualization that best fits your workflow: + + * ``--pstats``: Detailed tabular statistics compatible with :mod:`pstats`. Shows function-level + timing with direct and cumulative samples. Best for detailed analysis and integration with + existing Python profiling tools. + * ``--collapsed``: Generates collapsed stack traces (one line per stack). This format is + specifically designed for creating flamegraphs with external tools like Brendan Gregg's + FlameGraph scripts or speedscope. + * ``--flamegraph``: Generates a self-contained interactive HTML flamegraph using D3.js. + Opens directly in your browser for immediate visual analysis. Flamegraphs show the call + hierarchy where width represents time spent, making it easy to spot bottlenecks at a glance. + * ``--gecko``: Generates Gecko Profiler format compatible with Firefox Profiler + (https://profiler.firefox.com). Upload the output to Firefox Profiler for advanced + timeline-based analysis with features like stack charts, markers, and network activity. + * ``--heatmap``: Generates an interactive HTML heatmap visualization with line-level sample + counts. Creates a directory with per-file heatmaps showing exactly where time is spent + at the source code level. + +* **Live interactive mode**: Real-time TUI profiler with a top-like interface (``--live``). + Monitor performance as your application runs with interactive sorting and filtering. + +* **Async-aware profiling**: Profile async/await code with task-based stack reconstruction + (``--async-aware``). See which coroutines are consuming time, with options to show only + running tasks or all tasks including those waiting. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) + + +.. _whatsnew315-improved-error-messages: + +Improved error messages +----------------------- + +* The interpreter now provides more helpful suggestions in :exc:`AttributeError` + exceptions when accessing an attribute on an object that does not exist, but + a similar attribute is available through one of its members. + + For example, if the object has an attribute that itself exposes the requested + name, the error message will suggest accessing it via that inner attribute: + + .. code-block:: python + + @dataclass + class Circle: + radius: float + + @property + def area(self) -> float: + return pi * self.radius**2 + + class Container: + def __init__(self, inner: Circle) -> None: + self.inner = inner + + circle = Circle(radius=4.0) + container = Container(circle) + print(container.area) + + Running this code now produces a clearer suggestion: + + .. code-block:: pycon + + Traceback (most recent call last): + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ + AttributeError: 'Container' object has no attribute 'area'. Did you mean: 'inner.area'? + + +Other language changes +====================== + +.. _whatsnew315-utf8-default: + +* Python now uses UTF-8_ as the default encoding, independent of the system's + environment. This means that I/O operations without an explicit encoding, + for example, ``open('flying-circus.txt')``, will use UTF-8. + UTF-8 is a widely-supported Unicode_ character encoding that has become a + *de facto* standard for representing text, including nearly every webpage + on the internet, many common file formats, programming languages, and more. + + This only applies when no ``encoding`` argument is given. For best + compatibility between versions of Python, ensure that an explicit ``encoding`` + argument is always provided. The :ref:`opt-in encoding warning ` + can be used to identify code that may be affected by this change. + The special ``encoding='locale'`` argument uses the current locale + encoding, and has been supported since Python 3.10. + + To retain the previous behaviour, Python's UTF-8 mode may be disabled with + the :envvar:`PYTHONUTF8=0 ` environment variable or the + :option:`-X utf8=0 <-X>` command-line option. + + .. seealso:: :pep:`686` for further details. + + .. _UTF-8: https://en.wikipedia.org/wiki/UTF-8 + .. _Unicode: https://home.unicode.org/ + + (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) + +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) + +* The interpreter now tries to provide a suggestion when + :func:`delattr` fails due to a missing attribute. + When an attribute name that closely resembles an existing attribute is used, + the interpreter will suggest the correct attribute name in the error message. + For example: + + .. doctest:: + + >>> class A: + ... pass + >>> a = A() + >>> a.abcde = 1 + >>> del a.abcdf # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? + + (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + +* Unraisable exceptions are now highlighted with color by default. This can be + controlled by :ref:`environment variables `. + (Contributed by Peter Bierma in :gh:`134170`.) + +* The :meth:`~object.__repr__` of :class:`ImportError` and :class:`ModuleNotFoundError` + now shows "name" and "path" as ``name=`` and ``path=`` if they were given + as keyword arguments at construction time. + (Contributed by Serhiy Storchaka, Oleg Iarygin, and Yoav Nir in :gh:`74185`.) + +* The :attr:`~object.__dict__` and :attr:`!__weakref__` descriptors now use a + single descriptor instance per interpreter, shared across all types that + need them. + This speeds up class creation, and helps avoid reference cycles. + (Contributed by Petr Viktorin in :gh:`135228`.) + +* The :option:`-W` option and the :envvar:`PYTHONWARNINGS` environment variable + can now specify regular expressions instead of literal strings to match + the warning message and the module name, if the corresponding field starts + and ends with a forward slash (``/``). + (Contributed by Serhiy Storchaka in :gh:`134716`.) + +* Functions that take timestamp or timeout arguments now accept any real + numbers (such as :class:`~decimal.Decimal` and :class:`~fractions.Fraction`), + not only integers or floats, although this does not improve precision. + (Contributed by Serhiy Storchaka in :gh:`67795`.) + +.. _whatsnew315-bytearray-take-bytes: + +* Added :meth:`bytearray.take_bytes(n=None, /) ` to take + bytes out of a :class:`bytearray` without copying. This enables optimizing code + which must return :class:`bytes` after working with a mutable buffer of bytes + such as data buffering, network protocol parsing, encoding, decoding, + and compression. Common code patterns which can be optimized with + :func:`~bytearray.take_bytes` are listed below. + + .. list-table:: Suggested optimizing refactors + :header-rows: 1 + + * - Description + - Old + - New + + * - Return :class:`bytes` after working with :class:`bytearray` + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return bytes(buffer) + + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return buffer.take_bytes() + + * - Empty a buffer getting the bytes + - .. code:: python + + buffer = bytearray(1024) + ... + data = bytes(buffer) + buffer.clear() + + - .. code:: python + + buffer = bytearray(1024) + ... + data = buffer.take_bytes() + + * - Split a buffer at a specific separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n + 1]) + del buffer[:n + 1] + assert data == b'abc' + assert buffer == bytearray(b'def') + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = buffer.take_bytes(n + 1) + + * - Split a buffer at a specific separator; discard after the separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n]) + buffer.clear() + assert data == b'abc' + assert len(buffer) == 0 + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + buffer.resize(n) + data = buffer.take_bytes() + + (Contributed by Cody Maloney in :gh:`139871`.) + +* Many functions related to compiling or parsing Python code, such as + :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow the module + name to be passed. It is needed to unambiguously :ref:`filter ` + syntax warnings by module name. + (Contributed by Serhiy Storchaka in :gh:`135801`.) + +* Allowed defining the *__dict__* and *__weakref__* :ref:`__slots__ ` + for any class. + (Contributed by Serhiy Storchaka in :gh:`41779`.) + + +New modules +=========== + +math.integer +------------ + +This module provides access to the mathematical functions for integer +arguments (:pep:`791`). +(Contributed by Serhiy Storchaka in :gh:`81313`.) + + +Improved modules +================ + +argparse +-------- + +* The :class:`~argparse.BooleanOptionalAction` action supports now single-dash + long options and alternate prefix characters. + (Contributed by Serhiy Storchaka in :gh:`138525`.) + +* Changed the *suggest_on_error* parameter of :class:`argparse.ArgumentParser` to + default to ``True``. This enables suggestions for mistyped arguments by default. + (Contributed by Jakob Schluse in :gh:`140450`.) + +calendar +-------- + +* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support + dark mode and have been migrated to the HTML5 standard for improved accessibility. + (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) + +* The :mod:`calendar`'s :ref:`command-line ` HTML output now + accepts the year-month option: ``python -m calendar -t html 2009 06``. + (Contributed by Pål Grønås Drange in :gh:`140212`.) + + +collections +----------- + +* Added :meth:`!collections.Counter.__xor__` and + :meth:`!collections.Counter.__ixor__` to compute the symmetric difference + between :class:`~collections.Counter` objects. + (Contributed by Raymond Hettinger in :gh:`138682`.) + +collections.abc +--------------- + +* :class:`collections.abc.ByteString` has been removed from + ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been + deprecated since Python 3.12, and is scheduled for removal in Python 3.17. + +* The following statements now cause ``DeprecationWarning``\ s to be emitted at + runtime: + + * ``from collections.abc import ByteString`` + * ``import collections.abc; collections.abc.ByteString``. + + ``DeprecationWarning``\ s were already emitted if + :class:`collections.abc.ByteString` was subclassed or used as the second + argument to :func:`isinstance` or :func:`issubclass`, but warnings were not + previously emitted if it was merely imported or accessed from the + :mod:`!collections.abc` module. + + +concurrent.futures +------------------ + +* Improved error reporting when a child process in a + :class:`concurrent.futures.ProcessPoolExecutor` terminates abruptly. + The resulting traceback will now tell you the PID and exit code of the + terminated process. + (Contributed by Jonathan Berg in :gh:`139486`.) + + +dataclasses +----------- + +* Annotations for generated ``__init__`` methods no longer include internal + type names. + + +dbm +--- + +* Added new :meth:`!reorganize` methods to :mod:`dbm.dumb` and :mod:`dbm.sqlite3` + which allow to recover unused free space previously occupied by deleted entries. + (Contributed by Andrea Oliveri in :gh:`134004`.) + + + +difflib +------- + + .. _whatsnew315-color-difflib: + +* Introduced the optional *color* parameter to :func:`difflib.unified_diff`, + enabling color output similar to :program:`git diff`. + This can be controlled by :ref:`environment variables + `. + (Contributed by Douglas Thor in :gh:`133725`.) + +* Improved the styling of HTML diff pages generated by the :class:`difflib.HtmlDiff` + class, and migrated the output to the HTML5 standard. + (Contributed by Jiahao Li in :gh:`134580`.) + + +functools +--------- + +* :func:`~functools.singledispatchmethod` now supports non-:term:`descriptor` + callables. + (Contributed by Serhiy Storchaka in :gh:`140873`.) + + +hashlib +------- + +* Ensure that hash functions guaranteed to be always *available* exist as + attributes of :mod:`hashlib` even if they will not work at runtime due to + missing backend implementations. For instance, ``hashlib.md5`` will no + longer raise :exc:`AttributeError` if OpenSSL is not available and Python + has been built without MD5 support. + (Contributed by Bénédikt Tran in :gh:`136929`.) + + +http.client +----------- + +* A new *max_response_headers* keyword-only parameter has been added to + :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` + constructors. This parameter overrides the default maximum number of allowed + response headers. + (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) + + +http.cookies +------------ + +* Allow '``"``' double quotes in cookie values. + (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) + + +inspect +------- + +* Add parameters *inherit_class_doc* and *fallback_to_class_doc* + for :func:`~inspect.getdoc`. + (Contributed by Serhiy Storchaka in :gh:`132686`.) + + +locale +------ + +* :func:`~locale.setlocale` now supports language codes with ``@``-modifiers. + ``@``-modifiers are no longer silently removed in :func:`~locale.getlocale`, + but included in the language code. + (Contributed by Serhiy Storchaka in :gh:`137729`.) + + +math +---- + +* Add :func:`math.isnormal` and :func:`math.issubnormal` functions. + (Contributed by Sergey B Kirpichev in :gh:`132908`.) + +* Add :func:`math.fmax`, :func:`math.fmin` and :func:`math.signbit` functions. + (Contributed by Bénédikt Tran in :gh:`135853`.) + + +mimetypes +--------- + +* Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) +* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) +* Rename ``application/x-texinfo`` to ``application/texinfo``. + (Contributed by Charlie Lin in :gh:`140165`.) +* Changed the MIME type for ``.ai`` files to ``application/pdf``. + (Contributed by Stan Ulbrych in :gh:`141239`.) + + +mmap +---- + +* :class:`mmap.mmap` now has a *trackfd* parameter on Windows; + if it is ``False``, the file handle corresponding to *fileno* will + not be duplicated. + (Contributed by Serhiy Storchaka in :gh:`78502`.) + + +os +-- + +* Add :func:`os.statx` on Linux kernel versions 4.11 and later with + glibc versions 2.28 and later. + (Contributed by Jeffrey Bosboom and Victor Stinner in :gh:`83714`.) + + +os.path +------- + +* Add support of the all-but-last mode in :func:`~os.path.realpath`. + (Contributed by Serhiy Storchaka in :gh:`71189`.) + +* The *strict* parameter to :func:`os.path.realpath` accepts a new value, + :data:`os.path.ALLOW_MISSING`. + If used, errors other than :exc:`FileNotFoundError` will be re-raised; + the resulting path can be missing but it will be free of symlinks. + (Contributed by Petr Viktorin for :cve:`2025-4517`.) + + +resource +-------- + +* Add new constants: :data:`~resource.RLIMIT_NTHR`, + :data:`~resource.RLIMIT_UMTXP`, :data:`~resource.RLIMIT_THREADS`, + :data:`~resource.RLIM_SAVED_CUR`, and :data:`~resource.RLIM_SAVED_MAX`. + (Contributed by Serhiy Storchaka in :gh:`137512`.) + + +shelve +------ + +* Added new :meth:`!reorganize` method to :mod:`shelve` used to recover unused free + space previously occupied by deleted entries. + (Contributed by Andrea Oliveri in :gh:`134004`.) + + +socket +------ + +* Add constants for the ISO-TP CAN protocol. + (Contributed by Patrick Menschel and Stefan Tatschner in :gh:`86819`.) + + +sqlite3 +------- + +* The :ref:`command-line interface ` has several new features: + + * SQL keyword completion on . + (Contributed by Long Tan in :gh:`133393`.) + + * Prompts, error messages, and help text are now colored. + This is enabled by default, see :ref:`using-on-controlling-color` for + details. + (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`.) + + * Table, index, trigger, view, column, function, and schema completion on . + (Contributed by Long Tan in :gh:`136101`.) + + +ssl +--- + +* Indicate through :data:`ssl.HAS_PSK_TLS13` whether the :mod:`ssl` module + supports "External PSKs" in TLSv1.3, as described in RFC 9258. + (Contributed by Will Childs-Klein in :gh:`133624`.) + +* Added new methods for managing groups used for SSL key agreement + + * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing + key agreement, extending the previous + :meth:`ssl.SSLContext.set_ecdh_curve` method. + This new API provides the ability to list multiple groups and + supports fixed-field and post-quantum groups in addition to ECDH + curves. This method can also be used to control what key shares + are sent in the TLS handshake. + * :meth:`ssl.SSLSocket.group` returns the group selected for doing key + agreement on the current connection after the TLS handshake completes. + This call requires OpenSSL 3.2 or later. + * :meth:`ssl.SSLContext.get_groups` returns a list of all available key + agreement groups compatible with the minimum and maximum TLS versions + currently set in the context. This call requires OpenSSL 3.5 or later. + + (Contributed by Ron Frederick in :gh:`136306`.) + +* Added a new method :meth:`ssl.SSLContext.set_ciphersuites` for setting TLS 1.3 + ciphers. For TLS 1.2 or earlier, :meth:`ssl.SSLContext.set_ciphers` should + continue to be used. Both calls can be made on the same context and the + selected cipher suite will depend on the TLS version negotiated when a + connection is made. + (Contributed by Ron Frederick in :gh:`137197`.) + +* Added new methods for managing signature algorithms: + + * :func:`ssl.get_sigalgs` returns a list of all available TLS signature + algorithms. This call requires OpenSSL 3.4 or later. + * :meth:`ssl.SSLContext.set_client_sigalgs` sets the signature algorithms + allowed for certificate-based client authentication. + * :meth:`ssl.SSLContext.set_server_sigalgs` sets the signature algorithms + allowed for the server to complete the TLS handshake. + * :meth:`ssl.SSLSocket.client_sigalg` returns the signature algorithm + selected for client authentication on the current connection. This call + requires OpenSSL 3.5 or later. + * :meth:`ssl.SSLSocket.server_sigalg` returns the signature algorithm + selected for the server to complete the TLS handshake on the current + connection. This call requires OpenSSL 3.5 or later. + + (Contributed by Ron Frederick in :gh:`138252`.) + + +sys +--- + +* Add :data:`sys.abi_info` namespace to improve access to ABI information. + (Contributed by Klaus Zimmermann in :gh:`137476`.) + + +tarfile +------- + +* :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to + avoid path traversal attacks. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) +* :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes + when a directory was removed or replaced by another kind of file. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + now (re-)apply the extraction filter when substituting a link (hard or + symbolic) with a copy of another archive member, and when fixing up + directory attributes. + The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. + (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + no longer extract rejected members when + :func:`~tarfile.TarFile.errorlevel` is zero. + (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` + and :cve:`2025-4435`.) +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + now replace slashes by backslashes in symlink targets on Windows to prevent + creation of corrupted links. + (Contributed by Christoph Walcher in :gh:`57911`.) + + +timeit +------ + +* The command-line interface now colorizes error tracebacks + by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Yi Hong in :gh:`139374`.) + +tkinter +------- + +* The :meth:`!tkinter.Text.search` method now supports two additional + arguments: *nolinestop* which allows the search to + continue across line boundaries; + and *strictlimits* which restricts the search to within the specified range. + (Contributed by Rihaan Meher in :gh:`130848`) + +* A new method :meth:`!tkinter.Text.search_all` has been introduced. + This method allows for searching for all matches of a pattern + using Tcl's ``-all`` and ``-overlap`` options. + (Contributed by Rihaan Meher in :gh:`130848`) + +types +------ + +* Expose the write-through :func:`locals` proxy type + as :data:`types.FrameLocalsProxyType`. + This represents the type of the :attr:`frame.f_locals` attribute, + as described in :pep:`667`. + + +unicodedata +----------- + +* The Unicode database has been updated to Unicode 17.0.0. + +* Add :func:`unicodedata.isxidstart` and :func:`unicodedata.isxidcontinue` + functions to check whether a character can start or continue a + `Unicode Standard Annex #31 `_ identifier. + (Contributed by Stan Ulbrych in :gh:`129117`.) + + +unittest +-------- + +* :func:`unittest.TestCase.assertLogs` will now accept a formatter + to control how messages are formatted. + (Contributed by Garry Cairns in :gh:`134567`.) + + +venv +---- + +* On POSIX platforms, platlib directories will be created if needed when + creating virtual environments, instead of using ``lib64 -> lib`` symlink. + This means purelib and platlib of virtual environments no longer share the + same ``lib`` directory on platforms where :data:`sys.platlibdir` is not + equal to ``lib``. + (Contributed by Rui Xi in :gh:`133951`.) + + +warnings +-------- + +* Improve filtering by module in :func:`warnings.warn_explicit` if no *module* + argument is passed. + It now tests the module regular expression in the warnings filter not only + against the filename with ``.py`` stripped, but also against module names + constructed starting from different parent directories of the filename + (with ``/__init__.py``, ``.py`` and, on Windows, ``.pyw`` stripped). + (Contributed by Serhiy Storchaka in :gh:`135801`.) + + +xml.parsers.expat +----------------- + +* Add :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerActivationThreshold` + and :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerMaximumAmplification` + to :ref:`xmlparser ` objects to tune protections against + disproportional amounts of dynamic memory usage from within an Expat parser. + (Contributed by Bénédikt Tran in :gh:`90949`.) + +* Add :meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold` + and :meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification` + to :ref:`xmlparser ` objects to tune protections against + `billion laughs`_ attacks. + (Contributed by Bénédikt Tran in :gh:`90949`.) + + .. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack + + +zlib +---- + +* Allow combining two Adler-32 checksums via :func:`~zlib.adler32_combine`. + (Contributed by Callum Attryde and Bénédikt Tran in :gh:`134635`.) + +* Allow combining two CRC-32 checksums via :func:`~zlib.crc32_combine`. + (Contributed by Bénédikt Tran in :gh:`134635`.) + + +.. Add improved modules above alphabetically, not here at the end. + +Optimizations +============= + +csv +--- + +* :meth:`csv.Sniffer.sniff` delimiter detection is now up to 1.6x faster. + (Contributed by Maurycy Pawłowski-Wieroński in :gh:`137628`.) + + +Removed +======= + +ctypes +------ + +* Removed the undocumented function :func:`!ctypes.SetPointerType`, + which has been deprecated since Python 3.13. + (Contributed by Bénédikt Tran in :gh:`133866`.) + + +glob +---- + +* Removed the undocumented :func:`!glob.glob0` and :func:`!glob.glob1` + functions, which have been deprecated since Python 3.13. Use + :func:`glob.glob` and pass a directory to its *root_dir* argument instead. + (Contributed by Barney Gale in :gh:`137466`.) + + +http.server +----------- + +* Removed the :class:`!CGIHTTPRequestHandler` class + and the ``--cgi`` flag from the :program:`python -m http.server` + command-line interface. They were deprecated in Python 3.13. + (Contributed by Bénédikt Tran in :gh:`133810`.) + + +importlib.resources +------------------- + +* Removed deprecated ``package`` parameter + from :func:`importlib.resources.files` function. + (Contributed by Semyon Moroz in :gh:`138044`) + + +pathlib +------- + +* Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. + Use :func:`os.path.isreserved` to detect reserved paths on Windows. + (Contributed by Nikita Sobolev in :gh:`133875`.) + + +platform +-------- + +* Removed the :func:`!platform.java_ver` function, + which was deprecated since Python 3.13. + (Contributed by Alexey Makridenko in :gh:`133604`.) + + +sre_* +----- + +* Removed :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + (Contributed by Stan Ulbrych in :gh:`135994`.) + + +sysconfig +--------- + +* Removed the *check_home* parameter of :func:`sysconfig.is_python_build`. + (Contributed by Filipe Laíns in :gh:`92897`.) + + +threading +--------- + +* Remove support for arbitrary positional or keyword arguments in the C + implementation of :class:`~threading.RLock` objects. This was deprecated + in Python 3.14. + (Contributed by Bénédikt Tran in :gh:`134087`.) + + +typing +------ + +* The undocumented keyword argument syntax for creating + :class:`~typing.NamedTuple` classes (for example, + ``Point = NamedTuple("Point", x=int, y=int)``) is no longer supported. + Use the class-based syntax or the functional syntax instead. + (Contributed by Bénédikt Tran in :gh:`133817`.) + +* Using ``TD = TypedDict("TD")`` or ``TD = TypedDict("TD", None)`` to + construct a :class:`~typing.TypedDict` type with zero fields is no + longer supported. Use ``class TD(TypedDict): pass`` + or ``TD = TypedDict("TD", {})`` instead. + (Contributed by Bénédikt Tran in :gh:`133823`.) + +* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises + a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles + type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` + as it was incorrectly inferred in runtime before. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* :class:`typing.ByteString` has been removed from ``typing.__all__``. + :class:`!typing.ByteString` has been deprecated since Python 3.9, and is + scheduled for removal in Python 3.17. + +* The following statements now cause ``DeprecationWarning``\ s to be emitted at + runtime: + + * ``from typing import ByteString`` + * ``import typing; typing.ByteString``. + + ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` + was subclassed or used as the second argument to :func:`isinstance` or + :func:`issubclass`, but warnings were not previously emitted if it was merely + imported or accessed from the :mod:`!typing` module. + +* Deprecated :func:`!typing.no_type_check_decorator` has been removed. + (Contributed by Nikita Sobolev in :gh:`133601`.) + + +wave +---- + +* Removed the ``getmark()``, ``setmark()`` and ``getmarkers()`` methods + of the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes, + which were deprecated since Python 3.13. + (Contributed by Bénédikt Tran in :gh:`133873`.) + + +zipimport +--------- + +* Remove deprecated :meth:`!zipimport.zipimporter.load_module`. + Use :meth:`zipimport.zipimporter.exec_module` instead. + (Contributed by Jiahao Li in :gh:`133656`.) + + +Deprecated +========== + +New deprecations +---------------- + +* CLI: + + * Deprecate :option:`-b` and :option:`!-bb` command-line options + and schedule them to become no-ops in Python 3.17. + These were primarily helpers for the Python 2 -> 3 transition. + Starting with Python 3.17, no :exc:`BytesWarning` will be raised + for these cases; use a type checker instead. + + (Contributed by Nikita Sobolev in :gh:`136355`.) + +* :mod:`hashlib`: + + * In hash function constructors such as :func:`~hashlib.new` or the + direct hash-named constructors such as :func:`~hashlib.md5` and + :func:`~hashlib.sha256`, the optional initial data parameter could + also be passed as a keyword argument named ``data=`` or ``string=`` in + various :mod:`hashlib` implementations. + + Support for the ``string`` keyword argument name is now deprecated and + is slated for removal in Python 3.19. Prefer passing the initial data as + a positional argument for maximum backwards compatibility. + + (Contributed by Bénédikt Tran in :gh:`134978`.) + +* ``__version__`` + + * The ``__version__`` attribute has been deprecated in these standard library + modules and will be removed in Python 3.20. + Use :py:data:`sys.version_info` instead. + + - :mod:`argparse` + - :mod:`csv` + - :mod:`!ctypes.macholib` + - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`imaplib` + - :mod:`ipaddress` + - :mod:`json` + - :mod:`logging` (``__date__`` also deprecated) + - :mod:`optparse` + - :mod:`pickle` + - :mod:`platform` + - :mod:`re` + - :mod:`socketserver` + - :mod:`tabnanny` + - :mod:`tkinter.font` + - :mod:`tkinter.ttk` + - :mod:`zlib` + + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.16.rst + +.. include:: ../deprecations/pending-removal-in-3.17.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + + +C API changes +============= + +New features +------------ + +* Add :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, + :c:func:`PySys_GetOptionalAttr`, and :c:func:`PySys_GetOptionalAttrString` + functions as replacements for :c:func:`PySys_GetObject`. + (Contributed by Serhiy Storchaka in :gh:`108512`.) + +* Add :c:type:`PyUnstable_Unicode_GET_CACHED_HASH` to get the cached hash of + a string. See the documentation for caveats. + (Contributed by Petr Viktorin in :gh:`131510`.) + +* Add API for checking an extension module's ABI compatibility: + :c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, :c:macro:`PyABIInfo_VAR` + and :c:data:`Py_mod_abi`. + (Contributed by Petr Viktorin in :gh:`137210`.) + +.. _whatsnew315-pep782: + +* Implement :pep:`782`, the :ref:`PyBytesWriter API `. + Add functions: + + * :c:func:`PyBytesWriter_Create` + * :c:func:`PyBytesWriter_Discard` + * :c:func:`PyBytesWriter_FinishWithPointer` + * :c:func:`PyBytesWriter_FinishWithSize` + * :c:func:`PyBytesWriter_Finish` + * :c:func:`PyBytesWriter_Format` + * :c:func:`PyBytesWriter_GetData` + * :c:func:`PyBytesWriter_GetSize` + * :c:func:`PyBytesWriter_GrowAndUpdatePointer` + * :c:func:`PyBytesWriter_Grow` + * :c:func:`PyBytesWriter_Resize` + * :c:func:`PyBytesWriter_WriteBytes` + + (Contributed by Victor Stinner in :gh:`129813`.) + +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + +* Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. + (Contributed by Victor Stinner in :gh:`111489`.) + +* Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. + It should only be used for debugging. + (Contributed by Victor Stinner in :gh:`141070`.) + +* Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and + :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set + the stack protection base address and stack protection size of a Python + thread state. + (Contributed by Victor Stinner in :gh:`139653`.) + + +Changed C APIs +-------------- + +* If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` or :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` + flag is set then :c:macro:`Py_TPFLAGS_HAVE_GC` must be set too. + (Contributed by Sergey Miryanov in :gh:`134786`) + + +Porting to Python 3.15 +---------------------- + +* Private functions promoted to public C APIs: + + The |pythoncapi_compat_project| can be used to get most of these new + functions on Python 3.14 and older. + + +Removed C APIs +-------------- + +* Remove deprecated ``PyUnicode`` functions: + + * :c:func:`!PyUnicode_AsDecodedObject`: + Use :c:func:`PyCodec_Decode` instead. + * :c:func:`!PyUnicode_AsDecodedUnicode`: + Use :c:func:`PyCodec_Decode` instead; Note that some codecs (for example, "base64") + may return a type other than :class:`str`, such as :class:`bytes`. + * :c:func:`!PyUnicode_AsEncodedObject`: + Use :c:func:`PyCodec_Encode` instead. + * :c:func:`!PyUnicode_AsEncodedUnicode`: + Use :c:func:`PyCodec_Encode` instead; Note that some codecs (for example, "base64") + may return a type other than :class:`bytes`, such as :class:`str`. + + (Contributed by Stan Ulbrych in :gh:`133612`.) + +* :c:func:`!PyImport_ImportModuleNoBlock`: deprecated alias + of :c:func:`PyImport_ImportModule`. + (Contributed by Bénédikt Tran in :gh:`133644`.) + +* :c:func:`!PyWeakref_GetObject` and :c:macro:`!PyWeakref_GET_OBJECT`: + use :c:func:`PyWeakref_GetRef` instead. The |pythoncapi_compat_project| + can be used to get :c:func:`!PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Bénédikt Tran in :gh:`133644`.) + +* Remove deprecated :c:func:`!PySys_ResetWarnOptions`. + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + + (Contributed by Nikita Sobolev in :gh:`138886`.) + +The following functions are removed in favor of :c:func:`PyConfig_Get`. +The |pythoncapi_compat_project| can be used to get :c:func:`!PyConfig_Get` +on Python 3.13 and older. + +* Python initialization functions: + + * :c:func:`!Py_GetExecPrefix`: + use :c:func:`PyConfig_Get("base_exec_prefix") ` + (:data:`sys.base_exec_prefix`) instead. + Use :c:func:`PyConfig_Get("exec_prefix") ` + (:data:`sys.exec_prefix`) if :ref:`virtual environments ` + need to be handled. + * :c:func:`!Py_GetPath`: + use :c:func:`PyConfig_Get("module_search_paths") ` + (:data:`sys.path`) instead. + * :c:func:`!Py_GetPrefix`: + use :c:func:`PyConfig_Get("base_prefix") ` + (:data:`sys.base_prefix`) instead. + Use :c:func:`PyConfig_Get("prefix") ` + (:data:`sys.prefix`) if :ref:`virtual environments ` + need to be handled. + * :c:func:`!Py_GetProgramFullPath`: + use :c:func:`PyConfig_Get("executable") ` + (:data:`sys.executable`) instead. + * :c:func:`!Py_GetProgramName`: + use :c:func:`PyConfig_Get("executable") ` + (:data:`sys.executable`) instead. + * :c:func:`!Py_GetPythonHome`: + use :c:func:`PyConfig_Get("home") ` or the + :envvar:`PYTHONHOME` environment variable instead. + + (Contributed by Bénédikt Tran in :gh:`133644`.) + +.. |pythoncapi_compat_project| replace:: |pythoncapi_compat_project_link|_ +.. |pythoncapi_compat_project_link| replace:: pythoncapi-compat project +.. _pythoncapi_compat_project_link: https://github.com/python/pythoncapi-compat + + +Deprecated C APIs +----------------- + +* For unsigned integer formats in :c:func:`PyArg_ParseTuple`, + accepting Python integers with value that is larger than the maximal value + for the C type or less than the minimal value for the corresponding + signed integer type of the same size is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`132629`.) + +* :c:func:`PyBytes_FromStringAndSize(NULL, len) ` + and :c:func:`_PyBytes_Resize` are :term:`soft deprecated`, + use the :c:type:`PyBytesWriter` API instead. + (Contributed by Victor Stinner in :gh:`129813`.) + +* Deprecate :c:member:`~PyComplexObject.cval` field of the + :c:type:`PyComplexObject` type. + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + +* Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, + :c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and + :c:func:`_Py_c_abs` are :term:`soft deprecated`. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + +* :c:member:`~PyConfig.bytes_warning` is deprecated + since 3.15 and will be removed in 3.17. + (Contributed by Nikita Sobolev in :gh:`136355`.) + +* :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`, + use the C11 standard ```` :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + +* :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated + since 3.15 and will be removed in 3.20. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + + +.. Add C API deprecations above alphabetically, not here at the end. + + +Build changes +============= + +* Removed implicit fallback to the bundled copy of the ``libmpdec`` library. + Now this should be explicitly enabled with :option:`--with-system-libmpdec` + set to ``no`` or with :option:`!--without-system-libmpdec`. + (Contributed by Sergey B Kirpichev in :gh:`115119`.) + +* The new configure option :option:`--with-missing-stdlib-config=FILE` allows + distributors to pass a `JSON `_ + configuration file containing custom error messages for :term:`standard library` + modules that are missing or packaged separately. + (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.) + + +Porting to Python 3.15 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +* :class:`sqlite3.Connection` APIs have been cleaned up. + + * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. + * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` + and :meth:`~sqlite3.Connection.create_aggregate` are now positional-only. + * The first parameter of methods :meth:`~sqlite3.Connection.set_authorizer`, + :meth:`~sqlite3.Connection.set_progress_handler` and + :meth:`~sqlite3.Connection.set_trace_callback` is now positional-only. + + (Contributed by Serhiy Storchaka in :gh:`133595`.) + +* :data:`resource.RLIM_INFINITY` is now always positive. + Passing a negative integer value that corresponded to its old value + (such as ``-1`` or ``-3``, depending on platform) to + :func:`resource.setrlimit` and :func:`resource.prlimit` is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`137044`.) + +* :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the + underlying syscall, instead of raising a :exc:`SystemError`. + +* A resource warning is now emitted for an unclosed + :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. + Use its :meth:`!close` method or the :func:`contextlib.closing` context + manager to close it. + (Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.) + +* If a short option and a single-dash long option are passed to + :meth:`argparse.ArgumentParser.add_argument`, *dest* is now inferred from + the single-dash long option. For example, in ``add_argument('-f', '-foo')``, + *dest* is now ``'foo'`` instead of ``'f'``. + Pass an explicit *dest* argument to preserve the old behavior. + (Contributed by Serhiy Storchaka in :gh:`138697`.) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 7104904c956..47c4d9acbc8 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -458,7 +458,7 @@ Some smaller changes made to the core Python language are: :class:`~collections.defaultdict`, :class:`~shelve.Shelf`, :class:`~configparser.ConfigParser`, or :mod:`dbm`. It is also useful with custom :class:`dict` subclasses that normalize keys before look-up or that - supply a :meth:`__missing__` method for unknown keys:: + supply a :meth:`~object.__missing__` method for unknown keys:: >>> import shelve >>> d = shelve.open('tmp.shl') diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 7a8eb47cbdb..1bb79bce2c3 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -331,7 +331,7 @@ simplified and finer-grained. You don't have to worry anymore about choosing the appropriate exception type between :exc:`OSError`, :exc:`IOError`, :exc:`EnvironmentError`, -:exc:`WindowsError`, :exc:`mmap.error`, :exc:`socket.error` or +:exc:`WindowsError`, :exc:`!mmap.error`, :exc:`socket.error` or :exc:`select.error`. All these exception types are now only one: :exc:`OSError`. The other names are kept as aliases for compatibility reasons. @@ -805,7 +805,7 @@ Some smaller changes made to the core Python language are: * New methods have been added to :class:`list` and :class:`bytearray`: ``copy()`` and ``clear()`` (:issue:`10516`). Consequently, :class:`~collections.abc.MutableSequence` now also defines a - :meth:`~collections.abc.MutableSequence.clear` method (:issue:`11388`). + :meth:`!clear` method (:issue:`11388`). * Raw bytes literals can now be written ``rb"..."`` as well as ``br"..."``. @@ -829,7 +829,7 @@ Previous versions of CPython have always relied on a global import lock. This led to unexpected annoyances, such as deadlocks when importing a module would trigger code execution in a different thread as a side-effect. Clumsy workarounds were sometimes employed, such as the -:c:func:`PyImport_ImportModuleNoBlock` C API function. +:c:func:`!PyImport_ImportModuleNoBlock` C API function. In Python 3.3, importing a module takes a per-module lock. This correctly serializes importation of a given module from multiple threads (preventing @@ -869,10 +869,10 @@ faulthandler This new debug module :mod:`faulthandler` contains functions to dump Python tracebacks explicitly, on a fault (a crash like a segmentation fault), after a timeout, or on a user signal. Call :func:`faulthandler.enable` to install fault handlers for the -:const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS`, and -:const:`SIGILL` signals. You can also enable them at startup by setting the -:envvar:`PYTHONFAULTHANDLER` environment variable or by using :option:`-X` -``faulthandler`` command line option. +:const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, +:const:`~signal.SIGBUS`, and :const:`~signal.SIGILL` signals. +You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` +environment variable or by using :option:`-X` ``faulthandler`` command line option. Example of a segmentation fault on Linux: @@ -916,7 +916,7 @@ abc Improved support for abstract base classes containing descriptors composed with abstract methods. The recommended approach to declaring abstract descriptors is -now to provide :attr:`__isabstractmethod__` as a dynamically updated +now to provide :attr:`!__isabstractmethod__` as a dynamically updated property. The built-in descriptors have been updated accordingly. * :class:`abc.abstractproperty` has been deprecated, use :class:`property` @@ -979,7 +979,7 @@ new features have been added: (Contributed by Nir Aides in :issue:`1625`.) * :class:`bz2.BZ2File` now implements all of the :class:`io.BufferedIOBase` API, - except for the :meth:`detach` and :meth:`truncate` methods. + except for the :meth:`!detach` and :meth:`!truncate` methods. codecs @@ -1064,7 +1064,7 @@ curses * If the :mod:`curses` module is linked to the ncursesw library, use Unicode functions when Unicode strings or characters are passed (e.g. - :c:func:`waddwstr`), and bytes functions otherwise (e.g. :c:func:`waddstr`). + :c:func:`!waddwstr`), and bytes functions otherwise (e.g. :c:func:`!waddstr`). * Use the locale encoding instead of ``utf-8`` to encode Unicode strings. * :class:`curses.window` has a new :attr:`curses.window.encoding` attribute. * The :class:`curses.window` class has a new :meth:`~curses.window.get_wch` @@ -1137,15 +1137,15 @@ API changes * The C module has the following context limits, depending on the machine architecture: - +-------------------+----------------+-------------------------+ - | | 32-bit | 64-bit | - +===================+================+=========================+ - | :const:`MAX_PREC` | ``425000000`` | ``999999999999999999`` | - +-------------------+----------------+-------------------------+ - | :const:`MAX_EMAX` | ``425000000`` | ``999999999999999999`` | - +-------------------+----------------+-------------------------+ - | :const:`MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | - +-------------------+----------------+-------------------------+ + +----------------------------+----------------+-------------------------+ + | | 32-bit | 64-bit | + +============================+================+=========================+ + | :const:`~decimal.MAX_PREC` | ``425000000`` | ``999999999999999999`` | + +----------------------------+----------------+-------------------------+ + | :const:`~decimal.MAX_EMAX` | ``425000000`` | ``999999999999999999`` | + +----------------------------+----------------+-------------------------+ + | :const:`~decimal.MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | + +----------------------------+----------------+-------------------------+ * In the context templates (:const:`~decimal.DefaultContext`, :const:`~decimal.BasicContext` and :const:`~decimal.ExtendedContext`) @@ -1434,7 +1434,7 @@ html :class:`html.parser.HTMLParser` is now able to parse broken markup without raising errors, therefore the *strict* argument of the constructor and the -:exc:`~html.parser.HTMLParseError` exception are now deprecated. +:exc:`!HTMLParseError` exception are now deprecated. The ability to parse broken markup is the result of a number of bug fixes that are also available on the latest bug fix releases of Python 2.7/3.2. (Contributed by Ezio Melotti in :issue:`15114`, and :issue:`14538`, @@ -1486,7 +1486,7 @@ already exists. It is based on the C11 'x' mode to fopen(). The constructor of the :class:`~io.TextIOWrapper` class has a new *write_through* optional argument. If *write_through* is ``True``, calls to -:meth:`~io.TextIOWrapper.write` are guaranteed not to be buffered: any data +:meth:`!write` are guaranteed not to be buffered: any data written on the :class:`~io.TextIOWrapper` object is immediately handled to its underlying binary buffer. @@ -1504,7 +1504,7 @@ logging The :func:`~logging.basicConfig` function now supports an optional ``handlers`` argument taking an iterable of handlers to be added to the root logger. -A class level attribute :attr:`~logging.handlers.SysLogHandler.append_nul` has +A class level attribute :attr:`!append_nul` has been added to :class:`~logging.handlers.SysLogHandler` to allow control of the appending of the ``NUL`` (``\000``) byte to syslog records, since for some daemons it is required while for others it is passed through to the log. @@ -1536,8 +1536,8 @@ The new :func:`multiprocessing.connection.wait` function allows polling multiple objects (such as connections, sockets and pipes) with a timeout. (Contributed by Richard Oudkerk in :issue:`12328`.) -:class:`multiprocessing.Connection` objects can now be transferred over -multiprocessing connections. +:class:`multiprocessing.connection.Connection` objects can now be transferred +over multiprocessing connections. (Contributed by Richard Oudkerk in :issue:`4892`.) :class:`multiprocessing.Process` now accepts a ``daemon`` keyword argument @@ -1611,7 +1611,7 @@ os :func:`~os.rename`, :func:`~os.replace`, :func:`~os.rmdir`, :func:`~os.stat`, :func:`~os.symlink`, :func:`~os.unlink`, :func:`~os.utime`. Platform support for using these parameters can be checked via the sets - :data:`os.supports_dir_fd` and :data:`os.supports_follows_symlinks`. + :data:`os.supports_dir_fd` and :data:`os.supports_follow_symlinks`. - The following functions now support a file descriptor for their path argument: :func:`~os.chdir`, :func:`~os.chmod`, :func:`~os.chown`, @@ -1698,7 +1698,7 @@ os :const:`~os.RTLD_NOLOAD`, and :const:`~os.RTLD_DEEPBIND` are available on platforms that support them. These are for use with the :func:`sys.setdlopenflags` function, and supersede the similar constants - defined in :mod:`ctypes` and :mod:`DLFCN`. (Contributed by Victor Stinner + defined in :mod:`ctypes` and :mod:`!DLFCN`. (Contributed by Victor Stinner in :issue:`13226`.) * :func:`os.symlink` now accepts (and ignores) the ``target_is_directory`` @@ -1728,8 +1728,8 @@ reduction functions to be set. pydoc ----- -The Tk GUI and the :func:`~pydoc.serve` function have been removed from the -:mod:`pydoc` module: ``pydoc -g`` and :func:`~pydoc.serve` have been deprecated +The Tk GUI and the :func:`!serve` function have been removed from the +:mod:`pydoc` module: ``pydoc -g`` and :func:`!serve` have been deprecated in Python 3.2. @@ -1931,7 +1931,7 @@ ssl * :func:`~ssl.RAND_bytes`: generate cryptographically strong pseudo-random bytes. - * :func:`~ssl.RAND_pseudo_bytes`: generate pseudo-random bytes. + * :func:`!RAND_pseudo_bytes`: generate pseudo-random bytes. (Contributed by Victor Stinner in :issue:`12049`.) @@ -2020,8 +2020,7 @@ tarfile tempfile -------- -:class:`tempfile.SpooledTemporaryFile`\'s -:meth:`~tempfile.SpooledTemporaryFile.truncate` method now accepts +:class:`tempfile.SpooledTemporaryFile`\'s :meth:`!truncate` method now accepts a ``size`` parameter. (Contributed by Ryan Kelly in :issue:`9957`.) @@ -2129,7 +2128,7 @@ xml.etree.ElementTree The :mod:`xml.etree.ElementTree` module now imports its C accelerator by default; there is no longer a need to explicitly import -:mod:`xml.etree.cElementTree` (this module stays for backwards compatibility, +:mod:`!xml.etree.cElementTree` (this module stays for backwards compatibility, but is now deprecated). In addition, the ``iter`` family of methods of :class:`~xml.etree.ElementTree.Element` has been optimized (rewritten in C). The module's documentation has also been greatly improved with added examples @@ -2197,7 +2196,7 @@ Changes to Python's build process and to the C API include: * :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsUCS4Copy` * :c:macro:`PyUnicode_DATA`, :c:macro:`PyUnicode_1BYTE_DATA`, :c:macro:`PyUnicode_2BYTE_DATA`, :c:macro:`PyUnicode_4BYTE_DATA` - * :c:macro:`PyUnicode_KIND` with :c:enum:`PyUnicode_Kind` enum: + * :c:macro:`PyUnicode_KIND` with :c:enum:`!PyUnicode_Kind` enum: :c:data:`!PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, :c:data:`PyUnicode_2BYTE_KIND`, :c:data:`PyUnicode_4BYTE_KIND` * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` @@ -2232,17 +2231,17 @@ Deprecated Python modules, functions and methods (``utf-32-le`` or ``utf-32-be``) * :meth:`ftplib.FTP.nlst` and :meth:`ftplib.FTP.dir`: use :meth:`ftplib.FTP.mlsd` -* :func:`platform.popen`: use the :mod:`subprocess` module. Check especially +* :func:`!platform.popen`: use the :mod:`subprocess` module. Check especially the :ref:`subprocess-replacements` section (:issue:`11377`). * :issue:`13374`: The Windows bytes API has been deprecated in the :mod:`os` module. Use Unicode filenames, instead of bytes filenames, to not depend on the ANSI code page anymore and to support any filename. -* :issue:`13988`: The :mod:`xml.etree.cElementTree` module is deprecated. The +* :issue:`13988`: The :mod:`!xml.etree.cElementTree` module is deprecated. The accelerator is used automatically whenever available. -* The behaviour of :func:`time.clock` depends on the platform: use the new +* The behaviour of :func:`!time.clock` depends on the platform: use the new :func:`time.perf_counter` or :func:`time.process_time` function instead, depending on your requirements, to have a well defined behaviour. -* The :func:`os.stat_float_times` function is deprecated. +* The :func:`!os.stat_float_times` function is deprecated. * :mod:`abc` module: * :class:`abc.abstractproperty` has been deprecated, use :class:`property` @@ -2466,7 +2465,7 @@ Porting C code * In the course of changes to the buffer API the undocumented :c:member:`!smalltable` member of the :c:type:`Py_buffer` structure has been removed and the - layout of the :c:type:`PyMemoryViewObject` has changed. + layout of the :c:type:`!PyMemoryViewObject` has changed. All extensions relying on the relevant parts in ``memoryobject.h`` or ``object.h`` must be rebuilt. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index e4f602a17ee..9f4068116e3 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2,7 +2,7 @@ What's New In Python 3.4 **************************** -:Author: R. David Murray (Editor) +:Author: \R. David Murray (Editor) .. Rules for maintenance: @@ -122,7 +122,7 @@ Significantly improved library modules: on Unix ` (:issue:`8713`). * :mod:`email` has a new submodule, :mod:`~email.contentmanager`, and a new :mod:`~email.message.Message` subclass - (:class:`~email.contentmanager.EmailMessage`) that :ref:`simplify MIME + (:class:`~email.message.EmailMessage`) that :ref:`simplify MIME handling ` (:issue:`18891`). * The :mod:`inspect` and :mod:`pydoc` modules are now capable of correct introspection of a much wider variety of callable objects, @@ -154,7 +154,7 @@ Security improvements: `. * All modules in the standard library that support SSL now support server certificate verification, including hostname matching - (:func:`ssl.match_hostname`) and CRLs (Certificate Revocation lists, see + (:func:`!ssl.match_hostname`) and CRLs (Certificate Revocation lists, see :func:`ssl.SSLContext.load_verify_locations`). CPython implementation improvements: @@ -739,7 +739,7 @@ these new components. In addition, a new application-friendly class :class:`~dis.Bytecode` provides an object-oriented API for inspecting bytecode in both in human-readable form and for iterating over instructions. The :class:`~dis.Bytecode` constructor -takes the same arguments that :func:`~dis.get_instruction` does (plus an +takes the same arguments that :func:`~dis.get_instructions` does (plus an optional *current_offset*), and the resulting object can be iterated to produce :class:`~dis.Instruction` objects. But it also has a :mod:`~dis.Bytecode.dis` method, equivalent to calling :mod:`~dis.dis` on the constructor argument, but @@ -958,8 +958,9 @@ http optional additional *explain* parameter which can be used to provide an extended error description, overriding the hardcoded default if there is one. This extended error description will be formatted using the -:attr:`~http.server.HTTP.error_message_format` attribute and sent as the body -of the error response. (Contributed by Karl Cow in :issue:`12921`.) +:attr:`~http.server.BaseHTTPRequestHandler.error_message_format` attribute +and sent as the body of the error response. +(Contributed by Karl Cow in :issue:`12921`.) The :mod:`http.server` :ref:`command line interface ` now has a ``-b/--bind`` option that causes the server to listen on a specific address. @@ -1038,7 +1039,7 @@ As part of the implementation of the new :mod:`enum` module, the metaclasses. (Contributed by Ethan Furman in :issue:`18929` and :issue:`19030`.) -:func:`~inspect.getfullargspec` and :func:`~inspect.getargspec` +:func:`~inspect.getfullargspec` and :func:`!getargspec` now use the :func:`~inspect.signature` API. This allows them to support a much broader range of callables, including those with ``__signature__`` attributes, those with metadata provided by argument @@ -1221,7 +1222,7 @@ pickle :mod:`pickle` now supports (but does not use by default) a new pickle protocol, protocol 4. This new protocol addresses a number of issues that were present in previous protocols, such as the serialization of nested classes, very large -strings and containers, and classes whose :meth:`__new__` method takes +strings and containers, and classes whose :meth:`~object.__new__` method takes keyword-only arguments. It also provides some efficiency improvements. .. seealso:: @@ -1299,7 +1300,7 @@ affect the behaviour of :func:`help`. re -- -New :func:`~re.fullmatch` function and :meth:`.regex.fullmatch` method anchor +New :func:`~re.fullmatch` function and :meth:`.Pattern.fullmatch` method anchor the pattern at both ends of the string to match. This provides a way to be explicit about the goal of the match, which avoids a class of subtle bugs where ``$`` characters get lost during code changes or the addition of alternatives @@ -1519,7 +1520,7 @@ subprocess be used to provide the contents of ``stdin`` for the command that is run. (Contributed by Zack Weinberg in :issue:`16624`.) -:func:`~subprocess.getstatus` and :func:`~subprocess.getstatusoutput` now +:func:`~subprocess.getoutput` and :func:`~subprocess.getstatusoutput` now work on Windows. This change was actually inadvertently made in 3.3.4. (Contributed by Tim Golden in :issue:`10197`.) @@ -1535,7 +1536,7 @@ plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) called automatically at the end of the block. (Contributed by Serhiy Storchaka in :issue:`18878`.) -:meth:`.AU_write.setsampwidth` now supports 24 bit samples, thus adding +:meth:`!AU_write.setsampwidth` now supports 24 bit samples, thus adding support for writing 24 sample using the module. (Contributed by Serhiy Storchaka in :issue:`19261`.) @@ -1615,7 +1616,7 @@ A new :func:`~types.DynamicClassAttribute` descriptor provides a way to define an attribute that acts normally when looked up through an instance object, but which is routed to the *class* ``__getattr__`` when looked up through the class. This allows one to have properties active on a class, and have virtual -attributes on the class with the same name (see :mod:`Enum` for an example). +attributes on the class with the same name (see :mod:`enum` for an example). (Contributed by Ethan Furman in :issue:`19030`.) @@ -1709,7 +1710,7 @@ matching calls, which means an argument can now be matched by either position or name, instead of only by position. (Contributed by Antoine Pitrou in :issue:`17015`.) -:func:`~mock.mock_open` objects now have ``readline`` and ``readlines`` +:func:`~unittest.mock.mock_open` objects now have ``readline`` and ``readlines`` methods. (Contributed by Toshio Kuratomi in :issue:`17467`.) @@ -1729,8 +1730,8 @@ installed in the virtual environment. (Contributed by Nick Coghlan in wave ---- -The :meth:`~wave.getparams` method now returns a namedtuple rather than a -plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) +The :meth:`~wave.Wave_read.getparams` method now returns a namedtuple rather +than a plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) :meth:`wave.open` now supports the context management protocol. (Contributed by Claudiu Popa in :issue:`17616`.) @@ -1788,7 +1789,7 @@ example, this could be used to exclude test files from the archive. (Contributed by Christian Tismer in :issue:`19274`.) The *allowZip64* parameter to :class:`~zipfile.ZipFile` and -:class:`~zipfile.PyZipfile` is now ``True`` by default. (Contributed by +:class:`~zipfile.PyZipFile` is now ``True`` by default. (Contributed by William Mallard in :issue:`17201`.) @@ -1817,7 +1818,7 @@ PEP 442: Safe Object Finalization --------------------------------- :pep:`442` removes the current limitations and quirks of object finalization -in CPython. With it, objects with :meth:`__del__` methods, as well as +in CPython. With it, objects with :meth:`~object.__del__` methods, as well as generators with :keyword:`finally` clauses, can be finalized when they are part of a reference cycle. @@ -2043,7 +2044,7 @@ Significant Optimizations strings is now significantly faster. (Contributed by Victor Stinner and Antoine Pitrou in :issue:`15596`.) -* A performance issue in :meth:`io.FileIO.readall` has been solved. This +* A performance issue in :meth:`!io.FileIO.readall` has been solved. This particularly affects Windows, and significantly speeds up the case of piping significant amounts of data through :mod:`subprocess`. (Contributed by Richard Oudkerk in :issue:`15758`.) @@ -2103,7 +2104,7 @@ Deprecations in the Python API * The :mod:`!imp` module is pending deprecation. To keep compatibility with Python 2/3 code bases, the module's removal is currently not scheduled. -* The :mod:`formatter` module is pending deprecation and is slated for removal +* The :mod:`!formatter` module is pending deprecation and is slated for removal in Python 3.6. * ``MD5`` as the default *digestmod* for the :func:`hmac.new` function is @@ -2120,11 +2121,11 @@ Deprecations in the Python API * The *strict* argument of :class:`~html.parser.HTMLParser` is deprecated. -* The :mod:`plistlib` :func:`~plistlib.readPlist`, - :func:`~plistlib.writePlist`, :func:`~plistlib.readPlistFromBytes`, and - :func:`~plistlib.writePlistToBytes` functions are deprecated in favor of the +* The :mod:`plistlib` :func:`!readPlist`, + :func:`!writePlist`, :func:`!readPlistFromBytes`, and + :func:`!writePlistToBytes` functions are deprecated in favor of the corresponding new functions :func:`~plistlib.load`, :func:`~plistlib.dump`, - :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`~plistlib.Data` + :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`!Data` is deprecated in favor of just using the :class:`bytes` constructor. * The :mod:`sysconfig` key ``SO`` is deprecated, it has been replaced by @@ -2212,8 +2213,8 @@ removed: that do not have a __format__ method that handles it. See :issue:`7994` for background. -* :meth:`difflib.SequenceMatcher.isbjunk` and - :meth:`difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have +* :meth:`!difflib.SequenceMatcher.isbjunk` and + :meth:`!difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have now been removed: use ``x in sm.bjunk`` and ``x in sm.bpopular``, where *sm* is a :class:`~difflib.SequenceMatcher` object (:issue:`13248`). @@ -2280,7 +2281,7 @@ Changes in the Python API * :meth:`!importlib.util.module_for_loader` now sets ``__loader__`` and ``__package__`` unconditionally to properly support reloading. If this is not desired then you will need to set these attributes manually. You can use - :func:`importlib.util.module_to_load` for module management. + :func:`!importlib.util.module_to_load` for module management. * Import now resets relevant attributes (e.g. ``__name__``, ``__loader__``, ``__package__``, ``__file__``, ``__cached__``) unconditionally when reloading. @@ -2428,7 +2429,7 @@ Changes in the Python API disallowed command forms didn't make any sense and are unlikely to be in use. * The :func:`re.split`, :func:`re.findall`, and :func:`re.sub` functions, and - the :meth:`~re.match.group` and :meth:`~re.match.groups` methods of + the :meth:`~re.Match.group` and :meth:`~re.Match.groups` methods of ``match`` objects now always return a *bytes* object when the string to be matched is a :term:`bytes-like object`. Previously the return type matched the input type, so if your code was depending on the return value diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index db3f1db3bd7..6009dd8a71e 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -181,7 +181,7 @@ Coroutine functions are declared using the new :keyword:`async def` syntax:: Inside a coroutine function, the new :keyword:`await` expression can be used to suspend coroutine execution until the result is available. Any object can be *awaited*, as long as it implements the :term:`awaitable` protocol by -defining the :meth:`__await__` method. +defining the :meth:`~object.__await__` method. PEP 492 also adds :keyword:`async for` statement for convenient iteration over asynchronous iterables. @@ -273,9 +273,10 @@ PEP 465 - A dedicated infix operator for matrix multiplication :pep:`465` adds the ``@`` infix operator for matrix multiplication. Currently, no builtin Python types implement the new operator, however, it -can be implemented by defining :meth:`__matmul__`, :meth:`__rmatmul__`, -and :meth:`__imatmul__` for regular, reflected, and in-place matrix -multiplication. The semantics of these methods is similar to that of +can be implemented by defining :meth:`~object.__matmul__`, +:meth:`~object.__rmatmul__`, and :meth:`~object.__imatmul__` for regular, +reflected, and in-place matrix multiplication. +The semantics of these methods is similar to that of methods defining other infix arithmetic operators. Matrix multiplication is a notably common operation in many fields of @@ -800,7 +801,7 @@ Notable changes in the :mod:`asyncio` module since Python 3.4.0: control. (Contributed by Victor Stinner.) -* The :func:`~asyncio.async` function is deprecated in favor of +* The :func:`!async` function is deprecated in favor of :func:`~asyncio.ensure_future`. (Contributed by Yury Selivanov.) @@ -905,10 +906,8 @@ collections The :class:`~collections.OrderedDict` class is now implemented in C, which makes it 4 to 100 times faster. (Contributed by Eric Snow in :issue:`16991`.) -:meth:`OrderedDict.items() `, -:meth:`OrderedDict.keys() `, -:meth:`OrderedDict.values() ` views now support -:func:`reversed` iteration. +:meth:`!OrderedDict.items`, :meth:`!OrderedDict.keys`, +and :meth:`!OrderedDict.values` views now support :func:`reversed` iteration. (Contributed by Serhiy Storchaka in :issue:`19505`.) The :class:`~collections.deque` class now defines @@ -928,7 +927,7 @@ Docstrings produced by :func:`~collections.namedtuple` can now be updated:: (Contributed by Berker Peksag in :issue:`24064`.) The :class:`~collections.UserString` class now implements the -:meth:`__getnewargs__`, :meth:`__rmod__`, :meth:`~str.casefold`, +:meth:`~object.__getnewargs__`, :meth:`~object.__rmod__`, :meth:`~str.casefold`, :meth:`~str.format_map`, :meth:`~str.isprintable`, and :meth:`~str.maketrans` methods to match the corresponding methods of :class:`str`. (Contributed by Joe Jevnik in :issue:`22189`.) @@ -937,7 +936,7 @@ methods to match the corresponding methods of :class:`str`. collections.abc --------------- -The :meth:`Sequence.index() ` method now +The :meth:`!Sequence.index` method now accepts *start* and *stop* arguments to match the corresponding methods of :class:`tuple`, :class:`list`, etc. (Contributed by Devin Jeanpierre in :issue:`23086`.) @@ -1045,8 +1044,8 @@ not just sequences. (Contributed by Serhiy Storchaka in :issue:`23171`.) curses ------ -The new :func:`~curses.update_lines_cols` function updates the :data:`LINES` -and :data:`COLS` module variables. This is useful for detecting +The new :func:`~curses.update_lines_cols` function updates the :data:`~curses.LINES` +and :data:`~curses.COLS` module variables. This is useful for detecting manual screen resizing. (Contributed by Arnon Yaari in :issue:`4254`.) @@ -1347,8 +1346,8 @@ network objects from existing addresses:: (Contributed by Peter Moody and Antoine Pitrou in :issue:`16531`.) -A new :attr:`~ipaddress.IPv4Network.reverse_pointer` attribute for the -:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` classes +A new :attr:`~ipaddress.IPv4Address.reverse_pointer` attribute for the +:class:`~ipaddress.IPv4Address` and :class:`~ipaddress.IPv6Address` classes returns the name of the reverse DNS PTR record:: >>> import ipaddress @@ -1451,7 +1450,7 @@ and :data:`~math.nan`. (Contributed by Mark Dickinson in :issue:`23185`.) A new function :func:`~math.isclose` provides a way to test for approximate equality. (Contributed by Chris Barker and Tal Einat in :issue:`24270`.) -A new :func:`~math.gcd` function has been added. The :func:`fractions.gcd` +A new :func:`~math.gcd` function has been added. The :func:`!fractions.gcd` function is now deprecated. (Contributed by Mark Dickinson and Serhiy Storchaka in :issue:`22486`.) @@ -1602,10 +1601,10 @@ The :func:`~re.sub` and :func:`~re.subn` functions now replace unmatched groups with empty strings instead of raising an exception. (Contributed by Serhiy Storchaka in :issue:`1519638`.) -The :class:`re.error` exceptions have new attributes, -:attr:`~re.error.msg`, :attr:`~re.error.pattern`, -:attr:`~re.error.pos`, :attr:`~re.error.lineno`, -and :attr:`~re.error.colno`, that provide better context +The :class:`re.error ` exceptions have new attributes, +:attr:`~re.PatternError.msg`, :attr:`~re.PatternError.pattern`, +:attr:`~re.PatternError.pos`, :attr:`~re.PatternError.lineno`, +and :attr:`~re.PatternError.colno`, that provide better context information about the error:: >>> re.compile(""" @@ -1794,10 +1793,10 @@ query the actual protocol version in use. (Contributed by Antoine Pitrou in :issue:`20421`.) The :class:`~ssl.SSLSocket` class now implements -a :meth:`SSLSocket.sendfile() ` method. +a :meth:`!SSLSocket.sendfile` method. (Contributed by Giampaolo Rodola' in :issue:`17552`.) -The :meth:`SSLSocket.send() ` method now raises either +The :meth:`!SSLSocket.send` method now raises either the :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` exception on a non-blocking socket if the operation would block. Previously, it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) @@ -1806,20 +1805,20 @@ The :func:`~ssl.cert_time_to_seconds` function now interprets the input time as UTC and not as local time, per :rfc:`5280`. Additionally, the return value is always an :class:`int`. (Contributed by Akira Li in :issue:`19940`.) -New :meth:`SSLObject.shared_ciphers() ` and +New :meth:`!SSLObject.shared_ciphers` and :meth:`SSLSocket.shared_ciphers() ` methods return the list of ciphers sent by the client during the handshake. (Contributed by Benjamin Peterson in :issue:`23186`.) The :meth:`SSLSocket.do_handshake() `, :meth:`SSLSocket.read() `, -:meth:`SSLSocket.shutdown() `, and +:meth:`!SSLSocket.shutdown`, and :meth:`SSLSocket.write() ` methods of the :class:`~ssl.SSLSocket` class no longer reset the socket timeout every time bytes are received or sent. The socket timeout is now the maximum total duration of the method. (Contributed by Victor Stinner in :issue:`23853`.) -The :func:`~ssl.match_hostname` function now supports matching of IP addresses. +The :func:`!match_hostname` function now supports matching of IP addresses. (Contributed by Antoine Pitrou in :issue:`23239`.) @@ -1863,10 +1862,10 @@ Examples:: sys --- -A new :func:`~sys.set_coroutine_wrapper` function allows setting a global +A new :func:`!set_coroutine_wrapper` function allows setting a global hook that will be called whenever a :term:`coroutine object ` is created by an :keyword:`async def` function. A corresponding -:func:`~sys.get_coroutine_wrapper` can be used to obtain a currently set +:func:`!get_coroutine_wrapper` can be used to obtain a currently set wrapper. Both functions are :term:`provisional `, and are intended for debugging purposes only. (Contributed by Yury Selivanov in :issue:`24017`.) @@ -2014,8 +2013,9 @@ The :class:`~unittest.mock.Mock` class has the following improvements: method to check if the mock object was called. (Contributed by Kushal Das in :issue:`21262`.) -The :class:`~unittest.mock.MagicMock` class now supports :meth:`__truediv__`, -:meth:`__divmod__` and :meth:`__matmul__` operators. +The :class:`~unittest.mock.MagicMock` class now supports +:meth:`~object.__truediv__`, :meth:`~object.__divmod__` +and :meth:`~object.__matmul__` operators. (Contributed by Johannes Baiter in :issue:`20968`, and Håkan Lövdahl in :issue:`23581` and :issue:`23568`.) @@ -2290,10 +2290,10 @@ Windows XP is no longer supported by Microsoft, thus, per :PEP:`11`, CPython Deprecated Python modules, functions and methods ------------------------------------------------ -The :mod:`formatter` module has now graduated to full deprecation and is still +The :mod:`!formatter` module has now graduated to full deprecation and is still slated for removal in Python 3.6. -The :func:`asyncio.async` function is deprecated in favor of +The :func:`!asyncio.async` function is deprecated in favor of :func:`~asyncio.ensure_future`. The :mod:`!smtpd` module has in the past always decoded the DATA portion of @@ -2314,7 +2314,7 @@ Passing a format string as keyword argument *format_string* to the class has been deprecated. (Contributed by Serhiy Storchaka in :issue:`23671`.) -The :func:`platform.dist` and :func:`platform.linux_distribution` functions +The :func:`!platform.dist` and :func:`!platform.linux_distribution` functions are now deprecated. Linux distributions use too many different ways of describing themselves, so the functionality is left to a package. (Contributed by Vajrasky Kok and Berker Peksag in :issue:`1322`.) @@ -2324,11 +2324,11 @@ The previously undocumented ``from_function`` and ``from_builtin`` methods of :meth:`Signature.from_callable() ` method instead. (Contributed by Yury Selivanov in :issue:`24248`.) -The :func:`inspect.getargspec` function is deprecated and scheduled to be +The :func:`!inspect.getargspec` function is deprecated and scheduled to be removed in Python 3.6. (See :issue:`20438` for details.) The :mod:`inspect` :func:`~inspect.getfullargspec`, -:func:`~inspect.getcallargs`, and :func:`~inspect.formatargspec` functions are +:func:`~inspect.getcallargs`, and :func:`!formatargspec` functions are deprecated in favor of the :func:`inspect.signature` API. (Contributed by Yury Selivanov in :issue:`20438`.) @@ -2405,7 +2405,7 @@ Changes in the Python API error-prone and has been removed in Python 3.5. See :issue:`13936` for full details. -* The :meth:`ssl.SSLSocket.send` method now raises either +* The :meth:`!ssl.SSLSocket.send` method now raises either :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation would block. Previously, it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) @@ -2440,12 +2440,12 @@ Changes in the Python API :mod:`http.client` and :mod:`http.server` remain available for backwards compatibility. (Contributed by Demian Brecht in :issue:`21793`.) -* When an import loader defines :meth:`importlib.machinery.Loader.exec_module` +* When an import loader defines :meth:`~importlib.abc.Loader.exec_module` it is now expected to also define - :meth:`~importlib.machinery.Loader.create_module` (raises a + :meth:`~importlib.abc.Loader.create_module` (raises a :exc:`DeprecationWarning` now, will be an error in Python 3.6). If the loader inherits from :class:`importlib.abc.Loader` then there is nothing to do, else - simply define :meth:`~importlib.machinery.Loader.create_module` to return + simply define :meth:`~importlib.abc.Loader.create_module` to return ``None``. (Contributed by Brett Cannon in :issue:`23014`.) * The :func:`re.split` function always ignored empty pattern matches, so the @@ -2488,7 +2488,7 @@ Changes in the Python API the POT-Creation-Date header. * The :mod:`smtplib` module now uses :data:`sys.stderr` instead of the previous - module-level :data:`stderr` variable for debug output. If your (test) + module-level :data:`!stderr` variable for debug output. If your (test) program depends on patching the module-level variable to capture the debug output, you will need to update it to capture sys.stderr instead. @@ -2514,11 +2514,11 @@ Changes in the C API -------------------- * The undocumented :c:member:`!format` member of the - (non-public) :c:type:`PyMemoryViewObject` structure has been removed. + (non-public) :c:type:`!PyMemoryViewObject` structure has been removed. All extensions relying on the relevant parts in ``memoryobject.h`` must be rebuilt. -* The :c:type:`PyMemAllocator` structure was renamed to +* The :c:type:`!PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. * Removed non-documented macro :c:macro:`!PyObject_REPR()` which leaked references. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 050c9103b00..b1e3269239d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -745,7 +745,7 @@ Some smaller changes made to the core Python language are: * It is now possible to set a :ref:`special method ` to ``None`` to indicate that the corresponding operation is not available. - For example, if a class sets :meth:`__iter__` to ``None``, the class + For example, if a class sets :meth:`~object.__iter__` to ``None``, the class is not iterable. (Contributed by Andrew Barnert and Ivan Levkivskyi in :issue:`25958`.) @@ -871,7 +871,7 @@ Notable changes in the :mod:`asyncio` module since Python 3.5.0 of the last iteration will be discarded. (Contributed by Guido van Rossum in :issue:`25593`.) -* :meth:`Future.set_exception ` +* :meth:`Future.set_exception ` will now raise :exc:`TypeError` when passed an instance of the :exc:`StopIteration` exception. (Contributed by Chris Angelico in :issue:`26221`.) @@ -925,7 +925,7 @@ added to represent sized iterable container classes. (Contributed by Ivan Levkivskyi, docs by Neil Girdhar in :issue:`27598`.) The new :class:`~collections.abc.Reversible` abstract base class represents -iterable classes that also provide the :meth:`__reversed__` method. +iterable classes that also provide the :meth:`~object.__reversed__` method. (Contributed by Ivan Levkivskyi in :issue:`25987`.) The new :class:`~collections.abc.AsyncGenerator` abstract base class represents @@ -971,7 +971,7 @@ datetime -------- The :class:`~datetime.datetime` and :class:`~datetime.time` classes have -the new :attr:`~time.fold` attribute used to disambiguate local time +the new :attr:`~datetime.time.fold` attribute used to disambiguate local time when necessary. Many functions in the :mod:`datetime` have been updated to support local time disambiguation. See :ref:`Local Time Disambiguation ` section for more @@ -1052,12 +1052,12 @@ enum ---- Two new enumeration base classes have been added to the :mod:`enum` module: -:class:`~enum.Flag` and :class:`~enum.IntFlags`. Both are used to define +:class:`~enum.Flag` and :class:`~enum.IntFlag`. Both are used to define constants that can be combined using the bitwise operators. (Contributed by Ethan Furman in :issue:`23591`.) Many standard library modules have been updated to use the -:class:`~enum.IntFlags` class for their constants. +:class:`~enum.IntFlag` class for their constants. The new :class:`enum.auto` value can be used to assign values to enum members automatically:: @@ -1224,7 +1224,7 @@ generator expression scopes as if they were positional-only parameters called ``implicit0``. (Contributed by Jelle Zijlstra in :issue:`19611`.) To reduce code churn when upgrading from Python 2.7 and the legacy -:func:`inspect.getargspec` API, the previously documented deprecation of +:func:`!inspect.getargspec` API, the previously documented deprecation of :func:`inspect.getfullargspec` has been reversed. While this function is convenient for single/source Python 2/3 code bases, the richer :func:`inspect.signature` interface remains the recommended approach for new @@ -1275,7 +1275,7 @@ See the summary of :ref:`PEP 519 ` for details on how the A new :meth:`~os.scandir.close` method allows explicitly closing a :func:`~os.scandir` iterator. The :func:`~os.scandir` iterator now -supports the :term:`context manager` protocol. If a :func:`scandir` +supports the :term:`context manager` protocol. If a :func:`!scandir` iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted in its destructor. (Contributed by Serhiy Storchaka in :issue:`25994`.) @@ -1434,7 +1434,7 @@ defined in :mod:`http.server`, :mod:`xmlrpc.server` and protocol. (Contributed by Aviv Palivoda in :issue:`26404`.) -The :attr:`~socketserver.StreamRequestHandler.wfile` attribute of +The :attr:`wfile ` attribute of :class:`~socketserver.StreamRequestHandler` classes now implements the :class:`io.BufferedIOBase` writable interface. In particular, calling :meth:`~io.BufferedIOBase.write` is now guaranteed to send the @@ -1465,7 +1465,7 @@ The new :meth:`~ssl.SSLContext.get_ciphers` method can be used to get a list of enabled ciphers in order of cipher priority. All constants and flags have been converted to :class:`~enum.IntEnum` and -:class:`~enum.IntFlags`. +:class:`~enum.IntFlag`. (Contributed by Christian Heimes in :issue:`28025`.) Server and client-side specific TLS protocols for :class:`~ssl.SSLContext` @@ -1531,8 +1531,8 @@ Stéphane Wirtel in :issue:`25485`). time ---- -The :class:`~time.struct_time` attributes :attr:`tm_gmtoff` and -:attr:`tm_zone` are now available on all platforms. +The :class:`~time.struct_time` attributes :attr:`!tm_gmtoff` and +:attr:`!tm_zone` are now available on all platforms. timeit @@ -1551,12 +1551,12 @@ between best and worst times. tkinter ------- -Added methods :meth:`~tkinter.Variable.trace_add`, -:meth:`~tkinter.Variable.trace_remove` and :meth:`~tkinter.Variable.trace_info` -in the :class:`tkinter.Variable` class. They replace old methods -:meth:`~tkinter.Variable.trace_variable`, :meth:`~tkinter.Variable.trace`, -:meth:`~tkinter.Variable.trace_vdelete` and -:meth:`~tkinter.Variable.trace_vinfo` that use obsolete Tcl commands and might +Added methods :meth:`!Variable.trace_add`, +:meth:`!Variable.trace_remove` and :meth:`!trace_info` +in the :class:`!tkinter.Variable` class. They replace old methods +:meth:`!trace_variable`, :meth:`!trace`, +:meth:`!trace_vdelete` and +:meth:`!trace_vinfo` that use obsolete Tcl commands and might not work in future versions of Tcl. (Contributed by Serhiy Storchaka in :issue:`22115`). @@ -1674,8 +1674,8 @@ urllib.request If a HTTP request has a file or iterable body (other than a bytes object) but no ``Content-Length`` header, rather than -throwing an error, :class:`~urllib.request.AbstractHTTPHandler` now -falls back to use chunked transfer encoding. +throwing an error, :class:`AbstractHTTPHandler ` +now falls back to use chunked transfer encoding. (Contributed by Demian Brecht and Rolf Krahl in :issue:`12319`.) @@ -1701,7 +1701,7 @@ warnings A new optional *source* parameter has been added to the :func:`warnings.warn_explicit` function: the destroyed object which emitted a :exc:`ResourceWarning`. A *source* attribute has also been added to -:class:`warnings.WarningMessage` (contributed by Victor Stinner in +:class:`!warnings.WarningMessage` (contributed by Victor Stinner in :issue:`26568` and :issue:`26567`). When a :exc:`ResourceWarning` warning is logged, the :mod:`tracemalloc` module is now @@ -1942,7 +1942,7 @@ Raising the :exc:`StopIteration` exception inside a generator will now generate a :exc:`DeprecationWarning`, and will trigger a :exc:`RuntimeError` in Python 3.7. See :ref:`whatsnew-pep-479` for details. -The :meth:`__aiter__` method is now expected to return an asynchronous +The :meth:`~object.__aiter__` method is now expected to return an asynchronous iterator directly instead of returning an awaitable as previously. Doing the former will trigger a :exc:`DeprecationWarning`. Backward compatibility will be removed in Python 3.7. @@ -2189,7 +2189,7 @@ Changes in the Python API booleans being a subclass of integers, this should only be an issue if you were doing identity checks for ``1`` or ``0``. See :issue:`25768`. -* Reading the :attr:`~urllib.parse.SplitResult.port` attribute of +* Reading the :attr:`!port` attribute of :func:`urllib.parse.urlsplit` and :func:`~urllib.parse.urlparse` results now raises :exc:`ValueError` for out-of-range values, rather than returning :const:`None`. See :issue:`20059`. @@ -2197,8 +2197,8 @@ Changes in the Python API * The :mod:`!imp` module now raises a :exc:`DeprecationWarning` instead of :exc:`PendingDeprecationWarning`. -* The following modules have had missing APIs added to their :attr:`__all__` - attributes to match the documented APIs: +* The following modules have had missing APIs added to their + :attr:`~module.__all__` attributes to match the documented APIs: :mod:`calendar`, :mod:`!cgi`, :mod:`csv`, :mod:`~xml.etree.ElementTree`, :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`mailbox`, @@ -2253,11 +2253,13 @@ Changes in the Python API * As part of :pep:`487`, the handling of keyword arguments passed to :class:`type` (other than the metaclass hint, ``metaclass``) is now consistently delegated to :meth:`object.__init_subclass__`. This means that - :meth:`type.__new__` and :meth:`type.__init__` both now accept arbitrary - keyword arguments, but :meth:`object.__init_subclass__` (which is called from - :meth:`type.__new__`) will reject them by default. Custom metaclasses - accepting additional keyword arguments will need to adjust their calls to - :meth:`type.__new__` (whether direct or via :class:`super`) accordingly. + :meth:`type.__new__ ` and :meth:`type.__init__ + ` both now accept arbitrary keyword arguments, + but :meth:`object.__init_subclass__` (which is called from + :meth:`type.__new__ `) will reject them by default. + Custom metaclasses accepting additional keyword arguments will need to adjust + their calls to :meth:`type.__new__ ` + (whether direct or via :class:`super`) accordingly. * In ``distutils.command.sdist.sdist``, the ``default_format`` attribute has been removed and is no longer honored. Instead, the @@ -2305,7 +2307,7 @@ Changes in the Python API real-world compatibility. (Contributed by Lita Cho in :issue:`21815`.) -* The :func:`mmap.write() ` function now returns the number +* The :func:`mmap.mmap.write` function now returns the number of bytes written like other write methods. (Contributed by Jakub Stasiak in :issue:`26335`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index f420fa5c044..9ac3bf53833 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -320,9 +320,9 @@ effort will be made to add such support. PEP 562: Customization of Access to Module Attributes ----------------------------------------------------- -Python 3.7 allows defining :meth:`__getattr__` on modules and will call +Python 3.7 allows defining :meth:`~module.__getattr__` on modules and will call it whenever a module attribute is otherwise not found. Defining -:meth:`__dir__` on modules is now also allowed. +:meth:`~module.__dir__` on modules is now also allowed. A typical example of where this may be useful is module attribute deprecation and lazy loading. @@ -409,8 +409,8 @@ PEP 560: Core Support for ``typing`` module and Generic Types Initially :pep:`484` was designed in such way that it would not introduce *any* changes to the core CPython interpreter. Now type hints and the :mod:`typing` module are extensively used by the community, so this restriction is removed. -The PEP introduces two special methods :meth:`__class_getitem__` and -``__mro_entries__``, these methods are now used by most classes and special +The PEP introduces two special methods :meth:`~object.__class_getitem__` and +:meth:`~object.__mro_entries__`, these methods are now used by most classes and special constructs in :mod:`typing`. As a result, the speed of various operations with types increased up to 7 times, the generic types can be used without metaclass conflicts, and several long standing bugs in :mod:`typing` module are @@ -603,7 +603,7 @@ The new :mod:`importlib.resources` module provides several new APIs and one new ABC for access to, opening, and reading *resources* inside packages. Resources are roughly similar to files inside packages, but they needn't be actual files on the physical file system. Module loaders can provide a -:meth:`get_resource_reader` function which returns +:meth:`!get_resource_reader` function which returns a :class:`importlib.abc.ResourceReader` instance to support this new API. Built-in file path loaders and zip file loaders both support this. @@ -910,9 +910,9 @@ which allows listing the names of properties which should not become enum members. (Contributed by Ethan Furman in :issue:`31801`.) -In Python 3.8, attempting to check for non-Enum objects in :class:`Enum` +In Python 3.8, attempting to check for non-Enum objects in :class:`~enum.Enum` classes will raise a :exc:`TypeError` (e.g. ``1 in Color``); similarly, -attempting to check for non-Flag objects in a :class:`Flag` member will +attempting to check for non-Flag objects in a :class:`~enum.Flag` member will raise :exc:`TypeError` (e.g. ``1 in Perm.RW``); currently, both operations return :const:`False` instead and are deprecated. (Contributed by Ethan Furman in :issue:`33217`.) @@ -969,7 +969,7 @@ uses the current working directory. (Contributed by Stéphane Wirtel and Julien Palard in :issue:`28707`.) The new :class:`ThreadingHTTPServer ` class -uses threads to handle requests using :class:`~socketserver.ThreadingMixin`. +uses threads to handle requests using :class:`~socketserver.ThreadingMixIn`. It is used when ``http.server`` is run with ``-m``. (Contributed by Julien Palard in :issue:`31639`.) @@ -1052,12 +1052,12 @@ support the loading of resources from packages. See also lacks a spec. (Contributed by Garvit Khatri in :issue:`29851`.) -:func:`importlib.find_spec` now raises :exc:`ModuleNotFoundError` instead of +:func:`importlib.util.find_spec` now raises :exc:`ModuleNotFoundError` instead of :exc:`AttributeError` if the specified parent module is not a package (i.e. lacks a ``__path__`` attribute). (Contributed by Milan Oberkirch in :issue:`30436`.) -The new :func:`importlib.source_hash` can be used to compute the hash of +The new :func:`importlib.util.source_hash` can be used to compute the hash of the passed source. A :ref:`hash-based .pyc file ` embeds the value returned by this function. @@ -1148,7 +1148,7 @@ running. (Contributed by Antoine Pitrou in :issue:`30596`.) The new :meth:`Process.kill() ` method can -be used to terminate the process using the :data:`SIGKILL` signal on Unix. +be used to terminate the process using the :data:`~signal.SIGKILL` signal on Unix. (Contributed by Vitor Pereira in :issue:`30794`.) Non-daemonic threads created by :class:`~multiprocessing.Process` are now @@ -1280,9 +1280,10 @@ This function should be used instead of :func:`os.close` for better compatibility across platforms. (Contributed by Christian Heimes in :issue:`32454`.) -The :mod:`socket` module now exposes the :const:`socket.TCP_CONGESTION` -(Linux 2.6.13), :const:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37), and -:const:`socket.TCP_NOTSENT_LOWAT` (Linux 3.12) constants. +The :mod:`socket` module now exposes the :ref:`socket.TCP_CONGESTION +` (Linux 2.6.13), :ref:`socket.TCP_USER_TIMEOUT +` (Linux 2.6.37), and :ref:`socket.TCP_NOTSENT_LOWAT +` (Linux 3.12) constants. (Contributed by Omar Sandoval in :issue:`26273` and Nathaniel J. Smith in :issue:`29728`.) @@ -1298,11 +1299,14 @@ by default. socketserver ------------ -:meth:`socketserver.ThreadingMixIn.server_close` now waits until all non-daemon -threads complete. :meth:`socketserver.ForkingMixIn.server_close` now waits +:meth:`socketserver.ThreadingMixIn.server_close +` now waits until all non-daemon +threads complete. :meth:`socketserver.ForkingMixIn.server_close +` now waits until all child processes complete. -Add a new :attr:`socketserver.ForkingMixIn.block_on_close` class attribute to +Add a new :attr:`socketserver.ForkingMixIn.block_on_close +` class attribute to :class:`socketserver.ForkingMixIn` and :class:`socketserver.ThreadingMixIn` classes. Set the class attribute to ``False`` to get the pre-3.7 behaviour. @@ -1323,7 +1327,7 @@ ssl --- The :mod:`ssl` module now uses OpenSSL's builtin API instead of -:func:`~ssl.match_hostname` to check a host name or an IP address. Values +:func:`!match_hostname` to check a host name or an IP address. Values are validated during TLS handshake. Any certificate validation error including failing the host name check now raises :exc:`~ssl.SSLCertVerificationError` and aborts the handshake with a proper @@ -1341,7 +1345,7 @@ Host name validation can be customized with The ``ssl`` module no longer sends IP addresses in SNI TLS extension. (Contributed by Christian Heimes in :issue:`32185`.) -:func:`~ssl.match_hostname` no longer supports partial wildcards like +:func:`!match_hostname` no longer supports partial wildcards like ``www*.example.org``. (Contributed by Mandeep Singh in :issue:`23033` and Christian Heimes in :issue:`31399`.) @@ -1438,7 +1442,7 @@ The new :func:`sys.get_coroutine_origin_tracking_depth` function returns the current coroutine origin tracking depth, as set by the new :func:`sys.set_coroutine_origin_tracking_depth`. :mod:`asyncio` has been converted to use this new API instead of -the deprecated :func:`sys.set_coroutine_wrapper`. +the deprecated :func:`!sys.set_coroutine_wrapper`. (Contributed by Nathaniel J. Smith in :issue:`32591`.) @@ -1615,7 +1619,7 @@ external entities by default. xml.etree --------- -:ref:`ElementPath ` predicates in the :meth:`find` +:ref:`ElementPath ` predicates in the :meth:`!find` methods can now compare text of the current node with ``[. = "text"]``, not only text in children. Predicates also allow adding spaces for better readability. (Contributed by Stefan Behnel in :issue:`31648`.) @@ -1624,7 +1628,7 @@ better readability. (Contributed by Stefan Behnel in :issue:`31648`.) xmlrpc.server ------------- -:meth:`SimpleXMLRPCDispatcher.register_function ` +:meth:`!SimpleXMLRPCDispatcher.register_function` can now be used as a decorator. (Contributed by Xiang Zhang in :issue:`7769`.) @@ -1682,15 +1686,15 @@ The :mod:`tracemalloc` now exposes a C API through the new functions. (Contributed by Victor Stinner in :issue:`30054`.) -The new :c:func:`import__find__load__start` and -:c:func:`import__find__load__done` static markers can be used to trace -module imports. +The new :ref:`import__find__load__start ` and +:ref:`import__find__load__done ` static markers can be used +to trace module imports. (Contributed by Christian Heimes in :issue:`31574`.) The fields :c:member:`!name` and :c:member:`!doc` of structures :c:type:`PyMemberDef`, :c:type:`PyGetSetDef`, :c:type:`PyStructSequence_Field`, :c:type:`PyStructSequence_Desc`, -and :c:struct:`wrapperbase` are now of type ``const char *`` rather of +and :c:struct:`!wrapperbase` are now of type ``const char *`` rather of ``char *``. (Contributed by Serhiy Storchaka in :issue:`28761`.) The result of :c:func:`PyUnicode_AsUTF8AndSize` and :c:func:`PyUnicode_AsUTF8` @@ -1719,8 +1723,8 @@ Added C API support for timezones with timezone constructors and access to the UTC singleton with :c:data:`PyDateTime_TimeZone_UTC`. Contributed by Paul Ganssle in :issue:`10381`. -The type of results of :c:func:`PyThread_start_new_thread` and -:c:func:`PyThread_get_thread_ident`, and the *id* parameter of +The type of results of :c:func:`!PyThread_start_new_thread` and +:c:func:`!PyThread_get_thread_ident`, and the *id* parameter of :c:func:`PyThreadState_SetAsyncExc` changed from :c:expr:`long` to :c:expr:`unsigned long`. (Contributed by Serhiy Storchaka in :issue:`6532`.) @@ -1847,8 +1851,8 @@ make the creation of named tuples 4 to 6 times faster. (Contributed by Jelle Zijlstra with further improvements by INADA Naoki, Serhiy Storchaka, and Raymond Hettinger in :issue:`28638`.) -:meth:`date.fromordinal` and :meth:`date.fromtimestamp` are now up to -30% faster in the common case. +:meth:`datetime.date.fromordinal` and :meth:`datetime.date.fromtimestamp` +are now up to 30% faster in the common case. (Contributed by Paul Ganssle in :issue:`32403`.) The :func:`os.fwalk` function is now up to 2 times faster thanks to @@ -1997,9 +2001,9 @@ modes (this will be an error in future Python releases). enum ---- -In Python 3.8, attempting to check for non-Enum objects in :class:`Enum` +In Python 3.8, attempting to check for non-Enum objects in :class:`~enum.Enum` classes will raise a :exc:`TypeError` (e.g. ``1 in Color``); similarly, -attempting to check for non-Flag objects in a :class:`Flag` member will +attempting to check for non-Flag objects in a :class:`~enum.Flag` member will raise :exc:`TypeError` (e.g. ``1 in Perm.RW``); currently, both operations return :const:`False` instead. (Contributed by Ethan Furman in :issue:`33217`.) @@ -2034,14 +2038,14 @@ favour of :class:`importlib.abc.ResourceReader`. locale ------ -:func:`locale.format` has been deprecated, use :meth:`locale.format_string` +:func:`!locale.format` has been deprecated, use :meth:`locale.format_string` instead. (Contributed by Garvit in :issue:`10379`.) macpath ------- -The :mod:`macpath` is now deprecated and will be removed in Python 3.8. +The :mod:`!macpath` is now deprecated and will be removed in Python 3.8. (Contributed by Chi Hsuan Yen in :issue:`9850`.) @@ -2066,7 +2070,7 @@ if the passed argument is larger than 16 bits, an exception will be raised. ssl --- -:func:`ssl.wrap_socket` is deprecated. Use +:func:`!ssl.wrap_socket` is deprecated. Use :meth:`ssl.SSLContext.wrap_socket` instead. (Contributed by Christian Heimes in :issue:`28124`.) @@ -2082,8 +2086,8 @@ Use :func:`!sunau.open` instead. sys --- -Deprecated :func:`sys.set_coroutine_wrapper` and -:func:`sys.get_coroutine_wrapper`. +Deprecated :func:`!sys.set_coroutine_wrapper` and +:func:`!sys.get_coroutine_wrapper`. The undocumented ``sys.callstats()`` function has been deprecated and will be removed in a future Python version. @@ -2093,7 +2097,7 @@ will be removed in a future Python version. wave ---- -:func:`wave.openfp` has been deprecated and will be removed in Python 3.9. +:func:`!wave.openfp` has been deprecated and will be removed in Python 3.9. Use :func:`wave.open` instead. (Contributed by Brian Curtin in :issue:`31985`.) @@ -2173,8 +2177,8 @@ The following features and APIs have been removed from Python 3.7: * Removed previously deprecated in Python 2.4 classes ``Plist``, ``Dict`` and ``_InternalDict`` in the :mod:`plistlib` module. Dict values in the result - of functions :func:`~plistlib.readPlist` and - :func:`~plistlib.readPlistFromBytes` are now normal dicts. You no longer + of functions :func:`!readPlist` and + :func:`!readPlistFromBytes` are now normal dicts. You no longer can use attribute access to access items of these dictionaries. * The ``asyncio.windows_utils.socketpair()`` function has been @@ -2191,7 +2195,7 @@ The following features and APIs have been removed from Python 3.7: * Direct instantiation of :class:`ssl.SSLSocket` and :class:`ssl.SSLObject` objects is now prohibited. The constructors were never documented, tested, or designed as public constructors. Users were supposed to use - :func:`ssl.wrap_socket` or :class:`ssl.SSLContext`. + :func:`!ssl.wrap_socket` or :class:`ssl.SSLContext`. (Contributed by Christian Heimes in :issue:`32951`.) * The unused ``distutils`` ``install_misc`` command has been removed. @@ -2275,15 +2279,18 @@ Changes in Python Behavior Changes in the Python API ------------------------- -* :meth:`socketserver.ThreadingMixIn.server_close` now waits until all +* :meth:`socketserver.ThreadingMixIn.server_close + ` now waits until all non-daemon threads complete. Set the new :attr:`socketserver.ThreadingMixIn.block_on_close` class attribute to ``False`` to get the pre-3.7 behaviour. (Contributed by Victor Stinner in :issue:`31233` and :issue:`33540`.) -* :meth:`socketserver.ForkingMixIn.server_close` now waits until all +* :meth:`socketserver.ForkingMixIn.server_close + ` now waits until all child processes complete. Set the new - :attr:`socketserver.ForkingMixIn.block_on_close` class attribute to ``False`` + :attr:`socketserver.ForkingMixIn.block_on_close + ` class attribute to ``False`` to get the pre-3.7 behaviour. (Contributed by Victor Stinner in :issue:`31151` and :issue:`33540`.) @@ -2476,7 +2483,7 @@ avoiding possible problems use new functions :c:func:`PySlice_Unpack` and CPython bytecode changes ------------------------ -There are two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`!CALL_METHOD`. +There are two new opcodes: :opcode:`!LOAD_METHOD` and :opcode:`!CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) The :opcode:`!STORE_ANNOTATION` opcode has been removed. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 7aca35b2959..545a17aecab 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -536,7 +536,7 @@ Other Language Changes element is a callable with a ``(obj, state)`` signature. This allows the direct control over the state-updating behavior of a specific object. If not *None*, this callable will have priority over the object's - :meth:`~__setstate__` method. + :meth:`~object.__setstate__` method. (Contributed by Pierre Glaser and Olivier Grisel in :issue:`35900`.) New Modules @@ -1035,9 +1035,9 @@ symlinks and directory junctions) has been delegated to the operating system. Specifically, :func:`os.stat` will now traverse anything supported by the operating system, while :func:`os.lstat` will only open reparse points that identify as "name surrogates" while others are opened as for :func:`os.stat`. -In all cases, :attr:`stat_result.st_mode` will only have ``S_IFLNK`` set for +In all cases, :attr:`os.stat_result.st_mode` will only have ``S_IFLNK`` set for symbolic links and not other kinds of reparse points. To identify other kinds -of reparse point, check the new :attr:`stat_result.st_reparse_tag` attribute. +of reparse point, check the new :attr:`os.stat_result.st_reparse_tag` attribute. On Windows, :func:`os.readlink` is now able to read directory junctions. Note that :func:`~os.path.islink` will return ``False`` for directory junctions, @@ -1285,20 +1285,20 @@ now matches what the C tokenizer does internally. tkinter ------- -Added methods :meth:`~tkinter.Spinbox.selection_from`, -:meth:`~tkinter.Spinbox.selection_present`, -:meth:`~tkinter.Spinbox.selection_range` and -:meth:`~tkinter.Spinbox.selection_to` -in the :class:`tkinter.Spinbox` class. +Added methods :meth:`!selection_from`, +:meth:`!selection_present`, +:meth:`!selection_range` and +:meth:`!selection_to` +in the :class:`!tkinter.Spinbox` class. (Contributed by Juliette Monsel in :issue:`34829`.) -Added method :meth:`~tkinter.Canvas.moveto` -in the :class:`tkinter.Canvas` class. +Added method :meth:`!moveto` +in the :class:`!tkinter.Canvas` class. (Contributed by Juliette Monsel in :issue:`23831`.) -The :class:`tkinter.PhotoImage` class now has -:meth:`~tkinter.PhotoImage.transparency_get` and -:meth:`~tkinter.PhotoImage.transparency_set` methods. (Contributed by +The :class:`!tkinter.PhotoImage` class now has +:meth:`!transparency_get` and +:meth:`!transparency_set` methods. (Contributed by Zackery Spytz in :issue:`25451`.) @@ -1432,7 +1432,7 @@ and ``{namespace}*`` which returns all tags in the given namespace. (Contributed by Stefan Behnel in :issue:`28238`.) The :mod:`xml.etree.ElementTree` module provides a new function -:func:`–xml.etree.ElementTree.canonicalize` that implements C14N 2.0. +:func:`~xml.etree.ElementTree.canonicalize` that implements C14N 2.0. (Contributed by Stefan Behnel in :issue:`13611`.) The target object of :class:`xml.etree.ElementTree.XMLParser` can @@ -1573,7 +1573,7 @@ Build and C API Changes * :c:func:`Py_INCREF`, :c:func:`Py_DECREF` * :c:func:`Py_XINCREF`, :c:func:`Py_XDECREF` - * :c:func:`PyObject_INIT`, :c:func:`PyObject_INIT_VAR` + * :c:macro:`!PyObject_INIT`, :c:macro:`!PyObject_INIT_VAR` * Private functions: :c:func:`!_PyObject_GC_TRACK`, :c:func:`!_PyObject_GC_UNTRACK`, :c:func:`!_Py_Dealloc` @@ -1629,8 +1629,8 @@ Build and C API Changes (Contributed by Pablo Galindo in :issue:`37221`.) * :c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full - path (:c:func:`Py_GetProgramFullPath`) rather than to the program name - (:c:func:`Py_GetProgramName`). + path (:c:func:`!Py_GetProgramFullPath`) rather than to the program name + (:c:func:`!Py_GetProgramName`). (Contributed by Victor Stinner in :issue:`38234`.) @@ -1677,7 +1677,7 @@ Deprecated constant nodes. (Contributed by Serhiy Storchaka in :issue:`36917`.) -* The :func:`asyncio.coroutine` :term:`decorator` is deprecated and will be +* The :deco:`!asyncio.coroutine` :term:`decorator` is deprecated and will be removed in version 3.10. Instead of ``@asyncio.coroutine``, use :keyword:`async def` instead. (Contributed by Andrew Svetlov in :issue:`36921`.) @@ -1697,22 +1697,22 @@ Deprecated (Contributed by Yury Selivanov in :issue:`34790`.) * The following functions and methods are deprecated in the :mod:`gettext` - module: :func:`~gettext.lgettext`, :func:`~gettext.ldgettext`, - :func:`~gettext.lngettext` and :func:`~gettext.ldngettext`. + module: :func:`!lgettext`, :func:`!ldgettext`, + :func:`!lngettext` and :func:`!ldngettext`. They return encoded bytes, and it's possible that you will get unexpected Unicode-related exceptions if there are encoding problems with the translated strings. It's much better to use alternatives which return Unicode strings in Python 3. These functions have been broken for a long time. - Function :func:`~gettext.bind_textdomain_codeset`, methods - :meth:`~gettext.NullTranslations.output_charset` and - :meth:`~gettext.NullTranslations.set_output_charset`, and the *codeset* + Function :func:`!bind_textdomain_codeset`, methods + :meth:`!NullTranslations.output_charset` and + :meth:`!NullTranslations.set_output_charset`, and the *codeset* parameter of functions :func:`~gettext.translation` and :func:`~gettext.install` are also deprecated, since they are only used for the ``l*gettext()`` functions. (Contributed by Serhiy Storchaka in :issue:`33710`.) -* The :meth:`~threading.Thread.isAlive` method of :class:`threading.Thread` +* The :meth:`!isAlive` method of :class:`threading.Thread` has been deprecated. (Contributed by Donghee Na in :issue:`35283`.) @@ -1727,7 +1727,7 @@ Deprecated * Deprecated passing the following arguments as keyword arguments: - *func* in :func:`functools.partialmethod`, :func:`weakref.finalize`, - :meth:`profile.Profile.runcall`, :meth:`cProfile.Profile.runcall`, + :meth:`profile.Profile.runcall`, :meth:`!cProfile.Profile.runcall`, :meth:`bdb.Bdb.runcall`, :meth:`trace.Trace.runfunc` and :func:`curses.wrapper`. - *function* in :meth:`unittest.TestCase.addCleanup`. @@ -1735,11 +1735,11 @@ Deprecated :class:`concurrent.futures.ThreadPoolExecutor` and :class:`concurrent.futures.ProcessPoolExecutor`. - *callback* in :meth:`contextlib.ExitStack.callback`, - :meth:`contextlib.AsyncExitStack.callback` and + :meth:`!contextlib.AsyncExitStack.callback` and :meth:`contextlib.AsyncExitStack.push_async_callback`. - - *c* and *typeid* in the :meth:`~multiprocessing.managers.Server.create` - method of :class:`multiprocessing.managers.Server` and - :class:`multiprocessing.managers.SharedMemoryServer`. + - *c* and *typeid* in the :meth:`!create` + method of :class:`!multiprocessing.managers.Server` and + :class:`!multiprocessing.managers.SharedMemoryServer`. - *obj* in :func:`weakref.finalize`. In future releases of Python, they will be :ref:`positional-only @@ -1757,14 +1757,14 @@ The following features and APIs have been removed from Python 3.8: able to import from collections was marked for removal in 3.8, but has been delayed to 3.9. (See :gh:`81134`.) -* The :mod:`macpath` module, deprecated in Python 3.7, has been removed. +* The :mod:`!macpath` module, deprecated in Python 3.7, has been removed. (Contributed by Victor Stinner in :issue:`35471`.) -* The function :func:`platform.popen` has been removed, after having been +* The function :func:`!platform.popen` has been removed, after having been deprecated since Python 3.3: use :func:`os.popen` instead. (Contributed by Victor Stinner in :issue:`35345`.) -* The function :func:`time.clock` has been removed, after having been +* The function :func:`!time.clock` has been removed, after having been deprecated since Python 3.3: use :func:`time.perf_counter` or :func:`time.process_time` instead, depending on your requirements, to have well-defined behavior. @@ -1800,8 +1800,8 @@ The following features and APIs have been removed from Python 3.8: :func:`fileinput.FileInput` which was ignored and deprecated since Python 3.6 has been removed. :issue:`36952` (Contributed by Matthias Bussonnier.) -* The functions :func:`sys.set_coroutine_wrapper` and - :func:`sys.get_coroutine_wrapper` deprecated in Python 3.7 have been removed; +* The functions :func:`!sys.set_coroutine_wrapper` and + :func:`!sys.get_coroutine_wrapper` deprecated in Python 3.7 have been removed; :issue:`36933` (Contributed by Matthias Bussonnier.) @@ -1864,9 +1864,10 @@ Changes in the Python API * :class:`subprocess.Popen` can now use :func:`os.posix_spawn` in some cases for better performance. On Windows Subsystem for Linux and QEMU User - Emulation, the :class:`Popen` constructor using :func:`os.posix_spawn` no longer raises an - exception on errors like "missing program". Instead the child process fails with a - non-zero :attr:`~Popen.returncode`. + Emulation, the :class:`~subprocess.Popen` constructor using + :func:`os.posix_spawn` no longer raises an exception on errors like + "missing program". Instead the child process fails with a + non-zero :attr:`~subprocess.Popen.returncode`. (Contributed by Joannah Nanjekye and Victor Stinner in :issue:`35537`.) * The *preexec_fn* argument of * :class:`subprocess.Popen` is no longer @@ -1875,11 +1876,11 @@ Changes in the Python API (Contributed by Eric Snow in :issue:`34651`, modified by Christian Heimes in :issue:`37951`.) -* The :meth:`imap.IMAP4.logout` method no longer silently ignores arbitrary +* The :meth:`imaplib.IMAP4.logout` method no longer silently ignores arbitrary exceptions. (Contributed by Victor Stinner in :issue:`36348`.) -* The function :func:`platform.popen` has been removed, after having been deprecated since +* The function :func:`!platform.popen` has been removed, after having been deprecated since Python 3.3: use :func:`os.popen` instead. (Contributed by Victor Stinner in :issue:`35345`.) @@ -1894,9 +1895,11 @@ Changes in the Python API specialized methods like :meth:`~tkinter.ttk.Treeview.selection_set` for changing the selection. (Contributed by Serhiy Storchaka in :issue:`31508`.) -* The :meth:`writexml`, :meth:`toxml` and :meth:`toprettyxml` methods of - :mod:`xml.dom.minidom`, and the :meth:`write` method of :mod:`xml.etree`, - now preserve the attribute order specified by the user. +* The :meth:`~xml.dom.minidom.Node.writexml`, :meth:`~xml.dom.minidom.Node.toxml` + and :meth:`~xml.dom.minidom.Node.toprettyxml` methods of + :mod:`xml.dom.minidom` and the :meth:`~xml.etree.ElementTree.ElementTree.write` + method of :mod:`xml.etree.ElementTree` now preserve the attribute order + specified by the user. (Contributed by Diego Rojas and Raymond Hettinger in :issue:`34160`.) * A :mod:`dbm.dumb` database opened with flags ``'r'`` is now read-only. @@ -1916,8 +1919,8 @@ Changes in the Python API ``type.__new__``. A :exc:`DeprecationWarning` was emitted in Python 3.6--3.7. (Contributed by Serhiy Storchaka in :issue:`23722`.) -* The :class:`cProfile.Profile` class can now be used as a context - manager. (Contributed by Scott Sanderson in :issue:`29235`.) +* The :class:`cProfile.Profile ` class can now be used as + a context manager. (Contributed by Scott Sanderson in :issue:`29235`.) * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`, :func:`shutil.copytree` and :func:`shutil.move` use platform-specific @@ -1952,7 +1955,7 @@ Changes in the Python API (Contributed by Christian Heimes in :issue:`17239`.) * Deleting a key from a read-only :mod:`dbm` database (:mod:`dbm.dumb`, - :mod:`dbm.gnu` or :mod:`dbm.ndbm`) raises :attr:`error` (:exc:`dbm.dumb.error`, + :mod:`dbm.gnu` or :mod:`dbm.ndbm`) raises :attr:`!error` (:exc:`dbm.dumb.error`, :exc:`dbm.gnu.error` or :exc:`dbm.ndbm.error`) instead of :exc:`KeyError`. (Contributed by Xiang Zhang in :issue:`33106`.) @@ -2044,7 +2047,7 @@ Changes in the C API :c:func:`PyType_FromSpec`) hold a reference to their type object. Increasing the reference count of these type objects has been moved from :c:func:`PyType_GenericAlloc` to the more low-level functions, - :c:func:`PyObject_Init` and :c:func:`PyObject_INIT`. + :c:func:`PyObject_Init` and :c:macro:`!PyObject_INIT`. This makes types created through :c:func:`PyType_FromSpec` behave like other classes in managed code. @@ -2064,7 +2067,7 @@ Changes in the C API This may happen after calling :c:macro:`PyObject_New`, :c:macro:`PyObject_NewVar`, :c:func:`PyObject_GC_New`, :c:func:`PyObject_GC_NewVar`, or any other custom allocator that uses - :c:func:`PyObject_Init` or :c:func:`PyObject_INIT`. + :c:func:`PyObject_Init` or :c:macro:`!PyObject_INIT`. Example: diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 896e8f4a489..40d4a27bff9 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -423,8 +423,8 @@ digests. It skips MD5 on platforms that block MD5 digest. fcntl ----- -Added constants :const:`~fcntl.F_OFD_GETLK`, :const:`~fcntl.F_OFD_SETLK` -and :const:`~fcntl.F_OFD_SETLKW`. +Added constants :const:`!fcntl.F_OFD_GETLK`, :const:`!fcntl.F_OFD_SETLK` +and :const:`!fcntl.F_OFD_SETLKW`. (Contributed by Donghee Na in :issue:`38602`.) ftplib @@ -644,7 +644,7 @@ attribute. random ------ -Added a new :attr:`random.Random.randbytes` method: generate random bytes. +Added a new :meth:`random.Random.randbytes` method: generate random bytes. (Contributed by Victor Stinner in :issue:`40286`.) signal @@ -776,7 +776,7 @@ Optimizations :pep:`590` vectorcall protocol. (Contributed by Donghee Na, Mark Shannon, Jeroen Demeyer and Petr Viktorin in :issue:`37207`.) -* Optimized :func:`~set.difference_update` for the case when the other set +* Optimized :meth:`!set.difference_update` for the case when the other set is much larger than the base set. (Suggested by Evgeny Kapun with code contributed by Michele Orrù in :issue:`8425`.) diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index 6ff722a1894..38194db670b 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.15.rst 3.14.rst 3.13.rst 3.12.rst diff --git a/Grammar/Tokens b/Grammar/Tokens index e40a4437afb..0547e6ed08f 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -1,3 +1,8 @@ +# When adding new tokens, remember to update the PEG generator in +# Tools/peg_generator/pegen/parser_generator.py +# This will ensure that older versions of Python can generate a Python parser +# using "python -m pegen python ". + ENDMARKER NAME NUMBER diff --git a/Grammar/python.gram b/Grammar/python.gram index f3ef990923e..7ae00c6f005 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -94,14 +94,16 @@ func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMA # GENERAL STATEMENTS # ================== -statements[asdl_stmt_seq*]: a=statement+ { _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a)) } +statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } -statement[asdl_stmt_seq*]: - | a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } +statement[asdl_stmt_seq*]: + | a=compound_stmt { _PyPegen_register_stmts(p , + (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) + ) } | a[asdl_stmt_seq*]=simple_stmts { a } single_compound_stmt[asdl_stmt_seq*]: - | a=compound_stmt { + | a=compound_stmt { _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a)) } statement_newline[asdl_stmt_seq*]: @@ -184,7 +186,9 @@ return_stmt[stmt_ty]: | 'return' a=[star_expressions] { _PyAST_Return(a, EXTRA) } raise_stmt[stmt_ty]: - | 'raise' a=expression b=['from' z=expression { z }] { _PyAST_Raise(a, b, EXTRA) } + | 'raise' a=expression 'from' b=expression { _PyAST_Raise(a, b, EXTRA) } + | invalid_raise_stmt + | 'raise' a=expression { _PyAST_Raise(a, NULL, EXTRA) } | 'raise' { _PyAST_Raise(NULL, NULL, EXTRA) } pass_stmt[stmt_ty]: @@ -208,7 +212,9 @@ del_stmt[stmt_ty]: yield_stmt[stmt_ty]: y=yield_expr { _PyAST_Expr(y, EXTRA) } -assert_stmt[stmt_ty]: 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } +assert_stmt[stmt_ty]: + | invalid_assert_stmt + | 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } import_stmt[stmt_ty]: | invalid_import @@ -449,9 +455,9 @@ except_block[excepthandler_ty]: _PyAST_ExceptHandler(e, ((expr_ty) t)->v.Name.id, b, EXTRA) } | 'except' e=expressions ':' b=block { CHECK_VERSION( - excepthandler_ty, - 14, - "except expressions without parentheses are", + excepthandler_ty, + 14, + "except expressions without parentheses are", _PyAST_ExceptHandler(e, NULL, b, EXTRA)) } | 'except' ':' b=block { _PyAST_ExceptHandler(NULL, NULL, b, EXTRA) } | invalid_except_stmt @@ -463,9 +469,9 @@ except_star_block[excepthandler_ty]: _PyAST_ExceptHandler(e, ((expr_ty) t)->v.Name.id, b, EXTRA) } | 'except' '*' e=expressions ':' b=block { CHECK_VERSION( - excepthandler_ty, - 14, - "except expressions without parentheses are", + excepthandler_ty, + 14, + "except expressions without parentheses are", _PyAST_ExceptHandler(e, NULL, b, EXTRA)) } | invalid_except_star_stmt finally_block[asdl_stmt_seq*]: @@ -620,6 +626,7 @@ mapping_pattern[pattern_ty]: CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, items)), NULL, EXTRA) } + | invalid_mapping_pattern items_pattern[asdl_seq*]: | ','.key_value_pattern+ @@ -977,15 +984,18 @@ tstring_middle[expr_ty]: | tstring_replacement_field | t=TSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } tstring[expr_ty] (memo): - | a=TSTRING_START b=tstring_middle* c=TSTRING_END { + | a=TSTRING_START b=tstring_middle* c=TSTRING_END { CHECK_VERSION( - expr_ty, - 14, - "t-strings are", + expr_ty, + 14, + "t-strings are", _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) } string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) } -strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } +strings[expr_ty] (memo): + | invalid_string_tstring_concat + | a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } + | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) } list[expr_ty]: | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } @@ -1287,9 +1297,33 @@ invalid_ann_assign_target[expr_ty]: | list | tuple | '(' a=invalid_ann_assign_target ')' { a } +invalid_raise_stmt: + | a='raise' b='from' { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget an expression between 'raise' and 'from'?") } + | 'raise' expression a='from' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "did you forget an expression after 'from'?") } invalid_del_stmt: | 'del' a=star_expressions { RAISE_SYNTAX_ERROR_INVALID_TARGET(DEL_TARGETS, a) } +invalid_assert_stmt: + | 'assert' a=expression '=' b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "cannot assign to %s here. Maybe you meant '==' instead of '='?", + _PyPegen_get_expr_name(a)) } + | 'assert' expression ',' a=expression '=' b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "cannot assign to %s here. Maybe you meant '==' instead of '='?", + _PyPegen_get_expr_name(a)) } + | 'assert' a=expression ':=' b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "cannot use named expression without parentheses here") } + | 'assert' expression ',' a=expression ':=' b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "cannot use named expression without parentheses here") } invalid_block: | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") } invalid_comprehension: @@ -1305,7 +1339,7 @@ invalid_dict_comprehension: RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "dict unpacking cannot be used in dict comprehension") } invalid_parameters: | a="/" ',' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one argument must precede /") } + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") } | (slash_no_default | slash_with_default) param_maybe_default* a='/' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ may appear only once") } | slash_no_default? param_no_default* invalid_parameters_helper a=param_no_default { @@ -1319,21 +1353,21 @@ invalid_parameters: invalid_default: | a='=' &(')'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expected default value expression") } invalid_star_etc: - | a='*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "named arguments must follow bare *") } + | a='*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "named parameters must follow bare *") } | '*' ',' TYPE_COMMENT { RAISE_SYNTAX_ERROR("bare * has associated type comment") } - | '*' param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-positional argument cannot have default value") } + | '*' param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-positional parameter cannot have default value") } | '*' (param_no_default | ',') param_maybe_default* a='*' (param_no_default | ',') { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "* argument may appear only once") } + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "* may appear only once") } invalid_kwds: - | '**' param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-keyword argument cannot have default value") } - | '**' param ',' a=param { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "arguments cannot follow var-keyword argument") } - | '**' param ',' a[Token*]=('*'|'**'|'/') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "arguments cannot follow var-keyword argument") } + | '**' param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-keyword parameter cannot have default value") } + | '**' param ',' a=param { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") } + | '**' param ',' a[Token*]=('*'|'**'|'/') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") } invalid_parameters_helper: # This is only there to avoid type errors | a=slash_with_default { _PyPegen_singleton_seq(p, a) } | param_with_default+ invalid_lambda_parameters: | a="/" ',' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one argument must precede /") } + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") } | (lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* a='/' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "/ may appear only once") } | lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper a=lambda_param_no_default { @@ -1348,14 +1382,14 @@ invalid_lambda_parameters_helper: | a=lambda_slash_with_default { _PyPegen_singleton_seq(p, a) } | lambda_param_with_default+ invalid_lambda_star_etc: - | '*' (':' | ',' (':' | '**')) { RAISE_SYNTAX_ERROR("named arguments must follow bare *") } - | '*' lambda_param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-positional argument cannot have default value") } + | '*' (':' | ',' (':' | '**')) { RAISE_SYNTAX_ERROR("named parameters must follow bare *") } + | '*' lambda_param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-positional parameter cannot have default value") } | '*' (lambda_param_no_default | ',') lambda_param_maybe_default* a='*' (lambda_param_no_default | ',') { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "* argument may appear only once") } + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "* may appear only once") } invalid_lambda_kwds: - | '**' lambda_param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-keyword argument cannot have default value") } - | '**' lambda_param ',' a=lambda_param { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "arguments cannot follow var-keyword argument") } - | '**' lambda_param ',' a[Token*]=('*'|'**'|'/') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "arguments cannot follow var-keyword argument") } + | '**' lambda_param a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "var-keyword parameter cannot have default value") } + | '**' lambda_param ',' a=lambda_param { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") } + | '**' lambda_param ',' a[Token*]=('*'|'**'|'/') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "parameters cannot follow var-keyword parameter") } invalid_double_type_comments: | TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT { RAISE_SYNTAX_ERROR("Cannot have two type comments on def") } @@ -1382,12 +1416,12 @@ invalid_import: | 'import' token=NEWLINE { RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_dotted_as_name: - | dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) a=expression { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, + | dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } invalid_import_from_as_name: - | NAME 'as' !(NAME (',' | ')' | NEWLINE)) a=expression { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, + | NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } invalid_import_from_targets: @@ -1418,7 +1452,7 @@ invalid_except_stmt: RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") } | a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } - | 'except' expression 'as' a=expression { + | 'except' expression 'as' a=expression ':' block { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use except statement with %s", _PyPegen_get_expr_name(a)) } invalid_except_star_stmt: @@ -1426,7 +1460,7 @@ invalid_except_star_stmt: RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") } | a='except' '*' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' '*' (NEWLINE | ':') { RAISE_SYNTAX_ERROR("expected one or more exception types") } - | 'except' '*' expression 'as' a=expression { + | 'except' '*' expression 'as' a=expression ':' block { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use except* statement with %s", _PyPegen_get_expr_name(a)) } invalid_finally_stmt: @@ -1443,6 +1477,10 @@ invalid_match_stmt: | "match" subject_expr NEWLINE { CHECK_VERSION(void*, 10, "Pattern matching is", RAISE_SYNTAX_ERROR("expected ':'") ) } | a="match" subject=subject_expr ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after 'match' statement on line %d", a->lineno) } + | a="case" patterns guard? b=':' block { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, + "case statement must be inside match statement") } invalid_case_block: | "case" patterns guard? NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a="case" patterns guard? ':' NEWLINE !INDENT { @@ -1457,6 +1495,10 @@ invalid_class_pattern: PyPegen_first_item(a, pattern_ty), PyPegen_last_item(a, pattern_ty), "positional patterns follow keyword patterns") } +invalid_mapping_pattern: + | '{' (items_pattern ',')? rest=double_star_pattern ',' items_pattern ','? '}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + rest, + "double star pattern must be the last (right-most) subpattern in the mapping pattern") } invalid_class_argument_pattern[asdl_pattern_seq*]: | [positional_patterns ','] keyword_patterns ',' a=positional_patterns { a } invalid_if_stmt: @@ -1546,6 +1588,12 @@ invalid_tstring_conversion_character: | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") } | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") } +invalid_string_tstring_concat: + | a=(fstring|string)+ b[expr_ty]=tstring { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") } + | a=tstring+ b[expr_ty]=(fstring|string) { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") } + invalid_arithmetic: | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } invalid_factor: diff --git a/Include/Python.h b/Include/Python.h index 64be8014589..78083bbf31d 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -16,6 +16,7 @@ // Include standard header files +// When changing these files, remember to update Doc/extending/extending.rst. #include // assert() #include // uintptr_t #include // INT_MAX @@ -45,19 +46,28 @@ # endif #endif -// gh-111506: The free-threaded build is not compatible with the limited API -// or the stable ABI. -#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) -# error "The limited API is not currently supported in the free-threaded build" +#if defined(Py_GIL_DISABLED) +# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT) +# error "Py_LIMITED_API is not currently supported in the free-threaded build" +# endif + +# if defined(_MSC_VER) +# include // __readgsqword() +# endif + +# if defined(__MINGW32__) +# include // __readgsqword() +# endif +#endif // Py_GIL_DISABLED + +#ifdef _MSC_VER +// Ignore MSC warning C4201: "nonstandard extension used: nameless +// struct/union". (Only generated for C standard versions less than C11, which +// we don't *officially* support.) +__pragma(warning(push)) +__pragma(warning(disable: 4201)) #endif -#if defined(Py_GIL_DISABLED) && defined(_MSC_VER) -# include // __readgsqword() -#endif - -#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) -# include // __readgsqword() -#endif // Include Python header files #include "pyport.h" @@ -68,7 +78,7 @@ #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" -#include "lock.h" +#include "cpython/pylock.h" #include "critical_section.h" #include "object.h" #include "refcount.h" @@ -95,7 +105,7 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" -#include "monitoring.h" +#include "cpython/monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" @@ -138,4 +148,8 @@ #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" +#ifdef _MSC_VER +__pragma(warning(pop)) // warning(disable: 4201) +#endif + #endif /* !Py_PYTHON_H */ diff --git a/Include/abstract.h b/Include/abstract.h index b9199fc03a3..80f3298701d 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -138,7 +138,12 @@ extern "C" { Delete attribute named attr_name, for object o. Returns -1 on failure. - This is the equivalent of the Python statement: del o.attr_name. */ + This is the equivalent of the Python statement: del o.attr_name. + + Implemented as a macro in the limited C API 3.12 and older. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030d0000 +# define PyObject_DelAttrString(O, A) PyObject_SetAttrString((O), (A), NULL) +#endif /* Implemented elsewhere: @@ -147,7 +152,12 @@ extern "C" { Delete attribute named attr_name, for object o. Returns -1 on failure. This is the equivalent of the Python - statement: del o.attr_name. */ + statement: del o.attr_name. + + Implemented as a macro in the limited C API 3.12 and older. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030d0000 +# define PyObject_DelAttr(O, A) PyObject_SetAttr((O), (A), NULL) +#endif /* Implemented elsewhere: diff --git a/Include/audit.h b/Include/audit.h index 793b7077e10..9be54ad4411 100644 --- a/Include/audit.h +++ b/Include/audit.h @@ -1,5 +1,5 @@ -#ifndef Py_AUDIT_H -#define Py_AUDIT_H +#ifndef _Py_AUDIT_H +#define _Py_AUDIT_H #ifdef __cplusplus extern "C" { #endif @@ -18,13 +18,13 @@ PyAPI_FUNC(int) PySys_AuditTuple( #ifndef Py_LIMITED_API -# define Py_CPYTHON_AUDIT_H +# define _Py_CPYTHON_AUDIT_H # include "cpython/audit.h" -# undef Py_CPYTHON_AUDIT_H +# undef _Py_CPYTHON_AUDIT_H #endif #ifdef __cplusplus } #endif -#endif /* !Py_AUDIT_H */ +#endif /* !_Py_AUDIT_H */ diff --git a/Include/boolobject.h b/Include/boolobject.h index 3037e61bbf6..b56e2baecaa 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -34,9 +34,16 @@ PyAPI_FUNC(int) Py_IsTrue(PyObject *x); PyAPI_FUNC(int) Py_IsFalse(PyObject *x); #define Py_IsFalse(x) Py_Is((x), Py_False) -/* Macros for returning Py_True or Py_False, respectively */ -#define Py_RETURN_TRUE return Py_True -#define Py_RETURN_FALSE return Py_False +/* Macros for returning Py_True or Py_False, respectively. + * Only treat Py_True and Py_False as immortal in the limited C API 3.12 + * and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_TRUE return Py_NewRef(Py_True) +# define Py_RETURN_FALSE return Py_NewRef(Py_False) +#else +# define Py_RETURN_TRUE return Py_True +# define Py_RETURN_FALSE return Py_False +#endif /* Function to return a bool from a C long */ PyAPI_FUNC(PyObject *) PyBool_FromLong(long); diff --git a/Include/ceval.h b/Include/ceval.h index 32ab38972e5..e9df8684996 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -133,13 +133,6 @@ PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); #define FVS_MASK 0x4 #define FVS_HAVE_SPEC 0x4 -/* Special methods used by LOAD_SPECIAL */ -#define SPECIAL___ENTER__ 0 -#define SPECIAL___EXIT__ 1 -#define SPECIAL___AENTER__ 2 -#define SPECIAL___AEXIT__ 3 -#define SPECIAL_MAX 3 - #ifndef Py_LIMITED_API # define Py_CPYTHON_CEVAL_H # include "cpython/ceval.h" diff --git a/Include/cpython/audit.h b/Include/cpython/audit.h index 3c5c7a8c060..536f9248632 100644 --- a/Include/cpython/audit.h +++ b/Include/cpython/audit.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_AUDIT_H +#ifndef _Py_CPYTHON_AUDIT_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/bytearrayobject.h b/Include/cpython/bytearrayobject.h index 4dddef713ce..1edd0820742 100644 --- a/Include/cpython/bytearrayobject.h +++ b/Include/cpython/bytearrayobject.h @@ -5,25 +5,25 @@ /* Object layout */ typedef struct { PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ + /* How many bytes allocated in ob_bytes + + In the current implementation this is equivalent to Py_SIZE(ob_bytes_object). + The value is always loaded and stored atomically for thread safety. + There are API compatibilty concerns with removing so keeping for now. */ + Py_ssize_t ob_alloc; char *ob_bytes; /* Physical backing buffer */ char *ob_start; /* Logical start inside ob_bytes */ Py_ssize_t ob_exports; /* How many buffer exports */ + PyObject *ob_bytes_object; /* PyBytes for zero-copy bytes conversion */ } PyByteArrayObject; -PyAPI_DATA(char) _PyByteArray_empty_string[]; - /* Macros and static inline functions, trading safety for speed */ #define _PyByteArray_CAST(op) \ (assert(PyByteArray_Check(op)), _Py_CAST(PyByteArrayObject*, op)) static inline char* PyByteArray_AS_STRING(PyObject *op) { - PyByteArrayObject *self = _PyByteArray_CAST(op); - if (Py_SIZE(self)) { - return self->ob_start; - } - return _PyByteArray_empty_string; + return _PyByteArray_CAST(op)->ob_start; } #define PyByteArray_AS_STRING(self) PyByteArray_AS_STRING(_PyObject_CAST(self)) diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 71c133f173f..85bc2b827df 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -40,3 +40,46 @@ _PyBytes_Join(PyObject *sep, PyObject *iterable) { return PyBytes_Join(sep, iterable); } + + +// --- PyBytesWriter API ----------------------------------------------------- + +typedef struct PyBytesWriter PyBytesWriter; + +PyAPI_FUNC(PyBytesWriter *) PyBytesWriter_Create( + Py_ssize_t size); +PyAPI_FUNC(void) PyBytesWriter_Discard( + PyBytesWriter *writer); +PyAPI_FUNC(PyObject*) PyBytesWriter_Finish( + PyBytesWriter *writer); +PyAPI_FUNC(PyObject*) PyBytesWriter_FinishWithSize( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(PyObject*) PyBytesWriter_FinishWithPointer( + PyBytesWriter *writer, + void *buf); + +PyAPI_FUNC(void*) PyBytesWriter_GetData( + PyBytesWriter *writer); +PyAPI_FUNC(Py_ssize_t) PyBytesWriter_GetSize( + PyBytesWriter *writer); + +PyAPI_FUNC(int) PyBytesWriter_WriteBytes( + PyBytesWriter *writer, + const void *bytes, + Py_ssize_t size); +PyAPI_FUNC(int) PyBytesWriter_Format( + PyBytesWriter *writer, + const char *format, + ...); + +PyAPI_FUNC(int) PyBytesWriter_Resize( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(int) PyBytesWriter_Grow( + PyBytesWriter *writer, + Py_ssize_t size); +PyAPI_FUNC(void*) PyBytesWriter_GrowAndUpdatePointer( + PyBytesWriter *writer, + Py_ssize_t size, + void *buf); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 3f0dce03455..84456a709a6 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -282,15 +282,6 @@ typedef struct _line_offsets { */ PyAPI_FUNC(int) _PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds); -/* Create a comparable key used to compare constants taking in account the - * object type. It is used to make sure types are not coerced (e.g., float and - * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms - * - * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) - * depending on the type and the value. The type is the first item to not - * compare bytes and str which can raise a BytesWarning exception. */ -PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj); - PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyObject *lnotab); diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index fbdc6a91fe8..58da80140dc 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -7,7 +7,8 @@ typedef struct { double imag; } Py_complex; -// Operations on complex numbers. +/* Operations on complex numbers (soft deprecated + since Python 3.15). */ PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 35db3fb6a59..4fc46fefb93 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -73,22 +73,32 @@ typedef struct PyCriticalSection2 PyCriticalSection2; PyAPI_FUNC(void) PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); +PyAPI_FUNC(void) +PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); + PyAPI_FUNC(void) PyCriticalSection_End(PyCriticalSection *c); PyAPI_FUNC(void) PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); +PyAPI_FUNC(void) +PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); + PyAPI_FUNC(void) PyCriticalSection2_End(PyCriticalSection2 *c); #ifndef Py_GIL_DISABLED # define Py_BEGIN_CRITICAL_SECTION(op) \ { +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { # define Py_END_CRITICAL_SECTION() \ } # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ { +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { # define Py_END_CRITICAL_SECTION2() \ } #else /* !Py_GIL_DISABLED */ @@ -118,6 +128,11 @@ struct PyCriticalSection2 { PyCriticalSection _py_cs; \ PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_BeginMutex(&_py_cs, mutex) + # define Py_END_CRITICAL_SECTION() \ PyCriticalSection_End(&_py_cs); \ } @@ -127,6 +142,11 @@ struct PyCriticalSection2 { PyCriticalSection2 _py_cs2; \ PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + # define Py_END_CRITICAL_SECTION2() \ PyCriticalSection2_End(&_py_cs2); \ } diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index df9ec7050fc..5f2f7b6d4f5 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -39,16 +39,6 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObje PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -// Inserts `key` with a value `default_value`, if `key` is not already present -// in the dictionary. If `result` is not NULL, then the value associated -// with `key` is returned in `*result` (either the existing value, or the now -// inserted `default_value`). -// Returns: -// -1 on error -// 0 if `key` was not present and `default_value` was inserted -// 1 if `key` was present and `default_value` was not inserted -PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); - /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 18249b95bef..9e1599a7648 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -97,11 +97,6 @@ static inline PyObject* PyFunction_GET_GLOBALS(PyObject *func) { } #define PyFunction_GET_GLOBALS(func) PyFunction_GET_GLOBALS(_PyObject_CAST(func)) -static inline PyObject* PyFunction_GET_BUILTINS(PyObject *func) { - return _PyFunction_CAST(func)->func_builtins; -} -#define PyFunction_GET_BUILTINS(func) PyFunction_GET_BUILTINS(_PyObject_CAST(func)) - static inline PyObject* PyFunction_GET_MODULE(PyObject *func) { return _PyFunction_CAST(func)->func_module; } @@ -139,7 +134,8 @@ PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *); V(DESTROY) \ V(MODIFY_CODE) \ V(MODIFY_DEFAULTS) \ - V(MODIFY_KWDEFAULTS) + V(MODIFY_KWDEFAULTS) \ + V(MODIFY_QUALNAME) typedef enum { #define PY_DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cc..149a20af8b9 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,13 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( + PyObject *spec, + PyObject *(*initfunc)(void)); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 7ce4acfeb71..1c979d91a40 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -102,15 +102,14 @@ typedef struct PyPreConfig { /* Enable UTF-8 mode? (PEP 540) - Disabled by default (equals to 0). + If equal to 1, use the UTF-8 encoding and use "surrogateescape" for the + stdin & stdout error handlers. - Set to 1 by "-X utf8" and "-X utf8=1" command line options. - Set to 1 by PYTHONUTF8=1 environment variable. + Enabled by default (equal to 1; PEP 686), or if Py_UTF8Mode=1, + or if "-X utf8=1" or PYTHONUTF8=1. - Set to 0 by "-X utf8=0" and PYTHONUTF8=0. - - If equals to -1, it is set to 1 if the LC_CTYPE locale is "C" or - "POSIX", otherwise it is set to 0. Inherit Py_UTF8Mode value value. */ + Set to 0 by "-X utf8=0" or PYTHONUTF8=0. + */ int utf8_mode; /* If non-zero, enable the Python Development Mode. diff --git a/Include/cpython/marshal.h b/Include/cpython/marshal.h new file mode 100644 index 00000000000..6c1f7f96b6a --- /dev/null +++ b/Include/cpython/marshal.h @@ -0,0 +1,17 @@ +#ifndef _Py_CPYTHON_MARSHAL_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, + Py_ssize_t); +PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); + +#define Py_MARSHAL_VERSION 5 + +PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); +PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); +PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); + +PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); +PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h index d3b88f58c82..61344421064 100644 --- a/Include/cpython/modsupport.h +++ b/Include/cpython/modsupport.h @@ -24,3 +24,15 @@ typedef struct _PyArg_Parser { PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); + +#ifdef Py_BUILD_CORE +// Internal; defined here to avoid explicitly including pycore_modsupport.h +#define _Py_INTERNAL_ABI_SLOT \ + {Py_mod_abi, (void*) &(PyABIInfo) { \ + .abiinfo_major_version = 1, \ + .abiinfo_minor_version = 0, \ + .flags = PyABIInfo_INTERNAL, \ + .build_version = PY_VERSION_HEX, \ + .abi_version = PY_VERSION_HEX }} \ + /////////////////////////////////////////////////////// +#endif diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index ce92942404c..5094c8c23ae 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -1,7 +1,13 @@ -#ifndef Py_CPYTHON_MONITORING_H -# error "this header file must not be included directly" +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif +// There is currently no limited API for monitoring + + /* Local events. * These require bytecode instrumentation */ @@ -267,3 +273,9 @@ PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike } #undef _PYMONITORING_IF_ACTIVE + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_MONITORING_H diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 973d358ed8e..8693390aeda 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -295,7 +295,10 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); -PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(void) PyUnstable_Object_Dump(PyObject *); + +// Alias for backward compatibility +#define _PyObject_Dump PyUnstable_Object_Dump PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); @@ -387,10 +390,11 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); process with a message on stderr if the given condition fails to hold, but compile away to nothing if NDEBUG is defined. - However, before aborting, Python will also try to call _PyObject_Dump() on - the given object. This may be of use when investigating bugs in which a - particular object is corrupt (e.g. buggy a tp_visit method in an extension - module breaking the garbage collector), to help locate the broken objects. + However, before aborting, Python will also try to call + PyUnstable_Object_Dump() on the given object. This may be of use when + investigating bugs in which a particular object is corrupt (e.g. buggy a + tp_visit method in an extension module breaking the garbage collector), to + help locate the broken objects. The WITH_MSG variant allows you to supply an additional message that Python will attempt to print to stderr, after the object dump. */ @@ -432,8 +436,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed( PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); -PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count); - /* For backwards compatibility with the old trashcan mechanism */ #define Py_TRASHCAN_BEGIN(op, dealloc) #define Py_TRASHCAN_END @@ -442,7 +444,6 @@ PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int m PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); -PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); @@ -463,6 +464,7 @@ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); typedef enum { PyRefTracer_CREATE = 0, PyRefTracer_DESTROY = 1, + PyRefTracer_TRACKER_REMOVED = 2, } PyRefTracerEvent; typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615e..7176f667a40 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -948,14 +948,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) memory_order_relaxed); } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_relaxed); -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1031,6 +1023,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 6b63d304b0d..be2e3b641c2 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -18,6 +18,7 @@ typedef struct { PyException_HEAD PyObject *msg; PyObject *excs; + PyObject *excs_str; } PyBaseExceptionGroupObject; typedef struct { diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index a33ba10b8d3..dac223368db 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -7,7 +7,7 @@ /* Parameters used for the numeric hash implementation. See notes for _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ + reduction modulo the prime 2**PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 # define PyHASH_BITS 61 @@ -15,7 +15,7 @@ # define PyHASH_BITS 31 #endif -#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_MODULUS (((size_t)1 << PyHASH_BITS) - 1) #define PyHASH_INF 314159 #define PyHASH_IMAG PyHASH_MULTIPLIER diff --git a/Include/cpython/lock.h b/Include/cpython/pylock.h similarity index 77% rename from Include/cpython/lock.h rename to Include/cpython/pylock.h index 8ee03e82f74..460ac2c9f80 100644 --- a/Include/cpython/lock.h +++ b/Include/cpython/pylock.h @@ -1,7 +1,11 @@ -#ifndef Py_CPYTHON_LOCK_H -# error "this header file must not be included directly" +#ifndef Py_LOCK_H +#define Py_LOCK_H +#ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { #endif + #define _Py_UNLOCKED 0 #define _Py_LOCKED 1 @@ -36,6 +40,9 @@ PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m); // exported function for unlocking the mutex PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m); +// exported function for checking if the mutex is locked +PyAPI_FUNC(int) PyMutex_IsLocked(PyMutex *m); + // Locks the mutex. // // If the mutex is currently locked, the calling thread will be parked until @@ -61,3 +68,18 @@ _PyMutex_Unlock(PyMutex *m) } } #define PyMutex_Unlock _PyMutex_Unlock + +// Checks if the mutex is currently locked. +static inline int +_PyMutex_IsLocked(PyMutex *m) +{ + return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; +} +#define PyMutex_IsLocked _PyMutex_IsLocked + + +#ifdef __cplusplus +} +#endif +#endif // !Py_LIMITED_API +#endif // !Py_LOCK_H diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 7f1bc363861..22df26bd37a 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -28,10 +28,10 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_OPCODE 7 /* Remote debugger support */ -#define MAX_SCRIPT_PATH_SIZE 512 -typedef struct _remote_debugger_support { +#define _Py_MAX_SCRIPT_PATH_SIZE 512 +typedef struct { int32_t debugger_pending_call; - char debugger_script_path[MAX_SCRIPT_PATH_SIZE]; + char debugger_script_path[_Py_MAX_SCRIPT_PATH_SIZE]; } _PyRemoteDebuggerSupport; typedef struct _err_stackitem { @@ -61,6 +61,8 @@ typedef struct _stack_chunk { PyObject * data[1]; /* Variable sized */ } _PyStackChunk; +/* Minimum size of data stack chunk */ +#define _PY_DATA_STACK_CHUNK_SIZE (16*1024) struct _ts { /* See Python/ceval.c for comments explaining most fields */ @@ -105,11 +107,15 @@ struct _ts { # define _PyThreadState_WHENCE_THREADING 3 # define _PyThreadState_WHENCE_GILSTATE 4 # define _PyThreadState_WHENCE_EXEC 5 +# define _PyThreadState_WHENCE_THREADING_DAEMON 6 #endif /* Currently holds the GIL. Must be its own field to avoid data races */ int holds_gil; + /* Currently requesting the GIL */ + int gil_requested; + int _whence; /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). @@ -129,6 +135,15 @@ struct _ts { /* Pointer to currently executing frame. */ struct _PyInterpreterFrame *current_frame; + /* Pointer to the base frame (bottommost sentinel frame). + Used by profilers to validate complete stack unwinding. + Points to the embedded base_frame in _PyThreadStateImpl. + The frame is embedded there rather than here because _PyInterpreterFrame + is defined in internal headers that cannot be exposed in the public API. */ + struct _PyInterpreterFrame *base_frame; + + struct _PyInterpreterFrame *last_profiled_frame; + Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; PyObject *c_profileobj; @@ -155,6 +170,11 @@ struct _ts { */ unsigned long native_thread_id; + /* List of objects that still need to be cleaned up, singly linked + * via their gc headers' gc_next pointers. The list is populated by + * _PyTrash_thread_deposit_object and cleaned up by + * _PyTrash_thread_destroy_chain. + */ PyObject *delete_later; /* Tagged pointer to top-most critical section, or zero if there is no @@ -196,6 +216,9 @@ struct _ts { PyObject *current_executor; + /* Internal to the JIT */ + struct _PyExitData *jit_exit; + uint64_t dict_global_version; /* Used to store/retrieve `threading.local` keys/values for this thread */ @@ -206,6 +229,15 @@ struct _ts { */ PyObject *threading_local_sentinel; _PyRemoteDebuggerSupport remote_debugger_support; + +#ifdef Py_STATS + // Pointer to PyStats structure, NULL if recording is off. For the + // free-threaded build, the structure is per-thread (stored as a pointer + // in _PyThreadStateImpl). For the default build, the structure is stored + // in the PyInterpreterState structure (threads do not have their own + // structure and all share the same per-interpreter structure). + PyStats *pystats; +#endif }; /* other API */ @@ -228,6 +260,21 @@ PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); // function is set, otherwise disable them. PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); +#ifdef Py_STATS +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +extern _Py_thread_local PyThreadState *_Py_tss_tstate; + +static inline PyStats* +_PyThreadState_GetStatsFast(void) +{ + if (_Py_tss_tstate == NULL) { + return NULL; // no attached thread state + } + return _Py_tss_tstate->pystats; +} +#endif +#endif // Py_STATS + /* PyGILState */ /* Helper/diagnostic function - return 1 if the current thread @@ -241,6 +288,18 @@ PyAPI_FUNC(int) PyGILState_Check(void); */ PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); +// Set the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(int) PyUnstable_ThreadState_SetStackProtection( + PyThreadState *tstate, + void *stack_start_addr, // Stack start address + size_t stack_size); // Stack size (in bytes) + +// Reset the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(void) PyUnstable_ThreadState_ResetStackProtection( + PyThreadState *tstate); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index cf830b6066f..1c94603c08b 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -4,7 +4,7 @@ // // - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() // and Py_DECREF(). -// - _Py_stats variable +// - _PyStats_GET() // // Functions of the sys module: // @@ -14,7 +14,7 @@ // - sys._stats_dump() // // Python must be built with ./configure --enable-pystats to define the -// Py_STATS macro. +// _PyStats_GET() macro. // // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. @@ -109,6 +109,18 @@ typedef struct _gc_stats { uint64_t objects_not_transitively_reachable; } GCStats; +#ifdef Py_GIL_DISABLED +// stats specific to free-threaded build +typedef struct _ft_stats { + // number of times interpreter had to spin or park when trying to acquire a mutex + uint64_t mutex_sleeps; + // number of times that the QSBR mechanism polled (compute read sequence value) + uint64_t qsbr_polls; + // number of times stop-the-world mechanism was used + uint64_t world_stops; +} FTStats; +#endif + typedef struct _uop_stats { uint64_t execution_count; uint64_t miss; @@ -138,6 +150,8 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimizer_attempts; uint64_t optimizer_successes; + uint64_t optimizer_contradiction; + uint64_t optimizer_frame_overflow; uint64_t optimizer_failure_reason_no_memory; uint64_t remove_globals_builtins_changed; uint64_t remove_globals_incorrect_keys; @@ -173,22 +187,48 @@ typedef struct _stats { CallStats call_stats; ObjectStats object_stats; OptimizationStats optimization_stats; +#ifdef Py_GIL_DISABLED + FTStats ft_stats; +#endif RareEventStats rare_event_stats; - GCStats *gc_stats; + GCStats gc_stats[3]; // must match NUM_GENERATIONS } PyStats; +// Export for most shared extensions +PyAPI_FUNC(PyStats *) _PyStats_GetLocal(void); -// Export for shared extensions like 'math' -PyAPI_DATA(PyStats*) _Py_stats; +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +// use inline function version defined in cpython/pystate.h +static inline PyStats *_PyThreadState_GetStatsFast(void); +#define _PyStats_GET _PyThreadState_GetStatsFast +#else +#define _PyStats_GET _PyStats_GetLocal +#endif + +#define _Py_STATS_EXPR(expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL) { \ + s->expr; \ + } \ + } while (0) + +#define _Py_STATS_COND_EXPR(cond, expr) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s != NULL && (cond)) { \ + s->expr; \ + } \ + } while (0) #ifdef _PY_INTERPRETER -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_decrefs++) #else -# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) -# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) -# define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0) -# define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0) +# define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.increfs++) +# define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.decrefs++) +# define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_increfs++) +# define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_decrefs++) #endif diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h new file mode 100644 index 00000000000..4c3ea1faceb --- /dev/null +++ b/Include/cpython/sliceobject.h @@ -0,0 +1,20 @@ +#ifndef Py_CPYTHON_SLICEOBJECT_H +# error "this header file must not be included directly" +#endif + +/* Slice object interface */ + +/* +A slice object containing start, stop, and step data members (the +names are from range). After much talk with Guido, it was decided to +let these be any arbitrary python type. Py_None stands for omitted values. +*/ +typedef struct { + PyObject_HEAD + PyObject *start, *stop, *step; /* not NULL */ +} PySliceObject; + +PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); +PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, + PyObject **start_ptr, PyObject **stop_ptr, + PyObject **step_ptr); diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h new file mode 100644 index 00000000000..328fbe86143 --- /dev/null +++ b/Include/cpython/structseq.h @@ -0,0 +1,12 @@ +#ifndef Py_CPYTHON_STRUCTSEQ_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, + PyStructSequence_Desc *desc); +PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, + PyStructSequence_Desc *desc); + +typedef PyTupleObject PyStructSequence; +#define PyStructSequence_SET_ITEM PyStructSequence_SetItem +#define PyStructSequence_GET_ITEM PyStructSequence_GetItem diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index afb98ccbb81..888baaf3358 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -38,3 +38,7 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { } #define PyTuple_SET_ITEM(op, index, value) \ PyTuple_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) + +PyAPI_FUNC(PyObject*) PyTuple_FromArray( + PyObject *const *array, + Py_ssize_t size); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 136f5d5c5f8..2853d24c34b 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -47,12 +47,69 @@ static inline Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) { /* --- Unicode Type ------------------------------------------------------- */ +struct _PyUnicodeObject_state { + /* If interned is non-zero, the two references from the + dictionary to this object are *not* counted in ob_refcnt. + The possible values here are: + 0: Not Interned + 1: Interned + 2: Interned and Immortal + 3: Interned, Immortal, and Static + This categorization allows the runtime to determine the right + cleanup mechanism at runtime shutdown. */ +#ifdef Py_GIL_DISABLED + // Needs to be accessed atomically, so can't be a bit field. + unsigned char interned; +#else + unsigned int interned:2; +#endif + /* Character size: + + - PyUnicode_1BYTE_KIND (1): + + * character type = Py_UCS1 (8 bits, unsigned) + * all characters are in the range U+0000-U+00FF (latin1) + * if ascii is set, all characters are in the range U+0000-U+007F + (ASCII), otherwise at least one character is in the range + U+0080-U+00FF + + - PyUnicode_2BYTE_KIND (2): + + * character type = Py_UCS2 (16 bits, unsigned) + * all characters are in the range U+0000-U+FFFF (BMP) + * at least one character is in the range U+0100-U+FFFF + + - PyUnicode_4BYTE_KIND (4): + + * character type = Py_UCS4 (32 bits, unsigned) + * all characters are in the range U+0000-U+10FFFF + * at least one character is in the range U+10000-U+10FFFF + */ + unsigned int kind:3; + /* Compact is with respect to the allocation scheme. Compact unicode + objects only require one memory block while non-compact objects use + one block for the PyUnicodeObject struct and another for its data + buffer. */ + unsigned int compact:1; + /* The string only contains characters in the range U+0000-U+007F (ASCII) + and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is + set, use the PyASCIIObject structure. */ + unsigned int ascii:1; + /* The object is statically allocated. */ + unsigned int statically_allocated:1; +#ifndef Py_GIL_DISABLED + /* Historical: padding to ensure that PyUnicode_DATA() is always aligned to + 4 bytes (see issue gh-63736 on m68k) */ + unsigned int :24; +#endif +}; + /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject structure. state.ascii and state.compact are set, and the data immediately follow the structure. utf8_length can be found in the length field; the utf8 pointer is equal to the data pointer. */ typedef struct { - /* There are 4 forms of Unicode strings: + /* There are 3 forms of Unicode strings: - compact ascii: @@ -99,67 +156,8 @@ typedef struct { PyObject_HEAD Py_ssize_t length; /* Number of code points in the string */ Py_hash_t hash; /* Hash value; -1 if not set */ -#ifdef Py_GIL_DISABLED - /* Ensure 4 byte alignment for PyUnicode_DATA(), see gh-63736 on m68k. - In the non-free-threaded build, we'll use explicit padding instead */ - _Py_ALIGN_AS(4) -#endif - struct { - /* If interned is non-zero, the two references from the - dictionary to this object are *not* counted in ob_refcnt. - The possible values here are: - 0: Not Interned - 1: Interned - 2: Interned and Immortal - 3: Interned, Immortal, and Static - This categorization allows the runtime to determine the right - cleanup mechanism at runtime shutdown. */ -#ifdef Py_GIL_DISABLED - // Needs to be accessed atomically, so can't be a bit field. - unsigned char interned; -#else - unsigned int interned:2; -#endif - /* Character size: - - - PyUnicode_1BYTE_KIND (1): - - * character type = Py_UCS1 (8 bits, unsigned) - * all characters are in the range U+0000-U+00FF (latin1) - * if ascii is set, all characters are in the range U+0000-U+007F - (ASCII), otherwise at least one character is in the range - U+0080-U+00FF - - - PyUnicode_2BYTE_KIND (2): - - * character type = Py_UCS2 (16 bits, unsigned) - * all characters are in the range U+0000-U+FFFF (BMP) - * at least one character is in the range U+0100-U+FFFF - - - PyUnicode_4BYTE_KIND (4): - - * character type = Py_UCS4 (32 bits, unsigned) - * all characters are in the range U+0000-U+10FFFF - * at least one character is in the range U+10000-U+10FFFF - */ - unsigned int kind:3; - /* Compact is with respect to the allocation scheme. Compact unicode - objects only require one memory block while non-compact objects use - one block for the PyUnicodeObject struct and another for its data - buffer. */ - unsigned int compact:1; - /* The string only contains characters in the range U+0000-U+007F (ASCII) - and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is - set, use the PyASCIIObject structure. */ - unsigned int ascii:1; - /* The object is statically allocated. */ - unsigned int statically_allocated:1; -#ifndef Py_GIL_DISABLED - /* Padding to ensure that PyUnicode_DATA() is always aligned to - 4 bytes (see issue gh-63736 on m68k) */ - unsigned int :24; -#endif - } state; + /* Ensure 4 byte alignment for PyUnicode_DATA(), see gh-63736 on m68k. */ + _Py_ALIGNED_DEF(4, struct _PyUnicodeObject_state) state; } PyASCIIObject; /* Non-ASCII strings allocated through PyUnicode_New use the @@ -300,6 +298,16 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { } #define PyUnicode_GET_LENGTH(op) PyUnicode_GET_LENGTH(_PyObject_CAST(op)) +/* Returns the cached hash, or -1 if not cached yet. */ +static inline Py_hash_t +PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) { +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&_PyASCIIObject_CAST(op)->hash); +#else + return _PyASCIIObject_CAST(op)->hash; +#endif +} + /* Write into the canonical representation, this function does not do any sanity checks and is intended for usage in loops. The caller should cache the kind and data pointers obtained from other function calls. @@ -478,6 +486,10 @@ PyAPI_FUNC(int) PyUnicodeWriter_WriteUTF8( PyUnicodeWriter *writer, const char *str, Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteASCII( + PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( PyUnicodeWriter *writer, const wchar_t *str, diff --git a/Include/cpython/warnings.h b/Include/cpython/warnings.h index 8731fd2e96b..4e3eb88e8ff 100644 --- a/Include/cpython/warnings.h +++ b/Include/cpython/warnings.h @@ -18,9 +18,3 @@ PyAPI_FUNC(int) PyErr_WarnExplicitFormat( // DEPRECATED: Use PyErr_WarnEx() instead. #define PyErr_Warn(category, msg) PyErr_WarnEx((category), (msg), 1) - -int _PyErr_WarnExplicitObjectWithContext( - PyObject *category, - PyObject *message, - PyObject *filename, - int lineno); diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h index da8e77cddac..e0711407cee 100644 --- a/Include/cpython/weakrefobject.h +++ b/Include/cpython/weakrefobject.h @@ -47,20 +47,3 @@ PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); // Test if a weak reference is dead. PyAPI_FUNC(int) PyWeakref_IsDead(PyObject *ref); - -Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) -{ - PyWeakReference *ref = _PyWeakref_CAST(ref_obj); - PyObject *obj = ref->wr_object; - // Explanation for the Py_REFCNT() check: when a weakref's target is part - // of a long chain of deallocations which triggers the trashcan mechanism, - // clearing the weakrefs can be delayed long after the target's refcount - // has dropped to zero. In the meantime, code accessing the weakref will - // be able to "see" the target object even though it is supposed to be - // unreachable. See issue gh-60806. - if (Py_REFCNT(obj) > 0) { - return obj; - } - return Py_None; -} -#define PyWeakref_GET_OBJECT(ref) PyWeakref_GET_OBJECT(_PyObject_CAST(ref)) diff --git a/Include/datetime.h b/Include/datetime.h index b78cc0e8e2e..ed36e6e48c8 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -1,8 +1,8 @@ /* datetime.h */ #ifndef Py_LIMITED_API -#ifndef DATETIME_H -#define DATETIME_H +#ifndef Py_DATETIME_H +#define Py_DATETIME_H #ifdef __cplusplus extern "C" { #endif @@ -263,5 +263,5 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; #ifdef __cplusplus } #endif -#endif +#endif /* !Py_DATETIME_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/dictobject.h b/Include/dictobject.h index 1bbeec1ab69..0384e3131dc 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -68,6 +68,18 @@ PyAPI_FUNC(int) PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result PyAPI_FUNC(int) PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030F0000 +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); +#endif + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); #endif diff --git a/Include/exports.h b/Include/exports.h index 0c646d5beb6..62feb09ed2b 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -9,6 +9,7 @@ inside the Python core, they are private to the core. If in an extension module, it may be declared with external linkage depending on the platform. + PyMODEXPORT_FUNC: Like PyMODINIT_FUNC, but for a slots array As a number of platforms support/require "__declspec(dllimport/dllexport)", we support a HAVE_DECLSPEC_DLL macro to save duplication. @@ -62,9 +63,9 @@ /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ # if defined(__CYGWIN__) -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # else /* __CYGWIN__ */ -# define PyMODINIT_FUNC PyObject* +# define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ # else /* Py_BUILD_CORE */ /* Building an extension module, or an embedded situation */ @@ -78,9 +79,9 @@ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE /* module init functions outside the core must be exported */ # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # endif /* __cplusplus */ # endif /* Py_BUILD_CORE */ # endif /* HAVE_DECLSPEC_DLL */ @@ -93,13 +94,15 @@ #ifndef PyAPI_DATA # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE #endif -#ifndef PyMODINIT_FUNC +#ifndef _PyINIT_FUNC_DECLSPEC # if defined(__cplusplus) -# define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL # else /* __cplusplus */ -# define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* +# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL # endif /* __cplusplus */ #endif +#define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* +#define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* #endif /* Py_EXPORTS_H */ diff --git a/Include/fileutils.h b/Include/fileutils.h index 1509198e45f..00c37fa4602 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -16,7 +16,7 @@ # define S_IFMT 0170000 #endif #ifndef S_IFLNK - // Windows doesn't define S_IFLNK, but posixmodule.c maps + // Windows doesn't define S_IFLNK, but fileutils.c maps // IO_REPARSE_TAG_SYMLINK to S_IFLNK. # define S_IFLNK 0120000 #endif diff --git a/Include/floatobject.h b/Include/floatobject.h index 4d24a76edd5..814337b070a 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -18,14 +18,14 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) -#define Py_RETURN_INF(sign) \ - do { \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_INFINITY); \ - } \ - else { \ - return PyFloat_FromDouble(-Py_INFINITY); \ - } \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(INFINITY); \ + } \ + else { \ + return PyFloat_FromDouble(-INFINITY); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); diff --git a/Include/import.h b/Include/import.h index 24b23b91191..d91ebe96ca8 100644 --- a/Include/import.h +++ b/Include/import.h @@ -51,9 +51,6 @@ PyAPI_FUNC(PyObject *) PyImport_AddModuleRef( PyAPI_FUNC(PyObject *) PyImport_ImportModule( const char *name /* UTF-8 encoded string */ ); -Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock( - const char *name /* UTF-8 encoded string */ - ); PyAPI_FUNC(PyObject *) PyImport_ImportModuleLevel( const char *name, /* UTF-8 encoded string */ PyObject *globals, diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index d97f51b8eef..a7daa3a40a4 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -634,10 +634,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; #ifdef MI_ENCODE_FREELIST - next = (mi_block_t*)mi_ptr_decode(null, block->next, keys); + next = (mi_block_t*)mi_ptr_decode(null, mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next), keys); #else MI_UNUSED(keys); MI_UNUSED(null); - next = (mi_block_t*)block->next; + next = (mi_block_t*)mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); return next; @@ -646,10 +646,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { mi_track_mem_undefined(block,sizeof(mi_block_t)); #ifdef MI_ENCODE_FREELIST - block->next = mi_ptr_encode(null, next, keys); + mi_atomic_store_relaxed(&block->next, mi_ptr_encode(null, next, keys)); #else MI_UNUSED(keys); MI_UNUSED(null); - block->next = (mi_encoded_t)next; + mi_atomic_store_relaxed(&block->next, (mi_encoded_t)next); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); } diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 354839ba955..19e93224174 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -50,6 +50,32 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_cache_align #endif +#if (MI_DEBUG) +#if defined(_MSC_VER) +#define mi_decl_noreturn __declspec(noreturn) +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) +#define mi_decl_noreturn __attribute__((__noreturn__)) +#else +#define mi_decl_noreturn +#endif + +/* + * 'cold' attribute seems to have been fully supported since GCC 4.x. + * See https://github.com/gcc-mirror/gcc/commit/52bf96d2f299e9e6. + */ +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) +#define mi_decl_cold __attribute__((cold)) +#else +#define mi_decl_cold +#endif + +#if (defined(__GNUC__) && defined(__THROW)) +#define mi_decl_throw __THROW +#else +#define mi_decl_throw +#endif +#endif + // ------------------------------------------------------ // Variants // ------------------------------------------------------ @@ -235,7 +261,7 @@ typedef size_t mi_threadid_t; // free lists contain blocks typedef struct mi_block_s { - mi_encoded_t next; + _Atomic(mi_encoded_t) next; } mi_block_t; @@ -455,7 +481,7 @@ typedef struct mi_segment_s { struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`) size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) - size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long) + size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim if it is too long) size_t used; // count of pages in use uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` @@ -582,7 +608,8 @@ struct mi_heap_s { #if (MI_DEBUG) // use our own assertion to print without memory allocation -void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); +mi_decl_noreturn mi_decl_cold mi_decl_throw +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func); #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) #else #define mi_assert(x) @@ -678,7 +705,7 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); // Thread Local data // ------------------------------------------------------ -// A "span" is is an available range of slices. The span queues keep +// A "span" is an available range of slices. The span queues keep // track of slice spans of at most the given `slice_count` (but more than the previous size class). typedef struct mi_span_queue_s { mi_slice_t* first; diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 3cc0afac4bd..30809e09700 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -51,6 +51,10 @@ extern int _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls); // Export for '_bisect' shared extension. PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); +// Convert Python int to Py_ssize_t. Do nothing if the argument is None. +// Raises ValueError if argument is negative. +PyAPI_FUNC(int) _Py_convert_optional_to_non_negative_ssize_t(PyObject *, void *); + // Same as PyNumber_Index() but can return an instance of a subclass of int. // Export for 'math' shared extension. PyAPI_FUNC(PyObject*) _PyNumber_Index(PyObject *o); diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 942d8b107a7..7f60eb49508 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -22,33 +22,48 @@ extern "C" { Another use is for the Tier 2 optimizer to decide when to create a new Tier 2 trace (executor). Again, exponential backoff is used. - The 16-bit counter is structured as a 12-bit unsigned 'value' - and a 4-bit 'backoff' field. When resetting the counter, the + The 16-bit counter is structured as a 13-bit unsigned 'value' + and a 3-bit 'backoff' field. When resetting the counter, the backoff field is incremented (until it reaches a limit) and the - value is set to a bit mask representing the value 2**backoff - 1. - The maximum backoff is 12 (the number of bits in the value). + value is set to a bit mask representing some prime value - 1. + New values and backoffs for each backoff are calculated once + at compile time and saved to value_and_backoff_next table. + The maximum backoff is 6, since 7 is an UNREACHABLE_BACKOFF. There is an exceptional value which must not be updated, 0xFFFF. */ -#define BACKOFF_BITS 4 -#define MAX_BACKOFF 12 -#define UNREACHABLE_BACKOFF 15 +#define BACKOFF_BITS 3 +#define BACKOFF_MASK 7 +#define MAX_BACKOFF 6 +#define UNREACHABLE_BACKOFF 7 +#define MAX_VALUE 0x1FFF -static inline bool -is_unreachable_backoff_counter(_Py_BackoffCounter counter) -{ - return counter.value_and_backoff == UNREACHABLE_BACKOFF; -} +#define MAKE_VALUE_AND_BACKOFF(value, backoff) \ + ((value << BACKOFF_BITS) | backoff) + +// For previous backoff b we use value x such that +// x + 1 is near to 2**(2*b+1) and x + 1 is prime. +static const uint16_t value_and_backoff_next[] = { + MAKE_VALUE_AND_BACKOFF(1, 1), + MAKE_VALUE_AND_BACKOFF(6, 2), + MAKE_VALUE_AND_BACKOFF(30, 3), + MAKE_VALUE_AND_BACKOFF(126, 4), + MAKE_VALUE_AND_BACKOFF(508, 5), + MAKE_VALUE_AND_BACKOFF(2052, 6), + // We use the same backoff counter for all backoffs >= MAX_BACKOFF. + MAKE_VALUE_AND_BACKOFF(8190, 6), + MAKE_VALUE_AND_BACKOFF(8190, 6), +}; static inline _Py_BackoffCounter make_backoff_counter(uint16_t value, uint16_t backoff) { - assert(backoff <= 15); - assert(value <= 0xFFF); - _Py_BackoffCounter result; - result.value_and_backoff = (value << BACKOFF_BITS) | backoff; - return result; + assert(backoff <= UNREACHABLE_BACKOFF); + assert(value <= MAX_VALUE); + return ((_Py_BackoffCounter){ + .value_and_backoff = MAKE_VALUE_AND_BACKOFF(value, backoff) + }); } static inline _Py_BackoffCounter @@ -62,14 +77,11 @@ forge_backoff_counter(uint16_t counter) static inline _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter counter) { - assert(!is_unreachable_backoff_counter(counter)); - int backoff = counter.value_and_backoff & 15; - if (backoff < MAX_BACKOFF) { - return make_backoff_counter((1 << (backoff + 1)) - 1, backoff + 1); - } - else { - return make_backoff_counter((1 << MAX_BACKOFF) - 1, MAX_BACKOFF); - } + uint16_t backoff = counter.value_and_backoff & BACKOFF_MASK; + assert(backoff <= MAX_BACKOFF); + return ((_Py_BackoffCounter){ + .value_and_backoff = value_and_backoff_next[backoff] + }); } static inline _Py_BackoffCounter @@ -95,10 +107,25 @@ backoff_counter_triggers(_Py_BackoffCounter counter) return counter.value_and_backoff < UNREACHABLE_BACKOFF; } -/* Initial JUMP_BACKWARD counter. - * This determines when we create a trace for a loop. */ -#define JUMP_BACKWARD_INITIAL_VALUE 4095 -#define JUMP_BACKWARD_INITIAL_BACKOFF 12 +static inline _Py_BackoffCounter +trigger_backoff_counter(void) +{ + _Py_BackoffCounter result; + result.value_and_backoff = 0; + return result; +} + +// Initial JUMP_BACKWARD counter. +// Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is +// invalidated we may construct a new trace before the bytecode has properly +// re-specialized: +// Note: this should be a prime number-1. This increases the likelihood of +// finding a "good" loop iteration to trace. +// For example, 4095 does not work for the nqueens benchmark on pyperformance +// as we always end up tracing the loop iteration's +// exhaustion iteration. Which aborts our current tracer. +#define JUMP_BACKWARD_INITIAL_VALUE 4000 +#define JUMP_BACKWARD_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) { @@ -110,8 +137,8 @@ initial_jump_backoff_counter(void) * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. */ -#define SIDE_EXIT_INITIAL_VALUE 4095 -#define SIDE_EXIT_INITIAL_BACKOFF 12 +#define SIDE_EXIT_INITIAL_VALUE 4000 +#define SIDE_EXIT_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter initial_temperature_backoff_counter(void) diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 573e10359b7..016e7a18665 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -45,12 +45,14 @@ extern "C" { #endif typedef struct { - // List of bytes objects - PyObject *list; + // Bytes writer managing output buffer + PyBytesWriter *writer; // Number of whole allocated size Py_ssize_t allocated; - // Max length of the buffer, negative number means unlimited length. + // Max length of the buffer, negative number means unlimited length Py_ssize_t max_length; + // Number of blocks of bytes. Used to calculate next allocation size + size_t num_blocks; } _BlocksOutputBuffer; static const char unable_allocate_msg[] = "Unable to allocate output buffer."; @@ -107,11 +109,10 @@ _BlocksOutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, const Py_ssize_t max_length, void **next_out) { - PyObject *b; Py_ssize_t block_size; - // ensure .list was set to NULL - assert(buffer->list == NULL); + // ensure .writer was set to NULL + assert(buffer->writer == NULL); // get block size if (0 <= max_length && max_length < BUFFER_BLOCK_SIZE[0]) { @@ -120,25 +121,17 @@ _BlocksOutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, block_size = BUFFER_BLOCK_SIZE[0]; } - // the first block - b = PyBytes_FromStringAndSize(NULL, block_size); - if (b == NULL) { + buffer->writer = PyBytesWriter_Create(block_size); + if (buffer->writer == NULL) { return -1; } - // create the list - buffer->list = PyList_New(1); - if (buffer->list == NULL) { - Py_DECREF(b); - return -1; - } - PyList_SET_ITEM(buffer->list, 0, b); - // set variables buffer->allocated = block_size; buffer->max_length = max_length; + buffer->num_blocks = 1; - *next_out = PyBytes_AS_STRING(b); + *next_out = PyBytesWriter_GetData(buffer->writer); return block_size; } @@ -155,31 +148,21 @@ _BlocksOutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, const Py_ssize_t init_size, void **next_out) { - PyObject *b; - // ensure .list was set to NULL - assert(buffer->list == NULL); + // ensure .writer was set to NULL + assert(buffer->writer == NULL); - // the first block - b = PyBytes_FromStringAndSize(NULL, init_size); - if (b == NULL) { - PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + buffer->writer = PyBytesWriter_Create(init_size); + if (buffer->writer == NULL) { return -1; } - // create the list - buffer->list = PyList_New(1); - if (buffer->list == NULL) { - Py_DECREF(b); - return -1; - } - PyList_SET_ITEM(buffer->list, 0, b); - // set variables buffer->allocated = init_size; buffer->max_length = -1; + buffer->num_blocks = 1; - *next_out = PyBytes_AS_STRING(b); + *next_out = PyBytesWriter_GetData(buffer->writer); return init_size; } @@ -193,8 +176,6 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, void **next_out, const Py_ssize_t avail_out) { - PyObject *b; - const Py_ssize_t list_len = Py_SIZE(buffer->list); Py_ssize_t block_size; // ensure no gaps in the data @@ -205,11 +186,10 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, } // get block size - if (list_len < (Py_ssize_t) Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE)) { - block_size = BUFFER_BLOCK_SIZE[list_len]; - } else { - block_size = BUFFER_BLOCK_SIZE[Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE) - 1]; - } + size_t maxblock = Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE); + assert(maxblock >= 1); + size_t block_index = Py_MIN(buffer->num_blocks, maxblock - 1); + block_size = BUFFER_BLOCK_SIZE[block_index]; // check max_length if (buffer->max_length >= 0) { @@ -229,22 +209,19 @@ _BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, return -1; } - // create the block - b = PyBytes_FromStringAndSize(NULL, block_size); - if (b == NULL) { + if (PyBytesWriter_Grow(buffer->writer, block_size)) { PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); return -1; } - if (PyList_Append(buffer->list, b) < 0) { - Py_DECREF(b); - return -1; - } - Py_DECREF(b); + + Py_ssize_t current_size = buffer->allocated; // set variables buffer->allocated += block_size; + buffer->num_blocks += 1; - *next_out = PyBytes_AS_STRING(b); + char *data = PyBytesWriter_GetData(buffer->writer); + *next_out = data + current_size; return block_size; } @@ -265,54 +242,17 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { - PyObject *result, *block; - const Py_ssize_t list_len = Py_SIZE(buffer->list); - - // fast path for single block - if ((list_len == 1 && avail_out == 0) || - (list_len == 2 && Py_SIZE(PyList_GET_ITEM(buffer->list, 1)) == avail_out)) - { - block = PyList_GET_ITEM(buffer->list, 0); - Py_INCREF(block); - - Py_CLEAR(buffer->list); - return block; - } - - // final bytes object - result = PyBytes_FromStringAndSize(NULL, buffer->allocated - avail_out); - if (result == NULL) { - PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); - return NULL; - } - - // memory copy - if (list_len > 0) { - char *posi = PyBytes_AS_STRING(result); - - // blocks except the last one - Py_ssize_t i = 0; - for (; i < list_len-1; i++) { - block = PyList_GET_ITEM(buffer->list, i); - memcpy(posi, PyBytes_AS_STRING(block), Py_SIZE(block)); - posi += Py_SIZE(block); - } - // the last block - block = PyList_GET_ITEM(buffer->list, i); - memcpy(posi, PyBytes_AS_STRING(block), Py_SIZE(block) - avail_out); - } else { - assert(Py_SIZE(result) == 0); - } - - Py_CLEAR(buffer->list); - return result; + assert(buffer->writer != NULL); + return PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); } /* Clean up the buffer when an error occurred. */ static inline void _BlocksOutputBuffer_OnError(_BlocksOutputBuffer *buffer) { - Py_CLEAR(buffer->list); + PyBytesWriter_Discard(buffer->writer); + buffer->writer = NULL; } #ifdef __cplusplus diff --git a/Include/internal/pycore_bytes_methods.h b/Include/internal/pycore_bytes_methods.h index 059dc2599bb..3e1474c1c01 100644 --- a/Include/internal/pycore_bytes_methods.h +++ b/Include/internal/pycore_bytes_methods.h @@ -47,6 +47,9 @@ extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); +/* Helper for repr(bytes) and repr(bytearray). */ +extern PyObject *_Py_bytes_repr(const char *, Py_ssize_t, int, const char *); + /* Shared __doc__ strings. */ extern const char _Py_isspace__doc__[]; extern const char _Py_isalpha__doc__[]; diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 300e7f4896a..8e8fa696ee0 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -20,8 +20,9 @@ extern PyObject* _PyBytes_FromHex( // Helper for PyBytes_DecodeEscape that detects invalid escape chars. // Export for test_peg_generator. -PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, const char **); +PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape2(const char *, Py_ssize_t, + const char *, + int *, const char **); // Substring Search. @@ -59,88 +60,46 @@ PyAPI_FUNC(void) _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); -/* --- _PyBytesWriter ----------------------------------------------------- */ +/* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation + for a bytes object of length n should request PyBytesObject_SIZE + n bytes. -/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes, bytearray or NULL (when the small buffer is used) */ - PyObject *buffer; + Using _PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves + 3 or 7 bytes per bytes object allocation on a typical system. +*/ +#define _PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) - /* Number of allocated size. */ - Py_ssize_t allocated; +/* --- PyBytesWriter ------------------------------------------------------ */ - /* Minimum number of allocated bytes, - incremented by _PyBytesWriter_Prepare() */ - Py_ssize_t min_size; - - /* If non-zero, use a bytearray instead of a bytes object for buffer. */ +struct PyBytesWriter { + char small_buffer[256]; + PyObject *obj; + Py_ssize_t size; int use_bytearray; - - /* If non-zero, overallocate the buffer (default: 0). - This flag must be zero if use_bytearray is non-zero. */ int overallocate; +}; - /* Stack buffer */ - int use_small_buffer; - char small_buffer[512]; -} _PyBytesWriter; +// Export for '_testcapi' shared extension +PyAPI_FUNC(PyBytesWriter*) _PyBytesWriter_CreateByteArray(Py_ssize_t size); -/* Initialize a bytes writer +static inline Py_ssize_t +_PyBytesWriter_GetSize(PyBytesWriter *writer) +{ + return writer->size; +} - By default, the overallocation is disabled. Set the overallocate attribute - to control the allocation of the buffer. - - Export _PyBytesWriter API for '_pickle' shared extension. */ -PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); - -/* Get the buffer content and reset the writer. - Return a bytes object, or a bytearray object if use_bytearray is non-zero. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - void *str); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, - Py_ssize_t size); - -/* Ensure that the buffer is large enough to write *size* bytes. - Add size to the writer minimum size (min_size attribute). - - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Resize the buffer to make it larger. - The new buffer may be larger than size bytes because of overallocation. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. - - Note: size must be greater than the number of allocated bytes in the writer. - - This function doesn't use the writer minimum size (min_size attribute). - - See also _PyBytesWriter_Prepare(). - */ -PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Write bytes. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - void *str, - const void *bytes, - Py_ssize_t size); +static inline char* +_PyBytesWriter_GetData(PyBytesWriter *writer) +{ + if (writer->obj == NULL) { + return writer->small_buffer; + } + else if (writer->use_bytearray) { + return PyByteArray_AS_STRING(writer->obj); + } + else { + return PyBytes_AS_STRING(writer->obj); + } +} #ifdef __cplusplus } diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 32ac3d17f22..4f4cf02f64b 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -64,39 +64,6 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *name, const char *format, ...); -extern PyObject* _PyObject_CallMethodIdObjArgs( - PyObject *obj, - _Py_Identifier *name, - ...); - -static inline PyObject * -_PyObject_VectorcallMethodId( - _Py_Identifier *name, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return _Py_NULL; - } - return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); -} - -static inline PyObject * -_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) -{ - size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); -} - -static inline PyObject * -_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) -{ - PyObject *args[2] = {self, arg}; - size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - assert(arg != NULL); - return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); -} - /* === Vectorcall protocol (PEP 590) ============================= */ @@ -186,7 +153,7 @@ _PyObject_CallNoArgs(PyObject *func) { } -extern PyObject *const * +PyAPI_FUNC(PyObject *const *) _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames); @@ -196,7 +163,7 @@ extern void _PyStack_UnpackDict_Free( Py_ssize_t nargs, PyObject *kwnames); -extern void _PyStack_UnpackDict_FreeNoDecRef( +PyAPI_FUNC(void) _PyStack_UnpackDict_FreeNoDecRef( PyObject *const *stack, PyObject *kwnames); diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 3d8247df31c..6bf33bddd5b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -22,8 +22,10 @@ struct _ceval_runtime_state; // Export for '_lsprof' shared extension PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetProfileAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg); extern int _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetTraceAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg); extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); @@ -31,8 +33,6 @@ extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); // Export for 'array' shared extension PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); -extern PyObject* _PyEval_GetBuiltinId(_Py_Identifier *); - extern void _PyEval_SetSwitchInterval(unsigned long microseconds); extern unsigned long _PyEval_GetSwitchInterval(void); @@ -121,6 +121,22 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl return tstate->interp->eval_frame(tstate, frame, throwflag); } +#ifdef _Py_TIER2 +#ifdef _Py_JIT +_Py_CODEUNIT *_Py_LazyJitTrampoline( + struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); +#else +_Py_CODEUNIT *_PyTier2Interpreter( + struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); +#endif +#endif + +extern _PyJitEntryFuncPtr _Py_jit_entry; + extern PyObject* _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, @@ -199,7 +215,14 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - return here_addr < _tstate->c_stack_soft_limit; + // Overflow if stack pointer is between soft limit and the base of the hardware stack. + // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. + // We could have the wrong stack limits because of limited platform support, or user-space threads. +#if _Py_STACK_GROWS_DOWN + return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; +#else + return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; +#endif } // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() @@ -208,7 +231,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCall( PyThreadState *tstate, const char *where); -int _Py_CheckRecursiveCallPy( +PyAPI_FUNC(int) _Py_CheckRecursiveCallPy( PyThreadState *tstate); static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, @@ -231,14 +254,33 @@ static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN return here_addr <= _tstate->c_stack_soft_limit; +#else + return here_addr >= _tstate->c_stack_soft_limit; +#endif } +// Export for test_peg_generator +PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( + PyThreadState *tstate, + int margin_count); + static inline void _Py_LeaveRecursiveCall(void) { } extern _PyInterpreterFrame* _PyEval_GetFrame(void); +extern PyObject * _PyEval_GetGlobalsFromRunningMain(PyThreadState *); +extern int _PyEval_EnsureBuiltins( + PyThreadState *, + PyObject *, + PyObject **p_builtins); +extern int _PyEval_EnsureBuiltinsWithModule( + PyThreadState *, + PyObject *, + PyObject **p_builtins); + PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); /* Handle signals, pending calls, GIL drop request @@ -273,6 +315,7 @@ PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame * PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); @@ -353,6 +396,76 @@ PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyS extern int _PyRunRemoteDebugger(PyThreadState *tstate); #endif +PyAPI_FUNC(_PyStackRef) +_PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef *index_ptr); + +/* Special methods used by LOAD_SPECIAL */ +#define SPECIAL___ENTER__ 0 +#define SPECIAL___EXIT__ 1 +#define SPECIAL___AENTER__ 2 +#define SPECIAL___AEXIT__ 3 +#define SPECIAL_MAX 3 + +PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; + +/* Helper functions for large uops */ + +PyAPI_FUNC(PyObject *) +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFast_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFastWithKeywords_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFast_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_CallBuiltinClass_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args); + +PyAPI_FUNC(void) +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index b135e30b7ad..cb9c0aa27a1 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -262,7 +262,7 @@ extern PyObject* _PyCode_GetFreevars(PyCodeObject *); extern PyObject* _PyCode_GetCode(PyCodeObject *); /** API for initializing the line number tables. */ -extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); +PyAPI_FUNC(int) _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); /** Out of process API for initializing the location table. */ extern void _PyLineTable_InitAddressRange( @@ -272,9 +272,16 @@ extern void _PyLineTable_InitAddressRange( PyCodeAddressRange *range); /** API for traversing the line number table. */ -extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); +PyAPI_FUNC(int) _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); +// Similar to PyCode_Addr2Line(), but return -1 if the code object is invalid +// and can be called without an attached tstate. Used by dump_frame() in +// Python/traceback.c. The function uses heuristics to detect freed memory, +// it's not 100% reliable. +extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr); + + /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); @@ -291,33 +298,34 @@ extern void _PyCode_Clear_Executors(PyCodeObject *code); #define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION #endif -/* Specialization functions */ +/* Specialization functions, these are exported only for other re-generated + * interpreters to call */ -extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, +PyAPI_FUNC(void) _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, _Py_CODEUNIT *instr, int load_method); -extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, +PyAPI_FUNC(void) _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name); -extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, +PyAPI_FUNC(void) _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, - int nargs); -extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _PyStackRef self_or_null, + _Py_CODEUNIT *instr, int nargs); +PyAPI_FUNC(void) _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); -extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); -extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, +PyAPI_FUNC(void) _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, +PyAPI_FUNC(void) _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); -extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); -extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); -extern void _Py_GatherStats_GetIter(_PyStackRef iterable); +PyAPI_FUNC(void) _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); +PyAPI_FUNC(void) _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable); // Utility functions for reading/writing 32/64-bit values in the inline caches. // Great care should be taken to ensure that these functions remain correct and @@ -451,6 +459,9 @@ write_location_entry_start(uint8_t *ptr, int code, int length) #define ADAPTIVE_COOLDOWN_BACKOFF 0 // Can't assert this in pycore_backoff.h because of header order dependencies +#if JUMP_BACKWARD_INITIAL_VALUE <= ADAPTIVE_COOLDOWN_VALUE +# error "JIT threshold value should be larger than adaptive cooldown value" +#endif #if SIDE_EXIT_INITIAL_VALUE <= ADAPTIVE_COOLDOWN_VALUE # error "Cold exit value should be larger than adaptive cooldown value" #endif @@ -509,7 +520,7 @@ typedef struct { #define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN) -extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); +PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); @@ -614,8 +625,58 @@ PyAPI_FUNC(int) _PyCode_SetUnboundVarCounts( PyObject *globalsns, PyObject *builtinsns); + +/* "Stateless" code is a function or code object which does not rely on + * external state or internal state. It may rely on arguments and + * builtins, but not globals or a closure. Thus it does not rely + * on __globals__ or __closure__, and a stateless function + * is equivalent to its code object. + * + * Stateless code also does not keep any persistent state + * of its own, so it can't have any executors, monitoring, + * instrumentation, or "extras" (i.e. co_extra). + * + * Stateless code may create nested functions, including closures. + * However, nested functions must themselves be stateless, except they + * *can* close on the enclosing locals. + * + * Stateless code may return any value, including nested functions and closures. + * + * Stateless code that takes no arguments and doesn't return anything + * may be treated like a script. + * + * We consider stateless code to be "portable" if it does not return + * any object that holds a reference to any of the code's locals. Thus + * generators and coroutines are not portable. Likewise a function + * that returns a closure is not portable. The concept of + * portability is useful in cases where the code is run + * in a different execution context than where + * the return value will be used. */ + +PyAPI_FUNC(int) _PyCode_CheckNoInternalState(PyCodeObject *, const char **); +PyAPI_FUNC(int) _PyCode_CheckNoExternalState( + PyCodeObject *, + _PyCode_var_counts_t *, + const char **); +PyAPI_FUNC(int) _PyCode_VerifyStateless( + PyThreadState *, + PyCodeObject *, + PyObject *globalnames, + PyObject *globalsns, + PyObject *builtinsns); + +PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **); PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); +/* Create a comparable key used to compare constants taking in account the + * object type. It is used to make sure types are not coerced (e.g., float and + * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms + * + * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) + * depending on the type and the value. The type is the first item to not + * compare bytes and str which can raise a BytesWarning exception. */ +extern PyObject* _PyCode_ConstantKey(PyObject *obj); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index aecc50be1e6..527141b54d0 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -32,7 +32,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + PyObject *module); /* AST preprocessing */ extern int _PyCompile_AstPreprocess( @@ -41,7 +42,8 @@ extern int _PyCompile_AstPreprocess( PyCompilerFlags *flags, int optimize, struct _arena *arena, - int syntax_check_only); + int syntax_check_only, + PyObject *module); extern int _PyAST_Preprocess( struct _mod *, @@ -49,7 +51,9 @@ extern int _PyAST_Preprocess( PyObject *filename, int optimize, int ff_features, - int syntax_check_only); + int syntax_check_only, + int enable_warnings, + PyObject *module); typedef struct { @@ -95,6 +99,7 @@ typedef enum { enum _PyCompile_FBlockType { COMPILE_FBLOCK_WHILE_LOOP, COMPILE_FBLOCK_FOR_LOOP, + COMPILE_FBLOCK_ASYNC_FOR_LOOP, COMPILE_FBLOCK_TRY_EXCEPT, COMPILE_FBLOCK_FINALLY_TRY, COMPILE_FBLOCK_FINALLY_END, diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 42f06b935bd..60b6fc4a72e 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -21,16 +21,6 @@ extern "C" { #define _Py_CRITICAL_SECTION_MASK 0x3 #ifdef Py_GIL_DISABLED -# define Py_BEGIN_CRITICAL_SECTION_MUT(mutex) \ - { \ - PyCriticalSection _py_cs; \ - _PyCriticalSection_BeginMutex(&_py_cs, mutex) - -# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) \ - { \ - PyCriticalSection2 _py_cs2; \ - _PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) - // Specialized version of critical section locking to safely use // PySequence_Fast APIs without the GIL. For performance, the argument *to* // PySequence_Fast() is provided to the macro, not the *result* of @@ -42,7 +32,7 @@ extern "C" { const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \ PyCriticalSection _cs; \ if (_should_lock_cs) { \ - _PyCriticalSection_Begin(&_cs, _orig_seq); \ + PyCriticalSection_Begin(&_cs, _orig_seq); \ } # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \ @@ -64,7 +54,7 @@ extern "C" { # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ if (Py_REFCNT(op) != 1) { \ - _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \ + _PyCriticalSection_AssertHeldObj(_PyObject_CAST(op)); \ } #else /* Py_DEBUG */ @@ -75,8 +65,6 @@ extern "C" { #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. -# define Py_BEGIN_CRITICAL_SECTION_MUT(mut) { -# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) { # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) { # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() } # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) @@ -89,10 +77,10 @@ _PyCriticalSection_Resume(PyThreadState *tstate); // (private) slow path for locking the mutex PyAPI_FUNC(void) -_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m); +_PyCriticalSection_BeginSlow(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m); PyAPI_FUNC(void) -_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, +_PyCriticalSection2_BeginSlow(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, int is_m1_locked); PyAPI_FUNC(void) @@ -107,33 +95,30 @@ _PyCriticalSection_IsActive(uintptr_t tag) } static inline void -_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +_PyCriticalSection_BeginMutex(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m) { if (PyMutex_LockFast(m)) { - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_mutex = m; c->_cs_prev = tstate->critical_section; tstate->critical_section = (uintptr_t)c; } else { - _PyCriticalSection_BeginSlow(c, m); + _PyCriticalSection_BeginSlow(tstate, c, m); } } static inline void -_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) +_PyCriticalSection_Begin(PyThreadState *tstate, PyCriticalSection *c, PyObject *op) { - _PyCriticalSection_BeginMutex(c, &op->ob_mutex); + _PyCriticalSection_BeginMutex(tstate, c, &op->ob_mutex); } -#define PyCriticalSection_Begin _PyCriticalSection_Begin // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. static inline void -_PyCriticalSection_Pop(PyCriticalSection *c) +_PyCriticalSection_Pop(PyThreadState *tstate, PyCriticalSection *c) { - PyThreadState *tstate = _PyThreadState_GET(); uintptr_t prev = c->_cs_prev; tstate->critical_section = prev; @@ -143,7 +128,7 @@ _PyCriticalSection_Pop(PyCriticalSection *c) } static inline void -_PyCriticalSection_End(PyCriticalSection *c) +_PyCriticalSection_End(PyThreadState *tstate, PyCriticalSection *c) { // If the mutex is NULL, we used the fast path in // _PyCriticalSection_BeginSlow for locks already held in the top-most @@ -152,18 +137,17 @@ _PyCriticalSection_End(PyCriticalSection *c) return; } PyMutex_Unlock(c->_cs_mutex); - _PyCriticalSection_Pop(c); + _PyCriticalSection_Pop(tstate, c); } -#define PyCriticalSection_End _PyCriticalSection_End static inline void -_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +_PyCriticalSection2_BeginMutex(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { if (m1 == m2) { // If the two mutex arguments are the same, treat this as a critical // section with a single mutex. c->_cs_mutex2 = NULL; - _PyCriticalSection_BeginMutex(&c->_cs_base, m1); + _PyCriticalSection_BeginMutex(tstate, &c->_cs_base, m1); return; } @@ -178,7 +162,6 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) if (PyMutex_LockFast(m1)) { if (PyMutex_LockFast(m2)) { - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_base._cs_mutex = m1; c->_cs_mutex2 = m2; c->_cs_base._cs_prev = tstate->critical_section; @@ -187,23 +170,22 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) tstate->critical_section = p; } else { - _PyCriticalSection2_BeginSlow(c, m1, m2, 1); + _PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 1); } } else { - _PyCriticalSection2_BeginSlow(c, m1, m2, 0); + _PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 0); } } static inline void -_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) +_PyCriticalSection2_Begin(PyThreadState *tstate, PyCriticalSection2 *c, PyObject *a, PyObject *b) { - _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex); + _PyCriticalSection2_BeginMutex(tstate, c, &a->ob_mutex, &b->ob_mutex); } -#define PyCriticalSection2_Begin _PyCriticalSection2_Begin static inline void -_PyCriticalSection2_End(PyCriticalSection2 *c) +_PyCriticalSection2_End(PyThreadState *tstate, PyCriticalSection2 *c) { // if mutex1 is NULL, we used the fast path in // _PyCriticalSection_BeginSlow for mutexes that are already held, @@ -217,9 +199,8 @@ _PyCriticalSection2_End(PyCriticalSection2 *c) PyMutex_Unlock(c->_cs_mutex2); } PyMutex_Unlock(c->_cs_base._cs_mutex); - _PyCriticalSection_Pop(&c->_cs_base); + _PyCriticalSection_Pop(tstate, &c->_cs_base); } -#define PyCriticalSection2_End _PyCriticalSection2_End static inline void _PyCriticalSection_AssertHeld(PyMutex *mutex) @@ -239,6 +220,67 @@ _PyCriticalSection_AssertHeld(PyMutex *mutex) #endif } +static inline void +_PyCriticalSection_AssertHeldObj(PyObject *op) +{ +#ifdef Py_DEBUG + PyMutex *mutex = &_PyObject_CAST(op)->ob_mutex; + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t prev = tstate->critical_section; + if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *cs = (PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK); + _PyObject_ASSERT_WITH_MSG(op, + (cs != NULL && (cs->_cs_base._cs_mutex == mutex || cs->_cs_mutex2 == mutex)), + "Critical section of object is not held"); + } + else { + PyCriticalSection *cs = (PyCriticalSection *)(prev & ~_Py_CRITICAL_SECTION_MASK); + _PyObject_ASSERT_WITH_MSG(op, + (cs != NULL && cs->_cs_mutex == mutex), + "Critical section of object is not held"); + } + +#endif +} + +#undef Py_BEGIN_CRITICAL_SECTION +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection_Begin(_cs_tstate, &_py_cs, _PyObject_CAST(op)) + +#undef Py_BEGIN_CRITICAL_SECTION_MUTEX +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { \ + PyCriticalSection _py_cs; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection_BeginMutex(_cs_tstate, &_py_cs, mutex) + +#undef Py_END_CRITICAL_SECTION +# define Py_END_CRITICAL_SECTION() \ + _PyCriticalSection_End(_cs_tstate, &_py_cs); \ + } + +#undef Py_BEGIN_CRITICAL_SECTION2 +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection2_Begin(_cs_tstate, &_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +#undef Py_BEGIN_CRITICAL_SECTION2_MUTEX +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection2_BeginMutex(_cs_tstate, &_py_cs2, m1, m2) + +#undef Py_END_CRITICAL_SECTION2 +# define Py_END_CRITICAL_SECTION2() \ + _PyCriticalSection2_End(_cs_tstate, &_py_cs2); \ + } + #endif /* Py_GIL_DISABLED */ #ifdef __cplusplus diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 9de61ef5412..81faffac194 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -131,7 +131,23 @@ PyAPI_FUNC(void) _PyXIData_Clear(PyInterpreterState *, _PyXIData_t *); /* getting cross-interpreter data */ -typedef int (*xidatafunc)(PyThreadState *tstate, PyObject *, _PyXIData_t *); +typedef int xidata_fallback_t; +#define _PyXIDATA_XIDATA_ONLY (0) +#define _PyXIDATA_FULL_FALLBACK (1) + +// Technically, we don't need two different function types; +// we could go with just the fallback one. However, only container +// types like tuple need it, so always having the extra arg would be +// a bit unfortunate. It's also nice to be able to clearly distinguish +// between types that might call _PyObject_GetXIData() and those that won't. +// +typedef int (*xidatafunc)(PyThreadState *, PyObject *, _PyXIData_t *); +typedef int (*xidatafbfunc)( + PyThreadState *, PyObject *, xidata_fallback_t, _PyXIData_t *); +typedef struct { + xidatafunc basic; + xidatafbfunc fallback; +} _PyXIData_getdata_t; PyAPI_FUNC(PyObject *) _PyXIData_GetNotShareableErrorType(PyThreadState *); PyAPI_FUNC(void) _PyXIData_SetNotShareableError(PyThreadState *, const char *); @@ -140,16 +156,21 @@ PyAPI_FUNC(void) _PyXIData_FormatNotShareableError( const char *, ...); -PyAPI_FUNC(xidatafunc) _PyXIData_Lookup( +PyAPI_FUNC(_PyXIData_getdata_t) _PyXIData_Lookup( PyThreadState *, PyObject *); PyAPI_FUNC(int) _PyObject_CheckXIData( PyThreadState *, PyObject *); +PyAPI_FUNC(int) _PyObject_GetXIDataNoFallback( + PyThreadState *, + PyObject *, + _PyXIData_t *); PyAPI_FUNC(int) _PyObject_GetXIData( PyThreadState *, PyObject *, + xidata_fallback_t, _PyXIData_t *); // _PyObject_GetXIData() for bytes @@ -191,6 +212,21 @@ PyAPI_FUNC(int) _PyCode_GetXIData( PyThreadState *, PyObject *, _PyXIData_t *); +PyAPI_FUNC(int) _PyCode_GetScriptXIData( + PyThreadState *, + PyObject *, + _PyXIData_t *); +PyAPI_FUNC(int) _PyCode_GetPureScriptXIData( + PyThreadState *, + PyObject *, + _PyXIData_t *); + +// _PyObject_GetXIData() for functions +PyAPI_FUNC(PyObject *) _PyFunction_FromXIData(_PyXIData_t *); +PyAPI_FUNC(int) _PyFunction_GetXIData( + PyThreadState *, + PyObject *, + _PyXIData_t *); /* using cross-interpreter data */ @@ -267,10 +303,10 @@ typedef struct _excinfo { const char *errdisplay; } _PyXI_excinfo; -PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc); +PyAPI_FUNC(_PyXI_excinfo *) _PyXI_NewExcInfo(PyObject *exc); +PyAPI_FUNC(void) _PyXI_FreeExcInfo(_PyXI_excinfo *info); PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info); PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info); -PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info); typedef enum error_code { @@ -281,42 +317,30 @@ typedef enum error_code { _PyXI_ERR_ALREADY_RUNNING = -4, _PyXI_ERR_MAIN_NS_FAILURE = -5, _PyXI_ERR_APPLY_NS_FAILURE = -6, - _PyXI_ERR_NOT_SHAREABLE = -7, + _PyXI_ERR_PRESERVE_FAILURE = -7, + _PyXI_ERR_EXC_PROPAGATION_FAILURE = -8, + _PyXI_ERR_NOT_SHAREABLE = -9, } _PyXI_errcode; +typedef struct xi_failure _PyXI_failure; -typedef struct _sharedexception { - // The originating interpreter. - PyInterpreterState *interp; - // The kind of error to propagate. - _PyXI_errcode code; - // The exception information to propagate, if applicable. - // This is populated only for some error codes, - // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. - _PyXI_excinfo uncaught; -} _PyXI_error; +PyAPI_FUNC(_PyXI_failure *) _PyXI_NewFailure(void); +PyAPI_FUNC(void) _PyXI_FreeFailure(_PyXI_failure *); +PyAPI_FUNC(_PyXI_errcode) _PyXI_GetFailureCode(_PyXI_failure *); +PyAPI_FUNC(int) _PyXI_InitFailure(_PyXI_failure *, _PyXI_errcode, PyObject *); +PyAPI_FUNC(void) _PyXI_InitFailureUTF8( + _PyXI_failure *, + _PyXI_errcode, + const char *); -PyAPI_FUNC(PyObject *) _PyXI_ApplyError(_PyXI_error *err); - - -typedef struct xi_session _PyXI_session; -typedef struct _sharedns _PyXI_namespace; - -PyAPI_FUNC(void) _PyXI_FreeNamespace(_PyXI_namespace *ns); -PyAPI_FUNC(_PyXI_namespace *) _PyXI_NamespaceFromNames(PyObject *names); -PyAPI_FUNC(int) _PyXI_FillNamespaceFromDict( - _PyXI_namespace *ns, - PyObject *nsobj, - _PyXI_session *session); -PyAPI_FUNC(int) _PyXI_ApplyNamespace( - _PyXI_namespace *ns, - PyObject *nsobj, - PyObject *dflt); +PyAPI_FUNC(int) _PyXI_UnwrapNotShareableError( + PyThreadState *, + _PyXI_failure *); // A cross-interpreter session involves entering an interpreter -// (_PyXI_Enter()), doing some work with it, and finally exiting -// that interpreter (_PyXI_Exit()). +// with _PyXI_Enter(), doing some work with it, and finally exiting +// that interpreter with _PyXI_Exit(). // // At the boundaries of the session, both entering and exiting, // data may be exchanged between the previous interpreter and the @@ -324,48 +348,40 @@ PyAPI_FUNC(int) _PyXI_ApplyNamespace( // isolation between interpreters. This includes setting objects // in the target's __main__ module on the way in, and capturing // uncaught exceptions on the way out. -struct xi_session { - // Once a session has been entered, this is the tstate that was - // current before the session. If it is different from cur_tstate - // then we must have switched interpreters. Either way, this will - // be the current tstate once we exit the session. - PyThreadState *prev_tstate; - // Once a session has been entered, this is the current tstate. - // It must be current when the session exits. - PyThreadState *init_tstate; - // This is true if init_tstate needs cleanup during exit. - int own_init_tstate; +typedef struct xi_session _PyXI_session; - // This is true if, while entering the session, init_thread took - // "ownership" of the interpreter's __main__ module. This means - // it is the only thread that is allowed to run code there. - // (Caveat: for now, users may still run exec() against the - // __main__ module's dict, though that isn't advisable.) - int running; - // This is a cached reference to the __dict__ of the entered - // interpreter's __main__ module. It is looked up when at the - // beginning of the session as a convenience. - PyObject *main_ns; +PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void); +PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *); - // This is set if the interpreter is entered and raised an exception - // that needs to be handled in some special way during exit. - _PyXI_errcode *error_override; - // This is set if exit captured an exception to propagate. - _PyXI_error *error; - - // -- pre-allocated memory -- - _PyXI_error _error; - _PyXI_errcode _error_override; -}; +typedef struct { + PyObject *preserved; + PyObject *excinfo; + _PyXI_errcode errcode; +} _PyXI_session_result; +PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *); PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session *session, PyInterpreterState *interp, - PyObject *nsupdates); -PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session); + PyObject *nsupdates, + _PyXI_session_result *); +PyAPI_FUNC(int) _PyXI_Exit( + _PyXI_session *, + _PyXI_failure *, + _PyXI_session_result *); -PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); -PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); +PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace( + _PyXI_session *, + _PyXI_failure *); + +PyAPI_FUNC(int) _PyXI_Preserve( + _PyXI_session *, + const char *, + PyObject *, + _PyXI_failure *); +PyAPI_FUNC(PyObject *) _PyXI_GetPreserved( + _PyXI_session_result *, + const char *); /*************/ diff --git a/Include/internal/pycore_crossinterp_data_registry.h b/Include/internal/pycore_crossinterp_data_registry.h index 8f4bcb948e5..fbb4cad5cac 100644 --- a/Include/internal/pycore_crossinterp_data_registry.h +++ b/Include/internal/pycore_crossinterp_data_registry.h @@ -17,7 +17,7 @@ typedef struct _xid_regitem { /* This is NULL for builtin types. */ PyObject *weakref; size_t refcount; - xidatafunc getdata; + _PyXIData_getdata_t getdata; } _PyXIData_regitem_t; typedef struct { @@ -30,7 +30,7 @@ typedef struct { PyAPI_FUNC(int) _PyXIData_RegisterClass( PyThreadState *, PyTypeObject *, - xidatafunc); + _PyXIData_getdata_t); PyAPI_FUNC(int) _PyXIData_UnregisterClass( PyThreadState *, PyTypeObject *); diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 59d2c9d5377..1cdc4449b17 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -52,9 +52,15 @@ extern "C" { #ifdef Py_GIL_DISABLED # define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) # define _Py_Debug_Free_Threaded 1 +# define _Py_Debug_code_object_co_tlbc offsetof(PyCodeObject, co_tlbc) +# define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyInterpreterFrame, tlbc_index) +# define _Py_Debug_interpreter_state_tlbc_generation offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) #else # define _Py_Debug_gilruntimestate_enabled 0 # define _Py_Debug_Free_Threaded 0 +# define _Py_Debug_code_object_co_tlbc 0 +# define _Py_Debug_interpreter_frame_tlbc_index 0 +# define _Py_Debug_interpreter_state_tlbc_generation 0 #endif @@ -85,6 +91,8 @@ typedef struct _Py_DebugOffsets { uint64_t gil_runtime_state_enabled; uint64_t gil_runtime_state_locked; uint64_t gil_runtime_state_holder; + uint64_t code_object_generation; + uint64_t tlbc_generation; } interpreter_state; // Thread state offset; @@ -94,10 +102,14 @@ typedef struct _Py_DebugOffsets { uint64_t next; uint64_t interp; uint64_t current_frame; + uint64_t base_frame; + uint64_t last_profiled_frame; uint64_t thread_id; uint64_t native_thread_id; uint64_t datastack_chunk; uint64_t status; + uint64_t holds_gil; + uint64_t gil_requested; } thread_state; // InterpreterFrame offset; @@ -109,6 +121,7 @@ typedef struct _Py_DebugOffsets { uint64_t localsplus; uint64_t owner; uint64_t stackpointer; + uint64_t tlbc_index; } interpreter_frame; // Code object offset; @@ -123,6 +136,7 @@ typedef struct _Py_DebugOffsets { uint64_t localsplusnames; uint64_t localspluskinds; uint64_t co_code_adaptive; + uint64_t co_tlbc; } code_object; // PyObject offset; @@ -200,6 +214,7 @@ typedef struct _Py_DebugOffsets { struct _gc { uint64_t size; uint64_t collecting; + uint64_t frame; } gc; // Generator object offset; @@ -210,6 +225,11 @@ typedef struct _Py_DebugOffsets { uint64_t gi_frame_state; } gen_object; + struct _llist_node { + uint64_t next; + uint64_t prev; + } llist_node; + struct _debugger_support { uint64_t eval_breaker; uint64_t remote_debugger_support; @@ -245,6 +265,8 @@ typedef struct _Py_DebugOffsets { .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + .code_object_generation = offsetof(PyInterpreterState, _code_object_generation), \ + .tlbc_generation = _Py_Debug_interpreter_state_tlbc_generation, \ }, \ .thread_state = { \ .size = sizeof(PyThreadState), \ @@ -252,10 +274,14 @@ typedef struct _Py_DebugOffsets { .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ .current_frame = offsetof(PyThreadState, current_frame), \ + .base_frame = offsetof(PyThreadState, base_frame), \ + .last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ .status = offsetof(PyThreadState, _status), \ + .holds_gil = offsetof(PyThreadState, holds_gil), \ + .gil_requested = offsetof(PyThreadState, gil_requested), \ }, \ .interpreter_frame = { \ .size = sizeof(_PyInterpreterFrame), \ @@ -265,6 +291,7 @@ typedef struct _Py_DebugOffsets { .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \ + .tlbc_index = _Py_Debug_interpreter_frame_tlbc_index, \ }, \ .code_object = { \ .size = sizeof(PyCodeObject), \ @@ -277,6 +304,7 @@ typedef struct _Py_DebugOffsets { .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + .co_tlbc = _Py_Debug_code_object_co_tlbc, \ }, \ .pyobject = { \ .size = sizeof(PyObject), \ @@ -332,6 +360,7 @@ typedef struct _Py_DebugOffsets { .gc = { \ .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ + .frame = offsetof(struct _gc_runtime_state, frame), \ }, \ .gen_object = { \ .size = sizeof(PyGenObject), \ @@ -339,13 +368,17 @@ typedef struct _Py_DebugOffsets { .gi_iframe = offsetof(PyGenObject, gi_iframe), \ .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ }, \ + .llist_node = { \ + .next = offsetof(struct llist_node, next), \ + .prev = offsetof(struct llist_node, prev), \ + }, \ .debugger_support = { \ .eval_breaker = offsetof(PyThreadState, eval_breaker), \ .remote_debugger_support = offsetof(PyThreadState, remote_debugger_support), \ .remote_debugging_enabled = offsetof(PyInterpreterState, config.remote_debug), \ .debugger_pending_call = offsetof(_PyRemoteDebuggerSupport, debugger_pending_call), \ .debugger_script_path = offsetof(_PyRemoteDebuggerSupport, debugger_script_path), \ - .debugger_script_path_size = MAX_SCRIPT_PATH_SIZE, \ + .debugger_script_path_size = _Py_MAX_SCRIPT_PATH_SIZE, \ }, \ } diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 754eb88a85c..1193f496da1 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -30,14 +30,11 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); -extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); -// "Id" variants -extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); -extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); -extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); -extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); +extern int _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, + Py_hash_t hash); + +extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); extern int _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); @@ -47,6 +44,8 @@ extern int _PyDict_HasOnlyStringKeys(PyObject *mp); // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); +extern Py_ssize_t _PyDict_SizeOf_LockHeld(PyDictObject *); + #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) /* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, @@ -113,6 +112,8 @@ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t has extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr); +extern int _PyDict_GetMethodStackRef(PyDictObject *dict, PyObject *name, _PyStackRef *method); + extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); @@ -142,7 +143,7 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *k PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); +PyAPI_FUNC(int) _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( PyDictObject *dict, @@ -150,6 +151,8 @@ extern int _PyDict_Pop_KnownHash( Py_hash_t hash, PyObject **result); +extern void _PyDict_Clear_LockHeld(PyObject *op); + #ifdef Py_GIL_DISABLED PyAPI_FUNC(void) _PyDict_EnsureSharedOnRead(PyDictObject *mp); #endif diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index f3c9a669ad3..3a41ec4b54b 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -17,15 +17,16 @@ extern "C" { static inline struct _Py_freelists * _Py_freelists_GET(void) { - PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG - _Py_EnsureTstateNotNULL(tstate); + _Py_AssertHoldsTstate(); #endif #ifdef Py_GIL_DISABLED + PyThreadState *tstate = _PyThreadState_GET(); return &((_PyThreadStateImpl*)tstate)->freelists; #else - return &tstate->interp->object_state.freelists; + PyInterpreterState *interp = _PyInterpreterState_GET(); + return &interp->object_state.freelists; #endif } diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 4828dfd948f..46e2a82ea03 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -16,6 +16,7 @@ extern "C" { # define Py_dicts_MAXFREELIST 80 # define Py_dictkeys_MAXFREELIST 80 # define Py_floats_MAXFREELIST 100 +# define Py_complexes_MAXFREELIST 100 # define Py_ints_MAXFREELIST 100 # define Py_slices_MAXFREELIST 1 # define Py_ranges_MAXFREELIST 6 @@ -26,6 +27,7 @@ extern "C" { # define Py_futureiters_MAXFREELIST 255 # define Py_object_stack_chunks_MAXFREELIST 4 # define Py_unicode_writers_MAXFREELIST 1 +# define Py_bytes_writers_MAXFREELIST 1 # define Py_pycfunctionobject_MAXFREELIST 16 # define Py_pycmethodobject_MAXFREELIST 16 # define Py_pymethodobjects_MAXFREELIST 20 @@ -43,6 +45,7 @@ struct _Py_freelist { struct _Py_freelists { struct _Py_freelist floats; + struct _Py_freelist complexes; struct _Py_freelist ints; struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; struct _Py_freelist lists; @@ -59,6 +62,7 @@ struct _Py_freelists { struct _Py_freelist futureiters; struct _Py_freelist object_stack_chunks; struct _Py_freelist unicode_writers; + struct _Py_freelist bytes_writers; struct _Py_freelist pycfunctionobject; struct _Py_freelist pycmethodobject; struct _Py_freelist pymethodobjects; diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 209252b2ddc..e89f4b5c8a4 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern PyObject* _PyFunction_Vectorcall( +PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( PyObject *func, PyObject *const *stack, size_t nargsf, @@ -35,6 +35,18 @@ PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_cod extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); + +/* See pycore_code.h for explanation about what "stateless" means. */ + +PyAPI_FUNC(int) +_PyFunction_VerifyStateless(PyThreadState *, PyObject *); + +static inline PyObject* _PyFunction_GET_BUILTINS(PyObject *func) { + return _PyFunction_CAST(func)->func_builtins; +} +#define _PyFunction_GET_BUILTINS(func) _PyFunction_GET_BUILTINS(_PyObject_CAST(func)) + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index a6519aa0863..fd284d0e4ec 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -205,6 +205,12 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #endif } +extern void _Py_ScheduleGC(PyThreadState *tstate); + +#ifndef Py_GIL_DISABLED +extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); +#endif + /* Tell the GC to track this object. * @@ -238,14 +244,19 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = &interp->gc.young.head; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + PyGC_Head *generation0 = &gcstate->young.head; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ interp->gc.visited_space; + uintptr_t not_visited = 1 ^ gcstate->visited_space; gc->_gc_next = ((uintptr_t)generation0) | not_visited; generation0->_gc_prev = (uintptr_t)gc; + gcstate->young.count++; /* number of tracked GC objects */ + gcstate->heap_size++; + if (gcstate->young.count > gcstate->young.threshold) { + _Py_TriggerGC(gcstate); + } #endif } @@ -280,6 +291,11 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + if (gcstate->young.count > 0) { + gcstate->young.count--; + } + gcstate->heap_size--; #endif } @@ -343,7 +359,6 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index c1fc3511f84..b08c8c52221 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -31,7 +31,7 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); -extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9bde4faaf5a..d23d6d4f91b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -13,7 +13,7 @@ static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); - _PyObject_Dump(obj); + PyUnstable_Object_Dump(obj); } } #endif @@ -286,6 +286,774 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 254]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 255]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 256]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 257]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 258]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 259]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 260]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 261]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 262]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 263]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 264]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 265]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 266]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 267]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 268]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 269]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 270]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 271]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 272]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 273]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 274]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 275]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 276]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 277]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 278]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 279]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 280]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 281]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 282]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 283]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 284]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 285]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 286]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 287]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 288]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 289]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 290]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 291]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 292]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 293]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 294]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 295]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 296]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 297]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 298]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 299]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 300]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 301]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 302]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 303]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 304]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 305]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 306]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 307]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 308]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 309]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 310]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 311]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 312]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 313]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 314]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 315]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 316]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 317]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 318]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 319]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 320]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 321]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 322]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 323]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 324]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 325]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 326]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 327]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 328]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 329]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 330]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 331]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 332]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 333]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 334]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 335]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 336]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 337]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 338]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 339]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 340]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 341]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 342]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 343]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 344]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 345]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 346]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 347]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 348]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 349]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 350]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 351]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 352]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 353]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 354]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 355]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 356]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 357]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 358]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 359]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 360]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 361]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 362]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 363]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 364]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 365]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 366]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 367]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 368]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 369]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 370]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 371]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 372]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 373]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 374]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 375]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 376]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 377]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 378]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 379]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 380]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 381]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 382]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 383]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 384]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 385]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 386]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 387]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 388]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 389]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 390]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 391]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 392]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 393]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 394]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 395]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 396]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 397]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 398]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 399]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 400]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 401]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 402]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 403]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 404]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 405]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 406]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 407]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 408]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 409]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 410]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 411]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 412]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 413]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 414]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 415]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 416]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 417]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 418]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 419]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 420]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 421]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 422]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 423]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 424]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 425]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 426]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 427]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 428]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 429]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 430]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 431]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 432]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 433]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 434]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 435]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 436]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 437]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 438]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 439]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 440]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 441]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 442]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 443]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 444]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 445]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 446]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 447]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 448]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 449]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 450]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 451]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 452]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 453]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 454]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 455]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 456]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 457]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 458]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 459]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 460]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 461]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 462]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 463]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 464]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 465]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 466]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 467]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 468]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 469]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 470]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 471]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 472]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 473]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 474]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 475]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 476]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 477]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 478]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 479]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 480]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 481]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 482]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 483]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 484]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 485]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 486]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 487]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 488]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 489]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 490]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 491]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 492]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 493]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 494]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 495]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 496]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 497]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 498]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 499]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 500]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 501]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 502]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 503]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 504]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 505]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 506]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 507]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 508]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 509]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 510]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 511]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 512]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 513]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 514]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 515]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 516]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 517]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 518]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 519]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 520]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 521]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 522]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 523]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 524]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 525]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 526]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 527]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 528]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 529]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 530]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 531]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 532]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 533]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 534]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 535]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 536]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 537]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 538]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 539]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 540]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 541]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 542]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 543]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 544]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 545]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 546]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 547]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 548]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 549]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 550]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 551]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 552]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 553]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 554]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 555]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 556]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 557]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 558]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 559]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 560]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 561]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 562]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 563]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 564]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 565]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 566]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 567]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 568]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 569]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 570]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 571]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 572]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 573]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 574]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 575]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 576]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 577]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 578]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 579]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 580]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 581]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 582]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 583]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 584]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 585]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 586]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 587]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 588]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 589]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 590]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 591]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 592]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 593]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 594]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 595]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 596]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 597]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 598]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 599]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 600]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 601]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 602]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 603]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 604]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 605]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 606]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 607]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 608]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 609]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 610]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 611]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 612]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 613]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 614]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 615]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 616]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 617]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 618]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 619]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 620]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 621]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 622]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 623]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 624]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 625]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 626]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 627]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 628]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 629]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 630]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 631]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 632]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 633]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 634]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 635]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 636]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 637]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 638]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 639]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 640]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 641]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 642]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 643]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 644]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 645]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 646]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 647]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 648]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 649]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 650]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 651]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 652]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 653]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 654]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 655]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 656]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 657]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 658]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 659]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 660]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 661]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 662]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 663]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 664]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 665]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 666]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 667]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 668]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 669]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 670]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 671]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 672]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 673]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 674]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 675]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 676]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 677]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 678]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 679]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 680]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 681]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 682]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 683]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 684]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 685]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 686]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 687]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 688]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 689]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 690]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 691]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 692]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 693]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 694]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 695]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 696]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 697]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 698]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 699]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 700]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 701]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 702]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 703]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 704]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 705]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 706]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 707]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 708]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 709]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 710]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 711]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 712]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 713]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 714]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 715]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 716]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 717]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 718]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 719]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 720]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 721]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 722]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 723]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 724]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 725]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 726]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 727]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 728]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 729]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 730]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 731]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 732]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 733]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 734]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 735]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 736]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 737]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 738]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 739]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 740]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 741]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 742]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 743]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 744]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 745]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 746]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 747]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 748]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 749]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 750]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 751]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 752]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 753]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 754]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 755]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 756]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 757]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 758]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 759]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 760]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 761]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 762]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 763]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 764]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 765]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 766]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 767]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 768]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 769]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 770]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 771]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 772]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 773]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 774]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 775]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 776]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 777]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 778]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 779]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 780]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 781]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 782]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 783]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 784]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 785]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 786]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 787]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 788]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 789]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 790]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 791]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 792]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 793]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 794]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 795]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 796]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 797]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 798]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 799]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 800]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 801]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 802]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 803]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 804]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 805]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 806]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 807]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 808]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 809]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 810]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 811]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 812]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 813]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 814]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 815]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 816]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 817]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 818]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 819]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 820]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 821]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 822]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 823]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 824]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 825]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 826]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 827]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 828]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 829]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 830]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 831]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 832]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 833]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 834]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 835]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 836]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 837]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 838]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 839]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 840]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 841]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 842]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 843]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 844]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 845]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 846]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 847]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 848]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 849]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 850]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 851]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 852]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 853]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 854]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 855]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 856]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 857]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 858]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 859]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 860]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 861]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 862]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 863]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 864]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 865]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 866]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 867]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 868]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 869]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 870]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 871]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 872]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 873]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 874]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 875]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 876]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 877]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 878]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 879]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 880]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 881]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 882]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 883]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 884]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 885]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 886]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 887]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 888]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 889]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 890]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 891]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 892]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 893]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 894]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 895]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 896]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 897]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 898]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 899]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 900]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 901]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 902]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 903]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 904]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 905]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 906]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 907]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 908]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 909]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 910]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 911]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 912]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 913]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 914]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 915]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 916]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 917]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 918]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 919]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 920]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 921]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 922]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 923]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 924]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 925]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 926]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 927]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 928]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 929]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 930]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 931]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 932]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 933]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 934]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 935]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 936]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 937]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 938]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 939]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 940]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 941]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 942]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 943]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 944]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 945]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 946]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 947]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 948]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 949]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 950]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 951]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 952]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 953]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 954]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 955]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 956]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 957]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 958]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 959]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 960]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 961]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 962]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 963]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 964]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 965]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 966]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 967]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 968]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 969]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 970]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 971]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 972]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 973]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 974]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 975]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 976]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 977]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 978]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 979]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 980]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 981]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 982]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 983]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 984]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 985]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 986]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 987]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 988]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 989]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 990]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 991]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 992]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 993]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 994]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 995]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 996]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 997]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 998]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 999]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1000]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1001]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1002]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1003]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1004]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1005]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1006]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1007]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1008]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1009]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1010]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1011]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1012]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1013]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1014]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1015]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1016]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1017]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1018]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1019]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1020]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1021]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1022]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1023]); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + 1024]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[0]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[1]); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(bytes_characters)[2]); @@ -558,14 +1326,18 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Emax)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Emin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(False)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(JSONDecodeError)); @@ -792,10 +1564,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add_done_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aggregate_class)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); @@ -809,7 +1581,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ast)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(athrow)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(authorizer_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(autocommit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(backtick)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(base)); @@ -819,6 +1590,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(block)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(blocking)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bound)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffer_callback)); @@ -836,6 +1608,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_stack)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cache_frames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); @@ -843,13 +1617,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_exception_handler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(call_soon)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capitals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cb_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(certfile)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(chain)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(check_same_thread)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(clamp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(clear)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(close)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(closed)); @@ -878,6 +1656,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(config)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(contravariant)); @@ -888,11 +1667,15 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(coro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(count)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(covariant)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ctx)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cwd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(date)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(days)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(debug)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); @@ -904,8 +1687,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict_content)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dictcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest_size)); @@ -923,6 +1704,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(emptyerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end)); @@ -935,17 +1717,20 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(expression)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fallback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(family)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fanout)); @@ -972,15 +1757,17 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format_spec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(frame_buffer)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(free_threaded)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(from_param)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromtimestamp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fromutc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fullerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_event_loop)); @@ -1001,13 +1788,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hours)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(imag)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(implieslink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(in_fd)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(include_aliases)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(incoming)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(index)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup)); @@ -1054,9 +1844,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwargs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_exc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_node)); @@ -1065,6 +1855,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(latin1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(leaf_size)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(legacy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(len)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(level)); @@ -1072,7 +1863,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(line_buffering)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lineno)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(listcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(little)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(locale)); @@ -1081,12 +1871,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxmem)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxsplit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxvalue)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(memLevel)); @@ -1096,19 +1888,21 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minutes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module_globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modules)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(modulo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mutex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_arg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_sequence_fields)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_unnamed_fields)); @@ -1116,7 +1910,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(name_from)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespace_separator)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(narg)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); @@ -1139,6 +1933,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(offset_src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(on_type_read)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(onceregistry)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_active_thread)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_keys)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(oparg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcode)); @@ -1149,6 +1944,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(options)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(order)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(origin)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(other)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(out_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outpath)); @@ -1166,21 +1962,26 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(person)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pi_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pointer_bits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(policy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(posix)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(prec)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(preserve_exc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_handler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_routine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(proto)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(protocol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps2)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(qid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(qualname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(query)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(queuetype)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(quotetabs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(raw)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read)); @@ -1197,19 +1998,25 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rel_tol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(release)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reload)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repeat)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reqrefs)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(require_ready)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reserved)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(resetids)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(restrict)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reverse)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rounding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(security_attributes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); @@ -1221,18 +2028,21 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(server_hostname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(server_side)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(session)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setpgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setsid)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setsigdef)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setsigmask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setstate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(shape)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(shared)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(short)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(show_cmd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(signed)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(signum)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sizehint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(skip_file_prefixes)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(skip_non_matching_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sleep)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sock)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sort)); @@ -1244,6 +2054,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(statement)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stats)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(status)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stderr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdin)); @@ -1261,6 +2072,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(symmetric_difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tabsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tag)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(take_bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target_is_directory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(task)); @@ -1272,18 +2084,22 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(template)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(term)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(text)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(third)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(threading)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(throw)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(time)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeout)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(times)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timespec)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeunit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trace_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traceback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(trailers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(translate)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(traps)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(true)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(truncate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(twice)); @@ -1294,8 +2110,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzinfo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tzname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unboundop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unlink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(unraisablehook)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(updates)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uri)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(usedforsecurity)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); @@ -1309,6 +2127,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(week)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(weekday)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(weeks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(which)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(who)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(withdata)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3a83fd6b604..5c3ea474ad0 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -46,10 +46,12 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") STRUCT_FOR_STR(format, ".format") + STRUCT_FOR_STR(gc, "") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") @@ -57,6 +59,8 @@ struct _Py_global_strings { struct { STRUCT_FOR_ID(CANCELLED) + STRUCT_FOR_ID(Emax) + STRUCT_FOR_ID(Emin) STRUCT_FOR_ID(FINISHED) STRUCT_FOR_ID(False) STRUCT_FOR_ID(JSONDecodeError) @@ -283,10 +287,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(add_done_callback) STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) - STRUCT_FOR_ID(aggregate_class) STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) @@ -300,7 +304,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(ast) STRUCT_FOR_ID(athrow) STRUCT_FOR_ID(attribute) - STRUCT_FOR_ID(authorizer_callback) STRUCT_FOR_ID(autocommit) STRUCT_FOR_ID(backtick) STRUCT_FOR_ID(base) @@ -310,6 +313,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(bit_offset) STRUCT_FOR_ID(bit_size) STRUCT_FOR_ID(block) + STRUCT_FOR_ID(blocking) STRUCT_FOR_ID(bound) STRUCT_FOR_ID(buffer) STRUCT_FOR_ID(buffer_callback) @@ -327,6 +331,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_parameter_type) STRUCT_FOR_ID(c_return) + STRUCT_FOR_ID(c_stack) + STRUCT_FOR_ID(cache_frames) STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) @@ -334,13 +340,17 @@ struct _Py_global_strings { STRUCT_FOR_ID(call) STRUCT_FOR_ID(call_exception_handler) STRUCT_FOR_ID(call_soon) + STRUCT_FOR_ID(callable) STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) STRUCT_FOR_ID(capath) + STRUCT_FOR_ID(capitals) STRUCT_FOR_ID(category) STRUCT_FOR_ID(cb_type) STRUCT_FOR_ID(certfile) + STRUCT_FOR_ID(chain) STRUCT_FOR_ID(check_same_thread) + STRUCT_FOR_ID(clamp) STRUCT_FOR_ID(clear) STRUCT_FOR_ID(close) STRUCT_FOR_ID(closed) @@ -369,6 +379,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) + STRUCT_FOR_ID(config) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) STRUCT_FOR_ID(contravariant) @@ -379,11 +390,15 @@ struct _Py_global_strings { STRUCT_FOR_ID(coro) STRUCT_FOR_ID(count) STRUCT_FOR_ID(covariant) + STRUCT_FOR_ID(ctx) STRUCT_FOR_ID(cwd) STRUCT_FOR_ID(d_parameter_type) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) + STRUCT_FOR_ID(date) STRUCT_FOR_ID(day) + STRUCT_FOR_ID(days) + STRUCT_FOR_ID(debug) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) STRUCT_FOR_ID(default) @@ -395,8 +410,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(device) STRUCT_FOR_ID(dict) - STRUCT_FOR_ID(dict_content) - STRUCT_FOR_ID(dictcomp) STRUCT_FOR_ID(difference_update) STRUCT_FOR_ID(digest) STRUCT_FOR_ID(digest_size) @@ -414,6 +427,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) + STRUCT_FOR_ID(emptyerror) STRUCT_FOR_ID(encode) STRUCT_FOR_ID(encoding) STRUCT_FOR_ID(end) @@ -426,17 +440,20 @@ struct _Py_global_strings { STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) + STRUCT_FOR_ID(exc) STRUCT_FOR_ID(exc_type) STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) STRUCT_FOR_ID(existing_file_name) + STRUCT_FOR_ID(exit) STRUCT_FOR_ID(exp) STRUCT_FOR_ID(expression) STRUCT_FOR_ID(extend) STRUCT_FOR_ID(extra_tokens) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) + STRUCT_FOR_ID(fallback) STRUCT_FOR_ID(false) STRUCT_FOR_ID(family) STRUCT_FOR_ID(fanout) @@ -463,15 +480,17 @@ struct _Py_global_strings { STRUCT_FOR_ID(format) STRUCT_FOR_ID(format_spec) STRUCT_FOR_ID(frame_buffer) + STRUCT_FOR_ID(free_threaded) STRUCT_FOR_ID(from_param) STRUCT_FOR_ID(fromlist) STRUCT_FOR_ID(fromtimestamp) STRUCT_FOR_ID(fromutc) STRUCT_FOR_ID(fset) + STRUCT_FOR_ID(fullerror) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(gc) STRUCT_FOR_ID(generation) - STRUCT_FOR_ID(genexpr) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) STRUCT_FOR_ID(get_event_loop) @@ -492,13 +511,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) + STRUCT_FOR_ID(hours) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) STRUCT_FOR_ID(imag) + STRUCT_FOR_ID(implieslink) STRUCT_FOR_ID(importlib) STRUCT_FOR_ID(in_fd) + STRUCT_FOR_ID(include_aliases) STRUCT_FOR_ID(incoming) STRUCT_FOR_ID(index) STRUCT_FOR_ID(indexgroup) @@ -545,9 +567,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw) STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) + STRUCT_FOR_ID(kwargs) STRUCT_FOR_ID(kwdefaults) STRUCT_FOR_ID(label) - STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_exc) STRUCT_FOR_ID(last_node) @@ -556,6 +578,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(last_value) STRUCT_FOR_ID(latin1) STRUCT_FOR_ID(leaf_size) + STRUCT_FOR_ID(legacy) STRUCT_FOR_ID(len) STRUCT_FOR_ID(length) STRUCT_FOR_ID(level) @@ -563,7 +586,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(line) STRUCT_FOR_ID(line_buffering) STRUCT_FOR_ID(lineno) - STRUCT_FOR_ID(listcomp) STRUCT_FOR_ID(little) STRUCT_FOR_ID(lo) STRUCT_FOR_ID(locale) @@ -572,12 +594,14 @@ struct _Py_global_strings { STRUCT_FOR_ID(loop) STRUCT_FOR_ID(manual_reset) STRUCT_FOR_ID(mapping) + STRUCT_FOR_ID(mask) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) STRUCT_FOR_ID(maxlen) STRUCT_FOR_ID(maxmem) + STRUCT_FOR_ID(maxsize) STRUCT_FOR_ID(maxsplit) STRUCT_FOR_ID(maxvalue) STRUCT_FOR_ID(memLevel) @@ -587,19 +611,21 @@ struct _Py_global_strings { STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(microseconds) STRUCT_FOR_ID(milliseconds) STRUCT_FOR_ID(minute) + STRUCT_FOR_ID(minutes) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) STRUCT_FOR_ID(module_globals) STRUCT_FOR_ID(modules) + STRUCT_FOR_ID(modulo) STRUCT_FOR_ID(month) STRUCT_FOR_ID(mro) STRUCT_FOR_ID(msg) STRUCT_FOR_ID(mutex) STRUCT_FOR_ID(mycmp) - STRUCT_FOR_ID(n_arg) STRUCT_FOR_ID(n_fields) STRUCT_FOR_ID(n_sequence_fields) STRUCT_FOR_ID(n_unnamed_fields) @@ -607,7 +633,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(name_from) STRUCT_FOR_ID(namespace_separator) STRUCT_FOR_ID(namespaces) - STRUCT_FOR_ID(narg) + STRUCT_FOR_ID(native) STRUCT_FOR_ID(ndigits) STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) @@ -630,6 +656,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(offset_src) STRUCT_FOR_ID(on_type_read) STRUCT_FOR_ID(onceregistry) + STRUCT_FOR_ID(only_active_thread) STRUCT_FOR_ID(only_keys) STRUCT_FOR_ID(oparg) STRUCT_FOR_ID(opcode) @@ -640,6 +667,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(options) STRUCT_FOR_ID(order) STRUCT_FOR_ID(origin) + STRUCT_FOR_ID(other) STRUCT_FOR_ID(out_fd) STRUCT_FOR_ID(outgoing) STRUCT_FOR_ID(outpath) @@ -657,21 +685,26 @@ struct _Py_global_strings { STRUCT_FOR_ID(person) STRUCT_FOR_ID(pi_factory) STRUCT_FOR_ID(pid) + STRUCT_FOR_ID(pointer_bits) STRUCT_FOR_ID(policy) STRUCT_FOR_ID(pos) STRUCT_FOR_ID(pos1) STRUCT_FOR_ID(pos2) STRUCT_FOR_ID(posix) + STRUCT_FOR_ID(prec) + STRUCT_FOR_ID(preserve_exc) STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) - STRUCT_FOR_ID(progress_handler) STRUCT_FOR_ID(progress_routine) STRUCT_FOR_ID(proto) STRUCT_FOR_ID(protocol) STRUCT_FOR_ID(ps1) STRUCT_FOR_ID(ps2) + STRUCT_FOR_ID(qid) + STRUCT_FOR_ID(qualname) STRUCT_FOR_ID(query) + STRUCT_FOR_ID(queuetype) STRUCT_FOR_ID(quotetabs) STRUCT_FOR_ID(raw) STRUCT_FOR_ID(read) @@ -688,19 +721,25 @@ struct _Py_global_strings { STRUCT_FOR_ID(rel_tol) STRUCT_FOR_ID(release) STRUCT_FOR_ID(reload) + STRUCT_FOR_ID(repeat) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) + STRUCT_FOR_ID(reqrefs) + STRUCT_FOR_ID(require_ready) STRUCT_FOR_ID(reserved) STRUCT_FOR_ID(reset) STRUCT_FOR_ID(resetids) + STRUCT_FOR_ID(restrict) STRUCT_FOR_ID(return) STRUCT_FOR_ID(reverse) STRUCT_FOR_ID(reversed) + STRUCT_FOR_ID(rounding) STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(script) STRUCT_FOR_ID(second) + STRUCT_FOR_ID(seconds) STRUCT_FOR_ID(security_attributes) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) @@ -712,18 +751,21 @@ struct _Py_global_strings { STRUCT_FOR_ID(server_hostname) STRUCT_FOR_ID(server_side) STRUCT_FOR_ID(session) - STRUCT_FOR_ID(setcomp) STRUCT_FOR_ID(setpgroup) STRUCT_FOR_ID(setsid) STRUCT_FOR_ID(setsigdef) STRUCT_FOR_ID(setsigmask) STRUCT_FOR_ID(setstate) STRUCT_FOR_ID(shape) + STRUCT_FOR_ID(shared) + STRUCT_FOR_ID(short) STRUCT_FOR_ID(show_cmd) STRUCT_FOR_ID(signed) + STRUCT_FOR_ID(signum) STRUCT_FOR_ID(size) STRUCT_FOR_ID(sizehint) STRUCT_FOR_ID(skip_file_prefixes) + STRUCT_FOR_ID(skip_non_matching_threads) STRUCT_FOR_ID(sleep) STRUCT_FOR_ID(sock) STRUCT_FOR_ID(sort) @@ -735,6 +777,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(stacklevel) STRUCT_FOR_ID(start) STRUCT_FOR_ID(statement) + STRUCT_FOR_ID(stats) STRUCT_FOR_ID(status) STRUCT_FOR_ID(stderr) STRUCT_FOR_ID(stdin) @@ -752,6 +795,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(symmetric_difference_update) STRUCT_FOR_ID(tabsize) STRUCT_FOR_ID(tag) + STRUCT_FOR_ID(take_bytes) STRUCT_FOR_ID(target) STRUCT_FOR_ID(target_is_directory) STRUCT_FOR_ID(task) @@ -763,18 +807,22 @@ struct _Py_global_strings { STRUCT_FOR_ID(template) STRUCT_FOR_ID(term) STRUCT_FOR_ID(text) + STRUCT_FOR_ID(third) STRUCT_FOR_ID(threading) STRUCT_FOR_ID(throw) + STRUCT_FOR_ID(time) STRUCT_FOR_ID(timeout) STRUCT_FOR_ID(timer) STRUCT_FOR_ID(times) + STRUCT_FOR_ID(timespec) + STRUCT_FOR_ID(timestamp) STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(timeunit) STRUCT_FOR_ID(top) - STRUCT_FOR_ID(trace_callback) STRUCT_FOR_ID(traceback) STRUCT_FOR_ID(trailers) STRUCT_FOR_ID(translate) + STRUCT_FOR_ID(traps) STRUCT_FOR_ID(true) STRUCT_FOR_ID(truncate) STRUCT_FOR_ID(twice) @@ -785,8 +833,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(tzinfo) STRUCT_FOR_ID(tzname) STRUCT_FOR_ID(uid) + STRUCT_FOR_ID(unboundop) STRUCT_FOR_ID(unlink) STRUCT_FOR_ID(unraisablehook) + STRUCT_FOR_ID(updates) STRUCT_FOR_ID(uri) STRUCT_FOR_ID(usedforsecurity) STRUCT_FOR_ID(value) @@ -800,6 +850,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(wbits) STRUCT_FOR_ID(week) STRUCT_FOR_ID(weekday) + STRUCT_FOR_ID(weeks) STRUCT_FOR_ID(which) STRUCT_FOR_ID(who) STRUCT_FOR_ID(withdata) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 13fbff4eb65..4c8b8c0ed86 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -17,7 +17,8 @@ extern int _PyImport_IsInitialized(PyInterpreterState *); // Export for 'pyexpat' shared extension PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); -extern int _PyImport_SetModuleString(const char *name, PyObject* module); +// Export for 'math' shared extension +PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); extern void _PyImport_AcquireLock(PyInterpreterState *interp); extern void _PyImport_ReleaseLock(PyInterpreterState *interp); @@ -127,11 +128,18 @@ PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); // state of the module argument: // - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, // call _PyEval_DisableGIL(). -// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already -// enabled permanently, issue a warning referencing the module's name. +// - Otherwise, call _PyImport_EnableGILAndWarn // // This function may raise an exception. extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), call _PyEval_EnableGILPermanent(). +// If the GIL was not already enabled permanently, issue a warning referencing +// the module's name. +// Leave a message in verbose mode. +// +// This function may raise an exception. +extern int _PyImport_EnableGILAndWarn(PyThreadState *, PyObject *module_name); #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index 525a16f6b97..f60c5510d20 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -14,6 +14,34 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; +#ifdef HAVE_DYNAMIC_LOADING +/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is + supported on this platform. configure will then compile and link in one + of the dynload_*.c files, as appropriate. We will call a function in + those modules to get a function pointer to the module's init function. + + The function should return: + - The function pointer on success + - NULL with exception set if the library cannot be loaded + - NULL *without* an extension set if the library could be loaded but the + function cannot be found in it. +*/ +#ifdef MS_WINDOWS +#include +typedef FARPROC dl_funcptr; +extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, + const char *shortname, + PyObject *pathname, + FILE *fp); +#else +typedef void (*dl_funcptr)(void); +extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, + const char *shortname, + const char *pathname, FILE *fp); +#endif + +#endif /* HAVE_DYNAMIC_LOADING */ + typedef enum ext_module_kind { _Py_ext_module_kind_UNKNOWN = 0, @@ -28,6 +56,11 @@ typedef enum ext_module_origin { _Py_ext_module_origin_DYNAMIC = 3, } _Py_ext_module_origin; +struct hook_prefixes { + const char *const init_prefix; + const char *const export_prefix; +}; + /* Input for loading an extension module. */ struct _Py_ext_module_loader_info { PyObject *filename; @@ -40,7 +73,7 @@ struct _Py_ext_module_loader_info { * depending on if it's builtin or not. */ PyObject *path; _Py_ext_module_origin origin; - const char *hook_prefix; + const struct hook_prefixes *hook_prefixes; const char *newcontext; }; extern void _Py_ext_module_loader_info_clear( @@ -62,7 +95,9 @@ extern int _Py_ext_module_loader_info_init_from_spec( PyObject *spec); #endif -/* The result from running an extension module's init function. */ +/* The result from running an extension module's init function. + * Not used for modules defined via PyModExport (slots array). + */ struct _Py_ext_module_loader_result { PyModuleDef *def; PyObject *module; @@ -89,10 +124,11 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); +typedef PyModuleDef_Slot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING -extern PyModInitFunction _PyImport_GetModInitFunc( +extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, - FILE *fp); + FILE *fp, PyModInitFunction *modinit, PyModExportFunction *modexport); #endif extern int _PyImport_RunModInitFunc( PyModInitFunction p0, @@ -104,10 +140,8 @@ extern int _PyImport_RunModInitFunc( #define MAXSUFFIXSIZE 12 #ifdef MS_WINDOWS -#include -typedef FARPROC dl_funcptr; -#ifdef _DEBUG +#ifdef Py_DEBUG # define PYD_DEBUG_SUFFIX "_d" #else # define PYD_DEBUG_SUFFIX "" @@ -128,8 +162,6 @@ typedef FARPROC dl_funcptr; #define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" -#else -typedef void (*dl_funcptr)(void); #endif diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 368dafb9063..183b2d45c5e 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,10 +153,8 @@ typedef enum { } _PyConfigInitEnum; typedef enum { - /* For now, this means the GIL is enabled. - - gh-116329: This will eventually change to "the GIL is disabled but can - be re-enabled by loading an incompatible extension module." */ + /* In free threaded builds, this means that the GIL is disabled at startup, + but may be enabled by loading an incompatible extension module. */ _PyConfig_GIL_DEFAULT = -1, /* The GIL has been forced off or on, and will not be affected by module loading. */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 7658adca719..ebc5622912f 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -33,32 +33,34 @@ int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events); int _PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events); -extern int + +// these are exported only for other re-generated interpreters to call +PyAPI_FUNC(int) _Py_call_instrumentation(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_instruction( PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); -_Py_CODEUNIT * +PyAPI_FUNC(_Py_CODEUNIT *) _Py_call_instrumentation_jump( _Py_CODEUNIT *instr, PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_arg(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); -extern int +PyAPI_FUNC(int) _Py_call_instrumentation_2args(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern void +PyAPI_FUNC(void) _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index c3e6c77405b..6b3d5711b92 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -15,7 +15,6 @@ extern "C" { #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState - #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 #define FUNC_MAX_WATCHERS 8 @@ -73,6 +72,7 @@ struct trampoline_api_st { int (*free_state)(void* state); void *state; Py_ssize_t code_padding; + Py_ssize_t code_alignment; }; #endif @@ -87,6 +87,7 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; + _PyFrameEvalFunction prev_eval_frame; #else int _not_used; #endif @@ -97,7 +98,6 @@ struct _ceval_runtime_state { // For example, we use a preallocated array // for the list of pending calls. struct _pending_calls pending_mainthread; - PyMutex sys_trace_profile_mutex; }; @@ -129,8 +129,6 @@ struct _atexit_runtime_state { //################### // interpreter atexit -typedef void (*atexit_datacallbackfunc)(void *); - typedef struct atexit_callback { atexit_datacallbackfunc func; void *data; @@ -159,10 +157,11 @@ struct atexit_state { typedef struct { // Tagged pointer to next object in the list. // 0 means the object is not tracked - uintptr_t _gc_next; + _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, uintptr_t) _gc_next; // Tagged pointer to previous object in the list. // Lowest two bits are used for flags documented later. + // Those bits are made available by the struct's minimum alignment. uintptr_t _gc_prev; } PyGC_Head; @@ -180,6 +179,10 @@ struct gc_collection_stats { Py_ssize_t collected; /* total number of uncollectable objects (put into gc.garbage) */ Py_ssize_t uncollectable; + // Total number of objects considered for collection and traversed: + Py_ssize_t candidates; + // Duration of the collection in seconds: + double duration; }; /* Running stats per generation */ @@ -190,6 +193,10 @@ struct gc_generation_stats { Py_ssize_t collected; /* total number of uncollectable objects (put into gc.garbage) */ Py_ssize_t uncollectable; + // Total number of objects considered for collection and traversed: + Py_ssize_t candidates; + // Duration of the collection in seconds: + double duration; }; enum _GCPhase { @@ -198,16 +205,10 @@ enum _GCPhase { }; /* If we change this, we need to change the default value in the - signature of gc.collect. */ + signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 struct _gc_runtime_state { - /* List of objects that still need to be cleaned up, singly linked - * via their gc headers' gc_prev pointers. */ - PyObject *trash_delete_later; - /* Current call-stack depth of tp_dealloc calls. */ - int trash_delete_nesting; - /* Is automatic collection enabled? */ int enabled; int debug; @@ -219,6 +220,9 @@ struct _gc_runtime_state { struct gc_generation_stats generation_stats[NUM_GENERATIONS]; /* true if we are currently running the collector */ int collecting; + // The frame that started the current collection. It might be NULL even when + // collecting (if no Python frame is running): + _PyInterpreterFrame *frame; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ @@ -677,8 +681,6 @@ struct _Py_interp_cached_objects { /* object.__reduce__ */ PyObject *objreduce; - PyObject *type_slots_pname; - pytype_slotdef *type_slots_ptrs[MAX_EQUIV]; /* TypeVar and related types */ PyTypeObject *generic_type; @@ -688,6 +690,13 @@ struct _Py_interp_cached_objects { PyTypeObject *paramspecargs_type; PyTypeObject *paramspeckwargs_type; PyTypeObject *constevaluator_type; + + /* Descriptors for __dict__ and __weakref__ */ +#ifdef Py_GIL_DISABLED + PyMutex descriptor_mutex; +#endif + PyObject *dict_descriptor; + PyObject *weakref_descriptor; }; struct _Py_interp_static_objects { @@ -726,6 +735,10 @@ typedef struct _PyIndexPool { // Next index to allocate if no free indices are available int32_t next_index; + + // Generation counter incremented on thread creation/destruction + // Used for TLBC cache invalidation in remote debugging + uint32_t tlbc_generation; } _PyIndexPool; typedef union _Py_unique_id_entry { @@ -751,6 +764,7 @@ struct _Py_unique_id_pool { #endif +typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -764,12 +778,6 @@ struct _is { * and should be placed at the beginning. */ struct _ceval_state ceval; - /* This structure is carefully allocated so that it's correctly aligned - * to avoid undefined behaviors during LOAD and STORE. The '_malloced' - * field stores the allocated pointer address that will later be freed. - */ - void *_malloced; - PyInterpreterState *next; int64_t id; @@ -843,6 +851,8 @@ struct _is { /* The per-interpreter GIL, which might not be used. */ struct _gil_runtime_state _gil; + uint64_t _code_object_generation; + /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as possible to facilitate out-of-process observability @@ -932,16 +942,19 @@ struct _is { struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; bool jit; + bool compiling; struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; + struct _PyExecutorObject *cold_executor; + struct _PyExecutorObject *cold_dynamic_executor; int executor_deletion_list_remaining_capacity; - size_t trace_run_counter; + size_t executor_creation_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; _Py_GlobalMonitors monitors; - bool sys_profile_initialized; - bool sys_trace_initialized; + _PyOnceFlag sys_profile_once_flag; + _PyOnceFlag sys_trace_once_flag; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; @@ -959,6 +972,18 @@ struct _is { # ifdef Py_STACKREF_CLOSE_DEBUG _Py_hashtable_t *closed_stackrefs_table; # endif +#endif + +#ifdef Py_STATS + // true if recording of pystats is on, this is used when new threads + // are created to decide if recording should be on for them + int pystats_enabled; + // allocated when (and if) stats are first enabled + PyStats *pystats_struct; +#ifdef Py_GIL_DISABLED + // held when pystats related interpreter state is being updated + PyMutex pystats_mutex; +#endif #endif /* the initial PyInterpreterState.threads.head */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index d3fd218b27e..8949d6cc2fc 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -24,6 +24,36 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +// Similar to _PyFrame_GetCode(), but return NULL if the frame is invalid or +// freed. Used by dump_frame() in Python/traceback.c. The function uses +// heuristics to detect freed memory, it's not 100% reliable. +static inline PyCodeObject* +_PyFrame_SafeGetCode(_PyInterpreterFrame *f) +{ + // globals and builtins may be NULL on a legit frame, but it's unlikely. + // It's more likely that it's a sign of an invalid frame. + if (f->f_globals == NULL || f->f_builtins == NULL) { + return NULL; + } + + if (PyStackRef_IsNull(f->f_executable)) { + return NULL; + } + void *ptr; + memcpy(&ptr, &f->f_executable, sizeof(f->f_executable)); + if (_PyMem_IsPtrFreed(ptr)) { + return NULL; + } + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + if (_PyObject_IsFreed(executable)) { + return NULL; + } + if (!PyCode_Check(executable)) { + return NULL; + } + return (PyCodeObject *)executable; +} + static inline _Py_CODEUNIT * _PyFrame_GetBytecode(_PyInterpreterFrame *f) { @@ -37,6 +67,31 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) #endif } +// Similar to PyUnstable_InterpreterFrame_GetLasti(), but return NULL if the +// frame is invalid or freed. Used by dump_frame() in Python/traceback.c. The +// function uses heuristics to detect freed memory, it's not 100% reliable. +static inline int +_PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) +{ + // Code based on _PyFrame_GetBytecode() but replace _PyFrame_GetCode() + // with _PyFrame_SafeGetCode(). + PyCodeObject *co = _PyFrame_SafeGetCode(f); + if (co == NULL) { + return -1; + } + + _Py_CODEUNIT *bytecode; +#ifdef Py_GIL_DISABLED + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); + assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); + bytecode = (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; +#else + bytecode = _PyCode_CODE(co); +#endif + + return (int)(f->instr_ptr - bytecode) * sizeof(_Py_CODEUNIT); +} + static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); assert(PyFunction_Check(func)); @@ -48,13 +103,13 @@ static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { } static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { - assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + assert(f->stackpointer > _PyFrame_Stackbase(f)); assert(!PyStackRef_IsNull(f->stackpointer[-1])); return f->stackpointer[-1]; } static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { - assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + assert(f->stackpointer > _PyFrame_Stackbase(f)); f->stackpointer--; return *f->stackpointer; } diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 835b8e58194..38510685f40 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -24,7 +24,6 @@ enum _frameowner { FRAME_OWNED_BY_GENERATOR = 1, FRAME_OWNED_BY_FRAME_OBJECT = 2, FRAME_OWNED_BY_INTERPRETER = 3, - FRAME_OWNED_BY_CSTACK = 4, }; struct _PyInterpreterFrame { diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 8a88cbf607b..b1550a6ddcf 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -13,6 +13,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +/* To be able to reason about code layout and branches, keep code size below 1 MB */ +#define PY_MAX_JIT_CODE_SIZE ((1 << 20)-1) + #ifdef _Py_JIT typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 7484b05d7f2..c4e007e744c 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -13,7 +13,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -//_Py_UNLOCKED is defined as 0 and _Py_LOCKED as 1 in Include/cpython/lock.h +//_Py_UNLOCKED is defined as 0 and _Py_LOCKED as 1 in Include/cpython/pylock.h #define _Py_HAS_PARKED 2 #define _Py_ONCE_INITIALIZED 4 @@ -25,13 +25,6 @@ PyMutex_LockFast(PyMutex *m) return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); } -// Checks if the mutex is currently locked. -static inline int -PyMutex_IsLocked(PyMutex *m) -{ - return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; -} - // Re-initializes the mutex after a fork to the unlocked state. static inline void _PyMutex_at_fork_reinit(PyMutex *m) @@ -48,6 +41,14 @@ typedef enum _PyLockFlags { // Handle signals if interrupted while waiting on the lock. _PY_LOCK_HANDLE_SIGNALS = 2, + + // Fail if interrupted by a signal while waiting on the lock. + _PY_FAIL_IF_INTERRUPTED = 4, + + // Locking & unlocking this lock requires attached thread state. + // If locking returns PY_LOCK_FAILURE, a Python exception *may* be raised. + // (Intended for use with _PY_LOCK_HANDLE_SIGNALS and _PY_LOCK_DETACH.) + _PY_LOCK_PYTHONLOCK = 8, } _PyLockFlags; // Lock a mutex with an optional timeout and additional options. See diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index ed6c4353167..d545ba0c3ab 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -112,9 +112,9 @@ PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, int64_t); // Export for 'math' shared extension PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); -PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right); -PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right); -PyAPI_FUNC(PyObject*) _PyLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(_PyStackRef) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(_PyStackRef) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(_PyStackRef) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; @@ -135,7 +135,7 @@ extern int _PyLong_FormatWriter( int alternate); extern char* _PyLong_FormatBytesWriter( - _PyBytesWriter *writer, + PyBytesWriter *writer, char *str, PyObject *obj, int base, @@ -158,6 +158,11 @@ PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); // Export for '_testclinic' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt8_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt16_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt32_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt64_Converter(PyObject *, void *); + /* Long value tag bits: * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1. * 2: Set to 1 for the small ints @@ -208,7 +213,6 @@ _PyLong_BothAreCompact(const PyLongObject* a, const PyLongObject* b) { assert(PyLong_Check(b)); return (a->long_value.lv_tag | b->long_value.lv_tag) < (2 << NON_SIZE_BITS); } - static inline bool _PyLong_IsZero(const PyLongObject *op) { @@ -308,6 +312,12 @@ _PyLong_FlipSign(PyLongObject *op) { #define _PyLong_FALSE_TAG TAG_FROM_SIGN_AND_SIZE(0, 0) #define _PyLong_TRUE_TAG TAG_FROM_SIGN_AND_SIZE(1, 1) +static inline int +_PyLong_CheckExactAndCompact(PyObject *op) +{ + return PyLong_CheckExact(op) && _PyLong_IsCompact((const PyLongObject *)op); +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 1cc897c5a69..2fb46a6df50 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -277,8 +277,19 @@ Known values: Python 3.14a7 3622 (Store annotations in different class dict keys) Python 3.14a7 3623 (Add BUILD_INTERPOLATION & BUILD_TEMPLATE opcodes) Python 3.14b1 3624 (Don't optimize LOAD_FAST when local is killed by DELETE_FAST) + Python 3.14b3 3625 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) + Python 3.14rc2 3626 (Fix missing exception handlers in logical expression) + Python 3.14rc3 3627 (Fix miscompilation of some module-level annotations) + Python 3.15a0 3650 (Initial version) + Python 3.15a1 3651 (Simplify LOAD_CONST) + Python 3.15a1 3652 (Virtual iterators) + Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) + Python 3.15a1 3654 (Fix missing exception handlers in logical expression) + Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) - Python 3.15 will start with 3650 + + Python 3.16 will start with 3700 Please don't copy-paste the same pre-release tag for new entries above!!! You should always use the *upcoming* tag. For example, if 3.12a6 came out @@ -289,7 +300,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3624 +#define PYC_MAGIC_NUMBER 3656 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_modsupport.h b/Include/internal/pycore_modsupport.h index 614e9f93751..d90f42e9cd8 100644 --- a/Include/internal/pycore_modsupport.h +++ b/Include/internal/pycore_modsupport.h @@ -27,9 +27,8 @@ PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kwargs); // Export for 'zlib' shared extension PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, Py_ssize_t, Py_ssize_t); -#define _Py_ANY_VARARGS(n) ((n) == PY_SSIZE_T_MAX) #define _PyArg_CheckPositional(funcname, nargs, min, max) \ - ((!_Py_ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ + (((min) <= (nargs) && (nargs) <= (max)) \ || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) extern PyObject ** _Py_VaBuildStack( diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index b170d7bce70..9a62daf6621 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -1,5 +1,8 @@ #ifndef Py_INTERNAL_MODULEOBJECT_H #define Py_INTERNAL_MODULEOBJECT_H + +#include + #ifdef __cplusplus extern "C" { #endif @@ -16,32 +19,49 @@ extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); +typedef int (*_Py_modexecfunc)(PyObject *); + typedef struct { PyObject_HEAD PyObject *md_dict; - PyModuleDef *md_def; void *md_state; PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; + bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED - void *md_gil; + bool md_requires_gil; #endif + Py_ssize_t md_state_size; + traverseproc md_state_traverse; + inquiry md_state_clear; + freefunc md_state_free; + void *md_token; + _Py_modexecfunc md_exec; /* only set if md_token_is_def is true */ } PyModuleObject; -static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_def; +#define _PyModule_CAST(op) \ + (assert(PyModule_Check(op)), _Py_CAST(PyModuleObject*, (op))) + +static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + if (mod->md_token_is_def) { + return (PyModuleDef *)mod->md_token; + } + return NULL; +} + +static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) { + PyModuleObject *mod = _PyModule_CAST(arg); + return (PyModuleDef *)mod->md_token; } static inline void* _PyModule_GetState(PyObject* mod) { - assert(PyModule_Check(mod)); - return ((PyModuleObject *)mod)->md_state; + return _PyModule_CAST(mod)->md_state; } static inline PyObject* _PyModule_GetDict(PyObject *mod) { - assert(PyModule_Check(mod)); - PyObject *dict = ((PyModuleObject *)mod) -> md_dict; + PyObject *dict = _PyModule_CAST(mod)->md_dict; // _PyModule_GetDict(mod) must not be used after calling module_clear(mod) assert(dict != NULL); return dict; // borrowed reference diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b7e162c8abc..fb50acd62da 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -313,7 +313,7 @@ extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); // Fast inlined version of PyType_HasFeature() static inline int _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { - return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0); + return ((type->tp_flags) & feature) != 0; } extern void _PyType_InitCache(PyInterpreterState *interp); @@ -614,7 +614,7 @@ static inline PyObject * _Py_XGetRef(PyObject **ptr) { for (;;) { - PyObject *value = _Py_atomic_load_ptr(ptr); + PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); if (value == NULL) { return value; } @@ -629,7 +629,7 @@ _Py_XGetRef(PyObject **ptr) static inline PyObject * _Py_TryXGetRef(PyObject **ptr) { - PyObject *value = _Py_atomic_load_ptr(ptr); + PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); if (value == NULL) { return value; } @@ -767,6 +767,27 @@ _Py_TryIncref(PyObject *op) #endif } +// Enqueue an object to be freed possibly after some delay +#ifdef Py_GIL_DISABLED +PyAPI_FUNC(void) _PyObject_XDecRefDelayed(PyObject *obj); +#else +static inline void _PyObject_XDecRefDelayed(PyObject *obj) +{ + Py_XDECREF(obj); +} +#endif + +#ifdef Py_GIL_DISABLED +// Same as `Py_XSETREF` but in free-threading, it stores the object atomically +// and queues the old object to be decrefed at a safe point using QSBR. +PyAPI_FUNC(void) _PyObject_XSetRefDelayed(PyObject **p_obj, PyObject *obj); +#else +static inline void _PyObject_XSetRefDelayed(PyObject **p_obj, PyObject *obj) +{ + Py_XSETREF(*p_obj, obj); +} +#endif + #ifdef Py_REF_DEBUG extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); extern void _Py_FinalizeRefTotal(_PyRuntimeState *); @@ -842,8 +863,7 @@ static inline Py_hash_t _PyObject_HashFast(PyObject *op) { if (PyUnicode_CheckExact(op)) { - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( - _PyASCIIObject_CAST(op)->hash); + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); if (hash != -1) { return hash; } @@ -897,6 +917,9 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, extern unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); +PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, + PyObject *name, _PyStackRef *method); + // Cache the provided init method in the specialization cache of type if the // provided type version matches the current version of the type. // @@ -1007,6 +1030,27 @@ enum _PyAnnotateFormat { _Py_ANNOTATE_FORMAT_STRING = 4, }; +extern int _PyObject_SetDict(PyObject *obj, PyObject *value); +extern int _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); + +#ifndef Py_GIL_DISABLED +static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) +{ + assert(!_Py_IsStaticImmortal(op)); + op->ob_refcnt++; + _Py_INCREF_STAT_INC(); +#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) + if (!_Py_IsImmortal(op)) { + _Py_INCREF_IncRefTotal(); + } +#endif +} +#endif + +/* Utility for the tp_traverse slot of mutable heap types that have no other + * references. */ +PyAPI_FUNC(int) _PyObject_VisitType(PyObject *op, visitproc visit, void *arg); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 0e34074f160..cca88818c57 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -113,7 +113,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_INTRINSIC_2: return 2; case CALL_ISINSTANCE: - return 2 + oparg; + return 4; case CALL_KW: return 3 + oparg; case CALL_KW_BOUND_METHOD: @@ -205,15 +205,15 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case FORMAT_WITH_SPEC: return 2; case FOR_ITER: - return 1; + return 2; case FOR_ITER_GEN: - return 1; + return 2; case FOR_ITER_LIST: - return 1; + return 2; case FOR_ITER_RANGE: - return 1; + return 2; case FOR_ITER_TUPLE: - return 1; + return 2; case GET_AITER: return 1; case GET_ANEXT: @@ -239,11 +239,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_END_ASYNC_FOR: return 2; case INSTRUMENTED_END_FOR: - return 2; + return 3; case INSTRUMENTED_END_SEND: return 2; case INSTRUMENTED_FOR_ITER: - return 1; + return 2; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_BACKWARD: @@ -257,7 +257,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_NOT_TAKEN: return 0; case INSTRUMENTED_POP_ITER: - return 1; + return 2; case INSTRUMENTED_POP_JUMP_IF_FALSE: return 1; case INSTRUMENTED_POP_JUMP_IF_NONE: @@ -334,10 +334,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case LOAD_CONST: return 0; - case LOAD_CONST_IMMORTAL: - return 0; - case LOAD_CONST_MORTAL: - return 0; case LOAD_DEREF: return 0; case LOAD_FAST: @@ -399,7 +395,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case POP_EXCEPT: return 1; case POP_ITER: - return 1; + return 2; case POP_JUMP_IF_FALSE: return 1; case POP_JUMP_IF_NONE: @@ -492,6 +488,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -692,15 +690,15 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case FORMAT_WITH_SPEC: return 1; case FOR_ITER: - return 2; + return 3; case FOR_ITER_GEN: - return 1; + return 2; case FOR_ITER_LIST: - return 2; + return 3; case FOR_ITER_RANGE: - return 2; + return 3; case FOR_ITER_TUPLE: - return 2; + return 3; case GET_AITER: return 1; case GET_ANEXT: @@ -708,7 +706,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_AWAITABLE: return 1; case GET_ITER: - return 1; + return 2; case GET_LEN: return 2; case GET_YIELD_FROM_ITER: @@ -726,11 +724,11 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case INSTRUMENTED_END_ASYNC_FOR: return 0; case INSTRUMENTED_END_FOR: - return 1; + return 2; case INSTRUMENTED_END_SEND: return 1; case INSTRUMENTED_FOR_ITER: - return 2; + return 3; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_BACKWARD: @@ -821,10 +819,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case LOAD_CONST: return 1; - case LOAD_CONST_IMMORTAL: - return 1; - case LOAD_CONST_MORTAL: - return 1; case LOAD_DEREF: return 1; case LOAD_FAST: @@ -979,6 +973,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -1036,10 +1032,11 @@ enum InstructionFormat { #define HAS_ESCAPES_FLAG (512) #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) -#define HAS_PASSTHROUGH_FLAG (4096) -#define HAS_OPARG_AND_1_FLAG (8192) -#define HAS_ERROR_NO_POP_FLAG (16384) -#define HAS_NO_SAVE_IP_FLAG (32768) +#define HAS_ERROR_NO_POP_FLAG (4096) +#define HAS_NO_SAVE_IP_FLAG (8192) +#define HAS_PERIODIC_FLAG (16384) +#define HAS_UNPREDICTABLE_JUMP_FLAG (32768) +#define HAS_NEEDS_GUARD_IP_FLAG (65536) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1052,10 +1049,11 @@ enum InstructionFormat { #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) -#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) -#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) +#define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) +#define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) +#define OPCODE_HAS_NEEDS_GUARD_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NEEDS_GUARD_IP_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1072,7 +1070,7 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; uint8_t instr_format; - uint16_t flags; + uint32_t flags; }; extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; @@ -1080,55 +1078,55 @@ extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [BUILD_MAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_SET] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1137,7 +1135,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1153,7 +1151,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1161,11 +1159,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1174,13 +1172,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1193,10 +1191,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, @@ -1205,9 +1203,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -1215,27 +1213,25 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [LOAD_CONST_IMMORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [LOAD_CONST_MORTAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1252,23 +1248,23 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [NOT_TAKEN] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, + [POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, + [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ESCAPES_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_TOP] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1292,19 +1288,20 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1357,27 +1354,27 @@ _PyOpcode_macro_expansion[256] = { [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 10, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 7, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 } } }, - [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, + [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_KW_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 } } }, - [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_LIST_APPEND] = { .nuops = 4, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_PY_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TYPE_1] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, OPARG_SIMPLE, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, OPARG_SIMPLE, 0 } } }, @@ -1418,6 +1415,9 @@ _PyOpcode_macro_expansion[256] = { [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, @@ -1435,8 +1435,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, OPARG_SIMPLE, 0 } } }, [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, OPARG_SIMPLE, 0 } } }, - [LOAD_CONST_IMMORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_IMMORTAL, OPARG_SIMPLE, 0 } } }, - [LOAD_CONST_MORTAL] = { .nuops = 1, .uops = { { _LOAD_CONST_MORTAL, OPARG_SIMPLE, 0 } } }, + [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, OPARG_SIMPLE, 0 } } }, [LOAD_DEREF] = { .nuops = 1, .uops = { { _LOAD_DEREF, OPARG_SIMPLE, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { _LOAD_FAST, OPARG_SIMPLE, 0 } } }, [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { _LOAD_FAST_AND_CLEAR, OPARG_SIMPLE, 0 } } }, @@ -1464,7 +1463,7 @@ _PyOpcode_macro_expansion[256] = { [NOP] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, OPARG_SIMPLE, 0 } } }, - [POP_ITER] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [POP_ITER] = { .nuops = 1, .uops = { { _POP_ITER, OPARG_SIMPLE, 0 } } }, [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_TRUE, OPARG_REPLACED, 1 } } }, [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, @@ -1667,8 +1666,6 @@ const char *_PyOpcode_OpName[267] = { [LOAD_CLOSURE] = "LOAD_CLOSURE", [LOAD_COMMON_CONSTANT] = "LOAD_COMMON_CONSTANT", [LOAD_CONST] = "LOAD_CONST", - [LOAD_CONST_IMMORTAL] = "LOAD_CONST_IMMORTAL", - [LOAD_CONST_MORTAL] = "LOAD_CONST_MORTAL", [LOAD_DEREF] = "LOAD_DEREF", [LOAD_FAST] = "LOAD_FAST", [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", @@ -1746,6 +1743,7 @@ const char *_PyOpcode_OpName[267] = { [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [TRACE_RECORD] = "TRACE_RECORD", [UNARY_INVERT] = "UNARY_INVERT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", @@ -1787,6 +1785,36 @@ const uint8_t _PyOpcode_Caches[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [121] = 121, + [122] = 122, + [123] = 123, + [124] = 124, + [125] = 125, + [126] = 126, + [127] = 127, + [210] = 210, + [211] = 211, + [212] = 212, + [213] = 213, + [214] = 214, + [215] = 215, + [216] = 216, + [217] = 217, + [218] = 218, + [219] = 219, + [220] = 220, + [221] = 221, + [222] = 222, + [223] = 223, + [224] = 224, + [225] = 225, + [226] = 226, + [227] = 227, + [228] = 228, + [229] = 229, + [230] = 230, + [231] = 231, + [232] = 232, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -1930,8 +1958,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, [LOAD_COMMON_CONSTANT] = LOAD_COMMON_CONSTANT, [LOAD_CONST] = LOAD_CONST, - [LOAD_CONST_IMMORTAL] = LOAD_CONST, - [LOAD_CONST_MORTAL] = LOAD_CONST, [LOAD_DEREF] = LOAD_DEREF, [LOAD_FAST] = LOAD_FAST, [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, @@ -2004,6 +2030,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [TO_BOOL_LIST] = TO_BOOL, [TO_BOOL_NONE] = TO_BOOL, [TO_BOOL_STR] = TO_BOOL, + [TRACE_RECORD] = TRACE_RECORD, [UNARY_INVERT] = UNARY_INVERT, [UNARY_NEGATIVE] = UNARY_NEGATIVE, [UNARY_NOT] = UNARY_NOT, @@ -2026,6 +2053,8 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ + case 210: \ + case 211: \ case 212: \ case 213: \ case 214: \ @@ -2047,7 +2076,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 230: \ case 231: \ case 232: \ - case 233: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 62af06dc01c..79a1a242556 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -56,6 +56,8 @@ extern "C" { #define IS_RETURN_OPCODE(opcode) \ (opcode == RETURN_VALUE) +#define IS_RAISE_OPCODE(opcode) \ + (opcode == RAISE_VARARGS || opcode == RERAISE) /* Flags used in the oparg for MAKE_FUNCTION */ diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d3674726997..e7177552cf6 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -9,7 +9,9 @@ extern "C" { #endif #include "pycore_typedefs.h" // _PyInterpreterFrame +#include "pycore_uop.h" // _PyUOpInstruction #include "pycore_uop_ids.h" +#include "pycore_stackref.h" // _PyStackRef #include @@ -19,14 +21,6 @@ typedef struct _PyExecutorLinkListNode { } _PyExecutorLinkListNode; -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; @@ -40,34 +34,11 @@ typedef struct { PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). } _PyVMData; -/* Depending on the format, - * the 32 bits between the oparg and operand are: - * UOP_FORMAT_TARGET: - * uint32_t target; - * UOP_FORMAT_JUMP - * uint16_t jump_target; - * uint16_t error_target; - */ -typedef struct { - uint16_t opcode:15; - uint16_t format:1; - uint16_t oparg; - union { - uint32_t target; - struct { - uint16_t jump_target; - uint16_t error_target; - }; - }; - uint64_t operand0; // A cache entry - uint64_t operand1; -#ifdef Py_STATS - uint64_t execution_count; -#endif -} _PyUOpInstruction; - -typedef struct { +typedef struct _PyExitData { uint32_t target; + uint16_t index:14; + uint16_t is_dynamic:1; + uint16_t is_control_flow:1; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -80,7 +51,6 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; - void *jit_side_entry; _PyExitData exits[1]; } _PyExecutorObject; @@ -114,15 +84,12 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #endif // Used as the threshold to trigger executor invalidation when -// trace_run_counter is greater than this value. -#define JIT_CLEANUP_THRESHOLD 100000 +// executor_creation_counter is greater than this value. +// This value is arbitrary and was not optimized. +#define JIT_CLEANUP_THRESHOLD 1000 -// This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 800 - -#define TRACE_STACK_SIZE 5 - -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, +int _Py_uop_analyze_and_optimize( + PyFunctionObject *func, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); @@ -156,7 +123,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) #define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) // Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) +#define MAX_ABSTRACT_FRAME_DEPTH (16) // The maximum number of side exits that we can take before requiring forward // progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this @@ -178,6 +145,7 @@ typedef enum _JitSymType { JIT_SYM_KNOWN_VALUE_TAG = 7, JIT_SYM_TUPLE_TAG = 8, JIT_SYM_TRUTHINESS_TAG = 9, + JIT_SYM_COMPACT_INT = 10, } JitSymType; typedef struct _jit_opt_known_class { @@ -210,6 +178,10 @@ typedef struct { uint16_t value; } JitOptTruthiness; +typedef struct { + uint8_t tag; +} JitOptCompactInt; + typedef union _jit_opt_symbol { uint8_t tag; JitOptKnownClass cls; @@ -217,18 +189,73 @@ typedef union _jit_opt_symbol { JitOptKnownVersion version; JitOptTuple tuple; JitOptTruthiness truthiness; + JitOptCompactInt compact; } JitOptSymbol; +// This mimics the _PyStackRef API +typedef union { + uintptr_t bits; +} JitOptRef; + +#define REF_IS_BORROWED 1 + +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED))) + +static inline JitOptSymbol * +PyJitRef_Unwrap(JitOptRef ref) +{ + return JIT_BITS_TO_PTR_MASKED(ref); +} + +bool _Py_uop_symbol_is_immortal(JitOptSymbol *sym); + + +static inline JitOptRef +PyJitRef_Wrap(JitOptSymbol *sym) +{ + return (JitOptRef){.bits=(uintptr_t)sym}; +} + +static inline JitOptRef +PyJitRef_StripReferenceInfo(JitOptRef ref) +{ + return PyJitRef_Wrap(PyJitRef_Unwrap(ref)); +} + +static inline JitOptRef +PyJitRef_Borrow(JitOptRef ref) +{ + return (JitOptRef){ .bits = ref.bits | REF_IS_BORROWED }; +} + +static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; + +static inline bool +PyJitRef_IsNull(JitOptRef ref) +{ + return ref.bits == PyJitRef_NULL.bits; +} + +static inline int +PyJitRef_IsBorrowed(JitOptRef ref) +{ + return (ref.bits & REF_IS_BORROWED) == REF_IS_BORROWED; +} struct _Py_UOpsAbstractFrame { + bool globals_watched; + // The version number of the globals dicts, once checked. 0 if unchecked. + uint32_t globals_checked_version; // Max stacklen int stack_len; int locals_len; + PyFunctionObject *func; + PyCodeObject *code; - JitOptSymbol **stack_pointer; - JitOptSymbol **stack; - JitOptSymbol **locals; + JitOptRef *stack_pointer; + JitOptRef *stack; + JitOptRef *locals; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -243,6 +270,8 @@ typedef struct _JitOptContext { char done; char out_of_space; bool contradiction; + // Has the builtins dict been watched? + bool builtins_watched; // The current "executing" frame. _Py_UOpsAbstractFrame *frame; _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; @@ -251,37 +280,43 @@ typedef struct _JitOptContext { // Arena for the symbolic types. ty_arena t_arena; - JitOptSymbol **n_consumed; - JitOptSymbol **limit; - JitOptSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + JitOptRef *n_consumed; + JitOptRef *limit; + JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; } JitOptContext; -extern bool _Py_uop_sym_is_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_not_null(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym); -extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_unknown(JitOptContext *ctx); -extern JitOptSymbol *_Py_uop_sym_new_not_null(JitOptContext *ctx); -extern JitOptSymbol *_Py_uop_sym_new_type( +extern bool _Py_uop_sym_is_null(JitOptRef sym); +extern bool _Py_uop_sym_is_not_null(JitOptRef sym); +extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef sym); +extern PyObject *_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_unknown(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_not_null(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_type( JitOptContext *ctx, PyTypeObject *typ); -extern JitOptSymbol *_Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); -extern JitOptSymbol *_Py_uop_sym_new_null(JitOptContext *ctx); -extern bool _Py_uop_sym_has_type(JitOptSymbol *sym); -extern bool _Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym); -extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val); -extern bool _Py_uop_sym_is_bottom(JitOptSymbol *sym); -extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym); -extern PyTypeObject *_Py_uop_sym_get_type(JitOptSymbol *sym); -extern bool _Py_uop_sym_is_immortal(JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args); -extern JitOptSymbol *_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item); -extern int _Py_uop_sym_tuple_length(JitOptSymbol *sym); -extern JitOptSymbol *_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy); + +extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); +extern JitOptRef _Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val); +bool _Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym); +_PyStackRef _Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); +extern bool _Py_uop_sym_has_type(JitOptRef sym); +extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version); +extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef sym); +extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef sym, unsigned int version); +extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(JitOptRef sym); +extern int _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef sym); +extern PyTypeObject *_Py_uop_sym_get_type(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args); +extern JitOptRef _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef sym, Py_ssize_t item); +extern Py_ssize_t _Py_uop_sym_tuple_length(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value, bool truthy); +extern bool _Py_uop_sym_is_compact_int(JitOptRef sym); +extern JitOptRef _Py_uop_sym_new_compact_int(JitOptContext *ctx); +extern void _Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); @@ -290,28 +325,55 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len); -extern int _Py_uop_frame_pop(JitOptContext *ctx); +extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, PyThreadState *tstate); + +static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) +{ + _PyExitData *exit0 = exit - exit->index; + return (_PyExecutorObject *)(((char *)exit0) - offsetof(_PyExecutorObject, exits)); +} + +extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); +extern _PyExecutorObject *_PyExecutor_GetColdDynamicExecutor(void); + +PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); static inline int is_terminator(const _PyUOpInstruction *uop) { int opcode = uop->opcode; return ( opcode == _EXIT_TRACE || - opcode == _JUMP_TO_TOP + opcode == _DEOPT || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT ); } +extern void _PyExecutor_Free(_PyExecutorObject *self); + PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); #ifdef _Py_TIER2 extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); + +PyAPI_FUNC(int) +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, + int oparg); + +void _PyJit_FinalizeTracing(PyThreadState *tstate); + +void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2885dee63dc..2c46f59ab7d 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -48,7 +48,8 @@ extern struct _mod* _PyParser_ASTFromString( PyObject* filename, int mode, PyCompilerFlags *flags, - PyArena *arena); + PyArena *arena, + PyObject *module); extern struct _mod* _PyParser_ASTFromFile( FILE *fp, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3e41e2fd156..2ae0185226f 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -77,6 +77,10 @@ extern "C" { _Py_atomic_store_ushort_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_LOAD_INT(value) \ + _Py_atomic_load_int(&value) +#define FT_ATOMIC_STORE_INT(value, new_value) \ + _Py_atomic_store_int(&value, new_value) #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ _Py_atomic_store_int_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ @@ -111,6 +115,8 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -142,6 +148,8 @@ extern "C" { #define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value #define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT(value) value +#define FT_ATOMIC_STORE_INT(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value @@ -159,6 +167,8 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index f357b88e220..f80808fcc8c 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -94,13 +94,13 @@ extern void _PyErr_Fetch( PyObject **value, PyObject **traceback); -extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate); +PyAPI_FUNC(PyObject*) _PyErr_GetRaisedException(PyThreadState *tstate); PyAPI_FUNC(int) _PyErr_ExceptionMatches( PyThreadState *tstate, PyObject *exc); -extern void _PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc); +PyAPI_FUNC(void) _PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc); extern void _PyErr_Restore( PyThreadState *tstate, @@ -123,7 +123,8 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset); + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 6e89ca33e42..8faf7a4d403 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -41,6 +41,7 @@ extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); +extern PyStatus _PyDateTime_InitTypes(PyInterpreterState *interp); /* Various internal finalizers */ diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index eea8996ba68..4fcac3aab8b 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_INFINITY || x == -Py_INFINITY) { + if (x == INFINITY || x == -INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_INFINITY || x == -Py_INFINITY || - y == Py_INFINITY || y == -Py_INFINITY) + if (x == INFINITY || x == -INFINITY || + y == INFINITY || y == -INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 02537bdfef8..05484e847f1 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -54,20 +54,43 @@ static inline int _PyMem_IsPtrFreed(const void *ptr) { uintptr_t value = (uintptr_t)ptr; #if SIZEOF_VOID_P == 8 - return (value == 0 + return (value <= 0xff // NULL, 0x1, 0x2, ..., 0xff || value == (uintptr_t)0xCDCDCDCDCDCDCDCD || value == (uintptr_t)0xDDDDDDDDDDDDDDDD - || value == (uintptr_t)0xFDFDFDFDFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFDFDFDFDFD + || value >= (uintptr_t)0xFFFFFFFFFFFFFF00); // -0xff, ..., -2, -1 #elif SIZEOF_VOID_P == 4 - return (value == 0 + return (value <= 0xff || value == (uintptr_t)0xCDCDCDCD || value == (uintptr_t)0xDDDDDDDD - || value == (uintptr_t)0xFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFD + || value >= (uintptr_t)0xFFFFFF00); #else # error "unknown pointer size" #endif } +// Similar to _PyMem_IsPtrFreed() but expects an 'unsigned long' instead of a +// pointer. +static inline int _PyMem_IsULongFreed(unsigned long value) +{ +#if SIZEOF_LONG == 8 + return (value == 0 + || value == (unsigned long)0xCDCDCDCDCDCDCDCD + || value == (unsigned long)0xDDDDDDDDDDDDDDDD + || value == (unsigned long)0xFDFDFDFDFDFDFDFD + || value == (unsigned long)0xFFFFFFFFFFFFFFFF); +#elif SIZEOF_LONG == 4 + return (value == 0 + || value == (unsigned long)0xCDCDCDCD + || value == (unsigned long)0xDDDDDDDD + || value == (unsigned long)0xFDFDFDFD + || value == (unsigned long)0xFFFFFFFF); +#else +# error "unknown long size" +#endif +} + extern int _PyMem_GetAllocatorName( const char *name, PyMemAllocatorName *allocator); @@ -88,17 +111,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str); extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. -extern void _PyMem_FreeDelayed(void *ptr); - -// Enqueue an object to be freed possibly after some delay -#ifdef Py_GIL_DISABLED -PyAPI_FUNC(void) _PyObject_XDecRefDelayed(PyObject *obj); -#else -static inline void _PyObject_XDecRefDelayed(PyObject *obj) -{ - Py_XDECREF(obj); -} -#endif +extern void _PyMem_FreeDelayed(void *ptr, size_t size); // Periodically process delayed free requests. extern void _PyMem_ProcessDelayed(PyThreadState *tstate); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 633e5cf77db..189a8dde9f0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_pythonrun.h" // _PyOS_STACK_MARGIN_SHIFT #include "pycore_typedefs.h" // _PyRuntimeState #include "pycore_tstate.h" @@ -88,8 +89,9 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) /* Variable and static inline functions for in-line access to current thread and interpreter state */ -#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +#if !defined(Py_BUILD_CORE_MODULE) extern _Py_thread_local PyThreadState *_Py_tss_tstate; +extern _Py_thread_local PyInterpreterState *_Py_tss_interp; #endif #ifndef NDEBUG @@ -113,7 +115,7 @@ PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); static inline PyThreadState* _PyThreadState_GET(void) { -#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +#if !defined(Py_BUILD_CORE_MODULE) return _Py_tss_tstate; #else return _PyThreadState_GetCurrent(); @@ -203,11 +205,15 @@ _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) See also PyInterpreterState_Get() and _PyGILState_GetInterpreterStateUnsafe(). */ static inline PyInterpreterState* _PyInterpreterState_GET(void) { - PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); #endif - return tstate->interp; +#if !defined(Py_BUILD_CORE_MODULE) + return _Py_tss_interp; +#else + return _PyThreadState_GET()->interp; +#endif } @@ -325,7 +331,11 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); - return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, PYOS_STACK_MARGIN_SHIFT); +#if _Py_STACK_GROWS_DOWN + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); +#else + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)_tstate->c_stack_soft_limit - here_addr, _PyOS_STACK_MARGIN_SHIFT); +#endif } #ifdef __cplusplus diff --git a/Include/internal/pycore_pystats.h b/Include/internal/pycore_pystats.h index f8af398a560..50ab21aa0f1 100644 --- a/Include/internal/pycore_pystats.h +++ b/Include/internal/pycore_pystats.h @@ -9,7 +9,7 @@ extern "C" { #endif #ifdef Py_STATS -extern void _Py_StatsOn(void); +extern int _Py_StatsOn(void); extern void _Py_StatsOff(void); extern void _Py_StatsClear(void); extern int _Py_PrintSpecializationStats(int to_file); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index 0bfc5704dc4..04a557e1204 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -25,6 +25,7 @@ extern int _PyRun_InteractiveLoopObject( PyObject *filename, PyCompilerFlags *flags); +extern int _PyObject_SupportedAsScript(PyObject *); extern const char* _Py_SourceAsString( PyObject *cmd, const char *funcname, @@ -32,6 +33,40 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); +extern PyObject * _Py_CompileStringObjectWithModule( + const char *str, + PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, + PyObject *module); + + +/* Stack size, in "pointers". This must be large enough, so + * no two calls to check recursion depth are more than this far + * apart. In practice, that means it must be larger than the C + * stack consumption of PyEval_EvalDefault */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) +# define _PyOS_LOG2_STACK_MARGIN 12 +#elif defined(Py_DEBUG) && defined(WIN32) +# define _PyOS_LOG2_STACK_MARGIN 12 +#else +# define _PyOS_LOG2_STACK_MARGIN 11 +#endif +#define _PyOS_STACK_MARGIN (1 << _PyOS_LOG2_STACK_MARGIN) +#define _PyOS_STACK_MARGIN_BYTES (_PyOS_STACK_MARGIN * sizeof(void *)) + +#if SIZEOF_VOID_P == 8 +# define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 3) +#else +# define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2) +#endif + +#ifdef _Py_THREAD_SANITIZER +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 6) +#else +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 3) +#endif + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index b835c3abaf5..1f9b3fcf777 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -48,8 +48,21 @@ struct _qsbr_thread_state { // Thread state (or NULL) PyThreadState *tstate; - // Used to defer advancing write sequence a fixed number of times - int deferrals; + // Number of held items added by this thread since the last write sequence + // advance + int deferred_count; + + // Estimate for the amount of memory that is held by this thread since + // the last write sequence advance + size_t deferred_memory; + + // Amount of memory in mimalloc pages deferred from collection. When + // deferred, they are prevented from being used for a different size class + // and in a different thread. + size_t deferred_page_memory; + + // True if the deferred memory frees should be processed. + bool should_process; // Is this thread state allocated? bool allocated; @@ -109,11 +122,17 @@ _Py_qbsr_goal_reached(struct _qsbr_thread_state *qsbr, uint64_t goal) extern uint64_t _Py_qsbr_advance(struct _qsbr_shared *shared); -// Batches requests to advance the write sequence. This advances the write -// sequence every N calls, which reduces overhead but increases time to -// reclamation. Returns the new goal. +// Return the next value for the write sequence (current plus the increment). extern uint64_t -_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr); +_Py_qsbr_shared_next(struct _qsbr_shared *shared); + +// Return true if deferred memory frees held by QSBR should be processed to +// determine if they can be safely freed. +static inline bool +_Py_qsbr_should_process(struct _qsbr_thread_state *qsbr) +{ + return qsbr->should_process; +} // Have the read sequences advanced to the given goal? If this returns true, // it safe to reclaim any memory tagged with the goal (or earlier goal). diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 4200d91a2fc..b182f7825a2 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -61,9 +61,6 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ }, \ - /* A TSS key must be initialized with Py_tss_NEEDS_INIT \ - in accordance with the specification. */ \ - .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ .ceval = { \ .pending_mainthread = { \ @@ -236,4 +233,4 @@ extern PyTypeObject _PyExc_MemoryError; #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_RUNTIME_INIT_H */ +#endif /* !Py_INTERNAL_RUNTIME_INIT_H */ \ No newline at end of file diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4a34ffa559e..31d88339a13 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -275,6 +275,774 @@ extern "C" { _PyLong_DIGIT_INIT(254), \ _PyLong_DIGIT_INIT(255), \ _PyLong_DIGIT_INIT(256), \ + _PyLong_DIGIT_INIT(257), \ + _PyLong_DIGIT_INIT(258), \ + _PyLong_DIGIT_INIT(259), \ + _PyLong_DIGIT_INIT(260), \ + _PyLong_DIGIT_INIT(261), \ + _PyLong_DIGIT_INIT(262), \ + _PyLong_DIGIT_INIT(263), \ + _PyLong_DIGIT_INIT(264), \ + _PyLong_DIGIT_INIT(265), \ + _PyLong_DIGIT_INIT(266), \ + _PyLong_DIGIT_INIT(267), \ + _PyLong_DIGIT_INIT(268), \ + _PyLong_DIGIT_INIT(269), \ + _PyLong_DIGIT_INIT(270), \ + _PyLong_DIGIT_INIT(271), \ + _PyLong_DIGIT_INIT(272), \ + _PyLong_DIGIT_INIT(273), \ + _PyLong_DIGIT_INIT(274), \ + _PyLong_DIGIT_INIT(275), \ + _PyLong_DIGIT_INIT(276), \ + _PyLong_DIGIT_INIT(277), \ + _PyLong_DIGIT_INIT(278), \ + _PyLong_DIGIT_INIT(279), \ + _PyLong_DIGIT_INIT(280), \ + _PyLong_DIGIT_INIT(281), \ + _PyLong_DIGIT_INIT(282), \ + _PyLong_DIGIT_INIT(283), \ + _PyLong_DIGIT_INIT(284), \ + _PyLong_DIGIT_INIT(285), \ + _PyLong_DIGIT_INIT(286), \ + _PyLong_DIGIT_INIT(287), \ + _PyLong_DIGIT_INIT(288), \ + _PyLong_DIGIT_INIT(289), \ + _PyLong_DIGIT_INIT(290), \ + _PyLong_DIGIT_INIT(291), \ + _PyLong_DIGIT_INIT(292), \ + _PyLong_DIGIT_INIT(293), \ + _PyLong_DIGIT_INIT(294), \ + _PyLong_DIGIT_INIT(295), \ + _PyLong_DIGIT_INIT(296), \ + _PyLong_DIGIT_INIT(297), \ + _PyLong_DIGIT_INIT(298), \ + _PyLong_DIGIT_INIT(299), \ + _PyLong_DIGIT_INIT(300), \ + _PyLong_DIGIT_INIT(301), \ + _PyLong_DIGIT_INIT(302), \ + _PyLong_DIGIT_INIT(303), \ + _PyLong_DIGIT_INIT(304), \ + _PyLong_DIGIT_INIT(305), \ + _PyLong_DIGIT_INIT(306), \ + _PyLong_DIGIT_INIT(307), \ + _PyLong_DIGIT_INIT(308), \ + _PyLong_DIGIT_INIT(309), \ + _PyLong_DIGIT_INIT(310), \ + _PyLong_DIGIT_INIT(311), \ + _PyLong_DIGIT_INIT(312), \ + _PyLong_DIGIT_INIT(313), \ + _PyLong_DIGIT_INIT(314), \ + _PyLong_DIGIT_INIT(315), \ + _PyLong_DIGIT_INIT(316), \ + _PyLong_DIGIT_INIT(317), \ + _PyLong_DIGIT_INIT(318), \ + _PyLong_DIGIT_INIT(319), \ + _PyLong_DIGIT_INIT(320), \ + _PyLong_DIGIT_INIT(321), \ + _PyLong_DIGIT_INIT(322), \ + _PyLong_DIGIT_INIT(323), \ + _PyLong_DIGIT_INIT(324), \ + _PyLong_DIGIT_INIT(325), \ + _PyLong_DIGIT_INIT(326), \ + _PyLong_DIGIT_INIT(327), \ + _PyLong_DIGIT_INIT(328), \ + _PyLong_DIGIT_INIT(329), \ + _PyLong_DIGIT_INIT(330), \ + _PyLong_DIGIT_INIT(331), \ + _PyLong_DIGIT_INIT(332), \ + _PyLong_DIGIT_INIT(333), \ + _PyLong_DIGIT_INIT(334), \ + _PyLong_DIGIT_INIT(335), \ + _PyLong_DIGIT_INIT(336), \ + _PyLong_DIGIT_INIT(337), \ + _PyLong_DIGIT_INIT(338), \ + _PyLong_DIGIT_INIT(339), \ + _PyLong_DIGIT_INIT(340), \ + _PyLong_DIGIT_INIT(341), \ + _PyLong_DIGIT_INIT(342), \ + _PyLong_DIGIT_INIT(343), \ + _PyLong_DIGIT_INIT(344), \ + _PyLong_DIGIT_INIT(345), \ + _PyLong_DIGIT_INIT(346), \ + _PyLong_DIGIT_INIT(347), \ + _PyLong_DIGIT_INIT(348), \ + _PyLong_DIGIT_INIT(349), \ + _PyLong_DIGIT_INIT(350), \ + _PyLong_DIGIT_INIT(351), \ + _PyLong_DIGIT_INIT(352), \ + _PyLong_DIGIT_INIT(353), \ + _PyLong_DIGIT_INIT(354), \ + _PyLong_DIGIT_INIT(355), \ + _PyLong_DIGIT_INIT(356), \ + _PyLong_DIGIT_INIT(357), \ + _PyLong_DIGIT_INIT(358), \ + _PyLong_DIGIT_INIT(359), \ + _PyLong_DIGIT_INIT(360), \ + _PyLong_DIGIT_INIT(361), \ + _PyLong_DIGIT_INIT(362), \ + _PyLong_DIGIT_INIT(363), \ + _PyLong_DIGIT_INIT(364), \ + _PyLong_DIGIT_INIT(365), \ + _PyLong_DIGIT_INIT(366), \ + _PyLong_DIGIT_INIT(367), \ + _PyLong_DIGIT_INIT(368), \ + _PyLong_DIGIT_INIT(369), \ + _PyLong_DIGIT_INIT(370), \ + _PyLong_DIGIT_INIT(371), \ + _PyLong_DIGIT_INIT(372), \ + _PyLong_DIGIT_INIT(373), \ + _PyLong_DIGIT_INIT(374), \ + _PyLong_DIGIT_INIT(375), \ + _PyLong_DIGIT_INIT(376), \ + _PyLong_DIGIT_INIT(377), \ + _PyLong_DIGIT_INIT(378), \ + _PyLong_DIGIT_INIT(379), \ + _PyLong_DIGIT_INIT(380), \ + _PyLong_DIGIT_INIT(381), \ + _PyLong_DIGIT_INIT(382), \ + _PyLong_DIGIT_INIT(383), \ + _PyLong_DIGIT_INIT(384), \ + _PyLong_DIGIT_INIT(385), \ + _PyLong_DIGIT_INIT(386), \ + _PyLong_DIGIT_INIT(387), \ + _PyLong_DIGIT_INIT(388), \ + _PyLong_DIGIT_INIT(389), \ + _PyLong_DIGIT_INIT(390), \ + _PyLong_DIGIT_INIT(391), \ + _PyLong_DIGIT_INIT(392), \ + _PyLong_DIGIT_INIT(393), \ + _PyLong_DIGIT_INIT(394), \ + _PyLong_DIGIT_INIT(395), \ + _PyLong_DIGIT_INIT(396), \ + _PyLong_DIGIT_INIT(397), \ + _PyLong_DIGIT_INIT(398), \ + _PyLong_DIGIT_INIT(399), \ + _PyLong_DIGIT_INIT(400), \ + _PyLong_DIGIT_INIT(401), \ + _PyLong_DIGIT_INIT(402), \ + _PyLong_DIGIT_INIT(403), \ + _PyLong_DIGIT_INIT(404), \ + _PyLong_DIGIT_INIT(405), \ + _PyLong_DIGIT_INIT(406), \ + _PyLong_DIGIT_INIT(407), \ + _PyLong_DIGIT_INIT(408), \ + _PyLong_DIGIT_INIT(409), \ + _PyLong_DIGIT_INIT(410), \ + _PyLong_DIGIT_INIT(411), \ + _PyLong_DIGIT_INIT(412), \ + _PyLong_DIGIT_INIT(413), \ + _PyLong_DIGIT_INIT(414), \ + _PyLong_DIGIT_INIT(415), \ + _PyLong_DIGIT_INIT(416), \ + _PyLong_DIGIT_INIT(417), \ + _PyLong_DIGIT_INIT(418), \ + _PyLong_DIGIT_INIT(419), \ + _PyLong_DIGIT_INIT(420), \ + _PyLong_DIGIT_INIT(421), \ + _PyLong_DIGIT_INIT(422), \ + _PyLong_DIGIT_INIT(423), \ + _PyLong_DIGIT_INIT(424), \ + _PyLong_DIGIT_INIT(425), \ + _PyLong_DIGIT_INIT(426), \ + _PyLong_DIGIT_INIT(427), \ + _PyLong_DIGIT_INIT(428), \ + _PyLong_DIGIT_INIT(429), \ + _PyLong_DIGIT_INIT(430), \ + _PyLong_DIGIT_INIT(431), \ + _PyLong_DIGIT_INIT(432), \ + _PyLong_DIGIT_INIT(433), \ + _PyLong_DIGIT_INIT(434), \ + _PyLong_DIGIT_INIT(435), \ + _PyLong_DIGIT_INIT(436), \ + _PyLong_DIGIT_INIT(437), \ + _PyLong_DIGIT_INIT(438), \ + _PyLong_DIGIT_INIT(439), \ + _PyLong_DIGIT_INIT(440), \ + _PyLong_DIGIT_INIT(441), \ + _PyLong_DIGIT_INIT(442), \ + _PyLong_DIGIT_INIT(443), \ + _PyLong_DIGIT_INIT(444), \ + _PyLong_DIGIT_INIT(445), \ + _PyLong_DIGIT_INIT(446), \ + _PyLong_DIGIT_INIT(447), \ + _PyLong_DIGIT_INIT(448), \ + _PyLong_DIGIT_INIT(449), \ + _PyLong_DIGIT_INIT(450), \ + _PyLong_DIGIT_INIT(451), \ + _PyLong_DIGIT_INIT(452), \ + _PyLong_DIGIT_INIT(453), \ + _PyLong_DIGIT_INIT(454), \ + _PyLong_DIGIT_INIT(455), \ + _PyLong_DIGIT_INIT(456), \ + _PyLong_DIGIT_INIT(457), \ + _PyLong_DIGIT_INIT(458), \ + _PyLong_DIGIT_INIT(459), \ + _PyLong_DIGIT_INIT(460), \ + _PyLong_DIGIT_INIT(461), \ + _PyLong_DIGIT_INIT(462), \ + _PyLong_DIGIT_INIT(463), \ + _PyLong_DIGIT_INIT(464), \ + _PyLong_DIGIT_INIT(465), \ + _PyLong_DIGIT_INIT(466), \ + _PyLong_DIGIT_INIT(467), \ + _PyLong_DIGIT_INIT(468), \ + _PyLong_DIGIT_INIT(469), \ + _PyLong_DIGIT_INIT(470), \ + _PyLong_DIGIT_INIT(471), \ + _PyLong_DIGIT_INIT(472), \ + _PyLong_DIGIT_INIT(473), \ + _PyLong_DIGIT_INIT(474), \ + _PyLong_DIGIT_INIT(475), \ + _PyLong_DIGIT_INIT(476), \ + _PyLong_DIGIT_INIT(477), \ + _PyLong_DIGIT_INIT(478), \ + _PyLong_DIGIT_INIT(479), \ + _PyLong_DIGIT_INIT(480), \ + _PyLong_DIGIT_INIT(481), \ + _PyLong_DIGIT_INIT(482), \ + _PyLong_DIGIT_INIT(483), \ + _PyLong_DIGIT_INIT(484), \ + _PyLong_DIGIT_INIT(485), \ + _PyLong_DIGIT_INIT(486), \ + _PyLong_DIGIT_INIT(487), \ + _PyLong_DIGIT_INIT(488), \ + _PyLong_DIGIT_INIT(489), \ + _PyLong_DIGIT_INIT(490), \ + _PyLong_DIGIT_INIT(491), \ + _PyLong_DIGIT_INIT(492), \ + _PyLong_DIGIT_INIT(493), \ + _PyLong_DIGIT_INIT(494), \ + _PyLong_DIGIT_INIT(495), \ + _PyLong_DIGIT_INIT(496), \ + _PyLong_DIGIT_INIT(497), \ + _PyLong_DIGIT_INIT(498), \ + _PyLong_DIGIT_INIT(499), \ + _PyLong_DIGIT_INIT(500), \ + _PyLong_DIGIT_INIT(501), \ + _PyLong_DIGIT_INIT(502), \ + _PyLong_DIGIT_INIT(503), \ + _PyLong_DIGIT_INIT(504), \ + _PyLong_DIGIT_INIT(505), \ + _PyLong_DIGIT_INIT(506), \ + _PyLong_DIGIT_INIT(507), \ + _PyLong_DIGIT_INIT(508), \ + _PyLong_DIGIT_INIT(509), \ + _PyLong_DIGIT_INIT(510), \ + _PyLong_DIGIT_INIT(511), \ + _PyLong_DIGIT_INIT(512), \ + _PyLong_DIGIT_INIT(513), \ + _PyLong_DIGIT_INIT(514), \ + _PyLong_DIGIT_INIT(515), \ + _PyLong_DIGIT_INIT(516), \ + _PyLong_DIGIT_INIT(517), \ + _PyLong_DIGIT_INIT(518), \ + _PyLong_DIGIT_INIT(519), \ + _PyLong_DIGIT_INIT(520), \ + _PyLong_DIGIT_INIT(521), \ + _PyLong_DIGIT_INIT(522), \ + _PyLong_DIGIT_INIT(523), \ + _PyLong_DIGIT_INIT(524), \ + _PyLong_DIGIT_INIT(525), \ + _PyLong_DIGIT_INIT(526), \ + _PyLong_DIGIT_INIT(527), \ + _PyLong_DIGIT_INIT(528), \ + _PyLong_DIGIT_INIT(529), \ + _PyLong_DIGIT_INIT(530), \ + _PyLong_DIGIT_INIT(531), \ + _PyLong_DIGIT_INIT(532), \ + _PyLong_DIGIT_INIT(533), \ + _PyLong_DIGIT_INIT(534), \ + _PyLong_DIGIT_INIT(535), \ + _PyLong_DIGIT_INIT(536), \ + _PyLong_DIGIT_INIT(537), \ + _PyLong_DIGIT_INIT(538), \ + _PyLong_DIGIT_INIT(539), \ + _PyLong_DIGIT_INIT(540), \ + _PyLong_DIGIT_INIT(541), \ + _PyLong_DIGIT_INIT(542), \ + _PyLong_DIGIT_INIT(543), \ + _PyLong_DIGIT_INIT(544), \ + _PyLong_DIGIT_INIT(545), \ + _PyLong_DIGIT_INIT(546), \ + _PyLong_DIGIT_INIT(547), \ + _PyLong_DIGIT_INIT(548), \ + _PyLong_DIGIT_INIT(549), \ + _PyLong_DIGIT_INIT(550), \ + _PyLong_DIGIT_INIT(551), \ + _PyLong_DIGIT_INIT(552), \ + _PyLong_DIGIT_INIT(553), \ + _PyLong_DIGIT_INIT(554), \ + _PyLong_DIGIT_INIT(555), \ + _PyLong_DIGIT_INIT(556), \ + _PyLong_DIGIT_INIT(557), \ + _PyLong_DIGIT_INIT(558), \ + _PyLong_DIGIT_INIT(559), \ + _PyLong_DIGIT_INIT(560), \ + _PyLong_DIGIT_INIT(561), \ + _PyLong_DIGIT_INIT(562), \ + _PyLong_DIGIT_INIT(563), \ + _PyLong_DIGIT_INIT(564), \ + _PyLong_DIGIT_INIT(565), \ + _PyLong_DIGIT_INIT(566), \ + _PyLong_DIGIT_INIT(567), \ + _PyLong_DIGIT_INIT(568), \ + _PyLong_DIGIT_INIT(569), \ + _PyLong_DIGIT_INIT(570), \ + _PyLong_DIGIT_INIT(571), \ + _PyLong_DIGIT_INIT(572), \ + _PyLong_DIGIT_INIT(573), \ + _PyLong_DIGIT_INIT(574), \ + _PyLong_DIGIT_INIT(575), \ + _PyLong_DIGIT_INIT(576), \ + _PyLong_DIGIT_INIT(577), \ + _PyLong_DIGIT_INIT(578), \ + _PyLong_DIGIT_INIT(579), \ + _PyLong_DIGIT_INIT(580), \ + _PyLong_DIGIT_INIT(581), \ + _PyLong_DIGIT_INIT(582), \ + _PyLong_DIGIT_INIT(583), \ + _PyLong_DIGIT_INIT(584), \ + _PyLong_DIGIT_INIT(585), \ + _PyLong_DIGIT_INIT(586), \ + _PyLong_DIGIT_INIT(587), \ + _PyLong_DIGIT_INIT(588), \ + _PyLong_DIGIT_INIT(589), \ + _PyLong_DIGIT_INIT(590), \ + _PyLong_DIGIT_INIT(591), \ + _PyLong_DIGIT_INIT(592), \ + _PyLong_DIGIT_INIT(593), \ + _PyLong_DIGIT_INIT(594), \ + _PyLong_DIGIT_INIT(595), \ + _PyLong_DIGIT_INIT(596), \ + _PyLong_DIGIT_INIT(597), \ + _PyLong_DIGIT_INIT(598), \ + _PyLong_DIGIT_INIT(599), \ + _PyLong_DIGIT_INIT(600), \ + _PyLong_DIGIT_INIT(601), \ + _PyLong_DIGIT_INIT(602), \ + _PyLong_DIGIT_INIT(603), \ + _PyLong_DIGIT_INIT(604), \ + _PyLong_DIGIT_INIT(605), \ + _PyLong_DIGIT_INIT(606), \ + _PyLong_DIGIT_INIT(607), \ + _PyLong_DIGIT_INIT(608), \ + _PyLong_DIGIT_INIT(609), \ + _PyLong_DIGIT_INIT(610), \ + _PyLong_DIGIT_INIT(611), \ + _PyLong_DIGIT_INIT(612), \ + _PyLong_DIGIT_INIT(613), \ + _PyLong_DIGIT_INIT(614), \ + _PyLong_DIGIT_INIT(615), \ + _PyLong_DIGIT_INIT(616), \ + _PyLong_DIGIT_INIT(617), \ + _PyLong_DIGIT_INIT(618), \ + _PyLong_DIGIT_INIT(619), \ + _PyLong_DIGIT_INIT(620), \ + _PyLong_DIGIT_INIT(621), \ + _PyLong_DIGIT_INIT(622), \ + _PyLong_DIGIT_INIT(623), \ + _PyLong_DIGIT_INIT(624), \ + _PyLong_DIGIT_INIT(625), \ + _PyLong_DIGIT_INIT(626), \ + _PyLong_DIGIT_INIT(627), \ + _PyLong_DIGIT_INIT(628), \ + _PyLong_DIGIT_INIT(629), \ + _PyLong_DIGIT_INIT(630), \ + _PyLong_DIGIT_INIT(631), \ + _PyLong_DIGIT_INIT(632), \ + _PyLong_DIGIT_INIT(633), \ + _PyLong_DIGIT_INIT(634), \ + _PyLong_DIGIT_INIT(635), \ + _PyLong_DIGIT_INIT(636), \ + _PyLong_DIGIT_INIT(637), \ + _PyLong_DIGIT_INIT(638), \ + _PyLong_DIGIT_INIT(639), \ + _PyLong_DIGIT_INIT(640), \ + _PyLong_DIGIT_INIT(641), \ + _PyLong_DIGIT_INIT(642), \ + _PyLong_DIGIT_INIT(643), \ + _PyLong_DIGIT_INIT(644), \ + _PyLong_DIGIT_INIT(645), \ + _PyLong_DIGIT_INIT(646), \ + _PyLong_DIGIT_INIT(647), \ + _PyLong_DIGIT_INIT(648), \ + _PyLong_DIGIT_INIT(649), \ + _PyLong_DIGIT_INIT(650), \ + _PyLong_DIGIT_INIT(651), \ + _PyLong_DIGIT_INIT(652), \ + _PyLong_DIGIT_INIT(653), \ + _PyLong_DIGIT_INIT(654), \ + _PyLong_DIGIT_INIT(655), \ + _PyLong_DIGIT_INIT(656), \ + _PyLong_DIGIT_INIT(657), \ + _PyLong_DIGIT_INIT(658), \ + _PyLong_DIGIT_INIT(659), \ + _PyLong_DIGIT_INIT(660), \ + _PyLong_DIGIT_INIT(661), \ + _PyLong_DIGIT_INIT(662), \ + _PyLong_DIGIT_INIT(663), \ + _PyLong_DIGIT_INIT(664), \ + _PyLong_DIGIT_INIT(665), \ + _PyLong_DIGIT_INIT(666), \ + _PyLong_DIGIT_INIT(667), \ + _PyLong_DIGIT_INIT(668), \ + _PyLong_DIGIT_INIT(669), \ + _PyLong_DIGIT_INIT(670), \ + _PyLong_DIGIT_INIT(671), \ + _PyLong_DIGIT_INIT(672), \ + _PyLong_DIGIT_INIT(673), \ + _PyLong_DIGIT_INIT(674), \ + _PyLong_DIGIT_INIT(675), \ + _PyLong_DIGIT_INIT(676), \ + _PyLong_DIGIT_INIT(677), \ + _PyLong_DIGIT_INIT(678), \ + _PyLong_DIGIT_INIT(679), \ + _PyLong_DIGIT_INIT(680), \ + _PyLong_DIGIT_INIT(681), \ + _PyLong_DIGIT_INIT(682), \ + _PyLong_DIGIT_INIT(683), \ + _PyLong_DIGIT_INIT(684), \ + _PyLong_DIGIT_INIT(685), \ + _PyLong_DIGIT_INIT(686), \ + _PyLong_DIGIT_INIT(687), \ + _PyLong_DIGIT_INIT(688), \ + _PyLong_DIGIT_INIT(689), \ + _PyLong_DIGIT_INIT(690), \ + _PyLong_DIGIT_INIT(691), \ + _PyLong_DIGIT_INIT(692), \ + _PyLong_DIGIT_INIT(693), \ + _PyLong_DIGIT_INIT(694), \ + _PyLong_DIGIT_INIT(695), \ + _PyLong_DIGIT_INIT(696), \ + _PyLong_DIGIT_INIT(697), \ + _PyLong_DIGIT_INIT(698), \ + _PyLong_DIGIT_INIT(699), \ + _PyLong_DIGIT_INIT(700), \ + _PyLong_DIGIT_INIT(701), \ + _PyLong_DIGIT_INIT(702), \ + _PyLong_DIGIT_INIT(703), \ + _PyLong_DIGIT_INIT(704), \ + _PyLong_DIGIT_INIT(705), \ + _PyLong_DIGIT_INIT(706), \ + _PyLong_DIGIT_INIT(707), \ + _PyLong_DIGIT_INIT(708), \ + _PyLong_DIGIT_INIT(709), \ + _PyLong_DIGIT_INIT(710), \ + _PyLong_DIGIT_INIT(711), \ + _PyLong_DIGIT_INIT(712), \ + _PyLong_DIGIT_INIT(713), \ + _PyLong_DIGIT_INIT(714), \ + _PyLong_DIGIT_INIT(715), \ + _PyLong_DIGIT_INIT(716), \ + _PyLong_DIGIT_INIT(717), \ + _PyLong_DIGIT_INIT(718), \ + _PyLong_DIGIT_INIT(719), \ + _PyLong_DIGIT_INIT(720), \ + _PyLong_DIGIT_INIT(721), \ + _PyLong_DIGIT_INIT(722), \ + _PyLong_DIGIT_INIT(723), \ + _PyLong_DIGIT_INIT(724), \ + _PyLong_DIGIT_INIT(725), \ + _PyLong_DIGIT_INIT(726), \ + _PyLong_DIGIT_INIT(727), \ + _PyLong_DIGIT_INIT(728), \ + _PyLong_DIGIT_INIT(729), \ + _PyLong_DIGIT_INIT(730), \ + _PyLong_DIGIT_INIT(731), \ + _PyLong_DIGIT_INIT(732), \ + _PyLong_DIGIT_INIT(733), \ + _PyLong_DIGIT_INIT(734), \ + _PyLong_DIGIT_INIT(735), \ + _PyLong_DIGIT_INIT(736), \ + _PyLong_DIGIT_INIT(737), \ + _PyLong_DIGIT_INIT(738), \ + _PyLong_DIGIT_INIT(739), \ + _PyLong_DIGIT_INIT(740), \ + _PyLong_DIGIT_INIT(741), \ + _PyLong_DIGIT_INIT(742), \ + _PyLong_DIGIT_INIT(743), \ + _PyLong_DIGIT_INIT(744), \ + _PyLong_DIGIT_INIT(745), \ + _PyLong_DIGIT_INIT(746), \ + _PyLong_DIGIT_INIT(747), \ + _PyLong_DIGIT_INIT(748), \ + _PyLong_DIGIT_INIT(749), \ + _PyLong_DIGIT_INIT(750), \ + _PyLong_DIGIT_INIT(751), \ + _PyLong_DIGIT_INIT(752), \ + _PyLong_DIGIT_INIT(753), \ + _PyLong_DIGIT_INIT(754), \ + _PyLong_DIGIT_INIT(755), \ + _PyLong_DIGIT_INIT(756), \ + _PyLong_DIGIT_INIT(757), \ + _PyLong_DIGIT_INIT(758), \ + _PyLong_DIGIT_INIT(759), \ + _PyLong_DIGIT_INIT(760), \ + _PyLong_DIGIT_INIT(761), \ + _PyLong_DIGIT_INIT(762), \ + _PyLong_DIGIT_INIT(763), \ + _PyLong_DIGIT_INIT(764), \ + _PyLong_DIGIT_INIT(765), \ + _PyLong_DIGIT_INIT(766), \ + _PyLong_DIGIT_INIT(767), \ + _PyLong_DIGIT_INIT(768), \ + _PyLong_DIGIT_INIT(769), \ + _PyLong_DIGIT_INIT(770), \ + _PyLong_DIGIT_INIT(771), \ + _PyLong_DIGIT_INIT(772), \ + _PyLong_DIGIT_INIT(773), \ + _PyLong_DIGIT_INIT(774), \ + _PyLong_DIGIT_INIT(775), \ + _PyLong_DIGIT_INIT(776), \ + _PyLong_DIGIT_INIT(777), \ + _PyLong_DIGIT_INIT(778), \ + _PyLong_DIGIT_INIT(779), \ + _PyLong_DIGIT_INIT(780), \ + _PyLong_DIGIT_INIT(781), \ + _PyLong_DIGIT_INIT(782), \ + _PyLong_DIGIT_INIT(783), \ + _PyLong_DIGIT_INIT(784), \ + _PyLong_DIGIT_INIT(785), \ + _PyLong_DIGIT_INIT(786), \ + _PyLong_DIGIT_INIT(787), \ + _PyLong_DIGIT_INIT(788), \ + _PyLong_DIGIT_INIT(789), \ + _PyLong_DIGIT_INIT(790), \ + _PyLong_DIGIT_INIT(791), \ + _PyLong_DIGIT_INIT(792), \ + _PyLong_DIGIT_INIT(793), \ + _PyLong_DIGIT_INIT(794), \ + _PyLong_DIGIT_INIT(795), \ + _PyLong_DIGIT_INIT(796), \ + _PyLong_DIGIT_INIT(797), \ + _PyLong_DIGIT_INIT(798), \ + _PyLong_DIGIT_INIT(799), \ + _PyLong_DIGIT_INIT(800), \ + _PyLong_DIGIT_INIT(801), \ + _PyLong_DIGIT_INIT(802), \ + _PyLong_DIGIT_INIT(803), \ + _PyLong_DIGIT_INIT(804), \ + _PyLong_DIGIT_INIT(805), \ + _PyLong_DIGIT_INIT(806), \ + _PyLong_DIGIT_INIT(807), \ + _PyLong_DIGIT_INIT(808), \ + _PyLong_DIGIT_INIT(809), \ + _PyLong_DIGIT_INIT(810), \ + _PyLong_DIGIT_INIT(811), \ + _PyLong_DIGIT_INIT(812), \ + _PyLong_DIGIT_INIT(813), \ + _PyLong_DIGIT_INIT(814), \ + _PyLong_DIGIT_INIT(815), \ + _PyLong_DIGIT_INIT(816), \ + _PyLong_DIGIT_INIT(817), \ + _PyLong_DIGIT_INIT(818), \ + _PyLong_DIGIT_INIT(819), \ + _PyLong_DIGIT_INIT(820), \ + _PyLong_DIGIT_INIT(821), \ + _PyLong_DIGIT_INIT(822), \ + _PyLong_DIGIT_INIT(823), \ + _PyLong_DIGIT_INIT(824), \ + _PyLong_DIGIT_INIT(825), \ + _PyLong_DIGIT_INIT(826), \ + _PyLong_DIGIT_INIT(827), \ + _PyLong_DIGIT_INIT(828), \ + _PyLong_DIGIT_INIT(829), \ + _PyLong_DIGIT_INIT(830), \ + _PyLong_DIGIT_INIT(831), \ + _PyLong_DIGIT_INIT(832), \ + _PyLong_DIGIT_INIT(833), \ + _PyLong_DIGIT_INIT(834), \ + _PyLong_DIGIT_INIT(835), \ + _PyLong_DIGIT_INIT(836), \ + _PyLong_DIGIT_INIT(837), \ + _PyLong_DIGIT_INIT(838), \ + _PyLong_DIGIT_INIT(839), \ + _PyLong_DIGIT_INIT(840), \ + _PyLong_DIGIT_INIT(841), \ + _PyLong_DIGIT_INIT(842), \ + _PyLong_DIGIT_INIT(843), \ + _PyLong_DIGIT_INIT(844), \ + _PyLong_DIGIT_INIT(845), \ + _PyLong_DIGIT_INIT(846), \ + _PyLong_DIGIT_INIT(847), \ + _PyLong_DIGIT_INIT(848), \ + _PyLong_DIGIT_INIT(849), \ + _PyLong_DIGIT_INIT(850), \ + _PyLong_DIGIT_INIT(851), \ + _PyLong_DIGIT_INIT(852), \ + _PyLong_DIGIT_INIT(853), \ + _PyLong_DIGIT_INIT(854), \ + _PyLong_DIGIT_INIT(855), \ + _PyLong_DIGIT_INIT(856), \ + _PyLong_DIGIT_INIT(857), \ + _PyLong_DIGIT_INIT(858), \ + _PyLong_DIGIT_INIT(859), \ + _PyLong_DIGIT_INIT(860), \ + _PyLong_DIGIT_INIT(861), \ + _PyLong_DIGIT_INIT(862), \ + _PyLong_DIGIT_INIT(863), \ + _PyLong_DIGIT_INIT(864), \ + _PyLong_DIGIT_INIT(865), \ + _PyLong_DIGIT_INIT(866), \ + _PyLong_DIGIT_INIT(867), \ + _PyLong_DIGIT_INIT(868), \ + _PyLong_DIGIT_INIT(869), \ + _PyLong_DIGIT_INIT(870), \ + _PyLong_DIGIT_INIT(871), \ + _PyLong_DIGIT_INIT(872), \ + _PyLong_DIGIT_INIT(873), \ + _PyLong_DIGIT_INIT(874), \ + _PyLong_DIGIT_INIT(875), \ + _PyLong_DIGIT_INIT(876), \ + _PyLong_DIGIT_INIT(877), \ + _PyLong_DIGIT_INIT(878), \ + _PyLong_DIGIT_INIT(879), \ + _PyLong_DIGIT_INIT(880), \ + _PyLong_DIGIT_INIT(881), \ + _PyLong_DIGIT_INIT(882), \ + _PyLong_DIGIT_INIT(883), \ + _PyLong_DIGIT_INIT(884), \ + _PyLong_DIGIT_INIT(885), \ + _PyLong_DIGIT_INIT(886), \ + _PyLong_DIGIT_INIT(887), \ + _PyLong_DIGIT_INIT(888), \ + _PyLong_DIGIT_INIT(889), \ + _PyLong_DIGIT_INIT(890), \ + _PyLong_DIGIT_INIT(891), \ + _PyLong_DIGIT_INIT(892), \ + _PyLong_DIGIT_INIT(893), \ + _PyLong_DIGIT_INIT(894), \ + _PyLong_DIGIT_INIT(895), \ + _PyLong_DIGIT_INIT(896), \ + _PyLong_DIGIT_INIT(897), \ + _PyLong_DIGIT_INIT(898), \ + _PyLong_DIGIT_INIT(899), \ + _PyLong_DIGIT_INIT(900), \ + _PyLong_DIGIT_INIT(901), \ + _PyLong_DIGIT_INIT(902), \ + _PyLong_DIGIT_INIT(903), \ + _PyLong_DIGIT_INIT(904), \ + _PyLong_DIGIT_INIT(905), \ + _PyLong_DIGIT_INIT(906), \ + _PyLong_DIGIT_INIT(907), \ + _PyLong_DIGIT_INIT(908), \ + _PyLong_DIGIT_INIT(909), \ + _PyLong_DIGIT_INIT(910), \ + _PyLong_DIGIT_INIT(911), \ + _PyLong_DIGIT_INIT(912), \ + _PyLong_DIGIT_INIT(913), \ + _PyLong_DIGIT_INIT(914), \ + _PyLong_DIGIT_INIT(915), \ + _PyLong_DIGIT_INIT(916), \ + _PyLong_DIGIT_INIT(917), \ + _PyLong_DIGIT_INIT(918), \ + _PyLong_DIGIT_INIT(919), \ + _PyLong_DIGIT_INIT(920), \ + _PyLong_DIGIT_INIT(921), \ + _PyLong_DIGIT_INIT(922), \ + _PyLong_DIGIT_INIT(923), \ + _PyLong_DIGIT_INIT(924), \ + _PyLong_DIGIT_INIT(925), \ + _PyLong_DIGIT_INIT(926), \ + _PyLong_DIGIT_INIT(927), \ + _PyLong_DIGIT_INIT(928), \ + _PyLong_DIGIT_INIT(929), \ + _PyLong_DIGIT_INIT(930), \ + _PyLong_DIGIT_INIT(931), \ + _PyLong_DIGIT_INIT(932), \ + _PyLong_DIGIT_INIT(933), \ + _PyLong_DIGIT_INIT(934), \ + _PyLong_DIGIT_INIT(935), \ + _PyLong_DIGIT_INIT(936), \ + _PyLong_DIGIT_INIT(937), \ + _PyLong_DIGIT_INIT(938), \ + _PyLong_DIGIT_INIT(939), \ + _PyLong_DIGIT_INIT(940), \ + _PyLong_DIGIT_INIT(941), \ + _PyLong_DIGIT_INIT(942), \ + _PyLong_DIGIT_INIT(943), \ + _PyLong_DIGIT_INIT(944), \ + _PyLong_DIGIT_INIT(945), \ + _PyLong_DIGIT_INIT(946), \ + _PyLong_DIGIT_INIT(947), \ + _PyLong_DIGIT_INIT(948), \ + _PyLong_DIGIT_INIT(949), \ + _PyLong_DIGIT_INIT(950), \ + _PyLong_DIGIT_INIT(951), \ + _PyLong_DIGIT_INIT(952), \ + _PyLong_DIGIT_INIT(953), \ + _PyLong_DIGIT_INIT(954), \ + _PyLong_DIGIT_INIT(955), \ + _PyLong_DIGIT_INIT(956), \ + _PyLong_DIGIT_INIT(957), \ + _PyLong_DIGIT_INIT(958), \ + _PyLong_DIGIT_INIT(959), \ + _PyLong_DIGIT_INIT(960), \ + _PyLong_DIGIT_INIT(961), \ + _PyLong_DIGIT_INIT(962), \ + _PyLong_DIGIT_INIT(963), \ + _PyLong_DIGIT_INIT(964), \ + _PyLong_DIGIT_INIT(965), \ + _PyLong_DIGIT_INIT(966), \ + _PyLong_DIGIT_INIT(967), \ + _PyLong_DIGIT_INIT(968), \ + _PyLong_DIGIT_INIT(969), \ + _PyLong_DIGIT_INIT(970), \ + _PyLong_DIGIT_INIT(971), \ + _PyLong_DIGIT_INIT(972), \ + _PyLong_DIGIT_INIT(973), \ + _PyLong_DIGIT_INIT(974), \ + _PyLong_DIGIT_INIT(975), \ + _PyLong_DIGIT_INIT(976), \ + _PyLong_DIGIT_INIT(977), \ + _PyLong_DIGIT_INIT(978), \ + _PyLong_DIGIT_INIT(979), \ + _PyLong_DIGIT_INIT(980), \ + _PyLong_DIGIT_INIT(981), \ + _PyLong_DIGIT_INIT(982), \ + _PyLong_DIGIT_INIT(983), \ + _PyLong_DIGIT_INIT(984), \ + _PyLong_DIGIT_INIT(985), \ + _PyLong_DIGIT_INIT(986), \ + _PyLong_DIGIT_INIT(987), \ + _PyLong_DIGIT_INIT(988), \ + _PyLong_DIGIT_INIT(989), \ + _PyLong_DIGIT_INIT(990), \ + _PyLong_DIGIT_INIT(991), \ + _PyLong_DIGIT_INIT(992), \ + _PyLong_DIGIT_INIT(993), \ + _PyLong_DIGIT_INIT(994), \ + _PyLong_DIGIT_INIT(995), \ + _PyLong_DIGIT_INIT(996), \ + _PyLong_DIGIT_INIT(997), \ + _PyLong_DIGIT_INIT(998), \ + _PyLong_DIGIT_INIT(999), \ + _PyLong_DIGIT_INIT(1000), \ + _PyLong_DIGIT_INIT(1001), \ + _PyLong_DIGIT_INIT(1002), \ + _PyLong_DIGIT_INIT(1003), \ + _PyLong_DIGIT_INIT(1004), \ + _PyLong_DIGIT_INIT(1005), \ + _PyLong_DIGIT_INIT(1006), \ + _PyLong_DIGIT_INIT(1007), \ + _PyLong_DIGIT_INIT(1008), \ + _PyLong_DIGIT_INIT(1009), \ + _PyLong_DIGIT_INIT(1010), \ + _PyLong_DIGIT_INIT(1011), \ + _PyLong_DIGIT_INIT(1012), \ + _PyLong_DIGIT_INIT(1013), \ + _PyLong_DIGIT_INIT(1014), \ + _PyLong_DIGIT_INIT(1015), \ + _PyLong_DIGIT_INIT(1016), \ + _PyLong_DIGIT_INIT(1017), \ + _PyLong_DIGIT_INIT(1018), \ + _PyLong_DIGIT_INIT(1019), \ + _PyLong_DIGIT_INIT(1020), \ + _PyLong_DIGIT_INIT(1021), \ + _PyLong_DIGIT_INIT(1022), \ + _PyLong_DIGIT_INIT(1023), \ + _PyLong_DIGIT_INIT(1024), \ } #define _Py_bytes_characters_INIT { \ @@ -553,10 +1321,12 @@ extern "C" { INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ INIT_STR(format, ".format"), \ + INIT_STR(gc, ""), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ @@ -564,6 +1334,8 @@ extern "C" { #define _Py_str_identifiers_INIT { \ INIT_ID(CANCELLED), \ + INIT_ID(Emax), \ + INIT_ID(Emin), \ INIT_ID(FINISHED), \ INIT_ID(False), \ INIT_ID(JSONDecodeError), \ @@ -790,10 +1562,10 @@ extern "C" { INIT_ID(add_done_callback), \ INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ - INIT_ID(aggregate_class), \ INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_threads), \ INIT_ID(allow_code), \ INIT_ID(any), \ INIT_ID(append), \ @@ -807,7 +1579,6 @@ extern "C" { INIT_ID(ast), \ INIT_ID(athrow), \ INIT_ID(attribute), \ - INIT_ID(authorizer_callback), \ INIT_ID(autocommit), \ INIT_ID(backtick), \ INIT_ID(base), \ @@ -817,6 +1588,7 @@ extern "C" { INIT_ID(bit_offset), \ INIT_ID(bit_size), \ INIT_ID(block), \ + INIT_ID(blocking), \ INIT_ID(bound), \ INIT_ID(buffer), \ INIT_ID(buffer_callback), \ @@ -834,6 +1606,8 @@ extern "C" { INIT_ID(c_exception), \ INIT_ID(c_parameter_type), \ INIT_ID(c_return), \ + INIT_ID(c_stack), \ + INIT_ID(cache_frames), \ INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ @@ -841,13 +1615,17 @@ extern "C" { INIT_ID(call), \ INIT_ID(call_exception_handler), \ INIT_ID(call_soon), \ + INIT_ID(callable), \ INIT_ID(callback), \ INIT_ID(cancel), \ INIT_ID(capath), \ + INIT_ID(capitals), \ INIT_ID(category), \ INIT_ID(cb_type), \ INIT_ID(certfile), \ + INIT_ID(chain), \ INIT_ID(check_same_thread), \ + INIT_ID(clamp), \ INIT_ID(clear), \ INIT_ID(close), \ INIT_ID(closed), \ @@ -876,6 +1654,7 @@ extern "C" { INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ + INIT_ID(config), \ INIT_ID(consts), \ INIT_ID(context), \ INIT_ID(contravariant), \ @@ -886,11 +1665,15 @@ extern "C" { INIT_ID(coro), \ INIT_ID(count), \ INIT_ID(covariant), \ + INIT_ID(ctx), \ INIT_ID(cwd), \ INIT_ID(d_parameter_type), \ INIT_ID(data), \ INIT_ID(database), \ + INIT_ID(date), \ INIT_ID(day), \ + INIT_ID(days), \ + INIT_ID(debug), \ INIT_ID(decode), \ INIT_ID(decoder), \ INIT_ID(default), \ @@ -902,8 +1685,6 @@ extern "C" { INIT_ID(deterministic), \ INIT_ID(device), \ INIT_ID(dict), \ - INIT_ID(dict_content), \ - INIT_ID(dictcomp), \ INIT_ID(difference_update), \ INIT_ID(digest), \ INIT_ID(digest_size), \ @@ -921,6 +1702,7 @@ extern "C" { INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ + INIT_ID(emptyerror), \ INIT_ID(encode), \ INIT_ID(encoding), \ INIT_ID(end), \ @@ -933,17 +1715,20 @@ extern "C" { INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ + INIT_ID(exc), \ INIT_ID(exc_type), \ INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ INIT_ID(existing_file_name), \ + INIT_ID(exit), \ INIT_ID(exp), \ INIT_ID(expression), \ INIT_ID(extend), \ INIT_ID(extra_tokens), \ INIT_ID(facility), \ INIT_ID(factory), \ + INIT_ID(fallback), \ INIT_ID(false), \ INIT_ID(family), \ INIT_ID(fanout), \ @@ -970,15 +1755,17 @@ extern "C" { INIT_ID(format), \ INIT_ID(format_spec), \ INIT_ID(frame_buffer), \ + INIT_ID(free_threaded), \ INIT_ID(from_param), \ INIT_ID(fromlist), \ INIT_ID(fromtimestamp), \ INIT_ID(fromutc), \ INIT_ID(fset), \ + INIT_ID(fullerror), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(gc), \ INIT_ID(generation), \ - INIT_ID(genexpr), \ INIT_ID(get), \ INIT_ID(get_debug), \ INIT_ID(get_event_loop), \ @@ -999,13 +1786,16 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ + INIT_ID(hours), \ INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ INIT_ID(imag), \ + INIT_ID(implieslink), \ INIT_ID(importlib), \ INIT_ID(in_fd), \ + INIT_ID(include_aliases), \ INIT_ID(incoming), \ INIT_ID(index), \ INIT_ID(indexgroup), \ @@ -1052,9 +1842,9 @@ extern "C" { INIT_ID(kw), \ INIT_ID(kw1), \ INIT_ID(kw2), \ + INIT_ID(kwargs), \ INIT_ID(kwdefaults), \ INIT_ID(label), \ - INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_exc), \ INIT_ID(last_node), \ @@ -1063,6 +1853,7 @@ extern "C" { INIT_ID(last_value), \ INIT_ID(latin1), \ INIT_ID(leaf_size), \ + INIT_ID(legacy), \ INIT_ID(len), \ INIT_ID(length), \ INIT_ID(level), \ @@ -1070,7 +1861,6 @@ extern "C" { INIT_ID(line), \ INIT_ID(line_buffering), \ INIT_ID(lineno), \ - INIT_ID(listcomp), \ INIT_ID(little), \ INIT_ID(lo), \ INIT_ID(locale), \ @@ -1079,12 +1869,14 @@ extern "C" { INIT_ID(loop), \ INIT_ID(manual_reset), \ INIT_ID(mapping), \ + INIT_ID(mask), \ INIT_ID(match), \ INIT_ID(max_length), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ INIT_ID(maxlen), \ INIT_ID(maxmem), \ + INIT_ID(maxsize), \ INIT_ID(maxsplit), \ INIT_ID(maxvalue), \ INIT_ID(memLevel), \ @@ -1094,19 +1886,21 @@ extern "C" { INIT_ID(metadata), \ INIT_ID(method), \ INIT_ID(microsecond), \ + INIT_ID(microseconds), \ INIT_ID(milliseconds), \ INIT_ID(minute), \ + INIT_ID(minutes), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ INIT_ID(module_globals), \ INIT_ID(modules), \ + INIT_ID(modulo), \ INIT_ID(month), \ INIT_ID(mro), \ INIT_ID(msg), \ INIT_ID(mutex), \ INIT_ID(mycmp), \ - INIT_ID(n_arg), \ INIT_ID(n_fields), \ INIT_ID(n_sequence_fields), \ INIT_ID(n_unnamed_fields), \ @@ -1114,7 +1908,7 @@ extern "C" { INIT_ID(name_from), \ INIT_ID(namespace_separator), \ INIT_ID(namespaces), \ - INIT_ID(narg), \ + INIT_ID(native), \ INIT_ID(ndigits), \ INIT_ID(nested), \ INIT_ID(new_file_name), \ @@ -1137,6 +1931,7 @@ extern "C" { INIT_ID(offset_src), \ INIT_ID(on_type_read), \ INIT_ID(onceregistry), \ + INIT_ID(only_active_thread), \ INIT_ID(only_keys), \ INIT_ID(oparg), \ INIT_ID(opcode), \ @@ -1147,6 +1942,7 @@ extern "C" { INIT_ID(options), \ INIT_ID(order), \ INIT_ID(origin), \ + INIT_ID(other), \ INIT_ID(out_fd), \ INIT_ID(outgoing), \ INIT_ID(outpath), \ @@ -1164,21 +1960,26 @@ extern "C" { INIT_ID(person), \ INIT_ID(pi_factory), \ INIT_ID(pid), \ + INIT_ID(pointer_bits), \ INIT_ID(policy), \ INIT_ID(pos), \ INIT_ID(pos1), \ INIT_ID(pos2), \ INIT_ID(posix), \ + INIT_ID(prec), \ + INIT_ID(preserve_exc), \ INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ - INIT_ID(progress_handler), \ INIT_ID(progress_routine), \ INIT_ID(proto), \ INIT_ID(protocol), \ INIT_ID(ps1), \ INIT_ID(ps2), \ + INIT_ID(qid), \ + INIT_ID(qualname), \ INIT_ID(query), \ + INIT_ID(queuetype), \ INIT_ID(quotetabs), \ INIT_ID(raw), \ INIT_ID(read), \ @@ -1195,19 +1996,25 @@ extern "C" { INIT_ID(rel_tol), \ INIT_ID(release), \ INIT_ID(reload), \ + INIT_ID(repeat), \ INIT_ID(repl), \ INIT_ID(replace), \ + INIT_ID(reqrefs), \ + INIT_ID(require_ready), \ INIT_ID(reserved), \ INIT_ID(reset), \ INIT_ID(resetids), \ + INIT_ID(restrict), \ INIT_ID(return), \ INIT_ID(reverse), \ INIT_ID(reversed), \ + INIT_ID(rounding), \ INIT_ID(salt), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ INIT_ID(script), \ INIT_ID(second), \ + INIT_ID(seconds), \ INIT_ID(security_attributes), \ INIT_ID(seek), \ INIT_ID(seekable), \ @@ -1219,18 +2026,21 @@ extern "C" { INIT_ID(server_hostname), \ INIT_ID(server_side), \ INIT_ID(session), \ - INIT_ID(setcomp), \ INIT_ID(setpgroup), \ INIT_ID(setsid), \ INIT_ID(setsigdef), \ INIT_ID(setsigmask), \ INIT_ID(setstate), \ INIT_ID(shape), \ + INIT_ID(shared), \ + INIT_ID(short), \ INIT_ID(show_cmd), \ INIT_ID(signed), \ + INIT_ID(signum), \ INIT_ID(size), \ INIT_ID(sizehint), \ INIT_ID(skip_file_prefixes), \ + INIT_ID(skip_non_matching_threads), \ INIT_ID(sleep), \ INIT_ID(sock), \ INIT_ID(sort), \ @@ -1242,6 +2052,7 @@ extern "C" { INIT_ID(stacklevel), \ INIT_ID(start), \ INIT_ID(statement), \ + INIT_ID(stats), \ INIT_ID(status), \ INIT_ID(stderr), \ INIT_ID(stdin), \ @@ -1259,6 +2070,7 @@ extern "C" { INIT_ID(symmetric_difference_update), \ INIT_ID(tabsize), \ INIT_ID(tag), \ + INIT_ID(take_bytes), \ INIT_ID(target), \ INIT_ID(target_is_directory), \ INIT_ID(task), \ @@ -1270,18 +2082,22 @@ extern "C" { INIT_ID(template), \ INIT_ID(term), \ INIT_ID(text), \ + INIT_ID(third), \ INIT_ID(threading), \ INIT_ID(throw), \ + INIT_ID(time), \ INIT_ID(timeout), \ INIT_ID(timer), \ INIT_ID(times), \ + INIT_ID(timespec), \ + INIT_ID(timestamp), \ INIT_ID(timetuple), \ INIT_ID(timeunit), \ INIT_ID(top), \ - INIT_ID(trace_callback), \ INIT_ID(traceback), \ INIT_ID(trailers), \ INIT_ID(translate), \ + INIT_ID(traps), \ INIT_ID(true), \ INIT_ID(truncate), \ INIT_ID(twice), \ @@ -1292,8 +2108,10 @@ extern "C" { INIT_ID(tzinfo), \ INIT_ID(tzname), \ INIT_ID(uid), \ + INIT_ID(unboundop), \ INIT_ID(unlink), \ INIT_ID(unraisablehook), \ + INIT_ID(updates), \ INIT_ID(uri), \ INIT_ID(usedforsecurity), \ INIT_ID(value), \ @@ -1307,6 +2125,7 @@ extern "C" { INIT_ID(wbits), \ INIT_ID(week), \ INIT_ID(weekday), \ + INIT_ID(weeks), \ INIT_ID(which), \ INIT_ID(who), \ INIT_ID(withdata), \ diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 6bf3aae7175..995f49e78dc 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -106,7 +106,7 @@ struct _Py_cached_objects { }; // These would be in pycore_long.h if it weren't for an include cycle. -#define _PY_NSMALLPOSINTS 257 +#define _PY_NSMALLPOSINTS 1025 #define _PY_NSMALLNEGINTS 5 #include "pycore_global_strings.h" // struct _Py_global_strings @@ -223,9 +223,6 @@ struct pyruntimestate { struct _pythread_runtime_state threads; struct _signals_runtime_state signals; - /* Used for the thread state bound to the current thread. */ - Py_tss_t autoTSSkey; - /* Used instead of PyThreadState.trash when there is not current tstate. */ Py_tss_t trashTSSkey; @@ -279,12 +276,6 @@ struct pyruntimestate { struct _types_runtime_state types; struct _Py_time_runtime_state time; -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) - // Used in "Python/emscripten_trampoline.c" to choose between type - // reflection trampoline and EM_JS trampoline. - int (*emscripten_count_args_function)(PyCFunctionWithKeywords func); -#endif - /* All the objects that are shared by the runtime's interpreters. */ struct _Py_cached_objects cached_objects; struct _Py_static_objects static_objects; diff --git a/Include/internal/pycore_semaphore.h b/Include/internal/pycore_semaphore.h index 26953838460..66b4939dcac 100644 --- a/Include/internal/pycore_semaphore.h +++ b/Include/internal/pycore_semaphore.h @@ -46,10 +46,8 @@ typedef struct _PySemaphore { } _PySemaphore; // Puts the current thread to sleep until _PySemaphore_Wakeup() is called. -// If `detach` is true, then the thread will detach/release the GIL while -// sleeping. PyAPI_FUNC(int) -_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns, int detach); +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout_ns); // Wakes up a single thread waiting on sema. Note that _PySemaphore_Wakeup() // can be called before _PySemaphore_Wait(). diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index a2acf311ff4..e59611c07fa 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -50,26 +50,60 @@ extern "C" { CPython refcounting operations on it! */ +#define Py_INT_TAG 3 +#define Py_TAG_INVALID 2 +#define Py_TAG_REFCNT 1 +#define Py_TAG_BITS 3 + +#define Py_TAGGED_SHIFT 2 #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) -#define Py_TAG_BITS 0 - PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber); -PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber); +PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, uint16_t flags, const char *filename, int linenumber); PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(_PyStackRef) _Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber); +PyAPI_FUNC(void) _Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber); extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); static const _PyStackRef PyStackRef_NULL = { .index = 0 }; +static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) }; -// Use the first 3 even numbers for None, True and False. -// Odd numbers are reserved for (tagged) integers -#define PyStackRef_None ((_PyStackRef){ .index = 2 } ) -#define PyStackRef_False ((_PyStackRef){ .index = 4 }) -#define PyStackRef_True ((_PyStackRef){ .index = 6 }) +#define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } ) +#define PyStackRef_False ((_PyStackRef){ .index = (3 << Py_TAGGED_SHIFT) }) +#define PyStackRef_True ((_PyStackRef){ .index = (4 << Py_TAGGED_SHIFT) }) -#define INITIAL_STACKREF_INDEX 8 +#define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT) + +static inline _PyStackRef +PyStackRef_Wrap(void *ptr) +{ + assert(ptr != NULL); +#ifdef Py_DEBUG + assert(((uint64_t)ptr & Py_TAG_BITS) == 0); + return (_PyStackRef){ .index = ((uint64_t)ptr) | Py_TAG_INVALID }; +#else + return (_PyStackRef){ .index = (uint64_t)ptr }; +#endif +} + +static inline void * +PyStackRef_Unwrap(_PyStackRef ref) +{ +#ifdef Py_DEBUG + assert ((ref.index & Py_TAG_BITS) == Py_TAG_INVALID); + return (void *)(ref.index & ~Py_TAG_BITS); +#else + return (void *)(ref.index); +#endif +} + +static inline int +PyStackRef_RefcountOnObject(_PyStackRef ref) +{ + return (ref.index & Py_TAG_REFCNT) == 0; +} static inline int PyStackRef_IsNull(_PyStackRef ref) @@ -77,6 +111,25 @@ PyStackRef_IsNull(_PyStackRef ref) return ref.index == 0; } +static inline bool +PyStackRef_IsError(_PyStackRef ref) +{ + return ref.index == (1 << Py_TAGGED_SHIFT); +} + +static inline bool +PyStackRef_IsMalformed(_PyStackRef ref) +{ + return (ref.index & Py_TAG_BITS) == Py_TAG_INVALID; +} + +static inline bool +PyStackRef_IsValid(_PyStackRef ref) +{ + /* Invalid values are ERROR and NULL */ + return !PyStackRef_IsError(ref) && !PyStackRef_IsNull(ref); +} + static inline int PyStackRef_IsTrue(_PyStackRef ref) { @@ -95,67 +148,86 @@ PyStackRef_IsNone(_PyStackRef ref) return _Py_stackref_get_object(ref) == Py_None; } +static inline bool +PyStackRef_IsTaggedInt(_PyStackRef ref) +{ + return (ref.index & Py_TAG_BITS) == Py_INT_TAG; +} + static inline PyObject * _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber) { - assert((ref.index & 1) == 0); + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); _Py_stackref_record_borrow(ref, filename, linenumber); return _Py_stackref_get_object(ref); } - #define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__) static inline PyObject * _PyStackRef_AsPyObjectSteal(_PyStackRef ref, const char *filename, int linenumber) { - return _Py_stackref_close(ref, filename, linenumber); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + if (PyStackRef_RefcountOnObject(ref)) { + return obj; + } + return Py_NewRef(obj); } #define PyStackRef_AsPyObjectSteal(REF) _PyStackRef_AsPyObjectSteal((REF), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber) { - Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + assert(obj != NULL); + uint16_t flags = 0; + if (!_Py_IsImmortal(obj)) { + _Py_INCREF_MORTAL(obj); + } else { + flags = Py_TAG_REFCNT; + } + return _Py_stackref_create(obj, flags, filename, linenumber); } #define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber) { - return _Py_stackref_create(obj, filename, linenumber); + assert(obj != NULL); + uint16_t flags = 0; + if (_Py_IsImmortal(obj)) { + flags = Py_TAG_REFCNT; + } + return _Py_stackref_create(obj, flags, filename, linenumber); } #define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__) static inline _PyStackRef -_PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber) +_PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber) { - assert(_Py_IsImmortal(obj)); - return _Py_stackref_create(obj, filename, linenumber); -} -#define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__) - -static inline bool -PyStackRef_IsTaggedInt(_PyStackRef ref) -{ - return (ref.index & 1) == 1; + return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); } +#define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__) static inline void _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); if (PyStackRef_IsTaggedInt(ref)) { return; } PyObject *obj = _Py_stackref_close(ref, filename, linenumber); - Py_DECREF(obj); + assert(Py_REFCNT(obj) > 0); + if (PyStackRef_RefcountOnObject(ref)) { + Py_DECREF(obj); + } } #define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__) - static inline void _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); if (PyStackRef_IsNull(ref)) { return; } @@ -166,31 +238,57 @@ _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) static inline _PyStackRef _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); if (PyStackRef_IsTaggedInt(ref)) { return ref; } - else { - PyObject *obj = _Py_stackref_get_object(ref); + PyObject *obj = _Py_stackref_get_object(ref); + uint16_t flags = 0; + if (PyStackRef_RefcountOnObject(ref)) { Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + } else { + flags = Py_TAG_REFCNT; } + _PyStackRef new_ref = _Py_stackref_create(obj, flags, filename, linenumber); + if (flags == Py_TAG_REFCNT && !_Py_IsImmortal(obj)) { + _PyStackRef borrowed_from = _Py_stackref_get_borrowed_from(ref, filename, linenumber); + _Py_stackref_set_borrowed_from(new_ref, borrowed_from, filename, linenumber); + } + return new_ref; } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) -extern void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber); +static inline void +_PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + assert(Py_REFCNT(obj) > 0); + if (PyStackRef_RefcountOnObject(ref)) { + _Py_DECREF_SPECIALIZED(obj, destruct); + } +} #define PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT) _PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT, __FILE__, __LINE__) static inline _PyStackRef -PyStackRef_MakeHeapSafe(_PyStackRef ref) +_PyStackRef_Borrow(_PyStackRef ref, const char *filename, int linenumber) { - return ref; -} - -static inline _PyStackRef -PyStackRef_Borrow(_PyStackRef ref) -{ - return PyStackRef_DUP(ref); + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsNull(ref)); + if (PyStackRef_IsTaggedInt(ref)) { + return ref; + } + PyObject *obj = _Py_stackref_get_object(ref); + _PyStackRef new_ref = _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); + if (!_Py_IsImmortal(obj)) { + _Py_stackref_set_borrowed_from(new_ref, ref, filename, linenumber); + } + return new_ref; } +#define PyStackRef_Borrow(REF) _PyStackRef_Borrow((REF), __FILE__, __LINE__) #define PyStackRef_CLEAR(REF) \ do { \ @@ -203,28 +301,56 @@ PyStackRef_Borrow(_PyStackRef ref) static inline _PyStackRef _PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber) { + assert(obj != NULL); assert(!_Py_IsImmortal(obj)); - return _Py_stackref_create(obj, filename, linenumber); + return _Py_stackref_create(obj, 0, filename, linenumber); } #define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__) static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return true; + if ((ref.index & Py_TAG_BITS) != Py_TAG_REFCNT || PyStackRef_IsNull(ref)) { + // Tagged ints and ERROR are included. + return true; + } + + PyObject *obj = _Py_stackref_get_object(ref); + return _Py_IsImmortal(obj); } +static inline _PyStackRef +_PyStackRef_MakeHeapSafe(_PyStackRef ref, const char *filename, int linenumber) +{ + // Special references that can't be closed. + if (ref.index < INITIAL_STACKREF_INDEX) { + return ref; + } + + bool heap_safe = PyStackRef_IsHeapSafe(ref); + PyObject *obj = _Py_stackref_close(ref, filename, linenumber); + uint16_t flags = 0; + if (heap_safe) { + // Close old ref and create a new one with the same flags. + // This is necessary for correct borrow checking. + flags = ref.index & Py_TAG_BITS; + } else { + Py_INCREF(obj); + } + return _Py_stackref_create(obj, flags, filename, linenumber); +} +#define PyStackRef_MakeHeapSafe(REF) _PyStackRef_MakeHeapSafe(REF, __FILE__, __LINE__) + static inline _PyStackRef _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linenumber) { + assert(obj != NULL); assert(!_Py_IsStaticImmortal(obj)); Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + return _Py_stackref_create(obj, 0, filename, linenumber); } #define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj), __FILE__, __LINE__) -#define PyStackRef_RefcountOnObject(REF) 1 - extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b); extern bool PyStackRef_IsTaggedInt(_PyStackRef ref); @@ -233,44 +359,104 @@ extern intptr_t PyStackRef_UntagInt(_PyStackRef ref); extern _PyStackRef PyStackRef_TagInt(intptr_t i); +/* Increments a tagged int, but does not check for overflow */ +extern _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref); + extern bool PyStackRef_IsNullOrInt(_PyStackRef ref); #else -#define Py_INT_TAG 3 +static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; + +/* Wrap a pointer in a stack ref. + * The resulting stack reference is not safe and should only be used + * in the interpreter to pass values from one uop to another. + * The GC should never see one of these stack refs. */ +static inline _PyStackRef +PyStackRef_Wrap(void *ptr) +{ + assert(ptr != NULL); +#ifdef Py_DEBUG + assert(((uintptr_t)ptr & Py_TAG_BITS) == 0); + return (_PyStackRef){ .bits = ((uintptr_t)ptr) | Py_TAG_INVALID }; +#else + return (_PyStackRef){ .bits = (uintptr_t)ptr }; +#endif +} + +static inline void * +PyStackRef_Unwrap(_PyStackRef ref) +{ +#ifdef Py_DEBUG + assert ((ref.bits & Py_TAG_BITS) == Py_TAG_INVALID); + return (void *)(ref.bits & ~Py_TAG_BITS); +#else + return (void *)(ref.bits); +#endif +} + +static inline bool +PyStackRef_IsError(_PyStackRef ref) +{ + return ref.bits == Py_TAG_INVALID; +} + +static inline bool +PyStackRef_IsMalformed(_PyStackRef ref) +{ + return (ref.bits & Py_TAG_BITS) == Py_TAG_INVALID; +} + +static inline bool +PyStackRef_IsValid(_PyStackRef ref) +{ + /* Invalid values are ERROR and NULL */ + return ref.bits >= Py_INT_TAG; +} static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) { - return (i.bits & Py_INT_TAG) == Py_INT_TAG; + return (i.bits & Py_TAG_BITS) == Py_INT_TAG; } static inline _PyStackRef PyStackRef_TagInt(intptr_t i) { - assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i); - return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) }; + assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)(((uintptr_t)i) << Py_TAGGED_SHIFT), + Py_TAGGED_SHIFT) == i); + return (_PyStackRef){ .bits = ((((uintptr_t)i) << Py_TAGGED_SHIFT) | Py_INT_TAG) }; } static inline intptr_t PyStackRef_UntagInt(_PyStackRef i) { - assert((i.bits & Py_INT_TAG) == Py_INT_TAG); + assert(PyStackRef_IsTaggedInt(i)); intptr_t val = (intptr_t)i.bits; - return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2); + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, Py_TAGGED_SHIFT); } +static inline _PyStackRef +PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) +{ + assert((ref.bits & Py_TAG_BITS) == Py_INT_TAG); // Is tagged int + assert((ref.bits & (~Py_TAG_BITS)) != (INTPTR_MAX & (~Py_TAG_BITS))); // Isn't about to overflow + return (_PyStackRef){ .bits = ref.bits + (1 << Py_TAGGED_SHIFT) }; +} + +#define PyStackRef_IsDeferredOrTaggedInt(ref) (((ref).bits & Py_TAG_REFCNT) != 0) + #ifdef Py_GIL_DISABLED -#define Py_TAG_DEFERRED (1) +#define Py_TAG_DEFERRED Py_TAG_REFCNT #define Py_TAG_PTR ((uintptr_t)0) -#define Py_TAG_BITS ((uintptr_t)1) static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; + #define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) #define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) @@ -286,6 +472,7 @@ static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; static inline PyObject * PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) { + assert(!PyStackRef_IsTaggedInt(stackref)); PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); return cleared; } @@ -365,21 +552,20 @@ PyStackRef_FromPyObjectNew(PyObject *obj) #define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) static inline _PyStackRef -PyStackRef_FromPyObjectImmortal(PyObject *obj) +PyStackRef_FromPyObjectBorrow(PyObject *obj) { // Make sure we don't take an already tagged value. assert(((uintptr_t)obj & Py_TAG_BITS) == 0); assert(obj != NULL); - assert(_Py_IsImmortal(obj)); return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; } -#define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) +#define PyStackRef_FromPyObjectBorrow(obj) PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj)) #define PyStackRef_CLOSE(REF) \ do { \ _PyStackRef _close_tmp = (REF); \ assert(!PyStackRef_IsNull(_close_tmp)); \ - if (!PyStackRef_IsDeferred(_close_tmp)) { \ + if (!PyStackRef_IsDeferredOrTaggedInt(_close_tmp)) { \ Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ } \ } while (0) @@ -391,11 +577,17 @@ PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) PyStackRef_CLOSE(ref); } +static inline int +PyStackRef_RefcountOnObject(_PyStackRef ref) +{ + return (ref.bits & Py_TAG_REFCNT) == 0; +} + static inline _PyStackRef PyStackRef_DUP(_PyStackRef stackref) { assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferred(stackref)) { + if (PyStackRef_IsDeferredOrTaggedInt(stackref)) { return stackref; } Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); @@ -442,14 +634,12 @@ PyStackRef_AsStrongReference(_PyStackRef stackref) /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT * as they can (must) have their reclamation deferred */ -#define Py_TAG_BITS 1 -#define Py_TAG_REFCNT 1 #if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT # error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT" #endif #define BITS_TO_PTR(REF) ((PyObject *)((REF).bits)) -#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_BITS))) +#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_REFCNT))) #define PyStackRef_NULL_BITS Py_TAG_REFCNT static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS }; @@ -529,7 +719,7 @@ PyStackRef_FromPyObjectSteal(PyObject *obj) { assert(obj != NULL); #if SIZEOF_VOID_P > 4 - unsigned int tag = obj->ob_flags & Py_TAG_BITS; + unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; #else unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0; #endif @@ -548,12 +738,6 @@ PyStackRef_FromPyObjectStealMortal(PyObject *obj) return ref; } -// Check if a stackref is exactly the same as another stackref, including the -// the deferred bit. This can only be used safely if you know that the deferred -// bits of `a` and `b` match. -#define PyStackRef_IsExactly(a, b) \ - (assert(((a).bits & Py_TAG_BITS) == ((b).bits & Py_TAG_BITS)), (a).bits == (b).bits) - static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj) { @@ -561,7 +745,7 @@ _PyStackRef_FromPyObjectNew(PyObject *obj) if (_Py_IsImmortal(obj)) { return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT}; } - Py_INCREF_MORTAL(obj); + _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; @@ -572,7 +756,7 @@ static inline _PyStackRef _PyStackRef_FromPyObjectNewMortal(PyObject *obj) { assert(obj != NULL); - Py_INCREF_MORTAL(obj); + _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; @@ -581,23 +765,22 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj) /* Create a new reference from an object with an embedded reference count */ static inline _PyStackRef -PyStackRef_FromPyObjectImmortal(PyObject *obj) +PyStackRef_FromPyObjectBorrow(PyObject *obj) { - assert(_Py_IsImmortal(obj)); return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; } /* WARNING: This macro evaluates its argument more than once */ #ifdef _WIN32 #define PyStackRef_DUP(REF) \ - (PyStackRef_RefcountOnObject(REF) ? (Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF)) + (PyStackRef_RefcountOnObject(REF) ? (_Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF)) #else static inline _PyStackRef PyStackRef_DUP(_PyStackRef ref) { assert(!PyStackRef_IsNull(ref)); if (PyStackRef_RefcountOnObject(ref)) { - Py_INCREF_MORTAL(BITS_TO_PTR(ref)); + _Py_INCREF_MORTAL(BITS_TO_PTR(ref)); } return ref; } @@ -606,7 +789,7 @@ PyStackRef_DUP(_PyStackRef ref) static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return (ref.bits & Py_TAG_BITS) == 0 || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); + return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); } static inline _PyStackRef @@ -681,12 +864,18 @@ PyStackRef_XCLOSE(_PyStackRef ref) // Note: this is a macro because MSVC (Windows) has trouble inlining it. -#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_BITS)) == ((b).bits & (~Py_TAG_BITS))) +#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT))) #endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) -#define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) +static inline PyTypeObject * +PyStackRef_TYPE(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return &PyLong_Type; + } + return Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)); +} // Converts a PyStackRef back to a PyObject *, converting the // stackref to a new reference. @@ -694,42 +883,30 @@ PyStackRef_XCLOSE(_PyStackRef ref) // StackRef type checks -static inline bool -PyStackRef_GenCheck(_PyStackRef stackref) -{ - return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} +#define STACKREF_CHECK_FUNC(T) \ + static inline bool \ + PyStackRef_ ## T ## Check(_PyStackRef stackref) { \ + if (PyStackRef_IsTaggedInt(stackref)) { \ + return false; \ + } \ + return Py ## T ## _Check(PyStackRef_AsPyObjectBorrow(stackref)); \ + } -static inline bool -PyStackRef_BoolCheck(_PyStackRef stackref) -{ - return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} +STACKREF_CHECK_FUNC(Gen) +STACKREF_CHECK_FUNC(Bool) +STACKREF_CHECK_FUNC(ExceptionInstance) +STACKREF_CHECK_FUNC(Code) +STACKREF_CHECK_FUNC(Function) static inline bool PyStackRef_LongCheck(_PyStackRef stackref) { + if (PyStackRef_IsTaggedInt(stackref)) { + return true; + } return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); } -static inline bool -PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) -{ - return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - -static inline bool -PyStackRef_CodeCheck(_PyStackRef stackref) -{ - return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - -static inline bool -PyStackRef_FunctionCheck(_PyStackRef stackref) -{ - return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); -} - static inline void _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) { @@ -771,7 +948,7 @@ _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) static inline int _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) { - PyObject *op = _Py_atomic_load_ptr_relaxed(src); + PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src)); if (op == NULL) { *out = PyStackRef_NULL; return 1; @@ -781,6 +958,13 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) #endif +#define PyStackRef_XSETREF(dst, src) \ + do { \ + _PyStackRef _tmp_dst_ref = (dst); \ + (dst) = (src); \ + PyStackRef_XCLOSE(_tmp_dst_ref); \ + } while(0) + // Like Py_VISIT but for _PyStackRef fields #define _Py_VISIT_STACKREF(ref) \ do { \ diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h index ab649574f33..850e6ea4552 100644 --- a/Include/internal/pycore_stats.h +++ b/Include/internal/pycore_stats.h @@ -15,39 +15,56 @@ extern "C" { #include "pycore_bitutils.h" // _Py_bit_length -#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0) -#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0) -#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) -#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) -#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) -#define OBJECT_STAT_INC_COND(name, cond) \ - do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) -#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ - do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0) -#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) -#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) -#define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0) -#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) -#define UOP_PAIR_INC(uopcode, lastuop) \ - do { \ - if (lastuop && _Py_stats) { \ - _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ - } \ - lastuop = uopcode; \ - } while (0) -#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) -#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) -#define OPT_HIST(length, name) \ +#define STAT_INC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name++) +#define STAT_DEC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name--) +#define OPCODE_EXE_INC(opname) _Py_STATS_EXPR(opcode_stats[opname].execution_count++) +#define CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.name++) +#define OBJECT_STAT_INC(name) _Py_STATS_EXPR(object_stats.name++) +#define OBJECT_STAT_INC_COND(name, cond) _Py_STATS_COND_EXPR(cond, object_stats.name++) +#define EVAL_CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.eval_calls[name]++) +#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) _Py_STATS_COND_EXPR(PyFunction_Check(callable), call_stats.eval_calls[name]++) +#define GC_STAT_ADD(gen, name, n) _Py_STATS_EXPR(gc_stats[(gen)].name += (n)) +#define OPT_STAT_INC(name) _Py_STATS_EXPR(optimization_stats.name++) +#define OPT_STAT_ADD(name, n) _Py_STATS_EXPR(optimization_stats.name += (n)) +#define UOP_STAT_INC(opname, name) \ do { \ - if (_Py_stats) { \ - int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ - bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ - _Py_stats->optimization_stats.name[bucket]++; \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + assert(opname < 512); \ + s->optimization_stats.opcode[opname].name++; \ } \ } while (0) -#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0) -#define OPCODE_DEFERRED_INC(opname) do { if (_Py_stats && opcode == opname) _Py_stats->opcode_stats[opname].specialization.deferred++; } while (0) +#define UOP_PAIR_INC(uopcode, lastuop) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (lastuop && s) { \ + s->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \ + } \ + lastuop = uopcode; \ + } while (0) +#define OPT_UNSUPPORTED_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.unsupported_opcode[opname]++) +#define OPT_ERROR_IN_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.error_in_opcode[opname]++) +#define OPT_HIST(length, name) \ + do { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ + int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \ + bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \ + s->optimization_stats.name[bucket]++; \ + } \ + } while (0) +#define RARE_EVENT_STAT_INC(name) _Py_STATS_EXPR(rare_event_stats.name++) +#define OPCODE_DEFERRED_INC(opname) _Py_STATS_COND_EXPR(opcode==opname, opcode_stats[opname].specialization.deferred++) + +#ifdef Py_GIL_DISABLED +#define FT_STAT_MUTEX_SLEEP_INC() _Py_STATS_EXPR(ft_stats.mutex_sleeps++) +#define FT_STAT_QSBR_POLL_INC() _Py_STATS_EXPR(ft_stats.qsbr_polls++) +#define FT_STAT_WORLD_STOP_INC() _Py_STATS_EXPR(ft_stats.world_stops++) +#else +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() +#endif // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -71,13 +88,16 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) #define OPCODE_DEFERRED_INC(opname) ((void)0) +#define FT_STAT_MUTEX_SLEEP_INC() +#define FT_STAT_QSBR_POLL_INC() +#define FT_STAT_WORLD_STOP_INC() #endif // !Py_STATS #define RARE_EVENT_INTERP_INC(interp, name) \ do { \ /* saturating add */ \ - int val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ + uint8_t val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ if (val < UINT8_MAX) { \ FT_ATOMIC_STORE_UINT8(interp->rare_events.name, val + 1); \ } \ @@ -90,6 +110,11 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); RARE_EVENT_INTERP_INC(interp, name); \ } while (0); \ +PyStatus _PyStats_InterpInit(PyInterpreterState *); +bool _PyStats_ThreadInit(PyInterpreterState *, _PyThreadStateImpl *); +void _PyStats_ThreadFini(_PyThreadStateImpl *); +void _PyStats_Attach(_PyThreadStateImpl *); +void _PyStats_Detach(_PyThreadStateImpl *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 98099b4a497..9dbfa913219 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -188,7 +188,8 @@ extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, PyObject *filename, int start, - PyCompilerFlags *flags); + PyCompilerFlags *flags, + PyObject *module); int _PyFuture_FromAST( struct _mod * mod, diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 008a2da0d04..347b0a7a790 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -8,11 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -PyAPI_FUNC(int) _PySys_GetOptionalAttr(PyObject *, PyObject **); -PyAPI_FUNC(int) _PySys_GetOptionalAttrString(const char *, PyObject **); -PyAPI_FUNC(PyObject *) _PySys_GetRequiredAttr(PyObject *); -PyAPI_FUNC(PyObject *) _PySys_GetRequiredAttrString(const char *); - // Export for '_pickle' shared extension PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 23312471c65..b671225ca6e 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -147,11 +147,6 @@ extern int _PyTime_FromSecondsDouble( // Clamp to [PyTime_MIN; PyTime_MAX] on overflow. extern PyTime_t _PyTime_FromMicrosecondsClamp(PyTime_t us); -// Create a timestamp from a Python int object (number of nanoseconds). -// Export for '_lsprof' shared extension. -PyAPI_FUNC(int) _PyTime_FromLong(PyTime_t *t, - PyObject *obj); - // Convert a number of seconds (Python float or int) to a timestamp. // Raise an exception and return -1 on error, return 0 on success. // Export for '_socket' shared extension. @@ -182,10 +177,6 @@ extern PyTime_t _PyTime_As100Nanoseconds(PyTime_t t, _PyTime_round_t round); #endif -// Convert a timestamp (number of nanoseconds) as a Python int object. -// Export for '_testinternalcapi' shared extension. -PyAPI_FUNC(PyObject*) _PyTime_AsLong(PyTime_t t); - #ifndef MS_WINDOWS // Create a timestamp from a timeval structure. // Raise an exception and return -1 on overflow, return 0 on success. diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index d71dd288699..8357cce9d89 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -100,8 +100,11 @@ extern int _Py_WriteIndentedMargin(int, const char*, PyObject *); extern int _Py_WriteIndent(int, PyObject *); // Export for the faulthandler module +PyAPI_FUNC(void) _Py_InitDumpStack(void); PyAPI_FUNC(void) _Py_DumpStack(int fd); +extern void _Py_DumpTraceback_Init(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 572e8025876..693385f9a46 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -30,8 +30,8 @@ struct _PyTraceMalloc_Config { }; -/* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. */ +/* Pack the tracemalloc_frame and tracemalloc_traceback structures to reduce + the memory footprint on 64-bit architectures: 12 bytes instead of 16. */ #if defined(_MSC_VER) #pragma pack(push, 4) #endif @@ -46,18 +46,22 @@ tracemalloc_frame { PyObject *filename; unsigned int lineno; }; -#ifdef _MSC_VER -#pragma pack(pop) -#endif -struct tracemalloc_traceback { +struct +#ifdef __GNUC__ +__attribute__((packed)) +#endif +tracemalloc_traceback { Py_uhash_t hash; /* Number of frames stored */ uint16_t nframe; /* Total number of frames the traceback had */ uint16_t total_nframe; - struct tracemalloc_frame frames[1]; + struct tracemalloc_frame frames[]; }; +#ifdef _MSC_VER +#pragma pack(pop) +#endif struct _tracemalloc_runtime_state { @@ -95,7 +99,7 @@ struct _tracemalloc_runtime_state { Protected by TABLES_LOCK(). */ _Py_hashtable_t *domains; - struct tracemalloc_traceback empty_traceback; + struct tracemalloc_traceback *empty_traceback; Py_tss_t reentrant_key; }; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index bad968428c7..c4f723ac8ab 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -10,9 +10,11 @@ extern "C" { #include "pycore_brc.h" // struct _brc_thread_state #include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_interpframe_structs.h" // _PyInterpreterFrame #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr - +#include "pycore_uop.h" // struct _PyUOpInstruction +#include "pycore_structs.h" #ifdef Py_GIL_DISABLED struct _gc_thread_state { @@ -21,6 +23,38 @@ struct _gc_thread_state { }; #endif +#if _Py_TIER2 +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +typedef struct _PyJitTracerPreviousState { + bool dependencies_still_valid; + bool instr_is_super; + int code_max_size; + int code_curr_size; + int instr_oparg; + int instr_stacklevel; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + struct _PyInterpreterFrame *instr_frame; + _PyBloomFilter dependencies; +} _PyJitTracerPreviousState; + +typedef struct _PyJitTracerState { + _PyUOpInstruction *code_buffer; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; +} _PyJitTracerState; +#endif + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -28,6 +62,10 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + // Embedded base frame - sentinel at the bottom of the frame stack. + // Used by profiling/sampling to detect incomplete stack traces. + _PyInterpreterFrame base_frame; + // The reference count field is used to synchronize deallocation of the // thread state during runtime finalization. Py_ssize_t refcount; @@ -37,6 +75,10 @@ typedef struct _PyThreadStateImpl { uintptr_t c_stack_soft_limit; uintptr_t c_stack_hard_limit; + // PyUnstable_ThreadState_ResetStackProtection() values + uintptr_t c_stack_init_base; + uintptr_t c_stack_init_top; + PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference @@ -70,12 +112,20 @@ typedef struct _PyThreadStateImpl { // When >1, code objects do not immortalize their non-string constants. int suppress_co_const_immortalization; + +#ifdef Py_STATS + // per-thread stats, will be merged into interp->pystats_struct + PyStats *pystats_struct; // allocated by _PyStats_ThreadInit() #endif +#endif // Py_GIL_DISABLED + #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif - +#if _Py_TIER2 + _PyJitTracerState jit_tracer_state; +#endif } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index acf1bec4602..46db02593ad 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -23,7 +23,6 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) -PyAPI_FUNC(PyObject *)_PyTuple_FromArray(PyObject *const *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 1a4f89fd244..3661f171e2b 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -40,6 +40,7 @@ extern void _PyTypes_FiniTypes(PyInterpreterState *); extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); extern void _PyTypes_AfterFork(void); +extern void _PyTypes_FiniCachedDescriptors(PyInterpreterState *); static inline PyObject ** _PyStaticType_GET_WEAKREFS_LISTPTR(managed_static_type_state *state) @@ -89,6 +90,9 @@ _PyType_GetModuleState(PyTypeObject *type) // function PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_LookupSubclasses(PyTypeObject *); +PyAPI_FUNC(PyObject *) _PyType_InitSubclasses(PyTypeObject *); + extern PyObject * _PyType_GetBases(PyTypeObject *type); extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); @@ -134,7 +138,6 @@ extern int _PyType_AddMethod(PyTypeObject *, PyMethodDef *); extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags); -extern unsigned int _PyType_GetVersionForCurrentState(PyTypeObject *tp); PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); PyTypeObject *_PyType_LookupByVersion(unsigned int version); @@ -149,6 +152,9 @@ typedef int (*_py_validate_type)(PyTypeObject *); extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); +// Precalculates count of non-unique slots and fills wrapperbase.name_count. +extern int _PyType_InitSlotDefs(PyInterpreterState *interp); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodectype.h b/Include/internal/pycore_unicodectype.h new file mode 100644 index 00000000000..523bdb56b09 --- /dev/null +++ b/Include/internal/pycore_unicodectype.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_UNICODECTYPE_H +#define Py_INTERNAL_UNICODECTYPE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res); +extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch); +extern int _PyUnicode_IsCased(Py_UCS4 ch); + +// Export for 'unicodedata' shared extension. +PyAPI_FUNC(int) _PyUnicode_IsXidStart(Py_UCS4 ch); +PyAPI_FUNC(int) _PyUnicode_IsXidContinue(Py_UCS4 ch); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNICODECTYPE_H */ diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index c85d53b89ac..97dda73f9b5 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -11,16 +11,108 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI -/* --- Characters Type APIs ----------------------------------------------- */ -extern int _PyUnicode_IsXidStart(Py_UCS4 ch); -extern int _PyUnicode_IsXidContinue(Py_UCS4 ch); -extern int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res); -extern int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch); -extern int _PyUnicode_IsCased(Py_UCS4 ch); +// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). +#define _Py_MAX_UNICODE 0x10ffff + + +extern int _PyUnicode_IsModifiable(PyObject *unicode); +extern void _PyUnicodeWriter_InitWithBuffer( + _PyUnicodeWriter *writer, + PyObject *buffer); +extern PyObject* _PyUnicode_Result(PyObject *unicode); +extern int _PyUnicode_DecodeUTF8Writer( + _PyUnicodeWriter *writer, + const char *s, + Py_ssize_t size, + _Py_error_handler error_handler, + const char *errors, + Py_ssize_t *consumed); +extern PyObject* _PyUnicode_ResizeCompact( + PyObject *unicode, + Py_ssize_t length); +extern PyObject* _PyUnicode_GetEmpty(void); + + +/* Generic helper macro to convert characters of different types. + from_type and to_type have to be valid type names, begin and end + are pointers to the source characters which should be of type + "from_type *". to is a pointer of type "to_type *" and points to the + buffer where the result characters are written to. */ +#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ + do { \ + to_type *_to = (to_type *)(to); \ + const from_type *_iter = (const from_type *)(begin);\ + const from_type *_end = (const from_type *)(end);\ + Py_ssize_t n = (_end) - (_iter); \ + const from_type *_unrolled_end = \ + _iter + _Py_SIZE_ROUND_DOWN(n, 4); \ + while (_iter < (_unrolled_end)) { \ + _to[0] = (to_type) _iter[0]; \ + _to[1] = (to_type) _iter[1]; \ + _to[2] = (to_type) _iter[2]; \ + _to[3] = (to_type) _iter[3]; \ + _iter += 4; _to += 4; \ + } \ + while (_iter < (_end)) \ + *_to++ = (to_type) *_iter++; \ + } while (0) + + +static inline void +_PyUnicode_Fill(int kind, void *data, Py_UCS4 value, + Py_ssize_t start, Py_ssize_t length) +{ + assert(0 <= start); + switch (kind) { + case PyUnicode_1BYTE_KIND: { + assert(value <= 0xff); + Py_UCS1 ch = (unsigned char)value; + Py_UCS1 *to = (Py_UCS1 *)data + start; + memset(to, ch, length); + break; + } + case PyUnicode_2BYTE_KIND: { + assert(value <= 0xffff); + Py_UCS2 ch = (Py_UCS2)value; + Py_UCS2 *to = (Py_UCS2 *)data + start; + const Py_UCS2 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + case PyUnicode_4BYTE_KIND: { + assert(value <= _Py_MAX_UNICODE); + Py_UCS4 ch = value; + Py_UCS4 * to = (Py_UCS4 *)data + start; + const Py_UCS4 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + default: Py_UNREACHABLE(); + } +} + +static inline int +_PyUnicode_EnsureUnicode(PyObject *obj) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "must be str, not %T", obj); + return -1; + } + return 0; +} + +static inline int +_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) +{ + assert(ch <= _Py_MAX_UNICODE); + if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) + return -1; + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); + writer->pos++; + return 0; +} /* --- Unicode API -------------------------------------------------------- */ @@ -82,12 +174,16 @@ extern int _PyUnicode_FormatAdvancedWriter( Py_ssize_t start, Py_ssize_t end); +/* PyUnicodeWriter_Format, with va_list instead of `...` */ +extern int _PyUnicodeWriter_FormatV( + PyUnicodeWriter *writer, + const char *format, + va_list vargs); + /* --- UTF-7 Codecs ------------------------------------------------------- */ extern PyObject* _PyUnicode_EncodeUTF7( PyObject *unicode, /* Unicode object */ - int base64SetO, /* Encode RFC2152 Set O characters in base64 */ - int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ const char *errors); /* error handling */ /* --- UTF-8 Codecs ------------------------------------------------------- */ @@ -139,14 +235,18 @@ extern PyObject* _PyUnicode_DecodeUnicodeEscapeStateful( // Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape // chars. // Export for test_peg_generator. -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal2( const char *string, /* Unicode-Escape encoded string */ Py_ssize_t length, /* size of string */ const char *errors, /* error handling */ Py_ssize_t *consumed, /* bytes consumed */ - const char **first_invalid_escape); /* on return, points to first - invalid escaped char in - string. */ + int *first_invalid_escape_char, /* on return, if not -1, contain the first + invalid escaped char (<= 0xff) or invalid + octal escape (> 0xff) in string. */ + const char **first_invalid_escape_ptr); /* on return, if not NULL, may + point to the first invalid escaped + char in string. + May be NULL if errors is not NULL. */ /* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */ @@ -207,14 +307,6 @@ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( Py_ssize_t seqlen ); -/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, - 0 otherwise. The right argument must be ASCII identifier. - Any error occurs inside will be cleared before return. */ -extern int _PyUnicode_EqualToASCIIId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); - // Test whether a unicode is equal to ASCII string. Return 1 if true, // 0 otherwise. The right argument must be ASCII-encoded string. // Any error occurs inside will be cleared before return. @@ -232,21 +324,6 @@ extern PyObject* _PyUnicode_XStrip( ); -/* Using explicit passed-in values, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ -extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar, - int forward); - /* Dedent a string. Behaviour is expected to be an exact match of `textwrap.dedent`. Return a new reference on success, NULL with exception set on error. diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index fefacef77c8..c5b01ff9876 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -16,6 +16,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(Emax); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(Emin); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(FINISHED); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -920,10 +928,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(aggregate_class); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(alias); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -936,6 +940,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(allow_code); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -988,10 +996,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(authorizer_callback); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(autocommit); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1028,6 +1032,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(blocking); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bound); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1096,6 +1104,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(c_stack); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cache_frames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_datetime_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1124,6 +1140,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(callable); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(callback); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1136,6 +1156,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(capitals); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(category); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1148,10 +1172,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(chain); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(check_same_thread); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(clamp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(clear); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1264,6 +1296,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(config); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(consts); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1304,6 +1340,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(ctx); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cwd); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1320,10 +1360,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(date); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(day); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(days); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(debug); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decode); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1368,14 +1420,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(dict_content); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(dictcomp); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(difference_update); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1444,6 +1488,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(emptyerror); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(encode); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1492,6 +1540,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_type); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1512,6 +1564,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exit); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1536,6 +1592,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fallback); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(false); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1640,6 +1700,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(free_threaded); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(from_param); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1660,6 +1724,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(fullerror); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(func); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1668,11 +1736,11 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(generation); + string = &_Py_ID(gc); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(genexpr); + string = &_Py_ID(generation); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); @@ -1756,6 +1824,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(hours); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(id); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1776,6 +1848,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(implieslink); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(importlib); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1784,6 +1860,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(include_aliases); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(incoming); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1968,6 +2048,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(kwargs); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(kwdefaults); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1976,10 +2060,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(lambda); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2012,6 +2092,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(legacy); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(len); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2040,10 +2124,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(listcomp); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(little); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2076,6 +2156,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mask); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(match); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2100,6 +2184,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(maxsize); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxsplit); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2136,6 +2224,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(microseconds); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(milliseconds); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2144,6 +2236,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(minutes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2164,6 +2260,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(modulo); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(month); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2184,10 +2284,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(n_arg); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(n_fields); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2216,7 +2312,7 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(narg); + string = &_Py_ID(native); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); @@ -2308,6 +2404,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(only_active_thread); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(only_keys); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2348,6 +2448,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(other); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(out_fd); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2416,6 +2520,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pointer_bits); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(policy); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2436,6 +2544,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(prec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(preserve_exc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(print_file_and_line); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2448,10 +2564,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(progress_handler); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_routine); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2472,10 +2584,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(qid); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(qualname); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(query); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(queuetype); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(quotetabs); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2540,6 +2664,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(repeat); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(repl); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2548,6 +2676,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(reqrefs); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(require_ready); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reserved); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2560,6 +2696,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(restrict); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(return); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2572,6 +2712,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(rounding); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(salt); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2592,6 +2736,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(seconds); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(security_attributes); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2636,10 +2784,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(setcomp); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setpgroup); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2664,6 +2808,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(shared); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(short); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(show_cmd); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2672,6 +2824,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(signum); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(size); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2684,6 +2840,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(skip_non_matching_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sleep); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2728,6 +2888,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stats); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(status); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2796,6 +2960,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(take_bytes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2840,6 +3008,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(third); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(threading); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2848,6 +3020,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(time); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timeout); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2860,6 +3036,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timespec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timestamp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2872,10 +3056,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(trace_callback); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(traceback); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2888,6 +3068,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(traps); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(true); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2928,6 +3112,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(unboundop); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(unlink); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2936,6 +3124,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(updates); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(uri); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2988,6 +3180,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(weeks); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(which); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3060,6 +3256,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_null); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3084,6 +3284,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_setcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h new file mode 100644 index 00000000000..70576046385 --- /dev/null +++ b/Include/internal/pycore_uop.h @@ -0,0 +1,58 @@ +#ifndef Py_CORE_UOP_H +#define Py_CORE_UOP_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +/* Depending on the format, + * the 32 bits between the oparg and operand are: + * UOP_FORMAT_TARGET: + * uint32_t target; + * UOP_FORMAT_JUMP + * uint16_t jump_target; + * uint16_t error_target; + */ +typedef struct _PyUOpInstruction{ + uint16_t opcode:15; + uint16_t format:1; + uint16_t oparg; + union { + uint32_t target; + struct { + uint16_t jump_target; + uint16_t error_target; + }; + }; + uint64_t operand0; // A cache entry + uint64_t operand1; +#ifdef Py_STATS + uint64_t execution_count; +#endif +} _PyUOpInstruction; + +// This is the length of the trace we translate initially. +#ifdef Py_DEBUG + // With asserts, the stencils are a lot larger +#define UOP_MAX_TRACE_LENGTH 1000 +#else +#define UOP_MAX_TRACE_LENGTH 3000 +#endif +#define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) + +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UOP_H */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d3d2b716e2c..c38f28f9db1 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,22 +13,25 @@ extern "C" { #define _SET_IP 301 #define _BINARY_OP 302 #define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_ADD_UNICODE 305 -#define _BINARY_OP_EXTEND 306 -#define _BINARY_OP_INPLACE_ADD_UNICODE 307 -#define _BINARY_OP_MULTIPLY_FLOAT 308 -#define _BINARY_OP_MULTIPLY_INT 309 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 310 -#define _BINARY_OP_SUBSCR_DICT 311 -#define _BINARY_OP_SUBSCR_INIT_CALL 312 -#define _BINARY_OP_SUBSCR_LIST_INT 313 -#define _BINARY_OP_SUBSCR_LIST_SLICE 314 -#define _BINARY_OP_SUBSCR_STR_INT 315 -#define _BINARY_OP_SUBSCR_TUPLE_INT 316 -#define _BINARY_OP_SUBTRACT_FLOAT 317 -#define _BINARY_OP_SUBTRACT_INT 318 -#define _BINARY_SLICE 319 +#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS 304 +#define _BINARY_OP_ADD_INT 305 +#define _BINARY_OP_ADD_UNICODE 306 +#define _BINARY_OP_EXTEND 307 +#define _BINARY_OP_INPLACE_ADD_UNICODE 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS 310 +#define _BINARY_OP_MULTIPLY_INT 311 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 312 +#define _BINARY_OP_SUBSCR_DICT 313 +#define _BINARY_OP_SUBSCR_INIT_CALL 314 +#define _BINARY_OP_SUBSCR_LIST_INT 315 +#define _BINARY_OP_SUBSCR_LIST_SLICE 316 +#define _BINARY_OP_SUBSCR_STR_INT 317 +#define _BINARY_OP_SUBSCR_TUPLE_INT 318 +#define _BINARY_OP_SUBTRACT_FLOAT 319 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 320 +#define _BINARY_OP_SUBTRACT_INT 321 +#define _BINARY_SLICE 322 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -37,131 +40,148 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 320 -#define _CALL_BUILTIN_FAST 321 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 322 -#define _CALL_BUILTIN_O 323 +#define _CALL_BUILTIN_CLASS 323 +#define _CALL_BUILTIN_FAST 324 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 +#define _CALL_BUILTIN_O 326 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_KW_NON_PY 324 -#define _CALL_LEN 325 -#define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 326 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 327 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 328 -#define _CALL_METHOD_DESCRIPTOR_O 329 -#define _CALL_NON_PY_GENERAL 330 -#define _CALL_STR_1 331 -#define _CALL_TUPLE_1 332 -#define _CALL_TYPE_1 333 -#define _CHECK_AND_ALLOCATE_OBJECT 334 -#define _CHECK_ATTR_CLASS 335 -#define _CHECK_ATTR_METHOD_LAZY_DICT 336 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 337 +#define _CALL_ISINSTANCE 327 +#define _CALL_KW_NON_PY 328 +#define _CALL_LEN 329 +#define _CALL_LIST_APPEND 330 +#define _CALL_METHOD_DESCRIPTOR_FAST 331 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 +#define _CALL_METHOD_DESCRIPTOR_O 334 +#define _CALL_NON_PY_GENERAL 335 +#define _CALL_STR_1 336 +#define _CALL_TUPLE_1 337 +#define _CALL_TYPE_1 338 +#define _CHECK_AND_ALLOCATE_OBJECT 339 +#define _CHECK_ATTR_CLASS 340 +#define _CHECK_ATTR_METHOD_LAZY_DICT 341 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 338 -#define _CHECK_FUNCTION_EXACT_ARGS 339 -#define _CHECK_FUNCTION_VERSION 340 -#define _CHECK_FUNCTION_VERSION_INLINE 341 -#define _CHECK_FUNCTION_VERSION_KW 342 -#define _CHECK_IS_NOT_PY_CALLABLE 343 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 344 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 345 -#define _CHECK_METHOD_VERSION 346 -#define _CHECK_METHOD_VERSION_KW 347 -#define _CHECK_PEP_523 348 -#define _CHECK_PERIODIC 349 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 350 -#define _CHECK_RECURSION_REMAINING 351 -#define _CHECK_STACK_SPACE 352 -#define _CHECK_STACK_SPACE_OPERAND 353 -#define _CHECK_VALIDITY 354 -#define _COMPARE_OP 355 -#define _COMPARE_OP_FLOAT 356 -#define _COMPARE_OP_INT 357 -#define _COMPARE_OP_STR 358 -#define _CONTAINS_OP 359 -#define _CONTAINS_OP_DICT 360 -#define _CONTAINS_OP_SET 361 +#define _CHECK_FUNCTION_EXACT_ARGS 343 +#define _CHECK_FUNCTION_VERSION 344 +#define _CHECK_FUNCTION_VERSION_INLINE 345 +#define _CHECK_FUNCTION_VERSION_KW 346 +#define _CHECK_IS_NOT_PY_CALLABLE 347 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 348 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 349 +#define _CHECK_METHOD_VERSION 350 +#define _CHECK_METHOD_VERSION_KW 351 +#define _CHECK_PEP_523 352 +#define _CHECK_PERIODIC 353 +#define _CHECK_PERIODIC_AT_END 354 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 +#define _CHECK_RECURSION_REMAINING 356 +#define _CHECK_STACK_SPACE 357 +#define _CHECK_STACK_SPACE_OPERAND 358 +#define _CHECK_VALIDITY 359 +#define _COLD_DYNAMIC_EXIT 360 +#define _COLD_EXIT 361 +#define _COMPARE_OP 362 +#define _COMPARE_OP_FLOAT 363 +#define _COMPARE_OP_INT 364 +#define _COMPARE_OP_STR 365 +#define _CONTAINS_OP 366 +#define _CONTAINS_OP_DICT 367 +#define _CONTAINS_OP_SET 368 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY COPY +#define _COPY 369 +#define _COPY_1 370 +#define _COPY_2 371 +#define _COPY_3 372 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 362 +#define _CREATE_INIT_FRAME 373 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 363 +#define _DEOPT 374 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 364 -#define _DO_CALL_FUNCTION_EX 365 -#define _DO_CALL_KW 366 +#define _DO_CALL 375 +#define _DO_CALL_FUNCTION_EX 376 +#define _DO_CALL_KW 377 +#define _DYNAMIC_EXIT 378 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 367 +#define _ERROR_POP_N 379 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 368 -#define _EXPAND_METHOD_KW 369 -#define _FATAL_ERROR 370 +#define _EXPAND_METHOD 380 +#define _EXPAND_METHOD_KW 381 +#define _FATAL_ERROR 382 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 371 -#define _FOR_ITER_GEN_FRAME 372 -#define _FOR_ITER_TIER_TWO 373 +#define _FOR_ITER 383 +#define _FOR_ITER_GEN_FRAME 384 +#define _FOR_ITER_TIER_TWO 385 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 374 -#define _GUARD_CALLABLE_LEN 375 -#define _GUARD_CALLABLE_STR_1 376 -#define _GUARD_CALLABLE_TUPLE_1 377 -#define _GUARD_CALLABLE_TYPE_1 378 -#define _GUARD_DORV_NO_DICT 379 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 380 -#define _GUARD_GLOBALS_VERSION 381 -#define _GUARD_IS_FALSE_POP 382 -#define _GUARD_IS_NONE_POP 383 -#define _GUARD_IS_NOT_NONE_POP 384 -#define _GUARD_IS_TRUE_POP 385 -#define _GUARD_KEYS_VERSION 386 -#define _GUARD_NOS_DICT 387 -#define _GUARD_NOS_FLOAT 388 -#define _GUARD_NOS_INT 389 -#define _GUARD_NOS_LIST 390 -#define _GUARD_NOS_NULL 391 -#define _GUARD_NOS_TUPLE 392 -#define _GUARD_NOS_UNICODE 393 -#define _GUARD_NOT_EXHAUSTED_LIST 394 -#define _GUARD_NOT_EXHAUSTED_RANGE 395 -#define _GUARD_NOT_EXHAUSTED_TUPLE 396 -#define _GUARD_TOS_ANY_SET 397 -#define _GUARD_TOS_DICT 398 -#define _GUARD_TOS_FLOAT 399 -#define _GUARD_TOS_INT 400 -#define _GUARD_TOS_LIST 401 -#define _GUARD_TOS_SLICE 402 -#define _GUARD_TOS_TUPLE 403 -#define _GUARD_TOS_UNICODE 404 -#define _GUARD_TYPE_VERSION 405 -#define _GUARD_TYPE_VERSION_AND_LOCK 406 +#define _GUARD_BINARY_OP_EXTEND 386 +#define _GUARD_CALLABLE_ISINSTANCE 387 +#define _GUARD_CALLABLE_LEN 388 +#define _GUARD_CALLABLE_LIST_APPEND 389 +#define _GUARD_CALLABLE_STR_1 390 +#define _GUARD_CALLABLE_TUPLE_1 391 +#define _GUARD_CALLABLE_TYPE_1 392 +#define _GUARD_DORV_NO_DICT 393 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 394 +#define _GUARD_GLOBALS_VERSION 395 +#define _GUARD_IP_RETURN_GENERATOR 396 +#define _GUARD_IP_RETURN_VALUE 397 +#define _GUARD_IP_YIELD_VALUE 398 +#define _GUARD_IP__PUSH_FRAME 399 +#define _GUARD_IS_FALSE_POP 400 +#define _GUARD_IS_NONE_POP 401 +#define _GUARD_IS_NOT_NONE_POP 402 +#define _GUARD_IS_TRUE_POP 403 +#define _GUARD_KEYS_VERSION 404 +#define _GUARD_NOS_DICT 405 +#define _GUARD_NOS_FLOAT 406 +#define _GUARD_NOS_INT 407 +#define _GUARD_NOS_LIST 408 +#define _GUARD_NOS_NOT_NULL 409 +#define _GUARD_NOS_NULL 410 +#define _GUARD_NOS_OVERFLOWED 411 +#define _GUARD_NOS_TUPLE 412 +#define _GUARD_NOS_UNICODE 413 +#define _GUARD_NOT_EXHAUSTED_LIST 414 +#define _GUARD_NOT_EXHAUSTED_RANGE 415 +#define _GUARD_NOT_EXHAUSTED_TUPLE 416 +#define _GUARD_THIRD_NULL 417 +#define _GUARD_TOS_ANY_SET 418 +#define _GUARD_TOS_DICT 419 +#define _GUARD_TOS_FLOAT 420 +#define _GUARD_TOS_INT 421 +#define _GUARD_TOS_LIST 422 +#define _GUARD_TOS_OVERFLOWED 423 +#define _GUARD_TOS_SLICE 424 +#define _GUARD_TOS_TUPLE 425 +#define _GUARD_TOS_UNICODE 426 +#define _GUARD_TYPE_VERSION 427 +#define _GUARD_TYPE_VERSION_AND_LOCK 428 +#define _HANDLE_PENDING_AND_DEOPT 429 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 407 -#define _INIT_CALL_PY_EXACT_ARGS 408 -#define _INIT_CALL_PY_EXACT_ARGS_0 409 -#define _INIT_CALL_PY_EXACT_ARGS_1 410 -#define _INIT_CALL_PY_EXACT_ARGS_2 411 -#define _INIT_CALL_PY_EXACT_ARGS_3 412 -#define _INIT_CALL_PY_EXACT_ARGS_4 413 -#define _INSERT_NULL 414 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 430 +#define _INIT_CALL_PY_EXACT_ARGS 431 +#define _INIT_CALL_PY_EXACT_ARGS_0 432 +#define _INIT_CALL_PY_EXACT_ARGS_1 433 +#define _INIT_CALL_PY_EXACT_ARGS_2 434 +#define _INIT_CALL_PY_EXACT_ARGS_3 435 +#define _INIT_CALL_PY_EXACT_ARGS_4 436 +#define _INSERT_NULL 437 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -171,163 +191,179 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 415 +#define _IS_NONE 438 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 416 -#define _ITER_CHECK_RANGE 417 -#define _ITER_CHECK_TUPLE 418 -#define _ITER_JUMP_LIST 419 -#define _ITER_JUMP_RANGE 420 -#define _ITER_JUMP_TUPLE 421 -#define _ITER_NEXT_LIST 422 -#define _ITER_NEXT_LIST_TIER_TWO 423 -#define _ITER_NEXT_RANGE 424 -#define _ITER_NEXT_TUPLE 425 -#define _JUMP_TO_TOP 426 +#define _ITER_CHECK_LIST 439 +#define _ITER_CHECK_RANGE 440 +#define _ITER_CHECK_TUPLE 441 +#define _ITER_JUMP_LIST 442 +#define _ITER_JUMP_RANGE 443 +#define _ITER_JUMP_TUPLE 444 +#define _ITER_NEXT_LIST 445 +#define _ITER_NEXT_LIST_TIER_TWO 446 +#define _ITER_NEXT_RANGE 447 +#define _ITER_NEXT_TUPLE 448 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _JUMP_TO_TOP 449 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 427 -#define _LOAD_ATTR_CLASS 428 +#define _LOAD_ATTR 450 +#define _LOAD_ATTR_CLASS 451 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 429 -#define _LOAD_ATTR_METHOD_LAZY_DICT 430 -#define _LOAD_ATTR_METHOD_NO_DICT 431 -#define _LOAD_ATTR_METHOD_WITH_VALUES 432 -#define _LOAD_ATTR_MODULE 433 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 434 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 435 -#define _LOAD_ATTR_PROPERTY_FRAME 436 -#define _LOAD_ATTR_SLOT 437 -#define _LOAD_ATTR_WITH_HINT 438 +#define _LOAD_ATTR_INSTANCE_VALUE 452 +#define _LOAD_ATTR_METHOD_LAZY_DICT 453 +#define _LOAD_ATTR_METHOD_NO_DICT 454 +#define _LOAD_ATTR_METHOD_WITH_VALUES 455 +#define _LOAD_ATTR_MODULE 456 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 457 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 458 +#define _LOAD_ATTR_PROPERTY_FRAME 459 +#define _LOAD_ATTR_SLOT 460 +#define _LOAD_ATTR_WITH_HINT 461 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 439 +#define _LOAD_BYTECODE 462 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 440 -#define _LOAD_CONST_INLINE_BORROW 441 -#define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL +#define _LOAD_CONST_INLINE 463 +#define _LOAD_CONST_INLINE_BORROW 464 +#define _LOAD_CONST_UNDER_INLINE 465 +#define _LOAD_CONST_UNDER_INLINE_BORROW 466 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 442 -#define _LOAD_FAST_0 443 -#define _LOAD_FAST_1 444 -#define _LOAD_FAST_2 445 -#define _LOAD_FAST_3 446 -#define _LOAD_FAST_4 447 -#define _LOAD_FAST_5 448 -#define _LOAD_FAST_6 449 -#define _LOAD_FAST_7 450 +#define _LOAD_FAST 467 +#define _LOAD_FAST_0 468 +#define _LOAD_FAST_1 469 +#define _LOAD_FAST_2 470 +#define _LOAD_FAST_3 471 +#define _LOAD_FAST_4 472 +#define _LOAD_FAST_5 473 +#define _LOAD_FAST_6 474 +#define _LOAD_FAST_7 475 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 451 -#define _LOAD_FAST_BORROW_0 452 -#define _LOAD_FAST_BORROW_1 453 -#define _LOAD_FAST_BORROW_2 454 -#define _LOAD_FAST_BORROW_3 455 -#define _LOAD_FAST_BORROW_4 456 -#define _LOAD_FAST_BORROW_5 457 -#define _LOAD_FAST_BORROW_6 458 -#define _LOAD_FAST_BORROW_7 459 +#define _LOAD_FAST_BORROW 476 +#define _LOAD_FAST_BORROW_0 477 +#define _LOAD_FAST_BORROW_1 478 +#define _LOAD_FAST_BORROW_2 479 +#define _LOAD_FAST_BORROW_3 480 +#define _LOAD_FAST_BORROW_4 481 +#define _LOAD_FAST_BORROW_5 482 +#define _LOAD_FAST_BORROW_6 483 +#define _LOAD_FAST_BORROW_7 484 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 460 -#define _LOAD_GLOBAL_BUILTINS 461 -#define _LOAD_GLOBAL_MODULE 462 +#define _LOAD_GLOBAL 485 +#define _LOAD_GLOBAL_BUILTINS 486 +#define _LOAD_GLOBAL_MODULE 487 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 463 -#define _LOAD_SMALL_INT_0 464 -#define _LOAD_SMALL_INT_1 465 -#define _LOAD_SMALL_INT_2 466 -#define _LOAD_SMALL_INT_3 467 -#define _LOAD_SPECIAL 468 +#define _LOAD_SMALL_INT 488 +#define _LOAD_SMALL_INT_0 489 +#define _LOAD_SMALL_INT_1 490 +#define _LOAD_SMALL_INT_2 491 +#define _LOAD_SMALL_INT_3 492 +#define _LOAD_SPECIAL 493 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 469 +#define _MAKE_CALLARGS_A_TUPLE 494 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 470 +#define _MAKE_WARM 495 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 471 -#define _MAYBE_EXPAND_METHOD_KW 472 -#define _MONITOR_CALL 473 -#define _MONITOR_CALL_KW 474 -#define _MONITOR_JUMP_BACKWARD 475 -#define _MONITOR_RESUME 476 +#define _MAYBE_EXPAND_METHOD 496 +#define _MAYBE_EXPAND_METHOD_KW 497 +#define _MONITOR_CALL 498 +#define _MONITOR_CALL_KW 499 +#define _MONITOR_JUMP_BACKWARD 500 +#define _MONITOR_RESUME 501 #define _NOP NOP +#define _POP_CALL 502 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 503 +#define _POP_CALL_ONE 504 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 505 +#define _POP_CALL_TWO 506 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 507 #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 477 -#define _POP_JUMP_IF_TRUE 478 +#define _POP_ITER POP_ITER +#define _POP_JUMP_IF_FALSE 508 +#define _POP_JUMP_IF_TRUE 509 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 479 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 480 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 481 +#define _POP_TOP_FLOAT 510 +#define _POP_TOP_INT 511 +#define _POP_TOP_LOAD_CONST_INLINE 512 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 513 +#define _POP_TOP_NOP 514 +#define _POP_TOP_UNICODE 515 +#define _POP_TWO 516 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 517 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 482 +#define _PUSH_FRAME 518 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 483 -#define _PY_FRAME_GENERAL 484 -#define _PY_FRAME_KW 485 -#define _QUICKEN_RESUME 486 -#define _REPLACE_WITH_TRUE 487 +#define _PUSH_NULL_CONDITIONAL 519 +#define _PY_FRAME_GENERAL 520 +#define _PY_FRAME_KW 521 +#define _QUICKEN_RESUME 522 +#define _REPLACE_WITH_TRUE 523 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 488 -#define _SEND 489 -#define _SEND_GEN_FRAME 490 +#define _SAVE_RETURN_OFFSET 524 +#define _SEND 525 +#define _SEND_GEN_FRAME 526 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 491 -#define _STORE_ATTR 492 -#define _STORE_ATTR_INSTANCE_VALUE 493 -#define _STORE_ATTR_SLOT 494 -#define _STORE_ATTR_WITH_HINT 495 +#define _START_EXECUTOR 527 +#define _STORE_ATTR 528 +#define _STORE_ATTR_INSTANCE_VALUE 529 +#define _STORE_ATTR_SLOT 530 +#define _STORE_ATTR_WITH_HINT 531 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 496 -#define _STORE_FAST_0 497 -#define _STORE_FAST_1 498 -#define _STORE_FAST_2 499 -#define _STORE_FAST_3 500 -#define _STORE_FAST_4 501 -#define _STORE_FAST_5 502 -#define _STORE_FAST_6 503 -#define _STORE_FAST_7 504 +#define _STORE_FAST 532 +#define _STORE_FAST_0 533 +#define _STORE_FAST_1 534 +#define _STORE_FAST_2 535 +#define _STORE_FAST_3 536 +#define _STORE_FAST_4 537 +#define _STORE_FAST_5 538 +#define _STORE_FAST_6 539 +#define _STORE_FAST_7 540 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 505 -#define _STORE_SUBSCR 506 -#define _STORE_SUBSCR_DICT 507 -#define _STORE_SUBSCR_LIST_INT 508 -#define _SWAP SWAP -#define _TIER2_RESUME_CHECK 509 -#define _TO_BOOL 510 +#define _STORE_SLICE 541 +#define _STORE_SUBSCR 542 +#define _STORE_SUBSCR_DICT 543 +#define _STORE_SUBSCR_LIST_INT 544 +#define _SWAP 545 +#define _SWAP_2 546 +#define _SWAP_3 547 +#define _TIER2_RESUME_CHECK 548 +#define _TO_BOOL 549 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 511 +#define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 512 +#define _TO_BOOL_STR 551 +#define _TRACE_RECORD TRACE_RECORD #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 513 -#define _UNPACK_SEQUENCE_LIST 514 -#define _UNPACK_SEQUENCE_TUPLE 515 -#define _UNPACK_SEQUENCE_TWO_TUPLE 516 +#define _UNPACK_SEQUENCE 552 +#define _UNPACK_SEQUENCE_LIST 553 +#define _UNPACK_SEQUENCE_TUPLE 554 +#define _UNPACK_SEQUENCE_TWO_TUPLE 555 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 516 +#define MAX_UOP_ID 555 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 912b1e56692..d5a3c362d87 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,19 +11,20 @@ extern "C" { #include #include "pycore_uop_ids.h" -extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; -extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1]; +extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1]; +typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; +extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA -const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { +const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, - [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -45,8 +46,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_CONST_MORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG, - [_LOAD_CONST_IMMORTAL] = HAS_ARG_FLAG | HAS_CONST_FLAG, + [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG, [_LOAD_SMALL_INT_0] = 0, [_LOAD_SMALL_INT_1] = 0, [_LOAD_SMALL_INT_2] = 0, @@ -64,8 +64,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_POP_TOP_NOP] = 0, + [_POP_TOP_INT] = 0, + [_POP_TOP_FLOAT] = 0, + [_POP_TOP_UNICODE] = 0, + [_POP_TWO] = HAS_ESCAPES_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, + [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NOT] = HAS_PURE_FLAG, @@ -75,7 +81,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_NOS_LIST] = HAS_EXIT_FLAG, [_GUARD_TOS_LIST] = HAS_EXIT_FLAG, [_GUARD_TOS_SLICE] = HAS_EXIT_FLAG, - [_TO_BOOL_LIST] = 0, + [_TO_BOOL_LIST] = HAS_ESCAPES_FLAG, [_TO_BOOL_NONE] = HAS_EXIT_FLAG, [_GUARD_NOS_UNICODE] = HAS_EXIT_FLAG, [_GUARD_TOS_UNICODE] = HAS_EXIT_FLAG, @@ -84,18 +90,23 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, + [_GUARD_TOS_OVERFLOWED] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -103,7 +114,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, [_GUARD_TOS_TUPLE] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, + [_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -117,12 +128,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -130,14 +141,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_LOCALS] = HAS_ERROR_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL_CONDITIONAL] = HAS_ARG_FLAG, @@ -151,7 +162,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, - [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_INTERPOLATION] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_TEMPLATE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, @@ -173,9 +184,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_CHECK_ATTR_CLASS] = HAS_EXIT_FLAG, - [_LOAD_ATTR_CLASS] = 0, + [_LOAD_ATTR_CLASS] = HAS_ESCAPES_FLAG, [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, @@ -183,9 +194,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, - [_COMPARE_OP_INT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_COMPARE_OP_INT] = HAS_ARG_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, - [_IS_OP] = HAS_ARG_FLAG, + [_IS_OP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TOS_ANY_SET] = HAS_DEOPT_FLAG, [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -194,7 +205,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_IS_NONE] = 0, + [_IS_NONE] = HAS_ESCAPES_FLAG, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_MAPPING] = 0, @@ -205,7 +216,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, - [_ITER_NEXT_LIST_TIER_TWO] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, [_ITER_NEXT_TUPLE] = 0, @@ -245,8 +256,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, + [_PUSH_FRAME] = HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, + [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, + [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, [_GUARD_CALLABLE_TYPE_1] = HAS_DEOPT_FLAG, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_STR_1] = HAS_DEOPT_FLAG, @@ -262,7 +275,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_LEN] = HAS_DEOPT_FLAG, [_CALL_LEN] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_ISINSTANCE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_ISINSTANCE] = HAS_DEOPT_FLAG, + [_CALL_ISINSTANCE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_LIST_APPEND] = HAS_DEOPT_FLAG, [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -278,54 +293,78 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_WITH_SPEC] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_COPY_1] = HAS_PURE_FLAG, + [_COPY_2] = HAS_PURE_FLAG, + [_COPY_3] = HAS_PURE_FLAG, [_COPY] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_BINARY_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_SWAP_2] = HAS_PURE_FLAG, + [_SWAP_3] = HAS_PURE_FLAG, [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, - [_START_EXECUTOR] = 0, + [_POP_CALL] = HAS_ESCAPES_FLAG, + [_POP_CALL_ONE] = HAS_ESCAPES_FLAG, + [_POP_CALL_TWO] = HAS_ESCAPES_FLAG, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_POP_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_POP_CALL_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_LOAD_CONST_UNDER_INLINE] = 0, + [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, + [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_DEOPT] = 0, + [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, [_ERROR_POP_N] = HAS_ARG_FLAG, - [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, + [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, + [_COLD_EXIT] = 0, + [_COLD_DYNAMIC_EXIT] = 0, + [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, }; -const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { - [_LOAD_FAST] = 8, - [_LOAD_FAST_BORROW] = 8, - [_LOAD_SMALL_INT] = 4, - [_STORE_FAST] = 8, - [_INIT_CALL_PY_EXACT_ARGS] = 5, +const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { + [_LOAD_FAST] = { 0, 8 }, + [_LOAD_FAST_BORROW] = { 0, 8 }, + [_LOAD_SMALL_INT] = { 0, 4 }, + [_STORE_FAST] = { 0, 8 }, + [_INIT_CALL_PY_EXACT_ARGS] = { 0, 5 }, + [_COPY] = { 1, 4 }, + [_SWAP] = { 2, 4 }, }; const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", @@ -335,6 +374,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", [_BINARY_SLICE] = "_BINARY_SLICE", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", @@ -369,7 +409,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", - [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", @@ -386,6 +425,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COLD_DYNAMIC_EXIT] = "_COLD_DYNAMIC_EXIT", + [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", @@ -395,6 +436,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", [_CONVERT_VALUE] = "_CONVERT_VALUE", [_COPY] = "_COPY", + [_COPY_1] = "_COPY_1", + [_COPY_2] = "_COPY_2", + [_COPY_3] = "_COPY_3", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", [_DELETE_ATTR] = "_DELETE_ATTR", @@ -406,6 +450,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", @@ -425,13 +470,19 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GET_LEN] = "_GET_LEN", [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", + [_GUARD_CALLABLE_ISINSTANCE] = "_GUARD_CALLABLE_ISINSTANCE", [_GUARD_CALLABLE_LEN] = "_GUARD_CALLABLE_LEN", + [_GUARD_CALLABLE_LIST_APPEND] = "_GUARD_CALLABLE_LIST_APPEND", [_GUARD_CALLABLE_STR_1] = "_GUARD_CALLABLE_STR_1", [_GUARD_CALLABLE_TUPLE_1] = "_GUARD_CALLABLE_TUPLE_1", [_GUARD_CALLABLE_TYPE_1] = "_GUARD_CALLABLE_TYPE_1", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", + [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", + [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", + [_GUARD_IP__PUSH_FRAME] = "_GUARD_IP__PUSH_FRAME", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -441,22 +492,27 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", [_GUARD_NOS_INT] = "_GUARD_NOS_INT", [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", + [_GUARD_NOS_NOT_NULL] = "_GUARD_NOS_NOT_NULL", [_GUARD_NOS_NULL] = "_GUARD_NOS_NULL", + [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED", [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", [_GUARD_TOS_ANY_SET] = "_GUARD_TOS_ANY_SET", [_GUARD_TOS_DICT] = "_GUARD_TOS_DICT", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", + [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", [_GUARD_TOS_SLICE] = "_GUARD_TOS_SLICE", [_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE", [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_HANDLE_PENDING_AND_DEOPT] = "_HANDLE_PENDING_AND_DEOPT", [_IMPORT_FROM] = "_IMPORT_FROM", [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", @@ -492,10 +548,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", - [_LOAD_CONST_IMMORTAL] = "_LOAD_CONST_IMMORTAL", + [_LOAD_CONST] = "_LOAD_CONST", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", - [_LOAD_CONST_MORTAL] = "_LOAD_CONST_MORTAL", + [_LOAD_CONST_UNDER_INLINE] = "_LOAD_CONST_UNDER_INLINE", + [_LOAD_CONST_UNDER_INLINE_BORROW] = "_LOAD_CONST_UNDER_INLINE_BORROW", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_FAST] = "_LOAD_FAST", [_LOAD_FAST_0] = "_LOAD_FAST_0", @@ -545,10 +602,22 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", [_NOP] = "_NOP", + [_POP_CALL] = "_POP_CALL", + [_POP_CALL_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_LOAD_CONST_INLINE_BORROW", + [_POP_CALL_ONE] = "_POP_CALL_ONE", + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", + [_POP_CALL_TWO] = "_POP_CALL_TWO", + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_ITER] = "_POP_ITER", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", + [_POP_TOP_INT] = "_POP_TOP_INT", [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_NOP] = "_POP_TOP_NOP", + [_POP_TOP_UNICODE] = "_POP_TOP_UNICODE", + [_POP_TWO] = "_POP_TWO", [_POP_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_TWO_LOAD_CONST_INLINE_BORROW", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", @@ -591,6 +660,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_SWAP] = "_SWAP", + [_SWAP_2] = "_SWAP_2", + [_SWAP_3] = "_SWAP_3", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TO_BOOL] = "_TO_BOOL", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", @@ -664,9 +735,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_FAST_BORROW_LOAD_FAST_BORROW: return 0; - case _LOAD_CONST_MORTAL: - return 0; - case _LOAD_CONST_IMMORTAL: + case _LOAD_CONST: return 0; case _LOAD_SMALL_INT_0: return 0; @@ -702,10 +771,22 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _POP_TOP: return 1; + case _POP_TOP_NOP: + return 1; + case _POP_TOP_INT: + return 1; + case _POP_TOP_FLOAT: + return 1; + case _POP_TOP_UNICODE: + return 1; + case _POP_TWO: + return 2; case _PUSH_NULL: return 0; case _END_FOR: return 1; + case _POP_ITER: + return 2; case _END_SEND: return 2; case _UNARY_NEGATIVE: @@ -742,6 +823,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TOS_INT: return 0; + case _GUARD_NOS_OVERFLOWED: + return 0; + case _GUARD_TOS_OVERFLOWED: + return 0; case _BINARY_OP_MULTIPLY_INT: return 2; case _BINARY_OP_ADD_INT: @@ -758,6 +843,12 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: + return 2; + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: + return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: @@ -1068,6 +1159,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_NOS_NULL: return 0; + case _GUARD_NOS_NOT_NULL: + return 0; + case _GUARD_THIRD_NULL: + return 0; case _GUARD_CALLABLE_TYPE_1: return 0; case _CALL_TYPE_1: @@ -1098,8 +1193,12 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_LEN: return 3; + case _GUARD_CALLABLE_ISINSTANCE: + return 0; case _CALL_ISINSTANCE: - return 2 + oparg; + return 4; + case _GUARD_CALLABLE_LIST_APPEND: + return 0; case _CALL_LIST_APPEND: return 3; case _CALL_METHOD_DESCRIPTOR_O: @@ -1140,10 +1239,20 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _FORMAT_WITH_SPEC: return 2; + case _COPY_1: + return 0; + case _COPY_2: + return 0; + case _COPY_3: + return 0; case _COPY: return 0; case _BINARY_OP: return 2; + case _SWAP_2: + return 0; + case _SWAP_3: + return 0; case _SWAP: return 0; case _GUARD_IS_TRUE_POP: @@ -1164,6 +1273,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _EXIT_TRACE: return 0; + case _DYNAMIC_EXIT: + return 0; case _CHECK_VALIDITY: return 0; case _LOAD_CONST_INLINE: @@ -1172,12 +1283,26 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_CONST_INLINE_BORROW: return 0; + case _POP_CALL: + return 2; + case _POP_CALL_ONE: + return 3; + case _POP_CALL_TWO: + return 4; case _POP_TOP_LOAD_CONST_INLINE_BORROW: return 1; case _POP_TWO_LOAD_CONST_INLINE_BORROW: return 2; - case _CHECK_FUNCTION: - return 0; + case _POP_CALL_LOAD_CONST_INLINE_BORROW: + return 2; + case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: + return 3; + case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: + return 4; + case _LOAD_CONST_UNDER_INLINE: + return 1; + case _LOAD_CONST_UNDER_INLINE_BORROW: + return 1; case _START_EXECUTOR: return 0; case _MAKE_WARM: @@ -1186,10 +1311,24 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _DEOPT: return 0; + case _HANDLE_PENDING_AND_DEOPT: + return 0; case _ERROR_POP_N: return 0; case _TIER2_RESUME_CHECK: return 0; + case _COLD_EXIT: + return 0; + case _COLD_DYNAMIC_EXIT: + return 0; + case _GUARD_IP__PUSH_FRAME: + return 0; + case _GUARD_IP_YIELD_VALUE: + return 0; + case _GUARD_IP_RETURN_VALUE: + return 0; + case _GUARD_IP_RETURN_GENERATOR: + return 0; default: return -1; } diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index 950aa0af290..4ed8928c0b9 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -29,6 +29,12 @@ extern "C" { PyMutex_LockFlags(wr->weakrefs_lock, _Py_LOCK_DONT_DETACH) #define UNLOCK_WEAKREFS_FOR_WR(wr) PyMutex_Unlock(wr->weakrefs_lock) +#define FT_CLEAR_WEAKREFS(obj, weakref_list) \ + do { \ + assert(Py_REFCNT(obj) == 0); \ + PyObject_ClearWeakRefs(obj); \ + } while (0) + #else #define LOCK_WEAKREFS(obj) @@ -37,6 +43,14 @@ extern "C" { #define LOCK_WEAKREFS_FOR_WR(wr) #define UNLOCK_WEAKREFS_FOR_WR(wr) +#define FT_CLEAR_WEAKREFS(obj, weakref_list) \ + do { \ + assert(Py_REFCNT(obj) == 0); \ + if (weakref_list != NULL) { \ + PyObject_ClearWeakRefs(obj); \ + } \ + } while (0) + #endif static inline int _is_dead(PyObject *obj) diff --git a/Include/lock.h b/Include/lock.h deleted file mode 100644 index 782b9dbc70d..00000000000 --- a/Include/lock.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef Py_LOCK_H -#define Py_LOCK_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_LOCK_H -# include "cpython/lock.h" -# undef Py_CPYTHON_LOCK_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_LOCK_H */ diff --git a/Include/marshal.h b/Include/marshal.h index f773587bdd0..2ccb112b40c 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -1,31 +1,18 @@ - /* Interface for marshal.c */ #ifndef Py_MARSHAL_H #define Py_MARSHAL_H -#ifndef Py_LIMITED_API - #ifdef __cplusplus extern "C" { #endif -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, - Py_ssize_t); -PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); - -#define Py_MARSHAL_VERSION 5 - -PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); -PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); -PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); - -PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); -PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); +#ifndef Py_LIMITED_API +# define _Py_CPYTHON_MARSHAL_H +# include "cpython/marshal.h" +# undef _Py_CPYTHON_MARSHAL_H +#endif #ifdef __cplusplus } #endif - -#endif /* Py_LIMITED_API */ #endif /* !Py_MARSHAL_H */ diff --git a/Include/modsupport.h b/Include/modsupport.h index af995f567b0..cb47ad8cd27 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -56,58 +56,6 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); #define Py_CLEANUP_SUPPORTED 0x20000 -#define PYTHON_API_VERSION 1013 -#define PYTHON_API_STRING "1013" -/* The API version is maintained (independently from the Python version) - so we can detect mismatches between the interpreter and dynamically - loaded modules. These are diagnosed by an error message but - the module is still loaded (because the mismatch can only be tested - after loading the module). The error message is intended to - explain the core dump a few seconds later. - - The symbol PYTHON_API_STRING defines the same value as a string - literal. *** PLEASE MAKE SURE THE DEFINITIONS MATCH. *** - - Please add a line or two to the top of this log for each API - version change: - - 22-Feb-2006 MvL 1013 PEP 353 - long indices for sequence lengths - - 19-Aug-2002 GvR 1012 Changes to string object struct for - interning changes, saving 3 bytes. - - 17-Jul-2001 GvR 1011 Descr-branch, just to be on the safe side - - 25-Jan-2001 FLD 1010 Parameters added to PyCode_New() and - PyFrame_New(); Python 2.1a2 - - 14-Mar-2000 GvR 1009 Unicode API added - - 3-Jan-1999 GvR 1007 Decided to change back! (Don't reuse 1008!) - - 3-Dec-1998 GvR 1008 Python 1.5.2b1 - - 18-Jan-1997 GvR 1007 string interning and other speedups - - 11-Oct-1996 GvR renamed Py_Ellipses to Py_Ellipsis :-( - - 30-Jul-1996 GvR Slice and ellipses syntax added - - 23-Jul-1996 GvR For 1.4 -- better safe than sorry this time :-) - - 7-Nov-1995 GvR Keyword arguments (should've been done at 1.3 :-( ) - - 10-Jan-1995 GvR Renamed globals to new naming scheme - - 9-Jan-1995 GvR Initial version (incompatible with older API) -*/ - -/* The PYTHON_ABI_VERSION is introduced in PEP 384. For the lifetime of - Python 3, it will stay at the value of 3; changes to the limited API - must be performed in a strictly backwards-compatible manner. */ -#define PYTHON_ABI_VERSION 3 -#define PYTHON_ABI_STRING "3" - PyAPI_FUNC(PyObject *) PyModule_Create2(PyModuleDef*, int apiver); #ifdef Py_LIMITED_API @@ -134,6 +82,72 @@ PyAPI_FUNC(PyObject *) PyModule_FromDefAndSpec2(PyModuleDef *def, #endif /* New in 3.5 */ +/* ABI info & checking (new in 3.15) */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +typedef struct PyABIInfo { + uint8_t abiinfo_major_version; + uint8_t abiinfo_minor_version; + uint16_t flags; + uint32_t build_version; + uint32_t abi_version; +} PyABIInfo; +#define PyABIInfo_STABLE 0x0001 +#define PyABIInfo_GIL 0x0002 +#define PyABIInfo_FREETHREADED 0x0004 +#define PyABIInfo_INTERNAL 0x0008 + +#define PyABIInfo_FREETHREADING_AGNOSTIC (PyABIInfo_GIL|PyABIInfo_FREETHREADED) + +PyAPI_FUNC(int) PyABIInfo_Check(PyABIInfo *info, const char *module_name); + +// Define the defaults +#ifdef Py_LIMITED_API + #define _PyABIInfo_DEFAULT_FLAG_STABLE PyABIInfo_STABLE + #if Py_LIMITED_API == 3 + #define PyABIInfo_DEFAULT_ABI_VERSION _Py_PACK_VERSION(3, 2) + #else + #define PyABIInfo_DEFAULT_ABI_VERSION Py_LIMITED_API + #endif +#else + #define _PyABIInfo_DEFAULT_FLAG_STABLE 0 + #define PyABIInfo_DEFAULT_ABI_VERSION PY_VERSION_HEX +#endif +#if defined(Py_LIMITED_API) && defined(_Py_OPAQUE_PYOBJECT) + #define _PyABIInfo_DEFAULT_FLAG_FT PyABIInfo_FREETHREADING_AGNOSTIC +#elif defined(Py_GIL_DISABLED) + #define _PyABIInfo_DEFAULT_FLAG_FT PyABIInfo_FREETHREADED +#else + #define _PyABIInfo_DEFAULT_FLAG_FT PyABIInfo_GIL +#endif +#if defined(Py_BUILD_CORE) + #define _PyABIInfo_DEFAULT_FLAG_INTERNAL PyABIInfo_INTERNAL +#else + #define _PyABIInfo_DEFAULT_FLAG_INTERNAL 0 +#endif + +#define PyABIInfo_DEFAULT_FLAGS ( \ + _PyABIInfo_DEFAULT_FLAG_STABLE \ + | _PyABIInfo_DEFAULT_FLAG_FT \ + | _PyABIInfo_DEFAULT_FLAG_INTERNAL \ + ) \ + ///////////////////////////////////////////////////////// + +#define _PyABIInfo_DEFAULT { \ + 1, 0, \ + PyABIInfo_DEFAULT_FLAGS, \ + PY_VERSION_HEX, \ + PyABIInfo_DEFAULT_ABI_VERSION } \ + ///////////////////////////////////////////////////////// + +#define PyABIInfo_VAR(NAME) \ + static PyABIInfo NAME = _PyABIInfo_DEFAULT; + +#undef _PyABIInfo_DEFAULT_STABLE +#undef _PyABIInfo_DEFAULT_FT +#undef _PyABIInfo_DEFAULT_INTERNAL + +#endif /* ABI info (new in 3.15) */ + #ifndef Py_LIMITED_API # define Py_CPYTHON_MODSUPPORT_H # include "cpython/modsupport.h" diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 2a17c891dda..e83bc395aa4 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -36,6 +36,7 @@ PyAPI_FUNC(PyObject *) PyModuleDef_Init(PyModuleDef*); PyAPI_DATA(PyTypeObject) PyModuleDef_Type; #endif +#ifndef _Py_OPAQUE_PYOBJECT typedef struct PyModuleDef_Base { PyObject_HEAD /* The function used to re-initialize the module. @@ -63,6 +64,7 @@ typedef struct PyModuleDef_Base { 0, /* m_index */ \ _Py_NULL, /* m_copy */ \ } +#endif // _Py_OPAQUE_PYOBJECT #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 /* New in 3.5 */ @@ -79,10 +81,21 @@ struct PyModuleDef_Slot { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 # define Py_mod_gil 4 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +# define Py_mod_abi 5 +# define Py_mod_name 6 +# define Py_mod_doc 7 +# define Py_mod_state_size 8 +# define Py_mod_methods 9 +# define Py_mod_state_traverse 10 +# define Py_mod_state_clear 11 +# define Py_mod_state_free 12 +# define Py_mod_token 13 +#endif #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 4 +#define _Py_mod_LAST_SLOT 13 #endif #endif /* New in 3.5 */ @@ -104,6 +117,15 @@ struct PyModuleDef_Slot { PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *, + PyObject *spec); +PyAPI_FUNC(int) PyModule_Exec(PyObject *mod); +PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *mod, Py_ssize_t *result); +PyAPI_FUNC(int) PyModule_GetToken(PyObject *, void **result); +#endif + +#ifndef _Py_OPAQUE_PYOBJECT struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; @@ -115,6 +137,7 @@ struct PyModuleDef { inquiry m_clear; freefunc m_free; }; +#endif // _Py_OPAQUE_PYOBJECT #ifdef __cplusplus } diff --git a/Include/monitoring.h b/Include/monitoring.h deleted file mode 100644 index 985f7f230e4..00000000000 --- a/Include/monitoring.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef Py_MONITORING_H -#define Py_MONITORING_H -#ifdef __cplusplus -extern "C" { -#endif - -// There is currently no limited API for monitoring - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_MONITORING_H -# include "cpython/monitoring.h" -# undef Py_CPYTHON_MONITORING_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_MONITORING_H */ diff --git a/Include/object.h b/Include/object.h index 8cc83abb857..ad452be8405 100644 --- a/Include/object.h +++ b/Include/object.h @@ -56,6 +56,11 @@ whose size is determined when the object is allocated. # define Py_REF_DEBUG #endif +#if defined(_Py_OPAQUE_PYOBJECT) && !defined(Py_LIMITED_API) +# error "_Py_OPAQUE_PYOBJECT only makes sense with Py_LIMITED_API" +#endif + +#ifndef _Py_OPAQUE_PYOBJECT /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; @@ -66,6 +71,8 @@ whose size is determined when the object is allocated. * * Statically allocated objects might be shared between * interpreters, so must be marked as immortal. + * + * Before changing this, see the check in PyModuleDef_Init(). */ #if defined(Py_GIL_DISABLED) #define PyObject_HEAD_INIT(type) \ @@ -99,27 +106,26 @@ whose size is determined when the object is allocated. * not necessarily a byte count. */ #define PyObject_VAR_HEAD PyVarObject ob_base; +#endif // !defined(_Py_OPAQUE_PYOBJECT) + #define Py_INVALID_SIZE (Py_ssize_t)-1 +/* PyObjects are given a minimum alignment so that the least significant bits + * of an object pointer become available for other purposes. + * This must be an integer literal with the value (1 << _PyGC_PREV_SHIFT), number of bytes. + */ +#define _PyObject_MIN_ALIGNMENT 4 + /* Nothing is actually declared to be a PyObject, but every pointer to * a Python object can be cast to a PyObject*. This is inheritance built * by hand. Similarly every pointer to a variable-size Python object can, * in addition, be cast to PyVarObject*. */ -#ifndef Py_GIL_DISABLED +#ifdef _Py_OPAQUE_PYOBJECT + /* PyObject is opaque */ +#elif !defined(Py_GIL_DISABLED) struct _object { -#if (defined(__GNUC__) || defined(__clang__)) \ - && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L) - // On C99 and older, anonymous union is a GCC and clang extension - __extension__ -#endif -#ifdef _MSC_VER - // Ignore MSC warning C4201: "nonstandard extension used: - // nameless struct/union" - __pragma(warning(push)) - __pragma(warning(disable: 4201)) -#endif - union { + _Py_ANONYMOUS union { #if SIZEOF_VOID_P > 4 PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ struct { @@ -134,14 +140,12 @@ struct _object { # endif }; #else - Py_ssize_t ob_refcnt; + Py_ssize_t ob_refcnt; // part of stable ABI; do not change #endif + _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, char) _aligner; }; -#ifdef _MSC_VER - __pragma(warning(pop)) -#endif - PyTypeObject *ob_type; + PyTypeObject *ob_type; // part of stable ABI; do not change }; #else // Objects that are not owned by any thread use a thread id (tid) of zero. @@ -153,7 +157,7 @@ struct _object { // ob_tid stores the thread id (or zero). It is also used by the GC and the // trashcan mechanism as a linked list pointer and by the GC to store the // computed "gc_refs" refcount. - uintptr_t ob_tid; + _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, uintptr_t) ob_tid; uint16_t ob_flags; PyMutex ob_mutex; // per-object lock uint8_t ob_gc_bits; // gc-related state @@ -161,15 +165,18 @@ struct _object { Py_ssize_t ob_ref_shared; // shared (atomic) reference count PyTypeObject *ob_type; }; -#endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) /* Cast argument to PyObject* type. */ #define _PyObject_CAST(op) _Py_CAST(PyObject*, (op)) -typedef struct { +#ifndef _Py_OPAQUE_PYOBJECT +struct PyVarObject { PyObject ob_base; - Py_ssize_t ob_size; /* Number of items in variable part */ -} PyVarObject; + Py_ssize_t ob_size; // Number of items in variable part. Part of stable ABI +}; +#endif +typedef struct PyVarObject PyVarObject; /* Cast argument to PyVarObject* type. */ #define _PyVarObject_CAST(op) _Py_CAST(PyVarObject*, (op)) @@ -258,53 +265,72 @@ _Py_IsOwnedByCurrentThread(PyObject *ob) } #endif -// Py_TYPE() implementation for the stable ABI -PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); - -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 - // Stable ABI implements Py_TYPE() as a function call - // on limited C API version 3.14 and newer. -#else - static inline PyTypeObject* _Py_TYPE(PyObject *ob) - { - return ob->ob_type; - } - #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 - # define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST(ob)) - #else - # define Py_TYPE(ob) _Py_TYPE(ob) - #endif -#endif - PyAPI_DATA(PyTypeObject) PyLong_Type; PyAPI_DATA(PyTypeObject) PyBool_Type; +/* Definitions for the stable ABI */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 14) +PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); +#endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(Py_ssize_t) Py_SIZE(PyObject *ob); +PyAPI_FUNC(int) Py_IS_TYPE(PyObject *ob, PyTypeObject *type); +PyAPI_FUNC(void) Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size); +#endif + +#ifndef _Py_OPAQUE_PYOBJECT + +static inline void +Py_SET_TYPE(PyObject *ob, PyTypeObject *type) +{ + ob->ob_type = type; +} + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 11) +// Non-limited API & limited API 3.11 & below: use static inline functions and +// use _PyObject_CAST so that users don't need their own casts +# define Py_TYPE(ob) _Py_TYPE_impl(_PyObject_CAST(ob)) +# define Py_SIZE(ob) _Py_SIZE_impl(_PyObject_CAST(ob)) +# define Py_IS_TYPE(ob, type) _Py_IS_TYPE_impl(_PyObject_CAST(ob), (type)) +# define Py_SET_SIZE(ob, size) _Py_SET_SIZE_impl(_PyVarObject_CAST(ob), (size)) +# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) +#elif Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 15) +// Limited API 3.11-3.14: use static inline functions, without casts +# define Py_SIZE(ob) _Py_SIZE_impl(ob) +# define Py_IS_TYPE(ob, type) _Py_IS_TYPE_impl((ob), (type)) +# define Py_SET_SIZE(ob, size) _Py_SET_SIZE_impl((ob), (size)) +# if Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 14) +// Py_TYPE() is static inline only on Limited API 3.13 and below +# define Py_TYPE(ob) _Py_TYPE_impl(ob) +# endif +#else +// Limited API 3.15+: use function calls +#endif + +static inline +PyTypeObject* _Py_TYPE_impl(PyObject *ob) +{ + return ob->ob_type; +} + // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. -static inline Py_ssize_t Py_SIZE(PyObject *ob) { +static inline Py_ssize_t +_Py_SIZE_impl(PyObject *ob) +{ assert(Py_TYPE(ob) != &PyLong_Type); assert(Py_TYPE(ob) != &PyBool_Type); return _PyVarObject_CAST(ob)->ob_size; } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) -#endif -static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { +static inline int +_Py_IS_TYPE_impl(PyObject *ob, PyTypeObject *type) +{ return Py_TYPE(ob) == type; } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), (type)) -#endif - -static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { - ob->ob_type = type; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) -#endif - -static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { +static inline void +_Py_SET_SIZE_impl(PyVarObject *ob, Py_ssize_t size) +{ assert(Py_TYPE(_PyObject_CAST(ob)) != &PyLong_Type); assert(Py_TYPE(_PyObject_CAST(ob)) != &PyBool_Type); #ifdef Py_GIL_DISABLED @@ -313,9 +339,8 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { ob->ob_size = size; #endif } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) -#endif + +#endif // !defined(_Py_OPAQUE_PYOBJECT) /* @@ -518,7 +543,7 @@ given type object has a specified feature. #define Py_TPFLAGS_INLINE_VALUES (1 << 2) /* Placement of weakref pointers are managed by the VM, not by the type. - * The VM will automatically set tp_weaklistoffset. + * The VM will automatically set tp_weaklistoffset. Implies Py_TPFLAGS_HAVE_GC. */ #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3) @@ -527,6 +552,9 @@ given type object has a specified feature. */ #define Py_TPFLAGS_MANAGED_DICT (1 << 4) +/* Type has dictionary or weakref pointers that are managed by VM and has + * to allocate space to store these. + */ #define Py_TPFLAGS_PREHEADER (Py_TPFLAGS_MANAGED_WEAKREF | Py_TPFLAGS_MANAGED_DICT) /* Set if instances of the type object are treated as sequences for pattern matching */ @@ -620,6 +648,13 @@ given type object has a specified feature. #define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0) #define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) +// Flag values for ob_flags (16 bits available, if SIZEOF_VOID_P > 4). +#define _Py_IMMORTAL_FLAGS (1 << 0) +#define _Py_LEGACY_ABI_CHECK_FLAG (1 << 1) /* see PyModuleDef_Init() */ +#define _Py_STATICALLY_ALLOCATED_FLAG (1 << 2) +#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) +#define _Py_TYPE_REVEALED_FLAG (1 << 3) +#endif #define Py_CONSTANT_NONE 0 #define Py_CONSTANT_FALSE 1 @@ -654,8 +689,13 @@ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ PyAPI_FUNC(int) Py_IsNone(PyObject *x); #define Py_IsNone(x) Py_Is((x), Py_None) -/* Macro for returning Py_None from a function */ -#define Py_RETURN_NONE return Py_None +/* Macro for returning Py_None from a function. + * Only treat Py_None as immortal in the limited C API 3.12 and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_NONE return Py_NewRef(Py_None) +#else +# define Py_RETURN_NONE return Py_None +#endif /* Py_NotImplemented is a singleton used to signal that an operation is @@ -669,8 +709,13 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ # define Py_NotImplemented (&_Py_NotImplementedStruct) #endif -/* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +/* Macro for returning Py_NotImplemented from a function. Only treat + * Py_NotImplemented as immortal in the limited C API 3.12 and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) +#else +# define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +#endif /* Rich comparison opcodes */ #define Py_LT 0 @@ -776,11 +821,7 @@ PyType_HasFeature(PyTypeObject *type, unsigned long feature) // PyTypeObject is opaque in the limited C API flags = PyType_GetFlags(type); #else -# ifdef Py_GIL_DISABLED - flags = _Py_atomic_load_ulong_relaxed(&type->tp_flags); -# else - flags = type->tp_flags; -# endif + flags = type->tp_flags; #endif return ((flags & feature) != 0); } @@ -812,6 +853,11 @@ PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyType_GetModuleByToken(PyTypeObject *type, + const void *token); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 209103c83b3..0d066c16901 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -193,50 +193,49 @@ extern "C" { #define LOAD_ATTR_PROPERTY 187 #define LOAD_ATTR_SLOT 188 #define LOAD_ATTR_WITH_HINT 189 -#define LOAD_CONST_IMMORTAL 190 -#define LOAD_CONST_MORTAL 191 -#define LOAD_GLOBAL_BUILTIN 192 -#define LOAD_GLOBAL_MODULE 193 -#define LOAD_SUPER_ATTR_ATTR 194 -#define LOAD_SUPER_ATTR_METHOD 195 -#define RESUME_CHECK 196 -#define SEND_GEN 197 -#define STORE_ATTR_INSTANCE_VALUE 198 -#define STORE_ATTR_SLOT 199 -#define STORE_ATTR_WITH_HINT 200 -#define STORE_SUBSCR_DICT 201 -#define STORE_SUBSCR_LIST_INT 202 -#define TO_BOOL_ALWAYS_TRUE 203 -#define TO_BOOL_BOOL 204 -#define TO_BOOL_INT 205 -#define TO_BOOL_LIST 206 -#define TO_BOOL_NONE 207 -#define TO_BOOL_STR 208 -#define UNPACK_SEQUENCE_LIST 209 -#define UNPACK_SEQUENCE_TUPLE 210 -#define UNPACK_SEQUENCE_TWO_TUPLE 211 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 +#define LOAD_GLOBAL_BUILTIN 190 +#define LOAD_GLOBAL_MODULE 191 +#define LOAD_SUPER_ATTR_ATTR 192 +#define LOAD_SUPER_ATTR_METHOD 193 +#define RESUME_CHECK 194 +#define SEND_GEN 195 +#define STORE_ATTR_INSTANCE_VALUE 196 +#define STORE_ATTR_SLOT 197 +#define STORE_ATTR_WITH_HINT 198 +#define STORE_SUBSCR_DICT 199 +#define STORE_SUBSCR_LIST_INT 200 +#define TO_BOOL_ALWAYS_TRUE 201 +#define TO_BOOL_BOOL 202 +#define TO_BOOL_INT 203 +#define TO_BOOL_LIST 204 +#define TO_BOOL_NONE 205 +#define TO_BOOL_STR 206 +#define UNPACK_SEQUENCE_LIST 207 +#define UNPACK_SEQUENCE_TUPLE 208 +#define UNPACK_SEQUENCE_TWO_TUPLE 209 +#define INSTRUMENTED_END_FOR 233 +#define INSTRUMENTED_POP_ITER 234 +#define INSTRUMENTED_END_SEND 235 +#define INSTRUMENTED_FOR_ITER 236 +#define INSTRUMENTED_INSTRUCTION 237 +#define INSTRUMENTED_JUMP_FORWARD 238 +#define INSTRUMENTED_NOT_TAKEN 239 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 +#define INSTRUMENTED_POP_JUMP_IF_NONE 242 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 +#define INSTRUMENTED_RESUME 244 +#define INSTRUMENTED_RETURN_VALUE 245 +#define INSTRUMENTED_YIELD_VALUE 246 +#define INSTRUMENTED_END_ASYNC_FOR 247 +#define INSTRUMENTED_LOAD_SUPER_ATTR 248 +#define INSTRUMENTED_CALL 249 +#define INSTRUMENTED_CALL_KW 250 +#define INSTRUMENTED_CALL_FUNCTION_EX 251 +#define INSTRUMENTED_JUMP_BACKWARD 252 +#define INSTRUMENTED_LINE 253 +#define ENTER_EXECUTOR 254 +#define TRACE_RECORD 255 #define ANNOTATIONS_PLACEHOLDER 256 #define JUMP 257 #define JUMP_IF_FALSE 258 @@ -251,7 +250,7 @@ extern "C" { #define HAVE_ARGUMENT 43 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 +#define MIN_INSTRUMENTED_OPCODE 233 #ifdef __cplusplus } diff --git a/Include/patchlevel.h b/Include/patchlevel.h index c9dd659046a..804aa1a0427 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -6,6 +6,9 @@ configure.ac must also be changed. There is also (independent) API version information in modsupport.h. + + This header should self-contained; PC/python_ver_rc.h includes it + without the rest of Python.h. */ /* Values for PY_RELEASE_LEVEL */ @@ -18,13 +21,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 14 +#define PY_MINOR_VERSION 15 #define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.14.0b1" +#define PY_VERSION "3.15.0a2+" /*--end constants--*/ @@ -46,4 +49,16 @@ // Public Py_PACK_VERSION is declared in pymacro.h; it needs . + +/* The API and ABI versions are left for backwards compatibility. + They've not been updated since 2006 and 2010, respectively. + API/ABI versioning is now tied to the CPython version. + The *_VERSION and *_STRING symbols should define the same value; as + number and string literal respectively. Make sure the definitions match. +*/ +#define PYTHON_API_VERSION 1013 +#define PYTHON_API_STRING "1013" +#define PYTHON_ABI_VERSION 3 +#define PYTHON_ABI_STRING "3" + #endif //_Py_PATCHLEVEL_H diff --git a/Include/py_curses.h b/Include/py_curses.h index 49fc3c9d127..0948aabedd4 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -109,6 +109,13 @@ static void **PyCurses_API; static const char catchall_ERR[] = "curses function returned ERR"; static const char catchall_NULL[] = "curses function returned NULL"; +#if defined(CURSES_MODULE) || defined(CURSES_PANEL_MODULE) +/* Error messages shared by the curses package */ +# define CURSES_ERROR_FORMAT "%s() returned %s" +# define CURSES_ERROR_VERBOSE_FORMAT "%s() (called by %s()) returned %s" +# define CURSES_ERROR_MUST_CALL_FORMAT "must call %s() first" +#endif + #ifdef __cplusplus } #endif diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 9824d099c3d..f523f8bb273 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -52,6 +52,16 @@ struct PyExpat_CAPI int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); /* might be NULL for expat < 2.6.0 */ XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); + /* might be NULL for expat < 2.7.2 */ + XML_Bool (*SetAllocTrackerActivationThreshold)( + XML_Parser parser, unsigned long long activationThresholdBytes); + XML_Bool (*SetAllocTrackerMaximumAmplification)( + XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.4.0 */ + XML_Bool (*SetBillionLaughsAttackProtectionActivationThreshold)( + XML_Parser parser, unsigned long long activationThresholdBytes); + XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)( + XML_Parser parser, float maxAmplificationFactor); /* always add new stuff to the end! */ }; diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index de1bcb1d2cb..4b3474035ce 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -35,15 +35,8 @@ PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); /* In pathconfig.c */ Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); -Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); - Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); -Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); -Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); -Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); -Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); -Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetPath(void); #ifdef MS_WINDOWS int _Py_CheckPython3(void); #endif diff --git a/Include/pymacro.h b/Include/pymacro.h index 218987a80b0..7ecce44a0d2 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -24,46 +24,90 @@ #endif -// _Py_ALIGN_AS: this compiler's spelling of `alignas` keyword, -// We currently use alignas for free-threaded builds only; additional compat -// checking would be great before we add it to the default build. -// Standards/compiler support: +// _Py_ALIGNED_DEF(N, T): Define a variable/member with increased alignment +// +// `N`: the desired minimum alignment, an integer literal, number of bytes +// `T`: the type of the defined variable +// (or a type with at least the defined variable's alignment) +// +// May not be used on a struct definition. +// +// Standards/compiler support for `alignas` alternatives: // - `alignas` is a keyword in C23 and C++11. // - `_Alignas` is a keyword in C11 // - GCC & clang has __attribute__((aligned)) // (use that for older standards in pedantic mode) // - MSVC has __declspec(align) // - `_Alignas` is common C compiler extension -// Older compilers may name it differently; to allow compilation on such -// unsupported platforms, we don't redefine _Py_ALIGN_AS if it's already +// Older compilers may name `alignas` differently; to allow compilation on such +// unsupported platforms, we don't redefine _Py_ALIGNED_DEF if it's already // defined. Note that defining it wrong (including defining it to nothing) will // cause ABI incompatibilities. -#ifdef Py_GIL_DISABLED -# ifndef _Py_ALIGN_AS -# ifdef __cplusplus -# if __cplusplus >= 201103L -# define _Py_ALIGN_AS(V) alignas(V) -# elif defined(__GNUC__) || defined(__clang__) -# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) -# elif defined(_MSC_VER) -# define _Py_ALIGN_AS(V) __declspec(align(V)) -# else -# define _Py_ALIGN_AS(V) alignas(V) -# endif -# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L -# define _Py_ALIGN_AS(V) alignas(V) -# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -# define _Py_ALIGN_AS(V) _Alignas(V) -# elif (defined(__GNUC__) || defined(__clang__)) -# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) -# elif defined(_MSC_VER) -# define _Py_ALIGN_AS(V) __declspec(align(V)) -# else -# define _Py_ALIGN_AS(V) _Alignas(V) -# endif +// +// Behavior of `alignas` alternatives: +// - `alignas` & `_Alignas`: +// - Can be used multiple times; the greatest alignment applies. +// - It is an *error* if the combined effect of all `alignas` modifiers would +// decrease the alignment. +// - Takes types or numbers. +// - May not be used on a struct definition, unless also defining a variable. +// - `__declspec(align)`: +// - Has no effect if it would decrease alignment. +// - Only takes an integer literal. +// - May be used on struct or variable definitions. +// However, when defining both the struct and the variable at once, +// `declspec(aligned)` causes compiler warning 5274 and possible ABI +// incompatibility. +// - ` __attribute__((aligned))`: +// - Has no effect if it would decrease alignment. +// - Takes types or numbers +// - May be used on struct or variable definitions. +#ifndef _Py_ALIGNED_DEF +# ifdef __cplusplus +# if __cplusplus >= 201103L +# define _Py_ALIGNED_DEF(N, T) alignas(N) alignas(T) T +# elif defined(__GNUC__) || defined(__clang__) +# define _Py_ALIGNED_DEF(N, T) __attribute__((aligned(N))) T +# elif defined(_MSC_VER) +# define _Py_ALIGNED_DEF(N, T) __declspec(align(N)) T +# else +# define _Py_ALIGNED_DEF(N, T) alignas(N) alignas(T) T +# endif +# elif defined(_MSC_VER) +# define _Py_ALIGNED_DEF(N, T) __declspec(align(N)) T +# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +# define _Py_ALIGNED_DEF(N, T) alignas(N) alignas(T) T +# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define _Py_ALIGNED_DEF(N, T) _Alignas(N) _Alignas(T) T +# elif (defined(__GNUC__) || defined(__clang__)) +# define _Py_ALIGNED_DEF(N, T) __attribute__((aligned(N))) T +# else +# define _Py_ALIGNED_DEF(N, T) _Alignas(N) _Alignas(T) T +# endif +#endif + + +// _Py_ANONYMOUS: modifier for declaring an anonymous union. +// Usage: _Py_ANONYMOUS union { ... }; +// Standards/compiler support: +// - C++ allows anonymous unions, but not structs +// - C11 and above allows anonymous unions and structs +// - MSVC has warning(disable: 4201) "nonstandard extension used : nameless +// struct/union". This is specific enough that we disable it for all of +// Python.h. +// - GCC & clang needs __extension__ before C11 +// To allow unsupported platforms which need other spellings, we use a +// predefined value of _Py_ANONYMOUS if it exists. +#ifndef _Py_ANONYMOUS +# if (defined(__GNUC__) || defined(__clang__)) \ + && !(defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) +# define _Py_ANONYMOUS __extension__ +# else +# define _Py_ANONYMOUS # endif #endif + /* Minimum value between x and y */ #define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) @@ -72,6 +116,12 @@ /* Absolute value of the number x */ #define Py_ABS(x) ((x) < 0 ? -(x) : (x)) +/* Safer implementation that avoids an undefined behavior for the minimal + value of the signed integer type if its absolute value is larger than + the maximal value of the signed integer type (in the two's complement + representations, which is common). + */ +#define _Py_ABS_CAST(T, x) ((x) >= 0 ? ((T) (x)) : ((T) (((T) -((x) + 1)) + 1u))) #define _Py_XSTRINGIFY(x) #x @@ -231,12 +281,13 @@ // "comparison of unsigned expression in '< 0' is always false". #define _Py_IS_TYPE_SIGNED(type) ((type)(-1) <= 0) -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 // 3.14 // Version helpers. These are primarily macros, but have exported equivalents. +#define _Py_PACK_VERSION(X, Y) _Py_PACK_FULL_VERSION(X, Y, 0, 0, 0) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 14) PyAPI_FUNC(uint32_t) Py_PACK_FULL_VERSION(int x, int y, int z, int level, int serial); PyAPI_FUNC(uint32_t) Py_PACK_VERSION(int x, int y); #define Py_PACK_FULL_VERSION _Py_PACK_FULL_VERSION -#define Py_PACK_VERSION(X, Y) Py_PACK_FULL_VERSION(X, Y, 0, 0, 0) +#define Py_PACK_VERSION _Py_PACK_VERSION #endif // Py_LIMITED_API < 3.14 diff --git a/Include/pymath.h b/Include/pymath.h index 0ead1f95670..7cfe441365d 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -7,6 +7,7 @@ /* High precision definition of pi and e (Euler) * The values are taken from libc6's math.h. */ +// Deprecated since Python 3.15. #ifndef Py_MATH_PIl #define Py_MATH_PIl 3.1415926535897932384626433832795029L #endif @@ -14,6 +15,7 @@ #define Py_MATH_PI 3.14159265358979323846 #endif +// Deprecated since Python 3.15. #ifndef Py_MATH_El #define Py_MATH_El 2.7182818284590452353602874713526625L #endif @@ -43,13 +45,14 @@ #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. +// Soft deprecated since Python 3.15, use INFINITY instead. #ifndef Py_INFINITY # define Py_INFINITY ((double)INFINITY) #endif /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. + * conformity. The macro was soft deprecated in Python 3.14, use INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL @@ -57,9 +60,24 @@ /* Py_NAN: Value that evaluates to a quiet Not-a-Number (NaN). The sign is * undefined and normally not relevant, but e.g. fixed for float("nan"). + * + * Note: On Solaris, NAN is a function address, hence arithmetic is impossible. + * For that reason, we instead use the built-in call if available or fallback + * to a generic NaN computed from strtod() as a last resort. + * + * See https://github.com/python/cpython/issues/136006 for details. */ #if !defined(Py_NAN) -# define Py_NAN ((double)NAN) +# if defined(__sun) +# if _Py__has_builtin(__builtin_nanf) +# define Py_NAN ((double)__builtin_nanf("")) +# else +# include +# define Py_NAN (strtod("NAN", NULL)) +# endif +# else +# define Py_NAN ((double)NAN) +# endif #endif #endif /* Py_PYMATH_H */ diff --git a/Include/pyport.h b/Include/pyport.h index 3eac119bf8e..61e2317976e 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -49,8 +49,9 @@ // Static inline functions should use _Py_NULL rather than using directly NULL // to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, // _Py_NULL is defined as nullptr. -#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ - || (defined(__cplusplus) && __cplusplus >= 201103) +#if !defined(_MSC_VER) && \ + ((defined (__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \ + || (defined(__cplusplus) && __cplusplus >= 201103)) # define _Py_NULL nullptr #else # define _Py_NULL NULL @@ -503,28 +504,30 @@ extern "C" { * Thread support is stubbed and any attempt to create a new thread fails. */ #if (!defined(HAVE_PTHREAD_STUBS) && \ + !defined(__wasi__) && \ (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) # define Py_CAN_START_THREADS 1 #endif + +/* gh-142163: Some libraries rely on HAVE_THREAD_LOCAL being undefined, so + * we can only define it only when Py_BUILD_CORE is set.*/ +#ifdef Py_BUILD_CORE +// This is no longer coupled to _Py_thread_local. +# define HAVE_THREAD_LOCAL 1 +#endif + #ifdef WITH_THREAD -# ifdef Py_BUILD_CORE -# ifdef HAVE_THREAD_LOCAL -# error "HAVE_THREAD_LOCAL is already defined" -# endif -# define HAVE_THREAD_LOCAL 1 -# ifdef thread_local -# define _Py_thread_local thread_local -# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) -# define _Py_thread_local _Thread_local -# elif defined(_MSC_VER) /* AKA NT_THREADS */ -# define _Py_thread_local __declspec(thread) -# elif defined(__GNUC__) /* includes clang */ -# define _Py_thread_local __thread -# else - // fall back to the PyThread_tss_*() API, or ignore. -# undef HAVE_THREAD_LOCAL -# endif +# ifdef thread_local +# define _Py_thread_local thread_local +# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) +# define _Py_thread_local _Thread_local +# elif defined(_MSC_VER) /* AKA NT_THREADS */ +# define _Py_thread_local __declspec(thread) +# elif defined(__GNUC__) /* includes clang */ +# define _Py_thread_local __thread +# else +# error "no supported thread-local variable storage classifier" # endif #endif @@ -666,25 +669,6 @@ extern "C" { #endif -// _Py_NO_SANITIZE_UNDEFINED(): Disable Undefined Behavior sanitizer (UBsan) -// on a function. -// -// Clang and GCC 9.0+ use __attribute__((no_sanitize("undefined"))). -// GCC 4.9+ uses __attribute__((no_sanitize_undefined)). -#if defined(__has_feature) -# if __has_feature(undefined_behavior_sanitizer) -# define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) -# endif -#endif -#if !defined(_Py_NO_SANITIZE_UNDEFINED) && defined(__GNUC__) \ - && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 9)) -# define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined)) -#endif -#ifndef _Py_NO_SANITIZE_UNDEFINED -# define _Py_NO_SANITIZE_UNDEFINED -#endif - - // _Py_NONSTRING: The nonstring variable attribute specifies that an object or // member declaration with type array of char, signed char, or unsigned char, // or pointer to such a type is intended to store character arrays that do not @@ -700,4 +684,10 @@ extern "C" { #endif +// Assume the stack grows down unless specified otherwise +#ifndef _Py_STACK_GROWS_DOWN +# define _Py_STACK_GROWS_DOWN 1 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index fad2b3c7747..92b50aa807b 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -21,39 +21,15 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *); /* Stuff with no proper home (yet) */ PyAPI_DATA(int) (*PyOS_InputHook)(void); -/* Stack size, in "pointers". This must be large enough, so - * no two calls to check recursion depth are more than this far - * apart. In practice, that means it must be larger than the C - * stack consumption of PyEval_EvalDefault */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) -# define PYOS_LOG2_STACK_MARGIN 12 -#elif defined(Py_DEBUG) && defined(WIN32) -# define PYOS_LOG2_STACK_MARGIN 12 -#elif defined(__wasi__) - /* Web assembly has two stacks, so this isn't really a size */ -# define PYOS_LOG2_STACK_MARGIN 9 -#else -# define PYOS_LOG2_STACK_MARGIN 11 -#endif -#define PYOS_STACK_MARGIN (1 << PYOS_LOG2_STACK_MARGIN) -#define PYOS_STACK_MARGIN_BYTES (PYOS_STACK_MARGIN * sizeof(void *)) - -#if SIZEOF_VOID_P == 8 -#define PYOS_STACK_MARGIN_SHIFT (PYOS_LOG2_STACK_MARGIN + 3) -#else -#define PYOS_STACK_MARGIN_SHIFT (PYOS_LOG2_STACK_MARGIN + 2) -#endif - - #if defined(WIN32) -#define USE_STACKCHECK +# define USE_STACKCHECK #endif - #ifdef USE_STACKCHECK /* Check that we aren't overflowing our stack */ PyAPI_FUNC(int) PyOS_CheckStack(void); #endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYTHONRUN_H # include "cpython/pythonrun.h" diff --git a/Include/pythread.h b/Include/pythread.h index 82247daf8e0..a8a28b8572a 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -42,7 +42,8 @@ PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void); #if (defined(__APPLE__) || defined(__linux__) || defined(_WIN32) \ || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \ || defined(__OpenBSD__) || defined(__NetBSD__) \ - || defined(__DragonFly__) || defined(_AIX)) + || defined(__DragonFly__) || defined(_AIX) \ + || (defined(__sun__) && SIZEOF_LONG >= 8)) #define PY_HAVE_THREAD_NATIVE_ID PyAPI_FUNC(unsigned long) PyThread_get_thread_native_id(void); #endif diff --git a/Include/refcount.h b/Include/refcount.h index 177bbdaf0c5..51346c7e519 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -1,5 +1,5 @@ -#ifndef Py_REFCOUNT_H -#define Py_REFCOUNT_H +#ifndef _Py_REFCOUNT_H +#define _Py_REFCOUNT_H #ifdef __cplusplus extern "C" { #endif @@ -19,9 +19,6 @@ immortal. The latter should be the only instances that require cleanup during runtime finalization. */ -#define _Py_STATICALLY_ALLOCATED_FLAG 4 -#define _Py_IMMORTAL_FLAGS 1 - #if SIZEOF_VOID_P > 4 /* In 64+ bit systems, any object whose 32 bit reference count is >= 2**31 @@ -33,7 +30,7 @@ increase and decrease the objects reference count. In order to offer sufficient resilience to C extensions using the stable ABI compiled against 3.11 or earlier, we set the initial value near the -middle of the range (2**31, 2**32). That way the the refcount can be +middle of the range (2**31, 2**32). That way the refcount can be off by ~1 billion without affecting immortality. Reference count increases will use saturated arithmetic, taking advantage of @@ -117,9 +114,12 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST(ob)) + #else + # define Py_REFCNT(ob) _Py_REFCNT(ob) #endif #endif +#ifndef _Py_OPAQUE_PYOBJECT static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) { #if defined(Py_GIL_DISABLED) @@ -143,6 +143,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op) #endif } #define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op)) +#endif // !defined(_Py_OPAQUE_PYOBJECT) // Py_SET_REFCNT() implementation for stable ABI PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); @@ -247,20 +248,6 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *); PyAPI_FUNC(void) _Py_IncRef(PyObject *); PyAPI_FUNC(void) _Py_DecRef(PyObject *); -#ifndef Py_GIL_DISABLED -static inline Py_ALWAYS_INLINE void Py_INCREF_MORTAL(PyObject *op) -{ - assert(!_Py_IsStaticImmortal(op)); - op->ob_refcnt++; - _Py_INCREF_STAT_INC(); -#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) - if (!_Py_IsImmortal(op)) { - _Py_INCREF_IncRefTotal(); - } -#endif -} -#endif - static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { #if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) @@ -564,4 +551,4 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) #ifdef __cplusplus } #endif -#endif // !Py_REFCOUNT_H +#endif // !_Py_REFCOUNT_H diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 35e2ea254ca..00c70a6e911 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -16,19 +16,6 @@ PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ /* Slice object interface */ -/* - -A slice object containing start, stop, and step data members (the -names are from range). After much talk with Guido, it was decided to -let these be any arbitrary python type. Py_None stands for omitted values. -*/ -#ifndef Py_LIMITED_API -typedef struct { - PyObject_HEAD - PyObject *start, *stop, *step; /* not NULL */ -} PySliceObject; -#endif - PyAPI_DATA(PyTypeObject) PySlice_Type; PyAPI_DATA(PyTypeObject) PyEllipsis_Type; @@ -36,12 +23,6 @@ PyAPI_DATA(PyTypeObject) PyEllipsis_Type; PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop, PyObject* step); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); -PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, - PyObject **start_ptr, PyObject **stop_ptr, - PyObject **step_ptr); -#endif PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); Py_DEPRECATED(3.7) @@ -63,6 +44,12 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, Py_ssize_t step); #endif +#ifndef Py_LIMITED_API +# define Py_CPYTHON_SLICEOBJECT_H +# include "cpython/sliceobject.h" +# undef Py_CPYTHON_SLICEOBJECT_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/structseq.h b/Include/structseq.h index 29e24fee54e..e52d6188030 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -21,12 +21,6 @@ typedef struct PyStructSequence_Desc { PyAPI_DATA(const char * const) PyStructSequence_UnnamedField; -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, - PyStructSequence_Desc *desc); -PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, - PyStructSequence_Desc *desc); -#endif PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc); PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type); @@ -35,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -typedef PyTupleObject PyStructSequence; -#define PyStructSequence_SET_ITEM PyStructSequence_SetItem -#define PyStructSequence_GET_ITEM PyStructSequence_GetItem +# define Py_CPYTHON_STRUCTSEQ_H +# include "cpython/structseq.h" +# undef Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Include/sysmodule.h b/Include/sysmodule.h index c1d5f610fe0..b7d800c5e5d 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -4,6 +4,12 @@ extern "C" { #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030f0000 +PyAPI_FUNC(PyObject *) PySys_GetAttr(PyObject *); +PyAPI_FUNC(PyObject *) PySys_GetAttrString(const char *); +PyAPI_FUNC(int) PySys_GetOptionalAttr(PyObject *, PyObject **); +PyAPI_FUNC(int) PySys_GetOptionalAttrString(const char *, PyObject **); +#endif PyAPI_FUNC(PyObject *) PySys_GetObject(const char *); PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *); @@ -17,8 +23,6 @@ PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); -Py_DEPRECATED(3.13) PyAPI_FUNC(void) PySys_ResetWarnOptions(void); - PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); #ifdef __cplusplus diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index f8bcaecb98f..b72d581ec25 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -341,49 +341,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_Decode( const char *errors /* error handling */ ); -/* Decode a Unicode object unicode and return the result as Python - object. - - This API is DEPRECATED and will be removed in 3.15. - The only supported standard encoding is rot13. - Use PyCodec_Decode() to decode with rot13 and non-standard codecs - that decode from str. */ - -Py_DEPRECATED(3.6) PyAPI_FUNC(PyObject*) PyUnicode_AsDecodedObject( - PyObject *unicode, /* Unicode object */ - const char *encoding, /* encoding */ - const char *errors /* error handling */ - ); - -/* Decode a Unicode object unicode and return the result as Unicode - object. - - This API is DEPRECATED and will be removed in 3.15. - The only supported standard encoding is rot13. - Use PyCodec_Decode() to decode with rot13 and non-standard codecs - that decode from str to str. */ - -Py_DEPRECATED(3.6) PyAPI_FUNC(PyObject*) PyUnicode_AsDecodedUnicode( - PyObject *unicode, /* Unicode object */ - const char *encoding, /* encoding */ - const char *errors /* error handling */ - ); - -/* Encodes a Unicode object and returns the result as Python - object. - - This API is DEPRECATED and will be removed in 3.15. - It is superseded by PyUnicode_AsEncodedString() - since all standard encodings (except rot13) encode str to bytes. - Use PyCodec_Encode() for encoding with rot13 and non-standard codecs - that encode form str to non-bytes. */ - -Py_DEPRECATED(3.6) PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedObject( - PyObject *unicode, /* Unicode object */ - const char *encoding, /* encoding */ - const char *errors /* error handling */ - ); - /* Encodes a Unicode object and returns the result as Python string object. */ @@ -393,20 +350,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedString( const char *errors /* error handling */ ); -/* Encodes a Unicode object and returns the result as Unicode - object. - - This API is DEPRECATED and will be removed in 3.15. - The only supported standard encodings is rot13. - Use PyCodec_Encode() to encode with rot13 and non-standard codecs - that encode from str to str. */ - -Py_DEPRECATED(3.6) PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedUnicode( - PyObject *unicode, /* Unicode object */ - const char *encoding, /* encoding */ - const char *errors /* error handling */ - ); - /* Build an encoding map. */ PyAPI_FUNC(PyObject*) PyUnicode_BuildEncodingMap( diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index a6e71eb178b..17fac62961c 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -27,7 +27,6 @@ PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, PyObject *callback); PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, PyObject *callback); -Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 PyAPI_FUNC(int) PyWeakref_GetRef(PyObject *ref, PyObject **pobj); diff --git a/InternalDocs/README.md b/InternalDocs/README.md index 4502902307c..a6e2df5ae4a 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -41,3 +41,12 @@ # CPython Internals Documentation - [Garbage Collector Design](garbage_collector.md) - [Exception Handling](exception_handling.md) + +- [Quiescent-State Based Reclamation (QSBR)](qsbr.md) + +- [Stack protection](stack_protection.md) + +Modules +--- + +- [asyncio](asyncio.md) diff --git a/InternalDocs/asyncio.md b/InternalDocs/asyncio.md new file mode 100644 index 00000000000..ca7e8a92dec --- /dev/null +++ b/InternalDocs/asyncio.md @@ -0,0 +1,328 @@ +asyncio +======= + + +This document describes the working and implementation details of the +[`asyncio`](https://docs.python.org/3/library/asyncio.html) module. + +**The following section describes the implementation details of the C implementation**. + +# Task management + +## Pre-Python 3.14 implementation + +Before Python 3.14, the C implementation of `asyncio` used a +[`WeakSet`](https://docs.python.org/3/library/weakref.html#weakref.WeakSet) +to store all the tasks created by the event loop. `WeakSet` was used +so that the event loop doesn't hold strong references to the tasks, +allowing them to be garbage collected when they are no longer needed. +The current task of the event loop was stored in a dict mapping the +event loop to the current task. + +```c + /* Dictionary containing tasks that are currently active in + all running event loops. {EventLoop: Task} */ + PyObject *current_tasks; + + /* WeakSet containing all tasks scheduled to run on event loops. */ + PyObject *scheduled_tasks; +``` + +This implementation had a few drawbacks: + +1. **Performance**: Using a `WeakSet` for storing tasks is +inefficient, as it requires maintaining a full set of weak references +to tasks along with corresponding weakref callback to cleanup the +tasks when they are garbage collected. This increases the work done +by the garbage collector, and in applications with a large number of +tasks, this becomes a bottleneck, with increased memory usage and +lower performance. Looking up the current task was slow as it required +a dictionary lookup on the `current_tasks` dict. + +2. **Thread safety**: Before Python 3.14, concurrent iterations over +`WeakSet` was not thread safe[^1]. This meant calling APIs like +`asyncio.all_tasks()` could lead to inconsistent results or even +`RuntimeError` if used in multiple threads[^2]. + +3. **Poor scaling in free-threading**: Using global `WeakSet` for +storing all tasks across all threads lead to contention when adding +and removing tasks from the set which is a frequent operation. As such +it performed poorly in free-threading and did not scale well with the +number of threads. Similarly, accessing the current task in multiple +threads did not scale due to contention on the global `current_tasks` +dictionary. + +## Python 3.14 implementation + +To address these issues, Python 3.14 implements several changes to +improve the performance and thread safety of tasks management. + +- **Per-thread double linked list for tasks**: Python 3.14 introduces + a per-thread circular double linked list implementation for + storing tasks. This allows each thread to maintain its own list of + tasks and allows for lock free addition and removal of tasks. This + is designed to be efficient, and thread-safe and scales well with + the number of threads in free-threading. This also allows external + introspection tools such as `python -m asyncio pstree` to inspect + tasks running in all threads and was implemented as part of [Audit + asyncio thread + safety](https://github.com/python/cpython/issues/128002). + +- **Per-thread current task**: Python 3.14 stores the current task on + the current thread state instead of a global dictionary. This + allows for faster access to the current task without the need for + a dictionary lookup. Each thread maintains its own current task, + which is stored in the `PyThreadState` structure. This was + implemented in https://github.com/python/cpython/issues/129898. + +Storing the current task and list of all tasks per-thread instead of +storing it per-loop was chosen primarily to support external +introspection tools such as `python -m asyncio pstree` as looking up +arbitrary attributes on the loop object is not possible +externally. Storing data per-thread also makes it easy to support +third party event loop implementations such as `uvloop`, and is more +efficient for the single threaded asyncio use-case as it avoids the +overhead of attribute lookups on the loop object and several other +calls on the performance critical path of adding and removing tasks +from the per-loop task list. + +## Per-thread double linked list for tasks + +This implementation uses a circular doubly linked list to store tasks +on the thread states. This is used for all tasks which are instances +of `asyncio.Task` or subclasses of it, for third-party tasks a +fallback `WeakSet` implementation is used. The linked list is +implemented using an embedded `llist_node` structure within each +`TaskObj`. By embedding the list node directly into the task object, +the implementation avoids additional memory allocations for linked +list nodes. + +The `PyThreadState` structure gained a new field `asyncio_tasks_head`, +which serves as the head of the circular linked list of tasks. This +allows for lock free addition and removal of tasks from the list. + +It is possible that when a thread state is deallocated, there are +lingering tasks in its list; this can happen if another thread has +references to the tasks of this thread. Therefore, the +`PyInterpreterState` structure also gains a new `asyncio_tasks_head` +field to store any lingering tasks. When a thread state is +deallocated, any remaining lingering tasks are moved to the +interpreter state tasks list, and the thread state tasks list is +cleared. The `asyncio_tasks_lock` is used protect the interpreter's +tasks list from concurrent modifications. + +```c +typedef struct TaskObj { + ... + struct llist_node asyncio_node; +} TaskObj; + +typedef struct PyThreadState { + ... + struct llist_node asyncio_tasks_head; +} PyThreadState; + +typedef struct PyInterpreterState { + ... + struct llist_node asyncio_tasks_head; + PyMutex asyncio_tasks_lock; +} PyInterpreterState; +``` + +When a task is created, it is added to the current thread's list of +tasks by the `register_task` function. When the task is done, it is +removed from the list by the `unregister_task` function. In +free-threading, the thread id of the thread which created the task is +stored in `task_tid` field of the `TaskObj`. This is used to check if +the task is being removed from the correct thread's task list. If the +current thread is same as the thread which created it then no locking +is required, otherwise in free-threading, the `stop-the-world` pause +is used to pause all other threads and then safely remove the task +from the tasks list. + +```mermaid + +flowchart TD + subgraph one["Executing Thread"] + A["task = asyncio.create_task(coro())"] -->B("register_task(task)") + B --> C{"task->task_state?"} + C -->|pending| D["task_step(task)"] + C -->|done| F["unregister_task(task)"] + C -->|cancelled| F["unregister_task(task)"] + D --> C + F --> G{"free-threading?"} + G --> |false| H["unregister_task_safe(task)"] + G --> |true| J{"correct thread?
task->task_tid == _Py_ThreadId()"} + J --> |true| H + J --> |false| I["stop the world
pause all threads"] + I --> H["unregister_task_safe(task)"] + end + subgraph two["Thread deallocating"] + A1{"thread's task list empty?
llist_empty(tstate->asyncio_tasks_head)"} + A1 --> |true| B1["deallocate thread
free_threadstate(tstate)"] + A1 --> |false| C1["add tasks to interpreter's task list
llist_concat(&tstate->interp->asyncio_tasks_head, + &tstate->asyncio_tasks_head)"] + C1 --> B1 + end + + one --> two +``` + +`asyncio.all_tasks` now iterates over the per-thread task lists of all +threads and the interpreter's task list to get all the tasks. In +free-threading, this is done by pausing all the threads using the +`stop-the-world` pause to ensure that no tasks are being added or +removed while iterating over the lists. This allows for a consistent +view of all task lists across all threads and is thread safe. + +This design allows for lock free execution and scales well in +free-threading with multiple event loops running in different threads. + +## Per-thread current task + +This implementation stores the current task in the `PyThreadState` +structure, which allows for faster access to the current task without +the need for a dictionary lookup. + +```c +typedef struct PyThreadState { + ... + PyObject *asyncio_current_loop; + PyObject *asyncio_current_task; +} PyThreadState; +``` + +When a task is entered or left, the current task is updated in the +thread state using `enter_task` and `leave_task` functions. When +`current_task(loop)` is called where `loop` is the current running +event loop of the current thread, no locking is required as the +current task is stored in the thread state and is returned directly +(general case). Otherwise, if the `loop` is not current running event +loop, the `stop-the-world` pause is used to pause all threads in +free-threading and then by iterating over all the thread states and +checking if the `loop` matches with `tstate->asyncio_current_loop`, +the current task is found and returned. If no matching thread state is +found, `None` is returned. + +In free-threading, it avoids contention on a global dictionary as +threads can access the current task of their running loop without any +locking. + +--- + +**The following section describes the implementation details of the Python implementation**. + +# async generators + +This section describes the implementation details of async generators in `asyncio`. + +Since async generators are meant to be used from coroutines, +their finalization (execution of finally blocks) needs +to be done while the loop is running. +Most async generators are closed automatically +when they are fully iterated over and exhausted; however, +if the async generator is not fully iterated over, +it may not be closed properly, leading to the `finally` blocks not being executed. + +Consider the following code: +```py +import asyncio + +async def agen(): + try: + yield 1 + finally: + await asyncio.sleep(1) + print("finally executed") + + +async def main(): + async for i in agen(): + break + +loop = asyncio.EventLoop() +loop.run_until_complete(main()) +``` + +The above code will not print "finally executed", because the +async generator `agen` is not fully iterated over +and it is not closed manually by awaiting `agen.aclose()`. + +To solve this, `asyncio` uses the `sys.set_asyncgen_hooks` function to +set hooks for finalizing async generators as described in +[PEP 525](https://peps.python.org/pep-0525/). + +- **firstiter hook**: When the async generator is iterated over for the first time, +the *firstiter hook* is called. The async generator is added to `loop._asyncgens` WeakSet +and the event loop tracks all active async generators. + +- **finalizer hook**: When the async generator is about to be finalized, +the *finalizer hook* is called. The event loop removes the async generator +from `loop._asyncgens` WeakSet, and schedules the finalization of the async +generator by creating a task calling `agen.aclose()`. This ensures that the +finally block is executed while the event loop is running. When the loop is +shutting down, the loop checks if there are active async generators and if so, +it similarly schedules the finalization of all active async generators by calling +`agen.aclose()` on each of them and waits for them to complete before shutting +down the loop. + +This ensures that the async generator's `finally` blocks are executed even +if the generator is not explicitly closed. + +Consider the following example: + +```python +import asyncio + +async def agen(): + try: + yield 1 + yield 2 + finally: + print("executing finally block") + +async def main(): + async for item in agen(): + print(item) + break # not fully iterated + +asyncio.run(main()) +``` + +```mermaid +flowchart TD + subgraph one["Loop running"] + A["asyncio.run(main())"] --> B + B["set async generator hooks
sys.set_asyncgen_hooks()"] --> C + C["async for item in agen"] --> F + F{"first iteration?"} --> |true|D + F{"first iteration?"} --> |false|H + D["calls firstiter hook
loop._asyncgen_firstiter_hook(agen)"] --> E + E["add agen to WeakSet
loop._asyncgens.add(agen)"] --> H + H["item = await agen.\_\_anext\_\_()"] --> J + J{"StopAsyncIteration?"} --> |true|M + J{"StopAsyncIteration?"} --> |false|I + I["print(item)"] --> S + S{"continue iterating?"} --> |true|C + S{"continue iterating?"} --> |false|M + M{"agen is no longer referenced?"} --> |true|N + M{"agen is no longer referenced?"} --> |false|two + N["finalize agen
_PyGen_Finalize(agen)"] --> O + O["calls finalizer hook
loop._asyncgen_finalizer_hook(agen)"] --> P + P["remove agen from WeakSet
loop._asyncgens.discard(agen)"] --> Q + Q["schedule task to close it
self.create_task(agen.aclose())"] --> R + R["print('executing finally block')"] --> E1 + + end + + subgraph two["Loop shutting down"] + A1{"check for alive async generators?"} --> |true|B1 + B1["close all async generators
await asyncio.gather\(*\[ag.aclose\(\) for ag in loop._asyncgens\]"] --> R + A1{"check for alive async generators?"} --> |false|E1 + E1["loop.close()"] + end + +``` + +[^1]: https://github.com/python/cpython/issues/123089 +[^2]: https://github.com/python/cpython/issues/80788 diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 02bbdf6071f..742af5efcf5 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -604,4 +604,4 @@ pp. 213--227, 1997. [^2]: The Zephyr Abstract Syntax Description Language.: - https://www.cs.princeton.edu/research/techreps/TR-554-97 + https://www.cs.princeton.edu/research/techreps/254 diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md index 28589787e1f..9e38da4c862 100644 --- a/InternalDocs/exception_handling.md +++ b/InternalDocs/exception_handling.md @@ -8,7 +8,7 @@ The following code: -``` +```python try: g(0) except: @@ -18,7 +18,7 @@ compiles into intermediate code like the following: -``` +```python RESUME 0 1 SETUP_FINALLY 8 (to L1) @@ -118,13 +118,13 @@ We want the format to be compact, but quickly searchable. For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. -For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. +For it to be searchable quickly, we need to support binary search giving us `log(n)` performance in all cases. Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: `start, size, target, depth, push-lasti`. -Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. +Also, sizes are limited to `2**30` as the code length cannot exceed `2**31` and each code unit takes 2 bytes. It also happens that depth is generally quite small. So, we need to encode: @@ -140,7 +140,7 @@ We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. -The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. +The 8 bits of a byte are (msb left) `SXdddddd` where `S` is the start bit. `X` is the extend bit meaning that the next byte is required to extend the offset. In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 804d7436018..60ab2055afa 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -111,6 +111,55 @@ ### Shim frames instruction which cleans up the shim frame and returns. +### Base frame + +Each thread state contains an embedded `_PyInterpreterFrame` called the "base frame" +that serves as a sentinel at the bottom of the frame stack. This frame is allocated +in `_PyThreadStateImpl` (the internal extension of `PyThreadState`) and initialized +when the thread state is created. The `owner` field is set to `FRAME_OWNED_BY_INTERPRETER`. + +External profilers and sampling tools can validate that they have successfully unwound +the complete call stack by checking that the frame chain terminates at the base frame. +The `PyThreadState.base_frame` pointer provides the expected address to compare against. +If a stack walk doesn't reach this frame, the sample is incomplete (possibly due to a +race condition) and should be discarded. + +The base frame is embedded in `_PyThreadStateImpl` rather than `PyThreadState` because +`_PyInterpreterFrame` is defined in internal headers that cannot be exposed in the +public API. A pointer (`PyThreadState.base_frame`) is provided for profilers to access +the address without needing internal headers. + +See the initialization in `new_threadstate()` in [Python/pystate.c](../Python/pystate.c). + +#### How profilers should use the base frame + +External profilers should read `tstate->base_frame` before walking the stack, then +walk from `tstate->current_frame` following `frame->previous` pointers until reaching +a frame with `owner == FRAME_OWNED_BY_INTERPRETER`. After the walk, verify that the +last frame address matches `base_frame`. If not, discard the sample as incomplete +since the frame chain may have been in an inconsistent state due to concurrent updates. + + +### Remote Profiling Frame Cache + +The `last_profiled_frame` field in `PyThreadState` supports an optimization for +remote profilers that sample call stacks from external processes. When a remote +profiler reads the call stack, it writes the current frame address to this field. +The eval loop then keeps this pointer valid by updating it to the parent frame +whenever a frame returns (in `_PyEval_FrameClearAndPop`). + +This creates a "high-water mark" that always points to a frame still on the stack. +On subsequent samples, the profiler can walk from `current_frame` until it reaches +`last_profiled_frame`, knowing that frames from that point downward are unchanged +and can be retrieved from a cache. This significantly reduces the amount of remote +memory reads needed when call stacks are deep and stable at their base. + +The update in `_PyEval_FrameClearAndPop` is guarded: it only writes when +`last_profiled_frame` is non-NULL AND matches the frame being popped. This +prevents transient frames (called and returned between profiler samples) from +corrupting the cache pointer, while avoiding any overhead when profiling is inactive. + + ### The Instruction Pointer `_PyInterpreterFrame` has two fields which are used to maintain the instruction diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 4da6cd47dc8..a7d872f3ec4 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -286,7 +286,7 @@ Notice that an object that was marked as "tentatively unreachable" and was later moved back to the reachable list will be visited again by the garbage collector -as now all the references that that object has need to be processed as well. This +as now all the references that the object has need to be processed as well. This process is really a breadth first search over the object graph. Once all the objects are scanned, the GC knows that all container objects in the tentatively unreachable list are really unreachable and can thus be garbage collected. @@ -329,15 +329,16 @@ with the objective of completely destroying these objects. Roughly, the process follows these steps in order: -1. Handle and clear weak references (if any). Weak references to unreachable objects - are set to `None`. If the weak reference has an associated callback, the callback - is enqueued to be called once the clearing of weak references is finished. We only - invoke callbacks for weak references that are themselves reachable. If both the weak - reference and the pointed-to object are unreachable we do not execute the callback. - This is partly for historical reasons: the callback could resurrect an unreachable - object and support for weak references predates support for object resurrection. - Ignoring the weak reference's callback is fine because both the object and the weakref - are going away, so it's legitimate to say the weak reference is going away first. +1. Handle weak references with callbacks (if any). If the weak reference has + an associated callback, the callback is enqueued to be called after the weak + reference is cleared. We only invoke callbacks for weak references that + are themselves reachable. If both the weak reference and the pointed-to + object are unreachable we do not execute the callback. This is partly for + historical reasons: the callback could resurrect an unreachable object + and support for weak references predates support for object resurrection. + Ignoring the weak reference's callback is fine because both the object and + the weakref are going away, so it's legitimate to say the weak reference is + going away first. 2. If an object has legacy finalizers (`tp_del` slot) move it to the `gc.garbage` list. 3. Call the finalizers (`tp_finalize` slot) and mark the objects as already @@ -346,7 +347,12 @@ 4. Deal with resurrected objects. If some objects have been resurrected, the GC finds the new subset of objects that are still unreachable by running the cycle detection algorithm again and continues with them. -5. Call the `tp_clear` slot of every object so all internal links are broken and +5. Clear any weak references that still refer to unreachable objects. The + `wr_object` attribute for these weakrefs are set to `None`. Note that some + of these weak references maybe have been newly created during the running of + finalizers in step 3. Also, clear any weak references that are part of the + unreachable set. +6. Call the `tp_clear` slot of every object so all internal links are broken and the reference counts fall to 0, triggering the destruction of all unreachable objects. diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index 87fbb912368..fa652cd231c 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -28,9 +28,9 @@ The `frame` of a generator is embedded in the generator object struct as a [`_PyInterpreterFrame`](frames.md) (see `_PyGenObject_HEAD` in -[`pycore_genobject.h`](../Include/internal/pycore_genobject.h)). +[`pycore_interpframe_structs.h`](../Include/internal/pycore_interpframe_structs.h)). This means that we can get the frame from the generator or the generator -from the frame (see `_PyGen_GetGeneratorFromFrame` in the same file). +from the frame (see `_PyGen_GetGeneratorFromFrame` in [`pycore_genobject.h`](../Include/internal/pycore_genobject.h)). Other fields of the generator struct include metadata (such as the name of the generator function) and runtime state information (such as whether its frame is executing, suspended, cleared, etc.). @@ -64,7 +64,7 @@ The [`FOR_ITER`](https://docs.python.org/dev/library/dis.html#opcode-FOR_ITER) instruction calls `__next__` on the iterator which is on the top of the stack, -and pushes the result to the stack. It has [`specializations`](adaptive.md) +and pushes the result to the stack. It has [`specializations`](interpreter.md) for a few common iterator types, including `FOR_ITER_GEN`, for iterating over a generator. `FOR_ITER_GEN` bypasses the call to `__next__`, and instead directly pushes the generator stack and resumes its execution from the diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index c98ccb3ace8..1740b22b85f 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -53,7 +53,7 @@ ## The micro-op optimizer ## The JIT interpreter After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop -executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. +executor, it transfers control to this executor via the `TIER1_TO_TIER2` macro. CPython implements two executors. Here we describe the JIT interpreter, which is the simpler of them and is therefore useful for debugging and analyzing @@ -89,7 +89,7 @@ ## The JIT [`--enable-experimental-jit`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit), the uop executor's `jit_code` field is populated with a pointer to a compiled C function that implements the executor logic. This function's signature is -defined by `jit_func` in [`pycore_jit.h`](Include/internal/pycore_jit.h). +defined by `jit_func` in [`pycore_jit.h`](../Include/internal/pycore_jit.h). When the executor is invoked by `ENTER_EXECUTOR`, instead of jumping to the uop interpreter at `tier2_dispatch`, the executor runs the function that `jit_code` points to. This function returns the instruction pointer diff --git a/InternalDocs/qsbr.md b/InternalDocs/qsbr.md new file mode 100644 index 00000000000..1c4a79a7b44 --- /dev/null +++ b/InternalDocs/qsbr.md @@ -0,0 +1,129 @@ +# Quiescent-State Based Reclamation (QSBR) + +## Introduction + +When implementing lock-free data structures, a key challenge is determining +when it is safe to free memory that has been logically removed from a +structure. Freeing memory too early can lead to use-after-free bugs if another +thread is still accessing it. Freeing it too late results in excessive memory +consumption. + +Safe memory reclamation (SMR) schemes address this by delaying the free +operation until all concurrent read accesses are guaranteed to have completed. +Quiescent-State Based Reclamation (QSBR) is a SMR scheme used in Python's +free-threaded build to manage the lifecycle of shared memory. + +QSBR requires threads to periodically report that they are in a quiescent +state. A thread is in a quiescent state if it holds no references to shared +objects that might be reclaimed. Think of it as a checkpoint where a thread +signals, "I am not in the middle of any operation that relies on a shared +resource." In Python, the eval_breaker provides a natural and convenient place +for threads to report this state. + + +## Use in Free-Threaded Python + +While CPython's memory management is dominated by reference counting and a +tracing garbage collector, these mechanisms are not suitable for all data +structures. For example, the backing array of a list object is not individually +reference-counted but may have a shorter lifetime than the `PyListObject` that +contains it. We could delay reclamation until the next GC run, but we want +reclamation to be prompt and to run the GC less frequently in the free-threaded +build, as it requires pausing all threads. + +Many operations in the free-threaded build are protected by locks. However, for +performance-critical code, we want to allow reads to happen concurrently with +updates. For instance, we want to avoid locking during most list read accesses. +If a list is resized while another thread is reading it, QSBR provides the +mechanism to determine when it is safe to free the list's old backing array. + +Specific use cases for QSBR include: + +* Dictionary keys (`PyDictKeysObject`) and list arrays (`_PyListArray`): When a +dictionary or list that may be shared between threads is resized, we use QSBR +to delay freeing the old keys or array until it's safe. For dicts and lists +that are not shared, their storage can be freed immediately upon resize. + +* Mimalloc `mi_page_t`: Non-locking dictionary and list accesses require +cooperation from the memory allocator. If an object is freed and its memory is +reused, we must ensure the new object's reference count field is at the same +memory location. In practice, this means when a mimalloc page (`mi_page_t`) +becomes empty, we don't immediately allow it to be reused for allocations of a +different size class. QSBR is used to determine when it's safe to repurpose the +page or return its memory to the OS. + + +## Implementation Details + + +### Core Implementation + +The proposal to add QSBR to Python is contained in +[Github issue 115103](https://github.com/python/cpython/issues/115103). +Many details of that proposal have been copied here, so they can be kept +up-to-date with the actual implementation. + +Python's QSBR implementation is based on FreeBSD's "Global Unbounded +Sequences." [^1][^2][^3]. It relies on a few key counters: + +* Global Write Sequence (`wr_seq`): A per-interpreter counter, `wr_seq`, is started +at 1 and incremented by 2 each time it is advanced. This ensures its value is +always odd, which can be used to distinguish it from other state values. When +an object needs to be reclaimed, `wr_seq` is advanced, and the object is tagged +with this new sequence number. + +* Per-Thread Read Sequence: Each thread has a local read sequence counter. When +a thread reaches a quiescent state (e.g., at the eval_breaker), it copies the +current global `wr_seq` to its local counter. + +* Global Read Sequence (`rd_seq`): This per-interpreter value stores the minimum +of all per-thread read sequence counters (excluding detached threads). It is +updated by a "polling" operation. + +To free an object, the following steps are taken: + +1. Advance the global `wr_seq`. + +2. Add the object's pointer to a deferred-free list, tagging it with the new + `wr_seq` value as its qsbr_goal. + +Periodically, a polling mechanism processes this deferred-free list: + +1. The minimum read sequence value across all active threads is calculated and + stored as the global `rd_seq`. + +2. For each item on the deferred-free list, if its qsbr_goal is less than or + equal to the new `rd_seq`, its memory is freed, and it is removed from the: + list. Otherwise, it remains on the list for a future attempt. + + +### Deferred Advance Optimization + +To reduce memory contention from frequent updates to the global `wr_seq`, its +advancement is sometimes deferred. Instead of incrementing `wr_seq` on every +reclamation request, each thread tracks its number of deferrals locally. Once +the deferral count reaches a limit (QSBR_DEFERRED_LIMIT, currently 10), the +thread advances the global `wr_seq` and resets its local count. + +When an object is added to the deferred-free list, its qsbr_goal is set to +`wr_seq` + 2. By setting the goal to the next sequence value, we ensure it's safe +to defer the global counter advancement. This optimization improves runtime +speed but may increase peak memory usage by slightly delaying when memory can +be reclaimed. + + +## Limitations + +Determining the `rd_seq` requires scanning over all thread states. This operation +could become a bottleneck in applications with a very large number of threads +(e.g., >1,000). Future work may address this with more advanced mechanisms, +such as a tree-based structure or incremental scanning. For now, the +implementation prioritizes simplicity, with plans for refinement if +multi-threaded benchmarks reveal performance issues. + + +## References + +[^1]: https://youtu.be/ZXUIFj4nRjk?t=694 +[^2]: https://people.kernel.org/joelfernandes/gus-vs-rcu +[^3]: http://bxr.su/FreeBSD/sys/kern/subr_smr.c#44 diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md new file mode 100644 index 00000000000..14802e57d09 --- /dev/null +++ b/InternalDocs/stack_protection.md @@ -0,0 +1,68 @@ +# Stack Protection + +CPython protects against stack overflow in the form of runaway, or just very deep, recursion by raising a `RecursionError` instead of just crashing. +Protection against pure Python stack recursion has existed since very early, but in 3.12 we added protection against stack overflow +in C code. This was initially implemented using a counter and later improved in 3.14 to use the actual stack depth. +For those platforms that support it (Windows, Mac, and most Linuxes) we query the operating system to find the stack bounds. +For other platforms we use conservative estimates. + + +The C stack looks like this: + +``` + +-------+ <--- Top of machine stack + | | + | | + + ~~ + + | | + | | + +-------+ <--- Soft limit + | | + | | _PyOS_STACK_MARGIN_BYTES + | | + +-------+ <--- Hard limit + | | + | | _PyOS_STACK_MARGIN_BYTES + | | + +-------+ <--- Bottom of machine stack +``` + + +We get the current stack pointer using compiler intrinsics where available, or by taking the address of a C local variable. See `_Py_get_machine_stack_pointer()`. + +The soft and hard limits pointers are set by calling `_Py_InitializeRecursionLimits()` during thread initialization. + +Recursion checks are performed by `_Py_EnterRecursiveCall()` or `_Py_EnterRecursiveCallTstate()` which compare the stack pointer to the soft limit. If the stack pointer is lower than the soft limit, then `_Py_CheckRecursiveCall()` is called which checks against both the hard and soft limits: + +```python +kb_used = (stack_top - stack_pointer)>>10 +if stack_pointer < bottom_of_machine_stack: + pass # Our stack limits could be wrong so it is safest to do nothing. +elif stack_pointer < hard_limit: + FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") +elif stack_pointer < soft_limit: + raise RecursionError(f"Stack overflow (used {kb_used} kB)") +``` + +### User space threads and other oddities + +Some libraries provide user-space threads. These will change the C stack at runtime. +To guard against this we only raise if the stack pointer is in the window between the expected stack base and the soft limit. + +### Diagnosing and fixing stack overflows + +For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. + +If you see a traceback ending in: `RecursionError: Stack overflow (used ... kB)` then the stack protection is working as intended. If you don't expect to see the error, then check the amount of stack used. If it seems low then CPython may not be configured properly. + +However, if you see a fatal error or crash, then something is not right. +Either a recursive call is not checking `_Py_EnterRecursiveCall()`, or the amount of C stack consumed by a single call exceeds `_PyOS_STACK_MARGIN_BYTES`. If a hard crash occurs, it probably means that the amount of C stack consumed is more than double `_PyOS_STACK_MARGIN_BYTES`. + +Likely causes: +* Recursive code is not calling `_Py_EnterRecursiveCall()` +* `-O0` compilation flags, especially for Clang. With no optimization, C calls can consume a lot of stack space +* Giant, complex functions in third-party C extensions. This is unlikely as the function in question would need to be more complicated than the bytecode interpreter. +* `_PyOS_STACK_MARGIN_BYTES` is just too low. +* `_Py_InitializeRecursionLimits()` is not setting the soft and hard limits correctly for that platform. diff --git a/Lib/_android_support.py b/Lib/_android_support.py index ae506f6a4b5..a439d03a144 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -29,15 +29,19 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): global logcat logcat = Logcat(android_log_write) - - sys.stdout = TextLogStream( - stdout_prio, "python.stdout", sys.stdout.fileno()) - sys.stderr = TextLogStream( - stderr_prio, "python.stderr", sys.stderr.fileno()) + sys.stdout = TextLogStream(stdout_prio, "python.stdout", sys.stdout) + sys.stderr = TextLogStream(stderr_prio, "python.stderr", sys.stderr) class TextLogStream(io.TextIOWrapper): - def __init__(self, prio, tag, fileno=None, **kwargs): + def __init__(self, prio, tag, original=None, **kwargs): + # Respect the -u option. + if original: + kwargs.setdefault("write_through", original.write_through) + fileno = original.fileno() + else: + fileno = None + # The default is surrogateescape for stdout and backslashreplace for # stderr, but in the context of an Android log, readability is more # important than reversibility. diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 0b669edb2ff..1c8741b5a55 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -626,32 +626,11 @@ def _write_ftstring(self, values, prefix): ) self._ftstring_helper(fstring_parts) - def _tstring_helper(self, node): - last_idx = 0 - for i, value in enumerate(node.values): - # This can happen if we have an implicit concat of a t-string - # with an f-string - if isinstance(value, FormattedValue): - if i > last_idx: - # Write t-string until here - self._write_ftstring(node.values[last_idx:i], "t") - self.write(" ") - # Write f-string with the current formatted value - self._write_ftstring([node.values[i]], "f") - if i + 1 < len(node.values): - # Only add a space if there are more values after this - self.write(" ") - last_idx = i + 1 - - if last_idx < len(node.values): - # Write t-string from last_idx to end - self._write_ftstring(node.values[last_idx:], "t") - def visit_JoinedStr(self, node): self._write_ftstring(node.values, "f") def visit_TemplateStr(self, node): - self._tstring_helper(node) + self._write_ftstring(node.values, "t") def _write_ftstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): @@ -679,9 +658,12 @@ def _unparse_interpolation_value(self, inner): unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) - def _write_interpolation(self, node): + def _write_interpolation(self, node, use_str_attr=False): with self.delimit("{", "}"): - expr = self._unparse_interpolation_value(node.value) + if use_str_attr: + expr = node.str + else: + expr = self._unparse_interpolation_value(node.value) if expr.startswith("{"): # Separate pair of opening brackets as "{ {" self.write(" ") @@ -696,7 +678,8 @@ def visit_FormattedValue(self, node): self._write_interpolation(node) def visit_Interpolation(self, node): - self._write_interpolation(node) + # If `str` is set to `None`, use the `value` to generate the source code. + self._write_interpolation(node, use_str_attr=node.str is not None) def visit_Name(self, node): self.write(node.id) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 51263d696a1..60b471317ce 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -1061,6 +1061,41 @@ def count(self, value): Sequence.register(range) Sequence.register(memoryview) +class _DeprecateByteStringMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + if name != "ByteString": + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__new__(cls, name, bases, namespace, **kwargs) + + def __instancecheck__(cls, instance): + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__instancecheck__(instance) + +class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): + """Deprecated ABC serving as a common supertype of ``bytes`` and ``bytearray``. + + This ABC is scheduled for removal in Python 3.17. + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the buffer protocol at runtime. For use in type annotations, + either use ``Buffer`` or a union that explicitly specifies the types your + code supports (e.g., ``bytes | bytearray | memoryview``). + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + class MutableSequence(Sequence): """All the operations on a read-write sequence. @@ -1130,3 +1165,13 @@ def __iadd__(self, values): MutableSequence.register(list) MutableSequence.register(bytearray) + +_deprecated_ByteString = globals().pop("ByteString") + +def __getattr__(attr): + if attr == "ByteString": + import warnings + warnings._deprecated("collections.abc.ByteString", remove=(3, 17)) + globals()["ByteString"] = _deprecated_ByteString + return _deprecated_ByteString + raise AttributeError(f"module 'collections.abc' has no attribute {attr!r}") diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 4a310a40235..29d7cc67b6e 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,4 +1,3 @@ -import io import os import sys @@ -155,7 +154,7 @@ def __iter__(self) -> Iterator[str]: return iter(self.__dataclass_fields__) -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Argparse(ThemeSection): usage: str = ANSIColors.BOLD_BLUE prog: str = ANSIColors.BOLD_MAGENTA @@ -169,13 +168,30 @@ class Argparse(ThemeSection): short_option: str = ANSIColors.BOLD_GREEN label: str = ANSIColors.BOLD_YELLOW action: str = ANSIColors.BOLD_GREEN + default: str = ANSIColors.GREY + default_value: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + error: str = ANSIColors.BOLD_MAGENTA + warning: str = ANSIColors.BOLD_YELLOW + message: str = ANSIColors.MAGENTA + + +@dataclass(frozen=True, kw_only=True) +class Difflib(ThemeSection): + """A 'git diff'-like theme for `difflib.unified_diff`.""" + added: str = ANSIColors.GREEN + context: str = ANSIColors.RESET # context lines + header: str = ANSIColors.BOLD # eg "---" and "+++" lines + hunk: str = ANSIColors.CYAN # the "@@" lines + removed: str = ANSIColors.RED reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE builtin: str = ANSIColors.CYAN comment: str = ANSIColors.RED string: str = ANSIColors.GREEN @@ -186,7 +202,7 @@ class Syntax(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA message: str = ANSIColors.MAGENTA @@ -198,7 +214,7 @@ class Traceback(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Unittest(ThemeSection): passed: str = ANSIColors.GREEN warn: str = ANSIColors.YELLOW @@ -207,7 +223,7 @@ class Unittest(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Theme: """A suite of themes for all sections of Python. @@ -215,6 +231,7 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + difflib: Difflib = field(default_factory=Difflib) syntax: Syntax = field(default_factory=Syntax) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -223,6 +240,7 @@ def copy_with( self, *, argparse: Argparse | None = None, + difflib: Difflib | None = None, syntax: Syntax | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, @@ -234,6 +252,7 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + difflib=difflib or self.difflib, syntax=syntax or self.syntax, traceback=traceback or self.traceback, unittest=unittest or self.unittest, @@ -249,6 +268,7 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + difflib=Difflib.no_colors(), syntax=Syntax.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), @@ -272,21 +292,29 @@ def decolor(text: str) -> str: def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + if file is None: file = sys.stdout if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": + if _safe_getenv("PYTHON_COLORS") == "0": return False - if os.environ.get("PYTHON_COLORS") == "1": + if _safe_getenv("PYTHON_COLORS") == "1": return True - if os.environ.get("NO_COLOR"): + if _safe_getenv("NO_COLOR"): return False if not COLORIZE: return False - if os.environ.get("FORCE_COLOR"): + if _safe_getenv("FORCE_COLOR"): return True - if os.environ.get("TERM") == "dumb": + if _safe_getenv("TERM") == "dumb": return False if not hasattr(file, "fileno"): @@ -303,7 +331,7 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: try: return os.isatty(file.fileno()) - except io.UnsupportedOperation: + except OSError: return hasattr(file, "isatty") and file.isatty() @@ -329,7 +357,8 @@ def get_theme( environment (including environment variable state and console configuration on Windows) can also change in the course of the application life cycle. """ - if force_color or (not force_no_color and can_colorize(file=tty_file)): + if force_color or (not force_no_color and + can_colorize(file=tty_file)): return _theme return theme_no_color diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 439f8c02f4b..a9813264324 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -175,7 +175,6 @@ 'SimpleDialog': 'tkinter.simpledialog', 'DocXMLRPCServer': 'xmlrpc.server', 'SimpleHTTPServer': 'http.server', - 'CGIHTTPServer': 'http.server', # For compatibility with broken pickles saved in old Python 3 versions 'UserDict': 'collections', 'UserList': 'collections', @@ -217,8 +216,6 @@ ('DocXMLRPCServer', 'DocCGIXMLRPCRequestHandler'), ('http.server', 'SimpleHTTPRequestHandler'): ('SimpleHTTPServer', 'SimpleHTTPRequestHandler'), - ('http.server', 'CGIHTTPRequestHandler'): - ('CGIHTTPServer', 'CGIHTTPRequestHandler'), ('_socket', 'socket'): ('socket', '_socketobject'), }) diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index b9304ec3c03..e681cb17e43 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -6,10 +6,6 @@ _specializations = { "RESUME": [ "RESUME_CHECK", ], - "LOAD_CONST": [ - "LOAD_CONST_MORTAL", - "LOAD_CONST_IMMORTAL", - ], "TO_BOOL": [ "TO_BOOL_ALWAYS_TRUE", "TO_BOOL_BOOL", @@ -186,36 +182,35 @@ _specialized_opmap = { 'LOAD_ATTR_PROPERTY': 187, 'LOAD_ATTR_SLOT': 188, 'LOAD_ATTR_WITH_HINT': 189, - 'LOAD_CONST_IMMORTAL': 190, - 'LOAD_CONST_MORTAL': 191, - 'LOAD_GLOBAL_BUILTIN': 192, - 'LOAD_GLOBAL_MODULE': 193, - 'LOAD_SUPER_ATTR_ATTR': 194, - 'LOAD_SUPER_ATTR_METHOD': 195, - 'RESUME_CHECK': 196, - 'SEND_GEN': 197, - 'STORE_ATTR_INSTANCE_VALUE': 198, - 'STORE_ATTR_SLOT': 199, - 'STORE_ATTR_WITH_HINT': 200, - 'STORE_SUBSCR_DICT': 201, - 'STORE_SUBSCR_LIST_INT': 202, - 'TO_BOOL_ALWAYS_TRUE': 203, - 'TO_BOOL_BOOL': 204, - 'TO_BOOL_INT': 205, - 'TO_BOOL_LIST': 206, - 'TO_BOOL_NONE': 207, - 'TO_BOOL_STR': 208, - 'UNPACK_SEQUENCE_LIST': 209, - 'UNPACK_SEQUENCE_TUPLE': 210, - 'UNPACK_SEQUENCE_TWO_TUPLE': 211, + 'LOAD_GLOBAL_BUILTIN': 190, + 'LOAD_GLOBAL_MODULE': 191, + 'LOAD_SUPER_ATTR_ATTR': 192, + 'LOAD_SUPER_ATTR_METHOD': 193, + 'RESUME_CHECK': 194, + 'SEND_GEN': 195, + 'STORE_ATTR_INSTANCE_VALUE': 196, + 'STORE_ATTR_SLOT': 197, + 'STORE_ATTR_WITH_HINT': 198, + 'STORE_SUBSCR_DICT': 199, + 'STORE_SUBSCR_LIST_INT': 200, + 'TO_BOOL_ALWAYS_TRUE': 201, + 'TO_BOOL_BOOL': 202, + 'TO_BOOL_INT': 203, + 'TO_BOOL_LIST': 204, + 'TO_BOOL_NONE': 205, + 'TO_BOOL_STR': 206, + 'UNPACK_SEQUENCE_LIST': 207, + 'UNPACK_SEQUENCE_TUPLE': 208, + 'UNPACK_SEQUENCE_TWO_TUPLE': 209, } opmap = { 'CACHE': 0, 'RESERVED': 17, 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, + 'INSTRUMENTED_LINE': 253, + 'ENTER_EXECUTOR': 254, + 'TRACE_RECORD': 255, 'BINARY_SLICE': 1, 'BUILD_TEMPLATE': 2, 'CALL_FUNCTION_EX': 4, @@ -334,26 +329,26 @@ opmap = { 'UNPACK_EX': 118, 'UNPACK_SEQUENCE': 119, 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, + 'INSTRUMENTED_END_FOR': 233, + 'INSTRUMENTED_POP_ITER': 234, + 'INSTRUMENTED_END_SEND': 235, + 'INSTRUMENTED_FOR_ITER': 236, + 'INSTRUMENTED_INSTRUCTION': 237, + 'INSTRUMENTED_JUMP_FORWARD': 238, + 'INSTRUMENTED_NOT_TAKEN': 239, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 240, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 241, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 242, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 243, + 'INSTRUMENTED_RESUME': 244, + 'INSTRUMENTED_RETURN_VALUE': 245, + 'INSTRUMENTED_YIELD_VALUE': 246, + 'INSTRUMENTED_END_ASYNC_FOR': 247, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 248, + 'INSTRUMENTED_CALL': 249, + 'INSTRUMENTED_CALL_KW': 250, + 'INSTRUMENTED_CALL_FUNCTION_EX': 251, + 'INSTRUMENTED_JUMP_BACKWARD': 252, 'ANNOTATIONS_PLACEHOLDER': 256, 'JUMP': 257, 'JUMP_IF_FALSE': 258, @@ -368,4 +363,4 @@ opmap = { } HAVE_ARGUMENT = 43 -MIN_INSTRUMENTED_OPCODE = 234 +MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index cbaa9445862..67c74fdd2d0 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -369,9 +369,15 @@ def _setoption(arg): if message or module: import re if message: - message = re.escape(message) + if len(message) >= 2 and message[0] == message[-1] == '/': + message = message[1:-1] + else: + message = re.escape(message) if module: - module = re.escape(module) + r'\z' + if len(module) >= 2 and module[0] == module[-1] == '/': + module = module[1:-1] + else: + module = re.escape(module) + r'\z' if lineno: try: lineno = int(lineno) @@ -381,7 +387,23 @@ def _setoption(arg): raise _wm._OptionError("invalid lineno %r" % (lineno,)) from None else: lineno = 0 - _wm.filterwarnings(action, message, category, module, lineno) + try: + _wm.filterwarnings(action, message, category, module, lineno) + except re.PatternError if message or module else (): + if message: + try: + re.compile(message) + except re.PatternError: + raise _wm._OptionError(f"invalid regular expression for " + f"message: {message!r}") from None + if module: + try: + re.compile(module) + except re.PatternError: + raise _wm._OptionError(f"invalid regular expression for " + f"module: {module!r}") from None + # Should never happen. + raise # Helper for _setoption() @@ -449,9 +471,12 @@ def warn(message, category=None, stacklevel=1, source=None, # Check category argument if category is None: category = UserWarning - if not (isinstance(category, type) and issubclass(category, Warning)): - raise TypeError("category must be a Warning subclass, " - "not '{:s}'".format(type(category).__name__)) + elif not isinstance(category, type): + raise TypeError(f"category must be a Warning subclass, not " + f"'{type(category).__name__}'") + elif not issubclass(category, Warning): + raise TypeError(f"category must be a Warning subclass, not " + f"class '{category.__name__}'") if not isinstance(skip_file_prefixes, tuple): # The C version demands a tuple for implementation performance. raise TypeError('skip_file_prefixes must be a tuple of strs.') @@ -495,20 +520,50 @@ def warn(message, category=None, stacklevel=1, source=None, ) +def _match_filename(pattern, filename, *, MS_WINDOWS=(sys.platform == 'win32')): + if not filename: + return pattern.match('') is not None + if filename[0] == '<' and filename[-1] == '>': + return pattern.match(filename) is not None + + is_py = (filename[-3:].lower() == '.py' + if MS_WINDOWS else + filename.endswith('.py')) + if is_py: + filename = filename[:-3] + if pattern.match(filename): # for backward compatibility + return True + if MS_WINDOWS: + if not is_py and filename[-4:].lower() == '.pyw': + filename = filename[:-4] + is_py = True + if is_py and filename[-9:].lower() in (r'\__init__', '/__init__'): + filename = filename[:-9] + filename = filename.replace('\\', '/') + else: + if is_py and filename.endswith('/__init__'): + filename = filename[:-9] + filename = filename.replace('/', '.') + i = 0 + while True: + if pattern.match(filename, i): + return True + i = filename.find('.', i) + 1 + if not i: + return False + + def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None): lineno = int(lineno) - if module is None: - module = filename or "" - if module[-3:].lower() == ".py": - module = module[:-3] # XXX What about leading pathname? if isinstance(message, Warning): text = str(message) category = message.__class__ else: text = message message = category(message) + modules = None key = (text, category, lineno) with _wm._lock: if registry is None: @@ -524,9 +579,11 @@ def warn_explicit(message, category, filename, lineno, action, msg, cat, mod, ln = item if ((msg is None or msg.match(text)) and issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break + (ln == 0 or lineno == ln) and + (mod is None or (_match_filename(mod, filename) + if module is None else + mod.match(module)))): + break else: action = _wm.defaultaction # Early exit actions @@ -589,6 +646,9 @@ def __str__(self): "line : %r}" % (self.message, self._category_name, self.filename, self.lineno, self.line)) + def __repr__(self): + return f'<{type(self).__qualname__} {self}>' + class catch_warnings(object): @@ -762,27 +822,27 @@ def __new__(cls, /, *args, **kwargs): arg.__new__ = staticmethod(__new__) - original_init_subclass = arg.__init_subclass__ - # We need slightly different behavior if __init_subclass__ - # is a bound method (likely if it was implemented in Python) - if isinstance(original_init_subclass, MethodType): - original_init_subclass = original_init_subclass.__func__ + if "__init_subclass__" in arg.__dict__: + # __init_subclass__ is directly present on the decorated class. + # Synthesize a wrapper that calls this method directly. + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python). + # Otherwise, it likely means it's a builtin such as + # object's implementation of __init_subclass__. + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ @functools.wraps(original_init_subclass) def __init_subclass__(*args, **kwargs): _wm.warn(msg, category=category, stacklevel=stacklevel + 1) return original_init_subclass(*args, **kwargs) - - arg.__init_subclass__ = classmethod(__init_subclass__) - # Or otherwise, which likely means it's a builtin such as - # object's implementation of __init_subclass__. else: - @functools.wraps(original_init_subclass) - def __init_subclass__(*args, **kwargs): + def __init_subclass__(cls, *args, **kwargs): _wm.warn(msg, category=category, stacklevel=stacklevel + 1) - return original_init_subclass(*args, **kwargs) + return super(arg, cls).__init_subclass__(*args, **kwargs) - arg.__init_subclass__ = __init_subclass__ + arg.__init_subclass__ = classmethod(__init_subclass__) arg.__deprecated__ = __new__.__deprecated__ = msg __init_subclass__.__deprecated__ = msg diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index e3db1b52b11..b6d68f23728 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -213,17 +213,6 @@ def _need_normalize_century(): _normalize_century = True return _normalize_century -_supports_c99 = None -def _can_support_c99(): - global _supports_c99 - if _supports_c99 is None: - try: - _supports_c99 = ( - _time.strftime("%F", (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == "1900-01-01") - except ValueError: - _supports_c99 = False - return _supports_c99 - # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -283,7 +272,7 @@ def _wrap_strftime(object, format, timetuple): newformat.append(Zreplace) # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so # year 1000 for %G can go on the fast path. - elif ((ch in 'YG' or ch in 'FC' and _can_support_c99()) and + elif ((ch in 'YG' or ch in 'FC') and object.year < 1000 and _need_normalize_century()): if ch == 'G': year = int(_time.strftime("%G", timetuple)) @@ -467,6 +456,7 @@ def _parse_isoformat_time(tstr): hour, minute, second, microsecond = time_comps became_next_day = False error_from_components = False + error_from_tz = None if (hour == 24): if all(time_comp == 0 for time_comp in time_comps[1:]): hour = 0 @@ -500,14 +490,22 @@ def _parse_isoformat_time(tstr): else: tzsign = -1 if tstr[tz_pos - 1] == '-' else 1 - td = timedelta(hours=tz_comps[0], minutes=tz_comps[1], - seconds=tz_comps[2], microseconds=tz_comps[3]) - - tzi = timezone(tzsign * td) + try: + # This function is intended to validate datetimes, but because + # we restrict time zones to ±24h, it serves here as well. + _check_time_fields(hour=tz_comps[0], minute=tz_comps[1], + second=tz_comps[2], microsecond=tz_comps[3], + fold=0) + except ValueError as e: + error_from_tz = e + else: + td = timedelta(hours=tz_comps[0], minutes=tz_comps[1], + seconds=tz_comps[2], microseconds=tz_comps[3]) + tzi = timezone(tzsign * td) time_comps.append(tzi) - return time_comps, became_next_day, error_from_components + return time_comps, became_next_day, error_from_components, error_from_tz # tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar def _isoweek_to_gregorian(year, week, day): @@ -1074,7 +1072,11 @@ def fromisocalendar(cls, year, week, day): @classmethod def strptime(cls, date_string, format): - """Parse a date string according to the given format (like time.strptime()).""" + """Parse string according to the given date format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_date(cls, date_string, format) @@ -1111,6 +1113,8 @@ def strftime(self, format): Format using strftime(). Example: "%d/%m/%Y, %H:%M:%S" + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes """ return _wrap_strftime(self, format, self.timetuple()) @@ -1127,8 +1131,8 @@ def isoformat(self): This is 'YYYY-MM-DD'. References: - - http://www.w3.org/TR/NOTE-datetime - - http://www.cl.cam.ac.uk/~mgk25/iso-time.html + - https://www.w3.org/TR/NOTE-datetime + - https://www.cl.cam.ac.uk/~mgk25/iso-time.html """ return "%04d-%02d-%02d" % (self._year, self._month, self._day) @@ -1262,7 +1266,7 @@ def isocalendar(self): The first week is 1; Monday is 1 ... Sunday is 7. ISO calendar algorithm taken from - http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm + https://www.phys.uu.nl/~vgent/calendar/isocalendar.htm (used with permission) """ year = self._year @@ -1301,7 +1305,7 @@ def __reduce__(self): class tzinfo: - """Abstract base class for time zone info classes. + """Abstract base class for time zone info objects. Subclasses must override the tzname(), utcoffset() and dst() methods. """ @@ -1458,8 +1462,13 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold return self @classmethod + def strptime(cls, date_string, format): - """string, format -> new time parsed from a string (like time.strptime()).""" + """Parse string according to the given time format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_time(cls, date_string, format) @@ -1633,13 +1642,28 @@ def fromisoformat(cls, time_string): time_string = time_string.removeprefix('T') try: - return cls(*_parse_isoformat_time(time_string)[0]) - except Exception: - raise ValueError(f'Invalid isoformat string: {time_string!r}') + time_components, _, error_from_components, error_from_tz = ( + _parse_isoformat_time(time_string) + ) + except ValueError: + raise ValueError( + f'Invalid isoformat string: {time_string!r}') from None + else: + if error_from_tz: + raise error_from_tz + if error_from_components: + raise ValueError( + "Minute, second, and microsecond must be 0 when hour is 24" + ) + + return cls(*time_components) def strftime(self, format): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes """ # The year must be >= 1000 else Python's strftime implementation # can raise a bogus exception. @@ -1755,7 +1779,7 @@ def __reduce__(self): class datetime(date): - """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) + """A combination of a date and a time. The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. @@ -1947,11 +1971,16 @@ def fromisoformat(cls, date_string): if tstr: try: - time_components, became_next_day, error_from_components = _parse_isoformat_time(tstr) + (time_components, + became_next_day, + error_from_components, + error_from_tz) = _parse_isoformat_time(tstr) except ValueError: raise ValueError( f'Invalid isoformat string: {date_string!r}') from None else: + if error_from_tz: + raise error_from_tz if error_from_components: raise ValueError("minute, second, and microsecond must be 0 when hour is 24") @@ -2089,7 +2118,6 @@ def _local_timezone(self): else: ts = (self - _EPOCH) // timedelta(seconds=1) localtm = _time.localtime(ts) - local = datetime(*localtm[:6]) # Extract TZ data gmtoff = localtm.tm_gmtoff zone = localtm.tm_zone @@ -2139,7 +2167,7 @@ def isoformat(self, sep='T', timespec='auto'): By default, the fractional part is omitted if self.microsecond == 0. If self.tzinfo is not None, the UTC offset is also attached, giving - giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. + a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. Optional argument sep specifies the separator between date and time, default 'T'. @@ -2184,7 +2212,11 @@ def __str__(self): @classmethod def strptime(cls, date_string, format): - 'string, format -> new datetime parsed from a string (like time.strptime()).' + """Parse string according to the given time format (like time.strptime()). + + For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes + """ import _strptime return _strptime._strptime_datetime_datetime(cls, date_string, format) @@ -2410,6 +2442,8 @@ def _isoweek1monday(year): class timezone(tzinfo): + """Fixed offset from UTC implementation of tzinfo.""" + __slots__ = '_offset', '_name' # Sentinel value to disallow None diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 46fa9ffcb1e..ef889ea0cc8 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -47,13 +47,16 @@ 'HAVE_THREADS', # C version: compile time choice that enables the coroutine local context - 'HAVE_CONTEXTVAR' + 'HAVE_CONTEXTVAR', + + # Highest version of the spec this module complies with + 'SPEC_VERSION', ] __xname__ = __name__ # sys.modules lookup (--without-threads) __name__ = 'decimal' # For pickling -__version__ = '1.70' # Highest version of the spec this complies with - # See http://speleotrove.com/decimal/ +SPEC_VERSION = '1.70' # Highest version of the spec this complies with + # See https://speleotrove.com/decimal/decarith.html __libmpdec_version__ = "2.4.2" # compatible libmpdec version import math as _math @@ -3340,7 +3343,10 @@ def _fill_logical(self, context, opa, opb): return opa, opb def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" + """Applies an 'and' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3357,14 +3363,20 @@ def logical_and(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_invert(self, context=None): - """Invert all its digits.""" + """Invert all its digits. + + The self must be logical number. + """ if context is None: context = getcontext() return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), context) def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" + """Applies an 'or' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3381,7 +3393,10 @@ def logical_or(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" + """Applies an 'xor' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -6120,9 +6135,13 @@ def _convert_for_comparison(self, other, equality_op=False): (?Pz)? (?P\#)? (?P0)? -(?P(?!0)\d+)? +(?P\d+)? (?P[,_])? -(?:\.(?P0|(?!0)\d+))? +(?:\. + (?=[\d,_]) # lookahead for digit or separator + (?P\d+)? + (?P[,_])? +)? (?P[eEfFgGn%])? \z """, re.VERBOSE|re.DOTALL) @@ -6215,6 +6234,9 @@ def _parse_format_specifier(format_spec, _localeconv=None): format_dict['grouping'] = [3, 0] format_dict['decimal_point'] = '.' + if format_dict['frac_separators'] is None: + format_dict['frac_separators'] = '' + return format_dict def _format_align(sign, body, spec): @@ -6334,6 +6356,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): sign = _format_sign(is_negative, spec) + frac_sep = spec['frac_separators'] + if fracpart and frac_sep: + fracpart = frac_sep.join(fracpart[pos:pos + 3] + for pos in range(0, len(fracpart), 3)) + if fracpart or spec['alt']: fracpart = spec['decimal_point'] + fracpart @@ -6375,3 +6402,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): # _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS _PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) del sys + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return SPEC_VERSION + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/_pyio.py b/Lib/_pyio.py index a870de5b532..69a088df8fc 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -407,6 +407,9 @@ def __del__(self): if closed: return + if dealloc_warn := getattr(self, "_dealloc_warn", None): + dealloc_warn(self) + # If close() fails, the caller logs the exception with # sys.unraisablehook. close() must be called at the end at __del__(). self.close() @@ -543,7 +546,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -614,8 +617,10 @@ def read(self, size=-1): n = self.readinto(b) if n is None: return None + if n < 0 or n > len(b): + raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -623,7 +628,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -645,8 +650,6 @@ def write(self, b): self._unsupported("write") io.RawIOBase.register(RawIOBase) -from _io import FileIO -RawIOBase.register(FileIO) class BufferedIOBase(IOBase): @@ -853,6 +856,10 @@ def __repr__(self): else: return "<{}.{} name={!r}>".format(modname, clsname, name) + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.raw, "_dealloc_warn", None): + dealloc_warn(source) + ### Lower-level APIs ### def fileno(self): @@ -871,16 +878,28 @@ class BytesIO(BufferedIOBase): _buffer = None def __init__(self, initial_bytes=None): + # Use to keep self._buffer and self._pos consistent. + self._lock = Lock() + buf = bytearray() if initial_bytes is not None: buf += initial_bytes - self._buffer = buf - self._pos = 0 + + with self._lock: + self._buffer = buf + self._pos = 0 def __getstate__(self): if self.closed: raise ValueError("__getstate__ on closed file") - return self.__dict__.copy() + with self._lock: + state = self.__dict__.copy() + del state['_lock'] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self._lock = Lock() def getvalue(self): """Return the bytes value (contents) of the buffer @@ -913,14 +932,16 @@ def read(self, size=-1): raise TypeError(f"{size!r} is not an integer") else: size = size_index() - if size < 0: - size = len(self._buffer) - if len(self._buffer) <= self._pos: - return b"" - newpos = min(len(self._buffer), self._pos + size) - b = self._buffer[self._pos : newpos] - self._pos = newpos - return bytes(b) + + with self._lock: + if size < 0: + size = len(self._buffer) + if len(self._buffer) <= self._pos: + return b"" + newpos = min(len(self._buffer), self._pos + size) + b = self._buffer[self._pos : newpos] + self._pos = newpos + return bytes(b) def read1(self, size=-1): """This is the same as read. @@ -936,12 +957,14 @@ def write(self, b): n = view.nbytes # Size of any bytes-like object if n == 0: return 0 - pos = self._pos - if pos > len(self._buffer): - # Pad buffer to pos with null bytes. - self._buffer.resize(pos) - self._buffer[pos:pos + n] = b - self._pos += n + + with self._lock: + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = b + self._pos += n return n def seek(self, pos, whence=0): @@ -958,9 +981,11 @@ def seek(self, pos, whence=0): raise ValueError("negative seek position %r" % (pos,)) self._pos = pos elif whence == 1: - self._pos = max(0, self._pos + pos) + with self._lock: + self._pos = max(0, self._pos + pos) elif whence == 2: - self._pos = max(0, len(self._buffer) + pos) + with self._lock: + self._pos = max(0, len(self._buffer) + pos) else: raise ValueError("unsupported whence value") return self._pos @@ -973,18 +998,20 @@ def tell(self): def truncate(self, pos=None): if self.closed: raise ValueError("truncate on closed file") - if pos is None: - pos = self._pos - else: - try: - pos_index = pos.__index__ - except AttributeError: - raise TypeError(f"{pos!r} is not an integer") + + with self._lock: + if pos is None: + pos = self._pos else: - pos = pos_index() - if pos < 0: - raise ValueError("negative truncate position %r" % (pos,)) - del self._buffer[pos:] + try: + pos_index = pos.__index__ + except AttributeError: + raise TypeError(f"{pos!r} is not an integer") + else: + pos = pos_index() + if pos < 0: + raise ValueError("negative truncate position %r" % (pos,)) + del self._buffer[pos:] return pos def readable(self): @@ -1473,6 +1500,7 @@ class FileIO(RawIOBase): _writable = False _appending = False _seekable = None + _truncate = False _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): @@ -1528,6 +1556,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): flags = 0 elif 'w' in mode: self._writable = True + self._truncate = True flags = os.O_CREAT | os.O_TRUNC elif 'a' in mode: self._writable = True @@ -1563,7 +1592,8 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if not isinstance(fd, int): raise TypeError('expected integer from opener') if fd < 0: - raise OSError('Negative file descriptor') + # bpo-27066: Raise a ValueError for bad value. + raise ValueError(f'opener returned {fd}') owned_fd = fd if not noinherit_flag: os.set_inheritable(fd, False) @@ -1600,12 +1630,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): raise self._fd = fd - def __del__(self): + def _dealloc_warn(self, source): if self._fd >= 0 and self._closefd and not self.closed: import warnings - warnings.warn('unclosed file %r' % (self,), ResourceWarning, + warnings.warn(f'unclosed file {source!r}', ResourceWarning, stacklevel=2, source=self) - self.close() def __getstate__(self): raise TypeError(f"cannot pickle {self.__class__.__name__!r} object") @@ -1709,7 +1738,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" @@ -1780,7 +1809,7 @@ def close(self): if not self.closed: self._stat_atopen = None try: - if self._closefd: + if self._closefd and self._fd >= 0: os.close(self._fd) finally: super().close() @@ -1852,7 +1881,10 @@ def mode(self): return 'ab' elif self._readable: if self._writable: - return 'rb+' + if self._truncate: + return 'wb+' + else: + return 'rb+' else: return 'rb' else: @@ -2689,6 +2721,10 @@ def readline(self, size=None): def newlines(self): return self._decoder.newlines if self._decoder else None + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.buffer, "_dealloc_warn", None): + dealloc_warn(source) + class StringIO(TextIOWrapper): """Text I/O implementation using an in-memory buffer. diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py deleted file mode 100644 index d884f880f50..00000000000 --- a/Lib/_pyrepl/_minimal_curses.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Minimal '_curses' module, the low-level interface for curses module -which is not meant to be used directly. - -Based on ctypes. It's too incomplete to be really called '_curses', so -to use it, you have to import it and stick it in sys.modules['_curses'] -manually. - -Note that there is also a built-in module _minimal_curses which will -hide this one if compiled in. -""" - -import ctypes -import ctypes.util - - -class error(Exception): - pass - - -def _find_clib() -> str: - trylibs = ["ncursesw", "ncurses", "curses"] - - for lib in trylibs: - path = ctypes.util.find_library(lib) - if path: - return path - raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") - - -_clibpath = _find_clib() -clib = ctypes.cdll.LoadLibrary(_clibpath) - -clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] -clib.setupterm.restype = ctypes.c_int - -clib.tigetstr.argtypes = [ctypes.c_char_p] -clib.tigetstr.restype = ctypes.c_ssize_t - -clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] -clib.tparm.restype = ctypes.c_char_p - -OK = 0 -ERR = -1 - -# ____________________________________________________________ - - -def setupterm(termstr, fd): - err = ctypes.c_int(0) - result = clib.setupterm(termstr, fd, ctypes.byref(err)) - if result == ERR: - raise error("setupterm() failed (err=%d)" % err.value) - - -def tigetstr(cap): - if not isinstance(cap, bytes): - cap = cap.encode("ascii") - result = clib.tigetstr(cap) - if result == ERR: - return None - return ctypes.cast(result, ctypes.c_char_p).value - - -def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): - result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) - if result is None: - raise error("tparm() returned NULL") - return result diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 347f05607c7..cf59e007f4d 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -1,9 +1,12 @@ from __future__ import annotations +import importlib +import os import pkgutil import sys import token import tokenize +from importlib.machinery import FileFinder from io import StringIO from contextlib import contextmanager from dataclasses import dataclass @@ -16,9 +19,18 @@ from typing import Any, Iterable, Iterator, Mapping +HARDCODED_SUBMODULES = { + # Standard library submodules that are not detected by pkgutil.iter_modules + # but can be imported, so should be proposed in completion + "collections": ["abc"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], +} + + def make_default_module_completer() -> ModuleCompleter: - # Inside pyrepl, __package__ is set to '_pyrepl' - return ModuleCompleter(namespace={'__package__': '_pyrepl'}) + # Inside pyrepl, __package__ is set to None by default + return ModuleCompleter(namespace={'__package__': None}) class ModuleCompleter: @@ -41,12 +53,13 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self.namespace = namespace or {} self._global_cache: list[pkgutil.ModuleInfo] = [] self._curr_sys_path: list[str] = sys.path[:] + self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> list[str]: + def get_completions(self, line: str) -> list[str] | None: """Return the next possible import completions for 'line'.""" result = ImportParser(line).parse() if not result: - return [] + return None try: return self.complete(*result) except Exception: @@ -81,8 +94,11 @@ def find_modules(self, path: str, prefix: str) -> list[str]: def _find_modules(self, path: str, prefix: str) -> list[str]: if not path: # Top-level import (e.g. `import foo`` or `from foo`)` - return [name for _, name, _ in self.global_cache - if name.startswith(prefix)] + builtin_modules = [name for name in sys.builtin_module_names + if self.is_suggestion_match(name, prefix)] + third_party_modules = [module.name for module in self.global_cache + if self.is_suggestion_match(module.name, prefix)] + return sorted(builtin_modules + third_party_modules) if path.startswith('.'): # Convert relative path to absolute path @@ -92,12 +108,33 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: return [] modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + is_stdlib_import: bool | None = None for segment in path.split('.'): modules = [mod_info for mod_info in modules if mod_info.ispkg and mod_info.name == segment] + if is_stdlib_import is None: + # Top-level import decide if we import from stdlib or not + is_stdlib_import = all( + self._is_stdlib_module(mod_info) for mod_info in modules + ) modules = self.iter_submodules(modules) - return [module.name for module in modules - if module.name.startswith(prefix)] + + module_names = [module.name for module in modules] + if is_stdlib_import: + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) + return [module_name for module_name in module_names + if self.is_suggestion_match(module_name, prefix)] + + def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: + return (isinstance(module_info.module_finder, FileFinder) + and module_info.module_finder.path == self._stdlib_path) + + def is_suggestion_match(self, module_name: str, prefix: str) -> bool: + if prefix: + return module_name.startswith(prefix) + # For consistency with attribute completion, which + # does not suggest private attributes unless requested. + return not module_name.startswith("_") def iter_submodules(self, parent_modules: list[pkgutil.ModuleInfo]) -> Iterator[pkgutil.ModuleInfo]: """Iterate over all submodules of the given parent modules.""" diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index 842599bd187..0589a0f437e 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -87,7 +87,7 @@ def push(self, char: int | bytes) -> None: if isinstance(k, dict): self.keymap = k else: - self.insert(Event('key', k, self.flush_buf())) + self.insert(Event('key', k, bytes(self.flush_buf()))) self.keymap = self.compiled_keymap elif self.buf and self.buf[0] == 27: # escape @@ -96,7 +96,7 @@ def push(self, char: int | bytes) -> None: # the docstring in keymap.py trace('unrecognized escape sequence, propagating...') self.keymap = self.compiled_keymap - self.insert(Event('key', '\033', bytearray(b'\033'))) + self.insert(Event('key', '\033', b'\033')) for _c in self.flush_buf()[1:]: self.push(_c) @@ -106,5 +106,5 @@ def push(self, char: int | bytes) -> None: except UnicodeError: return else: - self.insert(Event('key', decoded, self.flush_buf())) + self.insert(Event('key', decoded, bytes(self.flush_buf()))) self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 2354fbb2ec2..10127e58897 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -370,6 +370,13 @@ def do(self) -> None: r = self.reader text = self.event * r.get_arg() r.insert(text) + if r.paste_mode: + data = "" + ev = r.console.getpending() + data += ev.data + if data: + r.insert(data) + r.last_refresh_cache.invalidated = True class insert_nl(EditCommand): @@ -413,14 +420,17 @@ class delete(EditCommand): def do(self) -> None: r = self.reader b = r.buffer - if ( - r.pos == 0 - and len(b) == 0 # this is something of a hack - and self.event[-1] == "\004" - ): - r.update_screen() - r.console.finish() - raise EOFError + if self.event[-1] == "\004": + if b and b[-1].endswith("\n"): + self.finish = True + elif ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] @@ -484,7 +494,6 @@ def do(self) -> None: data = "" start = time.time() while done not in data: - self.reader.console.wait(100) ev = self.reader.console.getpending() data += ev.data trace( diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 8956fb1242e..e0535d50396 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -27,6 +27,7 @@ import linecache from dataclasses import dataclass, field import os.path +import re import sys @@ -195,7 +196,19 @@ def runsource(self, source, filename="", symbol="single"): ast.PyCF_ONLY_AST, incomplete_input=False, ) - except (SyntaxError, OverflowError, ValueError): + except SyntaxError as e: + # If it looks like pip install was entered (a common beginner + # mistake), provide a hint to use the system command prompt. + if re.match(r"^\s*(pip3?|py(thon3?)? -m pip) install.*", source): + e.add_note( + "The Python package manager (pip) can only be used" + " outside of the Python REPL.\n" + "Try the 'pip' command in a separate terminal or" + " command prompt." + ) + self.showsyntaxerror(filename, source=source) + return False + except (OverflowError, ValueError): self.showsyntaxerror(filename, source=source) return False if tree.body: diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py deleted file mode 100644 index 3a624d9f683..00000000000 --- a/Lib/_pyrepl/curses.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2000-2010 Michael Hudson-Doyle -# Armin Rigo -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -try: - import _curses -except ImportError: - try: - import curses as _curses # type: ignore[no-redef] - except ImportError: - from . import _minimal_curses as _curses # type: ignore[no-redef] - -setupterm = _curses.setupterm -tigetstr = _curses.tigetstr -tparm = _curses.tparm -error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 0468b9a2670..8d5bd183f21 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -20,19 +20,25 @@ import termios -class TermState: - def __init__(self, tuples): - ( - self.iflag, - self.oflag, - self.cflag, - self.lflag, - self.ispeed, - self.ospeed, - self.cc, - ) = tuples +TYPE_CHECKING = False - def as_list(self): +if TYPE_CHECKING: + from typing import cast +else: + cast = lambda typ, val: val + + +class TermState: + def __init__(self, attrs: list[int | list[bytes]]) -> None: + self.iflag = cast(int, attrs[0]) + self.oflag = cast(int, attrs[1]) + self.cflag = cast(int, attrs[2]) + self.lflag = cast(int, attrs[3]) + self.ispeed = cast(int, attrs[4]) + self.ospeed = cast(int, attrs[5]) + self.cc = cast(list[bytes], attrs[6]) + + def as_list(self) -> list[int | list[bytes]]: return [ self.iflag, self.oflag, @@ -45,32 +51,32 @@ def as_list(self): self.cc[:], ] - def copy(self): + def copy(self) -> "TermState": return self.__class__(self.as_list()) -def tcgetattr(fd): +def tcgetattr(fd: int) -> TermState: return TermState(termios.tcgetattr(fd)) -def tcsetattr(fd, when, attrs): +def tcsetattr(fd: int, when: int, attrs: TermState) -> None: termios.tcsetattr(fd, when, attrs.as_list()) class Term(TermState): TS__init__ = TermState.__init__ - def __init__(self, fd=0): + def __init__(self, fd: int = 0) -> None: self.TS__init__(termios.tcgetattr(fd)) self.fd = fd - self.stack = [] + self.stack: list[list[int | list[bytes]]] = [] - def save(self): + def save(self) -> None: self.stack.append(self.as_list()) - def set(self, when=termios.TCSANOW): + def set(self, when: int = termios.TCSANOW) -> None: termios.tcsetattr(self.fd, when, self.as_list()) - def restore(self): + def restore(self) -> None: self.TS__init__(self.stack.pop()) self.set() diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py index a6f824dcc4a..447eb1e551e 100644 --- a/Lib/_pyrepl/main.py +++ b/Lib/_pyrepl/main.py @@ -1,6 +1,7 @@ import errno import os import sys +import types CAN_USE_PYREPL: bool @@ -29,12 +30,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): print(FAIL_REASON, file=sys.stderr) return sys._baserepl() - if mainmodule: - namespace = mainmodule.__dict__ - else: - import __main__ - namespace = __main__.__dict__ - namespace.pop("__pyrepl_interactive_console", None) + if not mainmodule: + mainmodule = types.ModuleType("__main__") + + namespace = mainmodule.__dict__ # sys._baserepl() above does this internally, we do it here startup_path = os.getenv("PYTHONSTARTUP") diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 560a9db1921..23b8fa6b9c7 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -43,10 +43,11 @@ Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] -try: - from .unix_console import UnixConsole as Console, _error -except ImportError: + +if os.name == "nt": from .windows_console import WindowsConsole as Console, _error +else: + from .unix_console import UnixConsole as Console, _error ENCODING = sys.getdefaultencoding() or "latin1" @@ -134,7 +135,8 @@ def get_stem(self) -> str: return "".join(b[p + 1 : self.pos]) def get_completions(self, stem: str) -> list[str]: - if module_completions := self.get_module_completions(): + module_completions = self.get_module_completions() + if module_completions is not None: return module_completions if len(stem) == 0 and self.more_lines is not None: b = self.buffer @@ -165,7 +167,7 @@ def get_completions(self, stem: str) -> list[str]: result.sort() return result - def get_module_completions(self) -> list[str]: + def get_module_completions(self) -> list[str] | None: line = self.get_line() return self.config.module_completer.get_completions(line) @@ -606,6 +608,7 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) + _wrapper.config.module_completer = ModuleCompleter(namespace) _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index b3848833e14..3b0debf2ba0 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -110,6 +110,10 @@ def run_multiline_interactive_console( more_lines = functools.partial(_more_lines, console) input_n = 0 + _is_x_showrefcount_set = sys._xoptions.get("showrefcount") + _is_pydebug_build = hasattr(sys, "gettotalrefcount") + show_ref_count = _is_x_showrefcount_set and _is_pydebug_build + def maybe_run_command(statement: str) -> bool: statement = statement.strip() if statement in console.locals or statement not in REPL_COMMANDS: @@ -149,9 +153,11 @@ def maybe_run_command(statement: str) -> bool: append_history_file() except (FileNotFoundError, PermissionError, OSError) as e: warnings.warn(f"failed to open the history file for writing: {e}") + input_n += 1 except KeyboardInterrupt: r = _get_reader() + r.cmpltn_reset() if r.input_trans is r.isearch_trans: r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) @@ -167,3 +173,8 @@ def maybe_run_command(statement: str) -> bool: except: console.showtraceback() console.resetbuffer() + if show_ref_count: + console.write( + f"[{sys.gettotalrefcount()} refs," + f" {sys.getallocatedblocks()} blocks]\n" + ) diff --git a/Lib/_pyrepl/terminfo.py b/Lib/_pyrepl/terminfo.py new file mode 100644 index 00000000000..d02ef69cce0 --- /dev/null +++ b/Lib/_pyrepl/terminfo.py @@ -0,0 +1,488 @@ +"""Pure Python curses-like terminal capability queries.""" + +from dataclasses import dataclass, field +import errno +import os +from pathlib import Path +import re +import struct + + +# Terminfo constants +MAGIC16 = 0o432 # Magic number for 16-bit terminfo format +MAGIC32 = 0o1036 # Magic number for 32-bit terminfo format + +# Special values for absent/cancelled capabilities +ABSENT_BOOLEAN = -1 +ABSENT_NUMERIC = -1 +CANCELLED_NUMERIC = -2 +ABSENT_STRING = None +CANCELLED_STRING = None + + +# Standard string capability names from ncurses Caps file +# This matches the order used by ncurses when compiling terminfo +# fmt: off +_STRING_NAMES: tuple[str, ...] = ( + "cbt", "bel", "cr", "csr", "tbc", "clear", "el", "ed", "hpa", "cmdch", + "cup", "cud1", "home", "civis", "cub1", "mrcup", "cnorm", "cuf1", "ll", + "cuu1", "cvvis", "dch1", "dl1", "dsl", "hd", "smacs", "blink", "bold", + "smcup", "smdc", "dim", "smir", "invis", "prot", "rev", "smso", "smul", + "ech", "rmacs", "sgr0", "rmcup", "rmdc", "rmir", "rmso", "rmul", "flash", + "ff", "fsl", "is1", "is2", "is3", "if", "ich1", "il1", "ip", "kbs", "ktbc", + "kclr", "kctab", "kdch1", "kdl1", "kcud1", "krmir", "kel", "ked", "kf0", + "kf1", "kf10", "kf2", "kf3", "kf4", "kf5", "kf6", "kf7", "kf8", "kf9", + "khome", "kich1", "kil1", "kcub1", "kll", "knp", "kpp", "kcuf1", "kind", + "kri", "khts", "kcuu1", "rmkx", "smkx", "lf0", "lf1", "lf10", "lf2", "lf3", + "lf4", "lf5", "lf6", "lf7", "lf8", "lf9", "rmm", "smm", "nel", "pad", "dch", + "dl", "cud", "ich", "indn", "il", "cub", "cuf", "rin", "cuu", "pfkey", + "pfloc", "pfx", "mc0", "mc4", "mc5", "rep", "rs1", "rs2", "rs3", "rf", "rc", + "vpa", "sc", "ind", "ri", "sgr", "hts", "wind", "ht", "tsl", "uc", "hu", + "iprog", "ka1", "ka3", "kb2", "kc1", "kc3", "mc5p", "rmp", "acsc", "pln", + "kcbt", "smxon", "rmxon", "smam", "rmam", "xonc", "xoffc", "enacs", "smln", + "rmln", "kbeg", "kcan", "kclo", "kcmd", "kcpy", "kcrt", "kend", "kent", + "kext", "kfnd", "khlp", "kmrk", "kmsg", "kmov", "knxt", "kopn", "kopt", + "kprv", "kprt", "krdo", "kref", "krfr", "krpl", "krst", "kres", "ksav", + "kspd", "kund", "kBEG", "kCAN", "kCMD", "kCPY", "kCRT", "kDC", "kDL", + "kslt", "kEND", "kEOL", "kEXT", "kFND", "kHLP", "kHOM", "kIC", "kLFT", + "kMSG", "kMOV", "kNXT", "kOPT", "kPRV", "kPRT", "kRDO", "kRPL", "kRIT", + "kRES", "kSAV", "kSPD", "kUND", "rfi", "kf11", "kf12", "kf13", "kf14", + "kf15", "kf16", "kf17", "kf18", "kf19", "kf20", "kf21", "kf22", "kf23", + "kf24", "kf25", "kf26", "kf27", "kf28", "kf29", "kf30", "kf31", "kf32", + "kf33", "kf34", "kf35", "kf36", "kf37", "kf38", "kf39", "kf40", "kf41", + "kf42", "kf43", "kf44", "kf45", "kf46", "kf47", "kf48", "kf49", "kf50", + "kf51", "kf52", "kf53", "kf54", "kf55", "kf56", "kf57", "kf58", "kf59", + "kf60", "kf61", "kf62", "kf63", "el1", "mgc", "smgl", "smgr", "fln", "sclk", + "dclk", "rmclk", "cwin", "wingo", "hup","dial", "qdial", "tone", "pulse", + "hook", "pause", "wait", "u0", "u1", "u2", "u3", "u4", "u5", "u6", "u7", + "u8", "u9", "op", "oc", "initc", "initp", "scp", "setf", "setb", "cpi", + "lpi", "chr", "cvr", "defc", "swidm", "sdrfq", "sitm", "slm", "smicm", + "snlq", "snrmq", "sshm", "ssubm", "ssupm", "sum", "rwidm", "ritm", "rlm", + "rmicm", "rshm", "rsubm", "rsupm", "rum", "mhpa", "mcud1", "mcub1", "mcuf1", + "mvpa", "mcuu1", "porder", "mcud", "mcub", "mcuf", "mcuu", "scs", "smgb", + "smgbp", "smglp", "smgrp", "smgt", "smgtp", "sbim", "scsd", "rbim", "rcsd", + "subcs", "supcs", "docr", "zerom", "csnm", "kmous", "minfo", "reqmp", + "getm", "setaf", "setab", "pfxl", "devt", "csin", "s0ds", "s1ds", "s2ds", + "s3ds", "smglr", "smgtb", "birep", "binel", "bicr", "colornm", "defbi", + "endbi", "setcolor", "slines", "dispc", "smpch", "rmpch", "smsc", "rmsc", + "pctrm", "scesc", "scesa", "ehhlm", "elhlm", "elohlm", "erhlm", "ethlm", + "evhlm", "sgr1", "slength", "OTi2", "OTrs", "OTnl", "OTbc", "OTko", "OTma", + "OTG2", "OTG3", "OTG1", "OTG4", "OTGR", "OTGL", "OTGU", "OTGD", "OTGH", + "OTGV", "OTGC","meml", "memu", "box1" +) +# fmt: on + + +def _get_terminfo_dirs() -> list[Path]: + """Get list of directories to search for terminfo files. + + Based on ncurses behavior in: + - ncurses/tinfo/db_iterator.c:_nc_next_db() + - ncurses/tinfo/read_entry.c:_nc_read_entry() + """ + dirs = [] + + terminfo = os.environ.get("TERMINFO") + if terminfo: + dirs.append(terminfo) + + try: + home = Path.home() + dirs.append(str(home / ".terminfo")) + except RuntimeError: + pass + + # Check TERMINFO_DIRS + terminfo_dirs = os.environ.get("TERMINFO_DIRS", "") + if terminfo_dirs: + for d in terminfo_dirs.split(":"): + if d: + dirs.append(d) + + dirs.extend( + [ + "/etc/terminfo", + "/lib/terminfo", + "/usr/lib/terminfo", + "/usr/share/terminfo", + "/usr/share/lib/terminfo", + "/usr/share/misc/terminfo", + "/usr/local/lib/terminfo", + "/usr/local/share/terminfo", + ] + ) + + return [Path(d) for d in dirs if Path(d).is_dir()] + + +def _validate_terminal_name_or_raise(terminal_name: str) -> None: + if not isinstance(terminal_name, str): + raise TypeError("`terminal_name` must be a string") + + if not terminal_name: + raise ValueError("`terminal_name` cannot be empty") + + if "\x00" in terminal_name: + raise ValueError("NUL character found in `terminal_name`") + + t = Path(terminal_name) + if len(t.parts) > 1: + raise ValueError("`terminal_name` cannot contain path separators") + + +def _read_terminfo_file(terminal_name: str) -> bytes: + """Find and read terminfo file for given terminal name. + + Terminfo files are stored in directories using the first character + of the terminal name as a subdirectory. + """ + _validate_terminal_name_or_raise(terminal_name) + first_char = terminal_name[0].lower() + filename = terminal_name + + for directory in _get_terminfo_dirs(): + path = directory / first_char / filename + if path.is_file(): + return path.read_bytes() + + # Try with hex encoding of first char (for special chars) + hex_dir = "%02x" % ord(first_char) + path = directory / hex_dir / filename + if path.is_file(): + return path.read_bytes() + + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename) + + +# Hard-coded terminal capabilities for common terminals +# This is a minimal subset needed by PyREPL +_TERMINAL_CAPABILITIES = { + # ANSI/xterm-compatible terminals + "ansi": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column + "cud1": b"\n", # Move cursor down 1 row + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[2J", # Clear screen and home cursor + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l", # Make cursor invisible + "cnorm": b"\x1b[?12l\x1b[?25h", # Make cursor normal (visible) + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Padding (not used in modern terminals) + "pad": b"", + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1bOB", # Down arrow + "kend": b"\x1bOF", # End key + "kent": b"\x1bOM", # Enter key + "khome": b"\x1bOH", # Home key + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1bOD", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1bOC", # Right arrow + "kcuu1": b"\x1bOA", # Up arrow + # Function keys F1-F20 + "kf1": b"\x1bOP", + "kf2": b"\x1bOQ", + "kf3": b"\x1bOR", + "kf4": b"\x1bOS", + "kf5": b"\x1b[15~", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[1;2P", + "kf14": b"\x1b[1;2Q", + "kf15": b"\x1b[1;2R", + "kf16": b"\x1b[1;2S", + "kf17": b"\x1b[15;2~", + "kf18": b"\x1b[17;2~", + "kf19": b"\x1b[18;2~", + "kf20": b"\x1b[19;2~", + }, + # Dumb terminal - minimal capabilities + "dumb": { + "bel": b"\x07", # Bell + "cud1": b"\n", # Move down 1 row (newline) + "ind": b"\n", # Scroll up one line (newline) + }, + # Linux console + "linux": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column (backspace) + "cud1": b"\n", # Move cursor down 1 row (newline) + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[J", # Clear screen and home cursor (different from ansi!) + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"\x1b[@", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l\x1b[?1c", # Make cursor invisible + "cnorm": b"\x1b[?25h\x1b[?0c", # Make cursor normal + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1b[B", # Down arrow + "kend": b"\x1b[4~", # End key (different from ansi!) + "khome": b"\x1b[1~", # Home key (different from ansi!) + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1b[D", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1b[C", # Right arrow + "kcuu1": b"\x1b[A", # Up arrow + # Function keys + "kf1": b"\x1b[[A", + "kf2": b"\x1b[[B", + "kf3": b"\x1b[[C", + "kf4": b"\x1b[[D", + "kf5": b"\x1b[[E", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[25~", + "kf14": b"\x1b[26~", + "kf15": b"\x1b[28~", + "kf16": b"\x1b[29~", + "kf17": b"\x1b[31~", + "kf18": b"\x1b[32~", + "kf19": b"\x1b[33~", + "kf20": b"\x1b[34~", + }, +} + +# Map common TERM values to capability sets +_TERM_ALIASES = { + "xterm": "ansi", + "xterm-color": "ansi", + "xterm-256color": "ansi", + "screen": "ansi", + "screen-256color": "ansi", + "tmux": "ansi", + "tmux-256color": "ansi", + "vt100": "ansi", + "vt220": "ansi", + "rxvt": "ansi", + "rxvt-unicode": "ansi", + "rxvt-unicode-256color": "ansi", + "unknown": "dumb", +} + + +@dataclass +class TermInfo: + terminal_name: str | bytes | None + fallback: bool = True + + _capabilities: dict[str, bytes] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Initialize terminal capabilities for the given terminal type. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_setup.c:setupterm() and _nc_setupterm() + - ncurses/tinfo/lib_setup.c:TINFO_SETUP_TERM() + + This version first attempts to read terminfo database files like ncurses, + then, if `fallback` is True, falls back to hardcoded capabilities for + common terminal types. + """ + # If termstr is None or empty, try to get from environment + if not self.terminal_name: + self.terminal_name = os.environ.get("TERM") or "ANSI" + + if isinstance(self.terminal_name, bytes): + self.terminal_name = self.terminal_name.decode("ascii") + + try: + self._parse_terminfo_file(self.terminal_name) + except (OSError, ValueError): + if not self.fallback: + raise + + term_type = _TERM_ALIASES.get( + self.terminal_name, self.terminal_name + ) + if term_type not in _TERMINAL_CAPABILITIES: + term_type = "dumb" + self._capabilities = _TERMINAL_CAPABILITIES[term_type].copy() + + def _parse_terminfo_file(self, terminal_name: str) -> None: + """Parse a terminfo file. + + Populate the _capabilities dict for easy retrieval + + Based on ncurses implementation in: + - ncurses/tinfo/read_entry.c:_nc_read_termtype() + - ncurses/tinfo/read_entry.c:_nc_read_file_entry() + - ncurses/tinfo/lib_ti.c:tigetstr() + """ + data = _read_terminfo_file(terminal_name) + too_short = f"TermInfo file for {terminal_name!r} too short" + offset = 12 + if len(data) < offset: + raise ValueError(too_short) + + magic, name_size, bool_count, num_count, str_count, str_size = ( + struct.unpack(" len(data): + raise ValueError(too_short) + + # Read string offsets + end_offset = offset + 2 * str_count + if offset > len(data): + raise ValueError(too_short) + string_offset_data = data[offset:end_offset] + string_offsets = [ + off for [off] in struct.iter_unpack(" len(data): + raise ValueError(too_short) + string_table = data[offset : offset + str_size] + + # Extract strings from string table + capabilities = {} + for cap, off in zip(_STRING_NAMES, string_offsets): + if off < 0: + # CANCELLED_STRING; we do not store those + continue + elif off < len(string_table): + # Find null terminator + end = string_table.find(0, off) + if end >= 0: + capabilities[cap] = string_table[off:end] + # in other cases this is ABSENT_STRING; we don't store those. + + # Note: we don't support extended capabilities since PyREPL doesn't + # need them. + + self._capabilities = capabilities + + def get(self, cap: str) -> bytes | None: + """Get terminal capability string by name. + """ + if not isinstance(cap, str): + raise TypeError(f"`cap` must be a string, not {type(cap)}") + + return self._capabilities.get(cap) + + +def tparm(cap_bytes: bytes, *params: int) -> bytes: + """Parameterize a terminal capability string. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_tparm.c:tparm() + - ncurses/tinfo/lib_tparm.c:tparam_internal() + + The ncurses version implements a full stack-based interpreter for + terminfo parameter strings. This pure Python version implements only + the subset of parameter substitution operations needed by PyREPL: + - %i (increment parameters for 1-based indexing) + - %p[1-9]%d (parameter substitution) + - %p[1-9]%{n}%+%d (parameter plus constant) + """ + if not isinstance(cap_bytes, bytes): + raise TypeError(f"`cap` must be bytes, not {type(cap_bytes)}") + + result = cap_bytes + + # %i - increment parameters (1-based instead of 0-based) + increment = b"%i" in result + if increment: + result = result.replace(b"%i", b"") + + # Replace %p1%d, %p2%d, etc. with actual parameter values + for i in range(len(params)): + pattern = b"%%p%d%%d" % (i + 1) + if pattern in result: + value = params[i] + if increment: + value += 1 + result = result.replace(pattern, str(value).encode("ascii")) + + # Handle %p1%{1}%+%d (parameter plus constant) + # Used in some cursor positioning sequences + pattern_re = re.compile(rb"%p(\d)%\{(\d+)\}%\+%d") + matches = list(pattern_re.finditer(result)) + for match in reversed(matches): # reversed to maintain positions + param_idx = int(match.group(1)) + constant = int(match.group(2)) + value = params[param_idx] + constant + result = ( + result[: match.start()] + + str(value).encode("ascii") + + result[match.end() :] + ) + + return result diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index a8eb2433cd3..943ee12f964 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import sys # types if False: @@ -12,10 +13,22 @@ trace_file = open(trace_filename, "a") -def trace(line: str, *k: object, **kw: object) -> None: - if trace_file is None: - return - if k or kw: - line = line.format(*k, **kw) - trace_file.write(line + "\n") - trace_file.flush() + +if sys.platform == "emscripten": + from posix import _emscripten_log + + def trace(line: str, *k: object, **kw: object) -> None: + if "PYREPL_TRACE" not in os.environ: + return + if k or kw: + line = line.format(*k, **kw) + _emscripten_log(line) + +else: + def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index d21cdd9b076..09247de748e 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -33,9 +33,9 @@ import platform from fcntl import ioctl -from . import curses +from . import terminfo from .console import Console, Event -from .fancy_termios import tcgetattr, tcsetattr +from .fancy_termios import tcgetattr, tcsetattr, TermState from .trace import trace from .unix_eventqueue import EventQueue from .utils import wlen @@ -51,16 +51,19 @@ # types if TYPE_CHECKING: - from typing import IO, Literal, overload + from typing import AbstractSet, IO, Literal, overload, cast else: overload = lambda func: None + cast = lambda typ, val: val class InvalidTerminal(RuntimeError): - pass + def __init__(self, message: str) -> None: + super().__init__(errno.EIO, message) -_error = (termios.error, curses.error, InvalidTerminal) +_error = (termios.error, InvalidTerminal) +_error_codes_to_ignore = frozenset([errno.EIO, errno.ENXIO, errno.EPERM]) SIGWINCH_EVENT = "repaint" @@ -125,12 +128,13 @@ def __init__(self): def register(self, fd, flag): self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* def poll(self, timeout: float | None = None) -> list[int]: if timeout is None: r, w, e = select.select([self.fd], [], []) else: - r, w, e = select.select([self.fd], [], [], timeout/1000) + r, w, e = select.select([self.fd], [], [], timeout / 1000) return r poll = MinimalPoll # type: ignore[assignment] @@ -157,17 +161,28 @@ def __init__( self.pollob = poll() self.pollob.register(self.input_fd, select.POLLIN) - curses.setupterm(term or None, self.output_fd) + self.terminfo = terminfo.TermInfo(term or None) self.term = term + self.is_apple_terminal = ( + platform.system() == "Darwin" + and os.getenv("TERM_PROGRAM") == "Apple_Terminal" + ) + + try: + self.__input_fd_set(tcgetattr(self.input_fd), ignore=frozenset()) + except _error as e: + raise RuntimeError(f"termios failure ({e.args[1]})") @overload - def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + def _my_getstr( + cap: str, optional: Literal[False] = False + ) -> bytes: ... @overload def _my_getstr(cap: str, optional: bool) -> bytes | None: ... def _my_getstr(cap: str, optional: bool = False) -> bytes | None: - r = curses.tigetstr(cap) + r = self.terminfo.get(cap) if not optional and r is None: raise InvalidTerminal( f"terminal doesn't have the required {cap} capability" @@ -201,7 +216,9 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.__setup_movement() - self.event_queue = EventQueue(self.input_fd, self.encoding) + self.event_queue = EventQueue( + self.input_fd, self.encoding, self.terminfo + ) self.cursor_visible = 1 signal.signal(signal.SIGCONT, self._sigcont_handler) @@ -213,7 +230,6 @@ def _sigcont_handler(self, signum, frame): def __read(self, n: int) -> bytes: return os.read(self.input_fd, n) - def change_encoding(self, encoding: str) -> None: """ Change the encoding used for I/O operations. @@ -325,6 +341,8 @@ def prepare(self): """ Prepare the console for input/output operations. """ + self.__buffer = [] + self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) @@ -336,17 +354,15 @@ def prepare(self): raw.lflag |= termios.ISIG raw.cc[termios.VMIN] = 1 raw.cc[termios.VTIME] = 0 - tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + self.__input_fd_set(raw) # In macOS terminal we need to deactivate line wrap via ANSI escape code - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7l") self.screen = [] self.height, self.width = self.getheightwidth() - self.__buffer = [] - self.posxy = 0, 0 self.__gone_tall = 0 self.__move = self.__move_short @@ -368,13 +384,18 @@ def restore(self): self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() - tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + self.__input_fd_set(self.__svtermstate) - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7h") if hasattr(self, "old_sigwinch"): - signal.signal(signal.SIGWINCH, self.old_sigwinch) + try: + signal.signal(signal.SIGWINCH, self.old_sigwinch) + except ValueError as e: + import threading + if threading.current_thread() is threading.main_thread(): + raise e del self.old_sigwinch def push_char(self, char: int | bytes) -> None: @@ -407,6 +428,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.get() else: continue + elif err.errno == errno.EIO: + raise SystemExit(errno.EIO) else: raise else: @@ -597,14 +620,14 @@ def __setup_movement(self): if self._dch1: self.dch1 = self._dch1 elif self._dch: - self.dch1 = curses.tparm(self._dch, 1) + self.dch1 = terminfo.tparm(self._dch, 1) else: self.dch1 = None if self._ich1: self.ich1 = self._ich1 elif self._ich: - self.ich1 = curses.tparm(self._ich, 1) + self.ich1 = terminfo.tparm(self._ich, 1) else: self.ich1 = None @@ -701,7 +724,7 @@ def __write(self, text): self.__buffer.append((text, 0)) def __write_code(self, fmt, *args): - self.__buffer.append((curses.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), 1)) def __maybe_write_code(self, fmt, *args): if fmt: @@ -803,3 +826,17 @@ def __tputs(self, fmt, prog=delayprog): os.write(self.output_fd, self._pad * nchars) else: time.sleep(float(delay) / 1000.0) + + def __input_fd_set( + self, + state: TermState, + ignore: AbstractSet[int] = _error_codes_to_ignore, + ) -> bool: + try: + tcsetattr(self.input_fd, termios.TCSADRAIN, state) + except termios.error as te: + if te.args[0] not in ignore: + raise + return False + else: + return True diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 29b3e9dd5ef..2a9cca59e74 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -18,7 +18,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from . import curses +from .terminfo import TermInfo from .trace import trace from .base_eventqueue import BaseEventQueue from termios import tcgetattr, VERASE @@ -54,22 +54,23 @@ b'\033Oc': 'ctrl right', } -def get_terminal_keycodes() -> dict[bytes, str]: +def get_terminal_keycodes(ti: TermInfo) -> dict[bytes, str]: """ Generates a dictionary mapping terminal keycodes to human-readable names. """ keycodes = {} for key, terminal_code in TERMINAL_KEYNAMES.items(): - keycode = curses.tigetstr(terminal_code) + keycode = ti.get(terminal_code) trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) if keycode: keycodes[keycode] = key keycodes.update(CTRL_ARROW_KEYCODES) return keycodes + class EventQueue(BaseEventQueue): - def __init__(self, fd: int, encoding: str) -> None: - keycodes = get_terminal_keycodes() + def __init__(self, fd: int, encoding: str, ti: TermInfo) -> None: + keycodes = get_terminal_keycodes(ti) if os.isatty(fd): backspace = tcgetattr(fd)[6][VERASE] keycodes[backspace] = "backspace" diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 38cf6b5a08e..06cddef851b 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -20,6 +20,7 @@ ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) IDENTIFIERS_AFTER = {"def", "class"} +KEYWORD_CONSTANTS = {"True", "False", "None"} BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} @@ -41,9 +42,15 @@ def from_re(cls, m: Match[str], group: int | str) -> Self: @classmethod def from_token(cls, token: TI, line_len: list[int]) -> Self: + end_offset = -1 + if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE} + and token.string.endswith(("{", "}"))): + # gh-134158: a visible trailing brace comes from a double brace in input + end_offset += 1 + return cls( line_len[token.start[0] - 1] + token.start[1], - line_len[token.end[0] - 1] + token.end[1] - 1, + line_len[token.end[0] - 1] + token.end[1] + end_offset, ) @@ -56,6 +63,12 @@ class ColorSpan(NamedTuple): def str_width(c: str) -> int: if ord(c) < 128: return 1 + # gh-139246 for zero-width joiner and combining characters + if unicodedata.combining(c): + return 0 + category = unicodedata.category(c) + if category == "Cf" and c != "\u00ad": + return 0 w = unicodedata.east_asian_width(c) if w in ("N", "Na", "H", "A"): return 1 @@ -102,6 +115,8 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]: for color in gen_colors_from_token_stream(gen, line_lengths): yield color last_emitted = color + except SyntaxError: + return except tokenize.TokenError as te: yield from recover_unterminated_string( te, line_lengths, last_emitted, buffer @@ -189,8 +204,11 @@ def gen_colors_from_token_stream( span = Span.from_token(token, line_lengths) yield ColorSpan(span, "definition") elif keyword.iskeyword(token.string): + span_cls = "keyword" + if token.string in KEYWORD_CONSTANTS: + span_cls = "keyword_constant" span = Span.from_token(token, line_lengths) - yield ColorSpan(span, "keyword") + yield ColorSpan(span, span_cls) if token.string in IDENTIFIERS_AFTER: is_def_name = True elif ( @@ -200,7 +218,10 @@ def gen_colors_from_token_stream( ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "soft_keyword") - elif token.string in BUILTINS: + elif ( + token.string in BUILTINS + and not (prev_token and prev_token.exact_type == T.DOT) + ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "builtin") @@ -233,14 +254,14 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return s in keyword_first_sets_match return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) | TI(T.OP, string="(" | "*" | "-" | "[" | "{") ): return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NAME, string=s) ): @@ -249,6 +270,12 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return True case (TI(string="case"), TI(string="_"), TI(string=":")): return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="type"), + TI(T.NAME, string=s) + ): + return not keyword.iskeyword(s) case _: return False diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index 95749198b3b..f9f5988af0b 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -249,22 +249,10 @@ def input_hook(self): def __write_changed_line( self, y: int, oldline: str, newline: str, px_coord: int ) -> None: - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) minlen = min(wlen(oldline), wlen(newline)) x_pos = 0 x_coord = 0 - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 - # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape # sequence @@ -358,7 +346,6 @@ def prepare(self) -> None: self.height, self.width = self.getheightwidth() self.posxy = 0, 0 - self.__gone_tall = 0 self.__offset = 0 if self.__vt_support: @@ -419,10 +406,7 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self, block: bool = True) -> INPUT_RECORD | None: - if not block and not self.wait(timeout=0): - return None - + def _read_input(self) -> INPUT_RECORD | None: rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): @@ -431,14 +415,10 @@ def _read_input(self, block: bool = True) -> INPUT_RECORD | None: return rec def _read_input_bulk( - self, block: bool, n: int + self, n: int ) -> tuple[ctypes.Array[INPUT_RECORD], int]: rec = (n * INPUT_RECORD)() read = DWORD() - - if not block and not self.wait(timeout=0): - return rec, 0 - if not ReadConsoleInput(InHandle, rec, n, read): raise WinError(GetLastError()) @@ -449,8 +429,11 @@ def get_event(self, block: bool = True) -> Event | None: and there is no event pending, otherwise waits for the completion of an event.""" + if not block and not self.wait(timeout=0): + return None + while self.event_queue.empty(): - rec = self._read_input(block) + rec = self._read_input() if rec is None: return None @@ -551,12 +534,20 @@ def getpending(self) -> Event: if e2: e.data += e2.data - recs, rec_count = self._read_input_bulk(False, 1024) + recs, rec_count = self._read_input_bulk(1024) for i in range(rec_count): rec = recs[i] + # In case of a legacy console, we do not only receive a keydown + # event, but also a keyup event - and for uppercase letters + # an additional SHIFT_PRESSED event. if rec and rec.EventType == KEY_EVENT: key_event = rec.Event.KeyEvent + if not key_event.bKeyDown: + continue ch = key_event.uChar.UnicodeChar + if ch == "\x00": + # ignore SHIFT_PRESSED and special keys + continue if ch == "\r": ch += "\n" e.data += ch diff --git a/Lib/_strptime.py b/Lib/_strptime.py index aa63933a49d..d011ddf8b18 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -14,6 +14,7 @@ import time import locale import calendar +import re from re import compile as re_compile from re import sub as re_sub from re import IGNORECASE @@ -41,6 +42,29 @@ def _findall(haystack, needle): yield i i += len(needle) +def _fixmonths(months): + yield from months + # The lower case of 'İ' ('\u0130') is 'i\u0307'. + # The re module only supports 1-to-1 character matching in + # case-insensitive mode. + for s in months: + if 'i\u0307' in s: + yield s.replace('i\u0307', '\u0130') + +lzh_TW_alt_digits = ( + # 〇:一:二:三:四:五:六:七:八:九 + '\u3007', '\u4e00', '\u4e8c', '\u4e09', '\u56db', + '\u4e94', '\u516d', '\u4e03', '\u516b', '\u4e5d', + # 十:十一:十二:十三:十四:十五:十六:十七:十八:十九 + '\u5341', '\u5341\u4e00', '\u5341\u4e8c', '\u5341\u4e09', '\u5341\u56db', + '\u5341\u4e94', '\u5341\u516d', '\u5341\u4e03', '\u5341\u516b', '\u5341\u4e5d', + # 廿:廿一:廿二:廿三:廿四:廿五:廿六:廿七:廿八:廿九 + '\u5eff', '\u5eff\u4e00', '\u5eff\u4e8c', '\u5eff\u4e09', '\u5eff\u56db', + '\u5eff\u4e94', '\u5eff\u516d', '\u5eff\u4e03', '\u5eff\u516b', '\u5eff\u4e5d', + # 卅:卅一 + '\u5345', '\u5345\u4e00') + + class LocaleTime(object): """Stores and handles locale-specific information related to time. @@ -84,6 +108,7 @@ def __init__(self): self.__calc_weekday() self.__calc_month() self.__calc_am_pm() + self.__calc_alt_digits() self.__calc_timezone() self.__calc_date_time() if _getlang() != self.lang: @@ -119,9 +144,43 @@ def __calc_am_pm(self): am_pm.append(time.strftime("%p", time_tuple).lower().strip()) self.am_pm = am_pm + def __calc_alt_digits(self): + # Set self.LC_alt_digits by using time.strftime(). + + # The magic data should contain all decimal digits. + time_tuple = time.struct_time((1998, 1, 27, 10, 43, 56, 1, 27, 0)) + s = time.strftime("%x%X", time_tuple) + if s.isascii(): + # Fast path -- all digits are ASCII. + self.LC_alt_digits = () + return + + digits = ''.join(sorted(set(re.findall(r'\d', s)))) + if len(digits) == 10 and ord(digits[-1]) == ord(digits[0]) + 9: + # All 10 decimal digits from the same set. + if digits.isascii(): + # All digits are ASCII. + self.LC_alt_digits = () + return + + self.LC_alt_digits = [a + b for a in digits for b in digits] + # Test whether the numbers contain leading zero. + time_tuple2 = time.struct_time((2000, 1, 1, 1, 1, 1, 5, 1, 0)) + if self.LC_alt_digits[1] not in time.strftime("%x %X", time_tuple2): + self.LC_alt_digits[:10] = digits + return + + # Either non-Gregorian calendar or non-decimal numbers. + if {'\u4e00', '\u4e03', '\u4e5d', '\u5341', '\u5eff'}.issubset(s): + # lzh_TW + self.LC_alt_digits = lzh_TW_alt_digits + return + + self.LC_alt_digits = None + def __calc_date_time(self): - # Set self.date_time, self.date, & self.time by using - # time.strftime(). + # Set self.LC_date_time, self.LC_date, self.LC_time and + # self.LC_time_ampm by using time.strftime(). # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of # overloaded numbers is minimized. The order in which searches for @@ -129,26 +188,32 @@ def __calc_date_time(self): # possible ambiguity for what something represents. time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0)) - replacement_pairs = [ + replacement_pairs = [] + + # Non-ASCII digits + if self.LC_alt_digits or self.LC_alt_digits is None: + for n, d in [(19, '%OC'), (99, '%Oy'), (22, '%OH'), + (44, '%OM'), (55, '%OS'), (17, '%Od'), + (3, '%Om'), (2, '%Ow'), (10, '%OI')]: + if self.LC_alt_digits is None: + s = chr(0x660 + n // 10) + chr(0x660 + n % 10) + replacement_pairs.append((s, d)) + if n < 10: + replacement_pairs.append((s[1], d)) + elif len(self.LC_alt_digits) > n: + replacement_pairs.append((self.LC_alt_digits[n], d)) + else: + replacement_pairs.append((time.strftime(d, time_tuple), d)) + replacement_pairs += [ ('1999', '%Y'), ('99', '%y'), ('22', '%H'), ('44', '%M'), ('55', '%S'), ('76', '%j'), ('17', '%d'), ('03', '%m'), ('3', '%m'), # '3' needed for when no leading zero. ('2', '%w'), ('10', '%I'), - # Non-ASCII digits - ('\u0661\u0669\u0669\u0669', '%Y'), - ('\u0669\u0669', '%Oy'), - ('\u0662\u0662', '%OH'), - ('\u0664\u0664', '%OM'), - ('\u0665\u0665', '%OS'), - ('\u0661\u0667', '%Od'), - ('\u0660\u0663', '%Om'), - ('\u0663', '%Om'), - ('\u0662', '%Ow'), - ('\u0661\u0660', '%OI'), ] + date_time = [] - for directive in ('%c', '%x', '%X'): + for directive in ('%c', '%x', '%X', '%r'): current_format = time.strftime(directive, time_tuple).lower() current_format = current_format.replace('%', '%%') # The month and the day of the week formats are treated specially @@ -172,9 +237,10 @@ def __calc_date_time(self): if tz: current_format = current_format.replace(tz, "%Z") # Transform all non-ASCII digits to digits in range U+0660 to U+0669. - current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", 'f': r"(?P[0-9]{1,6})", - 'H': r"(?P2[0-3]|[0-1]\d|\d)", + 'H': r"(?P2[0-3]|[0-1]\d|\d| \d)", + 'k': r"(?P2[0-3]|[0-1]\d|\d| \d)", 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", + 'l': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", @@ -302,26 +371,61 @@ def __init__(self, locale_time=None): # W is set below by using 'U' 'y': r"(?P\d\d)", 'Y': r"(?P\d\d\d\d)", - 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", + # See gh-121237: "z" must support colons for backwards compatibility. + 'z': r"(?P([+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?)|(?-i:Z))?", + ':z': r"(?P([+-]\d\d:[0-5]\d(:[0-5]\d(\.\d{1,6})?)?)|(?-i:Z))?", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), - 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'), - 'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'), + 'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'), + 'b': self.__seqToRE(_fixmonths(self.locale_time.a_month[1:]), 'b'), 'p': self.__seqToRE(self.locale_time.am_pm, 'p'), 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), '%': '%'} - for d in 'dmyHIMS': - mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d - mapping['Ow'] = r'(?P\d)' + if self.locale_time.LC_alt_digits is None: + for d in 'dmyCHIMS': + mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d + mapping['Ow'] = r'(?P\d)' + else: + mapping.update({ + 'Od': self.__seqToRE(self.locale_time.LC_alt_digits[1:32], 'd', + '3[0-1]|[1-2][0-9]|0[1-9]|[1-9]'), + 'Om': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'm', + '1[0-2]|0[1-9]|[1-9]'), + 'Ow': self.__seqToRE(self.locale_time.LC_alt_digits[:7], 'w', + '[0-6]'), + 'Oy': self.__seqToRE(self.locale_time.LC_alt_digits, 'y', + '[0-9][0-9]'), + 'OC': self.__seqToRE(self.locale_time.LC_alt_digits, 'C', + '[0-9][0-9]'), + 'OH': self.__seqToRE(self.locale_time.LC_alt_digits[:24], 'H', + '2[0-3]|[0-1][0-9]|[0-9]'), + 'OI': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'I', + '1[0-2]|0[1-9]|[1-9]'), + 'OM': self.__seqToRE(self.locale_time.LC_alt_digits[:60], 'M', + '[0-5][0-9]|[0-9]'), + 'OS': self.__seqToRE(self.locale_time.LC_alt_digits[:62], 'S', + '6[0-1]|[0-5][0-9]|[0-9]'), + }) + mapping.update({ + 'e': mapping['d'], + 'Oe': mapping['Od'], + 'P': mapping['p'], + 'Op': mapping['p'], + 'W': mapping['U'].replace('U', 'W'), + }) mapping['W'] = mapping['U'].replace('U', 'W') + base.__init__(mapping) + base.__setitem__('T', self.pattern('%H:%M:%S')) + base.__setitem__('R', self.pattern('%H:%M')) + base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm)) base.__setitem__('X', self.pattern(self.locale_time.LC_time)) base.__setitem__('x', self.pattern(self.locale_time.LC_date)) base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - def __seqToRE(self, to_convert, directive): + def __seqToRE(self, to_convert, directive, altregex=None): """Convert a list to a regex string for matching a directive. Want possible matching values to be from longest to shortest. This @@ -337,8 +441,9 @@ def __seqToRE(self, to_convert, directive): else: return '' regex = '|'.join(re_escape(stuff) for stuff in to_convert) - regex = '(?P<%s>%s' % (directive, regex) - return '%s)' % regex + if altregex is not None: + regex += '|' + altregex + return '(?P<%s>%s)' % (directive, regex) def pattern(self, format): """Return regex pattern for the format string. @@ -356,16 +461,16 @@ def pattern(self, format): year_in_format = False day_of_month_in_format = False def repl(m): - format_char = m[1] - match format_char: + directive = m.group()[1:] # exclude `%` symbol + match directive: case 'Y' | 'y' | 'G': nonlocal year_in_format year_in_format = True case 'd': nonlocal day_of_month_in_format day_of_month_in_format = True - return self[format_char] - format = re_sub(r'%([OE]?\\?.?)', repl, format) + return self[directive] + format = re_sub(r'%[-_0^#]*[0-9]*([OE]?[:\\]?.?)', repl, format) if day_of_month_in_format and not year_in_format: import warnings warnings.warn("""\ @@ -452,8 +557,17 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): raise ValueError("time data %r does not match format %r" % (data_string, format)) if len(data_string) != found.end(): - raise ValueError("unconverted data remains: %s" % - data_string[found.end():]) + rest = data_string[found.end():] + # Specific check for '%:z' directive + if ( + "colon_z" in found.re.groupindex + and found.group("colon_z") is not None + and rest[0] != ":" + ): + raise ValueError( + f"Missing colon in %:z before '{rest}', got '{data_string}'" + ) + raise ValueError("unconverted data remains: %s" % rest) iso_year = year = None month = day = 1 @@ -467,6 +581,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # values weekday = julian = None found_dict = found.groupdict() + if locale_time.LC_alt_digits: + def parse_int(s): + try: + return locale_time.LC_alt_digits.index(s) + except ValueError: + return int(s) + else: + parse_int = int + for group_key in found_dict.keys(): # Directives not explicitly handled below: # c, x, X @@ -474,48 +597,52 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # U, W # worthless without day of the week if group_key == 'y': - year = int(found_dict['y']) - # Open Group specification for strptime() states that a %y - #value in the range of [00, 68] is in the century 2000, while - #[69,99] is in the century 1900 - if year <= 68: - year += 2000 + year = parse_int(found_dict['y']) + if 'C' in found_dict: + century = parse_int(found_dict['C']) + year += century * 100 else: - year += 1900 + # Open Group specification for strptime() states that a %y + #value in the range of [00, 68] is in the century 2000, while + #[69,99] is in the century 1900 + if year <= 68: + year += 2000 + else: + year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) elif group_key == 'G': iso_year = int(found_dict['G']) elif group_key == 'm': - month = int(found_dict['m']) + month = parse_int(found_dict['m']) elif group_key == 'B': month = locale_time.f_month.index(found_dict['B'].lower()) elif group_key == 'b': month = locale_time.a_month.index(found_dict['b'].lower()) elif group_key == 'd': - day = int(found_dict['d']) + day = parse_int(found_dict['d']) elif group_key == 'H': - hour = int(found_dict['H']) + hour = parse_int(found_dict['H']) elif group_key == 'I': - hour = int(found_dict['I']) + hour = parse_int(found_dict['I']) ampm = found_dict.get('p', '').lower() # If there was no AM/PM indicator, we'll treat this like AM - if ampm in ('', locale_time.am_pm[0]): - # We're in AM so the hour is correct unless we're - # looking at 12 midnight. - # 12 midnight == 12 AM == hour 0 - if hour == 12: - hour = 0 - elif ampm == locale_time.am_pm[1]: + if ampm == locale_time.am_pm[1]: # We're in PM so we need to add 12 to the hour unless # we're looking at 12 noon. # 12 noon == 12 PM == hour 12 if hour != 12: hour += 12 + else: + # We're in AM so the hour is correct unless we're + # looking at 12 midnight. + # 12 midnight == 12 AM == hour 0 + if hour == 12: + hour = 0 elif group_key == 'M': - minute = int(found_dict['M']) + minute = parse_int(found_dict['M']) elif group_key == 'S': - second = int(found_dict['S']) + second = parse_int(found_dict['S']) elif group_key == 'f': s = found_dict['f'] # Pad to always return microseconds. @@ -546,29 +673,30 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): week_of_year_start = 0 elif group_key == 'V': iso_week = int(found_dict['V']) - elif group_key == 'z': - z = found_dict['z'] - if z == 'Z': - gmtoff = 0 - else: - if z[3] == ':': - z = z[:3] + z[4:] - if len(z) > 5: - if z[5] != ':': - msg = f"Inconsistent use of : in {found_dict['z']}" - raise ValueError(msg) - z = z[:5] + z[6:] - hours = int(z[1:3]) - minutes = int(z[3:5]) - seconds = int(z[5:7] or 0) - gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds - gmtoff_remainder = z[8:] - # Pad to always return microseconds. - gmtoff_remainder_padding = "0" * (6 - len(gmtoff_remainder)) - gmtoff_fraction = int(gmtoff_remainder + gmtoff_remainder_padding) - if z.startswith("-"): - gmtoff = -gmtoff - gmtoff_fraction = -gmtoff_fraction + elif group_key in ('z', 'colon_z'): + z = found_dict[group_key] + if z: + if z == 'Z': + gmtoff = 0 + else: + if z[3] == ':': + z = z[:3] + z[4:] + if len(z) > 5: + if z[5] != ':': + msg = f"Inconsistent use of : in {found_dict[group_key]}" + raise ValueError(msg) + z = z[:5] + z[6:] + hours = int(z[1:3]) + minutes = int(z[3:5]) + seconds = int(z[5:7] or 0) + gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds + gmtoff_remainder = z[8:] + # Pad to always return microseconds. + gmtoff_remainder_padding = "0" * (6 - len(gmtoff_remainder)) + gmtoff_fraction = int(gmtoff_remainder + gmtoff_remainder_padding) + if z.startswith("-"): + gmtoff = -gmtoff + gmtoff_fraction = -gmtoff_fraction elif group_key == 'Z': # Since -1 is default value only need to worry about setting tz if # it can be something other than -1. diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index c0b1d4395d1..a5788cdbfae 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -27,6 +27,9 @@ class Format(enum.IntEnum): _sentinel = object() +# Following `NAME_ERROR_MSG` in `ceval_macros.h`: +_NAME_ERROR_MSG = "name '{name:.200}' is not defined" + # Slots shared by ForwardRef and _Stringifier. The __forward__ names must be # preserved for compatibility with the old typing.ForwardRef class. The remaining @@ -82,6 +85,9 @@ def __init__( # These are always set to None here but may be non-None if a ForwardRef # is created through __class__ assignment on a _Stringifier object. self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). self.__cell__ = None self.__extra_names__ = None # These are initially None but serve as a cache and may be set to a non-None @@ -114,7 +120,7 @@ def evaluate( is_forwardref_format = True case _: raise NotImplementedError(format) - if self.__cell__ is not None: + if isinstance(self.__cell__, types.CellType): try: return self.__cell__.cell_contents except ValueError: @@ -144,34 +150,42 @@ def evaluate( if globals is None: globals = {} + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + if locals is None: locals = {} if isinstance(owner, type): locals.update(vars(owner)) - - if type_params is None and owner is not None: - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - type_params = getattr(owner, "__type_params__", None) - - # type parameters require some special handling, - # as they exist in their own scope - # but `eval()` does not have a dedicated parameter for that scope. - # For classes, names in type parameter scopes should override - # names in the global scope (which here are called `localns`!), - # but should in turn be overridden by names in the class scope - # (which here are called `globalns`!) - if type_params is not None: - globals = dict(globals) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. locals = dict(locals) + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params is not None: for param in type_params: - param_name = param.__name__ - if not self.__forward_is_class__ or param_name not in globals: - globals[param_name] = param - locals.pop(param_name, None) + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + if self.__extra_names__: - locals = {**locals, **self.__extra_names__} + locals.update(self.__extra_names__) arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): @@ -184,7 +198,7 @@ def evaluate( elif is_forwardref_format: return self else: - raise NameError(arg) + raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg) else: code = self.__forward_code__ try: @@ -192,8 +206,11 @@ def evaluate( except Exception: if not is_forwardref_format: raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. new_locals = _StringifierDict( - {**builtins.__dict__, **locals}, + {**builtins.__dict__, **globals, **locals}, globals=globals, owner=owner, is_class=self.__forward_is_class__, @@ -204,7 +221,7 @@ def evaluate( except Exception: return self else: - new_locals.transmogrify() + new_locals.transmogrify(self.__cell__) return result def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): @@ -246,15 +263,8 @@ def __forward_code__(self): if self.__code__ is not None: return self.__code__ arg = self.__forward_arg__ - # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. - # Unfortunately, this isn't a valid expression on its own, so we - # do the unpacking manually. - if arg.startswith("*"): - arg_to_compile = f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - arg_to_compile = arg try: - self.__code__ = compile(arg_to_compile, "", "eval") + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") except SyntaxError: raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") return self.__code__ @@ -283,7 +293,7 @@ def __hash__(self): self.__forward_module__, id(self.__globals__), # dictionaries are not hashable, so hash by identity self.__forward_is_class__, - self.__cell__, + tuple(sorted(self.__cell__.items())) if isinstance(self.__cell__, dict) else self.__cell__, self.__owner__, tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) @@ -305,6 +315,9 @@ def __repr__(self): return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" +_Template = type(t"") + + class _Stringifier: # Must match the slots on ForwardRef, so we can turn an instance of one into an # instance of the other in place. @@ -341,6 +354,8 @@ def __convert_to_ast(self, other): if isinstance(other.__ast_node__, str): return ast.Name(id=other.__ast_node__), other.__extra_names__ return other.__ast_node__, other.__extra_names__ + elif type(other) is _Template: + return _template_to_ast(other), None elif ( # In STRING format we don't bother with the create_unique_name() dance; # it's better to emit the repr() of the object instead of an opaque name. @@ -560,6 +575,70 @@ def unary_op(self): del _make_unary_op +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" + values = [] + interp_count = 0 + for part in template: + match part: + case str(): + values.append(ast.Constant(value=part)) + case _: + interp = ast.Interpolation( + str=part.expression, + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, + ) + values.append(interp) + interp_count += 1 + return ast.TemplateStr(values=values) + + +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + class _StringifierDict(dict): def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): super().__init__(namespace) @@ -582,13 +661,15 @@ def __missing__(self, key): self.stringifiers.append(fwdref) return fwdref - def transmogrify(self): + def transmogrify(self, cell_dict): for obj in self.stringifiers: obj.__class__ = ForwardRef obj.__stringifier_dict__ = None # not needed for ForwardRef if isinstance(obj.__ast_node__, str): obj.__arg__ = obj.__ast_node__ obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict def create_unique_name(self): name = f"__annotationlib_name_{self.next_id}__" @@ -638,9 +719,21 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + globals = _StringifierDict({}, format=format) is_class = isinstance(owner, type) - closure = _build_closure( + closure, _ = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -684,7 +777,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=True ) func = types.FunctionType( @@ -696,10 +789,13 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) try: result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) except Exception: pass else: - globals.transmogrify() + globals.transmogrify(cell_dict) return result # Try again, but do not provide any globals. This allows us to return @@ -711,7 +807,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -722,7 +818,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): kwdefaults=annotate.__kwdefaults__, ) result = func(Format.VALUE_WITH_FAKE_GLOBALS) - globals.transmogrify() + globals.transmogrify(cell_dict) if _is_evaluate: if isinstance(result, ForwardRef): return result.evaluate(format=Format.FORWARDREF) @@ -747,14 +843,11 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): if not annotate.__closure__: - return None - freevars = annotate.__code__.co_freevars + return None, None new_closure = [] - for i, cell in enumerate(annotate.__closure__): - if i < len(freevars): - name = freevars[i] - else: - name = "__cell__" + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell new_cell = None if allow_evaluation: try: @@ -775,7 +868,7 @@ def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluat stringifier_dict.stringifiers.append(fwdref) new_cell = types.CellType(fwdref) new_closure.append(new_cell) - return tuple(new_closure) + return tuple(new_closure), cell_dict def _stringify_single(anno): @@ -784,6 +877,8 @@ def _stringify_single(anno): # We have to handle str specially to support PEP 563 stringified annotations. elif isinstance(anno, str): return anno + elif isinstance(anno, _Template): + return ast.unparse(_template_to_ast(anno)) else: return repr(anno) @@ -906,48 +1001,49 @@ def get_annotations( if not eval_str: return dict(ann) - if isinstance(obj, type): - # class - obj_globals = None - module_name = getattr(obj, "__module__", None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - obj_globals = getattr(module, "__dict__", None) - obj_locals = dict(vars(obj)) - unwrap = obj - elif isinstance(obj, types.ModuleType): - # module - obj_globals = getattr(obj, "__dict__") - obj_locals = None - unwrap = None - elif callable(obj): - # this includes types.Function, types.BuiltinFunctionType, - # types.BuiltinMethodType, functools.partial, functools.singledispatch, - # "class funclike" from Lib/test/test_inspect... on and on it goes. - obj_globals = getattr(obj, "__globals__", None) - obj_locals = None - unwrap = obj - else: - obj_globals = obj_locals = unwrap = None + if globals is None or locals is None: + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + else: + obj_globals = obj_locals = unwrap = None - if unwrap is not None: - while True: - if hasattr(unwrap, "__wrapped__"): - unwrap = unwrap.__wrapped__ - continue - if functools := sys.modules.get("functools"): - if isinstance(unwrap, functools.partial): - unwrap = unwrap.func + if unwrap is not None: + while True: + if hasattr(unwrap, "__wrapped__"): + unwrap = unwrap.__wrapped__ continue - break - if hasattr(unwrap, "__globals__"): - obj_globals = unwrap.__globals__ + if functools := sys.modules.get("functools"): + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ - if globals is None: - globals = obj_globals - if locals is None: - locals = obj_locals + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals # "Inject" type parameters into the local namespace # (unless they are shadowed by assignments *in* the local namespace), @@ -958,7 +1054,8 @@ def get_annotations( locals = {param.__name__: param for param in type_params} | locals return_value = { - key: value if not isinstance(value, str) else eval(value, globals, locals) + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) for key, value in ann.items() } return return_value @@ -976,6 +1073,9 @@ def type_repr(value): if value.__module__ == "builtins": return value.__qualname__ return f"{value.__module__}.{value.__qualname__}" + elif isinstance(value, _Template): + tree = _template_to_ast(value) + return ast.unparse(tree) if value is ...: return "..." return repr(value) @@ -992,6 +1092,16 @@ def annotations_to_string(annotations): } +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + def _get_and_call_annotate(obj, format): """Get the __annotate__ function and call it. @@ -1006,14 +1116,27 @@ def _get_and_call_annotate(obj, format): return None +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + def _get_dunder_annotations(obj): """Return the annotations for an object, checking that it is a dictionary. Does not return a fresh dictionary. """ - ann = getattr(obj, "__annotations__", None) - if ann is None: - return None + # This special case is needed to support types defined under + # from __future__ import annotations, where accessing the __annotations__ + # attribute directly might return annotations for the wrong class. + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return None + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return None if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") diff --git a/Lib/argparse.py b/Lib/argparse.py index f13ac82dbc5..1d550264ae4 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -64,7 +64,6 @@ still considered an implementation detail.) """ -__version__ = '1.1' __all__ = [ 'ArgumentParser', 'ArgumentError', @@ -90,8 +89,8 @@ import os as _os import re as _re import sys as _sys - -from gettext import gettext as _, ngettext +from gettext import gettext as _ +from gettext import ngettext SUPPRESS = '==SUPPRESS==' @@ -167,8 +166,6 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - prefix_chars='-', - color=False, ): # default setting for width if width is None: @@ -176,16 +173,6 @@ def __init__( width = shutil.get_terminal_size().columns width -= 2 - from _colorize import can_colorize, decolor, get_theme - - if color and can_colorize(): - self._theme = get_theme(force_color=True).argparse - self._decolor = decolor - else: - self._theme = get_theme(force_no_color=True).argparse - self._decolor = lambda text: text - - self._prefix_chars = prefix_chars self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -202,9 +189,22 @@ def __init__( self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') + self._set_color(False) + + def _set_color(self, color, *, file=None): + from _colorize import can_colorize, decolor, get_theme + + if color and can_colorize(file=file): + self._theme = get_theme(force_color=True).argparse + self._decolor = decolor + else: + self._theme = get_theme(force_no_color=True).argparse + self._decolor = lambda text: text + # =============================== # Section and indentation methods # =============================== + def _indent(self): self._current_indent += self._indent_increment self._level += 1 @@ -256,6 +256,7 @@ def _add_item(self, func, args): # ======================== # Message building methods # ======================== + def start_section(self, heading): self._indent() section = self._Section(self, self._current_section, heading) @@ -279,7 +280,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) @@ -299,6 +300,7 @@ def add_arguments(self, actions): # ======================= # Help-formatting methods # ======================= + def format_help(self): help = self._root_section.format_help() if help: @@ -334,27 +336,17 @@ def _format_usage(self, usage, actions, groups, prefix): elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -411,120 +403,114 @@ def get_lines(parts, indent, prefix=None): # prefix with 'usage:' return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) - def _is_long_option(self, string): - return len(string) >= 2 and string[1] in self._prefix_chars - - def _is_short_option(self, string): - return ( - not self._is_long_option(string) - and len(string) >= 1 - and string[0] in self._prefix_chars - ) + return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} + """Get usage parts with split index for optionals/positionals. + + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') - - if all(action.help is SUPPRESS for action in group._group_actions): - continue - - try: - start = min(actions.index(item) for item in group._group_actions) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if set(actions[start:end]) == set(group._group_actions): - group_actions.update(group._group_actions) - inserts[start, end] = group + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] t = self._theme - for action in actions: + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = ( - t.summary_action - + self._format_args(action, default) - + t.reset - ) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() - if self._is_long_option(part): - part = f"{t.summary_long_option}{part}{t.reset}" - elif self._is_short_option(part): - part = f"{t.summary_short_option}{part}{t.reset}" - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS + # produce the first way to invoke the option in brackets else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) + option_string = action.option_strings[0] if self._is_long_option(option_string): option_color = t.summary_long_option - elif self._is_short_option(option_string): + else: option_color = t.summary_short_option - part = ( - f"{option_color}{option_string} " - f"{t.summary_label}{args_string}{t.reset}" - ) - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # add the action string to the list - parts.append(part) + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part - # return the usage parts - return [item for item in parts if item is not None] + # add the action string to the list + parts.append(part) + + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' + + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: @@ -606,10 +592,8 @@ def color_option_strings(strings): for s in strings: if self._is_long_option(s): parts.append(f"{t.long_option}{s}{t.reset}") - elif self._is_short_option(s): - parts.append(f"{t.short_option}{s}{t.reset}") else: - parts.append(s) + parts.append(f"{t.short_option}{s}{t.reset}") return parts # if the Optional doesn't take a value, format is: @@ -758,7 +742,14 @@ def _get_help_string(self, action): if action.default is not SUPPRESS: defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: - help += _(' (default: %(default)s)') + t = self._theme + default_str = _(" (default: %(default)s)") + prefix, suffix = default_str.split("%(default)s") + help += ( + f" {t.default}{prefix.lstrip()}" + f"{t.default_value}%(default)s" + f"{t.default}{suffix}{t.reset}" + ) return help @@ -942,15 +933,26 @@ def __init__(self, deprecated=False): _option_strings = [] + neg_option_strings = [] for option_string in option_strings: _option_strings.append(option_string) - if option_string.startswith('--'): - if option_string.startswith('--no-'): + if len(option_string) > 2 and option_string[0] == option_string[1]: + # two-dash long option: '--foo' -> '--no-foo' + if option_string.startswith('no-', 2): raise ValueError(f'invalid option name {option_string!r} ' f'for BooleanOptionalAction') - option_string = '--no-' + option_string[2:] + option_string = option_string[:2] + 'no-' + option_string[2:] _option_strings.append(option_string) + neg_option_strings.append(option_string) + elif len(option_string) > 2 and option_string[0] != option_string[1]: + # single-dash long option: '-foo' -> '-nofoo' + if option_string.startswith('no', 1): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') + option_string = option_string[:1] + 'no' + option_string[1:] + _option_strings.append(option_string) + neg_option_strings.append(option_string) super().__init__( option_strings=_option_strings, @@ -960,11 +962,12 @@ def __init__(self, required=required, help=help, deprecated=deprecated) + self.neg_option_strings = neg_option_strings def __call__(self, parser, namespace, values, option_string=None): if option_string in self.option_strings: - setattr(namespace, self.dest, not option_string.startswith('--no-')) + setattr(namespace, self.dest, option_string not in self.neg_option_strings) def format_usage(self): return ' | '.join(self.option_strings) @@ -1240,7 +1243,7 @@ def __init__(self, self._name_parser_map = {} self._choices_actions = [] self._deprecated = set() - self._color = False + self._color = True super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1479,6 +1482,7 @@ def __init__(self, # ==================== # Registration methods # ==================== + def register(self, registry_name, value, object): registry = self._registries.setdefault(registry_name, {}) registry[value] = object @@ -1489,6 +1493,7 @@ def _registry_get(self, registry_name, value, default=None): # ================================== # Namespace default accessor methods # ================================== + def set_defaults(self, **kwargs): self._defaults.update(kwargs) @@ -1508,6 +1513,7 @@ def get_default(self, dest): # ======================= # Adding argument actions # ======================= + def add_argument(self, *args, **kwargs): """ add_argument(dest, ..., name=value, ...) @@ -1540,7 +1546,7 @@ def add_argument(self, *args, **kwargs): action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action {action_class!r}') + raise ValueError(f'unknown action {action_class!r}') action = action_class(**kwargs) # raise an error if action for positional argument does not @@ -1558,8 +1564,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1667,29 +1673,35 @@ def _get_positional_kwargs(self, dest, **kwargs): def _get_optional_kwargs(self, *args, **kwargs): # determine short and long option strings option_strings = [] - long_option_strings = [] for option_string in args: # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: + if option_string[0] not in self.prefix_chars: raise ValueError( f'invalid option string {option_string!r}: ' f'must start with a character {self.prefix_chars!r}') - - # strings starting with two prefix characters are long options option_strings.append(option_string) - if len(option_string) > 1 and option_string[1] in self.prefix_chars: - long_option_strings.append(option_string) # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' dest = kwargs.pop('dest', None) if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip(self.prefix_chars) + priority = 0 + for option_string in option_strings: + if len(option_string) <= 2: + # short option: '-x' -> 'x' + if priority < 1: + dest = option_string.lstrip(self.prefix_chars) + priority = 1 + elif option_string[1] not in self.prefix_chars: + # single-dash long option: '-foo' -> 'foo' + if priority < 2: + dest = option_string.lstrip(self.prefix_chars) + priority = 2 + else: + # two-dash long option: '--foo' -> 'foo' + dest = option_string.lstrip(self.prefix_chars) + break if not dest: - msg = f'dest= is required for options like {option_string!r}' + msg = f'dest= is required for options like {repr(option_strings)[1:-1]}' raise TypeError(msg) dest = dest.replace('-', '_') @@ -1747,8 +1759,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1864,7 +1876,7 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs - suggest_on_error - Enables suggestions for mistyped argument choices - and subparser names (default: ``False``) + and subparser names (default: ``True``) - color - Allow color output in help messages (default: ``False``) """ @@ -1883,8 +1895,8 @@ def __init__(self, allow_abbrev=True, exit_on_error=True, *, - suggest_on_error=False, - color=False, + suggest_on_error=True, + color=True, ): superinit = super(ArgumentParser, self).__init__ superinit(description=description, @@ -1903,6 +1915,9 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) @@ -1933,6 +1948,7 @@ def identity(string): # ======================= # Pretty __repr__ methods # ======================= + def _get_kwargs(self): names = [ 'prog', @@ -1947,6 +1963,7 @@ def _get_kwargs(self): # ================================== # Optional/Positional adding methods # ================================== + def add_subparsers(self, **kwargs): if self._subparsers is not None: raise ValueError('cannot have multiple subparser arguments') @@ -1962,12 +1979,16 @@ def add_subparsers(self, **kwargs): self._subparsers = self._positionals # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix + # non-required optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() + required_optionals = [action for action in self._get_optional_actions() + if action.required] groups = self._mutually_exclusive_groups - formatter.add_usage(None, positionals, groups, '') + formatter.add_usage(None, required_optionals + positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list @@ -2000,6 +2021,7 @@ def _get_positional_actions(self): # ===================================== # Command line argument parsing methods # ===================================== + def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: @@ -2433,7 +2455,7 @@ def _parse_optional(self, arg_string): return None # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: + if arg_string[0] not in self.prefix_chars: return None # if the option string is present in the parser, return the action @@ -2594,6 +2616,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # ======================== # Value conversion methods # ======================== + def _get_values(self, action, arg_strings): # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: @@ -2693,14 +2716,17 @@ def _check_value(self, action, value): # ======================= # Help-formatting methods # ======================= - def format_usage(self): - formatter = self._get_formatter() + + def format_usage(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) return formatter.format_help() - def format_help(self): - formatter = self._get_formatter() + def format_help(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() # usage formatter.add_usage(self.usage, self._actions, @@ -2722,30 +2748,45 @@ def format_help(self): # determine help from format above return formatter.format_help() - def _get_formatter(self): - if isinstance(self.formatter_class, type) and issubclass( - self.formatter_class, HelpFormatter - ): - return self.formatter_class( - prog=self.prog, - prefix_chars=self.prefix_chars, - color=self.color, - ) - else: - return self.formatter_class(prog=self.prog) + def _get_formatter(self, file=None): + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(self.color, file=file) + return formatter + + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter # ===================== # Help-printing methods # ===================== + def print_usage(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_usage(), file) + formatter = self._get_formatter(file=file) + try: + usage_text = self.format_usage(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + usage_text = self.format_usage() + self._print_message(usage_text, file) def print_help(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_help(), file) + formatter = self._get_formatter(file=file) + try: + help_text = self.format_help(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + help_text = self.format_help() + self._print_message(help_text, file) def _print_message(self, message, file=None): if message: @@ -2755,9 +2796,18 @@ def _print_message(self, message, file=None): except (AttributeError, OSError): pass + def _get_theme(self, file=None): + from _colorize import can_colorize, get_theme + + if self.color and can_colorize(file=file): + return get_theme(force_color=True).argparse + else: + return get_theme(force_no_color=True).argparse + # =============== # Exiting methods # =============== + def exit(self, status=0, message=None): if message: self._print_message(message, _sys.stderr) @@ -2773,9 +2823,26 @@ def error(self, message): should either exit or raise an exception. """ self.print_usage(_sys.stderr) + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: error: %(message)s\n') + fmt = fmt.replace('error: %(message)s', + f'{theme.error}error:{theme.reset} {theme.message}%(message)s{theme.reset}') + args = {'prog': self.prog, 'message': message} - self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + self.exit(2, fmt % args) def _warning(self, message): + theme = self._get_theme(file=_sys.stderr) + fmt = _('%(prog)s: warning: %(message)s\n') + fmt = fmt.replace('warning: %(message)s', + f'{theme.warning}warning:{theme.reset} {theme.message}%(message)s{theme.reset}') args = {'prog': self.prog, 'message': message} - self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) + self._print_message(fmt % args, _sys.stderr) + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "1.1" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/ast.py b/Lib/ast.py index b9791bf52d3..d9743ba7ab4 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -24,7 +24,7 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None, optimize=-1): + type_comments=False, feature_version=None, optimize=-1, module=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). @@ -44,7 +44,8 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version, optimize=optimize) + _feature_version=feature_version, optimize=optimize, + module=module) def literal_eval(node_or_string): @@ -57,53 +58,60 @@ def literal_eval(node_or_string): Caution: A complex expression can overflow the C stack and cause a crash. """ if isinstance(node_or_string, str): - node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval') - if isinstance(node_or_string, Expression): + node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval').body + elif isinstance(node_or_string, Expression): node_or_string = node_or_string.body - def _raise_malformed_node(node): - msg = "malformed node or string" - if lno := getattr(node, 'lineno', None): - msg += f' on line {lno}' - raise ValueError(msg + f': {node!r}') - def _convert_num(node): - if not isinstance(node, Constant) or type(node.value) not in (int, float, complex): - _raise_malformed_node(node) + return _convert_literal(node_or_string) + + +def _convert_literal(node): + """ + Used by `literal_eval` to convert an AST node into a value. + """ + if isinstance(node, Constant): return node.value - def _convert_signed_num(node): - if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): - operand = _convert_num(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand - return _convert_num(node) - def _convert(node): - if isinstance(node, Constant): - return node.value - elif isinstance(node, Tuple): - return tuple(map(_convert, node.elts)) - elif isinstance(node, List): - return list(map(_convert, node.elts)) - elif isinstance(node, Set): - return set(map(_convert, node.elts)) - elif (isinstance(node, Call) and isinstance(node.func, Name) and - node.func.id == 'set' and node.args == node.keywords == []): - return set() - elif isinstance(node, Dict): - if len(node.keys) != len(node.values): - _raise_malformed_node(node) - return dict(zip(map(_convert, node.keys), - map(_convert, node.values))) - elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): - left = _convert_signed_num(node.left) - right = _convert_num(node.right) - if isinstance(left, (int, float)) and isinstance(right, complex): - if isinstance(node.op, Add): - return left + right - else: - return left - right - return _convert_signed_num(node) - return _convert(node_or_string) + if isinstance(node, Dict) and len(node.keys) == len(node.values): + return dict(zip( + map(_convert_literal, node.keys), + map(_convert_literal, node.values), + )) + if isinstance(node, Tuple): + return tuple(map(_convert_literal, node.elts)) + if isinstance(node, List): + return list(map(_convert_literal, node.elts)) + if isinstance(node, Set): + return set(map(_convert_literal, node.elts)) + if ( + isinstance(node, Call) and isinstance(node.func, Name) + and node.func.id == 'set' and node.args == node.keywords == [] + ): + return set() + if ( + isinstance(node, UnaryOp) + and isinstance(node.op, (UAdd, USub)) + and isinstance(node.operand, Constant) + and type(operand := node.operand.value) in (int, float, complex) + ): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + if ( + isinstance(node, BinOp) + and isinstance(node.op, (Add, Sub)) + and isinstance(node.left, (Constant, UnaryOp)) + and isinstance(node.right, Constant) + and type(left := _convert_literal(node.left)) in (int, float) + and type(right := _convert_literal(node.right)) is complex + ): + if isinstance(node.op, Add): + return left + right + else: + return left - right + msg = "malformed node or string" + if lno := getattr(node, 'lineno', None): + msg += f' on line {lno}' + raise ValueError(msg + f': {node!r}') def dump( @@ -147,18 +155,22 @@ def _format(node, level=0): if value is None and getattr(cls, name, ...) is None: keywords = True continue - if ( - not show_empty - and (value is None or value == []) - # Special cases: - # `Constant(value=None)` and `MatchSingleton(value=None)` - and not isinstance(node, (Constant, MatchSingleton)) - ): - args_buffer.append(repr(value)) - continue - elif not keywords: - args.extend(args_buffer) - args_buffer = [] + if not show_empty: + if value == []: + field_type = cls._field_types.get(name, object) + if getattr(field_type, '__origin__', ...) is list: + if not keywords: + args_buffer.append(repr(value)) + continue + elif isinstance(value, Load): + field_type = cls._field_types.get(name, object) + if field_type is expr_context: + if not keywords: + args_buffer.append(repr(value)) + continue + if not keywords: + args.extend(args_buffer) + args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 4be7112fa01..32a5dbae03a 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -51,15 +51,24 @@ def __getattr__(name: str): import warnings - deprecated = { - "AbstractEventLoopPolicy", - "DefaultEventLoopPolicy", - "WindowsSelectorEventLoopPolicy", - "WindowsProactorEventLoopPolicy", - } - if name in deprecated: - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - # deprecated things have underscores in front of them - return globals()["_" + name] + match name: + case "AbstractEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return events._AbstractEventLoopPolicy + case "DefaultEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + if sys.platform == 'win32': + return windows_events._DefaultEventLoopPolicy + return unix_events._DefaultEventLoopPolicy + case "WindowsSelectorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsSelectorEventLoopPolicy + # Else fall through to the AttributeError below. + case "WindowsProactorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsProactorEventLoopPolicy + # Else fall through to the AttributeError below. raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 21ca5c5f62a..d078ebfa4ce 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -64,7 +64,7 @@ def callback(): except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback, context=self.context) + self.loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() @@ -74,7 +74,8 @@ def callback(): return except BaseException: if keyboard_interrupted: - self.write("\nKeyboardInterrupt\n") + if not CAN_USE_PYREPL: + self.write("\nKeyboardInterrupt\n") else: self.showtraceback() return self.STATEMENT_FAILED @@ -106,13 +107,17 @@ def run(self): if CAN_USE_PYREPL: theme = get_theme().syntax ps1 = f"{theme.prompt}{ps1}{theme.reset}" - console.write(f"{ps1}import asyncio\n") + import_line = f'{theme.keyword}import{theme.reset} asyncio' + else: + import_line = "import asyncio" + console.write(f"{ps1}{import_line}\n") if CAN_USE_PYREPL: from _pyrepl.simple_interact import ( run_multiline_interactive_console, ) try: + sys.ps1 = ps1 run_multiline_interactive_console(console) except SystemExit: # expected via the `exit` and `quit` commands diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 04fb961e998..8cbb71f7085 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -636,7 +636,7 @@ def _check_running(self): def _run_forever_setup(self): """Prepare the run loop to process events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop setup logic. """ @@ -656,7 +656,7 @@ def _run_forever_setup(self): def _run_forever_cleanup(self): """Clean up after an event loop finishes the looping over events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop cleanup logic. """ @@ -1016,38 +1016,43 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): family, type_, proto, _, address = addr_info sock = None try: - sock = socket.socket(family=family, type=type_, proto=proto) - sock.setblocking(False) - if local_addr_infos is not None: - for lfamily, _, _, _, laddr in local_addr_infos: - # skip local addresses of different family - if lfamily != family: - continue - try: - sock.bind(laddr) - break - except OSError as exc: - msg = ( - f'error while attempting to bind on ' - f'address {laddr!r}: {str(exc).lower()}' - ) - exc = OSError(exc.errno, msg) - my_exceptions.append(exc) - else: # all bind attempts failed - if my_exceptions: - raise my_exceptions.pop() - else: - raise OSError(f"no matching local address with {family=} found") - await self.sock_connect(sock, address) - return sock - except OSError as exc: - my_exceptions.append(exc) - if sock is not None: - sock.close() - raise + try: + sock = socket.socket(family=family, type=type_, proto=proto) + sock.setblocking(False) + if local_addr_infos is not None: + for lfamily, _, _, _, laddr in local_addr_infos: + # skip local addresses of different family + if lfamily != family: + continue + try: + sock.bind(laddr) + break + except OSError as exc: + msg = ( + f'error while attempting to bind on ' + f'address {laddr!r}: {str(exc).lower()}' + ) + exc = OSError(exc.errno, msg) + my_exceptions.append(exc) + else: # all bind attempts failed + if my_exceptions: + raise my_exceptions.pop() + else: + raise OSError(f"no matching local address with {family=} found") + await self.sock_connect(sock, address) + return sock + except OSError as exc: + my_exceptions.append(exc) + raise except: if sock is not None: - sock.close() + try: + sock.close() + except OSError: + # An error when closing a newly created socket is + # not important, but it can overwrite more important + # non-OSError error. So ignore it. + pass raise finally: exceptions = my_exceptions = None @@ -1161,7 +1166,7 @@ async def create_connection( raise ExceptionGroup("create_connection failed", exceptions) if len(exceptions) == 1: raise exceptions[0] - else: + elif exceptions: # If they all have the same str(), raise one. model = str(exceptions[0]) if all(str(exc) == model for exc in exceptions): @@ -1170,6 +1175,9 @@ async def create_connection( # the various error messages. raise OSError('Multiple exceptions: {}'.format( ', '.join(str(exc) for exc in exceptions))) + else: + # No exceptions were collected, raise a timeout error + raise TimeoutError('create_connection failed') finally: exceptions = None diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 9c2ba679ce2..321a4e5d5d1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -26,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -104,7 +105,12 @@ def close(self): for proto in self._pipes.values(): if proto is None: continue - proto.pipe.close() + # See gh-114177 + # skip closing the pipe if loop is already closed + # this can happen e.g. when loop is closed immediately after + # process is killed + if self._loop and not self._loop.is_closed(): + proto.pipe.close() if (self._proc is not None and # has the child process finished? @@ -208,6 +214,7 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -251,6 +258,15 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 2913f901dca..a7fb55982ab 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,14 +5,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - "_AbstractEventLoopPolicy", "AbstractEventLoop", "AbstractServer", "Handle", "TimerHandle", - "_get_event_loop_policy", "get_event_loop_policy", - "_set_event_loop_policy", "set_event_loop_policy", "get_event_loop", "set_event_loop", @@ -791,7 +788,10 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import _DefaultEventLoopPolicy + if sys.platform == 'win32': + from .windows_events import _DefaultEventLoopPolicy + else: + from .unix_events import _DefaultEventLoopPolicy _event_loop_policy = _DefaultEventLoopPolicy() diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index d1df6707302..29652295218 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -351,22 +351,19 @@ def _set_concurrent_future_state(concurrent, source): def _copy_future_state(source, dest): """Internal helper to copy state from another Future. - The other Future may be a concurrent.futures.Future. + The other Future must be a concurrent.futures.Future. """ - assert source.done() if dest.cancelled(): return assert not dest.done() - if source.cancelled(): + done, cancelled, result, exception = source._get_snapshot() + assert done + if cancelled: dest.cancel() + elif exception is not None: + dest.set_exception(_convert_future_exc(exception)) else: - exception = source.exception() - if exception is not None: - dest.set_exception(_convert_future_exc(exception)) - else: - result = source.result() - dest.set_result(result) - + dest.set_result(result) def _chain_future(source, destination): """Chain two futures so that when one completes, so does the other. @@ -392,7 +389,7 @@ def _set_state(future, other): def _call_check_cancel(destination): if destination.cancelled(): - if source_loop is None or source_loop is dest_loop: + if source_loop is None or source_loop is events._get_running_loop(): source.cancel() else: source_loop.call_soon_threadsafe(source.cancel) @@ -401,7 +398,7 @@ def _call_set_state(source): if (destination.cancelled() and dest_loop is not None and dest_loop.is_closed()): return - if dest_loop is None or dest_loop is source_loop: + if dest_loop is None or dest_loop is events._get_running_loop(): _set_state(destination, source) else: if dest_loop.is_closed(): diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index d8df7c9919a..b5bfeb1630a 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -1,6 +1,7 @@ """Introspection utils for tasks call graphs.""" import dataclasses +import io import sys import types @@ -16,9 +17,6 @@ 'FutureCallGraph', ) -if False: # for type checkers - from typing import TextIO - # Sadly, we can't re-use the traceback module's datastructures as those # are tailored for error reporting, whereas we need to represent an # async call graph. @@ -270,7 +268,7 @@ def print_call_graph( future: futures.Future | None = None, /, *, - file: TextIO | None = None, + file: io.Writer[str] | None = None, depth: int = 1, limit: int | None = None, ) -> None: diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 7eb55bd63dd..f404273c3ae 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -460,6 +460,8 @@ def _pipe_closed(self, fut): class _ProactorDatagramTransport(_ProactorBasePipeTransport, transports.DatagramTransport): max_size = 256 * 1024 + _header_size = 8 + def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): self._address = address @@ -499,7 +501,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + 8 # include header bytes + self._buffer_size += len(data) + self._header_size if self._write_fut is None: # No current write operations are active, kick one off @@ -526,7 +528,7 @@ def _loop_writing(self, fut=None): return data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size if self._address is not None: self._write_fut = self._loop._proactor.send(self._sock, data) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 2f3865114a8..084fccaaff2 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -227,9 +227,6 @@ def task_done(self): been processed (meaning that a task_done() call was received for every item that had been put() into the queue). - shutdown(immediate=True) calls task_done() for each remaining item in - the queue. - Raises ValueError if called more times than there were items placed in the queue. """ @@ -256,9 +253,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', a task is marked as done for each item remaining in - the queue, which may unblock callers of join(). + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. """ self._is_shutdown = True if immediate: diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 22147451fa7..ff7e16df3c6 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -173,7 +173,7 @@ def _accept_connection( # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. # See https://bugs.python.org/issue27906 for more details. - for _ in range(backlog): + for _ in range(backlog + 1): try: conn, addr = sock.accept() if self._debug: @@ -1050,8 +1050,8 @@ def _read_ready__on_eof(self): def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError(f'data argument must be a bytes-like object, ' - f'not {type(data).__name__!r}') + raise TypeError(f'data argument must be a bytes, bytearray, or memoryview ' + f'object, not {type(data).__name__!r}') if self._eof: raise RuntimeError('Cannot call write() after write_eof()') if self._empty_waiter is not None: @@ -1174,6 +1174,13 @@ def writelines(self, list_of_data): raise RuntimeError('unable to writelines; sendfile is in progress') if not list_of_data: return + + if self._conn_lost: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.send() raised exception.') + self._conn_lost += 1 + return + self._buffer.extend([memoryview(data) for data in list_of_data]) self._write_ready() # If the entire buffer couldn't be written, register a write handler @@ -1212,6 +1219,7 @@ def close(self): class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): _buffer_factory = collections.deque + _header_size = 8 def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): @@ -1285,13 +1293,13 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + 8 # include header bytes + self._buffer_size += len(data) + self._header_size self._maybe_pause_protocol() def _sendto_ready(self): while self._buffer: data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size try: if self._extra['peername']: self._sock.send(data) @@ -1299,7 +1307,7 @@ def _sendto_ready(self): self._sock.sendto(data, addr) except (BlockingIOError, InterruptedError): self._buffer.appendleft((data, addr)) # Try again later. - self._buffer_size += len(data) + self._buffer_size += len(data) + self._header_size break except OSError as exc: self._protocol.error_received(exc) diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 2ad65d8648e..845aed4c6a3 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -62,7 +62,6 @@ async def staggered_race(coro_fns, delay, *, loop=None): coroutine's entry is ``None``. """ - # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 64aac4cc50d..d2db1a930c2 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -214,7 +214,6 @@ def _stream_reader(self): return self._stream_reader_wr() def _replace_transport(self, transport): - loop = self._loop self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None @@ -271,7 +270,6 @@ def connection_lost(self, exc): self._closed.set_exception(exc) super().connection_lost(exc) self._stream_reader_wr = None - self._stream_writer = None self._task = None self._transport = None @@ -669,8 +667,7 @@ async def readuntil(self, separator=b'\n'): # adds data which makes separator be found. That's why we check for # EOF *after* inspecting the buffer. if self._eof: - chunk = bytes(self._buffer) - self._buffer.clear() + chunk = self._buffer.take_bytes() raise exceptions.IncompleteReadError(chunk, None) # _wait_for_data() will resume reading if stream was paused. @@ -680,10 +677,9 @@ async def readuntil(self, separator=b'\n'): raise exceptions.LimitOverrunError( 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:match_end] - del self._buffer[:match_end] + chunk = self._buffer.take_bytes(match_end) self._maybe_resume_transport() - return bytes(chunk) + return chunk async def read(self, n=-1): """Read up to `n` bytes from the stream. @@ -718,20 +714,16 @@ async def read(self, n=-1): # collect everything in self._buffer, but that would # deadlock if the subprocess sends more than self.limit # bytes. So just call self.read(self._limit) until EOF. - blocks = [] - while True: - block = await self.read(self._limit) - if not block: - break - blocks.append(block) - return b''.join(blocks) + joined = bytearray() + while block := await self.read(self._limit): + joined += block + return joined.take_bytes() if not self._buffer and not self._eof: await self._wait_for_data('read') # This will work right even if buffer is less than n bytes - data = bytes(memoryview(self._buffer)[:n]) - del self._buffer[:n] + data = self._buffer.take_bytes(min(len(self._buffer), n)) self._maybe_resume_transport() return data @@ -762,18 +754,12 @@ async def readexactly(self, n): while len(self._buffer) < n: if self._eof: - incomplete = bytes(self._buffer) - self._buffer.clear() + incomplete = self._buffer.take_bytes() raise exceptions.IncompleteReadError(incomplete, n) await self._wait_for_data('readexactly') - if len(self._buffer) == n: - data = bytes(self._buffer) - self._buffer.clear() - else: - data = bytes(memoryview(self._buffer)[:n]) - del self._buffer[:n] + data = self._buffer.take_bytes(n) self._maybe_resume_transport() return data diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 888615f8e5e..fbd5c39a7c5 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -908,6 +908,25 @@ def _done_callback(fut, cur_task=cur_task): return outer +def _log_on_exception(fut): + if fut.cancelled(): + return + + exc = fut.exception() + if exc is None: + return + + context = { + 'message': + f'{exc.__class__.__name__} exception in shielded future', + 'exception': exc, + 'future': fut, + } + if fut._source_traceback: + context['source_traceback'] = fut._source_traceback + fut._loop.call_exception_handler(context) + + def shield(arg): """Wait for a future, shielding it from cancellation. @@ -953,14 +972,11 @@ def shield(arg): else: cur_task = None - def _inner_done_callback(inner, cur_task=cur_task): - if cur_task is not None: - futures.future_discard_from_awaited_by(inner, cur_task) + def _clear_awaited_by_callback(inner): + futures.future_discard_from_awaited_by(inner, cur_task) + def _inner_done_callback(inner): if outer.cancelled(): - if not inner.cancelled(): - # Mark inner's result as retrieved. - inner.exception() return if inner.cancelled(): @@ -972,10 +988,16 @@ def _inner_done_callback(inner, cur_task=cur_task): else: outer.set_result(inner.result()) - def _outer_done_callback(outer): if not inner.done(): inner.remove_done_callback(_inner_done_callback) + # Keep only one callback to log on cancel + inner.remove_done_callback(_log_on_exception) + inner.add_done_callback(_log_on_exception) + + if cur_task is not None: + inner.add_done_callback(_clear_awaited_by_callback) + inner.add_done_callback(_inner_done_callback) outer.add_done_callback(_outer_done_callback) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index bf1cb5e64cb..1d463ea09ba 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,71 +1,99 @@ """Tools to analyze tasks running in asyncio programs.""" -from dataclasses import dataclass from collections import defaultdict from itertools import count from enum import Enum import sys -from _remote_debugging import get_all_awaited_by - +from _remote_debugging import RemoteUnwinder, FrameInfo class NodeType(Enum): COROUTINE = 1 TASK = 2 -@dataclass(frozen=True) class CycleFoundException(Exception): """Raised when there is a cycle when drawing the call tree.""" - cycles: list[list[int]] - id2name: dict[int, str] + def __init__( + self, + cycles: list[list[int]], + id2name: dict[int, str], + ) -> None: + super().__init__(cycles, id2name) + self.cycles = cycles + self.id2name = id2name + # ─── indexing helpers ─────────────────────────────────────────── -def _format_stack_entry(elem: tuple[str, str, int] | str) -> str: - if isinstance(elem, tuple): - fqname, path, line_no = elem - return f"{fqname} {path}:{line_no}" - +def _format_stack_entry(elem: str|FrameInfo) -> str: + if not isinstance(elem, str): + if elem.lineno == 0 and elem.filename == "": + return f"{elem.funcname}" + else: + return f"{elem.funcname} {elem.filename}:{elem.lineno}" return elem def _index(result): - id2name, awaits = {}, [] - for _thr_id, tasks in result: - for tid, tname, awaited in tasks: - id2name[tid] = tname - for stack, parent_id in awaited: - stack = [_format_stack_entry(elem) for elem in stack] - awaits.append((parent_id, stack, tid)) - return id2name, awaits + id2name, awaits, task_stacks = {}, [], {} + for awaited_info in result: + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_name = task_info.task_name + id2name[task_id] = task_name + + # Store the internal coroutine stack for this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + call_stack = coro_info.call_stack + internal_stack = [_format_stack_entry(frame) for frame in call_stack] + task_stacks[task_id] = internal_stack + + # Add the awaited_by relationships (external dependencies) + if task_info.awaited_by: + for coro_info in task_info.awaited_by: + call_stack = coro_info.call_stack + parent_task_id = coro_info.task_name + stack = [_format_stack_entry(frame) for frame in call_stack] + awaits.append((parent_task_id, stack, task_id)) + return id2name, awaits, task_stacks -def _build_tree(id2name, awaits): +def _build_tree(id2name, awaits, task_stacks): id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()} children = defaultdict(list) - cor_names = defaultdict(dict) # (parent) -> {frame: node} - cor_id_seq = count(1) + cor_nodes = defaultdict(dict) # Maps parent -> {frame_name: node_key} + next_cor_id = count(1) - def _cor_node(parent_key, frame_name): - """Return an existing or new (NodeType.COROUTINE, …) node under *parent_key*.""" - bucket = cor_names[parent_key] - if frame_name in bucket: - return bucket[frame_name] - node_key = (NodeType.COROUTINE, f"c{next(cor_id_seq)}") - id2label[node_key] = frame_name - children[parent_key].append(node_key) - bucket[frame_name] = node_key + def get_or_create_cor_node(parent, frame): + """Get existing coroutine node or create new one under parent""" + if frame in cor_nodes[parent]: + return cor_nodes[parent][frame] + + node_key = (NodeType.COROUTINE, f"c{next(next_cor_id)}") + id2label[node_key] = frame + children[parent].append(node_key) + cor_nodes[parent][frame] = node_key return node_key - # lay down parent ➜ …frames… ➜ child paths + # Build task dependency tree with coroutine frames for parent_id, stack, child_id in awaits: cur = (NodeType.TASK, parent_id) - for frame in reversed(stack): # outer-most → inner-most - cur = _cor_node(cur, frame) + for frame in reversed(stack): + cur = get_or_create_cor_node(cur, frame) + child_key = (NodeType.TASK, child_id) if child_key not in children[cur]: children[cur].append(child_key) + # Add coroutine stacks for leaf tasks + awaiting_tasks = {parent_id for parent_id, _, _ in awaits} + for task_id in id2name: + if task_id not in awaiting_tasks and task_id in task_stacks: + cur = (NodeType.TASK, task_id) + for frame in reversed(task_stacks[task_id]): + cur = get_or_create_cor_node(cur, frame) + return id2label, children @@ -112,6 +140,11 @@ def dfs(v): # ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_all_awaited_by() + + def build_async_tree(result, task_emoji="(T)", cor_emoji=""): """ Build a list of strings for pretty-print an async call tree. @@ -119,12 +152,12 @@ def build_async_tree(result, task_emoji="(T)", cor_emoji=""): The call tree is produced by `get_all_async_stacks()`, prefixing tasks with `task_emoji` and coroutine frames with `cor_emoji`. """ - id2name, awaits = _index(result) + id2name, awaits, task_stacks = _index(result) g = _task_graph(awaits) cycles = _find_cycles(g) if cycles: raise CycleFoundException(cycles, id2name) - labels, children = _build_tree(id2name, awaits) + labels, children = _build_tree(id2name, awaits, task_stacks) def pretty(node): flag = task_emoji if node[0] == NodeType.TASK else cor_emoji @@ -144,35 +177,40 @@ def render(node, prefix="", last=True, buf=None): def build_task_table(result): - id2name, awaits = _index(result) + id2name, _, _ = _index(result) table = [] - for tid, tasks in result: - for task_id, task_name, awaited in tasks: - if not awaited: - table.append( - [ - tid, - hex(task_id), - task_name, - "", - "", - "0x0" - ] - ) - for stack, awaiter_id in awaited: - stack = [elem[0] if isinstance(elem, tuple) else elem for elem in stack] - coroutine_chain = " -> ".join(stack) - awaiter_name = id2name.get(awaiter_id, "Unknown") - table.append( - [ - tid, - hex(task_id), - task_name, - coroutine_chain, - awaiter_name, - hex(awaiter_id), - ] - ) + + for awaited_info in result: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + # Get task info + task_id = task_info.task_id + task_name = task_info.task_name + + # Build coroutine stack string + frames = [frame for coro in task_info.coroutine_stack + for frame in coro.call_stack] + coro_stack = " -> ".join(_format_stack_entry(x).split(" ")[0] + for x in frames) + + # Handle tasks with no awaiters + if not task_info.awaited_by: + table.append([thread_id, hex(task_id), task_name, coro_stack, + "", "", "0x0"]) + continue + + # Handle tasks with awaiters + for coro_info in task_info.awaited_by: + parent_id = coro_info.task_name + awaiter_frames = [_format_stack_entry(x).split(" ")[0] + for x in coro_info.call_stack] + awaiter_chain = " -> ".join(awaiter_frames) + awaiter_name = id2name.get(parent_id, "Unknown") + parent_id_str = (hex(parent_id) if isinstance(parent_id, int) + else str(parent_id)) + + table.append([thread_id, hex(task_id), task_name, coro_stack, + awaiter_chain, awaiter_name, parent_id_str]) return table @@ -184,6 +222,20 @@ def _print_cycle_exception(exception: CycleFoundException): print(f"cycle: {inames}", file=sys.stderr) +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + ) + sys.exit(1) + + def _get_awaited_by_tasks(pid: int) -> list: try: return get_all_awaited_by(pid) @@ -192,6 +244,8 @@ def _get_awaited_by_tasks(pid: int) -> list: e = e.__context__ print(f"Error retrieving tasks: {e}") sys.exit(1) + except PermissionError as e: + exit_with_permission_help_text() def display_awaited_by_tasks_table(pid: int) -> None: @@ -201,11 +255,11 @@ def display_awaited_by_tasks_table(pid: int) -> None: table = build_task_table(tasks) # Print the table in a simple tabular format print( - f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine chain':<50} {'awaiter name':<20} {'awaiter id':<15}" + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}" ) - print("-" * 135) + print("-" * 180) for row in table: - print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<20} {row[5]:<15}") + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") def display_awaited_by_tasks_tree(pid: int) -> None: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index f69c6a64c39..1c1458127db 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,7 +28,6 @@ __all__ = ( 'SelectorEventLoop', - '_DefaultEventLoopPolicy', 'EventLoop', ) diff --git a/Lib/base64.py b/Lib/base64.py index 5d78cc09f40..341bf8eaf18 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -193,7 +193,7 @@ def _b32encode(alphabet, s): encoded[-3:] = b'===' elif leftover == 4: encoded[-1:] = b'=' - return bytes(encoded) + return encoded.take_bytes() def _b32decode(alphabet, s, casefold=False, map01=None): # Delay the initialization of the table to not waste memory @@ -238,7 +238,7 @@ def _b32decode(alphabet, s, casefold=False, map01=None): last = acc.to_bytes(5) # big endian leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 decoded[-5:] = last[:leftover] - return bytes(decoded) + return decoded.take_bytes() def b32encode(s): @@ -462,9 +462,12 @@ def b85decode(b): # Delay the initialization of tables to not waste memory # if the function is never called if _b85dec is None: - _b85dec = [None] * 256 + # we don't assign to _b85dec directly to avoid issues when + # multiple threads call this function simultaneously + b85dec_tmp = [None] * 256 for i, c in enumerate(_b85alphabet): - _b85dec[c] = i + b85dec_tmp[c] = i + _b85dec = b85dec_tmp b = _bytes_from_decode_data(b) padding = (-len(b)) % 5 @@ -601,7 +604,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Lib/bdb.py b/Lib/bdb.py index 4290ef22302..50cf2b3f5b3 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -199,6 +199,8 @@ def __init__(self, skip=None, backend='settrace'): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self.code_linenos = weakref.WeakKeyDictionary() self.backend = backend if backend == 'monitoring': @@ -267,12 +269,9 @@ def trace_dispatch(self, frame, event, arg): is entered. return: A function or other code block is about to return. exception: An exception has occurred. - c_call: A C function is about to be called. - c_return: A C function has returned. - c_exception: A C function has raised an exception. - For the Python events, specialized functions (see the dispatch_*() - methods) are called. For the C events, no action is taken. + For all the events, specialized functions (see the dispatch_*() + methods) are called. The arg parameter depends on the previous event. """ @@ -288,12 +287,6 @@ def trace_dispatch(self, frame, event, arg): return self.dispatch_return(frame, arg) if event == 'exception': return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': - return self.trace_dispatch if event == 'opcode': return self.dispatch_opcode(frame, arg) print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) @@ -306,7 +299,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) self.restart_events() if self.quitting: raise BdbQuit @@ -535,7 +533,8 @@ def _set_trace_opcodes(self, trace_opcodes): if self.monitoring_tracer: self.monitoring_tracer.update_local_events() - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -548,6 +547,10 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno self._set_trace_opcodes(opcode) def _set_caller_tracefunc(self, current_frame): @@ -573,7 +576,9 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) def set_stepinstr(self): """Stop before the next instruction.""" @@ -581,7 +586,7 @@ def set_stepinstr(self): def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 770d26f7962..cc6255f61ae 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -1,205 +1,13 @@ -"""Python interface for the 'lsprof' profiler. - Compatible with the 'profile' module. +"""Compatibility wrapper for cProfile module. + +This module maintains backward compatibility by importing from the new +profiling.tracing module. """ +from profiling.tracing import run, runctx, Profile + __all__ = ["run", "runctx", "Profile"] -import _lsprof -import importlib.machinery -import importlib.util -import io -import profile as _pyprofile - -# ____________________________________________________________ -# Simple interface - -def run(statement, filename=None, sort=-1): - return _pyprofile._Utils(Profile).run(statement, filename, sort) - -def runctx(statement, globals, locals, filename=None, sort=-1): - return _pyprofile._Utils(Profile).runctx(statement, globals, locals, - filename, sort) - -run.__doc__ = _pyprofile.run.__doc__ -runctx.__doc__ = _pyprofile.runctx.__doc__ - -# ____________________________________________________________ - -class Profile(_lsprof.Profiler): - """Profile(timer=None, timeunit=None, subcalls=True, builtins=True) - - Builds a profiler object using the specified timer function. - The default timer is a fast built-in one based on real time. - For custom timer functions returning integers, timeunit can - be a float specifying a scale (i.e. how long each integer unit - is, in seconds). - """ - - # Most of the functionality is in the base class. - # This subclass only adds convenient and backward-compatible methods. - - def print_stats(self, sort=-1): - import pstats - if not isinstance(sort, tuple): - sort = (sort,) - pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() - - def dump_stats(self, file): - import marshal - with open(file, 'wb') as f: - self.create_stats() - marshal.dump(self.stats, f) - - def create_stats(self): - self.disable() - self.snapshot_stats() - - def snapshot_stats(self): - entries = self.getstats() - self.stats = {} - callersdicts = {} - # call information - for entry in entries: - func = label(entry.code) - nc = entry.callcount # ncalls column of pstats (before '/') - cc = nc - entry.reccallcount # ncalls column of pstats (after '/') - tt = entry.inlinetime # tottime column of pstats - ct = entry.totaltime # cumtime column of pstats - callers = {} - callersdicts[id(entry.code)] = callers - self.stats[func] = cc, nc, tt, ct, callers - # subcall information - for entry in entries: - if entry.calls: - func = label(entry.code) - for subentry in entry.calls: - try: - callers = callersdicts[id(subentry.code)] - except KeyError: - continue - nc = subentry.callcount - cc = nc - subentry.reccallcount - tt = subentry.inlinetime - ct = subentry.totaltime - if func in callers: - prev = callers[func] - nc += prev[0] - cc += prev[1] - tt += prev[2] - ct += prev[3] - callers[func] = nc, cc, tt, ct - - # The following two methods can be called by clients to use - # a profiler to profile a statement, given as a string. - - def run(self, cmd): - import __main__ - dict = __main__.__dict__ - return self.runctx(cmd, dict, dict) - - def runctx(self, cmd, globals, locals): - self.enable() - try: - exec(cmd, globals, locals) - finally: - self.disable() - return self - - # This method is more useful to profile a single function call. - def runcall(self, func, /, *args, **kw): - self.enable() - try: - return func(*args, **kw) - finally: - self.disable() - - def __enter__(self): - self.enable() - return self - - def __exit__(self, *exc_info): - self.disable() - -# ____________________________________________________________ - -def label(code): - if isinstance(code, str): - return ('~', 0, code) # built-in functions ('~' sorts at the end) - else: - return (code.co_filename, code.co_firstlineno, code.co_name) - -# ____________________________________________________________ - -def main(): - import os - import sys - import runpy - import pstats - from optparse import OptionParser - usage = "cProfile.py [-o output_file_path] [-s sort] [-m module | scriptfile] [arg] ..." - parser = OptionParser(usage=usage) - parser.allow_interspersed_args = False - parser.add_option('-o', '--outfile', dest="outfile", - help="Save stats to ", default=None) - parser.add_option('-s', '--sort', dest="sort", - help="Sort order when printing to stdout, based on pstats.Stats class", - default=2, - choices=sorted(pstats.Stats.sort_arg_dict_default)) - parser.add_option('-m', dest="module", action="store_true", - help="Profile a library module", default=False) - - if not sys.argv[1:]: - parser.print_usage() - sys.exit(2) - - (options, args) = parser.parse_args() - sys.argv[:] = args - - # The script that we're profiling may chdir, so capture the absolute path - # to the output file at startup. - if options.outfile is not None: - options.outfile = os.path.abspath(options.outfile) - - if len(args) > 0: - if options.module: - code = "run_module(modname, run_name='__main__')" - globs = { - 'run_module': runpy.run_module, - 'modname': args[0] - } - else: - progname = args[0] - sys.path.insert(0, os.path.dirname(progname)) - with io.open_code(progname) as fp: - code = compile(fp.read(), progname, 'exec') - spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, - origin=progname) - module = importlib.util.module_from_spec(spec) - # Set __main__ so that importing __main__ in the profiled code will - # return the same namespace that the code is executing under. - sys.modules['__main__'] = module - # Ensure that we're using the same __dict__ instance as the module - # for the global variables so that updates to globals are reflected - # in the module's namespace. - globs = module.__dict__ - globs.update({ - '__spec__': spec, - '__file__': spec.origin, - '__name__': spec.name, - '__package__': None, - '__cached__': None, - }) - - try: - runctx(code, globs, None, options.outfile, options.sort) - except BrokenPipeError as exc: - # Prevent "Exception ignored" during interpreter shutdown. - sys.stdout = None - sys.exit(exc.errno) - else: - parser.print_usage() - return parser - -# When invoked as main program, invoke the profiler on a script -if __name__ == '__main__': +if __name__ == "__main__": + from profiling.tracing.__main__ import main main() diff --git a/Lib/calendar.py b/Lib/calendar.py index 18f76d52ff8..d80c3fd9524 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -14,8 +14,9 @@ __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", - "timegm", "month_name", "month_abbr", "day_name", "day_abbr", - "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", + "timegm", "month_name", "month_abbr", "standalone_month_name", + "standalone_month_abbr", "day_name", "day_abbr", "Calendar", + "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", "LocaleHTMLCalendar", "weekheader", "Day", "Month", "JANUARY", "FEBRUARY", "MARCH", "APRIL", "MAY", "JUNE", "JULY", @@ -139,6 +140,24 @@ def __len__(self): month_name = _localized_month('%B') month_abbr = _localized_month('%b') +# On platforms that support the %OB and %Ob specifiers, they are used +# to get the standalone form of the month name. This is required for +# some languages such as Greek, Slavic, and Baltic languages. +try: + standalone_month_name = _localized_month('%OB') + standalone_month_abbr = _localized_month('%Ob') +except ValueError: + standalone_month_name = month_name + standalone_month_abbr = month_abbr +else: + # Some systems that do not support '%OB' will keep it as-is (i.e., + # we get [..., '%OB', '%OB', '%OB']), so for non-distinct names, + # we fall back to month_name/month_abbr. + if len(set(standalone_month_name)) != len(set(month_name)): + standalone_month_name = month_name + if len(set(standalone_month_abbr)) != len(set(month_abbr)): + standalone_month_abbr = month_abbr + def isleap(year): """Return True for leap years, False for non-leap years.""" @@ -359,7 +378,7 @@ def formatweekday(self, day, width): """ Returns a formatted week day name. """ - if width >= 9: + if width >= max(map(len, day_name)): names = day_name else: names = day_abbr @@ -377,7 +396,7 @@ def formatmonthname(self, theyear, themonth, width, withyear=True): """ _validate_month(themonth) - s = month_name[themonth] + s = standalone_month_name[themonth] if withyear: s = "%s %r" % (s, theyear) return s.center(width) @@ -479,30 +498,29 @@ def formatday(self, day, weekday): """ if day == 0: # day outside month - return '		  %d{day}
%s{day_abbr[day]}
%s
{s}
' % ( - self.cssclass_month)) + a(f'
') a('\n') a(self.formatmonthname(theyear, themonth, withyear=withyear)) a('\n') @@ -543,11 +559,9 @@ def formatyear(self, theyear, width=3): v = [] a = v.append width = max(width, 1) - a('
' % - self.cssclass_year) + a(f'
') a('\n') - a('' % ( - width, self.cssclass_year_head, theyear)) + a(f'') for i in range(JANUARY, JANUARY+12, width): # months in this row months = range(i, min(i+width, 13)) @@ -560,28 +574,47 @@ def formatyear(self, theyear, width=3): a('
%s
{theyear}
') return ''.join(v) + def _format_html_page(self, theyear, content, css, encoding): + """ + Return a complete HTML page with the given content. + """ + if encoding is None: + encoding = 'utf-8' + v = [] + a = v.append + a('\n') + a('\n') + a('\n') + a(f'\n') + a('\n') + a(f'Calendar for {theyear}\n') + a('\n') + if css is not None: + a(f'\n') + a('\n') + a('\n') + a(content) + a('\n') + a('\n') + return ''.join(v).encode(encoding, "xmlcharrefreplace") + def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None): """ Return a formatted year as a complete HTML page. """ - if encoding is None: - encoding = sys.getdefaultencoding() - v = [] - a = v.append - a('\n' % encoding) - a('\n') - a('\n') - a('\n') - a('\n' % encoding) - if css is not None: - a('\n' % css) - a('Calendar for %d\n' % theyear) - a('\n') - a('\n') - a(self.formatyear(theyear, width)) - a('\n') - a('\n') - return ''.join(v).encode(encoding, "xmlcharrefreplace") + content = self.formatyear(theyear, width) + return self._format_html_page(theyear, content, css, encoding) + + def formatmonthpage(self, theyear, themonth, width=3, css='calendar.css', encoding=None): + """ + Return a formatted month as a complete HTML page. + """ + content = self.formatmonth(theyear, themonth, width) + return self._format_html_page(theyear, content, css, encoding) class different_locale: @@ -846,7 +879,7 @@ def main(args=None): parser.add_argument( "-e", "--encoding", default=None, - help="encoding to use for output" + help="encoding to use for output (default utf-8)" ) parser.add_argument( "-t", "--type", @@ -867,7 +900,7 @@ def main(args=None): parser.add_argument( "month", nargs='?', type=int, - help="month number (1-12, text only)" + help="month number (1-12)" ) options = parser.parse_args(args) @@ -880,9 +913,6 @@ def main(args=None): today = datetime.date.today() if options.type == "html": - if options.month: - parser.error("incorrect number of arguments") - sys.exit(1) if options.locale: cal = LocaleHTMLCalendar(locale=locale) else: @@ -890,13 +920,17 @@ def main(args=None): cal.setfirstweekday(options.first_weekday) encoding = options.encoding if encoding is None: - encoding = sys.getdefaultencoding() + encoding = 'utf-8' optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write + if options.year is None: write(cal.formatyearpage(today.year, **optdict)) else: - write(cal.formatyearpage(options.year, **optdict)) + if options.month: + write(cal.formatmonthpage(options.year, options.month, **optdict)) + else: + write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: cal = _CLIDemoLocaleCalendar(highlight_day=today, locale=locale) diff --git a/Lib/code.py b/Lib/code.py index b134886dc26..f7e275d8801 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -224,7 +224,7 @@ def interact(self, banner=None, exitmsg=None): sys.ps1 = ">>> " delete_ps1_after = True try: - _ps2 = sys.ps2 + sys.ps2 delete_ps2_after = False except AttributeError: sys.ps2 = "... " diff --git a/Lib/codecs.py b/Lib/codecs.py index fc38e922257..e4a8010aba9 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -618,7 +618,7 @@ def readlines(self, sizehint=None, keepends=True): method and are included in the list entries. sizehint, if given, is ignored since there is no efficient - way to finding the true end-of-line. + way of finding the true end-of-line. """ data = self.read() @@ -709,13 +709,13 @@ def read(self, size=-1): return self.reader.read(size) - def readline(self, size=None): + def readline(self, size=None, keepends=True): - return self.reader.readline(size) + return self.reader.readline(size, keepends) - def readlines(self, sizehint=None): + def readlines(self, sizehint=None, keepends=True): - return self.reader.readlines(sizehint) + return self.reader.readlines(sizehint, keepends) def __next__(self): diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index d2ddc1cd9ec..55ffc36ea5b 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -776,23 +776,27 @@ def __repr__(self): # When the multiplicities are all zero or one, multiset operations # are guaranteed to be equivalent to the corresponding operations # for regular sets. + # # Given counter multisets such as: # cp = Counter(a=1, b=0, c=1) # cq = Counter(c=1, d=0, e=1) + # # The corresponding regular sets would be: # sp = {'a', 'c'} # sq = {'c', 'e'} + # # All of the following relations would hold: - # set(cp + cq) == sp | sq - # set(cp - cq) == sp - sq - # set(cp | cq) == sp | sq - # set(cp & cq) == sp & sq # (cp == cq) == (sp == sq) # (cp != cq) == (sp != sq) # (cp <= cq) == (sp <= sq) # (cp < cq) == (sp < sq) # (cp >= cq) == (sp >= sq) # (cp > cq) == (sp > sq) + # set(cp + cq) == sp | sq + # set(cp - cq) == sp - sq + # set(cp | cq) == sp | sq + # set(cp & cq) == sp & sq + # set(cp ^ cq) == sp ^ sq def __eq__(self, other): 'True if all counts agree. Missing counts are treated as zero.' @@ -905,6 +909,33 @@ def __and__(self, other): result[elem] = newcount return result + def __xor__(self, other): + '''Symmetric difference. Absolute value of count differences. + + The symmetric difference p ^ q is equivalent to: + + (p - q) | (q - p). + + For each element, symmetric difference gives the same result as: + + max(p[elem], q[elem]) - min(p[elem], q[elem]) + + >>> Counter(a=5, b=3, c=2, d=2) ^ Counter(a=1, b=3, c=5, e=1) + Counter({'a': 4, 'c': 3, 'd': 2, 'e': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + newcount = abs(count - other[elem]) + if newcount: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count: + result[elem] = abs(count) + return result + def __pos__(self): 'Adds an empty counter, effectively stripping negative and zero counts' result = Counter() @@ -987,6 +1018,22 @@ def __iand__(self, other): self[elem] = other_count return self._keep_positive() + def __ixor__(self, other): + '''Inplace symmetric difference. Absolute value of count differences. + + >>> c = Counter(a=5, b=3, c=2, d=2) + >>> c ^= Counter(a=1, b=3, c=5, e=1) + >>> c + Counter({'a': 4, 'c': 3, 'd': 2, 'e': 1}) + + ''' + for elem, count in self.items(): + self[elem] = abs(count - other[elem]) + for elem, count in other.items(): + if elem not in self: + self[elem] = abs(count) + return self._keep_positive() + ######################################################################## ### ChainMap @@ -1495,6 +1542,8 @@ def format_map(self, mapping): return self.data.format_map(mapping) def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.index(sub, start, end) def isalpha(self): @@ -1563,6 +1612,8 @@ def rfind(self, sub, start=0, end=_sys.maxsize): return self.data.rfind(sub, start, end) def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.rindex(sub, start, end) def rjust(self, width, *args): diff --git a/Lib/compression/bz2/__init__.py b/Lib/compression/bz2.py similarity index 100% rename from Lib/compression/bz2/__init__.py rename to Lib/compression/bz2.py diff --git a/Lib/compression/gzip/__init__.py b/Lib/compression/gzip.py similarity index 100% rename from Lib/compression/gzip/__init__.py rename to Lib/compression/gzip.py diff --git a/Lib/compression/lzma/__init__.py b/Lib/compression/lzma.py similarity index 100% rename from Lib/compression/lzma/__init__.py rename to Lib/compression/lzma.py diff --git a/Lib/compression/zlib/__init__.py b/Lib/compression/zlib.py similarity index 100% rename from Lib/compression/zlib/__init__.py rename to Lib/compression/zlib.py diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 4f734eb07b0..84b25914b0a 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -2,41 +2,48 @@ __all__ = ( # compression.zstd - "COMPRESSION_LEVEL_DEFAULT", - "compress", - "CompressionParameter", - "decompress", - "DecompressionParameter", - "finalize_dict", - "get_frame_info", - "Strategy", - "train_dict", + 'COMPRESSION_LEVEL_DEFAULT', + 'compress', + 'CompressionParameter', + 'decompress', + 'DecompressionParameter', + 'finalize_dict', + 'get_frame_info', + 'Strategy', + 'train_dict', # compression.zstd._zstdfile - "open", - "ZstdFile", + 'open', + 'ZstdFile', # _zstd - "get_frame_size", - "zstd_version", - "zstd_version_info", - "ZstdCompressor", - "ZstdDecompressor", - "ZstdDict", - "ZstdError", + 'get_frame_size', + 'zstd_version', + 'zstd_version_info', + 'ZstdCompressor', + 'ZstdDecompressor', + 'ZstdDict', + 'ZstdError', ) import _zstd import enum -from _zstd import * +from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError, + get_frame_size, zstd_version) from compression.zstd._zstdfile import ZstdFile, open, _nbytes -COMPRESSION_LEVEL_DEFAULT = _zstd._compressionLevel_values[0] +# zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE) +zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100), + _zstd.zstd_version_number % 100) +"""Version number of the runtime zstd library as a tuple of integers.""" + +COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT """The default compression level for Zstandard, currently '3'.""" class FrameInfo: """Information about a Zstandard frame.""" + __slots__ = 'decompressed_size', 'dictionary_id' def __init__(self, decompressed_size, dictionary_id): @@ -65,7 +72,7 @@ def get_frame_info(frame_buffer): the frame may or may not need a dictionary to be decoded, and the ID of such a dictionary is not specified. """ - return FrameInfo(*_zstd._get_frame_info(frame_buffer)) + return FrameInfo(*_zstd.get_frame_info(frame_buffer)) def train_dict(samples, dict_size): @@ -85,7 +92,7 @@ def train_dict(samples, dict_size): chunk_sizes = tuple(_nbytes(sample) for sample in samples) if not chunks: raise ValueError("samples contained no data; can't train dictionary.") - dict_content = _zstd._train_dict(chunks, chunk_sizes, dict_size) + dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size) return ZstdDict(dict_content) @@ -119,13 +126,13 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): chunks = b''.join(samples) chunk_sizes = tuple(_nbytes(sample) for sample in samples) if not chunks: - raise ValueError("The samples are empty content, can't finalize the" + raise ValueError("The samples are empty content, can't finalize the " "dictionary.") - dict_content = _zstd._finalize_dict(zstd_dict.dict_content, - chunks, chunk_sizes, - dict_size, level) + dict_content = _zstd.finalize_dict(zstd_dict.dict_content, chunks, + chunk_sizes, dict_size, level) return ZstdDict(dict_content) + def compress(data, level=None, options=None, zstd_dict=None): """Return Zstandard compressed *data* as bytes. @@ -141,6 +148,7 @@ def compress(data, level=None, options=None, zstd_dict=None): comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME) + def decompress(data, zstd_dict=None, options=None): """Decompress one or more frames of Zstandard compressed *data*. @@ -156,59 +164,59 @@ def decompress(data, zstd_dict=None, options=None): decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) results.append(decomp.decompress(data)) if not decomp.eof: - raise ZstdError("Compressed data ended before the " - "end-of-stream marker was reached") + raise ZstdError('Compressed data ended before the ' + 'end-of-stream marker was reached') data = decomp.unused_data if not data: break - return b"".join(results) + return b''.join(results) class CompressionParameter(enum.IntEnum): """Compression parameters.""" - compression_level = _zstd._ZSTD_c_compressionLevel - window_log = _zstd._ZSTD_c_windowLog - hash_log = _zstd._ZSTD_c_hashLog - chain_log = _zstd._ZSTD_c_chainLog - search_log = _zstd._ZSTD_c_searchLog - min_match = _zstd._ZSTD_c_minMatch - target_length = _zstd._ZSTD_c_targetLength - strategy = _zstd._ZSTD_c_strategy + compression_level = _zstd.ZSTD_c_compressionLevel + window_log = _zstd.ZSTD_c_windowLog + hash_log = _zstd.ZSTD_c_hashLog + chain_log = _zstd.ZSTD_c_chainLog + search_log = _zstd.ZSTD_c_searchLog + min_match = _zstd.ZSTD_c_minMatch + target_length = _zstd.ZSTD_c_targetLength + strategy = _zstd.ZSTD_c_strategy - enable_long_distance_matching = _zstd._ZSTD_c_enableLongDistanceMatching - ldm_hash_log = _zstd._ZSTD_c_ldmHashLog - ldm_min_match = _zstd._ZSTD_c_ldmMinMatch - ldm_bucket_size_log = _zstd._ZSTD_c_ldmBucketSizeLog - ldm_hash_rate_log = _zstd._ZSTD_c_ldmHashRateLog + enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching + ldm_hash_log = _zstd.ZSTD_c_ldmHashLog + ldm_min_match = _zstd.ZSTD_c_ldmMinMatch + ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog + ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog - content_size_flag = _zstd._ZSTD_c_contentSizeFlag - checksum_flag = _zstd._ZSTD_c_checksumFlag - dict_id_flag = _zstd._ZSTD_c_dictIDFlag + content_size_flag = _zstd.ZSTD_c_contentSizeFlag + checksum_flag = _zstd.ZSTD_c_checksumFlag + dict_id_flag = _zstd.ZSTD_c_dictIDFlag - nb_workers = _zstd._ZSTD_c_nbWorkers - job_size = _zstd._ZSTD_c_jobSize - overlap_log = _zstd._ZSTD_c_overlapLog + nb_workers = _zstd.ZSTD_c_nbWorkers + job_size = _zstd.ZSTD_c_jobSize + overlap_log = _zstd.ZSTD_c_overlapLog def bounds(self): """Return the (lower, upper) int bounds of a compression parameter. Both the lower and upper bounds are inclusive. """ - return _zstd._get_param_bounds(self.value, is_compress=True) + return _zstd.get_param_bounds(self.value, is_compress=True) class DecompressionParameter(enum.IntEnum): """Decompression parameters.""" - window_log_max = _zstd._ZSTD_d_windowLogMax + window_log_max = _zstd.ZSTD_d_windowLogMax def bounds(self): """Return the (lower, upper) int bounds of a decompression parameter. Both the lower and upper bounds are inclusive. """ - return _zstd._get_param_bounds(self.value, is_compress=False) + return _zstd.get_param_bounds(self.value, is_compress=False) class Strategy(enum.IntEnum): @@ -219,16 +227,16 @@ class Strategy(enum.IntEnum): the numeric value might change. """ - fast = _zstd._ZSTD_fast - dfast = _zstd._ZSTD_dfast - greedy = _zstd._ZSTD_greedy - lazy = _zstd._ZSTD_lazy - lazy2 = _zstd._ZSTD_lazy2 - btlazy2 = _zstd._ZSTD_btlazy2 - btopt = _zstd._ZSTD_btopt - btultra = _zstd._ZSTD_btultra - btultra2 = _zstd._ZSTD_btultra2 + fast = _zstd.ZSTD_fast + dfast = _zstd.ZSTD_dfast + greedy = _zstd.ZSTD_greedy + lazy = _zstd.ZSTD_lazy + lazy2 = _zstd.ZSTD_lazy2 + btlazy2 = _zstd.ZSTD_btlazy2 + btopt = _zstd.ZSTD_btopt + btultra = _zstd.ZSTD_btultra + btultra2 = _zstd.ZSTD_btultra2 # Check validity of the CompressionParameter & DecompressionParameter types -_zstd._set_parameter_types(CompressionParameter, DecompressionParameter) +_zstd.set_parameter_types(CompressionParameter, DecompressionParameter) diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index fbc9e02a733..d709f5efc65 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -1,12 +1,9 @@ import io from os import PathLike -from _zstd import (ZstdCompressor, ZstdDecompressor, _ZSTD_DStreamSizes, - ZstdError) +from _zstd import ZstdCompressor, ZstdDecompressor, ZSTD_DStreamOutSize from compression._common import _streams -__all__ = ("ZstdFile", "open") - -_ZSTD_DStreamOutSize = _ZSTD_DStreamSizes[1] +__all__ = ('ZstdFile', 'open') _MODE_CLOSED = 0 _MODE_READ = 1 @@ -33,15 +30,15 @@ class ZstdFile(_streams.BaseStream): FLUSH_BLOCK = ZstdCompressor.FLUSH_BLOCK FLUSH_FRAME = ZstdCompressor.FLUSH_FRAME - def __init__(self, file, /, mode="r", *, + def __init__(self, file, /, mode='r', *, level=None, options=None, zstd_dict=None): """Open a Zstandard compressed file in binary mode. *file* can be either an file-like object, or a file name to open. - *mode* can be "r" for reading (default), "w" for (over)writing, "x" for - creating exclusively, or "a" for appending. These can equivalently be - given as "rb", "wb", "xb" and "ab" respectively. + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for + creating exclusively, or 'a' for appending. These can equivalently be + given as 'rb', 'wb', 'xb' and 'ab' respectively. *level* is an optional int specifying the compression level to use, or COMPRESSION_LEVEL_DEFAULT if not given. @@ -59,39 +56,38 @@ def __init__(self, file, /, mode="r", *, self._buffer = None if not isinstance(mode, str): - raise ValueError("mode must be a str") + raise ValueError('mode must be a str') if options is not None and not isinstance(options, dict): - raise TypeError("options must be a dict or None") - mode = mode.removesuffix("b") # handle rb, wb, xb, ab - if mode == "r": + raise TypeError('options must be a dict or None') + mode = mode.removesuffix('b') # handle rb, wb, xb, ab + if mode == 'r': if level is not None: - raise TypeError("level is illegal in read mode") + raise TypeError('level is illegal in read mode') self._mode = _MODE_READ - elif mode in {"w", "a", "x"}: + elif mode in {'w', 'a', 'x'}: if level is not None and not isinstance(level, int): - raise TypeError("level must be int or None") + raise TypeError('level must be int or None') self._mode = _MODE_WRITE self._compressor = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) self._pos = 0 else: - raise ValueError(f"Invalid mode: {mode!r}") + raise ValueError(f'Invalid mode: {mode!r}') if isinstance(file, (str, bytes, PathLike)): self._fp = io.open(file, f'{mode}b') self._close_fp = True - elif ((mode == 'r' and hasattr(file, "read")) - or (mode != 'r' and hasattr(file, "write"))): + elif ((mode == 'r' and hasattr(file, 'read')) + or (mode != 'r' and hasattr(file, 'write'))): self._fp = file else: - raise TypeError("file must be a file-like object " - "or a str, bytes, or PathLike object") + raise TypeError('file must be a file-like object ' + 'or a str, bytes, or PathLike object') if self._mode == _MODE_READ: raw = _streams.DecompressReader( self._fp, ZstdDecompressor, - trailing_error=ZstdError, zstd_dict=zstd_dict, options=options, ) @@ -154,22 +150,22 @@ def flush(self, mode=FLUSH_BLOCK): return self._check_not_closed() if mode not in {self.FLUSH_BLOCK, self.FLUSH_FRAME}: - raise ValueError("Invalid mode argument, expected either " - "ZstdFile.FLUSH_FRAME or " - "ZstdFile.FLUSH_BLOCK") + raise ValueError('Invalid mode argument, expected either ' + 'ZstdFile.FLUSH_FRAME or ' + 'ZstdFile.FLUSH_BLOCK') if self._compressor.last_mode == mode: return # Flush zstd block/frame, and write. data = self._compressor.flush(mode) self._fp.write(data) - if hasattr(self._fp, "flush"): + if hasattr(self._fp, 'flush'): self._fp.flush() def read(self, size=-1): """Read up to size uncompressed bytes from the file. If size is negative or omitted, read until EOF is reached. - Returns b"" if the file is already at EOF. + Returns b'' if the file is already at EOF. """ if size is None: size = -1 @@ -181,14 +177,14 @@ def read1(self, size=-1): making multiple reads from the underlying stream. Reads up to a buffer's worth of data if size is negative. - Returns b"" if the file is at EOF. + Returns b'' if the file is at EOF. """ self._check_can_read() if size < 0: # Note this should *not* be io.DEFAULT_BUFFER_SIZE. # ZSTD_DStreamOutSize is the minimum amount to read guaranteeing # a full block is read. - size = _ZSTD_DStreamOutSize + size = ZSTD_DStreamOutSize return self._buffer.read1(size) def readinto(self, b): @@ -296,7 +292,7 @@ def writable(self): return self._mode == _MODE_WRITE -def open(file, /, mode="rb", *, level=None, options=None, zstd_dict=None, +def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, encoding=None, errors=None, newline=None): """Open a Zstandard compressed file in binary or text mode. @@ -304,8 +300,8 @@ def open(file, /, mode="rb", *, level=None, options=None, zstd_dict=None, in which case the named file is opened, or it can be an existing file object to read from or write to. - The mode parameter can be "r", "rb" (default), "w", "wb", "x", "xb", "a", - "ab" for binary mode, or "rt", "wt", "xt", "at" for text mode. + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', + 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. The level, options, and zstd_dict parameters specify the settings the same as ZstdFile. @@ -326,19 +322,19 @@ def open(file, /, mode="rb", *, level=None, options=None, zstd_dict=None, behavior, and line ending(s). """ - text_mode = "t" in mode - mode = mode.replace("t", "") + text_mode = 't' in mode + mode = mode.replace('t', '') if text_mode: - if "b" in mode: - raise ValueError(f"Invalid mode: {mode!r}") + if 'b' in mode: + raise ValueError(f'Invalid mode: {mode!r}') else: if encoding is not None: - raise ValueError("Argument 'encoding' not supported in binary mode") + raise ValueError('Argument "encoding" not supported in binary mode') if errors is not None: - raise ValueError("Argument 'errors' not supported in binary mode") + raise ValueError('Argument "errors" not supported in binary mode') if newline is not None: - raise ValueError("Argument 'newline' not supported in binary mode") + raise ValueError('Argument "newline" not supported in binary mode') binary_file = ZstdFile(file, mode, level=level, options=options, zstd_dict=zstd_dict) diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 7ada7431c1a..d6ac4b3e0b6 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -17,7 +17,7 @@ wait, as_completed) -__all__ = ( +__all__ = [ 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', @@ -29,36 +29,37 @@ 'Executor', 'wait', 'as_completed', - 'InterpreterPoolExecutor', 'ProcessPoolExecutor', 'ThreadPoolExecutor', -) +] + + +try: + import _interpreters +except ImportError: + _interpreters = None + +if _interpreters: + __all__.append('InterpreterPoolExecutor') def __dir__(): - return __all__ + ('__author__', '__doc__') + return __all__ + ['__author__', '__doc__'] def __getattr__(name): global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor if name == 'ProcessPoolExecutor': - from .process import ProcessPoolExecutor as pe - ProcessPoolExecutor = pe - return pe + from .process import ProcessPoolExecutor + return ProcessPoolExecutor if name == 'ThreadPoolExecutor': - from .thread import ThreadPoolExecutor as te - ThreadPoolExecutor = te - return te + from .thread import ThreadPoolExecutor + return ThreadPoolExecutor - if name == 'InterpreterPoolExecutor': - try: - from .interpreter import InterpreterPoolExecutor as ie - except ModuleNotFoundError: - ie = InterpreterPoolExecutor = None - else: - InterpreterPoolExecutor = ie - return ie + if _interpreters and name == 'InterpreterPoolExecutor': + from .interpreter import InterpreterPoolExecutor + return InterpreterPoolExecutor raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index d98b1ebdd58..f506ce68aea 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -558,6 +558,33 @@ def set_exception(self, exception): self._condition.notify_all() self._invoke_callbacks() + def _get_snapshot(self): + """Get a snapshot of the future's current state. + + This method atomically retrieves the state in one lock acquisition, + which is significantly faster than multiple method calls. + + Returns: + Tuple of (done, cancelled, result, exception) + - done: True if the future is done (cancelled or finished) + - cancelled: True if the future was cancelled + - result: The result if available and not cancelled + - exception: The exception if available and not cancelled + """ + # Fast path: check if already finished without lock + if self._state == FINISHED: + return True, False, self._result, self._exception + + # Need lock for other states since they can change + with self._condition: + # We have to check the state again after acquiring the lock + # because it may have changed in the meantime. + if self._state == FINISHED: + return True, False, self._result, self._exception + if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED}: + return True, True, None, None + return False, False, None, None + __class_getitem__ = classmethod(types.GenericAlias) class Executor(object): diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index d17688dc9d7..85c1da2c722 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -1,69 +1,38 @@ """Implements InterpreterPoolExecutor.""" -import contextlib -import pickle -import textwrap +from concurrent import interpreters +import sys from . import thread as _thread -import _interpreters -import _interpqueues +import traceback -class ExecutionFailed(_interpreters.InterpreterError): - """An unhandled exception happened during execution.""" - - def __init__(self, excinfo): - msg = excinfo.formatted - if not msg: - if excinfo.type and excinfo.msg: - msg = f'{excinfo.type.__name__}: {excinfo.msg}' - else: - msg = excinfo.type.__name__ or excinfo.msg - super().__init__(msg) - self.excinfo = excinfo - - def __str__(self): +def do_call(results, func, args, kwargs): + try: + return func(*args, **kwargs) + except BaseException as exc: + # Send the captured exception out on the results queue, + # but still leave it unhandled for the interpreter to handle. try: - formatted = self.excinfo.errdisplay - except Exception: - return super().__str__() - else: - return textwrap.dedent(f""" -{super().__str__()} - -Uncaught in the interpreter: - -{formatted} - """.strip()) - - -UNBOUND = 2 # error; this should not happen. + results.put(exc) + except interpreters.NotShareableError: + # The exception is not shareable. + print('exception is not shareable:', file=sys.stderr) + traceback.print_exception(exc) + results.put(None) + raise # re-raise class WorkerContext(_thread.WorkerContext): @classmethod - def prepare(cls, initializer, initargs, shared): + def prepare(cls, initializer, initargs): def resolve_task(fn, args, kwargs): if isinstance(fn, str): # XXX Circle back to this later. raise TypeError('scripts not supported') - if args or kwargs: - raise ValueError(f'a script does not take args or kwargs, got {args!r} and {kwargs!r}') - data = textwrap.dedent(fn) - kind = 'script' - # Make sure the script compiles. - # Ideally we wouldn't throw away the resulting code - # object. However, there isn't much to be done until - # code objects are shareable and/or we do a better job - # of supporting code objects in _interpreters.exec(). - compile(data, '', 'exec') else: - # Functions defined in the __main__ module can't be pickled, - # so they can't be used here. In the future, we could possibly - # borrow from multiprocessing to work around this. - data = pickle.dumps((fn, args, kwargs)) - kind = 'function' - return (data, kind) + task = (fn, args, kwargs) + return task if initializer is not None: try: @@ -75,73 +44,24 @@ def resolve_task(fn, args, kwargs): else: initdata = None def create_context(): - return cls(initdata, shared) + return cls(initdata) return create_context, resolve_task - @classmethod - @contextlib.contextmanager - def _capture_exc(cls, resultsid): - try: - yield - except BaseException as exc: - # Send the captured exception out on the results queue, - # but still leave it unhandled for the interpreter to handle. - err = pickle.dumps(exc) - _interpqueues.put(resultsid, (None, err), 1, UNBOUND) - raise # re-raise - - @classmethod - def _send_script_result(cls, resultsid): - _interpqueues.put(resultsid, (None, None), 0, UNBOUND) - - @classmethod - def _call(cls, func, args, kwargs, resultsid): - with cls._capture_exc(resultsid): - res = func(*args or (), **kwargs or {}) - # Send the result back. - try: - _interpqueues.put(resultsid, (res, None), 0, UNBOUND) - except _interpreters.NotShareableError: - res = pickle.dumps(res) - _interpqueues.put(resultsid, (res, None), 1, UNBOUND) - - @classmethod - def _call_pickled(cls, pickled, resultsid): - with cls._capture_exc(resultsid): - fn, args, kwargs = pickle.loads(pickled) - cls._call(fn, args, kwargs, resultsid) - - def __init__(self, initdata, shared=None): + def __init__(self, initdata): self.initdata = initdata - self.shared = dict(shared) if shared else None - self.interpid = None - self.resultsid = None + self.interp = None + self.results = None def __del__(self): - if self.interpid is not None: + if self.interp is not None: self.finalize() - def _exec(self, script): - assert self.interpid is not None - excinfo = _interpreters.exec(self.interpid, script, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) - def initialize(self): - assert self.interpid is None, self.interpid - self.interpid = _interpreters.create(reqrefs=True) + assert self.interp is None, self.interp + self.interp = interpreters.create() try: - _interpreters.incref(self.interpid) - maxsize = 0 - fmt = 0 - self.resultsid = _interpqueues.create(maxsize, fmt, UNBOUND) - - self._exec(f'from {__name__} import WorkerContext') - - if self.shared: - _interpreters.set___main___attrs( - self.interpid, self.shared, restrict=True) + self.results = interpreters.create_queue(maxsize) if self.initdata: self.run(self.initdata) @@ -150,64 +70,25 @@ def initialize(self): raise # re-raise def finalize(self): - interpid = self.interpid - resultsid = self.resultsid - self.resultsid = None - self.interpid = None - if resultsid is not None: - try: - _interpqueues.destroy(resultsid) - except _interpqueues.QueueNotFoundError: - pass - if interpid is not None: - try: - _interpreters.decref(interpid) - except _interpreters.InterpreterNotFoundError: - pass + interp = self.interp + results = self.results + self.results = None + self.interp = None + if results is not None: + del results + if interp is not None: + interp.close() def run(self, task): - data, kind = task - if kind == 'script': - raise NotImplementedError('script kind disabled') - script = f""" -with WorkerContext._capture_exc({self.resultsid}): -{textwrap.indent(data, ' ')} -WorkerContext._send_script_result({self.resultsid})""" - elif kind == 'function': - script = f'WorkerContext._call_pickled({data!r}, {self.resultsid})' - else: - raise NotImplementedError(kind) - try: - self._exec(script) - except ExecutionFailed as exc: - exc_wrapper = exc - else: - exc_wrapper = None - - # Return the result, or raise the exception. - while True: - try: - obj = _interpqueues.get(self.resultsid) - except _interpqueues.QueueNotFoundError: + return self.interp.call(do_call, self.results, *task) + except interpreters.ExecutionFailed as wrapper: + # Wait for the exception data to show up. + exc = self.results.get() + if exc is None: + # The exception must have been not shareable. raise # re-raise - except _interpqueues.QueueError: - continue - except ModuleNotFoundError: - # interpreters.queues doesn't exist, which means - # QueueEmpty doesn't. Act as though it does. - continue - else: - break - (res, excdata), pickled, unboundop = obj - assert unboundop is None, unboundop - if excdata is not None: - assert res is None, res - assert pickled - assert exc_wrapper is not None - exc = pickle.loads(excdata) - raise exc from exc_wrapper - return pickle.loads(res) if pickled else res + raise exc from wrapper class BrokenInterpreterPool(_thread.BrokenThreadPool): @@ -221,11 +102,11 @@ class InterpreterPoolExecutor(_thread.ThreadPoolExecutor): BROKEN = BrokenInterpreterPool @classmethod - def prepare_context(cls, initializer, initargs, shared): - return WorkerContext.prepare(initializer, initargs, shared) + def prepare_context(cls, initializer, initargs): + return WorkerContext.prepare(initializer, initargs) def __init__(self, max_workers=None, thread_name_prefix='', - initializer=None, initargs=(), shared=None): + initializer=None, initargs=()): """Initializes a new InterpreterPoolExecutor instance. Args: @@ -235,8 +116,8 @@ def __init__(self, max_workers=None, thread_name_prefix='', initializer: A callable or script used to initialize each worker interpreter. initargs: A tuple of arguments to pass to the initializer. - shared: A mapping of shareabled objects to be inserted into - each worker interpreter. """ + thread_name_prefix = (thread_name_prefix or + (f"InterpreterPoolExecutor-{self._counter()}")) super().__init__(max_workers, thread_name_prefix, - initializer, initargs, shared=shared) + initializer, initargs) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 76b7b2abe83..a42afa68efc 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -474,9 +474,23 @@ def _terminate_broken(self, cause): bpe = BrokenProcessPool("A process in the process pool was " "terminated abruptly while the future was " "running or pending.") + cause_str = None if cause is not None: - bpe.__cause__ = _RemoteTraceback( - f"\n'''\n{''.join(cause)}'''") + cause_str = ''.join(cause) + else: + # No cause known, so report any processes that have + # terminated with nonzero exit codes, e.g. from a + # segfault. Multiple may terminate simultaneously, + # so include all of them in the traceback. + errors = [] + for p in self.processes.values(): + if p.exitcode is not None and p.exitcode != 0: + errors.append(f"Process {p.pid} terminated abruptly " + f"with exit code {p.exitcode}") + if errors: + cause_str = "\n".join(errors) + if cause_str: + bpe.__cause__ = _RemoteTraceback(f"\n'''\n{cause_str}'''") # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): @@ -755,6 +769,11 @@ def _start_executor_manager_thread(self): self._executor_manager_thread_wakeup def _adjust_process_count(self): + # gh-132969: avoid error when state is reset and executor is still running, + # which will happen when shutdown(wait=False) is called. + if self._processes is None: + return + # if there's an idle process, we don't need to spawn a new one. if self._idle_worker_semaphore.acquire(blocking=False): return diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py similarity index 84% rename from Lib/test/support/interpreters/__init__.py rename to Lib/concurrent/interpreters/__init__.py index e067f259364..ea4147ee9a2 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -9,6 +9,10 @@ InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) +from ._queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, +) __all__ = [ @@ -20,21 +24,6 @@ ] -_queuemod = None - -def __getattr__(name): - if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): - global create_queue, Queue, QueueEmpty, QueueFull - ns = globals() - from .queues import ( - create as create_queue, - Queue, QueueEmpty, QueueFull, - ) - return ns[name] - else: - raise AttributeError(name) - - _EXEC_FAILURE_STR = """ {superstr} @@ -157,19 +146,20 @@ def __del__(self): self._decref() # for pickling: - def __getnewargs__(self): - return (self._id,) + def __reduce__(self): + return (type(self), (self._id,)) - # for pickling: - def __getstate__(self): - return None - - def _decref(self): + # gh-135729: Globals might be destroyed by the time this is called, so we + # need to keep references ourself + def _decref(self, *, + InterpreterNotFoundError=InterpreterNotFoundError, + _interp_decref=_interpreters.decref, + ): if not self._ownsref: return self._ownsref = False try: - _interpreters.decref(self._id) + _interp_decref(self._id) except InterpreterNotFoundError: pass @@ -226,33 +216,32 @@ def exec(self, code, /): if excinfo is not None: raise ExecutionFailed(excinfo) - def call(self, callable, /): + def _call(self, callable, args, kwargs): + res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + return res + + def call(self, callable, /, *args, **kwargs): """Call the object in the interpreter with given args/kwargs. - Only functions that take no arguments and have no closure - are supported. - - The return value is discarded. + Nearly all callables, args, kwargs, and return values are + supported. All "shareable" objects are supported, as are + "stateless" functions (meaning non-closures that do not use + any globals). This method will fall back to pickle. If the callable raises an exception then the error display - (including full traceback) is send back between the interpreters + (including full traceback) is sent back between the interpreters and an ExecutionFailed exception is raised, much like what happens with Interpreter.exec(). """ - # XXX Support args and kwargs. - # XXX Support arbitrary callables. - # XXX Support returning the return value (e.g. via pickle). - excinfo = _interpreters.call(self._id, callable, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) + return self._call(callable, args, kwargs) - def call_in_thread(self, callable, /): + def call_in_thread(self, callable, /, *args, **kwargs): """Return a new thread that calls the object in the interpreter. The return value and any raised exception are discarded. """ - def task(): - self.call(callable) - t = threading.Thread(target=task) + t = threading.Thread(target=self._call, args=(callable, args, kwargs)) t.start() return t diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/concurrent/interpreters/_crossinterp.py similarity index 86% rename from Lib/test/support/interpreters/_crossinterp.py rename to Lib/concurrent/interpreters/_crossinterp.py index 544e197ba4c..a5f46b20fbb 100644 --- a/Lib/test/support/interpreters/_crossinterp.py +++ b/Lib/concurrent/interpreters/_crossinterp.py @@ -40,16 +40,21 @@ class UnboundItem: @classonly def singleton(cls, kind, module, name='UNBOUND'): - doc = cls.__doc__.replace('cross-interpreter container', kind) - doc = doc.replace('cross-interpreter', kind) + doc = cls.__doc__ + if doc: + doc = doc.replace( + 'cross-interpreter container', kind, + ).replace( + 'cross-interpreter', kind, + ) subclass = type( f'Unbound{kind.capitalize()}Item', (cls,), - dict( - _MODULE=module, - _NAME=name, - __doc__=doc, - ), + { + "_MODULE": module, + "_NAME": name, + "__doc__": doc, + }, ) return object.__new__(subclass) @@ -61,7 +66,7 @@ def __new__(cls): def __repr__(self): return f'{self._MODULE}.{self._NAME}' -# return f'interpreters.queues.UNBOUND' +# return f'interpreters._queues.UNBOUND' UNBOUND = object.__new__(UnboundItem) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/concurrent/interpreters/_queues.py similarity index 58% rename from Lib/test/support/interpreters/queues.py rename to Lib/concurrent/interpreters/_queues.py index deb8e8613af..b5cc0b89449 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -1,6 +1,5 @@ """Cross-interpreter Queues High Level Module.""" -import pickle import queue import time import weakref @@ -63,29 +62,34 @@ def _resolve_unbound(flag): return resolved -def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND): +def create(maxsize=0, *, unbounditems=UNBOUND): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. - "syncobj" sets the default for Queue.put() - and Queue.put_nowait(). - - "unbounditems" likewise sets the default. See Queue.put() for + "unbounditems" sets the default for Queue.put(); see that method for supported values. The default value is UNBOUND, which replaces the unbound item. """ - fmt = _SHARED_ONLY if syncobj else _PICKLED unbound = _serialize_unbound(unbounditems) unboundop, = unbound - qid = _queues.create(maxsize, fmt, unboundop) - return Queue(qid, _fmt=fmt, _unbound=unbound) + qid = _queues.create(maxsize, unboundop, -1) + self = Queue(qid) + self._set_unbound(unboundop, unbounditems) + return self def list_all(): """Return a list of all open queues.""" - return [Queue(qid, _fmt=fmt, _unbound=(unboundop,)) - for qid, fmt, unboundop in _queues.list_all()] + queues = [] + for qid, unboundop, _ in _queues.list_all(): + self = Queue(qid) + if not hasattr(self, '_unbound'): + self._set_unbound(unboundop) + else: + assert self._unbound[0] == unboundop + queues.append(self) + return queues _known_queues = weakref.WeakValueDictionary() @@ -93,28 +97,17 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /, *, _fmt=None, _unbound=None): + def __new__(cls, id, /): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') - if _fmt is None: - if _unbound is None: - _fmt, op = _queues.get_queue_defaults(id) - _unbound = (op,) - else: - _fmt, _ = _queues.get_queue_defaults(id) - elif _unbound is None: - _, op = _queues.get_queue_defaults(id) - _unbound = (op,) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id - self._fmt = _fmt - self._unbound = _unbound _known_queues[id] = self _queues.bind(id) return self @@ -136,17 +129,30 @@ def __hash__(self): return hash(self._id) # for pickling: - def __getnewargs__(self): - return (self._id,) + def __reduce__(self): + return (type(self), (self._id,)) - # for pickling: - def __getstate__(self): - return None + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound @property def id(self): return self._id + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _queues.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items + @property def maxsize(self): try: @@ -164,78 +170,59 @@ def full(self): def qsize(self): return _queues.get_count(self._id) - def put(self, obj, timeout=None, *, - syncobj=None, - unbound=None, + def put(self, obj, block=True, timeout=None, *, + unbounditems=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. - This blocks while the queue is full. + If "block" is true, this blocks while the queue is full. - If "syncobj" is None (the default) then it uses the - queue's default, set with create_queue(). + For most objects, the object received through Queue.get() will + be a new one, equivalent to the original and not sharing any + actual underlying data. The notable exceptions include + cross-interpreter types (like Queue) and memoryview, where the + underlying data is actually shared. Furthermore, some types + can be sent through a queue more efficiently than others. This + group includes various immutable types like int, str, bytes, and + tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable(). - If "syncobj" is false then all objects are supported, - at the expense of worse performance. - - If "syncobj" is true then the object must be "shareable". - Examples of "shareable" objects include the builtin singletons, - str, and memoryview. One benefit is that such objects are - passed through the queue efficiently. - - The key difference, though, is conceptual: the corresponding - object returned from Queue.get() will be strictly equivalent - to the given obj. In other words, the two objects will be - effectively indistinguishable from each other, even if the - object is mutable. The received object may actually be the - same object, or a copy (immutable values only), or a proxy. - Regardless, the received object should be treated as though - the original has been shared directly, whether or not it - actually is. That's a slightly different and stronger promise - than just (initial) equality, which is all "syncobj=False" - can promise. - - "unbound" controls the behavior of Queue.get() for the given + "unbounditems" controls the behavior of Queue.get() for the given object if the current interpreter (calling put()) is later destroyed. - If "unbound" is None (the default) then it uses the + If "unbounditems" is None (the default) then it uses the queue's default, set with create_queue(), which is usually UNBOUND. - If "unbound" is UNBOUND_ERROR then get() will raise an + If "unbounditems" is UNBOUND_ERROR then get() will raise an ItemInterpreterDestroyed exception if the original interpreter has been destroyed. This does not otherwise affect the queue; the next call to put() will work like normal, returning the next item in the queue. - If "unbound" is UNBOUND_REMOVE then the item will be removed + If "unbounditems" is UNBOUND_REMOVE then the item will be removed from the queue as soon as the original interpreter is destroyed. Be aware that this will introduce an imbalance between put() and get() calls. - If "unbound" is UNBOUND then it is returned by get() in place + If "unbounditems" is UNBOUND then it is returned by get() in place of the unbound item. """ - if syncobj is None: - fmt = self._fmt + if not block: + return self.put_nowait(obj, unbounditems=unbounditems) + if unbounditems is None: + unboundop = -1 else: - fmt = _SHARED_ONLY if syncobj else _PICKLED - if unbound is None: - unboundop, = self._unbound - else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) if timeout is not None: timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout - if fmt is _PICKLED: - obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj, fmt, unboundop) + _queues.put(self._id, obj, unboundop) except QueueFull as exc: if timeout is not None and time.time() >= end: raise # re-raise @@ -243,30 +230,26 @@ def put(self, obj, timeout=None, *, else: break - def put_nowait(self, obj, *, syncobj=None, unbound=None): - if syncobj is None: - fmt = self._fmt + def put_nowait(self, obj, *, unbounditems=None): + if unbounditems is None: + unboundop = -1 else: - fmt = _SHARED_ONLY if syncobj else _PICKLED - if unbound is None: - unboundop, = self._unbound - else: - unboundop, = _serialize_unbound(unbound) - if fmt is _PICKLED: - obj = pickle.dumps(obj) - _queues.put(self._id, obj, fmt, unboundop) + unboundop, = _serialize_unbound(unbounditems) + _queues.put(self._id, obj, unboundop) - def get(self, timeout=None, *, + def get(self, block=True, timeout=None, *, _delay=10 / 1000, # 10 milliseconds ): """Return the next object from the queue. - This blocks while the queue is empty. + If "block" is true, this blocks while the queue is empty. If the next item's original interpreter has been destroyed then the "next object" is determined by the value of the - "unbound" argument to put(). + "unbounditems" argument to put(). """ + if not block: + return self.get_nowait() if timeout is not None: timeout = int(timeout) if timeout < 0: @@ -274,7 +257,7 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - obj, fmt, unboundop = _queues.get(self._id) + obj, unboundop = _queues.get(self._id) except QueueEmpty as exc: if timeout is not None and time.time() >= end: raise # re-raise @@ -284,10 +267,6 @@ def get(self, timeout=None, *, if unboundop is not None: assert obj is None, repr(obj) return _resolve_unbound(unboundop) - if fmt == _PICKLED: - obj = pickle.loads(obj) - else: - assert fmt == _SHARED_ONLY return obj def get_nowait(self): @@ -297,16 +276,12 @@ def get_nowait(self): is the same as get(). """ try: - obj, fmt, unboundop = _queues.get(self._id) + obj, unboundop = _queues.get(self._id) except QueueEmpty as exc: raise # re-raise if unboundop is not None: assert obj is None, repr(obj) return _resolve_unbound(unboundop) - if fmt == _PICKLED: - obj = pickle.loads(obj) - else: - assert fmt == _SHARED_ONLY return obj diff --git a/Lib/configparser.py b/Lib/configparser.py index 239fda60a02..18af1eadaad 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -1218,11 +1218,14 @@ def _convert_to_boolean(self, value): def _validate_key_contents(self, key): """Raises an InvalidWriteError for any keys containing - delimiters or that match the section header pattern""" + delimiters or that begins with the section header pattern""" if re.match(self.SECTCRE, key): - raise InvalidWriteError("Cannot write keys matching section pattern") - if any(delim in key for delim in self._delimiters): - raise InvalidWriteError("Cannot write key that contains delimiters") + raise InvalidWriteError( + f"Cannot write key {key}; begins with section pattern") + for delim in self._delimiters: + if delim in key: + raise InvalidWriteError( + f"Cannot write key {key}; contains delimiter {delim}") def _validate_value_types(self, *, section="", option="", value=""): """Raises a TypeError for illegal non-string values. diff --git a/Lib/copy.py b/Lib/copy.py index c64fc076179..fff7e93c2a1 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -100,14 +100,14 @@ def copy(x): return _reconstruct(x, None, *rv) -_copy_atomic_types = {types.NoneType, int, float, bool, complex, str, tuple, +_copy_atomic_types = frozenset({types.NoneType, int, float, bool, complex, str, tuple, bytes, frozenset, type, range, slice, property, types.BuiltinFunctionType, types.EllipsisType, types.NotImplementedType, types.FunctionType, types.CodeType, - weakref.ref, super} -_copy_builtin_containers = {list, dict, set, bytearray} + weakref.ref, super}) +_copy_builtin_containers = frozenset({list, dict, set, bytearray}) -def deepcopy(x, memo=None, _nil=[]): +def deepcopy(x, memo=None): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. @@ -122,8 +122,8 @@ def deepcopy(x, memo=None, _nil=[]): if memo is None: memo = {} else: - y = memo.get(d, _nil) - if y is not _nil: + y = memo.get(d, None) + if y is not None: return y copier = _deepcopy_dispatch.get(cls) @@ -162,9 +162,9 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y -_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, +_atomic_types = frozenset({types.NoneType, types.EllipsisType, types.NotImplementedType, int, float, bool, complex, bytes, str, types.CodeType, type, range, - types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property}) _deepcopy_dispatch = d = {} diff --git a/Lib/csv.py b/Lib/csv.py index 0a627ba7a51..b2aaf5fd9fa 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -81,8 +81,6 @@ class excel: "unregister_dialect", "DictReader", "DictWriter", "unix_dialect"] -__version__ = "1.0" - class Dialect: """Describe a CSV dialect. @@ -364,31 +362,33 @@ def _guess_delimiter(self, data, delimiters): try and evaluate the smallest portion of the data possible, evaluating additional chunks as necessary. """ + from collections import Counter, defaultdict data = list(filter(None, data.split('\n'))) - ascii = [chr(c) for c in range(127)] # 7-bit ASCII - # build frequency tables chunkLength = min(10, len(data)) iteration = 0 - charFrequency = {} + num_lines = 0 + # {char -> {count_per_line -> num_lines_with_that_count}} + char_frequency = defaultdict(Counter) modes = {} delims = {} start, end = 0, chunkLength while start < len(data): iteration += 1 for line in data[start:end]: - for char in ascii: - metaFrequency = charFrequency.get(char, {}) - # must count even if frequency is 0 - freq = line.count(char) - # value is the mode - metaFrequency[freq] = metaFrequency.get(freq, 0) + 1 - charFrequency[char] = metaFrequency + num_lines += 1 + for char, count in Counter(line).items(): + if char.isascii(): + char_frequency[char][count] += 1 - for char in charFrequency.keys(): - items = list(charFrequency[char].items()) + for char, counts in char_frequency.items(): + items = list(counts.items()) + missed_lines = num_lines - sum(counts.values()) + if missed_lines: + # Store the number of lines 'char' was missing from. + items.append((0, missed_lines)) if len(items) == 1 and items[0][0] == 0: continue # get the mode of the frequencies @@ -511,3 +511,12 @@ def has_header(self, sample): hasHeader -= 1 return hasHeader > 0 + + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "1.0" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 823a3692fd1..ab5b656e6e5 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -1,6 +1,8 @@ """create and manipulate C data types in Python""" -import os as _os, sys as _sys +import os as _os +import sys as _sys +import sysconfig as _sysconfig import types as _types __version__ = "1.1.0" @@ -108,7 +110,7 @@ class CFunctionType(_CFuncPtr): return CFunctionType if _os.name == "nt": - from _ctypes import LoadLibrary as _dlopen + from _ctypes import LoadLibrary as _LoadLibrary from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL _win_functype_cache = {} @@ -379,12 +381,6 @@ def create_unicode_buffer(init, size=None): return buf raise TypeError(init) - -def SetPointerType(pointer, cls): - import warnings - warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) - pointer.set_type(cls) - def ARRAY(typ, len): return typ * len @@ -416,52 +412,59 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None): + class _FuncPtr(_CFuncPtr): + _flags_ = self._func_flags_ + _restype_ = self._func_restype_ + if use_errno: + _flags_ |= _FUNCFLAG_USE_ERRNO + if use_last_error: + _flags_ |= _FUNCFLAG_USE_LASTERROR + + self._FuncPtr = _FuncPtr if name: name = _os.fspath(name) + self._handle = self._load_library(name, mode, handle, winmode) + + if _os.name == "nt": + def _load_library(self, name, mode, handle, winmode): + if winmode is None: + import nt as _nt + winmode = _nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS + # WINAPI LoadLibrary searches for a DLL if the given name + # is not fully qualified with an explicit drive. For POSIX + # compatibility, and because the DLL search path no longer + # contains the working directory, begin by fully resolving + # any name that contains a path separator. + if name is not None and ('/' in name or '\\' in name): + name = _nt._getfullpathname(name) + winmode |= _nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR + self._name = name + if handle is not None: + return handle + return _LoadLibrary(self._name, winmode) + + else: + def _load_library(self, name, mode, handle, winmode): # If the filename that has been provided is an iOS/tvOS/watchOS # .fwork file, dereference the location to the true origin of the # binary. - if name.endswith(".fwork"): + if name and name.endswith(".fwork"): with open(name) as f: name = _os.path.join( _os.path.dirname(_sys.executable), f.read().strip() ) - - self._name = name - flags = self._func_flags_ - if use_errno: - flags |= _FUNCFLAG_USE_ERRNO - if use_last_error: - flags |= _FUNCFLAG_USE_LASTERROR - if _sys.platform.startswith("aix"): - """When the name contains ".a(" and ends with ")", - e.g., "libFOO.a(libFOO.so)" - this is taken to be an - archive(member) syntax for dlopen(), and the mode is adjusted. - Otherwise, name is presented to dlopen() as a file argument. - """ - if name and name.endswith(")") and ".a(" in name: - mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW ) - if _os.name == "nt": - if winmode is not None: - mode = winmode - else: - import nt - mode = nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS - if '/' in name or '\\' in name: - self._name = nt._getfullpathname(self._name) - mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR - - class _FuncPtr(_CFuncPtr): - _flags_ = flags - _restype_ = self._func_restype_ - self._FuncPtr = _FuncPtr - - if handle is None: - self._handle = _dlopen(self._name, mode) - else: - self._handle = handle + if _sys.platform.startswith("aix"): + """When the name contains ".a(" and ends with ")", + e.g., "libFOO.a(libFOO.so)" - this is taken to be an + archive(member) syntax for dlopen(), and the mode is adjusted. + Otherwise, name is presented to dlopen() as a file argument. + """ + if name and name.endswith(")") and ".a(" in name: + mode |= _os.RTLD_MEMBER | _os.RTLD_NOW + self._name = name + return _dlopen(name, mode) def __repr__(self): return "<%s '%s', handle %x at %#x>" % \ @@ -549,10 +552,9 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif _sys.platform == "android": - pythonapi = PyDLL("libpython%d.%d.so" % _sys.version_info[:2]) -elif _sys.platform == "cygwin": - pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) +elif _sys.platform in ["android", "cygwin"]: + # These are Unix-like platforms which use a dynamically-linked libpython. + pythonapi = PyDLL(_sysconfig.get_config_var("LDLIBRARY")) else: pythonapi = PyDLL(None) diff --git a/Lib/ctypes/macholib/__init__.py b/Lib/ctypes/macholib/__init__.py index 5621defccd6..f00e55f8131 100644 --- a/Lib/ctypes/macholib/__init__.py +++ b/Lib/ctypes/macholib/__init__.py @@ -6,4 +6,10 @@ And also Apple's documentation. """ -__version__ = '1.0' +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "1.0" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 99504911a3d..378f12167c6 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -173,6 +173,25 @@ def find_library(name): fname = f"{directory}/lib{name}.so" return fname if os.path.isfile(fname) else None +elif sys.platform == "emscripten": + def _is_wasm(filename): + # Return True if the given file is an WASM module + wasm_header = b"\x00asm" + with open(filename, 'br') as thefile: + return thefile.read(4) == wasm_header + + def find_library(name): + candidates = [f"lib{name}.so", f"lib{name}.wasm"] + paths = os.environ.get("LD_LIBRARY_PATH", "") + for libdir in paths.split(":"): + for name in candidates: + libfile = os.path.join(libdir, name) + + if os.path.isfile(libfile) and _is_wasm(libfile): + return libfile + + return None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py index 6165fe6c987..605d5fcbec5 100644 --- a/Lib/curses/__init__.py +++ b/Lib/curses/__init__.py @@ -30,9 +30,8 @@ def initscr(): fd=_sys.__stdout__.fileno()) stdscr = _curses.initscr() for key, value in _curses.__dict__.items(): - if key[0:4] == 'ACS_' or key in ('LINES', 'COLS'): + if key.startswith('ACS_') or key in ('LINES', 'COLS'): setattr(curses, key, value) - return stdscr # This is a similar wrapper for start_color(), which adds the COLORS and @@ -41,12 +40,9 @@ def initscr(): def start_color(): import _curses, curses - retval = _curses.start_color() - if hasattr(_curses, 'COLORS'): - curses.COLORS = _curses.COLORS - if hasattr(_curses, 'COLOR_PAIRS'): - curses.COLOR_PAIRS = _curses.COLOR_PAIRS - return retval + _curses.start_color() + curses.COLORS = _curses.COLORS + curses.COLOR_PAIRS = _curses.COLOR_PAIRS # Import Python has_key() implementation if _curses doesn't contain has_key() @@ -85,10 +81,11 @@ def wrapper(func, /, *args, **kwds): # Start color, too. Harmless if the terminal doesn't have # color; user can test with has_color() later on. The try/catch # works around a minor bit of over-conscientiousness in the curses - # module -- the error return from C start_color() is ignorable. + # module -- the error return from C start_color() is ignorable, + # unless they are raised by the interpreter due to other issues. try: start_color() - except: + except _curses.error: pass return func(stdscr, *args, **kwds) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 86d29df0639..730ced72998 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -441,9 +441,11 @@ def __init__(self, globals): self.locals = {} self.overwrite_errors = {} self.unconditional_adds = {} + self.method_annotations = {} def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, - overwrite_error=False, unconditional_add=False, decorator=None): + overwrite_error=False, unconditional_add=False, decorator=None, + annotation_fields=None): if locals is not None: self.locals.update(locals) @@ -464,16 +466,14 @@ def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, self.names.append(name) - if return_type is not MISSING: - self.locals[f'__dataclass_{name}_return_type__'] = return_type - return_annotation = f'->__dataclass_{name}_return_type__' - else: - return_annotation = '' + if annotation_fields is not None: + self.method_annotations[name] = (annotation_fields, return_type) + args = ','.join(args) body = '\n'.join(body) # Compute the text of the entire function, add it to the text we're generating. - self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}):\n{body}') def add_fns_to_class(self, cls): # The source to all of the functions we're generating. @@ -509,6 +509,15 @@ def add_fns_to_class(self, cls): # Now that we've generated the functions, assign them into cls. for name, fn in zip(self.names, fns): fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + + try: + annotation_fields, return_type = self.method_annotations[name] + except KeyError: + pass + else: + annotate_fn = _make_annotate_function(cls, name, annotation_fields, return_type) + fn.__annotate__ = annotate_fn + if self.unconditional_adds.get(name, False): setattr(cls, name, fn) else: @@ -524,6 +533,49 @@ def add_fns_to_class(self, cls): raise TypeError(error_msg) +def _make_annotate_function(__class__, method_name, annotation_fields, return_type): + # Create an __annotate__ function for a dataclass + # Try to return annotations in the same format as they would be + # from a regular __init__ function + + def __annotate__(format, /): + Format = annotationlib.Format + match format: + case Format.VALUE | Format.FORWARDREF | Format.STRING: + cls_annotations = {} + for base in reversed(__class__.__mro__): + cls_annotations.update( + annotationlib.get_annotations(base, format=format) + ) + + new_annotations = {} + for k in annotation_fields: + # gh-142214: The annotation may be missing in unusual dynamic cases. + # If so, just skip it. + try: + new_annotations[k] = cls_annotations[k] + except KeyError: + pass + + if return_type is not MISSING: + if format == Format.STRING: + new_annotations["return"] = annotationlib.type_repr(return_type) + else: + new_annotations["return"] = return_type + + return new_annotations + + case _: + raise NotImplementedError(format) + + # This is a flag for _add_slots to know it needs to regenerate this method + # In order to remove references to the original class when it is replaced + __annotate__.__generated_by_dataclasses__ = True + __annotate__.__qualname__ = f"{__class__.__qualname__}.{method_name}.__annotate__" + + return __annotate__ + + def _field_assign(frozen, name, value, self_name): # If we're a frozen class, then assign to our fields in __init__ # via object.__setattr__. Otherwise, just use a simple @@ -612,7 +664,7 @@ def _init_param(f): elif f.default_factory is not MISSING: # There's a factory function. Set a marker. default = '=__dataclass_HAS_DEFAULT_FACTORY__' - return f'{f.name}:__dataclass_type_{f.name}__{default}' + return f'{f.name}{default}' def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, @@ -635,11 +687,10 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, raise TypeError(f'non-default argument {f.name!r} ' f'follows default argument {seen_default.name!r}') - locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, - **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - } - } + annotation_fields = [f.name for f in fields if f.init] + + locals = {'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object} body_lines = [] for f in fields: @@ -670,7 +721,8 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, [self_name] + _init_params, body_lines, locals=locals, - return_type=None) + return_type=None, + annotation_fields=annotation_fields) def _frozen_get_del_attr(cls, fields, func_builder): @@ -1265,7 +1317,7 @@ def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): doc = getattr(defined_fields.get(slot), 'doc', None) if doc is not None: seen_docs = True - slots.update({slot: doc}) + slots[slot] = doc # We only return dict if there's at least one doc member, # otherwise we return tuple, which is the old default format. @@ -1300,10 +1352,9 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # available in _MARKER. cls_dict.pop(field_name, None) - # Remove __dict__ itself. + # Remove __dict__ and `__weakref__` descriptors. + # They'll be added back if applicable. cls_dict.pop('__dict__', None) - - # Clear existing `__weakref__` descriptor, it belongs to a previous type: cls_dict.pop('__weakref__', None) # gh-102069 # And finally create the class. @@ -1338,6 +1389,26 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): or _update_func_cell_for__class__(member.fdel, cls, newcls)): break + # Get new annotations to remove references to the original class + # in forward references + newcls_ann = annotationlib.get_annotations( + newcls, format=annotationlib.Format.FORWARDREF) + + # Fix references in dataclass Fields + for f in getattr(newcls, _FIELDS).values(): + try: + ann = newcls_ann[f.name] + except KeyError: + pass + else: + f.type = ann + + # Fix the class reference in the __annotate__ method + init = newcls.__init__ + if init_annotate := getattr(init, "__annotate__", None): + if getattr(init_annotate, "__generated_by_dataclasses__", False): + _update_func_cell_for__class__(init_annotate, cls, newcls) + return newcls diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index def120ffc37..1bc239a84ff 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -9,7 +9,7 @@ - seems to contain a bug when updating... - reclaim free space (currently, space once occupied by deleted or expanded -items is never reused) +items is not reused exept if .reorganize() is called) - support concurrent access (currently, if two processes take turns making updates, they can mess up the index) @@ -17,8 +17,6 @@ - support efficient access to large databases (currently, the whole index is read when the database is opened, and some updates rewrite the whole index) -- support opening for read-only (flag = 'm') - """ import ast as _ast @@ -289,6 +287,34 @@ def __enter__(self): def __exit__(self, *args): self.close() + def reorganize(self): + if self._readonly: + raise error('The database is opened for reading only') + self._verify_open() + # Ensure all changes are committed before reorganizing. + self._commit() + # Open file in r+ to allow changing in-place. + with _io.open(self._datfile, 'rb+') as f: + reorganize_pos = 0 + + # Iterate over existing keys, sorted by starting byte. + for key in sorted(self._index, key = lambda k: self._index[k][0]): + pos, siz = self._index[key] + f.seek(pos) + val = f.read(siz) + + f.seek(reorganize_pos) + f.write(val) + self._index[key] = (reorganize_pos, siz) + + blocks_occupied = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE + reorganize_pos += blocks_occupied * _BLOCKSIZE + + f.truncate(reorganize_pos) + # Commit changes to index, which were not in-place. + self._commit() + + def open(file, flag='c', mode=0o666): """Open the database file, filename, and return corresponding object. diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py index 7e0ae2a29e3..c8ee6f184b3 100644 --- a/Lib/dbm/sqlite3.py +++ b/Lib/dbm/sqlite3.py @@ -15,6 +15,7 @@ STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)" ITER_KEYS = "SELECT key FROM Dict" +REORGANIZE = "VACUUM" class error(OSError): @@ -59,18 +60,22 @@ def __init__(self, path, /, *, flag, mode): # We use the URI format when opening the database. uri = _normalize_uri(path) uri = f"{uri}?mode={flag}" + if flag == "ro": + # Add immutable=1 to allow read-only SQLite access even if wal/shm missing + uri += "&immutable=1" try: self._cx = sqlite3.connect(uri, autocommit=True, uri=True) except sqlite3.Error as exc: raise error(str(exc)) - # This is an optimization only; it's ok if it fails. - with suppress(sqlite3.OperationalError): - self._cx.execute("PRAGMA journal_mode = wal") + if flag != "ro": + # This is an optimization only; it's ok if it fails. + with suppress(sqlite3.OperationalError): + self._cx.execute("PRAGMA journal_mode = wal") - if flag == "rwc": - self._execute(BUILD_TABLE) + if flag == "rwc": + self._execute(BUILD_TABLE) def _execute(self, *args, **kwargs): if not self._cx: @@ -122,6 +127,9 @@ def __enter__(self): def __exit__(self, *args): self.close() + def reorganize(self): + self._execute(REORGANIZE) + def open(filename, /, flag="r", mode=0o666): """Open a dbm.sqlite3 database and return the dbm object. diff --git a/Lib/decimal.py b/Lib/decimal.py index 530bdfb3895..cf13050bfef 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -100,8 +100,8 @@ try: from _decimal import * - from _decimal import __version__ # noqa: F401 from _decimal import __libmpdec_version__ # noqa: F401 + from _decimal import __getattr__ # noqa: F401 except ImportError: import _pydecimal import sys diff --git a/Lib/difflib.py b/Lib/difflib.py index f1f4e62514a..4a0600e4ebb 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -30,6 +30,7 @@ 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match'] +from _colorize import can_colorize, get_theme from heapq import nlargest as _nlargest from collections import namedtuple as _namedtuple from types import GenericAlias @@ -78,8 +79,8 @@ class SequenceMatcher: sequences. As a rule of thumb, a .ratio() value over 0.6 means the sequences are close matches: - >>> print(round(s.ratio(), 3)) - 0.866 + >>> print(round(s.ratio(), 2)) + 0.87 >>> If you're only interested in where the sequences match, @@ -1094,7 +1095,7 @@ def _format_range_unified(start, stop): return '{},{}'.format(beginning, length) def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', - tofiledate='', n=3, lineterm='\n'): + tofiledate='', n=3, lineterm='\n', *, color=False): r""" Compare two sequences of lines; generate the delta as a unified diff. @@ -1111,6 +1112,10 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', For inputs that do not have trailing newlines, set the lineterm argument to "" so that the output will be uniformly newline free. + Set 'color' to True to enable output in color, similar to + 'git diff --color'. Even if enabled, it can be + controlled using environment variables such as 'NO_COLOR'. + The unidiff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. @@ -1134,6 +1139,11 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', four """ + if color and can_colorize(): + t = get_theme(force_color=True).difflib + else: + t = get_theme(force_no_color=True).difflib + _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): @@ -1141,25 +1151,25 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' todate = '\t{}'.format(tofiledate) if tofiledate else '' - yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) - yield '+++ {}{}{}'.format(tofile, todate, lineterm) + yield f'{t.header}--- {fromfile}{fromdate}{lineterm}{t.reset}' + yield f'{t.header}+++ {tofile}{todate}{lineterm}{t.reset}' first, last = group[0], group[-1] file1_range = _format_range_unified(first[1], last[2]) file2_range = _format_range_unified(first[3], last[4]) - yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) + yield f'{t.hunk}@@ -{file1_range} +{file2_range} @@{lineterm}{t.reset}' for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: - yield ' ' + line + yield f'{t.context} {line}{t.reset}' continue if tag in {'replace', 'delete'}: for line in a[i1:i2]: - yield '-' + line + yield f'{t.removed}-{line}{t.reset}' if tag in {'replace', 'insert'}: for line in b[j1:j2]: - yield '+' + line + yield f'{t.added}+{line}{t.reset}' ######################################################################## @@ -1615,16 +1625,13 @@ def _line_pair_iterator(): _file_template = """ - - - - + + - - - @@ -1636,13 +1643,36 @@ def _line_pair_iterator(): _styles = """ :root {color-scheme: light dark} - table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium} - .diff_header {background-color:#e0e0e0} - td.diff_header {text-align:right} - .diff_next {background-color:#c0c0c0} + table.diff { + font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; + border: medium; + } + .diff_header { + background-color: #e0e0e0; + font-weight: bold; + } + td.diff_header { + text-align: right; + padding: 0 8px; + } + .diff_next { + background-color: #c0c0c0; + padding: 4px 0; + } .diff_add {background-color:palegreen} .diff_chg {background-color:#ffff77} .diff_sub {background-color:#ffaaaa} + table.diff[summary="Legends"] { + margin-top: 20px; + border: 1px solid #ccc; + } + table.diff[summary="Legends"] th { + background-color: #e0e0e0; + padding: 4px 8px; + } + table.diff[summary="Legends"] td { + padding: 4px 8px; + } @media (prefers-color-scheme: dark) { .diff_header {background-color:#666} @@ -1650,6 +1680,8 @@ def _line_pair_iterator(): .diff_add {background-color:darkgreen} .diff_chg {background-color:#847415} .diff_sub {background-color:darkred} + table.diff[summary="Legends"] {border-color:#555} + table.diff[summary="Legends"] th{background-color:#666} }""" _table_template = """ @@ -1692,7 +1724,7 @@ class HtmlDiff(object): make_table -- generates HTML for a single side by side table make_file -- generates complete HTML file with a single side by side table - See tools/scripts/diff.py for an example usage of this class. + See Doc/includes/diff.py for an example usage of this class. """ _file_template = _file_template @@ -1892,8 +1924,11 @@ def _format_line(self,side,flag,linenum,text): # make space non-breakable so they don't get compressed or line wrapped text = text.replace(' ',' ').rstrip() - return '%s%s' \ - % (id,linenum,text) + # add a class to the td tag if there is a difference on the line + css_class = ' class="diff_changed" ' if flag else ' ' + + return f'{linenum}' \ + + f'{text}' def _make_prefix(self): """Create unique anchor prefixes""" diff --git a/Lib/doctest.py b/Lib/doctest.py index 2acb6cb79f3..0fcfa1e3e97 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -94,6 +94,7 @@ def _test(): import __future__ import difflib +import functools import inspect import linecache import os @@ -101,8 +102,9 @@ def _test(): import re import sys import traceback +import types import unittest -from io import StringIO, IncrementalNewlineDecoder +from io import StringIO, TextIOWrapper, BytesIO from collections import namedtuple import _colorize # Used in doctests from _colorize import ANSIColors, can_colorize @@ -235,10 +237,6 @@ def _normalize_module(module, depth=2): else: raise TypeError("Expected a module, string, or None") -def _newline_convert(data): - # The IO module provides a handy decoder for universal newline conversion - return IncrementalNewlineDecoder(None, True).decode(data, True) - def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) @@ -250,10 +248,9 @@ def _load_testfile(filename, package, module_relative, encoding): pass if hasattr(loader, 'get_data'): file_contents = loader.get_data(filename) - file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + return TextIOWrapper(BytesIO(file_contents), encoding=encoding, newline=None).read(), filename with open(filename, encoding=encoding) as f: return f.read(), filename @@ -385,7 +382,7 @@ def __init__(self, out): self.__out = out self.__debugger_used = False # do not play signal games in the pdb - pdb.Pdb.__init__(self, stdout=out, nosigint=True) + super().__init__(stdout=out, nosigint=True) # still use input() to get user input self.use_rawinput = 1 @@ -1140,7 +1137,9 @@ def _find_lineno(self, obj, source_lines): if inspect.ismethod(obj): obj = obj.__func__ if isinstance(obj, property): obj = obj.fget - if inspect.isfunction(obj) and getattr(obj, '__doc__', None): + if isinstance(obj, functools.cached_property): + obj = obj.func + if inspect.isroutine(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. obj = inspect.unwrap(obj) try: @@ -1168,6 +1167,32 @@ def _find_lineno(self, obj, source_lines): if pat.match(source_lines[lineno]): return lineno + # Handle __test__ string doctests formatted as triple-quoted + # strings. Find a non-blank line in the test string and match it + # in the source, verifying subsequent lines also match to handle + # duplicate lines. + if isinstance(obj, str) and source_lines is not None: + obj_lines = obj.splitlines(keepends=True) + # Skip the first line (may be on same line as opening quotes) + # and any blank lines to find a meaningful line to match. + start_index = 1 + while (start_index < len(obj_lines) + and not obj_lines[start_index].strip()): + start_index += 1 + if start_index < len(obj_lines): + target_line = obj_lines[start_index] + for lineno, source_line in enumerate(source_lines): + if source_line == target_line: + # Verify subsequent lines also match + for i in range(start_index + 1, len(obj_lines) - 1): + source_idx = lineno + i - start_index + if source_idx >= len(source_lines): + break + if obj_lines[i] != source_lines[source_idx]: + break + else: + return lineno - start_index + # We couldn't find the line number. return None @@ -1278,6 +1303,11 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Reporting methods #///////////////////////////////////////////////////////////////// + def report_skip(self, out, test, example): + """ + Report that the given example was skipped. + """ + def report_start(self, out, test, example): """ Report that the test runner is about to process the given @@ -1375,6 +1405,8 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + if not quiet: + self.report_skip(out, test, example) skips += 1 continue @@ -1395,11 +1427,11 @@ def __run(self, test, compileflags, out): exec(compile(example.source, filename, "single", compileflags, True), test.globs) self.debugger.set_continue() # ==== Example Finished ==== - exception = None + exc_info = None except KeyboardInterrupt: raise - except: - exception = sys.exc_info() + except BaseException as exc: + exc_info = type(exc), exc, exc.__traceback__.tb_next self.debugger.set_continue() # ==== Example Finished ==== got = self._fakeout.getvalue() # the actual output @@ -1408,21 +1440,21 @@ def __run(self, test, compileflags, out): # If the example executed without raising any exceptions, # verify its output. - if exception is None: + if exc_info is None: if check(example.want, got, self.optionflags): outcome = SUCCESS # The example raised an exception: check if it was expected. else: - formatted_ex = traceback.format_exception_only(*exception[:2]) - if issubclass(exception[0], SyntaxError): + formatted_ex = traceback.format_exception_only(*exc_info[:2]) + if issubclass(exc_info[0], SyntaxError): # SyntaxError / IndentationError is special: # we don't care about the carets / suggestions / etc # We only care about the error message and notes. # They start with `SyntaxError:` (or any other class name) exception_line_prefixes = ( - f"{exception[0].__qualname__}:", - f"{exception[0].__module__}.{exception[0].__qualname__}:", + f"{exc_info[0].__qualname__}:", + f"{exc_info[0].__module__}.{exc_info[0].__qualname__}:", ) exc_msg_index = next( index @@ -1433,7 +1465,7 @@ def __run(self, test, compileflags, out): exc_msg = "".join(formatted_ex) if not quiet: - got += _exception_traceback(exception) + got += _exception_traceback(exc_info) # If `example.exc_msg` is None, then we weren't expecting # an exception. @@ -1462,7 +1494,7 @@ def __run(self, test, compileflags, out): elif outcome is BOOM: if not quiet: self.report_unexpected_exception(out, test, example, - exception) + exc_info) failures += 1 else: assert False, ("unknown outcome", outcome) @@ -1989,8 +2021,8 @@ def testmod(m=None, name=None, globs=None, verbose=None, from module m (or the current module if m is not supplied), starting with m.__doc__. - Also test examples reachable from dict m.__test__ if it exists and is - not None. m.__test__ maps names to functions, classes and strings; + Also test examples reachable from dict m.__test__ if it exists. + m.__test__ maps names to functions, classes and strings; function and class docstrings are tested even if the name is private; strings are tested directly, as if they were docstrings. @@ -2272,12 +2304,63 @@ def set_unittest_reportflags(flags): return old +class _DocTestCaseRunner(DocTestRunner): + + def __init__(self, *args, test_case, test_result, **kwargs): + super().__init__(*args, **kwargs) + self._test_case = test_case + self._test_result = test_result + self._examplenum = 0 + + def _subTest(self): + subtest = unittest.case._SubTest(self._test_case, str(self._examplenum), {}) + self._examplenum += 1 + return subtest + + def report_skip(self, out, test, example): + unittest.case._addSkip(self._test_result, self._subTest(), '') + + def report_success(self, out, test, example, got): + self._test_result.addSubTest(self._test_case, self._subTest(), None) + + def report_unexpected_exception(self, out, test, example, exc_info): + tb = self._add_traceback(exc_info[2], test, example) + exc_info = (*exc_info[:2], tb) + self._test_result.addSubTest(self._test_case, self._subTest(), exc_info) + + def report_failure(self, out, test, example, got): + msg = ('Failed example:\n' + _indent(example.source) + + self._checker.output_difference(example, got, self.optionflags).rstrip('\n')) + exc = self._test_case.failureException(msg) + tb = self._add_traceback(None, test, example) + exc_info = (type(exc), exc, tb) + self._test_result.addSubTest(self._test_case, self._subTest(), exc_info) + + def _add_traceback(self, traceback, test, example): + if test.lineno is None or example.lineno is None: + lineno = None + else: + lineno = test.lineno + example.lineno + 1 + return types.SimpleNamespace( + tb_frame = types.SimpleNamespace( + f_globals=test.globs, + f_code=types.SimpleNamespace( + co_filename=test.filename, + co_name=test.name, + ), + ), + tb_next = traceback, + tb_lasti = -1, + tb_lineno = lineno, + ) + + class DocTestCase(unittest.TestCase): def __init__(self, test, optionflags=0, setUp=None, tearDown=None, checker=None): - unittest.TestCase.__init__(self) + super().__init__() self._dt_optionflags = optionflags self._dt_checker = checker self._dt_test = test @@ -2301,30 +2384,28 @@ def tearDown(self): test.globs.clear() test.globs.update(self._dt_globs) + def run(self, result=None): + self._test_result = result + return super().run(result) + def runTest(self): test = self._dt_test - old = sys.stdout - new = StringIO() optionflags = self._dt_optionflags + result = self._test_result if not (optionflags & REPORTING_FLAGS): # The option flags don't include any reporting flags, # so add the default reporting flags optionflags |= _unittest_reportflags + if getattr(result, 'failfast', False): + optionflags |= FAIL_FAST - runner = DocTestRunner(optionflags=optionflags, - checker=self._dt_checker, verbose=False) - - try: - runner.DIVIDER = "-"*70 - results = runner.run(test, out=new.write, clear_globs=False) - if results.skipped == results.attempted: - raise unittest.SkipTest("all examples were skipped") - finally: - sys.stdout = old - - if results.failed: - raise self.failureException(self.format_failure(new.getvalue())) + runner = _DocTestCaseRunner(optionflags=optionflags, + checker=self._dt_checker, verbose=False, + test_case=self, test_result=result) + results = runner.run(test, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") def format_failure(self, err): test = self._dt_test @@ -2439,7 +2520,7 @@ def shortDescription(self): class SkipDocTestCase(DocTestCase): def __init__(self, module): self.module = module - DocTestCase.__init__(self, None) + super().__init__(None) def setUp(self): self.skipTest("DocTestSuite will not work with -O2 and above") diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9a51b943733..cbff9694742 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -796,6 +796,10 @@ def params(self): value = urllib.parse.unquote(value, encoding='latin-1') else: try: + # Explicitly look up the codec for warning generation, see gh-140030 + # Can be removed in 3.17 + import codecs + codecs.lookup(charset) value = value.decode(charset, 'surrogateescape') except (LookupError, UnicodeEncodeError): # XXX: there should really be a custom defect for @@ -1020,6 +1024,8 @@ def _get_ptext_to_endchars(value, endchars): a flag that is True iff there were any quoted printables decoded. """ + if not value: + return '', '', False fragment, *remainder = _wsp_splitter(value, 1) vchars = [] escape = False @@ -1573,7 +1579,7 @@ def get_dtext(value): def _check_for_early_dl_end(value, domain_literal): if value: return False - domain_literal.append(errors.InvalidHeaderDefect( + domain_literal.defects.append(errors.InvalidHeaderDefect( "end of input inside domain-literal")) domain_literal.append(ValueTerminal(']', 'domain-literal-end')) return True @@ -1592,9 +1598,9 @@ def get_domain_literal(value): raise errors.HeaderParseError("expected '[' at start of domain-literal " "but found '{}'".format(value)) value = value[1:] + domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if _check_for_early_dl_end(value, domain_literal): return domain_literal, value - domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if value[0] in WSP: token, value = get_fws(value) domain_literal.append(token) @@ -2786,6 +2792,9 @@ def _steal_trailing_WSP_if_exists(lines): if lines and lines[-1] and lines[-1][-1] in WSP: wsp = lines[-1][-1] lines[-1] = lines[-1][:-1] + # gh-142006: if the line is now empty, remove it entirely. + if not lines[-1]: + lines.pop() return wsp def _refold_parse_tree(parse_tree, *, policy): diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 84917038874..6a7c5fa06d2 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -146,8 +146,9 @@ def _parsedate_tz(data): return None # Check for a yy specified in two-digit format, then convert it to the # appropriate four-digit format, according to the POSIX standard. RFC 822 - # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) - # mandates a 4-digit yy. For more information, see the documentation for + # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) already + # mandated a 4-digit yy, and RFC 5322 (which obsoletes RFC 2822) continues + # this requirement. For more information, see the documentation for # the time module. if yy < 100: # The year is between 1969 and 1999 (inclusive). @@ -233,9 +234,11 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies '.' as obs-phrase, meaning that it - # is obsolete syntax. RFC 2822 requires that we recognize obsolete - # syntax, so allow dots in phrases. + # Note that RFC 2822 section 4.1 introduced '.' as obs-phrase to handle + # existing practice (periods in display names), even though it was not + # allowed in RFC 822. RFC 5322 section 4.1 (which obsoletes RFC 2822) + # continues this requirement. We must recognize obsolete syntax, so + # allow dots in phrases. self.phraseends = self.atomends.replace('.', '') self.field = field self.commentlist = [] diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index 95e79b8938b..e23843df448 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -380,7 +380,7 @@ def _fold(self, name, value, sanitize): h = value if h is not None: # The Header class interprets a value of None for maxlinelen as the - # default value of 78, as recommended by RFC 2822. + # default value of 78, as recommended by RFC 5322 section 2.1.1. maxlinelen = 0 if self.max_line_length is not None: maxlinelen = self.max_line_length diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index b4f5830bead..11d1536db27 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -2,6 +2,7 @@ import email.charset import email.message import email.errors +import sys from email import quoprimime class ContentManager: @@ -142,13 +143,15 @@ def _encode_base64(data, max_line_length): def _encode_text(string, charset, cte, policy): + # If max_line_length is 0 or None, there is no limit. + maxlen = policy.max_line_length or sys.maxsize lines = string.encode(charset).splitlines() linesep = policy.linesep.encode('ascii') def embedded_body(lines): return linesep.join(lines) + linesep def normal_body(lines): return b'\n'.join(lines) + b'\n' if cte is None: # Use heuristics to decide on the "best" encoding. - if max((len(x) for x in lines), default=0) <= policy.max_line_length: + if max(map(len, lines), default=0) <= maxlen: try: return '7bit', normal_body(lines).decode('ascii') except UnicodeDecodeError: @@ -156,8 +159,7 @@ def normal_body(lines): return b'\n'.join(lines) + b'\n' if policy.cte_type == '8bit': return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') sniff = embedded_body(lines[:10]) - sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), - policy.max_line_length) + sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), maxlen) sniff_base64 = binascii.b2a_base64(sniff) # This is a little unfair to qp; it includes lineseps, base64 doesn't. if len(sniff_qp) > len(sniff_base64): @@ -172,9 +174,9 @@ def normal_body(lines): return b'\n'.join(lines) + b'\n' data = normal_body(lines).decode('ascii', 'surrogateescape') elif cte == 'quoted-printable': data = quoprimime.body_encode(normal_body(lines).decode('latin-1'), - policy.max_line_length) + maxlen) elif cte == 'base64': - data = _encode_base64(embedded_body(lines), policy.max_line_length) + data = _encode_base64(embedded_body(lines), maxlen) else: raise ValueError("Unknown content transfer encoding {}".format(cte)) return cte, data diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 9d80a5822af..ae8ef32792b 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -32,7 +32,7 @@ NLCRE_bol = re.compile(r'(\r\n|\r|\n)') NLCRE_eol = re.compile(r'(\r\n|\r|\n)\z') NLCRE_crack = re.compile(r'(\r\n|\r|\n)') -# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character +# RFC 5322 section 3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character # except controls, SP, and ":". headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') EMPTYSTRING = '' @@ -294,7 +294,7 @@ def _parsegen(self): return if self._cur.get_content_maintype() == 'message': # The message claims to be a message/* type, then what follows is - # another RFC 2822 message. + # another RFC 5322 message. for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData @@ -504,10 +504,9 @@ def _parse_headers(self, lines): self._input.unreadline(line) return else: - # Weirdly placed unix-from line. Note this as a defect - # and ignore it. + # Weirdly placed unix-from line. defect = errors.MisplacedEnvelopeHeaderDefect(line) - self._cur.defects.append(defect) + self.policy.handle_defect(self._cur, defect) continue # Split the line on the colon separating field name from value. # There will always be a colon, because if there wasn't the part of @@ -519,7 +518,7 @@ def _parse_headers(self, lines): # message. Track the error but keep going. if i == 0: defect = errors.InvalidHeaderDefect("Missing header name.") - self._cur.defects.append(defect) + self.policy.handle_defect(self._cur, defect) continue assert i>0, "_parse_headers fed line with no : and no leading WS" diff --git a/Lib/email/generator.py b/Lib/email/generator.py index ab5bd0653e4..03524c96559 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -50,7 +50,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, expanded to 8 spaces) than maxheaderlen, the header will split as defined in the Header class. Set maxheaderlen to zero to disable header wrapping. The default is 78, as recommended (but not required) - by RFC 2822. + by RFC 5322 section 2.1.1. The policy keyword specifies a policy object that controls a number of aspects of the generator's operation. If no policy is specified, diff --git a/Lib/email/header.py b/Lib/email/header.py index 113a81f4131..220a84a7454 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -59,16 +59,22 @@ def decode_header(header): """Decode a message header value without converting charset. - Returns a list of (string, charset) pairs containing each of the decoded - parts of the header. Charset is None for non-encoded parts of the header, - otherwise a lower-case string containing the name of the character set - specified in the encoded string. + For historical reasons, this function may return either: + + 1. A list of length 1 containing a pair (str, None). + 2. A list of (bytes, charset) pairs containing each of the decoded + parts of the header. Charset is None for non-encoded parts of the header, + otherwise a lower-case string containing the name of the character set + specified in the encoded string. header may be a string that may or may not contain RFC2047 encoded words, or it may be a Header object. An email.errors.HeaderParseError may be raised when certain decoding error occurs (e.g. a base64 decoding exception). + + This function exists for backwards compatibility only. For new code, we + recommend using email.headerregistry.HeaderRegistry instead. """ # If it is a Header object, we can just return the encoded chunks. if hasattr(header, '_chunks'): @@ -161,6 +167,9 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, This function takes one of those sequence of pairs and returns a Header instance. Optional maxlinelen, header_name, and continuation_ws are as in the Header constructor. + + This function exists for backwards compatibility only, and is not + recommended for use in new code. """ h = Header(maxlinelen=maxlinelen, header_name=header_name, continuation_ws=continuation_ws) diff --git a/Lib/email/message.py b/Lib/email/message.py index 87fcab68868..641fb2e944d 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -74,19 +74,25 @@ def _parseparam(s): # RDM This might be a Header, so for now stringify it. s = ';' + str(s) plist = [] - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) + start = 0 + while s.find(';', start) == start: + start += 1 + end = s.find(';', start) + ind, diff = start, 0 + while end > 0: + diff += s.count('"', ind, end) - s.count('\\"', ind, end) + if diff % 2 == 0: + break + end, ind = ind, s.find(';', end + 1) if end < 0: end = len(s) - f = s[:end] - if '=' in f: - i = f.index('=') - f = f[:i].strip().lower() + '=' + f[i+1:].strip() + i = s.find('=', start, end) + if i == -1: + f = s[start:end] + else: + f = s[start:i].rstrip().lower() + '=' + s[i+1:end].lstrip() plist.append(f.strip()) - s = s[end:] + start = end return plist @@ -135,7 +141,7 @@ def _decode_uu(encoded): class Message: """Basic message object. - A message object is defined as something that has a bunch of RFC 2822 + A message object is defined as something that has a bunch of RFC 5322 headers and a payload. It may optionally have an envelope header (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a multipart or a message/rfc822), then the payload is a list of Message @@ -313,6 +319,8 @@ def get_payload(self, i=None, decode=False): # If it does happen, turn the string into bytes in a way # guaranteed not to fail. bpayload = payload.encode('raw-unicode-escape') + else: + bpayload = payload if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': @@ -564,7 +572,7 @@ def add_header(self, _name, _value, **_params): msg.add_header('content-disposition', 'attachment', filename='bud.gif') msg.add_header('content-disposition', 'attachment', - filename=('utf-8', '', Fußballer.ppt')) + filename=('utf-8', '', 'Fußballer.ppt')) msg.add_header('content-disposition', 'attachment', filename='Fußballer.ppt')) """ diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 039f03cba74..c6a51dd8e37 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -2,7 +2,7 @@ # Author: Barry Warsaw, Thomas Wouters, Anthony Baxter # Contact: email-sig@python.org -"""A parser of RFC 2822 and MIME email messages.""" +"""A parser of RFC 5322 and MIME email messages.""" __all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser', 'FeedParser', 'BytesFeedParser'] @@ -15,13 +15,13 @@ class Parser: def __init__(self, _class=None, *, policy=compat32): - """Parser of RFC 2822 and MIME email messages. + """Parser of RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The string must be formatted as a block of RFC 2822 headers and header + The string must be formatted as a block of RFC 5322 headers and header continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -75,13 +75,13 @@ def parsestr(self, text, headersonly=True): class BytesParser: def __init__(self, *args, **kw): - """Parser of binary RFC 2822 and MIME email messages. + """Parser of binary RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The input must be formatted as a block of RFC 2822 headers and header + The input must be formatted as a block of RFC 5322 headers and header continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/email/utils.py b/Lib/email/utils.py index 7eab74dc0db..d4824dc3601 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -417,8 +417,14 @@ def decode_params(params): for name, continuations in rfc2231_params.items(): value = [] extended = False - # Sort by number - continuations.sort() + # Sort by number, treating None as 0 if there is no 0, + # and ignore it if there is already a 0. + has_zero = any(x[0] == 0 for x in continuations) + if has_zero: + continuations = [x for x in continuations if x[0] is not None] + else: + continuations = [(x[0] or 0, x[1], x[2]) for x in continuations] + continuations.sort(key=lambda x: x[0]) # And now append all values in numerical order, converting # %-encodings for the encoded segments. If any of the # continuation names ends in a *, then the entire string, after @@ -454,6 +460,10 @@ def collapse_rfc2231_value(value, errors='replace', charset = fallback_charset rawbytes = bytes(text, 'raw-unicode-escape') try: + # Explicitly look up the codec for warning generation, see gh-140030 + # Can be removed in 3.17 + import codecs + codecs.lookup(charset) return str(rawbytes, charset, errors) except LookupError: # charset is not a known codec. diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index 298177eb800..e205ec32637 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -26,10 +26,11 @@ (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. -"""#" +""" import codecs import sys +from _codecs import _normalize_encoding from . import aliases _cache = {} @@ -55,18 +56,13 @@ def normalize_encoding(encoding): if isinstance(encoding, bytes): encoding = str(encoding, "ascii") - chars = [] - punct = False - for c in encoding: - if c.isalnum() or c == '.': - if punct and chars: - chars.append('_') - if c.isascii(): - chars.append(c) - punct = False - else: - punct = True - return ''.join(chars) + if not encoding.isascii(): + import warnings + warnings.warn( + "Support for non-ascii encoding names will be removed in 3.17", + DeprecationWarning, stacklevel=2) + + return _normalize_encoding(encoding) def search_function(encoding): diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index a94bb270671..f4b1b8dd43f 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -17,7 +17,7 @@ """ aliases = { - # Please keep this list sorted alphabetically by value ! + # Please keep this list sorted alphabetically by value! # ascii codec '646' : 'ascii', @@ -318,6 +318,7 @@ 'iso_ir_157' : 'iso8859_10', 'l6' : 'iso8859_10', 'latin6' : 'iso8859_10', + 'latin_6' : 'iso8859_10', # iso8859_11 codec 'thai' : 'iso8859_11', @@ -328,6 +329,7 @@ 'iso_8859_13' : 'iso8859_13', 'l7' : 'iso8859_13', 'latin7' : 'iso8859_13', + 'latin_7' : 'iso8859_13', # iso8859_14 codec 'iso_8859_14' : 'iso8859_14', @@ -336,11 +338,13 @@ 'iso_ir_199' : 'iso8859_14', 'l8' : 'iso8859_14', 'latin8' : 'iso8859_14', + 'latin_8' : 'iso8859_14', # iso8859_15 codec 'iso_8859_15' : 'iso8859_15', 'l9' : 'iso8859_15', 'latin9' : 'iso8859_15', + 'latin_9' : 'iso8859_15', # iso8859_16 codec 'iso_8859_16' : 'iso8859_16', @@ -348,6 +352,7 @@ 'iso_ir_226' : 'iso8859_16', 'l10' : 'iso8859_16', 'latin10' : 'iso8859_16', + 'latin_10' : 'iso8859_16', # iso8859_2 codec 'csisolatin2' : 'iso8859_2', @@ -356,6 +361,7 @@ 'iso_ir_101' : 'iso8859_2', 'l2' : 'iso8859_2', 'latin2' : 'iso8859_2', + 'latin_2' : 'iso8859_2', # iso8859_3 codec 'csisolatin3' : 'iso8859_3', @@ -364,6 +370,7 @@ 'iso_ir_109' : 'iso8859_3', 'l3' : 'iso8859_3', 'latin3' : 'iso8859_3', + 'latin_3' : 'iso8859_3', # iso8859_4 codec 'csisolatin4' : 'iso8859_4', @@ -372,6 +379,7 @@ 'iso_ir_110' : 'iso8859_4', 'l4' : 'iso8859_4', 'latin4' : 'iso8859_4', + 'latin_4' : 'iso8859_4', # iso8859_5 codec 'csisolatincyrillic' : 'iso8859_5', @@ -405,6 +413,8 @@ 'iso_8859_8' : 'iso8859_8', 'iso_8859_8_1988' : 'iso8859_8', 'iso_ir_138' : 'iso8859_8', + 'iso_8859_8_i' : 'iso8859_8', + 'iso_8859_8_e' : 'iso8859_8', # iso8859_9 codec 'csisolatin5' : 'iso8859_9', @@ -413,6 +423,7 @@ 'iso_ir_148' : 'iso8859_9', 'l5' : 'iso8859_9', 'latin5' : 'iso8859_9', + 'latin_5' : 'iso8859_9', # johab codec 'cp1361' : 'johab', @@ -543,6 +554,9 @@ 'utf8_ucs4' : 'utf_8', 'cp65001' : 'utf_8', + # utf_8_sig codec + 'utf8_sig' : 'utf_8_sig', + # uu_codec codec 'uu' : 'uu_codec', diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 60a8d5eb227..d31ee07ab45 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -226,7 +226,8 @@ def encode(self, input, errors='strict'): offset + exc.end, exc.reason, ) - return bytes(result+trailing_dot), len(input) + result += trailing_dot + return result.take_bytes(), len(input) def decode(self, input, errors='strict'): @@ -311,12 +312,12 @@ def _buffer_encode(self, input, errors, final): result += trailing_dot size += len(trailing_dot) - return (bytes(result), size) + return (result.take_bytes(), size) class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling: {errors}") + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return ("", 0) diff --git a/Lib/encodings/palmos.py b/Lib/encodings/palmos.py index c506d654523..df164ca5b95 100644 --- a/Lib/encodings/palmos.py +++ b/Lib/encodings/palmos.py @@ -201,7 +201,7 @@ def getregentry(): '\u02dc' # 0x98 -> SMALL TILDE '\u2122' # 0x99 -> TRADE MARK SIGN '\u0161' # 0x9A -> LATIN SMALL LETTER S WITH CARON - '\x9b' # 0x9B -> + '\u203a' # 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK '\u0153' # 0x9C -> LATIN SMALL LIGATURE OE '\x9d' # 0x9D -> '\x9e' # 0x9E -> diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 4622fc8c920..268fccbd539 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -17,7 +17,7 @@ def segregate(str): else: extended.add(c) extended = sorted(extended) - return bytes(base), extended + return base.take_bytes(), extended def selective_len(str, max): """Return the length of str, considering only characters below max.""" @@ -83,7 +83,7 @@ def generate_generalized_integer(N, bias): t = T(j, bias) if N < t: result.append(digits[N]) - return bytes(result) + return result.take_bytes() result.append(digits[t + ((N - t) % (36 - t))]) N = (N - t) // (36 - t) j += 1 @@ -112,7 +112,7 @@ def generate_integers(baselen, deltas): s = generate_generalized_integer(delta, bias) result.extend(s) bias = adapt(delta, points==0, baselen+points+1) - return bytes(result) + return result.take_bytes() def punycode_encode(text): base, extended = segregate(text) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index aa641e94a8b..f9f905f46ff 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "25.1.1" +_PIP_VERSION = "25.3" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora @@ -130,6 +130,15 @@ def _bootstrap(*, root=None, upgrade=False, user=False, Note that calling this function will alter both sys.path and os.environ. """ + + try: + import zlib + except ImportError: + raise ModuleNotFoundError( + "ensurepip requires the standard library module 'zlib' " + "to install pip." + ) from None + if altinstall and default_pip: raise ValueError("Cannot use altinstall and default_pip together") diff --git a/Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl similarity index 57% rename from Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl index 2fdcfbf9ff8..755e1aa0c3d 100644 Binary files a/Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 01fecca3e5a..ad782b8c41e 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -10,14 +10,15 @@ 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', 'global_flag_repr', 'global_enum_repr', 'global_str', 'global_enum', 'EnumCheck', 'CONTINUOUS', 'NAMED_FLAGS', 'UNIQUE', - 'pickle_by_global_name', 'pickle_by_enum_name', + 'pickle_by_global_name', 'pickle_by_enum_name', 'show_flag_values', + 'bin', ] # Dummy value for Enum and Flag as there are explicit checks for them # before they have been created. # This is also why there are checks in EnumType like `if Enum is not None` -Enum = Flag = EJECT = _stdlib_enums = ReprEnum = None +Enum = Flag = EJECT = ReprEnum = None class nonmember(object): """ @@ -535,7 +536,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) # - # Flag structures (will be removed if final class is not a Flag + # Flag structures (will be removed if final class is not a Flag) classdict['_boundary_'] = ( boundary or getattr(first_enum, '_boundary_', None) @@ -544,6 +545,29 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_singles_mask_'] = 0 classdict['_all_bits_'] = 0 classdict['_inverted_'] = None + # check for negative flag values and invert if found (using _proto_members) + if Flag is not None and bases and issubclass(bases[-1], Flag): + bits = 0 + inverted = [] + for n in member_names: + p = classdict[n] + if isinstance(p.value, int): + if p.value < 0: + inverted.append(p) + else: + bits |= p.value + elif p.value is None: + pass + elif isinstance(p.value, tuple) and p.value and isinstance(p.value[0], int): + if p.value[0] < 0: + inverted.append(p) + else: + bits |= p.value[0] + for p in inverted: + if isinstance(p.value, int): + p.value = bits & p.value + else: + p.value = (bits & p.value[0], ) + p.value[1:] try: classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) @@ -1487,7 +1511,10 @@ def _missing_(cls, value): ) if value < 0: neg_value = value - value = all_bits + 1 + value + if cls._boundary_ in (EJECT, KEEP): + value = all_bits + 1 + value + else: + value = singles_mask & value # get members and unknown unknown = value & ~flag_mask aliases = value & ~singles_mask @@ -1965,7 +1992,7 @@ def __call__(self, enumeration): if 2**i not in values: missing.append(2**i) elif enum_type == 'enum': - # check for powers of one + # check for missing consecutive integers for i in range(low+1, high): if i not in values: missing.append(i) @@ -2163,5 +2190,3 @@ def _old_convert_(etype, name, module, filter, source=None, *, boundary=None): members.sort(key=lambda t: t[0]) cls = etype(name, members, module=module, boundary=boundary or KEEP) return cls - -_stdlib_enums = IntEnum, StrEnum, IntFlag diff --git a/Lib/fractions.py b/Lib/fractions.py index 8163e3bb594..c1b12e7a1c0 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -168,9 +168,13 @@ def _round_to_figures(n, d, figures): # A '0' that's *not* followed by another digit is parsed as a minimum width # rather than a zeropad flag. (?P0(?=[0-9]))? - (?P0|[1-9][0-9]*)? + (?P[0-9]+)? (?P[,_])? - (?:\.(?P0|[1-9][0-9]*))? + (?:\. + (?=[,_0-9]) # lookahead for digit or separator + (?P[0-9]+)? + (?P[,_])? + )? (?P[eEfFgG%]) """, re.DOTALL | re.VERBOSE).fullmatch @@ -238,11 +242,6 @@ def __new__(cls, numerator=0, denominator=None): self._denominator = 1 return self - elif isinstance(numerator, numbers.Rational): - self._numerator = numerator.numerator - self._denominator = numerator.denominator - return self - elif (isinstance(numerator, float) or (not isinstance(numerator, type) and hasattr(numerator, 'as_integer_ratio'))): @@ -278,6 +277,11 @@ def __new__(cls, numerator=0, denominator=None): if m.group('sign') == '-': numerator = -numerator + elif isinstance(numerator, numbers.Rational): + self._numerator = numerator.numerator + self._denominator = numerator.denominator + return self + else: raise TypeError("argument should be a string or a Rational " "instance or have the as_integer_ratio() method") @@ -499,11 +503,15 @@ def _format_float_style(self, match): minimumwidth = int(match["minimumwidth"] or "0") thousands_sep = match["thousands_sep"] precision = int(match["precision"] or "6") + frac_sep = match["frac_separators"] or "" presentation_type = match["presentation_type"] trim_zeros = presentation_type in "gG" and not alternate_form trim_point = not alternate_form exponent_indicator = "E" if presentation_type in "EFG" else "e" + if align == '=' and fill == '0': + zeropad = True + # Round to get the digits we need, figure out where to place the point, # and decide whether to use scientific notation. 'point_pos' is the # relative to the _end_ of the digit string: that is, it's the number @@ -552,6 +560,9 @@ def _format_float_style(self, match): if trim_zeros: frac_part = frac_part.rstrip("0") separator = "" if trim_point and not frac_part else "." + if frac_sep: + frac_part = frac_sep.join(frac_part[pos:pos + 3] + for pos in range(0, len(frac_part), 3)) trailing = separator + frac_part + suffix # Do zero padding if required. diff --git a/Lib/functools.py b/Lib/functools.py index 714070c6ac9..8063eb5ffc3 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -323,6 +323,9 @@ def _partial_new(cls, func, /, *args, **keywords): "or a descriptor") if args and args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") + for value in keywords.values(): + if value is Placeholder: + raise TypeError("Placeholder cannot be passed as a keyword argument") if isinstance(func, base_cls): pto_phcount = func._phcount tot_args = func.args @@ -577,12 +580,14 @@ def lru_cache(maxsize=128, typed=False): # Negative maxsize is treated as 0 if maxsize < 0: maxsize = 0 + elif callable(maxsize) and isinstance(typed, bool): # The user_function was passed in directly via the maxsize argument user_function, maxsize = maxsize, 128 wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) wrapper.cache_parameters = lambda : {'maxsize': maxsize, 'typed': typed} return update_wrapper(wrapper, user_function) + elif maxsize is not None: raise TypeError( 'Expected first argument to be an integer, a callable, or None') @@ -614,6 +619,7 @@ def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): def wrapper(*args, **kwds): # No caching -- just a statistics update nonlocal misses + misses += 1 result = user_function(*args, **kwds) return result @@ -623,6 +629,7 @@ def wrapper(*args, **kwds): def wrapper(*args, **kwds): # Simple caching without ordering or size limit nonlocal hits, misses + key = make_key(args, kwds, typed) result = cache_get(key, sentinel) if result is not sentinel: @@ -638,7 +645,9 @@ def wrapper(*args, **kwds): def wrapper(*args, **kwds): # Size limited caching that tracks accesses by recency nonlocal root, hits, misses, full + key = make_key(args, kwds, typed) + with lock: link = cache_get(key) if link is not None: @@ -653,7 +662,9 @@ def wrapper(*args, **kwds): hits += 1 return result misses += 1 + result = user_function(*args, **kwds) + with lock: if key in cache: # Getting here means that this same key was added to the @@ -661,11 +672,13 @@ def wrapper(*args, **kwds): # update is already done, we need only return the # computed result and update the count of misses. pass + elif full: # Use the old root to store the new key and result. oldroot = root oldroot[KEY] = key oldroot[RESULT] = result + # Empty the oldest link and make it the new root. # Keep a reference to the old key and old result to # prevent their ref counts from going to zero during the @@ -676,20 +689,25 @@ def wrapper(*args, **kwds): oldkey = root[KEY] oldresult = root[RESULT] root[KEY] = root[RESULT] = None + # Now update the cache dictionary. del cache[oldkey] + # Save the potentially reentrant cache[key] assignment # for last, after the root and links have been put in # a consistent state. cache[key] = oldroot + else: # Put result in a new link at the front of the queue. last = root[PREV] link = [last, root, key, result] last[NEXT] = root[PREV] = cache[key] = link + # Use the cache_len bound method instead of the len() function # which could potentially be wrapped in an lru_cache itself. full = (cache_len() >= maxsize) + return result def cache_info(): @@ -700,6 +718,7 @@ def cache_info(): def cache_clear(): """Clear the cache and cache statistics""" nonlocal hits, misses, full + with lock: cache.clear() root[:] = [root, root, None, None] @@ -1064,7 +1083,10 @@ def __call__(self, /, *args, **kwargs): 'singledispatchmethod method') raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return self._dispatch(args[0].__class__).__get__(self._obj, self._cls)(*args, **kwargs) + method = self._dispatch(args[0].__class__) + if hasattr(method, "__get__"): + method = method.__get__(self._obj, self._cls) + return method(*args, **kwargs) def __getattr__(self, name): # Resolve these attributes lazily to speed up creation of diff --git a/Lib/genericpath.py b/Lib/genericpath.py index ba7b0a13c7f..7588fe5e802 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -8,7 +8,8 @@ __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', 'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink', - 'lexists', 'samefile', 'sameopenfile', 'samestat'] + 'lexists', 'samefile', 'sameopenfile', 'samestat', + 'ALL_BUT_LAST', 'ALLOW_MISSING'] # Does a path exist? @@ -81,28 +82,28 @@ def isdevdrive(path): return False -def getsize(filename): +def getsize(filename, /): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size -def getmtime(filename): +def getmtime(filename, /): """Return the last modification time of a file, reported by os.stat().""" return os.stat(filename).st_mtime -def getatime(filename): +def getatime(filename, /): """Return the last access time of a file, reported by os.stat().""" return os.stat(filename).st_atime -def getctime(filename): +def getctime(filename, /): """Return the metadata change time of a file, reported by os.stat().""" return os.stat(filename).st_ctime # Return the longest prefix of all list elements. -def commonprefix(m): +def commonprefix(m, /): "Given a list of pathnames, returns the longest common leading component" if not m: return '' # Some people pass in a list of pathname parts to operate in an OS-agnostic @@ -120,14 +121,14 @@ def commonprefix(m): # Are two stat buffers (obtained from stat, fstat or lstat) # describing the same file? -def samestat(s1, s2): +def samestat(s1, s2, /): """Test whether two stat buffers reference the same file""" return (s1.st_ino == s2.st_ino and s1.st_dev == s2.st_dev) # Are two filenames really pointing to the same file? -def samefile(f1, f2): +def samefile(f1, f2, /): """Test whether two pathnames reference the same actual file or directory This is determined by the device number and i-node number and @@ -189,3 +190,22 @@ def _check_arg_types(funcname, *args): f'os.PathLike object, not {s.__class__.__name__!r}') from None if hasstr and hasbytes: raise TypeError("Can't mix strings and bytes in path components") from None + + +# Singletons with a true boolean value. + +@object.__new__ +class ALL_BUT_LAST: + """Special value for use in realpath().""" + def __repr__(self): + return 'os.path.ALL_BUT_LAST' + def __reduce__(self): + return self.__class__.__name__ + +@object.__new__ +class ALLOW_MISSING: + """Special value for use in realpath().""" + def __repr__(self): + return 'os.path.ALLOW_MISSING' + def __reduce__(self): + return self.__class__.__name__ diff --git a/Lib/getpass.py b/Lib/getpass.py index f571425e541..3d9bb1f0d14 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -33,8 +33,8 @@ def unix_getpass(prompt='Password: ', stream=None, *, echo_char=None): prompt: Written on stream to ask for the input. Default: 'Password: ' stream: A writable file object to display the prompt. Defaults to the tty. If no tty is available defaults to sys.stderr. - echo_char: A string used to mask input (e.g., '*'). If None, input is - hidden. + echo_char: A single ASCII character to mask input (e.g., '*'). + If None, input is hidden. Returns: The seKr3t input. Raises: @@ -119,9 +119,9 @@ def win_getpass(prompt='Password: ', stream=None, *, echo_char=None): raise KeyboardInterrupt if c == '\b': if echo_char and pw: - msvcrt.putch('\b') - msvcrt.putch(' ') - msvcrt.putch('\b') + msvcrt.putwch('\b') + msvcrt.putwch(' ') + msvcrt.putwch('\b') pw = pw[:-1] else: pw = pw + c @@ -132,21 +132,31 @@ def win_getpass(prompt='Password: ', stream=None, *, echo_char=None): return pw -def fallback_getpass(prompt='Password: ', stream=None): +def fallback_getpass(prompt='Password: ', stream=None, *, echo_char=None): + _check_echo_char(echo_char) import warnings warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: stream = sys.stderr print("Warning: Password input may be echoed.", file=stream) - return _raw_input(prompt, stream) + return _raw_input(prompt, stream, echo_char=echo_char) def _check_echo_char(echo_char): - # ASCII excluding control characters - if echo_char and not (echo_char.isprintable() and echo_char.isascii()): - raise ValueError("'echo_char' must be a printable ASCII string, " - f"got: {echo_char!r}") + # Single-character ASCII excluding control characters + if echo_char is None: + return + if not isinstance(echo_char, str): + raise TypeError("'echo_char' must be a str or None, not " + f"{type(echo_char).__name__}") + if not ( + len(echo_char) == 1 + and echo_char.isprintable() + and echo_char.isascii() + ): + raise ValueError("'echo_char' must be a single printable ASCII " + f"character, got: {echo_char!r}") def _raw_input(prompt="", stream=None, input=None, echo_char=None): diff --git a/Lib/glob.py b/Lib/glob.py index 341524282ba..c2f8ce279ab 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -22,6 +22,9 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, dot are special cases that are not matched by '*' and '?' patterns by default. + The order of the returned list is undefined. Sort it if you need a + particular order. + If `include_hidden` is true, the patterns '*', '?', '**' will match hidden directories. @@ -40,6 +43,9 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, dot are special cases that are not matched by '*' and '?' patterns. + The order of the returned paths is undefined. Sort them if you need a + particular order. + If recursive is true, the pattern '**' will match any files and zero or more directories and subdirectories. """ @@ -122,21 +128,6 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -_deprecated_function_message = ( - "{name} is deprecated and will be removed in Python {remove}. Use " - "glob.glob and pass a directory to its root_dir argument instead." -) - -def glob0(dirname, pattern): - import warnings - warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) - return _glob0(dirname, pattern, None, False) - -def glob1(dirname, pattern): - import warnings - warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) - return _glob1(dirname, pattern, None, False) - # This helper function recursively yields relative pathnames inside a literal # directory. @@ -358,6 +349,12 @@ def concat_path(path, text): """ raise NotImplementedError + @staticmethod + def stringify_path(path): + """Converts the path to a string object + """ + raise NotImplementedError + # High-level methods def compile(self, pat, altsep=None): @@ -466,8 +463,9 @@ def recursive_selector(self, part, parts): select_next = self.selector(parts) def select_recursive(path, exists=False): - match_pos = len(str(path)) - if match is None or match(str(path), match_pos): + path_str = self.stringify_path(path) + match_pos = len(path_str) + if match is None or match(path_str, match_pos): yield from select_next(path, exists) stack = [path] while stack: @@ -489,7 +487,7 @@ def select_recursive_step(stack, match_pos): pass if is_dir or not dir_only: - entry_path_str = str(entry_path) + entry_path_str = self.stringify_path(entry_path) if dir_only: entry_path = self.concat_path(entry_path, self.sep) if match is None or match(entry_path_str, match_pos): @@ -529,19 +527,6 @@ def scandir(path): entries = list(scandir_it) return ((entry, entry.name, entry.path) for entry in entries) - -class _PathGlobber(_GlobberBase): - """Provides shell-style pattern matching and globbing for pathlib paths. - """ - @staticmethod - def lexists(path): - return path.info.exists(follow_symlinks=False) - - @staticmethod - def scandir(path): - return ((child.info, child.name, child) for child in path.iterdir()) - - @staticmethod - def concat_path(path, text): - return path.with_segments(str(path) + text) + def stringify_path(path): + return path # Already a string. diff --git a/Lib/gzip.py b/Lib/gzip.py index c00f51858de..8a2faf846bf 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -30,7 +30,7 @@ _WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE -def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, +def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF, encoding=None, errors=None, newline=None): """Open a gzip-compressed file in binary or text mode. @@ -158,7 +158,7 @@ class GzipFile(_streams.BaseStream): myfileobj = None def __init__(self, filename=None, mode=None, - compresslevel=_COMPRESS_LEVEL_BEST, fileobj=None, mtime=None): + compresslevel=_COMPRESS_LEVEL_TRADEOFF, fileobj=None, mtime=None): """Constructor for the GzipFile class. At least one of fileobj and filename must be given a @@ -268,8 +268,6 @@ def _init_write(self, filename): self.name = filename self.crc = zlib.crc32(b"") self.size = 0 - self.writebuf = [] - self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc def tell(self): @@ -621,7 +619,7 @@ def _rewind(self): self._new_member = True -def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0): +def compress(data, compresslevel=_COMPRESS_LEVEL_TRADEOFF, *, mtime=0): """Compress data in one shot and return the compressed string. compresslevel sets the compression level in range of 0-9. diff --git a/Lib/hashlib.py b/Lib/hashlib.py index abacac22ea0..6c73eb9f31f 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -2,7 +2,7 @@ # Licensed to PSF under a Contributor Agreement. # -__doc__ = """hashlib module - A common interface to many hash functions. +__doc__ = r"""hashlib module - A common interface to many hash functions. new(name, data=b'', **kwargs) - returns a new hash object implementing the given hash function; initializing the hash @@ -12,7 +12,7 @@ than using new(name): md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), blake2s(), -sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256. +sha3_224(), sha3_256(), sha3_384(), sha3_512(), shake_128(), and shake_256(). More algorithms may be available on your platform but the above are guaranteed to exist. See the algorithms_guaranteed and algorithms_available attributes @@ -21,8 +21,8 @@ NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. -Choose your hash function wisely. Some have known collision weaknesses. -sha384 and sha512 will be slow on 32 bit platforms. +Choose your hash function wisely. Some have known collision weaknesses, +while others may be slower depending on the CPU architecture. Hash objects have these methods: - update(data): Update the hash object with the bytes in data. Repeated calls @@ -36,21 +36,21 @@ efficiently compute the digests of data that share a common initial substring. -For example, to obtain the digest of the byte string 'Nobody inspects the -spammish repetition': +Assuming that Python has been built with SHA-2 support, the SHA-256 digest +of the byte string b'Nobody inspects the spammish repetition' is computed +as follows: >>> import hashlib - >>> m = hashlib.md5() + >>> m = hashlib.sha256() >>> m.update(b"Nobody inspects") >>> m.update(b" the spammish repetition") - >>> m.digest() - b'\\xbbd\\x9c\\x83\\xdd\\x1e\\xa5\\xc9\\xd9\\xde\\xc9\\xa1\\x8d\\xf0\\xff\\xe9' + >>> m.digest() # doctest: +ELLIPSIS + b'\x03\x1e\xdd}Ae\x15\x93\xc5\xfe\\\x00o\xa5u+7...' More condensed: - >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest() - 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' - + >>> hashlib.sha256(b"Nobody inspects the spammish repetition").hexdigest() + '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' """ # This tuple and __get_builtin_constructor() must be modified if a new @@ -80,6 +80,11 @@ } def __get_builtin_constructor(name): + if not isinstance(name, str): + # Since this function is only used by new(), we use the same + # exception as _hashlib.new() when 'name' is of incorrect type. + err = f"new() argument 'name' must be str, not {type(name).__name__}" + raise TypeError(err) cache = __builtin_constructor_cache constructor = cache.get(name) if constructor is not None: @@ -120,20 +125,33 @@ def __get_builtin_constructor(name): if constructor is not None: return constructor - raise ValueError('unsupported hash type ' + name) + # Keep the message in sync with hashlib.h::HASHLIB_UNSUPPORTED_ALGORITHM. + raise ValueError(f'unsupported hash algorithm {name}') def __get_openssl_constructor(name): + # This function is only used until the module has been initialized. + assert isinstance(name, str), "invalid call to __get_openssl_constructor()" if name in __block_openssl_constructor: # Prefer our builtin blake2 implementation. return __get_builtin_constructor(name) try: - # MD5, SHA1, and SHA2 are in all supported OpenSSL versions - # SHA3/shake are available in OpenSSL 1.1.1+ + # Fetch the OpenSSL hash function if it exists, + # independently of the context security policy. f = getattr(_hashlib, 'openssl_' + name) - # Allow the C module to raise ValueError. The function will be - # defined but the hash not actually available. Don't fall back to - # builtin if the current security policy blocks a digest, bpo#40695. + # Check if the context security policy blocks the digest or not + # by allowing the C module to raise a ValueError. The function + # will be defined but the hash will not be available at runtime. + # + # We use "usedforsecurity=False" to prevent falling back to the + # built-in function in case the security policy does not allow it. + # + # Note that this only affects the explicit named constructors, + # and not the algorithms exposed through hashlib.new() which + # can still be resolved to a built-in function even if the + # current security policy does not allow it. + # + # See https://github.com/python/cpython/issues/84872. f(usedforsecurity=False) # Use the C function directly (very fast) return f @@ -141,29 +159,31 @@ def __get_openssl_constructor(name): return __get_builtin_constructor(name) -def __py_new(name, data=b'', **kwargs): +def __py_new(name, *args, **kwargs): """new(name, data=b'', **kwargs) - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ - return __get_builtin_constructor(name)(data, **kwargs) + return __get_builtin_constructor(name)(*args, **kwargs) -def __hash_new(name, data=b'', **kwargs): +def __hash_new(name, *args, **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; optionally initialized with data (which must be a bytes-like object). """ if name in __block_openssl_constructor: + # __block_openssl_constructor is expected to contain strings only + assert isinstance(name, str), f"unexpected name: {name}" # Prefer our builtin blake2 implementation. - return __get_builtin_constructor(name)(data, **kwargs) + return __get_builtin_constructor(name)(*args, **kwargs) try: - return _hashlib.new(name, data, **kwargs) + return _hashlib.new(name, *args, **kwargs) except ValueError: # If the _hashlib module (OpenSSL) doesn't support the named # hash, try using our builtin implementations. # This allows for SHA224/256 and SHA384/512 support even though # the OpenSSL library prior to 0.9.8 doesn't provide them. - return __get_builtin_constructor(name)(data) + return __get_builtin_constructor(name)(*args, **kwargs) try: @@ -187,7 +207,8 @@ def __hash_new(name, data=b'', **kwargs): try: # OpenSSL's scrypt requires OpenSSL 1.1+ - from _hashlib import scrypt # noqa: F401 + from _hashlib import scrypt + __all__ += ('scrypt',) except ImportError: pass @@ -203,7 +224,7 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): *digest* must either be a hash algorithm name as a *str*, a hash constructor, or a callable that returns a hash object. """ - # On Linux we could use AF_ALG sockets and sendfile() to archive zero-copy + # On Linux we could use AF_ALG sockets and sendfile() to achieve zero-copy # hashing with hardware acceleration. if isinstance(digest, str): digestobj = new(digest) @@ -240,16 +261,39 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): return digestobj +__logging = None for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL # version not supporting that algorithm. try: globals()[__func_name] = __get_hash(__func_name) - except ValueError: - import logging - logging.exception('code for hash %s was not found.', __func_name) - + except ValueError as __exc: + import logging as __logging + __logging.error('hash algorithm %s will not be supported at runtime ' + '[reason: %s]', __func_name, __exc) + # The following code can be simplified in Python 3.19 + # once "string" is removed from the signature. + __code = f'''\ +def {__func_name}(data=__UNSET, *, usedforsecurity=True, string=__UNSET): + if data is __UNSET and string is not __UNSET: + import warnings + warnings.warn( + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", + DeprecationWarning, stacklevel=2) + if data is not __UNSET and string is not __UNSET: + raise TypeError("'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version.") + raise ValueError("unsupported hash algorithm {__func_name}") +''' + exec(__code, {"__UNSET": object()}, __locals := {}) + globals()[__func_name] = __locals[__func_name] + del __exc, __code, __locals # Cleanup locals() del __always_supported, __func_name, __get_hash del __py_new, __hash_new, __get_openssl_constructor +del __logging diff --git a/Lib/heapq.py b/Lib/heapq.py index 6ceb211f1ca..f944376bcd2 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -126,8 +126,9 @@ From all times, sorting has always been a Great Art! :-) """ -__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', - 'nlargest', 'nsmallest', 'heappushpop'] +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'heappushpop', + 'heappush_max', 'heappop_max', 'heapify_max', 'heapreplace_max', + 'heappushpop_max', 'nlargest', 'nsmallest', 'merge'] def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" @@ -493,7 +494,7 @@ def nsmallest(n, iterable, key=None): pass else: if n >= size: - return sorted(iterable, key=key)[:n] + return sorted(iterable, key=key) # When key is none, use simpler decoration if key is None: @@ -553,7 +554,7 @@ def nlargest(n, iterable, key=None): pass else: if n >= size: - return sorted(iterable, key=key, reverse=True)[:n] + return sorted(iterable, key=key, reverse=True) # When key is none, use simpler decoration if key is None: diff --git a/Lib/hmac.py b/Lib/hmac.py index 3683a4aa653..9d3fae8b1b1 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -26,6 +26,16 @@ digest_size = None +def _is_shake_constructor(digest_like): + if isinstance(digest_like, str): + name = digest_like + else: + h = digest_like() if callable(digest_like) else digest_like.new() + if not isinstance(name := getattr(h, "name", None), str): + return False + return name.startswith(("shake", "SHAKE")) + + def _get_digest_constructor(digest_like): if callable(digest_like): return digest_like @@ -109,6 +119,8 @@ def _init_old(self, key, msg, digestmod): import warnings digest_cons = _get_digest_constructor(digestmod) + if _is_shake_constructor(digest_cons): + raise ValueError(f"unsupported hash algorithm {digestmod}") self._hmac = None self._outer = digest_cons() @@ -229,6 +241,14 @@ def digest(key, msg, digest): if _hashopenssl and isinstance(digest, (str, _functype)): try: return _hashopenssl.hmac_digest(key, msg, digest) + except OverflowError: + # OpenSSL's HMAC limits the size of the key to INT_MAX. + # Instead of falling back to HACL* implementation which + # may still not be supported due to a too large key, we + # directly switch to the pure Python fallback instead + # even if we could have used streaming HMAC for small keys + # but large messages. + return _compute_digest_fallback(key, msg, digest) except _hashopenssl.UnsupportedDigestmodError: pass @@ -236,6 +256,10 @@ def digest(key, msg, digest): try: return _hmac.compute_digest(key, msg, digest) except (OverflowError, _hmac.UnknownHashError): + # HACL* HMAC limits the size of the key to UINT32_MAX + # so we fallback to the pure Python implementation even + # if streaming HMAC may have been used for small keys + # and large messages. pass return _compute_digest_fallback(key, msg, digest) @@ -243,6 +267,8 @@ def digest(key, msg, digest): def _compute_digest_fallback(key, msg, digest): digest_cons = _get_digest_constructor(digest) + if _is_shake_constructor(digest_cons): + raise ValueError(f"unsupported hash algorithm {digest}") inner = digest_cons() outer = digest_cons() blocksize = getattr(inner, 'block_size', 64) diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 13c95c34e50..80fb8c3f929 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -12,6 +12,7 @@ import _markupbase from html import unescape +from html.entities import html5 as html5_entities __all__ = ['HTMLParser'] @@ -23,20 +24,52 @@ entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +incomplete_charref = re.compile('&#(?:[0-9]|[xX][0-9a-fA-F])') +attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') +endtagopen = re.compile('') -commentclose = re.compile(r'--\s*>') +commentclose = re.compile(r'--!?>') +commentabruptclose = re.compile(r'-?>') # Note: -# 1) if you change tagfind/attrfind remember to update locatestarttagend too; -# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# 1) if you change tagfind/attrfind remember to update locatetagend too; +# 2) if you change tagfind/attrfind and/or locatetagend the parser will # explode, so don't do it. -# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state -# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state -tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') -attrfind_tolerant = re.compile( - r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' - r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +# see the HTML5 specs section "13.2.5.6 Tag open state", +# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state". +# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state +# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state +# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*') +attrfind_tolerant = re.compile(r""" + ( + (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + ) + ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + ('[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space +""", re.VERBOSE) +locatetagend = re.compile(r""" + [a-zA-Z][^\t\n\r\f />]* # tag name + [\t\n\r\f /]* # optional whitespace before attribute name + (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + [\t\n\r\f /]* # possibly followed by a space + )* + >? +""", re.VERBOSE) +# The following variables are not used, but are temporarily left for +# backward compatibility. locatestarttagend_tolerant = re.compile(r""" <[a-zA-Z][^\t\n\r\f />\x00]* # tag name (?:[\s/]* # optional whitespace before attribute name @@ -53,10 +86,24 @@ \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') -# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between -# ') +# Character reference processing logic specific to attribute values +# See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state +def _replace_attr_charref(match): + ref = match.group(0) + # Numeric / hex char refs must always be unescaped + if ref.startswith('&#'): + return unescape(ref) + # Named character / entity references must only be unescaped + # if they are an exact match, and they are not followed by an equals sign + if not ref.endswith('=') and ref[1:] in html5_entities: + return unescape(ref) + # Otherwise do not unescape + return ref + +def _unescape_attrvalue(s): + return attr_charref.sub(_replace_attr_charref, s) class HTMLParser(_markupbase.ParserBase): @@ -81,16 +128,25 @@ class HTMLParser(_markupbase.ParserBase): argument. """ - CDATA_CONTENT_ELEMENTS = ("script", "style") + # See the HTML5 specs section "13.4 Parsing HTML fragments". + # https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments + # CDATA_CONTENT_ELEMENTS are parsed in RAWTEXT mode + CDATA_CONTENT_ELEMENTS = ("script", "style", "xmp", "iframe", "noembed", "noframes") + RCDATA_CONTENT_ELEMENTS = ("textarea", "title") - def __init__(self, *, convert_charrefs=True): + def __init__(self, *, convert_charrefs=True, scripting=False): """Initialize and reset this instance. - If convert_charrefs is True (the default), all character references + If convert_charrefs is true (the default), all character references are automatically converted to the corresponding Unicode characters. + + If *scripting* is false (the default), the content of the + ``noscript`` element is parsed normally; if it's true, + it's returned as is without being parsed. """ super().__init__() self.convert_charrefs = convert_charrefs + self.scripting = scripting self.reset() def reset(self): @@ -99,6 +155,8 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None + self._support_cdata = True + self._escapable = True super().reset() def feed(self, data): @@ -120,13 +178,35 @@ def get_starttag_text(self): """Return full source of start tag: '<...>'.""" return self.__starttag_text - def set_cdata_mode(self, elem): + def set_cdata_mode(self, elem, *, escapable=False): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'' % self.cdata_elem, re.I) + self._escapable = escapable + if self.cdata_elem == 'plaintext': + self.interesting = re.compile(r'\z') + elif escapable and not self.convert_charrefs: + self.interesting = re.compile(r'&|])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) + else: + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True + + def _set_support_cdata(self, flag=True): + """Enable or disable support of the CDATA sections. + If enabled, "<[CDATA[" starts a CDATA section which ends with "]]>". + If disabled, "<[CDATA[" starts a bogus comments which ends with ">". + + This method is not called by default. Its purpose is to be called + in custom handle_starttag() and handle_endtag() methods, with + value that depends on the adjusted current node. + See https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state + for details. + """ + self._support_cdata = flag # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is @@ -147,7 +227,7 @@ def goahead(self, end): # & near the end and see if it's followed by a space or ;. amppos = rawdata.rfind('&', max(i, n-34)) if (amppos >= 0 and - not re.compile(r'[\s;]').search(rawdata, amppos)): + not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)): break # wait till we get all the text j = n else: @@ -159,7 +239,7 @@ def goahead(self, end): break j = n if i < j: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:j])) else: self.handle_data(rawdata[i:j]) @@ -177,7 +257,7 @@ def goahead(self, end): k = self.parse_pi(i) elif startswith("', i + 1) - if k < 0: - k = rawdata.find('<', i + 1) - if k < 0: - k = i + 1 + if starttagopen.match(rawdata, i): # < + letter + pass + elif startswith("', i+9) + if j < 0: + return -1 + self.unknown_decl(rawdata[i+3: j]) + return j + 3 elif rawdata[i:i+9].lower() == ' gtpos = rawdata.find('>', i+9) @@ -272,12 +382,27 @@ def parse_html_declaration(self, i): else: return self.parse_bogus_comment(i) + # Internal -- parse comment, return length or -1 if not terminated + # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state + def parse_comment(self, i, report=True): + rawdata = self.rawdata + assert rawdata.startswith(' + + + + + + +
+ +
+
+ Tachyon + + Profiler +
+
+ + +
+
+ + + +
+
+ + +
+ + + + +
+
+
+
+ + +
+ + + + +
+
+ + + + diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.css b/Lib/profiling/sampling/_heatmap_assets/heatmap.css new file mode 100644 index 00000000000..ada6d2f2ee1 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.css @@ -0,0 +1,1162 @@ +/* ========================================================================== + Heatmap Viewer - Component-Specific CSS + + DEPENDENCY: Requires _shared_assets/base.css to be loaded first + This file extends the shared foundation with heatmap-specific styles. + ========================================================================== */ + +/* -------------------------------------------------------------------------- + Layout Overrides (Heatmap-specific) + -------------------------------------------------------------------------- */ + +.app-layout { + min-height: 100vh; +} + +/* Sticky top bar for heatmap views */ +.top-bar { + position: sticky; + top: 0; + z-index: 100; +} + +/* Back link in toolbar */ +.back-link { + color: white; + text-decoration: none; + padding: 6px 14px; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + font-size: 13px; + font-weight: 500; + transition: all var(--transition-fast); +} + +.back-link:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(255, 255, 255, 0.35); +} + +/* -------------------------------------------------------------------------- + Main Content Area + -------------------------------------------------------------------------- */ + +.main-content { + flex: 1; + padding: 24px 3%; + width: 100%; + max-width: 100%; +} + +/* -------------------------------------------------------------------------- + Stats Summary Cards - Enhanced with Icons & Animations + -------------------------------------------------------------------------- */ + +.stats-summary { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 12px; + margin-bottom: 24px; +} + +.stat-card { + display: flex; + align-items: center; + gap: 12px; + background: var(--bg-primary); + border: 2px solid var(--border); + border-radius: 10px; + padding: 14px 16px; + transition: all var(--transition-fast); + animation: slideUp 0.5s ease-out backwards; + animation-delay: calc(var(--i, 0) * 0.08s); + position: relative; + overflow: hidden; +} + +.stat-card::before { + content: ''; + position: absolute; + top: 0; + left: 0; + right: 0; + height: 3px; + background: linear-gradient(90deg, var(--python-blue), var(--python-gold)); + opacity: 0; + transition: opacity var(--transition-fast); +} + +.stat-card:nth-child(1) { --i: 0; --card-color: 55, 118, 171; } +.stat-card:nth-child(2) { --i: 1; --card-color: 40, 167, 69; } +.stat-card:nth-child(3) { --i: 2; --card-color: 255, 193, 7; } +.stat-card:nth-child(4) { --i: 3; --card-color: 111, 66, 193; } +.stat-card:nth-child(5) { --i: 4; --card-color: 220, 53, 69; } +.stat-card:nth-child(6) { --i: 5; --card-color: 23, 162, 184; } + +.stat-card:hover { + border-color: rgba(var(--card-color), 0.6); + background: linear-gradient(135deg, rgba(var(--card-color), 0.08) 0%, var(--bg-primary) 100%); + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(var(--card-color), 0.15); +} + +.stat-card:hover::before { + opacity: 1; +} + +.stat-icon { + width: 40px; + height: 40px; + display: flex; + align-items: center; + justify-content: center; + font-size: 18px; + background: linear-gradient(135deg, rgba(var(--card-color), 0.15) 0%, rgba(var(--card-color), 0.05) 100%); + border: 1px solid rgba(var(--card-color), 0.2); + border-radius: 10px; + flex-shrink: 0; + transition: all var(--transition-fast); +} + +.stat-card:hover .stat-icon { + transform: scale(1.05) rotate(-2deg); + background: linear-gradient(135deg, rgba(var(--card-color), 0.25) 0%, rgba(var(--card-color), 0.1) 100%); +} + +.stat-data { + flex: 1; + min-width: 0; +} + +.stat-value { + font-family: var(--font-mono); + font-size: 1.35em; + font-weight: 800; + color: rgb(var(--card-color)); + display: block; + line-height: 1.1; + letter-spacing: -0.3px; +} + +.stat-label { + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.3px; + margin-top: 2px; +} + +/* Sparkline decoration for stats */ +.stat-sparkline { + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 30px; + opacity: 0.1; + background: linear-gradient(180deg, + transparent 0%, + rgba(var(--card-color), 0.3) 100% + ); + pointer-events: none; +} + +/* -------------------------------------------------------------------------- + Rate Cards (Error Rate, Missed Samples) with Progress Bars + -------------------------------------------------------------------------- */ + +.rate-card { + display: flex; + flex-direction: column; + gap: 12px; + background: var(--bg-primary); + border: 2px solid var(--border); + border-radius: 12px; + padding: 18px 20px; + transition: all var(--transition-fast); + animation: slideUp 0.5s ease-out backwards; + position: relative; + overflow: hidden; +} + +.rate-card:nth-child(5) { animation-delay: 0.32s; --rate-color: 220, 53, 69; } +.rate-card:nth-child(6) { animation-delay: 0.40s; --rate-color: 255, 152, 0; } + +.rate-card:hover { + border-color: rgba(var(--rate-color), 0.5); + transform: translateY(-2px); + box-shadow: 0 6px 20px rgba(var(--rate-color), 0.15); +} + +.rate-header { + display: flex; + justify-content: space-between; + align-items: center; +} + +.rate-info { + display: flex; + align-items: center; + gap: 10px; +} + +.rate-icon { + width: 36px; + height: 36px; + display: flex; + align-items: center; + justify-content: center; + font-size: 18px; + background: linear-gradient(135deg, rgba(var(--rate-color), 0.15) 0%, rgba(var(--rate-color), 0.05) 100%); + border: 1px solid rgba(var(--rate-color), 0.2); + border-radius: 10px; + flex-shrink: 0; +} + +.rate-label { + font-size: 12px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.3px; +} + +.rate-value { + font-family: var(--font-mono); + font-size: 1.4em; + font-weight: 800; + color: rgb(var(--rate-color)); +} + +.rate-bar { + height: 8px; + background: var(--bg-tertiary); + border-radius: 4px; + overflow: hidden; + position: relative; +} + +.rate-fill { + height: 100%; + border-radius: 4px; + transition: width 0.8s ease-out; + position: relative; + overflow: hidden; +} + +.rate-fill.error { + background: linear-gradient(90deg, #dc3545 0%, #ff6b6b 100%); +} + +.rate-fill.warning { + background: linear-gradient(90deg, #ff9800 0%, #ffc107 100%); +} + +.rate-fill.good { + background: linear-gradient(90deg, #28a745 0%, #20c997 100%); +} + +/* Shimmer animation on rate bars */ +.rate-fill::after { + content: ''; + position: absolute; + top: 0; + left: -100%; + width: 100%; + height: 100%; + background: linear-gradient( + 90deg, + transparent 0%, + rgba(255, 255, 255, 0.4) 50%, + transparent 100% + ); + animation: shimmer 2.5s ease-in-out infinite; +} + +/* -------------------------------------------------------------------------- + Section Headers + -------------------------------------------------------------------------- */ + +.section-header { + display: flex; + align-items: center; + gap: 12px; + margin-bottom: 16px; + padding-bottom: 12px; + border-bottom: 2px solid var(--python-gold); +} + +.section-title { + font-size: 18px; + font-weight: 700; + color: var(--text-primary); + margin: 0; + flex: 1; +} + +/* -------------------------------------------------------------------------- + Filter Controls + -------------------------------------------------------------------------- */ + +.filter-controls { + display: flex; + gap: 8px; + flex-wrap: wrap; + align-items: center; + margin-bottom: 16px; +} + +.control-btn { + padding: 8px 16px; + background: var(--bg-secondary); + color: var(--text-primary); + border: 1px solid var(--border); + border-radius: 6px; + font-size: 13px; + font-weight: 500; + cursor: pointer; + transition: all var(--transition-fast); +} + +.control-btn:hover { + background: var(--accent); + color: white; + border-color: var(--accent); +} + +/* -------------------------------------------------------------------------- + Type Sections (stdlib, project, etc) + -------------------------------------------------------------------------- */ + +.type-section { + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + overflow: hidden; + margin-bottom: 12px; +} + +.type-header { + padding: 12px 16px; + background: var(--header-gradient); + color: white; + cursor: pointer; + display: flex; + align-items: center; + gap: 10px; + user-select: none; + transition: all var(--transition-fast); + font-weight: 600; +} + +.type-header:hover { + opacity: 0.95; +} + +.type-icon { + font-size: 12px; + transition: transform var(--transition-fast); + min-width: 12px; +} + +.type-title { + font-size: 14px; + flex: 1; +} + +.type-stats { + font-size: 12px; + opacity: 0.9; + background: rgba(255, 255, 255, 0.15); + padding: 4px 10px; + border-radius: 4px; + font-family: var(--font-mono); +} + +.type-content { + padding: 12px; +} + +/* -------------------------------------------------------------------------- + Folder Nodes (hierarchical structure) + -------------------------------------------------------------------------- */ + +.folder-node { + margin-bottom: 6px; +} + +.folder-header { + padding: 8px 12px; + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: 6px; + cursor: pointer; + display: flex; + align-items: center; + gap: 8px; + user-select: none; + transition: all var(--transition-fast); +} + +.folder-header:hover { + background: var(--accent-glow); + border-color: var(--accent); +} + +.folder-icon { + font-size: 10px; + color: var(--accent); + transition: transform var(--transition-fast); + min-width: 12px; +} + +.folder-name { + flex: 1; + font-weight: 500; + color: var(--text-primary); + font-size: 13px; +} + +.folder-stats { + font-size: 11px; + color: var(--text-secondary); + background: var(--bg-tertiary); + padding: 2px 8px; + border-radius: 4px; + font-family: var(--font-mono); +} + +.folder-content { + padding-left: 20px; + margin-top: 6px; +} + +/* -------------------------------------------------------------------------- + File Items + -------------------------------------------------------------------------- */ + +.files-list { + display: flex; + flex-direction: column; + gap: 4px; + margin-top: 8px; +} + +.file-item { + display: flex; + align-items: center; + gap: 12px; + padding: 8px 12px; + background: var(--bg-primary); + border: 1px solid var(--border-subtle); + border-radius: 6px; + transition: all var(--transition-fast); +} + +.file-item:hover { + background: var(--bg-secondary); + border-color: var(--border); +} + +.file-item .file-link { + flex: 1; + min-width: 0; + font-size: 13px; +} + +.file-samples { + font-size: 12px; + color: var(--text-secondary); + font-weight: 600; + white-space: nowrap; + width: 130px; + flex-shrink: 0; + text-align: right; + font-family: var(--font-mono); +} + +.heatmap-bar-container { + width: 120px; + flex-shrink: 0; + display: flex; + align-items: center; +} + +.heatmap-bar { + flex-shrink: 0; + border-radius: 2px; +} + +/* Links */ +.file-link { + color: var(--accent); + text-decoration: none; + font-weight: 500; + transition: color var(--transition-fast); +} + +.file-link:hover { + color: var(--accent-hover); + text-decoration: underline; +} + +/* -------------------------------------------------------------------------- + Module Badges + -------------------------------------------------------------------------- */ + +.module-badge { + display: inline-block; + padding: 3px 8px; + border-radius: 4px; + font-size: 11px; + font-weight: 600; +} + +.badge-stdlib { + background: rgba(40, 167, 69, 0.15); + color: #28a745; +} + +.badge-site-packages { + background: rgba(0, 123, 255, 0.15); + color: #007bff; +} + +.badge-project { + background: rgba(255, 193, 7, 0.2); + color: #d39e00; +} + +.badge-other { + background: var(--bg-tertiary); + color: var(--text-secondary); +} + +[data-theme="dark"] .badge-stdlib { + background: rgba(40, 167, 69, 0.25); + color: #5dd879; +} + +[data-theme="dark"] .badge-site-packages { + background: rgba(88, 166, 255, 0.25); + color: #79b8ff; +} + +[data-theme="dark"] .badge-project { + background: rgba(255, 212, 59, 0.25); + color: #ffd43b; +} + +/* ========================================================================== + FILE VIEW STYLES (Code Display) + ========================================================================== */ + +.code-view { + font-family: var(--font-mono); + min-height: 100vh; +} + +/* Code Header (Top Bar for file view) */ +.code-header { + height: var(--topbar-height); + background: var(--header-gradient); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); + border-bottom: 2px solid var(--python-gold); + position: sticky; + top: 0; + z-index: 100; +} + +.code-header-content { + display: flex; + align-items: center; + justify-content: space-between; + width: 94%; + max-width: 100%; + margin: 0 auto; +} + +.code-header h1 { + font-size: 14px; + font-weight: 600; + color: white; + margin: 0; + font-family: var(--font-mono); + display: flex; + align-items: center; + gap: 8px; +} + +/* File Stats Bar */ +.file-stats { + background: var(--bg-secondary); + padding: 16px 24px; + border-bottom: 1px solid var(--border); +} + +.file-stats .stats-grid { + width: 94%; + max-width: 100%; + margin: 0 auto; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); + gap: 12px; +} + +.stat-item { + background: var(--bg-primary); + padding: 12px; + border-radius: 8px; + box-shadow: var(--shadow-sm); + text-align: center; + border: 1px solid var(--border); + transition: all var(--transition-fast); +} + +.stat-item:hover { + transform: translateY(-2px); + box-shadow: var(--shadow-md); + border-color: var(--accent); +} + +.stat-item .stat-value { + font-size: 1.4em; + font-weight: 700; + color: var(--accent); +} + +.stat-item .stat-label { + color: var(--text-muted); + font-size: 10px; + margin-top: 2px; +} + +/* Legend */ +.legend { + background: var(--bg-secondary); + padding: 12px 24px; + border-bottom: 1px solid var(--border); +} + +.legend-content { + width: 94%; + max-width: 100%; + margin: 0 auto; + display: flex; + align-items: center; + gap: 20px; + flex-wrap: wrap; +} + +.legend-title { + font-weight: 600; + color: var(--text-primary); + font-size: 13px; + font-family: var(--font-sans); +} + +.legend-gradient { + flex: 1; + max-width: 300px; + height: 24px; + background: linear-gradient(90deg, + var(--bg-tertiary) 0%, + var(--heat-2) 25%, + var(--heat-4) 50%, + var(--heat-6) 75%, + var(--heat-8) 100% + ); + border-radius: 4px; + border: 1px solid var(--border); +} + +.legend-labels { + display: flex; + gap: 12px; + font-size: 11px; + color: var(--text-muted); + font-family: var(--font-sans); +} + +/* Toggle Switch Styles */ +.toggle-switch { + display: inline-flex; + align-items: center; + gap: 8px; + cursor: pointer; + user-select: none; + font-family: var(--font-sans); + transition: opacity var(--transition-fast); +} + +.toggle-switch:hover { + opacity: 0.85; +} + +.toggle-switch .toggle-label { + font-size: 11px; + font-weight: 500; + color: var(--text-muted); + min-width: 55px; + text-align: right; + transition: color var(--transition-fast); +} + +.toggle-switch .toggle-label:last-child { + text-align: left; +} + +.toggle-switch .toggle-label.active { + color: var(--text-primary); + font-weight: 600; +} + +.toggle-track { + position: relative; + width: 36px; + height: 20px; + background: var(--bg-tertiary); + border: 2px solid var(--border); + border-radius: 12px; + transition: all var(--transition-fast); + box-shadow: inset var(--shadow-sm); +} + +.toggle-track:hover { + border-color: var(--text-muted); +} + +.toggle-track.on { + background: var(--accent); + border-color: var(--accent); + box-shadow: 0 0 8px var(--accent-glow); +} + +.toggle-track::after { + content: ''; + position: absolute; + top: 1px; + left: 1px; + width: 14px; + height: 14px; + background: white; + border-radius: 50%; + box-shadow: var(--shadow-sm); + transition: all var(--transition-fast); +} + +.toggle-track.on::after { + transform: translateX(16px); + box-shadow: var(--shadow-md); +} + +/* Specific toggle overrides */ +#toggle-color-mode .toggle-track.on { + background: #8e44ad; + border-color: #8e44ad; + box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); +} + +#toggle-cold .toggle-track.on { + background: #e67e22; + border-color: #e67e22; + box-shadow: 0 0 8px rgba(230, 126, 34, 0.3); +} + +/* Code Container */ +.code-container { + width: 94%; + max-width: 100%; + margin: 16px auto; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px 8px 8px 8px; + box-shadow: var(--shadow-sm); + /* Allow horizontal scroll for long lines, but don't clip sticky header */ +} + +/* Code Header Row */ +.code-header-row { + position: sticky; + top: var(--topbar-height); + z-index: 50; + display: flex; + background: var(--bg-secondary); + border-bottom: 2px solid var(--border); + font-weight: 700; + font-size: 11px; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.5px; + border-radius: 8px 8px 0 0; +} + +.header-line-number { + flex-shrink: 0; + width: 60px; + padding: 8px 10px; + text-align: right; + border-right: 1px solid var(--border); +} + +.header-samples-self, +.header-samples-cumulative { + flex-shrink: 0; + width: 90px; + padding: 8px 10px; + text-align: right; + border-right: 1px solid var(--border); +} + +.header-samples-self { + color: var(--heat-8); +} + +.header-samples-cumulative { + color: var(--accent); +} + +.header-content { + flex: 1; + padding: 8px 15px; +} + +/* Code Lines */ +.code-line { + position: relative; + display: flex; + min-height: 20px; + line-height: 20px; + font-size: 13px; + transition: background var(--transition-fast); + scroll-margin-top: calc(var(--topbar-height) + 50px); +} + +.code-line:hover { + filter: brightness(0.97); +} + +[data-theme="dark"] .code-line:hover { + filter: brightness(1.1); +} + +.line-number { + flex-shrink: 0; + width: 60px; + padding: 0 10px; + text-align: right; + color: var(--text-muted); + background: var(--bg-secondary); + border-right: 1px solid var(--border); + user-select: none; + transition: all var(--transition-fast); +} + +.line-number:hover { + background: var(--accent); + color: white; + cursor: pointer; +} + +.line-samples-self, +.line-samples-cumulative { + flex-shrink: 0; + width: 90px; + padding: 0 10px; + text-align: right; + background: var(--bg-secondary); + border-right: 1px solid var(--border); + font-weight: 600; + user-select: none; + font-size: 12px; +} + +.line-samples-self { + color: var(--heat-8); +} + +.line-samples-cumulative { + color: var(--accent); +} + +.line-content { + flex: 1; + padding: 0 15px; + white-space: pre; + overflow-x: auto; +} + +/* Scrollbar Styling */ +.line-content::-webkit-scrollbar { + height: 6px; +} + +.line-content::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 3px; +} + +.line-content::-webkit-scrollbar-thumb:hover { + background: var(--text-muted); +} + +/* Navigation Buttons */ +.line-nav-buttons { + position: absolute; + right: 8px; + top: 50%; + transform: translateY(-50%); + display: flex; + gap: 4px; + align-items: center; +} + +.nav-btn { + padding: 2px 6px; + font-size: 12px; + font-weight: 500; + border: 1px solid var(--accent); + border-radius: 4px; + background: var(--bg-primary); + color: var(--accent); + cursor: pointer; + transition: all var(--transition-fast); + user-select: none; + line-height: 1; +} + +.nav-btn:hover:not(:disabled) { + background: var(--accent); + color: white; + transform: translateY(-1px); + box-shadow: var(--shadow-sm); +} + +.nav-btn:active:not(:disabled) { + transform: translateY(0); +} + +.nav-btn:disabled { + opacity: 0.3; + cursor: not-allowed; + color: var(--text-muted); + background: var(--bg-secondary); + border-color: var(--border); +} + +.nav-btn.caller { + color: var(--nav-caller); + border-color: var(--nav-caller); +} + +.nav-btn.callee { + color: var(--nav-callee); + border-color: var(--nav-callee); +} + +.nav-btn.caller:hover:not(:disabled) { + background: var(--nav-caller-hover); + color: white; +} + +.nav-btn.callee:hover:not(:disabled) { + background: var(--nav-callee-hover); + color: white; +} + +/* Highlighted target line */ +.code-line:target { + animation: highlight-line 2s ease-out; +} + +@keyframes highlight-line { + 0% { + background: rgba(255, 212, 59, 0.6) !important; + outline: 3px solid var(--python-gold); + outline-offset: -3px; + } + 50% { + background: rgba(255, 212, 59, 0.5) !important; + outline: 3px solid var(--python-gold); + outline-offset: -3px; + } + 100% { + background: inherit; + outline: 3px solid transparent; + outline-offset: -3px; + } +} + +/* Popup menu for multiple callees */ +.callee-menu { + position: absolute; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + box-shadow: var(--shadow-lg); + padding: 8px; + z-index: 1000; + min-width: 250px; + max-width: 400px; + max-height: 300px; + overflow-y: auto; +} + +.callee-menu-header { + font-weight: 600; + color: var(--text-primary); + margin-bottom: 8px; + padding-bottom: 8px; + border-bottom: 1px solid var(--border); + font-size: 13px; + font-family: var(--font-sans); +} + +.callee-menu-item { + padding: 8px; + margin: 4px 0; + border-radius: 6px; + cursor: pointer; + transition: background var(--transition-fast); + display: flex; + flex-direction: column; + gap: 4px; +} + +.callee-menu-item:hover { + background: var(--bg-secondary); +} + +.callee-menu-func { + font-weight: 500; + color: var(--accent); + font-size: 12px; +} + +.callee-menu-file { + font-size: 11px; + color: var(--text-muted); +} + +.count-badge { + display: inline-block; + background: var(--accent); + color: white; + font-size: 10px; + padding: 2px 6px; + border-radius: 4px; + font-weight: 600; + margin-left: 6px; +} + +/* Callee menu scrollbar */ +.callee-menu::-webkit-scrollbar { + width: 6px; +} + +.callee-menu::-webkit-scrollbar-track { + background: var(--bg-secondary); + border-radius: 3px; +} + +.callee-menu::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 3px; +} + +/* -------------------------------------------------------------------------- + Scroll Minimap Marker + -------------------------------------------------------------------------- */ + +#scroll_marker { + position: fixed; + z-index: 1000; + right: 0; + top: 0; + width: 12px; + height: 100%; + background: var(--bg-secondary); + border-left: 1px solid var(--border); + pointer-events: none; +} + +#scroll_marker .marker { + position: absolute; + min-height: 3px; + width: 100%; + pointer-events: none; +} + +#scroll_marker .marker.cold { + background: var(--heat-1); +} + +#scroll_marker .marker.cool { + background: var(--heat-2); +} + +#scroll_marker .marker.mild { + background: var(--heat-3); +} + +#scroll_marker .marker.warm { + background: var(--heat-4); +} + +#scroll_marker .marker.hot { + background: var(--heat-5); +} + +#scroll_marker .marker.very-hot { + background: var(--heat-6); +} + +#scroll_marker .marker.intense { + background: var(--heat-7); +} + +#scroll_marker .marker.extreme { + background: var(--heat-8); +} + +/* -------------------------------------------------------------------------- + Responsive (Heatmap-specific) + -------------------------------------------------------------------------- */ + +@media (max-width: 1100px) { + .stats-summary { + grid-template-columns: repeat(2, 1fr); + } +} + +@media (max-width: 900px) { + .main-content { + padding: 16px; + } +} + +@media (max-width: 600px) { + .stats-summary { + grid-template-columns: 1fr; + } + + .file-stats .stats-grid { + grid-template-columns: repeat(2, 1fr); + } + + .legend-content { + flex-direction: column; + gap: 12px; + } + + .legend-gradient { + width: 100%; + max-width: none; + } +} diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js new file mode 100644 index 00000000000..5a7ff5dd61a --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -0,0 +1,338 @@ +// Tachyon Profiler - Heatmap JavaScript +// Interactive features for the heatmap visualization +// Aligned with Flamegraph viewer design patterns + +// ============================================================================ +// State Management +// ============================================================================ + +let currentMenu = null; +let colorMode = 'self'; // 'self' or 'cumulative' - default to self +let coldCodeHidden = false; + +// ============================================================================ +// Theme Support +// ============================================================================ + +function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme') || 'light'; + const next = current === 'light' ? 'dark' : 'light'; + html.setAttribute('data-theme', next); + localStorage.setItem('heatmap-theme', next); + + // Update theme button icon + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + } + applyLineColors(); + + // Rebuild scroll marker with new theme colors + buildScrollMarker(); +} + +function restoreUIState() { + // Restore theme + const savedTheme = localStorage.getItem('heatmap-theme'); + if (savedTheme) { + document.documentElement.setAttribute('data-theme', savedTheme); + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + } + } +} + +// ============================================================================ +// Utility Functions +// ============================================================================ + +function createElement(tag, className, textContent = '') { + const el = document.createElement(tag); + if (className) el.className = className; + if (textContent) el.textContent = textContent; + return el; +} + +function calculateMenuPosition(buttonRect, menuWidth, menuHeight) { + const viewport = { width: window.innerWidth, height: window.innerHeight }; + const scroll = { + x: window.pageXOffset || document.documentElement.scrollLeft, + y: window.pageYOffset || document.documentElement.scrollTop + }; + + const left = buttonRect.right + menuWidth + 10 < viewport.width + ? buttonRect.right + scroll.x + 10 + : Math.max(scroll.x + 10, buttonRect.left + scroll.x - menuWidth - 10); + + const top = buttonRect.bottom + menuHeight + 10 < viewport.height + ? buttonRect.bottom + scroll.y + 5 + : Math.max(scroll.y + 10, buttonRect.top + scroll.y - menuHeight - 10); + + return { left, top }; +} + +// ============================================================================ +// Menu Management +// ============================================================================ + +function closeMenu() { + if (currentMenu) { + currentMenu.remove(); + currentMenu = null; + } +} + +function showNavigationMenu(button, items, title) { + closeMenu(); + + const menu = createElement('div', 'callee-menu'); + menu.appendChild(createElement('div', 'callee-menu-header', title)); + + items.forEach(linkData => { + const item = createElement('div', 'callee-menu-item'); + + const funcDiv = createElement('div', 'callee-menu-func'); + funcDiv.textContent = linkData.func; + + if (linkData.count !== undefined && linkData.count > 0) { + const countBadge = createElement('span', 'count-badge'); + countBadge.textContent = linkData.count.toLocaleString(); + countBadge.title = `${linkData.count.toLocaleString()} samples`; + funcDiv.appendChild(document.createTextNode(' ')); + funcDiv.appendChild(countBadge); + } + + item.appendChild(funcDiv); + item.appendChild(createElement('div', 'callee-menu-file', linkData.file)); + item.addEventListener('click', () => window.location.href = linkData.link); + menu.appendChild(item); + }); + + const pos = calculateMenuPosition(button.getBoundingClientRect(), 350, 300); + menu.style.left = `${pos.left}px`; + menu.style.top = `${pos.top}px`; + + document.body.appendChild(menu); + currentMenu = menu; +} + +// ============================================================================ +// Navigation +// ============================================================================ + +function handleNavigationClick(button, e) { + e.stopPropagation(); + + const navData = button.getAttribute('data-nav'); + if (navData) { + window.location.href = JSON.parse(navData).link; + return; + } + + const navMulti = button.getAttribute('data-nav-multi'); + if (navMulti) { + const items = JSON.parse(navMulti); + const title = button.classList.contains('caller') ? 'Choose a caller:' : 'Choose a callee:'; + showNavigationMenu(button, items, title); + } +} + +function scrollToTargetLine() { + if (!window.location.hash) return; + const target = document.querySelector(window.location.hash); + if (target) { + target.scrollIntoView({ behavior: 'smooth', block: 'start' }); + } +} + +// ============================================================================ +// Sample Count & Intensity +// ============================================================================ + +function getSampleCount(line) { + let text; + if (colorMode === 'self') { + text = line.querySelector('.line-samples-self')?.textContent.trim().replace(/,/g, ''); + } else { + text = line.querySelector('.line-samples-cumulative')?.textContent.trim().replace(/,/g, ''); + } + return parseInt(text) || 0; +} + +// ============================================================================ +// Scroll Minimap +// ============================================================================ + +function buildScrollMarker() { + const existing = document.getElementById('scroll_marker'); + if (existing) existing.remove(); + + if (document.body.scrollHeight <= window.innerHeight) return; + + const allLines = document.querySelectorAll('.code-line'); + const lines = Array.from(allLines).filter(line => line.style.display !== 'none'); + const markerScale = window.innerHeight / document.body.scrollHeight; + const lineHeight = Math.min(Math.max(3, window.innerHeight / lines.length), 10); + const maxSamples = Math.max(...Array.from(lines, getSampleCount)); + + const scrollMarker = createElement('div', ''); + scrollMarker.id = 'scroll_marker'; + + let prevLine = -99, lastMark, lastTop; + + lines.forEach((line, index) => { + const samples = getSampleCount(line); + if (samples === 0) return; + + const lineTop = Math.floor(line.offsetTop * markerScale); + const lineNumber = index + 1; + const intensityClass = maxSamples > 0 ? (intensityToClass(samples / maxSamples) || 'cold') : 'cold'; + + if (lineNumber === prevLine + 1 && lastMark?.classList.contains(intensityClass)) { + lastMark.style.height = `${lineTop + lineHeight - lastTop}px`; + } else { + lastMark = createElement('div', `marker ${intensityClass}`); + lastMark.style.height = `${lineHeight}px`; + lastMark.style.top = `${lineTop}px`; + scrollMarker.appendChild(lastMark); + lastTop = lineTop; + } + + prevLine = lineNumber; + }); + + document.body.appendChild(scrollMarker); +} + +function applyLineColors() { + const lines = document.querySelectorAll('.code-line'); + lines.forEach(line => { + let intensity; + if (colorMode === 'self') { + intensity = parseFloat(line.getAttribute('data-self-intensity')) || 0; + } else { + intensity = parseFloat(line.getAttribute('data-cumulative-intensity')) || 0; + } + + const color = intensityToColor(intensity); + line.style.background = color; + }); +} + +// ============================================================================ +// Toggle Controls +// ============================================================================ + +function updateToggleUI(toggleId, isOn) { + const toggle = document.getElementById(toggleId); + if (toggle) { + const track = toggle.querySelector('.toggle-track'); + const labels = toggle.querySelectorAll('.toggle-label'); + if (isOn) { + track.classList.add('on'); + labels[0].classList.remove('active'); + labels[1].classList.add('active'); + } else { + track.classList.remove('on'); + labels[0].classList.add('active'); + labels[1].classList.remove('active'); + } + } +} + +function toggleColdCode() { + coldCodeHidden = !coldCodeHidden; + applyHotFilter(); + updateToggleUI('toggle-cold', coldCodeHidden); + buildScrollMarker(); +} + +function applyHotFilter() { + const lines = document.querySelectorAll('.code-line'); + + lines.forEach(line => { + const selfSamples = line.querySelector('.line-samples-self')?.textContent.trim(); + const cumulativeSamples = line.querySelector('.line-samples-cumulative')?.textContent.trim(); + + let isCold; + if (colorMode === 'self') { + isCold = !selfSamples || selfSamples === ''; + } else { + isCold = !cumulativeSamples || cumulativeSamples === ''; + } + + if (isCold) { + line.style.display = coldCodeHidden ? 'none' : 'flex'; + } else { + line.style.display = 'flex'; + } + }); +} + +function toggleColorMode() { + colorMode = colorMode === 'self' ? 'cumulative' : 'self'; + applyLineColors(); + + updateToggleUI('toggle-color-mode', colorMode === 'cumulative'); + + if (coldCodeHidden) { + applyHotFilter(); + } + + buildScrollMarker(); +} + +// ============================================================================ +// Initialization +// ============================================================================ + +document.addEventListener('DOMContentLoaded', function() { + // Restore UI state (theme, etc.) + restoreUIState(); + applyLineColors(); + + // Initialize navigation buttons + document.querySelectorAll('.nav-btn').forEach(button => { + button.addEventListener('click', e => handleNavigationClick(button, e)); + }); + + // Initialize line number permalink handlers + document.querySelectorAll('.line-number').forEach(lineNum => { + lineNum.style.cursor = 'pointer'; + lineNum.addEventListener('click', e => { + window.location.hash = `line-${e.target.textContent.trim()}`; + }); + }); + + // Initialize toggle buttons + const toggleColdBtn = document.getElementById('toggle-cold'); + if (toggleColdBtn) { + toggleColdBtn.addEventListener('click', toggleColdCode); + } + + const colorModeBtn = document.getElementById('toggle-color-mode'); + if (colorModeBtn) { + colorModeBtn.addEventListener('click', toggleColorMode); + } + + // Build scroll marker + setTimeout(buildScrollMarker, 200); + + // Setup scroll-to-line behavior + setTimeout(scrollToTargetLine, 100); +}); + +// Close menu when clicking outside +document.addEventListener('click', e => { + if (currentMenu && !currentMenu.contains(e.target) && !e.target.classList.contains('nav-btn')) { + closeMenu(); + } +}); + +// Handle hash changes +window.addEventListener('hashchange', () => setTimeout(scrollToTargetLine, 50)); + +// Rebuild scroll marker on resize +window.addEventListener('resize', buildScrollMarker); diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js new file mode 100644 index 00000000000..4ddacca5173 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js @@ -0,0 +1,127 @@ +// Tachyon Profiler - Heatmap Index JavaScript +// Index page specific functionality + +// ============================================================================ +// Heatmap Bar Coloring +// ============================================================================ + +function applyHeatmapBarColors() { + const bars = document.querySelectorAll('.heatmap-bar[data-intensity]'); + bars.forEach(bar => { + const intensity = parseFloat(bar.getAttribute('data-intensity')) || 0; + const color = intensityToColor(intensity); + bar.style.backgroundColor = color; + }); +} + +// ============================================================================ +// Theme Support +// ============================================================================ + +function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme') || 'light'; + const next = current === 'light' ? 'dark' : 'light'; + html.setAttribute('data-theme', next); + localStorage.setItem('heatmap-theme', next); + + // Update theme button icon + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + } + + applyHeatmapBarColors(); +} + +function restoreUIState() { + // Restore theme + const savedTheme = localStorage.getItem('heatmap-theme'); + if (savedTheme) { + document.documentElement.setAttribute('data-theme', savedTheme); + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + } + } +} + +// ============================================================================ +// Type Section Toggle (stdlib, project, etc) +// ============================================================================ + +function toggleTypeSection(header) { + const section = header.parentElement; + const content = section.querySelector('.type-content'); + const icon = header.querySelector('.type-icon'); + + if (content.style.display === 'none') { + content.style.display = 'block'; + icon.textContent = '\u25BC'; + } else { + content.style.display = 'none'; + icon.textContent = '\u25B6'; + } +} + +// ============================================================================ +// Folder Toggle +// ============================================================================ + +function toggleFolder(header) { + const folder = header.parentElement; + const content = folder.querySelector('.folder-content'); + const icon = header.querySelector('.folder-icon'); + + if (content.style.display === 'none') { + content.style.display = 'block'; + icon.textContent = '\u25BC'; + folder.classList.remove('collapsed'); + } else { + content.style.display = 'none'; + icon.textContent = '\u25B6'; + folder.classList.add('collapsed'); + } +} + +// ============================================================================ +// Expand/Collapse All +// ============================================================================ + +function expandAll() { + // Expand all type sections + document.querySelectorAll('.type-section').forEach(section => { + const content = section.querySelector('.type-content'); + const icon = section.querySelector('.type-icon'); + content.style.display = 'block'; + icon.textContent = '\u25BC'; + }); + + // Expand all folders + document.querySelectorAll('.folder-node').forEach(folder => { + const content = folder.querySelector('.folder-content'); + const icon = folder.querySelector('.folder-icon'); + content.style.display = 'block'; + icon.textContent = '\u25BC'; + folder.classList.remove('collapsed'); + }); +} + +function collapseAll() { + document.querySelectorAll('.folder-node').forEach(folder => { + const content = folder.querySelector('.folder-content'); + const icon = folder.querySelector('.folder-icon'); + content.style.display = 'none'; + icon.textContent = '\u25B6'; + folder.classList.add('collapsed'); + }); +} + +// ============================================================================ +// Initialization +// ============================================================================ + +document.addEventListener('DOMContentLoaded', function() { + restoreUIState(); + applyHeatmapBarColors(); +}); diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html b/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html new file mode 100644 index 00000000000..b71bd94c661 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html @@ -0,0 +1,118 @@ + + + + + + Tachyon Profiler - Heatmap Report + + + +
+ +
+
+ Tachyon + + Heatmap Report +
+
+ +
+
+ + +
+ +
+
+
📄
+
+ + Files Profiled +
+
+
+
+
📊
+
+ + Total Snapshots +
+
+
+
+
⏱
+
+ + Duration +
+
+
+
+
⚡
+
+ + Samples/sec +
+
+
+
+
+
+
⚠
+ Error Rate +
+ +
+
+
+
+
+
+
+
+
💥
+ Missed Samples +
+ +
+
+
+
+
+
+ + +
+

Profiled Files

+
+ +
+ + +
+ +
+ +
+
+ + +
+ + Tachyon Profiler + + + Python Sampling Profiler + +
+
+ + + + diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html b/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html new file mode 100644 index 00000000000..d8b26adfb02 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html @@ -0,0 +1,96 @@ + + + + + + <!-- FILENAME --> - Heatmap + + + +
+ +
+
+ Tachyon + + +
+
+ Back to Index + +
+
+ + +
+
+
+
+
Self Samples
+
+
+
+
Cumulative
+
+
+
+
Lines Hit
+
+
+
%
+
% of Total
+
+
+
+
Max Self
+
+
+
+
Max Total
+
+
+
+ + +
+
+ Intensity: +
+
+ Cold + → + Hot +
+
+ Self Time +
+ Total Time +
+
+ Show All +
+ Hot Only +
+
+
+ + +
+
+
Line
+
Self
+
Total
+
Code
+
+ +
+
+ + + + diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_shared.js b/Lib/profiling/sampling/_heatmap_assets/heatmap_shared.js new file mode 100644 index 00000000000..f44ebcff4ff --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_shared.js @@ -0,0 +1,40 @@ +// Tachyon Profiler - Shared Heatmap JavaScript +// Common utilities shared between index and file views + +// ============================================================================ +// Heat Level Mapping (Single source of truth for intensity thresholds) +// ============================================================================ + +// Maps intensity (0-1) to heat level (0-8). Level 0 = no heat, 1-8 = heat levels. +function intensityToHeatLevel(intensity) { + if (intensity <= 0) return 0; + if (intensity <= 0.125) return 1; + if (intensity <= 0.25) return 2; + if (intensity <= 0.375) return 3; + if (intensity <= 0.5) return 4; + if (intensity <= 0.625) return 5; + if (intensity <= 0.75) return 6; + if (intensity <= 0.875) return 7; + return 8; +} + +// Class names corresponding to heat levels 1-8 (used by scroll marker) +const HEAT_CLASS_NAMES = ['cold', 'cool', 'mild', 'warm', 'hot', 'very-hot', 'intense', 'extreme']; + +function intensityToClass(intensity) { + const level = intensityToHeatLevel(intensity); + return level === 0 ? null : HEAT_CLASS_NAMES[level - 1]; +} + +// ============================================================================ +// Color Mapping (Intensity to Heat Color) +// ============================================================================ + +function intensityToColor(intensity) { + const level = intensityToHeatLevel(intensity); + if (level === 0) { + return 'transparent'; + } + const rootStyle = getComputedStyle(document.documentElement); + return rootStyle.getPropertyValue(`--heat-${level}`).trim(); +} diff --git a/Lib/profiling/sampling/_shared_assets/base.css b/Lib/profiling/sampling/_shared_assets/base.css new file mode 100644 index 00000000000..54e3d78f8eb --- /dev/null +++ b/Lib/profiling/sampling/_shared_assets/base.css @@ -0,0 +1,369 @@ +/* ========================================================================== + Python Profiler - Shared CSS Foundation + Design system shared between Flamegraph and Heatmap viewers + ========================================================================== */ + +/* -------------------------------------------------------------------------- + CSS Variables & Theme System + -------------------------------------------------------------------------- */ + +:root { + /* Typography */ + --font-sans: "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", + "Geneva", "Verdana", sans-serif; + --font-mono: 'SF Mono', 'Monaco', 'Consolas', 'Liberation Mono', monospace; + + /* Python brand colors (theme-independent) */ + --python-blue: #3776ab; + --python-blue-light: #4584bb; + --python-blue-lighter: #5592cc; + --python-gold: #ffd43b; + --python-gold-dark: #ffcd02; + --python-gold-light: #ffdc5c; + + /* Heat palette - defined per theme below */ + + /* Layout */ + --sidebar-width: 280px; + --sidebar-collapsed: 44px; + --topbar-height: 56px; + --statusbar-height: 32px; + + /* Transitions */ + --transition-fast: 0.15s ease; + --transition-normal: 0.25s ease; +} + +/* Light theme (default) */ +:root, [data-theme="light"] { + --bg-primary: #ffffff; + --bg-secondary: #f8f9fa; + --bg-tertiary: #e9ecef; + --border: #e9ecef; + --border-subtle: #f0f2f5; + + --text-primary: #2e3338; + --text-secondary: #5a6c7d; + --text-muted: #6f767e; + + --accent: #3776ab; + --accent-hover: #2d5aa0; + --accent-glow: rgba(55, 118, 171, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.1); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.15); + + --header-gradient: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); + + /* Light mode heat palette - blue to yellow to orange to red (cold to hot) */ + --heat-1: #7ba3d1; + --heat-2: #a8d0ef; + --heat-3: #d6e9f8; + --heat-4: #ffe6a8; + --heat-5: #ffd43b; + --heat-6: #ffb84d; + --heat-7: #ff9966; + --heat-8: #ff6347; + + /* Code view specific */ + --code-bg: #ffffff; + --code-bg-line: #f8f9fa; + --code-border: #e9ecef; + --code-text: #2e3338; + --code-text-muted: #8b949e; + --code-accent: #3776ab; + + /* Navigation colors */ + --nav-caller: #2563eb; + --nav-caller-hover: #1d4ed8; + --nav-callee: #dc2626; + --nav-callee-hover: #b91c1c; +} + +/* Dark theme */ +[data-theme="dark"] { + --bg-primary: #0d1117; + --bg-secondary: #161b22; + --bg-tertiary: #21262d; + --border: #30363d; + --border-subtle: #21262d; + + --text-primary: #e6edf3; + --text-secondary: #8b949e; + --text-muted: #757e8a; + + --accent: #58a6ff; + --accent-hover: #79b8ff; + --accent-glow: rgba(88, 166, 255, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.4); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.5); + + --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); + + /* Dark mode heat palette - dark blue to teal to yellow to orange (cold to hot) */ + --heat-1: #4a7ba7; + --heat-2: #5a9fa8; + --heat-3: #6ab5b5; + --heat-4: #7ec488; + --heat-5: #a0d878; + --heat-6: #c4de6a; + --heat-7: #f4d44d; + --heat-8: #ff6b35; + + /* Code view specific - dark mode */ + --code-bg: #0d1117; + --code-bg-line: #161b22; + --code-border: #30363d; + --code-text: #e6edf3; + --code-text-muted: #6e7681; + --code-accent: #58a6ff; + + /* Navigation colors - dark theme friendly */ + --nav-caller: #58a6ff; + --nav-caller-hover: #4184e4; + --nav-callee: #f87171; + --nav-callee-hover: #e53e3e; +} + +/* -------------------------------------------------------------------------- + Base Styles + -------------------------------------------------------------------------- */ + +*, *::before, *::after { + box-sizing: border-box; +} + +html, body { + margin: 0; + padding: 0; +} + +body { + font-family: var(--font-sans); + font-size: 14px; + line-height: 1.6; + color: var(--text-primary); + background: var(--bg-primary); + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + transition: background var(--transition-normal), color var(--transition-normal); +} + +/* -------------------------------------------------------------------------- + Layout Structure + -------------------------------------------------------------------------- */ + +.app-layout { + display: flex; + flex-direction: column; +} + +/* -------------------------------------------------------------------------- + Top Bar + -------------------------------------------------------------------------- */ + +.top-bar { + height: var(--topbar-height); + background: var(--header-gradient); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + flex-shrink: 0; + box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); + border-bottom: 2px solid var(--python-gold); +} + +/* Brand / Logo */ +.brand { + display: flex; + align-items: center; + gap: 12px; + color: white; + text-decoration: none; + flex-shrink: 0; +} + +.brand-logo { + display: flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + flex-shrink: 0; +} + +/* Style the inlined SVG/img inside brand-logo */ +.brand-logo svg, +.brand-logo img { + width: 28px; + height: 28px; + display: block; + object-fit: contain; + filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.2)); +} + +.brand-info { + display: flex; + flex-direction: column; + line-height: 1.15; +} + +.brand-text { + font-weight: 700; + font-size: 16px; + letter-spacing: -0.3px; + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.15); +} + +.brand-subtitle { + font-weight: 500; + font-size: 10px; + opacity: 0.9; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.brand-divider { + width: 1px; + height: 16px; + background: rgba(255, 255, 255, 0.3); +} + +/* Toolbar */ +.toolbar { + display: flex; + align-items: center; + gap: 6px; + margin-left: auto; +} + +.toolbar-btn { + display: flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + padding: 0; + font-size: 15px; + color: white; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + cursor: pointer; + transition: all var(--transition-fast); +} + +.toolbar-btn:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(255, 255, 255, 0.35); +} + +.toolbar-btn:active { + transform: scale(0.95); +} + +/* -------------------------------------------------------------------------- + Status Bar + -------------------------------------------------------------------------- */ + +.status-bar { + height: var(--statusbar-height); + background: var(--bg-secondary); + border-top: 1px solid var(--border); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-secondary); + flex-shrink: 0; +} + +.status-item { + display: flex; + align-items: center; + gap: 5px; +} + +.status-item::before { + content: ''; + width: 4px; + height: 4px; + background: var(--python-gold); + border-radius: 50%; +} + +.status-item:first-child::before { + display: none; +} + +.status-label { + color: var(--text-muted); +} + +.status-value { + color: var(--text-primary); + font-weight: 500; +} + +.status-value.accent { + color: var(--accent); + font-weight: 600; +} + +/* -------------------------------------------------------------------------- + Animations + -------------------------------------------------------------------------- */ + +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +@keyframes slideUp { + from { + opacity: 0; + transform: translateY(12px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +@keyframes shimmer { + 0% { left: -100%; } + 100% { left: 100%; } +} + +/* -------------------------------------------------------------------------- + Focus States (Accessibility) + -------------------------------------------------------------------------- */ + +button:focus-visible, +select:focus-visible, +input:focus-visible { + outline: 2px solid var(--python-gold); + outline-offset: 2px; +} + +/* -------------------------------------------------------------------------- + Shared Responsive + -------------------------------------------------------------------------- */ + +@media (max-width: 900px) { + .brand-subtitle { + display: none; + } +} + +@media (max-width: 600px) { + .toolbar-btn:not(.theme-toggle) { + display: none; + } +} diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py new file mode 100644 index 00000000000..be63dbe3e90 --- /dev/null +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -0,0 +1,251 @@ +""" +Internal synchronization coordinator for the sample profiler. + +This module is used internally by the sample profiler to coordinate +the startup of target processes. It should not be called directly by users. +""" + +import os +import sys +import socket +import runpy +import time +import types +from typing import List, NoReturn + + +class CoordinatorError(Exception): + """Base exception for coordinator errors.""" + pass + + +class ArgumentError(CoordinatorError): + """Raised when invalid arguments are provided.""" + pass + + +class SyncError(CoordinatorError): + """Raised when synchronization with profiler fails.""" + pass + + +class TargetError(CoordinatorError): + """Raised when target execution fails.""" + pass + + +def _validate_arguments(args: List[str]) -> tuple[int, str, List[str]]: + """ + Validate and parse command line arguments. + + Args: + args: Command line arguments including script name + + Returns: + Tuple of (sync_port, working_directory, target_args) + + Raises: + ArgumentError: If arguments are invalid + """ + if len(args) < 4: + raise ArgumentError( + "Insufficient arguments. Expected: [args...]" + ) + + try: + sync_port = int(args[1]) + if not (1 <= sync_port <= 65535): + raise ValueError("Port out of range") + except ValueError as e: + raise ArgumentError(f"Invalid sync port '{args[1]}': {e}") from e + + cwd = args[2] + if not os.path.isdir(cwd): + raise ArgumentError(f"Working directory does not exist: {cwd}") + + target_args = args[3:] + if not target_args: + raise ArgumentError("No target specified") + + return sync_port, cwd, target_args + + +# Constants for socket communication +_MAX_RETRIES = 3 +_INITIAL_RETRY_DELAY = 0.1 +_SOCKET_TIMEOUT = 2.0 +_READY_MESSAGE = b"ready" + + +def _signal_readiness(sync_port: int) -> None: + """ + Signal readiness to the profiler via TCP socket. + + Args: + sync_port: Port number where profiler is listening + + Raises: + SyncError: If unable to signal readiness + """ + last_error = None + + for attempt in range(_MAX_RETRIES): + try: + # Use context manager for automatic cleanup + with socket.create_connection(("127.0.0.1", sync_port), timeout=_SOCKET_TIMEOUT) as sock: + sock.send(_READY_MESSAGE) + return + except (socket.error, OSError) as e: + last_error = e + if attempt < _MAX_RETRIES - 1: + # Exponential backoff before retry + time.sleep(_INITIAL_RETRY_DELAY * (2 ** attempt)) + + # If we get here, all retries failed + raise SyncError(f"Failed to signal readiness after {_MAX_RETRIES} attempts: {last_error}") from last_error + + +def _setup_environment(cwd: str) -> None: + """ + Set up the execution environment. + + Args: + cwd: Working directory to change to + + Raises: + TargetError: If unable to set up environment + """ + try: + os.chdir(cwd) + except OSError as e: + raise TargetError(f"Failed to change to directory {cwd}: {e}") from e + + # Add current directory to sys.path if not present (for module imports) + if cwd not in sys.path: + sys.path.insert(0, cwd) + + +def _execute_module(module_name: str, module_args: List[str]) -> None: + """ + Execute a Python module. + + Args: + module_name: Name of the module to execute + module_args: Arguments to pass to the module + + Raises: + TargetError: If module execution fails + """ + # Replace sys.argv to match how Python normally runs modules + # When running 'python -m module args', sys.argv is ["__main__.py", "args"] + sys.argv = [f"__main__.py"] + module_args + + try: + runpy.run_module(module_name, run_name="__main__", alter_sys=True) + except ImportError as e: + raise TargetError(f"Module '{module_name}' not found: {e}") from e + except SystemExit: + # SystemExit is normal for modules + pass + except Exception as e: + raise TargetError(f"Error executing module '{module_name}': {e}") from e + + +def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: + """ + Execute a Python script. + + Args: + script_path: Path to the script to execute + script_args: Arguments to pass to the script + cwd: Current working directory for path resolution + + Raises: + TargetError: If script execution fails + """ + # Make script path absolute if it isn't already + if not os.path.isabs(script_path): + script_path = os.path.join(cwd, script_path) + + if not os.path.isfile(script_path): + raise TargetError(f"Script not found: {script_path}") + + # Replace sys.argv to match original script call + sys.argv = [script_path] + script_args + + try: + with open(script_path, 'rb') as f: + source_code = f.read() + + except FileNotFoundError as e: + raise TargetError(f"Script file not found: {script_path}") from e + except PermissionError as e: + raise TargetError(f"Permission denied reading script: {script_path}") from e + + try: + main_module = types.ModuleType("__main__") + main_module.__file__ = script_path + main_module.__builtins__ = __builtins__ + # gh-140729: Create a __mp_main__ module to allow pickling + sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + + code = compile(source_code, script_path, 'exec', module='__main__') + exec(code, main_module.__dict__) + except SyntaxError as e: + raise TargetError(f"Syntax error in script {script_path}: {e}") from e + except SystemExit: + # SystemExit is normal for scripts + pass + except Exception as e: + raise TargetError(f"Error executing script '{script_path}': {e}") from e + + +def main() -> NoReturn: + """ + Main coordinator function. + + This function coordinates the startup of a target Python process + with the sample profiler by signaling when the process is ready + to be profiled. + """ + try: + # Parse and validate arguments + sync_port, cwd, target_args = _validate_arguments(sys.argv) + + # Set up execution environment + _setup_environment(cwd) + + # Signal readiness to profiler + _signal_readiness(sync_port) + + # Execute the target + if target_args[0] == "-m": + # Module execution + if len(target_args) < 2: + raise ArgumentError("Module name required after -m") + + module_name = target_args[1] + module_args = target_args[2:] + _execute_module(module_name, module_args) + else: + # Script execution + script_path = target_args[0] + script_args = target_args[1:] + _execute_script(script_path, script_args, cwd) + + except CoordinatorError as e: + print(f"Profiler coordinator error: {e}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + print("Interrupted", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Unexpected error in profiler coordinator: {e}", file=sys.stderr) + sys.exit(1) + + # Normal exit + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/Lib/profiling/sampling/_vendor/d3-flame-graph/4.1.3/d3-flamegraph-tooltip.min.js b/Lib/profiling/sampling/_vendor/d3-flame-graph/4.1.3/d3-flamegraph-tooltip.min.js new file mode 100644 index 00000000000..7468437093b --- /dev/null +++ b/Lib/profiling/sampling/_vendor/d3-flame-graph/4.1.3/d3-flamegraph-tooltip.min.js @@ -0,0 +1 @@ +!function(t,n){"object"==typeof exports&&"object"==typeof module?module.exports=n():"function"==typeof define&&define.amd?define([],n):"object"==typeof exports?exports.flamegraph=n():(t.flamegraph=t.flamegraph||{},t.flamegraph.tooltip=n())}(self,(function(){return(()=>{"use strict";var t={d:(n,e)=>{for(var r in e)t.o(e,r)&&!t.o(n,r)&&Object.defineProperty(n,r,{enumerable:!0,get:e[r]})},o:(t,n)=>Object.prototype.hasOwnProperty.call(t,n),r:t=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})}},n={};function e(){}function r(t){return null==t?e:function(){return this.querySelector(t)}}function i(t){return null==t?[]:Array.isArray(t)?t:Array.from(t)}function o(){return[]}function u(t){return null==t?o:function(){return this.querySelectorAll(t)}}function a(t){return function(){return this.matches(t)}}function s(t){return function(n){return n.matches(t)}}t.r(n),t.d(n,{defaultFlamegraphTooltip:()=>Ae});var l=Array.prototype.find;function c(){return this.firstElementChild}var f=Array.prototype.filter;function h(){return Array.from(this.children)}function p(t){return new Array(t.length)}function d(t,n){this.ownerDocument=t.ownerDocument,this.namespaceURI=t.namespaceURI,this._next=null,this._parent=t,this.__data__=n}function y(t){return function(){return t}}function v(t,n,e,r,i,o){for(var u,a=0,s=n.length,l=o.length;an?1:t>=n?0:NaN}d.prototype={constructor:d,appendChild:function(t){return this._parent.insertBefore(t,this._next)},insertBefore:function(t,n){return this._parent.insertBefore(t,n)},querySelector:function(t){return this._parent.querySelector(t)},querySelectorAll:function(t){return this._parent.querySelectorAll(t)}};var b="http://www.w3.org/1999/xhtml";const x={svg:"http://www.w3.org/2000/svg",xhtml:b,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};function A(t){var n=t+="",e=n.indexOf(":");return e>=0&&"xmlns"!==(n=t.slice(0,e))&&(t=t.slice(e+1)),x.hasOwnProperty(n)?{space:x[n],local:t}:t}function N(t){return function(){this.removeAttribute(t)}}function k(t){return function(){this.removeAttributeNS(t.space,t.local)}}function M(t,n){return function(){this.setAttribute(t,n)}}function E(t,n){return function(){this.setAttributeNS(t.space,t.local,n)}}function S(t,n){return function(){var e=n.apply(this,arguments);null==e?this.removeAttribute(t):this.setAttribute(t,e)}}function C(t,n){return function(){var e=n.apply(this,arguments);null==e?this.removeAttributeNS(t.space,t.local):this.setAttributeNS(t.space,t.local,e)}}function P(t){return t.ownerDocument&&t.ownerDocument.defaultView||t.document&&t||t.defaultView}function O(t){return function(){this.style.removeProperty(t)}}function q(t,n,e){return function(){this.style.setProperty(t,n,e)}}function j(t,n,e){return function(){var r=n.apply(this,arguments);null==r?this.style.removeProperty(t):this.style.setProperty(t,r,e)}}function T(t,n){return t.style.getPropertyValue(n)||P(t).getComputedStyle(t,null).getPropertyValue(n)}function X(t){return function(){delete this[t]}}function R(t,n){return function(){this[t]=n}}function I(t,n){return function(){var e=n.apply(this,arguments);null==e?delete this[t]:this[t]=e}}function L(t){return t.trim().split(/^|\s+/)}function D(t){return t.classList||new H(t)}function H(t){this._node=t,this._names=L(t.getAttribute("class")||"")}function Y(t,n){for(var e=D(t),r=-1,i=n.length;++r=0&&(n=t.slice(e+1),t=t.slice(0,e)),{type:t,name:n}}))}function st(t){return function(){var n=this.__on;if(n){for(var e,r=0,i=-1,o=n.length;r=0&&(this._names.splice(n,1),this._node.setAttribute("class",this._names.join(" ")))},contains:function(t){return this._names.indexOf(t)>=0}};var pt=[null];function dt(t,n){this._groups=t,this._parents=n}function yt(){return new dt([[document.documentElement]],pt)}dt.prototype=yt.prototype={constructor:dt,select:function(t){"function"!=typeof t&&(t=r(t));for(var n=this._groups,e=n.length,i=new Array(e),o=0;o=M&&(M=k+1);!(N=b[M])&&++M=0;)(r=i[o])&&(u&&4^r.compareDocumentPosition(u)&&u.parentNode.insertBefore(r,u),u=r);return this},sort:function(t){function n(n,e){return n&&e?t(n.__data__,e.__data__):!n-!e}t||(t=w);for(var e=this._groups,r=e.length,i=new Array(r),o=0;o1?this.each((null==n?O:"function"==typeof n?j:q)(t,n,null==e?"":e)):T(this.node(),t)},property:function(t,n){return arguments.length>1?this.each((null==n?X:"function"==typeof n?I:R)(t,n)):this.node()[t]},classed:function(t,n){var e=L(t+"");if(arguments.length<2){for(var r=D(this.node()),i=-1,o=e.length;++i{}};function _t(){for(var t,n=0,e=arguments.length,r={};n=0&&(e=t.slice(r+1),t=t.slice(0,r)),t&&!n.hasOwnProperty(t))throw new Error("unknown type: "+t);return{type:t,name:e}}))}function bt(t,n){for(var e,r=0,i=t.length;r0)for(var e,r,i=new Array(e),o=0;o=0&&n._call.call(void 0,t),n=n._next;--Mt}()}finally{Mt=0,function(){var t,n,e=Nt,r=1/0;for(;e;)e._call?(r>e._time&&(r=e._time),t=e,e=e._next):(n=e._next,e._next=null,e=t?t._next=n:Nt=n);kt=t,Ht(r)}(),Pt=0}}function Dt(){var t=qt.now(),n=t-Ct;n>1e3&&(Ot-=n,Ct=t)}function Ht(t){Mt||(Et&&(Et=clearTimeout(Et)),t-Pt>24?(t<1/0&&(Et=setTimeout(Lt,t-qt.now()-Ot)),St&&(St=clearInterval(St))):(St||(Ct=qt.now(),St=setInterval(Dt,1e3)),Mt=1,jt(Lt)))}function Yt(t,n,e){var r=new Rt;return n=null==n?0:+n,r.restart((e=>{r.stop(),t(e+n)}),n,e),r}Rt.prototype=It.prototype={constructor:Rt,restart:function(t,n,e){if("function"!=typeof t)throw new TypeError("callback is not a function");e=(null==e?Tt():+e)+(null==n?0:+n),this._next||kt===this||(kt?kt._next=this:Nt=this,kt=this),this._call=t,this._time=e,Ht()},stop:function(){this._call&&(this._call=null,this._time=1/0,Ht())}};var Bt=At("start","end","cancel","interrupt"),$t=[];function Vt(t,n,e,r,i,o){var u=t.__transition;if(u){if(e in u)return}else t.__transition={};!function(t,n,e){var r,i=t.__transition;function o(t){e.state=1,e.timer.restart(u,e.delay,e.time),e.delay<=t&&u(t-e.delay)}function u(o){var l,c,f,h;if(1!==e.state)return s();for(l in i)if((h=i[l]).name===e.name){if(3===h.state)return Yt(u);4===h.state?(h.state=6,h.timer.stop(),h.on.call("interrupt",t,t.__data__,h.index,h.group),delete i[l]):+l0)throw new Error("too late; already scheduled");return e}function Ut(t,n){var e=Ft(t,n);if(e.state>3)throw new Error("too late; already running");return e}function Ft(t,n){var e=t.__transition;if(!e||!(e=e[n]))throw new Error("transition not found");return e}function Kt(t,n){return t=+t,n=+n,function(e){return t*(1-e)+n*e}}var Wt,Gt=180/Math.PI,Jt={translateX:0,translateY:0,rotate:0,skewX:0,scaleX:1,scaleY:1};function Qt(t,n,e,r,i,o){var u,a,s;return(u=Math.sqrt(t*t+n*n))&&(t/=u,n/=u),(s=t*e+n*r)&&(e-=t*s,r-=n*s),(a=Math.sqrt(e*e+r*r))&&(e/=a,r/=a,s/=a),t*r180?n+=360:n-t>180&&(t+=360),o.push({i:e.push(i(e)+"rotate(",null,r)-2,x:Kt(t,n)})):n&&e.push(i(e)+"rotate("+n+r)}(o.rotate,u.rotate,a,s),function(t,n,e,o){t!==n?o.push({i:e.push(i(e)+"skewX(",null,r)-2,x:Kt(t,n)}):n&&e.push(i(e)+"skewX("+n+r)}(o.skewX,u.skewX,a,s),function(t,n,e,r,o,u){if(t!==e||n!==r){var a=o.push(i(o)+"scale(",null,",",null,")");u.push({i:a-4,x:Kt(t,e)},{i:a-2,x:Kt(n,r)})}else 1===e&&1===r||o.push(i(o)+"scale("+e+","+r+")")}(o.scaleX,o.scaleY,u.scaleX,u.scaleY,a,s),o=u=null,function(t){for(var n,e=-1,r=s.length;++e>8&15|n>>4&240,n>>4&15|240&n,(15&n)<<4|15&n,1):8===e?Mn(n>>24&255,n>>16&255,n>>8&255,(255&n)/255):4===e?Mn(n>>12&15|n>>8&240,n>>8&15|n>>4&240,n>>4&15|240&n,((15&n)<<4|15&n)/255):null):(n=yn.exec(t))?new Cn(n[1],n[2],n[3],1):(n=vn.exec(t))?new Cn(255*n[1]/100,255*n[2]/100,255*n[3]/100,1):(n=gn.exec(t))?Mn(n[1],n[2],n[3],n[4]):(n=_n.exec(t))?Mn(255*n[1]/100,255*n[2]/100,255*n[3]/100,n[4]):(n=mn.exec(t))?jn(n[1],n[2]/100,n[3]/100,1):(n=wn.exec(t))?jn(n[1],n[2]/100,n[3]/100,n[4]):bn.hasOwnProperty(t)?kn(bn[t]):"transparent"===t?new Cn(NaN,NaN,NaN,0):null}function kn(t){return new Cn(t>>16&255,t>>8&255,255&t,1)}function Mn(t,n,e,r){return r<=0&&(t=n=e=NaN),new Cn(t,n,e,r)}function En(t){return t instanceof sn||(t=Nn(t)),t?new Cn((t=t.rgb()).r,t.g,t.b,t.opacity):new Cn}function Sn(t,n,e,r){return 1===arguments.length?En(t):new Cn(t,n,e,null==r?1:r)}function Cn(t,n,e,r){this.r=+t,this.g=+n,this.b=+e,this.opacity=+r}function Pn(){return"#"+qn(this.r)+qn(this.g)+qn(this.b)}function On(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"rgb(":"rgba(")+Math.max(0,Math.min(255,Math.round(this.r)||0))+", "+Math.max(0,Math.min(255,Math.round(this.g)||0))+", "+Math.max(0,Math.min(255,Math.round(this.b)||0))+(1===t?")":", "+t+")")}function qn(t){return((t=Math.max(0,Math.min(255,Math.round(t)||0)))<16?"0":"")+t.toString(16)}function jn(t,n,e,r){return r<=0?t=n=e=NaN:e<=0||e>=1?t=n=NaN:n<=0&&(t=NaN),new Xn(t,n,e,r)}function Tn(t){if(t instanceof Xn)return new Xn(t.h,t.s,t.l,t.opacity);if(t instanceof sn||(t=Nn(t)),!t)return new Xn;if(t instanceof Xn)return t;var n=(t=t.rgb()).r/255,e=t.g/255,r=t.b/255,i=Math.min(n,e,r),o=Math.max(n,e,r),u=NaN,a=o-i,s=(o+i)/2;return a?(u=n===o?(e-r)/a+6*(e0&&s<1?0:u,new Xn(u,a,s,t.opacity)}function Xn(t,n,e,r){this.h=+t,this.s=+n,this.l=+e,this.opacity=+r}function Rn(t,n,e){return 255*(t<60?n+(e-n)*t/60:t<180?e:t<240?n+(e-n)*(240-t)/60:n)}function In(t,n,e,r,i){var o=t*t,u=o*t;return((1-3*t+3*o-u)*n+(4-6*o+3*u)*e+(1+3*t+3*o-3*u)*r+u*i)/6}un(sn,Nn,{copy:function(t){return Object.assign(new this.constructor,this,t)},displayable:function(){return this.rgb().displayable()},hex:xn,formatHex:xn,formatHsl:function(){return Tn(this).formatHsl()},formatRgb:An,toString:An}),un(Cn,Sn,an(sn,{brighter:function(t){return t=null==t?cn:Math.pow(cn,t),new Cn(this.r*t,this.g*t,this.b*t,this.opacity)},darker:function(t){return t=null==t?ln:Math.pow(ln,t),new Cn(this.r*t,this.g*t,this.b*t,this.opacity)},rgb:function(){return this},displayable:function(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:Pn,formatHex:Pn,formatRgb:On,toString:On})),un(Xn,(function(t,n,e,r){return 1===arguments.length?Tn(t):new Xn(t,n,e,null==r?1:r)}),an(sn,{brighter:function(t){return t=null==t?cn:Math.pow(cn,t),new Xn(this.h,this.s,this.l*t,this.opacity)},darker:function(t){return t=null==t?ln:Math.pow(ln,t),new Xn(this.h,this.s,this.l*t,this.opacity)},rgb:function(){var t=this.h%360+360*(this.h<0),n=isNaN(t)||isNaN(this.s)?0:this.s,e=this.l,r=e+(e<.5?e:1-e)*n,i=2*e-r;return new Cn(Rn(t>=240?t-240:t+120,i,r),Rn(t,i,r),Rn(t<120?t+240:t-120,i,r),this.opacity)},displayable:function(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl:function(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"hsl(":"hsla(")+(this.h||0)+", "+100*(this.s||0)+"%, "+100*(this.l||0)+"%"+(1===t?")":", "+t+")")}}));const Ln=t=>()=>t;function Dn(t,n){return function(e){return t+e*n}}function Hn(t){return 1==(t=+t)?Yn:function(n,e){return e-n?function(t,n,e){return t=Math.pow(t,e),n=Math.pow(n,e)-t,e=1/e,function(r){return Math.pow(t+r*n,e)}}(n,e,t):Ln(isNaN(n)?e:n)}}function Yn(t,n){var e=n-t;return e?Dn(t,e):Ln(isNaN(t)?n:t)}const Bn=function t(n){var e=Hn(n);function r(t,n){var r=e((t=Sn(t)).r,(n=Sn(n)).r),i=e(t.g,n.g),o=e(t.b,n.b),u=Yn(t.opacity,n.opacity);return function(n){return t.r=r(n),t.g=i(n),t.b=o(n),t.opacity=u(n),t+""}}return r.gamma=t,r}(1);function $n(t){return function(n){var e,r,i=n.length,o=new Array(i),u=new Array(i),a=new Array(i);for(e=0;e=1?(e=1,n-1):Math.floor(e*n),i=t[r],o=t[r+1],u=r>0?t[r-1]:2*i-o,a=ro&&(i=n.slice(o,i),a[u]?a[u]+=i:a[++u]=i),(e=e[0])===(r=r[0])?a[u]?a[u]+=r:a[++u]=r:(a[++u]=null,s.push({i:u,x:Kt(e,r)})),o=zn.lastIndex;return o=0&&(t=t.slice(0,n)),!t||"start"===t}))}(n)?zt:Ut;return function(){var u=o(this,t),a=u.on;a!==r&&(i=(r=a).copy()).on(n,e),u.on=i}}var ce=vt.prototype.constructor;function fe(t){return function(){this.style.removeProperty(t)}}function he(t,n,e){return function(r){this.style.setProperty(t,n.call(this,r),e)}}function pe(t,n,e){var r,i;function o(){var o=n.apply(this,arguments);return o!==i&&(r=(i=o)&&he(t,o,e)),r}return o._value=n,o}function de(t){return function(n){this.textContent=t.call(this,n)}}function ye(t){var n,e;function r(){var r=t.apply(this,arguments);return r!==e&&(n=(e=r)&&de(r)),n}return r._value=t,r}var ve=0;function ge(t,n,e,r){this._groups=t,this._parents=n,this._name=e,this._id=r}function _e(){return++ve}var me=vt.prototype;ge.prototype=function(t){return vt().transition(t)}.prototype={constructor:ge,select:function(t){var n=this._name,e=this._id;"function"!=typeof t&&(t=r(t));for(var i=this._groups,o=i.length,u=new Array(o),a=0;a2&&e.state<5,e.state=6,e.timer.stop(),e.on.call(r?"interrupt":"cancel",t,t.__data__,e.index,e.group),delete o[i]):u=!1;u&&delete t.__transition}}(this,t)}))},vt.prototype.transition=function(t){var n,e;t instanceof ge?(n=t._id,t=t._name):(n=_e(),(e=we).time=Tt(),t=null==t?null:t+"");for(var r=this._groups,i=r.length,o=0;o{"use strict";var t={d:(n,e)=>{for(var r in e)t.o(e,r)&&!t.o(n,r)&&Object.defineProperty(n,r,{enumerable:!0,get:e[r]})},o:(t,n)=>Object.prototype.hasOwnProperty.call(t,n)},n={};function e(){}function r(t){return null==t?e:function(){return this.querySelector(t)}}function i(t){return null==t?[]:Array.isArray(t)?t:Array.from(t)}function o(){return[]}function u(t){return null==t?o:function(){return this.querySelectorAll(t)}}function a(t){return function(){return this.matches(t)}}function l(t){return function(n){return n.matches(t)}}t.d(n,{default:()=>xr});var s=Array.prototype.find;function c(){return this.firstElementChild}var h=Array.prototype.filter;function f(){return Array.from(this.children)}function p(t){return new Array(t.length)}function d(t,n){this.ownerDocument=t.ownerDocument,this.namespaceURI=t.namespaceURI,this._next=null,this._parent=t,this.__data__=n}function g(t){return function(){return t}}function v(t,n,e,r,i,o){for(var u,a=0,l=n.length,s=o.length;an?1:t>=n?0:NaN}d.prototype={constructor:d,appendChild:function(t){return this._parent.insertBefore(t,this._next)},insertBefore:function(t,n){return this._parent.insertBefore(t,n)},querySelector:function(t){return this._parent.querySelector(t)},querySelectorAll:function(t){return this._parent.querySelectorAll(t)}};var b="http://www.w3.org/1999/xhtml";const x={svg:"http://www.w3.org/2000/svg",xhtml:b,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};function M(t){var n=t+="",e=n.indexOf(":");return e>=0&&"xmlns"!==(n=t.slice(0,e))&&(t=t.slice(e+1)),x.hasOwnProperty(n)?{space:x[n],local:t}:t}function A(t){return function(){this.removeAttribute(t)}}function N(t){return function(){this.removeAttributeNS(t.space,t.local)}}function k(t,n){return function(){this.setAttribute(t,n)}}function E(t,n){return function(){this.setAttributeNS(t.space,t.local,n)}}function S(t,n){return function(){var e=n.apply(this,arguments);null==e?this.removeAttribute(t):this.setAttribute(t,e)}}function C(t,n){return function(){var e=n.apply(this,arguments);null==e?this.removeAttributeNS(t.space,t.local):this.setAttributeNS(t.space,t.local,e)}}function j(t){return t.ownerDocument&&t.ownerDocument.defaultView||t.document&&t||t.defaultView}function q(t){return function(){this.style.removeProperty(t)}}function O(t,n,e){return function(){this.style.setProperty(t,n,e)}}function P(t,n,e){return function(){var r=n.apply(this,arguments);null==r?this.style.removeProperty(t):this.style.setProperty(t,r,e)}}function H(t,n){return t.style.getPropertyValue(n)||j(t).getComputedStyle(t,null).getPropertyValue(n)}function L(t){return function(){delete this[t]}}function T(t,n){return function(){this[t]=n}}function D(t,n){return function(){var e=n.apply(this,arguments);null==e?delete this[t]:this[t]=e}}function B(t){return t.trim().split(/^|\s+/)}function X(t){return t.classList||new z(t)}function z(t){this._node=t,this._names=B(t.getAttribute("class")||"")}function R(t,n){for(var e=X(t),r=-1,i=n.length;++r=0&&(n=t.slice(e+1),t=t.slice(0,e)),{type:t,name:n}}))}function lt(t){return function(){var n=this.__on;if(n){for(var e,r=0,i=-1,o=n.length;r=0&&(this._names.splice(n,1),this._node.setAttribute("class",this._names.join(" ")))},contains:function(t){return this._names.indexOf(t)>=0}};var pt=[null];function dt(t,n){this._groups=t,this._parents=n}function gt(){return new dt([[document.documentElement]],pt)}dt.prototype=gt.prototype={constructor:dt,select:function(t){"function"!=typeof t&&(t=r(t));for(var n=this._groups,e=n.length,i=new Array(e),o=0;o=k&&(k=N+1);!(A=b[k])&&++k=0;)(r=i[o])&&(u&&4^r.compareDocumentPosition(u)&&u.parentNode.insertBefore(r,u),u=r);return this},sort:function(t){function n(n,e){return n&&e?t(n.__data__,e.__data__):!n-!e}t||(t=_);for(var e=this._groups,r=e.length,i=new Array(r),o=0;o1?this.each((null==n?q:"function"==typeof n?P:O)(t,n,null==e?"":e)):H(this.node(),t)},property:function(t,n){return arguments.length>1?this.each((null==n?L:"function"==typeof n?D:T)(t,n)):this.node()[t]},classed:function(t,n){var e=B(t+"");if(arguments.length<2){for(var r=X(this.node()),i=-1,o=e.length;++i1?r[0]+r.slice(2):r,+t.slice(e+1)]}function wt(t){return(t=mt(Math.abs(t)))?t[1]:NaN}var _t,bt=/^(?:(.)?([<>=^]))?([+\-( ])?([$#])?(0)?(\d+)?(,)?(\.\d+)?(~)?([a-z%])?$/i;function xt(t){if(!(n=bt.exec(t)))throw new Error("invalid format: "+t);var n;return new Mt({fill:n[1],align:n[2],sign:n[3],symbol:n[4],zero:n[5],width:n[6],comma:n[7],precision:n[8]&&n[8].slice(1),trim:n[9],type:n[10]})}function Mt(t){this.fill=void 0===t.fill?" ":t.fill+"",this.align=void 0===t.align?">":t.align+"",this.sign=void 0===t.sign?"-":t.sign+"",this.symbol=void 0===t.symbol?"":t.symbol+"",this.zero=!!t.zero,this.width=void 0===t.width?void 0:+t.width,this.comma=!!t.comma,this.precision=void 0===t.precision?void 0:+t.precision,this.trim=!!t.trim,this.type=void 0===t.type?"":t.type+""}function At(t,n){var e=mt(t,n);if(!e)return t+"";var r=e[0],i=e[1];return i<0?"0."+new Array(-i).join("0")+r:r.length>i+1?r.slice(0,i+1)+"."+r.slice(i+1):r+new Array(i-r.length+2).join("0")}xt.prototype=Mt.prototype,Mt.prototype.toString=function(){return this.fill+this.align+this.sign+this.symbol+(this.zero?"0":"")+(void 0===this.width?"":Math.max(1,0|this.width))+(this.comma?",":"")+(void 0===this.precision?"":"."+Math.max(0,0|this.precision))+(this.trim?"~":"")+this.type};const Nt={"%":(t,n)=>(100*t).toFixed(n),b:t=>Math.round(t).toString(2),c:t=>t+"",d:function(t){return Math.abs(t=Math.round(t))>=1e21?t.toLocaleString("en").replace(/,/g,""):t.toString(10)},e:(t,n)=>t.toExponential(n),f:(t,n)=>t.toFixed(n),g:(t,n)=>t.toPrecision(n),o:t=>Math.round(t).toString(8),p:(t,n)=>At(100*t,n),r:At,s:function(t,n){var e=mt(t,n);if(!e)return t+"";var r=e[0],i=e[1],o=i-(_t=3*Math.max(-8,Math.min(8,Math.floor(i/3))))+1,u=r.length;return o===u?r:o>u?r+new Array(o-u+1).join("0"):o>0?r.slice(0,o)+"."+r.slice(o):"0."+new Array(1-o).join("0")+mt(t,Math.max(0,n+o-1))[0]},X:t=>Math.round(t).toString(16).toUpperCase(),x:t=>Math.round(t).toString(16)};function kt(t){return t}var Et,St,Ct,jt=Array.prototype.map,qt=["y","z","a","f","p","n","µ","m","","k","M","G","T","P","E","Z","Y"];function Ot(t){var n,e,r=void 0===t.grouping||void 0===t.thousands?kt:(n=jt.call(t.grouping,Number),e=t.thousands+"",function(t,r){for(var i=t.length,o=[],u=0,a=n[0],l=0;i>0&&a>0&&(l+a+1>r&&(a=Math.max(1,r-l)),o.push(t.substring(i-=a,i+a)),!((l+=a+1)>r));)a=n[u=(u+1)%n.length];return o.reverse().join(e)}),i=void 0===t.currency?"":t.currency[0]+"",o=void 0===t.currency?"":t.currency[1]+"",u=void 0===t.decimal?".":t.decimal+"",a=void 0===t.numerals?kt:function(t){return function(n){return n.replace(/[0-9]/g,(function(n){return t[+n]}))}}(jt.call(t.numerals,String)),l=void 0===t.percent?"%":t.percent+"",s=void 0===t.minus?"−":t.minus+"",c=void 0===t.nan?"NaN":t.nan+"";function h(t){var n=(t=xt(t)).fill,e=t.align,h=t.sign,f=t.symbol,p=t.zero,d=t.width,g=t.comma,v=t.precision,y=t.trim,m=t.type;"n"===m?(g=!0,m="g"):Nt[m]||(void 0===v&&(v=12),y=!0,m="g"),(p||"0"===n&&"="===e)&&(p=!0,n="0",e="=");var w="$"===f?i:"#"===f&&/[boxX]/.test(m)?"0"+m.toLowerCase():"",_="$"===f?o:/[%p]/.test(m)?l:"",b=Nt[m],x=/[defgprs%]/.test(m);function M(t){var i,o,l,f=w,M=_;if("c"===m)M=b(t)+M,t="";else{var A=(t=+t)<0||1/t<0;if(t=isNaN(t)?c:b(Math.abs(t),v),y&&(t=function(t){t:for(var n,e=t.length,r=1,i=-1;r0&&(i=0)}return i>0?t.slice(0,i)+t.slice(n+1):t}(t)),A&&0==+t&&"+"!==h&&(A=!1),f=(A?"("===h?h:s:"-"===h||"("===h?"":h)+f,M=("s"===m?qt[8+_t/3]:"")+M+(A&&"("===h?")":""),x)for(i=-1,o=t.length;++i(l=t.charCodeAt(i))||l>57){M=(46===l?u+t.slice(i+1):t.slice(i))+M,t=t.slice(0,i);break}}g&&!p&&(t=r(t,1/0));var N=f.length+t.length+M.length,k=N>1)+f+t+M+k.slice(N);break;default:t=k+f+t+M}return a(t)}return v=void 0===v?6:/[gprs]/.test(m)?Math.max(1,Math.min(21,v)):Math.max(0,Math.min(20,v)),M.toString=function(){return t+""},M}return{format:h,formatPrefix:function(t,n){var e=h(((t=xt(t)).type="f",t)),r=3*Math.max(-8,Math.min(8,Math.floor(wt(n)/3))),i=Math.pow(10,-r),o=qt[8+r/3];return function(t){return e(i*t)+o}}}}function Pt(t,n){return null==t||null==n?NaN:tn?1:t>=n?0:NaN}function Ht(t){t.x0=Math.round(t.x0),t.y0=Math.round(t.y0),t.x1=Math.round(t.x1),t.y1=Math.round(t.y1)}function Lt(){var t=1,n=1,e=0,r=!1;function i(i){var o=i.height+1;return i.x0=i.y0=e,i.x1=t,i.y1=n/o,i.eachBefore(function(t,n){return function(r){r.children&&function(t,n,e,r,i){for(var o,u=t.children,a=-1,l=u.length,s=t.value&&(r-n)/t.value;++a=0;)n+=e[r].value;else n=1;t.value=n}function Dt(t,n){t instanceof Map?(t=[void 0,t],void 0===n&&(n=Xt)):void 0===n&&(n=Bt);for(var e,r,i,o,u,a=new It(t),l=[a];e=l.pop();)if((i=n(e.data))&&(u=(i=Array.from(i)).length))for(e.children=i,o=u-1;o>=0;--o)l.push(r=i[o]=new It(i[o])),r.parent=e,r.depth=e.depth+1;return a.eachBefore(Rt)}function Bt(t){return t.children}function Xt(t){return Array.isArray(t)?t[1]:null}function zt(t){void 0!==t.data.value&&(t.value=t.data.value),t.data=t.data.data}function Rt(t){var n=0;do{t.height=n}while((t=t.parent)&&t.height<++n)}function It(t){this.data=t,this.depth=this.height=0,this.parent=null}Et=Ot({thousands:",",grouping:[3],currency:["$",""]}),St=Et.format,Ct=Et.formatPrefix,It.prototype=Dt.prototype={constructor:It,count:function(){return this.eachAfter(Tt)},each:function(t,n){let e=-1;for(const r of this)t.call(n,r,++e,this);return this},eachAfter:function(t,n){for(var e,r,i,o=this,u=[o],a=[],l=-1;o=u.pop();)if(a.push(o),e=o.children)for(r=0,i=e.length;r=0;--r)o.push(e[r]);return this},find:function(t,n){let e=-1;for(const r of this)if(t.call(n,r,++e,this))return r},sum:function(t){return this.eachAfter((function(n){for(var e=+t(n.data)||0,r=n.children,i=r&&r.length;--i>=0;)e+=r[i].value;n.value=e}))},sort:function(t){return this.eachBefore((function(n){n.children&&n.children.sort(t)}))},path:function(t){for(var n=this,e=function(t,n){if(t===n)return t;var e=t.ancestors(),r=n.ancestors(),i=null;t=e.pop(),n=r.pop();for(;t===n;)i=t,t=e.pop(),n=r.pop();return i}(n,t),r=[n];n!==e;)n=n.parent,r.push(n);for(var i=r.length;t!==e;)r.splice(i,0,t),t=t.parent;return r},ancestors:function(){for(var t=this,n=[t];t=t.parent;)n.push(t);return n},descendants:function(){return Array.from(this)},leaves:function(){var t=[];return this.eachBefore((function(n){n.children||t.push(n)})),t},links:function(){var t=this,n=[];return t.each((function(e){e!==t&&n.push({source:e.parent,target:e})})),n},copy:function(){return Dt(this).eachBefore(zt)},[Symbol.iterator]:function*(){var t,n,e,r,i=this,o=[i];do{for(t=o.reverse(),o=[];i=t.pop();)if(yield i,n=i.children)for(e=0,r=n.length;e=0?(o>=$t?10:o>=Vt?5:o>=Yt?2:1)*Math.pow(10,i):-Math.pow(10,-i)/(o>=$t?10:o>=Vt?5:o>=Yt?2:1)}function Ut(t){let n=t,e=t,r=t;function i(t,n,i=0,o=t.length){if(i>>1;r(t[e],n)<0?i=e+1:o=e}while(it(n)-e,e=Pt,r=(n,e)=>Pt(t(n),e)),{left:i,center:function(t,e,r=0,o=t.length){const u=i(t,e,r,o-1);return u>r&&n(t[u-1],e)>-n(t[u],e)?u-1:u},right:function(t,n,i=0,o=t.length){if(i>>1;r(t[e],n)<=0?i=e+1:o=e}while(i>8&15|n>>4&240,n>>4&15|240&n,(15&n)<<4|15&n,1):8===e?mn(n>>24&255,n>>16&255,n>>8&255,(255&n)/255):4===e?mn(n>>12&15|n>>8&240,n>>8&15|n>>4&240,n>>4&15|240&n,((15&n)<<4|15&n)/255):null):(n=an.exec(t))?new bn(n[1],n[2],n[3],1):(n=ln.exec(t))?new bn(255*n[1]/100,255*n[2]/100,255*n[3]/100,1):(n=sn.exec(t))?mn(n[1],n[2],n[3],n[4]):(n=cn.exec(t))?mn(255*n[1]/100,255*n[2]/100,255*n[3]/100,n[4]):(n=hn.exec(t))?Nn(n[1],n[2]/100,n[3]/100,1):(n=fn.exec(t))?Nn(n[1],n[2]/100,n[3]/100,n[4]):pn.hasOwnProperty(t)?yn(pn[t]):"transparent"===t?new bn(NaN,NaN,NaN,0):null}function yn(t){return new bn(t>>16&255,t>>8&255,255&t,1)}function mn(t,n,e,r){return r<=0&&(t=n=e=NaN),new bn(t,n,e,r)}function wn(t){return t instanceof Qt||(t=vn(t)),t?new bn((t=t.rgb()).r,t.g,t.b,t.opacity):new bn}function _n(t,n,e,r){return 1===arguments.length?wn(t):new bn(t,n,e,null==r?1:r)}function bn(t,n,e,r){this.r=+t,this.g=+n,this.b=+e,this.opacity=+r}function xn(){return"#"+An(this.r)+An(this.g)+An(this.b)}function Mn(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"rgb(":"rgba(")+Math.max(0,Math.min(255,Math.round(this.r)||0))+", "+Math.max(0,Math.min(255,Math.round(this.g)||0))+", "+Math.max(0,Math.min(255,Math.round(this.b)||0))+(1===t?")":", "+t+")")}function An(t){return((t=Math.max(0,Math.min(255,Math.round(t)||0)))<16?"0":"")+t.toString(16)}function Nn(t,n,e,r){return r<=0?t=n=e=NaN:e<=0||e>=1?t=n=NaN:n<=0&&(t=NaN),new En(t,n,e,r)}function kn(t){if(t instanceof En)return new En(t.h,t.s,t.l,t.opacity);if(t instanceof Qt||(t=vn(t)),!t)return new En;if(t instanceof En)return t;var n=(t=t.rgb()).r/255,e=t.g/255,r=t.b/255,i=Math.min(n,e,r),o=Math.max(n,e,r),u=NaN,a=o-i,l=(o+i)/2;return a?(u=n===o?(e-r)/a+6*(e0&&l<1?0:u,new En(u,a,l,t.opacity)}function En(t,n,e,r){this.h=+t,this.s=+n,this.l=+e,this.opacity=+r}function Sn(t,n,e){return 255*(t<60?n+(e-n)*t/60:t<180?e:t<240?n+(e-n)*(240-t)/60:n)}function Cn(t,n,e,r,i){var o=t*t,u=o*t;return((1-3*t+3*o-u)*n+(4-6*o+3*u)*e+(1+3*t+3*o-3*u)*r+u*i)/6}Wt(Qt,vn,{copy:function(t){return Object.assign(new this.constructor,this,t)},displayable:function(){return this.rgb().displayable()},hex:dn,formatHex:dn,formatHsl:function(){return kn(this).formatHsl()},formatRgb:gn,toString:gn}),Wt(bn,_n,Jt(Qt,{brighter:function(t){return t=null==t?nn:Math.pow(nn,t),new bn(this.r*t,this.g*t,this.b*t,this.opacity)},darker:function(t){return t=null==t?tn:Math.pow(tn,t),new bn(this.r*t,this.g*t,this.b*t,this.opacity)},rgb:function(){return this},displayable:function(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:xn,formatHex:xn,formatRgb:Mn,toString:Mn})),Wt(En,(function(t,n,e,r){return 1===arguments.length?kn(t):new En(t,n,e,null==r?1:r)}),Jt(Qt,{brighter:function(t){return t=null==t?nn:Math.pow(nn,t),new En(this.h,this.s,this.l*t,this.opacity)},darker:function(t){return t=null==t?tn:Math.pow(tn,t),new En(this.h,this.s,this.l*t,this.opacity)},rgb:function(){var t=this.h%360+360*(this.h<0),n=isNaN(t)||isNaN(this.s)?0:this.s,e=this.l,r=e+(e<.5?e:1-e)*n,i=2*e-r;return new bn(Sn(t>=240?t-240:t+120,i,r),Sn(t,i,r),Sn(t<120?t+240:t-120,i,r),this.opacity)},displayable:function(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl:function(){var t=this.opacity;return(1===(t=isNaN(t)?1:Math.max(0,Math.min(1,t)))?"hsl(":"hsla(")+(this.h||0)+", "+100*(this.s||0)+"%, "+100*(this.l||0)+"%"+(1===t?")":", "+t+")")}}));const jn=t=>()=>t;function qn(t,n){return function(e){return t+e*n}}function On(t){return 1==(t=+t)?Pn:function(n,e){return e-n?function(t,n,e){return t=Math.pow(t,e),n=Math.pow(n,e)-t,e=1/e,function(r){return Math.pow(t+r*n,e)}}(n,e,t):jn(isNaN(n)?e:n)}}function Pn(t,n){var e=n-t;return e?qn(t,e):jn(isNaN(t)?n:t)}const Hn=function t(n){var e=On(n);function r(t,n){var r=e((t=_n(t)).r,(n=_n(n)).r),i=e(t.g,n.g),o=e(t.b,n.b),u=Pn(t.opacity,n.opacity);return function(n){return t.r=r(n),t.g=i(n),t.b=o(n),t.opacity=u(n),t+""}}return r.gamma=t,r}(1);function Ln(t){return function(n){var e,r,i=n.length,o=new Array(i),u=new Array(i),a=new Array(i);for(e=0;e=1?(e=1,n-1):Math.floor(e*n),i=t[r],o=t[r+1],u=r>0?t[r-1]:2*i-o,a=ro&&(i=n.slice(o,i),a[u]?a[u]+=i:a[++u]=i),(e=e[0])===(r=r[0])?a[u]?a[u]+=r:a[++u]=r:(a[++u]=null,l.push({i:u,x:Bn(e,r)})),o=Rn.lastIndex;return on&&(e=t,t=n,n=e),s=function(e){return Math.max(t,Math.min(n,e))}),r=l>2?Wn:Kn,i=o=null,h}function h(n){return null==n||isNaN(n=+n)?e:(i||(i=r(u.map(t),a,l)))(t(s(n)))}return h.invert=function(e){return s(n((o||(o=r(a,u.map(t),Bn)))(e)))},h.domain=function(t){return arguments.length?(u=Array.from(t,Fn),c()):u.slice()},h.range=function(t){return arguments.length?(a=Array.from(t),c()):a.slice()},h.rangeRound=function(t){return a=Array.from(t),l=Yn,c()},h.clamp=function(t){return arguments.length?(s=!!t||Zn,c()):s!==Zn},h.interpolate=function(t){return arguments.length?(l=t,c()):l},h.unknown=function(t){return arguments.length?(e=t,h):e},function(e,r){return t=e,n=r,c()}}function te(){return Qn()(Zn,Zn)}function ne(t,n){switch(arguments.length){case 0:break;case 1:this.range(t);break;default:this.range(n).domain(t)}return this}function ee(t,n,e,r){var i,o=function(t,n,e){var r=Math.abs(n-t)/Math.max(0,e),i=Math.pow(10,Math.floor(Math.log(r)/Math.LN10)),o=r/i;return o>=$t?i*=10:o>=Vt?i*=5:o>=Yt&&(i*=2),n0)return[t];if((r=n0){let e=Math.round(t/u),r=Math.round(n/u);for(e*un&&--r,o=new Array(i=r-e+1);++an&&--r,o=new Array(i=r-e+1);++a0;){if((i=Ft(l,s,e))===r)return o[u]=l,o[a]=s,n(o);if(i>0)l=Math.floor(l/i)*i,s=Math.ceil(s/i)*i;else{if(!(i<0))break;l=Math.ceil(l*i)/i,s=Math.floor(s*i)/i}r=i}return t},t}function ie(){var t=te();return t.copy=function(){return Jn(t,ie())},ne.apply(t,arguments),re(t)}function oe(t){return((t*=2)<=1?t*t*t:(t-=2)*t*t+2)/2}var ue={value:()=>{}};function ae(){for(var t,n=0,e=arguments.length,r={};n=0&&(e=t.slice(r+1),t=t.slice(0,r)),t&&!n.hasOwnProperty(t))throw new Error("unknown type: "+t);return{type:t,name:e}}))}function ce(t,n){for(var e,r=0,i=t.length;r0)for(var e,r,i=new Array(e),o=0;o=0&&n._call.call(void 0,t),n=n._next;--ge}()}finally{ge=0,function(){var t,n,e=pe,r=1/0;for(;e;)e._call?(r>e._time&&(r=e._time),t=e,e=e._next):(n=e._next,e._next=null,e=t?t._next=n:pe=n);de=t,Ce(r)}(),we=0}}function Se(){var t=be.now(),n=t-me;n>1e3&&(_e-=n,me=t)}function Ce(t){ge||(ve&&(ve=clearTimeout(ve)),t-we>24?(t<1/0&&(ve=setTimeout(Ee,t-be.now()-_e)),ye&&(ye=clearInterval(ye))):(ye||(me=be.now(),ye=setInterval(Se,1e3)),ge=1,xe(Ee)))}function je(t,n,e){var r=new Ne;return n=null==n?0:+n,r.restart((e=>{r.stop(),t(e+n)}),n,e),r}Ne.prototype=ke.prototype={constructor:Ne,restart:function(t,n,e){if("function"!=typeof t)throw new TypeError("callback is not a function");e=(null==e?Me():+e)+(null==n?0:+n),this._next||de===this||(de?de._next=this:pe=this,de=this),this._call=t,this._time=e,Ce()},stop:function(){this._call&&(this._call=null,this._time=1/0,Ce())}};var qe=fe("start","end","cancel","interrupt"),Oe=[];function Pe(t,n,e,r,i,o){var u=t.__transition;if(u){if(e in u)return}else t.__transition={};!function(t,n,e){var r,i=t.__transition;function o(t){e.state=1,e.timer.restart(u,e.delay,e.time),e.delay<=t&&u(t-e.delay)}function u(o){var s,c,h,f;if(1!==e.state)return l();for(s in i)if((f=i[s]).name===e.name){if(3===f.state)return je(u);4===f.state?(f.state=6,f.timer.stop(),f.on.call("interrupt",t,t.__data__,f.index,f.group),delete i[s]):+s0)throw new Error("too late; already scheduled");return e}function Le(t,n){var e=Te(t,n);if(e.state>3)throw new Error("too late; already running");return e}function Te(t,n){var e=t.__transition;if(!e||!(e=e[n]))throw new Error("transition not found");return e}var De,Be=180/Math.PI,Xe={translateX:0,translateY:0,rotate:0,skewX:0,scaleX:1,scaleY:1};function ze(t,n,e,r,i,o){var u,a,l;return(u=Math.sqrt(t*t+n*n))&&(t/=u,n/=u),(l=t*e+n*r)&&(e-=t*l,r-=n*l),(a=Math.sqrt(e*e+r*r))&&(e/=a,r/=a,l/=a),t*r180?n+=360:n-t>180&&(t+=360),o.push({i:e.push(i(e)+"rotate(",null,r)-2,x:Bn(t,n)})):n&&e.push(i(e)+"rotate("+n+r)}(o.rotate,u.rotate,a,l),function(t,n,e,o){t!==n?o.push({i:e.push(i(e)+"skewX(",null,r)-2,x:Bn(t,n)}):n&&e.push(i(e)+"skewX("+n+r)}(o.skewX,u.skewX,a,l),function(t,n,e,r,o,u){if(t!==e||n!==r){var a=o.push(i(o)+"scale(",null,",",null,")");u.push({i:a-4,x:Bn(t,e)},{i:a-2,x:Bn(n,r)})}else 1===e&&1===r||o.push(i(o)+"scale("+e+","+r+")")}(o.scaleX,o.scaleY,u.scaleX,u.scaleY,a,l),o=u=null,function(t){for(var n,e=-1,r=l.length;++e=0&&(t=t.slice(0,n)),!t||"start"===t}))}(n)?He:Le;return function(){var u=o(this,t),a=u.on;a!==r&&(i=(r=a).copy()).on(n,e),u.on=i}}var cr=vt.prototype.constructor;function hr(t){return function(){this.style.removeProperty(t)}}function fr(t,n,e){return function(r){this.style.setProperty(t,n.call(this,r),e)}}function pr(t,n,e){var r,i;function o(){var o=n.apply(this,arguments);return o!==i&&(r=(i=o)&&fr(t,o,e)),r}return o._value=n,o}function dr(t){return function(n){this.textContent=t.call(this,n)}}function gr(t){var n,e;function r(){var r=t.apply(this,arguments);return r!==e&&(n=(e=r)&&dr(r)),n}return r._value=t,r}var vr=0;function yr(t,n,e,r){this._groups=t,this._parents=n,this._name=e,this._id=r}function mr(){return++vr}var wr=vt.prototype;yr.prototype=function(t){return vt().transition(t)}.prototype={constructor:yr,select:function(t){var n=this._name,e=this._id;"function"!=typeof t&&(t=r(t));for(var i=this._groups,o=i.length,u=new Array(o),a=0;a{p&&(p.textContent="search: "+n+" of "+e+" total samples ( "+St(".3f")(n/e*100,3)+"%)")},d()};const E=k;let S=(t,n,e=!1)=>{if(!n)return!1;let r=b(t);e&&(n=n.toLowerCase(),r=r.toLowerCase());const i=new RegExp(n);return void 0!==r&&r&&r.match(i)};const C=S;let j=function(t){p&&(t?p.textContent=t:"function"==typeof d?d():p.textContent="")};const q=j;let O=function(t){return b(t)+" ("+St(".3f")(100*(t.x1-t.x0),3)+"%, "+x(t)+" samples)"},P=function(t){return t.highlight?"#E600E6":function(t,n){let e=_||"warm";_||void 0===n||""===n||(e="red",void 0!==t&&t&&t.match(/::/)&&(e="yellow"),"kernel"===n?e="orange":"jit"===n?e="green":"inlined"===n&&(e="aqua"));const r=function(t){let n=0;if(t){const e=t.split("`");e.length>1&&(t=e[e.length-1]),n=function(t){let n=0,e=0,r=1;if(t){for(let i=0;i6);i++)n+=r*(t.charCodeAt(i)%10),e+=9*r,r*=.7;e>0&&(n/=e)}return n}(t=t.split("(")[0])}return n}(t);return function(t,n){let e,r,i;return"red"===t?(e=200+Math.round(55*n),r=50+Math.round(80*n),i=r):"orange"===t?(e=190+Math.round(65*n),r=90+Math.round(65*n),i=0):"yellow"===t?(e=175+Math.round(55*n),r=e,i=50+Math.round(20*n)):"green"===t?(e=50+Math.round(60*n),r=200+Math.round(55*n),i=e):"pastelgreen"===t?(e=163+Math.round(75*n),r=195+Math.round(49*n),i=72+Math.round(149*n)):"blue"===t?(e=91+Math.round(126*n),r=156+Math.round(76*n),i=221+Math.round(26*n)):"aqua"===t?(e=50+Math.round(60*n),r=165+Math.round(55*n),i=r):"cold"===t?(e=0+Math.round(55*(1-n)),r=0+Math.round(230*(1-n)),i=200+Math.round(55*n)):(e=200+Math.round(55*n),r=0+Math.round(230*(1-n)),i=0+Math.round(55*(1-n))),"rgb("+e+","+r+","+i+")"}(e,r)}(b(t),A(t))};const H=P;function L(t){t.data.fade=!1,t.data.hide=!1,t.children&&t.children.forEach(L)}function T(t){t.parent&&(t.parent.data.fade=!0,T(t.parent))}function D(t){if(i&&i.hide(),function(t){let n,e,r,i=t,o=i.parent;for(;o;){for(n=o.children,e=n.length;e--;)r=n[e],r!==i&&(r.data.hide=!0);i=o,o=i.parent}}(t),L(t),T(t),I(),y){const n=yt(this).select("svg")._groups[0][0].parentNode.offsetTop,r=(window.innerHeight-n)/e,i=(t.height-r+10)*e;window.scrollTo({top:n+i,left:0,behavior:"smooth"})}"function"==typeof c&&c(t)}function B(t,n){if(t.id===n)return t;{const e=M(t);if(e)for(let t=0;t0){const r=t/(n.x1-n.x0);e=e.filter((function(t){return(t.x1-t.x0)*r>f}))}return e}(r),y=yt(this).select("svg");y.attr("width",t);let w=y.selectAll("g").data(g,(function(t){return t.id}));if(!n||v){const t=Math.max.apply(null,g.map((function(t){return t.depth})));n=(t+3)*e,n{D(n)})),w.exit().remove(),w.on("mouseover",(function(t,n){i&&i.show(n,this),j(O(n)),"function"==typeof h&&h(n)})).on("mouseout",(function(){i&&i.hide(),j(null)}))}))}function $(t,n){n.forEach((function(n){const e=t.find((function(t){return t.name===n.name}));e?(e.value+=n.value,n.children&&(e.children||(e.children=[]),$(e.children,n.children))):t.push(n)}))}function V(t){let n,e,r,i,o,u,a,l;const s=[],c=[],h=[],f=!g;let p=t.data;for(p.hide?(t.value=0,e=t.children,e&&h.push(e)):(t.value=p.fade?0:x(p),s.push(t));n=s.pop();)if(e=n.children,e&&(o=e.length)){for(i=0;o--;)a=e[o],p=a.data,p.hide?(a.value=0,r=a.children,r&&h.push(r)):(p.fade?a.value=0:(l=x(p),a.value=l,i+=l),s.push(a));f&&n.value&&(n.value-=i),c.push(e)}for(o=c.length;o--;){for(e=c[o],i=0,u=e.length;u--;)i+=e[u].value;e[0].parent.value+=i}for(;h.length;)for(e=h.pop(),u=e.length;u--;)a=e[u],a.value=0,r=a.children,r&&h.push(r)}function Y(){r.datum((t=>{if("Node"!==t.constructor.name){const n=Dt(t,M);return function(t){let n=0;!function(t,n){n(t);let e=t.children;if(e){const t=[e];let r,i,o;for(;t.length;)for(e=t.pop(),r=e.length;r--;)i=e[r],n(i),o=i.children,o&&t.push(o)}}(t,(function(t){t.id=n++}))}(n),V(n),n.originalValue=n.value,w&&n.eachAfter((t=>{let n=N(t);const e=t.children;let r=e&&e.length;for(;--r>=0;)n+=e[r].delta;t.delta=n})),n}}))}function F(e){if(!arguments.length)return F;r=e,Y(),r.each((function(e){if(0===yt(this).select("svg").size()){const e=yt(this).append("svg:svg").attr("width",t).attr("class","partition d3-flame-graph");n&&(n($([n.data],[t]),n.data))),Y(),I(),F):F},F.update=function(t){return r?(t&&(r.datum(t),Y()),I(),F):F},F.destroy=function(){return r?(i&&(i.hide(),"function"==typeof i.destroy&&i.destroy()),r.selectAll("svg").remove(),F):F},F.setColorMapper=function(t){return arguments.length?(P=n=>{const e=H(n);return t(n,e)},F):(P=H,F)},F.color=F.setColorMapper,F.setColorHue=function(t){return arguments.length?(_=t,F):(_=null,F)},F.minFrameSize=function(t){return arguments.length?(f=t,F):f},F.setDetailsElement=function(t){return arguments.length?(p=t,F):p},F.details=F.setDetailsElement,F.selfValue=function(t){return arguments.length?(g=t,F):g},F.resetHeightOnZoom=function(t){return arguments.length?(v=t,F):v},F.scrollOnZoom=function(t){return arguments.length?(y=t,F):y},F.getName=function(t){return arguments.length?(b=t,F):b},F.getValue=function(t){return arguments.length?(x=t,F):x},F.getChildren=function(t){return arguments.length?(M=t,F):M},F.getLibtype=function(t){return arguments.length?(A=t,F):A},F.getDelta=function(t){return arguments.length?(N=t,F):N},F.setSearchHandler=function(t){return arguments.length?(k=t,F):(k=E,F)},F.setDetailsHandler=function(t){return arguments.length?(j=t,F):(j=q,F)},F.setSearchMatch=function(t){return arguments.length?(S=t,F):(S=C,F)},F}return vt.prototype.interrupt=function(t){return this.each((function(){!function(t,n){var e,r,i,o=t.__transition,u=!0;if(o){for(i in n=null==n?null:n+"",o)(e=o[i]).name===n?(r=e.state>2&&e.state<5,e.state=6,e.timer.stop(),e.on.call(r?"interrupt":"cancel",t,t.__data__,e.index,e.group),delete o[i]):u=!1;u&&delete t.__transition}}(this,t)}))},vt.prototype.transition=function(t){var n,e;t instanceof yr?(n=t._id,t=t._name):(n=mr(),(e=_r).time=Me(),t=null==t?null:t+"");for(var r=this._groups,i=r.length,o=0;on?1:t>=n?0:NaN}function e(t,n){return null==t||null==n?NaN:nt?1:n>=t?0:NaN}function r(t){let r,o,a;function u(t,n,e=0,i=t.length){if(e>>1;o(t[r],n)<0?e=r+1:i=r}while(en(t(e),r),a=(n,e)=>t(n)-e):(r=t===n||t===e?t:i,o=t,a=t),{left:u,center:function(t,n,e=0,r=t.length){const i=u(t,n,e,r-1);return i>e&&a(t[i-1],n)>-a(t[i],n)?i-1:i},right:function(t,n,e=0,i=t.length){if(e>>1;o(t[r],n)<=0?e=r+1:i=r}while(e{n(t,e,(r<<=2)+0,(i<<=2)+0,o<<=2),n(t,e,r+1,i+1,o),n(t,e,r+2,i+2,o),n(t,e,r+3,i+3,o)}}));function d(t){return function(n,e,r=e){if(!((e=+e)>=0))throw new RangeError("invalid rx");if(!((r=+r)>=0))throw new RangeError("invalid ry");let{data:i,width:o,height:a}=n;if(!((o=Math.floor(o))>=0))throw new RangeError("invalid width");if(!((a=Math.floor(void 0!==a?a:i.length/o))>=0))throw new RangeError("invalid height");if(!o||!a||!e&&!r)return n;const u=e&&t(e),c=r&&t(r),f=i.slice();return u&&c?(p(u,f,i,o,a),p(u,i,f,o,a),p(u,f,i,o,a),g(c,i,f,o,a),g(c,f,i,o,a),g(c,i,f,o,a)):u?(p(u,i,f,o,a),p(u,f,i,o,a),p(u,i,f,o,a)):c&&(g(c,i,f,o,a),g(c,f,i,o,a),g(c,i,f,o,a)),n}}function p(t,n,e,r,i){for(let o=0,a=r*i;o{if(!((o-=a)>=i))return;let u=t*r[i];const c=a*t;for(let t=i,n=i+c;t{if(!((a-=u)>=o))return;let c=n*i[o];const f=u*n,s=f+u;for(let t=o,n=o+f;t=n&&++e;else{let r=-1;for(let i of t)null!=(i=n(i,++r,t))&&(i=+i)>=i&&++e}return e}function _(t){return 0|t.length}function b(t){return!(t>0)}function m(t){return"object"!=typeof t||"length"in t?t:Array.from(t)}function x(t,n){let e,r=0,i=0,o=0;if(void 0===n)for(let n of t)null!=n&&(n=+n)>=n&&(e=n-i,i+=e/++r,o+=e*(n-i));else{let a=-1;for(let u of t)null!=(u=n(u,++a,t))&&(u=+u)>=u&&(e=u-i,i+=e/++r,o+=e*(u-i))}if(r>1)return o/(r-1)}function w(t,n){const e=x(t,n);return e?Math.sqrt(e):e}function M(t,n){let e,r;if(void 0===n)for(const n of t)null!=n&&(void 0===e?n>=n&&(e=r=n):(e>n&&(e=n),r=o&&(e=r=o):(e>o&&(e=o),r0){for(o=t[--i];i>0&&(n=o,e=t[--i],o=n+e,r=e-(o-n),!r););i>0&&(r<0&&t[i-1]<0||r>0&&t[i-1]>0)&&(e=2*r,n=o+e,e==n-o&&(o=n))}return o}}class InternMap extends Map{constructor(t,n=N){if(super(),Object.defineProperties(this,{_intern:{value:new Map},_key:{value:n}}),null!=t)for(const[n,e]of t)this.set(n,e)}get(t){return super.get(A(this,t))}has(t){return super.has(A(this,t))}set(t,n){return super.set(S(this,t),n)}delete(t){return super.delete(E(this,t))}}class InternSet extends Set{constructor(t,n=N){if(super(),Object.defineProperties(this,{_intern:{value:new Map},_key:{value:n}}),null!=t)for(const n of t)this.add(n)}has(t){return super.has(A(this,t))}add(t){return super.add(S(this,t))}delete(t){return super.delete(E(this,t))}}function A({_intern:t,_key:n},e){const r=n(e);return t.has(r)?t.get(r):e}function S({_intern:t,_key:n},e){const r=n(e);return t.has(r)?t.get(r):(t.set(r,e),e)}function E({_intern:t,_key:n},e){const r=n(e);return t.has(r)&&(e=t.get(r),t.delete(r)),e}function N(t){return null!==t&&"object"==typeof t?t.valueOf():t}function k(t){return t}function C(t,...n){return F(t,k,k,n)}function P(t,...n){return F(t,Array.from,k,n)}function z(t,n){for(let e=1,r=n.length;et.pop().map((([n,e])=>[...t,n,e]))));return t}function $(t,n,...e){return F(t,k,n,e)}function D(t,n,...e){return F(t,Array.from,n,e)}function R(t){if(1!==t.length)throw new Error("duplicate key");return t[0]}function F(t,n,e,r){return function t(i,o){if(o>=r.length)return e(i);const a=new InternMap,u=r[o++];let c=-1;for(const t of i){const n=u(t,++c,i),e=a.get(n);e?e.push(t):a.set(n,[t])}for(const[n,e]of a)a.set(n,t(e,o));return n(a)}(t,0)}function q(t,n){return Array.from(n,(n=>t[n]))}function U(t,...n){if("function"!=typeof t[Symbol.iterator])throw new TypeError("values is not iterable");t=Array.from(t);let[e]=n;if(e&&2!==e.length||n.length>1){const r=Uint32Array.from(t,((t,n)=>n));return n.length>1?(n=n.map((n=>t.map(n))),r.sort(((t,e)=>{for(const r of n){const n=O(r[t],r[e]);if(n)return n}}))):(e=t.map(e),r.sort(((t,n)=>O(e[t],e[n])))),q(t,r)}return t.sort(I(e))}function I(t=n){if(t===n)return O;if("function"!=typeof t)throw new TypeError("compare is not a function");return(n,e)=>{const r=t(n,e);return r||0===r?r:(0===t(e,e))-(0===t(n,n))}}function O(t,n){return(null==t||!(t>=t))-(null==n||!(n>=n))||(tn?1:0)}var B=Array.prototype.slice;function Y(t){return()=>t}const L=Math.sqrt(50),j=Math.sqrt(10),H=Math.sqrt(2);function X(t,n,e){const r=(n-t)/Math.max(0,e),i=Math.floor(Math.log10(r)),o=r/Math.pow(10,i),a=o>=L?10:o>=j?5:o>=H?2:1;let u,c,f;return i<0?(f=Math.pow(10,-i)/a,u=Math.round(t*f),c=Math.round(n*f),u/fn&&--c,f=-f):(f=Math.pow(10,i)*a,u=Math.round(t/f),c=Math.round(n/f),u*fn&&--c),c0))return[];if((t=+t)===(n=+n))return[t];const r=n=i))return[];const u=o-i+1,c=new Array(u);if(r)if(a<0)for(let t=0;t0?(t=Math.floor(t/i)*i,n=Math.ceil(n/i)*i):i<0&&(t=Math.ceil(t*i)/i,n=Math.floor(n*i)/i),r=i}}function K(t){return Math.max(1,Math.ceil(Math.log(v(t))/Math.LN2)+1)}function Q(){var t=k,n=M,e=K;function r(r){Array.isArray(r)||(r=Array.from(r));var i,o,a,u=r.length,c=new Array(u);for(i=0;i=h)if(t>=h&&n===M){const t=V(l,h,e);isFinite(t)&&(t>0?h=(Math.floor(h/t)+1)*t:t<0&&(h=(Math.ceil(h*-t)+1)/-t))}else d.pop()}for(var p=d.length,g=0,y=p;d[g]<=l;)++g;for(;d[y-1]>h;)--y;(g||y0?d[i-1]:l,v.x1=i0)for(i=0;i=n)&&(e=n);else{let r=-1;for(let i of t)null!=(i=n(i,++r,t))&&(e=i)&&(e=i)}return e}function tt(t,n){let e,r=-1,i=-1;if(void 0===n)for(const n of t)++i,null!=n&&(e=n)&&(e=n,r=i);else for(let o of t)null!=(o=n(o,++i,t))&&(e=o)&&(e=o,r=i);return r}function nt(t,n){let e;if(void 0===n)for(const n of t)null!=n&&(e>n||void 0===e&&n>=n)&&(e=n);else{let r=-1;for(let i of t)null!=(i=n(i,++r,t))&&(e>i||void 0===e&&i>=i)&&(e=i)}return e}function et(t,n){let e,r=-1,i=-1;if(void 0===n)for(const n of t)++i,null!=n&&(e>n||void 0===e&&n>=n)&&(e=n,r=i);else for(let o of t)null!=(o=n(o,++i,t))&&(e>o||void 0===e&&o>=o)&&(e=o,r=i);return r}function rt(t,n,e=0,r=1/0,i){if(n=Math.floor(n),e=Math.floor(Math.max(0,e)),r=Math.floor(Math.min(t.length-1,r)),!(e<=n&&n<=r))return t;for(i=void 0===i?O:I(i);r>e;){if(r-e>600){const o=r-e+1,a=n-e+1,u=Math.log(o),c=.5*Math.exp(2*u/3),f=.5*Math.sqrt(u*c*(o-c)/o)*(a-o/2<0?-1:1);rt(t,n,Math.max(e,Math.floor(n-a*c/o+f)),Math.min(r,Math.floor(n+(o-a)*c/o+f)),i)}const o=t[n];let a=e,u=r;for(it(t,e,n),i(t[r],o)>0&&it(t,e,r);a0;)--u}0===i(t[e],o)?it(t,e,u):(++u,it(t,u,r)),u<=n&&(e=u+1),n<=u&&(r=u-1)}return t}function it(t,n,e){const r=t[n];t[n]=t[e],t[e]=r}function ot(t,e=n){let r,i=!1;if(1===e.length){let o;for(const a of t){const t=e(a);(i?n(t,o)>0:0===n(t,t))&&(r=a,o=t,i=!0)}}else for(const n of t)(i?e(n,r)>0:0===e(n,n))&&(r=n,i=!0);return r}function at(t,n,e){if(t=Float64Array.from(function*(t,n){if(void 0===n)for(let n of t)null!=n&&(n=+n)>=n&&(yield n);else{let e=-1;for(let r of t)null!=(r=n(r,++e,t))&&(r=+r)>=r&&(yield r)}}(t,e)),(r=t.length)&&!isNaN(n=+n)){if(n<=0||r<2)return nt(t);if(n>=1)return J(t);var r,i=(r-1)*n,o=Math.floor(i),a=J(rt(t,o).subarray(0,o+1));return a+(nt(t.subarray(o+1))-a)*(i-o)}}function ut(t,n,e=o){if((r=t.length)&&!isNaN(n=+n)){if(n<=0||r<2)return+e(t[0],0,t);if(n>=1)return+e(t[r-1],r-1,t);var r,i=(r-1)*n,a=Math.floor(i),u=+e(t[a],a,t);return u+(+e(t[a+1],a+1,t)-u)*(i-a)}}function ct(t,n,e=o){if(!isNaN(n=+n)){if(r=Float64Array.from(t,((n,r)=>o(e(t[r],r,t)))),n<=0)return et(r);if(n>=1)return tt(r);var r,i=Uint32Array.from(t,((t,n)=>n)),a=r.length-1,u=Math.floor(a*n);return rt(i,u,0,a,((t,n)=>O(r[t],r[n]))),(u=ot(i.subarray(0,u+1),(t=>r[t])))>=0?u:-1}}function ft(t){return Array.from(function*(t){for(const n of t)yield*n}(t))}function st(t,n){return[t,n]}function lt(t,n,e){t=+t,n=+n,e=(i=arguments.length)<2?(n=t,t=0,1):i<3?1:+e;for(var r=-1,i=0|Math.max(0,Math.ceil((n-t)/e)),o=new Array(i);++r+t(n)}function kt(t,n){return n=Math.max(0,t.bandwidth()-2*n)/2,t.round()&&(n=Math.round(n)),e=>+t(e)+n}function Ct(){return!this.__axis}function Pt(t,n){var e=[],r=null,i=null,o=6,a=6,u=3,c="undefined"!=typeof window&&window.devicePixelRatio>1?0:.5,f=t===xt||t===Tt?-1:1,s=t===Tt||t===wt?"x":"y",l=t===xt||t===Mt?St:Et;function h(h){var d=null==r?n.ticks?n.ticks.apply(n,e):n.domain():r,p=null==i?n.tickFormat?n.tickFormat.apply(n,e):mt:i,g=Math.max(o,0)+u,y=n.range(),v=+y[0]+c,_=+y[y.length-1]+c,b=(n.bandwidth?kt:Nt)(n.copy(),c),m=h.selection?h.selection():h,x=m.selectAll(".domain").data([null]),w=m.selectAll(".tick").data(d,n).order(),M=w.exit(),T=w.enter().append("g").attr("class","tick"),A=w.select("line"),S=w.select("text");x=x.merge(x.enter().insert("path",".tick").attr("class","domain").attr("stroke","currentColor")),w=w.merge(T),A=A.merge(T.append("line").attr("stroke","currentColor").attr(s+"2",f*o)),S=S.merge(T.append("text").attr("fill","currentColor").attr(s,f*g).attr("dy",t===xt?"0em":t===Mt?"0.71em":"0.32em")),h!==m&&(x=x.transition(h),w=w.transition(h),A=A.transition(h),S=S.transition(h),M=M.transition(h).attr("opacity",At).attr("transform",(function(t){return isFinite(t=b(t))?l(t+c):this.getAttribute("transform")})),T.attr("opacity",At).attr("transform",(function(t){var n=this.parentNode.__axis;return l((n&&isFinite(n=n(t))?n:b(t))+c)}))),M.remove(),x.attr("d",t===Tt||t===wt?a?"M"+f*a+","+v+"H"+c+"V"+_+"H"+f*a:"M"+c+","+v+"V"+_:a?"M"+v+","+f*a+"V"+c+"H"+_+"V"+f*a:"M"+v+","+c+"H"+_),w.attr("opacity",1).attr("transform",(function(t){return l(b(t)+c)})),A.attr(s+"2",f*o),S.attr(s,f*g).text(p),m.filter(Ct).attr("fill","none").attr("font-size",10).attr("font-family","sans-serif").attr("text-anchor",t===wt?"start":t===Tt?"end":"middle"),m.each((function(){this.__axis=b}))}return h.scale=function(t){return arguments.length?(n=t,h):n},h.ticks=function(){return e=Array.from(arguments),h},h.tickArguments=function(t){return arguments.length?(e=null==t?[]:Array.from(t),h):e.slice()},h.tickValues=function(t){return arguments.length?(r=null==t?null:Array.from(t),h):r&&r.slice()},h.tickFormat=function(t){return arguments.length?(i=t,h):i},h.tickSize=function(t){return arguments.length?(o=a=+t,h):o},h.tickSizeInner=function(t){return arguments.length?(o=+t,h):o},h.tickSizeOuter=function(t){return arguments.length?(a=+t,h):a},h.tickPadding=function(t){return arguments.length?(u=+t,h):u},h.offset=function(t){return arguments.length?(c=+t,h):c},h}var zt={value:()=>{}};function $t(){for(var t,n=0,e=arguments.length,r={};n=0&&(n=t.slice(e+1),t=t.slice(0,e)),t&&!r.hasOwnProperty(t))throw new Error("unknown type: "+t);return{type:t,name:n}}))),a=-1,u=o.length;if(!(arguments.length<2)){if(null!=n&&"function"!=typeof n)throw new Error("invalid callback: "+n);for(;++a0)for(var e,r,i=new Array(e),o=0;o=0&&"xmlns"!==(n=t.slice(0,e))&&(t=t.slice(e+1)),Ut.hasOwnProperty(n)?{space:Ut[n],local:t}:t}function Ot(t){return function(){var n=this.ownerDocument,e=this.namespaceURI;return e===qt&&n.documentElement.namespaceURI===qt?n.createElement(t):n.createElementNS(e,t)}}function Bt(t){return function(){return this.ownerDocument.createElementNS(t.space,t.local)}}function Yt(t){var n=It(t);return(n.local?Bt:Ot)(n)}function Lt(){}function jt(t){return null==t?Lt:function(){return this.querySelector(t)}}function Ht(t){return null==t?[]:Array.isArray(t)?t:Array.from(t)}function Xt(){return[]}function Gt(t){return null==t?Xt:function(){return this.querySelectorAll(t)}}function Vt(t){return function(){return this.matches(t)}}function Wt(t){return function(n){return n.matches(t)}}var Zt=Array.prototype.find;function Kt(){return this.firstElementChild}var Qt=Array.prototype.filter;function Jt(){return Array.from(this.children)}function tn(t){return new Array(t.length)}function nn(t,n){this.ownerDocument=t.ownerDocument,this.namespaceURI=t.namespaceURI,this._next=null,this._parent=t,this.__data__=n}function en(t,n,e,r,i,o){for(var a,u=0,c=n.length,f=o.length;un?1:t>=n?0:NaN}function cn(t){return function(){this.removeAttribute(t)}}function fn(t){return function(){this.removeAttributeNS(t.space,t.local)}}function sn(t,n){return function(){this.setAttribute(t,n)}}function ln(t,n){return function(){this.setAttributeNS(t.space,t.local,n)}}function hn(t,n){return function(){var e=n.apply(this,arguments);null==e?this.removeAttribute(t):this.setAttribute(t,e)}}function dn(t,n){return function(){var e=n.apply(this,arguments);null==e?this.removeAttributeNS(t.space,t.local):this.setAttributeNS(t.space,t.local,e)}}function pn(t){return t.ownerDocument&&t.ownerDocument.defaultView||t.document&&t||t.defaultView}function gn(t){return function(){this.style.removeProperty(t)}}function yn(t,n,e){return function(){this.style.setProperty(t,n,e)}}function vn(t,n,e){return function(){var r=n.apply(this,arguments);null==r?this.style.removeProperty(t):this.style.setProperty(t,r,e)}}function _n(t,n){return t.style.getPropertyValue(n)||pn(t).getComputedStyle(t,null).getPropertyValue(n)}function bn(t){return function(){delete this[t]}}function mn(t,n){return function(){this[t]=n}}function xn(t,n){return function(){var e=n.apply(this,arguments);null==e?delete this[t]:this[t]=e}}function wn(t){return t.trim().split(/^|\s+/)}function Mn(t){return t.classList||new Tn(t)}function Tn(t){this._node=t,this._names=wn(t.getAttribute("class")||"")}function An(t,n){for(var e=Mn(t),r=-1,i=n.length;++r=0&&(this._names.splice(n,1),this._node.setAttribute("class",this._names.join(" ")))},contains:function(t){return this._names.indexOf(t)>=0}};var Gn=[null];function Vn(t,n){this._groups=t,this._parents=n}function Wn(){return new Vn([[document.documentElement]],Gn)}function Zn(t){return"string"==typeof t?new Vn([[document.querySelector(t)]],[document.documentElement]):new Vn([[t]],Gn)}Vn.prototype=Wn.prototype={constructor:Vn,select:function(t){"function"!=typeof t&&(t=jt(t));for(var n=this._groups,e=n.length,r=new Array(e),i=0;i=m&&(m=b+1);!(_=y[m])&&++m=0;)(r=i[o])&&(a&&4^r.compareDocumentPosition(a)&&a.parentNode.insertBefore(r,a),a=r);return this},sort:function(t){function n(n,e){return n&&e?t(n.__data__,e.__data__):!n-!e}t||(t=un);for(var e=this._groups,r=e.length,i=new Array(r),o=0;o1?this.each((null==n?gn:"function"==typeof n?vn:yn)(t,n,null==e?"":e)):_n(this.node(),t)},property:function(t,n){return arguments.length>1?this.each((null==n?bn:"function"==typeof n?xn:mn)(t,n)):this.node()[t]},classed:function(t,n){var e=wn(t+"");if(arguments.length<2){for(var r=Mn(this.node()),i=-1,o=e.length;++i=0&&(n=t.slice(e+1),t=t.slice(0,e)),{type:t,name:n}}))}(t+""),a=o.length;if(!(arguments.length<2)){for(u=n?Ln:Yn,r=0;r()=>t;function fe(t,{sourceEvent:n,subject:e,target:r,identifier:i,active:o,x:a,y:u,dx:c,dy:f,dispatch:s}){Object.defineProperties(this,{type:{value:t,enumerable:!0,configurable:!0},sourceEvent:{value:n,enumerable:!0,configurable:!0},subject:{value:e,enumerable:!0,configurable:!0},target:{value:r,enumerable:!0,configurable:!0},identifier:{value:i,enumerable:!0,configurable:!0},active:{value:o,enumerable:!0,configurable:!0},x:{value:a,enumerable:!0,configurable:!0},y:{value:u,enumerable:!0,configurable:!0},dx:{value:c,enumerable:!0,configurable:!0},dy:{value:f,enumerable:!0,configurable:!0},_:{value:s}})}function se(t){return!t.ctrlKey&&!t.button}function le(){return this.parentNode}function he(t,n){return null==n?{x:t.x,y:t.y}:n}function de(){return navigator.maxTouchPoints||"ontouchstart"in this}function pe(t,n,e){t.prototype=n.prototype=e,e.constructor=t}function ge(t,n){var e=Object.create(t.prototype);for(var r in n)e[r]=n[r];return e}function ye(){}fe.prototype.on=function(){var t=this._.on.apply(this._,arguments);return t===this._?this:t};var ve=.7,_e=1/ve,be="\\s*([+-]?\\d+)\\s*",me="\\s*([+-]?(?:\\d*\\.)?\\d+(?:[eE][+-]?\\d+)?)\\s*",xe="\\s*([+-]?(?:\\d*\\.)?\\d+(?:[eE][+-]?\\d+)?)%\\s*",we=/^#([0-9a-f]{3,8})$/,Me=new RegExp(`^rgb\\(${be},${be},${be}\\)$`),Te=new RegExp(`^rgb\\(${xe},${xe},${xe}\\)$`),Ae=new RegExp(`^rgba\\(${be},${be},${be},${me}\\)$`),Se=new RegExp(`^rgba\\(${xe},${xe},${xe},${me}\\)$`),Ee=new RegExp(`^hsl\\(${me},${xe},${xe}\\)$`),Ne=new RegExp(`^hsla\\(${me},${xe},${xe},${me}\\)$`),ke={aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,rebeccapurple:6697881,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074};function Ce(){return this.rgb().formatHex()}function Pe(){return this.rgb().formatRgb()}function ze(t){var n,e;return t=(t+"").trim().toLowerCase(),(n=we.exec(t))?(e=n[1].length,n=parseInt(n[1],16),6===e?$e(n):3===e?new qe(n>>8&15|n>>4&240,n>>4&15|240&n,(15&n)<<4|15&n,1):8===e?De(n>>24&255,n>>16&255,n>>8&255,(255&n)/255):4===e?De(n>>12&15|n>>8&240,n>>8&15|n>>4&240,n>>4&15|240&n,((15&n)<<4|15&n)/255):null):(n=Me.exec(t))?new qe(n[1],n[2],n[3],1):(n=Te.exec(t))?new qe(255*n[1]/100,255*n[2]/100,255*n[3]/100,1):(n=Ae.exec(t))?De(n[1],n[2],n[3],n[4]):(n=Se.exec(t))?De(255*n[1]/100,255*n[2]/100,255*n[3]/100,n[4]):(n=Ee.exec(t))?Le(n[1],n[2]/100,n[3]/100,1):(n=Ne.exec(t))?Le(n[1],n[2]/100,n[3]/100,n[4]):ke.hasOwnProperty(t)?$e(ke[t]):"transparent"===t?new qe(NaN,NaN,NaN,0):null}function $e(t){return new qe(t>>16&255,t>>8&255,255&t,1)}function De(t,n,e,r){return r<=0&&(t=n=e=NaN),new qe(t,n,e,r)}function Re(t){return t instanceof ye||(t=ze(t)),t?new qe((t=t.rgb()).r,t.g,t.b,t.opacity):new qe}function Fe(t,n,e,r){return 1===arguments.length?Re(t):new qe(t,n,e,null==r?1:r)}function qe(t,n,e,r){this.r=+t,this.g=+n,this.b=+e,this.opacity=+r}function Ue(){return`#${Ye(this.r)}${Ye(this.g)}${Ye(this.b)}`}function Ie(){const t=Oe(this.opacity);return`${1===t?"rgb(":"rgba("}${Be(this.r)}, ${Be(this.g)}, ${Be(this.b)}${1===t?")":`, ${t})`}`}function Oe(t){return isNaN(t)?1:Math.max(0,Math.min(1,t))}function Be(t){return Math.max(0,Math.min(255,Math.round(t)||0))}function Ye(t){return((t=Be(t))<16?"0":"")+t.toString(16)}function Le(t,n,e,r){return r<=0?t=n=e=NaN:e<=0||e>=1?t=n=NaN:n<=0&&(t=NaN),new Xe(t,n,e,r)}function je(t){if(t instanceof Xe)return new Xe(t.h,t.s,t.l,t.opacity);if(t instanceof ye||(t=ze(t)),!t)return new Xe;if(t instanceof Xe)return t;var n=(t=t.rgb()).r/255,e=t.g/255,r=t.b/255,i=Math.min(n,e,r),o=Math.max(n,e,r),a=NaN,u=o-i,c=(o+i)/2;return u?(a=n===o?(e-r)/u+6*(e0&&c<1?0:a,new Xe(a,u,c,t.opacity)}function He(t,n,e,r){return 1===arguments.length?je(t):new Xe(t,n,e,null==r?1:r)}function Xe(t,n,e,r){this.h=+t,this.s=+n,this.l=+e,this.opacity=+r}function Ge(t){return(t=(t||0)%360)<0?t+360:t}function Ve(t){return Math.max(0,Math.min(1,t||0))}function We(t,n,e){return 255*(t<60?n+(e-n)*t/60:t<180?e:t<240?n+(e-n)*(240-t)/60:n)}pe(ye,ze,{copy(t){return Object.assign(new this.constructor,this,t)},displayable(){return this.rgb().displayable()},hex:Ce,formatHex:Ce,formatHex8:function(){return this.rgb().formatHex8()},formatHsl:function(){return je(this).formatHsl()},formatRgb:Pe,toString:Pe}),pe(qe,Fe,ge(ye,{brighter(t){return t=null==t?_e:Math.pow(_e,t),new qe(this.r*t,this.g*t,this.b*t,this.opacity)},darker(t){return t=null==t?ve:Math.pow(ve,t),new qe(this.r*t,this.g*t,this.b*t,this.opacity)},rgb(){return this},clamp(){return new qe(Be(this.r),Be(this.g),Be(this.b),Oe(this.opacity))},displayable(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:Ue,formatHex:Ue,formatHex8:function(){return`#${Ye(this.r)}${Ye(this.g)}${Ye(this.b)}${Ye(255*(isNaN(this.opacity)?1:this.opacity))}`},formatRgb:Ie,toString:Ie})),pe(Xe,He,ge(ye,{brighter(t){return t=null==t?_e:Math.pow(_e,t),new Xe(this.h,this.s,this.l*t,this.opacity)},darker(t){return t=null==t?ve:Math.pow(ve,t),new Xe(this.h,this.s,this.l*t,this.opacity)},rgb(){var t=this.h%360+360*(this.h<0),n=isNaN(t)||isNaN(this.s)?0:this.s,e=this.l,r=e+(e<.5?e:1-e)*n,i=2*e-r;return new qe(We(t>=240?t-240:t+120,i,r),We(t,i,r),We(t<120?t+240:t-120,i,r),this.opacity)},clamp(){return new Xe(Ge(this.h),Ve(this.s),Ve(this.l),Oe(this.opacity))},displayable(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl(){const t=Oe(this.opacity);return`${1===t?"hsl(":"hsla("}${Ge(this.h)}, ${100*Ve(this.s)}%, ${100*Ve(this.l)}%${1===t?")":`, ${t})`}`}}));const Ze=Math.PI/180,Ke=180/Math.PI,Qe=.96422,Je=1,tr=.82521,nr=4/29,er=6/29,rr=3*er*er,ir=er*er*er;function or(t){if(t instanceof ur)return new ur(t.l,t.a,t.b,t.opacity);if(t instanceof pr)return gr(t);t instanceof qe||(t=Re(t));var n,e,r=lr(t.r),i=lr(t.g),o=lr(t.b),a=cr((.2225045*r+.7168786*i+.0606169*o)/Je);return r===i&&i===o?n=e=a:(n=cr((.4360747*r+.3850649*i+.1430804*o)/Qe),e=cr((.0139322*r+.0971045*i+.7141733*o)/tr)),new ur(116*a-16,500*(n-a),200*(a-e),t.opacity)}function ar(t,n,e,r){return 1===arguments.length?or(t):new ur(t,n,e,null==r?1:r)}function ur(t,n,e,r){this.l=+t,this.a=+n,this.b=+e,this.opacity=+r}function cr(t){return t>ir?Math.pow(t,1/3):t/rr+nr}function fr(t){return t>er?t*t*t:rr*(t-nr)}function sr(t){return 255*(t<=.0031308?12.92*t:1.055*Math.pow(t,1/2.4)-.055)}function lr(t){return(t/=255)<=.04045?t/12.92:Math.pow((t+.055)/1.055,2.4)}function hr(t){if(t instanceof pr)return new pr(t.h,t.c,t.l,t.opacity);if(t instanceof ur||(t=or(t)),0===t.a&&0===t.b)return new pr(NaN,0=1?(e=1,n-1):Math.floor(e*n),i=t[r],o=t[r+1],a=r>0?t[r-1]:2*i-o,u=r()=>t;function Cr(t,n){return function(e){return t+e*n}}function Pr(t,n){var e=n-t;return e?Cr(t,e>180||e<-180?e-360*Math.round(e/360):e):kr(isNaN(t)?n:t)}function zr(t){return 1==(t=+t)?$r:function(n,e){return e-n?function(t,n,e){return t=Math.pow(t,e),n=Math.pow(n,e)-t,e=1/e,function(r){return Math.pow(t+r*n,e)}}(n,e,t):kr(isNaN(n)?e:n)}}function $r(t,n){var e=n-t;return e?Cr(t,e):kr(isNaN(t)?n:t)}var Dr=function t(n){var e=zr(n);function r(t,n){var r=e((t=Fe(t)).r,(n=Fe(n)).r),i=e(t.g,n.g),o=e(t.b,n.b),a=$r(t.opacity,n.opacity);return function(n){return t.r=r(n),t.g=i(n),t.b=o(n),t.opacity=a(n),t+""}}return r.gamma=t,r}(1);function Rr(t){return function(n){var e,r,i=n.length,o=new Array(i),a=new Array(i),u=new Array(i);for(e=0;eo&&(i=n.slice(o,i),u[a]?u[a]+=i:u[++a]=i),(e=e[0])===(r=r[0])?u[a]?u[a]+=r:u[++a]=r:(u[++a]=null,c.push({i:a,x:Yr(e,r)})),o=Hr.lastIndex;return o180?n+=360:n-t>180&&(t+=360),o.push({i:e.push(i(e)+"rotate(",null,r)-2,x:Yr(t,n)})):n&&e.push(i(e)+"rotate("+n+r)}(o.rotate,a.rotate,u,c),function(t,n,e,o){t!==n?o.push({i:e.push(i(e)+"skewX(",null,r)-2,x:Yr(t,n)}):n&&e.push(i(e)+"skewX("+n+r)}(o.skewX,a.skewX,u,c),function(t,n,e,r,o,a){if(t!==e||n!==r){var u=o.push(i(o)+"scale(",null,",",null,")");a.push({i:u-4,x:Yr(t,e)},{i:u-2,x:Yr(n,r)})}else 1===e&&1===r||o.push(i(o)+"scale("+e+","+r+")")}(o.scaleX,o.scaleY,a.scaleX,a.scaleY,u,c),o=a=null,function(t){for(var n,e=-1,r=c.length;++e=0&&n._call.call(void 0,t),n=n._next;--yi}function Ci(){xi=(mi=Mi.now())+wi,yi=vi=0;try{ki()}finally{yi=0,function(){var t,n,e=pi,r=1/0;for(;e;)e._call?(r>e._time&&(r=e._time),t=e,e=e._next):(n=e._next,e._next=null,e=t?t._next=n:pi=n);gi=t,zi(r)}(),xi=0}}function Pi(){var t=Mi.now(),n=t-mi;n>bi&&(wi-=n,mi=t)}function zi(t){yi||(vi&&(vi=clearTimeout(vi)),t-xi>24?(t<1/0&&(vi=setTimeout(Ci,t-Mi.now()-wi)),_i&&(_i=clearInterval(_i))):(_i||(mi=Mi.now(),_i=setInterval(Pi,bi)),yi=1,Ti(Ci)))}function $i(t,n,e){var r=new Ei;return n=null==n?0:+n,r.restart((e=>{r.stop(),t(e+n)}),n,e),r}Ei.prototype=Ni.prototype={constructor:Ei,restart:function(t,n,e){if("function"!=typeof t)throw new TypeError("callback is not a function");e=(null==e?Ai():+e)+(null==n?0:+n),this._next||gi===this||(gi?gi._next=this:pi=this,gi=this),this._call=t,this._time=e,zi()},stop:function(){this._call&&(this._call=null,this._time=1/0,zi())}};var Di=$t("start","end","cancel","interrupt"),Ri=[],Fi=0,qi=1,Ui=2,Ii=3,Oi=4,Bi=5,Yi=6;function Li(t,n,e,r,i,o){var a=t.__transition;if(a){if(e in a)return}else t.__transition={};!function(t,n,e){var r,i=t.__transition;function o(t){e.state=qi,e.timer.restart(a,e.delay,e.time),e.delay<=t&&a(t-e.delay)}function a(o){var f,s,l,h;if(e.state!==qi)return c();for(f in i)if((h=i[f]).name===e.name){if(h.state===Ii)return $i(a);h.state===Oi?(h.state=Yi,h.timer.stop(),h.on.call("interrupt",t,t.__data__,h.index,h.group),delete i[f]):+fFi)throw new Error("too late; already scheduled");return e}function Hi(t,n){var e=Xi(t,n);if(e.state>Ii)throw new Error("too late; already running");return e}function Xi(t,n){var e=t.__transition;if(!e||!(e=e[n]))throw new Error("transition not found");return e}function Gi(t,n){var e,r,i,o=t.__transition,a=!0;if(o){for(i in n=null==n?null:n+"",o)(e=o[i]).name===n?(r=e.state>Ui&&e.state=0&&(t=t.slice(0,n)),!t||"start"===t}))}(n)?ji:Hi;return function(){var a=o(this,t),u=a.on;u!==r&&(i=(r=u).copy()).on(n,e),a.on=i}}(e,t,n))},attr:function(t,n){var e=It(t),r="transform"===e?ni:Ki;return this.attrTween(t,"function"==typeof n?(e.local?ro:eo)(e,r,Zi(this,"attr."+t,n)):null==n?(e.local?Ji:Qi)(e):(e.local?no:to)(e,r,n))},attrTween:function(t,n){var e="attr."+t;if(arguments.length<2)return(e=this.tween(e))&&e._value;if(null==n)return this.tween(e,null);if("function"!=typeof n)throw new Error;var r=It(t);return this.tween(e,(r.local?io:oo)(r,n))},style:function(t,n,e){var r="transform"==(t+="")?ti:Ki;return null==n?this.styleTween(t,function(t,n){var e,r,i;return function(){var o=_n(this,t),a=(this.style.removeProperty(t),_n(this,t));return o===a?null:o===e&&a===r?i:i=n(e=o,r=a)}}(t,r)).on("end.style."+t,lo(t)):"function"==typeof n?this.styleTween(t,function(t,n,e){var r,i,o;return function(){var a=_n(this,t),u=e(this),c=u+"";return null==u&&(this.style.removeProperty(t),c=u=_n(this,t)),a===c?null:a===r&&c===i?o:(i=c,o=n(r=a,u))}}(t,r,Zi(this,"style."+t,n))).each(function(t,n){var e,r,i,o,a="style."+n,u="end."+a;return function(){var c=Hi(this,t),f=c.on,s=null==c.value[a]?o||(o=lo(n)):void 0;f===e&&i===s||(r=(e=f).copy()).on(u,i=s),c.on=r}}(this._id,t)):this.styleTween(t,function(t,n,e){var r,i,o=e+"";return function(){var a=_n(this,t);return a===o?null:a===r?i:i=n(r=a,e)}}(t,r,n),e).on("end.style."+t,null)},styleTween:function(t,n,e){var r="style."+(t+="");if(arguments.length<2)return(r=this.tween(r))&&r._value;if(null==n)return this.tween(r,null);if("function"!=typeof n)throw new Error;return this.tween(r,function(t,n,e){var r,i;function o(){var o=n.apply(this,arguments);return o!==i&&(r=(i=o)&&function(t,n,e){return function(r){this.style.setProperty(t,n.call(this,r),e)}}(t,o,e)),r}return o._value=n,o}(t,n,null==e?"":e))},text:function(t){return this.tween("text","function"==typeof t?function(t){return function(){var n=t(this);this.textContent=null==n?"":n}}(Zi(this,"text",t)):function(t){return function(){this.textContent=t}}(null==t?"":t+""))},textTween:function(t){var n="text";if(arguments.length<1)return(n=this.tween(n))&&n._value;if(null==t)return this.tween(n,null);if("function"!=typeof t)throw new Error;return this.tween(n,function(t){var n,e;function r(){var r=t.apply(this,arguments);return r!==e&&(n=(e=r)&&function(t){return function(n){this.textContent=t.call(this,n)}}(r)),n}return r._value=t,r}(t))},remove:function(){return this.on("end.remove",function(t){return function(){var n=this.parentNode;for(var e in this.__transition)if(+e!==t)return;n&&n.removeChild(this)}}(this._id))},tween:function(t,n){var e=this._id;if(t+="",arguments.length<2){for(var r,i=Xi(this.node(),e).tween,o=0,a=i.length;o()=>t;function Qo(t,{sourceEvent:n,target:e,selection:r,mode:i,dispatch:o}){Object.defineProperties(this,{type:{value:t,enumerable:!0,configurable:!0},sourceEvent:{value:n,enumerable:!0,configurable:!0},target:{value:e,enumerable:!0,configurable:!0},selection:{value:r,enumerable:!0,configurable:!0},mode:{value:i,enumerable:!0,configurable:!0},_:{value:o}})}function Jo(t){t.preventDefault(),t.stopImmediatePropagation()}var ta={name:"drag"},na={name:"space"},ea={name:"handle"},ra={name:"center"};const{abs:ia,max:oa,min:aa}=Math;function ua(t){return[+t[0],+t[1]]}function ca(t){return[ua(t[0]),ua(t[1])]}var fa={name:"x",handles:["w","e"].map(va),input:function(t,n){return null==t?null:[[+t[0],n[0][1]],[+t[1],n[1][1]]]},output:function(t){return t&&[t[0][0],t[1][0]]}},sa={name:"y",handles:["n","s"].map(va),input:function(t,n){return null==t?null:[[n[0][0],+t[0]],[n[1][0],+t[1]]]},output:function(t){return t&&[t[0][1],t[1][1]]}},la={name:"xy",handles:["n","w","e","s","nw","ne","sw","se"].map(va),input:function(t){return null==t?null:ca(t)},output:function(t){return t}},ha={overlay:"crosshair",selection:"move",n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},da={e:"w",w:"e",nw:"ne",ne:"nw",se:"sw",sw:"se"},pa={n:"s",s:"n",nw:"sw",ne:"se",se:"ne",sw:"nw"},ga={overlay:1,selection:1,n:null,e:1,s:null,w:-1,nw:-1,ne:1,se:1,sw:-1},ya={overlay:1,selection:1,n:-1,e:null,s:1,w:null,nw:-1,ne:-1,se:1,sw:1};function va(t){return{type:t}}function _a(t){return!t.ctrlKey&&!t.button}function ba(){var t=this.ownerSVGElement||this;return t.hasAttribute("viewBox")?[[(t=t.viewBox.baseVal).x,t.y],[t.x+t.width,t.y+t.height]]:[[0,0],[t.width.baseVal.value,t.height.baseVal.value]]}function ma(){return navigator.maxTouchPoints||"ontouchstart"in this}function xa(t){for(;!t.__brush;)if(!(t=t.parentNode))return;return t.__brush}function wa(t){var n,e=ba,r=_a,i=ma,o=!0,a=$t("start","brush","end"),u=6;function c(n){var e=n.property("__brush",g).selectAll(".overlay").data([va("overlay")]);e.enter().append("rect").attr("class","overlay").attr("pointer-events","all").attr("cursor",ha.overlay).merge(e).each((function(){var t=xa(this).extent;Zn(this).attr("x",t[0][0]).attr("y",t[0][1]).attr("width",t[1][0]-t[0][0]).attr("height",t[1][1]-t[0][1])})),n.selectAll(".selection").data([va("selection")]).enter().append("rect").attr("class","selection").attr("cursor",ha.selection).attr("fill","#777").attr("fill-opacity",.3).attr("stroke","#fff").attr("shape-rendering","crispEdges");var r=n.selectAll(".handle").data(t.handles,(function(t){return t.type}));r.exit().remove(),r.enter().append("rect").attr("class",(function(t){return"handle handle--"+t.type})).attr("cursor",(function(t){return ha[t.type]})),n.each(f).attr("fill","none").attr("pointer-events","all").on("mousedown.brush",h).filter(i).on("touchstart.brush",h).on("touchmove.brush",d).on("touchend.brush touchcancel.brush",p).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function f(){var t=Zn(this),n=xa(this).selection;n?(t.selectAll(".selection").style("display",null).attr("x",n[0][0]).attr("y",n[0][1]).attr("width",n[1][0]-n[0][0]).attr("height",n[1][1]-n[0][1]),t.selectAll(".handle").style("display",null).attr("x",(function(t){return"e"===t.type[t.type.length-1]?n[1][0]-u/2:n[0][0]-u/2})).attr("y",(function(t){return"s"===t.type[0]?n[1][1]-u/2:n[0][1]-u/2})).attr("width",(function(t){return"n"===t.type||"s"===t.type?n[1][0]-n[0][0]+u:u})).attr("height",(function(t){return"e"===t.type||"w"===t.type?n[1][1]-n[0][1]+u:u}))):t.selectAll(".selection,.handle").style("display","none").attr("x",null).attr("y",null).attr("width",null).attr("height",null)}function s(t,n,e){var r=t.__brush.emitter;return!r||e&&r.clean?new l(t,n,e):r}function l(t,n,e){this.that=t,this.args=n,this.state=t.__brush,this.active=0,this.clean=e}function h(e){if((!n||e.touches)&&r.apply(this,arguments)){var i,a,u,c,l,h,d,p,g,y,v,_=this,b=e.target.__data__.type,m="selection"===(o&&e.metaKey?b="overlay":b)?ta:o&&e.altKey?ra:ea,x=t===sa?null:ga[b],w=t===fa?null:ya[b],M=xa(_),T=M.extent,A=M.selection,S=T[0][0],E=T[0][1],N=T[1][0],k=T[1][1],C=0,P=0,z=x&&w&&o&&e.shiftKey,$=Array.from(e.touches||[e],(t=>{const n=t.identifier;return(t=ne(t,_)).point0=t.slice(),t.identifier=n,t}));Gi(_);var D=s(_,arguments,!0).beforestart();if("overlay"===b){A&&(g=!0);const n=[$[0],$[1]||$[0]];M.selection=A=[[i=t===sa?S:aa(n[0][0],n[1][0]),u=t===fa?E:aa(n[0][1],n[1][1])],[l=t===sa?N:oa(n[0][0],n[1][0]),d=t===fa?k:oa(n[0][1],n[1][1])]],$.length>1&&I(e)}else i=A[0][0],u=A[0][1],l=A[1][0],d=A[1][1];a=i,c=u,h=l,p=d;var R=Zn(_).attr("pointer-events","none"),F=R.selectAll(".overlay").attr("cursor",ha[b]);if(e.touches)D.moved=U,D.ended=O;else{var q=Zn(e.view).on("mousemove.brush",U,!0).on("mouseup.brush",O,!0);o&&q.on("keydown.brush",(function(t){switch(t.keyCode){case 16:z=x&&w;break;case 18:m===ea&&(x&&(l=h-C*x,i=a+C*x),w&&(d=p-P*w,u=c+P*w),m=ra,I(t));break;case 32:m!==ea&&m!==ra||(x<0?l=h-C:x>0&&(i=a-C),w<0?d=p-P:w>0&&(u=c-P),m=na,F.attr("cursor",ha.selection),I(t));break;default:return}Jo(t)}),!0).on("keyup.brush",(function(t){switch(t.keyCode){case 16:z&&(y=v=z=!1,I(t));break;case 18:m===ra&&(x<0?l=h:x>0&&(i=a),w<0?d=p:w>0&&(u=c),m=ea,I(t));break;case 32:m===na&&(t.altKey?(x&&(l=h-C*x,i=a+C*x),w&&(d=p-P*w,u=c+P*w),m=ra):(x<0?l=h:x>0&&(i=a),w<0?d=p:w>0&&(u=c),m=ea),F.attr("cursor",ha[b]),I(t));break;default:return}Jo(t)}),!0),ae(e.view)}f.call(_),D.start(e,m.name)}function U(t){for(const n of t.changedTouches||[t])for(const t of $)t.identifier===n.identifier&&(t.cur=ne(n,_));if(z&&!y&&!v&&1===$.length){const t=$[0];ia(t.cur[0]-t[0])>ia(t.cur[1]-t[1])?v=!0:y=!0}for(const t of $)t.cur&&(t[0]=t.cur[0],t[1]=t.cur[1]);g=!0,Jo(t),I(t)}function I(t){const n=$[0],e=n.point0;var r;switch(C=n[0]-e[0],P=n[1]-e[1],m){case na:case ta:x&&(C=oa(S-i,aa(N-l,C)),a=i+C,h=l+C),w&&(P=oa(E-u,aa(k-d,P)),c=u+P,p=d+P);break;case ea:$[1]?(x&&(a=oa(S,aa(N,$[0][0])),h=oa(S,aa(N,$[1][0])),x=1),w&&(c=oa(E,aa(k,$[0][1])),p=oa(E,aa(k,$[1][1])),w=1)):(x<0?(C=oa(S-i,aa(N-i,C)),a=i+C,h=l):x>0&&(C=oa(S-l,aa(N-l,C)),a=i,h=l+C),w<0?(P=oa(E-u,aa(k-u,P)),c=u+P,p=d):w>0&&(P=oa(E-d,aa(k-d,P)),c=u,p=d+P));break;case ra:x&&(a=oa(S,aa(N,i-C*x)),h=oa(S,aa(N,l+C*x))),w&&(c=oa(E,aa(k,u-P*w)),p=oa(E,aa(k,d+P*w)))}ht+e))}function za(t,n){var e=0,r=null,i=null,o=null;function a(a){var u,c=a.length,f=new Array(c),s=Pa(0,c),l=new Array(c*c),h=new Array(c),d=0;a=Float64Array.from({length:c*c},n?(t,n)=>a[n%c][n/c|0]:(t,n)=>a[n/c|0][n%c]);for(let n=0;nr(f[t],f[n])));for(const e of s){const r=n;if(t){const t=Pa(1+~c,c).filter((t=>t<0?a[~t*c+e]:a[e*c+t]));i&&t.sort(((t,n)=>i(t<0?-a[~t*c+e]:a[e*c+t],n<0?-a[~n*c+e]:a[e*c+n])));for(const r of t)if(r<0){(l[~r*c+e]||(l[~r*c+e]={source:null,target:null})).target={index:e,startAngle:n,endAngle:n+=a[~r*c+e]*d,value:a[~r*c+e]}}else{(l[e*c+r]||(l[e*c+r]={source:null,target:null})).source={index:e,startAngle:n,endAngle:n+=a[e*c+r]*d,value:a[e*c+r]}}h[e]={index:e,startAngle:r,endAngle:n,value:f[e]}}else{const t=Pa(0,c).filter((t=>a[e*c+t]||a[t*c+e]));i&&t.sort(((t,n)=>i(a[e*c+t],a[e*c+n])));for(const r of t){let t;if(e=0))throw new Error(`invalid digits: ${t}`);if(n>15)return qa;const e=10**n;return function(t){this._+=t[0];for(let n=1,r=t.length;nRa)if(Math.abs(s*u-c*f)>Ra&&i){let h=e-o,d=r-a,p=u*u+c*c,g=h*h+d*d,y=Math.sqrt(p),v=Math.sqrt(l),_=i*Math.tan(($a-Math.acos((p+l-g)/(2*y*v)))/2),b=_/v,m=_/y;Math.abs(b-1)>Ra&&this._append`L${t+b*f},${n+b*s}`,this._append`A${i},${i},0,0,${+(s*h>f*d)},${this._x1=t+m*u},${this._y1=n+m*c}`}else this._append`L${this._x1=t},${this._y1=n}`;else;}arc(t,n,e,r,i,o){if(t=+t,n=+n,o=!!o,(e=+e)<0)throw new Error(`negative radius: ${e}`);let a=e*Math.cos(r),u=e*Math.sin(r),c=t+a,f=n+u,s=1^o,l=o?r-i:i-r;null===this._x1?this._append`M${c},${f}`:(Math.abs(this._x1-c)>Ra||Math.abs(this._y1-f)>Ra)&&this._append`L${c},${f}`,e&&(l<0&&(l=l%Da+Da),l>Fa?this._append`A${e},${e},0,1,${s},${t-a},${n-u}A${e},${e},0,1,${s},${this._x1=c},${this._y1=f}`:l>Ra&&this._append`A${e},${e},0,${+(l>=$a)},${s},${this._x1=t+e*Math.cos(i)},${this._y1=n+e*Math.sin(i)}`)}rect(t,n,e,r){this._append`M${this._x0=this._x1=+t},${this._y0=this._y1=+n}h${e=+e}v${+r}h${-e}Z`}toString(){return this._}};function Ia(){return new Ua}Ia.prototype=Ua.prototype;var Oa=Array.prototype.slice;function Ba(t){return function(){return t}}function Ya(t){return t.source}function La(t){return t.target}function ja(t){return t.radius}function Ha(t){return t.startAngle}function Xa(t){return t.endAngle}function Ga(){return 0}function Va(){return 10}function Wa(t){var n=Ya,e=La,r=ja,i=ja,o=Ha,a=Xa,u=Ga,c=null;function f(){var f,s=n.apply(this,arguments),l=e.apply(this,arguments),h=u.apply(this,arguments)/2,d=Oa.call(arguments),p=+r.apply(this,(d[0]=s,d)),g=o.apply(this,d)-Ea,y=a.apply(this,d)-Ea,v=+i.apply(this,(d[0]=l,d)),_=o.apply(this,d)-Ea,b=a.apply(this,d)-Ea;if(c||(c=f=Ia()),h>Ca&&(Ma(y-g)>2*h+Ca?y>g?(g+=h,y-=h):(g-=h,y+=h):g=y=(g+y)/2,Ma(b-_)>2*h+Ca?b>_?(_+=h,b-=h):(_-=h,b+=h):_=b=(_+b)/2),c.moveTo(p*Ta(g),p*Aa(g)),c.arc(0,0,p,g,y),g!==_||y!==b)if(t){var m=v-+t.apply(this,arguments),x=(_+b)/2;c.quadraticCurveTo(0,0,m*Ta(_),m*Aa(_)),c.lineTo(v*Ta(x),v*Aa(x)),c.lineTo(m*Ta(b),m*Aa(b))}else c.quadraticCurveTo(0,0,v*Ta(_),v*Aa(_)),c.arc(0,0,v,_,b);if(c.quadraticCurveTo(0,0,p*Ta(g),p*Aa(g)),c.closePath(),f)return c=null,f+""||null}return t&&(f.headRadius=function(n){return arguments.length?(t="function"==typeof n?n:Ba(+n),f):t}),f.radius=function(t){return arguments.length?(r=i="function"==typeof t?t:Ba(+t),f):r},f.sourceRadius=function(t){return arguments.length?(r="function"==typeof t?t:Ba(+t),f):r},f.targetRadius=function(t){return arguments.length?(i="function"==typeof t?t:Ba(+t),f):i},f.startAngle=function(t){return arguments.length?(o="function"==typeof t?t:Ba(+t),f):o},f.endAngle=function(t){return arguments.length?(a="function"==typeof t?t:Ba(+t),f):a},f.padAngle=function(t){return arguments.length?(u="function"==typeof t?t:Ba(+t),f):u},f.source=function(t){return arguments.length?(n=t,f):n},f.target=function(t){return arguments.length?(e=t,f):e},f.context=function(t){return arguments.length?(c=null==t?null:t,f):c},f}var Za=Array.prototype.slice;function Ka(t,n){return t-n}var Qa=t=>()=>t;function Ja(t,n){for(var e,r=-1,i=n.length;++rr!=d>r&&e<(h-f)*(r-s)/(d-s)+f&&(i=-i)}return i}function nu(t,n,e){var r,i,o,a;return function(t,n,e){return(n[0]-t[0])*(e[1]-t[1])==(e[0]-t[0])*(n[1]-t[1])}(t,n,e)&&(i=t[r=+(t[0]===n[0])],o=e[r],a=n[r],i<=o&&o<=a||a<=o&&o<=i)}function eu(){}var ru=[[],[[[1,1.5],[.5,1]]],[[[1.5,1],[1,1.5]]],[[[1.5,1],[.5,1]]],[[[1,.5],[1.5,1]]],[[[1,1.5],[.5,1]],[[1,.5],[1.5,1]]],[[[1,.5],[1,1.5]]],[[[1,.5],[.5,1]]],[[[.5,1],[1,.5]]],[[[1,1.5],[1,.5]]],[[[.5,1],[1,.5]],[[1.5,1],[1,1.5]]],[[[1.5,1],[1,.5]]],[[[.5,1],[1.5,1]]],[[[1,1.5],[1.5,1]]],[[[.5,1],[1,1.5]]],[]];function iu(){var t=1,n=1,e=K,r=u;function i(t){var n=e(t);if(Array.isArray(n))n=n.slice().sort(Ka);else{const e=M(t,ou);for(n=G(...Z(e[0],e[1],n),n);n[n.length-1]>=e[1];)n.pop();for(;n[1]o(t,n)))}function o(e,i){const o=null==i?NaN:+i;if(isNaN(o))throw new Error(`invalid value: ${i}`);var u=[],c=[];return function(e,r,i){var o,u,c,f,s,l,h=new Array,d=new Array;o=u=-1,f=au(e[0],r),ru[f<<1].forEach(p);for(;++o=r,ru[s<<2].forEach(p);for(;++o0?u.push([t]):c.push(t)})),c.forEach((function(t){for(var n,e=0,r=u.length;e0&&o0&&a=0&&o>=0))throw new Error("invalid size");return t=r,n=o,i},i.thresholds=function(t){return arguments.length?(e="function"==typeof t?t:Array.isArray(t)?Qa(Za.call(t)):Qa(t),i):e},i.smooth=function(t){return arguments.length?(r=t?u:eu,i):r===u},i}function ou(t){return isFinite(t)?t:NaN}function au(t,n){return null!=t&&+t>=n}function uu(t){return null==t||isNaN(t=+t)?-1/0:t}function cu(t,n,e,r){const i=r-n,o=e-n,a=isFinite(i)||isFinite(o)?i/o:Math.sign(i)/Math.sign(o);return isNaN(a)?t:t+a-.5}function fu(t){return t[0]}function su(t){return t[1]}function lu(){return 1}const hu=134217729,du=33306690738754706e-32;function pu(t,n,e,r,i){let o,a,u,c,f=n[0],s=r[0],l=0,h=0;s>f==s>-f?(o=f,f=n[++l]):(o=s,s=r[++h]);let d=0;if(lf==s>-f?(a=f+o,u=o-(a-f),f=n[++l]):(a=s+o,u=o-(a-s),s=r[++h]),o=a,0!==u&&(i[d++]=u);lf==s>-f?(a=o+f,c=a-o,u=o-(a-c)+(f-c),f=n[++l]):(a=o+s,c=a-o,u=o-(a-c)+(s-c),s=r[++h]),o=a,0!==u&&(i[d++]=u);for(;l=33306690738754716e-32*f?c:-function(t,n,e,r,i,o,a){let u,c,f,s,l,h,d,p,g,y,v,_,b,m,x,w,M,T;const A=t-i,S=e-i,E=n-o,N=r-o;m=A*N,h=hu*A,d=h-(h-A),p=A-d,h=hu*N,g=h-(h-N),y=N-g,x=p*y-(m-d*g-p*g-d*y),w=E*S,h=hu*E,d=h-(h-E),p=E-d,h=hu*S,g=h-(h-S),y=S-g,M=p*y-(w-d*g-p*g-d*y),v=x-M,l=x-v,_u[0]=x-(v+l)+(l-M),_=m+v,l=_-m,b=m-(_-l)+(v-l),v=b-w,l=b-v,_u[1]=b-(v+l)+(l-w),T=_+v,l=T-_,_u[2]=_-(T-l)+(v-l),_u[3]=T;let k=function(t,n){let e=n[0];for(let r=1;r=C||-k>=C)return k;if(l=t-A,u=t-(A+l)+(l-i),l=e-S,f=e-(S+l)+(l-i),l=n-E,c=n-(E+l)+(l-o),l=r-N,s=r-(N+l)+(l-o),0===u&&0===c&&0===f&&0===s)return k;if(C=vu*a+du*Math.abs(k),k+=A*s+N*u-(E*f+S*c),k>=C||-k>=C)return k;m=u*N,h=hu*u,d=h-(h-u),p=u-d,h=hu*N,g=h-(h-N),y=N-g,x=p*y-(m-d*g-p*g-d*y),w=c*S,h=hu*c,d=h-(h-c),p=c-d,h=hu*S,g=h-(h-S),y=S-g,M=p*y-(w-d*g-p*g-d*y),v=x-M,l=x-v,wu[0]=x-(v+l)+(l-M),_=m+v,l=_-m,b=m-(_-l)+(v-l),v=b-w,l=b-v,wu[1]=b-(v+l)+(l-w),T=_+v,l=T-_,wu[2]=_-(T-l)+(v-l),wu[3]=T;const P=pu(4,_u,4,wu,bu);m=A*s,h=hu*A,d=h-(h-A),p=A-d,h=hu*s,g=h-(h-s),y=s-g,x=p*y-(m-d*g-p*g-d*y),w=E*f,h=hu*E,d=h-(h-E),p=E-d,h=hu*f,g=h-(h-f),y=f-g,M=p*y-(w-d*g-p*g-d*y),v=x-M,l=x-v,wu[0]=x-(v+l)+(l-M),_=m+v,l=_-m,b=m-(_-l)+(v-l),v=b-w,l=b-v,wu[1]=b-(v+l)+(l-w),T=_+v,l=T-_,wu[2]=_-(T-l)+(v-l),wu[3]=T;const z=pu(P,bu,4,wu,mu);m=u*s,h=hu*u,d=h-(h-u),p=u-d,h=hu*s,g=h-(h-s),y=s-g,x=p*y-(m-d*g-p*g-d*y),w=c*f,h=hu*c,d=h-(h-c),p=c-d,h=hu*f,g=h-(h-f),y=f-g,M=p*y-(w-d*g-p*g-d*y),v=x-M,l=x-v,wu[0]=x-(v+l)+(l-M),_=m+v,l=_-m,b=m-(_-l)+(v-l),v=b-w,l=b-v,wu[1]=b-(v+l)+(l-w),T=_+v,l=T-_,wu[2]=_-(T-l)+(v-l),wu[3]=T;const $=pu(z,mu,4,wu,xu);return xu[$-1]}(t,n,e,r,i,o,f)}const Tu=Math.pow(2,-52),Au=new Uint32Array(512);class Su{static from(t,n=zu,e=$u){const r=t.length,i=new Float64Array(2*r);for(let o=0;o>1;if(n>0&&"number"!=typeof t[0])throw new Error("Expected coords to contain numbers.");this.coords=t;const e=Math.max(2*n-5,0);this._triangles=new Uint32Array(3*e),this._halfedges=new Int32Array(3*e),this._hashSize=Math.ceil(Math.sqrt(n)),this._hullPrev=new Uint32Array(n),this._hullNext=new Uint32Array(n),this._hullTri=new Uint32Array(n),this._hullHash=new Int32Array(this._hashSize).fill(-1),this._ids=new Uint32Array(n),this._dists=new Float64Array(n),this.update()}update(){const{coords:t,_hullPrev:n,_hullNext:e,_hullTri:r,_hullHash:i}=this,o=t.length>>1;let a=1/0,u=1/0,c=-1/0,f=-1/0;for(let n=0;nc&&(c=e),r>f&&(f=r),this._ids[n]=n}const s=(a+c)/2,l=(u+f)/2;let h,d,p,g=1/0;for(let n=0;n0&&(d=n,g=e)}let _=t[2*d],b=t[2*d+1],m=1/0;for(let n=0;nr&&(n[e++]=i,r=this._dists[i])}return this.hull=n.subarray(0,e),this.triangles=new Uint32Array(0),void(this.halfedges=new Uint32Array(0))}if(Mu(y,v,_,b,x,w)<0){const t=d,n=_,e=b;d=p,_=x,b=w,p=t,x=n,w=e}const M=function(t,n,e,r,i,o){const a=e-t,u=r-n,c=i-t,f=o-n,s=a*a+u*u,l=c*c+f*f,h=.5/(a*f-u*c),d=t+(f*s-u*l)*h,p=n+(a*l-c*s)*h;return{x:d,y:p}}(y,v,_,b,x,w);this._cx=M.x,this._cy=M.y;for(let n=0;n0&&Math.abs(f-o)<=Tu&&Math.abs(s-a)<=Tu)continue;if(o=f,a=s,c===h||c===d||c===p)continue;let l=0;for(let t=0,n=this._hashKey(f,s);t=0;)if(y=g,y===l){y=-1;break}if(-1===y)continue;let v=this._addTriangle(y,c,e[y],-1,-1,r[y]);r[c]=this._legalize(v+2),r[y]=v,T++;let _=e[y];for(;g=e[_],Mu(f,s,t[2*_],t[2*_+1],t[2*g],t[2*g+1])<0;)v=this._addTriangle(_,c,g,r[c],-1,r[_]),r[c]=this._legalize(v+2),e[_]=_,T--,_=g;if(y===l)for(;g=n[y],Mu(f,s,t[2*g],t[2*g+1],t[2*y],t[2*y+1])<0;)v=this._addTriangle(g,c,y,-1,r[y],r[g]),this._legalize(v+2),r[g]=v,e[y]=y,T--,y=g;this._hullStart=n[c]=y,e[y]=n[_]=c,e[c]=_,i[this._hashKey(f,s)]=c,i[this._hashKey(t[2*y],t[2*y+1])]=y}this.hull=new Uint32Array(T);for(let t=0,n=this._hullStart;t0?3-e:1+e)/4}(t-this._cx,n-this._cy)*this._hashSize)%this._hashSize}_legalize(t){const{_triangles:n,_halfedges:e,coords:r}=this;let i=0,o=0;for(;;){const a=e[t],u=t-t%3;if(o=u+(t+2)%3,-1===a){if(0===i)break;t=Au[--i];continue}const c=a-a%3,f=u+(t+1)%3,s=c+(a+2)%3,l=n[o],h=n[t],d=n[f],p=n[s];if(Nu(r[2*l],r[2*l+1],r[2*h],r[2*h+1],r[2*d],r[2*d+1],r[2*p],r[2*p+1])){n[t]=p,n[a]=l;const r=e[s];if(-1===r){let n=this._hullStart;do{if(this._hullTri[n]===s){this._hullTri[n]=t;break}n=this._hullPrev[n]}while(n!==this._hullStart)}this._link(t,r),this._link(a,e[o]),this._link(o,s);const u=c+(a+1)%3;i=e&&n[t[a]]>o;)t[a+1]=t[a--];t[a+1]=r}else{let i=e+1,o=r;Pu(t,e+r>>1,i),n[t[e]]>n[t[r]]&&Pu(t,e,r),n[t[i]]>n[t[r]]&&Pu(t,i,r),n[t[e]]>n[t[i]]&&Pu(t,e,i);const a=t[i],u=n[a];for(;;){do{i++}while(n[t[i]]u);if(o=o-e?(Cu(t,n,i,r),Cu(t,n,e,o-1)):(Cu(t,n,e,o-1),Cu(t,n,i,r))}}function Pu(t,n,e){const r=t[n];t[n]=t[e],t[e]=r}function zu(t){return t[0]}function $u(t){return t[1]}const Du=1e-6;class Ru{constructor(){this._x0=this._y0=this._x1=this._y1=null,this._=""}moveTo(t,n){this._+=`M${this._x0=this._x1=+t},${this._y0=this._y1=+n}`}closePath(){null!==this._x1&&(this._x1=this._x0,this._y1=this._y0,this._+="Z")}lineTo(t,n){this._+=`L${this._x1=+t},${this._y1=+n}`}arc(t,n,e){const r=(t=+t)+(e=+e),i=n=+n;if(e<0)throw new Error("negative radius");null===this._x1?this._+=`M${r},${i}`:(Math.abs(this._x1-r)>Du||Math.abs(this._y1-i)>Du)&&(this._+="L"+r+","+i),e&&(this._+=`A${e},${e},0,1,1,${t-e},${n}A${e},${e},0,1,1,${this._x1=r},${this._y1=i}`)}rect(t,n,e,r){this._+=`M${this._x0=this._x1=+t},${this._y0=this._y1=+n}h${+e}v${+r}h${-e}Z`}value(){return this._||null}}class Fu{constructor(){this._=[]}moveTo(t,n){this._.push([t,n])}closePath(){this._.push(this._[0].slice())}lineTo(t,n){this._.push([t,n])}value(){return this._.length?this._:null}}class qu{constructor(t,[n,e,r,i]=[0,0,960,500]){if(!((r=+r)>=(n=+n)&&(i=+i)>=(e=+e)))throw new Error("invalid bounds");this.delaunay=t,this._circumcenters=new Float64Array(2*t.points.length),this.vectors=new Float64Array(2*t.points.length),this.xmax=r,this.xmin=n,this.ymax=i,this.ymin=e,this._init()}update(){return this.delaunay.update(),this._init(),this}_init(){const{delaunay:{points:t,hull:n,triangles:e},vectors:r}=this;let i,o;const a=this.circumcenters=this._circumcenters.subarray(0,e.length/3*2);for(let r,u,c=0,f=0,s=e.length;c1;)i-=2;for(let t=2;t0){if(n>=this.ymax)return null;(i=(this.ymax-n)/r)0){if(t>=this.xmax)return null;(i=(this.xmax-t)/e)this.xmax?2:0)|(nthis.ymax?8:0)}_simplify(t){if(t&&t.length>4){for(let n=0;n2&&function(t){const{triangles:n,coords:e}=t;for(let t=0;t1e-10)return!1}return!0}(t)){this.collinear=Int32Array.from({length:n.length/2},((t,n)=>n)).sort(((t,e)=>n[2*t]-n[2*e]||n[2*t+1]-n[2*e+1]));const t=this.collinear[0],e=this.collinear[this.collinear.length-1],r=[n[2*t],n[2*t+1],n[2*e],n[2*e+1]],i=1e-8*Math.hypot(r[3]-r[1],r[2]-r[0]);for(let t=0,e=n.length/2;t0&&(this.triangles=new Int32Array(3).fill(-1),this.halfedges=new Int32Array(3).fill(-1),this.triangles[0]=r[0],o[r[0]]=1,2===r.length&&(o[r[1]]=0,this.triangles[1]=r[1],this.triangles[2]=r[1]))}voronoi(t){return new qu(this,t)}*neighbors(t){const{inedges:n,hull:e,_hullIndex:r,halfedges:i,triangles:o,collinear:a}=this;if(a){const n=a.indexOf(t);return n>0&&(yield a[n-1]),void(n=0&&i!==e&&i!==r;)e=i;return i}_step(t,n,e){const{inedges:r,hull:i,_hullIndex:o,halfedges:a,triangles:u,points:c}=this;if(-1===r[t]||!c.length)return(t+1)%(c.length>>1);let f=t,s=Iu(n-c[2*t],2)+Iu(e-c[2*t+1],2);const l=r[t];let h=l;do{let r=u[h];const l=Iu(n-c[2*r],2)+Iu(e-c[2*r+1],2);if(l9999?"+"+Ku(n,6):Ku(n,4))+"-"+Ku(t.getUTCMonth()+1,2)+"-"+Ku(t.getUTCDate(),2)+(o?"T"+Ku(e,2)+":"+Ku(r,2)+":"+Ku(i,2)+"."+Ku(o,3)+"Z":i?"T"+Ku(e,2)+":"+Ku(r,2)+":"+Ku(i,2)+"Z":r||e?"T"+Ku(e,2)+":"+Ku(r,2)+"Z":"")}function Ju(t){var n=new RegExp('["'+t+"\n\r]"),e=t.charCodeAt(0);function r(t,n){var r,i=[],o=t.length,a=0,u=0,c=o<=0,f=!1;function s(){if(c)return Hu;if(f)return f=!1,ju;var n,r,i=a;if(t.charCodeAt(i)===Xu){for(;a++=o?c=!0:(r=t.charCodeAt(a++))===Gu?f=!0:r===Vu&&(f=!0,t.charCodeAt(a)===Gu&&++a),t.slice(i+1,n-1).replace(/""/g,'"')}for(;amc(n,e).then((n=>(new DOMParser).parseFromString(n,t)))}var Sc=Ac("application/xml"),Ec=Ac("text/html"),Nc=Ac("image/svg+xml");function kc(t,n,e,r){if(isNaN(n)||isNaN(e))return t;var i,o,a,u,c,f,s,l,h,d=t._root,p={data:r},g=t._x0,y=t._y0,v=t._x1,_=t._y1;if(!d)return t._root=p,t;for(;d.length;)if((f=n>=(o=(g+v)/2))?g=o:v=o,(s=e>=(a=(y+_)/2))?y=a:_=a,i=d,!(d=d[l=s<<1|f]))return i[l]=p,t;if(u=+t._x.call(null,d.data),c=+t._y.call(null,d.data),n===u&&e===c)return p.next=d,i?i[l]=p:t._root=p,t;do{i=i?i[l]=new Array(4):t._root=new Array(4),(f=n>=(o=(g+v)/2))?g=o:v=o,(s=e>=(a=(y+_)/2))?y=a:_=a}while((l=s<<1|f)==(h=(c>=a)<<1|u>=o));return i[h]=d,i[l]=p,t}function Cc(t,n,e,r,i){this.node=t,this.x0=n,this.y0=e,this.x1=r,this.y1=i}function Pc(t){return t[0]}function zc(t){return t[1]}function $c(t,n,e){var r=new Dc(null==n?Pc:n,null==e?zc:e,NaN,NaN,NaN,NaN);return null==t?r:r.addAll(t)}function Dc(t,n,e,r,i,o){this._x=t,this._y=n,this._x0=e,this._y0=r,this._x1=i,this._y1=o,this._root=void 0}function Rc(t){for(var n={data:t.data},e=n;t=t.next;)e=e.next={data:t.data};return n}var Fc=$c.prototype=Dc.prototype;function qc(t){return function(){return t}}function Uc(t){return 1e-6*(t()-.5)}function Ic(t){return t.x+t.vx}function Oc(t){return t.y+t.vy}function Bc(t){return t.index}function Yc(t,n){var e=t.get(n);if(!e)throw new Error("node not found: "+n);return e}Fc.copy=function(){var t,n,e=new Dc(this._x,this._y,this._x0,this._y0,this._x1,this._y1),r=this._root;if(!r)return e;if(!r.length)return e._root=Rc(r),e;for(t=[{source:r,target:e._root=new Array(4)}];r=t.pop();)for(var i=0;i<4;++i)(n=r.source[i])&&(n.length?t.push({source:n,target:r.target[i]=new Array(4)}):r.target[i]=Rc(n));return e},Fc.add=function(t){const n=+this._x.call(null,t),e=+this._y.call(null,t);return kc(this.cover(n,e),n,e,t)},Fc.addAll=function(t){var n,e,r,i,o=t.length,a=new Array(o),u=new Array(o),c=1/0,f=1/0,s=-1/0,l=-1/0;for(e=0;es&&(s=r),il&&(l=i));if(c>s||f>l)return this;for(this.cover(c,f).cover(s,l),e=0;et||t>=i||r>n||n>=o;)switch(u=(nh||(o=c.y0)>d||(a=c.x1)=v)<<1|t>=y)&&(c=p[p.length-1],p[p.length-1]=p[p.length-1-f],p[p.length-1-f]=c)}else{var _=t-+this._x.call(null,g.data),b=n-+this._y.call(null,g.data),m=_*_+b*b;if(m=(u=(p+y)/2))?p=u:y=u,(s=a>=(c=(g+v)/2))?g=c:v=c,n=d,!(d=d[l=s<<1|f]))return this;if(!d.length)break;(n[l+1&3]||n[l+2&3]||n[l+3&3])&&(e=n,h=l)}for(;d.data!==t;)if(r=d,!(d=d.next))return this;return(i=d.next)&&delete d.next,r?(i?r.next=i:delete r.next,this):n?(i?n[l]=i:delete n[l],(d=n[0]||n[1]||n[2]||n[3])&&d===(n[3]||n[2]||n[1]||n[0])&&!d.length&&(e?e[h]=d:this._root=d),this):(this._root=i,this)},Fc.removeAll=function(t){for(var n=0,e=t.length;n1?r[0]+r.slice(2):r,+t.slice(e+1)]}function Zc(t){return(t=Wc(Math.abs(t)))?t[1]:NaN}var Kc,Qc=/^(?:(.)?([<>=^]))?([+\-( ])?([$#])?(0)?(\d+)?(,)?(\.\d+)?(~)?([a-z%])?$/i;function Jc(t){if(!(n=Qc.exec(t)))throw new Error("invalid format: "+t);var n;return new tf({fill:n[1],align:n[2],sign:n[3],symbol:n[4],zero:n[5],width:n[6],comma:n[7],precision:n[8]&&n[8].slice(1),trim:n[9],type:n[10]})}function tf(t){this.fill=void 0===t.fill?" ":t.fill+"",this.align=void 0===t.align?">":t.align+"",this.sign=void 0===t.sign?"-":t.sign+"",this.symbol=void 0===t.symbol?"":t.symbol+"",this.zero=!!t.zero,this.width=void 0===t.width?void 0:+t.width,this.comma=!!t.comma,this.precision=void 0===t.precision?void 0:+t.precision,this.trim=!!t.trim,this.type=void 0===t.type?"":t.type+""}function nf(t,n){var e=Wc(t,n);if(!e)return t+"";var r=e[0],i=e[1];return i<0?"0."+new Array(-i).join("0")+r:r.length>i+1?r.slice(0,i+1)+"."+r.slice(i+1):r+new Array(i-r.length+2).join("0")}Jc.prototype=tf.prototype,tf.prototype.toString=function(){return this.fill+this.align+this.sign+this.symbol+(this.zero?"0":"")+(void 0===this.width?"":Math.max(1,0|this.width))+(this.comma?",":"")+(void 0===this.precision?"":"."+Math.max(0,0|this.precision))+(this.trim?"~":"")+this.type};var ef={"%":(t,n)=>(100*t).toFixed(n),b:t=>Math.round(t).toString(2),c:t=>t+"",d:function(t){return Math.abs(t=Math.round(t))>=1e21?t.toLocaleString("en").replace(/,/g,""):t.toString(10)},e:(t,n)=>t.toExponential(n),f:(t,n)=>t.toFixed(n),g:(t,n)=>t.toPrecision(n),o:t=>Math.round(t).toString(8),p:(t,n)=>nf(100*t,n),r:nf,s:function(t,n){var e=Wc(t,n);if(!e)return t+"";var r=e[0],i=e[1],o=i-(Kc=3*Math.max(-8,Math.min(8,Math.floor(i/3))))+1,a=r.length;return o===a?r:o>a?r+new Array(o-a+1).join("0"):o>0?r.slice(0,o)+"."+r.slice(o):"0."+new Array(1-o).join("0")+Wc(t,Math.max(0,n+o-1))[0]},X:t=>Math.round(t).toString(16).toUpperCase(),x:t=>Math.round(t).toString(16)};function rf(t){return t}var of,af=Array.prototype.map,uf=["y","z","a","f","p","n","µ","m","","k","M","G","T","P","E","Z","Y"];function cf(t){var n,e,r=void 0===t.grouping||void 0===t.thousands?rf:(n=af.call(t.grouping,Number),e=t.thousands+"",function(t,r){for(var i=t.length,o=[],a=0,u=n[0],c=0;i>0&&u>0&&(c+u+1>r&&(u=Math.max(1,r-c)),o.push(t.substring(i-=u,i+u)),!((c+=u+1)>r));)u=n[a=(a+1)%n.length];return o.reverse().join(e)}),i=void 0===t.currency?"":t.currency[0]+"",o=void 0===t.currency?"":t.currency[1]+"",a=void 0===t.decimal?".":t.decimal+"",u=void 0===t.numerals?rf:function(t){return function(n){return n.replace(/[0-9]/g,(function(n){return t[+n]}))}}(af.call(t.numerals,String)),c=void 0===t.percent?"%":t.percent+"",f=void 0===t.minus?"−":t.minus+"",s=void 0===t.nan?"NaN":t.nan+"";function l(t){var n=(t=Jc(t)).fill,e=t.align,l=t.sign,h=t.symbol,d=t.zero,p=t.width,g=t.comma,y=t.precision,v=t.trim,_=t.type;"n"===_?(g=!0,_="g"):ef[_]||(void 0===y&&(y=12),v=!0,_="g"),(d||"0"===n&&"="===e)&&(d=!0,n="0",e="=");var b="$"===h?i:"#"===h&&/[boxX]/.test(_)?"0"+_.toLowerCase():"",m="$"===h?o:/[%p]/.test(_)?c:"",x=ef[_],w=/[defgprs%]/.test(_);function M(t){var i,o,c,h=b,M=m;if("c"===_)M=x(t)+M,t="";else{var T=(t=+t)<0||1/t<0;if(t=isNaN(t)?s:x(Math.abs(t),y),v&&(t=function(t){t:for(var n,e=t.length,r=1,i=-1;r0&&(i=0)}return i>0?t.slice(0,i)+t.slice(n+1):t}(t)),T&&0==+t&&"+"!==l&&(T=!1),h=(T?"("===l?l:f:"-"===l||"("===l?"":l)+h,M=("s"===_?uf[8+Kc/3]:"")+M+(T&&"("===l?")":""),w)for(i=-1,o=t.length;++i(c=t.charCodeAt(i))||c>57){M=(46===c?a+t.slice(i+1):t.slice(i))+M,t=t.slice(0,i);break}}g&&!d&&(t=r(t,1/0));var A=h.length+t.length+M.length,S=A>1)+h+t+M+S.slice(A);break;default:t=S+h+t+M}return u(t)}return y=void 0===y?6:/[gprs]/.test(_)?Math.max(1,Math.min(21,y)):Math.max(0,Math.min(20,y)),M.toString=function(){return t+""},M}return{format:l,formatPrefix:function(t,n){var e=l(((t=Jc(t)).type="f",t)),r=3*Math.max(-8,Math.min(8,Math.floor(Zc(n)/3))),i=Math.pow(10,-r),o=uf[8+r/3];return function(t){return e(i*t)+o}}}}function ff(n){return of=cf(n),t.format=of.format,t.formatPrefix=of.formatPrefix,of}function sf(t){return Math.max(0,-Zc(Math.abs(t)))}function lf(t,n){return Math.max(0,3*Math.max(-8,Math.min(8,Math.floor(Zc(n)/3)))-Zc(Math.abs(t)))}function hf(t,n){return t=Math.abs(t),n=Math.abs(n)-t,Math.max(0,Zc(n)-Zc(t))+1}t.format=void 0,t.formatPrefix=void 0,ff({thousands:",",grouping:[3],currency:["$",""]});var df=1e-6,pf=1e-12,gf=Math.PI,yf=gf/2,vf=gf/4,_f=2*gf,bf=180/gf,mf=gf/180,xf=Math.abs,wf=Math.atan,Mf=Math.atan2,Tf=Math.cos,Af=Math.ceil,Sf=Math.exp,Ef=Math.hypot,Nf=Math.log,kf=Math.pow,Cf=Math.sin,Pf=Math.sign||function(t){return t>0?1:t<0?-1:0},zf=Math.sqrt,$f=Math.tan;function Df(t){return t>1?0:t<-1?gf:Math.acos(t)}function Rf(t){return t>1?yf:t<-1?-yf:Math.asin(t)}function Ff(t){return(t=Cf(t/2))*t}function qf(){}function Uf(t,n){t&&Of.hasOwnProperty(t.type)&&Of[t.type](t,n)}var If={Feature:function(t,n){Uf(t.geometry,n)},FeatureCollection:function(t,n){for(var e=t.features,r=-1,i=e.length;++r=0?1:-1,i=r*e,o=Tf(n=(n*=mf)/2+vf),a=Cf(n),u=Vf*a,c=Gf*o+u*Tf(i),f=u*r*Cf(i);as.add(Mf(f,c)),Xf=t,Gf=o,Vf=a}function ds(t){return[Mf(t[1],t[0]),Rf(t[2])]}function ps(t){var n=t[0],e=t[1],r=Tf(e);return[r*Tf(n),r*Cf(n),Cf(e)]}function gs(t,n){return t[0]*n[0]+t[1]*n[1]+t[2]*n[2]}function ys(t,n){return[t[1]*n[2]-t[2]*n[1],t[2]*n[0]-t[0]*n[2],t[0]*n[1]-t[1]*n[0]]}function vs(t,n){t[0]+=n[0],t[1]+=n[1],t[2]+=n[2]}function _s(t,n){return[t[0]*n,t[1]*n,t[2]*n]}function bs(t){var n=zf(t[0]*t[0]+t[1]*t[1]+t[2]*t[2]);t[0]/=n,t[1]/=n,t[2]/=n}var ms,xs,ws,Ms,Ts,As,Ss,Es,Ns,ks,Cs,Ps,zs,$s,Ds,Rs,Fs={point:qs,lineStart:Is,lineEnd:Os,polygonStart:function(){Fs.point=Bs,Fs.lineStart=Ys,Fs.lineEnd=Ls,rs=new T,cs.polygonStart()},polygonEnd:function(){cs.polygonEnd(),Fs.point=qs,Fs.lineStart=Is,Fs.lineEnd=Os,as<0?(Wf=-(Kf=180),Zf=-(Qf=90)):rs>df?Qf=90:rs<-df&&(Zf=-90),os[0]=Wf,os[1]=Kf},sphere:function(){Wf=-(Kf=180),Zf=-(Qf=90)}};function qs(t,n){is.push(os=[Wf=t,Kf=t]),nQf&&(Qf=n)}function Us(t,n){var e=ps([t*mf,n*mf]);if(es){var r=ys(es,e),i=ys([r[1],-r[0],0],r);bs(i),i=ds(i);var o,a=t-Jf,u=a>0?1:-1,c=i[0]*bf*u,f=xf(a)>180;f^(u*JfQf&&(Qf=o):f^(u*Jf<(c=(c+360)%360-180)&&cQf&&(Qf=n)),f?tjs(Wf,Kf)&&(Kf=t):js(t,Kf)>js(Wf,Kf)&&(Wf=t):Kf>=Wf?(tKf&&(Kf=t)):t>Jf?js(Wf,t)>js(Wf,Kf)&&(Kf=t):js(t,Kf)>js(Wf,Kf)&&(Wf=t)}else is.push(os=[Wf=t,Kf=t]);nQf&&(Qf=n),es=e,Jf=t}function Is(){Fs.point=Us}function Os(){os[0]=Wf,os[1]=Kf,Fs.point=qs,es=null}function Bs(t,n){if(es){var e=t-Jf;rs.add(xf(e)>180?e+(e>0?360:-360):e)}else ts=t,ns=n;cs.point(t,n),Us(t,n)}function Ys(){cs.lineStart()}function Ls(){Bs(ts,ns),cs.lineEnd(),xf(rs)>df&&(Wf=-(Kf=180)),os[0]=Wf,os[1]=Kf,es=null}function js(t,n){return(n-=t)<0?n+360:n}function Hs(t,n){return t[0]-n[0]}function Xs(t,n){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:ngf&&(t-=Math.round(t/_f)*_f),[t,n]}function ul(t,n,e){return(t%=_f)?n||e?ol(fl(t),sl(n,e)):fl(t):n||e?sl(n,e):al}function cl(t){return function(n,e){return xf(n+=t)>gf&&(n-=Math.round(n/_f)*_f),[n,e]}}function fl(t){var n=cl(t);return n.invert=cl(-t),n}function sl(t,n){var e=Tf(t),r=Cf(t),i=Tf(n),o=Cf(n);function a(t,n){var a=Tf(n),u=Tf(t)*a,c=Cf(t)*a,f=Cf(n),s=f*e+u*r;return[Mf(c*i-s*o,u*e-f*r),Rf(s*i+c*o)]}return a.invert=function(t,n){var a=Tf(n),u=Tf(t)*a,c=Cf(t)*a,f=Cf(n),s=f*i-c*o;return[Mf(c*i+f*o,u*e+s*r),Rf(s*e-u*r)]},a}function ll(t){function n(n){return(n=t(n[0]*mf,n[1]*mf))[0]*=bf,n[1]*=bf,n}return t=ul(t[0]*mf,t[1]*mf,t.length>2?t[2]*mf:0),n.invert=function(n){return(n=t.invert(n[0]*mf,n[1]*mf))[0]*=bf,n[1]*=bf,n},n}function hl(t,n,e,r,i,o){if(e){var a=Tf(n),u=Cf(n),c=r*e;null==i?(i=n+r*_f,o=n-c/2):(i=dl(a,i),o=dl(a,o),(r>0?io)&&(i+=r*_f));for(var f,s=i;r>0?s>o:s1&&n.push(n.pop().concat(n.shift()))},result:function(){var e=n;return n=[],t=null,e}}}function gl(t,n){return xf(t[0]-n[0])=0;--o)i.point((s=f[o])[0],s[1]);else r(h.x,h.p.x,-1,i);h=h.p}f=(h=h.o).z,d=!d}while(!h.v);i.lineEnd()}}}function _l(t){if(n=t.length){for(var n,e,r=0,i=t[0];++r=0?1:-1,E=S*A,N=E>gf,k=y*w;if(c.add(Mf(k*S*Cf(E),v*M+k*Tf(E))),a+=N?A+S*_f:A,N^p>=e^m>=e){var C=ys(ps(d),ps(b));bs(C);var P=ys(o,C);bs(P);var z=(N^A>=0?-1:1)*Rf(P[2]);(r>z||r===z&&(C[0]||C[1]))&&(u+=N^A>=0?1:-1)}}return(a<-df||a0){for(l||(i.polygonStart(),l=!0),i.lineStart(),t=0;t1&&2&c&&h.push(h.pop().concat(h.shift())),a.push(h.filter(wl))}return h}}function wl(t){return t.length>1}function Ml(t,n){return((t=t.x)[0]<0?t[1]-yf-df:yf-t[1])-((n=n.x)[0]<0?n[1]-yf-df:yf-n[1])}al.invert=al;var Tl=xl((function(){return!0}),(function(t){var n,e=NaN,r=NaN,i=NaN;return{lineStart:function(){t.lineStart(),n=1},point:function(o,a){var u=o>0?gf:-gf,c=xf(o-e);xf(c-gf)0?yf:-yf),t.point(i,r),t.lineEnd(),t.lineStart(),t.point(u,r),t.point(o,r),n=0):i!==u&&c>=gf&&(xf(e-i)df?wf((Cf(n)*(o=Tf(r))*Cf(e)-Cf(r)*(i=Tf(n))*Cf(t))/(i*o*a)):(n+r)/2}(e,r,o,a),t.point(i,r),t.lineEnd(),t.lineStart(),t.point(u,r),n=0),t.point(e=o,r=a),i=u},lineEnd:function(){t.lineEnd(),e=r=NaN},clean:function(){return 2-n}}}),(function(t,n,e,r){var i;if(null==t)i=e*yf,r.point(-gf,i),r.point(0,i),r.point(gf,i),r.point(gf,0),r.point(gf,-i),r.point(0,-i),r.point(-gf,-i),r.point(-gf,0),r.point(-gf,i);else if(xf(t[0]-n[0])>df){var o=t[0]0,i=xf(n)>df;function o(t,e){return Tf(t)*Tf(e)>n}function a(t,e,r){var i=[1,0,0],o=ys(ps(t),ps(e)),a=gs(o,o),u=o[0],c=a-u*u;if(!c)return!r&&t;var f=n*a/c,s=-n*u/c,l=ys(i,o),h=_s(i,f);vs(h,_s(o,s));var d=l,p=gs(h,d),g=gs(d,d),y=p*p-g*(gs(h,h)-1);if(!(y<0)){var v=zf(y),_=_s(d,(-p-v)/g);if(vs(_,h),_=ds(_),!r)return _;var b,m=t[0],x=e[0],w=t[1],M=e[1];x0^_[1]<(xf(_[0]-m)gf^(m<=_[0]&&_[0]<=x)){var S=_s(d,(-p+v)/g);return vs(S,h),[_,ds(S)]}}}function u(n,e){var i=r?t:gf-t,o=0;return n<-i?o|=1:n>i&&(o|=2),e<-i?o|=4:e>i&&(o|=8),o}return xl(o,(function(t){var n,e,c,f,s;return{lineStart:function(){f=c=!1,s=1},point:function(l,h){var d,p=[l,h],g=o(l,h),y=r?g?0:u(l,h):g?u(l+(l<0?gf:-gf),h):0;if(!n&&(f=c=g)&&t.lineStart(),g!==c&&(!(d=a(n,p))||gl(n,d)||gl(p,d))&&(p[2]=1),g!==c)s=0,g?(t.lineStart(),d=a(p,n),t.point(d[0],d[1])):(d=a(n,p),t.point(d[0],d[1],2),t.lineEnd()),n=d;else if(i&&n&&r^g){var v;y&e||!(v=a(p,n,!0))||(s=0,r?(t.lineStart(),t.point(v[0][0],v[0][1]),t.point(v[1][0],v[1][1]),t.lineEnd()):(t.point(v[1][0],v[1][1]),t.lineEnd(),t.lineStart(),t.point(v[0][0],v[0][1],3)))}!g||n&&gl(n,p)||t.point(p[0],p[1]),n=p,c=g,e=y},lineEnd:function(){c&&t.lineEnd(),n=null},clean:function(){return s|(f&&c)<<1}}}),(function(n,r,i,o){hl(o,t,e,i,n,r)}),r?[0,-t]:[-gf,t-gf])}var Sl,El,Nl,kl,Cl=1e9,Pl=-Cl;function zl(t,n,e,r){function i(i,o){return t<=i&&i<=e&&n<=o&&o<=r}function o(i,o,u,f){var s=0,l=0;if(null==i||(s=a(i,u))!==(l=a(o,u))||c(i,o)<0^u>0)do{f.point(0===s||3===s?t:e,s>1?r:n)}while((s=(s+u+4)%4)!==l);else f.point(o[0],o[1])}function a(r,i){return xf(r[0]-t)0?0:3:xf(r[0]-e)0?2:1:xf(r[1]-n)0?1:0:i>0?3:2}function u(t,n){return c(t.x,n.x)}function c(t,n){var e=a(t,1),r=a(n,1);return e!==r?e-r:0===e?n[1]-t[1]:1===e?t[0]-n[0]:2===e?t[1]-n[1]:n[0]-t[0]}return function(a){var c,f,s,l,h,d,p,g,y,v,_,b=a,m=pl(),x={point:w,lineStart:function(){x.point=M,f&&f.push(s=[]);v=!0,y=!1,p=g=NaN},lineEnd:function(){c&&(M(l,h),d&&y&&m.rejoin(),c.push(m.result()));x.point=w,y&&b.lineEnd()},polygonStart:function(){b=m,c=[],f=[],_=!0},polygonEnd:function(){var n=function(){for(var n=0,e=0,i=f.length;er&&(h-o)*(r-a)>(d-a)*(t-o)&&++n:d<=r&&(h-o)*(r-a)<(d-a)*(t-o)&&--n;return n}(),e=_&&n,i=(c=ft(c)).length;(e||i)&&(a.polygonStart(),e&&(a.lineStart(),o(null,null,1,a),a.lineEnd()),i&&vl(c,u,n,o,a),a.polygonEnd());b=a,c=f=s=null}};function w(t,n){i(t,n)&&b.point(t,n)}function M(o,a){var u=i(o,a);if(f&&s.push([o,a]),v)l=o,h=a,d=u,v=!1,u&&(b.lineStart(),b.point(o,a));else if(u&&y)b.point(o,a);else{var c=[p=Math.max(Pl,Math.min(Cl,p)),g=Math.max(Pl,Math.min(Cl,g))],m=[o=Math.max(Pl,Math.min(Cl,o)),a=Math.max(Pl,Math.min(Cl,a))];!function(t,n,e,r,i,o){var a,u=t[0],c=t[1],f=0,s=1,l=n[0]-u,h=n[1]-c;if(a=e-u,l||!(a>0)){if(a/=l,l<0){if(a0){if(a>s)return;a>f&&(f=a)}if(a=i-u,l||!(a<0)){if(a/=l,l<0){if(a>s)return;a>f&&(f=a)}else if(l>0){if(a0)){if(a/=h,h<0){if(a0){if(a>s)return;a>f&&(f=a)}if(a=o-c,h||!(a<0)){if(a/=h,h<0){if(a>s)return;a>f&&(f=a)}else if(h>0){if(a0&&(t[0]=u+f*l,t[1]=c+f*h),s<1&&(n[0]=u+s*l,n[1]=c+s*h),!0}}}}}(c,m,t,n,e,r)?u&&(b.lineStart(),b.point(o,a),_=!1):(y||(b.lineStart(),b.point(c[0],c[1])),b.point(m[0],m[1]),u||b.lineEnd(),_=!1)}p=o,g=a,y=u}return x}}var $l={sphere:qf,point:qf,lineStart:function(){$l.point=Rl,$l.lineEnd=Dl},lineEnd:qf,polygonStart:qf,polygonEnd:qf};function Dl(){$l.point=$l.lineEnd=qf}function Rl(t,n){El=t*=mf,Nl=Cf(n*=mf),kl=Tf(n),$l.point=Fl}function Fl(t,n){t*=mf;var e=Cf(n*=mf),r=Tf(n),i=xf(t-El),o=Tf(i),a=r*Cf(i),u=kl*e-Nl*r*o,c=Nl*e+kl*r*o;Sl.add(Mf(zf(a*a+u*u),c)),El=t,Nl=e,kl=r}function ql(t){return Sl=new T,Lf(t,$l),+Sl}var Ul=[null,null],Il={type:"LineString",coordinates:Ul};function Ol(t,n){return Ul[0]=t,Ul[1]=n,ql(Il)}var Bl={Feature:function(t,n){return Ll(t.geometry,n)},FeatureCollection:function(t,n){for(var e=t.features,r=-1,i=e.length;++r0&&(i=Ol(t[o],t[o-1]))>0&&e<=i&&r<=i&&(e+r-i)*(1-Math.pow((e-r)/i,2))df})).map(c)).concat(lt(Af(o/d)*d,i,d).filter((function(t){return xf(t%g)>df})).map(f))}return v.lines=function(){return _().map((function(t){return{type:"LineString",coordinates:t}}))},v.outline=function(){return{type:"Polygon",coordinates:[s(r).concat(l(a).slice(1),s(e).reverse().slice(1),l(u).reverse().slice(1))]}},v.extent=function(t){return arguments.length?v.extentMajor(t).extentMinor(t):v.extentMinor()},v.extentMajor=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],u=+t[0][1],a=+t[1][1],r>e&&(t=r,r=e,e=t),u>a&&(t=u,u=a,a=t),v.precision(y)):[[r,u],[e,a]]},v.extentMinor=function(e){return arguments.length?(n=+e[0][0],t=+e[1][0],o=+e[0][1],i=+e[1][1],n>t&&(e=n,n=t,t=e),o>i&&(e=o,o=i,i=e),v.precision(y)):[[n,o],[t,i]]},v.step=function(t){return arguments.length?v.stepMajor(t).stepMinor(t):v.stepMinor()},v.stepMajor=function(t){return arguments.length?(p=+t[0],g=+t[1],v):[p,g]},v.stepMinor=function(t){return arguments.length?(h=+t[0],d=+t[1],v):[h,d]},v.precision=function(h){return arguments.length?(y=+h,c=Wl(o,i,90),f=Zl(n,t,y),s=Wl(u,a,90),l=Zl(r,e,y),v):y},v.extentMajor([[-180,-90+df],[180,90-df]]).extentMinor([[-180,-80-df],[180,80+df]])}var Ql,Jl,th,nh,eh=t=>t,rh=new T,ih=new T,oh={point:qf,lineStart:qf,lineEnd:qf,polygonStart:function(){oh.lineStart=ah,oh.lineEnd=fh},polygonEnd:function(){oh.lineStart=oh.lineEnd=oh.point=qf,rh.add(xf(ih)),ih=new T},result:function(){var t=rh/2;return rh=new T,t}};function ah(){oh.point=uh}function uh(t,n){oh.point=ch,Ql=th=t,Jl=nh=n}function ch(t,n){ih.add(nh*t-th*n),th=t,nh=n}function fh(){ch(Ql,Jl)}var sh=oh,lh=1/0,hh=lh,dh=-lh,ph=dh,gh={point:function(t,n){tdh&&(dh=t);nph&&(ph=n)},lineStart:qf,lineEnd:qf,polygonStart:qf,polygonEnd:qf,result:function(){var t=[[lh,hh],[dh,ph]];return dh=ph=-(hh=lh=1/0),t}};var yh,vh,_h,bh,mh=gh,xh=0,wh=0,Mh=0,Th=0,Ah=0,Sh=0,Eh=0,Nh=0,kh=0,Ch={point:Ph,lineStart:zh,lineEnd:Rh,polygonStart:function(){Ch.lineStart=Fh,Ch.lineEnd=qh},polygonEnd:function(){Ch.point=Ph,Ch.lineStart=zh,Ch.lineEnd=Rh},result:function(){var t=kh?[Eh/kh,Nh/kh]:Sh?[Th/Sh,Ah/Sh]:Mh?[xh/Mh,wh/Mh]:[NaN,NaN];return xh=wh=Mh=Th=Ah=Sh=Eh=Nh=kh=0,t}};function Ph(t,n){xh+=t,wh+=n,++Mh}function zh(){Ch.point=$h}function $h(t,n){Ch.point=Dh,Ph(_h=t,bh=n)}function Dh(t,n){var e=t-_h,r=n-bh,i=zf(e*e+r*r);Th+=i*(_h+t)/2,Ah+=i*(bh+n)/2,Sh+=i,Ph(_h=t,bh=n)}function Rh(){Ch.point=Ph}function Fh(){Ch.point=Uh}function qh(){Ih(yh,vh)}function Uh(t,n){Ch.point=Ih,Ph(yh=_h=t,vh=bh=n)}function Ih(t,n){var e=t-_h,r=n-bh,i=zf(e*e+r*r);Th+=i*(_h+t)/2,Ah+=i*(bh+n)/2,Sh+=i,Eh+=(i=bh*t-_h*n)*(_h+t),Nh+=i*(bh+n),kh+=3*i,Ph(_h=t,bh=n)}var Oh=Ch;function Bh(t){this._context=t}Bh.prototype={_radius:4.5,pointRadius:function(t){return this._radius=t,this},polygonStart:function(){this._line=0},polygonEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){0===this._line&&this._context.closePath(),this._point=NaN},point:function(t,n){switch(this._point){case 0:this._context.moveTo(t,n),this._point=1;break;case 1:this._context.lineTo(t,n);break;default:this._context.moveTo(t+this._radius,n),this._context.arc(t,n,this._radius,0,_f)}},result:qf};var Yh,Lh,jh,Hh,Xh,Gh=new T,Vh={point:qf,lineStart:function(){Vh.point=Wh},lineEnd:function(){Yh&&Zh(Lh,jh),Vh.point=qf},polygonStart:function(){Yh=!0},polygonEnd:function(){Yh=null},result:function(){var t=+Gh;return Gh=new T,t}};function Wh(t,n){Vh.point=Zh,Lh=Hh=t,jh=Xh=n}function Zh(t,n){Hh-=t,Xh-=n,Gh.add(zf(Hh*Hh+Xh*Xh)),Hh=t,Xh=n}var Kh=Vh;let Qh,Jh,td,nd;class ed{constructor(t){this._append=null==t?rd:function(t){const n=Math.floor(t);if(!(n>=0))throw new RangeError(`invalid digits: ${t}`);if(n>15)return rd;if(n!==Qh){const t=10**n;Qh=n,Jh=function(n){let e=1;this._+=n[0];for(const r=n.length;e4*n&&g--){var m=a+h,x=u+d,w=c+p,M=zf(m*m+x*x+w*w),T=Rf(w/=M),A=xf(xf(w)-1)n||xf((v*k+_*C)/b-.5)>.3||a*h+u*d+c*p2?t[2]%360*mf:0,k()):[y*bf,v*bf,_*bf]},E.angle=function(t){return arguments.length?(b=t%360*mf,k()):b*bf},E.reflectX=function(t){return arguments.length?(m=t?-1:1,k()):m<0},E.reflectY=function(t){return arguments.length?(x=t?-1:1,k()):x<0},E.precision=function(t){return arguments.length?(a=dd(u,S=t*t),C()):zf(S)},E.fitExtent=function(t,n){return ud(E,t,n)},E.fitSize=function(t,n){return cd(E,t,n)},E.fitWidth=function(t,n){return fd(E,t,n)},E.fitHeight=function(t,n){return sd(E,t,n)},function(){return n=t.apply(this,arguments),E.invert=n.invert&&N,k()}}function _d(t){var n=0,e=gf/3,r=vd(t),i=r(n,e);return i.parallels=function(t){return arguments.length?r(n=t[0]*mf,e=t[1]*mf):[n*bf,e*bf]},i}function bd(t,n){var e=Cf(t),r=(e+Cf(n))/2;if(xf(r)0?n<-yf+df&&(n=-yf+df):n>yf-df&&(n=yf-df);var e=i/kf(Nd(n),r);return[e*Cf(r*t),i-e*Tf(r*t)]}return o.invert=function(t,n){var e=i-n,o=Pf(r)*zf(t*t+e*e),a=Mf(t,xf(e))*Pf(e);return e*r<0&&(a-=gf*Pf(t)*Pf(e)),[a/r,2*wf(kf(i/o,1/r))-yf]},o}function Cd(t,n){return[t,n]}function Pd(t,n){var e=Tf(t),r=t===n?Cf(t):(e-Tf(n))/(n-t),i=e/r+t;if(xf(r)=0;)n+=e[r].value;else n=1;t.value=n}function Gd(t,n){t instanceof Map?(t=[void 0,t],void 0===n&&(n=Wd)):void 0===n&&(n=Vd);for(var e,r,i,o,a,u=new Qd(t),c=[u];e=c.pop();)if((i=n(e.data))&&(a=(i=Array.from(i)).length))for(e.children=i,o=a-1;o>=0;--o)c.push(r=i[o]=new Qd(i[o])),r.parent=e,r.depth=e.depth+1;return u.eachBefore(Kd)}function Vd(t){return t.children}function Wd(t){return Array.isArray(t)?t[1]:null}function Zd(t){void 0!==t.data.value&&(t.value=t.data.value),t.data=t.data.data}function Kd(t){var n=0;do{t.height=n}while((t=t.parent)&&t.height<++n)}function Qd(t){this.data=t,this.depth=this.height=0,this.parent=null}function Jd(t){return null==t?null:tp(t)}function tp(t){if("function"!=typeof t)throw new Error;return t}function np(){return 0}function ep(t){return function(){return t}}qd.invert=function(t,n){for(var e,r=n,i=r*r,o=i*i*i,a=0;a<12&&(o=(i=(r-=e=(r*(zd+$d*i+o*(Dd+Rd*i))-n)/(zd+3*$d*i+o*(7*Dd+9*Rd*i)))*r)*i*i,!(xf(e)df&&--i>0);return[t/(.8707+(o=r*r)*(o*(o*o*o*(.003971-.001529*o)-.013791)-.131979)),r]},Od.invert=Md(Rf),Bd.invert=Md((function(t){return 2*wf(t)})),Yd.invert=function(t,n){return[-n,2*wf(Sf(t))-yf]},Qd.prototype=Gd.prototype={constructor:Qd,count:function(){return this.eachAfter(Xd)},each:function(t,n){let e=-1;for(const r of this)t.call(n,r,++e,this);return this},eachAfter:function(t,n){for(var e,r,i,o=this,a=[o],u=[],c=-1;o=a.pop();)if(u.push(o),e=o.children)for(r=0,i=e.length;r=0;--r)o.push(e[r]);return this},find:function(t,n){let e=-1;for(const r of this)if(t.call(n,r,++e,this))return r},sum:function(t){return this.eachAfter((function(n){for(var e=+t(n.data)||0,r=n.children,i=r&&r.length;--i>=0;)e+=r[i].value;n.value=e}))},sort:function(t){return this.eachBefore((function(n){n.children&&n.children.sort(t)}))},path:function(t){for(var n=this,e=function(t,n){if(t===n)return t;var e=t.ancestors(),r=n.ancestors(),i=null;t=e.pop(),n=r.pop();for(;t===n;)i=t,t=e.pop(),n=r.pop();return i}(n,t),r=[n];n!==e;)n=n.parent,r.push(n);for(var i=r.length;t!==e;)r.splice(i,0,t),t=t.parent;return r},ancestors:function(){for(var t=this,n=[t];t=t.parent;)n.push(t);return n},descendants:function(){return Array.from(this)},leaves:function(){var t=[];return this.eachBefore((function(n){n.children||t.push(n)})),t},links:function(){var t=this,n=[];return t.each((function(e){e!==t&&n.push({source:e.parent,target:e})})),n},copy:function(){return Gd(this).eachBefore(Zd)},[Symbol.iterator]:function*(){var t,n,e,r,i=this,o=[i];do{for(t=o.reverse(),o=[];i=t.pop();)if(yield i,n=i.children)for(e=0,r=n.length;e(t=(rp*t+ip)%op)/op}function up(t,n){for(var e,r,i=0,o=(t=function(t,n){let e,r,i=t.length;for(;i;)r=n()*i--|0,e=t[i],t[i]=t[r],t[r]=e;return t}(Array.from(t),n)).length,a=[];i0&&e*e>r*r+i*i}function lp(t,n){for(var e=0;e1e-6?(E+Math.sqrt(E*E-4*S*N))/(2*S):N/E);return{x:r+w+M*k,y:i+T+A*k,r:k}}function gp(t,n,e){var r,i,o,a,u=t.x-n.x,c=t.y-n.y,f=u*u+c*c;f?(i=n.r+e.r,i*=i,a=t.r+e.r,i>(a*=a)?(r=(f+a-i)/(2*f),o=Math.sqrt(Math.max(0,a/f-r*r)),e.x=t.x-r*u-o*c,e.y=t.y-r*c+o*u):(r=(f+i-a)/(2*f),o=Math.sqrt(Math.max(0,i/f-r*r)),e.x=n.x+r*u-o*c,e.y=n.y+r*c+o*u)):(e.x=n.x+e.r,e.y=n.y)}function yp(t,n){var e=t.r+n.r-1e-6,r=n.x-t.x,i=n.y-t.y;return e>0&&e*e>r*r+i*i}function vp(t){var n=t._,e=t.next._,r=n.r+e.r,i=(n.x*e.r+e.x*n.r)/r,o=(n.y*e.r+e.y*n.r)/r;return i*i+o*o}function _p(t){this._=t,this.next=null,this.previous=null}function bp(t,n){if(!(o=(t=function(t){return"object"==typeof t&&"length"in t?t:Array.from(t)}(t)).length))return 0;var e,r,i,o,a,u,c,f,s,l,h;if((e=t[0]).x=0,e.y=0,!(o>1))return e.r;if(r=t[1],e.x=-r.r,r.x=e.r,r.y=0,!(o>2))return e.r+r.r;gp(r,e,i=t[2]),e=new _p(e),r=new _p(r),i=new _p(i),e.next=i.previous=r,r.next=e.previous=i,i.next=r.previous=e;t:for(c=3;c1&&!zp(t,n););return t.slice(0,n)}function zp(t,n){if("/"===t[n]){let e=0;for(;n>0&&"\\"===t[--n];)++e;if(0==(1&e))return!0}return!1}function $p(t,n){return t.parent===n.parent?1:2}function Dp(t){var n=t.children;return n?n[0]:t.t}function Rp(t){var n=t.children;return n?n[n.length-1]:t.t}function Fp(t,n,e){var r=e/(n.i-t.i);n.c-=r,n.s+=e,t.c+=r,n.z+=e,n.m+=e}function qp(t,n,e){return t.a.parent===n.parent?t.a:e}function Up(t,n){this._=t,this.parent=null,this.children=null,this.A=null,this.a=this,this.z=0,this.m=0,this.c=0,this.s=0,this.t=null,this.i=n}function Ip(t,n,e,r,i){for(var o,a=t.children,u=-1,c=a.length,f=t.value&&(i-e)/t.value;++uh&&(h=u),y=s*s*g,(d=Math.max(h/y,y/l))>p){s-=u;break}p=d}v.push(a={value:s,dice:c1?n:1)},e}(Op);var Lp=function t(n){function e(t,e,r,i,o){if((a=t._squarify)&&a.ratio===n)for(var a,u,c,f,s,l=-1,h=a.length,d=t.value;++l1?n:1)},e}(Op);function jp(t,n,e){return(n[0]-t[0])*(e[1]-t[1])-(n[1]-t[1])*(e[0]-t[0])}function Hp(t,n){return t[0]-n[0]||t[1]-n[1]}function Xp(t){const n=t.length,e=[0,1];let r,i=2;for(r=2;r1&&jp(t[e[i-2]],t[e[i-1]],t[r])<=0;)--i;e[i++]=r}return e.slice(0,i)}var Gp=Math.random,Vp=function t(n){function e(t,e){return t=null==t?0:+t,e=null==e?1:+e,1===arguments.length?(e=t,t=0):e-=t,function(){return n()*e+t}}return e.source=t,e}(Gp),Wp=function t(n){function e(t,e){return arguments.length<2&&(e=t,t=0),t=Math.floor(t),e=Math.floor(e)-t,function(){return Math.floor(n()*e+t)}}return e.source=t,e}(Gp),Zp=function t(n){function e(t,e){var r,i;return t=null==t?0:+t,e=null==e?1:+e,function(){var o;if(null!=r)o=r,r=null;else do{r=2*n()-1,o=2*n()-1,i=r*r+o*o}while(!i||i>1);return t+e*o*Math.sqrt(-2*Math.log(i)/i)}}return e.source=t,e}(Gp),Kp=function t(n){var e=Zp.source(n);function r(){var t=e.apply(this,arguments);return function(){return Math.exp(t())}}return r.source=t,r}(Gp),Qp=function t(n){function e(t){return(t=+t)<=0?()=>0:function(){for(var e=0,r=t;r>1;--r)e+=n();return e+r*n()}}return e.source=t,e}(Gp),Jp=function t(n){var e=Qp.source(n);function r(t){if(0==(t=+t))return n;var r=e(t);return function(){return r()/t}}return r.source=t,r}(Gp),tg=function t(n){function e(t){return function(){return-Math.log1p(-n())/t}}return e.source=t,e}(Gp),ng=function t(n){function e(t){if((t=+t)<0)throw new RangeError("invalid alpha");return t=1/-t,function(){return Math.pow(1-n(),t)}}return e.source=t,e}(Gp),eg=function t(n){function e(t){if((t=+t)<0||t>1)throw new RangeError("invalid p");return function(){return Math.floor(n()+t)}}return e.source=t,e}(Gp),rg=function t(n){function e(t){if((t=+t)<0||t>1)throw new RangeError("invalid p");return 0===t?()=>1/0:1===t?()=>1:(t=Math.log1p(-t),function(){return 1+Math.floor(Math.log1p(-n())/t)})}return e.source=t,e}(Gp),ig=function t(n){var e=Zp.source(n)();function r(t,r){if((t=+t)<0)throw new RangeError("invalid k");if(0===t)return()=>0;if(r=null==r?1:+r,1===t)return()=>-Math.log1p(-n())*r;var i=(t<1?t+1:t)-1/3,o=1/(3*Math.sqrt(i)),a=t<1?()=>Math.pow(n(),1/t):()=>1;return function(){do{do{var t=e(),u=1+o*t}while(u<=0);u*=u*u;var c=1-n()}while(c>=1-.0331*t*t*t*t&&Math.log(c)>=.5*t*t+i*(1-u+Math.log(u)));return i*u*a()*r}}return r.source=t,r}(Gp),og=function t(n){var e=ig.source(n);function r(t,n){var r=e(t),i=e(n);return function(){var t=r();return 0===t?0:t/(t+i())}}return r.source=t,r}(Gp),ag=function t(n){var e=rg.source(n),r=og.source(n);function i(t,n){return t=+t,(n=+n)>=1?()=>t:n<=0?()=>0:function(){for(var i=0,o=t,a=n;o*a>16&&o*(1-a)>16;){var u=Math.floor((o+1)*a),c=r(u,o-u+1)();c<=a?(i+=u,o-=u,a=(a-c)/(1-c)):(o=u-1,a/=c)}for(var f=a<.5,s=e(f?a:1-a),l=s(),h=0;l<=o;++h)l+=s();return i+(f?h:o-h)}}return i.source=t,i}(Gp),ug=function t(n){function e(t,e,r){var i;return 0==(t=+t)?i=t=>-Math.log(t):(t=1/t,i=n=>Math.pow(n,t)),e=null==e?0:+e,r=null==r?1:+r,function(){return e+r*i(-Math.log1p(-n()))}}return e.source=t,e}(Gp),cg=function t(n){function e(t,e){return t=null==t?0:+t,e=null==e?1:+e,function(){return t+e*Math.tan(Math.PI*n())}}return e.source=t,e}(Gp),fg=function t(n){function e(t,e){return t=null==t?0:+t,e=null==e?1:+e,function(){var r=n();return t+e*Math.log(r/(1-r))}}return e.source=t,e}(Gp),sg=function t(n){var e=ig.source(n),r=ag.source(n);function i(t){return function(){for(var i=0,o=t;o>16;){var a=Math.floor(.875*o),u=e(a)();if(u>o)return i+r(a-1,o/u)();i+=a,o-=u}for(var c=-Math.log1p(-n()),f=0;c<=o;++f)c-=Math.log1p(-n());return i+f}}return i.source=t,i}(Gp);const lg=1/4294967296;function hg(t,n){switch(arguments.length){case 0:break;case 1:this.range(t);break;default:this.range(n).domain(t)}return this}function dg(t,n){switch(arguments.length){case 0:break;case 1:"function"==typeof t?this.interpolator(t):this.range(t);break;default:this.domain(t),"function"==typeof n?this.interpolator(n):this.range(n)}return this}const pg=Symbol("implicit");function gg(){var t=new InternMap,n=[],e=[],r=pg;function i(i){let o=t.get(i);if(void 0===o){if(r!==pg)return r;t.set(i,o=n.push(i)-1)}return e[o%e.length]}return i.domain=function(e){if(!arguments.length)return n.slice();n=[],t=new InternMap;for(const r of e)t.has(r)||t.set(r,n.push(r)-1);return i},i.range=function(t){return arguments.length?(e=Array.from(t),i):e.slice()},i.unknown=function(t){return arguments.length?(r=t,i):r},i.copy=function(){return gg(n,e).unknown(r)},hg.apply(i,arguments),i}function yg(){var t,n,e=gg().unknown(void 0),r=e.domain,i=e.range,o=0,a=1,u=!1,c=0,f=0,s=.5;function l(){var e=r().length,l=an&&(e=t,t=n,n=e),function(e){return Math.max(t,Math.min(n,e))}}(a[0],a[t-1])),r=t>2?Mg:wg,i=o=null,l}function l(n){return null==n||isNaN(n=+n)?e:(i||(i=r(a.map(t),u,c)))(t(f(n)))}return l.invert=function(e){return f(n((o||(o=r(u,a.map(t),Yr)))(e)))},l.domain=function(t){return arguments.length?(a=Array.from(t,_g),s()):a.slice()},l.range=function(t){return arguments.length?(u=Array.from(t),s()):u.slice()},l.rangeRound=function(t){return u=Array.from(t),c=Vr,s()},l.clamp=function(t){return arguments.length?(f=!!t||mg,s()):f!==mg},l.interpolate=function(t){return arguments.length?(c=t,s()):c},l.unknown=function(t){return arguments.length?(e=t,l):e},function(e,r){return t=e,n=r,s()}}function Sg(){return Ag()(mg,mg)}function Eg(n,e,r,i){var o,a=W(n,e,r);switch((i=Jc(null==i?",f":i)).type){case"s":var u=Math.max(Math.abs(n),Math.abs(e));return null!=i.precision||isNaN(o=lf(a,u))||(i.precision=o),t.formatPrefix(i,u);case"":case"e":case"g":case"p":case"r":null!=i.precision||isNaN(o=hf(a,Math.max(Math.abs(n),Math.abs(e))))||(i.precision=o-("e"===i.type));break;case"f":case"%":null!=i.precision||isNaN(o=sf(a))||(i.precision=o-2*("%"===i.type))}return t.format(i)}function Ng(t){var n=t.domain;return t.ticks=function(t){var e=n();return G(e[0],e[e.length-1],null==t?10:t)},t.tickFormat=function(t,e){var r=n();return Eg(r[0],r[r.length-1],null==t?10:t,e)},t.nice=function(e){null==e&&(e=10);var r,i,o=n(),a=0,u=o.length-1,c=o[a],f=o[u],s=10;for(f0;){if((i=V(c,f,e))===r)return o[a]=c,o[u]=f,n(o);if(i>0)c=Math.floor(c/i)*i,f=Math.ceil(f/i)*i;else{if(!(i<0))break;c=Math.ceil(c*i)/i,f=Math.floor(f*i)/i}r=i}return t},t}function kg(t,n){var e,r=0,i=(t=t.slice()).length-1,o=t[r],a=t[i];return a-t(-n,e)}function Fg(n){const e=n(Cg,Pg),r=e.domain;let i,o,a=10;function u(){return i=function(t){return t===Math.E?Math.log:10===t&&Math.log10||2===t&&Math.log2||(t=Math.log(t),n=>Math.log(n)/t)}(a),o=function(t){return 10===t?Dg:t===Math.E?Math.exp:n=>Math.pow(t,n)}(a),r()[0]<0?(i=Rg(i),o=Rg(o),n(zg,$g)):n(Cg,Pg),e}return e.base=function(t){return arguments.length?(a=+t,u()):a},e.domain=function(t){return arguments.length?(r(t),u()):r()},e.ticks=t=>{const n=r();let e=n[0],u=n[n.length-1];const c=u0){for(;l<=h;++l)for(f=1;fu)break;p.push(s)}}else for(;l<=h;++l)for(f=a-1;f>=1;--f)if(s=l>0?f/o(-l):f*o(l),!(su)break;p.push(s)}2*p.length{if(null==n&&(n=10),null==r&&(r=10===a?"s":","),"function"!=typeof r&&(a%1||null!=(r=Jc(r)).precision||(r.trim=!0),r=t.format(r)),n===1/0)return r;const u=Math.max(1,a*n/e.ticks().length);return t=>{let n=t/o(Math.round(i(t)));return n*ar(kg(r(),{floor:t=>o(Math.floor(i(t))),ceil:t=>o(Math.ceil(i(t)))})),e}function qg(t){return function(n){return Math.sign(n)*Math.log1p(Math.abs(n/t))}}function Ug(t){return function(n){return Math.sign(n)*Math.expm1(Math.abs(n))*t}}function Ig(t){var n=1,e=t(qg(n),Ug(n));return e.constant=function(e){return arguments.length?t(qg(n=+e),Ug(n)):n},Ng(e)}function Og(t){return function(n){return n<0?-Math.pow(-n,t):Math.pow(n,t)}}function Bg(t){return t<0?-Math.sqrt(-t):Math.sqrt(t)}function Yg(t){return t<0?-t*t:t*t}function Lg(t){var n=t(mg,mg),e=1;return n.exponent=function(n){return arguments.length?1===(e=+n)?t(mg,mg):.5===e?t(Bg,Yg):t(Og(e),Og(1/e)):e},Ng(n)}function jg(){var t=Lg(Ag());return t.copy=function(){return Tg(t,jg()).exponent(t.exponent())},hg.apply(t,arguments),t}function Hg(t){return Math.sign(t)*t*t}const Xg=new Date,Gg=new Date;function Vg(t,n,e,r){function i(n){return t(n=0===arguments.length?new Date:new Date(+n)),n}return i.floor=n=>(t(n=new Date(+n)),n),i.ceil=e=>(t(e=new Date(e-1)),n(e,1),t(e),e),i.round=t=>{const n=i(t),e=i.ceil(t);return t-n(n(t=new Date(+t),null==e?1:Math.floor(e)),t),i.range=(e,r,o)=>{const a=[];if(e=i.ceil(e),o=null==o?1:Math.floor(o),!(e0))return a;let u;do{a.push(u=new Date(+e)),n(e,o),t(e)}while(uVg((n=>{if(n>=n)for(;t(n),!e(n);)n.setTime(n-1)}),((t,r)=>{if(t>=t)if(r<0)for(;++r<=0;)for(;n(t,-1),!e(t););else for(;--r>=0;)for(;n(t,1),!e(t););})),e&&(i.count=(n,r)=>(Xg.setTime(+n),Gg.setTime(+r),t(Xg),t(Gg),Math.floor(e(Xg,Gg))),i.every=t=>(t=Math.floor(t),isFinite(t)&&t>0?t>1?i.filter(r?n=>r(n)%t==0:n=>i.count(0,n)%t==0):i:null)),i}const Wg=Vg((()=>{}),((t,n)=>{t.setTime(+t+n)}),((t,n)=>n-t));Wg.every=t=>(t=Math.floor(t),isFinite(t)&&t>0?t>1?Vg((n=>{n.setTime(Math.floor(n/t)*t)}),((n,e)=>{n.setTime(+n+e*t)}),((n,e)=>(e-n)/t)):Wg:null);const Zg=Wg.range,Kg=1e3,Qg=6e4,Jg=36e5,ty=864e5,ny=6048e5,ey=2592e6,ry=31536e6,iy=Vg((t=>{t.setTime(t-t.getMilliseconds())}),((t,n)=>{t.setTime(+t+n*Kg)}),((t,n)=>(n-t)/Kg),(t=>t.getUTCSeconds())),oy=iy.range,ay=Vg((t=>{t.setTime(t-t.getMilliseconds()-t.getSeconds()*Kg)}),((t,n)=>{t.setTime(+t+n*Qg)}),((t,n)=>(n-t)/Qg),(t=>t.getMinutes())),uy=ay.range,cy=Vg((t=>{t.setUTCSeconds(0,0)}),((t,n)=>{t.setTime(+t+n*Qg)}),((t,n)=>(n-t)/Qg),(t=>t.getUTCMinutes())),fy=cy.range,sy=Vg((t=>{t.setTime(t-t.getMilliseconds()-t.getSeconds()*Kg-t.getMinutes()*Qg)}),((t,n)=>{t.setTime(+t+n*Jg)}),((t,n)=>(n-t)/Jg),(t=>t.getHours())),ly=sy.range,hy=Vg((t=>{t.setUTCMinutes(0,0,0)}),((t,n)=>{t.setTime(+t+n*Jg)}),((t,n)=>(n-t)/Jg),(t=>t.getUTCHours())),dy=hy.range,py=Vg((t=>t.setHours(0,0,0,0)),((t,n)=>t.setDate(t.getDate()+n)),((t,n)=>(n-t-(n.getTimezoneOffset()-t.getTimezoneOffset())*Qg)/ty),(t=>t.getDate()-1)),gy=py.range,yy=Vg((t=>{t.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCDate(t.getUTCDate()+n)}),((t,n)=>(n-t)/ty),(t=>t.getUTCDate()-1)),vy=yy.range,_y=Vg((t=>{t.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCDate(t.getUTCDate()+n)}),((t,n)=>(n-t)/ty),(t=>Math.floor(t/ty))),by=_y.range;function my(t){return Vg((n=>{n.setDate(n.getDate()-(n.getDay()+7-t)%7),n.setHours(0,0,0,0)}),((t,n)=>{t.setDate(t.getDate()+7*n)}),((t,n)=>(n-t-(n.getTimezoneOffset()-t.getTimezoneOffset())*Qg)/ny))}const xy=my(0),wy=my(1),My=my(2),Ty=my(3),Ay=my(4),Sy=my(5),Ey=my(6),Ny=xy.range,ky=wy.range,Cy=My.range,Py=Ty.range,zy=Ay.range,$y=Sy.range,Dy=Ey.range;function Ry(t){return Vg((n=>{n.setUTCDate(n.getUTCDate()-(n.getUTCDay()+7-t)%7),n.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCDate(t.getUTCDate()+7*n)}),((t,n)=>(n-t)/ny))}const Fy=Ry(0),qy=Ry(1),Uy=Ry(2),Iy=Ry(3),Oy=Ry(4),By=Ry(5),Yy=Ry(6),Ly=Fy.range,jy=qy.range,Hy=Uy.range,Xy=Iy.range,Gy=Oy.range,Vy=By.range,Wy=Yy.range,Zy=Vg((t=>{t.setDate(1),t.setHours(0,0,0,0)}),((t,n)=>{t.setMonth(t.getMonth()+n)}),((t,n)=>n.getMonth()-t.getMonth()+12*(n.getFullYear()-t.getFullYear())),(t=>t.getMonth())),Ky=Zy.range,Qy=Vg((t=>{t.setUTCDate(1),t.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCMonth(t.getUTCMonth()+n)}),((t,n)=>n.getUTCMonth()-t.getUTCMonth()+12*(n.getUTCFullYear()-t.getUTCFullYear())),(t=>t.getUTCMonth())),Jy=Qy.range,tv=Vg((t=>{t.setMonth(0,1),t.setHours(0,0,0,0)}),((t,n)=>{t.setFullYear(t.getFullYear()+n)}),((t,n)=>n.getFullYear()-t.getFullYear()),(t=>t.getFullYear()));tv.every=t=>isFinite(t=Math.floor(t))&&t>0?Vg((n=>{n.setFullYear(Math.floor(n.getFullYear()/t)*t),n.setMonth(0,1),n.setHours(0,0,0,0)}),((n,e)=>{n.setFullYear(n.getFullYear()+e*t)})):null;const nv=tv.range,ev=Vg((t=>{t.setUTCMonth(0,1),t.setUTCHours(0,0,0,0)}),((t,n)=>{t.setUTCFullYear(t.getUTCFullYear()+n)}),((t,n)=>n.getUTCFullYear()-t.getUTCFullYear()),(t=>t.getUTCFullYear()));ev.every=t=>isFinite(t=Math.floor(t))&&t>0?Vg((n=>{n.setUTCFullYear(Math.floor(n.getUTCFullYear()/t)*t),n.setUTCMonth(0,1),n.setUTCHours(0,0,0,0)}),((n,e)=>{n.setUTCFullYear(n.getUTCFullYear()+e*t)})):null;const rv=ev.range;function iv(t,n,e,i,o,a){const u=[[iy,1,Kg],[iy,5,5e3],[iy,15,15e3],[iy,30,3e4],[a,1,Qg],[a,5,3e5],[a,15,9e5],[a,30,18e5],[o,1,Jg],[o,3,108e5],[o,6,216e5],[o,12,432e5],[i,1,ty],[i,2,1728e5],[e,1,ny],[n,1,ey],[n,3,7776e6],[t,1,ry]];function c(n,e,i){const o=Math.abs(e-n)/i,a=r((([,,t])=>t)).right(u,o);if(a===u.length)return t.every(W(n/ry,e/ry,i));if(0===a)return Wg.every(Math.max(W(n,e,i),1));const[c,f]=u[o/u[a-1][2]=12)]},q:function(t){return 1+~~(t.getMonth()/3)},Q:k_,s:C_,S:Zv,u:Kv,U:Qv,V:t_,w:n_,W:e_,x:null,X:null,y:r_,Y:o_,Z:u_,"%":N_},m={a:function(t){return a[t.getUTCDay()]},A:function(t){return o[t.getUTCDay()]},b:function(t){return c[t.getUTCMonth()]},B:function(t){return u[t.getUTCMonth()]},c:null,d:c_,e:c_,f:d_,g:T_,G:S_,H:f_,I:s_,j:l_,L:h_,m:p_,M:g_,p:function(t){return i[+(t.getUTCHours()>=12)]},q:function(t){return 1+~~(t.getUTCMonth()/3)},Q:k_,s:C_,S:y_,u:v_,U:__,V:m_,w:x_,W:w_,x:null,X:null,y:M_,Y:A_,Z:E_,"%":N_},x={a:function(t,n,e){var r=d.exec(n.slice(e));return r?(t.w=p.get(r[0].toLowerCase()),e+r[0].length):-1},A:function(t,n,e){var r=l.exec(n.slice(e));return r?(t.w=h.get(r[0].toLowerCase()),e+r[0].length):-1},b:function(t,n,e){var r=v.exec(n.slice(e));return r?(t.m=_.get(r[0].toLowerCase()),e+r[0].length):-1},B:function(t,n,e){var r=g.exec(n.slice(e));return r?(t.m=y.get(r[0].toLowerCase()),e+r[0].length):-1},c:function(t,e,r){return T(t,n,e,r)},d:zv,e:zv,f:Uv,g:Nv,G:Ev,H:Dv,I:Dv,j:$v,L:qv,m:Pv,M:Rv,p:function(t,n,e){var r=f.exec(n.slice(e));return r?(t.p=s.get(r[0].toLowerCase()),e+r[0].length):-1},q:Cv,Q:Ov,s:Bv,S:Fv,u:Mv,U:Tv,V:Av,w:wv,W:Sv,x:function(t,n,r){return T(t,e,n,r)},X:function(t,n,e){return T(t,r,n,e)},y:Nv,Y:Ev,Z:kv,"%":Iv};function w(t,n){return function(e){var r,i,o,a=[],u=-1,c=0,f=t.length;for(e instanceof Date||(e=new Date(+e));++u53)return null;"w"in o||(o.w=1),"Z"in o?(i=(r=sv(lv(o.y,0,1))).getUTCDay(),r=i>4||0===i?qy.ceil(r):qy(r),r=yy.offset(r,7*(o.V-1)),o.y=r.getUTCFullYear(),o.m=r.getUTCMonth(),o.d=r.getUTCDate()+(o.w+6)%7):(i=(r=fv(lv(o.y,0,1))).getDay(),r=i>4||0===i?wy.ceil(r):wy(r),r=py.offset(r,7*(o.V-1)),o.y=r.getFullYear(),o.m=r.getMonth(),o.d=r.getDate()+(o.w+6)%7)}else("W"in o||"U"in o)&&("w"in o||(o.w="u"in o?o.u%7:"W"in o?1:0),i="Z"in o?sv(lv(o.y,0,1)).getUTCDay():fv(lv(o.y,0,1)).getDay(),o.m=0,o.d="W"in o?(o.w+6)%7+7*o.W-(i+5)%7:o.w+7*o.U-(i+6)%7);return"Z"in o?(o.H+=o.Z/100|0,o.M+=o.Z%100,sv(o)):fv(o)}}function T(t,n,e,r){for(var i,o,a=0,u=n.length,c=e.length;a=c)return-1;if(37===(i=n.charCodeAt(a++))){if(i=n.charAt(a++),!(o=x[i in pv?n.charAt(a++):i])||(r=o(t,e,r))<0)return-1}else if(i!=e.charCodeAt(r++))return-1}return r}return b.x=w(e,b),b.X=w(r,b),b.c=w(n,b),m.x=w(e,m),m.X=w(r,m),m.c=w(n,m),{format:function(t){var n=w(t+="",b);return n.toString=function(){return t},n},parse:function(t){var n=M(t+="",!1);return n.toString=function(){return t},n},utcFormat:function(t){var n=w(t+="",m);return n.toString=function(){return t},n},utcParse:function(t){var n=M(t+="",!0);return n.toString=function(){return t},n}}}var dv,pv={"-":"",_:" ",0:"0"},gv=/^\s*\d+/,yv=/^%/,vv=/[\\^$*+?|[\]().{}]/g;function _v(t,n,e){var r=t<0?"-":"",i=(r?-t:t)+"",o=i.length;return r+(o[t.toLowerCase(),n])))}function wv(t,n,e){var r=gv.exec(n.slice(e,e+1));return r?(t.w=+r[0],e+r[0].length):-1}function Mv(t,n,e){var r=gv.exec(n.slice(e,e+1));return r?(t.u=+r[0],e+r[0].length):-1}function Tv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.U=+r[0],e+r[0].length):-1}function Av(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.V=+r[0],e+r[0].length):-1}function Sv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.W=+r[0],e+r[0].length):-1}function Ev(t,n,e){var r=gv.exec(n.slice(e,e+4));return r?(t.y=+r[0],e+r[0].length):-1}function Nv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.y=+r[0]+(+r[0]>68?1900:2e3),e+r[0].length):-1}function kv(t,n,e){var r=/^(Z)|([+-]\d\d)(?::?(\d\d))?/.exec(n.slice(e,e+6));return r?(t.Z=r[1]?0:-(r[2]+(r[3]||"00")),e+r[0].length):-1}function Cv(t,n,e){var r=gv.exec(n.slice(e,e+1));return r?(t.q=3*r[0]-3,e+r[0].length):-1}function Pv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.m=r[0]-1,e+r[0].length):-1}function zv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.d=+r[0],e+r[0].length):-1}function $v(t,n,e){var r=gv.exec(n.slice(e,e+3));return r?(t.m=0,t.d=+r[0],e+r[0].length):-1}function Dv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.H=+r[0],e+r[0].length):-1}function Rv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.M=+r[0],e+r[0].length):-1}function Fv(t,n,e){var r=gv.exec(n.slice(e,e+2));return r?(t.S=+r[0],e+r[0].length):-1}function qv(t,n,e){var r=gv.exec(n.slice(e,e+3));return r?(t.L=+r[0],e+r[0].length):-1}function Uv(t,n,e){var r=gv.exec(n.slice(e,e+6));return r?(t.L=Math.floor(r[0]/1e3),e+r[0].length):-1}function Iv(t,n,e){var r=yv.exec(n.slice(e,e+1));return r?e+r[0].length:-1}function Ov(t,n,e){var r=gv.exec(n.slice(e));return r?(t.Q=+r[0],e+r[0].length):-1}function Bv(t,n,e){var r=gv.exec(n.slice(e));return r?(t.s=+r[0],e+r[0].length):-1}function Yv(t,n){return _v(t.getDate(),n,2)}function Lv(t,n){return _v(t.getHours(),n,2)}function jv(t,n){return _v(t.getHours()%12||12,n,2)}function Hv(t,n){return _v(1+py.count(tv(t),t),n,3)}function Xv(t,n){return _v(t.getMilliseconds(),n,3)}function Gv(t,n){return Xv(t,n)+"000"}function Vv(t,n){return _v(t.getMonth()+1,n,2)}function Wv(t,n){return _v(t.getMinutes(),n,2)}function Zv(t,n){return _v(t.getSeconds(),n,2)}function Kv(t){var n=t.getDay();return 0===n?7:n}function Qv(t,n){return _v(xy.count(tv(t)-1,t),n,2)}function Jv(t){var n=t.getDay();return n>=4||0===n?Ay(t):Ay.ceil(t)}function t_(t,n){return t=Jv(t),_v(Ay.count(tv(t),t)+(4===tv(t).getDay()),n,2)}function n_(t){return t.getDay()}function e_(t,n){return _v(wy.count(tv(t)-1,t),n,2)}function r_(t,n){return _v(t.getFullYear()%100,n,2)}function i_(t,n){return _v((t=Jv(t)).getFullYear()%100,n,2)}function o_(t,n){return _v(t.getFullYear()%1e4,n,4)}function a_(t,n){var e=t.getDay();return _v((t=e>=4||0===e?Ay(t):Ay.ceil(t)).getFullYear()%1e4,n,4)}function u_(t){var n=t.getTimezoneOffset();return(n>0?"-":(n*=-1,"+"))+_v(n/60|0,"0",2)+_v(n%60,"0",2)}function c_(t,n){return _v(t.getUTCDate(),n,2)}function f_(t,n){return _v(t.getUTCHours(),n,2)}function s_(t,n){return _v(t.getUTCHours()%12||12,n,2)}function l_(t,n){return _v(1+yy.count(ev(t),t),n,3)}function h_(t,n){return _v(t.getUTCMilliseconds(),n,3)}function d_(t,n){return h_(t,n)+"000"}function p_(t,n){return _v(t.getUTCMonth()+1,n,2)}function g_(t,n){return _v(t.getUTCMinutes(),n,2)}function y_(t,n){return _v(t.getUTCSeconds(),n,2)}function v_(t){var n=t.getUTCDay();return 0===n?7:n}function __(t,n){return _v(Fy.count(ev(t)-1,t),n,2)}function b_(t){var n=t.getUTCDay();return n>=4||0===n?Oy(t):Oy.ceil(t)}function m_(t,n){return t=b_(t),_v(Oy.count(ev(t),t)+(4===ev(t).getUTCDay()),n,2)}function x_(t){return t.getUTCDay()}function w_(t,n){return _v(qy.count(ev(t)-1,t),n,2)}function M_(t,n){return _v(t.getUTCFullYear()%100,n,2)}function T_(t,n){return _v((t=b_(t)).getUTCFullYear()%100,n,2)}function A_(t,n){return _v(t.getUTCFullYear()%1e4,n,4)}function S_(t,n){var e=t.getUTCDay();return _v((t=e>=4||0===e?Oy(t):Oy.ceil(t)).getUTCFullYear()%1e4,n,4)}function E_(){return"+0000"}function N_(){return"%"}function k_(t){return+t}function C_(t){return Math.floor(+t/1e3)}function P_(n){return dv=hv(n),t.timeFormat=dv.format,t.timeParse=dv.parse,t.utcFormat=dv.utcFormat,t.utcParse=dv.utcParse,dv}t.timeFormat=void 0,t.timeParse=void 0,t.utcFormat=void 0,t.utcParse=void 0,P_({dateTime:"%x, %X",date:"%-m/%-d/%Y",time:"%-I:%M:%S %p",periods:["AM","PM"],days:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],shortDays:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],months:["January","February","March","April","May","June","July","August","September","October","November","December"],shortMonths:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]});var z_="%Y-%m-%dT%H:%M:%S.%LZ";var $_=Date.prototype.toISOString?function(t){return t.toISOString()}:t.utcFormat(z_),D_=$_;var R_=+new Date("2000-01-01T00:00:00.000Z")?function(t){var n=new Date(t);return isNaN(n)?null:n}:t.utcParse(z_),F_=R_;function q_(t){return new Date(t)}function U_(t){return t instanceof Date?+t:+new Date(+t)}function I_(t,n,e,r,i,o,a,u,c,f){var s=Sg(),l=s.invert,h=s.domain,d=f(".%L"),p=f(":%S"),g=f("%I:%M"),y=f("%I %p"),v=f("%a %d"),_=f("%b %d"),b=f("%B"),m=f("%Y");function x(t){return(c(t)Fr(t[t.length-1]),rb=new Array(3).concat("d8b365f5f5f55ab4ac","a6611adfc27d80cdc1018571","a6611adfc27df5f5f580cdc1018571","8c510ad8b365f6e8c3c7eae55ab4ac01665e","8c510ad8b365f6e8c3f5f5f5c7eae55ab4ac01665e","8c510abf812ddfc27df6e8c3c7eae580cdc135978f01665e","8c510abf812ddfc27df6e8c3f5f5f5c7eae580cdc135978f01665e","5430058c510abf812ddfc27df6e8c3c7eae580cdc135978f01665e003c30","5430058c510abf812ddfc27df6e8c3f5f5f5c7eae580cdc135978f01665e003c30").map(H_),ib=eb(rb),ob=new Array(3).concat("af8dc3f7f7f77fbf7b","7b3294c2a5cfa6dba0008837","7b3294c2a5cff7f7f7a6dba0008837","762a83af8dc3e7d4e8d9f0d37fbf7b1b7837","762a83af8dc3e7d4e8f7f7f7d9f0d37fbf7b1b7837","762a839970abc2a5cfe7d4e8d9f0d3a6dba05aae611b7837","762a839970abc2a5cfe7d4e8f7f7f7d9f0d3a6dba05aae611b7837","40004b762a839970abc2a5cfe7d4e8d9f0d3a6dba05aae611b783700441b","40004b762a839970abc2a5cfe7d4e8f7f7f7d9f0d3a6dba05aae611b783700441b").map(H_),ab=eb(ob),ub=new Array(3).concat("e9a3c9f7f7f7a1d76a","d01c8bf1b6dab8e1864dac26","d01c8bf1b6daf7f7f7b8e1864dac26","c51b7de9a3c9fde0efe6f5d0a1d76a4d9221","c51b7de9a3c9fde0eff7f7f7e6f5d0a1d76a4d9221","c51b7dde77aef1b6dafde0efe6f5d0b8e1867fbc414d9221","c51b7dde77aef1b6dafde0eff7f7f7e6f5d0b8e1867fbc414d9221","8e0152c51b7dde77aef1b6dafde0efe6f5d0b8e1867fbc414d9221276419","8e0152c51b7dde77aef1b6dafde0eff7f7f7e6f5d0b8e1867fbc414d9221276419").map(H_),cb=eb(ub),fb=new Array(3).concat("998ec3f7f7f7f1a340","5e3c99b2abd2fdb863e66101","5e3c99b2abd2f7f7f7fdb863e66101","542788998ec3d8daebfee0b6f1a340b35806","542788998ec3d8daebf7f7f7fee0b6f1a340b35806","5427888073acb2abd2d8daebfee0b6fdb863e08214b35806","5427888073acb2abd2d8daebf7f7f7fee0b6fdb863e08214b35806","2d004b5427888073acb2abd2d8daebfee0b6fdb863e08214b358067f3b08","2d004b5427888073acb2abd2d8daebf7f7f7fee0b6fdb863e08214b358067f3b08").map(H_),sb=eb(fb),lb=new Array(3).concat("ef8a62f7f7f767a9cf","ca0020f4a58292c5de0571b0","ca0020f4a582f7f7f792c5de0571b0","b2182bef8a62fddbc7d1e5f067a9cf2166ac","b2182bef8a62fddbc7f7f7f7d1e5f067a9cf2166ac","b2182bd6604df4a582fddbc7d1e5f092c5de4393c32166ac","b2182bd6604df4a582fddbc7f7f7f7d1e5f092c5de4393c32166ac","67001fb2182bd6604df4a582fddbc7d1e5f092c5de4393c32166ac053061","67001fb2182bd6604df4a582fddbc7f7f7f7d1e5f092c5de4393c32166ac053061").map(H_),hb=eb(lb),db=new Array(3).concat("ef8a62ffffff999999","ca0020f4a582bababa404040","ca0020f4a582ffffffbababa404040","b2182bef8a62fddbc7e0e0e09999994d4d4d","b2182bef8a62fddbc7ffffffe0e0e09999994d4d4d","b2182bd6604df4a582fddbc7e0e0e0bababa8787874d4d4d","b2182bd6604df4a582fddbc7ffffffe0e0e0bababa8787874d4d4d","67001fb2182bd6604df4a582fddbc7e0e0e0bababa8787874d4d4d1a1a1a","67001fb2182bd6604df4a582fddbc7ffffffe0e0e0bababa8787874d4d4d1a1a1a").map(H_),pb=eb(db),gb=new Array(3).concat("fc8d59ffffbf91bfdb","d7191cfdae61abd9e92c7bb6","d7191cfdae61ffffbfabd9e92c7bb6","d73027fc8d59fee090e0f3f891bfdb4575b4","d73027fc8d59fee090ffffbfe0f3f891bfdb4575b4","d73027f46d43fdae61fee090e0f3f8abd9e974add14575b4","d73027f46d43fdae61fee090ffffbfe0f3f8abd9e974add14575b4","a50026d73027f46d43fdae61fee090e0f3f8abd9e974add14575b4313695","a50026d73027f46d43fdae61fee090ffffbfe0f3f8abd9e974add14575b4313695").map(H_),yb=eb(gb),vb=new Array(3).concat("fc8d59ffffbf91cf60","d7191cfdae61a6d96a1a9641","d7191cfdae61ffffbfa6d96a1a9641","d73027fc8d59fee08bd9ef8b91cf601a9850","d73027fc8d59fee08bffffbfd9ef8b91cf601a9850","d73027f46d43fdae61fee08bd9ef8ba6d96a66bd631a9850","d73027f46d43fdae61fee08bffffbfd9ef8ba6d96a66bd631a9850","a50026d73027f46d43fdae61fee08bd9ef8ba6d96a66bd631a9850006837","a50026d73027f46d43fdae61fee08bffffbfd9ef8ba6d96a66bd631a9850006837").map(H_),_b=eb(vb),bb=new Array(3).concat("fc8d59ffffbf99d594","d7191cfdae61abdda42b83ba","d7191cfdae61ffffbfabdda42b83ba","d53e4ffc8d59fee08be6f59899d5943288bd","d53e4ffc8d59fee08bffffbfe6f59899d5943288bd","d53e4ff46d43fdae61fee08be6f598abdda466c2a53288bd","d53e4ff46d43fdae61fee08bffffbfe6f598abdda466c2a53288bd","9e0142d53e4ff46d43fdae61fee08be6f598abdda466c2a53288bd5e4fa2","9e0142d53e4ff46d43fdae61fee08bffffbfe6f598abdda466c2a53288bd5e4fa2").map(H_),mb=eb(bb),xb=new Array(3).concat("e5f5f999d8c92ca25f","edf8fbb2e2e266c2a4238b45","edf8fbb2e2e266c2a42ca25f006d2c","edf8fbccece699d8c966c2a42ca25f006d2c","edf8fbccece699d8c966c2a441ae76238b45005824","f7fcfde5f5f9ccece699d8c966c2a441ae76238b45005824","f7fcfde5f5f9ccece699d8c966c2a441ae76238b45006d2c00441b").map(H_),wb=eb(xb),Mb=new Array(3).concat("e0ecf49ebcda8856a7","edf8fbb3cde38c96c688419d","edf8fbb3cde38c96c68856a7810f7c","edf8fbbfd3e69ebcda8c96c68856a7810f7c","edf8fbbfd3e69ebcda8c96c68c6bb188419d6e016b","f7fcfde0ecf4bfd3e69ebcda8c96c68c6bb188419d6e016b","f7fcfde0ecf4bfd3e69ebcda8c96c68c6bb188419d810f7c4d004b").map(H_),Tb=eb(Mb),Ab=new Array(3).concat("e0f3dba8ddb543a2ca","f0f9e8bae4bc7bccc42b8cbe","f0f9e8bae4bc7bccc443a2ca0868ac","f0f9e8ccebc5a8ddb57bccc443a2ca0868ac","f0f9e8ccebc5a8ddb57bccc44eb3d32b8cbe08589e","f7fcf0e0f3dbccebc5a8ddb57bccc44eb3d32b8cbe08589e","f7fcf0e0f3dbccebc5a8ddb57bccc44eb3d32b8cbe0868ac084081").map(H_),Sb=eb(Ab),Eb=new Array(3).concat("fee8c8fdbb84e34a33","fef0d9fdcc8afc8d59d7301f","fef0d9fdcc8afc8d59e34a33b30000","fef0d9fdd49efdbb84fc8d59e34a33b30000","fef0d9fdd49efdbb84fc8d59ef6548d7301f990000","fff7ecfee8c8fdd49efdbb84fc8d59ef6548d7301f990000","fff7ecfee8c8fdd49efdbb84fc8d59ef6548d7301fb300007f0000").map(H_),Nb=eb(Eb),kb=new Array(3).concat("ece2f0a6bddb1c9099","f6eff7bdc9e167a9cf02818a","f6eff7bdc9e167a9cf1c9099016c59","f6eff7d0d1e6a6bddb67a9cf1c9099016c59","f6eff7d0d1e6a6bddb67a9cf3690c002818a016450","fff7fbece2f0d0d1e6a6bddb67a9cf3690c002818a016450","fff7fbece2f0d0d1e6a6bddb67a9cf3690c002818a016c59014636").map(H_),Cb=eb(kb),Pb=new Array(3).concat("ece7f2a6bddb2b8cbe","f1eef6bdc9e174a9cf0570b0","f1eef6bdc9e174a9cf2b8cbe045a8d","f1eef6d0d1e6a6bddb74a9cf2b8cbe045a8d","f1eef6d0d1e6a6bddb74a9cf3690c00570b0034e7b","fff7fbece7f2d0d1e6a6bddb74a9cf3690c00570b0034e7b","fff7fbece7f2d0d1e6a6bddb74a9cf3690c00570b0045a8d023858").map(H_),zb=eb(Pb),$b=new Array(3).concat("e7e1efc994c7dd1c77","f1eef6d7b5d8df65b0ce1256","f1eef6d7b5d8df65b0dd1c77980043","f1eef6d4b9dac994c7df65b0dd1c77980043","f1eef6d4b9dac994c7df65b0e7298ace125691003f","f7f4f9e7e1efd4b9dac994c7df65b0e7298ace125691003f","f7f4f9e7e1efd4b9dac994c7df65b0e7298ace125698004367001f").map(H_),Db=eb($b),Rb=new Array(3).concat("fde0ddfa9fb5c51b8a","feebe2fbb4b9f768a1ae017e","feebe2fbb4b9f768a1c51b8a7a0177","feebe2fcc5c0fa9fb5f768a1c51b8a7a0177","feebe2fcc5c0fa9fb5f768a1dd3497ae017e7a0177","fff7f3fde0ddfcc5c0fa9fb5f768a1dd3497ae017e7a0177","fff7f3fde0ddfcc5c0fa9fb5f768a1dd3497ae017e7a017749006a").map(H_),Fb=eb(Rb),qb=new Array(3).concat("edf8b17fcdbb2c7fb8","ffffcca1dab441b6c4225ea8","ffffcca1dab441b6c42c7fb8253494","ffffccc7e9b47fcdbb41b6c42c7fb8253494","ffffccc7e9b47fcdbb41b6c41d91c0225ea80c2c84","ffffd9edf8b1c7e9b47fcdbb41b6c41d91c0225ea80c2c84","ffffd9edf8b1c7e9b47fcdbb41b6c41d91c0225ea8253494081d58").map(H_),Ub=eb(qb),Ib=new Array(3).concat("f7fcb9addd8e31a354","ffffccc2e69978c679238443","ffffccc2e69978c67931a354006837","ffffccd9f0a3addd8e78c67931a354006837","ffffccd9f0a3addd8e78c67941ab5d238443005a32","ffffe5f7fcb9d9f0a3addd8e78c67941ab5d238443005a32","ffffe5f7fcb9d9f0a3addd8e78c67941ab5d238443006837004529").map(H_),Ob=eb(Ib),Bb=new Array(3).concat("fff7bcfec44fd95f0e","ffffd4fed98efe9929cc4c02","ffffd4fed98efe9929d95f0e993404","ffffd4fee391fec44ffe9929d95f0e993404","ffffd4fee391fec44ffe9929ec7014cc4c028c2d04","ffffe5fff7bcfee391fec44ffe9929ec7014cc4c028c2d04","ffffe5fff7bcfee391fec44ffe9929ec7014cc4c02993404662506").map(H_),Yb=eb(Bb),Lb=new Array(3).concat("ffeda0feb24cf03b20","ffffb2fecc5cfd8d3ce31a1c","ffffb2fecc5cfd8d3cf03b20bd0026","ffffb2fed976feb24cfd8d3cf03b20bd0026","ffffb2fed976feb24cfd8d3cfc4e2ae31a1cb10026","ffffccffeda0fed976feb24cfd8d3cfc4e2ae31a1cb10026","ffffccffeda0fed976feb24cfd8d3cfc4e2ae31a1cbd0026800026").map(H_),jb=eb(Lb),Hb=new Array(3).concat("deebf79ecae13182bd","eff3ffbdd7e76baed62171b5","eff3ffbdd7e76baed63182bd08519c","eff3ffc6dbef9ecae16baed63182bd08519c","eff3ffc6dbef9ecae16baed64292c62171b5084594","f7fbffdeebf7c6dbef9ecae16baed64292c62171b5084594","f7fbffdeebf7c6dbef9ecae16baed64292c62171b508519c08306b").map(H_),Xb=eb(Hb),Gb=new Array(3).concat("e5f5e0a1d99b31a354","edf8e9bae4b374c476238b45","edf8e9bae4b374c47631a354006d2c","edf8e9c7e9c0a1d99b74c47631a354006d2c","edf8e9c7e9c0a1d99b74c47641ab5d238b45005a32","f7fcf5e5f5e0c7e9c0a1d99b74c47641ab5d238b45005a32","f7fcf5e5f5e0c7e9c0a1d99b74c47641ab5d238b45006d2c00441b").map(H_),Vb=eb(Gb),Wb=new Array(3).concat("f0f0f0bdbdbd636363","f7f7f7cccccc969696525252","f7f7f7cccccc969696636363252525","f7f7f7d9d9d9bdbdbd969696636363252525","f7f7f7d9d9d9bdbdbd969696737373525252252525","fffffff0f0f0d9d9d9bdbdbd969696737373525252252525","fffffff0f0f0d9d9d9bdbdbd969696737373525252252525000000").map(H_),Zb=eb(Wb),Kb=new Array(3).concat("efedf5bcbddc756bb1","f2f0f7cbc9e29e9ac86a51a3","f2f0f7cbc9e29e9ac8756bb154278f","f2f0f7dadaebbcbddc9e9ac8756bb154278f","f2f0f7dadaebbcbddc9e9ac8807dba6a51a34a1486","fcfbfdefedf5dadaebbcbddc9e9ac8807dba6a51a34a1486","fcfbfdefedf5dadaebbcbddc9e9ac8807dba6a51a354278f3f007d").map(H_),Qb=eb(Kb),Jb=new Array(3).concat("fee0d2fc9272de2d26","fee5d9fcae91fb6a4acb181d","fee5d9fcae91fb6a4ade2d26a50f15","fee5d9fcbba1fc9272fb6a4ade2d26a50f15","fee5d9fcbba1fc9272fb6a4aef3b2ccb181d99000d","fff5f0fee0d2fcbba1fc9272fb6a4aef3b2ccb181d99000d","fff5f0fee0d2fcbba1fc9272fb6a4aef3b2ccb181da50f1567000d").map(H_),tm=eb(Jb),nm=new Array(3).concat("fee6cefdae6be6550d","feeddefdbe85fd8d3cd94701","feeddefdbe85fd8d3ce6550da63603","feeddefdd0a2fdae6bfd8d3ce6550da63603","feeddefdd0a2fdae6bfd8d3cf16913d948018c2d04","fff5ebfee6cefdd0a2fdae6bfd8d3cf16913d948018c2d04","fff5ebfee6cefdd0a2fdae6bfd8d3cf16913d94801a636037f2704").map(H_),em=eb(nm);var rm=hi(Tr(300,.5,0),Tr(-240,.5,1)),im=hi(Tr(-100,.75,.35),Tr(80,1.5,.8)),om=hi(Tr(260,.75,.35),Tr(80,1.5,.8)),am=Tr();var um=Fe(),cm=Math.PI/3,fm=2*Math.PI/3;function sm(t){var n=t.length;return function(e){return t[Math.max(0,Math.min(n-1,Math.floor(e*n)))]}}var lm=sm(H_("44015444025645045745055946075a46085c460a5d460b5e470d60470e6147106347116447136548146748166848176948186a481a6c481b6d481c6e481d6f481f70482071482173482374482475482576482677482878482979472a7a472c7a472d7b472e7c472f7d46307e46327e46337f463480453581453781453882443983443a83443b84433d84433e85423f854240864241864142874144874045884046883f47883f48893e49893e4a893e4c8a3d4d8a3d4e8a3c4f8a3c508b3b518b3b528b3a538b3a548c39558c39568c38588c38598c375a8c375b8d365c8d365d8d355e8d355f8d34608d34618d33628d33638d32648e32658e31668e31678e31688e30698e306a8e2f6b8e2f6c8e2e6d8e2e6e8e2e6f8e2d708e2d718e2c718e2c728e2c738e2b748e2b758e2a768e2a778e2a788e29798e297a8e297b8e287c8e287d8e277e8e277f8e27808e26818e26828e26828e25838e25848e25858e24868e24878e23888e23898e238a8d228b8d228c8d228d8d218e8d218f8d21908d21918c20928c20928c20938c1f948c1f958b1f968b1f978b1f988b1f998a1f9a8a1e9b8a1e9c891e9d891f9e891f9f881fa0881fa1881fa1871fa28720a38620a48621a58521a68522a78522a88423a98324aa8325ab8225ac8226ad8127ad8128ae8029af7f2ab07f2cb17e2db27d2eb37c2fb47c31b57b32b67a34b67935b77937b87838b9773aba763bbb753dbc743fbc7340bd7242be7144bf7046c06f48c16e4ac16d4cc26c4ec36b50c46a52c56954c56856c66758c7655ac8645cc8635ec96260ca6063cb5f65cb5e67cc5c69cd5b6ccd5a6ece5870cf5773d05675d05477d1537ad1517cd2507fd34e81d34d84d44b86d54989d5488bd6468ed64590d74393d74195d84098d83e9bd93c9dd93ba0da39a2da37a5db36a8db34aadc32addc30b0dd2fb2dd2db5de2bb8de29bade28bddf26c0df25c2df23c5e021c8e020cae11fcde11dd0e11cd2e21bd5e21ad8e219dae319dde318dfe318e2e418e5e419e7e419eae51aece51befe51cf1e51df4e61ef6e620f8e621fbe723fde725")),hm=sm(H_("00000401000501010601010802010902020b02020d03030f03031204041405041606051806051a07061c08071e0907200a08220b09240c09260d0a290e0b2b100b2d110c2f120d31130d34140e36150e38160f3b180f3d19103f1a10421c10441d11471e114920114b21114e22115024125325125527125829115a2a115c2c115f2d11612f116331116533106734106936106b38106c390f6e3b0f703d0f713f0f72400f74420f75440f764510774710784910784a10794c117a4e117b4f127b51127c52137c54137d56147d57157e59157e5a167e5c167f5d177f5f187f601880621980641a80651a80671b80681c816a1c816b1d816d1d816e1e81701f81721f817320817521817621817822817922827b23827c23827e24828025828125818326818426818627818827818928818b29818c29818e2a81902a81912b81932b80942c80962c80982d80992d809b2e7f9c2e7f9e2f7fa02f7fa1307ea3307ea5317ea6317da8327daa337dab337cad347cae347bb0357bb2357bb3367ab5367ab73779b83779ba3878bc3978bd3977bf3a77c03a76c23b75c43c75c53c74c73d73c83e73ca3e72cc3f71cd4071cf4070d0416fd2426fd3436ed5446dd6456cd8456cd9466bdb476adc4869de4968df4a68e04c67e24d66e34e65e44f64e55064e75263e85362e95462ea5661eb5760ec5860ed5a5fee5b5eef5d5ef05f5ef1605df2625df2645cf3655cf4675cf4695cf56b5cf66c5cf66e5cf7705cf7725cf8745cf8765cf9785df9795df97b5dfa7d5efa7f5efa815ffb835ffb8560fb8761fc8961fc8a62fc8c63fc8e64fc9065fd9266fd9467fd9668fd9869fd9a6afd9b6bfe9d6cfe9f6dfea16efea36ffea571fea772fea973feaa74feac76feae77feb078feb27afeb47bfeb67cfeb77efeb97ffebb81febd82febf84fec185fec287fec488fec68afec88cfeca8dfecc8ffecd90fecf92fed194fed395fed597fed799fed89afdda9cfddc9efddea0fde0a1fde2a3fde3a5fde5a7fde7a9fde9aafdebacfcecaefceeb0fcf0b2fcf2b4fcf4b6fcf6b8fcf7b9fcf9bbfcfbbdfcfdbf")),dm=sm(H_("00000401000501010601010802010a02020c02020e03021004031204031405041706041907051b08051d09061f0a07220b07240c08260d08290e092b10092d110a30120a32140b34150b37160b39180c3c190c3e1b0c411c0c431e0c451f0c48210c4a230c4c240c4f260c51280b53290b552b0b572d0b592f0a5b310a5c320a5e340a5f3609613809623909633b09643d09653e0966400a67420a68440a68450a69470b6a490b6a4a0c6b4c0c6b4d0d6c4f0d6c510e6c520e6d540f6d550f6d57106e59106e5a116e5c126e5d126e5f136e61136e62146e64156e65156e67166e69166e6a176e6c186e6d186e6f196e71196e721a6e741a6e751b6e771c6d781c6d7a1d6d7c1d6d7d1e6d7f1e6c801f6c82206c84206b85216b87216b88226a8a226a8c23698d23698f24699025689225689326679526679727669827669a28659b29649d29649f2a63a02a63a22b62a32c61a52c60a62d60a82e5fa92e5eab2f5ead305dae305cb0315bb1325ab3325ab43359b63458b73557b93556ba3655bc3754bd3853bf3952c03a51c13a50c33b4fc43c4ec63d4dc73e4cc83f4bca404acb4149cc4248ce4347cf4446d04545d24644d34743d44842d54a41d74b3fd84c3ed94d3dda4e3cdb503bdd513ade5238df5337e05536e15635e25734e35933e45a31e55c30e65d2fe75e2ee8602de9612bea632aeb6429eb6628ec6726ed6925ee6a24ef6c23ef6e21f06f20f1711ff1731df2741cf3761bf37819f47918f57b17f57d15f67e14f68013f78212f78410f8850ff8870ef8890cf98b0bf98c0af98e09fa9008fa9207fa9407fb9606fb9706fb9906fb9b06fb9d07fc9f07fca108fca309fca50afca60cfca80dfcaa0ffcac11fcae12fcb014fcb216fcb418fbb61afbb81dfbba1ffbbc21fbbe23fac026fac228fac42afac62df9c72ff9c932f9cb35f8cd37f8cf3af7d13df7d340f6d543f6d746f5d949f5db4cf4dd4ff4df53f4e156f3e35af3e55df2e661f2e865f2ea69f1ec6df1ed71f1ef75f1f179f2f27df2f482f3f586f3f68af4f88ef5f992f6fa96f8fb9af9fc9dfafda1fcffa4")),pm=sm(H_("0d088710078813078916078a19068c1b068d1d068e20068f2206902406912605912805922a05932c05942e05952f059631059733059735049837049938049a3a049a3c049b3e049c3f049c41049d43039e44039e46039f48039f4903a04b03a14c02a14e02a25002a25102a35302a35502a45601a45801a45901a55b01a55c01a65e01a66001a66100a76300a76400a76600a76700a86900a86a00a86c00a86e00a86f00a87100a87201a87401a87501a87701a87801a87a02a87b02a87d03a87e03a88004a88104a78305a78405a78606a68707a68808a68a09a58b0aa58d0ba58e0ca48f0da4910ea3920fa39410a29511a19613a19814a099159f9a169f9c179e9d189d9e199da01a9ca11b9ba21d9aa31e9aa51f99a62098a72197a82296aa2395ab2494ac2694ad2793ae2892b02991b12a90b22b8fb32c8eb42e8db52f8cb6308bb7318ab83289ba3388bb3488bc3587bd3786be3885bf3984c03a83c13b82c23c81c33d80c43e7fc5407ec6417dc7427cc8437bc9447aca457acb4679cc4778cc4977cd4a76ce4b75cf4c74d04d73d14e72d24f71d35171d45270d5536fd5546ed6556dd7566cd8576bd9586ada5a6ada5b69db5c68dc5d67dd5e66de5f65de6164df6263e06363e16462e26561e26660e3685fe4695ee56a5de56b5de66c5ce76e5be76f5ae87059e97158e97257ea7457eb7556eb7655ec7754ed7953ed7a52ee7b51ef7c51ef7e50f07f4ff0804ef1814df1834cf2844bf3854bf3874af48849f48948f58b47f58c46f68d45f68f44f79044f79143f79342f89441f89540f9973ff9983ef99a3efa9b3dfa9c3cfa9e3bfb9f3afba139fba238fca338fca537fca636fca835fca934fdab33fdac33fdae32fdaf31fdb130fdb22ffdb42ffdb52efeb72dfeb82cfeba2cfebb2bfebd2afebe2afec029fdc229fdc328fdc527fdc627fdc827fdca26fdcb26fccd25fcce25fcd025fcd225fbd324fbd524fbd724fad824fada24f9dc24f9dd25f8df25f8e125f7e225f7e425f6e626f6e826f5e926f5eb27f4ed27f3ee27f3f027f2f227f1f426f1f525f0f724f0f921"));function gm(t){return function(){return t}}const ym=Math.abs,vm=Math.atan2,_m=Math.cos,bm=Math.max,mm=Math.min,xm=Math.sin,wm=Math.sqrt,Mm=1e-12,Tm=Math.PI,Am=Tm/2,Sm=2*Tm;function Em(t){return t>=1?Am:t<=-1?-Am:Math.asin(t)}function Nm(t){let n=3;return t.digits=function(e){if(!arguments.length)return n;if(null==e)n=null;else{const t=Math.floor(e);if(!(t>=0))throw new RangeError(`invalid digits: ${e}`);n=t}return t},()=>new Ua(n)}function km(t){return t.innerRadius}function Cm(t){return t.outerRadius}function Pm(t){return t.startAngle}function zm(t){return t.endAngle}function $m(t){return t&&t.padAngle}function Dm(t,n,e,r,i,o,a){var u=t-e,c=n-r,f=(a?o:-o)/wm(u*u+c*c),s=f*c,l=-f*u,h=t+s,d=n+l,p=e+s,g=r+l,y=(h+p)/2,v=(d+g)/2,_=p-h,b=g-d,m=_*_+b*b,x=i-o,w=h*g-p*d,M=(b<0?-1:1)*wm(bm(0,x*x*m-w*w)),T=(w*b-_*M)/m,A=(-w*_-b*M)/m,S=(w*b+_*M)/m,E=(-w*_+b*M)/m,N=T-y,k=A-v,C=S-y,P=E-v;return N*N+k*k>C*C+P*P&&(T=S,A=E),{cx:T,cy:A,x01:-s,y01:-l,x11:T*(i/x-1),y11:A*(i/x-1)}}var Rm=Array.prototype.slice;function Fm(t){return"object"==typeof t&&"length"in t?t:Array.from(t)}function qm(t){this._context=t}function Um(t){return new qm(t)}function Im(t){return t[0]}function Om(t){return t[1]}function Bm(t,n){var e=gm(!0),r=null,i=Um,o=null,a=Nm(u);function u(u){var c,f,s,l=(u=Fm(u)).length,h=!1;for(null==r&&(o=i(s=a())),c=0;c<=l;++c)!(c=l;--h)u.point(v[h],_[h]);u.lineEnd(),u.areaEnd()}y&&(v[s]=+t(d,s,f),_[s]=+n(d,s,f),u.point(r?+r(d,s,f):v[s],e?+e(d,s,f):_[s]))}if(p)return u=null,p+""||null}function s(){return Bm().defined(i).curve(a).context(o)}return t="function"==typeof t?t:void 0===t?Im:gm(+t),n="function"==typeof n?n:gm(void 0===n?0:+n),e="function"==typeof e?e:void 0===e?Om:gm(+e),f.x=function(n){return arguments.length?(t="function"==typeof n?n:gm(+n),r=null,f):t},f.x0=function(n){return arguments.length?(t="function"==typeof n?n:gm(+n),f):t},f.x1=function(t){return arguments.length?(r=null==t?null:"function"==typeof t?t:gm(+t),f):r},f.y=function(t){return arguments.length?(n="function"==typeof t?t:gm(+t),e=null,f):n},f.y0=function(t){return arguments.length?(n="function"==typeof t?t:gm(+t),f):n},f.y1=function(t){return arguments.length?(e=null==t?null:"function"==typeof t?t:gm(+t),f):e},f.lineX0=f.lineY0=function(){return s().x(t).y(n)},f.lineY1=function(){return s().x(t).y(e)},f.lineX1=function(){return s().x(r).y(n)},f.defined=function(t){return arguments.length?(i="function"==typeof t?t:gm(!!t),f):i},f.curve=function(t){return arguments.length?(a=t,null!=o&&(u=a(o)),f):a},f.context=function(t){return arguments.length?(null==t?o=u=null:u=a(o=t),f):o},f}function Lm(t,n){return nt?1:n>=t?0:NaN}function jm(t){return t}qm.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;default:this._context.lineTo(t,n)}}};var Hm=Gm(Um);function Xm(t){this._curve=t}function Gm(t){function n(n){return new Xm(t(n))}return n._curve=t,n}function Vm(t){var n=t.curve;return t.angle=t.x,delete t.x,t.radius=t.y,delete t.y,t.curve=function(t){return arguments.length?n(Gm(t)):n()._curve},t}function Wm(){return Vm(Bm().curve(Hm))}function Zm(){var t=Ym().curve(Hm),n=t.curve,e=t.lineX0,r=t.lineX1,i=t.lineY0,o=t.lineY1;return t.angle=t.x,delete t.x,t.startAngle=t.x0,delete t.x0,t.endAngle=t.x1,delete t.x1,t.radius=t.y,delete t.y,t.innerRadius=t.y0,delete t.y0,t.outerRadius=t.y1,delete t.y1,t.lineStartAngle=function(){return Vm(e())},delete t.lineX0,t.lineEndAngle=function(){return Vm(r())},delete t.lineX1,t.lineInnerRadius=function(){return Vm(i())},delete t.lineY0,t.lineOuterRadius=function(){return Vm(o())},delete t.lineY1,t.curve=function(t){return arguments.length?n(Gm(t)):n()._curve},t}function Km(t,n){return[(n=+n)*Math.cos(t-=Math.PI/2),n*Math.sin(t)]}Xm.prototype={areaStart:function(){this._curve.areaStart()},areaEnd:function(){this._curve.areaEnd()},lineStart:function(){this._curve.lineStart()},lineEnd:function(){this._curve.lineEnd()},point:function(t,n){this._curve.point(n*Math.sin(t),n*-Math.cos(t))}};class Qm{constructor(t,n){this._context=t,this._x=n}areaStart(){this._line=0}areaEnd(){this._line=NaN}lineStart(){this._point=0}lineEnd(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line}point(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;default:this._x?this._context.bezierCurveTo(this._x0=(this._x0+t)/2,this._y0,this._x0,n,t,n):this._context.bezierCurveTo(this._x0,this._y0=(this._y0+n)/2,t,this._y0,t,n)}this._x0=t,this._y0=n}}class Jm{constructor(t){this._context=t}lineStart(){this._point=0}lineEnd(){}point(t,n){if(t=+t,n=+n,0===this._point)this._point=1;else{const e=Km(this._x0,this._y0),r=Km(this._x0,this._y0=(this._y0+n)/2),i=Km(t,this._y0),o=Km(t,n);this._context.moveTo(...e),this._context.bezierCurveTo(...r,...i,...o)}this._x0=t,this._y0=n}}function tx(t){return new Qm(t,!0)}function nx(t){return new Qm(t,!1)}function ex(t){return new Jm(t)}function rx(t){return t.source}function ix(t){return t.target}function ox(t){let n=rx,e=ix,r=Im,i=Om,o=null,a=null,u=Nm(c);function c(){let c;const f=Rm.call(arguments),s=n.apply(this,f),l=e.apply(this,f);if(null==o&&(a=t(c=u())),a.lineStart(),f[0]=s,a.point(+r.apply(this,f),+i.apply(this,f)),f[0]=l,a.point(+r.apply(this,f),+i.apply(this,f)),a.lineEnd(),c)return a=null,c+""||null}return c.source=function(t){return arguments.length?(n=t,c):n},c.target=function(t){return arguments.length?(e=t,c):e},c.x=function(t){return arguments.length?(r="function"==typeof t?t:gm(+t),c):r},c.y=function(t){return arguments.length?(i="function"==typeof t?t:gm(+t),c):i},c.context=function(n){return arguments.length?(null==n?o=a=null:a=t(o=n),c):o},c}const ax=wm(3);var ux={draw(t,n){const e=.59436*wm(n+mm(n/28,.75)),r=e/2,i=r*ax;t.moveTo(0,e),t.lineTo(0,-e),t.moveTo(-i,-r),t.lineTo(i,r),t.moveTo(-i,r),t.lineTo(i,-r)}},cx={draw(t,n){const e=wm(n/Tm);t.moveTo(e,0),t.arc(0,0,e,0,Sm)}},fx={draw(t,n){const e=wm(n/5)/2;t.moveTo(-3*e,-e),t.lineTo(-e,-e),t.lineTo(-e,-3*e),t.lineTo(e,-3*e),t.lineTo(e,-e),t.lineTo(3*e,-e),t.lineTo(3*e,e),t.lineTo(e,e),t.lineTo(e,3*e),t.lineTo(-e,3*e),t.lineTo(-e,e),t.lineTo(-3*e,e),t.closePath()}};const sx=wm(1/3),lx=2*sx;var hx={draw(t,n){const e=wm(n/lx),r=e*sx;t.moveTo(0,-e),t.lineTo(r,0),t.lineTo(0,e),t.lineTo(-r,0),t.closePath()}},dx={draw(t,n){const e=.62625*wm(n);t.moveTo(0,-e),t.lineTo(e,0),t.lineTo(0,e),t.lineTo(-e,0),t.closePath()}},px={draw(t,n){const e=.87559*wm(n-mm(n/7,2));t.moveTo(-e,0),t.lineTo(e,0),t.moveTo(0,e),t.lineTo(0,-e)}},gx={draw(t,n){const e=wm(n),r=-e/2;t.rect(r,r,e,e)}},yx={draw(t,n){const e=.4431*wm(n);t.moveTo(e,e),t.lineTo(e,-e),t.lineTo(-e,-e),t.lineTo(-e,e),t.closePath()}};const vx=xm(Tm/10)/xm(7*Tm/10),_x=xm(Sm/10)*vx,bx=-_m(Sm/10)*vx;var mx={draw(t,n){const e=wm(.8908130915292852*n),r=_x*e,i=bx*e;t.moveTo(0,-e),t.lineTo(r,i);for(let n=1;n<5;++n){const o=Sm*n/5,a=_m(o),u=xm(o);t.lineTo(u*e,-a*e),t.lineTo(a*r-u*i,u*r+a*i)}t.closePath()}};const xx=wm(3);var wx={draw(t,n){const e=-wm(n/(3*xx));t.moveTo(0,2*e),t.lineTo(-xx*e,-e),t.lineTo(xx*e,-e),t.closePath()}};const Mx=wm(3);var Tx={draw(t,n){const e=.6824*wm(n),r=e/2,i=e*Mx/2;t.moveTo(0,-e),t.lineTo(i,r),t.lineTo(-i,r),t.closePath()}};const Ax=-.5,Sx=wm(3)/2,Ex=1/wm(12),Nx=3*(Ex/2+1);var kx={draw(t,n){const e=wm(n/Nx),r=e/2,i=e*Ex,o=r,a=e*Ex+e,u=-o,c=a;t.moveTo(r,i),t.lineTo(o,a),t.lineTo(u,c),t.lineTo(Ax*r-Sx*i,Sx*r+Ax*i),t.lineTo(Ax*o-Sx*a,Sx*o+Ax*a),t.lineTo(Ax*u-Sx*c,Sx*u+Ax*c),t.lineTo(Ax*r+Sx*i,Ax*i-Sx*r),t.lineTo(Ax*o+Sx*a,Ax*a-Sx*o),t.lineTo(Ax*u+Sx*c,Ax*c-Sx*u),t.closePath()}},Cx={draw(t,n){const e=.6189*wm(n-mm(n/6,1.7));t.moveTo(-e,-e),t.lineTo(e,e),t.moveTo(-e,e),t.lineTo(e,-e)}};const Px=[cx,fx,hx,gx,mx,wx,kx],zx=[cx,px,Cx,Tx,ux,yx,dx];function $x(){}function Dx(t,n,e){t._context.bezierCurveTo((2*t._x0+t._x1)/3,(2*t._y0+t._y1)/3,(t._x0+2*t._x1)/3,(t._y0+2*t._y1)/3,(t._x0+4*t._x1+n)/6,(t._y0+4*t._y1+e)/6)}function Rx(t){this._context=t}function Fx(t){this._context=t}function qx(t){this._context=t}function Ux(t,n){this._basis=new Rx(t),this._beta=n}Rx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._y0=this._y1=NaN,this._point=0},lineEnd:function(){switch(this._point){case 3:Dx(this,this._x1,this._y1);case 2:this._context.lineTo(this._x1,this._y1)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;break;case 2:this._point=3,this._context.lineTo((5*this._x0+this._x1)/6,(5*this._y0+this._y1)/6);default:Dx(this,t,n)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=n}},Fx.prototype={areaStart:$x,areaEnd:$x,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._y0=this._y1=this._y2=this._y3=this._y4=NaN,this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x2,this._y2),this._context.closePath();break;case 2:this._context.moveTo((this._x2+2*this._x3)/3,(this._y2+2*this._y3)/3),this._context.lineTo((this._x3+2*this._x2)/3,(this._y3+2*this._y2)/3),this._context.closePath();break;case 3:this.point(this._x2,this._y2),this.point(this._x3,this._y3),this.point(this._x4,this._y4)}},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._x2=t,this._y2=n;break;case 1:this._point=2,this._x3=t,this._y3=n;break;case 2:this._point=3,this._x4=t,this._y4=n,this._context.moveTo((this._x0+4*this._x1+t)/6,(this._y0+4*this._y1+n)/6);break;default:Dx(this,t,n)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=n}},qx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._y0=this._y1=NaN,this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3;var e=(this._x0+4*this._x1+t)/6,r=(this._y0+4*this._y1+n)/6;this._line?this._context.lineTo(e,r):this._context.moveTo(e,r);break;case 3:this._point=4;default:Dx(this,t,n)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=n}},Ux.prototype={lineStart:function(){this._x=[],this._y=[],this._basis.lineStart()},lineEnd:function(){var t=this._x,n=this._y,e=t.length-1;if(e>0)for(var r,i=t[0],o=n[0],a=t[e]-i,u=n[e]-o,c=-1;++c<=e;)r=c/e,this._basis.point(this._beta*t[c]+(1-this._beta)*(i+r*a),this._beta*n[c]+(1-this._beta)*(o+r*u));this._x=this._y=null,this._basis.lineEnd()},point:function(t,n){this._x.push(+t),this._y.push(+n)}};var Ix=function t(n){function e(t){return 1===n?new Rx(t):new Ux(t,n)}return e.beta=function(n){return t(+n)},e}(.85);function Ox(t,n,e){t._context.bezierCurveTo(t._x1+t._k*(t._x2-t._x0),t._y1+t._k*(t._y2-t._y0),t._x2+t._k*(t._x1-n),t._y2+t._k*(t._y1-e),t._x2,t._y2)}function Bx(t,n){this._context=t,this._k=(1-n)/6}Bx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._point=0},lineEnd:function(){switch(this._point){case 2:this._context.lineTo(this._x2,this._y2);break;case 3:Ox(this,this._x1,this._y1)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2,this._x1=t,this._y1=n;break;case 2:this._point=3;default:Ox(this,t,n)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Yx=function t(n){function e(t){return new Bx(t,n)}return e.tension=function(n){return t(+n)},e}(0);function Lx(t,n){this._context=t,this._k=(1-n)/6}Lx.prototype={areaStart:$x,areaEnd:$x,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._x5=this._y0=this._y1=this._y2=this._y3=this._y4=this._y5=NaN,this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x3,this._y3),this._context.closePath();break;case 2:this._context.lineTo(this._x3,this._y3),this._context.closePath();break;case 3:this.point(this._x3,this._y3),this.point(this._x4,this._y4),this.point(this._x5,this._y5)}},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._x3=t,this._y3=n;break;case 1:this._point=2,this._context.moveTo(this._x4=t,this._y4=n);break;case 2:this._point=3,this._x5=t,this._y5=n;break;default:Ox(this,t,n)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var jx=function t(n){function e(t){return new Lx(t,n)}return e.tension=function(n){return t(+n)},e}(0);function Hx(t,n){this._context=t,this._k=(1-n)/6}Hx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3,this._line?this._context.lineTo(this._x2,this._y2):this._context.moveTo(this._x2,this._y2);break;case 3:this._point=4;default:Ox(this,t,n)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Xx=function t(n){function e(t){return new Hx(t,n)}return e.tension=function(n){return t(+n)},e}(0);function Gx(t,n,e){var r=t._x1,i=t._y1,o=t._x2,a=t._y2;if(t._l01_a>Mm){var u=2*t._l01_2a+3*t._l01_a*t._l12_a+t._l12_2a,c=3*t._l01_a*(t._l01_a+t._l12_a);r=(r*u-t._x0*t._l12_2a+t._x2*t._l01_2a)/c,i=(i*u-t._y0*t._l12_2a+t._y2*t._l01_2a)/c}if(t._l23_a>Mm){var f=2*t._l23_2a+3*t._l23_a*t._l12_a+t._l12_2a,s=3*t._l23_a*(t._l23_a+t._l12_a);o=(o*f+t._x1*t._l23_2a-n*t._l12_2a)/s,a=(a*f+t._y1*t._l23_2a-e*t._l12_2a)/s}t._context.bezierCurveTo(r,i,o,a,t._x2,t._y2)}function Vx(t,n){this._context=t,this._alpha=n}Vx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){switch(this._point){case 2:this._context.lineTo(this._x2,this._y2);break;case 3:this.point(this._x2,this._y2)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){if(t=+t,n=+n,this._point){var e=this._x2-t,r=this._y2-n;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(e*e+r*r,this._alpha))}switch(this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;break;case 2:this._point=3;default:Gx(this,t,n)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Wx=function t(n){function e(t){return n?new Vx(t,n):new Bx(t,0)}return e.alpha=function(n){return t(+n)},e}(.5);function Zx(t,n){this._context=t,this._alpha=n}Zx.prototype={areaStart:$x,areaEnd:$x,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._x5=this._y0=this._y1=this._y2=this._y3=this._y4=this._y5=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x3,this._y3),this._context.closePath();break;case 2:this._context.lineTo(this._x3,this._y3),this._context.closePath();break;case 3:this.point(this._x3,this._y3),this.point(this._x4,this._y4),this.point(this._x5,this._y5)}},point:function(t,n){if(t=+t,n=+n,this._point){var e=this._x2-t,r=this._y2-n;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(e*e+r*r,this._alpha))}switch(this._point){case 0:this._point=1,this._x3=t,this._y3=n;break;case 1:this._point=2,this._context.moveTo(this._x4=t,this._y4=n);break;case 2:this._point=3,this._x5=t,this._y5=n;break;default:Gx(this,t,n)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Kx=function t(n){function e(t){return n?new Zx(t,n):new Lx(t,0)}return e.alpha=function(n){return t(+n)},e}(.5);function Qx(t,n){this._context=t,this._alpha=n}Qx.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,n){if(t=+t,n=+n,this._point){var e=this._x2-t,r=this._y2-n;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(e*e+r*r,this._alpha))}switch(this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3,this._line?this._context.lineTo(this._x2,this._y2):this._context.moveTo(this._x2,this._y2);break;case 3:this._point=4;default:Gx(this,t,n)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=n}};var Jx=function t(n){function e(t){return n?new Qx(t,n):new Hx(t,0)}return e.alpha=function(n){return t(+n)},e}(.5);function tw(t){this._context=t}function nw(t){return t<0?-1:1}function ew(t,n,e){var r=t._x1-t._x0,i=n-t._x1,o=(t._y1-t._y0)/(r||i<0&&-0),a=(e-t._y1)/(i||r<0&&-0),u=(o*i+a*r)/(r+i);return(nw(o)+nw(a))*Math.min(Math.abs(o),Math.abs(a),.5*Math.abs(u))||0}function rw(t,n){var e=t._x1-t._x0;return e?(3*(t._y1-t._y0)/e-n)/2:n}function iw(t,n,e){var r=t._x0,i=t._y0,o=t._x1,a=t._y1,u=(o-r)/3;t._context.bezierCurveTo(r+u,i+u*n,o-u,a-u*e,o,a)}function ow(t){this._context=t}function aw(t){this._context=new uw(t)}function uw(t){this._context=t}function cw(t){this._context=t}function fw(t){var n,e,r=t.length-1,i=new Array(r),o=new Array(r),a=new Array(r);for(i[0]=0,o[0]=2,a[0]=t[0]+2*t[1],n=1;n=0;--n)i[n]=(a[n]-i[n+1])/o[n];for(o[r-1]=(t[r]+i[r-1])/2,n=0;n1)for(var e,r,i,o=1,a=t[n[0]],u=a.length;o=0;)e[n]=n;return e}function dw(t,n){return t[n]}function pw(t){const n=[];return n.key=t,n}function gw(t){var n=t.map(yw);return hw(t).sort((function(t,e){return n[t]-n[e]}))}function yw(t){for(var n,e=-1,r=0,i=t.length,o=-1/0;++eo&&(o=n,r=e);return r}function vw(t){var n=t.map(_w);return hw(t).sort((function(t,e){return n[t]-n[e]}))}function _w(t){for(var n,e=0,r=-1,i=t.length;++r=0&&(this._t=1-this._t,this._line=1-this._line)},point:function(t,n){switch(t=+t,n=+n,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,n):this._context.moveTo(t,n);break;case 1:this._point=2;default:if(this._t<=0)this._context.lineTo(this._x,n),this._context.lineTo(t,n);else{var e=this._x*(1-this._t)+t*this._t;this._context.lineTo(e,this._y),this._context.lineTo(e,n)}}this._x=t,this._y=n}};var bw=t=>()=>t;function mw(t,{sourceEvent:n,target:e,transform:r,dispatch:i}){Object.defineProperties(this,{type:{value:t,enumerable:!0,configurable:!0},sourceEvent:{value:n,enumerable:!0,configurable:!0},target:{value:e,enumerable:!0,configurable:!0},transform:{value:r,enumerable:!0,configurable:!0},_:{value:i}})}function xw(t,n,e){this.k=t,this.x=n,this.y=e}xw.prototype={constructor:xw,scale:function(t){return 1===t?this:new xw(this.k*t,this.x,this.y)},translate:function(t,n){return 0===t&0===n?this:new xw(this.k,this.x+this.k*t,this.y+this.k*n)},apply:function(t){return[t[0]*this.k+this.x,t[1]*this.k+this.y]},applyX:function(t){return t*this.k+this.x},applyY:function(t){return t*this.k+this.y},invert:function(t){return[(t[0]-this.x)/this.k,(t[1]-this.y)/this.k]},invertX:function(t){return(t-this.x)/this.k},invertY:function(t){return(t-this.y)/this.k},rescaleX:function(t){return t.copy().domain(t.range().map(this.invertX,this).map(t.invert,t))},rescaleY:function(t){return t.copy().domain(t.range().map(this.invertY,this).map(t.invert,t))},toString:function(){return"translate("+this.x+","+this.y+") scale("+this.k+")"}};var ww=new xw(1,0,0);function Mw(t){for(;!t.__zoom;)if(!(t=t.parentNode))return ww;return t.__zoom}function Tw(t){t.stopImmediatePropagation()}function Aw(t){t.preventDefault(),t.stopImmediatePropagation()}function Sw(t){return!(t.ctrlKey&&"wheel"!==t.type||t.button)}function Ew(){var t=this;return t instanceof SVGElement?(t=t.ownerSVGElement||t).hasAttribute("viewBox")?[[(t=t.viewBox.baseVal).x,t.y],[t.x+t.width,t.y+t.height]]:[[0,0],[t.width.baseVal.value,t.height.baseVal.value]]:[[0,0],[t.clientWidth,t.clientHeight]]}function Nw(){return this.__zoom||ww}function kw(t){return-t.deltaY*(1===t.deltaMode?.05:t.deltaMode?1:.002)*(t.ctrlKey?10:1)}function Cw(){return navigator.maxTouchPoints||"ontouchstart"in this}function Pw(t,n,e){var r=t.invertX(n[0][0])-e[0][0],i=t.invertX(n[1][0])-e[1][0],o=t.invertY(n[0][1])-e[0][1],a=t.invertY(n[1][1])-e[1][1];return t.translate(i>r?(r+i)/2:Math.min(0,r)||Math.max(0,i),a>o?(o+a)/2:Math.min(0,o)||Math.max(0,a))}Mw.prototype=xw.prototype,t.Adder=T,t.Delaunay=Lu,t.FormatSpecifier=tf,t.InternMap=InternMap,t.InternSet=InternSet,t.Node=Qd,t.Path=Ua,t.Voronoi=qu,t.ZoomTransform=xw,t.active=function(t,n){var e,r,i=t.__transition;if(i)for(r in n=null==n?null:n+"",i)if((e=i[r]).state>qi&&e.name===n)return new po([[t]],Zo,n,+r);return null},t.arc=function(){var t=km,n=Cm,e=gm(0),r=null,i=Pm,o=zm,a=$m,u=null,c=Nm(f);function f(){var f,s,l=+t.apply(this,arguments),h=+n.apply(this,arguments),d=i.apply(this,arguments)-Am,p=o.apply(this,arguments)-Am,g=ym(p-d),y=p>d;if(u||(u=f=c()),hMm)if(g>Sm-Mm)u.moveTo(h*_m(d),h*xm(d)),u.arc(0,0,h,d,p,!y),l>Mm&&(u.moveTo(l*_m(p),l*xm(p)),u.arc(0,0,l,p,d,y));else{var v,_,b=d,m=p,x=d,w=p,M=g,T=g,A=a.apply(this,arguments)/2,S=A>Mm&&(r?+r.apply(this,arguments):wm(l*l+h*h)),E=mm(ym(h-l)/2,+e.apply(this,arguments)),N=E,k=E;if(S>Mm){var C=Em(S/l*xm(A)),P=Em(S/h*xm(A));(M-=2*C)>Mm?(x+=C*=y?1:-1,w-=C):(M=0,x=w=(d+p)/2),(T-=2*P)>Mm?(b+=P*=y?1:-1,m-=P):(T=0,b=m=(d+p)/2)}var z=h*_m(b),$=h*xm(b),D=l*_m(w),R=l*xm(w);if(E>Mm){var F,q=h*_m(m),U=h*xm(m),I=l*_m(x),O=l*xm(x);if(g1?0:t<-1?Tm:Math.acos(t)}((B*L+Y*j)/(wm(B*B+Y*Y)*wm(L*L+j*j)))/2),X=wm(F[0]*F[0]+F[1]*F[1]);N=mm(E,(l-X)/(H-1)),k=mm(E,(h-X)/(H+1))}else N=k=0}T>Mm?k>Mm?(v=Dm(I,O,z,$,h,k,y),_=Dm(q,U,D,R,h,k,y),u.moveTo(v.cx+v.x01,v.cy+v.y01),kMm&&M>Mm?N>Mm?(v=Dm(D,R,q,U,l,-N,y),_=Dm(z,$,I,O,l,-N,y),u.lineTo(v.cx+v.x01,v.cy+v.y01),N=0))throw new RangeError("invalid r");let e=t.length;if(!((e=Math.floor(e))>=0))throw new RangeError("invalid length");if(!e||!n)return t;const r=y(n),i=t.slice();return r(t,i,0,e,1),r(i,t,0,e,1),r(t,i,0,e,1),t},t.blur2=l,t.blurImage=h,t.brush=function(){return wa(la)},t.brushSelection=function(t){var n=t.__brush;return n?n.dim.output(n.selection):null},t.brushX=function(){return wa(fa)},t.brushY=function(){return wa(sa)},t.buffer=function(t,n){return fetch(t,n).then(_c)},t.chord=function(){return za(!1,!1)},t.chordDirected=function(){return za(!0,!1)},t.chordTranspose=function(){return za(!1,!0)},t.cluster=function(){var t=Ld,n=1,e=1,r=!1;function i(i){var o,a=0;i.eachAfter((function(n){var e=n.children;e?(n.x=function(t){return t.reduce(jd,0)/t.length}(e),n.y=function(t){return 1+t.reduce(Hd,0)}(e)):(n.x=o?a+=t(n,o):0,n.y=0,o=n)}));var u=function(t){for(var n;n=t.children;)t=n[0];return t}(i),c=function(t){for(var n;n=t.children;)t=n[n.length-1];return t}(i),f=u.x-t(u,c)/2,s=c.x+t(c,u)/2;return i.eachAfter(r?function(t){t.x=(t.x-i.x)*n,t.y=(i.y-t.y)*e}:function(t){t.x=(t.x-f)/(s-f)*n,t.y=(1-(i.y?t.y/i.y:1))*e})}return i.separation=function(n){return arguments.length?(t=n,i):t},i.size=function(t){return arguments.length?(r=!1,n=+t[0],e=+t[1],i):r?null:[n,e]},i.nodeSize=function(t){return arguments.length?(r=!0,n=+t[0],e=+t[1],i):r?[n,e]:null},i},t.color=ze,t.contourDensity=function(){var t=fu,n=su,e=lu,r=960,i=500,o=20,a=2,u=3*o,c=r+2*u>>a,f=i+2*u>>a,s=Qa(20);function h(r){var i=new Float32Array(c*f),s=Math.pow(2,-a),h=-1;for(const o of r){var d=(t(o,++h,r)+u)*s,p=(n(o,h,r)+u)*s,g=+e(o,h,r);if(g&&d>=0&&d=0&&pt*r)))(n).map(((t,n)=>(t.value=+e[n],p(t))))}function p(t){return t.coordinates.forEach(g),t}function g(t){t.forEach(y)}function y(t){t.forEach(v)}function v(t){t[0]=t[0]*Math.pow(2,a)-u,t[1]=t[1]*Math.pow(2,a)-u}function _(){return c=r+2*(u=3*o)>>a,f=i+2*u>>a,d}return d.contours=function(t){var n=h(t),e=iu().size([c,f]),r=Math.pow(2,2*a),i=t=>{t=+t;var i=p(e.contour(n,t*r));return i.value=t,i};return Object.defineProperty(i,"max",{get:()=>J(n)/r}),i},d.x=function(n){return arguments.length?(t="function"==typeof n?n:Qa(+n),d):t},d.y=function(t){return arguments.length?(n="function"==typeof t?t:Qa(+t),d):n},d.weight=function(t){return arguments.length?(e="function"==typeof t?t:Qa(+t),d):e},d.size=function(t){if(!arguments.length)return[r,i];var n=+t[0],e=+t[1];if(!(n>=0&&e>=0))throw new Error("invalid size");return r=n,i=e,_()},d.cellSize=function(t){if(!arguments.length)return 1<=1))throw new Error("invalid cell size");return a=Math.floor(Math.log(t)/Math.LN2),_()},d.thresholds=function(t){return arguments.length?(s="function"==typeof t?t:Array.isArray(t)?Qa(Za.call(t)):Qa(t),d):s},d.bandwidth=function(t){if(!arguments.length)return Math.sqrt(o*(o+1));if(!((t=+t)>=0))throw new Error("invalid bandwidth");return o=(Math.sqrt(4*t*t+1)-1)/2,_()},d},t.contours=iu,t.count=v,t.create=function(t){return Zn(Yt(t).call(document.documentElement))},t.creator=Yt,t.cross=function(...t){const n="function"==typeof t[t.length-1]&&function(t){return n=>t(...n)}(t.pop()),e=(t=t.map(m)).map(_),r=t.length-1,i=new Array(r+1).fill(0),o=[];if(r<0||e.some(b))return o;for(;;){o.push(i.map(((n,e)=>t[e][n])));let a=r;for(;++i[a]===e[a];){if(0===a)return n?o.map(n):o;i[a--]=0}}},t.csv=wc,t.csvFormat=rc,t.csvFormatBody=ic,t.csvFormatRow=ac,t.csvFormatRows=oc,t.csvFormatValue=uc,t.csvParse=nc,t.csvParseRows=ec,t.cubehelix=Tr,t.cumsum=function(t,n){var e=0,r=0;return Float64Array.from(t,void 0===n?t=>e+=+t||0:i=>e+=+n(i,r++,t)||0)},t.curveBasis=function(t){return new Rx(t)},t.curveBasisClosed=function(t){return new Fx(t)},t.curveBasisOpen=function(t){return new qx(t)},t.curveBumpX=tx,t.curveBumpY=nx,t.curveBundle=Ix,t.curveCardinal=Yx,t.curveCardinalClosed=jx,t.curveCardinalOpen=Xx,t.curveCatmullRom=Wx,t.curveCatmullRomClosed=Kx,t.curveCatmullRomOpen=Jx,t.curveLinear=Um,t.curveLinearClosed=function(t){return new tw(t)},t.curveMonotoneX=function(t){return new ow(t)},t.curveMonotoneY=function(t){return new aw(t)},t.curveNatural=function(t){return new cw(t)},t.curveStep=function(t){return new sw(t,.5)},t.curveStepAfter=function(t){return new sw(t,1)},t.curveStepBefore=function(t){return new sw(t,0)},t.descending=e,t.deviation=w,t.difference=function(t,...n){t=new InternSet(t);for(const e of n)for(const n of e)t.delete(n);return t},t.disjoint=function(t,n){const e=n[Symbol.iterator](),r=new InternSet;for(const n of t){if(r.has(n))return!1;let t,i;for(;({value:t,done:i}=e.next())&&!i;){if(Object.is(n,t))return!1;r.add(t)}}return!0},t.dispatch=$t,t.drag=function(){var t,n,e,r,i=se,o=le,a=he,u=de,c={},f=$t("start","drag","end"),s=0,l=0;function h(t){t.on("mousedown.drag",d).filter(u).on("touchstart.drag",y).on("touchmove.drag",v,ee).on("touchend.drag touchcancel.drag",_).style("touch-action","none").style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function d(a,u){if(!r&&i.call(this,a,u)){var c=b(this,o.call(this,a,u),a,u,"mouse");c&&(Zn(a.view).on("mousemove.drag",p,re).on("mouseup.drag",g,re),ae(a.view),ie(a),e=!1,t=a.clientX,n=a.clientY,c("start",a))}}function p(r){if(oe(r),!e){var i=r.clientX-t,o=r.clientY-n;e=i*i+o*o>l}c.mouse("drag",r)}function g(t){Zn(t.view).on("mousemove.drag mouseup.drag",null),ue(t.view,e),oe(t),c.mouse("end",t)}function y(t,n){if(i.call(this,t,n)){var e,r,a=t.changedTouches,u=o.call(this,t,n),c=a.length;for(e=0;e+t,t.easePoly=wo,t.easePolyIn=mo,t.easePolyInOut=wo,t.easePolyOut=xo,t.easeQuad=_o,t.easeQuadIn=function(t){return t*t},t.easeQuadInOut=_o,t.easeQuadOut=function(t){return t*(2-t)},t.easeSin=Ao,t.easeSinIn=function(t){return 1==+t?1:1-Math.cos(t*To)},t.easeSinInOut=Ao,t.easeSinOut=function(t){return Math.sin(t*To)},t.every=function(t,n){if("function"!=typeof n)throw new TypeError("test is not a function");let e=-1;for(const r of t)if(!n(r,++e,t))return!1;return!0},t.extent=M,t.fcumsum=function(t,n){const e=new T;let r=-1;return Float64Array.from(t,void 0===n?t=>e.add(+t||0):i=>e.add(+n(i,++r,t)||0))},t.filter=function(t,n){if("function"!=typeof n)throw new TypeError("test is not a function");const e=[];let r=-1;for(const i of t)n(i,++r,t)&&e.push(i);return e},t.flatGroup=function(t,...n){return z(P(t,...n),n)},t.flatRollup=function(t,n,...e){return z(D(t,n,...e),e)},t.forceCenter=function(t,n){var e,r=1;function i(){var i,o,a=e.length,u=0,c=0;for(i=0;if+p||os+p||ac.index){var g=f-u.x-u.vx,y=s-u.y-u.vy,v=g*g+y*y;vt.r&&(t.r=t[n].r)}function c(){if(n){var r,i,o=n.length;for(e=new Array(o),r=0;r[u(t,n,r),t])));for(a=0,i=new Array(f);a=u)){(t.data!==n||t.next)&&(0===l&&(p+=(l=Uc(e))*l),0===h&&(p+=(h=Uc(e))*h),p(t=(Lc*t+jc)%Hc)/Hc}();function l(){h(),f.call("tick",n),e1?(null==e?u.delete(t):u.set(t,p(e)),n):u.get(t)},find:function(n,e,r){var i,o,a,u,c,f=0,s=t.length;for(null==r?r=1/0:r*=r,f=0;f1?(f.on(t,e),n):f.on(t)}}},t.forceX=function(t){var n,e,r,i=qc(.1);function o(t){for(var i,o=0,a=n.length;o=.12&&i<.234&&r>=-.425&&r<-.214?u:i>=.166&&i<.234&&r>=-.214&&r<-.115?c:a).invert(t)},s.stream=function(e){return t&&n===e?t:(r=[a.stream(n=e),u.stream(e),c.stream(e)],i=r.length,t={point:function(t,n){for(var e=-1;++ejs(r[0],r[1])&&(r[1]=i[1]),js(i[0],r[1])>js(r[0],r[1])&&(r[0]=i[0])):o.push(r=i);for(a=-1/0,n=0,r=o[e=o.length-1];n<=e;r=i,++n)i=o[n],(u=js(r[1],i[0]))>a&&(a=u,Wf=i[0],Kf=r[1])}return is=os=null,Wf===1/0||Zf===1/0?[[NaN,NaN],[NaN,NaN]]:[[Wf,Zf],[Kf,Qf]]},t.geoCentroid=function(t){ms=xs=ws=Ms=Ts=As=Ss=Es=0,Ns=new T,ks=new T,Cs=new T,Lf(t,Gs);var n=+Ns,e=+ks,r=+Cs,i=Ef(n,e,r);return i=0))throw new RangeError(`invalid digits: ${t}`);i=n}return null===n&&(r=new ed(i)),a},a.projection(t).digits(i).context(n)},t.geoProjection=yd,t.geoProjectionMutator=vd,t.geoRotation=ll,t.geoStereographic=function(){return yd(Bd).scale(250).clipAngle(142)},t.geoStereographicRaw=Bd,t.geoStream=Lf,t.geoTransform=function(t){return{stream:id(t)}},t.geoTransverseMercator=function(){var t=Ed(Yd),n=t.center,e=t.rotate;return t.center=function(t){return arguments.length?n([-t[1],t[0]]):[(t=n())[1],-t[0]]},t.rotate=function(t){return arguments.length?e([t[0],t[1],t.length>2?t[2]+90:90]):[(t=e())[0],t[1],t[2]-90]},e([0,0,90]).scale(159.155)},t.geoTransverseMercatorRaw=Yd,t.gray=function(t,n){return new ur(t,0,0,null==n?1:n)},t.greatest=ot,t.greatestIndex=function(t,e=n){if(1===e.length)return tt(t,e);let r,i=-1,o=-1;for(const n of t)++o,(i<0?0===e(n,n):e(n,r)>0)&&(r=n,i=o);return i},t.group=C,t.groupSort=function(t,e,r){return(2!==e.length?U($(t,e,r),(([t,e],[r,i])=>n(e,i)||n(t,r))):U(C(t,r),(([t,r],[i,o])=>e(r,o)||n(t,i)))).map((([t])=>t))},t.groups=P,t.hcl=dr,t.hierarchy=Gd,t.histogram=Q,t.hsl=He,t.html=Ec,t.image=function(t,n){return new Promise((function(e,r){var i=new Image;for(var o in n)i[o]=n[o];i.onerror=r,i.onload=function(){e(i)},i.src=t}))},t.index=function(t,...n){return F(t,k,R,n)},t.indexes=function(t,...n){return F(t,Array.from,R,n)},t.interpolate=Gr,t.interpolateArray=function(t,n){return(Ir(n)?Ur:Or)(t,n)},t.interpolateBasis=Er,t.interpolateBasisClosed=Nr,t.interpolateBlues=Xb,t.interpolateBrBG=ib,t.interpolateBuGn=wb,t.interpolateBuPu=Tb,t.interpolateCividis=function(t){return t=Math.max(0,Math.min(1,t)),"rgb("+Math.max(0,Math.min(255,Math.round(-4.54-t*(35.34-t*(2381.73-t*(6402.7-t*(7024.72-2710.57*t)))))))+", "+Math.max(0,Math.min(255,Math.round(32.49+t*(170.73+t*(52.82-t*(131.46-t*(176.58-67.37*t)))))))+", "+Math.max(0,Math.min(255,Math.round(81.24+t*(442.36-t*(2482.43-t*(6167.24-t*(6614.94-2475.67*t)))))))+")"},t.interpolateCool=om,t.interpolateCubehelix=li,t.interpolateCubehelixDefault=rm,t.interpolateCubehelixLong=hi,t.interpolateDate=Br,t.interpolateDiscrete=function(t){var n=t.length;return function(e){return t[Math.max(0,Math.min(n-1,Math.floor(e*n)))]}},t.interpolateGnBu=Sb,t.interpolateGreens=Vb,t.interpolateGreys=Zb,t.interpolateHcl=ci,t.interpolateHclLong=fi,t.interpolateHsl=oi,t.interpolateHslLong=ai,t.interpolateHue=function(t,n){var e=Pr(+t,+n);return function(t){var n=e(t);return n-360*Math.floor(n/360)}},t.interpolateInferno=dm,t.interpolateLab=function(t,n){var e=$r((t=ar(t)).l,(n=ar(n)).l),r=$r(t.a,n.a),i=$r(t.b,n.b),o=$r(t.opacity,n.opacity);return function(n){return t.l=e(n),t.a=r(n),t.b=i(n),t.opacity=o(n),t+""}},t.interpolateMagma=hm,t.interpolateNumber=Yr,t.interpolateNumberArray=Ur,t.interpolateObject=Lr,t.interpolateOrRd=Nb,t.interpolateOranges=em,t.interpolatePRGn=ab,t.interpolatePiYG=cb,t.interpolatePlasma=pm,t.interpolatePuBu=zb,t.interpolatePuBuGn=Cb,t.interpolatePuOr=sb,t.interpolatePuRd=Db,t.interpolatePurples=Qb,t.interpolateRainbow=function(t){(t<0||t>1)&&(t-=Math.floor(t));var n=Math.abs(t-.5);return am.h=360*t-100,am.s=1.5-1.5*n,am.l=.8-.9*n,am+""},t.interpolateRdBu=hb,t.interpolateRdGy=pb,t.interpolateRdPu=Fb,t.interpolateRdYlBu=yb,t.interpolateRdYlGn=_b,t.interpolateReds=tm,t.interpolateRgb=Dr,t.interpolateRgbBasis=Fr,t.interpolateRgbBasisClosed=qr,t.interpolateRound=Vr,t.interpolateSinebow=function(t){var n;return t=(.5-t)*Math.PI,um.r=255*(n=Math.sin(t))*n,um.g=255*(n=Math.sin(t+cm))*n,um.b=255*(n=Math.sin(t+fm))*n,um+""},t.interpolateSpectral=mb,t.interpolateString=Xr,t.interpolateTransformCss=ti,t.interpolateTransformSvg=ni,t.interpolateTurbo=function(t){return t=Math.max(0,Math.min(1,t)),"rgb("+Math.max(0,Math.min(255,Math.round(34.61+t*(1172.33-t*(10793.56-t*(33300.12-t*(38394.49-14825.05*t)))))))+", "+Math.max(0,Math.min(255,Math.round(23.31+t*(557.33+t*(1225.33-t*(3574.96-t*(1073.77+707.56*t)))))))+", "+Math.max(0,Math.min(255,Math.round(27.2+t*(3211.1-t*(15327.97-t*(27814-t*(22569.18-6838.66*t)))))))+")"},t.interpolateViridis=lm,t.interpolateWarm=im,t.interpolateYlGn=Ob,t.interpolateYlGnBu=Ub,t.interpolateYlOrBr=Yb,t.interpolateYlOrRd=jb,t.interpolateZoom=ri,t.interrupt=Gi,t.intersection=function(t,...n){t=new InternSet(t),n=n.map(vt);t:for(const e of t)for(const r of n)if(!r.has(e)){t.delete(e);continue t}return t},t.interval=function(t,n,e){var r=new Ei,i=n;return null==n?(r.restart(t,n,e),r):(r._restart=r.restart,r.restart=function(t,n,e){n=+n,e=null==e?Ai():+e,r._restart((function o(a){a+=i,r._restart(o,i+=n,e),t(a)}),n,e)},r.restart(t,n,e),r)},t.isoFormat=D_,t.isoParse=F_,t.json=function(t,n){return fetch(t,n).then(Tc)},t.lab=ar,t.lch=function(t,n,e,r){return 1===arguments.length?hr(t):new pr(e,n,t,null==r?1:r)},t.least=function(t,e=n){let r,i=!1;if(1===e.length){let o;for(const a of t){const t=e(a);(i?n(t,o)<0:0===n(t,t))&&(r=a,o=t,i=!0)}}else for(const n of t)(i?e(n,r)<0:0===e(n,n))&&(r=n,i=!0);return r},t.leastIndex=ht,t.line=Bm,t.lineRadial=Wm,t.link=ox,t.linkHorizontal=function(){return ox(tx)},t.linkRadial=function(){const t=ox(ex);return t.angle=t.x,delete t.x,t.radius=t.y,delete t.y,t},t.linkVertical=function(){return ox(nx)},t.local=Qn,t.map=function(t,n){if("function"!=typeof t[Symbol.iterator])throw new TypeError("values is not iterable");if("function"!=typeof n)throw new TypeError("mapper is not a function");return Array.from(t,((e,r)=>n(e,r,t)))},t.matcher=Vt,t.max=J,t.maxIndex=tt,t.mean=function(t,n){let e=0,r=0;if(void 0===n)for(let n of t)null!=n&&(n=+n)>=n&&(++e,r+=n);else{let i=-1;for(let o of t)null!=(o=n(o,++i,t))&&(o=+o)>=o&&(++e,r+=o)}if(e)return r/e},t.median=function(t,n){return at(t,.5,n)},t.medianIndex=function(t,n){return ct(t,.5,n)},t.merge=ft,t.min=nt,t.minIndex=et,t.mode=function(t,n){const e=new InternMap;if(void 0===n)for(let n of t)null!=n&&n>=n&&e.set(n,(e.get(n)||0)+1);else{let r=-1;for(let i of t)null!=(i=n(i,++r,t))&&i>=i&&e.set(i,(e.get(i)||0)+1)}let r,i=0;for(const[t,n]of e)n>i&&(i=n,r=t);return r},t.namespace=It,t.namespaces=Ut,t.nice=Z,t.now=Ai,t.pack=function(){var t=null,n=1,e=1,r=np;function i(i){const o=ap();return i.x=n/2,i.y=e/2,t?i.eachBefore(xp(t)).eachAfter(wp(r,.5,o)).eachBefore(Mp(1)):i.eachBefore(xp(mp)).eachAfter(wp(np,1,o)).eachAfter(wp(r,i.r/Math.min(n,e),o)).eachBefore(Mp(Math.min(n,e)/(2*i.r))),i}return i.radius=function(n){return arguments.length?(t=Jd(n),i):t},i.size=function(t){return arguments.length?(n=+t[0],e=+t[1],i):[n,e]},i.padding=function(t){return arguments.length?(r="function"==typeof t?t:ep(+t),i):r},i},t.packEnclose=function(t){return up(t,ap())},t.packSiblings=function(t){return bp(t,ap()),t},t.pairs=function(t,n=st){const e=[];let r,i=!1;for(const o of t)i&&e.push(n(r,o)),r=o,i=!0;return e},t.partition=function(){var t=1,n=1,e=0,r=!1;function i(i){var o=i.height+1;return i.x0=i.y0=e,i.x1=t,i.y1=n/o,i.eachBefore(function(t,n){return function(r){r.children&&Ap(r,r.x0,t*(r.depth+1)/n,r.x1,t*(r.depth+2)/n);var i=r.x0,o=r.y0,a=r.x1-e,u=r.y1-e;a0&&(d+=l);for(null!=n?p.sort((function(t,e){return n(g[t],g[e])})):null!=e&&p.sort((function(t,n){return e(a[t],a[n])})),u=0,f=d?(v-h*b)/d:0;u0?l*f:0)+b,g[c]={data:a[c],index:u,value:l,startAngle:y,endAngle:s,padAngle:_};return g}return a.value=function(n){return arguments.length?(t="function"==typeof n?n:gm(+n),a):t},a.sortValues=function(t){return arguments.length?(n=t,e=null,a):n},a.sort=function(t){return arguments.length?(e=t,n=null,a):e},a.startAngle=function(t){return arguments.length?(r="function"==typeof t?t:gm(+t),a):r},a.endAngle=function(t){return arguments.length?(i="function"==typeof t?t:gm(+t),a):i},a.padAngle=function(t){return arguments.length?(o="function"==typeof t?t:gm(+t),a):o},a},t.piecewise=di,t.pointRadial=Km,t.pointer=ne,t.pointers=function(t,n){return t.target&&(t=te(t),void 0===n&&(n=t.currentTarget),t=t.touches||[t]),Array.from(t,(t=>ne(t,n)))},t.polygonArea=function(t){for(var n,e=-1,r=t.length,i=t[r-1],o=0;++eu!=f>u&&a<(c-e)*(u-r)/(f-r)+e&&(s=!s),c=e,f=r;return s},t.polygonHull=function(t){if((e=t.length)<3)return null;var n,e,r=new Array(e),i=new Array(e);for(n=0;n=0;--n)f.push(t[r[o[n]][2]]);for(n=+u;n(n=1664525*n+1013904223|0,lg*(n>>>0))},t.randomLogNormal=Kp,t.randomLogistic=fg,t.randomNormal=Zp,t.randomPareto=ng,t.randomPoisson=sg,t.randomUniform=Vp,t.randomWeibull=ug,t.range=lt,t.rank=function(t,e=n){if("function"!=typeof t[Symbol.iterator])throw new TypeError("values is not iterable");let r=Array.from(t);const i=new Float64Array(r.length);2!==e.length&&(r=r.map(e),e=n);const o=(t,n)=>e(r[t],r[n]);let a,u;return(t=Uint32Array.from(r,((t,n)=>n))).sort(e===n?(t,n)=>O(r[t],r[n]):I(o)),t.forEach(((t,n)=>{const e=o(t,void 0===a?t:a);e>=0?((void 0===a||e>0)&&(a=t,u=n),i[t]=u):i[t]=NaN})),i},t.reduce=function(t,n,e){if("function"!=typeof n)throw new TypeError("reducer is not a function");const r=t[Symbol.iterator]();let i,o,a=-1;if(arguments.length<3){if(({done:i,value:e}=r.next()),i)return;++a}for(;({done:i,value:o}=r.next()),!i;)e=n(e,o,++a,t);return e},t.reverse=function(t){if("function"!=typeof t[Symbol.iterator])throw new TypeError("values is not iterable");return Array.from(t).reverse()},t.rgb=Fe,t.ribbon=function(){return Wa()},t.ribbonArrow=function(){return Wa(Va)},t.rollup=$,t.rollups=D,t.scaleBand=yg,t.scaleDiverging=function t(){var n=Ng(L_()(mg));return n.copy=function(){return B_(n,t())},dg.apply(n,arguments)},t.scaleDivergingLog=function t(){var n=Fg(L_()).domain([.1,1,10]);return n.copy=function(){return B_(n,t()).base(n.base())},dg.apply(n,arguments)},t.scaleDivergingPow=j_,t.scaleDivergingSqrt=function(){return j_.apply(null,arguments).exponent(.5)},t.scaleDivergingSymlog=function t(){var n=Ig(L_());return n.copy=function(){return B_(n,t()).constant(n.constant())},dg.apply(n,arguments)},t.scaleIdentity=function t(n){var e;function r(t){return null==t||isNaN(t=+t)?e:t}return r.invert=r,r.domain=r.range=function(t){return arguments.length?(n=Array.from(t,_g),r):n.slice()},r.unknown=function(t){return arguments.length?(e=t,r):e},r.copy=function(){return t(n).unknown(e)},n=arguments.length?Array.from(n,_g):[0,1],Ng(r)},t.scaleImplicit=pg,t.scaleLinear=function t(){var n=Sg();return n.copy=function(){return Tg(n,t())},hg.apply(n,arguments),Ng(n)},t.scaleLog=function t(){const n=Fg(Ag()).domain([1,10]);return n.copy=()=>Tg(n,t()).base(n.base()),hg.apply(n,arguments),n},t.scaleOrdinal=gg,t.scalePoint=function(){return vg(yg.apply(null,arguments).paddingInner(1))},t.scalePow=jg,t.scaleQuantile=function t(){var e,r=[],i=[],o=[];function a(){var t=0,n=Math.max(1,i.length);for(o=new Array(n-1);++t0?o[n-1]:r[0],n=i?[o[i-1],r]:[o[n-1],o[n]]},u.unknown=function(t){return arguments.length?(n=t,u):u},u.thresholds=function(){return o.slice()},u.copy=function(){return t().domain([e,r]).range(a).unknown(n)},hg.apply(Ng(u),arguments)},t.scaleRadial=function t(){var n,e=Sg(),r=[0,1],i=!1;function o(t){var r=function(t){return Math.sign(t)*Math.sqrt(Math.abs(t))}(e(t));return isNaN(r)?n:i?Math.round(r):r}return o.invert=function(t){return e.invert(Hg(t))},o.domain=function(t){return arguments.length?(e.domain(t),o):e.domain()},o.range=function(t){return arguments.length?(e.range((r=Array.from(t,_g)).map(Hg)),o):r.slice()},o.rangeRound=function(t){return o.range(t).round(!0)},o.round=function(t){return arguments.length?(i=!!t,o):i},o.clamp=function(t){return arguments.length?(e.clamp(t),o):e.clamp()},o.unknown=function(t){return arguments.length?(n=t,o):n},o.copy=function(){return t(e.domain(),r).round(i).clamp(e.clamp()).unknown(n)},hg.apply(o,arguments),Ng(o)},t.scaleSequential=function t(){var n=Ng(O_()(mg));return n.copy=function(){return B_(n,t())},dg.apply(n,arguments)},t.scaleSequentialLog=function t(){var n=Fg(O_()).domain([1,10]);return n.copy=function(){return B_(n,t()).base(n.base())},dg.apply(n,arguments)},t.scaleSequentialPow=Y_,t.scaleSequentialQuantile=function t(){var e=[],r=mg;function i(t){if(null!=t&&!isNaN(t=+t))return r((s(e,t,1)-1)/(e.length-1))}return i.domain=function(t){if(!arguments.length)return e.slice();e=[];for(let n of t)null==n||isNaN(n=+n)||e.push(n);return e.sort(n),i},i.interpolator=function(t){return arguments.length?(r=t,i):r},i.range=function(){return e.map(((t,n)=>r(n/(e.length-1))))},i.quantiles=function(t){return Array.from({length:t+1},((n,r)=>at(e,r/t)))},i.copy=function(){return t(r).domain(e)},dg.apply(i,arguments)},t.scaleSequentialSqrt=function(){return Y_.apply(null,arguments).exponent(.5)},t.scaleSequentialSymlog=function t(){var n=Ig(O_());return n.copy=function(){return B_(n,t()).constant(n.constant())},dg.apply(n,arguments)},t.scaleSqrt=function(){return jg.apply(null,arguments).exponent(.5)},t.scaleSymlog=function t(){var n=Ig(Ag());return n.copy=function(){return Tg(n,t()).constant(n.constant())},hg.apply(n,arguments)},t.scaleThreshold=function t(){var n,e=[.5],r=[0,1],i=1;function o(t){return null!=t&&t<=t?r[s(e,t,0,i)]:n}return o.domain=function(t){return arguments.length?(e=Array.from(t),i=Math.min(e.length,r.length-1),o):e.slice()},o.range=function(t){return arguments.length?(r=Array.from(t),i=Math.min(e.length,r.length-1),o):r.slice()},o.invertExtent=function(t){var n=r.indexOf(t);return[e[n-1],e[n]]},o.unknown=function(t){return arguments.length?(n=t,o):n},o.copy=function(){return t().domain(e).range(r).unknown(n)},hg.apply(o,arguments)},t.scaleTime=function(){return hg.apply(I_(uv,cv,tv,Zy,xy,py,sy,ay,iy,t.timeFormat).domain([new Date(2e3,0,1),new Date(2e3,0,2)]),arguments)},t.scaleUtc=function(){return hg.apply(I_(ov,av,ev,Qy,Fy,yy,hy,cy,iy,t.utcFormat).domain([Date.UTC(2e3,0,1),Date.UTC(2e3,0,2)]),arguments)},t.scan=function(t,n){const e=ht(t,n);return e<0?void 0:e},t.schemeAccent=G_,t.schemeBlues=Hb,t.schemeBrBG=rb,t.schemeBuGn=xb,t.schemeBuPu=Mb,t.schemeCategory10=X_,t.schemeDark2=V_,t.schemeGnBu=Ab,t.schemeGreens=Gb,t.schemeGreys=Wb,t.schemeOrRd=Eb,t.schemeOranges=nm,t.schemePRGn=ob,t.schemePaired=W_,t.schemePastel1=Z_,t.schemePastel2=K_,t.schemePiYG=ub,t.schemePuBu=Pb,t.schemePuBuGn=kb,t.schemePuOr=fb,t.schemePuRd=$b,t.schemePurples=Kb,t.schemeRdBu=lb,t.schemeRdGy=db,t.schemeRdPu=Rb,t.schemeRdYlBu=gb,t.schemeRdYlGn=vb,t.schemeReds=Jb,t.schemeSet1=Q_,t.schemeSet2=J_,t.schemeSet3=tb,t.schemeSpectral=bb,t.schemeTableau10=nb,t.schemeYlGn=Ib,t.schemeYlGnBu=qb,t.schemeYlOrBr=Bb,t.schemeYlOrRd=Lb,t.select=Zn,t.selectAll=function(t){return"string"==typeof t?new Vn([document.querySelectorAll(t)],[document.documentElement]):new Vn([Ht(t)],Gn)},t.selection=Wn,t.selector=jt,t.selectorAll=Gt,t.shuffle=dt,t.shuffler=pt,t.some=function(t,n){if("function"!=typeof n)throw new TypeError("test is not a function");let e=-1;for(const r of t)if(n(r,++e,t))return!0;return!1},t.sort=U,t.stack=function(){var t=gm([]),n=hw,e=lw,r=dw;function i(i){var o,a,u=Array.from(t.apply(this,arguments),pw),c=u.length,f=-1;for(const t of i)for(o=0,++f;o0)for(var e,r,i,o,a,u,c=0,f=t[n[0]].length;c0?(r[0]=o,r[1]=o+=i):i<0?(r[1]=a,r[0]=a+=i):(r[0]=0,r[1]=i)},t.stackOffsetExpand=function(t,n){if((r=t.length)>0){for(var e,r,i,o=0,a=t[0].length;o0){for(var e,r=0,i=t[n[0]],o=i.length;r0&&(r=(e=t[n[0]]).length)>0){for(var e,r,i,o=0,a=1;afunction(t){t=`${t}`;let n=t.length;zp(t,n-1)&&!zp(t,n-2)&&(t=t.slice(0,-1));return"/"===t[0]?t:`/${t}`}(t(n,e,r)))),e=n.map(Pp),i=new Set(n).add("");for(const t of e)i.has(t)||(i.add(t),n.push(t),e.push(Pp(t)),h.push(Np));d=(t,e)=>n[e],p=(t,n)=>e[n]}for(a=0,i=h.length;a=0&&(f=h[t]).data===Np;--t)f.data=null}if(u.parent=Sp,u.eachBefore((function(t){t.depth=t.parent.depth+1,--i})).eachBefore(Kd),u.parent=null,i>0)throw new Error("cycle");return u}return r.id=function(t){return arguments.length?(n=Jd(t),r):n},r.parentId=function(t){return arguments.length?(e=Jd(t),r):e},r.path=function(n){return arguments.length?(t=Jd(n),r):t},r},t.style=_n,t.subset=function(t,n){return _t(n,t)},t.sum=function(t,n){let e=0;if(void 0===n)for(let n of t)(n=+n)&&(e+=n);else{let r=-1;for(let i of t)(i=+n(i,++r,t))&&(e+=i)}return e},t.superset=_t,t.svg=Nc,t.symbol=function(t,n){let e=null,r=Nm(i);function i(){let i;if(e||(e=i=r()),t.apply(this,arguments).draw(e,+n.apply(this,arguments)),i)return e=null,i+""||null}return t="function"==typeof t?t:gm(t||cx),n="function"==typeof n?n:gm(void 0===n?64:+n),i.type=function(n){return arguments.length?(t="function"==typeof n?n:gm(n),i):t},i.size=function(t){return arguments.length?(n="function"==typeof t?t:gm(+t),i):n},i.context=function(t){return arguments.length?(e=null==t?null:t,i):e},i},t.symbolAsterisk=ux,t.symbolCircle=cx,t.symbolCross=fx,t.symbolDiamond=hx,t.symbolDiamond2=dx,t.symbolPlus=px,t.symbolSquare=gx,t.symbolSquare2=yx,t.symbolStar=mx,t.symbolTimes=Cx,t.symbolTriangle=wx,t.symbolTriangle2=Tx,t.symbolWye=kx,t.symbolX=Cx,t.symbols=Px,t.symbolsFill=Px,t.symbolsStroke=zx,t.text=mc,t.thresholdFreedmanDiaconis=function(t,n,e){const r=v(t),i=at(t,.75)-at(t,.25);return r&&i?Math.ceil((e-n)/(2*i*Math.pow(r,-1/3))):1},t.thresholdScott=function(t,n,e){const r=v(t),i=w(t);return r&&i?Math.ceil((e-n)*Math.cbrt(r)/(3.49*i)):1},t.thresholdSturges=K,t.tickFormat=Eg,t.tickIncrement=V,t.tickStep=W,t.ticks=G,t.timeDay=py,t.timeDays=gy,t.timeFormatDefaultLocale=P_,t.timeFormatLocale=hv,t.timeFriday=Sy,t.timeFridays=$y,t.timeHour=sy,t.timeHours=ly,t.timeInterval=Vg,t.timeMillisecond=Wg,t.timeMilliseconds=Zg,t.timeMinute=ay,t.timeMinutes=uy,t.timeMonday=wy,t.timeMondays=ky,t.timeMonth=Zy,t.timeMonths=Ky,t.timeSaturday=Ey,t.timeSaturdays=Dy,t.timeSecond=iy,t.timeSeconds=oy,t.timeSunday=xy,t.timeSundays=Ny,t.timeThursday=Ay,t.timeThursdays=zy,t.timeTickInterval=cv,t.timeTicks=uv,t.timeTuesday=My,t.timeTuesdays=Cy,t.timeWednesday=Ty,t.timeWednesdays=Py,t.timeWeek=xy,t.timeWeeks=Ny,t.timeYear=tv,t.timeYears=nv,t.timeout=$i,t.timer=Ni,t.timerFlush=ki,t.transition=go,t.transpose=gt,t.tree=function(){var t=$p,n=1,e=1,r=null;function i(i){var c=function(t){for(var n,e,r,i,o,a=new Up(t,0),u=[a];n=u.pop();)if(r=n._.children)for(n.children=new Array(o=r.length),i=o-1;i>=0;--i)u.push(e=n.children[i]=new Up(r[i],i)),e.parent=n;return(a.parent=new Up(null,0)).children=[a],a}(i);if(c.eachAfter(o),c.parent.m=-c.z,c.eachBefore(a),r)i.eachBefore(u);else{var f=i,s=i,l=i;i.eachBefore((function(t){t.xs.x&&(s=t),t.depth>l.depth&&(l=t)}));var h=f===s?1:t(f,s)/2,d=h-f.x,p=n/(s.x+h+d),g=e/(l.depth||1);i.eachBefore((function(t){t.x=(t.x+d)*p,t.y=t.depth*g}))}return i}function o(n){var e=n.children,r=n.parent.children,i=n.i?r[n.i-1]:null;if(e){!function(t){for(var n,e=0,r=0,i=t.children,o=i.length;--o>=0;)(n=i[o]).z+=e,n.m+=e,e+=n.s+(r+=n.c)}(n);var o=(e[0].z+e[e.length-1].z)/2;i?(n.z=i.z+t(n._,i._),n.m=n.z-o):n.z=o}else i&&(n.z=i.z+t(n._,i._));n.parent.A=function(n,e,r){if(e){for(var i,o=n,a=n,u=e,c=o.parent.children[0],f=o.m,s=a.m,l=u.m,h=c.m;u=Rp(u),o=Dp(o),u&&o;)c=Dp(c),(a=Rp(a)).a=n,(i=u.z+l-o.z-f+t(u._,o._))>0&&(Fp(qp(u,n,r),n,i),f+=i,s+=i),l+=u.m,f+=o.m,h+=c.m,s+=a.m;u&&!Rp(a)&&(a.t=u,a.m+=l-s),o&&!Dp(c)&&(c.t=o,c.m+=f-h,r=n)}return r}(n,i,n.parent.A||r[0])}function a(t){t._.x=t.z+t.parent.m,t.m+=t.parent.m}function u(t){t.x*=n,t.y=t.depth*e}return i.separation=function(n){return arguments.length?(t=n,i):t},i.size=function(t){return arguments.length?(r=!1,n=+t[0],e=+t[1],i):r?null:[n,e]},i.nodeSize=function(t){return arguments.length?(r=!0,n=+t[0],e=+t[1],i):r?[n,e]:null},i},t.treemap=function(){var t=Yp,n=!1,e=1,r=1,i=[0],o=np,a=np,u=np,c=np,f=np;function s(t){return t.x0=t.y0=0,t.x1=e,t.y1=r,t.eachBefore(l),i=[0],n&&t.eachBefore(Tp),t}function l(n){var e=i[n.depth],r=n.x0+e,s=n.y0+e,l=n.x1-e,h=n.y1-e;l=e-1){var s=u[n];return s.x0=i,s.y0=o,s.x1=a,void(s.y1=c)}var l=f[n],h=r/2+l,d=n+1,p=e-1;for(;d>>1;f[g]c-o){var _=r?(i*v+a*y)/r:a;t(n,d,y,i,o,_,c),t(d,e,v,_,o,a,c)}else{var b=r?(o*v+c*y)/r:c;t(n,d,y,i,o,a,b),t(d,e,v,i,b,a,c)}}(0,c,t.value,n,e,r,i)},t.treemapDice=Ap,t.treemapResquarify=Lp,t.treemapSlice=Ip,t.treemapSliceDice=function(t,n,e,r,i){(1&t.depth?Ip:Ap)(t,n,e,r,i)},t.treemapSquarify=Yp,t.tsv=Mc,t.tsvFormat=lc,t.tsvFormatBody=hc,t.tsvFormatRow=pc,t.tsvFormatRows=dc,t.tsvFormatValue=gc,t.tsvParse=fc,t.tsvParseRows=sc,t.union=function(...t){const n=new InternSet;for(const e of t)for(const t of e)n.add(t);return n},t.unixDay=_y,t.unixDays=by,t.utcDay=yy,t.utcDays=vy,t.utcFriday=By,t.utcFridays=Vy,t.utcHour=hy,t.utcHours=dy,t.utcMillisecond=Wg,t.utcMilliseconds=Zg,t.utcMinute=cy,t.utcMinutes=fy,t.utcMonday=qy,t.utcMondays=jy,t.utcMonth=Qy,t.utcMonths=Jy,t.utcSaturday=Yy,t.utcSaturdays=Wy,t.utcSecond=iy,t.utcSeconds=oy,t.utcSunday=Fy,t.utcSundays=Ly,t.utcThursday=Oy,t.utcThursdays=Gy,t.utcTickInterval=av,t.utcTicks=ov,t.utcTuesday=Uy,t.utcTuesdays=Hy,t.utcWednesday=Iy,t.utcWednesdays=Xy,t.utcWeek=Fy,t.utcWeeks=Ly,t.utcYear=ev,t.utcYears=rv,t.variance=x,t.version="7.8.5",t.window=pn,t.xml=Sc,t.zip=function(){return gt(arguments)},t.zoom=function(){var t,n,e,r=Sw,i=Ew,o=Pw,a=kw,u=Cw,c=[0,1/0],f=[[-1/0,-1/0],[1/0,1/0]],s=250,l=ri,h=$t("start","zoom","end"),d=500,p=150,g=0,y=10;function v(t){t.property("__zoom",Nw).on("wheel.zoom",T,{passive:!1}).on("mousedown.zoom",A).on("dblclick.zoom",S).filter(u).on("touchstart.zoom",E).on("touchmove.zoom",N).on("touchend.zoom touchcancel.zoom",k).style("-webkit-tap-highlight-color","rgba(0,0,0,0)")}function _(t,n){return(n=Math.max(c[0],Math.min(c[1],n)))===t.k?t:new xw(n,t.x,t.y)}function b(t,n,e){var r=n[0]-e[0]*t.k,i=n[1]-e[1]*t.k;return r===t.x&&i===t.y?t:new xw(t.k,r,i)}function m(t){return[(+t[0][0]+ +t[1][0])/2,(+t[0][1]+ +t[1][1])/2]}function x(t,n,e,r){t.on("start.zoom",(function(){w(this,arguments).event(r).start()})).on("interrupt.zoom end.zoom",(function(){w(this,arguments).event(r).end()})).tween("zoom",(function(){var t=this,o=arguments,a=w(t,o).event(r),u=i.apply(t,o),c=null==e?m(u):"function"==typeof e?e.apply(t,o):e,f=Math.max(u[1][0]-u[0][0],u[1][1]-u[0][1]),s=t.__zoom,h="function"==typeof n?n.apply(t,o):n,d=l(s.invert(c).concat(f/s.k),h.invert(c).concat(f/h.k));return function(t){if(1===t)t=h;else{var n=d(t),e=f/n[2];t=new xw(e,c[0]-n[0]*e,c[1]-n[1]*e)}a.zoom(null,t)}}))}function w(t,n,e){return!e&&t.__zooming||new M(t,n)}function M(t,n){this.that=t,this.args=n,this.active=0,this.sourceEvent=null,this.extent=i.apply(t,n),this.taps=0}function T(t,...n){if(r.apply(this,arguments)){var e=w(this,n).event(t),i=this.__zoom,u=Math.max(c[0],Math.min(c[1],i.k*Math.pow(2,a.apply(this,arguments)))),s=ne(t);if(e.wheel)e.mouse[0][0]===s[0]&&e.mouse[0][1]===s[1]||(e.mouse[1]=i.invert(e.mouse[0]=s)),clearTimeout(e.wheel);else{if(i.k===u)return;e.mouse=[s,i.invert(s)],Gi(this),e.start()}Aw(t),e.wheel=setTimeout((function(){e.wheel=null,e.end()}),p),e.zoom("mouse",o(b(_(i,u),e.mouse[0],e.mouse[1]),e.extent,f))}}function A(t,...n){if(!e&&r.apply(this,arguments)){var i=t.currentTarget,a=w(this,n,!0).event(t),u=Zn(t.view).on("mousemove.zoom",(function(t){if(Aw(t),!a.moved){var n=t.clientX-s,e=t.clientY-l;a.moved=n*n+e*e>g}a.event(t).zoom("mouse",o(b(a.that.__zoom,a.mouse[0]=ne(t,i),a.mouse[1]),a.extent,f))}),!0).on("mouseup.zoom",(function(t){u.on("mousemove.zoom mouseup.zoom",null),ue(t.view,a.moved),Aw(t),a.event(t).end()}),!0),c=ne(t,i),s=t.clientX,l=t.clientY;ae(t.view),Tw(t),a.mouse=[c,this.__zoom.invert(c)],Gi(this),a.start()}}function S(t,...n){if(r.apply(this,arguments)){var e=this.__zoom,a=ne(t.changedTouches?t.changedTouches[0]:t,this),u=e.invert(a),c=e.k*(t.shiftKey?.5:2),l=o(b(_(e,c),a,u),i.apply(this,n),f);Aw(t),s>0?Zn(this).transition().duration(s).call(x,l,a,t):Zn(this).call(v.transform,l,a,t)}}function E(e,...i){if(r.apply(this,arguments)){var o,a,u,c,f=e.touches,s=f.length,l=w(this,i,e.changedTouches.length===s).event(e);for(Tw(e),a=0;a --help' for command-specific help.""" + + +# Constants for socket synchronization +_SYNC_TIMEOUT = 5.0 +_PROCESS_KILL_TIMEOUT = 2.0 +_READY_MESSAGE = b"ready" +_RECV_BUFFER_SIZE = 1024 + +# Format configuration +FORMAT_EXTENSIONS = { + "pstats": "pstats", + "collapsed": "txt", + "flamegraph": "html", + "gecko": "json", + "heatmap": "html", +} + +COLLECTOR_MAP = { + "pstats": PstatsCollector, + "collapsed": CollapsedStackCollector, + "flamegraph": FlamegraphCollector, + "gecko": GeckoCollector, + "heatmap": HeatmapCollector, +} + + +def _parse_mode(mode_string): + """Convert mode string to mode constant.""" + mode_map = { + "wall": PROFILING_MODE_WALL, + "cpu": PROFILING_MODE_CPU, + "gil": PROFILING_MODE_GIL, + } + return mode_map[mode_string] + + +def _run_with_sync(original_cmd, suppress_output=False): + """Run a command with socket-based synchronization and return the process.""" + # Create a TCP socket for synchronization with better socket options + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sync_sock: + # Set SO_REUSEADDR to avoid "Address already in use" errors + sync_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sync_sock.bind(("127.0.0.1", 0)) # Let OS choose a free port + sync_port = sync_sock.getsockname()[1] + sync_sock.listen(1) + sync_sock.settimeout(_SYNC_TIMEOUT) + + # Get current working directory to preserve it + cwd = os.getcwd() + + # Build command using the sync coordinator + target_args = original_cmd[1:] # Remove python executable + cmd = ( + sys.executable, + "-m", + "profiling.sampling._sync_coordinator", + str(sync_port), + cwd, + ) + tuple(target_args) + + # Start the process with coordinator + # Suppress stdout/stderr if requested (for live mode) + popen_kwargs = {} + if suppress_output: + popen_kwargs["stdin"] = subprocess.DEVNULL + popen_kwargs["stdout"] = subprocess.DEVNULL + popen_kwargs["stderr"] = subprocess.DEVNULL + + process = subprocess.Popen(cmd, **popen_kwargs) + + try: + # Wait for ready signal with timeout + with sync_sock.accept()[0] as conn: + ready_signal = conn.recv(_RECV_BUFFER_SIZE) + + if ready_signal != _READY_MESSAGE: + raise RuntimeError( + f"Invalid ready signal received: {ready_signal!r}" + ) + + except socket.timeout: + # If we timeout, kill the process and raise an error + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=_PROCESS_KILL_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise RuntimeError( + "Process failed to signal readiness within timeout" + ) + + return process + + +def _add_sampling_options(parser): + """Add sampling configuration options to a parser.""" + sampling_group = parser.add_argument_group("Sampling configuration") + sampling_group.add_argument( + "-i", + "--interval", + type=int, + default=100, + metavar="MICROSECONDS", + help="sampling interval", + ) + sampling_group.add_argument( + "-d", + "--duration", + type=int, + default=10, + metavar="SECONDS", + help="Sampling duration", + ) + sampling_group.add_argument( + "-a", + "--all-threads", + action="store_true", + help="Sample all threads in the process instead of just the main thread", + ) + sampling_group.add_argument( + "--realtime-stats", + action="store_true", + help="Print real-time sampling statistics (Hz, mean, min, max) during profiling", + ) + sampling_group.add_argument( + "--native", + action="store_true", + help='Include artificial "" frames to denote calls to non-Python code', + ) + sampling_group.add_argument( + "--no-gc", + action="store_false", + dest="gc", + help='Don\'t include artificial "" frames to denote active garbage collection', + ) + sampling_group.add_argument( + "--async-aware", + action="store_true", + help="Enable async-aware profiling (uses task-based stack reconstruction)", + ) + + +def _add_mode_options(parser): + """Add mode options to a parser.""" + mode_group = parser.add_argument_group("Mode options") + mode_group.add_argument( + "--mode", + choices=["wall", "cpu", "gil"], + default="wall", + help="Sampling mode: wall (all samples), cpu (only samples when thread is on CPU), " + "gil (only samples when thread holds the GIL). Incompatible with --async-aware", + ) + mode_group.add_argument( + "--async-mode", + choices=["running", "all"], + default="running", + help='Async profiling mode: "running" (only running task) ' + 'or "all" (all tasks including waiting). Requires --async-aware', + ) + + +def _add_format_options(parser): + """Add output format options to a parser.""" + output_group = parser.add_argument_group("Output options") + format_group = output_group.add_mutually_exclusive_group() + format_group.add_argument( + "--pstats", + action="store_const", + const="pstats", + dest="format", + help="Generate pstats output (default)", + ) + format_group.add_argument( + "--collapsed", + action="store_const", + const="collapsed", + dest="format", + help="Generate collapsed stack traces for flamegraphs", + ) + format_group.add_argument( + "--flamegraph", + action="store_const", + const="flamegraph", + dest="format", + help="Generate interactive HTML flamegraph visualization", + ) + format_group.add_argument( + "--gecko", + action="store_const", + const="gecko", + dest="format", + help="Generate Gecko format for Firefox Profiler", + ) + format_group.add_argument( + "--heatmap", + action="store_const", + const="heatmap", + dest="format", + help="Generate interactive HTML heatmap visualization with line-level sample counts", + ) + parser.set_defaults(format="pstats") + + output_group.add_argument( + "-o", + "--output", + dest="outfile", + help="Output path (default: stdout for pstats, auto-generated for others). " + "For heatmap: directory name (default: heatmap_PID)", + ) + + +def _add_pstats_options(parser): + """Add pstats-specific display options to a parser.""" + pstats_group = parser.add_argument_group("pstats format options") + pstats_group.add_argument( + "--sort", + choices=[ + "nsamples", + "tottime", + "cumtime", + "sample-pct", + "cumul-pct", + "nsamples-cumul", + "name", + ], + default=None, + help="Sort order for pstats output (default: nsamples)", + ) + pstats_group.add_argument( + "-l", + "--limit", + type=int, + default=None, + help="Limit the number of rows in the output (default: 15)", + ) + pstats_group.add_argument( + "--no-summary", + action="store_true", + help="Disable the summary section in the pstats output", + ) + + +def _sort_to_mode(sort_choice): + """Convert sort choice string to SORT_MODE constant.""" + sort_map = { + "nsamples": SORT_MODE_NSAMPLES, + "tottime": SORT_MODE_TOTTIME, + "cumtime": SORT_MODE_CUMTIME, + "sample-pct": SORT_MODE_SAMPLE_PCT, + "cumul-pct": SORT_MODE_CUMUL_PCT, + "nsamples-cumul": SORT_MODE_NSAMPLES_CUMUL, + "name": -1, + } + return sort_map.get(sort_choice, SORT_MODE_NSAMPLES) + + +def _create_collector(format_type, interval, skip_idle): + """Create the appropriate collector based on format type. + + Args: + format_type: The output format ('pstats', 'collapsed', 'flamegraph', 'gecko') + interval: Sampling interval in microseconds + skip_idle: Whether to skip idle samples + + Returns: + A collector instance of the appropriate type + """ + collector_class = COLLECTOR_MAP.get(format_type) + if collector_class is None: + raise ValueError(f"Unknown format: {format_type}") + + # Gecko format never skips idle (it needs both GIL and CPU data) + if format_type == "gecko": + skip_idle = False + + return collector_class(interval, skip_idle=skip_idle) + + +def _generate_output_filename(format_type, pid): + """Generate output filename based on format and PID. + + Args: + format_type: The output format + pid: Process ID + + Returns: + Generated filename + """ + extension = FORMAT_EXTENSIONS.get(format_type, "txt") + # For heatmap, use cleaner directory name without extension + if format_type == "heatmap": + return f"heatmap_{pid}" + return f"{format_type}.{pid}.{extension}" + + +def _handle_output(collector, args, pid, mode): + """Handle output for the collector based on format and arguments. + + Args: + collector: The collector instance with profiling data + args: Parsed command-line arguments + pid: Process ID (for generating filenames) + mode: Profiling mode used + """ + if args.format == "pstats": + if args.outfile: + collector.export(args.outfile) + else: + # Print to stdout with defaults applied + sort_choice = args.sort if args.sort is not None else "nsamples" + limit = args.limit if args.limit is not None else 15 + sort_mode = _sort_to_mode(sort_choice) + collector.print_stats( + sort_mode, limit, not args.no_summary, mode + ) + else: + # Export to file + filename = args.outfile or _generate_output_filename(args.format, pid) + collector.export(filename) + + +def _validate_args(args, parser): + """Validate format-specific options and live mode requirements. + + Args: + args: Parsed command-line arguments + parser: ArgumentParser instance for error reporting + """ + # Check if live mode is available + if hasattr(args, 'live') and args.live and LiveStatsCollector is None: + parser.error( + "Live mode requires the curses module, which is not available." + ) + + # Async-aware mode is incompatible with --native, --no-gc, --mode, and --all-threads + if args.async_aware: + issues = [] + if args.native: + issues.append("--native") + if not args.gc: + issues.append("--no-gc") + if hasattr(args, 'mode') and args.mode != "wall": + issues.append(f"--mode={args.mode}") + if hasattr(args, 'all_threads') and args.all_threads: + issues.append("--all-threads") + if issues: + parser.error( + f"Options {', '.join(issues)} are incompatible with --async-aware. " + "Async-aware profiling uses task-based stack reconstruction." + ) + + # --async-mode requires --async-aware + if hasattr(args, 'async_mode') and args.async_mode != "running" and not args.async_aware: + parser.error("--async-mode requires --async-aware to be enabled.") + + # Live mode is incompatible with format options + if hasattr(args, 'live') and args.live: + if args.format != "pstats": + format_flag = f"--{args.format}" + parser.error( + f"--live is incompatible with {format_flag}. Live mode uses a TUI interface." + ) + + # Live mode is also incompatible with pstats-specific options + issues = [] + if args.sort is not None: + issues.append("--sort") + if args.limit is not None: + issues.append("--limit") + if args.no_summary: + issues.append("--no-summary") + + if issues: + parser.error( + f"Options {', '.join(issues)} are incompatible with --live. " + "Live mode uses a TUI interface with its own controls." + ) + return + + # Validate gecko mode doesn't use non-wall mode + if args.format == "gecko" and args.mode != "wall": + parser.error( + "--mode option is incompatible with --gecko. " + "Gecko format automatically includes both GIL-holding and CPU status analysis." + ) + + # Validate pstats-specific options are only used with pstats format + if args.format != "pstats": + issues = [] + if args.sort is not None: + issues.append("--sort") + if args.limit is not None: + issues.append("--limit") + if args.no_summary: + issues.append("--no-summary") + + if issues: + format_flag = f"--{args.format}" + parser.error( + f"Options {', '.join(issues)} are only valid with --pstats, not {format_flag}" + ) + + +def main(): + """Main entry point for the CLI.""" + # Create the main parser + parser = argparse.ArgumentParser( + description=_HELP_DESCRIPTION, + formatter_class=CustomFormatter, + ) + + # Create subparsers for commands + subparsers = parser.add_subparsers( + dest="command", required=True, help="Command to run" + ) + + # === RUN COMMAND === + run_parser = subparsers.add_parser( + "run", + help="Run and profile a script or module", + formatter_class=CustomFormatter, + description="""Run and profile a Python script or module + +Examples: + # Run and profile a module + python -m profiling.sampling run -m mymodule arg1 arg2 + + # Generate flamegraph from a script + python -m profiling.sampling run --flamegraph -o output.html script.py + + # Profile with custom interval and duration + python -m profiling.sampling run -i 50 -d 30 script.py + + # Save collapsed stacks to file + python -m profiling.sampling run --collapsed -o stacks.txt script.py + + # Live interactive mode for a script + python -m profiling.sampling run --live script.py""", + ) + run_parser.add_argument( + "-m", + "--module", + action="store_true", + help="Run target as a module (like python -m)", + ) + run_parser.add_argument( + "target", + help="Script file or module name to profile", + ) + run_parser.add_argument( + "args", + nargs=argparse.REMAINDER, + help="Arguments to pass to the script or module", + ) + run_parser.add_argument( + "--live", + action="store_true", + help="Interactive TUI profiler (top-like interface, press 'q' to quit, 's' to cycle sort)", + ) + _add_sampling_options(run_parser) + _add_mode_options(run_parser) + _add_format_options(run_parser) + _add_pstats_options(run_parser) + + # === ATTACH COMMAND === + attach_parser = subparsers.add_parser( + "attach", + help="Attach to and profile a running process", + formatter_class=CustomFormatter, + description="""Attach to a running process and profile it + +Examples: + # Profile all threads, sort by total time + python -m profiling.sampling attach -a --sort tottime 1234 + + # Live interactive mode for a running process + python -m profiling.sampling attach --live 1234""", + ) + attach_parser.add_argument( + "pid", + type=int, + help="Process ID to attach to", + ) + attach_parser.add_argument( + "--live", + action="store_true", + help="Interactive TUI profiler (top-like interface, press 'q' to quit, 's' to cycle sort)", + ) + _add_sampling_options(attach_parser) + _add_mode_options(attach_parser) + _add_format_options(attach_parser) + _add_pstats_options(attach_parser) + + # Parse arguments + args = parser.parse_args() + + # Validate arguments + _validate_args(args, parser) + + # Command dispatch table + command_handlers = { + "run": _handle_run, + "attach": _handle_attach, + } + + # Execute the appropriate command + handler = command_handlers.get(args.command) + if handler: + handler(args) + else: + parser.error(f"Unknown command: {args.command}") + + +def _handle_attach(args): + """Handle the 'attach' command.""" + # Check if live mode is requested + if args.live: + _handle_live_attach(args, args.pid) + return + + # Use PROFILING_MODE_ALL for gecko format + mode = ( + PROFILING_MODE_ALL + if args.format == "gecko" + else _parse_mode(args.mode) + ) + + # Determine skip_idle based on mode + skip_idle = ( + mode != PROFILING_MODE_WALL if mode != PROFILING_MODE_ALL else False + ) + + # Create the appropriate collector + collector = _create_collector(args.format, args.interval, skip_idle) + + # Sample the process + collector = sample( + args.pid, + collector, + duration_sec=args.duration, + all_threads=args.all_threads, + realtime_stats=args.realtime_stats, + mode=mode, + async_aware=args.async_mode if args.async_aware else None, + native=args.native, + gc=args.gc, + ) + + # Handle output + _handle_output(collector, args, args.pid, mode) + + +def _handle_run(args): + """Handle the 'run' command.""" + # Check if live mode is requested + if args.live: + _handle_live_run(args) + return + + # Build the command to run + if args.module: + cmd = (sys.executable, "-m", args.target, *args.args) + else: + cmd = (sys.executable, args.target, *args.args) + + # Run with synchronization + process = _run_with_sync(cmd, suppress_output=False) + + # Use PROFILING_MODE_ALL for gecko format + mode = ( + PROFILING_MODE_ALL + if args.format == "gecko" + else _parse_mode(args.mode) + ) + + # Determine skip_idle based on mode + skip_idle = ( + mode != PROFILING_MODE_WALL if mode != PROFILING_MODE_ALL else False + ) + + # Create the appropriate collector + collector = _create_collector(args.format, args.interval, skip_idle) + + # Profile the subprocess + try: + collector = sample( + process.pid, + collector, + duration_sec=args.duration, + all_threads=args.all_threads, + realtime_stats=args.realtime_stats, + mode=mode, + async_aware=args.async_mode if args.async_aware else None, + native=args.native, + gc=args.gc, + ) + + # Handle output + _handle_output(collector, args, process.pid, mode) + finally: + # Clean up the subprocess + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=_PROCESS_KILL_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def _handle_live_attach(args, pid): + """Handle live mode for an existing process.""" + mode = _parse_mode(args.mode) + + # Determine skip_idle based on mode + skip_idle = mode != PROFILING_MODE_WALL + + # Create live collector with default settings + collector = LiveStatsCollector( + args.interval, + skip_idle=skip_idle, + sort_by="tottime", # Default initial sort + limit=20, # Default limit + pid=pid, + mode=mode, + async_aware=args.async_mode if args.async_aware else None, + ) + + # Sample in live mode + sample_live( + pid, + collector, + duration_sec=args.duration, + all_threads=args.all_threads, + realtime_stats=args.realtime_stats, + mode=mode, + async_aware=args.async_mode if args.async_aware else None, + native=args.native, + gc=args.gc, + ) + + +def _handle_live_run(args): + """Handle live mode for running a script/module.""" + # Build the command to run + if args.module: + cmd = (sys.executable, "-m", args.target, *args.args) + else: + cmd = (sys.executable, args.target, *args.args) + + # Run with synchronization, suppressing output for live mode + process = _run_with_sync(cmd, suppress_output=True) + + mode = _parse_mode(args.mode) + + # Determine skip_idle based on mode + skip_idle = mode != PROFILING_MODE_WALL + + # Create live collector with default settings + collector = LiveStatsCollector( + args.interval, + skip_idle=skip_idle, + sort_by="tottime", # Default initial sort + limit=20, # Default limit + pid=process.pid, + mode=mode, + async_aware=args.async_mode if args.async_aware else None, + ) + + # Profile the subprocess in live mode + try: + sample_live( + process.pid, + collector, + duration_sec=args.duration, + all_threads=args.all_threads, + realtime_stats=args.realtime_stats, + mode=mode, + async_aware=args.async_mode if args.async_aware else None, + native=args.native, + gc=args.gc, + ) + finally: + # Clean up the subprocess + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=_PROCESS_KILL_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +if __name__ == "__main__": + main() diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py new file mode 100644 index 00000000000..f63ea0afd8a --- /dev/null +++ b/Lib/profiling/sampling/collector.py @@ -0,0 +1,209 @@ +from abc import ABC, abstractmethod +from .constants import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + THREAD_STATUS_GIL_REQUESTED, + THREAD_STATUS_UNKNOWN, +) + +try: + from _remote_debugging import FrameInfo +except ImportError: + # Fallback definition if _remote_debugging is not available + FrameInfo = None + +class Collector(ABC): + @abstractmethod + def collect(self, stack_frames): + """Collect profiling data from stack frames.""" + + def collect_failed_sample(self): + """Collect data about a failed sample attempt.""" + + @abstractmethod + def export(self, filename): + """Export collected data to a file.""" + + def _iter_all_frames(self, stack_frames, skip_idle=False): + for interpreter_info in stack_frames: + for thread_info in interpreter_info.threads: + # skip_idle now means: skip if thread is not actively running + # A thread is "active" if it has the GIL OR is on CPU + if skip_idle: + status_flags = thread_info.status + has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) + on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) + if not (has_gil or on_cpu): + continue + frames = thread_info.frame_info + if frames: + yield frames, thread_info.thread_id + + def _iter_async_frames(self, awaited_info_list): + # Phase 1: Index tasks and build parent relationships with pre-computed selection + task_map, child_to_parent, all_task_ids, all_parent_ids = self._build_task_graph(awaited_info_list) + + # Phase 2: Find leaf tasks (tasks not awaited by anyone) + leaf_task_ids = self._find_leaf_tasks(all_task_ids, all_parent_ids) + + # Phase 3: Build linear stacks from each leaf to root (optimized - no sorting!) + yield from self._build_linear_stacks(leaf_task_ids, task_map, child_to_parent) + + def _build_task_graph(self, awaited_info_list): + task_map = {} + child_to_parent = {} # Maps child_id -> (selected_parent_id, parent_count) + all_task_ids = set() + all_parent_ids = set() # Track ALL parent IDs for leaf detection + + for awaited_info in awaited_info_list: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_map[task_id] = (task_info, thread_id) + all_task_ids.add(task_id) + + # Pre-compute selected parent and count for optimization + if task_info.awaited_by: + parent_ids = [p.task_name for p in task_info.awaited_by] + parent_count = len(parent_ids) + # Track ALL parents for leaf detection + all_parent_ids.update(parent_ids) + # Use min() for O(n) instead of sorted()[0] which is O(n log n) + selected_parent = min(parent_ids) if parent_count > 1 else parent_ids[0] + child_to_parent[task_id] = (selected_parent, parent_count) + + return task_map, child_to_parent, all_task_ids, all_parent_ids + + def _find_leaf_tasks(self, all_task_ids, all_parent_ids): + # Leaves are tasks that are not parents of any other task + return all_task_ids - all_parent_ids + + def _build_linear_stacks(self, leaf_task_ids, task_map, child_to_parent): + for leaf_id in leaf_task_ids: + frames = [] + visited = set() + current_id = leaf_id + thread_id = None + + # Follow the single parent chain from leaf to root + while current_id is not None: + # Cycle detection + if current_id in visited: + break + visited.add(current_id) + + # Check if task exists in task_map + if current_id not in task_map: + break + + task_info, tid = task_map[current_id] + + # Set thread_id from first task + if thread_id is None: + thread_id = tid + + # Add all frames from all coroutines in this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + for frame in coro_info.call_stack: + frames.append(frame) + + # Get pre-computed parent info (no sorting needed!) + parent_info = child_to_parent.get(current_id) + + # Add task boundary marker with parent count annotation if multiple parents + task_name = task_info.task_name or "Task-" + str(task_info.task_id) + if parent_info: + selected_parent, parent_count = parent_info + if parent_count > 1: + task_name = f"{task_name} ({parent_count} parents)" + frames.append(FrameInfo(("", 0, task_name))) + current_id = selected_parent + else: + # Root task - no parent + frames.append(FrameInfo(("", 0, task_name))) + current_id = None + + # Yield the complete stack if we collected any frames + if frames and thread_id is not None: + yield frames, thread_id, leaf_id + + def _is_gc_frame(self, frame): + if isinstance(frame, tuple): + funcname = frame[2] if len(frame) >= 3 else "" + else: + funcname = getattr(frame, "funcname", "") + + return "" in funcname or "gc_collect" in funcname + + def _collect_thread_status_stats(self, stack_frames): + """Collect aggregate and per-thread status statistics from a sample. + + Returns: + tuple: (aggregate_status_counts, has_gc_frame, per_thread_stats) + - aggregate_status_counts: dict with has_gil, on_cpu, etc. + - has_gc_frame: bool indicating if any thread has GC frames + - per_thread_stats: dict mapping thread_id to per-thread counts + """ + status_counts = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + } + has_gc_frame = False + per_thread_stats = {} + + for interpreter_info in stack_frames: + threads = getattr(interpreter_info, "threads", []) + for thread_info in threads: + status_counts["total"] += 1 + + # Track thread status using bit flags + status_flags = getattr(thread_info, "status", 0) + + if status_flags & THREAD_STATUS_HAS_GIL: + status_counts["has_gil"] += 1 + if status_flags & THREAD_STATUS_ON_CPU: + status_counts["on_cpu"] += 1 + if status_flags & THREAD_STATUS_GIL_REQUESTED: + status_counts["gil_requested"] += 1 + if status_flags & THREAD_STATUS_UNKNOWN: + status_counts["unknown"] += 1 + + # Track per-thread statistics + thread_id = getattr(thread_info, "thread_id", None) + if thread_id is not None: + if thread_id not in per_thread_stats: + per_thread_stats[thread_id] = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + "gc_samples": 0, + } + + thread_stats = per_thread_stats[thread_id] + thread_stats["total"] += 1 + + if status_flags & THREAD_STATUS_HAS_GIL: + thread_stats["has_gil"] += 1 + if status_flags & THREAD_STATUS_ON_CPU: + thread_stats["on_cpu"] += 1 + if status_flags & THREAD_STATUS_GIL_REQUESTED: + thread_stats["gil_requested"] += 1 + if status_flags & THREAD_STATUS_UNKNOWN: + thread_stats["unknown"] += 1 + + # Check for GC frames in this thread + frames = getattr(thread_info, "frame_info", None) + if frames: + for frame in frames: + if self._is_gc_frame(frame): + thread_stats["gc_samples"] += 1 + has_gc_frame = True + break + + return status_counts, has_gc_frame, per_thread_stats diff --git a/Lib/profiling/sampling/constants.py b/Lib/profiling/sampling/constants.py new file mode 100644 index 00000000000..be2ae60a88f --- /dev/null +++ b/Lib/profiling/sampling/constants.py @@ -0,0 +1,30 @@ +"""Constants for the sampling profiler.""" + +# Profiling mode constants +PROFILING_MODE_WALL = 0 +PROFILING_MODE_CPU = 1 +PROFILING_MODE_GIL = 2 +PROFILING_MODE_ALL = 3 # Combines GIL + CPU checks + +# Sort mode constants +SORT_MODE_NSAMPLES = 0 +SORT_MODE_TOTTIME = 1 +SORT_MODE_CUMTIME = 2 +SORT_MODE_SAMPLE_PCT = 3 +SORT_MODE_CUMUL_PCT = 4 +SORT_MODE_NSAMPLES_CUMUL = 5 + +# Thread status flags +try: + from _remote_debugging import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + THREAD_STATUS_UNKNOWN, + THREAD_STATUS_GIL_REQUESTED, + ) +except ImportError: + # Fallback for tests or when module is not available + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_UNKNOWN = (1 << 2) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py new file mode 100644 index 00000000000..921cd625f04 --- /dev/null +++ b/Lib/profiling/sampling/gecko_collector.py @@ -0,0 +1,670 @@ +import itertools +import json +import os +import platform +import sys +import threading +import time + +from .collector import Collector +try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED +except ImportError: + # Fallback if module not available (shouldn't happen in normal use) + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_UNKNOWN = (1 << 2) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) + + +# Categories matching Firefox Profiler expectations +GECKO_CATEGORIES = [ + {"name": "Other", "color": "grey", "subcategories": ["Other"]}, + {"name": "Python", "color": "yellow", "subcategories": ["Other"]}, + {"name": "Native", "color": "blue", "subcategories": ["Other"]}, + {"name": "GC", "color": "orange", "subcategories": ["Other"]}, + {"name": "GIL", "color": "green", "subcategories": ["Other"]}, + {"name": "CPU", "color": "purple", "subcategories": ["Other"]}, + {"name": "Code Type", "color": "red", "subcategories": ["Other"]}, +] + +# Category indices +CATEGORY_OTHER = 0 +CATEGORY_PYTHON = 1 +CATEGORY_NATIVE = 2 +CATEGORY_GC = 3 +CATEGORY_GIL = 4 +CATEGORY_CPU = 5 +CATEGORY_CODE_TYPE = 6 + +# Subcategory indices +DEFAULT_SUBCATEGORY = 0 + +GECKO_FORMAT_VERSION = 32 +GECKO_PREPROCESSED_VERSION = 57 + +# Resource type constants +RESOURCE_TYPE_LIBRARY = 1 + +# Frame constants +FRAME_ADDRESS_NONE = -1 +FRAME_INLINE_DEPTH_ROOT = 0 + +# Process constants +PROCESS_TYPE_MAIN = 0 +STACKWALK_DISABLED = 0 + + +class GeckoCollector(Collector): + def __init__(self, sample_interval_usec, *, skip_idle=False): + self.sample_interval_usec = sample_interval_usec + self.skip_idle = skip_idle + self.start_time = time.time() * 1000 # milliseconds since epoch + + # Global string table (shared across all threads) + self.global_strings = ["(root)"] # Start with root + self.global_string_map = {"(root)": 0} + + # Per-thread data structures + self.threads = {} # tid -> thread data + + # Global tables + self.libs = [] + + # Sampling interval tracking + self.sample_count = 0 + self.last_sample_time = 0 + self.interval = 1.0 # Will be calculated from actual sampling + + # State tracking for interval markers (tid -> start_time) + self.has_gil_start = {} # Thread has the GIL + self.no_gil_start = {} # Thread doesn't have the GIL + self.on_cpu_start = {} # Thread is running on CPU + self.off_cpu_start = {} # Thread is off CPU + self.python_code_start = {} # Thread running Python code (has GIL) + self.native_code_start = {} # Thread running native code (on CPU without GIL) + self.gil_wait_start = {} # Thread waiting for GIL + + # GC event tracking: track GC start time per thread + self.gc_start_per_thread = {} # tid -> start_time + + # Track which threads have been initialized for state tracking + self.initialized_threads = set() + + def _track_state_transition(self, tid, condition, active_dict, inactive_dict, + active_name, inactive_name, category, current_time): + """Track binary state transitions and emit markers. + + Args: + tid: Thread ID + condition: Whether the active state is true + active_dict: Dict tracking start time of active state + inactive_dict: Dict tracking start time of inactive state + active_name: Name for active state marker + inactive_name: Name for inactive state marker + category: Gecko category for the markers + current_time: Current timestamp + """ + # On first observation of a thread, just record the current state + # without creating a marker (we don't know what the previous state was) + if tid not in self.initialized_threads: + if condition: + active_dict[tid] = current_time + else: + inactive_dict[tid] = current_time + return + + # For already-initialized threads, track transitions + if condition: + active_dict.setdefault(tid, current_time) + if tid in inactive_dict: + self._add_marker(tid, inactive_name, inactive_dict.pop(tid), + current_time, category) + else: + inactive_dict.setdefault(tid, current_time) + if tid in active_dict: + self._add_marker(tid, active_name, active_dict.pop(tid), + current_time, category) + + def collect(self, stack_frames): + """Collect a sample from stack frames.""" + current_time = (time.time() * 1000) - self.start_time + + # Update interval calculation + if self.sample_count > 0 and self.last_sample_time > 0: + self.interval = ( + current_time - self.last_sample_time + ) / self.sample_count + self.last_sample_time = current_time + + # Process threads and track GC per thread + for interpreter_info in stack_frames: + for thread_info in interpreter_info.threads: + frames = thread_info.frame_info + tid = thread_info.thread_id + + # Initialize thread if needed + if tid not in self.threads: + self.threads[tid] = self._create_thread(tid) + + thread_data = self.threads[tid] + + # Decode status flags + status_flags = thread_info.status + has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) + on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) + gil_requested = bool(status_flags & THREAD_STATUS_GIL_REQUESTED) + + # Track GIL possession (Has GIL / No GIL) + self._track_state_transition( + tid, has_gil, self.has_gil_start, self.no_gil_start, + "Has GIL", "No GIL", CATEGORY_GIL, current_time + ) + + # Track CPU state (On CPU / Off CPU) + self._track_state_transition( + tid, on_cpu, self.on_cpu_start, self.off_cpu_start, + "On CPU", "Off CPU", CATEGORY_CPU, current_time + ) + + # Track code type (Python Code / Native Code) + # This is tri-state: Python (has_gil), Native (on_cpu without gil), or Neither + if has_gil: + self._track_state_transition( + tid, True, self.python_code_start, self.native_code_start, + "Python Code", "Native Code", CATEGORY_CODE_TYPE, current_time + ) + elif on_cpu: + self._track_state_transition( + tid, True, self.native_code_start, self.python_code_start, + "Native Code", "Python Code", CATEGORY_CODE_TYPE, current_time + ) + else: + # Thread is idle (neither has GIL nor on CPU) - close any open code markers + # This handles the third state that _track_state_transition doesn't cover + if tid in self.initialized_threads: + if tid in self.python_code_start: + self._add_marker(tid, "Python Code", self.python_code_start.pop(tid), + current_time, CATEGORY_CODE_TYPE) + if tid in self.native_code_start: + self._add_marker(tid, "Native Code", self.native_code_start.pop(tid), + current_time, CATEGORY_CODE_TYPE) + + # Track "Waiting for GIL" intervals (one-sided tracking) + if gil_requested: + self.gil_wait_start.setdefault(tid, current_time) + elif tid in self.gil_wait_start: + self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), + current_time, CATEGORY_GIL) + + # Track GC events by detecting frames in the stack trace + # This leverages the improved GC frame tracking from commit 336366fd7ca + # which precisely identifies the thread that initiated GC collection + has_gc_frame = any(frame[2] == "" for frame in frames) + if has_gc_frame: + # This thread initiated GC collection + if tid not in self.gc_start_per_thread: + self.gc_start_per_thread[tid] = current_time + elif tid in self.gc_start_per_thread: + # End GC marker when no more GC frames are detected + self._add_marker(tid, "GC Collecting", self.gc_start_per_thread.pop(tid), + current_time, CATEGORY_GC) + + # Mark thread as initialized after processing all state transitions + self.initialized_threads.add(tid) + + # Categorize: idle if neither has GIL nor on CPU + is_idle = not has_gil and not on_cpu + + # Skip idle threads if skip_idle is enabled + if self.skip_idle and is_idle: + continue + + if not frames: + continue + + # Process the stack + stack_index = self._process_stack(thread_data, frames) + + # Add sample - cache references to avoid dictionary lookups + samples = thread_data["samples"] + samples["stack"].append(stack_index) + samples["time"].append(current_time) + samples["eventDelay"].append(None) + + self.sample_count += 1 + + def _create_thread(self, tid): + """Create a new thread structure with processed profile format.""" + + # Determine if this is the main thread + try: + is_main = tid == threading.main_thread().ident + except (RuntimeError, AttributeError): + is_main = False + + thread = { + "name": f"Thread-{tid}", + "isMainThread": is_main, + "processStartupTime": 0, + "processShutdownTime": None, + "registerTime": 0, + "unregisterTime": None, + "pausedRanges": [], + "pid": str(os.getpid()), + "tid": tid, + "processType": "default", + "processName": "Python Process", + # Sample data - processed format with direct arrays + "samples": { + "stack": [], + "time": [], + "eventDelay": [], + "weight": None, + "weightType": "samples", + "length": 0, # Will be updated on export + }, + # Stack table - processed format + "stackTable": { + "frame": [], + "category": [], + "subcategory": [], + "prefix": [], + "length": 0, # Will be updated on export + }, + # Frame table - processed format + "frameTable": { + "address": [], + "category": [], + "subcategory": [], + "func": [], + "innerWindowID": [], + "implementation": [], + "optimizations": [], + "line": [], + "column": [], + "inlineDepth": [], + "nativeSymbol": [], + "length": 0, # Will be updated on export + }, + # Function table - processed format + "funcTable": { + "name": [], + "isJS": [], + "relevantForJS": [], + "resource": [], + "fileName": [], + "lineNumber": [], + "columnNumber": [], + "length": 0, # Will be updated on export + }, + # Resource table - processed format + "resourceTable": { + "lib": [], + "name": [], + "host": [], + "type": [], + "length": 0, # Will be updated on export + }, + # Native symbols table (empty for Python) + "nativeSymbols": { + "libIndex": [], + "address": [], + "name": [], + "functionSize": [], + "length": 0, + }, + # Markers - processed format (arrays) + "markers": { + "data": [], + "name": [], + "startTime": [], + "endTime": [], + "phase": [], + "category": [], + "length": 0, + }, + # Caches for deduplication + "_stackCache": {}, + "_frameCache": {}, + "_funcCache": {}, + "_resourceCache": {}, + } + + return thread + + def _is_python(self, filename: str) -> bool: + return not filename.startswith("<") or filename in ["", ""] + + def _get_category(self, filename: str) -> int: + return CATEGORY_PYTHON if self._is_python(filename) else CATEGORY_NATIVE + + def _intern_string(self, s): + """Intern a string in the global string table.""" + if s in self.global_string_map: + return self.global_string_map[s] + idx = len(self.global_strings) + self.global_strings.append(s) + self.global_string_map[s] = idx + return idx + + def _add_marker(self, tid, name, start_time, end_time, category): + """Add an interval marker for a specific thread.""" + if tid not in self.threads: + return + + thread_data = self.threads[tid] + duration = end_time - start_time + + name_idx = self._intern_string(name) + markers = thread_data["markers"] + markers["name"].append(name_idx) + markers["startTime"].append(start_time) + markers["endTime"].append(end_time) + markers["phase"].append(1) # 1 = interval marker + markers["category"].append(category) + markers["data"].append({ + "type": name.replace(" ", ""), + "duration": duration, + "tid": tid + }) + + def _process_stack(self, thread_data, frames): + """Process a stack and return the stack index.""" + if not frames: + return None + + # Cache references to avoid repeated dictionary lookups + stack_cache = thread_data["_stackCache"] + stack_table = thread_data["stackTable"] + stack_frames = stack_table["frame"] + stack_prefix = stack_table["prefix"] + stack_category = stack_table["category"] + stack_subcategory = stack_table["subcategory"] + + # Build stack bottom-up (from root to leaf) + prefix_stack_idx = None + + for frame_tuple in reversed(frames): + # frame_tuple is (filename, lineno, funcname) + filename, lineno, funcname = frame_tuple + + # Get or create function + func_idx = self._get_or_create_func( + thread_data, filename, funcname, lineno + ) + + # Get or create frame + frame_idx = self._get_or_create_frame( + thread_data, func_idx, lineno + ) + + # Check stack cache + stack_key = (frame_idx, prefix_stack_idx) + if stack_key in stack_cache: + prefix_stack_idx = stack_cache[stack_key] + else: + # Create new stack entry + stack_idx = len(stack_frames) + stack_frames.append(frame_idx) + stack_prefix.append(prefix_stack_idx) + + # Determine category + category = self._get_category(filename) + stack_category.append(category) + stack_subcategory.append(DEFAULT_SUBCATEGORY) + + stack_cache[stack_key] = stack_idx + prefix_stack_idx = stack_idx + + return prefix_stack_idx + + def _get_or_create_func(self, thread_data, filename, funcname, lineno): + """Get or create a function entry.""" + func_cache = thread_data["_funcCache"] + func_key = (filename, funcname) + + if func_key in func_cache: + return func_cache[func_key] + + # Cache references for func table + func_table = thread_data["funcTable"] + func_names = func_table["name"] + func_is_js = func_table["isJS"] + func_relevant = func_table["relevantForJS"] + func_resources = func_table["resource"] + func_filenames = func_table["fileName"] + func_line_numbers = func_table["lineNumber"] + func_column_numbers = func_table["columnNumber"] + + func_idx = len(func_names) + + # Intern strings in global table + name_idx = self._intern_string(funcname) + + # Determine if Python + is_python = self._is_python(filename) + + # Create resource + resource_idx = self._get_or_create_resource(thread_data, filename) + + # Add function + func_names.append(name_idx) + func_is_js.append(is_python) + func_relevant.append(is_python) + func_resources.append(resource_idx) + + if is_python: + filename_idx = self._intern_string(os.path.basename(filename)) + func_filenames.append(filename_idx) + func_line_numbers.append(lineno) + else: + func_filenames.append(None) + func_line_numbers.append(None) + func_column_numbers.append(None) + + func_cache[func_key] = func_idx + return func_idx + + def _get_or_create_resource(self, thread_data, filename): + """Get or create a resource entry.""" + resource_cache = thread_data["_resourceCache"] + + if filename in resource_cache: + return resource_cache[filename] + + # Cache references for resource table + resource_table = thread_data["resourceTable"] + resource_libs = resource_table["lib"] + resource_names = resource_table["name"] + resource_hosts = resource_table["host"] + resource_types = resource_table["type"] + + resource_idx = len(resource_names) + resource_name = ( + os.path.basename(filename) if "/" in filename else filename + ) + name_idx = self._intern_string(resource_name) + + resource_libs.append(None) + resource_names.append(name_idx) + resource_hosts.append(None) + resource_types.append(RESOURCE_TYPE_LIBRARY) + + resource_cache[filename] = resource_idx + return resource_idx + + def _get_or_create_frame(self, thread_data, func_idx, lineno): + """Get or create a frame entry.""" + frame_cache = thread_data["_frameCache"] + frame_key = (func_idx, lineno) + + if frame_key in frame_cache: + return frame_cache[frame_key] + + # Cache references for frame table + frame_table = thread_data["frameTable"] + frame_addresses = frame_table["address"] + frame_inline_depths = frame_table["inlineDepth"] + frame_categories = frame_table["category"] + frame_subcategories = frame_table["subcategory"] + frame_funcs = frame_table["func"] + frame_native_symbols = frame_table["nativeSymbol"] + frame_inner_window_ids = frame_table["innerWindowID"] + frame_implementations = frame_table["implementation"] + frame_lines = frame_table["line"] + frame_columns = frame_table["column"] + frame_optimizations = frame_table["optimizations"] + + frame_idx = len(frame_funcs) + + # Determine category based on function - use cached func table reference + is_python = thread_data["funcTable"]["isJS"][func_idx] + category = CATEGORY_PYTHON if is_python else CATEGORY_NATIVE + + frame_addresses.append(FRAME_ADDRESS_NONE) + frame_inline_depths.append(FRAME_INLINE_DEPTH_ROOT) + frame_categories.append(category) + frame_subcategories.append(DEFAULT_SUBCATEGORY) + frame_funcs.append(func_idx) + frame_native_symbols.append(None) + frame_inner_window_ids.append(None) + frame_implementations.append(None) + frame_lines.append(lineno if lineno else None) + frame_columns.append(None) + frame_optimizations.append(None) + + frame_cache[frame_key] = frame_idx + return frame_idx + + def _finalize_markers(self): + """Close any open markers at the end of profiling.""" + end_time = self.last_sample_time + + # Close all open markers for each thread using a generic approach + marker_states = [ + (self.has_gil_start, "Has GIL", CATEGORY_GIL), + (self.no_gil_start, "No GIL", CATEGORY_GIL), + (self.on_cpu_start, "On CPU", CATEGORY_CPU), + (self.off_cpu_start, "Off CPU", CATEGORY_CPU), + (self.python_code_start, "Python Code", CATEGORY_CODE_TYPE), + (self.native_code_start, "Native Code", CATEGORY_CODE_TYPE), + (self.gil_wait_start, "Waiting for GIL", CATEGORY_GIL), + (self.gc_start_per_thread, "GC Collecting", CATEGORY_GC), + ] + + for state_dict, marker_name, category in marker_states: + for tid in list(state_dict.keys()): + self._add_marker(tid, marker_name, state_dict[tid], end_time, category) + del state_dict[tid] + + def export(self, filename): + """Export the profile to a Gecko JSON file.""" + + if self.sample_count > 0 and self.last_sample_time > 0: + self.interval = self.last_sample_time / self.sample_count + + # Spinner for progress indication + spinner = itertools.cycle(['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']) + stop_spinner = threading.Event() + + def spin(): + message = 'Building Gecko profile...' + while not stop_spinner.is_set(): + sys.stderr.write(f'\r{next(spinner)} {message}') + sys.stderr.flush() + time.sleep(0.1) + # Clear the spinner line + sys.stderr.write('\r' + ' ' * (len(message) + 3) + '\r') + sys.stderr.flush() + + spinner_thread = threading.Thread(target=spin, daemon=True) + spinner_thread.start() + + try: + # Finalize any open markers before building profile + self._finalize_markers() + + profile = self._build_profile() + + with open(filename, "w") as f: + json.dump(profile, f, separators=(",", ":")) + finally: + stop_spinner.set() + spinner_thread.join(timeout=1.0) + # Small delay to ensure the clear happens + time.sleep(0.01) + + print(f"Gecko profile written to {filename}") + print( + f"Open in Firefox Profiler: https://profiler.firefox.com/" + ) + + def _build_profile(self): + """Build the complete profile structure in processed format.""" + # Convert thread data to final format + threads = [] + + for tid, thread_data in self.threads.items(): + # Update lengths + samples = thread_data["samples"] + stack_table = thread_data["stackTable"] + frame_table = thread_data["frameTable"] + func_table = thread_data["funcTable"] + resource_table = thread_data["resourceTable"] + + samples["length"] = len(samples["stack"]) + stack_table["length"] = len(stack_table["frame"]) + frame_table["length"] = len(frame_table["func"]) + func_table["length"] = len(func_table["name"]) + resource_table["length"] = len(resource_table["name"]) + thread_data["markers"]["length"] = len(thread_data["markers"]["name"]) + + # Clean up internal caches + del thread_data["_stackCache"] + del thread_data["_frameCache"] + del thread_data["_funcCache"] + del thread_data["_resourceCache"] + + threads.append(thread_data) + + # Main profile structure in processed format + profile = { + "meta": { + "interval": self.interval, + "startTime": self.start_time, + "abi": platform.machine(), + "misc": "Python profiler", + "oscpu": platform.machine(), + "platform": platform.system(), + "processType": PROCESS_TYPE_MAIN, + "categories": GECKO_CATEGORIES, + "stackwalk": STACKWALK_DISABLED, + "toolkit": "", + "version": GECKO_FORMAT_VERSION, + "preprocessedProfileVersion": GECKO_PREPROCESSED_VERSION, + "appBuildID": "", + "physicalCPUs": os.cpu_count() or 0, + "logicalCPUs": os.cpu_count() or 0, + "CPUName": "", + "product": "Python", + "symbolicated": True, + "markerSchema": [], + "importedFrom": "Tachyon Sampling Profiler", + "extensions": { + "id": [], + "name": [], + "baseURL": [], + "length": 0, + }, + }, + "libs": self.libs, + "threads": threads, + "pages": [], + "shared": { + "stringArray": self.global_strings, + "sources": {"length": 0, "uuid": [], "filename": []}, + }, + } + + return profile diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py new file mode 100644 index 00000000000..8a8ba9628df --- /dev/null +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -0,0 +1,944 @@ +"""Heatmap collector for Python profiling with line-level execution heat visualization.""" + +import base64 +import collections +import html +import importlib.resources +import json +import math +import os +import platform +import site +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Tuple + +from ._css_utils import get_combined_css +from .stack_collector import StackTraceCollector + + +# ============================================================================ +# Data Classes +# ============================================================================ + +@dataclass +class FileStats: + """Statistics for a single profiled file.""" + filename: str + module_name: str + module_type: str + total_samples: int + total_self_samples: int + num_lines: int + max_samples: int + max_self_samples: int + percentage: float = 0.0 + + +@dataclass +class TreeNode: + """Node in the hierarchical file tree structure.""" + files: List[FileStats] = field(default_factory=list) + samples: int = 0 + count: int = 0 + children: Dict[str, 'TreeNode'] = field(default_factory=dict) + + +# ============================================================================ +# Module Path Analysis +# ============================================================================ + +def get_python_path_info(): + """Get information about Python installation paths for module extraction. + + Returns: + dict: Dictionary containing stdlib path, site-packages paths, and sys.path entries. + """ + info = { + 'stdlib': None, + 'site_packages': [], + 'sys_path': [] + } + + # Get standard library path from os module location + try: + if hasattr(os, '__file__') and os.__file__: + info['stdlib'] = Path(os.__file__).parent + except (AttributeError, OSError): + pass # Silently continue if we can't determine stdlib path + + # Get site-packages directories + site_packages = [] + try: + site_packages.extend(Path(p) for p in site.getsitepackages()) + except (AttributeError, OSError): + pass # Continue without site packages if unavailable + + # Get user site-packages + try: + user_site = site.getusersitepackages() + if user_site and Path(user_site).exists(): + site_packages.append(Path(user_site)) + except (AttributeError, OSError): + pass # Continue without user site packages + + info['site_packages'] = site_packages + info['sys_path'] = [Path(p) for p in sys.path if p] + + return info + + +def extract_module_name(filename, path_info): + """Extract Python module name and type from file path. + + Args: + filename: Path to the Python file + path_info: Dictionary from get_python_path_info() + + Returns: + tuple: (module_name, module_type) where module_type is one of: + 'stdlib', 'site-packages', 'project', or 'other' + """ + if not filename: + return ('unknown', 'other') + + try: + file_path = Path(filename) + except (ValueError, OSError): + return (str(filename), 'other') + + # Check if it's in stdlib + if path_info['stdlib'] and _is_subpath(file_path, path_info['stdlib']): + try: + rel_path = file_path.relative_to(path_info['stdlib']) + return (_path_to_module(rel_path), 'stdlib') + except ValueError: + pass + + # Check site-packages + for site_pkg in path_info['site_packages']: + if _is_subpath(file_path, site_pkg): + try: + rel_path = file_path.relative_to(site_pkg) + return (_path_to_module(rel_path), 'site-packages') + except ValueError: + continue + + # Check other sys.path entries (project files) + if not str(file_path).startswith(('<', '[')): # Skip special files + for path_entry in path_info['sys_path']: + if _is_subpath(file_path, path_entry): + try: + rel_path = file_path.relative_to(path_entry) + return (_path_to_module(rel_path), 'project') + except ValueError: + continue + + # Fallback: just use the filename + return (_path_to_module(file_path), 'other') + + +def _is_subpath(file_path, parent_path): + try: + file_path.relative_to(parent_path) + return True + except (ValueError, OSError): + return False + + +def _path_to_module(path): + if isinstance(path, str): + path = Path(path) + + # Remove .py extension + if path.suffix == '.py': + path = path.with_suffix('') + + # Convert path separators to dots + parts = path.parts + + # Handle __init__ files - they represent the package itself + if parts and parts[-1] == '__init__': + parts = parts[:-1] + + return '.'.join(parts) if parts else path.stem + + +# ============================================================================ +# Helper Classes +# ============================================================================ + +class _TemplateLoader: + """Loads and caches HTML/CSS/JS templates for heatmap generation.""" + + def __init__(self): + """Load all templates and assets once.""" + self.index_template = None + self.file_template = None + self.index_css = None + self.index_js = None + self.file_css = None + self.file_js = None + self.logo_html = None + + self._load_templates() + + def _load_templates(self): + """Load all template files from _heatmap_assets.""" + try: + template_dir = importlib.resources.files(__package__) + assets_dir = template_dir / "_heatmap_assets" + + # Load HTML templates + self.index_template = (assets_dir / "heatmap_index_template.html").read_text(encoding="utf-8") + self.file_template = (assets_dir / "heatmap_pyfile_template.html").read_text(encoding="utf-8") + + # Load CSS (same file used for both index and file pages) + css_content = get_combined_css("heatmap") + self.index_css = css_content + self.file_css = css_content + + # Load JS + shared_js = (assets_dir / "heatmap_shared.js").read_text(encoding="utf-8") + self.index_js = f"{shared_js}\n{(assets_dir / 'heatmap_index.js').read_text(encoding='utf-8')}" + self.file_js = f"{shared_js}\n{(assets_dir / 'heatmap.js').read_text(encoding='utf-8')}" + + # Load Python logo + logo_dir = template_dir / "_assets" + try: + png_path = logo_dir / "python-logo-only.png" + b64_logo = base64.b64encode(png_path.read_bytes()).decode("ascii") + self.logo_html = f'' + except (FileNotFoundError, IOError) as e: + self.logo_html = '
' + print(f"Warning: Could not load Python logo: {e}") + + except (FileNotFoundError, IOError) as e: + raise RuntimeError(f"Failed to load heatmap template files: {e}") from e + + +class _TreeBuilder: + """Builds hierarchical tree structure from file statistics.""" + + @staticmethod + def build_file_tree(file_stats: List[FileStats]) -> Dict[str, TreeNode]: + """Build hierarchical tree grouped by module type, then by module structure. + + Args: + file_stats: List of FileStats objects + + Returns: + Dictionary mapping module types to their tree roots + """ + # Group by module type first + type_groups = {'stdlib': [], 'site-packages': [], 'project': [], 'other': []} + for stat in file_stats: + type_groups[stat.module_type].append(stat) + + # Build tree for each type + trees = {} + for module_type, stats in type_groups.items(): + if not stats: + continue + + root_node = TreeNode() + + for stat in stats: + module_name = stat.module_name + parts = module_name.split('.') + + # Navigate/create tree structure + current_node = root_node + for i, part in enumerate(parts): + if i == len(parts) - 1: + # Last part - store the file + current_node.files.append(stat) + else: + # Intermediate part - create or navigate + if part not in current_node.children: + current_node.children[part] = TreeNode() + current_node = current_node.children[part] + + # Calculate aggregate stats for this type's tree + _TreeBuilder._calculate_node_stats(root_node) + trees[module_type] = root_node + + return trees + + @staticmethod + def _calculate_node_stats(node: TreeNode) -> Tuple[int, int]: + """Recursively calculate aggregate statistics for tree nodes. + + Args: + node: TreeNode to calculate stats for + + Returns: + Tuple of (total_samples, file_count) + """ + total_samples = 0 + file_count = 0 + + # Count files at this level + for file_stat in node.files: + total_samples += file_stat.total_samples + file_count += 1 + + # Recursively process children + for child in node.children.values(): + child_samples, child_count = _TreeBuilder._calculate_node_stats(child) + total_samples += child_samples + file_count += child_count + + node.samples = total_samples + node.count = file_count + return total_samples, file_count + + +class _HtmlRenderer: + """Renders hierarchical tree structures as HTML.""" + + def __init__(self, file_index: Dict[str, str]): + """Initialize renderer with file index. + + Args: + file_index: Mapping from filenames to HTML file names + """ + self.file_index = file_index + self.heatmap_bar_height = 16 + + def render_hierarchical_html(self, trees: Dict[str, TreeNode]) -> str: + """Build hierarchical HTML with type sections and collapsible module folders. + + Args: + trees: Dictionary mapping module types to tree roots + + Returns: + Complete HTML string for all sections + """ + type_names = { + 'stdlib': '📚 Standard Library', + 'site-packages': '📦 Site Packages', + 'project': '🏗️ Project Files', + 'other': '📄 Other Files' + } + + sections = [] + for module_type in ['project', 'stdlib', 'site-packages', 'other']: + if module_type not in trees: + continue + + tree = trees[module_type] + + # Project starts expanded, others start collapsed + is_collapsed = module_type in {'stdlib', 'site-packages', 'other'} + icon = '▶' if is_collapsed else '▼' + content_style = ' style="display: none;"' if is_collapsed else '' + + section_html = f''' +
+
+ {icon} + {type_names[module_type]} + ({tree.count} files, {tree.samples:,} samples) +
+
+''' + + # Render root folders + root_folders = sorted(tree.children.items(), + key=lambda x: x[1].samples, reverse=True) + + for folder_name, folder_node in root_folders: + section_html += self._render_folder(folder_node, folder_name, level=1) + + # Render root files (files not in any module) + if tree.files: + sorted_files = sorted(tree.files, key=lambda x: x.total_samples, reverse=True) + section_html += '
\n' + for stat in sorted_files: + section_html += self._render_file_item(stat, indent=' ') + section_html += '
\n' + + section_html += '
\n
\n' + sections.append(section_html) + + return '\n'.join(sections) + + def _render_folder(self, node: TreeNode, name: str, level: int = 1) -> str: + """Render a single folder node recursively. + + Args: + node: TreeNode to render + name: Display name for the folder + level: Nesting level for indentation + + Returns: + HTML string for this folder and its contents + """ + indent = ' ' * level + parts = [] + + # Render folder header (collapsed by default) + parts.append(f'{indent}') + + return '\n'.join(parts) + + def _render_file_item(self, stat: FileStats, indent: str = '') -> str: + """Render a single file item with heatmap bar. + + Args: + stat: FileStats object + indent: Indentation string + + Returns: + HTML string for file item + """ + full_path = html.escape(stat.filename) + module_name = html.escape(stat.module_name) + + intensity = stat.percentage / 100.0 + bar_width = min(stat.percentage, 100) + + html_file = self.file_index[stat.filename] + + return (f'{indent}
\n' + f'{indent} 📄 {module_name}\n' + f'{indent} {stat.total_samples:,} samples\n' + f'{indent}
\n' + f'{indent}
\n') + + +# ============================================================================ +# Main Collector Class +# ============================================================================ + +class HeatmapCollector(StackTraceCollector): + """Collector that generates coverage.py-style heatmap HTML output with line intensity. + + This collector creates detailed HTML reports showing which lines of code + were executed most frequently during profiling, similar to coverage.py + but showing execution "heat" rather than just coverage. + """ + + # File naming and formatting constants + FILE_INDEX_FORMAT = "file_{:04d}.html" + + def __init__(self, *args, **kwargs): + """Initialize the heatmap collector with data structures for analysis.""" + super().__init__(*args, **kwargs) + + # Sample counting data structures + self.line_samples = collections.Counter() + self.file_samples = collections.defaultdict(collections.Counter) + self.line_self_samples = collections.Counter() + self.file_self_samples = collections.defaultdict(collections.Counter) + + # Call graph data structures for navigation + self.call_graph = collections.defaultdict(list) + self.callers_graph = collections.defaultdict(list) + self.function_definitions = {} + + # Edge counting for call path analysis + self.edge_samples = collections.Counter() + + # Statistics and metadata + self._total_samples = 0 + self._path_info = get_python_path_info() + self.stats = {} + + # Template loader (loads all templates once) + self._template_loader = _TemplateLoader() + + # File index (populated during export) + self.file_index = {} + + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, missed_samples=None, **kwargs): + """Set profiling statistics to include in heatmap output. + + Args: + sample_interval_usec: Sampling interval in microseconds + duration_sec: Total profiling duration in seconds + sample_rate: Effective sampling rate + error_rate: Optional error rate during profiling + missed_samples: Optional percentage of missed samples + **kwargs: Additional statistics to include + """ + self.stats = { + "sample_interval_usec": sample_interval_usec, + "duration_sec": duration_sec, + "sample_rate": sample_rate, + "error_rate": error_rate, + "missed_samples": missed_samples, + "python_version": sys.version, + "python_implementation": platform.python_implementation(), + "platform": platform.platform(), + } + self.stats.update(kwargs) + + def process_frames(self, frames, thread_id): + """Process stack frames and count samples per line. + + Args: + frames: List of frame tuples (filename, lineno, funcname) + frames[0] is the leaf (top of stack, where execution is) + thread_id: Thread ID for this stack trace + """ + self._total_samples += 1 + + # Count each line in the stack and build call graph + for i, frame_info in enumerate(frames): + filename, lineno, funcname = frame_info + + if not self._is_valid_frame(filename, lineno): + continue + + # frames[0] is the leaf - where execution is actually happening + is_leaf = (i == 0) + self._record_line_sample(filename, lineno, funcname, is_leaf=is_leaf) + + # Build call graph for adjacent frames + if i + 1 < len(frames): + self._record_call_relationship(frames[i], frames[i + 1]) + + def _is_valid_frame(self, filename, lineno): + """Check if a frame should be included in the heatmap.""" + # Skip internal or invalid files + if not filename or filename.startswith('<') or filename.startswith('['): + return False + + # Skip invalid frames with corrupted filename data + if filename == "__init__" and lineno == 0: + return False + + return True + + def _record_line_sample(self, filename, lineno, funcname, is_leaf=False): + """Record a sample for a specific line.""" + # Track cumulative samples (all occurrences in stack) + self.line_samples[(filename, lineno)] += 1 + self.file_samples[filename][lineno] += 1 + + # Track self/leaf samples (only when at top of stack) + if is_leaf: + self.line_self_samples[(filename, lineno)] += 1 + self.file_self_samples[filename][lineno] += 1 + + # Record function definition location + if funcname and (filename, funcname) not in self.function_definitions: + self.function_definitions[(filename, funcname)] = lineno + + def _record_call_relationship(self, callee_frame, caller_frame): + """Record caller/callee relationship between adjacent frames.""" + callee_filename, callee_lineno, callee_funcname = callee_frame + caller_filename, caller_lineno, caller_funcname = caller_frame + + # Skip internal files for call graph + if callee_filename.startswith('<') or callee_filename.startswith('['): + return + + # Get the callee's function definition line + callee_def_line = self.function_definitions.get( + (callee_filename, callee_funcname), callee_lineno + ) + + # Record caller -> callee relationship + caller_key = (caller_filename, caller_lineno) + callee_info = (callee_filename, callee_def_line, callee_funcname) + if callee_info not in self.call_graph[caller_key]: + self.call_graph[caller_key].append(callee_info) + + # Record callee <- caller relationship + callee_key = (callee_filename, callee_def_line) + caller_info = (caller_filename, caller_lineno, caller_funcname) + if caller_info not in self.callers_graph[callee_key]: + self.callers_graph[callee_key].append(caller_info) + + # Count this call edge for path analysis + edge_key = (caller_key, callee_key) + self.edge_samples[edge_key] += 1 + + def export(self, output_path): + """Export heatmap data as HTML files in a directory. + + Args: + output_path: Path where to create the heatmap output directory + """ + if not self.file_samples: + print("Warning: No heatmap data to export") + return + + try: + output_dir = self._prepare_output_directory(output_path) + file_stats = self._calculate_file_stats() + self._create_file_index(file_stats) + + # Generate individual file reports + self._generate_file_reports(output_dir, file_stats) + + # Generate index page + self._generate_index_html(output_dir / 'index.html', file_stats) + + self._print_export_summary(output_dir, file_stats) + + except Exception as e: + print(f"Error: Failed to export heatmap: {e}") + raise + + def _prepare_output_directory(self, output_path): + """Create output directory for heatmap files.""" + output_dir = Path(output_path) + if output_dir.suffix == '.html': + output_dir = output_dir.with_suffix('') + + try: + output_dir.mkdir(exist_ok=True, parents=True) + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to create output directory {output_dir}: {e}") from e + + return output_dir + + def _create_file_index(self, file_stats: List[FileStats]): + """Create mapping from filenames to HTML file names.""" + self.file_index = { + stat.filename: self.FILE_INDEX_FORMAT.format(i) + for i, stat in enumerate(file_stats) + } + + def _generate_file_reports(self, output_dir, file_stats: List[FileStats]): + """Generate HTML report for each source file.""" + for stat in file_stats: + file_path = output_dir / self.file_index[stat.filename] + line_counts = self.file_samples[stat.filename] + valid_line_counts = {line: count for line, count in line_counts.items() if line >= 0} + + self_counts = self.file_self_samples.get(stat.filename, {}) + valid_self_counts = {line: count for line, count in self_counts.items() if line >= 0} + + self._generate_file_html( + file_path, + stat.filename, + valid_line_counts, + valid_self_counts, + stat + ) + + def _print_export_summary(self, output_dir, file_stats: List[FileStats]): + """Print summary of exported heatmap.""" + print(f"Heatmap output written to {output_dir}/") + print(f" - Index: {output_dir / 'index.html'}") + print(f" - {len(file_stats)} source file(s) analyzed") + + def _calculate_file_stats(self) -> List[FileStats]: + """Calculate statistics for each file. + + Returns: + List of FileStats objects sorted by total samples + """ + file_stats = [] + for filename, line_counts in self.file_samples.items(): + # Skip special frames + if filename in ('~', '...', '.') or filename.startswith('<') or filename.startswith('['): + continue + + # Filter out lines with -1 (special frames) + valid_line_counts = {line: count for line, count in line_counts.items() if line >= 0} + if not valid_line_counts: + continue + + # Get self samples for this file + self_line_counts = self.file_self_samples.get(filename, {}) + valid_self_counts = {line: count for line, count in self_line_counts.items() if line >= 0} + + total_samples = sum(valid_line_counts.values()) + total_self_samples = sum(valid_self_counts.values()) + num_lines = len(valid_line_counts) + max_samples = max(valid_line_counts.values()) + max_self_samples = max(valid_self_counts.values()) if valid_self_counts else 0 + module_name, module_type = extract_module_name(filename, self._path_info) + + file_stats.append(FileStats( + filename=filename, + module_name=module_name, + module_type=module_type, + total_samples=total_samples, + total_self_samples=total_self_samples, + num_lines=num_lines, + max_samples=max_samples, + max_self_samples=max_self_samples, + percentage=0.0 + )) + + # Sort by total samples and calculate percentages + file_stats.sort(key=lambda x: x.total_samples, reverse=True) + if file_stats: + max_total = file_stats[0].total_samples + for stat in file_stats: + stat.percentage = (stat.total_samples / max_total * 100) if max_total > 0 else 0 + + return file_stats + + def _generate_index_html(self, index_path: Path, file_stats: List[FileStats]): + """Generate index.html with list of all profiled files.""" + # Build hierarchical tree + tree = _TreeBuilder.build_file_tree(file_stats) + + # Render tree as HTML + renderer = _HtmlRenderer(self.file_index) + sections_html = renderer.render_hierarchical_html(tree) + + # Format error rate and missed samples with bar classes + error_rate = self.stats.get('error_rate') + if error_rate is not None: + error_rate_str = f"{error_rate:.1f}%" + error_rate_width = min(error_rate, 100) + # Determine bar color class based on rate + if error_rate < 5: + error_rate_class = "good" + elif error_rate < 15: + error_rate_class = "warning" + else: + error_rate_class = "error" + else: + error_rate_str = "N/A" + error_rate_width = 0 + error_rate_class = "good" + + missed_samples = self.stats.get('missed_samples') + if missed_samples is not None: + missed_samples_str = f"{missed_samples:.1f}%" + missed_samples_width = min(missed_samples, 100) + if missed_samples < 5: + missed_samples_class = "good" + elif missed_samples < 15: + missed_samples_class = "warning" + else: + missed_samples_class = "error" + else: + missed_samples_str = "N/A" + missed_samples_width = 0 + missed_samples_class = "good" + + # Populate template + replacements = { + "": f"", + "": f"", + "": self._template_loader.logo_html, + "": str(len(file_stats)), + "": f"{self._total_samples:,}", + "": f"{self.stats.get('duration_sec', 0):.1f}s", + "": f"{self.stats.get('sample_rate', 0):.1f}", + "": error_rate_str, + "": str(error_rate_width), + "": error_rate_class, + "": missed_samples_str, + "": str(missed_samples_width), + "": missed_samples_class, + "": sections_html, + } + + html_content = self._template_loader.index_template + for placeholder, value in replacements.items(): + html_content = html_content.replace(placeholder, value) + + try: + index_path.write_text(html_content, encoding='utf-8') + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to write index file {index_path}: {e}") from e + + def _generate_file_html(self, output_path: Path, filename: str, + line_counts: Dict[int, int], self_counts: Dict[int, int], + file_stat: FileStats): + """Generate HTML for a single source file with heatmap coloring.""" + # Read source file + try: + source_lines = Path(filename).read_text(encoding='utf-8', errors='replace').splitlines() + except (IOError, OSError) as e: + if not (filename.startswith('<') or filename.startswith('[') or + filename in ('~', '...', '.') or len(filename) < 2): + print(f"Warning: Could not read source file {filename}: {e}") + source_lines = [f"# Source file not available: {filename}"] + + # Generate HTML for each line + max_samples = max(line_counts.values()) if line_counts else 1 + max_self_samples = max(self_counts.values()) if self_counts else 1 + code_lines_html = [ + self._build_line_html(line_num, line_content, line_counts, self_counts, + max_samples, max_self_samples, filename) + for line_num, line_content in enumerate(source_lines, start=1) + ] + + # Populate template + replacements = { + "": html.escape(filename), + "": f"{file_stat.total_samples:,}", + "": f"{file_stat.total_self_samples:,}", + "": str(file_stat.num_lines), + "": f"{file_stat.percentage:.2f}", + "": str(file_stat.max_samples), + "": str(file_stat.max_self_samples), + "": ''.join(code_lines_html), + "": f"", + "": f"", + } + + html_content = self._template_loader.file_template + for placeholder, value in replacements.items(): + html_content = html_content.replace(placeholder, value) + + try: + output_path.write_text(html_content, encoding='utf-8') + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to write file {output_path}: {e}") from e + + def _build_line_html(self, line_num: int, line_content: str, + line_counts: Dict[int, int], self_counts: Dict[int, int], + max_samples: int, max_self_samples: int, filename: str) -> str: + """Build HTML for a single line of source code.""" + cumulative_samples = line_counts.get(line_num, 0) + self_samples = self_counts.get(line_num, 0) + + # Calculate colors for both self and cumulative modes + if cumulative_samples > 0: + log_cumulative = math.log(cumulative_samples + 1) + log_max = math.log(max_samples + 1) + cumulative_intensity = log_cumulative / log_max if log_max > 0 else 0 + + if self_samples > 0 and max_self_samples > 0: + log_self = math.log(self_samples + 1) + log_max_self = math.log(max_self_samples + 1) + self_intensity = log_self / log_max_self if log_max_self > 0 else 0 + else: + self_intensity = 0 + + self_display = f"{self_samples:,}" if self_samples > 0 else "" + cumulative_display = f"{cumulative_samples:,}" + tooltip = f"Self: {self_samples:,}, Total: {cumulative_samples:,}" + else: + cumulative_intensity = 0 + self_intensity = 0 + self_display = "" + cumulative_display = "" + tooltip = "" + + # Get navigation buttons + nav_buttons_html = self._build_navigation_buttons(filename, line_num) + + # Build line HTML with intensity data attributes + line_html = html.escape(line_content.rstrip('\n')) + title_attr = f' title="{html.escape(tooltip)}"' if tooltip else "" + + return ( + f'
\n' + f'
{line_num}
\n' + f'
{self_display}
\n' + f'
{cumulative_display}
\n' + f'
{line_html}
\n' + f' {nav_buttons_html}\n' + f'
\n' + ) + + def _build_navigation_buttons(self, filename: str, line_num: int) -> str: + """Build navigation buttons for callers/callees.""" + line_key = (filename, line_num) + caller_list = self._deduplicate_by_function(self.callers_graph.get(line_key, [])) + callee_list = self._deduplicate_by_function(self.call_graph.get(line_key, [])) + + # Get edge counts for each caller/callee + callers_with_counts = self._get_edge_counts(line_key, caller_list, is_caller=True) + callees_with_counts = self._get_edge_counts(line_key, callee_list, is_caller=False) + + # Build navigation buttons with counts + caller_btn = self._create_navigation_button(callers_with_counts, 'caller', '▲') + callee_btn = self._create_navigation_button(callees_with_counts, 'callee', '▼') + + if caller_btn or callee_btn: + return f'
{caller_btn}{callee_btn}
' + return '' + + def _get_edge_counts(self, line_key: Tuple[str, int], + items: List[Tuple[str, int, str]], + is_caller: bool) -> List[Tuple[str, int, str, int]]: + """Get sample counts for each caller/callee edge.""" + result = [] + for file, line, func in items: + edge_line_key = (file, line) + if is_caller: + edge_key = (edge_line_key, line_key) + else: + edge_key = (line_key, edge_line_key) + + count = self.edge_samples.get(edge_key, 0) + result.append((file, line, func, count)) + + result.sort(key=lambda x: x[3], reverse=True) + return result + + def _deduplicate_by_function(self, items: List[Tuple[str, int, str]]) -> List[Tuple[str, int, str]]: + """Remove duplicate entries based on (file, function) key.""" + seen = {} + result = [] + for file, line, func in items: + key = (file, func) + if key not in seen: + seen[key] = True + result.append((file, line, func)) + return result + + def _create_navigation_button(self, items_with_counts: List[Tuple[str, int, str, int]], + btn_class: str, arrow: str) -> str: + """Create HTML for a navigation button with sample counts.""" + # Filter valid items + valid_items = [(f, l, fn, cnt) for f, l, fn, cnt in items_with_counts + if f in self.file_index and l > 0] + if not valid_items: + return "" + + if len(valid_items) == 1: + file, line, func, count = valid_items[0] + target_html = self.file_index[file] + nav_data = json.dumps({'link': f"{target_html}#line-{line}", 'func': func}) + title = f"Go to {btn_class}: {html.escape(func)} ({count:,} samples)" + return f'' + + # Multiple items - create menu + total_samples = sum(cnt for _, _, _, cnt in valid_items) + items_data = [ + { + 'file': os.path.basename(file), + 'func': func, + 'count': count, + 'link': f"{self.file_index[file]}#line-{line}" + } + for file, line, func, count in valid_items + ] + items_json = html.escape(json.dumps(items_data)) + title = f"{len(items_data)} {btn_class}s ({total_samples:,} samples)" + return f'' diff --git a/Lib/profiling/sampling/live_collector/__init__.py b/Lib/profiling/sampling/live_collector/__init__.py new file mode 100644 index 00000000000..175e4610d23 --- /dev/null +++ b/Lib/profiling/sampling/live_collector/__init__.py @@ -0,0 +1,200 @@ +"""Live profiling collector that displays top-like statistics using curses. + + ┌─────────────────────────────┐ + │ Target Python Process │ + │ (being profiled) │ + └──────────────┬──────────────┘ + │ Stack sampling at + │ configured interval + │ (e.g., 10000µs) + ▼ + ┌─────────────────────────────┐ + │ LiveStatsCollector │ + │ ┌───────────────────────┐ │ + │ │ collect() │ │ Aggregates samples + │ │ - Iterates frames │ │ into statistics + │ │ - Updates counters │ │ + │ └───────────┬───────────┘ │ + │ │ │ + │ ▼ │ + │ ┌───────────────────────┐ │ + │ │ Data Storage │ │ + │ │ - result dict │ │ Tracks per-function: + │ │ - direct_calls │ │ • Direct samples + │ │ - cumulative_calls │ │ • Cumulative samples + │ └───────────┬───────────┘ │ • Derived time stats + │ │ │ + │ ▼ │ + │ ┌───────────────────────┐ │ + │ │ Display Update │ │ + │ │ (10Hz by default) │ │ Rate-limited refresh + │ └───────────┬───────────┘ │ + └──────────────┼──────────────┘ + │ + ▼ + ┌─────────────────────────────┐ + │ DisplayInterface │ + │ (Abstract layer) │ + └──────────────┬──────────────┘ + ┌───────┴────────┐ + │ │ + ┌──────────▼────────┐ ┌───▼──────────┐ + │ CursesDisplay │ │ MockDisplay │ + │ - Real terminal │ │ - Testing │ + │ - ncurses backend │ │ - No UI │ + └─────────┬─────────┘ └──────────────┘ + │ + ▼ + ┌─────────────────────────────────────┐ + │ Widget-Based Rendering │ + │ ┌─────────────────────────────────┐ │ + │ │ HeaderWidget │ │ + │ │ • PID, uptime, time, interval │ │ + │ │ • Sample stats & progress bar │ │ + │ │ • Efficiency bar │ │ + │ │ • Thread status & GC stats │ │ + │ │ • Function summary │ │ + │ │ • Top 3 hottest functions │ │ + │ ├─────────────────────────────────┤ │ + │ │ TableWidget │ │ + │ │ • Column headers (sortable) │ │ Interactive display + │ │ • Stats rows (scrolling) │ │ with keyboard controls: + │ │ - nsamples % time │ │ s: sort, p: pause + │ │ - function file:line │ │ r: reset, /: filter + │ ├─────────────────────────────────┤ │ q: quit, h: help + │ │ FooterWidget │ │ + │ │ • Legend and status │ │ + │ │ • Filter input prompt │ │ + │ └─────────────────────────────────┘ │ + └─────────────────────────────────────┘ + +Architecture: + +The live collector is organized into four layers. The data collection layer +(LiveStatsCollector) aggregates stack samples into per-function statistics without +any knowledge of how they will be presented. The display abstraction layer +(DisplayInterface) defines rendering operations without coupling to curses or any +specific UI framework. The widget layer (Widget, HeaderWidget, TableWidget, +FooterWidget, HelpWidget, ProgressBarWidget) encapsulates individual UI components +with their own rendering logic, promoting modularity and reusability. The +presentation layer (CursesDisplay/MockDisplay) implements the actual rendering for +terminal output and testing. + +The system runs two independent update loops. The sampling loop is driven by the +profiler at the configured interval (e.g., 10000µs) and continuously collects +stack frames and updates statistics. The display loop runs at a fixed refresh rate +(default 10Hz) and updates the terminal independently of sampling frequency. This +separation allows high-frequency sampling without overwhelming the terminal with +constant redraws. + +Statistics are computed incrementally as samples arrive. The collector maintains +running counters (direct calls and cumulative calls) in a dictionary keyed by +function location. Derived metrics like time estimates and percentages are computed +on-demand during display updates rather than being stored, which minimizes memory +overhead as the number of tracked functions grows. + +User input is processed asynchronously during display updates using non-blocking I/O. +This allows interactive controls (sorting, filtering, pausing) without interrupting +the data collection pipeline. The collector maintains mode flags (paused, +filter_input_mode) that affect what gets displayed but not what gets collected. + +""" + +# Re-export all public classes and constants for backward compatibility +from .collector import LiveStatsCollector +from .display import DisplayInterface, CursesDisplay, MockDisplay +from .widgets import ( + Widget, + ProgressBarWidget, + HeaderWidget, + TableWidget, + FooterWidget, + HelpWidget, +) +from .constants import ( + MICROSECONDS_PER_SECOND, + DISPLAY_UPDATE_HZ, + DISPLAY_UPDATE_INTERVAL, + MIN_TERMINAL_WIDTH, + MIN_TERMINAL_HEIGHT, + WIDTH_THRESHOLD_SAMPLE_PCT, + WIDTH_THRESHOLD_TOTTIME, + WIDTH_THRESHOLD_CUMUL_PCT, + WIDTH_THRESHOLD_CUMTIME, + HEADER_LINES, + FOOTER_LINES, + SAFETY_MARGIN, + TOP_FUNCTIONS_DISPLAY_COUNT, + COL_WIDTH_NSAMPLES, + COL_SPACING, + COL_WIDTH_SAMPLE_PCT, + COL_WIDTH_TIME, + MIN_FUNC_NAME_WIDTH, + MAX_FUNC_NAME_WIDTH, + MIN_AVAILABLE_SPACE, + MIN_BAR_WIDTH, + MAX_SAMPLE_RATE_BAR_WIDTH, + MAX_EFFICIENCY_BAR_WIDTH, + MIN_SAMPLE_RATE_FOR_SCALING, + FINISHED_BANNER_EXTRA_LINES, + COLOR_PAIR_HEADER_BG, + COLOR_PAIR_CYAN, + COLOR_PAIR_YELLOW, + COLOR_PAIR_GREEN, + COLOR_PAIR_MAGENTA, + COLOR_PAIR_RED, + COLOR_PAIR_SORTED_HEADER, + DEFAULT_SORT_BY, + DEFAULT_DISPLAY_LIMIT, +) + +__all__ = [ + # Main collector + "LiveStatsCollector", + # Display interfaces + "DisplayInterface", + "CursesDisplay", + "MockDisplay", + # Widgets + "Widget", + "ProgressBarWidget", + "HeaderWidget", + "TableWidget", + "FooterWidget", + "HelpWidget", + # Constants + "MICROSECONDS_PER_SECOND", + "DISPLAY_UPDATE_HZ", + "DISPLAY_UPDATE_INTERVAL", + "MIN_TERMINAL_WIDTH", + "MIN_TERMINAL_HEIGHT", + "WIDTH_THRESHOLD_SAMPLE_PCT", + "WIDTH_THRESHOLD_TOTTIME", + "WIDTH_THRESHOLD_CUMUL_PCT", + "WIDTH_THRESHOLD_CUMTIME", + "HEADER_LINES", + "FOOTER_LINES", + "SAFETY_MARGIN", + "TOP_FUNCTIONS_DISPLAY_COUNT", + "COL_WIDTH_NSAMPLES", + "COL_SPACING", + "COL_WIDTH_SAMPLE_PCT", + "COL_WIDTH_TIME", + "MIN_FUNC_NAME_WIDTH", + "MAX_FUNC_NAME_WIDTH", + "MIN_AVAILABLE_SPACE", + "MIN_BAR_WIDTH", + "MAX_SAMPLE_RATE_BAR_WIDTH", + "MAX_EFFICIENCY_BAR_WIDTH", + "MIN_SAMPLE_RATE_FOR_SCALING", + "FINISHED_BANNER_EXTRA_LINES", + "COLOR_PAIR_HEADER_BG", + "COLOR_PAIR_CYAN", + "COLOR_PAIR_YELLOW", + "COLOR_PAIR_GREEN", + "COLOR_PAIR_MAGENTA", + "COLOR_PAIR_RED", + "COLOR_PAIR_SORTED_HEADER", + "DEFAULT_SORT_BY", + "DEFAULT_DISPLAY_LIMIT", +] diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py new file mode 100644 index 00000000000..5edb02e6e88 --- /dev/null +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -0,0 +1,993 @@ +"""LiveStatsCollector - Main collector class for live profiling.""" + +import collections +import contextlib +import curses +from dataclasses import dataclass, field +import os +import site +import sys +import sysconfig +import time +import _colorize + +from ..collector import Collector +from ..constants import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + THREAD_STATUS_UNKNOWN, + THREAD_STATUS_GIL_REQUESTED, + PROFILING_MODE_CPU, + PROFILING_MODE_GIL, + PROFILING_MODE_WALL, +) +from .constants import ( + MICROSECONDS_PER_SECOND, + DISPLAY_UPDATE_INTERVAL, + MIN_TERMINAL_WIDTH, + MIN_TERMINAL_HEIGHT, + HEADER_LINES, + FOOTER_LINES, + SAFETY_MARGIN, + FINISHED_BANNER_EXTRA_LINES, + DEFAULT_SORT_BY, + DEFAULT_DISPLAY_LIMIT, + COLOR_PAIR_HEADER_BG, + COLOR_PAIR_CYAN, + COLOR_PAIR_YELLOW, + COLOR_PAIR_GREEN, + COLOR_PAIR_MAGENTA, + COLOR_PAIR_RED, + COLOR_PAIR_SORTED_HEADER, +) +from .display import CursesDisplay +from .widgets import HeaderWidget, TableWidget, FooterWidget, HelpWidget +from .trend_tracker import TrendTracker + + +@dataclass +class ThreadData: + """Encapsulates all profiling data for a single thread.""" + + thread_id: int + + # Function call statistics: {location: {direct_calls: int, cumulative_calls: int}} + result: dict = field(default_factory=lambda: collections.defaultdict( + lambda: dict(direct_calls=0, cumulative_calls=0) + )) + + # Thread status statistics + has_gil: int = 0 + on_cpu: int = 0 + gil_requested: int = 0 + unknown: int = 0 + total: int = 0 # Total status samples for this thread + + # Sample counts + sample_count: int = 0 + gc_frame_samples: int = 0 + + def increment_status_flag(self, status_flags): + """Update status counts based on status bit flags.""" + if status_flags & THREAD_STATUS_HAS_GIL: + self.has_gil += 1 + if status_flags & THREAD_STATUS_ON_CPU: + self.on_cpu += 1 + if status_flags & THREAD_STATUS_GIL_REQUESTED: + self.gil_requested += 1 + if status_flags & THREAD_STATUS_UNKNOWN: + self.unknown += 1 + self.total += 1 + + def as_status_dict(self): + """Return status counts as a dict for compatibility.""" + return { + "has_gil": self.has_gil, + "on_cpu": self.on_cpu, + "gil_requested": self.gil_requested, + "unknown": self.unknown, + "total": self.total, + } + + +class LiveStatsCollector(Collector): + """Collector that displays live top-like statistics using ncurses.""" + + def __init__( + self, + sample_interval_usec, + *, + skip_idle=False, + sort_by=DEFAULT_SORT_BY, + limit=DEFAULT_DISPLAY_LIMIT, + pid=None, + display=None, + mode=None, + async_aware=None, + ): + """ + Initialize the live stats collector. + + Args: + sample_interval_usec: Sampling interval in microseconds + skip_idle: Whether to skip idle threads + sort_by: Sort key ('tottime', 'nsamples', 'cumtime', 'sample_pct', 'cumul_pct') + limit: Maximum number of functions to display + pid: Process ID being profiled + display: DisplayInterface implementation (None means curses will be used) + mode: Profiling mode ('cpu', 'gil', etc.) - affects what stats are shown + async_aware: Async tracing mode - None (sync only), "all" or "running" + """ + self.result = collections.defaultdict( + lambda: dict(total_rec_calls=0, direct_calls=0, cumulative_calls=0) + ) + self.sample_interval_usec = sample_interval_usec + self.sample_interval_sec = ( + sample_interval_usec / MICROSECONDS_PER_SECOND + ) + self.skip_idle = skip_idle + self.sort_by = sort_by + self.limit = limit + self.total_samples = 0 + self.start_time = None + self.stdscr = None + self.display = display # DisplayInterface implementation + self.running = True + self.pid = pid + self.mode = mode # Profiling mode + self.async_aware = async_aware # Async tracing mode + # Pre-select frame iterator method to avoid per-call dispatch overhead + self._get_frame_iterator = self._get_async_frame_iterator if async_aware else self._get_sync_frame_iterator + self._saved_stdout = None + self._saved_stderr = None + self._devnull = None + self._last_display_update = None + self.max_sample_rate = 0 # Track maximum sample rate seen + self.successful_samples = 0 # Track samples that captured frames + self.failed_samples = 0 # Track samples that failed to capture frames + self.display_update_interval = DISPLAY_UPDATE_INTERVAL # Instance variable for display refresh rate + + # Thread status statistics (bit flags) + self.thread_status_counts = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, # Total thread count across all samples + } + self.gc_frame_samples = 0 # Track samples with GC frames + + # Interactive controls state + self.paused = False # Pause UI updates (profiling continues) + self.show_help = False # Show help screen + self.filter_pattern = None # Glob pattern to filter functions + self.filter_input_mode = False # Currently entering filter text + self.filter_input_buffer = "" # Buffer for filter input + self.finished = False # Program has finished, showing final state + self.finish_timestamp = None # When profiling finished (for time freezing) + self.finish_wall_time = None # Wall clock time when profiling finished + + # Thread tracking state + self.thread_ids = [] # List of thread IDs seen + self.view_mode = "ALL" # "ALL" or "PER_THREAD" + self.current_thread_index = ( + 0 # Index into thread_ids when in PER_THREAD mode + ) + self.per_thread_data = {} # {thread_id: ThreadData} + + # Calculate common path prefixes to strip + self._path_prefixes = self._get_common_path_prefixes() + + # Widgets (initialized when display is available) + self.header_widget = None + self.table_widget = None + self.footer_widget = None + self.help_widget = None + + # Color mode + self._can_colorize = _colorize.can_colorize() + + # Trend tracking (initialized after colors are set up) + self._trend_tracker = None + + @property + def elapsed_time(self): + """Get the elapsed time, frozen when finished.""" + if self.finished and self.finish_timestamp is not None: + return self.finish_timestamp - self.start_time + return time.perf_counter() - self.start_time if self.start_time else 0 + + @property + def current_time_display(self): + """Get the current time for display, frozen when finished.""" + if self.finished and self.finish_wall_time is not None: + return time.strftime("%H:%M:%S", time.localtime(self.finish_wall_time)) + return time.strftime("%H:%M:%S") + + def _get_or_create_thread_data(self, thread_id): + """Get or create ThreadData for a thread ID.""" + if thread_id not in self.per_thread_data: + self.per_thread_data[thread_id] = ThreadData(thread_id=thread_id) + return self.per_thread_data[thread_id] + + def _get_current_thread_data(self): + """Get ThreadData for currently selected thread in PER_THREAD mode.""" + if self.view_mode == "PER_THREAD" and self.current_thread_index < len(self.thread_ids): + thread_id = self.thread_ids[self.current_thread_index] + return self.per_thread_data.get(thread_id) + return None + + def _get_current_result_source(self): + """Get result dict for current view mode (aggregated or per-thread).""" + if self.view_mode == "ALL": + return self.result + thread_data = self._get_current_thread_data() + return thread_data.result if thread_data else {} + + def _get_common_path_prefixes(self): + """Get common path prefixes to strip from file paths.""" + prefixes = [] + + # Get the actual stdlib location from the os module + # This works for both installed Python and development builds + os_module_file = os.__file__ + if os_module_file: + # os.__file__ points to os.py, get its directory + stdlib_dir = os.path.dirname(os.path.abspath(os_module_file)) + prefixes.append(stdlib_dir) + + # Get stdlib location from sysconfig (may be different or same) + stdlib_path = sysconfig.get_path("stdlib") + if stdlib_path: + prefixes.append(stdlib_path) + + # Get platstdlib location (platform-specific stdlib) + platstdlib_path = sysconfig.get_path("platstdlib") + if platstdlib_path: + prefixes.append(platstdlib_path) + + # Get site-packages locations + for site_path in site.getsitepackages(): + prefixes.append(site_path) + + # Also check user site-packages + user_site = site.getusersitepackages() + if user_site: + prefixes.append(user_site) + + # Remove duplicates and sort by length (longest first) to match most specific paths first + prefixes = list(set(prefixes)) + prefixes.sort(key=lambda x: len(x), reverse=True) + + return prefixes + + def simplify_path(self, filepath): + """Simplify a file path by removing common prefixes.""" + # Try to match against known prefixes + for prefix_path in self._path_prefixes: + if filepath.startswith(prefix_path): + # Remove the prefix completely + relative = filepath[len(prefix_path) :].lstrip(os.sep) + return relative + + # If no match, return the original path + return filepath + + def process_frames(self, frames, thread_id=None): + """Process a single thread's frame stack. + + Args: + frames: List of frame information + thread_id: Thread ID for per-thread tracking (optional) + """ + if not frames: + return + + # Get per-thread data if tracking per-thread + thread_data = self._get_or_create_thread_data(thread_id) if thread_id is not None else None + + # Process each frame in the stack to track cumulative calls + for frame in frames: + location = (frame.filename, frame.lineno, frame.funcname) + self.result[location]["cumulative_calls"] += 1 + if thread_data: + thread_data.result[location]["cumulative_calls"] += 1 + + # The top frame gets counted as an inline call (directly executing) + top_location = (frames[0].filename, frames[0].lineno, frames[0].funcname) + self.result[top_location]["direct_calls"] += 1 + if thread_data: + thread_data.result[top_location]["direct_calls"] += 1 + + def _get_sync_frame_iterator(self, stack_frames): + """Iterator for sync frames.""" + return self._iter_all_frames(stack_frames, skip_idle=self.skip_idle) + + def _get_async_frame_iterator(self, stack_frames): + """Iterator for async frames, yielding (frames, thread_id) tuples.""" + for frames, thread_id, task_id in self._iter_async_frames(stack_frames): + yield frames, thread_id + + def collect_failed_sample(self): + self.failed_samples += 1 + self.total_samples += 1 + + def collect(self, stack_frames): + """Collect and display profiling data.""" + if self.start_time is None: + self.start_time = time.perf_counter() + self._last_display_update = self.start_time + + has_gc_frame = False + + # Collect thread status stats (only available in sync mode) + if not self.async_aware: + status_counts, sample_has_gc, per_thread_stats = self._collect_thread_status_stats(stack_frames) + for key, count in status_counts.items(): + self.thread_status_counts[key] += count + if sample_has_gc: + has_gc_frame = True + + for thread_id, stats in per_thread_stats.items(): + thread_data = self._get_or_create_thread_data(thread_id) + thread_data.has_gil += stats.get("has_gil", 0) + thread_data.on_cpu += stats.get("on_cpu", 0) + thread_data.gil_requested += stats.get("gil_requested", 0) + thread_data.unknown += stats.get("unknown", 0) + thread_data.total += stats.get("total", 0) + if stats.get("gc_samples", 0): + thread_data.gc_frame_samples += stats["gc_samples"] + + # Process frames using pre-selected iterator + for frames, thread_id in self._get_frame_iterator(stack_frames): + if not frames: + continue + + self.process_frames(frames, thread_id=thread_id) + + # Track thread IDs + if thread_id is not None and thread_id not in self.thread_ids: + self.thread_ids.append(thread_id) + + if thread_id is not None: + thread_data = self._get_or_create_thread_data(thread_id) + thread_data.sample_count += 1 + + if has_gc_frame: + self.gc_frame_samples += 1 + + self.successful_samples += 1 + self.total_samples += 1 + + # Handle input on every sample for instant responsiveness + if self.display is not None: + self._handle_input() + + # Update display at configured rate if display is initialized and not paused + if self.display is not None and not self.paused: + current_time = time.perf_counter() + if ( + self._last_display_update is None + or (current_time - self._last_display_update) + >= self.display_update_interval + ): + self._update_display() + self._last_display_update = current_time + + def _prepare_display_data(self, height): + """Prepare data for display rendering.""" + elapsed = self.elapsed_time + stats_list = self.build_stats_list() + + # Calculate available space for stats + # Add extra lines for finished banner when in finished state + extra_header_lines = ( + FINISHED_BANNER_EXTRA_LINES if self.finished else 0 + ) + max_stats_lines = max( + 0, + height + - HEADER_LINES + - extra_header_lines + - FOOTER_LINES + - SAFETY_MARGIN, + ) + stats_list = stats_list[:max_stats_lines] + + return elapsed, stats_list + + def _initialize_widgets(self, colors): + """Initialize widgets with display and colors.""" + if self.header_widget is None: + # Initialize trend tracker with colors + if self._trend_tracker is None: + self._trend_tracker = TrendTracker(colors, enabled=True) + + self.header_widget = HeaderWidget(self.display, colors, self) + self.table_widget = TableWidget(self.display, colors, self) + self.footer_widget = FooterWidget(self.display, colors, self) + self.help_widget = HelpWidget(self.display, colors) + + def _render_display_sections( + self, height, width, elapsed, stats_list, colors + ): + """Render all display sections to the screen.""" + line = 0 + try: + # Initialize widgets if not already done + self._initialize_widgets(colors) + + # Render header + line = self.header_widget.render( + line, width, elapsed=elapsed, stats_list=stats_list + ) + + # Render table + line = self.table_widget.render( + line, width, height=height, stats_list=stats_list + ) + + except curses.error: + pass + + def _update_display(self): + """Update the display with current stats.""" + try: + # Clear screen and get dimensions + self.display.clear() + height, width = self.display.get_dimensions() + + # Check terminal size + if width < MIN_TERMINAL_WIDTH or height < MIN_TERMINAL_HEIGHT: + self._show_terminal_too_small(height, width) + self.display.refresh() + return + + # Setup colors and initialize widgets (needed for both help and normal display) + colors = self._setup_colors() + self._initialize_widgets(colors) + + # Show help screen if requested + if self.show_help: + self.help_widget.render(0, width, height=height) + self.display.refresh() + return + + # Prepare data + elapsed, stats_list = self._prepare_display_data(height) + + # Render all sections + self._render_display_sections( + height, width, elapsed, stats_list, colors + ) + + # Footer + self.footer_widget.render(height - 2, width) + + # Show filter input prompt if in filter input mode + if self.filter_input_mode: + self.footer_widget.render_filter_input_prompt( + height - 1, width + ) + + # Refresh display + self.display.redraw() + self.display.refresh() + + except Exception: + pass + + def _cycle_sort(self, reverse=False): + """Cycle through different sort modes in column order. + + Args: + reverse: If True, cycle backwards (right to left), otherwise forward (left to right) + """ + sort_modes = [ + "nsamples", + "sample_pct", + "tottime", + "cumul_pct", + "cumtime", + ] + try: + current_idx = sort_modes.index(self.sort_by) + if reverse: + self.sort_by = sort_modes[(current_idx - 1) % len(sort_modes)] + else: + self.sort_by = sort_modes[(current_idx + 1) % len(sort_modes)] + except ValueError: + self.sort_by = "nsamples" + + def _setup_colors(self): + """Set up color pairs and return color attributes.""" + + A_BOLD = self.display.get_attr("A_BOLD") + A_REVERSE = self.display.get_attr("A_REVERSE") + A_UNDERLINE = self.display.get_attr("A_UNDERLINE") + A_NORMAL = self.display.get_attr("A_NORMAL") + + # Check both curses color support and _colorize.can_colorize() + if self.display.has_colors() and self._can_colorize: + with contextlib.suppress(Exception): + # Color constants (using curses values for compatibility) + COLOR_CYAN = 6 + COLOR_GREEN = 2 + COLOR_YELLOW = 3 + COLOR_BLACK = 0 + COLOR_MAGENTA = 5 + COLOR_RED = 1 + + # Initialize all color pairs used throughout the UI + self.display.init_color_pair( + 1, COLOR_CYAN, -1 + ) # Data colors for stats rows + self.display.init_color_pair(2, COLOR_GREEN, -1) + self.display.init_color_pair(3, COLOR_YELLOW, -1) + self.display.init_color_pair( + COLOR_PAIR_HEADER_BG, COLOR_BLACK, COLOR_GREEN + ) + self.display.init_color_pair( + COLOR_PAIR_CYAN, COLOR_CYAN, COLOR_BLACK + ) + self.display.init_color_pair( + COLOR_PAIR_YELLOW, COLOR_YELLOW, COLOR_BLACK + ) + self.display.init_color_pair( + COLOR_PAIR_GREEN, COLOR_GREEN, COLOR_BLACK + ) + self.display.init_color_pair( + COLOR_PAIR_MAGENTA, COLOR_MAGENTA, COLOR_BLACK + ) + self.display.init_color_pair( + COLOR_PAIR_RED, COLOR_RED, COLOR_BLACK + ) + self.display.init_color_pair( + COLOR_PAIR_SORTED_HEADER, COLOR_BLACK, COLOR_YELLOW + ) + + return { + "header": self.display.get_color_pair(COLOR_PAIR_HEADER_BG) + | A_BOLD, + "cyan": self.display.get_color_pair(COLOR_PAIR_CYAN) + | A_BOLD, + "yellow": self.display.get_color_pair(COLOR_PAIR_YELLOW) + | A_BOLD, + "green": self.display.get_color_pair(COLOR_PAIR_GREEN) + | A_BOLD, + "magenta": self.display.get_color_pair(COLOR_PAIR_MAGENTA) + | A_BOLD, + "red": self.display.get_color_pair(COLOR_PAIR_RED) + | A_BOLD, + "sorted_header": self.display.get_color_pair( + COLOR_PAIR_SORTED_HEADER + ) + | A_BOLD, + "normal_header": A_REVERSE | A_BOLD, + "color_samples": self.display.get_color_pair(1), + "color_file": self.display.get_color_pair(2), + "color_func": self.display.get_color_pair(3), + # Trend colors (stock-like indicators) + "trend_up": self.display.get_color_pair(COLOR_PAIR_GREEN) | A_BOLD, + "trend_down": self.display.get_color_pair(COLOR_PAIR_RED) | A_BOLD, + "trend_stable": A_NORMAL, + } + + # Fallback to non-color attributes + return { + "header": A_REVERSE | A_BOLD, + "cyan": A_BOLD, + "yellow": A_BOLD, + "green": A_BOLD, + "magenta": A_BOLD, + "red": A_BOLD, + "sorted_header": A_REVERSE | A_BOLD | A_UNDERLINE, + "normal_header": A_REVERSE | A_BOLD, + "color_samples": A_NORMAL, + "color_file": A_NORMAL, + "color_func": A_NORMAL, + # Trend colors (fallback to bold/normal for monochrome) + "trend_up": A_BOLD, + "trend_down": A_BOLD, + "trend_stable": A_NORMAL, + } + + def build_stats_list(self): + """Build and sort the statistics list.""" + stats_list = [] + result_source = self._get_current_result_source() + + for func, call_counts in result_source.items(): + # Apply filter if set (using substring matching) + if self.filter_pattern: + filename, lineno, funcname = func + # Simple substring match (case-insensitive) + pattern_lower = self.filter_pattern.lower() + filename_lower = filename.lower() + funcname_lower = funcname.lower() + + # Match if pattern is substring of filename, funcname, or combined + matched = ( + pattern_lower in filename_lower + or pattern_lower in funcname_lower + or pattern_lower in f"{filename_lower}:{funcname_lower}" + ) + if not matched: + continue + + direct_calls = call_counts.get("direct_calls", 0) + cumulative_calls = call_counts.get("cumulative_calls", 0) + total_time = direct_calls * self.sample_interval_sec + cumulative_time = cumulative_calls * self.sample_interval_sec + + # Calculate sample percentages + sample_pct = (direct_calls / self.total_samples * 100) if self.total_samples > 0 else 0 + cumul_pct = (cumulative_calls / self.total_samples * 100) if self.total_samples > 0 else 0 + + # Calculate trends for all columns using TrendTracker + trends = {} + if self._trend_tracker is not None: + trends = self._trend_tracker.update_metrics( + func, + { + 'nsamples': direct_calls, + 'tottime': total_time, + 'cumtime': cumulative_time, + 'sample_pct': sample_pct, + 'cumul_pct': cumul_pct, + } + ) + + stats_list.append( + { + "func": func, + "direct_calls": direct_calls, + "cumulative_calls": cumulative_calls, + "total_time": total_time, + "cumulative_time": cumulative_time, + "trends": trends, # Dictionary of trends for all columns + } + ) + + # Sort the stats + if self.sort_by == "nsamples": + stats_list.sort(key=lambda x: x["direct_calls"], reverse=True) + elif self.sort_by == "tottime": + stats_list.sort(key=lambda x: x["total_time"], reverse=True) + elif self.sort_by == "cumtime": + stats_list.sort(key=lambda x: x["cumulative_time"], reverse=True) + elif self.sort_by == "sample_pct": + stats_list.sort( + key=lambda x: (x["direct_calls"] / self.total_samples * 100) + if self.total_samples > 0 + else 0, + reverse=True, + ) + elif self.sort_by == "cumul_pct": + stats_list.sort( + key=lambda x: ( + x["cumulative_calls"] / self.total_samples * 100 + ) + if self.total_samples > 0 + else 0, + reverse=True, + ) + + return stats_list + + def reset_stats(self): + """Reset all collected statistics.""" + self.result.clear() + self.per_thread_data.clear() + self.thread_ids.clear() + self.view_mode = "ALL" + self.current_thread_index = 0 + self.total_samples = 0 + self.successful_samples = 0 + self.failed_samples = 0 + self.max_sample_rate = 0 + self.thread_status_counts = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + } + self.gc_frame_samples = 0 + # Clear trend tracking + if self._trend_tracker is not None: + self._trend_tracker.clear() + # Reset finished state and finish timestamp + self.finished = False + self.finish_timestamp = None + self.finish_wall_time = None + self.start_time = time.perf_counter() + self._last_display_update = self.start_time + + def mark_finished(self): + """Mark the profiling session as finished.""" + self.finished = True + # Capture the finish timestamp to freeze all timing displays + self.finish_timestamp = time.perf_counter() + self.finish_wall_time = time.time() # Wall clock time for display + # Force a final display update to show the finished message + if self.display is not None: + self._update_display() + + def _handle_finished_input_update(self, had_input): + """Update display after input when program is finished.""" + if self.finished and had_input and self.display is not None: + self._update_display() + + def _show_terminal_too_small(self, height, width): + """Display a message when terminal is too small.""" + A_BOLD = self.display.get_attr("A_BOLD") + msg1 = "Terminal too small!" + msg2 = f"Need: {MIN_TERMINAL_WIDTH}x{MIN_TERMINAL_HEIGHT}" + msg3 = f"Have: {width}x{height}" + msg4 = "Please resize" + + # Center the messages + if height >= 4: + self.display.add_str( + height // 2 - 2, + max(0, (width - len(msg1)) // 2), + msg1[: width - 1], + A_BOLD, + ) + self.display.add_str( + height // 2 - 1, + max(0, (width - len(msg2)) // 2), + msg2[: width - 1], + ) + self.display.add_str( + height // 2, + max(0, (width - len(msg3)) // 2), + msg3[: width - 1], + ) + self.display.add_str( + height // 2 + 1, + max(0, (width - len(msg4)) // 2), + msg4[: width - 1], + ) + elif height >= 1: + self.display.add_str(0, 0, msg1[: width - 1], A_BOLD) + + def _show_terminal_size_warning_and_wait(self, height, width): + """Show terminal size warning during initialization and wait for user acknowledgment.""" + A_BOLD = self.display.get_attr("A_BOLD") + A_DIM = self.display.get_attr("A_DIM") + + self.display.clear() + msg1 = "WARNING: Terminal too small!" + msg2 = f"Required: {MIN_TERMINAL_WIDTH}x{MIN_TERMINAL_HEIGHT}" + msg3 = f"Current: {width}x{height}" + msg4 = "Please resize your terminal for best experience" + msg5 = "Press any key to continue..." + + # Center the messages + if height >= 5: + self.display.add_str( + height // 2 - 2, + max(0, (width - len(msg1)) // 2), + msg1[: width - 1], + A_BOLD, + ) + self.display.add_str( + height // 2 - 1, + max(0, (width - len(msg2)) // 2), + msg2[: width - 1], + ) + self.display.add_str( + height // 2, + max(0, (width - len(msg3)) // 2), + msg3[: width - 1], + ) + self.display.add_str( + height // 2 + 1, + max(0, (width - len(msg4)) // 2), + msg4[: width - 1], + ) + self.display.add_str( + height // 2 + 3, + max(0, (width - len(msg5)) // 2), + msg5[: width - 1], + A_DIM, + ) + elif height >= 1: + self.display.add_str(0, 0, msg1[: width - 1], A_BOLD) + + self.display.refresh() + # Wait for user acknowledgment (2 seconds timeout) + self.display.set_nodelay(False) + # Note: timeout is curses-specific, skipping for now + self.display.get_input() + self.display.set_nodelay(True) + + def _handle_input(self): + """Handle keyboard input (non-blocking).""" + from . import constants + + self.display.set_nodelay(True) + ch = self.display.get_input() + + # Handle filter input mode FIRST - takes precedence over all commands + if self.filter_input_mode: + if ch == 27: # ESC key + self.filter_input_mode = False + self.filter_input_buffer = "" + elif ch == 10 or ch == 13: # Enter key + self.filter_pattern = ( + self.filter_input_buffer + if self.filter_input_buffer + else None + ) + self.filter_input_mode = False + self.filter_input_buffer = "" + elif ch == 127 or ch == 263: # Backspace + if self.filter_input_buffer: + self.filter_input_buffer = self.filter_input_buffer[:-1] + elif ch >= 32 and ch < 127: # Printable characters + self.filter_input_buffer += chr(ch) + + # Update display if input was processed while finished + self._handle_finished_input_update(ch != -1) + return + + # Handle help toggle keys + if ch == ord("h") or ch == ord("H") or ch == ord("?"): + self.show_help = not self.show_help + return + + # If showing help, any other key closes it + if self.show_help and ch != -1: + self.show_help = False + return + + # Handle regular commands + if ch == ord("q") or ch == ord("Q"): + self.running = False + + elif ch == ord("s"): + self._cycle_sort(reverse=False) + + elif ch == ord("S"): + self._cycle_sort(reverse=True) + + elif ch == ord("p") or ch == ord("P"): + self.paused = not self.paused + + elif ch == ord("r") or ch == ord("R"): + # Don't allow reset when profiling is finished + if not self.finished: + self.reset_stats() + + elif ch == ord("+") or ch == ord("="): + # Decrease update interval (faster refresh) + self.display_update_interval = max( + 0.05, self.display_update_interval - 0.05 + ) # Min 20Hz + + elif ch == ord("-") or ch == ord("_"): + # Increase update interval (slower refresh) + self.display_update_interval = min( + 1.0, self.display_update_interval + 0.05 + ) # Max 1Hz + + elif ch == ord("c") or ch == ord("C"): + if self.filter_pattern: + self.filter_pattern = None + + elif ch == ord("/"): + self.filter_input_mode = True + self.filter_input_buffer = self.filter_pattern or "" + + elif ch == ord("t") or ch == ord("T"): + # Toggle between ALL and PER_THREAD modes + if self.view_mode == "ALL": + if len(self.thread_ids) > 0: + self.view_mode = "PER_THREAD" + self.current_thread_index = 0 + else: + self.view_mode = "ALL" + + elif ch == ord("x") or ch == ord("X"): + # Toggle trend colors on/off + if self._trend_tracker is not None: + self._trend_tracker.toggle() + + elif ch == curses.KEY_LEFT or ch == curses.KEY_UP: + # Navigate to previous thread in PER_THREAD mode, or switch from ALL to PER_THREAD + if len(self.thread_ids) > 0: + if self.view_mode == "ALL": + self.view_mode = "PER_THREAD" + self.current_thread_index = 0 + else: + self.current_thread_index = ( + self.current_thread_index - 1 + ) % len(self.thread_ids) + + elif ch == curses.KEY_RIGHT or ch == curses.KEY_DOWN: + # Navigate to next thread in PER_THREAD mode, or switch from ALL to PER_THREAD + if len(self.thread_ids) > 0: + if self.view_mode == "ALL": + self.view_mode = "PER_THREAD" + self.current_thread_index = 0 + else: + self.current_thread_index = ( + self.current_thread_index + 1 + ) % len(self.thread_ids) + + # Update display if input was processed while finished + self._handle_finished_input_update(ch != -1) + + def init_curses(self, stdscr): + """Initialize curses display and suppress stdout/stderr.""" + self.stdscr = stdscr + self.display = CursesDisplay(stdscr) + + # Check terminal size upfront and warn if too small + height, width = self.display.get_dimensions() + + if width < MIN_TERMINAL_WIDTH or height < MIN_TERMINAL_HEIGHT: + # Show warning and wait briefly for user to see it + self._show_terminal_size_warning_and_wait(height, width) + + curses.curs_set(0) # Hide cursor + stdscr.nodelay(True) # Non-blocking input + stdscr.scrollok(False) # Disable scrolling + stdscr.idlok(False) # Disable hardware insert/delete + stdscr.leaveok(True) # Don't care about cursor position + + if curses.has_colors(): + curses.start_color() + curses.use_default_colors() + + # Suppress stdout and stderr to prevent interfering with curses display + # Use contextlib.redirect_stdout/stderr for better resource management + self._saved_stdout = sys.stdout + self._saved_stderr = sys.stderr + # Open devnull and ensure it's cleaned up even if an exception occurs + try: + self._devnull = open(os.devnull, "w") + sys.stdout = self._devnull + sys.stderr = self._devnull + except Exception: + # If redirection fails, restore original streams + sys.stdout = self._saved_stdout + sys.stderr = self._saved_stderr + raise + + # Initial clear + self.display.clear() + self.display.refresh() + + def cleanup_curses(self): + """Clean up curses display and restore stdout/stderr.""" + # Restore stdout and stderr in reverse order + # Use try-finally to ensure cleanup even if restoration fails + try: + if self._saved_stdout is not None: + sys.stdout = self._saved_stdout + self._saved_stdout = None + if self._saved_stderr is not None: + sys.stderr = self._saved_stderr + self._saved_stderr = None + finally: + # Always close devnull, even if stdout/stderr restoration fails + if self._devnull is not None: + with contextlib.suppress(Exception): + self._devnull.close() + self._devnull = None + + if self.display is not None and self.stdscr is not None: + with contextlib.suppress(Exception): + curses.curs_set(1) # Show cursor + self.display.set_nodelay(False) + + def export(self, filename): + """Export is not supported in live mode.""" + raise NotImplementedError( + "Export to file is not supported in live mode. " + "Use the live TUI to view statistics in real-time." + ) diff --git a/Lib/profiling/sampling/live_collector/constants.py b/Lib/profiling/sampling/live_collector/constants.py new file mode 100644 index 00000000000..e4690c90baf --- /dev/null +++ b/Lib/profiling/sampling/live_collector/constants.py @@ -0,0 +1,59 @@ +"""Constants for the live profiling collector.""" + +# Time conversion constants +MICROSECONDS_PER_SECOND = 1_000_000 + +# Display update constants +DISPLAY_UPDATE_HZ = 10 +DISPLAY_UPDATE_INTERVAL = 1.0 / DISPLAY_UPDATE_HZ # 0.1 seconds + +# Terminal size constraints +MIN_TERMINAL_WIDTH = 60 +MIN_TERMINAL_HEIGHT = 12 + +# Column width thresholds +WIDTH_THRESHOLD_SAMPLE_PCT = 80 +WIDTH_THRESHOLD_TOTTIME = 100 +WIDTH_THRESHOLD_CUMUL_PCT = 120 +WIDTH_THRESHOLD_CUMTIME = 140 + +# Display layout constants +HEADER_LINES = 10 # Increased to include thread status line +FOOTER_LINES = 2 +SAFETY_MARGIN = 1 +TOP_FUNCTIONS_DISPLAY_COUNT = 3 + +# Column widths for data display +COL_WIDTH_NSAMPLES = 13 +COL_SPACING = 2 +COL_WIDTH_SAMPLE_PCT = 5 +COL_WIDTH_TIME = 10 + +# Function name display +MIN_FUNC_NAME_WIDTH = 10 +MAX_FUNC_NAME_WIDTH = 40 +MIN_AVAILABLE_SPACE = 10 + +# Progress bar display +MIN_BAR_WIDTH = 10 +MAX_SAMPLE_RATE_BAR_WIDTH = 30 +MAX_EFFICIENCY_BAR_WIDTH = 60 + +# Sample rate scaling +MIN_SAMPLE_RATE_FOR_SCALING = 100 + +# Finished banner display +FINISHED_BANNER_EXTRA_LINES = 3 # Blank line + banner + blank line + +# Color pair IDs +COLOR_PAIR_HEADER_BG = 4 +COLOR_PAIR_CYAN = 5 +COLOR_PAIR_YELLOW = 6 +COLOR_PAIR_GREEN = 7 +COLOR_PAIR_MAGENTA = 8 +COLOR_PAIR_RED = 9 +COLOR_PAIR_SORTED_HEADER = 10 + +# Default display settings +DEFAULT_SORT_BY = "nsamples" # Number of samples in leaf (self time) +DEFAULT_DISPLAY_LIMIT = 20 diff --git a/Lib/profiling/sampling/live_collector/display.py b/Lib/profiling/sampling/live_collector/display.py new file mode 100644 index 00000000000..d7f65ad73fd --- /dev/null +++ b/Lib/profiling/sampling/live_collector/display.py @@ -0,0 +1,236 @@ +"""Display interface abstractions for the live profiling collector.""" + +import contextlib +import curses +from abc import ABC, abstractmethod + + +class DisplayInterface(ABC): + """Abstract interface for display operations to enable testing.""" + + @abstractmethod + def get_dimensions(self): + """Get terminal dimensions as (height, width).""" + pass + + @abstractmethod + def clear(self): + """Clear the screen.""" + pass + + @abstractmethod + def refresh(self): + """Refresh the screen to show changes.""" + pass + + @abstractmethod + def redraw(self): + """Redraw the entire window.""" + pass + + @abstractmethod + def add_str(self, line, col, text, attr=0): + """Add a string at the specified position.""" + pass + + @abstractmethod + def get_input(self): + """Get a character from input (non-blocking). Returns -1 if no input.""" + pass + + @abstractmethod + def set_nodelay(self, flag): + """Set non-blocking mode for input.""" + pass + + @abstractmethod + def has_colors(self): + """Check if terminal supports colors.""" + pass + + @abstractmethod + def init_color_pair(self, pair_id, fg, bg): + """Initialize a color pair.""" + pass + + @abstractmethod + def get_color_pair(self, pair_id): + """Get a color pair attribute.""" + pass + + @abstractmethod + def get_attr(self, name): + """Get a display attribute by name (e.g., 'A_BOLD', 'A_REVERSE').""" + pass + + +class CursesDisplay(DisplayInterface): + """Real curses display implementation.""" + + def __init__(self, stdscr): + self.stdscr = stdscr + + def get_dimensions(self): + return self.stdscr.getmaxyx() + + def clear(self): + self.stdscr.clear() + + def refresh(self): + self.stdscr.refresh() + + def redraw(self): + self.stdscr.redrawwin() + + def add_str(self, line, col, text, attr=0): + try: + height, width = self.get_dimensions() + if 0 <= line < height and 0 <= col < width: + max_len = width - col - 1 + if len(text) > max_len: + text = text[:max_len] + self.stdscr.addstr(line, col, text, attr) + except curses.error: + pass + + def get_input(self): + try: + return self.stdscr.getch() + except (KeyError, curses.error): + return -1 + + def set_nodelay(self, flag): + self.stdscr.nodelay(flag) + + def has_colors(self): + return curses.has_colors() + + def init_color_pair(self, pair_id, fg, bg): + try: + curses.init_pair(pair_id, fg, bg) + except curses.error: + pass + + def get_color_pair(self, pair_id): + return curses.color_pair(pair_id) + + def get_attr(self, name): + return getattr(curses, name, 0) + + +class MockDisplay(DisplayInterface): + """Mock display for testing.""" + + def __init__(self, height=40, width=160): + self.height = height + self.width = width + self.buffer = {} + self.cleared = False + self.refreshed = False + self.redrawn = False + self.input_queue = [] + self.nodelay_flag = True + self.colors_supported = True + self.color_pairs = {} + + def get_dimensions(self): + return (self.height, self.width) + + def clear(self): + self.buffer.clear() + self.cleared = True + + def refresh(self): + self.refreshed = True + + def redraw(self): + self.redrawn = True + + def add_str(self, line, col, text, attr=0): + if 0 <= line < self.height and 0 <= col < self.width: + max_len = self.width - col - 1 + if len(text) > max_len: + text = text[:max_len] + self.buffer[(line, col)] = (text, attr) + + def get_input(self): + if self.input_queue: + return self.input_queue.pop(0) + return -1 + + def set_nodelay(self, flag): + self.nodelay_flag = flag + + def has_colors(self): + return self.colors_supported + + def init_color_pair(self, pair_id, fg, bg): + self.color_pairs[pair_id] = (fg, bg) + + def get_color_pair(self, pair_id): + return pair_id << 8 + + def get_attr(self, name): + attrs = { + "A_NORMAL": 0, + "A_BOLD": 1 << 16, + "A_REVERSE": 1 << 17, + "A_UNDERLINE": 1 << 18, + "A_DIM": 1 << 19, + } + return attrs.get(name, 0) + + def simulate_input(self, char): + """Helper method for tests to simulate keyboard input.""" + self.input_queue.append(char) + + def get_text_at(self, line, col): + """Helper method for tests to inspect buffer content.""" + if (line, col) in self.buffer: + return self.buffer[(line, col)][0] + return None + + def get_all_lines(self): + """Get all display content as a list of lines (for testing).""" + if not self.buffer: + return [] + + max_line = max(pos[0] for pos in self.buffer.keys()) + lines = [] + for line_num in range(max_line + 1): + line_parts = [] + for col in range(self.width): + if (line_num, col) in self.buffer: + text, _ = self.buffer[(line_num, col)] + line_parts.append((col, text)) + + # Reconstruct line from parts + if line_parts: + line_parts.sort(key=lambda x: x[0]) + line = "" + last_col = 0 + for col, text in line_parts: + if col > last_col: + line += " " * (col - last_col) + line += text + last_col = col + len(text) + lines.append(line.rstrip()) + else: + lines.append("") + + # Remove trailing empty lines + while lines and not lines[-1]: + lines.pop() + + return lines + + def find_text(self, pattern): + """Find text matching pattern in buffer (for testing). Returns (line, col) or None.""" + for (line, col), (text, _) in self.buffer.items(): + if pattern in text: + return (line, col) + return None + + def contains_text(self, text): + """Check if display contains the given text anywhere (for testing).""" + return self.find_text(text) is not None diff --git a/Lib/profiling/sampling/live_collector/trend_tracker.py b/Lib/profiling/sampling/live_collector/trend_tracker.py new file mode 100644 index 00000000000..c025b83a134 --- /dev/null +++ b/Lib/profiling/sampling/live_collector/trend_tracker.py @@ -0,0 +1,157 @@ +"""TrendTracker - Encapsulated trend tracking for live profiling metrics. + +This module provides trend tracking functionality for profiling metrics, +calculating direction indicators (up/down/stable) and managing associated +visual attributes like colors. +""" + +import curses +from typing import Dict, Literal, Any + +TrendDirection = Literal["up", "down", "stable"] + + +class TrendTracker: + """ + Tracks metric trends over time and provides visual indicators. + + This class encapsulates all logic for: + - Tracking previous values of metrics + - Calculating trend directions (up/down/stable) + - Determining visual attributes (colors) for trends + - Managing enable/disable state + + Example: + tracker = TrendTracker(colors_dict) + tracker.update("func1", "nsamples", 10) + trend = tracker.get_trend("func1", "nsamples") + color = tracker.get_color("func1", "nsamples") + """ + + # Threshold for determining if a value has changed significantly + CHANGE_THRESHOLD = 0.001 + + def __init__(self, colors: Dict[str, int], enabled: bool = True): + """ + Initialize the trend tracker. + + Args: + colors: Dictionary containing color attributes including + 'trend_up', 'trend_down', 'trend_stable' + enabled: Whether trend tracking is initially enabled + """ + self._previous_values: Dict[Any, Dict[str, float]] = {} + self._enabled = enabled + self._colors = colors + + @property + def enabled(self) -> bool: + """Whether trend tracking is enabled.""" + return self._enabled + + def toggle(self) -> bool: + """ + Toggle trend tracking on/off. + + Returns: + New enabled state + """ + self._enabled = not self._enabled + return self._enabled + + def set_enabled(self, enabled: bool) -> None: + """Set trend tracking enabled state.""" + self._enabled = enabled + + def update(self, key: Any, metric: str, value: float) -> TrendDirection: + """ + Update a metric value and calculate its trend. + + Args: + key: Identifier for the entity (e.g., function) + metric: Name of the metric (e.g., 'nsamples', 'tottime') + value: Current value of the metric + + Returns: + Trend direction: 'up', 'down', or 'stable' + """ + # Initialize storage for this key if needed + if key not in self._previous_values: + self._previous_values[key] = {} + + # Get previous value, defaulting to current if not tracked yet + prev_value = self._previous_values[key].get(metric, value) + + # Calculate trend + if value > prev_value + self.CHANGE_THRESHOLD: + trend = "up" + elif value < prev_value - self.CHANGE_THRESHOLD: + trend = "down" + else: + trend = "stable" + + # Update previous value for next iteration + self._previous_values[key][metric] = value + + return trend + + def get_trend(self, key: Any, metric: str) -> TrendDirection: + """ + Get the current trend for a metric without updating. + + Args: + key: Identifier for the entity + metric: Name of the metric + + Returns: + Trend direction, or 'stable' if not tracked + """ + # This would require storing trends separately, which we don't do + # For now, return stable if not found + return "stable" + + def get_color(self, trend: TrendDirection) -> int: + """ + Get the color attribute for a trend direction. + + Args: + trend: The trend direction + + Returns: + Curses color attribute (or A_NORMAL if disabled) + """ + if not self._enabled: + return curses.A_NORMAL + + if trend == "up": + return self._colors.get("trend_up", curses.A_BOLD) + elif trend == "down": + return self._colors.get("trend_down", curses.A_BOLD) + else: # stable + return self._colors.get("trend_stable", curses.A_NORMAL) + + def update_metrics(self, key: Any, metrics: Dict[str, float]) -> Dict[str, TrendDirection]: + """ + Update multiple metrics at once and get their trends. + + Args: + key: Identifier for the entity + metrics: Dictionary of metric_name -> value + + Returns: + Dictionary of metric_name -> trend_direction + """ + trends = {} + for metric, value in metrics.items(): + trends[metric] = self.update(key, metric, value) + return trends + + def clear(self) -> None: + """Clear all tracked values (useful on stats reset).""" + self._previous_values.clear() + + def __repr__(self) -> str: + """String representation for debugging.""" + status = "enabled" if self._enabled else "disabled" + tracked = len(self._previous_values) + return f"TrendTracker({status}, tracking {tracked} entities)" diff --git a/Lib/profiling/sampling/live_collector/widgets.py b/Lib/profiling/sampling/live_collector/widgets.py new file mode 100644 index 00000000000..2af8caa2c2f --- /dev/null +++ b/Lib/profiling/sampling/live_collector/widgets.py @@ -0,0 +1,963 @@ +"""Widget classes for the live profiling collector UI.""" + +import curses +import time +from abc import ABC, abstractmethod + +from .constants import ( + TOP_FUNCTIONS_DISPLAY_COUNT, + MIN_FUNC_NAME_WIDTH, + MAX_FUNC_NAME_WIDTH, + WIDTH_THRESHOLD_SAMPLE_PCT, + WIDTH_THRESHOLD_TOTTIME, + WIDTH_THRESHOLD_CUMUL_PCT, + WIDTH_THRESHOLD_CUMTIME, + MICROSECONDS_PER_SECOND, + DISPLAY_UPDATE_INTERVAL, + MIN_BAR_WIDTH, + MAX_SAMPLE_RATE_BAR_WIDTH, + MAX_EFFICIENCY_BAR_WIDTH, + MIN_SAMPLE_RATE_FOR_SCALING, + FOOTER_LINES, + FINISHED_BANNER_EXTRA_LINES, +) +from ..constants import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + THREAD_STATUS_UNKNOWN, + THREAD_STATUS_GIL_REQUESTED, + PROFILING_MODE_CPU, + PROFILING_MODE_GIL, + PROFILING_MODE_WALL, +) + + +class Widget(ABC): + """Base class for UI widgets.""" + + def __init__(self, display, colors): + """ + Initialize widget. + + Args: + display: DisplayInterface implementation + colors: Dictionary of color attributes + """ + self.display = display + self.colors = colors + + @abstractmethod + def render(self, line, width, **kwargs): + """ + Render the widget starting at the given line. + + Args: + line: Starting line number + width: Available width + **kwargs: Additional rendering parameters + + Returns: + Next available line number after rendering + """ + pass + + def add_str(self, line, col, text, attr=0): + """Add a string to the display at the specified position.""" + self.display.add_str(line, col, text, attr) + + +class ProgressBarWidget(Widget): + """Reusable progress bar widget.""" + + def render(self, line, width, **kwargs): + """Render is not used for progress bars - use render_bar instead.""" + raise NotImplementedError("Use render_bar method instead") + + def render_bar( + self, filled, total, max_width, fill_char="█", empty_char="░" + ): + """ + Render a progress bar and return the bar string and its length. + + Args: + filled: Current filled amount + total: Total amount (max value) + max_width: Maximum width for the bar + fill_char: Character to use for filled portion + empty_char: Character to use for empty portion + + Returns: + Tuple of (bar_string, bar_length) + """ + bar_width = min(max_width, max_width) + normalized = min(filled / max(total, 1), 1.0) + bar_fill = int(normalized * bar_width) + + bar = "[" + for i in range(bar_width): + if i < bar_fill: + bar += fill_char + else: + bar += empty_char + bar += "]" + return bar, len(bar) + + +class HeaderWidget(Widget): + """Widget for rendering the header section (lines 0-8).""" + + def __init__(self, display, colors, collector): + """ + Initialize header widget. + + Args: + display: DisplayInterface implementation + colors: Dictionary of color attributes + collector: Reference to LiveStatsCollector for accessing stats + """ + super().__init__(display, colors) + self.collector = collector + self.progress_bar = ProgressBarWidget(display, colors) + + def render(self, line, width, **kwargs): + """ + Render the complete header section. + + Args: + line: Starting line number + width: Available width + kwargs: Must contain 'elapsed' key + + Returns: + Next available line number + """ + elapsed = kwargs["elapsed"] + + line = self.draw_header_info(line, width, elapsed) + line = self.draw_sample_stats(line, width, elapsed) + line = self.draw_efficiency_bar(line, width) + line = self.draw_thread_status(line, width) + line = self.draw_function_stats( + line, width, kwargs.get("stats_list", []) + ) + line = self.draw_top_functions( + line, width, kwargs.get("stats_list", []) + ) + + # Show prominent finished banner if profiling is complete + if self.collector.finished: + line = self.draw_finished_banner(line, width) + + # Separator + A_DIM = self.display.get_attr("A_DIM") + separator = "─" * (width - 1) + self.add_str(line, 0, separator[: width - 1], A_DIM) + line += 1 + + return line + + def format_uptime(self, elapsed): + """Format elapsed time as uptime string.""" + uptime_sec = int(elapsed) + hours = uptime_sec // 3600 + minutes = (uptime_sec % 3600) // 60 + seconds = uptime_sec % 60 + if hours > 0: + return f"{hours}h{minutes:02d}m{seconds:02d}s" + else: + return f"{minutes}m{seconds:02d}s" + + def draw_header_info(self, line, width, elapsed): + """Draw the header information line with PID, uptime, time, and interval.""" + # Draw title + A_BOLD = self.display.get_attr("A_BOLD") + title = "Tachyon Profiler" + self.add_str(line, 0, title, A_BOLD | self.colors["cyan"]) + line += 1 + + current_time = self.collector.current_time_display + uptime = self.format_uptime(elapsed) + + # Calculate display refresh rate + refresh_hz = ( + 1.0 / self.collector.display_update_interval if self.collector.display_update_interval > 0 else 0 + ) + + # Get current view mode and thread display + if self.collector.view_mode == "ALL": + thread_name = "ALL" + thread_color = self.colors["green"] + else: + # PER_THREAD mode + if self.collector.current_thread_index < len( + self.collector.thread_ids + ): + thread_id = self.collector.thread_ids[ + self.collector.current_thread_index + ] + num_threads = len(self.collector.thread_ids) + thread_name = f"{thread_id} ({self.collector.current_thread_index + 1}/{num_threads})" + thread_color = self.colors["magenta"] + else: + thread_name = "ALL" + thread_color = self.colors["green"] + + header_parts = [ + ("PID: ", curses.A_BOLD), + (f"{self.collector.pid}", self.colors["cyan"]), + (" │ ", curses.A_DIM), + ("Thread: ", curses.A_BOLD), + (thread_name, thread_color), + (" │ ", curses.A_DIM), + ("Uptime: ", curses.A_BOLD), + (uptime, self.colors["green"]), + (" │ ", curses.A_DIM), + ("Time: ", curses.A_BOLD), + (current_time, self.colors["yellow"]), + (" │ ", curses.A_DIM), + ("Interval: ", curses.A_BOLD), + ( + f"{self.collector.sample_interval_usec}µs", + self.colors["magenta"], + ), + (" │ ", curses.A_DIM), + ("Display: ", curses.A_BOLD), + (f"{refresh_hz:.1f}Hz", self.colors["cyan"]), + ] + + col = 0 + for text, attr in header_parts: + if col < width - 1: + self.add_str(line, col, text, attr) + col += len(text) + return line + 1 + + def format_rate_with_units(self, rate_hz): + """Format a rate in Hz with appropriate units (Hz, KHz, MHz).""" + if rate_hz >= 1_000_000: + return f"{rate_hz / 1_000_000:.1f}MHz" + elif rate_hz >= 1_000: + return f"{rate_hz / 1_000:.1f}KHz" + else: + return f"{rate_hz:.1f}Hz" + + def draw_sample_stats(self, line, width, elapsed): + """Draw sample statistics with visual progress bar.""" + sample_rate = ( + self.collector.total_samples / elapsed if elapsed > 0 else 0 + ) + + # Update max sample rate + if sample_rate > self.collector.max_sample_rate: + self.collector.max_sample_rate = sample_rate + + col = 0 + self.add_str(line, col, "Samples: ", curses.A_BOLD) + col += 9 + self.add_str( + line, + col, + f"{self.collector.total_samples:>8}", + self.colors["cyan"], + ) + col += 8 + self.add_str( + line, col, f" total ({sample_rate:>7.1f}/s) ", curses.A_NORMAL + ) + col += 23 + + # Draw sample rate bar + target_rate = ( + MICROSECONDS_PER_SECOND / self.collector.sample_interval_usec + ) + + # Show current/target ratio with percentage + if sample_rate > 0 and target_rate > 0: + percentage = min((sample_rate / target_rate) * 100, 100) + current_formatted = self.format_rate_with_units(sample_rate) + target_formatted = self.format_rate_with_units(target_rate) + + if percentage >= 99.5: # Show 100% when very close + rate_label = f" {current_formatted}/{target_formatted} (100%)" + else: + rate_label = f" {current_formatted}/{target_formatted} ({percentage:>4.1f}%)" + else: + target_formatted = self.format_rate_with_units(target_rate) + rate_label = f" target: {target_formatted}" + + available_width = width - col - len(rate_label) - 3 + + if available_width >= MIN_BAR_WIDTH: + bar_width = min(MAX_SAMPLE_RATE_BAR_WIDTH, available_width) + # Use target rate as the reference, with a minimum for scaling + reference_rate = max(target_rate, MIN_SAMPLE_RATE_FOR_SCALING) + normalized_rate = min(sample_rate / reference_rate, 1.0) + bar_fill = int(normalized_rate * bar_width) + + bar = "[" + for i in range(bar_width): + bar += "█" if i < bar_fill else "░" + bar += "]" + self.add_str(line, col, bar, self.colors["green"]) + col += len(bar) + + if col + len(rate_label) < width - 1: + self.add_str(line, col + 1, rate_label, curses.A_DIM) + return line + 1 + + def draw_efficiency_bar(self, line, width): + """Draw sample efficiency bar showing success/failure rates.""" + success_pct = ( + self.collector.successful_samples + / max(1, self.collector.total_samples) + ) * 100 + failed_pct = ( + self.collector.failed_samples + / max(1, self.collector.total_samples) + ) * 100 + + col = 0 + self.add_str(line, col, "Efficiency:", curses.A_BOLD) + col += 11 + + label = f" {success_pct:>5.2f}% good, {failed_pct:>4.2f}% failed" + available_width = width - col - len(label) - 3 + + if available_width >= MIN_BAR_WIDTH: + bar_width = min(MAX_EFFICIENCY_BAR_WIDTH, available_width) + success_fill = int( + ( + self.collector.successful_samples + / max(1, self.collector.total_samples) + ) + * bar_width + ) + failed_fill = bar_width - success_fill + + self.add_str(line, col, "[", curses.A_NORMAL) + col += 1 + if success_fill > 0: + self.add_str( + line, col, "█" * success_fill, self.colors["green"] + ) + col += success_fill + if failed_fill > 0: + self.add_str(line, col, "█" * failed_fill, self.colors["red"]) + col += failed_fill + self.add_str(line, col, "]", curses.A_NORMAL) + col += 1 + + self.add_str(line, col + 1, label, curses.A_NORMAL) + return line + 1 + + def _add_percentage_stat( + self, line, col, value, label, color, add_separator=False + ): + """Add a percentage stat to the display. + + Args: + line: Line number + col: Starting column + value: Percentage value + label: Label text + color: Color attribute + add_separator: Whether to add separator before the stat + + Returns: + Updated column position + """ + if add_separator: + self.add_str(line, col, " │ ", curses.A_DIM) + col += 3 + + self.add_str(line, col, f"{value:>4.1f}", color) + col += 4 + self.add_str(line, col, f"% {label}", curses.A_NORMAL) + col += len(label) + 2 + + return col + + def draw_thread_status(self, line, width): + """Draw thread status statistics and GC information.""" + # Get status counts for current view mode + thread_data = self.collector._get_current_thread_data() + status_counts = thread_data.as_status_dict() if thread_data else self.collector.thread_status_counts + + # Calculate percentages + total_threads = max(1, status_counts["total"]) + pct_on_gil = (status_counts["has_gil"] / total_threads) * 100 + pct_off_gil = 100.0 - pct_on_gil + pct_gil_requested = (status_counts["gil_requested"] / total_threads) * 100 + + # Get GC percentage based on view mode + if thread_data: + total_samples = max(1, thread_data.sample_count) + pct_gc = (thread_data.gc_frame_samples / total_samples) * 100 + else: + total_samples = max(1, self.collector.total_samples) + pct_gc = (self.collector.gc_frame_samples / total_samples) * 100 + + col = 0 + self.add_str(line, col, "Threads: ", curses.A_BOLD) + col += 11 + + # Show GIL stats only if mode is not GIL (GIL mode filters to only GIL holders) + if self.collector.mode != PROFILING_MODE_GIL: + col = self._add_percentage_stat( + line, col, pct_on_gil, "on gil", self.colors["green"] + ) + col = self._add_percentage_stat( + line, + col, + pct_off_gil, + "off gil", + self.colors["red"], + add_separator=True, + ) + + # Show "waiting for gil" only if mode is not GIL + if self.collector.mode != PROFILING_MODE_GIL and col < width - 30: + col = self._add_percentage_stat( + line, + col, + pct_gil_requested, + "waiting for gil", + self.colors["yellow"], + add_separator=True, + ) + + # Always show GC stats + if col < width - 15: + col = self._add_percentage_stat( + line, + col, + pct_gc, + "GC", + self.colors["magenta"], + add_separator=(col > 11), + ) + + return line + 1 + + def draw_function_stats(self, line, width, stats_list): + """Draw function statistics summary.""" + result_set = self.collector._get_current_result_source() + total_funcs = len(result_set) + funcs_shown = len(stats_list) + executing_funcs = sum( + 1 for f in result_set.values() if f.get("direct_calls", 0) > 0 + ) + stack_only = total_funcs - executing_funcs + + col = 0 + self.add_str(line, col, "Functions: ", curses.A_BOLD) + col += 11 + self.add_str(line, col, f"{total_funcs:>5}", self.colors["cyan"]) + col += 5 + self.add_str(line, col, " total", curses.A_NORMAL) + col += 6 + + if col < width - 25: + self.add_str(line, col, " │ ", curses.A_DIM) + col += 3 + self.add_str( + line, col, f"{executing_funcs:>5}", self.colors["green"] + ) + col += 5 + self.add_str(line, col, " exec", curses.A_NORMAL) + col += 5 + + if col < width - 25: + self.add_str(line, col, " │ ", curses.A_DIM) + col += 3 + self.add_str(line, col, f"{stack_only:>5}", self.colors["yellow"]) + col += 5 + self.add_str(line, col, " stack", curses.A_NORMAL) + col += 6 + + if col < width - 20: + self.add_str(line, col, " │ ", curses.A_DIM) + col += 3 + self.add_str( + line, col, f"{funcs_shown:>5}", self.colors["magenta"] + ) + col += 5 + self.add_str(line, col, " shown", curses.A_NORMAL) + return line + 1 + + def draw_top_functions(self, line, width, stats_list): + """Draw top N hottest functions.""" + col = 0 + self.add_str( + line, + col, + f"Top {TOP_FUNCTIONS_DISPLAY_COUNT}: ", + curses.A_BOLD, + ) + col += 11 + + top_by_samples = sorted( + stats_list, key=lambda x: x["direct_calls"], reverse=True + ) + emojis = ["🥇", "🥈", "🥉"] + medal_colors = [ + self.colors["red"], + self.colors["yellow"], + self.colors["green"], + ] + + displayed = 0 + for func_data in top_by_samples: + if displayed >= TOP_FUNCTIONS_DISPLAY_COUNT: + break + if col >= width - 20: + break + if func_data["direct_calls"] == 0: + continue + + func_name = func_data["func"][2] + func_pct = ( + func_data["direct_calls"] + / max(1, self.collector.total_samples) + ) * 100 + + # Medal emoji + if col + 3 < width - 15: + self.add_str( + line, col, emojis[displayed] + " ", medal_colors[displayed] + ) + col += 3 + + # Function name (truncate to fit) + available_for_name = width - col - 15 + max_name_len = min(25, max(5, available_for_name)) + if len(func_name) > max_name_len: + func_name = func_name[: max_name_len - 3] + "..." + + if col + len(func_name) < width - 10: + self.add_str(line, col, func_name, medal_colors[displayed]) + col += len(func_name) + + pct_str = ( + f" ({func_pct:.1f}%)" + if func_pct >= 0.1 + else f" ({func_data['direct_calls']})" + ) + self.add_str(line, col, pct_str, curses.A_DIM) + col += len(pct_str) + + displayed += 1 + + if displayed < 3 and col < width - 30: + self.add_str(line, col, " │ ", curses.A_DIM) + col += 3 + + if displayed == 0 and col < width - 25: + self.add_str(line, col, "(collecting samples...)", curses.A_DIM) + + return line + 1 + + def draw_finished_banner(self, line, width): + """Draw a prominent banner when profiling is finished.""" + A_REVERSE = self.display.get_attr("A_REVERSE") + A_BOLD = self.display.get_attr("A_BOLD") + + # Add blank line for separation + line += 1 + + # Create the banner message + message = " ✓ PROFILING COMPLETE - Final Results Below - Press 'q' to Quit " + + # Center the message and fill the width with reverse video + if len(message) < width - 1: + padding_total = width - len(message) - 1 + padding_left = padding_total // 2 + padding_right = padding_total - padding_left + full_message = " " * padding_left + message + " " * padding_right + else: + full_message = message[: width - 1] + + # Draw the banner with reverse video and bold + self.add_str( + line, 0, full_message, A_REVERSE | A_BOLD | self.colors["green"] + ) + line += 1 + + # Add blank line for separation + line += 1 + + return line + + +class TableWidget(Widget): + """Widget for rendering column headers and data rows.""" + + def __init__(self, display, colors, collector): + """ + Initialize table widget. + + Args: + display: DisplayInterface implementation + colors: Dictionary of color attributes + collector: Reference to LiveStatsCollector for accessing stats + """ + super().__init__(display, colors) + self.collector = collector + + def render(self, line, width, **kwargs): + """ + Render column headers and data rows. + + Args: + line: Starting line number + width: Available width + kwargs: Must contain 'height' and 'stats_list' keys + + Returns: + Next available line number + """ + height = kwargs["height"] + stats_list = kwargs["stats_list"] + + # Draw column headers + line, show_sample_pct, show_tottime, show_cumul_pct, show_cumtime = ( + self.draw_column_headers(line, width) + ) + column_flags = ( + show_sample_pct, + show_tottime, + show_cumul_pct, + show_cumtime, + ) + + # Draw data rows + line = self.draw_stats_rows( + line, height, width, stats_list, column_flags + ) + + return line + + def draw_column_headers(self, line, width): + """Draw column headers with sort indicators.""" + col = 0 + + # Determine which columns to show based on width + show_sample_pct = width >= WIDTH_THRESHOLD_SAMPLE_PCT + show_tottime = width >= WIDTH_THRESHOLD_TOTTIME + show_cumul_pct = width >= WIDTH_THRESHOLD_CUMUL_PCT + show_cumtime = width >= WIDTH_THRESHOLD_CUMTIME + + sorted_header = self.colors["sorted_header"] + normal_header = self.colors["normal_header"] + + # Determine which column is sorted + sort_col = { + "nsamples": 0, + "sample_pct": 1, + "tottime": 2, + "cumul_pct": 3, + "cumtime": 4, + }.get(self.collector.sort_by, -1) + + # Column 0: nsamples + attr = sorted_header if sort_col == 0 else normal_header + text = f"{'▼nsamples' if sort_col == 0 else 'nsamples':>13}" + self.add_str(line, col, text, attr) + col += 15 + + # Column 1: sample % + if show_sample_pct: + attr = sorted_header if sort_col == 1 else normal_header + text = f"{'▼%' if sort_col == 1 else '%':>5}" + self.add_str(line, col, text, attr) + col += 7 + + # Column 2: tottime + if show_tottime: + attr = sorted_header if sort_col == 2 else normal_header + text = f"{'▼tottime' if sort_col == 2 else 'tottime':>10}" + self.add_str(line, col, text, attr) + col += 12 + + # Column 3: cumul % + if show_cumul_pct: + attr = sorted_header if sort_col == 3 else normal_header + text = f"{'▼%' if sort_col == 3 else '%':>5}" + self.add_str(line, col, text, attr) + col += 7 + + # Column 4: cumtime + if show_cumtime: + attr = sorted_header if sort_col == 4 else normal_header + text = f"{'▼cumtime' if sort_col == 4 else 'cumtime':>10}" + self.add_str(line, col, text, attr) + col += 12 + + # Remaining headers + if col < width - 15: + remaining_space = width - col - 1 + func_width = min( + MAX_FUNC_NAME_WIDTH, + max(MIN_FUNC_NAME_WIDTH, remaining_space // 2), + ) + self.add_str( + line, col, f"{'function':<{func_width}}", normal_header + ) + col += func_width + 2 + + if col < width - 10: + self.add_str(line, col, "file:line", normal_header) + + return ( + line + 1, + show_sample_pct, + show_tottime, + show_cumul_pct, + show_cumtime, + ) + + def draw_stats_rows(self, line, height, width, stats_list, column_flags): + """Draw the statistics data rows.""" + show_sample_pct, show_tottime, show_cumul_pct, show_cumtime = ( + column_flags + ) + + # Get color attributes from the colors dict (already initialized) + color_samples = self.colors.get("color_samples", curses.A_NORMAL) + color_file = self.colors.get("color_file", curses.A_NORMAL) + color_func = self.colors.get("color_func", curses.A_NORMAL) + + # Get trend tracker for color decisions + trend_tracker = self.collector._trend_tracker + + for stat in stats_list: + if line >= height - FOOTER_LINES: + break + + func = stat["func"] + direct_calls = stat["direct_calls"] + cumulative_calls = stat["cumulative_calls"] + total_time = stat["total_time"] + cumulative_time = stat["cumulative_time"] + trends = stat.get("trends", {}) + + sample_pct = ( + (direct_calls / self.collector.total_samples * 100) + if self.collector.total_samples > 0 + else 0 + ) + cum_pct = ( + (cumulative_calls / self.collector.total_samples * 100) + if self.collector.total_samples > 0 + else 0 + ) + + # Helper function to get trend color for a specific column + def get_trend_color(column_name): + trend = trends.get(column_name, "stable") + if trend_tracker is not None: + return trend_tracker.get_color(trend) + return curses.A_NORMAL + + filename, lineno, funcname = func[0], func[1], func[2] + samples_str = f"{direct_calls}/{cumulative_calls}" + col = 0 + + # Samples column - apply trend color based on nsamples trend + nsamples_color = get_trend_color("nsamples") + self.add_str(line, col, f"{samples_str:>13}", nsamples_color) + col += 15 + + # Sample % column + if show_sample_pct: + sample_pct_color = get_trend_color("sample_pct") + self.add_str(line, col, f"{sample_pct:>5.1f}", sample_pct_color) + col += 7 + + # Total time column + if show_tottime: + tottime_color = get_trend_color("tottime") + self.add_str(line, col, f"{total_time:>10.3f}", tottime_color) + col += 12 + + # Cumul % column + if show_cumul_pct: + cumul_pct_color = get_trend_color("cumul_pct") + self.add_str(line, col, f"{cum_pct:>5.1f}", cumul_pct_color) + col += 7 + + # Cumul time column + if show_cumtime: + cumtime_color = get_trend_color("cumtime") + self.add_str(line, col, f"{cumulative_time:>10.3f}", cumtime_color) + col += 12 + + # Function name column + if col < width - 15: + remaining_space = width - col - 1 + func_width = min( + MAX_FUNC_NAME_WIDTH, + max(MIN_FUNC_NAME_WIDTH, remaining_space // 2), + ) + + func_display = funcname + if len(funcname) > func_width: + func_display = funcname[: func_width - 3] + "..." + func_display = f"{func_display:<{func_width}}" + self.add_str(line, col, func_display, color_func) + col += func_width + 2 + + # File:line column + if col < width - 10: + simplified_path = self.collector.simplify_path(filename) + file_line = f"{simplified_path}:{lineno}" + remaining_width = width - col - 1 + self.add_str( + line, col, file_line[:remaining_width], color_file + ) + + line += 1 + + return line + + +class FooterWidget(Widget): + """Widget for rendering the footer section (legend and controls).""" + + def __init__(self, display, colors, collector): + """ + Initialize footer widget. + + Args: + display: DisplayInterface implementation + colors: Dictionary of color attributes + collector: Reference to LiveStatsCollector for accessing state + """ + super().__init__(display, colors) + self.collector = collector + + def render(self, line, width, **kwargs): + """ + Render the footer at the specified position. + + Args: + line: Starting line number (should be height - 2) + width: Available width + + Returns: + Next available line number + """ + A_DIM = self.display.get_attr("A_DIM") + A_BOLD = self.display.get_attr("A_BOLD") + + # Legend line + legend = "nsamples: direct/cumulative (direct=executing, cumulative=on stack)" + self.add_str(line, 0, legend[: width - 1], A_DIM) + line += 1 + + # Controls line with status + sort_names = { + "tottime": "Total Time", + "nsamples": "Direct Samples", + "cumtime": "Cumulative Time", + "sample_pct": "Sample %", + "cumul_pct": "Cumulative %", + } + sort_display = sort_names.get( + self.collector.sort_by, self.collector.sort_by + ) + + # Build status indicators + status = [] + if self.collector.finished: + status.append("[PROFILING FINISHED - Press 'q' to quit]") + elif self.collector.paused: + status.append("[PAUSED]") + if self.collector.filter_pattern: + status.append( + f"[Filter: {self.collector.filter_pattern} (c to clear)]" + ) + # Show trend colors status if disabled + if self.collector._trend_tracker is not None and not self.collector._trend_tracker.enabled: + status.append("[Trend colors: OFF]") + status_str = " ".join(status) + " " if status else "" + + if self.collector.finished: + footer = f"{status_str}" + else: + footer = f"{status_str}Sort: {sort_display} | 't':mode 'x':trends ←→:thread 'h':help 'q':quit" + self.add_str( + line, + 0, + footer[: width - 1], + A_BOLD + if (self.collector.paused or self.collector.finished) + else A_DIM, + ) + + return line + 1 + + def render_filter_input_prompt(self, line, width): + """Draw the filter input prompt at the bottom of the screen.""" + A_BOLD = self.display.get_attr("A_BOLD") + A_REVERSE = self.display.get_attr("A_REVERSE") + + # Draw prompt on last line + prompt = f"Function filter: {self.collector.filter_input_buffer}_" + self.add_str(line, 0, prompt[: width - 1], A_REVERSE | A_BOLD) + + +class HelpWidget(Widget): + """Widget for rendering the help screen overlay.""" + + def render(self, line, width, **kwargs): + """ + Render the help screen. + + Args: + line: Starting line number (ignored, help is centered) + width: Available width + kwargs: Must contain 'height' key + + Returns: + Next available line number (not used for overlays) + """ + height = kwargs["height"] + A_BOLD = self.display.get_attr("A_BOLD") + A_NORMAL = self.display.get_attr("A_NORMAL") + + help_lines = [ + ("Tachyon Profiler - Interactive Commands", A_BOLD), + ("", A_NORMAL), + ("Navigation & Display:", A_BOLD), + (" s - Cycle through sort modes (forward)", A_NORMAL), + (" S - Cycle through sort modes (backward)", A_NORMAL), + (" t - Toggle view mode (ALL / per-thread)", A_NORMAL), + (" x - Toggle trend colors (on/off)", A_NORMAL), + (" ← → ↑ ↓ - Navigate threads (in per-thread mode)", A_NORMAL), + (" + - Faster display refresh rate", A_NORMAL), + (" - - Slower display refresh rate", A_NORMAL), + ("", A_NORMAL), + ("Control:", A_BOLD), + (" p - Freeze display (snapshot)", A_NORMAL), + (" r - Reset all statistics", A_NORMAL), + ("", A_NORMAL), + ("Filtering:", A_BOLD), + (" / - Enter function filter (substring)", A_NORMAL), + (" c - Clear filter", A_NORMAL), + (" ESC - Cancel filter input", A_NORMAL), + ("", A_NORMAL), + ("Other:", A_BOLD), + (" h or ? - Show/hide this help", A_NORMAL), + (" q - Quit profiler", A_NORMAL), + ("", A_NORMAL), + ("Press any key to close this help screen", A_BOLD), + ] + + start_line = (height - len(help_lines)) // 2 + for i, (text, attr) in enumerate(help_lines): + if start_line + i < height - 1: + col = 2 # Left-align with small margin + self.add_str(start_line + i, col, text[: width - 3], attr) + + return line # Not used for overlays diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py new file mode 100644 index 00000000000..4fe3acfa9ff --- /dev/null +++ b/Lib/profiling/sampling/pstats_collector.py @@ -0,0 +1,418 @@ +import collections +import marshal + +from _colorize import ANSIColors +from .collector import Collector + + +class PstatsCollector(Collector): + def __init__(self, sample_interval_usec, *, skip_idle=False): + self.result = collections.defaultdict( + lambda: dict(total_rec_calls=0, direct_calls=0, cumulative_calls=0) + ) + self.stats = {} + self.sample_interval_usec = sample_interval_usec + self.callers = collections.defaultdict( + lambda: collections.defaultdict(int) + ) + self.skip_idle = skip_idle + + def _process_frames(self, frames): + """Process a single thread's frame stack.""" + if not frames: + return + + # Process each frame in the stack to track cumulative calls + for frame in frames: + location = (frame.filename, frame.lineno, frame.funcname) + self.result[location]["cumulative_calls"] += 1 + + # The top frame gets counted as an inline call (directly executing) + top_location = (frames[0].filename, frames[0].lineno, frames[0].funcname) + self.result[top_location]["direct_calls"] += 1 + + # Track caller-callee relationships for call graph + for i in range(1, len(frames)): + callee_frame = frames[i - 1] + caller_frame = frames[i] + + callee = (callee_frame.filename, callee_frame.lineno, callee_frame.funcname) + caller = (caller_frame.filename, caller_frame.lineno, caller_frame.funcname) + + self.callers[callee][caller] += 1 + + def collect(self, stack_frames): + if stack_frames and hasattr(stack_frames[0], "awaited_by"): + # Async frame processing + for frames, thread_id, task_id in self._iter_async_frames(stack_frames): + self._process_frames(frames) + else: + # Regular frame processing + for frames, thread_id in self._iter_all_frames(stack_frames, skip_idle=self.skip_idle): + self._process_frames(frames) + + def export(self, filename): + self.create_stats() + self._dump_stats(filename) + + def _dump_stats(self, file): + stats_with_marker = dict(self.stats) + stats_with_marker[("__sampled__",)] = True + with open(file, "wb") as f: + marshal.dump(stats_with_marker, f) + + # Needed for compatibility with pstats.Stats + def create_stats(self): + sample_interval_sec = self.sample_interval_usec / 1_000_000 + callers = {} + for fname, call_counts in self.result.items(): + total = call_counts["direct_calls"] * sample_interval_sec + cumulative_calls = call_counts["cumulative_calls"] + cumulative = cumulative_calls * sample_interval_sec + callers = dict(self.callers.get(fname, {})) + self.stats[fname] = ( + call_counts["direct_calls"], # cc = direct calls for sample percentage + cumulative_calls, # nc = cumulative calls for cumulative percentage + total, + cumulative, + callers, + ) + + def print_stats(self, sort=-1, limit=None, show_summary=True, mode=None): + """Print formatted statistics to stdout.""" + import pstats + from .constants import PROFILING_MODE_CPU + + # Create stats object + stats = pstats.SampledStats(self).strip_dirs() + if not stats.stats: + print("No samples were collected.") + if mode == PROFILING_MODE_CPU: + print("This can happen in CPU mode when all threads are idle.") + return + + # Get the stats data + stats_list = [] + for func, ( + direct_calls, + cumulative_calls, + total_time, + cumulative_time, + callers, + ) in stats.stats.items(): + stats_list.append( + ( + func, + direct_calls, + cumulative_calls, + total_time, + cumulative_time, + callers, + ) + ) + + # Calculate total samples for percentage calculations (using direct_calls) + total_samples = sum( + direct_calls for _, direct_calls, _, _, _, _ in stats_list + ) + + # Sort based on the requested field + sort_field = sort + if sort_field == -1: # stdname + stats_list.sort(key=lambda x: str(x[0])) + elif sort_field == 0: # nsamples (direct samples) + stats_list.sort(key=lambda x: x[1], reverse=True) # direct_calls + elif sort_field == 1: # tottime + stats_list.sort(key=lambda x: x[3], reverse=True) # total_time + elif sort_field == 2: # cumtime + stats_list.sort(key=lambda x: x[4], reverse=True) # cumulative_time + elif sort_field == 3: # sample% + stats_list.sort( + key=lambda x: (x[1] / total_samples * 100) + if total_samples > 0 + else 0, + reverse=True, # direct_calls percentage + ) + elif sort_field == 4: # cumul% + stats_list.sort( + key=lambda x: (x[2] / total_samples * 100) + if total_samples > 0 + else 0, + reverse=True, # cumulative_calls percentage + ) + elif sort_field == 5: # nsamples (cumulative samples) + stats_list.sort(key=lambda x: x[2], reverse=True) # cumulative_calls + + # Apply limit if specified + if limit is not None: + stats_list = stats_list[:limit] + + # Determine the best unit for time columns based on maximum values + max_total_time = max( + (total_time for _, _, _, total_time, _, _ in stats_list), default=0 + ) + max_cumulative_time = max( + (cumulative_time for _, _, _, _, cumulative_time, _ in stats_list), + default=0, + ) + + total_time_unit, total_time_scale = self._determine_best_unit(max_total_time) + cumulative_time_unit, cumulative_time_scale = self._determine_best_unit( + max_cumulative_time + ) + + # Define column widths for consistent alignment + col_widths = { + "nsamples": 15, # "nsamples" column (inline/cumulative format) + "sample_pct": 8, # "sample%" column + "tottime": max(12, len(f"tottime ({total_time_unit})")), + "cum_pct": 8, # "cumul%" column + "cumtime": max(12, len(f"cumtime ({cumulative_time_unit})")), + } + + # Print header with colors and proper alignment + print(f"{ANSIColors.BOLD_BLUE}Profile Stats:{ANSIColors.RESET}") + + header_nsamples = f"{ANSIColors.BOLD_BLUE}{'nsamples':>{col_widths['nsamples']}}{ANSIColors.RESET}" + header_sample_pct = f"{ANSIColors.BOLD_BLUE}{'sample%':>{col_widths['sample_pct']}}{ANSIColors.RESET}" + header_tottime = f"{ANSIColors.BOLD_BLUE}{f'tottime ({total_time_unit})':>{col_widths['tottime']}}{ANSIColors.RESET}" + header_cum_pct = f"{ANSIColors.BOLD_BLUE}{'cumul%':>{col_widths['cum_pct']}}{ANSIColors.RESET}" + header_cumtime = f"{ANSIColors.BOLD_BLUE}{f'cumtime ({cumulative_time_unit})':>{col_widths['cumtime']}}{ANSIColors.RESET}" + header_filename = ( + f"{ANSIColors.BOLD_BLUE}filename:lineno(function){ANSIColors.RESET}" + ) + + print( + f"{header_nsamples} {header_sample_pct} {header_tottime} {header_cum_pct} {header_cumtime} {header_filename}" + ) + + # Print each line with proper alignment + for ( + func, + direct_calls, + cumulative_calls, + total_time, + cumulative_time, + callers, + ) in stats_list: + # Calculate percentages + sample_pct = ( + (direct_calls / total_samples * 100) if total_samples > 0 else 0 + ) + cum_pct = ( + (cumulative_calls / total_samples * 100) + if total_samples > 0 + else 0 + ) + + # Format values with proper alignment - always use A/B format + nsamples_str = f"{direct_calls}/{cumulative_calls}" + nsamples_str = f"{nsamples_str:>{col_widths['nsamples']}}" + sample_pct_str = f"{sample_pct:{col_widths['sample_pct']}.1f}" + tottime = f"{total_time * total_time_scale:{col_widths['tottime']}.3f}" + cum_pct_str = f"{cum_pct:{col_widths['cum_pct']}.1f}" + cumtime = f"{cumulative_time * cumulative_time_scale:{col_widths['cumtime']}.3f}" + + # Format the function name with colors + func_name = ( + f"{ANSIColors.GREEN}{func[0]}{ANSIColors.RESET}:" + f"{ANSIColors.YELLOW}{func[1]}{ANSIColors.RESET}(" + f"{ANSIColors.CYAN}{func[2]}{ANSIColors.RESET})" + ) + + # Print the formatted line with consistent spacing + print( + f"{nsamples_str} {sample_pct_str} {tottime} {cum_pct_str} {cumtime} {func_name}" + ) + + # Print legend + print(f"\n{ANSIColors.BOLD_BLUE}Legend:{ANSIColors.RESET}") + print( + f" {ANSIColors.YELLOW}nsamples{ANSIColors.RESET}: Direct/Cumulative samples (direct executing / on call stack)" + ) + print( + f" {ANSIColors.YELLOW}sample%{ANSIColors.RESET}: Percentage of total samples this function was directly executing" + ) + print( + f" {ANSIColors.YELLOW}tottime{ANSIColors.RESET}: Estimated total time spent directly in this function" + ) + print( + f" {ANSIColors.YELLOW}cumul%{ANSIColors.RESET}: Percentage of total samples when this function was on the call stack" + ) + print( + f" {ANSIColors.YELLOW}cumtime{ANSIColors.RESET}: Estimated cumulative time (including time in called functions)" + ) + print( + f" {ANSIColors.YELLOW}filename:lineno(function){ANSIColors.RESET}: Function location and name" + ) + + # Print summary of interesting functions if enabled + if show_summary and stats_list: + self._print_summary(stats_list, total_samples) + + @staticmethod + def _determine_best_unit(max_value): + """Determine the best unit (s, ms, μs) and scale factor for a maximum value.""" + if max_value >= 1.0: + return "s", 1.0 + elif max_value >= 0.001: + return "ms", 1000.0 + else: + return "μs", 1000000.0 + + def _print_summary(self, stats_list, total_samples): + """Print summary of interesting functions.""" + print( + f"\n{ANSIColors.BOLD_BLUE}Summary of Interesting Functions:{ANSIColors.RESET}" + ) + + # Aggregate stats by fully qualified function name (ignoring line numbers) + func_aggregated = {} + for ( + func, + direct_calls, + cumulative_calls, + total_time, + cumulative_time, + callers, + ) in stats_list: + # Use filename:function_name as the key to get fully qualified name + qualified_name = f"{func[0]}:{func[2]}" + if qualified_name not in func_aggregated: + func_aggregated[qualified_name] = [ + 0, + 0, + 0, + 0, + ] # direct_calls, cumulative_calls, total_time, cumulative_time + func_aggregated[qualified_name][0] += direct_calls + func_aggregated[qualified_name][1] += cumulative_calls + func_aggregated[qualified_name][2] += total_time + func_aggregated[qualified_name][3] += cumulative_time + + # Convert aggregated data back to list format for processing + aggregated_stats = [] + for qualified_name, ( + prim_calls, + total_calls, + total_time, + cumulative_time, + ) in func_aggregated.items(): + # Parse the qualified name back to filename and function name + if ":" in qualified_name: + filename, func_name = qualified_name.rsplit(":", 1) + else: + filename, func_name = "", qualified_name + # Create a dummy func tuple with filename and function name for display + dummy_func = (filename, "", func_name) + aggregated_stats.append( + ( + dummy_func, + prim_calls, + total_calls, + total_time, + cumulative_time, + {}, + ) + ) + + # Determine best units for summary metrics + max_total_time = max( + (total_time for _, _, _, total_time, _, _ in aggregated_stats), + default=0, + ) + max_cumulative_time = max( + ( + cumulative_time + for _, _, _, _, cumulative_time, _ in aggregated_stats + ), + default=0, + ) + + total_unit, total_scale = self._determine_best_unit(max_total_time) + cumulative_unit, cumulative_scale = self._determine_best_unit( + max_cumulative_time + ) + + def _format_func_name(func): + """Format function name with colors.""" + return ( + f"{ANSIColors.GREEN}{func[0]}{ANSIColors.RESET}:" + f"{ANSIColors.YELLOW}{func[1]}{ANSIColors.RESET}(" + f"{ANSIColors.CYAN}{func[2]}{ANSIColors.RESET})" + ) + + def _print_top_functions(stats_list, title, key_func, format_line, n=3): + """Print top N functions sorted by key_func with formatted output.""" + print(f"\n{ANSIColors.BOLD_BLUE}{title}:{ANSIColors.RESET}") + sorted_stats = sorted(stats_list, key=key_func, reverse=True) + for stat in sorted_stats[:n]: + if line := format_line(stat): + print(f" {line}") + + # Functions with highest direct/cumulative ratio (hot spots) + def format_hotspots(stat): + func, direct_calls, cumulative_calls, total_time, _, _ = stat + if direct_calls > 0 and cumulative_calls > 0: + ratio = direct_calls / cumulative_calls + direct_pct = ( + (direct_calls / total_samples * 100) + if total_samples > 0 + else 0 + ) + return ( + f"{ratio:.3f} direct/cumulative ratio, " + f"{direct_pct:.1f}% direct samples: {_format_func_name(func)}" + ) + return None + + _print_top_functions( + aggregated_stats, + "Functions with Highest Direct/Cumulative Ratio (Hot Spots)", + key_func=lambda x: (x[1] / x[2]) if x[2] > 0 else 0, + format_line=format_hotspots, + ) + + # Functions with highest call frequency (cumulative/direct difference) + def format_call_frequency(stat): + func, direct_calls, cumulative_calls, total_time, _, _ = stat + if cumulative_calls > direct_calls: + call_frequency = cumulative_calls - direct_calls + cum_pct = ( + (cumulative_calls / total_samples * 100) + if total_samples > 0 + else 0 + ) + return ( + f"{call_frequency:d} indirect calls, " + f"{cum_pct:.1f}% total stack presence: {_format_func_name(func)}" + ) + return None + + _print_top_functions( + aggregated_stats, + "Functions with Highest Call Frequency (Indirect Calls)", + key_func=lambda x: x[2] - x[1], # Sort by (cumulative - direct) + format_line=format_call_frequency, + ) + + # Functions with highest cumulative-to-direct multiplier (call magnification) + def format_call_magnification(stat): + func, direct_calls, cumulative_calls, total_time, _, _ = stat + if direct_calls > 0 and cumulative_calls > direct_calls: + multiplier = cumulative_calls / direct_calls + indirect_calls = cumulative_calls - direct_calls + return ( + f"{multiplier:.1f}x call magnification, " + f"{indirect_calls:d} indirect calls from {direct_calls:d} direct: {_format_func_name(func)}" + ) + return None + + _print_top_functions( + aggregated_stats, + "Functions with Highest Call Magnification (Cumulative/Direct)", + key_func=lambda x: (x[2] / x[1]) + if x[1] > 0 + else 0, # Sort by cumulative/direct ratio + format_line=format_call_magnification, + ) diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py new file mode 100644 index 00000000000..dd4ea1edbf6 --- /dev/null +++ b/Lib/profiling/sampling/sample.py @@ -0,0 +1,407 @@ +import _remote_debugging +import os +import pstats +import statistics +import sys +import sysconfig +import time +from collections import deque +from _colorize import ANSIColors + +from .pstats_collector import PstatsCollector +from .stack_collector import CollapsedStackCollector, FlamegraphCollector +from .heatmap_collector import HeatmapCollector +from .gecko_collector import GeckoCollector +from .constants import ( + PROFILING_MODE_WALL, + PROFILING_MODE_CPU, + PROFILING_MODE_GIL, + PROFILING_MODE_ALL, +) +try: + from .live_collector import LiveStatsCollector +except ImportError: + LiveStatsCollector = None + +_FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") is not None + + +class SampleProfiler: + def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True, collect_stats=False): + self.pid = pid + self.sample_interval_usec = sample_interval_usec + self.all_threads = all_threads + self.mode = mode # Store mode for later use + self.collect_stats = collect_stats + if _FREE_THREADED_BUILD: + self.unwinder = _remote_debugging.RemoteUnwinder( + self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc, + skip_non_matching_threads=skip_non_matching_threads, cache_frames=True, + stats=collect_stats + ) + else: + only_active_threads = bool(self.all_threads) + self.unwinder = _remote_debugging.RemoteUnwinder( + self.pid, only_active_thread=only_active_threads, mode=mode, native=native, gc=gc, + skip_non_matching_threads=skip_non_matching_threads, cache_frames=True, + stats=collect_stats + ) + # Track sample intervals and total sample count + self.sample_intervals = deque(maxlen=100) + self.total_samples = 0 + self.realtime_stats = False + + def sample(self, collector, duration_sec=10, *, async_aware=False): + sample_interval_sec = self.sample_interval_usec / 1_000_000 + running_time = 0 + num_samples = 0 + errors = 0 + start_time = next_time = time.perf_counter() + last_sample_time = start_time + realtime_update_interval = 1.0 # Update every second + last_realtime_update = start_time + interrupted = False + + try: + while running_time < duration_sec: + # Check if live collector wants to stop + if hasattr(collector, 'running') and not collector.running: + break + + current_time = time.perf_counter() + if next_time < current_time: + try: + if async_aware == "all": + stack_frames = self.unwinder.get_all_awaited_by() + elif async_aware == "running": + stack_frames = self.unwinder.get_async_stack_trace() + else: + stack_frames = self.unwinder.get_stack_trace() + collector.collect(stack_frames) + except ProcessLookupError: + duration_sec = current_time - start_time + break + except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): + collector.collect_failed_sample() + errors += 1 + except Exception as e: + if not self._is_process_running(): + break + raise e from None + + # Track actual sampling intervals for real-time stats + if num_samples > 0: + actual_interval = current_time - last_sample_time + self.sample_intervals.append( + 1.0 / actual_interval + ) # Convert to Hz + self.total_samples += 1 + + # Print real-time statistics if enabled + if ( + self.realtime_stats + and (current_time - last_realtime_update) + >= realtime_update_interval + ): + self._print_realtime_stats() + last_realtime_update = current_time + + last_sample_time = current_time + num_samples += 1 + next_time += sample_interval_sec + + running_time = time.perf_counter() - start_time + except KeyboardInterrupt: + interrupted = True + running_time = time.perf_counter() - start_time + print("Interrupted by user.") + + # Clear real-time stats line if it was being displayed + if self.realtime_stats and len(self.sample_intervals) > 0: + print() # Add newline after real-time stats + + sample_rate = num_samples / running_time if running_time > 0 else 0 + error_rate = (errors / num_samples) * 100 if num_samples > 0 else 0 + expected_samples = int(duration_sec / sample_interval_sec) + missed_samples = (expected_samples - num_samples) / expected_samples * 100 if expected_samples > 0 else 0 + + # Don't print stats for live mode (curses is handling display) + is_live_mode = LiveStatsCollector is not None and isinstance(collector, LiveStatsCollector) + if not is_live_mode: + print(f"Captured {num_samples} samples in {running_time:.2f} seconds") + print(f"Sample rate: {sample_rate:.2f} samples/sec") + print(f"Error rate: {error_rate:.2f}%") + + # Print unwinder stats if stats collection is enabled + if self.collect_stats: + self._print_unwinder_stats() + + # Pass stats to flamegraph collector if it's the right type + if hasattr(collector, 'set_stats'): + collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, missed_samples, mode=self.mode) + + if num_samples < expected_samples and not is_live_mode and not interrupted: + print( + f"Warning: missed {expected_samples - num_samples} samples " + f"from the expected total of {expected_samples} " + f"({(expected_samples - num_samples) / expected_samples * 100:.2f}%)" + ) + + def _is_process_running(self): + if sys.platform == "linux" or sys.platform == "darwin": + try: + os.kill(self.pid, 0) + return True + except ProcessLookupError: + return False + elif sys.platform == "win32": + try: + _remote_debugging.RemoteUnwinder(self.pid) + except Exception: + return False + return True + else: + raise ValueError(f"Unsupported platform: {sys.platform}") + + def _print_realtime_stats(self): + """Print real-time sampling statistics.""" + if len(self.sample_intervals) < 2: + return + + # Calculate statistics on the Hz values (deque automatically maintains rolling window) + hz_values = list(self.sample_intervals) + mean_hz = statistics.mean(hz_values) + min_hz = min(hz_values) + max_hz = max(hz_values) + + # Calculate microseconds per sample for all metrics (1/Hz * 1,000,000) + mean_us_per_sample = (1.0 / mean_hz) * 1_000_000 if mean_hz > 0 else 0 + min_us_per_sample = ( + (1.0 / max_hz) * 1_000_000 if max_hz > 0 else 0 + ) # Min time = Max Hz + max_us_per_sample = ( + (1.0 / min_hz) * 1_000_000 if min_hz > 0 else 0 + ) # Max time = Min Hz + + # Build cache stats string if stats collection is enabled + cache_stats_str = "" + if self.collect_stats: + try: + stats = self.unwinder.get_stats() + hits = stats.get('frame_cache_hits', 0) + partial = stats.get('frame_cache_partial_hits', 0) + misses = stats.get('frame_cache_misses', 0) + total = hits + partial + misses + if total > 0: + hit_pct = (hits + partial) / total * 100 + cache_stats_str = f" {ANSIColors.MAGENTA}Cache: {hit_pct:.1f}% ({hits}+{partial}/{misses}){ANSIColors.RESET}" + except RuntimeError: + pass + + # Clear line and print stats + print( + f"\r\033[K{ANSIColors.BOLD_BLUE}Stats:{ANSIColors.RESET} " + f"{ANSIColors.YELLOW}{mean_hz:.1f}Hz ({mean_us_per_sample:.1f}µs){ANSIColors.RESET} " + f"{ANSIColors.GREEN}Min: {min_hz:.1f}Hz{ANSIColors.RESET} " + f"{ANSIColors.RED}Max: {max_hz:.1f}Hz{ANSIColors.RESET} " + f"{ANSIColors.CYAN}N={self.total_samples}{ANSIColors.RESET}" + f"{cache_stats_str}", + end="", + flush=True, + ) + + def _print_unwinder_stats(self): + """Print unwinder statistics including cache performance.""" + try: + stats = self.unwinder.get_stats() + except RuntimeError: + return # Stats not enabled + + print(f"\n{ANSIColors.BOLD_BLUE}{'='*50}{ANSIColors.RESET}") + print(f"{ANSIColors.BOLD_BLUE}Unwinder Statistics:{ANSIColors.RESET}") + + # Frame cache stats + total_samples = stats.get('total_samples', 0) + frame_cache_hits = stats.get('frame_cache_hits', 0) + frame_cache_partial_hits = stats.get('frame_cache_partial_hits', 0) + frame_cache_misses = stats.get('frame_cache_misses', 0) + total_lookups = frame_cache_hits + frame_cache_partial_hits + frame_cache_misses + + # Calculate percentages + hits_pct = (frame_cache_hits / total_lookups * 100) if total_lookups > 0 else 0 + partial_pct = (frame_cache_partial_hits / total_lookups * 100) if total_lookups > 0 else 0 + misses_pct = (frame_cache_misses / total_lookups * 100) if total_lookups > 0 else 0 + + print(f" {ANSIColors.CYAN}Frame Cache:{ANSIColors.RESET}") + print(f" Total samples: {total_samples:,}") + print(f" Full hits: {frame_cache_hits:,} ({ANSIColors.GREEN}{hits_pct:.1f}%{ANSIColors.RESET})") + print(f" Partial hits: {frame_cache_partial_hits:,} ({ANSIColors.YELLOW}{partial_pct:.1f}%{ANSIColors.RESET})") + print(f" Misses: {frame_cache_misses:,} ({ANSIColors.RED}{misses_pct:.1f}%{ANSIColors.RESET})") + + # Frame read stats + frames_from_cache = stats.get('frames_read_from_cache', 0) + frames_from_memory = stats.get('frames_read_from_memory', 0) + total_frames = frames_from_cache + frames_from_memory + cache_frame_pct = (frames_from_cache / total_frames * 100) if total_frames > 0 else 0 + memory_frame_pct = (frames_from_memory / total_frames * 100) if total_frames > 0 else 0 + + print(f" {ANSIColors.CYAN}Frame Reads:{ANSIColors.RESET}") + print(f" From cache: {frames_from_cache:,} ({ANSIColors.GREEN}{cache_frame_pct:.1f}%{ANSIColors.RESET})") + print(f" From memory: {frames_from_memory:,} ({ANSIColors.RED}{memory_frame_pct:.1f}%{ANSIColors.RESET})") + + # Code object cache stats + code_hits = stats.get('code_object_cache_hits', 0) + code_misses = stats.get('code_object_cache_misses', 0) + total_code = code_hits + code_misses + code_hits_pct = (code_hits / total_code * 100) if total_code > 0 else 0 + code_misses_pct = (code_misses / total_code * 100) if total_code > 0 else 0 + + print(f" {ANSIColors.CYAN}Code Object Cache:{ANSIColors.RESET}") + print(f" Hits: {code_hits:,} ({ANSIColors.GREEN}{code_hits_pct:.1f}%{ANSIColors.RESET})") + print(f" Misses: {code_misses:,} ({ANSIColors.RED}{code_misses_pct:.1f}%{ANSIColors.RESET})") + + # Memory operations + memory_reads = stats.get('memory_reads', 0) + memory_bytes = stats.get('memory_bytes_read', 0) + if memory_bytes >= 1024 * 1024: + memory_str = f"{memory_bytes / (1024 * 1024):.1f} MB" + elif memory_bytes >= 1024: + memory_str = f"{memory_bytes / 1024:.1f} KB" + else: + memory_str = f"{memory_bytes} B" + print(f" {ANSIColors.CYAN}Memory:{ANSIColors.RESET}") + print(f" Read operations: {memory_reads:,} ({memory_str})") + + # Stale invalidations + stale_invalidations = stats.get('stale_cache_invalidations', 0) + if stale_invalidations > 0: + print(f" {ANSIColors.YELLOW}Stale cache invalidations: {stale_invalidations}{ANSIColors.RESET}") + + +def sample( + pid, + collector, + *, + duration_sec=10, + all_threads=False, + realtime_stats=False, + mode=PROFILING_MODE_WALL, + async_aware=None, + native=False, + gc=True, +): + """Sample a process using the provided collector. + + Args: + pid: Process ID to sample + collector: Collector instance to use for gathering samples + duration_sec: How long to sample for (seconds) + all_threads: Whether to sample all threads + realtime_stats: Whether to print real-time sampling statistics + mode: Profiling mode - WALL (all samples), CPU (only when on CPU), + GIL (only when holding GIL), ALL (includes GIL and CPU status) + native: Whether to include native frames + gc: Whether to include GC frames + + Returns: + The collector with collected samples + """ + # Get sample interval from collector + sample_interval_usec = collector.sample_interval_usec + + # PROFILING_MODE_ALL implies no skipping at all + if mode == PROFILING_MODE_ALL: + skip_non_matching_threads = False + else: + # For most modes, skip non-matching threads + # Gecko collector overrides this by setting skip_idle=False + skip_non_matching_threads = True + + profiler = SampleProfiler( + pid, + sample_interval_usec, + all_threads=all_threads, + mode=mode, + native=native, + gc=gc, + skip_non_matching_threads=skip_non_matching_threads, + collect_stats=realtime_stats, + ) + profiler.realtime_stats = realtime_stats + + # Run the sampling + profiler.sample(collector, duration_sec, async_aware=async_aware) + + return collector + + +def sample_live( + pid, + collector, + *, + duration_sec=10, + all_threads=False, + realtime_stats=False, + mode=PROFILING_MODE_WALL, + async_aware=None, + native=False, + gc=True, +): + """Sample a process in live/interactive mode with curses TUI. + + Args: + pid: Process ID to sample + collector: LiveStatsCollector instance + duration_sec: How long to sample for (seconds) + all_threads: Whether to sample all threads + realtime_stats: Whether to print real-time sampling statistics + mode: Profiling mode - WALL (all samples), CPU (only when on CPU), + GIL (only when holding GIL), ALL (includes GIL and CPU status) + native: Whether to include native frames + gc: Whether to include GC frames + + Returns: + The collector with collected samples + """ + import curses + + # Get sample interval from collector + sample_interval_usec = collector.sample_interval_usec + + # PROFILING_MODE_ALL implies no skipping at all + if mode == PROFILING_MODE_ALL: + skip_non_matching_threads = False + else: + skip_non_matching_threads = True + + profiler = SampleProfiler( + pid, + sample_interval_usec, + all_threads=all_threads, + mode=mode, + native=native, + gc=gc, + skip_non_matching_threads=skip_non_matching_threads, + collect_stats=realtime_stats, + ) + profiler.realtime_stats = realtime_stats + + def curses_wrapper_func(stdscr): + collector.init_curses(stdscr) + try: + profiler.sample(collector, duration_sec, async_aware=async_aware) + # Mark as finished and keep the TUI running until user presses 'q' + collector.mark_finished() + # Keep processing input until user quits + while collector.running: + collector._handle_input() + time.sleep(0.05) # Small sleep to avoid busy waiting + finally: + collector.cleanup_curses() + + try: + curses.wrapper(curses_wrapper_func) + except KeyboardInterrupt: + pass + + return collector diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py new file mode 100644 index 00000000000..1f766682858 --- /dev/null +++ b/Lib/profiling/sampling/stack_collector.py @@ -0,0 +1,389 @@ +import base64 +import collections +import functools +import importlib.resources +import json +import linecache +import os + +from ._css_utils import get_combined_css +from .collector import Collector +from .string_table import StringTable + + +class StackTraceCollector(Collector): + def __init__(self, sample_interval_usec, *, skip_idle=False): + self.sample_interval_usec = sample_interval_usec + self.skip_idle = skip_idle + + def collect(self, stack_frames, skip_idle=False): + if stack_frames and hasattr(stack_frames[0], "awaited_by"): + # Async-aware mode: process async task frames + for frames, thread_id, task_id in self._iter_async_frames(stack_frames): + if not frames: + continue + self.process_frames(frames, thread_id) + else: + # Sync-only mode + for frames, thread_id in self._iter_all_frames(stack_frames, skip_idle=skip_idle): + if not frames: + continue + self.process_frames(frames, thread_id) + + def process_frames(self, frames, thread_id): + pass + + +class CollapsedStackCollector(StackTraceCollector): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.stack_counter = collections.Counter() + + def process_frames(self, frames, thread_id): + call_tree = tuple(reversed(frames)) + self.stack_counter[(call_tree, thread_id)] += 1 + + def export(self, filename): + lines = [] + for (call_tree, thread_id), count in self.stack_counter.items(): + parts = [f"tid:{thread_id}"] + for file, line, func in call_tree: + # This is what pstats does for "special" frames: + if file == "~" and line == 0: + part = func + else: + part = f"{os.path.basename(file)}:{func}:{line}" + parts.append(part) + stack_str = ";".join(parts) + lines.append((stack_str, count)) + + lines.sort(key=lambda x: (-x[1], x[0])) + + with open(filename, "w") as f: + for stack, count in lines: + f.write(f"{stack} {count}\n") + print(f"Collapsed stack output written to {filename}") + + +class FlamegraphCollector(StackTraceCollector): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.stats = {} + self._root = {"samples": 0, "children": {}, "threads": set()} + self._total_samples = 0 + self._sample_count = 0 # Track actual number of samples (not thread traces) + self._func_intern = {} + self._string_table = StringTable() + self._all_threads = set() + + # Thread status statistics (similar to LiveStatsCollector) + self.thread_status_counts = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + } + self.samples_with_gc_frames = 0 + + # Per-thread statistics + self.per_thread_stats = {} # {thread_id: {has_gil, on_cpu, gil_requested, unknown, total, gc_samples}} + + def collect(self, stack_frames, skip_idle=False): + """Override to track thread status statistics before processing frames.""" + # Increment sample count once per sample + self._sample_count += 1 + + # Collect both aggregate and per-thread statistics using base method + status_counts, has_gc_frame, per_thread_stats = self._collect_thread_status_stats(stack_frames) + + # Merge aggregate status counts + for key in status_counts: + self.thread_status_counts[key] += status_counts[key] + + # Update aggregate GC frame count + if has_gc_frame: + self.samples_with_gc_frames += 1 + + # Merge per-thread statistics + for thread_id, stats in per_thread_stats.items(): + if thread_id not in self.per_thread_stats: + self.per_thread_stats[thread_id] = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + "gc_samples": 0, + } + for key, value in stats.items(): + self.per_thread_stats[thread_id][key] += value + + # Call parent collect to process frames + super().collect(stack_frames, skip_idle=skip_idle) + + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, + error_rate=None, missed_samples=None, mode=None): + """Set profiling statistics to include in flamegraph data.""" + self.stats = { + "sample_interval_usec": sample_interval_usec, + "duration_sec": duration_sec, + "sample_rate": sample_rate, + "error_rate": error_rate, + "missed_samples": missed_samples, + "mode": mode + } + + def export(self, filename): + flamegraph_data = self._convert_to_flamegraph_format() + + # Debug output with string table statistics + num_functions = len(flamegraph_data.get("children", [])) + total_time = flamegraph_data.get("value", 0) + string_count = len(self._string_table) + print( + f"Flamegraph data: {num_functions} root functions, total samples: {total_time}, " + f"{string_count} unique strings" + ) + + if num_functions == 0: + print( + "Warning: No functions found in profiling data. Check if sampling captured any data." + ) + return + + html_content = self._create_flamegraph_html(flamegraph_data) + + with open(filename, "w", encoding="utf-8") as f: + f.write(html_content) + + print(f"Flamegraph saved to: {filename}") + + @staticmethod + @functools.lru_cache(maxsize=None) + def _format_function_name(func): + filename, lineno, funcname = func + + # Special frames like and should not show file:line + if filename == "~" and lineno == 0: + return funcname + + if len(filename) > 50: + parts = filename.split("/") + if len(parts) > 2: + filename = f".../{'/'.join(parts[-2:])}" + + return f"{funcname} ({filename}:{lineno})" + + def _convert_to_flamegraph_format(self): + if self._total_samples == 0: + return { + "name": self._string_table.intern("No Data"), + "value": 0, + "children": [], + "threads": [], + "strings": self._string_table.get_strings() + } + + def convert_children(children, min_samples): + out = [] + for func, node in children.items(): + samples = node["samples"] + if samples < min_samples: + continue + + # Intern all string components for maximum efficiency + filename_idx = self._string_table.intern(func[0]) + funcname_idx = self._string_table.intern(func[2]) + name_idx = self._string_table.intern(self._format_function_name(func)) + + child_entry = { + "name": name_idx, + "value": samples, + "children": [], + "filename": filename_idx, + "lineno": func[1], + "funcname": funcname_idx, + "threads": sorted(list(node.get("threads", set()))), + } + + source = self._get_source_lines(func) + if source: + # Intern source lines for memory efficiency + source_indices = [self._string_table.intern(line) for line in source] + child_entry["source"] = source_indices + + # Recurse + child_entry["children"] = convert_children( + node["children"], min_samples + ) + out.append(child_entry) + + # Sort by value (descending) then by name index for consistent ordering + out.sort(key=lambda x: (-x["value"], x["name"])) + return out + + # Filter out very small functions (less than 0.1% of total samples) + total_samples = self._total_samples + min_samples = max(1, int(total_samples * 0.001)) + + root_children = convert_children(self._root["children"], min_samples) + if not root_children: + return { + "name": self._string_table.intern("No significant data"), + "value": 0, + "children": [], + "strings": self._string_table.get_strings() + } + + # Calculate thread status percentages for display + total_threads = max(1, self.thread_status_counts["total"]) + thread_stats = { + "has_gil_pct": (self.thread_status_counts["has_gil"] / total_threads) * 100, + "on_cpu_pct": (self.thread_status_counts["on_cpu"] / total_threads) * 100, + "gil_requested_pct": (self.thread_status_counts["gil_requested"] / total_threads) * 100, + "gc_pct": (self.samples_with_gc_frames / max(1, self._sample_count)) * 100, + **self.thread_status_counts + } + + # Calculate per-thread statistics with percentages + per_thread_stats_with_pct = {} + total_samples_denominator = max(1, self._sample_count) + for thread_id, stats in self.per_thread_stats.items(): + total = max(1, stats["total"]) + per_thread_stats_with_pct[thread_id] = { + "has_gil_pct": (stats["has_gil"] / total) * 100, + "on_cpu_pct": (stats["on_cpu"] / total) * 100, + "gil_requested_pct": (stats["gil_requested"] / total) * 100, + "gc_pct": (stats["gc_samples"] / total_samples_denominator) * 100, + **stats + } + + # If we only have one root child, make it the root to avoid redundant level + if len(root_children) == 1: + main_child = root_children[0] + # Update the name to indicate it's the program root + old_name = self._string_table.get_string(main_child["name"]) + new_name = f"Program Root: {old_name}" + main_child["name"] = self._string_table.intern(new_name) + main_child["stats"] = { + **self.stats, + "thread_stats": thread_stats, + "per_thread_stats": per_thread_stats_with_pct + } + main_child["threads"] = sorted(list(self._all_threads)) + main_child["strings"] = self._string_table.get_strings() + return main_child + + return { + "name": self._string_table.intern("Program Root"), + "value": total_samples, + "children": root_children, + "stats": { + **self.stats, + "thread_stats": thread_stats, + "per_thread_stats": per_thread_stats_with_pct + }, + "threads": sorted(list(self._all_threads)), + "strings": self._string_table.get_strings() + } + + def process_frames(self, frames, thread_id): + # Reverse to root->leaf + call_tree = reversed(frames) + self._root["samples"] += 1 + self._total_samples += 1 + self._root["threads"].add(thread_id) + self._all_threads.add(thread_id) + + current = self._root + for func in call_tree: + func = self._func_intern.setdefault(func, func) + children = current["children"] + node = children.get(func) + if node is None: + node = {"samples": 0, "children": {}, "threads": set()} + children[func] = node + node["samples"] += 1 + node["threads"].add(thread_id) + current = node + + def _get_source_lines(self, func): + filename, lineno, _ = func + + try: + lines = [] + start_line = max(1, lineno - 2) + end_line = lineno + 3 + + for line_num in range(start_line, end_line): + line = linecache.getline(filename, line_num) + if line.strip(): + marker = "→ " if line_num == lineno else " " + lines.append(f"{marker}{line_num}: {line.rstrip()}") + + return lines if lines else None + + except Exception: + return None + + def _create_flamegraph_html(self, data): + data_json = json.dumps(data) + + template_dir = importlib.resources.files(__package__) + vendor_dir = template_dir / "_vendor" + assets_dir = template_dir / "_assets" + + d3_path = vendor_dir / "d3" / "7.8.5" / "d3.min.js" + d3_flame_graph_dir = vendor_dir / "d3-flame-graph" / "4.1.3" + fg_css_path = d3_flame_graph_dir / "d3-flamegraph.css" + fg_js_path = d3_flame_graph_dir / "d3-flamegraph.min.js" + fg_tooltip_js_path = d3_flame_graph_dir / "d3-flamegraph-tooltip.min.js" + + html_template = (template_dir / "_flamegraph_assets" / "flamegraph_template.html").read_text(encoding="utf-8") + css_content = get_combined_css("flamegraph") + js_content = (template_dir / "_flamegraph_assets" / "flamegraph.js").read_text(encoding="utf-8") + + # Inline first-party CSS/JS + html_template = html_template.replace( + "", f"" + ) + html_template = html_template.replace( + "", f"" + ) + + png_path = assets_dir / "python-logo-only.png" + b64_logo = base64.b64encode(png_path.read_bytes()).decode("ascii") + + # Let CSS control size; keep markup simple + logo_html = f'Python logo' + html_template = html_template.replace("", logo_html) + + d3_js = d3_path.read_text(encoding="utf-8") + fg_css = fg_css_path.read_text(encoding="utf-8") + fg_js = fg_js_path.read_text(encoding="utf-8") + fg_tooltip_js = fg_tooltip_js_path.read_text(encoding="utf-8") + + html_template = html_template.replace( + "", + f"", + ) + html_template = html_template.replace( + "", + f"", + ) + html_template = html_template.replace( + "", + f"", + ) + html_template = html_template.replace( + "", + f"", + ) + + # Replace the placeholder with actual data + html_content = html_template.replace( + "{{FLAMEGRAPH_DATA}}", data_json + ) + + return html_content diff --git a/Lib/profiling/sampling/string_table.py b/Lib/profiling/sampling/string_table.py new file mode 100644 index 00000000000..25c347f7ff1 --- /dev/null +++ b/Lib/profiling/sampling/string_table.py @@ -0,0 +1,53 @@ +"""String table implementation for memory-efficient string storage in profiling data.""" + +class StringTable: + """A string table for interning strings and reducing memory usage.""" + + def __init__(self): + self._strings = [] + self._string_to_index = {} + + def intern(self, string): + """Intern a string and return its index. + + Args: + string: The string to intern + + Returns: + int: The index of the string in the table + """ + if not isinstance(string, str): + string = str(string) + + if string in self._string_to_index: + return self._string_to_index[string] + + index = len(self._strings) + self._strings.append(string) + self._string_to_index[string] = index + return index + + def get_string(self, index): + """Get a string by its index. + + Args: + index: The index of the string + + Returns: + str: The string at the given index, or empty string if invalid + """ + if 0 <= index < len(self._strings): + return self._strings[index] + return "" + + def get_strings(self): + """Get the list of all strings in the table. + + Returns: + list: A copy of the strings list + """ + return self._strings.copy() + + def __len__(self): + """Return the number of strings in the table.""" + return len(self._strings) diff --git a/Lib/profiling/tracing/__init__.py b/Lib/profiling/tracing/__init__.py new file mode 100644 index 00000000000..a6b8edf7216 --- /dev/null +++ b/Lib/profiling/tracing/__init__.py @@ -0,0 +1,219 @@ +"""Tracing profiler for Python. + +This module provides deterministic profiling of Python programs by tracing +every function call and return. +""" + +__all__ = ("run", "runctx", "Profile") + +import _lsprof +import importlib.machinery +import importlib.util +import io +from profiling.tracing._utils import _Utils + +# ____________________________________________________________ +# Simple interface + +def run(statement, filename=None, sort=-1): + """Run statement under profiler optionally saving results in filename + + This function takes a single argument that can be passed to the + "exec" statement, and an optional file name. In all cases this + routine attempts to "exec" its first argument and gather profiling + statistics from the execution. If no file name is present, then this + function automatically prints a simple profiling report, sorted by the + standard name string (file/line/function-name) that is presented in + each line. + """ + return _Utils(Profile).run(statement, filename, sort) + +def runctx(statement, globals, locals, filename=None, sort=-1): + """Run statement under profiler, supplying your own globals and locals, + optionally saving results in filename. + + statement and filename have the same semantics as profile.run + """ + return _Utils(Profile).runctx(statement, globals, locals, + filename, sort) + +# ____________________________________________________________ + +class Profile(_lsprof.Profiler): + """Profile(timer=None, timeunit=None, subcalls=True, builtins=True) + + Builds a profiler object using the specified timer function. + The default timer is a fast built-in one based on real time. + For custom timer functions returning integers, timeunit can + be a float specifying a scale (i.e. how long each integer unit + is, in seconds). + """ + + # Most of the functionality is in the base class. + # This subclass only adds convenient and backward-compatible methods. + + def print_stats(self, sort=-1): + import pstats + if not isinstance(sort, tuple): + sort = (sort,) + pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats() + + def dump_stats(self, file): + import marshal + with open(file, 'wb') as f: + self.create_stats() + marshal.dump(self.stats, f) + + def create_stats(self): + self.disable() + self.snapshot_stats() + + def snapshot_stats(self): + entries = self.getstats() + self.stats = {} + callersdicts = {} + # call information + for entry in entries: + func = label(entry.code) + nc = entry.callcount # ncalls column of pstats (before '/') + cc = nc - entry.reccallcount # ncalls column of pstats (after '/') + tt = entry.inlinetime # tottime column of pstats + ct = entry.totaltime # cumtime column of pstats + callers = {} + callersdicts[id(entry.code)] = callers + self.stats[func] = cc, nc, tt, ct, callers + # subcall information + for entry in entries: + if entry.calls: + func = label(entry.code) + for subentry in entry.calls: + try: + callers = callersdicts[id(subentry.code)] + except KeyError: + continue + nc = subentry.callcount + cc = nc - subentry.reccallcount + tt = subentry.inlinetime + ct = subentry.totaltime + if func in callers: + prev = callers[func] + nc += prev[0] + cc += prev[1] + tt += prev[2] + ct += prev[3] + callers[func] = nc, cc, tt, ct + + # The following two methods can be called by clients to use + # a profiler to profile a statement, given as a string. + + def run(self, cmd): + import __main__ + dict = __main__.__dict__ + return self.runctx(cmd, dict, dict) + + def runctx(self, cmd, globals, locals): + self.enable() + try: + exec(cmd, globals, locals) + finally: + self.disable() + return self + + # This method is more useful to profile a single function call. + def runcall(self, func, /, *args, **kw): + self.enable() + try: + return func(*args, **kw) + finally: + self.disable() + + def __enter__(self): + self.enable() + return self + + def __exit__(self, *exc_info): + self.disable() + +# ____________________________________________________________ + +def label(code): + if isinstance(code, str): + return ('~', 0, code) # built-in functions ('~' sorts at the end) + else: + return (code.co_filename, code.co_firstlineno, code.co_name) + +# ____________________________________________________________ + +def main(): + import os + import sys + import runpy + import pstats + from optparse import OptionParser + usage = "cProfile.py [-o output_file_path] [-s sort] [-m module | scriptfile] [arg] ..." + parser = OptionParser(usage=usage) + parser.allow_interspersed_args = False + parser.add_option('-o', '--outfile', dest="outfile", + help="Save stats to ", default=None) + parser.add_option('-s', '--sort', dest="sort", + help="Sort order when printing to stdout, based on pstats.Stats class", + default=2, + choices=sorted(pstats.Stats.sort_arg_dict_default)) + parser.add_option('-m', dest="module", action="store_true", + help="Profile a library module", default=False) + + if not sys.argv[1:]: + parser.print_usage() + sys.exit(2) + + (options, args) = parser.parse_args() + sys.argv[:] = args + + # The script that we're profiling may chdir, so capture the absolute path + # to the output file at startup. + if options.outfile is not None: + options.outfile = os.path.abspath(options.outfile) + + if len(args) > 0: + if options.module: + code = "run_module(modname, run_name='__main__')" + globs = { + 'run_module': runpy.run_module, + 'modname': args[0] + } + else: + progname = args[0] + sys.path.insert(0, os.path.dirname(progname)) + with io.open_code(progname) as fp: + code = compile(fp.read(), progname, 'exec', module='__main__') + spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, + origin=progname) + module = importlib.util.module_from_spec(spec) + # Set __main__ so that importing __main__ in the profiled code will + # return the same namespace that the code is executing under. + sys.modules['__main__'] = module + # Ensure that we're using the same __dict__ instance as the module + # for the global variables so that updates to globals are reflected + # in the module's namespace. + globs = module.__dict__ + globs.update({ + '__spec__': spec, + '__file__': spec.origin, + '__name__': spec.name, + '__package__': None, + '__cached__': None, + }) + + try: + runctx(code, globs, None, options.outfile, options.sort) + except BrokenPipeError as exc: + # Prevent "Exception ignored" during interpreter shutdown. + sys.stdout = None + sys.exit(exc.errno) + else: + parser.print_usage() + return parser + +# When invoked as main program, invoke the profiler on a script +if __name__ == '__main__': + main() diff --git a/Lib/profiling/tracing/__main__.py b/Lib/profiling/tracing/__main__.py new file mode 100644 index 00000000000..95041d1368b --- /dev/null +++ b/Lib/profiling/tracing/__main__.py @@ -0,0 +1,6 @@ +"""Run the tracing profiler from the command line.""" + +from profiling.tracing import main + +if __name__ == '__main__': + main() diff --git a/Lib/profiling/tracing/_utils.py b/Lib/profiling/tracing/_utils.py new file mode 100644 index 00000000000..8c4b65889c7 --- /dev/null +++ b/Lib/profiling/tracing/_utils.py @@ -0,0 +1,32 @@ +class _Utils: + """Support class for utility functions which are shared by + profile.py and cProfile.py modules. + Not supposed to be used directly. + """ + + def __init__(self, profiler): + self.profiler = profiler + + def run(self, statement, filename, sort): + prof = self.profiler() + try: + prof.run(statement) + except SystemExit: + pass + finally: + self._show(prof, filename, sort) + + def runctx(self, statement, globals, locals, filename, sort): + prof = self.profiler() + try: + prof.runctx(statement, globals, locals) + except SystemExit: + pass + finally: + self._show(prof, filename, sort) + + def _show(self, prof, filename, sort): + if filename is not None: + prof.dump_stats(filename) + else: + prof.print_stats(sort) diff --git a/Lib/pstats.py b/Lib/pstats.py index becaf35580e..07ecda07796 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -139,7 +139,11 @@ def load_stats(self, arg): return elif isinstance(arg, str): with open(arg, 'rb') as f: - self.stats = marshal.load(f) + stats = marshal.load(f) + if (('__sampled__',)) in stats: + stats.pop((('__sampled__',))) + self.__class__ = SampledStats + self.stats = stats try: file_stats = os.stat(arg) arg = time.ctime(file_stats.st_mtime) + " " + arg @@ -150,6 +154,7 @@ def load_stats(self, arg): arg.create_stats() self.stats = arg.stats arg.stats = {} + return if not self.stats: raise TypeError("Cannot create or construct a %r object from %r" % (self.__class__, arg)) @@ -467,7 +472,10 @@ def print_call_heading(self, name_size, column_title): subheader = isinstance(value, tuple) break if subheader: - print(" "*name_size + " ncalls tottime cumtime", file=self.stream) + self.print_call_subheading(name_size) + + def print_call_subheading(self, name_size): + print(" "*name_size + " ncalls tottime cumtime", file=self.stream) def print_call_line(self, name_size, source, call_dict, arrow="->"): print(func_std_string(source).ljust(name_size) + arrow, end=' ', file=self.stream) @@ -516,6 +524,35 @@ def print_line(self, func): # hack: should print percentages print(f8(ct/cc), end=' ', file=self.stream) print(func_std_string(func), file=self.stream) + +class SampledStats(Stats): + def __init__(self, *args, stream=None): + super().__init__(*args, stream=stream) + + self.sort_arg_dict = { + "samples" : (((1,-1), ), "sample count"), + "nsamples" : (((1,-1), ), "sample count"), + "cumtime" : (((3,-1), ), "cumulative time"), + "cumulative": (((3,-1), ), "cumulative time"), + "filename" : (((4, 1), ), "file name"), + "line" : (((5, 1), ), "line number"), + "module" : (((4, 1), ), "file name"), + "name" : (((6, 1), ), "function name"), + "nfl" : (((6, 1),(4, 1),(5, 1),), "name/file/line"), + "psamples" : (((0,-1), ), "primitive call count"), + "stdname" : (((7, 1), ), "standard name"), + "time" : (((2,-1), ), "internal time"), + "tottime" : (((2,-1), ), "internal time"), + } + + def print_call_subheading(self, name_size): + print(" "*name_size + " nsamples tottime cumtime", file=self.stream) + + def print_title(self): + print(' nsamples tottime persample cumtime persample', end=' ', file=self.stream) + print('filename:lineno(function)', file=self.stream) + + class TupleComp: """This class provides a generic function for comparing any two tuples. Each instance records a list of tuple-indices (from most significant @@ -607,6 +644,24 @@ def f8(x): # Statistics browser added by ESR, April 2001 #************************************************************************** +class StatsLoaderShim: + """Compatibility shim implementing 'create_stats' needed by Stats classes + to handle already unmarshalled data.""" + def __init__(self, raw_stats): + self.stats = raw_stats + + def create_stats(self): + pass + +def stats_factory(raw_stats): + """Return a Stats or SampledStats instance based on the marker in raw_stats.""" + if (('__sampled__',)) in raw_stats: + raw_stats = dict(raw_stats) # avoid mutating caller's dict + raw_stats.pop((('__sampled__',))) + return SampledStats(StatsLoaderShim(raw_stats)) + else: + return Stats(StatsLoaderShim(raw_stats)) + if __name__ == '__main__': import cmd try: @@ -693,7 +748,15 @@ def help_quit(self): def do_read(self, line): if line: try: - self.stats = Stats(line) + with open(line, 'rb') as f: + raw_stats = marshal.load(f) + self.stats = stats_factory(raw_stats) + try: + file_stats = os.stat(line) + arg = time.ctime(file_stats.st_mtime) + " " + line + except Exception: + arg = line + self.stats.files = [arg] except OSError as err: print(err.args[1], file=self.stream) return diff --git a/Lib/pydoc.py b/Lib/pydoc.py index def76d076a2..45ff5fca308 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -108,96 +108,10 @@ def pathdirs(): normdirs.append(normdir) return dirs -def _findclass(func): - cls = sys.modules.get(func.__module__) - if cls is None: - return None - for name in func.__qualname__.split('.')[:-1]: - cls = getattr(cls, name) - if not inspect.isclass(cls): - return None - return cls - -def _finddoc(obj): - if inspect.ismethod(obj): - name = obj.__func__.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - getattr(getattr(self, name, None), '__func__') is obj.__func__): - # classmethod - cls = self - else: - cls = self.__class__ - elif inspect.isfunction(obj): - name = obj.__name__ - cls = _findclass(obj) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.isbuiltin(obj): - name = obj.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - self.__qualname__ + '.' + name == obj.__qualname__): - # classmethod - cls = self - else: - cls = self.__class__ - # Should be tested before isdatadescriptor(). - elif isinstance(obj, property): - name = obj.__name__ - cls = _findclass(obj.fget) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): - name = obj.__name__ - cls = obj.__objclass__ - if getattr(cls, name) is not obj: - return None - if inspect.ismemberdescriptor(obj): - slots = getattr(cls, '__slots__', None) - if isinstance(slots, dict) and name in slots: - return slots[name] - else: - return None - for base in cls.__mro__: - try: - doc = _getowndoc(getattr(base, name)) - except AttributeError: - continue - if doc is not None: - return doc - return None - -def _getowndoc(obj): - """Get the documentation string for an object if it is not - inherited from its class.""" - try: - doc = object.__getattribute__(obj, '__doc__') - if doc is None: - return None - if obj is not type: - typedoc = type(obj).__doc__ - if isinstance(typedoc, str) and typedoc == doc: - return None - return doc - except AttributeError: - return None - def _getdoc(object): - """Get the documentation string for an object. - - All tabs are expanded to spaces. To clean up docstrings that are - indented to line up with blocks of code, any whitespace than can be - uniformly removed from the second line onwards is removed.""" - doc = _getowndoc(object) - if doc is None: - try: - doc = _finddoc(object) - except (AttributeError, TypeError): - return None - if not isinstance(doc, str): - return None - return inspect.cleandoc(doc) + return inspect.getdoc(object, + fallback_to_class_doc=False, + inherit_class_doc=False) def getdoc(object): """Get the doc string or comments for an object.""" @@ -884,6 +798,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use a heuristic. if (all is not None + or inspect.isbuiltin(value) or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) @@ -1328,6 +1243,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use a heuristic. if (all is not None + or inspect.isbuiltin(value) or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) @@ -1812,7 +1728,6 @@ def writedocs(dir, pkgpath='', done=None): def _introdoc(): - import textwrap ver = '%d.%d' % sys.version_info[:2] if os.environ.get('PYTHON_BASIC_REPL'): pyrepl_keys = '' @@ -2110,7 +2025,7 @@ def intro(self): self.output.write(_introdoc()) def list(self, items, columns=4, width=80): - items = list(sorted(items)) + items = sorted(items) colw = width // columns rows = (len(items) + columns - 1) // columns for row in range(rows): @@ -2142,7 +2057,7 @@ def listtopics(self): Here is a list of available topics. Enter any topic name to get more help. ''') - self.list(self.topics.keys()) + self.list(self.topics.keys(), columns=3) def showtopic(self, topic, more_xrefs=''): try: @@ -2170,7 +2085,6 @@ def showtopic(self, topic, more_xrefs=''): if more_xrefs: xrefs = (xrefs or '') + ' ' + more_xrefs if xrefs: - import textwrap text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 5f7e14a79d3..11ffc6bf3a1 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue May 6 18:33:44 2025 +# Autogenerated by Sphinx on Tue Nov 18 16:51:09 2025 # as part of the release process. topics = { @@ -435,9 +435,9 @@ async def func(param1, param2): 'atom-identifiers': r'''Identifiers (Names) ******************* -An identifier occurring as an atom is a name. See section Identifiers -and keywords for lexical definition and section Naming and binding for -documentation of naming and binding. +An identifier occurring as an atom is a name. See section Names +(identifiers and keywords) for lexical definition and section Naming +and binding for documentation of naming and binding. When the name is bound to an object, evaluation of the atom yields that object. When a name is not bound, an attempt to evaluate it @@ -492,19 +492,65 @@ async def func(param1, param2): Python supports string and bytes literals and various numeric literals: - literal: stringliteral | bytesliteral - | integer | floatnumber | imagnumber + literal: strings | NUMBER Evaluation of a literal yields an object of the given type (string, bytes, integer, floating-point number, complex number) with the given value. The value may be approximated in the case of floating-point -and imaginary (complex) literals. See section Literals for details. +and imaginary (complex) literals. See section Literals for details. +See section String literal concatenation for details on "strings". All literals correspond to immutable data types, and hence the object’s identity is less important than its value. Multiple evaluations of literals with the same value (either the same occurrence in the program text or a different occurrence) may obtain the same object or a different object with the same value. + + +String literal concatenation +============================ + +Multiple adjacent string or bytes literals (delimited by whitespace), +possibly using different quoting conventions, are allowed, and their +meaning is the same as their concatenation: + + >>> "hello" 'world' + "helloworld" + +Formally: + + strings: ( STRING | fstring)+ | tstring+ + +This feature is defined at the syntactical level, so it only works +with literals. To concatenate string expressions at run time, the ‘+’ +operator may be used: + + >>> greeting = "Hello" + >>> space = " " + >>> name = "Blaise" + >>> print(greeting + space + name) # not: print(greeting space name) + Hello Blaise + +Literal concatenation can freely mix raw strings, triple-quoted +strings, and formatted string literals. For example: + + >>> "Hello" r', ' f"{name}!" + "Hello, Blaise!" + +This feature can be used to reduce the number of backslashes needed, +to split long strings conveniently across long lines, or even to add +comments to parts of strings. For example: + + re.compile("[A-Za-z_]" # letter or underscore + "[A-Za-z0-9_]*" # letter, digit or underscore + ) + +However, bytes literals may only be combined with other byte literals; +not with string literals of any kind. Also, template string literals +may only be combined with other template string literals: + + >>> t"Hello" t"{name}!" + Template(strings=('Hello', '!'), interpolations=(...)) ''', 'attribute-access': r'''Customizing attribute access **************************** @@ -588,6 +634,9 @@ class instances. Customizing module attribute access =================================== +module.__getattr__() +module.__dir__() + Special names "__getattr__" and "__dir__" can be also used to customize access to module attributes. The "__getattr__" function at the module level should accept one argument which is the name of an @@ -603,6 +652,8 @@ class instances. present, this function overrides the standard "dir()" search on a module. +module.__class__ + For a more fine grained customization of the module behavior (setting attributes, properties, etc.), one can set the "__class__" attribute of a module object to a subclass of "types.ModuleType". For example: @@ -1047,12 +1098,33 @@ class and instance attributes applies as for regular assignments. 'bltin-ellipsis-object': r'''The Ellipsis Object ******************* -This object is commonly used by slicing (see Slicings). It supports -no special operations. There is exactly one ellipsis object, named -"Ellipsis" (a built-in name). "type(Ellipsis)()" produces the +This object is commonly used to indicate that something is omitted. It +supports no special operations. There is exactly one ellipsis object, +named "Ellipsis" (a built-in name). "type(Ellipsis)()" produces the "Ellipsis" singleton. It is written as "Ellipsis" or "...". + +In typical use, "..." as the "Ellipsis" object appears in a few +different places, for instance: + +* In type annotations, such as callable arguments or tuple elements. + +* As the body of a function instead of a pass statement. + +* In third-party libraries, such as Numpy’s slicing and striding. + +Python also uses three dots in ways that are not "Ellipsis" objects, +for instance: + +* Doctest’s "ELLIPSIS", as a pattern for missing content. + +* The default Python prompt of the *interactive* shell when partial + input is incomplete. + +Lastly, the Python documentation often uses three dots in conventional +English usage to mean omitted content, even in code examples that also +use them as the "Ellipsis". ''', 'bltin-null-object': r'''The Null Object *************** @@ -1314,6 +1386,9 @@ class Foo: class Foo(object): pass +There may be one or more base classes; see Multiple inheritance below +for more information. + The class’s suite is then executed in a new execution frame (see Naming and binding), using a newly created local namespace and the original global namespace. (Usually, the suite contains mostly @@ -1377,6 +1452,115 @@ class attributes; they are shared by instances. Instance attributes **PEP 3129** - Class Decorators The proposal that added class decorators. Function and method decorators were introduced in **PEP 318**. + + +Multiple inheritance +==================== + +Python classes may have multiple base classes, a technique known as +*multiple inheritance*. The base classes are specified in the class +definition by listing them in parentheses after the class name, +separated by commas. For example, the following class definition: + + >>> class A: pass + >>> class B: pass + >>> class C(A, B): pass + +defines a class "C" that inherits from classes "A" and "B". + +The *method resolution order* (MRO) is the order in which base classes +are searched when looking up an attribute on a class. See The Python +2.3 Method Resolution Order for a description of how Python determines +the MRO for a class. + +Multiple inheritance is not always allowed. Attempting to define a +class with multiple inheritance will raise an error if one of the +bases does not allow subclassing, if a consistent MRO cannot be +created, if no valid metaclass can be determined, or if there is an +instance layout conflict. We’ll discuss each of these in turn. + +First, all base classes must allow subclassing. While most classes +allow subclassing, some built-in classes do not, such as "bool": + + >>> class SubBool(bool): # TypeError + ... pass + Traceback (most recent call last): + ... + TypeError: type 'bool' is not an acceptable base type + +In the resolved MRO of a class, the class’s bases appear in the order +they were specified in the class’s bases list. Additionally, the MRO +always lists a child class before any of its bases. A class definition +will fail if it is impossible to resolve a consistent MRO that +satisfies these rules from the list of bases provided: + + >>> class Base: pass + >>> class Child(Base): pass + >>> class Grandchild(Base, Child): pass # TypeError + Traceback (most recent call last): + ... + TypeError: Cannot create a consistent method resolution order (MRO) for bases Base, Child + +In the MRO of "Grandchild", "Base" must appear before "Child" because +it is first in the base class list, but it must also appear after +"Child" because it is a parent of "Child". This is a contradiction, so +the class cannot be defined. + +If some of the bases have a custom *metaclass*, the metaclass of the +resulting class is chosen among the metaclasses of the bases and the +explicitly specified metaclass of the child class. It must be a +metaclass that is a subclass of all other candidate metaclasses. If no +such metaclass exists among the candidates, the class cannot be +created, as explained in Determining the appropriate metaclass. + +Finally, the instance layouts of the bases must be compatible. This +means that it must be possible to compute a *solid base* for the +class. Exactly which classes are solid bases depends on the Python +implementation. + +**CPython implementation detail:** In CPython, a class is a solid base +if it has a nonempty "__slots__" definition. Many but not all classes +defined in C are also solid bases, including most builtins (such as +"int" or "BaseException") but excluding most concrete "Exception" +classes. Generally, a C class is a solid base if its underlying struct +is different in size from its base class. + +Every class has a solid base. "object", the base class, has itself as +its solid base. If there is a single base, the child class’s solid +base is that class if it is a solid base, or else the base class’s +solid base. If there are multiple bases, we first find the solid base +for each base class to produce a list of candidate solid bases. If +there is a unique solid base that is a subclass of all others, then +that class is the solid base. Otherwise, class creation fails. + +Example: + + >>> class Solid1: + ... __slots__ = ("solid1",) + >>> + >>> class Solid2: + ... __slots__ = ("solid2",) + >>> + >>> class SolidChild(Solid1): + ... __slots__ = ("solid_child",) + >>> + >>> class C1: # solid base is `object` + ... pass + >>> + >>> # OK: solid bases are `Solid1` and `object`, and `Solid1` is a subclass of `object`. + >>> class C2(Solid1, C1): # solid base is `Solid1` + ... pass + >>> + >>> # OK: solid bases are `SolidChild` and `Solid1`, and `SolidChild` is a subclass of `Solid1`. + >>> class C3(SolidChild, Solid1): # solid base is `SolidChild` + ... pass + >>> + >>> # Error: solid bases are `Solid1` and `Solid2`, but neither is a subclass of the other. + >>> class C4(Solid1, Solid2): # error: no single solid base + ... pass + Traceback (most recent call last): + ... + TypeError: multiple bases have instance lay-out conflict ''', 'comparisons': r'''Comparisons *********** @@ -1724,16 +1908,16 @@ class attributes; they are shared by instances. Instance attributes The "for" statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: - for_stmt: "for" target_list "in" starred_list ":" suite + for_stmt: "for" target_list "in" starred_expression_list ":" suite ["else" ":" suite] -The "starred_list" expression is evaluated once; it should yield an -*iterable* object. An *iterator* is created for that iterable. The -first item provided by the iterator is then assigned to the target -list using the standard rules for assignments (see Assignment -statements), and the suite is executed. This repeats for each item -provided by the iterator. When the iterator is exhausted, the suite -in the "else" clause, if present, is executed, and the loop +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop terminates. A "break" statement executed in the first suite terminates the loop @@ -1874,15 +2058,29 @@ class attributes; they are shared by instances. Instance attributes "except*" clause ---------------- -The "except*" clause(s) are used for handling "ExceptionGroup"s. The -exception type for matching is interpreted as in the case of "except", -but in the case of exception groups we can have partial matches when -the type matches some of the exceptions in the group. This means that -multiple "except*" clauses can execute, each handling part of the -exception group. Each clause executes at most once and handles an -exception group of all matching exceptions. Each exception in the -group is handled by at most one "except*" clause, the first that -matches it. +The "except*" clause(s) specify one or more handlers for groups of +exceptions ("BaseExceptionGroup" instances). A "try" statement can +have either "except" or "except*" clauses, but not both. The exception +type for matching is mandatory in the case of "except*", so "except*:" +is a syntax error. The type is interpreted as in the case of "except", +but matching is performed on the exceptions contained in the group +that is being handled. An "TypeError" is raised if a matching type is +a subclass of "BaseExceptionGroup", because that would have ambiguous +semantics. + +When an exception group is raised in the try block, each "except*" +clause splits (see "split()") it into the subgroups of matching and +non-matching exceptions. If the matching subgroup is not empty, it +becomes the handled exception (the value returned from +"sys.exception()") and assigned to the target of the "except*" clause +(if there is one). Then, the body of the "except*" clause executes. If +the non-matching subgroup is not empty, it is processed by the next +"except*" in the same manner. This continues until all exceptions in +the group have been matched, or the last "except*" clause has run. + +After all "except*" clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from +within "except*" clauses. This merged exception group propagates on.: >>> try: ... raise ExceptionGroup("eg", @@ -1895,20 +2093,19 @@ class attributes; they are shared by instances. Instance attributes caught with nested (TypeError(2),) caught with nested (OSError(3), OSError(4)) + Exception Group Traceback (most recent call last): - | File "", line 2, in - | ExceptionGroup: eg + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: 1 +------------------------------------ -Any remaining exceptions that were not handled by any "except*" clause -are re-raised at the end, along with all exceptions that were raised -from within the "except*" clauses. If this list contains more than one -exception to reraise, they are combined into an exception group. - -If the raised exception is not an exception group and its type matches -one of the "except*" clauses, it is caught and wrapped by an exception -group with an empty message string. +If the exception raised from the "try" block is not an exception group +and its type matches one of the "except*" clauses, it is caught and +wrapped by an exception group with an empty message string. This +ensures that the type of the target "e" is consistently +"BaseExceptionGroup": >>> try: ... raise BlockingIOError @@ -1917,11 +2114,6 @@ class attributes; they are shared by instances. Instance attributes ... ExceptionGroup('', (BlockingIOError())) -An "except*" clause must have a matching expression; it cannot be -"except*:". Furthermore, this expression cannot contain exception -group types, because that would have ambiguous semantics. - -It is not possible to mix "except" and "except*" in the same "try". "break", "continue" and "return" cannot appear in an "except*" clause. @@ -1938,11 +2130,11 @@ class attributes; they are shared by instances. Instance attributes ---------------- If "finally" is present, it specifies a ‘cleanup’ handler. The "try" -clause is executed, including any "except" and "else" clauses. If an +clause is executed, including any "except" and "else" clauses. If an exception occurs in any of the clauses and is not handled, the exception is temporarily saved. The "finally" clause is executed. If there is a saved exception it is re-raised at the end of the "finally" -clause. If the "finally" clause raises another exception, the saved +clause. If the "finally" clause raises another exception, the saved exception is set as the context of the new exception. If the "finally" clause executes a "return", "break" or "continue" statement, the saved exception is discarded. For example, this function returns 42. @@ -2097,9 +2289,9 @@ def foo(): The match statement is used for pattern matching. Syntax: match_stmt: 'match' subject_expr ":" NEWLINE INDENT case_block+ DEDENT - subject_expr: star_named_expression "," star_named_expressions? - | named_expression - case_block: 'case' patterns [guard] ":" block + subject_expr: `!star_named_expression` "," `!star_named_expressions`? + | `!named_expression` + case_block: 'case' patterns [guard] ":" `!block` Note: @@ -2190,7 +2382,7 @@ def foo(): Guards ------ - guard: "if" named_expression + guard: "if" `!named_expression` A "guard" (which is part of the "case") must succeed for code inside the "case" block to execute. It takes the form: "if" followed by an @@ -2329,7 +2521,8 @@ def foo(): The rule "strings" and the token "NUMBER" are defined in the standard Python grammar. Triple-quoted strings are supported. Raw strings and -byte strings are supported. f-strings are not supported. +byte strings are supported. f-strings and t-strings are not +supported. The forms "signed_number '+' NUMBER" and "signed_number '-' NUMBER" are for expressing complex numbers; they require a real number on the @@ -2483,7 +2676,7 @@ def foo(): Note: The length of the subject sequence is obtained via "len()" (i.e. - via the "__len__()" protocol). This length may be cached by the + via the "__len__()" protocol). This length may be cached by the interpreter in a similar manner as value patterns. In simple terms "[P1, P2, P3," … ", P]" matches only if all the @@ -2866,6 +3059,9 @@ class Foo: class Foo(object): pass +There may be one or more base classes; see Multiple inheritance below +for more information. + The class’s suite is then executed in a new execution frame (see Naming and binding), using a newly created local namespace and the original global namespace. (Usually, the suite contains mostly @@ -2931,6 +3127,115 @@ class attributes; they are shared by instances. Instance attributes decorators were introduced in **PEP 318**. +Multiple inheritance +-------------------- + +Python classes may have multiple base classes, a technique known as +*multiple inheritance*. The base classes are specified in the class +definition by listing them in parentheses after the class name, +separated by commas. For example, the following class definition: + + >>> class A: pass + >>> class B: pass + >>> class C(A, B): pass + +defines a class "C" that inherits from classes "A" and "B". + +The *method resolution order* (MRO) is the order in which base classes +are searched when looking up an attribute on a class. See The Python +2.3 Method Resolution Order for a description of how Python determines +the MRO for a class. + +Multiple inheritance is not always allowed. Attempting to define a +class with multiple inheritance will raise an error if one of the +bases does not allow subclassing, if a consistent MRO cannot be +created, if no valid metaclass can be determined, or if there is an +instance layout conflict. We’ll discuss each of these in turn. + +First, all base classes must allow subclassing. While most classes +allow subclassing, some built-in classes do not, such as "bool": + + >>> class SubBool(bool): # TypeError + ... pass + Traceback (most recent call last): + ... + TypeError: type 'bool' is not an acceptable base type + +In the resolved MRO of a class, the class’s bases appear in the order +they were specified in the class’s bases list. Additionally, the MRO +always lists a child class before any of its bases. A class definition +will fail if it is impossible to resolve a consistent MRO that +satisfies these rules from the list of bases provided: + + >>> class Base: pass + >>> class Child(Base): pass + >>> class Grandchild(Base, Child): pass # TypeError + Traceback (most recent call last): + ... + TypeError: Cannot create a consistent method resolution order (MRO) for bases Base, Child + +In the MRO of "Grandchild", "Base" must appear before "Child" because +it is first in the base class list, but it must also appear after +"Child" because it is a parent of "Child". This is a contradiction, so +the class cannot be defined. + +If some of the bases have a custom *metaclass*, the metaclass of the +resulting class is chosen among the metaclasses of the bases and the +explicitly specified metaclass of the child class. It must be a +metaclass that is a subclass of all other candidate metaclasses. If no +such metaclass exists among the candidates, the class cannot be +created, as explained in Determining the appropriate metaclass. + +Finally, the instance layouts of the bases must be compatible. This +means that it must be possible to compute a *solid base* for the +class. Exactly which classes are solid bases depends on the Python +implementation. + +**CPython implementation detail:** In CPython, a class is a solid base +if it has a nonempty "__slots__" definition. Many but not all classes +defined in C are also solid bases, including most builtins (such as +"int" or "BaseException") but excluding most concrete "Exception" +classes. Generally, a C class is a solid base if its underlying struct +is different in size from its base class. + +Every class has a solid base. "object", the base class, has itself as +its solid base. If there is a single base, the child class’s solid +base is that class if it is a solid base, or else the base class’s +solid base. If there are multiple bases, we first find the solid base +for each base class to produce a list of candidate solid bases. If +there is a unique solid base that is a subclass of all others, then +that class is the solid base. Otherwise, class creation fails. + +Example: + + >>> class Solid1: + ... __slots__ = ("solid1",) + >>> + >>> class Solid2: + ... __slots__ = ("solid2",) + >>> + >>> class SolidChild(Solid1): + ... __slots__ = ("solid_child",) + >>> + >>> class C1: # solid base is `object` + ... pass + >>> + >>> # OK: solid bases are `Solid1` and `object`, and `Solid1` is a subclass of `object`. + >>> class C2(Solid1, C1): # solid base is `Solid1` + ... pass + >>> + >>> # OK: solid bases are `SolidChild` and `Solid1`, and `SolidChild` is a subclass of `Solid1`. + >>> class C3(SolidChild, Solid1): # solid base is `SolidChild` + ... pass + >>> + >>> # Error: solid bases are `Solid1` and `Solid2`, but neither is a subclass of the other. + >>> class C4(Solid1, Solid2): # error: no single solid base + ... pass + Traceback (most recent call last): + ... + TypeError: multiple bases have instance lay-out conflict + + Coroutines ========== @@ -3304,7 +3609,7 @@ def f() -> annotation: ... introspects and uses the annotations (such as "dataclasses" or "functools.singledispatch()"). -By default, annotations are lazily evaluated in a annotation scope. +By default, annotations are lazily evaluated in an annotation scope. This means that they are not evaluated when the code containing the annotation is evaluated. Instead, the interpreter saves information that can be used to evaluate the annotation later if requested. The @@ -3318,6 +3623,12 @@ def f() -> annotation: ... >>> f.__annotations__ {'param': 'annotation'} +This future statement will be deprecated and removed in a future +version of Python, but not before Python 3.13 reaches its end of life +(see **PEP 749**). When it is used, introspection tools like +"annotationlib.get_annotations()" and "typing.get_type_hints()" are +less likely to be able to resolve annotations at runtime. + -[ Footnotes ]- [1] The exception is propagated to the invocation stack unless there @@ -3829,10 +4140,14 @@ def double(x): available for commands and command arguments, e.g. the current global and local names are offered as arguments of the "p" command. + +Command-line interface +====================== + You can also invoke "pdb" from the command line to debug other scripts. For example: - python -m pdb [-c command] (-m module | pyfile) [args ...] + python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] When invoked as a module, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post- @@ -3844,7 +4159,7 @@ def double(x): -c, --command To execute commands as if given in a ".pdbrc" file; see Debugger - Commands. + commands. Changed in version 3.2: Added the "-c" option. @@ -3856,6 +4171,23 @@ def double(x): Changed in version 3.7: Added the "-m" option. +-p, --pid + + Attach to the process with the specified PID. + + Added in version 3.14. + +To attach to a running Python process for remote debugging, use the +"-p" or "--pid" option with the target process’s PID: + + python -m pdb -p 1234 + +Note: + + Attaching to a process that is blocked in a system call or waiting + for I/O will only work once the next bytecode instruction is + executed or when the process receives a signal. + Typical usage to execute a statement under control of the debugger is: >>> import pdb @@ -4048,7 +4380,7 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa See the documentation for the functions explained above. -Debugger Commands +Debugger commands ================= The commands recognized by the debugger are listed below. Most @@ -4574,8 +4906,8 @@ def inner(x): Deletion of a name removes the binding of that name from the local or global namespace, depending on whether the name occurs in a "global" -statement in the same code block. If the name is unbound, a -"NameError" exception will be raised. +statement in the same code block. Trying to delete an unbound name +raises a "NameError" exception. Deletion of attribute references, subscriptions and slicings is passed to the primary object involved; deletion of a slicing is in general @@ -4711,11 +5043,6 @@ class of the instance or a *non-virtual base class* thereof. The See also the description of the "try" statement in section The try statement and "raise" statement in section The raise statement. - --[ Footnotes ]- - -[1] This limitation occurs because the code that is executed by these - operations is not available at the time the module is compiled. ''', 'execmodel': r'''Execution model *************** @@ -5069,6 +5396,181 @@ class of the instance or a *non-virtual base class* thereof. The See also the description of the "try" statement in section The try statement and "raise" statement in section The raise statement. + +Runtime Components +================== + + +General Computing Model +----------------------- + +Python’s execution model does not operate in a vacuum. It runs on a +host machine and through that host’s runtime environment, including +its operating system (OS), if there is one. When a program runs, the +conceptual layers of how it runs on the host look something like this: + + **host machine** + **process** (global resources) + **thread** (runs machine code) + +Each process represents a program running on the host. Think of each +process itself as the data part of its program. Think of the process’ +threads as the execution part of the program. This distinction will +be important to understand the conceptual Python runtime. + +The process, as the data part, is the execution context in which the +program runs. It mostly consists of the set of resources assigned to +the program by the host, including memory, signals, file handles, +sockets, and environment variables. + +Processes are isolated and independent from one another. (The same is +true for hosts.) The host manages the process’ access to its assigned +resources, in addition to coordinating between processes. + +Each thread represents the actual execution of the program’s machine +code, running relative to the resources assigned to the program’s +process. It’s strictly up to the host how and when that execution +takes place. + +From the point of view of Python, a program always starts with exactly +one thread. However, the program may grow to run in multiple +simultaneous threads. Not all hosts support multiple threads per +process, but most do. Unlike processes, threads in a process are not +isolated and independent from one another. Specifically, all threads +in a process share all of the process’ resources. + +The fundamental point of threads is that each one does *run* +independently, at the same time as the others. That may be only +conceptually at the same time (“concurrently”) or physically (“in +parallel”). Either way, the threads effectively run at a non- +synchronized rate. + +Note: + + That non-synchronized rate means none of the process’ memory is + guaranteed to stay consistent for the code running in any given + thread. Thus multi-threaded programs must take care to coordinate + access to intentionally shared resources. Likewise, they must take + care to be absolutely diligent about not accessing any *other* + resources in multiple threads; otherwise two threads running at the + same time might accidentally interfere with each other’s use of some + shared data. All this is true for both Python programs and the + Python runtime.The cost of this broad, unstructured requirement is + the tradeoff for the kind of raw concurrency that threads provide. + The alternative to the required discipline generally means dealing + with non-deterministic bugs and data corruption. + + +Python Runtime Model +-------------------- + +The same conceptual layers apply to each Python program, with some +extra data layers specific to Python: + + **host machine** + **process** (global resources) + Python global runtime (*state*) + Python interpreter (*state*) + **thread** (runs Python bytecode and “C-API”) + Python thread *state* + +At the conceptual level: when a Python program starts, it looks +exactly like that diagram, with one of each. The runtime may grow to +include multiple interpreters, and each interpreter may grow to +include multiple thread states. + +Note: + + A Python implementation won’t necessarily implement the runtime + layers distinctly or even concretely. The only exception is places + where distinct layers are directly specified or exposed to users, + like through the "threading" module. + +Note: + + The initial interpreter is typically called the “main” interpreter. + Some Python implementations, like CPython, assign special roles to + the main interpreter.Likewise, the host thread where the runtime was + initialized is known as the “main” thread. It may be different from + the process’ initial thread, though they are often the same. In + some cases “main thread” may be even more specific and refer to the + initial thread state. A Python runtime might assign specific + responsibilities to the main thread, such as handling signals. + +As a whole, the Python runtime consists of the global runtime state, +interpreters, and thread states. The runtime ensures all that state +stays consistent over its lifetime, particularly when used with +multiple host threads. + +The global runtime, at the conceptual level, is just a set of +interpreters. While those interpreters are otherwise isolated and +independent from one another, they may share some data or other +resources. The runtime is responsible for managing these global +resources safely. The actual nature and management of these resources +is implementation-specific. Ultimately, the external utility of the +global runtime is limited to managing interpreters. + +In contrast, an “interpreter” is conceptually what we would normally +think of as the (full-featured) “Python runtime”. When machine code +executing in a host thread interacts with the Python runtime, it calls +into Python in the context of a specific interpreter. + +Note: + + The term “interpreter” here is not the same as the “bytecode + interpreter”, which is what regularly runs in threads, executing + compiled Python code.In an ideal world, “Python runtime” would refer + to what we currently call “interpreter”. However, it’s been called + “interpreter” at least since introduced in 1997 (CPython:a027efa5b). + +Each interpreter completely encapsulates all of the non-process- +global, non-thread-specific state needed for the Python runtime to +work. Notably, the interpreter’s state persists between uses. It +includes fundamental data like "sys.modules". The runtime ensures +multiple threads using the same interpreter will safely share it +between them. + +A Python implementation may support using multiple interpreters at the +same time in the same process. They are independent and isolated from +one another. For example, each interpreter has its own "sys.modules". + +For thread-specific runtime state, each interpreter has a set of +thread states, which it manages, in the same way the global runtime +contains a set of interpreters. It can have thread states for as many +host threads as it needs. It may even have multiple thread states for +the same host thread, though that isn’t as common. + +Each thread state, conceptually, has all the thread-specific runtime +data an interpreter needs to operate in one host thread. The thread +state includes the current raised exception and the thread’s Python +call stack. It may include other thread-specific resources. + +Note: + + The term “Python thread” can sometimes refer to a thread state, but + normally it means a thread created using the "threading" module. + +Each thread state, over its lifetime, is always tied to exactly one +interpreter and exactly one host thread. It will only ever be used in +that thread and with that interpreter. + +Multiple thread states may be tied to the same host thread, whether +for different interpreters or even the same interpreter. However, for +any given host thread, only one of the thread states tied to it can be +used by the thread at a time. + +Thread states are isolated and independent from one another and don’t +share any data, except for possibly sharing an interpreter and objects +or other resources belonging to that interpreter. + +Once a program is running, new Python threads can be created using the +"threading" module (on platforms and Python implementations that +support threads). Additional processes can be created using the "os", +"subprocess", and "multiprocessing" modules. Interpreters can be +created and used with the "interpreters" module. Coroutines (async) +can be run using "asyncio" in each interpreter, typically only in a +single thread (often the main thread). + -[ Footnotes ]- [1] This limitation occurs because the code that is executed by these @@ -5077,7 +5579,7 @@ class of the instance or a *non-virtual base class* thereof. The 'exprlists': r'''Expression lists **************** - starred_expression: ["*"] or_expr + starred_expression: "*" or_expr | expression flexible_expression: assignment_expression | starred_expression flexible_expression_list: flexible_expression ("," flexible_expression)* [","] starred_expression_list: starred_expression ("," starred_expression)* [","] @@ -5109,25 +5611,53 @@ class of the instance or a *non-virtual base class* thereof. The 'floating': r'''Floating-point literals *********************** -Floating-point literals are described by the following lexical -definitions: +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. - floatnumber: pointfloat | exponentfloat - pointfloat: [digitpart] fraction | digitpart "." - exponentfloat: (digitpart | pointfloat) exponent - digitpart: digit (["_"] digit)* - fraction: "." digitpart - exponent: ("e" | "E") ["+" | "-"] digitpart +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": -Note that the integer and exponent parts are always interpreted using -radix 10. For example, "077e010" is legal, and denotes the same number -as "77e10". The allowed range of floating-point literals is -implementation-dependent. As in integer literals, underscores are -supported for digit grouping. + 2.71828 + 4.0 -Some examples of floating-point literals: +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". - 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart Changed in version 3.6: Underscores are now allowed for grouping purposes in literals. @@ -5138,16 +5668,16 @@ class of the instance or a *non-virtual base class* thereof. The The "for" statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: - for_stmt: "for" target_list "in" starred_list ":" suite + for_stmt: "for" target_list "in" starred_expression_list ":" suite ["else" ":" suite] -The "starred_list" expression is evaluated once; it should yield an -*iterable* object. An *iterator* is created for that iterable. The -first item provided by the iterator is then assigned to the target -list using the standard rules for assignments (see Assignment -statements), and the suite is executed. This repeats for each item -provided by the iterator. When the iterator is exhausted, the suite -in the "else" clause, if present, is executed, and the loop +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop terminates. A "break" statement executed in the first suite terminates the loop @@ -5181,9 +5711,9 @@ class of the instance or a *non-virtual base class* thereof. The The "str.format()" method and the "Formatter" class share the same syntax for format strings (although in the case of "Formatter", subclasses can define their own format string syntax). The syntax is -related to that of formatted string literals, but it is less -sophisticated and, in particular, does not support arbitrary -expressions. +related to that of formatted string literals and template string +literals, but it is less sophisticated and, in particular, does not +support arbitrary expressions in interpolations. Format strings contain “replacement fields” surrounded by curly braces "{}". Anything that is not contained in braces is considered literal @@ -5283,9 +5813,9 @@ class of the instance or a *non-virtual base class* thereof. The “Format specifications” are used within replacement fields contained within a format string to define how individual values are presented -(see Format String Syntax and f-strings). They can also be passed -directly to the built-in "format()" function. Each formattable type -may define how the format specification is to be interpreted. +(see Format String Syntax, f-strings, and t-strings). They can also be +passed directly to the built-in "format()" function. Each formattable +type may define how the format specification is to be interpreted. Most built-in types implement the following options for format specifications, although some of the formatting options are only @@ -5304,7 +5834,7 @@ class of the instance or a *non-virtual base class* thereof. The sign: "+" | "-" | " " width_and_precision: [width_with_grouping][precision_with_grouping] width_with_grouping: [width][grouping] - precision_with_grouping: "." [precision][grouping] + precision_with_grouping: "." [precision][grouping] | "." grouping width: digit+ precision: digit+ grouping: "," | "_" @@ -5886,9 +6416,15 @@ def whats_on_the_telly(penguin=None): without "global", although free variables may refer to globals without being declared global. -The "global" statement applies to the entire scope of a function or -class body. A "SyntaxError" is raised if a variable is used or -assigned to prior to its global declaration in the scope. +The "global" statement applies to the entire current scope (module, +function body or class definition). A "SyntaxError" is raised if a +variable is used or assigned to prior to its global declaration in the +scope. + +At the module level, all variables are global, so a "global" statement +has no effect. However, variables must still not be used or assigned +to prior to their "global" declaration. This requirement is relaxed in +the interactive prompt (*REPL*). **Programmer’s note:** "global" is a directive to the parser. It applies only to code parsed at the same time as the "global" @@ -5942,73 +6478,92 @@ class body. A "SyntaxError" is raised if a variable is used or to help avoid name clashes between “private” attributes of base and derived classes. See section Identifiers (Names). ''', - 'identifiers': r'''Identifiers and keywords -************************ + 'identifiers': r'''Names (identifiers and keywords) +******************************** -Identifiers (also referred to as *names*) are described by the -following lexical definitions. - -The syntax of identifiers in Python is based on the Unicode standard -annex UAX-31, with elaboration and changes as defined below; see also -**PEP 3131** for further details. +"NAME" tokens represent *identifiers*, *keywords*, and *soft +keywords*. Within the ASCII range (U+0001..U+007F), the valid characters for -identifiers include the uppercase and lowercase letters "A" through -"Z", the underscore "_" and, except for the first character, the -digits "0" through "9". Python 3.0 introduced additional characters -from outside the ASCII range (see **PEP 3131**). For these -characters, the classification uses the version of the Unicode -Character Database as included in the "unicodedata" module. +names include the uppercase and lowercase letters ("A-Z" and "a-z"), +the underscore "_" and, except for the first character, the digits "0" +through "9". -Identifiers are unlimited in length. Case is significant. +Names must contain at least one character, but have no upper length +limit. Case is significant. - identifier: xid_start xid_continue* - id_start: - id_continue: - xid_start: - xid_continue: +Besides "A-Z", "a-z", "_" and "0-9", names can also use “letter-like” +and “number-like” characters from outside the ASCII range, as detailed +below. -The Unicode category codes mentioned above stand for: +All identifiers are converted into the normalization form NFKC while +parsing; comparison of identifiers is based on NFKC. -* *Lu* - uppercase letters +Formally, the first character of a normalized identifier must belong +to the set "id_start", which is the union of: -* *Ll* - lowercase letters +* Unicode category "" - uppercase letters (includes "A" to "Z") -* *Lt* - titlecase letters +* Unicode category "" - lowercase letters (includes "a" to "z") -* *Lm* - modifier letters +* Unicode category "" - titlecase letters -* *Lo* - other letters +* Unicode category "" - modifier letters -* *Nl* - letter numbers +* Unicode category "" - other letters -* *Mn* - nonspacing marks +* Unicode category "" - letter numbers -* *Mc* - spacing combining marks +* {""_""} - the underscore -* *Nd* - decimal numbers +* "" - an explicit set of characters in PropList.txt + to support backwards compatibility -* *Pc* - connector punctuations +The remaining characters must belong to the set "id_continue", which +is the union of: -* *Other_ID_Start* - explicit list of characters in PropList.txt to - support backwards compatibility +* all characters in "id_start" -* *Other_ID_Continue* - likewise +* Unicode category "" - decimal numbers (includes "0" to "9") -All identifiers are converted into the normal form NFKC while parsing; -comparison of identifiers is based on NFKC. +* Unicode category "" - connector punctuations -A non-normative HTML file listing all valid identifier characters for -Unicode 16.0.0 can be found at -https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt +* Unicode category "" - nonspacing marks + +* Unicode category "" - spacing combining marks + +* "" - another explicit set of characters in + PropList.txt to support backwards compatibility + +Unicode categories use the version of the Unicode Character Database +as included in the "unicodedata" module. + +These sets are based on the Unicode standard annex UAX-31. See also +**PEP 3131** for further details. + +Even more formally, names are described by the following lexical +definitions: + + NAME: xid_start xid_continue* + id_start: | | | | | | "_" | + id_continue: id_start | | | | | + xid_start: + xid_continue: + identifier: + +A non-normative listing of all valid identifier characters as defined +by Unicode is available in the DerivedCoreProperties.txt file in the +Unicode Character Database. Keywords ======== -The following identifiers are used as reserved words, or *keywords* of -the language, and cannot be used as ordinary identifiers. They must -be spelled exactly as written here: +The following names are used as reserved words, or *keywords* of the +language, and cannot be used as ordinary identifiers. They must be +spelled exactly as written here: False await else import pass None break except in raise @@ -6024,18 +6579,20 @@ class body. A "SyntaxError" is raised if a variable is used or Added in version 3.10. -Some identifiers are only reserved under specific contexts. These are -known as *soft keywords*. The identifiers "match", "case", "type" and -"_" can syntactically act as keywords in certain contexts, but this -distinction is done at the parser level, not when tokenizing. +Some names are only reserved under specific contexts. These are known +as *soft keywords*: + +* "match", "case", and "_", when used in the "match" statement. + +* "type", when used in the "type" statement. + +These syntactically act as keywords in their specific contexts, but +this distinction is done at the parser level, not when tokenizing. As soft keywords, their use in the grammar is possible while still preserving compatibility with existing code that uses these names as identifier names. -"match", "case", and "_" are used in the "match" statement. "type" is -used in the "type" statement. - Changed in version 3.12: "type" is now a soft keyword. @@ -6101,17 +6658,53 @@ class body. A "SyntaxError" is raised if a variable is used or 'imaginary': r'''Imaginary literals ****************** -Imaginary literals are described by the following lexical definitions: +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. + +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: imagnumber: (floatnumber | digitpart) ("j" | "J") - -An imaginary literal yields a complex number with a real part of 0.0. -Complex numbers are represented as a pair of floating-point numbers -and have the same restrictions on their range. To create a complex -number with a nonzero real part, add a floating-point number to it, -e.g., "(3+4j)". Some examples of imaginary literals: - - 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j ''', 'import': r'''The "import" statement ********************** @@ -6353,37 +6946,62 @@ class body. A "SyntaxError" is raised if a variable is used or 'integers': r'''Integer literals **************** -Integer literals are described by the following lexical definitions: +Integer literals denote whole numbers. For example: - integer: decinteger | bininteger | octinteger | hexinteger - decinteger: nonzerodigit (["_"] digit)* | "0"+ (["_"] "0")* + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* bininteger: "0" ("b" | "B") (["_"] bindigit)+ octinteger: "0" ("o" | "O") (["_"] octdigit)+ hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* nonzerodigit: "1"..."9" digit: "0"..."9" bindigit: "0" | "1" octdigit: "0"..."7" hexdigit: digit | "a"..."f" | "A"..."F" -There is no limit for the length of integer literals apart from what -can be stored in available memory. - -Underscores are ignored for determining the numeric value of the -literal. They can be used to group digits for enhanced readability. -One underscore can occur between digits, and after base specifiers -like "0x". - -Note that leading zeros in a non-zero decimal number are not allowed. -This is for disambiguation with C-style octal literals, which Python -used before version 3.0. - -Some examples of integer literals: - - 7 2147483647 0o177 0b100110111 - 3 79228162514264337593543950336 0o377 0xdeadbeef - 100_000_000_000 0b_1110_0101 - Changed in version 3.6: Underscores are now allowed for grouping purposes in literals. ''', @@ -6730,14 +7348,189 @@ class body. A "SyntaxError" is raised if a variable is used or 'numbers': r'''Numeric literals **************** -There are three types of numeric literals: integers, floating-point -numbers, and imaginary numbers. There are no complex literals -(complex numbers can be formed by adding a real number and an -imaginary number). +"NUMBER" tokens represent numeric literals, of which there are three +types: integers, floating-point numbers, and imaginary numbers. -Note that numeric literals do not include a sign; a phrase like "-1" -is actually an expression composed of the unary operator ‘"-"’ and the -literal "1". + NUMBER: integer | floatnumber | imagnumber + +The numeric value of a numeric literal is the same as if it were +passed as a string to the "int", "float" or "complex" class +constructor, respectively. Note that not all valid inputs for those +constructors are also valid literals. + +Numeric literals do not include a sign; a phrase like "-1" is actually +an expression composed of the unary operator ‘"-"’ and the literal +"1". + + +Integer literals +================ + +Integer literals denote whole numbers. For example: + + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* + bininteger: "0" ("b" | "B") (["_"] bindigit)+ + octinteger: "0" ("o" | "O") (["_"] octdigit)+ + hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* + nonzerodigit: "1"..."9" + digit: "0"..."9" + bindigit: "0" | "1" + octdigit: "0"..."7" + hexdigit: digit | "a"..."f" | "A"..."F" + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Floating-point literals +======================= + +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Imaginary literals +================== + +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. + +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") ''', 'numeric-types': r'''Emulating numeric types *********************** @@ -6807,9 +7600,9 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is third argument if the three-argument version of the built-in "pow()" function is to be supported. - Changed in version 3.14.0a7 (unreleased): Three-argument "pow()" - now try calling "__rpow__()" if necessary. Previously it was only - called in two-argument "pow()" and the binary power operator. + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. Note: @@ -6894,9 +7687,8 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is ************************* *Objects* are Python’s abstraction for data. All data in a Python -program is represented by objects or by relations between objects. (In -a sense, and in conformance to Von Neumann’s model of a “stored -program computer”, code is also represented by objects.) +program is represented by objects or by relations between objects. +Even code is represented by objects. Every object has an identity, a type and a value. An object’s *identity* never changes once it has been created; you may think of it @@ -7283,21 +8075,24 @@ class C: pass # a class with no methods (yet) The "collections.abc" module provides a "MutableMapping" *abstract base class* to help create those methods from a base set of "__getitem__()", "__setitem__()", "__delitem__()", and "keys()". -Mutable sequences should provide methods "append()", "count()", -"index()", "extend()", "insert()", "pop()", "remove()", "reverse()" -and "sort()", like Python standard "list" objects. Finally, sequence + +Mutable sequences should provide methods "append()", "clear()", +"count()", "extend()", "index()", "insert()", "pop()", "remove()", and +"reverse()", like Python standard "list" objects. Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods "__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described below; they should not define other numerical -operators. It is recommended that both mappings and sequences -implement the "__contains__()" method to allow efficient use of the -"in" operator; for mappings, "in" should search the mapping’s keys; -for sequences, it should search through the values. It is further -recommended that both mappings and sequences implement the -"__iter__()" method to allow efficient iteration through the -container; for mappings, "__iter__()" should iterate through the -object’s keys; for sequences, it should iterate through the values. +operators. + +It is recommended that both mappings and sequences implement the +"__contains__()" method to allow efficient use of the "in" operator; +for mappings, "in" should search the mapping’s keys; for sequences, it +should search through the values. It is further recommended that both +mappings and sequences implement the "__iter__()" method to allow +efficient iteration through the container; for mappings, "__iter__()" +should iterate through the object’s keys; for sequences, it should +iterate through the values. object.__len__(self) @@ -7534,8 +8329,8 @@ class C: pass # a class with no methods (yet) important that the emulation only be implemented to the degree that it makes sense for the object being modelled. For example, some sequences may work well with retrieval of individual elements, but -extracting a slice may not make sense. (One example of this is the -"NodeList" interface in the W3C’s Document Object Model.) +extracting a slice may not make sense. (One example of this is the +NodeList interface in the W3C’s Document Object Model.) Basic customization @@ -7930,6 +8725,9 @@ class instances. Customizing module attribute access ----------------------------------- +module.__getattr__() +module.__dir__() + Special names "__getattr__" and "__dir__" can be also used to customize access to module attributes. The "__getattr__" function at the module level should accept one argument which is the name of an @@ -7945,6 +8743,8 @@ class instances. present, this function overrides the standard "dir()" search on a module. +module.__class__ + For a more fine grained customization of the module behavior (setting attributes, properties, etc.), one can set the "__class__" attribute of a module object to a subclass of "types.ModuleType". For example: @@ -8640,21 +9440,24 @@ class of a class is known as that class’s *metaclass*, and most The "collections.abc" module provides a "MutableMapping" *abstract base class* to help create those methods from a base set of "__getitem__()", "__setitem__()", "__delitem__()", and "keys()". -Mutable sequences should provide methods "append()", "count()", -"index()", "extend()", "insert()", "pop()", "remove()", "reverse()" -and "sort()", like Python standard "list" objects. Finally, sequence + +Mutable sequences should provide methods "append()", "clear()", +"count()", "extend()", "index()", "insert()", "pop()", "remove()", and +"reverse()", like Python standard "list" objects. Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods "__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described below; they should not define other numerical -operators. It is recommended that both mappings and sequences -implement the "__contains__()" method to allow efficient use of the -"in" operator; for mappings, "in" should search the mapping’s keys; -for sequences, it should search through the values. It is further -recommended that both mappings and sequences implement the -"__iter__()" method to allow efficient iteration through the -container; for mappings, "__iter__()" should iterate through the -object’s keys; for sequences, it should iterate through the values. +operators. + +It is recommended that both mappings and sequences implement the +"__contains__()" method to allow efficient use of the "in" operator; +for mappings, "in" should search the mapping’s keys; for sequences, it +should search through the values. It is further recommended that both +mappings and sequences implement the "__iter__()" method to allow +efficient iteration through the container; for mappings, "__iter__()" +should iterate through the object’s keys; for sequences, it should +iterate through the values. object.__len__(self) @@ -8845,9 +9648,9 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is third argument if the three-argument version of the built-in "pow()" function is to be supported. - Changed in version 3.14.0a7 (unreleased): Three-argument "pow()" - now try calling "__rpow__()" if necessary. Previously it was only - called in two-argument "pow()" and the binary power operator. + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. Note: @@ -9205,17 +10008,24 @@ class is used in a class pattern with positional arguments, each Since it is already lowercase, "lower()" would do nothing to "'ß'"; "casefold()" converts it to ""ss"". - The casefolding algorithm is described in section 3.13 ‘Default + The casefolding algorithm is described in section 3.13.3 ‘Default Case Folding’ of the Unicode Standard. Added in version 3.3. -str.center(width[, fillchar]) +str.center(width, fillchar=' ', /) Return centered in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to - "len(s)". + "len(s)". For example: + + >>> 'Python'.center(10) + ' Python ' + >>> 'Python'.center(10, '-') + '--Python--' + >>> 'Python'.center(4) + 'Python' str.count(sub[, start[, end]]) @@ -9224,7 +10034,18 @@ class is used in a class pattern with positional arguments, each *end* are interpreted as in slice notation. If *sub* is empty, returns the number of empty strings between - characters which is the length of the string plus one. + characters which is the length of the string plus one. For example: + + >>> 'spam, spam, spam'.count('spam') + 3 + >>> 'spam, spam, spam'.count('spam', 5) + 2 + >>> 'spam, spam, spam'.count('spam', 5, 10) + 1 + >>> 'spam, spam, spam'.count('eggs') + 0 + >>> 'spam, spam, spam'.count('') + 17 str.encode(encoding='utf-8', errors='strict') @@ -9241,7 +10062,13 @@ class is used in a class pattern with positional arguments, each For performance reasons, the value of *errors* is not checked for validity unless an encoding error actually occurs, Python - Development Mode is enabled or a debug build is used. + Development Mode is enabled or a debug build is used. For example: + + >>> encoded_str_to_bytes = 'Python'.encode() + >>> type(encoded_str_to_bytes) + + >>> encoded_str_to_bytes + b'Python' Changed in version 3.1: Added support for keyword arguments. @@ -9254,6 +10081,19 @@ class is used in a class pattern with positional arguments, each otherwise return "False". *suffix* can also be a tuple of suffixes to look for. With optional *start*, test beginning at that position. With optional *end*, stop comparing at that position. + Using *start* and *end* is equivalent to + "str[start:end].endswith(suffix)". For example: + + >>> 'Python'.endswith('on') + True + >>> 'a tuple of suffixes'.endswith(('at', 'in')) + False + >>> 'a tuple of suffixes'.endswith(('at', 'es')) + True + >>> 'Python is amazing'.endswith('is', 0, 9) + True + + See also "startswith()" and "removesuffix()". str.expandtabs(tabsize=8) @@ -9269,19 +10109,29 @@ class is used in a class pattern with positional arguments, each ("\n") or return ("\r"), it is copied and the current column is reset to zero. Any other character is copied unchanged and the current column is incremented by one regardless of how the - character is represented when printed. + character is represented when printed. For example: - >>> '01\t012\t0123\t01234'.expandtabs() - '01 012 0123 01234' - >>> '01\t012\t0123\t01234'.expandtabs(4) - '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs() + '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs(4) + '01 012 0123 01234' + >>> print('01\t012\n0123\t01234'.expandtabs(4)) + 01 012 + 0123 01234 str.find(sub[, start[, end]]) Return the lowest index in the string where substring *sub* is found within the slice "s[start:end]". Optional arguments *start* and *end* are interpreted as in slice notation. Return "-1" if - *sub* is not found. + *sub* is not found. For example: + + >>> 'spam, spam, spam'.find('sp') + 0 + >>> 'spam, spam, spam'.find('sp', 5) + 6 + + See also "rfind()" and "index()". Note: @@ -9389,7 +10239,7 @@ class is used in a class pattern with positional arguments, each str.isidentifier() Return "True" if the string is a valid identifier according to the - language definition, section Identifiers and keywords. + language definition, section Names (identifiers and keywords). "keyword.iskeyword()" can be used to test whether string "s" is a reserved identifier, such as "def" and "class". @@ -9421,8 +10271,8 @@ class is used in a class pattern with positional arguments, each str.isprintable() - Return true if all characters in the string are printable, false if - it contains at least one non-printable character. + Return "True" if all characters in the string are printable, + "False" if it contains at least one non-printable character. Here “printable” means the character is suitable for "repr()" to use in its output; “non-printable” means that "repr()" on built-in @@ -9452,6 +10302,17 @@ class is used in a class pattern with positional arguments, each follow uncased characters and lowercase characters only cased ones. Return "False" otherwise. + For example: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also "title()". + str.isupper() Return "True" if all cased characters [4] in the string are @@ -9467,14 +10328,14 @@ class is used in a class pattern with positional arguments, each >>> ' '.isupper() False -str.join(iterable) +str.join(iterable, /) Return a string which is the concatenation of the strings in *iterable*. A "TypeError" will be raised if there are any non- string values in *iterable*, including "bytes" objects. The separator between elements is the string providing this method. -str.ljust(width[, fillchar]) +str.ljust(width, fillchar=' ', /) Return the string left justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII @@ -9486,10 +10347,10 @@ class is used in a class pattern with positional arguments, each Return a copy of the string with all the cased characters [4] converted to lowercase. - The lowercasing algorithm used is described in section 3.13 - ‘Default Case Folding’ of the Unicode Standard. + The lowercasing algorithm used is described in section 3.13.2 + ‘Default Case Conversion’ of the Unicode Standard. -str.lstrip([chars]) +str.lstrip(chars=None, /) Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be @@ -9510,7 +10371,8 @@ class is used in a class pattern with positional arguments, each >>> 'Arthur: three!'.removeprefix('Arthur: ') 'three!' -static str.maketrans(x[, y[, z]]) +static str.maketrans(dict, /) +static str.maketrans(from, to, remove='', /) This static method returns a translation table usable for "str.translate()". @@ -9521,12 +10383,12 @@ class is used in a class pattern with positional arguments, each Character keys will then be converted to ordinals. If there are two arguments, they must be strings of equal length, - and in the resulting dictionary, each character in x will be mapped - to the character at the same position in y. If there is a third - argument, it must be a string, whose characters will be mapped to - "None" in the result. + and in the resulting dictionary, each character in *from* will be + mapped to the character at the same position in *to*. If there is + a third argument, it must be a string, whose characters will be + mapped to "None" in the result. -str.partition(sep) +str.partition(sep, /) Split the string at the first occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator @@ -9560,7 +10422,7 @@ class is used in a class pattern with positional arguments, each Added in version 3.9. -str.replace(old, new, count=-1) +str.replace(old, new, /, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by *new*. If *count* is given, only the first *count* @@ -9582,14 +10444,14 @@ class is used in a class pattern with positional arguments, each Like "rfind()" but raises "ValueError" when the substring *sub* is not found. -str.rjust(width[, fillchar]) +str.rjust(width, fillchar=' ', /) Return the string right justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to "len(s)". -str.rpartition(sep) +str.rpartition(sep, /) Split the string at the last occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator @@ -9606,7 +10468,7 @@ class is used in a class pattern with positional arguments, each from the right, "rsplit()" behaves like "split()" which is described in detail below. -str.rstrip([chars]) +str.rstrip(chars=None, /) Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be @@ -9669,6 +10531,18 @@ class is used in a class pattern with positional arguments, each >>> ' 1 2 3 '.split() ['1', '2', '3'] + If *sep* is not specified or is "None" and *maxsplit* is "0", only + leading runs of consecutive whitespace are considered. + + For example: + + >>> "".split(None, 0) + [] + >>> " ".split(None, 0) + [] + >>> " foo ".split(maxsplit=0) + ['foo '] + str.splitlines(keepends=False) Return a list of the lines in the string, breaking at line @@ -9737,7 +10611,7 @@ class is used in a class pattern with positional arguments, each With optional *start*, test string beginning at that position. With optional *end*, stop comparing string at that position. -str.strip([chars]) +str.strip(chars=None, /) Return a copy of the string with the leading and trailing characters removed. The *chars* argument is a string specifying the @@ -9801,7 +10675,9 @@ class is used in a class pattern with positional arguments, each >>> titlecase("they're bill's friends.") "They're Bill's Friends." -str.translate(table) + See also "istitle()". + +str.translate(table, /) Return a copy of the string in which each character has been mapped through the given translation table. The table must be an object @@ -9826,10 +10702,10 @@ class is used in a class pattern with positional arguments, each category of the resulting character(s) is not “Lu” (Letter, uppercase), but e.g. “Lt” (Letter, titlecase). - The uppercasing algorithm used is described in section 3.13 - ‘Default Case Folding’ of the Unicode Standard. + The uppercasing algorithm used is described in section 3.13.2 + ‘Default Case Conversion’ of the Unicode Standard. -str.zfill(width) +str.zfill(width, /) Return a copy of the string left filled with ASCII "'0'" digits to make a string of length *width*. A leading sign prefix @@ -9847,174 +10723,313 @@ class is used in a class pattern with positional arguments, each 'strings': '''String and Bytes literals ************************* -String literals are described by the following lexical definitions: +String literals are text enclosed in single quotes ("'") or double +quotes ("""). For example: - stringliteral: [stringprefix](shortstring | longstring) - stringprefix: "r" | "u" | "R" | "U" | "f" | "F" - | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" - shortstring: "'" shortstringitem* "'" | '"' shortstringitem* '"' - longstring: "\'\'\'" longstringitem* "\'\'\'" | '"""' longstringitem* '"""' - shortstringitem: shortstringchar | stringescapeseq - longstringitem: longstringchar | stringescapeseq - shortstringchar: - longstringchar: - stringescapeseq: "\\" + "spam" + 'eggs' - bytesliteral: bytesprefix(shortbytes | longbytes) - bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" - shortbytes: "'" shortbytesitem* "'" | '"' shortbytesitem* '"' - longbytes: "\'\'\'" longbytesitem* "\'\'\'" | '"""' longbytesitem* '"""' - shortbytesitem: shortbyteschar | bytesescapeseq - longbytesitem: longbyteschar | bytesescapeseq - shortbyteschar: - longbyteschar: - bytesescapeseq: "\\" +The quote used to start the literal also terminates it, so a string +literal can only contain the other quote (except with escape +sequences, see below). For example: -One syntactic restriction not indicated by these productions is that -whitespace is not allowed between the "stringprefix" or "bytesprefix" -and the rest of the literal. The source character set is defined by -the encoding declaration; it is UTF-8 if no encoding declaration is -given in the source file; see section Encoding declarations. + 'Say "Hello", please.' + "Don't do that!" -In plain English: Both types of literals can be enclosed in matching -single quotes ("'") or double quotes ("""). They can also be enclosed -in matching groups of three single or double quotes (these are -generally referred to as *triple-quoted strings*). The backslash ("\\") -character is used to give special meaning to otherwise ordinary -characters like "n", which means ‘newline’ when escaped ("\\n"). It can -also be used to escape characters that otherwise have a special -meaning, such as newline, backslash itself, or the quote character. -See escape sequences below for examples. +Except for this limitation, the choice of quote character ("'" or """) +does not affect how the literal is parsed. -Bytes literals are always prefixed with "'b'" or "'B'"; they produce -an instance of the "bytes" type instead of the "str" type. They may -only contain ASCII characters; bytes with a numeric value of 128 or -greater must be expressed with escapes. +Inside a string literal, the backslash ("\\") character introduces an +*escape sequence*, which has special meaning depending on the +character after the backslash. For example, "\\"" denotes the double +quote character, and does *not* end the string: -Both string and bytes literals may optionally be prefixed with a -letter "'r'" or "'R'"; such constructs are called *raw string -literals* and *raw bytes literals* respectively and treat backslashes -as literal characters. As a result, in raw string literals, "'\\U'" -and "'\\u'" escapes are not treated specially. + >>> print("Say \\"Hello\\" to everyone!") + Say "Hello" to everyone! + +See escape sequences below for a full list of such sequences, and more +details. + + +Triple-quoted strings +===================== + +Strings can also be enclosed in matching groups of three single or +double quotes. These are generally referred to as *triple-quoted +strings*: + + """This is a triple-quoted string.""" + +In triple-quoted literals, unescaped quotes are allowed (and are +retained), except that three unescaped quotes in a row terminate the +literal, if they are of the same kind ("'" or """) used at the start: + + """This string has "quotes" inside.""" + +Unescaped newlines are also allowed and retained: + + \'\'\'This triple-quoted string + continues on the next line.\'\'\' + + +String prefixes +=============== + +String literals can have an optional *prefix* that influences how the +content of the literal is parsed, for example: + + b"data" + f'{result=}' + +The allowed prefixes are: + +* "b": Bytes literal + +* "r": Raw string + +* "f": Formatted string literal (“f-string”) + +* "t": Template string literal (“t-string”) + +* "u": No effect (allowed for backwards compatibility) + +See the linked sections for details on each type. + +Prefixes are case-insensitive (for example, ‘"B"’ works the same as +‘"b"’). The ‘"r"’ prefix can be combined with ‘"f"’, ‘"t"’ or ‘"b"’, +so ‘"fr"’, ‘"rf"’, ‘"tr"’, ‘"rt"’, ‘"br"’, and ‘"rb"’ are also valid +prefixes. Added in version 3.3: The "'rb'" prefix of raw bytes literals has been added as a synonym of "'br'".Support for the unicode legacy literal ("u'value'") was reintroduced to simplify the maintenance of dual Python 2.x and 3.x codebases. See **PEP 414** for more information. -A string literal with "'f'" or "'F'" in its prefix is a *formatted -string literal*; see f-strings. The "'f'" may be combined with "'r'", -but not with "'b'" or "'u'", therefore raw formatted strings are -possible, but formatted bytes literals are not. -In triple-quoted literals, unescaped newlines and quotes are allowed -(and are retained), except that three unescaped quotes in a row -terminate the literal. (A “quote” is the character used to open the -literal, i.e. either "'" or """.) +Formal grammar +============== + +String literals, except “f-strings” and “t-strings”, are described by +the following lexical definitions. + +These definitions use negative lookaheads ("!") to indicate that an +ending quote ends the literal. + + STRING: [stringprefix] (stringcontent) + stringprefix: <("r" | "u" | "b" | "br" | "rb"), case-insensitive> + stringcontent: + | "\'\'\'" ( !"\'\'\'" longstringitem)* "\'\'\'" + | '"""' ( !'"""' longstringitem)* '"""' + | "'" ( !"'" stringitem)* "'" + | '"' ( !'"' stringitem)* '"' + stringitem: stringchar | stringescapeseq + stringchar: + longstringitem: stringitem | newline + stringescapeseq: "\\" + +Note that as in all lexical definitions, whitespace is significant. In +particular, the prefix (if any) must be immediately followed by the +starting quote. Escape sequences ================ -Unless an "'r'" or "'R'" prefix is present, escape sequences in string +Unless an ‘"r"’ or ‘"R"’ prefix is present, escape sequences in string and bytes literals are interpreted according to rules similar to those used by Standard C. The recognized escape sequences are: -+---------------------------+-----------------------------------+---------+ -| Escape Sequence | Meaning | Notes | -|===========================|===================================|=========| -| "\\" | Backslash and newline ignored | (1) | -+---------------------------+-----------------------------------+---------+ -| "\\\\" | Backslash ("\\") | | -+---------------------------+-----------------------------------+---------+ -| "\\'" | Single quote ("'") | | -+---------------------------+-----------------------------------+---------+ -| "\\"" | Double quote (""") | | -+---------------------------+-----------------------------------+---------+ -| "\\a" | ASCII Bell (BEL) | | -+---------------------------+-----------------------------------+---------+ -| "\\b" | ASCII Backspace (BS) | | -+---------------------------+-----------------------------------+---------+ -| "\\f" | ASCII Formfeed (FF) | | -+---------------------------+-----------------------------------+---------+ -| "\\n" | ASCII Linefeed (LF) | | -+---------------------------+-----------------------------------+---------+ -| "\\r" | ASCII Carriage Return (CR) | | -+---------------------------+-----------------------------------+---------+ -| "\\t" | ASCII Horizontal Tab (TAB) | | -+---------------------------+-----------------------------------+---------+ -| "\\v" | ASCII Vertical Tab (VT) | | -+---------------------------+-----------------------------------+---------+ -| "\\*ooo*" | Character with octal value *ooo* | (2,4) | -+---------------------------+-----------------------------------+---------+ -| "\\x*hh*" | Character with hex value *hh* | (3,4) | -+---------------------------+-----------------------------------+---------+ ++----------------------------------------------------+----------------------------------------------------+ +| Escape Sequence | Meaning | +|====================================================|====================================================| +| "\\" | Ignored end of line | ++----------------------------------------------------+----------------------------------------------------+ +| "\\\\" | Backslash | ++----------------------------------------------------+----------------------------------------------------+ +| "\\'" | Single quote | ++----------------------------------------------------+----------------------------------------------------+ +| "\\"" | Double quote | ++----------------------------------------------------+----------------------------------------------------+ +| "\\a" | ASCII Bell (BEL) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\b" | ASCII Backspace (BS) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\f" | ASCII Formfeed (FF) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\n" | ASCII Linefeed (LF) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\r" | ASCII Carriage Return (CR) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\t" | ASCII Horizontal Tab (TAB) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\v" | ASCII Vertical Tab (VT) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\*ooo*" | Octal character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\x*hh*" | Hexadecimal character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\N{*name*}" | Named Unicode character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\u*xxxx*" | Hexadecimal Unicode character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\U*xxxxxxxx*" | Hexadecimal Unicode character | ++----------------------------------------------------+----------------------------------------------------+ -Escape sequences only recognized in string literals are: -+---------------------------+-----------------------------------+---------+ -| Escape Sequence | Meaning | Notes | -|===========================|===================================|=========| -| "\\N{*name*}" | Character named *name* in the | (5) | -| | Unicode database | | -+---------------------------+-----------------------------------+---------+ -| "\\u*xxxx*" | Character with 16-bit hex value | (6) | -| | *xxxx* | | -+---------------------------+-----------------------------------+---------+ -| "\\U*xxxxxxxx*" | Character with 32-bit hex value | (7) | -| | *xxxxxxxx* | | -+---------------------------+-----------------------------------+---------+ +Ignored end of line +------------------- -Notes: +A backslash can be added at the end of a line to ignore the newline: -1. A backslash can be added at the end of a line to ignore the - newline: + >>> 'This string will not include \\ + ... backslashes or newline characters.' + 'This string will not include backslashes or newline characters.' - >>> 'This string will not include \\ - ... backslashes or newline characters.' - 'This string will not include backslashes or newline characters.' +The same result can be achieved using triple-quoted strings, or +parentheses and string literal concatenation. - The same result can be achieved using triple-quoted strings, or - parentheses and string literal concatenation. -2. As in Standard C, up to three octal digits are accepted. +Escaped characters +------------------ - Changed in version 3.11: Octal escapes with value larger than - "0o377" produce a "DeprecationWarning". +To include a backslash in a non-raw Python string literal, it must be +doubled. The "\\\\" escape sequence denotes a single backslash +character: - Changed in version 3.12: Octal escapes with value larger than - "0o377" produce a "SyntaxWarning". In a future Python version they - will be eventually a "SyntaxError". + >>> print('C:\\\\Program Files') + C:\\Program Files -3. Unlike in Standard C, exactly two hex digits are required. +Similarly, the "\\'" and "\\"" sequences denote the single and double +quote character, respectively: -4. In a bytes literal, hexadecimal and octal escapes denote the byte - with the given value. In a string literal, these escapes denote a - Unicode character with the given value. + >>> print('\\' and \\"') + ' and " -5. Changed in version 3.3: Support for name aliases [1] has been - added. -6. Exactly four hex digits are required. +Octal character +--------------- -7. Any Unicode character can be encoded this way. Exactly eight hex - digits are required. +The sequence "\\*ooo*" denotes a *character* with the octal (base 8) +value *ooo*: -Unlike Standard C, all unrecognized escape sequences are left in the -string unchanged, i.e., *the backslash is left in the result*. (This -behavior is useful when debugging: if an escape sequence is mistyped, -the resulting output is more easily recognized as broken.) It is also -important to note that the escape sequences only recognized in string -literals fall into the category of unrecognized escapes for bytes -literals. + >>> '\\120' + 'P' + +Up to three octal digits (0 through 7) are accepted. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given +value. + +Changed in version 3.11: Octal escapes with value larger than "0o377" +(255) produce a "DeprecationWarning". + +Changed in version 3.12: Octal escapes with value larger than "0o377" +(255) produce a "SyntaxWarning". In a future Python version they will +raise a "SyntaxError". + + +Hexadecimal character +--------------------- + +The sequence "\\x*hh*" denotes a *character* with the hex (base 16) +value *hh*: + + >>> '\\x50' + 'P' + +Unlike in Standard C, exactly two hex digits are required. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given +value. + + +Named Unicode character +----------------------- + +The sequence "\\N{*name*}" denotes a Unicode character with the given +*name*: + + >>> '\\N{LATIN CAPITAL LETTER P}' + 'P' + >>> '\\N{SNAKE}' + '🐍' + +This sequence cannot appear in bytes literals. + +Changed in version 3.3: Support for name aliases has been added. + + +Hexadecimal Unicode characters +------------------------------ + +These sequences "\\u*xxxx*" and "\\U*xxxxxxxx*" denote the Unicode +character with the given hex (base 16) value. Exactly four digits are +required for "\\u"; exactly eight digits are required for "\\U". The +latter can encode any Unicode character. + + >>> '\\u1234' + 'ሴ' + >>> '\\U0001f40d' + '🐍' + +These sequences cannot appear in bytes literals. + + +Unrecognized escape sequences +----------------------------- + +Unlike in Standard C, all unrecognized escape sequences are left in +the string unchanged, that is, *the backslash is left in the result*: + + >>> print('\\q') + \\q + >>> list('\\q') + ['\\\\', 'q'] + +Note that for bytes literals, the escape sequences only recognized in +string literals ("\\N...", "\\u...", "\\U...") fall into the category of +unrecognized escapes. Changed in version 3.6: Unrecognized escape sequences produce a "DeprecationWarning". Changed in version 3.12: Unrecognized escape sequences produce a -"SyntaxWarning". In a future Python version they will be eventually a +"SyntaxWarning". In a future Python version they will raise a "SyntaxError". + +Bytes literals +============== + +*Bytes literals* are always prefixed with ‘"b"’ or ‘"B"’; they produce +an instance of the "bytes" type instead of the "str" type. They may +only contain ASCII characters; bytes with a numeric value of 128 or +greater must be expressed with escape sequences (typically Hexadecimal +character or Octal character): + + >>> b'\\x89PNG\\r\\n\\x1a\\n' + b'\\x89PNG\\r\\n\\x1a\\n' + >>> list(b'\\x89PNG\\r\\n\\x1a\\n') + [137, 80, 78, 71, 13, 10, 26, 10] + +Similarly, a zero byte must be expressed using an escape sequence +(typically "\\0" or "\\x00"). + + +Raw string literals +=================== + +Both string and bytes literals may optionally be prefixed with a +letter ‘"r"’ or ‘"R"’; such constructs are called *raw string +literals* and *raw bytes literals* respectively and treat backslashes +as literal characters. As a result, in raw string literals, escape +sequences are not treated specially: + + >>> r'\\d{4}-\\d{2}-\\d{2}' + '\\\\d{4}-\\\\d{2}-\\\\d{2}' + Even in a raw literal, quotes can be escaped with a backslash, but the backslash remains in the result; for example, "r"\\""" is a valid string literal consisting of two characters: a backslash and a double @@ -10024,6 +11039,199 @@ class is used in a class pattern with positional arguments, each the following quote character). Note also that a single backslash followed by a newline is interpreted as those two characters as part of the literal, *not* as a line continuation. + + +f-strings +========= + +Added in version 3.6. + +A *formatted string literal* or *f-string* is a string literal that is +prefixed with ‘"f"’ or ‘"F"’. These strings may contain replacement +fields, which are expressions delimited by curly braces "{}". While +other string literals always have a constant value, formatted strings +are really expressions evaluated at run time. + +Escape sequences are decoded like in ordinary string literals (except +when a literal is also marked as a raw string). After decoding, the +grammar for the contents of the string is: + + f_string: (literal_char | "{{" | "}}" | replacement_field)* + replacement_field: "{" f_expression ["="] ["!" conversion] [":" format_spec] "}" + f_expression: (conditional_expression | "*" or_expr) + ("," conditional_expression | "," "*" or_expr)* [","] + | yield_expression + conversion: "s" | "r" | "a" + format_spec: (literal_char | replacement_field)* + literal_char: + +The parts of the string outside curly braces are treated literally, +except that any doubled curly braces "'{{'" or "'}}'" are replaced +with the corresponding single curly brace. A single opening curly +bracket "'{'" marks a replacement field, which starts with a Python +expression. To display both the expression text and its value after +evaluation, (useful in debugging), an equal sign "'='" may be added +after the expression. A conversion field, introduced by an exclamation +point "'!'" may follow. A format specifier may also be appended, +introduced by a colon "':'". A replacement field ends with a closing +curly bracket "'}'". + +Expressions in formatted string literals are treated like regular +Python expressions surrounded by parentheses, with a few exceptions. +An empty expression is not allowed, and both "lambda" and assignment +expressions ":=" must be surrounded by explicit parentheses. Each +expression is evaluated in the context where the formatted string +literal appears, in order from left to right. Replacement expressions +can contain newlines in both single-quoted and triple-quoted f-strings +and they can contain comments. Everything that comes after a "#" +inside a replacement field is a comment (even closing braces and +quotes). In that case, replacement fields must be closed in a +different line. + + >>> f"abc{a # This is a comment }" + ... + 3}" + 'abc5' + +Changed in version 3.7: Prior to Python 3.7, an "await" expression and +comprehensions containing an "async for" clause were illegal in the +expressions in formatted string literals due to a problem with the +implementation. + +Changed in version 3.12: Prior to Python 3.12, comments were not +allowed inside f-string replacement fields. + +When the equal sign "'='" is provided, the output will have the +expression text, the "'='" and the evaluated value. Spaces after the +opening brace "'{'", within the expression and after the "'='" are all +retained in the output. By default, the "'='" causes the "repr()" of +the expression to be provided, unless there is a format specified. +When a format is specified it defaults to the "str()" of the +expression unless a conversion "'!r'" is declared. + +Added in version 3.8: The equal sign "'='". + +If a conversion is specified, the result of evaluating the expression +is converted before formatting. Conversion "'!s'" calls "str()" on +the result, "'!r'" calls "repr()", and "'!a'" calls "ascii()". + +The result is then formatted using the "format()" protocol. The +format specifier is passed to the "__format__()" method of the +expression or conversion result. An empty string is passed when the +format specifier is omitted. The formatted result is then included in +the final value of the whole string. + +Top-level format specifiers may include nested replacement fields. +These nested fields may include their own conversion fields and format +specifiers, but may not include more deeply nested replacement fields. +The format specifier mini-language is the same as that used by the +"str.format()" method. + +Formatted string literals may be concatenated, but replacement fields +cannot be split across literals. + +Some examples of formatted string literals: + + >>> name = "Fred" + >>> f"He said his name is {name!r}." + "He said his name is 'Fred'." + >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r + "He said his name is 'Fred'." + >>> width = 10 + >>> precision = 4 + >>> value = decimal.Decimal("12.34567") + >>> f"result: {value:{width}.{precision}}" # nested fields + 'result: 12.35' + >>> today = datetime(year=2017, month=1, day=27) + >>> f"{today:%B %d, %Y}" # using date format specifier + 'January 27, 2017' + >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging + 'today=January 27, 2017' + >>> number = 1024 + >>> f"{number:#0x}" # using integer format specifier + '0x400' + >>> foo = "bar" + >>> f"{ foo = }" # preserves whitespace + " foo = 'bar'" + >>> line = "The mill's closed" + >>> f"{line = }" + 'line = "The mill\\'s closed"' + >>> f"{line = :20}" + "line = The mill's closed " + >>> f"{line = !r:20}" + 'line = "The mill\\'s closed" ' + +Reusing the outer f-string quoting type inside a replacement field is +permitted: + + >>> a = dict(x=2) + >>> f"abc {a["x"]} def" + 'abc 2 def' + +Changed in version 3.12: Prior to Python 3.12, reuse of the same +quoting type of the outer f-string inside a replacement field was not +possible. + +Backslashes are also allowed in replacement fields and are evaluated +the same way as in any other context: + + >>> a = ["a", "b", "c"] + >>> print(f"List a contains:\\n{"\\n".join(a)}") + List a contains: + a + b + c + +Changed in version 3.12: Prior to Python 3.12, backslashes were not +permitted inside an f-string replacement field. + +Formatted string literals cannot be used as docstrings, even if they +do not include expressions. + + >>> def foo(): + ... f"Not a docstring" + ... + >>> foo.__doc__ is None + True + +See also **PEP 498** for the proposal that added formatted string +literals, and "str.format()", which uses a related format string +mechanism. + + +t-strings +========= + +Added in version 3.14. + +A *template string literal* or *t-string* is a string literal that is +prefixed with ‘"t"’ or ‘"T"’. These strings follow the same syntax and +evaluation rules as formatted string literals, with the following +differences: + +* Rather than evaluating to a "str" object, template string literals + evaluate to a "string.templatelib.Template" object. + +* The "format()" protocol is not used. Instead, the format specifier + and conversions (if any) are passed to a new "Interpolation" object + that is created for each evaluated expression. It is up to code that + processes the resulting "Template" object to decide how to handle + format specifiers and conversions. + +* Format specifiers containing nested replacement fields are evaluated + eagerly, prior to being passed to the "Interpolation" object. For + instance, an interpolation of the form "{amount:.{precision}f}" will + evaluate the inner expression "{precision}" to determine the value + of the "format_spec" attribute. If "precision" were to be "2", the + resulting format specifier would be "'.2f'". + +* When the equals sign "'='" is provided in an interpolation + expression, the text of the expression is appended to the literal + string that precedes the relevant interpolation. This includes the + equals sign and any surrounding whitespace. The "Interpolation" + instance for the expression will be created as normal, except that + "conversion" will be set to ‘"r"’ ("repr()") by default. If an + explicit conversion or format specifier are provided, this will + override the default behaviour. ''', 'subscriptions': r'''Subscriptions ************* @@ -10218,15 +11426,29 @@ class is used in a class pattern with positional arguments, each "except*" clause ================ -The "except*" clause(s) are used for handling "ExceptionGroup"s. The -exception type for matching is interpreted as in the case of "except", -but in the case of exception groups we can have partial matches when -the type matches some of the exceptions in the group. This means that -multiple "except*" clauses can execute, each handling part of the -exception group. Each clause executes at most once and handles an -exception group of all matching exceptions. Each exception in the -group is handled by at most one "except*" clause, the first that -matches it. +The "except*" clause(s) specify one or more handlers for groups of +exceptions ("BaseExceptionGroup" instances). A "try" statement can +have either "except" or "except*" clauses, but not both. The exception +type for matching is mandatory in the case of "except*", so "except*:" +is a syntax error. The type is interpreted as in the case of "except", +but matching is performed on the exceptions contained in the group +that is being handled. An "TypeError" is raised if a matching type is +a subclass of "BaseExceptionGroup", because that would have ambiguous +semantics. + +When an exception group is raised in the try block, each "except*" +clause splits (see "split()") it into the subgroups of matching and +non-matching exceptions. If the matching subgroup is not empty, it +becomes the handled exception (the value returned from +"sys.exception()") and assigned to the target of the "except*" clause +(if there is one). Then, the body of the "except*" clause executes. If +the non-matching subgroup is not empty, it is processed by the next +"except*" in the same manner. This continues until all exceptions in +the group have been matched, or the last "except*" clause has run. + +After all "except*" clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from +within "except*" clauses. This merged exception group propagates on.: >>> try: ... raise ExceptionGroup("eg", @@ -10239,20 +11461,19 @@ class is used in a class pattern with positional arguments, each caught with nested (TypeError(2),) caught with nested (OSError(3), OSError(4)) + Exception Group Traceback (most recent call last): - | File "", line 2, in - | ExceptionGroup: eg + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: 1 +------------------------------------ -Any remaining exceptions that were not handled by any "except*" clause -are re-raised at the end, along with all exceptions that were raised -from within the "except*" clauses. If this list contains more than one -exception to reraise, they are combined into an exception group. - -If the raised exception is not an exception group and its type matches -one of the "except*" clauses, it is caught and wrapped by an exception -group with an empty message string. +If the exception raised from the "try" block is not an exception group +and its type matches one of the "except*" clauses, it is caught and +wrapped by an exception group with an empty message string. This +ensures that the type of the target "e" is consistently +"BaseExceptionGroup": >>> try: ... raise BlockingIOError @@ -10261,11 +11482,6 @@ class is used in a class pattern with positional arguments, each ... ExceptionGroup('', (BlockingIOError())) -An "except*" clause must have a matching expression; it cannot be -"except*:". Furthermore, this expression cannot contain exception -group types, because that would have ambiguous semantics. - -It is not possible to mix "except" and "except*" in the same "try". "break", "continue" and "return" cannot appear in an "except*" clause. @@ -10282,11 +11498,11 @@ class is used in a class pattern with positional arguments, each ================ If "finally" is present, it specifies a ‘cleanup’ handler. The "try" -clause is executed, including any "except" and "else" clauses. If an +clause is executed, including any "except" and "else" clauses. If an exception occurs in any of the clauses and is not handled, the exception is temporarily saved. The "finally" clause is executed. If there is a saved exception it is re-raised at the end of the "finally" -clause. If the "finally" clause raises another exception, the saved +clause. If the "finally" clause raises another exception, the saved exception is set as the context of the new exception. If the "finally" clause executes a "return", "break" or "continue" statement, the saved exception is discarded. For example, this function returns 42. @@ -10574,7 +11790,7 @@ def foo(): Sets These represent a mutable set. They are created by the built-in "set()" constructor and can be modified afterwards by several - methods, such as "add()". + methods, such as "add". Frozen sets These represent an immutable set. They are created by the built-in @@ -10979,7 +12195,7 @@ def foo(): "ImportWarning" when falling back to "__package__" during import resolution. - Deprecated since version 3.13, will be removed in version 3.15: + Deprecated since version 3.13, removed in version 3.15: "__package__" will cease to be set or taken into consideration by the import system or standard library. @@ -11053,11 +12269,10 @@ def foo(): It is **strongly** recommended that you use "module.__spec__.cached" instead of "module.__cached__". - Deprecated since version 3.13, will be removed in version 3.15: - Setting "__cached__" on a module while failing to set - "__spec__.cached" is deprecated. In Python 3.15, "__cached__" will - cease to be set or taken into consideration by the import system or - standard library. + Deprecated since version 3.13, removed in version 3.15: Setting + "__cached__" on a module while failing to set "__spec__.cached" is + deprecated. In Python 3.15, "__cached__" will cease to be set or + taken into consideration by the import system or standard library. Other writable attributes on module objects @@ -11161,6 +12376,11 @@ class method object, it is transformed into an instance method object | | "X.__bases__" will be exactly equal to "(A, B, | | | C)". | +----------------------------------------------------+----------------------------------------------------+ +| type.__base__ | **CPython implementation detail:** The single base | +| | class in the inheritance chain that is responsible | +| | for the memory layout of instances. This attribute | +| | corresponds to "tp_base" at the C level. | ++----------------------------------------------------+----------------------------------------------------+ | type.__doc__ | The class’s documentation string, or "None" if | | | undefined. Not inherited by subclasses. | +----------------------------------------------------+----------------------------------------------------+ @@ -11168,11 +12388,20 @@ class method object, it is transformed into an instance method object | | collected during class body execution. See also: | | | "__annotations__ attributes". For best practices | | | on working with "__annotations__", please see | -| | "annotationlib". Where possible, use | +| | "annotationlib". Use | | | "annotationlib.get_annotations()" instead of | -| | accessing this attribute directly. Changed in | -| | version 3.14: Annotations are now lazily | -| | evaluated. See **PEP 649**. | +| | accessing this attribute directly. Warning: | +| | Accessing the "__annotations__" attribute directly | +| | on a class object may return annotations for the | +| | wrong class, specifically in certain cases where | +| | the class, its base class, or a metaclass is | +| | defined under "from __future__ import | +| | annotations". See **749** for details.This | +| | attribute does not exist on certain builtin | +| | classes. On user-defined classes without | +| | "__annotations__", it is an empty dictionary. | +| | Changed in version 3.14: Annotations are now | +| | lazily evaluated. See **PEP 649**. | +----------------------------------------------------+----------------------------------------------------+ | type.__annotate__() | The *annotate function* for this class, or "None" | | | if the class has no annotations. See also: | @@ -11510,6 +12739,10 @@ class instance has a namespace implemented as a dictionary which is | | (this is an index into the *bytecode* string of | | | the code object) | +----------------------------------------------------+----------------------------------------------------+ +| frame.f_generator | The *generator* or *coroutine* object that owns | +| | this frame, or "None" if the frame is a normal | +| | function. Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ Special writable attributes @@ -11690,8 +12923,8 @@ class instance has a namespace implemented as a dictionary which is dictionary entry. class dict(**kwargs) -class dict(mapping, **kwargs) -class dict(iterable, **kwargs) +class dict(mapping, /, **kwargs) +class dict(iterable, /, **kwargs) Return a new dictionary initialized from an optional positional argument and a possibly empty set of keyword arguments. @@ -11724,8 +12957,15 @@ class dict(iterable, **kwargs) the keyword argument replaces the value from the positional argument. - To illustrate, the following examples all return a dictionary equal - to "{"one": 1, "two": 2, "three": 3}": + Providing keyword arguments as in the first example only works for + keys that are valid Python identifiers. Otherwise, any valid keys + can be used. + + Dictionaries compare equal if and only if they have the same "(key, + value)" pairs (regardless of ordering). Order comparisons (‘<’, + ‘<=’, ‘>=’, ‘>’) raise "TypeError". To illustrate dictionary + creation and equality, the following examples all return a + dictionary equal to "{"one": 1, "two": 2, "three": 3}": >>> a = dict(one=1, two=2, three=3) >>> b = {'one': 1, 'two': 2, 'three': 3} @@ -11740,6 +12980,29 @@ class dict(iterable, **kwargs) keys that are valid Python identifiers. Otherwise, any valid keys can be used. + Dictionaries preserve insertion order. Note that updating a key + does not affect the order. Keys added after deletion are inserted + at the end. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. This behavior was an implementation detail of + CPython from 3.6. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -11777,8 +13040,8 @@ class dict(iterable, **kwargs) 1 The example above shows part of the implementation of - "collections.Counter". A different "__missing__" method is used - by "collections.defaultdict". + "collections.Counter". A different "__missing__()" method is + used by "collections.defaultdict". d[key] = value @@ -11837,7 +13100,8 @@ class dict(iterable, **kwargs) Return a new view of the dictionary’s keys. See the documentation of view objects. - pop(key[, default]) + pop(key, /) + pop(key, default, /) If *key* is in the dictionary, remove it and return its value, else return *default*. If *default* is not given and *key* is @@ -11868,10 +13132,13 @@ class dict(iterable, **kwargs) *key* with a value of *default* and return *default*. *default* defaults to "None". - update([other]) + update(**kwargs) + update(mapping, /, **kwargs) + update(iterable, /, **kwargs) - Update the dictionary with the key/value pairs from *other*, - overwriting existing keys. Return "None". + Update the dictionary with the key/value pairs from *mapping* or + *iterable* and *kwargs*, overwriting existing keys. Return + "None". "update()" accepts either another object with a "keys()" method (in which case "__getitem__()" is called with every key returned @@ -11910,33 +13177,6 @@ class dict(iterable, **kwargs) Added in version 3.9. - Dictionaries compare equal if and only if they have the same "(key, - value)" pairs (regardless of ordering). Order comparisons (‘<’, - ‘<=’, ‘>=’, ‘>’) raise "TypeError". - - Dictionaries preserve insertion order. Note that updating a key - does not affect the order. Keys added after deletion are inserted - at the end. - - >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} - >>> d - {'one': 1, 'two': 2, 'three': 3, 'four': 4} - >>> list(d) - ['one', 'two', 'three', 'four'] - >>> list(d.values()) - [1, 2, 3, 4] - >>> d["one"] = 42 - >>> d - {'one': 42, 'two': 2, 'three': 3, 'four': 4} - >>> del d["two"] - >>> d["two"] = None - >>> d - {'one': 42, 'three': 3, 'four': 4, 'two': None} - - Changed in version 3.7: Dictionary order is guaranteed to be - insertion order. This behavior was an implementation detail of - CPython from 3.6. - Dictionaries and dictionary views are reversible. >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} @@ -12163,7 +13403,7 @@ class dict(iterable, **kwargs) | "s * n" or "n * s" | equivalent to adding *s* to | (2)(7) | | | itself *n* times | | +----------------------------+----------------------------------+------------+ -| "s[i]" | *i*th item of *s*, origin 0 | (3) | +| "s[i]" | *i*th item of *s*, origin 0 | (3)(8) | +----------------------------+----------------------------------+------------+ | "s[i:j]" | slice of *s* from *i* to *j* | (3)(4) | +----------------------------+----------------------------------+------------+ @@ -12176,13 +13416,6 @@ class dict(iterable, **kwargs) +----------------------------+----------------------------------+------------+ | "max(s)" | largest item of *s* | | +----------------------------+----------------------------------+------------+ -| "s.index(x[, i[, j]])" | index of the first occurrence of | (8) | -| | *x* in *s* (at or after index | | -| | *i* and before index *j*) | | -+----------------------------+----------------------------------+------------+ -| "s.count(x)" | total number of occurrences of | | -| | *x* in *s* | | -+----------------------------+----------------------------------+------------+ Sequences of the same type also support comparisons. In particular, tuples and lists are compared lexicographically by comparing @@ -12279,13 +13512,31 @@ class dict(iterable, **kwargs) that follow specific patterns, and hence don’t support sequence concatenation or repetition. -8. "index" raises "ValueError" when *x* is not found in *s*. Not all - implementations support passing the additional arguments *i* and - *j*. These arguments allow efficient searching of subsections of - the sequence. Passing the extra arguments is roughly equivalent to - using "s[i:j].index(x)", only without copying any data and with the - returned index being relative to the start of the sequence rather - than the start of the slice. +8. An "IndexError" is raised if *i* is outside the sequence range. + +-[ Sequence Methods ]- + +Sequence types also support the following methods: + +sequence.count(value, /) + + Return the total number of occurrences of *value* in *sequence*. + +sequence.index(value[, start[, stop]) + + Return the index of the first occurrence of *value* in *sequence*. + + Raises "ValueError" if *value* is not found in *sequence*. + + The *start* or *stop* arguments allow for efficient searching of + subsections of the sequence, beginning at *start* and ending at + *stop*. This is roughly equivalent to "start + + sequence[start:stop].index(value)", only without copying any data. + + Caution: + + Not all sequence types support passing the *start* and *stop* + arguments. Immutable Sequence Types @@ -12321,11 +13572,15 @@ class dict(iterable, **kwargs) | "s[i] = x" | item *i* of *s* is replaced by | | | | *x* | | +--------------------------------+----------------------------------+-----------------------+ +| "del s[i]" | removes item *i* of *s* | | ++--------------------------------+----------------------------------+-----------------------+ | "s[i:j] = t" | slice of *s* from *i* to *j* is | | | | replaced by the contents of the | | | | iterable *t* | | +--------------------------------+----------------------------------+-----------------------+ -| "del s[i:j]" | same as "s[i:j] = []" | | +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | +--------------------------------+----------------------------------+-----------------------+ | "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | | | replaced by those of *t* | | @@ -12333,64 +13588,80 @@ class dict(iterable, **kwargs) | "del s[i:j:k]" | removes the elements of | | | | "s[i:j:k]" from the list | | +--------------------------------+----------------------------------+-----------------------+ -| "s.append(x)" | appends *x* to the end of the | | -| | sequence (same as | | -| | "s[len(s):len(s)] = [x]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.clear()" | removes all items from *s* (same | (5) | -| | as "del s[:]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.copy()" | creates a shallow copy of *s* | (5) | -| | (same as "s[:]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.extend(t)" or "s += t" | extends *s* with the contents of | | +| "s += t" | extends *s* with the contents of | | | | *t* (for the most part the same | | | | as "s[len(s):len(s)] = t") | | +--------------------------------+----------------------------------+-----------------------+ -| "s *= n" | updates *s* with its contents | (6) | +| "s *= n" | updates *s* with its contents | (2) | | | repeated *n* times | | +--------------------------------+----------------------------------+-----------------------+ -| "s.insert(i, x)" | inserts *x* into *s* at the | | -| | index given by *i* (same as | | -| | "s[i:i] = [x]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.pop()" or "s.pop(i)" | retrieves the item at *i* and | (2) | -| | also removes it from *s* | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.remove(x)" | removes the first item from *s* | (3) | -| | where "s[i]" is equal to *x* | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.reverse()" | reverses the items of *s* in | (4) | -| | place | | -+--------------------------------+----------------------------------+-----------------------+ Notes: 1. If *k* is not equal to "1", *t* must have the same length as the slice it is replacing. -2. The optional argument *i* defaults to "-1", so that by default the - last item is removed and returned. - -3. "remove()" raises "ValueError" when *x* is not found in *s*. - -4. The "reverse()" method modifies the sequence in place for economy - of space when reversing a large sequence. To remind users that it - operates by side effect, it does not return the reversed sequence. - -5. "clear()" and "copy()" are included for consistency with the - interfaces of mutable containers that don’t support slicing - operations (such as "dict" and "set"). "copy()" is not part of the - "collections.abc.MutableSequence" ABC, but most concrete mutable - sequence classes provide it. - - Added in version 3.3: "clear()" and "copy()" methods. - -6. The value *n* is an integer, or an object implementing +2. The value *n* is an integer, or an object implementing "__index__()". Zero and negative values of *n* clear the sequence. Items in the sequence are not copied; they are referenced multiple times, as explained for "s * n" under Common Sequence Operations. +-[ Mutable Sequence Methods ]- + +Mutable sequence types also support the following methods: + +sequence.append(value, /) + + Append *value* to the end of the sequence This is equivalent to + writing "seq[len(seq):len(seq)] = [value]". + +sequence.clear() + + Added in version 3.3. + + Remove all items from *sequence*. This is equivalent to writing + "del sequence[:]". + +sequence.copy() + + Added in version 3.3. + + Create a shallow copy of *sequence*. This is equivalent to writing + "sequence[:]". + + Hint: + + The "copy()" method is not part of the "MutableSequence" "ABC", + but most concrete mutable sequence types provide it. + +sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. For the most + part, this is the same as writing "seq[len(seq):len(seq)] = + iterable". + +sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. This is + equivalent to writing "sequence[index:index] = [value]". + +sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +sequence.remove(value, /) + + Remove the first item from *sequence* where "sequence[i] == value". + + Raises "ValueError" if *value* is not found in *sequence*. + +sequence.reverse() + + Reverse the items of *sequence* in place. This method maintains + economy of space when reversing a large sequence. To remind users + that it operates by side-effect, it returns "None". + Lists ===== @@ -12399,7 +13670,7 @@ class dict(iterable, **kwargs) homogeneous items (where the precise degree of similarity will vary by application). -class list([iterable]) +class list(iterable=(), /) Lists may be constructed in several ways: @@ -12480,7 +13751,7 @@ class list([iterable]) of homogeneous data is needed (such as allowing storage in a "set" or "dict" instance). -class tuple([iterable]) +class tuple(iterable=(), /) Tuples may be constructed in a number of ways: @@ -12520,8 +13791,8 @@ class tuple([iterable]) The "range" type represents an immutable sequence of numbers and is commonly used for looping a specific number of times in "for" loops. -class range(stop) -class range(start, stop[, step]) +class range(stop, /) +class range(start, stop, step=1, /) The arguments to the range constructor must be integers (either built-in "int" or any object that implements the "__index__()" @@ -12649,11 +13920,15 @@ class range(start, stop[, step]) | "s[i] = x" | item *i* of *s* is replaced by | | | | *x* | | +--------------------------------+----------------------------------+-----------------------+ +| "del s[i]" | removes item *i* of *s* | | ++--------------------------------+----------------------------------+-----------------------+ | "s[i:j] = t" | slice of *s* from *i* to *j* is | | | | replaced by the contents of the | | | | iterable *t* | | +--------------------------------+----------------------------------+-----------------------+ -| "del s[i:j]" | same as "s[i:j] = []" | | +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | +--------------------------------+----------------------------------+-----------------------+ | "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | | | replaced by those of *t* | | @@ -12661,63 +13936,79 @@ class range(start, stop[, step]) | "del s[i:j:k]" | removes the elements of | | | | "s[i:j:k]" from the list | | +--------------------------------+----------------------------------+-----------------------+ -| "s.append(x)" | appends *x* to the end of the | | -| | sequence (same as | | -| | "s[len(s):len(s)] = [x]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.clear()" | removes all items from *s* (same | (5) | -| | as "del s[:]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.copy()" | creates a shallow copy of *s* | (5) | -| | (same as "s[:]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.extend(t)" or "s += t" | extends *s* with the contents of | | +| "s += t" | extends *s* with the contents of | | | | *t* (for the most part the same | | | | as "s[len(s):len(s)] = t") | | +--------------------------------+----------------------------------+-----------------------+ -| "s *= n" | updates *s* with its contents | (6) | +| "s *= n" | updates *s* with its contents | (2) | | | repeated *n* times | | +--------------------------------+----------------------------------+-----------------------+ -| "s.insert(i, x)" | inserts *x* into *s* at the | | -| | index given by *i* (same as | | -| | "s[i:i] = [x]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.pop()" or "s.pop(i)" | retrieves the item at *i* and | (2) | -| | also removes it from *s* | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.remove(x)" | removes the first item from *s* | (3) | -| | where "s[i]" is equal to *x* | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.reverse()" | reverses the items of *s* in | (4) | -| | place | | -+--------------------------------+----------------------------------+-----------------------+ Notes: 1. If *k* is not equal to "1", *t* must have the same length as the slice it is replacing. -2. The optional argument *i* defaults to "-1", so that by default the - last item is removed and returned. - -3. "remove()" raises "ValueError" when *x* is not found in *s*. - -4. The "reverse()" method modifies the sequence in place for economy - of space when reversing a large sequence. To remind users that it - operates by side effect, it does not return the reversed sequence. - -5. "clear()" and "copy()" are included for consistency with the - interfaces of mutable containers that don’t support slicing - operations (such as "dict" and "set"). "copy()" is not part of the - "collections.abc.MutableSequence" ABC, but most concrete mutable - sequence classes provide it. - - Added in version 3.3: "clear()" and "copy()" methods. - -6. The value *n* is an integer, or an object implementing +2. The value *n* is an integer, or an object implementing "__index__()". Zero and negative values of *n* clear the sequence. Items in the sequence are not copied; they are referenced multiple times, as explained for "s * n" under Common Sequence Operations. + +-[ Mutable Sequence Methods ]- + +Mutable sequence types also support the following methods: + +sequence.append(value, /) + + Append *value* to the end of the sequence This is equivalent to + writing "seq[len(seq):len(seq)] = [value]". + +sequence.clear() + + Added in version 3.3. + + Remove all items from *sequence*. This is equivalent to writing + "del sequence[:]". + +sequence.copy() + + Added in version 3.3. + + Create a shallow copy of *sequence*. This is equivalent to writing + "sequence[:]". + + Hint: + + The "copy()" method is not part of the "MutableSequence" "ABC", + but most concrete mutable sequence types provide it. + +sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. For the most + part, this is the same as writing "seq[len(seq):len(seq)] = + iterable". + +sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. This is + equivalent to writing "sequence[index:index] = [value]". + +sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +sequence.remove(value, /) + + Remove the first item from *sequence* where "sequence[i] == value". + + Raises "ValueError" if *value* is not found in *sequence*. + +sequence.reverse() + + Reverse the items of *sequence* in place. This method maintains + economy of space when reversing a large sequence. To remind users + that it operates by side-effect, it returns "None". ''', 'unary': r'''Unary arithmetic and bitwise operations *************************************** diff --git a/Lib/queue.py b/Lib/queue.py index 25beb46e30d..c0b35987654 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -80,9 +80,6 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). - shutdown(immediate=True) calls task_done() for each remaining item in - the queue. - Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -239,9 +236,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', a task is marked as done for each item remaining in - the queue, which may unblock callers of join(). + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. ''' with self.mutex: self.is_shutdown = True diff --git a/Lib/random.py b/Lib/random.py index 86d562f0b8a..c89cbb755ab 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -844,8 +844,8 @@ def binomialvariate(self, n=1, p=0.5): # BTRS: Transformed rejection with squeeze method by Wolfgang Hörmann # https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.47.8407&rep=rep1&type=pdf assert n*p >= 10.0 and p <= 0.5 - setup_complete = False + setup_complete = False spq = _sqrt(n * p * (1.0 - p)) # Standard deviation of the distribution b = 1.15 + 2.53 * spq a = -0.0873 + 0.0248 * b + 0.01 * p @@ -860,22 +860,23 @@ def binomialvariate(self, n=1, p=0.5): k = _floor((2.0 * a / us + b) * u + c) if k < 0 or k > n: continue + v = random() # The early-out "squeeze" test substantially reduces # the number of acceptance condition evaluations. - v = random() if us >= 0.07 and v <= vr: return k - # Acceptance-rejection test. - # Note, the original paper erroneously omits the call to log(v) - # when comparing to the log of the rescaled binomial distribution. if not setup_complete: alpha = (2.83 + 5.1 / b) * spq lpq = _log(p / (1.0 - p)) m = _floor((n + 1) * p) # Mode of the distribution h = _lgamma(m + 1) + _lgamma(n - m + 1) setup_complete = True # Only needs to be done once + + # Acceptance-rejection test. + # Note, the original paper erroneously omits the call to log(v) + # when comparing to the log of the rescaled binomial distribution. v *= alpha / (a / (us * us) + b) if _log(v) <= h - _lgamma(k + 1) - _lgamma(n - k + 1) + (k - m) * lpq: return k diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index af2808a77da..ecec16e9005 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -137,8 +137,6 @@ "UNICODE", "NOFLAG", "RegexFlag", "PatternError" ] -__version__ = "2.2.1" - @enum.global_enum @enum._simple_enum(enum.IntFlag, boundary=enum.KEEP) class RegexFlag: @@ -399,9 +397,12 @@ def __init__(self, lexicon, flags=0): s = _parser.State() s.flags = flags for phrase, action in lexicon: + sub_pattern = _parser.parse(phrase, flags) + if sub_pattern.state.groups != 1: + raise ValueError("Cannot use capturing groups in re.Scanner") gid = s.opengroup() p.append(_parser.SubPattern(s, [ - (SUBPATTERN, (gid, 0, 0, _parser.parse(phrase, flags))), + (SUBPATTERN, (gid, 0, 0, sub_pattern)), ])) s.closegroup(gid, p[-1]) p = _parser.SubPattern(s, [(BRANCH, (None, p))]) @@ -426,3 +427,12 @@ def scan(self, string): append(action) i = j return result, string[i:] + + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "2.2.1" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index 20dd561d1c1..c2ca8e25abe 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -375,7 +375,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): # less significant byte is a bit index in the chunk (just like the # CHARSET matching). - charmap = bytes(charmap) # should be hashable + charmap = charmap.take_bytes() # should be hashable comps = {} mapping = bytearray(256) block = 0 diff --git a/Lib/reprlib.py b/Lib/reprlib.py index 441d1be4bde..ab18247682b 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -181,7 +181,22 @@ def repr_str(self, x, level): return s def repr_int(self, x, level): - s = builtins.repr(x) # XXX Hope this isn't too slow... + try: + s = builtins.repr(x) + except ValueError as exc: + assert 'sys.set_int_max_str_digits()' in str(exc) + # Those imports must be deferred due to Python's build system + # where the reprlib module is imported before the math module. + import math, sys + # Integers with more than sys.get_int_max_str_digits() digits + # are rendered differently as their repr() raises a ValueError. + # See https://github.com/python/cpython/issues/135487. + k = 1 + int(math.log10(abs(x))) + # Note: math.log10(abs(x)) may be overestimated or underestimated, + # but for simplicity, we do not compute the exact number of digits. + max_digits = sys.get_int_max_str_digits() + return (f'<{x.__class__.__name__} instance with roughly {k} ' + f'digits (limit at {max_digits}) at 0x{id(x):x}>') if len(s) > self.maxlong: i = max(0, (self.maxlong-3)//2) j = max(0, self.maxlong-3-i) diff --git a/Lib/runpy.py b/Lib/runpy.py index ef54d3282ee..f072498f6cb 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -247,7 +247,7 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(fname): +def _get_code_from_file(fname, module): # Check for a compiled file first from pkgutil import read_code code_path = os.path.abspath(fname) @@ -256,7 +256,7 @@ def _get_code_from_file(fname): if code is None: # That didn't work, so try it as normal source code with io.open_code(code_path) as f: - code = compile(f.read(), fname, 'exec') + code = compile(f.read(), fname, 'exec', module=module) return code def run_path(path_name, init_globals=None, run_name=None): @@ -283,7 +283,7 @@ def run_path(path_name, init_globals=None, run_name=None): if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code = _get_code_from_file(path_name) + code = _get_code_from_file(path_name, run_name) return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=path_name) else: diff --git a/Lib/shelve.py b/Lib/shelve.py index 50584716e9e..9f6296667fd 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -56,12 +56,16 @@ the persistent dictionary on disk, if feasible). """ -from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler -from io import BytesIO +from pickle import DEFAULT_PROTOCOL, dumps, loads import collections.abc -__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] +__all__ = ["ShelveError", "Shelf", "BsdDbShelf", "DbfilenameShelf", "open"] + + +class ShelveError(Exception): + pass + class _ClosedDict(collections.abc.MutableMapping): 'Marker for a closed dict. Access attempts raise a ValueError.' @@ -82,7 +86,7 @@ class Shelf(collections.abc.MutableMapping): """ def __init__(self, dict, protocol=None, writeback=False, - keyencoding="utf-8"): + keyencoding="utf-8", *, serializer=None, deserializer=None): self.dict = dict if protocol is None: protocol = DEFAULT_PROTOCOL @@ -91,6 +95,16 @@ def __init__(self, dict, protocol=None, writeback=False, self.cache = {} self.keyencoding = keyencoding + if serializer is None and deserializer is None: + self.serializer = dumps + self.deserializer = loads + elif (serializer is None) ^ (deserializer is None): + raise ShelveError("serializer and deserializer must be " + "defined together") + else: + self.serializer = serializer + self.deserializer = deserializer + def __iter__(self): for k in self.dict.keys(): yield k.decode(self.keyencoding) @@ -110,8 +124,8 @@ def __getitem__(self, key): try: value = self.cache[key] except KeyError: - f = BytesIO(self.dict[key.encode(self.keyencoding)]) - value = Unpickler(f).load() + f = self.dict[key.encode(self.keyencoding)] + value = self.deserializer(f) if self.writeback: self.cache[key] = value return value @@ -119,10 +133,8 @@ def __getitem__(self, key): def __setitem__(self, key, value): if self.writeback: self.cache[key] = value - f = BytesIO() - p = Pickler(f, self._protocol) - p.dump(value) - self.dict[key.encode(self.keyencoding)] = f.getvalue() + serialized_value = self.serializer(value, self._protocol) + self.dict[key.encode(self.keyencoding)] = serialized_value def __delitem__(self, key): del self.dict[key.encode(self.keyencoding)] @@ -171,6 +183,11 @@ def sync(self): if hasattr(self.dict, 'sync'): self.dict.sync() + def reorganize(self): + self.sync() + if hasattr(self.dict, 'reorganize'): + self.dict.reorganize() + class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface. @@ -186,33 +203,29 @@ class BsdDbShelf(Shelf): """ def __init__(self, dict, protocol=None, writeback=False, - keyencoding="utf-8"): - Shelf.__init__(self, dict, protocol, writeback, keyencoding) + keyencoding="utf-8", *, serializer=None, deserializer=None): + Shelf.__init__(self, dict, protocol, writeback, keyencoding, + serializer=serializer, deserializer=deserializer) def set_location(self, key): (key, value) = self.dict.set_location(key) - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def next(self): (key, value) = next(self.dict) - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def previous(self): (key, value) = self.dict.previous() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def first(self): (key, value) = self.dict.first() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) def last(self): (key, value) = self.dict.last() - f = BytesIO(value) - return (key.decode(self.keyencoding), Unpickler(f).load()) + return (key.decode(self.keyencoding), self.deserializer(value)) class DbfilenameShelf(Shelf): @@ -222,9 +235,11 @@ class DbfilenameShelf(Shelf): See the module's __doc__ string for an overview of the interface. """ - def __init__(self, filename, flag='c', protocol=None, writeback=False): + def __init__(self, filename, flag='c', protocol=None, writeback=False, *, + serializer=None, deserializer=None): import dbm - Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) + Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback, + serializer=serializer, deserializer=deserializer) def clear(self): """Remove all items from the shelf.""" @@ -233,8 +248,8 @@ def clear(self): self.cache.clear() self.dict.clear() - -def open(filename, flag='c', protocol=None, writeback=False): +def open(filename, flag='c', protocol=None, writeback=False, *, + serializer=None, deserializer=None): """Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying @@ -247,4 +262,5 @@ def open(filename, flag='c', protocol=None, writeback=False): See the module's __doc__ string for an overview of the interface. """ - return DbfilenameShelf(filename, flag, protocol, writeback) + return DbfilenameShelf(filename, flag, protocol, writeback, + serializer=serializer, deserializer=deserializer) diff --git a/Lib/shlex.py b/Lib/shlex.py index 5bf6e0d70e0..5959f52dd12 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -322,6 +322,9 @@ def quote(s): if not s: return "''" + if not isinstance(s, str): + raise TypeError(f"expected string object, got {type(s).__name__!r}") + # Use bytes.translate() for performance safe_chars = (b'%+,-./0123456789:=@' b'ABCDEFGHIJKLMNOPQRSTUVWXYZ_' diff --git a/Lib/shutil.py b/Lib/shutil.py index ca0a2ea2f7f..8d8fe145567 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -994,14 +994,14 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, """Create a (possibly compressed) tar file from all the files under 'base_dir'. - 'compress' must be "gzip" (the default), "bzip2", "xz", or None. + 'compress' must be "gzip" (the default), "bzip2", "xz", "zst", or None. 'owner' and 'group' can be used to define an owner and a group for the archive that is being built. If not provided, the current owner and group will be used. The output tar file will be named 'base_name' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", or ".xz"). + the appropriate compression extension (".gz", ".bz2", ".xz", or ".zst"). Returns the output filename. """ @@ -1187,7 +1187,7 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, 'base_name' is the name of the file to create, minus any format-specific extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", "zstdtar", or "xztar". Or any other registered format. + "bztar", "xztar", or "zstdtar". Or any other registered format. 'root_dir' is a directory that will be the root directory of the archive; ie. we typically chdir into 'root_dir' before creating the @@ -1337,7 +1337,7 @@ def _unpack_zipfile(filename, extract_dir): zip.close() def _unpack_tarfile(filename, extract_dir, *, filter=None): - """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` + """Unpack tar/tar.gz/tar.bz2/tar.xz/tar.zst `filename` to `extract_dir` """ import tarfile # late import for breaking circular dependency try: @@ -1392,7 +1392,7 @@ def unpack_archive(filename, extract_dir=None, format=None, *, filter=None): is unpacked. If not provided, the current working directory is used. `format` is the archive format: one of "zip", "tar", "gztar", "bztar", - or "xztar". Or any other registered format. If not provided, + "xztar", or "zstdtar". Or any other registered format. If not provided, unpack_archive will use the filename extension and see if an unpacker was registered for that extension. diff --git a/Lib/site.py b/Lib/site.py index 5c38b1b17d5..7c6810792cf 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -75,6 +75,7 @@ import _sitebuiltins import _io as io import stat +import errno # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -332,7 +333,7 @@ def _get_path(userbase): if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/python{version[0]}.{version[1]}{abi_thread}/site-packages' + return f'{userbase}/lib/{implementation_lower}{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -448,9 +449,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software - Foundation, and a cast of thousands for supporting Python - development. See www.python.org for more information.""") +Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software +Foundation, and a cast of thousands for supporting Python +development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -578,10 +579,15 @@ def register_readline(): def write_history(): try: readline_module.write_history_file(history) - except (FileNotFoundError, PermissionError): + except FileNotFoundError, PermissionError: # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass + except OSError: + if errno.EROFS: + pass # gh-128066: read-only file system + else: + raise atexit.register(write_history) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 84d6d858e7d..72093f7f8b0 100644 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -177,6 +177,15 @@ def _quote_periods(bindata): def _fix_eols(data): return re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data) + +try: + hmac.digest(b'', b'', 'md5') +except ValueError: + _have_cram_md5_support = False +else: + _have_cram_md5_support = True + + try: import ssl except ImportError: @@ -665,8 +674,11 @@ def auth_cram_md5(self, challenge=None): # CRAM-MD5 does not support initial-response. if challenge is None: return None - return self.user + " " + hmac.HMAC( - self.password.encode('ascii'), challenge, 'md5').hexdigest() + if not _have_cram_md5_support: + raise SMTPException("CRAM-MD5 is not supported") + password = self.password.encode('ascii') + authcode = hmac.HMAC(password, challenge, 'md5') + return f"{self.user} {authcode.hexdigest()}" def auth_plain(self, challenge=None): """ Authobject to use with PLAIN authentication. Requires self.user and @@ -718,8 +730,10 @@ def login(self, user, password, *, initial_response_ok=True): advertised_authlist = self.esmtp_features["auth"].split() # Authentication methods we can handle in our preferred order: - preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] - + if _have_cram_md5_support: + preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] + else: + preferred_auths = ['PLAIN', 'LOGIN'] # We try the supported authentications in our preferred order, if # the server supports them. authlist = [auth for auth in preferred_auths @@ -903,7 +917,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, The arguments are as for sendmail, except that msg is an email.message.Message object. If from_addr is None or to_addrs is None, these arguments are taken from the headers of the Message as - described in RFC 2822 (a ValueError is raised if there is more than + described in RFC 5322 (a ValueError is raised if there is more than one set of 'Resent-' headers). Regardless of the values of from_addr and to_addr, any Bcc field (or Resent-Bcc field, when the Message is a resent) of the Message object won't be transmitted. The Message @@ -917,7 +931,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, policy. """ - # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822 + # 'Resent-Date' is a mandatory field if the Message is resent (RFC 5322 # Section 3.6.6). In such a case, we use the 'Resent-*' fields. However, # if there is more than one 'Resent-' block there's no way to # unambiguously determine which one is the most recent in all cases, @@ -936,7 +950,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, else: raise ValueError("message has more than one 'Resent-' header block") if from_addr is None: - # Prefer the sender field per RFC 2822:3.6.2. + # Prefer the sender field per RFC 5322 section 3.6.2. from_addr = (msg[header_prefix + 'Sender'] if (header_prefix + 'Sender') in msg else msg[header_prefix + 'From']) diff --git a/Lib/socket.py b/Lib/socket.py index 727b0e75f03..3073c012b19 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -56,6 +56,7 @@ import os import sys from enum import IntEnum, IntFlag +from functools import partial try: import errno @@ -348,75 +349,83 @@ def makefile(self, mode="r", buffering=None, *, text.mode = mode return text + def _sendfile_zerocopy(self, zerocopy_func, giveup_exc_type, file, + offset=0, count=None): + """ + Send a file using a zero-copy function. + """ + import selectors + + self._check_sendfile_params(file, offset, count) + sockno = self.fileno() + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise giveup_exc_type(err) # not a regular file + try: + fsize = os.fstat(fileno).st_size + except OSError as err: + raise giveup_exc_type(err) # not a regular file + if not fsize: + return 0 # empty file + # Truncate to 1GiB to avoid OverflowError, see bpo-38319. + blocksize = min(count or fsize, 2 ** 30) + timeout = self.gettimeout() + if timeout == 0: + raise ValueError("non-blocking sockets are not supported") + # poll/select have the advantage of not requiring any + # extra file descriptor, contrarily to epoll/kqueue + # (also, they require a single syscall). + if hasattr(selectors, 'PollSelector'): + selector = selectors.PollSelector() + else: + selector = selectors.SelectSelector() + selector.register(sockno, selectors.EVENT_WRITE) + + total_sent = 0 + # localize variable access to minimize overhead + selector_select = selector.select + try: + while True: + if timeout and not selector_select(timeout): + raise TimeoutError('timed out') + if count: + blocksize = min(count - total_sent, blocksize) + if blocksize <= 0: + break + try: + sent = zerocopy_func(fileno, offset, blocksize) + except BlockingIOError: + if not timeout: + # Block until the socket is ready to send some + # data; avoids hogging CPU resources. + selector_select() + continue + except OSError as err: + if total_sent == 0: + # We can get here for different reasons, the main + # one being 'file' is not a regular mmap(2)-like + # file, in which case we'll fall back on using + # plain send(). + raise giveup_exc_type(err) + raise err from None + else: + if sent == 0: + break # EOF + offset += sent + total_sent += sent + return total_sent + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset) + if hasattr(os, 'sendfile'): - def _sendfile_use_sendfile(self, file, offset=0, count=None): - # Lazy import to improve module import time - import selectors - - self._check_sendfile_params(file, offset, count) - sockno = self.fileno() - try: - fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: - raise _GiveupOnSendfile(err) # not a regular file - try: - fsize = os.fstat(fileno).st_size - except OSError as err: - raise _GiveupOnSendfile(err) # not a regular file - if not fsize: - return 0 # empty file - # Truncate to 1GiB to avoid OverflowError, see bpo-38319. - blocksize = min(count or fsize, 2 ** 30) - timeout = self.gettimeout() - if timeout == 0: - raise ValueError("non-blocking sockets are not supported") - # poll/select have the advantage of not requiring any - # extra file descriptor, contrarily to epoll/kqueue - # (also, they require a single syscall). - if hasattr(selectors, 'PollSelector'): - selector = selectors.PollSelector() - else: - selector = selectors.SelectSelector() - selector.register(sockno, selectors.EVENT_WRITE) - - total_sent = 0 - # localize variable access to minimize overhead - selector_select = selector.select - os_sendfile = os.sendfile - try: - while True: - if timeout and not selector_select(timeout): - raise TimeoutError('timed out') - if count: - blocksize = min(count - total_sent, blocksize) - if blocksize <= 0: - break - try: - sent = os_sendfile(sockno, fileno, offset, blocksize) - except BlockingIOError: - if not timeout: - # Block until the socket is ready to send some - # data; avoids hogging CPU resources. - selector_select() - continue - except OSError as err: - if total_sent == 0: - # We can get here for different reasons, the main - # one being 'file' is not a regular mmap(2)-like - # file, in which case we'll fall back on using - # plain send(). - raise _GiveupOnSendfile(err) - raise err from None - else: - if sent == 0: - break # EOF - offset += sent - total_sent += sent - return total_sent - finally: - if total_sent > 0 and hasattr(file, 'seek'): - file.seek(offset) + return self._sendfile_zerocopy( + partial(os.sendfile, self.fileno()), + _GiveupOnSendfile, + file, offset, count, + ) else: def _sendfile_use_sendfile(self, file, offset=0, count=None): raise _GiveupOnSendfile( diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 35b2723de3b..ec389457ef5 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -120,9 +120,6 @@ class will essentially render the service "deaf" while one request is # Author of the BaseServer patch: Luke Kenneth Casson Leighton -__version__ = "0.4" - - import socket import selectors import os @@ -441,7 +438,7 @@ class TCPServer(BaseServer): socket_type = socket.SOCK_STREAM - request_queue_size = 5 + request_queue_size = getattr(socket, "SOMAXCONN", 5) allow_reuse_address = False @@ -861,3 +858,12 @@ def setup(self): def finish(self): self.socket.sendto(self.wfile.getvalue(), self.client_address) + + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "0.4" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 002f1986cdd..b3746ed7573 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -10,9 +10,12 @@ from argparse import ArgumentParser from code import InteractiveConsole from textwrap import dedent +from _colorize import get_theme, theme_no_color + +from ._completer import completer -def execute(c, sql, suppress_errors=True): +def execute(c, sql, suppress_errors=True, theme=theme_no_color): """Helper that wraps execution of SQL code. This is used both by the REPL and by direct execution from the CLI. @@ -25,11 +28,15 @@ def execute(c, sql, suppress_errors=True): for row in c.execute(sql): print(row) except sqlite3.Error as e: + t = theme.traceback tp = type(e).__name__ try: - print(f"{tp} ({e.sqlite_errorname}): {e}", file=sys.stderr) + tp += f" ({e.sqlite_errorname})" except AttributeError: - print(f"{tp}: {e}", file=sys.stderr) + pass + print( + f"{t.type}{tp}{t.reset}: {t.message}{e}{t.reset}", file=sys.stderr + ) if not suppress_errors: sys.exit(1) @@ -37,10 +44,11 @@ def execute(c, sql, suppress_errors=True): class SqliteInteractiveConsole(InteractiveConsole): """A simple SQLite REPL.""" - def __init__(self, connection): + def __init__(self, connection, use_color=False): super().__init__() self._con = connection self._cur = connection.cursor() + self._use_color = use_color def runsource(self, source, filename="", symbol="single"): """Override runsource, the core of the InteractiveConsole REPL. @@ -48,17 +56,33 @@ def runsource(self, source, filename="", symbol="single"): Return True if more input is needed; buffering is done automatically. Return False if input is a complete statement ready for execution. """ - match source: - case ".version": - print(f"{sqlite3.sqlite_version}") - case ".help": - print("Enter SQL code and press enter.") - case ".quit": - sys.exit(0) - case _: - if not sqlite3.complete_statement(source): - return True - execute(self._cur, source) + theme = get_theme(force_no_color=not self._use_color) + + if not source or source.isspace(): + return False + # Remember to update CLI_COMMANDS in _completer.py + if source[0] == ".": + match source[1:].strip(): + case "version": + print(sqlite3.sqlite_version) + case "help": + t = theme.syntax + print(f"Enter SQL code or one of the below commands, and press enter.\n\n" + f"{t.builtin}.version{t.reset} Print underlying SQLite library version\n" + f"{t.builtin}.help{t.reset} Print this help message\n" + f"{t.builtin}.quit{t.reset} Exit the CLI, equivalent to CTRL-D\n") + case "quit": + sys.exit(0) + case "": + pass + case _ as unknown: + t = theme.traceback + self.write(f'{t.type}Error{t.reset}: {t.message}unknown ' + f'command: "{unknown}"{t.reset}\n') + else: + if not sqlite3.complete_statement(source): + return True + execute(self._cur, source, theme=theme) return False @@ -105,22 +129,23 @@ def main(*args): Each command will be run using execute() on the cursor. Type ".help" for more information; type ".quit" or {eofkey} to quit. """).strip() - sys.ps1 = "sqlite> " - sys.ps2 = " ... " + + theme = get_theme() + s = theme.syntax + + sys.ps1 = f"{s.prompt}sqlite> {s.reset}" + sys.ps2 = f"{s.prompt} ... {s.reset}" con = sqlite3.connect(args.filename, isolation_level=None) try: if args.sql: # SQL statement provided on the command-line; execute it directly. - execute(con, args.sql, suppress_errors=False) + execute(con, args.sql, suppress_errors=False, theme=theme) else: # No SQL provided; start the REPL. - console = SqliteInteractiveConsole(con) - try: - import readline # noqa: F401 - except ImportError: - pass - console.interact(banner, exitmsg="") + with completer(con): + console = SqliteInteractiveConsole(con, use_color=True) + console.interact(banner, exitmsg="") finally: con.close() diff --git a/Lib/sqlite3/_completer.py b/Lib/sqlite3/_completer.py new file mode 100644 index 00000000000..ba580f968bf --- /dev/null +++ b/Lib/sqlite3/_completer.py @@ -0,0 +1,111 @@ +from _sqlite3 import OperationalError +from contextlib import contextmanager + +try: + from _sqlite3 import SQLITE_KEYWORDS +except ImportError: + SQLITE_KEYWORDS = () + +CLI_COMMANDS = ('.quit', '.help', '.version') + +_completion_matches = [] + + +def _complete(con, text, state): + global _completion_matches + + if state == 0: + if text.startswith("."): + _completion_matches = [ + c + " " for c in CLI_COMMANDS if c.startswith(text) + ] + else: + text_upper = text.upper() + _completion_matches = [ + c + " " for c in SQLITE_KEYWORDS if c.startswith(text_upper) + ] + + cursor = con.cursor() + schemata = tuple(row[1] for row + in cursor.execute("PRAGMA database_list")) + # tables, indexes, triggers, and views + # escape '_' which can appear in attached database names + select_clauses = ( + f"""\ + SELECT name || ' ' FROM \"{schema}\".sqlite_master + WHERE name LIKE REPLACE(:text, '_', '^_') || '%' ESCAPE '^'""" + for schema in schemata + ) + _completion_matches.extend( + row[0] + for row in cursor.execute( + " UNION ".join(select_clauses), {"text": text} + ) + ) + # columns + try: + select_clauses = ( + f"""\ + SELECT pti.name || ' ' FROM "{schema}".sqlite_master AS sm + JOIN pragma_table_xinfo(sm.name,'{schema}') AS pti + WHERE sm.type='table' AND + pti.name LIKE REPLACE(:text, '_', '^_') || '%' ESCAPE '^'""" + for schema in schemata + ) + _completion_matches.extend( + row[0] + for row in cursor.execute( + " UNION ".join(select_clauses), {"text": text} + ) + ) + except OperationalError: + # skip on SQLite<3.16.0 where pragma table-valued function is + # not supported yet + pass + # functions + try: + _completion_matches.extend( + row[0] for row in cursor.execute("""\ + SELECT DISTINCT UPPER(name) || '(' + FROM pragma_function_list() + WHERE name NOT IN ('->', '->>') AND + name LIKE REPLACE(:text, '_', '^_') || '%' ESCAPE '^'""", + {"text": text}, + ) + ) + except OperationalError: + # skip on SQLite<3.30.0 where function_list is not supported yet + pass + # schemata + text_lower = text.lower() + _completion_matches.extend(c for c in schemata + if c.lower().startswith(text_lower)) + _completion_matches = sorted(set(_completion_matches)) + try: + return _completion_matches[state] + except IndexError: + return None + + +@contextmanager +def completer(con): + try: + import readline + except ImportError: + yield + return + + old_completer = readline.get_completer() + def complete(text, state): + return _complete(con, text, state) + try: + readline.set_completer(complete) + if readline.backend == "editline": + # libedit uses "^I" instead of "tab" + command_string = "bind ^I rl_complete" + else: + command_string = "tab: complete" + readline.parse_and_bind(command_string) + yield + finally: + readline.set_completer(old_completer) diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py deleted file mode 100644 index f9da61e6487..00000000000 --- a/Lib/sre_compile.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -warnings.warn(f"module {__name__!r} is deprecated", - DeprecationWarning, - stacklevel=2) - -from re import _compiler as _ -globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'}) diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py deleted file mode 100644 index fa09d044292..00000000000 --- a/Lib/sre_constants.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -warnings.warn(f"module {__name__!r} is deprecated", - DeprecationWarning, - stacklevel=2) - -from re import _constants as _ -globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'}) diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py deleted file mode 100644 index 25a3f557d44..00000000000 --- a/Lib/sre_parse.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -warnings.warn(f"module {__name__!r} is deprecated", - DeprecationWarning, - stacklevel=2) - -from re import _parser as _ -globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'}) diff --git a/Lib/ssl.py b/Lib/ssl.py index 05df4ad7f0f..67a2990b281 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -13,6 +13,9 @@ Functions: + get_sigalgs -- return a list of all available TLS signature + algorithms (requires OpenSSL 3.4 or later) + cert_time_to_seconds -- convert time string used for certificate notBefore and notAfter functions to integer seconds past the Epoch (the time values @@ -107,16 +110,12 @@ ) from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj from _ssl import RAND_status, RAND_add, RAND_bytes -try: - from _ssl import RAND_egd -except ImportError: - # RAND_egd is not supported on some platforms - pass +from _ssl import get_sigalgs from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK, HAS_PHA + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK, HAS_PSK_TLS13, HAS_PHA ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION @@ -186,7 +185,7 @@ class _TLSContentType: class _TLSAlertType: """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 @@ -931,6 +930,18 @@ def cipher(self): ssl_version, secret_bits)``.""" return self._sslobj.cipher() + def group(self): + """Return the currently selected key agreement group name.""" + return self._sslobj.group() + + def client_sigalg(self): + """Return the selected client authentication signature algorithm.""" + return self._sslobj.client_sigalg() + + def server_sigalg(self): + """Return the selected server handshake signature algorithm.""" + return self._sslobj.server_sigalg() + def shared_ciphers(self): """Return a list of ciphers shared by the client during the handshake or None if this is not a valid server connection. @@ -975,6 +986,10 @@ def _sslcopydoc(func): return func +class _GiveupOnSSLSendfile(Exception): + pass + + class SSLSocket(socket): """This class implements a subtype of socket.socket that wraps the underlying OS socket in an SSL context when necessary, and @@ -1206,6 +1221,30 @@ def cipher(self): else: return self._sslobj.cipher() + @_sslcopydoc + def group(self): + self._checkClosed() + if self._sslobj is None: + return None + else: + return self._sslobj.group() + + @_sslcopydoc + def client_sigalg(self): + self._checkClosed() + if self._sslobj is None: + return None + else: + return self._sslobj.client_sigalg() + + @_sslcopydoc + def server_sigalg(self): + self._checkClosed() + if self._sslobj is None: + return None + else: + return self._sslobj.server_sigalg() + @_sslcopydoc def shared_ciphers(self): self._checkClosed() @@ -1266,15 +1305,26 @@ def sendall(self, data, flags=0): return super().sendall(data, flags) def sendfile(self, file, offset=0, count=None): - """Send a file, possibly by using os.sendfile() if this is a - clear-text socket. Return the total number of bytes sent. + """Send a file, possibly by using an efficient sendfile() call if + the system supports it. Return the total number of bytes sent. """ - if self._sslobj is not None: - return self._sendfile_use_send(file, offset, count) - else: - # os.sendfile() works with plain sockets only + if self._sslobj is None: return super().sendfile(file, offset, count) + if not self._sslobj.uses_ktls_for_send(): + return self._sendfile_use_send(file, offset, count) + + sendfile = getattr(self._sslobj, "sendfile", None) + if sendfile is None: + return self._sendfile_use_send(file, offset, count) + + try: + return self._sendfile_zerocopy( + sendfile, _GiveupOnSSLSendfile, file, offset, count, + ) + except _GiveupOnSSLSendfile: + return self._sendfile_use_send(file, offset, count) + def recv(self, buflen=1024, flags=0): self._checkClosed() if self._sslobj is not None: diff --git a/Lib/stat.py b/Lib/stat.py index 1b4ed1ebc94..ab1b25b9d63 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -200,6 +200,21 @@ def filemode(mode): FILE_ATTRIBUTE_VIRTUAL = 65536 +# Linux STATX_ATTR constants for interpreting os.statx()'s +# "stx_attributes" and "stx_attributes_mask" members + +STATX_ATTR_COMPRESSED = 0x00000004 +STATX_ATTR_IMMUTABLE = 0x00000010 +STATX_ATTR_APPEND = 0x00000020 +STATX_ATTR_NODUMP = 0x00000040 +STATX_ATTR_ENCRYPTED = 0x00000800 +STATX_ATTR_AUTOMOUNT = 0x00001000 +STATX_ATTR_MOUNT_ROOT = 0x00002000 +STATX_ATTR_VERITY = 0x00100000 +STATX_ATTR_DAX = 0x00200000 +STATX_ATTR_WRITE_ATOMIC = 0x00400000 + + # If available, use C implementation try: from _stat import * diff --git a/Lib/statistics.py b/Lib/statistics.py index 3d805cb0739..26cf925529e 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -619,9 +619,14 @@ def stdev(data, xbar=None): if n < 2: raise StatisticsError('stdev requires at least two data points') mss = ss / (n - 1) + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) def pstdev(data, mu=None): @@ -637,9 +642,14 @@ def pstdev(data, mu=None): if n < 1: raise StatisticsError('pstdev requires at least one data point') mss = ss / n + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) ## Statistics for relations between two inputs ############################# diff --git a/Lib/string/__init__.py b/Lib/string/__init__.py index eab5067c9b1..b788d7136f1 100644 --- a/Lib/string/__init__.py +++ b/Lib/string/__init__.py @@ -191,16 +191,14 @@ def get_identifiers(self): ######################################################################## -# the Formatter class -# see PEP 3101 for details and purpose of this class - -# The hard parts are reused from the C implementation. They're exposed as "_" -# prefixed methods of str. - +# The Formatter class (PEP 3101). +# # The overall parser is implemented in _string.formatter_parser. -# The field name parser is implemented in _string.formatter_field_name_split +# The field name parser is implemented in _string.formatter_field_name_split. class Formatter: + """See PEP 3101 for details and purpose of this class.""" + def format(self, format_string, /, *args, **kwargs): return self.vformat(format_string, args, kwargs) @@ -264,22 +262,18 @@ def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, return ''.join(result), auto_arg_index - def get_value(self, key, args, kwargs): if isinstance(key, int): return args[key] else: return kwargs[key] - def check_unused_args(self, used_args, args, kwargs): pass - def format_field(self, value, format_spec): return format(value, format_spec) - def convert_field(self, value, conversion): # do any conversion on the resulting object if conversion is None: @@ -292,28 +286,26 @@ def convert_field(self, value, conversion): return ascii(value) raise ValueError("Unknown conversion specifier {0!s}".format(conversion)) - - # returns an iterable that contains tuples of the form: - # (literal_text, field_name, format_spec, conversion) - # literal_text can be zero length - # field_name can be None, in which case there's no - # object to format and output - # if field_name is not None, it is looked up, formatted - # with format_spec and conversion and then used def parse(self, format_string): + """ + Return an iterable that contains tuples of the form + (literal_text, field_name, format_spec, conversion). + + *field_name* can be None, in which case there's no object + to format and output; otherwise, it is looked up and + formatted with *format_spec* and *conversion*. + """ return _string.formatter_parser(format_string) - - # given a field_name, find the object it references. - # field_name: the field being looked up, e.g. "0.name" - # or "lookup[3]" - # used_args: a set of which args have been used - # args, kwargs: as passed in to vformat def get_field(self, field_name, args, kwargs): + """Find the object referenced by a given field name. + + The field name *field_name* can be for instance "0.name" + or "lookup[3]". The *args* and *kwargs* arguments are + passed to get_value(). + """ first, rest = _string.formatter_field_name_split(field_name) - obj = self.get_value(first, args, kwargs) - # loop through the rest of the field_name, doing # getattr or getitem as needed for is_attr, i in rest: @@ -321,5 +313,4 @@ def get_field(self, field_name, args, kwargs): obj = getattr(obj, i) else: obj = obj[i] - return obj, first diff --git a/Lib/string/templatelib.py b/Lib/string/templatelib.py index 14b40e1e36e..8164872432a 100644 --- a/Lib/string/templatelib.py +++ b/Lib/string/templatelib.py @@ -1,15 +1,22 @@ """Support for template string literals (t-strings).""" -__all__ = [ - "Interpolation", - "Template", -] - t = t"{0}" Template = type(t) Interpolation = type(t.interpolations[0]) del t +def convert(obj, /, conversion): + """Convert *obj* using formatted string literal semantics.""" + if conversion is None: + return obj + if conversion == 'r': + return repr(obj) + if conversion == 's': + return str(obj) + if conversion == 'a': + return ascii(obj) + raise ValueError(f'invalid conversion specifier: {conversion}') + def _template_unpickle(*args): import itertools diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 54c2eb515b6..4d5ab6fbff0 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -380,8 +380,7 @@ def _text_encoding(): if sys.flags.utf8_mode: return "utf-8" - else: - return locale.getencoding() + return locale.getencoding() def call(*popenargs, timeout=None, **kwargs): @@ -1614,6 +1613,10 @@ def _readerthread(self, fh, buffer): fh.close() + def _writerthread(self, input): + self._stdin_write(input) + + def _communicate(self, input, endtime, orig_timeout): # Start reader threads feeding into a list hanging off of this # object, unless they've already been started. @@ -1632,8 +1635,23 @@ def _communicate(self, input, endtime, orig_timeout): self.stderr_thread.daemon = True self.stderr_thread.start() - if self.stdin: - self._stdin_write(input) + # Start writer thread to send input to stdin, unless already + # started. The thread writes input and closes stdin when done, + # or continues in the background on timeout. + if self.stdin and not hasattr(self, "_stdin_thread"): + self._stdin_thread = \ + threading.Thread(target=self._writerthread, + args=(input,)) + self._stdin_thread.daemon = True + self._stdin_thread.start() + + # Wait for the writer thread, or time out. If we time out, the + # thread remains writing and the fd left open in case the user + # calls communicate again. + if hasattr(self, "_stdin_thread"): + self._stdin_thread.join(self._remaining_time(endtime)) + if self._stdin_thread.is_alive(): + raise TimeoutExpired(self.args, orig_timeout) # Wait for the reader threads, or time out. If we time out, the # threads remain reading and the fds left open in case the user @@ -2078,6 +2096,10 @@ def _communicate(self, input, endtime, orig_timeout): self.stdin.flush() except BrokenPipeError: pass # communicate() must ignore BrokenPipeError. + except ValueError: + # ignore ValueError: I/O operation on closed file. + if not self.stdin.closed: + raise if not input: try: self.stdin.close() @@ -2103,10 +2125,13 @@ def _communicate(self, input, endtime, orig_timeout): self._save_input(input) if self._input: - input_view = memoryview(self._input) + if not isinstance(self._input, memoryview): + input_view = memoryview(self._input) + else: + input_view = self._input.cast("b") # byte input required with _PopenSelector() as selector: - if self.stdin and input: + if self.stdin and not self.stdin.closed and self._input: selector.register(self.stdin, selectors.EVENT_WRITE) if self.stdout and not self.stdout.closed: selector.register(self.stdout, selectors.EVENT_READ) @@ -2139,7 +2164,7 @@ def _communicate(self, input, endtime, orig_timeout): selector.unregister(key.fileobj) key.fileobj.close() else: - if self._input_offset >= len(self._input): + if self._input_offset >= len(input_view): selector.unregister(key.fileobj) key.fileobj.close() elif key.fileobj in (self.stdout, self.stderr): diff --git a/Lib/symtable.py b/Lib/symtable.py index 7a30e1ac4ca..4c832e68f94 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -17,13 +17,13 @@ __all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] -def symtable(code, filename, compile_type): +def symtable(code, filename, compile_type, *, module=None): """ Return the toplevel *SymbolTable* for the source code. *filename* is the name of the file with the code and *compile_type* is the *compile()* mode argument. """ - top = _symtable.symtable(code, filename, compile_type) + top = _symtable.symtable(code, filename, compile_type, module=module) return _newSymbolTable(top, filename) class SymbolTableFactory: @@ -255,11 +255,6 @@ def is_local_symbol(ident): if is_local_symbol(st.name): match st.type: case _symtable.TYPE_FUNCTION: - # generators are of type TYPE_FUNCTION with a ".0" - # parameter as a first parameter (which makes them - # distinguishable from a function named 'genexpr') - if st.name == 'genexpr' and '.0' in st.varnames: - continue d[st.name] = 1 case _symtable.TYPE_TYPE_PARAMETERS: # Get the function-def block in the annotation @@ -267,13 +262,6 @@ def is_local_symbol(ident): scope_name = st.name for c in st.children: if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: - # A generic generator of type TYPE_FUNCTION - # cannot be a direct child of 'st' (but it - # can be a descendant), e.g.: - # - # class A: - # type genexpr[genexpr] = (x for x in []) - assert scope_name != 'genexpr' or '.0' not in c.varnames d[scope_name] = 1 break self.__methods = tuple(d) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index dad715eb087..8ff9c99435b 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -219,18 +219,7 @@ def _safe_realpath(path): if "_PYTHON_PROJECT_BASE" in os.environ: _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) -def is_python_build(check_home=None): - if check_home is not None: - import warnings - warnings.warn( - ( - 'The check_home argument of sysconfig.is_python_build is ' - 'deprecated and its value is ignored. ' - 'It will be removed in Python 3.15.' - ), - DeprecationWarning, - stacklevel=2, - ) +def is_python_build(): for fn in ("Setup", "Setup.local"): if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): return True @@ -423,7 +412,13 @@ def _init_non_posix(vars): vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) - vars['TZPATH'] = '' + # No standard path exists on Windows for this, but we'll check + # whether someone is imitating a POSIX-like layout + check_tzpath = os.path.join(vars['prefix'], 'share', 'zoneinfo') + if os.path.exists(check_tzpath): + vars['TZPATH'] = check_tzpath + else: + vars['TZPATH'] = '' # # public APIs @@ -468,7 +463,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.dirname(sys._base_executable) + inc_dir = os.path.join(_PROJECT_BASE, 'PC') else: inc_dir = _PROJECT_BASE else: @@ -650,18 +645,25 @@ def get_platform(): isn't particularly important. Examples of returned values: - linux-i586 - linux-alpha (?) + linux-x86_64 + linux-aarch64 solaris-2.6-sun4u - Windows will return one of: - win-amd64 (64-bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) - win-arm64 (64-bit Windows on ARM64 (aka AArch64) - win32 (all others - specifically, sys.platform is returned) - For other non-POSIX platforms, currently just returns 'sys.platform'. + Windows: - """ + - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) + - win-arm64 (64-bit Windows on ARM64, aka AArch64) + - win32 (all others - specifically, sys.platform is returned) + + POSIX based OS: + + - linux-x86_64 + - macosx-15.5-arm64 + - macosx-26.0-universal2 (macOS on Apple Silicon or Intel) + - android-24-arm64_v8a + + For other non-POSIX platforms, currently just returns :data:`sys.platform`.""" if os.name == 'nt': if 'amd64' in sys.version.lower(): return 'win-amd64' diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index bc2197cfe79..0cf0cf4dbb9 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -21,8 +21,9 @@ # Regexes needed for parsing Makefile (and similar syntaxes, # like old-style Setup files). _variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" -_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" -_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" +_findvar_rx = (r"\$(\([A-Za-z][A-Za-z0-9_]*\)" + r"|\{[A-Za-z][A-Za-z0-9_]*\}" + r"|\$?)") def _parse_makefile(filename, vars=None, keep_unresolved=True): @@ -49,26 +50,7 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): m = re.match(_variable_rx, line) if m: n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - if n in _ALWAYS_STR: - raise ValueError - - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # do variable interpolation here - variables = list(notdone.keys()) + notdone[n] = v.strip() # Variables with a 'PY_' prefix in the makefile. These need to # be made available without that prefix through sysconfig. @@ -76,72 +58,64 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): # if the expansion uses the name without a prefix. renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - while len(variables) > 0: - for name in tuple(variables): - value = notdone[name] - m1 = re.search(_findvar1_rx, value) - m2 = re.search(_findvar2_rx, value) - if m1 and m2: - m = m1 if m1.start() < m2.start() else m2 - else: - m = m1 if m1 else m2 - if m is not None: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if (name.startswith('PY_') and - name[3:] in renamed_variables): - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - - else: - done[n] = item = "" - - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: - if name in _ALWAYS_STR: - raise ValueError - value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - variables.remove(name) - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - - else: - # Adds unresolved variables to the done dict. - # This is disabled when called from distutils.sysconfig + def resolve_var(name): + def repl(m): + n = m[1] + if n == '$': + return '$' + elif n == '': + # bogus variable reference (e.g. "prefix=$/opt/python") if keep_unresolved: - done[name] = value - # bogus variable reference (e.g. "prefix=$/opt/python"); - # just drop it since we can't deal - variables.remove(name) + return m[0] + raise ValueError + elif n[0] == '(' and n[-1] == ')': + n = n[1:-1] + elif n[0] == '{' and n[-1] == '}': + n = n[1:-1] + + if n in done: + return str(done[n]) + elif n in notdone: + return str(resolve_var(n)) + elif n in os.environ: + # do it like make: fall back to environment + return os.environ[n] + elif n in renamed_variables: + if name.startswith('PY_') and name[3:] in renamed_variables: + return "" + n = 'PY_' + n + if n in notdone: + return str(resolve_var(n)) + else: + assert n not in done + return "" + else: + done[n] = "" + return "" + + assert name not in done + done[name] = "" + try: + value = re.sub(_findvar_rx, repl, notdone[name]) + except ValueError: + del done[name] + return "" + value = value.strip() + if name not in _ALWAYS_STR: + try: + value = int(value) + except ValueError: + pass + done[name] = value + if name.startswith('PY_') and name[3:] in renamed_variables: + name = name[3:] + if name not in done: + done[name] = value + return value + + for n in notdone: + if n not in done: + resolve_var(n) # strip spurious spaces for k, v in done.items(): diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index c0097351b26..272e8e33e0c 100644 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -16,8 +16,6 @@ # XXX The API needs to undergo changes however; the current code is too # XXX script-like. This will be addressed later. -__version__ = "6" - import os import sys import tokenize @@ -334,5 +332,14 @@ def _process_tokens(tokens): raise NannyNag(start[0], msg, line) +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "6" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + if __name__ == '__main__': main() diff --git a/Lib/tarfile.py b/Lib/tarfile.py index c0f5a609b9f..7db3a40c9b3 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -67,7 +67,7 @@ "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", "tar_filter", "FilterError", "AbsoluteLinkError", "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", - "LinkOutsideDestinationError"] + "LinkOutsideDestinationError", "LinkFallbackError"] #--------------------------------------------------------- @@ -201,7 +201,6 @@ def itn(n, digits=8, format=DEFAULT_FORMAT): # base-256 representation. This allows values up to (256**(digits-1))-1. # A 0o200 byte indicates a positive number, a 0o377 byte a negative # number. - original_n = n n = int(n) if 0 <= n < 8 ** (digits - 1): s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL @@ -353,7 +352,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, fileobj = _StreamProxy(fileobj) comptype = fileobj.getcomptype() - self.name = name or "" + self.name = os.fspath(name) if name is not None else "" self.mode = mode self.comptype = comptype self.fileobj = fileobj @@ -766,10 +765,22 @@ def __init__(self, tarinfo, path): super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' + 'which is outside the destination') +class LinkFallbackError(FilterError): + def __init__(self, tarinfo, path): + self.tarinfo = tarinfo + self._path = path + super().__init__(f'link {tarinfo.name!r} would be extracted as a ' + + f'copy of {path!r}, which was rejected') + +# Errors caused by filters -- both "fatal" and "non-fatal" -- that +# we consider to be issues with the argument, rather than a bug in the +# filter function +_FILTER_ERRORS = (FilterError, OSError, ExtractError) + def _get_filtered_attrs(member, dest_path, for_data=True): new_attrs = {} name = member.name - dest_path = os.path.realpath(dest_path) + dest_path = os.path.realpath(dest_path, strict=os.path.ALLOW_MISSING) # Strip leading / (tar's directory separator) from filenames. # Include os.sep (target OS directory separator) as well. if name.startswith(('/', os.sep)): @@ -779,7 +790,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): # For example, 'C:/foo' on Windows. raise AbsolutePathError(member) # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) + target_path = os.path.realpath(os.path.join(dest_path, name), + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise OutsideDestinationError(member, target_path) # Limit permissions (no high bits, and go-w) @@ -817,6 +829,9 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + normalized = os.path.normpath(member.linkname) + if normalized != member.linkname: + new_attrs['linkname'] = normalized if member.issym(): target_path = os.path.join(dest_path, os.path.dirname(name), @@ -824,7 +839,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): else: target_path = os.path.join(dest_path, member.linkname) - target_path = os.path.realpath(target_path) + target_path = os.path.realpath(target_path, + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs @@ -1630,6 +1646,9 @@ def _block(self, count): """Round up a byte count by BLOCKSIZE and return it, e.g. _block(834) => 1024. """ + # Only non-negative offsets are allowed + if count < 0: + raise InvalidHeaderError("invalid offset") blocks, remainder = divmod(count, BLOCKSIZE) if remainder: blocks += 1 @@ -1910,7 +1929,7 @@ def not_compressed(comptype): if "preset" in kwargs and comptype not in ("xz",): raise ValueError("preset is only valid for w|xz mode") - compresslevel = kwargs.pop("compresslevel", 9) + compresslevel = kwargs.pop("compresslevel", 6) preset = kwargs.pop("preset", None) stream = _Stream(name, filemode, comptype, fileobj, bufsize, compresslevel, preset) @@ -1936,7 +1955,7 @@ def taropen(cls, name, mode="r", fileobj=None, **kwargs): return cls(name, mode, fileobj, **kwargs) @classmethod - def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): + def gzopen(cls, name, mode="r", fileobj=None, compresslevel=6, **kwargs): """Open gzip compressed tar archive name for reading or writing. Appending is not allowed. """ @@ -2065,7 +2084,7 @@ def zstopen(cls, name, mode="r", fileobj=None, level=None, options=None, "gz": "gzopen", # gzip compressed tar "bz2": "bz2open", # bzip2 compressed tar "xz": "xzopen", # lzma compressed tar - "zst": "zstopen" # zstd compressed tar + "zst": "zstopen", # zstd compressed tar } #-------------------------------------------------------------------------- @@ -2386,30 +2405,58 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, members = self for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is None: continue if tarinfo.isdir(): # For directories, delay setting attributes until later, # since permissions can interfere with extraction and # extracting contents can reset mtime. - directories.append(tarinfo) + directories.append(unfiltered) self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) + numeric_owner=numeric_owner, + filter_function=filter_function) # Reverse sort directories. directories.sort(key=lambda a: a.name, reverse=True) + # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) + for unfiltered in directories: try: + # Need to re-apply any filter, to take the *current* filesystem + # state into account. + try: + tarinfo = filter_function(unfiltered, path) + except _FILTER_ERRORS as exc: + self._log_no_directory_fixup(unfiltered, repr(exc)) + continue + if tarinfo is None: + self._log_no_directory_fixup(unfiltered, + 'excluded by filter') + continue + dirpath = os.path.join(path, tarinfo.name) + try: + lstat = os.lstat(dirpath) + except FileNotFoundError: + self._log_no_directory_fixup(tarinfo, 'missing') + continue + if not stat.S_ISDIR(lstat.st_mode): + # This is no longer a directory; presumably a later + # member overwrote the entry. + self._log_no_directory_fixup(tarinfo, 'not a directory') + continue self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: self._handle_nonfatal_error(e) + def _log_no_directory_fixup(self, member, reason): + self._dbg(2, "tarfile: Not fixing up directory %r (%s)" % + (member.name, reason)) + def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, @@ -2425,42 +2472,57 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, String names of common filters are accepted. """ filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is not None: self._extract_one(tarinfo, path, set_attrs, numeric_owner) def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" - if isinstance(member, str): - tarinfo = self.getmember(member) - else: - tarinfo = member + """Get (filtered, unfiltered) TarInfos from *member* - unfiltered = tarinfo + *member* might be a string. + + Return (None, None) if not found. + """ + + if isinstance(member, str): + unfiltered = self.getmember(member) + else: + unfiltered = member + + filtered = None try: - tarinfo = filter_function(tarinfo, path) - except (OSError, FilterError) as e: + filtered = filter_function(unfiltered, path) + except (OSError, UnicodeEncodeError, FilterError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) - if tarinfo is None: + if filtered is None: self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None - # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo + return None, None - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" + # Prepare the link target for makelink(). + if filtered.islnk(): + filtered = copy.copy(filtered) + filtered._link_target = os.path.join(path, filtered.linkname) + return filtered, unfiltered + + def _extract_one(self, tarinfo, path, set_attrs, numeric_owner, + filter_function=None): + """Extract from filtered tarinfo to disk. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) + """ self._check("r") try: self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, - numeric_owner=numeric_owner) - except OSError as e: + numeric_owner=numeric_owner, + filter_function=filter_function, + extraction_root=path) + except (OSError, UnicodeEncodeError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) @@ -2517,9 +2579,13 @@ def extractfile(self, member): return None def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical + numeric_owner=False, *, filter_function=None, + extraction_root=None): + """Extract the filtered TarInfo object tarinfo to a physical file called targetpath. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) """ # Fetch the TarInfo object for the given name # and build the destination pathname, replacing @@ -2548,7 +2614,10 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, elif tarinfo.ischr() or tarinfo.isblk(): self.makedev(tarinfo, targetpath) elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) + self.makelink_with_filter( + tarinfo, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) elif tarinfo.type not in SUPPORTED_TYPES: self.makeunknown(tarinfo, targetpath) else: @@ -2631,29 +2700,66 @@ def makedev(self, tarinfo, targetpath): os.makedev(tarinfo.devmajor, tarinfo.devminor)) def makelink(self, tarinfo, targetpath): + return self.makelink_with_filter(tarinfo, targetpath, None, None) + + def makelink_with_filter(self, tarinfo, targetpath, + filter_function, extraction_root): """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a link). """ + keyerror_to_extracterror = False try: # For systems that support symbolic and hard links. if tarinfo.issym(): if os.path.lexists(targetpath): # Avoid FileExistsError on following os.symlink. os.unlink(targetpath) - os.symlink(tarinfo.linkname, targetpath) + link_target = tarinfo.linkname + if os.name == "nt": + # gh-57911: Posix-flavoured forward-slash path separators in + # symlink targets aren't acknowledged by Windows, resulting + # in corrupted links. + link_target = link_target.replace("/", os.path.sep) + os.symlink(link_target, targetpath) + return else: if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + # Avoid FileExistsError on following os.link. + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) + return except symlink_exception: + keyerror_to_extracterror = True + + try: + unfiltered = self._find_link_target(tarinfo) + except KeyError: + if keyerror_to_extracterror: + raise ExtractError( + "unable to resolve link inside archive") from None + else: + raise + + if filter_function is None: + filtered = unfiltered + else: + if extraction_root is None: + raise ExtractError( + "makelink_with_filter: if filter_function is not None, " + + "extraction_root must also not be None") try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None + filtered = filter_function(unfiltered, extraction_root) + except _FILTER_ERRORS as cause: + raise LinkFallbackError(tarinfo, unfiltered.name) from cause + if filtered is not None: + self._extract_member(filtered, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) def chown(self, tarinfo, targetpath, numeric_owner): """Set owner of targetpath according to tarinfo. If numeric_owner diff --git a/Lib/tempfile.py b/Lib/tempfile.py index cadb0bed3cc..53d14ff5c67 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -180,7 +180,7 @@ def _candidate_tempdir_list(): return dirlist -def _get_default_tempdir(): +def _get_default_tempdir(dirlist=None): """Calculate the default directory to use for temporary files. This routine should be called exactly once. @@ -190,7 +190,8 @@ def _get_default_tempdir(): service, the name of the test file must be randomized.""" namer = _RandomNameSequence() - dirlist = _candidate_tempdir_list() + if dirlist is None: + dirlist = _candidate_tempdir_list() for dir in dirlist: if dir != _os.curdir: @@ -655,7 +656,7 @@ def TemporaryFile(mode='w+b', buffering=-1, encoding=None, fd = None def opener(*args): nonlocal fd - flags2 = (flags | _os.O_TMPFILE) & ~_os.O_CREAT + flags2 = (flags | _os.O_TMPFILE) & ~_os.O_CREAT & ~_os.O_EXCL fd = _os.open(dir, flags2, 0o600) return fd try: diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index a1eac32a83a..a1b749798fa 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -1,5 +1,7 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings +target-version = "py312" + extend-exclude = [ # Excluded (run with the other AC files in its own separate ruff job in pre-commit) "test_clinic.py", @@ -8,15 +10,28 @@ extend-exclude = [ # Non UTF-8 files "encoded_modules/module_iso_8859_1.py", "encoded_modules/module_koi8_r.py", - # SyntaxError because of t-strings - "test_tstring.py", - "test_string/test_templatelib.py", # New grammar constructions may not yet be recognized by Ruff, # and tests re-use the same names as only the grammar is being checked. "test_grammar.py", ] +[per-file-target-version] +# Type parameter defaults +"test_type_params.py" = "py313" + +# Template string literals +"test_annotationlib.py" = "py314" +"test_string/test_templatelib.py" = "py314" +"test_tstring.py" = "py314" + [lint] select = [ + "F401", # Unused import "F811", # Redefinition of unused variable (useful for finding test methods with the same name) ] + +[lint.per-file-ignores] +"*/**/__main__.py" = ["F401"] # Unused import +"test_import/*.py" = ["F401"] # Unused import +"test_importlib/*.py" = ["F401"] # Unused import +"typinganndata/partialexecution/*.py" = ["F401"] # Unused import diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py index c3daa0dccf5..70c44da2ec6 100644 --- a/Lib/test/_code_definitions.py +++ b/Lib/test/_code_definitions.py @@ -1,4 +1,32 @@ +def simple_script(): + assert True + + +def complex_script(): + obj = 'a string' + pickle = __import__('pickle') + def spam_minimal(): + pass + spam_minimal() + data = pickle.dumps(obj) + res = pickle.loads(data) + assert res == obj, (res, obj) + + +def script_with_globals(): + obj1, obj2 = spam(42) + assert obj1 == 42 + assert obj2 is None + + +def script_with_explicit_empty_return(): + return None + + +def script_with_return(): + return True + def spam_minimal(): # no arg defaults or kwarg defaults @@ -29,6 +57,22 @@ def spam_with_globals_and_builtins(): print(res) +def spam_with_global_and_attr_same_name(): + try: + spam_minimal.spam_minimal + except AttributeError: + pass + + +def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + +def spam_full_args_with_defaults(a=-1, b=-2, /, c=-3, d=-4, *args, + e=-5, f=-6, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + def spam_args_attrs_and_builtins(a, b, /, c, d, *args, e, f, **kwargs): if args.__len__() > 2: return None @@ -39,6 +83,10 @@ def spam_returns_arg(x): return x +def spam_raises(): + raise Exception('spam!') + + def spam_with_inner_not_closure(): def eggs(): pass @@ -141,11 +189,20 @@ def ham_C_closure(z): TOP_FUNCTIONS = [ # shallow + simple_script, + complex_script, + script_with_globals, + script_with_explicit_empty_return, + script_with_return, spam_minimal, spam_with_builtins, spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, + spam_full_args, + spam_full_args_with_defaults, spam_args_attrs_and_builtins, spam_returns_arg, + spam_raises, spam_with_inner_not_closure, spam_with_inner_closure, spam_annotated, @@ -178,6 +235,58 @@ def ham_C_closure(z): *NESTED_FUNCTIONS, ] +STATELESS_FUNCTIONS = [ + simple_script, + complex_script, + script_with_explicit_empty_return, + script_with_return, + spam, + spam_minimal, + spam_with_builtins, + spam_full_args, + spam_args_attrs_and_builtins, + spam_returns_arg, + spam_raises, + spam_annotated, + spam_with_inner_not_closure, + spam_with_inner_closure, + spam_N, + spam_C, + spam_NN, + spam_NC, + spam_CN, + spam_CC, + eggs_nested, + eggs_nested_N, + ham_nested, + ham_C_nested +] +STATELESS_CODE = [ + *STATELESS_FUNCTIONS, + script_with_globals, + spam_full_args_with_defaults, + spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, + spam_full, +] + +PURE_SCRIPT_FUNCTIONS = [ + simple_script, + complex_script, + script_with_explicit_empty_return, + spam_minimal, + spam_with_builtins, + spam_raises, + spam_with_inner_not_closure, + spam_with_inner_closure, +] +SCRIPT_FUNCTIONS = [ + *PURE_SCRIPT_FUNCTIONS, + script_with_globals, + spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, +] + # generators diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 0ce42276bfe..4a050792df7 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -380,6 +380,8 @@ def os_open(self, path): @unittest.skipIf(sys.platform == "darwin", "hangs under macOS; see bpo-25234, bpo-35363") + @unittest.skipIf(sys.platform.startswith('netbsd'), + "hangs on NetBSD; see gh-137397") def test_os_open(self): self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", self.os_open) diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 154662efce9..4cac84d7a46 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -11,7 +11,7 @@ def check_structseq(self, obj_type): # ob_refcnt self.assertGreaterEqual(sys.getrefcount(obj_type), 1) # tp_base - self.assertTrue(issubclass(obj_type, tuple)) + self.assertIsSubclass(obj_type, tuple) # tp_bases self.assertEqual(obj_type.__bases__, (tuple,)) # tp_dict diff --git a/Lib/test/_test_gc_fast_cycles.py b/Lib/test/_test_gc_fast_cycles.py new file mode 100644 index 00000000000..4e2c7d72a02 --- /dev/null +++ b/Lib/test/_test_gc_fast_cycles.py @@ -0,0 +1,48 @@ +# Run by test_gc. +from test import support +import _testinternalcapi +import gc +import unittest + +class IncrementalGCTests(unittest.TestCase): + + # Use small increments to emulate longer running process in a shorter time + @support.gc_threshold(200, 10) + def test_incremental_gc_handles_fast_cycle_creation(self): + + class LinkedList: + + #Use slots to reduce number of implicit objects + __slots__ = "next", "prev", "surprise" + + def __init__(self, next=None, prev=None): + self.next = next + if next is not None: + next.prev = self + self.prev = prev + if prev is not None: + prev.next = self + + def make_ll(depth): + head = LinkedList() + for i in range(depth): + head = LinkedList(head, head.prev) + return head + + head = make_ll(1000) + + assert(gc.isenabled()) + olds = [] + initial_heap_size = _testinternalcapi.get_tracked_heap_size() + for i in range(20_000): + newhead = make_ll(20) + newhead.surprise = head + olds.append(newhead) + if len(olds) == 20: + new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size + self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") + del olds[:] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4dc9a31d22f..d03eb1dfb25 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -39,7 +39,8 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper - +from test.support import subTests +from test.support.script_helper import assert_python_failure, assert_python_ok # Skip tests if _multiprocessing wasn't built. _multiprocessing = import_helper.import_module('_multiprocessing') @@ -325,6 +326,7 @@ def test_current(self): self.assertEqual(current.ident, os.getpid()) self.assertEqual(current.exitcode, None) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_set_executable(self): if self.TYPE == 'threads': self.skipTest(f'test not appropriate for {self.TYPE}') @@ -341,6 +343,7 @@ def test_set_executable(self): p.join() self.assertEqual(p.exitcode, 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_resource('cpu') def test_args_argument(self): # bpo-45735: Using list or tuple as *args* in constructor could @@ -388,6 +391,7 @@ def _test(cls, q, *args, **kwds): q.put(bytes(current.authkey)) q.put(current.pid) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_parent_process_attributes(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -408,6 +412,7 @@ def _test_send_parent_process(cls, wconn): from multiprocessing.process import parent_process wconn.send([parent_process().pid, parent_process().name]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_parent_process(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -446,6 +451,7 @@ def _test_report_parent_status(cls, wconn): parent_process().join(timeout=support.SHORT_TIMEOUT) wconn.send("alive" if parent_process().is_alive() else "not alive") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_process(self): q = self.Queue(1) e = self.Event() @@ -486,6 +492,7 @@ def test_process(self): self.assertNotIn(p, self.active_children()) close_queue(q) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(threading._HAVE_THREAD_NATIVE_ID, "needs native_id") def test_process_mainthread_native_id(self): if self.TYPE == 'threads': @@ -513,19 +520,28 @@ def _sleep_some(cls): time.sleep(100) @classmethod - def _sleep_no_int_handler(cls): + def _sleep_some_event(cls, event): + event.set() + time.sleep(100) + + @classmethod + def _sleep_no_int_handler(cls, event): signal.signal(signal.SIGINT, signal.SIG_DFL) - cls._sleep_some() + cls._sleep_some_event(event) @classmethod def _test_sleep(cls, delay): time.sleep(delay) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def _kill_process(self, meth, target=None): if self.TYPE == 'threads': self.skipTest('test not appropriate for {}'.format(self.TYPE)) - p = self.Process(target=target or self._sleep_some) + event = self.Event() + if not target: + target = self._sleep_some_event + p = self.Process(target=target, args=(event,)) p.daemon = True p.start() @@ -543,8 +559,11 @@ def _kill_process(self, meth, target=None): self.assertTimingAlmostEqual(join.elapsed, 0.0) self.assertEqual(p.is_alive(), True) - # XXX maybe terminating too soon causes the problems on Gentoo... - time.sleep(1) + timeout = support.SHORT_TIMEOUT + if not event.wait(timeout): + p.terminate() + p.join() + self.fail(f"event not signaled in {timeout} seconds") meth(p) @@ -572,6 +591,7 @@ def handler(*args): return p.exitcode + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipIf(os.name == 'nt', "POSIX only") def test_interrupt(self): exitcode = self._kill_process(multiprocessing.Process.interrupt) @@ -580,15 +600,18 @@ def test_interrupt(self): # (KeyboardInterrupt in this case) # in multiprocessing.BaseProcess._bootstrap + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipIf(os.name == 'nt', "POSIX only") def test_interrupt_no_handler(self): exitcode = self._kill_process(multiprocessing.Process.interrupt, target=self._sleep_no_int_handler) self.assertEqual(exitcode, -signal.SIGINT) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_terminate(self): exitcode = self._kill_process(multiprocessing.Process.terminate) self.assertEqual(exitcode, -signal.SIGTERM) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_kill(self): exitcode = self._kill_process(multiprocessing.Process.kill) if os.name != 'nt': @@ -604,6 +627,7 @@ def test_cpu_count(self): self.assertIsInstance(cpus, int) self.assertGreaterEqual(cpus, 1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_active_children(self): self.assertEqual(type(self.active_children()), list) @@ -632,6 +656,7 @@ def _test_recursion(cls, wconn, id): p.start() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_recursion(self): rconn, wconn = self.Pipe(duplex=False) self._test_recursion(wconn, []) @@ -656,6 +681,7 @@ def test_recursion(self): def _test_sentinel(cls, event): event.wait(10.0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_sentinel(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -678,6 +704,7 @@ def _test_close(cls, rc=0, q=None): q.get() sys.exit(rc) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_close(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -710,6 +737,7 @@ def test_close(self): close_queue(q) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_resource('walltime') def test_many_processes(self): if self.TYPE == 'threads': @@ -746,6 +774,7 @@ def test_many_processes(self): for p in procs: self.assertIn(p.exitcode, exitcodes) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_lose_target_ref(self): c = DummyCallable() wr = weakref.ref(c) @@ -808,6 +837,7 @@ def func2(): threading.Thread(target=func1).start() threading.Thread(target=func2, daemon=True).start() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_wait_for_threads(self): # A child process should wait for non-daemonic threads to end # before exiting @@ -832,6 +862,7 @@ def _test_error_on_stdio_flush(self, evt, break_std_streams={}): setattr(sys, stream_name, None) evt.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_error_on_stdio_flush_1(self): # Check that Process works with broken standard streams streams = [io.StringIO(), None] @@ -851,6 +882,7 @@ def test_error_on_stdio_flush_1(self): finally: setattr(sys, stream_name, old_stream) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_error_on_stdio_flush_2(self): # Same as test_error_on_stdio_flush_1(), but standard streams are # broken by the child process @@ -1001,6 +1033,7 @@ class _TestSubclassingProcess(BaseTestCase): ALLOWED_TYPES = ('processes',) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_subclassing(self): uppercaser = _UpperCaser() uppercaser.daemon = True @@ -1010,6 +1043,7 @@ def test_subclassing(self): uppercaser.stop() uppercaser.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_stderr_flush(self): # sys.stderr is flushed at process shutdown (issue #13812) if self.TYPE == "threads": @@ -1040,6 +1074,7 @@ def _test_sys_exit(cls, reason, testfn): sys.stderr = open(fd, 'w', encoding="utf-8", closefd=False) sys.exit(reason) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_sys_exit(self): # See Issue 13854 if self.TYPE == 'threads': @@ -1107,6 +1142,7 @@ def _test_put(cls, queue, child_can_start, parent_can_continue): queue.get() parent_can_continue.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_put(self): MAXSIZE = 6 queue = self.Queue(maxsize=MAXSIZE) @@ -1169,13 +1205,14 @@ def test_put(self): @classmethod def _test_get(cls, queue, child_can_start, parent_can_continue): child_can_start.wait() - #queue.put(1) + queue.put(1) queue.put(2) queue.put(3) queue.put(4) queue.put(5) parent_can_continue.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_get(self): queue = self.Queue() child_can_start = self.Event() @@ -1193,15 +1230,16 @@ def test_get(self): child_can_start.set() parent_can_continue.wait() - time.sleep(DELTA) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if not queue_empty(queue): + break self.assertEqual(queue_empty(queue), False) - # Hangs unexpectedly, remove for now - #self.assertEqual(queue.get(), 1) + self.assertEqual(queue.get_nowait(), 1) self.assertEqual(queue.get(True, None), 2) self.assertEqual(queue.get(True), 3) self.assertEqual(queue.get(timeout=1), 4) - self.assertEqual(queue.get_nowait(), 5) + self.assertEqual(queue.get(), 5) self.assertEqual(queue_empty(queue), True) @@ -1237,6 +1275,7 @@ def _test_fork(cls, queue): # process cannot shutdown until the feeder thread has finished # pushing items onto the pipe. + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_fork(self): # Old versions of Queue would fail to create a new feeder # thread for a forked process if the original process had its @@ -1287,6 +1326,7 @@ def _test_task_done(cls, q): time.sleep(DELTA) q.task_done() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_task_done(self): queue = self.JoinableQueue() @@ -1330,6 +1370,7 @@ def test_no_import_lock_contention(self): self.fail("Probable regression on import lock contention;" " see Issue #22853") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_timeout(self): q = multiprocessing.Queue() start = time.monotonic() @@ -1453,6 +1494,7 @@ def _acquire_event(lock, event): event.set() time.sleep(1.0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_repr_lock(self): if self.TYPE != 'processes': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -1516,6 +1558,7 @@ def _test_lock_locked_2processes(cls, lock, event, res): res.value = lock.locked() event.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') def test_lock_locked_2processes(self): if self.TYPE != 'processes': @@ -1542,6 +1585,7 @@ def _acquire_release(lock, timeout, l=None, n=1): for _ in range(n): lock.release() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_repr_rlock(self): if self.TYPE != 'processes': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -1601,6 +1645,7 @@ def test_rlock(self): self.assertFalse(lock.locked()) self.assertRaises((AssertionError, RuntimeError), lock.release) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') def test_rlock_locked_2processes(self): if self.TYPE != 'processes': @@ -1712,6 +1757,7 @@ def check_invariant(self, cond): except NotImplementedError: pass + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_notify(self): cond = self.Condition() sleeping = self.Semaphore(0) @@ -1754,6 +1800,7 @@ def test_notify(self): threading_helper.join_thread(t) join_process(p) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_notify_all(self): cond = self.Condition() sleeping = self.Semaphore(0) @@ -1823,6 +1870,7 @@ def test_notify_all(self): # NOTE: join_process and join_thread are the same threading_helper.join_thread(w) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_notify_n(self): cond = self.Condition() sleeping = self.Semaphore(0) @@ -1896,6 +1944,7 @@ def _test_waitfor_f(cls, cond, state): if not result or state.value != 4: sys.exit(1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') def test_waitfor(self): # based on test in test/lock_tests.py @@ -1931,6 +1980,7 @@ def _test_waitfor_timeout_f(cls, cond, state, success, sem): if not result and (expected - CLOCK_RES) <= dt: success.value = True + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') def test_waitfor_timeout(self): # based on test in test/lock_tests.py @@ -1963,6 +2013,7 @@ def _test_wait_result(cls, c, pid): if pid is not None: os.kill(pid, signal.SIGINT) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_wait_result(self): if isinstance(self, ProcessesMixin) and sys.platform != 'win32': pid = os.getpid() @@ -1991,6 +2042,7 @@ def _test_event(cls, event): time.sleep(TIMEOUT2) event.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_event(self): event = self.Event() wait = TimingWrapper(event.wait) @@ -2187,6 +2239,7 @@ def multipass(cls, barrier, results, n): pass assert not barrier.broken + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_barrier(self, passes=1): """ Test that a barrier is passed in lockstep @@ -2194,6 +2247,7 @@ def test_barrier(self, passes=1): results = [self.DummyList(), self.DummyList()] self.run_threads(self.multipass, (self.barrier, results, passes)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_barrier_10(self): """ Test that a barrier works for 10 consecutive runs @@ -2205,6 +2259,7 @@ def _test_wait_return_f(cls, barrier, queue): res = barrier.wait() queue.put(res) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_wait_return(self): """ test the return value from barrier.wait @@ -2221,6 +2276,7 @@ def _test_action_f(cls, barrier, results): if len(results) != 1: raise RuntimeError + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_action(self): """ Test the 'action' callback @@ -2243,6 +2299,7 @@ def _test_abort_f(cls, barrier, results1, results2): except RuntimeError: barrier.abort() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_abort(self): """ Test that an abort will put the barrier in a broken state @@ -2273,6 +2330,7 @@ def _test_reset_f(cls, barrier, results1, results2, results3): barrier.wait() results3.append(True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_reset(self): """ Test that a 'reset' on a barrier frees the waiting threads @@ -2308,6 +2366,7 @@ def _test_abort_and_reset_f(cls, barrier, barrier2, barrier.wait() results3.append(True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_abort_and_reset(self): """ Test that a barrier can be reset after being broken. @@ -2334,6 +2393,7 @@ def _test_timeout_f(cls, barrier, results): except threading.BrokenBarrierError: results.append(True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_timeout(self): """ Test wait(timeout) @@ -2353,6 +2413,7 @@ def _test_default_timeout_f(cls, barrier, results): except threading.BrokenBarrierError: results.append(True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_default_timeout(self): """ Test the barrier's default timeout @@ -2374,6 +2435,7 @@ def _test_thousand_f(cls, barrier, passes, conn, lock): with lock: conn.send(i) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_thousand(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -2415,7 +2477,7 @@ def _test(cls, values): for sv, cv in zip(values, cls.codes_values): sv.value = cv[2] - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_value(self, raw=False): if raw: values = [self.RawValue(code, value) @@ -2463,6 +2525,12 @@ def test_getobj_getlock(self): self.assertNotHasAttr(arr5, 'get_lock') self.assertNotHasAttr(arr5, 'get_obj') + @unittest.skipIf(c_int is None, "requires _ctypes") + def test_invalid_typecode(self): + with self.assertRaisesRegex(TypeError, 'bad typecode'): + self.Value('x', None) + with self.assertRaisesRegex(TypeError, 'bad typecode'): + self.RawValue('x', None) class _TestArray(BaseTestCase): @@ -2473,6 +2541,7 @@ def f(cls, seq): for i in range(1, len(seq)): seq[i] += seq[i-1] + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipIf(c_int is None, "requires _ctypes") def test_array(self, raw=False): seq = [680, 626, 934, 821, 150, 233, 548, 982, 714, 831] @@ -2543,6 +2612,12 @@ def test_getobj_getlock_obj(self): self.assertNotHasAttr(arr5, 'get_lock') self.assertNotHasAttr(arr5, 'get_obj') + @unittest.skipIf(c_int is None, "requires _ctypes") + def test_invalid_typecode(self): + with self.assertRaisesRegex(TypeError, 'bad typecode'): + self.Array('x', []) + with self.assertRaisesRegex(TypeError, 'bad typecode'): + self.RawArray('x', []) # # # @@ -2778,8 +2853,9 @@ class _TestPool(BaseTestCase): @classmethod def setUpClass(cls): - super().setUpClass() - cls.pool = cls.Pool(4) + with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): + super().setUpClass() + cls.pool = cls.Pool(4) @classmethod def tearDownClass(cls): @@ -2878,6 +2954,7 @@ def test_async(self): self.assertEqual(get(), 49) self.assertTimingAlmostEqual(get.elapsed, TIMEOUT1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_async_timeout(self): p = self.Pool(3) try: @@ -2975,6 +3052,7 @@ def test_imap_unordered_handle_iterable_exception(self): self.assertIn(value, expected_values) expected_values.remove(value) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_make_pool(self): expected_error = (RemoteError if self.TYPE == 'manager' else ValueError) @@ -2990,6 +3068,7 @@ def test_make_pool(self): p.close() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_terminate(self): # Simulate slow tasks which take "forever" to complete sleep_time = support.LONG_TIMEOUT @@ -3007,6 +3086,7 @@ def test_terminate(self): p.terminate() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_empty_iterable(self): # See Issue 12157 p = self.Pool(1) @@ -3019,6 +3099,7 @@ def test_empty_iterable(self): p.close() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_context(self): if self.TYPE == 'processes': L = list(range(10)) @@ -3033,6 +3114,7 @@ def test_context(self): def _test_traceback(cls): raise RuntimeError(123) # some comment + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_traceback(self): # We want ensure that the traceback from the child process is # contained in the traceback raised in the main process. @@ -3072,9 +3154,11 @@ def test_traceback(self): p.join() @classmethod + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def _test_wrapped_exception(cls): raise RuntimeError('foo') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_wrapped_exception(self): # Issue #20980: Should not wrap exception when using thread pool with self.Pool(1) as p: @@ -3082,6 +3166,7 @@ def test_wrapped_exception(self): p.apply(self._test_wrapped_exception) p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_no_failfast(self): # Issue #23992: the fail-fast behaviour when an exception is raised # during map() would make Pool.join() deadlock, because a worker @@ -3117,6 +3202,7 @@ def test_release_task_refs(self): # they were released too. self.assertEqual(CountedObject.n_instances, 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_enter(self): if self.TYPE == 'manager': self.skipTest("test not applicable to manager") @@ -3133,6 +3219,7 @@ def test_enter(self): pass pool.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_resource_warning(self): if self.TYPE == 'manager': self.skipTest("test not applicable to manager") @@ -3158,6 +3245,7 @@ def unpickleable_result(): class _TestPoolWorkerErrors(BaseTestCase): ALLOWED_TYPES = ('processes', ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_async_error_callback(self): p = multiprocessing.Pool(2) @@ -3173,6 +3261,7 @@ def errback(exc): p.close() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_unpickleable_result(self): from multiprocessing.pool import MaybeEncodingError p = multiprocessing.Pool(2) @@ -3198,6 +3287,7 @@ def errback(exc): class _TestPoolWorkerLifetime(BaseTestCase): ALLOWED_TYPES = ('processes', ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_pool_worker_lifetime(self): p = multiprocessing.Pool(3, maxtasksperchild=10) self.assertEqual(3, len(p._pool)) @@ -3227,6 +3317,7 @@ def test_pool_worker_lifetime(self): p.close() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_pool_worker_lifetime_early_close(self): # Issue #10332: closing a pool whose workers have limited lifetimes # before all the tasks completed would make join() hang. @@ -3300,6 +3391,7 @@ class _TestMyManager(BaseTestCase): ALLOWED_TYPES = ('manager',) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_mymanager(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) manager.start() @@ -3311,6 +3403,7 @@ def test_mymanager(self): # which happens on slow buildbots. self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_mymanager_context(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) with manager: @@ -3320,6 +3413,7 @@ def test_mymanager_context(self): # which happens on slow buildbots. self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_mymanager_context_prestarted(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) manager.start() @@ -3390,6 +3484,7 @@ def _putter(cls, address, authkey): # Note that xmlrpclib will deserialize object as a list not a tuple queue.put(tuple(cls.values)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_remote(self): authkey = os.urandom(32) @@ -3431,6 +3526,7 @@ def _putter(cls, address, authkey): queue = manager.get_queue() queue.put('hello world') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_rapid_restart(self): authkey = os.urandom(32) manager = QueueManager( @@ -3483,12 +3579,14 @@ def recv(self): class TestManagerExceptions(unittest.TestCase): # Issue 106558: Manager exceptions avoids creating cyclic references. def setUp(self): - self.mgr = multiprocessing.Manager() + with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): + self.mgr = multiprocessing.Manager() def tearDown(self): self.mgr.shutdown() self.mgr.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_queue_get(self): queue = self.mgr.Queue() if gc.isenabled(): @@ -3500,6 +3598,7 @@ def test_queue_get(self): wr = weakref.ref(e) self.assertEqual(wr(), None) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_dispatch(self): if gc.isenabled(): gc.disable() @@ -3526,6 +3625,7 @@ def _echo(cls, conn): conn.send_bytes(msg) conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_connection(self): conn, child_conn = self.Pipe() @@ -3618,6 +3718,7 @@ def test_duplex_false(self): self.assertRaises(OSError, writer.recv) self.assertRaises(OSError, writer.poll) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_spawn_close(self): # We test that a pipe connection can be closed by parent # process immediately after child is spawned. On Windows this @@ -3694,6 +3795,7 @@ def _writefd(cls, conn, data, create_dummy_fds=False): os.write(fd, data) os.close(fd) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction") def test_fd_transfer(self): if self.TYPE != 'processes': @@ -3713,6 +3815,7 @@ def test_fd_transfer(self): with open(os_helper.TESTFN, "rb") as f: self.assertEqual(f.read(), b"foo") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction") @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") @@ -3750,6 +3853,7 @@ def test_large_fd_transfer(self): def _send_data_without_fd(self, conn): os.write(conn.fileno(), b"\0") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction") @unittest.skipIf(sys.platform == "win32", "doesn't make sense on Windows") def test_missing_fd_transfer(self): @@ -3849,6 +3953,7 @@ def _test(cls, address): conn.send('hello') conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_listener_client(self): for family in self.connection.families: l = self.connection.Listener(family=family) @@ -3860,6 +3965,7 @@ def test_listener_client(self): p.join() l.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_issue14725(self): l = self.connection.Listener() p = self.Process(target=self._test, args=(l.address,)) @@ -3905,6 +4011,7 @@ def _child_strings(cls, conn, strings): conn.send_bytes(s) conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_strings(self): strings = (b'hello', b'', b'a', b'b', b'', b'bye', b'', b'lop') a, b = self.Pipe() @@ -3928,6 +4035,7 @@ def _child_boundaries(cls, r): # read from it. r.poll(5) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_boundaries(self): r, w = self.Pipe(False) p = self.Process(target=self._child_boundaries, args=(r,)) @@ -3946,6 +4054,7 @@ def _child_dont_merge(cls, b): b.send_bytes(b'b') b.send_bytes(b'cd') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_dont_merge(self): a, b = self.Pipe() self.assertEqual(a.poll(0.0), False) @@ -4014,6 +4123,7 @@ def _remote(cls, conn): conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_pickling(self): families = self.connection.families @@ -4072,6 +4182,7 @@ def child_access(cls, conn): conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_access(self): # On Windows, if we do not specify a destination pid when # using DupHandle then we need to be careful to use the @@ -4235,6 +4346,7 @@ def _double(cls, x, y, z, foo, arr, string): for i in range(len(arr)): arr[i] *= 2 + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_sharedctypes(self, lock=False): x = Value('i', 7, lock=lock) y = Value(c_double, 1.0/3.0, lock=lock) @@ -4272,6 +4384,19 @@ def test_copy(self): self.assertEqual(bar.z, 2 ** 33) +def resource_tracker_format_subtests(func): + """Run given test using both resource tracker communication formats""" + def _inner(self, *args, **kwargs): + tracker = resource_tracker._resource_tracker + for use_simple_format in False, True: + with ( + self.subTest(use_simple_format=use_simple_format), + unittest.mock.patch.object( + tracker, '_use_simple_format', use_simple_format) + ): + func(self, *args, **kwargs) + return _inner + @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") @hashlib_helper.requires_hashdigest('sha256') class _TestSharedMemory(BaseTestCase): @@ -4496,6 +4621,7 @@ def test_shared_memory_pickle_unpickle_dead_object(self): with self.assertRaises(FileNotFoundError): pickle.loads(pickled_sms) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_shared_memory_across_processes(self): # bpo-40135: don't define shared memory block's name in case of # the failure when we run multiprocessing tests in parallel. @@ -4524,6 +4650,7 @@ def test_shared_memory_across_processes(self): sms.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms") def test_shared_memory_SharedMemoryServer_ignores_sigint(self): # bpo-36368: protect SharedMemoryManager server process from @@ -4549,6 +4676,7 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self): smm.shutdown() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # bpo-36867: test that a SharedMemoryManager uses the # same resource_tracker process as its parent. @@ -4569,6 +4697,7 @@ def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # properly released sl. self.assertFalse(err) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_shared_memory_SharedMemoryManager_basics(self): smm1 = multiprocessing.managers.SharedMemoryManager() with self.assertRaises(ValueError): @@ -4799,6 +4928,7 @@ def test_shared_memory_cleaned_after_process_termination(self): "shared_memory objects to clean up at shutdown", err) @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=False, it must not cause the memory to be deleted @@ -4826,6 +4956,7 @@ def test_shared_memory_untracking(self): mem.close() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_tracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=True, it must cause the memory to be deleted when @@ -4908,6 +5039,7 @@ class Foo(object): conn.close() os._exit(0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_finalize(self): conn, child_conn = self.Pipe() @@ -5035,6 +5167,7 @@ def _test_level(cls, conn): logger = multiprocessing.get_logger() conn.send(logger.getEffectiveLevel()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_level(self): LEVEL1 = 32 LEVEL2 = 37 @@ -5119,6 +5252,7 @@ def _killer(cls, pid): time.sleep(0.1) os.kill(pid, signal.SIGUSR1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') def test_poll_eintr(self): got_signal = [False] @@ -5162,6 +5296,23 @@ def test_invalid_handles(self): multiprocessing.connection.Connection, -1) +# +# Regression tests for BaseProcess kwargs handling +# + +class TestBaseProcessKwargs(unittest.TestCase): + def test_default_kwargs_not_shared_between_instances(self): + # Creating multiple Process instances without passing kwargs + # must create independent empty dicts (no shared state). + p1 = multiprocessing.Process(target=lambda: None) + p2 = multiprocessing.Process(target=lambda: None) + self.assertIsInstance(p1._kwargs, dict) + self.assertIsInstance(p2._kwargs, dict) + self.assertIsNot(p1._kwargs, p2._kwargs) + # Mutating one should not affect the other + p1._kwargs['x'] = 1 + self.assertNotIn('x', p2._kwargs) + @hashlib_helper.requires_hashdigest('sha256') class OtherTest(unittest.TestCase): @@ -5241,14 +5392,16 @@ def initializer(ns): @hashlib_helper.requires_hashdigest('sha256') class TestInitializers(unittest.TestCase): def setUp(self): - self.mgr = multiprocessing.Manager() - self.ns = self.mgr.Namespace() - self.ns.test = 0 + with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): + self.mgr = multiprocessing.Manager() + self.ns = self.mgr.Namespace() + self.ns.test = 0 def tearDown(self): self.mgr.shutdown() self.mgr.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_manager_initializer(self): m = multiprocessing.managers.SyncManager() self.assertRaises(TypeError, m.start, 1) @@ -5257,6 +5410,7 @@ def test_manager_initializer(self): m.shutdown() m.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_pool_initializer(self): self.assertRaises(TypeError, multiprocessing.Pool, initializer=1) p = multiprocessing.Pool(1, initializer, (self.ns,)) @@ -5314,16 +5468,19 @@ def flush(self): class TestStdinBadfiledescriptor(unittest.TestCase): + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_queue_in_process(self): proc = multiprocessing.Process(target=_test_process) proc.start() proc.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_pool_in_process(self): p = multiprocessing.Process(target=pool_in_process) p.start() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_flushing(self): sio = io.StringIO() flike = _file_like(sio) @@ -5343,6 +5500,7 @@ def _child_test_wait(cls, w, slow): w.send((i, os.getpid())) w.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_wait(self, slow=False): from multiprocessing.connection import wait readers = [] @@ -5383,6 +5541,7 @@ def _child_test_wait_socket(cls, address, slow): s.sendall(('%s\n' % i).encode('ascii')) s.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_wait_socket(self, slow=False): from multiprocessing.connection import wait l = socket.create_server((socket_helper.HOST, 0)) @@ -5447,6 +5606,7 @@ def signal_and_sleep(cls, sem, period): sem.release() time.sleep(period) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_resource('walltime') def test_wait_integer(self): from multiprocessing.connection import wait @@ -5491,6 +5651,7 @@ def test_wait_integer(self): p.terminate() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_neg_timeout(self): from multiprocessing.connection import wait a, b = multiprocessing.Pipe() @@ -5568,6 +5729,7 @@ def _test_timeout(cls, child, address): conn.send(456) conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_timeout(self): old_timeout = socket.getdefaulttimeout() try: @@ -5625,6 +5787,7 @@ def child(cls, n, conn): conn.send(len(util._afterfork_registry)) conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_lock(self): r, w = multiprocessing.Pipe(False) l = util.ForkAwareThreadLock() @@ -5676,6 +5839,7 @@ def _test_closefds(cls, conn, fd): s.close() conn.send(None) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_closefd(self): if not HAS_REDUCTION: raise unittest.SkipTest('requires fd pickling') @@ -5721,6 +5885,7 @@ def handler(signum, frame): conn.send(x) conn.send_bytes(b'x' * cls.CONN_MAX_SIZE) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') def test_ignore(self): conn, child_conn = multiprocessing.Pipe() @@ -5754,6 +5919,7 @@ def handler(signum, frame): a = l.accept() a.send('welcome') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') def test_ignore_listener(self): conn, child_conn = multiprocessing.Pipe() @@ -5788,6 +5954,7 @@ def check_context(self, ctx): p.join() self.assertEqual(child_method, ctx.get_start_method()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_context(self): for method in ('fork', 'spawn', 'forkserver'): try: @@ -5808,6 +5975,7 @@ def test_context_check_module_types(self): with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'): ctx.set_forkserver_preload([1, 2, 3]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_set_get(self): multiprocessing.set_forkserver_preload(PRELOAD) count = 0 @@ -5865,6 +6033,7 @@ def test_preload_resources(self): print(err) self.fail("failed spawning forkserver or grandchild") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipIf(sys.platform == "win32", "Only Spawn on windows so no risk of mixing") @only_run_in_spawn_testsuite("avoids redundant testing.") @@ -5898,6 +6067,7 @@ def _put_two_and_nest_once(cls, queue): process.start() process.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_nested_startmethod(self): # gh-108520: Regression test to ensure that child process can send its # arguments to another process @@ -6053,6 +6223,7 @@ def _is_resource_tracker_reused(conn, pid): reused &= _resource_tracker._check_alive() conn.send(reused) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_resource_tracker_reused(self): from multiprocessing.resource_tracker import _resource_tracker _resource_tracker.ensure_running() @@ -6154,6 +6325,7 @@ def test_empty_exceptions(self): with self.assertRaisesRegex(OSError, 'is closed'): q.empty() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_empty(self): queue = multiprocessing.SimpleQueue() child_can_start = multiprocessing.Event() @@ -6261,7 +6433,8 @@ def test_list(self): def setUp(self): self.manager = self.manager_class() - self.manager.start() + with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): + self.manager.start() self.proc = None def tearDown(self): @@ -6310,6 +6483,7 @@ def _test_event(cls, obj): obj.clear() obj.wait(0.001) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_event(self): o = self.manager.Event() o.set() @@ -6322,6 +6496,7 @@ def _test_lock(cls, obj): obj.acquire() obj.locked() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_lock(self, lname="Lock"): o = getattr(self.manager, lname)() self.run_worker(self._test_lock, o) @@ -6334,6 +6509,7 @@ def _test_rlock(cls, obj): obj.release() obj.locked() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_rlock(self, lname="RLock"): o = getattr(self.manager, lname)() self.run_worker(self._test_rlock, o) @@ -6342,6 +6518,7 @@ def test_rlock(self, lname="RLock"): def _test_semaphore(cls, obj): obj.acquire() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_semaphore(self, sname="Semaphore"): o = getattr(self.manager, sname)() self.run_worker(self._test_semaphore, o) @@ -6355,6 +6532,7 @@ def _test_condition(cls, obj): obj.acquire() obj.release() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_condition(self): o = self.manager.Condition() self.run_worker(self._test_condition, o) @@ -6364,6 +6542,7 @@ def _test_barrier(cls, obj): assert obj.parties == 5 obj.reset() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_barrier(self): o = self.manager.Barrier(5) self.run_worker(self._test_barrier, o) @@ -6374,6 +6553,7 @@ def _test_pool(cls, obj): with obj: pass + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_pool(self): o = self.manager.Pool(processes=4) self.run_worker(self._test_pool, o) @@ -6388,6 +6568,7 @@ def _test_queue(cls, obj): assert obj.get() == 6 assert obj.empty() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_queue(self, qname="Queue"): o = getattr(self.manager, qname)(2) o.put(5) @@ -6396,6 +6577,7 @@ def test_queue(self, qname="Queue"): assert o.empty() assert not o.full() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_joinable_queue(self): self.test_queue("JoinableQueue") @@ -6430,6 +6612,7 @@ def _test_list(cls, obj): obj.clear() case.assertEqual(len(obj), 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_list(self): o = self.manager.list() o.append(5) @@ -6471,6 +6654,7 @@ def _test_dict(cls, obj): obj.clear() case.assertEqual(len(obj), 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_dict(self): o = self.manager.dict() o['foo'] = 5 @@ -6485,6 +6669,7 @@ def _test_value(cls, obj): case.assertEqual(obj.get(), 1) obj.set(2) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_value(self): o = self.manager.Value('i', 1) self.run_worker(self._test_value, o) @@ -6499,6 +6684,7 @@ def _test_array(cls, obj): case.assertEqual(len(obj), 2) case.assertListEqual(list(obj), [0, 1]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_array(self): o = self.manager.Array('i', [0, 1]) self.run_worker(self._test_array, o) @@ -6509,6 +6695,7 @@ def _test_namespace(cls, obj): case.assertEqual(obj.x, 0) case.assertEqual(obj.y, 1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_namespace(self): o = self.manager.Namespace() o.x = 0 @@ -6628,6 +6815,7 @@ def _test_set_comparisons(cls, obj): case.assertGreater(obj, {'a'}) case.assertGreaterEqual(obj, {'a', 'b'}) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_set(self): o = self.manager.set() self.run_worker(self._test_set_operator_symbols, o) @@ -6645,6 +6833,7 @@ def test_set_init(self): self.assertSetEqual(o, {"a", "b", "c"}) self.assertRaises(RemoteError, self.manager.set, 1234) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_set_contain_all_method(self): o = self.manager.set() set_methods = { @@ -6698,6 +6887,7 @@ def exit_handler(): f.write("deadbeef") atexit.register(exit_handler) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_atexit(self): # gh-83856 with os_helper.temp_dir() as temp_dir: @@ -6778,6 +6968,20 @@ def test_child_sys_path(self): self.assertEqual(child_sys_path[1:], sys.path[1:]) self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}") + def test_std_streams_flushed_after_preload(self): + # gh-135335: Check fork server flushes standard streams after + # preloading modules + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_flush.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + + # Check stderr first, as it is more likely to be useful to see in the + # event of a failure. + self.assertEqual(err.decode().rstrip(), '__main____mp_main__') + self.assertEqual(out.decode().rstrip(), '__main____mp_main__') + class MiscTestCase(unittest.TestCase): def test__all__(self): @@ -6821,6 +7025,41 @@ def f(x): return x*x self.assertEqual("332833500", out.decode('utf-8').strip()) self.assertFalse(err, msg=err.decode('utf-8')) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_forked_thread_not_started(self): + # gh-134381: Ensure that a thread that has not been started yet in + # the parent process can be started within a forked child process. + + if multiprocessing.get_start_method() != "fork": + self.skipTest("fork specific test") + + q = multiprocessing.Queue() + t = threading.Thread(target=lambda: q.put("done"), daemon=True) + + def child(): + t.start() + t.join() + + p = multiprocessing.Process(target=child) + p.start() + p.join(support.SHORT_TIMEOUT) + + self.assertEqual(p.exitcode, 0) + self.assertEqual(q.get_nowait(), "done") + close_queue(q) + + def test_preload_main(self): + # gh-126631: Check that __main__ can be pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_main.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + self.assertEqual(err, b'') + + # The trailing empty string comes from split() on output ending with \n + out = out.decode().split("\n") + self.assertEqual(out, ['__main__', '__mp_main__', 'f', 'f', '']) # # Mixins @@ -6900,8 +7139,9 @@ def Pool(cls, *args, **kwds): @classmethod def setUpClass(cls): - super().setUpClass() - cls.manager = multiprocessing.Manager() + with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): + super().setUpClass() + cls.manager = multiprocessing.Manager() @classmethod def tearDownClass(cls): @@ -7068,3 +7308,110 @@ class SemLock(_multiprocessing.SemLock): name = f'test_semlock_subclass-{os.getpid()}' s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) + + +@unittest.skipIf(sys.platform != "linux", "Linux only") +class ForkInThreads(unittest.TestCase): + + def test_fork(self): + code = """ + import os, sys, threading, time + + t = threading.Thread(target=time.sleep, args=(1,), daemon=True) + t.start() + + assert threading.active_count() == 2 + + pid = os.fork() + if pid < 0: + print("Fork failed") + elif pid == 0: + print("In child") + sys.exit(0) + print("In parent") + """ + + res = assert_python_ok("-c", code, PYTHONWARNINGS='always') + self.assertIn(b'In child', res.out) + self.assertIn(b'In parent', res.out) + self.assertIn(b'DeprecationWarning', res.err) + self.assertIn(b'is multi-threaded, use of fork() may lead to deadlocks in the child', res.err) + + res = assert_python_failure("-c", code, PYTHONWARNINGS='error') + self.assertIn(b'DeprecationWarning', res.err) + self.assertIn(b'is multi-threaded, use of fork() may lead to deadlocks in the child', res.err) + + def test_forkpty(self): + code = """ + import os, sys, threading, time + + t = threading.Thread(target=time.sleep, args=(1,), daemon=True) + t.start() + + assert threading.active_count() == 2 + + pid, _ = os.forkpty() + if pid < 0: + print(f"forkpty failed") + elif pid == 0: + print(f"In child") + sys.exit(0) + print(f"In parent") + """ + + res = assert_python_ok("-c", code, PYTHONWARNINGS='always') + self.assertIn(b'In parent', res.out) + self.assertIn(b'DeprecationWarning', res.err) + self.assertIn(b'is multi-threaded, use of forkpty() may lead to deadlocks in the child', res.err) + + res = assert_python_failure("-c", code, PYTHONWARNINGS='error') + self.assertIn(b'DeprecationWarning', res.err) + self.assertIn(b'is multi-threaded, use of forkpty() may lead to deadlocks in the child', res.err) + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +class TestSharedMemoryNames(unittest.TestCase): + @subTests('use_simple_format', (True, False)) + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors( + self, use_simple_format): + # Test script that creates and cleans up shared memory with colon in name + test_script = textwrap.dedent(""" + import sys + from multiprocessing import shared_memory + from multiprocessing import resource_tracker + import time + + resource_tracker._resource_tracker._use_simple_format = %s + + # Test various patterns of colons in names + test_names = [ + "a:b", + "a:b:c", + "test:name:with:many:colons", + ":starts:with:colon", + "ends:with:colon:", + "::double::colons::", + "name\\nwithnewline", + "name-with-trailing-newline\\n", + "\\nname-starts-with-newline", + "colons:and\\nnewlines:mix", + "multi\\nline\\nname", + ] + + for name in test_names: + try: + shm = shared_memory.SharedMemory(create=True, size=100, name=name) + shm.buf[:5] = b'hello' # Write something to the shared memory + shm.close() + shm.unlink() + + except Exception as e: + print(f"Error with name '{name}': {e}", file=sys.stderr) + sys.exit(1) + + print("SUCCESS") + """ % use_simple_format) + + rc, out, err = assert_python_ok("-c", test_script) + self.assertIn(b"SUCCESS", out) + self.assertNotIn(b"traceback", err.lower(), err) + self.assertNotIn(b"resource_tracker.py", err, err) diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 08b638e4b8d..a893932169a 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -8,6 +8,8 @@ import contextlib import os import sys +import unittest.mock +from test.support import swap_item class TestHook: @@ -643,6 +645,112 @@ def test_assert_unicode(): else: raise RuntimeError("Expected sys.audit(9) to fail.") +def test_sys_remote_exec(): + import tempfile + + pid = os.getpid() + event_pid = -1 + event_script_path = "" + remote_event_script_path = "" + def hook(event, args): + if event not in ["sys.remote_exec", "cpython.remote_debugger_script"]: + return + print(event, args) + match event: + case "sys.remote_exec": + nonlocal event_pid, event_script_path + event_pid = args[0] + event_script_path = args[1] + case "cpython.remote_debugger_script": + nonlocal remote_event_script_path + remote_event_script_path = args[0] + + sys.addaudithook(hook) + with tempfile.NamedTemporaryFile(mode='w+', delete=True) as tmp_file: + tmp_file.write("a = 1+1\n") + tmp_file.flush() + sys.remote_exec(pid, tmp_file.name) + assertEqual(event_pid, pid) + assertEqual(event_script_path, tmp_file.name) + assertEqual(remote_event_script_path, tmp_file.name) + +def test_import_module(): + import importlib + + with TestHook() as hook: + importlib.import_module("importlib") # already imported, won't get logged + importlib.import_module("email") # standard library module + importlib.import_module("pythoninfo") # random module + importlib.import_module(".audit_test_data.submodule", "test") # relative import + importlib.import_module("test.audit_test_data.submodule2") # absolute import + importlib.import_module("_testcapi") # extension module + + actual = [a for e, a in hook.seen if e == "import"] + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) + +def test_builtin__import__(): + import importlib # noqa: F401 + + with TestHook() as hook: + __import__("importlib") + __import__("email") + __import__("pythoninfo") + __import__("audit_test_data.submodule", level=1, globals={"__package__": "test"}) + __import__("test.audit_test_data.submodule2") + __import__("_testcapi") + + actual = [a for e, a in hook.seen if e == "import"] + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) + +def test_import_statement(): + import importlib # noqa: F401 + # Set __package__ so relative imports work + with swap_item(globals(), "__package__", "test"): + with TestHook() as hook: + import importlib # noqa: F401 + import email # noqa: F401 + import pythoninfo # noqa: F401 + from .audit_test_data import submodule # noqa: F401 + import test.audit_test_data.submodule2 # noqa: F401 + import _testcapi # noqa: F401 + + actual = [a for e, a in hook.seen if e == "import"] + # Import statement ordering is different because the package is + # loaded first and then the submodule + assertSequenceEqual( + [ + ("email", None, sys.path, sys.meta_path, sys.path_hooks), + ("pythoninfo", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule", None, sys.path, sys.meta_path, sys.path_hooks), + ("test.audit_test_data.submodule2", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", None, sys.path, sys.meta_path, sys.path_hooks), + ("_testcapi", unittest.mock.ANY, None, None, None) + ], + actual, + ) if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/audit_test_data/__init__.py b/Lib/test/audit_test_data/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/audit_test_data/submodule.py b/Lib/test/audit_test_data/submodule.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/audit_test_data/submodule2.py b/Lib/test/audit_test_data/submodule2.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 4a67fcd2c3e..4729708efd3 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -1020,12 +1020,19 @@ test_unsigned_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t goto skip_optional; } { - unsigned long ival = PyLong_AsUnsignedLongMask(args[2]); - if (ival == (unsigned long)-1 && PyErr_Occurred()) { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned char), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { goto exit; } - else { - c = (unsigned char) ival; + if ((size_t)_bytes > sizeof(unsigned char)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } } } skip_optional: @@ -1038,7 +1045,7 @@ test_unsigned_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_unsigned_char_converter_impl(PyObject *module, unsigned char a, unsigned char b, unsigned char c) -/*[clinic end generated code: output=45920dbedc22eb55 input=021414060993e289]*/ +/*[clinic end generated code: output=49eda9faaf53372a input=021414060993e289]*/ /*[clinic input] @@ -1151,9 +1158,21 @@ test_unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_ if (nargs < 3) { goto skip_optional; } - c = (unsigned short)PyLong_AsUnsignedLongMask(args[2]); - if (c == (unsigned short)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned short), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned short)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = test_unsigned_short_converter_impl(module, a, b, c); @@ -1165,7 +1184,7 @@ test_unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_ static PyObject * test_unsigned_short_converter_impl(PyObject *module, unsigned short a, unsigned short b, unsigned short c) -/*[clinic end generated code: output=e6e990df729114fc input=cdfd8eff3d9176b4]*/ +/*[clinic end generated code: output=f591c7797e150f49 input=cdfd8eff3d9176b4]*/ /*[clinic input] @@ -1298,9 +1317,21 @@ test_unsigned_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t if (nargs < 3) { goto skip_optional; } - c = (unsigned int)PyLong_AsUnsignedLongMask(args[2]); - if (c == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = test_unsigned_int_converter_impl(module, a, b, c); @@ -1312,7 +1343,7 @@ test_unsigned_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_unsigned_int_converter_impl(PyObject *module, unsigned int a, unsigned int b, unsigned int c) -/*[clinic end generated code: output=f9cdbe410ccc98a3 input=5533534828b62fc0]*/ +/*[clinic end generated code: output=50a413f1cc82dc11 input=5533534828b62fc0]*/ /*[clinic input] @@ -1414,7 +1445,22 @@ test_unsigned_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t _PyArg_BadArgument("test_unsigned_long_converter", "argument 3", "int", args[2]); goto exit; } - c = PyLong_AsUnsignedLongMask(args[2]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } skip_optional: return_value = test_unsigned_long_converter_impl(module, a, b, c); @@ -1425,7 +1471,7 @@ test_unsigned_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t static PyObject * test_unsigned_long_converter_impl(PyObject *module, unsigned long a, unsigned long b, unsigned long c) -/*[clinic end generated code: output=d74eed227d77a31b input=f450d94cae1ef73b]*/ +/*[clinic end generated code: output=1bbf5620093cc914 input=f450d94cae1ef73b]*/ /*[clinic input] @@ -1529,7 +1575,22 @@ test_unsigned_long_long_converter(PyObject *module, PyObject *const *args, Py_ss _PyArg_BadArgument("test_unsigned_long_long_converter", "argument 3", "int", args[2]); goto exit; } - c = PyLong_AsUnsignedLongLongMask(args[2]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned long long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } skip_optional: return_value = test_unsigned_long_long_converter_impl(module, a, b, c); @@ -1542,7 +1603,7 @@ test_unsigned_long_long_converter_impl(PyObject *module, unsigned long long a, unsigned long long b, unsigned long long c) -/*[clinic end generated code: output=5ca4e4dfb3db644b input=a15115dc41866ff4]*/ +/*[clinic end generated code: output=582a6623dc845824 input=a15115dc41866ff4]*/ /*[clinic input] @@ -1551,12 +1612,17 @@ test_Py_ssize_t_converter a: Py_ssize_t = 12 b: Py_ssize_t(accept={int}) = 34 c: Py_ssize_t(accept={int, NoneType}) = 56 + d: Py_ssize_t(accept={int}, allow_negative=False) = 78 + e: Py_ssize_t(accept={int, NoneType}, allow_negative=False) = 90 + f: Py_ssize_t(accept={int}, allow_negative=True) = -12 + g: Py_ssize_t(accept={int, NoneType}, allow_negative=True) = -34 / [clinic start generated code]*/ PyDoc_STRVAR(test_Py_ssize_t_converter__doc__, -"test_Py_ssize_t_converter($module, a=12, b=34, c=56, /)\n" +"test_Py_ssize_t_converter($module, a=12, b=34, c=56, d=78, e=90, f=-12,\n" +" g=-34, /)\n" "--\n" "\n"); @@ -1565,7 +1631,8 @@ PyDoc_STRVAR(test_Py_ssize_t_converter__doc__, static PyObject * test_Py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, - Py_ssize_t c); + Py_ssize_t c, Py_ssize_t d, Py_ssize_t e, + Py_ssize_t f, Py_ssize_t g); static PyObject * test_Py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -1574,8 +1641,12 @@ test_Py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t na Py_ssize_t a = 12; Py_ssize_t b = 34; Py_ssize_t c = 56; + Py_ssize_t d = 78; + Py_ssize_t e = 90; + Py_ssize_t f = -12; + Py_ssize_t g = -34; - if (!_PyArg_CheckPositional("test_Py_ssize_t_converter", nargs, 0, 3)) { + if (!_PyArg_CheckPositional("test_Py_ssize_t_converter", nargs, 0, 7)) { goto exit; } if (nargs < 1) { @@ -1614,8 +1685,55 @@ test_Py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t na if (!_Py_convert_optional_to_ssize_t(args[2], &c)) { goto exit; } + if (nargs < 4) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[3]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + d = ival; + if (d < 0) { + PyErr_SetString(PyExc_ValueError, + "d cannot be negative"); + goto exit; + } + } + if (nargs < 5) { + goto skip_optional; + } + if (!_Py_convert_optional_to_non_negative_ssize_t(args[4], &e)) { + goto exit; + } + if (nargs < 6) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[5]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + f = ival; + } + if (nargs < 7) { + goto skip_optional; + } + if (!_Py_convert_optional_to_ssize_t(args[6], &g)) { + goto exit; + } skip_optional: - return_value = test_Py_ssize_t_converter_impl(module, a, b, c); + return_value = test_Py_ssize_t_converter_impl(module, a, b, c, d, e, f, g); exit: return return_value; @@ -1623,8 +1741,9 @@ test_Py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t na static PyObject * test_Py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, - Py_ssize_t c) -/*[clinic end generated code: output=48214bc3d01f4dd7 input=3855f184bb3f299d]*/ + Py_ssize_t c, Py_ssize_t d, Py_ssize_t e, + Py_ssize_t f, Py_ssize_t g) +/*[clinic end generated code: output=4ae0a56a1447fba9 input=a25bac8ecf2890aa]*/ /*[clinic input] @@ -4019,13 +4138,14 @@ test_preprocessor_guarded_if_with_continuation_impl(PyObject *module) #if CONDITION_E || CONDITION_F #warning "different type of CPP directive" /*[clinic input] +@permit_long_summary test_preprocessor_guarded_if_e_or_f Makes sure cpp.Monitor handles other directives than preprocessor conditionals. [clinic start generated code]*/ static PyObject * test_preprocessor_guarded_if_e_or_f_impl(PyObject *module) -/*[clinic end generated code: output=e49d24ff64ad88bc input=57b9c37f938bc4f1]*/ +/*[clinic end generated code: output=e49d24ff64ad88bc input=3ca9ab4e883300ed]*/ #endif /*[clinic input] @@ -4221,7 +4341,7 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg goto exit; } a = args[0]; - __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + __clinic_args = PyTuple_FromArray(args + 1, nargs - 1); if (__clinic_args == NULL) { goto exit; } @@ -4236,7 +4356,7 @@ test_vararg_and_posonly(PyObject *module, PyObject *const *args, Py_ssize_t narg static PyObject * test_vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=0c11c475e240869e input=2c49a482f68545c0]*/ +/*[clinic end generated code: output=83cbe9554d04add2 input=2c49a482f68545c0]*/ /*[clinic input] test_vararg @@ -4301,7 +4421,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } a = fastargs[0]; __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -4317,7 +4437,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=17ba625cdd0369c1 input=7448995636d9186a]*/ +/*[clinic end generated code: output=d773f7b54e61f73a input=7448995636d9186a]*/ /*[clinic input] test_vararg_with_default @@ -4394,7 +4514,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar } skip_optional_kwonly: __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -4411,7 +4531,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=3f2b06ab08d5d0be input=3a0f9f557ce1f712]*/ +/*[clinic end generated code: output=d25e56802c197344 input=3a0f9f557ce1f712]*/ /*[clinic input] test_vararg_with_only_defaults @@ -4492,7 +4612,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize } c = fastargs[1]; skip_optional_kwonly: - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -4508,7 +4628,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize static PyObject * test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b, PyObject *c) -/*[clinic end generated code: output=f46666f0b1bf86b9 input=6983e66817f82924]*/ +/*[clinic end generated code: output=7366943a7df42e05 input=6983e66817f82924]*/ /*[clinic input] test_paramname_module @@ -5023,14 +5143,18 @@ Test_an_metho_arg_named_arg_impl(TestObj *self, int arg) Test.__init__ *args: tuple -Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE. +Varargs init method. + +For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ PyDoc_STRVAR(Test___init____doc__, "Test(*args)\n" "--\n" "\n" -"Varargs init method. For example, nargs is translated to PyTuple_GET_SIZE."); +"Varargs init method.\n" +"\n" +"For example, nargs is translated to PyTuple_GET_SIZE."); static int Test___init___impl(TestObj *self, PyObject *args); @@ -5059,21 +5183,25 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) static int Test___init___impl(TestObj *self, PyObject *args) -/*[clinic end generated code: output=f172425cec373cd6 input=4b8388c4e6baab6f]*/ +/*[clinic end generated code: output=0e5836c40dbc2397 input=a615a4485c0fc3e2]*/ /*[clinic input] @classmethod Test.__new__ *args: tuple -Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE. +Varargs new method. + +For example, nargs is translated to PyTuple_GET_SIZE. [clinic start generated code]*/ PyDoc_STRVAR(Test__doc__, "Test(*args)\n" "--\n" "\n" -"Varargs new method. For example, nargs is translated to PyTuple_GET_SIZE."); +"Varargs new method.\n" +"\n" +"For example, nargs is translated to PyTuple_GET_SIZE."); static PyObject * Test_impl(PyTypeObject *type, PyObject *args); @@ -5101,7 +5229,7 @@ Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) static PyObject * Test_impl(PyTypeObject *type, PyObject *args) -/*[clinic end generated code: output=ee1e8892a67abd4a input=a8259521129cad20]*/ +/*[clinic end generated code: output=e6fba0c8951882fd input=8ce30adb836aeacb]*/ /*[clinic input] diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 55844ec35a9..ace56aab7ac 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -3,6 +3,7 @@ import contextlib import copy import decimal +import fractions import io import itertools import os @@ -183,7 +184,7 @@ class NotEnough(tzinfo): def __init__(self, offset, name): self.__offset = offset self.__name = name - self.assertTrue(issubclass(NotEnough, tzinfo)) + self.assertIsSubclass(NotEnough, tzinfo) ne = NotEnough(3, "NotByALongShot") self.assertIsInstance(ne, tzinfo) @@ -232,7 +233,7 @@ def test_pickling_subclass(self): self.assertIs(type(derived), otype) self.assertEqual(derived.utcoffset(None), offset) self.assertEqual(derived.tzname(None), oname) - self.assertFalse(hasattr(derived, 'spam')) + self.assertNotHasAttr(derived, 'spam') def test_issue23600(self): DSTDIFF = DSTOFFSET = timedelta(hours=1) @@ -773,6 +774,9 @@ def test_str(self): microseconds=999999)), "999999999 days, 23:59:59.999999") + # test the Doc/library/datetime.rst recipe + eq(f'-({-td(hours=-1)!s})', "-(1:00:00)") + def test_repr(self): name = 'datetime.' + self.theclass.__name__ self.assertEqual(repr(self.theclass(1)), @@ -810,7 +814,7 @@ def test_roundtrip(self): # Verify td -> string -> td identity. s = repr(td) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] td2 = eval(s) self.assertEqual(td, td2) @@ -1228,7 +1232,7 @@ def test_roundtrip(self): self.theclass.today()): # Verify dt -> string -> date identity. s = repr(dt) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) @@ -1804,7 +1808,7 @@ def test_bool(self): self.assertTrue(self.theclass.min) self.assertTrue(self.theclass.max) - def test_strftime_y2k(self): + def check_strftime_y2k(self, specifier): # Test that years less than 1000 are 0-padded; note that the beginning # of an ISO 8601 year may fall in an ISO week of the year before, and # therefore needs an offset of -1 when formatting with '%G'. @@ -1818,22 +1822,28 @@ def test_strftime_y2k(self): (1000, 0), (1970, 0), ) - specifiers = 'YG' - if _time.strftime('%F', (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == '1900-01-01': - specifiers += 'FC' for year, g_offset in dataset: - for specifier in specifiers: - with self.subTest(year=year, specifier=specifier): - d = self.theclass(year, 1, 1) - if specifier == 'G': - year += g_offset - if specifier == 'C': - expected = f"{year // 100:02d}" - else: - expected = f"{year:04d}" - if specifier == 'F': - expected += f"-01-01" - self.assertEqual(d.strftime(f"%{specifier}"), expected) + with self.subTest(year=year, specifier=specifier): + d = self.theclass(year, 1, 1) + if specifier == 'G': + year += g_offset + if specifier == 'C': + expected = f"{year // 100:02d}" + else: + expected = f"{year:04d}" + if specifier == 'F': + expected += f"-01-01" + self.assertEqual(d.strftime(f"%{specifier}"), expected) + + def test_strftime_y2k(self): + self.check_strftime_y2k('Y') + self.check_strftime_y2k('G') + + def test_strftime_y2k_c99(self): + # CPython requires C11; specifiers new in C99 must work. + # (Other implementations may want to disable this test.) + self.check_strftime_y2k('F') + self.check_strftime_y2k('C') def test_replace(self): cls = self.theclass @@ -2144,14 +2154,20 @@ def test_fromisocalendar_value_errors(self): (10000, 1, 1), (0, 1, 1), (9999999, 1, 1), + ] + for isocal in isocals: + with self.subTest(isocal=isocal): + with self.assertRaises(ValueError): + self.theclass.fromisocalendar(*isocal) + + isocals = [ (2<<32, 1, 1), (2019, 2<<32, 1), (2019, 1, 2<<32), ] - for isocal in isocals: with self.subTest(isocal=isocal): - with self.assertRaises(ValueError): + with self.assertRaises((ValueError, OverflowError)): self.theclass.fromisocalendar(*isocal) def test_fromisocalendar_type_errors(self): @@ -2215,7 +2231,7 @@ def test_roundtrip(self): self.theclass.now()): # Verify dt -> string -> datetime identity. s = repr(dt) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) @@ -2298,7 +2314,7 @@ def test_isoformat_timezone(self): dt = dt_base.replace(tzinfo=tzi) exp = exp_base + exp_tz with self.subTest(tzi=tzi): - assert dt.isoformat() == exp + self.assertEqual(dt.isoformat(), exp) def test_format(self): dt = self.theclass(2007, 9, 10, 4, 5, 1, 123) @@ -2611,6 +2627,10 @@ def test_fromtimestamp(self): expected = time.localtime(ts) got = self.theclass.fromtimestamp(ts) self.verify_field_equality(expected, got) + got = self.theclass.fromtimestamp(decimal.Decimal(ts)) + self.verify_field_equality(expected, got) + got = self.theclass.fromtimestamp(fractions.Fraction(ts)) + self.verify_field_equality(expected, got) def test_fromtimestamp_keyword_arg(self): import time @@ -2626,6 +2646,12 @@ def test_utcfromtimestamp(self): with self.assertWarns(DeprecationWarning): got = self.theclass.utcfromtimestamp(ts) self.verify_field_equality(expected, got) + with self.assertWarns(DeprecationWarning): + got = self.theclass.utcfromtimestamp(decimal.Decimal(ts)) + self.verify_field_equality(expected, got) + with self.assertWarns(DeprecationWarning): + got = self.theclass.utcfromtimestamp(fractions.Fraction(ts)) + self.verify_field_equality(expected, got) # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). @@ -2713,6 +2739,108 @@ def utcfromtimestamp(*args, **kwargs): self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 7812) + @support.run_with_tz('MSK-03') # Something east of Greenwich + def test_microsecond_rounding_decimal(self): + D = decimal.Decimal + def utcfromtimestamp(*args, **kwargs): + with self.assertWarns(DeprecationWarning): + return self.theclass.utcfromtimestamp(*args, **kwargs) + + for fts in [self.theclass.fromtimestamp, + utcfromtimestamp]: + zero = fts(D(0)) + self.assertEqual(zero.second, 0) + self.assertEqual(zero.microsecond, 0) + one = fts(D('0.000_001')) + try: + minus_one = fts(D('-0.000_001')) + except OSError: + # localtime(-1) and gmtime(-1) is not supported on Windows + pass + else: + self.assertEqual(minus_one.second, 59) + self.assertEqual(minus_one.microsecond, 999_999) + + t = fts(D('-0.000_000_1')) + self.assertEqual(t, zero) + t = fts(D('-0.000_000_9')) + self.assertEqual(t, minus_one) + t = fts(D(-1)/2**7) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992188) + + t = fts(D('0.000_000_1')) + self.assertEqual(t, zero) + t = fts(D('0.000_000_5')) + self.assertEqual(t, zero) + t = fts(D('0.000_000_500_000_000_000_000_1')) + self.assertEqual(t, one) + t = fts(D('0.000_000_9')) + self.assertEqual(t, one) + t = fts(D('0.999_999_499_999_999_9')) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 999_999) + t = fts(D('0.999_999_5')) + self.assertEqual(t.second, 1) + self.assertEqual(t.microsecond, 0) + t = fts(D('0.999_999_9')) + self.assertEqual(t.second, 1) + self.assertEqual(t.microsecond, 0) + t = fts(D(1)/2**7) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 7812) + + @support.run_with_tz('MSK-03') # Something east of Greenwich + def test_microsecond_rounding_fraction(self): + F = fractions.Fraction + def utcfromtimestamp(*args, **kwargs): + with self.assertWarns(DeprecationWarning): + return self.theclass.utcfromtimestamp(*args, **kwargs) + + for fts in [self.theclass.fromtimestamp, + utcfromtimestamp]: + zero = fts(F(0)) + self.assertEqual(zero.second, 0) + self.assertEqual(zero.microsecond, 0) + one = fts(F(1, 1_000_000)) + try: + minus_one = fts(F(-1, 1_000_000)) + except OSError: + # localtime(-1) and gmtime(-1) is not supported on Windows + pass + else: + self.assertEqual(minus_one.second, 59) + self.assertEqual(minus_one.microsecond, 999_999) + + t = fts(F(-1, 10_000_000)) + self.assertEqual(t, zero) + t = fts(F(-9, 10_000_000)) + self.assertEqual(t, minus_one) + t = fts(F(-1, 2**7)) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992188) + + t = fts(F(1, 10_000_000)) + self.assertEqual(t, zero) + t = fts(F(5, 10_000_000)) + self.assertEqual(t, zero) + t = fts(F(5_000_000_000, 9_999_999_999_999_999)) + self.assertEqual(t, one) + t = fts(F(9, 10_000_000)) + self.assertEqual(t, one) + t = fts(F(9_999_995_000_000_000, 10_000_000_000_000_001)) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 999_999) + t = fts(F(9_999_995, 10_000_000)) + self.assertEqual(t.second, 1) + self.assertEqual(t.microsecond, 0) + t = fts(F(9_999_999, 10_000_000)) + self.assertEqual(t.second, 1) + self.assertEqual(t.microsecond, 0) + t = fts(F(1, 2**7)) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 7812) + def test_timestamp_limits(self): with self.subTest("minimum UTC"): min_dt = self.theclass.min.replace(tzinfo=timezone.utc) @@ -2892,6 +3020,12 @@ def test_strptime(self): strptime("-00:02:01.000003", "%z").utcoffset(), -timedelta(minutes=2, seconds=1, microseconds=3) ) + self.assertEqual(strptime("+01:07", "%:z").utcoffset(), + 1 * HOUR + 7 * MINUTE) + self.assertEqual(strptime("-10:02", "%:z").utcoffset(), + -(10 * HOUR + 2 * MINUTE)) + self.assertEqual(strptime("-00:00:01.00001", "%:z").utcoffset(), + -timedelta(seconds=1, microseconds=10)) # Only local timezone and UTC are supported for tzseconds, tzname in ((0, 'UTC'), (0, 'GMT'), (-_time.timezone, _time.tzname[0])): @@ -2921,6 +3055,16 @@ def test_strptime(self): with self.assertRaises(ValueError): strptime("-000", "%z") with self.assertRaises(ValueError): strptime("z", "%z") + def test_strptime_ampm(self): + dt = datetime(1999, 3, 17, 0, 44, 55, 2) + for hour in range(0, 24): + with self.subTest(hour=hour): + new_dt = dt.replace(hour=hour) + dt_str = new_dt.strftime("%I %p") + + self.assertEqual(self.theclass.strptime(dt_str, "%I %p").hour, + hour) + def test_strptime_single_digit(self): # bpo-34903: Check that single digit dates and times are allowed. @@ -2969,6 +3113,17 @@ def test_strptime_leap_year(self): with self._assertNotWarns(DeprecationWarning): self.theclass.strptime('02-29,2024', '%m-%d,%Y') + def test_strptime_z_empty(self): + for directive in ('z', ':z'): + string = '2025-04-25 11:42:47' + format = f'%Y-%m-%d %H:%M:%S%{directive}' + target = self.theclass(2025, 4, 25, 11, 42, 47) + with self.subTest(string=string, + format=format, + target=target): + result = self.theclass.strptime(string, format) + self.assertEqual(result, target) + def test_more_timetuple(self): # This tests fields beyond those tested by the TestDate.test_timetuple. t = self.theclass(2004, 12, 31, 6, 22, 33) @@ -3335,7 +3490,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=dtstr): dt_rt = self.theclass.fromisoformat(dtstr) - assert dt == dt_rt, dt_rt + self.assertEqual(dt_rt, dt) def test_fromisoformat_separators(self): separators = [ @@ -3568,6 +3723,10 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) '2009-04-19T12:30:45.400 ', # Trailing space (gh-130959) '2009-04-19T12:30:45. 400', # Space before fraction (gh-130959) + '2009-04-19T12:30:45+00:90:00', # Time zone field out from range + '2009-04-19T12:30:45+00:00:90', # Time zone field out from range + '2009-04-19T12:30:45-00:90:00', # Time zone field out from range + '2009-04-19T12:30:45-00:00:90', # Time zone field out from range ] for bad_str in bad_strs: @@ -3669,7 +3828,7 @@ def test_roundtrip(self): # Verify t -> string -> time identity. s = repr(t) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] t2 = eval(s) self.assertEqual(t, t2) @@ -3847,7 +4006,7 @@ def test_isoformat_timezone(self): t = t_base.replace(tzinfo=tzi) exp = exp_base + exp_tz with self.subTest(tzi=tzi): - assert t.isoformat() == exp + self.assertEqual(t.isoformat(), exp) def test_1653736(self): # verify it doesn't accept extra keyword arguments @@ -4023,6 +4182,12 @@ def test_strptime_tz(self): strptime("-00:02:01.000003", "%z").utcoffset(), -timedelta(minutes=2, seconds=1, microseconds=3) ) + self.assertEqual(strptime("+01:07", "%:z").utcoffset(), + 1 * HOUR + 7 * MINUTE) + self.assertEqual(strptime("-10:02", "%:z").utcoffset(), + -(10 * HOUR + 2 * MINUTE)) + self.assertEqual(strptime("-00:00:01.00001", "%:z").utcoffset(), + -timedelta(seconds=1, microseconds=10)) # Only local timezone and UTC are supported for tzseconds, tzname in ((0, 'UTC'), (0, 'GMT'), (-_time.timezone, _time.tzname[0])): @@ -4052,9 +4217,11 @@ def test_strptime_tz(self): self.assertEqual(strptime("UTC", "%Z").tzinfo, None) def test_strptime_errors(self): - for tzstr in ("-2400", "-000", "z"): + for tzstr in ("-2400", "-000", "z", "24:00"): with self.assertRaises(ValueError): self.theclass.strptime(tzstr, "%z") + with self.assertRaises(ValueError): + self.theclass.strptime(tzstr, "%:z") def test_strptime_single_digit(self): # bpo-34903: Check that single digit times are allowed. @@ -4332,7 +4499,7 @@ def utcoffset(self, t): elif x is d2: expected = -1 else: - assert y is d2 + self.assertIs(y, d2) expected = 1 self.assertEqual(got, expected) @@ -4660,7 +4827,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=tstr): t_rt = self.theclass.fromisoformat(tstr) - assert t == t_rt + self.assertEqual(t_rt, t) def test_fromisoformat_timespecs(self): time_bases = [ @@ -4792,6 +4959,11 @@ def test_fromisoformat_fails(self): '12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) '12:30:45.400 ', # Trailing space (gh-130959) '12:30:45. 400', # Space before fraction (gh-130959) + '24:00:00.000001', # Has non-zero microseconds on 24:00 + '24:00:01.000000', # Has non-zero seconds on 24:00 + '24:01:00.000000', # Has non-zero minutes on 24:00 + '12:30:45+00:90:00', # Time zone field out from range + '12:30:45+00:00:90', # Time zone field out from range ] for bad_str in bad_strs: @@ -5492,7 +5664,7 @@ def utcoffset(self, t): elif x is d2: expected = timedelta(minutes=(11-59)-0) else: - assert y is d2 + self.assertIs(y, d2) expected = timedelta(minutes=0-(11-59)) self.assertEqual(got, expected) @@ -6128,21 +6300,21 @@ def test_vilnius_1941_fromutc(self): gdt = datetime(1941, 6, 23, 20, 59, 59, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:59:59 1941 MSK+0300') self.assertEqual(ldt.fold, 0) self.assertFalse(ldt.dst()) gdt = datetime(1941, 6, 23, 21, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 1) self.assertTrue(ldt.dst()) gdt = datetime(1941, 6, 23, 22, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Tue Jun 24 00:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 0) self.assertTrue(ldt.dst()) @@ -6152,22 +6324,22 @@ def test_vilnius_1941_toutc(self): ldt = datetime(1941, 6, 23, 22, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 19:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 20:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius, fold=1) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 21:59:59 1941 UTC') ldt = datetime(1941, 6, 24, 0, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 22:00:00 1941 UTC') def test_constructors(self): @@ -7272,6 +7444,34 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) + def test_concurrent_initialization_subinterpreter(self): + # gh-136421: Concurrent initialization of _datetime across multiple + # interpreters wasn't thread-safe due to its static types. + + # Run in a subprocess to ensure we get a clean version of _datetime + script = """if True: + from concurrent.futures import InterpreterPoolExecutor + + def func(): + import _datetime + print('a', end='') + + with InterpreterPoolExecutor() as executor: + for _ in range(8): + executor.submit(func) + """ + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + + # Now test against concurrent reinitialization + script = "import _datetime\n" + script + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 07681d75448..e7a12e4d0b6 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -271,6 +271,9 @@ def _create_parser(): group = parser.add_argument_group('Selecting tests') group.add_argument('-r', '--randomize', action='store_true', help='randomize test execution order.' + more_details) + group.add_argument('--no-randomize', dest='no_randomize', action='store_true', + help='do not randomize test execution order, even if ' + 'it would be implied by another option') group.add_argument('--prioritize', metavar='TEST1,TEST2,...', action='append', type=priority_list, help='select these tests first, even if the order is' @@ -461,7 +464,11 @@ def _parse_args(args, **kwargs): if ns.python is None: ns.rerun = True ns.print_slow = True - ns.verbose3 = True + if not ns.verbose: + ns.verbose3 = True + else: + # --verbose has the priority over --verbose3 + pass else: ns._add_python_opts = False @@ -539,6 +546,8 @@ def _parse_args(args, **kwargs): ns.use_resources.append(r) if ns.random_seed is not None: ns.randomize = True + if ns.no_randomize: + ns.randomize = False if ns.verbose: ns.header = True diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index f01c1240774..6c0e50846a4 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -24,10 +24,12 @@ "test_future_stmt", "test_gdb", "test_inspect", - "test_pydoc", + "test_io", "test_multiprocessing_fork", "test_multiprocessing_forkserver", "test_multiprocessing_spawn", + "test_os", + "test_pydoc", } diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 713cbedb299..0fc2548789e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -190,6 +190,12 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList strip_py_suffix(tests) + exclude_tests = set() + if self.exclude: + for arg in self.cmdline_args: + exclude_tests.add(arg) + self.cmdline_args = [] + if self.pgo: # add default PGO tests if no tests are specified setup_pgo_tests(self.cmdline_args, self.pgo_extended) @@ -200,17 +206,15 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList if self.tsan_parallel: setup_tsan_parallel_tests(self.cmdline_args) - exclude_tests = set() - if self.exclude: - for arg in self.cmdline_args: - exclude_tests.add(arg) - self.cmdline_args = [] - alltests = findtests(testdir=self.test_dir, exclude=exclude_tests) if not self.fromfile: selected = tests or self.cmdline_args + if exclude_tests: + # Support "--pgo/--tsan -x test_xxx" command + selected = [name for name in selected + if name not in exclude_tests] if selected: selected = split_test_packages(selected) else: @@ -543,8 +547,6 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: self.first_runtests = runtests self.logger.set_tests(runtests) - setup_process() - if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interferes with refleak check use_load_tracker = False @@ -644,15 +646,23 @@ def _add_cross_compile_opts(self, regrtest_opts): return (environ, keep_environ) def _add_ci_python_opts(self, python_opts, keep_environ): - # --fast-ci and --slow-ci add options to Python: - # "-u -W default -bb -E" + # --fast-ci and --slow-ci add options to Python. + # + # Some platforms run tests in embedded mode and cannot change options + # after startup, so if this function changes, consider also updating: + # * gradle_task in Android/android.py - # Unbuffered stdout and stderr - if not sys.stdout.write_through: + # Unbuffered stdout and stderr. This isn't helpful on Android, because + # it would cause lines to be split into multiple log messages. + if not sys.stdout.write_through and sys.platform != "android": python_opts.append('-u') - # Add warnings filter 'error' - if 'default' not in sys.warnoptions: + # Add warnings filter 'error', unless the user specified a different + # filter. Ignore BytesWarning since it's controlled by '-b' below. + if not [ + opt for opt in sys.warnoptions + if not opt.endswith("::BytesWarning") + ]: python_opts.extend(('-W', 'error')) # Error on bytes/str comparison @@ -671,8 +681,12 @@ def _execute_python(self, cmd, environ): cmd_text = shlex.join(cmd) try: - print(f"+ {cmd_text}", flush=True) + # Android and iOS run tests in embedded mode. To update their + # Python options, see the comment in _add_ci_python_opts. + if not cmd[0]: + raise ValueError("No Python executable is present") + print(f"+ {cmd_text}", flush=True) if hasattr(os, 'execv') and not MS_WINDOWS: os.execv(cmd[0], cmd) # On success, execv() do no return. @@ -721,10 +735,7 @@ def _add_python_opts(self) -> None: self._execute_python(cmd, environ) def _init(self): - # Set sys.stdout encoder error handler to backslashreplace, - # similar to sys.stderr error handler, to avoid UnicodeEncodeError - # when printing a traceback or any other non-encodable character. - sys.stdout.reconfigure(errors="backslashreplace") + setup_process() if self.junit_filename and not os.path.isabs(self.junit_filename): self.junit_filename = os.path.abspath(self.junit_filename) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 5c78515506d..e7da17e500e 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -93,6 +93,13 @@ def runtest_refleak(test_name, test_func, for obj in abc.__subclasses__() + [abc]: abcs[obj] = _get_dump(obj)[0] + # `ByteString` is not included in `collections.abc.__all__` + with warnings.catch_warnings(action='ignore', category=DeprecationWarning): + ByteString = collections.abc.ByteString + # Mypy doesn't even think `ByteString` is a class, hence the `type: ignore` + for obj in ByteString.__subclasses__() + [ByteString]: # type: ignore[attr-defined] + abcs[obj] = _get_dump(obj)[0] + # bpo-31217: Integer pool to get a single integer object for the same # value. The pool is used to prevent false alarm when checking for memory # block leaks. Fill the pool with values in -1000..1000 which are the most @@ -254,6 +261,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data): # Clear ABC registries, restoring previously saved ABC registries. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + with warnings.catch_warnings(action='ignore', category=DeprecationWarning): + abs_classes.append(collections.abc.ByteString) abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index ffc29fa8dc6..138465012a2 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -9,6 +9,13 @@ from .utils import print_warning +# Import termios to save and restore terminal echo. This is only available on +# Unix, and it's fine if the module can't be found. +try: + import termios # noqa: F401 +except ModuleNotFoundError: + pass + class SkipTestEnvironment(Exception): pass @@ -65,6 +72,7 @@ def __init__(self, test_name, verbose, quiet, *, pgo): 'shutil_archive_formats', 'shutil_unpack_formats', 'asyncio.events._event_loop_policy', 'urllib.requests._url_tempfiles', 'urllib.requests._opener', + 'stty_echo', ) def get_module(self, name): @@ -97,7 +105,7 @@ def get_asyncio_events__event_loop_policy(self): return support.maybe_get_event_loop_policy() def restore_asyncio_events__event_loop_policy(self, policy): asyncio = self.get_module('asyncio') - asyncio._set_event_loop_policy(policy) + asyncio.events._set_event_loop_policy(policy) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] @@ -292,6 +300,24 @@ def restore_warnings_showwarning(self, fxn): warnings = self.get_module('warnings') warnings.showwarning = fxn + def get_stty_echo(self): + termios = self.try_get_module('termios') + if not os.isatty(fd := sys.__stdin__.fileno()): + return None + attrs = termios.tcgetattr(fd) + lflags = attrs[3] + return bool(lflags & termios.ECHO) + def restore_stty_echo(self, echo): + termios = self.get_module('termios') + attrs = termios.tcgetattr(fd := sys.__stdin__.fileno()) + if echo: + # Turn echo on. + attrs[3] |= termios.ECHO + else: + # Turn echo off. + attrs[3] &= ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, attrs) + def resource_info(self): for name in self.resources: method_suffix = name.replace('.', '_') diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index c3d1f60a400..b9b76a44e3b 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -1,5 +1,6 @@ import faulthandler import gc +import io import os import random import signal @@ -7,6 +8,7 @@ import unittest from test import support from test.support.os_helper import TESTFN_UNDECODABLE, FS_NONASCII +from _colorize import can_colorize # type: ignore[import-not-found] from .filter import set_match_tests from .runtests import RunTests @@ -52,6 +54,14 @@ def setup_process() -> None: support.record_original_stdout(sys.stdout) + # Set sys.stdout encoder error handler to backslashreplace, + # similar to sys.stderr error handler, to avoid UnicodeEncodeError + # when printing a traceback or any other non-encodable character. + # + # Use an assertion to fix mypy error. + assert isinstance(sys.stdout, io.TextIOWrapper) + sys.stdout.reconfigure(errors="backslashreplace") + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). @@ -130,3 +140,10 @@ def setup_tests(runtests: RunTests) -> None: gc.set_threshold(runtests.gc_threshold) random.seed(runtests.random_seed) + + # sys.stdout is redirected to a StringIO in single process mode on which + # color auto-detect fails as StringIO is not a TTY. If the original + # sys.stdout supports color pass that through with FORCE_COLOR so that when + # results are printed, such as with -W, they get color. + if can_colorize(file=sys.stdout): + os.environ['FORCE_COLOR'] = "1" diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 57d7b649d2e..958a915626a 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -283,7 +283,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: try: setup_tests(runtests) - if output_on_failure: + if output_on_failure or runtests.pgo: support.verbose = True stream = io.StringIO() diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index d984a735bdf..f1f8c8bde92 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -3,12 +3,10 @@ TSAN_TESTS = [ 'test_asyncio', - # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). - 'test_capi.test_mem', - 'test_capi.test_pyatomic', + 'test_capi', 'test_code', 'test_ctypes', - # 'test_concurrent_futures', # gh-130605: too many data races + 'test_concurrent_futures', 'test_enum', 'test_functools', 'test_httpservers', diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 63a2e427d18..cfb009c203e 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -31,7 +31,7 @@ EXIT_TIMEOUT = 120.0 -ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', +ALL_RESOURCES = ('audio', 'console', 'curses', 'largefile', 'network', 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') # Other resources excluded from --use=all: @@ -294,6 +294,25 @@ def clear_caches(): else: importlib_metadata.FastPath.__new__.cache_clear() + try: + encodings = sys.modules['encodings'] + except KeyError: + pass + else: + encodings._cache.clear() + + try: + codecs = sys.modules['codecs'] + except KeyError: + pass + else: + # There's no direct API to clear the codecs search cache, but + # `unregister` clears it implicitly. + def noop_search_function(name): + return None + codecs.register(noop_search_function) + codecs.unregister(noop_search_function) + def get_build_info(): # Get most important configure and build options as a list of strings. @@ -536,7 +555,7 @@ def normalize_test_name(test_full_name: str, *, if is_error and short_name in _TEST_LIFECYCLE_HOOKS: if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): # if setUpModule() or tearDownModule() failed, don't filter - # tests with the test file name, don't use use filters. + # tests with the test file name, don't use filters. return None # This means that we have a failure in a life-cycle hook, diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 5d75bf7ae78..1ad67e1cebf 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -1,6 +1,7 @@ import subprocess import sys import os +from _colorize import can_colorize # type: ignore[import-not-found] from typing import Any, NoReturn from test.support import os_helper, Py_DEBUG @@ -32,6 +33,12 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, env['TEMP'] = tmp_dir env['TMP'] = tmp_dir + # The subcommand is run with a temporary output which means it is not a TTY + # and won't auto-color. The test results are printed to stdout so if we can + # color that have the subprocess use color. + if can_colorize(file=sys.stdout): + env['FORCE_COLOR'] = '1' + # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 68d6bad2094..e76f79c274e 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -32,13 +32,13 @@ def test_init(self): self.assertEqual(a, b) def test_getitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] def test_setitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] = "python" @@ -561,7 +561,7 @@ def test_constructor_exception_handling(self): class F(object): def __iter__(self): raise KeyboardInterrupt - self.assertRaises(KeyboardInterrupt, list, F()) + self.assertRaises(KeyboardInterrupt, self.type2test, F()) def test_exhausted_iterator(self): a = self.type2test([1, 2, 3]) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 009e04e9c0b..691029a1a54 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -124,6 +124,11 @@ def test_constructor(self): lock = self.locktype() del lock + def test_constructor_noargs(self): + self.assertRaises(TypeError, self.locktype, 1) + self.assertRaises(TypeError, self.locktype, x=1) + self.assertRaises(TypeError, self.locktype, 1, x=2) + def test_repr(self): lock = self.locktype() self.assertRegex(repr(lock), "") @@ -332,6 +337,26 @@ class RLockTests(BaseLockTests): """ Tests for recursive locks. """ + def test_repr_count(self): + # see gh-134322: check that count values are correct: + # when a rlock is just created, + # in a second thread when rlock is acquired in the main thread. + lock = self.locktype() + self.assertIn("count=0", repr(lock)) + self.assertIn(" memo_len * 2) instead of allocating + # a massive array. This test verifies large sparse indices work without + # causing memory exhaustion. + # + # The following simple pickle creates an empty list, memoizes it + # using a large index, then loads it back on the stack, builds + # a tuple containing 2 identical empty lists and returns it. + data = lambda n: (b'((lp' + str(n).encode() + b'\n' + + b'g' + str(n).encode() + b'\nt.') + # 0: ( MARK + # 1: ( MARK + # 2: l LIST (MARK at 1) + # 3: p PUT 1000000000000 + # 18: g GET 1000000000000 + # 33: t TUPLE (MARK at 0) + # 34: . STOP + for idx in [10**6, 10**9, 10**12]: + if idx > sys.maxsize: + continue + self.assertEqual(self.loads(data(idx)), ([],)*2) + + def test_too_large_long_binput(self): + # Test that LONG_BINPUT with large id does not cause allocation of + # too large memo table. The C implementation uses a dict-based memo + # for sparse indices (when idx > memo_len * 2) instead of allocating + # a massive array. This test verifies large sparse indices work without + # causing memory exhaustion. + # + # The following simple pickle creates an empty list, memoizes it + # using a large index, then loads it back on the stack, builds + # a tuple containing 2 identical empty lists and returns it. + data = lambda n: (b'(]r' + struct.pack(' sys.maxsize')) + + def test_truncated_large_binunicode8(self): + data = lambda size: b'\x8d' + struct.pack('= 2: proto_header = pickle.PROTO + bytes([proto]) - self.assertTrue(pickled.startswith(proto_header)) + self.assertStartsWith(pickled, proto_header) else: self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) @@ -4998,7 +5183,7 @@ def test_default_dispatch_table(self): p = self.pickler_class(f, 0) with self.assertRaises(AttributeError): p.dispatch_table - self.assertFalse(hasattr(p, 'dispatch_table')) + self.assertNotHasAttr(p, 'dispatch_table') def test_class_dispatch_table(self): # A dispatch_table attribute can be specified class-wide diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 682815c3fdd..3befc0f2e63 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -291,6 +291,8 @@ def format_groups(groups): "DISTUTILS_USE_SDK", "DYLD_LIBRARY_PATH", "ENSUREPIP_OPTIONS", + "FORCE_COLOR", + "GITHUB_ACTIONS", "HISTORY_FILE", "HOME", "HOMEDRIVE", @@ -307,11 +309,13 @@ def format_groups(groups): "MAKEFLAGS", "MIXERDEV", "MSSDK", + "NO_COLOR", "PATH", "PATHEXT", "PIP_CONFIG_FILE", "PLAT", "POSIXLY_CORRECT", + "PYTHON_COLORS", "PY_SAX_PARSER", "ProgramFiles", "ProgramFiles(x86)", @@ -658,6 +662,16 @@ def collect_zlib(info_add): copy_attributes(info_add, zlib, 'zlib.%s', attributes) +def collect_zstd(info_add): + try: + import _zstd + except ImportError: + return + + attributes = ('zstd_version',) + copy_attributes(info_add, _zstd, 'zstd.%s', attributes) + + def collect_expat(info_add): try: from xml.parsers import expat @@ -752,6 +766,7 @@ def collect_support(info_add): 'is_emscripten', 'is_jython', 'is_wasi', + 'is_wasm32', ) copy_attributes(info_add, support, 'support.%s', attributes) @@ -910,10 +925,17 @@ def collect_windows(info_add): try: import _winapi - dll_path = _winapi.GetModuleFileName(sys.dllhandle) - info_add('windows.dll_path', dll_path) - except (ImportError, AttributeError): + except ImportError: pass + else: + try: + dll_path = _winapi.GetModuleFileName(sys.dllhandle) + info_add('windows.dll_path', dll_path) + except AttributeError: + pass + + call_func(info_add, 'windows.ansi_code_page', _winapi, 'GetACP') + call_func(info_add, 'windows.oem_code_page', _winapi, 'GetOEMCP') # windows.version_caption: "wmic os get Caption,Version /value" command import subprocess @@ -1051,6 +1073,7 @@ def collect_info(info): collect_tkinter, collect_windows, collect_zlib, + collect_zstd, collect_libregrtest_utils, # Collecting from tests should be last as they have side effects. diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 4b82d51b450..185aa3fce39 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -90,6 +90,18 @@ def checkcall(self, obj, methodname, *args): args = self.fixtype(args) getattr(obj, methodname)(*args) + def _get_teststrings(self, charset, digits): + base = len(charset) + teststrings = set() + for i in range(base ** digits): + entry = [] + for j in range(digits): + i, m = divmod(i, base) + entry.append(charset[m]) + teststrings.add(''.join(entry)) + teststrings = [self.fixtype(ts) for ts in teststrings] + return teststrings + def test_count(self): self.checkequal(3, 'aaa', 'count', 'a') self.checkequal(0, 'aaa', 'count', 'b') @@ -130,17 +142,7 @@ def test_count(self): # For a variety of combinations, # verify that str.count() matches an equivalent function # replacing all occurrences and then differencing the string lengths - charset = ['', 'a', 'b'] - digits = 7 - base = len(charset) - teststrings = set() - for i in range(base ** digits): - entry = [] - for j in range(digits): - i, m = divmod(i, base) - entry.append(charset[m]) - teststrings.add(''.join(entry)) - teststrings = [self.fixtype(ts) for ts in teststrings] + teststrings = self._get_teststrings(['', 'a', 'b'], 7) for i in teststrings: n = len(i) for j in teststrings: @@ -197,17 +199,7 @@ def test_find(self): # For a variety of combinations, # verify that str.find() matches __contains__ # and that the found substring is really at that location - charset = ['', 'a', 'b', 'c'] - digits = 5 - base = len(charset) - teststrings = set() - for i in range(base ** digits): - entry = [] - for j in range(digits): - i, m = divmod(i, base) - entry.append(charset[m]) - teststrings.add(''.join(entry)) - teststrings = [self.fixtype(ts) for ts in teststrings] + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) for i in teststrings: for j in teststrings: loc = i.find(j) @@ -244,17 +236,7 @@ def test_rfind(self): # For a variety of combinations, # verify that str.rfind() matches __contains__ # and that the found substring is really at that location - charset = ['', 'a', 'b', 'c'] - digits = 5 - base = len(charset) - teststrings = set() - for i in range(base ** digits): - entry = [] - for j in range(digits): - i, m = divmod(i, base) - entry.append(charset[m]) - teststrings.add(''.join(entry)) - teststrings = [self.fixtype(ts) for ts in teststrings] + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) for i in teststrings: for j in teststrings: loc = i.rfind(j) @@ -295,6 +277,19 @@ def test_index(self): else: self.checkraises(TypeError, 'hello', 'index', 42) + # For a variety of combinations, + # verify that str.index() matches __contains__ + # and that the found substring is really at that location + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) + for i in teststrings: + for j in teststrings: + if j in i: + loc = i.index(j) + self.assertGreaterEqual(loc, 0) + self.assertEqual(i[loc:loc+len(j)], j) + else: + self.assertRaises(ValueError, i.index, j) + def test_rindex(self): self.checkequal(12, 'abcdefghiabc', 'rindex', '') self.checkequal(3, 'abcdefghiabc', 'rindex', 'def') @@ -321,6 +316,19 @@ def test_rindex(self): else: self.checkraises(TypeError, 'hello', 'rindex', 42) + # For a variety of combinations, + # verify that str.rindex() matches __contains__ + # and that the found substring is really at that location + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) + for i in teststrings: + for j in teststrings: + if j in i: + loc = i.rindex(j) + self.assertGreaterEqual(loc, 0) + self.assertEqual(i[loc:loc+len(j)], j) + else: + self.assertRaises(ValueError, i.rindex, j) + def test_find_periodic_pattern(self): """Cover the special path for periodic patterns.""" def reference_find(p, s): @@ -767,6 +775,15 @@ def test_replace(self): self.checkraises(TypeError, 'hello', 'replace', 42, 'h') self.checkraises(TypeError, 'hello', 'replace', 'h', 42) + def test_replacement_on_buffer_boundary(self): + # gh-127971: Check we don't read past the end of the buffer when a + # potential match misses on the last character. + any_3_nonblank_codepoints = '!!!' + seven_codepoints = any_3_nonblank_codepoints + ' ' + any_3_nonblank_codepoints + a = (' ' * 243) + seven_codepoints + (' ' * 7) + b = ' ' * 6 + chr(256) + a.replace(seven_codepoints, b) + def test_replace_uses_two_way_maxcount(self): # Test that maxcount works in _two_way_count in fastsearch.h A, B = "A"*1000, "B"*1000 diff --git a/Lib/test/subprocessdata/fd_status.py b/Lib/test/subprocessdata/fd_status.py index d12bd95abee..90e785981ae 100644 --- a/Lib/test/subprocessdata/fd_status.py +++ b/Lib/test/subprocessdata/fd_status.py @@ -2,7 +2,7 @@ file descriptors on stdout. Usage: -fd_stats.py: check all file descriptors +fd_status.py: check all file descriptors (up to 255) fd_status.py fd1 fd2 ...: check only specified file descriptors """ @@ -18,7 +18,7 @@ _MAXFD = os.sysconf("SC_OPEN_MAX") except: _MAXFD = 256 - test_fds = range(0, _MAXFD) + test_fds = range(0, min(_MAXFD, 256)) else: test_fds = map(int, sys.argv[1:]) for fd in test_fds: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c74c3a31909..0a50912ff0e 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -46,6 +46,7 @@ # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", + "support_remote_exec_only", # os "get_pagesize", # network @@ -67,7 +68,7 @@ "BrokenIter", "in_systemd_nspawn_sync_suppressed", "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", - "reset_code", + "reset_code", "on_github_actions" ] @@ -309,6 +310,16 @@ def requires(resource, msg=None): if resource == 'gui' and not _is_gui_available(): raise ResourceDenied(_is_gui_available.reason) +def _get_kernel_version(sysname="Linux"): + import platform + if platform.system() != sysname: + return None + version_txt = platform.release().split('-', 1)[0] + try: + return tuple(map(int, version_txt.split('.'))) + except ValueError: + return None + def _requires_unix_version(sysname, min_version): """Decorator raising SkipTest if the OS is `sysname` and the version is less than `min_version`. @@ -540,10 +551,14 @@ def has_no_debug_ranges(): except ImportError: raise unittest.SkipTest("_testinternalcapi required") return not _testcapi.config_get('code_debug_ranges') - return not bool(config['code_debug_ranges']) def requires_debug_ranges(reason='requires co_positions / debug_ranges'): - return unittest.skipIf(has_no_debug_ranges(), reason) + try: + skip = has_no_debug_ranges() + except unittest.SkipTest as e: + skip = True + reason = e.args[0] if e.args else reason + return unittest.skipIf(skip, reason) MS_WINDOWS = (sys.platform == 'win32') @@ -568,8 +583,11 @@ def skip_android_selinux(name): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" +# Use is_wasm32 as a generic check for WebAssembly platforms. +is_wasm32 = is_emscripten or is_wasi + def skip_emscripten_stack_overflow(): - return unittest.skipIf(is_emscripten, "Exhausts limited stack on Emscripten") + return unittest.skipIf(is_emscripten, "Exhausts stack on Emscripten") def skip_wasi_stack_overflow(): return unittest.skipIf(is_wasi, "Exhausts stack on WASI") @@ -696,9 +714,11 @@ def sortdict(dict): return "{%s}" % withcommas -def run_code(code: str) -> dict[str, object]: +def run_code(code: str, extra_names: dict[str, object] | None = None) -> dict[str, object]: """Run a piece of code after dedenting it, and return its global namespace.""" ns = {} + if extra_names: + ns.update(extra_names) exec(textwrap.dedent(code), ns) return ns @@ -943,6 +963,31 @@ def check_sizeof(test, o, size): % (type(o), result, size) test.assertEqual(result, size, msg) +def subTests(arg_names, arg_values, /, *, _do_cleanups=False): + """Run multiple subtests with different parameters. + """ + single_param = False + if isinstance(arg_names, str): + arg_names = arg_names.replace(',',' ').split() + if len(arg_names) == 1: + single_param = True + arg_values = tuple(arg_values) + def decorator(func): + if isinstance(func, type): + raise TypeError('subTests() can only decorate methods, not classes') + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + for values in arg_values: + if single_param: + values = (values,) + subtest_kwargs = dict(zip(arg_names, values)) + with self.subTest(**subtest_kwargs): + func(self, *args, **kwargs, **subtest_kwargs) + if _do_cleanups: + self.doCleanups() + return wrapper + return decorator + #======================================================================= # Decorator/context manager for running a code in a different locale, # correctly resetting it afterwards. @@ -1082,7 +1127,7 @@ def set_memlimit(limit: str) -> None: global real_max_memuse memlimit = _parse_memlimit(limit) if memlimit < _2G - 1: - raise ValueError('Memory limit {limit!r} too low to be useful') + raise ValueError(f'Memory limit {limit!r} too low to be useful') real_max_memuse = memlimit memlimit = min(memlimit, MAX_Py_ssize_t) @@ -1099,7 +1144,6 @@ def __init__(self): self.started = False def start(self): - import warnings try: f = open(self.procfile, 'r') except OSError as e: @@ -1325,6 +1369,7 @@ def reset_code(f: types.FunctionType) -> types.FunctionType: f.__code__ = f.__code__.replace() return f +on_github_actions = "GITHUB_ACTIONS" in os.environ #======================================================================= # Check for the presence of docstrings. @@ -1651,7 +1696,7 @@ def check__all__(test_case, module, name_of_module=None, extra=(), 'module'. The 'name_of_module' argument can specify (as a string or tuple thereof) - what module(s) an API could be defined in in order to be detected as a + what module(s) an API could be defined in order to be detected as a public API. One case for this is when 'module' imports part of its public API from other modules, possibly a C backend (like 'csv' and its '_csv'). @@ -2306,6 +2351,7 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): qualname = f"{name}" msg = f"cannot create '{re.escape(qualname)}' instances" testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) + testcase.assertRaisesRegex(TypeError, msg, tp.__new__, tp, *args, **kwds) def get_recursion_depth(): """Get the recursion depth of the caller function. @@ -2357,7 +2403,7 @@ def infinite_recursion(max_depth=None): # very deep recursion. max_depth = 20_000 elif max_depth < 3: - raise ValueError("max_depth must be at least 3, got {max_depth}") + raise ValueError(f"max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. limit = depth + max_depth @@ -2726,7 +2772,7 @@ def iter_builtin_types(): # Fall back to making a best-effort guess. if hasattr(object, '__flags__'): # Look for any type object with the Py_TPFLAGS_STATIC_BUILTIN flag set. - import datetime + import datetime # noqa: F401 seen = set() for cls, subs in walk_class_hierarchy(object): if cls in seen: @@ -2863,7 +2909,7 @@ def force_color(color: bool): from .os_helper import EnvironmentVarGuard with ( - swap_attr(_colorize, "can_colorize", lambda file=None: color), + swap_attr(_colorize, "can_colorize", lambda *, file=None: color), EnvironmentVarGuard() as env, ): env.unset("FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS") @@ -2927,12 +2973,6 @@ def make_clean_env() -> dict[str, str]: return clean_env -def initialized_with_pyrepl(): - """Detect whether PyREPL was used during Python initialization.""" - # If the main module has a __file__ attribute it's a Python module, which means PyREPL. - return hasattr(sys.modules["__main__"], "__file__") - - WINDOWS_STATUS = { 0xC0000005: "STATUS_ACCESS_VIOLATION", 0xC00000FD: "STATUS_STACK_OVERFLOW", @@ -3049,6 +3089,27 @@ def is_libssl_fips_mode(): return False # more of a maybe, unless we add this to the _ssl module. return get_fips_mode() != 0 +def _supports_remote_attaching(): + PROCESS_VM_READV_SUPPORTED = False + + try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED + except ImportError: + pass + + return PROCESS_VM_READV_SUPPORTED + +def _support_remote_exec_only_impl(): + if not sys.is_remote_debug_enabled(): + return unittest.skip("Remote debugging is not enabled") + if sys.platform not in ("darwin", "linux", "win32"): + return unittest.skip("Test only runs on Linux, Windows and macOS") + if sys.platform == "linux" and not _supports_remote_attaching(): + return unittest.skip("Test only runs on Linux with process_vm_readv support") + return _id + +def support_remote_exec_only(test): + return _support_remote_exec_only_impl()(test) class EqualToForwardRef: """Helper to ease use of annotationlib.ForwardRef in tests. @@ -3105,7 +3166,7 @@ def linked_to_musl(): # emscripten (at least as far as we're concerned) and wasi use musl, # but platform doesn't know how to get the version, so set it to zero. - if is_emscripten or is_wasi: + if is_wasm32: _linked_to_musl = (0, 0, 0) return _linked_to_musl diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py index 6ba5bb814b9..6fa013b55b2 100644 --- a/Lib/test/support/_hypothesis_stubs/__init__.py +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -24,7 +24,13 @@ def decorator(f): @functools.wraps(f) def test_function(self): for example_args, example_kwargs in examples: - with self.subTest(*example_args, **example_kwargs): + if len(example_args) < 2: + subtest_args = example_args + else: + # subTest takes up to one positional argument. + # When there are more, display them as a tuple + subtest_args = [example_args] + with self.subTest(*subtest_args, **example_kwargs): f(self, *example_args, **example_kwargs) else: diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/channels.py similarity index 73% rename from Lib/test/support/interpreters/channels.py rename to Lib/test/support/channels.py index d2bd93d77f7..fab1797659b 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/channels.py @@ -2,14 +2,14 @@ import time import _interpchannels as _channels -from . import _crossinterp +from concurrent.interpreters import _crossinterp # aliases: from _interpchannels import ( - ChannelError, ChannelNotFoundError, ChannelClosedError, - ChannelEmptyError, ChannelNotEmptyError, + ChannelError, ChannelNotFoundError, ChannelClosedError, # noqa: F401 + ChannelEmptyError, ChannelNotEmptyError, # noqa: F401 ) -from ._crossinterp import ( +from concurrent.interpreters._crossinterp import ( UNBOUND_ERROR, UNBOUND_REMOVE, ) @@ -55,15 +55,23 @@ def create(*, unbounditems=UNBOUND): """ unbound = _serialize_unbound(unbounditems) unboundop, = unbound - cid = _channels.create(unboundop) - recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound) + cid = _channels.create(unboundop, -1) + recv, send = RecvChannel(cid), SendChannel(cid) + send._set_unbound(unboundop, unbounditems) return recv, send def list_all(): """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound)) - for cid, unbound in _channels.list_all()] + channels = [] + for cid, unboundop, _ in _channels.list_all(): + chan = _, send = RecvChannel(cid), SendChannel(cid) + if not hasattr(send, '_unboundop'): + send._set_unbound(unboundop) + else: + assert send._unbound[0] == unboundop + channels.append(chan) + return channels class _ChannelEnd: @@ -97,12 +105,8 @@ def __eq__(self, other): return other._id == self._id # for pickling: - def __getnewargs__(self): - return (int(self._id),) - - # for pickling: - def __getstate__(self): - return None + def __reduce__(self): + return (type(self), (int(self._id),)) @property def id(self): @@ -175,16 +179,33 @@ class SendChannel(_ChannelEnd): _end = 'send' - def __new__(cls, cid, *, _unbound=None): - if _unbound is None: - try: - op = _channels.get_channel_defaults(cid) - _unbound = (op,) - except ChannelNotFoundError: - _unbound = _serialize_unbound(UNBOUND) - self = super().__new__(cls, cid) - self._unbound = _unbound - return self +# def __new__(cls, cid, *, _unbound=None): +# if _unbound is None: +# try: +# op = _channels.get_channel_defaults(cid) +# _unbound = (op,) +# except ChannelNotFoundError: +# _unbound = _serialize_unbound(UNBOUND) +# self = super().__new__(cls, cid) +# self._unbound = _unbound +# return self + + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound + + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _channels.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items @property def is_closed(self): @@ -192,61 +213,61 @@ def is_closed(self): return info.closed or info.closing def send(self, obj, timeout=None, *, - unbound=None, + unbounditems=None, ): """Send the object (i.e. its data) to the channel's receiving end. This blocks until the object is received. """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) def send_nowait(self, obj, *, - unbound=None, + unbounditems=None, ): """Send the object to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) # XXX Note that at the moment channel_send() only ever returns # None. This should be fixed when channel_send_wait() is added. # See bpo-32604 and gh-19829. return _channels.send(self._id, obj, unboundop, blocking=False) def send_buffer(self, obj, timeout=None, *, - unbound=None, + unbounditems=None, ): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) _channels.send_buffer(self._id, obj, unboundop, timeout=timeout, blocking=True) def send_buffer_nowait(self, obj, *, - unbound=None, + unbounditems=None, ): """Send the object's buffer to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) return _channels.send_buffer(self._id, obj, unboundop, blocking=False) def close(self): diff --git a/Lib/test/support/hashlib_helper.py b/Lib/test/support/hashlib_helper.py index 5043f08dd93..49077d7cb4d 100644 --- a/Lib/test/support/hashlib_helper.py +++ b/Lib/test/support/hashlib_helper.py @@ -1,29 +1,516 @@ +import contextlib +import enum import functools -import hashlib import importlib +import inspect import unittest -from test.support.import_helper import import_module - -try: - import _hashlib -except ImportError: - _hashlib = None - -try: - import _hmac -except ImportError: - _hmac = None +import unittest.mock +from test.support import import_helper +from types import MappingProxyType -def requires_hashlib(): - return unittest.skipIf(_hashlib is None, "requires _hashlib") +def _parse_fullname(fullname, *, strict=False): + """Parse a fully-qualified name ``.``. + + The ``module_name`` component contains one or more dots. + The ``member_name`` component does not contain any dot. + + If *strict* is true, *fullname* must be a string. Otherwise, + it can be None, and, ``module_name`` and ``member_name`` will + also be None. + """ + if fullname is None: + assert not strict + return None, None + assert isinstance(fullname, str), fullname + assert fullname.count(".") >= 1, fullname + module_name, member_name = fullname.rsplit(".", maxsplit=1) + return module_name, member_name -def requires_builtin_hmac(): - return unittest.skipIf(_hmac is None, "requires _hmac") +def _import_module(module_name, *, strict=False): + """Import a module from its fully-qualified name. + + If *strict* is false, import failures are suppressed and None is returned. + """ + if module_name is None: + # To prevent a TypeError in importlib.import_module + if strict: + raise ImportError("no module to import") + return None + try: + return importlib.import_module(module_name) + except ImportError as exc: + if strict: + raise exc + return None -def _decorate_func_or_class(func_or_class, decorator_func): +def _import_member(module_name, member_name, *, strict=False): + """Import a member from a module. + + If *strict* is false, import failures are suppressed and None is returned. + """ + if member_name is None: + if strict: + raise ImportError(f"no member to import from {module_name}") + return None + module = _import_module(module_name, strict=strict) + if strict: + return getattr(module, member_name) + return getattr(module, member_name, None) + + +class Implementation(enum.StrEnum): + # Indicate that the hash function is implemented by a built-in module. + builtin = enum.auto() + # Indicate that the hash function is implemented by OpenSSL. + openssl = enum.auto() + # Indicate that the hash function is provided through the public API. + hashlib = enum.auto() + + +class _HashId(enum.StrEnum): + """Enumeration containing the canonical digest names. + + Those names should only be used by hashlib.new() or hmac.new(). + Their support by _hashlib.new() is not necessarily guaranteed. + """ + + md5 = enum.auto() + sha1 = enum.auto() + + sha224 = enum.auto() + sha256 = enum.auto() + sha384 = enum.auto() + sha512 = enum.auto() + + sha3_224 = enum.auto() + sha3_256 = enum.auto() + sha3_384 = enum.auto() + sha3_512 = enum.auto() + + shake_128 = enum.auto() + shake_256 = enum.auto() + + blake2s = enum.auto() + blake2b = enum.auto() + + def __repr__(self): + return str(self) + + @property + def is_xof(self): + """Indicate whether the hash is an extendable-output hash function.""" + return self.startswith("shake_") + + @property + def is_keyed(self): + """Indicate whether the hash is a keyed hash function.""" + return self.startswith("blake2") + + +CANONICAL_DIGEST_NAMES = frozenset(map(str, _HashId.__members__)) +NON_HMAC_DIGEST_NAMES = frozenset(( + _HashId.shake_128, _HashId.shake_256, + _HashId.blake2s, _HashId.blake2b, +)) + + +class _HashInfoItem: + """Interface for interacting with a named object. + + The object is entirely described by its fully-qualified *fullname*. + + *fullname* must be None or a string ".". + """ + + def __init__(self, fullname=None, *, strict=False): + module_name, member_name = _parse_fullname(fullname, strict=strict) + self.fullname = fullname + self.module_name = module_name + self.member_name = member_name + + def import_module(self, *, strict=False): + """Import the described module. + + If *strict* is true, an ImportError may be raised if importing fails, + otherwise, None is returned on error. + """ + return _import_module(self.module_name, strict=strict) + + def import_member(self, *, strict=False): + """Import the described member. + + If *strict* is true, an AttributeError or an ImportError may be + raised if importing fails; otherwise, None is returned on error. + """ + return _import_member( + self.module_name, self.member_name, strict=strict + ) + + +class _HashInfoBase: + """Base dataclass containing "backend" information. + + Subclasses may define an attribute named after one of the known + implementations ("builtin", "openssl" or "hashlib") which stores + an _HashInfoItem object. + + Those attributes can be retrieved through __getitem__(), e.g., + ``info["builtin"]`` returns the _HashInfoItem corresponding to + the builtin implementation. + """ + + def __init__(self, canonical_name): + assert isinstance(canonical_name, _HashId), canonical_name + self.canonical_name = canonical_name + + def __getitem__(self, implementation): + try: + attrname = Implementation(implementation) + except ValueError: + raise self.invalid_implementation_error(implementation) from None + + try: + provider = getattr(self, attrname) + except AttributeError: + raise self.invalid_implementation_error(implementation) from None + + if not isinstance(provider, _HashInfoItem): + raise KeyError(implementation) + return provider + + def invalid_implementation_error(self, implementation): + msg = f"no implementation {implementation} for {self.canonical_name}" + return AssertionError(msg) + + +class _HashTypeInfo(_HashInfoBase): + """Dataclass containing information for hash functions types. + + - *canonical_name* must be a _HashId. + + - *builtin* is the fully-qualified name for the builtin HACL* type, + e.g., "_md5.MD5Type". + + - *openssl* is the fully-qualified name for the OpenSSL wrapper type, + e.g., "_hashlib.HASH". + """ + + def __init__(self, canonical_name, builtin, openssl): + super().__init__(canonical_name) + self.builtin = _HashInfoItem(builtin, strict=True) + self.openssl = _HashInfoItem(openssl, strict=True) + + def fullname(self, implementation): + """Get the fully qualified name of a given implementation. + + This returns a string of the form "MODULE_NAME.OBJECT_NAME" or None + if the hash function does not have a corresponding implementation. + + *implementation* must be "builtin" or "openssl". + """ + return self[implementation].fullname + + def module_name(self, implementation): + """Get the name of the module containing the hash object type.""" + return self[implementation].module_name + + def object_type_name(self, implementation): + """Get the name of the hash object class name.""" + return self[implementation].member_name + + def import_module(self, implementation, *, allow_skip=False): + """Import the module containing the hash object type. + + On error, return None if *allow_skip* is false, or raise SkipNoHash. + """ + target = self[implementation] + module = target.import_module() + if allow_skip and module is None: + reason = f"cannot import module {target.module_name}" + raise SkipNoHash(self.canonical_name, implementation, reason) + return module + + def import_object_type(self, implementation, *, allow_skip=False): + """Get the runtime hash object type. + + On error, return None if *allow_skip* is false, or raise SkipNoHash. + """ + target = self[implementation] + member = target.import_member() + if allow_skip and member is None: + reason = f"cannot import class {target.fullname}" + raise SkipNoHash(self.canonical_name, implementation, reason) + return member + + +class _HashFuncInfo(_HashInfoBase): + """Dataclass containing information for hash functions constructors. + + - *canonical_name* must be a _HashId. + + - *builtin* is the fully-qualified name of the HACL* + hash constructor function, e.g., "_md5.md5". + + - *openssl* is the fully-qualified name of the "_hashlib" method + for the OpenSSL named constructor, e.g., "_hashlib.openssl_md5". + + - *hashlib* is the fully-qualified name of the "hashlib" method + for the explicit named hash constructor, e.g., "hashlib.md5". + """ + + def __init__(self, canonical_name, builtin, openssl=None, hashlib=None): + super().__init__(canonical_name) + self.builtin = _HashInfoItem(builtin, strict=True) + self.openssl = _HashInfoItem(openssl, strict=False) + self.hashlib = _HashInfoItem(hashlib, strict=False) + + def fullname(self, implementation): + """Get the fully qualified name of a given implementation. + + This returns a string of the form "MODULE_NAME.METHOD_NAME" or None + if the hash function does not have a corresponding implementation. + + *implementation* must be "builtin", "openssl" or "hashlib". + """ + return self[implementation].fullname + + def module_name(self, implementation): + """Get the name of the constructor function module. + + The *implementation* must be "builtin", "openssl" or "hashlib". + """ + return self[implementation].module_name + + def method_name(self, implementation): + """Get the name of the constructor function module method. + + Use fullname() to get the constructor function fully-qualified name. + + The *implementation* must be "builtin", "openssl" or "hashlib". + """ + return self[implementation].member_name + + +class _HashInfo: + """Dataclass containing information for supported hash functions. + + Attributes + ---------- + canonical_name : _HashId + The hash function canonical name. + type : _HashTypeInfo + The hash object types information. + func : _HashTypeInfo + The hash object constructors information. + """ + + def __init__( + self, + canonical_name, + builtin_object_type_fullname, + openssl_object_type_fullname, + builtin_method_fullname, + openssl_method_fullname=None, + hashlib_method_fullname=None, + ): + """ + - *canonical_name* must be a _HashId. + + - *builtin_object_type_fullname* is the fully-qualified name + for the builtin HACL* type, e.g., "_md5.MD5Type". + + - *openssl_object_type_fullname* is the fully-qualified name + for the OpenSSL wrapper type, e.g., "_hashlib.HASH". + + - *builtin_method_fullname* is the fully-qualified name + of the HACL* hash constructor function, e.g., "_md5.md5". + + - *openssl_method_fullname* is the fully-qualified name + of the "_hashlib" module method for the explicit OpenSSL + hash constructor function, e.g., "_hashlib.openssl_md5". + + - *hashlib_method_fullname* is the fully-qualified name + of the "hashlib" module method for the explicit hash + constructor function, e.g., "hashlib.md5". + """ + assert isinstance(canonical_name, _HashId), canonical_name + self.canonical_name = canonical_name + self.type = _HashTypeInfo( + canonical_name, + builtin_object_type_fullname, + openssl_object_type_fullname, + ) + self.func = _HashFuncInfo( + canonical_name, + builtin_method_fullname, + openssl_method_fullname, + hashlib_method_fullname, + ) + + +_HASHINFO_DATABASE = MappingProxyType({ + _HashId.md5: _HashInfo( + _HashId.md5, + "_md5.MD5Type", + "_hashlib.HASH", + "_md5.md5", + "_hashlib.openssl_md5", + "hashlib.md5", + ), + _HashId.sha1: _HashInfo( + _HashId.sha1, + "_sha1.SHA1Type", + "_hashlib.HASH", + "_sha1.sha1", + "_hashlib.openssl_sha1", + "hashlib.sha1", + ), + _HashId.sha224: _HashInfo( + _HashId.sha224, + "_sha2.SHA224Type", + "_hashlib.HASH", + "_sha2.sha224", + "_hashlib.openssl_sha224", + "hashlib.sha224", + ), + _HashId.sha256: _HashInfo( + _HashId.sha256, + "_sha2.SHA256Type", + "_hashlib.HASH", + "_sha2.sha256", + "_hashlib.openssl_sha256", + "hashlib.sha256", + ), + _HashId.sha384: _HashInfo( + _HashId.sha384, + "_sha2.SHA384Type", + "_hashlib.HASH", + "_sha2.sha384", + "_hashlib.openssl_sha384", + "hashlib.sha384", + ), + _HashId.sha512: _HashInfo( + _HashId.sha512, + "_sha2.SHA512Type", + "_hashlib.HASH", + "_sha2.sha512", + "_hashlib.openssl_sha512", + "hashlib.sha512", + ), + _HashId.sha3_224: _HashInfo( + _HashId.sha3_224, + "_sha3.sha3_224", + "_hashlib.HASH", + "_sha3.sha3_224", + "_hashlib.openssl_sha3_224", + "hashlib.sha3_224", + ), + _HashId.sha3_256: _HashInfo( + _HashId.sha3_256, + "_sha3.sha3_256", + "_hashlib.HASH", + "_sha3.sha3_256", + "_hashlib.openssl_sha3_256", + "hashlib.sha3_256", + ), + _HashId.sha3_384: _HashInfo( + _HashId.sha3_384, + "_sha3.sha3_384", + "_hashlib.HASH", + "_sha3.sha3_384", + "_hashlib.openssl_sha3_384", + "hashlib.sha3_384", + ), + _HashId.sha3_512: _HashInfo( + _HashId.sha3_512, + "_sha3.sha3_512", + "_hashlib.HASH", + "_sha3.sha3_512", + "_hashlib.openssl_sha3_512", + "hashlib.sha3_512", + ), + _HashId.shake_128: _HashInfo( + _HashId.shake_128, + "_sha3.shake_128", + "_hashlib.HASHXOF", + "_sha3.shake_128", + "_hashlib.openssl_shake_128", + "hashlib.shake_128", + ), + _HashId.shake_256: _HashInfo( + _HashId.shake_256, + "_sha3.shake_256", + "_hashlib.HASHXOF", + "_sha3.shake_256", + "_hashlib.openssl_shake_256", + "hashlib.shake_256", + ), + _HashId.blake2s: _HashInfo( + _HashId.blake2s, + "_blake2.blake2s", + "_hashlib.HASH", + "_blake2.blake2s", + None, + "hashlib.blake2s", + ), + _HashId.blake2b: _HashInfo( + _HashId.blake2b, + "_blake2.blake2b", + "_hashlib.HASH", + "_blake2.blake2b", + None, + "hashlib.blake2b", + ), +}) +assert _HASHINFO_DATABASE.keys() == CANONICAL_DIGEST_NAMES + + +def get_hash_type_info(name): + info = _HASHINFO_DATABASE[name] + assert isinstance(info, _HashInfo), info + return info.type + + +def get_hash_func_info(name): + info = _HASHINFO_DATABASE[name] + assert isinstance(info, _HashInfo), info + return info.func + + +def _iter_hash_func_info(excluded): + for name, info in _HASHINFO_DATABASE.items(): + if name not in excluded: + yield info.func + + +# Mapping from canonical hash names to their explicit HACL* HMAC constructor. +# There is currently no OpenSSL one-shot named function and there will likely +# be none in the future. +_HMACINFO_DATABASE = { + _HashId(canonical_name): _HashInfoItem(f"_hmac.compute_{canonical_name}") + for canonical_name in CANONICAL_DIGEST_NAMES +} +# Neither HACL* nor OpenSSL supports HMAC over XOFs. +_HMACINFO_DATABASE[_HashId.shake_128] = _HashInfoItem() +_HMACINFO_DATABASE[_HashId.shake_256] = _HashInfoItem() +# Strictly speaking, HMAC-BLAKE is meaningless as BLAKE2 is already a +# keyed hash function. However, as it's exposed by HACL*, we test it. +_HMACINFO_DATABASE[_HashId.blake2s] = _HashInfoItem('_hmac.compute_blake2s_32') +_HMACINFO_DATABASE[_HashId.blake2b] = _HashInfoItem('_hmac.compute_blake2b_32') +_HMACINFO_DATABASE = MappingProxyType(_HMACINFO_DATABASE) +assert _HMACINFO_DATABASE.keys() == CANONICAL_DIGEST_NAMES + + +def get_hmac_item_info(name): + info = _HMACINFO_DATABASE[name] + assert isinstance(info, _HashInfoItem), info + return info + + +def _decorate_func_or_class(decorator_func, func_or_class): if not isinstance(func_or_class, type): return decorator_func(func_or_class) @@ -41,7 +528,167 @@ def setUpClass(cls): return decorated_class -def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): +def _chain_decorators(decorators): + """Obtain a decorator by chaining multiple decorators. + + The decorators are applied in the order they are given. + """ + def decorator_func(func): + return functools.reduce(lambda w, deco: deco(w), decorators, func) + return functools.partial(_decorate_func_or_class, decorator_func) + + +def _ensure_wrapper_signature(wrapper, wrapped): + """Ensure that a wrapper has the same signature as the wrapped function. + + This is used to guarantee that a TypeError raised due to a bad API call + is raised consistently (using variadic signatures would hide such errors). + """ + try: + wrapped_sig = inspect.signature(wrapped) + except ValueError: # built-in signature cannot be found + return + + wrapper_sig = inspect.signature(wrapper) + if wrapped_sig != wrapper_sig: + fullname = f"{wrapped.__module__}.{wrapped.__qualname__}" + raise AssertionError( + f"signature for {fullname}() is incorrect:\n" + f" expect: {wrapped_sig}\n" + f" actual: {wrapper_sig}" + ) + + +def _make_conditional_decorator(test, /, *test_args, **test_kwargs): + def decorator_func(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + test(*test_args, **test_kwargs) + return func(*args, **kwargs) + return wrapper + return functools.partial(_decorate_func_or_class, decorator_func) + + +def requires_openssl_hashlib(): + _hashlib = _import_module("_hashlib") + return unittest.skipIf(_hashlib is None, "requires _hashlib") + + +def requires_builtin_hmac(): + _hmac = _import_module("_hmac") + return unittest.skipIf(_hmac is None, "requires _hmac") + + +class SkipNoHash(unittest.SkipTest): + """A SkipTest exception raised when a hash is not available.""" + + def __init__(self, digestname, implementation=None, reason=None): + parts = ["missing", implementation, f"hash algorithm {digestname!r}"] + if reason is not None: + parts.insert(0, f"{reason}: ") + super().__init__(" ".join(filter(None, parts))) + + +class SkipNoHashInCall(SkipNoHash): + + def __init__(self, func, digestname, implementation=None): + super().__init__(digestname, implementation, f"cannot use {func}") + + +def _hashlib_new(digestname, openssl, /, **kwargs): + """Check availability of [hashlib|_hashlib].new(digestname, **kwargs). + + If *openssl* is True, module is "_hashlib" (C extension module), + otherwise it is "hashlib" (pure Python interface). + + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. + """ + assert isinstance(digestname, str), digestname + # Re-import 'hashlib' in case it was mocked, but propagate + # exceptions as it should be unconditionally available. + hashlib = importlib.import_module("hashlib") + # re-import '_hashlib' in case it was mocked + _hashlib = _import_module("_hashlib") + module = _hashlib if openssl and _hashlib is not None else hashlib + try: + module.new(digestname, **kwargs) + except ValueError as exc: + raise SkipNoHashInCall(f"{module.__name__}.new", digestname) from exc + return functools.partial(module.new, digestname) + + +def _builtin_hash(module_name, digestname, /, **kwargs): + """Check availability of .(**kwargs). + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + + The constructor function is returned, or SkipTest is raised if none exists. + """ + assert isinstance(module_name, str), module_name + assert isinstance(digestname, str), digestname + fullname = f'{module_name}.{digestname}' + try: + builtin_module = importlib.import_module(module_name) + except ImportError as exc: + raise SkipNoHash(fullname, "builtin") from exc + try: + constructor = getattr(builtin_module, digestname) + except AttributeError as exc: + raise SkipNoHash(fullname, "builtin") from exc + try: + constructor(**kwargs) + except ValueError as exc: + raise SkipNoHash(fullname, "builtin") from exc + return constructor + + +def _openssl_new(digestname, /, **kwargs): + """Check availability of _hashlib.new(digestname, **kwargs). + + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. + """ + assert isinstance(digestname, str), digestname + try: + # re-import '_hashlib' in case it was mocked + _hashlib = importlib.import_module("_hashlib") + except ImportError as exc: + raise SkipNoHash(digestname, "openssl") from exc + try: + _hashlib.new(digestname, **kwargs) + except ValueError as exc: + raise SkipNoHashInCall("_hashlib.new", digestname) from exc + return functools.partial(_hashlib.new, digestname) + + +def _openssl_hash(digestname, /, **kwargs): + """Check availability of _hashlib.openssl_(**kwargs). + + The constructor function is returned (without binding **kwargs), + or SkipTest is raised if none exists. + """ + assert isinstance(digestname, str), digestname + method_name = f"openssl_{digestname}" + fullname = f"_hashlib.{method_name}" + try: + # re-import '_hashlib' in case it was mocked + _hashlib = importlib.import_module("_hashlib") + except ImportError as exc: + raise SkipNoHash(fullname, "openssl") from exc + try: + constructor = getattr(_hashlib, method_name) + except AttributeError as exc: + raise SkipNoHash(fullname, "openssl") from exc + try: + constructor(**kwargs) + except ValueError as exc: + raise SkipNoHash(fullname, "openssl") from exc + return constructor + + +def requires_hashdigest(digestname, openssl=None, *, usedforsecurity=True): """Decorator raising SkipTest if a hashing algorithm is not available. The hashing algorithm may be missing, blocked by a strict crypto policy, @@ -58,27 +705,9 @@ def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS ValueError: unsupported hash type md4 """ - if openssl and _hashlib is not None: - def test_availability(): - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - else: - def test_availability(): - hashlib.new(digestname, usedforsecurity=usedforsecurity) - - def decorator_func(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - test_availability() - except ValueError as exc: - msg = f"missing hash algorithm: {digestname!r}" - raise unittest.SkipTest(msg) from exc - return func(*args, **kwargs) - return wrapper - - def decorator(func_or_class): - return _decorate_func_or_class(func_or_class, decorator_func) - return decorator + return _make_conditional_decorator( + _hashlib_new, digestname, openssl, usedforsecurity=usedforsecurity + ) def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): @@ -86,21 +715,118 @@ def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): The hashing algorithm may be missing or blocked by a strict crypto policy. """ - def decorator_func(func): - @requires_hashlib() - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - except ValueError: - msg = f"missing OpenSSL hash algorithm: {digestname!r}" - raise unittest.SkipTest(msg) - return func(*args, **kwargs) - return wrapper + return _make_conditional_decorator( + _openssl_new, digestname, usedforsecurity=usedforsecurity + ) - def decorator(func_or_class): - return _decorate_func_or_class(func_or_class, decorator_func) - return decorator + +def _make_requires_builtin_hashdigest_decorator(item, *, usedforsecurity=True): + assert isinstance(item, _HashInfoItem), item + return _make_conditional_decorator( + _builtin_hash, + item.module_name, + item.member_name, + usedforsecurity=usedforsecurity, + ) + + +def requires_builtin_hashdigest(canonical_name, *, usedforsecurity=True): + """Decorator raising SkipTest if a HACL* hashing algorithm is missing.""" + info = get_hash_func_info(canonical_name) + return _make_requires_builtin_hashdigest_decorator( + info.builtin, usedforsecurity=usedforsecurity + ) + + +def requires_builtin_hashes(*, exclude=(), usedforsecurity=True): + """Decorator raising SkipTest if one HACL* hashing algorithm is missing.""" + return _chain_decorators(( + _make_requires_builtin_hashdigest_decorator( + info.builtin, usedforsecurity=usedforsecurity + ) for info in _iter_hash_func_info(exclude) + )) + + +class HashFunctionsTrait: + """Mixin trait class containing hash functions. + + This class is assumed to have all unitest.TestCase methods but should + not directly inherit from it to prevent the test suite being run on it. + + Subclasses should implement the hash functions by returning an object + that can be recognized as a valid digestmod parameter for both hashlib + and HMAC. In particular, it cannot be a lambda function as it will not + be recognized by hashlib (it will still be accepted by the pure Python + implementation of HMAC). + """ + + # Default 'usedforsecurity' to use when checking a hash function. + # When the trait properties are callables (e.g., _md5.md5) and + # not strings, they must be called with the same 'usedforsecurity'. + usedforsecurity = True + + def is_valid_digest_name(self, digestname): + self.assertIn(digestname, _HashId) + + def _find_constructor(self, digestname): + # By default, a missing algorithm skips the test that uses it. + self.is_valid_digest_name(digestname) + self.skipTest(f"missing hash function: {digestname}") + + md5 = property(lambda self: self._find_constructor("md5")) + sha1 = property(lambda self: self._find_constructor("sha1")) + + sha224 = property(lambda self: self._find_constructor("sha224")) + sha256 = property(lambda self: self._find_constructor("sha256")) + sha384 = property(lambda self: self._find_constructor("sha384")) + sha512 = property(lambda self: self._find_constructor("sha512")) + + sha3_224 = property(lambda self: self._find_constructor("sha3_224")) + sha3_256 = property(lambda self: self._find_constructor("sha3_256")) + sha3_384 = property(lambda self: self._find_constructor("sha3_384")) + sha3_512 = property(lambda self: self._find_constructor("sha3_512")) + + +class NamedHashFunctionsTrait(HashFunctionsTrait): + """Trait containing named hash functions. + + Hash functions are available if and only if they are available in hashlib. + """ + + def _find_constructor(self, digestname): + self.is_valid_digest_name(digestname) + return str(digestname) # ensure that we are an exact string + + +class OpenSSLHashFunctionsTrait(HashFunctionsTrait): + """Trait containing OpenSSL hash functions. + + Hash functions are available if and only if they are available in _hashlib. + """ + + def _find_constructor(self, digestname): + self.is_valid_digest_name(digestname) + # This returns a function of the form _hashlib.openssl_ and + # not a lambda function as it is rejected by _hashlib.hmac_new(). + return _openssl_hash(digestname, usedforsecurity=self.usedforsecurity) + + +class BuiltinHashFunctionsTrait(HashFunctionsTrait): + """Trait containing HACL* hash functions. + + Hash functions are available if and only if they are available in C. + In particular, HACL* HMAC-MD5 may be available even though HACL* md5 + is not since the former is unconditionally built. + """ + + def _find_constructor(self, digestname): + self.is_valid_digest_name(digestname) + info = get_hash_func_info(digestname) + return _builtin_hash( + info.builtin.module_name, + info.builtin.member_name, + usedforsecurity=self.usedforsecurity, + ) def find_gil_minsize(modules_names, default=2048): @@ -114,9 +840,257 @@ def find_gil_minsize(modules_names, default=2048): """ sizes = [] for module_name in modules_names: - try: - module = importlib.import_module(module_name) - except ImportError: - continue - sizes.append(getattr(module, '_GIL_MINSIZE', default)) + module = _import_module(module_name) + if module is not None: + sizes.append(getattr(module, '_GIL_MINSIZE', default)) return max(sizes, default=default) + + +def _block_openssl_hash_new(blocked_name): + """Block OpenSSL implementation of _hashlib.new().""" + assert isinstance(blocked_name, str), blocked_name + + # re-import '_hashlib' in case it was mocked + if (_hashlib := _import_module("_hashlib")) is None: + return contextlib.nullcontext() + + @functools.wraps(wrapped := _hashlib.new) + def _hashlib_new(name, data=b'', *, usedforsecurity=True, string=None): + if name == blocked_name: + raise _hashlib.UnsupportedDigestmodError(blocked_name) + return wrapped(name, data, + usedforsecurity=usedforsecurity, string=string) + + _ensure_wrapper_signature(_hashlib_new, wrapped) + return unittest.mock.patch('_hashlib.new', _hashlib_new) + + +def _block_openssl_hmac_new(blocked_name): + """Block OpenSSL HMAC-HASH implementation.""" + assert isinstance(blocked_name, str), blocked_name + + # re-import '_hashlib' in case it was mocked + if (_hashlib := _import_module("_hashlib")) is None: + return contextlib.nullcontext() + + @functools.wraps(wrapped := _hashlib.hmac_new) + def wrapper(key, msg=b'', digestmod=None): + if digestmod == blocked_name: + raise _hashlib.UnsupportedDigestmodError(blocked_name) + return wrapped(key, msg, digestmod) + + _ensure_wrapper_signature(wrapper, wrapped) + return unittest.mock.patch('_hashlib.hmac_new', wrapper) + + +def _block_openssl_hmac_digest(blocked_name): + """Block OpenSSL HMAC-HASH one-shot digest implementation.""" + assert isinstance(blocked_name, str), blocked_name + + # re-import '_hashlib' in case it was mocked + if (_hashlib := _import_module("_hashlib")) is None: + return contextlib.nullcontext() + + @functools.wraps(wrapped := _hashlib.hmac_digest) + def _hashlib_hmac_digest(key, msg, digest): + if digest == blocked_name: + raise _hashlib.UnsupportedDigestmodError(blocked_name) + return wrapped(key, msg, digest) + + _ensure_wrapper_signature(_hashlib_hmac_digest, wrapped) + return unittest.mock.patch('_hashlib.hmac_digest', _hashlib_hmac_digest) + + +def _block_builtin_hash_new(name): + """Block a buitin-in hash name from the hashlib.new() interface.""" + assert isinstance(name, str), name + assert name.lower() == name, f"invalid name: {name}" + assert name in _HashId, f"invalid hash: {name}" + + # Re-import 'hashlib' in case it was mocked + hashlib = importlib.import_module('hashlib') + builtin_constructor_cache = getattr(hashlib, '__builtin_constructor_cache') + builtin_constructor_cache_mock = builtin_constructor_cache.copy() + builtin_constructor_cache_mock.pop(name, None) + builtin_constructor_cache_mock.pop(name.upper(), None) + + # __get_builtin_constructor() imports the HACL* modules on demand, + # so we need to block the possibility of importing it, but only + # during the call to __get_builtin_constructor(). + get_builtin_constructor = getattr(hashlib, '__get_builtin_constructor') + builtin_module_name = get_hash_func_info(name).builtin.module_name + + @functools.wraps(get_builtin_constructor) + def get_builtin_constructor_mock(name): + with import_helper.isolated_modules(): + sys = importlib.import_module("sys") + sys.modules[builtin_module_name] = None # block module's import + return get_builtin_constructor(name) + + return unittest.mock.patch.multiple( + hashlib, + __get_builtin_constructor=get_builtin_constructor_mock, + __builtin_constructor_cache=builtin_constructor_cache_mock, + ) + + +def _block_builtin_hmac_new(blocked_name): + assert isinstance(blocked_name, str), blocked_name + + # re-import '_hmac' in case it was mocked + if (_hmac := _import_module("_hmac")) is None: + return contextlib.nullcontext() + + @functools.wraps(wrapped := _hmac.new) + def _hmac_new(key, msg=None, digestmod=None): + if digestmod == blocked_name: + raise _hmac.UnknownHashError(blocked_name) + return wrapped(key, msg, digestmod) + + _ensure_wrapper_signature(_hmac_new, wrapped) + return unittest.mock.patch('_hmac.new', _hmac_new) + + +def _block_builtin_hmac_digest(blocked_name): + assert isinstance(blocked_name, str), blocked_name + + # re-import '_hmac' in case it was mocked + if (_hmac := _import_module("_hmac")) is None: + return contextlib.nullcontext() + + @functools.wraps(wrapped := _hmac.compute_digest) + def _hmac_compute_digest(key, msg, digest): + if digest == blocked_name: + raise _hmac.UnknownHashError(blocked_name) + return wrapped(key, msg, digest) + + _ensure_wrapper_signature(_hmac_compute_digest, wrapped) + return unittest.mock.patch('_hmac.compute_digest', _hmac_compute_digest) + + +def _make_hash_constructor_blocker(name, dummy, implementation): + info = get_hash_func_info(name)[implementation] + if (wrapped := info.import_member()) is None: + # function shouldn't exist for this implementation + return contextlib.nullcontext() + wrapper = functools.wraps(wrapped)(dummy) + _ensure_wrapper_signature(wrapper, wrapped) + return unittest.mock.patch(info.fullname, wrapper) + + +def _block_hashlib_hash_constructor(name): + """Block explicit public constructors.""" + def dummy(data=b'', *, usedforsecurity=True, string=None): + raise ValueError(f"blocked explicit public hash name: {name}") + return _make_hash_constructor_blocker(name, dummy, 'hashlib') + + +def _block_openssl_hash_constructor(name): + """Block explicit OpenSSL constructors.""" + def dummy(data=b'', *, usedforsecurity=True, string=None): + raise ValueError(f"blocked explicit OpenSSL hash name: {name}") + return _make_hash_constructor_blocker(name, dummy, 'openssl') + + +def _block_builtin_hash_constructor(name): + """Block explicit HACL* constructors.""" + def dummy(data=b'', *, usedforsecurity=True, string=b''): + raise ValueError(f"blocked explicit builtin hash name: {name}") + return _make_hash_constructor_blocker(name, dummy, 'builtin') + + +def _block_builtin_hmac_constructor(name): + """Block explicit HACL* HMAC constructors.""" + info = get_hmac_item_info(name) + assert info.module_name is None or info.module_name == "_hmac", info + if (wrapped := info.import_member()) is None: + # function shouldn't exist for this implementation + return contextlib.nullcontext() + + @functools.wraps(wrapped) + def wrapper(key, obj): + raise ValueError(f"blocked hash name: {name}") + + _ensure_wrapper_signature(wrapper, wrapped) + return unittest.mock.patch(info.fullname, wrapper) + + +@contextlib.contextmanager +def block_algorithm(name, *, allow_openssl=False, allow_builtin=False): + """Block a hash algorithm for both hashing and HMAC. + + Be careful with this helper as a function may be allowed, but can + still raise a ValueError at runtime if the OpenSSL security policy + disables it, e.g., if allow_openssl=True and FIPS mode is on. + """ + with contextlib.ExitStack() as stack: + if not (allow_openssl or allow_builtin): + # Named constructors have a different behavior in the sense + # that they are either built-ins or OpenSSL ones, but not + # "agile" ones (namely once "hashlib" has been imported, + # they are fixed). + # + # If OpenSSL is not available, hashes fall back to built-in ones, + # in which case we don't need to block the explicit public hashes + # as they will call a mocked one. + # + # If OpenSSL is available, hashes fall back to "openssl_*" ones, + # except for BLAKE2b and BLAKE2s. + stack.enter_context(_block_hashlib_hash_constructor(name)) + elif ( + # In FIPS mode, hashlib.() functions may raise if they use + # the OpenSSL implementation, except with usedforsecurity=False. + # However, blocking such functions also means blocking them + # so we again need to block them if we want to. + (_hashlib := _import_module("_hashlib")) + and _hashlib.get_fips_mode() + and not allow_openssl + ) or ( + # Without OpenSSL, hashlib.() functions are aliases + # to built-in functions, so both of them must be blocked + # as the module may have been imported before the HACL ones. + not (_hashlib := _import_module("_hashlib")) + and not allow_builtin + ): + stack.enter_context(_block_hashlib_hash_constructor(name)) + + if not allow_openssl: + # _hashlib.new() + stack.enter_context(_block_openssl_hash_new(name)) + # _hashlib.openssl_*() + stack.enter_context(_block_openssl_hash_constructor(name)) + # _hashlib.hmac_new() + stack.enter_context(_block_openssl_hmac_new(name)) + # _hashlib.hmac_digest() + stack.enter_context(_block_openssl_hmac_digest(name)) + + if not allow_builtin: + # __get_builtin_constructor(name) + stack.enter_context(_block_builtin_hash_new(name)) + # .() + stack.enter_context(_block_builtin_hash_constructor(name)) + # _hmac.new(..., name) + stack.enter_context(_block_builtin_hmac_new(name)) + # _hmac.compute_() + stack.enter_context(_block_builtin_hmac_constructor(name)) + # _hmac.compute_digest(..., name) + stack.enter_context(_block_builtin_hmac_digest(name)) + yield + + +@contextlib.contextmanager +def block_openssl_algorithms(*, exclude=()): + """Block OpenSSL implementations, except those given in *exclude*.""" + with contextlib.ExitStack() as stack: + for name in CANONICAL_DIGEST_NAMES.difference(exclude): + stack.enter_context(block_algorithm(name, allow_builtin=True)) + yield + + +@contextlib.contextmanager +def block_builtin_algorithms(*, exclude=()): + """Block HACL* implementations, except those given in *exclude*.""" + with contextlib.ExitStack() as stack: + for name in CANONICAL_DIGEST_NAMES.difference(exclude): + stack.enter_context(block_algorithm(name, allow_openssl=True)) + yield diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index edb734d294f..0af63501f93 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -438,5 +438,5 @@ def ensure_module_imported(name, *, clearnone=True): if sys.modules.get(name) is not None: mod = sys.modules[name] else: - mod, _, _ = _force_import(name, False, True, clearnone) + mod, _, _ = _ensure_module(name, False, True, clearnone) return mod diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index 87941ee1791..a41e487f3e4 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -259,6 +259,10 @@ def filter_error(err): # raise OSError('socket error', msg) from msg elif len(a) >= 2 and isinstance(a[1], OSError): err = a[1] + # The error can also be wrapped as __cause__: + # raise URLError(f"ftp error: {exp}") from exp + elif isinstance(err, urllib.error.URLError) and err.__cause__: + err = err.__cause__ else: break filter_error(err) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index 798d6c68869..cf95f7bdc7d 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -38,7 +38,7 @@ def events(self): This assumes the program under inspection doesn't print any non-utf8 strings which would mix into the strace output.""" - decoded_events = self.event_bytes.decode('utf-8') + decoded_events = self.event_bytes.decode('utf-8', 'surrogateescape') matches = [ _syscall_regex.match(event) for event in decoded_events.splitlines() @@ -178,7 +178,10 @@ def get_syscalls(code, strace_flags, prelude="", cleanup="", # Moderately expensive (spawns a subprocess), so share results when possible. @cache def _can_strace(): - res = strace_python("import sys; sys.exit(0)", [], check=False) + res = strace_python("import sys; sys.exit(0)", + # --trace option needs strace 5.5 (gh-133741) + ["--trace=%process"], + check=False) if res.strace_returncode == 0 and res.python_returncode == 0: assert res.events(), "Should have parsed multiple calls" return True diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index afa25a76f63..3e04c344a0d 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -248,3 +248,27 @@ def requires_working_threading(*, module=False): raise unittest.SkipTest(msg) else: return unittest.skipUnless(can_start_thread, msg) + + +def run_concurrently(worker_func, nthreads, args=(), kwargs={}): + """ + Run the worker function concurrently in multiple threads. + """ + barrier = threading.Barrier(nthreads) + + def wrapper_func(*args, **kwargs): + # Wait for all threads to reach this point before proceeding. + barrier.wait() + worker_func(*args, **kwargs) + + with catch_threading_exception() as cm: + workers = [ + threading.Thread(target=wrapper_func, args=args, kwargs=kwargs) + for _ in range(nthreads) + ] + with start_threads(workers): + pass + + # If a worker thread raises an exception, re-raise it. + if cm.exc_value is not None: + raise cm.exc_value diff --git a/Lib/test/support/warnings_helper.py b/Lib/test/support/warnings_helper.py index a6e43dff200..c046f39fbff 100644 --- a/Lib/test/support/warnings_helper.py +++ b/Lib/test/support/warnings_helper.py @@ -1,11 +1,11 @@ import contextlib -import functools import importlib import re import sys import warnings + def import_deprecated(name): """Import *name* while suppressing DeprecationWarning.""" with warnings.catch_warnings(): @@ -23,8 +23,7 @@ def check_syntax_warning(testcase, statement, errtext='', testcase.assertEqual(len(warns), 1, warns) warn, = warns - testcase.assertTrue(issubclass(warn.category, SyntaxWarning), - warn.category) + testcase.assertIsSubclass(warn.category, SyntaxWarning) if errtext: testcase.assertRegex(str(warn.message), errtext) testcase.assertEqual(warn.filename, '') @@ -43,20 +42,32 @@ def check_syntax_warning(testcase, statement, errtext='', testcase.assertEqual(warns, []) -def ignore_warnings(*, category): +@contextlib.contextmanager +def ignore_warnings(*, category, message=''): """Decorator to suppress warnings. - Use of context managers to hide warnings make diffs - more noisy and tools like 'git blame' less useful. + Can also be used as a context manager. This is not preferred, + because it makes diffs more noisy and tools like 'git blame' less useful. + But, it's useful for async functions. """ - def decorator(test): - @functools.wraps(test) - def wrapper(self, *args, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=category) - return test(self, *args, **kwargs) - return wrapper - return decorator + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=category, message=message) + yield + + +@contextlib.contextmanager +def ignore_fork_in_thread_deprecation_warnings(): + """Suppress deprecation warnings related to forking in multi-threaded code. + + See gh-135427 + + Can be used as decorator (preferred) or context manager. + """ + with ignore_warnings( + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning, + ): + yield class WarningsRecorder(object): diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index f35b1194308..8ded9f99248 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -72,6 +72,8 @@ def check_all(self, modname): all_set = set(all_list) self.assertCountEqual(all_set, all_list, "in module {}".format(modname)) self.assertEqual(keys, all_set, "in module {}".format(modname)) + # Verify __dir__ is non-empty and doesn't produce an error + self.assertTrue(dir(sys.modules[modname])) def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index b2f0bb1386f..67e0595943d 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -1,4 +1,5 @@ import contextlib +import dataclasses import io import sys import unittest @@ -21,6 +22,42 @@ def supports_virtual_terminal(): return contextlib.nullcontext() +class TestTheme(unittest.TestCase): + + def test_attributes(self): + # only theme configurations attributes by default + for field in dataclasses.fields(_colorize.Theme): + with self.subTest(field.name): + self.assertIsSubclass(field.type, _colorize.ThemeSection) + self.assertIsNotNone(field.default_factory) + + def test_copy_with(self): + theme = _colorize.Theme() + + copy = theme.copy_with() + self.assertEqual(theme, copy) + + unittest_no_colors = _colorize.Unittest.no_colors() + copy = theme.copy_with(unittest=unittest_no_colors) + self.assertEqual(copy.argparse, theme.argparse) + self.assertEqual(copy.difflib, theme.difflib) + self.assertEqual(copy.syntax, theme.syntax) + self.assertEqual(copy.traceback, theme.traceback) + self.assertEqual(copy.unittest, unittest_no_colors) + + def test_no_colors(self): + # idempotence test + theme_no_colors = _colorize.Theme().no_colors() + theme_no_colors_no_colors = theme_no_colors.no_colors() + self.assertEqual(theme_no_colors, theme_no_colors_no_colors) + + # attributes check + for section in dataclasses.fields(_colorize.Theme): + with self.subTest(section.name): + section_theme = getattr(theme_no_colors, section.name) + self.assertEqual(section_theme, section.type.no_colors()) + + class TestColorizeFunction(unittest.TestCase): def test_colorized_detection_checks_for_environment_variables(self): def check(env, fallback, expected): @@ -129,6 +166,17 @@ def test_colorized_detection_checks_for_file(self): file.isatty.return_value = False self.assertEqual(_colorize.can_colorize(file=file), False) + # The documentation for file.fileno says: + # > An OSError is raised if the IO object does not use a file descriptor. + # gh-141570: Check OSError is caught and handled + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = OSError + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index e4c1ad85451..d7cf77368ef 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -6,10 +6,10 @@ import time import unittest -from test.support import import_helper, skip_if_sanitizer +from test.support import import_helper _channels = import_helper.import_module('_interpchannels') -from test.support.interpreters import _crossinterp +from concurrent.interpreters import _crossinterp from test.test__interpreters import ( _interpreters, _run_output, @@ -247,7 +247,7 @@ def _run_action(cid, action, end, state): def clean_up_channels(): - for cid, _ in _channels.list_all(): + for cid, _, _ in _channels.list_all(): try: _channels.destroy(cid) except _channels.ChannelNotFoundError: @@ -365,7 +365,6 @@ def test_shareable(self): #self.assertIsNot(got, obj) -@skip_if_sanitizer('gh-129824: race on _waiting_release', thread=True) class ChannelTests(TestBase): def test_create_cid(self): @@ -373,11 +372,11 @@ def test_create_cid(self): self.assertIsInstance(cid, _channels.ChannelID) def test_sequential_ids(self): - before = [cid for cid, _ in _channels.list_all()] + before = [cid for cid, _, _ in _channels.list_all()] id1 = _channels.create(REPLACE) id2 = _channels.create(REPLACE) id3 = _channels.create(REPLACE) - after = [cid for cid, _ in _channels.list_all()] + after = [cid for cid, _, _ in _channels.list_all()] self.assertEqual(id2, int(id1) + 1) self.assertEqual(id3, int(id2) + 1) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 0c43f46300f..ec5a26bc1c0 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -474,15 +474,32 @@ def setUp(self): def test_signatures(self): # See https://github.com/python/cpython/issues/126654 - msg = "expected 'shared' to be a dict" + msg = r"exec\(\) argument 'shared' must be dict, not int" with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', 1) with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', shared=1) + msg = r"run_string\(\) argument 'shared' must be dict, not int" with self.assertRaisesRegex(TypeError, msg): _interpreters.run_string(self.id, 'a', shared=1) + msg = r"run_func\(\) argument 'shared' must be dict, not int" with self.assertRaisesRegex(TypeError, msg): _interpreters.run_func(self.id, lambda: None, shared=1) + # See https://github.com/python/cpython/issues/135855 + msg = r"set___main___attrs\(\) argument 'updates' must be dict, not int" + with self.assertRaisesRegex(TypeError, msg): + _interpreters.set___main___attrs(self.id, 1) + + def test_invalid_shared_none(self): + msg = r'must be dict, not None' + with self.assertRaisesRegex(TypeError, msg): + _interpreters.exec(self.id, 'a', shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_string(self.id, 'a', shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_func(self.id, lambda: None, shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.set___main___attrs(self.id, None) def test_invalid_shared_encoding(self): # See https://github.com/python/cpython/issues/127196 @@ -952,7 +969,8 @@ def test_invalid_syntax(self): """) with self.subTest('script'): - self.assert_run_failed(SyntaxError, script) + with self.assertRaises(SyntaxError): + _interpreters.run_string(self.id, script) with self.subTest('module'): modname = 'spam_spam_spam' @@ -1019,12 +1037,19 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') + failed = None def f(): - _interpreters.set___main___attrs(self.id, dict(w=w)) - _interpreters.run_func(self.id, script) + nonlocal failed + try: + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) + except Exception as exc: + failed = exc t = threading.Thread(target=f) t.start() t.join() + if failed: + raise Exception from failed with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1053,18 +1078,16 @@ def test_closure(self): spam = True def script(): assert spam - with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) - # XXX This hasn't been fixed yet. - @unittest.expectedFailure def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) +# @unittest.skip("we're not quite there yet") def test_args(self): with self.subTest('args'): def script(a, b=0): diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index 53aa26620a6..0813c4804c1 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -66,8 +66,8 @@ def test__find_build_tool(self): 'cc not found - check xcode-select') def test__get_system_version(self): - self.assertTrue(platform.mac_ver()[0].startswith( - _osx_support._get_system_version())) + self.assertStartsWith(platform.mac_ver()[0], + _osx_support._get_system_version()) def test__remove_original_values(self): config_vars = { diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 72232b670cd..cf071d2c933 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -24,11 +24,11 @@ def not_implemented(*args, **kwargs): class TestNumbers(unittest.TestCase): def test_int(self): - self.assertTrue(issubclass(int, Integral)) - self.assertTrue(issubclass(int, Rational)) - self.assertTrue(issubclass(int, Real)) - self.assertTrue(issubclass(int, Complex)) - self.assertTrue(issubclass(int, Number)) + self.assertIsSubclass(int, Integral) + self.assertIsSubclass(int, Rational) + self.assertIsSubclass(int, Real) + self.assertIsSubclass(int, Complex) + self.assertIsSubclass(int, Number) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -38,11 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): - self.assertFalse(issubclass(float, Integral)) - self.assertFalse(issubclass(float, Rational)) - self.assertTrue(issubclass(float, Real)) - self.assertTrue(issubclass(float, Complex)) - self.assertTrue(issubclass(float, Number)) + self.assertNotIsSubclass(float, Integral) + self.assertNotIsSubclass(float, Rational) + self.assertIsSubclass(float, Real) + self.assertIsSubclass(float, Complex) + self.assertIsSubclass(float, Number) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -50,11 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): - self.assertFalse(issubclass(complex, Integral)) - self.assertFalse(issubclass(complex, Rational)) - self.assertFalse(issubclass(complex, Real)) - self.assertTrue(issubclass(complex, Complex)) - self.assertTrue(issubclass(complex, Number)) + self.assertNotIsSubclass(complex, Integral) + self.assertNotIsSubclass(complex, Rational) + self.assertNotIsSubclass(complex, Real) + self.assertIsSubclass(complex, Complex) + self.assertIsSubclass(complex, Number) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index de83ce081c2..c6c4a15a7ee 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -91,34 +91,38 @@ def tearDown(self): self.logcat_thread = None @contextmanager - def unbuffered(self, stream): - stream.reconfigure(write_through=True) + def reconfigure(self, stream, **settings): + original_settings = {key: getattr(stream, key, None) for key in settings.keys()} + stream.reconfigure(**settings) try: yield finally: - stream.reconfigure(write_through=False) + stream.reconfigure(**original_settings) - # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't - # test them directly. Detect this mode and use some temporary streams with - # the same properties. def stream_context(self, stream_name, level): - # https://developer.android.com/ndk/reference/group/logging - prio = {"I": 4, "W": 5}[level] - stack = ExitStack() stack.enter_context(self.subTest(stream_name)) + + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. stream = getattr(sys, stream_name) native_stream = getattr(sys, f"__{stream_name}__") if isinstance(stream, io.StringIO): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] stack.enter_context( patch( f"sys.{stream_name}", - TextLogStream( - prio, f"python.{stream_name}", native_stream.fileno(), - errors="backslashreplace" + stream := TextLogStream( + prio, f"python.{stream_name}", native_stream, ), ) ) + + # The tests assume the stream is initially buffered. + stack.enter_context(self.reconfigure(stream, write_through=False)) + return stack def test_str(self): @@ -145,7 +149,7 @@ def write(s, lines=None, *, write_len=None): self.assert_logs(level, tag, lines) # Single-line messages, - with self.unbuffered(stream): + with self.reconfigure(stream, write_through=True): write("", []) write("a") @@ -192,7 +196,7 @@ def write(s, lines=None, *, write_len=None): # However, buffering can be turned off completely if you want a # flush after every write. - with self.unbuffered(stream): + with self.reconfigure(stream, write_through=True): write("\nx", ["", "x"]) write("\na\n", ["", "a"]) write("\n", [""]) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index c3c245ddaf8..8208d0e9c94 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -7,7 +7,9 @@ import functools import itertools import pickle +from string.templatelib import Template, Interpolation import typing +import sys import unittest from annotationlib import ( Format, @@ -72,6 +74,30 @@ def inner(arg: x): anno = get_annotations(inner, format=Format.FORWARDREF) self.assertEqual(anno["arg"], x) + def test_multiple_closure(self): + def inner(arg: x[y]): + pass + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x[y]") + with self.assertRaises(NameError): + fwdref.evaluate() + + y = str + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + extra_name, extra_val = next(iter(fwdref.__extra_names__.items())) + self.assertEqual(fwdref.__forward_arg__.replace(extra_name, extra_val.__name__), "x[str]") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = list + self.assertEqual(fwdref.evaluate(), x[y]) + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertEqual(fwdref, x[y]) + def test_function(self): def f(x: int, y: doesntexist): pass @@ -93,6 +119,10 @@ def f( alpha: some | obj, beta: +some, gamma: some < obj, + delta: some | {obj: module}, + epsilon: some | {obj, module}, + zeta: some | [obj], + eta: some | (), ): pass @@ -121,6 +151,69 @@ def f( self.assertIsInstance(gamma_anno, ForwardRef) self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + delta_anno = anno["delta"] + self.assertIsInstance(delta_anno, ForwardRef) + self.assertEqual(delta_anno, support.EqualToForwardRef("some | {obj: module}", owner=f)) + + epsilon_anno = anno["epsilon"] + self.assertIsInstance(epsilon_anno, ForwardRef) + self.assertEqual(epsilon_anno, support.EqualToForwardRef("some | {obj, module}", owner=f)) + + zeta_anno = anno["zeta"] + self.assertIsInstance(zeta_anno, ForwardRef) + self.assertEqual(zeta_anno, support.EqualToForwardRef("some | [obj]", owner=f)) + + eta_anno = anno["eta"] + self.assertIsInstance(eta_anno, ForwardRef) + self.assertEqual(eta_anno, support.EqualToForwardRef("some | ()", owner=f)) + + def test_partially_nonexistent(self): + # These annotations start with a non-existent variable and then use + # global types with defined values. This partially evaluates by putting + # those globals into `fwdref.__extra_names__`. + def f( + x: obj | int, + y: container[int:obj, int], + z: dict_val | {str: int}, + alpha: set_val | {str, int}, + beta: obj | bool | int, + gamma: obj | call_func(int, kwd=bool), + ): + pass + + def func(*args, **kwargs): + return Union[*args, *(kwargs.values())] + + anno = get_annotations(f, format=Format.FORWARDREF) + globals_ = { + "obj": str, "container": list, "dict_val": {1: 2}, "set_val": {1, 2}, + "call_func": func + } + + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno.evaluate(globals=globals_), str | int) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno.evaluate(globals=globals_), list[int:str, int]) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno.evaluate(globals=globals_), {1: 2} | {str: int}) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno.evaluate(globals=globals_), {1, 2} | {str, int}) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno.evaluate(globals=globals_), str | bool | int) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno.evaluate(globals=globals_), str | func(int, kwd=bool)) + def test_partially_nonexistent_union(self): # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs class UnionForwardrefs: @@ -273,6 +366,45 @@ def f( }, ) + def test_template_str(self): + def f( + x: t"{a}", + y: list[t"{a}"], + z: t"{a:b} {c!r} {d!s:t}", + a: t"a{b}c{d}e{f}g", + b: t"{a:{1}}", + c: t"{a | b * c}", + gh138558: t"{ 0}", + ): pass + + annos = get_annotations(f, format=Format.STRING) + self.assertEqual(annos, { + "x": "t'{a}'", + "y": "list[t'{a}']", + "z": "t'{a:b} {c!r} {d!s:t}'", + "a": "t'a{b}c{d}e{f}g'", + # interpolations in the format spec are eagerly evaluated so we can't recover the source + "b": "t'{a:1}'", + "c": "t'{a | b * c}'", + "gh138558": "t'{ 0}'", + }) + + def g( + x: t"{a}", + ): ... + + annos = get_annotations(g, format=Format.FORWARDREF) + templ = annos["x"] + # Template and Interpolation don't have __eq__ so we have to compare manually + self.assertIsInstance(templ, Template) + self.assertEqual(templ.strings, ("", "")) + self.assertEqual(len(templ.interpolations), 1) + interp = templ.interpolations[0] + self.assertEqual(interp.value, support.EqualToForwardRef("a", owner=g)) + self.assertEqual(interp.expression, "a") + self.assertIsNone(interp.conversion) + self.assertEqual(interp.format_spec, "") + def test_getitem(self): def f(x: undef1[str, undef2]): pass @@ -340,7 +472,7 @@ def f(x: a[[int, str], float]): def g( w: a[[int, str], float], - x: a[{int, str}, 3], + x: a[{int}, 3], y: a[{int: str}, 4], z: a[(int, str), 5], ): @@ -350,7 +482,7 @@ def g( anno, { "w": "a[[int, str], float]", - "x": "a[{int, str}, 3]", + "x": "a[{int}, 3]", "y": "a[{int: str}, 4]", "z": "a[(int, str), 5]", }, @@ -715,6 +847,8 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {"eval_str": True}, + {"eval_str": True, "globals": isa.__dict__, "locals": {}}, + {"eval_str": True, "globals": {}, "locals": isa.__dict__}, {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): @@ -747,6 +881,12 @@ def test_stringized_annotations_in_empty_module(self): self.assertEqual(get_annotations(isa2, eval_str=True), {}) self.assertEqual(get_annotations(isa2, eval_str=False), {}) + def test_stringized_annotations_with_star_unpack(self): + def f(*args: "*tuple[int, ...]"): ... + self.assertEqual(get_annotations(f, eval_str=True), + {'args': (*tuple[int, ...],)[0]}) + + def test_stringized_annotations_on_wrapper(self): isa = inspect_stringized_annotations wrapped = times_three(isa.function) @@ -765,6 +905,44 @@ def test_stringized_annotations_on_wrapper(self): {"a": "int", "b": "str", "return": "MyClass"}, ) + def test_stringized_annotations_on_partial_wrapper(self): + isa = inspect_stringized_annotations + + def times_three_str(fn: typing.Callable[[str], isa.MyClass]): + @functools.wraps(fn) + def wrapper(b: "str") -> "MyClass": + return fn(b * 3) + + return wrapper + + wrapped = times_three_str(functools.partial(isa.function, 1)) + self.assertEqual(wrapped("x"), isa.MyClass(1, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + # If functools is not loaded, names will be evaluated in the current + # module instead of being unwrapped to the original. + functools_mod = sys.modules["functools"] + del sys.modules["functools"] + + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + sys.modules["functools"] = functools_mod + def test_stringized_annotations_on_class(self): isa = inspect_stringized_annotations # test that local namespace lookups work @@ -777,6 +955,80 @@ def test_stringized_annotations_on_class(self): {"x": int}, ) + def test_stringized_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": "int"} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha), {"x": "int"}) + self.assertEqual(get_annotations(ha, eval_str=True), {"x": int}) + + def test_stringized_annotation_permutations(self): + def define_class(name, has_future, has_annos, base_text, extra_names=None): + lines = [] + if has_future: + lines.append("from __future__ import annotations") + lines.append(f"class {name}({base_text}):") + if has_annos: + lines.append(f" {name}_attr: int") + else: + lines.append(" pass") + code = "\n".join(lines) + ns = support.run_code(code, extra_names=extra_names) + return ns[name] + + def check_annotations(cls, has_future, has_annos): + if has_annos: + if has_future: + anno = "int" + else: + anno = int + self.assertEqual(get_annotations(cls), {f"{cls.__name__}_attr": anno}) + else: + self.assertEqual(get_annotations(cls), {}) + + for meta_future, base_future, child_future, meta_has_annos, base_has_annos, child_has_annos in itertools.product( + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + ): + with self.subTest( + meta_future=meta_future, + base_future=base_future, + child_future=child_future, + meta_has_annos=meta_has_annos, + base_has_annos=base_has_annos, + child_has_annos=child_has_annos, + ): + meta = define_class( + "Meta", + has_future=meta_future, + has_annos=meta_has_annos, + base_text="type", + ) + base = define_class( + "Base", + has_future=base_future, + has_annos=base_has_annos, + base_text="metaclass=Meta", + extra_names={"Meta": meta}, + ) + child = define_class( + "Child", + has_future=child_future, + has_annos=child_has_annos, + base_text="Base", + extra_names={"Base": base}, + ) + check_annotations(meta, meta_future, meta_has_annos) + check_annotations(base, base_future, base_has_annos) + check_annotations(child, child_future, child_has_annos) + def test_modify_annotations(self): def f(x: int): pass @@ -880,6 +1132,23 @@ def __annotate__(self): {"x": "int"}, ) + def test_non_dict_annotate(self): + class WeirdAnnotate: + def __annotate__(self, *args, **kwargs): + return "not a dict" + + wa = WeirdAnnotate() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotate__ returned a non-dict" + ), + ): + get_annotations(wa, format=format) + def test_no_annotations(self): class CustomClass: pass @@ -1084,6 +1353,40 @@ class RaisesAttributeError: }, ) + def test_nonlocal_in_annotation_scope(self): + class Demo: + nonlocal sequence_b + x: sequence_b + y: sequence_b[int] + + fwdrefs = get_annotations(Demo, format=Format.FORWARDREF) + + self.assertIsInstance(fwdrefs["x"], ForwardRef) + self.assertIsInstance(fwdrefs["y"], ForwardRef) + + sequence_b = list + self.assertIs(fwdrefs["x"].evaluate(), list) + self.assertEqual(fwdrefs["y"].evaluate(), list[int]) + + def test_raises_error_from_value(self): + # test that if VALUE is the only supported format, but raises an error + # that error is propagated from get_annotations + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + def f(): ... + + f.__annotate__ = annotate + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + get_annotations(f, format=fmt) + class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): @@ -1103,6 +1406,206 @@ def evaluate(format, exc=NotImplementedError): "undefined", ) + def test_fake_global_evaluation(self): + # This will raise an AttributeError + def evaluate_union(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + # Return a ForwardRef + return builtins.undefined | list[int] + raise exc + + self.assertEqual( + annotationlib.call_evaluate_function(evaluate_union, Format.FORWARDREF), + support.EqualToForwardRef("builtins.undefined | list[int]"), + ) + + # This will raise an AttributeError + def evaluate_intermediate(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + intermediate = builtins.undefined + # Return a literal + return intermediate is None + raise exc + + self.assertIs( + annotationlib.call_evaluate_function(evaluate_intermediate, Format.FORWARDREF), + False, + ) + + +class TestCallAnnotateFunction(unittest.TestCase): + # Tests for user defined annotate functions. + + # Format and NotImplementedError are provided as arguments so they exist in + # the fake globals namespace. + # This avoids non-matching conditions passing by being converted to stringifiers. + # See: https://github.com/python/cpython/issues/138764 + + def test_user_annotate_value(self): + def annotate(format, /): + if format == Format.VALUE: + return {"x": str} + else: + raise NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_forwardref_supported(self): + # If Format.FORWARDREF is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.FORWARDREF: + return {'x': float} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": float}) + + def test_user_annotate_forwardref_fakeglobals(self): + # If Format.FORWARDREF is not supported, use Format.VALUE_WITH_FAKE_GLOBALS + # before falling back to Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": int}) + + def test_user_annotate_forwardref_value_fallback(self): + # If Format.FORWARDREF and Format.VALUE_WITH_FAKE_GLOBALS are not supported + # use Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_string_supported(self): + # If Format.STRING is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "float"}) + + def test_user_annotate_string_fakeglobals(self): + # If Format.STRING is not supported but Format.VALUE_WITH_FAKE_GLOBALS is + # prefer that over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "int"}) + + def test_user_annotate_string_value_fallback(self): + # If Format.STRING and Format.VALUE_WITH_FAKE_GLOBALS are not + # supported fall back to Format.VALUE and convert to strings + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "str"}) + + def test_condition_not_stringified(self): + # Make sure the first condition isn't evaluated as True by being converted + # to a _Stringifier + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(NotImplementedError): + annotationlib.call_annotate_function(annotate, Format.STRING) + + def test_unsupported_formats(self): + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(ValueError): + annotationlib.call_annotate_function(annotate, Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(RuntimeError): + annotationlib.call_annotate_function(annotate, Format.VALUE) + + with self.assertRaises(ValueError): + # Some non-Format value + annotationlib.call_annotate_function(annotate, 7) + + def test_error_from_value_raised(self): + # Test that the error from format.VALUE is raised + # if all formats fail + + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + annotationlib.call_annotate_function(annotate, format=fmt) + class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): @@ -1248,6 +1751,24 @@ def nested(): self.assertEqual(type_repr("1"), "'1'") self.assertEqual(type_repr(Format.VALUE), repr(Format.VALUE)) self.assertEqual(type_repr(MyClass()), "my repr") + # gh138558 tests + self.assertEqual(type_repr(t'''{ 0 + & 1 + | 2 + }'''), 't"""{ 0\n & 1\n | 2}"""') + self.assertEqual( + type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'" + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42))), + "Template('hi', Interpolation(42, '', None, ''))", + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42, " "))), + "Template('hi', Interpolation(42, ' ', None, ''))", + ) + # gh138558: perhaps in the future, we can improve this behavior: + self.assertEqual(type_repr(Template(Interpolation(42, "99"))), "t'{99}'") class TestAnnotationsToString(unittest.TestCase): @@ -1263,6 +1784,11 @@ def test_annotations_to_string(self): class A: pass +TypeParamsAlias1 = int + +class TypeParamsSample[TypeParamsAlias1, TypeParamsAlias2]: + TypeParamsAlias2 = str + class TestForwardRefClass(unittest.TestCase): def test_forwardref_instance_type_error(self): @@ -1364,6 +1890,14 @@ def test_forward_repr(self): repr(List[ForwardRef("int", module="mod")]), "typing.List[ForwardRef('int', module='mod')]", ) + self.assertEqual( + repr(List[ForwardRef("int", module="mod", is_class=True)]), + "typing.List[ForwardRef('int', module='mod', is_class=True)]", + ) + self.assertEqual( + repr(List[ForwardRef("int", owner="class")]), + "typing.List[ForwardRef('int', owner='class')]", + ) def test_forward_recursion_actually(self): def namespace1(): @@ -1469,6 +2003,19 @@ def test_evaluate_forwardref_format(self): support.EqualToForwardRef('"a" + 1'), ) + def test_evaluate_notimplemented_format(self): + class C: + x: alias + + fwdref = get_annotations(C, format=Format.FORWARDREF)["x"] + + with self.assertRaises(NotImplementedError): + fwdref.evaluate(format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(NotImplementedError): + # Some other unsupported value + fwdref.evaluate(format=7) + def test_evaluate_with_type_params(self): class Gen[T]: alias = int @@ -1495,6 +2042,21 @@ class Gen[T]: ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str ) + def test_evaluate_with_type_params_and_scope_conflict(self): + for is_class in (False, True): + with self.subTest(is_class=is_class): + fwdref1 = ForwardRef("TypeParamsAlias1", owner=TypeParamsSample, is_class=is_class) + fwdref2 = ForwardRef("TypeParamsAlias2", owner=TypeParamsSample, is_class=is_class) + + self.assertIs( + fwdref1.evaluate(), + TypeParamsSample.__type_params__[0], + ) + self.assertIs( + fwdref2.evaluate(), + TypeParamsSample.TypeParamsAlias2, + ) + def test_fwdref_with_module(self): self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) self.assertIs( @@ -1548,9 +2110,37 @@ def test_name_lookup_without_eval(self): with support.swap_attr(builtins, "int", dict): self.assertIs(ForwardRef("int").evaluate(), dict) - with self.assertRaises(NameError): + with self.assertRaises(NameError, msg="name 'doesntexist' is not defined") as exc: ForwardRef("doesntexist").evaluate() + self.assertEqual(exc.exception.name, "doesntexist") + + def test_evaluate_undefined_generic(self): + # Test the codepath where have to eval() with undefined variables. + class C: + x: alias[int, undef] + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": Union}, + locals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + def test_fwdref_invalid_syntax(self): fr = ForwardRef("if") with self.assertRaises(SyntaxError): @@ -1559,6 +2149,56 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() + def test_re_evaluate_generics(self): + global global_alias + + # If we've already run this test before, + # ensure the variable is still undefined + if "global_alias" in globals(): + del global_alias + + class C: + x: global_alias[int] + + # Evaluate the ForwardRef once + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF + ) + + # Now define the global and ensure that the ForwardRef evaluates + global_alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + + def test_fwdref_evaluate_argument_mutation(self): + class C[T]: + nonlocal alias + x: alias[T] + + # Mutable arguments + globals_ = globals() + globals_copy = globals_.copy() + locals_ = locals() + locals_copy = locals_.copy() + + # Evaluate the ForwardRef, ensuring we use __cell__ and type params + get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + globals=globals_, + locals=locals_, + type_params=C.__type_params__, + format=Format.FORWARDREF, + ) + + # Check if the passed in mutable arguments equal the originals + self.assertEqual(globals_, globals_copy) + self.assertEqual(locals_, locals_copy) + + alias = list + + def test_fwdref_final_class(self): + with self.assertRaises(TypeError): + class C(ForwardRef): + pass + class TestAnnotationLib(unittest.TestCase): def test__all__(self): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 5a6be1180c1..ab5382e41e7 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -18,7 +18,12 @@ import warnings from enum import StrEnum -from test.support import captured_stderr +from test.support import ( + captured_stderr, + force_not_colorized, + force_not_colorized_test_class, + swap_attr, +) from test.support import import_helper from test.support import os_helper from test.support import script_helper @@ -576,13 +581,22 @@ class TestOptionalsShortLong(ParserTestCase): class TestOptionalsDest(ParserTestCase): """Tests various means of setting destination""" - argument_signatures = [Sig('--foo-bar'), Sig('--baz', dest='zabbaz')] + argument_signatures = [ + Sig('-x', '-foobar', '--foo-bar', '-barfoo', '-X'), + Sig('--baz', dest='zabbaz'), + Sig('-y', '-qux', '-Y'), + Sig('-z'), + ] failures = ['a'] successes = [ - ('--foo-bar f', NS(foo_bar='f', zabbaz=None)), - ('--baz g', NS(foo_bar=None, zabbaz='g')), - ('--foo-bar h --baz i', NS(foo_bar='h', zabbaz='i')), - ('--baz j --foo-bar k', NS(foo_bar='k', zabbaz='j')), + ('--foo-bar f', NS(foo_bar='f', zabbaz=None, qux=None, z=None)), + ('-x f', NS(foo_bar='f', zabbaz=None, qux=None, z=None)), + ('--baz g', NS(foo_bar=None, zabbaz='g', qux=None, z=None)), + ('--foo-bar h --baz i', NS(foo_bar='h', zabbaz='i', qux=None, z=None)), + ('--baz j --foo-bar k', NS(foo_bar='k', zabbaz='j', qux=None, z=None)), + ('-qux l', NS(foo_bar=None, zabbaz=None, qux='l', z=None)), + ('-y l', NS(foo_bar=None, zabbaz=None, qux='l', z=None)), + ('-z m', NS(foo_bar=None, zabbaz=None, qux=None, z='m')), ] @@ -791,6 +805,76 @@ def test_invalid_name(self): self.assertEqual(str(cm.exception), "invalid option name '--no-foo' for BooleanOptionalAction") +class TestBooleanOptionalActionSingleDash(ParserTestCase): + """Tests BooleanOptionalAction with single dash""" + + argument_signatures = [ + Sig('-foo', '-x', action=argparse.BooleanOptionalAction), + ] + failures = ['--foo', '--no-foo', '-no-foo', '-no-x', '-nox'] + successes = [ + ('', NS(foo=None)), + ('-foo', NS(foo=True)), + ('-nofoo', NS(foo=False)), + ('-x', NS(foo=True)), + ] + + def test_invalid_name(self): + parser = argparse.ArgumentParser() + with self.assertRaises(ValueError) as cm: + parser.add_argument('-nofoo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '-nofoo' for BooleanOptionalAction") + +class TestBooleanOptionalActionAlternatePrefixChars(ParserTestCase): + """Tests BooleanOptionalAction with custom prefixes""" + + parser_signature = Sig(prefix_chars='+-', add_help=False) + argument_signatures = [Sig('++foo', action=argparse.BooleanOptionalAction)] + failures = ['--foo', '--no-foo'] + successes = [ + ('', NS(foo=None)), + ('++foo', NS(foo=True)), + ('++no-foo', NS(foo=False)), + ] + + def test_invalid_name(self): + parser = argparse.ArgumentParser(prefix_chars='+/') + with self.assertRaisesRegex(ValueError, + 'BooleanOptionalAction.*is not valid for positional arguments'): + parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + with self.assertRaises(ValueError) as cm: + parser.add_argument('++no-foo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '++no-foo' for BooleanOptionalAction") + +class TestBooleanOptionalActionSingleAlternatePrefixChar(ParserTestCase): + """Tests BooleanOptionalAction with single alternate prefix char""" + + parser_signature = Sig(prefix_chars='+/', add_help=False) + argument_signatures = [ + Sig('+foo', '+x', action=argparse.BooleanOptionalAction), + ] + failures = ['++foo', '++no-foo', '++nofoo', + '-no-foo', '-nofoo', '+no-foo', '-nofoo', + '+no-x', '+nox', '-no-x', '-nox'] + successes = [ + ('', NS(foo=None)), + ('+foo', NS(foo=True)), + ('+nofoo', NS(foo=False)), + ('+x', NS(foo=True)), + ] + + def test_invalid_name(self): + parser = argparse.ArgumentParser(prefix_chars='+/') + with self.assertRaisesRegex(ValueError, + 'BooleanOptionalAction.*is not valid for positional arguments'): + parser.add_argument('-foo', action=argparse.BooleanOptionalAction) + with self.assertRaises(ValueError) as cm: + parser.add_argument('+nofoo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '+nofoo' for BooleanOptionalAction") + class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -1007,6 +1091,7 @@ def test_parse_enum_value(self): args = parser.parse_args(['--color', 'red']) self.assertEqual(args.color, self.Color.RED) + @force_not_colorized def test_help_message_contains_enum_choices(self): parser = argparse.ArgumentParser() parser.add_argument('--color', choices=self.Color, help='Choose a color') @@ -1829,7 +1914,7 @@ def test_r_1_replace(self): class StdStreamComparer: def __init__(self, attr): # We try to use the actual stdXXX.buffer attribute as our - # marker, but but under some test environments, + # marker, but under some test environments, # sys.stdout/err are replaced by io.StringIO which won't have .buffer, # so we use a sentinel simply to show that the tests do the right thing # for any buffer supporting object @@ -2277,11 +2362,12 @@ class TestNegativeNumber(ParserTestCase): ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), ] +@force_not_colorized_test_class class TestArgumentAndSubparserSuggestions(TestCase): """Test error handling and suggestion when a user makes a typo""" def test_wrong_argument_error_with_suggestions(self): - parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser = ErrorRaisingArgumentParser() parser.add_argument('foo', choices=['bar', 'baz']) with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) @@ -2301,7 +2387,7 @@ def test_wrong_argument_error_no_suggestions(self): ) def test_wrong_argument_subparsers_with_suggestions(self): - parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers(required=True) subparsers.add_parser('foo') subparsers.add_parser('bar') @@ -2325,18 +2411,19 @@ def test_wrong_argument_subparsers_no_suggestions(self): excinfo.exception.stderr, ) - def test_wrong_argument_no_suggestion_implicit(self): - parser = ErrorRaisingArgumentParser() + def test_wrong_argument_with_suggestion_explicit(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) parser.add_argument('foo', choices=['bar', 'baz']) with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) self.assertIn( - "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + "error: argument foo: invalid choice: 'bazz', maybe you meant" + " 'baz'? (choose from bar, baz)", excinfo.exception.stderr, ) def test_suggestions_choices_empty(self): - parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser = ErrorRaisingArgumentParser() parser.add_argument('foo', choices=[]) with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) @@ -2346,7 +2433,7 @@ def test_suggestions_choices_empty(self): ) def test_suggestions_choices_int(self): - parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser = ErrorRaisingArgumentParser() parser.add_argument('foo', choices=[1, 2]) with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('3',)) @@ -2356,7 +2443,7 @@ def test_suggestions_choices_int(self): ) def test_suggestions_choices_mixed_types(self): - parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser = ErrorRaisingArgumentParser() parser.add_argument('foo', choices=[1, '2']) with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('3',)) @@ -2403,6 +2490,7 @@ def test_modified_invalid_action(self): # Subparsers tests # ================ +@force_not_colorized_test_class class TestAddSubparsers(TestCase): """Test the add_subparsers method""" @@ -2682,6 +2770,16 @@ def test_optional_subparsers(self): ret = parser.parse_args(()) self.assertIsNone(ret.command) + def test_subparser_help_with_parent_required_optional(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + parser.add_argument('--foo', required=True) + parser.add_argument('--bar') + subparsers = parser.add_subparsers() + parser_sub = subparsers.add_parser('sub') + parser_sub.add_argument('arg') + self.assertEqual(parser_sub.format_usage(), + 'usage: PROG --foo FOO sub [-h] arg\n') + def test_help(self): self.assertEqual(self.parser.format_usage(), 'usage: PROG [-h] [--foo] bar {1,2,3} ...\n') @@ -3009,6 +3107,7 @@ def test_nested_argument_group(self): # Parent parser tests # =================== +@force_not_colorized_test_class class TestParentParsers(TestCase): """Tests that parsers can be created with parent parsers""" @@ -3216,6 +3315,7 @@ def test_mutex_groups_parents(self): # Mutually exclusive group tests # ============================== +@force_not_colorized_test_class class TestMutuallyExclusiveGroupErrors(TestCase): def test_invalid_add_argument_group(self): @@ -3298,12 +3398,11 @@ def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): ''' self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) - def test_empty_group(self): + def test_usage_empty_group(self): # See issue 26952 - parser = argparse.ArgumentParser() + parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group() - with self.assertRaises(ValueError): - parser.parse_args(['-h']) + self.assertEqual(parser.format_usage(), 'usage: PROG [-h]\n') def test_nested_mutex_groups(self): parser = argparse.ArgumentParser(prog='PROG') @@ -3344,21 +3443,25 @@ def test_successes_when_required(self): actual_ns = parse_args(args_string.split()) self.assertEqual(actual_ns, expected_ns) + @force_not_colorized def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage expected_usage = self.usage_when_not_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_usage_when_required(self): format_usage = self.get_parser(required=True).format_usage expected_usage = self.usage_when_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_help_when_not_required(self): format_help = self.get_parser(required=False).format_help help = self.usage_when_not_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) + @force_not_colorized def test_help_when_required(self): format_help = self.get_parser(required=True).format_help help = self.usage_when_required + self.help @@ -3567,25 +3670,29 @@ def get_parser(self, required): group.add_argument('-b', action='store_true', help='b help') parser.add_argument('-y', action='store_true', help='y help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['-a -b', '-b -c', '-a -c', '-a -b -c'] successes = [ - ('-a', NS(a=True, b=False, c=False, x=False, y=False)), - ('-b', NS(a=False, b=True, c=False, x=False, y=False)), - ('-c', NS(a=False, b=False, c=True, x=False, y=False)), - ('-a -x', NS(a=True, b=False, c=False, x=True, y=False)), - ('-y -b', NS(a=False, b=True, c=False, x=False, y=True)), - ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True)), + ('-a', NS(a=True, b=False, c=False, x=False, y=False, z=False)), + ('-b', NS(a=False, b=True, c=False, x=False, y=False, z=False)), + ('-c', NS(a=False, b=False, c=True, x=False, y=False, z=False)), + ('-a -x', NS(a=True, b=False, c=False, x=True, y=False, z=False)), + ('-y -b', NS(a=False, b=True, c=False, x=False, y=True, z=False)), + ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True, z=False)), ] successes_when_not_required = [ - ('', NS(a=False, b=False, c=False, x=False, y=False)), - ('-x', NS(a=False, b=False, c=False, x=True, y=False)), - ('-y', NS(a=False, b=False, c=False, x=False, y=True)), + ('', NS(a=False, b=False, c=False, x=False, y=False, z=False)), + ('-x', NS(a=False, b=False, c=False, x=True, y=False, z=False)), + ('-y', NS(a=False, b=False, c=False, x=False, y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-x] [-a] [-b] [-y] [-c] + usage_when_not_required = '''\ + usage: PROG [-h] [-x] [-a | -b | -c] [-y] [-z] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-x] (-a | -b | -c) [-y] [-z] ''' help = '''\ @@ -3596,6 +3703,7 @@ def get_parser(self, required): -b b help -y y help -c c help + -z z help ''' @@ -3649,23 +3757,27 @@ def get_parser(self, required): group.add_argument('a', nargs='?', help='a help') group.add_argument('-b', action='store_true', help='b help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['X A -b', '-b -c', '-c X A'] successes = [ - ('X A', NS(a='A', b=False, c=False, x='X', y=False)), - ('X -b', NS(a=None, b=True, c=False, x='X', y=False)), - ('X -c', NS(a=None, b=False, c=True, x='X', y=False)), - ('X A -y', NS(a='A', b=False, c=False, x='X', y=True)), - ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True)), + ('X A', NS(a='A', b=False, c=False, x='X', y=False, z=False)), + ('X -b', NS(a=None, b=True, c=False, x='X', y=False, z=False)), + ('X -c', NS(a=None, b=False, c=True, x='X', y=False, z=False)), + ('X A -y', NS(a='A', b=False, c=False, x='X', y=True, z=False)), + ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True, z=False)), ] successes_when_not_required = [ - ('X', NS(a=None, b=False, c=False, x='X', y=False)), - ('X -y', NS(a=None, b=False, c=False, x='X', y=True)), + ('X', NS(a=None, b=False, c=False, x='X', y=False, z=False)), + ('X -y', NS(a=None, b=False, c=False, x='X', y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-y] [-b] [-c] x [a] + usage_when_not_required = '''\ + usage: PROG [-h] [-y] [-z] x [-b | -c | a] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-y] [-z] x (-b | -c | a) ''' help = '''\ @@ -3678,6 +3790,7 @@ def get_parser(self, required): -y y help -b b help -c c help + -z z help ''' @@ -4030,11 +4143,13 @@ def _test(self, tester, parser_text): tester.maxDiff = None tester.assertEqual(expected_text, parser_text) + @force_not_colorized def test_format(self, tester): parser = self._get_parser(tester) format = getattr(parser, 'format_%s' % self.func_suffix) self._test(tester, format()) + @force_not_colorized def test_print(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4047,6 +4162,7 @@ def test_print(self, tester): setattr(sys, self.std_name, old_stream) self._test(tester, parser_text) + @force_not_colorized def test_print_file(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4788,6 +4904,7 @@ class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): version = '' +@force_not_colorized_test_class class TestHelpUsageNoWhitespaceCrash(TestCase): def test_all_suppressed_mutex_followed_by_long_arg(self): @@ -4868,6 +4985,25 @@ def test_long_mutex_groups_wrap(self): ''') self.assertEqual(parser.format_usage(), usage) + def test_mutex_groups_with_mixed_optionals_positionals_wrap(self): + # https://github.com/python/cpython/issues/75949 + # Mutually exclusive groups containing both optionals and positionals + # should preserve pipe separators when the usage line wraps. + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('-v', '--verbose', action='store_true') + g.add_argument('-q', '--quiet', action='store_true') + g.add_argument('-x', '--extra-long-option-name', nargs='?') + g.add_argument('-y', '--yet-another-long-option', nargs='?') + g.add_argument('positional', nargs='?') + + usage = textwrap.dedent('''\ + usage: PROG [-h] + [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | + -y [YET_ANOTHER_LONG_OPTION] | positional] + ''') + self.assertEqual(parser.format_usage(), usage) + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" @@ -5469,11 +5605,61 @@ def custom_type(string): version = '' -class TestHelpUsageLongSubparserCommand(TestCase): - """Test that subparser commands are formatted correctly in help""" +@force_not_colorized_test_class +class TestHelpCustomHelpFormatter(TestCase): maxDiff = None - def test_parent_help(self): + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=custom_formatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=CustomFormatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_usage_long_subparser_command(self): + """Test that subparser commands are formatted correctly in help""" def custom_formatter(prog): return argparse.RawTextHelpFormatter(prog, max_help_position=50) @@ -5508,6 +5694,11 @@ def custom_formatter(prog): a-very-long-command command that does something ''')) + def test_direct_formatter_instantiation(self): + formatter = argparse.HelpFormatter(prog="program") + formatter.add_usage(usage=None, actions=[], groups=[]) + help_text = formatter.format_help() + self.assertEqual(help_text, "usage: program\n") # ===================================== # Optional/Positional constructor tests @@ -5542,6 +5733,8 @@ def test_invalid_option_strings(self): self.assertTypeError('-', errmsg='dest= is required') self.assertTypeError('--', errmsg='dest= is required') self.assertTypeError('---', errmsg='dest= is required') + self.assertTypeError('-', '--', '---', + errmsg="dest= is required for options like '-', '--', '---'") def test_invalid_prefix(self): self.assertValueError('--foo', '+foo', @@ -5716,6 +5909,7 @@ def test_conflict_error(self): self.assertRaises(argparse.ArgumentError, parser.add_argument, '--spam') + @force_not_colorized def test_resolve_error(self): get_parser = argparse.ArgumentParser parser = get_parser(prog='PROG', conflict_handler='resolve') @@ -5763,6 +5957,7 @@ def test_subparser_conflict(self): # Help and Version option tests # ============================= +@force_not_colorized_test_class class TestOptionalsHelpVersionActions(TestCase): """Test the help and version actions""" @@ -5982,6 +6177,7 @@ def test_argument_error(self): class TestArgumentTypeError(TestCase): + @force_not_colorized def test_argument_type_error(self): def spam(string): @@ -6076,6 +6272,7 @@ def spam(string_to_convert): # Check that deprecated arguments output warning # ============================================== +@force_not_colorized_test_class class TestDeprecatedArguments(TestCase): def test_deprecated_option(self): @@ -6756,7 +6953,7 @@ class TestImportStar(TestCase): def test(self): for name in argparse.__all__: - self.assertTrue(hasattr(argparse, name)) + self.assertHasAttr(argparse, name) def test_all_exports_everything_but_modules(self): items = [ @@ -6780,6 +6977,7 @@ def setUp(self): metavar = '' self.parser.add_argument('--proxy', metavar=metavar) + @force_not_colorized def test_help_with_metavar(self): help_text = self.parser.format_help() self.assertEqual(help_text, textwrap.dedent('''\ @@ -6945,6 +7143,7 @@ def test_os_error(self): self.parser.parse_args, ['@no-such-file']) +@force_not_colorized_test_class class TestProgName(TestCase): source = textwrap.dedent('''\ import argparse @@ -6973,7 +7172,7 @@ def make_zip_script(self, script_name, name_in_zip=None): def check_usage(self, expected, *args, **kwargs): res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) - self.assertEqual(res.out.splitlines()[0].decode(), + self.assertEqual(os.fsdecode(res.out.splitlines()[0]), f'usage: {expected} [-h]') def test_script(self, compiled=False): @@ -7053,11 +7252,13 @@ def test_translations(self): class TestColorized(TestCase): + maxDiff = None def setUp(self): super().setUp() # Ensure color even if ran with NO_COLOR=1 - _colorize.can_colorize = lambda *args, **kwargs: True + self.enterContext(swap_attr(_colorize, 'can_colorize', + lambda *args, **kwargs: True)) self.theme = _colorize.get_theme(force_color=True).argparse def test_argparse_color(self): @@ -7130,6 +7331,8 @@ def test_argparse_color(self): short_b = self.theme.short_option label_b = self.theme.label pos_b = self.theme.action + default = self.theme.default + default_value = self.theme.default_value reset = self.theme.reset # Act @@ -7156,17 +7359,17 @@ def test_argparse_color(self): {heading}options:{reset} {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit - {short_b}-v{reset}, {long_b}--verbose{reset} more spam (default: False) - {short_b}-q{reset}, {long_b}--quiet{reset} less spam (default: False) + {short_b}-v{reset}, {long_b}--verbose{reset} more spam {default}(default: {default_value}False{default}){reset} + {short_b}-q{reset}, {long_b}--quiet{reset} less spam {default}(default: {default_value}False{default}){reset} {short_b}-o{reset}, {long_b}--optional1{reset} {long_b}--optional2{reset} {label_b}OPTIONAL2{reset} - pick one (default: None) + pick one {default}(default: {default_value}None{default}){reset} {long_b}--optional3{reset} {label_b}{{X,Y,Z}}{reset} - {long_b}--optional4{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) - {long_b}--optional5{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) - {long_b}--optional6{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional4{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {default_value}None{default}){reset} + {long_b}--optional5{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {default_value}None{default}){reset} + {long_b}--optional6{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {default_value}None{default}){reset} {short_b}-p{reset}, {long_b}--optional7{reset} {label_b}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset} - pick one (default: None) + pick one {default}(default: {default_value}None{default}){reset} {short_b}+f{reset} {label_b}F{reset} {long_b}++bar{reset} {label_b}BAR{reset} {long_b}-+baz{reset} {label_b}BAZ{reset} @@ -7182,7 +7385,28 @@ def test_argparse_color(self): ), ) - def test_argparse_color_usage(self): + def test_argparse_color_mutually_exclusive_group_usage(self): + parser = argparse.ArgumentParser(color=True, prog="PROG") + group = parser.add_mutually_exclusive_group() + group.add_argument('--foo', action='store_true', help='FOO') + group.add_argument('--spam', help='SPAM') + group.add_argument('badger', nargs='*', help='BADGER') + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + reset = self.theme.reset + + self.assertEqual(parser.format_usage(), + f"{heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] " + f"[{long}--foo{reset} | " + f"{long}--spam {label}SPAM{reset} | " + f"{pos}badger ...{reset}]\n") + + def test_argparse_color_custom_usage(self): # Arrange parser = argparse.ArgumentParser( add_help=False, @@ -7211,6 +7435,173 @@ def test_argparse_color_usage(self): ), ) + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=custom_formatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=CustomFormatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_subparser_prog_is_stored_without_color(self): + parser = argparse.ArgumentParser(prog='complex', color=True) + sub = parser.add_subparsers(dest='command') + demo_parser = sub.add_parser('demo') + + self.assertNotIn('\x1b[', demo_parser.prog) + + demo_parser.color = False + help_text = demo_parser.format_help() + self.assertNotIn('\x1b[', help_text) + + def test_error_and_warning_keywords_colorized(self): + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('foo') + + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + parser.parse_args([]) + + err = stderr.getvalue() + error_color = self.theme.error + reset = self.theme.reset + self.assertIn(f'{error_color}error:{reset}', err) + + with captured_stderr() as stderr: + parser._warning('test warning') + + warn = stderr.getvalue() + warning_color = self.theme.warning + self.assertIn(f'{warning_color}warning:{reset}', warn) + + def test_error_and_warning_not_colorized_when_disabled(self): + parser = argparse.ArgumentParser(prog='PROG', color=False) + parser.add_argument('foo') + + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + parser.parse_args([]) + + err = stderr.getvalue() + self.assertNotIn('\x1b[', err) + self.assertIn('error:', err) + + with captured_stderr() as stderr: + parser._warning('test warning') + + warn = stderr.getvalue() + self.assertNotIn('\x1b[', warn) + self.assertIn('warning:', warn) + + def test_print_help_uses_target_file_for_color_decision(self): + parser = argparse.ArgumentParser(prog='PROG', color=True) + parser.add_argument('--opt') + output = io.StringIO() + calls = [] + + def fake_can_colorize(*, file=None): + calls.append(file) + return file is None + + with swap_attr(_colorize, 'can_colorize', fake_can_colorize): + parser.print_help(file=output) + + self.assertIs(calls[-1], output) + self.assertIn(output, calls) + self.assertNotIn('\x1b[', output.getvalue()) + + def test_print_usage_uses_target_file_for_color_decision(self): + parser = argparse.ArgumentParser(prog='PROG', color=True) + parser.add_argument('--opt') + output = io.StringIO() + calls = [] + + def fake_can_colorize(*, file=None): + calls.append(file) + return file is None + + with swap_attr(_colorize, 'can_colorize', fake_can_colorize): + parser.print_usage(file=output) + + self.assertIs(calls[-1], output) + self.assertIn(output, calls) + self.assertNotIn('\x1b[', output.getvalue()) + + +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(argparse, "__version__") + self.assertEqual(cm.filename, __file__) + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 58ea89c4fac..83b3c978da3 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1255,6 +1255,14 @@ def test_typecode_u_deprecation(self): with self.assertWarns(DeprecationWarning): array.array("u") + def test_empty_string_mem_leak_gh140474(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + for _ in range(1000): + a = array.array('u', '') + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, 'u') + class UCS4Test(UnicodeTest): typecode = 'w' diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 09cf3186e05..d2b76b46dbe 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -13,6 +13,7 @@ import textwrap import types import unittest +import warnings import weakref from io import StringIO from pathlib import Path @@ -131,7 +132,7 @@ def test_snippets(self): (eval_tests, eval_results, "eval")): for i, o in zip(input, output): with self.subTest(action="parsing", input=i): - ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) + ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST, optimize=False) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) with self.subTest(action="compiling", input=i, kind=kind): @@ -141,7 +142,7 @@ def test_ast_validation(self): # compile() is the only function that calls PyAST_Validate snippets_to_validate = exec_tests + single_tests + eval_tests for snippet in snippets_to_validate: - tree = ast.parse(snippet) + tree = ast.parse(snippet, optimize=False) compile(tree, '', 'exec') def test_parse_invalid_ast(self): @@ -220,6 +221,131 @@ def test_negative_locations_for_compile(self): # This also must not crash: ast.parse(tree, optimize=2) + def test_docstring_optimization_single_node(self): + # https://github.com/python/cpython/issues/137308 + class_example1 = textwrap.dedent(''' + class A: + """Docstring""" + ''') + class_example2 = textwrap.dedent(''' + class A: + """ + Docstring""" + ''') + def_example1 = textwrap.dedent(''' + def some(): + """Docstring""" + ''') + def_example2 = textwrap.dedent(''' + def some(): + """Docstring + """ + ''') + async_def_example1 = textwrap.dedent(''' + async def some(): + """Docstring""" + ''') + async_def_example2 = textwrap.dedent(''' + async def some(): + """ + Docstring + """ + ''') + for code in [ + class_example1, + class_example2, + def_example1, + def_example2, + async_def_example1, + async_def_example2, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + self.assertEqual(len(mod.body[0].body), 1) + if opt_level == 2: + pass_stmt = mod.body[0].body[0] + self.assertIsInstance(pass_stmt, ast.Pass) + self.assertEqual( + vars(pass_stmt), + { + 'lineno': 3, + 'col_offset': 4, + 'end_lineno': 3, + 'end_col_offset': 8, + }, + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + + def test_docstring_optimization_multiple_nodes(self): + # https://github.com/python/cpython/issues/137308 + class_example = textwrap.dedent( + """ + class A: + ''' + Docstring + ''' + x = 1 + """ + ) + + def_example = textwrap.dedent( + """ + def some(): + ''' + Docstring + + ''' + x = 1 + """ + ) + + async_def_example = textwrap.dedent( + """ + async def some(): + + '''Docstring + + ''' + x = 1 + """ + ) + + for code in [ + class_example, + def_example, + async_def_example, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + if opt_level == 2: + self.assertNotIsInstance( + mod.body[0].body[0], + (ast.Pass, ast.Expr), + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice self.assertIsNone(slc.upper) @@ -275,12 +401,12 @@ def test_alias(self): self.assertEqual(alias.end_col_offset, 17) def test_base_classes(self): - self.assertTrue(issubclass(ast.For, ast.stmt)) - self.assertTrue(issubclass(ast.Name, ast.expr)) - self.assertTrue(issubclass(ast.stmt, ast.AST)) - self.assertTrue(issubclass(ast.expr, ast.AST)) - self.assertTrue(issubclass(ast.comprehension, ast.AST)) - self.assertTrue(issubclass(ast.Gt, ast.AST)) + self.assertIsSubclass(ast.For, ast.stmt) + self.assertIsSubclass(ast.Name, ast.expr) + self.assertIsSubclass(ast.stmt, ast.AST) + self.assertIsSubclass(ast.expr, ast.AST) + self.assertIsSubclass(ast.comprehension, ast.AST) + self.assertIsSubclass(ast.Gt, ast.AST) def test_field_attr_existence(self): for name, item in ast.__dict__.items(): @@ -821,6 +947,17 @@ def test_constant_as_name(self): with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): compile(expr, "", "eval") + def test_constant_as_unicode_name(self): + constants = [ + ("True", b"Tru\xe1\xb5\x89"), + ("False", b"Fal\xc5\xbfe"), + ("None", b"N\xc2\xbane"), + ] + for constant in constants: + with self.assertRaisesRegex(ValueError, + f"identifier field can't represent '{constant[0]}' constant"): + ast.parse(constant[1], mode="eval") + def test_precedence_enum(self): class _Precedence(enum.IntEnum): """Precedence table that originated from python grammar.""" @@ -855,7 +992,8 @@ def next(self): @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): - crash_depth = 500_000 + # Android test devices have less memory. + crash_depth = 100_000 if sys.platform == "android" else 500_000 success_depth = 200 if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() @@ -912,7 +1050,7 @@ def test_repr(self) -> None: snapshots = AST_REPR_DATA_FILE.read_text().split("\n") for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): with self.subTest(test_input=test): - self.assertEqual(repr(ast.parse(test)), snapshot) + self.assertEqual(repr(ast.parse(test, optimize=False)), snapshot) def test_repr_large_input_crash(self): # gh-125010: Fix use-after-free in ast repr() @@ -921,61 +1059,6 @@ def test_repr_large_input_crash(self): r"Exceeds the limit \(\d+ digits\)"): repr(ast.Constant(value=eval(source))) - def test_pep_765_warnings(self): - srcs = [ - textwrap.dedent(""" - def f(): - try: - pass - finally: - return 42 - """), - textwrap.dedent(""" - for x in y: - try: - pass - finally: - break - """), - textwrap.dedent(""" - for x in y: - try: - pass - finally: - continue - """), - ] - for src in srcs: - with self.assertWarnsRegex(SyntaxWarning, 'finally'): - ast.parse(src) - - def test_pep_765_no_warnings(self): - srcs = [ - textwrap.dedent(""" - try: - pass - finally: - def f(): - return 42 - """), - textwrap.dedent(""" - try: - pass - finally: - for x in y: - break - """), - textwrap.dedent(""" - try: - pass - finally: - for x in y: - continue - """), - ] - for src in srcs: - ast.parse(src) - def test_tstring(self): # Test AST structure for simple t-string tree = ast.parse('t"Hello"') @@ -988,12 +1071,28 @@ def test_tstring(self): self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - # Test AST for implicit concat of t-string with f-string - tree = ast.parse('t"Hello {name}" f"{name}"') - self.assertIsInstance(tree.body[0].value, ast.TemplateStr) - self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) - self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue) + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + ast.parse(source) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=r'') + ast.parse(source, filename, module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) class CopyTests(unittest.TestCase): @@ -1090,7 +1189,7 @@ def test_copy_with_parents(self): def test_replace_interface(self): for klass in self.iter_ast_classes(): with self.subTest(klass=klass): - self.assertTrue(hasattr(klass, '__replace__')) + self.assertHasAttr(klass, '__replace__') fields = set(klass._fields) with self.subTest(klass=klass, fields=fields): @@ -1304,13 +1403,22 @@ def test_replace_reject_missing_field(self): self.assertIs(repl.id, 'y') self.assertIs(repl.ctx, context) + def test_replace_accept_missing_field_with_default(self): + node = ast.FunctionDef(name="foo", args=ast.arguments()) + self.assertIs(node.returns, None) + self.assertEqual(node.decorator_list, []) + node2 = copy.replace(node, name="bar") + self.assertEqual(node2.name, "bar") + self.assertIs(node2.returns, None) + self.assertEqual(node2.decorator_list, []) + def test_replace_reject_known_custom_instance_fields_commits(self): node = ast.parse('x').body[0].value node.extra = extra = object() # add instance 'extra' field context = node.ctx # explicit rejection of known instance fields - self.assertTrue(hasattr(node, 'extra')) + self.assertHasAttr(node, 'extra') msg = "Name.__replace__ got an unexpected keyword argument 'extra'." with self.assertRaisesRegex(TypeError, re.escape(msg)): copy.replace(node, extra=1) @@ -1352,17 +1460,17 @@ def test_parse_in_error(self): def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" + "Module(body=[Expr(value=Call(func=Name(id='spam'), " + "args=[Name(id='eggs'), Constant(value='and cheese')]))])" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " + "Module([Expr(Call(Name('spam'), [Name('eggs'), " "Constant('and cheese')]))])" ) self.assertEqual(ast.dump(node, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "Module(body=[Expr(value=Call(func=Name(id='spam', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " - "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " + "args=[Name(id='eggs', lineno=1, col_offset=5, " "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " @@ -1376,18 +1484,18 @@ def test_dump_indent(self): body=[ Expr( value=Call( - func=Name(id='spam', ctx=Load()), + func=Name(id='spam'), args=[ - Name(id='eggs', ctx=Load()), + Name(id='eggs'), Constant(value='and cheese')]))])""") self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ Module( \t[ \t\tExpr( \t\t\tCall( -\t\t\t\tName('spam', Load()), +\t\t\t\tName('spam'), \t\t\t\t[ -\t\t\t\t\tName('eggs', Load()), +\t\t\t\t\tName('eggs'), \t\t\t\t\tConstant('and cheese')]))])""") self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ Module( @@ -1396,7 +1504,6 @@ def test_dump_indent(self): value=Call( func=Name( id='spam', - ctx=Load(), lineno=1, col_offset=0, end_lineno=1, @@ -1404,7 +1511,6 @@ def test_dump_indent(self): args=[ Name( id='eggs', - ctx=Load(), lineno=1, col_offset=5, end_lineno=1, @@ -1434,23 +1540,23 @@ def test_dump_incomplete(self): ) node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) self.assertEqual(ast.dump(node), - "Raise(exc=Name(id='e', ctx=Load()))" + "Raise(exc=Name(id='e'))" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(Name('e', Load()))" + "Raise(Name('e'))" ) self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" + "Raise(exc=Name(id='e'), lineno=3, col_offset=4)" ) self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), - "Raise(Name('e', Load()), lineno=3, col_offset=4)" + "Raise(Name('e'), lineno=3, col_offset=4)" ) node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) self.assertEqual(ast.dump(node), - "Raise(cause=Name(id='e', ctx=Load()))" + "Raise(cause=Name(id='e'))" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(cause=Name('e', Load()))" + "Raise(cause=Name('e'))" ) # Arguments: node = ast.arguments(args=[ast.arg("x")]) @@ -1482,10 +1588,10 @@ def test_dump_incomplete(self): [ast.Name('dataclass', ctx=ast.Load())], ) self.assertEqual(ast.dump(node), - "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass')])", ) self.assertEqual(ast.dump(node, annotate_fields=False), - "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass')])", ) def test_dump_show_empty(self): @@ -1513,7 +1619,7 @@ def check_text(code, empty, full, **kwargs): check_node( # Corner case: there are no real `Name` instances with `id=''`: ast.Name(id='', ctx=ast.Load()), - empty="Name(id='', ctx=Load())", + empty="Name(id='')", full="Name(id='', ctx=Load())", ) @@ -1523,40 +1629,64 @@ def check_text(code, empty, full, **kwargs): full="MatchSingleton(value=None)", ) + check_node( + ast.MatchSingleton(value=[]), + empty="MatchSingleton(value=[])", + full="MatchSingleton(value=[])", + ) + check_node( ast.Constant(value=None), empty="Constant(value=None)", full="Constant(value=None)", ) + check_node( + ast.Constant(value=[]), + empty="Constant(value=[])", + full="Constant(value=[])", + ) + check_node( ast.Constant(value=''), empty="Constant(value='')", full="Constant(value='')", ) + check_node( + ast.Interpolation(value=ast.Constant(42), str=None, conversion=-1), + empty="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + full="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + ) + + check_node( + ast.Interpolation(value=ast.Constant(42), str=[], conversion=-1), + empty="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + full="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + ) + check_text( "def a(b: int = 0, *, c): ...", - empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int'))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))], decorator_list=[], type_params=[])], type_ignores=[])", ) check_text( "def a(b: int = 0, *, c): ...", - empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], decorator_list=[], type_params=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)], type_ignores=[])", include_attributes=True, ) check_text( 'spam(eggs, "and cheese")', - empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + empty="Module(body=[Expr(value=Call(func=Name(id='spam'), args=[Name(id='eggs'), Constant(value='and cheese')]))])", full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], keywords=[]))], type_ignores=[])", ) check_text( 'spam(eggs, text="and cheese")', - empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", + empty="Module(body=[Expr(value=Call(func=Name(id='spam'), args=[Name(id='eggs')], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))], type_ignores=[])", ) @@ -1590,12 +1720,12 @@ def test_fix_missing_locations(self): self.assertEqual(src, ast.fix_missing_locations(src)) self.maxDiff = None self.assertEqual(ast.dump(src, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " + "Module(body=[Expr(value=Call(func=Name(id='write', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " "end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "end_col_offset=13), Expr(value=Call(func=Name(id='spam', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " @@ -1656,22 +1786,22 @@ def test_iter_child_nodes(self): ) def test_get_docstring(self): - node = ast.parse('"""line one\n line two"""') + node = ast.parse('"""line one\n line two"""', optimize=False) self.assertEqual(ast.get_docstring(node), 'line one\nline two') - node = ast.parse('class foo:\n """line one\n line two"""') + node = ast.parse('class foo:\n """line one\n line two"""', optimize=False) self.assertEqual(ast.get_docstring(node.body[0]), 'line one\nline two') - node = ast.parse('def foo():\n """line one\n line two"""') + node = ast.parse('def foo():\n """line one\n line two"""', optimize=False) self.assertEqual(ast.get_docstring(node.body[0]), 'line one\nline two') - node = ast.parse('async def foo():\n """spam\n ham"""') + node = ast.parse('async def foo():\n """spam\n ham"""', optimize=False) self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') - node = ast.parse('async def foo():\n """spam\n ham"""') + node = ast.parse('async def foo():\n """spam\n ham"""', optimize=False) self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') node = ast.parse('x') @@ -1710,7 +1840,8 @@ def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): 'def foo():\n """line one\n line two"""\n\n' ' def bar():\n """line one\n line two"""\n' ' """line one\n line two"""\n' - '"""line one\nline two"""\n\n' + '"""line one\nline two"""\n\n', + optimize=False ) self.assertEqual(node.body[0].col_offset, 0) self.assertEqual(node.body[0].lineno, 1) @@ -2279,9 +2410,9 @@ def test_stdlib_validates(self): fn = os.path.join(STDLIB, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() - mod = ast.parse(source, fn) + mod = ast.parse(source, fn, optimize=False) compile(mod, fn, "exec") - mod2 = ast.parse(source, fn) + mod2 = ast.parse(source, fn, optimize=False) self.assertTrue(ast.compare(mod, mod2)) constant_1 = ast.Constant(1) @@ -2495,7 +2626,7 @@ def test_assign_to_constant(self): "to in Store context") def test_get_docstring(self): - tree = ast.parse("'docstring'\nx = 1") + tree = ast.parse("'docstring'\nx = 1", optimize=False) self.assertEqual(ast.get_docstring(tree), 'docstring') def get_load_const(self, tree): @@ -2927,8 +3058,8 @@ def test_source_segment_missing_info(self): class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, transformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) + code, expected_code): + initial_ast = ast.parse(dedent(code)) expected_ast = ast.parse(dedent(expected_code)) transformer = transformer_class() @@ -3051,7 +3182,7 @@ def test_FunctionDef(self): with self.assertWarnsRegex(DeprecationWarning, r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): node = ast.FunctionDef(args=args) - self.assertFalse(hasattr(node, "name")) + self.assertNotHasAttr(node, "name") self.assertEqual(node.decorator_list, []) node = ast.FunctionDef(name='foo', args=args) self.assertEqual(node.name, 'foo') @@ -3147,6 +3278,15 @@ class MoreFieldsThanTypes(ast.AST): self.assertEqual(obj.a, 1) self.assertEqual(obj.b, 2) + def test_malformed_fields_with_bytes(self): + class BadFields(ast.AST): + _fields = (b'\xff'*64,) + _field_types = {'a': int} + + # This should not crash + with self.assertWarnsRegex(DeprecationWarning, r"Field b'\\xff\\xff.*' .*"): + obj = BadFields() + def test_complete_field_types(self): class _AllFieldTypes(ast.AST): _fields = ('a', 'b') @@ -3272,6 +3412,7 @@ def check_output(self, source, expect, *flags): expect = self.text_normalize(expect) self.assertEqual(res, expect) + @support.requires_resource('cpu') def test_invocation(self): # test various combinations of parameters base_flags = ( @@ -3314,7 +3455,7 @@ def test_exec_mode_flag(self): body=[ AnnAssign( target=Name(id='x', ctx=Store()), - annotation=Name(id='bool', ctx=Load()), + annotation=Name(id='bool'), value=Constant(value=1), simple=1)], type_ignores=[ @@ -3342,7 +3483,7 @@ def test_eval_mode_flag(self): expect = ''' Expression( body=Call( - func=Name(id='print', ctx=Load()), + func=Name(id='print'), args=[ Constant(value=1), Constant(value=2), @@ -3358,12 +3499,11 @@ def test_func_type_mode_flag(self): expect = ''' FunctionType( argtypes=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], + Name(id='int'), + Name(id='str')], returns=Subscript( - value=Name(id='list', ctx=Load()), - slice=Name(id='int', ctx=Load()), - ctx=Load())) + value=Name(id='list'), + slice=Name(id='int'))) ''' for flag in ('-m=func_type', '--mode=func_type'): with self.subTest(flag=flag): @@ -3377,7 +3517,7 @@ def test_no_type_comments_flag(self): body=[ AnnAssign( target=Name(id='x', ctx=Store()), - annotation=Name(id='bool', ctx=Load()), + annotation=Name(id='bool'), value=Constant(value=1), simple=1)]) ''' @@ -3422,7 +3562,7 @@ def test_feature_version_flag(self): Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchValue( @@ -3445,7 +3585,7 @@ def test_no_optimize_flag(self): Module( body=[ Match( - subject=Name(id='a', ctx=Load()), + subject=Name(id='a'), cases=[ match_case( pattern=MatchValue( @@ -3471,7 +3611,7 @@ def test_optimize_flag(self): Module( body=[ Match( - subject=Name(id='a', ctx=Load()), + subject=Name(id='a'), cases=[ match_case( pattern=MatchValue( @@ -3502,7 +3642,7 @@ def test_show_empty_flag(self): self.check_output(source, expect, '--show-empty') -class ASTOptimiziationTests(unittest.TestCase): +class ASTOptimizationTests(unittest.TestCase): def wrap_expr(self, expr): return ast.Module(body=[ast.Expr(value=expr)]) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 2c44647bf3e..cd33878d6c7 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -629,7 +629,7 @@ def setUp(self): def tearDown(self): self.loop.close() self.loop = None - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def check_async_iterator_anext(self, ait_class): with self.subTest(anext="pure-Python"): @@ -2021,6 +2021,15 @@ async def gen(): g.athrow(RuntimeError) gc_collect() + def test_athrow_throws_immediately(self): + async def gen(): + yield 1 + + g = gen() + msg = "athrow expected at least 1 argument, got 0" + with self.assertRaisesRegex(TypeError, msg): + g.athrow() + def test_aclose(self): async def gen(): yield 1 diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 2ca5c4c6719..8c02de77c24 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -24,8 +24,12 @@ MOCK_ANY = mock.ANY +class CustomError(Exception): + pass + + def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def mock_socket_module(): @@ -146,6 +150,29 @@ def test_ipaddr_info_no_inet_pton(self, m_socket): socket.SOCK_STREAM, socket.IPPROTO_TCP)) + def test_interleave_addrinfos(self): + self.maxDiff = None + SIX_A = (socket.AF_INET6, 0, 0, '', ('2001:db8::1', 1)) + SIX_B = (socket.AF_INET6, 0, 0, '', ('2001:db8::2', 2)) + SIX_C = (socket.AF_INET6, 0, 0, '', ('2001:db8::3', 3)) + SIX_D = (socket.AF_INET6, 0, 0, '', ('2001:db8::4', 4)) + FOUR_A = (socket.AF_INET, 0, 0, '', ('192.0.2.1', 5)) + FOUR_B = (socket.AF_INET, 0, 0, '', ('192.0.2.2', 6)) + FOUR_C = (socket.AF_INET, 0, 0, '', ('192.0.2.3', 7)) + FOUR_D = (socket.AF_INET, 0, 0, '', ('192.0.2.4', 8)) + + addrinfos = [SIX_A, SIX_B, SIX_C, FOUR_A, FOUR_B, FOUR_C, FOUR_D, SIX_D] + expected = [SIX_A, FOUR_A, SIX_B, FOUR_B, SIX_C, FOUR_C, SIX_D, FOUR_D] + + self.assertEqual(expected, base_events._interleave_addrinfos(addrinfos)) + + expected_fafc_2 = [SIX_A, SIX_B, FOUR_A, SIX_C, FOUR_B, SIX_D, FOUR_C, FOUR_D] + self.assertEqual( + expected_fafc_2, + base_events._interleave_addrinfos(addrinfos, first_address_family_count=2), + ) + + class BaseEventLoopTests(test_utils.TestCase): @@ -1049,6 +1076,71 @@ def test_asyncgen_finalization_by_gc_in_other_thread(self): test_utils.run_briefly(self.loop) self.assertTrue(status['finalized']) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'no IPv6 support') + @patch_socket + def test_create_connection_happy_eyeballs(self, m_socket): + + class MyProto(asyncio.Protocol): + pass + + async def getaddrinfo(*args, **kw): + return [(socket.AF_INET6, 0, 0, '', ('2001:db8::1', 1)), + (socket.AF_INET, 0, 0, '', ('192.0.2.1', 5))] + + async def sock_connect(sock, address): + if address[0] == '2001:db8::1': + await asyncio.sleep(1) + sock.connect(address) + + loop = asyncio.new_event_loop() + loop._add_writer = mock.Mock() + loop._add_writer = mock.Mock() + loop._add_reader = mock.Mock() + loop.getaddrinfo = getaddrinfo + loop.sock_connect = sock_connect + + coro = loop.create_connection(MyProto, 'example.com', 80, happy_eyeballs_delay=0.3) + transport, protocol = loop.run_until_complete(coro) + try: + sock = transport._sock + sock.connect.assert_called_with(('192.0.2.1', 5)) + finally: + transport.close() + test_utils.run_briefly(loop) # allow transport to close + loop.close() + + @patch_socket + def test_create_connection_happy_eyeballs_ipv4_only(self, m_socket): + + class MyProto(asyncio.Protocol): + pass + + async def getaddrinfo(*args, **kw): + return [(socket.AF_INET, 0, 0, '', ('192.0.2.1', 5)), + (socket.AF_INET, 0, 0, '', ('192.0.2.2', 6))] + + async def sock_connect(sock, address): + if address[0] == '192.0.2.1': + await asyncio.sleep(1) + sock.connect(address) + + loop = asyncio.new_event_loop() + loop._add_writer = mock.Mock() + loop._add_writer = mock.Mock() + loop._add_reader = mock.Mock() + loop.getaddrinfo = getaddrinfo + loop.sock_connect = sock_connect + + coro = loop.create_connection(MyProto, 'example.com', 80, happy_eyeballs_delay=0.3) + transport, protocol = loop.run_until_complete(coro) + try: + sock = transport._sock + sock.connect.assert_called_with(('192.0.2.2', 6)) + finally: + transport.close() + test_utils.run_briefly(loop) # allow transport to close + loop.close() + class MyProto(asyncio.Protocol): done = None @@ -1190,6 +1282,36 @@ def getaddrinfo(*args, **kw): self.loop.run_until_complete(coro) self.assertTrue(sock.close.called) + @patch_socket + def test_create_connection_happy_eyeballs_empty_exceptions(self, m_socket): + # See gh-135836: Fix IndexError when Happy Eyeballs algorithm + # results in empty exceptions list + + async def getaddrinfo(*args, **kw): + return [(socket.AF_INET, socket.SOCK_STREAM, 0, '', ('127.0.0.1', 80)), + (socket.AF_INET6, socket.SOCK_STREAM, 0, '', ('::1', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + + # Mock staggered_race to return empty exceptions list + # This simulates the scenario where Happy Eyeballs algorithm + # cancels all attempts but doesn't properly collect exceptions + with mock.patch('asyncio.staggered.staggered_race') as mock_staggered: + # Return (None, []) - no winner, empty exceptions list + async def mock_race(coro_fns, delay, loop): + return None, [] + mock_staggered.side_effect = mock_race + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, happy_eyeballs_delay=0.1) + + # Should raise TimeoutError instead of IndexError + with self.assertRaisesRegex(TimeoutError, "create_connection failed"): + self.loop.run_until_complete(coro) + def test_create_connection_host_port_sock(self): coro = self.loop.create_connection( MyProto, 'example.com', 80, sock=object()) @@ -1296,6 +1418,31 @@ def getaddrinfo_task(*args, **kwds): self.assertEqual(len(cm.exception.exceptions), 1) self.assertIsInstance(cm.exception.exceptions[0], OSError) + @patch_socket + def test_create_connection_connect_non_os_err_close_err(self, m_socket): + # Test the case when sock_connect() raises non-OSError exception + # and sock.close() raises OSError. + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = CustomError + sock = mock.Mock() + m_socket.socket.return_value = sock + sock.close.side_effect = OSError + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + CustomError, self.loop.run_until_complete, coro) + + coro = self.loop.create_connection(MyProto, 'example.com', 80, all_errors=True) + self.assertRaises( + CustomError, self.loop.run_until_complete, coro) + def test_create_connection_multiple(self): async def getaddrinfo(*args, **kw): return [(2, 1, 6, '', ('0.0.0.1', 80)), diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py index 9c386dd2e63..6d3edcc36f5 100644 --- a/Lib/test/test_asyncio/test_buffered_proto.py +++ b/Lib/test/test_asyncio/test_buffered_proto.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ReceiveStuffProto(asyncio.BufferedProtocol): diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py index ad394f44e7e..f85f39839cb 100644 --- a/Lib/test/test_asyncio/test_context.py +++ b/Lib/test/test_asyncio/test_context.py @@ -4,7 +4,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 9f3b6f9acef..0561b54a3f1 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -13,7 +13,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class EagerTaskFactoryLoopTests: @@ -316,11 +316,9 @@ def tearDown(self): asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks return super().tearDown() - - @unittest.skip("skip") def test_issue105987(self): code = """if 1: - from _asyncio import _swap_current_task + from _asyncio import _swap_current_task, _set_running_loop class DummyTask: pass @@ -329,6 +327,7 @@ class DummyLoop: pass l = DummyLoop() + _set_running_loop(l) _swap_current_task(l, DummyTask()) t = _swap_current_task(l, None) """ diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 873c503fa02..919d543b032 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -38,7 +38,7 @@ from test.support import ALWAYS_EQ, LARGEST, SMALLEST def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def broken_unix_getsockname(): @@ -2843,13 +2843,13 @@ def test_default_event_loop_policy_deprecation(self): self.assertIsInstance(policy, asyncio.DefaultEventLoopPolicy) def test_event_loop_policy(self): - policy = asyncio._AbstractEventLoopPolicy() + policy = asyncio.events._AbstractEventLoopPolicy() self.assertRaises(NotImplementedError, policy.get_event_loop) self.assertRaises(NotImplementedError, policy.set_event_loop, object()) self.assertRaises(NotImplementedError, policy.new_event_loop) def test_get_event_loop(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) with self.assertRaises(RuntimeError): @@ -2857,7 +2857,7 @@ def test_get_event_loop(self): self.assertIsNone(policy._local._loop) def test_get_event_loop_does_not_call_set_event_loop(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() with mock.patch.object( policy, "set_event_loop", @@ -2869,7 +2869,7 @@ def test_get_event_loop_does_not_call_set_event_loop(self): m_set_event_loop.assert_not_called() def test_get_event_loop_after_set_none(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() policy.set_event_loop(None) self.assertRaises(RuntimeError, policy.get_event_loop) @@ -2877,7 +2877,7 @@ def test_get_event_loop_after_set_none(self): def test_get_event_loop_thread(self, m_current_thread): def f(): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() self.assertRaises(RuntimeError, policy.get_event_loop) th = threading.Thread(target=f) @@ -2885,14 +2885,14 @@ def f(): th.join() def test_new_event_loop(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() loop = policy.new_event_loop() self.assertIsInstance(loop, asyncio.AbstractEventLoop) loop.close() def test_set_event_loop(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() old_loop = policy.new_event_loop() policy.set_event_loop(old_loop) @@ -2909,7 +2909,7 @@ def test_get_event_loop_policy(self): with self.assertWarnsRegex( DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): policy = asyncio.get_event_loop_policy() - self.assertIsInstance(policy, asyncio._AbstractEventLoopPolicy) + self.assertIsInstance(policy, asyncio.events._AbstractEventLoopPolicy) self.assertIs(policy, asyncio.get_event_loop_policy()) def test_set_event_loop_policy(self): @@ -2922,7 +2922,7 @@ def test_set_event_loop_policy(self): DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): old_policy = asyncio.get_event_loop_policy() - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() with self.assertWarnsRegex( DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): asyncio.set_event_loop_policy(policy) @@ -3034,13 +3034,13 @@ def test_get_event_loop_returns_running_loop(self): class TestError(Exception): pass - class Policy(asyncio._DefaultEventLoopPolicy): + class Policy(test_utils.DefaultEventLoopPolicy): def get_event_loop(self): raise TestError - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: - asyncio._set_event_loop_policy(Policy()) + asyncio.events._set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() with self.assertRaises(TestError): @@ -3068,7 +3068,7 @@ async def func(): asyncio.get_event_loop() finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if loop is not None: loop.close() @@ -3078,9 +3078,9 @@ async def func(): self.assertIs(asyncio._get_running_loop(), None) def test_get_event_loop_returns_running_loop2(self): - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: - asyncio._set_event_loop_policy(asyncio._DefaultEventLoopPolicy()) + asyncio.events._set_event_loop_policy(test_utils.DefaultEventLoopPolicy()) loop = asyncio.new_event_loop() self.addCleanup(loop.close) @@ -3106,7 +3106,7 @@ async def func(): asyncio.get_event_loop() finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if loop is not None: loop.close() diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py index 110996c3485..d874ed00bd7 100644 --- a/Lib/test/test_asyncio/test_free_threading.py +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -15,7 +15,7 @@ class MyException(Exception): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestFreeThreading: diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 8b51522278a..666f9c9ee18 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -17,7 +17,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def _fakefunc(f): @@ -413,7 +413,7 @@ def func_repr(func): def test_copy_state(self): from asyncio.futures import _copy_future_state - f = self._new_future(loop=self.loop) + f = concurrent.futures.Future() f.set_result(10) newf = self._new_future(loop=self.loop) @@ -421,7 +421,7 @@ def test_copy_state(self): self.assertTrue(newf.done()) self.assertEqual(newf.result(), 10) - f_exception = self._new_future(loop=self.loop) + f_exception = concurrent.futures.Future() f_exception.set_exception(RuntimeError()) newf_exception = self._new_future(loop=self.loop) @@ -429,7 +429,7 @@ def test_copy_state(self): self.assertTrue(newf_exception.done()) self.assertRaises(RuntimeError, newf_exception.result) - f_cancelled = self._new_future(loop=self.loop) + f_cancelled = concurrent.futures.Future() f_cancelled.cancel() newf_cancelled = self._new_future(loop=self.loop) @@ -441,7 +441,7 @@ def test_copy_state(self): except BaseException as e: f_exc = e - f_conexc = self._new_future(loop=self.loop) + f_conexc = concurrent.futures.Future() f_conexc.set_exception(f_exc) newf_conexc = self._new_future(loop=self.loop) @@ -454,6 +454,56 @@ def test_copy_state(self): newf_tb = ''.join(traceback.format_tb(newf_exc.__traceback__)) self.assertEqual(newf_tb.count('raise concurrent.futures.InvalidStateError'), 1) + def test_copy_state_from_concurrent_futures(self): + """Test _copy_future_state from concurrent.futures.Future. + + This tests the optimized path using _get_snapshot when available. + """ + from asyncio.futures import _copy_future_state + + # Test with a result + f_concurrent = concurrent.futures.Future() + f_concurrent.set_result(42) + f_asyncio = self._new_future(loop=self.loop) + _copy_future_state(f_concurrent, f_asyncio) + self.assertTrue(f_asyncio.done()) + self.assertEqual(f_asyncio.result(), 42) + + # Test with an exception + f_concurrent_exc = concurrent.futures.Future() + f_concurrent_exc.set_exception(ValueError("test exception")) + f_asyncio_exc = self._new_future(loop=self.loop) + _copy_future_state(f_concurrent_exc, f_asyncio_exc) + self.assertTrue(f_asyncio_exc.done()) + with self.assertRaises(ValueError) as cm: + f_asyncio_exc.result() + self.assertEqual(str(cm.exception), "test exception") + + # Test with cancelled state + f_concurrent_cancelled = concurrent.futures.Future() + f_concurrent_cancelled.cancel() + f_asyncio_cancelled = self._new_future(loop=self.loop) + _copy_future_state(f_concurrent_cancelled, f_asyncio_cancelled) + self.assertTrue(f_asyncio_cancelled.cancelled()) + + # Test that destination already cancelled prevents copy + f_concurrent_result = concurrent.futures.Future() + f_concurrent_result.set_result(10) + f_asyncio_precancelled = self._new_future(loop=self.loop) + f_asyncio_precancelled.cancel() + _copy_future_state(f_concurrent_result, f_asyncio_precancelled) + self.assertTrue(f_asyncio_precancelled.cancelled()) + + # Test exception type conversion + f_concurrent_invalid = concurrent.futures.Future() + f_concurrent_invalid.set_exception(concurrent.futures.InvalidStateError("invalid")) + f_asyncio_invalid = self._new_future(loop=self.loop) + _copy_future_state(f_concurrent_invalid, f_asyncio_invalid) + self.assertTrue(f_asyncio_invalid.done()) + with self.assertRaises(asyncio.exceptions.InvalidStateError) as cm: + f_asyncio_invalid.result() + self.assertEqual(str(cm.exception), "invalid") + def test_iter(self): fut = self._new_future(loop=self.loop) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index e2cddea01ec..c7c0ebdac1b 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -7,7 +7,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class FutureTests: diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 62f6593c31d..2f22fbccba4 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -5,7 +5,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def capture_test_stack(*, fut=None, depth=1): diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 047f03cbb14..e025d2990a3 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -20,7 +20,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class LockTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index 48f4a75e0fd..a0c8434c945 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) # Test that asyncio.iscoroutine() uses collections.abc.Coroutine diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index 24c4e8546b1..edfad5e11db 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -18,7 +18,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def close_transport(transport): @@ -566,6 +566,8 @@ def test_sendto(self): self.assertTrue(self.proactor.sendto.called) self.proactor.sendto.assert_called_with( self.sock, data, addr=('0.0.0.0', 1234)) + self.assertFalse(transport._buffer) + self.assertEqual(0, transport._buffer_size) def test_sendto_bytearray(self): data = bytearray(b'data') diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py index 4484a031988..29d3bd22705 100644 --- a/Lib/test/test_asyncio/test_protocols.py +++ b/Lib/test/test_asyncio/test_protocols.py @@ -7,7 +7,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ProtocolsAbsTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 090b9774c22..54bbe79f81f 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -6,7 +6,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class QueueBasicTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 21f277bc2d8..8a4d7f5c796 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -12,14 +12,14 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def interrupt_self(): _thread.interrupt_main() -class TestPolicy(asyncio._AbstractEventLoopPolicy): +class TestPolicy(asyncio.events._AbstractEventLoopPolicy): def __init__(self, loop_factory): self.loop_factory = loop_factory @@ -61,15 +61,15 @@ def setUp(self): super().setUp() policy = TestPolicy(self.new_loop) - asyncio._set_event_loop_policy(policy) + asyncio.events._set_event_loop_policy(policy) def tearDown(self): - policy = asyncio._get_event_loop_policy() + policy = asyncio.events._get_event_loop_policy() if policy.loop is not None: self.assertTrue(policy.loop.is_closed()) self.assertTrue(policy.loop.shutdown_ag_run) - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) super().tearDown() @@ -208,7 +208,7 @@ async def main(): await asyncio.sleep(0) return 42 - policy = asyncio._get_event_loop_policy() + policy = asyncio.events._get_event_loop_policy() policy.set_event_loop = mock.Mock() asyncio.run(main()) self.assertTrue(policy.set_event_loop.called) @@ -259,7 +259,7 @@ def new_event_loop(): loop.set_task_factory(Task) return loop - asyncio._set_event_loop_policy(TestPolicy(new_event_loop)) + asyncio.events._set_event_loop_policy(TestPolicy(new_event_loop)) with self.assertRaises(asyncio.CancelledError): asyncio.run(main()) @@ -495,7 +495,7 @@ def test_set_event_loop_called_once(self): async def coro(): pass - policy = asyncio._get_event_loop_policy() + policy = asyncio.events._get_event_loop_policy() policy.set_event_loop = mock.Mock() runner = asyncio.Runner() runner.run(coro()) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index de81936b745..4bb5d4fb816 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -24,7 +24,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestBaseSelectorEventLoop(BaseSelectorEventLoop): @@ -347,6 +347,18 @@ def test_process_events_write_cancelled(self): selectors.EVENT_WRITE)]) self.loop._remove_writer.assert_called_with(1) + def test_accept_connection_zero_one(self): + for backlog in [0, 1]: + sock = mock.Mock() + sock.accept.return_value = (mock.Mock(), mock.Mock()) + with self.subTest(backlog): + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog + 1) + def test_accept_connection_multiple(self): sock = mock.Mock() sock.accept.return_value = (mock.Mock(), mock.Mock()) @@ -362,7 +374,7 @@ def test_accept_connection_multiple(self): self.loop._accept_connection( mock.Mock(), sock, backlog=backlog) self.loop.run_until_complete(asyncio.sleep(0)) - self.assertEqual(sock.accept.call_count, backlog) + self.assertEqual(sock.accept.call_count, backlog + 1) def test_accept_connection_skip_connectionabortederror(self): sock = mock.Mock() @@ -388,7 +400,7 @@ def mock_sock_accept(): # as in test_accept_connection_multiple avoid task pending # warnings by using asyncio.sleep(0) self.loop.run_until_complete(asyncio.sleep(0)) - self.assertEqual(sock.accept.call_count, backlog) + self.assertEqual(sock.accept.call_count, backlog + 1) class SelectorTransportTests(test_utils.TestCase): @@ -842,6 +854,22 @@ def test_writelines_pauses_protocol(self): self.assertTrue(self.sock.send.called) self.assertTrue(self.loop.writers) + def test_writelines_after_connection_lost(self): + # GH-136234 + transport = self.socket_transport() + self.sock.send = mock.Mock() + self.sock.send.side_effect = ConnectionResetError + transport.write(b'data1') # Will fail immediately, causing connection lost + + transport.writelines([b'data2']) + self.assertFalse(transport._buffer) + self.assertFalse(self.loop.writers) + + test_utils.run_briefly(self.loop) # Allow _call_connection_lost to run + transport.writelines([b'data2']) + self.assertFalse(transport._buffer) + self.assertFalse(self.loop.writers) + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') def test_write_sendmsg_full(self): data = memoryview(b'data') @@ -1485,6 +1513,47 @@ def test_sendto_closing(self): transport.sendto(b'data', (1,)) self.assertEqual(transport._conn_lost, 2) + def test_sendto_sendto_ready(self): + data = b'data' + + # First queue up the buffer by having the socket blocked + self.sock.sendto.side_effect = BlockingIOError + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + # Now let the socket send the buffer + self.sock.sendto.side_effect = None + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertFalse(self.loop.writers) + self.assertFalse(transport._buffer) + self.assertEqual(transport._buffer_size, 0) + + def test_sendto_sendto_ready_blocked(self): + data = b'data' + + # First queue up the buffer by having the socket blocked + self.sock.sendto.side_effect = BlockingIOError + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + # Now try to send the buffer, it will be added to buffer again if it fails + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertTrue(self.loop.writers) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + def test_sendto_ready(self): data = b'data' self.sock.sendto.return_value = len(data) diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index e1b766d06cb..dcd963b3355 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -22,7 +22,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MySendfileProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 32211f4cba3..5bd0f7e2af4 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class BaseStartServer(func_tests.FunctionalTestCaseMixin): diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index 4f7b9a1dda6..df4ec794897 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MyProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index 986ecc2c5a9..06118f3a615 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -30,7 +30,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MyBaseProto(asyncio.Protocol): @@ -195,9 +195,10 @@ async def wait_closed(self, obj): except (BrokenPipeError, ConnectionError): pass - def test_create_server_ssl_1(self): + @support.bigmemtest(size=25, memuse=90*2**20, dry_run=False) + def test_create_server_ssl_1(self, size): CNT = 0 # number of clients that were successful - TOTAL_CNT = 25 # total number of clients that test will create + TOTAL_CNT = size # total number of clients that test will create TIMEOUT = support.LONG_TIMEOUT # timeout for this test A_DATA = b'A' * 1024 * BUF_MULTIPLIER @@ -1038,9 +1039,10 @@ async def run_main(): self.loop.run_until_complete(run_main()) - def test_create_server_ssl_over_ssl(self): + @support.bigmemtest(size=25, memuse=90*2**20, dry_run=False) + def test_create_server_ssl_over_ssl(self, size): CNT = 0 # number of clients that were successful - TOTAL_CNT = 25 # total number of clients that test will create + TOTAL_CNT = size # total number of clients that test will create TIMEOUT = support.LONG_TIMEOUT # timeout for this test A_DATA = b'A' * 1024 * BUF_MULTIPLIER diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index aa248c5786f..3e304c16642 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -21,7 +21,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @unittest.skipIf(ssl is None, 'No ssl module') diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index ad34aa6da01..32e4817b70d 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class StaggeredTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 4fa4384346f..f93ee54abc6 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -18,7 +18,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class StreamTests(test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 341e3e979e0..bf301740741 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -11,7 +11,7 @@ from asyncio import subprocess from test.test_asyncio import utils as test_utils from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper, gc_collect if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -37,7 +37,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestSubprocessTransport(base_subprocess.BaseSubprocessTransport): @@ -879,6 +879,44 @@ async def main(): self.loop.run_until_complete(main()) + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + asyncio.run(main()) + gc_collect() if sys.platform != 'win32': # Unix diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 0d69a436fdb..91f6b03b459 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -15,7 +15,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MyExc(Exception): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 44498ef790e..9809621a324 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -24,7 +24,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) async def coroutine_function(): @@ -2116,6 +2116,46 @@ def test_shield_cancel_outer(self): self.assertTrue(outer.cancelled()) self.assertEqual(0, 0 if outer._callbacks is None else len(outer._callbacks)) + def test_shield_cancel_outer_result(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_result(1) + test_utils.run_briefly(self.loop) + mock_handler.assert_not_called() + + def test_shield_cancel_outer_exception(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + + def test_shield_duplicate_log_once(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + def test_shield_shortcut(self): fut = self.new_future(self.loop) fut.set_result(42) @@ -3640,6 +3680,30 @@ def task_factory(loop, coro): (loop, context), kwargs = callback.call_args self.assertEqual(context['exception'], exc_context.exception) + def test_run_coroutine_threadsafe_and_cancel(self): + task = None + thread_future = None + # Use a custom task factory to capture the created Task + def task_factory(loop, coro): + nonlocal task + task = asyncio.Task(coro, loop=loop) + return task + + self.addCleanup(self.loop.set_task_factory, + self.loop.get_task_factory()) + + async def target(): + nonlocal thread_future + self.loop.set_task_factory(task_factory) + thread_future = asyncio.run_coroutine_threadsafe(asyncio.sleep(10), self.loop) + await asyncio.sleep(0) + + thread_future.cancel() + + self.loop.run_until_complete(target()) + self.assertTrue(task.cancelled()) + self.assertTrue(thread_future.cancelled()) + class SleepTests(test_utils.TestCase): def setUp(self): diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index c98c9a9b395..8ad5f9b2c9e 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ToThreadTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index 3ba84d63b2c..f60722c48b7 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -9,7 +9,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TimeoutTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 0413e236c27..34e94830204 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -2,6 +2,13 @@ from asyncio import tools +from collections import namedtuple + +FrameInfo = namedtuple('FrameInfo', ['funcname', 'filename', 'lineno']) +CoroInfo = namedtuple('CoroInfo', ['call_stack', 'task_name']) +TaskInfo = namedtuple('TaskInfo', ['task_id', 'task_name', 'coroutine_stack', 'awaited_by']) +AwaitedInfo = namedtuple('AwaitedInfo', ['thread_id', 'awaited_by']) + # mock output of get_all_awaited_by function. TEST_INPUTS_TREE = [ @@ -10,81 +17,151 @@ # different subtasks part of a TaskGroup (root1 and root2) which call # awaiter functions. ( - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "timer", - [ - [[("awaiter3", "/path/to/app.py", 130), - ("awaiter2", "/path/to/app.py", 120), - ("awaiter", "/path/to/app.py", 110)], 4], - [[("awaiterB3", "/path/to/app.py", 190), - ("awaiterB2", "/path/to/app.py", 180), - ("awaiterB", "/path/to/app.py", 170)], 5], - [[("awaiterB3", "/path/to/app.py", 190), - ("awaiterB2", "/path/to/app.py", 180), - ("awaiterB", "/path/to/app.py", 170)], 6], - [[("awaiter3", "/path/to/app.py", 130), - ("awaiter2", "/path/to/app.py", 120), - ("awaiter", "/path/to/app.py", 110)], 7], - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ( - 8, - "root1", - [[["_aexit", "__aexit__", "main"], 2]], + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=7 + ) + ] ), - ( - 9, - "root2", - [[["_aexit", "__aexit__", "main"], 2]], + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] ), - ( - 4, - "child1_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] ), - ( - 6, - "child2_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] ), - ( - 7, - "child1_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] ), - ( - 5, - "child2_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] ), - ], + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ @@ -130,26 +207,96 @@ [ # test case containing two roots ( - ( - 9, - [ - (5, "Task-5", []), - (6, "Task-6", [[["main2"], 5]]), - (7, "Task-7", [[["main2"], 5]]), - (8, "Task-8", [[["main2"], 5]]), - ], + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] ), - ( - 10, - [ - (1, "Task-1", []), - (2, "Task-2", [[["main"], 1]]), - (3, "Task-3", [[["main"], 1]]), - (4, "Task-4", [[["main"], 1]]), - ], + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] ), - (11, []), - (0, []), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ @@ -174,18 +321,63 @@ # test case containing two roots, one of them without subtasks ( [ - (1, [(2, "Task-5", [])]), - ( - 3, - [ - (4, "Task-1", []), - (5, "Task-2", [[["main"], 4]]), - (6, "Task-3", [[["main"], 4]]), - (7, "Task-4", [[["main"], 4]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] ), - (8, []), - (0, []), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( @@ -208,19 +400,44 @@ # this test case contains a cycle: two tasks awaiting each other. ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "a", - [[["awaiter2"], 4], [["main"], 2]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - (4, "b", [[["awaiter"], 3]]), - ], + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ([[4, 3, 4]]), @@ -229,32 +446,85 @@ # this test case contains two cycles ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "A", - [[["nested", "nested", "task_b"], 4]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ( - 4, - "B", - [ - [["nested", "nested", "task_c"], 5], - [["nested", "nested", "task_a"], 3], - ], + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] ), - (5, "C", [[["nested", "nested"], 6]]), - ( - 6, - "Task-2", - [[["nested", "nested", "task_b"], 4]], + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] ), - ], + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ([[4, 3, 4], [4, 6, 5, 4]]), @@ -267,81 +537,160 @@ # different subtasks part of a TaskGroup (root1 and root2) which call # awaiter functions. ( - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "timer", - [ - [["awaiter3", "awaiter2", "awaiter"], 4], - [["awaiter1_3", "awaiter1_2", "awaiter1"], 5], - [["awaiter1_3", "awaiter1_2", "awaiter1"], 6], - [["awaiter3", "awaiter2", "awaiter"], 7], - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ( - 8, - "root1", - [[["_aexit", "__aexit__", "main"], 2]], + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=7 + ) + ] ), - ( - 9, - "root2", - [[["_aexit", "__aexit__", "main"], 2]], + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] ), - ( - 4, - "child1_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] ), - ( - 6, - "child2_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] ), - ( - 7, - "child1_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] ), - ( - 5, - "child2_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] ), - ], + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], + [1, "0x2", "Task-1", "", "", "", "0x0"], [ 1, "0x3", "timer", + "", "awaiter3 -> awaiter2 -> awaiter", "child1_1", "0x4", @@ -350,6 +699,7 @@ 1, "0x3", "timer", + "", "awaiter1_3 -> awaiter1_2 -> awaiter1", "child2_2", "0x5", @@ -358,6 +708,7 @@ 1, "0x3", "timer", + "", "awaiter1_3 -> awaiter1_2 -> awaiter1", "child2_1", "0x6", @@ -366,6 +717,7 @@ 1, "0x3", "timer", + "", "awaiter3 -> awaiter2 -> awaiter", "child1_2", "0x7", @@ -374,6 +726,7 @@ 1, "0x8", "root1", + "", "_aexit -> __aexit__ -> main", "Task-1", "0x2", @@ -382,6 +735,7 @@ 1, "0x9", "root2", + "", "_aexit -> __aexit__ -> main", "Task-1", "0x2", @@ -390,6 +744,7 @@ 1, "0x4", "child1_1", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root1", "0x8", @@ -398,6 +753,7 @@ 1, "0x6", "child2_1", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root1", "0x8", @@ -406,6 +762,7 @@ 1, "0x7", "child1_2", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root2", "0x9", @@ -414,6 +771,7 @@ 1, "0x5", "child2_2", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root2", "0x9", @@ -424,37 +782,107 @@ [ # test case containing two roots ( - ( - 9, - [ - (5, "Task-5", []), - (6, "Task-6", [[["main2"], 5]]), - (7, "Task-7", [[["main2"], 5]]), - (8, "Task-8", [[["main2"], 5]]), - ], + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] ), - ( - 10, - [ - (1, "Task-1", []), - (2, "Task-2", [[["main"], 1]]), - (3, "Task-3", [[["main"], 1]]), - (4, "Task-4", [[["main"], 1]]), - ], + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] ), - (11, []), - (0, []), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ - [9, "0x5", "Task-5", "", "", "0x0"], - [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], - [9, "0x7", "Task-7", "main2", "Task-5", "0x5"], - [9, "0x8", "Task-8", "main2", "Task-5", "0x5"], - [10, "0x1", "Task-1", "", "", "0x0"], - [10, "0x2", "Task-2", "main", "Task-1", "0x1"], - [10, "0x3", "Task-3", "main", "Task-1", "0x1"], - [10, "0x4", "Task-4", "main", "Task-1", "0x1"], + [9, "0x5", "Task-5", "", "", "", "0x0"], + [9, "0x6", "Task-6", "", "main2", "Task-5", "0x5"], + [9, "0x7", "Task-7", "", "main2", "Task-5", "0x5"], + [9, "0x8", "Task-8", "", "main2", "Task-5", "0x5"], + [10, "0x1", "Task-1", "", "", "", "0x0"], + [10, "0x2", "Task-2", "", "main", "Task-1", "0x1"], + [10, "0x3", "Task-3", "", "main", "Task-1", "0x1"], + [10, "0x4", "Task-4", "", "main", "Task-1", "0x1"], ] ), ], @@ -462,27 +890,72 @@ # test case containing two roots, one of them without subtasks ( [ - (1, [(2, "Task-5", [])]), - ( - 3, - [ - (4, "Task-1", []), - (5, "Task-2", [[["main"], 4]]), - (6, "Task-3", [[["main"], 4]]), - (7, "Task-4", [[["main"], 4]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] ), - (8, []), - (0, []), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-5", "", "", "0x0"], - [3, "0x4", "Task-1", "", "", "0x0"], - [3, "0x5", "Task-2", "main", "Task-1", "0x4"], - [3, "0x6", "Task-3", "main", "Task-1", "0x4"], - [3, "0x7", "Task-4", "main", "Task-1", "0x4"], + [1, "0x2", "Task-5", "", "", "", "0x0"], + [3, "0x4", "Task-1", "", "", "", "0x0"], + [3, "0x5", "Task-2", "", "main", "Task-1", "0x4"], + [3, "0x6", "Task-3", "", "main", "Task-1", "0x4"], + [3, "0x7", "Task-4", "", "main", "Task-1", "0x4"], ] ), ], @@ -491,27 +964,52 @@ # this test case contains a cycle: two tasks awaiting each other. ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "a", - [[["awaiter2"], 4], [["main"], 2]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - (4, "b", [[["awaiter"], 3]]), - ], + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], - [1, "0x3", "a", "awaiter2", "b", "0x4"], - [1, "0x3", "a", "main", "Task-1", "0x2"], - [1, "0x4", "b", "awaiter", "a", "0x3"], + [1, "0x2", "Task-1", "", "", "", "0x0"], + [1, "0x3", "a", "", "awaiter2", "b", "0x4"], + [1, "0x3", "a", "", "main", "Task-1", "0x2"], + [1, "0x4", "b", "", "awaiter", "a", "0x3"], ] ), ], @@ -519,41 +1017,95 @@ # this test case contains two cycles ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "A", - [[["nested", "nested", "task_b"], 4]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ( - 4, - "B", - [ - [["nested", "nested", "task_c"], 5], - [["nested", "nested", "task_a"], 3], - ], + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] ), - (5, "C", [[["nested", "nested"], 6]]), - ( - 6, - "Task-2", - [[["nested", "nested", "task_b"], 4]], + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] ), - ], + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], + [1, "0x2", "Task-1", "", "", "", "0x0"], [ 1, "0x3", "A", + "", "nested -> nested -> task_b", "B", "0x4", @@ -562,6 +1114,7 @@ 1, "0x4", "B", + "", "nested -> nested -> task_c", "C", "0x5", @@ -570,6 +1123,7 @@ 1, "0x4", "B", + "", "nested -> nested -> task_a", "A", "0x3", @@ -578,6 +1132,7 @@ 1, "0x5", "C", + "", "nested -> nested", "Task-2", "0x6", @@ -586,6 +1141,7 @@ 1, "0x6", "Task-2", + "", "nested -> nested -> task_b", "B", "0x4", @@ -600,7 +1156,8 @@ class TestAsyncioToolsTree(unittest.TestCase): def test_asyncio_utils(self): for input_, tree in TEST_INPUTS_TREE: with self.subTest(input_): - self.assertEqual(tools.build_async_tree(input_), tree) + result = tools.build_async_tree(input_) + self.assertEqual(result, tree) def test_asyncio_utils_cycles(self): for input_, cycles in TEST_INPUTS_CYCLES_TREE: @@ -615,7 +1172,8 @@ class TestAsyncioToolsTable(unittest.TestCase): def test_asyncio_utils(self): for input_, table in TEST_INPUTS_TABLE: with self.subTest(input_): - self.assertEqual(tools.build_task_table(input_), table) + result = tools.build_task_table(input_) + self.assertEqual(result, table) class TestAsyncioToolsBasic(unittest.TestCase): @@ -632,26 +1190,67 @@ def test_empty_input_table(self): self.assertEqual(tools.build_task_table(result), expected_output) def test_only_independent_tasks_tree(self): - input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] expected = [["└── (T) taskA"], ["└── (T) taskB"]] result = tools.build_async_tree(input_) self.assertEqual(sorted(result), sorted(expected)) def test_only_independent_tasks_table(self): - input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] self.assertEqual( tools.build_task_table(input_), - [[1, "0xa", "taskA", "", "", "0x0"], [1, "0xb", "taskB", "", "", "0x0"]], + [[1, '0xa', 'taskA', '', '', '', '0x0'], [1, '0xb', 'taskB', '', '', '', '0x0']] ) def test_single_task_tree(self): """Test build_async_tree with a single task and no awaits.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] expected_output = [ @@ -664,25 +1263,50 @@ def test_single_task_tree(self): def test_single_task_table(self): """Test build_task_table with a single task and no awaits.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] - expected_output = [[1, "0x2", "Task-1", "", "", "0x0"]] + expected_output = [[1, '0x2', 'Task-1', '', '', '', '0x0']] self.assertEqual(tools.build_task_table(result), expected_output) def test_cycle_detection(self): """Test build_async_tree raises CycleFoundException for cyclic input.""" result = [ - ( - 1, - [ - (2, "Task-1", [[["main"], 3]]), - (3, "Task-2", [[["main"], 2]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ) + ] ) ] with self.assertRaises(tools.CycleFoundException) as context: @@ -692,13 +1316,38 @@ def test_cycle_detection(self): def test_complex_tree(self): """Test build_async_tree with a more complex tree structure.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - (3, "Task-2", [[["main"], 2]]), - (4, "Task-3", [[["main"], 3]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] ) ] expected_output = [ @@ -715,30 +1364,76 @@ def test_complex_tree(self): def test_complex_table(self): """Test build_task_table with a more complex tree structure.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - (3, "Task-2", [[["main"], 2]]), - (4, "Task-3", [[["main"], 3]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] ) ] expected_output = [ - [1, "0x2", "Task-1", "", "", "0x0"], - [1, "0x3", "Task-2", "main", "Task-1", "0x2"], - [1, "0x4", "Task-3", "main", "Task-2", "0x3"], + [1, '0x2', 'Task-1', '', '', '', '0x0'], + [1, '0x3', 'Task-2', '', 'main', 'Task-1', '0x2'], + [1, '0x4', 'Task-3', '', 'main', 'Task-2', '0x3'] ] self.assertEqual(tools.build_task_table(result), expected_output) def test_deep_coroutine_chain(self): input_ = [ - ( - 1, - [ - (10, "leaf", [[["c1", "c2", "c3", "c4", "c5"], 11]]), - (11, "root", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="leaf", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("c1", "", 0), + FrameInfo("c2", "", 0), + FrameInfo("c3", "", 0), + FrameInfo("c4", "", 0), + FrameInfo("c5", "", 0) + ], + task_name=11 + ) + ] + ), + TaskInfo( + task_id=11, + task_name="root", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] expected = [ @@ -757,13 +1452,47 @@ def test_deep_coroutine_chain(self): def test_multiple_cycles_same_node(self): input_ = [ - ( - 1, - [ - (1, "Task-A", [[["call1"], 2]]), - (2, "Task-B", [[["call2"], 3]]), - (3, "Task-C", [[["call3"], 1], [["call4"], 2]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call2", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call3", "", 0)], + task_name=1 + ), + CoroInfo( + call_stack=[FrameInfo("call4", "", 0)], + task_name=2 + ) + ] + ) + ] ) ] with self.assertRaises(tools.CycleFoundException) as ctx: @@ -772,48 +1501,130 @@ def test_multiple_cycles_same_node(self): self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) def test_table_output_format(self): - input_ = [(1, [(1, "Task-A", [[["foo"], 2]]), (2, "Task-B", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("foo", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] table = tools.build_task_table(input_) for row in table: - self.assertEqual(len(row), 6) + self.assertEqual(len(row), 7) self.assertIsInstance(row[0], int) # thread ID self.assertTrue( isinstance(row[1], str) and row[1].startswith("0x") ) # hex task ID self.assertIsInstance(row[2], str) # task name - self.assertIsInstance(row[3], str) # coroutine chain - self.assertIsInstance(row[4], str) # awaiter name + self.assertIsInstance(row[3], str) # coroutine stack + self.assertIsInstance(row[4], str) # coroutine chain + self.assertIsInstance(row[5], str) # awaiter name self.assertTrue( - isinstance(row[5], str) and row[5].startswith("0x") + isinstance(row[6], str) and row[6].startswith("0x") ) # hex awaiter ID class TestAsyncioToolsEdgeCases(unittest.TestCase): def test_task_awaits_self(self): - """A task directly awaits itself – should raise a cycle.""" - input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])] + """A task directly awaits itself - should raise a cycle.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Self-Awaiter", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("loopback", "", 0)], + task_name=1 + ) + ] + ) + ] + ) + ] with self.assertRaises(tools.CycleFoundException) as ctx: tools.build_async_tree(input_) self.assertIn([1, 1], ctx.exception.cycles) def test_task_with_missing_awaiter_id(self): - """Awaiter ID not in task list – should not crash, just show 'Unknown'.""" - input_ = [(1, [(1, "Task-A", [[["coro"], 999]])])] # 999 not defined + """Awaiter ID not in task list - should not crash, just show 'Unknown'.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("coro", "", 0)], + task_name=999 + ) + ] + ) + ] + ) + ] table = tools.build_task_table(input_) self.assertEqual(len(table), 1) - self.assertEqual(table[0][4], "Unknown") + self.assertEqual(table[0][5], "Unknown") def test_duplicate_coroutine_frames(self): - """Same coroutine frame repeated under a parent – should deduplicate.""" + """Same coroutine frame repeated under a parent - should deduplicate.""" input_ = [ - ( - 1, - [ - (1, "Task-1", [[["frameA"], 2], [["frameA"], 3]]), - (2, "Task-2", []), - (3, "Task-3", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=2 + ), + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] tree = tools.build_async_tree(input_) @@ -829,15 +1640,64 @@ def test_duplicate_coroutine_frames(self): self.assertIn("Task-1", flat) def test_task_with_no_name(self): - """Task with no name in id2name – should still render with fallback.""" - input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])] + """Task with no name in id2name - should still render with fallback.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="root", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("f1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name=None, + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] # If name is None, fallback to string should not crash tree = tools.build_async_tree(input_) self.assertIn("(T) None", "\n".join(tree[0])) def test_tree_rendering_with_custom_emojis(self): """Pass custom emojis to the tree renderer.""" - input_ = [(1, [(1, "MainTask", [[["f1", "f2"], 2]]), (2, "SubTask", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="MainTask", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("f1", "", 0), + FrameInfo("f2", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="SubTask", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] tree = tools.build_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") flat = "\n".join(tree[0]) self.assertIn("🧵 MainTask", flat) diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index af10d3dc2a8..dbb572e2e15 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -10,7 +10,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TransportTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index e020c1f3e4f..d2b3de3b9a4 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -15,7 +15,7 @@ from unittest import mock from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper from test.support import socket_helper from test.support import wait_process from test.support import hashlib_helper @@ -30,7 +30,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) MOCK_ANY = mock.ANY @@ -1180,11 +1180,48 @@ async def runner(): @support.requires_fork() -class TestFork(unittest.IsolatedAsyncioTestCase): +class TestFork(unittest.TestCase): - async def test_fork_not_share_event_loop(self): + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_fork_not_share_current_task(self): + loop = object() + task = object() + asyncio._set_running_loop(loop) + self.addCleanup(asyncio._set_running_loop, None) + asyncio.tasks._enter_task(loop, task) + self.addCleanup(asyncio.tasks._leave_task, loop, task) + self.assertIs(asyncio.current_task(), task) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + asyncio._set_running_loop(loop) + current_task = asyncio.current_task() + if current_task is None: + os.write(w, b'NO TASK') + else: + os.write(w, b'TASK:' + str(id(current_task)).encode()) + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + asyncio._set_running_loop(None) + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO TASK') + wait_process(pid, exitcode=0) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_fork_not_share_event_loop(self): # The forked process should not share the event loop with the parent - loop = asyncio.get_running_loop() + loop = object() + asyncio._set_running_loop(loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.addCleanup(asyncio._set_running_loop, None) r, w = os.pipe() self.addCleanup(os.close, r) self.addCleanup(os.close, w) @@ -1206,6 +1243,7 @@ async def test_fork_not_share_event_loop(self): self.assertEqual(result, b'NO LOOP') wait_process(pid, exitcode=0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_signal_handling(self): @@ -1253,6 +1291,7 @@ async def func(): self.assertFalse(parent_handled.is_set()) self.assertTrue(child_handled.is_set()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_run(self): @@ -1273,6 +1312,7 @@ async def child_main(): self.assertEqual(result.value, 42) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_subprocess(self): diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index d083f6b4d2a..dedc6bf69d7 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) # The following value can be used as a very small timeout: diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 69e9905205e..0af3368627a 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -19,7 +19,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class UpperProto(asyncio.Protocol): @@ -330,16 +330,16 @@ def test_selector_win_policy(self): async def main(): self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: with self.assertWarnsRegex( DeprecationWarning, "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", ): - asyncio._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.events._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) def test_proactor_win_policy(self): async def main(): @@ -347,16 +347,16 @@ async def main(): asyncio.get_running_loop(), asyncio.ProactorEventLoop) - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: with self.assertWarnsRegex( DeprecationWarning, "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", ): - asyncio._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + asyncio.events._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index a6b207567c4..97f078ff911 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -16,7 +16,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class PipeTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 0a96573a81c..a480e16e81b 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -601,3 +601,9 @@ def func(): await asyncio.sleep(0) if exc is not None: raise exc + + +if sys.platform == 'win32': + DefaultEventLoopPolicy = asyncio.windows_events._DefaultEventLoopPolicy +else: + DefaultEventLoopPolicy = asyncio.unix_events._DefaultEventLoopPolicy diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index eb01da6e88a..8256ff183f2 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -1,9 +1,11 @@ import atexit import os +import subprocess import textwrap import unittest from test import support -from test.support import script_helper +from test.support import SuppressCrashReport, script_helper +from test.support import os_helper from test.support import threading_helper class GeneralTest(unittest.TestCase): @@ -79,6 +81,62 @@ def thready(): # want them to affect the rest of the tests. script_helper.assert_python_ok("-c", textwrap.dedent(source)) + @threading_helper.requires_working_threading() + def test_thread_created_in_atexit(self): + source = """if True: + import atexit + import threading + import time + + + def run(): + print(24) + time.sleep(1) + print(42) + + @atexit.register + def start_thread(): + threading.Thread(target=run).start() + """ + return_code, stdout, stderr = script_helper.assert_python_ok("-c", source) + self.assertEqual(return_code, 0) + self.assertEqual(stdout, f"24{os.linesep}42{os.linesep}".encode("utf-8")) + self.assertEqual(stderr, b"") + + @threading_helper.requires_working_threading() + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_thread_created_in_atexit_subinterpreter(self): + try: + from concurrent import interpreters + except ImportError: + self.skipTest("subinterpreters are not available") + + read, write = os.pipe() + source = f"""if True: + import atexit + import threading + import time + import os + + def run(): + os.write({write}, b'spanish') + time.sleep(1) + os.write({write}, b'inquisition') + + @atexit.register + def start_thread(): + threading.Thread(target=run).start() + """ + interp = interpreters.create() + try: + interp.exec(source) + + # Close the interpreter to invoke atexit callbacks + interp.close() + self.assertEqual(os.read(read, 100), b"spanishinquisition") + finally: + os.close(read) + os.close(write) @support.cpython_only class SubinterpreterTest(unittest.TestCase): @@ -133,6 +191,37 @@ def callback(): self.assertEqual(os.read(r, len(expected)), expected) os.close(r) + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_atexit_with_low_memory(self): + # gh-140080: Test that setting low memory after registering an atexit + # callback doesn't cause an infinite loop during finalization. + code = textwrap.dedent(""" + import atexit + import _testcapi + + def callback(): + print("hello") + + atexit.register(callback) + # Simulate low memory condition + _testcapi.set_nomemory(0) + """) + + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script(temp_dir, 'test_atexit_script', code) + with SuppressCrashReport(): + with script_helper.spawn_python(script, + stderr=subprocess.PIPE) as proc: + proc.wait() + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + self.assertIn(proc.returncode, (0, 1)) + self.assertNotIn(b"hello", stdout) + self.assertIn(b"MemoryError", stderr) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 2b24b5d7927..db4e1eb9999 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -134,7 +134,7 @@ def test_socket(self): self.assertEqual(events[0][0], "socket.gethostname") self.assertEqual(events[1][0], "socket.__new__") self.assertEqual(events[2][0], "socket.bind") - self.assertTrue(events[2][2].endswith("('127.0.0.1', 8080)")) + self.assertEndsWith(events[2][2], "('127.0.0.1', 8080)") def test_gc(self): returncode, events, stderr = self.run_python("test_gc") @@ -322,6 +322,23 @@ def test_assert_unicode(self): if returncode: self.fail(stderr) + @support.support_remote_exec_only + @support.cpython_only + def test_sys_remote_exec(self): + returncode, events, stderr = self.run_python("test_sys_remote_exec") + self.assertTrue(any(["sys.remote_exec" in event for event in events])) + self.assertTrue(any(["cpython.remote_debugger_script" in event for event in events])) + if returncode: + self.fail(stderr) + + def test_import_module(self): + self.do_test("test_import_module") + + def test_builtin__import__(self): + self.do_test("test_builtin__import__") + + def test_import_statement(self): + self.do_test("test_import_statement") if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 9efebc43d91..ac3f0940545 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -1,6 +1,7 @@ import unittest import base64 import binascii +import string import os from array import array from test.support import cpython_only @@ -14,6 +15,8 @@ class LazyImportTest(unittest.TestCase): def test_lazy_import(self): ensure_lazy_imports("base64", {"re", "getopt"}) +from test.support.hypothesis_helper import hypothesis + class LegacyBase64TestCase(unittest.TestCase): @@ -68,6 +71,13 @@ def test_decodebytes(self): eq(base64.decodebytes(array('B', b'YWJj\n')), b'abc') self.check_type_errors(base64.decodebytes) + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz') + def test_bytes_encode_decode_round_trip(self, payload): + encoded = base64.encodebytes(payload) + decoded = base64.decodebytes(encoded) + self.assertEqual(payload, decoded) + def test_encode(self): eq = self.assertEqual from io import BytesIO, StringIO @@ -96,6 +106,19 @@ def test_decode(self): self.assertRaises(TypeError, base64.encode, BytesIO(b'YWJj\n'), StringIO()) self.assertRaises(TypeError, base64.encode, StringIO('YWJj\n'), StringIO()) + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz') + def test_legacy_encode_decode_round_trip(self, payload): + from io import BytesIO + payload_file_r = BytesIO(payload) + encoded_file_w = BytesIO() + base64.encode(payload_file_r, encoded_file_w) + encoded_file_r = BytesIO(encoded_file_w.getvalue()) + decoded_file_w = BytesIO() + base64.decode(encoded_file_r, decoded_file_w) + decoded = decoded_file_w.getvalue() + self.assertEqual(payload, decoded) + class BaseXYTestCase(unittest.TestCase): @@ -208,18 +231,6 @@ def test_b64decode(self): self.check_other_types(base64.b64decode, b"YWJj", b"abc") self.check_decode_type_errors(base64.b64decode) - # Test with arbitrary alternative characters - tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d', - } - for (data, altchars), res in tests_altchars.items(): - data_str = data.decode('ascii') - altchars_str = altchars.decode('ascii') - - eq(base64.b64decode(data, altchars=altchars), res) - eq(base64.b64decode(data_str, altchars=altchars), res) - eq(base64.b64decode(data, altchars=altchars_str), res) - eq(base64.b64decode(data_str, altchars=altchars_str), res) - # Test standard alphabet for data, res in tests.items(): eq(base64.standard_b64decode(data), res) @@ -240,6 +251,20 @@ def test_b64decode(self): b'\xd3V\xbeo\xf7\x1d') self.check_decode_type_errors(base64.urlsafe_b64decode) + def test_b64decode_altchars(self): + # Test with arbitrary alternative characters + eq = self.assertEqual + res = b'\xd3V\xbeo\xf7\x1d' + for altchars in b'*$', b'+/', b'/+', b'+_', b'-+', b'-/', b'/_': + data = b'01a%cb%ccd' % tuple(altchars) + data_str = data.decode('ascii') + altchars_str = altchars.decode('ascii') + + eq(base64.b64decode(data, altchars=altchars), res) + eq(base64.b64decode(data_str, altchars=altchars), res) + eq(base64.b64decode(data, altchars=altchars_str), res) + eq(base64.b64decode(data_str, altchars=altchars_str), res) + def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') self.assertRaises(binascii.Error, base64.b64decode, 'abc') @@ -272,9 +297,49 @@ def test_b64decode_invalid_chars(self): base64.b64decode(bstr.decode('ascii'), validate=True) # Normal alphabet characters not discarded when alternative given - res = b'\xFB\xEF\xBE\xFF\xFF\xFF' - self.assertEqual(base64.b64decode(b'++[[//]]', b'[]'), res) - self.assertEqual(base64.urlsafe_b64decode(b'++--//__'), res) + res = b'\xfb\xef\xff' + self.assertEqual(base64.b64decode(b'++//', validate=True), res) + self.assertEqual(base64.b64decode(b'++//', '-_', validate=True), res) + self.assertEqual(base64.b64decode(b'--__', '-_', validate=True), res) + self.assertEqual(base64.urlsafe_b64decode(b'++//'), res) + self.assertEqual(base64.urlsafe_b64decode(b'--__'), res) + + def _altchars_strategy(): + """Generate 'altchars' for base64 encoding.""" + reserved_chars = (string.digits + string.ascii_letters + "=").encode() + allowed_chars = hypothesis.strategies.sampled_from( + [n for n in range(256) if n not in reserved_chars]) + two_bytes_strategy = hypothesis.strategies.lists( + allowed_chars, min_size=2, max_size=2, unique=True).map(bytes) + return (hypothesis.strategies.none() + | hypothesis.strategies.just(b"_-") + | two_bytes_strategy) + + @hypothesis.given( + payload=hypothesis.strategies.binary(), + altchars=_altchars_strategy(), + validate=hypothesis.strategies.booleans()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', b"_-", True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', b"_-", False) + def test_b64_encode_decode_round_trip(self, payload, altchars, validate): + encoded = base64.b64encode(payload, altchars=altchars) + decoded = base64.b64decode(encoded, altchars=altchars, + validate=validate) + self.assertEqual(payload, decoded) + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz') + def test_standard_b64_encode_decode_round_trip(self, payload): + encoded = base64.standard_b64encode(payload) + decoded = base64.standard_b64decode(encoded) + self.assertEqual(payload, decoded) + + @hypothesis.given(payload=hypothesis.strategies.binary()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz') + def test_urlsafe_b64_encode_decode_round_trip(self, payload): + encoded = base64.urlsafe_b64encode(payload) + decoded = base64.urlsafe_b64decode(encoded) + self.assertEqual(payload, decoded) def test_b32encode(self): eq = self.assertEqual @@ -329,23 +394,33 @@ def test_b32decode_casefold(self): self.assertRaises(binascii.Error, base64.b32decode, b'me======') self.assertRaises(binascii.Error, base64.b32decode, 'me======') + def test_b32decode_map01(self): # Mapping zero and one - eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') - eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe') + eq = self.assertEqual + res_L = b'b\xdd\xad\xf3\xbe' + res_I = b'b\x1d\xad\xf3\xbe' + eq(base64.b32decode(b'MLO23456'), res_L) + eq(base64.b32decode('MLO23456'), res_L) + eq(base64.b32decode(b'MIO23456'), res_I) + eq(base64.b32decode('MIO23456'), res_I) + self.assertRaises(binascii.Error, base64.b32decode, b'M1023456') + self.assertRaises(binascii.Error, base64.b32decode, b'M1O23456') + self.assertRaises(binascii.Error, base64.b32decode, b'ML023456') + self.assertRaises(binascii.Error, base64.b32decode, b'MI023456') - map_tests = {(b'M1023456', b'L'): b'b\xdd\xad\xf3\xbe', - (b'M1023456', b'I'): b'b\x1d\xad\xf3\xbe', - } - for (data, map01), res in map_tests.items(): - data_str = data.decode('ascii') + data = b'M1023456' + data_str = data.decode('ascii') + for map01, res in [(b'L', res_L), (b'I', res_I)]: map01_str = map01.decode('ascii') eq(base64.b32decode(data, map01=map01), res) eq(base64.b32decode(data_str, map01=map01), res) eq(base64.b32decode(data, map01=map01_str), res) eq(base64.b32decode(data_str, map01=map01_str), res) - self.assertRaises(binascii.Error, base64.b32decode, data) - self.assertRaises(binascii.Error, base64.b32decode, data_str) + + eq(base64.b32decode(b'M1O23456', map01=map01), res) + eq(base64.b32decode(b'M%c023456' % map01, map01=map01), res) + eq(base64.b32decode(b'M%cO23456' % map01, map01=map01), res) def test_b32decode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF'] @@ -363,6 +438,19 @@ def test_b32decode_error(self): with self.assertRaises(binascii.Error): base64.b32decode(data.decode('ascii')) + @hypothesis.given( + payload=hypothesis.strategies.binary(), + casefold=hypothesis.strategies.booleans(), + map01=( + hypothesis.strategies.none() + | hypothesis.strategies.binary(min_size=1, max_size=1))) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True, None) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False, None) + def test_b32_encode_decode_round_trip(self, payload, casefold, map01): + encoded = base64.b32encode(payload) + decoded = base64.b32decode(encoded, casefold=casefold, map01=map01) + self.assertEqual(payload, decoded) + def test_b32hexencode(self): test_cases = [ # to_encode, expected @@ -432,6 +520,15 @@ def test_b32hexdecode_error(self): with self.assertRaises(binascii.Error): base64.b32hexdecode(data.decode('ascii')) + @hypothesis.given( + payload=hypothesis.strategies.binary(), + casefold=hypothesis.strategies.booleans()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False) + def test_b32_hexencode_decode_round_trip(self, payload, casefold): + encoded = base64.b32hexencode(payload) + decoded = base64.b32hexdecode(encoded, casefold=casefold) + self.assertEqual(payload, decoded) def test_b16encode(self): eq = self.assertEqual @@ -469,6 +566,16 @@ def test_b16decode(self): # Incorrect "padding" self.assertRaises(binascii.Error, base64.b16decode, '010') + @hypothesis.given( + payload=hypothesis.strategies.binary(), + casefold=hypothesis.strategies.booleans()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False) + def test_b16_encode_decode_round_trip(self, payload, casefold): + endoded = base64.b16encode(payload) + decoded = base64.b16decode(endoded, casefold=casefold) + self.assertEqual(payload, decoded) + def test_a85encode(self): eq = self.assertEqual @@ -799,6 +906,61 @@ def test_z85decode_errors(self): self.assertRaises(ValueError, base64.z85decode, b'%nSc') self.assertRaises(ValueError, base64.z85decode, b'%nSc1') + def add_padding(self, payload): + """Add the expected padding for test_?85_encode_decode_round_trip.""" + if len(payload) % 4 != 0: + padding = b"\0" * ((-len(payload)) % 4) + payload = payload + padding + return payload + + @hypothesis.given( + payload=hypothesis.strategies.binary(), + foldspaces=hypothesis.strategies.booleans(), + wrapcol=( + hypothesis.strategies.just(0) + | hypothesis.strategies.integers(1, 1000)), + pad=hypothesis.strategies.booleans(), + adobe=hypothesis.strategies.booleans(), + ) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False, 0, False, False) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False, 20, True, True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True, 0, False, True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True, 20, True, False) + def test_a85_encode_decode_round_trip( + self, payload, foldspaces, wrapcol, pad, adobe + ): + encoded = base64.a85encode( + payload, foldspaces=foldspaces, wrapcol=wrapcol, + pad=pad, adobe=adobe, + ) + if wrapcol: + if adobe and wrapcol == 1: + # "adobe" needs wrapcol to be at least 2. + # a85decode quietly uses 2 if 1 is given; it's not worth + # loudly deprecating this behavior. + wrapcol = 2 + for line in encoded.splitlines(keepends=False): + self.assertLessEqual(len(line), wrapcol) + if adobe: + self.assertTrue(encoded.startswith(b'<~')) + self.assertTrue(encoded.endswith(b'~>')) + decoded = base64.a85decode(encoded, foldspaces=foldspaces, adobe=adobe) + if pad: + payload = self.add_padding(payload) + self.assertEqual(payload, decoded) + + @hypothesis.given( + payload=hypothesis.strategies.binary(), + pad=hypothesis.strategies.booleans()) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', True) + @hypothesis.example(b'abcdefghijklmnopqrstuvwxyz', False) + def test_b85_encode_decode_round_trip(self, payload, pad): + encoded = base64.b85encode(payload, pad=pad) + if pad: + payload = self.add_padding(payload) + decoded = base64.b85decode(encoded) + self.assertEqual(payload, decoded) + def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, base64.standard_b64decode, @@ -812,7 +974,7 @@ def test_decode_nonascii_str(self): self.assertRaises(ValueError, f, 'with non-ascii \xcb') def test_ErrorHeritage(self): - self.assertTrue(issubclass(binascii.Error, ValueError)) + self.assertIsSubclass(binascii.Error, ValueError) def test_RFC4648_test_cases(self): # test cases from RFC 4648 section 10 diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index e599b02c17d..12d4088842b 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -10,13 +10,11 @@ class ExceptionClassTests(unittest.TestCase): inheritance hierarchy)""" def test_builtins_new_style(self): - self.assertTrue(issubclass(Exception, object)) + self.assertIsSubclass(Exception, object) def verify_instance_interface(self, ins): for attr in ("args", "__str__", "__repr__"): - self.assertTrue(hasattr(ins, attr), - "%s missing %s attribute" % - (ins.__class__.__name__, attr)) + self.assertHasAttr(ins, attr) def test_inheritance(self): # Make sure the inheritance hierarchy matches the documentation @@ -65,7 +63,7 @@ def test_inheritance(self): elif last_depth > depth: while superclasses[-1][0] >= depth: superclasses.pop() - self.assertTrue(issubclass(exc, superclasses[-1][1]), + self.assertIsSubclass(exc, superclasses[-1][1], "%s is not a subclass of %s" % (exc.__name__, superclasses[-1][1].__name__)) try: # Some exceptions require arguments; just skip them diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 1f3b6746ce4..7ed7d7c47b6 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -38,13 +38,13 @@ def assertConversion(self, original, converted, restored, **kwargs): def test_exceptions(self): # Check module exceptions - self.assertTrue(issubclass(binascii.Error, Exception)) - self.assertTrue(issubclass(binascii.Incomplete, Exception)) + self.assertIsSubclass(binascii.Error, Exception) + self.assertIsSubclass(binascii.Incomplete, Exception) def test_functions(self): # Check presence of all functions for name in all_functions: - self.assertTrue(hasattr(getattr(binascii, name), '__call__')) + self.assertHasAttr(getattr(binascii, name), '__call__') self.assertRaises(TypeError, getattr(binascii, name)) def test_returned_value(self): diff --git a/Lib/test/test_binop.py b/Lib/test/test_binop.py index 299af09c498..b224c3d4e60 100644 --- a/Lib/test/test_binop.py +++ b/Lib/test/test_binop.py @@ -383,7 +383,7 @@ def test_comparison_orders(self): self.assertEqual(op_sequence(le, B, C), ['C.__ge__', 'B.__le__']) self.assertEqual(op_sequence(le, C, B), ['C.__le__', 'B.__ge__']) - self.assertTrue(issubclass(V, B)) + self.assertIsSubclass(V, B) self.assertEqual(op_sequence(eq, B, V), ['B.__eq__', 'V.__eq__']) self.assertEqual(op_sequence(le, B, V), ['B.__le__', 'V.__ge__']) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 61921e93e85..19582e75716 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -2879,11 +2879,11 @@ def test_memoryview_tolist(self): def test_memoryview_repr(self): m = memoryview(bytearray(9)) r = m.__repr__() - self.assertTrue(r.startswith(" +).resolve() +MODULE_PATH = BASE_PATH / 'Tools' / 'build' / 'generate-build-details.py' + +try: + # Import "generate-build-details.py" as "generate_build_details" + spec = importlib.util.spec_from_file_location( + "generate_build_details", MODULE_PATH + ) + generate_build_details = importlib.util.module_from_spec(spec) + sys.modules["generate_build_details"] = generate_build_details + spec.loader.exec_module(generate_build_details) +except (FileNotFoundError, ImportError): + generate_build_details = None class FormatTestsBase: @@ -31,16 +53,15 @@ def key(self, name): value = value[part] return value - def test_parse(self): - self.data - def test_top_level_container(self): self.assertIsInstance(self.data, dict) for key, value in self.data.items(): with self.subTest(key=key): - if key in ('schema_version', 'base_prefix', 'base_interpreter', 'platform'): + if key in ('schema_version', 'base_prefix', 'base_interpreter', + 'platform'): self.assertIsInstance(value, str) - elif key in ('language', 'implementation', 'abi', 'suffixes', 'libpython', 'c_api', 'arbitrary_data'): + elif key in ('language', 'implementation', 'abi', 'suffixes', + 'libpython', 'c_api', 'arbitrary_data'): self.assertIsInstance(value, dict) def test_base_prefix(self): @@ -71,15 +92,20 @@ def test_language_version_info(self): self.assertEqual(len(value), sys.version_info.n_fields) for part_name, part_value in value.items(): with self.subTest(part=part_name): - self.assertEqual(part_value, getattr(sys.version_info, part_name)) + sys_version_value = getattr(sys.version_info, part_name) + self.assertEqual(part_value, sys_version_value) def test_implementation(self): + impl_ver = sys.implementation.version for key, value in self.key('implementation').items(): with self.subTest(part=key): if key == 'version': - self.assertEqual(len(value), len(sys.implementation.version)) + self.assertEqual(len(value), len(impl_ver)) for part_name, part_value in value.items(): - self.assertEqual(getattr(sys.implementation.version, part_name), part_value) + self.assertFalse(isinstance(sys.implementation.version, dict)) + getattr(sys.implementation.version, part_name) + sys_implementation_value = getattr(impl_ver, part_name) + self.assertEqual(sys_implementation_value, part_value) else: self.assertEqual(getattr(sys.implementation, key), value) @@ -91,7 +117,7 @@ def test_implementation(self): @unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') -@unittest.skipIf(is_wasi or is_emscripten, 'Feature not available on WebAssembly builds') +@unittest.skipIf(is_wasm32, 'Feature not available on WebAssembly builds') class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): """Test CPython's install details file implementation.""" @@ -99,7 +125,8 @@ class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): def location(self): if sysconfig.is_python_build(): projectdir = sysconfig.get_config_var('projectbase') - with open(os.path.join(projectdir, 'pybuilddir.txt')) as f: + pybuilddir = os.path.join(projectdir, 'pybuilddir.txt') + with open(pybuilddir, encoding='utf-8') as f: dirname = os.path.join(projectdir, f.read()) else: dirname = sysconfig.get_path('stdlib') @@ -107,7 +134,7 @@ def location(self): @property def contents(self): - with open(self.location, 'r') as f: + with open(self.location, 'r', encoding='utf-8') as f: return f.read() @needs_installed_python @@ -117,12 +144,94 @@ def test_location(self): # Override generic format tests with tests for our specific implemenation. @needs_installed_python - @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS run tests via a custom testbed method that changes sys.executable') + @unittest.skipIf( + is_android or is_apple_mobile, + 'Android and iOS run tests via a custom testbed method that changes sys.executable' + ) def test_base_interpreter(self): value = self.key('base_interpreter') + # Skip check if installation is relocated + if sysconfig._installation_is_relocated(): + self.skipTest("Installation is relocated") + self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable)) + @needs_installed_python + @unittest.skipIf( + is_android or is_apple_mobile, + "Android and iOS run tests via a custom testbed method that doesn't ship headers" + ) + def test_c_api(self): + value = self.key('c_api') + + # Skip check if installation is relocated + if sysconfig._installation_is_relocated(): + self.skipTest("Installation is relocated") + + self.assertTrue(os.path.exists(os.path.join(value['headers'], 'Python.h'))) + version = sysconfig.get_config_var('VERSION') + self.assertTrue(os.path.exists(os.path.join(value['pkgconfig_path'], f'python-{version}.pc'))) + + +@unittest.skipIf( + generate_build_details is None, + "Failed to import generate-build-details" +) +@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') +@unittest.skipIf(is_wasm32, 'Feature not available on WebAssembly builds') +class BuildDetailsRelativePathsTests(unittest.TestCase): + @property + def build_details_absolute_paths(self): + data = generate_build_details.generate_data(schema_version='1.0') + return json.loads(json.dumps(data)) + + @property + def build_details_relative_paths(self): + data = self.build_details_absolute_paths + generate_build_details.make_paths_relative(data, config_path=None) + return data + + def test_round_trip(self): + data_abs_path = self.build_details_absolute_paths + data_rel_path = self.build_details_relative_paths + + self.assertEqual(data_abs_path['base_prefix'], + data_rel_path['base_prefix']) + + base_prefix = data_abs_path['base_prefix'] + + top_level_keys = ('base_interpreter',) + for key in top_level_keys: + self.assertEqual(key in data_abs_path, key in data_rel_path) + if key not in data_abs_path: + continue + + abs_rel_path = os.path.join(base_prefix, data_rel_path[key]) + abs_rel_path = os.path.normpath(abs_rel_path) + self.assertEqual(data_abs_path[key], abs_rel_path) + + second_level_keys = ( + ('libpython', 'dynamic'), + ('libpython', 'dynamic_stableabi'), + ('libpython', 'static'), + ('c_api', 'headers'), + ('c_api', 'pkgconfig_path'), + + ) + for part, key in second_level_keys: + self.assertEqual(part in data_abs_path, part in data_rel_path) + if part not in data_abs_path: + continue + self.assertEqual(key in data_abs_path[part], + key in data_rel_path[part]) + if key not in data_abs_path[part]: + continue + + abs_rel_path = os.path.join(base_prefix, data_rel_path[part][key]) + abs_rel_path = os.path.normpath(abs_rel_path) + self.assertEqual(data_abs_path[part][key], abs_rel_path) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 31597a320d4..ce60a5d095d 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -31,6 +31,7 @@ from test import support from test.support import cpython_only, swap_attr from test.support import async_yield, run_yielding_async_fn +from test.support import warnings_helper from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok @@ -393,7 +394,7 @@ def test_chr(self): self.assertRaises(ValueError, chr, -2**1000) def test_cmp(self): - self.assertTrue(not hasattr(builtins, "cmp")) + self.assertNotHasAttr(builtins, "cmp") def test_compile(self): compile('print(1)\n', '', 'exec') @@ -436,7 +437,7 @@ def f(): """doc""" # test both direct compilation and compilation via AST codeobjs = [] codeobjs.append(compile(codestr, "", "exec", optimize=optval)) - tree = ast.parse(codestr) + tree = ast.parse(codestr, optimize=optval) codeobjs.append(compile(tree, "", "exec", optimize=optval)) for code in codeobjs: ns = {} @@ -624,7 +625,7 @@ def test_compile_ast(self): for opt in [opt1, opt2]: opt_right = opt.value.right self.assertIsInstance(opt_right, ast.Constant) - self.assertEqual(opt_right.value, True) + self.assertEqual(opt_right.value, __debug__) def test_delattr(self): sys.spam = 1 @@ -1087,6 +1088,29 @@ def four_freevars(): three_freevars.__globals__, closure=my_closure) + def test_exec_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + exec(source, {}) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package.module\z') + warnings.filterwarnings('error', module=r'') + exec(source, {'__name__': 'package.module', '__file__': filename}) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + def test_filter(self): self.assertEqual(list(filter(lambda c: 'a' <= c <= 'z', 'Hello World')), list('elloorld')) @@ -1120,6 +1144,7 @@ def test_filter_pickle(self): self.check_iter_pickle(f1, list(f2), proto) @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @support.requires_resource('cpu') def test_filter_dealloc(self): # Tests recursive deallocation of nested filter objects using the @@ -1182,6 +1207,16 @@ def __hash__(self): return self self.assertEqual(hash(Z(42)), hash(42)) + def test_invalid_hash_typeerror(self): + # GH-140406: The returned object from __hash__() would leak if it + # wasn't an integer. + class A: + def __hash__(self): + return 1.0 + + with self.assertRaises(TypeError): + hash(A()) + def test_hex(self): self.assertEqual(hex(16), '0x10') self.assertEqual(hex(-16), '-0x10') @@ -1373,6 +1408,22 @@ def test_map_strict(self): self.assertRaises(ValueError, tuple, map(pack, (1, 2), (1, 2), 'abc', strict=True)) + # gh-140517: Testing refleaks with mortal objects. + t1 = (None, object()) + t2 = (object(), object()) + t3 = (object(),) + + self.assertRaises(ValueError, tuple, + map(pack, t1, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, t3, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t1, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t2, t3, strict=True)) + def test_map_strict_iterators(self): x = iter(range(5)) y = [0] @@ -2303,7 +2354,7 @@ def __format__(self, format_spec): # tests for object.__format__ really belong elsewhere, but # there's no good place to put them x = object().__format__('') - self.assertTrue(x.startswith(' +:root { color-scheme: light dark; } +table.year { border: solid; } +table.year > tbody > tr > td { border: solid; vertical-align: top; } +""" + result_2004_html = """\ - - - + + - - + + Calendar for 2004 +{css_styles} + - -
2004
+
+
2004
@@ -132,7 +140,7 @@
January
MonTueWedThuFriSatSun
   1234
19202122232425
262728293031 
-		 +
@@ -141,7 +149,7 @@
February
MonTueWedThuFriSatSun
      1
16171819202122
23242526272829
-		 +
@@ -150,7 +158,7 @@
March
MonTueWedThuFriSatSun
1234567
22232425262728
293031    
-
+
@@ -159,7 +167,7 @@
April
MonTueWedThuFriSatSun
   1234
19202122232425
2627282930  
-		 +
@@ -169,7 +177,7 @@
May
MonTueWedThuFriSatSun
     12
24252627282930
31      
-		 +
@@ -178,7 +186,7 @@
June
MonTueWedThuFriSatSun
 123456
21222324252627
282930    
-
+
@@ -187,7 +195,7 @@
July
MonTueWedThuFriSatSun
   1234
19202122232425
262728293031 
-		 + - - + + - - - - - - + + + + + + - - + + - - - - - - - - - + + + + + + + + +
@@ -197,7 +205,7 @@
August
MonTueWedThuFriSatSun
      1
23242526272829
3031     
-		 + - - + + - - - - - - + + + + + + - - + + - - - - - - - - - + + + + + + + + +
@@ -206,7 +214,7 @@
September
MonTueWedThuFriSatSun
  12345
20212223242526
27282930   
-
+ - - + + - - - - - - + + + + + + - - + + - - - - - - - - - + + + + + + + + +
@@ -215,7 +223,7 @@
October
MonTueWedThuFriSatSun
    123
18192021222324
25262728293031
-		 + - - - - - + + + + +
@@ -224,7 +232,7 @@
November
MonTueWedThuFriSatSun
1234567
22232425262728
2930     
-		 + - - - - - + + + + +
@@ -237,6 +245,34 @@ """ +result_2009_6_html = """\ + + + + + +Calendar for 2009 + + + + +
December
MonTueWedThuFriSatSun
  12345
+ + + + + + + +
June 2009
MonTueWedThuFriSatSun
1234567
891011121314
15161718192021
22232425262728
2930     
+ + +""" + result_2004_days = [ [[[0, 0, 0, 1, 2, 3, 4], [5, 6, 7, 8, 9, 10, 11], @@ -384,10 +420,12 @@ def check_htmlcalendar_encoding(self, req, res): cal = calendar.HTMLCalendar() format_ = default_format.copy() format_["encoding"] = req or 'utf-8' + format_with_css = {**format_, "css_styles": result_2004_css} + formatted_html = result_2004_html.format(**format_with_css) output = cal.formatyearpage(2004, encoding=req) self.assertEqual( output, - result_2004_html.format(**format_).encode(res) + formatted_html.encode(res) ) def test_output(self): @@ -417,7 +455,7 @@ def test_output_htmlcalendar_encoding_utf8(self): self.check_htmlcalendar_encoding('utf-8', 'utf-8') def test_output_htmlcalendar_encoding_default(self): - self.check_htmlcalendar_encoding(None, sys.getdefaultencoding()) + self.check_htmlcalendar_encoding(None, 'utf-8') def test_yeardatescalendar(self): def shrink(cal): @@ -496,6 +534,13 @@ def test_format(self): calendar.format(["1", "2", "3"], colwidth=3, spacing=1) self.assertEqual(out.getvalue().strip(), "1 2 3") + def test_format_html_year_with_month(self): + self.assertEqual( + calendar.HTMLCalendar().formatmonthpage(2009, 6).decode("ascii"), + result_2009_6_html + ) + + class CalendarTestCase(unittest.TestCase): def test_deprecation_warning(self): @@ -546,7 +591,8 @@ def test_days(self): self.assertEqual(value[::-1], list(reversed(value))) def test_months(self): - for attr in "month_name", "month_abbr": + for attr in ("month_name", "month_abbr", "standalone_month_name", + "standalone_month_abbr"): value = getattr(calendar, attr) self.assertEqual(len(value), 13) self.assertEqual(len(value[:]), 13) @@ -556,6 +602,38 @@ def test_months(self): # verify it "acts like a sequence" in two forms of iteration self.assertEqual(value[::-1], list(reversed(value))) + @support.run_with_locale('LC_ALL', 'pl_PL') + @unittest.skipUnless(sys.platform == 'darwin' or platform.libc_ver()[0] == 'glibc', + "Guaranteed to work with glibc and macOS") + def test_standalone_month_name_and_abbr_pl_locale(self): + expected_standalone_month_names = [ + "", "styczeń", "luty", "marzec", "kwiecień", "maj", "czerwiec", + "lipiec", "sierpień", "wrzesień", "październik", "listopad", + "grudzień" + ] + expected_standalone_month_abbr = [ + "", "sty", "lut", "mar", "kwi", "maj", "cze", + "lip", "sie", "wrz", "paź", "lis", "gru" + ] + self.assertEqual( + list(calendar.standalone_month_name), + expected_standalone_month_names + ) + self.assertEqual( + list(calendar.standalone_month_abbr), + expected_standalone_month_abbr + ) + + def test_standalone_month_name_and_abbr_C_locale(self): + # Ensure that the standalone month names and abbreviations are + # equal to the regular month names and abbreviations for + # the "C" locale. + with calendar.different_locale("C"): + self.assertListEqual(list(calendar.month_name), + list(calendar.standalone_month_name)) + self.assertListEqual(list(calendar.month_abbr), + list(calendar.standalone_month_abbr)) + def test_locale_text_calendar(self): try: cal = calendar.LocaleTextCalendar(locale='') @@ -662,6 +740,57 @@ def test_locale_calendar_formatweekday(self): except locale.Error: raise unittest.SkipTest('cannot set the en_US locale') + # These locales have weekday names all shorter than English's longest + # 'Wednesday'. They should not be abbreviated unnecessarily + @support.run_with_locales("LC_ALL", + 'Chinese', 'zh_CN.UTF-8', + 'French', 'fr_FR.UTF-8', + 'Norwegian', 'nb_NO.UTF-8', + 'Malay', 'ms_MY.UTF8' + ) + def test_locale_calendar_short_weekday_names(self): + names = (datetime.date(2001, 1, i+1).strftime('%A') for i in range(7)) + max_length = max(map(len, names)) + if max_length >= 9: + self.skipTest('weekday names are too long') + + def get_weekday_names(width): + return calendar.TextCalendar().formatweekheader(width).split() + + # Weekday names should not be abbreviated if the width is sufficient + self.assertEqual( + get_weekday_names(max_length), + get_weekday_names(max_length + 10) + ) + + # Any width shorter than necessary should produce abbreviations + self.assertNotEqual( + get_weekday_names(max_length), + get_weekday_names(max_length - 1) + ) + + # These locales have a weekday name longer than 'Wednesday' + # They should be properly abbreviated rather than truncated + @support.run_with_locales("LC_ALL", + 'Portuguese', 'pt_PT.UTF-8', + 'German', 'de_DE.UTF-8', + 'Russian', 'ru_RU.UTF-8', + ) + def test_locale_calendar_long_weekday_names(self): + names = (datetime.date(2001, 1, i+1).strftime('%A') for i in range(7)) + max_length = max(map(len, names)) + abbrev_names = (datetime.date(2001, 1, i+1).strftime('%a') for i in range(7)) + abbrev_max_length = max(map(len, abbrev_names)) + + if max_length <= 9: + self.skipTest('weekday names are too short') + if abbrev_max_length >= 9: + self.skipTest('abbreviated weekday names are too long') + + def get_weekday_names(width): + return calendar.TextCalendar().formatweekheader(width).split() + self.assertEqual(get_weekday_names(abbrev_max_length), get_weekday_names(max_length-1)) + def test_locale_calendar_formatmonthname(self): try: # formatmonthname uses the same month names regardless of the width argument. @@ -1008,7 +1137,6 @@ def test_illegal_arguments(self): self.assertFailure('2004', '1', 'spam') self.assertFailure('2004', '1', '1') self.assertFailure('2004', '1', '1', 'spam') - self.assertFailure('-t', 'html', '2004', '1') def test_output_current_year(self): for run in self.runners: @@ -1098,7 +1226,7 @@ def test_option_type(self): output = run('--type', 'text', '2004') self.assertEqual(output, conv(result_2004_text)) output = run('--type', 'html', '2004') - self.assertEqual(output[:6], b'') self.assertIn(b'Calendar for 2004', output) def test_html_output_current_year(self): @@ -1111,15 +1239,16 @@ def test_html_output_current_year(self): def test_html_output_year_encoding(self): for run in self.runners: output = run('-t', 'html', '--encoding', 'ascii', '2004') - self.assertEqual(output, result_2004_html.format(**default_format).encode('ascii')) + format_with_css = default_format.copy() + format_with_css["css_styles"] = result_2004_css + self.assertEqual(output, result_2004_html.format(**format_with_css).encode('ascii')) def test_html_output_year_css(self): self.assertFailure('-t', 'html', '-c') self.assertFailure('-t', 'html', '--css') for run in self.runners: output = run('-t', 'html', '--css', 'custom.css', '2004') - self.assertIn(b'', output) + self.assertIn(b'', output) class MiscTestCase(unittest.TestCase): @@ -1173,7 +1302,7 @@ def test_formatweek_head(self): def test_format_year(self): self.assertIn( - ('' % + ('
' % self.cal.cssclass_year), self.cal.formatyear(2017)) def test_format_year_head(self): diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 185ae84dc4d..f42526aee19 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -12,6 +12,10 @@ import _testlimitedcapi except ImportError: _testlimitedcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None import struct import collections import itertools @@ -695,8 +699,8 @@ class DerivedType(SuperType): UnaffectedType2 = _testcapi.make_vectorcall_class(SuperType) # Aside: Quickly check that the C helper actually made derived types - self.assertTrue(issubclass(UnaffectedType1, DerivedType)) - self.assertTrue(issubclass(UnaffectedType2, SuperType)) + self.assertIsSubclass(UnaffectedType1, DerivedType) + self.assertIsSubclass(UnaffectedType2, SuperType) # Initial state: tp_call self.assertEqual(instance(), "tp_call") @@ -1037,6 +1041,23 @@ def test_unexpected_keyword_suggestion_via_getargs(self): @cpython_only class TestRecursion(unittest.TestCase): + def test_margin_is_sufficient(self): + + def get_sp(): + return _testinternalcapi.get_stack_pointer() + + this_sp = _testinternalcapi.get_stack_pointer() + lower_sp = _testcapi.pyobject_vectorcall(get_sp, (), ()) + if _testcapi._Py_STACK_GROWS_DOWN: + self.assertLess(lower_sp, this_sp) + safe_margin = this_sp - lower_sp + else: + self.assertGreater(lower_sp, this_sp) + safe_margin = lower_sp - this_sp + # Add an (arbitrary) extra 25% for safety + safe_margin = safe_margin * 5 / 4 + self.assertLess(safe_margin, _testinternalcapi.get_stack_margin()) + @skip_on_s390x @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") @skip_if_sanitizer("requires deep stack", thread=True) @@ -1074,6 +1095,14 @@ def c_py_recurse(m): with self.assertRaises(RecursionError): c_py_recurse(100_000) + def test_recursion_with_kwargs(self): + # GH-137883: The interpreter forgot to check the recursion limit when + # calling with keywords. + def recurse_kw(a=0): + recurse_kw(a=0) + with self.assertRaises(RecursionError): + recurse_kw() + class TestFunctionWithManyArgs(unittest.TestCase): def test_function_with_many_args(self): diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 7d548ae87c0..3a2ed9f5db8 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -1077,6 +1077,31 @@ def test_iter_nextitem(self): with self.assertRaisesRegex(TypeError, regex): PyIter_NextItem(10) + def test_object_setattr_null_exc(self): + class Obj: + pass + obj = Obj() + obj.attr = 123 + + exc = ValueError("error") + with self.assertRaises(SystemError) as cm: + _testcapi.object_setattr_null_exc(obj, 'attr', exc) + self.assertIs(cm.exception.__context__, exc) + self.assertIsNone(cm.exception.__cause__) + self.assertHasAttr(obj, 'attr') + + with self.assertRaises(SystemError) as cm: + _testcapi.object_setattrstring_null_exc(obj, 'attr', exc) + self.assertIs(cm.exception.__context__, exc) + self.assertIsNone(cm.exception.__cause__) + self.assertHasAttr(obj, 'attr') + + with self.assertRaises(SystemError) as cm: + # undecodable name + _testcapi.object_setattrstring_null_exc(obj, b'\xff', exc) + self.assertIs(cm.exception.__context__, exc) + self.assertIsNone(cm.exception.__cause__) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index dfa98de9f00..cb7ad8b2225 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,3 +1,4 @@ +import sys import unittest from test.support import import_helper @@ -55,7 +56,9 @@ def test_fromstringandsize(self): self.assertEqual(fromstringandsize(b'', 0), bytearray()) self.assertEqual(fromstringandsize(NULL, 0), bytearray()) self.assertEqual(len(fromstringandsize(NULL, 3)), 3) - self.assertRaises(MemoryError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, fromstringandsize, NULL, + PY_SSIZE_T_MAX-sys.getsizeof(b'') + 1) self.assertRaises(SystemError, fromstringandsize, b'abc', -1) self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) @@ -66,6 +69,7 @@ def test_fromobject(self): # Test PyByteArray_FromObject() fromobject = _testlimitedcapi.bytearray_fromobject + self.assertEqual(fromobject(b''), bytearray(b'')) self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) self.assertEqual(fromobject(ByteArraySubclass(b'abc')), bytearray(b'abc')) @@ -115,6 +119,7 @@ def test_concat(self): self.assertEqual(concat(b'abc', bytearray(b'def')), bytearray(b'abcdef')) self.assertEqual(concat(bytearray(b'abc'), b''), bytearray(b'abc')) self.assertEqual(concat(b'', bytearray(b'def')), bytearray(b'def')) + self.assertEqual(concat(bytearray(b''), bytearray(b'')), bytearray(b'')) self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), bytearray(b'abcdef')) self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), @@ -150,6 +155,10 @@ def test_resize(self): self.assertEqual(resize(ba, 0), 0) self.assertEqual(ba, bytearray()) + ba = bytearray(b'') + self.assertEqual(resize(ba, 0), 0) + self.assertEqual(ba, bytearray()) + ba = ByteArraySubclass(b'abcdef') self.assertEqual(resize(ba, 3), 0) self.assertEqual(ba, bytearray(b'abc')) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 5b61c733815..410ebab729c 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -22,6 +22,7 @@ def test_check(self): # Test PyBytes_Check() check = _testlimitedcapi.bytes_check self.assertTrue(check(b'abc')) + self.assertTrue(check(b'')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) self.assertTrue(check(BytesSubclass(b'abc'))) @@ -36,6 +37,7 @@ def test_checkexact(self): # Test PyBytes_CheckExact() check = _testlimitedcapi.bytes_checkexact self.assertTrue(check(b'abc')) + self.assertTrue(check(b'')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) self.assertFalse(check(BytesSubclass(b'abc'))) @@ -79,6 +81,7 @@ def test_fromobject(self): # Test PyBytes_FromObject() fromobject = _testlimitedcapi.bytes_fromobject + self.assertEqual(fromobject(b''), b'') self.assertEqual(fromobject(b'abc'), b'abc') self.assertEqual(fromobject(bytearray(b'abc')), b'abc') self.assertEqual(fromobject(BytesSubclass(b'abc')), b'abc') @@ -108,6 +111,7 @@ def test_asstring(self): self.assertEqual(asstring(b'abc', 4), b'abc\0') self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') + self.assertEqual(asstring(b'', 1), b'\0') self.assertRaises(TypeError, asstring, 'abc', 0) self.assertRaises(TypeError, asstring, object(), 0) @@ -120,6 +124,7 @@ def test_asstringandsize(self): self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) + self.assertEqual(asstringandsize(b'', 1), (b'\0', 0)) self.assertEqual(asstringandsize_null(b'abc', 4), b'abc\0') self.assertRaises(ValueError, asstringandsize_null, b'abc\0def', 8) self.assertRaises(TypeError, asstringandsize, 'abc', 0) @@ -134,6 +139,7 @@ def test_repr(self): # Test PyBytes_Repr() bytes_repr = _testlimitedcapi.bytes_repr + self.assertEqual(bytes_repr(b'', 0), r"""b''""") self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") self.assertEqual(bytes_repr(b'''a'b"c"d''', 0), r"""b'a\'b"c"d'""") @@ -163,6 +169,7 @@ def test_concat(self, concat=None): self.assertEqual(concat(b'', bytearray(b'def')), b'def') self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), b'abcdef') self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), b'abcdef') + self.assertEqual(concat(b'', b''), b'') self.assertEqual(concat(b'abc', b'def', True), b'abcdef') self.assertEqual(concat(b'abc', bytearray(b'def'), True), b'abcdef') @@ -192,6 +199,7 @@ def test_decodeescape(self): """Test PyBytes_DecodeEscape()""" decodeescape = _testlimitedcapi.bytes_decodeescape + self.assertEqual(decodeescape(b''), b'') self.assertEqual(decodeescape(b'abc'), b'abc') self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), b'''\t\n\r\v\f\0\\'"''') @@ -291,5 +299,95 @@ def test_join(self): bytes_join(b'', NULL) +class BytesWriterTest(unittest.TestCase): + result_type = bytes + + def create_writer(self, alloc=0, string=b''): + return _testcapi.PyBytesWriter(alloc, string, 0) + + def test_create(self): + # Test PyBytesWriter_Create() + writer = self.create_writer() + self.assertEqual(writer.get_size(), 0) + self.assertEqual(writer.finish(), self.result_type(b'')) + + writer = self.create_writer(3, b'abc') + self.assertEqual(writer.get_size(), 3) + self.assertEqual(writer.finish(), self.result_type(b'abc')) + + writer = self.create_writer(10, b'abc') + self.assertEqual(writer.get_size(), 10) + self.assertEqual(writer.finish_with_size(3), self.result_type(b'abc')) + + def test_write_bytes(self): + # Test PyBytesWriter_WriteBytes() + writer = self.create_writer() + writer.write_bytes(b'Hello World!', -1) + self.assertEqual(writer.finish(), self.result_type(b'Hello World!')) + + writer = self.create_writer() + writer.write_bytes(b'Hello ', -1) + writer.write_bytes(b'World! ', 6) + self.assertEqual(writer.finish(), self.result_type(b'Hello World!')) + + def test_resize(self): + # Test PyBytesWriter_Resize() + writer = self.create_writer() + writer.resize(len(b'number=123456'), b'number=123456') + writer.resize(len(b'number=123456'), b'') + self.assertEqual(writer.get_size(), len(b'number=123456')) + self.assertEqual(writer.finish(), self.result_type(b'number=123456')) + + writer = self.create_writer() + writer.resize(0, b'') + writer.resize(len(b'number=123456'), b'number=123456') + self.assertEqual(writer.finish(), self.result_type(b'number=123456')) + + writer = self.create_writer() + writer.resize(len(b'number='), b'number=') + writer.resize(len(b'number=123456'), b'123456') + self.assertEqual(writer.finish(), self.result_type(b'number=123456')) + + writer = self.create_writer() + writer.resize(len(b'number='), b'number=') + writer.resize(len(b'number='), b'') + writer.resize(len(b'number=123456'), b'123456') + self.assertEqual(writer.finish(), self.result_type(b'number=123456')) + + writer = self.create_writer() + writer.resize(len(b'number'), b'number') + writer.resize(len(b'number='), b'=') + writer.resize(len(b'number=123'), b'123') + writer.resize(len(b'number=123456'), b'456') + self.assertEqual(writer.finish(), self.result_type(b'number=123456')) + + def test_format_i(self): + # Test PyBytesWriter_Format() + writer = self.create_writer() + writer.format_i(b'x=%i', 123456) + self.assertEqual(writer.finish(), self.result_type(b'x=123456')) + + writer = self.create_writer() + writer.format_i(b'x=%i, ', 123) + writer.format_i(b'y=%i', 456) + self.assertEqual(writer.finish(), self.result_type(b'x=123, y=456')) + + def test_example_abc(self): + self.assertEqual(_testcapi.byteswriter_abc(), b'abc') + + def test_example_resize(self): + self.assertEqual(_testcapi.byteswriter_resize(), b'Hello World') + + def test_example_highlevel(self): + self.assertEqual(_testcapi.byteswriter_highlevel(), b'Hello World!') + + +class ByteArrayWriterTest(BytesWriterTest): + result_type = bytearray + + def create_writer(self, alloc=0, string=b''): + return _testcapi.PyBytesWriter(alloc, string, 1) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index a0355c7a388..1a3f476ed0f 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -630,7 +630,6 @@ def test_codec_known_encoding(self): for name in [ encoding_name, encoding_name.upper(), - encoding_name.replace('_', '-'), ]: with self.subTest(name): self.assertTrue(_testcapi.codec_known_encoding(name)) diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index a2d70dd3af4..04a27de8d84 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -3,7 +3,6 @@ """ import os import sys -import sysconfig import types import unittest from test import support diff --git a/Lib/test/test_capi/test_emscripten.py b/Lib/test/test_capi/test_emscripten.py new file mode 100644 index 00000000000..272d9a10ceb --- /dev/null +++ b/Lib/test/test_capi/test_emscripten.py @@ -0,0 +1,25 @@ +import unittest +from test.support import is_emscripten + +if not is_emscripten: + raise unittest.SkipTest("Emscripten-only test") + +from _testinternalcapi import emscripten_set_up_async_input_device +from pathlib import Path + + +class EmscriptenAsyncInputDeviceTest(unittest.TestCase): + def test_emscripten_async_input_device(self): + jspi_supported = emscripten_set_up_async_input_device() + p = Path("/dev/blah") + self.addCleanup(p.unlink) + if not jspi_supported: + with open(p, "r") as f: + self.assertRaises(OSError, f.readline) + return + + with open(p, "r") as f: + for _ in range(10): + self.assertEqual(f.readline().strip(), "ab") + self.assertEqual(f.readline().strip(), "fi") + self.assertEqual(f.readline().strip(), "xy") diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index ade55338e63..4967f02b007 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -6,7 +6,7 @@ import textwrap from test import support -from test.support import import_helper +from test.support import import_helper, force_not_colorized from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE from test.support.script_helper import assert_python_failure, assert_python_ok from test.support.testcase import ExceptionIsLikeMixin @@ -337,6 +337,10 @@ def test_err_writeunraisable(self): self.assertIsNone(cm.unraisable.err_msg) self.assertIsNone(cm.unraisable.object) + @force_not_colorized + def test_err_writeunraisable_lines(self): + writeunraisable = _testcapi.err_writeunraisable + with (support.swap_attr(sys, 'unraisablehook', None), support.captured_stderr() as stderr): writeunraisable(CustomError('oops!'), hex) @@ -387,6 +391,10 @@ def test_err_formatunraisable(self): self.assertIsNone(cm.unraisable.err_msg) self.assertIsNone(cm.unraisable.object) + @force_not_colorized + def test_err_formatunraisable_lines(self): + formatunraisable = _testcapi.err_formatunraisable + with (support.swap_attr(sys, 'unraisablehook', None), support.captured_stderr() as stderr): formatunraisable(CustomError('oops!'), b'Error in %R', []) diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index f7efe0d0254..df7017e6436 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -1,5 +1,6 @@ import math import random +import platform import sys import unittest import warnings @@ -28,6 +29,23 @@ NAN = float("nan") +def make_nan(size, sign, quiet, payload=None): + if size == 8: + payload_mask = 0x7ffffffffffff + i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + elif size == 4: + payload_mask = 0x3fffff + i = (sign << 31) + (0xff << 23) + (quiet << 22) + elif size == 2: + payload_mask = 0x1ff + i = (sign << 15) + (0x1f << 10) + (quiet << 9) + else: + raise ValueError("size must be either 2, 4, or 8") + if payload is None: + payload = random.randint(not quiet, payload_mask) + return i + payload + + class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() @@ -197,16 +215,11 @@ def test_pack_unpack_roundtrip_for_nans(self): # PyFloat_Pack/Unpack*() API. See also gh-130317 and # e.g. https://developercommunity.visualstudio.com/t/155064 signaling = 0 - quiet = int(not signaling) - if size == 8: - payload = random.randint(signaling, 0x7ffffffffffff) - i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + payload - elif size == 4: - payload = random.randint(signaling, 0x3fffff) - i = (sign << 31) + (0xff << 23) + (quiet << 22) + payload - elif size == 2: - payload = random.randint(signaling, 0x1ff) - i = (sign << 15) + (0x1f << 10) + (quiet << 9) + payload + if platform.machine().startswith('parisc'): + # HP PA RISC uses 0 for quiet, see: + # https://en.wikipedia.org/wiki/NaN#Encoding + signaling = 1 + i = make_nan(size, sign, not signaling) data = bytes.fromhex(f'{i:x}') for endian in (BIG_ENDIAN, LITTLE_ENDIAN): with self.subTest(data=data, size=size, endian=endian): @@ -216,6 +229,32 @@ def test_pack_unpack_roundtrip_for_nans(self): self.assertTrue(math.isnan(value)) self.assertEqual(data1, data2) + @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") + @unittest.skipUnless(sys.maxsize != 2147483647, "requires 64-bit mode") + def test_pack_unpack_nans_for_different_formats(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(endian=endian): + byteorder = "big" if endian == BIG_ENDIAN else "little" + + # Convert sNaN to qNaN, if payload got truncated + data = make_nan(8, 0, False, 0x80001).to_bytes(8, byteorder) + snan_low = unpack(data, endian) + qnan4 = make_nan(4, 0, True, 0).to_bytes(4, byteorder) + qnan2 = make_nan(2, 0, True, 0).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_low, endian), qnan4) + self.assertEqual(pack(2, snan_low, endian), qnan2) + + # Preserve NaN type, if payload not truncated + data = make_nan(8, 0, False, 0x80000000001).to_bytes(8, byteorder) + snan_high = unpack(data, endian) + snan4 = make_nan(4, 0, False, 16384).to_bytes(4, byteorder) + snan2 = make_nan(2, 0, False, 2).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_high, endian), snan4) + self.assertEqual(pack(2, snan_high, endian), snan2) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 67a8da75995..0b2473bac2b 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -48,8 +48,8 @@ LARGE = 0x7FFFFFFF VERY_LARGE = 0xFF0000121212121212121242 -from _testcapi import UCHAR_MAX, USHRT_MAX, UINT_MAX, ULONG_MAX, INT_MAX, \ - INT_MIN, LONG_MIN, LONG_MAX, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, \ +from _testcapi import UCHAR_MAX, USHRT_MAX, UINT_MAX, ULONG_MAX, ULLONG_MAX, INT_MAX, \ + INT_MIN, LONG_MIN, LONG_MAX, LLONG_MIN, LLONG_MAX, PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, \ SHRT_MIN, SHRT_MAX, FLT_MIN, FLT_MAX, DBL_MIN, DBL_MAX DBL_MAX_EXP = sys.float_info.max_exp @@ -57,9 +57,8 @@ NAN = float('nan') # fake, they are not defined in Python's header files -LLONG_MAX = 2**63-1 -LLONG_MIN = -2**63 -ULLONG_MAX = 2**64-1 +SCHAR_MAX = UCHAR_MAX // 2 +SCHAR_MIN = SCHAR_MAX - UCHAR_MAX NULL = None @@ -209,10 +208,23 @@ def test_B(self): self.assertEqual(UCHAR_MAX, getargs_B(-1)) self.assertEqual(0, getargs_B(0)) self.assertEqual(UCHAR_MAX, getargs_B(UCHAR_MAX)) - self.assertEqual(0, getargs_B(UCHAR_MAX+1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(0, getargs_B(UCHAR_MAX+1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(1, getargs_B(-UCHAR_MAX)) + self.assertEqual(SCHAR_MAX+1, getargs_B(SCHAR_MIN)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(SCHAR_MAX, getargs_B(SCHAR_MIN-1)) + + self.assertEqual(128, getargs_B(-2**7)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(127, getargs_B(-2**7-1)) self.assertEqual(42, getargs_B(42)) - self.assertEqual(UCHAR_MAX & VERY_LARGE, getargs_B(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(UCHAR_MAX & VERY_LARGE, getargs_B(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(UCHAR_MAX & -VERY_LARGE, getargs_B(-VERY_LARGE)) def test_H(self): from _testcapi import getargs_H @@ -233,11 +245,18 @@ def test_H(self): self.assertEqual(USHRT_MAX, getargs_H(-1)) self.assertEqual(0, getargs_H(0)) self.assertEqual(USHRT_MAX, getargs_H(USHRT_MAX)) - self.assertEqual(0, getargs_H(USHRT_MAX+1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(0, getargs_H(USHRT_MAX+1)) + self.assertEqual(SHRT_MAX+1, getargs_H(SHRT_MIN)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(SHRT_MAX, getargs_H(SHRT_MIN-1)) self.assertEqual(42, getargs_H(42)) - self.assertEqual(VERY_LARGE & USHRT_MAX, getargs_H(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(USHRT_MAX & VERY_LARGE, getargs_H(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(USHRT_MAX & -VERY_LARGE, getargs_H(-VERY_LARGE)) def test_I(self): from _testcapi import getargs_I @@ -258,11 +277,18 @@ def test_I(self): self.assertEqual(UINT_MAX, getargs_I(-1)) self.assertEqual(0, getargs_I(0)) self.assertEqual(UINT_MAX, getargs_I(UINT_MAX)) - self.assertEqual(0, getargs_I(UINT_MAX+1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(0, getargs_I(UINT_MAX+1)) + self.assertEqual(INT_MAX+1, getargs_I(INT_MIN)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(INT_MAX, getargs_I(INT_MIN-1)) self.assertEqual(42, getargs_I(42)) - self.assertEqual(VERY_LARGE & UINT_MAX, getargs_I(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(UINT_MAX & VERY_LARGE, getargs_I(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(UINT_MAX & -VERY_LARGE, getargs_I(-VERY_LARGE)) def test_k(self): from _testcapi import getargs_k @@ -283,11 +309,18 @@ def test_k(self): self.assertEqual(ULONG_MAX, getargs_k(-1)) self.assertEqual(0, getargs_k(0)) self.assertEqual(ULONG_MAX, getargs_k(ULONG_MAX)) - self.assertEqual(0, getargs_k(ULONG_MAX+1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(0, getargs_k(ULONG_MAX+1)) + self.assertEqual(LONG_MAX+1, getargs_k(LONG_MIN)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(LONG_MAX, getargs_k(LONG_MIN-1)) self.assertEqual(42, getargs_k(42)) - self.assertEqual(VERY_LARGE & ULONG_MAX, getargs_k(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ULONG_MAX & VERY_LARGE, getargs_k(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ULONG_MAX & -VERY_LARGE, getargs_k(-VERY_LARGE)) class Signed_TestCase(unittest.TestCase): def test_h(self): @@ -434,11 +467,18 @@ def test_K(self): self.assertEqual(ULLONG_MAX, getargs_K(ULLONG_MAX)) self.assertEqual(0, getargs_K(0)) self.assertEqual(ULLONG_MAX, getargs_K(ULLONG_MAX)) - self.assertEqual(0, getargs_K(ULLONG_MAX+1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(0, getargs_K(ULLONG_MAX+1)) + self.assertEqual(LLONG_MAX+1, getargs_K(LLONG_MIN)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(LLONG_MAX, getargs_K(LLONG_MIN-1)) self.assertEqual(42, getargs_K(42)) - self.assertEqual(VERY_LARGE & ULLONG_MAX, getargs_K(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ULLONG_MAX & VERY_LARGE, getargs_K(VERY_LARGE)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ULLONG_MAX & -VERY_LARGE, getargs_K(-VERY_LARGE)) class Float_TestCase(unittest.TestCase, FloatsAreIdenticalMixin): @@ -1389,123 +1429,6 @@ def test_nested_sequence(self): "argument 1 must be sequence of length 1, not 0"): parse(([],), {}, '(' + f + ')', ['a']) - def test_specific_type_errors(self): - parse = _testcapi.parse_tuple_and_keywords - - def check(format, arg, expected, got='list'): - errmsg = f'must be {expected}, not {got}' - with self.assertRaisesRegex(TypeError, errmsg): - parse((arg,), {}, format, ['a']) - - check('k', [], 'int') - check('k?', [], 'int or None') - check('K', [], 'int') - check('K?', [], 'int or None') - check('c', [], 'a byte string of length 1') - check('c?', [], 'a byte string of length 1 or None') - check('c', b'abc', 'a byte string of length 1', - 'a bytes object of length 3') - check('c?', b'abc', 'a byte string of length 1 or None', - 'a bytes object of length 3') - check('c', bytearray(b'abc'), 'a byte string of length 1', - 'a bytearray object of length 3') - check('c?', bytearray(b'abc'), 'a byte string of length 1 or None', - 'a bytearray object of length 3') - check('C', [], 'a unicode character') - check('C?', [], 'a unicode character or None') - check('C', 'abc', 'a unicode character', - 'a string of length 3') - check('C?', 'abc', 'a unicode character or None', - 'a string of length 3') - check('s', [], 'str') - check('s?', [], 'str or None') - check('z', [], 'str or None') - check('z?', [], 'str or None') - check('es', [], 'str') - check('es?', [], 'str or None') - check('es#', [], 'str') - check('es#?', [], 'str or None') - check('et', [], 'str, bytes or bytearray') - check('et?', [], 'str, bytes, bytearray or None') - check('et#', [], 'str, bytes or bytearray') - check('et#?', [], 'str, bytes, bytearray or None') - check('w*', [], 'read-write bytes-like object') - check('w*?', [], 'read-write bytes-like object or None') - check('S', [], 'bytes') - check('S?', [], 'bytes or None') - check('U', [], 'str') - check('U?', [], 'str or None') - check('Y', [], 'bytearray') - check('Y?', [], 'bytearray or None') - check('(OO)', 42, '2-item tuple', 'int') - check('(OO)?', 42, '2-item tuple or None', 'int') - check('(OO)', (1, 2, 3), 'tuple of length 2', '3') - - def test_nullable(self): - parse = _testcapi.parse_tuple_and_keywords - - def check(format, arg, allows_none=False): - # Because some format units (such as y*) require cleanup, - # we force the parsing code to perform the cleanup by adding - # an argument that always fails. - # By checking for an exception, we ensure that the parsing - # of the first argument was successful. - self.assertRaises(OverflowError, parse, - (arg, 256), {}, format + '?b', ['a', 'b']) - self.assertRaises(OverflowError, parse, - (None, 256), {}, format + '?b', ['a', 'b']) - self.assertRaises(OverflowError, parse, - (arg, 256), {}, format + 'b', ['a', 'b']) - self.assertRaises(OverflowError if allows_none else TypeError, parse, - (None, 256), {}, format + 'b', ['a', 'b']) - - check('b', 42) - check('B', 42) - check('h', 42) - check('H', 42) - check('i', 42) - check('I', 42) - check('n', 42) - check('l', 42) - check('k', 42) - check('L', 42) - check('K', 42) - check('f', 2.5) - check('d', 2.5) - check('D', 2.5j) - check('c', b'a') - check('C', 'a') - check('p', True, allows_none=True) - check('y', b'buffer') - check('y*', b'buffer') - check('y#', b'buffer') - check('s', 'string') - check('s*', 'string') - check('s#', 'string') - check('z', 'string', allows_none=True) - check('z*', 'string', allows_none=True) - check('z#', 'string', allows_none=True) - check('w*', bytearray(b'buffer')) - check('U', 'string') - check('S', b'bytes') - check('Y', bytearray(b'bytearray')) - check('O', object, allows_none=True) - - check('(OO)', (1, 2)) - self.assertEqual(parse((((1, 2), 3),), {}, '((OO)?O)', ['a']), (1, 2, 3)) - self.assertEqual(parse(((None, 3),), {}, '((OO)?O)', ['a']), (NULL, NULL, 3)) - self.assertEqual(parse((((1, 2), 3),), {}, '((OO)O)', ['a']), (1, 2, 3)) - self.assertRaises(TypeError, parse, ((None, 3),), {}, '((OO)O)', ['a']) - - parse((None,), {}, 'es?', ['a']) - parse((None,), {}, 'es#?', ['a']) - parse((None,), {}, 'et?', ['a']) - parse((None,), {}, 'et#?', ['a']) - parse((None,), {}, 'O!?', ['a']) - parse((None,), {}, 'O&?', ['a']) - - # TODO: More tests for es?, es#?, et?, et#?, O!, O& - @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') def test_gh_119213(self): rc, out, err = script_helper.assert_python_ok("-c", """if True: diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py index 660e8a0e789..6b882d65a42 100644 --- a/Lib/test/test_capi/test_immortal.py +++ b/Lib/test/test_capi/test_immortal.py @@ -25,14 +25,14 @@ def test_immortal(self): class TestInternalCAPI(unittest.TestCase): def test_immortal_builtins(self): - for obj in range(-5, 256): + for obj in range(-5, 1025): self.assertTrue(_testinternalcapi.is_static_immortal(obj)) self.assertTrue(_testinternalcapi.is_static_immortal(None)) self.assertTrue(_testinternalcapi.is_static_immortal(False)) self.assertTrue(_testinternalcapi.is_static_immortal(True)) self.assertTrue(_testinternalcapi.is_static_immortal(...)) self.assertTrue(_testinternalcapi.is_static_immortal(())) - for obj in range(300, 400): + for obj in range(1025, 1125): self.assertFalse(_testinternalcapi.is_static_immortal(obj)) for obj in ([], {}, set()): self.assertFalse(_testinternalcapi.is_static_immortal(obj)) diff --git a/Lib/test/test_capi/test_import.py b/Lib/test/test_capi/test_import.py index 25136624ca4..57e0316fda8 100644 --- a/Lib/test/test_capi/test_import.py +++ b/Lib/test/test_capi/test_import.py @@ -134,7 +134,7 @@ def test_importmodule(self): # CRASHES importmodule(NULL) def test_importmodulenoblock(self): - # Test deprecated PyImport_ImportModuleNoBlock() + # Test deprecated (stable ABI only) PyImport_ImportModuleNoBlock() importmodulenoblock = _testlimitedcapi.PyImport_ImportModuleNoBlock with check_warnings(('', DeprecationWarning)): self.check_import_func(importmodulenoblock) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index a597f23a992..3997acbdf84 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -22,6 +22,7 @@ from test import support from test.support import MISSING_C_DOCSTRINGS from test.support import import_helper +from test.support import script_helper from test.support import threading_helper from test.support import warnings_helper from test.support import requires_limited_api @@ -306,7 +307,7 @@ def test_getitem_with_error(self): CURRENT_THREAD_REGEX + r' File .*, line 6 in \n' r'\n' - r'Extension modules: _testcapi \(total: 1\)\n') + r'Extension modules: ') else: # Python built with NDEBUG macro defined: # test _Py_CheckFunctionResult() instead. @@ -412,10 +413,14 @@ def test_trashcan_subclass(self): L = MyList((L,)) @support.requires_resource('cpu') + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_trashcan_python_class1(self): self.do_test_trashcan_python_class(list) @support.requires_resource('cpu') + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_trashcan_python_class2(self): from _testcapi import MyList self.do_test_trashcan_python_class(MyList) @@ -1637,6 +1642,36 @@ def subthread(): self.assertEqual(actual, int(interpid)) + @threading_helper.requires_working_threading() + def test_pending_call_creates_thread(self): + source = """ + import _testinternalcapi + import threading + import time + + + def output(): + print(24) + time.sleep(1) + print(42) + + + def callback(): + threading.Thread(target=output).start() + + + def create_pending_call(): + time.sleep(1) + _testinternalcapi.simple_pending_call(callback) + + + threading.Thread(target=create_pending_call).start() + """ + return_code, stdout, stderr = script_helper.assert_python_ok('-c', textwrap.dedent(source)) + self.assertEqual(return_code, 0) + self.assertEqual(stdout, f"24{os.linesep}42{os.linesep}".encode("utf-8")) + self.assertEqual(stderr, b"") + class SubinterpreterTest(unittest.TestCase): @@ -1945,6 +1980,43 @@ def test_module_state_shared_in_global(self): subinterp_attr_id = os.read(r, 100) self.assertEqual(main_attr_id, subinterp_attr_id) + @threading_helper.requires_working_threading() + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @requires_subinterpreters + def test_pending_call_creates_thread_subinterpreter(self): + interpreters = import_helper.import_module("concurrent.interpreters") + r, w = os.pipe() + source = f"""if True: + import _testinternalcapi + import threading + import time + import os + + + def output(): + time.sleep(1) + os.write({w}, b"x") + os.close({w}) + + + def callback(): + threading.Thread(target=output).start() + + + def create_pending_call(): + time.sleep(1) + _testinternalcapi.simple_pending_call(callback) + + + threading.Thread(target=create_pending_call).start() + """ + interp = interpreters.create() + interp.exec(source) + interp.close() + data = os.read(r, 1) + self.assertEqual(data, b"x") + os.close(r) + @requires_subinterpreters class InterpreterConfigTests(unittest.TestCase): diff --git a/Lib/test/test_capi/test_modsupport.py b/Lib/test/test_capi/test_modsupport.py new file mode 100644 index 00000000000..1520489f843 --- /dev/null +++ b/Lib/test/test_capi/test_modsupport.py @@ -0,0 +1,154 @@ +import sys +import unittest +import sysconfig + +from test.support import subTests +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class Test_ABIInfo_Check(unittest.TestCase): + @subTests('modname', (None, 'test_mod')) + def test_zero(self, modname): + _testcapi.pyabiinfo_check(modname, 0, 0, 0, 0, 0) + _testcapi.pyabiinfo_check(modname, 1, 0, 0, 0, 0) + + def test_large_major_version(self): + with self.assertRaisesRegex(ImportError, + '^PyABIInfo version too high$'): + _testcapi.pyabiinfo_check(None, 2, 0, 0, 0, 0) + with self.assertRaisesRegex(ImportError, + '^test_mod: PyABIInfo version too high$'): + _testcapi.pyabiinfo_check("test_mod", 2, 0, 0, 0, 0) + + @subTests('modname', (None, 'test_mod')) + def test_large_minor_version(self, modname): + _testcapi.pyabiinfo_check(modname, 1, 2, 0, 0, 0) + + @subTests('modname', (None, 'test_mod')) + @subTests('major', (0, 1)) + @subTests('minor', (0, 1, 9)) + @subTests('build', (0, sys.hexversion)) + def test_positive_regular(self, modname, major, minor, build): + ver = sys.hexversion + truncated = ver & 0xffff0000 + filled = truncated | 0x12b8 + maxed = truncated | 0xffff + for abi_version in (0, ver, truncated, filled, maxed): + with self.subTest(abi_version=abi_version): + _testcapi.pyabiinfo_check(modname, major, minor, 0, + build, abi_version) + + @subTests('modname', (None, 'test_mod')) + @subTests('minor', (0, 1, 9)) + @subTests('build', (0, sys.hexversion)) + @subTests('offset', (+0x00010000, -0x00010000)) + def test_negative_regular(self, modname, minor, build, offset): + ver = sys.hexversion + offset + truncated = ver & 0xffff0000 + filled = truncated | 0x12b8 + maxed = truncated | 0xffff + for abi_version in (ver, truncated, filled, maxed): + with self.subTest(abi_version=abi_version): + with self.assertRaisesRegex( + ImportError, + r'incompatible ABI version \(3\.\d+\)$'): + _testcapi.pyabiinfo_check(modname, 1, minor, 0, + build, + abi_version) + + @subTests('modname', (None, 'test_mod')) + @subTests('major', (0, 1)) + @subTests('minor', (0, 1, 9)) + @subTests('build', (0, sys.hexversion)) + @subTests('abi_version', ( + 0, + 0x03020000, + sys.hexversion, + sys.hexversion & 0xffff0000, + sys.hexversion - 0x00010000, + )) + def test_positive_stable(self, modname, major, minor, build, abi_version): + _testcapi.pyabiinfo_check(modname, major, minor, + _testcapi.PyABIInfo_STABLE, + build, + abi_version) + + @subTests('modname', (None, 'test_mod')) + @subTests('minor', (0, 1, 9)) + @subTests('build', (0, sys.hexversion)) + @subTests('abi_version_and_msg', ( + (1, 'invalid'), + (3, 'invalid'), + (0x0301ffff, 'invalid'), + ((sys.hexversion & 0xffff0000) + 0x00010000, 'incompatible future'), + (sys.hexversion + 0x00010000, 'incompatible future'), + (0x04000000, 'incompatible future'), + )) + def test_negative_stable(self, modname, minor, build, abi_version_and_msg): + abi_version, msg = abi_version_and_msg + with self.assertRaisesRegex( + ImportError, + rf'{msg} stable ABI version \(\d+\.\d+\)$'): + _testcapi.pyabiinfo_check(modname, 1, minor, + _testcapi.PyABIInfo_STABLE, + build, + abi_version) + + @subTests('modname', (None, 'test_mod')) + @subTests('major', (0, 1)) + @subTests('minor', (0, 1, 9)) + @subTests('build', (0, sys.hexversion)) + @subTests('abi_version', (0, sys.hexversion)) + def test_positive_internal(self, modname, major, minor, build, abi_version): + _testcapi.pyabiinfo_check(modname, major, minor, + _testcapi.PyABIInfo_INTERNAL, + build, + abi_version) + + @subTests('modname', (None, 'test_mod')) + @subTests('minor', (0, 1, 9)) + @subTests('build', (0, sys.hexversion)) + @subTests('abi_version', ( + sys.hexversion - 0x00010000, + sys.hexversion - 1, + sys.hexversion + 1, + sys.hexversion + 0x00010000, + )) + def test_negative_internal(self, modname, minor, build, abi_version): + with self.assertRaisesRegex( + ImportError, + r'incompatible internal ABI \(0x[\da-f]+ != 0x[\da-f]+\)$'): + _testcapi.pyabiinfo_check(modname, 1, minor, + _testcapi.PyABIInfo_INTERNAL, + build, + abi_version) + + @subTests('modname', (None, 'test_mod')) + @subTests('minor', (0, 1, 9)) + @subTests('build', (0, sys.hexversion)) + @subTests('ft_flag', ( + 0, + (_testcapi.PyABIInfo_FREETHREADED + if sysconfig.get_config_var("Py_GIL_DISABLED") + else _testcapi.PyABIInfo_GIL), + _testcapi.PyABIInfo_FREETHREADING_AGNOSTIC, + )) + def test_positive_freethreading(self, modname, minor, build, ft_flag): + self.assertEqual(ft_flag & _testcapi.PyABIInfo_FREETHREADING_AGNOSTIC, + ft_flag) + _testcapi.pyabiinfo_check(modname, 1, minor, ft_flag, build, 0) + + @subTests('modname', (None, 'test_mod')) + @subTests('minor', (0, 1, 9)) + @subTests('build', (0, sys.hexversion)) + def test_negative_freethreading(self, modname, minor, build): + if sysconfig.get_config_var("Py_GIL_DISABLED"): + ft_flag = _testcapi.PyABIInfo_GIL + msg = "incompatible with free-threaded CPython" + else: + ft_flag = _testcapi.PyABIInfo_FREETHREADED + msg = "only compatible with free-threaded CPython" + with self.assertRaisesRegex(ImportError, msg): + _testcapi.pyabiinfo_check(modname, 1, minor, ft_flag, build, 0) diff --git a/Lib/test/test_capi/test_module.py b/Lib/test/test_capi/test_module.py new file mode 100644 index 00000000000..823e2ab6b2e --- /dev/null +++ b/Lib/test/test_capi/test_module.py @@ -0,0 +1,186 @@ +# The C functions used by this module are in: +# Modules/_testcapi/module.c + +import unittest +import types +from test.support import import_helper, subTests, requires_gil_enabled + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + + +class FakeSpec: + name = 'testmod' + +DEF_SLOTS = ( + 'Py_mod_name', 'Py_mod_doc', 'Py_mod_state_size', 'Py_mod_methods', + 'Py_mod_state_traverse', 'Py_mod_state_clear', 'Py_mod_state_free', + 'Py_mod_token', +) + +def def_and_token(mod): + return ( + _testcapi.pymodule_get_def(mod), + _testcapi.pymodule_get_token(mod), + ) + +class TestModFromSlotsAndSpec(unittest.TestCase): + @requires_gil_enabled("empty slots re-enable GIL") + def test_empty(self): + mod = _testcapi.module_from_slots_empty(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, 0) + + def test_null_slots(self): + with self.assertRaises(SystemError): + _testcapi.module_from_slots_null(FakeSpec()) + + def test_none_spec(self): + # The spec currently must contain a name + with self.assertRaises(AttributeError): + _testcapi.module_from_slots_empty(None) + with self.assertRaises(AttributeError): + _testcapi.module_from_slots_name(None) + + def test_name(self): + # Py_mod_name (and PyModuleDef.m_name) are currently ignored when + # spec is given. + # We still test that it's accepted. + mod = _testcapi.module_from_slots_name(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_doc(self): + mod = _testcapi.module_from_slots_doc(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, 'the docstring') + + def test_size(self): + mod = _testcapi.module_from_slots_size(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, 123) + + def test_methods(self): + mod = _testcapi.module_from_slots_methods(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.a_method(456), (mod, 456)) + + def test_gc(self): + mod = _testcapi.module_from_slots_gc(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + # Check that the requested hook functions (which module_from_slots_gc + # stores as attributes) match what's in the module (as retrieved by + # _testinternalcapi.module_get_gc_hooks) + _testinternalcapi = import_helper.import_module('_testinternalcapi') + traverse, clear, free = _testinternalcapi.module_get_gc_hooks(mod) + self.assertEqual(traverse, mod.traverse) + self.assertEqual(clear, mod.clear) + self.assertEqual(free, mod.free) + + def test_token(self): + mod = _testcapi.module_from_slots_token(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, _testcapi.module_test_token)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + + def test_exec(self): + mod = _testcapi.module_from_slots_exec(FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(def_and_token(mod), (0, 0)) + self.assertEqual(mod.__name__, 'testmod') + self.assertEqual(mod.__doc__, None) + self.assertEqual(mod.a_number, 456) + + def test_create(self): + spec = FakeSpec() + spec._gimme_this = "not a module object" + mod = _testcapi.module_from_slots_create(spec) + self.assertIsInstance(mod, str) + self.assertEqual(mod, "not a module object") + with self.assertRaises(TypeError): + _testcapi.pymodule_get_def(mod), + with self.assertRaises(TypeError): + _testcapi.pymodule_get_token(mod) + + def test_def_slot(self): + """Slots that replace PyModuleDef fields can't be used with PyModuleDef + """ + for name in DEF_SLOTS: + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_def_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("PyModuleDef", str(cm.exception)) + + def test_repeated_def_slot(self): + """Slots that replace PyModuleDef fields can't be repeated""" + for name in (*DEF_SLOTS, 'Py_mod_exec'): + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_slots_repeat_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("more than one", str(cm.exception)) + + def test_null_def_slot(self): + """Slots that replace PyModuleDef fields can't be NULL""" + for name in (*DEF_SLOTS, 'Py_mod_exec'): + with self.subTest(name): + spec = FakeSpec() + spec._test_slot_id = getattr(_testcapi, name) + with self.assertRaises(SystemError) as cm: + _testcapi.module_from_slots_null_slot(spec) + self.assertIn(name, str(cm.exception)) + self.assertIn("NULL", str(cm.exception)) + + def test_def_multiple_exec(self): + """PyModule_Exec runs all exec slots of PyModuleDef-defined module""" + mod = _testcapi.module_from_def_multiple_exec(FakeSpec()) + self.assertFalse(hasattr(mod, 'a_number')) + _testcapi.pymodule_exec(mod) + self.assertEqual(mod.a_number, 456) + self.assertEqual(mod.another_number, 789) + _testcapi.pymodule_exec(mod) + self.assertEqual(mod.a_number, 456) + self.assertEqual(mod.another_number, -789) + def_ptr, token = def_and_token(mod) + self.assertEqual(def_ptr, token) + + def test_def_token(self): + """In PyModuleDef-defined modules, the def is the token""" + mod = _testcapi.module_from_def_multiple_exec(FakeSpec()) + def_ptr, token = def_and_token(mod) + self.assertEqual(def_ptr, token) + self.assertGreater(def_ptr, 0) + + @subTests('name, expected_size', [ + (__name__, 0), # Python module + ('_testsinglephase', -1), # single-phase init + ('sys', -1), + ]) + def test_get_state_size(self, name, expected_size): + mod = import_helper.import_module(name) + size = _testcapi.pymodule_get_state_size(mod) + self.assertEqual(size, expected_size) diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 127862546b1..c5040913e9e 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -1,4 +1,5 @@ import enum +import os import sys import textwrap import unittest @@ -13,6 +14,9 @@ _testcapi = import_helper.import_module('_testcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') +NULL = None +STDERR_FD = 2 + class Constant(enum.IntEnum): Py_CONSTANT_NONE = 0 @@ -180,7 +184,7 @@ def test_is_uniquely_referenced(self): self.assertTrue(_testcapi.is_uniquely_referenced(object())) self.assertTrue(_testcapi.is_uniquely_referenced([])) # Immortals - self.assertFalse(_testcapi.is_uniquely_referenced("spanish inquisition")) + self.assertFalse(_testcapi.is_uniquely_referenced(())) self.assertFalse(_testcapi.is_uniquely_referenced(42)) # CRASHES is_uniquely_referenced(NULL) @@ -221,6 +225,7 @@ def test_decref_freed_object(self): """ self.check_negative_refcount(code) + @support.requires_resource('cpu') def test_decref_delayed(self): # gh-130519: Test that _PyObject_XDecRefDelayed() and QSBR code path # handles destructors that are possibly re-entrant or trigger a GC. @@ -246,5 +251,53 @@ def func(x): func(object()) + def pyobject_dump(self, obj, release_gil=False): + pyobject_dump = _testcapi.pyobject_dump + + try: + old_stderr = os.dup(STDERR_FD) + except OSError as exc: + # os.dup(STDERR_FD) is not supported on WASI + self.skipTest(f"os.dup() failed with {exc!r}") + + filename = os_helper.TESTFN + try: + try: + with open(filename, "wb") as fp: + fd = fp.fileno() + os.dup2(fd, STDERR_FD) + pyobject_dump(obj, release_gil) + finally: + os.dup2(old_stderr, STDERR_FD) + os.close(old_stderr) + + with open(filename) as fp: + return fp.read().rstrip() + finally: + os_helper.unlink(filename) + + def test_pyobject_dump(self): + # test string object + str_obj = 'test string' + output = self.pyobject_dump(str_obj) + hex_regex = r'(0x)?[0-9a-fA-F]+' + regex = ( + fr"object address : {hex_regex}\n" + r"object refcount : [0-9]+\n" + fr"object type : {hex_regex}\n" + r"object type name: str\n" + r"object repr : 'test string'" + ) + self.assertRegex(output, regex) + + # release the GIL + output = self.pyobject_dump(str_obj, release_gil=True) + self.assertRegex(output, regex) + + # test NULL object + output = self.pyobject_dump(NULL) + self.assertRegex(output, r'') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index ba7bcb4540a..51234a2e40f 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5,6 +5,7 @@ import unittest import gc import os +import types import _opcode @@ -14,8 +15,10 @@ _testinternalcapi = import_helper.import_module("_testinternalcapi") -from _testinternalcapi import TIER2_THRESHOLD +from _testinternalcapi import _PY_NSMALLPOSINTS, TIER2_THRESHOLD +#For test of issue 136154 +GLOBAL_136154 = 42 @contextlib.contextmanager def clear_executors(func): @@ -37,6 +40,17 @@ def get_first_executor(func): pass return None +def get_all_executors(func): + code = func.__code__ + co_code = code.co_code + executors = [] + for i in range(0, len(co_code), 2): + try: + executors.append(_opcode.get_executor(code, i)) + except ValueError: + pass + return executors + def iter_opnames(ex): for item in ex: @@ -407,44 +421,18 @@ def testfunc(n, m): x = 0 for i in range(m): for j in MyIter(n): - x += 1000*i + j + x += j return x - x = testfunc(TIER2_THRESHOLD, TIER2_THRESHOLD) + x = testfunc(TIER2_THRESHOLD, 2) - self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * TIER2_THRESHOLD * 1001) + self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * 2) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) - def test_confidence_score(self): - def testfunc(n): - bits = 0 - for i in range(n): - if i & 0x01: - bits += 1 - if i & 0x02: - bits += 1 - if i&0x04: - bits += 1 - if i&0x08: - bits += 1 - if i&0x10: - bits += 1 - return bits - - x = testfunc(TIER2_THRESHOLD * 2) - - self.assertEqual(x, TIER2_THRESHOLD * 5) - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - ops = list(iter_opnames(ex)) - #Since branch is 50/50 the trace could go either way. - count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") - self.assertLessEqual(count, 2) - @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @@ -678,7 +666,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) def test_float_subtract_constant_propagation(self): def testfunc(n): @@ -700,7 +688,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", uops) def test_float_multiply_constant_propagation(self): def testfunc(n): @@ -722,7 +710,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", uops) def test_add_unicode_propagation(self): def testfunc(n): @@ -844,37 +832,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_unicode_count), 1) self.assertIn("_COMPARE_OP_STR", uops) - def test_type_inconsistency(self): - ns = {} - src = textwrap.dedent(""" - def testfunc(n): - for i in range(n): - x = _test_global + _test_global - """) - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertIn("_BINARY_OP_ADD_INT", uops) - # Try again, but between the runs, set the global to a float. - # This should result in no executor the second time. - ns = {} - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_sequential(self): def dummy12(x): return x - 1 @@ -903,6 +861,7 @@ def testfunc(n): largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_nested(self): def dummy12(x): return x + 3 @@ -933,6 +892,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls(self): def dummy12(x): return x + 3 @@ -968,6 +928,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls_different_order(self): # same as `several_calls` but with top-level calls reversed def dummy12(x): @@ -1004,6 +965,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_complex(self): def dummy0(x): return x @@ -1053,6 +1015,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_large_framesize(self): # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is # actually doing its job. Note that the resulting trace hits @@ -1114,6 +1077,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_recursion(self): def dummy15(x): while x > 0: @@ -1183,6 +1147,17 @@ def testfunc(n): self.assertIsNotNone(ex) self.assertIn("_RETURN_GENERATOR", get_opnames(ex)) + def test_for_iter(self): + def testfunc(n): + t = 0 + for i in set(range(n)): + t += i + return t + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD * (TIER2_THRESHOLD - 1) // 2) + self.assertIsNotNone(ex) + self.assertIn("_FOR_ITER_TIER_TWO", get_opnames(ex)) + @unittest.skip("Tracing into generators currently isn't supported.") def test_for_iter_gen(self): def gen(n): @@ -1280,8 +1255,8 @@ class Bar: self.assertIsNotNone(ex) self.assertEqual(res, TIER2_THRESHOLD * 6 + 1) call = opnames.index("_CALL_BUILTIN_FAST") - load_attr_top = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", 0, call) - load_attr_bottom = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", call) + load_attr_top = opnames.index("_POP_TOP_LOAD_CONST_INLINE_BORROW", 0, call) + load_attr_bottom = opnames.index("_POP_TOP_LOAD_CONST_INLINE_BORROW", call) self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2) @@ -1303,8 +1278,8 @@ class Foo: self.assertIsNotNone(ex) self.assertEqual(res, TIER2_THRESHOLD * 2) call = opnames.index("_CALL_BUILTIN_FAST_WITH_KEYWORDS") - load_attr_top = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", 0, call) - load_attr_bottom = opnames.index("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", call) + load_attr_top = opnames.index("_POP_TOP_LOAD_CONST_INLINE_BORROW", 0, call) + load_attr_bottom = opnames.index("_POP_TOP_LOAD_CONST_INLINE_BORROW", call) self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2) @@ -1370,6 +1345,21 @@ def testfunc(n): # Removed guard self.assertNotIn("_CHECK_FUNCTION_EXACT_ARGS", uops) + def test_method_guards_removed_or_reduced(self): + def testfunc(n): + result = 0 + for i in range(n): + result += test_bound_method(i) + return result + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, sum(range(TIER2_THRESHOLD))) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + # Strength reduced version + self.assertIn("_CHECK_FUNCTION_VERSION_INLINE", uops) + self.assertNotIn("_CHECK_METHOD_VERSION", uops) + def test_jit_error_pops(self): """ Tests that the correct number of pops are inserted into the @@ -1436,7 +1426,8 @@ def testfunc(n): self.assertEqual(res, 3) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertNotIn("_BINARY_OP_ADD_INT", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) self.assertNotIn("_GUARD_NOS_INT", uops) self.assertNotIn("_GUARD_TOS_INT", uops) @@ -1586,7 +1577,75 @@ def f(n): # But all of the appends we care about are still there: self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) - def test_compare_pop_two_load_const_inline_borrow(self): + def test_unary_negative_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 1 + result = -a + if result < 0: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_NEGATIVE", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_unary_not_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 42 + result = not a + if result: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_NOT", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_unary_invert_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 0 + result = ~a + if result < 0: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_INVERT", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_compare_op_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = 10 + b = 10.0 + if a == b: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_COMPARE_OP", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + + def test_compare_op_int_pop_two_load_const_inline_borrow(self): def testfunc(n): x = 0 for _ in range(n): @@ -1603,6 +1662,57 @@ def testfunc(n): self.assertNotIn("_COMPARE_OP_INT", uops) self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + def test_compare_op_str_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = "foo" + b = "foo" + if a == b: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_COMPARE_OP_STR", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + + def test_compare_op_float_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = 1.0 + b = 1.0 + if a == b: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_COMPARE_OP_FLOAT", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + + def test_contains_op_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = "foo" + s = "foo bar baz" + if a in s: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_CONTAINS_OP", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + def test_to_bool_bool_contains_op_set(self): """ Test that _TO_BOOL_BOOL is removed from code like: @@ -1655,13 +1765,11 @@ def testfunc(n): self.assertIn("_CONTAINS_OP_DICT", uops) self.assertNotIn("_TO_BOOL_BOOL", uops) - def test_remove_guard_for_known_type_str(self): def f(n): for i in range(n): false = i == TIER2_THRESHOLD empty = "X"[:false] - empty += "" # Make JIT realize this is a string. if empty: return 1 return 0 @@ -1767,7 +1875,23 @@ def testfunc(n): self.assertNotIn("_GUARD_TOS_UNICODE", uops) self.assertIn("_BINARY_OP_ADD_UNICODE", uops) - def test_call_type_1(self): + def test_call_type_1_guards_removed(self): + def testfunc(n): + x = 0 + for _ in range(n): + foo = eval('42') + x += type(foo) is int + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_TYPE_1", uops) + self.assertNotIn("_GUARD_NOS_NULL", uops) + self.assertNotIn("_GUARD_CALLABLE_TYPE_1", uops) + + def test_call_type_1_known_type(self): def testfunc(n): x = 0 for _ in range(n): @@ -1778,9 +1902,13 @@ def testfunc(n): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_CALL_TYPE_1", uops) - self.assertNotIn("_GUARD_NOS_NULL", uops) - self.assertNotIn("_GUARD_CALLABLE_TYPE_1", uops) + # When the result of type(...) is known, _CALL_TYPE_1 is replaced with + # _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW which is optimized away in + # remove_unneeded_uops. + self.assertNotIn("_CALL_TYPE_1", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) def test_call_type_1_result_is_const(self): def testfunc(n): @@ -1795,7 +1923,6 @@ def testfunc(n): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_CALL_TYPE_1", uops) self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_call_str_1(self): @@ -1925,6 +2052,101 @@ def testfunc(n): self.assertNotIn("_GUARD_NOS_INT", uops) self.assertNotIn("_GUARD_TOS_INT", uops) + def test_call_len_known_length_small_int(self): + # Make sure that len(t) is optimized for a tuple of length 5. + # See https://github.com/python/cpython/issues/139393. + self.assertGreater(_PY_NSMALLPOSINTS, 5) + + def testfunc(n): + x = 0 + for _ in range(n): + t = (1, 2, 3, 4, 5) + if len(t) == 5: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # When the length is < _PY_NSMALLPOSINTS, the len() call is replaced + # with just an inline load. + self.assertNotIn("_CALL_LEN", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_call_len_known_length(self): + # Make sure that len(t) is not optimized for a tuple of length 2048. + # See https://github.com/python/cpython/issues/139393. + self.assertLess(_PY_NSMALLPOSINTS, 2048) + + def testfunc(n): + class C: + t = tuple(range(2048)) + + x = 0 + for _ in range(n): + if len(C.t) == 2048: # comparison + guard removed + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # When the length is >= _PY_NSMALLPOSINTS, we cannot replace + # the len() call with an inline load, but knowing the exact + # length allows us to optimize more code, such as conditionals + # in this case + self.assertIn("_CALL_LEN", uops) + self.assertNotIn("_COMPARE_OP_INT", uops) + self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + + def test_get_len_with_const_tuple(self): + def testfunc(n): + x = 0.0 + for _ in range(n): + match (1, 2, 3, 4): + case [_, _, _, _]: + x += 1.0 + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(int(res), TIER2_THRESHOLD) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GET_LEN", uops) + self.assertIn("_LOAD_CONST_INLINE_BORROW", uops) + + def test_get_len_with_non_const_tuple(self): + def testfunc(n): + x = 0.0 + for _ in range(n): + match object(), object(): + case [_, _]: + x += 1.0 + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(int(res), TIER2_THRESHOLD) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GET_LEN", uops) + self.assertIn("_LOAD_CONST_INLINE_BORROW", uops) + + def test_get_len_with_non_tuple(self): + def testfunc(n): + x = 0.0 + for _ in range(n): + match [1, 2, 3, 4]: + case [_, _, _, _]: + x += 1.0 + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(int(res), TIER2_THRESHOLD) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertIn("_GET_LEN", uops) + def test_binary_op_subscr_tuple_int(self): def testfunc(n): x = 0 @@ -1942,9 +2164,547 @@ def testfunc(n): self.assertNotIn("_COMPARE_OP_INT", uops) self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + def test_call_isinstance_guards_removed(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = isinstance(42, int) + if y: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_CALL_ISINSTANCE", uops) + self.assertNotIn("_GUARD_THIRD_NULL", uops) + self.assertNotIn("_GUARD_CALLABLE_ISINSTANCE", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", uops) + + def test_call_list_append(self): + def testfunc(n): + a = [] + for i in range(n): + a.append(i) + return sum(a) + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, sum(range(TIER2_THRESHOLD))) + uops = get_opnames(ex) + self.assertIn("_CALL_LIST_APPEND", uops) + # We should remove these in the future + self.assertIn("_GUARD_NOS_LIST", uops) + self.assertIn("_GUARD_CALLABLE_LIST_APPEND", uops) + + def test_call_isinstance_is_true(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = isinstance(42, int) + if y: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_CALL_ISINSTANCE", uops) + self.assertNotIn("_TO_BOOL_BOOL", uops) + self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", uops) + + def test_call_isinstance_is_false(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = isinstance(42, str) + if not y: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_CALL_ISINSTANCE", uops) + self.assertNotIn("_TO_BOOL_BOOL", uops) + self.assertNotIn("_GUARD_IS_FALSE_POP", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", uops) + + def test_call_isinstance_subclass(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = isinstance(True, int) + if y: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_CALL_ISINSTANCE", uops) + self.assertNotIn("_TO_BOOL_BOOL", uops) + self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", uops) + + def test_call_isinstance_unknown_object(self): + def testfunc(n): + x = 0 + for _ in range(n): + # The optimizer doesn't know the return type here: + bar = eval("42") + # This will only narrow to bool: + y = isinstance(bar, int) + if y: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_ISINSTANCE", uops) + self.assertNotIn("_TO_BOOL_BOOL", uops) + self.assertIn("_GUARD_IS_TRUE_POP", uops) + + def test_call_isinstance_tuple_of_classes(self): + def testfunc(n): + x = 0 + for _ in range(n): + # A tuple of classes is currently not optimized, + # so this is only narrowed to bool: + y = isinstance(42, (int, str)) + if y: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_ISINSTANCE", uops) + self.assertNotIn("_TO_BOOL_BOOL", uops) + self.assertIn("_GUARD_IS_TRUE_POP", uops) + + def test_call_isinstance_metaclass(self): + class EvenNumberMeta(type): + def __instancecheck__(self, number): + return number % 2 == 0 + + class EvenNumber(metaclass=EvenNumberMeta): + pass + + def testfunc(n): + x = 0 + for _ in range(n): + # Only narrowed to bool + y = isinstance(42, EvenNumber) + if y: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_ISINSTANCE", uops) + self.assertNotIn("_TO_BOOL_BOOL", uops) + self.assertIn("_GUARD_IS_TRUE_POP", uops) + + def test_set_type_version_sets_type(self): + class C: + A = 1 + + def testfunc(n): + x = 0 + c = C() + for _ in range(n): + x += c.A # Guarded. + x += type(c).A # Unguarded! + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 2 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_GUARD_TYPE_VERSION", uops) + self.assertNotIn("_CHECK_ATTR_CLASS", uops) + + def test_load_small_int(self): + def testfunc(n): + x = 0 + for i in range(n): + x += 1 + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_LOAD_SMALL_INT", uops) + self.assertIn("_LOAD_CONST_INLINE_BORROW", uops) + + def test_cached_attributes(self): + class C: + A = 1 + def m(self): + return 1 + class D: + __slots__ = () + A = 1 + def m(self): + return 1 + class E(Exception): + def m(self): + return 1 + def f(n): + x = 0 + c = C() + d = D() + e = E() + for _ in range(n): + x += C.A # _LOAD_ATTR_CLASS + x += c.A # _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + x += d.A # _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + x += c.m() # _LOAD_ATTR_METHOD_WITH_VALUES + x += d.m() # _LOAD_ATTR_METHOD_NO_DICT + x += e.m() # _LOAD_ATTR_METHOD_LAZY_DICT + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 6 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_LOAD_ATTR_CLASS", uops) + self.assertNotIn("_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", uops) + self.assertNotIn("_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", uops) + self.assertNotIn("_LOAD_ATTR_METHOD_WITH_VALUES", uops) + self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) + self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + + def test_float_op_refcount_elimination(self): + def testfunc(args): + a, b, n = args + c = 0.0 + for _ in range(n): + c += a + b + return c + + res, ex = self._run_with_optimizer(testfunc, (0.1, 0.1, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * (0.1 + 0.1)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) + + def test_remove_guard_for_slice_list(self): + def f(n): + for i in range(n): + false = i == TIER2_THRESHOLD + sliced = [1, 2, 3][:false] + if sliced: + return 1 + return 0 + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_TO_BOOL_LIST", uops) + self.assertNotIn("_GUARD_TOS_LIST", uops) + + def test_remove_guard_for_slice_tuple(self): + def f(n): + for i in range(n): + false = i == TIER2_THRESHOLD + a, b = (1, 2, 3)[: false + 2] + + _, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) + self.assertNotIn("_GUARD_TOS_TUPLE", uops) + + def test_unary_invert_long_type(self): + def testfunc(n): + for _ in range(n): + a = 9397 + x = ~a + ~a + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertNotIn("_GUARD_TOS_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + + def test_attr_promotion_failure(self): + # We're not testing for any specific uops here, just + # testing it doesn't crash. + script_helper.assert_python_ok('-c', textwrap.dedent(""" + import _testinternalcapi + import _opcode + import email + + def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + def testfunc(n): + for _ in range(n): + email.jit_testing = None + prompt = email.jit_testing + del email.jit_testing + + + testfunc(_testinternalcapi.TIER2_THRESHOLD) + """)) + + def test_pop_top_specialize_none(self): + def testfunc(n): + for _ in range(n): + global_identity(None) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_POP_TOP_NOP", uops) + + def test_pop_top_specialize_int(self): + def testfunc(n): + for _ in range(n): + global_identity(100000) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_POP_TOP_INT", uops) + + def test_pop_top_specialize_float(self): + def testfunc(n): + for _ in range(n): + global_identity(1e6) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_POP_TOP_FLOAT", uops) + + + def test_unary_negative_long_float_type(self): + def testfunc(n): + for _ in range(n): + a = 9397 + f = 9397.0 + x = -a + -a + y = -f + -f + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertNotIn("_GUARD_TOS_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GUARD_TOS_FLOAT", uops) + self.assertNotIn("_GUARD_NOS_FLOAT", uops) + + def test_binary_op_constant_evaluate(self): + def testfunc(n): + for _ in range(n): + 2 ** 65 + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + # For now... until we constant propagate it away. + self.assertIn("_BINARY_OP", uops) + + def test_jitted_code_sees_changed_globals(self): + "Issue 136154: Check that jitted code spots the change in the globals" + + def make_f(): + def f(): + return GLOBAL_136154 + return f + + make_f_with_bad_globals = types.FunctionType(make_f.__code__, {}) + + def jitted(funcs): + for func in funcs: + func() + + # Make a "good" f: + f = make_f() + # Compile jitted for the "good" f: + jitted([f] * TIER2_THRESHOLD) + # This "bad" f has different globals, but the *same* code/function versions: + f_with_bad_globals = make_f_with_bad_globals() + # A "good" f to enter the JIT code, and a "bad" f to trigger the bug: + with self.assertRaises(NameError): + jitted([f, f_with_bad_globals]) + + def test_reference_tracking_across_call_doesnt_crash(self): + + def f1(): + for _ in range(TIER2_THRESHOLD + 1): + # Choose a value that won't occur elsewhere to avoid sharing + str("value that won't occur elsewhere to avoid sharing") + + f1() + + def f2(): + for _ in range(TIER2_THRESHOLD + 1): + # Choose a value that won't occur elsewhere to avoid sharing + tuple((31, -17, 25, "won't occur elsewhere")) + + f2() + + def test_next_instr_for_exception_handler_set(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + for i in range(TIER2_THRESHOLD + 3): + try: + undefined_variable(i) + except Exception: + pass + + f() + + def test_next_instr_for_exception_handler_set_lasts_instr(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + a_list = [] + for _ in range(TIER2_THRESHOLD + 3): + try: + a_list[""] = 0 + except Exception: + pass + + f() + + def test_interpreter_finalization_with_generator_alive(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + t = tuple(range(%d)) + def simple_for(): + for x in t: + x + + def gen(): + try: + yield + except: + simple_for() + + sys.settrace(lambda *args: None) + simple_for() + g = gen() + next(g) + """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) + + def test_executor_side_exits_create_another_executor(self): + def f(): + for x in range(TIER2_THRESHOLD + 3): + for y in range(TIER2_THRESHOLD + 3): + z = x + y + + f() + all_executors = get_all_executors(f) + # Inner loop warms up first. + # Outer loop warms up later, linking to the inner one. + # Therefore, we have at least two executors. + self.assertGreaterEqual(len(all_executors), 2) + for executor in all_executors: + opnames = list(get_opnames(executor)) + # Assert all executors first terminator ends in + # _EXIT_TRACE or _JUMP_TO_TOP, not _DEOPT + for idx, op in enumerate(opnames): + if op == "_EXIT_TRACE" or op == "_JUMP_TO_TOP": + break + elif op == "_DEOPT": + self.fail(f"_DEOPT encountered first at executor" + f" {executor} at offset {idx} rather" + f" than expected _EXIT_TRACE") + + def test_enter_executor_valid_op_arg(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + sys.setrecursionlimit(30) # reduce time of the run + + str_v1 = '' + tuple_v2 = (None, None, None, None, None) + small_int_v3 = 4 + + def f1(): + + for _ in range(10): + abs(0) + + tuple_v2[small_int_v3] + tuple_v2[small_int_v3] + tuple_v2[small_int_v3] + + def recursive_wrapper_4569(): + str_v1 > str_v1 + str_v1 > str_v1 + str_v1 > str_v1 + recursive_wrapper_4569() + + recursive_wrapper_4569() + + for i_f1 in range(19000): + try: + f1() + except RecursionError: + pass + """)) + def global_identity(x): return x +class TestObject: + def test(self, *args, **kwargs): + return args[0] + +test_object = TestObject() +test_bound_method = TestObject.test.__get__(test_object) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_sys.py b/Lib/test/test_capi/test_sys.py index d3a9b378e77..3793ce2461e 100644 --- a/Lib/test/test_capi/test_sys.py +++ b/Lib/test/test_capi/test_sys.py @@ -19,6 +19,68 @@ class CAPITest(unittest.TestCase): maxDiff = None + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getattr(self): + # Test PySys_GetAttr() + sys_getattr = _testlimitedcapi.sys_getattr + + self.assertIs(sys_getattr('stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(sys_getattr('\U0001f40d'), 42) + + with self.assertRaisesRegex(RuntimeError, r'lost sys\.nonexistent'): + sys_getattr('nonexistent') + with self.assertRaisesRegex(RuntimeError, r'lost sys\.\U0001f40d'): + sys_getattr('\U0001f40d') + self.assertRaises(TypeError, sys_getattr, 1) + self.assertRaises(TypeError, sys_getattr, []) + # CRASHES sys_getattr(NULL) + + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getattrstring(self): + # Test PySys_GetAttrString() + getattrstring = _testlimitedcapi.sys_getattrstring + + self.assertIs(getattrstring(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getattrstring('\U0001f40d'.encode()), 42) + + with self.assertRaisesRegex(RuntimeError, r'lost sys\.nonexistent'): + getattrstring(b'nonexistent') + with self.assertRaisesRegex(RuntimeError, r'lost sys\.\U0001f40d'): + getattrstring('\U0001f40d'.encode()) + self.assertRaises(UnicodeDecodeError, getattrstring, b'\xff') + # CRASHES getattrstring(NULL) + + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getoptionalattr(self): + # Test PySys_GetOptionalAttr() + getoptionalattr = _testlimitedcapi.sys_getoptionalattr + + self.assertIs(getoptionalattr('stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getoptionalattr('\U0001f40d'), 42) + + self.assertIs(getoptionalattr('nonexistent'), AttributeError) + self.assertIs(getoptionalattr('\U0001f40d'), AttributeError) + self.assertRaises(TypeError, getoptionalattr, 1) + self.assertRaises(TypeError, getoptionalattr, []) + # CRASHES getoptionalattr(NULL) + + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') + def test_sys_getoptionalattrstring(self): + # Test PySys_GetOptionalAttrString() + getoptionalattrstring = _testlimitedcapi.sys_getoptionalattrstring + + self.assertIs(getoptionalattrstring(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getoptionalattrstring('\U0001f40d'.encode()), 42) + + self.assertIs(getoptionalattrstring(b'nonexistent'), AttributeError) + self.assertIs(getoptionalattrstring('\U0001f40d'.encode()), AttributeError) + self.assertRaises(UnicodeDecodeError, getoptionalattrstring, b'\xff') + # CRASHES getoptionalattrstring(NULL) + @support.cpython_only @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_sys_getobject(self): @@ -29,7 +91,7 @@ def test_sys_getobject(self): with support.swap_attr(sys, '\U0001f40d', 42): self.assertEqual(getobject('\U0001f40d'.encode()), 42) - self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'nonexistent'), AttributeError) with support.catch_unraisable_exception() as cm: self.assertIs(getobject(b'\xff'), AttributeError) self.assertEqual(cm.unraisable.exc_type, UnicodeDecodeError) diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index 7c07bc64e24..d6669d7802c 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -14,6 +14,12 @@ class TupleSubclass(tuple): class CAPITest(unittest.TestCase): + def _not_tracked(self, t): + self.assertFalse(gc.is_tracked(t), t) + + def _tracked(self, t): + self.assertTrue(gc.is_tracked(t), t) + def test_check(self): # Test PyTuple_Check() check = _testlimitedcapi.tuple_check @@ -52,16 +58,47 @@ def test_tuple_new(self): self.assertEqual(tup1, ()) self.assertEqual(size(tup1), 0) self.assertIs(type(tup1), tuple) + self._not_tracked(tup1) + tup2 = tuple_new(1) self.assertIs(type(tup2), tuple) self.assertEqual(size(tup2), 1) self.assertIsNot(tup2, tup1) self.assertTrue(checknull(tup2, 0)) + self._tracked(tup2) self.assertRaises(SystemError, tuple_new, -1) self.assertRaises(SystemError, tuple_new, PY_SSIZE_T_MIN) self.assertRaises(MemoryError, tuple_new, PY_SSIZE_T_MAX) + def test_tuple_fromarray(self): + # Test PyTuple_FromArray() + tuple_fromarray = _testcapi.tuple_fromarray + + tup = tuple([i] for i in range(5)) + copy = tuple_fromarray(tup) + self.assertEqual(copy, tup) + self._tracked(copy) + + tup = tuple(42**i for i in range(5)) + copy = tuple_fromarray(tup) + self.assertEqual(copy, tup) + self._not_tracked(copy) + + tup = () + copy = tuple_fromarray(tup) + self.assertIs(copy, tup) + + copy = tuple_fromarray(NULL, 0) + self.assertIs(copy, ()) + + with self.assertRaises(SystemError): + tuple_fromarray(NULL, -1) + with self.assertRaises(SystemError): + tuple_fromarray(NULL, PY_SSIZE_T_MIN) + with self.assertRaises(MemoryError): + tuple_fromarray(NULL, PY_SSIZE_T_MAX) + def test_tuple_pack(self): # Test PyTuple_Pack() pack = _testlimitedcapi.tuple_pack @@ -70,6 +107,10 @@ def test_tuple_pack(self): self.assertEqual(pack(1, [1]), ([1],)) self.assertEqual(pack(2, [1], [2]), ([1], [2])) + self._tracked(pack(1, [1])) + self._tracked(pack(2, [1], b'abc')) + self._not_tracked(pack(2, 42, b'abc')) + self.assertRaises(SystemError, pack, PY_SSIZE_T_MIN) self.assertRaises(SystemError, pack, -1) self.assertRaises(MemoryError, pack, PY_SSIZE_T_MAX) diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py index 3c9974c7387..e6a8ef9eed6 100644 --- a/Lib/test/test_capi/test_type.py +++ b/Lib/test/test_capi/test_type.py @@ -195,6 +195,24 @@ class H2(int): pass with self.assertRaises(TypeError): _testcapi.pytype_getmodulebydef(H2) + def test_get_module_by_token(self): + token = _testcapi.pymodule_get_token(_testcapi) + + heaptype = _testcapi.create_type_with_token('_testcapi.H', 0) + mod = _testcapi.pytype_getmodulebytoken(heaptype, token) + self.assertIs(mod, _testcapi) + + class H1(heaptype): pass + mod = _testcapi.pytype_getmodulebytoken(H1, token) + self.assertIs(mod, _testcapi) + + with self.assertRaises(TypeError): + _testcapi.pytype_getmodulebytoken(int, token) + + class H2(int): pass + with self.assertRaises(TypeError): + _testcapi.pytype_getmodulebytoken(H2, token) + def test_freeze(self): # test PyType_Freeze() type_freeze = _testcapi.type_freeze @@ -264,3 +282,20 @@ def test_manual_heap_type(self): ManualHeapType = _testcapi.ManualHeapType for i in range(100): self.assertIsInstance(ManualHeapType(), ManualHeapType) + + def test_extension_managed_dict_type(self): + ManagedDictType = _testcapi.ManagedDictType + obj = ManagedDictType() + obj.foo = 42 + self.assertEqual(obj.foo, 42) + self.assertEqual(obj.__dict__, {'foo': 42}) + obj.__dict__ = {'bar': 3} + self.assertEqual(obj.__dict__, {'bar': 3}) + self.assertEqual(obj.bar, 3) + + def test_extension_managed_weakref_nogc_type(self): + msg = ("type _testcapi.ManagedWeakrefNoGCType " + "has the Py_TPFLAGS_MANAGED_WEAKREF " + "flag but not Py_TPFLAGS_HAVE_GC flag") + with self.assertRaisesRegex(SystemError, msg): + _testcapi.create_managed_weakref_nogc_type() diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 3408c10f426..6a9c60f3a6d 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1739,6 +1739,20 @@ def test_pep393_utf8_caching_bug(self): # Check that the second call returns the same result self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_GET_CACHED_HASH(self): + from _testcapi import unicode_GET_CACHED_HASH + content_bytes = b'some new string' + # avoid parser interning & constant folding + obj = str(content_bytes, 'ascii') + # impl detail: fresh strings do not have cached hash + self.assertEqual(unicode_GET_CACHED_HASH(obj), -1) + # impl detail: adding string to a dict caches its hash + {obj: obj} + # impl detail: ASCII string hashes are equal to bytes ones + self.assertEqual(unicode_GET_CACHED_HASH(obj), hash(content_bytes)) + class PyUnicodeWriterTest(unittest.TestCase): def create_writer(self, size): @@ -1776,6 +1790,13 @@ def test_utf8(self): self.assertEqual(writer.finish(), "ascii-latin1=\xE9-euro=\u20AC.") + def test_ascii(self): + writer = self.create_writer(0) + writer.write_ascii(b"Hello ", -1) + writer.write_ascii(b"", 0) + writer.write_ascii(b"Python! ", 6) + self.assertEqual(writer.finish(), "Hello Python") + def test_invalid_utf8(self): writer = self.create_writer(0) with self.assertRaises(UnicodeDecodeError): diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 8644479d83d..bef72032513 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -514,6 +514,10 @@ def myfunc(): _testcapi.set_func_kwdefaults_via_capi(myfunc, new_kwdefaults) self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_KWDEFAULTS, myfunc, new_kwdefaults), events) + new_qualname = "foo.bar" + myfunc.__qualname__ = new_qualname + self.assertIn((_testcapi.PYFUNC_EVENT_MODIFY_QUALNAME, myfunc, new_qualname), events) + # Clear events reference to func events = [] del myfunc diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 46fde541494..a52c2241f5d 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -12,7 +12,9 @@ from test import support -SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SOURCES = [ + os.path.join(os.path.dirname(__file__), 'extension.c'), +] SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @@ -28,40 +30,29 @@ @support.requires_venv_with_pip() @support.requires_subprocess() @support.requires_resource('cpu') -class TestExt(unittest.TestCase): +class BaseTests: + TEST_INTERNAL_C_API = False + # Default build with no options def test_build(self): self.check_build('_test_cext') - def test_build_c11(self): - self.check_build('_test_c11_cext', std='c11') - - @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") - def test_build_c99(self): - # In public docs, we say C API is compatible with C11. However, - # in practice we do maintain C99 compatibility in public headers. - # Please ask the C API WG before adding a new C11-only feature. - self.check_build('_test_c99_cext', std='c99') - - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited(self): - self.check_build('_test_limited_cext', limited=True) - - @support.requires_gil_enabled('broken for now with Free Threading') - def test_build_limited_c11(self): - self.check_build('_test_limited_c11_cext', limited=True, std='c11') - - def check_build(self, extension_name, std=None, limited=False): + def check_build(self, extension_name, std=None, limited=False, + opaque_pyobject=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: self._check_build(extension_name, python_exe, - std=std, limited=limited) + std=std, limited=limited, + opaque_pyobject=opaque_pyobject) - def _check_build(self, extension_name, python_exe, std, limited): + def _check_build(self, extension_name, python_exe, std, limited, + opaque_pyobject): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) - shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + for source in SOURCES: + dest = os.path.join(pkg_dir, os.path.basename(source)) + shutil.copy(source, dest) def run_cmd(operation, cmd): env = os.environ.copy() @@ -69,7 +60,10 @@ def run_cmd(operation, cmd): env['CPYTHON_TEST_STD'] = std if limited: env['CPYTHON_TEST_LIMITED'] = '1' + if opaque_pyobject: + env['CPYTHON_TEST_OPAQUE_PYOBJECT'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name + env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) if support.verbose: print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) @@ -110,5 +104,34 @@ def run_cmd(operation, cmd): run_cmd('Import', cmd) +class TestPublicCAPI(BaseTests, unittest.TestCase): + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_test_limited_cext', limited=True) + + @support.requires_gil_enabled('broken for now with Free Threading') + def test_build_limited_c11(self): + self.check_build('_test_limited_c11_cext', limited=True, std='c11') + + def test_build_c11(self): + self.check_build('_test_c11_cext', std='c11') + + def test_build_opaque_pyobject(self): + # Test with _Py_OPAQUE_PYOBJECT + self.check_build('_test_limited_opaque_cext', limited=True, + opaque_pyobject=True) + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") + def test_build_c99(self): + # In public docs, we say C API is compatible with C11. However, + # in practice we do maintain C99 compatibility in public headers. + # Please ask the C API WG before adding a new C11-only feature. + self.check_build('_test_c99_cext', std='c99') + + +class TestInteralCAPI(BaseTests, unittest.TestCase): + TEST_INTERNAL_C_API = True + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 64629c5a6da..0f668c1da32 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -1,11 +1,31 @@ // gh-116869: Basic C test extension to check that the Python C API // does not emit C compiler warnings. +// +// Test also the internal C API if the TEST_INTERNAL_C_API macro is defined. // Always enable assertions #undef NDEBUG +#ifdef TEST_INTERNAL_C_API +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" +#ifdef TEST_INTERNAL_C_API + // gh-135906: Check for compiler warnings in the internal C API. + // - Cython uses pycore_frame.h. + // - greenlet uses pycore_frame.h, pycore_interpframe_structs.h and + // pycore_interpframe.h. +# include "internal/pycore_frame.h" +# include "internal/pycore_gc.h" +# include "internal/pycore_interp.h" +# include "internal/pycore_interpframe.h" +# include "internal/pycore_interpframe_structs.h" +# include "internal/pycore_object.h" +# include "internal/pycore_pystate.h" +#endif + #ifndef MODULE_NAME # error "MODULE_NAME macro must be defined" #endif @@ -58,6 +78,9 @@ _testcext_exec( return 0; } +#define _FUNC_NAME(NAME) PyModExport_ ## NAME +#define FUNC_NAME(NAME) _FUNC_NAME(NAME) + // Converting from function pointer to void* has undefined behavior, but // works on all known platforms, and CPython's module and type slots currently // need it. @@ -65,38 +88,40 @@ _testcext_exec( _Py_COMP_DIAG_PUSH #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wcast-qual" #elif defined(__clang__) #pragma clang diagnostic ignored "-Wpedantic" +#pragma clang diagnostic ignored "-Wcast-qual" #endif +PyDoc_STRVAR(_testcext_doc, "C test extension."); + static PyModuleDef_Slot _testcext_slots[] = { + {Py_mod_name, STR(MODULE_NAME)}, + {Py_mod_doc, (void*)(char*)_testcext_doc}, {Py_mod_exec, (void*)_testcext_exec}, + {Py_mod_methods, _testcext_methods}, {0, NULL} }; _Py_COMP_DIAG_POP - -PyDoc_STRVAR(_testcext_doc, "C test extension."); - -static struct PyModuleDef _testcext_module = { - PyModuleDef_HEAD_INIT, // m_base - STR(MODULE_NAME), // m_name - _testcext_doc, // m_doc - 0, // m_size - _testcext_methods, // m_methods - _testcext_slots, // m_slots - NULL, // m_traverse - NULL, // m_clear - NULL, // m_free -}; - - -#define _FUNC_NAME(NAME) PyInit_ ## NAME -#define FUNC_NAME(NAME) _FUNC_NAME(NAME) - -PyMODINIT_FUNC +PyMODEXPORT_FUNC FUNC_NAME(MODULE_NAME)(void) { - return PyModuleDef_Init(&_testcext_module); + return _testcext_slots; +} + +// Also define the soft-deprecated entrypoint to ensure it isn't called + +#define _INITFUNC_NAME(NAME) PyInit_ ## NAME +#define INITFUNC_NAME(NAME) _INITFUNC_NAME(NAME) + +PyMODINIT_FUNC +INITFUNC_NAME(MODULE_NAME)(void) +{ + PyErr_SetString( + PyExc_AssertionError, + "PyInit_* function called while a PyModExport_* one is available"); + return NULL; } diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 1275282983f..67dfddec751 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -14,10 +14,15 @@ if not support.MS_WINDOWS: # C compiler flags for GCC and clang - CFLAGS = [ + BASE_CFLAGS = [ # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', + ] + + # C compiler flags for GCC and clang + PUBLIC_CFLAGS = [ + *BASE_CFLAGS, # gh-120593: Check the 'const' qualifier '-Wcast-qual', @@ -26,27 +31,43 @@ '-pedantic-errors', ] if not support.Py_GIL_DISABLED: - CFLAGS.append( + PUBLIC_CFLAGS.append( # gh-116869: The Python C API must be compatible with building # with the -Werror=declaration-after-statement compiler flag. '-Werror=declaration-after-statement', ) + INTERNAL_CFLAGS = [*BASE_CFLAGS] else: # MSVC compiler flags - CFLAGS = [ - # Display warnings level 1 to 4 - '/W4', + BASE_CFLAGS = [ # Treat all compiler warnings as compiler errors '/WX', ] + PUBLIC_CFLAGS = [ + *BASE_CFLAGS, + # Display warnings level 1 to 4 + '/W4', + ] + INTERNAL_CFLAGS = [ + *BASE_CFLAGS, + # Display warnings level 1 to 3 + '/W3', + ] def main(): std = os.environ.get("CPYTHON_TEST_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + opaque_pyobject = bool(os.environ.get("CPYTHON_TEST_OPAQUE_PYOBJECT", "")) + internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) - cflags = list(CFLAGS) + sources = [SOURCE] + + if not internal: + cflags = list(PUBLIC_CFLAGS) + else: + cflags = list(INTERNAL_CFLAGS) cflags.append(f'-DMODULE_NAME={module_name}') # Add -std=STD or /std:STD (MSVC) compiler flag @@ -75,6 +96,13 @@ def main(): version = sys.hexversion cflags.append(f'-DPy_LIMITED_API={version:#x}') + # Define _Py_OPAQUE_PYOBJECT macro + if opaque_pyobject: + cflags.append(f'-D_Py_OPAQUE_PYOBJECT') + + if internal: + cflags.append('-DTEST_INTERNAL_C_API=1') + # On Windows, add PCbuild\amd64\ to include and library directories include_dirs = [] library_dirs = [] @@ -99,7 +127,7 @@ def main(): ext = Extension( module_name, - sources=[SOURCE], + sources=sources, extra_compile_args=cflags, include_dirs=include_dirs, library_dirs=library_dirs) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 4c12d43556f..64222555166 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -652,6 +652,7 @@ class B(A): a = A(hash(A.f)^(-1)) hash(a.f) + @cpython_only def testSetattrWrapperNameIntern(self): # Issue #25794: __setattr__ should intern the attribute name class A: @@ -858,7 +859,12 @@ def __init__(self, arg): from _testinternalcapi import has_inline_values -Py_TPFLAGS_MANAGED_DICT = (1 << 2) +Py_TPFLAGS_INLINE_VALUES = (1 << 2) +Py_TPFLAGS_MANAGED_DICT = (1 << 4) + +class NoManagedDict: + __slots__ = ('a',) + class Plain: pass @@ -873,11 +879,31 @@ def __init__(self): self.d = 4 +class VarSizedSubclass(tuple): + pass + + class TestInlineValues(unittest.TestCase): - def test_flags(self): - self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) - self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + def test_no_flags_for_slots_class(self): + flags = NoManagedDict.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, 0) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(NoManagedDict())) + + def test_both_flags_for_regular_class(self): + for cls in (Plain, WithAttrs): + with self.subTest(cls=cls.__name__): + flags = cls.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, Py_TPFLAGS_INLINE_VALUES) + self.assertTrue(has_inline_values(cls())) + + def test_managed_dict_only_for_varsized_subclass(self): + flags = VarSizedSubclass.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(VarSizedSubclass())) def test_has_inline_values(self): c = Plain() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 0c99620e27c..e71f9fc181b 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -4,6 +4,7 @@ from functools import partial from test import support, test_tools +from test.support import force_not_colorized_test_class from test.support import os_helper from test.support.os_helper import TESTFN, unlink, rmtree from textwrap import dedent @@ -238,11 +239,11 @@ def test_directive_output_print(self): # The generated output will differ for every run, but we can check that # it starts with the clinic block, we check that it contains all the # expected fields, and we check that it contains the checksum line. - self.assertTrue(out.startswith(dedent(""" + self.assertStartsWith(out, dedent(""" /*[clinic input] output print 'I told you once.' [clinic start generated code]*/ - """))) + """)) fields = { "cpp_endif", "cpp_if", @@ -259,9 +260,7 @@ def test_directive_output_print(self): with self.subTest(field=field): self.assertIn(field, out) last_line = out.rstrip().split("\n")[-1] - self.assertTrue( - last_line.startswith("/*[clinic end generated code: output=") - ) + self.assertStartsWith(last_line, "/*[clinic end generated code: output=") def test_directive_wrong_arg_number(self): raw = dedent(""" @@ -359,6 +358,32 @@ def test_vararg_after_star(self): """ self.expect_failure(block, err, lineno=6) + def test_double_star_after_var_keyword(self): + err = "Function 'my_test_func' has an invalid parameter declaration (**kwargs?): '**kwds: dict'" + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + **kwds: dict + ** + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=5) + + def test_var_keyword_after_star(self): + err = "Function 'my_test_func' has an invalid parameter declaration: '**'" + block = """ + /*[clinic input] + my_test_func + + pos_arg: object + ** + **kwds: dict + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=5) + def test_module_already_got_one(self): err = "Already defined module 'm'!" block = """ @@ -750,6 +775,16 @@ def test_ignore_preprocessor_in_comments(self): """) self.clinic.parse(raw) + def test_var_keyword_non_dict(self): + err = "'var_keyword_object' is not a valid converter" + block = """ + /*[clinic input] + my_test_func + + **kwds: object + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=4) class ParseFileUnitTest(TestCase): def expect_parsing_failure( @@ -1279,12 +1314,8 @@ def test_base_invalid_syntax(self): os.stat invalid syntax: int = 42 """ - err = dedent(r""" - Function 'stat' has an invalid parameter declaration: - \s+'invalid syntax: int = 42' - """).strip() - with self.assertRaisesRegex(ClinicError, err): - self.parse_function(block) + err = "Function 'stat' has an invalid parameter declaration: 'invalid syntax: int = 42'" + self.expect_failure(block, err, lineno=2) def test_param_default_invalid_syntax(self): block = """ @@ -1292,7 +1323,7 @@ def test_param_default_invalid_syntax(self): os.stat x: int = invalid syntax """ - err = r"Syntax error: 'x = invalid syntax\n'" + err = "Function 'stat' has an invalid parameter declaration:" self.expect_failure(block, err, lineno=2) def test_cloning_nonexistent_function_correctly_fails(self): @@ -1614,6 +1645,11 @@ def test_disallowed_grouping__must_be_position_only(self): [ a: object ] + """, """ + with_kwds + [ + **kwds: dict + ] """) err = ( "You cannot use optional groups ('[' and ']') unless all " @@ -1997,6 +2033,44 @@ def test_slash_after_vararg(self): err = "Function 'bar': '/' must precede '*'" self.expect_failure(block, err) + def test_slash_after_var_keyword(self): + block = """ + module foo + foo.bar + x: int + y: int + **kwds: dict + z: int + / + """ + err = "Function 'bar' has an invalid parameter declaration (**kwargs?): '**kwds: dict'" + self.expect_failure(block, err) + + def test_star_after_var_keyword(self): + block = """ + module foo + foo.bar + x: int + y: int + **kwds: dict + z: int + * + """ + err = "Function 'bar' has an invalid parameter declaration (**kwargs?): '**kwds: dict'" + self.expect_failure(block, err) + + def test_parameter_after_var_keyword(self): + block = """ + module foo + foo.bar + x: int + y: int + **kwds: dict + z: int + """ + err = "Function 'bar' has an invalid parameter declaration (**kwargs?): '**kwds: dict'" + self.expect_failure(block, err) + def test_depr_star_must_come_after_slash(self): block = """ module foo @@ -2085,6 +2159,16 @@ def test_parameters_no_more_than_one_vararg(self): """ self.expect_failure(block, err, lineno=3) + def test_parameters_no_more_than_one_var_keyword(self): + err = "Encountered parameter line when not expecting parameters: **var_keyword_2: dict" + block = """ + module foo + foo.bar + **var_keyword_1: dict + **var_keyword_2: dict + """ + self.expect_failure(block, err, lineno=3) + def test_function_not_at_column_0(self): function = self.parse_function(""" module foo @@ -2512,13 +2596,21 @@ def test_cannot_specify_pydefault_without_default(self): self.expect_failure(block, err, lineno=1) def test_vararg_cannot_take_default_value(self): - err = "Vararg can't take a default value!" + err = "Function 'fn' has an invalid parameter declaration:" block = """ fn *args: tuple = None """ self.expect_failure(block, err, lineno=1) + def test_var_keyword_cannot_take_default_value(self): + err = "Function 'fn' has an invalid parameter declaration:" + block = """ + fn + **kwds: dict = None + """ + self.expect_failure(block, err, lineno=1) + def test_default_is_not_of_correct_type(self): err = ("int_converter: default value 2.5 for field 'a' " "is not of type 'int'") @@ -2616,7 +2708,58 @@ def test_disallow_defining_class_at_module_level(self): """ self.expect_failure(block, err, lineno=2) + def test_var_keyword_with_pos_or_kw(self): + block = """ + module foo + foo.bar + x: int + **kwds: dict + """ + err = "Function 'bar' has an invalid parameter declaration (**kwargs?): '**kwds: dict'" + self.expect_failure(block, err) + def test_var_keyword_with_kw_only(self): + block = """ + module foo + foo.bar + x: int + / + * + y: int + **kwds: dict + """ + err = "Function 'bar' has an invalid parameter declaration (**kwargs?): '**kwds: dict'" + self.expect_failure(block, err) + + def test_var_keyword_with_pos_or_kw_and_kw_only(self): + block = """ + module foo + foo.bar + x: int + / + y: int + * + z: int + **kwds: dict + """ + err = "Function 'bar' has an invalid parameter declaration (**kwargs?): '**kwds: dict'" + self.expect_failure(block, err) + + def test_allow_negative_accepted_by_py_ssize_t_converter_only(self): + errmsg = re.escape("converter_init() got an unexpected keyword argument 'allow_negative'") + unsupported_converters = [converter_name for converter_name in converters.keys() + if converter_name != "Py_ssize_t"] + for converter in unsupported_converters: + with self.subTest(converter=converter): + block = f""" + module m + m.func + a: {converter}(allow_negative=True) + """ + with self.assertRaisesRegex((AssertionError, TypeError), errmsg): + self.parse_function(block) + +@force_not_colorized_test_class class ClinicExternalTest(TestCase): maxDiff = None @@ -2705,8 +2848,7 @@ def test_cli_force(self): # Note, we cannot check the entire fail msg, because the path to # the tmp file will change for every run. _, err = self.expect_failure(fn) - self.assertTrue(err.endswith(fail_msg), - f"{err!r} does not end with {fail_msg!r}") + self.assertEndsWith(err, fail_msg) # Then, force regeneration; success expected. out = self.expect_success("-f", fn) self.assertEqual(out, "") @@ -2717,8 +2859,7 @@ def test_cli_force(self): ) with open(fn, encoding='utf-8') as f: generated = f.read() - self.assertTrue(generated.endswith(checksum), - (generated, checksum)) + self.assertEndsWith(generated, checksum) def test_cli_make(self): c_code = dedent(""" @@ -2800,6 +2941,7 @@ def test_cli_verbose(self): out = self.expect_success("-v", fn) self.assertEqual(out.strip(), fn) + @support.force_not_colorized def test_cli_help(self): out = self.expect_success("-h") self.assertIn("usage: clinic.py", out) @@ -2835,7 +2977,13 @@ def test_cli_converters(self): "size_t", "slice_index", "str", + "uint16", + "uint32", + "uint64", + "uint8", "unicode", + "unicode_fs_decoded", + "unicode_fs_encoded", "unsigned_char", "unsigned_int", "unsigned_long", @@ -2863,8 +3011,8 @@ def test_cli_converters(self): # param may change (it's a set, thus unordered). So, let's compare the # start and end of the expected output, and then assert that the # converters appear lined up in alphabetical order. - self.assertTrue(out.startswith(prelude), out) - self.assertTrue(out.endswith(finale), out) + self.assertStartsWith(out, prelude) + self.assertEndsWith(out, finale) out = out.removeprefix(prelude) out = out.removesuffix(finale) @@ -2872,10 +3020,7 @@ def test_cli_converters(self): for converter, line in zip(expected_converters, lines): line = line.lstrip() with self.subTest(converter=converter): - self.assertTrue( - line.startswith(converter), - f"expected converter {converter!r}, got {line!r}" - ) + self.assertStartsWith(line, converter) def test_cli_fail_converters_and_filename(self): _, err = self.expect_failure("--converters", "test.c") @@ -3051,6 +3196,8 @@ def test_char_converter(self): def test_unsigned_char_converter(self): from _testcapi import UCHAR_MAX + SCHAR_MAX = UCHAR_MAX // 2 + SCHAR_MIN = SCHAR_MAX - UCHAR_MAX with self.assertRaises(OverflowError): ac_tester.unsigned_char_converter(-1) with self.assertRaises(OverflowError): @@ -3060,8 +3207,13 @@ def test_unsigned_char_converter(self): with self.assertRaises(TypeError): ac_tester.unsigned_char_converter([]) self.assertEqual(ac_tester.unsigned_char_converter(), (12, 34, 56)) - self.assertEqual(ac_tester.unsigned_char_converter(0, 0, UCHAR_MAX + 1), (0, 0, 0)) - self.assertEqual(ac_tester.unsigned_char_converter(0, 0, (UCHAR_MAX + 1) * 3 + 123), (0, 0, 123)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, UCHAR_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, SCHAR_MIN), (0, 0, SCHAR_MAX + 1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, SCHAR_MIN - 1), (0, 0, SCHAR_MAX)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_char_converter(0, 0, (UCHAR_MAX + 1) * 3 + 123), (0, 0, 123)) def test_short_converter(self): from _testcapi import SHRT_MIN, SHRT_MAX @@ -3075,7 +3227,7 @@ def test_short_converter(self): self.assertEqual(ac_tester.short_converter(4321), (4321,)) def test_unsigned_short_converter(self): - from _testcapi import USHRT_MAX + from _testcapi import SHRT_MIN, SHRT_MAX, USHRT_MAX with self.assertRaises(ValueError): ac_tester.unsigned_short_converter(-1) with self.assertRaises(OverflowError): @@ -3085,8 +3237,13 @@ def test_unsigned_short_converter(self): with self.assertRaises(TypeError): ac_tester.unsigned_short_converter([]) self.assertEqual(ac_tester.unsigned_short_converter(), (12, 34, 56)) - self.assertEqual(ac_tester.unsigned_short_converter(0, 0, USHRT_MAX + 1), (0, 0, 0)) - self.assertEqual(ac_tester.unsigned_short_converter(0, 0, (USHRT_MAX + 1) * 3 + 123), (0, 0, 123)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, USHRT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, SHRT_MIN), (0, 0, SHRT_MAX + 1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, SHRT_MIN - 1), (0, 0, SHRT_MAX)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_short_converter(0, 0, (USHRT_MAX + 1) * 3 + 123), (0, 0, 123)) def test_int_converter(self): from _testcapi import INT_MIN, INT_MAX @@ -3102,7 +3259,7 @@ def test_int_converter(self): self.assertEqual(ac_tester.int_converter(1, 2, '3'), (1, 2, ord('3'))) def test_unsigned_int_converter(self): - from _testcapi import UINT_MAX + from _testcapi import INT_MIN, INT_MAX, UINT_MAX with self.assertRaises(ValueError): ac_tester.unsigned_int_converter(-1) with self.assertRaises(OverflowError): @@ -3112,8 +3269,13 @@ def test_unsigned_int_converter(self): with self.assertRaises(TypeError): ac_tester.unsigned_int_converter([]) self.assertEqual(ac_tester.unsigned_int_converter(), (12, 34, 56)) - self.assertEqual(ac_tester.unsigned_int_converter(0, 0, UINT_MAX + 1), (0, 0, 0)) - self.assertEqual(ac_tester.unsigned_int_converter(0, 0, (UINT_MAX + 1) * 3 + 123), (0, 0, 123)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, UINT_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, INT_MIN), (0, 0, INT_MAX + 1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, INT_MIN - 1), (0, 0, INT_MAX)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_int_converter(0, 0, (UINT_MAX + 1) * 3 + 123), (0, 0, 123)) def test_long_converter(self): from _testcapi import LONG_MIN, LONG_MAX @@ -3127,7 +3289,7 @@ def test_long_converter(self): self.assertEqual(ac_tester.long_converter(-1234), (-1234,)) def test_unsigned_long_converter(self): - from _testcapi import ULONG_MAX + from _testcapi import LONG_MIN, LONG_MAX, ULONG_MAX with self.assertRaises(ValueError): ac_tester.unsigned_long_converter(-1) with self.assertRaises(OverflowError): @@ -3137,8 +3299,13 @@ def test_unsigned_long_converter(self): with self.assertRaises(TypeError): ac_tester.unsigned_long_converter([]) self.assertEqual(ac_tester.unsigned_long_converter(), (12, 34, 56)) - self.assertEqual(ac_tester.unsigned_long_converter(0, 0, ULONG_MAX + 1), (0, 0, 0)) - self.assertEqual(ac_tester.unsigned_long_converter(0, 0, (ULONG_MAX + 1) * 3 + 123), (0, 0, 123)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, ULONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, LONG_MIN), (0, 0, LONG_MAX + 1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, LONG_MIN - 1), (0, 0, LONG_MAX)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_long_converter(0, 0, (ULONG_MAX + 1) * 3 + 123), (0, 0, 123)) def test_long_long_converter(self): from _testcapi import LLONG_MIN, LLONG_MAX @@ -3152,7 +3319,7 @@ def test_long_long_converter(self): self.assertEqual(ac_tester.long_long_converter(-1234), (-1234,)) def test_unsigned_long_long_converter(self): - from _testcapi import ULLONG_MAX + from _testcapi import LLONG_MIN, LLONG_MAX, ULLONG_MAX with self.assertRaises(ValueError): ac_tester.unsigned_long_long_converter(-1) with self.assertRaises(OverflowError): @@ -3162,8 +3329,13 @@ def test_unsigned_long_long_converter(self): with self.assertRaises(TypeError): ac_tester.unsigned_long_long_converter([]) self.assertEqual(ac_tester.unsigned_long_long_converter(), (12, 34, 56)) - self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, ULLONG_MAX + 1), (0, 0, 0)) - self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, (ULLONG_MAX + 1) * 3 + 123), (0, 0, 123)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, ULLONG_MAX + 1), (0, 0, 0)) + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, LLONG_MIN), (0, 0, LLONG_MAX + 1)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, LLONG_MIN - 1), (0, 0, LLONG_MAX)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ac_tester.unsigned_long_long_converter(0, 0, (ULLONG_MAX + 1) * 3 + 123), (0, 0, 123)) def test_py_ssize_t_converter(self): from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX @@ -3173,8 +3345,12 @@ def test_py_ssize_t_converter(self): ac_tester.py_ssize_t_converter(PY_SSIZE_T_MAX + 1) with self.assertRaises(TypeError): ac_tester.py_ssize_t_converter([]) - self.assertEqual(ac_tester.py_ssize_t_converter(), (12, 34, 56)) - self.assertEqual(ac_tester.py_ssize_t_converter(1, 2, None), (1, 2, 56)) + with self.assertRaises(ValueError): + ac_tester.py_ssize_t_converter(12, 34, 56, -1) + with self.assertRaises(ValueError): + ac_tester.py_ssize_t_converter(12, 34, 56, 78, -1) + self.assertEqual(ac_tester.py_ssize_t_converter(), (12, 34, 56, 78, 90, -12, -34)) + self.assertEqual(ac_tester.py_ssize_t_converter(1, 2, None, 3, None, 4, None), (1, 2, 56, 3, 90, 4, -34)) def test_slice_index_converter(self): from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX @@ -3916,6 +4092,49 @@ def test_depr_multi(self): check("a", b="b", c="c", d="d", e="e", f="f", g="g") self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") + def test_lone_kwds(self): + with self.assertRaises(TypeError): + ac_tester.lone_kwds(1, 2) + self.assertEqual(ac_tester.lone_kwds(), ({},)) + self.assertEqual(ac_tester.lone_kwds(y='y'), ({'y': 'y'},)) + kwds = {'y': 'y', 'z': 'z'} + self.assertEqual(ac_tester.lone_kwds(y='y', z='z'), (kwds,)) + self.assertEqual(ac_tester.lone_kwds(**kwds), (kwds,)) + + def test_kwds_with_pos_only(self): + with self.assertRaises(TypeError): + ac_tester.kwds_with_pos_only() + with self.assertRaises(TypeError): + ac_tester.kwds_with_pos_only(y='y') + with self.assertRaises(TypeError): + ac_tester.kwds_with_pos_only(1, y='y') + self.assertEqual(ac_tester.kwds_with_pos_only(1, 2), (1, 2, {})) + self.assertEqual(ac_tester.kwds_with_pos_only(1, 2, y='y'), (1, 2, {'y': 'y'})) + kwds = {'y': 'y', 'z': 'z'} + self.assertEqual(ac_tester.kwds_with_pos_only(1, 2, y='y', z='z'), (1, 2, kwds)) + self.assertEqual(ac_tester.kwds_with_pos_only(1, 2, **kwds), (1, 2, kwds)) + + def test_kwds_with_stararg(self): + self.assertEqual(ac_tester.kwds_with_stararg(), ((), {})) + self.assertEqual(ac_tester.kwds_with_stararg(1, 2), ((1, 2), {})) + self.assertEqual(ac_tester.kwds_with_stararg(y='y'), ((), {'y': 'y'})) + args = (1, 2) + kwds = {'y': 'y', 'z': 'z'} + self.assertEqual(ac_tester.kwds_with_stararg(1, 2, y='y', z='z'), (args, kwds)) + self.assertEqual(ac_tester.kwds_with_stararg(*args, **kwds), (args, kwds)) + + def test_kwds_with_pos_only_and_stararg(self): + with self.assertRaises(TypeError): + ac_tester.kwds_with_pos_only_and_stararg() + with self.assertRaises(TypeError): + ac_tester.kwds_with_pos_only_and_stararg(y='y') + self.assertEqual(ac_tester.kwds_with_pos_only_and_stararg(1, 2), (1, 2, (), {})) + self.assertEqual(ac_tester.kwds_with_pos_only_and_stararg(1, 2, y='y'), (1, 2, (), {'y': 'y'})) + args = ('lobster', 'thermidor') + kwds = {'y': 'y', 'z': 'z'} + self.assertEqual(ac_tester.kwds_with_pos_only_and_stararg(1, 2, 'lobster', 'thermidor', y='y', z='z'), (1, 2, args, kwds)) + self.assertEqual(ac_tester.kwds_with_pos_only_and_stararg(1, 2, *args, **kwds), (1, 2, args, kwds)) + class LimitedCAPIOutputTests(unittest.TestCase): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1b40e0d05fe..3ed7a360d64 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -39,7 +39,8 @@ def test_directories(self): def verify_valid_flag(self, cmd_line): rc, out, err = assert_python_ok(cmd_line) - self.assertTrue(out == b'' or out.endswith(b'\n')) + if out != b'': + self.assertEndsWith(out, b'\n') self.assertNotIn(b'Traceback', out) self.assertNotIn(b'Traceback', err) return out @@ -89,8 +90,8 @@ def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': rc, out, err = assert_python_ok(switch) - self.assertFalse(err.startswith(version)) - self.assertTrue(out.startswith(version)) + self.assertNotStartsWith(err, version) + self.assertStartsWith(out, version) def test_verbose(self): # -v causes imports to write to stderr. If the write to @@ -299,6 +300,10 @@ def run_utf8_mode(arg): cmd = [sys.executable, '-X', 'utf8', '-c', code, arg] return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) + def run_no_utf8_mode(arg): + cmd = [sys.executable, '-X', 'utf8=0', '-c', code, arg] + return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) + valid_utf8 = 'e:\xe9, euro:\u20ac, non-bmp:\U0010ffff'.encode('utf-8') # invalid UTF-8 byte sequences with a valid UTF-8 sequence # in the middle. @@ -311,7 +316,8 @@ def run_utf8_mode(arg): ) test_args = [valid_utf8, invalid_utf8] - for run_cmd in (run_default, run_c_locale, run_utf8_mode): + for run_cmd in (run_default, run_c_locale, run_utf8_mode, + run_no_utf8_mode): with self.subTest(run_cmd=run_cmd): for arg in test_args: proc = run_cmd(arg) @@ -380,7 +386,7 @@ def test_unbuffered_input(self): p.stdin.flush() data, rc = _kill_python_and_exit_code(p) self.assertEqual(rc, 0) - self.assertTrue(data.startswith(b'x'), data) + self.assertStartsWith(data, b'x') def test_large_PYTHONPATH(self): path1 = "ABCDE" * 100 @@ -483,6 +489,7 @@ def test_unmached_quote(self): self.assertRegex(err.decode('ascii', 'ignore'), 'SyntaxError') self.assertEqual(b'', out) + @force_not_colorized def test_stdout_flush_at_shutdown(self): # Issue #5319: if stdout.flush() fails at shutdown, an error should # be printed out. @@ -972,10 +979,25 @@ def test_python_legacy_windows_fs_encoding(self): @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') def test_python_legacy_windows_stdio(self): - code = "import sys; print(sys.stdin.encoding, sys.stdout.encoding)" - expected = 'cp' - rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSSTDIO='1') - self.assertIn(expected.encode(), out) + # Test that _WindowsConsoleIO is used when PYTHONLEGACYWINDOWSSTDIO + # is not set. + # We cannot use PIPE becase it prevents creating new console. + # So we use exit code. + code = "import sys; sys.exit(type(sys.stdout.buffer.raw).__name__ != '_WindowsConsoleIO')" + env = os.environ.copy() + env["PYTHONLEGACYWINDOWSSTDIO"] = "" + p = subprocess.run([sys.executable, "-c", code], + creationflags=subprocess.CREATE_NEW_CONSOLE, + env=env) + self.assertEqual(p.returncode, 0) + + # Then test that FIleIO is used when PYTHONLEGACYWINDOWSSTDIO is set. + code = "import sys; sys.exit(type(sys.stdout.buffer.raw).__name__ != 'FileIO')" + env["PYTHONLEGACYWINDOWSSTDIO"] = "1" + p = subprocess.run([sys.executable, "-c", code], + creationflags=subprocess.CREATE_NEW_CONSOLE, + env=env) + self.assertEqual(p.returncode, 0) @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), "PYTHONMALLOCSTATS doesn't work with ASAN") @@ -1024,7 +1046,7 @@ def test_parsing_error(self): stderr=subprocess.PIPE, text=True) err_msg = "Unknown option: --unknown-option\nusage: " - self.assertTrue(proc.stderr.startswith(err_msg), proc.stderr) + self.assertStartsWith(proc.stderr, err_msg) self.assertNotEqual(proc.returncode, 0) def test_int_max_str_digits(self): diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 53dc9b1a7ef..cc1a625a509 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -553,9 +553,9 @@ def test_pep_409_verbiage(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = stderr.decode('ascii').split('\n') self.assertEqual(len(text), 5) - self.assertTrue(text[0].startswith('Traceback')) - self.assertTrue(text[1].startswith(' File ')) - self.assertTrue(text[3].startswith('NameError')) + self.assertStartsWith(text[0], 'Traceback') + self.assertStartsWith(text[1], ' File ') + self.assertStartsWith(text[3], 'NameError') def test_non_ascii(self): # Apple platforms deny the creation of a file with an invalid UTF-8 name. @@ -708,9 +708,8 @@ def test_syntaxerror_does_not_crash(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() # It used to crash in https://github.com/python/cpython/issues/111132 - self.assertTrue(text.endswith( - 'SyntaxError: nonlocal declaration not allowed at module level\n', - ), text) + self.assertEndsWith(text, + 'SyntaxError: nonlocal declaration not allowed at module level\n') def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same @@ -811,6 +810,30 @@ def test_script_as_dev_fd(self): out, err = p.communicate() self.assertEqual(out, b"12345678912345678912345\n") + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + rc, out, err = assert_python_ok( + '-Werror', + '-Walways:::__main__', + '-Werror:::test.test_import.data.syntax_warnings', + '-Werror:::syntax_warnings', + filename) + self.assertEqual(err.count(b': SyntaxWarning: '), 6) + + def test_zipfile_run_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with os_helper.temp_dir() as script_dir: + zip_name, _ = make_zip_pkg( + script_dir, 'test_zip', 'test_pkg', '__main__', source) + rc, out, err = assert_python_ok( + '-Werror', + '-Walways:::__main__', + '-Werror:::test_pkg.__main__', + os.path.join(zip_name, 'test_pkg') + ) + self.assertEqual(err.count(b': SyntaxWarning: '), 12) def tearDownModule(): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index b646042a3b8..655f5a9be7f 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -220,6 +220,7 @@ import _testinternalcapi except ModuleNotFoundError: _testinternalcapi = None +import test._code_definitions as defs COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -671,8 +672,21 @@ def test_local_kinds(self): VARARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_POS VARKWARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_KW - import test._code_definitions as defs funcs = { + defs.simple_script: {}, + defs.complex_script: { + 'obj': CO_FAST_LOCAL, + 'pickle': CO_FAST_LOCAL, + 'spam_minimal': CO_FAST_LOCAL, + 'data': CO_FAST_LOCAL, + 'res': CO_FAST_LOCAL, + }, + defs.script_with_globals: { + 'obj1': CO_FAST_LOCAL, + 'obj2': CO_FAST_LOCAL, + }, + defs.script_with_explicit_empty_return: {}, + defs.script_with_return: {}, defs.spam_minimal: {}, defs.spam_with_builtins: { 'x': CO_FAST_LOCAL, @@ -687,6 +701,27 @@ def test_local_kinds(self): 'checks': CO_FAST_LOCAL, 'res': CO_FAST_LOCAL, }, + defs.spam_with_global_and_attr_same_name: {}, + defs.spam_full_args: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, + defs.spam_full_args_with_defaults: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, defs.spam_args_attrs_and_builtins: { 'a': POSONLY, 'b': POSONLY, @@ -700,6 +735,7 @@ def test_local_kinds(self): defs.spam_returns_arg: { 'x': POSORKW, }, + defs.spam_raises: {}, defs.spam_with_inner_not_closure: { 'eggs': CO_FAST_LOCAL, }, @@ -897,8 +933,20 @@ def new_var_counts(*, }, } - import test._code_definitions as defs funcs = { + defs.simple_script: new_var_counts(), + defs.complex_script: new_var_counts( + purelocals=5, + globalvars=1, + attrs=2, + ), + defs.script_with_globals: new_var_counts( + purelocals=2, + globalvars=1, + ), + defs.script_with_explicit_empty_return: new_var_counts(), + defs.script_with_return: new_var_counts(), + defs.spam_minimal: new_var_counts(), defs.spam_minimal: new_var_counts(), defs.spam_with_builtins: new_var_counts( purelocals=4, @@ -908,6 +956,24 @@ def new_var_counts(*, purelocals=5, globalvars=6, ), + defs.spam_with_global_and_attr_same_name: new_var_counts( + globalvars=2, + attrs=1, + ), + defs.spam_full_args: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), + defs.spam_full_args_with_defaults: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), defs.spam_args_attrs_and_builtins: new_var_counts( posonly=2, posorkw=2, @@ -919,6 +985,9 @@ def new_var_counts(*, defs.spam_returns_arg: new_var_counts( posorkw=1, ), + defs.spam_raises: new_var_counts( + globalvars=1, + ), defs.spam_with_inner_not_closure: new_var_counts( purelocals=1, ), @@ -1025,42 +1094,35 @@ def new_var_counts(*, counts = _testinternalcapi.get_code_var_counts(func.__code__) self.assertEqual(counts, expected) - def func_with_globals_and_builtins(): - mod1 = _testinternalcapi - mod2 = dis - mods = (mod1, mod2) - checks = tuple(callable(m) for m in mods) - return callable(mod2), tuple(mods), list(mods), checks - - func = func_with_globals_and_builtins + func = defs.spam_with_globals_and_builtins with self.subTest(f'{func} code'): expected = new_var_counts( - purelocals=4, - globalvars=5, + purelocals=5, + globalvars=6, ) counts = _testinternalcapi.get_code_var_counts(func.__code__) self.assertEqual(counts, expected) with self.subTest(f'{func} with own globals and builtins'): expected = new_var_counts( - purelocals=4, - globalvars=(2, 3), + purelocals=5, + globalvars=(2, 4), ) counts = _testinternalcapi.get_code_var_counts(func) self.assertEqual(counts, expected) with self.subTest(f'{func} without globals'): expected = new_var_counts( - purelocals=4, - globalvars=(0, 3, 2), + purelocals=5, + globalvars=(0, 4, 2), ) counts = _testinternalcapi.get_code_var_counts(func, globalsns={}) self.assertEqual(counts, expected) with self.subTest(f'{func} without both'): expected = new_var_counts( - purelocals=4, - globalvars=5, + purelocals=5, + globalvars=6, ) counts = _testinternalcapi.get_code_var_counts(func, globalsns={}, builtinsns={}) @@ -1068,12 +1130,40 @@ def func_with_globals_and_builtins(): with self.subTest(f'{func} without builtins'): expected = new_var_counts( - purelocals=4, - globalvars=(2, 0, 3), + purelocals=5, + globalvars=(2, 0, 4), ) counts = _testinternalcapi.get_code_var_counts(func, builtinsns={}) self.assertEqual(counts, expected) + @unittest.skipIf(_testinternalcapi is None, "missing _testinternalcapi") + def test_stateless(self): + self.maxDiff = None + + STATELESS_FUNCTIONS = [ + *defs.STATELESS_FUNCTIONS, + # stateless with defaults + defs.spam_full_args_with_defaults, + ] + + for func in defs.STATELESS_CODE: + with self.subTest((func, '(code)')): + _testinternalcapi.verify_stateless_code(func.__code__) + for func in STATELESS_FUNCTIONS: + with self.subTest((func, '(func)')): + _testinternalcapi.verify_stateless_code(func) + + for func in defs.FUNCTIONS: + if func not in defs.STATELESS_CODE: + with self.subTest((func, '(code)')): + with self.assertRaises(Exception): + _testinternalcapi.verify_stateless_code(func.__code__) + + if func not in STATELESS_FUNCTIONS: + with self.subTest((func, '(func)')): + with self.assertRaises(Exception): + _testinternalcapi.verify_stateless_code(func) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 57fb130070b..3642b47c2c1 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -133,7 +133,7 @@ def test_unicode_error(self): output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) output = output[output.index('(InteractiveConsole)'):] output = output[output.index('\n') + 1:] - self.assertTrue(output.startswith('UnicodeEncodeError: '), output) + self.assertStartsWith(output, 'UnicodeEncodeError: ') self.assertIs(self.sysmod.last_type, UnicodeEncodeError) self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) self.assertIsNone(self.sysmod.last_traceback) diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 86e5e5c1474..65d54d1004d 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1125,7 +1125,7 @@ def test_bug828737(self): text = 'abcghi'*n text.translate(charmap) - def test_mutatingdecodehandler(self): + def test_mutating_decode_handler(self): baddata = [ ("ascii", b"\xff"), ("utf-7", b"++"), @@ -1160,6 +1160,42 @@ def mutating(exc): for (encoding, data) in baddata: self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242") + def test_mutating_decode_handler_unicode_escape(self): + decode = codecs.unicode_escape_decode + def mutating(exc): + if isinstance(exc, UnicodeDecodeError): + r = data.get(exc.object[:exc.end]) + if r is not None: + exc.object = r[0] + exc.object[exc.end:] + return ('\u0404', r[1]) + raise AssertionError("don't know how to handle %r" % exc) + + codecs.register_error('test.mutating2', mutating) + data = { + br'\x0': (b'\\', 0), + br'\x3': (b'xxx\\', 3), + br'\x5': (b'x\\', 1), + } + def check(input, expected, msg): + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(decode(input, 'test.mutating2'), (expected, len(input))) + self.assertIn(msg, str(cm.warning)) + + check(br'\x0n\z', '\u0404\n\\z', r'"\z" is an invalid escape sequence') + check(br'\x0n\501', '\u0404\n\u0141', r'"\501" is an invalid octal escape sequence') + check(br'\x0z', '\u0404\\z', r'"\z" is an invalid escape sequence') + + check(br'\x3n\zr', '\u0404\n\\zr', r'"\z" is an invalid escape sequence') + check(br'\x3zr', '\u0404\\zr', r'"\z" is an invalid escape sequence') + check(br'\x3z5', '\u0404\\z5', r'"\z" is an invalid escape sequence') + check(memoryview(br'\x3z5x')[:-1], '\u0404\\z5', r'"\z" is an invalid escape sequence') + check(memoryview(br'\x3z5xy')[:-2], '\u0404\\z5', r'"\z" is an invalid escape sequence') + + check(br'\x5n\z', '\u0404\n\\z', r'"\z" is an invalid escape sequence') + check(br'\x5n\501', '\u0404\n\u0141', r'"\501" is an invalid octal escape sequence') + check(br'\x5z', '\u0404\\z', r'"\z" is an invalid escape sequence') + check(memoryview(br'\x5zy')[:-1], '\u0404\\z', r'"\z" is an invalid escape sequence') + # issue32583 def test_crashing_decode_handler(self): # better generating one more character to fill the extra space slot diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 94fcf98e757..c31faec9ee5 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,8 +1,10 @@ import codecs import contextlib import copy +import importlib import io import pickle +import os import sys import unittest import encodings @@ -11,6 +13,7 @@ from test import support from test.support import os_helper +from test.support import warnings_helper try: import _testlimitedcapi @@ -1196,23 +1199,39 @@ def test_escape(self): check(br"[\1010]", b"[A0]") check(br"[\x41]", b"[A]") check(br"[\x410]", b"[A0]") + + def test_warnings(self): + decode = codecs.escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, b"\\" + b) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), b"\\" + b.upper()) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\8" is an invalid escape sequence'): check(br"\8", b"\\8") with self.assertWarns(DeprecationWarning): check(br"\9", b"\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", b"\\\xfa") for i in range(0o400, 0o1000): - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%o" is an invalid octal escape sequence' % i): check(rb'\%o' % i, bytes([i & 0o377])) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\z" is an invalid escape sequence'): + self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\501" is an invalid octal escape sequence'): + self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6)) + def test_errors(self): decode = codecs.escape_decode self.assertRaises(ValueError, decode, br"\x") @@ -2661,24 +2680,40 @@ def test_escape_decode(self): check(br"[\x410]", "[A0]") check(br"\u20ac", "\u20ac") check(br"\U0001d120", "\U0001d120") + + def test_decode_warnings(self): + decode = codecs.unicode_escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtuvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, "\\" + chr(i)) if b.upper() not in b'UN': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), "\\" + chr(i-32)) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\8" is an invalid escape sequence'): check(br"\8", "\\8") with self.assertWarns(DeprecationWarning): check(br"\9", "\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", "\\\xfa") for i in range(0o400, 0o1000): - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%o" is an invalid octal escape sequence' % i): check(rb'\%o' % i, chr(i)) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\z" is an invalid escape sequence'): + self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\501" is an invalid octal escape sequence'): + self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6)) + def test_decode_errors(self): decode = codecs.unicode_escape_decode for c, d in (b'x', 2), (b'u', 4), (b'U', 4): @@ -3075,6 +3110,13 @@ def test_aliases(self): info = codecs.lookup(alias) self.assertEqual(info.name, expected_name) + def test_alias_modules_exist(self): + encodings_dir = os.path.dirname(encodings.__file__) + for value in encodings.aliases.aliases.values(): + codec_mod = f"encodings.{value}" + self.assertIsNotNone(importlib.util.find_spec(codec_mod), + f"Codec module not found: {codec_mod}") + def test_quopri_stateless(self): # Should encode with quotetabs=True encoded = codecs.encode(b"space tab\teol \n", "quopri-codec") @@ -3252,7 +3294,7 @@ def test_code_page_name(self): codecs.code_page_encode, 932, '\xff') self.assertRaisesRegex(UnicodeDecodeError, 'cp932', codecs.code_page_decode, 932, b'\x81\x00', 'strict', True) - self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8', + self.assertRaisesRegex(UnicodeDecodeError, 'cp65001', codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): @@ -3762,7 +3804,7 @@ def check_decode_strings(self, errors): with self.assertRaises(RuntimeError) as cm: self.decode(encoded, errors) errmsg = str(cm.exception) - self.assertTrue(errmsg.startswith("decode error: "), errmsg) + self.assertStartsWith(errmsg, "decode error: ") else: decoded = self.decode(encoded, errors) self.assertEqual(decoded, expected) @@ -3832,32 +3874,27 @@ def test_rot13_func(self): class CodecNameNormalizationTest(unittest.TestCase): """Test codec name normalization""" def test_codecs_lookup(self): - FOUND = (1, 2, 3, 4) - NOT_FOUND = (None, None, None, None) def search_function(encoding): - if encoding == "aaa_8": - return FOUND + if encoding.startswith("test."): + return (encoding, 2, 3, 4) else: - return NOT_FOUND + return None codecs.register(search_function) self.addCleanup(codecs.unregister, search_function) - self.assertEqual(FOUND, codecs.lookup('aaa_8')) - self.assertEqual(FOUND, codecs.lookup('AAA-8')) - self.assertEqual(FOUND, codecs.lookup('AAA---8')) - self.assertEqual(FOUND, codecs.lookup('AAA 8')) - self.assertEqual(FOUND, codecs.lookup('aaa\xe9\u20ac-8')) - self.assertEqual(NOT_FOUND, codecs.lookup('AAA.8')) - self.assertEqual(NOT_FOUND, codecs.lookup('AAA...8')) - self.assertEqual(NOT_FOUND, codecs.lookup('BBB-8')) - self.assertEqual(NOT_FOUND, codecs.lookup('BBB.8')) - self.assertEqual(NOT_FOUND, codecs.lookup('a\xe9\u20ac-8')) + self.assertEqual(codecs.lookup('test.aaa_8'), ('test.aaa_8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA-8'), ('test.aaa-8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa-8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA---8'), ('test.aaa---8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA 8'), ('test.aaa---8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA.8'), ('test.aaa.8', 2, 3, 4)) + self.assertEqual(codecs.lookup('TEST.AAA...8'), ('test.aaa...8', 2, 3, 4)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(codecs.lookup('TEST.AAA\xe9\u20ac-8'), ('test.aaa\xe9\u20ac-8', 2, 3, 4)) def test_encodings_normalize_encoding(self): - # encodings.normalize_encoding() ignores non-ASCII characters. normalize = encodings.normalize_encoding self.assertEqual(normalize('utf_8'), 'utf_8') - self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') self.assertEqual(normalize('utf 8'), 'utf_8') # encodings.normalize_encoding() doesn't convert # characters to lower case. @@ -3865,6 +3902,11 @@ def test_encodings_normalize_encoding(self): self.assertEqual(normalize('utf.8'), 'utf.8') self.assertEqual(normalize('utf...8'), 'utf...8') + # Non-ASCII *encoding* is deprecated. + msg = "Support for non-ascii encoding names will be removed in 3.17" + with warnings_helper.check_warnings((msg, DeprecationWarning)): + self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8') + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 0eefc22d11b..ed10bd3dcb6 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -322,7 +322,7 @@ def test_syntax_errors(self): dedent("""\ def foo(x,x): pass - """), "duplicate argument 'x' in function definition") + """), "duplicate parameter 'x' in function definition") diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 1e93530398b..22595239252 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -12,6 +12,7 @@ import string import sys from test import support +from test.support.import_helper import import_fresh_module import types import unittest @@ -542,6 +543,8 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) + @support.requires_resource('cpu') + def test_large_size(self): n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) @@ -734,7 +737,7 @@ def validate_abstract_methods(self, abc, *names): stubs = methodstubs.copy() del stubs[name] C = type('C', (abc,), stubs) - self.assertRaises(TypeError, C, name) + self.assertRaises(TypeError, C) def validate_isinstance(self, abc, name): stub = lambda s, *args: 0 @@ -961,7 +964,7 @@ class AnextOnly: async def __anext__(self): raise StopAsyncIteration self.assertNotIsInstance(AnextOnly(), AsyncIterator) - self.validate_abstract_methods(AsyncIterator, '__anext__', '__aiter__') + self.validate_abstract_methods(AsyncIterator, '__anext__') def test_Iterable(self): # Check some non-iterables @@ -1157,7 +1160,7 @@ def test_Iterator(self): for x in samples: self.assertIsInstance(x, Iterator) self.assertIsSubclass(type(x), Iterator) - self.validate_abstract_methods(Iterator, '__next__', '__iter__') + self.validate_abstract_methods(Iterator, '__next__') # Issue 10565 class NextOnly: @@ -1841,8 +1844,7 @@ def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) self.assertIsSubclass(sample, Mapping) - self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): def __len__(self): return 0 @@ -1857,7 +1859,7 @@ def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) self.assertIsSubclass(sample, MutableMapping) - self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__', + self.validate_abstract_methods(MutableMapping, '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') def test_MutableMapping_subclass(self): @@ -1896,8 +1898,7 @@ def test_Sequence(self): self.assertIsInstance(memoryview(b""), Sequence) self.assertIsSubclass(memoryview, Sequence) self.assertIsSubclass(str, Sequence) - self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Sequence, '__len__', '__getitem__') def test_Sequence_mixins(self): class SequenceSubclass(Sequence): @@ -1934,6 +1935,44 @@ def assert_index_same(seq1, seq2, index_args): assert_index_same( nativeseq, seqseq, (letter, start, stop)) + def test_ByteString(self): + previous_sys_modules = sys.modules.copy() + self.addCleanup(sys.modules.update, previous_sys_modules) + + for module in "collections", "_collections_abc", "collections.abc": + sys.modules.pop(module, None) + + with self.assertWarns(DeprecationWarning): + from collections.abc import ByteString + for sample in [bytes, bytearray]: + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(sample(), ByteString) + self.assertTrue(issubclass(sample, ByteString)) + for sample in [str, list, tuple]: + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(sample(), ByteString) + self.assertFalse(issubclass(sample, ByteString)) + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(memoryview(b""), ByteString) + self.assertFalse(issubclass(memoryview, ByteString)) + with self.assertWarns(DeprecationWarning): + self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + + with self.assertWarns(DeprecationWarning): + class X(ByteString): pass + + with self.assertWarns(DeprecationWarning): + # No metaclass conflict + class Z(ByteString, Awaitable): pass + + def test_ByteString_attribute_access(self): + collections_abc = import_fresh_module( + "collections.abc", + fresh=("collections", "_collections_abc") + ) + with self.assertWarns(DeprecationWarning): + collections_abc.ByteString + def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) @@ -1952,8 +1991,8 @@ def test_MutableSequence(self): self.assertIsSubclass(sample, MutableSequence) self.assertIsSubclass(array.array, MutableSequence) self.assertNotIsSubclass(str, MutableSequence) - self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', - '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__', + '__setitem__', '__delitem__', 'insert') def test_MutableSequence_mixins(self): # Test the mixins of MutableSequence by creating a minimal concrete @@ -2140,6 +2179,7 @@ def correctly_ordered(seq): self.assertTrue(correctly_ordered(p - q)) self.assertTrue(correctly_ordered(p | q)) self.assertTrue(correctly_ordered(p & q)) + self.assertTrue(correctly_ordered(p ^ q)) p, q = Counter(ps), Counter(qs) p += q @@ -2157,6 +2197,10 @@ def correctly_ordered(seq): p &= q self.assertTrue(correctly_ordered(p)) + p, q = Counter(ps), Counter(qs) + p ^= q + self.assertTrue(correctly_ordered(p)) + p, q = Counter(ps), Counter(qs) p.update(q) self.assertTrue(correctly_ordered(p)) @@ -2239,6 +2283,7 @@ def test_multiset_operations(self): (Counter.__sub__, lambda x, y: max(0, x-y)), (Counter.__or__, lambda x, y: max(0,x,y)), (Counter.__and__, lambda x, y: max(0, min(x,y))), + (Counter.__xor__, lambda x, y: max(0, max(x,y) - min(x,y))), ]: result = counterop(p, q) for x in elements: @@ -2256,6 +2301,7 @@ def test_multiset_operations(self): (Counter.__sub__, set.__sub__), (Counter.__or__, set.__or__), (Counter.__and__, set.__and__), + (Counter.__xor__, set.__xor__), ]: counter_result = counterop(p, q) set_result = setop(set(p.elements()), set(q.elements())) @@ -2274,6 +2320,7 @@ def test_inplace_operations(self): (Counter.__isub__, Counter.__sub__), (Counter.__ior__, Counter.__or__), (Counter.__iand__, Counter.__and__), + (Counter.__ixor__, Counter.__xor__), ]: c = p.copy() c_id = id(c) @@ -2349,6 +2396,7 @@ def test_multiset_operations_equivalent_to_set_operations(self): self.assertEqual(set(cp - cq), sp - sq) self.assertEqual(set(cp | cq), sp | sq) self.assertEqual(set(cp & cq), sp & sq) + self.assertEqual(set(cp ^ cq), sp ^ sq) self.assertEqual(cp == cq, sp == sq) self.assertEqual(cp != cq, sp != sq) self.assertEqual(cp <= cq, sp <= sq) @@ -2376,6 +2424,40 @@ def test_gt(self): self.assertTrue(Counter(a=3, b=2, c=0) > Counter('aab')) self.assertFalse(Counter(a=2, b=1, c=0) > Counter('aab')) + def test_symmetric_difference(self): + population = (-4, -3, -2, -1, 0, 1, 2, 3, 4) + + for a, b1, b2, c in product(population, repeat=4): + p = Counter(a=a, b=b1) + q = Counter(b=b2, c=c) + r = p ^ q + + # Elementwise invariants + for k in ('a', 'b', 'c'): + self.assertEqual(r[k], max(p[k], q[k]) - min(p[k], q[k])) + self.assertEqual(r[k], abs(p[k] - q[k])) + + # Invariant for all positive, negative, and zero counts + self.assertEqual(r, (p - q) | (q - p)) + + # Invariant for non-negative counts + if a >= 0 and b1 >= 0 and b2 >= 0 and c >= 0: + self.assertEqual(r, (p | q) - (p & q)) + + # Zeros and negatives eliminated + self.assertTrue(all(value > 0 for value in r.values())) + + # Output preserves input order: p first and then q + keys = list(p) + list(q) + indices = [keys.index(k) for k in r] + self.assertEqual(indices, sorted(indices)) + + # Inplace operation matches binary operation + pp = Counter(p) + qq = Counter(q) + pp ^= qq + self.assertEqual(pp, r) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(collections)) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 57e5f29b015..fa611f480d6 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -651,6 +651,21 @@ def test_compile_filename(self): compile('pass', filename, 'exec') self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') + def test_compile_filename_refleak(self): + # Regression tests for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'mode' AC conversion failed + self.assertRaises(TypeError, compile, b'', mortal_str, mode=1234) + # check error path when 'optimize' AC conversion failed + self.assertRaises(OverflowError, compile, b'', mortal_str, + 'exec', optimize=1 << 1000) + # check error path when 'dont_inherit' AC conversion failed + class EvilBool: + def __bool__(self): raise ValueError + self.assertRaises(ValueError, compile, b'', mortal_str, + 'exec', dont_inherit=EvilBool()) + @support.cpython_only def test_same_filename_used(self): s = """def f(): pass\ndef g(): pass""" @@ -713,7 +728,8 @@ def test_yet_more_evil_still_undecodable(self): def test_compiler_recursion_limit(self): # Compiler frames are small limit = 100 - crash_depth = limit * 5000 + # Android test devices have less memory. + crash_depth = limit * (1000 if sys.platform == "android" else 5000) success_depth = limit def check_limit(prefix, repeated, mode="single"): @@ -823,8 +839,10 @@ def f(): else: return "unused" - self.assertEqual(f.__code__.co_consts, - (f.__doc__, "used")) + if f.__doc__ is None: + self.assertEqual(f.__code__.co_consts, (True, "used")) + else: + self.assertEqual(f.__code__.co_consts, (f.__doc__, "used")) @support.cpython_only def test_remove_unused_consts_no_docstring(self): @@ -869,7 +887,11 @@ def test_strip_unused_None(self): def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, (f1.__doc__,)) + + if f1.__doc__ is None: + self.assertEqual(f1.__code__.co_consts, (42,)) + else: + self.assertEqual(f1.__code__.co_consts, (f1.__doc__,)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -1015,11 +1037,13 @@ def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). compile("42", FakePath("test_compile_pathlike"), "single") + # bpo-31113: Stack overflow when compile a long sequence of + # complex statements. @support.requires_resource('cpu') def test_stack_overflow(self): - # bpo-31113: Stack overflow when compile a long sequence of - # complex statements. - compile("if a: b\n" * 200000, "", "exec") + # Android test devices have less memory. + size = 100_000 if sys.platform == "android" else 200_000 + compile("if a: b\n" * size, "", "exec") # Multiple users rely on the fact that CPython does not generate # bytecode for dead code blocks. See bpo-37500 for more context. @@ -1609,6 +1633,17 @@ def test_remove_redundant_nop_edge_case(self): def f(): a if (1 if b else c) else d + def test_lineno_propagation_empty_blocks(self): + # Smoke test. See gh-138714. + def f(): + while name: + try: + break + except: + pass + else: + 1 if 1 else 1 + def test_global_declaration_in_except_used_in_else(self): # See gh-111123 code = textwrap.dedent("""\ @@ -1647,22 +1682,21 @@ class WeirdDict(dict): self.assertRaises(NameError, ns['foo']) def test_compile_warnings(self): - # See gh-131927 - # Compile warnings originating from the same file and - # line are now only emitted once. + # Each invocation of compile() emits compiler warnings, even if they + # have the same message and line number. + source = textwrap.dedent(r""" + # tokenizer + 1or 0 # line 3 + # code generator + 1 is 1 # line 5 + """) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("default") - compile('1 is 1', '', 'eval') - compile('1 is 1', '', 'eval') + for i in range(2): + # Even if compile() is at the same line. + compile(source, '', 'exec') - self.assertEqual(len(caught), 1) - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - compile('1 is 1', '', 'eval') - compile('1 is 1', '', 'eval') - - self.assertEqual(len(caught), 2) + self.assertEqual([wm.lineno for wm in caught], [3, 5] * 2) def test_compile_warning_in_finally(self): # Ensure that warnings inside finally blocks are @@ -1673,16 +1707,131 @@ def test_compile_warning_in_finally(self): try: pass finally: - 1 is 1 + 1 is 1 # line 5 + try: + pass + finally: # nested + 1 is 1 # line 9 """) with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("default") + warnings.simplefilter("always") compile(source, '', 'exec') - self.assertEqual(len(caught), 1) - self.assertEqual(caught[0].category, SyntaxWarning) - self.assertIn("\"is\" with 'int' literal", str(caught[0].message)) + self.assertEqual(sorted(wm.lineno for wm in caught), [5, 9]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + # Other code path is used for "try" with "except*". + source = textwrap.dedent(""" + try: + pass + except *Exception: + pass + finally: + 1 is 1 # line 7 + try: + pass + except *Exception: + pass + finally: # nested + 1 is 1 # line 13 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + compile(source, '', 'exec') + + self.assertEqual(sorted(wm.lineno for wm in caught), [7, 13]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + compile(source, filename, 'exec') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + compile(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + @support.subTests('src', [ + textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + break + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + continue + """), + ]) + def test_pep_765_warnings(self, src): + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(src, '', 'exec') + with warnings.catch_warnings(): + warnings.simplefilter("error") + tree = ast.parse(src) + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(tree, '', 'exec') + + @support.subTests('src', [ + textwrap.dedent(""" + try: + pass + finally: + def f(): + return 42 + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + break + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + continue + """), + ]) + def test_pep_765_no_warnings(self, src): + with warnings.catch_warnings(): + warnings.simplefilter("error") + compile(src, '', 'exec') + class TestBooleanExpression(unittest.TestCase): class Value: @@ -1723,6 +1872,21 @@ def test_compound(self): self.assertIs(res, v[3]) self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + def test_exception(self): + # See gh-137288 + class Foo: + def __bool__(self): + raise NotImplementedError() + + a = Foo() + b = Foo() + + with self.assertRaises(NotImplementedError): + bool(a) + + with self.assertRaises(NotImplementedError): + c = a or b + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index a580a240d9f..8384c183dd9 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -316,7 +316,7 @@ def _test_ddir_only(self, *, ddir, parallel=True): self.assertTrue(mods) for mod in mods: - self.assertTrue(mod.startswith(self.directory), mod) + self.assertStartsWith(mod, self.directory) modcode = importlib.util.cache_from_source(mod) modpath = mod[len(self.directory+os.sep):] _, _, err = script_helper.assert_python_failure(modcode) diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index c4962e35999..99a11e99d56 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -146,4 +146,4 @@ def test_exception_table(self): L1 to L2 -> L2 [0] L2 to L3 -> L3 [1] lasti """) - self.assertTrue(output.getvalue().endswith(exc_table)) + self.assertEndsWith(output.getvalue(), exc_table) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 95bf8fcd25b..a37c4d45f07 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -5,7 +5,7 @@ from concurrent import futures from operator import add from test import support -from test.support import Py_GIL_DISABLED +from test.support import Py_GIL_DISABLED, warnings_helper def mul(x, y): @@ -43,10 +43,12 @@ class ExecutorTest: # Executor.shutdown() and context manager usage is tested by # ExecutorShutdownTest. + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_submit(self): future = self.executor.submit(pow, 2, 8) self.assertEqual(256, future.result()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_submit_keyword(self): future = self.executor.submit(mul, 2, y=8) self.assertEqual(16, future.result()) @@ -57,6 +59,7 @@ def test_submit_keyword(self): with self.assertRaises(TypeError): self.executor.submit(arg=1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map(self): self.assertEqual( list(self.executor.map(pow, range(10), range(10))), @@ -66,6 +69,7 @@ def test_map(self): list(self.executor.map(pow, range(10), range(10), chunksize=3)), list(map(pow, range(10), range(10)))) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_exception(self): i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) self.assertEqual(i.__next__(), (0, 1)) @@ -73,6 +77,7 @@ def test_map_exception(self): with self.assertRaises(ZeroDivisionError): i.__next__() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_resource('walltime') def test_map_timeout(self): results = [] @@ -108,6 +113,7 @@ def test_map_buffersize_value_validation(self): ): self.executor.map(str, range(4), buffersize=buffersize) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_buffersize(self): ints = range(4) for buffersize in (1, 2, len(ints), len(ints) * 2): @@ -115,6 +121,7 @@ def test_map_buffersize(self): res = self.executor.map(str, ints, buffersize=buffersize) self.assertListEqual(list(res), ["0", "1", "2", "3"]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_buffersize_on_multiple_iterables(self): ints = range(4) for buffersize in (1, 2, len(ints), len(ints) * 2): @@ -122,12 +129,14 @@ def test_map_buffersize_on_multiple_iterables(self): res = self.executor.map(add, ints, ints, buffersize=buffersize) self.assertListEqual(list(res), [0, 2, 4, 6]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_buffersize_on_infinite_iterable(self): res = self.executor.map(str, itertools.count(), buffersize=2) self.assertEqual(next(res, None), "0") self.assertEqual(next(res, None), "1") self.assertEqual(next(res, None), "2") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_buffersize_on_multiple_infinite_iterables(self): res = self.executor.map( add, @@ -147,6 +156,7 @@ def test_map_buffersize_without_iterable(self): res = self.executor.map(str, buffersize=2) self.assertIsNone(next(res, None)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_buffersize_when_buffer_is_full(self): ints = iter(range(4)) buffersize = 2 @@ -158,6 +168,7 @@ def test_map_buffersize_when_buffer_is_full(self): msg="should have fetched only `buffersize` elements from `ints`.", ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_shutdown_race_issue12456(self): # Issue #12456: race condition at shutdown where trying to post a # sentinel in the call queue blocks (the queue is full while processes @@ -165,6 +176,7 @@ def test_shutdown_race_issue12456(self): self.executor.map(str, [2] * (self.worker_count + 1)) self.executor.shutdown() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.cpython_only def test_no_stale_references(self): # Issue #16284: check that the executors don't unnecessarily hang onto @@ -209,6 +221,7 @@ def test_max_workers_negative(self): "than 0"): self.executor_type(max_workers=number) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_free_reference(self): # Issue #14406: Result iterator should not keep an internal # reference to result objects. @@ -221,6 +234,7 @@ def test_free_reference(self): if wr() is None: break + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_swallows_falsey_exceptions(self): # see gh-132063: Prevent exceptions that evaluate as falsey # from being ignored. diff --git a/Lib/test/test_concurrent_futures/test_as_completed.py b/Lib/test/test_concurrent_futures/test_as_completed.py index c90b0021d85..31c7bb3ebd8 100644 --- a/Lib/test/test_concurrent_futures/test_as_completed.py +++ b/Lib/test/test_concurrent_futures/test_as_completed.py @@ -7,6 +7,7 @@ CANCELLED_AND_NOTIFIED, FINISHED, Future) from test import support +from test.support import warnings_helper from .util import ( PENDING_FUTURE, RUNNING_FUTURE, @@ -19,6 +20,7 @@ def mul(x, y): class AsCompletedTests: + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_no_timeout(self): future1 = self.executor.submit(mul, 2, 21) future2 = self.executor.submit(mul, 7, 6) @@ -35,6 +37,7 @@ def test_no_timeout(self): future1, future2]), completed) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_future_times_out(self): """Test ``futures.as_completed`` timing out before completing it's final future.""" @@ -62,6 +65,7 @@ def test_future_times_out(self): # Check that ``future`` wasn't completed. self.assertEqual(completed_futures, already_completed) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_duplicate_futures(self): # Issue 20367. Duplicate futures should not raise exceptions or give # duplicate responses. diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py index a465400509d..5cd84f7e430 100644 --- a/Lib/test/test_concurrent_futures/test_deadlock.py +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -10,6 +10,7 @@ from concurrent.futures.process import BrokenProcessPool, _ThreadWakeup from test import support +from test.support import warnings_helper from .util import ( create_executor_tests, setup_module, @@ -111,6 +112,7 @@ def _fail_on_deadlock(self, executor): print(f"\nTraceback:\n {tb}", file=sys.__stderr__) self.fail(f"Executor deadlock:\n\n{tb}") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def _check_error(self, error, func, *args, ignore_stderr=False): # test for deadlock caused by crashes or exiting in a pool self.executor.shutdown(wait=True) @@ -199,6 +201,7 @@ def test_exit_during_result_unpickle_in_result_handler(self): # the result_handler thread self._check_error(BrokenProcessPool, _return_instance, ExitAtUnpickle) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True) def test_shutdown_deadlock(self): # Test that the pool calling shutdown do not cause deadlock @@ -212,6 +215,7 @@ def test_shutdown_deadlock(self): with self.assertRaises(BrokenProcessPool): f.result() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_shutdown_deadlock_pickle(self): # Test that the pool calling shutdown with wait=False does not cause # a deadlock if a task fails at pickle after the shutdown call. @@ -238,6 +242,7 @@ def test_shutdown_deadlock_pickle(self): # dangling threads executor_manager.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True) def test_crash_big_data(self): # Test that there is a clean exception instead of a deadlock when a @@ -254,6 +259,7 @@ def test_crash_big_data(self): executor.shutdown(wait=True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_gh105829_should_not_deadlock_if_wakeup_pipe_full(self): # Issue #105829: The _ExecutorManagerThread wakeup pipe could # fill up and block. See: https://github.com/python/cpython/issues/105829 diff --git a/Lib/test/test_concurrent_futures/test_future.py b/Lib/test/test_concurrent_futures/test_future.py index 4066ea1ee4b..06b11a3bacf 100644 --- a/Lib/test/test_concurrent_futures/test_future.py +++ b/Lib/test/test_concurrent_futures/test_future.py @@ -6,6 +6,7 @@ PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future) from test import support +from test.support import threading_helper from .util import ( PENDING_FUTURE, RUNNING_FUTURE, CANCELLED_FUTURE, @@ -282,6 +283,62 @@ def test_multiple_set_exception(self): self.assertEqual(f.exception(), e) + def test_get_snapshot(self): + """Test the _get_snapshot method for atomic state retrieval.""" + # Test with a pending future + f = Future() + done, cancelled, result, exception = f._get_snapshot() + self.assertFalse(done) + self.assertFalse(cancelled) + self.assertIsNone(result) + self.assertIsNone(exception) + + # Test with a finished future (successful result) + f = Future() + f.set_result(42) + done, cancelled, result, exception = f._get_snapshot() + self.assertTrue(done) + self.assertFalse(cancelled) + self.assertEqual(result, 42) + self.assertIsNone(exception) + + # Test with a finished future (exception) + f = Future() + exc = ValueError("test error") + f.set_exception(exc) + done, cancelled, result, exception = f._get_snapshot() + self.assertTrue(done) + self.assertFalse(cancelled) + self.assertIsNone(result) + self.assertIs(exception, exc) + + # Test with a cancelled future + f = Future() + f.cancel() + done, cancelled, result, exception = f._get_snapshot() + self.assertTrue(done) + self.assertTrue(cancelled) + self.assertIsNone(result) + self.assertIsNone(exception) + + # Test concurrent access (basic thread safety check) + f = Future() + f.set_result(100) + results = [] + + def get_snapshot(): + for _ in range(1000): + snapshot = f._get_snapshot() + results.append(snapshot) + + threads = [threading.Thread(target=get_snapshot) for _ in range(4)] + with threading_helper.start_threads(threads): + pass + # All snapshots should be identical for a finished future + expected = (True, False, 100, None) + for result in results: + self.assertEqual(result, expected) + def setUpModule(): setup_module() diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index df640929309..5ea543bf748 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -11,6 +11,7 @@ from logging.handlers import QueueHandler from test import support +from test.support import warnings_helper from .util import ExecutorMixin, create_executor_tests, setup_module @@ -20,6 +21,10 @@ def init(x): global INITIALIZER_STATUS INITIALIZER_STATUS = x + # InterpreterPoolInitializerTest.test_initializer fails + # if we don't have a LOAD_GLOBAL. (It could be any global.) + # We will address this separately. + INITIALIZER_STATUS def get_init_status(): return INITIALIZER_STATUS @@ -44,6 +49,7 @@ def setUp(self): initargs=('initialized',)) super().setUp() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_initializer(self): futures = [self.executor.submit(get_init_status) for _ in range(self.worker_count)] @@ -70,6 +76,7 @@ def setUp(self): self.executor_kwargs = dict(initializer=init_fail) super().setUp() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_initializer(self): with self._assert_logged('ValueError: error in initializer'): try: diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index f6c62ae4b20..7241fcc4b1e 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -1,36 +1,82 @@ +import _thread import asyncio import contextlib import io import os -import pickle +import subprocess +import sys +import textwrap import time import unittest -from concurrent.futures.interpreter import ( - ExecutionFailed, BrokenInterpreterPool, -) +from concurrent.futures.interpreter import BrokenInterpreterPool +from concurrent import interpreters +from concurrent.interpreters import _queues as queues import _interpreters from test import support +from test.support import os_helper +from test.support import script_helper import test.test_asyncio.utils as testasyncio_utils -from test.support.interpreters import queues from .executor import ExecutorTest, mul from .util import BaseTestCase, InterpreterPoolMixin, setup_module +WINDOWS = sys.platform.startswith('win') + + +@contextlib.contextmanager +def nonblocking(fd): + blocking = os.get_blocking(fd) + if blocking: + os.set_blocking(fd, False) + try: + yield + finally: + if blocking: + os.set_blocking(fd, blocking) + + +def read_file_with_timeout(fd, nbytes, timeout): + with nonblocking(fd): + end = time.time() + timeout + try: + return os.read(fd, nbytes) + except BlockingIOError: + pass + while time.time() < end: + try: + return os.read(fd, nbytes) + except BlockingIOError: + continue + else: + raise TimeoutError('nothing to read') + + +if not WINDOWS: + import select + def read_file_with_timeout(fd, nbytes, timeout): + r, _, _ = select.select([fd], [], [], timeout) + if fd not in r: + raise TimeoutError('nothing to read') + return os.read(fd, nbytes) + + def noop(): pass def write_msg(fd, msg): + import os os.write(fd, msg + b'\0') -def read_msg(fd): +def read_msg(fd, timeout=10.0): msg = b'' - while ch := os.read(fd, 1): - if ch == b'\0': - return msg + ch = read_file_with_timeout(fd, 1, timeout) + while ch != b'\0': msg += ch + ch = os.read(fd, 1) + return msg def get_current_name(): @@ -113,6 +159,38 @@ def test_init_func(self): self.assertEqual(before, b'\0') self.assertEqual(after, msg) + def test_init_with___main___global(self): + # See https://github.com/python/cpython/pull/133957#issuecomment-2927415311. + text = """if True: + from concurrent.futures import InterpreterPoolExecutor + + INITIALIZER_STATUS = 'uninitialized' + + def init(x): + global INITIALIZER_STATUS + INITIALIZER_STATUS = x + INITIALIZER_STATUS + + def get_init_status(): + return INITIALIZER_STATUS + + if __name__ == "__main__": + exe = InterpreterPoolExecutor(initializer=init, + initargs=('initialized',)) + fut = exe.submit(get_init_status) + print(fut.result()) # 'initialized' + exe.shutdown(wait=True) + print(INITIALIZER_STATUS) # 'uninitialized' + """ + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + self.assertEqual(stdout.splitlines(), [ + 'initialized', + 'uninitialized', + ]) + def test_init_closure(self): count = 0 def init1(): @@ -121,10 +199,19 @@ def init2(): nonlocal count count += 1 - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=init1) - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=init2) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=init1) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() + + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=init2) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() def test_init_instance_method(self): class Spam: @@ -132,26 +219,12 @@ def initializer(self): raise NotImplementedError spam = Spam() - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=spam.initializer) - - def test_init_shared(self): - msg = b'eggs' - r, w = self.pipe() - script = f"""if True: - import os - if __name__ != '__main__': - import __main__ - spam = __main__.spam - os.write({w}, spam + b'\\0') - """ - - executor = self.executor_type(shared={'spam': msg}) - fut = executor.submit(exec, script) - fut.result() - after = read_msg(r) - - self.assertEqual(after, msg) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=spam.initializer) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() @unittest.expectedFailure def test_init_exception_in_script(self): @@ -178,8 +251,6 @@ def test_init_exception_in_func(self): stderr = stderr.getvalue() self.assertIn('ExecutionFailed: Exception: spam', stderr) self.assertIn('Uncaught in the interpreter:', stderr) - self.assertIn('The above exception was the direct cause of the following exception:', - stderr) @unittest.expectedFailure def test_submit_script(self): @@ -208,10 +279,14 @@ def task2(): return spam executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(task1) - with self.assertRaises(pickle.PicklingError): - executor.submit(task2) + + fut = executor.submit(task1) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() + + fut = executor.submit(task2) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_local_instance(self): class Spam: @@ -219,8 +294,9 @@ def __init__(self): self.value = True executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(Spam) + fut = executor.submit(Spam) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_instance_method(self): class Spam: @@ -229,8 +305,9 @@ def run(self): spam = Spam() executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(spam.run) + fut = executor.submit(spam.run) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_func_globals(self): executor = self.executor_type() @@ -242,13 +319,14 @@ def test_submit_func_globals(self): @unittest.expectedFailure def test_submit_exception_in_script(self): + # Scripts are not supported currently. fut = self.executor.submit('raise Exception("spam")') with self.assertRaises(Exception) as captured: fut.result() self.assertIs(type(captured.exception), Exception) self.assertEqual(str(captured.exception), 'spam') cause = captured.exception.__cause__ - self.assertIs(type(cause), ExecutionFailed) + self.assertIs(type(cause), interpreters.ExecutionFailed) for attr in ('__name__', '__qualname__', '__module__'): self.assertEqual(getattr(cause.excinfo.type, attr), getattr(Exception, attr)) @@ -261,7 +339,7 @@ def test_submit_exception_in_func(self): self.assertIs(type(captured.exception), Exception) self.assertEqual(str(captured.exception), 'spam') cause = captured.exception.__cause__ - self.assertIs(type(cause), ExecutionFailed) + self.assertIs(type(cause), interpreters.ExecutionFailed) for attr in ('__name__', '__qualname__', '__module__'): self.assertEqual(getattr(cause.excinfo.type, attr), getattr(Exception, attr)) @@ -269,16 +347,93 @@ def test_submit_exception_in_func(self): def test_saturation(self): blocker = queues.create() - executor = self.executor_type(4, shared=dict(blocker=blocker)) + executor = self.executor_type(4) for i in range(15 * executor._max_workers): - executor.submit(exec, 'import __main__; __main__.blocker.get()') - #executor.submit('blocker.get()') + executor.submit(blocker.get) self.assertEqual(len(executor._threads), executor._max_workers) for i in range(15 * executor._max_workers): blocker.put_nowait(None) executor.shutdown(wait=True) + def test_blocking(self): + # There is no guarantee that a worker will be created for every + # submitted task. That's because there's a race between: + # + # * a new worker thread, created when task A was just submitted, + # becoming non-idle when it picks up task A + # * after task B is added to the queue, a new worker thread + # is started only if there are no idle workers + # (the check in ThreadPoolExecutor._adjust_thread_count()) + # + # That means we must not block waiting for *all* tasks to report + # "ready" before we unblock the known-ready workers. + ready = queues.create() + blocker = queues.create() + + def run(taskid, ready, blocker): + # There can't be any globals here. + ready.put_nowait(taskid) + blocker.get() # blocking + + numtasks = 10 + futures = [] + with self.executor_type() as executor: + # Request the jobs. + for i in range(numtasks): + fut = executor.submit(run, i, ready, blocker) + futures.append(fut) + pending = numtasks + while pending > 0: + # Wait for any to be ready. + done = 0 + for _ in range(pending): + try: + ready.get(timeout=1) # blocking + except interpreters.QueueEmpty: + pass + else: + done += 1 + pending -= done + # Unblock the workers. + for _ in range(done): + blocker.put_nowait(None) + + def test_blocking_with_limited_workers(self): + # This is essentially the same as test_blocking, + # but we explicitly force a limited number of workers, + # instead of it happening implicitly sometimes due to a race. + ready = queues.create() + blocker = queues.create() + + def run(taskid, ready, blocker): + # There can't be any globals here. + ready.put_nowait(taskid) + blocker.get() # blocking + + numtasks = 10 + futures = [] + with self.executor_type(4) as executor: + # Request the jobs. + for i in range(numtasks): + fut = executor.submit(run, i, ready, blocker) + futures.append(fut) + pending = numtasks + while pending > 0: + # Wait for any to be ready. + done = 0 + for _ in range(pending): + try: + ready.get(timeout=1) # blocking + except interpreters.QueueEmpty: + pass + else: + done += 1 + pending -= done + # Unblock the workers. + for _ in range(done): + blocker.put_nowait(None) + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_thread_reuse(self): executor = self.executor_type() @@ -289,13 +444,75 @@ def test_idle_thread_reuse(self): executor.shutdown(wait=True) def test_pickle_errors_propagate(self): - # GH-125864: Pickle errors happen before the script tries to execute, so the - # queue used to wait infinitely. - + # GH-125864: Pickle errors happen before the script tries to execute, + # so the queue used to wait infinitely. fut = self.executor.submit(PickleShenanigans(0)) - with self.assertRaisesRegex(RuntimeError, "gotcha"): + expected = interpreters.NotShareableError + with self.assertRaisesRegex(expected, 'args not shareable') as cm: fut.result() + self.assertRegex(str(cm.exception.__cause__), 'unpickled') + def test_no_stale_references(self): + # Weak references don't cross between interpreters. + raise unittest.SkipTest('not applicable') + + def test_free_reference(self): + # Weak references don't cross between interpreters. + raise unittest.SkipTest('not applicable') + + @support.requires_subprocess() + def test_import_interpreter_pool_executor(self): + # Test the import behavior normally if _interpreters is unavailable. + code = textwrap.dedent(""" + import sys + # Set it to None to emulate the case when _interpreter is unavailable. + sys.modules['_interpreters'] = None + from concurrent import futures + + try: + futures.InterpreterPoolExecutor + except AttributeError: + pass + else: + print('AttributeError not raised!', file=sys.stderr) + sys.exit(1) + + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + pass + else: + print('ImportError not raised!', file=sys.stderr) + sys.exit(1) + + from concurrent.futures import * + + if 'InterpreterPoolExecutor' in globals(): + print('InterpreterPoolExecutor should not be imported!', + file=sys.stderr) + sys.exit(1) + """) + + cmd = [sys.executable, '-c', code] + p = subprocess.run(cmd, capture_output=True) + self.assertEqual(p.returncode, 0, p.stderr.decode()) + self.assertEqual(p.stdout.decode(), '') + self.assertEqual(p.stderr.decode(), '') + + def test_thread_name_prefix(self): + self.assertStartsWith(self.executor._thread_name_prefix, + "InterpreterPoolExecutor-") + + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_thread_name_prefix_with_thread_get_name(self): + def get_thread_name(): + import _thread + return _thread._get_name() + + # Some platforms (Linux) are using 16 bytes to store the thread name, + # so only compare the first 15 bytes (without the trailing \n). + self.assertStartsWith(self.executor.submit(get_thread_name).result(), + "InterpreterPoolExecutor-"[:15]) class AsyncioTest(InterpretersMixin, testasyncio_utils.TestCase): @@ -310,7 +527,7 @@ def setUpClass(cls): # tests left a policy in place, just in case. policy = support.maybe_get_event_loop_policy() assert policy is None, policy - cls.addClassCleanup(lambda: asyncio._set_event_loop_policy(None)) + cls.addClassCleanup(lambda: asyncio.events._set_event_loop_policy(None)) def setUp(self): super().setUp() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 3f13a1900a4..731419a48bd 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -9,7 +9,7 @@ from concurrent.futures.process import BrokenProcessPool from test import support -from test.support import hashlib_helper +from test.support import hashlib_helper, warnings_helper from test.test_importlib.metadata.fixtures import parameterize from .executor import ExecutorTest, mul @@ -49,6 +49,7 @@ def test_max_workers_too_large(self): "max_workers must be <= 61"): futures.ProcessPoolExecutor(max_workers=62) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_killed_child(self): # When a child process is abruptly terminated, the whole pool gets # "broken". @@ -61,6 +62,7 @@ def test_killed_child(self): # Submitting other jobs fails as well. self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_chunksize(self): def bad_map(): list(self.executor.map(pow, range(40), range(40), chunksize=-1)) @@ -81,6 +83,7 @@ def bad_map(): def _test_traceback(cls): raise RuntimeError(123) # some comment + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_traceback(self): # We want ensure that the traceback from the child process is # contained in the traceback raised in the main process. @@ -103,6 +106,22 @@ def test_traceback(self): self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + def test_traceback_when_child_process_terminates_abruptly(self): + # gh-139462 enhancement - BrokenProcessPool exceptions + # should describe which process terminated. + exit_code = 99 + with self.executor_type(max_workers=1) as executor: + future = executor.submit(os._exit, exit_code) + with self.assertRaises(BrokenProcessPool) as bpe: + future.result() + + cause = bpe.exception.__cause__ + self.assertIsInstance(cause, futures.process._RemoteTraceback) + self.assertIn( + f"terminated abruptly with exit code {exit_code}", cause.tb + ) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') def test_ressources_gced_in_workers(self): # Ensure that argument for a job are correctly gc-ed after the job @@ -123,6 +142,7 @@ def test_ressources_gced_in_workers(self): mgr.shutdown() mgr.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_saturation(self): executor = self.executor mp_context = self.get_context() @@ -208,6 +228,7 @@ def test_max_tasks_early_shutdown(self): for i, future in enumerate(futures): self.assertEqual(future.result(), mul(i, i)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_python_finalization_error(self): # gh-109047: Catch RuntimeError on thread creation # during Python finalization. @@ -258,6 +279,7 @@ def test_force_shutdown_workers_invalid_op(self): executor._force_shutdown, operation='invalid operation'), + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @parameterize(*FORCE_SHUTDOWN_PARAMS) def test_force_shutdown_workers(self, function_name): manager = self.get_context().Manager() diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 7a4065afd46..c576df1068e 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -6,6 +6,7 @@ from concurrent import futures from test import support +from test.support import warnings_helper from test.support.script_helper import assert_python_ok from .util import ( @@ -49,6 +50,7 @@ def test_interpreter_shutdown(self): self.assertFalse(err) self.assertEqual(out.strip(), b"apple") + @support.force_not_colorized def test_submit_after_interpreter_shutdown(self): # Test the atexit hook for shutdown of worker threads and processes rc, out, err = assert_python_ok('-c', """if 1: @@ -77,12 +79,14 @@ def run_last(): self.assertIn("RuntimeError: cannot schedule new futures", err.decode()) self.assertEqual(out.strip(), b"runtime-error") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_hang_issue12364(self): fs = [self.executor.submit(time.sleep, 0.1) for _ in range(50)] self.executor.shutdown() for f in fs: f.result() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_cancel_futures(self): assert self.worker_count <= 5, "test needs few workers" fs = [self.executor.submit(time.sleep, .1) for _ in range(50)] @@ -128,6 +132,7 @@ def test_hang_gh83386(self): self.assertFalse(err) self.assertEqual(out.strip(), b"apple") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_hang_gh94440(self): """shutdown(wait=True) doesn't hang when a future was submitted and quickly canceled right before shutdown. @@ -171,6 +176,7 @@ def acquire_lock(lock): for t in self.executor._threads: t.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_context_manager_shutdown(self): with futures.ThreadPoolExecutor(max_workers=5) as e: executor = e @@ -180,6 +186,7 @@ def test_context_manager_shutdown(self): for t in executor._threads: t.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_del_shutdown(self): executor = futures.ThreadPoolExecutor(max_workers=5) res = executor.map(abs, range(-5, 5)) @@ -193,6 +200,7 @@ def test_del_shutdown(self): # executor got shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_shutdown_no_wait(self): # Ensure that the executor cleans up the threads when calling # shutdown with wait=False @@ -207,7 +215,7 @@ def test_shutdown_no_wait(self): # executor got shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_thread_names_assigned(self): executor = futures.ThreadPoolExecutor( max_workers=5, thread_name_prefix='SpecialPool') @@ -220,6 +228,7 @@ def test_thread_names_assigned(self): self.assertRegex(t.name, r'^SpecialPool_[0-4]$') t.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_thread_names_default(self): executor = futures.ThreadPoolExecutor(max_workers=5) executor.map(abs, range(-5, 5)) @@ -253,6 +262,7 @@ def test_cancel_futures_wait_false(self): class ProcessPoolShutdownTest(ExecutorShutdownTest): + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_processes_terminate(self): def acquire_lock(lock): lock.acquire() @@ -276,6 +286,7 @@ def acquire_lock(lock): for p in processes.values(): p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_context_manager_shutdown(self): with futures.ProcessPoolExecutor( max_workers=5, mp_context=self.get_context()) as e: @@ -286,6 +297,7 @@ def test_context_manager_shutdown(self): for p in processes.values(): p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_del_shutdown(self): executor = futures.ProcessPoolExecutor( max_workers=5, mp_context=self.get_context()) @@ -308,6 +320,7 @@ def test_del_shutdown(self): # executor got shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_shutdown_no_wait(self): # Ensure that the executor cleans up the processes when calling # shutdown with wait=False @@ -330,6 +343,64 @@ def test_shutdown_no_wait(self): # shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + @classmethod + def _failing_task_gh_132969(cls, n): + raise ValueError("failing task") + + @classmethod + def _good_task_gh_132969(cls, n): + time.sleep(0.1 * n) + return n + + def _run_test_issue_gh_132969(self, max_workers): + # max_workers=2 will repro exception + # max_workers=4 will repro exception and then hang + + # Repro conditions + # max_tasks_per_child=1 + # a task ends abnormally + # shutdown(wait=False) is called + start_method = self.get_context().get_start_method() + if (start_method == "fork" or + (start_method == "forkserver" and sys.platform.startswith("win"))): + self.skipTest(f"Skipping test for {start_method = }") + executor = futures.ProcessPoolExecutor( + max_workers=max_workers, + max_tasks_per_child=1, + mp_context=self.get_context()) + f1 = executor.submit(ProcessPoolShutdownTest._good_task_gh_132969, 1) + f2 = executor.submit(ProcessPoolShutdownTest._failing_task_gh_132969, 2) + f3 = executor.submit(ProcessPoolShutdownTest._good_task_gh_132969, 3) + result = 0 + try: + result += f1.result() + result += f2.result() + result += f3.result() + except ValueError: + # stop processing results upon first exception + pass + + # Ensure that the executor cleans up after called + # shutdown with wait=False + executor_manager_thread = executor._executor_manager_thread + executor.shutdown(wait=False) + time.sleep(0.2) + executor_manager_thread.join() + return result + + def test_shutdown_gh_132969_case_1(self): + # gh-132969: test that exception "object of type 'NoneType' has no len()" + # is not raised when shutdown(wait=False) is called. + result = self._run_test_issue_gh_132969(2) + self.assertEqual(result, 1) + + def test_shutdown_gh_132969_case_2(self): + # gh-132969: test that process does not hang and + # exception "object of type 'NoneType' has no len()" is not raised + # when shutdown(wait=False) is called. + result = self._run_test_issue_gh_132969(4) + self.assertEqual(result, 1) + create_executor_tests(globals(), ProcessPoolShutdownTest, executor_mixins=(ProcessPoolForkMixin, diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 4324241b374..226b7acb64f 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -7,6 +7,7 @@ import unittest from concurrent import futures from test import support +from test.support import warnings_helper from .executor import ExecutorTest, mul from .util import BaseTestCase, ThreadPoolMixin, setup_module @@ -53,6 +54,7 @@ def test_idle_thread_reuse(self): @support.requires_fork() @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') @support.requires_resource('cpu') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_hang_global_shutdown_lock(self): # bpo-45021: _global_shutdown_lock should be reinitialized in the child # process, otherwise it will never exit @@ -68,6 +70,7 @@ def submit(pool): @support.requires_fork() @unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_process_fork_from_a_threadpool(self): # bpo-43944: clear concurrent.futures.thread._threads_queues after fork, # otherwise child process will try to join parent thread diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index cc387883141..b8250cec7ab 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -3,7 +3,7 @@ import unittest from concurrent import futures from test import support -from test.support import threading_helper +from test.support import threading_helper, warnings_helper from .util import ( CANCELLED_FUTURE, CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, @@ -22,6 +22,7 @@ def wait_and_raise(e): class WaitTests: + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_20369(self): # See https://bugs.python.org/issue20369 future = self.executor.submit(mul, 1, 2) @@ -30,7 +31,7 @@ def test_20369(self): self.assertEqual({future}, done) self.assertEqual(set(), not_done) - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_first_completed(self): event = self.create_event() future1 = self.executor.submit(mul, 21, 2) @@ -47,6 +48,7 @@ def test_first_completed(self): event.set() future2.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_first_completed_some_already_completed(self): event = self.create_event() future1 = self.executor.submit(event.wait) @@ -64,6 +66,7 @@ def test_first_completed_some_already_completed(self): event.set() future1.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_first_exception(self): event1 = self.create_event() event2 = self.create_event() @@ -93,6 +96,7 @@ def wait_for_future1(): event2.set() future3.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_first_exception_some_already_complete(self): event = self.create_event() future1 = self.executor.submit(divmod, 21, 0) @@ -114,6 +118,7 @@ def test_first_exception_some_already_complete(self): event.set() future2.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_first_exception_one_already_failed(self): event = self.create_event() future1 = self.executor.submit(event.wait) @@ -129,6 +134,7 @@ def test_first_exception_one_already_failed(self): event.set() future1.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_all_completed(self): future1 = self.executor.submit(divmod, 2, 0) future2 = self.executor.submit(mul, 2, 21) @@ -148,6 +154,7 @@ def test_all_completed(self): future2]), finished) self.assertEqual(set(), pending) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_timeout(self): short_timeout = 0.050 diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index b12940414d9..2a9e55152b8 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -10,7 +10,7 @@ from concurrent.futures.process import _check_system_limits from test import support -from test.support import threading_helper +from test.support import threading_helper, warnings_helper def create_future(state=PENDING, exception=None, result=None): @@ -51,7 +51,8 @@ def setUp(self): max_workers=self.worker_count, mp_context=self.get_context(), **self.executor_kwargs) - self.manager = self.get_context().Manager() + with warnings_helper.ignore_fork_in_thread_deprecation_warnings(): + self.manager = self.get_context().Manager() else: self.executor = self.executor_type( max_workers=self.worker_count, diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 23904d17d32..e7364e18742 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -986,12 +986,12 @@ def test_add_section_default(self): def test_defaults_keyword(self): """bpo-23835 fix for ConfigParser""" - cf = self.newconfig(defaults={1: 2.4}) - self.assertEqual(cf[self.default_section]['1'], '2.4') - self.assertAlmostEqual(cf[self.default_section].getfloat('1'), 2.4) - cf = self.newconfig(defaults={"A": 5.2}) - self.assertEqual(cf[self.default_section]['a'], '5.2') - self.assertAlmostEqual(cf[self.default_section].getfloat('a'), 5.2) + cf = self.newconfig(defaults={1: 2.5}) + self.assertEqual(cf[self.default_section]['1'], '2.5') + self.assertAlmostEqual(cf[self.default_section].getfloat('1'), 2.5) + cf = self.newconfig(defaults={"A": 5.25}) + self.assertEqual(cf[self.default_section]['a'], '5.25') + self.assertAlmostEqual(cf[self.default_section].getfloat('a'), 5.25) class ConfigParserTestCaseNoInterpolation(BasicTestCase, unittest.TestCase): diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index cf651959803..6a3329fa5aa 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -48,23 +48,23 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): return None - self.assertTrue(issubclass(ManagerFromScratch, AbstractContextManager)) + self.assertIsSubclass(ManagerFromScratch, AbstractContextManager) class DefaultEnter(AbstractContextManager): def __exit__(self, *args): super().__exit__(*args) - self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + self.assertIsSubclass(DefaultEnter, AbstractContextManager) class NoEnter(ManagerFromScratch): __enter__ = None - self.assertFalse(issubclass(NoEnter, AbstractContextManager)) + self.assertNotIsSubclass(NoEnter, AbstractContextManager) class NoExit(ManagerFromScratch): __exit__ = None - self.assertFalse(issubclass(NoExit, AbstractContextManager)) + self.assertNotIsSubclass(NoExit, AbstractContextManager) class ContextManagerTestCase(unittest.TestCase): diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 7750186e56a..dcd00720379 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -77,23 +77,23 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_value, traceback): return None - self.assertTrue(issubclass(ManagerFromScratch, AbstractAsyncContextManager)) + self.assertIsSubclass(ManagerFromScratch, AbstractAsyncContextManager) class DefaultEnter(AbstractAsyncContextManager): async def __aexit__(self, *args): await super().__aexit__(*args) - self.assertTrue(issubclass(DefaultEnter, AbstractAsyncContextManager)) + self.assertIsSubclass(DefaultEnter, AbstractAsyncContextManager) class NoneAenter(ManagerFromScratch): __aenter__ = None - self.assertFalse(issubclass(NoneAenter, AbstractAsyncContextManager)) + self.assertNotIsSubclass(NoneAenter, AbstractAsyncContextManager) class NoneAexit(ManagerFromScratch): __aexit__ = None - self.assertFalse(issubclass(NoneAexit, AbstractAsyncContextManager)) + self.assertNotIsSubclass(NoneAexit, AbstractAsyncContextManager) class AsyncContextManagerTestCase(unittest.TestCase): diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index d76341417e9..cfef24727e8 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -19,7 +19,7 @@ class TestCopy(unittest.TestCase): def test_exceptions(self): self.assertIs(copy.Error, copy.error) - self.assertTrue(issubclass(copy.Error, Exception)) + self.assertIsSubclass(copy.Error, Exception) # The copy() method @@ -372,6 +372,7 @@ def test_deepcopy_list(self): self.assertIsNot(x[0], y[0]) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_list(self): x = [] x.append(x) @@ -400,6 +401,7 @@ def test_deepcopy_tuple_of_immutables(self): self.assertIs(x, y) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_tuple(self): x = ([],) x[0].append(x) @@ -418,6 +420,7 @@ def test_deepcopy_dict(self): self.assertIsNot(x["foo"], y["foo"]) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_dict(self): x = {} x['foo'] = x @@ -669,7 +672,7 @@ def __eq__(self, other): def test_reduce_5tuple(self): class C(dict): def __reduce__(self): - return (C, (), self.__dict__, None, self.items()) + return (C, (), self.__dict__, None, iter(self.items())) def __eq__(self, other): return (dict(self) == dict(other) and self.__dict__ == other.__dict__) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 761cb230277..6ad7e7994f3 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -527,7 +527,7 @@ class CoroutineTest(unittest.TestCase): def test_gen_1(self): def gen(): yield - self.assertFalse(hasattr(gen, '__await__')) + self.assertNotHasAttr(gen, '__await__') def test_func_1(self): async def foo(): @@ -1008,7 +1008,7 @@ async def foo(): return (await Awaitable()) with self.assertRaisesRegex( - TypeError, "__await__.*returned non-iterator of type"): + TypeError, "__await__.*must return an iterator, not"): run_async(foo()) @@ -1106,7 +1106,7 @@ async def foo(): return await Awaitable() with self.assertRaisesRegex( - TypeError, r"__await__\(\) returned a coroutine"): + TypeError, r"__await__\(\) must return an iterator, not coroutine"): run_async(foo()) c.close() @@ -1120,7 +1120,7 @@ async def foo(): return await Awaitable() with self.assertRaisesRegex( - TypeError, "__await__.*returned non-iterator of type"): + TypeError, "__await__.*must return an iterator, not"): run_async(foo()) @@ -2307,7 +2307,7 @@ async def f(): pass finally: loop.close() - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) self.assertEqual(buffer, [1, 2, 'MyException']) @@ -2490,7 +2490,7 @@ async def foo(): return (await future) with self.assertRaisesRegex( - TypeError, "__await__.*returned non-iterator of type 'int'"): + TypeError, "__await__.*must return an iterator, not int"): self.assertEqual(foo().send(None), 1) diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 2b7adac4bcc..2f54b3ccb35 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -24,7 +24,7 @@ @support.requires_venv_with_pip() @support.requires_subprocess() @support.requires_resource('cpu') -class TestCPPExt(unittest.TestCase): +class BaseTests: def test_build(self): self.check_build('_testcppext') @@ -34,10 +34,6 @@ def test_build_cpp03(self): # Please ask the C API WG before adding a new C++11-only feature. self.check_build('_testcpp03ext', std='c++03') - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited_cpp03(self): - self.check_build('_test_limited_cpp03ext', std='c++03', limited=True) - @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") def test_build_cpp11(self): self.check_build('_testcpp11ext', std='c++11') @@ -48,10 +44,6 @@ def test_build_cpp11(self): def test_build_cpp14(self): self.check_build('_testcpp14ext', std='c++14') - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited(self): - self.check_build('_testcppext_limited', limited=True) - def check_build(self, extension_name, std=None, limited=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: @@ -111,5 +103,19 @@ def run_cmd(operation, cmd): run_cmd('Import', cmd) +class TestPublicCAPI(BaseTests, unittest.TestCase): + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited_cpp03(self): + self.check_build('_test_limited_cpp03ext', std='c++03', limited=True) + + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_testcppext_limited', limited=True) + + +class TestInteralCAPI(BaseTests, unittest.TestCase): + TEST_INTERNAL_C_API = True + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 5b3571b295b..1affa176088 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -6,8 +6,17 @@ // Always enable assertions #undef NDEBUG +#ifdef TEST_INTERNAL_C_API +# define Py_BUILD_CORE 1 +#endif + #include "Python.h" +#ifdef TEST_INTERNAL_C_API + // gh-135906: Check for compiler warnings in the internal C API +# include "internal/pycore_frame.h" +#endif + #ifndef MODULE_NAME # error "MODULE_NAME macro must be defined" #endif diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index ea1ed64bf7a..98442b106b6 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -47,6 +47,7 @@ def main(): std = os.environ.get("CPYTHON_TEST_CPP_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) cppflags = list(CPPFLAGS) cppflags.append(f'-DMODULE_NAME={module_name}') @@ -82,6 +83,9 @@ def main(): version = sys.hexversion cppflags.append(f'-DPy_LIMITED_API={version:#x}') + if internal: + cppflags.append('-DTEST_INTERNAL_C_API=1') + # On Windows, add PCbuild\amd64\ to include and library directories include_dirs = [] library_dirs = [] diff --git a/Lib/test/test_crossinterp.py b/Lib/test/test_crossinterp.py index 5ac0080db43..2fa0077a09b 100644 --- a/Lib/test/test_crossinterp.py +++ b/Lib/test/test_crossinterp.py @@ -1,10 +1,9 @@ import contextlib -import importlib -import importlib.util import itertools import sys import types import unittest +import warnings from test.support import import_helper @@ -16,13 +15,281 @@ from test import _crossinterp_definitions as defs -BUILTIN_TYPES = [o for _, o in __builtins__.items() - if isinstance(o, type)] -EXCEPTION_TYPES = [cls for cls in BUILTIN_TYPES +@contextlib.contextmanager +def ignore_byteswarning(): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=BytesWarning) + yield + + +# builtin types + +BUILTINS_TYPES = [o for _, o in __builtins__.items() if isinstance(o, type)] +EXCEPTION_TYPES = [cls for cls in BUILTINS_TYPES if issubclass(cls, BaseException)] OTHER_TYPES = [o for n, o in vars(types).items() if (isinstance(o, type) and - n not in ('DynamicClassAttribute', '_GeneratorWrapper'))] + n not in ('DynamicClassAttribute', '_GeneratorWrapper'))] +BUILTIN_TYPES = [ + *BUILTINS_TYPES, + *OTHER_TYPES, +] + +# builtin exceptions + +try: + raise Exception +except Exception as exc: + CAUGHT = exc +EXCEPTIONS_WITH_SPECIAL_SIG = { + BaseExceptionGroup: (lambda msg: (msg, [CAUGHT])), + ExceptionGroup: (lambda msg: (msg, [CAUGHT])), + UnicodeError: (lambda msg: (None, msg, None, None, None)), + UnicodeEncodeError: (lambda msg: ('utf-8', '', 1, 3, msg)), + UnicodeDecodeError: (lambda msg: ('utf-8', b'', 1, 3, msg)), + UnicodeTranslateError: (lambda msg: ('', 1, 3, msg)), +} +BUILTIN_EXCEPTIONS = [ + *(cls(*sig('error!')) for cls, sig in EXCEPTIONS_WITH_SPECIAL_SIG.items()), + *(cls('error!') for cls in EXCEPTION_TYPES + if cls not in EXCEPTIONS_WITH_SPECIAL_SIG), +] + +# other builtin objects + +METHOD = defs.SpamOkay().okay +BUILTIN_METHOD = [].append +METHOD_DESCRIPTOR_WRAPPER = str.join +METHOD_WRAPPER = object().__str__ +WRAPPER_DESCRIPTOR = object.__init__ +BUILTIN_WRAPPERS = { + METHOD: types.MethodType, + BUILTIN_METHOD: types.BuiltinMethodType, + dict.__dict__['fromkeys']: types.ClassMethodDescriptorType, + types.FunctionType.__code__: types.GetSetDescriptorType, + types.FunctionType.__globals__: types.MemberDescriptorType, + METHOD_DESCRIPTOR_WRAPPER: types.MethodDescriptorType, + METHOD_WRAPPER: types.MethodWrapperType, + WRAPPER_DESCRIPTOR: types.WrapperDescriptorType, + staticmethod(defs.SpamOkay.okay): None, + classmethod(defs.SpamOkay.okay): None, + property(defs.SpamOkay.okay): None, +} +BUILTIN_FUNCTIONS = [ + # types.BuiltinFunctionType + len, + sys.is_finalizing, + sys.exit, + _testinternalcapi.get_crossinterp_data, +] +assert 'emptymod' not in sys.modules +with import_helper.ready_to_import('emptymod', ''): + import emptymod as EMPTYMOD +MODULES = [ + sys, + defs, + unittest, + EMPTYMOD, +] +OBJECT = object() +EXCEPTION = Exception() +LAMBDA = (lambda: None) +BUILTIN_SIMPLE = [ + OBJECT, + # singletons + None, + True, + False, + Ellipsis, + NotImplemented, + # bytes + *(i.to_bytes(2, 'little', signed=True) + for i in range(-1, 258)), + # str + 'hello world', + '你好世界', + '', + # int + sys.maxsize + 1, + sys.maxsize, + -sys.maxsize - 1, + -sys.maxsize - 2, + *range(-1, 258), + 2**1000, + # float + 0.0, + 1.1, + -1.0, + 0.12345678, + -0.12345678, +] +TUPLE_EXCEPTION = (0, 1.0, EXCEPTION) +TUPLE_OBJECT = (0, 1.0, OBJECT) +TUPLE_NESTED_EXCEPTION = (0, 1.0, (EXCEPTION,)) +TUPLE_NESTED_OBJECT = (0, 1.0, (OBJECT,)) +MEMORYVIEW_EMPTY = memoryview(b'') +MEMORYVIEW_NOT_EMPTY = memoryview(b'spam'*42) +MAPPING_PROXY_EMPTY = types.MappingProxyType({}) +BUILTIN_CONTAINERS = [ + # tuple (flat) + (), + (1,), + ("hello", "world", ), + (1, True, "hello"), + TUPLE_EXCEPTION, + TUPLE_OBJECT, + # tuple (nested) + ((1,),), + ((1, 2), (3, 4)), + ((1, 2), (3, 4), (5, 6)), + TUPLE_NESTED_EXCEPTION, + TUPLE_NESTED_OBJECT, + # buffer + MEMORYVIEW_EMPTY, + MEMORYVIEW_NOT_EMPTY, + # list + [], + [1, 2, 3], + [[1], (2,), {3: 4}], + # dict + {}, + {1: 7, 2: 8, 3: 9}, + {1: [1], 2: (2,), 3: {3: 4}}, + # set + set(), + {1, 2, 3}, + {frozenset({1}), (2,)}, + # frozenset + frozenset([]), + frozenset({frozenset({1}), (2,)}), + # bytearray + bytearray(b''), + # other + MAPPING_PROXY_EMPTY, + types.SimpleNamespace(), +] +ns = {} +exec(""" +try: + raise Exception +except Exception as exc: + TRACEBACK = exc.__traceback__ + FRAME = TRACEBACK.tb_frame +""", ns, ns) +BUILTIN_OTHER = [ + # types.CellType + types.CellType(), + # types.FrameType + ns['FRAME'], + # types.TracebackType + ns['TRACEBACK'], +] +del ns + +# user-defined objects + +USER_TOP_INSTANCES = [c(*a) for c, a in defs.TOP_CLASSES.items()] +USER_NESTED_INSTANCES = [c(*a) for c, a in defs.NESTED_CLASSES.items()] +USER_INSTANCES = [ + *USER_TOP_INSTANCES, + *USER_NESTED_INSTANCES, +] +USER_EXCEPTIONS = [ + defs.MimimalError('error!'), +] + +# shareable objects + +TUPLES_WITHOUT_EQUALITY = [ + TUPLE_EXCEPTION, + TUPLE_OBJECT, + TUPLE_NESTED_EXCEPTION, + TUPLE_NESTED_OBJECT, +] +_UNSHAREABLE_SIMPLE = [ + Ellipsis, + NotImplemented, + OBJECT, + sys.maxsize + 1, + -sys.maxsize - 2, + 2**1000, +] +with ignore_byteswarning(): + _SHAREABLE_SIMPLE = [o for o in BUILTIN_SIMPLE + if o not in _UNSHAREABLE_SIMPLE] + _SHAREABLE_CONTAINERS = [ + *(o for o in BUILTIN_CONTAINERS if type(o) is memoryview), + *(o for o in BUILTIN_CONTAINERS + if type(o) is tuple and o not in TUPLES_WITHOUT_EQUALITY), + ] + _UNSHAREABLE_CONTAINERS = [o for o in BUILTIN_CONTAINERS + if o not in _SHAREABLE_CONTAINERS] +SHAREABLE = [ + *_SHAREABLE_SIMPLE, + *_SHAREABLE_CONTAINERS, +] +NOT_SHAREABLE = [ + *_UNSHAREABLE_SIMPLE, + *_UNSHAREABLE_CONTAINERS, + *BUILTIN_TYPES, + *BUILTIN_WRAPPERS, + *BUILTIN_EXCEPTIONS, + *BUILTIN_FUNCTIONS, + *MODULES, + *BUILTIN_OTHER, + # types.CodeType + *(f.__code__ for f in defs.FUNCTIONS), + *(f.__code__ for f in defs.FUNCTION_LIKE), + # types.FunctionType + *defs.FUNCTIONS, + defs.SpamOkay.okay, + LAMBDA, + *defs.FUNCTION_LIKE, + # coroutines and generators + *defs.FUNCTION_LIKE_APPLIED, + # user classes + *defs.CLASSES, + *USER_INSTANCES, + # user exceptions + *USER_EXCEPTIONS, +] + +# pickleable objects + +PICKLEABLE = [ + *BUILTIN_SIMPLE, + *(o for o in BUILTIN_CONTAINERS if o not in [ + MEMORYVIEW_EMPTY, + MEMORYVIEW_NOT_EMPTY, + MAPPING_PROXY_EMPTY, + ] or type(o) is dict), + *BUILTINS_TYPES, + *BUILTIN_EXCEPTIONS, + *BUILTIN_FUNCTIONS, + *defs.TOP_FUNCTIONS, + defs.SpamOkay.okay, + *defs.FUNCTION_LIKE, + *defs.TOP_CLASSES, + *USER_TOP_INSTANCES, + *USER_EXCEPTIONS, + # from OTHER_TYPES + types.NoneType, + types.EllipsisType, + types.NotImplementedType, + types.GenericAlias, + types.UnionType, + types.SimpleNamespace, + # from BUILTIN_WRAPPERS + METHOD, + BUILTIN_METHOD, + METHOD_DESCRIPTOR_WRAPPER, + METHOD_WRAPPER, + WRAPPER_DESCRIPTOR, +] +assert not any(isinstance(o, types.MappingProxyType) for o in PICKLEABLE) + + +# helpers DEFS = defs with open(code_defs.__file__) as infile: @@ -111,6 +378,77 @@ class _GetXIDataTests(unittest.TestCase): MODE = None + def assert_functions_equal(self, func1, func2): + assert type(func1) is types.FunctionType, repr(func1) + assert type(func2) is types.FunctionType, repr(func2) + self.assertEqual(func1.__name__, func2.__name__) + self.assertEqual(func1.__code__, func2.__code__) + self.assertEqual(func1.__defaults__, func2.__defaults__) + self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__) + # We don't worry about __globals__ for now. + + def assert_exc_args_equal(self, exc1, exc2): + args1 = exc1.args + args2 = exc2.args + if isinstance(exc1, ExceptionGroup): + self.assertIs(type(args1), type(args2)) + self.assertEqual(len(args1), 2) + self.assertEqual(len(args1), len(args2)) + self.assertEqual(args1[0], args2[0]) + group1 = args1[1] + group2 = args2[1] + self.assertEqual(len(group1), len(group2)) + for grouped1, grouped2 in zip(group1, group2): + # Currently the "extra" attrs are not preserved + # (via __reduce__). + self.assertIs(type(exc1), type(exc2)) + self.assert_exc_equal(grouped1, grouped2) + else: + self.assertEqual(args1, args2) + + def assert_exc_equal(self, exc1, exc2): + self.assertIs(type(exc1), type(exc2)) + + if type(exc1).__eq__ is not object.__eq__: + self.assertEqual(exc1, exc2) + + self.assert_exc_args_equal(exc1, exc2) + # XXX For now we do not preserve tracebacks. + if exc1.__traceback__ is not None: + self.assertEqual(exc1.__traceback__, exc2.__traceback__) + self.assertEqual( + getattr(exc1, '__notes__', None), + getattr(exc2, '__notes__', None), + ) + # We assume there are no cycles. + if exc1.__cause__ is None: + self.assertIs(exc1.__cause__, exc2.__cause__) + else: + self.assert_exc_equal(exc1.__cause__, exc2.__cause__) + if exc1.__context__ is None: + self.assertIs(exc1.__context__, exc2.__context__) + else: + self.assert_exc_equal(exc1.__context__, exc2.__context__) + + def assert_equal_or_equalish(self, obj, expected): + cls = type(expected) + if cls.__eq__ is not object.__eq__: + self.assertEqual(obj, expected) + elif cls is types.FunctionType: + self.assert_functions_equal(obj, expected) + elif isinstance(expected, BaseException): + self.assert_exc_equal(obj, expected) + elif cls is types.MethodType: + raise NotImplementedError(cls) + elif cls is types.BuiltinMethodType: + raise NotImplementedError(cls) + elif cls is types.MethodWrapperType: + raise NotImplementedError(cls) + elif cls.__bases__ == (object,): + self.assertEqual(obj.__dict__, expected.__dict__) + else: + raise NotImplementedError(cls) + def get_xidata(self, obj, *, mode=None): mode = self._resolve_mode(mode) return _testinternalcapi.get_crossinterp_data(obj, mode) @@ -126,35 +464,37 @@ def _get_roundtrip(self, obj, mode): def assert_roundtrip_identical(self, values, *, mode=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIs(got, obj) def assert_roundtrip_equal(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) - self.assertEqual(got, obj) + if got is obj: + continue self.assertIs(type(got), type(obj) if expecttype is None else expecttype) + self.assert_equal_or_equalish(got, obj) def assert_roundtrip_equal_not_identical(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIsNot(got, obj) self.assertIs(type(got), type(obj) if expecttype is None else expecttype) - self.assertEqual(got, obj) + self.assert_equal_or_equalish(got, obj) def assert_roundtrip_not_equal(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIsNot(got, obj) self.assertIs(type(got), @@ -164,7 +504,7 @@ def assert_roundtrip_not_equal(self, values, *, def assert_not_shareable(self, values, exctype=None, *, mode=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): with self.assertRaises(NotShareableError) as cm: _testinternalcapi.get_crossinterp_data(obj, mode) if exctype is not None: @@ -182,49 +522,26 @@ class PickleTests(_GetXIDataTests): MODE = 'pickle' def test_shareable(self): - self.assert_roundtrip_equal([ - # singletons - None, - True, - False, - # bytes - *(i.to_bytes(2, 'little', signed=True) - for i in range(-1, 258)), - # str - 'hello world', - '你好世界', - '', - # int - sys.maxsize, - -sys.maxsize - 1, - *range(-1, 258), - # float - 0.0, - 1.1, - -1.0, - 0.12345678, - -0.12345678, - # tuple - (), - (1,), - ("hello", "world", ), - (1, True, "hello"), - ((1,),), - ((1, 2), (3, 4)), - ((1, 2), (3, 4), (5, 6)), - ]) - # not shareable using xidata - self.assert_roundtrip_equal([ - # int - sys.maxsize + 1, - -sys.maxsize - 2, - 2**1000, - # tuple - (0, 1.0, []), - (0, 1.0, {}), - (0, 1.0, ([],)), - (0, 1.0, ({},)), - ]) + with ignore_byteswarning(): + for obj in SHAREABLE: + if obj in PICKLEABLE: + self.assert_roundtrip_equal([obj]) + else: + self.assert_not_shareable([obj]) + + def test_not_shareable(self): + with ignore_byteswarning(): + for obj in NOT_SHAREABLE: + if type(obj) is types.MappingProxyType: + self.assert_not_shareable([obj]) + elif obj in PICKLEABLE: + with self.subTest(repr(obj)): + # We don't worry about checking the actual value. + # The other tests should cover that well enough. + got = self.get_roundtrip(obj) + self.assertIs(type(got), type(obj)) + else: + self.assert_not_shareable([obj]) def test_list(self): self.assert_roundtrip_equal_not_identical([ @@ -266,7 +583,7 @@ def assert_class_defs_same(self, defs): if cls not in defs.CLASSES_WITHOUT_EQUALITY: continue instances.append(cls(*args)) - self.assert_roundtrip_not_equal(instances) + self.assert_roundtrip_equal(instances) def assert_class_defs_other_pickle(self, defs, mod): # Pickle relative to a different module than the original. @@ -286,7 +603,7 @@ def assert_class_defs_other_unpickle(self, defs, mod, *, fail=False): instances = [] for cls, args in defs.TOP_CLASSES.items(): - with self.subTest(cls): + with self.subTest(repr(cls)): setattr(mod, cls.__name__, cls) xid = self.get_xidata(cls) inst = cls(*args) @@ -295,7 +612,7 @@ def assert_class_defs_other_unpickle(self, defs, mod, *, fail=False): (cls, xid, inst, instxid)) for cls, xid, inst, instxid in instances: - with self.subTest(cls): + with self.subTest(repr(cls)): delattr(mod, cls.__name__) if fail: with self.assertRaises(NotShareableError): @@ -403,13 +720,13 @@ def assert_func_defs_same(self, defs): def assert_func_defs_other_pickle(self, defs, mod): # Pickle relative to a different module than the original. for func in defs.TOP_FUNCTIONS: - assert not hasattr(mod, func.__name__), (cls, getattr(mod, func.__name__)) + assert not hasattr(mod, func.__name__), (getattr(mod, func.__name__),) self.assert_not_shareable(defs.TOP_FUNCTIONS) def assert_func_defs_other_unpickle(self, defs, mod, *, fail=False): # Unpickle relative to a different module than the original. for func in defs.TOP_FUNCTIONS: - assert not hasattr(mod, func.__name__), (cls, getattr(mod, func.__name__)) + assert not hasattr(mod, func.__name__), (getattr(mod, func.__name__),) captured = [] for func in defs.TOP_FUNCTIONS: @@ -434,7 +751,7 @@ def assert_func_defs_not_shareable(self, defs): self.assert_not_shareable(defs.TOP_FUNCTIONS) def test_user_function_normal(self): -# self.assert_roundtrip_equal(defs.TOP_FUNCTIONS) + self.assert_roundtrip_equal(defs.TOP_FUNCTIONS) self.assert_func_defs_same(defs) def test_user_func_in___main__(self): @@ -505,7 +822,7 @@ def test_nested_function(self): # exceptions def test_user_exception_normal(self): - self.assert_roundtrip_not_equal([ + self.assert_roundtrip_equal([ defs.MimimalError('error!'), ]) self.assert_roundtrip_equal_not_identical([ @@ -521,7 +838,7 @@ def test_builtin_exception(self): special = { BaseExceptionGroup: (msg, [caught]), ExceptionGroup: (msg, [caught]), -# UnicodeError: (None, msg, None, None, None), + UnicodeError: (None, msg, None, None, None), UnicodeEncodeError: ('utf-8', '', 1, 3, msg), UnicodeDecodeError: ('utf-8', b'', 1, 3, msg), UnicodeTranslateError: ('', 1, 3, msg), @@ -531,7 +848,7 @@ def test_builtin_exception(self): args = special.get(cls) or (msg,) exceptions.append(cls(*args)) - self.assert_roundtrip_not_equal(exceptions) + self.assert_roundtrip_equal(exceptions) class MarshalTests(_GetXIDataTests): @@ -576,7 +893,7 @@ def test_simple_builtin_objects(self): '', ]) self.assert_not_shareable([ - object(), + OBJECT, types.SimpleNamespace(), ]) @@ -647,10 +964,7 @@ def test_builtin_type(self): shareable = [ StopIteration, ] - types = [ - *BUILTIN_TYPES, - *OTHER_TYPES, - ] + types = BUILTIN_TYPES self.assert_not_shareable(cls for cls in types if cls not in shareable) self.assert_roundtrip_identical(cls for cls in types @@ -758,10 +1072,203 @@ def test_other_objects(self): ]) +class ShareableFuncTests(_GetXIDataTests): + + MODE = 'func' + + def test_stateless(self): + self.assert_roundtrip_equal([ + *defs.STATELESS_FUNCTIONS, + # Generators can be stateless too. + *defs.FUNCTION_LIKE, + ]) + + def test_not_stateless(self): + self.assert_not_shareable([ + *(f for f in defs.FUNCTIONS + if f not in defs.STATELESS_FUNCTIONS), + ]) + + def test_other_objects(self): + self.assert_not_shareable([ + None, + True, + False, + Ellipsis, + NotImplemented, + 9999, + 'spam', + b'spam', + (), + [], + {}, + object(), + ]) + + +class PureShareableScriptTests(_GetXIDataTests): + + MODE = 'script-pure' + + VALID_SCRIPTS = [ + '', + 'spam', + '# a comment', + 'print("spam")', + 'raise Exception("spam")', + """if True: + do_something() + """, + """if True: + def spam(x): + return x + class Spam: + def eggs(self): + return 42 + x = Spam().eggs() + raise ValueError(spam(x)) + """, + ] + INVALID_SCRIPTS = [ + ' pass', # IndentationError + '----', # SyntaxError + """if True: + def spam(): + # no body + spam() + """, # IndentationError + ] + + def test_valid_str(self): + self.assert_roundtrip_not_equal([ + *self.VALID_SCRIPTS, + ], expecttype=types.CodeType) + + def test_invalid_str(self): + self.assert_not_shareable([ + *self.INVALID_SCRIPTS, + ]) + + def test_valid_bytes(self): + self.assert_roundtrip_not_equal([ + *(s.encode('utf8') for s in self.VALID_SCRIPTS), + ], expecttype=types.CodeType) + + def test_invalid_bytes(self): + self.assert_not_shareable([ + *(s.encode('utf8') for s in self.INVALID_SCRIPTS), + ]) + + def test_pure_script_code(self): + self.assert_roundtrip_equal_not_identical([ + *(f.__code__ for f in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_impure_script_code(self): + self.assert_not_shareable([ + *(f.__code__ for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_other_code(self): + self.assert_not_shareable([ + *(f.__code__ for f in defs.FUNCTIONS + if f not in defs.SCRIPT_FUNCTIONS), + *(f.__code__ for f in defs.FUNCTION_LIKE), + ]) + + def test_pure_script_function(self): + self.assert_roundtrip_not_equal([ + *defs.PURE_SCRIPT_FUNCTIONS, + ], expecttype=types.CodeType) + + def test_impure_script_function(self): + self.assert_not_shareable([ + *(f for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_other_function(self): + self.assert_not_shareable([ + *(f for f in defs.FUNCTIONS + if f not in defs.SCRIPT_FUNCTIONS), + *defs.FUNCTION_LIKE, + ]) + + def test_other_objects(self): + self.assert_not_shareable([ + None, + True, + False, + Ellipsis, + NotImplemented, + (), + [], + {}, + object(), + ]) + + +class ShareableScriptTests(PureShareableScriptTests): + + MODE = 'script' + + def test_impure_script_code(self): + self.assert_roundtrip_equal_not_identical([ + *(f.__code__ for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_impure_script_function(self): + self.assert_roundtrip_not_equal([ + *(f for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ], expecttype=types.CodeType) + + +class ShareableFallbackTests(_GetXIDataTests): + + MODE = 'fallback' + + def test_shareable(self): + self.assert_roundtrip_equal(SHAREABLE) + + def test_not_shareable(self): + okay = [ + *PICKLEABLE, + *defs.STATELESS_FUNCTIONS, + LAMBDA, + ] + ignored = [ + *TUPLES_WITHOUT_EQUALITY, + OBJECT, + METHOD, + BUILTIN_METHOD, + METHOD_WRAPPER, + ] + with ignore_byteswarning(): + self.assert_roundtrip_equal([ + *(o for o in NOT_SHAREABLE + if o in okay and o not in ignored + and o is not MAPPING_PROXY_EMPTY), + ]) + self.assert_roundtrip_not_equal([ + *(o for o in NOT_SHAREABLE + if o in ignored and o is not MAPPING_PROXY_EMPTY), + ]) + self.assert_not_shareable([ + *(o for o in NOT_SHAREABLE if o not in okay), + MAPPING_PROXY_EMPTY, + ]) + + class ShareableTypeTests(_GetXIDataTests): MODE = 'xidata' + def test_shareable(self): + self.assert_roundtrip_equal(SHAREABLE) + def test_singletons(self): self.assert_roundtrip_identical([ None, @@ -829,8 +1336,8 @@ def test_tuple(self): def test_tuples_containing_non_shareable_types(self): non_shareables = [ - Exception(), - object(), + EXCEPTION, + OBJECT, ] for s in non_shareables: value = tuple([0, 1.0, s]) @@ -845,6 +1352,9 @@ def test_tuples_containing_non_shareable_types(self): # The rest are not shareable. + def test_not_shareable(self): + self.assert_not_shareable(NOT_SHAREABLE) + def test_object(self): self.assert_not_shareable([ object(), @@ -861,12 +1371,12 @@ def test_function_object(self): for func in defs.FUNCTIONS: assert type(func) is types.FunctionType, func assert type(defs.SpamOkay.okay) is types.FunctionType, func - assert type(lambda: None) is types.LambdaType + assert type(LAMBDA) is types.LambdaType self.assert_not_shareable([ *defs.FUNCTIONS, defs.SpamOkay.okay, - (lambda: None), + LAMBDA, ]) def test_builtin_function(self): @@ -931,10 +1441,7 @@ def test_class(self): self.assert_not_shareable(instances) def test_builtin_type(self): - self.assert_not_shareable([ - *BUILTIN_TYPES, - *OTHER_TYPES, - ]) + self.assert_not_shareable(BUILTIN_TYPES) def test_exception(self): self.assert_not_shareable([ @@ -973,7 +1480,7 @@ def test_builtin_objects(self): """, ns, ns) self.assert_not_shareable([ - types.MappingProxyType({}), + MAPPING_PROXY_EMPTY, types.SimpleNamespace(), # types.CellType types.CellType(), diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 9aace57633b..df79840088a 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -918,6 +918,14 @@ def test_dict_reader_fieldnames_accepts_list(self): reader = csv.DictReader(f, fieldnames) self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_reader_set_fieldnames(self): + fieldnames = ["a", "b", "c"] + f = StringIO() + reader = csv.DictReader(f) + self.assertIsNone(reader.fieldnames) + reader.fieldnames = fieldnames + self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_writer_fieldnames_rejects_iter(self): fieldnames = ["a", "b", "c"] f = StringIO() @@ -933,6 +941,7 @@ def test_dict_writer_fieldnames_accepts_list(self): def test_dict_reader_fieldnames_is_optional(self): f = StringIO() reader = csv.DictReader(f, fieldnames=None) + self.assertIsNone(reader.fieldnames) def test_read_dict_fields(self): with TemporaryFile("w+", encoding="utf-8") as fileobj: @@ -1122,19 +1131,22 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be a 1-character string') + '"quotechar" must be a unicode character or None, ' + 'not a string of length 0') mydialect.quotechar = "''" with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be a 1-character string') + '"quotechar" must be a unicode character or None, ' + 'not a string of length 2') mydialect.quotechar = 4 with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be string or None, not int') + '"quotechar" must be a unicode character or None, ' + 'not int') def test_delimiter(self): class mydialect(csv.Dialect): @@ -1151,31 +1163,32 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be a 1-character string') + '"delimiter" must be a unicode character, ' + 'not a string of length 3') mydialect.delimiter = "" with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be a 1-character string') + '"delimiter" must be a unicode character, not a string of length 0') mydialect.delimiter = b"," with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not bytes') + '"delimiter" must be a unicode character, not bytes') mydialect.delimiter = 4 with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not int') + '"delimiter" must be a unicode character, not int') mydialect.delimiter = None with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not NoneType') + '"delimiter" must be a unicode character, not NoneType') def test_escapechar(self): class mydialect(csv.Dialect): @@ -1189,20 +1202,32 @@ class mydialect(csv.Dialect): self.assertEqual(d.escapechar, "\\") mydialect.escapechar = "" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not a string of length 0') mydialect.escapechar = "**" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not a string of length 2') mydialect.escapechar = b"*" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not bytes'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not bytes') mydialect.escapechar = 4 - with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not int'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not int') def test_lineterminator(self): class mydialect(csv.Dialect): @@ -1223,7 +1248,13 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"lineterminator" must be a string') + '"lineterminator" must be a string, not int') + + mydialect.lineterminator = None + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + '"lineterminator" must be a string, not NoneType') def test_invalid_chars(self): def create_invalid(field_name, value, **kwargs): @@ -1331,6 +1362,19 @@ class TestSniffer(unittest.TestCase): ghi\0jkl """ + sample15 = "\n\n\n" + sample16 = "abc\ndef\nghi" + + sample17 = ["letter,offset"] + sample17.extend(f"{chr(ord('a') + i)},{i}" for i in range(20)) + sample17.append("v,twenty_one") # 'u' was skipped + sample17 = '\n'.join(sample17) + + sample18 = ["letter,offset"] + sample18.extend(f"{chr(ord('a') + i)},{i}" for i in range(21)) + sample18.append("v,twenty_one") # 'u' was not skipped + sample18 = '\n'.join(sample18) + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1352,6 +1396,11 @@ def test_has_header_regex_special_delimiter(self): self.assertIs(sniffer.has_header(self.sample8), False) self.assertIs(sniffer.has_header(self.header2 + self.sample8), True) + def test_has_header_checks_20_rows(self): + sniffer = csv.Sniffer() + self.assertFalse(sniffer.has_header(self.sample17)) + self.assertTrue(sniffer.has_header(self.sample18)) + def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): @@ -1401,6 +1450,10 @@ def test_delimiters(self): self.assertEqual(dialect.quotechar, "'") dialect = sniffer.sniff(self.sample14) self.assertEqual(dialect.delimiter, '\0') + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample15) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample16) def test_doublequote(self): sniffer = csv.Sniffer() @@ -1415,6 +1468,56 @@ def test_doublequote(self): dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + def test_guess_delimiter_crlf_not_chosen(self): + # Ensure that we pick the real delimiter ("|") over "\r" in a tie. + sniffer = csv.Sniffer() + sample = "a|b\r\nc|d\r\ne|f\r\n" + self.assertEqual(sniffer.sniff(sample).delimiter, "|") + self.assertNotEqual(sniffer.sniff(sample).delimiter, "\r") + + def test_zero_mode_tie_order_independence(self): + sniffer = csv.Sniffer() + # ":" appears in half the rows (1, 0, 1, 0) - a tie between + # 0 and 1 per line. + # "," appears once every row (true delimiter). + # + # Even if the zero-frequency bucket is appended vs. inserted, the tie + # yields an adjusted score of 0, so ":" should not be promoted and + # "," must be selected. + sample = ( + "a,b:c\n" + "d,e\n" + "f,g:c\n" + "h,i\n" + ) + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ",") + + def test_zero_mode_tie_order_comma_first(self): + sniffer = csv.Sniffer() + pattern = ( + "a,b\n" + "c:d\n" + "e,f\n" + "g:h\n" + ) + sample = pattern * 10 + with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"): + sniffer.sniff(sample) + + def test_zero_mode_tie_order_colon_first(self): + sniffer = csv.Sniffer() + pattern = ( + "a:b\n" + "c,d\n" + "e:f\n" + "g,h\n" + ) + sample = pattern * 10 + with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"): + sniffer.sniff(sample) + + class NUL: def write(s, *args): pass @@ -1581,5 +1684,16 @@ def test_disallow_instantiation(self): with self.subTest(tp=tp): check_disallow_instantiation(self, tp) + +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(csv, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/_support.py b/Lib/test/test_ctypes/_support.py index 946d654a19a..700657a4e41 100644 --- a/Lib/test/test_ctypes/_support.py +++ b/Lib/test/test_ctypes/_support.py @@ -3,7 +3,6 @@ import ctypes from _ctypes import Structure, Union, _Pointer, Array, _SimpleCData, CFuncPtr import sys -from test import support _CData = Structure.__base__ diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index ea5951603f9..f14e1aa32e1 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -1,5 +1,4 @@ import binascii -import ctypes import math import struct import sys diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 8af34e62b94..5658234f9ec 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -32,6 +32,7 @@ @unittest.skipUnless(sys.platform.startswith('linux'), 'test requires GNU IFUNC support') +@unittest.skipIf(test.support.linked_to_musl(), "Requires glibc") class TestNullDlsym(unittest.TestCase): """GH-126554: Ensure that we catch NULL dlsym return values diff --git a/Lib/test/test_ctypes/test_find.py b/Lib/test/test_ctypes/test_find.py index 3bd41a0e435..8bc84c3d2ef 100644 --- a/Lib/test/test_ctypes/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -153,5 +153,73 @@ def test_find(self): self.assertIsNone(find_library(name)) +@unittest.skipUnless(test.support.is_emscripten, + 'Test only valid for Emscripten') +class FindLibraryEmscripten(unittest.TestCase): + @classmethod + def setUpClass(cls): + import tempfile + + # A very simple wasm module + # In WAT format: (module) + cls.wasm_module = b'\x00asm\x01\x00\x00\x00\x00\x08\x04name\x02\x01\x00' + + cls.non_wasm_content = b'This is not a WASM file' + + cls.temp_dir = tempfile.mkdtemp() + cls.libdummy_so_path = os.path.join(cls.temp_dir, 'libdummy.so') + with open(cls.libdummy_so_path, 'wb') as f: + f.write(cls.wasm_module) + + cls.libother_wasm_path = os.path.join(cls.temp_dir, 'libother.wasm') + with open(cls.libother_wasm_path, 'wb') as f: + f.write(cls.wasm_module) + + cls.libnowasm_so_path = os.path.join(cls.temp_dir, 'libnowasm.so') + with open(cls.libnowasm_so_path, 'wb') as f: + f.write(cls.non_wasm_content) + + @classmethod + def tearDownClass(cls): + import shutil + shutil.rmtree(cls.temp_dir) + + def test_find_wasm_file_with_so_extension(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('dummy') + self.assertEqual(result, self.libdummy_so_path) + def test_find_wasm_file_with_wasm_extension(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('other') + self.assertEqual(result, self.libother_wasm_path) + + def test_ignore_non_wasm_file(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('nowasm') + self.assertIsNone(result) + + def test_find_nothing_without_ld_library_path(self): + with os_helper.EnvironmentVarGuard() as env: + if 'LD_LIBRARY_PATH' in env: + del env['LD_LIBRARY_PATH'] + result = find_library('dummy') + self.assertIsNone(result) + result = find_library('other') + self.assertIsNone(result) + + def test_find_nothing_with_wrong_ld_library_path(self): + import tempfile + with tempfile.TemporaryDirectory() as empty_dir: + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', empty_dir) + result = find_library('dummy') + self.assertIsNone(result) + result = find_library('other') + self.assertIsNone(result) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index aa448fad5bb..1cb46a82701 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -10,7 +10,7 @@ """ import unittest -from test.support import import_helper, verbose +from test.support import import_helper import re from dataclasses import dataclass from functools import cached_property diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py index fefdfe9102e..3189fcd1bd1 100644 --- a/Lib/test/test_ctypes/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -1,6 +1,5 @@ import ctypes import unittest -import warnings from ctypes import Structure, POINTER, pointer, c_char_p # String-based "incomplete pointers" were implemented in ctypes 0.6.3 (2003, when @@ -21,9 +20,7 @@ class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - ctypes.SetPointerType(lpcell, cell) + lpcell.set_type(cell) self.assertIs(POINTER(cell), lpcell) @@ -50,10 +47,9 @@ class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] - with self.assertWarns(DeprecationWarning): - ctypes.SetPointerType(lpcell, cell) - + lpcell.set_type(cell) self.assertIs(POINTER(cell), lpcell) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 13ed813ad98..3b8332fbb30 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -100,6 +100,12 @@ def test_load_ordinal_functions(self): self.assertRaises(AttributeError, dll.__getitem__, 1234) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') + def test_load_without_name_and_with_handle(self): + handle = ctypes.windll.kernel32._handle + lib = ctypes.WinDLL(name=None, handle=handle) + self.assertIs(handle, lib._handle) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_A(self): # On winXP 64-bit, advapi32 loads at an address that does diff --git a/Lib/test/test_ctypes/test_macholib.py b/Lib/test/test_ctypes/test_macholib.py index 9d906179956..9a5e18ed589 100644 --- a/Lib/test/test_ctypes/test_macholib.py +++ b/Lib/test/test_ctypes/test_macholib.py @@ -108,5 +108,17 @@ def test_framework_info(self): d('P', 'F.framework/Versions/A/F_debug', 'F', 'A', 'debug')) +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + import ctypes.macholib + + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(ctypes.macholib, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index f89521cf8b3..46f8ff93efa 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,3 +1,4 @@ +import sys import unittest import test.support from ctypes import (CDLL, PyDLL, ArgumentError, @@ -240,7 +241,8 @@ def test_parameter_repr(self): self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") self.assertEqual(repr(c_float.from_param(1.5)), "") self.assertEqual(repr(c_double.from_param(1.5)), "") - self.assertEqual(repr(c_double.from_param(1e300)), "") + if sys.float_repr_style == 'short': + self.assertEqual(repr(c_double.from_param(1e300)), "") self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") diff --git a/Lib/test/test_ctypes/test_prototypes.py b/Lib/test/test_ctypes/test_prototypes.py index 63ae799ea86..d976e8da0e2 100644 --- a/Lib/test/test_ctypes/test_prototypes.py +++ b/Lib/test/test_ctypes/test_prototypes.py @@ -72,6 +72,32 @@ def test_paramflags(self): self.assertEqual(func(None), None) self.assertEqual(func(input=None), None) + def test_invalid_paramflags(self): + proto = CFUNCTYPE(c_int, c_char_p) + with self.assertRaises(ValueError): + func = proto(("myprintf", testdll), ((1, "fmt"), (1, "arg1"))) + + def test_invalid_setattr_argtypes(self): + proto = CFUNCTYPE(c_int, c_char_p) + func = proto(("myprintf", testdll), ((1, "fmt"),)) + + with self.assertRaisesRegex(TypeError, "_argtypes_ must be a sequence of types"): + func.argtypes = 123 + self.assertEqual(func.argtypes, (c_char_p,)) + + with self.assertRaisesRegex(ValueError, "paramflags must have the same length as argtypes"): + func.argtypes = (c_char_p, c_int) + self.assertEqual(func.argtypes, (c_char_p,)) + + def test_paramflags_outarg(self): + proto = CFUNCTYPE(c_int, c_char_p, c_int) + with self.assertRaisesRegex(TypeError, "must be a pointer type"): + func = proto(("myprintf", testdll), ((1, "fmt"), (2, "out"))) + + proto = CFUNCTYPE(c_int, c_char_p, c_void_p) + func = proto(("myprintf", testdll), ((1, "fmt"), (2, "out"))) + with self.assertRaisesRegex(TypeError, "must be a pointer type"): + func.argtypes = (c_char_p, c_int) def test_int_pointer_arg(self): func = testdll._testfunc_p_p diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index c307258e565..d5ca7f2ca1a 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -130,6 +130,9 @@ def test_use_env(self): curses.use_env(False) curses.use_env(True) + def test_error(self): + self.assertIsSubclass(curses.error, Exception) + def test_create_windows(self): win = curses.newwin(5, 10) self.assertEqual(win.getbegyx(), (0, 0)) @@ -1257,7 +1260,7 @@ class TestAscii(unittest.TestCase): def test_controlnames(self): for name in curses.ascii.controlnames: - self.assertTrue(hasattr(curses.ascii, name), name) + self.assertHasAttr(curses.ascii, name) def test_ctypes(self): def check(func, expected): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index ac78f8327b8..3b335429b98 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -120,7 +120,7 @@ class Some: pass for param in inspect.signature(dataclass).parameters: if param == 'cls': continue - self.assertTrue(hasattr(Some.__dataclass_params__, param), msg=param) + self.assertHasAttr(Some.__dataclass_params__, param) def test_named_init_params(self): @dataclass @@ -671,7 +671,7 @@ class C: self.assertEqual(the_fields[0].name, 'x') self.assertEqual(the_fields[0].type, int) - self.assertFalse(hasattr(C, 'x')) + self.assertNotHasAttr(C, 'x') self.assertTrue (the_fields[0].init) self.assertTrue (the_fields[0].repr) self.assertEqual(the_fields[1].name, 'y') @@ -681,7 +681,7 @@ class C: self.assertTrue (the_fields[1].repr) self.assertEqual(the_fields[2].name, 'z') self.assertEqual(the_fields[2].type, str) - self.assertFalse(hasattr(C, 'z')) + self.assertNotHasAttr(C, 'z') self.assertTrue (the_fields[2].init) self.assertFalse(the_fields[2].repr) @@ -732,8 +732,8 @@ class C: z: object = default t: int = field(default=100) - self.assertFalse(hasattr(C, 'x')) - self.assertFalse(hasattr(C, 'y')) + self.assertNotHasAttr(C, 'x') + self.assertNotHasAttr(C, 'y') self.assertIs (C.z, default) self.assertEqual(C.t, 100) @@ -927,6 +927,20 @@ class C: validate_class(C) + def test_incomplete_annotations(self): + # gh-142214 + @dataclass + class C: + "doc" # needed because otherwise we fetch the annotations at the wrong time + x: int + + C.__annotate__ = lambda _: {} + + self.assertEqual( + annotationlib.get_annotations(C.__init__), + {"return": None} + ) + def test_missing_default(self): # Test that MISSING works the same as a default not being # specified. @@ -2471,6 +2485,149 @@ def __init__(self, a): self.assertEqual(D(5).a, 10) +class TestInitAnnotate(unittest.TestCase): + # Tests for the generated __annotate__ function for __init__ + # See: https://github.com/python/cpython/issues/137530 + + def test_annotate_function(self): + # No forward references + @dataclass + class A: + a: int + + value_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'a': int, 'return': None}) + self.assertEqual(forwardref_annos, {'a': int, 'return': None}) + self.assertEqual(string_annos, {'a': 'int', 'return': 'None'}) + + self.assertTrue(getattr(A.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_annotate_function_forwardref(self): + # With forward references + @dataclass + class B: + b: undefined + + # VALUE annotations should raise while unresolvable + with self.assertRaises(NameError): + _ = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(forwardref_annos, {'b': support.EqualToForwardRef('undefined', owner=B, is_class=True), 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + # Now VALUE and FORWARDREF should resolve, STRING should be unchanged + undefined = int + + value_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'b': int, 'return': None}) + self.assertEqual(forwardref_annos, {'b': int, 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + def test_annotate_function_init_false(self): + # Check `init=False` attributes don't get into the annotations of the __init__ function + @dataclass + class C: + c: str = field(init=False) + + self.assertEqual(annotationlib.get_annotations(C.__init__), {'return': None}) + + def test_annotate_function_contains_forwardref(self): + # Check string annotations on objects containing a ForwardRef + @dataclass + class D: + d: list[undefined] + + with self.assertRaises(NameError): + annotationlib.get_annotations(D.__init__) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[support.EqualToForwardRef("undefined", is_class=True, owner=D)], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + # Now test when it is defined + undefined = str + + # VALUE should now resolve + self.assertEqual( + annotationlib.get_annotations(D.__init__), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + def test_annotate_function_not_replaced(self): + # Check that __annotate__ is not replaced on non-generated __init__ functions + @dataclass(slots=True) + class E: + x: str + def __init__(self, x: int) -> None: + self.x = x + + self.assertEqual( + annotationlib.get_annotations(E.__init__), {"x": int, "return": None} + ) + + self.assertFalse(hasattr(E.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_slots_true_init_false(self): + # Test that slots=True and init=False work together and + # that __annotate__ is not added to __init__. + + @dataclass(slots=True, init=False) + class F: + x: int + + f = F() + f.x = 10 + self.assertEqual(f.x, 10) + + self.assertFalse(hasattr(F.__init__, "__annotate__")) + + def test_init_false_forwardref(self): + # Test forward references in fields not required for __init__ annotations. + + # At the moment this raises a NameError for VALUE annotations even though the + # undefined annotation is not required for the __init__ annotations. + # Ideally this will be fixed but currently there is no good way to resolve this + + @dataclass + class F: + not_in_init: list[undefined] = field(init=False, default=None) + in_init: int + + annos = annotationlib.get_annotations(F.__init__, format=annotationlib.Format.FORWARDREF) + self.assertEqual( + annos, + {"in_init": int, "return": None}, + ) + + with self.assertRaises(NameError): + annos = annotationlib.get_annotations(F.__init__) # NameError on not_in_init + + class TestRepr(unittest.TestCase): def test_repr(self): @dataclass @@ -2912,10 +3069,10 @@ class C: pass c = C() - self.assertFalse(hasattr(c, 'i')) + self.assertNotHasAttr(c, 'i') with self.assertRaises(FrozenInstanceError): c.i = 5 - self.assertFalse(hasattr(c, 'i')) + self.assertNotHasAttr(c, 'i') with self.assertRaises(FrozenInstanceError): del c.i @@ -3144,7 +3301,7 @@ class S(D): del s.y self.assertEqual(s.y, 10) del s.cached - self.assertFalse(hasattr(s, 'cached')) + self.assertNotHasAttr(s, 'cached') with self.assertRaises(AttributeError) as cm: del s.cached self.assertNotIsInstance(cm.exception, FrozenInstanceError) @@ -3158,12 +3315,12 @@ class S(D): pass s = S() - self.assertFalse(hasattr(s, 'x')) + self.assertNotHasAttr(s, 'x') s.x = 5 self.assertEqual(s.x, 5) del s.x - self.assertFalse(hasattr(s, 'x')) + self.assertNotHasAttr(s, 'x') with self.assertRaises(AttributeError) as cm: del s.x self.assertNotIsInstance(cm.exception, FrozenInstanceError) @@ -3393,8 +3550,8 @@ class A: B = dataclass(A, slots=True) self.assertIsNot(A, B) - self.assertFalse(hasattr(A, "__slots__")) - self.assertTrue(hasattr(B, "__slots__")) + self.assertNotHasAttr(A, "__slots__") + self.assertHasAttr(B, "__slots__") # Can't be local to test_frozen_pickle. @dataclass(frozen=True, slots=True) @@ -3804,6 +3961,49 @@ class WithCorrectSuper(CorrectSuper): # that we create internally. self.assertEqual(CorrectSuper.args, ["default", "default"]) + def test_original_class_is_gced(self): + # gh-135228: Make sure when we replace the class with slots=True, the original class + # gets garbage collected. + def make_simple(): + @dataclass(slots=True) + class SlotsTest: + pass + + return SlotsTest + + def make_with_annotations(): + @dataclass(slots=True) + class SlotsTest: + x: int + + return SlotsTest + + def make_with_annotations_and_method(): + @dataclass(slots=True) + class SlotsTest: + x: int + + def method(self) -> int: + return self.x + + return SlotsTest + + def make_with_forwardref(): + @dataclass(slots=True) + class SlotsTest: + x: undefined + y: list[undefined] + + return SlotsTest + + for make in (make_simple, make_with_annotations, make_with_annotations_and_method, make_with_forwardref): + with self.subTest(make=make): + C = make() + support.gc_collect() + candidates = [cls for cls in object.__subclasses__() if cls.__name__ == 'SlotsTest' + and cls.__firstlineno__ == make.__code__.co_firstlineno + 1] + self.assertEqual(candidates, [C]) + class TestDescriptors(unittest.TestCase): def test_set_name(self): diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 4be7c5649da..ae9faabd536 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -66,7 +66,7 @@ def keys_helper(self, f): return keys def test_error(self): - self.assertTrue(issubclass(self.module.error, OSError)) + self.assertIsSubclass(self.module.error, OSError) def test_anydbm_not_existing(self): self.assertRaises(dbm.error, dbm.open, _fname) @@ -135,6 +135,67 @@ def test_anydbm_access(self): assert(f[key] == b"Python:") f.close() + def test_anydbm_readonly_reorganize(self): + self.init_db() + with dbm.open(_fname, 'r') as d: + # Early stopping. + if not hasattr(d, 'reorganize'): + self.skipTest("method reorganize not available this dbm submodule") + + self.assertRaises(dbm.error, lambda: d.reorganize()) + + def test_anydbm_reorganize_not_changed_content(self): + self.init_db() + with dbm.open(_fname, 'c') as d: + # Early stopping. + if not hasattr(d, 'reorganize'): + self.skipTest("method reorganize not available this dbm submodule") + + keys_before = sorted(d.keys()) + values_before = [d[k] for k in keys_before] + d.reorganize() + keys_after = sorted(d.keys()) + values_after = [d[k] for k in keys_before] + self.assertEqual(keys_before, keys_after) + self.assertEqual(values_before, values_after) + + def test_anydbm_reorganize_decreased_size(self): + + def _calculate_db_size(db_path): + if os.path.isfile(db_path): + return os.path.getsize(db_path) + total_size = 0 + for root, _, filenames in os.walk(db_path): + for filename in filenames: + file_path = os.path.join(root, filename) + total_size += os.path.getsize(file_path) + return total_size + + # This test requires relatively large databases to reliably show difference in size before and after reorganizing. + with dbm.open(_fname, 'n') as f: + # Early stopping. + if not hasattr(f, 'reorganize'): + self.skipTest("method reorganize not available this dbm submodule") + + for k in self._dict: + f[k.encode('ascii')] = self._dict[k] * 100000 + db_keys = list(f.keys()) + + # Make sure to calculate size of database only after file is closed to ensure file content are flushed to disk. + size_before = _calculate_db_size(os.path.dirname(_fname)) + + # Delete some elements from the start of the database. + keys_to_delete = db_keys[:len(db_keys) // 2] + with dbm.open(_fname, 'c') as f: + for k in keys_to_delete: + del f[k] + f.reorganize() + + # Make sure to calculate size of database only after file is closed to ensure file content are flushed to disk. + size_after = _calculate_db_size(os.path.dirname(_fname)) + + self.assertLess(size_after, size_before) + def test_open_with_bytes(self): dbm.open(os.fsencode(_fname), "c").close() @@ -213,7 +274,8 @@ def test_whichdb(self): @unittest.skipUnless(ndbm, reason='Test requires ndbm') def test_whichdb_ndbm(self): # Issue 17198: check that ndbm which is referenced in whichdb is defined - with open(_fname + '.db', 'wb'): pass + with open(_fname + '.db', 'wb') as f: + f.write(b'spam') _bytes_fname = os.fsencode(_fname) fnames = [_fname, os_helper.FakePath(_fname), _bytes_fname, os_helper.FakePath(_bytes_fname)] diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 2e1f2d32924..f367a98865d 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -1,3 +1,5 @@ +import os +import stat import sys import unittest from contextlib import closing @@ -14,6 +16,11 @@ from dbm.sqlite3 import _normalize_uri +root_in_posix = False +if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + + class _SQLiteDbmTests(unittest.TestCase): def setUp(self): @@ -36,7 +43,7 @@ def test_uri_substitutions(self): ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): - self.assertTrue(_normalize_uri(path).endswith(normalized)) + self.assertEndsWith(_normalize_uri(path), normalized) @unittest.skipUnless(sys.platform == "win32", "requires Windows") def test_uri_windows(self): @@ -55,7 +62,7 @@ def test_uri_windows(self): with self.subTest(path=path, normalized=normalized): if not Path(path).is_absolute(): self.skipTest(f"skipping relative path: {path!r}") - self.assertTrue(_normalize_uri(path).endswith(normalized)) + self.assertEndsWith(_normalize_uri(path), normalized) class ReadOnly(_SQLiteDbmTests): @@ -90,6 +97,50 @@ def test_readonly_iter(self): self.assertEqual([k for k in self.db], [b"key1", b"key2"]) +@unittest.skipIf(root_in_posix, "test is meanless with root privilege") +class ReadOnlyFilesystem(unittest.TestCase): + + def setUp(self): + self.test_dir = os_helper.TESTFN + self.addCleanup(os_helper.rmtree, self.test_dir) + os.mkdir(self.test_dir) + self.db_path = os.path.join(self.test_dir, "test.db") + + db = dbm_sqlite3.open(self.db_path, "c") + db[b"key"] = b"value" + db.close() + + def test_readonly_file_read(self): + os.chmod(self.db_path, stat.S_IREAD) + with dbm_sqlite3.open(self.db_path, "r") as db: + self.assertEqual(db[b"key"], b"value") + + def test_readonly_file_write(self): + os.chmod(self.db_path, stat.S_IREAD) + with dbm_sqlite3.open(self.db_path, "w") as db: + with self.assertRaises(dbm_sqlite3.error): + db[b"newkey"] = b"newvalue" + + def test_readonly_dir_read(self): + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + with dbm_sqlite3.open(self.db_path, "r") as db: + self.assertEqual(db[b"key"], b"value") + + def test_readonly_dir_write(self): + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + with dbm_sqlite3.open(self.db_path, "w") as db: + try: + db[b"newkey"] = b"newvalue" + modified = True # on Windows and macOS + except dbm_sqlite3.error: + modified = False + with dbm_sqlite3.open(self.db_path, "r") as db: + if modified: + self.assertEqual(db[b"newkey"], b"newvalue") + else: + self.assertNotIn(b"newkey", db) + + class ReadWrite(_SQLiteDbmTests): def setUp(self): diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 9e298401dc3..b520b062ebc 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -28,7 +28,6 @@ import math import os, sys import operator -import warnings import pickle, copy import unittest import numbers @@ -982,6 +981,7 @@ def test_formatting(self): ('.0f', '0e-2', '0'), ('.0f', '3.14159265', '3'), ('.1f', '3.14159265', '3.1'), + ('.01f', '3.14159265', '3.1'), # leading zero in precision ('.4f', '3.14159265', '3.1416'), ('.6f', '3.14159265', '3.141593'), ('.7f', '3.14159265', '3.1415926'), # round-half-even! @@ -1067,6 +1067,7 @@ def test_formatting(self): ('8,', '123456', ' 123,456'), ('08,', '123456', '0,123,456'), # special case: extra 0 needed ('+08,', '123456', '+123,456'), # but not if there's a sign + ('008,', '123456', '0,123,456'), # leading zero in width (' 08,', '123456', ' 123,456'), ('08,', '-123456', '-123,456'), ('+09,', '123456', '+0,123,456'), @@ -1088,6 +1089,15 @@ def test_formatting(self): ('07_', '1234.56', '1_234.56'), ('_', '1.23456789', '1.23456789'), ('_%', '123.456789', '12_345.6789%'), + # and now for something completely different... + ('.,', '1.23456789', '1.234,567,89'), + ('._', '1.23456789', '1.234_567_89'), + ('.6_f', '12345.23456789', '12345.234_568'), + (',._%', '123.456789', '12,345.678_9%'), + (',._e', '123456', '1.234_56e+5'), + (',.4_e', '123456', '1.234_6e+5'), + (',.3_e', '123456', '1.235e+5'), + (',._E', '123456', '1.234_56E+5'), # negative zero: default behavior ('.1f', '-0', '-0.0'), @@ -1161,6 +1171,10 @@ def test_formatting(self): # bytes format argument self.assertRaises(TypeError, Decimal(1).__format__, b'-020') + # precision or fractional part separator should follow after dot + self.assertRaises(ValueError, format, Decimal(1), '.f') + self.assertRaises(ValueError, format, Decimal(1), '._6f') + def test_negative_zero_format_directed_rounding(self): with self.decimal.localcontext() as ctx: ctx.rounding = ROUND_CEILING @@ -4460,7 +4474,7 @@ def test_module_attributes(self): self.assertTrue(C.HAVE_THREADS is True or C.HAVE_THREADS is False) self.assertTrue(P.HAVE_THREADS is True or P.HAVE_THREADS is False) - self.assertEqual(C.__version__, P.__version__) + self.assertEqual(C.SPEC_VERSION, P.SPEC_VERSION) self.assertLessEqual(set(dir(C)), set(dir(P))) self.assertEqual([n for n in dir(C) if n[:2] != '__'], sorted(P.__all__)) @@ -5915,6 +5929,23 @@ def doit(ty): doit('Context') +class TestModule: + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(self.decimal, "__version__") + self.assertEqual(cm.filename, __file__) + + +@requires_cdecimal +class CTestModule(TestModule, unittest.TestCase): + decimal = C +class PyTestModule(TestModule, unittest.TestCase): + decimal = P + + def load_tests(loader, tests, pattern): if TODO_TESTS is not None: # Run only Arithmetic tests diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 4679f297fd7..4e1a489205a 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -838,7 +838,7 @@ def test_copy_pickle(self): self.assertEqual(list(d), list(e)) self.assertEqual(e.x, d.x) self.assertEqual(e.z, d.z) - self.assertFalse(hasattr(e, 'y')) + self.assertNotHasAttr(e, 'y') def test_pickle_recursive(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 76937432a43..82a48ad4d1a 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -409,7 +409,7 @@ class ClassPropertiesAndMethods(unittest.TestCase): def test_python_dicts(self): # Testing Python subclass of dict... - self.assertTrue(issubclass(dict, dict)) + self.assertIsSubclass(dict, dict) self.assertIsInstance({}, dict) d = dict() self.assertEqual(d, {}) @@ -433,7 +433,7 @@ def setstate(self, state): self.state = state def getstate(self): return self.state - self.assertTrue(issubclass(C, dict)) + self.assertIsSubclass(C, dict) a1 = C(12) self.assertEqual(a1.state, 12) a2 = C(foo=1, bar=2) @@ -1048,15 +1048,15 @@ class SubType(types.ModuleType): m = types.ModuleType("m") self.assertTrue(m.__class__ is types.ModuleType) - self.assertFalse(hasattr(m, "a")) + self.assertNotHasAttr(m, "a") m.__class__ = SubType self.assertTrue(m.__class__ is SubType) - self.assertTrue(hasattr(m, "a")) + self.assertHasAttr(m, "a") m.__class__ = types.ModuleType self.assertTrue(m.__class__ is types.ModuleType) - self.assertFalse(hasattr(m, "a")) + self.assertNotHasAttr(m, "a") # Make sure that builtin immutable objects don't support __class__ # assignment, because the object instances may be interned. @@ -1329,18 +1329,17 @@ class D(object): self.assertNotHasAttr(a, "__weakref__") a.foo = 42 self.assertEqual(a.__dict__, {"foo": 42}) + with self.assertRaises(TypeError): + weakref.ref(a) class W(object): __slots__ = ["__weakref__"] a = W() self.assertHasAttr(a, "__weakref__") self.assertNotHasAttr(a, "__dict__") - try: + with self.assertRaises(AttributeError): a.foo = 42 - except AttributeError: - pass - else: - self.fail("shouldn't be allowed to set a.foo") + self.assertIs(weakref.ref(a)(), a) class C1(W, D): __slots__ = [] @@ -1349,6 +1348,7 @@ class C1(W, D): self.assertHasAttr(a, "__weakref__") a.foo = 42 self.assertEqual(a.__dict__, {"foo": 42}) + self.assertIs(weakref.ref(a)(), a) class C2(D, W): __slots__ = [] @@ -1357,6 +1357,77 @@ class C2(D, W): self.assertHasAttr(a, "__weakref__") a.foo = 42 self.assertEqual(a.__dict__, {"foo": 42}) + self.assertIs(weakref.ref(a)(), a) + + @unittest.skipIf(_testcapi is None, 'need the _testcapi module') + def test_slots_special_before_items(self): + class D(_testcapi.HeapCCollection): + __slots__ = ["__dict__"] + a = D(1, 2, 3) + self.assertHasAttr(a, "__dict__") + self.assertNotHasAttr(a, "__weakref__") + a.foo = 42 + self.assertEqual(a.__dict__, {"foo": 42}) + with self.assertRaises(TypeError): + weakref.ref(a) + del a.__dict__ + self.assertNotHasAttr(a, "foo") + self.assertEqual(a.__dict__, {}) + self.assertEqual(list(a), [1, 2, 3]) + + class W(_testcapi.HeapCCollection): + __slots__ = ["__weakref__"] + a = W(1, 2, 3) + self.assertHasAttr(a, "__weakref__") + self.assertNotHasAttr(a, "__dict__") + with self.assertRaises(AttributeError): + a.foo = 42 + self.assertIs(weakref.ref(a)(), a) + + with self.assertRaises(TypeError): + class X(_testcapi.HeapCCollection): + __slots__ = ['x'] + + with self.assertRaises(TypeError): + class X(_testcapi.HeapCCollection): + __slots__ = ['__dict__', 'x'] + + @support.subTests(('base', 'arg'), [ + (tuple, (1, 2, 3)), + (int, 9876543210**2), + (bytes, b'ab'), + ]) + def test_slots_special_after_items(self, base, arg): + class D(base): + __slots__ = ["__dict__"] + a = D(arg) + self.assertHasAttr(a, "__dict__") + self.assertNotHasAttr(a, "__weakref__") + a.foo = 42 + self.assertEqual(a.__dict__, {"foo": 42}) + with self.assertRaises(TypeError): + weakref.ref(a) + del a.__dict__ + self.assertNotHasAttr(a, "foo") + self.assertEqual(a.__dict__, {}) + self.assertEqual(a, base(arg)) + + class W(base): + __slots__ = ["__weakref__"] + a = W(arg) + self.assertHasAttr(a, "__weakref__") + self.assertNotHasAttr(a, "__dict__") + with self.assertRaises(AttributeError): + a.foo = 42 + self.assertIs(weakref.ref(a)(), a) + self.assertEqual(a, base(arg)) + + with self.assertRaises(TypeError): + class X(base): + __slots__ = ['x'] + with self.assertRaises(TypeError): + class X(base): + __slots__ = ['__dict__', 'x'] def test_slots_special2(self): # Testing __qualname__ and __classcell__ in __slots__ @@ -1780,7 +1851,7 @@ class D(C): class E: # *not* subclassing from C foo = C.foo self.assertEqual(E().foo.__func__, C.foo) # i.e., unbound - self.assertTrue(repr(C.foo.__get__(C())).startswith("' + class A_with_dict: + __slots__ = ('__dict__',) + def __repr__(self): + return '' + class A_with_dict_weakref: + def __repr__(self): + return '' + class A_with_slots: + __slots__ = ('x',) + def __repr__(self): + return '' + class A_with_slots_dict: + __slots__ = ('x', '__dict__') + def __repr__(self): + return '' + + class B: + __slots__ = () + b = B() + r = repr(b) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (int,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (A_with_dict_weakref,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (A_with_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (A_with_slots,) + B.__bases__ = (A,) + self.assertNotHasAttr(b, '__dict__') + self.assertNotHasAttr(b, '__weakref__') + self.assertEqual(repr(b), '') + B.__bases__ = (object,) + self.assertEqual(repr(b), r) + + class B_with_dict_weakref: pass - else: - self.fail("shouldn't be able to create inheritance cycles") + b = B_with_dict_weakref() + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (A_with_slots,) + B_with_dict_weakref.__bases__ = (A_with_dict_weakref,) + self.assertEqual(repr(b), '') + B_with_dict_weakref.__bases__ = (A_with_dict,) + self.assertEqual(repr(b), '') + B_with_dict_weakref.__bases__ = (A,) + self.assertEqual(repr(b), '') + B_with_dict_weakref.__bases__ = (object,) + + class B_with_slots: + __slots__ = ('x',) + b = B_with_slots() + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots.__bases__ = (A_with_dict_weakref,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots.__bases__ = (A_with_dict,) + B_with_slots.__bases__ = (A,) + self.assertEqual(repr(b), '') + + class B_with_slots_dict: + __slots__ = ('x', '__dict__') + b = B_with_slots_dict() + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots_dict.__bases__ = (A_with_dict_weakref,) + B_with_slots_dict.__bases__ = (A_with_dict,) + self.assertEqual(repr(b), '') + B_with_slots_dict.__bases__ = (A,) + self.assertEqual(repr(b), '') + + class B_with_slots_dict_weakref: + __slots__ = ('x', '__dict__', '__weakref__') + b = B_with_slots_dict_weakref() + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots_dict_weakref.__bases__ = (A_with_slots_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots_dict_weakref.__bases__ = (A_with_slots,) + B_with_slots_dict_weakref.__bases__ = (A_with_dict_weakref,) + self.assertEqual(repr(b), '') + B_with_slots_dict_weakref.__bases__ = (A_with_dict,) + self.assertEqual(repr(b), '') + B_with_slots_dict_weakref.__bases__ = (A,) + self.assertEqual(repr(b), '') + + class C_with_slots(A_with_slots): + __slots__ = () + c = C_with_slots() + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots.__bases__ = (A_with_slots_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots.__bases__ = (A_with_dict_weakref,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots.__bases__ = (A_with_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots.__bases__ = (A,) + C_with_slots.__bases__ = (A_with_slots,) + self.assertEqual(repr(c), '') + + class C_with_slots_dict(A_with_slots): + pass + c = C_with_slots_dict() + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots_dict.__bases__ = (A_with_dict_weakref,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots_dict.__bases__ = (A_with_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots_dict.__bases__ = (A,) + C_with_slots_dict.__bases__ = (A_with_slots_dict,) + self.assertEqual(repr(c), '') + C_with_slots_dict.__bases__ = (A_with_slots,) + self.assertEqual(repr(c), '') + + class A_int(int): + __slots__ = () + def __repr__(self): + return '' + class B_int(int): + __slots__ = () + b = B_int(42) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_int.__bases__ = (object,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_int.__bases__ = (tuple,) + with self.assertRaisesRegex(TypeError, 'is not an acceptable base type'): + B_int.__bases__ = (bool,) + B_int.__bases__ = (A_int,) + self.assertEqual(repr(b), '') + B_int.__bases__ = (int,) + self.assertEqual(repr(b), '42') + + class A_tuple(tuple): + __slots__ = () + def __repr__(self): + return '' + class B_tuple(tuple): + __slots__ = () + b = B_tuple((1, 2)) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_tuple.__bases__ = (object,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_tuple.__bases__ = (int,) + B_tuple.__bases__ = (A_tuple,) + self.assertEqual(repr(b), '') + B_tuple.__bases__ = (tuple,) + self.assertEqual(repr(b), '(1, 2)') + + def test_assign_bases_many_subclasses(self): + # This is intended to check that typeobject.c:queue_slot_update() can + # handle updating many subclasses when a slot method is re-assigned. + class A: + x = 'hello' + def __call__(self): + return 123 + def __getitem__(self, index): + return None + + class X: + x = 'bye' + + class B(A): + pass + + subclasses = [] + for i in range(1000): + sc = type(f'Sub{i}', (B,), {}) + subclasses.append(sc) + + self.assertEqual(subclasses[0]()(), 123) + self.assertEqual(subclasses[0]().x, 'hello') + B.__bases__ = (X,) + with self.assertRaises(TypeError): + subclasses[0]()() + self.assertEqual(subclasses[0]().x, 'bye') def test_builtin_bases(self): # Make sure all the builtin types can have their base queried without @@ -4136,26 +4361,14 @@ class C(object): class D(C): pass - try: + with self.assertRaisesRegex(TypeError, 'layout differs'): L.__bases__ = (dict,) - except TypeError: - pass - else: - self.fail("shouldn't turn list subclass into dict subclass") - try: + with self.assertRaisesRegex(TypeError, 'immutable type'): list.__bases__ = (dict,) - except TypeError: - pass - else: - self.fail("shouldn't be able to assign to list.__bases__") - try: + with self.assertRaisesRegex(TypeError, 'layout differs'): D.__bases__ = (C, list) - except TypeError: - pass - else: - self.fail("best_base calculation found wanting") def test_unsubclassable_types(self): with self.assertRaises(TypeError): @@ -4523,6 +4736,7 @@ class Oops(object): del o @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @support.requires_resource('cpu') def test_wrapper_segfault(self): # SF 927248: deeply nested wrappers could cause stack overflow @@ -4867,6 +5081,7 @@ class Thing: deque.append(thing, thing) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_repr_as_str(self): # Issue #11603: crash or infinite loop when rebinding __str__ as # __repr__. @@ -4918,7 +5133,7 @@ class X: with self.assertRaises(TypeError) as cm: type(X).__dict__["__doc__"].__delete__(X) - self.assertIn("cannot delete '__doc__' attribute of immutable type 'X'", str(cm.exception)) + self.assertIn("cannot delete '__doc__' attribute of type 'X'", str(cm.exception)) self.assertEqual(X.__doc__, "banana") def test_qualname(self): @@ -5194,8 +5409,8 @@ def test_repr(self): # We can't blindly compare with the repr of another dict as ordering # of keys and values is arbitrary and may differ. r = repr(self.C.__dict__) - self.assertTrue(r.startswith('mappingproxy('), r) - self.assertTrue(r.endswith(')'), r) + self.assertStartsWith(r, 'mappingproxy(') + self.assertEndsWith(r, ')') for k, v in self.C.__dict__.items(): self.assertIn('{!r}: {!r}'.format(k, v), r) @@ -5982,5 +6197,69 @@ class A(metaclass=M): pass +class TestGenericDescriptors(unittest.TestCase): + def test___dict__(self): + class CustomClass: + pass + class SlotClass: + __slots__ = ['foo'] + class SlotSubClass(SlotClass): + pass + class IntSubclass(int): + pass + + dict_descriptor = CustomClass.__dict__['__dict__'] + self.assertEqual(dict_descriptor.__objclass__, object) + + for cls in CustomClass, SlotSubClass, IntSubclass: + with self.subTest(cls=cls): + self.assertIs(cls.__dict__['__dict__'], dict_descriptor) + instance = cls() + instance.attr = 123 + self.assertEqual( + dict_descriptor.__get__(instance, cls), + {'attr': 123}, + ) + with self.assertRaises(AttributeError): + print(dict_descriptor.__get__(True, bool)) + with self.assertRaises(AttributeError): + print(dict_descriptor.__get__(SlotClass(), SlotClass)) + + # delegation to type.__dict__ + self.assertIsInstance( + dict_descriptor.__get__(type, type), + types.MappingProxyType, + ) + + def test___weakref__(self): + class CustomClass: + pass + class SlotClass: + __slots__ = ['foo'] + class SlotSubClass(SlotClass): + pass + class IntSubclass(int): + pass + + weakref_descriptor = CustomClass.__dict__['__weakref__'] + self.assertEqual(weakref_descriptor.__objclass__, object) + + for cls in CustomClass, SlotSubClass: + with self.subTest(cls=cls): + self.assertIs(cls.__dict__['__weakref__'], weakref_descriptor) + instance = cls() + instance.attr = 123 + self.assertEqual( + weakref_descriptor.__get__(instance, cls), + None, + ) + with self.assertRaises(AttributeError): + weakref_descriptor.__get__(True, bool) + with self.assertRaises(AttributeError): + weakref_descriptor.__get__(SlotClass(), SlotClass) + with self.assertRaises(AttributeError): + weakref_descriptor.__get__(IntSubclass(), IntSubclass) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 3104cbc66cb..665b3e843dd 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -290,6 +290,38 @@ def badgen(): ['Cannot convert dictionary update sequence element #0 to a sequence'], ) + def test_update_shared_keys(self): + class MyClass: pass + + # Subclass str to enable us to create an object during the + # dict.update() call. + class MyStr(str): + def __hash__(self): + return super().__hash__() + + def __eq__(self, other): + # Create an object that shares the same PyDictKeysObject as + # obj.__dict__. + obj2 = MyClass() + obj2.a = "a" + obj2.b = "b" + obj2.c = "c" + return super().__eq__(other) + + obj = MyClass() + obj.a = "a" + obj.b = "b" + + x = {} + x[MyStr("a")] = MyStr("a") + + # gh-132617: this previously raised "dict mutated during update" error + x.update(obj.__dict__) + + self.assertEqual(x, { + MyStr("a"): "a", + "b": "b", + }) def test_fromkeys(self): self.assertEqual(dict.fromkeys('abc'), {'a':None, 'b':None, 'c':None}) @@ -338,17 +370,34 @@ def __setitem__(self, key, value): self.assertRaises(Exc, baddict2.fromkeys, [1]) # test fast path for dictionary inputs + res = dict(zip(range(6), [0]*6)) d = dict(zip(range(6), range(6))) - self.assertEqual(dict.fromkeys(d, 0), dict(zip(range(6), [0]*6))) + self.assertEqual(dict.fromkeys(d, 0), res) + # test fast path for set inputs + d = set(range(6)) + self.assertEqual(dict.fromkeys(d, 0), res) + # test slow path for other iterable inputs + d = list(range(6)) + self.assertEqual(dict.fromkeys(d, 0), res) + # test fast path when object's constructor returns large non-empty dict class baddict3(dict): def __new__(cls): return d - d = {i : i for i in range(10)} + d = {i : i for i in range(1000)} res = d.copy() res.update(a=None, b=None, c=None) self.assertEqual(baddict3.fromkeys({"a", "b", "c"}), res) + # test slow path when object is a proper subclass of dict + class baddict4(dict): + def __init__(self): + dict.__init__(self, d) + d = {i : i for i in range(1000)} + res = d.copy() + res.update(a=None, b=None, c=None) + self.assertEqual(baddict4.fromkeys({"a", "b", "c"}), res) + def test_copy(self): d = {1: 1, 2: 2, 3: 3} self.assertIsNot(d.copy(), d) @@ -770,8 +819,8 @@ def test_dictview_mixed_set_operations(self): def test_missing(self): # Make sure dict doesn't have a __missing__ method - self.assertFalse(hasattr(dict, "__missing__")) - self.assertFalse(hasattr({}, "__missing__")) + self.assertNotHasAttr(dict, "__missing__") + self.assertNotHasAttr({}, "__missing__") # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError @@ -1022,10 +1071,8 @@ class C: a = C() a.x = 1 d = a.__dict__ - before_resize = sys.getsizeof(d) d[2] = 2 # split table is resized to a generic combined table - self.assertGreater(sys.getsizeof(d), before_resize) self.assertEqual(list(d), ['x', 2]) def test_iterator_pickling(self): @@ -1554,6 +1601,34 @@ def __hash__(self): with self.assertRaises(KeyError): d.get(key2) + def test_clear_at_lookup(self): + class X: + def __hash__(self): + return 1 + def __eq__(self, other): + nonlocal d + d.clear() + + d = {} + for _ in range(10): + d[X()] = None + + self.assertEqual(len(d), 1) + + d = {} + for _ in range(10): + d.setdefault(X(), None) + + self.assertEqual(len(d), 1) + + def test_split_table_update_with_str_subclass(self): + class MyStr(str): pass + class MyClass: pass + obj = MyClass() + obj.attr = 1 + obj.__dict__[MyStr('attr')] = 2 + self.assertEqual(obj.attr, 2) + class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index 26b56dac503..a7a46216787 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -132,7 +132,7 @@ def test_star_expression(self): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: {x:x for x in BrokenIter(init_raises=True)} diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 9e217249be7..771fd46e042 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -1,5 +1,5 @@ import difflib -from test.support import findfile +from test.support import findfile, force_colorized import unittest import doctest import sys @@ -29,6 +29,16 @@ def test_one_delete(self): ('delete', 40, 41, 40, 40), ('equal', 41, 81, 40, 80)]) + def test_opcode_caching(self): + sm = difflib.SequenceMatcher(None, 'b' * 100, 'a' + 'b' * 100) + opcode = sm.get_opcodes() + self.assertEqual(opcode, + [ ('insert', 0, 0, 0, 1), + ('equal', 0, 100, 1, 101)]) + # Implementation detail: opcodes are cached; + # `get_opcodes()` returns the same object + self.assertIs(opcode, sm.get_opcodes()) + def test_bjunk(self): sm = difflib.SequenceMatcher(isjunk=lambda x: x == ' ', a='a' * 40 + 'b' * 40, b='a' * 44 + 'b' * 40) @@ -255,21 +265,21 @@ def test_make_file_default_charset(self): html_diff = difflib.HtmlDiff() output = html_diff.make_file(patch914575_from1.splitlines(), patch914575_to1.splitlines()) - self.assertIn('content="text/html; charset=utf-8"', output) + self.assertIn('charset="utf-8"', output) def test_make_file_iso88591_charset(self): html_diff = difflib.HtmlDiff() output = html_diff.make_file(patch914575_from1.splitlines(), patch914575_to1.splitlines(), charset='iso-8859-1') - self.assertIn('content="text/html; charset=iso-8859-1"', output) + self.assertIn('charset="iso-8859-1"', output) def test_make_file_usascii_charset_with_nonascii_input(self): html_diff = difflib.HtmlDiff() output = html_diff.make_file(patch914575_nonascii_from1.splitlines(), patch914575_nonascii_to1.splitlines(), charset='us-ascii') - self.assertIn('content="text/html; charset=us-ascii"', output) + self.assertIn('charset="us-ascii"', output) self.assertIn('ımplıcıt', output) class TestDiffer(unittest.TestCase): @@ -293,6 +303,15 @@ def test_close_matches_aligned(self): '+ kitten\n', '+ puppy\n']) + def test_one_insert(self): + m = difflib.Differ().compare('b' * 2, 'a' + 'b' * 2) + self.assertEqual(list(m), ['+ a', ' b', ' b']) + + def test_one_delete(self): + m = difflib.Differ().compare('a' + 'b' * 2, 'b' * 2) + self.assertEqual(list(m), ['- a', ' b', ' b']) + + class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): args = [['one'], ['two'], 'Original', 'Current', @@ -355,6 +374,22 @@ def test_range_format_context(self): self.assertEqual(fmt(3,6), '4,6') self.assertEqual(fmt(0,0), '0') + @force_colorized + def test_unified_diff_colored_output(self): + args = [['one', 'three'], ['two', 'three'], 'Original', 'Current', + '2005-01-26 23:30:50', '2010-04-02 10:20:52'] + actual = list(difflib.unified_diff(*args, lineterm='', color=True)) + + expect = [ + "\033[1m--- Original\t2005-01-26 23:30:50\033[0m", + "\033[1m+++ Current\t2010-04-02 10:20:52\033[0m", + "\033[36m@@ -1,2 +1,2 @@\033[0m", + "\033[31m-one\033[0m", + "\033[32m+two\033[0m", + "\033[0m three\033[0m", + ] + self.assertEqual(expect, actual) + class TestBytes(unittest.TestCase): # don't really care about the content of the output, just the fact @@ -585,6 +620,26 @@ def test_longest_match_with_popular_chars(self): self.assertFalse(self.longer_match_exists(a, b, match.size)) +class TestCloseMatches(unittest.TestCase): + # Happy paths are tested in the doctests of `difflib.get_close_matches`. + + def test_invalid_inputs(self): + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], n=0) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], n=-1) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], cutoff=1.1) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], cutoff=-0.1) + + +class TestRestore(unittest.TestCase): + # Happy paths are tested in the doctests of `difflib.restore`. + + def test_invalid_input(self): + with self.assertRaises(ValueError): + ''.join(difflib.restore([], 0)) + with self.assertRaises(ValueError): + ''.join(difflib.restore([], 3)) + + def setUpModule(): difflib.HtmlDiff._default_prefix = 0 diff --git a/Lib/test/test_difflib_expect.html b/Lib/test/test_difflib_expect.html index 9f33a9e9c9c..240e2c33368 100644 --- a/Lib/test/test_difflib_expect.html +++ b/Lib/test/test_difflib_expect.html @@ -1,22 +1,42 @@ - - - - + + - - - @@ -37,11 +59,11 @@ - - - - - + + + + + @@ -53,11 +75,11 @@ - - - - - + + + + + @@ -69,11 +91,11 @@ - - - - - + + + + + @@ -111,11 +133,11 @@

Context (first diff within numlines=5(default))

 - - - - - + + + + + @@ -128,11 +150,11 @@

Context (first diff within numlines=5(default))

 - - - - - + + + + + @@ -145,11 +167,11 @@

Context (first diff within numlines=5(default))

 - - - - - + + + + + @@ -170,11 +192,11 @@

Context (first diff after numlines=5(default))

 - - - - - + + + + + @@ -187,11 +209,11 @@

Context (first diff after numlines=5(default))

 - - - - - + + + + + @@ -204,11 +226,11 @@

Context (first diff after numlines=5(default))

 - - - - - + + + + + @@ -225,11 +247,11 @@

Context (numlines=6)

 - - - - - + + + + + @@ -241,11 +263,11 @@

Context (numlines=6)

 - - - - - + + + + + @@ -257,11 +279,11 @@

Context (numlines=6)

 - - - - - + + + + + @@ -278,25 +300,25 @@

Context (numlines=0)

 - - - - - + + + + + - - - - - + + + + + - - - - - + + + + +

from
to
f1f1
n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly.
3   2. Explicit is better than implicit.
4   3. Simple is better than complex.3   3.   Simple is better than complex.
5   4. Complex is better than complicated.4   4. Complicated is better than complex.
5   5. Flat is better than nested.
n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly.
3   2. Explicit is better than implicit.
4   3. Simple is better than complex.3   3.   Simple is better than complex.
5   4. Complex is better than complicated.4   4. Complicated is better than complex.
5   5. Flat is better than nested.
61236123
71237123
81238123
1412314123
1512315123
1616
n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly.
18   2. Explicit is better than implicit.
19   3. Simple is better than complex.18   3.   Simple is better than complex.
20   4. Complex is better than complicated.19   4. Complicated is better than complex.
20   5. Flat is better than nested.
n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly.
18   2. Explicit is better than implicit.
19   3. Simple is better than complex.18   3.   Simple is better than complex.
20   4. Complex is better than complicated.19   4. Complicated is better than complex.
20   5. Flat is better than nested.
2112321123
2212322123
2312323123
2912329123
3012330123
3131
t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly.
33   2. Explicit is better than implicit.
34   3. Simple is better than complex.33   3.   Simple is better than complex.
35   4. Complex is better than complicated.34   4. Complicated is better than complex.
35   5. Flat is better than nested.
t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly.
33   2. Explicit is better than implicit.
34   3. Simple is better than complex.33   3.   Simple is better than complex.
35   4. Complex is better than complicated.34   4. Complicated is better than complex.
35   5. Flat is better than nested.
3612336123
3712337123
3812338123

from
to
f1f1
n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly.
3   2. Explicit is better than implicit.
4   3. Simple is better than complex.3   3.   Simple is better than complex.
5   4. Complex is better than complicated.4   4. Complicated is better than complex.
5   5. Flat is better than nested.
n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly.
3   2. Explicit is better than implicit.
4   3. Simple is better than complex.3   3.   Simple is better than complex.
5   4. Complex is better than complicated.4   4. Complicated is better than complex.
5   5. Flat is better than nested.
61236123
71237123
81238123
1412314123
1512315123
1616
n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly.
18   2. Explicit is better than implicit.
19   3. Simple is better than complex.18   3.   Simple is better than complex.
20   4. Complex is better than complicated.19   4. Complicated is better than complex.
20   5. Flat is better than nested.
n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly.
18   2. Explicit is better than implicit.
19   3. Simple is better than complex.18   3.   Simple is better than complex.
20   4. Complex is better than complicated.19   4. Complicated is better than complex.
20   5. Flat is better than nested.
2112321123
2212322123
2312323123
2912329123
3012330123
3131
t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly.
33   2. Explicit is better than implicit.
34   3. Simple is better than complex.33   3.   Simple is better than complex.
35   4. Complex is better than complicated.34   4. Complicated is better than complex.
35   5. Flat is better than nested.
t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly.
33   2. Explicit is better than implicit.
34   3. Simple is better than complex.33   3.   Simple is better than complex.
35   4. Complex is better than complicated.34   4. Complicated is better than complex.
35   5. Flat is better than nested.
3612336123
3712337123
3812338123
94569456
1045610456
1111
n12   1. Beautiful is beTTer than ugly.n12   1. Beautiful is better than ugly.
13   2. Explicit is better than implicit.
14   3. Simple is better than complex.13   3.   Simple is better than complex.
15   4. Complex is better than complicated.14   4. Complicated is better than complex.
15   5. Flat is better than nested.
n12   1. Beautiful is beTTer than ugly.n12   1. Beautiful is better than ugly.
13   2. Explicit is better than implicit.
14   3. Simple is better than complex.13   3.   Simple is better than complex.
15   4. Complex is better than complicated.14   4. Complicated is better than complex.
15   5. Flat is better than nested.
1612316123
1712317123
1812318123
2412324123
2512325123
2626
n27   1. Beautiful is beTTer than ugly.n27   1. Beautiful is better than ugly.
28   2. Explicit is better than implicit.
29   3. Simple is better than complex.28   3.   Simple is better than complex.
30   4. Complex is better than complicated.29   4. Complicated is better than complex.
30   5. Flat is better than nested.
n27   1. Beautiful is beTTer than ugly.n27   1. Beautiful is better than ugly.
28   2. Explicit is better than implicit.
29   3. Simple is better than complex.28   3.   Simple is better than complex.
30   4. Complex is better than complicated.29   4. Complicated is better than complex.
30   5. Flat is better than nested.
3112331123
3212332123
3312333123
3912339123
4012340123
4141
t42   1. Beautiful is beTTer than ugly.t42   1. Beautiful is better than ugly.
43   2. Explicit is better than implicit.
44   3. Simple is better than complex.43   3.   Simple is better than complex.
45   4. Complex is better than complicated.44   4. Complicated is better than complex.
45   5. Flat is better than nested.
t42   1. Beautiful is beTTer than ugly.t42   1. Beautiful is better than ugly.
43   2. Explicit is better than implicit.
44   3. Simple is better than complex.43   3.   Simple is better than complex.
45   4. Complex is better than complicated.44   4. Complicated is better than complex.
45   5. Flat is better than nested.
4612346123
4712347123
4812348123

from
to
f1f1
n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly.
3   2. Explicit is better than implicit.
4   3. Simple is better than complex.3   3.   Simple is better than complex.
5   4. Complex is better than complicated.4   4. Complicated is better than complex.
5   5. Flat is better than nested.
n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly.
3   2. Explicit is better than implicit.
4   3. Simple is better than complex.3   3.   Simple is better than complex.
5   4. Complex is better than complicated.4   4. Complicated is better than complex.
5   5. Flat is better than nested.
61236123
71237123
81238123
1412314123
1512315123
1616
n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly.
18   2. Explicit is better than implicit.
19   3. Simple is better than complex.18   3.   Simple is better than complex.
20   4. Complex is better than complicated.19   4. Complicated is better than complex.
20   5. Flat is better than nested.
n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly.
18   2. Explicit is better than implicit.
19   3. Simple is better than complex.18   3.   Simple is better than complex.
20   4. Complex is better than complicated.19   4. Complicated is better than complex.
20   5. Flat is better than nested.
2112321123
2212322123
2312323123
2912329123
3012330123
3131
t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly.
33   2. Explicit is better than implicit.
34   3. Simple is better than complex.33   3.   Simple is better than complex.
35   4. Complex is better than complicated.34   4. Complicated is better than complex.
35   5. Flat is better than nested.
t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly.
33   2. Explicit is better than implicit.
34   3. Simple is better than complex.33   3.   Simple is better than complex.
35   4. Complex is better than complicated.34   4. Complicated is better than complex.
35   5. Flat is better than nested.
3612336123
3712337123
3812338123

from
to
n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly.
3   2. Explicit is better than implicit.
4   3. Simple is better than complex.3   3.   Simple is better than complex.
5   4. Complex is better than complicated.4   4. Complicated is better than complex.
5   5. Flat is better than nested.
n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly.
3   2. Explicit is better than implicit.
4   3. Simple is better than complex.3   3.   Simple is better than complex.
5   4. Complex is better than complicated.4   4. Complicated is better than complex.
5   5. Flat is better than nested.
n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly.
18   2. Explicit is better than implicit.
19   3. Simple is better than complex.18   3.   Simple is better than complex.
20   4. Complex is better than complicated.19   4. Complicated is better than complex.
20   5. Flat is better than nested.
n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly.
18   2. Explicit is better than implicit.
19   3. Simple is better than complex.18   3.   Simple is better than complex.
20   4. Complex is better than complicated.19   4. Complicated is better than complex.
20   5. Flat is better than nested.
t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly.
33   2. Explicit is better than implicit.
34   3. Simple is better than complex.33   3.   Simple is better than complex.
35   4. Complex is better than complicated.34   4. Complicated is better than complex.
35   5. Flat is better than nested.
t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly.
33   2. Explicit is better than implicit.
34   3. Simple is better than complex.33   3.   Simple is better than complex.
35   4. Complex is better than complicated.34   4. Complicated is better than complex.
35   5. Flat is better than nested.

Same Context

@@ -396,11 +418,11 @@

tabsize=2
f1f1
t2    Line 1: preceded by from:[tt] to:[ssss]t2    Line 1: preceded by from:[tt] to:[ssss]
3      Line 2: preceded by from:[sstt] to:[sssst]3      Line 2: preceded by from:[sstt] to:[sssst]
4      Line 3: preceded by from:[sstst] to:[ssssss]4      Line 3: preceded by from:[sstst] to:[ssssss]
5Line 4:   has from:[sst] to:[sss] after :5Line 4:   has from:[sst] to:[sss] after :
6Line 5: has from:[t] to:[ss] at end 6Line 5: has from:[t] to:[ss] at end
t2    Line 1: preceded by from:[tt] to:[ssss]t2    Line 1: preceded by from:[tt] to:[ssss]
3      Line 2: preceded by from:[sstt] to:[sssst]3      Line 2: preceded by from:[sstt] to:[sssst]
4      Line 3: preceded by from:[sstst] to:[ssssss]4      Line 3: preceded by from:[sstst] to:[ssssss]
5Line 4:   has from:[sst] to:[sss] after :5Line 4:   has from:[sst] to:[sss] after :
6Line 5: has from:[t] to:[ss] at end 6Line 5: has from:[t] to:[ss] at end

tabsize=default

@@ -412,11 +434,11 @@

tabsize=default
f1f1
t2                Line 1: preceded by from:[tt] to:[ssss]t2    Line 1: preceded by from:[tt] to:[ssss]
3                Line 2: preceded by from:[sstt] to:[sssst]3        Line 2: preceded by from:[sstt] to:[sssst]
4                Line 3: preceded by from:[sstst] to:[ssssss]4      Line 3: preceded by from:[sstst] to:[ssssss]
5Line 4:         has from:[sst] to:[sss] after :5Line 4:   has from:[sst] to:[sss] after :
6Line 5: has from:[t] to:[ss] at end     6Line 5: has from:[t] to:[ss] at end
t2                Line 1: preceded by from:[tt] to:[ssss]t2    Line 1: preceded by from:[tt] to:[ssss]
3                Line 2: preceded by from:[sstt] to:[sssst]3        Line 2: preceded by from:[sstt] to:[sssst]
4                Line 3: preceded by from:[sstst] to:[ssssss]4      Line 3: preceded by from:[sstst] to:[ssssss]
5Line 4:         has from:[sst] to:[sss] after :5Line 4:   has from:[sst] to:[sss] after :
6Line 5: has from:[t] to:[ss] at end     6Line 5: has from:[t] to:[ss] at end

Context (wrapcolumn=14,numlines=0)

@@ -427,31 +449,31 @@

Context (wrapcolumn=14,numlines=0)
n4line 2n4line 2    adde
 >d
n4line 2n4line 2    adde
 >d
n6line 4   changn6line 4   chanG
>ed>Ed
7line 5   chang7line 5a  chanG
>ed>ed
8line 6   chang8line 6a  chang
>ed>Ed
n6line 4   changn6line 4   chanG
>ed>Ed
7line 5   chang7line 5a  chanG
>ed>ed
8line 6   chang8line 6a  chang
>ed>Ed
n10line 8  subtran10line 8
>cted 
n10line 8  subtran10line 8
>cted 
t1212345678901234t121234567890
>56789012345689 
>012345 
13short line13another long l
 >ine that needs
 > to be wrapped
14just fits in!!14just fitS in!!
15just fits in t15just fits in t
>wo lines yup!!>wo lineS yup!!
t1212345678901234t121234567890
>56789012345689 
>012345 
13short line13another long l
 >ine that needs
 > to be wrapped
14just fits in!!14just fitS in!!
15just fits in t15just fits in t
>wo lines yup!!>wo lineS yup!!

wrapcolumn=14,splitlines()

@@ -467,28 +489,28 @@

wrapcolumn=14,splitlines()

>56789012345689>56789012345689
>012345>012345
3line 13line 1
n4line 2n4line 2    adde
 >d
n4line 2n4line 2    adde
 >d
5line 35line 3
n6line 4   changn6line 4   chanG
>ed>Ed
7line 5   chang7line 5a  chanG
>ed>ed
8line 6   chang8line 6a  chang
>ed>Ed
n6line 4   changn6line 4   chanG
>ed>Ed
7line 5   chang7line 5a  chanG
>ed>ed
8line 6   chang8line 6a  chang
>ed>Ed
9line 79line 7
n10line 8  subtran10line 8
>cted 
n10line 8  subtran10line 8
>cted 
11line 911line 9
t1212345678901234t121234567890
>56789012345689 
>012345 
13short line13another long l
 >ine that needs
 > to be wrapped
14just fits in!!14just fitS in!!
15just fits in t15just fits in t
>wo lines yup!!>wo lineS yup!!
t1212345678901234t121234567890
>56789012345689 
>012345 
13short line13another long l
 >ine that needs
 > to be wrapped
14just fits in!!14just fitS in!!
15just fits in t15just fits in t
>wo lines yup!!>wo lineS yup!!
16the end16the end
@@ -505,28 +527,28 @@

wrapcolumn=14,splitlines(True)

>56789012345689>56789012345689
>012345>012345
3line 13line 1
n4line 2n4line 2    adde
 >d
n4line 2n4line 2    adde
 >d
5line 35line 3
n6line 4   changn6line 4   chanG
>ed>Ed
7line 5   chang7line 5a  chanG
>ed>ed
8line 6   chang8line 6a  chang
>ed>Ed
n6line 4   changn6line 4   chanG
>ed>Ed
7line 5   chang7line 5a  chanG
>ed>ed
8line 6   chang8line 6a  chang
>ed>Ed
9line 79line 7
n10line 8  subtran10line 8
>cted 
n10line 8  subtran10line 8
>cted 
11line 911line 9
t1212345678901234t121234567890
>56789012345689 
>012345 
13short line13another long l
 >ine that needs
 > to be wrapped
14just fits in!!14just fitS in!!
15just fits in t15just fits in t
>wo lines yup!!>wo lineS yup!!
t1212345678901234t121234567890
>56789012345689 
>012345 
13short line13another long l
 >ine that needs
 > to be wrapped
14just fits in!!14just fitS in!!
15just fits in t15just fits in t
>wo lines yup!!>wo lineS yup!!
16the end16the end
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index ae68c1dd75c..3e747748720 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -453,52 +453,53 @@ def foo(a: int, b: str) -> str: """ dis_traceback = """\ -%4d RESUME 0 +%4d RESUME 0 -%4d NOP +%4d NOP -%4d L1: LOAD_SMALL_INT 1 - LOAD_SMALL_INT 0 - --> BINARY_OP 11 (/) - POP_TOP +%4d L1: LOAD_SMALL_INT 1 + LOAD_SMALL_INT 0 + --> BINARY_OP 11 (/) + POP_TOP -%4d L2: LOAD_FAST_CHECK 1 (tb) - RETURN_VALUE +%4d L2: LOAD_FAST_CHECK 1 (tb) + RETURN_VALUE - -- L3: PUSH_EXC_INFO + -- L3: PUSH_EXC_INFO -%4d LOAD_GLOBAL 0 (Exception) - CHECK_EXC_MATCH - POP_JUMP_IF_FALSE 24 (to L7) - NOT_TAKEN - STORE_FAST 0 (e) +%4d LOAD_GLOBAL 0 (Exception) + CHECK_EXC_MATCH + POP_JUMP_IF_FALSE 24 (to L9) + L4: NOT_TAKEN + L5: STORE_FAST 0 (e) -%4d L4: LOAD_FAST 0 (e) - LOAD_ATTR 2 (__traceback__) - STORE_FAST 1 (tb) - L5: POP_EXCEPT - LOAD_CONST 1 (None) - STORE_FAST 0 (e) - DELETE_FAST 0 (e) +%4d L6: LOAD_FAST 0 (e) + LOAD_ATTR 2 (__traceback__) + STORE_FAST 1 (tb) + L7: POP_EXCEPT + LOAD_CONST 1 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) -%4d LOAD_FAST 1 (tb) - RETURN_VALUE +%4d LOAD_FAST 1 (tb) + RETURN_VALUE - -- L6: LOAD_CONST 1 (None) - STORE_FAST 0 (e) - DELETE_FAST 0 (e) - RERAISE 1 + -- L8: LOAD_CONST 1 (None) + STORE_FAST 0 (e) + DELETE_FAST 0 (e) + RERAISE 1 -%4d L7: RERAISE 0 +%4d L9: RERAISE 0 - -- L8: COPY 3 - POP_EXCEPT - RERAISE 1 + -- L10: COPY 3 + POP_EXCEPT + RERAISE 1 ExceptionTable: L1 to L2 -> L3 [0] - L3 to L4 -> L8 [1] lasti - L4 to L5 -> L6 [1] lasti - L6 to L8 -> L8 [1] lasti + L3 to L4 -> L10 [1] lasti + L5 to L6 -> L10 [1] lasti + L6 to L7 -> L8 [1] lasti + L8 to L10 -> L10 [1] lasti """ % (TRACEBACK_CODE.co_firstlineno, TRACEBACK_CODE.co_firstlineno + 1, TRACEBACK_CODE.co_firstlineno + 2, @@ -567,11 +568,11 @@ def _with(c): %4d L3: PUSH_EXC_INFO WITH_EXCEPT_START TO_BOOL - POP_JUMP_IF_TRUE 2 (to L4) - NOT_TAKEN - RERAISE 2 - L4: POP_TOP - L5: POP_EXCEPT + POP_JUMP_IF_TRUE 2 (to L6) + L4: NOT_TAKEN + L5: RERAISE 2 + L6: POP_TOP + L7: POP_EXCEPT POP_TOP POP_TOP POP_TOP @@ -581,12 +582,13 @@ def _with(c): LOAD_CONST 1 (None) RETURN_VALUE - -- L6: COPY 3 + -- L8: COPY 3 POP_EXCEPT RERAISE 1 ExceptionTable: L1 to L2 -> L3 [2] lasti - L3 to L5 -> L6 [4] lasti + L3 to L4 -> L8 [4] lasti + L5 to L7 -> L8 [4] lasti """ % (_with.__code__.co_firstlineno, _with.__code__.co_firstlineno + 1, _with.__code__.co_firstlineno + 2, @@ -606,7 +608,7 @@ async def _asyncwith(c): POP_TOP L1: RESUME 0 -%4d LOAD_FAST_BORROW 0 (c) +%4d LOAD_FAST 0 (c) COPY 1 LOAD_SPECIAL 3 (__aexit__) SWAP 2 @@ -828,7 +830,7 @@ def foo(x): %4d LOAD_GLOBAL 1 (list + NULL) LOAD_FAST_BORROW 0 (x) BUILD_TUPLE 1 - LOAD_CONST 1 ( at 0x..., file "%s", line %d>) + LOAD_CONST %d ( at 0x..., file "%s", line %d>) MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) LOAD_DEREF 1 (y) @@ -840,6 +842,7 @@ def foo(x): _h.__code__.co_firstlineno + 1, _h.__code__.co_firstlineno + 1, _h.__code__.co_firstlineno + 3, + 1 if __debug__ else 0, __file__, _h.__code__.co_firstlineno + 3, ) @@ -851,7 +854,7 @@ def foo(x): %4d RETURN_GENERATOR POP_TOP L1: RESUME 0 - LOAD_FAST_BORROW 0 (.0) + LOAD_FAST 0 (.0) GET_ITER L2: FOR_ITER 14 (to L3) STORE_FAST 1 (z) @@ -902,7 +905,7 @@ def loop_test(): %3d RESUME_CHECK 0 %3d BUILD_LIST 0 - LOAD_CONST_MORTAL 2 ((1, 2, 3)) + LOAD_CONST 2 ((1, 2, 3)) LIST_EXTEND 1 LOAD_SMALL_INT 3 BINARY_OP 5 (*) @@ -918,7 +921,7 @@ def loop_test(): %3d L2: END_FOR POP_ITER - LOAD_CONST_IMMORTAL 1 (None) + LOAD_CONST 1 (None) RETURN_VALUE """ % (loop_test.__code__.co_firstlineno, loop_test.__code__.co_firstlineno + 1, @@ -1304,7 +1307,7 @@ def test_load_attr_specialize(self): load_attr_quicken = """\ 0 RESUME_CHECK 0 - 1 LOAD_CONST_IMMORTAL 0 ('a') + 1 LOAD_CONST 0 ('a') LOAD_ATTR_SLOT 0 (__class__) RETURN_VALUE """ @@ -1466,7 +1469,7 @@ def get_disassembly(self, func, lasti=-1, wrapper=True, **kwargs): Kw-only arguments: 0 Number of locals: 1 Stack size: \\d+ -Flags: OPTIMIZED, NEWLOCALS, HAS_DOCSTRING +Flags: OPTIMIZED, NEWLOCALS(, HAS_DOCSTRING)? Constants: {code_info_consts} Names: @@ -1821,7 +1824,7 @@ def _prepare_test_cases(): make_inst(opname='LOAD_SMALL_INT', arg=10, argval=10, argrepr='', offset=12, start_offset=12, starts_line=False, line_number=3), make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='GET_ITER', arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=False, line_number=3), - make_inst(opname='FOR_ITER', arg=32, argval=92, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='FOR_ITER', arg=33, argval=94, argrepr='to L4', offset=24, start_offset=24, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=False, line_number=3), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=True, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=False, line_number=4), @@ -1840,110 +1843,111 @@ def _prepare_test_cases(): make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=82, start_offset=82, starts_line=False, line_number=7), make_inst(opname='JUMP_BACKWARD', arg=32, argval=24, argrepr='to L1', offset=84, start_offset=84, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=88, start_offset=88, starts_line=True, line_number=8, label=3), - make_inst(opname='JUMP_FORWARD', arg=13, argval=118, argrepr='to L5', offset=90, start_offset=90, starts_line=False, line_number=8), - make_inst(opname='END_FOR', arg=None, argval=None, argrepr='', offset=92, start_offset=92, starts_line=True, line_number=3, label=4), - make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=False, line_number=3), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=96, start_offset=96, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=106, start_offset=106, starts_line=False, line_number=10), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=108, start_offset=108, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=116, start_offset=116, starts_line=False, line_number=10), - make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=118, start_offset=118, starts_line=True, line_number=11, label=5), - make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=120, start_offset=120, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_JUMP_IF_FALSE', arg=40, argval=212, argrepr='to L8', offset=128, start_offset=128, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=132, start_offset=132, starts_line=False, line_number=11), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=134, start_offset=134, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=144, start_offset=144, starts_line=False, line_number=12), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=146, start_offset=146, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=154, start_offset=154, starts_line=False, line_number=12), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=156, start_offset=156, starts_line=True, line_number=13), - make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=158, start_offset=158, starts_line=False, line_number=13), - make_inst(opname='BINARY_OP', arg=23, argval=23, argrepr='-=', offset=160, start_offset=160, starts_line=False, line_number=13, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), - make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=172, start_offset=172, starts_line=False, line_number=13), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=True, line_number=14), - make_inst(opname='LOAD_SMALL_INT', arg=6, argval=6, argrepr='', offset=176, start_offset=176, starts_line=False, line_number=14), - make_inst(opname='COMPARE_OP', arg=148, argval='>', argrepr='bool(>)', offset=178, start_offset=178, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='POP_JUMP_IF_FALSE', arg=3, argval=192, argrepr='to L6', offset=182, start_offset=182, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=186, start_offset=186, starts_line=False, line_number=14), - make_inst(opname='JUMP_BACKWARD', arg=37, argval=118, argrepr='to L5', offset=188, start_offset=188, starts_line=True, line_number=15, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=192, start_offset=192, starts_line=True, line_number=16, label=6), - make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=194, start_offset=194, starts_line=False, line_number=16), - make_inst(opname='COMPARE_OP', arg=18, argval='<', argrepr='bool(<)', offset=196, start_offset=196, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='POP_JUMP_IF_TRUE', arg=3, argval=210, argrepr='to L7', offset=200, start_offset=200, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=204, start_offset=204, starts_line=False, line_number=16), - make_inst(opname='JUMP_BACKWARD', arg=46, argval=118, argrepr='to L5', offset=206, start_offset=206, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='JUMP_FORWARD', arg=11, argval=234, argrepr='to L9', offset=210, start_offset=210, starts_line=True, line_number=17, label=7), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=212, start_offset=212, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=222, start_offset=222, starts_line=False, line_number=19), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=224, start_offset=224, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=232, start_offset=232, starts_line=False, line_number=19), - make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=True, line_number=20, label=9), - make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=21), - make_inst(opname='LOAD_SMALL_INT', arg=0, argval=0, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=21), - make_inst(opname='BINARY_OP', arg=11, argval=11, argrepr='/', offset=240, start_offset=240, starts_line=False, line_number=21, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=252, start_offset=252, starts_line=False, line_number=21), - make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=254, start_offset=254, starts_line=True, line_number=25), - make_inst(opname='COPY', arg=1, argval=1, argrepr='', offset=256, start_offset=256, starts_line=False, line_number=25), - make_inst(opname='LOAD_SPECIAL', arg=1, argval=1, argrepr='__exit__', offset=258, start_offset=258, starts_line=False, line_number=25), - make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=260, start_offset=260, starts_line=False, line_number=25), - make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25), - make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=264, start_offset=264, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=274, start_offset=274, starts_line=False, line_number=25), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=276, start_offset=276, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=286, start_offset=286, starts_line=False, line_number=26), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=288, start_offset=288, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=False, line_number=26), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=298, start_offset=298, starts_line=True, line_number=25), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=90, start_offset=90, starts_line=False, line_number=8), + make_inst(opname='JUMP_FORWARD', arg=13, argval=120, argrepr='to L5', offset=92, start_offset=92, starts_line=False, line_number=8), + make_inst(opname='END_FOR', arg=None, argval=None, argrepr='', offset=94, start_offset=94, starts_line=True, line_number=3, label=4), + make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=96, start_offset=96, starts_line=False, line_number=3), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=98, start_offset=98, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=108, start_offset=108, starts_line=False, line_number=10), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=110, start_offset=110, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=118, start_offset=118, starts_line=False, line_number=10), + make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=120, start_offset=120, starts_line=True, line_number=11, label=5), + make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=122, start_offset=122, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=40, argval=214, argrepr='to L8', offset=130, start_offset=130, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=134, start_offset=134, starts_line=False, line_number=11), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=136, start_offset=136, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=146, start_offset=146, starts_line=False, line_number=12), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=148, start_offset=148, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=156, start_offset=156, starts_line=False, line_number=12), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=158, start_offset=158, starts_line=True, line_number=13), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=160, start_offset=160, starts_line=False, line_number=13), + make_inst(opname='BINARY_OP', arg=23, argval=23, argrepr='-=', offset=162, start_offset=162, starts_line=False, line_number=13, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=174, start_offset=174, starts_line=False, line_number=13), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=176, start_offset=176, starts_line=True, line_number=14), + make_inst(opname='LOAD_SMALL_INT', arg=6, argval=6, argrepr='', offset=178, start_offset=178, starts_line=False, line_number=14), + make_inst(opname='COMPARE_OP', arg=148, argval='>', argrepr='bool(>)', offset=180, start_offset=180, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_FALSE', arg=3, argval=194, argrepr='to L6', offset=184, start_offset=184, starts_line=False, line_number=14, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=188, start_offset=188, starts_line=False, line_number=14), + make_inst(opname='JUMP_BACKWARD', arg=37, argval=120, argrepr='to L5', offset=190, start_offset=190, starts_line=True, line_number=15, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=194, start_offset=194, starts_line=True, line_number=16, label=6), + make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=196, start_offset=196, starts_line=False, line_number=16), + make_inst(opname='COMPARE_OP', arg=18, argval='<', argrepr='bool(<)', offset=198, start_offset=198, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=3, argval=212, argrepr='to L7', offset=202, start_offset=202, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=206, start_offset=206, starts_line=False, line_number=16), + make_inst(opname='JUMP_BACKWARD', arg=46, argval=120, argrepr='to L5', offset=208, start_offset=208, starts_line=False, line_number=16, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='JUMP_FORWARD', arg=11, argval=236, argrepr='to L9', offset=212, start_offset=212, starts_line=True, line_number=17, label=7), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=214, start_offset=214, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=224, start_offset=224, starts_line=False, line_number=19), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=226, start_offset=226, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=234, start_offset=234, starts_line=False, line_number=19), + make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=236, start_offset=236, starts_line=True, line_number=20, label=9), + make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=238, start_offset=238, starts_line=True, line_number=21), + make_inst(opname='LOAD_SMALL_INT', arg=0, argval=0, argrepr='', offset=240, start_offset=240, starts_line=False, line_number=21), + make_inst(opname='BINARY_OP', arg=11, argval=11, argrepr='/', offset=242, start_offset=242, starts_line=False, line_number=21, cache_info=[('counter', 1, b'\x00\x00'), ('descr', 4, b'\x00\x00\x00\x00\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=False, line_number=21), + make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=256, start_offset=256, starts_line=True, line_number=25), + make_inst(opname='COPY', arg=1, argval=1, argrepr='', offset=258, start_offset=258, starts_line=False, line_number=25), + make_inst(opname='LOAD_SPECIAL', arg=1, argval=1, argrepr='__exit__', offset=260, start_offset=260, starts_line=False, line_number=25), + make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=False, line_number=25), + make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=264, start_offset=264, starts_line=False, line_number=25), + make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=266, start_offset=266, starts_line=False, line_number=25), + make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=276, start_offset=276, starts_line=False, line_number=25), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=278, start_offset=278, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=288, start_offset=288, starts_line=False, line_number=26), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=290, start_offset=290, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=False, line_number=26), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=300, start_offset=300, starts_line=True, line_number=25), make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=302, start_offset=302, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=304, start_offset=304, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=False, line_number=25), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=314, start_offset=314, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=324, start_offset=324, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=326, start_offset=326, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=334, start_offset=334, starts_line=False, line_number=28), - make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=336, start_offset=336, starts_line=False, line_number=28), - make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=338, start_offset=338, starts_line=False, line_number=28), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=True, line_number=25), - make_inst(opname='WITH_EXCEPT_START', arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=False, line_number=25), - make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_JUMP_IF_TRUE', arg=2, argval=360, argrepr='to L11', offset=352, start_offset=352, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=356, start_offset=356, starts_line=False, line_number=25), - make_inst(opname='RERAISE', arg=2, argval=2, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25, label=11), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=304, start_offset=304, starts_line=False, line_number=25), + make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=306, start_offset=306, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=False, line_number=25), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=316, start_offset=316, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=326, start_offset=326, starts_line=False, line_number=28), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=336, start_offset=336, starts_line=False, line_number=28), + make_inst(opname='LOAD_CONST', arg=4, argval=None, argrepr='None', offset=338, start_offset=338, starts_line=False, line_number=28), + make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=28), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=True, line_number=25), + make_inst(opname='WITH_EXCEPT_START', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=25), + make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_JUMP_IF_TRUE', arg=2, argval=362, argrepr='to L11', offset=354, start_offset=354, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=358, start_offset=358, starts_line=False, line_number=25), + make_inst(opname='RERAISE', arg=2, argval=2, argrepr='', offset=360, start_offset=360, starts_line=False, line_number=25), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=362, start_offset=362, starts_line=False, line_number=25, label=11), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=364, start_offset=364, starts_line=False, line_number=25), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=366, start_offset=366, starts_line=False, line_number=25), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=False, line_number=25), - make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=29, argval=314, argrepr='to L10', offset=370, start_offset=370, starts_line=False, line_number=25), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=372, start_offset=372, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=374, start_offset=374, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None), - make_inst(opname='LOAD_GLOBAL', arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=380, start_offset=380, starts_line=True, line_number=22, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='CHECK_EXC_MATCH', arg=None, argval=None, argrepr='', offset=390, start_offset=390, starts_line=False, line_number=22), - make_inst(opname='POP_JUMP_IF_FALSE', arg=15, argval=426, argrepr='to L12', offset=392, start_offset=392, starts_line=False, line_number=22, cache_info=[('counter', 1, b'\x00\x00')]), - make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=396, start_offset=396, starts_line=False, line_number=22), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=400, start_offset=400, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=5, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=410, start_offset=410, starts_line=False, line_number=23), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=412, start_offset=412, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=420, start_offset=420, starts_line=False, line_number=23), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23), - make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=314, argrepr='to L10', offset=424, start_offset=424, starts_line=False, line_number=23), - make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=426, start_offset=426, starts_line=True, line_number=22, label=12), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=430, start_offset=430, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None), - make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None), - make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=436, start_offset=436, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), - make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=446, start_offset=446, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=448, start_offset=448, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), - make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=456, start_offset=456, starts_line=False, line_number=28), - make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28), - make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=460, start_offset=460, starts_line=True, line_number=None), - make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=462, start_offset=462, starts_line=False, line_number=None), - make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=False, line_number=25), + make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=29, argval=316, argrepr='to L10', offset=372, start_offset=372, starts_line=False, line_number=25), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=374, start_offset=374, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=376, start_offset=376, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=378, start_offset=378, starts_line=False, line_number=None), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=False, line_number=None), + make_inst(opname='LOAD_GLOBAL', arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=382, start_offset=382, starts_line=True, line_number=22, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='CHECK_EXC_MATCH', arg=None, argval=None, argrepr='', offset=392, start_offset=392, starts_line=False, line_number=22), + make_inst(opname='POP_JUMP_IF_FALSE', arg=15, argval=428, argrepr='to L12', offset=394, start_offset=394, starts_line=False, line_number=22, cache_info=[('counter', 1, b'\x00\x00')]), + make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=398, start_offset=398, starts_line=False, line_number=22), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=400, start_offset=400, starts_line=False, line_number=22), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=402, start_offset=402, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=5, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=412, start_offset=412, starts_line=False, line_number=23), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=422, start_offset=422, starts_line=False, line_number=23), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=424, start_offset=424, starts_line=False, line_number=23), + make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=316, argrepr='to L10', offset=426, start_offset=426, starts_line=False, line_number=23), + make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=428, start_offset=428, starts_line=True, line_number=22, label=12), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=430, start_offset=430, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=432, start_offset=432, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=434, start_offset=434, starts_line=False, line_number=None), + make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=436, start_offset=436, starts_line=False, line_number=None), + make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=438, start_offset=438, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), + make_inst(opname='LOAD_CONST', arg=6, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=448, start_offset=448, starts_line=False, line_number=28), + make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=450, start_offset=450, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=458, start_offset=458, starts_line=False, line_number=28), + make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=460, start_offset=460, starts_line=False, line_number=28), + make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=462, start_offset=462, starts_line=True, line_number=None), + make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=None), + make_inst(opname='RERAISE', arg=1, argval=1, argrepr='', offset=466, start_offset=466, starts_line=False, line_number=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Lib/test/test_doctest/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py index 0dbcd9a11ea..0bd402e9828 100644 --- a/Lib/test/test_doctest/doctest_lineno.py +++ b/Lib/test/test_doctest/doctest_lineno.py @@ -76,3 +76,32 @@ def property_with_doctest(self): @decorator def func_with_docstring_wrapped(): """Some unrelated info.""" + + +# https://github.com/python/cpython/issues/136914 +import functools + + +@functools.cache +def cached_func_with_doctest(value): + """ + >>> cached_func_with_doctest(1) + -1 + """ + return -value + + +@functools.cache +def cached_func_without_docstring(value): + return value + 1 + + +class ClassWithACachedProperty: + + @functools.cached_property + def cached(self): + """ + >>> X().cached + -1 + """ + return 0 diff --git a/Lib/test/test_doctest/sample_doctest_errors.py b/Lib/test/test_doctest/sample_doctest_errors.py new file mode 100644 index 00000000000..4a6f07af2d4 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_errors.py @@ -0,0 +1,46 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving errors. + +>>> 2 + 2 +5 +>>> 1/0 +1 +""" + +def g(): + [][0] # line 12 + +def errors(): + """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> g() + 1 + """ + +def syntax_error(): + """ + >>> 2+*3 + 5 + """ + +__test__ = { + 'bad': """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + """, +} + +def test_suite(): + import doctest + return doctest.DocTestSuite() diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index a4a49298bab..241d09db1fa 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -678,6 +678,8 @@ def basics(): r""" >>> for t in tests: ... print('%5s %s' % (t.lineno, t.name)) None test.test_doctest.doctest_lineno + None test.test_doctest.doctest_lineno.ClassWithACachedProperty + 102 test.test_doctest.doctest_lineno.ClassWithACachedProperty.cached 22 test.test_doctest.doctest_lineno.ClassWithDocstring 30 test.test_doctest.doctest_lineno.ClassWithDoctest None test.test_doctest.doctest_lineno.ClassWithoutDocstring @@ -687,6 +689,8 @@ def basics(): r""" 45 test.test_doctest.doctest_lineno.MethodWrapper.method_with_doctest None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest + 86 test.test_doctest.doctest_lineno.cached_func_with_doctest + None test.test_doctest.doctest_lineno.cached_func_without_docstring 4 test.test_doctest.doctest_lineno.func_with_docstring 77 test.test_doctest.doctest_lineno.func_with_docstring_wrapped 12 test.test_doctest.doctest_lineno.func_with_doctest @@ -829,6 +833,118 @@ def test_empty_namespace_package(self): self.assertEqual(len(include_empty_finder.find(mod)), 1) self.assertEqual(len(exclude_empty_finder.find(mod)), 0) + def test_lineno_of_test_dict_strings(self): + """Test line numbers are found for __test__ dict strings.""" + module_content = '''\ +"""Module docstring.""" + +def dummy_function(): + """Dummy function docstring.""" + pass + +__test__ = { + 'test_string': """ + This is a test string. + >>> 1 + 1 + 2 + """, +} +''' + with tempfile.TemporaryDirectory() as tmpdir: + module_path = os.path.join(tmpdir, 'test_module_lineno.py') + with open(module_path, 'w') as f: + f.write(module_content) + + sys.path.insert(0, tmpdir) + try: + import test_module_lineno + finder = doctest.DocTestFinder() + tests = finder.find(test_module_lineno) + + test_dict_test = None + for test in tests: + if '__test__' in test.name: + test_dict_test = test + break + + self.assertIsNotNone( + test_dict_test, + "__test__ dict test not found" + ) + # gh-69113: line number should not be None for __test__ strings + self.assertIsNotNone( + test_dict_test.lineno, + "Line number should not be None for __test__ dict strings" + ) + self.assertGreater( + test_dict_test.lineno, + 0, + "Line number should be positive" + ) + finally: + if 'test_module_lineno' in sys.modules: + del sys.modules['test_module_lineno'] + sys.path.pop(0) + + def test_lineno_multiline_matching(self): + """Test multi-line matching when no unique line exists.""" + # gh-69113: test that line numbers are found even when lines + # appear multiple times (e.g., ">>> x = 1" in both test entries) + module_content = '''\ +"""Module docstring.""" + +__test__ = { + 'test_one': """ + >>> x = 1 + >>> x + 1 + """, + 'test_two': """ + >>> x = 1 + >>> x + 2 + """, +} +''' + with tempfile.TemporaryDirectory() as tmpdir: + module_path = os.path.join(tmpdir, 'test_module_multiline.py') + with open(module_path, 'w') as f: + f.write(module_content) + + sys.path.insert(0, tmpdir) + try: + import test_module_multiline + finder = doctest.DocTestFinder() + tests = finder.find(test_module_multiline) + + test_one = None + test_two = None + for test in tests: + if 'test_one' in test.name: + test_one = test + elif 'test_two' in test.name: + test_two = test + + self.assertIsNotNone(test_one, "test_one not found") + self.assertIsNotNone(test_two, "test_two not found") + self.assertIsNotNone( + test_one.lineno, + "Line number should not be None for test_one" + ) + self.assertIsNotNone( + test_two.lineno, + "Line number should not be None for test_two" + ) + self.assertNotEqual( + test_one.lineno, + test_two.lineno, + "test_one and test_two should have different line numbers" + ) + finally: + if 'test_module_multiline' in sys.modules: + del sys.modules['test_module_multiline'] + sys.path.pop(0) + def test_DocTestParser(): r""" Unit tests for the `DocTestParser` class. @@ -2267,14 +2383,24 @@ def test_DocTestSuite(): >>> import unittest >>> import test.test_doctest.sample_doctest >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) - >>> suite.run(unittest.TestResult()) - + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> for tst, _ in result.failures: + ... print(tst) + bad (test.test_doctest.sample_doctest.__test__) [0] + foo (test.test_doctest.sample_doctest) [0] + >>> for tst, _ in result.errors: + ... print(tst) + test_silly_setup (test.test_doctest.sample_doctest) [1] + y_is_one (test.test_doctest.sample_doctest) [0] We can also supply the module by name: >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') - >>> suite.run(unittest.TestResult()) - + >>> result = suite.run(unittest.TestResult()) + >>> result + The module need not contain any doctest examples: @@ -2296,13 +2422,26 @@ def test_DocTestSuite(): >>> result >>> len(result.skipped) - 2 + 7 + >>> for tst, _ in result.skipped: + ... print(tst) + double_skip (test.test_doctest.sample_doctest_skip) [0] + double_skip (test.test_doctest.sample_doctest_skip) [1] + double_skip (test.test_doctest.sample_doctest_skip) + partial_skip_fail (test.test_doctest.sample_doctest_skip) [0] + partial_skip_pass (test.test_doctest.sample_doctest_skip) [0] + single_skip (test.test_doctest.sample_doctest_skip) [0] + single_skip (test.test_doctest.sample_doctest_skip) + >>> for tst, _ in result.failures: + ... print(tst) + no_skip_fail (test.test_doctest.sample_doctest_skip) [0] + partial_skip_fail (test.test_doctest.sample_doctest_skip) [1] We can use the current module: >>> suite = test.test_doctest.sample_doctest.test_suite() >>> suite.run(unittest.TestResult()) - + We can also provide a DocTestFinder: @@ -2310,7 +2449,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... test_finder=finder) >>> suite.run(unittest.TestResult()) - + The DocTestFinder need not return any tests: @@ -2326,7 +2465,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', globs={}) >>> suite.run(unittest.TestResult()) - + Alternatively, we can provide extra globals. Here we'll make an error go away by providing an extra global variable: @@ -2334,7 +2473,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... extraglobs={'y': 1}) >>> suite.run(unittest.TestResult()) - + You can pass option flags. Here we'll cause an extra error by disabling the blank-line feature: @@ -2342,7 +2481,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) >>> suite.run(unittest.TestResult()) - + You can supply setUp and tearDown functions: @@ -2359,7 +2498,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... setUp=setUp, tearDown=tearDown) >>> suite.run(unittest.TestResult()) - + But the tearDown restores sanity: @@ -2377,13 +2516,117 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', setUp=setUp) >>> suite.run(unittest.TestResult()) - + Here, we didn't need to use a tearDown function because we modified the test globals, which are a copy of the sample_doctest module dictionary. The test globals are automatically cleared for us after a test. - """ + """ + +def test_DocTestSuite_errors(): + """Tests for error reporting in DocTestSuite. + + >>> import unittest + >>> import test.test_doctest.sample_doctest_errors as mod + >>> suite = doctest.DocTestSuite(mod) + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + >...>> 2 + 2 + AssertionError: Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + + >>> print(result.failures[1][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 37, in test.test_doctest.sample_doctest_errors.__test__.bad + >...>> 2 + 2 + AssertionError: Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + + >>> print(result.failures[2][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + >...>> 2 + 2 + AssertionError: Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + + >>> print(result.errors[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + >...>> 1/0 + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[1][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 39, in test.test_doctest.sample_doctest_errors.__test__.bad + >...>> 1/0 + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[2][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + >...>> 1/0 + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[3][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + >...>> f() + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + + >>> print(result.errors[4][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + >...>> g() + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range + + >>> print(result.errors[5][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + >...>> 2+*3 + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + + """ def test_DocFileSuite(): """We can test tests found in text files using a DocFileSuite. @@ -2396,7 +2639,7 @@ def test_DocFileSuite(): ... 'test_doctest2.txt', ... 'test_doctest4.txt') >>> suite.run(unittest.TestResult()) - + The test files are looked for in the directory containing the calling module. A package keyword argument can be provided to @@ -2408,14 +2651,14 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... package='test.test_doctest') >>> suite.run(unittest.TestResult()) - + '/' should be used as a path separator. It will be converted to a native separator at run time: >>> suite = doctest.DocFileSuite('../test_doctest/test_doctest.txt') >>> suite.run(unittest.TestResult()) - + If DocFileSuite is used from an interactive session, then files are resolved relative to the directory of sys.argv[0]: @@ -2441,7 +2684,7 @@ def test_DocFileSuite(): >>> suite = doctest.DocFileSuite(test_file, module_relative=False) >>> suite.run(unittest.TestResult()) - + It is an error to specify `package` when `module_relative=False`: @@ -2455,12 +2698,19 @@ def test_DocFileSuite(): >>> suite = doctest.DocFileSuite('test_doctest.txt', ... 'test_doctest4.txt', - ... 'test_doctest_skip.txt') + ... 'test_doctest_skip.txt', + ... 'test_doctest_skip2.txt') >>> result = suite.run(unittest.TestResult()) >>> result - - >>> len(result.skipped) - 1 + + >>> len(result.skipped) + 4 + >>> for tst, _ in result.skipped: # doctest: +ELLIPSIS + ... print('=', tst) + = ...test_doctest_skip.txt [0] + = ...test_doctest_skip.txt [1] + = ...test_doctest_skip.txt + = ...test_doctest_skip2.txt [0] You can specify initial global variables: @@ -2469,7 +2719,7 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... globs={'favorite_color': 'blue'}) >>> suite.run(unittest.TestResult()) - + In this case, we supplied a missing favorite color. You can provide doctest options: @@ -2480,7 +2730,7 @@ def test_DocFileSuite(): ... optionflags=doctest.DONT_ACCEPT_BLANKLINE, ... globs={'favorite_color': 'blue'}) >>> suite.run(unittest.TestResult()) - + And, you can provide setUp and tearDown functions: @@ -2499,7 +2749,7 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... setUp=setUp, tearDown=tearDown) >>> suite.run(unittest.TestResult()) - + But the tearDown restores sanity: @@ -2541,9 +2791,60 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... encoding='utf-8') >>> suite.run(unittest.TestResult()) - + + """ - """ +def test_DocFileSuite_errors(): + """Tests for error reporting in DocTestSuite. + + >>> import unittest + >>> suite = doctest.DocFileSuite('test_doctest_errors.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + >...>> 2 + 2 + AssertionError: Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + + >>> print(result.errors[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + >...>> 1/0 + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[1][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + >...>> f() + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + + >>> print(result.errors[2][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + >...>> 2+*3 + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + + + """ def test_trailing_space_in_test(): """ @@ -2612,14 +2913,26 @@ def test_unittest_reportflags(): ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) >>> import unittest >>> result = suite.run(unittest.TestResult()) + >>> result + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback ... - Failed example: - favorite_color - ... - Failed example: + Traceback (most recent call last): + File ... + >...>> if 1: + AssertionError: Failed example: if 1: - ... + print('a') + print() + print('b') + Expected: + a + + b + Got: + a + + b + Note that we see both failures displayed. @@ -2628,16 +2941,8 @@ def test_unittest_reportflags(): Now, when we run the test: - >>> result = suite.run(unittest.TestResult()) - >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback ... - Failed example: - favorite_color - Exception raised: - ... - NameError: name 'favorite_color' is not defined - - + >>> suite.run(unittest.TestResult()) + We get only the first failure. @@ -2647,19 +2952,20 @@ def test_unittest_reportflags(): >>> suite = doctest.DocFileSuite('test_doctest.txt', ... optionflags=doctest.DONT_ACCEPT_BLANKLINE | doctest.REPORT_NDIFF) - Then the default eporting options are ignored: + Then the default reporting options are ignored: >>> result = suite.run(unittest.TestResult()) + >>> result + *NOTE*: These doctest are intentionally not placed in raw string to depict the trailing whitespace using `\x20` in the diff below. >>> print(result.failures[0][1]) # doctest: +ELLIPSIS Traceback ... - Failed example: - favorite_color - ... - Failed example: + File ... + >...>> if 1: + AssertionError: Failed example: if 1: print('a') print() @@ -2670,7 +2976,6 @@ def test_unittest_reportflags(): +\x20 b - Test runners can restore the formatting flags after they run: @@ -2860,6 +3165,57 @@ def test_testfile(): r""" >>> _colorize.COLORIZE = save_colorize """ +def test_testfile_errors(): r""" +Tests for error reporting in the testfile() function. + + >>> doctest.testfile('test_doctest_errors.txt', verbose=False) # doctest: +ELLIPSIS + ********************************************************************** + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ********************************************************************** + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + ********************************************************************** + 1 item had failures: + 4 of 5 in test_doctest_errors.txt + ***Test Failed*** 4 failures. + TestResults(failed=4, attempted=5) +""" + class TestImporter(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path, target=None): @@ -2990,6 +3346,110 @@ def test_testmod(): r""" TestResults(failed=0, attempted=0) """ +def test_testmod_errors(): r""" +Tests for error reporting in the testmod() function. + + >>> import test.test_doctest.sample_doctest_errors as mod + >>> doctest.testmod(mod, verbose=False) # doctest: +ELLIPSIS + ********************************************************************** + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 37, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 39, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ********************************************************************** + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + Failed example: + g() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range + ********************************************************************** + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + ********************************************************************** + 4 items had failures: + 2 of 2 in test.test_doctest.sample_doctest_errors + 2 of 2 in test.test_doctest.sample_doctest_errors.__test__.bad + 4 of 5 in test.test_doctest.sample_doctest_errors.errors + 1 of 1 in test.test_doctest.sample_doctest_errors.syntax_error + ***Test Failed*** 9 failures. + TestResults(failed=9, attempted=10) +""" + try: os.fsencode("foo-bär@baz.py") supports_unicode = True @@ -3021,11 +3481,6 @@ def test_unicode(): """ raise Exception('clé') Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in raise Exception('clé') Exception: clé @@ -3318,9 +3773,9 @@ def test_run_doctestsuite_multiple_times(): >>> import test.test_doctest.sample_doctest >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> suite.run(unittest.TestResult()) - + >>> suite.run(unittest.TestResult()) - + """ diff --git a/Lib/test/test_doctest/test_doctest_errors.txt b/Lib/test/test_doctest/test_doctest_errors.txt new file mode 100644 index 00000000000..93c3c106e60 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_errors.txt @@ -0,0 +1,14 @@ +This is a sample doctest in a text file, in which all examples fail +or raise an exception. + + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> 2+*3 + 5 diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt index f340e2b8141..06c23d06e60 100644 --- a/Lib/test/test_doctest/test_doctest_skip.txt +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -2,3 +2,5 @@ This is a sample doctest in a text file, in which all examples are skipped. >>> 2 + 2 # doctest: +SKIP 5 + >>> 2 + 2 # doctest: +SKIP + 4 diff --git a/Lib/test/test_doctest/test_doctest_skip2.txt b/Lib/test/test_doctest/test_doctest_skip2.txt new file mode 100644 index 00000000000..85e4938c346 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip2.txt @@ -0,0 +1,6 @@ +This is a sample doctest in a text file, in which some examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 + >>> 2 + 2 + 4 diff --git a/Lib/test/test_dynamicclassattribute.py b/Lib/test/test_dynamicclassattribute.py index 9f694d9eb46..b19be33c72f 100644 --- a/Lib/test/test_dynamicclassattribute.py +++ b/Lib/test/test_dynamicclassattribute.py @@ -104,8 +104,8 @@ def test_property_decorator_baseclass(self): self.assertEqual(base.spam, 10) self.assertEqual(base._spam, 10) delattr(base, "spam") - self.assertTrue(not hasattr(base, "spam")) - self.assertTrue(not hasattr(base, "_spam")) + self.assertNotHasAttr(base, "spam") + self.assertNotHasAttr(base, "_spam") base.spam = 20 self.assertEqual(base.spam, 20) self.assertEqual(base._spam, 20) diff --git a/Lib/test/test_email/data/msg_35.txt b/Lib/test/test_email/data/msg_35.txt index be7d5a2f7b9..0e2bbcaf718 100644 --- a/Lib/test/test_email/data/msg_35.txt +++ b/Lib/test/test_email/data/msg_35.txt @@ -1,4 +1,4 @@ From: aperson@dom.ain To: bperson@dom.ain Subject: here's something interesting -counter to RFC 2822, there's no separating newline here +counter to RFC 5322, there's no separating newline here diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index ac12c3b2306..f7f9f9c4e2f 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -463,6 +463,19 @@ def test_get_qp_ctext_non_printables(self): [errors.NonPrintableDefect], ')') self.assertEqual(ptext.defects[0].non_printables[0], '\x00') + def test_get_qp_ctext_close_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + ')', '', ' ', [], ')') + + def test_get_qp_ctext_open_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + '(', '', ' ', [], '(') + + def test_get_qp_ctext_no_end_char(self): + self._test_get_x(parser.get_qp_ctext, + '', '', ' ', [], '') + + # get_qcontent def test_get_qcontent_only(self): @@ -503,6 +516,14 @@ def test_get_qcontent_non_printables(self): [errors.NonPrintableDefect], '"') self.assertEqual(ptext.defects[0].non_printables[0], '\x00') + def test_get_qcontent_empty(self): + self._test_get_x(parser.get_qcontent, + '"', '', '', [], '"') + + def test_get_qcontent_no_end_char(self): + self._test_get_x(parser.get_qcontent, + '', '', '', [], '') + # get_atext def test_get_atext_only(self): @@ -1283,6 +1304,18 @@ def test_get_dtext_open_bracket_mid_word(self): self._test_get_x(parser.get_dtext, 'foo[bar', 'foo', 'foo', [], '[bar') + def test_get_dtext_open_bracket_only(self): + self._test_get_x(parser.get_dtext, + '[', '', '', [], '[') + + def test_get_dtext_close_bracket_only(self): + self._test_get_x(parser.get_dtext, + ']', '', '', [], ']') + + def test_get_dtext_empty(self): + self._test_get_x(parser.get_dtext, + '', '', '', [], '') + # get_domain_literal def test_get_domain_literal_only(self): @@ -2458,6 +2491,38 @@ def test_get_address_quoted_strings_in_atom_list(self): self.assertEqual(address.all_mailboxes[0].domain, 'example.com') self.assertEqual(address.all_mailboxes[0].addr_spec, '"example example"@example.com') + def test_get_address_with_invalid_domain(self): + address = self._test_get_x(parser.get_address, + '', + '', + [errors.InvalidHeaderDefect, # missing trailing '>' on angle-addr + errors.InvalidHeaderDefect, # end of input inside domain-literal + ], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertEqual(address.all_mailboxes[0].domain, '[]') + self.assertEqual(address.all_mailboxes[0].local_part, 'T') + self.assertEqual(address.all_mailboxes[0].token_type, 'invalid-mailbox') + self.assertEqual(address[0].token_type, 'invalid-mailbox') + + address = self._test_get_x(parser.get_address, + '!an??:=m==fr2@[C', + '!an??:=m==fr2@[C];', + '!an??:=m==fr2@[C];', + [errors.InvalidHeaderDefect, # end of header in group + errors.InvalidHeaderDefect, # end of input inside domain-literal + ], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertEqual(address.all_mailboxes[0].domain, '[C]') + self.assertEqual(address.all_mailboxes[0].local_part, '=m==fr2') + self.assertEqual(address.all_mailboxes[0].token_type, 'invalid-mailbox') + self.assertEqual(address[0].token_type, 'group') # get_address_list @@ -2732,6 +2797,19 @@ def test_parse_valid_message_id(self): ) self.assertEqual(message_id.token_type, 'message-id') + def test_parse_message_id_with_invalid_domain(self): + message_id = self._test_parse_x( + parser.parse_message_id, + "", + "", + [errors.ObsoleteHeaderDefect] + [errors.InvalidHeaderDefect] * 2, + [], + ) + self.assertEqual(message_id.token_type, 'message-id') + self.assertEqual(str(message_id.all_defects[-1]), + "end of input inside domain-literal") + def test_parse_message_id_with_remaining(self): message_id = self._test_parse_x( parser.parse_message_id, @@ -3177,5 +3255,15 @@ def test_long_filename_attachment(self): " filename*1*=_TEST_TES.txt\n", ) + def test_fold_unfoldable_element_stealing_whitespace(self): + # gh-142006: When an element is too long to fit on the current line + # the previous line's trailing whitespace should not trigger a double newline. + policy = self.policy.clone(max_line_length=10) + # The non-whitespace text needs to exactly fill the max_line_length (10). + text = ("a" * 9) + ", " + ("b" * 20) + expected = ("a" * 9) + ",\n " + ("b" * 20) + "\n" + token = parser.get_address_list(text)[0] + self._test(token, expected, policy=policy) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_defect_handling.py b/Lib/test/test_email/test_defect_handling.py index 44e76c8ce5e..acc4accccac 100644 --- a/Lib/test/test_email/test_defect_handling.py +++ b/Lib/test/test_email/test_defect_handling.py @@ -126,12 +126,10 @@ def test_multipart_invalid_cte(self): errors.InvalidMultipartContentTransferEncodingDefect) def test_multipart_no_cte_no_defect(self): - if self.raise_expected: return msg = self._str_msg(self.multipart_msg.format('')) self.assertEqual(len(self.get_defects(msg)), 0) def test_multipart_valid_cte_no_defect(self): - if self.raise_expected: return for cte in ('7bit', '8bit', 'BINary'): msg = self._str_msg( self.multipart_msg.format("\nContent-Transfer-Encoding: "+cte)) @@ -300,6 +298,47 @@ def test_missing_ending_boundary(self): self.assertDefectsEqual(self.get_defects(msg), [errors.CloseBoundaryNotFoundDefect]) + def test_line_beginning_colon(self): + string = ( + "Subject: Dummy subject\r\n: faulty header line\r\n\r\nbody\r\n" + ) + + with self._raise_point(errors.InvalidHeaderDefect): + msg = self._str_msg(string) + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertDefectsEqual( + self.get_defects(msg), [errors.InvalidHeaderDefect] + ) + + if msg: + self.assertEqual(msg.items(), [("Subject", "Dummy subject")]) + self.assertEqual(msg.get_payload(), "body\r\n") + + def test_misplaced_envelope(self): + string = ( + "Subject: Dummy subject\r\nFrom wtf\r\nTo: abc\r\n\r\nbody\r\n" + ) + with self._raise_point(errors.MisplacedEnvelopeHeaderDefect): + msg = self._str_msg(string) + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertDefectsEqual( + self.get_defects(msg), [errors.MisplacedEnvelopeHeaderDefect] + ) + + if msg: + headers = [("Subject", "Dummy subject"), ("To", "abc")] + self.assertEqual(msg.items(), headers) + self.assertEqual(msg.get_payload(), "body\r\n") + + + +class TestCompat32(TestDefectsBase, TestEmailBase): + + policy = policy.compat32 + + def get_defects(self, obj): + return obj.defects + class TestDefectDetection(TestDefectsBase, TestEmailBase): @@ -332,6 +371,9 @@ def _raise_point(self, defect): with self.assertRaises(defect): yield + def get_defects(self, obj): + return obj.defects + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 7b14305f997..4e6c213510c 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -41,6 +41,7 @@ from test import support from test.support import threading_helper +from test.support import warnings_helper from test.support.os_helper import unlink from test.test_email import openfile, TestEmailBase @@ -389,6 +390,24 @@ def test_bad_param(self): msg = email.message_from_string("Content-Type: blarg; baz; boo\n") self.assertEqual(msg.get_param('baz'), '') + def test_continuation_sorting_part_order(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"ignored\"; " + "filename*0*=\"utf-8''foo%20\"; " + "filename*1*=\"bar.txt\"\n" + ) + filename = msg.get_filename() + self.assertEqual(filename, 'foo bar.txt') + + def test_sorting_no_continuations(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"bar.txt\"; " + ) + filename = msg.get_filename() + self.assertEqual(filename, 'bar.txt') + def test_missing_filename(self): msg = email.message_from_string("From: foo\n") self.assertEqual(msg.get_filename(), None) @@ -463,6 +482,27 @@ def test_get_param_with_quotes(self): "Content-Type: foo; bar*0=\"baz\\\"foobar\"; bar*1=\"\\\"baz\"") self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') + def test_get_param_linear_complexity(self): + # Ensure that email.message._parseparam() is fast. + # See https://github.com/python/cpython/issues/136063. + N = 100_000 + for s, r in [ + ("", ""), + ("foo=bar", "foo=bar"), + (" FOO = bar ", "foo=bar"), + ]: + with self.subTest(s=s, r=r, N=N): + src = f'{s};' * (N - 1) + s + res = email.message._parseparam(src) + self.assertEqual(len(res), N) + self.assertEqual(len(set(res)), 1) + self.assertEqual(res[0], r) + + # This will be considered as a single parameter. + malformed = 's="' + ';' * (N - 1) + res = email.message._parseparam(malformed) + self.assertEqual(res, [malformed]) + def test_field_containment(self): msg = email.message_from_string('Header: exists') self.assertIn('header', msg) @@ -2223,70 +2263,6 @@ def test_parse_missing_minor_type(self): eq(msg.get_content_maintype(), 'text') eq(msg.get_content_subtype(), 'plain') - # test_defect_handling - def test_same_boundary_inner_outer(self): - msg = self._msgobj('msg_15.txt') - # XXX We can probably eventually do better - inner = msg.get_payload(0) - self.assertHasAttr(inner, 'defects') - self.assertEqual(len(inner.defects), 1) - self.assertIsInstance(inner.defects[0], - errors.StartBoundaryNotFoundDefect) - - # test_defect_handling - def test_multipart_no_boundary(self): - msg = self._msgobj('msg_25.txt') - self.assertIsInstance(msg.get_payload(), str) - self.assertEqual(len(msg.defects), 2) - self.assertIsInstance(msg.defects[0], - errors.NoBoundaryInMultipartDefect) - self.assertIsInstance(msg.defects[1], - errors.MultipartInvariantViolationDefect) - - multipart_msg = textwrap.dedent("""\ - Date: Wed, 14 Nov 2007 12:56:23 GMT - From: foo@bar.invalid - To: foo@bar.invalid - Subject: Content-Transfer-Encoding: base64 and multipart - MIME-Version: 1.0 - Content-Type: multipart/mixed; - boundary="===============3344438784458119861=="{} - - --===============3344438784458119861== - Content-Type: text/plain - - Test message - - --===============3344438784458119861== - Content-Type: application/octet-stream - Content-Transfer-Encoding: base64 - - YWJj - - --===============3344438784458119861==-- - """) - - # test_defect_handling - def test_multipart_invalid_cte(self): - msg = self._str_msg( - self.multipart_msg.format("\nContent-Transfer-Encoding: base64")) - self.assertEqual(len(msg.defects), 1) - self.assertIsInstance(msg.defects[0], - errors.InvalidMultipartContentTransferEncodingDefect) - - # test_defect_handling - def test_multipart_no_cte_no_defect(self): - msg = self._str_msg(self.multipart_msg.format('')) - self.assertEqual(len(msg.defects), 0) - - # test_defect_handling - def test_multipart_valid_cte_no_defect(self): - for cte in ('7bit', '8bit', 'BINary'): - msg = self._str_msg( - self.multipart_msg.format( - "\nContent-Transfer-Encoding: {}".format(cte))) - self.assertEqual(len(msg.defects), 0) - # test_headerregistry.TestContentTypeHeader invalid_1 and invalid_2. def test_invalid_content_type(self): eq = self.assertEqual @@ -2334,7 +2310,7 @@ def test_no_separating_blank_line(self): To: bperson@dom.ain Subject: here's something interesting -counter to RFC 2822, there's no separating newline here +counter to RFC 5322, there's no separating newline here """) # test_defect_handling @@ -2363,30 +2339,6 @@ def test_missing_start_boundary(self): self.assertIsInstance(bad.defects[0], errors.StartBoundaryNotFoundDefect) - # test_defect_handling - def test_first_line_is_continuation_header(self): - eq = self.assertEqual - m = ' Line 1\nSubject: test\n\nbody' - msg = email.message_from_string(m) - eq(msg.keys(), ['Subject']) - eq(msg.get_payload(), 'body') - eq(len(msg.defects), 1) - self.assertDefectsEqual(msg.defects, - [errors.FirstHeaderLineIsContinuationDefect]) - eq(msg.defects[0].line, ' Line 1\n') - - # test_defect_handling - def test_missing_header_body_separator(self): - # Our heuristic if we see a line that doesn't look like a header (no - # leading whitespace but no ':') is to assume that the blank line that - # separates the header from the body is missing, and to stop parsing - # headers and start parsing the body. - msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n') - self.assertEqual(msg.keys(), ['Subject']) - self.assertEqual(msg.get_payload(), 'not a header\nTo: abc\n\nb\n') - self.assertDefectsEqual(msg.defects, - [errors.MissingHeaderBodySeparatorDefect]) - def test_string_payload_with_extra_space_after_cte(self): # https://github.com/python/cpython/issues/98188 cte = "base64 " @@ -2490,49 +2442,49 @@ def test_rfc2047_Q_invalid_digits(self): [(b'andr\xe9=zz', 'iso-8859-1')]) def test_rfc2047_rfc2047_1(self): - # 1st testcase at end of rfc2047 + # 1st testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_2(self): - # 2nd testcase at end of rfc2047 + # 2nd testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= b)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b' b)', None)]) def test_rfc2047_rfc2047_3(self): - # 3rd testcase at end of rfc2047 + # 3rd testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_4(self): - # 4th testcase at end of rfc2047 + # 4th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_5a(self): - # 5th testcase at end of rfc2047 newline is \r\n + # 5th testcase at end of RFC 2047 newline is \r\n s = '(=?ISO-8859-1?Q?a?=\r\n =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_5b(self): - # 5th testcase at end of rfc2047 newline is \n + # 5th testcase at end of RFC 2047 newline is \n s = '(=?ISO-8859-1?Q?a?=\n =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_6(self): - # 6th testcase at end of rfc2047 + # 6th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a_b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a b', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_7(self): - # 7th testcase at end of rfc2047 + # 7th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b' b', 'iso-8859-2'), @@ -2550,6 +2502,18 @@ def test_multiline_header(self): self.assertEqual(str(make_header(decode_header(s))), '"Müller T" ') + def test_unencoded_ascii(self): + # bpo-22833/gh-67022: returns [(str, None)] rather than [(bytes, None)] + s = 'header without encoded words' + self.assertEqual(decode_header(s), + [('header without encoded words', None)]) + + def test_unencoded_utf8(self): + # bpo-22833/gh-67022: returns [(str, None)] rather than [(bytes, None)] + s = 'header with unexpected non ASCII caract\xe8res' + self.assertEqual(decode_header(s), + [('header with unexpected non ASCII caract\xe8res', None)]) + # Test the MIMEMessage class class TestMIMEMessage(TestEmailBase): @@ -3222,8 +3186,8 @@ def test_parsedate_y2k(self): """Test for parsing a date with a two-digit year. Parsing a date with a two-digit year should return the correct - four-digit year. RFC822 allows two-digit years, but RFC2822 (which - obsoletes RFC822) requires four-digit years. + four-digit year. RFC 822 allows two-digit years, but RFC 5322 (which + obsoletes RFC 2822, which obsoletes RFC 822) requires four-digit years. """ self.assertEqual(utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), @@ -3274,7 +3238,7 @@ def test_escape_backslashes(self): self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) def test_quotes_unicode_names(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " @@ -3284,7 +3248,7 @@ def test_quotes_unicode_names(self): latin1_quopri) def test_accepts_any_charset_like_object(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " @@ -3299,7 +3263,7 @@ def header_encode(self, string): utf8_base64) def test_invalid_charset_like_object_raises_error(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' # An object without a header_encode method: @@ -3308,7 +3272,7 @@ def test_invalid_charset_like_object_raises_error(self): bad_charset) def test_unicode_address_raises_error(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. addr = 'pers\u00f6n@dom.in' self.assertRaises(UnicodeError, utils.formataddr, (None, addr)) self.assertRaises(UnicodeError, utils.formataddr, ("Name", addr)) @@ -3329,7 +3293,7 @@ def test_parseaddr_preserves_quoted_pairs_in_addresses(self): # string containing a quoted backslash, followed by 'example' and two # backslashes, followed by another quoted string containing a space and # the word 'example'. parseaddr copies those two backslashes - # literally. Per rfc5322 this is not technically correct since a \ may + # literally. Per RFC 5322 this is not technically correct since a \ may # not appear in an address outside of a quoted string. It is probably # a sensible Postel interpretation, though. eq = self.assertEqual @@ -3341,12 +3305,12 @@ def test_parseaddr_preserves_quoted_pairs_in_addresses(self): ('', '"\\\\"example\\\\" example"@example.com')) def test_parseaddr_preserves_spaces_in_local_part(self): - # issue 9286. A normal RFC5322 local part should not contain any + # issue 9286. A normal RFC 5322 local part should not contain any # folding white space, but legacy local parts can (they are a sequence # of atoms, not dotatoms). On the other hand we strip whitespace from # before the @ and around dots, on the assumption that the whitespace # around the punctuation is a mistake in what would otherwise be - # an RFC5322 local part. Leading whitespace is, usual, stripped as well. + # an RFC 5322 local part. Leading whitespace is, usual, stripped as well. self.assertEqual(('', "merwok wok@xample.com"), utils.parseaddr("merwok wok@xample.com")) self.assertEqual(('', "merwok wok@xample.com"), @@ -5687,7 +5651,8 @@ def test_rfc2231_bad_character_in_encoding(self): """ msg = email.message_from_string(m) - self.assertEqual(msg.get_filename(), 'myfile.txt') + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.assertEqual(msg.get_filename(), 'myfile.txt') def test_rfc2231_single_tick_in_filename_extended(self): eq = self.assertEqual diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index ff7a6da644d..7138aa4c556 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -8,6 +8,7 @@ from email import headerregistry from email.headerregistry import Address, Group from test.support import ALWAYS_EQ +from test.support import warnings_helper DITTO = object() @@ -247,7 +248,15 @@ def content_type_as_value(self, decoded = args[2] if l>2 and args[2] is not DITTO else source header = 'Content-Type:' + ' ' if source else '' folded = args[3] if l>3 else header + decoded + '\n' - h = self.make_header('Content-Type', source) + # Both rfc2231 test cases with utf-8%E2%80%9D raise warnings, + # clear encoding cache to ensure test isolation. + if 'utf-8%E2%80%9D' in source and 'ascii' not in source: + import encodings + encodings._cache.clear() + with warnings_helper.check_warnings(('', DeprecationWarning)): + h = self.make_header('Content-Type', source) + else: + h = self.make_header('Content-Type', source) self.assertEqual(h.content_type, content_type) self.assertEqual(h.maintype, maintype) self.assertEqual(h.subtype, subtype) diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 23c39775a8b..56ad446694d 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -1004,6 +1004,32 @@ def test_folding_with_long_nospace_http_policy_1(self): parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + def test_no_wrapping_max_line_length(self): + # Test that falsey 'max_line_length' are converted to sys.maxsize. + for n in [0, None]: + with self.subTest(max_line_length=n): + self.do_test_no_wrapping_max_line_length(n) + + def do_test_no_wrapping_max_line_length(self, falsey): + self.assertFalse(falsey) + pol = policy.default.clone(max_line_length=falsey) + subj = "S" * 100 + body = "B" * 100 + msg = EmailMessage(policy=pol) + msg["From"] = "a@ex.com" + msg["To"] = "b@ex.com" + msg["Subject"] = subj + msg.set_content(body) + + raw = msg.as_bytes() + self.assertNotIn(b"=\n", raw, + "Found fold indicator; wrapping not disabled") + + parsed = message_from_bytes(raw, policy=policy.default) + self.assertEqual(parsed["Subject"], subj) + parsed_body = parsed.get_body().get_content().rstrip('\n') + self.assertEqual(parsed_body, body) + def test_invalid_header_names(self): invalid_headers = [ ('Invalid Header', 'contains space'), @@ -1055,6 +1081,15 @@ def test_get_body_malformed(self): # AttributeError: 'str' object has no attribute 'is_attachment' m.get_body() + def test_get_bytes_payload_with_quoted_printable_encoding(self): + # We use a memoryview to avoid directly changing the private payload + # and to prevent using the dedicated paths for string or bytes objects. + payload = memoryview(b'Some payload') + m = self._make_message() + m.add_header('Content-Transfer-Encoding', 'quoted-printable') + m.set_payload(payload) + self.assertEqual(m.get_payload(decode=True), payload) + class TestMIMEPart(TestEmailMessageBase, TestEmailBase): # Doing the full test run here may seem a bit redundant, since the two diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 95b2d80464c..b5367941227 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -239,6 +239,37 @@ def test_repeated_init_and_inittab(self): lines = "\n".join(lines) + "\n" self.assertEqual(out, lines) + def test_create_module_from_initfunc(self): + out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") + self.assertEqual(self._nogil_filtered_err(err, "embedded_ext"), "") + self.assertEqual(out, + "\n" + "my_test_extension.executed='yes'\n" + "my_test_extension.exec_slot_ran='yes'\n" + "\n" + "embedded_ext.executed='yes'\n" + ) + + def test_inittab_submodule_multiphase(self): + out, err = self.run_embedded_interpreter("test_inittab_submodule_multiphase") + self.assertEqual(err, "") + self.assertEqual(out, + "\n" + "\n" + "Hello from sub-module\n" + "mp_pkg.mp_submod.mp_submod_exec_slot_ran='yes'\n" + "mp_pkg.mp_pkg_exec_slot_ran='yes'\n" + ) + + def test_inittab_submodule_singlephase(self): + out, err = self.run_embedded_interpreter("test_inittab_submodule_singlephase") + self.assertEqual(self._nogil_filtered_err(err, "sp_pkg"), "") + self.assertEqual(out, + "\n" + "\n" + "Hello from sub-module\n" + ) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") @@ -296,7 +327,7 @@ def test_pre_initialization_api(self): if MS_WINDOWS: expected_path = self.test_exe else: - expected_path = os.path.join(os.getcwd(), "spam") + expected_path = os.path.join(os.getcwd(), "_testembed") expected_output = f"sys.executable: {expected_path}\n" self.assertIn(expected_output, out) self.assertEqual(err, '') @@ -516,6 +547,24 @@ def test_getargs_reset_static_parser(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '1\n2\n3\n' * INIT_LOOPS) + @staticmethod + def _nogil_filtered_err(err: str, mod_name: str) -> str: + if not support.Py_GIL_DISABLED: + return err + + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + f" has been enabled to load module '{mod_name}'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + return "\n".join(filtered_err_lines) + def config_dev_mode(preconfig, config): preconfig['allocator'] = PYMEM_ALLOCATOR_DEBUG @@ -543,7 +592,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'configure_locale': True, 'coerce_c_locale': False, 'coerce_c_locale_warn': False, - 'utf8_mode': False, + 'utf8_mode': True, } if MS_WINDOWS: PRE_CONFIG_COMPAT.update({ @@ -560,7 +609,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): configure_locale=False, isolated=True, use_environment=False, - utf8_mode=False, + utf8_mode=True, dev_mode=False, coerce_c_locale=False, ) @@ -805,12 +854,6 @@ def get_expected_config(self, expected_preconfig, expected, 'stdio_encoding', 'stdio_errors'): expected[key] = self.IGNORE_CONFIG - if not expected_preconfig['configure_locale']: - # UTF-8 Mode depends on the locale. There is no easy way - # to guess if UTF-8 Mode will be enabled or not if the locale - # is not configured. - expected_preconfig['utf8_mode'] = self.IGNORE_CONFIG - if expected_preconfig['utf8_mode'] == 1: if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG: expected['filesystem_encoding'] = 'utf-8' @@ -969,7 +1012,6 @@ def test_init_global_config(self): 'utf8_mode': True, } config = { - 'program_name': './globalvar', 'site_import': False, 'bytes_warning': True, 'warnoptions': ['default::BytesWarning'], @@ -1239,21 +1281,6 @@ def test_init_dont_configure_locale(self): self.check_all_configs("test_init_dont_configure_locale", {}, preconfig, api=API_PYTHON) - @unittest.skip('as of 3.11 this test no longer works because ' - 'path calculations do not occur on read') - def test_init_read_set(self): - config = { - 'program_name': './init_read_set', - 'executable': 'my_executable', - 'base_executable': 'my_executable', - } - def modify_path(path): - path.insert(1, "test_path_insert1") - path.append("test_path_append") - self.check_all_configs("test_init_read_set", config, - api=API_PYTHON, - modify_path_cb=modify_path) - def test_init_sys_add(self): config = { 'faulthandler': 1, @@ -1916,6 +1943,10 @@ def test_get_incomplete_frame(self): self.run_embedded_interpreter("test_get_incomplete_frame") + def test_gilstate_after_finalization(self): + self.run_embedded_interpreter("test_gilstate_after_finalization") + + class MiscTests(EmbeddingTestsMixin, unittest.TestCase): def test_unicode_id_init(self): # bpo-42882: Test that _PyUnicode_FromId() works diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 6d3c91b0b6d..f6743d57ca2 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -60,6 +60,11 @@ def setUp(self): self.run_pip.return_value = 0 self.addCleanup(run_pip_patch.stop) + # Allow testing on zlib-less platforms by avoiding the check for zlib in _bootstrap() + zlib_patch = unittest.mock.patch.dict('sys.modules', {'zlib': unittest.mock.MagicMock()}) + zlib_patch.start() + self.addCleanup(zlib_patch.stop) + # Avoid side effects on the actual os module real_devnull = os.devnull os_patch = unittest.mock.patch("ensurepip.os") @@ -185,6 +190,16 @@ def test_pip_config_file_disabled(self): ensurepip.bootstrap() self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull) + def test_missing_zlib(self): + with unittest.mock.patch.dict('sys.modules', {'zlib': None}): + with self.assertRaises(ModuleNotFoundError) as cm: + ensurepip.bootstrap() + + error_msg = str(cm.exception) + self.assertIn("ensurepip requires the standard library module 'zlib'", error_msg) + + self.assertFalse(self.run_pip.called) + @contextlib.contextmanager def fake_pip(version=ensurepip.version()): if version is None: diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index d8cb5261244..66d78980c41 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -36,7 +36,7 @@ def load_tests(loader, tests, ignore): optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) howto_tests = os.path.join(REPO_ROOT, 'Doc/howto/enum.rst') - if os.path.exists(howto_tests): + if os.path.exists(howto_tests) and sys.float_repr_style == 'short': tests.addTests(doctest.DocFileSuite( howto_tests, module_relative=False, @@ -434,9 +434,9 @@ class Season(self.enum_type): def spam(cls): pass # - self.assertTrue(hasattr(Season, 'spam')) + self.assertHasAttr(Season, 'spam') del Season.spam - self.assertFalse(hasattr(Season, 'spam')) + self.assertNotHasAttr(Season, 'spam') # with self.assertRaises(AttributeError): del Season.SPRING @@ -1002,12 +1002,18 @@ class OpenAB(self.enum_type): self.assertIs(~(A|B), OpenAB(252)) self.assertIs(~AB_MASK, OpenAB(0)) self.assertIs(~OpenAB(0), AB_MASK) + self.assertIs(OpenAB(~4), OpenAB(251)) else: self.assertIs(~A, B) self.assertIs(~B, A) + self.assertIs(OpenAB(~1), B) + self.assertIs(OpenAB(~2), A) self.assertIs(~(A|B), OpenAB(0)) self.assertIs(~AB_MASK, OpenAB(0)) self.assertIs(~OpenAB(0), (A|B)) + self.assertIs(OpenAB(~3), OpenAB(0)) + self.assertIs(OpenAB(~4), OpenAB(3)) + self.assertIs(OpenAB(~33), B) # class OpenXYZ(self.enum_type): X = 4 @@ -1031,6 +1037,9 @@ class OpenXYZ(self.enum_type): self.assertIs(~X, Y|Z) self.assertIs(~Y, X|Z) self.assertIs(~Z, X|Y) + self.assertIs(OpenXYZ(~4), Y|Z) + self.assertIs(OpenXYZ(~2), X|Z) + self.assertIs(OpenXYZ(~1), X|Y) self.assertIs(~(X|Y), Z) self.assertIs(~(X|Z), Y) self.assertIs(~(Y|Z), X) @@ -1038,6 +1047,28 @@ class OpenXYZ(self.enum_type): self.assertIs(~XYZ_MASK, OpenXYZ(0)) self.assertTrue(~OpenXYZ(0), (X|Y|Z)) + def test_assigned_negative_value(self): + class X(self.enum_type): + A = auto() + B = auto() + C = A | B + D = ~A + self.assertEqual(list(X), [X.A, X.B]) + self.assertIs(~X.A, X.B) + self.assertIs(X.D, X.B) + self.assertEqual(X.D.value, 2) + # + class Y(self.enum_type): + A = auto() + B = auto() + C = A | B + D = ~A + E = auto() + self.assertEqual(list(Y), [Y.A, Y.B, Y.E]) + self.assertIs(~Y.A, Y.B|Y.E) + self.assertIs(Y.D, Y.B|Y.E) + self.assertEqual(Y.D.value, 6) + class TestPlainEnumClass(_EnumTests, _PlainOutputTests, unittest.TestCase): enum_type = Enum @@ -2652,12 +2683,12 @@ def __new__(cls, value, period): OneDay = day_1 OneWeek = week_1 OneMonth = month_1 - self.assertFalse(hasattr(Period, '_ignore_')) - self.assertFalse(hasattr(Period, 'Period')) - self.assertFalse(hasattr(Period, 'i')) - self.assertTrue(isinstance(Period.day_1, timedelta)) - self.assertTrue(Period.month_1 is Period.day_30) - self.assertTrue(Period.week_4 is Period.day_28) + self.assertNotHasAttr(Period, '_ignore_') + self.assertNotHasAttr(Period, 'Period') + self.assertNotHasAttr(Period, 'i') + self.assertIsInstance(Period.day_1, timedelta) + self.assertIs(Period.month_1, Period.day_30) + self.assertIs(Period.week_4, Period.day_28) def test_nonhash_value(self): class AutoNumberInAList(Enum): @@ -2877,7 +2908,7 @@ class ReformedColor(StrMixin, IntEnum, SomeEnum, AnotherEnum): self.assertEqual(str(ReformedColor.BLUE), 'blue') self.assertEqual(ReformedColor.RED.behavior(), 'booyah') self.assertEqual(ConfusedColor.RED.social(), "what's up?") - self.assertTrue(issubclass(ReformedColor, int)) + self.assertIsSubclass(ReformedColor, int) def test_multiple_inherited_mixin(self): @unique @@ -3680,6 +3711,8 @@ class SkipFlag(enum.Flag): C = 4 | B # self.assertTrue(SkipFlag.C in (SkipFlag.A|SkipFlag.C)) + self.assertTrue(SkipFlag.B in SkipFlag.C) + self.assertIs(SkipFlag(~1), SkipFlag.B) self.assertRaisesRegex(ValueError, 'SkipFlag.. invalid value 42', SkipFlag, 42) # class SkipIntFlag(enum.IntFlag): @@ -3688,6 +3721,8 @@ class SkipIntFlag(enum.IntFlag): C = 4 | B # self.assertTrue(SkipIntFlag.C in (SkipIntFlag.A|SkipIntFlag.C)) + self.assertTrue(SkipIntFlag.B in SkipIntFlag.C) + self.assertIs(SkipIntFlag(~1), SkipIntFlag.B|SkipIntFlag.C) self.assertEqual(SkipIntFlag(42).value, 42) # class MethodHint(Flag): @@ -4727,6 +4762,8 @@ class Color(Flag): BLUE = 4 WHITE = -1 # no error means success + self.assertEqual(list(Color.WHITE), [Color.RED, Color.GREEN, Color.BLUE]) + self.assertEqual(Color.WHITE.value, 7) class TestInternals(unittest.TestCase): @@ -4850,7 +4887,7 @@ class Color(Enum): def _generate_next_value_(name, start, count, last): return name - def test_auto_order_wierd(self): + def test_auto_order_weird(self): weird_auto = auto() weird_auto.value = 'pathological case' class Color(Enum): @@ -5287,7 +5324,7 @@ def __new__(cls, value, label): class MiscTestCase(unittest.TestCase): def test__all__(self): - support.check__all__(self, enum, not_exported={'bin', 'show_flag_values'}) + support.check__all__(self, enum) @cpython_only def test_lazy_import(self): diff --git a/Lib/test/test_errno.py b/Lib/test/test_errno.py index 5c437e9ccea..e7f185c6b1a 100644 --- a/Lib/test/test_errno.py +++ b/Lib/test/test_errno.py @@ -12,14 +12,12 @@ class ErrnoAttributeTests(unittest.TestCase): def test_for_improper_attributes(self): # No unexpected attributes should be on the module. for error_code in std_c_errors: - self.assertTrue(hasattr(errno, error_code), - "errno is missing %s" % error_code) + self.assertHasAttr(errno, error_code) def test_using_errorcode(self): # Every key value in errno.errorcode should be on the module. for value in errno.errorcode.values(): - self.assertTrue(hasattr(errno, value), - 'no %s attr in errno' % value) + self.assertHasAttr(errno, value) class ErrorcodeTests(unittest.TestCase): diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 92bbf791764..ace7ec72917 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,13 +1,13 @@ -import collections.abc +import collections import types import unittest -from test.support import skip_emscripten_stack_overflow, exceeds_recursion_limit +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): - self.assertTrue(issubclass(ExceptionGroup, Exception)) - self.assertTrue(issubclass(ExceptionGroup, BaseExceptionGroup)) - self.assertTrue(issubclass(BaseExceptionGroup, BaseException)) + self.assertIsSubclass(ExceptionGroup, Exception) + self.assertIsSubclass(ExceptionGroup, BaseExceptionGroup) + self.assertIsSubclass(BaseExceptionGroup, BaseException) def test_exception_is_not_generic_type(self): with self.assertRaisesRegex(TypeError, 'Exception'): @@ -193,6 +193,77 @@ class MyEG(ExceptionGroup): "MyEG('flat', [ValueError(1), TypeError(2)]), " "TypeError(2)])")) + def test_exceptions_mutation(self): + class MyEG(ExceptionGroup): + pass + + excs = [ValueError(1), TypeError(2)] + eg = MyEG('test', excs) + + self.assertEqual(repr(eg), "MyEG('test', [ValueError(1), TypeError(2)])") + excs.clear() + + # Ensure that clearing the exceptions sequence doesn't change the repr. + self.assertEqual(repr(eg), "MyEG('test', [ValueError(1), TypeError(2)])") + + # Ensure that the args are still as passed. + self.assertEqual(eg.args, ('test', [])) + + excs = (ValueError(1), KeyboardInterrupt(2)) + eg = BaseExceptionGroup('test', excs) + + # Ensure that immutable sequences still work fine. + self.assertEqual( + repr(eg), + "BaseExceptionGroup('test', (ValueError(1), KeyboardInterrupt(2)))" + ) + + # Test non-standard custom sequences. + excs = collections.deque([ValueError(1), TypeError(2)]) + eg = ExceptionGroup('test', excs) + + self.assertEqual( + repr(eg), + "ExceptionGroup('test', deque([ValueError(1), TypeError(2)]))" + ) + excs.clear() + + # Ensure that clearing the exceptions sequence doesn't change the repr. + self.assertEqual( + repr(eg), + "ExceptionGroup('test', deque([ValueError(1), TypeError(2)]))" + ) + + def test_repr_raises(self): + class MySeq(collections.abc.Sequence): + def __init__(self, raises): + self.raises = raises + + def __len__(self): + return 1 + + def __getitem__(self, index): + if index == 0: + return ValueError(1) + raise IndexError + + def __repr__(self): + if self.raises: + raise self.raises + return None + + seq = MySeq(None) + with self.assertRaisesRegex( + TypeError, + r".*MySeq\.__repr__\(\) must return a str, not NoneType" + ): + ExceptionGroup("test", seq) + + seq = MySeq(ValueError) + with self.assertRaises(ValueError): + BaseExceptionGroup("test", seq) + + def create_simple_eg(): excs = [] @@ -465,12 +536,14 @@ def make_deep_eg(self): return e @skip_emscripten_stack_overflow() + @skip_wasi_stack_overflow() def test_deep_split(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): e.split(TypeError) @skip_emscripten_stack_overflow() + @skip_wasi_stack_overflow() def test_deep_subgroup(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): @@ -812,8 +885,8 @@ def test_split_does_not_copy_non_sequence_notes(self): eg = ExceptionGroup("eg", [ValueError(1), TypeError(2)]) eg.__notes__ = 123 match, rest = eg.split(TypeError) - self.assertFalse(hasattr(match, '__notes__')) - self.assertFalse(hasattr(rest, '__notes__')) + self.assertNotHasAttr(match, '__notes__') + self.assertNotHasAttr(rest, '__notes__') def test_drive_invalid_return_value(self): class MyEg(ExceptionGroup): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index d177e3dc0f5..6f212d2f91e 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -224,6 +224,8 @@ def check(self, src, lineno, offset, end_lineno=None, end_offset=None, encoding= if not isinstance(src, str): src = src.decode(encoding, 'replace') line = src.split('\n')[lineno-1] + if lineno == 1: + line = line.removeprefix('\ufeff') self.assertIn(line, cm.exception.text) def test_error_offset_continuation_characters(self): @@ -239,7 +241,9 @@ def testSyntaxErrorOffset(self): check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) check(b'# -*- coding: cp1251 -*-\nPython = "\xcf\xb3\xf2\xee\xed" +', 2, 19, encoding='cp1251') - check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 10) + check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 12) + check(b'\n\n\nPython = "\xcf\xb3\xf2\xee\xed" +', 4, 12) + check(b'\xef\xbb\xbfPython = "\xcf\xb3\xf2\xee\xed" +', 1, 12) check('x = "a', 1, 5) check('lambda x: x = 2', 1, 1) check('f{a + b + c}', 1, 2) @@ -248,7 +252,16 @@ def testSyntaxErrorOffset(self): check('[\nfile\nfor str(file)\nin\n[]\n]', 3, 5) check('[file for\n str(file) in []]', 2, 2) check("ages = {'Alice'=22, 'Bob'=23}", 1, 9) - check('match ...:\n case {**rest, "key": value}:\n ...', 2, 19) + check(dedent("""\ + match ...: + case {**rest1, "after": after}: + ... + """), 2, 11) + check(dedent("""\ + match ...: + case {"before": before, **rest2, "after": after}: + ... + """), 2, 29) check("[a b c d e f]", 1, 2) check("for x yfff:", 1, 7) check("f(a for a in b, c)", 1, 3, 1, 15) @@ -287,7 +300,7 @@ def baz(): check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) check("(1+)", 1, 4) check("[interesting\nfoo()\n", 1, 1) - check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1) + check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 1, 0) check("""f''' { (123_a) @@ -357,7 +370,7 @@ def test_capi1(): except TypeError as err: co = err.__traceback__.tb_frame.f_code self.assertEqual(co.co_name, "test_capi1") - self.assertTrue(co.co_filename.endswith('test_exceptions.py')) + self.assertEndsWith(co.co_filename, 'test_exceptions.py') else: self.fail("Expected exception") @@ -369,7 +382,7 @@ def test_capi2(): tb = err.__traceback__.tb_next co = tb.tb_frame.f_code self.assertEqual(co.co_name, "__init__") - self.assertTrue(co.co_filename.endswith('test_exceptions.py')) + self.assertEndsWith(co.co_filename, 'test_exceptions.py') co2 = tb.tb_frame.f_back.f_code self.assertEqual(co2.co_name, "test_capi2") else: @@ -598,7 +611,7 @@ def test_invalid_setstate(self): def test_notes(self): for e in [BaseException(1), Exception(2), ValueError(3)]: with self.subTest(e=e): - self.assertFalse(hasattr(e, '__notes__')) + self.assertNotHasAttr(e, '__notes__') e.add_note("My Note") self.assertEqual(e.__notes__, ["My Note"]) @@ -610,7 +623,7 @@ def test_notes(self): self.assertEqual(e.__notes__, ["My Note", "Your Note"]) del e.__notes__ - self.assertFalse(hasattr(e, '__notes__')) + self.assertNotHasAttr(e, '__notes__') e.add_note("Our Note") self.assertEqual(e.__notes__, ["Our Note"]) @@ -1429,6 +1442,7 @@ def g(): self.assertIn("maximum recursion depth exceeded", str(exc)) @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @cpython_only @support.requires_resource('cpu') def test_trashcan_recursion(self): @@ -1444,6 +1458,7 @@ def foo(): foo() support.gc_collect() + @support.skip_emscripten_stack_overflow() @cpython_only def test_recursion_normalizing_exception(self): import_module("_testinternalcapi") @@ -1521,6 +1536,7 @@ def test_recursion_normalizing_infinite_exception(self): self.assertIn(b'Done.', out) + @support.skip_emscripten_stack_overflow() def test_recursion_in_except_handler(self): def set_relative_recursion_limit(n): @@ -1626,7 +1642,7 @@ def test_exception_with_doc(self): # test basic usage of PyErr_NewException error1 = _testcapi.make_exception_with_doc("_testcapi.error1") self.assertIs(type(error1), type) - self.assertTrue(issubclass(error1, Exception)) + self.assertIsSubclass(error1, Exception) self.assertIsNone(error1.__doc__) # test with given docstring @@ -1636,21 +1652,21 @@ def test_exception_with_doc(self): # test with explicit base (without docstring) error3 = _testcapi.make_exception_with_doc("_testcapi.error3", base=error2) - self.assertTrue(issubclass(error3, error2)) + self.assertIsSubclass(error3, error2) # test with explicit base tuple class C(object): pass error4 = _testcapi.make_exception_with_doc("_testcapi.error4", doc4, (error3, C)) - self.assertTrue(issubclass(error4, error3)) - self.assertTrue(issubclass(error4, C)) + self.assertIsSubclass(error4, error3) + self.assertIsSubclass(error4, C) self.assertEqual(error4.__doc__, doc4) # test with explicit dictionary error5 = _testcapi.make_exception_with_doc("_testcapi.error5", "", error4, {'a': 1}) - self.assertTrue(issubclass(error5, error4)) + self.assertIsSubclass(error5, error4) self.assertEqual(error5.a, 1) self.assertEqual(error5.__doc__, "") @@ -1743,7 +1759,7 @@ def test_unhandled(self): self.assertIn("", report) else: self.assertIn("test message", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") @cpython_only # Python built with Py_TRACE_REFS fail with a fatal error in @@ -1907,6 +1923,39 @@ def test_keyerror_context(self): exc2 = None + @cpython_only + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_exec_set_nomemory_hang(self): + import_module("_testcapi") + # gh-134163: A MemoryError inside code that was wrapped by a try/except + # block would lead to an infinite loop. + + # The frame_lasti needs to be greater than 257 to prevent + # PyLong_FromLong() from returning cached integers, which + # don't require a memory allocation. Prepend some dummy code + # to artificially increase the instruction index. + warmup_code = "a = list(range(0, 1))\n" * 60 + user_input = warmup_code + dedent(""" + try: + import _testcapi + _testcapi.set_nomemory(0) + b = list(range(1000, 2000)) + except Exception as e: + import traceback + traceback.print_exc() + """) + with SuppressCrashReport(): + with script_helper.spawn_python('-c', user_input) as p: + p.wait() + output = p.stdout.read() + + self.assertIn(p.returncode, (0, 1)) + self.assertGreater(len(output), 0) # At minimum, should not hang + self.assertIn(b"MemoryError", output) + + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): try: @@ -2076,6 +2125,50 @@ def test_copy_pickle(self): self.assertEqual(exc.name, orig.name) self.assertEqual(exc.path, orig.path) + def test_repr(self): + exc = ImportError() + self.assertEqual(repr(exc), "ImportError()") + + exc = ImportError('test') + self.assertEqual(repr(exc), "ImportError('test')") + + exc = ImportError('test', 'case') + self.assertEqual(repr(exc), "ImportError('test', 'case')") + + exc = ImportError(name='somemodule') + self.assertEqual(repr(exc), "ImportError(name='somemodule')") + + exc = ImportError('test', name='somemodule') + self.assertEqual(repr(exc), "ImportError('test', name='somemodule')") + + exc = ImportError(path='somepath') + self.assertEqual(repr(exc), "ImportError(path='somepath')") + + exc = ImportError('test', path='somepath') + self.assertEqual(repr(exc), "ImportError('test', path='somepath')") + + exc = ImportError(name='somename', path='somepath') + self.assertEqual(repr(exc), + "ImportError(name='somename', path='somepath')") + + exc = ImportError('test', name='somename', path='somepath') + self.assertEqual(repr(exc), + "ImportError('test', name='somename', path='somepath')") + + exc = ModuleNotFoundError('test', name='somename', path='somepath') + self.assertEqual(repr(exc), + "ModuleNotFoundError('test', name='somename', path='somepath')") + + def test_ModuleNotFoundError_repr_with_failed_import(self): + with self.assertRaises(ModuleNotFoundError) as cm: + import does_not_exist # type: ignore[import] # noqa: F401 + + self.assertEqual(cm.exception.name, "does_not_exist") + self.assertIsNone(cm.exception.path) + + self.assertEqual(repr(cm.exception), + "ModuleNotFoundError(\"No module named 'does_not_exist'\", name='does_not_exist')") + def run_script(source): if isinstance(source, str): diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index d9d85fe79af..f003a5837ae 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -137,7 +137,7 @@ >>> g(*Nothing()) Traceback (most recent call last): ... - TypeError: test.test_extcall.g() argument after * must be an iterable, not Nothing + TypeError: Value after * must be an iterable, not Nothing >>> class Nothing: ... def __len__(self): return 5 @@ -146,7 +146,7 @@ >>> g(*Nothing()) Traceback (most recent call last): ... - TypeError: test.test_extcall.g() argument after * must be an iterable, not Nothing + TypeError: Value after * must be an iterable, not Nothing >>> class Nothing(): ... def __len__(self): return 5 @@ -266,7 +266,7 @@ >>> h(*h) Traceback (most recent call last): ... - TypeError: test.test_extcall.h() argument after * must be an iterable, not function + TypeError: Value after * must be an iterable, not function >>> h(1, *h) Traceback (most recent call last): @@ -281,55 +281,53 @@ >>> dir(*h) Traceback (most recent call last): ... - TypeError: dir() argument after * must be an iterable, not function + TypeError: Value after * must be an iterable, not function >>> nothing = None >>> nothing(*h) Traceback (most recent call last): ... - TypeError: None argument after * must be an iterable, \ -not function + TypeError: Value after * must be an iterable, not function >>> h(**h) Traceback (most recent call last): ... - TypeError: test.test_extcall.h() argument after ** must be a mapping, not function + TypeError: Value after ** must be a mapping, not function >>> h(**[]) Traceback (most recent call last): ... - TypeError: test.test_extcall.h() argument after ** must be a mapping, not list + TypeError: Value after ** must be a mapping, not list >>> h(a=1, **h) Traceback (most recent call last): ... - TypeError: test.test_extcall.h() argument after ** must be a mapping, not function + TypeError: Value after ** must be a mapping, not function >>> h(a=1, **[]) Traceback (most recent call last): ... - TypeError: test.test_extcall.h() argument after ** must be a mapping, not list + TypeError: Value after ** must be a mapping, not list >>> h(**{'a': 1}, **h) Traceback (most recent call last): ... - TypeError: test.test_extcall.h() argument after ** must be a mapping, not function + TypeError: Value after ** must be a mapping, not function >>> h(**{'a': 1}, **[]) Traceback (most recent call last): ... - TypeError: test.test_extcall.h() argument after ** must be a mapping, not list + TypeError: Value after ** must be a mapping, not list >>> dir(**h) Traceback (most recent call last): ... - TypeError: dir() argument after ** must be a mapping, not function + TypeError: Value after ** must be a mapping, not function >>> nothing(**h) Traceback (most recent call last): ... - TypeError: None argument after ** must be a mapping, \ -not function + TypeError: Value after ** must be a mapping, not function >>> dir(b=1, **{'b': 1}) Traceback (most recent call last): diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index ad3f669a030..a97242483a8 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -1,26 +1,61 @@ import unittest import os import textwrap +import contextlib import importlib import sys import socket -from asyncio import staggered, taskgroups +import threading +import time +from contextlib import contextmanager +from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY -from test.support import os_helper, SHORT_TIMEOUT, busy_retry +from test.support import ( + os_helper, + SHORT_TIMEOUT, + busy_retry, + requires_gil_enabled, +) from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port import subprocess +# Profiling mode constants +PROFILING_MODE_WALL = 0 +PROFILING_MODE_CPU = 1 +PROFILING_MODE_GIL = 2 +PROFILING_MODE_ALL = 3 + +# Thread status flags +THREAD_STATUS_HAS_GIL = 1 << 0 +THREAD_STATUS_ON_CPU = 1 << 1 +THREAD_STATUS_UNKNOWN = 1 << 2 + +# Maximum number of retry attempts for operations that may fail transiently +MAX_TRIES = 10 + +try: + from concurrent import interpreters +except ImportError: + interpreters = None + PROCESS_VM_READV_SUPPORTED = False try: from _remote_debugging import PROCESS_VM_READV_SUPPORTED - from _remote_debugging import get_stack_trace - from _remote_debugging import get_async_stack_trace - from _remote_debugging import get_all_awaited_by + from _remote_debugging import RemoteUnwinder + from _remote_debugging import FrameInfo, CoroInfo, TaskInfo except ImportError: - raise unittest.SkipTest("Test only runs when _remote_debugging is available") + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + + +# ============================================================================ +# Module-level helper functions +# ============================================================================ + def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) @@ -28,26 +63,340 @@ def _make_test_script(script_dir, script_basename, source): return to_return +def _create_server_socket(port, backlog=1): + """Create and configure a server socket for test communication.""" + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(backlog) + return server_socket + + +def _wait_for_signal(sock, expected_signals, timeout=SHORT_TIMEOUT): + """ + Wait for expected signal(s) from a socket with proper timeout and EOF handling. + + Args: + sock: Connected socket to read from + expected_signals: Single bytes object or list of bytes objects to wait for + timeout: Socket timeout in seconds + + Returns: + bytes: Complete accumulated response buffer + + Raises: + RuntimeError: If connection closed before signal received or timeout + """ + if isinstance(expected_signals, bytes): + expected_signals = [expected_signals] + + sock.settimeout(timeout) + buffer = b"" + + while True: + # Check if all expected signals are in buffer + if all(sig in buffer for sig in expected_signals): + return buffer + + try: + chunk = sock.recv(4096) + if not chunk: + # EOF - connection closed + raise RuntimeError( + f"Connection closed before receiving expected signals. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) + buffer += chunk + except socket.timeout: + raise RuntimeError( + f"Timeout waiting for signals. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) + + +def _wait_for_n_signals(sock, signal_pattern, count, timeout=SHORT_TIMEOUT): + """ + Wait for N occurrences of a signal pattern. + + Args: + sock: Connected socket to read from + signal_pattern: bytes pattern to count (e.g., b"ready") + count: Number of occurrences expected + timeout: Socket timeout in seconds + + Returns: + bytes: Complete accumulated response buffer + + Raises: + RuntimeError: If connection closed or timeout before receiving all signals + """ + sock.settimeout(timeout) + buffer = b"" + found_count = 0 + + while found_count < count: + try: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError( + f"Connection closed after {found_count}/{count} signals. " + f"Last 200 bytes: {buffer[-200:]!r}" + ) + buffer += chunk + # Count occurrences in entire buffer + found_count = buffer.count(signal_pattern) + except socket.timeout: + raise RuntimeError( + f"Timeout waiting for {count} signals (found {found_count}). " + f"Last 200 bytes: {buffer[-200:]!r}" + ) + + return buffer + + +@contextmanager +def _managed_subprocess(args, timeout=SHORT_TIMEOUT): + """ + Context manager for subprocess lifecycle management. + + Ensures process is properly terminated and cleaned up even on exceptions. + Uses graceful termination first, then forceful kill if needed. + """ + p = subprocess.Popen(args) + try: + yield p + finally: + try: + p.terminate() + try: + p.wait(timeout=timeout) + except subprocess.TimeoutExpired: + p.kill() + try: + p.wait(timeout=timeout) + except subprocess.TimeoutExpired: + pass # Process refuses to die, nothing more we can do + except OSError: + pass # Process already dead + + +def _cleanup_sockets(*sockets): + """Safely close multiple sockets, ignoring errors.""" + for sock in sockets: + if sock is not None: + try: + sock.close() + except OSError: + pass + + +# ============================================================================ +# Decorators and skip conditions +# ============================================================================ + skip_if_not_supported = unittest.skipIf( - (sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32"), + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), "Test only runs on Linux, Windows and MacOS", ) -class TestGetStackTrace(unittest.TestCase): +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(interpreters is None, "subinterpreters required")( + meth + ) + +# ============================================================================ +# Simple wrapper functions for RemoteUnwinder +# ============================================================================ + +def get_stack_trace(pid): + for _ in busy_retry(SHORT_TIMEOUT): + try: + unwinder = RemoteUnwinder(pid, all_threads=True, debug=True) + return unwinder.get_stack_trace() + except RuntimeError as e: + continue + raise RuntimeError("Failed to get stack trace after retries") + + +def get_async_stack_trace(pid): + for _ in busy_retry(SHORT_TIMEOUT): + try: + unwinder = RemoteUnwinder(pid, debug=True) + return unwinder.get_async_stack_trace() + except RuntimeError as e: + continue + raise RuntimeError("Failed to get async stack trace after retries") + + +def get_all_awaited_by(pid): + for _ in busy_retry(SHORT_TIMEOUT): + try: + unwinder = RemoteUnwinder(pid, debug=True) + return unwinder.get_all_awaited_by() + except RuntimeError as e: + continue + raise RuntimeError("Failed to get all awaited_by after retries") + + +# ============================================================================ +# Base test class with shared infrastructure +# ============================================================================ + + +class RemoteInspectionTestBase(unittest.TestCase): + """Base class for remote inspection tests with common helpers.""" + + maxDiff = None + + def _run_script_and_get_trace( + self, + script, + trace_func, + wait_for_signals=None, + port=None, + backlog=1, + ): + """ + Common pattern: run a script, wait for signals, get trace. + + Args: + script: Script content (will be formatted with port if {port} present) + trace_func: Function to call with pid to get trace (e.g., get_stack_trace) + wait_for_signals: Signal(s) to wait for before getting trace + port: Port to use (auto-selected if None) + backlog: Socket listen backlog + + Returns: + tuple: (trace_result, script_name) + """ + if port is None: + port = find_unused_port() + + # Format script with port if needed + if "{port}" in script or "{{port}}" in script: + script = script.replace("{{port}}", "{port}").format(port=port) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port, backlog) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + if wait_for_signals: + _wait_for_signal(client_socket, wait_for_signals) + + try: + trace = trace_func(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + return trace, script_name + finally: + _cleanup_sockets(client_socket, server_socket) + + def _find_frame_in_trace(self, stack_trace, predicate): + """ + Find a frame matching predicate in stack trace. + + Args: + stack_trace: List of InterpreterInfo objects + predicate: Function(frame) -> bool + + Returns: + FrameInfo or None + """ + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + for frame in thread_info.frame_info: + if predicate(frame): + return frame + return None + + def _find_thread_by_id(self, stack_trace, thread_id): + """Find a thread by its native thread ID.""" + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + if thread_info.thread_id == thread_id: + return thread_info + return None + + def _find_thread_with_frame(self, stack_trace, frame_predicate): + """Find a thread containing a frame matching predicate.""" + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + for frame in thread_info.frame_info: + if frame_predicate(frame): + return thread_info + return None + + def _get_thread_statuses(self, stack_trace): + """Extract thread_id -> status mapping from stack trace.""" + statuses = {} + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + statuses[thread_info.thread_id] = thread_info.status + return statuses + + def _get_task_id_map(self, stack_trace): + """Create task_id -> task mapping from async stack trace.""" + return {task.task_id: task for task in stack_trace[0].awaited_by} + + def _get_awaited_by_relationships(self, stack_trace): + """Extract task name to awaited_by set mapping.""" + id_to_task = self._get_task_id_map(stack_trace) + return { + task.task_name: set( + id_to_task[awaited.task_name].task_name + for awaited in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + + def _extract_coroutine_stacks(self, stack_trace): + """Extract and format coroutine stacks from tasks.""" + return { + task.task_name: sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + for task in stack_trace[0].awaited_by + } + + +# ============================================================================ +# Test classes +# ============================================================================ + + +class TestGetStackTrace(RemoteInspectionTestBase): @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support", ) def test_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ - import time, sys, socket - # Connect to the test process + import time, sys, socket, threading + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) @@ -55,52 +404,83 @@ def bar(): for x in range(100): if x == 50: baz() + def baz(): foo() def foo(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready:thread\\n"); time.sleep(10_000) - bar() + t = threading.Thread(target=bar) + t.start() + sock.sendall(b"ready:main\\n"); t.join() """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_stack_trace(p.pid) - except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - expected_stack_trace = [ - ("foo", script_name, 14), - ("baz", script_name, 11), - ("bar", script_name, 9), - ("", script_name, 16), - ] - self.assertEqual(stack_trace, expected_stack_trace) + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + _wait_for_signal( + client_socket, [b"ready:main", b"ready:thread"] + ) + + try: + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + thread_expected_stack_trace = [ + FrameInfo([script_name, 15, "foo"]), + FrameInfo([script_name, 12, "baz"]), + FrameInfo([script_name, 9, "bar"]), + FrameInfo([threading.__file__, ANY, "Thread.run"]), + FrameInfo( + [ + threading.__file__, + ANY, + "Thread._bootstrap_inner", + ] + ), + FrameInfo( + [threading.__file__, ANY, "Thread._bootstrap"] + ), + ] + + # Find expected thread stack + found_thread = self._find_thread_with_frame( + stack_trace, + lambda f: f.funcname == "foo" and f.lineno == 15, + ) + self.assertIsNotNone( + found_thread, "Expected thread stack trace not found" + ) + self.assertEqual( + found_thread.frame_info, thread_expected_stack_trace + ) + + # Check main thread + main_frame = FrameInfo([script_name, 19, ""]) + found_main = self._find_frame_in_trace( + stack_trace, lambda f: f == main_frame + ) + self.assertIsNotNone( + found_main, "Main thread stack trace not found" + ) + finally: + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -108,7 +488,6 @@ def foo(): "Test only runs on Linux with process_vm_readv support", ) def test_async_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ @@ -116,12 +495,12 @@ def test_async_remote_stack_trace(self): import time import sys import socket - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) def c5(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready"); time.sleep(10_000) async def c4(): await asyncio.sleep(0) @@ -152,7 +531,7 @@ def new_eager_loop(): asyncio.run(main(), loop_factory={{TASK_FACTORY}}) """ ) - stack_trace = None + for task_factory_variant in "asyncio.new_event_loop", "new_eager_loop": with ( self.subTest(task_factory_variant=task_factory_variant), @@ -160,112 +539,203 @@ def new_eager_loop(): ): script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script( script_dir, "script", script.format(TASK_FACTORY=task_factory_variant), ) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) + with _managed_subprocess( + [sys.executable, script_name] + ) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) + response = _wait_for_signal(client_socket, b"ready") + self.assertIn(b"ready", response) - root_task = "Task-1" - expected_stack_trace = [ - [ - ("c5", script_name, 10), - ("c4", script_name, 14), - ("c3", script_name, 17), - ("c2", script_name, 20), - ], - "c2_root", - [ - [ - [ - ( - "TaskGroup._aexit", - taskgroups.__file__, - ANY, - ), - ( - "TaskGroup.__aexit__", - taskgroups.__file__, - ANY, - ), - ("main", script_name, 26), - ], - "Task-1", - [], - ], - [ - [("c1", script_name, 23)], + try: + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # Check all tasks are present + tasks_names = [ + task.task_name + for task in stack_trace[0].awaited_by + ] + for task_name in [ + "c2_root", "sub_main_1", - [ - [ - [ - ( - "TaskGroup._aexit", - taskgroups.__file__, - ANY, - ), - ( - "TaskGroup.__aexit__", - taskgroups.__file__, - ANY, - ), - ("main", script_name, 26), - ], - "Task-1", - [], - ] - ], - ], - [ - [("c1", script_name, 23)], "sub_main_2", - [ - [ - [ - ( - "TaskGroup._aexit", - taskgroups.__file__, - ANY, - ), - ( - "TaskGroup.__aexit__", - taskgroups.__file__, - ANY, - ), - ("main", script_name, 26), - ], + ]: + self.assertIn(task_name, tasks_names) + + # Check awaited_by relationships + relationships = self._get_awaited_by_relationships( + stack_trace + ) + self.assertEqual( + relationships, + { + "c2_root": { "Task-1", - [], - ] - ], - ], - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) + "sub_main_1", + "sub_main_2", + }, + "Task-1": set(), + "sub_main_1": {"Task-1"}, + "sub_main_2": {"Task-1"}, + }, + ) + + # Check coroutine stacks + coroutine_stacks = self._extract_coroutine_stacks( + stack_trace + ) + self.assertEqual( + coroutine_stacks, + { + "Task-1": [ + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ) + ], + "c2_root": [ + ( + tuple([script_name, 10, "c5"]), + tuple([script_name, 14, "c4"]), + tuple([script_name, 17, "c3"]), + tuple([script_name, 20, "c2"]), + ) + ], + "sub_main_1": [ + (tuple([script_name, 23, "c1"]),) + ], + "sub_main_2": [ + (tuple([script_name, 23, "c1"]),) + ], + }, + ) + + # Check awaited_by coroutine stacks + id_to_task = self._get_task_id_map(stack_trace) + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple( + tuple(frame) + for frame in coro.call_stack + ), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "c2_root": [ + ( + "Task-1", + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ), + ( + "sub_main_1", + (tuple([script_name, 23, "c1"]),), + ), + ( + "sub_main_2", + (tuple([script_name, 23, "c1"]),), + ), + ], + "sub_main_1": [ + ( + "Task-1", + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ) + ], + "sub_main_2": [ + ( + "Task-1", + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ) + ], + }, + ) + finally: + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -273,7 +743,6 @@ def new_eager_loop(): "Test only runs on Linux with process_vm_readv support", ) def test_asyncgen_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ @@ -281,12 +750,12 @@ def test_asyncgen_remote_stack_trace(self): import time import sys import socket - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) async def gen_nested_call(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready"); time.sleep(10_000) async def gen(): for num in range(2): @@ -301,47 +770,56 @@ async def main(): asyncio.run(main()) """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + response = _wait_for_signal(client_socket, b"ready") + self.assertIn(b"ready", response) + + try: + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # For this simple asyncgen test, we only expect one task + self.assertEqual(len(stack_trace[0].awaited_by), 1) + task = stack_trace[0].awaited_by[0] + self.assertEqual(task.task_name, "Task-1") + + # Check the coroutine stack + coroutine_stack = sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + self.assertEqual( + coroutine_stack, + [ + ( + tuple([script_name, 10, "gen_nested_call"]), + tuple([script_name, 16, "gen"]), + tuple([script_name, 19, "main"]), + ) + ], + ) + + # No awaited_by relationships expected + self.assertEqual(task.awaited_by, []) finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - [ - ("gen_nested_call", script_name, 10), - ("gen", script_name, 16), - ("main", script_name, 19), - ], - "Task-1", - [], - ] - self.assertEqual(stack_trace, expected_stack_trace) + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -349,7 +827,6 @@ async def main(): "Test only runs on Linux with process_vm_readv support", ) def test_async_gather_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ @@ -357,13 +834,13 @@ def test_async_gather_remote_stack_trace(self): import time import sys import socket - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) async def deep(): await asyncio.sleep(0) - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready"); time.sleep(10_000) async def c1(): await asyncio.sleep(0) @@ -378,43 +855,92 @@ async def main(): asyncio.run(main()) """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + response = _wait_for_signal(client_socket, b"ready") + self.assertIn(b"ready", response) + + try: + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # Check all tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by + ] + for task_name in ["Task-1", "Task-2"]: + self.assertIn(task_name, tasks_names) + + # Check awaited_by relationships + relationships = self._get_awaited_by_relationships( + stack_trace + ) + self.assertEqual( + relationships, + { + "Task-1": set(), + "Task-2": {"Task-1"}, + }, + ) + + # Check coroutine stacks + coroutine_stacks = self._extract_coroutine_stacks( + stack_trace + ) + self.assertEqual( + coroutine_stacks, + { + "Task-1": [(tuple([script_name, 21, "main"]),)], + "Task-2": [ + ( + tuple([script_name, 11, "deep"]), + tuple([script_name, 15, "c1"]), + ) + ], + }, + ) + + # Check awaited_by coroutine stacks + id_to_task = self._get_task_id_map(stack_trace) + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple( + tuple(frame) for frame in coro.call_stack + ), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "Task-2": [ + ("Task-1", (tuple([script_name, 21, "main"]),)) + ], + }, + ) finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - [("deep", script_name, 11), ("c1", script_name, 15)], - "Task-2", - [[[("main", script_name, 21)], "Task-1", []]], - ] - self.assertEqual(stack_trace, expected_stack_trace) + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -422,7 +948,6 @@ async def main(): "Test only runs on Linux with process_vm_readv support", ) def test_async_staggered_race_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ @@ -430,13 +955,13 @@ def test_async_staggered_race_remote_stack_trace(self): import time import sys import socket - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) async def deep(): await asyncio.sleep(0) - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready"); time.sleep(10_000) async def c1(): await asyncio.sleep(0) @@ -454,55 +979,122 @@ async def main(): asyncio.run(main()) """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [ - [ - ("deep", script_name, 11), - ("c1", script_name, 15), - ("staggered_race..run_one_coro", staggered.__file__, ANY), - ], - "Task-2", - [ - [ - [ - ("staggered_race", staggered.__file__, ANY), - ("main", script_name, 21), - ], - "Task-1", - [], + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + response = _wait_for_signal(client_socket, b"ready") + self.assertIn(b"ready", response) + + try: + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # Check all tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by ] - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) + for task_name in ["Task-1", "Task-2"]: + self.assertIn(task_name, tasks_names) + + # Check awaited_by relationships + relationships = self._get_awaited_by_relationships( + stack_trace + ) + self.assertEqual( + relationships, + { + "Task-1": set(), + "Task-2": {"Task-1"}, + }, + ) + + # Check coroutine stacks + coroutine_stacks = self._extract_coroutine_stacks( + stack_trace + ) + self.assertEqual( + coroutine_stacks, + { + "Task-1": [ + ( + tuple( + [ + staggered.__file__, + ANY, + "staggered_race", + ] + ), + tuple([script_name, 21, "main"]), + ) + ], + "Task-2": [ + ( + tuple([script_name, 11, "deep"]), + tuple([script_name, 15, "c1"]), + tuple( + [ + staggered.__file__, + ANY, + "staggered_race..run_one_coro", + ] + ), + ) + ], + }, + ) + + # Check awaited_by coroutine stacks + id_to_task = self._get_task_id_map(stack_trace) + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple( + tuple(frame) for frame in coro.call_stack + ), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "Task-2": [ + ( + "Task-1", + ( + tuple( + [ + staggered.__file__, + ANY, + "staggered_race", + ] + ), + tuple([script_name, 21, "main"]), + ), + ) + ], + }, + ) + finally: + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -510,6 +1102,10 @@ async def main(): "Test only runs on Linux with process_vm_readv support", ) def test_async_global_awaited_by(self): + # Reduced from 1000 to 100 to avoid file descriptor exhaustion + # when running tests in parallel (e.g., -j 20) + NUM_TASKS = 100 + port = find_unused_port() script = textwrap.dedent( f"""\ @@ -525,7 +1121,6 @@ def test_async_global_awaited_by(self): PORT = socket_helper.find_unused_port() connections = 0 - # Connect to the test process sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) @@ -548,23 +1143,16 @@ async def echo_client(message): assert message == data.decode() writer.close() await writer.wait_closed() - # Signal we are ready to sleep sock.sendall(b"ready") await asyncio.sleep(SHORT_TIMEOUT) async def echo_client_spam(server): async with asyncio.TaskGroup() as tg: - while connections < 1000: + while connections < {NUM_TASKS}: msg = list(ascii_lowercase + digits) random.shuffle(msg) tg.create_task(echo_client("".join(msg))) await asyncio.sleep(0) - # at least a 1000 tasks created. Each task will signal - # when is ready to avoid the race caused by the fact that - # tasks are waited on tg.__exit__ and we cannot signal when - # that happens otherwise - # at this point all client tasks completed without assertion errors - # let's wrap up the test server.close() await server.wait_closed() @@ -579,119 +1167,216 @@ async def main(): asyncio.run(main()) """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - for _ in range(1000): - expected_response = b"ready" - response = client_socket.recv(len(expected_response)) - self.assertEqual(response, expected_response) - for _ in busy_retry(SHORT_TIMEOUT): + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + # Wait for NUM_TASKS "ready" signals + try: + _wait_for_n_signals(client_socket, b"ready", NUM_TASKS) + except RuntimeError as e: + self.fail(str(e)) + try: all_awaited_by = get_all_awaited_by(p.pid) - except RuntimeError as re: - # This call reads a linked list in another process with - # no synchronization. That occasionally leads to invalid - # reads. Here we avoid making the test flaky. - msg = str(re) - if msg.startswith("Task list appears corrupted"): - continue - elif msg.startswith( - "Invalid linked list structure reading remote memory" - ): - continue - elif msg.startswith("Unknown error reading memory"): - continue - elif msg.startswith("Unhandled frame owner"): - continue - raise # Unrecognized exception, safest not to ignore it - else: - break - # expected: a list of two elements: 1 thread, 1 interp - self.assertEqual(len(all_awaited_by), 2) - # expected: a tuple with the thread ID and the awaited_by list - self.assertEqual(len(all_awaited_by[0]), 2) - # expected: no tasks in the fallback per-interp task list - self.assertEqual(all_awaited_by[1], (0, [])) - entries = all_awaited_by[0][1] - # expected: at least 1000 pending tasks - self.assertGreaterEqual(len(entries), 1000) - # the first three tasks stem from the code structure - self.assertIn((ANY, "Task-1", []), entries) - main_stack = [ - ( - "TaskGroup._aexit", - taskgroups.__file__, - ANY, - ), - ( - "TaskGroup.__aexit__", - taskgroups.__file__, - ANY, - ), - ("main", script_name, 60), - ] - self.assertIn( - (ANY, "server task", [[main_stack, ANY]]), - entries, - ) - self.assertIn( - (ANY, "echo client spam", [[main_stack, ANY]]), - entries, - ) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) - expected_stack = [ - [ - [ - ( - "TaskGroup._aexit", - taskgroups.__file__, - ANY, - ), - ( - "TaskGroup.__aexit__", - taskgroups.__file__, - ANY, - ), - ("echo_client_spam", script_name, 41), - ], - ANY, + # Expected: a list of two elements: 1 thread, 1 interp + self.assertEqual(len(all_awaited_by), 2) + # Expected: a tuple with the thread ID and the awaited_by list + self.assertEqual(len(all_awaited_by[0]), 2) + # Expected: no tasks in the fallback per-interp task list + self.assertEqual(all_awaited_by[1], (0, [])) + + entries = all_awaited_by[0][1] + # Expected: at least NUM_TASKS pending tasks + self.assertGreaterEqual(len(entries), NUM_TASKS) + + # Check the main task structure + main_stack = [ + FrameInfo( + [taskgroups.__file__, ANY, "TaskGroup._aexit"] + ), + FrameInfo( + [taskgroups.__file__, ANY, "TaskGroup.__aexit__"] + ), + FrameInfo([script_name, 52, "main"]), ] - ] - tasks_with_stack = [ - task for task in entries if task[2] == expected_stack - ] - self.assertGreaterEqual(len(tasks_with_stack), 1000) - - # the final task will have some random number, but it should for - # sure be one of the echo client spam horde (In windows this is not true - # for some reason) - if sys.platform != "win32": - self.assertEqual( - expected_stack, - entries[-1][2], + self.assertIn( + TaskInfo( + [ANY, "Task-1", [CoroInfo([main_stack, ANY])], []] + ), + entries, ) - except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.assertIn( + TaskInfo( + [ + ANY, + "server task", + [ + CoroInfo( + [ + [ + FrameInfo( + [ + base_events.__file__, + ANY, + "Server.serve_forever", + ] + ) + ], + ANY, + ] + ) + ], + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, ANY, "main"] + ), + ], + ANY, + ] + ) + ], + ] + ), + entries, + ) + self.assertIn( + TaskInfo( + [ + ANY, + "Task-4", + [ + CoroInfo( + [ + [ + FrameInfo( + [ + tasks.__file__, + ANY, + "sleep", + ] + ), + FrameInfo( + [ + script_name, + 36, + "echo_client", + ] + ), + ], + ANY, + ] + ) + ], + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [ + script_name, + 39, + "echo_client_spam", + ] + ), + ], + ANY, + ] + ) + ], + ] + ), + entries, + ) + + expected_awaited_by = [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, 39, "echo_client_spam"] + ), + ], + ANY, + ] + ) + ] + tasks_with_awaited = [ + task + for task in entries + if task.awaited_by == expected_awaited_by + ] + self.assertGreaterEqual(len(tasks_with_awaited), NUM_TASKS) + + # Final task should be from echo client spam (not on Windows) + if sys.platform != "win32": + self.assertEqual( + tasks_with_awaited[-1].awaited_by, + entries[-1].awaited_by, + ) finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -700,15 +1385,1680 @@ async def main(): ) def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) + + this_thread_stack = None + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + if thread_info.thread_id == threading.get_native_id(): + this_thread_stack = thread_info.frame_info + break + if this_thread_stack: + break + + self.assertIsNotNone(this_thread_stack) self.assertEqual( - stack_trace[0], - ( - "TestGetStackTrace.test_self_trace", - __file__, - self.test_self_trace.__code__.co_firstlineno + 6, - ), + this_thread_stack[:2], + [ + FrameInfo( + [ + __file__, + get_stack_trace.__code__.co_firstlineno + 4, + "get_stack_trace", + ] + ), + FrameInfo( + [ + __file__, + self.test_self_trace.__code__.co_firstlineno + 6, + "TestGetStackTrace.test_self_trace", + ] + ), + ], ) + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + @requires_subinterpreters + def test_subinterpreter_stack_trace(self): + port = find_unused_port() + + import pickle + + subinterp_code = textwrap.dedent(f""" + import socket + import time + + def sub_worker(): + def nested_func(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + sock.sendall(b"ready:sub\\n") + time.sleep(10_000) + nested_func() + + sub_worker() + """).strip() + + pickled_code = pickle.dumps(subinterp_code) + + script = textwrap.dedent( + f""" + from concurrent import interpreters + import time + import sys + import socket + import threading + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def main_worker(): + sock.sendall(b"ready:main\\n") + time.sleep(10_000) + + def run_subinterp(): + subinterp = interpreters.create() + import pickle + pickled_code = {pickled_code!r} + subinterp_code = pickle.loads(pickled_code) + subinterp.exec(subinterp_code) + + sub_thread = threading.Thread(target=run_subinterp) + sub_thread.start() + + main_thread = threading.Thread(target=main_worker) + main_thread.start() + + main_thread.join() + sub_thread.join() + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_sockets = [] + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + # Accept connections from both main and subinterpreter + responses = set() + while len(responses) < 2: + try: + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + response = client_socket.recv(1024) + if b"ready:main" in response: + responses.add("main") + if b"ready:sub" in response: + responses.add("sub") + except socket.timeout: + break + + server_socket.close() + server_socket = None + + try: + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # Verify we have at least one interpreter + self.assertGreaterEqual(len(stack_trace), 1) + + # Look for main interpreter (ID 0) and subinterpreter (ID > 0) + main_interp = None + sub_interp = None + for interpreter_info in stack_trace: + if interpreter_info.interpreter_id == 0: + main_interp = interpreter_info + elif interpreter_info.interpreter_id > 0: + sub_interp = interpreter_info + + self.assertIsNotNone( + main_interp, "Main interpreter should be present" + ) + + # Check main interpreter has expected stack trace + main_found = self._find_frame_in_trace( + [main_interp], lambda f: f.funcname == "main_worker" + ) + self.assertIsNotNone( + main_found, + "Main interpreter should have main_worker in stack", + ) + + # If subinterpreter is present, check its stack trace + if sub_interp: + sub_found = self._find_frame_in_trace( + [sub_interp], + lambda f: f.funcname + in ("sub_worker", "nested_func"), + ) + self.assertIsNotNone( + sub_found, + "Subinterpreter should have sub_worker or nested_func in stack", + ) + finally: + _cleanup_sockets(*client_sockets, server_socket) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + @requires_subinterpreters + def test_multiple_subinterpreters_with_threads(self): + port = find_unused_port() + + import pickle + + subinterp1_code = textwrap.dedent(f""" + import socket + import time + import threading + + def worker1(): + def nested_func(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + sock.sendall(b"ready:sub1-t1\\n") + time.sleep(10_000) + nested_func() + + def worker2(): + def nested_func(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + sock.sendall(b"ready:sub1-t2\\n") + time.sleep(10_000) + nested_func() + + t1 = threading.Thread(target=worker1) + t2 = threading.Thread(target=worker2) + t1.start() + t2.start() + t1.join() + t2.join() + """).strip() + + subinterp2_code = textwrap.dedent(f""" + import socket + import time + import threading + + def worker1(): + def nested_func(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + sock.sendall(b"ready:sub2-t1\\n") + time.sleep(10_000) + nested_func() + + def worker2(): + def nested_func(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + sock.sendall(b"ready:sub2-t2\\n") + time.sleep(10_000) + nested_func() + + t1 = threading.Thread(target=worker1) + t2 = threading.Thread(target=worker2) + t1.start() + t2.start() + t1.join() + t2.join() + """).strip() + + pickled_code1 = pickle.dumps(subinterp1_code) + pickled_code2 = pickle.dumps(subinterp2_code) + + script = textwrap.dedent( + f""" + from concurrent import interpreters + import time + import sys + import socket + import threading + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def main_worker(): + sock.sendall(b"ready:main\\n") + time.sleep(10_000) + + def run_subinterp1(): + subinterp = interpreters.create() + import pickle + pickled_code = {pickled_code1!r} + subinterp_code = pickle.loads(pickled_code) + subinterp.exec(subinterp_code) + + def run_subinterp2(): + subinterp = interpreters.create() + import pickle + pickled_code = {pickled_code2!r} + subinterp_code = pickle.loads(pickled_code) + subinterp.exec(subinterp_code) + + sub1_thread = threading.Thread(target=run_subinterp1) + sub2_thread = threading.Thread(target=run_subinterp2) + sub1_thread.start() + sub2_thread.start() + + main_thread = threading.Thread(target=main_worker) + main_thread.start() + + main_thread.join() + sub1_thread.join() + sub2_thread.join() + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port, backlog=5) + script_name = _make_test_script(script_dir, "script", script) + client_sockets = [] + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + # Accept connections from main and all subinterpreter threads + expected_responses = { + "ready:main", + "ready:sub1-t1", + "ready:sub1-t2", + "ready:sub2-t1", + "ready:sub2-t2", + } + responses = set() + + while len(responses) < 5: + try: + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + response = client_socket.recv(1024) + response_str = response.decode().strip() + if response_str in expected_responses: + responses.add(response_str) + except socket.timeout: + break + + server_socket.close() + server_socket = None + + try: + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # Verify we have multiple interpreters + self.assertGreaterEqual(len(stack_trace), 2) + + # Count interpreters by ID + interpreter_ids = { + interp.interpreter_id for interp in stack_trace + } + self.assertIn( + 0, + interpreter_ids, + "Main interpreter should be present", + ) + self.assertGreaterEqual(len(interpreter_ids), 3) + + # Count total threads + total_threads = sum( + len(interp.threads) for interp in stack_trace + ) + self.assertGreaterEqual(total_threads, 5) + + # Look for expected function names + all_funcnames = set() + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + for frame in thread_info.frame_info: + all_funcnames.add(frame.funcname) + + expected_funcs = { + "main_worker", + "worker1", + "worker2", + "nested_func", + } + found_funcs = expected_funcs.intersection(all_funcnames) + self.assertGreater(len(found_funcs), 0) + finally: + _cleanup_sockets(*client_sockets, server_socket) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + @requires_gil_enabled("Free threaded builds don't have an 'active thread'") + def test_only_active_thread(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import time, sys, socket, threading + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def worker_thread(name, barrier, ready_event): + barrier.wait() + ready_event.wait() + time.sleep(10_000) + + def main_work(): + sock.sendall(b"working\\n") + count = 0 + while count < 100000000: + count += 1 + if count % 10000000 == 0: + pass + sock.sendall(b"done\\n") + + num_threads = 3 + barrier = threading.Barrier(num_threads + 1) + ready_event = threading.Event() + + threads = [] + for i in range(num_threads): + t = threading.Thread(target=worker_thread, args=(f"Worker-{{i}}", barrier, ready_event)) + t.start() + threads.append(t) + + barrier.wait() + sock.sendall(b"ready\\n") + ready_event.set() + main_work() + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + # Wait for ready and working signals + _wait_for_signal(client_socket, [b"ready", b"working"]) + + try: + # Get stack trace with all threads + unwinder_all = RemoteUnwinder(p.pid, all_threads=True) + for _ in range(MAX_TRIES): + all_traces = unwinder_all.get_stack_trace() + found = self._find_frame_in_trace( + all_traces, + lambda f: f.funcname == "main_work" + and f.lineno > 12, + ) + if found: + break + time.sleep(0.1) + else: + self.fail( + "Main thread did not start its busy work on time" + ) + + # Get stack trace with only GIL holder + unwinder_gil = RemoteUnwinder( + p.pid, only_active_thread=True + ) + gil_traces = unwinder_gil.get_stack_trace() + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # Count threads + total_threads = sum( + len(interp.threads) for interp in all_traces + ) + self.assertGreater(total_threads, 1) + + total_gil_threads = sum( + len(interp.threads) for interp in gil_traces + ) + self.assertEqual(total_gil_threads, 1) + + # Get the GIL holder thread ID + gil_thread_id = None + for interpreter_info in gil_traces: + if interpreter_info.threads: + gil_thread_id = interpreter_info.threads[ + 0 + ].thread_id + break + + # Get all thread IDs + all_thread_ids = [] + for interpreter_info in all_traces: + for thread_info in interpreter_info.threads: + all_thread_ids.append(thread_info.thread_id) + + self.assertIn(gil_thread_id, all_thread_ids) + finally: + _cleanup_sockets(client_socket, server_socket) + + +class TestUnsupportedPlatformHandling(unittest.TestCase): + @unittest.skipIf( + sys.platform in ("linux", "darwin", "win32"), + "Test only runs on unsupported platforms (not Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_unsupported_platform_error(self): + with self.assertRaises(RuntimeError) as cm: + RemoteUnwinder(os.getpid()) + + self.assertIn( + "Reading the PyRuntime section is not supported on this platform", + str(cm.exception), + ) + + +class TestDetectionOfThreadStatus(RemoteInspectionTestBase): + def _run_thread_status_test(self, mode, check_condition): + """ + Common pattern for thread status detection tests. + + Args: + mode: Profiling mode (PROFILING_MODE_CPU, PROFILING_MODE_GIL, etc.) + check_condition: Function(statuses, sleeper_tid, busy_tid) -> bool + """ + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import time, sys, socket, threading + import os + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def sleeper(): + tid = threading.get_native_id() + sock.sendall(f'ready:sleeper:{{tid}}\\n'.encode()) + time.sleep(10000) + + def busy(): + tid = threading.get_native_id() + sock.sendall(f'ready:busy:{{tid}}\\n'.encode()) + x = 0 + while True: + x = x + 1 + time.sleep(0.5) + + t1 = threading.Thread(target=sleeper) + t2 = threading.Thread(target=busy) + t1.start() + t2.start() + sock.sendall(b'ready:main\\n') + t1.join() + t2.join() + sock.close() + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script( + script_dir, "thread_status_script", script + ) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + # Wait for all ready signals and parse TIDs + response = _wait_for_signal( + client_socket, + [b"ready:main", b"ready:sleeper", b"ready:busy"], + ) + + sleeper_tid = None + busy_tid = None + for line in response.split(b"\n"): + if line.startswith(b"ready:sleeper:"): + try: + sleeper_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + elif line.startswith(b"ready:busy:"): + try: + busy_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + + self.assertIsNotNone( + sleeper_tid, "Sleeper thread id not received" + ) + self.assertIsNotNone( + busy_tid, "Busy thread id not received" + ) + + # Sample until we see expected thread states + statuses = {} + try: + unwinder = RemoteUnwinder( + p.pid, + all_threads=True, + mode=mode, + skip_non_matching_threads=False, + ) + for _ in range(MAX_TRIES): + traces = unwinder.get_stack_trace() + statuses = self._get_thread_statuses(traces) + + if check_condition( + statuses, sleeper_tid, busy_tid + ): + break + time.sleep(0.5) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + return statuses, sleeper_tid, busy_tid + finally: + _cleanup_sockets(client_socket, server_socket) + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_thread_status_detection(self): + def check_cpu_status(statuses, sleeper_tid, busy_tid): + return ( + sleeper_tid in statuses + and busy_tid in statuses + and not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) + and (statuses[busy_tid] & THREAD_STATUS_ON_CPU) + ) + + statuses, sleeper_tid, busy_tid = self._run_thread_status_test( + PROFILING_MODE_CPU, check_cpu_status + ) + + self.assertIn(sleeper_tid, statuses) + self.assertIn(busy_tid, statuses) + self.assertFalse( + statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, + "Sleeper thread should be off CPU", + ) + self.assertTrue( + statuses[busy_tid] & THREAD_STATUS_ON_CPU, + "Busy thread should be on CPU", + ) + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_thread_status_gil_detection(self): + def check_gil_status(statuses, sleeper_tid, busy_tid): + return ( + sleeper_tid in statuses + and busy_tid in statuses + and not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) + and (statuses[busy_tid] & THREAD_STATUS_HAS_GIL) + ) + + statuses, sleeper_tid, busy_tid = self._run_thread_status_test( + PROFILING_MODE_GIL, check_gil_status + ) + + self.assertIn(sleeper_tid, statuses) + self.assertIn(busy_tid, statuses) + self.assertFalse( + statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, + "Sleeper thread should not have GIL", + ) + self.assertTrue( + statuses[busy_tid] & THREAD_STATUS_HAS_GIL, + "Busy thread should have GIL", + ) + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_thread_status_all_mode_detection(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import socket + import threading + import time + import sys + + def sleeper_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"sleeper:" + str(threading.get_native_id()).encode()) + while True: + time.sleep(1) + + def busy_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"busy:" + str(threading.get_native_id()).encode()) + while True: + sum(range(100000)) + + t1 = threading.Thread(target=sleeper_thread) + t2 = threading.Thread(target=busy_thread) + t1.start() + t2.start() + t1.join() + t2.join() + """ + ) + + with os_helper.temp_dir() as tmp_dir: + script_file = make_script(tmp_dir, "script", script) + server_socket = _create_server_socket(port, backlog=2) + client_sockets = [] + + try: + with _managed_subprocess( + [sys.executable, script_file], + ) as p: + sleeper_tid = None + busy_tid = None + + # Receive thread IDs from the child process + for _ in range(2): + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + line = client_socket.recv(1024) + if line: + if line.startswith(b"sleeper:"): + try: + sleeper_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + elif line.startswith(b"busy:"): + try: + busy_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + + server_socket.close() + server_socket = None + + statuses = {} + try: + unwinder = RemoteUnwinder( + p.pid, + all_threads=True, + mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False, + ) + for _ in range(MAX_TRIES): + traces = unwinder.get_stack_trace() + statuses = self._get_thread_statuses(traces) + + # Check ALL mode provides both GIL and CPU info + if ( + sleeper_tid in statuses + and busy_tid in statuses + and not ( + statuses[sleeper_tid] + & THREAD_STATUS_ON_CPU + ) + and not ( + statuses[sleeper_tid] + & THREAD_STATUS_HAS_GIL + ) + and (statuses[busy_tid] & THREAD_STATUS_ON_CPU) + and ( + statuses[busy_tid] & THREAD_STATUS_HAS_GIL + ) + ): + break + time.sleep(0.5) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + self.assertIsNotNone( + sleeper_tid, "Sleeper thread id not received" + ) + self.assertIsNotNone( + busy_tid, "Busy thread id not received" + ) + self.assertIn(sleeper_tid, statuses) + self.assertIn(busy_tid, statuses) + + # Sleeper: off CPU, no GIL + self.assertFalse( + statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, + "Sleeper should be off CPU", + ) + self.assertFalse( + statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, + "Sleeper should not have GIL", + ) + + # Busy: on CPU, has GIL + self.assertTrue( + statuses[busy_tid] & THREAD_STATUS_ON_CPU, + "Busy should be on CPU", + ) + self.assertTrue( + statuses[busy_tid] & THREAD_STATUS_HAS_GIL, + "Busy should have GIL", + ) + finally: + _cleanup_sockets(*client_sockets, server_socket) + + +class TestFrameCaching(RemoteInspectionTestBase): + """Test that frame caching produces correct results. + + Uses socket-based synchronization for deterministic testing. + All tests verify cache reuse via object identity checks (assertIs). + """ + + @contextmanager + def _target_process(self, script_body): + """Context manager for running a target process with socket sync.""" + port = find_unused_port() + script = f"""\ +import socket +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +sock.connect(('localhost', {port})) +{textwrap.dedent(script_body)} +""" + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + def make_unwinder(cache_frames=True): + return RemoteUnwinder( + p.pid, all_threads=True, cache_frames=cache_frames + ) + + yield p, client_socket, make_unwinder + + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + _cleanup_sockets(client_socket, server_socket) + + def _get_frames_with_retry(self, unwinder, required_funcs): + """Get frames containing required_funcs, with retry for transient errors.""" + for _ in range(MAX_TRIES): + with contextlib.suppress(OSError, RuntimeError): + traces = unwinder.get_stack_trace() + for interp in traces: + for thread in interp.threads: + funcs = {f.funcname for f in thread.frame_info} + if required_funcs.issubset(funcs): + return thread.frame_info + time.sleep(0.1) + return None + + def _sample_frames( + self, + client_socket, + unwinder, + wait_signal, + send_ack, + required_funcs, + expected_frames=1, + ): + """Wait for signal, sample frames with retry until required funcs present, send ack.""" + _wait_for_signal(client_socket, wait_signal) + frames = None + for _ in range(MAX_TRIES): + frames = self._get_frames_with_retry(unwinder, required_funcs) + if frames and len(frames) >= expected_frames: + break + time.sleep(0.1) + client_socket.sendall(send_ack) + return frames + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_hit_same_stack(self): + """Test that consecutive samples reuse cached parent frame objects. + + The current frame (index 0) is always re-read from memory to get + updated line numbers, so it may be a different object. Parent frames + (index 1+) should be identical objects from cache. + """ + script_body = """\ + def level3(): + sock.sendall(b"sync1") + sock.recv(16) + sock.sendall(b"sync2") + sock.recv(16) + sock.sendall(b"sync3") + sock.recv(16) + + def level2(): + level3() + + def level1(): + level2() + + level1() + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + unwinder = make_unwinder(cache_frames=True) + expected = {"level1", "level2", "level3"} + + frames1 = self._sample_frames( + client_socket, unwinder, b"sync1", b"ack", expected + ) + frames2 = self._sample_frames( + client_socket, unwinder, b"sync2", b"ack", expected + ) + frames3 = self._sample_frames( + client_socket, unwinder, b"sync3", b"done", expected + ) + + self.assertIsNotNone(frames1) + self.assertIsNotNone(frames2) + self.assertIsNotNone(frames3) + self.assertEqual(len(frames1), len(frames2)) + self.assertEqual(len(frames2), len(frames3)) + + # Current frame (index 0) is always re-read, so check value equality + self.assertEqual(frames1[0].funcname, frames2[0].funcname) + self.assertEqual(frames2[0].funcname, frames3[0].funcname) + + # Parent frames (index 1+) must be identical objects (cache reuse) + for i in range(1, len(frames1)): + f1, f2, f3 = frames1[i], frames2[i], frames3[i] + self.assertIs( + f1, f2, f"Frame {i}: samples 1-2 must be same object" + ) + self.assertIs( + f2, f3, f"Frame {i}: samples 2-3 must be same object" + ) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_line_number_updates_in_same_frame(self): + """Test that line numbers are correctly updated when execution moves within a function. + + When the profiler samples at different points within the same function, + it must report the correct line number for each sample, not stale cached values. + """ + script_body = """\ + def outer(): + inner() + + def inner(): + sock.sendall(b"line_a"); sock.recv(16) + sock.sendall(b"line_b"); sock.recv(16) + sock.sendall(b"line_c"); sock.recv(16) + sock.sendall(b"line_d"); sock.recv(16) + + outer() + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + unwinder = make_unwinder(cache_frames=True) + + frames_a = self._sample_frames( + client_socket, unwinder, b"line_a", b"ack", {"inner"} + ) + frames_b = self._sample_frames( + client_socket, unwinder, b"line_b", b"ack", {"inner"} + ) + frames_c = self._sample_frames( + client_socket, unwinder, b"line_c", b"ack", {"inner"} + ) + frames_d = self._sample_frames( + client_socket, unwinder, b"line_d", b"done", {"inner"} + ) + + self.assertIsNotNone(frames_a) + self.assertIsNotNone(frames_b) + self.assertIsNotNone(frames_c) + self.assertIsNotNone(frames_d) + + # Get the 'inner' frame from each sample (should be index 0) + inner_a = frames_a[0] + inner_b = frames_b[0] + inner_c = frames_c[0] + inner_d = frames_d[0] + + self.assertEqual(inner_a.funcname, "inner") + self.assertEqual(inner_b.funcname, "inner") + self.assertEqual(inner_c.funcname, "inner") + self.assertEqual(inner_d.funcname, "inner") + + # Line numbers must be different and increasing (execution moves forward) + self.assertLess( + inner_a.lineno, inner_b.lineno, "Line B should be after line A" + ) + self.assertLess( + inner_b.lineno, inner_c.lineno, "Line C should be after line B" + ) + self.assertLess( + inner_c.lineno, inner_d.lineno, "Line D should be after line C" + ) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_invalidation_on_return(self): + """Test cache invalidation when stack shrinks (function returns).""" + script_body = """\ + def inner(): + sock.sendall(b"at_inner") + sock.recv(16) + + def outer(): + inner() + sock.sendall(b"at_outer") + sock.recv(16) + + outer() + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + unwinder = make_unwinder(cache_frames=True) + + frames_deep = self._sample_frames( + client_socket, + unwinder, + b"at_inner", + b"ack", + {"inner", "outer"}, + ) + frames_shallow = self._sample_frames( + client_socket, unwinder, b"at_outer", b"done", {"outer"} + ) + + self.assertIsNotNone(frames_deep) + self.assertIsNotNone(frames_shallow) + + funcs_deep = [f.funcname for f in frames_deep] + funcs_shallow = [f.funcname for f in frames_shallow] + + self.assertIn("inner", funcs_deep) + self.assertIn("outer", funcs_deep) + self.assertNotIn("inner", funcs_shallow) + self.assertIn("outer", funcs_shallow) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_invalidation_on_call(self): + """Test cache invalidation when stack grows (new function called).""" + script_body = """\ + def deeper(): + sock.sendall(b"at_deeper") + sock.recv(16) + + def middle(): + sock.sendall(b"at_middle") + sock.recv(16) + deeper() + + def top(): + middle() + + top() + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + unwinder = make_unwinder(cache_frames=True) + + frames_before = self._sample_frames( + client_socket, + unwinder, + b"at_middle", + b"ack", + {"middle", "top"}, + ) + frames_after = self._sample_frames( + client_socket, + unwinder, + b"at_deeper", + b"done", + {"deeper", "middle", "top"}, + ) + + self.assertIsNotNone(frames_before) + self.assertIsNotNone(frames_after) + + funcs_before = [f.funcname for f in frames_before] + funcs_after = [f.funcname for f in frames_after] + + self.assertIn("middle", funcs_before) + self.assertIn("top", funcs_before) + self.assertNotIn("deeper", funcs_before) + + self.assertIn("deeper", funcs_after) + self.assertIn("middle", funcs_after) + self.assertIn("top", funcs_after) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_partial_stack_reuse(self): + """Test that unchanged bottom frames are reused when top changes (A→B→C to A→B→D).""" + script_body = """\ + def func_c(): + sock.sendall(b"at_c") + sock.recv(16) + + def func_d(): + sock.sendall(b"at_d") + sock.recv(16) + + def func_b(): + func_c() + func_d() + + def func_a(): + func_b() + + func_a() + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + unwinder = make_unwinder(cache_frames=True) + + # Sample at C: stack is A→B→C + frames_c = self._sample_frames( + client_socket, + unwinder, + b"at_c", + b"ack", + {"func_a", "func_b", "func_c"}, + ) + # Sample at D: stack is A→B→D (C returned, D called) + frames_d = self._sample_frames( + client_socket, + unwinder, + b"at_d", + b"done", + {"func_a", "func_b", "func_d"}, + ) + + self.assertIsNotNone(frames_c) + self.assertIsNotNone(frames_d) + + # Find func_a and func_b frames in both samples + def find_frame(frames, funcname): + for f in frames: + if f.funcname == funcname: + return f + return None + + frame_a_in_c = find_frame(frames_c, "func_a") + frame_b_in_c = find_frame(frames_c, "func_b") + frame_a_in_d = find_frame(frames_d, "func_a") + frame_b_in_d = find_frame(frames_d, "func_b") + + self.assertIsNotNone(frame_a_in_c) + self.assertIsNotNone(frame_b_in_c) + self.assertIsNotNone(frame_a_in_d) + self.assertIsNotNone(frame_b_in_d) + + # The bottom frames (A, B) should be the SAME objects (cache reuse) + self.assertIs( + frame_a_in_c, + frame_a_in_d, + "func_a frame should be reused from cache", + ) + self.assertIs( + frame_b_in_c, + frame_b_in_d, + "func_b frame should be reused from cache", + ) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_recursive_frames(self): + """Test caching with same function appearing multiple times (recursion).""" + script_body = """\ + def recurse(n): + if n <= 0: + sock.sendall(b"sync1") + sock.recv(16) + sock.sendall(b"sync2") + sock.recv(16) + else: + recurse(n - 1) + + recurse(5) + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + unwinder = make_unwinder(cache_frames=True) + + frames1 = self._sample_frames( + client_socket, unwinder, b"sync1", b"ack", {"recurse"} + ) + frames2 = self._sample_frames( + client_socket, unwinder, b"sync2", b"done", {"recurse"} + ) + + self.assertIsNotNone(frames1) + self.assertIsNotNone(frames2) + + # Should have multiple "recurse" frames (6 total: recurse(5) down to recurse(0)) + recurse_count = sum(1 for f in frames1 if f.funcname == "recurse") + self.assertEqual(recurse_count, 6, "Should have 6 recursive frames") + + self.assertEqual(len(frames1), len(frames2)) + + # Current frame (index 0) is re-read, check value equality + self.assertEqual(frames1[0].funcname, frames2[0].funcname) + + # Parent frames (index 1+) should be identical objects (cache reuse) + for i in range(1, len(frames1)): + self.assertIs( + frames1[i], + frames2[i], + f"Frame {i}: recursive frames must be same object", + ) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_vs_no_cache_equivalence(self): + """Test that cache_frames=True and cache_frames=False produce equivalent results.""" + script_body = """\ + def level3(): + sock.sendall(b"ready"); sock.recv(16) + + def level2(): + level3() + + def level1(): + level2() + + level1() + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + _wait_for_signal(client_socket, b"ready") + + # Sample with cache + unwinder_cache = make_unwinder(cache_frames=True) + frames_cached = self._get_frames_with_retry( + unwinder_cache, {"level1", "level2", "level3"} + ) + + # Sample without cache + unwinder_no_cache = make_unwinder(cache_frames=False) + frames_no_cache = self._get_frames_with_retry( + unwinder_no_cache, {"level1", "level2", "level3"} + ) + + client_socket.sendall(b"done") + + self.assertIsNotNone(frames_cached) + self.assertIsNotNone(frames_no_cache) + + # Same number of frames + self.assertEqual(len(frames_cached), len(frames_no_cache)) + + # Same function names in same order + funcs_cached = [f.funcname for f in frames_cached] + funcs_no_cache = [f.funcname for f in frames_no_cache] + self.assertEqual(funcs_cached, funcs_no_cache) + + # Same line numbers + lines_cached = [f.lineno for f in frames_cached] + lines_no_cache = [f.lineno for f in frames_no_cache] + self.assertEqual(lines_cached, lines_no_cache) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_per_thread_isolation(self): + """Test that frame cache is per-thread and cache invalidation works independently.""" + script_body = """\ + import threading + + lock = threading.Lock() + + def sync(msg): + with lock: + sock.sendall(msg + b"\\n") + sock.recv(1) + + # Thread 1 functions + def baz1(): + sync(b"t1:baz1") + + def bar1(): + baz1() + + def blech1(): + sync(b"t1:blech1") + + def foo1(): + bar1() # Goes down to baz1, syncs + blech1() # Returns up, goes down to blech1, syncs + + # Thread 2 functions + def baz2(): + sync(b"t2:baz2") + + def bar2(): + baz2() + + def blech2(): + sync(b"t2:blech2") + + def foo2(): + bar2() # Goes down to baz2, syncs + blech2() # Returns up, goes down to blech2, syncs + + t1 = threading.Thread(target=foo1) + t2 = threading.Thread(target=foo2) + t1.start() + t2.start() + t1.join() + t2.join() + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + unwinder = make_unwinder(cache_frames=True) + + # Message dispatch table: signal -> required functions for that thread + dispatch = { + b"t1:baz1": {"baz1", "bar1", "foo1"}, + b"t2:baz2": {"baz2", "bar2", "foo2"}, + b"t1:blech1": {"blech1", "foo1"}, + b"t2:blech2": {"blech2", "foo2"}, + } + + # Track results for each sync point + results = {} + + # Process 4 sync points (order depends on thread scheduling) + buffer = _wait_for_signal(client_socket, b"\n") + for i in range(4): + # Extract first message from buffer + msg, sep, buffer = buffer.partition(b"\n") + self.assertIn(msg, dispatch, f"Unexpected message: {msg!r}") + + # Sample frames for the thread at this sync point + required_funcs = dispatch[msg] + frames = self._get_frames_with_retry(unwinder, required_funcs) + self.assertIsNotNone(frames, f"Thread not found for {msg!r}") + results[msg] = [f.funcname for f in frames] + + # Release thread and wait for next message (if not last) + client_socket.sendall(b"k") + if i < 3: + buffer += _wait_for_signal(client_socket, b"\n") + + # Validate Phase 1: baz snapshots + t1_baz = results.get(b"t1:baz1") + t2_baz = results.get(b"t2:baz2") + self.assertIsNotNone(t1_baz, "Missing t1:baz1 snapshot") + self.assertIsNotNone(t2_baz, "Missing t2:baz2 snapshot") + + # Thread 1 at baz1: should have foo1->bar1->baz1 + self.assertIn("baz1", t1_baz) + self.assertIn("bar1", t1_baz) + self.assertIn("foo1", t1_baz) + self.assertNotIn("blech1", t1_baz) + # No cross-contamination + self.assertNotIn("baz2", t1_baz) + self.assertNotIn("bar2", t1_baz) + self.assertNotIn("foo2", t1_baz) + + # Thread 2 at baz2: should have foo2->bar2->baz2 + self.assertIn("baz2", t2_baz) + self.assertIn("bar2", t2_baz) + self.assertIn("foo2", t2_baz) + self.assertNotIn("blech2", t2_baz) + # No cross-contamination + self.assertNotIn("baz1", t2_baz) + self.assertNotIn("bar1", t2_baz) + self.assertNotIn("foo1", t2_baz) + + # Validate Phase 2: blech snapshots (cache invalidation test) + t1_blech = results.get(b"t1:blech1") + t2_blech = results.get(b"t2:blech2") + self.assertIsNotNone(t1_blech, "Missing t1:blech1 snapshot") + self.assertIsNotNone(t2_blech, "Missing t2:blech2 snapshot") + + # Thread 1 at blech1: bar1/baz1 should be GONE (cache invalidated) + self.assertIn("blech1", t1_blech) + self.assertIn("foo1", t1_blech) + self.assertNotIn( + "bar1", t1_blech, "Cache not invalidated: bar1 still present" + ) + self.assertNotIn( + "baz1", t1_blech, "Cache not invalidated: baz1 still present" + ) + # No cross-contamination + self.assertNotIn("blech2", t1_blech) + + # Thread 2 at blech2: bar2/baz2 should be GONE (cache invalidated) + self.assertIn("blech2", t2_blech) + self.assertIn("foo2", t2_blech) + self.assertNotIn( + "bar2", t2_blech, "Cache not invalidated: bar2 still present" + ) + self.assertNotIn( + "baz2", t2_blech, "Cache not invalidated: baz2 still present" + ) + # No cross-contamination + self.assertNotIn("blech1", t2_blech) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_new_unwinder_with_stale_last_profiled_frame(self): + """Test that a new unwinder returns complete stack when cache lookup misses.""" + script_body = """\ + def level4(): + sock.sendall(b"sync1") + sock.recv(16) + sock.sendall(b"sync2") + sock.recv(16) + + def level3(): + level4() + + def level2(): + level3() + + def level1(): + level2() + + level1() + """ + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + expected = {"level1", "level2", "level3", "level4"} + + # First unwinder samples - this sets last_profiled_frame in target + unwinder1 = make_unwinder(cache_frames=True) + frames1 = self._sample_frames( + client_socket, unwinder1, b"sync1", b"ack", expected + ) + + # Create NEW unwinder (empty cache) and sample + # The target still has last_profiled_frame set from unwinder1 + unwinder2 = make_unwinder(cache_frames=True) + frames2 = self._sample_frames( + client_socket, unwinder2, b"sync2", b"done", expected + ) + + self.assertIsNotNone(frames1) + self.assertIsNotNone(frames2) + + funcs1 = [f.funcname for f in frames1] + funcs2 = [f.funcname for f in frames2] + + # Both should have all levels + for level in ["level1", "level2", "level3", "level4"]: + self.assertIn(level, funcs1, f"{level} missing from first sample") + self.assertIn(level, funcs2, f"{level} missing from second sample") + + # Should have same stack depth + self.assertEqual( + len(frames1), + len(frames2), + "New unwinder should return complete stack despite stale last_profiled_frame", + ) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_exhaustion(self): + """Test cache works when frame limit (1024) is exceeded. + + FRAME_CACHE_MAX_FRAMES=1024. With 1100 recursive frames, + the cache can't store all of them but should still work. + """ + # Use 1100 to exceed FRAME_CACHE_MAX_FRAMES=1024 + depth = 1100 + script_body = f"""\ +import sys +sys.setrecursionlimit(2000) + +def recurse(n): + if n <= 0: + sock.sendall(b"ready") + sock.recv(16) # wait for ack + sock.sendall(b"ready2") + sock.recv(16) # wait for done + return + recurse(n - 1) + +recurse({depth}) +""" + + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + unwinder_cache = make_unwinder(cache_frames=True) + unwinder_no_cache = make_unwinder(cache_frames=False) + + frames_cached = self._sample_frames( + client_socket, + unwinder_cache, + b"ready", + b"ack", + {"recurse"}, + expected_frames=1102, + ) + # Sample again with no cache for comparison + frames_no_cache = self._sample_frames( + client_socket, + unwinder_no_cache, + b"ready2", + b"done", + {"recurse"}, + expected_frames=1102, + ) + + self.assertIsNotNone(frames_cached) + self.assertIsNotNone(frames_no_cache) + + # Both should have many recurse frames (> 1024 limit) + cached_count = [f.funcname for f in frames_cached].count("recurse") + no_cache_count = [f.funcname for f in frames_no_cache].count("recurse") + + self.assertGreater( + cached_count, 1000, "Should have >1000 recurse frames" + ) + self.assertGreater( + no_cache_count, 1000, "Should have >1000 recurse frames" + ) + + # Both modes should produce same frame count + self.assertEqual( + len(frames_cached), + len(frames_no_cache), + "Cache exhaustion should not affect stack completeness", + ) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_get_stats(self): + """Test that get_stats() returns statistics when stats=True.""" + script_body = """\ + sock.sendall(b"ready") + sock.recv(16) + """ + + with self._target_process(script_body) as (p, client_socket, _): + unwinder = RemoteUnwinder(p.pid, all_threads=True, stats=True) + _wait_for_signal(client_socket, b"ready") + + # Take a sample + unwinder.get_stack_trace() + + stats = unwinder.get_stats() + client_socket.sendall(b"done") + + # Verify expected keys exist + expected_keys = [ + "total_samples", + "frame_cache_hits", + "frame_cache_misses", + "frame_cache_partial_hits", + "frames_read_from_cache", + "frames_read_from_memory", + "frame_cache_hit_rate", + ] + for key in expected_keys: + self.assertIn(key, stats) + + self.assertEqual(stats["total_samples"], 1) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_get_stats_disabled_raises(self): + """Test that get_stats() raises RuntimeError when stats=False.""" + script_body = """\ + sock.sendall(b"ready") + sock.recv(16) + """ + + with self._target_process(script_body) as (p, client_socket, _): + unwinder = RemoteUnwinder( + p.pid, all_threads=True + ) # stats=False by default + _wait_for_signal(client_socket, b"ready") + + with self.assertRaises(RuntimeError): + unwinder.get_stats() + + client_socket.sendall(b"done") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 371c63adce9..874f52af857 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -166,29 +166,6 @@ def check_windows_exception(self, code, line_number, name_regex, **kw): fatal_error = 'Windows fatal exception: %s' % name_regex self.check_error(code, line_number, fatal_error, **kw) - @unittest.skipIf(sys.platform.startswith('aix'), - "the first page of memory is a mapped read-only on AIX") - def test_read_null(self): - if not MS_WINDOWS: - self.check_fatal_error(""" - import faulthandler - faulthandler.enable() - faulthandler._read_null() - """, - 3, - # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion - '(?:Segmentation fault' - '|Bus error' - '|Illegal instruction)') - else: - self.check_windows_exception(""" - import faulthandler - faulthandler.enable() - faulthandler._read_null() - """, - 3, - 'access violation') - @skip_segfault_on_android def test_sigsegv(self): self.check_fatal_error(""" @@ -383,6 +360,17 @@ def test_enable_single_thread(self): 'Segmentation fault', all_threads=False) + @skip_segfault_on_android + def test_enable_without_c_stack(self): + self.check_fatal_error(""" + import faulthandler + faulthandler.enable(c_stack=False) + faulthandler._sigsegv() + """, + 3, + 'Segmentation fault', + c_stack=False) + @skip_segfault_on_android def test_disable(self): code = """ diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index b84c98ef3a2..222b69a6d25 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -8,10 +8,10 @@ import sys import unittest from test.support import ( - cpython_only, get_pagesize, is_apple, requires_subprocess, verbose + cpython_only, get_pagesize, is_apple, requires_subprocess, verbose, is_emscripten ) from test.support.import_helper import import_module -from test.support.os_helper import TESTFN, unlink +from test.support.os_helper import TESTFN, unlink, make_bad_fd # Skip test if no fcntl module. @@ -211,6 +211,7 @@ def test_fcntl_f_getpath(self): @unittest.skipUnless( hasattr(fcntl, "F_SETPIPE_SZ") and hasattr(fcntl, "F_GETPIPE_SZ"), "F_SETPIPE_SZ and F_GETPIPE_SZ are not available on all platforms.") + @unittest.skipIf(is_emscripten, "Emscripten pipefs doesn't support these") def test_fcntl_f_pipesize(self): test_pipe_r, test_pipe_w = os.pipe() try: @@ -228,6 +229,65 @@ def test_fcntl_f_pipesize(self): os.close(test_pipe_r) os.close(test_pipe_w) + def _check_fcntl_not_mutate_len(self, nbytes=None): + self.f = open(TESTFN, 'wb') + buf = struct.pack('ii', fcntl.F_OWNER_PID, os.getpid()) + if nbytes is not None: + buf += b' ' * (nbytes - len(buf)) + else: + nbytes = len(buf) + save_buf = bytes(buf) + r = fcntl.fcntl(self.f, fcntl.F_SETOWN_EX, buf) + self.assertIsInstance(r, bytes) + self.assertEqual(len(r), len(save_buf)) + self.assertEqual(buf, save_buf) + type, pid = memoryview(r).cast('i')[:2] + self.assertEqual(type, fcntl.F_OWNER_PID) + self.assertEqual(pid, os.getpid()) + + buf = b' ' * nbytes + r = fcntl.fcntl(self.f, fcntl.F_GETOWN_EX, buf) + self.assertIsInstance(r, bytes) + self.assertEqual(len(r), len(save_buf)) + self.assertEqual(buf, b' ' * nbytes) + type, pid = memoryview(r).cast('i')[:2] + self.assertEqual(type, fcntl.F_OWNER_PID) + self.assertEqual(pid, os.getpid()) + + buf = memoryview(b' ' * nbytes) + r = fcntl.fcntl(self.f, fcntl.F_GETOWN_EX, buf) + self.assertIsInstance(r, bytes) + self.assertEqual(len(r), len(save_buf)) + self.assertEqual(bytes(buf), b' ' * nbytes) + type, pid = memoryview(r).cast('i')[:2] + self.assertEqual(type, fcntl.F_OWNER_PID) + self.assertEqual(pid, os.getpid()) + + @unittest.skipUnless( + hasattr(fcntl, "F_SETOWN_EX") and hasattr(fcntl, "F_GETOWN_EX"), + "requires F_SETOWN_EX and F_GETOWN_EX") + @unittest.skipIf(is_emscripten, "Emscripten doesn't actually support these") + def test_fcntl_small_buffer(self): + self._check_fcntl_not_mutate_len() + + @unittest.skipUnless( + hasattr(fcntl, "F_SETOWN_EX") and hasattr(fcntl, "F_GETOWN_EX"), + "requires F_SETOWN_EX and F_GETOWN_EX") + @unittest.skipIf(is_emscripten, "Emscripten doesn't actually support these") + def test_fcntl_large_buffer(self): + self._check_fcntl_not_mutate_len(2024) + + @unittest.skipUnless(hasattr(fcntl, 'F_DUPFD'), 'need fcntl.F_DUPFD') + def test_bad_fd(self): + # gh-134744: Test error handling + fd = make_bad_fd() + with self.assertRaises(OSError): + fcntl.fcntl(fd, fcntl.F_DUPFD, 0) + with self.assertRaises(OSError): + fcntl.fcntl(fd, fcntl.F_DUPFD, b'\0' * 10) + with self.assertRaises(OSError): + fcntl.fcntl(fd, fcntl.F_DUPFD, b'\0' * 2048) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index b340ef7ed16..6524baabe7f 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -245,7 +245,7 @@ def test_detached_stdin_binary_mode(self): orig_stdin = sys.stdin try: sys.stdin = BytesIO(b'spam, bacon, sausage, and spam') - self.assertFalse(hasattr(sys.stdin, 'buffer')) + self.assertNotHasAttr(sys.stdin, 'buffer') fi = FileInput(files=['-'], mode='rb') lines = list(fi) self.assertEqual(lines, [b'spam, bacon, sausage, and spam']) diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py index 42871f8a09b..9dd68cf8d57 100644 --- a/Lib/test/test_finalization.py +++ b/Lib/test/test_finalization.py @@ -174,7 +174,7 @@ def test_simple(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) @@ -188,12 +188,12 @@ def test_simple_resurrect(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors(ids) - self.assertIsNot(wr(), None) + self.assertIsNotNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) @support.cpython_only def test_non_gc(self): @@ -265,7 +265,7 @@ def test_simple(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) @@ -276,19 +276,24 @@ def test_simple_resurrect(self): s = SelfCycleResurrector() ids = [id(s)] wr = weakref.ref(s) + wrc = weakref.ref(s, lambda x: None) del s gc.collect() self.assert_del_calls(ids) self.assert_survivors(ids) - # XXX is this desirable? - self.assertIs(wr(), None) + # This used to be None because weakrefs were cleared before + # calling finalizers. Now they are cleared after. + self.assertIsNotNone(wr()) + # A weakref with a callback is still cleared before calling + # finalizers. + self.assertIsNone(wrc()) # When trying to destroy the object a second time, __del__ # isn't called anymore (and the object isn't resurrected). self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) def test_simple_suicide(self): # Test the GC is able to deal with an object that kills its last @@ -301,11 +306,11 @@ def test_simple_suicide(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) class ChainedBase: @@ -378,18 +383,27 @@ def check_non_resurrecting_chain(self, classes): def check_resurrecting_chain(self, classes): N = len(classes) + def dummy_callback(ref): + pass with SimpleBase.test(): nodes = self.build_chain(classes) N = len(nodes) ids = [id(s) for s in nodes] survivor_ids = [id(s) for s in nodes if isinstance(s, SimpleResurrector)] wrs = [weakref.ref(s) for s in nodes] + wrcs = [weakref.ref(s, dummy_callback) for s in nodes] del nodes gc.collect() self.assert_del_calls(ids) self.assert_survivors(survivor_ids) - # XXX desirable? - self.assertEqual([wr() for wr in wrs], [None] * N) + for wr in wrs: + # These values used to be None because weakrefs were cleared + # before calling finalizers. Now they are cleared after. + self.assertIsNotNone(wr()) + for wr in wrcs: + # Weakrefs with callbacks are still cleared before calling + # finalizers. + self.assertIsNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) @@ -491,7 +505,7 @@ def test_legacy(self): self.assert_del_calls(ids) self.assert_tp_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_tp_del_calls(ids) @@ -507,13 +521,13 @@ def test_legacy_resurrect(self): self.assert_tp_del_calls(ids) self.assert_survivors(ids) # weakrefs are cleared before tp_del is called. - self.assertIs(wr(), None) + self.assertIsNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_tp_del_calls(ids * 2) self.assert_survivors(ids) - self.assertIs(wr(), None) + self.assertIsNone(wr()) def test_legacy_self_cycle(self): # Self-cycles with legacy finalizers end up in gc.garbage. @@ -527,11 +541,11 @@ def test_legacy_self_cycle(self): self.assert_tp_del_calls([]) self.assert_survivors([]) self.assert_garbage(ids) - self.assertIsNot(wr(), None) + self.assertIsNotNone(wr()) # Break the cycle to allow collection gc.garbage[0].ref = None self.assert_garbage([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) if __name__ == "__main__": diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 237d7b5d35e..00518abcb11 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -795,6 +795,8 @@ def test_format(self): self.assertRaises(ValueError, format, x, '.6,n') @support.requires_IEEE_754 + @unittest.skipUnless(sys.float_repr_style == 'short', + "applies only when using short float repr style") def test_format_testfile(self): with open(format_testfile, encoding="utf-8") as testfile: for line in testfile: diff --git a/Lib/test/test_fork1.py b/Lib/test/test_fork1.py index a6523bbc518..550faa8a174 100644 --- a/Lib/test/test_fork1.py +++ b/Lib/test/test_fork1.py @@ -11,6 +11,7 @@ from test.fork_wait import ForkWait from test import support +from test.support import warnings_helper # Skip test if fork does not exist. @@ -19,6 +20,7 @@ class ForkTest(ForkWait): + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_threaded_import_lock_fork(self): """Check fork() in main thread works while a subthread is doing an import""" import_started = threading.Event() @@ -61,7 +63,7 @@ def importer(): except OSError: pass - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_nested_import_lock_fork(self): """Check fork() in main thread works while the main thread is doing an import""" exitcode = 42 diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index c7cc32e0949..1f626d87fa6 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -346,12 +346,12 @@ def __bytes__(self): testcommon(b"%s", memoryview(b"abc"), b"abc") # %a will give the equivalent of # repr(some_obj).encode('ascii', 'backslashreplace') - testcommon(b"%a", 3.14, b"3.14") + testcommon(b"%a", 3.25, b"3.25") testcommon(b"%a", b"ghi", b"b'ghi'") testcommon(b"%a", "jkl", b"'jkl'") testcommon(b"%a", "\u0544", b"'\\u0544'") # %r is an alias for %a - testcommon(b"%r", 3.14, b"3.14") + testcommon(b"%r", 3.25, b"3.25") testcommon(b"%r", b"ghi", b"b'ghi'") testcommon(b"%r", "jkl", b"'jkl'") testcommon(b"%r", "\u0544", b"'\\u0544'") @@ -407,19 +407,19 @@ def test_non_ascii(self): self.assertEqual(format("abc", "\u2007<5"), "abc\u2007\u2007") self.assertEqual(format(123, "\u2007<5"), "123\u2007\u2007") - self.assertEqual(format(12.3, "\u2007<6"), "12.3\u2007\u2007") + self.assertEqual(format(12.5, "\u2007<6"), "12.5\u2007\u2007") self.assertEqual(format(0j, "\u2007<4"), "0j\u2007\u2007") self.assertEqual(format(1+2j, "\u2007<8"), "(1+2j)\u2007\u2007") self.assertEqual(format("abc", "\u2007>5"), "\u2007\u2007abc") self.assertEqual(format(123, "\u2007>5"), "\u2007\u2007123") - self.assertEqual(format(12.3, "\u2007>6"), "\u2007\u200712.3") + self.assertEqual(format(12.5, "\u2007>6"), "\u2007\u200712.5") self.assertEqual(format(1+2j, "\u2007>8"), "\u2007\u2007(1+2j)") self.assertEqual(format(0j, "\u2007>4"), "\u2007\u20070j") self.assertEqual(format("abc", "\u2007^5"), "\u2007abc\u2007") self.assertEqual(format(123, "\u2007^5"), "\u2007123\u2007") - self.assertEqual(format(12.3, "\u2007^6"), "\u200712.3\u2007") + self.assertEqual(format(12.5, "\u2007^6"), "\u200712.5\u2007") self.assertEqual(format(1+2j, "\u2007^8"), "\u2007(1+2j)\u2007") self.assertEqual(format(0j, "\u2007^4"), "\u20070j\u2007") diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 84faa636064..cf42b86358d 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1,7 +1,7 @@ """Tests for Lib/fractions.py.""" from decimal import Decimal -from test.support import requires_IEEE_754 +from test.support import requires_IEEE_754, adjust_int_max_str_digits import math import numbers import operator @@ -395,12 +395,14 @@ class B(metaclass=M): def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) + self.assertEqual((5, 1), _components(F("005"))) self.assertEqual((3, 2), _components(F("3/2"))) self.assertEqual((3, 2), _components(F("3 / 2"))) self.assertEqual((3, 2), _components(F(" \n +3/2"))) self.assertEqual((-3, 2), _components(F("-3/2 "))) - self.assertEqual((13, 2), _components(F(" 013/02 \n "))) + self.assertEqual((13, 2), _components(F(" 0013/002 \n "))) self.assertEqual((16, 5), _components(F(" 3.2 "))) + self.assertEqual((16, 5), _components(F("003.2"))) self.assertEqual((-16, 5), _components(F(" -3.2 "))) self.assertEqual((-3, 1), _components(F(" -3. "))) self.assertEqual((3, 5), _components(F(" .6 "))) @@ -419,116 +421,102 @@ def testFromString(self): self.assertRaisesMessage( ZeroDivisionError, "Fraction(3, 0)", F, "3/0") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '3/'", - F, "3/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/2'", - F, "/2") - self.assertRaisesMessage( - # Denominators don't need a sign. - ValueError, "Invalid literal for Fraction: '3/+2'", - F, "3/+2") - self.assertRaisesMessage( - # Imitate float's parsing. - ValueError, "Invalid literal for Fraction: '+ 3/2'", - F, "+ 3/2") - self.assertRaisesMessage( - # Avoid treating '.' as a regex special character. - ValueError, "Invalid literal for Fraction: '3a2'", - F, "3a2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3/7.2'", - F, "3/7.2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3.2/7'", - F, "3.2/7") - self.assertRaisesMessage( - # Allow 3. and .3, but not . - ValueError, "Invalid literal for Fraction: '.'", - F, ".") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_'", - F, "_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1'", - F, "_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2'", - F, "1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/_'", - F, "/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1_/'", - F, "1_/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1/'", - F, "_1/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2/'", - F, "1__2/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_'", - F, "1/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_1'", - F, "1/_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/1__2'", - F, "1/1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1._111'", - F, "1._111") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1__1'", - F, "1.1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+_1'", - F, "1.1e+_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+1__1'", - F, "1.1e+1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.dd'", - F, "123.dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.5_dd'", - F, "123.5_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: 'dd.5'", - F, "dd.5") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '7_dd'", - F, "7_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/dd'", - F, "1/dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/123_dd'", - F, "1/123_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789edd'", - F, "789edd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789e2_dd'", - F, "789e2_dd") + + def check_invalid(s): + msg = "Invalid literal for Fraction: " + repr(s) + self.assertRaisesMessage(ValueError, msg, F, s) + + check_invalid("3/") + check_invalid("/2") + # Denominators don't need a sign. + check_invalid("3/+2") + check_invalid("3/-2") + # Imitate float's parsing. + check_invalid("+ 3/2") + check_invalid("- 3/2") + # Avoid treating '.' as a regex special character. + check_invalid("3a2") + # Don't accept combinations of decimals and rationals. + check_invalid("3/7.2") + check_invalid("3.2/7") + # No space around dot. + check_invalid("3 .2") + check_invalid("3. 2") + # No space around e. + check_invalid("3.2 e1") + check_invalid("3.2e 1") + # Fractional part don't need a sign. + check_invalid("3.+2") + check_invalid("3.-2") + # Only accept base 10. + check_invalid("0x10") + check_invalid("0x10/1") + check_invalid("1/0x10") + check_invalid("0x10.") + check_invalid("0x10.1") + check_invalid("1.0x10") + check_invalid("1.0e0x10") + # Only accept decimal digits. + check_invalid("³") + check_invalid("³/2") + check_invalid("3/²") + check_invalid("³.2") + check_invalid("3.²") + check_invalid("3.2e²") + check_invalid("¼") + # Allow 3. and .3, but not . + check_invalid(".") + check_invalid("_") + check_invalid("_1") + check_invalid("1__2") + check_invalid("/_") + check_invalid("1_/") + check_invalid("_1/") + check_invalid("1__2/") + check_invalid("1/_") + check_invalid("1/_1") + check_invalid("1/1__2") + check_invalid("1._111") + check_invalid("1.1__1") + check_invalid("1.1e+_1") + check_invalid("1.1e+1__1") + check_invalid("123.dd") + check_invalid("123.5_dd") + check_invalid("dd.5") + check_invalid("7_dd") + check_invalid("1/dd") + check_invalid("1/123_dd") + check_invalid("789edd") + check_invalid("789e2_dd") # Test catastrophic backtracking. val = "9"*50 + "_" - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '" + val + "'", - F, val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/" + val + "'", - F, "1/" + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1." + val + "'", - F, "1." + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1+e" + val + "'", - F, "1.1+e" + val) + check_invalid(val) + check_invalid("1/" + val) + check_invalid("1." + val) + check_invalid("." + val) + check_invalid("1.1+e" + val) + check_invalid("1.1e" + val) + + def test_limit_int(self): + maxdigits = 5000 + with adjust_int_max_str_digits(maxdigits): + msg = 'Exceeds the limit' + val = '1' * maxdigits + num = (10**maxdigits - 1)//9 + self.assertEqual((num, 1), _components(F(val))) + self.assertRaisesRegex(ValueError, msg, F, val + '1') + self.assertEqual((num, 2), _components(F(val + '/2'))) + self.assertRaisesRegex(ValueError, msg, F, val + '1/2') + self.assertEqual((1, num), _components(F('1/' + val))) + self.assertRaisesRegex(ValueError, msg, F, '1/1' + val) + self.assertEqual(((10**(maxdigits+1) - 1)//9, 10**maxdigits), + _components(F('1.' + val))) + self.assertRaisesRegex(ValueError, msg, F, '1.1' + val) + self.assertEqual((num, 10**maxdigits), _components(F('.' + val))) + self.assertRaisesRegex(ValueError, msg, F, '.1' + val) + self.assertRaisesRegex(ValueError, msg, F, '1.1e1' + val) + self.assertEqual((11, 10), _components(F('1.1e' + '0' * maxdigits))) + self.assertRaisesRegex(ValueError, msg, F, '1.1e' + '0' * (maxdigits+1)) def testImmutable(self): r = F(7, 3) @@ -1334,6 +1322,8 @@ def test_format_e_presentation_type(self): # Thousands separators (F('1234567.123456'), ',.5e', '1.23457e+06'), (F('123.123456'), '012_.2e', '0_001.23e+02'), + # Thousands separators for fractional part (or for integral too) + (F('1234567.123456'), '.5_e', '1.234_57e+06'), # z flag is legal, but never makes a difference to the output (F(-1, 7**100), 'z.6e', '-3.091690e-85'), ] @@ -1459,6 +1449,12 @@ def test_format_f_presentation_type(self): (F('1234567'), ',.2f', '1,234,567.00'), (F('12345678'), ',.2f', '12,345,678.00'), (F('12345678'), ',f', '12,345,678.000000'), + # Thousands separators for fractional part (or for integral too) + (F('123456.789123123'), '._f', '123456.789_123'), + (F('123456.789123123'), '.7_f', '123456.789_123_1'), + (F('123456.789123123'), '.9_f', '123456.789_123_123'), + (F('123456.789123123'), '.,f', '123456.789,123'), + (F('123456.789123123'), '_.,f', '123_456.789,123'), # Underscore as thousands separator (F(2, 3), '_.2f', '0.67'), (F(2, 3), '_.7f', '0.6666667'), @@ -1492,11 +1488,8 @@ def test_format_f_presentation_type(self): (F('-1234.5678'), '08,.0f', '-001,235'), (F('-1234.5678'), '09,.0f', '-0,001,235'), # Corner-case - zero-padding specified through fill and align - # instead of the zero-pad character - in this case, treat '0' as a - # regular fill character and don't attempt to insert commas into - # the filled portion. This differs from the int and float - # behaviour. - (F('1234.5678'), '0=12,.2f', '00001,234.57'), + # instead of the zero-pad character. + (F('1234.5678'), '0=12,.2f', '0,001,234.57'), # Corner case where it's not clear whether the '0' indicates zero # padding or gives the minimum width, but there's still an obvious # answer to give. We want this to work in case the minimum width @@ -1530,6 +1523,8 @@ def test_format_f_presentation_type(self): (F(51, 1000), '.1f', '0.1'), (F(149, 1000), '.1f', '0.1'), (F(151, 1000), '.1f', '0.2'), + (F(22, 7), '.02f', '3.14'), # issue gh-130662 + (F(22, 7), '005.02f', '03.14'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1628,17 +1623,16 @@ def test_invalid_formats(self): '=010%', '>00.2f', '>00f', - # Too many zeros - minimum width should not have leading zeros - '006f', - # Leading zeros in precision - '.010f', - '.02f', - '.000f', # Missing precision '.e', '.f', '.g', '.%', + # Thousands separators before precision + '._6e', + '._6f', + '._6g', + '._6%', # Z instead of z for negative zero suppression 'Z.2f' # z flag not supported for general formatting diff --git a/Lib/test/test_free_threading/test_bisect.py b/Lib/test/test_free_threading/test_bisect.py new file mode 100644 index 00000000000..bd7732da035 --- /dev/null +++ b/Lib/test/test_free_threading/test_bisect.py @@ -0,0 +1,56 @@ +import unittest +from test.support import import_helper, threading_helper +import random + +py_bisect = import_helper.import_fresh_module('bisect', blocked=['_bisect']) +c_bisect = import_helper.import_fresh_module('bisect', fresh=['_bisect']) + + +NTHREADS = 4 +OBJECT_COUNT = 500 + + +class TestBase: + def do_racing_insort(self, insert_method): + def insert(data): + for _ in range(OBJECT_COUNT): + x = random.randint(-OBJECT_COUNT, OBJECT_COUNT) + insert_method(data, x) + + data = list(range(OBJECT_COUNT)) + threading_helper.run_concurrently( + worker_func=insert, args=(data,), nthreads=NTHREADS + ) + if False: + # These functions are not thread-safe and so the list can become + # unsorted. However, we don't want Python to crash if these + # functions are used concurrently on the same sequence. This + # should also not produce any TSAN warnings. + self.assertTrue(self.is_sorted_ascending(data)) + + def test_racing_insert_right(self): + self.do_racing_insort(self.mod.insort_right) + + def test_racing_insert_left(self): + self.do_racing_insort(self.mod.insort_left) + + @staticmethod + def is_sorted_ascending(lst): + """ + Check if the list is sorted in ascending order (non-decreasing). + """ + return all(lst[i - 1] <= lst[i] for i in range(1, len(lst))) + + +@threading_helper.requires_working_threading() +class TestPyBisect(unittest.TestCase, TestBase): + mod = py_bisect + + +@threading_helper.requires_working_threading() +class TestCBisect(unittest.TestCase, TestBase): + mod = c_bisect + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_bz2.py b/Lib/test/test_free_threading/test_bz2.py new file mode 100644 index 00000000000..0e09c64d561 --- /dev/null +++ b/Lib/test/test_free_threading/test_bz2.py @@ -0,0 +1,53 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +bz2 = import_helper.import_module("bz2") +from bz2 import BZ2Compressor, BZ2Decompressor + +from test.test_bz2 import ext_decompress, BaseTest + + +NTHREADS = 10 +TEXT = BaseTest.TEXT + + +@threading_helper.requires_working_threading() +class TestBZ2(unittest.TestCase): + def test_compressor(self): + bz2c = BZ2Compressor() + + def worker(): + # it should return empty bytes as it buffers data internally + data = bz2c.compress(TEXT) + self.assertEqual(data, b"") + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + data = bz2c.flush() + # The decompressed data should be TEXT repeated NTHREADS times + decompressed = ext_decompress(data) + self.assertEqual(decompressed, TEXT * NTHREADS) + + def test_decompressor(self): + chunk_size = 128 + chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)] + input_data = b"".join(chunks) + compressed = bz2.compress(input_data) + + bz2d = BZ2Decompressor() + output = [] + + def worker(): + data = bz2d.decompress(compressed, chunk_size) + self.assertEqual(len(data), chunk_size) + output.append(data) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(output), NTHREADS) + # Verify the expected chunks (order doesn't matter due to append race) + self.assertEqual(set(output), set(chunks)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_capi.py b/Lib/test/test_free_threading/test_capi.py new file mode 100644 index 00000000000..146d7cfc97a --- /dev/null +++ b/Lib/test/test_free_threading/test_capi.py @@ -0,0 +1,47 @@ +import ctypes +import sys +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + + +_PyImport_AddModuleRef = ctypes.pythonapi.PyImport_AddModuleRef +_PyImport_AddModuleRef.argtypes = (ctypes.c_char_p,) +_PyImport_AddModuleRef.restype = ctypes.py_object + + +@threading_helper.requires_working_threading() +class TestImportCAPI(unittest.TestCase): + def test_pyimport_addmoduleref_thread_safe(self): + # gh-137422: Concurrent calls to PyImport_AddModuleRef with the same + # module name must return the same module object. + + NUM_ITERS = 10 + NTHREADS = 4 + + module_name = f"test_free_threading_addmoduleref_{id(self)}" + module_name_bytes = module_name.encode() + sys.modules.pop(module_name, None) + results = [] + + def worker(): + module = _PyImport_AddModuleRef(module_name_bytes) + results.append(module) + + for _ in range(NUM_ITERS): + try: + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(results), NTHREADS) + reference = results[0] + for module in results[1:]: + self.assertIs(module, reference) + self.assertIn(module_name, sys.modules) + self.assertIs(sys.modules[module_name], reference) + finally: + results.clear() + sys.modules.pop(module_name, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_cprofile.py b/Lib/test/test_free_threading/test_cprofile.py new file mode 100644 index 00000000000..361b800d6b9 --- /dev/null +++ b/Lib/test/test_free_threading/test_cprofile.py @@ -0,0 +1,43 @@ +import unittest + +from test.support import threading_helper + +import cProfile +import pstats + + +NTHREADS = 10 +INSERT_PER_THREAD = 1000 + + +@threading_helper.requires_working_threading() +class TestCProfile(unittest.TestCase): + def test_cprofile_racing_list_insert(self): + def list_insert(lst): + for i in range(INSERT_PER_THREAD): + lst.insert(0, i) + + lst = [] + + with cProfile.Profile() as pr: + threading_helper.run_concurrently( + worker_func=list_insert, nthreads=NTHREADS, args=(lst,) + ) + pr.create_stats() + ps = pstats.Stats(pr) + stats_profile = ps.get_stats_profile() + list_insert_profile = stats_profile.func_profiles[ + "" + ] + # Even though there is no explicit recursive call to insert, + # cProfile may record some calls as recursive due to limitations + # in its handling of multithreaded programs. This issue is not + # directly related to FT Python itself; however, it tends to be + # more noticeable when using FT Python. Therefore, consider only + # the calls section and disregard the recursive part. + list_insert_ncalls = list_insert_profile.ncalls.split("/")[0] + self.assertEqual( + int(list_insert_ncalls), NTHREADS * INSERT_PER_THREAD + ) + + self.assertEqual(len(lst), NTHREADS * INSERT_PER_THREAD) diff --git a/Lib/test/test_free_threading/test_csv.py b/Lib/test/test_free_threading/test_csv.py new file mode 100644 index 00000000000..beb4510a128 --- /dev/null +++ b/Lib/test/test_free_threading/test_csv.py @@ -0,0 +1,50 @@ +import csv +import io +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestCSV(unittest.TestCase): + def test_concurrent_reader_next(self): + input_rows = [f"{i},{i},{i}" for i in range(50)] + input_stream = io.StringIO("\n".join(input_rows)) + reader = csv.reader(input_stream) + output_rows = [] + + def read_row(): + for row in reader: + self.assertEqual(len(row), 3) + output_rows.append(",".join(row)) + + run_concurrently(worker_func=read_row, nthreads=NTHREADS) + self.assertSetEqual(set(input_rows), set(output_rows)) + + def test_concurrent_writer_writerow(self): + output_stream = io.StringIO() + writer = csv.writer(output_stream) + row_per_thread = 10 + expected_rows = [] + + def write_row(): + for i in range(row_per_thread): + writer.writerow([i, i, i]) + expected_rows.append(f"{i},{i},{i}") + + run_concurrently(worker_func=write_row, nthreads=NTHREADS) + + # Rewind to the start of the stream and parse the rows + output_stream.seek(0) + output_rows = [line.strip() for line in output_stream.readlines()] + + self.assertEqual(len(output_rows), NTHREADS * row_per_thread) + self.assertListEqual(sorted(output_rows), sorted(expected_rows)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_dbm_gnu.py b/Lib/test/test_free_threading/test_dbm_gnu.py new file mode 100644 index 00000000000..d2d7b78be71 --- /dev/null +++ b/Lib/test/test_free_threading/test_dbm_gnu.py @@ -0,0 +1,79 @@ +import unittest + +from test.support import import_helper, os_helper, threading_helper +from test.support.threading_helper import run_concurrently + +import threading + +gdbm = import_helper.import_module("dbm.gnu") + +NTHREADS = 10 +KEY_PER_THREAD = 1000 + +gdbm_filename = "test_gdbm_file" + + +@threading_helper.requires_working_threading() +class TestGdbm(unittest.TestCase): + def test_racing_dbm_gnu(self): + def gdbm_multi_op_worker(db): + # Each thread sets, gets, and iterates + tid = threading.get_ident() + + # Insert keys + for i in range(KEY_PER_THREAD): + db[f"key_{tid}_{i}"] = f"value_{tid}_{i}" + + for i in range(KEY_PER_THREAD): + # Keys and values are stored as bytes; encode values for + # comparison + key = f"key_{tid}_{i}" + value = f"value_{tid}_{i}".encode() + self.assertIn(key, db) + self.assertEqual(db[key], value) + self.assertEqual(db.get(key), value) + self.assertIsNone(db.get("not_exist")) + with self.assertRaises(KeyError): + db["not_exist"] + + # Iterate over the database keys and verify only those belonging + # to this thread. Other threads may concurrently delete their keys. + key_prefix = f"key_{tid}".encode() + key = db.firstkey() + key_count = 0 + while key: + if key.startswith(key_prefix): + self.assertIn(key, db) + key_count += 1 + key = db.nextkey(key) + + # Can't assert key_count == KEY_PER_THREAD because concurrent + # threads may insert or delete keys during iteration. This can + # cause keys to be skipped or counted multiple times, making the + # count unreliable. + # See: https://www.gnu.org.ua/software/gdbm/manual/Sequential.html + # self.assertEqual(key_count, KEY_PER_THREAD) + + # Delete this thread's keys + for i in range(KEY_PER_THREAD): + key = f"key_{tid}_{i}" + del db[key] + self.assertNotIn(key, db) + with self.assertRaises(KeyError): + del db["not_exist"] + + # Re-insert keys + for i in range(KEY_PER_THREAD): + db[f"key_{tid}_{i}"] = f"value_{tid}_{i}" + + with os_helper.temp_dir() as tmpdirname: + db = gdbm.open(f"{tmpdirname}/{gdbm_filename}", "c") + run_concurrently( + worker_func=gdbm_multi_op_worker, nthreads=NTHREADS, args=(db,) + ) + self.assertEqual(len(db), NTHREADS * KEY_PER_THREAD) + db.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 476cc3178d8..5d5d4e226ca 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -228,6 +228,22 @@ def reader_func(): self.assertEqual(count, 0) + def test_racing_object_get_set_dict(self): + e = Exception() + + def writer(): + for i in range(10000): + e.__dict__ = {1:2} + + def reader(): + for i in range(10000): + e.__dict__ + + t1 = Thread(target=writer) + t2 = Thread(target=reader) + + with threading_helper.start_threads([t1, t2]): + pass if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_functools.py b/Lib/test/test_free_threading/test_functools.py new file mode 100644 index 00000000000..a442fe056ce --- /dev/null +++ b/Lib/test/test_free_threading/test_functools.py @@ -0,0 +1,75 @@ +import random +import unittest + +from functools import lru_cache +from threading import Barrier, Thread + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestLRUCache(unittest.TestCase): + + def _test_concurrent_operations(self, maxsize): + num_threads = 10 + b = Barrier(num_threads) + @lru_cache(maxsize=maxsize) + def func(arg=0): + return object() + + + def thread_func(): + b.wait() + for i in range(1000): + r = random.randint(0, 1000) + if i < 800: + func(i) + elif i < 900: + func.cache_info() + else: + func.cache_clear() + + threads = [] + for i in range(num_threads): + t = Thread(target=thread_func) + threads.append(t) + + with threading_helper.start_threads(threads): + pass + + def test_concurrent_operations_unbounded(self): + self._test_concurrent_operations(maxsize=None) + + def test_concurrent_operations_bounded(self): + self._test_concurrent_operations(maxsize=128) + + def _test_reentrant_cache_clear(self, maxsize): + num_threads = 10 + b = Barrier(num_threads) + @lru_cache(maxsize=maxsize) + def func(arg=0): + func.cache_clear() + return object() + + + def thread_func(): + b.wait() + for i in range(1000): + func(random.randint(0, 10000)) + + threads = [] + for i in range(num_threads): + t = Thread(target=thread_func) + threads.append(t) + + with threading_helper.start_threads(threads): + pass + + def test_reentrant_cache_clear_unbounded(self): + self._test_reentrant_cache_clear(maxsize=None) + + def test_reentrant_cache_clear_bounded(self): + self._test_reentrant_cache_clear(maxsize=128) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_generators.py b/Lib/test/test_free_threading/test_generators.py new file mode 100644 index 00000000000..d01675eb38b --- /dev/null +++ b/Lib/test/test_free_threading/test_generators.py @@ -0,0 +1,51 @@ +import concurrent.futures +import unittest +from threading import Barrier +from unittest import TestCase +import random +import time + +from test.support import threading_helper, Py_GIL_DISABLED + +threading_helper.requires_working_threading(module=True) + + +def random_sleep(): + delay_us = random.randint(50, 100) + time.sleep(delay_us * 1e-6) + +def random_string(): + return ''.join(random.choice('0123456789ABCDEF') for _ in range(10)) + +def set_gen_name(g, b): + b.wait() + random_sleep() + g.__name__ = random_string() + return g.__name__ + +def set_gen_qualname(g, b): + b.wait() + random_sleep() + g.__qualname__ = random_string() + return g.__qualname__ + + +@unittest.skipUnless(Py_GIL_DISABLED, "Enable only in FT build") +class TestFTGenerators(TestCase): + NUM_THREADS = 4 + + def concurrent_write_with_func(self, func): + gen = (x for x in range(42)) + for j in range(1000): + with concurrent.futures.ThreadPoolExecutor(max_workers=self.NUM_THREADS) as executor: + b = Barrier(self.NUM_THREADS) + futures = {executor.submit(func, gen, b): i for i in range(self.NUM_THREADS)} + for fut in concurrent.futures.as_completed(futures): + gen_name = fut.result() + self.assertEqual(len(gen_name), 10) + + def test_concurrent_write(self): + with self.subTest(func=set_gen_name): + self.concurrent_write_with_func(func=set_gen_name) + with self.subTest(func=set_gen_qualname): + self.concurrent_write_with_func(func=set_gen_qualname) diff --git a/Lib/test/test_free_threading/test_grp.py b/Lib/test/test_free_threading/test_grp.py new file mode 100644 index 00000000000..1a47a975770 --- /dev/null +++ b/Lib/test/test_free_threading/test_grp.py @@ -0,0 +1,35 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +grp = import_helper.import_module("grp") + +from test import test_grp + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestGrp(unittest.TestCase): + def setUp(self): + self.test_grp = test_grp.GroupDatabaseTestCase() + + def test_racing_test_values(self): + # test_grp.test_values() calls grp.getgrall() and checks the entries + run_concurrently( + worker_func=self.test_grp.test_values, nthreads=NTHREADS + ) + + def test_racing_test_values_extended(self): + # test_grp.test_values_extended() calls grp.getgrall(), grp.getgrgid(), + # grp.getgrnam() and checks the entries + run_concurrently( + worker_func=self.test_grp.test_values_extended, + nthreads=NTHREADS, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py new file mode 100644 index 00000000000..d771333ffcc --- /dev/null +++ b/Lib/test/test_free_threading/test_heapq.py @@ -0,0 +1,247 @@ +import unittest + +import heapq + +from enum import Enum +from threading import Barrier, Lock +from random import shuffle, randint + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently +from test import test_heapq + + +NTHREADS = 10 +OBJECT_COUNT = 5_000 + + +class Heap(Enum): + MIN = 1 + MAX = 2 + + +@threading_helper.requires_working_threading() +class TestHeapq(unittest.TestCase): + def setUp(self): + self.test_heapq = test_heapq.TestHeapPython() + + def test_racing_heapify(self): + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + + run_concurrently( + worker_func=heapq.heapify, nthreads=NTHREADS, args=(heap,) + ) + self.test_heapq.check_invariant(heap) + + def test_racing_heappush(self): + heap = [] + + def heappush_func(heap): + for item in reversed(range(OBJECT_COUNT)): + heapq.heappush(heap, item) + + run_concurrently( + worker_func=heappush_func, nthreads=NTHREADS, args=(heap,) + ) + self.test_heapq.check_invariant(heap) + + def test_racing_heappop(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + + # Each thread pops (OBJECT_COUNT / NTHREADS) items + self.assertEqual(OBJECT_COUNT % NTHREADS, 0) + per_thread_pop_count = OBJECT_COUNT // NTHREADS + + def heappop_func(heap, pop_count): + local_list = [] + for _ in range(pop_count): + item = heapq.heappop(heap) + local_list.append(item) + + # Each local list should be sorted + self.assertTrue(self.is_sorted_ascending(local_list)) + + run_concurrently( + worker_func=heappop_func, + nthreads=NTHREADS, + args=(heap, per_thread_pop_count), + ) + self.assertEqual(len(heap), 0) + + def test_racing_heappushpop(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heappushpop_func(heap, pushpop_items): + for item in pushpop_items: + popped_item = heapq.heappushpop(heap, item) + self.assertTrue(popped_item <= item) + + run_concurrently( + worker_func=heappushpop_func, + nthreads=NTHREADS, + args=(heap, pushpop_items), + ) + self.assertEqual(len(heap), OBJECT_COUNT) + self.test_heapq.check_invariant(heap) + + def test_racing_heapreplace(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heapreplace_func(heap, replace_items): + for item in replace_items: + heapq.heapreplace(heap, item) + + run_concurrently( + worker_func=heapreplace_func, + nthreads=NTHREADS, + args=(heap, replace_items), + ) + self.assertEqual(len(heap), OBJECT_COUNT) + self.test_heapq.check_invariant(heap) + + def test_racing_heapify_max(self): + max_heap = list(range(OBJECT_COUNT)) + shuffle(max_heap) + + run_concurrently( + worker_func=heapq.heapify_max, nthreads=NTHREADS, args=(max_heap,) + ) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heappush_max(self): + max_heap = [] + + def heappush_max_func(max_heap): + for item in range(OBJECT_COUNT): + heapq.heappush_max(max_heap, item) + + run_concurrently( + worker_func=heappush_max_func, nthreads=NTHREADS, args=(max_heap,) + ) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heappop_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + + # Each thread pops (OBJECT_COUNT / NTHREADS) items + self.assertEqual(OBJECT_COUNT % NTHREADS, 0) + per_thread_pop_count = OBJECT_COUNT // NTHREADS + + def heappop_max_func(max_heap, pop_count): + local_list = [] + for _ in range(pop_count): + item = heapq.heappop_max(max_heap) + local_list.append(item) + + # Each local list should be sorted + self.assertTrue(self.is_sorted_descending(local_list)) + + run_concurrently( + worker_func=heappop_max_func, + nthreads=NTHREADS, + args=(max_heap, per_thread_pop_count), + ) + self.assertEqual(len(max_heap), 0) + + def test_racing_heappushpop_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heappushpop_max_func(max_heap, pushpop_items): + for item in pushpop_items: + popped_item = heapq.heappushpop_max(max_heap, item) + self.assertTrue(popped_item >= item) + + run_concurrently( + worker_func=heappushpop_max_func, + nthreads=NTHREADS, + args=(max_heap, pushpop_items), + ) + self.assertEqual(len(max_heap), OBJECT_COUNT) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heapreplace_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heapreplace_max_func(max_heap, replace_items): + for item in replace_items: + heapq.heapreplace_max(max_heap, item) + + run_concurrently( + worker_func=heapreplace_max_func, + nthreads=NTHREADS, + args=(max_heap, replace_items), + ) + self.assertEqual(len(max_heap), OBJECT_COUNT) + self.test_heapq.check_max_invariant(max_heap) + + def test_lock_free_list_read(self): + n, n_threads = 1_000, 10 + l = [] + barrier = Barrier(n_threads * 2) + + count = 0 + lock = Lock() + + def worker(): + with lock: + nonlocal count + x = count + count += 1 + + barrier.wait() + for i in range(n): + if x % 2: + heapq.heappush(l, 1) + heapq.heappop(l) + else: + try: + l[0] + except IndexError: + pass + + run_concurrently(worker, n_threads * 2) + + @staticmethod + def is_sorted_ascending(lst): + """ + Check if the list is sorted in ascending order (non-decreasing). + """ + return all(lst[i - 1] <= lst[i] for i in range(1, len(lst))) + + @staticmethod + def is_sorted_descending(lst): + """ + Check if the list is sorted in descending order (non-increasing). + """ + return all(lst[i - 1] >= lst[i] for i in range(1, len(lst))) + + @staticmethod + def create_heap(size, heap_kind): + """ + Create a min/max heap where elements are in the range (0, size - 1) and + shuffled before heapify. + """ + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + if heap_kind == Heap.MIN: + heapq.heapify(heap) + else: + heapq.heapify_max(heap) + + return heap + + @staticmethod + def create_random_list(a, b, size): + """ + Create a list of random numbers between a and b (inclusive). + """ + return [randint(-a, b) for _ in range(size)] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_free_threading/test_io.py new file mode 100644 index 00000000000..c67aaff31b3 --- /dev/null +++ b/Lib/test/test_free_threading/test_io.py @@ -0,0 +1,117 @@ +import io +import _pyio as pyio +import threading +from unittest import TestCase +from test.support import threading_helper +from random import randint +from sys import getsizeof + + +class ThreadSafetyMixin: + # Test pretty much everything that can break under free-threading. + # Non-deterministic, but at least one of these things will fail if + # BytesIO object is not free-thread safe. + + def check(self, funcs, *args): + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier, *args)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_free_threading(self): + """Test for segfaults and aborts.""" + + def write(barrier, b, *ignore): + barrier.wait() + try: b.write(b'0' * randint(100, 1000)) + except ValueError: pass # ignore write fail to closed file + + def writelines(barrier, b, *ignore): + barrier.wait() + b.write(b'0\n' * randint(100, 1000)) + + def truncate(barrier, b, *ignore): + barrier.wait() + try: b.truncate(0) + except BufferError: pass # ignore exported buffer + + def read(barrier, b, *ignore): + barrier.wait() + b.read() + + def read1(barrier, b, *ignore): + barrier.wait() + b.read1() + + def readline(barrier, b, *ignore): + barrier.wait() + b.readline() + + def readlines(barrier, b, *ignore): + barrier.wait() + b.readlines() + + def readinto(barrier, b, into, *ignore): + barrier.wait() + b.readinto(into) + + def close(barrier, b, *ignore): + barrier.wait() + b.close() + + def getvalue(barrier, b, *ignore): + barrier.wait() + b.getvalue() + + def getbuffer(barrier, b, *ignore): + barrier.wait() + b.getbuffer() + + def iter(barrier, b, *ignore): + barrier.wait() + list(b) + + def getstate(barrier, b, *ignore): + barrier.wait() + b.__getstate__() + + def setstate(barrier, b, st, *ignore): + barrier.wait() + b.__setstate__(st) + + def sizeof(barrier, b, *ignore): + barrier.wait() + getsizeof(b) + + self.check([write] * 10, self.ioclass()) + self.check([writelines] * 10, self.ioclass()) + self.check([write] * 10 + [truncate] * 10, self.ioclass()) + self.check([truncate] + [read] * 10, self.ioclass(b'0\n'*204800)) + self.check([truncate] + [read1] * 10, self.ioclass(b'0\n'*204800)) + self.check([truncate] + [readline] * 10, self.ioclass(b'0\n'*20480)) + self.check([truncate] + [readlines] * 10, self.ioclass(b'0\n'*20480)) + self.check([truncate] + [readinto] * 10, self.ioclass(b'0\n'*204800), bytearray(b'0\n'*204800)) + self.check([close] + [write] * 10, self.ioclass()) + self.check([truncate] + [getvalue] * 10, self.ioclass(b'0\n'*204800)) + self.check([truncate] + [getbuffer] * 10, self.ioclass(b'0\n'*204800)) + self.check([truncate] + [iter] * 10, self.ioclass(b'0\n'*20480)) + self.check([truncate] + [getstate] * 10, self.ioclass(b'0\n'*204800)) + state = self.ioclass(b'123').__getstate__() + self.check([truncate] + [setstate] * 10, self.ioclass(b'0\n'*204800), state) + self.check([truncate] + [sizeof] * 10, self.ioclass(b'0\n'*204800)) + + # no tests for seek or tell because they don't break anything + +class CBytesIOTest(ThreadSafetyMixin, TestCase): + ioclass = io.BytesIO + +class PyBytesIOTest(ThreadSafetyMixin, TestCase): + ioclass = pyio.BytesIO diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py new file mode 100644 index 00000000000..9d366041917 --- /dev/null +++ b/Lib/test/test_free_threading/test_itertools.py @@ -0,0 +1,95 @@ +import unittest +from threading import Thread, Barrier +from itertools import batched, chain, cycle +from test.support import threading_helper + + +threading_helper.requires_working_threading(module=True) + +class ItertoolsThreading(unittest.TestCase): + + @threading_helper.reap_threads + def test_batched(self): + number_of_threads = 10 + number_of_iterations = 20 + barrier = Barrier(number_of_threads) + def work(it): + barrier.wait() + while True: + try: + next(it) + except StopIteration: + break + + data = tuple(range(1000)) + for it in range(number_of_iterations): + batch_iterator = batched(data, 2) + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=work, args=[batch_iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + + @threading_helper.reap_threads + def test_cycle(self): + number_of_threads = 6 + number_of_iterations = 10 + number_of_cycles = 400 + + barrier = Barrier(number_of_threads) + def work(it): + barrier.wait() + for _ in range(number_of_cycles): + try: + next(it) + except StopIteration: + pass + + data = (1, 2, 3, 4) + for it in range(number_of_iterations): + cycle_iterator = cycle(data) + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=work, args=[cycle_iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + + @threading_helper.reap_threads + def test_chain(self): + number_of_threads = 6 + number_of_iterations = 20 + + barrier = Barrier(number_of_threads) + def work(it): + barrier.wait() + while True: + try: + next(it) + except StopIteration: + break + + data = [(1, )] * 200 + for it in range(number_of_iterations): + chain_iterator = chain(*data) + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=work, args=[chain_iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_itertools_batched.py b/Lib/test/test_free_threading/test_itertools_batched.py deleted file mode 100644 index a754b4f9ea9..00000000000 --- a/Lib/test/test_free_threading/test_itertools_batched.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest -from threading import Thread, Barrier -from itertools import batched -from test.support import threading_helper - - -threading_helper.requires_working_threading(module=True) - -class EnumerateThreading(unittest.TestCase): - - @threading_helper.reap_threads - def test_threading(self): - number_of_threads = 10 - number_of_iterations = 20 - barrier = Barrier(number_of_threads) - def work(it): - barrier.wait() - while True: - try: - _ = next(it) - except StopIteration: - break - - data = tuple(range(1000)) - for it in range(number_of_iterations): - batch_iterator = batched(data, 2) - worker_threads = [] - for ii in range(number_of_threads): - worker_threads.append( - Thread(target=work, args=[batch_iterator])) - - with threading_helper.start_threads(worker_threads): - pass - - barrier.reset() - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_free_threading/test_itertools_combinatoric.py b/Lib/test/test_free_threading/test_itertools_combinatoric.py new file mode 100644 index 00000000000..5b3b88deedd --- /dev/null +++ b/Lib/test/test_free_threading/test_itertools_combinatoric.py @@ -0,0 +1,51 @@ +import unittest +from threading import Thread, Barrier +from itertools import combinations, product +from test.support import threading_helper + + +threading_helper.requires_working_threading(module=True) + +def test_concurrent_iteration(iterator, number_of_threads): + barrier = Barrier(number_of_threads) + def iterator_worker(it): + barrier.wait() + while True: + try: + _ = next(it) + except StopIteration: + return + + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=iterator_worker, args=[iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + +class ItertoolsThreading(unittest.TestCase): + + @threading_helper.reap_threads + def test_combinations(self): + number_of_threads = 10 + number_of_iterations = 24 + + for it in range(number_of_iterations): + iterator = combinations((1, 2, 3, 4, 5), 2) + test_concurrent_iteration(iterator, number_of_threads) + + @threading_helper.reap_threads + def test_product(self): + number_of_threads = 10 + number_of_iterations = 24 + + for it in range(number_of_iterations): + iterator = product((1, 2, 3, 4, 5), (10, 20, 30)) + test_concurrent_iteration(iterator, number_of_threads) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_json.py b/Lib/test/test_free_threading/test_json.py new file mode 100644 index 00000000000..010eb322a15 --- /dev/null +++ b/Lib/test/test_free_threading/test_json.py @@ -0,0 +1,79 @@ +from threading import Barrier, Thread +from test.test_json import CTest +from test.support import threading_helper + + +def encode_json_helper( + json, worker, data, number_of_threads=12, number_of_json_encodings=100 +): + worker_threads = [] + barrier = Barrier(number_of_threads) + for index in range(number_of_threads): + worker_threads.append( + Thread(target=worker, args=[barrier, data, index]) + ) + for t in worker_threads: + t.start() + for ii in range(number_of_json_encodings): + json.dumps(data) + data.clear() + for t in worker_threads: + t.join() + + +class MyMapping(dict): + def __init__(self): + self.mapping = [] + + def items(self): + return self.mapping + + +@threading_helper.reap_threads +@threading_helper.requires_working_threading() +class TestJsonEncoding(CTest): + # Test encoding json with concurrent threads modifying the data cannot + # corrupt the interpreter + + def test_json_mutating_list(self): + def worker(barrier, data, index): + barrier.wait() + while data: + for d in data: + if len(d) > 5: + d.clear() + else: + d.append(index) + + data = [[], []] + encode_json_helper(self.json, worker, data) + + def test_json_mutating_exact_dict(self): + def worker(barrier, data, index): + barrier.wait() + while data: + for d in data: + if len(d) > 5: + try: + key = list(d)[0] + d.pop(key) + except (KeyError, IndexError): + pass + else: + d[index] = index + + data = [{}, {}] + encode_json_helper(self.json, worker, data) + + def test_json_mutating_mapping(self): + def worker(barrier, data, index): + barrier.wait() + while data: + for d in data: + if len(d.mapping) > 3: + d.mapping.clear() + else: + d.mapping.append((index, index)) + + data = [MyMapping(), MyMapping()] + encode_json_helper(self.json, worker, data) diff --git a/Lib/test/test_free_threading/test_lzma.py b/Lib/test/test_free_threading/test_lzma.py new file mode 100644 index 00000000000..38d7e5db489 --- /dev/null +++ b/Lib/test/test_free_threading/test_lzma.py @@ -0,0 +1,56 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +lzma = import_helper.import_module("lzma") +from lzma import LZMACompressor, LZMADecompressor + +from test.test_lzma import INPUT + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestLZMA(unittest.TestCase): + def test_compressor(self): + lzc = LZMACompressor() + + # First compress() outputs LZMA header + header = lzc.compress(INPUT) + self.assertGreater(len(header), 0) + + def worker(): + # it should return empty bytes as it buffers data internally + data = lzc.compress(INPUT) + self.assertEqual(data, b"") + + run_concurrently(worker_func=worker, nthreads=NTHREADS - 1) + full_compressed = header + lzc.flush() + decompressed = lzma.decompress(full_compressed) + # The decompressed data should be INPUT repeated NTHREADS times + self.assertEqual(decompressed, INPUT * NTHREADS) + + def test_decompressor(self): + chunk_size = 128 + chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)] + input_data = b"".join(chunks) + compressed = lzma.compress(input_data) + + lzd = LZMADecompressor() + output = [] + + def worker(): + data = lzd.decompress(compressed, chunk_size) + self.assertEqual(len(data), chunk_size) + output.append(data) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(output), NTHREADS) + # Verify the expected chunks (order doesn't matter due to append race) + self.assertSetEqual(set(output), set(chunks)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_mmap.py b/Lib/test/test_free_threading/test_mmap.py new file mode 100644 index 00000000000..ece13d33f3b --- /dev/null +++ b/Lib/test/test_free_threading/test_mmap.py @@ -0,0 +1,315 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +import os +import string +import tempfile +import threading + +from collections import Counter + +mmap = import_helper.import_module("mmap") + +NTHREADS = 10 +ANONYMOUS_MEM = -1 + + +@threading_helper.requires_working_threading() +class MmapTests(unittest.TestCase): + def test_read_and_read_byte(self): + ascii_uppercase = string.ascii_uppercase.encode() + # Choose a total mmap size that evenly divides across threads and the + # read pattern (3 bytes per loop). + mmap_size = 3 * NTHREADS * len(ascii_uppercase) + num_bytes_to_read_per_thread = mmap_size // NTHREADS + bytes_read_from_mmap = [] + + def read(mm_obj): + nread = 0 + while nread < num_bytes_to_read_per_thread: + b = mm_obj.read_byte() + bytes_read_from_mmap.append(b) + b = mm_obj.read(2) + bytes_read_from_mmap.extend(b) + nread += 3 + + with mmap.mmap(ANONYMOUS_MEM, mmap_size) as mm_obj: + for i in range(mmap_size // len(ascii_uppercase)): + mm_obj.write(ascii_uppercase) + + mm_obj.seek(0) + run_concurrently( + worker_func=read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + self.assertEqual(len(bytes_read_from_mmap), mmap_size) + # Count each letter/byte to verify read correctness + counter = Counter(bytes_read_from_mmap) + self.assertEqual(len(counter), len(ascii_uppercase)) + # Each letter/byte should be read (3 * NTHREADS) times + for letter in ascii_uppercase: + self.assertEqual(counter[letter], 3 * NTHREADS) + + def test_readline(self): + num_lines = 1000 + lines_read_from_mmap = [] + expected_lines = [] + + def readline(mm_obj): + for i in range(num_lines // NTHREADS): + line = mm_obj.readline() + lines_read_from_mmap.append(line) + + # Allocate mmap enough for num_lines (max line 5 bytes including NL) + with mmap.mmap(ANONYMOUS_MEM, num_lines * 5) as mm_obj: + for i in range(num_lines): + line = b"%d\n" % i + mm_obj.write(line) + expected_lines.append(line) + + mm_obj.seek(0) + run_concurrently( + worker_func=readline, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + self.assertEqual(len(lines_read_from_mmap), num_lines) + # Every line should be read once by threads; order is non-deterministic + # Sort numerically by integer value + lines_read_from_mmap.sort(key=lambda x: int(x)) + self.assertEqual(lines_read_from_mmap, expected_lines) + + def test_write_and_write_byte(self): + thread_letters = list(string.ascii_uppercase) + self.assertLessEqual(NTHREADS, len(thread_letters)) + per_thread_write_loop = 100 + + def write(mm_obj): + # Each thread picks a unique letter to write + thread_letter = thread_letters.pop(0) + thread_bytes = (thread_letter * 2).encode() + for _ in range(per_thread_write_loop): + mm_obj.write_byte(thread_bytes[0]) + mm_obj.write(thread_bytes) + + with mmap.mmap( + ANONYMOUS_MEM, per_thread_write_loop * 3 * NTHREADS + ) as mm_obj: + run_concurrently( + worker_func=write, + args=(mm_obj,), + nthreads=NTHREADS, + ) + mm_obj.seek(0) + data = mm_obj.read() + self.assertEqual(len(data), NTHREADS * per_thread_write_loop * 3) + counter = Counter(data) + self.assertEqual(len(counter), NTHREADS) + # Each thread letter should be written `per_thread_write_loop` * 3 + for letter in counter: + self.assertEqual(counter[letter], per_thread_write_loop * 3) + + def test_move(self): + ascii_uppercase = string.ascii_uppercase.encode() + num_letters = len(ascii_uppercase) + + def move(mm_obj): + for i in range(num_letters): + # Move 1 byte from the first half to the second half + mm_obj.move(0 + i, num_letters + i, 1) + + with mmap.mmap(ANONYMOUS_MEM, 2 * num_letters) as mm_obj: + mm_obj.write(ascii_uppercase) + run_concurrently( + worker_func=move, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_seek_and_tell(self): + seek_per_thread = 10 + + def seek(mm_obj): + self.assertTrue(mm_obj.seekable()) + for _ in range(seek_per_thread): + before_seek = mm_obj.tell() + mm_obj.seek(1, os.SEEK_CUR) + self.assertLess(before_seek, mm_obj.tell()) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=seek, + args=(mm_obj,), + nthreads=NTHREADS, + ) + # Each thread seeks from current position, the end position should + # be the sum of all seeks from all threads. + self.assertEqual(mm_obj.tell(), NTHREADS * seek_per_thread) + + def test_slice_update_and_slice_read(self): + thread_letters = list(string.ascii_uppercase) + self.assertLessEqual(NTHREADS, len(thread_letters)) + + def slice_update_and_slice_read(mm_obj): + # Each thread picks a unique letter to write + thread_letter = thread_letters.pop(0) + thread_bytes = (thread_letter * 1024).encode() + for _ in range(100): + mm_obj[:] = thread_bytes + read_bytes = mm_obj[:] + # Read bytes should be all the same letter, showing no + # interleaving + self.assertTrue(all_same(read_bytes)) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=slice_update_and_slice_read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_item_update_and_item_read(self): + thread_indexes = [i for i in range(NTHREADS)] + + def item_update_and_item_read(mm_obj): + # Each thread picks a unique index to write + thread_index = thread_indexes.pop() + for i in range(100): + mm_obj[thread_index] = i + self.assertEqual(mm_obj[thread_index], i) + + # Read values set by other threads, all values + # should be less than '100' + for val in mm_obj: + self.assertLess(int.from_bytes(val), 100) + + with mmap.mmap(ANONYMOUS_MEM, NTHREADS + 1) as mm_obj: + run_concurrently( + worker_func=item_update_and_item_read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + @unittest.skipUnless(os.name == "posix", "requires Posix") + @unittest.skipUnless(hasattr(mmap.mmap, "resize"), "requires mmap.resize") + def test_resize_and_size(self): + thread_indexes = [i for i in range(NTHREADS)] + + def resize_and_item_update(mm_obj): + # Each thread picks a unique index to write + thread_index = thread_indexes.pop() + mm_obj.resize(2048) + self.assertEqual(mm_obj.size(), 2048) + for i in range(100): + mm_obj[thread_index] = i + self.assertEqual(mm_obj[thread_index], i) + + with mmap.mmap(ANONYMOUS_MEM, 1024, flags=mmap.MAP_PRIVATE) as mm_obj: + run_concurrently( + worker_func=resize_and_item_update, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_close_and_closed(self): + def close_mmap(mm_obj): + mm_obj.close() + self.assertTrue(mm_obj.closed) + + with mmap.mmap(ANONYMOUS_MEM, 1) as mm_obj: + run_concurrently( + worker_func=close_mmap, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_find_and_rfind(self): + per_thread_loop = 10 + + def find_and_rfind(mm_obj): + pattern = b'Thread-Ident:"%d"' % threading.get_ident() + mm_obj.write(pattern) + for _ in range(per_thread_loop): + found_at = mm_obj.find(pattern, 0) + self.assertNotEqual(found_at, -1) + # Should not find it after the `found_at` + self.assertEqual(mm_obj.find(pattern, found_at + 1), -1) + found_at_rev = mm_obj.rfind(pattern, 0) + self.assertEqual(found_at, found_at_rev) + # Should not find it after the `found_at` + self.assertEqual(mm_obj.rfind(pattern, found_at + 1), -1) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=find_and_rfind, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + @unittest.skipUnless(os.name == "posix", "requires Posix") + @unittest.skipUnless(hasattr(mmap.mmap, "resize"), "requires mmap.resize") + def test_flush(self): + mmap_filename = "test_mmap_file" + resize_to = 1024 + + def resize_and_flush(mm_obj): + mm_obj.resize(resize_to) + mm_obj.flush() + + with tempfile.TemporaryDirectory() as tmpdirname: + file_path = f"{tmpdirname}/{mmap_filename}" + with open(file_path, "wb+") as file: + file.write(b"CPython") + file.flush() + with mmap.mmap(file.fileno(), 1) as mm_obj: + run_concurrently( + worker_func=resize_and_flush, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + self.assertEqual(os.path.getsize(file_path), resize_to) + + def test_mmap_export_as_memoryview(self): + """ + Each thread creates a memoryview and updates the internal state of the + mmap object. + """ + buffer_size = 42 + + def create_memoryview_from_mmap(mm_obj): + memoryviews = [] + for _ in range(100): + mv = memoryview(mm_obj) + memoryviews.append(mv) + self.assertEqual(len(mv), buffer_size) + self.assertEqual(mv[:7], b"CPython") + + # Cannot close the mmap while it is exported as buffers + with self.assertRaisesRegex( + BufferError, "cannot close exported pointers exist" + ): + mm_obj.close() + + with mmap.mmap(ANONYMOUS_MEM, 42) as mm_obj: + mm_obj.write(b"CPython") + run_concurrently( + worker_func=create_memoryview_from_mmap, + args=(mm_obj,), + nthreads=NTHREADS, + ) + # Implicit mm_obj.close() verifies all exports (memoryviews) are + # properly freed. + + +def all_same(lst): + return all(item == lst[0] for item in lst) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index a480e398722..2cd6e7b035e 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -2,10 +2,12 @@ environment to verify things are thread-safe in a free-threaded build""" import sys +import threading import time import unittest import weakref +from contextlib import contextmanager from sys import monitoring from test.support import threading_helper from threading import Thread, _PyRLock, Barrier @@ -33,10 +35,10 @@ def work(self, n, funcs): return n return self.work(n - 1, funcs) + self.work(n - 2, funcs) - def start_work(self, n, funcs): + def start_work(self, n, funcs, barrier): # With the GIL builds we need to make sure that the hooks have # a chance to run as it's possible to run w/o releasing the GIL. - time.sleep(0.1) + barrier.wait() self.work(n, funcs) def after_test(self): @@ -51,14 +53,16 @@ def test_instrumentation(self): exec("def f(): pass", x) funcs.append(x["f"]) + barrier = Barrier(self.thread_count + 1) threads = [] for i in range(self.thread_count): # Each thread gets a copy of the func list to avoid contention - t = Thread(target=self.start_work, args=(self.fib, list(funcs))) + t = Thread(target=self.start_work, args=(self.fib, list(funcs), barrier)) t.start() threads.append(t) self.after_threads() + barrier.wait() while True: any_alive = False @@ -71,6 +75,9 @@ def test_instrumentation(self): break self.during_threads() + # Sleep to avoid setting monitoring events too rapidly and + # overflowing the global version counter + time.sleep(0.0001) self.after_test() @@ -115,7 +122,6 @@ class MonitoringMultiThreaded( def setUp(self): super().setUp() self.set = False - self.called = False monitoring.register_callback( self.tool_id, monitoring.events.LINE, self.callback ) @@ -125,10 +131,7 @@ def tearDown(self): super().tearDown() def callback(self, *args): - self.called = True - - def after_test(self): - self.assertTrue(self.called) + pass def during_threads(self): if self.set: @@ -146,16 +149,11 @@ class SetTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): def setUp(self): self.set = False - self.called = False - - def after_test(self): - self.assertTrue(self.called) def tearDown(self): sys.settrace(None) def trace_func(self, frame, event, arg): - self.called = True return self.trace_func def during_threads(self): @@ -172,16 +170,11 @@ class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): def setUp(self): self.set = False - self.called = False - - def after_test(self): - self.assertTrue(self.called) def tearDown(self): sys.setprofile(None) def trace_func(self, frame, event, arg): - self.called = True return self.trace_func def during_threads(self): @@ -192,6 +185,70 @@ def during_threads(self): self.set = not self.set +@threading_helper.requires_working_threading() +class SetProfileAllThreadsMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses threading.setprofile_all_threads and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + + def tearDown(self): + threading.setprofile_all_threads(None) + + def trace_func(self, frame, event, arg): + return self.trace_func + + def during_threads(self): + if self.set: + threading.setprofile_all_threads(self.trace_func) + else: + threading.setprofile_all_threads(None) + self.set = not self.set + + +class SetProfileAllMultiThreaded(TestCase): + def test_profile_all_threads(self): + done = threading.Event() + + def func(): + pass + + def bg_thread(): + while not done.is_set(): + func() + func() + func() + func() + func() + + def my_profile(frame, event, arg): + return None + + bg_threads = [] + for i in range(10): + t = threading.Thread(target=bg_thread) + t.start() + bg_threads.append(t) + + for i in range(100): + threading.setprofile_all_threads(my_profile) + threading.setprofile_all_threads(None) + + done.set() + for t in bg_threads: + t.join() + + +class TraceBuf: + def __init__(self): + self.traces = [] + self.traces_lock = threading.Lock() + + def append(self, trace): + with self.traces_lock: + self.traces.append(trace) + + @threading_helper.requires_working_threading() class MonitoringMisc(MonitoringTestMixin, TestCase): def register_callback(self, barrier): @@ -246,6 +303,167 @@ def f(): finally: sys.settrace(None) + def test_toggle_setprofile_no_new_events(self): + # gh-136396: Make sure that profile functions are called for newly + # created threads when profiling is toggled but the set of monitoring + # events doesn't change + traces = [] + + def profiler(frame, event, arg): + traces.append((frame.f_code.co_name, event, arg)) + + def a(x, y): + return b(x, y) + + def b(x, y): + return max(x, y) + + sys.setprofile(profiler) + try: + a(1, 2) + finally: + sys.setprofile(None) + traces.clear() + + def thread_main(x, y): + sys.setprofile(profiler) + try: + a(x, y) + finally: + sys.setprofile(None) + t = Thread(target=thread_main, args=(100, 200)) + t.start() + t.join() + + expected = [ + ("a", "call", None), + ("b", "call", None), + ("b", "c_call", max), + ("b", "c_return", max), + ("b", "return", 200), + ("a", "return", 200), + ("thread_main", "c_call", sys.setprofile), + ] + self.assertEqual(traces, expected) + + def observe_threads(self, observer, buf): + def in_child(ident): + return ident + + def child(ident): + with observer(): + in_child(ident) + + def in_parent(ident): + return ident + + def parent(barrier, ident): + barrier.wait() + with observer(): + t = Thread(target=child, args=(ident,)) + t.start() + t.join() + in_parent(ident) + + num_threads = 5 + barrier = Barrier(num_threads) + threads = [] + for i in range(num_threads): + t = Thread(target=parent, args=(barrier, i)) + t.start() + threads.append(t) + for t in threads: + t.join() + + for i in range(num_threads): + self.assertIn(("in_parent", "return", i), buf.traces) + self.assertIn(("in_child", "return", i), buf.traces) + + def test_profile_threads(self): + buf = TraceBuf() + + def profiler(frame, event, arg): + buf.append((frame.f_code.co_name, event, arg)) + + @contextmanager + def profile(): + sys.setprofile(profiler) + try: + yield + finally: + sys.setprofile(None) + + self.observe_threads(profile, buf) + + def test_trace_threads(self): + buf = TraceBuf() + + def tracer(frame, event, arg): + buf.append((frame.f_code.co_name, event, arg)) + return tracer + + @contextmanager + def trace(): + sys.settrace(tracer) + try: + yield + finally: + sys.settrace(None) + + self.observe_threads(trace, buf) + + def test_monitor_threads(self): + buf = TraceBuf() + + def monitor_py_return(code, off, retval): + buf.append((code.co_name, "return", retval)) + + monitoring.register_callback( + self.tool_id, monitoring.events.PY_RETURN, monitor_py_return + ) + + monitoring.set_events( + self.tool_id, monitoring.events.PY_RETURN + ) + + @contextmanager + def noop(): + yield + + self.observe_threads(noop, buf) + + def test_trace_concurrent(self): + # Test calling a function concurrently from a tracing and a non-tracing + # thread + b = threading.Barrier(2) + + def func(): + for _ in range(100): + pass + + def noop(): + pass + + def bg_thread(): + b.wait() + func() # this may instrument `func` + + def tracefunc(frame, event, arg): + # These calls run under tracing can race with the background thread + for _ in range(10): + func() + return tracefunc + + t = Thread(target=bg_thread) + t.start() + try: + sys.settrace(tracefunc) + b.wait() + noop() + finally: + sys.settrace(None) + t.join() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_pwd.py b/Lib/test/test_free_threading/test_pwd.py new file mode 100644 index 00000000000..58ab22bae39 --- /dev/null +++ b/Lib/test/test_free_threading/test_pwd.py @@ -0,0 +1,33 @@ +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + +from test import test_pwd + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestPwd(unittest.TestCase): + def setUp(self): + self.test_pwd = test_pwd.PwdTest() + + def test_racing_test_values(self): + # test_pwd.test_values() calls pwd.getpwall() and checks the entries + run_concurrently( + worker_func=self.test_pwd.test_values, nthreads=NTHREADS + ) + + def test_racing_test_values_extended(self): + # test_pwd.test_values_extended() calls pwd.getpwall(), pwd.getpwnam(), + # pwd.getpwduid() and checks the entries + run_concurrently( + worker_func=self.test_pwd.test_values_extended, + nthreads=NTHREADS, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_re.py b/Lib/test/test_free_threading/test_re.py new file mode 100644 index 00000000000..56f25045d1b --- /dev/null +++ b/Lib/test/test_free_threading/test_re.py @@ -0,0 +1,62 @@ +import re +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestRe(unittest.TestCase): + def test_pattern_sub(self): + """Pattern substitution should work across threads""" + pattern = re.compile(r"\w+@\w+\.\w+") + text = "e-mail: test@python.org or user@pycon.org. " * 5 + results = [] + + def worker(): + substituted = pattern.sub("(redacted)", text) + results.append(substituted.count("(redacted)")) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(results, [2 * 5] * NTHREADS) + + def test_pattern_search(self): + """Pattern search should work across threads.""" + emails = ["alice@python.org", "bob@pycon.org"] * 10 + pattern = re.compile(r"\w+@\w+\.\w+") + results = [] + + def worker(): + matches = [pattern.search(e).group() for e in emails] + results.append(len(matches)) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(results, [2 * 10] * NTHREADS) + + def test_scanner_concurrent_access(self): + """Shared scanner should reject concurrent access.""" + pattern = re.compile(r"\w+") + scanner = pattern.scanner("word " * 10) + + def worker(): + for _ in range(100): + try: + scanner.search() + except ValueError as e: + if "already executing" in str(e): + pass + else: + raise + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + # This test has no assertions. Its purpose is to catch crashes and + # enable thread sanitizer to detect race conditions. While "already + # executing" errors are very likely, they're not guaranteed due to + # non-deterministic thread scheduling, so we can't assert errors > 0. + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_resource.py b/Lib/test/test_free_threading/test_resource.py new file mode 100644 index 00000000000..ecd0e535c44 --- /dev/null +++ b/Lib/test/test_free_threading/test_resource.py @@ -0,0 +1,41 @@ +import unittest +from test.support import import_helper, threading_helper + +resource = import_helper.import_module("resource") + + +NTHREADS = 10 +LOOP_PER_THREAD = 1000 + + +@threading_helper.requires_working_threading() +class ResourceTest(unittest.TestCase): + @unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage") + @unittest.skipUnless( + hasattr(resource, "RUSAGE_THREAD"), "needs RUSAGE_THREAD" + ) + def test_getrusage(self): + ru_utime_lst = [] + + def dummy_work(ru_utime_lst): + for _ in range(LOOP_PER_THREAD): + pass + + usage_process = resource.getrusage(resource.RUSAGE_SELF) + usage_thread = resource.getrusage(resource.RUSAGE_THREAD) + # Process user time should be greater than thread user time + self.assertGreater(usage_process.ru_utime, usage_thread.ru_utime) + ru_utime_lst.append(usage_thread.ru_utime) + + threading_helper.run_concurrently( + worker_func=dummy_work, args=(ru_utime_lst,), nthreads=NTHREADS + ) + + usage_process = resource.getrusage(resource.RUSAGE_SELF) + self.assertEqual(len(ru_utime_lst), NTHREADS) + # Process user time should be greater than sum of all thread user times + self.assertGreater(usage_process.ru_utime, sum(ru_utime_lst)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_suggestions.py b/Lib/test/test_free_threading/test_suggestions.py new file mode 100755 index 00000000000..2c10a511b86 --- /dev/null +++ b/Lib/test/test_free_threading/test_suggestions.py @@ -0,0 +1,24 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +suggestions = import_helper.import_module("_suggestions") + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class SuggestionsTests(unittest.TestCase): + def test_generate_suggestions(self): + candidates = [str(i) for i in range(100)] + + def worker(): + _ = suggestions._generate_suggestions(candidates, "42") + candidates.clear() + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_syslog.py b/Lib/test/test_free_threading/test_syslog.py new file mode 100644 index 00000000000..b374a98b96e --- /dev/null +++ b/Lib/test/test_free_threading/test_syslog.py @@ -0,0 +1,44 @@ +import unittest +import threading + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +syslog = import_helper.import_module("syslog") + +NTHREADS = 32 + +# Similar to Lib/test/test_syslog.py, this test's purpose is to verify that +# the code neither crashes nor leaks. + + +@threading_helper.requires_working_threading() +class TestSyslog(unittest.TestCase): + def test_racing_syslog(self): + def worker(): + """ + The syslog module provides the following functions: + openlog(), syslog(), closelog(), and setlogmask(). + """ + thread_id = threading.get_ident() + syslog.openlog(f"thread-id: {thread_id}") + try: + for _ in range(5): + syslog.syslog("logline") + syslog.setlogmask(syslog.LOG_MASK(syslog.LOG_INFO)) + syslog.syslog(syslog.LOG_INFO, "logline LOG_INFO") + syslog.setlogmask(syslog.LOG_MASK(syslog.LOG_ERR)) + syslog.syslog(syslog.LOG_ERR, "logline LOG_ERR") + syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG)) + finally: + syslog.closelog() + + # Run the worker concurrently to exercise all these syslog functions + run_concurrently( + worker_func=worker, + nthreads=NTHREADS, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index ae996e7db3c..1255d842dbf 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -127,6 +127,39 @@ class ClassB(Base): obj.__class__ = ClassB + def test_name_change(self): + class Foo: + pass + + def writer(): + for _ in range(1000): + Foo.__name__ = 'Bar' + + def reader(): + for _ in range(1000): + Foo.__name__ + + self.run_one(writer, reader) + + def test_bases_change(self): + class BaseA: + pass + + class Derived(BaseA): + pass + + def writer(): + for _ in range(1000): + class BaseB: + pass + Derived.__bases__ = (BaseB,) + + def reader(): + for _ in range(1000): + Derived.__base__ + + self.run_one(writer, reader) + def run_one(self, writer_func, reader_func): barrier = threading.Barrier(NTHREADS) diff --git a/Lib/test/test_free_threading/test_uuid.py b/Lib/test/test_free_threading/test_uuid.py new file mode 100755 index 00000000000..d794afc552a --- /dev/null +++ b/Lib/test/test_free_threading/test_uuid.py @@ -0,0 +1,60 @@ +import os +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently +from uuid import SafeUUID + +c_uuid = import_helper.import_module("_uuid") + +NTHREADS = 10 +UUID_PER_THREAD = 1000 + + +@threading_helper.requires_working_threading() +class UUIDTests(unittest.TestCase): + @unittest.skipUnless(os.name == "posix", "POSIX only") + def test_generate_time_safe(self): + uuids = [] + + def worker(): + local_uuids = [] + for _ in range(UUID_PER_THREAD): + uuid, is_safe = c_uuid.generate_time_safe() + self.assertIs(type(uuid), bytes) + self.assertEqual(len(uuid), 16) + # Collect the UUID only if it is safe. If not, we cannot ensure + # UUID uniqueness. According to uuid_generate_time_safe() man + # page, it is theoretically possible for two concurrently + # running processes to generate the same UUID(s) if the return + # value is not 0. + if is_safe == SafeUUID.safe: + local_uuids.append(uuid) + + # Merge all safe uuids + uuids.extend(local_uuids) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(uuids), len(set(uuids))) + + @unittest.skipUnless(os.name == "nt", "Windows only") + def test_UuidCreate(self): + uuids = [] + + def worker(): + local_uuids = [] + for _ in range(UUID_PER_THREAD): + uuid = c_uuid.UuidCreate() + self.assertIs(type(uuid), bytes) + self.assertEqual(len(uuid), 16) + local_uuids.append(uuid) + + # Merge all uuids + uuids.extend(local_uuids) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(uuids), len(set(uuids))) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index dd58e032a8b..05d0cbd2445 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1336,9 +1336,9 @@ def test_equal_equal(self): def test_conversions(self): self.assertEqual(f'{3.14:10.10}', ' 3.14') - self.assertEqual(f'{3.14!s:10.10}', '3.14 ') - self.assertEqual(f'{3.14!r:10.10}', '3.14 ') - self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + self.assertEqual(f'{1.25!s:10.10}', '1.25 ') + self.assertEqual(f'{1.25!r:10.10}', '1.25 ') + self.assertEqual(f'{1.25!a:10.10}', '1.25 ') self.assertEqual(f'{"a"}', 'a') self.assertEqual(f'{"a"!r}', "'a'") @@ -1347,7 +1347,7 @@ def test_conversions(self): # Conversions can have trailing whitespace after them since it # does not provide any significance self.assertEqual(f"{3!s }", "3") - self.assertEqual(f'{3.14!s :10.10}', '3.14 ') + self.assertEqual(f'{1.25!s :10.10}', '1.25 ') # Not a conversion. self.assertEqual(f'{"a!r"}', "a!r") @@ -1380,7 +1380,7 @@ def test_conversions(self): for conv in ' s', ' s ': self.assertAllRaise(SyntaxError, "f-string: conversion type must come right after the" - " exclamanation mark", + " exclamation mark", ["f'{3!" + conv + "}'"]) self.assertAllRaise(SyntaxError, @@ -1651,6 +1651,18 @@ def __repr__(self): self.assertEqual(f"{1+2 = # my comment }", '1+2 = \n 3') + self.assertEqual(f'{""" # booo + """=}', '""" # booo\n """=\' # booo\\n \'') + + self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'') + self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'') + + self.assertEqual(f'{ # some comment goes here + """hello"""=}', ' \n """hello"""=\'hello\'') + self.assertEqual(f'{"""# this is not a comment + a""" # this is a comment + }', '# this is not a comment\n a') + # These next lines contains tabs. Backslash escapes don't # work in f-strings. # patchcheck doesn't like these tabs. So the only way to test @@ -1819,6 +1831,41 @@ def test_newlines_in_format_specifiers(self): for case in valid_cases: compile(case, "", "exec") + def test_raw_fstring_format_spec(self): + # Test raw f-string format spec behavior (Issue #137314). + # + # Raw f-strings should preserve literal backslashes in format specifications, + # not interpret them as escape sequences. + class UnchangedFormat: + """Test helper that returns the format spec unchanged.""" + def __format__(self, format): + return format + + # Test basic escape sequences + self.assertEqual(f"{UnchangedFormat():\xFF}", 'ÿ') + self.assertEqual(rf"{UnchangedFormat():\xFF}", '\\xFF') + + # Test nested expressions with raw/non-raw combinations + self.assertEqual(rf"{UnchangedFormat():{'\xFF'}}", 'ÿ') + self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + + # Test continuation character in format specs + self.assertEqual(f"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + self.assertEqual(rf"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + + # Test multiple format specs in same raw f-string + self.assertEqual(rf"{UnchangedFormat():\xFF} {UnchangedFormat():\n}", '\\xFF \\n') + + def test_gh139516(self): + with temp_cwd(): + script = 'script.py' + with open(script, 'wb') as f: + f.write('''def f(a): pass\nf"{f(a=lambda: 'à'\n)}"'''.encode()) + assert_python_ok(script) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 2e794b0fc95..090926fd8d8 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -21,6 +21,7 @@ import contextlib from inspect import Signature +from test.support import ALWAYS_EQ from test.support import import_helper from test.support import threading_helper from test.support import cpython_only @@ -244,6 +245,13 @@ def test_placeholders(self): actual_args, actual_kwds = p('x', 'y') self.assertEqual(actual_args, ('x', 0, 'y', 1)) self.assertEqual(actual_kwds, {}) + # Checks via `is` and not `eq` + # thus ALWAYS_EQ isn't treated as Placeholder + p = self.partial(capture, ALWAYS_EQ) + actual_args, actual_kwds = p() + self.assertEqual(len(actual_args), 1) + self.assertIs(actual_args[0], ALWAYS_EQ) + self.assertEqual(actual_kwds, {}) def test_placeholders_optimization(self): PH = self.module.Placeholder @@ -260,6 +268,17 @@ def test_placeholders_optimization(self): self.assertEqual(p2.args, (PH, 0)) self.assertEqual(p2(1), ((1, 0), {})) + def test_placeholders_kw_restriction(self): + PH = self.module.Placeholder + with self.assertRaisesRegex(TypeError, "Placeholder"): + self.partial(capture, a=PH) + # Passes, as checks via `is` and not `eq` + p = self.partial(capture, a=ALWAYS_EQ) + actual_args, actual_kwds = p() + self.assertEqual(actual_args, ()) + self.assertEqual(len(actual_kwds), 1) + self.assertIs(actual_kwds['a'], ALWAYS_EQ) + def test_construct_placeholder_singleton(self): PH = self.module.Placeholder tp = type(PH) @@ -387,6 +406,7 @@ def test_setstate(self): def test_setstate_errors(self): f = self.partial(signature) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, {}, None)) self.assertRaises(TypeError, f.__setstate__, [capture, (), {}, None]) @@ -394,6 +414,8 @@ def test_setstate_errors(self): self.assertRaises(TypeError, f.__setstate__, (capture, None, {}, None)) self.assertRaises(TypeError, f.__setstate__, (capture, [], {}, None)) self.assertRaises(TypeError, f.__setstate__, (capture, (), [], None)) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, ())) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, 'test')) def test_setstate_subclasses(self): f = self.partial(signature) @@ -2763,7 +2785,7 @@ class Slot: @functools.singledispatchmethod @classmethod def go(cls, item, arg): - pass + return item - arg @go.register @classmethod @@ -2772,7 +2794,9 @@ def _(cls, item: int, arg): s = Slot() self.assertEqual(s.go(1, 1), 2) + self.assertEqual(s.go(1.5, 1), 0.5) self.assertEqual(Slot.go(1, 1), 2) + self.assertEqual(Slot.go(1.5, 1), 0.5) def test_staticmethod_slotted_class(self): class A: @@ -3463,6 +3487,37 @@ def _(item, arg: bytes) -> str: self.assertEqual(str(Signature.from_callable(A.static_func)), '(item, arg: int) -> str') + def test_method_non_descriptor(self): + class Callable: + def __init__(self, value): + self.value = value + def __call__(self, arg): + return self.value, arg + + class A: + t = functools.singledispatchmethod(Callable('general')) + t.register(int, Callable('special')) + + @functools.singledispatchmethod + def u(self, arg): + return 'general', arg + u.register(int, Callable('special')) + + v = functools.singledispatchmethod(Callable('general')) + @v.register(int) + def _(self, arg): + return 'special', arg + + a = A() + self.assertEqual(a.t(0), ('special', 0)) + self.assertEqual(a.t(2.5), ('general', 2.5)) + self.assertEqual(A.t(0), ('special', 0)) + self.assertEqual(A.t(2.5), ('general', 2.5)) + self.assertEqual(a.u(0), ('special', 0)) + self.assertEqual(a.u(2.5), ('general', 2.5)) + self.assertEqual(a.v(0), ('special', 0)) + self.assertEqual(a.v(2.5), ('general', 2.5)) + class CachedCostItem: _cost = 1 diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 42c6cb3fefa..71f1e616116 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -422,6 +422,11 @@ def test_annotations(self): eq('(((a)))', 'a') eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") + eq("t''") + eq("t'{a + b}'") + eq("t'{a!s}'") + eq("t'{a:b}'") + eq("t'{a:b=}'") def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 8fae12c478c..ec5df4d20e7 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -7,7 +7,7 @@ Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink -from test.support.script_helper import assert_python_ok, make_script +from test.support.script_helper import assert_python_ok, make_script, run_test_script from test.support import threading_helper, gc_threshold import gc @@ -262,9 +262,11 @@ class Cyclic(tuple): # finalizer. def __del__(self): - # 5. Create a weakref to `func` now. If we had created - # it earlier, it would have been cleared by the - # garbage collector before calling the finalizers. + # 5. Create a weakref to `func` now. In previous + # versions of Python, this would avoid having it + # cleared by the garbage collector before calling + # the finalizers. Now, weakrefs get cleared after + # calling finalizers. self[1].ref = weakref.ref(self[0]) # 6. Drop the global reference to `latefin`. The only @@ -293,16 +295,40 @@ def func(): # which will find `cyc` and `func` as garbage. gc.collect() - # 9. Previously, this would crash because `func_qualname` - # had been NULL-ed out by func_clear(). + # 9. Previously, this would crash because the weakref + # created in the finalizer revealed the function after + # `tp_clear` was called and `func_qualname` + # had been NULL-ed out by func_clear(). Now, we clear + # weakrefs to unreachable objects before calling `tp_clear` + # but after calling finalizers. print(f"{func=}") """ - # We're mostly just checking that this doesn't crash. rc, stdout, stderr = assert_python_ok("-c", code) self.assertEqual(rc, 0) - self.assertRegex(stdout, rb"""\A\s*func=\s*\z""") + # The `func` global is None because the weakref was cleared. + self.assertRegex(stdout, rb"""\A\s*func=None""") self.assertFalse(stderr) + def test_datetime_weakref_cycle(self): + # https://github.com/python/cpython/issues/132413 + # If the weakref used by the datetime extension gets cleared by the GC (due to being + # in an unreachable cycle) then datetime functions would crash (get_module_state() + # was returning a NULL pointer). This bug is fixed by clearing weakrefs without + # callbacks *after* running finalizers. + code = """if 1: + import _datetime + class C: + def __del__(self): + print('__del__ called') + _datetime.timedelta(days=1) # crash? + + l = [C()] + l.append(l) + """ + rc, stdout, stderr = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(stdout.strip(), b'__del__ called') + @refcount_test def test_frame(self): def f(): @@ -652,9 +678,8 @@ def callback(ignored): gc.collect() self.assertEqual(len(ouch), 2) # else the callbacks didn't run for x in ouch: - # If the callback resurrected one of these guys, the instance - # would be damaged, with an empty __dict__. - self.assertEqual(x, None) + # The weakref should be cleared before executing the callback. + self.assertIsNone(x) def test_bug21435(self): # This is a poor test - its only virtue is that it happened to @@ -726,6 +751,9 @@ def run_command(code): self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " b"shutdown; use", stderr) self.assertNotIn(b"", stderr) + one_line_re = b"gc: uncollectable " + expected_re = one_line_re + b"\r?\n" + one_line_re + self.assertNotRegex(stderr, expected_re) # With DEBUG_UNCOLLECTABLE, the garbage list gets printed stderr = run_command(code % "gc.DEBUG_UNCOLLECTABLE") self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " @@ -733,6 +761,8 @@ def run_command(code): self.assertTrue( (b"[, ]" in stderr) or (b"[, ]" in stderr), stderr) + # we expect two lines with uncollectable objects + self.assertRegex(stderr, expected_re) # With DEBUG_SAVEALL, no additional message should get printed # (because gc.garbage also contains normally reclaimable cyclic # references, and its elements get printed at runtime anyway). @@ -771,6 +801,32 @@ def __del__(self): rc, out, err = assert_python_ok('-c', code) self.assertEqual(out.strip(), b'__del__ called') + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") + def test_gc_debug_stats(self): + # Checks that debug information is printed to stderr + # when DEBUG_STATS is set. + code = """if 1: + import gc + gc.set_debug(%s) + gc.collect() + """ + _, _, err = assert_python_ok("-c", code % "gc.DEBUG_STATS") + self.assertRegex(err, b"gc: collecting generation [0-9]+") + self.assertRegex( + err, + b"gc: objects in each generation: [0-9]+ [0-9]+ [0-9]+", + ) + self.assertRegex( + err, b"gc: objects in permanent generation: [0-9]+" + ) + self.assertRegex( + err, + b"gc: done, .* unreachable, .* uncollectable, .* elapsed", + ) + + _, _, err = assert_python_ok("-c", code % "0") + self.assertNotIn(b"elapsed", err) + def test_global_del_SystemExit(self): code = """if 1: class ClassWithDel: @@ -790,11 +846,15 @@ def test_get_stats(self): self.assertEqual(len(stats), 3) for st in stats: self.assertIsInstance(st, dict) - self.assertEqual(set(st), - {"collected", "collections", "uncollectable"}) + self.assertEqual( + set(st), + {"collected", "collections", "uncollectable", "candidates", "duration"} + ) self.assertGreaterEqual(st["collected"], 0) self.assertGreaterEqual(st["collections"], 0) self.assertGreaterEqual(st["uncollectable"], 0) + self.assertGreaterEqual(st["candidates"], 0) + self.assertGreaterEqual(st["duration"], 0) # Check that collection counts are incremented correctly if gc.isenabled(): self.addCleanup(gc.enable) @@ -805,11 +865,25 @@ def test_get_stats(self): self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) self.assertEqual(new[1]["collections"], old[1]["collections"]) self.assertEqual(new[2]["collections"], old[2]["collections"]) + self.assertGreater(new[0]["duration"], old[0]["duration"]) + self.assertEqual(new[1]["duration"], old[1]["duration"]) + self.assertEqual(new[2]["duration"], old[2]["duration"]) + for stat in ["collected", "uncollectable", "candidates"]: + self.assertGreaterEqual(new[0][stat], old[0][stat]) + self.assertEqual(new[1][stat], old[1][stat]) + self.assertEqual(new[2][stat], old[2][stat]) gc.collect(2) - new = gc.get_stats() - self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) + old, new = new, gc.get_stats() + self.assertEqual(new[0]["collections"], old[0]["collections"]) self.assertEqual(new[1]["collections"], old[1]["collections"]) self.assertEqual(new[2]["collections"], old[2]["collections"] + 1) + self.assertEqual(new[0]["duration"], old[0]["duration"]) + self.assertEqual(new[1]["duration"], old[1]["duration"]) + self.assertGreater(new[2]["duration"], old[2]["duration"]) + for stat in ["collected", "uncollectable", "candidates"]: + self.assertEqual(new[0][stat], old[0][stat]) + self.assertEqual(new[1][stat], old[1][stat]) + self.assertGreaterEqual(new[2][stat], old[2][stat]) def test_freeze(self): gc.freeze() @@ -914,7 +988,7 @@ def __del__(self): gc.collect() self.assertEqual(len(Lazarus.resurrected_instances), 1) instance = Lazarus.resurrected_instances.pop() - self.assertTrue(hasattr(instance, "cargo")) + self.assertHasAttr(instance, "cargo") self.assertEqual(id(instance.cargo), cargo_id) gc.collect() @@ -1125,66 +1199,47 @@ def test_something(self): """) assert_python_ok("-c", source) + def test_do_not_cleanup_type_subclasses_before_finalization(self): + # See https://github.com/python/cpython/issues/135552 + # If we cleanup weakrefs for tp_subclasses before calling + # the finalizer (__del__) then the line `fail = BaseNode.next.next` + # should fail because we are trying to access a subclass + # attribute. But subclass type cache was not properly invalidated. + code = """ + class BaseNode: + def __del__(self): + BaseNode.next = BaseNode.next.next + fail = BaseNode.next.next + + class Node(BaseNode): + pass + + BaseNode.next = Node() + BaseNode.next.next = Node() + """ + # this test checks garbage collection while interp + # finalization + assert_python_ok("-c", textwrap.dedent(code)) + + code_inside_function = textwrap.dedent(F""" + def test(): + {textwrap.indent(code, ' ')} + + test() + """) + # this test checks regular garbage collection + assert_python_ok("-c", code_inside_function) + class IncrementalGCTests(unittest.TestCase): - - def setUp(self): - # Reenable GC as it is disabled module-wide - gc.enable() - - def tearDown(self): - gc.disable() - @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @requires_gil_enabled("Free threading does not support incremental GC") - # Use small increments to emulate longer running process in a shorter time - @gc_threshold(200, 10) def test_incremental_gc_handles_fast_cycle_creation(self): - - class LinkedList: - - #Use slots to reduce number of implicit objects - __slots__ = "next", "prev", "surprise" - - def __init__(self, next=None, prev=None): - self.next = next - if next is not None: - next.prev = self - self.prev = prev - if prev is not None: - prev.next = self - - def make_ll(depth): - head = LinkedList() - for i in range(depth): - head = LinkedList(head, head.prev) - return head - - head = make_ll(1000) - count = 1000 - - # There will be some objects we aren't counting, - # e.g. the gc stats dicts. This test checks - # that the counts don't grow, so we try to - # correct for the uncounted objects - # This is just an estimate. - CORRECTION = 20 - - enabled = gc.isenabled() - gc.enable() - olds = [] - initial_heap_size = _testinternalcapi.get_tracked_heap_size() - for i in range(20_000): - newhead = make_ll(20) - count += 20 - newhead.surprise = head - olds.append(newhead) - if len(olds) == 20: - new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size - self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") - del olds[:] - if not enabled: - gc.disable() + # Run this test in a fresh process. The number of alive objects (which can + # be from unit tests run before this one) can influence how quickly cyclic + # garbage is found. + script = support.findfile("_test_gc_fast_cycles.py") + run_test_script(script) class GCCallbackTests(unittest.TestCase): @@ -1261,9 +1316,11 @@ def test_collect(self): # Check that we got the right info dict for all callbacks for v in self.visit: info = v[2] - self.assertTrue("generation" in info) - self.assertTrue("collected" in info) - self.assertTrue("uncollectable" in info) + self.assertIn("generation", info) + self.assertIn("collected", info) + self.assertIn("uncollectable", info) + self.assertIn("candidates", info) + self.assertIn("duration", info) def test_collect_generation(self): self.preclean() @@ -1374,6 +1431,7 @@ def setUp(self): def tearDown(self): gc.disable() + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820c(self): # Corresponds to temp2c.py in the bug report. This is pretty # elaborate. @@ -1435,10 +1493,11 @@ def callback(ignored): # The free-threaded build doesn't have multiple generations, so # just trigger a GC manually. gc.collect() + assert not detector.gc_happened while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 100000: + self.fail("gc didn't happen after 100000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1449,6 +1508,7 @@ def callback(ignored): self.assertEqual(x, None) @gc_threshold(1000, 0, 0) + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref @@ -1509,8 +1569,8 @@ def __del__(self): gc.collect() while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 50000: + self.fail("gc didn't happen after 50000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1527,8 +1587,8 @@ def test_indirect_calls_with_gc_disabled(self): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 100000: + self.fail("gc didn't happen after 100000 iterations") junk.append([]) # this will eventually trigger gc try: @@ -1538,14 +1598,28 @@ def test_indirect_calls_with_gc_disabled(self): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 10000: + if i > 100000: break junk.append([]) # this may eventually trigger gc (if it is enabled) - self.assertEqual(i, 10001) + self.assertEqual(i, 100001) finally: gc.enable() + # Ensure that setting *threshold0* to zero disables collection. + @gc_threshold(0) + def test_threshold_zero(self): + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 50000: + break + junk.append([]) # this may eventually trigger gc (if it is enabled) + + self.assertEqual(i, 50001) + class PythonFinalizationTests(unittest.TestCase): def test_ast_fini(self): @@ -1567,6 +1641,19 @@ def test_ast_fini(self): """) assert_python_ok("-c", code) + def test_warnings_fini(self): + # See https://github.com/python/cpython/issues/137384 + code = textwrap.dedent(''' + import asyncio + from contextvars import ContextVar + + context_loop = ContextVar("context_loop", default=None) + loop = asyncio.new_event_loop() + context_loop.set(loop) + ''') + + assert_python_ok("-c", code) + def setUpModule(): global enabled, debug diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index d481cb07f75..de0dbab480f 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1,11 +1,9 @@ import contextlib import os -import re import sys import tempfile import unittest -from io import StringIO from test import support from test import test_tools @@ -31,12 +29,11 @@ def skip_if_different_mount_drives(): test_tools.skip_if_missing("cases_generator") with test_tools.imports_under_tool("cases_generator"): - from analyzer import analyze_forest, StackItem + from analyzer import StackItem from cwriter import CWriter import parser from stack import Local, Stack import tier1_generator - import opcode_metadata_generator import optimizer_generator @@ -59,14 +56,14 @@ class TestEffects(unittest.TestCase): def test_effect_sizes(self): stack = Stack() inputs = [ - x := StackItem("x", None, "1"), - y := StackItem("y", None, "oparg"), - z := StackItem("z", None, "oparg*2"), + x := StackItem("x", "1"), + y := StackItem("y", "oparg"), + z := StackItem("z", "oparg*2"), ] outputs = [ - StackItem("x", None, "1"), - StackItem("b", None, "oparg*4"), - StackItem("c", None, "1"), + StackItem("x", "1"), + StackItem("b", "oparg*4"), + StackItem("c", "1"), ] null = CWriter.null() stack.pop(z, null) @@ -135,7 +132,7 @@ def test_inst_no_args(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -157,7 +154,7 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -168,7 +165,7 @@ def test_inst_one_pop(self): value = stack_pointer[-1]; SPAM(value); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -182,7 +179,7 @@ def test_inst_one_push(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -193,7 +190,7 @@ def test_inst_one_push(self): res = SPAM(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -208,7 +205,7 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -235,7 +232,7 @@ def test_binary_op(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -250,7 +247,7 @@ def test_binary_op(self): res = SPAM(left, right); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -265,7 +262,7 @@ def test_overlap(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -299,7 +296,7 @@ def test_predictions(self): """ output = """ TARGET(OP1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP1; (void)(opcode); #endif @@ -316,7 +313,7 @@ def test_predictions(self): } TARGET(OP3) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP3; (void)(opcode); #endif @@ -358,7 +355,7 @@ def test_sync_sp(self): """ output = """ TARGET(A) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = A; (void)(opcode); #endif @@ -369,19 +366,19 @@ def test_sync_sp(self): _PyStackRef res; arg = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); escaping_call(); stack_pointer = _PyFrame_GetStackPointer(frame); res = Py_None; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(B) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = B; (void)(opcode); #endif @@ -424,7 +421,7 @@ def test_error_if_plain(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -447,7 +444,7 @@ def test_error_if_plain_with_comment(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -473,7 +470,7 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -492,7 +489,7 @@ def test_error_if_pop(self): res = 0; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -508,7 +505,7 @@ def test_error_if_pop_with_result(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -526,7 +523,7 @@ def test_error_if_pop_with_result(self): } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -540,7 +537,7 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -556,7 +553,7 @@ def test_cache_effect(self): uint32_t extra = read_u32(&this_instr[2].cache); (void)extra; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -573,7 +570,7 @@ def test_suppress_dispatch(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -607,7 +604,7 @@ def test_macro_instruction(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -643,12 +640,12 @@ def test_macro_instruction(self): } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(OP1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP1; (void)(opcode); #endif @@ -670,7 +667,7 @@ def test_macro_instruction(self): } TARGET(OP3) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP3; (void)(opcode); #endif @@ -691,7 +688,7 @@ def test_macro_instruction(self): stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -705,7 +702,7 @@ def test_unused_caches(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -731,7 +728,7 @@ def test_pseudo_instruction_no_flags(self): """ output = """ TARGET(OP1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP1; (void)(opcode); #endif @@ -754,7 +751,7 @@ def test_pseudo_instruction_with_flags(self): """ output = """ TARGET(OP1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP1; (void)(opcode); #endif @@ -780,7 +777,7 @@ def test_pseudo_instruction_as_sequence(self): """ output = """ TARGET(OP1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP1; (void)(opcode); #endif @@ -791,7 +788,7 @@ def test_pseudo_instruction_as_sequence(self): } TARGET(OP2) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP2; (void)(opcode); #endif @@ -815,7 +812,7 @@ def test_array_input(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -830,7 +827,7 @@ def test_array_input(self): below = stack_pointer[-2 - oparg*2]; SPAM(values, oparg); stack_pointer += -2 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -846,7 +843,7 @@ def test_array_output(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -863,7 +860,7 @@ def test_array_output(self): stack_pointer[-2] = below; stack_pointer[-1 + oparg*3] = above; stack_pointer += oparg*3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -878,7 +875,7 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -892,7 +889,7 @@ def test_array_input_output(self): above = 0; stack_pointer[0] = above; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -908,7 +905,7 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -921,11 +918,11 @@ def test_array_error_if(self): extra = stack_pointer[-1 - oparg]; if (oparg == 0) { stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -943,7 +940,7 @@ def test_macro_push_push(self): """ output = """ TARGET(M) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = M; (void)(opcode); #endif @@ -963,7 +960,7 @@ def test_macro_push_push(self): stack_pointer[0] = val1; stack_pointer[1] = val2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -980,7 +977,7 @@ def test_override_inst(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -1005,7 +1002,7 @@ def test_override_op(self): """ output = """ TARGET(M) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = M; (void)(opcode); #endif @@ -1026,7 +1023,7 @@ def test_annotated_inst(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -1048,7 +1045,7 @@ def test_annotated_op(self): """ output = """ TARGET(M) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = M; (void)(opcode); #endif @@ -1089,7 +1086,7 @@ def test_array_of_one(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -1106,32 +1103,6 @@ def test_array_of_one(self): """ self.run_cases_test(input, output) - def test_pointer_to_stackref(self): - input = """ - inst(OP, (arg: _PyStackRef * -- out)) { - out = *arg; - DEAD(arg); - } - """ - output = """ - TARGET(OP) { - #if Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef *arg; - _PyStackRef out; - arg = (_PyStackRef *)stack_pointer[-1].bits; - out = *arg; - stack_pointer[-1] = out; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - def test_unused_cached_value(self): input = """ op(FIRST, (arg1 -- out)) { @@ -1157,7 +1128,7 @@ def test_unused_named_values(self): """ output = """ TARGET(INST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INST; (void)(opcode); #endif @@ -1187,7 +1158,7 @@ def test_used_unused_used(self): """ output = """ TARGET(TEST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TEST; (void)(opcode); #endif @@ -1231,7 +1202,7 @@ def test_unused_used_used(self): """ output = """ TARGET(TEST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TEST; (void)(opcode); #endif @@ -1274,7 +1245,7 @@ def test_flush(self): """ output = """ TARGET(TEST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TEST; (void)(opcode); #endif @@ -1292,13 +1263,13 @@ def test_flush(self): stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); // SECOND { USE(a, b); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1326,7 +1297,7 @@ def test_pop_on_error_peeks(self): """ output = """ TARGET(TEST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TEST; (void)(opcode); #endif @@ -1354,7 +1325,7 @@ def test_pop_on_error_peeks(self): } } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1377,7 +1348,7 @@ def test_push_then_error(self): output = """ TARGET(TEST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TEST; (void)(opcode); #endif @@ -1397,14 +1368,14 @@ def test_push_then_error(self): stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1422,7 +1393,7 @@ def test_error_if_true(self): """ output = """ TARGET(OP1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP1; (void)(opcode); #endif @@ -1433,7 +1404,7 @@ def test_error_if_true(self): } TARGET(OP2) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP2; (void)(opcode); #endif @@ -1495,7 +1466,7 @@ def test_stack_save_reload(self): output = """ TARGET(BALANCED) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BALANCED; (void)(opcode); #endif @@ -1521,7 +1492,7 @@ def test_stack_save_reload_paired(self): output = """ TARGET(BALANCED) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BALANCED; (void)(opcode); #endif @@ -1543,7 +1514,7 @@ def test_stack_reload_only(self): output = """ TARGET(BALANCED) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BALANCED; (void)(opcode); #endif @@ -1568,7 +1539,7 @@ def test_stack_save_only(self): output = """ TARGET(BALANCED) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BALANCED; (void)(opcode); #endif @@ -1594,14 +1565,14 @@ def test_instruction_size_macro(self): output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - frame->return_offset = 1 ; + frame->return_offset = 1u ; DISPATCH(); } """ @@ -1633,7 +1604,7 @@ def test_escaping_call_next_to_cmacro(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -1671,7 +1642,7 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -1690,7 +1661,7 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self): stack_pointer[0] = out1; stack_pointer[1] = out2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1869,7 +1840,7 @@ def test_reassigning_live_inputs(self): output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -1896,7 +1867,7 @@ def test_reassigning_dead_inputs(self): """ output = """ TARGET(OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = OP; (void)(opcode); #endif @@ -1910,7 +1881,7 @@ def test_reassigning_dead_inputs(self): stack_pointer = _PyFrame_GetStackPointer(frame); in = temp; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(in); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2005,8 +1976,8 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - JitOptSymbol *arg1; - JitOptSymbol *out; + JitOptRef arg1; + JitOptRef out; arg1 = stack_pointer[-1]; out = EGGS(arg1); stack_pointer[-1] = out; @@ -2014,7 +1985,7 @@ def test_overridden_abstract_args(self): } case OP2: { - JitOptSymbol *out; + JitOptRef out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; @@ -2039,14 +2010,14 @@ def test_no_overridden_case(self): """ output = """ case OP: { - JitOptSymbol *out; + JitOptRef out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - JitOptSymbol *out; + JitOptRef out; out = NULL; stack_pointer[-1] = out; break; @@ -2066,9 +2037,440 @@ def test_missing_override_failure(self): """ output = """ """ - with self.assertRaisesRegex(AssertionError, "All abstract uops"): + with self.assertRaisesRegex(ValueError, "All abstract uops"): self.run_cases_test(input, input2, output) + def test_validate_uop_input_length_mismatch(self): + input = """ + op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1, arg2 -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Must have the same number of inputs"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_length_mismatch(self): + input = """ + op(OP, (arg1 -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1 -- out1, out2)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Must have the same number of outputs"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_input_name_mismatch(self): + input = """ + op(OP, (foo -- out)) { + SPAM(); + } + """ + input2 = """ + op(OP, (bar -- out)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal names"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_name_mismatch(self): + input = """ + op(OP, (arg1 -- foo)) { + SPAM(); + } + """ + input2 = """ + op(OP, (arg1 -- bar)) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Outputs must have equal names"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_input(self): + input = """ + op(OP, (unused -- )) { + } + """ + input2 = """ + op(OP, (foo -- )) { + } + """ + output = """ + case OP: { + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + input = """ + op(OP, (foo -- )) { + } + """ + input2 = """ + op(OP, (unused -- )) { + } + """ + output = """ + case OP: { + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_output(self): + input = """ + op(OP, ( -- unused)) { + } + """ + input2 = """ + op(OP, ( -- foo)) { + foo = NULL; + } + """ + output = """ + case OP: { + JitOptRef foo; + foo = NULL; + CHECK_STACK_BOUNDS(1); + stack_pointer[0] = foo; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + input = """ + op(OP, ( -- foo)) { + foo = NULL; + } + """ + input2 = """ + op(OP, ( -- unused)) { + } + """ + output = """ + case OP: { + CHECK_STACK_BOUNDS(1); + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_validate_uop_input_size_mismatch(self): + input = """ + op(OP, (arg1[2] -- )) { + } + """ + input2 = """ + op(OP, (arg1[4] -- )) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal sizes"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_output_size_mismatch(self): + input = """ + op(OP, ( -- out[2])) { + } + """ + input2 = """ + op(OP, ( -- out[4])) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Outputs must have equal sizes"): + self.run_cases_test(input, input2, output) + + def test_validate_uop_unused_size_mismatch(self): + input = """ + op(OP, (foo[2] -- )) { + } + """ + input2 = """ + op(OP, (unused[4] -- )) { + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Inputs must have equal sizes"): + self.run_cases_test(input, input2, output) + + def test_pure_uop_body_copied_in(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + res = PyStackRef_IsNone(foo); + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = sym_new_known(ctx, foo); + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + res_stackref = PyStackRef_IsNone(foo); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = sym_new_known(ctx, foo); + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_pure_uop_body_copied_in_deopt(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + DEOPT_IF(PyStackRef_IsNull(foo)); + res = foo; + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = foo; + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (PyStackRef_IsNull(foo)) { + ctx->done = true; + break; + } + res_stackref = foo; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = foo; + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_pure_uop_body_copied_in_error_if(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + ERROR_IF(PyStackRef_IsNull(foo)); + res = foo; + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = foo; + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (PyStackRef_IsNull(foo)) { + goto error; + } + res_stackref = foo; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = foo; + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + + def test_replace_opcode_uop_body_copied_in_complex(self): + input = """ + pure op(OP, (foo -- res)) { + if (foo) { + res = PyStackRef_IsNone(foo); + } + else { + res = 1; + } + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = sym_new_known(ctx, foo); + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (foo) { + res_stackref = PyStackRef_IsNone(foo); + } + else { + res_stackref = 1; + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = sym_new_known(ctx, foo); + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_replace_opcode_escaping_uop_body_copied_in_complex(self): + input = """ + pure op(OP, (foo -- res)) { + if (foo) { + res = ESCAPING_CODE(foo); + } + else { + res = 1; + } + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = sym_new_known(ctx, foo); + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (foo) { + res_stackref = ESCAPING_CODE(foo); + } + else { + res_stackref = 1; + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = sym_new_known(ctx, foo); + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_replace_opocode_uop_reject_array_effects(self): + input = """ + pure op(OP, (foo[2] -- res)) { + if (foo) { + res = PyStackRef_IsNone(foo); + } + else { + res = 1; + } + } + """ + input2 = """ + op(OP, (foo[2] -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = sym_new_unknown(ctx); + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Pure evaluation cannot take array-like inputs"): + self.run_cases_test(input, input2, output) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 8d21ded4501..9df9296e26a 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -17,7 +17,7 @@ from functools import partial, partialmethod, cached_property from graphlib import TopologicalSorter from logging import LoggerAdapter, StreamHandler -from mailbox import Mailbox, _PartialFile +from mailbox import Mailbox try: import ctypes except ImportError: @@ -61,6 +61,7 @@ from tkinter import Event except ImportError: Event = None +from string.templatelib import Template, Interpolation from typing import TypeVar T = TypeVar('T') @@ -116,7 +117,7 @@ class BaseTest(unittest.TestCase): Iterable, Iterator, Reversible, Container, Collection, - Mailbox, _PartialFile, + Mailbox, ContextVar, Token, Field, Set, MutableSet, @@ -139,7 +140,10 @@ class BaseTest(unittest.TestCase): DictReader, DictWriter, array, staticmethod, - classmethod] + classmethod, + Template, + Interpolation, + ] if ctypes is not None: generic_types.extend((ctypes.Array, ctypes.LibraryLoader, ctypes.py_object)) if ValueProxy is not None: @@ -232,13 +236,13 @@ class MyGeneric: self.assertEqual(repr(x2), 'tuple[*tuple[int, str]]') x3 = tuple[*tuple[int, ...]] self.assertEqual(repr(x3), 'tuple[*tuple[int, ...]]') - self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr..MyList[int]')) + self.assertEndsWith(repr(MyList[int]), '.BaseTest.test_repr..MyList[int]') self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr # gh-105488 - self.assertTrue(repr(MyGeneric[int]).endswith('MyGeneric[int]')) - self.assertTrue(repr(MyGeneric[[]]).endswith('MyGeneric[[]]')) - self.assertTrue(repr(MyGeneric[[int, str]]).endswith('MyGeneric[[int, str]]')) + self.assertEndsWith(repr(MyGeneric[int]), 'MyGeneric[int]') + self.assertEndsWith(repr(MyGeneric[[]]), 'MyGeneric[[]]') + self.assertEndsWith(repr(MyGeneric[[int, str]]), 'MyGeneric[[int, str]]') def test_exposed_type(self): import types @@ -358,7 +362,7 @@ def test_isinstance(self): def test_issubclass(self): class L(list): ... - self.assertTrue(issubclass(L, list)) + self.assertIsSubclass(L, list) with self.assertRaises(TypeError): issubclass(L, list[str]) @@ -398,7 +402,10 @@ def __deepcopy__(self, memo): aliases = [ GenericAlias(list, T), GenericAlias(deque, T), - GenericAlias(X, T) + GenericAlias(X, T), + X[T], + list[T], + deque[T], ] + _UNPACKED_TUPLES for alias in aliases: with self.subTest(alias=alias): @@ -428,10 +435,26 @@ def test_union_generic(self): self.assertEqual(a.__parameters__, (T,)) def test_dir(self): - dir_of_gen_alias = set(dir(list[int])) + ga = list[int] + dir_of_gen_alias = set(dir(ga)) self.assertTrue(dir_of_gen_alias.issuperset(dir(list))) - for generic_alias_property in ("__origin__", "__args__", "__parameters__"): - self.assertIn(generic_alias_property, dir_of_gen_alias) + for generic_alias_property in ( + "__origin__", "__args__", "__parameters__", + "__unpacked__", + ): + with self.subTest(generic_alias_property=generic_alias_property): + self.assertIn(generic_alias_property, dir_of_gen_alias) + for blocked in ( + "__bases__", + "__copy__", + "__deepcopy__", + ): + with self.subTest(blocked=blocked): + self.assertNotIn(blocked, dir_of_gen_alias) + + for entry in dir_of_gen_alias: + with self.subTest(entry=entry): + getattr(ga, entry) # must not raise `AttributeError` def test_weakref(self): for t in self.generic_types: diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 6c3abe602f5..dfc0817da45 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -2,14 +2,16 @@ Tests common to genericpath, ntpath and posixpath """ +import copy import genericpath import os +import pickle import sys import unittest import warnings -from test.support import ( - is_apple, is_emscripten, os_helper, warnings_helper -) +from test import support +from test.support import os_helper +from test.support import warnings_helper from test.support.script_helper import assert_python_ok from test.support.os_helper import FakePath @@ -92,8 +94,8 @@ def test_commonprefix(self): for s1 in testlist: for s2 in testlist: p = commonprefix([s1, s2]) - self.assertTrue(s1.startswith(p)) - self.assertTrue(s2.startswith(p)) + self.assertStartsWith(s1, p) + self.assertStartsWith(s2, p) if s1 != s2: n = len(p) self.assertNotEqual(s1[n:n+1], s2[n:n+1]) @@ -320,6 +322,21 @@ def test_sameopenfile(self): fd2 = fp2.fileno() self.assertTrue(self.pathmodule.sameopenfile(fd1, fd2)) + def test_realpath_mode_values(self): + for name in 'ALL_BUT_LAST', 'ALLOW_MISSING': + with self.subTest(name): + mode = getattr(self.pathmodule, name) + self.assertEqual(repr(mode), 'os.path.' + name) + self.assertEqual(str(mode), 'os.path.' + name) + self.assertTrue(mode) + self.assertIs(copy.copy(mode), mode) + self.assertIs(copy.deepcopy(mode), mode) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(mode, proto) + unpickled = pickle.loads(pickled) + self.assertIs(unpickled, mode) + class TestGenericTest(GenericTest, unittest.TestCase): # Issue 16852: GenericTest can't inherit from unittest.TestCase @@ -445,6 +462,19 @@ def check(value, expected): os.fsencode('$bar%s bar' % nonascii)) check(b'$spam}bar', os.fsencode('%s}bar' % nonascii)) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = self.pathmodule.expandvars + with os_helper.EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('$A'*n), 'B'*n) + self.assertEqual(expandvars('${A}'*n), 'B'*n) + self.assertEqual(expandvars('$A!'*n), 'B!'*n) + self.assertEqual(expandvars('${A}A'*n), 'BA'*n) + self.assertEqual(expandvars('${'*10*n), '${'*10*n) + def test_abspath(self): self.assertIn("foo", self.pathmodule.abspath("foo")) with warnings.catch_warnings(): @@ -502,7 +532,7 @@ def test_nonascii_abspath(self): # directory (when the bytes name is used). and sys.platform not in { "win32", "emscripten", "wasi" - } and not is_apple + } and not support.is_apple ): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index ab36535a1cf..9c3def2c3be 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -201,5 +201,41 @@ def test_control_chars_with_echo_char(self): self.assertEqual('Password: *******\x08 \x08', mock_output.getvalue()) +class GetpassEchoCharTest(unittest.TestCase): + + def test_accept_none(self): + getpass._check_echo_char(None) + + @support.subTests('echo_char', ["*", "A", " "]) + def test_accept_single_printable_ascii(self, echo_char): + getpass._check_echo_char(echo_char) + + def test_reject_empty_string(self): + self.assertRaises(ValueError, getpass.getpass, echo_char="") + + @support.subTests('echo_char', ["***", "AA", "aA*!"]) + def test_reject_multi_character_strings(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + @support.subTests('echo_char', [ + '\N{LATIN CAPITAL LETTER AE}', # non-ASCII single character + '\N{HEAVY BLACK HEART}', # non-ASCII multibyte character + ]) + def test_reject_non_ascii(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + @support.subTests('echo_char', [ + ch for ch in map(chr, range(0, 128)) + if not ch.isprintable() + ]) + def test_reject_non_printable_characters(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + # TypeError Rejection + @support.subTests('echo_char', [b"*", 0, 0.0, [], {}]) + def test_reject_non_string(self, echo_char): + self.assertRaises(TypeError, getpass.getpass, echo_char=echo_char) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index f86df9d0d03..83f09f34955 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -354,6 +354,27 @@ def test_venv_posix(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_venv_posix_without_home_key(self): + ns = MockPosixNamespace( + argv0="/venv/bin/python3", + PREFIX="/usr", + ENV_PATH="/usr/bin", + ) + # Setup the bare minimum venv + ns.add_known_xfile("/usr/bin/python3") + ns.add_known_xfile("/venv/bin/python3") + ns.add_known_link("/venv/bin/python3", "/usr/bin/python3") + ns.add_known_file("/venv/pyvenv.cfg", [ + # home = key intentionally omitted + ]) + expected = dict( + executable="/venv/bin/python3", + prefix="/venv", + base_prefix="/usr", + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_venv_changed_name_posix(self): "Test a venv layout on *nix." ns = MockPosixNamespace( diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 33b7d75e3ff..9ad37909a8e 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -937,6 +937,13 @@ def test_lazy_import(self): ensure_lazy_imports("gettext", {"re", "warnings", "locale"}) +class TranslationFallbackTestCase(unittest.TestCase): + def test_translation_fallback(self): + with os_helper.temp_cwd() as tempdir: + t = gettext.translation('gettext', localedir=tempdir, fallback=True) + self.assertIsInstance(t, gettext.NullTranslations) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index d0ed5129253..9e4e82b2542 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -4,7 +4,6 @@ import shutil import sys import unittest -import warnings from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, @@ -393,36 +392,6 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) - def test_glob0(self): - with self.assertWarns(DeprecationWarning): - glob.glob0(self.tempdir, 'a') - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - eq = self.assertSequencesEqual_noorder - eq(glob.glob0(self.tempdir, 'a'), ['a']) - eq(glob.glob0(self.tempdir, '.bb'), ['.bb']) - eq(glob.glob0(self.tempdir, '.b*'), []) - eq(glob.glob0(self.tempdir, 'b'), []) - eq(glob.glob0(self.tempdir, '?'), []) - eq(glob.glob0(self.tempdir, '*a'), []) - eq(glob.glob0(self.tempdir, 'a*'), []) - - def test_glob1(self): - with self.assertWarns(DeprecationWarning): - glob.glob1(self.tempdir, 'a') - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - eq = self.assertSequencesEqual_noorder - eq(glob.glob1(self.tempdir, 'a'), ['a']) - eq(glob.glob1(self.tempdir, '.bb'), ['.bb']) - eq(glob.glob1(self.tempdir, '.b*'), ['.bb']) - eq(glob.glob1(self.tempdir, 'b'), []) - eq(glob.glob1(self.tempdir, '?'), ['a']) - eq(glob.glob1(self.tempdir, '*a'), ['a', 'aaa']) - eq(glob.glob1(self.tempdir, 'a*'), ['a', 'aaa', 'aab']) - def test_translate_matching(self): match = re.compile(glob.translate('*')).match self.assertIsNotNone(match('foo')) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index c39565144bf..cfb24a5c457 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1,7 +1,7 @@ # Python test set -- part 1, grammar. # This just tests whether the parser accepts them all. -from test.support import check_syntax_error +from test.support import check_syntax_error, skip_wasi_stack_overflow from test.support import import_helper import annotationlib import inspect @@ -249,6 +249,18 @@ def test_eof_error(self): compile(s, "", "exec") self.assertIn("was never closed", str(cm.exception)) + @skip_wasi_stack_overflow() + def test_max_level(self): + # Macro defined in Parser/lexer/state.h + MAXLEVEL = 200 + + result = eval("(" * MAXLEVEL + ")" * MAXLEVEL) + self.assertEqual(result, ()) + + with self.assertRaises(SyntaxError) as cm: + eval("(" * (MAXLEVEL + 1) + ")" * (MAXLEVEL + 1)) + self.assertStartsWith(str(cm.exception), 'too many nested parentheses') + var_annot_global: int # a global annotated is necessary for test_var_annot @@ -1542,6 +1554,8 @@ def check(test): check('[None [i, j]]') check('[True [i, j]]') check('[... [i, j]]') + check('[t"{x}" [i, j]]') + check('[t"x={x}" [i, j]]') msg=r'indices must be integers or slices, not tuple; perhaps you missed a comma\?' check('[(1, 2) [i, j]]') @@ -1552,8 +1566,6 @@ def check(test): check('[f"x={x}" [i, j]]') check('["abc" [i, j]]') check('[b"abc" [i, j]]') - check('[t"{x}" [i, j]]') - check('[t"x={x}" [i, j]]') msg=r'indices must be integers or slices, not tuple;' check('[[1, 2] [3, 4]]') @@ -1574,6 +1586,7 @@ def check(test): check('[[1, 2] [f"{x}"]]') check('[[1, 2] [f"x={x}"]]') check('[[1, 2] ["abc"]]') + msg=r'indices must be integers or slices, not string.templatelib.Template;' check('[[1, 2] [t"{x}"]]') check('[[1, 2] [t"x={x}"]]') msg=r'indices must be integers or slices, not' diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index fa5de7c190e..442d30fc970 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -9,10 +9,9 @@ import struct import sys import unittest -import warnings from subprocess import PIPE, Popen from test.support import catch_unraisable_exception -from test.support import import_helper +from test.support import force_not_colorized_test_class, import_helper from test.support import os_helper from test.support import _4G, bigmemtest, requires_subprocess from test.support.script_helper import assert_python_ok, assert_python_failure @@ -331,13 +330,13 @@ def test_mode(self): def test_1647484(self): for mode in ('wb', 'rb'): with gzip.GzipFile(self.filename, mode) as f: - self.assertTrue(hasattr(f, "name")) + self.assertHasAttr(f, "name") self.assertEqual(f.name, self.filename) def test_paddedfile_getattr(self): self.test_write() with gzip.GzipFile(self.filename, 'rb') as f: - self.assertTrue(hasattr(f.fileobj, "name")) + self.assertHasAttr(f.fileobj, "name") self.assertEqual(f.fileobj.name, self.filename) def test_mtime(self): @@ -345,7 +344,7 @@ def test_mtime(self): with gzip.GzipFile(self.filename, 'w', mtime = mtime) as fWrite: fWrite.write(data1) with gzip.GzipFile(self.filename) as fRead: - self.assertTrue(hasattr(fRead, 'mtime')) + self.assertHasAttr(fRead, 'mtime') self.assertIsNone(fRead.mtime) dataRead = fRead.read() self.assertEqual(dataRead, data1) @@ -354,7 +353,7 @@ def test_mtime(self): def test_metadata(self): mtime = 123456789 - with gzip.GzipFile(self.filename, 'w', mtime = mtime) as fWrite: + with gzip.GzipFile(self.filename, 'w', mtime = mtime, compresslevel = 9) as fWrite: fWrite.write(data1) with open(self.filename, 'rb') as fRead: @@ -460,7 +459,7 @@ def test_zero_padded_file(self): self.assertEqual(d, data1 * 50, "Incorrect data in file") def test_gzip_BadGzipFile_exception(self): - self.assertTrue(issubclass(gzip.BadGzipFile, OSError)) + self.assertIsSubclass(gzip.BadGzipFile, OSError) def test_bad_gzip_file(self): with open(self.filename, 'wb') as file: @@ -640,7 +639,7 @@ def test_fileobj_mode(self): with open(self.filename, mode) as f: with gzip.GzipFile(fileobj=f) as g: self.assertEqual(g.mode, gzip.READ) - for mode in "wb", "ab", "xb": + for mode in "wb", "ab", "xb", "wb+", "ab+", "xb+": if "x" in mode: os_helper.unlink(self.filename) with open(self.filename, mode) as f: @@ -1058,6 +1057,7 @@ def wrapper(*args, **kwargs): return decorator +@force_not_colorized_test_class class TestCommandLine(unittest.TestCase): data = b'This is a simple test with gzip' diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 5e3356a02f3..489bb049d2f 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -12,12 +12,12 @@ import itertools import logging import os +import re import sys import sysconfig import tempfile import threading import unittest -import warnings from test import support from test.support import _4G, bigmemtest from test.support import hashlib_helper @@ -27,32 +27,28 @@ from http.client import HTTPException -default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'} +default_builtin_hashes = {'md5', 'sha1', 'sha2', 'sha3', 'blake2'} # --with-builtin-hashlib-hashes override builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES") if builtin_hashes is None: builtin_hashes = default_builtin_hashes else: - builtin_hashes = { - m.strip() for m in builtin_hashes.strip('"').lower().split(",") - } + builtin_hash_names = builtin_hashes.strip('"').lower().split(",") + builtin_hashes = set(map(str.strip, builtin_hash_names)) -# hashlib with and without OpenSSL backend for PBKDF2 -# only import builtin_hashlib when all builtin hashes are available. -# Otherwise import prints noise on stderr +# Public 'hashlib' module with OpenSSL backend for PBKDF2. openssl_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) -if builtin_hashes == default_builtin_hashes: - builtin_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) -else: - builtin_hashlib = None try: - from _hashlib import HASH, HASHXOF, openssl_md_meth_names, get_fips_mode + import _hashlib except ImportError: - HASH = None - HASHXOF = None - openssl_md_meth_names = frozenset() - + _hashlib = None +# The extension module may exist but only define some of these. gh-141907 +HASH = getattr(_hashlib, 'HASH', None) +HASHXOF = getattr(_hashlib, 'HASHXOF', None) +openssl_md_meth_names = getattr(_hashlib, 'openssl_md_meth_names', frozenset()) +get_fips_mode = getattr(_hashlib, 'get_fips_mode', None) +if not get_fips_mode: def get_fips_mode(): return 0 @@ -98,6 +94,14 @@ def read_vectors(hash_name): yield parts +DEPRECATED_STRING_PARAMETER = re.escape( + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead" +) + + class HashLibTestCase(unittest.TestCase): supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', 'sha224', 'SHA224', 'sha256', 'SHA256', @@ -141,19 +145,18 @@ def __init__(self, *args, **kwargs): # of hashlib.new given the algorithm name. for algorithm, constructors in self.constructors_to_test.items(): constructors.add(getattr(hashlib, algorithm)) - def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): - if data is None: - return hashlib.new(_alg, **kwargs) - return hashlib.new(_alg, data, **kwargs) - constructors.add(_test_algorithm_via_hashlib_new) + def c(*args, __algorithm_name=algorithm, **kwargs): + return hashlib.new(__algorithm_name, *args, **kwargs) + c.__name__ = f'do_test_algorithm_via_hashlib_new_{algorithm}' + constructors.add(c) _hashlib = self._conditional_import_module('_hashlib') self._hashlib = _hashlib if _hashlib: # These algorithms should always be present when this module # is compiled. If not, something was compiled wrong. - self.assertTrue(hasattr(_hashlib, 'openssl_md5')) - self.assertTrue(hasattr(_hashlib, 'openssl_sha1')) + self.assertHasAttr(_hashlib, 'openssl_md5') + self.assertHasAttr(_hashlib, 'openssl_sha1') for algorithm, constructors in self.constructors_to_test.items(): constructor = getattr(_hashlib, 'openssl_'+algorithm, None) if constructor: @@ -200,6 +203,11 @@ def hash_constructors(self): constructors = self.constructors_to_test.values() return itertools.chain.from_iterable(constructors) + @property + def shake_constructors(self): + for shake_name in self.shakes: + yield from self.constructors_to_test.get(shake_name, ()) + @property def is_fips_mode(self): return get_fips_mode() @@ -250,9 +258,90 @@ def test_usedforsecurity_false(self): self._hashlib.new("md5", usedforsecurity=False) self._hashlib.openssl_md5(usedforsecurity=False) + @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") + def test_clinic_signature(self): + for constructor in self.hash_constructors: + with self.subTest(constructor.__name__): + constructor(b'') + constructor(data=b'') + with self.assertWarnsRegex(DeprecationWarning, + DEPRECATED_STRING_PARAMETER): + constructor(string=b'') + + digest_name = constructor(b'').name + with self.subTest(digest_name): + hashlib.new(digest_name, b'') + hashlib.new(digest_name, data=b'') + with self.assertWarnsRegex(DeprecationWarning, + DEPRECATED_STRING_PARAMETER): + hashlib.new(digest_name, string=b'') + # Make sure that _hashlib contains the constructor + # to test when using a combination of libcrypto and + # interned hash implementations. + if self._hashlib and digest_name in self._hashlib._constructors: + self._hashlib.new(digest_name, b'') + self._hashlib.new(digest_name, data=b'') + with self.assertWarnsRegex(DeprecationWarning, + DEPRECATED_STRING_PARAMETER): + self._hashlib.new(digest_name, string=b'') + + @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") + def test_clinic_signature_errors(self): + nomsg = b'' + mymsg = b'msg' + conflicting_call = re.escape( + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version." + ) + duplicated_param = re.escape("given by name ('data') and position") + unexpected_param = re.escape("got an unexpected keyword argument '_'") + for args, kwds, errmsg in [ + # Reject duplicated arguments before unknown keyword arguments. + ((nomsg,), dict(data=nomsg, _=nomsg), duplicated_param), + ((mymsg,), dict(data=nomsg, _=nomsg), duplicated_param), + # Reject duplicated arguments before conflicting ones. + *itertools.product( + [[nomsg], [mymsg]], + [dict(data=nomsg), dict(data=nomsg, string=nomsg)], + [duplicated_param] + ), + # Reject unknown keyword arguments before conflicting ones. + *itertools.product( + [()], + [ + dict(_=None), + dict(data=nomsg, _=None), + dict(string=nomsg, _=None), + dict(string=nomsg, data=nomsg, _=None), + ], + [unexpected_param] + ), + ((nomsg,), dict(_=None), unexpected_param), + ((mymsg,), dict(_=None), unexpected_param), + # Reject conflicting arguments. + [(nomsg,), dict(string=nomsg), conflicting_call], + [(mymsg,), dict(string=nomsg), conflicting_call], + [(), dict(data=nomsg, string=nomsg), conflicting_call], + ]: + for constructor in self.hash_constructors: + digest_name = constructor(b'').name + with self.subTest(constructor.__name__, args=args, kwds=kwds): + with self.assertRaisesRegex(TypeError, errmsg): + constructor(*args, **kwds) + with self.subTest(digest_name, args=args, kwds=kwds): + with self.assertRaisesRegex(TypeError, errmsg): + hashlib.new(digest_name, *args, **kwds) + if (self._hashlib and + digest_name in self._hashlib._constructors): + with self.assertRaisesRegex(TypeError, errmsg): + self._hashlib.new(digest_name, *args, **kwds) + def test_unknown_hash(self): self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') - self.assertRaises(TypeError, hashlib.new, 1) + # ensure that the exception message remains consistent + err = re.escape("new() argument 'name' must be str, not int") + self.assertRaisesRegex(TypeError, err, hashlib.new, 1) def test_new_upper_to_lower(self): self.assertEqual(hashlib.new("SHA256").name, "sha256") @@ -279,11 +368,23 @@ def test_get_builtin_constructor(self): sys.modules['_md5'] = _md5 else: del sys.modules['_md5'] - self.assertRaises(TypeError, get_builtin_constructor, 3) + # ensure that the exception message remains consistent + err = re.escape("new() argument 'name' must be str, not int") + self.assertRaises(TypeError, err, get_builtin_constructor, 3) constructor = get_builtin_constructor('md5') self.assertIs(constructor, _md5.md5) self.assertEqual(sorted(builtin_constructor_cache), ['MD5', 'md5']) + def test_copy(self): + for cons in self.hash_constructors: + h1 = cons(os.urandom(16), usedforsecurity=False) + h2 = h1.copy() + self.assertIs(type(h1), type(h2)) + self.assertEqual(h1.name, h2.name) + size = (16,) if h1.name in self.shakes else () + self.assertEqual(h1.digest(*size), h2.digest(*size)) + self.assertEqual(h1.hexdigest(*size), h2.hexdigest(*size)) + def test_hexdigest(self): for cons in self.hash_constructors: h = cons(usedforsecurity=False) @@ -294,21 +395,50 @@ def test_hexdigest(self): self.assertIsInstance(h.digest(), bytes) self.assertEqual(hexstr(h.digest()), h.hexdigest()) - def test_digest_length_overflow(self): - # See issue #34922 - large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10) - for cons in self.hash_constructors: - h = cons(usedforsecurity=False) - if h.name not in self.shakes: - continue - if HASH is not None and isinstance(h, HASH): - # _hashopenssl's take a size_t - continue - for digest in h.digest, h.hexdigest: - self.assertRaises(ValueError, digest, -10) - for length in large_sizes: - with self.assertRaises((ValueError, OverflowError)): - digest(length) + def test_shakes_zero_digest_length(self): + for constructor in self.shake_constructors: + with self.subTest(constructor=constructor): + h = constructor(b'abcdef', usedforsecurity=False) + self.assertEqual(h.digest(0), b'') + self.assertEqual(h.hexdigest(0), '') + + def test_shakes_invalid_digest_length(self): + # See https://github.com/python/cpython/issues/79103. + for constructor in self.shake_constructors: + with self.subTest(constructor=constructor): + h = constructor(usedforsecurity=False) + # Note: digest() and hexdigest() take a signed input and + # raise if it is negative; the rationale is that we use + # internally PyBytes_FromStringAndSize() and _Py_strhex() + # which both take a Py_ssize_t. + for negative_size in (-1, -10, -(1 << 31), -sys.maxsize): + self.assertRaises(ValueError, h.digest, negative_size) + self.assertRaises(ValueError, h.hexdigest, negative_size) + + def test_shakes_overflow_digest_length(self): + # See https://github.com/python/cpython/issues/135759. + + exc_types = (OverflowError, ValueError) + # HACL* accepts an 'uint32_t' while OpenSSL accepts a 'size_t'. + openssl_overflown_sizes = (sys.maxsize + 1, 2 * sys.maxsize) + # https://github.com/python/cpython/issues/79103 restricts + # the accepted built-in lengths to 2 ** 29, even if OpenSSL + # accepts such lengths. + builtin_overflown_sizes = openssl_overflown_sizes + ( + 2 ** 29, 2 ** 32 - 10, 2 ** 32, 2 ** 32 + 10, + 2 ** 61, 2 ** 64 - 10, 2 ** 64, 2 ** 64 + 10, + ) + + for constructor in self.shake_constructors: + with self.subTest(constructor=constructor): + h = constructor(usedforsecurity=False) + if HASH is not None and isinstance(h, HASH): + overflown_sizes = openssl_overflown_sizes + else: + overflown_sizes = builtin_overflown_sizes + for invalid_size in overflown_sizes: + self.assertRaises(exc_types, h.digest, invalid_size) + self.assertRaises(exc_types, h.hexdigest, invalid_size) def test_name_attribute(self): for cons in self.hash_constructors: @@ -411,13 +541,17 @@ def check(self, name, data, hexdigest, shake=False, **kwargs): def check_file_digest(self, name, data, hexdigest): hexdigest = hexdigest.lower() - try: - hashlib.new(name) - except ValueError: - # skip, algorithm is blocked by security policy. - return - digests = [name] - digests.extend(self.constructors_to_test[name]) + digests = [] + for digest in [name, *self.constructors_to_test[name]]: + try: + if callable(digest): + digest(b"") + else: + hashlib.new(digest) + except ValueError: + # skip, algorithm is blocked by security policy. + continue + digests.append(digest) with tempfile.TemporaryFile() as f: f.write(data) @@ -500,9 +634,14 @@ def check_sha3(self, name, capacity, rate, suffix): constructors = self.constructors_to_test[name] for hash_object_constructor in constructors: m = hash_object_constructor() - if HASH is not None and isinstance(m, HASH): - # _hashopenssl's variant does not have extra SHA3 attributes - continue + if name.startswith('shake_'): + if HASHXOF is not None and isinstance(m, HASHXOF): + # _hashopenssl's variant does not have extra SHA3 attributes + continue + else: + if HASH is not None and isinstance(m, HASH): + # _hashopenssl's variant does not have extra SHA3 attributes + continue self.assertEqual(capacity + rate, 1600) self.assertEqual(m._capacity_bits, capacity) self.assertEqual(m._rate_bits, rate) @@ -719,8 +858,6 @@ def check_blake2(self, constructor, salt_size, person_size, key_size, self.assertRaises(ValueError, constructor, node_offset=-1) self.assertRaises(OverflowError, constructor, node_offset=max_offset+1) - self.assertRaises(TypeError, constructor, data=b'') - self.assertRaises(TypeError, constructor, string=b'') self.assertRaises(TypeError, constructor, '') constructor( @@ -929,49 +1066,67 @@ def test_gil(self): def test_sha256_gil(self): gil_minsize = hashlib_helper.find_gil_minsize(['_sha2', '_hashlib']) + data = b'1' + b'#' * gil_minsize + b'1' + expected = hashlib.sha256(data).hexdigest() + m = hashlib.sha256() m.update(b'1') m.update(b'#' * gil_minsize) m.update(b'1') - self.assertEqual( - m.hexdigest(), - '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94' - ) - - m = hashlib.sha256(b'1' + b'#' * gil_minsize + b'1') - self.assertEqual( - m.hexdigest(), - '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94' - ) + self.assertEqual(m.hexdigest(), expected) @threading_helper.reap_threads @threading_helper.requires_working_threading() - def test_threaded_hashing(self): + def test_threaded_hashing_fast(self): + # Same as test_threaded_hashing_slow() but only tests some functions + # since otherwise test_hashlib.py becomes too slow during development. + for name in ['md5', 'sha1', 'sha256', 'sha3_256', 'blake2s']: + if constructor := getattr(hashlib, name, None): + with self.subTest(name): + self.do_test_threaded_hashing(constructor, is_shake=False) + if shake_128 := getattr(hashlib, 'shake_128', None): + self.do_test_threaded_hashing(shake_128, is_shake=True) + + @requires_resource('cpu') + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_threaded_hashing_slow(self): + for algorithm, constructors in self.constructors_to_test.items(): + is_shake = algorithm in self.shakes + for constructor in constructors: + with self.subTest(constructor.__name__, is_shake=is_shake): + self.do_test_threaded_hashing(constructor, is_shake) + + def do_test_threaded_hashing(self, constructor, is_shake): # Updating the same hash object from several threads at once # using data chunk sizes containing the same byte sequences. # # If the internal locks are working to prevent multiple # updates on the same object from running at once, the resulting # hash will be the same as doing it single threaded upfront. - hasher = hashlib.sha1() - num_threads = 5 - smallest_data = b'swineflu' - data = smallest_data * 200000 - expected_hash = hashlib.sha1(data*num_threads).hexdigest() - def hash_in_chunks(chunk_size): - index = 0 - while index < len(data): - hasher.update(data[index:index + chunk_size]) - index += chunk_size + # The data to hash has length s|M|q^N and the chunk size for the i-th + # thread is s|M|q^(N-i), where N is the number of threads, M is a fixed + # message of small length, and s >= 1 and q >= 2 are small integers. + smallest_size, num_threads, s, q = 8, 5, 2, 10 + + smallest_data = os.urandom(smallest_size) + data = s * smallest_data * (q ** num_threads) + + h1 = constructor(usedforsecurity=False) + h2 = constructor(data * num_threads, usedforsecurity=False) + + def update(chunk_size): + for index in range(0, len(data), chunk_size): + h1.update(data[index:index + chunk_size]) threads = [] - for threadnum in range(num_threads): - chunk_size = len(data) // (10 ** threadnum) + for thread_num in range(num_threads): + # chunk_size = len(data) // (q ** thread_num) + chunk_size = s * smallest_size * q ** (num_threads - thread_num) self.assertGreater(chunk_size, 0) - self.assertEqual(chunk_size % len(smallest_data), 0) - thread = threading.Thread(target=hash_in_chunks, - args=(chunk_size,)) + self.assertEqual(chunk_size % smallest_size, 0) + thread = threading.Thread(target=update, args=(chunk_size,)) threads.append(thread) for thread in threads: @@ -979,7 +1134,10 @@ def hash_in_chunks(chunk_size): for thread in threads: thread.join() - self.assertEqual(expected_hash, hasher.hexdigest()) + if is_shake: + self.assertEqual(h1.hexdigest(16), h2.hexdigest(16)) + else: + self.assertEqual(h1.hexdigest(), h2.hexdigest()) def test_get_fips_mode(self): fips_mode = self.is_fips_mode @@ -1006,7 +1164,8 @@ def test_disallow_instantiation(self): def test_hash_disallow_instantiation(self): # internal types like _hashlib.HASH are not constructable support.check_disallow_instantiation(self, HASH) - support.check_disallow_instantiation(self, HASHXOF) + if HASHXOF is not None: + support.check_disallow_instantiation(self, HASHXOF) def test_readonly_types(self): for algorithm, constructors in self.constructors_to_test.items(): @@ -1035,12 +1194,6 @@ class KDFTests(unittest.TestCase): (b'pass\0word', b'sa\0lt', 4096, 16), ] - scrypt_test_vectors = [ - (b'', b'', 16, 1, 1, unhexlify('77d6576238657b203b19ca42c18a0497f16b4844e3074ae8dfdffa3fede21442fcd0069ded0948f8326a753a0fc81f17e8d3e0fb2e0d3628cf35e20c38d18906')), - (b'password', b'NaCl', 1024, 8, 16, unhexlify('fdbabe1c9d3472007856e7190d01e9fe7c6ad7cbc8237830e77376634b3731622eaf30d92e22a3886ff109279d9830dac727afb94a83ee6d8360cbdfa2cc0640')), - (b'pleaseletmein', b'SodiumChloride', 16384, 8, 1, unhexlify('7023bdcb3afd7348461c06cd81fd38ebfda8fbba904f8e3ea9b543f6545da1f2d5432955613f0fcf62d49705242a9af9e61e85dc0d651e40dfcf017b45575887')), - ] - pbkdf2_results = { "sha1": [ # official test vectors from RFC 6070 @@ -1132,46 +1285,6 @@ def _test_pbkdf2_hmac(self, pbkdf2, supported): def test_pbkdf2_hmac_c(self): self._test_pbkdf2_hmac(openssl_hashlib.pbkdf2_hmac, openssl_md_meth_names) - @unittest.skipUnless(hasattr(hashlib, 'scrypt'), - ' test requires OpenSSL > 1.1') - @unittest.skipIf(get_fips_mode(), reason="scrypt is blocked in FIPS mode") - def test_scrypt(self): - for password, salt, n, r, p, expected in self.scrypt_test_vectors: - result = hashlib.scrypt(password, salt=salt, n=n, r=r, p=p) - self.assertEqual(result, expected) - - # this values should work - hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1) - # password and salt must be bytes-like - with self.assertRaises(TypeError): - hashlib.scrypt('password', salt=b'salt', n=2, r=8, p=1) - with self.assertRaises(TypeError): - hashlib.scrypt(b'password', salt='salt', n=2, r=8, p=1) - # require keyword args - with self.assertRaises(TypeError): - hashlib.scrypt(b'password') - with self.assertRaises(TypeError): - hashlib.scrypt(b'password', b'salt') - with self.assertRaises(TypeError): - hashlib.scrypt(b'password', 2, 8, 1, salt=b'salt') - for n in [-1, 0, 1, None]: - with self.assertRaises((ValueError, OverflowError, TypeError)): - hashlib.scrypt(b'password', salt=b'salt', n=n, r=8, p=1) - for r in [-1, 0, None]: - with self.assertRaises((ValueError, OverflowError, TypeError)): - hashlib.scrypt(b'password', salt=b'salt', n=2, r=r, p=1) - for p in [-1, 0, None]: - with self.assertRaises((ValueError, OverflowError, TypeError)): - hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=p) - for maxmem in [-1, None]: - with self.assertRaises((ValueError, OverflowError, TypeError)): - hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1, - maxmem=maxmem) - for dklen in [-1, None]: - with self.assertRaises((ValueError, OverflowError, TypeError)): - hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1, - dklen=dklen) - def test_normalized_name(self): self.assertNotIn("blake2b512", hashlib.algorithms_available) self.assertNotIn("sha3-512", hashlib.algorithms_available) @@ -1213,5 +1326,76 @@ def readable(self): hashlib.file_digest(NonBlocking(), hashlib.sha256) +@unittest.skipUnless(hasattr(hashlib, 'scrypt'), 'requires OpenSSL 1.1+') +@unittest.skipIf(get_fips_mode(), reason="scrypt is blocked in FIPS mode") +class TestScrypt(unittest.TestCase): + + scrypt_test_vectors = [ + (b'', b'', 16, 1, 1, unhexlify('77d6576238657b203b19ca42c18a0497f16b4844e3074ae8dfdffa3fede21442fcd0069ded0948f8326a753a0fc81f17e8d3e0fb2e0d3628cf35e20c38d18906')), + (b'password', b'NaCl', 1024, 8, 16, unhexlify('fdbabe1c9d3472007856e7190d01e9fe7c6ad7cbc8237830e77376634b3731622eaf30d92e22a3886ff109279d9830dac727afb94a83ee6d8360cbdfa2cc0640')), + (b'pleaseletmein', b'SodiumChloride', 16384, 8, 1, unhexlify('7023bdcb3afd7348461c06cd81fd38ebfda8fbba904f8e3ea9b543f6545da1f2d5432955613f0fcf62d49705242a9af9e61e85dc0d651e40dfcf017b45575887')), + ] + + def test_scrypt(self): + for password, salt, n, r, p, expected in self.scrypt_test_vectors: + result = hashlib.scrypt(password, salt=salt, n=n, r=r, p=p) + self.assertEqual(result, expected) + + # these parameters must be valid + hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1) + hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1, maxmem=0) + hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1, dklen=1) + + def test_scrypt_types(self): + # password and salt must be bytes-like + with self.assertRaises(TypeError): + hashlib.scrypt('password', salt=b'salt', n=2, r=8, p=1) + with self.assertRaises(TypeError): + hashlib.scrypt(b'password', salt='salt', n=2, r=8, p=1) + # require keyword args + with self.assertRaises(TypeError): + hashlib.scrypt(b'password') + with self.assertRaises(TypeError): + hashlib.scrypt(b'password', b'salt') + with self.assertRaises(TypeError): + hashlib.scrypt(b'password', 2, 8, 1, salt=b'salt') + + def test_scrypt_validate(self): + def scrypt(password=b"password", /, **kwargs): + # overwrite well-defined parameters with bad ones + kwargs = dict(salt=b'salt', n=2, r=8, p=1) | kwargs + return hashlib.scrypt(password, **kwargs) + + for param_name in ('n', 'r', 'p', 'maxmem', 'dklen'): + param = {param_name: None} + with self.subTest(**param): + self.assertRaises(TypeError, scrypt, **param) + + self.assertRaises(ValueError, scrypt, n=0) + self.assertRaises(ValueError, scrypt, n=-1) + self.assertRaises(ValueError, scrypt, n=1) + + self.assertRaises(ValueError, scrypt, r=0) + self.assertRaises(ValueError, scrypt, r=-1) + + self.assertRaises(ValueError, scrypt, p=-1) + self.assertRaises(ValueError, scrypt, p=0) + + self.assertRaises(ValueError, scrypt, maxmem=-1) + # OpenSSL hard limit for 'maxmem' is an 'uint64_t' but for now, + # we do not use the 'uint64' Clinic converter but the 'long' one. + self.assertRaises(OverflowError, scrypt, maxmem=(1 << 64)) + # Historically, Python allowed 'maxmem' to be at most INT_MAX, + # which is at most 2**32-1 (on Windows, sizeof(long) == 4, so + # an OverflowError will be raised instead of a ValueError). + numeric_exc_types = (OverflowError, ValueError) + self.assertRaises(numeric_exc_types, scrypt, maxmem=(1 << 32)) + + self.assertRaises(ValueError, scrypt, dklen=-1) + self.assertRaises(ValueError, scrypt, dklen=0) + MAX_DKLEN = ((1 << 32) - 1) * 32 # see RFC 7914 + self.assertRaises(numeric_exc_types, scrypt, dklen=MAX_DKLEN + 1) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 70c79437722..7634deeb1d8 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -1,16 +1,41 @@ +"""Test suite for HMAC. + +Python provides three different implementations of HMAC: + +- OpenSSL HMAC using OpenSSL hash functions. +- HACL* HMAC using HACL* hash functions. +- Generic Python HMAC using user-defined hash functions. + +The generic Python HMAC implementation is able to use OpenSSL +callables or names, HACL* named hash functions or arbitrary +objects implementing PEP 247 interface. + +In the two first cases, Python HMAC wraps a C HMAC object (either OpenSSL +or HACL*-based). As a last resort, HMAC is re-implemented in pure Python. +It is however interesting to test the pure Python implementation against +the OpenSSL and HACL* hash functions. +""" + import binascii import functools import hmac import hashlib import random -import test.support.hashlib_helper as hashlib_helper import types import unittest -import unittest.mock as mock import warnings from _operator import _compare_digest as operator_compare_digest +from test.support import _4G, bigmemtest from test.support import check_disallow_instantiation -from test.support.import_helper import import_fresh_module, import_module +from test.support import hashlib_helper, import_helper +from test.support.hashlib_helper import ( + BuiltinHashFunctionsTrait, + HashFunctionsTrait, + NamedHashFunctionsTrait, + OpenSSLHashFunctionsTrait, +) +from test.support.import_helper import import_fresh_module +from unittest.mock import patch try: import _hashlib @@ -136,7 +161,7 @@ def hmac_digest(self, key, msg=None, digestmod=DIGESTMOD_SENTINEL): return _call_digest_func(self.hmac.digest, key, msg, digestmod) -@hashlib_helper.requires_hashlib() +@hashlib_helper.requires_openssl_hashlib() class ThroughOpenSSLAPIMixin(CreatorMixin, DigestMixin): """Mixin delegating to _hashlib.hmac_new() and _hashlib.hmac_digest().""" @@ -382,50 +407,7 @@ class BuiltinAssertersMixin(ThroughBuiltinAPIMixin, AssertersMixin): pass -class HashFunctionsTrait: - """Trait class for 'hashfunc' in hmac_new() and hmac_digest().""" - - ALGORITHMS = [ - 'md5', 'sha1', - 'sha224', 'sha256', 'sha384', 'sha512', - 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - ] - - # By default, a missing algorithm skips the test that uses it. - _ = property(lambda self: self.skipTest("missing hash function")) - md5 = sha1 = _ - sha224 = sha256 = sha384 = sha512 = _ - sha3_224 = sha3_256 = sha3_384 = sha3_512 = _ - del _ - - -class WithOpenSSLHashFunctions(HashFunctionsTrait): - """Test a HMAC implementation with an OpenSSL-based callable 'hashfunc'.""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - - for name in cls.ALGORITHMS: - @property - @hashlib_helper.requires_openssl_hashdigest(name) - def func(self, *, __name=name): # __name needed to bind 'name' - return getattr(_hashlib, f'openssl_{__name}') - setattr(cls, name, func) - - -class WithNamedHashFunctions(HashFunctionsTrait): - """Test a HMAC implementation with a named 'hashfunc'.""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - - for name in cls.ALGORITHMS: - setattr(cls, name, name) - - -class RFCTestCaseMixin(AssertersMixin, HashFunctionsTrait): +class RFCTestCaseMixin(HashFunctionsTrait, AssertersMixin): """Test HMAC implementations against RFC 2202/4231 and NIST test vectors. - Test vectors for MD5 and SHA-1 are taken from RFC 2202. @@ -739,26 +721,83 @@ def test_sha3_512_nist(self): ) -class PyRFCTestCase(ThroughObjectMixin, PyAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, - unittest.TestCase): +class PurePythonInitHMAC(PyModuleMixin, HashFunctionsTrait): + + @classmethod + def setUpClass(cls): + super().setUpClass() + for meth in ['_init_openssl_hmac', '_init_builtin_hmac']: + fn = getattr(cls.hmac.HMAC, meth) + cm = patch.object(cls.hmac.HMAC, meth, autospec=True, wraps=fn) + cls.enterClassContext(cm) + + @classmethod + def tearDownClass(cls): + cls.hmac.HMAC._init_openssl_hmac.assert_not_called() + cls.hmac.HMAC._init_builtin_hmac.assert_not_called() + # Do not assert that HMAC._init_old() has been called as it's tricky + # to determine whether a test for a specific hash function has been + # executed or not. On regular builds, it will be called but if a + # hash function is not available, it's hard to detect for which + # test we should checj HMAC._init_old() or not. + super().tearDownClass() + + +class PyRFCOpenSSLTestCase(ThroughObjectMixin, + PyAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): """Python implementation of HMAC using hmac.HMAC(). - The underlying hash functions are OpenSSL-based. + The underlying hash functions are OpenSSL-based but + _init_old() is used instead of _init_openssl_hmac(). """ -class PyDotNewRFCTestCase(ThroughModuleAPIMixin, PyAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, - unittest.TestCase): +class PyRFCBuiltinTestCase(ThroughObjectMixin, + PyAssertersMixin, + BuiltinHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.HMAC(). + + The underlying hash functions are HACL*-based but + _init_old() is used instead of _init_builtin_hmac(). + """ + + +class PyDotNewOpenSSLRFCTestCase(ThroughModuleAPIMixin, + PyAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): """Python implementation of HMAC using hmac.new(). - The underlying hash functions are OpenSSL-based. + The underlying hash functions are OpenSSL-based but + _init_old() is used instead of _init_openssl_hmac(). + """ + + +class PyDotNewBuiltinRFCTestCase(ThroughModuleAPIMixin, + PyAssertersMixin, + BuiltinHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.new(). + + The underlying hash functions are HACL-based but + _init_old() is used instead of _init_openssl_hmac(). """ class OpenSSLRFCTestCase(OpenSSLAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, unittest.TestCase): """OpenSSL implementation of HMAC. @@ -767,7 +806,8 @@ class OpenSSLRFCTestCase(OpenSSLAssertersMixin, class BuiltinRFCTestCase(BuiltinAssertersMixin, - WithNamedHashFunctions, RFCTestCaseMixin, + NamedHashFunctionsTrait, + RFCTestCaseMixin, unittest.TestCase): """Built-in HACL* implementation of HMAC. @@ -784,12 +824,6 @@ def assert_hmac_extra_cases( self.check_hmac_hexdigest(key, msg, hexdigest, digest_size, func) -# TODO(picnixz): once we have a HACL* HMAC, we should also test the Python -# implementation of HMAC with a HACL*-based hash function. For now, we only -# test it partially via the '_sha2' module, but for completeness we could -# also test the RFC test vectors against all possible implementations. - - class DigestModTestCaseMixin(CreatorMixin, DigestMixin): """Tests for the 'digestmod' parameter for hmac_new() and hmac_digest().""" @@ -916,7 +950,11 @@ class PyConstructorTestCase(ThroughObjectMixin, PyConstructorBaseMixin, class PyModuleConstructorTestCase(ThroughModuleAPIMixin, PyConstructorBaseMixin, unittest.TestCase): - """Test the hmac.new() and hmac.digest() functions.""" + """Test the hmac.new() and hmac.digest() functions. + + Note that "self.hmac" is imported by blocking "_hashlib" and "_hmac". + For testing functions in "hmac", extend PyMiscellaneousTests instead. + """ def test_hmac_digest_digestmod_parameter(self): func = self.hmac_digest @@ -927,7 +965,7 @@ def raiser(): with self.assertRaisesRegex(RuntimeError, "custom exception"): func(b'key', b'msg', raiser) - with self.assertRaisesRegex(ValueError, 'hash type'): + with self.assertRaisesRegex(ValueError, 'unsupported hash algorithm'): func(b'key', b'msg', 'unknown') with self.assertRaisesRegex(AttributeError, 'new'): @@ -1393,7 +1431,7 @@ def test_compare_digest_func(self): self.assertIs(self.compare_digest, operator_compare_digest) -@hashlib_helper.requires_hashlib() +@hashlib_helper.requires_openssl_hashlib() class OpenSSLCompareDigestTestCase(CompareDigestMixin, unittest.TestCase): compare_digest = openssl_compare_digest @@ -1412,9 +1450,8 @@ def test_hmac_constructor_uses_builtin(self): hmac = import_fresh_module("hmac", blocked=["_hashlib"]) def watch_method(cls, name): - return mock.patch.object( - cls, name, autospec=True, wraps=getattr(cls, name) - ) + wraps = getattr(cls, name) + return patch.object(cls, name, autospec=True, wraps=wraps) with ( watch_method(hmac.HMAC, '_init_openssl_hmac') as f, @@ -1466,6 +1503,48 @@ def test_with_fallback(self): finally: cache.pop('foo') + @hashlib_helper.requires_openssl_hashdigest("md5") + @bigmemtest(size=_4G + 5, memuse=2, dry_run=False) + def test_hmac_digest_overflow_error_openssl_only(self, size): + hmac = import_fresh_module("hmac", blocked=["_hmac"]) + self.do_test_hmac_digest_overflow_error_switch_to_slow(hmac, size) + + @hashlib_helper.requires_builtin_hashdigest("md5") + @bigmemtest(size=_4G + 5, memuse=2, dry_run=False) + def test_hmac_digest_overflow_error_builtin_only(self, size): + hmac = import_fresh_module("hmac", blocked=["_hashlib"]) + self.do_test_hmac_digest_overflow_error_switch_to_slow(hmac, size) + + def do_test_hmac_digest_overflow_error_switch_to_slow(self, hmac, size): + """Check that hmac.digest() falls back to pure Python. + + The *hmac* argument implements the HMAC module interface. + The *size* argument is a large key size or message size that would + trigger an OverflowError in the C implementation(s) of hmac.digest(). + """ + + bigkey = b'K' * size + bigmsg = b'M' * size + + with patch.object(hmac, "_compute_digest_fallback") as slow: + hmac.digest(bigkey, b'm', "md5") + slow.assert_called_once() + + with patch.object(hmac, "_compute_digest_fallback") as slow: + hmac.digest(b'k', bigmsg, "md5") + slow.assert_called_once() + + @hashlib_helper.requires_hashdigest("md5", openssl=True) + @bigmemtest(size=_4G + 5, memuse=2, dry_run=False) + def test_hmac_digest_no_overflow_error_in_fallback(self, size): + hmac = import_fresh_module("hmac", blocked=["_hashlib", "_hmac"]) + + for key, msg in [(b'K' * size, b'm'), (b'k', b'M' * size)]: + with self.subTest(keysize=len(key), msgsize=len(msg)): + with patch.object(hmac, "_compute_digest_fallback") as slow: + hmac.digest(key, msg, "md5") + slow.assert_called_once() + class BuiltinMiscellaneousTests(BuiltinModuleMixin, unittest.TestCase): """HMAC-BLAKE2 is not standardized as BLAKE2 is a keyed hash function. @@ -1478,7 +1557,7 @@ class BuiltinMiscellaneousTests(BuiltinModuleMixin, unittest.TestCase): @classmethod def setUpClass(cls): super().setUpClass() - cls.blake2 = import_module("_blake2") + cls.blake2 = import_helper.import_module("_blake2") cls.blake2b = cls.blake2.blake2b cls.blake2s = cls.blake2.blake2s diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index b42a611c62c..e4eff1ea17a 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -5,14 +5,30 @@ import unittest from unittest.mock import patch +from test import support + + +SAMPLE_RCDATA = ( + '' + "" + '' + '' + '' + '\u2603' +) + +SAMPLE_RAWTEXT = SAMPLE_RCDATA + '&☺' class EventCollector(html.parser.HTMLParser): - def __init__(self, *args, **kw): + def __init__(self, *args, autocdata=False, **kw): + self.autocdata = autocdata self.events = [] self.append = self.events.append html.parser.HTMLParser.__init__(self, *args, **kw) + if autocdata: + self._set_support_cdata(False) def get_events(self): # Normalize the list of events so that buffer artefacts don't @@ -33,12 +49,16 @@ def get_events(self): def handle_starttag(self, tag, attrs): self.append(("starttag", tag, attrs)) + if self.autocdata and tag == 'svg': + self._set_support_cdata(True) def handle_startendtag(self, tag, attrs): self.append(("startendtag", tag, attrs)) def handle_endtag(self, tag): self.append(("endtag", tag)) + if self.autocdata and tag == 'svg': + self._set_support_cdata(False) # all other markup @@ -80,14 +100,22 @@ def handle_entityref(self, data): self.fail('This should never be called with convert_charrefs=True') +# The normal event collector normalizes the events in get_events, +# so we override it to return the original list of events. +class EventCollectorNoNormalize(EventCollector): + def get_events(self): + return self.events + + class TestCaseBase(unittest.TestCase): - def get_collector(self): - return EventCollector(convert_charrefs=False) + def get_collector(self, convert_charrefs=False): + return EventCollector(convert_charrefs=convert_charrefs) - def _run_check(self, source, expected_events, collector=None): + def _run_check(self, source, expected_events, + *, collector=None, convert_charrefs=False): if collector is None: - collector = self.get_collector() + collector = self.get_collector(convert_charrefs=convert_charrefs) parser = collector for s in source: parser.feed(s) @@ -101,7 +129,7 @@ def _run_check(self, source, expected_events, collector=None): def _run_check_extra(self, source, events): self._run_check(source, events, - EventCollectorExtra(convert_charrefs=False)) + collector=EventCollectorExtra(convert_charrefs=False)) class HTMLParserTestCase(TestCaseBase): @@ -160,10 +188,87 @@ def test_malformatted_charref(self): ]) def test_unclosed_entityref(self): - self._run_check("&entityref foo", [ - ("entityref", "entityref"), - ("data", " foo"), - ]) + self._run_check('> <', [('entityref', 'gt'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('> <', [('data', '> <')], convert_charrefs=True) + + self._run_check('&undefined <', + [('entityref', 'undefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('&undefined <', [('data', '&undefined <')], + convert_charrefs=True) + + self._run_check('>undefined <', + [('entityref', 'gtundefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('>undefined <', [('data', '>undefined <')], + convert_charrefs=True) + + self._run_check('& <', [('data', '& '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('& <', [('data', '& <')], convert_charrefs=True) + + def test_eof_in_entityref(self): + self._run_check('>', [('entityref', 'gt')], convert_charrefs=False) + self._run_check('>', [('data', '>')], convert_charrefs=True) + + self._run_check('&g', [('entityref', 'g')], convert_charrefs=False) + self._run_check('&g', [('data', '&g')], convert_charrefs=True) + + self._run_check('&undefined', [('entityref', 'undefined')], + convert_charrefs=False) + self._run_check('&undefined', [('data', '&undefined')], + convert_charrefs=True) + + self._run_check('>undefined', [('entityref', 'gtundefined')], + convert_charrefs=False) + self._run_check('>undefined', [('data', '>undefined')], + convert_charrefs=True) + + self._run_check('&', [('data', '&')], convert_charrefs=False) + self._run_check('&', [('data', '&')], convert_charrefs=True) + + def test_unclosed_charref(self): + self._run_check('{ <', [('charref', '123'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('{ <', [('data', '{ <')], convert_charrefs=True) + self._run_check('« <', [('charref', 'xab'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('« <', [('data', '\xab <')], convert_charrefs=True) + + self._run_check('� <', + [('charref', '123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + self._run_check('� <', + [('charref', 'x123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + + self._run_check('&# <', [('data', '&# '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&# <', [('data', '&# <')], convert_charrefs=True) + self._run_check('&#x <', [('data', '&#x '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&#x <', [('data', '&#x <')], convert_charrefs=True) + + def test_eof_in_charref(self): + self._run_check('{', [('charref', '123')], convert_charrefs=False) + self._run_check('{', [('data', '{')], convert_charrefs=True) + self._run_check('«', [('charref', 'xab')], convert_charrefs=False) + self._run_check('«', [('data', '\xab')], convert_charrefs=True) + + self._run_check('�', [('charref', '123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + self._run_check('�', [('charref', 'x123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + + self._run_check('&#', [('data', '&#')], convert_charrefs=False) + self._run_check('&#', [('data', '&#')], convert_charrefs=True) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=False) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=True) def test_bad_nesting(self): # Strangely, this *is* supposed to test that overlapping @@ -264,8 +369,7 @@ def test_get_starttag_text(self): ("starttag", "foo:bar", [("one", "1"), ("two", "2")]), ("starttag_text", s)]) - def test_cdata_content(self): - contents = [ + @support.subTests('content', [ ' ¬-an-entity-ref;', "", '

', @@ -278,60 +382,238 @@ def test_cdata_content(self): 'src="http://www.example.org/r=\'+new ' 'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'), '\n\n', - 'foo = "";', '', - # these two should be invalid according to the HTML 5 spec, - # section 8.1.2.2 - #'foo = ', - #'foo = ', - ] - elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style'] - for content in contents: - for element in elements: - element_lower = element.lower() - s = '<{element}>{content}'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)]) + ]) + def test_script_content(self, content): + s = f'' + self._run_check(s, [ + ("starttag", "script", []), + ("data", content), + ("endtag", "script"), + ]) - def test_cdata_with_closing_tags(self): + @support.subTests('content', [ + 'a::before { content: ""; }', + 'a::before { content: "¬-an-entity-ref;"; }', + 'a::before { content: ""; }', + 'a::before { content: "\u2603"; }', + ]) + def test_style_content(self, content): + s = f'' + self._run_check(s, [("starttag", "style", []), + ("data", content), + ("endtag", "style")]) + + @support.subTests('tag', ['title', 'textarea']) + def test_rcdata_content(self, tag): + source = f"<{tag}>{SAMPLE_RCDATA}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", SAMPLE_RCDATA), + ("endtag", tag), + ]) + source = f"<{tag}>&" + self._run_check(source, [ + ("starttag", tag, []), + ('entityref', 'amp'), + ("endtag", tag), + ]) + + @support.subTests('tag', + ['style', 'xmp', 'iframe', 'noembed', 'noframes', 'script']) + def test_rawtext_content(self, tag): + source = f"<{tag}>{SAMPLE_RAWTEXT}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", SAMPLE_RAWTEXT), + ("endtag", tag), + ]) + + def test_noscript_content(self): + source = f"" + # scripting=False -- normal mode + self._run_check(source, [ + ('starttag', 'noscript', []), + ('comment', ' not a comment '), + ('starttag', 'not', [('a', 'start tag')]), + ('unknown decl', 'CDATA[not a cdata'), + ('comment', 'not a bogus comment'), + ('endtag', 'not'), + ('data', '☃'), + ('entityref', 'amp'), + ('charref', '9786'), + ('endtag', 'noscript'), + ]) + # scripting=True -- RAWTEXT mode + self._run_check(source, [ + ("starttag", "noscript", []), + ("data", SAMPLE_RAWTEXT), + ("endtag", "noscript"), + ], collector=EventCollector(scripting=True)) + + def test_plaintext_content(self): + content = SAMPLE_RAWTEXT + '' # not closing + source = f"

{content}" + self._run_check(source, [ + ("starttag", "plaintext", []), + ("data", content), + ]) + + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', + 'script/', 'script foo=bar', 'script foo=">"']) + def test_script_closing_tag(self, endtag): # see issue #13358 # make sure that HTMLParser calls handle_data only once for each CDATA. - # The normal event collector normalizes the events in get_events, - # so we override it to return the original list of events. - class Collector(EventCollector): - def get_events(self): - return self.events - content = """<!-- not a comment --> &not-an-entity-ref; <a href="" /> </p><p> <span></span></style> '</script' + '>'""" - for element in [' script', 'script ', ' script ', - '\nscript', 'script\n', '\nscript\n']: - element_lower = element.lower().strip() - s = '<script>{content}</{element}>'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)], - collector=Collector(convert_charrefs=False)) + s = f'<ScrIPt>{content}</{endtag}>' + self._run_check(s, [("starttag", "script", []), + ("data", content), + ("endtag", "script")], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tag', [ + 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', + 'textarea', 'title', 'noscript', + ]) + def test_closing_tag(self, tag): + for endtag in [tag, tag.upper(), f'{tag} ', f'{tag}\n', + f'{tag}/', f'{tag} foo=bar', f'{tag} foo=">"']: + content = "<!-- not a comment --><i>Spam</i>" + s = f'<{tag.upper()}>{content}</{endtag}>' + self._run_check(s, [ + ("starttag", tag, []), + ('data', content), + ("endtag", tag), + ], collector=EventCollectorNoNormalize(convert_charrefs=False, scripting=True)) + + @support.subTests('tag', [ + 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', + 'textarea', 'title', 'noscript', + ]) + def test_invalid_closing_tag(self, tag): + content = ( + f'< /{tag}>' + f'</ {tag}>' + f'</{tag}x>' + f'</{tag}\v>' + f'</{tag}\xa0>' + ) + source = f"<{tag}>{content}</{tag}>" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ("endtag", tag), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + + @support.subTests('tag,endtag', [ + ('title', 'tıtle'), + ('style', 'ſtyle'), + ('style', 'ſtyle'), + ('style', 'style'), + ('iframe', 'ıframe'), + ('noframes', 'noframeſ'), + ('noscript', 'noſcript'), + ('noscript', 'noscrıpt'), + ('script', 'ſcript'), + ('script', 'scrıpt'), + ]) + def test_invalid_nonascii_closing_tag(self, tag, endtag): + content = f"<br></{endtag}>" + source = f"<{tag}>{content}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + source = f"<{tag}>{content}</{tag}>" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ("endtag", tag), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('</', False), + ('</s', False), + ('</script', False), + ('</script ', True), + ('</script foo=bar', True), + ('</script foo=">', True), + ]) + def test_eof_in_script(self, tail, end): + content = "a = 123" + s = f'<ScrIPt>{content}{tail}' + self._run_check(s, [("starttag", "script", []), + ("data", content if end else content + tail)], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('</', False), + ('</t', False), + ('</title', False), + ('</title ', True), + ('</title foo=bar', True), + ('</title foo=">', True), + ]) + def test_eof_in_title(self, tail, end): + s = f'<TitLe>Egg &amp; Spam{tail}' + self._run_check(s, [("starttag", "title", []), + ("data", "Egg & Spam" + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "title", []), + ('data', 'Egg '), + ('entityref', 'amp'), + ('data', ' Spam' + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=False)) def test_comments(self): html = ("<!-- I'm a valid comment -->" '<!--me too!-->' '<!------>' + '<!----->' '<!---->' + # abrupt-closing-of-empty-comment + '<!--->' + '<!-->' '<!----I have many hyphens---->' '<!-- I have a > in the middle -->' - '<!-- and I have -- in the middle! -->') + '<!-- and I have -- in the middle! -->' + '<!--incorrectly-closed-comment--!>' + '<!----!>' + '<!----!-->' + '<!---- >-->' + '<!---!>-->' + '<!--!>-->' + # nested-comment + '<!-- <!-- nested --> -->' + '<!--<!-->' + '<!--<!--!>' + ) expected = [('comment', " I'm a valid comment "), ('comment', 'me too!'), ('comment', '--'), + ('comment', '-'), + ('comment', ''), + ('comment', ''), ('comment', ''), ('comment', '--I have many hyphens--'), ('comment', ' I have a > in the middle '), - ('comment', ' and I have -- in the middle! ')] + ('comment', ' and I have -- in the middle! '), + ('comment', 'incorrectly-closed-comment'), + ('comment', ''), + ('comment', '--!'), + ('comment', '-- >'), + ('comment', '-!>'), + ('comment', '!>'), + ('comment', ' <!-- nested '), ('data', ' -->'), + ('comment', '<!'), + ('comment', '<!'), + ] self._run_check(html, expected) def test_condcoms(self): @@ -348,18 +630,16 @@ def test_convert_charrefs(self): collector = lambda: EventCollectorCharrefs() self.assertTrue(collector().convert_charrefs) charrefs = ['&quot;', '&#34;', '&#x22;', '&quot', '&#34', '&#x22'] - # check charrefs in the middle of the text/attributes - expected = [('starttag', 'a', [('href', 'foo"zar')]), - ('data', 'a"z'), ('endtag', 'a')] + # check charrefs in the middle of the text + expected = [('starttag', 'a', []), ('data', 'a"z'), ('endtag', 'a')] for charref in charrefs: - self._run_check('<a href="foo{0}zar">a{0}z</a>'.format(charref), + self._run_check('<a>a{0}z</a>'.format(charref), expected, collector=collector()) - # check charrefs at the beginning/end of the text/attributes - expected = [('data', '"'), - ('starttag', 'a', [('x', '"'), ('y', '"X'), ('z', 'X"')]), + # check charrefs at the beginning/end of the text + expected = [('data', '"'), ('starttag', 'a', []), ('data', '"'), ('endtag', 'a'), ('data', '"')] for charref in charrefs: - self._run_check('{0}<a x="{0}" y="{0}X" z="X{0}">' + self._run_check('{0}<a>' '{0}</a>{0}'.format(charref), expected, collector=collector()) # check charrefs in <script>/<style> elements @@ -382,6 +662,35 @@ def test_convert_charrefs(self): self._run_check('no charrefs here', [('data', 'no charrefs here')], collector=collector()) + def test_convert_charrefs_in_attribute_values(self): + # default value for convert_charrefs is now True + collector = lambda: EventCollectorCharrefs() + self.assertTrue(collector().convert_charrefs) + + # always unescape terminated entity refs, numeric and hex char refs: + # - regardless whether they are at start, middle, end of attribute + # - or followed by alphanumeric, non-alphanumeric, or equals char + charrefs = ['&cent;', '&#xa2;', '&#xa2', '&#162;', '&#162'] + expected = [('starttag', 'a', + [('x', '¢'), ('x', 'z¢'), ('x', '¢z'), + ('x', 'z¢z'), ('x', '¢ z'), ('x', '¢=z')]), + ('endtag', 'a')] + for charref in charrefs: + self._run_check('<a x="{0}" x="z{0}" x="{0}z" ' + ' x="z{0}z" x="{0} z" x="{0}=z"></a>' + .format(charref), expected, collector=collector()) + + # only unescape unterminated entity matches if they are not followed by + # an alphanumeric or an equals sign + charref = '&cent' + expected = [('starttag', 'a', + [('x', '¢'), ('x', 'z¢'), ('x', '&centz'), + ('x', 'z&centz'), ('x', '¢ z'), ('x', '&cent=z')]), + ('endtag', 'a')] + self._run_check('<a x="{0}" x="z{0}" x="{0}z" ' + ' x="z{0}z" x="{0} z" x="{0}=z"></a>' + .format(charref), expected, collector=collector()) + # the remaining tests were for the "tolerant" parser (which is now # the default), and check various kind of broken markup def test_tolerant_parsing(self): @@ -393,28 +702,34 @@ def test_tolerant_parsing(self): ('data', '<'), ('starttag', 'bc<', [('a', None)]), ('endtag', 'html'), - ('data', '\n<img src="URL>'), - ('comment', '/img'), - ('endtag', 'html<')]) + ('data', '\n')]) def test_starttag_junk_chars(self): + self._run_check("<", [('data', '<')]) + self._run_check("<>", [('data', '<>')]) + self._run_check("< >", [('data', '< >')]) + self._run_check("< ", [('data', '< ')]) self._run_check("</>", []) + self._run_check("<$>", [('data', '<$>')]) self._run_check("</$>", [('comment', '$')]) self._run_check("</", [('data', '</')]) - self._run_check("</a", [('data', '</a')]) + self._run_check("</a", []) + self._run_check("</ a>", [('comment', ' a')]) + self._run_check("</ a", [('comment', ' a')]) self._run_check("<a<a>", [('starttag', 'a<a', [])]) self._run_check("</a<a>", [('endtag', 'a<a')]) - self._run_check("<!", [('data', '<!')]) - self._run_check("<a", [('data', '<a')]) - self._run_check("<a foo='bar'", [('data', "<a foo='bar'")]) - self._run_check("<a foo='bar", [('data', "<a foo='bar")]) - self._run_check("<a foo='>'", [('data', "<a foo='>'")]) - self._run_check("<a foo='>", [('data', "<a foo='>")]) + self._run_check("<!", [('comment', '')]) + self._run_check("<a", []) + self._run_check("<a foo='bar'", []) + self._run_check("<a foo='bar", []) + self._run_check("<a foo='>'", []) + self._run_check("<a foo='>", []) self._run_check("<a$>", [('starttag', 'a$', [])]) self._run_check("<a$b>", [('starttag', 'a$b', [])]) self._run_check("<a$b/>", [('startendtag', 'a$b', [])]) self._run_check("<a$b >", [('starttag', 'a$b', [])]) self._run_check("<a$b />", [('startendtag', 'a$b', [])]) + self._run_check("</a$b>", [('endtag', 'a$b')]) def test_slashes_in_starttag(self): self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])]) @@ -447,6 +762,10 @@ def test_slashes_in_starttag(self): ] self._run_check(html, expected) + def test_slashes_in_endtag(self): + self._run_check('</a/>', [('endtag', 'a')]) + self._run_check('</a foo="var"/>', [('endtag', 'a')]) + def test_declaration_junk_chars(self): self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')]) @@ -481,15 +800,11 @@ def test_invalid_end_tags(self): self._run_check(html, expected) def test_broken_invalid_end_tag(self): - # This is technically wrong (the "> shouldn't be included in the 'data') - # but is probably not worth fixing it (in addition to all the cases of - # the previous test, it would require a full attribute parsing). - # see #13993 html = '<b>This</b attr=">"> confuses the parser' expected = [('starttag', 'b', []), ('data', 'This'), ('endtag', 'b'), - ('data', '"> confuses the parser')] + ('data', ' confuses the parser')] self._run_check(html, expected) def test_correct_detection_of_start_tags(self): @@ -525,69 +840,165 @@ def test_correct_detection_of_start_tags(self): ] self._run_check(html, expected) - def test_EOF_in_charref(self): - # see #17802 - # This test checks that the UnboundLocalError reported in the issue - # is not raised, however I'm not sure the returned values are correct. - # Maybe HTMLParser should use self.unescape for these + def test_eof_in_comments(self): data = [ - ('a&', [('data', 'a&')]), - ('a&b', [('data', 'ab')]), - ('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]), - ('a&b;', [('data', 'a'), ('entityref', 'b')]), + ('<!--', [('comment', '')]), + ('<!---', [('comment', '')]), + ('<!----', [('comment', '')]), + ('<!-----', [('comment', '-')]), + ('<!------', [('comment', '--')]), + ('<!----!', [('comment', '')]), + ('<!---!', [('comment', '-!')]), + ('<!---!>', [('comment', '-!>')]), + ('<!--foo', [('comment', 'foo')]), + ('<!--foo-', [('comment', 'foo')]), + ('<!--foo--', [('comment', 'foo')]), + ('<!--foo--!', [('comment', 'foo')]), + ('<!--<!--', [('comment', '<!')]), + ('<!--<!--!', [('comment', '<!')]), ] for html, expected in data: self._run_check(html, expected) - def test_broken_comments(self): - html = ('<! not really a comment >' + def test_eof_in_declarations(self): + data = [ + ('<!', [('comment', '')]), + ('<!-', [('comment', '-')]), + ('<![', [('comment', '[')]), + ('<!DOCTYPE', [('decl', 'DOCTYPE')]), + ('<!DOCTYPE ', [('decl', 'DOCTYPE ')]), + ('<!DOCTYPE html', [('decl', 'DOCTYPE html')]), + ('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]), + ('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]), + ('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]), + ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo', + [('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]), + ] + for html, expected in data: + self._run_check(html, expected) + + @support.subTests('content', ['', 'x', 'x]', 'x]]']) + def test_eof_in_cdata(self, content): + self._run_check('<![CDATA[' + content, + [('unknown decl', 'CDATA[' + content)]) + self._run_check('<![CDATA[' + content, + [('comment', '[CDATA[' + content)], + collector=EventCollector(autocdata=True)) + self._run_check('<svg><text y="100"><![CDATA[' + content, + [('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content)]) + + def test_bogus_comments(self): + html = ('<!ELEMENT br EMPTY>' + '<! not really a comment >' '<! not a comment either -->' '<! -- close enough -->' '<!><!<-- this was an empty comment>' - '<!!! another bogus comment !!!>') + '<!!! another bogus comment !!!>' + # see #32876 + '<![with square brackets]!>' + '<![\nmultiline\nbogusness\n]!>' + '<![more brackets]-[and a hyphen]!>' + '<![cdata[should be uppercase]]>' + '<![CDATA [whitespaces are not ignored]]>' + '<![CDATA]]>' # required '[' after CDATA + ) expected = [ + ('comment', 'ELEMENT br EMPTY'), ('comment', ' not really a comment '), ('comment', ' not a comment either --'), ('comment', ' -- close enough --'), ('comment', ''), ('comment', '<-- this was an empty comment'), ('comment', '!! another bogus comment !!!'), + ('comment', '[with square brackets]!'), + ('comment', '[\nmultiline\nbogusness\n]!'), + ('comment', '[more brackets]-[and a hyphen]!'), + ('comment', '[cdata[should be uppercase]]'), + ('comment', '[CDATA [whitespaces are not ignored]]'), + ('comment', '[CDATA]]'), ] self._run_check(html, expected) def test_broken_condcoms(self): # these condcoms are missing the '--' after '<!' and before the '>' + # and they are considered bogus comments according to + # "8.2.4.42. Markup declaration open state" html = ('<![if !(IE)]>broken condcom<![endif]>' '<![if ! IE]><link href="favicon.tiff"/><![endif]>' '<![if !IE 6]><img src="firefox.png" /><![endif]>' '<![if !ie 6]><b>foo</b><![endif]>' '<![if (!IE)|(lt IE 9)]><img src="mammoth.bmp" /><![endif]>') - # According to the HTML5 specs sections "8.2.4.44 Bogus comment state" - # and "8.2.4.45 Markup declaration open state", comment tokens should - # be emitted instead of 'unknown decl', but calling unknown_decl - # provides more flexibility. - # See also Lib/_markupbase.py:parse_declaration expected = [ - ('unknown decl', 'if !(IE)'), + ('comment', '[if !(IE)]'), ('data', 'broken condcom'), - ('unknown decl', 'endif'), - ('unknown decl', 'if ! IE'), + ('comment', '[endif]'), + ('comment', '[if ! IE]'), ('startendtag', 'link', [('href', 'favicon.tiff')]), - ('unknown decl', 'endif'), - ('unknown decl', 'if !IE 6'), + ('comment', '[endif]'), + ('comment', '[if !IE 6]'), ('startendtag', 'img', [('src', 'firefox.png')]), - ('unknown decl', 'endif'), - ('unknown decl', 'if !ie 6'), + ('comment', '[endif]'), + ('comment', '[if !ie 6]'), ('starttag', 'b', []), ('data', 'foo'), ('endtag', 'b'), - ('unknown decl', 'endif'), - ('unknown decl', 'if (!IE)|(lt IE 9)'), + ('comment', '[endif]'), + ('comment', '[if (!IE)|(lt IE 9)]'), ('startendtag', 'img', [('src', 'mammoth.bmp')]), - ('unknown decl', 'endif') + ('comment', '[endif]') ] self._run_check(html, expected) + @support.subTests('content', [ + 'just some plain text', + '<!-- not a comment -->', + '&not-an-entity-ref;', + "<not a='start tag'>", + '', + '[[I have many brackets]]', + 'I have a > in the middle', + 'I have a ]] in the middle', + '] ]>', + ']] >', + ('\n' + ' if (a < b && a > b) {\n' + ' printf("[<marquee>How?</marquee>]");\n' + ' }\n'), + ]) + def test_cdata_section_content(self, content): + # See "13.2.5.42 Markup declaration open state", + # "13.2.5.69 CDATA section state", and issue bpo-32876. + html = f'<svg><text y="100"><![CDATA[{content}]]></text></svg>' + expected = [ + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content), + ('endtag', 'text'), + ('endtag', 'svg'), + ] + self._run_check(html, expected) + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + + def test_cdata_section(self): + # See "13.2.5.42 Markup declaration open state". + html = ('<![CDATA[foo<br>bar]]>' + '<svg><text y="100"><![CDATA[foo<br>bar]]></text></svg>' + '<![CDATA[foo<br>bar]]>') + expected = [ + ('comment', '[CDATA[foo<br'), + ('data', 'bar]]>'), + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[foo<br>bar'), + ('endtag', 'text'), + ('endtag', 'svg'), + ('comment', '[CDATA[foo<br'), + ('data', 'bar]]>'), + ] + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + def test_convert_charrefs_dropped_text(self): # #23144: make sure that all the events are triggered when # convert_charrefs is True, even if we don't call .close() @@ -600,6 +1011,26 @@ def test_convert_charrefs_dropped_text(self): ('endtag', 'a'), ('data', ' bar & baz')] ) + @support.requires_resource('cpu') + def test_eof_no_quadratic_complexity(self): + # Each of these examples used to take about an hour. + # Now they take a fraction of a second. + def check(source): + parser = html.parser.HTMLParser() + parser.feed(source) + parser.close() + n = 120_000 + check("<a " * n) + check("<a a=" * n) + check("</a " * 14 * n) + check("</a a=" * 11 * n) + check("<!--" * 4 * n) + check("<!" * 60 * n) + check("<?" * 19 * n) + check("</$" * 15 * n) + check("<![CDATA[" * 9 * n) + check("<!doctype" * 35 * n) + class AttributesTestCase(TestCaseBase): @@ -608,9 +1039,15 @@ def test_attr_syntax(self): ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) ] self._run_check("""<a b='v' c="v" d=v e>""", output) - self._run_check("""<a b = 'v' c = "v" d = v e>""", output) - self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output) - self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output) + self._run_check("<a foo==bar>", [('starttag', 'a', [('foo', '=bar')])]) + self._run_check("<a foo =bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\t=bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\v=bar>", [('starttag', 'a', [('foo\v', 'bar')])]) + self._run_check("<a foo\xa0=bar>", [('starttag', 'a', [('foo\xa0', 'bar')])]) + self._run_check("<a foo= bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\tbar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\vbar>", [('starttag', 'a', [('foo', '\vbar')])]) + self._run_check("<a foo=\xa0bar>", [('starttag', 'a', [('foo', '\xa0bar')])]) def test_attr_values(self): self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""", @@ -619,6 +1056,10 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""<a b='' c="">""", [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("<a b=\tx c=\ny>", + [('starttag', 'a', [('b', 'x'), ('c', 'y')])]) + self._run_check("<a b=\v c=\xa0>", + [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. self._run_check("<e a=rgb(1,2,3)>", [("starttag", "e", [("a", "rgb(1,2,3)")])]) @@ -685,13 +1126,17 @@ def test_malformed_attributes(self): ) expected = [ ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), - ('data', 'test - bad1'), ('endtag', 'a'), + ('data', 'test - bad1'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), - ('data', 'test - bad2'), ('endtag', 'a'), + ('data', 'test - bad2'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), - ('data', 'test - bad3'), ('endtag', 'a'), + ('data', 'test - bad3'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), - ('data', 'test - bad4'), ('endtag', 'a') + ('data', 'test - bad4'), + ('endtag', 'a'), ] self._run_check(html, expected) diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 6bc33b15ec3..04cb440cd4c 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -4,6 +4,7 @@ import stat import sys import re +from test import support from test.support import os_helper from test.support import warnings_helper import time @@ -105,8 +106,7 @@ def test_http2time_formats(self): self.assertEqual(http2time(s.lower()), test_t, s.lower()) self.assertEqual(http2time(s.upper()), test_t, s.upper()) - def test_http2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Mandag 16. September 1996', @@ -121,10 +121,9 @@ def test_http2time_garbage(self): '08-01-3697739', '09 Feb 19942632 22:23:32 GMT', 'Wed, 09 Feb 1994834 22:23:32 GMT', - ]: - self.assertIsNone(http2time(test), - "http2time(%s) is not None\n" - "http2time(test) %s" % (test, http2time(test))) + ]) + def test_http2time_garbage(self, test): + self.assertIsNone(http2time(test)) def test_http2time_redos_regression_actually_completes(self): # LOOSE_HTTP_DATE_RE was vulnerable to malicious input which caused catastrophic backtracking (REDoS). @@ -149,9 +148,7 @@ def parse_date(text): self.assertEqual(parse_date("1994-02-03 19:45:29 +0530"), (1994, 2, 3, 14, 15, 29)) - def test_iso2time_formats(self): - # test iso2time for supported dates. - tests = [ + @support.subTests('s', [ '1994-02-03 00:00:00 -0000', # ISO 8601 format '1994-02-03 00:00:00 +0000', # ISO 8601 format '1994-02-03 00:00:00', # zone is optional @@ -164,16 +161,15 @@ def test_iso2time_formats(self): # A few tests with extra space at various places ' 1994-02-03 ', ' 1994-02-03T00:00:00 ', - ] - + ]) + def test_iso2time_formats(self, s): + # test iso2time for supported dates. test_t = 760233600 # assume broken POSIX counting of seconds - for s in tests: - self.assertEqual(iso2time(s), test_t, s) - self.assertEqual(iso2time(s.lower()), test_t, s.lower()) - self.assertEqual(iso2time(s.upper()), test_t, s.upper()) + self.assertEqual(iso2time(s), test_t, s) + self.assertEqual(iso2time(s.lower()), test_t, s.lower()) + self.assertEqual(iso2time(s.upper()), test_t, s.upper()) - def test_iso2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Thursday, 03-Feb-94 00:00:00 GMT', @@ -186,9 +182,9 @@ def test_iso2time_garbage(self): '01-01-1980 00:00:62', '01-01-1980T00:00:62', '19800101T250000Z', - ]: - self.assertIsNone(iso2time(test), - "iso2time(%r)" % test) + ]) + def test_iso2time_garbage(self, test): + self.assertIsNone(iso2time(test)) def test_iso2time_performance_regression(self): # If ISO_DATE_RE regresses to quadratic complexity, this test will take a very long time to succeed. @@ -199,24 +195,23 @@ def test_iso2time_performance_regression(self): class HeaderTests(unittest.TestCase): - def test_parse_ns_headers(self): - # quotes should be stripped - expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] - for hdr in [ + @support.subTests('hdr', [ 'foo=bar; expires=01 Jan 2040 22:23:32 GMT', 'foo=bar; expires="01 Jan 2040 22:23:32 GMT"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) - - def test_parse_ns_headers_version(self): - + ]) + def test_parse_ns_headers(self, hdr): # quotes should be stripped - expected = [[('foo', 'bar'), ('version', '1')]] - for hdr in [ + expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] + self.assertEqual(parse_ns_headers([hdr]), expected) + + @support.subTests('hdr', [ 'foo=bar; version="1"', 'foo=bar; Version="1"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) + ]) + def test_parse_ns_headers_version(self, hdr): + # quotes should be stripped + expected = [[('foo', 'bar'), ('version', '1')]] + self.assertEqual(parse_ns_headers([hdr]), expected) def test_parse_ns_headers_special_names(self): # names such as 'expires' are not special in first name=value pair @@ -226,8 +221,7 @@ def test_parse_ns_headers_special_names(self): expected = [[("expires", "01 Jan 2040 22:23:32 GMT"), ("version", "0")]] self.assertEqual(parse_ns_headers([hdr]), expected) - def test_join_header_words(self): - for src, expected in [ + @support.subTests('src,expected', [ ([[("foo", None), ("bar", "baz")]], "foo; bar=baz"), (([]), ""), (([[]]), ""), @@ -237,12 +231,11 @@ def test_join_header_words(self): 'n; foo="foo;_", bar=foo_bar'), ([[("n", "m"), ("foo", None)], [("bar", "foo_bar")]], 'n=m; foo, bar=foo_bar'), - ]: - with self.subTest(src=src): - self.assertEqual(join_header_words(src), expected) + ]) + def test_join_header_words(self, src, expected): + self.assertEqual(join_header_words(src), expected) - def test_split_header_words(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", [[("foo", None)]]), ("foo=bar", [[("foo", "bar")]]), (" foo ", [[("foo", None)]]), @@ -259,24 +252,22 @@ def test_split_header_words(self): (r'foo; bar=baz, spam=, foo="\,\;\"", bar= ', [[("foo", None), ("bar", "baz")], [("spam", "")], [("foo", ',;"')], [("bar", "")]]), - ] - - for arg, expect in tests: - try: - result = split_header_words([arg]) - except: - import traceback, io - f = io.StringIO() - traceback.print_exc(None, f) - result = "(error -- traceback follows)\n\n%s" % f.getvalue() - self.assertEqual(result, expect, """ + ]) + def test_split_header_words(self, arg, expect): + try: + result = split_header_words([arg]) + except: + import traceback, io + f = io.StringIO() + traceback.print_exc(None, f) + result = "(error -- traceback follows)\n\n%s" % f.getvalue() + self.assertEqual(result, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' """ % (arg, expect, result)) - def test_roundtrip(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", "foo"), ("foo=bar", "foo=bar"), (" foo ", "foo"), @@ -309,12 +300,11 @@ def test_roundtrip(self): ('n; foo="foo;_", bar="foo,_"', 'n; foo="foo;_", bar="foo,_"'), - ] - - for arg, expect in tests: - input = split_header_words([arg]) - res = join_header_words(input) - self.assertEqual(res, expect, """ + ]) + def test_roundtrip(self, arg, expect): + input = split_header_words([arg]) + res = join_header_words(input) + self.assertEqual(res, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' @@ -516,14 +506,7 @@ class CookieTests(unittest.TestCase): ## just the 7 special TLD's listed in their spec. And folks rely on ## that... - def test_domain_return_ok(self): - # test optimization: .domain_return_ok() should filter out most - # domains in the CookieJar before we try to access them (because that - # may require disk access -- in particular, with MSIECookieJar) - # This is only a rough check for performance reasons, so it's not too - # critical as long as it's sufficiently liberal. - pol = DefaultCookiePolicy() - for url, domain, ok in [ + @support.subTests('url,domain,ok', [ ("http://foo.bar.com/", "blah.com", False), ("http://foo.bar.com/", "rhubarb.blah.com", False), ("http://foo.bar.com/", "rhubarb.foo.bar.com", False), @@ -543,11 +526,18 @@ def test_domain_return_ok(self): ("http://foo/", ".local", True), ("http://barfoo.com", ".foo.com", False), ("http://barfoo.com", "foo.com", False), - ]: - request = urllib.request.Request(url) - r = pol.domain_return_ok(domain, request) - if ok: self.assertTrue(r) - else: self.assertFalse(r) + ]) + def test_domain_return_ok(self, url, domain, ok): + # test optimization: .domain_return_ok() should filter out most + # domains in the CookieJar before we try to access them (because that + # may require disk access -- in particular, with MSIECookieJar) + # This is only a rough check for performance reasons, so it's not too + # critical as long as it's sufficiently liberal. + pol = DefaultCookiePolicy() + request = urllib.request.Request(url) + r = pol.domain_return_ok(domain, request) + if ok: self.assertTrue(r) + else: self.assertFalse(r) def test_missing_value(self): # missing = sign in Cookie: header is regarded by Mozilla as a missing @@ -581,10 +571,7 @@ def test_missing_value(self): self.assertEqual(interact_netscape(c, "http://www.acme.com/foo/"), '"spam"; eggs') - def test_rfc2109_handling(self): - # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, - # dependent on policy settings - for rfc2109_as_netscape, rfc2965, version in [ + @support.subTests('rfc2109_as_netscape,rfc2965,version', [ # default according to rfc2965 if not explicitly specified (None, False, 0), (None, True, 1), @@ -593,24 +580,27 @@ def test_rfc2109_handling(self): (False, True, 1), (True, False, 0), (True, True, 0), - ]: - policy = DefaultCookiePolicy( - rfc2109_as_netscape=rfc2109_as_netscape, - rfc2965=rfc2965) - c = CookieJar(policy) - interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") - try: - cookie = c._cookies["www.example.com"]["/"]["ni"] - except KeyError: - self.assertIsNone(version) # didn't expect a stored cookie - else: - self.assertEqual(cookie.version, version) - # 2965 cookies are unaffected - interact_2965(c, "http://www.example.com/", - "foo=bar; Version=1") - if rfc2965: - cookie2965 = c._cookies["www.example.com"]["/"]["foo"] - self.assertEqual(cookie2965.version, 1) + ]) + def test_rfc2109_handling(self, rfc2109_as_netscape, rfc2965, version): + # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, + # dependent on policy settings + policy = DefaultCookiePolicy( + rfc2109_as_netscape=rfc2109_as_netscape, + rfc2965=rfc2965) + c = CookieJar(policy) + interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") + try: + cookie = c._cookies["www.example.com"]["/"]["ni"] + except KeyError: + self.assertIsNone(version) # didn't expect a stored cookie + else: + self.assertEqual(cookie.version, version) + # 2965 cookies are unaffected + interact_2965(c, "http://www.example.com/", + "foo=bar; Version=1") + if rfc2965: + cookie2965 = c._cookies["www.example.com"]["/"]["foo"] + self.assertEqual(cookie2965.version, 1) def test_ns_parser(self): c = CookieJar() @@ -778,8 +768,7 @@ def test_default_path_with_query(self): # Cookie is sent back to the same URI. self.assertEqual(interact_netscape(cj, uri), value) - def test_escape_path(self): - cases = [ + @support.subTests('arg,result', [ # quoted safe ("/foo%2f/bar", "/foo%2F/bar"), ("/foo%2F/bar", "/foo%2F/bar"), @@ -799,9 +788,9 @@ def test_escape_path(self): ("/foo/bar\u00fc", "/foo/bar%C3%BC"), # UTF-8 encoded # unicode ("/foo/bar\uabcd", "/foo/bar%EA%AF%8D"), # UTF-8 encoded - ] - for arg, result in cases: - self.assertEqual(escape_path(arg), result) + ]) + def test_escape_path(self, arg, result): + self.assertEqual(escape_path(arg), result) def test_request_path(self): # with parameters diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 2fbc142de2f..c2ed30831b2 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -48,6 +48,29 @@ def test_basic(self): 'Set-Cookie: d=r', 'Set-Cookie: f=h' )) + }, + + # gh-92936: allow double quote in cookie values + { + 'data': 'cookie="{"key": "value"}"', + 'dict': {'cookie': '{"key": "value"}'}, + 'repr': "<SimpleCookie: cookie='{\"key\": \"value\"}'>", + 'output': 'Set-Cookie: cookie="{"key": "value"}"', + }, + { + 'data': 'key="some value; surrounded by quotes"', + 'dict': {'key': 'some value; surrounded by quotes'}, + 'repr': "<SimpleCookie: key='some value; surrounded by quotes'>", + 'output': 'Set-Cookie: key="some value; surrounded by quotes"', + }, + { + 'data': 'session="user123"; preferences="{"theme": "dark"}"', + 'dict': {'session': 'user123', 'preferences': '{"theme": "dark"}'}, + 'repr': "<SimpleCookie: preferences='{\"theme\": \"dark\"}' session='user123'>", + 'output': '\n'.join(( + 'Set-Cookie: preferences="{"theme": "dark"}"', + 'Set-Cookie: session="user123"', + )) } ] diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 38429ad480f..44044d0385c 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -386,6 +386,52 @@ def test_headers_debuglevel(self): self.assertEqual(lines[2], "header: Second: val1") self.assertEqual(lines[3], "header: Second: val2") + def test_max_response_headers(self): + max_headers = client._MAXHEADERS + 20 + headers = [f"Name{i}: Value{i}".encode() for i in range(max_headers)] + body = b"HTTP/1.1 200 OK\r\n" + b"\r\n".join(headers) + + with self.subTest(max_headers=None): + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + with self.assertRaisesRegex( + client.HTTPException, f"got more than 100 headers" + ): + resp.begin() + + with self.subTest(max_headers=max_headers): + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin(_max_headers=max_headers) + + def test_max_connection_headers(self): + max_headers = client._MAXHEADERS + 20 + headers = ( + f"Name{i}: Value{i}".encode() for i in range(max_headers - 1) + ) + body = ( + b"HTTP/1.1 200 OK\r\n" + + b"\r\n".join(headers) + + b"\r\nContent-Length: 12\r\n\r\nDummy body\r\n" + ) + + with self.subTest(max_headers=None): + conn = client.HTTPConnection("example.com") + conn.sock = FakeSocket(body) + conn.request("GET", "/") + with self.assertRaisesRegex( + client.HTTPException, f"got more than {client._MAXHEADERS} headers" + ): + response = conn.getresponse() + + with self.subTest(max_headers=None): + conn = client.HTTPConnection( + "example.com", max_response_headers=max_headers + ) + conn.sock = FakeSocket(body) + conn.request("GET", "/") + response = conn.getresponse() + response.read() class HttpMethodTests(TestCase): def test_invalid_method_names(self): @@ -1465,6 +1511,72 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_large_content_length(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + [conn, address] = serv.accept() + with conn: + while conn.recv(1024): + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" % size) + conn.sendall(b'A' * (size//3)) + conn.sendall(b'B' * (size - size//3)) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(15, 27): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertEqual(len(response.read()), size) + finally: + conn.close() + thread.join(1.0) + + def test_large_content_length_truncated(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + while True: + [conn, address] = serv.accept() + with conn: + conn.recv(1024) + if not size: + break + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"Text" % size) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(18, 65): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertRaises(client.IncompleteRead, response.read) + conn.close() + finally: + conn.close() + size = 0 + conn.request("GET", "/") + conn.close() + thread.join(1.0) + def test_putrequest_override_domain_validation(self): """ It should be possible to override the default validation diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 2cafa4e45a1..7da5e3a1957 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -3,16 +3,16 @@ Written by Cody A.W. Somerville <cody-somerville@ubuntu.com>, Josip Dzolonga, and Michael Otteneder for the 2007/08 GHOP contest. """ -from collections import OrderedDict + from http.server import BaseHTTPRequestHandler, HTTPServer, HTTPSServer, \ - SimpleHTTPRequestHandler, CGIHTTPRequestHandler + SimpleHTTPRequestHandler from http import server, HTTPStatus +import contextlib import os import socket import sys import re -import base64 import ntpath import pathlib import shutil @@ -21,6 +21,7 @@ import html import http, http.client import urllib.parse +import urllib.request import tempfile import time import datetime @@ -31,8 +32,10 @@ import unittest from test import support from test.support import ( - is_apple, import_helper, os_helper, requires_subprocess, threading_helper + is_apple, import_helper, os_helper, threading_helper ) +from test.support.script_helper import kill_python, spawn_python +from test.support.socket_helper import find_unused_port try: import ssl @@ -359,6 +362,44 @@ def test_head_via_send_error(self): self.assertEqual(b'', data) +class HTTP09ServerTestCase(BaseTestCase): + + class request_handler(NoLogRequestHandler, BaseHTTPRequestHandler): + """Request handler for HTTP/0.9 server.""" + + def do_GET(self): + self.wfile.write(f'OK: here is {self.path}\r\n'.encode()) + + def setUp(self): + super().setUp() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock = self.enterContext(self.sock) + self.sock.connect((self.HOST, self.PORT)) + + def test_simple_get(self): + self.sock.send(b'GET /index.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /index.html\r\n") + + def test_invalid_request(self): + self.sock.send(b'POST /index.html\r\n') + res = self.sock.recv(1024) + self.assertIn(b"Bad HTTP/0.9 request type ('POST')", res) + + def test_single_request(self): + self.sock.send(b'GET /foo.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /foo.html\r\n") + + # Ignore errors if the connection is already closed, + # as this is the expected behavior of HTTP/0.9. + with contextlib.suppress(OSError): + self.sock.send(b'GET /bar.html\r\n') + res = self.sock.recv(1024) + # The server should not process our request. + self.assertEqual(res, b'') + + def certdata_file(*path): return os.path.join(os.path.dirname(__file__), "certdata", *path) @@ -522,42 +563,120 @@ def close_conn(): reader.close() return body + def check_list_dir_dirname(self, dirname, quotedname=None): + fullpath = os.path.join(self.tempdir, dirname) + try: + os.mkdir(os.path.join(self.tempdir, dirname)) + except (OSError, UnicodeEncodeError): + self.skipTest(f'Can not create directory {dirname!a} ' + f'on current file system') + + if quotedname is None: + quotedname = urllib.parse.quote(dirname, errors='surrogatepass') + response = self.request(self.base_url + '/' + quotedname + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + displaypath = html.escape(f'{self.base_url}/{dirname}/', quote=False) + enc = sys.getfilesystemencoding() + prefix = f'listing for {displaypath}</'.encode(enc, 'surrogateescape') + self.assertIn(prefix + b'title>', body) + self.assertIn(prefix + b'h1>', body) + + def check_list_dir_filename(self, filename): + fullpath = os.path.join(self.tempdir, filename) + content = ascii(fullpath).encode() + (os_helper.TESTFN_UNDECODABLE or b'\xff') + try: + with open(fullpath, 'wb') as f: + f.write(content) + except OSError: + self.skipTest(f'Can not create file {filename!a} ' + f'on current file system') + + response = self.request(self.base_url + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + quotedname = urllib.parse.quote(filename, errors='surrogatepass') + enc = response.headers.get_content_charset() + self.assertIsNotNone(enc) + self.assertIn((f'href="{quotedname}"').encode('ascii'), body) + displayname = html.escape(filename, quote=False) + self.assertIn(f'>{displayname}<'.encode(enc, 'surrogateescape'), body) + + response = self.request(self.base_url + '/' + quotedname) + self.check_status_and_reason(response, HTTPStatus.OK, data=content) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_dirname(self): + dirname = os_helper.TESTFN_NONASCII + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_filename(self): + filename = os_helper.TESTFN_NONASCII + '.txt' + self.check_list_dir_filename(filename) + @unittest.skipIf(is_apple, 'undecodable name cannot always be decoded on Apple platforms') @unittest.skipIf(sys.platform == 'win32', 'undecodable name cannot be decoded on win32') @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, 'need os_helper.TESTFN_UNDECODABLE') - def test_undecodable_filename(self): - enc = sys.getfilesystemencoding() - filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' - with open(os.path.join(self.tempdir, filename), 'wb') as f: - f.write(os_helper.TESTFN_UNDECODABLE) - response = self.request(self.base_url + '/') - if is_apple: - # On Apple platforms the HFS+ filesystem replaces bytes that - # aren't valid UTF-8 into a percent-encoded value. - for name in os.listdir(self.tempdir): - if name != 'test': # Ignore a filename created in setUp(). - filename = name - break - body = self.check_status_and_reason(response, HTTPStatus.OK) - quotedname = urllib.parse.quote(filename, errors='surrogatepass') - self.assertIn(('href="%s"' % quotedname) - .encode(enc, 'surrogateescape'), body) - self.assertIn(('>%s<' % html.escape(filename, quote=False)) - .encode(enc, 'surrogateescape'), body) - response = self.request(self.base_url + '/' + quotedname) - self.check_status_and_reason(response, HTTPStatus.OK, - data=os_helper.TESTFN_UNDECODABLE) + def test_list_dir_undecodable_dirname(self): + dirname = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.dir' + self.check_list_dir_dirname(dirname) - def test_undecodable_parameter(self): - # sanity check using a valid parameter + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') + @unittest.skipIf(sys.platform == 'win32', + 'undecodable name cannot be decoded on win32') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'need os_helper.TESTFN_UNDECODABLE') + def test_list_dir_undecodable_filename(self): + filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_undecodable_dirname2(self): + dirname = '\ufffd.dir' + self.check_list_dir_dirname(dirname, quotedname='%ff.dir') + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_dirname(self): + dirname = os_helper.TESTFN_UNENCODABLE + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_filename(self): + filename = os_helper.TESTFN_UNENCODABLE + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_escape_dirname(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&amp;', '&amp', '<i>', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + dirname = name + '.dir' + self.check_list_dir_dirname(dirname, + quotedname=urllib.parse.quote(dirname, safe='&<>\'"')) + + def test_list_dir_escape_filename(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&amp;', '&amp', '<i>', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + filename = name + '.txt' + self.check_list_dir_filename(filename) + os_helper.unlink(os.path.join(self.tempdir, filename)) + + def test_list_dir_with_query_and_fragment(self): + prefix = f'listing for {self.base_url}/</'.encode('latin1') + response = self.request(self.base_url + '/#123').read() + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) response = self.request(self.base_url + '/?x=123').read() - self.assertRegex(response, rf'listing for {self.base_url}/\?x=123'.encode('latin1')) - # now the bogus encoding - response = self.request(self.base_url + '/?x=%bb').read() - self.assertRegex(response, rf'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) def test_get_dir_redirect_location_domain_injection_bug(self): """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. @@ -615,10 +734,19 @@ def test_get(self): # check for trailing "/" which should return 404. See Issue17324 response = self.request(self.base_url + '/test/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2f') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2F') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request(self.base_url + '/') self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2f') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2F') + self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.base_url) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), self.base_url + "/") self.assertEqual(response.getheader("Content-Length"), "0") response = self.request(self.base_url + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) @@ -724,6 +852,8 @@ def test_path_without_leading_slash(self): self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), + self.tempdir_name + "/") response = self.request(self.tempdir_name + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name + '?hi=1') @@ -731,350 +861,6 @@ def test_path_without_leading_slash(self): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") - def test_html_escape_filename(self): - filename = '<test&>.txt' - fullpath = os.path.join(self.tempdir, filename) - - try: - open(fullpath, 'wb').close() - except OSError: - raise unittest.SkipTest('Can not create file %s on current file ' - 'system' % filename) - - try: - response = self.request(self.base_url + '/') - body = self.check_status_and_reason(response, HTTPStatus.OK) - enc = response.headers.get_content_charset() - finally: - os.unlink(fullpath) # avoid affecting test_undecodable_filename - - self.assertIsNotNone(enc) - html_text = '>%s<' % html.escape(filename, quote=False) - self.assertIn(html_text.encode(enc), body) - - -cgi_file1 = """\ -#!%s - -print("Content-type: text/html") -print() -print("Hello World") -""" - -cgi_file2 = """\ -#!%s -import os -import sys -import urllib.parse - -print("Content-type: text/html") -print() - -content_length = int(os.environ["CONTENT_LENGTH"]) -query_string = sys.stdin.buffer.read(content_length) -params = {key.decode("utf-8"): val.decode("utf-8") - for key, val in urllib.parse.parse_qsl(query_string)} - -print("%%s, %%s, %%s" %% (params["spam"], params["eggs"], params["bacon"])) -""" - -cgi_file4 = """\ -#!%s -import os - -print("Content-type: text/html") -print() - -print(os.environ["%s"]) -""" - -cgi_file6 = """\ -#!%s -import os - -print("X-ambv: was here") -print("Content-type: text/html") -print() -print("<pre>") -for k, v in os.environ.items(): - try: - k.encode('ascii') - v.encode('ascii') - except UnicodeEncodeError: - continue # see: BPO-44647 - print(f"{k}={v}") -print("</pre>") -""" - - -@unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, - "This test can't be run reliably as root (issue #13308).") -@requires_subprocess() -class CGIHTTPServerTestCase(BaseTestCase): - class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): - _test_case_self = None # populated by each setUp() method call. - - def __init__(self, *args, **kwargs): - with self._test_case_self.assertWarnsRegex( - DeprecationWarning, - r'http\.server\.CGIHTTPRequestHandler'): - # This context also happens to catch and silence the - # threading DeprecationWarning from os.fork(). - super().__init__(*args, **kwargs) - - linesep = os.linesep.encode('ascii') - - def setUp(self): - self.request_handler._test_case_self = self # practical, but yuck. - BaseTestCase.setUp(self) - self.cwd = os.getcwd() - self.parent_dir = tempfile.mkdtemp() - self.cgi_dir = os.path.join(self.parent_dir, 'cgi-bin') - self.cgi_child_dir = os.path.join(self.cgi_dir, 'child-dir') - self.sub_dir_1 = os.path.join(self.parent_dir, 'sub') - self.sub_dir_2 = os.path.join(self.sub_dir_1, 'dir') - self.cgi_dir_in_sub_dir = os.path.join(self.sub_dir_2, 'cgi-bin') - os.mkdir(self.cgi_dir) - os.mkdir(self.cgi_child_dir) - os.mkdir(self.sub_dir_1) - os.mkdir(self.sub_dir_2) - os.mkdir(self.cgi_dir_in_sub_dir) - self.nocgi_path = None - self.file1_path = None - self.file2_path = None - self.file3_path = None - self.file4_path = None - self.file5_path = None - - # The shebang line should be pure ASCII: use symlink if possible. - # See issue #7668. - self._pythonexe_symlink = None - if os_helper.can_symlink(): - self.pythonexe = os.path.join(self.parent_dir, 'python') - self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__() - else: - self.pythonexe = sys.executable - - try: - # The python executable path is written as the first line of the - # CGI Python script. The encoding cookie cannot be used, and so the - # path should be encodable to the default script encoding (utf-8) - self.pythonexe.encode('utf-8') - except UnicodeEncodeError: - self.tearDown() - self.skipTest("Python executable path is not encodable to utf-8") - - self.nocgi_path = os.path.join(self.parent_dir, 'nocgi.py') - with open(self.nocgi_path, 'w', encoding='utf-8') as fp: - fp.write(cgi_file1 % self.pythonexe) - os.chmod(self.nocgi_path, 0o777) - - self.file1_path = os.path.join(self.cgi_dir, 'file1.py') - with open(self.file1_path, 'w', encoding='utf-8') as file1: - file1.write(cgi_file1 % self.pythonexe) - os.chmod(self.file1_path, 0o777) - - self.file2_path = os.path.join(self.cgi_dir, 'file2.py') - with open(self.file2_path, 'w', encoding='utf-8') as file2: - file2.write(cgi_file2 % self.pythonexe) - os.chmod(self.file2_path, 0o777) - - self.file3_path = os.path.join(self.cgi_child_dir, 'file3.py') - with open(self.file3_path, 'w', encoding='utf-8') as file3: - file3.write(cgi_file1 % self.pythonexe) - os.chmod(self.file3_path, 0o777) - - self.file4_path = os.path.join(self.cgi_dir, 'file4.py') - with open(self.file4_path, 'w', encoding='utf-8') as file4: - file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING')) - os.chmod(self.file4_path, 0o777) - - self.file5_path = os.path.join(self.cgi_dir_in_sub_dir, 'file5.py') - with open(self.file5_path, 'w', encoding='utf-8') as file5: - file5.write(cgi_file1 % self.pythonexe) - os.chmod(self.file5_path, 0o777) - - self.file6_path = os.path.join(self.cgi_dir, 'file6.py') - with open(self.file6_path, 'w', encoding='utf-8') as file6: - file6.write(cgi_file6 % self.pythonexe) - os.chmod(self.file6_path, 0o777) - - os.chdir(self.parent_dir) - - def tearDown(self): - self.request_handler._test_case_self = None - try: - os.chdir(self.cwd) - if self._pythonexe_symlink: - self._pythonexe_symlink.__exit__(None, None, None) - if self.nocgi_path: - os.remove(self.nocgi_path) - if self.file1_path: - os.remove(self.file1_path) - if self.file2_path: - os.remove(self.file2_path) - if self.file3_path: - os.remove(self.file3_path) - if self.file4_path: - os.remove(self.file4_path) - if self.file5_path: - os.remove(self.file5_path) - if self.file6_path: - os.remove(self.file6_path) - os.rmdir(self.cgi_child_dir) - os.rmdir(self.cgi_dir) - os.rmdir(self.cgi_dir_in_sub_dir) - os.rmdir(self.sub_dir_2) - os.rmdir(self.sub_dir_1) - # The 'gmon.out' file can be written in the current working - # directory if C-level code profiling with gprof is enabled. - os_helper.unlink(os.path.join(self.parent_dir, 'gmon.out')) - os.rmdir(self.parent_dir) - finally: - BaseTestCase.tearDown(self) - - def test_url_collapse_path(self): - # verify tail is the last portion and head is the rest on proper urls - test_vectors = { - '': '//', - '..': IndexError, - '/.//..': IndexError, - '/': '//', - '//': '//', - '/\\': '//\\', - '/.//': '//', - 'cgi-bin/file1.py': '/cgi-bin/file1.py', - '/cgi-bin/file1.py': '/cgi-bin/file1.py', - 'a': '//a', - '/a': '//a', - '//a': '//a', - './a': '//a', - './C:/': '/C:/', - '/a/b': '/a/b', - '/a/b/': '/a/b/', - '/a/b/.': '/a/b/', - '/a/b/c/..': '/a/b/', - '/a/b/c/../d': '/a/b/d', - '/a/b/c/../d/e/../f': '/a/b/d/f', - '/a/b/c/../d/e/../../f': '/a/b/f', - '/a/b/c/../d/e/.././././..//f': '/a/b/f', - '../a/b/c/../d/e/.././././..//f': IndexError, - '/a/b/c/../d/e/../../../f': '/a/f', - '/a/b/c/../d/e/../../../../f': '//f', - '/a/b/c/../d/e/../../../../../f': IndexError, - '/a/b/c/../d/e/../../../../f/..': '//', - '/a/b/c/../d/e/../../../../f/../.': '//', - } - for path, expected in test_vectors.items(): - if isinstance(expected, type) and issubclass(expected, Exception): - self.assertRaises(expected, - server._url_collapse_path, path) - else: - actual = server._url_collapse_path(path) - self.assertEqual(expected, actual, - msg='path = %r\nGot: %r\nWanted: %r' % - (path, actual, expected)) - - def test_headers_and_content(self): - res = self.request('/cgi-bin/file1.py') - self.assertEqual( - (res.read(), res.getheader('Content-type'), res.status), - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK)) - - def test_issue19435(self): - res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh') - self.assertEqual(res.status, HTTPStatus.NOT_FOUND) - - def test_post(self): - params = urllib.parse.urlencode( - {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) - headers = {'Content-type' : 'application/x-www-form-urlencoded'} - res = self.request('/cgi-bin/file2.py', 'POST', params, headers) - - self.assertEqual(res.read(), b'1, python, 123456' + self.linesep) - - def test_invaliduri(self): - res = self.request('/cgi-bin/invalid') - res.read() - self.assertEqual(res.status, HTTPStatus.NOT_FOUND) - - def test_authorization(self): - headers = {b'Authorization' : b'Basic ' + - base64.b64encode(b'username:pass')} - res = self.request('/cgi-bin/file1.py', 'GET', headers=headers) - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_no_leading_slash(self): - # http://bugs.python.org/issue2254 - res = self.request('cgi-bin/file1.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_os_environ_is_not_altered(self): - signature = "Test CGI Server" - os.environ['SERVER_SOFTWARE'] = signature - res = self.request('/cgi-bin/file1.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) - - def test_urlquote_decoding_in_cgi_check(self): - res = self.request('/cgi-bin%2ffile1.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_nested_cgi_path_issue21323(self): - res = self.request('/cgi-bin/child-dir/file3.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_query_with_multiple_question_mark(self): - res = self.request('/cgi-bin/file4.py?a=b?c=d') - self.assertEqual( - (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_query_with_continuous_slashes(self): - res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//') - self.assertEqual( - (b'k=aa%2F%2Fbb&//q//p//=//a//b//' + self.linesep, - 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_cgi_path_in_sub_directories(self): - try: - CGIHTTPRequestHandler.cgi_directories.append('/sub/dir/cgi-bin') - res = self.request('/sub/dir/cgi-bin/file5.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - finally: - CGIHTTPRequestHandler.cgi_directories.remove('/sub/dir/cgi-bin') - - def test_accept(self): - browser_accept = \ - 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - tests = ( - ((('Accept', browser_accept),), browser_accept), - ((), ''), - # Hack case to get two values for the one header - ((('Accept', 'text/html'), ('ACCEPT', 'text/plain')), - 'text/html,text/plain'), - ) - for headers, expected in tests: - headers = OrderedDict(headers) - with self.subTest(headers): - res = self.request('/cgi-bin/file6.py', 'GET', headers=headers) - self.assertEqual(http.HTTPStatus.OK, res.status) - expected = f"HTTP_ACCEPT={expected}".encode('ascii') - self.assertIn(expected, res.read()) - class SocketlessRequestHandler(SimpleHTTPRequestHandler): def __init__(self, directory=None): @@ -1095,6 +881,7 @@ def do_GET(self): def log_message(self, format, *args): pass + class RejectingSocketlessRequestHandler(SocketlessRequestHandler): def handle_expect_100(self): self.send_error(HTTPStatus.EXPECTATION_FAILED) @@ -1536,6 +1323,243 @@ def test_server_test_ipv4(self, _): self.assertEqual(mock_server.address_family, socket.AF_INET) +class CommandLineTestCase(unittest.TestCase): + default_port = 8000 + default_bind = None + default_protocol = 'HTTP/1.0' + default_handler = SimpleHTTPRequestHandler + default_server = unittest.mock.ANY + tls_cert = certdata_file('ssl_cert.pem') + tls_key = certdata_file('ssl_key.pem') + tls_password = 'somepass' + tls_cert_options = ['--tls-cert'] + tls_key_options = ['--tls-key'] + tls_password_options = ['--tls-password-file'] + args = { + 'HandlerClass': default_handler, + 'ServerClass': default_server, + 'protocol': default_protocol, + 'port': default_port, + 'bind': default_bind, + 'tls_cert': None, + 'tls_key': None, + 'tls_password': None, + } + + def setUp(self): + super().setUp() + self.tls_password_file = tempfile.mktemp() + with open(self.tls_password_file, 'wb') as f: + f.write(self.tls_password.encode()) + self.addCleanup(os_helper.unlink, self.tls_password_file) + + def invoke_httpd(self, *args, stdout=None, stderr=None): + stdout = StringIO() if stdout is None else stdout + stderr = StringIO() if stderr is None else stderr + with contextlib.redirect_stdout(stdout), \ + contextlib.redirect_stderr(stderr): + server._main(args) + return stdout.getvalue(), stderr.getvalue() + + @mock.patch('http.server.test') + def test_port_flag(self, mock_func): + ports = [8000, 65535] + for port in ports: + with self.subTest(port=port): + self.invoke_httpd(str(port)) + call_args = self.args | dict(port=port) + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_directory_flag(self, mock_func): + options = ['-d', '--directory'] + directories = ['.', '/foo', '\\bar', '/', + 'C:\\', 'C:\\foo', 'C:\\bar', + '/home/user', './foo/foo2', 'D:\\foo\\bar'] + for flag in options: + for directory in directories: + with self.subTest(flag=flag, directory=directory): + self.invoke_httpd(flag, directory) + mock_func.assert_called_once_with(**self.args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_bind_flag(self, mock_func): + options = ['-b', '--bind'] + bind_addresses = ['localhost', '127.0.0.1', '::1', + '0.0.0.0', '8.8.8.8'] + for flag in options: + for bind_address in bind_addresses: + with self.subTest(flag=flag, bind_address=bind_address): + self.invoke_httpd(flag, bind_address) + call_args = self.args | dict(bind=bind_address) + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_protocol_flag(self, mock_func): + options = ['-p', '--protocol'] + protocols = ['HTTP/1.0', 'HTTP/1.1', 'HTTP/2.0', 'HTTP/3.0'] + for flag in options: + for protocol in protocols: + with self.subTest(flag=flag, protocol=protocol): + self.invoke_httpd(flag, protocol) + call_args = self.args | dict(protocol=protocol) + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @unittest.skipIf(ssl is None, "requires ssl") + @mock.patch('http.server.test') + def test_tls_cert_and_key_flags(self, mock_func): + for tls_cert_option in self.tls_cert_options: + for tls_key_option in self.tls_key_options: + self.invoke_httpd(tls_cert_option, self.tls_cert, + tls_key_option, self.tls_key) + call_args = self.args | { + 'tls_cert': self.tls_cert, + 'tls_key': self.tls_key, + } + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @unittest.skipIf(ssl is None, "requires ssl") + @mock.patch('http.server.test') + def test_tls_cert_and_key_and_password_flags(self, mock_func): + for tls_cert_option in self.tls_cert_options: + for tls_key_option in self.tls_key_options: + for tls_password_option in self.tls_password_options: + self.invoke_httpd(tls_cert_option, + self.tls_cert, + tls_key_option, + self.tls_key, + tls_password_option, + self.tls_password_file) + call_args = self.args | { + 'tls_cert': self.tls_cert, + 'tls_key': self.tls_key, + 'tls_password': self.tls_password, + } + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @unittest.skipIf(ssl is None, "requires ssl") + @mock.patch('http.server.test') + def test_missing_tls_cert_flag(self, mock_func): + for tls_key_option in self.tls_key_options: + with self.assertRaises(SystemExit): + self.invoke_httpd(tls_key_option, self.tls_key) + mock_func.reset_mock() + + for tls_password_option in self.tls_password_options: + with self.assertRaises(SystemExit): + self.invoke_httpd(tls_password_option, self.tls_password) + mock_func.reset_mock() + + @unittest.skipIf(ssl is None, "requires ssl") + @mock.patch('http.server.test') + def test_invalid_password_file(self, mock_func): + non_existent_file = 'non_existent_file' + for tls_password_option in self.tls_password_options: + for tls_cert_option in self.tls_cert_options: + with self.assertRaises(SystemExit): + self.invoke_httpd(tls_cert_option, + self.tls_cert, + tls_password_option, + non_existent_file) + + @mock.patch('http.server.test') + def test_no_arguments(self, mock_func): + self.invoke_httpd() + mock_func.assert_called_once_with(**self.args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_help_flag(self, _): + options = ['-h', '--help'] + for option in options: + stdout, stderr = StringIO(), StringIO() + with self.assertRaises(SystemExit): + self.invoke_httpd(option, stdout=stdout, stderr=stderr) + self.assertIn('usage', stdout.getvalue()) + self.assertEqual(stderr.getvalue(), '') + + @mock.patch('http.server.test') + def test_unknown_flag(self, _): + stdout, stderr = StringIO(), StringIO() + with self.assertRaises(SystemExit): + self.invoke_httpd('--unknown-flag', stdout=stdout, stderr=stderr) + self.assertEqual(stdout.getvalue(), '') + self.assertIn('error', stderr.getvalue()) + + +class CommandLineRunTimeTestCase(unittest.TestCase): + served_data = os.urandom(32) + served_filename = 'served_filename' + tls_cert = certdata_file('ssl_cert.pem') + tls_key = certdata_file('ssl_key.pem') + tls_password = b'somepass' + tls_password_file = 'ssl_key_password' + + def setUp(self): + super().setUp() + server_dir_context = os_helper.temp_cwd() + server_dir = self.enterContext(server_dir_context) + with open(self.served_filename, 'wb') as f: + f.write(self.served_data) + with open(self.tls_password_file, 'wb') as f: + f.write(self.tls_password) + + def fetch_file(self, path, context=None): + req = urllib.request.Request(path, method='GET') + with urllib.request.urlopen(req, context=context) as res: + return res.read() + + def parse_cli_output(self, output): + match = re.search(r'Serving (HTTP|HTTPS) on (.+) port (\d+)', output) + if match is None: + return None, None, None + return match.group(1).lower(), match.group(2), int(match.group(3)) + + def wait_for_server(self, proc, protocol, bind, port): + """Check that the server has been successfully started.""" + line = proc.stdout.readline().strip() + if support.verbose: + print() + print('python -m http.server: ', line) + return self.parse_cli_output(line) == (protocol, bind, port) + + def test_http_client(self): + bind, port = '127.0.0.1', find_unused_port() + proc = spawn_python('-u', '-m', 'http.server', str(port), '-b', bind, + bufsize=1, text=True) + self.addCleanup(kill_python, proc) + self.addCleanup(proc.terminate) + self.assertTrue(self.wait_for_server(proc, 'http', bind, port)) + res = self.fetch_file(f'http://{bind}:{port}/{self.served_filename}') + self.assertEqual(res, self.served_data) + + @unittest.skipIf(ssl is None, "requires ssl") + def test_https_client(self): + context = ssl.create_default_context() + # allow self-signed certificates + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + bind, port = '127.0.0.1', find_unused_port() + proc = spawn_python('-u', '-m', 'http.server', str(port), '-b', bind, + '--tls-cert', self.tls_cert, + '--tls-key', self.tls_key, + '--tls-password-file', self.tls_password_file, + bufsize=1, text=True) + self.addCleanup(kill_python, proc) + self.addCleanup(proc.terminate) + self.assertTrue(self.wait_for_server(proc, 'https', bind, port)) + url = f'https://{bind}:{port}/{self.served_filename}' + res = self.fetch_file(url, context=context) + self.assertEqual(res, self.served_data) + + def setUpModule(): unittest.addModuleCleanup(os.chdir, os.getcwd()) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 3d8b7ecc0ec..ebf572ac5ca 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -16,7 +16,7 @@ # Unittest.main and test.libregrtest.runtest.runtest_inner # call load_tests, when present here, to discover tests to run. -from idlelib.idle_test import load_tests +from idlelib.idle_test import load_tests # noqa: F401 if __name__ == '__main__': tk.NoDefaultRoot() diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index a13ee58d650..430fa71fa29 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -12,8 +12,7 @@ import socket from test.support import verbose, run_with_tz, run_with_locale, cpython_only -from test.support import hashlib_helper -from test.support import threading_helper +from test.support import hashlib_helper, threading_helper import unittest from unittest import mock from datetime import datetime, timezone, timedelta @@ -256,7 +255,20 @@ def cmd_IDLE(self, tag, args): self._send_tagged(tag, 'BAD', 'Expected DONE') -class NewIMAPTestsMixin(): +class AuthHandler_CRAM_MD5(SimpleIMAPHandler): + capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' + 'VzdG9uLm1jaS5uZXQ=') + r = yield + if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' + b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): + self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') + else: + self._send_tagged(tag, 'NO', 'No access') + + +class NewIMAPTestsMixin: client = None def _setup(self, imap_handler, connect=True): @@ -360,7 +372,11 @@ def cmd_AUTHENTICATE(self, tag, args): self._send_tagged(tag, 'OK', 'FAKEAUTH successful') def cmd_APPEND(self, tag, args): self._send_textline('+') - self.server.response = yield + self.server.response = args + literal = yield + self.server.response.append(literal) + literal = yield + self.server.response.append(literal) self._send_tagged(tag, 'OK', 'okay') client, server = self._setup(UTF8AppendServer) self.assertEqual(client._encoding, 'ascii') @@ -371,10 +387,13 @@ def cmd_APPEND(self, tag, args): self.assertEqual(code, 'OK') self.assertEqual(client._encoding, 'utf-8') msg_string = 'Subject: üñí©öðé' - typ, data = client.append(None, None, None, msg_string.encode('utf-8')) + typ, data = client.append( + None, None, None, (msg_string + '\n').encode('utf-8')) self.assertEqual(typ, 'OK') self.assertEqual(server.response, - ('UTF8 (%s)\r\n' % msg_string).encode('utf-8')) + ['INBOX', 'UTF8', + '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'), + b')\r\n' ]) def test_search_disallows_charset_in_utf8_mode(self): class UTF8Server(SimpleIMAPHandler): @@ -439,40 +458,26 @@ def cmd_AUTHENTICATE(self, tag, args): @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_login_cram_md5_bytes(self): - class AuthHandler(SimpleIMAPHandler): - capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' - def cmd_AUTHENTICATE(self, tag, args): - self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' - 'VzdG9uLm1jaS5uZXQ=') - r = yield - if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' - b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): - self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') - else: - self._send_tagged(tag, 'NO', 'No access') - client, _ = self._setup(AuthHandler) - self.assertTrue('AUTH=CRAM-MD5' in client.capabilities) + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf") self.assertEqual(ret, "OK") @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_login_cram_md5_plain_text(self): - class AuthHandler(SimpleIMAPHandler): - capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' - def cmd_AUTHENTICATE(self, tag, args): - self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' - 'VzdG9uLm1jaS5uZXQ=') - r = yield - if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' - b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): - self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') - else: - self._send_tagged(tag, 'NO', 'No access') - client, _ = self._setup(AuthHandler) - self.assertTrue('AUTH=CRAM-MD5' in client.capabilities) + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) ret, _ = client.login_cram_md5("tim", "tanstaaftanstaaf") self.assertEqual(ret, "OK") + @hashlib_helper.block_algorithm("md5") + def test_login_cram_md5_blocked(self): + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) + msg = re.escape("CRAM-MD5 authentication is not supported") + with self.assertRaisesRegex(imaplib.IMAP4.error, msg): + client.login_cram_md5("tim", b"tanstaaftanstaaf") + def test_aborted_authentication(self): class MyServer(SimpleIMAPHandler): def cmd_AUTHENTICATE(self, tag, args): @@ -883,7 +888,11 @@ def test_enable_UTF8_True_append(self): class UTF8AppendServer(self.UTF8Server): def cmd_APPEND(self, tag, args): self._send_textline('+') - self.server.response = yield + self.server.response = args + literal = yield + self.server.response.append(literal) + literal = yield + self.server.response.append(literal) self._send_tagged(tag, 'OK', 'okay') with self.reaped_pair(UTF8AppendServer) as (server, client): @@ -897,12 +906,12 @@ def cmd_APPEND(self, tag, args): self.assertEqual(client._encoding, 'utf-8') msg_string = 'Subject: üñí©öðé' typ, data = client.append( - None, None, None, msg_string.encode('utf-8')) + None, None, None, (msg_string + '\n').encode('utf-8')) self.assertEqual(typ, 'OK') - self.assertEqual( - server.response, - ('UTF8 (%s)\r\n' % msg_string).encode('utf-8') - ) + self.assertEqual(server.response, + ['INBOX', 'UTF8', + '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'), + b')\r\n' ]) # XXX also need a test that makes sure that the Literal and Untagged_status # regexes uses unicode in UTF8 mode instead of the default ASCII. @@ -1108,5 +1117,15 @@ def test_ssl_verified(self): client.shutdown() +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(imaplib, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 6e34094c5aa..271361ae816 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -15,6 +15,7 @@ import os import py_compile import random +import re import shutil import stat import subprocess @@ -23,6 +24,7 @@ import threading import time import types +import warnings import unittest from unittest import mock import _imp @@ -35,7 +37,7 @@ cpython_only, is_apple_mobile, is_emscripten, - is_wasi, + is_wasm32, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS, @@ -51,7 +53,7 @@ TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE) from test.support import script_helper from test.support import threading_helper -from test.test_importlib.util import uncache +from test.test_importlib.util import uncache, temporary_pycache_prefix from types import ModuleType try: import _testsinglephase @@ -63,8 +65,10 @@ _testmultiphase = None try: import _interpreters + import concurrent.interpreters except ModuleNotFoundError: _interpreters = None + concurrent = None try: import _testinternalcapi except ImportError: @@ -412,7 +416,6 @@ def test_from_import_missing_attr_path_is_canonical(self): self.assertIsNotNone(cm.exception) def test_from_import_star_invalid_type(self): - import re with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("__all__ = [b'invalid_type']") @@ -1187,6 +1190,7 @@ class substr(str): @unittest.skipIf(sys.platform == 'win32', 'Cannot delete cwd on Windows') @unittest.skipIf(sys.platform == 'sunos5', 'Cannot delete cwd on Solaris/Illumos') + @unittest.skipIf(sys.platform.startswith('aix'), 'Cannot delete cwd on AIX') def test_script_shadowing_stdlib_cwd_failure(self): with os_helper.temp_dir() as tmp: subtmp = os.path.join(tmp, "subtmp") @@ -1249,6 +1253,22 @@ class Spec2: origin = "a\x00b" _imp.create_dynamic(Spec2()) + def test_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + unload('test.test_import.data.syntax_warnings') + with (os_helper.temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + import test.test_import.data.syntax_warnings + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = test.test_import.data.syntax_warnings.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -1257,7 +1277,7 @@ class FilePermissionTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Emscripten's/WASI's umask is a stub." ) def test_creation_mode(self): @@ -2466,6 +2486,21 @@ def test_multi_init_extension_per_interpreter_gil_compat(self): self.check_compatible_here( modname, filename, strict=False, isolated=False) + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_testmultiphase_exec_multiple(self): + modname = '_testmultiphase_exec_multiple' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + # All three exec's were called. + self.assertEqual(module.a, 1) + self.assertEqual(module.b, 2) + self.assertEqual(module.c, 3) + # They were called in order. + keys = list(module.__dict__) + self.assertLess(keys.index('a'), keys.index('b')) + self.assertLess(keys.index('b'), keys.index('c')) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_python_compat(self): module = 'threading' @@ -3159,6 +3194,7 @@ def test_check_state_first(self): # Also, we test with a single-phase module that has global state, # which is shared by all interpreters. + @no_rerun(reason="module state is not cleared (see gh-140657)") @requires_subinterpreters def test_basic_multiple_interpreters_main_no_reset(self): # without resetting; already loaded in main interpreter @@ -3227,6 +3263,7 @@ def test_basic_multiple_interpreters_main_no_reset(self): # * m_copy was copied from interp2 (was from interp1) # * module's global state was updated, not reset + @unittest.skip("gh-131229: This is suddenly very flaky") @no_rerun(reason="rerun not possible; module state is never cleared (see gh-102251)") @requires_subinterpreters def test_basic_multiple_interpreters_deleted_no_reset(self): @@ -3362,6 +3399,150 @@ def test_basic_multiple_interpreters_reset_each(self): # * module's global state was initialized, not reset +@unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") +class ModexportTests(unittest.TestCase): + def test_from_modexport(self): + modname = '_test_from_modexport' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + self.assertEqual(module.__name__, modname) + + @requires_subinterpreters + def test_from_modexport_gil_used(self): + # Test that a module with Py_MOD_GIL_USED (re-)enables the GIL. + # Do this in a new interpreter to avoid interfering with global state. + modname = '_test_from_modexport_gil_used' + filename = _testmultiphase.__file__ + interp = concurrent.interpreters.create() + self.addCleanup(interp.close) + queue = concurrent.interpreters.create_queue() + interp.prepare_main( + modname=modname, + filename=filename, + queue=queue, + ) + enabled_before = sys._is_gil_enabled() + interp.exec(f"""if True: + import sys + from test.support.warnings_helper import check_warnings + from {__name__} import import_extension_from_file + with check_warnings((".*GIL..has been enabled.*", RuntimeWarning), + quiet=True): + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + queue.put(module.__name__) + queue.put(sys._is_gil_enabled()) + """) + + self.assertEqual(queue.get(), modname) + self.assertEqual(queue.get(), True) + self.assertTrue(queue.empty()) + + self.assertEqual(enabled_before, sys._is_gil_enabled()) + + def test_from_modexport_null(self): + modname = '_test_from_modexport_null' + filename = _testmultiphase.__file__ + with self.assertRaises(SystemError): + import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + def test_from_modexport_exception(self): + modname = '_test_from_modexport_exception' + filename = _testmultiphase.__file__ + with self.assertRaises(ValueError): + import_extension_from_file(modname, filename, + put_in_sys_modules=False) + + def test_from_modexport_create_nonmodule(self): + modname = '_test_from_modexport_create_nonmodule' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + self.assertIsInstance(module, str) + + @requires_subinterpreters + def test_from_modexport_create_nonmodule_gil_used(self): + # Test that a module with Py_MOD_GIL_USED (re-)enables the GIL. + # Do this in a new interpreter to avoid interfering with global state. + modname = '_test_from_modexport_create_nonmodule_gil_used' + filename = _testmultiphase.__file__ + interp = concurrent.interpreters.create() + self.addCleanup(interp.close) + queue = concurrent.interpreters.create_queue() + interp.prepare_main( + modname=modname, + filename=filename, + queue=queue, + ) + enabled_before = sys._is_gil_enabled() + interp.exec(f"""if True: + import sys + from test.support.warnings_helper import check_warnings + from {__name__} import import_extension_from_file + with check_warnings((".*GIL..has been enabled.*", RuntimeWarning), + quiet=True): + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + queue.put(module) + queue.put(sys._is_gil_enabled()) + """) + + self.assertIsInstance(queue.get(), str) + self.assertEqual(queue.get(), True) + self.assertTrue(queue.empty()) + + self.assertEqual(enabled_before, sys._is_gil_enabled()) + + def test_from_modexport_smoke(self): + # General positive test for sundry features + # (PyModule_FromSlotsAndSpec tests exercise these more carefully) + modname = '_test_from_modexport_smoke' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + self.assertEqual(module.__doc__, "the expected docstring") + self.assertEqual(module.number, 147) + self.assertEqual(module.get_state_int(), 258) + self.assertGreater(module.get_test_token(), 0) + + def test_from_modexport_smoke_token(self): + _testcapi = import_module("_testcapi") + + modname = '_test_from_modexport_smoke' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename, + put_in_sys_modules=False) + token = module.get_test_token() + self.assertEqual(_testcapi.pymodule_get_token(module), token) + + tp = module.Example + self.assertEqual(_testcapi.pytype_getmodulebytoken(tp, token), module) + class Sub(tp): + pass + self.assertEqual(_testcapi.pytype_getmodulebytoken(Sub, token), module) + + @requires_gil_enabled("empty slots re-enable GIL") + def test_from_modexport_empty_slots(self): + # Module to test that: + # - no slots are mandatory for PyModExport + # - the slots array is used as the default token + modname = '_test_from_modexport_empty_slots' + filename = _testmultiphase.__file__ + module = import_extension_from_file( + modname, filename, put_in_sys_modules=False) + + self.assertEqual(module.__name__, modname) + self.assertEqual(module.__doc__, None) + + _testcapi = import_module("_testcapi") + smoke_mod = import_extension_from_file( + '_test_from_modexport_smoke', filename, put_in_sys_modules=False) + self.assertEqual(_testcapi.pymodule_get_token(module), + smoke_mod.get_modexport_empty_slots()) + @cpython_only class TestMagicNumber(unittest.TestCase): def test_magic_number_endianness(self): diff --git a/Lib/test/test_import/data/syntax_warnings.py b/Lib/test/test_import/data/syntax_warnings.py new file mode 100644 index 00000000000..103f07b6187 --- /dev/null +++ b/Lib/test/test_import/data/syntax_warnings.py @@ -0,0 +1,21 @@ +# Syntax warnings emitted in different parts of the Python compiler. + +# Parser/lexer/lexer.c +x = 1or 0 # line 4 + +# Parser/tokenizer/helpers.c +'\z' # line 7 + +# Parser/string_parser.c +'\400' # line 10 + +# _PyCompile_Warn() in Python/codegen.c +assert(x, 'message') # line 13 +x is 1 # line 14 + +# _PyErr_EmitSyntaxWarning() in Python/ast_preprocess.c +def f(): + try: + pass + finally: + return 42 # line 21 diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index e535d119763..1549cbe96ce 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -223,6 +223,21 @@ def test_relative_import_no_package_exists_absolute(self): self.__import__('sys', {'__package__': '', '__spec__': None}, level=1) + def test_malicious_relative_import(self): + # https://github.com/python/cpython/issues/134100 + # Test to make sure UAF bug with error msg doesn't come back to life + import sys + loooong = "".ljust(0x23000, "b") + name = f"a.{loooong}.c" + + with util.uncache(name): + sys.modules[name] = {} + with self.assertRaisesRegex( + KeyError, + r"'a\.b+' not in sys\.modules as expected" + ): + __import__(f"{loooong}.c", {"__package__": "a"}, level=1) + (Frozen_RelativeImports, Source_RelativeImports diff --git a/Lib/test/test_importlib/metadata/_path.py b/Lib/test/test_importlib/metadata/_path.py index b3cfb9cd549..e63d889f96b 100644 --- a/Lib/test/test_importlib/metadata/_path.py +++ b/Lib/test/test_importlib/metadata/_path.py @@ -1,9 +1,14 @@ -# from jaraco.path 3.7 +# from jaraco.path 3.7.2 + +from __future__ import annotations import functools import pathlib -from typing import Dict, Protocol, Union -from typing import runtime_checkable +from collections.abc import Mapping +from typing import TYPE_CHECKING, Protocol, Union, runtime_checkable + +if TYPE_CHECKING: + from typing_extensions import Self class Symlink(str): @@ -12,29 +17,25 @@ class Symlink(str): """ -FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] # type: ignore +FilesSpec = Mapping[str, Union[str, bytes, Symlink, 'FilesSpec']] @runtime_checkable class TreeMaker(Protocol): - def __truediv__(self, *args, **kwargs): ... # pragma: no cover - - def mkdir(self, **kwargs): ... # pragma: no cover - - def write_text(self, content, **kwargs): ... # pragma: no cover - - def write_bytes(self, content): ... # pragma: no cover - - def symlink_to(self, target): ... # pragma: no cover + def __truediv__(self, other, /) -> Self: ... + def mkdir(self, *, exist_ok) -> object: ... + def write_text(self, content, /, *, encoding) -> object: ... + def write_bytes(self, content, /) -> object: ... + def symlink_to(self, target, /) -> object: ... -def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: - return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore +def _ensure_tree_maker(obj: str | TreeMaker) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) def build( spec: FilesSpec, - prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore + prefix: str | TreeMaker = pathlib.Path(), ): """ Build a set of files/directories, as described by the spec. @@ -66,23 +67,24 @@ def build( @functools.singledispatch -def create(content: Union[str, bytes, FilesSpec], path): +def create(content: str | bytes | FilesSpec, path: TreeMaker) -> None: path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore + # Mypy only looks at the signature of the main singledispatch method. So it must contain the complete Union + build(content, prefix=path) # type: ignore[arg-type] # python/mypy#11727 @create.register -def _(content: bytes, path): +def _(content: bytes, path: TreeMaker) -> None: path.write_bytes(content) @create.register -def _(content: str, path): +def _(content: str, path: TreeMaker) -> None: path.write_text(content, encoding='utf-8') @create.register -def _(content: Symlink, path): +def _(content: Symlink, path: TreeMaker) -> None: path.symlink_to(content) diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py index 826b1b3259b..ad0ab42e089 100644 --- a/Lib/test/test_importlib/metadata/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -1,11 +1,11 @@ -import sys -import copy -import json -import shutil -import pathlib -import textwrap -import functools import contextlib +import copy +import functools +import json +import pathlib +import shutil +import sys +import textwrap from test.support import import_helper from test.support import os_helper @@ -14,14 +14,10 @@ from . import _path from ._path import FilesSpec - -try: - from importlib import resources # type: ignore - - getattr(resources, 'files') - getattr(resources, 'as_file') -except (ImportError, AttributeError): - import importlib_resources as resources # type: ignore +if sys.version_info >= (3, 9): + from importlib import resources +else: + import importlib_resources as resources @contextlib.contextmanager @@ -378,6 +374,8 @@ def setUp(self): # Add self.zip_name to the front of sys.path. self.resources = contextlib.ExitStack() self.addCleanup(self.resources.close) + # workaround for #138313 + self.addCleanup(lambda: None) def parameterize(*args_set): diff --git a/Lib/test/test_importlib/metadata/test_api.py b/Lib/test/test_importlib/metadata/test_api.py index 2256e0c502e..9f6e12c87e8 100644 --- a/Lib/test/test_importlib/metadata/test_api.py +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -1,11 +1,8 @@ +import importlib import re import textwrap import unittest -import warnings -import importlib -import contextlib -from . import fixtures from importlib.metadata import ( Distribution, PackageNotFoundError, @@ -17,12 +14,7 @@ version, ) - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx +from . import fixtures class APITests( @@ -153,13 +145,13 @@ def test_metadata_for_this_package(self): classifiers = md.get_all('Classifier') assert 'Topic :: Software Development :: Libraries' in classifiers - def test_missing_key_legacy(self): + def test_missing_key(self): """ - Requesting a missing key will still return None, but warn. + Requesting a missing key raises KeyError. """ md = metadata('distinfo-pkg') - with suppress_known_deprecation(): - assert md['does-not-exist'] is None + with self.assertRaises(KeyError): + md['does-not-exist'] def test_get_key(self): """ diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py index e4218076f8c..83b686babfd 100644 --- a/Lib/test/test_importlib/metadata/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -1,10 +1,7 @@ -import re -import pickle -import unittest -import warnings import importlib -import importlib.metadata -import contextlib +import pickle +import re +import unittest from test.support import os_helper try: @@ -12,9 +9,6 @@ except ImportError: from .stubs import fake_filesystem_unittest as ffs -from . import fixtures -from ._context import suppress -from ._path import Symlink from importlib.metadata import ( Distribution, EntryPoint, @@ -27,12 +21,8 @@ version, ) - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx +from . import fixtures +from ._path import Symlink class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): @@ -59,9 +49,6 @@ def test_package_not_found_mentions_metadata(self): assert "metadata" in str(ctx.exception) - # expected to fail until ABC is enforced - @suppress(AssertionError) - @suppress_known_deprecation() def test_abc_enforced(self): with self.assertRaises(TypeError): type('DistributionSubclass', (Distribution,), {})() @@ -146,6 +133,41 @@ def test_unique_distributions(self): assert len(after) == len(before) +class InvalidMetadataTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def make_pkg(name, files=dict(METADATA="VERSION: 1.0")): + """ + Create metadata for a dist-info package with name and files. + """ + return { + f'{name}.dist-info': files, + } + + def test_valid_dists_preferred(self): + """ + Dists with metadata should be preferred when discovered by name. + + Ref python/importlib_metadata#489. + """ + # create three dists with the valid one in the middle (lexicographically) + # such that on most file systems, the valid one is never naturally first. + fixtures.build_files(self.make_pkg('foo-4.0', files={}), self.site_dir) + fixtures.build_files(self.make_pkg('foo-4.1'), self.site_dir) + fixtures.build_files(self.make_pkg('foo-4.2', files={}), self.site_dir) + dist = Distribution.from_name('foo') + assert dist.version == "1.0" + + def test_missing_metadata(self): + """ + Dists with a missing metadata file should return None. + + Ref python/importlib_metadata#493. + """ + fixtures.build_files(self.make_pkg('foo-4.3', files={}), self.site_dir) + assert Distribution.from_name('foo').metadata is None + assert metadata('foo') is None + + class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod def pkg_with_non_ascii_description(site_dir): diff --git a/Lib/test/test_importlib/metadata/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py index 276f6288c91..fcb649f3736 100644 --- a/Lib/test/test_importlib/metadata/test_zip.py +++ b/Lib/test/test_importlib/metadata/test_zip.py @@ -1,7 +1,6 @@ import sys import unittest -from . import fixtures from importlib.metadata import ( PackageNotFoundError, distribution, @@ -11,6 +10,8 @@ version, ) +from . import fixtures + class TestZip(fixtures.ZipFixtures, unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_importlib/namespace_pkgs/foo/README.md b/Lib/test/test_importlib/namespace_pkgs/foo/README.md new file mode 100644 index 00000000000..9f6bc74941d --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/foo/README.md @@ -0,0 +1,2 @@ +This directory should not be a package, but should share a name with an +unrelated subpackage. diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 3ce44999f98..c935b1e10ac 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -38,14 +38,6 @@ def test_joinpath_with_multiple_args(self): binfile = files.joinpath('subdirectory', 'binary.file') self.assertTrue(binfile.is_file()) - def test_old_parameter(self): - """ - Files used to take a 'package' parameter. Make sure anyone - passing by name is still supported. - """ - with suppress_known_deprecation(): - resources.files(package=self.data) - class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): pass diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 070920d0da7..bd1540ce403 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -224,15 +224,7 @@ class ResourceLoaderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(ResourceLoader) def test_get_data(self): - with ( - self.assertRaises(IOError), - self.assertWarnsRegex( - DeprecationWarning, - r"importlib\.abc\.ResourceLoader is deprecated in favour of " - r"supporting resource loading through importlib\.resources" - r"\.abc\.TraversableResources.", - ), - ): + with self.assertRaises(IOError): self.ins.get_data('/some/path') @@ -912,7 +904,7 @@ def test_universal_newlines(self): mock = self.SourceOnlyLoaderMock('mod.file') source = "x = 42\r\ny = -13\r\n" mock.source = source.encode('utf-8') - expect = io.IncrementalNewlineDecoder(None, True).decode(source) + expect = io.StringIO(source, newline=None).getvalue() self.assertEqual(mock.get_source(name), expect) @@ -936,13 +928,8 @@ def get_filename(self, fullname): def path_stats(self, path): return {'mtime': 1} - with self.assertWarnsRegex( - DeprecationWarning, - r"importlib\.abc\.ResourceLoader is deprecated in favour of " - r"supporting resource loading through importlib\.resources" - r"\.abc\.TraversableResources.", - ): - loader = DummySourceLoader() + + loader = DummySourceLoader() with self.assertWarnsRegex( DeprecationWarning, @@ -952,17 +939,5 @@ def path_stats(self, path): loader.path_mtime('foo.py') -class ResourceLoaderDeprecationWarningsTests(unittest.TestCase): - """Tests ResourceLoader deprecation warnings.""" - - def test_deprecated_resource_loader(self): - from importlib.abc import ResourceLoader - class DummyLoader(ResourceLoader): - def get_data(self, path): - return b'' - - with self.assertWarns(DeprecationWarning): - DummyLoader() - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index befac5d62b0..655e5881a15 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -34,6 +34,7 @@ class ModuleLockAsRLockTests: # lock status in repr unsupported test_repr = None test_locked_repr = None + test_repr_count = None def tearDown(self): for splitinit in init.values(): diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 6ca0978f9bc..522ed9fa43f 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -318,6 +318,17 @@ def test_module_before_namespace_package(self): self.assertEqual(a_test.attr, 'in module') +class NamespaceSubpackageSameName(NamespacePackageTest): + paths = [''] + + def test_namespace_subpackage_shares_name_with_directory(self): + submodule_path = 'project4.foo' + with self.assertRaises(ModuleNotFoundError) as cm: + importlib.machinery.PathFinder.find_spec(submodule_path) + + self.assertEqual(cm.exception.name, 'project4') + + class ReloadTests(NamespacePackageTest): paths = ['portion1'] diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 9af1e4d505c..f78dc399720 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -135,10 +135,12 @@ def check_parallel_module_init(self, mock_os): if verbose: print("OK.") - def test_parallel_module_init(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_module_init(self, size): self.check_parallel_module_init() - def test_parallel_meta_path(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_meta_path(self, size): finder = Finder() sys.meta_path.insert(0, finder) try: @@ -148,7 +150,8 @@ def test_parallel_meta_path(self): finally: sys.meta_path.remove(finder) - def test_parallel_path_hooks(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_path_hooks(self, size): # Here the Finder instance is only used to check concurrent calls # to path_hook(). finder = Finder() @@ -242,13 +245,15 @@ def target(): __import__(TESTFN) del sys.modules[TESTFN] - def test_concurrent_futures_circular_import(self): + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_concurrent_futures_circular_import(self, size): # Regression test for bpo-43515 fn = os.path.join(os.path.dirname(__file__), 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) - def test_multiprocessing_pool_circular_import(self): + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_multiprocessing_pool_circular_import(self, size): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), 'partial', 'pool_in_threads.py') diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 5de89714eb5..0adab8d14e0 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -580,6 +580,18 @@ def test_cache_from_source_respects_pycache_prefix_relative(self): self.util.cache_from_source(path, optimization=''), os.path.normpath(expect)) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_cache_from_source_in_root_with_pycache_prefix(self): + # Regression test for gh-82916 + pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode') + path = 'qux.py' + expect = os.path.join(os.path.sep, 'tmp', 'bytecode', + f'qux.{self.tag}.pyc') + with util.temporary_pycache_prefix(pycache_prefix): + with os_helper.change_cwd('/'): + self.assertEqual(self.util.cache_from_source(path), expect) + @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_inside_pycache_prefix(self): @@ -635,7 +647,7 @@ def test_magic_number(self): # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3495 + EXPECTED_MAGIC_NUMBER = 3625 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( @@ -776,31 +788,70 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) -class MiscTests(unittest.TestCase): - def test_atomic_write_should_notice_incomplete_writes(self): +class PatchAtomicWrites: + def __init__(self, truncate_at_length, never_complete=False): + self.truncate_at_length = truncate_at_length + self.never_complete = never_complete + self.seen_write = False + self._children = [] + + def __enter__(self): import _pyio oldwrite = os.write - seen_write = False - - truncate_at_length = 100 # Emulate an os.write that only writes partial data. def write(fd, data): - nonlocal seen_write - seen_write = True - return oldwrite(fd, data[:truncate_at_length]) + if self.seen_write and self.never_complete: + return None + self.seen_write = True + return oldwrite(fd, data[:self.truncate_at_length]) # Need to patch _io to be _pyio, so that io.FileIO is affected by the # os.write patch. - with (support.swap_attr(_bootstrap_external, '_io', _pyio), - support.swap_attr(os, 'write', write)): - with self.assertRaises(OSError): - # Make sure we write something longer than the point where we - # truncate. - content = b'x' * (truncate_at_length * 2) - _bootstrap_external._write_atomic(os_helper.TESTFN, content) - assert seen_write + self.children = [ + support.swap_attr(_bootstrap_external, '_io', _pyio), + support.swap_attr(os, 'write', write) + ] + for child in self.children: + child.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for child in self.children: + child.__exit__(exc_type, exc_val, exc_tb) + + +class MiscTests(unittest.TestCase): + + def test_atomic_write_retries_incomplete_writes(self): + truncate_at_length = 100 + length = truncate_at_length * 2 + + with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm: + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * length + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length) + os.unlink(support.os_helper.TESTFN) + + def test_atomic_write_errors_if_unable_to_complete(self): + truncate_at_length = 100 + + with ( + PatchAtomicWrites( + truncate_at_length=truncate_at_length, never_complete=True, + ) as cm, + self.assertRaises(OSError) + ): + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * (truncate_at_length * 2) + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) with self.assertRaises(OSError): os.stat(support.os_helper.TESTFN) # Check that the file did not get written. diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index edbe78545a2..efbec667317 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -15,7 +15,8 @@ import tempfile import types -_testsinglephase = import_helper.import_module("_testsinglephase") +# gh-116303: Skip test module dependent tests if test modules are unavailable +import_helper.import_module("_testmultiphase") BUILTINS = types.SimpleNamespace() diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 43fda662253..157e12167b5 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -369,3 +369,35 @@ class dc364: # line 369 dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) + +import inspect +import itertools + +# line 376 +ge377 = ( + inspect.currentframe() + for i in itertools.count() +) + +# line 382 +def func383(): + # line 384 + ge385 = ( + inspect.currentframe() + for i in itertools.count() + ) + return ge385 + +# line 391 +@decorator +# comment +def func394(): + return 395 + +# line 397 +@decorator + +def func400(): + return 401 + +pass # end of file diff --git a/Lib/test/test_inspect/inspect_fodder3.py b/Lib/test/test_inspect/inspect_fodder3.py new file mode 100644 index 00000000000..ea2481edf93 --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder3.py @@ -0,0 +1,39 @@ +from functools import cached_property + +# docstring in parent, inherited in child +class ParentInheritDoc: + @cached_property + def foo(self): + """docstring for foo defined in parent""" + +class ChildInheritDoc(ParentInheritDoc): + pass + +class ChildInheritDefineDoc(ParentInheritDoc): + @cached_property + def foo(self): + pass + +# Redefine foo as something other than cached_property +class ChildPropertyFoo(ParentInheritDoc): + @property + def foo(self): + """docstring for the property foo""" + +class ChildMethodFoo(ParentInheritDoc): + def foo(self): + """docstring for the method foo""" + +# docstring in child but not parent +class ParentNoDoc: + @cached_property + def foo(self): + pass + +class ChildNoDoc(ParentNoDoc): + pass + +class ChildDefineDoc(ParentNoDoc): + @cached_property + def foo(self): + """docstring for foo defined in child""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c9b37fcd8f6..dd3b7d9c5b4 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -46,6 +46,7 @@ from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_fodder3 as mod3 from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_deferred_annotations @@ -148,6 +149,29 @@ def meth_self_o(self, object, /): pass def meth_type_noargs(type, /): pass def meth_type_o(type, object, /): pass +# Decorator decorator that returns a simple wrapped function +def identity_wrapper(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + return func(*args, **kwargs) + return wrapped + +# Original signature of the simple wrapped function returned by +# identity_wrapper(). +varargs_signature = ( + (('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ..., +) + +# Decorator decorator that returns a simple descriptor +class custom_descriptor: + def __init__(self, func): + self.func = func + + def __get__(self, instance, owner): + return self.func.__get__(instance, owner) + class TestPredicates(IsTestBase): @@ -665,10 +689,56 @@ def test_getdoc_inherited(self): self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), 'The automatic gainsaying.') + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc_inherited_class_doc(self): + class A: + """Common base class""" + class B(A): + pass + + a = A() + self.assertEqual(inspect.getdoc(A), 'Common base class') + self.assertEqual(inspect.getdoc(A, inherit_class_doc=False), + 'Common base class') + self.assertEqual(inspect.getdoc(a), 'Common base class') + self.assertIsNone(inspect.getdoc(a, fallback_to_class_doc=False)) + a.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(a, fallback_to_class_doc=False), + 'Instance') + + b = B() + self.assertEqual(inspect.getdoc(B), 'Common base class') + self.assertIsNone(inspect.getdoc(B, inherit_class_doc=False)) + self.assertIsNone(inspect.getdoc(b)) + self.assertIsNone(inspect.getdoc(b, fallback_to_class_doc=False)) + b.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + + def test_getdoc_inherited_cached_property(self): + doc = inspect.getdoc(mod3.ParentInheritDoc.foo) + self.assertEqual(doc, 'docstring for foo defined in parent') + self.assertEqual(inspect.getdoc(mod3.ChildInheritDoc.foo), doc) + self.assertEqual(inspect.getdoc(mod3.ChildInheritDefineDoc.foo), doc) + + def test_getdoc_redefine_cached_property_as_other(self): + self.assertEqual(inspect.getdoc(mod3.ChildPropertyFoo.foo), + 'docstring for the property foo') + self.assertEqual(inspect.getdoc(mod3.ChildMethodFoo.foo), + 'docstring for the method foo') + + def test_getdoc_define_cached_property(self): + self.assertEqual(inspect.getdoc(mod3.ChildDefineDoc.foo), + 'docstring for foo defined in child') + + def test_getdoc_nodoc_inherited(self): + self.assertIsNone(inspect.getdoc(mod3.ChildNoDoc.foo)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc self.assertEqual(finddoc(int), int.__doc__) + self.assertIsNone(finddoc(int, search_in_class=False)) self.assertEqual(finddoc(int.to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int().to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) @@ -786,12 +856,12 @@ def test_getfile(self): def test_getfile_builtin_module(self): with self.assertRaises(TypeError) as e: inspect.getfile(sys) - self.assertTrue(str(e.exception).startswith('<module')) + self.assertStartsWith(str(e.exception), '<module') def test_getfile_builtin_class(self): with self.assertRaises(TypeError) as e: inspect.getfile(int) - self.assertTrue(str(e.exception).startswith('<class')) + self.assertStartsWith(str(e.exception), '<class') def test_getfile_builtin_function_or_method(self): with self.assertRaises(TypeError) as e_abs: @@ -1189,12 +1259,22 @@ def test_nested_class_definition_inside_async_function(self): self.assertSourceEqual(run(mod2.func225), 226, 227) self.assertSourceEqual(mod2.cls226, 231, 235) + self.assertSourceEqual(mod2.cls226.func232, 232, 235) self.assertSourceEqual(run(mod2.cls226().func232), 233, 234) def test_class_definition_same_name_diff_methods(self): self.assertSourceEqual(mod2.cls296, 296, 298) self.assertSourceEqual(mod2.cls310, 310, 312) + def test_generator_expression(self): + self.assertSourceEqual(next(mod2.ge377), 377, 380) + self.assertSourceEqual(next(mod2.func383()), 385, 388) + + def test_comment_or_empty_line_after_decorator(self): + self.assertSourceEqual(mod2.func394, 392, 395) + self.assertSourceEqual(mod2.func400, 398, 401) + + class TestNoEOL(GetSourceBase): def setUp(self): self.tempdir = TESTFN + '_dir' @@ -1749,14 +1829,55 @@ class C(metaclass=M): class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): - from test.typinganndata.ann_module9 import ann, ann1 + from test.typinganndata.ann_module9 import A, ann, ann1 self.assertEqual(inspect.formatannotation(ann), 'List[str] | int') self.assertEqual(inspect.formatannotation(ann1), 'List[testModule.typing.A] | int') + self.assertEqual(inspect.formatannotation(A, 'testModule.typing'), 'A') + self.assertEqual(inspect.formatannotation(A, 'other'), 'testModule.typing.A') + self.assertEqual( + inspect.formatannotation(ann1, 'testModule.typing'), + 'List[testModule.typing.A] | int', + ) + def test_forwardref(self): fwdref = ForwardRef('fwdref') self.assertEqual(inspect.formatannotation(fwdref), 'fwdref') + def test_formatannotationrelativeto(self): + from test.typinganndata.ann_module9 import A, ann1 + + # Builtin types: + self.assertEqual( + inspect.formatannotationrelativeto(object)(type), + 'type', + ) + + # Custom types: + self.assertEqual( + inspect.formatannotationrelativeto(None)(A), + 'testModule.typing.A', + ) + + class B: ... + B.__module__ = 'testModule.typing' + + self.assertEqual( + inspect.formatannotationrelativeto(B)(A), + 'A', + ) + + self.assertEqual( + inspect.formatannotationrelativeto(object)(A), + 'testModule.typing.A', + ) + + # Not an instance of "type": + self.assertEqual( + inspect.formatannotationrelativeto(A)(ann1), + 'List[testModule.typing.A] | int', + ) + class TestIsMethodDescriptor(unittest.TestCase): @@ -2820,7 +2941,7 @@ async def asyncTearDown(self): @classmethod def tearDownClass(cls): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def _asyncgenstate(self): return inspect.getasyncgenstate(self.asyncgen) @@ -2949,7 +3070,7 @@ def test(po, /, pk, pkd=100, *args, ko, kod=10, **kwargs): pass sig = inspect.signature(test) - self.assertTrue(repr(sig).startswith('<Signature')) + self.assertStartsWith(repr(sig), '<Signature') self.assertTrue('(po, /, pk' in repr(sig)) # We need two functions, because it is impossible to represent @@ -2958,7 +3079,7 @@ def test2(pod=42, /): pass sig2 = inspect.signature(test2) - self.assertTrue(repr(sig2).startswith('<Signature')) + self.assertStartsWith(repr(sig2), '<Signature') self.assertTrue('(pod=42, /)' in repr(sig2)) po = sig.parameters['po'] @@ -4021,44 +4142,272 @@ def __init__(self, b): ('bar', 2, ..., "keyword_only")), ...)) - def test_signature_on_class_with_decorated_new(self): - def identity(func): - @functools.wraps(func) - def wrapped(*args, **kwargs): - return func(*args, **kwargs) - return wrapped - - class Foo: - @identity - def __new__(cls, a, b): + def test_signature_on_class_with_wrapped_metaclass_call(self): + class CM(type): + @identity_wrapper + def __call__(cls, a): + pass + class C(metaclass=CM): + def __init__(self, b): pass - self.assertEqual(self.signature(Foo), - ((('a', ..., ..., "positional_or_keyword"), - ('b', ..., ..., "positional_or_keyword")), + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), ...)) - self.assertEqual(self.signature(Foo.__new__), - ((('cls', ..., ..., "positional_or_keyword"), - ('a', ..., ..., "positional_or_keyword"), - ('b', ..., ..., "positional_or_keyword")), - ...)) + with self.subTest('classmethod'): + class CM(type): + @classmethod + @identity_wrapper + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass - class Bar: - __new__ = identity(object.__new__) + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) - varargs_signature = ( - (('args', ..., ..., 'var_positional'), - ('kwargs', ..., ..., 'var_keyword')), - ..., - ) + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + @identity_wrapper + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass - self.assertEqual(self.signature(Bar), ((), ...)) - self.assertEqual(self.signature(Bar.__new__), varargs_signature) - self.assertEqual(self.signature(Bar, follow_wrapped=False), - varargs_signature) - self.assertEqual(self.signature(Bar.__new__, follow_wrapped=False), - varargs_signature) + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class CM(type): + @custom_descriptor + @identity_wrapper + def __call__(self, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__call__), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + self.assertEqual(self.signature(C.__call__, follow_wrapped=False), + varargs_signature) + + def test_signature_on_class_with_wrapped_init(self): + class C: + @identity_wrapper + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + @identity_wrapper + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + @identity_wrapper + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, a): + pass + + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(identity_wrapper(lambda x, a, b: None), 2) + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + @identity_wrapper + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class C: + @custom_descriptor + @identity_wrapper + def __init__(self, a): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__init__), + ((('self', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + if support.MISSING_C_DOCSTRINGS: + self.assertRaisesRegex( + ValueError, "no signature found", + self.signature, C.__new__, follow_wrapped=False, + ) + else: + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) + + def test_signature_on_class_with_wrapped_new(self): + with self.subTest('FunctionType'): + class C: + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + @identity_wrapper + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(identity_wrapper(lambda x, cls, a: (x, a)), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(identity_wrapper(lambda cls, x, a: (x, a)), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class C: + @custom_descriptor + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__new__), + ((('cls', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) def test_signature_on_class_with_init(self): class C: @@ -4997,6 +5346,37 @@ def test_signature_annotation_format(self): with self.assertRaisesRegex(NameError, "undefined"): signature_func(ida.f) + def test_signature_deferred_annotations(self): + def f(x: undef): + pass + + class C: + x: undef + + def __init__(self, x: undef): + self.x = x + + sig = inspect.signature(f, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + sig = inspect.signature(C, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + + class CallableWrapper: + def __init__(self, func): + self.func = func + self.__annotate__ = func.__annotate__ + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + @property + def __annotations__(self): + return self.__annotate__(Format.VALUE) + + cw = CallableWrapper(f) + sig = inspect.signature(cw, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['args', 'kwargs']) + def test_signature_none_annotation(self): class funclike: # Has to be callable, and have correct @@ -5102,7 +5482,7 @@ def test_signature_parameter_object(self): with self.assertRaisesRegex(ValueError, 'cannot have default values'): p.replace(kind=inspect.Parameter.VAR_POSITIONAL) - self.assertTrue(repr(p).startswith('<Parameter')) + self.assertStartsWith(repr(p), '<Parameter') self.assertTrue('"a=42"' in repr(p)) def test_signature_parameter_hashable(self): @@ -5755,6 +6135,7 @@ def test_types_module_has_signatures(self): 'AsyncGeneratorType': {'athrow'}, 'CoroutineType': {'throw'}, 'GeneratorType': {'throw'}, + 'FrameLocalsProxyType': {'setdefault', 'pop', 'get'}, } self._test_module_has_signatures(types, unsupported_signature=unsupported_signature, @@ -5799,6 +6180,21 @@ def test_collections_abc_module_has_signatures(self): import collections.abc self._test_module_has_signatures(collections.abc) + def test_datetime_module_has_signatures(self): + # Only test if the C implementation is available. + import_helper.import_module('_datetime') + import datetime + no_signature = {'tzinfo'} + unsupported_signature = {'timezone'} + methods_unsupported_signature = { + 'date': {'replace'}, + 'time': {'replace'}, + 'datetime': {'replace', 'combine'}, + } + self._test_module_has_signatures(datetime, + no_signature, unsupported_signature, + methods_unsupported_signature=methods_unsupported_signature) + def test_errno_module_has_signatures(self): import errno self._test_module_has_signatures(errno) @@ -5844,9 +6240,9 @@ def test_operator_module_has_signatures(self): self._test_module_has_signatures(operator) def test_os_module_has_signatures(self): - unsupported_signature = {'chmod', 'link', 'utime'} + unsupported_signature = {'chmod', 'utime'} unsupported_signature |= {name for name in - ['get_terminal_size', 'posix_spawn', 'posix_spawnp', + ['get_terminal_size', 'link', 'posix_spawn', 'posix_spawnp', 'register_at_fork', 'startfile'] if hasattr(os, name)} self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) @@ -5885,6 +6281,7 @@ def test_sysconfig_module_has_signatures(self): def test_threading_module_has_signatures(self): import threading self._test_module_has_signatures(threading) + self.assertIsNotNone(inspect.signature(threading.__excepthook__)) def test_thread_module_has_signatures(self): import _thread @@ -6146,12 +6543,14 @@ def spawn_repl(self, *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ** object. """ + # TODO(picnixz): refactor this as it's used by test_repl.py + # To run the REPL without using a terminal, spawn python with the command # line option '-i' and the process name set to '<stdin>'. # The directory of argv[0] must match the directory of the Python # executable for the Popen() call to python to succeed as the directory - # path may be used by Py_GetPath() to build the default module search - # path. + # path may be used by PyConfig_Get("module_search_paths") to build the + # default module search path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "<stdin>") cmd_line = [stdin_fname, '-E', '-i'] cmd_line.extend(args) diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 245528ce57a..7a7cb73f673 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -836,7 +836,7 @@ def test_pylong_roundtrip(self): n = hibit | getrandbits(bits - 1) assert n.bit_length() == bits sn = str(n) - self.assertFalse(sn.startswith('0')) + self.assertNotStartsWith(sn, '0') self.assertEqual(n, int(sn)) bits <<= 1 diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 66c7afce88f..fd9e46bf335 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,18 +1,25 @@ +import contextlib import os import pickle +import signal +import sys from textwrap import dedent import threading import types import unittest from test import support +from test.support import os_helper +from test.support import script_helper from test.support import import_helper +from test.support.script_helper import assert_python_ok, spawn_python # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_interpreters') +from concurrent import interpreters from test.support import Py_GIL_DISABLED -from test.support import interpreters from test.support import force_not_colorized -from test.support.interpreters import ( +import test._crossinterp_definitions as defs +from concurrent.interpreters import ( InterpreterError, InterpreterNotFoundError, ExecutionFailed, ) from .utils import ( @@ -29,6 +36,59 @@ WHENCE_STR_STDLIB = '_interpreters module' +def is_pickleable(obj): + try: + pickle.dumps(obj) + except Exception: + return False + return True + + +@contextlib.contextmanager +def defined_in___main__(name, script, *, remove=False): + import __main__ as mainmod + mainns = vars(mainmod) + assert name not in mainns + exec(script, mainns, mainns) + if remove: + yield mainns.pop(name) + else: + try: + yield mainns[name] + finally: + mainns.pop(name, None) + + +def build_excinfo(exctype, msg=None, formatted=None, errdisplay=None): + if isinstance(exctype, type): + assert issubclass(exctype, BaseException), exctype + exctype = types.SimpleNamespace( + __name__=exctype.__name__, + __qualname__=exctype.__qualname__, + __module__=exctype.__module__, + ) + elif isinstance(exctype, str): + module, _, name = exctype.rpartition(exctype) + if not module and name in __builtins__: + module = 'builtins' + exctype = types.SimpleNamespace( + __name__=name, + __qualname__=exctype, + __module__=module or None, + ) + else: + assert isinstance(exctype, types.SimpleNamespace) + assert msg is None or isinstance(msg, str), msg + assert formatted is None or isinstance(formatted, str), formatted + assert errdisplay is None or isinstance(errdisplay, str), errdisplay + return types.SimpleNamespace( + type=exctype, + msg=msg, + formatted=formatted, + errdisplay=errdisplay, + ) + + class ModuleTests(TestBase): def test_queue_aliases(self): @@ -75,7 +135,7 @@ def test_in_subinterpreter(self): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters interp = interpreters.create() print(interp.id) """)) @@ -138,7 +198,7 @@ def test_subinterpreter(self): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(cur.id) """)) @@ -155,7 +215,7 @@ def test_idempotent(self): with self.subTest('subinterpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) cur = interpreters.get_current() @@ -167,7 +227,7 @@ def test_idempotent(self): with self.subTest('per-interpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) """)) @@ -354,9 +414,57 @@ def test_equality(self): def test_pickle(self): interp = interpreters.create() - data = pickle.dumps(interp) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, interp) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(interp, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) + + @support.requires_subprocess() + @force_not_colorized + def test_cleanup_in_repl(self): + # GH-135729: Using a subinterpreter in the REPL would lead to an unraisable + # exception during finalization + repl = script_helper.spawn_python("-i") + script = b"""if True: + from concurrent import interpreters + interpreters.create() + exit()""" + stdout, stderr = repl.communicate(script) + self.assertIsNone(stderr) + self.assertIn(b"Interpreter.close()", stdout) + self.assertNotIn(b"Traceback", stdout) + + @support.requires_subprocess() + @unittest.skipIf(os.name == 'nt', "signals don't work well on windows") + def test_keyboard_interrupt_in_thread_running_interp(self): + import subprocess + source = f"""if True: + from concurrent import interpreters + from threading import Thread + + def test(): + import time + print('a', flush=True, end='') + time.sleep(10) + + interp = interpreters.create() + interp.call_in_thread(test) + """ + + with spawn_python("-c", source, stderr=subprocess.PIPE) as proc: + self.assertEqual(proc.stdout.read(1), b'a') + proc.send_signal(signal.SIGINT) + proc.stderr.flush() + error = proc.stderr.read() + self.assertIn(b"KeyboardInterrupt", error) + retcode = proc.wait() + # Sometimes we send the SIGINT after the subthread yields the GIL to + # the main thread, which results in the main thread getting the + # KeyboardInterrupt before finalization is reached. There's not + # any great way to protect against that, so we just allow a -2 + # return code as well. + self.assertIn(retcode, (0, -signal.SIGINT)) class TestInterpreterIsRunning(TestBase): @@ -524,7 +632,7 @@ def test_from_current(self): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(f""" - from test.support import interpreters + from concurrent import interpreters interp = interpreters.Interpreter({interp.id}) try: interp.close() @@ -541,7 +649,7 @@ def test_from_sibling(self): self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) interp1.exec(dedent(f""" - from test.support import interpreters + from concurrent import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() interp3 = interpreters.create() @@ -647,6 +755,68 @@ def test_created_with_capi(self): self.interp_exists(interpid)) + def test_remaining_threads(self): + r_interp, w_interp = self.pipe() + + FINISHED = b'F' + + # It's unlikely, but technically speaking, it's possible + # that the thread could've finished before interp.close() is + # reached, so this test might not properly exercise the case. + # However, it's quite unlikely and probably not worth bothering about. + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + import time + + def task(): + time.sleep(1) + os.write({w_interp}, {FINISHED!r}) + + threads = (threading.Thread(target=task) for _ in range(3)) + for t in threads: + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_remaining_daemon_threads(self): + # Daemon threads leak reference by nature, because they hang threads + # without allowing them to do cleanup (i.e., release refs). + # To prevent that from messing up the refleak hunter and whatnot, we + # run this in a subprocess. + code = '''if True: + import _interpreters + import types + interp = _interpreters.create( + types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=True, + gil='own', + ) + ) + _interpreters.exec(interp, f"""if True: + import threading + import time + + def task(): + time.sleep(3) + + threads = (threading.Thread(target=task, daemon=True) for _ in range(3)) + for t in threads: + t.start() + """) + _interpreters.destroy(interp) + ''' + assert_python_ok('-c', code) + + class TestInterpreterPrepareMain(TestBase): def test_empty(self): @@ -748,14 +918,17 @@ def eggs(): ham() """) scriptfile = self.make_script('script.py', tempdir, text=""" - from test.support import interpreters + from concurrent import interpreters def script(): import spam spam.eggs() interp = interpreters.create() - interp.exec(script) + try: + interp.exec(script) + finally: + interp.close() """) stdout, stderr = self.assert_python_failure(scriptfile) @@ -764,12 +937,12 @@ def script(): # File "{interpreters.__file__}", line 179, in exec self.assertEqual(stderr, dedent(f"""\ Traceback (most recent call last): - File "{scriptfile}", line 9, in <module> + File "{scriptfile}", line 10, in <module> interp.exec(script) ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} raise ExecutionFailed(excinfo) - test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! + concurrent.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -839,9 +1012,16 @@ def test_bad_script(self): interp.exec(10) def test_bytes_for_script(self): + r, w = self.pipe() + RAN = b'R' + DONE = b'D' interp = interpreters.create() - with self.assertRaises(TypeError): - interp.exec(b'print("spam")') + interp.exec(f"""if True: + import os + os.write({w}, {RAN!r}) + """) + os.write(w, DONE) + self.assertEqual(os.read(r, 1), RAN) def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -879,28 +1059,46 @@ def test_created_with_capi(self): with self.assertRaisesRegex(InterpreterError, 'unrecognized'): interp.exec('raise Exception("it worked!")') + def test_list_comprehension(self): + # gh-135450: List comprehensions caused an assertion failure + # in _PyCode_CheckNoExternalState() + import string + r_interp, w_interp = self.pipe() + + interp = interpreters.create() + interp.exec(f"""if True: + import os + comp = [str(i) for i in range(10)] + os.write({w_interp}, ''.join(comp).encode()) + """) + self.assertEqual(os.read(r_interp, 10).decode(), string.digits) + interp.close() + + # test__interpreters covers the remaining # Interpreter.exec() behavior. -def call_func_noop(): - pass +call_func_noop = defs.spam_minimal +call_func_ident = defs.spam_returns_arg +call_func_failure = defs.spam_raises def call_func_return_shareable(): return (1, None) -def call_func_return_not_shareable(): +def call_func_return_stateless_func(): + return (lambda x: x) + + +def call_func_return_pickleable(): return [1, 2, 3] -def call_func_failure(): - raise Exception('spam!') - - -def call_func_ident(value): - return value +def call_func_return_unpickleable(): + x = 42 + return (lambda: x) def get_call_func_closure(value): @@ -909,6 +1107,11 @@ def call_func_closure(): return call_func_closure +def call_func_exec_wrapper(script, ns): + res = exec(script, ns, ns) + return res, ns, id(ns) + + class Spam: @staticmethod @@ -1005,86 +1208,663 @@ class TestInterpreterCall(TestBase): # - preserves info (e.g. SyntaxError) # - matching error display - def test_call(self): + @contextlib.contextmanager + def assert_fails(self, expected): + with self.assertRaises(ExecutionFailed) as cm: + yield cm + uncaught = cm.exception.excinfo + self.assertEqual(uncaught.type.__name__, expected.__name__) + + def assert_fails_not_shareable(self): + return self.assert_fails(interpreters.NotShareableError) + + def assert_code_equal(self, code1, code2): + if code1 == code2: + return + self.assertEqual(code1.co_name, code2.co_name) + self.assertEqual(code1.co_flags, code2.co_flags) + self.assertEqual(code1.co_consts, code2.co_consts) + self.assertEqual(code1.co_varnames, code2.co_varnames) + self.assertEqual(code1.co_cellvars, code2.co_cellvars) + self.assertEqual(code1.co_freevars, code2.co_freevars) + self.assertEqual(code1.co_names, code2.co_names) + self.assertEqual( + _testinternalcapi.get_code_var_counts(code1), + _testinternalcapi.get_code_var_counts(code2), + ) + self.assertEqual(code1.co_code, code2.co_code) + + def assert_funcs_equal(self, func1, func2): + if func1 == func2: + return + self.assertIs(type(func1), type(func2)) + self.assertEqual(func1.__name__, func2.__name__) + self.assertEqual(func1.__defaults__, func2.__defaults__) + self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__) + self.assertEqual(func1.__closure__, func2.__closure__) + self.assert_code_equal(func1.__code__, func2.__code__) + self.assertEqual( + _testinternalcapi.get_code_var_counts(func1), + _testinternalcapi.get_code_var_counts(func2), + ) + + def assert_exceptions_equal(self, exc1, exc2): + assert isinstance(exc1, Exception) + assert isinstance(exc2, Exception) + if exc1 == exc2: + return + self.assertIs(type(exc1), type(exc2)) + self.assertEqual(exc1.args, exc2.args) + + def test_stateless_funcs(self): interp = interpreters.create() - for i, (callable, args, kwargs) in enumerate([ - (call_func_noop, (), {}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), - (Spam.noop, (), {}), - ]): - with self.subTest(f'success case #{i+1}'): - res = interp.call(callable) - self.assertIs(res, None) + func = call_func_noop + with self.subTest('no args, no return'): + res = interp.call(func) + self.assertIsNone(res) - for i, (callable, args, kwargs) in enumerate([ - (call_func_ident, ('spamspamspam',), {}), - (get_call_func_closure, (42,), {}), - (get_call_func_closure(42), (), {}), - (Spam.from_values, (), {}), - (Spam.from_values, (1, 2, 3), {}), - (Spam, ('???'), {}), - (Spam(101), (), {}), - (Spam(10101).run, (), {}), - (call_func_complex, ('ident', 'spam'), {}), - (call_func_complex, ('full-ident', 'spam'), {}), - (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), - (call_func_complex, ('globals',), {}), - (call_func_complex, ('interpid',), {}), - (call_func_complex, ('closure',), {'value': '~~~'}), - (call_func_complex, ('custom', 'spam!'), {}), - (call_func_complex, ('custom-inner', 'eggs!'), {}), - (call_func_complex, ('???',), {'exc': ValueError('spam')}), - ]): - with self.subTest(f'invalid case #{i+1}'): - with self.assertRaises(Exception): - if args or kwargs: - raise Exception((args, kwargs)) - interp.call(callable) + func = call_func_return_shareable + with self.subTest('no args, returns shareable'): + res = interp.call(func) + self.assertEqual(res, (1, None)) + func = call_func_return_stateless_func + expected = (lambda x: x) + with self.subTest('no args, returns stateless func'): + res = interp.call(func) + self.assert_funcs_equal(res, expected) + + func = call_func_return_pickleable + with self.subTest('no args, returns pickleable'): + res = interp.call(func) + self.assertEqual(res, [1, 2, 3]) + + func = call_func_return_unpickleable + with self.subTest('no args, returns unpickleable'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + def test_stateless_func_returns_arg(self): + interp = interpreters.create() + + for arg in [ + None, + 10, + 'spam!', + b'spam!', + (1, 2, 'spam!'), + memoryview(b'spam!'), + ]: + with self.subTest(f'shareable {arg!r}'): + assert _interpreters.is_shareable(arg) + res = interp.call(defs.spam_returns_arg, arg) + self.assertEqual(res, arg) + + for arg in defs.STATELESS_FUNCTIONS: + with self.subTest(f'stateless func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + + for arg in defs.TOP_FUNCTIONS: + if arg in defs.STATELESS_FUNCTIONS: + continue + with self.subTest(f'stateful func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + assert is_pickleable(arg) + + for arg in [ + Ellipsis, + NotImplemented, + object(), + 2**1000, + [1, 2, 3], + {'a': 1, 'b': 2}, + types.SimpleNamespace(x=42), + # builtin types + object, + type, + Exception, + ModuleNotFoundError, + # builtin exceptions + Exception('uh-oh!'), + ModuleNotFoundError('mymodule'), + # builtin fnctions + len, + sys.exit, + # user classes + *defs.TOP_CLASSES, + *(c(*a) for c, a in defs.TOP_CLASSES.items() + if c not in defs.CLASSES_WITHOUT_EQUALITY), + ]: + with self.subTest(f'pickleable {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + if type(arg) is object: + self.assertIs(type(res), object) + elif isinstance(arg, BaseException): + self.assert_exceptions_equal(res, arg) + else: + self.assertEqual(res, arg) + assert is_pickleable(arg) + + for arg in [ + types.MappingProxyType({}), + *(f for f in defs.NESTED_FUNCTIONS + if f not in defs.STATELESS_FUNCTIONS), + ]: + with self.subTest(f'unpickleable {arg!r}'): + assert not _interpreters.is_shareable(arg) + assert not is_pickleable(arg) + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_full_args(self): + interp = interpreters.create() + expected = (1, 2, 3, 4, 5, 6, ('?',), {'g': 7, 'h': 8}) + func = defs.spam_full_args + res = interp.call(func, 1, 2, 3, 4, '?', e=5, f=6, g=7, h=8) + self.assertEqual(res, expected) + + def test_full_defaults(self): + # pickleable, but not stateless + interp = interpreters.create() + expected = (-1, -2, -3, -4, -5, -6, (), {'g': 8, 'h': 9}) + res = interp.call(defs.spam_full_args_with_defaults, g=8, h=9) + self.assertEqual(res, expected) + + def test_modified_arg(self): + interp = interpreters.create() + script = dedent(""" + a = 7 + b = 2 + c = a ** b + """) + ns = {} + expected = {'a': 7, 'b': 2, 'c': 49} + res = interp.call(call_func_exec_wrapper, script, ns) + obj, resns, resid = res + del resns['__builtins__'] + self.assertIsNone(obj) + self.assertEqual(ns, {}) + self.assertEqual(resns, expected) + self.assertNotEqual(resid, id(ns)) + self.assertNotEqual(resid, id(resns)) + + def test_func_in___main___valid(self): + # pickleable, already there' + + with os_helper.temp_dir() as tempdir: + def new_mod(name, text): + script_helper.make_script(tempdir, name, dedent(text)) + + def run(text): + name = 'myscript' + text = dedent(f""" + import sys + sys.path.insert(0, {tempdir!r}) + + """) + dedent(text) + filename = script_helper.make_script(tempdir, name, text) + res = script_helper.assert_python_ok(filename) + return res.out.decode('utf-8').strip() + + # no module indirection + with self.subTest('no indirection'): + text = run(f""" + from concurrent import interpreters + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = interp.call(spam) + print(res) + """) + self.assertEqual(text, '<fake __main__>') + + # indirect as func, direct interp + new_mod('mymod', f""" + def run(interp, func): + return interp.call(func) + """) + with self.subTest('indirect as func, direct interp'): + text = run(f""" + from concurrent import interpreters + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = mymod.run(interp, spam) + print(res) + """) + self.assertEqual(text, '<fake __main__>') + + # indirect as func, indirect interp + new_mod('mymod', f""" + from concurrent import interpreters + def run(func): + interp = interpreters.create() + return interp.call(func) + """) + with self.subTest('indirect as func, indirect interp'): + text = run(f""" + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + res = mymod.run(spam) + print(res) + """) + self.assertEqual(text, '<fake __main__>') + + def test_func_in___main___invalid(self): + interp = interpreters.create() + + funcname = f'{__name__.replace(".", "_")}_spam_okay' + script = dedent(f""" + def {funcname}(): + # This a global var... + return __name__ + """) + + with self.subTest('pickleable, added dynamically'): + with defined_in___main__(funcname, script) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + with self.subTest('lying about __main__'): + with defined_in___main__(funcname, script, remove=True) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_func_in___main___hidden(self): + # When a top-level function that uses global variables is called + # through Interpreter.call(), it will be pickled, sent over, + # and unpickled. That requires that it be found in the other + # interpreter's __main__ module. However, the original script + # that defined the function is only run in the main interpreter, + # so pickle.loads() would normally fail. + # + # We work around this by running the script in the other + # interpreter. However, this is a one-off solution for the sake + # of unpickling, so we avoid modifying that interpreter's + # __main__ module by running the script in a hidden module. + # + # In this test we verify that the function runs with the hidden + # module as its __globals__ when called in the other interpreter, + # and that the interpreter's __main__ module is unaffected. + text = dedent(""" + eggs = True + + def spam(*, explicit=False): + if explicit: + import __main__ + ns = __main__.__dict__ + else: + # For now we have to have a LOAD_GLOBAL in the + # function in order for globals() to actually return + # spam.__globals__. Maybe it doesn't go through pickle? + # XXX We will fix this later. + spam + ns = globals() + + func = ns.get('spam') + return [ + id(ns), + ns.get('__name__'), + ns.get('__file__'), + id(func), + None if func is None else repr(func), + ns.get('eggs'), + ns.get('ham'), + ] + + if __name__ == "__main__": + from concurrent import interpreters + interp = interpreters.create() + + ham = True + print([ + [ + spam(explicit=True), + spam(), + ], + [ + interp.call(spam, explicit=True), + interp.call(spam), + ], + ]) + """) + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + local, remote = eval(stdout) + + # In the main interpreter. + main, unpickled = local + nsid, _, _, funcid, func, _, _ = main + self.assertEqual(main, [ + nsid, + '__main__', + filename, + funcid, + func, + True, + True, + ]) + self.assertIsNot(func, None) + self.assertRegex(func, '^<function spam at 0x.*>$') + self.assertEqual(unpickled, main) + + # In the subinterpreter. + main, unpickled = remote + nsid1, _, _, funcid1, _, _, _ = main + self.assertEqual(main, [ + nsid1, + '__main__', + None, + funcid1, + None, + None, + None, + ]) + nsid2, _, _, funcid2, func, _, _ = unpickled + self.assertEqual(unpickled, [ + nsid2, + '<fake __main__>', + filename, + funcid2, + func, + True, + None, + ]) + self.assertIsNot(func, None) + self.assertRegex(func, '^<function spam at 0x.*>$') + self.assertNotEqual(nsid2, nsid1) + self.assertNotEqual(funcid2, funcid1) + + def test_func_in___main___uses_globals(self): + # See the note in test_func_in___main___hidden about pickle + # and the __main__ module. + # + # Additionally, the solution to that problem must provide + # for global variables on which a pickled function might rely. + # + # To check that, we run a script that has two global functions + # and a global variable in the __main__ module. One of the + # functions sets the global variable and the other returns + # the value. + # + # The script calls those functions multiple times in another + # interpreter, to verify the following: + # + # * the global variable is properly initialized + # * the global variable retains state between calls + # * the setter modifies that persistent variable + # * the getter uses the variable + # * the calls in the other interpreter do not modify + # the main interpreter + # * those calls don't modify the interpreter's __main__ module + # * the functions and variable do not actually show up in the + # other interpreter's __main__ module + text = dedent(""" + count = 0 + + def inc(x=1): + global count + count += x + + def get_count(): + return count + + if __name__ == "__main__": + counts = [] + results = [count, counts] + + from concurrent import interpreters + interp = interpreters.create() + + val = interp.call(get_count) + counts.append(val) + + interp.call(inc) + val = interp.call(get_count) + counts.append(val) + + interp.call(inc, 3) + val = interp.call(get_count) + counts.append(val) + + results.append(count) + + modified = {name: interp.call(eval, f'{name!r} in vars()') + for name in ('count', 'inc', 'get_count')} + results.append(modified) + + print(results) + """) + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + before, counts, after, modified = eval(stdout) + self.assertEqual(modified, { + 'count': False, + 'inc': False, + 'get_count': False, + }) + self.assertEqual(before, 0) + self.assertEqual(after, 0) + self.assertEqual(counts, [0, 1, 4]) + + def test_raises(self): + interp = interpreters.create() with self.assertRaises(ExecutionFailed): interp.call(call_func_failure) + with self.assert_fails(ValueError): + interp.call(call_func_complex, '???', exc=ValueError('spam')) + + def test_call_valid(self): + interp = interpreters.create() + + for i, (callable, args, kwargs, expected) in enumerate([ + (call_func_noop, (), {}, None), + (call_func_ident, ('spamspamspam',), {}, 'spamspamspam'), + (call_func_return_shareable, (), {}, (1, None)), + (call_func_return_pickleable, (), {}, [1, 2, 3]), + (Spam.noop, (), {}, None), + (Spam.from_values, (), {}, Spam(())), + (Spam.from_values, (1, 2, 3), {}, Spam((1, 2, 3))), + (Spam, ('???',), {}, Spam('???')), + (Spam(101), (), {}, (101, (), {})), + (Spam(10101).run, (), {}, (10101, (), {})), + (call_func_complex, ('ident', 'spam'), {}, 'spam'), + (call_func_complex, ('full-ident', 'spam'), {}, ('spam', (), {})), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}, + ('spam', ('ham',), {'eggs': '!!!'})), + (call_func_complex, ('globals',), {}, __name__), + (call_func_complex, ('interpid',), {}, interp.id), + (call_func_complex, ('custom', 'spam!'), {}, Spam('spam!')), + ]): + with self.subTest(f'success case #{i+1}'): + res = interp.call(callable, *args, **kwargs) + self.assertEqual(res, expected) + + def test_call_invalid(self): + interp = interpreters.create() + + func = get_call_func_closure + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, 42) + + func = get_call_func_closure(42) + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + func = call_func_complex + op = 'closure' + with self.subTest(f'{func} ({op})'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, op, value='~~~') + + op = 'custom-inner' + with self.subTest(f'{func} ({op})'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, op, 'eggs!') + + def test_callable_requires_frame(self): + # There are various functions that require a current frame. + interp = interpreters.create() + for call, expected in [ + ((eval, '[1, 2, 3]'), + [1, 2, 3]), + ((eval, 'sum([1, 2, 3])'), + 6), + ((exec, '...'), + None), + ]: + with self.subTest(str(call)): + res = interp.call(*call) + self.assertEqual(res, expected) + + result_not_pickleable = [ + globals, + locals, + vars, + ] + for func, expectedtype in { + globals: dict, + locals: dict, + vars: dict, + dir: list, + }.items(): + with self.subTest(str(func)): + if func in result_not_pickleable: + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + else: + res = interp.call(func) + self.assertIsInstance(res, expectedtype) + self.assertIn('__builtins__', res) + + def test_globals_from_builtins(self): + # The builtins exec(), eval(), globals(), locals(), vars(), + # and dir() each runs relative to the target interpreter's + # __main__ module, when called directly. However, + # globals(), locals(), and vars() don't work when called + # directly so we don't check them. + from _frozen_importlib import BuiltinImporter + interp = interpreters.create() + + names = interp.call(dir) + self.assertEqual(names, [ + '__builtins__', + '__doc__', + '__loader__', + '__name__', + '__package__', + '__spec__', + ]) + + values = {name: interp.call(eval, name) + for name in names if name != '__builtins__'} + self.assertEqual(values, { + '__name__': '__main__', + '__doc__': None, + '__spec__': None, # It wasn't imported, so no module spec? + '__package__': None, + '__loader__': BuiltinImporter, + }) + with self.assertRaises(ExecutionFailed): + interp.call(eval, 'spam'), + + interp.call(exec, f'assert dir() == {names}') + + # Update the interpreter's __main__. + interp.prepare_main(spam=42) + expected = names + ['spam'] + + names = interp.call(dir) + self.assertEqual(names, expected) + + value = interp.call(eval, 'spam') + self.assertEqual(value, 42) + + interp.call(exec, f'assert dir() == {expected}, dir()') + + def test_globals_from_stateless_func(self): + # A stateless func, which doesn't depend on any globals, + # doesn't go through pickle, so it runs in __main__. + def set_global(name, value): + globals()[name] = value + + def get_global(name): + return globals().get(name) + + interp = interpreters.create() + + modname = interp.call(get_global, '__name__') + self.assertEqual(modname, '__main__') + + res = interp.call(get_global, 'spam') + self.assertIsNone(res) + + interp.exec('spam = True') + res = interp.call(get_global, 'spam') + self.assertTrue(res) + + interp.call(set_global, 'spam', 42) + res = interp.call(get_global, 'spam') + self.assertEqual(res, 42) + + interp.exec('assert spam == 42, repr(spam)') + def test_call_in_thread(self): interp = interpreters.create() for i, (callable, args, kwargs) in enumerate([ (call_func_noop, (), {}), (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), - (Spam.noop, (), {}), - ]): - with self.subTest(f'success case #{i+1}'): - with self.captured_thread_exception() as ctx: - t = interp.call_in_thread(callable) - t.join() - self.assertIsNone(ctx.caught) - - for i, (callable, args, kwargs) in enumerate([ - (call_func_ident, ('spamspamspam',), {}), - (get_call_func_closure, (42,), {}), - (get_call_func_closure(42), (), {}), + (call_func_return_pickleable, (), {}), (Spam.from_values, (), {}), (Spam.from_values, (1, 2, 3), {}), - (Spam, ('???'), {}), (Spam(101), (), {}), (Spam(10101).run, (), {}), + (Spam.noop, (), {}), (call_func_complex, ('ident', 'spam'), {}), (call_func_complex, ('full-ident', 'spam'), {}), (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), (call_func_complex, ('globals',), {}), (call_func_complex, ('interpid',), {}), - (call_func_complex, ('closure',), {'value': '~~~'}), (call_func_complex, ('custom', 'spam!'), {}), - (call_func_complex, ('custom-inner', 'eggs!'), {}), - (call_func_complex, ('???',), {'exc': ValueError('spam')}), + ]): + with self.subTest(f'success case #{i+1}'): + with self.captured_thread_exception() as ctx: + t = interp.call_in_thread(callable, *args, **kwargs) + t.join() + self.assertIsNone(ctx.caught) + + for i, (callable, args, kwargs) in enumerate([ + (get_call_func_closure, (42,), {}), + (get_call_func_closure(42), (), {}), ]): with self.subTest(f'invalid case #{i+1}'): - if args or kwargs: - continue with self.captured_thread_exception() as ctx: - t = interp.call_in_thread(callable) + t = interp.call_in_thread(callable, *args, **kwargs) t.join() self.assertIsNotNone(ctx.caught) @@ -1452,6 +2232,14 @@ def test_destroy(self): self.assertFalse( self.interp_exists(interpid)) + with self.subTest('basic C-API'): + interpid = _testinternalcapi.create_interpreter() + self.assertTrue( + self.interp_exists(interpid)) + _testinternalcapi.destroy_interpreter(interpid, basic=True) + self.assertFalse( + self.interp_exists(interpid)) + def test_get_config(self): # This test overlaps with # test.test_capi.test_misc.InterpreterConfigTests. @@ -1529,6 +2317,16 @@ def test_whence(self): whence = eval(text) self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI) + def test_contextvars_missing(self): + script = f""" + import contextvars + print(getattr(contextvars.Token, "MISSING", "'doesn't exist'")) + """ + + orig = _interpreters.create() + text = self.run_and_capture(orig, script) + self.assertEqual(text.strip(), "<Token.MISSING>") + def test_is_running(self): def check(interpid, expected): with self.assertRaisesRegex(InterpreterError, 'unrecognized'): @@ -1585,18 +2383,14 @@ def test_exec(self): with results: exc = _interpreters.exec(interpid, script) out = results.stdout() - self.assertEqual(out, '') - self.assert_ns_equal(exc, types.SimpleNamespace( - type=types.SimpleNamespace( - __name__='Exception', - __qualname__='Exception', - __module__='builtins', - ), - msg='uh-oh!', + expected = build_excinfo( + Exception, 'uh-oh!', # We check these in other tests. formatted=exc.formatted, errdisplay=exc.errdisplay, - )) + ) + self.assertEqual(out, '') + self.assert_ns_equal(exc, expected) with self.subTest('from C-API'): with self.interpreter_from_capi() as interpid: @@ -1608,25 +2402,50 @@ def test_exec(self): self.assertEqual(exc.msg, 'it worked!') def test_call(self): - with self.subTest('no args'): - interpid = _interpreters.create() - exc = _interpreters.call(interpid, call_func_return_shareable) - self.assertIs(exc, None) + interpid = _interpreters.create() + + # Here we focus on basic args and return values. + # See TestInterpreterCall for full operational coverage, + # including supported callables. + + with self.subTest('no args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('empty args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func, (), {}) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('no args, return non-None'): + func = defs.script_with_return + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIs(res, True) + + with self.subTest('full args, return non-None'): + expected = (1, 2, 3, 4, 5, 6, (7, 8), {'g': 9, 'h': 0}) + func = defs.spam_full_args + args = (1, 2, 3, 4, 7, 8) + kwargs = dict(e=5, f=6, g=9, h=0) + res, exc = _interpreters.call(interpid, func, args, kwargs) + self.assertIsNone(exc) + self.assertEqual(res, expected) with self.subTest('uncaught exception'): - interpid = _interpreters.create() - exc = _interpreters.call(interpid, call_func_failure) - self.assertEqual(exc, types.SimpleNamespace( - type=types.SimpleNamespace( - __name__='Exception', - __qualname__='Exception', - __module__='builtins', - ), - msg='spam!', + func = defs.spam_raises + res, exc = _interpreters.call(interpid, func) + expected = build_excinfo( + Exception, 'spam!', # We check these in other tests. formatted=exc.formatted, errdisplay=exc.errdisplay, - )) + ) + self.assertIsNone(res) + self.assertEqual(exc, expected) @requires_test_modules def test_set___main___attrs(self): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index eada18f99d0..52827357078 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -8,8 +8,8 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _channels = import_helper.import_module('_interpchannels') -from test.support import interpreters -from test.support.interpreters import channels +from concurrent import interpreters +from test.support import channels from .utils import _run_output, TestBase @@ -121,9 +121,11 @@ def test_equality(self): def test_pickle(self): ch, _ = channels.create() - data = pickle.dumps(ch) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, ch) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) class TestSendChannelAttrs(TestBase): @@ -152,9 +154,11 @@ def test_equality(self): def test_pickle(self): _, ch = channels.create() - data = pickle.dumps(ch) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, ch) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) class TestSendRecv(TestBase): @@ -171,7 +175,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -244,7 +248,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -377,17 +381,17 @@ def common(rch, sch, unbound=None, presize=0): if not unbound: extraargs = '' elif unbound is channels.UNBOUND: - extraargs = ', unbound=channels.UNBOUND' + extraargs = ', unbounditems=channels.UNBOUND' elif unbound is channels.UNBOUND_ERROR: - extraargs = ', unbound=channels.UNBOUND_ERROR' + extraargs = ', unbounditems=channels.UNBOUND_ERROR' elif unbound is channels.UNBOUND_REMOVE: - extraargs = ', unbound=channels.UNBOUND_REMOVE' + extraargs = ', unbounditems=channels.UNBOUND_REMOVE' else: raise NotImplementedError(repr(unbound)) interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) obj1 = b'spam' obj2 = b'eggs' @@ -454,11 +458,11 @@ def common(rch, sch, unbound=None, presize=0): with self.assertRaises(channels.ChannelEmptyError): rch.recv_nowait() - sch.send_nowait(b'ham', unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(b'ham', unbounditems=channels.UNBOUND_REMOVE) self.assertEqual(_channels.get_count(rch.id), 1) interp = common(rch, sch, channels.UNBOUND_REMOVE, 1) self.assertEqual(_channels.get_count(rch.id), 3) - sch.send_nowait(42, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(42, unbounditems=channels.UNBOUND_REMOVE) self.assertEqual(_channels.get_count(rch.id), 4) del interp self.assertEqual(_channels.get_count(rch.id), 2) @@ -482,13 +486,13 @@ def test_send_cleared_with_subinterpreter_mixed(self): self.assertEqual(_channels.get_count(rch.id), 0) _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) - sch.send_nowait(1, unbound=channels.UNBOUND) - sch.send_nowait(2, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(1, unbounditems=channels.UNBOUND) + sch.send_nowait(2, unbounditems=channels.UNBOUND_ERROR) sch.send_nowait(3) - sch.send_nowait(4, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(5, unbound=channels.UNBOUND) + sch.send_nowait(4, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(5, unbounditems=channels.UNBOUND) """)) self.assertEqual(_channels.get_count(rch.id), 5) @@ -518,15 +522,15 @@ def test_send_cleared_with_subinterpreter_multiple(self): sch.send_nowait(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj1 = rch.recv() - sch.send_nowait(2, unbound=channels.UNBOUND) - sch.send_nowait(obj1, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(2, unbounditems=channels.UNBOUND) + sch.send_nowait(obj1, unbounditems=channels.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj2 = rch.recv() @@ -535,21 +539,21 @@ def test_send_cleared_with_subinterpreter_multiple(self): self.assertEqual(_channels.get_count(rch.id), 0) sch.send_nowait(3) _run_output(interp1, dedent(""" - sch.send_nowait(4, unbound=channels.UNBOUND) + sch.send_nowait(4, unbounditems=channels.UNBOUND) # interp closed here - sch.send_nowait(5, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(6, unbound=channels.UNBOUND) + sch.send_nowait(5, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(6, unbounditems=channels.UNBOUND) """)) _run_output(interp2, dedent(""" - sch.send_nowait(7, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(7, unbounditems=channels.UNBOUND_ERROR) # interp closed here - sch.send_nowait(obj1, unbound=channels.UNBOUND_ERROR) - sch.send_nowait(obj2, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(8, unbound=channels.UNBOUND) + sch.send_nowait(obj1, unbounditems=channels.UNBOUND_ERROR) + sch.send_nowait(obj2, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(8, unbounditems=channels.UNBOUND) """)) _run_output(interp1, dedent(""" - sch.send_nowait(9, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(10, unbound=channels.UNBOUND) + sch.send_nowait(9, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(10, unbounditems=channels.UNBOUND) """)) self.assertEqual(_channels.get_count(rch.id), 10) diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index ac24f6568ac..39c1441b67c 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -119,7 +119,7 @@ def test_sys_path_0(self): # The main interpreter's sys.path[0] should be used by subinterpreters. script = ''' import sys - from test.support import interpreters + from concurrent import interpreters orig = sys.path[0] @@ -132,6 +132,7 @@ def test_sys_path_0(self): 'sub': sys.path[0], }}, indent=4), flush=True) """) + interp.close() ''' # <tmp>/ # pkg/ @@ -170,9 +171,12 @@ def test_gh_109793(self): # is reported, even when subinterpreters get cleaned up at the end. import subprocess argv = [sys.executable, '-c', '''if True: - from test.support import interpreters + from concurrent import interpreters interp = interpreters.create() - raise Exception + try: + raise Exception + finally: + interp.close() '''] proc = subprocess.run(argv, capture_output=True, text=True) self.assertIn('Traceback', proc.stderr) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 18f83d097eb..77334aea383 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -7,11 +7,11 @@ from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_interpqueues') -from test.support import interpreters -from test.support.interpreters import queues, _crossinterp +from concurrent import interpreters +from concurrent.interpreters import _queues as queues, _crossinterp from .utils import _run_output, TestBase as _TestBase - +HUGE_TIMEOUT = 3600 REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] @@ -42,7 +42,7 @@ def test_highlevel_reloaded(self): importlib.reload(queues) def test_create_destroy(self): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.destroy(qid) self.assertEqual(get_num_queues(), 0) with self.assertRaises(queues.QueueNotFoundError): @@ -56,7 +56,7 @@ def test_not_destroyed(self): '-c', dedent(f""" import {_queues.__name__} as _queues - _queues.create(2, 0, {REPLACE}) + _queues.create(2, {REPLACE}, -1) """), ) self.assertEqual(stdout, '') @@ -67,13 +67,13 @@ def test_not_destroyed(self): def test_bind_release(self): with self.subTest('typical'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.release(qid) self.assertEqual(get_num_queues(), 0) with self.subTest('bind too much'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.bind(qid) _queues.release(qid) @@ -81,7 +81,7 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('nested'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.bind(qid) _queues.release(qid) @@ -89,7 +89,7 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('release without binding'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) with self.assertRaises(queues.QueueError): _queues.release(qid) @@ -126,19 +126,19 @@ def test_shareable(self): interp = interpreters.create() interp.exec(dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) """)); with self.subTest('same interpreter'): queue2 = queues.create() - queue1.put(queue2, syncobj=True) + queue1.put(queue2) queue3 = queue1.get() self.assertIs(queue3, queue2) with self.subTest('from current interpreter'): queue4 = queues.create() - queue1.put(queue4, syncobj=True) + queue1.put(queue4) out = _run_output(interp, dedent(""" queue4 = queue1.get() print(queue4.id) @@ -149,7 +149,7 @@ def test_shareable(self): with self.subTest('from subinterpreter'): out = _run_output(interp, dedent(""" queue5 = queues.create() - queue1.put(queue5, syncobj=True) + queue1.put(queue5) print(queue5.id) """)) qid = int(out) @@ -188,9 +188,11 @@ def test_equality(self): def test_pickle(self): queue = queues.create() - data = pickle.dumps(queue) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, queue) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(queue, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) class TestQueueOps(TestBase): @@ -198,7 +200,7 @@ class TestQueueOps(TestBase): def test_empty(self): queue = queues.create() before = queue.empty() - queue.put(None, syncobj=True) + queue.put(None) during = queue.empty() queue.get() after = queue.empty() @@ -208,18 +210,64 @@ def test_empty(self): self.assertIs(after, True) def test_full(self): - expected = [False, False, False, True, False, False, False] - actual = [] - queue = queues.create(3) - for _ in range(3): - actual.append(queue.full()) - queue.put(None, syncobj=True) - actual.append(queue.full()) - for _ in range(3): - queue.get() - actual.append(queue.full()) + for maxsize in [1, 3, 11]: + with self.subTest(f'maxsize={maxsize}'): + num_to_add = maxsize + expected = [False] * (num_to_add * 2 + 3) + expected[maxsize] = True + expected[maxsize + 1] = True - self.assertEqual(actual, expected) + queue = queues.create(maxsize) + actual = [] + empty = [queue.empty()] + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.put_nowait(None) + actual.append(queue.full()) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + empty.append(queue.empty()) + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.get_nowait() + actual.append(queue.full()) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + actual.append(queue.full()) + empty.append(queue.empty()) + + self.assertEqual(actual, expected) + self.assertEqual(empty, [True, False, True]) + + # no max size + for args in [(), (0,), (-1,), (-10,)]: + with self.subTest(f'maxsize={args[0]}' if args else '<default>'): + num_to_add = 13 + expected = [False] * (num_to_add * 2 + 3) + + queue = queues.create(*args) + actual = [] + empty = [queue.empty()] + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.put_nowait(None) + actual.append(queue.full()) + empty.append(queue.empty()) + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.get_nowait() + actual.append(queue.full()) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + actual.append(queue.full()) + empty.append(queue.empty()) + + self.assertEqual(actual, expected) + self.assertEqual(empty, [True, False, True]) def test_qsize(self): expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0] @@ -227,16 +275,16 @@ def test_qsize(self): queue = queues.create() for _ in range(3): actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) for _ in range(3): queue.get() actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) @@ -245,70 +293,38 @@ def test_qsize(self): def test_put_get_main(self): expected = list(range(20)) - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create() - for i in range(20): - queue.put(i, **kwds) - actual = [queue.get() for _ in range(20)] + queue = queues.create() + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) def test_put_timeout(self): - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create(2) - queue.put(None, **kwds) - queue.put(None, **kwds) - with self.assertRaises(queues.QueueFull): - queue.put(None, timeout=0.1, **kwds) - queue.get() - queue.put(None, **kwds) + queue = queues.create(2) + queue.put(None) + queue.put(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1) + with self.assertRaises(queues.QueueFull): + queue.put(None, HUGE_TIMEOUT, 0.1) + queue.get() + queue.put(None) def test_put_nowait(self): - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create(2) - queue.put_nowait(None, **kwds) - queue.put_nowait(None, **kwds) - with self.assertRaises(queues.QueueFull): - queue.put_nowait(None, **kwds) - queue.get() - queue.put_nowait(None, **kwds) + queue = queues.create(2) + queue.put_nowait(None) + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, False) + with self.assertRaises(queues.QueueFull): + queue.put(None, False, timeout=HUGE_TIMEOUT) + queue.get() + queue.put_nowait(None) - def test_put_syncobj(self): - for obj in [ - None, - True, - 10, - 'spam', - b'spam', - (0, 'a'), - ]: - with self.subTest(repr(obj)): - queue = queues.create() - - queue.put(obj, syncobj=True) - obj2 = queue.get() - self.assertEqual(obj2, obj) - - queue.put(obj, syncobj=True) - obj2 = queue.get_nowait() - self.assertEqual(obj2, obj) - - for obj in [ - [1, 2, 3], - {'a': 13, 'b': 17}, - ]: - with self.subTest(repr(obj)): - queue = queues.create() - with self.assertRaises(interpreters.NotShareableError): - queue.put(obj, syncobj=True) - - def test_put_not_syncobj(self): + def test_put_full_fallback(self): for obj in [ None, True, @@ -323,11 +339,11 @@ def test_put_not_syncobj(self): with self.subTest(repr(obj)): queue = queues.create() - queue.put(obj, syncobj=False) + queue.put(obj) obj2 = queue.get() self.assertEqual(obj2, obj) - queue.put(obj, syncobj=False) + queue.put(obj) obj2 = queue.get_nowait() self.assertEqual(obj2, obj) @@ -335,30 +351,21 @@ def test_get_timeout(self): queue = queues.create() with self.assertRaises(queues.QueueEmpty): queue.get(timeout=0.1) + with self.assertRaises(queues.QueueEmpty): + queue.get(HUGE_TIMEOUT, 0.1) def test_get_nowait(self): queue = queues.create() with self.assertRaises(queues.QueueEmpty): queue.get_nowait() + with self.assertRaises(queues.QueueEmpty): + queue.get(False) + with self.assertRaises(queues.QueueEmpty): + queue.get(False, timeout=HUGE_TIMEOUT) - def test_put_get_default_syncobj(self): + def test_put_get_full_fallback(self): expected = list(range(20)) - queue = queues.create(syncobj=True) - for methname in ('get', 'get_nowait'): - with self.subTest(f'{methname}()'): - get = getattr(queue, methname) - for i in range(20): - queue.put(i) - actual = [get() for _ in range(20)] - self.assertEqual(actual, expected) - - obj = [1, 2, 3] # lists are not shareable - with self.assertRaises(interpreters.NotShareableError): - queue.put(obj) - - def test_put_get_default_not_syncobj(self): - expected = list(range(20)) - queue = queues.create(syncobj=False) + queue = queues.create() for methname in ('get', 'get_nowait'): with self.subTest(f'{methname}()'): get = getattr(queue, methname) @@ -377,14 +384,14 @@ def test_put_get_default_not_syncobj(self): def test_put_get_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.create() """)) for methname in ('get', 'get_nowait'): with self.subTest(f'{methname}()'): interp.exec(dedent(f""" orig = b'spam' - queue.put(orig, syncobj=True) + queue.put(orig) obj = queue.{methname}() assert obj == orig, 'expected: obj == orig' assert obj is not orig, 'expected: obj is not orig' @@ -399,12 +406,12 @@ def test_put_get_different_interpreters(self): for methname in ('get', 'get_nowait'): with self.subTest(f'{methname}()'): obj1 = b'spam' - queue1.put(obj1, syncobj=True) + queue1.put(obj1) out = _run_output( interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) queue2 = queues.Queue({queue2.id}) assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' @@ -416,7 +423,7 @@ def test_put_get_different_interpreters(self): obj2 = b'eggs' print(id(obj2)) assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' - queue2.put(obj2, syncobj=True) + queue2.put(obj2) assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' """)) self.assertEqual(len(queues.list_all()), 2) @@ -433,22 +440,22 @@ def common(queue, unbound=None, presize=0): if not unbound: extraargs = '' elif unbound is queues.UNBOUND: - extraargs = ', unbound=queues.UNBOUND' + extraargs = ', unbounditems=queues.UNBOUND' elif unbound is queues.UNBOUND_ERROR: - extraargs = ', unbound=queues.UNBOUND_ERROR' + extraargs = ', unbounditems=queues.UNBOUND_ERROR' elif unbound is queues.UNBOUND_REMOVE: - extraargs = ', unbound=queues.UNBOUND_REMOVE' + extraargs = ', unbounditems=queues.UNBOUND_REMOVE' else: raise NotImplementedError(repr(unbound)) interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' - queue.put(obj1, syncobj=True{extraargs}) - queue.put(obj2, syncobj=True{extraargs}) + queue.put(obj1{extraargs}) + queue.put(obj2{extraargs}) """)) self.assertEqual(queue.qsize(), presize + 2) @@ -501,11 +508,11 @@ def common(queue, unbound=None, presize=0): with self.assertRaises(queues.QueueEmpty): queue.get_nowait() - queue.put(b'ham', unbound=queues.UNBOUND_REMOVE) + queue.put(b'ham', unbounditems=queues.UNBOUND_REMOVE) self.assertEqual(queue.qsize(), 1) interp = common(queue, queues.UNBOUND_REMOVE, 1) self.assertEqual(queue.qsize(), 3) - queue.put(42, unbound=queues.UNBOUND_REMOVE) + queue.put(42, unbounditems=queues.UNBOUND_REMOVE) self.assertEqual(queue.qsize(), 4) del interp self.assertEqual(queue.qsize(), 2) @@ -521,13 +528,13 @@ def test_put_cleared_with_subinterpreter_mixed(self): queue = queues.create() interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) - queue.put(1, syncobj=True, unbound=queues.UNBOUND) - queue.put(2, syncobj=True, unbound=queues.UNBOUND_ERROR) - queue.put(3, syncobj=True) - queue.put(4, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(5, syncobj=True, unbound=queues.UNBOUND) + queue.put(1, unbounditems=queues.UNBOUND) + queue.put(2, unbounditems=queues.UNBOUND_ERROR) + queue.put(3) + queue.put(4, unbounditems=queues.UNBOUND_REMOVE) + queue.put(5, unbounditems=queues.UNBOUND) """)) self.assertEqual(queue.qsize(), 5) @@ -555,16 +562,16 @@ def test_put_cleared_with_subinterpreter_multiple(self): interp1 = interpreters.create() interp2 = interpreters.create() - queue.put(1, syncobj=True) + queue.put(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = queue.get() - queue.put(2, syncobj=True, unbound=queues.UNBOUND) - queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(2, unbounditems=queues.UNBOUND) + queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj2 = queue.get() obj1 = queue.get() @@ -572,21 +579,21 @@ def test_put_cleared_with_subinterpreter_multiple(self): self.assertEqual(queue.qsize(), 0) queue.put(3) _run_output(interp1, dedent(""" - queue.put(4, syncobj=True, unbound=queues.UNBOUND) + queue.put(4, unbounditems=queues.UNBOUND) # interp closed here - queue.put(5, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(6, syncobj=True, unbound=queues.UNBOUND) + queue.put(5, unbounditems=queues.UNBOUND_REMOVE) + queue.put(6, unbounditems=queues.UNBOUND) """)) _run_output(interp2, dedent(""" - queue.put(7, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(7, unbounditems=queues.UNBOUND_ERROR) # interp closed here - queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_ERROR) - queue.put(obj2, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(8, syncobj=True, unbound=queues.UNBOUND) + queue.put(obj1, unbounditems=queues.UNBOUND_ERROR) + queue.put(obj2, unbounditems=queues.UNBOUND_REMOVE) + queue.put(8, unbounditems=queues.UNBOUND) """)) _run_output(interp1, dedent(""" - queue.put(9, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(10, syncobj=True, unbound=queues.UNBOUND) + queue.put(9, unbounditems=queues.UNBOUND_REMOVE) + queue.put(10, unbounditems=queues.UNBOUND) """)) self.assertEqual(queue.qsize(), 10) @@ -642,12 +649,12 @@ def f(): break except queues.QueueEmpty: continue - queue2.put(obj, syncobj=True) + queue2.put(obj) t = threading.Thread(target=f) t.start() orig = b'spam' - queue1.put(orig, syncobj=True) + queue1.put(orig) obj = queue2.get() t.join() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index fae2f38cb55..6b40a536bd3 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -6,7 +6,8 @@ from test.support import threading_helper # Raise SkipTest if subinterpreters not supported. import_helper.import_module('_interpreters') -from test.support import interpreters +from concurrent import interpreters +from concurrent.interpreters import InterpreterError from .utils import TestBase @@ -74,6 +75,14 @@ def run(): start.set() support.gc_collect() + def test_create_interpreter_no_memory(self): + import _interpreters + _testcapi = import_helper.import_module("_testcapi") + + with self.assertRaises(InterpreterError): + _testcapi.set_nomemory(0, 1) + _interpreters.create() + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index fc4ad662e03..ae09aa457b4 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -12,7 +12,6 @@ import threading import types import unittest -import warnings from test import support @@ -22,7 +21,7 @@ import _interpreters except ImportError as exc: raise unittest.SkipTest(str(exc)) -from test.support import interpreters +from concurrent import interpreters try: diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py deleted file mode 100644 index 5a8f1949baa..00000000000 --- a/Lib/test/test_io.py +++ /dev/null @@ -1,5036 +0,0 @@ -"""Unit tests for the io module.""" - -# Tests of io are scattered over the test suite: -# * test_bufio - tests file buffering -# * test_memoryio - tests BytesIO and StringIO -# * test_fileio - tests FileIO -# * test_file - tests the file interface -# * test_io - tests everything else in the io module -# * test_univnewlines - tests universal newline support -# * test_largefile - tests operations on a file greater than 2**32 bytes -# (only enabled with -ulargefile) - -################################################################################ -# ATTENTION TEST WRITERS!!! -################################################################################ -# When writing tests for io, it's important to test both the C and Python -# implementations. This is usually done by writing a base test that refers to -# the type it is testing as an attribute. Then it provides custom subclasses to -# test both implementations. This file has lots of examples. -################################################################################ - -import abc -import array -import errno -import locale -import os -import pickle -import random -import signal -import sys -import textwrap -import threading -import time -import unittest -import warnings -import weakref -from collections import deque, UserList -from itertools import cycle, count -from test import support -from test.support.script_helper import ( - assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import ( - import_helper, is_apple, os_helper, threading_helper, warnings_helper, -) -from test.support.os_helper import FakePath - -import codecs -import io # C implementation of io -import _pyio as pyio # Python implementation of io - -try: - import ctypes -except ImportError: - def byteslike(*pos, **kw): - return array.array("b", bytes(*pos, **kw)) -else: - def byteslike(*pos, **kw): - """Create a bytes-like object having no string or sequence methods""" - data = bytes(*pos, **kw) - obj = EmptyStruct() - ctypes.resize(obj, len(data)) - memoryview(obj).cast("B")[:] = data - return obj - class EmptyStruct(ctypes.Structure): - pass - - -def _default_chunk_size(): - """Get the default TextIOWrapper chunk size""" - with open(__file__, "r", encoding="latin-1") as f: - return f._CHUNK_SIZE - -requires_alarm = unittest.skipUnless( - hasattr(signal, "alarm"), "test requires signal.alarm()" -) - - -class BadIndex: - def __index__(self): - 1/0 - -class MockRawIOWithoutRead: - """A RawIO implementation without read(), so as to exercise the default - RawIO.read() which calls readinto().""" - - def __init__(self, read_stack=()): - self._read_stack = list(read_stack) - self._write_stack = [] - self._reads = 0 - self._extraneous_reads = 0 - - def write(self, b): - self._write_stack.append(bytes(b)) - return len(b) - - def writable(self): - return True - - def fileno(self): - return 42 - - def readable(self): - return True - - def seekable(self): - return True - - def seek(self, pos, whence): - return 0 # wrong but we gotta return something - - def tell(self): - return 0 # same comment as above - - def readinto(self, buf): - self._reads += 1 - max_len = len(buf) - try: - data = self._read_stack[0] - except IndexError: - self._extraneous_reads += 1 - return 0 - if data is None: - del self._read_stack[0] - return None - n = len(data) - if len(data) <= max_len: - del self._read_stack[0] - buf[:n] = data - return n - else: - buf[:] = data[:max_len] - self._read_stack[0] = data[max_len:] - return max_len - - def truncate(self, pos=None): - return pos - -class CMockRawIOWithoutRead(MockRawIOWithoutRead, io.RawIOBase): - pass - -class PyMockRawIOWithoutRead(MockRawIOWithoutRead, pyio.RawIOBase): - pass - - -class MockRawIO(MockRawIOWithoutRead): - - def read(self, n=None): - self._reads += 1 - try: - return self._read_stack.pop(0) - except: - self._extraneous_reads += 1 - return b"" - -class CMockRawIO(MockRawIO, io.RawIOBase): - pass - -class PyMockRawIO(MockRawIO, pyio.RawIOBase): - pass - - -class MisbehavedRawIO(MockRawIO): - def write(self, b): - return super().write(b) * 2 - - def read(self, n=None): - return super().read(n) * 2 - - def seek(self, pos, whence): - return -123 - - def tell(self): - return -456 - - def readinto(self, buf): - super().readinto(buf) - return len(buf) * 5 - -class CMisbehavedRawIO(MisbehavedRawIO, io.RawIOBase): - pass - -class PyMisbehavedRawIO(MisbehavedRawIO, pyio.RawIOBase): - pass - - -class SlowFlushRawIO(MockRawIO): - def __init__(self): - super().__init__() - self.in_flush = threading.Event() - - def flush(self): - self.in_flush.set() - time.sleep(0.25) - -class CSlowFlushRawIO(SlowFlushRawIO, io.RawIOBase): - pass - -class PySlowFlushRawIO(SlowFlushRawIO, pyio.RawIOBase): - pass - - -class CloseFailureIO(MockRawIO): - closed = 0 - - def close(self): - if not self.closed: - self.closed = 1 - raise OSError - -class CCloseFailureIO(CloseFailureIO, io.RawIOBase): - pass - -class PyCloseFailureIO(CloseFailureIO, pyio.RawIOBase): - pass - - -class MockFileIO: - - def __init__(self, data): - self.read_history = [] - super().__init__(data) - - def read(self, n=None): - res = super().read(n) - self.read_history.append(None if res is None else len(res)) - return res - - def readinto(self, b): - res = super().readinto(b) - self.read_history.append(res) - return res - -class CMockFileIO(MockFileIO, io.BytesIO): - pass - -class PyMockFileIO(MockFileIO, pyio.BytesIO): - pass - - -class MockUnseekableIO: - def seekable(self): - return False - - def seek(self, *args): - raise self.UnsupportedOperation("not seekable") - - def tell(self, *args): - raise self.UnsupportedOperation("not seekable") - - def truncate(self, *args): - raise self.UnsupportedOperation("not seekable") - -class CMockUnseekableIO(MockUnseekableIO, io.BytesIO): - UnsupportedOperation = io.UnsupportedOperation - -class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): - UnsupportedOperation = pyio.UnsupportedOperation - - -class MockCharPseudoDevFileIO(MockFileIO): - # GH-95782 - # ftruncate() does not work on these special files (and CPython then raises - # appropriate exceptions), so truncate() does not have to be accounted for - # here. - def __init__(self, data): - super().__init__(data) - - def seek(self, *args): - return 0 - - def tell(self, *args): - return 0 - -class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): - pass - -class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): - pass - - -class MockNonBlockWriterIO: - - def __init__(self): - self._write_stack = [] - self._blocker_char = None - - def pop_written(self): - s = b"".join(self._write_stack) - self._write_stack[:] = [] - return s - - def block_on(self, char): - """Block when a given char is encountered.""" - self._blocker_char = char - - def readable(self): - return True - - def seekable(self): - return True - - def seek(self, pos, whence=0): - # naive implementation, enough for tests - return 0 - - def writable(self): - return True - - def write(self, b): - b = bytes(b) - n = -1 - if self._blocker_char: - try: - n = b.index(self._blocker_char) - except ValueError: - pass - else: - if n > 0: - # write data up to the first blocker - self._write_stack.append(b[:n]) - return n - else: - # cancel blocker and indicate would block - self._blocker_char = None - return None - self._write_stack.append(b) - return len(b) - -class CMockNonBlockWriterIO(MockNonBlockWriterIO, io.RawIOBase): - BlockingIOError = io.BlockingIOError - -class PyMockNonBlockWriterIO(MockNonBlockWriterIO, pyio.RawIOBase): - BlockingIOError = pyio.BlockingIOError - - -class IOTest(unittest.TestCase): - - def setUp(self): - os_helper.unlink(os_helper.TESTFN) - - def tearDown(self): - os_helper.unlink(os_helper.TESTFN) - - def write_ops(self, f): - self.assertEqual(f.write(b"blah."), 5) - f.truncate(0) - self.assertEqual(f.tell(), 5) - f.seek(0) - - self.assertEqual(f.write(b"blah."), 5) - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.write(b"Hello."), 6) - self.assertEqual(f.tell(), 6) - self.assertEqual(f.seek(-1, 1), 5) - self.assertEqual(f.tell(), 5) - buffer = bytearray(b" world\n\n\n") - self.assertEqual(f.write(buffer), 9) - buffer[:] = b"*" * 9 # Overwrite our copy of the data - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.write(b"h"), 1) - self.assertEqual(f.seek(-1, 2), 13) - self.assertEqual(f.tell(), 13) - - self.assertEqual(f.truncate(12), 12) - self.assertEqual(f.tell(), 13) - self.assertRaises(TypeError, f.seek, 0.0) - - def read_ops(self, f, buffered=False): - data = f.read(5) - self.assertEqual(data, b"hello") - data = byteslike(data) - self.assertEqual(f.readinto(data), 5) - self.assertEqual(bytes(data), b" worl") - data = bytearray(5) - self.assertEqual(f.readinto(data), 2) - self.assertEqual(len(data), 5) - self.assertEqual(data[:2], b"d\n") - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.read(20), b"hello world\n") - self.assertEqual(f.read(1), b"") - self.assertEqual(f.readinto(byteslike(b"x")), 0) - self.assertEqual(f.seek(-6, 2), 6) - self.assertEqual(f.read(5), b"world") - self.assertEqual(f.read(0), b"") - self.assertEqual(f.readinto(byteslike()), 0) - self.assertEqual(f.seek(-6, 1), 5) - self.assertEqual(f.read(5), b" worl") - self.assertEqual(f.tell(), 10) - self.assertRaises(TypeError, f.seek, 0.0) - if buffered: - f.seek(0) - self.assertEqual(f.read(), b"hello world\n") - f.seek(6) - self.assertEqual(f.read(), b"world\n") - self.assertEqual(f.read(), b"") - f.seek(0) - data = byteslike(5) - self.assertEqual(f.readinto1(data), 5) - self.assertEqual(bytes(data), b"hello") - - LARGE = 2**31 - - def large_file_ops(self, f): - assert f.readable() - assert f.writable() - try: - self.assertEqual(f.seek(self.LARGE), self.LARGE) - except (OverflowError, ValueError): - self.skipTest("no largefile support") - self.assertEqual(f.tell(), self.LARGE) - self.assertEqual(f.write(b"xxx"), 3) - self.assertEqual(f.tell(), self.LARGE + 3) - self.assertEqual(f.seek(-1, 1), self.LARGE + 2) - self.assertEqual(f.truncate(), self.LARGE + 2) - self.assertEqual(f.tell(), self.LARGE + 2) - self.assertEqual(f.seek(0, 2), self.LARGE + 2) - self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1) - self.assertEqual(f.tell(), self.LARGE + 2) - self.assertEqual(f.seek(0, 2), self.LARGE + 1) - self.assertEqual(f.seek(-1, 2), self.LARGE) - self.assertEqual(f.read(2), b"x") - - def test_invalid_operations(self): - # Try writing on a file opened in read mode and vice-versa. - exc = self.UnsupportedOperation - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as fp: - self.assertRaises(exc, fp.read) - self.assertRaises(exc, fp.readline) - with self.open(os_helper.TESTFN, "wb") as fp: - self.assertRaises(exc, fp.read) - self.assertRaises(exc, fp.readline) - with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: - self.assertRaises(exc, fp.read) - self.assertRaises(exc, fp.readline) - with self.open(os_helper.TESTFN, "rb", buffering=0) as fp: - self.assertRaises(exc, fp.write, b"blah") - self.assertRaises(exc, fp.writelines, [b"blah\n"]) - with self.open(os_helper.TESTFN, "rb") as fp: - self.assertRaises(exc, fp.write, b"blah") - self.assertRaises(exc, fp.writelines, [b"blah\n"]) - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as fp: - self.assertRaises(exc, fp.write, "blah") - self.assertRaises(exc, fp.writelines, ["blah\n"]) - # Non-zero seeking from current or end pos - self.assertRaises(exc, fp.seek, 1, self.SEEK_CUR) - self.assertRaises(exc, fp.seek, -1, self.SEEK_END) - - @support.cpython_only - def test_startup_optimization(self): - # gh-132952: Test that `io` is not imported at startup and that the - # __module__ of UnsupportedOperation is set to "io". - assert_python_ok("-S", "-c", textwrap.dedent( - """ - import sys - assert "io" not in sys.modules - try: - sys.stdin.truncate() - except Exception as e: - typ = type(e) - assert typ.__module__ == "io", (typ, typ.__module__) - assert typ.__name__ == "UnsupportedOperation", (typ, typ.__name__) - else: - raise AssertionError("Expected UnsupportedOperation") - """ - )) - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_optional_abilities(self): - # Test for OSError when optional APIs are not supported - # The purpose of this test is to try fileno(), reading, writing and - # seeking operations with various objects that indicate they do not - # support these operations. - - def pipe_reader(): - [r, w] = os.pipe() - os.close(w) # So that read() is harmless - return self.FileIO(r, "r") - - def pipe_writer(): - [r, w] = os.pipe() - self.addCleanup(os.close, r) - # Guarantee that we can write into the pipe without blocking - thread = threading.Thread(target=os.read, args=(r, 100)) - thread.start() - self.addCleanup(thread.join) - return self.FileIO(w, "w") - - def buffered_reader(): - return self.BufferedReader(self.MockUnseekableIO()) - - def buffered_writer(): - return self.BufferedWriter(self.MockUnseekableIO()) - - def buffered_random(): - return self.BufferedRandom(self.BytesIO()) - - def buffered_rw_pair(): - return self.BufferedRWPair(self.MockUnseekableIO(), - self.MockUnseekableIO()) - - def text_reader(): - class UnseekableReader(self.MockUnseekableIO): - writable = self.BufferedIOBase.writable - write = self.BufferedIOBase.write - return self.TextIOWrapper(UnseekableReader(), "ascii") - - def text_writer(): - class UnseekableWriter(self.MockUnseekableIO): - readable = self.BufferedIOBase.readable - read = self.BufferedIOBase.read - return self.TextIOWrapper(UnseekableWriter(), "ascii") - - tests = ( - (pipe_reader, "fr"), (pipe_writer, "fw"), - (buffered_reader, "r"), (buffered_writer, "w"), - (buffered_random, "rws"), (buffered_rw_pair, "rw"), - (text_reader, "r"), (text_writer, "w"), - (self.BytesIO, "rws"), (self.StringIO, "rws"), - ) - - def do_test(test, obj, abilities): - readable = "r" in abilities - self.assertEqual(obj.readable(), readable) - writable = "w" in abilities - self.assertEqual(obj.writable(), writable) - - if isinstance(obj, self.TextIOBase): - data = "3" - elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)): - data = b"3" - else: - self.fail("Unknown base class") - - if "f" in abilities: - obj.fileno() - else: - self.assertRaises(OSError, obj.fileno) - - if readable: - obj.read(1) - obj.read() - else: - self.assertRaises(OSError, obj.read, 1) - self.assertRaises(OSError, obj.read) - - if writable: - obj.write(data) - else: - self.assertRaises(OSError, obj.write, data) - - if sys.platform.startswith("win") and test in ( - pipe_reader, pipe_writer): - # Pipes seem to appear as seekable on Windows - return - seekable = "s" in abilities - self.assertEqual(obj.seekable(), seekable) - - if seekable: - obj.tell() - obj.seek(0) - else: - self.assertRaises(OSError, obj.tell) - self.assertRaises(OSError, obj.seek, 0) - - if writable and seekable: - obj.truncate() - obj.truncate(0) - else: - self.assertRaises(OSError, obj.truncate) - self.assertRaises(OSError, obj.truncate, 0) - - for [test, abilities] in tests: - with self.subTest(test): - if test == pipe_writer and not threading_helper.can_start_thread: - skipTest() - with test() as obj: - do_test(test, obj, abilities) - - - def test_open_handles_NUL_chars(self): - fn_with_NUL = 'foo\0bar' - self.assertRaises(ValueError, self.open, fn_with_NUL, 'w', encoding="utf-8") - - bytes_fn = bytes(fn_with_NUL, 'ascii') - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.assertRaises(ValueError, self.open, bytes_fn, 'w', encoding="utf-8") - - def test_raw_file_io(self): - with self.open(os_helper.TESTFN, "wb", buffering=0) as f: - self.assertEqual(f.readable(), False) - self.assertEqual(f.writable(), True) - self.assertEqual(f.seekable(), True) - self.write_ops(f) - with self.open(os_helper.TESTFN, "rb", buffering=0) as f: - self.assertEqual(f.readable(), True) - self.assertEqual(f.writable(), False) - self.assertEqual(f.seekable(), True) - self.read_ops(f) - - def test_buffered_file_io(self): - with self.open(os_helper.TESTFN, "wb") as f: - self.assertEqual(f.readable(), False) - self.assertEqual(f.writable(), True) - self.assertEqual(f.seekable(), True) - self.write_ops(f) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.readable(), True) - self.assertEqual(f.writable(), False) - self.assertEqual(f.seekable(), True) - self.read_ops(f, True) - - def test_readline(self): - with self.open(os_helper.TESTFN, "wb") as f: - f.write(b"abc\ndef\nxyzzy\nfoo\x00bar\nanother line") - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.readline(), b"abc\n") - self.assertEqual(f.readline(10), b"def\n") - self.assertEqual(f.readline(2), b"xy") - self.assertEqual(f.readline(4), b"zzy\n") - self.assertEqual(f.readline(), b"foo\x00bar\n") - self.assertEqual(f.readline(None), b"another line") - self.assertRaises(TypeError, f.readline, 5.3) - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - self.assertRaises(TypeError, f.readline, 5.3) - - def test_readline_nonsizeable(self): - # Issue #30061 - # Crash when readline() returns an object without __len__ - class R(self.IOBase): - def readline(self): - return None - self.assertRaises((TypeError, StopIteration), next, R()) - - def test_next_nonsizeable(self): - # Issue #30061 - # Crash when __next__() returns an object without __len__ - class R(self.IOBase): - def __next__(self): - return None - self.assertRaises(TypeError, R().readlines, 1) - - def test_raw_bytes_io(self): - f = self.BytesIO() - self.write_ops(f) - data = f.getvalue() - self.assertEqual(data, b"hello world\n") - f = self.BytesIO(data) - self.read_ops(f, True) - - def test_large_file_ops(self): - # On Windows and Apple platforms this test consumes large resources; It - # takes a long time to build the >2 GiB file and takes >2 GiB of disk - # space therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or is_apple: - support.requires( - 'largefile', - 'test requires %s bytes and a long time to run' % self.LARGE) - with self.open(os_helper.TESTFN, "w+b", 0) as f: - self.large_file_ops(f) - with self.open(os_helper.TESTFN, "w+b") as f: - self.large_file_ops(f) - - def test_with_open(self): - for bufsize in (0, 100): - with self.open(os_helper.TESTFN, "wb", bufsize) as f: - f.write(b"xxx") - self.assertEqual(f.closed, True) - try: - with self.open(os_helper.TESTFN, "wb", bufsize) as f: - 1/0 - except ZeroDivisionError: - self.assertEqual(f.closed, True) - else: - self.fail("1/0 didn't raise an exception") - - # issue 5008 - def test_append_mode_tell(self): - with self.open(os_helper.TESTFN, "wb") as f: - f.write(b"xxx") - with self.open(os_helper.TESTFN, "ab", buffering=0) as f: - self.assertEqual(f.tell(), 3) - with self.open(os_helper.TESTFN, "ab") as f: - self.assertEqual(f.tell(), 3) - with self.open(os_helper.TESTFN, "a", encoding="utf-8") as f: - self.assertGreater(f.tell(), 0) - - def test_destructor(self): - record = [] - class MyFileIO(self.FileIO): - def __del__(self): - record.append(1) - try: - f = super().__del__ - except AttributeError: - pass - else: - f() - def close(self): - record.append(2) - super().close() - def flush(self): - record.append(3) - super().flush() - with warnings_helper.check_warnings(('', ResourceWarning)): - f = MyFileIO(os_helper.TESTFN, "wb") - f.write(b"xxx") - del f - support.gc_collect() - self.assertEqual(record, [1, 2, 3]) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"xxx") - - def _check_base_destructor(self, base): - record = [] - class MyIO(base): - def __init__(self): - # This exercises the availability of attributes on object - # destruction. - # (in the C version, close() is called by the tp_dealloc - # function, not by __del__) - self.on_del = 1 - self.on_close = 2 - self.on_flush = 3 - def __del__(self): - record.append(self.on_del) - try: - f = super().__del__ - except AttributeError: - pass - else: - f() - def close(self): - record.append(self.on_close) - super().close() - def flush(self): - record.append(self.on_flush) - super().flush() - f = MyIO() - del f - support.gc_collect() - self.assertEqual(record, [1, 2, 3]) - - def test_IOBase_destructor(self): - self._check_base_destructor(self.IOBase) - - def test_RawIOBase_destructor(self): - self._check_base_destructor(self.RawIOBase) - - def test_BufferedIOBase_destructor(self): - self._check_base_destructor(self.BufferedIOBase) - - def test_TextIOBase_destructor(self): - self._check_base_destructor(self.TextIOBase) - - def test_close_flushes(self): - with self.open(os_helper.TESTFN, "wb") as f: - f.write(b"xxx") - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"xxx") - - def test_array_writes(self): - a = array.array('i', range(10)) - n = len(a.tobytes()) - def check(f): - with f: - self.assertEqual(f.write(a), n) - f.writelines((a,)) - check(self.BytesIO()) - check(self.FileIO(os_helper.TESTFN, "w")) - check(self.BufferedWriter(self.MockRawIO())) - check(self.BufferedRandom(self.MockRawIO())) - check(self.BufferedRWPair(self.MockRawIO(), self.MockRawIO())) - - def test_closefd(self): - self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'w', - encoding="utf-8", closefd=False) - - def test_read_closed(self): - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - f.write("egg\n") - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) - self.assertEqual(file.read(), "egg\n") - file.seek(0) - file.close() - self.assertRaises(ValueError, file.read) - with self.open(os_helper.TESTFN, "rb") as f: - file = self.open(f.fileno(), "rb", closefd=False) - self.assertEqual(file.read()[:3], b"egg") - file.close() - self.assertRaises(ValueError, file.readinto, bytearray(1)) - - def test_no_closefd_with_filename(self): - # can't use closefd in combination with a file name - self.assertRaises(ValueError, self.open, os_helper.TESTFN, "r", - encoding="utf-8", closefd=False) - - def test_closefd_attr(self): - with self.open(os_helper.TESTFN, "wb") as f: - f.write(b"egg\n") - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - self.assertEqual(f.buffer.raw.closefd, True) - file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) - self.assertEqual(file.buffer.raw.closefd, False) - - def test_garbage_collection(self): - # FileIO objects are collected, and collecting them flushes - # all data to disk. - with warnings_helper.check_warnings(('', ResourceWarning)): - f = self.FileIO(os_helper.TESTFN, "wb") - f.write(b"abcxxx") - f.f = f - wr = weakref.ref(f) - del f - support.gc_collect() - self.assertIsNone(wr(), wr) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"abcxxx") - - def test_unbounded_file(self): - # Issue #1174606: reading from an unbounded stream such as /dev/zero. - zero = "/dev/zero" - if not os.path.exists(zero): - self.skipTest("{0} does not exist".format(zero)) - if sys.maxsize > 0x7FFFFFFF: - self.skipTest("test can only run in a 32-bit address space") - if support.real_max_memuse < support._2G: - self.skipTest("test requires at least 2 GiB of memory") - with self.open(zero, "rb", buffering=0) as f: - self.assertRaises(OverflowError, f.read) - with self.open(zero, "rb") as f: - self.assertRaises(OverflowError, f.read) - with self.open(zero, "r") as f: - self.assertRaises(OverflowError, f.read) - - def check_flush_error_on_close(self, *args, **kwargs): - # Test that the file is closed despite failed flush - # and that flush() is called before file closed. - f = self.open(*args, **kwargs) - closed = [] - def bad_flush(): - closed[:] = [f.closed] - raise OSError() - f.flush = bad_flush - self.assertRaises(OSError, f.close) # exception not swallowed - self.assertTrue(f.closed) - self.assertTrue(closed) # flush() called - self.assertFalse(closed[0]) # flush() called before file closed - f.flush = lambda: None # break reference loop - - def test_flush_error_on_close(self): - # raw file - # Issue #5700: io.FileIO calls flush() after file closed - self.check_flush_error_on_close(os_helper.TESTFN, 'wb', buffering=0) - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'wb', buffering=0) - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'wb', buffering=0, closefd=False) - os.close(fd) - # buffered io - self.check_flush_error_on_close(os_helper.TESTFN, 'wb') - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'wb') - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'wb', closefd=False) - os.close(fd) - # text io - self.check_flush_error_on_close(os_helper.TESTFN, 'w', encoding="utf-8") - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'w', encoding="utf-8") - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'w', encoding="utf-8", closefd=False) - os.close(fd) - - def test_multi_close(self): - f = self.open(os_helper.TESTFN, "wb", buffering=0) - f.close() - f.close() - f.close() - self.assertRaises(ValueError, f.flush) - - def test_RawIOBase_read(self): - # Exercise the default limited RawIOBase.read(n) implementation (which - # calls readinto() internally). - rawio = self.MockRawIOWithoutRead((b"abc", b"d", None, b"efg", None)) - self.assertEqual(rawio.read(2), b"ab") - self.assertEqual(rawio.read(2), b"c") - self.assertEqual(rawio.read(2), b"d") - self.assertEqual(rawio.read(2), None) - self.assertEqual(rawio.read(2), b"ef") - self.assertEqual(rawio.read(2), b"g") - self.assertEqual(rawio.read(2), None) - self.assertEqual(rawio.read(2), b"") - - def test_types_have_dict(self): - test = ( - self.IOBase(), - self.RawIOBase(), - self.TextIOBase(), - self.StringIO(), - self.BytesIO() - ) - for obj in test: - self.assertTrue(hasattr(obj, "__dict__")) - - def test_opener(self): - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - f.write("egg\n") - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - def opener(path, flags): - return fd - with self.open("non-existent", "r", encoding="utf-8", opener=opener) as f: - self.assertEqual(f.read(), "egg\n") - - def test_bad_opener_negative_1(self): - # Issue #27066. - def badopener(fname, flags): - return -1 - with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) - self.assertEqual(str(cm.exception), 'opener returned -1') - - def test_bad_opener_other_negative(self): - # Issue #27066. - def badopener(fname, flags): - return -2 - with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) - self.assertEqual(str(cm.exception), 'opener returned -2') - - def test_opener_invalid_fd(self): - # Check that OSError is raised with error code EBADF if the - # opener returns an invalid file descriptor (see gh-82212). - fd = os_helper.make_bad_fd() - with self.assertRaises(OSError) as cm: - self.open('foo', opener=lambda name, flags: fd) - self.assertEqual(cm.exception.errno, errno.EBADF) - - def test_fileio_closefd(self): - # Issue #4841 - with self.open(__file__, 'rb') as f1, \ - self.open(__file__, 'rb') as f2: - fileio = self.FileIO(f1.fileno(), closefd=False) - # .__init__() must not close f1 - fileio.__init__(f2.fileno(), closefd=False) - f1.readline() - # .close() must not close f2 - fileio.close() - f2.readline() - - def test_nonbuffered_textio(self): - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError): - self.open(os_helper.TESTFN, 'w', encoding="utf-8", buffering=0) - - def test_invalid_newline(self): - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError): - self.open(os_helper.TESTFN, 'w', encoding="utf-8", newline='invalid') - - def test_buffered_readinto_mixin(self): - # Test the implementation provided by BufferedIOBase - class Stream(self.BufferedIOBase): - def read(self, size): - return b"12345" - read1 = read - stream = Stream() - for method in ("readinto", "readinto1"): - with self.subTest(method): - buffer = byteslike(5) - self.assertEqual(getattr(stream, method)(buffer), 5) - self.assertEqual(bytes(buffer), b"12345") - - def test_fspath_support(self): - def check_path_succeeds(path): - with self.open(path, "w", encoding="utf-8") as f: - f.write("egg\n") - - with self.open(path, "r", encoding="utf-8") as f: - self.assertEqual(f.read(), "egg\n") - - check_path_succeeds(FakePath(os_helper.TESTFN)) - check_path_succeeds(FakePath(os.fsencode(os_helper.TESTFN))) - - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - bad_path = FakePath(f.fileno()) - with self.assertRaises(TypeError): - self.open(bad_path, 'w', encoding="utf-8") - - bad_path = FakePath(None) - with self.assertRaises(TypeError): - self.open(bad_path, 'w', encoding="utf-8") - - bad_path = FakePath(FloatingPointError) - with self.assertRaises(FloatingPointError): - self.open(bad_path, 'w', encoding="utf-8") - - # ensure that refcounting is correct with some error conditions - with self.assertRaisesRegex(ValueError, 'read/write/append mode'): - self.open(FakePath(os_helper.TESTFN), 'rwxa', encoding="utf-8") - - def test_RawIOBase_readall(self): - # Exercise the default unlimited RawIOBase.read() and readall() - # implementations. - rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg")) - self.assertEqual(rawio.read(), b"abcdefg") - rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg")) - self.assertEqual(rawio.readall(), b"abcdefg") - - def test_BufferedIOBase_readinto(self): - # Exercise the default BufferedIOBase.readinto() and readinto1() - # implementations (which call read() or read1() internally). - class Reader(self.BufferedIOBase): - def __init__(self, avail): - self.avail = avail - def read(self, size): - result = self.avail[:size] - self.avail = self.avail[size:] - return result - def read1(self, size): - """Returns no more than 5 bytes at once""" - return self.read(min(size, 5)) - tests = ( - # (test method, total data available, read buffer size, expected - # read size) - ("readinto", 10, 5, 5), - ("readinto", 10, 6, 6), # More than read1() can return - ("readinto", 5, 6, 5), # Buffer larger than total available - ("readinto", 6, 7, 6), - ("readinto", 10, 0, 0), # Empty buffer - ("readinto1", 10, 5, 5), # Result limited to single read1() call - ("readinto1", 10, 6, 5), # Buffer larger than read1() can return - ("readinto1", 5, 6, 5), # Buffer larger than total available - ("readinto1", 6, 7, 5), - ("readinto1", 10, 0, 0), # Empty buffer - ) - UNUSED_BYTE = 0x81 - for test in tests: - with self.subTest(test): - method, avail, request, result = test - reader = Reader(bytes(range(avail))) - buffer = bytearray((UNUSED_BYTE,) * request) - method = getattr(reader, method) - self.assertEqual(method(buffer), result) - self.assertEqual(len(buffer), request) - self.assertSequenceEqual(buffer[:result], range(result)) - unused = (UNUSED_BYTE,) * (request - result) - self.assertSequenceEqual(buffer[result:], unused) - self.assertEqual(len(reader.avail), avail - result) - - def test_close_assert(self): - class R(self.IOBase): - def __setattr__(self, name, value): - pass - def flush(self): - raise OSError() - f = R() - # This would cause an assertion failure. - self.assertRaises(OSError, f.close) - - # Silence destructor error - R.flush = lambda self: None - - -class CIOTest(IOTest): - - def test_IOBase_finalize(self): - # Issue #12149: segmentation fault on _PyIOBase_finalize when both a - # class which inherits IOBase and an object of this class are caught - # in a reference cycle and close() is already in the method cache. - class MyIO(self.IOBase): - def close(self): - pass - - # create an instance to populate the method cache - MyIO() - obj = MyIO() - obj.obj = obj - wr = weakref.ref(obj) - del MyIO - del obj - support.gc_collect() - self.assertIsNone(wr(), wr) - -@support.cpython_only -class TestIOCTypes(unittest.TestCase): - def setUp(self): - _io = import_helper.import_module("_io") - self.types = [ - _io.BufferedRWPair, - _io.BufferedRandom, - _io.BufferedReader, - _io.BufferedWriter, - _io.BytesIO, - _io.FileIO, - _io.IncrementalNewlineDecoder, - _io.StringIO, - _io.TextIOWrapper, - _io._BufferedIOBase, - _io._BytesIOBuffer, - _io._IOBase, - _io._RawIOBase, - _io._TextIOBase, - ] - if sys.platform == "win32": - self.types.append(_io._WindowsConsoleIO) - self._io = _io - - def test_immutable_types(self): - for tp in self.types: - with self.subTest(tp=tp): - with self.assertRaisesRegex(TypeError, "immutable"): - tp.foo = "bar" - - def test_class_hierarchy(self): - def check_subs(types, base): - for tp in types: - with self.subTest(tp=tp, base=base): - self.assertTrue(issubclass(tp, base)) - - def recursive_check(d): - for k, v in d.items(): - if isinstance(v, dict): - recursive_check(v) - elif isinstance(v, set): - check_subs(v, k) - else: - self.fail("corrupt test dataset") - - _io = self._io - hierarchy = { - _io._IOBase: { - _io._BufferedIOBase: { - _io.BufferedRWPair, - _io.BufferedRandom, - _io.BufferedReader, - _io.BufferedWriter, - _io.BytesIO, - }, - _io._RawIOBase: { - _io.FileIO, - }, - _io._TextIOBase: { - _io.StringIO, - _io.TextIOWrapper, - }, - }, - } - if sys.platform == "win32": - hierarchy[_io._IOBase][_io._RawIOBase].add(_io._WindowsConsoleIO) - - recursive_check(hierarchy) - - def test_subclassing(self): - _io = self._io - dataset = {k: True for k in self.types} - dataset[_io._BytesIOBuffer] = False - - for tp, is_basetype in dataset.items(): - with self.subTest(tp=tp, is_basetype=is_basetype): - name = f"{tp.__name__}_subclass" - bases = (tp,) - if is_basetype: - _ = type(name, bases, {}) - else: - msg = "not an acceptable base type" - with self.assertRaisesRegex(TypeError, msg): - _ = type(name, bases, {}) - - def test_disallow_instantiation(self): - _io = self._io - support.check_disallow_instantiation(self, _io._BytesIOBuffer) - - def test_stringio_setstate(self): - # gh-127182: Calling __setstate__() with invalid arguments must not crash - obj = self._io.StringIO() - with self.assertRaisesRegex( - TypeError, - 'initial_value must be str or None, not int', - ): - obj.__setstate__((1, '', 0, {})) - - obj.__setstate__((None, '', 0, {})) # should not crash - self.assertEqual(obj.getvalue(), '') - - obj.__setstate__(('', '', 0, {})) - self.assertEqual(obj.getvalue(), '') - -class PyIOTest(IOTest): - pass - - -@support.cpython_only -class APIMismatchTest(unittest.TestCase): - - def test_RawIOBase_io_in_pyio_match(self): - """Test that pyio RawIOBase class has all c RawIOBase methods""" - mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__', '__static_attributes__')) - self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') - - def test_RawIOBase_pyio_in_io_match(self): - """Test that c RawIOBase class has all pyio RawIOBase methods""" - mismatch = support.detect_api_mismatch(io.RawIOBase, pyio.RawIOBase) - self.assertEqual(mismatch, set(), msg='C RawIOBase does not have all Python RawIOBase methods') - - -class CommonBufferedTests: - # Tests common to BufferedReader, BufferedWriter and BufferedRandom - - def test_detach(self): - raw = self.MockRawIO() - buf = self.tp(raw) - self.assertIs(buf.detach(), raw) - self.assertRaises(ValueError, buf.detach) - - repr(buf) # Should still work - - def test_fileno(self): - rawio = self.MockRawIO() - bufio = self.tp(rawio) - - self.assertEqual(42, bufio.fileno()) - - def test_invalid_args(self): - rawio = self.MockRawIO() - bufio = self.tp(rawio) - # Invalid whence - self.assertRaises(ValueError, bufio.seek, 0, -1) - self.assertRaises(ValueError, bufio.seek, 0, 9) - - def test_override_destructor(self): - tp = self.tp - record = [] - class MyBufferedIO(tp): - def __del__(self): - record.append(1) - try: - f = super().__del__ - except AttributeError: - pass - else: - f() - def close(self): - record.append(2) - super().close() - def flush(self): - record.append(3) - super().flush() - rawio = self.MockRawIO() - bufio = MyBufferedIO(rawio) - del bufio - support.gc_collect() - self.assertEqual(record, [1, 2, 3]) - - def test_context_manager(self): - # Test usability as a context manager - rawio = self.MockRawIO() - bufio = self.tp(rawio) - def _with(): - with bufio: - pass - _with() - # bufio should now be closed, and using it a second time should raise - # a ValueError. - self.assertRaises(ValueError, _with) - - def test_error_through_destructor(self): - # Test that the exception state is not modified by a destructor, - # even if close() fails. - rawio = self.CloseFailureIO() - with support.catch_unraisable_exception() as cm: - with self.assertRaises(AttributeError): - self.tp(rawio).xyzzy - - self.assertEqual(cm.unraisable.exc_type, OSError) - - def test_repr(self): - raw = self.MockRawIO() - b = self.tp(raw) - clsname = r"(%s\.)?%s" % (self.tp.__module__, self.tp.__qualname__) - self.assertRegex(repr(b), "<%s>" % clsname) - raw.name = "dummy" - self.assertRegex(repr(b), "<%s name='dummy'>" % clsname) - raw.name = b"dummy" - self.assertRegex(repr(b), "<%s name=b'dummy'>" % clsname) - - def test_recursive_repr(self): - # Issue #25455 - raw = self.MockRawIO() - b = self.tp(raw) - with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): - with self.assertRaises(RuntimeError): - repr(b) # Should not crash - - def test_flush_error_on_close(self): - # Test that buffered file is closed despite failed flush - # and that flush() is called before file closed. - raw = self.MockRawIO() - closed = [] - def bad_flush(): - closed[:] = [b.closed, raw.closed] - raise OSError() - raw.flush = bad_flush - b = self.tp(raw) - self.assertRaises(OSError, b.close) # exception not swallowed - self.assertTrue(b.closed) - self.assertTrue(raw.closed) - self.assertTrue(closed) # flush() called - self.assertFalse(closed[0]) # flush() called before file closed - self.assertFalse(closed[1]) - raw.flush = lambda: None # break reference loop - - def test_close_error_on_close(self): - raw = self.MockRawIO() - def bad_flush(): - raise OSError('flush') - def bad_close(): - raise OSError('close') - raw.close = bad_close - b = self.tp(raw) - b.flush = bad_flush - with self.assertRaises(OSError) as err: # exception not swallowed - b.close() - self.assertEqual(err.exception.args, ('close',)) - self.assertIsInstance(err.exception.__context__, OSError) - self.assertEqual(err.exception.__context__.args, ('flush',)) - self.assertFalse(b.closed) - - # Silence destructor error - raw.close = lambda: None - b.flush = lambda: None - - def test_nonnormalized_close_error_on_close(self): - # Issue #21677 - raw = self.MockRawIO() - def bad_flush(): - raise non_existing_flush - def bad_close(): - raise non_existing_close - raw.close = bad_close - b = self.tp(raw) - b.flush = bad_flush - with self.assertRaises(NameError) as err: # exception not swallowed - b.close() - self.assertIn('non_existing_close', str(err.exception)) - self.assertIsInstance(err.exception.__context__, NameError) - self.assertIn('non_existing_flush', str(err.exception.__context__)) - self.assertFalse(b.closed) - - # Silence destructor error - b.flush = lambda: None - raw.close = lambda: None - - def test_multi_close(self): - raw = self.MockRawIO() - b = self.tp(raw) - b.close() - b.close() - b.close() - self.assertRaises(ValueError, b.flush) - - def test_unseekable(self): - bufio = self.tp(self.MockUnseekableIO(b"A" * 10)) - self.assertRaises(self.UnsupportedOperation, bufio.tell) - self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) - - def test_readonly_attributes(self): - raw = self.MockRawIO() - buf = self.tp(raw) - x = self.MockRawIO() - with self.assertRaises(AttributeError): - buf.raw = x - - def test_pickling_subclass(self): - global MyBufferedIO - class MyBufferedIO(self.tp): - def __init__(self, raw, tag): - super().__init__(raw) - self.tag = tag - def __getstate__(self): - return self.tag, self.raw.getvalue() - def __setstate__(slf, state): - tag, value = state - slf.__init__(self.BytesIO(value), tag) - - raw = self.BytesIO(b'data') - buf = MyBufferedIO(raw, tag='ham') - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(protocol=proto): - pickled = pickle.dumps(buf, proto) - newbuf = pickle.loads(pickled) - self.assertEqual(newbuf.raw.getvalue(), b'data') - self.assertEqual(newbuf.tag, 'ham') - del MyBufferedIO - - -class SizeofTest: - - @support.cpython_only - def test_sizeof(self): - bufsize1 = 4096 - bufsize2 = 8192 - rawio = self.MockRawIO() - bufio = self.tp(rawio, buffer_size=bufsize1) - size = sys.getsizeof(bufio) - bufsize1 - rawio = self.MockRawIO() - bufio = self.tp(rawio, buffer_size=bufsize2) - self.assertEqual(sys.getsizeof(bufio), size + bufsize2) - - @support.cpython_only - def test_buffer_freeing(self) : - bufsize = 4096 - rawio = self.MockRawIO() - bufio = self.tp(rawio, buffer_size=bufsize) - size = sys.getsizeof(bufio) - bufsize - bufio.close() - self.assertEqual(sys.getsizeof(bufio), size) - -class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): - read_mode = "rb" - - def test_constructor(self): - rawio = self.MockRawIO([b"abc"]) - bufio = self.tp(rawio) - bufio.__init__(rawio) - bufio.__init__(rawio, buffer_size=1024) - bufio.__init__(rawio, buffer_size=16) - self.assertEqual(b"abc", bufio.read()) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) - rawio = self.MockRawIO([b"abc"]) - bufio.__init__(rawio) - self.assertEqual(b"abc", bufio.read()) - - def test_uninitialized(self): - bufio = self.tp.__new__(self.tp) - del bufio - bufio = self.tp.__new__(self.tp) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - bufio.read, 0) - bufio.__init__(self.MockRawIO()) - self.assertEqual(bufio.read(0), b'') - - def test_read(self): - for arg in (None, 7): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - self.assertEqual(b"abcdefg", bufio.read(arg)) - # Invalid args - self.assertRaises(ValueError, bufio.read, -2) - - def test_read1(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - self.assertEqual(b"a", bufio.read(1)) - self.assertEqual(b"b", bufio.read1(1)) - self.assertEqual(rawio._reads, 1) - self.assertEqual(b"", bufio.read1(0)) - self.assertEqual(b"c", bufio.read1(100)) - self.assertEqual(rawio._reads, 1) - self.assertEqual(b"d", bufio.read1(100)) - self.assertEqual(rawio._reads, 2) - self.assertEqual(b"efg", bufio.read1(100)) - self.assertEqual(rawio._reads, 3) - self.assertEqual(b"", bufio.read1(100)) - self.assertEqual(rawio._reads, 4) - - def test_read1_arbitrary(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - self.assertEqual(b"a", bufio.read(1)) - self.assertEqual(b"bc", bufio.read1()) - self.assertEqual(b"d", bufio.read1()) - self.assertEqual(b"efg", bufio.read1(-1)) - self.assertEqual(rawio._reads, 3) - self.assertEqual(b"", bufio.read1()) - self.assertEqual(rawio._reads, 4) - - def test_readinto(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - b = bytearray(2) - self.assertEqual(bufio.readinto(b), 2) - self.assertEqual(b, b"ab") - self.assertEqual(bufio.readinto(b), 2) - self.assertEqual(b, b"cd") - self.assertEqual(bufio.readinto(b), 2) - self.assertEqual(b, b"ef") - self.assertEqual(bufio.readinto(b), 1) - self.assertEqual(b, b"gf") - self.assertEqual(bufio.readinto(b), 0) - self.assertEqual(b, b"gf") - rawio = self.MockRawIO((b"abc", None)) - bufio = self.tp(rawio) - self.assertEqual(bufio.readinto(b), 2) - self.assertEqual(b, b"ab") - self.assertEqual(bufio.readinto(b), 1) - self.assertEqual(b, b"cb") - - def test_readinto1(self): - buffer_size = 10 - rawio = self.MockRawIO((b"abc", b"de", b"fgh", b"jkl")) - bufio = self.tp(rawio, buffer_size=buffer_size) - b = bytearray(2) - self.assertEqual(bufio.peek(3), b'abc') - self.assertEqual(rawio._reads, 1) - self.assertEqual(bufio.readinto1(b), 2) - self.assertEqual(b, b"ab") - self.assertEqual(rawio._reads, 1) - self.assertEqual(bufio.readinto1(b), 1) - self.assertEqual(b[:1], b"c") - self.assertEqual(rawio._reads, 1) - self.assertEqual(bufio.readinto1(b), 2) - self.assertEqual(b, b"de") - self.assertEqual(rawio._reads, 2) - b = bytearray(2*buffer_size) - self.assertEqual(bufio.peek(3), b'fgh') - self.assertEqual(rawio._reads, 3) - self.assertEqual(bufio.readinto1(b), 6) - self.assertEqual(b[:6], b"fghjkl") - self.assertEqual(rawio._reads, 4) - - def test_readinto_array(self): - buffer_size = 60 - data = b"a" * 26 - rawio = self.MockRawIO((data,)) - bufio = self.tp(rawio, buffer_size=buffer_size) - - # Create an array with element size > 1 byte - b = array.array('i', b'x' * 32) - assert len(b) != 16 - - # Read into it. We should get as many *bytes* as we can fit into b - # (which is more than the number of elements) - n = bufio.readinto(b) - self.assertGreater(n, len(b)) - - # Check that old contents of b are preserved - bm = memoryview(b).cast('B') - self.assertLess(n, len(bm)) - self.assertEqual(bm[:n], data[:n]) - self.assertEqual(bm[n:], b'x' * (len(bm[n:]))) - - def test_readinto1_array(self): - buffer_size = 60 - data = b"a" * 26 - rawio = self.MockRawIO((data,)) - bufio = self.tp(rawio, buffer_size=buffer_size) - - # Create an array with element size > 1 byte - b = array.array('i', b'x' * 32) - assert len(b) != 16 - - # Read into it. We should get as many *bytes* as we can fit into b - # (which is more than the number of elements) - n = bufio.readinto1(b) - self.assertGreater(n, len(b)) - - # Check that old contents of b are preserved - bm = memoryview(b).cast('B') - self.assertLess(n, len(bm)) - self.assertEqual(bm[:n], data[:n]) - self.assertEqual(bm[n:], b'x' * (len(bm[n:]))) - - def test_readlines(self): - def bufio(): - rawio = self.MockRawIO((b"abc\n", b"d\n", b"ef")) - return self.tp(rawio) - self.assertEqual(bufio().readlines(), [b"abc\n", b"d\n", b"ef"]) - self.assertEqual(bufio().readlines(5), [b"abc\n", b"d\n"]) - self.assertEqual(bufio().readlines(None), [b"abc\n", b"d\n", b"ef"]) - - def test_buffering(self): - data = b"abcdefghi" - dlen = len(data) - - tests = [ - [ 100, [ 3, 1, 4, 8 ], [ dlen, 0 ] ], - [ 100, [ 3, 3, 3], [ dlen ] ], - [ 4, [ 1, 2, 4, 2 ], [ 4, 4, 1 ] ], - ] - - for bufsize, buf_read_sizes, raw_read_sizes in tests: - rawio = self.MockFileIO(data) - bufio = self.tp(rawio, buffer_size=bufsize) - pos = 0 - for nbytes in buf_read_sizes: - self.assertEqual(bufio.read(nbytes), data[pos:pos+nbytes]) - pos += nbytes - # this is mildly implementation-dependent - self.assertEqual(rawio.read_history, raw_read_sizes) - - def test_read_non_blocking(self): - # Inject some None's in there to simulate EWOULDBLOCK - rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None)) - bufio = self.tp(rawio) - self.assertEqual(b"abcd", bufio.read(6)) - self.assertEqual(b"e", bufio.read(1)) - self.assertEqual(b"fg", bufio.read()) - self.assertEqual(b"", bufio.peek(1)) - self.assertIsNone(bufio.read()) - self.assertEqual(b"", bufio.read()) - - rawio = self.MockRawIO((b"a", None, None)) - self.assertEqual(b"a", rawio.readall()) - self.assertIsNone(rawio.readall()) - - def test_read_past_eof(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - - self.assertEqual(b"abcdefg", bufio.read(9000)) - - def test_read_all(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - - self.assertEqual(b"abcdefg", bufio.read()) - - @threading_helper.requires_working_threading() - @support.requires_resource('cpu') - def test_threads(self): - try: - # Write out many bytes with exactly the same number of 0's, - # 1's... 255's. This will help us check that concurrent reading - # doesn't duplicate or forget contents. - N = 1000 - l = list(range(256)) * N - random.shuffle(l) - s = bytes(bytearray(l)) - with self.open(os_helper.TESTFN, "wb") as f: - f.write(s) - with self.open(os_helper.TESTFN, self.read_mode, buffering=0) as raw: - bufio = self.tp(raw, 8) - errors = [] - results = [] - def f(): - try: - # Intra-buffer read then buffer-flushing read - for n in cycle([1, 19]): - s = bufio.read(n) - if not s: - break - # list.append() is atomic - results.append(s) - except Exception as e: - errors.append(e) - raise - threads = [threading.Thread(target=f) for x in range(20)] - with threading_helper.start_threads(threads): - time.sleep(0.02) # yield - self.assertFalse(errors, - "the following exceptions were caught: %r" % errors) - s = b''.join(results) - for i in range(256): - c = bytes(bytearray([i])) - self.assertEqual(s.count(c), N) - finally: - os_helper.unlink(os_helper.TESTFN) - - def test_unseekable(self): - bufio = self.tp(self.MockUnseekableIO(b"A" * 10)) - self.assertRaises(self.UnsupportedOperation, bufio.tell) - self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) - bufio.read(1) - self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) - self.assertRaises(self.UnsupportedOperation, bufio.tell) - - def test_misbehaved_io(self): - rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - self.assertRaises(OSError, bufio.seek, 0) - self.assertRaises(OSError, bufio.tell) - - # Silence destructor error - bufio.close = lambda: None - - def test_no_extraneous_read(self): - # Issue #9550; when the raw IO object has satisfied the read request, - # we should not issue any additional reads, otherwise it may block - # (e.g. socket). - bufsize = 16 - for n in (2, bufsize - 1, bufsize, bufsize + 1, bufsize * 2): - rawio = self.MockRawIO([b"x" * n]) - bufio = self.tp(rawio, bufsize) - self.assertEqual(bufio.read(n), b"x" * n) - # Simple case: one raw read is enough to satisfy the request. - self.assertEqual(rawio._extraneous_reads, 0, - "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) - # A more complex case where two raw reads are needed to satisfy - # the request. - rawio = self.MockRawIO([b"x" * (n - 1), b"x"]) - bufio = self.tp(rawio, bufsize) - self.assertEqual(bufio.read(n), b"x" * n) - self.assertEqual(rawio._extraneous_reads, 0, - "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) - - def test_read_on_closed(self): - # Issue #23796 - b = self.BufferedReader(self.BytesIO(b"12")) - b.read(1) - b.close() - with self.subTest('peek'): - self.assertRaises(ValueError, b.peek) - with self.subTest('read1'): - self.assertRaises(ValueError, b.read1, 1) - with self.subTest('read'): - self.assertRaises(ValueError, b.read) - with self.subTest('readinto'): - self.assertRaises(ValueError, b.readinto, bytearray()) - with self.subTest('readinto1'): - self.assertRaises(ValueError, b.readinto1, bytearray()) - with self.subTest('flush'): - self.assertRaises(ValueError, b.flush) - with self.subTest('truncate'): - self.assertRaises(ValueError, b.truncate) - with self.subTest('seek'): - self.assertRaises(ValueError, b.seek, 0) - - def test_truncate_on_read_only(self): - rawio = self.MockFileIO(b"abc") - bufio = self.tp(rawio) - self.assertFalse(bufio.writable()) - self.assertRaises(self.UnsupportedOperation, bufio.truncate) - self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) - - def test_tell_character_device_file(self): - # GH-95782 - # For the (former) bug in BufferedIO to manifest, the wrapped IO obj - # must be able to produce at least 2 bytes. - raw = self.MockCharPseudoDevFileIO(b"12") - buf = self.tp(raw) - self.assertEqual(buf.tell(), 0) - self.assertEqual(buf.read(1), b"1") - self.assertEqual(buf.tell(), 0) - - def test_seek_character_device_file(self): - raw = self.MockCharPseudoDevFileIO(b"12") - buf = self.tp(raw) - self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) - self.assertEqual(buf.seek(1, io.SEEK_SET), 0) - self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) - self.assertEqual(buf.read(1), b"1") - - # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, - # which means that the next seek() could return a negative offset if it - # does not sanity-check: - self.assertEqual(buf.tell(), 0) - self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) - - -class CBufferedReaderTest(BufferedReaderTest, SizeofTest): - tp = io.BufferedReader - - def test_initialization(self): - rawio = self.MockRawIO([b"abc"]) - bufio = self.tp(rawio) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) - self.assertRaises(ValueError, bufio.read) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) - self.assertRaises(ValueError, bufio.read) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) - self.assertRaises(ValueError, bufio.read) - - def test_misbehaved_io_read(self): - rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - # _pyio.BufferedReader seems to implement reading different, so that - # checking this is not so easy. - self.assertRaises(OSError, bufio.read, 10) - - def test_garbage_collection(self): - # C BufferedReader objects are collected. - # The Python version has __del__, so it ends into gc.garbage instead - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with warnings_helper.check_warnings(('', ResourceWarning)): - rawio = self.FileIO(os_helper.TESTFN, "w+b") - f = self.tp(rawio) - f.f = f - wr = weakref.ref(f) - del f - support.gc_collect() - self.assertIsNone(wr(), wr) - - def test_args_error(self): - # Issue #17275 - with self.assertRaisesRegex(TypeError, "BufferedReader"): - self.tp(self.BytesIO(), 1024, 1024, 1024) - - def test_bad_readinto_value(self): - rawio = self.tp(self.BytesIO(b"12")) - rawio.readinto = lambda buf: -1 - bufio = self.tp(rawio) - with self.assertRaises(OSError) as cm: - bufio.readline() - self.assertIsNone(cm.exception.__cause__) - - def test_bad_readinto_type(self): - rawio = self.tp(self.BytesIO(b"12")) - rawio.readinto = lambda buf: b'' - bufio = self.tp(rawio) - with self.assertRaises(OSError) as cm: - bufio.readline() - self.assertIsInstance(cm.exception.__cause__, TypeError) - - -class PyBufferedReaderTest(BufferedReaderTest): - tp = pyio.BufferedReader - - -class BufferedWriterTest(unittest.TestCase, CommonBufferedTests): - write_mode = "wb" - - def test_constructor(self): - rawio = self.MockRawIO() - bufio = self.tp(rawio) - bufio.__init__(rawio) - bufio.__init__(rawio, buffer_size=1024) - bufio.__init__(rawio, buffer_size=16) - self.assertEqual(3, bufio.write(b"abc")) - bufio.flush() - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) - bufio.__init__(rawio) - self.assertEqual(3, bufio.write(b"ghi")) - bufio.flush() - self.assertEqual(b"".join(rawio._write_stack), b"abcghi") - - def test_uninitialized(self): - bufio = self.tp.__new__(self.tp) - del bufio - bufio = self.tp.__new__(self.tp) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - bufio.write, b'') - bufio.__init__(self.MockRawIO()) - self.assertEqual(bufio.write(b''), 0) - - def test_detach_flush(self): - raw = self.MockRawIO() - buf = self.tp(raw) - buf.write(b"howdy!") - self.assertFalse(raw._write_stack) - buf.detach() - self.assertEqual(raw._write_stack, [b"howdy!"]) - - def test_write(self): - # Write to the buffered IO but don't overflow the buffer. - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.write(b"abc") - self.assertFalse(writer._write_stack) - buffer = bytearray(b"def") - bufio.write(buffer) - buffer[:] = b"***" # Overwrite our copy of the data - bufio.flush() - self.assertEqual(b"".join(writer._write_stack), b"abcdef") - - def test_write_overflow(self): - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - contents = b"abcdefghijklmnop" - for n in range(0, len(contents), 3): - bufio.write(contents[n:n+3]) - flushed = b"".join(writer._write_stack) - # At least (total - 8) bytes were implicitly flushed, perhaps more - # depending on the implementation. - self.assertTrue(flushed.startswith(contents[:-8]), flushed) - - def check_writes(self, intermediate_func): - # Lots of writes, test the flushed output is as expected. - contents = bytes(range(256)) * 1000 - n = 0 - writer = self.MockRawIO() - bufio = self.tp(writer, 13) - # Generator of write sizes: repeat each N 15 times then proceed to N+1 - def gen_sizes(): - for size in count(1): - for i in range(15): - yield size - sizes = gen_sizes() - while n < len(contents): - size = min(next(sizes), len(contents) - n) - self.assertEqual(bufio.write(contents[n:n+size]), size) - intermediate_func(bufio) - n += size - bufio.flush() - self.assertEqual(contents, b"".join(writer._write_stack)) - - def test_writes(self): - self.check_writes(lambda bufio: None) - - def test_writes_and_flushes(self): - self.check_writes(lambda bufio: bufio.flush()) - - def test_writes_and_seeks(self): - def _seekabs(bufio): - pos = bufio.tell() - bufio.seek(pos + 1, 0) - bufio.seek(pos - 1, 0) - bufio.seek(pos, 0) - self.check_writes(_seekabs) - def _seekrel(bufio): - pos = bufio.seek(0, 1) - bufio.seek(+1, 1) - bufio.seek(-1, 1) - bufio.seek(pos, 0) - self.check_writes(_seekrel) - - def test_writes_and_truncates(self): - self.check_writes(lambda bufio: bufio.truncate(bufio.tell())) - - def test_write_non_blocking(self): - raw = self.MockNonBlockWriterIO() - bufio = self.tp(raw, 8) - - self.assertEqual(bufio.write(b"abcd"), 4) - self.assertEqual(bufio.write(b"efghi"), 5) - # 1 byte will be written, the rest will be buffered - raw.block_on(b"k") - self.assertEqual(bufio.write(b"jklmn"), 5) - - # 8 bytes will be written, 8 will be buffered and the rest will be lost - raw.block_on(b"0") - try: - bufio.write(b"opqrwxyz0123456789") - except self.BlockingIOError as e: - written = e.characters_written - else: - self.fail("BlockingIOError should have been raised") - self.assertEqual(written, 16) - self.assertEqual(raw.pop_written(), - b"abcdefghijklmnopqrwxyz") - - self.assertEqual(bufio.write(b"ABCDEFGHI"), 9) - s = raw.pop_written() - # Previously buffered bytes were flushed - self.assertTrue(s.startswith(b"01234567A"), s) - - def test_write_and_rewind(self): - raw = self.BytesIO() - bufio = self.tp(raw, 4) - self.assertEqual(bufio.write(b"abcdef"), 6) - self.assertEqual(bufio.tell(), 6) - bufio.seek(0, 0) - self.assertEqual(bufio.write(b"XY"), 2) - bufio.seek(6, 0) - self.assertEqual(raw.getvalue(), b"XYcdef") - self.assertEqual(bufio.write(b"123456"), 6) - bufio.flush() - self.assertEqual(raw.getvalue(), b"XYcdef123456") - - def test_flush(self): - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.write(b"abc") - bufio.flush() - self.assertEqual(b"abc", writer._write_stack[0]) - - def test_writelines(self): - l = [b'ab', b'cd', b'ef'] - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.writelines(l) - bufio.flush() - self.assertEqual(b''.join(writer._write_stack), b'abcdef') - - def test_writelines_userlist(self): - l = UserList([b'ab', b'cd', b'ef']) - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.writelines(l) - bufio.flush() - self.assertEqual(b''.join(writer._write_stack), b'abcdef') - - def test_writelines_error(self): - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - self.assertRaises(TypeError, bufio.writelines, [1, 2, 3]) - self.assertRaises(TypeError, bufio.writelines, None) - self.assertRaises(TypeError, bufio.writelines, 'abc') - - def test_destructor(self): - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.write(b"abc") - del bufio - support.gc_collect() - self.assertEqual(b"abc", writer._write_stack[0]) - - def test_truncate(self): - # Truncate implicitly flushes the buffer. - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw: - bufio = self.tp(raw, 8) - bufio.write(b"abcdef") - self.assertEqual(bufio.truncate(3), 3) - self.assertEqual(bufio.tell(), 6) - with self.open(os_helper.TESTFN, "rb", buffering=0) as f: - self.assertEqual(f.read(), b"abc") - - def test_truncate_after_write(self): - # Ensure that truncate preserves the file position after - # writes longer than the buffer size. - # Issue: https://bugs.python.org/issue32228 - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with self.open(os_helper.TESTFN, "wb") as f: - # Fill with some buffer - f.write(b'\x00' * 10000) - buffer_sizes = [8192, 4096, 200] - for buffer_size in buffer_sizes: - with self.open(os_helper.TESTFN, "r+b", buffering=buffer_size) as f: - f.write(b'\x00' * (buffer_size + 1)) - # After write write_pos and write_end are set to 0 - f.read(1) - # read operation makes sure that pos != raw_pos - f.truncate() - self.assertEqual(f.tell(), buffer_size + 2) - - @threading_helper.requires_working_threading() - @support.requires_resource('cpu') - def test_threads(self): - try: - # Write out many bytes from many threads and test they were - # all flushed. - N = 1000 - contents = bytes(range(256)) * N - sizes = cycle([1, 19]) - n = 0 - queue = deque() - while n < len(contents): - size = next(sizes) - queue.append(contents[n:n+size]) - n += size - del contents - # We use a real file object because it allows us to - # exercise situations where the GIL is released before - # writing the buffer to the raw streams. This is in addition - # to concurrency issues due to switching threads in the middle - # of Python code. - with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw: - bufio = self.tp(raw, 8) - errors = [] - def f(): - try: - while True: - try: - s = queue.popleft() - except IndexError: - return - bufio.write(s) - except Exception as e: - errors.append(e) - raise - threads = [threading.Thread(target=f) for x in range(20)] - with threading_helper.start_threads(threads): - time.sleep(0.02) # yield - self.assertFalse(errors, - "the following exceptions were caught: %r" % errors) - bufio.close() - with self.open(os_helper.TESTFN, "rb") as f: - s = f.read() - for i in range(256): - self.assertEqual(s.count(bytes([i])), N) - finally: - os_helper.unlink(os_helper.TESTFN) - - def test_misbehaved_io(self): - rawio = self.MisbehavedRawIO() - bufio = self.tp(rawio, 5) - self.assertRaises(OSError, bufio.seek, 0) - self.assertRaises(OSError, bufio.tell) - self.assertRaises(OSError, bufio.write, b"abcdef") - - # Silence destructor error - bufio.close = lambda: None - - def test_max_buffer_size_removal(self): - with self.assertRaises(TypeError): - self.tp(self.MockRawIO(), 8, 12) - - def test_write_error_on_close(self): - raw = self.MockRawIO() - def bad_write(b): - raise OSError() - raw.write = bad_write - b = self.tp(raw) - b.write(b'spam') - self.assertRaises(OSError, b.close) # exception not swallowed - self.assertTrue(b.closed) - - @threading_helper.requires_working_threading() - def test_slow_close_from_thread(self): - # Issue #31976 - rawio = self.SlowFlushRawIO() - bufio = self.tp(rawio, 8) - t = threading.Thread(target=bufio.close) - t.start() - rawio.in_flush.wait() - self.assertRaises(ValueError, bufio.write, b'spam') - self.assertTrue(bufio.closed) - t.join() - - - -class CBufferedWriterTest(BufferedWriterTest, SizeofTest): - tp = io.BufferedWriter - - def test_initialization(self): - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) - self.assertRaises(ValueError, bufio.write, b"def") - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) - self.assertRaises(ValueError, bufio.write, b"def") - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) - self.assertRaises(ValueError, bufio.write, b"def") - - def test_garbage_collection(self): - # C BufferedWriter objects are collected, and collecting them flushes - # all data to disk. - # The Python version has __del__, so it ends into gc.garbage instead - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with warnings_helper.check_warnings(('', ResourceWarning)): - rawio = self.FileIO(os_helper.TESTFN, "w+b") - f = self.tp(rawio) - f.write(b"123xxx") - f.x = f - wr = weakref.ref(f) - del f - support.gc_collect() - self.assertIsNone(wr(), wr) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"123xxx") - - def test_args_error(self): - # Issue #17275 - with self.assertRaisesRegex(TypeError, "BufferedWriter"): - self.tp(self.BytesIO(), 1024, 1024, 1024) - - -class PyBufferedWriterTest(BufferedWriterTest): - tp = pyio.BufferedWriter - -class BufferedRWPairTest(unittest.TestCase): - - def test_constructor(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertFalse(pair.closed) - - def test_uninitialized(self): - pair = self.tp.__new__(self.tp) - del pair - pair = self.tp.__new__(self.tp) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - pair.read, 0) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - pair.write, b'') - pair.__init__(self.MockRawIO(), self.MockRawIO()) - self.assertEqual(pair.read(0), b'') - self.assertEqual(pair.write(b''), 0) - - def test_detach(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertRaises(self.UnsupportedOperation, pair.detach) - - def test_constructor_max_buffer_size_removal(self): - with self.assertRaises(TypeError): - self.tp(self.MockRawIO(), self.MockRawIO(), 8, 12) - - def test_constructor_with_not_readable(self): - class NotReadable(MockRawIO): - def readable(self): - return False - - self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO()) - - def test_constructor_with_not_writeable(self): - class NotWriteable(MockRawIO): - def writable(self): - return False - - self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable()) - - def test_read(self): - pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - - self.assertEqual(pair.read(3), b"abc") - self.assertEqual(pair.read(1), b"d") - self.assertEqual(pair.read(), b"ef") - pair = self.tp(self.BytesIO(b"abc"), self.MockRawIO()) - self.assertEqual(pair.read(None), b"abc") - - def test_readlines(self): - pair = lambda: self.tp(self.BytesIO(b"abc\ndef\nh"), self.MockRawIO()) - self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) - self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) - self.assertEqual(pair().readlines(5), [b"abc\n", b"def\n"]) - - def test_read1(self): - # .read1() is delegated to the underlying reader object, so this test - # can be shallow. - pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - - self.assertEqual(pair.read1(3), b"abc") - self.assertEqual(pair.read1(), b"def") - - def test_readinto(self): - for method in ("readinto", "readinto1"): - with self.subTest(method): - pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - - data = byteslike(b'\0' * 5) - self.assertEqual(getattr(pair, method)(data), 5) - self.assertEqual(bytes(data), b"abcde") - - def test_write(self): - w = self.MockRawIO() - pair = self.tp(self.MockRawIO(), w) - - pair.write(b"abc") - pair.flush() - buffer = bytearray(b"def") - pair.write(buffer) - buffer[:] = b"***" # Overwrite our copy of the data - pair.flush() - self.assertEqual(w._write_stack, [b"abc", b"def"]) - - def test_peek(self): - pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - - self.assertTrue(pair.peek(3).startswith(b"abc")) - self.assertEqual(pair.read(3), b"abc") - - def test_readable(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertTrue(pair.readable()) - - def test_writeable(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertTrue(pair.writable()) - - def test_seekable(self): - # BufferedRWPairs are never seekable, even if their readers and writers - # are. - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertFalse(pair.seekable()) - - # .flush() is delegated to the underlying writer object and has been - # tested in the test_write method. - - def test_close_and_closed(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertFalse(pair.closed) - pair.close() - self.assertTrue(pair.closed) - - def test_reader_close_error_on_close(self): - def reader_close(): - reader_non_existing - reader = self.MockRawIO() - reader.close = reader_close - writer = self.MockRawIO() - pair = self.tp(reader, writer) - with self.assertRaises(NameError) as err: - pair.close() - self.assertIn('reader_non_existing', str(err.exception)) - self.assertTrue(pair.closed) - self.assertFalse(reader.closed) - self.assertTrue(writer.closed) - - # Silence destructor error - reader.close = lambda: None - - def test_writer_close_error_on_close(self): - def writer_close(): - writer_non_existing - reader = self.MockRawIO() - writer = self.MockRawIO() - writer.close = writer_close - pair = self.tp(reader, writer) - with self.assertRaises(NameError) as err: - pair.close() - self.assertIn('writer_non_existing', str(err.exception)) - self.assertFalse(pair.closed) - self.assertTrue(reader.closed) - self.assertFalse(writer.closed) - - # Silence destructor error - writer.close = lambda: None - writer = None - - # Ignore BufferedWriter (of the BufferedRWPair) unraisable exception - with support.catch_unraisable_exception(): - # Ignore BufferedRWPair unraisable exception - with support.catch_unraisable_exception(): - pair = None - support.gc_collect() - support.gc_collect() - - def test_reader_writer_close_error_on_close(self): - def reader_close(): - reader_non_existing - def writer_close(): - writer_non_existing - reader = self.MockRawIO() - reader.close = reader_close - writer = self.MockRawIO() - writer.close = writer_close - pair = self.tp(reader, writer) - with self.assertRaises(NameError) as err: - pair.close() - self.assertIn('reader_non_existing', str(err.exception)) - self.assertIsInstance(err.exception.__context__, NameError) - self.assertIn('writer_non_existing', str(err.exception.__context__)) - self.assertFalse(pair.closed) - self.assertFalse(reader.closed) - self.assertFalse(writer.closed) - - # Silence destructor error - reader.close = lambda: None - writer.close = lambda: None - - def test_isatty(self): - class SelectableIsAtty(MockRawIO): - def __init__(self, isatty): - MockRawIO.__init__(self) - self._isatty = isatty - - def isatty(self): - return self._isatty - - pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(False)) - self.assertFalse(pair.isatty()) - - pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(False)) - self.assertTrue(pair.isatty()) - - pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(True)) - self.assertTrue(pair.isatty()) - - pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(True)) - self.assertTrue(pair.isatty()) - - def test_weakref_clearing(self): - brw = self.tp(self.MockRawIO(), self.MockRawIO()) - ref = weakref.ref(brw) - brw = None - ref = None # Shouldn't segfault. - -class CBufferedRWPairTest(BufferedRWPairTest): - tp = io.BufferedRWPair - -class PyBufferedRWPairTest(BufferedRWPairTest): - tp = pyio.BufferedRWPair - - -class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest): - read_mode = "rb+" - write_mode = "wb+" - - def test_constructor(self): - BufferedReaderTest.test_constructor(self) - BufferedWriterTest.test_constructor(self) - - def test_uninitialized(self): - BufferedReaderTest.test_uninitialized(self) - BufferedWriterTest.test_uninitialized(self) - - def test_read_and_write(self): - raw = self.MockRawIO((b"asdf", b"ghjk")) - rw = self.tp(raw, 8) - - self.assertEqual(b"as", rw.read(2)) - rw.write(b"ddd") - rw.write(b"eee") - self.assertFalse(raw._write_stack) # Buffer writes - self.assertEqual(b"ghjk", rw.read()) - self.assertEqual(b"dddeee", raw._write_stack[0]) - - def test_seek_and_tell(self): - raw = self.BytesIO(b"asdfghjkl") - rw = self.tp(raw) - - self.assertEqual(b"as", rw.read(2)) - self.assertEqual(2, rw.tell()) - rw.seek(0, 0) - self.assertEqual(b"asdf", rw.read(4)) - - rw.write(b"123f") - rw.seek(0, 0) - self.assertEqual(b"asdf123fl", rw.read()) - self.assertEqual(9, rw.tell()) - rw.seek(-4, 2) - self.assertEqual(5, rw.tell()) - rw.seek(2, 1) - self.assertEqual(7, rw.tell()) - self.assertEqual(b"fl", rw.read(11)) - rw.flush() - self.assertEqual(b"asdf123fl", raw.getvalue()) - - self.assertRaises(TypeError, rw.seek, 0.0) - - def check_flush_and_read(self, read_func): - raw = self.BytesIO(b"abcdefghi") - bufio = self.tp(raw) - - self.assertEqual(b"ab", read_func(bufio, 2)) - bufio.write(b"12") - self.assertEqual(b"ef", read_func(bufio, 2)) - self.assertEqual(6, bufio.tell()) - bufio.flush() - self.assertEqual(6, bufio.tell()) - self.assertEqual(b"ghi", read_func(bufio)) - raw.seek(0, 0) - raw.write(b"XYZ") - # flush() resets the read buffer - bufio.flush() - bufio.seek(0, 0) - self.assertEqual(b"XYZ", read_func(bufio, 3)) - - def test_flush_and_read(self): - self.check_flush_and_read(lambda bufio, *args: bufio.read(*args)) - - def test_flush_and_readinto(self): - def _readinto(bufio, n=-1): - b = bytearray(n if n >= 0 else 9999) - n = bufio.readinto(b) - return bytes(b[:n]) - self.check_flush_and_read(_readinto) - - def test_flush_and_peek(self): - def _peek(bufio, n=-1): - # This relies on the fact that the buffer can contain the whole - # raw stream, otherwise peek() can return less. - b = bufio.peek(n) - if n != -1: - b = b[:n] - bufio.seek(len(b), 1) - return b - self.check_flush_and_read(_peek) - - def test_flush_and_write(self): - raw = self.BytesIO(b"abcdefghi") - bufio = self.tp(raw) - - bufio.write(b"123") - bufio.flush() - bufio.write(b"45") - bufio.flush() - bufio.seek(0, 0) - self.assertEqual(b"12345fghi", raw.getvalue()) - self.assertEqual(b"12345fghi", bufio.read()) - - def test_threads(self): - BufferedReaderTest.test_threads(self) - BufferedWriterTest.test_threads(self) - - def test_writes_and_peek(self): - def _peek(bufio): - bufio.peek(1) - self.check_writes(_peek) - def _peek(bufio): - pos = bufio.tell() - bufio.seek(-1, 1) - bufio.peek(1) - bufio.seek(pos, 0) - self.check_writes(_peek) - - def test_writes_and_reads(self): - def _read(bufio): - bufio.seek(-1, 1) - bufio.read(1) - self.check_writes(_read) - - def test_writes_and_read1s(self): - def _read1(bufio): - bufio.seek(-1, 1) - bufio.read1(1) - self.check_writes(_read1) - - def test_writes_and_readintos(self): - def _read(bufio): - bufio.seek(-1, 1) - bufio.readinto(bytearray(1)) - self.check_writes(_read) - - def test_write_after_readahead(self): - # Issue #6629: writing after the buffer was filled by readahead should - # first rewind the raw stream. - for overwrite_size in [1, 5]: - raw = self.BytesIO(b"A" * 10) - bufio = self.tp(raw, 4) - # Trigger readahead - self.assertEqual(bufio.read(1), b"A") - self.assertEqual(bufio.tell(), 1) - # Overwriting should rewind the raw stream if it needs so - bufio.write(b"B" * overwrite_size) - self.assertEqual(bufio.tell(), overwrite_size + 1) - # If the write size was smaller than the buffer size, flush() and - # check that rewind happens. - bufio.flush() - self.assertEqual(bufio.tell(), overwrite_size + 1) - s = raw.getvalue() - self.assertEqual(s, - b"A" + b"B" * overwrite_size + b"A" * (9 - overwrite_size)) - - def test_write_rewind_write(self): - # Various combinations of reading / writing / seeking backwards / writing again - def mutate(bufio, pos1, pos2): - assert pos2 >= pos1 - # Fill the buffer - bufio.seek(pos1) - bufio.read(pos2 - pos1) - bufio.write(b'\x02') - # This writes earlier than the previous write, but still inside - # the buffer. - bufio.seek(pos1) - bufio.write(b'\x01') - - b = b"\x80\x81\x82\x83\x84" - for i in range(0, len(b)): - for j in range(i, len(b)): - raw = self.BytesIO(b) - bufio = self.tp(raw, 100) - mutate(bufio, i, j) - bufio.flush() - expected = bytearray(b) - expected[j] = 2 - expected[i] = 1 - self.assertEqual(raw.getvalue(), expected, - "failed result for i=%d, j=%d" % (i, j)) - - def test_truncate_after_read_or_write(self): - raw = self.BytesIO(b"A" * 10) - bufio = self.tp(raw, 100) - self.assertEqual(bufio.read(2), b"AA") # the read buffer gets filled - self.assertEqual(bufio.truncate(), 2) - self.assertEqual(bufio.write(b"BB"), 2) # the write buffer increases - self.assertEqual(bufio.truncate(), 4) - - def test_misbehaved_io(self): - BufferedReaderTest.test_misbehaved_io(self) - BufferedWriterTest.test_misbehaved_io(self) - - def test_interleaved_read_write(self): - # Test for issue #12213 - with self.BytesIO(b'abcdefgh') as raw: - with self.tp(raw, 100) as f: - f.write(b"1") - self.assertEqual(f.read(1), b'b') - f.write(b'2') - self.assertEqual(f.read1(1), b'd') - f.write(b'3') - buf = bytearray(1) - f.readinto(buf) - self.assertEqual(buf, b'f') - f.write(b'4') - self.assertEqual(f.peek(1), b'h') - f.flush() - self.assertEqual(raw.getvalue(), b'1b2d3f4h') - - with self.BytesIO(b'abc') as raw: - with self.tp(raw, 100) as f: - self.assertEqual(f.read(1), b'a') - f.write(b"2") - self.assertEqual(f.read(1), b'c') - f.flush() - self.assertEqual(raw.getvalue(), b'a2c') - - def test_read1_after_write(self): - with self.BytesIO(b'abcdef') as raw: - with self.tp(raw, 3) as f: - f.write(b"1") - self.assertEqual(f.read1(1), b'b') - f.flush() - self.assertEqual(raw.getvalue(), b'1bcdef') - with self.BytesIO(b'abcdef') as raw: - with self.tp(raw, 3) as f: - f.write(b"1") - self.assertEqual(f.read1(), b'bcd') - f.flush() - self.assertEqual(raw.getvalue(), b'1bcdef') - with self.BytesIO(b'abcdef') as raw: - with self.tp(raw, 3) as f: - f.write(b"1") - # XXX: read(100) returns different numbers of bytes - # in Python and C implementations. - self.assertEqual(f.read1(100)[:3], b'bcd') - f.flush() - self.assertEqual(raw.getvalue(), b'1bcdef') - - def test_interleaved_readline_write(self): - with self.BytesIO(b'ab\ncdef\ng\n') as raw: - with self.tp(raw) as f: - f.write(b'1') - self.assertEqual(f.readline(), b'b\n') - f.write(b'2') - self.assertEqual(f.readline(), b'def\n') - f.write(b'3') - self.assertEqual(f.readline(), b'\n') - f.flush() - self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n') - - # You can't construct a BufferedRandom over a non-seekable stream. - test_unseekable = None - - # writable() returns True, so there's no point to test it over - # a writable stream. - test_truncate_on_read_only = None - - -class CBufferedRandomTest(BufferedRandomTest, SizeofTest): - tp = io.BufferedRandom - - def test_garbage_collection(self): - CBufferedReaderTest.test_garbage_collection(self) - CBufferedWriterTest.test_garbage_collection(self) - - def test_args_error(self): - # Issue #17275 - with self.assertRaisesRegex(TypeError, "BufferedRandom"): - self.tp(self.BytesIO(), 1024, 1024, 1024) - - -class PyBufferedRandomTest(BufferedRandomTest): - tp = pyio.BufferedRandom - - -# To fully exercise seek/tell, the StatefulIncrementalDecoder has these -# properties: -# - A single output character can correspond to many bytes of input. -# - The number of input bytes to complete the character can be -# undetermined until the last input byte is received. -# - The number of input bytes can vary depending on previous input. -# - A single input byte can correspond to many characters of output. -# - The number of output characters can be undetermined until the -# last input byte is received. -# - The number of output characters can vary depending on previous input. - -class StatefulIncrementalDecoder(codecs.IncrementalDecoder): - """ - For testing seek/tell behavior with a stateful, buffering decoder. - - Input is a sequence of words. Words may be fixed-length (length set - by input) or variable-length (period-terminated). In variable-length - mode, extra periods are ignored. Possible words are: - - 'i' followed by a number sets the input length, I (maximum 99). - When I is set to 0, words are space-terminated. - - 'o' followed by a number sets the output length, O (maximum 99). - - Any other word is converted into a word followed by a period on - the output. The output word consists of the input word truncated - or padded out with hyphens to make its length equal to O. If O - is 0, the word is output verbatim without truncating or padding. - I and O are initially set to 1. When I changes, any buffered input is - re-scanned according to the new I. EOF also terminates the last word. - """ - - def __init__(self, errors='strict'): - codecs.IncrementalDecoder.__init__(self, errors) - self.reset() - - def __repr__(self): - return '<SID %x>' % id(self) - - def reset(self): - self.i = 1 - self.o = 1 - self.buffer = bytearray() - - def getstate(self): - i, o = self.i ^ 1, self.o ^ 1 # so that flags = 0 after reset() - return bytes(self.buffer), i*100 + o - - def setstate(self, state): - buffer, io = state - self.buffer = bytearray(buffer) - i, o = divmod(io, 100) - self.i, self.o = i ^ 1, o ^ 1 - - def decode(self, input, final=False): - output = '' - for b in input: - if self.i == 0: # variable-length, terminated with period - if b == ord('.'): - if self.buffer: - output += self.process_word() - else: - self.buffer.append(b) - else: # fixed-length, terminate after self.i bytes - self.buffer.append(b) - if len(self.buffer) == self.i: - output += self.process_word() - if final and self.buffer: # EOF terminates the last word - output += self.process_word() - return output - - def process_word(self): - output = '' - if self.buffer[0] == ord('i'): - self.i = min(99, int(self.buffer[1:] or 0)) # set input length - elif self.buffer[0] == ord('o'): - self.o = min(99, int(self.buffer[1:] or 0)) # set output length - else: - output = self.buffer.decode('ascii') - if len(output) < self.o: - output += '-'*self.o # pad out with hyphens - if self.o: - output = output[:self.o] # truncate to output length - output += '.' - self.buffer = bytearray() - return output - - codecEnabled = False - - -# bpo-41919: This method is separated from StatefulIncrementalDecoder to avoid a resource leak -# when registering codecs and cleanup functions. -def lookupTestDecoder(name): - if StatefulIncrementalDecoder.codecEnabled and name == 'test_decoder': - latin1 = codecs.lookup('latin-1') - return codecs.CodecInfo( - name='test_decoder', encode=latin1.encode, decode=None, - incrementalencoder=None, - streamreader=None, streamwriter=None, - incrementaldecoder=StatefulIncrementalDecoder) - - -class StatefulIncrementalDecoderTest(unittest.TestCase): - """ - Make sure the StatefulIncrementalDecoder actually works. - """ - - test_cases = [ - # I=1, O=1 (fixed-length input == fixed-length output) - (b'abcd', False, 'a.b.c.d.'), - # I=0, O=0 (variable-length input, variable-length output) - (b'oiabcd', True, 'abcd.'), - # I=0, O=0 (should ignore extra periods) - (b'oi...abcd...', True, 'abcd.'), - # I=0, O=6 (variable-length input, fixed-length output) - (b'i.o6.x.xyz.toolongtofit.', False, 'x-----.xyz---.toolon.'), - # I=2, O=6 (fixed-length input < fixed-length output) - (b'i.i2.o6xyz', True, 'xy----.z-----.'), - # I=6, O=3 (fixed-length input > fixed-length output) - (b'i.o3.i6.abcdefghijklmnop', True, 'abc.ghi.mno.'), - # I=0, then 3; O=29, then 15 (with longer output) - (b'i.o29.a.b.cde.o15.abcdefghijabcdefghij.i3.a.b.c.d.ei00k.l.m', True, - 'a----------------------------.' + - 'b----------------------------.' + - 'cde--------------------------.' + - 'abcdefghijabcde.' + - 'a.b------------.' + - '.c.------------.' + - 'd.e------------.' + - 'k--------------.' + - 'l--------------.' + - 'm--------------.') - ] - - def test_decoder(self): - # Try a few one-shot test cases. - for input, eof, output in self.test_cases: - d = StatefulIncrementalDecoder() - self.assertEqual(d.decode(input, eof), output) - - # Also test an unfinished decode, followed by forcing EOF. - d = StatefulIncrementalDecoder() - self.assertEqual(d.decode(b'oiabcd'), '') - self.assertEqual(d.decode(b'', 1), 'abcd.') - -class TextIOWrapperTest(unittest.TestCase): - - def setUp(self): - self.testdata = b"AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n" - self.normalized = b"AAA\nBBB\nCCC\nDDD\nEEE\n".decode("ascii") - os_helper.unlink(os_helper.TESTFN) - codecs.register(lookupTestDecoder) - self.addCleanup(codecs.unregister, lookupTestDecoder) - - def tearDown(self): - os_helper.unlink(os_helper.TESTFN) - - def test_constructor(self): - r = self.BytesIO(b"\xc3\xa9\n\n") - b = self.BufferedReader(r, 1000) - t = self.TextIOWrapper(b, encoding="utf-8") - t.__init__(b, encoding="latin-1", newline="\r\n") - self.assertEqual(t.encoding, "latin-1") - self.assertEqual(t.line_buffering, False) - t.__init__(b, encoding="utf-8", line_buffering=True) - self.assertEqual(t.encoding, "utf-8") - self.assertEqual(t.line_buffering, True) - self.assertEqual("\xe9\n", t.readline()) - invalid_type = TypeError if self.is_C else ValueError - with self.assertRaises(invalid_type): - t.__init__(b, encoding=42) - with self.assertRaises(UnicodeEncodeError): - t.__init__(b, encoding='\udcfe') - with self.assertRaises(ValueError): - t.__init__(b, encoding='utf-8\0') - with self.assertRaises(invalid_type): - t.__init__(b, encoding="utf-8", errors=42) - if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: - with self.assertRaises(UnicodeEncodeError): - t.__init__(b, encoding="utf-8", errors='\udcfe') - if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: - with self.assertRaises(ValueError): - t.__init__(b, encoding="utf-8", errors='replace\0') - with self.assertRaises(TypeError): - t.__init__(b, encoding="utf-8", newline=42) - with self.assertRaises(ValueError): - t.__init__(b, encoding="utf-8", newline='\udcfe') - with self.assertRaises(ValueError): - t.__init__(b, encoding="utf-8", newline='\n\0') - with self.assertRaises(ValueError): - t.__init__(b, encoding="utf-8", newline='xyzzy') - - def test_uninitialized(self): - t = self.TextIOWrapper.__new__(self.TextIOWrapper) - del t - t = self.TextIOWrapper.__new__(self.TextIOWrapper) - self.assertRaises(Exception, repr, t) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - t.read, 0) - t.__init__(self.MockRawIO(), encoding="utf-8") - self.assertEqual(t.read(0), '') - - def test_non_text_encoding_codecs_are_rejected(self): - # Ensure the constructor complains if passed a codec that isn't - # marked as a text encoding - # http://bugs.python.org/issue20404 - r = self.BytesIO() - b = self.BufferedWriter(r) - with self.assertRaisesRegex(LookupError, "is not a text encoding"): - self.TextIOWrapper(b, encoding="hex") - - def test_detach(self): - r = self.BytesIO() - b = self.BufferedWriter(r) - t = self.TextIOWrapper(b, encoding="ascii") - self.assertIs(t.detach(), b) - - t = self.TextIOWrapper(b, encoding="ascii") - t.write("howdy") - self.assertFalse(r.getvalue()) - t.detach() - self.assertEqual(r.getvalue(), b"howdy") - self.assertRaises(ValueError, t.detach) - - # Operations independent of the detached stream should still work - repr(t) - self.assertEqual(t.encoding, "ascii") - self.assertEqual(t.errors, "strict") - self.assertFalse(t.line_buffering) - self.assertFalse(t.write_through) - - def test_repr(self): - raw = self.BytesIO("hello".encode("utf-8")) - b = self.BufferedReader(raw) - t = self.TextIOWrapper(b, encoding="utf-8") - modname = self.TextIOWrapper.__module__ - self.assertRegex(repr(t), - r"<(%s\.)?TextIOWrapper encoding='utf-8'>" % modname) - raw.name = "dummy" - self.assertRegex(repr(t), - r"<(%s\.)?TextIOWrapper name='dummy' encoding='utf-8'>" % modname) - t.mode = "r" - self.assertRegex(repr(t), - r"<(%s\.)?TextIOWrapper name='dummy' mode='r' encoding='utf-8'>" % modname) - raw.name = b"dummy" - self.assertRegex(repr(t), - r"<(%s\.)?TextIOWrapper name=b'dummy' mode='r' encoding='utf-8'>" % modname) - - t.buffer.detach() - repr(t) # Should not raise an exception - - def test_recursive_repr(self): - # Issue #25455 - raw = self.BytesIO() - t = self.TextIOWrapper(raw, encoding="utf-8") - with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): - with self.assertRaises(RuntimeError): - repr(t) # Should not crash - - def test_subclass_repr(self): - class TestSubclass(self.TextIOWrapper): - pass - - f = TestSubclass(self.StringIO()) - self.assertIn(TestSubclass.__name__, repr(f)) - - def test_line_buffering(self): - r = self.BytesIO() - b = self.BufferedWriter(r, 1000) - t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=True) - t.write("X") - self.assertEqual(r.getvalue(), b"") # No flush happened - t.write("Y\nZ") - self.assertEqual(r.getvalue(), b"XY\nZ") # All got flushed - t.write("A\rB") - self.assertEqual(r.getvalue(), b"XY\nZA\rB") - - def test_reconfigure_line_buffering(self): - r = self.BytesIO() - b = self.BufferedWriter(r, 1000) - t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=False) - t.write("AB\nC") - self.assertEqual(r.getvalue(), b"") - - t.reconfigure(line_buffering=True) # implicit flush - self.assertEqual(r.getvalue(), b"AB\nC") - t.write("DEF\nG") - self.assertEqual(r.getvalue(), b"AB\nCDEF\nG") - t.write("H") - self.assertEqual(r.getvalue(), b"AB\nCDEF\nG") - t.reconfigure(line_buffering=False) # implicit flush - self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH") - t.write("IJ") - self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH") - - # Keeping default value - t.reconfigure() - t.reconfigure(line_buffering=None) - self.assertEqual(t.line_buffering, False) - t.reconfigure(line_buffering=True) - t.reconfigure() - t.reconfigure(line_buffering=None) - self.assertEqual(t.line_buffering, True) - - @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") - def test_default_encoding(self): - with os_helper.EnvironmentVarGuard() as env: - # try to get a user preferred encoding different than the current - # locale encoding to check that TextIOWrapper() uses the current - # locale encoding and not the user preferred encoding - env.unset('LC_ALL', 'LANG', 'LC_CTYPE') - - current_locale_encoding = locale.getencoding() - b = self.BytesIO() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", EncodingWarning) - t = self.TextIOWrapper(b) - self.assertEqual(t.encoding, current_locale_encoding) - - def test_encoding(self): - # Check the encoding attribute is always set, and valid - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="utf-8") - self.assertEqual(t.encoding, "utf-8") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", EncodingWarning) - t = self.TextIOWrapper(b) - self.assertIsNotNone(t.encoding) - codecs.lookup(t.encoding) - - def test_encoding_errors_reading(self): - # (1) default - b = self.BytesIO(b"abc\n\xff\n") - t = self.TextIOWrapper(b, encoding="ascii") - self.assertRaises(UnicodeError, t.read) - # (2) explicit strict - b = self.BytesIO(b"abc\n\xff\n") - t = self.TextIOWrapper(b, encoding="ascii", errors="strict") - self.assertRaises(UnicodeError, t.read) - # (3) ignore - b = self.BytesIO(b"abc\n\xff\n") - t = self.TextIOWrapper(b, encoding="ascii", errors="ignore") - self.assertEqual(t.read(), "abc\n\n") - # (4) replace - b = self.BytesIO(b"abc\n\xff\n") - t = self.TextIOWrapper(b, encoding="ascii", errors="replace") - self.assertEqual(t.read(), "abc\n\ufffd\n") - - def test_encoding_errors_writing(self): - # (1) default - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="ascii") - self.assertRaises(UnicodeError, t.write, "\xff") - # (2) explicit strict - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="ascii", errors="strict") - self.assertRaises(UnicodeError, t.write, "\xff") - # (3) ignore - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="ascii", errors="ignore", - newline="\n") - t.write("abc\xffdef\n") - t.flush() - self.assertEqual(b.getvalue(), b"abcdef\n") - # (4) replace - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="ascii", errors="replace", - newline="\n") - t.write("abc\xffdef\n") - t.flush() - self.assertEqual(b.getvalue(), b"abc?def\n") - - def test_newlines(self): - input_lines = [ "unix\n", "windows\r\n", "os9\r", "last\n", "nonl" ] - - tests = [ - [ None, [ 'unix\n', 'windows\n', 'os9\n', 'last\n', 'nonl' ] ], - [ '', input_lines ], - [ '\n', [ "unix\n", "windows\r\n", "os9\rlast\n", "nonl" ] ], - [ '\r\n', [ "unix\nwindows\r\n", "os9\rlast\nnonl" ] ], - [ '\r', [ "unix\nwindows\r", "\nos9\r", "last\nnonl" ] ], - ] - encodings = ( - 'utf-8', 'latin-1', - 'utf-16', 'utf-16-le', 'utf-16-be', - 'utf-32', 'utf-32-le', 'utf-32-be', - ) - - # Try a range of buffer sizes to test the case where \r is the last - # character in TextIOWrapper._pending_line. - for encoding in encodings: - # XXX: str.encode() should return bytes - data = bytes(''.join(input_lines).encode(encoding)) - for do_reads in (False, True): - for bufsize in range(1, 10): - for newline, exp_lines in tests: - bufio = self.BufferedReader(self.BytesIO(data), bufsize) - textio = self.TextIOWrapper(bufio, newline=newline, - encoding=encoding) - if do_reads: - got_lines = [] - while True: - c2 = textio.read(2) - if c2 == '': - break - self.assertEqual(len(c2), 2) - got_lines.append(c2 + textio.readline()) - else: - got_lines = list(textio) - - for got_line, exp_line in zip(got_lines, exp_lines): - self.assertEqual(got_line, exp_line) - self.assertEqual(len(got_lines), len(exp_lines)) - - def test_newlines_input(self): - testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG" - normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - for newline, expected in [ - (None, normalized.decode("ascii").splitlines(keepends=True)), - ("", testdata.decode("ascii").splitlines(keepends=True)), - ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), - ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), - ("\r", ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]), - ]: - buf = self.BytesIO(testdata) - txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline) - self.assertEqual(txt.readlines(), expected) - txt.seek(0) - self.assertEqual(txt.read(), "".join(expected)) - - def test_newlines_output(self): - testdict = { - "": b"AAA\nBBB\nCCC\nX\rY\r\nZ", - "\n": b"AAA\nBBB\nCCC\nX\rY\r\nZ", - "\r": b"AAA\rBBB\rCCC\rX\rY\r\rZ", - "\r\n": b"AAA\r\nBBB\r\nCCC\r\nX\rY\r\r\nZ", - } - tests = [(None, testdict[os.linesep])] + sorted(testdict.items()) - for newline, expected in tests: - buf = self.BytesIO() - txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline) - txt.write("AAA\nB") - txt.write("BB\nCCC\n") - txt.write("X\rY\r\nZ") - txt.flush() - self.assertEqual(buf.closed, False) - self.assertEqual(buf.getvalue(), expected) - - def test_destructor(self): - l = [] - base = self.BytesIO - class MyBytesIO(base): - def close(self): - l.append(self.getvalue()) - base.close(self) - b = MyBytesIO() - t = self.TextIOWrapper(b, encoding="ascii") - t.write("abc") - del t - support.gc_collect() - self.assertEqual([b"abc"], l) - - def test_override_destructor(self): - record = [] - class MyTextIO(self.TextIOWrapper): - def __del__(self): - record.append(1) - try: - f = super().__del__ - except AttributeError: - pass - else: - f() - def close(self): - record.append(2) - super().close() - def flush(self): - record.append(3) - super().flush() - b = self.BytesIO() - t = MyTextIO(b, encoding="ascii") - del t - support.gc_collect() - self.assertEqual(record, [1, 2, 3]) - - def test_error_through_destructor(self): - # Test that the exception state is not modified by a destructor, - # even if close() fails. - rawio = self.CloseFailureIO() - with support.catch_unraisable_exception() as cm: - with self.assertRaises(AttributeError): - self.TextIOWrapper(rawio, encoding="utf-8").xyzzy - - self.assertEqual(cm.unraisable.exc_type, OSError) - - # Systematic tests of the text I/O API - - def test_basic_io(self): - for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65): - for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le": - f = self.open(os_helper.TESTFN, "w+", encoding=enc) - f._CHUNK_SIZE = chunksize - self.assertEqual(f.write("abc"), 3) - f.close() - f = self.open(os_helper.TESTFN, "r+", encoding=enc) - f._CHUNK_SIZE = chunksize - self.assertEqual(f.tell(), 0) - self.assertEqual(f.read(), "abc") - cookie = f.tell() - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.read(None), "abc") - f.seek(0) - self.assertEqual(f.read(2), "ab") - self.assertEqual(f.read(1), "c") - self.assertEqual(f.read(1), "") - self.assertEqual(f.read(), "") - self.assertEqual(f.tell(), cookie) - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.seek(0, 2), cookie) - self.assertEqual(f.write("def"), 3) - self.assertEqual(f.seek(cookie), cookie) - self.assertEqual(f.read(), "def") - if enc.startswith("utf"): - self.multi_line_test(f, enc) - f.close() - - def multi_line_test(self, f, enc): - f.seek(0) - f.truncate() - sample = "s\xff\u0fff\uffff" - wlines = [] - for size in (0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 62, 63, 64, 65, 1000): - chars = [] - for i in range(size): - chars.append(sample[i % len(sample)]) - line = "".join(chars) + "\n" - wlines.append((f.tell(), line)) - f.write(line) - f.seek(0) - rlines = [] - while True: - pos = f.tell() - line = f.readline() - if not line: - break - rlines.append((pos, line)) - self.assertEqual(rlines, wlines) - - def test_telling(self): - f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") - p0 = f.tell() - f.write("\xff\n") - p1 = f.tell() - f.write("\xff\n") - p2 = f.tell() - f.seek(0) - self.assertEqual(f.tell(), p0) - self.assertEqual(f.readline(), "\xff\n") - self.assertEqual(f.tell(), p1) - self.assertEqual(f.readline(), "\xff\n") - self.assertEqual(f.tell(), p2) - f.seek(0) - for line in f: - self.assertEqual(line, "\xff\n") - self.assertRaises(OSError, f.tell) - self.assertEqual(f.tell(), p2) - f.close() - - def test_seeking(self): - chunk_size = _default_chunk_size() - prefix_size = chunk_size - 2 - u_prefix = "a" * prefix_size - prefix = bytes(u_prefix.encode("utf-8")) - self.assertEqual(len(u_prefix), len(prefix)) - u_suffix = "\u8888\n" - suffix = bytes(u_suffix.encode("utf-8")) - line = prefix + suffix - with self.open(os_helper.TESTFN, "wb") as f: - f.write(line*2) - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - s = f.read(prefix_size) - self.assertEqual(s, str(prefix, "ascii")) - self.assertEqual(f.tell(), prefix_size) - self.assertEqual(f.readline(), u_suffix) - - def test_seeking_too(self): - # Regression test for a specific bug - data = b'\xe0\xbf\xbf\n' - with self.open(os_helper.TESTFN, "wb") as f: - f.write(data) - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - f._CHUNK_SIZE # Just test that it exists - f._CHUNK_SIZE = 2 - f.readline() - f.tell() - - def test_seek_and_tell(self): - #Test seek/tell using the StatefulIncrementalDecoder. - # Make test faster by doing smaller seeks - CHUNK_SIZE = 128 - - def test_seek_and_tell_with_data(data, min_pos=0): - """Tell/seek to various points within a data stream and ensure - that the decoded data returned by read() is consistent.""" - f = self.open(os_helper.TESTFN, 'wb') - f.write(data) - f.close() - f = self.open(os_helper.TESTFN, encoding='test_decoder') - f._CHUNK_SIZE = CHUNK_SIZE - decoded = f.read() - f.close() - - for i in range(min_pos, len(decoded) + 1): # seek positions - for j in [1, 5, len(decoded) - i]: # read lengths - f = self.open(os_helper.TESTFN, encoding='test_decoder') - self.assertEqual(f.read(i), decoded[:i]) - cookie = f.tell() - self.assertEqual(f.read(j), decoded[i:i + j]) - f.seek(cookie) - self.assertEqual(f.read(), decoded[i:]) - f.close() - - # Enable the test decoder. - StatefulIncrementalDecoder.codecEnabled = 1 - - # Run the tests. - try: - # Try each test case. - for input, _, _ in StatefulIncrementalDecoderTest.test_cases: - test_seek_and_tell_with_data(input) - - # Position each test case so that it crosses a chunk boundary. - for input, _, _ in StatefulIncrementalDecoderTest.test_cases: - offset = CHUNK_SIZE - len(input)//2 - prefix = b'.'*offset - # Don't bother seeking into the prefix (takes too long). - min_pos = offset*2 - test_seek_and_tell_with_data(prefix + input, min_pos) - - # Ensure our test decoder won't interfere with subsequent tests. - finally: - StatefulIncrementalDecoder.codecEnabled = 0 - - def test_multibyte_seek_and_tell(self): - f = self.open(os_helper.TESTFN, "w", encoding="euc_jp") - f.write("AB\n\u3046\u3048\n") - f.close() - - f = self.open(os_helper.TESTFN, "r", encoding="euc_jp") - self.assertEqual(f.readline(), "AB\n") - p0 = f.tell() - self.assertEqual(f.readline(), "\u3046\u3048\n") - p1 = f.tell() - f.seek(p0) - self.assertEqual(f.readline(), "\u3046\u3048\n") - self.assertEqual(f.tell(), p1) - f.close() - - def test_seek_with_encoder_state(self): - f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") - f.write("\u00e6\u0300") - p0 = f.tell() - f.write("\u00e6") - f.seek(p0) - f.write("\u0300") - f.close() - - f = self.open(os_helper.TESTFN, "r", encoding="euc_jis_2004") - self.assertEqual(f.readline(), "\u00e6\u0300\u0300") - f.close() - - def test_encoded_writes(self): - data = "1234567890" - tests = ("utf-16", - "utf-16-le", - "utf-16-be", - "utf-32", - "utf-32-le", - "utf-32-be") - for encoding in tests: - buf = self.BytesIO() - f = self.TextIOWrapper(buf, encoding=encoding) - # Check if the BOM is written only once (see issue1753). - f.write(data) - f.write(data) - f.seek(0) - self.assertEqual(f.read(), data * 2) - f.seek(0) - self.assertEqual(f.read(), data * 2) - self.assertEqual(buf.getvalue(), (data * 2).encode(encoding)) - - def test_unreadable(self): - class UnReadable(self.BytesIO): - def readable(self): - return False - txt = self.TextIOWrapper(UnReadable(), encoding="utf-8") - self.assertRaises(OSError, txt.read) - - def test_read_one_by_one(self): - txt = self.TextIOWrapper(self.BytesIO(b"AA\r\nBB"), encoding="utf-8") - reads = "" - while True: - c = txt.read(1) - if not c: - break - reads += c - self.assertEqual(reads, "AA\nBB") - - def test_readlines(self): - txt = self.TextIOWrapper(self.BytesIO(b"AA\nBB\nCC"), encoding="utf-8") - self.assertEqual(txt.readlines(), ["AA\n", "BB\n", "CC"]) - txt.seek(0) - self.assertEqual(txt.readlines(None), ["AA\n", "BB\n", "CC"]) - txt.seek(0) - self.assertEqual(txt.readlines(5), ["AA\n", "BB\n"]) - - # read in amounts equal to TextIOWrapper._CHUNK_SIZE which is 128. - def test_read_by_chunk(self): - # make sure "\r\n" straddles 128 char boundary. - txt = self.TextIOWrapper(self.BytesIO(b"A" * 127 + b"\r\nB"), encoding="utf-8") - reads = "" - while True: - c = txt.read(128) - if not c: - break - reads += c - self.assertEqual(reads, "A"*127+"\nB") - - def test_writelines(self): - l = ['ab', 'cd', 'ef'] - buf = self.BytesIO() - txt = self.TextIOWrapper(buf, encoding="utf-8") - txt.writelines(l) - txt.flush() - self.assertEqual(buf.getvalue(), b'abcdef') - - def test_writelines_userlist(self): - l = UserList(['ab', 'cd', 'ef']) - buf = self.BytesIO() - txt = self.TextIOWrapper(buf, encoding="utf-8") - txt.writelines(l) - txt.flush() - self.assertEqual(buf.getvalue(), b'abcdef') - - def test_writelines_error(self): - txt = self.TextIOWrapper(self.BytesIO(), encoding="utf-8") - self.assertRaises(TypeError, txt.writelines, [1, 2, 3]) - self.assertRaises(TypeError, txt.writelines, None) - self.assertRaises(TypeError, txt.writelines, b'abc') - - def test_issue1395_1(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - - # read one char at a time - reads = "" - while True: - c = txt.read(1) - if not c: - break - reads += c - self.assertEqual(reads, self.normalized) - - def test_issue1395_2(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt._CHUNK_SIZE = 4 - - reads = "" - while True: - c = txt.read(4) - if not c: - break - reads += c - self.assertEqual(reads, self.normalized) - - def test_issue1395_3(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt._CHUNK_SIZE = 4 - - reads = txt.read(4) - reads += txt.read(4) - reads += txt.readline() - reads += txt.readline() - reads += txt.readline() - self.assertEqual(reads, self.normalized) - - def test_issue1395_4(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt._CHUNK_SIZE = 4 - - reads = txt.read(4) - reads += txt.read() - self.assertEqual(reads, self.normalized) - - def test_issue1395_5(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt._CHUNK_SIZE = 4 - - reads = txt.read(4) - pos = txt.tell() - txt.seek(0) - txt.seek(pos) - self.assertEqual(txt.read(4), "BBB\n") - - def test_issue2282(self): - buffer = self.BytesIO(self.testdata) - txt = self.TextIOWrapper(buffer, encoding="ascii") - - self.assertEqual(buffer.seekable(), txt.seekable()) - - def test_append_bom(self): - # The BOM is not written again when appending to a non-empty file - filename = os_helper.TESTFN - for charset in ('utf-8-sig', 'utf-16', 'utf-32'): - with self.open(filename, 'w', encoding=charset) as f: - f.write('aaa') - pos = f.tell() - with self.open(filename, 'rb') as f: - self.assertEqual(f.read(), 'aaa'.encode(charset)) - - with self.open(filename, 'a', encoding=charset) as f: - f.write('xxx') - with self.open(filename, 'rb') as f: - self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) - - def test_seek_bom(self): - # Same test, but when seeking manually - filename = os_helper.TESTFN - for charset in ('utf-8-sig', 'utf-16', 'utf-32'): - with self.open(filename, 'w', encoding=charset) as f: - f.write('aaa') - pos = f.tell() - with self.open(filename, 'r+', encoding=charset) as f: - f.seek(pos) - f.write('zzz') - f.seek(0) - f.write('bbb') - with self.open(filename, 'rb') as f: - self.assertEqual(f.read(), 'bbbzzz'.encode(charset)) - - def test_seek_append_bom(self): - # Same test, but first seek to the start and then to the end - filename = os_helper.TESTFN - for charset in ('utf-8-sig', 'utf-16', 'utf-32'): - with self.open(filename, 'w', encoding=charset) as f: - f.write('aaa') - with self.open(filename, 'a', encoding=charset) as f: - f.seek(0) - f.seek(0, self.SEEK_END) - f.write('xxx') - with self.open(filename, 'rb') as f: - self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) - - def test_errors_property(self): - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - self.assertEqual(f.errors, "strict") - with self.open(os_helper.TESTFN, "w", encoding="utf-8", errors="replace") as f: - self.assertEqual(f.errors, "replace") - - @support.no_tracing - @threading_helper.requires_working_threading() - def test_threads_write(self): - # Issue6750: concurrent writes could duplicate data - event = threading.Event() - with self.open(os_helper.TESTFN, "w", encoding="utf-8", buffering=1) as f: - def run(n): - text = "Thread%03d\n" % n - event.wait() - f.write(text) - threads = [threading.Thread(target=run, args=(x,)) - for x in range(20)] - with threading_helper.start_threads(threads, event.set): - time.sleep(0.02) - with self.open(os_helper.TESTFN, encoding="utf-8") as f: - content = f.read() - for n in range(20): - self.assertEqual(content.count("Thread%03d\n" % n), 1) - - def test_flush_error_on_close(self): - # Test that text file is closed despite failed flush - # and that flush() is called before file closed. - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - closed = [] - def bad_flush(): - closed[:] = [txt.closed, txt.buffer.closed] - raise OSError() - txt.flush = bad_flush - self.assertRaises(OSError, txt.close) # exception not swallowed - self.assertTrue(txt.closed) - self.assertTrue(txt.buffer.closed) - self.assertTrue(closed) # flush() called - self.assertFalse(closed[0]) # flush() called before file closed - self.assertFalse(closed[1]) - txt.flush = lambda: None # break reference loop - - def test_close_error_on_close(self): - buffer = self.BytesIO(self.testdata) - def bad_flush(): - raise OSError('flush') - def bad_close(): - raise OSError('close') - buffer.close = bad_close - txt = self.TextIOWrapper(buffer, encoding="ascii") - txt.flush = bad_flush - with self.assertRaises(OSError) as err: # exception not swallowed - txt.close() - self.assertEqual(err.exception.args, ('close',)) - self.assertIsInstance(err.exception.__context__, OSError) - self.assertEqual(err.exception.__context__.args, ('flush',)) - self.assertFalse(txt.closed) - - # Silence destructor error - buffer.close = lambda: None - txt.flush = lambda: None - - def test_nonnormalized_close_error_on_close(self): - # Issue #21677 - buffer = self.BytesIO(self.testdata) - def bad_flush(): - raise non_existing_flush - def bad_close(): - raise non_existing_close - buffer.close = bad_close - txt = self.TextIOWrapper(buffer, encoding="ascii") - txt.flush = bad_flush - with self.assertRaises(NameError) as err: # exception not swallowed - txt.close() - self.assertIn('non_existing_close', str(err.exception)) - self.assertIsInstance(err.exception.__context__, NameError) - self.assertIn('non_existing_flush', str(err.exception.__context__)) - self.assertFalse(txt.closed) - - # Silence destructor error - buffer.close = lambda: None - txt.flush = lambda: None - - def test_multi_close(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt.close() - txt.close() - txt.close() - self.assertRaises(ValueError, txt.flush) - - def test_unseekable(self): - txt = self.TextIOWrapper(self.MockUnseekableIO(self.testdata), encoding="utf-8") - self.assertRaises(self.UnsupportedOperation, txt.tell) - self.assertRaises(self.UnsupportedOperation, txt.seek, 0) - - def test_readonly_attributes(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - buf = self.BytesIO(self.testdata) - with self.assertRaises(AttributeError): - txt.buffer = buf - - def test_rawio(self): - # Issue #12591: TextIOWrapper must work with raw I/O objects, so - # that subprocess.Popen() can have the required unbuffered - # semantics with universal_newlines=True. - raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n']) - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - # Reads - self.assertEqual(txt.read(4), 'abcd') - self.assertEqual(txt.readline(), 'efghi\n') - self.assertEqual(list(txt), ['jkl\n', 'opq\n']) - - def test_rawio_write_through(self): - # Issue #12591: with write_through=True, writes don't need a flush - raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n']) - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n', - write_through=True) - txt.write('1') - txt.write('23\n4') - txt.write('5') - self.assertEqual(b''.join(raw._write_stack), b'123\n45') - - def test_bufio_write_through(self): - # Issue #21396: write_through=True doesn't force a flush() - # on the underlying binary buffered object. - flush_called, write_called = [], [] - class BufferedWriter(self.BufferedWriter): - def flush(self, *args, **kwargs): - flush_called.append(True) - return super().flush(*args, **kwargs) - def write(self, *args, **kwargs): - write_called.append(True) - return super().write(*args, **kwargs) - - rawio = self.BytesIO() - data = b"a" - bufio = BufferedWriter(rawio, len(data)*2) - textio = self.TextIOWrapper(bufio, encoding='ascii', - write_through=True) - # write to the buffered io but don't overflow the buffer - text = data.decode('ascii') - textio.write(text) - - # buffer.flush is not called with write_through=True - self.assertFalse(flush_called) - # buffer.write *is* called with write_through=True - self.assertTrue(write_called) - self.assertEqual(rawio.getvalue(), b"") # no flush - - write_called = [] # reset - textio.write(text * 10) # total content is larger than bufio buffer - self.assertTrue(write_called) - self.assertEqual(rawio.getvalue(), data * 11) # all flushed - - def test_reconfigure_write_through(self): - raw = self.MockRawIO([]) - t = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - t.write('1') - t.reconfigure(write_through=True) # implied flush - self.assertEqual(t.write_through, True) - self.assertEqual(b''.join(raw._write_stack), b'1') - t.write('23') - self.assertEqual(b''.join(raw._write_stack), b'123') - t.reconfigure(write_through=False) - self.assertEqual(t.write_through, False) - t.write('45') - t.flush() - self.assertEqual(b''.join(raw._write_stack), b'12345') - # Keeping default value - t.reconfigure() - t.reconfigure(write_through=None) - self.assertEqual(t.write_through, False) - t.reconfigure(write_through=True) - t.reconfigure() - t.reconfigure(write_through=None) - self.assertEqual(t.write_through, True) - - def test_read_nonbytes(self): - # Issue #17106 - # Crash when underlying read() returns non-bytes - t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") - self.assertRaises(TypeError, t.read, 1) - t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") - self.assertRaises(TypeError, t.readline) - t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") - self.assertRaises(TypeError, t.read) - - def test_illegal_encoder(self): - # Issue 31271: Calling write() while the return value of encoder's - # encode() is invalid shouldn't cause an assertion failure. - rot13 = codecs.lookup("rot13") - with support.swap_attr(rot13, '_is_text_encoding', True): - t = self.TextIOWrapper(self.BytesIO(b'foo'), encoding="rot13") - self.assertRaises(TypeError, t.write, 'bar') - - def test_illegal_decoder(self): - # Issue #17106 - # Bypass the early encoding check added in issue 20404 - def _make_illegal_wrapper(): - quopri = codecs.lookup("quopri") - quopri._is_text_encoding = True - try: - t = self.TextIOWrapper(self.BytesIO(b'aaaaaa'), - newline='\n', encoding="quopri") - finally: - quopri._is_text_encoding = False - return t - # Crash when decoder returns non-string - t = _make_illegal_wrapper() - self.assertRaises(TypeError, t.read, 1) - t = _make_illegal_wrapper() - self.assertRaises(TypeError, t.readline) - t = _make_illegal_wrapper() - self.assertRaises(TypeError, t.read) - - # Issue 31243: calling read() while the return value of decoder's - # getstate() is invalid should neither crash the interpreter nor - # raise a SystemError. - def _make_very_illegal_wrapper(getstate_ret_val): - class BadDecoder: - def getstate(self): - return getstate_ret_val - def _get_bad_decoder(dummy): - return BadDecoder() - quopri = codecs.lookup("quopri") - with support.swap_attr(quopri, 'incrementaldecoder', - _get_bad_decoder): - return _make_illegal_wrapper() - t = _make_very_illegal_wrapper(42) - self.assertRaises(TypeError, t.read, 42) - t = _make_very_illegal_wrapper(()) - self.assertRaises(TypeError, t.read, 42) - t = _make_very_illegal_wrapper((1, 2)) - self.assertRaises(TypeError, t.read, 42) - - def _check_create_at_shutdown(self, **kwargs): - # Issue #20037: creating a TextIOWrapper at shutdown - # shouldn't crash the interpreter. - iomod = self.io.__name__ - code = """if 1: - import codecs - import {iomod} as io - - # Avoid looking up codecs at shutdown - codecs.lookup('utf-8') - - class C: - def __del__(self): - io.TextIOWrapper(io.BytesIO(), **{kwargs}) - print("ok") - c = C() - """.format(iomod=iomod, kwargs=kwargs) - return assert_python_ok("-c", code) - - def test_create_at_shutdown_without_encoding(self): - rc, out, err = self._check_create_at_shutdown() - if err: - # Can error out with a RuntimeError if the module state - # isn't found. - self.assertIn(self.shutdown_error, err.decode()) - else: - self.assertEqual("ok", out.decode().strip()) - - def test_create_at_shutdown_with_encoding(self): - rc, out, err = self._check_create_at_shutdown(encoding='utf-8', - errors='strict') - self.assertFalse(err) - self.assertEqual("ok", out.decode().strip()) - - def test_read_byteslike(self): - r = MemviewBytesIO(b'Just some random string\n') - t = self.TextIOWrapper(r, 'utf-8') - - # TextIOwrapper will not read the full string, because - # we truncate it to a multiple of the native int size - # so that we can construct a more complex memoryview. - bytes_val = _to_memoryview(r.getvalue()).tobytes() - - self.assertEqual(t.read(200), bytes_val.decode('utf-8')) - - def test_issue22849(self): - class F(object): - def readable(self): return True - def writable(self): return True - def seekable(self): return True - - for i in range(10): - try: - self.TextIOWrapper(F(), encoding='utf-8') - except Exception: - pass - - F.tell = lambda x: 0 - t = self.TextIOWrapper(F(), encoding='utf-8') - - def test_reconfigure_locale(self): - wrapper = self.TextIOWrapper(self.BytesIO(b"test")) - wrapper.reconfigure(encoding="locale") - - def test_reconfigure_encoding_read(self): - # latin1 -> utf8 - # (latin1 can decode utf-8 encoded string) - data = 'abc\xe9\n'.encode('latin1') + 'd\xe9f\n'.encode('utf8') - raw = self.BytesIO(data) - txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n') - self.assertEqual(txt.readline(), 'abc\xe9\n') - with self.assertRaises(self.UnsupportedOperation): - txt.reconfigure(encoding='utf-8') - with self.assertRaises(self.UnsupportedOperation): - txt.reconfigure(newline=None) - - def test_reconfigure_write_fromascii(self): - # ascii has a specific encodefunc in the C implementation, - # but utf-8-sig has not. Make sure that we get rid of the - # cached encodefunc when we switch encoders. - raw = self.BytesIO() - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - txt.write('foo\n') - txt.reconfigure(encoding='utf-8-sig') - txt.write('\xe9\n') - txt.flush() - self.assertEqual(raw.getvalue(), b'foo\n\xc3\xa9\n') - - def test_reconfigure_write(self): - # latin -> utf8 - raw = self.BytesIO() - txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n') - txt.write('abc\xe9\n') - txt.reconfigure(encoding='utf-8') - self.assertEqual(raw.getvalue(), b'abc\xe9\n') - txt.write('d\xe9f\n') - txt.flush() - self.assertEqual(raw.getvalue(), b'abc\xe9\nd\xc3\xa9f\n') - - # ascii -> utf-8-sig: ensure that no BOM is written in the middle of - # the file - raw = self.BytesIO() - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - txt.write('abc\n') - txt.reconfigure(encoding='utf-8-sig') - txt.write('d\xe9f\n') - txt.flush() - self.assertEqual(raw.getvalue(), b'abc\nd\xc3\xa9f\n') - - def test_reconfigure_write_non_seekable(self): - raw = self.BytesIO() - raw.seekable = lambda: False - raw.seek = None - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - txt.write('abc\n') - txt.reconfigure(encoding='utf-8-sig') - txt.write('d\xe9f\n') - txt.flush() - - # If the raw stream is not seekable, there'll be a BOM - self.assertEqual(raw.getvalue(), b'abc\n\xef\xbb\xbfd\xc3\xa9f\n') - - def test_reconfigure_defaults(self): - txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\n') - txt.reconfigure(encoding=None) - self.assertEqual(txt.encoding, 'ascii') - self.assertEqual(txt.errors, 'replace') - txt.write('LF\n') - - txt.reconfigure(newline='\r\n') - self.assertEqual(txt.encoding, 'ascii') - self.assertEqual(txt.errors, 'replace') - - txt.reconfigure(errors='ignore') - self.assertEqual(txt.encoding, 'ascii') - self.assertEqual(txt.errors, 'ignore') - txt.write('CRLF\n') - - txt.reconfigure(encoding='utf-8', newline=None) - self.assertEqual(txt.errors, 'strict') - txt.seek(0) - self.assertEqual(txt.read(), 'LF\nCRLF\n') - - self.assertEqual(txt.detach().getvalue(), b'LF\nCRLF\r\n') - - def test_reconfigure_errors(self): - txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\r') - with self.assertRaises(TypeError): # there was a crash - txt.reconfigure(encoding=42) - if self.is_C: - with self.assertRaises(UnicodeEncodeError): - txt.reconfigure(encoding='\udcfe') - with self.assertRaises(LookupError): - txt.reconfigure(encoding='locale\0') - # TODO: txt.reconfigure(encoding='utf-8\0') - # TODO: txt.reconfigure(encoding='nonexisting') - with self.assertRaises(TypeError): - txt.reconfigure(errors=42) - if self.is_C: - with self.assertRaises(UnicodeEncodeError): - txt.reconfigure(errors='\udcfe') - # TODO: txt.reconfigure(errors='ignore\0') - # TODO: txt.reconfigure(errors='nonexisting') - with self.assertRaises(TypeError): - txt.reconfigure(newline=42) - with self.assertRaises(ValueError): - txt.reconfigure(newline='\udcfe') - with self.assertRaises(ValueError): - txt.reconfigure(newline='xyz') - if not self.is_C: - # TODO: Should fail in C too. - with self.assertRaises(ValueError): - txt.reconfigure(newline='\n\0') - if self.is_C: - # TODO: Use __bool__(), not __index__(). - with self.assertRaises(ZeroDivisionError): - txt.reconfigure(line_buffering=BadIndex()) - with self.assertRaises(OverflowError): - txt.reconfigure(line_buffering=2**1000) - with self.assertRaises(ZeroDivisionError): - txt.reconfigure(write_through=BadIndex()) - with self.assertRaises(OverflowError): - txt.reconfigure(write_through=2**1000) - with self.assertRaises(ZeroDivisionError): # there was a crash - txt.reconfigure(line_buffering=BadIndex(), - write_through=BadIndex()) - self.assertEqual(txt.encoding, 'ascii') - self.assertEqual(txt.errors, 'replace') - self.assertIs(txt.line_buffering, False) - self.assertIs(txt.write_through, False) - - txt.reconfigure(encoding='latin1', errors='ignore', newline='\r\n', - line_buffering=True, write_through=True) - self.assertEqual(txt.encoding, 'latin1') - self.assertEqual(txt.errors, 'ignore') - self.assertIs(txt.line_buffering, True) - self.assertIs(txt.write_through, True) - - def test_reconfigure_newline(self): - raw = self.BytesIO(b'CR\rEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\n') - txt.reconfigure(newline=None) - self.assertEqual(txt.readline(), 'CR\n') - raw = self.BytesIO(b'CR\rEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\n') - txt.reconfigure(newline='') - self.assertEqual(txt.readline(), 'CR\r') - raw = self.BytesIO(b'CR\rLF\nEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\r') - txt.reconfigure(newline='\n') - self.assertEqual(txt.readline(), 'CR\rLF\n') - raw = self.BytesIO(b'LF\nCR\rEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\n') - txt.reconfigure(newline='\r') - self.assertEqual(txt.readline(), 'LF\nCR\r') - raw = self.BytesIO(b'CR\rCRLF\r\nEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\r') - txt.reconfigure(newline='\r\n') - self.assertEqual(txt.readline(), 'CR\rCRLF\r\n') - - txt = self.TextIOWrapper(self.BytesIO(), 'ascii', newline='\r') - txt.reconfigure(newline=None) - txt.write('linesep\n') - txt.reconfigure(newline='') - txt.write('LF\n') - txt.reconfigure(newline='\n') - txt.write('LF\n') - txt.reconfigure(newline='\r') - txt.write('CR\n') - txt.reconfigure(newline='\r\n') - txt.write('CRLF\n') - expected = 'linesep' + os.linesep + 'LF\nLF\nCR\rCRLF\r\n' - self.assertEqual(txt.detach().getvalue().decode('ascii'), expected) - - def test_issue25862(self): - # Assertion failures occurred in tell() after read() and write(). - t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii') - t.read(1) - t.read() - t.tell() - t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii') - t.read(1) - t.write('x') - t.tell() - - def test_issue35928(self): - p = self.BufferedRWPair(self.BytesIO(b'foo\nbar\n'), self.BytesIO()) - f = self.TextIOWrapper(p) - res = f.readline() - self.assertEqual(res, 'foo\n') - f.write(res) - self.assertEqual(res + f.readline(), 'foo\nbar\n') - - def test_pickling_subclass(self): - global MyTextIO - class MyTextIO(self.TextIOWrapper): - def __init__(self, raw, tag): - super().__init__(raw) - self.tag = tag - def __getstate__(self): - return self.tag, self.buffer.getvalue() - def __setstate__(slf, state): - tag, value = state - slf.__init__(self.BytesIO(value), tag) - - raw = self.BytesIO(b'data') - txt = MyTextIO(raw, 'ham') - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(protocol=proto): - pickled = pickle.dumps(txt, proto) - newtxt = pickle.loads(pickled) - self.assertEqual(newtxt.buffer.getvalue(), b'data') - self.assertEqual(newtxt.tag, 'ham') - del MyTextIO - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_read_non_blocking(self): - import os - r, w = os.pipe() - try: - os.set_blocking(r, False) - with self.io.open(r, 'rt') as textfile: - r = None - # Nothing has been written so a non-blocking read raises a BlockingIOError exception. - with self.assertRaises(BlockingIOError): - textfile.read() - finally: - if r is not None: - os.close(r) - os.close(w) - - -class MemviewBytesIO(io.BytesIO): - '''A BytesIO object whose read method returns memoryviews - rather than bytes''' - - def read1(self, len_): - return _to_memoryview(super().read1(len_)) - - def read(self, len_): - return _to_memoryview(super().read(len_)) - -def _to_memoryview(buf): - '''Convert bytes-object *buf* to a non-trivial memoryview''' - - arr = array.array('i') - idx = len(buf) - len(buf) % arr.itemsize - arr.frombytes(buf[:idx]) - return memoryview(arr) - - -class CTextIOWrapperTest(TextIOWrapperTest): - io = io - shutdown_error = "LookupError: unknown encoding: ascii" - - def test_initialization(self): - r = self.BytesIO(b"\xc3\xa9\n\n") - b = self.BufferedReader(r, 1000) - t = self.TextIOWrapper(b, encoding="utf-8") - self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') - self.assertRaises(ValueError, t.read) - - t = self.TextIOWrapper.__new__(self.TextIOWrapper) - self.assertRaises(Exception, repr, t) - - def test_garbage_collection(self): - # C TextIOWrapper objects are collected, and collecting them flushes - # all data to disk. - # The Python version has __del__, so it ends in gc.garbage instead. - with warnings_helper.check_warnings(('', ResourceWarning)): - rawio = self.FileIO(os_helper.TESTFN, "wb") - b = self.BufferedWriter(rawio) - t = self.TextIOWrapper(b, encoding="ascii") - t.write("456def") - t.x = t - wr = weakref.ref(t) - del t - support.gc_collect() - self.assertIsNone(wr(), wr) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"456def") - - def test_rwpair_cleared_before_textio(self): - # Issue 13070: TextIOWrapper's finalization would crash when called - # after the reference to the underlying BufferedRWPair's writer got - # cleared by the GC. - for i in range(1000): - b1 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()) - t1 = self.TextIOWrapper(b1, encoding="ascii") - b2 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()) - t2 = self.TextIOWrapper(b2, encoding="ascii") - # circular references - t1.buddy = t2 - t2.buddy = t1 - support.gc_collect() - - def test_del__CHUNK_SIZE_SystemError(self): - t = self.TextIOWrapper(self.BytesIO(), encoding='ascii') - with self.assertRaises(AttributeError): - del t._CHUNK_SIZE - - def test_internal_buffer_size(self): - # bpo-43260: TextIOWrapper's internal buffer should not store - # data larger than chunk size. - chunk_size = 8192 # default chunk size, updated later - - class MockIO(self.MockRawIO): - def write(self, data): - if len(data) > chunk_size: - raise RuntimeError - return super().write(data) - - buf = MockIO() - t = self.TextIOWrapper(buf, encoding="ascii") - chunk_size = t._CHUNK_SIZE - t.write("abc") - t.write("def") - # default chunk size is 8192 bytes so t don't write data to buf. - self.assertEqual([], buf._write_stack) - - with self.assertRaises(RuntimeError): - t.write("x"*(chunk_size+1)) - - self.assertEqual([b"abcdef"], buf._write_stack) - t.write("ghi") - t.write("x"*chunk_size) - self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) - - def test_issue119506(self): - chunk_size = 8192 - - class MockIO(self.MockRawIO): - written = False - def write(self, data): - if not self.written: - self.written = True - t.write("middle") - return super().write(data) - - buf = MockIO() - t = self.TextIOWrapper(buf) - t.write("abc") - t.write("def") - # writing data which size >= chunk_size cause flushing buffer before write. - t.write("g" * chunk_size) - t.flush() - - self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], - buf._write_stack) - - -class PyTextIOWrapperTest(TextIOWrapperTest): - io = pyio - shutdown_error = "LookupError: unknown encoding: ascii" - - -class IncrementalNewlineDecoderTest(unittest.TestCase): - - def check_newline_decoding_utf8(self, decoder): - # UTF-8 specific tests for a newline decoder - def _check_decode(b, s, **kwargs): - # We exercise getstate() / setstate() as well as decode() - state = decoder.getstate() - self.assertEqual(decoder.decode(b, **kwargs), s) - decoder.setstate(state) - self.assertEqual(decoder.decode(b, **kwargs), s) - - _check_decode(b'\xe8\xa2\x88', "\u8888") - - _check_decode(b'\xe8', "") - _check_decode(b'\xa2', "") - _check_decode(b'\x88', "\u8888") - - _check_decode(b'\xe8', "") - _check_decode(b'\xa2', "") - _check_decode(b'\x88', "\u8888") - - _check_decode(b'\xe8', "") - self.assertRaises(UnicodeDecodeError, decoder.decode, b'', final=True) - - decoder.reset() - _check_decode(b'\n', "\n") - _check_decode(b'\r', "") - _check_decode(b'', "\n", final=True) - _check_decode(b'\r', "\n", final=True) - - _check_decode(b'\r', "") - _check_decode(b'a', "\na") - - _check_decode(b'\r\r\n', "\n\n") - _check_decode(b'\r', "") - _check_decode(b'\r', "\n") - _check_decode(b'\na', "\na") - - _check_decode(b'\xe8\xa2\x88\r\n', "\u8888\n") - _check_decode(b'\xe8\xa2\x88', "\u8888") - _check_decode(b'\n', "\n") - _check_decode(b'\xe8\xa2\x88\r', "\u8888") - _check_decode(b'\n', "\n") - - def check_newline_decoding(self, decoder, encoding): - result = [] - if encoding is not None: - encoder = codecs.getincrementalencoder(encoding)() - def _decode_bytewise(s): - # Decode one byte at a time - for b in encoder.encode(s): - result.append(decoder.decode(bytes([b]))) - else: - encoder = None - def _decode_bytewise(s): - # Decode one char at a time - for c in s: - result.append(decoder.decode(c)) - self.assertEqual(decoder.newlines, None) - _decode_bytewise("abc\n\r") - self.assertEqual(decoder.newlines, '\n') - _decode_bytewise("\nabc") - self.assertEqual(decoder.newlines, ('\n', '\r\n')) - _decode_bytewise("abc\r") - self.assertEqual(decoder.newlines, ('\n', '\r\n')) - _decode_bytewise("abc") - self.assertEqual(decoder.newlines, ('\r', '\n', '\r\n')) - _decode_bytewise("abc\r") - self.assertEqual("".join(result), "abc\n\nabcabc\nabcabc") - decoder.reset() - input = "abc" - if encoder is not None: - encoder.reset() - input = encoder.encode(input) - self.assertEqual(decoder.decode(input), "abc") - self.assertEqual(decoder.newlines, None) - - def test_newline_decoder(self): - encodings = ( - # None meaning the IncrementalNewlineDecoder takes unicode input - # rather than bytes input - None, 'utf-8', 'latin-1', - 'utf-16', 'utf-16-le', 'utf-16-be', - 'utf-32', 'utf-32-le', 'utf-32-be', - ) - for enc in encodings: - decoder = enc and codecs.getincrementaldecoder(enc)() - decoder = self.IncrementalNewlineDecoder(decoder, translate=True) - self.check_newline_decoding(decoder, enc) - decoder = codecs.getincrementaldecoder("utf-8")() - decoder = self.IncrementalNewlineDecoder(decoder, translate=True) - self.check_newline_decoding_utf8(decoder) - self.assertRaises(TypeError, decoder.setstate, 42) - - def test_newline_bytes(self): - # Issue 5433: Excessive optimization in IncrementalNewlineDecoder - def _check(dec): - self.assertEqual(dec.newlines, None) - self.assertEqual(dec.decode("\u0D00"), "\u0D00") - self.assertEqual(dec.newlines, None) - self.assertEqual(dec.decode("\u0A00"), "\u0A00") - self.assertEqual(dec.newlines, None) - dec = self.IncrementalNewlineDecoder(None, translate=False) - _check(dec) - dec = self.IncrementalNewlineDecoder(None, translate=True) - _check(dec) - - def test_translate(self): - # issue 35062 - for translate in (-2, -1, 1, 2): - decoder = codecs.getincrementaldecoder("utf-8")() - decoder = self.IncrementalNewlineDecoder(decoder, translate) - self.check_newline_decoding_utf8(decoder) - decoder = codecs.getincrementaldecoder("utf-8")() - decoder = self.IncrementalNewlineDecoder(decoder, translate=0) - self.assertEqual(decoder.decode(b"\r\r\n"), "\r\r\n") - -class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): - @support.cpython_only - def test_uninitialized(self): - uninitialized = self.IncrementalNewlineDecoder.__new__( - self.IncrementalNewlineDecoder) - self.assertRaises(ValueError, uninitialized.decode, b'bar') - self.assertRaises(ValueError, uninitialized.getstate) - self.assertRaises(ValueError, uninitialized.setstate, (b'foo', 0)) - self.assertRaises(ValueError, uninitialized.reset) - - -class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): - pass - - -# XXX Tests for open() - -class MiscIOTest(unittest.TestCase): - - # for test__all__, actual values are set in subclasses - name_of_module = None - extra_exported = () - not_exported = () - - def tearDown(self): - os_helper.unlink(os_helper.TESTFN) - - def test___all__(self): - support.check__all__(self, self.io, self.name_of_module, - extra=self.extra_exported, - not_exported=self.not_exported) - - def test_attributes(self): - f = self.open(os_helper.TESTFN, "wb", buffering=0) - self.assertEqual(f.mode, "wb") - f.close() - - f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") - self.assertEqual(f.mode, "w+") - self.assertEqual(f.buffer.mode, "rb+") # Does it really matter? - self.assertEqual(f.buffer.raw.mode, "rb+") - - g = self.open(f.fileno(), "wb", closefd=False) - self.assertEqual(g.mode, "wb") - self.assertEqual(g.raw.mode, "wb") - self.assertEqual(g.name, f.fileno()) - self.assertEqual(g.raw.name, f.fileno()) - f.close() - g.close() - - def test_removed_u_mode(self): - # bpo-37330: The "U" mode has been removed in Python 3.11 - for mode in ("U", "rU", "r+U"): - with self.assertRaises(ValueError) as cm: - self.open(os_helper.TESTFN, mode) - self.assertIn('invalid mode', str(cm.exception)) - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_open_pipe_with_append(self): - # bpo-27805: Ignore ESPIPE from lseek() in open(). - r, w = os.pipe() - self.addCleanup(os.close, r) - f = self.open(w, 'a', encoding="utf-8") - self.addCleanup(f.close) - # Check that the file is marked non-seekable. On Windows, however, lseek - # somehow succeeds on pipes. - if sys.platform != 'win32': - self.assertFalse(f.seekable()) - - def test_io_after_close(self): - for kwargs in [ - {"mode": "w"}, - {"mode": "wb"}, - {"mode": "w", "buffering": 1}, - {"mode": "w", "buffering": 2}, - {"mode": "wb", "buffering": 0}, - {"mode": "r"}, - {"mode": "rb"}, - {"mode": "r", "buffering": 1}, - {"mode": "r", "buffering": 2}, - {"mode": "rb", "buffering": 0}, - {"mode": "w+"}, - {"mode": "w+b"}, - {"mode": "w+", "buffering": 1}, - {"mode": "w+", "buffering": 2}, - {"mode": "w+b", "buffering": 0}, - ]: - if "b" not in kwargs["mode"]: - kwargs["encoding"] = "utf-8" - f = self.open(os_helper.TESTFN, **kwargs) - f.close() - self.assertRaises(ValueError, f.flush) - self.assertRaises(ValueError, f.fileno) - self.assertRaises(ValueError, f.isatty) - self.assertRaises(ValueError, f.__iter__) - if hasattr(f, "peek"): - self.assertRaises(ValueError, f.peek, 1) - self.assertRaises(ValueError, f.read) - if hasattr(f, "read1"): - self.assertRaises(ValueError, f.read1, 1024) - self.assertRaises(ValueError, f.read1) - if hasattr(f, "readall"): - self.assertRaises(ValueError, f.readall) - if hasattr(f, "readinto"): - self.assertRaises(ValueError, f.readinto, bytearray(1024)) - if hasattr(f, "readinto1"): - self.assertRaises(ValueError, f.readinto1, bytearray(1024)) - self.assertRaises(ValueError, f.readline) - self.assertRaises(ValueError, f.readlines) - self.assertRaises(ValueError, f.readlines, 1) - self.assertRaises(ValueError, f.seek, 0) - self.assertRaises(ValueError, f.tell) - self.assertRaises(ValueError, f.truncate) - self.assertRaises(ValueError, f.write, - b"" if "b" in kwargs['mode'] else "") - self.assertRaises(ValueError, f.writelines, []) - self.assertRaises(ValueError, next, f) - - def test_blockingioerror(self): - # Various BlockingIOError issues - class C(str): - pass - c = C("") - b = self.BlockingIOError(1, c) - c.b = b - b.c = c - wr = weakref.ref(c) - del c, b - support.gc_collect() - self.assertIsNone(wr(), wr) - - def test_abcs(self): - # Test the visible base classes are ABCs. - self.assertIsInstance(self.IOBase, abc.ABCMeta) - self.assertIsInstance(self.RawIOBase, abc.ABCMeta) - self.assertIsInstance(self.BufferedIOBase, abc.ABCMeta) - self.assertIsInstance(self.TextIOBase, abc.ABCMeta) - - def _check_abc_inheritance(self, abcmodule): - with self.open(os_helper.TESTFN, "wb", buffering=0) as f: - self.assertIsInstance(f, abcmodule.IOBase) - self.assertIsInstance(f, abcmodule.RawIOBase) - self.assertNotIsInstance(f, abcmodule.BufferedIOBase) - self.assertNotIsInstance(f, abcmodule.TextIOBase) - with self.open(os_helper.TESTFN, "wb") as f: - self.assertIsInstance(f, abcmodule.IOBase) - self.assertNotIsInstance(f, abcmodule.RawIOBase) - self.assertIsInstance(f, abcmodule.BufferedIOBase) - self.assertNotIsInstance(f, abcmodule.TextIOBase) - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - self.assertIsInstance(f, abcmodule.IOBase) - self.assertNotIsInstance(f, abcmodule.RawIOBase) - self.assertNotIsInstance(f, abcmodule.BufferedIOBase) - self.assertIsInstance(f, abcmodule.TextIOBase) - - def test_abc_inheritance(self): - # Test implementations inherit from their respective ABCs - self._check_abc_inheritance(self) - - def test_abc_inheritance_official(self): - # Test implementations inherit from the official ABCs of the - # baseline "io" module. - self._check_abc_inheritance(io) - - def _check_warn_on_dealloc(self, *args, **kwargs): - f = open(*args, **kwargs) - r = repr(f) - with self.assertWarns(ResourceWarning) as cm: - f = None - support.gc_collect() - self.assertIn(r, str(cm.warning.args[0])) - - def test_warn_on_dealloc(self): - self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0) - self._check_warn_on_dealloc(os_helper.TESTFN, "wb") - self._check_warn_on_dealloc(os_helper.TESTFN, "w", encoding="utf-8") - - def _check_warn_on_dealloc_fd(self, *args, **kwargs): - fds = [] - def cleanup_fds(): - for fd in fds: - try: - os.close(fd) - except OSError as e: - if e.errno != errno.EBADF: - raise - self.addCleanup(cleanup_fds) - r, w = os.pipe() - fds += r, w - self._check_warn_on_dealloc(r, *args, **kwargs) - # When using closefd=False, there's no warning - r, w = os.pipe() - fds += r, w - with warnings_helper.check_no_resource_warning(self): - open(r, *args, closefd=False, **kwargs) - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_warn_on_dealloc_fd(self): - self._check_warn_on_dealloc_fd("rb", buffering=0) - self._check_warn_on_dealloc_fd("rb") - self._check_warn_on_dealloc_fd("r", encoding="utf-8") - - - def test_pickling(self): - # Pickling file objects is forbidden - msg = "cannot pickle" - for kwargs in [ - {"mode": "w"}, - {"mode": "wb"}, - {"mode": "wb", "buffering": 0}, - {"mode": "r"}, - {"mode": "rb"}, - {"mode": "rb", "buffering": 0}, - {"mode": "w+"}, - {"mode": "w+b"}, - {"mode": "w+b", "buffering": 0}, - ]: - if "b" not in kwargs["mode"]: - kwargs["encoding"] = "utf-8" - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(protocol=protocol, kwargs=kwargs): - with self.open(os_helper.TESTFN, **kwargs) as f: - with self.assertRaisesRegex(TypeError, msg): - pickle.dumps(f, protocol) - - @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") - def test_nonblock_pipe_write_bigbuf(self): - self._test_nonblock_pipe_write(16*1024) - - @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") - def test_nonblock_pipe_write_smallbuf(self): - self._test_nonblock_pipe_write(1024) - - @unittest.skipUnless(hasattr(os, 'set_blocking'), - 'os.set_blocking() required for this test') - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def _test_nonblock_pipe_write(self, bufsize): - sent = [] - received = [] - r, w = os.pipe() - os.set_blocking(r, False) - os.set_blocking(w, False) - - # To exercise all code paths in the C implementation we need - # to play with buffer sizes. For instance, if we choose a - # buffer size less than or equal to _PIPE_BUF (4096 on Linux) - # then we will never get a partial write of the buffer. - rf = self.open(r, mode='rb', closefd=True, buffering=bufsize) - wf = self.open(w, mode='wb', closefd=True, buffering=bufsize) - - with rf, wf: - for N in 9999, 73, 7574: - try: - i = 0 - while True: - msg = bytes([i % 26 + 97]) * N - sent.append(msg) - wf.write(msg) - i += 1 - - except self.BlockingIOError as e: - self.assertEqual(e.args[0], errno.EAGAIN) - self.assertEqual(e.args[2], e.characters_written) - sent[-1] = sent[-1][:e.characters_written] - received.append(rf.read()) - msg = b'BLOCKED' - wf.write(msg) - sent.append(msg) - - while True: - try: - wf.flush() - break - except self.BlockingIOError as e: - self.assertEqual(e.args[0], errno.EAGAIN) - self.assertEqual(e.args[2], e.characters_written) - self.assertEqual(e.characters_written, 0) - received.append(rf.read()) - - received += iter(rf.read, None) - - sent, received = b''.join(sent), b''.join(received) - self.assertEqual(sent, received) - self.assertTrue(wf.closed) - self.assertTrue(rf.closed) - - def test_create_fail(self): - # 'x' mode fails if file is existing - with self.open(os_helper.TESTFN, 'w', encoding="utf-8"): - pass - self.assertRaises(FileExistsError, self.open, os_helper.TESTFN, 'x', encoding="utf-8") - - def test_create_writes(self): - # 'x' mode opens for writing - with self.open(os_helper.TESTFN, 'xb') as f: - f.write(b"spam") - with self.open(os_helper.TESTFN, 'rb') as f: - self.assertEqual(b"spam", f.read()) - - def test_open_allargs(self): - # there used to be a buffer overflow in the parser for rawmode - self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'rwax+', encoding="utf-8") - - def test_check_encoding_errors(self): - # bpo-37388: open() and TextIOWrapper must check encoding and errors - # arguments in dev mode - mod = self.io.__name__ - filename = __file__ - invalid = 'Boom, Shaka Laka, Boom!' - code = textwrap.dedent(f''' - import sys - from {mod} import open, TextIOWrapper - - try: - open({filename!r}, encoding={invalid!r}) - except LookupError: - pass - else: - sys.exit(21) - - try: - open({filename!r}, errors={invalid!r}) - except LookupError: - pass - else: - sys.exit(22) - - fp = open({filename!r}, "rb") - with fp: - try: - TextIOWrapper(fp, encoding={invalid!r}) - except LookupError: - pass - else: - sys.exit(23) - - try: - TextIOWrapper(fp, errors={invalid!r}) - except LookupError: - pass - else: - sys.exit(24) - - sys.exit(10) - ''') - proc = assert_python_failure('-X', 'dev', '-c', code) - self.assertEqual(proc.rc, 10, proc) - - def test_check_encoding_warning(self): - # PEP 597: Raise warning when encoding is not specified - # and sys.flags.warn_default_encoding is set. - mod = self.io.__name__ - filename = __file__ - code = textwrap.dedent(f'''\ - import sys - from {mod} import open, TextIOWrapper - import pathlib - - with open({filename!r}) as f: # line 5 - pass - - pathlib.Path({filename!r}).read_text() # line 8 - ''') - proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) - warnings = proc.err.splitlines() - self.assertEqual(len(warnings), 2) - self.assertTrue( - warnings[0].startswith(b"<string>:5: EncodingWarning: ")) - self.assertTrue( - warnings[1].startswith(b"<string>:8: EncodingWarning: ")) - - def test_text_encoding(self): - # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" - # based on sys.flags.utf8_mode - code = "import io; print(io.text_encoding(None))" - - proc = assert_python_ok('-X', 'utf8=0', '-c', code) - self.assertEqual(b"locale", proc.out.strip()) - - proc = assert_python_ok('-X', 'utf8=1', '-c', code) - self.assertEqual(b"utf-8", proc.out.strip()) - - -class CMiscIOTest(MiscIOTest): - io = io - name_of_module = "io", "_io" - extra_exported = "BlockingIOError", - - def test_readinto_buffer_overflow(self): - # Issue #18025 - class BadReader(self.io.BufferedIOBase): - def read(self, n=-1): - return b'x' * 10**6 - bufio = BadReader() - b = bytearray(2) - self.assertRaises(ValueError, bufio.readinto, b) - - def check_daemon_threads_shutdown_deadlock(self, stream_name): - # Issue #23309: deadlocks at shutdown should be avoided when a - # daemon thread and the main thread both write to a file. - code = """if 1: - import sys - import time - import threading - from test.support import SuppressCrashReport - - file = sys.{stream_name} - - def run(): - while True: - file.write('.') - file.flush() - - crash = SuppressCrashReport() - crash.__enter__() - # don't call __exit__(): the crash occurs at Python shutdown - - thread = threading.Thread(target=run) - thread.daemon = True - thread.start() - - time.sleep(0.5) - file.write('!') - file.flush() - """.format_map(locals()) - res, _ = run_python_until_end("-c", code) - err = res.err.decode() - if res.rc != 0: - # Failure: should be a fatal error - pattern = (r"Fatal Python error: _enter_buffered_busy: " - r"could not acquire lock " - r"for <(_io\.)?BufferedWriter name='<{stream_name}>'> " - r"at interpreter shutdown, possibly due to " - r"daemon threads".format_map(locals())) - self.assertRegex(err, pattern) - else: - self.assertFalse(err.strip('.!')) - - @threading_helper.requires_working_threading() - @support.requires_resource('walltime') - def test_daemon_threads_shutdown_stdout_deadlock(self): - self.check_daemon_threads_shutdown_deadlock('stdout') - - @threading_helper.requires_working_threading() - @support.requires_resource('walltime') - def test_daemon_threads_shutdown_stderr_deadlock(self): - self.check_daemon_threads_shutdown_deadlock('stderr') - - -class PyMiscIOTest(MiscIOTest): - io = pyio - name_of_module = "_pyio", "io" - extra_exported = "BlockingIOError", "open_code", - not_exported = "valid_seek_flags", - - -@unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.') -class SignalsTest(unittest.TestCase): - - def setUp(self): - self.oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt) - - def tearDown(self): - signal.signal(signal.SIGALRM, self.oldalrm) - - def alarm_interrupt(self, sig, frame): - 1/0 - - def check_interrupted_write(self, item, bytes, **fdopen_kwargs): - """Check that a partial write, when it gets interrupted, properly - invokes the signal handler, and bubbles up the exception raised - in the latter.""" - - # XXX This test has three flaws that appear when objects are - # XXX not reference counted. - - # - if wio.write() happens to trigger a garbage collection, - # the signal exception may be raised when some __del__ - # method is running; it will not reach the assertRaises() - # call. - - # - more subtle, if the wio object is not destroyed at once - # and survives this function, the next opened file is likely - # to have the same fileno (since the file descriptor was - # actively closed). When wio.__del__ is finally called, it - # will close the other's test file... To trigger this with - # CPython, try adding "global wio" in this function. - - # - This happens only for streams created by the _pyio module, - # because a wio.close() that fails still consider that the - # file needs to be closed again. You can try adding an - # "assert wio.closed" at the end of the function. - - # Fortunately, a little gc.collect() seems to be enough to - # work around all these issues. - support.gc_collect() # For PyPy or other GCs. - - read_results = [] - def _read(): - s = os.read(r, 1) - read_results.append(s) - - t = threading.Thread(target=_read) - t.daemon = True - r, w = os.pipe() - fdopen_kwargs["closefd"] = False - large_data = item * (support.PIPE_MAX_SIZE // len(item) + 1) - try: - wio = self.io.open(w, **fdopen_kwargs) - if hasattr(signal, 'pthread_sigmask'): - # create the thread with SIGALRM signal blocked - signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM]) - t.start() - signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGALRM]) - else: - t.start() - - # Fill the pipe enough that the write will be blocking. - # It will be interrupted by the timer armed above. Since the - # other thread has read one byte, the low-level write will - # return with a successful (partial) result rather than an EINTR. - # The buffered IO layer must check for pending signal - # handlers, which in this case will invoke alarm_interrupt(). - signal.alarm(1) - try: - self.assertRaises(ZeroDivisionError, wio.write, large_data) - finally: - signal.alarm(0) - t.join() - # We got one byte, get another one and check that it isn't a - # repeat of the first one. - read_results.append(os.read(r, 1)) - self.assertEqual(read_results, [bytes[0:1], bytes[1:2]]) - finally: - os.close(w) - os.close(r) - # This is deliberate. If we didn't close the file descriptor - # before closing wio, wio would try to flush its internal - # buffer, and block again. - try: - wio.close() - except OSError as e: - if e.errno != errno.EBADF: - raise - - @requires_alarm - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_interrupted_write_unbuffered(self): - self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0) - - @requires_alarm - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_interrupted_write_buffered(self): - self.check_interrupted_write(b"xy", b"xy", mode="wb") - - @requires_alarm - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_interrupted_write_text(self): - self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") - - @support.no_tracing - def check_reentrant_write(self, data, **fdopen_kwargs): - def on_alarm(*args): - # Will be called reentrantly from the same thread - wio.write(data) - 1/0 - signal.signal(signal.SIGALRM, on_alarm) - r, w = os.pipe() - wio = self.io.open(w, **fdopen_kwargs) - try: - signal.alarm(1) - # Either the reentrant call to wio.write() fails with RuntimeError, - # or the signal handler raises ZeroDivisionError. - with self.assertRaises((ZeroDivisionError, RuntimeError)) as cm: - while 1: - for i in range(100): - wio.write(data) - wio.flush() - # Make sure the buffer doesn't fill up and block further writes - os.read(r, len(data) * 100) - exc = cm.exception - if isinstance(exc, RuntimeError): - self.assertTrue(str(exc).startswith("reentrant call"), str(exc)) - finally: - signal.alarm(0) - wio.close() - os.close(r) - - @requires_alarm - def test_reentrant_write_buffered(self): - self.check_reentrant_write(b"xy", mode="wb") - - @requires_alarm - def test_reentrant_write_text(self): - self.check_reentrant_write("xy", mode="w", encoding="ascii") - - def check_interrupted_read_retry(self, decode, **fdopen_kwargs): - """Check that a buffered read, when it gets interrupted (either - returning a partial result or EINTR), properly invokes the signal - handler and retries if the latter returned successfully.""" - r, w = os.pipe() - fdopen_kwargs["closefd"] = False - def alarm_handler(sig, frame): - os.write(w, b"bar") - signal.signal(signal.SIGALRM, alarm_handler) - try: - rio = self.io.open(r, **fdopen_kwargs) - os.write(w, b"foo") - signal.alarm(1) - # Expected behaviour: - # - first raw read() returns partial b"foo" - # - second raw read() returns EINTR - # - third raw read() returns b"bar" - self.assertEqual(decode(rio.read(6)), "foobar") - finally: - signal.alarm(0) - rio.close() - os.close(w) - os.close(r) - - @requires_alarm - @support.requires_resource('walltime') - def test_interrupted_read_retry_buffered(self): - self.check_interrupted_read_retry(lambda x: x.decode('latin1'), - mode="rb") - - @requires_alarm - @support.requires_resource('walltime') - def test_interrupted_read_retry_text(self): - self.check_interrupted_read_retry(lambda x: x, - mode="r", encoding="latin1") - - def check_interrupted_write_retry(self, item, **fdopen_kwargs): - """Check that a buffered write, when it gets interrupted (either - returning a partial result or EINTR), properly invokes the signal - handler and retries if the latter returned successfully.""" - select = import_helper.import_module("select") - - # A quantity that exceeds the buffer size of an anonymous pipe's - # write end. - N = support.PIPE_MAX_SIZE - r, w = os.pipe() - fdopen_kwargs["closefd"] = False - - # We need a separate thread to read from the pipe and allow the - # write() to finish. This thread is started after the SIGALRM is - # received (forcing a first EINTR in write()). - read_results = [] - write_finished = False - error = None - def _read(): - try: - while not write_finished: - while r in select.select([r], [], [], 1.0)[0]: - s = os.read(r, 1024) - read_results.append(s) - except BaseException as exc: - nonlocal error - error = exc - t = threading.Thread(target=_read) - t.daemon = True - def alarm1(sig, frame): - signal.signal(signal.SIGALRM, alarm2) - signal.alarm(1) - def alarm2(sig, frame): - t.start() - - large_data = item * N - signal.signal(signal.SIGALRM, alarm1) - try: - wio = self.io.open(w, **fdopen_kwargs) - signal.alarm(1) - # Expected behaviour: - # - first raw write() is partial (because of the limited pipe buffer - # and the first alarm) - # - second raw write() returns EINTR (because of the second alarm) - # - subsequent write()s are successful (either partial or complete) - written = wio.write(large_data) - self.assertEqual(N, written) - - wio.flush() - write_finished = True - t.join() - - self.assertIsNone(error) - self.assertEqual(N, sum(len(x) for x in read_results)) - finally: - signal.alarm(0) - write_finished = True - os.close(w) - os.close(r) - # This is deliberate. If we didn't close the file descriptor - # before closing wio, wio would try to flush its internal - # buffer, and could block (in case of failure). - try: - wio.close() - except OSError as e: - if e.errno != errno.EBADF: - raise - - @requires_alarm - @support.requires_resource('walltime') - def test_interrupted_write_retry_buffered(self): - self.check_interrupted_write_retry(b"x", mode="wb") - - @requires_alarm - @support.requires_resource('walltime') - def test_interrupted_write_retry_text(self): - self.check_interrupted_write_retry("x", mode="w", encoding="latin1") - - -class CSignalsTest(SignalsTest): - io = io - -class PySignalsTest(SignalsTest): - io = pyio - - # Handling reentrancy issues would slow down _pyio even more, so the - # tests are disabled. - test_reentrant_write_buffered = None - test_reentrant_write_text = None - - -class ProtocolsTest(unittest.TestCase): - class MyReader: - def read(self, sz=-1): - return b"" - - class MyWriter: - def write(self, b: bytes): - pass - - def test_reader_subclass(self): - self.assertIsSubclass(MyReader, io.Reader[bytes]) - self.assertNotIsSubclass(str, io.Reader[bytes]) - - def test_writer_subclass(self): - self.assertIsSubclass(MyWriter, io.Writer[bytes]) - self.assertNotIsSubclass(str, io.Writer[bytes]) - - -def load_tests(loader, tests, pattern): - tests = (CIOTest, PyIOTest, APIMismatchTest, - CBufferedReaderTest, PyBufferedReaderTest, - CBufferedWriterTest, PyBufferedWriterTest, - CBufferedRWPairTest, PyBufferedRWPairTest, - CBufferedRandomTest, PyBufferedRandomTest, - StatefulIncrementalDecoderTest, - CIncrementalNewlineDecoderTest, PyIncrementalNewlineDecoderTest, - CTextIOWrapperTest, PyTextIOWrapperTest, - CMiscIOTest, PyMiscIOTest, - CSignalsTest, PySignalsTest, TestIOCTypes, - ) - - # Put the namespaces of the IO module we are testing and some useful mock - # classes in the __dict__ of each test. - mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO, - MockNonBlockWriterIO, MockUnseekableIO, MockRawIOWithoutRead, - SlowFlushRawIO, MockCharPseudoDevFileIO) - all_members = io.__all__ - c_io_ns = {name : getattr(io, name) for name in all_members} - py_io_ns = {name : getattr(pyio, name) for name in all_members} - globs = globals() - c_io_ns.update((x.__name__, globs["C" + x.__name__]) for x in mocks) - py_io_ns.update((x.__name__, globs["Py" + x.__name__]) for x in mocks) - for test in tests: - if test.__name__.startswith("C"): - for name, obj in c_io_ns.items(): - setattr(test, name, obj) - test.is_C = True - elif test.__name__.startswith("Py"): - for name, obj in py_io_ns.items(): - setattr(test, name, obj) - test.is_C = False - - suite = loader.suiteClass() - for test in tests: - suite.addTest(loader.loadTestsFromTestCase(test)) - return suite - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_io/__init__.py b/Lib/test/test_io/__init__.py new file mode 100644 index 00000000000..c94fad3e779 --- /dev/null +++ b/Lib/test/test_io/__init__.py @@ -0,0 +1,28 @@ +"""Tests for the io module and its implementations (_io and _pyio) + +Tests are split across multiple files to increase +parallelism and focus on specific implementation pieces. + +* test_io + * test_bufferedio - tests file buffering + * test_memoryio - tests BytesIO and StringIO + * test_fileio - tests FileIO + * test_file - tests the file interface + * test_general - tests everything else in the io module + * test_univnewlines - tests universal newline support + * test_largefile - tests operations on a file greater than 2**32 bytes + (only enabled with -ulargefile) +* test_free_threading/test_io - tests thread safety of io objects + +.. attention:: + When writing tests for io, it's important to test both the C and Python + implementations. This is usually done by writing a base test that refers to + the type it is testing as an attribute. Then it provides custom subclasses to + test both implementations. This directory contains lots of examples. +""" + +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_io/__main__.py b/Lib/test/test_io/__main__.py new file mode 100644 index 00000000000..40a23a297ec --- /dev/null +++ b/Lib/test/test_io/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py new file mode 100644 index 00000000000..3278665bdc9 --- /dev/null +++ b/Lib/test/test_io/test_bufferedio.py @@ -0,0 +1,1548 @@ +import array +import pickle +import random +import sys +import threading +import time +import unittest +import warnings +import weakref +from collections import deque, UserList +from itertools import cycle, count +from test import support +from test.support import os_helper, threading_helper +from .utils import byteslike, CTestCase, PyTestCase + + +import io # C implementation. +import _pyio as pyio # Python implementation. + + +class CommonBufferedTests: + # Tests common to BufferedReader, BufferedWriter and BufferedRandom + + def test_detach(self): + raw = self.MockRawIO() + buf = self.tp(raw) + self.assertIs(buf.detach(), raw) + self.assertRaises(ValueError, buf.detach) + + repr(buf) # Should still work + + def test_fileno(self): + rawio = self.MockRawIO() + bufio = self.tp(rawio) + + self.assertEqual(42, bufio.fileno()) + + def test_invalid_args(self): + rawio = self.MockRawIO() + bufio = self.tp(rawio) + # Invalid whence + self.assertRaises(ValueError, bufio.seek, 0, -1) + self.assertRaises(ValueError, bufio.seek, 0, 9) + + def test_override_destructor(self): + tp = self.tp + record = [] + class MyBufferedIO(tp): + def __del__(self): + record.append(1) + try: + f = super().__del__ + except AttributeError: + pass + else: + f() + def close(self): + record.append(2) + super().close() + def flush(self): + record.append(3) + super().flush() + rawio = self.MockRawIO() + bufio = MyBufferedIO(rawio) + del bufio + support.gc_collect() + self.assertEqual(record, [1, 2, 3]) + + def test_context_manager(self): + # Test usability as a context manager + rawio = self.MockRawIO() + bufio = self.tp(rawio) + def _with(): + with bufio: + pass + _with() + # bufio should now be closed, and using it a second time should raise + # a ValueError. + self.assertRaises(ValueError, _with) + + def test_error_through_destructor(self): + # Test that the exception state is not modified by a destructor, + # even if close() fails. + rawio = self.CloseFailureIO() + with support.catch_unraisable_exception() as cm: + with self.assertRaises(AttributeError): + self.tp(rawio).xyzzy + + self.assertEqual(cm.unraisable.exc_type, OSError) + + def test_repr(self): + raw = self.MockRawIO() + b = self.tp(raw) + clsname = r"(%s\.)?%s" % (self.tp.__module__, self.tp.__qualname__) + self.assertRegex(repr(b), "<%s>" % clsname) + raw.name = "dummy" + self.assertRegex(repr(b), "<%s name='dummy'>" % clsname) + raw.name = b"dummy" + self.assertRegex(repr(b), "<%s name=b'dummy'>" % clsname) + + def test_recursive_repr(self): + # Issue #25455 + raw = self.MockRawIO() + b = self.tp(raw) + with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): + repr(b) # Should not crash + + def test_flush_error_on_close(self): + # Test that buffered file is closed despite failed flush + # and that flush() is called before file closed. + raw = self.MockRawIO() + closed = [] + def bad_flush(): + closed[:] = [b.closed, raw.closed] + raise OSError() + raw.flush = bad_flush + b = self.tp(raw) + self.assertRaises(OSError, b.close) # exception not swallowed + self.assertTrue(b.closed) + self.assertTrue(raw.closed) + self.assertTrue(closed) # flush() called + self.assertFalse(closed[0]) # flush() called before file closed + self.assertFalse(closed[1]) + raw.flush = lambda: None # break reference loop + + def test_close_error_on_close(self): + raw = self.MockRawIO() + def bad_flush(): + raise OSError('flush') + def bad_close(): + raise OSError('close') + raw.close = bad_close + b = self.tp(raw) + b.flush = bad_flush + with self.assertRaises(OSError) as err: # exception not swallowed + b.close() + self.assertEqual(err.exception.args, ('close',)) + self.assertIsInstance(err.exception.__context__, OSError) + self.assertEqual(err.exception.__context__.args, ('flush',)) + self.assertFalse(b.closed) + + # Silence destructor error + raw.close = lambda: None + b.flush = lambda: None + + def test_nonnormalized_close_error_on_close(self): + # Issue #21677 + raw = self.MockRawIO() + def bad_flush(): + raise non_existing_flush + def bad_close(): + raise non_existing_close + raw.close = bad_close + b = self.tp(raw) + b.flush = bad_flush + with self.assertRaises(NameError) as err: # exception not swallowed + b.close() + self.assertIn('non_existing_close', str(err.exception)) + self.assertIsInstance(err.exception.__context__, NameError) + self.assertIn('non_existing_flush', str(err.exception.__context__)) + self.assertFalse(b.closed) + + # Silence destructor error + b.flush = lambda: None + raw.close = lambda: None + + def test_multi_close(self): + raw = self.MockRawIO() + b = self.tp(raw) + b.close() + b.close() + b.close() + self.assertRaises(ValueError, b.flush) + + def test_unseekable(self): + bufio = self.tp(self.MockUnseekableIO(b"A" * 10)) + self.assertRaises(self.UnsupportedOperation, bufio.tell) + self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) + + def test_readonly_attributes(self): + raw = self.MockRawIO() + buf = self.tp(raw) + x = self.MockRawIO() + with self.assertRaises(AttributeError): + buf.raw = x + + def test_pickling_subclass(self): + global MyBufferedIO + class MyBufferedIO(self.tp): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.raw.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + buf = MyBufferedIO(raw, tag='ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(buf, proto) + newbuf = pickle.loads(pickled) + self.assertEqual(newbuf.raw.getvalue(), b'data') + self.assertEqual(newbuf.tag, 'ham') + del MyBufferedIO + + +class SizeofTest: + + @support.cpython_only + def test_sizeof(self): + bufsize1 = 4096 + bufsize2 = 8192 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize1) + size = sys.getsizeof(bufio) - bufsize1 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize2) + self.assertEqual(sys.getsizeof(bufio), size + bufsize2) + + @support.cpython_only + def test_buffer_freeing(self) : + bufsize = 4096 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize) + size = sys.getsizeof(bufio) - bufsize + bufio.close() + self.assertEqual(sys.getsizeof(bufio), size) + +class BufferedReaderTest(CommonBufferedTests): + read_mode = "rb" + + def test_constructor(self): + rawio = self.MockRawIO([b"abc"]) + bufio = self.tp(rawio) + bufio.__init__(rawio) + bufio.__init__(rawio, buffer_size=1024) + bufio.__init__(rawio, buffer_size=16) + self.assertEqual(b"abc", bufio.read()) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) + rawio = self.MockRawIO([b"abc"]) + bufio.__init__(rawio) + self.assertEqual(b"abc", bufio.read()) + + def test_uninitialized(self): + bufio = self.tp.__new__(self.tp) + del bufio + bufio = self.tp.__new__(self.tp) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + bufio.read, 0) + bufio.__init__(self.MockRawIO()) + self.assertEqual(bufio.read(0), b'') + + def test_read(self): + for arg in (None, 7): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + self.assertEqual(b"abcdefg", bufio.read(arg)) + # Invalid args + self.assertRaises(ValueError, bufio.read, -2) + + def test_read1(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + self.assertEqual(b"a", bufio.read(1)) + self.assertEqual(b"b", bufio.read1(1)) + self.assertEqual(rawio._reads, 1) + self.assertEqual(b"", bufio.read1(0)) + self.assertEqual(b"c", bufio.read1(100)) + self.assertEqual(rawio._reads, 1) + self.assertEqual(b"d", bufio.read1(100)) + self.assertEqual(rawio._reads, 2) + self.assertEqual(b"efg", bufio.read1(100)) + self.assertEqual(rawio._reads, 3) + self.assertEqual(b"", bufio.read1(100)) + self.assertEqual(rawio._reads, 4) + + def test_read1_arbitrary(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + self.assertEqual(b"a", bufio.read(1)) + self.assertEqual(b"bc", bufio.read1()) + self.assertEqual(b"d", bufio.read1()) + self.assertEqual(b"efg", bufio.read1(-1)) + self.assertEqual(rawio._reads, 3) + self.assertEqual(b"", bufio.read1()) + self.assertEqual(rawio._reads, 4) + + def test_readinto(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + b = bytearray(2) + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"ab") + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"cd") + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"ef") + self.assertEqual(bufio.readinto(b), 1) + self.assertEqual(b, b"gf") + self.assertEqual(bufio.readinto(b), 0) + self.assertEqual(b, b"gf") + rawio = self.MockRawIO((b"abc", None)) + bufio = self.tp(rawio) + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"ab") + self.assertEqual(bufio.readinto(b), 1) + self.assertEqual(b, b"cb") + + def test_readinto1(self): + buffer_size = 10 + rawio = self.MockRawIO((b"abc", b"de", b"fgh", b"jkl")) + bufio = self.tp(rawio, buffer_size=buffer_size) + b = bytearray(2) + self.assertEqual(bufio.peek(3), b'abc') + self.assertEqual(rawio._reads, 1) + self.assertEqual(bufio.readinto1(b), 2) + self.assertEqual(b, b"ab") + self.assertEqual(rawio._reads, 1) + self.assertEqual(bufio.readinto1(b), 1) + self.assertEqual(b[:1], b"c") + self.assertEqual(rawio._reads, 1) + self.assertEqual(bufio.readinto1(b), 2) + self.assertEqual(b, b"de") + self.assertEqual(rawio._reads, 2) + b = bytearray(2*buffer_size) + self.assertEqual(bufio.peek(3), b'fgh') + self.assertEqual(rawio._reads, 3) + self.assertEqual(bufio.readinto1(b), 6) + self.assertEqual(b[:6], b"fghjkl") + self.assertEqual(rawio._reads, 4) + + def test_readinto_array(self): + buffer_size = 60 + data = b"a" * 26 + rawio = self.MockRawIO((data,)) + bufio = self.tp(rawio, buffer_size=buffer_size) + + # Create an array with element size > 1 byte + b = array.array('i', b'x' * 32) + assert len(b) != 16 + + # Read into it. We should get as many *bytes* as we can fit into b + # (which is more than the number of elements) + n = bufio.readinto(b) + self.assertGreater(n, len(b)) + + # Check that old contents of b are preserved + bm = memoryview(b).cast('B') + self.assertLess(n, len(bm)) + self.assertEqual(bm[:n], data[:n]) + self.assertEqual(bm[n:], b'x' * (len(bm[n:]))) + + def test_readinto1_array(self): + buffer_size = 60 + data = b"a" * 26 + rawio = self.MockRawIO((data,)) + bufio = self.tp(rawio, buffer_size=buffer_size) + + # Create an array with element size > 1 byte + b = array.array('i', b'x' * 32) + assert len(b) != 16 + + # Read into it. We should get as many *bytes* as we can fit into b + # (which is more than the number of elements) + n = bufio.readinto1(b) + self.assertGreater(n, len(b)) + + # Check that old contents of b are preserved + bm = memoryview(b).cast('B') + self.assertLess(n, len(bm)) + self.assertEqual(bm[:n], data[:n]) + self.assertEqual(bm[n:], b'x' * (len(bm[n:]))) + + def test_readlines(self): + def bufio(): + rawio = self.MockRawIO((b"abc\n", b"d\n", b"ef")) + return self.tp(rawio) + self.assertEqual(bufio().readlines(), [b"abc\n", b"d\n", b"ef"]) + self.assertEqual(bufio().readlines(5), [b"abc\n", b"d\n"]) + self.assertEqual(bufio().readlines(None), [b"abc\n", b"d\n", b"ef"]) + + def test_buffering(self): + data = b"abcdefghi" + dlen = len(data) + + tests = [ + [ 100, [ 3, 1, 4, 8 ], [ dlen, 0 ] ], + [ 100, [ 3, 3, 3], [ dlen ] ], + [ 4, [ 1, 2, 4, 2 ], [ 4, 4, 1 ] ], + ] + + for bufsize, buf_read_sizes, raw_read_sizes in tests: + rawio = self.MockFileIO(data) + bufio = self.tp(rawio, buffer_size=bufsize) + pos = 0 + for nbytes in buf_read_sizes: + self.assertEqual(bufio.read(nbytes), data[pos:pos+nbytes]) + pos += nbytes + # this is mildly implementation-dependent + self.assertEqual(rawio.read_history, raw_read_sizes) + + def test_read_non_blocking(self): + # Inject some None's in there to simulate EWOULDBLOCK + rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None)) + bufio = self.tp(rawio) + self.assertEqual(b"abcd", bufio.read(6)) + self.assertEqual(b"e", bufio.read(1)) + self.assertEqual(b"fg", bufio.read()) + self.assertEqual(b"", bufio.peek(1)) + self.assertIsNone(bufio.read()) + self.assertEqual(b"", bufio.read()) + + rawio = self.MockRawIO((b"a", None, None)) + self.assertEqual(b"a", rawio.readall()) + self.assertIsNone(rawio.readall()) + + def test_read_past_eof(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + + self.assertEqual(b"abcdefg", bufio.read(9000)) + + def test_read_all(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + + self.assertEqual(b"abcdefg", bufio.read()) + + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') + def test_threads(self): + try: + # Write out many bytes with exactly the same number of 0's, + # 1's... 255's. This will help us check that concurrent reading + # doesn't duplicate or forget contents. + N = 1000 + l = list(range(256)) * N + random.shuffle(l) + s = bytes(bytearray(l)) + with self.open(os_helper.TESTFN, "wb") as f: + f.write(s) + with self.open(os_helper.TESTFN, self.read_mode, buffering=0) as raw: + bufio = self.tp(raw, 8) + errors = [] + results = [] + def f(): + try: + # Intra-buffer read then buffer-flushing read + for n in cycle([1, 19]): + s = bufio.read(n) + if not s: + break + # list.append() is atomic + results.append(s) + except Exception as e: + errors.append(e) + raise + threads = [threading.Thread(target=f) for x in range(20)] + with threading_helper.start_threads(threads): + time.sleep(0.02) # yield + self.assertFalse(errors, + "the following exceptions were caught: %r" % errors) + s = b''.join(results) + for i in range(256): + c = bytes(bytearray([i])) + self.assertEqual(s.count(c), N) + finally: + os_helper.unlink(os_helper.TESTFN) + + def test_unseekable(self): + bufio = self.tp(self.MockUnseekableIO(b"A" * 10)) + self.assertRaises(self.UnsupportedOperation, bufio.tell) + self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) + bufio.read(1) + self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) + self.assertRaises(self.UnsupportedOperation, bufio.tell) + + def test_misbehaved_io(self): + rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + self.assertRaises(OSError, bufio.seek, 0) + self.assertRaises(OSError, bufio.tell) + + # Silence destructor error + bufio.close = lambda: None + + def test_no_extraneous_read(self): + # Issue #9550; when the raw IO object has satisfied the read request, + # we should not issue any additional reads, otherwise it may block + # (e.g. socket). + bufsize = 16 + for n in (2, bufsize - 1, bufsize, bufsize + 1, bufsize * 2): + rawio = self.MockRawIO([b"x" * n]) + bufio = self.tp(rawio, bufsize) + self.assertEqual(bufio.read(n), b"x" * n) + # Simple case: one raw read is enough to satisfy the request. + self.assertEqual(rawio._extraneous_reads, 0, + "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) + # A more complex case where two raw reads are needed to satisfy + # the request. + rawio = self.MockRawIO([b"x" * (n - 1), b"x"]) + bufio = self.tp(rawio, bufsize) + self.assertEqual(bufio.read(n), b"x" * n) + self.assertEqual(rawio._extraneous_reads, 0, + "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) + + def test_read_on_closed(self): + # Issue #23796 + b = self.BufferedReader(self.BytesIO(b"12")) + b.read(1) + b.close() + with self.subTest('peek'): + self.assertRaises(ValueError, b.peek) + with self.subTest('read1'): + self.assertRaises(ValueError, b.read1, 1) + with self.subTest('read'): + self.assertRaises(ValueError, b.read) + with self.subTest('readinto'): + self.assertRaises(ValueError, b.readinto, bytearray()) + with self.subTest('readinto1'): + self.assertRaises(ValueError, b.readinto1, bytearray()) + with self.subTest('flush'): + self.assertRaises(ValueError, b.flush) + with self.subTest('truncate'): + self.assertRaises(ValueError, b.truncate) + with self.subTest('seek'): + self.assertRaises(ValueError, b.seek, 0) + + def test_truncate_on_read_only(self): + rawio = self.MockFileIO(b"abc") + bufio = self.tp(rawio) + self.assertFalse(bufio.writable()) + self.assertRaises(self.UnsupportedOperation, bufio.truncate) + self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) + + def test_tell_character_device_file(self): + # GH-95782 + # For the (former) bug in BufferedIO to manifest, the wrapped IO obj + # must be able to produce at least 2 bytes. + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.read(1), b"1") + self.assertEqual(buf.tell(), 0) + + def test_seek_character_device_file(self): + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.seek(1, io.SEEK_SET), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.read(1), b"1") + + # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, + # which means that the next seek() could return a negative offset if it + # does not sanity-check: + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + + +class CBufferedReaderTest(BufferedReaderTest, SizeofTest, CTestCase): + tp = io.BufferedReader + + def test_initialization(self): + rawio = self.MockRawIO([b"abc"]) + bufio = self.tp(rawio) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) + self.assertRaises(ValueError, bufio.read) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) + self.assertRaises(ValueError, bufio.read) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) + self.assertRaises(ValueError, bufio.read) + + def test_misbehaved_io_read(self): + rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + # _pyio.BufferedReader seems to implement reading different, so that + # checking this is not so easy. + self.assertRaises(OSError, bufio.read, 10) + + def test_garbage_collection(self): + # C BufferedReader objects are collected. + # The Python version has __del__, so it ends into gc.garbage instead + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + rawio = self.FileIO(os_helper.TESTFN, "w+b") + f = self.tp(rawio) + f.f = f + wr = weakref.ref(f) + del f + support.gc_collect() + self.assertIsNone(wr(), wr) + + def test_args_error(self): + # Issue #17275 + with self.assertRaisesRegex(TypeError, "BufferedReader"): + self.tp(self.BytesIO(), 1024, 1024, 1024) + + def test_bad_readinto_value(self): + rawio = self.tp(self.BytesIO(b"12")) + rawio.readinto = lambda buf: -1 + bufio = self.tp(rawio) + with self.assertRaises(OSError) as cm: + bufio.readline() + self.assertIsNone(cm.exception.__cause__) + + def test_bad_readinto_type(self): + rawio = self.tp(self.BytesIO(b"12")) + rawio.readinto = lambda buf: b'' + bufio = self.tp(rawio) + with self.assertRaises(OSError) as cm: + bufio.readline() + self.assertIsInstance(cm.exception.__cause__, TypeError) + + +class PyBufferedReaderTest(BufferedReaderTest, PyTestCase): + tp = pyio.BufferedReader + + +class BufferedWriterTest(CommonBufferedTests): + write_mode = "wb" + + def test_constructor(self): + rawio = self.MockRawIO() + bufio = self.tp(rawio) + bufio.__init__(rawio) + bufio.__init__(rawio, buffer_size=1024) + bufio.__init__(rawio, buffer_size=16) + self.assertEqual(3, bufio.write(b"abc")) + bufio.flush() + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) + bufio.__init__(rawio) + self.assertEqual(3, bufio.write(b"ghi")) + bufio.flush() + self.assertEqual(b"".join(rawio._write_stack), b"abcghi") + + def test_uninitialized(self): + bufio = self.tp.__new__(self.tp) + del bufio + bufio = self.tp.__new__(self.tp) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + bufio.write, b'') + bufio.__init__(self.MockRawIO()) + self.assertEqual(bufio.write(b''), 0) + + def test_detach_flush(self): + raw = self.MockRawIO() + buf = self.tp(raw) + buf.write(b"howdy!") + self.assertFalse(raw._write_stack) + buf.detach() + self.assertEqual(raw._write_stack, [b"howdy!"]) + + def test_write(self): + # Write to the buffered IO but don't overflow the buffer. + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.write(b"abc") + self.assertFalse(writer._write_stack) + buffer = bytearray(b"def") + bufio.write(buffer) + buffer[:] = b"***" # Overwrite our copy of the data + bufio.flush() + self.assertEqual(b"".join(writer._write_stack), b"abcdef") + + def test_write_overflow(self): + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + contents = b"abcdefghijklmnop" + for n in range(0, len(contents), 3): + bufio.write(contents[n:n+3]) + flushed = b"".join(writer._write_stack) + # At least (total - 8) bytes were implicitly flushed, perhaps more + # depending on the implementation. + self.assertStartsWith(flushed, contents[:-8]) + + def check_writes(self, intermediate_func): + # Lots of writes, test the flushed output is as expected. + contents = bytes(range(256)) * 1000 + n = 0 + writer = self.MockRawIO() + bufio = self.tp(writer, 13) + # Generator of write sizes: repeat each N 15 times then proceed to N+1 + def gen_sizes(): + for size in count(1): + for i in range(15): + yield size + sizes = gen_sizes() + while n < len(contents): + size = min(next(sizes), len(contents) - n) + self.assertEqual(bufio.write(contents[n:n+size]), size) + intermediate_func(bufio) + n += size + bufio.flush() + self.assertEqual(contents, b"".join(writer._write_stack)) + + def test_writes(self): + self.check_writes(lambda bufio: None) + + def test_writes_and_flushes(self): + self.check_writes(lambda bufio: bufio.flush()) + + def test_writes_and_seeks(self): + def _seekabs(bufio): + pos = bufio.tell() + bufio.seek(pos + 1, 0) + bufio.seek(pos - 1, 0) + bufio.seek(pos, 0) + self.check_writes(_seekabs) + def _seekrel(bufio): + pos = bufio.seek(0, 1) + bufio.seek(+1, 1) + bufio.seek(-1, 1) + bufio.seek(pos, 0) + self.check_writes(_seekrel) + + def test_writes_and_truncates(self): + self.check_writes(lambda bufio: bufio.truncate(bufio.tell())) + + def test_write_non_blocking(self): + raw = self.MockNonBlockWriterIO() + bufio = self.tp(raw, 8) + + self.assertEqual(bufio.write(b"abcd"), 4) + self.assertEqual(bufio.write(b"efghi"), 5) + # 1 byte will be written, the rest will be buffered + raw.block_on(b"k") + self.assertEqual(bufio.write(b"jklmn"), 5) + + # 8 bytes will be written, 8 will be buffered and the rest will be lost + raw.block_on(b"0") + try: + bufio.write(b"opqrwxyz0123456789") + except self.BlockingIOError as e: + written = e.characters_written + else: + self.fail("BlockingIOError should have been raised") + self.assertEqual(written, 16) + self.assertEqual(raw.pop_written(), + b"abcdefghijklmnopqrwxyz") + + self.assertEqual(bufio.write(b"ABCDEFGHI"), 9) + s = raw.pop_written() + # Previously buffered bytes were flushed + self.assertStartsWith(s, b"01234567A") + + def test_write_and_rewind(self): + raw = self.BytesIO() + bufio = self.tp(raw, 4) + self.assertEqual(bufio.write(b"abcdef"), 6) + self.assertEqual(bufio.tell(), 6) + bufio.seek(0, 0) + self.assertEqual(bufio.write(b"XY"), 2) + bufio.seek(6, 0) + self.assertEqual(raw.getvalue(), b"XYcdef") + self.assertEqual(bufio.write(b"123456"), 6) + bufio.flush() + self.assertEqual(raw.getvalue(), b"XYcdef123456") + + def test_flush(self): + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.write(b"abc") + bufio.flush() + self.assertEqual(b"abc", writer._write_stack[0]) + + def test_writelines(self): + l = [b'ab', b'cd', b'ef'] + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.writelines(l) + bufio.flush() + self.assertEqual(b''.join(writer._write_stack), b'abcdef') + + def test_writelines_userlist(self): + l = UserList([b'ab', b'cd', b'ef']) + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.writelines(l) + bufio.flush() + self.assertEqual(b''.join(writer._write_stack), b'abcdef') + + def test_writelines_error(self): + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + self.assertRaises(TypeError, bufio.writelines, [1, 2, 3]) + self.assertRaises(TypeError, bufio.writelines, None) + self.assertRaises(TypeError, bufio.writelines, 'abc') + + def test_destructor(self): + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.write(b"abc") + del bufio + support.gc_collect() + self.assertEqual(b"abc", writer._write_stack[0]) + + def test_truncate(self): + # Truncate implicitly flushes the buffer. + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw: + bufio = self.tp(raw, 8) + bufio.write(b"abcdef") + self.assertEqual(bufio.truncate(3), 3) + self.assertEqual(bufio.tell(), 6) + with self.open(os_helper.TESTFN, "rb", buffering=0) as f: + self.assertEqual(f.read(), b"abc") + + def test_truncate_after_write(self): + # Ensure that truncate preserves the file position after + # writes longer than the buffer size. + # Issue: https://bugs.python.org/issue32228 + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with self.open(os_helper.TESTFN, "wb") as f: + # Fill with some buffer + f.write(b'\x00' * 10000) + buffer_sizes = [8192, 4096, 200] + for buffer_size in buffer_sizes: + with self.open(os_helper.TESTFN, "r+b", buffering=buffer_size) as f: + f.write(b'\x00' * (buffer_size + 1)) + # After write write_pos and write_end are set to 0 + f.read(1) + # read operation makes sure that pos != raw_pos + f.truncate() + self.assertEqual(f.tell(), buffer_size + 2) + + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') + def test_threads(self): + try: + # Write out many bytes from many threads and test they were + # all flushed. + N = 1000 + contents = bytes(range(256)) * N + sizes = cycle([1, 19]) + n = 0 + queue = deque() + while n < len(contents): + size = next(sizes) + queue.append(contents[n:n+size]) + n += size + del contents + # We use a real file object because it allows us to + # exercise situations where the GIL is released before + # writing the buffer to the raw streams. This is in addition + # to concurrency issues due to switching threads in the middle + # of Python code. + with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw: + bufio = self.tp(raw, 8) + errors = [] + def f(): + try: + while True: + try: + s = queue.popleft() + except IndexError: + return + bufio.write(s) + except Exception as e: + errors.append(e) + raise + threads = [threading.Thread(target=f) for x in range(20)] + with threading_helper.start_threads(threads): + time.sleep(0.02) # yield + self.assertFalse(errors, + "the following exceptions were caught: %r" % errors) + bufio.close() + with self.open(os_helper.TESTFN, "rb") as f: + s = f.read() + for i in range(256): + self.assertEqual(s.count(bytes([i])), N) + finally: + os_helper.unlink(os_helper.TESTFN) + + def test_misbehaved_io(self): + rawio = self.MisbehavedRawIO() + bufio = self.tp(rawio, 5) + self.assertRaises(OSError, bufio.seek, 0) + self.assertRaises(OSError, bufio.tell) + self.assertRaises(OSError, bufio.write, b"abcdef") + + # Silence destructor error + bufio.close = lambda: None + + def test_max_buffer_size_removal(self): + with self.assertRaises(TypeError): + self.tp(self.MockRawIO(), 8, 12) + + def test_write_error_on_close(self): + raw = self.MockRawIO() + def bad_write(b): + raise OSError() + raw.write = bad_write + b = self.tp(raw) + b.write(b'spam') + self.assertRaises(OSError, b.close) # exception not swallowed + self.assertTrue(b.closed) + + @threading_helper.requires_working_threading() + def test_slow_close_from_thread(self): + # Issue #31976 + rawio = self.SlowFlushRawIO() + bufio = self.tp(rawio, 8) + t = threading.Thread(target=bufio.close) + t.start() + rawio.in_flush.wait() + self.assertRaises(ValueError, bufio.write, b'spam') + self.assertTrue(bufio.closed) + t.join() + + +class CBufferedWriterTest(BufferedWriterTest, SizeofTest, CTestCase): + tp = io.BufferedWriter + + def test_initialization(self): + rawio = self.MockRawIO() + bufio = self.tp(rawio) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) + self.assertRaises(ValueError, bufio.write, b"def") + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) + self.assertRaises(ValueError, bufio.write, b"def") + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) + self.assertRaises(ValueError, bufio.write, b"def") + + def test_garbage_collection(self): + # C BufferedWriter objects are collected, and collecting them flushes + # all data to disk. + # The Python version has __del__, so it ends into gc.garbage instead + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + rawio = self.FileIO(os_helper.TESTFN, "w+b") + f = self.tp(rawio) + f.write(b"123xxx") + f.x = f + wr = weakref.ref(f) + del f + support.gc_collect() + self.assertIsNone(wr(), wr) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"123xxx") + + def test_args_error(self): + # Issue #17275 + with self.assertRaisesRegex(TypeError, "BufferedWriter"): + self.tp(self.BytesIO(), 1024, 1024, 1024) + + def test_non_boolean_closed_attr(self): + # gh-140650: check TypeError is raised + class MockRawIOWithoutClosed(self.MockRawIO): + closed = NotImplemented + + bufio = self.tp(MockRawIOWithoutClosed()) + self.assertRaises(TypeError, bufio.write, b"") + self.assertRaises(TypeError, bufio.flush) + self.assertRaises(TypeError, bufio.close) + + def test_closed_attr_raises(self): + class MockRawIOClosedRaises(self.MockRawIO): + @property + def closed(self): + raise ValueError("test") + + bufio = self.tp(MockRawIOClosedRaises()) + self.assertRaisesRegex(ValueError, "test", bufio.write, b"") + self.assertRaisesRegex(ValueError, "test", bufio.flush) + self.assertRaisesRegex(ValueError, "test", bufio.close) + + +class PyBufferedWriterTest(BufferedWriterTest, PyTestCase): + tp = pyio.BufferedWriter + +class BufferedRWPairTest: + + def test_constructor(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertFalse(pair.closed) + + def test_uninitialized(self): + pair = self.tp.__new__(self.tp) + del pair + pair = self.tp.__new__(self.tp) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + pair.read, 0) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + pair.write, b'') + pair.__init__(self.MockRawIO(), self.MockRawIO()) + self.assertEqual(pair.read(0), b'') + self.assertEqual(pair.write(b''), 0) + + def test_detach(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertRaises(self.UnsupportedOperation, pair.detach) + + def test_constructor_max_buffer_size_removal(self): + with self.assertRaises(TypeError): + self.tp(self.MockRawIO(), self.MockRawIO(), 8, 12) + + def test_constructor_with_not_readable(self): + class NotReadable(self.MockRawIO): + def readable(self): + return False + + self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO()) + + def test_constructor_with_not_writeable(self): + class NotWriteable(self.MockRawIO): + def writable(self): + return False + + self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable()) + + def test_read(self): + pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) + + self.assertEqual(pair.read(3), b"abc") + self.assertEqual(pair.read(1), b"d") + self.assertEqual(pair.read(), b"ef") + pair = self.tp(self.BytesIO(b"abc"), self.MockRawIO()) + self.assertEqual(pair.read(None), b"abc") + + def test_readlines(self): + pair = lambda: self.tp(self.BytesIO(b"abc\ndef\nh"), self.MockRawIO()) + self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) + self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) + self.assertEqual(pair().readlines(5), [b"abc\n", b"def\n"]) + + def test_read1(self): + # .read1() is delegated to the underlying reader object, so this test + # can be shallow. + pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) + + self.assertEqual(pair.read1(3), b"abc") + self.assertEqual(pair.read1(), b"def") + + def test_readinto(self): + for method in ("readinto", "readinto1"): + with self.subTest(method): + pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) + + data = byteslike(b'\0' * 5) + self.assertEqual(getattr(pair, method)(data), 5) + self.assertEqual(bytes(data), b"abcde") + + # gh-138720: C BufferedRWPair would destruct in a bad order resulting in + # an unraisable exception. + support.gc_collect() + + def test_write(self): + w = self.MockRawIO() + pair = self.tp(self.MockRawIO(), w) + + pair.write(b"abc") + pair.flush() + buffer = bytearray(b"def") + pair.write(buffer) + buffer[:] = b"***" # Overwrite our copy of the data + pair.flush() + self.assertEqual(w._write_stack, [b"abc", b"def"]) + + def test_peek(self): + pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) + + self.assertStartsWith(pair.peek(3), b"abc") + self.assertEqual(pair.read(3), b"abc") + + def test_readable(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertTrue(pair.readable()) + + def test_writeable(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertTrue(pair.writable()) + + def test_seekable(self): + # BufferedRWPairs are never seekable, even if their readers and writers + # are. + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertFalse(pair.seekable()) + + # .flush() is delegated to the underlying writer object and has been + # tested in the test_write method. + + def test_close_and_closed(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertFalse(pair.closed) + pair.close() + self.assertTrue(pair.closed) + + def test_reader_close_error_on_close(self): + def reader_close(): + reader_non_existing + reader = self.MockRawIO() + reader.close = reader_close + writer = self.MockRawIO() + pair = self.tp(reader, writer) + with self.assertRaises(NameError) as err: + pair.close() + self.assertIn('reader_non_existing', str(err.exception)) + self.assertTrue(pair.closed) + self.assertFalse(reader.closed) + self.assertTrue(writer.closed) + + # Silence destructor error + reader.close = lambda: None + + def test_writer_close_error_on_close(self): + def writer_close(): + writer_non_existing + reader = self.MockRawIO() + writer = self.MockRawIO() + writer.close = writer_close + pair = self.tp(reader, writer) + with self.assertRaises(NameError) as err: + pair.close() + self.assertIn('writer_non_existing', str(err.exception)) + self.assertFalse(pair.closed) + self.assertTrue(reader.closed) + self.assertFalse(writer.closed) + + # Silence destructor error + writer.close = lambda: None + writer = None + + # Ignore BufferedWriter (of the BufferedRWPair) unraisable exception + with support.catch_unraisable_exception(): + # Ignore BufferedRWPair unraisable exception + with support.catch_unraisable_exception(): + pair = None + support.gc_collect() + support.gc_collect() + + def test_reader_writer_close_error_on_close(self): + def reader_close(): + reader_non_existing + def writer_close(): + writer_non_existing + reader = self.MockRawIO() + reader.close = reader_close + writer = self.MockRawIO() + writer.close = writer_close + pair = self.tp(reader, writer) + with self.assertRaises(NameError) as err: + pair.close() + self.assertIn('reader_non_existing', str(err.exception)) + self.assertIsInstance(err.exception.__context__, NameError) + self.assertIn('writer_non_existing', str(err.exception.__context__)) + self.assertFalse(pair.closed) + self.assertFalse(reader.closed) + self.assertFalse(writer.closed) + + # Silence destructor error + reader.close = lambda: None + writer.close = lambda: None + + def test_isatty(self): + class SelectableIsAtty(self.MockRawIO): + def __init__(self, isatty): + super().__init__() + self._isatty = isatty + + def isatty(self): + return self._isatty + + pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(False)) + self.assertFalse(pair.isatty()) + + pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(False)) + self.assertTrue(pair.isatty()) + + pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(True)) + self.assertTrue(pair.isatty()) + + pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(True)) + self.assertTrue(pair.isatty()) + + def test_weakref_clearing(self): + brw = self.tp(self.MockRawIO(), self.MockRawIO()) + ref = weakref.ref(brw) + brw = None + ref = None # Shouldn't segfault. + +class CBufferedRWPairTest(BufferedRWPairTest, CTestCase): + tp = io.BufferedRWPair + +class PyBufferedRWPairTest(BufferedRWPairTest, PyTestCase): + tp = pyio.BufferedRWPair + + +class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest): + read_mode = "rb+" + write_mode = "wb+" + + def test_constructor(self): + BufferedReaderTest.test_constructor(self) + BufferedWriterTest.test_constructor(self) + + def test_uninitialized(self): + BufferedReaderTest.test_uninitialized(self) + BufferedWriterTest.test_uninitialized(self) + + def test_read_and_write(self): + raw = self.MockRawIO((b"asdf", b"ghjk")) + rw = self.tp(raw, 8) + + self.assertEqual(b"as", rw.read(2)) + rw.write(b"ddd") + rw.write(b"eee") + self.assertFalse(raw._write_stack) # Buffer writes + self.assertEqual(b"ghjk", rw.read()) + self.assertEqual(b"dddeee", raw._write_stack[0]) + + def test_seek_and_tell(self): + raw = self.BytesIO(b"asdfghjkl") + rw = self.tp(raw) + + self.assertEqual(b"as", rw.read(2)) + self.assertEqual(2, rw.tell()) + rw.seek(0, 0) + self.assertEqual(b"asdf", rw.read(4)) + + rw.write(b"123f") + rw.seek(0, 0) + self.assertEqual(b"asdf123fl", rw.read()) + self.assertEqual(9, rw.tell()) + rw.seek(-4, 2) + self.assertEqual(5, rw.tell()) + rw.seek(2, 1) + self.assertEqual(7, rw.tell()) + self.assertEqual(b"fl", rw.read(11)) + rw.flush() + self.assertEqual(b"asdf123fl", raw.getvalue()) + + self.assertRaises(TypeError, rw.seek, 0.0) + + def check_flush_and_read(self, read_func): + raw = self.BytesIO(b"abcdefghi") + bufio = self.tp(raw) + + self.assertEqual(b"ab", read_func(bufio, 2)) + bufio.write(b"12") + self.assertEqual(b"ef", read_func(bufio, 2)) + self.assertEqual(6, bufio.tell()) + bufio.flush() + self.assertEqual(6, bufio.tell()) + self.assertEqual(b"ghi", read_func(bufio)) + raw.seek(0, 0) + raw.write(b"XYZ") + # flush() resets the read buffer + bufio.flush() + bufio.seek(0, 0) + self.assertEqual(b"XYZ", read_func(bufio, 3)) + + def test_flush_and_read(self): + self.check_flush_and_read(lambda bufio, *args: bufio.read(*args)) + + def test_flush_and_readinto(self): + def _readinto(bufio, n=-1): + b = bytearray(n if n >= 0 else 9999) + n = bufio.readinto(b) + b.resize(n) + return b.take_bytes() + self.check_flush_and_read(_readinto) + + def test_flush_and_peek(self): + def _peek(bufio, n=-1): + # This relies on the fact that the buffer can contain the whole + # raw stream, otherwise peek() can return less. + b = bufio.peek(n) + if n != -1: + b = b[:n] + bufio.seek(len(b), 1) + return b + self.check_flush_and_read(_peek) + + def test_flush_and_write(self): + raw = self.BytesIO(b"abcdefghi") + bufio = self.tp(raw) + + bufio.write(b"123") + bufio.flush() + bufio.write(b"45") + bufio.flush() + bufio.seek(0, 0) + self.assertEqual(b"12345fghi", raw.getvalue()) + self.assertEqual(b"12345fghi", bufio.read()) + + def test_threads(self): + BufferedReaderTest.test_threads(self) + BufferedWriterTest.test_threads(self) + + def test_writes_and_peek(self): + def _peek(bufio): + bufio.peek(1) + self.check_writes(_peek) + def _peek(bufio): + pos = bufio.tell() + bufio.seek(-1, 1) + bufio.peek(1) + bufio.seek(pos, 0) + self.check_writes(_peek) + + def test_writes_and_reads(self): + def _read(bufio): + bufio.seek(-1, 1) + bufio.read(1) + self.check_writes(_read) + + def test_writes_and_read1s(self): + def _read1(bufio): + bufio.seek(-1, 1) + bufio.read1(1) + self.check_writes(_read1) + + def test_writes_and_readintos(self): + def _read(bufio): + bufio.seek(-1, 1) + bufio.readinto(bytearray(1)) + self.check_writes(_read) + + def test_write_after_readahead(self): + # Issue #6629: writing after the buffer was filled by readahead should + # first rewind the raw stream. + for overwrite_size in [1, 5]: + raw = self.BytesIO(b"A" * 10) + bufio = self.tp(raw, 4) + # Trigger readahead + self.assertEqual(bufio.read(1), b"A") + self.assertEqual(bufio.tell(), 1) + # Overwriting should rewind the raw stream if it needs so + bufio.write(b"B" * overwrite_size) + self.assertEqual(bufio.tell(), overwrite_size + 1) + # If the write size was smaller than the buffer size, flush() and + # check that rewind happens. + bufio.flush() + self.assertEqual(bufio.tell(), overwrite_size + 1) + s = raw.getvalue() + self.assertEqual(s, + b"A" + b"B" * overwrite_size + b"A" * (9 - overwrite_size)) + + def test_write_rewind_write(self): + # Various combinations of reading / writing / seeking backwards / writing again + def mutate(bufio, pos1, pos2): + assert pos2 >= pos1 + # Fill the buffer + bufio.seek(pos1) + bufio.read(pos2 - pos1) + bufio.write(b'\x02') + # This writes earlier than the previous write, but still inside + # the buffer. + bufio.seek(pos1) + bufio.write(b'\x01') + + b = b"\x80\x81\x82\x83\x84" + for i in range(0, len(b)): + for j in range(i, len(b)): + raw = self.BytesIO(b) + bufio = self.tp(raw, 100) + mutate(bufio, i, j) + bufio.flush() + expected = bytearray(b) + expected[j] = 2 + expected[i] = 1 + self.assertEqual(raw.getvalue(), expected, + "failed result for i=%d, j=%d" % (i, j)) + + def test_truncate_after_read_or_write(self): + raw = self.BytesIO(b"A" * 10) + bufio = self.tp(raw, 100) + self.assertEqual(bufio.read(2), b"AA") # the read buffer gets filled + self.assertEqual(bufio.truncate(), 2) + self.assertEqual(bufio.write(b"BB"), 2) # the write buffer increases + self.assertEqual(bufio.truncate(), 4) + + def test_misbehaved_io(self): + BufferedReaderTest.test_misbehaved_io(self) + BufferedWriterTest.test_misbehaved_io(self) + + def test_interleaved_read_write(self): + # Test for issue #12213 + with self.BytesIO(b'abcdefgh') as raw: + with self.tp(raw, 100) as f: + f.write(b"1") + self.assertEqual(f.read(1), b'b') + f.write(b'2') + self.assertEqual(f.read1(1), b'd') + f.write(b'3') + buf = bytearray(1) + f.readinto(buf) + self.assertEqual(buf, b'f') + f.write(b'4') + self.assertEqual(f.peek(1), b'h') + f.flush() + self.assertEqual(raw.getvalue(), b'1b2d3f4h') + + with self.BytesIO(b'abc') as raw: + with self.tp(raw, 100) as f: + self.assertEqual(f.read(1), b'a') + f.write(b"2") + self.assertEqual(f.read(1), b'c') + f.flush() + self.assertEqual(raw.getvalue(), b'a2c') + + def test_read1_after_write(self): + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(1), b'b') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(), b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + # XXX: read(100) returns different numbers of bytes + # in Python and C implementations. + self.assertEqual(f.read1(100)[:3], b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + + def test_interleaved_readline_write(self): + with self.BytesIO(b'ab\ncdef\ng\n') as raw: + with self.tp(raw) as f: + f.write(b'1') + self.assertEqual(f.readline(), b'b\n') + f.write(b'2') + self.assertEqual(f.readline(), b'def\n') + f.write(b'3') + self.assertEqual(f.readline(), b'\n') + f.flush() + self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n') + + # You can't construct a BufferedRandom over a non-seekable stream. + test_unseekable = None + + # writable() returns True, so there's no point to test it over + # a writable stream. + test_truncate_on_read_only = None + + +class CBufferedRandomTest(BufferedRandomTest, SizeofTest, CTestCase): + tp = io.BufferedRandom + + def test_garbage_collection(self): + CBufferedReaderTest.test_garbage_collection(self) + CBufferedWriterTest.test_garbage_collection(self) + + def test_args_error(self): + # Issue #17275 + with self.assertRaisesRegex(TypeError, "BufferedRandom"): + self.tp(self.BytesIO(), 1024, 1024, 1024) + + +class PyBufferedRandomTest(BufferedRandomTest, PyTestCase): + tp = pyio.BufferedRandom + + +# Simple test to ensure that optimizations in the IO library deliver the +# expected results. For best testing, run this under a debug-build Python too +# (to exercise asserts in the C code). + +lengths = list(range(1, 257)) + [512, 1000, 1024, 2048, 4096, 8192, 10000, + 16384, 32768, 65536, 1000000] + +class BufferSizeTest: + def try_one(self, s): + # Write s + "\n" + s to file, then open it and ensure that successive + # .readline()s deliver what we wrote. + + # Ensure we can open TESTFN for writing. + os_helper.unlink(os_helper.TESTFN) + + # Since C doesn't guarantee we can write/read arbitrary bytes in text + # files, use binary mode. + f = self.open(os_helper.TESTFN, "wb") + try: + # write once with \n and once without + f.write(s) + f.write(b"\n") + f.write(s) + f.close() + f = self.open(os_helper.TESTFN, "rb") + line = f.readline() + self.assertEqual(line, s + b"\n") + line = f.readline() + self.assertEqual(line, s) + line = f.readline() + self.assertFalse(line) # Must be at EOF + f.close() + finally: + os_helper.unlink(os_helper.TESTFN) + + def drive_one(self, pattern): + for length in lengths: + # Repeat string 'pattern' as often as needed to reach total length + # 'length'. Then call try_one with that string, a string one larger + # than that, and a string one smaller than that. Try this with all + # small sizes and various powers of 2, so we exercise all likely + # stdio buffer sizes, and "off by one" errors on both sides. + q, r = divmod(length, len(pattern)) + teststring = pattern * q + pattern[:r] + self.assertEqual(len(teststring), length) + self.try_one(teststring) + self.try_one(teststring + b"x") + self.try_one(teststring[:-1]) + + def test_primepat(self): + # A pattern with prime length, to avoid simple relationships with + # stdio buffer sizes. + self.drive_one(b"1234567890\00\01\02\03\04\05\06") + + def test_nullpat(self): + self.drive_one(b'\0' * 1000) + + +class CBufferSizeTest(BufferSizeTest, unittest.TestCase): + open = io.open + +class PyBufferSizeTest(BufferSizeTest, unittest.TestCase): + open = staticmethod(pyio.open) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_file.py b/Lib/test/test_io/test_file.py similarity index 100% rename from Lib/test/test_file.py rename to Lib/test/test_io/test_file.py diff --git a/Lib/test/test_fileio.py b/Lib/test/test_io/test_fileio.py similarity index 98% rename from Lib/test/test_fileio.py rename to Lib/test/test_io/test_fileio.py index 5a0f033ebb8..e53c4749f58 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_io/test_fileio.py @@ -567,8 +567,8 @@ def testModeStrings(self): # test that the mode attribute is correct for various mode strings # given as init args try: - for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'), - ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'), + for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'wb+'), + ('w+b', 'wb+'), ('a', 'ab'), ('ab', 'ab'), ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'), ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]: # read modes are last so that TESTFN will exist first @@ -591,7 +591,7 @@ def testBytesOpen(self): try: f.write(b"abc") f.close() - with open(TESTFN_ASCII, "rb") as f: + with self.open(TESTFN_ASCII, "rb") as f: self.assertEqual(f.read(), b"abc") finally: os.unlink(TESTFN_ASCII) @@ -608,7 +608,7 @@ def testUtf8BytesOpen(self): try: f.write(b"abc") f.close() - with open(TESTFN_UNICODE, "rb") as f: + with self.open(TESTFN_UNICODE, "rb") as f: self.assertEqual(f.read(), b"abc") finally: os.unlink(TESTFN_UNICODE) @@ -692,13 +692,13 @@ def bug801631(): def testAppend(self): try: - f = open(TESTFN, 'wb') + f = self.FileIO(TESTFN, 'wb') f.write(b'spam') f.close() - f = open(TESTFN, 'ab') + f = self.FileIO(TESTFN, 'ab') f.write(b'eggs') f.close() - f = open(TESTFN, 'rb') + f = self.FileIO(TESTFN, 'rb') d = f.read() f.close() self.assertEqual(d, b'spameggs') @@ -734,6 +734,7 @@ def __setattr__(self, name, value): class COtherFileTests(OtherFileTests, unittest.TestCase): FileIO = _io.FileIO modulename = '_io' + open = _io.open @cpython_only def testInvalidFd_overflow(self): @@ -755,6 +756,7 @@ def test_open_code(self): class PyOtherFileTests(OtherFileTests, unittest.TestCase): FileIO = _pyio.FileIO modulename = '_pyio' + open = _pyio.open def test_open_code(self): # Check that the default behaviour of open_code matches diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py new file mode 100644 index 00000000000..085ed3ea6a9 --- /dev/null +++ b/Lib/test/test_io/test_general.py @@ -0,0 +1,1431 @@ +"""General tests for the io module. + +New tests should go in more specific modules; see test_io/__init__.py +""" + +import abc +import array +import errno +import os +import pickle +import sys +import textwrap +import threading +import unittest +import warnings +import weakref +from test import support +from test.support.script_helper import ( + assert_python_ok, assert_python_failure, run_python_until_end) +from test.support import ( + import_helper, is_apple, os_helper, threading_helper, warnings_helper, +) +from test.support.os_helper import FakePath +from .utils import byteslike, CTestCase, PyTestCase + +import io # C implementation of io +import _pyio as pyio # Python implementation of io + + +class IOTest: + + def setUp(self): + os_helper.unlink(os_helper.TESTFN) + + def tearDown(self): + os_helper.unlink(os_helper.TESTFN) + + def write_ops(self, f): + self.assertEqual(f.write(b"blah."), 5) + f.truncate(0) + self.assertEqual(f.tell(), 5) + f.seek(0) + + self.assertEqual(f.write(b"blah."), 5) + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.write(b"Hello."), 6) + self.assertEqual(f.tell(), 6) + self.assertEqual(f.seek(-1, 1), 5) + self.assertEqual(f.tell(), 5) + buffer = bytearray(b" world\n\n\n") + self.assertEqual(f.write(buffer), 9) + buffer[:] = b"*" * 9 # Overwrite our copy of the data + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.write(b"h"), 1) + self.assertEqual(f.seek(-1, 2), 13) + self.assertEqual(f.tell(), 13) + + self.assertEqual(f.truncate(12), 12) + self.assertEqual(f.tell(), 13) + self.assertRaises(TypeError, f.seek, 0.0) + + def read_ops(self, f, buffered=False): + data = f.read(5) + self.assertEqual(data, b"hello") + data = byteslike(data) + self.assertEqual(f.readinto(data), 5) + self.assertEqual(bytes(data), b" worl") + data = bytearray(5) + self.assertEqual(f.readinto(data), 2) + self.assertEqual(len(data), 5) + self.assertEqual(data[:2], b"d\n") + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.read(20), b"hello world\n") + self.assertEqual(f.read(1), b"") + self.assertEqual(f.readinto(byteslike(b"x")), 0) + self.assertEqual(f.seek(-6, 2), 6) + self.assertEqual(f.read(5), b"world") + self.assertEqual(f.read(0), b"") + self.assertEqual(f.readinto(byteslike()), 0) + self.assertEqual(f.seek(-6, 1), 5) + self.assertEqual(f.read(5), b" worl") + self.assertEqual(f.tell(), 10) + self.assertRaises(TypeError, f.seek, 0.0) + if buffered: + f.seek(0) + self.assertEqual(f.read(), b"hello world\n") + f.seek(6) + self.assertEqual(f.read(), b"world\n") + self.assertEqual(f.read(), b"") + f.seek(0) + data = byteslike(5) + self.assertEqual(f.readinto1(data), 5) + self.assertEqual(bytes(data), b"hello") + + LARGE = 2**31 + + def large_file_ops(self, f): + assert f.readable() + assert f.writable() + try: + self.assertEqual(f.seek(self.LARGE), self.LARGE) + except (OverflowError, ValueError): + self.skipTest("no largefile support") + self.assertEqual(f.tell(), self.LARGE) + self.assertEqual(f.write(b"xxx"), 3) + self.assertEqual(f.tell(), self.LARGE + 3) + self.assertEqual(f.seek(-1, 1), self.LARGE + 2) + self.assertEqual(f.truncate(), self.LARGE + 2) + self.assertEqual(f.tell(), self.LARGE + 2) + self.assertEqual(f.seek(0, 2), self.LARGE + 2) + self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1) + self.assertEqual(f.tell(), self.LARGE + 2) + self.assertEqual(f.seek(0, 2), self.LARGE + 1) + self.assertEqual(f.seek(-1, 2), self.LARGE) + self.assertEqual(f.read(2), b"x") + + def test_invalid_operations(self): + # Try writing on a file opened in read mode and vice-versa. + exc = self.UnsupportedOperation + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as fp: + self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readline) + with self.open(os_helper.TESTFN, "wb") as fp: + self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readline) + with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: + self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readall) + self.assertRaises(exc, fp.readline) + with self.open(os_helper.TESTFN, "rb", buffering=0) as fp: + self.assertRaises(exc, fp.write, b"blah") + self.assertRaises(exc, fp.writelines, [b"blah\n"]) + with self.open(os_helper.TESTFN, "rb") as fp: + self.assertRaises(exc, fp.write, b"blah") + self.assertRaises(exc, fp.writelines, [b"blah\n"]) + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as fp: + self.assertRaises(exc, fp.write, "blah") + self.assertRaises(exc, fp.writelines, ["blah\n"]) + # Non-zero seeking from current or end pos + self.assertRaises(exc, fp.seek, 1, self.SEEK_CUR) + self.assertRaises(exc, fp.seek, -1, self.SEEK_END) + + @support.cpython_only + def test_startup_optimization(self): + # gh-132952: Test that `io` is not imported at startup and that the + # __module__ of UnsupportedOperation is set to "io". + assert_python_ok("-S", "-c", textwrap.dedent( + """ + import sys + assert "io" not in sys.modules + try: + sys.stdin.truncate() + except Exception as e: + typ = type(e) + assert typ.__module__ == "io", (typ, typ.__module__) + assert typ.__name__ == "UnsupportedOperation", (typ, typ.__name__) + else: + raise AssertionError("Expected UnsupportedOperation") + """ + )) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_optional_abilities(self): + # Test for OSError when optional APIs are not supported + # The purpose of this test is to try fileno(), reading, writing and + # seeking operations with various objects that indicate they do not + # support these operations. + + def pipe_reader(): + [r, w] = os.pipe() + os.close(w) # So that read() is harmless + return self.FileIO(r, "r") + + def pipe_writer(): + [r, w] = os.pipe() + self.addCleanup(os.close, r) + # Guarantee that we can write into the pipe without blocking + thread = threading.Thread(target=os.read, args=(r, 100)) + thread.start() + self.addCleanup(thread.join) + return self.FileIO(w, "w") + + def buffered_reader(): + return self.BufferedReader(self.MockUnseekableIO()) + + def buffered_writer(): + return self.BufferedWriter(self.MockUnseekableIO()) + + def buffered_random(): + return self.BufferedRandom(self.BytesIO()) + + def buffered_rw_pair(): + return self.BufferedRWPair(self.MockUnseekableIO(), + self.MockUnseekableIO()) + + def text_reader(): + class UnseekableReader(self.MockUnseekableIO): + writable = self.BufferedIOBase.writable + write = self.BufferedIOBase.write + return self.TextIOWrapper(UnseekableReader(), "ascii") + + def text_writer(): + class UnseekableWriter(self.MockUnseekableIO): + readable = self.BufferedIOBase.readable + read = self.BufferedIOBase.read + return self.TextIOWrapper(UnseekableWriter(), "ascii") + + tests = ( + (pipe_reader, "fr"), (pipe_writer, "fw"), + (buffered_reader, "r"), (buffered_writer, "w"), + (buffered_random, "rws"), (buffered_rw_pair, "rw"), + (text_reader, "r"), (text_writer, "w"), + (self.BytesIO, "rws"), (self.StringIO, "rws"), + ) + + def do_test(test, obj, abilities): + readable = "r" in abilities + self.assertEqual(obj.readable(), readable) + writable = "w" in abilities + self.assertEqual(obj.writable(), writable) + + if isinstance(obj, self.TextIOBase): + data = "3" + elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)): + data = b"3" + else: + self.fail("Unknown base class") + + if "f" in abilities: + obj.fileno() + else: + self.assertRaises(OSError, obj.fileno) + + if readable: + obj.read(1) + obj.read() + else: + self.assertRaises(OSError, obj.read, 1) + self.assertRaises(OSError, obj.read) + + if writable: + obj.write(data) + else: + self.assertRaises(OSError, obj.write, data) + + if sys.platform.startswith("win") and test in ( + pipe_reader, pipe_writer): + # Pipes seem to appear as seekable on Windows + return + seekable = "s" in abilities + self.assertEqual(obj.seekable(), seekable) + + if seekable: + obj.tell() + obj.seek(0) + else: + self.assertRaises(OSError, obj.tell) + self.assertRaises(OSError, obj.seek, 0) + + if writable and seekable: + obj.truncate() + obj.truncate(0) + else: + self.assertRaises(OSError, obj.truncate) + self.assertRaises(OSError, obj.truncate, 0) + + for [test, abilities] in tests: + with self.subTest(test): + if test == pipe_writer and not threading_helper.can_start_thread: + self.skipTest("Need threads") + with test() as obj: + do_test(test, obj, abilities) + + + def test_open_handles_NUL_chars(self): + fn_with_NUL = 'foo\0bar' + self.assertRaises(ValueError, self.open, fn_with_NUL, 'w', encoding="utf-8") + + bytes_fn = bytes(fn_with_NUL, 'ascii') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertRaises(ValueError, self.open, bytes_fn, 'w', encoding="utf-8") + + def test_raw_file_io(self): + with self.open(os_helper.TESTFN, "wb", buffering=0) as f: + self.assertEqual(f.readable(), False) + self.assertEqual(f.writable(), True) + self.assertEqual(f.seekable(), True) + self.write_ops(f) + with self.open(os_helper.TESTFN, "rb", buffering=0) as f: + self.assertEqual(f.readable(), True) + self.assertEqual(f.writable(), False) + self.assertEqual(f.seekable(), True) + self.read_ops(f) + + def test_buffered_file_io(self): + with self.open(os_helper.TESTFN, "wb") as f: + self.assertEqual(f.readable(), False) + self.assertEqual(f.writable(), True) + self.assertEqual(f.seekable(), True) + self.write_ops(f) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.readable(), True) + self.assertEqual(f.writable(), False) + self.assertEqual(f.seekable(), True) + self.read_ops(f, True) + + def test_readline(self): + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"abc\ndef\nxyzzy\nfoo\x00bar\nanother line") + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.readline(), b"abc\n") + self.assertEqual(f.readline(10), b"def\n") + self.assertEqual(f.readline(2), b"xy") + self.assertEqual(f.readline(4), b"zzy\n") + self.assertEqual(f.readline(), b"foo\x00bar\n") + self.assertEqual(f.readline(None), b"another line") + self.assertRaises(TypeError, f.readline, 5.3) + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + self.assertRaises(TypeError, f.readline, 5.3) + + def test_readline_nonsizeable(self): + # Issue #30061 + # Crash when readline() returns an object without __len__ + class R(self.IOBase): + def readline(self): + return None + self.assertRaises((TypeError, StopIteration), next, R()) + + def test_next_nonsizeable(self): + # Issue #30061 + # Crash when __next__() returns an object without __len__ + class R(self.IOBase): + def __next__(self): + return None + self.assertRaises(TypeError, R().readlines, 1) + + def test_raw_bytes_io(self): + f = self.BytesIO() + self.write_ops(f) + data = f.getvalue() + self.assertEqual(data, b"hello world\n") + f = self.BytesIO(data) + self.read_ops(f, True) + + def test_large_file_ops(self): + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: + support.requires( + 'largefile', + 'test requires %s bytes and a long time to run' % self.LARGE) + with self.open(os_helper.TESTFN, "w+b", 0) as f: + self.large_file_ops(f) + with self.open(os_helper.TESTFN, "w+b") as f: + self.large_file_ops(f) + + def test_with_open(self): + for bufsize in (0, 100): + with self.open(os_helper.TESTFN, "wb", bufsize) as f: + f.write(b"xxx") + self.assertEqual(f.closed, True) + try: + with self.open(os_helper.TESTFN, "wb", bufsize) as f: + 1/0 + except ZeroDivisionError: + self.assertEqual(f.closed, True) + else: + self.fail("1/0 didn't raise an exception") + + # issue 5008 + def test_append_mode_tell(self): + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"xxx") + with self.open(os_helper.TESTFN, "ab", buffering=0) as f: + self.assertEqual(f.tell(), 3) + with self.open(os_helper.TESTFN, "ab") as f: + self.assertEqual(f.tell(), 3) + with self.open(os_helper.TESTFN, "a", encoding="utf-8") as f: + self.assertGreater(f.tell(), 0) + + def test_destructor(self): + record = [] + class MyFileIO(self.FileIO): + def __del__(self): + record.append(1) + try: + f = super().__del__ + except AttributeError: + pass + else: + f() + def close(self): + record.append(2) + super().close() + def flush(self): + record.append(3) + super().flush() + with warnings_helper.check_warnings(('', ResourceWarning)): + f = MyFileIO(os_helper.TESTFN, "wb") + f.write(b"xxx") + del f + support.gc_collect() + self.assertEqual(record, [1, 2, 3]) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"xxx") + + def _check_base_destructor(self, base): + record = [] + class MyIO(base): + def __init__(self): + # This exercises the availability of attributes on object + # destruction. + # (in the C version, close() is called by the tp_dealloc + # function, not by __del__) + self.on_del = 1 + self.on_close = 2 + self.on_flush = 3 + def __del__(self): + record.append(self.on_del) + try: + f = super().__del__ + except AttributeError: + pass + else: + f() + def close(self): + record.append(self.on_close) + super().close() + def flush(self): + record.append(self.on_flush) + super().flush() + f = MyIO() + del f + support.gc_collect() + self.assertEqual(record, [1, 2, 3]) + + def test_IOBase_destructor(self): + self._check_base_destructor(self.IOBase) + + def test_RawIOBase_destructor(self): + self._check_base_destructor(self.RawIOBase) + + def test_BufferedIOBase_destructor(self): + self._check_base_destructor(self.BufferedIOBase) + + def test_TextIOBase_destructor(self): + self._check_base_destructor(self.TextIOBase) + + def test_close_flushes(self): + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"xxx") + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"xxx") + + def test_array_writes(self): + a = array.array('i', range(10)) + n = len(a.tobytes()) + def check(f): + with f: + self.assertEqual(f.write(a), n) + f.writelines((a,)) + check(self.BytesIO()) + check(self.FileIO(os_helper.TESTFN, "w")) + check(self.BufferedWriter(self.MockRawIO())) + check(self.BufferedRandom(self.MockRawIO())) + check(self.BufferedRWPair(self.MockRawIO(), self.MockRawIO())) + + def test_closefd(self): + self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'w', + encoding="utf-8", closefd=False) + + def test_read_closed(self): + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + f.write("egg\n") + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) + self.assertEqual(file.read(), "egg\n") + file.seek(0) + file.close() + self.assertRaises(ValueError, file.read) + with self.open(os_helper.TESTFN, "rb") as f: + file = self.open(f.fileno(), "rb", closefd=False) + self.assertEqual(file.read()[:3], b"egg") + file.close() + self.assertRaises(ValueError, file.readinto, bytearray(1)) + + def test_no_closefd_with_filename(self): + # can't use closefd in combination with a file name + self.assertRaises(ValueError, self.open, os_helper.TESTFN, "r", + encoding="utf-8", closefd=False) + + def test_closefd_attr(self): + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"egg\n") + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + self.assertEqual(f.buffer.raw.closefd, True) + file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) + self.assertEqual(file.buffer.raw.closefd, False) + + def test_garbage_collection(self): + # FileIO objects are collected, and collecting them flushes + # all data to disk. + # + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + f = self.FileIO(os_helper.TESTFN, "wb") + f.write(b"abcxxx") + f.f = f + wr = weakref.ref(f) + del f + support.gc_collect() + self.assertIsNone(wr(), wr) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"abcxxx") + + def test_unbounded_file(self): + # Issue #1174606: reading from an unbounded stream such as /dev/zero. + zero = "/dev/zero" + if not os.path.exists(zero): + self.skipTest("{0} does not exist".format(zero)) + if sys.maxsize > 0x7FFFFFFF: + self.skipTest("test can only run in a 32-bit address space") + if support.real_max_memuse < support._2G: + self.skipTest("test requires at least 2 GiB of memory") + with self.open(zero, "rb", buffering=0) as f: + self.assertRaises(OverflowError, f.read) + with self.open(zero, "rb") as f: + self.assertRaises(OverflowError, f.read) + with self.open(zero, "r") as f: + self.assertRaises(OverflowError, f.read) + + def check_flush_error_on_close(self, *args, **kwargs): + # Test that the file is closed despite failed flush + # and that flush() is called before file closed. + f = self.open(*args, **kwargs) + closed = [] + def bad_flush(): + closed[:] = [f.closed] + raise OSError() + f.flush = bad_flush + self.assertRaises(OSError, f.close) # exception not swallowed + self.assertTrue(f.closed) + self.assertTrue(closed) # flush() called + self.assertFalse(closed[0]) # flush() called before file closed + f.flush = lambda: None # break reference loop + + def test_flush_error_on_close(self): + # raw file + # Issue #5700: io.FileIO calls flush() after file closed + self.check_flush_error_on_close(os_helper.TESTFN, 'wb', buffering=0) + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'wb', buffering=0) + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'wb', buffering=0, closefd=False) + os.close(fd) + # buffered io + self.check_flush_error_on_close(os_helper.TESTFN, 'wb') + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'wb') + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'wb', closefd=False) + os.close(fd) + # text io + self.check_flush_error_on_close(os_helper.TESTFN, 'w', encoding="utf-8") + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'w', encoding="utf-8") + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'w', encoding="utf-8", closefd=False) + os.close(fd) + + def test_multi_close(self): + f = self.open(os_helper.TESTFN, "wb", buffering=0) + f.close() + f.close() + f.close() + self.assertRaises(ValueError, f.flush) + + def test_RawIOBase_read(self): + # Exercise the default limited RawIOBase.read(n) implementation (which + # calls readinto() internally). + rawio = self.MockRawIOWithoutRead((b"abc", b"d", None, b"efg", None)) + self.assertEqual(rawio.read(2), b"ab") + self.assertEqual(rawio.read(2), b"c") + self.assertEqual(rawio.read(2), b"d") + self.assertEqual(rawio.read(2), None) + self.assertEqual(rawio.read(2), b"ef") + self.assertEqual(rawio.read(2), b"g") + self.assertEqual(rawio.read(2), None) + self.assertEqual(rawio.read(2), b"") + + def test_RawIOBase_read_bounds_checking(self): + # Make sure a `.readinto` call which returns a value outside + # (0, len(buffer)) raises. + class Misbehaved(self.io.RawIOBase): + def __init__(self, readinto_return) -> None: + self._readinto_return = readinto_return + def readinto(self, b): + return self._readinto_return + + with self.assertRaises(ValueError) as cm: + Misbehaved(2).read(1) + self.assertEqual(str(cm.exception), "readinto returned 2 outside buffer size 1") + for bad_size in (2147483647, sys.maxsize, -1, -1000): + with self.assertRaises(ValueError): + Misbehaved(bad_size).read() + + def test_RawIOBase_read_gh60107(self): + # gh-60107: Ensure a "Raw I/O" which keeps a reference to the + # mutable memory doesn't allow making a mutable bytes. + class RawIOKeepsReference(self.MockRawIOWithoutRead): + def __init__(self, *args, **kwargs): + self.buf = None + super().__init__(*args, **kwargs) + + def readinto(self, buf): + # buf is the bytearray so keeping a reference to it doesn't keep + # the memory alive; a memoryview does. + self.buf = memoryview(buf) + buf[0:4] = self._read_stack.pop() + return 3 + + with self.assertRaises(BufferError): + rawio = RawIOKeepsReference([b"1234"]) + rawio.read(4) + + def test_types_have_dict(self): + test = ( + self.IOBase(), + self.RawIOBase(), + self.TextIOBase(), + self.StringIO(), + self.BytesIO() + ) + for obj in test: + self.assertHasAttr(obj, "__dict__") + + def test_opener(self): + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + f.write("egg\n") + fd = os.open(os_helper.TESTFN, os.O_RDONLY) + def opener(path, flags): + return fd + with self.open("non-existent", "r", encoding="utf-8", opener=opener) as f: + self.assertEqual(f.read(), "egg\n") + + def test_bad_opener_negative_1(self): + # Issue #27066. + def badopener(fname, flags): + return -1 + with self.assertRaises(ValueError) as cm: + self.open('non-existent', 'r', opener=badopener) + self.assertEqual(str(cm.exception), 'opener returned -1') + + def test_bad_opener_other_negative(self): + # Issue #27066. + def badopener(fname, flags): + return -2 + with self.assertRaises(ValueError) as cm: + self.open('non-existent', 'r', opener=badopener) + self.assertEqual(str(cm.exception), 'opener returned -2') + + def test_opener_invalid_fd(self): + # Check that OSError is raised with error code EBADF if the + # opener returns an invalid file descriptor (see gh-82212). + fd = os_helper.make_bad_fd() + with self.assertRaises(OSError) as cm: + self.open('foo', opener=lambda name, flags: fd) + self.assertEqual(cm.exception.errno, errno.EBADF) + + def test_fileio_closefd(self): + # Issue #4841 + with self.open(__file__, 'rb') as f1, \ + self.open(__file__, 'rb') as f2: + fileio = self.FileIO(f1.fileno(), closefd=False) + # .__init__() must not close f1 + fileio.__init__(f2.fileno(), closefd=False) + f1.readline() + # .close() must not close f2 + fileio.close() + f2.readline() + + def test_nonbuffered_textio(self): + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError): + self.open(os_helper.TESTFN, 'w', encoding="utf-8", buffering=0) + + def test_invalid_newline(self): + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError): + self.open(os_helper.TESTFN, 'w', encoding="utf-8", newline='invalid') + + def test_buffered_readinto_mixin(self): + # Test the implementation provided by BufferedIOBase + class Stream(self.BufferedIOBase): + def read(self, size): + return b"12345" + read1 = read + stream = Stream() + for method in ("readinto", "readinto1"): + with self.subTest(method): + buffer = byteslike(5) + self.assertEqual(getattr(stream, method)(buffer), 5) + self.assertEqual(bytes(buffer), b"12345") + + def test_fspath_support(self): + def check_path_succeeds(path): + with self.open(path, "w", encoding="utf-8") as f: + f.write("egg\n") + + with self.open(path, "r", encoding="utf-8") as f: + self.assertEqual(f.read(), "egg\n") + + check_path_succeeds(FakePath(os_helper.TESTFN)) + check_path_succeeds(FakePath(os.fsencode(os_helper.TESTFN))) + + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + bad_path = FakePath(f.fileno()) + with self.assertRaises(TypeError): + self.open(bad_path, 'w', encoding="utf-8") + + bad_path = FakePath(None) + with self.assertRaises(TypeError): + self.open(bad_path, 'w', encoding="utf-8") + + bad_path = FakePath(FloatingPointError) + with self.assertRaises(FloatingPointError): + self.open(bad_path, 'w', encoding="utf-8") + + # ensure that refcounting is correct with some error conditions + with self.assertRaisesRegex(ValueError, 'read/write/append mode'): + self.open(FakePath(os_helper.TESTFN), 'rwxa', encoding="utf-8") + + def test_RawIOBase_readall(self): + # Exercise the default unlimited RawIOBase.read() and readall() + # implementations. + rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg")) + self.assertEqual(rawio.read(), b"abcdefg") + rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg")) + self.assertEqual(rawio.readall(), b"abcdefg") + + def test_BufferedIOBase_readinto(self): + # Exercise the default BufferedIOBase.readinto() and readinto1() + # implementations (which call read() or read1() internally). + class Reader(self.BufferedIOBase): + def __init__(self, avail): + self.avail = avail + def read(self, size): + result = self.avail[:size] + self.avail = self.avail[size:] + return result + def read1(self, size): + """Returns no more than 5 bytes at once""" + return self.read(min(size, 5)) + tests = ( + # (test method, total data available, read buffer size, expected + # read size) + ("readinto", 10, 5, 5), + ("readinto", 10, 6, 6), # More than read1() can return + ("readinto", 5, 6, 5), # Buffer larger than total available + ("readinto", 6, 7, 6), + ("readinto", 10, 0, 0), # Empty buffer + ("readinto1", 10, 5, 5), # Result limited to single read1() call + ("readinto1", 10, 6, 5), # Buffer larger than read1() can return + ("readinto1", 5, 6, 5), # Buffer larger than total available + ("readinto1", 6, 7, 5), + ("readinto1", 10, 0, 0), # Empty buffer + ) + UNUSED_BYTE = 0x81 + for test in tests: + with self.subTest(test): + method, avail, request, result = test + reader = Reader(bytes(range(avail))) + buffer = bytearray((UNUSED_BYTE,) * request) + method = getattr(reader, method) + self.assertEqual(method(buffer), result) + self.assertEqual(len(buffer), request) + self.assertSequenceEqual(buffer[:result], range(result)) + unused = (UNUSED_BYTE,) * (request - result) + self.assertSequenceEqual(buffer[result:], unused) + self.assertEqual(len(reader.avail), avail - result) + + def test_close_assert(self): + class R(self.IOBase): + def __setattr__(self, name, value): + pass + def flush(self): + raise OSError() + f = R() + # This would cause an assertion failure. + self.assertRaises(OSError, f.close) + + # Silence destructor error + R.flush = lambda self: None + + @threading_helper.requires_working_threading() + def test_write_readline_races(self): + # gh-134908: Concurrent iteration over a file caused races + thread_count = 2 + write_count = 100 + read_count = 100 + + def writer(file, barrier): + barrier.wait() + for _ in range(write_count): + file.write("x") + + def reader(file, barrier): + barrier.wait() + for _ in range(read_count): + for line in file: + self.assertEqual(line, "") + + with self.open(os_helper.TESTFN, "w+") as f: + barrier = threading.Barrier(thread_count + 1) + reader = threading.Thread(target=reader, args=(f, barrier)) + writers = [threading.Thread(target=writer, args=(f, barrier)) + for _ in range(thread_count)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(writers + [reader]): + pass + self.assertIsNone(cm.exc_type) + + self.assertEqual(os.stat(os_helper.TESTFN).st_size, + write_count * thread_count) + + +class CIOTest(IOTest, CTestCase): + + def test_IOBase_finalize(self): + # Issue #12149: segmentation fault on _PyIOBase_finalize when both a + # class which inherits IOBase and an object of this class are caught + # in a reference cycle and close() is already in the method cache. + class MyIO(self.IOBase): + def close(self): + pass + + # create an instance to populate the method cache + MyIO() + obj = MyIO() + obj.obj = obj + wr = weakref.ref(obj) + del MyIO + del obj + support.gc_collect() + self.assertIsNone(wr(), wr) + +@support.cpython_only +class TestIOCTypes(unittest.TestCase): + def setUp(self): + _io = import_helper.import_module("_io") + self.types = [ + _io.BufferedRWPair, + _io.BufferedRandom, + _io.BufferedReader, + _io.BufferedWriter, + _io.BytesIO, + _io.FileIO, + _io.IncrementalNewlineDecoder, + _io.StringIO, + _io.TextIOWrapper, + _io._BufferedIOBase, + _io._BytesIOBuffer, + _io._IOBase, + _io._RawIOBase, + _io._TextIOBase, + ] + if sys.platform == "win32": + self.types.append(_io._WindowsConsoleIO) + self._io = _io + + def test_immutable_types(self): + for tp in self.types: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = "bar" + + def test_class_hierarchy(self): + def check_subs(types, base): + for tp in types: + with self.subTest(tp=tp, base=base): + self.assertIsSubclass(tp, base) + + def recursive_check(d): + for k, v in d.items(): + if isinstance(v, dict): + recursive_check(v) + elif isinstance(v, set): + check_subs(v, k) + else: + self.fail("corrupt test dataset") + + _io = self._io + hierarchy = { + _io._IOBase: { + _io._BufferedIOBase: { + _io.BufferedRWPair, + _io.BufferedRandom, + _io.BufferedReader, + _io.BufferedWriter, + _io.BytesIO, + }, + _io._RawIOBase: { + _io.FileIO, + }, + _io._TextIOBase: { + _io.StringIO, + _io.TextIOWrapper, + }, + }, + } + if sys.platform == "win32": + hierarchy[_io._IOBase][_io._RawIOBase].add(_io._WindowsConsoleIO) + + recursive_check(hierarchy) + + def test_subclassing(self): + _io = self._io + dataset = {k: True for k in self.types} + dataset[_io._BytesIOBuffer] = False + + for tp, is_basetype in dataset.items(): + with self.subTest(tp=tp, is_basetype=is_basetype): + name = f"{tp.__name__}_subclass" + bases = (tp,) + if is_basetype: + _ = type(name, bases, {}) + else: + msg = "not an acceptable base type" + with self.assertRaisesRegex(TypeError, msg): + _ = type(name, bases, {}) + + def test_disallow_instantiation(self): + _io = self._io + support.check_disallow_instantiation(self, _io._BytesIOBuffer) + + def test_stringio_setstate(self): + # gh-127182: Calling __setstate__() with invalid arguments must not crash + obj = self._io.StringIO() + with self.assertRaisesRegex( + TypeError, + 'initial_value must be str or None, not int', + ): + obj.__setstate__((1, '', 0, {})) + + obj.__setstate__((None, '', 0, {})) # should not crash + self.assertEqual(obj.getvalue(), '') + + obj.__setstate__(('', '', 0, {})) + self.assertEqual(obj.getvalue(), '') + +class PyIOTest(IOTest, PyTestCase): + pass + + +@support.cpython_only +class APIMismatchTest(unittest.TestCase): + + def test_RawIOBase_io_in_pyio_match(self): + """Test that pyio RawIOBase class has all c RawIOBase methods""" + mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, + ignore=('__weakref__', '__static_attributes__')) + self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') + + def test_RawIOBase_pyio_in_io_match(self): + """Test that c RawIOBase class has all pyio RawIOBase methods""" + mismatch = support.detect_api_mismatch(io.RawIOBase, pyio.RawIOBase) + self.assertEqual(mismatch, set(), msg='C RawIOBase does not have all Python RawIOBase methods') + + +# XXX Tests for open() + +class MiscIOTest: + + # for test__all__, actual values are set in subclasses + name_of_module = None + extra_exported = () + not_exported = () + + def tearDown(self): + os_helper.unlink(os_helper.TESTFN) + + def test___all__(self): + support.check__all__(self, self.io, self.name_of_module, + extra=self.extra_exported, + not_exported=self.not_exported) + + def test_attributes(self): + f = self.open(os_helper.TESTFN, "wb", buffering=0) + self.assertEqual(f.mode, "wb") + f.close() + + f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") + self.assertEqual(f.mode, "w+") + self.assertEqual(f.buffer.mode, "wb+") + self.assertEqual(f.buffer.raw.mode, "wb+") + + g = self.open(f.fileno(), "wb", closefd=False) + self.assertEqual(g.mode, "wb") + self.assertEqual(g.raw.mode, "wb") + self.assertEqual(g.name, f.fileno()) + self.assertEqual(g.raw.name, f.fileno()) + f.close() + g.close() + + def test_removed_u_mode(self): + # bpo-37330: The "U" mode has been removed in Python 3.11 + for mode in ("U", "rU", "r+U"): + with self.assertRaises(ValueError) as cm: + self.open(os_helper.TESTFN, mode) + self.assertIn('invalid mode', str(cm.exception)) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_open_pipe_with_append(self): + # bpo-27805: Ignore ESPIPE from lseek() in open(). + r, w = os.pipe() + self.addCleanup(os.close, r) + f = self.open(w, 'a', encoding="utf-8") + self.addCleanup(f.close) + # Check that the file is marked non-seekable. On Windows, however, lseek + # somehow succeeds on pipes. + if sys.platform != 'win32': + self.assertFalse(f.seekable()) + + def test_io_after_close(self): + for kwargs in [ + {"mode": "w"}, + {"mode": "wb"}, + {"mode": "w", "buffering": 1}, + {"mode": "w", "buffering": 2}, + {"mode": "wb", "buffering": 0}, + {"mode": "r"}, + {"mode": "rb"}, + {"mode": "r", "buffering": 1}, + {"mode": "r", "buffering": 2}, + {"mode": "rb", "buffering": 0}, + {"mode": "w+"}, + {"mode": "w+b"}, + {"mode": "w+", "buffering": 1}, + {"mode": "w+", "buffering": 2}, + {"mode": "w+b", "buffering": 0}, + ]: + if "b" not in kwargs["mode"]: + kwargs["encoding"] = "utf-8" + f = self.open(os_helper.TESTFN, **kwargs) + f.close() + self.assertRaises(ValueError, f.flush) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.isatty) + self.assertRaises(ValueError, f.__iter__) + if hasattr(f, "peek"): + self.assertRaises(ValueError, f.peek, 1) + self.assertRaises(ValueError, f.read) + if hasattr(f, "read1"): + self.assertRaises(ValueError, f.read1, 1024) + self.assertRaises(ValueError, f.read1) + if hasattr(f, "readall"): + self.assertRaises(ValueError, f.readall) + if hasattr(f, "readinto"): + self.assertRaises(ValueError, f.readinto, bytearray(1024)) + if hasattr(f, "readinto1"): + self.assertRaises(ValueError, f.readinto1, bytearray(1024)) + self.assertRaises(ValueError, f.readline) + self.assertRaises(ValueError, f.readlines) + self.assertRaises(ValueError, f.readlines, 1) + self.assertRaises(ValueError, f.seek, 0) + self.assertRaises(ValueError, f.tell) + self.assertRaises(ValueError, f.truncate) + self.assertRaises(ValueError, f.write, + b"" if "b" in kwargs['mode'] else "") + self.assertRaises(ValueError, f.writelines, []) + self.assertRaises(ValueError, next, f) + + def test_blockingioerror(self): + # Various BlockingIOError issues + class C(str): + pass + c = C("") + b = self.BlockingIOError(1, c) + c.b = b + b.c = c + wr = weakref.ref(c) + del c, b + support.gc_collect() + self.assertIsNone(wr(), wr) + + def test_abcs(self): + # Test the visible base classes are ABCs. + self.assertIsInstance(self.IOBase, abc.ABCMeta) + self.assertIsInstance(self.RawIOBase, abc.ABCMeta) + self.assertIsInstance(self.BufferedIOBase, abc.ABCMeta) + self.assertIsInstance(self.TextIOBase, abc.ABCMeta) + + def _check_abc_inheritance(self, abcmodule): + with self.open(os_helper.TESTFN, "wb", buffering=0) as f: + self.assertIsInstance(f, abcmodule.IOBase) + self.assertIsInstance(f, abcmodule.RawIOBase) + self.assertNotIsInstance(f, abcmodule.BufferedIOBase) + self.assertNotIsInstance(f, abcmodule.TextIOBase) + with self.open(os_helper.TESTFN, "wb") as f: + self.assertIsInstance(f, abcmodule.IOBase) + self.assertNotIsInstance(f, abcmodule.RawIOBase) + self.assertIsInstance(f, abcmodule.BufferedIOBase) + self.assertNotIsInstance(f, abcmodule.TextIOBase) + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + self.assertIsInstance(f, abcmodule.IOBase) + self.assertNotIsInstance(f, abcmodule.RawIOBase) + self.assertNotIsInstance(f, abcmodule.BufferedIOBase) + self.assertIsInstance(f, abcmodule.TextIOBase) + + def test_abc_inheritance(self): + # Test implementations inherit from their respective ABCs + self._check_abc_inheritance(self) + + def test_abc_inheritance_official(self): + # Test implementations inherit from the official ABCs of the + # baseline "io" module. + self._check_abc_inheritance(io) + + def _check_warn_on_dealloc(self, *args, **kwargs): + f = self.open(*args, **kwargs) + r = repr(f) + with self.assertWarns(ResourceWarning) as cm: + f = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_warn_on_dealloc(self): + self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0) + self._check_warn_on_dealloc(os_helper.TESTFN, "wb") + self._check_warn_on_dealloc(os_helper.TESTFN, "w", encoding="utf-8") + + def _check_warn_on_dealloc_fd(self, *args, **kwargs): + fds = [] + def cleanup_fds(): + for fd in fds: + try: + os.close(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + self.addCleanup(cleanup_fds) + r, w = os.pipe() + fds += r, w + self._check_warn_on_dealloc(r, *args, **kwargs) + # When using closefd=False, there's no warning + r, w = os.pipe() + fds += r, w + with warnings_helper.check_no_resource_warning(self): + self.open(r, *args, closefd=False, **kwargs) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_warn_on_dealloc_fd(self): + self._check_warn_on_dealloc_fd("rb", buffering=0) + self._check_warn_on_dealloc_fd("rb") + self._check_warn_on_dealloc_fd("r", encoding="utf-8") + + + def test_pickling(self): + # Pickling file objects is forbidden + msg = "cannot pickle" + for kwargs in [ + {"mode": "w"}, + {"mode": "wb"}, + {"mode": "wb", "buffering": 0}, + {"mode": "r"}, + {"mode": "rb"}, + {"mode": "rb", "buffering": 0}, + {"mode": "w+"}, + {"mode": "w+b"}, + {"mode": "w+b", "buffering": 0}, + ]: + if "b" not in kwargs["mode"]: + kwargs["encoding"] = "utf-8" + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol, kwargs=kwargs): + with self.open(os_helper.TESTFN, **kwargs) as f: + with self.assertRaisesRegex(TypeError, msg): + pickle.dumps(f, protocol) + + @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") + def test_nonblock_pipe_write_bigbuf(self): + self._test_nonblock_pipe_write(16*1024) + + @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") + def test_nonblock_pipe_write_smallbuf(self): + self._test_nonblock_pipe_write(1024) + + @unittest.skipUnless(hasattr(os, 'set_blocking'), + 'os.set_blocking() required for this test') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def _test_nonblock_pipe_write(self, bufsize): + sent = [] + received = [] + r, w = os.pipe() + os.set_blocking(r, False) + os.set_blocking(w, False) + + # To exercise all code paths in the C implementation we need + # to play with buffer sizes. For instance, if we choose a + # buffer size less than or equal to _PIPE_BUF (4096 on Linux) + # then we will never get a partial write of the buffer. + rf = self.open(r, mode='rb', closefd=True, buffering=bufsize) + wf = self.open(w, mode='wb', closefd=True, buffering=bufsize) + + with rf, wf: + for N in 9999, 73, 7574: + try: + i = 0 + while True: + msg = bytes([i % 26 + 97]) * N + sent.append(msg) + wf.write(msg) + i += 1 + + except self.BlockingIOError as e: + self.assertEqual(e.args[0], errno.EAGAIN) + self.assertEqual(e.args[2], e.characters_written) + sent[-1] = sent[-1][:e.characters_written] + received.append(rf.read()) + msg = b'BLOCKED' + wf.write(msg) + sent.append(msg) + + while True: + try: + wf.flush() + break + except self.BlockingIOError as e: + self.assertEqual(e.args[0], errno.EAGAIN) + self.assertEqual(e.args[2], e.characters_written) + self.assertEqual(e.characters_written, 0) + received.append(rf.read()) + + received += iter(rf.read, None) + + sent, received = b''.join(sent), b''.join(received) + self.assertEqual(sent, received) + self.assertTrue(wf.closed) + self.assertTrue(rf.closed) + + def test_create_fail(self): + # 'x' mode fails if file is existing + with self.open(os_helper.TESTFN, 'w', encoding="utf-8"): + pass + self.assertRaises(FileExistsError, self.open, os_helper.TESTFN, 'x', encoding="utf-8") + + def test_create_writes(self): + # 'x' mode opens for writing + with self.open(os_helper.TESTFN, 'xb') as f: + f.write(b"spam") + with self.open(os_helper.TESTFN, 'rb') as f: + self.assertEqual(b"spam", f.read()) + + def test_open_allargs(self): + # there used to be a buffer overflow in the parser for rawmode + self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'rwax+', encoding="utf-8") + + def test_check_encoding_errors(self): + # bpo-37388: open() and TextIOWrapper must check encoding and errors + # arguments in dev mode + mod = self.io.__name__ + filename = __file__ + invalid = 'Boom, Shaka Laka, Boom!' + code = textwrap.dedent(f''' + import sys + from {mod} import open, TextIOWrapper + + try: + open({filename!r}, encoding={invalid!r}) + except LookupError: + pass + else: + sys.exit(21) + + try: + open({filename!r}, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(22) + + fp = open({filename!r}, "rb") + with fp: + try: + TextIOWrapper(fp, encoding={invalid!r}) + except LookupError: + pass + else: + sys.exit(23) + + try: + TextIOWrapper(fp, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(24) + + sys.exit(10) + ''') + proc = assert_python_failure('-X', 'dev', '-c', code) + self.assertEqual(proc.rc, 10, proc) + + def test_check_encoding_warning(self): + # PEP 597: Raise warning when encoding is not specified + # and sys.flags.warn_default_encoding is set. + mod = self.io.__name__ + filename = __file__ + code = textwrap.dedent(f'''\ + import sys + from {mod} import open, TextIOWrapper + import pathlib + + with open({filename!r}) as f: # line 5 + pass + + pathlib.Path({filename!r}).read_text() # line 8 + ''') + proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) + warnings = proc.err.splitlines() + self.assertEqual(len(warnings), 2) + self.assertStartsWith(warnings[0], b"<string>:5: EncodingWarning: ") + self.assertStartsWith(warnings[1], b"<string>:8: EncodingWarning: ") + + def test_text_encoding(self): + # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" + # based on sys.flags.utf8_mode + code = "import io; print(io.text_encoding(None))" + + proc = assert_python_ok('-X', 'utf8=0', '-c', code) + self.assertEqual(b"locale", proc.out.strip()) + + proc = assert_python_ok('-X', 'utf8=1', '-c', code) + self.assertEqual(b"utf-8", proc.out.strip()) + + +class CMiscIOTest(MiscIOTest, CTestCase): + name_of_module = "io", "_io" + extra_exported = "BlockingIOError", + + def test_readinto_buffer_overflow(self): + # Issue #18025 + class BadReader(self.io.BufferedIOBase): + def read(self, n=-1): + return b'x' * 10**6 + bufio = BadReader() + b = bytearray(2) + self.assertRaises(ValueError, bufio.readinto, b) + + def check_daemon_threads_shutdown_deadlock(self, stream_name): + # Issue #23309: deadlocks at shutdown should be avoided when a + # daemon thread and the main thread both write to a file. + code = """if 1: + import sys + import time + import threading + from test.support import SuppressCrashReport + + file = sys.{stream_name} + + def run(): + while True: + file.write('.') + file.flush() + + crash = SuppressCrashReport() + crash.__enter__() + # don't call __exit__(): the crash occurs at Python shutdown + + thread = threading.Thread(target=run) + thread.daemon = True + thread.start() + + time.sleep(0.5) + file.write('!') + file.flush() + """.format_map(locals()) + res, _ = run_python_until_end("-c", code) + err = res.err.decode() + if res.rc != 0: + # Failure: should be a fatal error + pattern = (r"Fatal Python error: _enter_buffered_busy: " + r"could not acquire lock " + r"for <(_io\.)?BufferedWriter name='<{stream_name}>'> " + r"at interpreter shutdown, possibly due to " + r"daemon threads".format_map(locals())) + self.assertRegex(err, pattern) + else: + self.assertFalse(err.strip('.!')) + + @threading_helper.requires_working_threading() + @support.requires_resource('walltime') + def test_daemon_threads_shutdown_stdout_deadlock(self): + self.check_daemon_threads_shutdown_deadlock('stdout') + + @threading_helper.requires_working_threading() + @support.requires_resource('walltime') + def test_daemon_threads_shutdown_stderr_deadlock(self): + self.check_daemon_threads_shutdown_deadlock('stderr') + + +class PyMiscIOTest(MiscIOTest, PyTestCase): + name_of_module = "_pyio", "io" + extra_exported = "BlockingIOError", "open_code", + not_exported = "valid_seek_flags", + + +class ProtocolsTest(unittest.TestCase): + class MyReader: + def read(self, sz=-1): + return b"" + + class MyWriter: + def write(self, b: bytes): + pass + + def test_reader_subclass(self): + self.assertIsSubclass(self.MyReader, io.Reader) + self.assertNotIsSubclass(str, io.Reader) + + def test_writer_subclass(self): + self.assertIsSubclass(self.MyWriter, io.Writer) + self.assertNotIsSubclass(str, io.Writer) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_largefile.py b/Lib/test/test_io/test_largefile.py similarity index 98% rename from Lib/test/test_largefile.py rename to Lib/test/test_io/test_largefile.py index 41f7b70e5cf..438a90a92ed 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_io/test_largefile.py @@ -56,9 +56,7 @@ class TestFileMethods(LargeFileTest): (i.e. > 2 GiB) files. """ - # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, - # so memuse=2 is needed - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: @@ -154,7 +152,7 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_seek_readall(self, _size): # Seek which doesn't change position should readall successfully. with self.open(TESTFN, 'rb') as f: diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_io/test_memoryio.py similarity index 98% rename from Lib/test/test_memoryio.py rename to Lib/test/test_io/test_memoryio.py index 95629ed862d..bb023735e21 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -54,6 +54,12 @@ def testSeek(self): self.assertEqual(buf[3:], bytesIo.read()) self.assertRaises(TypeError, bytesIo.seek, 0.0) + self.assertEqual(sys.maxsize, bytesIo.seek(sys.maxsize)) + self.assertEqual(self.EOF, bytesIo.read(4)) + + self.assertEqual(sys.maxsize - 2, bytesIo.seek(sys.maxsize - 2)) + self.assertEqual(self.EOF, bytesIo.read(4)) + def testTell(self): buf = self.buftype("1234567890") bytesIo = self.ioclass(buf) @@ -265,8 +271,8 @@ def test_iterator(self): memio = self.ioclass(buf * 10) self.assertEqual(iter(memio), memio) - self.assertTrue(hasattr(memio, '__iter__')) - self.assertTrue(hasattr(memio, '__next__')) + self.assertHasAttr(memio, '__iter__') + self.assertHasAttr(memio, '__next__') i = 0 for line in memio: self.assertEqual(line, buf) @@ -552,6 +558,14 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_issue141311(self): + memio = self.ioclass() + # Seek allows PY_SSIZE_T_MAX, read should handle that. + # Past end of buffer read should always return 0 (EOF). + self.assertEqual(sys.maxsize, memio.seek(sys.maxsize)) + buf = bytearray(2) + self.assertEqual(0, memio.readinto(buf)) + def test_unicode(self): memio = self.ioclass() diff --git a/Lib/test/test_io/test_signals.py b/Lib/test/test_io/test_signals.py new file mode 100644 index 00000000000..03f1da1eb1c --- /dev/null +++ b/Lib/test/test_io/test_signals.py @@ -0,0 +1,280 @@ +import errno +import os +import signal +import threading +import unittest +from test import support +from test.support import import_helper +from .utils import PyTestCase, CTestCase + + +requires_alarm = unittest.skipUnless( + hasattr(signal, "alarm"), "test requires signal.alarm()" +) + + +@unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.') +class SignalsTest: + + def setUp(self): + self.oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt) + + def tearDown(self): + signal.signal(signal.SIGALRM, self.oldalrm) + + def alarm_interrupt(self, sig, frame): + 1/0 + + def check_interrupted_write(self, item, bytes, **fdopen_kwargs): + """Check that a partial write, when it gets interrupted, properly + invokes the signal handler, and bubbles up the exception raised + in the latter.""" + + # XXX This test has three flaws that appear when objects are + # XXX not reference counted. + + # - if wio.write() happens to trigger a garbage collection, + # the signal exception may be raised when some __del__ + # method is running; it will not reach the assertRaises() + # call. + + # - more subtle, if the wio object is not destroyed at once + # and survives this function, the next opened file is likely + # to have the same fileno (since the file descriptor was + # actively closed). When wio.__del__ is finally called, it + # will close the other's test file... To trigger this with + # CPython, try adding "global wio" in this function. + + # - This happens only for streams created by the _pyio module, + # because a wio.close() that fails still consider that the + # file needs to be closed again. You can try adding an + # "assert wio.closed" at the end of the function. + + # Fortunately, a little gc.collect() seems to be enough to + # work around all these issues. + support.gc_collect() # For PyPy or other GCs. + + read_results = [] + def _read(): + s = os.read(r, 1) + read_results.append(s) + + t = threading.Thread(target=_read) + t.daemon = True + r, w = os.pipe() + fdopen_kwargs["closefd"] = False + large_data = item * (support.PIPE_MAX_SIZE // len(item) + 1) + try: + wio = self.io.open(w, **fdopen_kwargs) + if hasattr(signal, 'pthread_sigmask'): + # create the thread with SIGALRM signal blocked + signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM]) + t.start() + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGALRM]) + else: + t.start() + + # Fill the pipe enough that the write will be blocking. + # It will be interrupted by the timer armed above. Since the + # other thread has read one byte, the low-level write will + # return with a successful (partial) result rather than an EINTR. + # The buffered IO layer must check for pending signal + # handlers, which in this case will invoke alarm_interrupt(). + signal.alarm(1) + try: + self.assertRaises(ZeroDivisionError, wio.write, large_data) + finally: + signal.alarm(0) + t.join() + # We got one byte, get another one and check that it isn't a + # repeat of the first one. + read_results.append(os.read(r, 1)) + self.assertEqual(read_results, [bytes[0:1], bytes[1:2]]) + finally: + os.close(w) + os.close(r) + # This is deliberate. If we didn't close the file descriptor + # before closing wio, wio would try to flush its internal + # buffer, and block again. + try: + wio.close() + except OSError as e: + if e.errno != errno.EBADF: + raise + + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_interrupted_write_unbuffered(self): + self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0) + + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_interrupted_write_buffered(self): + self.check_interrupted_write(b"xy", b"xy", mode="wb") + + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_interrupted_write_text(self): + self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") + + @support.no_tracing + def check_reentrant_write(self, data, **fdopen_kwargs): + def on_alarm(*args): + # Will be called reentrantly from the same thread + wio.write(data) + 1/0 + signal.signal(signal.SIGALRM, on_alarm) + r, w = os.pipe() + wio = self.io.open(w, **fdopen_kwargs) + try: + signal.alarm(1) + # Either the reentrant call to wio.write() fails with RuntimeError, + # or the signal handler raises ZeroDivisionError. + with self.assertRaises((ZeroDivisionError, RuntimeError)) as cm: + while 1: + for i in range(100): + wio.write(data) + wio.flush() + # Make sure the buffer doesn't fill up and block further writes + os.read(r, len(data) * 100) + exc = cm.exception + if isinstance(exc, RuntimeError): + self.assertStartsWith(str(exc), "reentrant call") + finally: + signal.alarm(0) + wio.close() + os.close(r) + + @requires_alarm + def test_reentrant_write_buffered(self): + self.check_reentrant_write(b"xy", mode="wb") + + @requires_alarm + def test_reentrant_write_text(self): + self.check_reentrant_write("xy", mode="w", encoding="ascii") + + def check_interrupted_read_retry(self, decode, **fdopen_kwargs): + """Check that a buffered read, when it gets interrupted (either + returning a partial result or EINTR), properly invokes the signal + handler and retries if the latter returned successfully.""" + r, w = os.pipe() + fdopen_kwargs["closefd"] = False + def alarm_handler(sig, frame): + os.write(w, b"bar") + signal.signal(signal.SIGALRM, alarm_handler) + try: + rio = self.io.open(r, **fdopen_kwargs) + os.write(w, b"foo") + signal.alarm(1) + # Expected behaviour: + # - first raw read() returns partial b"foo" + # - second raw read() returns EINTR + # - third raw read() returns b"bar" + self.assertEqual(decode(rio.read(6)), "foobar") + finally: + signal.alarm(0) + rio.close() + os.close(w) + os.close(r) + + @requires_alarm + @support.requires_resource('walltime') + def test_interrupted_read_retry_buffered(self): + self.check_interrupted_read_retry(lambda x: x.decode('latin1'), + mode="rb") + + @requires_alarm + @support.requires_resource('walltime') + def test_interrupted_read_retry_text(self): + self.check_interrupted_read_retry(lambda x: x, + mode="r", encoding="latin1") + + def check_interrupted_write_retry(self, item, **fdopen_kwargs): + """Check that a buffered write, when it gets interrupted (either + returning a partial result or EINTR), properly invokes the signal + handler and retries if the latter returned successfully.""" + select = import_helper.import_module("select") + + # A quantity that exceeds the buffer size of an anonymous pipe's + # write end. + N = support.PIPE_MAX_SIZE + r, w = os.pipe() + fdopen_kwargs["closefd"] = False + + # We need a separate thread to read from the pipe and allow the + # write() to finish. This thread is started after the SIGALRM is + # received (forcing a first EINTR in write()). + read_results = [] + write_finished = False + error = None + def _read(): + try: + while not write_finished: + while r in select.select([r], [], [], 1.0)[0]: + s = os.read(r, 1024) + read_results.append(s) + except BaseException as exc: + nonlocal error + error = exc + t = threading.Thread(target=_read) + t.daemon = True + def alarm1(sig, frame): + signal.signal(signal.SIGALRM, alarm2) + signal.alarm(1) + def alarm2(sig, frame): + t.start() + + large_data = item * N + signal.signal(signal.SIGALRM, alarm1) + try: + wio = self.io.open(w, **fdopen_kwargs) + signal.alarm(1) + # Expected behaviour: + # - first raw write() is partial (because of the limited pipe buffer + # and the first alarm) + # - second raw write() returns EINTR (because of the second alarm) + # - subsequent write()s are successful (either partial or complete) + written = wio.write(large_data) + self.assertEqual(N, written) + + wio.flush() + write_finished = True + t.join() + + self.assertIsNone(error) + self.assertEqual(N, sum(len(x) for x in read_results)) + finally: + signal.alarm(0) + write_finished = True + os.close(w) + os.close(r) + # This is deliberate. If we didn't close the file descriptor + # before closing wio, wio would try to flush its internal + # buffer, and could block (in case of failure). + try: + wio.close() + except OSError as e: + if e.errno != errno.EBADF: + raise + + @requires_alarm + @support.requires_resource('walltime') + def test_interrupted_write_retry_buffered(self): + self.check_interrupted_write_retry(b"x", mode="wb") + + @requires_alarm + @support.requires_resource('walltime') + def test_interrupted_write_retry_text(self): + self.check_interrupted_write_retry("x", mode="w", encoding="latin1") + + +class CSignalsTest(SignalsTest, CTestCase): + pass + +class PySignalsTest(SignalsTest, PyTestCase): + pass + + # Handling reentrancy issues would slow down _pyio even more, so the + # tests are disabled. + test_reentrant_write_buffered = None + test_reentrant_write_text = None diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py new file mode 100644 index 00000000000..6331ed2b958 --- /dev/null +++ b/Lib/test/test_io/test_textio.py @@ -0,0 +1,1683 @@ +import array +import codecs +import locale +import os +import pickle +import sys +import threading +import time +import unittest +import warnings +import weakref +from collections import UserList +from test import support +from test.support import os_helper, threading_helper +from test.support.script_helper import assert_python_ok +from .utils import CTestCase, PyTestCase + +import io # C implementation of io +import _pyio as pyio # Python implementation of io + + +def _default_chunk_size(): + """Get the default TextIOWrapper chunk size""" + with open(__file__, "r", encoding="latin-1") as f: + return f._CHUNK_SIZE + + +class BadIndex: + def __index__(self): + 1/0 + + +# To fully exercise seek/tell, the StatefulIncrementalDecoder has these +# properties: +# - A single output character can correspond to many bytes of input. +# - The number of input bytes to complete the character can be +# undetermined until the last input byte is received. +# - The number of input bytes can vary depending on previous input. +# - A single input byte can correspond to many characters of output. +# - The number of output characters can be undetermined until the +# last input byte is received. +# - The number of output characters can vary depending on previous input. + +class StatefulIncrementalDecoder(codecs.IncrementalDecoder): + """ + For testing seek/tell behavior with a stateful, buffering decoder. + + Input is a sequence of words. Words may be fixed-length (length set + by input) or variable-length (period-terminated). In variable-length + mode, extra periods are ignored. Possible words are: + - 'i' followed by a number sets the input length, I (maximum 99). + When I is set to 0, words are space-terminated. + - 'o' followed by a number sets the output length, O (maximum 99). + - Any other word is converted into a word followed by a period on + the output. The output word consists of the input word truncated + or padded out with hyphens to make its length equal to O. If O + is 0, the word is output verbatim without truncating or padding. + I and O are initially set to 1. When I changes, any buffered input is + re-scanned according to the new I. EOF also terminates the last word. + """ + + def __init__(self, errors='strict'): + codecs.IncrementalDecoder.__init__(self, errors) + self.reset() + + def __repr__(self): + return '<SID %x>' % id(self) + + def reset(self): + self.i = 1 + self.o = 1 + self.buffer = bytearray() + + def getstate(self): + i, o = self.i ^ 1, self.o ^ 1 # so that flags = 0 after reset() + return bytes(self.buffer), i*100 + o + + def setstate(self, state): + buffer, io = state + self.buffer = bytearray(buffer) + i, o = divmod(io, 100) + self.i, self.o = i ^ 1, o ^ 1 + + def decode(self, input, final=False): + output = '' + for b in input: + if self.i == 0: # variable-length, terminated with period + if b == ord('.'): + if self.buffer: + output += self.process_word() + else: + self.buffer.append(b) + else: # fixed-length, terminate after self.i bytes + self.buffer.append(b) + if len(self.buffer) == self.i: + output += self.process_word() + if final and self.buffer: # EOF terminates the last word + output += self.process_word() + return output + + def process_word(self): + output = '' + if self.buffer[0] == ord('i'): + self.i = min(99, int(self.buffer[1:] or 0)) # set input length + elif self.buffer[0] == ord('o'): + self.o = min(99, int(self.buffer[1:] or 0)) # set output length + else: + output = self.buffer.decode('ascii') + if len(output) < self.o: + output += '-'*self.o # pad out with hyphens + if self.o: + output = output[:self.o] # truncate to output length + output += '.' + self.buffer = bytearray() + return output + + codecEnabled = False + + +# bpo-41919: This method is separated from StatefulIncrementalDecoder to avoid a resource leak +# when registering codecs and cleanup functions. +def lookupTestDecoder(name): + if StatefulIncrementalDecoder.codecEnabled and name == 'test_decoder': + latin1 = codecs.lookup('latin-1') + return codecs.CodecInfo( + name='test_decoder', encode=latin1.encode, decode=None, + incrementalencoder=None, + streamreader=None, streamwriter=None, + incrementaldecoder=StatefulIncrementalDecoder) + + +class StatefulIncrementalDecoderTest(unittest.TestCase): + """ + Make sure the StatefulIncrementalDecoder actually works. + """ + + test_cases = [ + # I=1, O=1 (fixed-length input == fixed-length output) + (b'abcd', False, 'a.b.c.d.'), + # I=0, O=0 (variable-length input, variable-length output) + (b'oiabcd', True, 'abcd.'), + # I=0, O=0 (should ignore extra periods) + (b'oi...abcd...', True, 'abcd.'), + # I=0, O=6 (variable-length input, fixed-length output) + (b'i.o6.x.xyz.toolongtofit.', False, 'x-----.xyz---.toolon.'), + # I=2, O=6 (fixed-length input < fixed-length output) + (b'i.i2.o6xyz', True, 'xy----.z-----.'), + # I=6, O=3 (fixed-length input > fixed-length output) + (b'i.o3.i6.abcdefghijklmnop', True, 'abc.ghi.mno.'), + # I=0, then 3; O=29, then 15 (with longer output) + (b'i.o29.a.b.cde.o15.abcdefghijabcdefghij.i3.a.b.c.d.ei00k.l.m', True, + 'a----------------------------.' + + 'b----------------------------.' + + 'cde--------------------------.' + + 'abcdefghijabcde.' + + 'a.b------------.' + + '.c.------------.' + + 'd.e------------.' + + 'k--------------.' + + 'l--------------.' + + 'm--------------.') + ] + + def test_decoder(self): + # Try a few one-shot test cases. + for input, eof, output in self.test_cases: + d = StatefulIncrementalDecoder() + self.assertEqual(d.decode(input, eof), output) + + # Also test an unfinished decode, followed by forcing EOF. + d = StatefulIncrementalDecoder() + self.assertEqual(d.decode(b'oiabcd'), '') + self.assertEqual(d.decode(b'', 1), 'abcd.') + +class TextIOWrapperTest: + + def setUp(self): + self.testdata = b"AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n" + self.normalized = b"AAA\nBBB\nCCC\nDDD\nEEE\n".decode("ascii") + os_helper.unlink(os_helper.TESTFN) + codecs.register(lookupTestDecoder) + self.addCleanup(codecs.unregister, lookupTestDecoder) + + def tearDown(self): + os_helper.unlink(os_helper.TESTFN) + + def test_constructor(self): + r = self.BytesIO(b"\xc3\xa9\n\n") + b = self.BufferedReader(r, 1000) + t = self.TextIOWrapper(b, encoding="utf-8") + t.__init__(b, encoding="latin-1", newline="\r\n") + self.assertEqual(t.encoding, "latin-1") + self.assertEqual(t.line_buffering, False) + t.__init__(b, encoding="utf-8", line_buffering=True) + self.assertEqual(t.encoding, "utf-8") + self.assertEqual(t.line_buffering, True) + self.assertEqual("\xe9\n", t.readline()) + invalid_type = TypeError if self.is_C else ValueError + with self.assertRaises(invalid_type): + t.__init__(b, encoding=42) + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding='utf-8\0') + with self.assertRaises(invalid_type): + t.__init__(b, encoding="utf-8", errors=42) + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding="utf-8", errors='\udcfe') + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", errors='replace\0') + with self.assertRaises(TypeError): + t.__init__(b, encoding="utf-8", newline=42) + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\n\0') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='xyzzy') + + def test_uninitialized(self): + t = self.TextIOWrapper.__new__(self.TextIOWrapper) + del t + t = self.TextIOWrapper.__new__(self.TextIOWrapper) + self.assertRaises(Exception, repr, t) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + t.read, 0) + t.__init__(self.MockRawIO(), encoding="utf-8") + self.assertEqual(t.read(0), '') + + def test_non_text_encoding_codecs_are_rejected(self): + # Ensure the constructor complains if passed a codec that isn't + # marked as a text encoding + # http://bugs.python.org/issue20404 + r = self.BytesIO() + b = self.BufferedWriter(r) + with self.assertRaisesRegex(LookupError, "is not a text encoding"): + self.TextIOWrapper(b, encoding="hex") + + def test_detach(self): + r = self.BytesIO() + b = self.BufferedWriter(r) + t = self.TextIOWrapper(b, encoding="ascii") + self.assertIs(t.detach(), b) + + t = self.TextIOWrapper(b, encoding="ascii") + t.write("howdy") + self.assertFalse(r.getvalue()) + t.detach() + self.assertEqual(r.getvalue(), b"howdy") + self.assertRaises(ValueError, t.detach) + + # Operations independent of the detached stream should still work + repr(t) + self.assertEqual(t.encoding, "ascii") + self.assertEqual(t.errors, "strict") + self.assertFalse(t.line_buffering) + self.assertFalse(t.write_through) + + def test_repr(self): + raw = self.BytesIO("hello".encode("utf-8")) + b = self.BufferedReader(raw) + t = self.TextIOWrapper(b, encoding="utf-8") + modname = self.TextIOWrapper.__module__ + self.assertRegex(repr(t), + r"<(%s\.)?TextIOWrapper encoding='utf-8'>" % modname) + raw.name = "dummy" + self.assertRegex(repr(t), + r"<(%s\.)?TextIOWrapper name='dummy' encoding='utf-8'>" % modname) + t.mode = "r" + self.assertRegex(repr(t), + r"<(%s\.)?TextIOWrapper name='dummy' mode='r' encoding='utf-8'>" % modname) + raw.name = b"dummy" + self.assertRegex(repr(t), + r"<(%s\.)?TextIOWrapper name=b'dummy' mode='r' encoding='utf-8'>" % modname) + + t.buffer.detach() + repr(t) # Should not raise an exception + + def test_recursive_repr(self): + # Issue #25455 + raw = self.BytesIO() + t = self.TextIOWrapper(raw, encoding="utf-8") + with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): + repr(t) # Should not crash + + def test_subclass_repr(self): + class TestSubclass(self.TextIOWrapper): + pass + + f = TestSubclass(self.StringIO()) + self.assertIn(TestSubclass.__name__, repr(f)) + + def test_line_buffering(self): + r = self.BytesIO() + b = self.BufferedWriter(r, 1000) + t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=True) + t.write("X") + self.assertEqual(r.getvalue(), b"") # No flush happened + t.write("Y\nZ") + self.assertEqual(r.getvalue(), b"XY\nZ") # All got flushed + t.write("A\rB") + self.assertEqual(r.getvalue(), b"XY\nZA\rB") + + def test_reconfigure_line_buffering(self): + r = self.BytesIO() + b = self.BufferedWriter(r, 1000) + t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=False) + t.write("AB\nC") + self.assertEqual(r.getvalue(), b"") + + t.reconfigure(line_buffering=True) # implicit flush + self.assertEqual(r.getvalue(), b"AB\nC") + t.write("DEF\nG") + self.assertEqual(r.getvalue(), b"AB\nCDEF\nG") + t.write("H") + self.assertEqual(r.getvalue(), b"AB\nCDEF\nG") + t.reconfigure(line_buffering=False) # implicit flush + self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH") + t.write("IJ") + self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH") + + # Keeping default value + t.reconfigure() + t.reconfigure(line_buffering=None) + self.assertEqual(t.line_buffering, False) + t.reconfigure(line_buffering=True) + t.reconfigure() + t.reconfigure(line_buffering=None) + self.assertEqual(t.line_buffering, True) + + @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") + def test_default_encoding(self): + with os_helper.EnvironmentVarGuard() as env: + # try to get a user preferred encoding different than the current + # locale encoding to check that TextIOWrapper() uses the current + # locale encoding and not the user preferred encoding + env.unset('LC_ALL', 'LANG', 'LC_CTYPE') + + current_locale_encoding = locale.getencoding() + b = self.BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", EncodingWarning) + t = self.TextIOWrapper(b) + self.assertEqual(t.encoding, current_locale_encoding) + + def test_encoding(self): + # Check the encoding attribute is always set, and valid + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="utf-8") + self.assertEqual(t.encoding, "utf-8") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", EncodingWarning) + t = self.TextIOWrapper(b) + self.assertIsNotNone(t.encoding) + codecs.lookup(t.encoding) + + def test_encoding_errors_reading(self): + # (1) default + b = self.BytesIO(b"abc\n\xff\n") + t = self.TextIOWrapper(b, encoding="ascii") + self.assertRaises(UnicodeError, t.read) + # (2) explicit strict + b = self.BytesIO(b"abc\n\xff\n") + t = self.TextIOWrapper(b, encoding="ascii", errors="strict") + self.assertRaises(UnicodeError, t.read) + # (3) ignore + b = self.BytesIO(b"abc\n\xff\n") + t = self.TextIOWrapper(b, encoding="ascii", errors="ignore") + self.assertEqual(t.read(), "abc\n\n") + # (4) replace + b = self.BytesIO(b"abc\n\xff\n") + t = self.TextIOWrapper(b, encoding="ascii", errors="replace") + self.assertEqual(t.read(), "abc\n\ufffd\n") + + def test_encoding_errors_writing(self): + # (1) default + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="ascii") + self.assertRaises(UnicodeError, t.write, "\xff") + # (2) explicit strict + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="ascii", errors="strict") + self.assertRaises(UnicodeError, t.write, "\xff") + # (3) ignore + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="ascii", errors="ignore", + newline="\n") + t.write("abc\xffdef\n") + t.flush() + self.assertEqual(b.getvalue(), b"abcdef\n") + # (4) replace + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="ascii", errors="replace", + newline="\n") + t.write("abc\xffdef\n") + t.flush() + self.assertEqual(b.getvalue(), b"abc?def\n") + + def test_newlines(self): + input_lines = [ "unix\n", "windows\r\n", "os9\r", "last\n", "nonl" ] + + tests = [ + [ None, [ 'unix\n', 'windows\n', 'os9\n', 'last\n', 'nonl' ] ], + [ '', input_lines ], + [ '\n', [ "unix\n", "windows\r\n", "os9\rlast\n", "nonl" ] ], + [ '\r\n', [ "unix\nwindows\r\n", "os9\rlast\nnonl" ] ], + [ '\r', [ "unix\nwindows\r", "\nos9\r", "last\nnonl" ] ], + ] + encodings = ( + 'utf-8', 'latin-1', + 'utf-16', 'utf-16-le', 'utf-16-be', + 'utf-32', 'utf-32-le', 'utf-32-be', + ) + + # Try a range of buffer sizes to test the case where \r is the last + # character in TextIOWrapper._pending_line. + for encoding in encodings: + # XXX: str.encode() should return bytes + data = bytes(''.join(input_lines).encode(encoding)) + for do_reads in (False, True): + for bufsize in range(1, 10): + for newline, exp_lines in tests: + bufio = self.BufferedReader(self.BytesIO(data), bufsize) + textio = self.TextIOWrapper(bufio, newline=newline, + encoding=encoding) + if do_reads: + got_lines = [] + while True: + c2 = textio.read(2) + if c2 == '': + break + self.assertEqual(len(c2), 2) + got_lines.append(c2 + textio.readline()) + else: + got_lines = list(textio) + + for got_line, exp_line in zip(got_lines, exp_lines): + self.assertEqual(got_line, exp_line) + self.assertEqual(len(got_lines), len(exp_lines)) + + def test_newlines_input(self): + testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG" + normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + for newline, expected in [ + (None, normalized.decode("ascii").splitlines(keepends=True)), + ("", testdata.decode("ascii").splitlines(keepends=True)), + ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), + ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), + ("\r", ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]), + ]: + buf = self.BytesIO(testdata) + txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline) + self.assertEqual(txt.readlines(), expected) + txt.seek(0) + self.assertEqual(txt.read(), "".join(expected)) + + def test_newlines_output(self): + testdict = { + "": b"AAA\nBBB\nCCC\nX\rY\r\nZ", + "\n": b"AAA\nBBB\nCCC\nX\rY\r\nZ", + "\r": b"AAA\rBBB\rCCC\rX\rY\r\rZ", + "\r\n": b"AAA\r\nBBB\r\nCCC\r\nX\rY\r\r\nZ", + } + tests = [(None, testdict[os.linesep])] + sorted(testdict.items()) + for newline, expected in tests: + buf = self.BytesIO() + txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline) + txt.write("AAA\nB") + txt.write("BB\nCCC\n") + txt.write("X\rY\r\nZ") + txt.flush() + self.assertEqual(buf.closed, False) + self.assertEqual(buf.getvalue(), expected) + + def test_destructor(self): + l = [] + base = self.BytesIO + class MyBytesIO(base): + def close(self): + l.append(self.getvalue()) + base.close(self) + b = MyBytesIO() + t = self.TextIOWrapper(b, encoding="ascii") + t.write("abc") + del t + support.gc_collect() + self.assertEqual([b"abc"], l) + + def test_override_destructor(self): + record = [] + class MyTextIO(self.TextIOWrapper): + def __del__(self): + record.append(1) + try: + f = super().__del__ + except AttributeError: + pass + else: + f() + def close(self): + record.append(2) + super().close() + def flush(self): + record.append(3) + super().flush() + b = self.BytesIO() + t = MyTextIO(b, encoding="ascii") + del t + support.gc_collect() + self.assertEqual(record, [1, 2, 3]) + + def test_error_through_destructor(self): + # Test that the exception state is not modified by a destructor, + # even if close() fails. + rawio = self.CloseFailureIO() + with support.catch_unraisable_exception() as cm: + with self.assertRaises(AttributeError): + self.TextIOWrapper(rawio, encoding="utf-8").xyzzy + + self.assertEqual(cm.unraisable.exc_type, OSError) + + # Systematic tests of the text I/O API + + def test_basic_io(self): + for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65): + for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le": + f = self.open(os_helper.TESTFN, "w+", encoding=enc) + f._CHUNK_SIZE = chunksize + self.assertEqual(f.write("abc"), 3) + f.close() + f = self.open(os_helper.TESTFN, "r+", encoding=enc) + f._CHUNK_SIZE = chunksize + self.assertEqual(f.tell(), 0) + self.assertEqual(f.read(), "abc") + cookie = f.tell() + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.read(None), "abc") + f.seek(0) + self.assertEqual(f.read(2), "ab") + self.assertEqual(f.read(1), "c") + self.assertEqual(f.read(1), "") + self.assertEqual(f.read(), "") + self.assertEqual(f.tell(), cookie) + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.seek(0, 2), cookie) + self.assertEqual(f.write("def"), 3) + self.assertEqual(f.seek(cookie), cookie) + self.assertEqual(f.read(), "def") + if enc.startswith("utf"): + self.multi_line_test(f, enc) + f.close() + + def multi_line_test(self, f, enc): + f.seek(0) + f.truncate() + sample = "s\xff\u0fff\uffff" + wlines = [] + for size in (0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 62, 63, 64, 65, 1000): + chars = [] + for i in range(size): + chars.append(sample[i % len(sample)]) + line = "".join(chars) + "\n" + wlines.append((f.tell(), line)) + f.write(line) + f.seek(0) + rlines = [] + while True: + pos = f.tell() + line = f.readline() + if not line: + break + rlines.append((pos, line)) + self.assertEqual(rlines, wlines) + + def test_telling(self): + f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") + p0 = f.tell() + f.write("\xff\n") + p1 = f.tell() + f.write("\xff\n") + p2 = f.tell() + f.seek(0) + self.assertEqual(f.tell(), p0) + self.assertEqual(f.readline(), "\xff\n") + self.assertEqual(f.tell(), p1) + self.assertEqual(f.readline(), "\xff\n") + self.assertEqual(f.tell(), p2) + f.seek(0) + for line in f: + self.assertEqual(line, "\xff\n") + self.assertRaises(OSError, f.tell) + self.assertEqual(f.tell(), p2) + f.close() + + def test_seeking(self): + chunk_size = _default_chunk_size() + prefix_size = chunk_size - 2 + u_prefix = "a" * prefix_size + prefix = bytes(u_prefix.encode("utf-8")) + self.assertEqual(len(u_prefix), len(prefix)) + u_suffix = "\u8888\n" + suffix = bytes(u_suffix.encode("utf-8")) + line = prefix + suffix + with self.open(os_helper.TESTFN, "wb") as f: + f.write(line*2) + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + s = f.read(prefix_size) + self.assertEqual(s, str(prefix, "ascii")) + self.assertEqual(f.tell(), prefix_size) + self.assertEqual(f.readline(), u_suffix) + + def test_seeking_too(self): + # Regression test for a specific bug + data = b'\xe0\xbf\xbf\n' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + f._CHUNK_SIZE # Just test that it exists + f._CHUNK_SIZE = 2 + f.readline() + f.tell() + + def test_seek_and_tell(self): + #Test seek/tell using the StatefulIncrementalDecoder. + # Make test faster by doing smaller seeks + CHUNK_SIZE = 128 + + def test_seek_and_tell_with_data(data, min_pos=0): + """Tell/seek to various points within a data stream and ensure + that the decoded data returned by read() is consistent.""" + f = self.open(os_helper.TESTFN, 'wb') + f.write(data) + f.close() + f = self.open(os_helper.TESTFN, encoding='test_decoder') + f._CHUNK_SIZE = CHUNK_SIZE + decoded = f.read() + f.close() + + for i in range(min_pos, len(decoded) + 1): # seek positions + for j in [1, 5, len(decoded) - i]: # read lengths + f = self.open(os_helper.TESTFN, encoding='test_decoder') + self.assertEqual(f.read(i), decoded[:i]) + cookie = f.tell() + self.assertEqual(f.read(j), decoded[i:i + j]) + f.seek(cookie) + self.assertEqual(f.read(), decoded[i:]) + f.close() + + # Enable the test decoder. + StatefulIncrementalDecoder.codecEnabled = 1 + + # Run the tests. + try: + # Try each test case. + for input, _, _ in StatefulIncrementalDecoderTest.test_cases: + test_seek_and_tell_with_data(input) + + # Position each test case so that it crosses a chunk boundary. + for input, _, _ in StatefulIncrementalDecoderTest.test_cases: + offset = CHUNK_SIZE - len(input)//2 + prefix = b'.'*offset + # Don't bother seeking into the prefix (takes too long). + min_pos = offset*2 + test_seek_and_tell_with_data(prefix + input, min_pos) + + # Ensure our test decoder won't interfere with subsequent tests. + finally: + StatefulIncrementalDecoder.codecEnabled = 0 + + def test_multibyte_seek_and_tell(self): + f = self.open(os_helper.TESTFN, "w", encoding="euc_jp") + f.write("AB\n\u3046\u3048\n") + f.close() + + f = self.open(os_helper.TESTFN, "r", encoding="euc_jp") + self.assertEqual(f.readline(), "AB\n") + p0 = f.tell() + self.assertEqual(f.readline(), "\u3046\u3048\n") + p1 = f.tell() + f.seek(p0) + self.assertEqual(f.readline(), "\u3046\u3048\n") + self.assertEqual(f.tell(), p1) + f.close() + + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + + + def test_seek_with_encoder_state(self): + f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") + f.write("\u00e6\u0300") + p0 = f.tell() + f.write("\u00e6") + f.seek(p0) + f.write("\u0300") + f.close() + + f = self.open(os_helper.TESTFN, "r", encoding="euc_jis_2004") + self.assertEqual(f.readline(), "\u00e6\u0300\u0300") + f.close() + + def test_encoded_writes(self): + data = "1234567890" + tests = ("utf-16", + "utf-16-le", + "utf-16-be", + "utf-32", + "utf-32-le", + "utf-32-be") + for encoding in tests: + buf = self.BytesIO() + f = self.TextIOWrapper(buf, encoding=encoding) + # Check if the BOM is written only once (see issue1753). + f.write(data) + f.write(data) + f.seek(0) + self.assertEqual(f.read(), data * 2) + f.seek(0) + self.assertEqual(f.read(), data * 2) + self.assertEqual(buf.getvalue(), (data * 2).encode(encoding)) + + def test_unreadable(self): + class UnReadable(self.BytesIO): + def readable(self): + return False + txt = self.TextIOWrapper(UnReadable(), encoding="utf-8") + self.assertRaises(OSError, txt.read) + + def test_read_one_by_one(self): + txt = self.TextIOWrapper(self.BytesIO(b"AA\r\nBB"), encoding="utf-8") + reads = "" + while True: + c = txt.read(1) + if not c: + break + reads += c + self.assertEqual(reads, "AA\nBB") + + def test_readlines(self): + txt = self.TextIOWrapper(self.BytesIO(b"AA\nBB\nCC"), encoding="utf-8") + self.assertEqual(txt.readlines(), ["AA\n", "BB\n", "CC"]) + txt.seek(0) + self.assertEqual(txt.readlines(None), ["AA\n", "BB\n", "CC"]) + txt.seek(0) + self.assertEqual(txt.readlines(5), ["AA\n", "BB\n"]) + + # read in amounts equal to TextIOWrapper._CHUNK_SIZE which is 128. + def test_read_by_chunk(self): + # make sure "\r\n" straddles 128 char boundary. + txt = self.TextIOWrapper(self.BytesIO(b"A" * 127 + b"\r\nB"), encoding="utf-8") + reads = "" + while True: + c = txt.read(128) + if not c: + break + reads += c + self.assertEqual(reads, "A"*127+"\nB") + + def test_writelines(self): + l = ['ab', 'cd', 'ef'] + buf = self.BytesIO() + txt = self.TextIOWrapper(buf, encoding="utf-8") + txt.writelines(l) + txt.flush() + self.assertEqual(buf.getvalue(), b'abcdef') + + def test_writelines_userlist(self): + l = UserList(['ab', 'cd', 'ef']) + buf = self.BytesIO() + txt = self.TextIOWrapper(buf, encoding="utf-8") + txt.writelines(l) + txt.flush() + self.assertEqual(buf.getvalue(), b'abcdef') + + def test_writelines_error(self): + txt = self.TextIOWrapper(self.BytesIO(), encoding="utf-8") + self.assertRaises(TypeError, txt.writelines, [1, 2, 3]) + self.assertRaises(TypeError, txt.writelines, None) + self.assertRaises(TypeError, txt.writelines, b'abc') + + def test_issue1395_1(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + + # read one char at a time + reads = "" + while True: + c = txt.read(1) + if not c: + break + reads += c + self.assertEqual(reads, self.normalized) + + def test_issue1395_2(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = "" + while True: + c = txt.read(4) + if not c: + break + reads += c + self.assertEqual(reads, self.normalized) + + def test_issue1395_3(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + reads += txt.read(4) + reads += txt.readline() + reads += txt.readline() + reads += txt.readline() + self.assertEqual(reads, self.normalized) + + def test_issue1395_4(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + reads += txt.read() + self.assertEqual(reads, self.normalized) + + def test_issue1395_5(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + pos = txt.tell() + txt.seek(0) + txt.seek(pos) + self.assertEqual(txt.read(4), "BBB\n") + + def test_issue2282(self): + buffer = self.BytesIO(self.testdata) + txt = self.TextIOWrapper(buffer, encoding="ascii") + + self.assertEqual(buffer.seekable(), txt.seekable()) + + def test_append_bom(self): + # The BOM is not written again when appending to a non-empty file + filename = os_helper.TESTFN + for charset in ('utf-8-sig', 'utf-16', 'utf-32'): + with self.open(filename, 'w', encoding=charset) as f: + f.write('aaa') + pos = f.tell() + with self.open(filename, 'rb') as f: + self.assertEqual(f.read(), 'aaa'.encode(charset)) + + with self.open(filename, 'a', encoding=charset) as f: + f.write('xxx') + with self.open(filename, 'rb') as f: + self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) + + def test_seek_bom(self): + # Same test, but when seeking manually + filename = os_helper.TESTFN + for charset in ('utf-8-sig', 'utf-16', 'utf-32'): + with self.open(filename, 'w', encoding=charset) as f: + f.write('aaa') + pos = f.tell() + with self.open(filename, 'r+', encoding=charset) as f: + f.seek(pos) + f.write('zzz') + f.seek(0) + f.write('bbb') + with self.open(filename, 'rb') as f: + self.assertEqual(f.read(), 'bbbzzz'.encode(charset)) + + def test_seek_append_bom(self): + # Same test, but first seek to the start and then to the end + filename = os_helper.TESTFN + for charset in ('utf-8-sig', 'utf-16', 'utf-32'): + with self.open(filename, 'w', encoding=charset) as f: + f.write('aaa') + with self.open(filename, 'a', encoding=charset) as f: + f.seek(0) + f.seek(0, self.SEEK_END) + f.write('xxx') + with self.open(filename, 'rb') as f: + self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) + + def test_errors_property(self): + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + self.assertEqual(f.errors, "strict") + with self.open(os_helper.TESTFN, "w", encoding="utf-8", errors="replace") as f: + self.assertEqual(f.errors, "replace") + + @support.no_tracing + @threading_helper.requires_working_threading() + def test_threads_write(self): + # Issue6750: concurrent writes could duplicate data + event = threading.Event() + with self.open(os_helper.TESTFN, "w", encoding="utf-8", buffering=1) as f: + def run(n): + text = "Thread%03d\n" % n + event.wait() + f.write(text) + threads = [threading.Thread(target=run, args=(x,)) + for x in range(20)] + with threading_helper.start_threads(threads, event.set): + time.sleep(0.02) + with self.open(os_helper.TESTFN, encoding="utf-8") as f: + content = f.read() + for n in range(20): + self.assertEqual(content.count("Thread%03d\n" % n), 1) + + def test_flush_error_on_close(self): + # Test that text file is closed despite failed flush + # and that flush() is called before file closed. + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + closed = [] + def bad_flush(): + closed[:] = [txt.closed, txt.buffer.closed] + raise OSError() + txt.flush = bad_flush + self.assertRaises(OSError, txt.close) # exception not swallowed + self.assertTrue(txt.closed) + self.assertTrue(txt.buffer.closed) + self.assertTrue(closed) # flush() called + self.assertFalse(closed[0]) # flush() called before file closed + self.assertFalse(closed[1]) + txt.flush = lambda: None # break reference loop + + def test_close_error_on_close(self): + buffer = self.BytesIO(self.testdata) + def bad_flush(): + raise OSError('flush') + def bad_close(): + raise OSError('close') + buffer.close = bad_close + txt = self.TextIOWrapper(buffer, encoding="ascii") + txt.flush = bad_flush + with self.assertRaises(OSError) as err: # exception not swallowed + txt.close() + self.assertEqual(err.exception.args, ('close',)) + self.assertIsInstance(err.exception.__context__, OSError) + self.assertEqual(err.exception.__context__.args, ('flush',)) + self.assertFalse(txt.closed) + + # Silence destructor error + buffer.close = lambda: None + txt.flush = lambda: None + + def test_nonnormalized_close_error_on_close(self): + # Issue #21677 + buffer = self.BytesIO(self.testdata) + def bad_flush(): + raise non_existing_flush + def bad_close(): + raise non_existing_close + buffer.close = bad_close + txt = self.TextIOWrapper(buffer, encoding="ascii") + txt.flush = bad_flush + with self.assertRaises(NameError) as err: # exception not swallowed + txt.close() + self.assertIn('non_existing_close', str(err.exception)) + self.assertIsInstance(err.exception.__context__, NameError) + self.assertIn('non_existing_flush', str(err.exception.__context__)) + self.assertFalse(txt.closed) + + # Silence destructor error + buffer.close = lambda: None + txt.flush = lambda: None + + def test_multi_close(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt.close() + txt.close() + txt.close() + self.assertRaises(ValueError, txt.flush) + + def test_unseekable(self): + txt = self.TextIOWrapper(self.MockUnseekableIO(self.testdata), encoding="utf-8") + self.assertRaises(self.UnsupportedOperation, txt.tell) + self.assertRaises(self.UnsupportedOperation, txt.seek, 0) + + def test_readonly_attributes(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + buf = self.BytesIO(self.testdata) + with self.assertRaises(AttributeError): + txt.buffer = buf + + def test_rawio(self): + # Issue #12591: TextIOWrapper must work with raw I/O objects, so + # that subprocess.Popen() can have the required unbuffered + # semantics with universal_newlines=True. + raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n']) + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + # Reads + self.assertEqual(txt.read(4), 'abcd') + self.assertEqual(txt.readline(), 'efghi\n') + self.assertEqual(list(txt), ['jkl\n', 'opq\n']) + + def test_rawio_write_through(self): + # Issue #12591: with write_through=True, writes don't need a flush + raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n']) + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n', + write_through=True) + txt.write('1') + txt.write('23\n4') + txt.write('5') + self.assertEqual(b''.join(raw._write_stack), b'123\n45') + + def test_bufio_write_through(self): + # Issue #21396: write_through=True doesn't force a flush() + # on the underlying binary buffered object. + flush_called, write_called = [], [] + class BufferedWriter(self.BufferedWriter): + def flush(self, *args, **kwargs): + flush_called.append(True) + return super().flush(*args, **kwargs) + def write(self, *args, **kwargs): + write_called.append(True) + return super().write(*args, **kwargs) + + rawio = self.BytesIO() + data = b"a" + bufio = BufferedWriter(rawio, len(data)*2) + textio = self.TextIOWrapper(bufio, encoding='ascii', + write_through=True) + # write to the buffered io but don't overflow the buffer + text = data.decode('ascii') + textio.write(text) + + # buffer.flush is not called with write_through=True + self.assertFalse(flush_called) + # buffer.write *is* called with write_through=True + self.assertTrue(write_called) + self.assertEqual(rawio.getvalue(), b"") # no flush + + write_called = [] # reset + textio.write(text * 10) # total content is larger than bufio buffer + self.assertTrue(write_called) + self.assertEqual(rawio.getvalue(), data * 11) # all flushed + + def test_reconfigure_write_through(self): + raw = self.MockRawIO([]) + t = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + t.write('1') + t.reconfigure(write_through=True) # implied flush + self.assertEqual(t.write_through, True) + self.assertEqual(b''.join(raw._write_stack), b'1') + t.write('23') + self.assertEqual(b''.join(raw._write_stack), b'123') + t.reconfigure(write_through=False) + self.assertEqual(t.write_through, False) + t.write('45') + t.flush() + self.assertEqual(b''.join(raw._write_stack), b'12345') + # Keeping default value + t.reconfigure() + t.reconfigure(write_through=None) + self.assertEqual(t.write_through, False) + t.reconfigure(write_through=True) + t.reconfigure() + t.reconfigure(write_through=None) + self.assertEqual(t.write_through, True) + + def test_read_nonbytes(self): + # Issue #17106 + # Crash when underlying read() returns non-bytes + t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") + self.assertRaises(TypeError, t.read, 1) + t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") + self.assertRaises(TypeError, t.readline) + t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") + self.assertRaises(TypeError, t.read) + + def test_illegal_encoder(self): + # Issue 31271: Calling write() while the return value of encoder's + # encode() is invalid shouldn't cause an assertion failure. + rot13 = codecs.lookup("rot13") + with support.swap_attr(rot13, '_is_text_encoding', True): + t = self.TextIOWrapper(self.BytesIO(b'foo'), encoding="rot13") + self.assertRaises(TypeError, t.write, 'bar') + + def test_illegal_decoder(self): + # Issue #17106 + # Bypass the early encoding check added in issue 20404 + def _make_illegal_wrapper(): + quopri = codecs.lookup("quopri") + quopri._is_text_encoding = True + try: + t = self.TextIOWrapper(self.BytesIO(b'aaaaaa'), + newline='\n', encoding="quopri") + finally: + quopri._is_text_encoding = False + return t + # Crash when decoder returns non-string + t = _make_illegal_wrapper() + self.assertRaises(TypeError, t.read, 1) + t = _make_illegal_wrapper() + self.assertRaises(TypeError, t.readline) + t = _make_illegal_wrapper() + self.assertRaises(TypeError, t.read) + + # Issue 31243: calling read() while the return value of decoder's + # getstate() is invalid should neither crash the interpreter nor + # raise a SystemError. + def _make_very_illegal_wrapper(getstate_ret_val): + class BadDecoder: + def getstate(self): + return getstate_ret_val + def _get_bad_decoder(dummy): + return BadDecoder() + quopri = codecs.lookup("quopri") + with support.swap_attr(quopri, 'incrementaldecoder', + _get_bad_decoder): + return _make_illegal_wrapper() + t = _make_very_illegal_wrapper(42) + self.assertRaises(TypeError, t.read, 42) + t = _make_very_illegal_wrapper(()) + self.assertRaises(TypeError, t.read, 42) + t = _make_very_illegal_wrapper((1, 2)) + self.assertRaises(TypeError, t.read, 42) + + def _check_create_at_shutdown(self, **kwargs): + # Issue #20037: creating a TextIOWrapper at shutdown + # shouldn't crash the interpreter. + iomod = self.io.__name__ + code = """if 1: + import codecs + import {iomod} as io + + # Avoid looking up codecs at shutdown + codecs.lookup('utf-8') + + class C: + def __del__(self): + io.TextIOWrapper(io.BytesIO(), **{kwargs}) + print("ok") + c = C() + """.format(iomod=iomod, kwargs=kwargs) + return assert_python_ok("-c", code) + + def test_create_at_shutdown_without_encoding(self): + rc, out, err = self._check_create_at_shutdown() + if err: + # Can error out with a RuntimeError if the module state + # isn't found. + self.assertIn(self.shutdown_error, err.decode()) + else: + self.assertEqual("ok", out.decode().strip()) + + def test_create_at_shutdown_with_encoding(self): + rc, out, err = self._check_create_at_shutdown(encoding='utf-8', + errors='strict') + self.assertFalse(err) + self.assertEqual("ok", out.decode().strip()) + + def test_read_byteslike(self): + r = MemviewBytesIO(b'Just some random string\n') + t = self.TextIOWrapper(r, 'utf-8') + + # TextIOwrapper will not read the full string, because + # we truncate it to a multiple of the native int size + # so that we can construct a more complex memoryview. + bytes_val = _to_memoryview(r.getvalue()).tobytes() + + self.assertEqual(t.read(200), bytes_val.decode('utf-8')) + + def test_issue22849(self): + class F(object): + def readable(self): return True + def writable(self): return True + def seekable(self): return True + + for i in range(10): + try: + self.TextIOWrapper(F(), encoding='utf-8') + except Exception: + pass + + F.tell = lambda x: 0 + t = self.TextIOWrapper(F(), encoding='utf-8') + + def test_reconfigure_locale(self): + wrapper = self.TextIOWrapper(self.BytesIO(b"test")) + wrapper.reconfigure(encoding="locale") + + def test_reconfigure_encoding_read(self): + # latin1 -> utf8 + # (latin1 can decode utf-8 encoded string) + data = 'abc\xe9\n'.encode('latin1') + 'd\xe9f\n'.encode('utf8') + raw = self.BytesIO(data) + txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n') + self.assertEqual(txt.readline(), 'abc\xe9\n') + with self.assertRaises(self.UnsupportedOperation): + txt.reconfigure(encoding='utf-8') + with self.assertRaises(self.UnsupportedOperation): + txt.reconfigure(newline=None) + + def test_reconfigure_write_fromascii(self): + # ascii has a specific encodefunc in the C implementation, + # but utf-8-sig has not. Make sure that we get rid of the + # cached encodefunc when we switch encoders. + raw = self.BytesIO() + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + txt.write('foo\n') + txt.reconfigure(encoding='utf-8-sig') + txt.write('\xe9\n') + txt.flush() + self.assertEqual(raw.getvalue(), b'foo\n\xc3\xa9\n') + + def test_reconfigure_write(self): + # latin -> utf8 + raw = self.BytesIO() + txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n') + txt.write('abc\xe9\n') + txt.reconfigure(encoding='utf-8') + self.assertEqual(raw.getvalue(), b'abc\xe9\n') + txt.write('d\xe9f\n') + txt.flush() + self.assertEqual(raw.getvalue(), b'abc\xe9\nd\xc3\xa9f\n') + + # ascii -> utf-8-sig: ensure that no BOM is written in the middle of + # the file + raw = self.BytesIO() + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + txt.write('abc\n') + txt.reconfigure(encoding='utf-8-sig') + txt.write('d\xe9f\n') + txt.flush() + self.assertEqual(raw.getvalue(), b'abc\nd\xc3\xa9f\n') + + def test_reconfigure_write_non_seekable(self): + raw = self.BytesIO() + raw.seekable = lambda: False + raw.seek = None + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + txt.write('abc\n') + txt.reconfigure(encoding='utf-8-sig') + txt.write('d\xe9f\n') + txt.flush() + + # If the raw stream is not seekable, there'll be a BOM + self.assertEqual(raw.getvalue(), b'abc\n\xef\xbb\xbfd\xc3\xa9f\n') + + def test_reconfigure_defaults(self): + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\n') + txt.reconfigure(encoding=None) + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + txt.write('LF\n') + + txt.reconfigure(newline='\r\n') + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + + txt.reconfigure(errors='ignore') + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'ignore') + txt.write('CRLF\n') + + txt.reconfigure(encoding='utf-8', newline=None) + self.assertEqual(txt.errors, 'strict') + txt.seek(0) + self.assertEqual(txt.read(), 'LF\nCRLF\n') + + self.assertEqual(txt.detach().getvalue(), b'LF\nCRLF\r\n') + + def test_reconfigure_errors(self): + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\r') + with self.assertRaises(TypeError): # there was a crash + txt.reconfigure(encoding=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(encoding='\udcfe') + with self.assertRaises(LookupError): + txt.reconfigure(encoding='locale\0') + # TODO: txt.reconfigure(encoding='utf-8\0') + # TODO: txt.reconfigure(encoding='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(errors=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(errors='\udcfe') + # TODO: txt.reconfigure(errors='ignore\0') + # TODO: txt.reconfigure(errors='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(newline=42) + with self.assertRaises(ValueError): + txt.reconfigure(newline='\udcfe') + with self.assertRaises(ValueError): + txt.reconfigure(newline='xyz') + if not self.is_C: + # TODO: Should fail in C too. + with self.assertRaises(ValueError): + txt.reconfigure(newline='\n\0') + if self.is_C: + # TODO: Use __bool__(), not __index__(). + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(line_buffering=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(line_buffering=2**1000) + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(write_through=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(write_through=2**1000) + with self.assertRaises(ZeroDivisionError): # there was a crash + txt.reconfigure(line_buffering=BadIndex(), + write_through=BadIndex()) + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + self.assertIs(txt.line_buffering, False) + self.assertIs(txt.write_through, False) + + txt.reconfigure(encoding='latin1', errors='ignore', newline='\r\n', + line_buffering=True, write_through=True) + self.assertEqual(txt.encoding, 'latin1') + self.assertEqual(txt.errors, 'ignore') + self.assertIs(txt.line_buffering, True) + self.assertIs(txt.write_through, True) + + def test_reconfigure_newline(self): + raw = self.BytesIO(b'CR\rEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\n') + txt.reconfigure(newline=None) + self.assertEqual(txt.readline(), 'CR\n') + raw = self.BytesIO(b'CR\rEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\n') + txt.reconfigure(newline='') + self.assertEqual(txt.readline(), 'CR\r') + raw = self.BytesIO(b'CR\rLF\nEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\r') + txt.reconfigure(newline='\n') + self.assertEqual(txt.readline(), 'CR\rLF\n') + raw = self.BytesIO(b'LF\nCR\rEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\n') + txt.reconfigure(newline='\r') + self.assertEqual(txt.readline(), 'LF\nCR\r') + raw = self.BytesIO(b'CR\rCRLF\r\nEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\r') + txt.reconfigure(newline='\r\n') + self.assertEqual(txt.readline(), 'CR\rCRLF\r\n') + + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', newline='\r') + txt.reconfigure(newline=None) + txt.write('linesep\n') + txt.reconfigure(newline='') + txt.write('LF\n') + txt.reconfigure(newline='\n') + txt.write('LF\n') + txt.reconfigure(newline='\r') + txt.write('CR\n') + txt.reconfigure(newline='\r\n') + txt.write('CRLF\n') + expected = 'linesep' + os.linesep + 'LF\nLF\nCR\rCRLF\r\n' + self.assertEqual(txt.detach().getvalue().decode('ascii'), expected) + + def test_issue25862(self): + # Assertion failures occurred in tell() after read() and write(). + t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii') + t.read(1) + t.read() + t.tell() + t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii') + t.read(1) + t.write('x') + t.tell() + + def test_issue35928(self): + p = self.BufferedRWPair(self.BytesIO(b'foo\nbar\n'), self.BytesIO()) + f = self.TextIOWrapper(p) + res = f.readline() + self.assertEqual(res, 'foo\n') + f.write(res) + self.assertEqual(res + f.readline(), 'foo\nbar\n') + + def test_pickling_subclass(self): + global MyTextIO + class MyTextIO(self.TextIOWrapper): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.buffer.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + txt = MyTextIO(raw, 'ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(txt, proto) + newtxt = pickle.loads(pickled) + self.assertEqual(newtxt.buffer.getvalue(), b'data') + self.assertEqual(newtxt.tag, 'ham') + del MyTextIO + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_read_non_blocking(self): + import os + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.io.open(r, 'rt') as textfile: + r = None + # Nothing has been written so a non-blocking read raises a BlockingIOError exception. + with self.assertRaises(BlockingIOError): + textfile.read() + finally: + if r is not None: + os.close(r) + os.close(w) + + +class MemviewBytesIO(io.BytesIO): + '''A BytesIO object whose read method returns memoryviews + rather than bytes''' + + def read1(self, len_): + return _to_memoryview(super().read1(len_)) + + def read(self, len_): + return _to_memoryview(super().read(len_)) + +def _to_memoryview(buf): + '''Convert bytes-object *buf* to a non-trivial memoryview''' + + arr = array.array('i') + idx = len(buf) - len(buf) % arr.itemsize + arr.frombytes(buf[:idx]) + return memoryview(arr) + + +class CTextIOWrapperTest(TextIOWrapperTest, CTestCase): + shutdown_error = "LookupError: unknown encoding: ascii" + + def test_initialization(self): + r = self.BytesIO(b"\xc3\xa9\n\n") + b = self.BufferedReader(r, 1000) + t = self.TextIOWrapper(b, encoding="utf-8") + self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') + self.assertRaises(ValueError, t.read) + + t = self.TextIOWrapper.__new__(self.TextIOWrapper) + self.assertRaises(Exception, repr, t) + + def test_garbage_collection(self): + # C TextIOWrapper objects are collected, and collecting them flushes + # all data to disk. + # The Python version has __del__, so it ends in gc.garbage instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + rawio = self.FileIO(os_helper.TESTFN, "wb") + b = self.BufferedWriter(rawio) + t = self.TextIOWrapper(b, encoding="ascii") + t.write("456def") + t.x = t + wr = weakref.ref(t) + del t + support.gc_collect() + self.assertIsNone(wr(), wr) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"456def") + + def test_rwpair_cleared_before_textio(self): + # Issue 13070: TextIOWrapper's finalization would crash when called + # after the reference to the underlying BufferedRWPair's writer got + # cleared by the GC. + for i in range(1000): + b1 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()) + t1 = self.TextIOWrapper(b1, encoding="ascii") + b2 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()) + t2 = self.TextIOWrapper(b2, encoding="ascii") + # circular references + t1.buddy = t2 + t2.buddy = t1 + support.gc_collect() + + def test_del__CHUNK_SIZE_SystemError(self): + t = self.TextIOWrapper(self.BytesIO(), encoding='ascii') + with self.assertRaises(AttributeError): + del t._CHUNK_SIZE + + def test_internal_buffer_size(self): + # bpo-43260: TextIOWrapper's internal buffer should not store + # data larger than chunk size. + chunk_size = 8192 # default chunk size, updated later + + class MockIO(self.MockRawIO): + def write(self, data): + if len(data) > chunk_size: + raise RuntimeError + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf, encoding="ascii") + chunk_size = t._CHUNK_SIZE + t.write("abc") + t.write("def") + # default chunk size is 8192 bytes so t don't write data to buf. + self.assertEqual([], buf._write_stack) + + with self.assertRaises(RuntimeError): + t.write("x"*(chunk_size+1)) + + self.assertEqual([b"abcdef"], buf._write_stack) + t.write("ghi") + t.write("x"*chunk_size) + self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + + +class PyTextIOWrapperTest(TextIOWrapperTest, PyTestCase): + shutdown_error = "LookupError: unknown encoding: ascii" + + +class IncrementalNewlineDecoderTest: + + def check_newline_decoding_utf8(self, decoder): + # UTF-8 specific tests for a newline decoder + def _check_decode(b, s, **kwargs): + # We exercise getstate() / setstate() as well as decode() + state = decoder.getstate() + self.assertEqual(decoder.decode(b, **kwargs), s) + decoder.setstate(state) + self.assertEqual(decoder.decode(b, **kwargs), s) + + _check_decode(b'\xe8\xa2\x88', "\u8888") + + _check_decode(b'\xe8', "") + _check_decode(b'\xa2', "") + _check_decode(b'\x88', "\u8888") + + _check_decode(b'\xe8', "") + _check_decode(b'\xa2', "") + _check_decode(b'\x88', "\u8888") + + _check_decode(b'\xe8', "") + self.assertRaises(UnicodeDecodeError, decoder.decode, b'', final=True) + + decoder.reset() + _check_decode(b'\n', "\n") + _check_decode(b'\r', "") + _check_decode(b'', "\n", final=True) + _check_decode(b'\r', "\n", final=True) + + _check_decode(b'\r', "") + _check_decode(b'a', "\na") + + _check_decode(b'\r\r\n', "\n\n") + _check_decode(b'\r', "") + _check_decode(b'\r', "\n") + _check_decode(b'\na', "\na") + + _check_decode(b'\xe8\xa2\x88\r\n', "\u8888\n") + _check_decode(b'\xe8\xa2\x88', "\u8888") + _check_decode(b'\n', "\n") + _check_decode(b'\xe8\xa2\x88\r', "\u8888") + _check_decode(b'\n', "\n") + + def check_newline_decoding(self, decoder, encoding): + result = [] + if encoding is not None: + encoder = codecs.getincrementalencoder(encoding)() + def _decode_bytewise(s): + # Decode one byte at a time + for b in encoder.encode(s): + result.append(decoder.decode(bytes([b]))) + else: + encoder = None + def _decode_bytewise(s): + # Decode one char at a time + for c in s: + result.append(decoder.decode(c)) + self.assertEqual(decoder.newlines, None) + _decode_bytewise("abc\n\r") + self.assertEqual(decoder.newlines, '\n') + _decode_bytewise("\nabc") + self.assertEqual(decoder.newlines, ('\n', '\r\n')) + _decode_bytewise("abc\r") + self.assertEqual(decoder.newlines, ('\n', '\r\n')) + _decode_bytewise("abc") + self.assertEqual(decoder.newlines, ('\r', '\n', '\r\n')) + _decode_bytewise("abc\r") + self.assertEqual("".join(result), "abc\n\nabcabc\nabcabc") + decoder.reset() + input = "abc" + if encoder is not None: + encoder.reset() + input = encoder.encode(input) + self.assertEqual(decoder.decode(input), "abc") + self.assertEqual(decoder.newlines, None) + + def test_newline_decoder(self): + encodings = ( + # None meaning the IncrementalNewlineDecoder takes unicode input + # rather than bytes input + None, 'utf-8', 'latin-1', + 'utf-16', 'utf-16-le', 'utf-16-be', + 'utf-32', 'utf-32-le', 'utf-32-be', + ) + for enc in encodings: + decoder = enc and codecs.getincrementaldecoder(enc)() + decoder = self.IncrementalNewlineDecoder(decoder, translate=True) + self.check_newline_decoding(decoder, enc) + decoder = codecs.getincrementaldecoder("utf-8")() + decoder = self.IncrementalNewlineDecoder(decoder, translate=True) + self.check_newline_decoding_utf8(decoder) + self.assertRaises(TypeError, decoder.setstate, 42) + + def test_newline_bytes(self): + # Issue 5433: Excessive optimization in IncrementalNewlineDecoder + def _check(dec): + self.assertEqual(dec.newlines, None) + self.assertEqual(dec.decode("\u0D00"), "\u0D00") + self.assertEqual(dec.newlines, None) + self.assertEqual(dec.decode("\u0A00"), "\u0A00") + self.assertEqual(dec.newlines, None) + dec = self.IncrementalNewlineDecoder(None, translate=False) + _check(dec) + dec = self.IncrementalNewlineDecoder(None, translate=True) + _check(dec) + + def test_translate(self): + # issue 35062 + for translate in (-2, -1, 1, 2): + decoder = codecs.getincrementaldecoder("utf-8")() + decoder = self.IncrementalNewlineDecoder(decoder, translate) + self.check_newline_decoding_utf8(decoder) + decoder = codecs.getincrementaldecoder("utf-8")() + decoder = self.IncrementalNewlineDecoder(decoder, translate=0) + self.assertEqual(decoder.decode(b"\r\r\n"), "\r\r\n") + +class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest, unittest.TestCase): + IncrementalNewlineDecoder = io.IncrementalNewlineDecoder + + @support.cpython_only + def test_uninitialized(self): + uninitialized = self.IncrementalNewlineDecoder.__new__( + self.IncrementalNewlineDecoder) + self.assertRaises(ValueError, uninitialized.decode, b'bar') + self.assertRaises(ValueError, uninitialized.getstate) + self.assertRaises(ValueError, uninitialized.setstate, (b'foo', 0)) + self.assertRaises(ValueError, uninitialized.reset) + + +class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest, unittest.TestCase): + IncrementalNewlineDecoder = pyio.IncrementalNewlineDecoder diff --git a/Lib/test/test_univnewlines.py b/Lib/test/test_io/test_univnewlines.py similarity index 100% rename from Lib/test/test_univnewlines.py rename to Lib/test/test_io/test_univnewlines.py diff --git a/Lib/test/test_io/utils.py b/Lib/test/test_io/utils.py new file mode 100644 index 00000000000..3b1faec2140 --- /dev/null +++ b/Lib/test/test_io/utils.py @@ -0,0 +1,318 @@ +import array +import threading +import time +import unittest + +import io # C implementation of io +import _pyio as pyio # Python implementation of io + + +try: + import ctypes +except ImportError: + def byteslike(*pos, **kw): + return array.array("b", bytes(*pos, **kw)) +else: + class EmptyStruct(ctypes.Structure): + pass + + def byteslike(*pos, **kw): + """Create a bytes-like object having no string or sequence methods""" + data = bytes(*pos, **kw) + obj = EmptyStruct() + ctypes.resize(obj, len(data)) + memoryview(obj).cast("B")[:] = data + return obj + + +class MockRawIOWithoutRead: + """A RawIO implementation without read(), so as to exercise the default + RawIO.read() which calls readinto().""" + + def __init__(self, read_stack=()): + self._read_stack = list(read_stack) + self._write_stack = [] + self._reads = 0 + self._extraneous_reads = 0 + + def write(self, b): + self._write_stack.append(bytes(b)) + return len(b) + + def writable(self): + return True + + def fileno(self): + return 42 + + def readable(self): + return True + + def seekable(self): + return True + + def seek(self, pos, whence): + return 0 # wrong but we gotta return something + + def tell(self): + return 0 # same comment as above + + def readinto(self, buf): + self._reads += 1 + max_len = len(buf) + try: + data = self._read_stack[0] + except IndexError: + self._extraneous_reads += 1 + return 0 + if data is None: + del self._read_stack[0] + return None + n = len(data) + if len(data) <= max_len: + del self._read_stack[0] + buf[:n] = data + return n + else: + buf[:] = data[:max_len] + self._read_stack[0] = data[max_len:] + return max_len + + def truncate(self, pos=None): + return pos + +class CMockRawIOWithoutRead(MockRawIOWithoutRead, io.RawIOBase): + pass + +class PyMockRawIOWithoutRead(MockRawIOWithoutRead, pyio.RawIOBase): + pass + + +class MockRawIO(MockRawIOWithoutRead): + + def read(self, n=None): + self._reads += 1 + try: + return self._read_stack.pop(0) + except: + self._extraneous_reads += 1 + return b"" + +class CMockRawIO(MockRawIO, io.RawIOBase): + pass + +class PyMockRawIO(MockRawIO, pyio.RawIOBase): + pass + + +class MisbehavedRawIO(MockRawIO): + def write(self, b): + return super().write(b) * 2 + + def read(self, n=None): + return super().read(n) * 2 + + def seek(self, pos, whence): + return -123 + + def tell(self): + return -456 + + def readinto(self, buf): + super().readinto(buf) + return len(buf) * 5 + +class CMisbehavedRawIO(MisbehavedRawIO, io.RawIOBase): + pass + +class PyMisbehavedRawIO(MisbehavedRawIO, pyio.RawIOBase): + pass + + +class SlowFlushRawIO(MockRawIO): + def __init__(self): + super().__init__() + self.in_flush = threading.Event() + + def flush(self): + self.in_flush.set() + time.sleep(0.25) + +class CSlowFlushRawIO(SlowFlushRawIO, io.RawIOBase): + pass + +class PySlowFlushRawIO(SlowFlushRawIO, pyio.RawIOBase): + pass + + +class CloseFailureIO(MockRawIO): + closed = 0 + + def close(self): + if not self.closed: + self.closed = 1 + raise OSError + +class CCloseFailureIO(CloseFailureIO, io.RawIOBase): + pass + +class PyCloseFailureIO(CloseFailureIO, pyio.RawIOBase): + pass + + +class MockFileIO: + + def __init__(self, data): + self.read_history = [] + super().__init__(data) + + def read(self, n=None): + res = super().read(n) + self.read_history.append(None if res is None else len(res)) + return res + + def readinto(self, b): + res = super().readinto(b) + self.read_history.append(res) + return res + +class CMockFileIO(MockFileIO, io.BytesIO): + pass + +class PyMockFileIO(MockFileIO, pyio.BytesIO): + pass + + +class MockUnseekableIO: + def seekable(self): + return False + + def seek(self, *args): + raise self.UnsupportedOperation("not seekable") + + def tell(self, *args): + raise self.UnsupportedOperation("not seekable") + + def truncate(self, *args): + raise self.UnsupportedOperation("not seekable") + +class CMockUnseekableIO(MockUnseekableIO, io.BytesIO): + UnsupportedOperation = io.UnsupportedOperation + +class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): + UnsupportedOperation = pyio.UnsupportedOperation + + +class MockCharPseudoDevFileIO(MockFileIO): + # GH-95782 + # ftruncate() does not work on these special files (and CPython then raises + # appropriate exceptions), so truncate() does not have to be accounted for + # here. + def __init__(self, data): + super().__init__(data) + + def seek(self, *args): + return 0 + + def tell(self, *args): + return 0 + +class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): + pass + +class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): + pass + + +class MockNonBlockWriterIO: + + def __init__(self): + self._write_stack = [] + self._blocker_char = None + + def pop_written(self): + s = b"".join(self._write_stack) + self._write_stack[:] = [] + return s + + def block_on(self, char): + """Block when a given char is encountered.""" + self._blocker_char = char + + def readable(self): + return True + + def seekable(self): + return True + + def seek(self, pos, whence=0): + # naive implementation, enough for tests + return 0 + + def writable(self): + return True + + def write(self, b): + b = bytes(b) + n = -1 + if self._blocker_char: + try: + n = b.index(self._blocker_char) + except ValueError: + pass + else: + if n > 0: + # write data up to the first blocker + self._write_stack.append(b[:n]) + return n + else: + # cancel blocker and indicate would block + self._blocker_char = None + return None + self._write_stack.append(b) + return len(b) + +class CMockNonBlockWriterIO(MockNonBlockWriterIO, io.RawIOBase): + BlockingIOError = io.BlockingIOError + +class PyMockNonBlockWriterIO(MockNonBlockWriterIO, pyio.RawIOBase): + BlockingIOError = pyio.BlockingIOError + + +# Build classes which point to all the right mocks per io implementation +class CTestCase(unittest.TestCase): + io = io + is_C = True + + MockRawIO = CMockRawIO + MisbehavedRawIO = CMisbehavedRawIO + MockFileIO = CMockFileIO + CloseFailureIO = CCloseFailureIO + MockNonBlockWriterIO = CMockNonBlockWriterIO + MockUnseekableIO = CMockUnseekableIO + MockRawIOWithoutRead = CMockRawIOWithoutRead + SlowFlushRawIO = CSlowFlushRawIO + MockCharPseudoDevFileIO = CMockCharPseudoDevFileIO + + # Use the class as a proxy to the io module members. + def __getattr__(self, name): + return getattr(io, name) + + +class PyTestCase(unittest.TestCase): + io = pyio + is_C = False + + MockRawIO = PyMockRawIO + MisbehavedRawIO = PyMisbehavedRawIO + MockFileIO = PyMockFileIO + CloseFailureIO = PyCloseFailureIO + MockNonBlockWriterIO = PyMockNonBlockWriterIO + MockUnseekableIO = PyMockUnseekableIO + MockRawIOWithoutRead = PyMockRawIOWithoutRead + SlowFlushRawIO = PySlowFlushRawIO + MockCharPseudoDevFileIO = PyMockCharPseudoDevFileIO + + # Use the class as a proxy to the _pyio module members. + def __getattr__(self, name): + return getattr(pyio, name) diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py index 7a986048bda..277d2fc99ea 100644 --- a/Lib/test/test_ioctl.py +++ b/Lib/test/test_ioctl.py @@ -5,7 +5,7 @@ import threading import unittest from test import support -from test.support import threading_helper +from test.support import os_helper, threading_helper from test.support.import_helper import import_module fcntl = import_module('fcntl') termios = import_module('termios') @@ -127,9 +127,8 @@ def test_ioctl_mutate_1024(self): self._check_ioctl_not_mutate_len(1024) def test_ioctl_mutate_2048(self): - # Test with a larger buffer, just for the record. self._check_ioctl_mutate_len(2048) - self.assertRaises(ValueError, self._check_ioctl_not_mutate_len, 2048) + self._check_ioctl_not_mutate_len(1024) @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") @@ -202,6 +201,17 @@ def test_ioctl_set_window_size(self): new_winsz = struct.unpack("HHHH", result) self.assertEqual(new_winsz[:2], (20, 40)) + @unittest.skipUnless(hasattr(fcntl, 'FICLONE'), 'need fcntl.FICLONE') + def test_bad_fd(self): + # gh-134744: Test error handling + fd = os_helper.make_bad_fd() + with self.assertRaises(OSError): + fcntl.ioctl(fd, fcntl.FICLONE, fd) + with self.assertRaises(OSError): + fcntl.ioctl(fd, fcntl.FICLONE, b'\0' * 10) + with self.assertRaises(OSError): + fcntl.ioctl(fd, fcntl.FICLONE, b'\0' * 2048) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index d04012d1afd..3f017b97dc2 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -12,6 +12,7 @@ import pickle import ipaddress import weakref +from collections.abc import Iterator from test.support import LARGEST, SMALLEST @@ -397,6 +398,19 @@ def assertBadSplit(addr): # A trailing IPv4 address is two parts assertBadSplit("10:9:8:7:6:5:4:3:42.42.42.42%scope") + def test_bad_address_split_v6_too_long(self): + def assertBadSplit(addr): + msg = r"At most 45 characters expected in '%s" + with self.assertAddressError(msg, re.escape(addr[:45])): + ipaddress.IPv6Address(addr) + + # Long IPv6 address + long_addr = ("0:" * 10000) + "0" + assertBadSplit(long_addr) + assertBadSplit(long_addr + "%zoneid") + assertBadSplit(long_addr + ":255.255.255.255") + assertBadSplit(long_addr + ":ffff:255.255.255.255") + def test_bad_address_split_v6_too_many_parts(self): def assertBadSplit(addr): msg = "Exactly 8 parts expected without '::' in %r" @@ -1459,18 +1473,27 @@ def testGetSupernet4(self): self.ipv6_scoped_network.supernet(new_prefix=62)) def testHosts(self): + hosts = self.ipv4_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), next(hosts)) hosts = list(self.ipv4_network.hosts()) self.assertEqual(254, len(hosts)) self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = ipv6_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), next(hosts)) hosts = list(ipv6_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) ipv6_scoped_network = ipaddress.IPv6Network('2001:658:22a:cafe::%scope/120') + hosts = ipv6_scoped_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual((ipaddress.IPv6Address('2001:658:22a:cafe::1')), next(hosts)) hosts = list(ipv6_scoped_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) @@ -1481,6 +1504,12 @@ def testHosts(self): ipaddress.IPv4Address('2.0.0.1')] str_args = '2.0.0.0/31' tpl_args = ('2.0.0.0', 31) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1490,6 +1519,12 @@ def testHosts(self): addrs = [ipaddress.IPv4Address('1.2.3.4')] str_args = '1.2.3.4/32' tpl_args = ('1.2.3.4', 32) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1499,6 +1534,12 @@ def testHosts(self): ipaddress.IPv6Address('2001:658:22a:cafe::1')] str_args = '2001:658:22a:cafe::/127' tpl_args = ('2001:658:22a:cafe::', 127) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1507,6 +1548,12 @@ def testHosts(self): addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::1'), ] str_args = '2001:658:22a:cafe::1/128' tpl_args = ('2001:658:22a:cafe::1', 128) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -2178,6 +2225,11 @@ def testIPv6AddressTooLarge(self): self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1'), ipaddress.ip_address('FFFF::c000:201')) + self.assertEqual(ipaddress.ip_address('0000:0000:0000:0000:0000:FFFF:192.168.255.255'), + ipaddress.ip_address('::ffff:c0a8:ffff')) + self.assertEqual(ipaddress.ip_address('FFFF:0000:0000:0000:0000:0000:192.168.255.255'), + ipaddress.ip_address('ffff::c0a8:ffff')) + self.assertEqual(ipaddress.ip_address('::FFFF:192.0.2.1%scope'), ipaddress.ip_address('::FFFF:c000:201%scope')) self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1%scope'), @@ -2190,6 +2242,10 @@ def testIPv6AddressTooLarge(self): ipaddress.ip_address('::FFFF:c000:201%scope')) self.assertNotEqual(ipaddress.ip_address('FFFF::192.0.2.1'), ipaddress.ip_address('FFFF::c000:201%scope')) + self.assertEqual(ipaddress.ip_address('0000:0000:0000:0000:0000:FFFF:192.168.255.255%scope'), + ipaddress.ip_address('::ffff:c0a8:ffff%scope')) + self.assertEqual(ipaddress.ip_address('FFFF:0000:0000:0000:0000:0000:192.168.255.255%scope'), + ipaddress.ip_address('ffff::c0a8:ffff%scope')) def testIPVersion(self): self.assertEqual(ipaddress.IPv4Address.version, 4) @@ -2247,6 +2303,10 @@ def testReservedIpv4(self): self.assertEqual(False, ipaddress.ip_network('240.0.0.0').is_multicast) self.assertEqual(True, ipaddress.ip_network('240.0.0.0').is_reserved) + self.assertTrue(ipaddress.ip_interface('0.0.0.0/32').is_unspecified) + self.assertFalse(ipaddress.ip_interface('0.0.0.0/31').is_unspecified) + self.assertFalse(ipaddress.ip_interface('1.2.3.4/32').is_unspecified) + self.assertEqual(True, ipaddress.ip_interface( '192.168.1.1/17').is_private) self.assertEqual(False, ipaddress.ip_network('192.169.0.0').is_private) @@ -2599,6 +2659,10 @@ def testCompressIPv6Address(self): '::7:6:5:4:3:2:0': '0:7:6:5:4:3:2:0/128', '7:6:5:4:3:2:1::': '7:6:5:4:3:2:1:0/128', '0:6:5:4:3:2:1::': '0:6:5:4:3:2:1:0/128', + '0000:0000:0000:0000:0000:0000:255.255.255.255': '::ffff:ffff/128', + '0000:0000:0000:0000:0000:ffff:255.255.255.255': '::ffff:255.255.255.255/128', + 'ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255': + 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128', } for uncompressed, compressed in list(test_addresses.items()): self.assertEqual(compressed, str(ipaddress.IPv6Interface( @@ -2762,6 +2826,44 @@ def testV6HashIsNotConstant(self): ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2") self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__()) + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV4HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.255/32").__hash__(), + ipaddress.IPv4Network("192.168.1.0/24").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("172.24.255.0/24").__hash__(), + ipaddress.IPv4Network("172.24.0.0/16").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.87/32").__hash__(), + ipaddress.IPv4Network("192.168.1.86/31").__hash__() + ) + + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV6HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv6Network("fe80::/64").__hash__(), + ipaddress.IPv6Network("fe80::ffff:ffff:ffff:0/112").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("10.0.0.0/8").__hash__(), + ipaddress.IPv6Network( + "ffff:ffff:ffff:ffff:ffff:ffff:aff:0/112" + ).__hash__() + ) + + +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(ipaddress, "__version__") + self.assertEqual(cm.filename, __file__) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index daad00e8643..f440fc28ee7 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -318,6 +318,7 @@ def __bases__(self): self.assertRaises(RecursionError, isinstance, 1, X()) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_infinite_recursion_via_bases_tuple(self): """Regression test for bpo-30570.""" class Failure(object): @@ -328,6 +329,7 @@ def __getattr__(self, attr): issubclass(Failure(), int) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_infinite_cycle_in_bases(self): """Regression test for bpo-30570.""" class X: diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 1b9f3cf7624..18e4b676c53 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -1147,7 +1147,7 @@ def test_error_iter(self): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: diff --git a/Lib/test/test_json/__init__.py b/Lib/test/test_json/__init__.py index 74b64ed86a3..e58fbee41af 100644 --- a/Lib/test/test_json/__init__.py +++ b/Lib/test/test_json/__init__.py @@ -47,6 +47,16 @@ def test_cjson(self): '_json') +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(json, "__version__") + self.assertEqual(cm.filename, __file__) + + def load_tests(loader, _, pattern): suite = unittest.TestSuite() for mod in (json, json.encoder, json.decoder): diff --git a/Lib/test/test_json/test_dump.py b/Lib/test/test_json/test_dump.py index 13b40020781..39470754003 100644 --- a/Lib/test/test_json/test_dump.py +++ b/Lib/test/test_json/test_dump.py @@ -22,6 +22,14 @@ def test_dump_skipkeys(self): self.assertIn('valid_key', o) self.assertNotIn(b'invalid_key', o) + def test_dump_skipkeys_indent_empty(self): + v = {b'invalid_key': False} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{}') + + def test_skipkeys_indent(self): + v = {b'invalid_key': False, 'valid_key': True} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}') + def test_encode_truefalse(self): self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 6a39b72a09d..c90d3e968e5 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -8,13 +8,12 @@ ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ('controls', '"controls"'), ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), + ('\x00\x1f\x7f', '"\\u0000\\u001f\\u007f"'), ('{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), (' s p a c e d ', '" s p a c e d "'), ('\U0001d120', '"\\ud834\\udd20"'), ('\u03b1\u03a9', '"\\u03b1\\u03a9"'), ("`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] class TestEncodeBasestringAscii: diff --git a/Lib/test/test_json/test_enum.py b/Lib/test/test_json/test_enum.py index 10f414898b8..196229897bd 100644 --- a/Lib/test/test_json/test_enum.py +++ b/Lib/test/test_json/test_enum.py @@ -26,7 +26,7 @@ class FloatNum(float, Enum): NEG_INF = float('-inf') NAN = float('nan') -class WierdNum(float, Enum): +class WeirdNum(float, Enum): inf = INF neg_inf = NEG_INF nan = NAN @@ -40,7 +40,7 @@ def test_floats(self): self.assertEqual(self.loads(self.dumps(enum)), enum) def test_weird_floats(self): - for enum, expected in zip(WierdNum, ('Infinity', '-Infinity', 'NaN')): + for enum, expected in zip(WeirdNum, ('Infinity', '-Infinity', 'NaN')): self.assertEqual(self.dumps(enum), expected) if not isnan(enum): self.assertEqual(float(self.dumps(enum)), enum) @@ -64,16 +64,16 @@ def test_list(self): str([E, PI, TAU])) self.assertEqual(self.loads(self.dumps(list(FloatNum))), list(FloatNum)) - self.assertEqual(self.dumps(list(WierdNum)), + self.assertEqual(self.dumps(list(WeirdNum)), '[Infinity, -Infinity, NaN]') - self.assertEqual(self.loads(self.dumps(list(WierdNum)))[:2], - list(WierdNum)[:2]) - self.assertTrue(isnan(self.loads(self.dumps(list(WierdNum)))[2])) + self.assertEqual(self.loads(self.dumps(list(WeirdNum)))[:2], + list(WeirdNum)[:2]) + self.assertTrue(isnan(self.loads(self.dumps(list(WeirdNum)))[2])) def test_dict_keys(self): s, b, h, r = BigNum e, p, t = FloatNum - i, j, n = WierdNum + i, j, n = WeirdNum d = { s:'tiny', b:'large', h:'larger', r:'largest', e:"Euler's number", p:'pi', t:'tau', @@ -100,9 +100,9 @@ def test_dict_values(self): e=FloatNum.e, pi=FloatNum.pi, tau=FloatNum.tau, - i=WierdNum.inf, - j=WierdNum.neg_inf, - n=WierdNum.nan, + i=WeirdNum.inf, + j=WeirdNum.neg_inf, + n=WeirdNum.nan, ) nd = self.loads(self.dumps(d)) self.assertEqual(nd['tiny'], SMALL) diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 7c1696cc66d..79c44af2fbf 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -102,7 +102,7 @@ def test_not_serializable(self): with self.assertRaisesRegex(TypeError, 'Object of type module is not JSON serializable') as cm: self.dumps(sys) - self.assertFalse(hasattr(cm.exception, '__notes__')) + self.assertNotHasAttr(cm.exception, '__notes__') with self.assertRaises(TypeError) as cm: self.dumps([1, [2, 3, sys]]) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index d82093f3895..5d7b56ff9ad 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -69,6 +69,7 @@ def default(self, o): @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_highly_nested_objects_decoding(self): very_deep = 200000 # test that loading highly-nested objects doesn't segfault when C @@ -85,6 +86,7 @@ def test_highly_nested_objects_decoding(self): @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() + @support.requires_resource('cpu') def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} @@ -98,6 +100,7 @@ def test_highly_nested_objects_encoding(self): self.dumps(d) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index cca556a3b95..9a6cdfe12d2 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -144,7 +144,7 @@ def test_bad_escapes(self): def test_overflow(self): with self.assertRaises(OverflowError): - self.json.decoder.scanstring(b"xxx", sys.maxsize+1) + self.json.decoder.scanstring("xxx", sys.maxsize+1) class TestPyScanstring(TestScanstring, PyTest): pass diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 72cde3f0d6c..30f9bb33316 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -160,7 +160,7 @@ def test_help_flag(self): rc, out, err = assert_python_ok('-m', self.module, '-h', PYTHON_COLORS='0') self.assertEqual(rc, 0) - self.assertTrue(out.startswith(b'usage: ')) + self.assertStartsWith(out, b'usage: ') self.assertEqual(err, b'') def test_sort_keys_flag(self): @@ -270,7 +270,7 @@ def test_colors(self): (r'" \"foo\" "', f'{t.string}" \\"foo\\" "{t.reset}'), ('"α"', f'{t.string}"\\u03b1"{t.reset}'), ('123', f'{t.number}123{t.reset}'), - ('-1.2345e+23', f'{t.number}-1.2345e+23{t.reset}'), + ('-1.25e+23', f'{t.number}-1.25e+23{t.reset}'), (r'{"\\": ""}', f'''\ {ob} diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 68629cceeb9..1aa9546dc46 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -32,6 +32,29 @@ def test_encoding7(self): j = self.dumps(u + "\n", ensure_ascii=False) self.assertEqual(j, f'"{u}\\n"') + def test_ascii_non_printable_encode(self): + u = '\b\t\n\f\r\x00\x1f\x7f' + self.assertEqual(self.dumps(u), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\\u007f"') + self.assertEqual(self.dumps(u, ensure_ascii=False), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\x7f"') + + def test_ascii_non_printable_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), + '\b\t\n\f\r') + s = ''.join(map(chr, range(32))) + for c in s: + self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') + self.assertEqual(self.loads(f'"{s}"', strict=False), s) + self.assertEqual(self.loads('"\x7f"'), '\x7f') + + def test_escaped_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), '\b\t\n\f\r') + self.assertEqual(self.loads('"\\"\\\\\\/"'), '"\\/') + for c in set(map(chr, range(0x100))) - set('"\\/bfnrt'): + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"') + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"', strict=False) + def test_big_unicode_encode(self): u = '\U0001d120' self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') @@ -48,6 +71,18 @@ def test_unicode_decode(self): s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) + def test_single_surrogate_encode(self): + self.assertEqual(self.dumps('\uD83D'), '"\\ud83d"') + self.assertEqual(self.dumps('\uD83D', ensure_ascii=False), '"\ud83d"') + self.assertEqual(self.dumps('\uDC0D'), '"\\udc0d"') + self.assertEqual(self.dumps('\uDC0D', ensure_ascii=False), '"\udc0d"') + + def test_single_surrogate_decode(self): + self.assertEqual(self.loads('"\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\udc0d"'), '\udc0d') + self.assertEqual(self.loads('"\\udc0d"'), '\udc0d') + def test_unicode_preservation(self): self.assertEqual(type(self.loads('""')), str) self.assertEqual(type(self.loads('"a"')), str) diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index e94edcbc107..d2ab45c4a5b 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -9,6 +9,8 @@ import time import unittest +from test.support import warnings_helper + if not hasattr(select, "kqueue"): raise unittest.SkipTest("test works only on BSD") @@ -257,6 +259,7 @@ def test_fd_non_inheritable(self): self.addCleanup(kqueue.close) self.assertEqual(os.get_inheritable(kqueue.fileno()), False) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_fork() def test_fork(self): # gh-110395: kqueue objects must be closed after fork diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 173fc743cf6..caa1603c78e 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -443,7 +443,7 @@ def test_search_major_3(self): except subprocess.CalledProcessError: raise unittest.SkipTest("requires at least one Python 3.x install") self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("3."), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "3.") def test_search_major_3_32(self): try: @@ -453,8 +453,8 @@ def test_search_major_3_32(self): raise unittest.SkipTest("requires at least one 32-bit Python 3.x install") raise self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("3."), data["env.tag"]) - self.assertTrue(data["env.tag"].endswith("-32"), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "3.") + self.assertEndsWith(data["env.tag"], "-32") def test_search_major_2(self): try: @@ -463,7 +463,7 @@ def test_search_major_2(self): if not is_installed("2.7"): raise unittest.SkipTest("requires at least one Python 2.x install") self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("2."), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "2.") def test_py_default(self): with self.py_ini(TEST_PY_DEFAULTS): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e4aa41ebb43..fcd94edc611 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -4,10 +4,12 @@ import unittest import os.path import tempfile +import threading import tokenize from importlib.machinery import ModuleSpec from test import support from test.support import os_helper +from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -257,22 +259,44 @@ def raise_memoryerror(*args, **kwargs): def test_loader(self): filename = 'scheme://path' - for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': None} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + for loader in object(), NoSourceLoader(): linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': loader} - self.assertEqual(linecache.getlines(filename, module_globals), []) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), []) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} - self.assertEqual(linecache.getlines(filename, module_globals), - ['source for a.b.c\n']) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') - for spec in (None, object(), ModuleSpec('', FakeLoader())): + for spec in None, object(): linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), '__spec__': spec} + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': ModuleSpec('', FakeLoader())} + with self.assertWarns(DeprecationWarning) as w: self.assertEqual(linecache.getlines(filename, module_globals), ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals; __loader__ != __spec__.loader') linecache.clearcache() spec = ModuleSpec('x.y.z', FakeLoader()) @@ -374,5 +398,40 @@ def test_checkcache_with_no_parameter(self): self.assertIn(self.unchanged_file, linecache.cache) +class MultiThreadingTest(unittest.TestCase): + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_read_write_safety(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + filenames = [] + for i in range(10): + name = os.path.join(tmpdirname, f"test_{i}.py") + with open(name, "w") as h: + h.write("import time\n") + h.write("import system\n") + filenames.append(name) + + def linecache_get_line(b): + b.wait() + for _ in range(100): + for name in filenames: + linecache.getline(name, 1) + + def check(funcs): + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + check([linecache_get_line] * 20) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 6894fba2ad1..642b54d3484 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -349,10 +349,12 @@ def test_deopt_from_append_list(self): # gh-132011: it used to crash, because # of `CALL_LIST_APPEND` specialization failure. code = textwrap.dedent(""" + import _testinternalcapi + l = [] def lappend(l, x, y): l.append((x, y)) - for x in range(3): + for x in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): lappend(l, None, None) try: lappend(list, None, None) @@ -365,5 +367,20 @@ def lappend(l, x, y): rc, _, _ = assert_python_ok("-c", code) self.assertEqual(rc, 0) + def test_list_overwrite_local(self): + """Test that overwriting the last reference to the + iterable doesn't prematurely free the iterable""" + + def foo(x): + self.assertEqual(sys.getrefcount(x), 1) + r = 0 + for i in x: + r += i + x = None + return r + + self.assertEqual(foo(list(range(10))), 45) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index cffdeeacc5d..70148dc30fc 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -716,7 +716,7 @@ def test_multiple_comprehension_name_reuse(self): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 455d2af37ef..d49f78c91da 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,10 +1,12 @@ from decimal import Decimal +from test import support from test.support import cpython_only, verbose, is_android, linked_to_musl, os_helper from test.support.warnings_helper import check_warnings from test.support.import_helper import ensure_lazy_imports, import_fresh_module from unittest import mock import unittest import locale +import os import sys import codecs @@ -349,8 +351,7 @@ def setUp(self): enc = codecs.lookup(locale.getencoding() or 'ascii').name if enc not in ('utf-8', 'iso8859-1', 'cp1252'): raise unittest.SkipTest('encoding not suitable') - if enc != 'iso8859-1' and (sys.platform == 'darwin' or is_android or - sys.platform.startswith('freebsd')): + if enc != 'iso8859-1' and is_android: raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs') BaseLocalizedTest.setUp(self) @@ -387,6 +388,10 @@ def test_c(self): self.check('c', 'C') self.check('posix', 'C') + def test_c_utf8(self): + self.check('c.utf8', 'C.UTF-8') + self.check('C.UTF-8', 'C.UTF-8') + def test_english(self): self.check('en', 'en_US.ISO8859-1') self.check('EN', 'en_US.ISO8859-1') @@ -420,8 +425,8 @@ def test_hyphenated_encoding(self): self.check('cs_CZ.ISO8859-2', 'cs_CZ.ISO8859-2') def test_euro_modifier(self): - self.check('de_DE@euro', 'de_DE.ISO8859-15') - self.check('en_US.ISO8859-15@euro', 'en_US.ISO8859-15') + self.check('de_DE@euro', 'de_DE.ISO8859-15@euro') + self.check('en_US.ISO8859-15@euro', 'en_US.ISO8859-15@euro') self.check('de_DE.utf8@euro', 'de_DE.UTF-8') def test_latin_modifier(self): @@ -482,6 +487,153 @@ def test_japanese(self): self.check('jp_jp', 'ja_JP.eucJP') +class TestRealLocales(unittest.TestCase): + def setUp(self): + oldlocale = locale.setlocale(locale.LC_CTYPE) + self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) + + def test_getsetlocale_issue1813(self): + # Issue #1813: setting and getting the locale under a Turkish locale + try: + locale.setlocale(locale.LC_CTYPE, 'tr_TR') + except locale.Error: + # Unsupported locale on this system + self.skipTest('test needs Turkish locale') + loc = locale.getlocale(locale.LC_CTYPE) + if verbose: + print('testing with %a' % (loc,), end=' ', flush=True) + try: + locale.setlocale(locale.LC_CTYPE, loc) + except locale.Error as exc: + # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) + # and the tr_TR locale on Windows. getlocale() builds a locale + # which is not recognize by setlocale(). + self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") + self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) + + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + def test_setlocale_long_encoding(self): + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'English.%016d' % 1252) + locale.setlocale(locale.LC_CTYPE, 'English.%015d' % 1252) + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.1252', loc) + loc2 = loc.replace('.1252', '.%016d' % 1252, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + loc2 = loc.replace('.1252', '.%015d' % 1252, 1) + locale.setlocale(locale.LC_ALL, loc2) + + # gh-137273: Debug assertion failure on Windows for long encoding. + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'en_US.' + 'x'*16) + locale.setlocale(locale.LC_CTYPE, 'en_US.UTF-8') + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.UTF-8', loc) + loc2 = loc.replace('.UTF-8', '.' + 'x'*16, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + + @support.subTests('localename,localetuple', [ + ('fr_FR.ISO8859-15@euro', ('fr_FR@euro', 'iso885915')), + ('fr_FR.ISO8859-15@euro', ('fr_FR@euro', 'iso88591')), + ('fr_FR.ISO8859-15@euro', ('fr_FR@euro', 'ISO8859-15')), + ('fr_FR.ISO8859-15@euro', ('fr_FR@euro', 'ISO8859-1')), + ('fr_FR.ISO8859-15@euro', ('fr_FR@euro', None)), + ('de_DE.ISO8859-15@euro', ('de_DE@euro', 'iso885915')), + ('de_DE.ISO8859-15@euro', ('de_DE@euro', 'iso88591')), + ('de_DE.ISO8859-15@euro', ('de_DE@euro', 'ISO8859-15')), + ('de_DE.ISO8859-15@euro', ('de_DE@euro', 'ISO8859-1')), + ('de_DE.ISO8859-15@euro', ('de_DE@euro', None)), + ('el_GR.ISO8859-7@euro', ('el_GR@euro', 'iso88597')), + ('el_GR.ISO8859-7@euro', ('el_GR@euro', 'ISO8859-7')), + ('el_GR.ISO8859-7@euro', ('el_GR@euro', None)), + ('ca_ES.ISO8859-15@euro', ('ca_ES@euro', 'iso885915')), + ('ca_ES.ISO8859-15@euro', ('ca_ES@euro', 'iso88591')), + ('ca_ES.ISO8859-15@euro', ('ca_ES@euro', 'ISO8859-15')), + ('ca_ES.ISO8859-15@euro', ('ca_ES@euro', 'ISO8859-1')), + ('ca_ES.ISO8859-15@euro', ('ca_ES@euro', None)), + ('ca_ES.UTF-8@valencia', ('ca_ES@valencia', 'utf8')), + ('ca_ES.UTF-8@valencia', ('ca_ES@valencia', 'UTF-8')), + ('ca_ES.UTF-8@valencia', ('ca_ES@valencia', None)), + ('ks_IN.UTF-8@devanagari', ('ks_IN@devanagari', 'utf8')), + ('ks_IN.UTF-8@devanagari', ('ks_IN@devanagari', 'UTF-8')), + ('ks_IN.UTF-8@devanagari', ('ks_IN@devanagari', None)), + ('sd_IN.UTF-8@devanagari', ('sd_IN@devanagari', 'utf8')), + ('sd_IN.UTF-8@devanagari', ('sd_IN@devanagari', 'UTF-8')), + ('sd_IN.UTF-8@devanagari', ('sd_IN@devanagari', None)), + ('be_BY.UTF-8@latin', ('be_BY@latin', 'utf8')), + ('be_BY.UTF-8@latin', ('be_BY@latin', 'UTF-8')), + ('be_BY.UTF-8@latin', ('be_BY@latin', None)), + ('sr_RS.UTF-8@latin', ('sr_RS@latin', 'utf8')), + ('sr_RS.UTF-8@latin', ('sr_RS@latin', 'UTF-8')), + ('sr_RS.UTF-8@latin', ('sr_RS@latin', None)), + ('ug_CN.UTF-8@latin', ('ug_CN@latin', 'utf8')), + ('ug_CN.UTF-8@latin', ('ug_CN@latin', 'UTF-8')), + ('ug_CN.UTF-8@latin', ('ug_CN@latin', None)), + ('uz_UZ.UTF-8@cyrillic', ('uz_UZ@cyrillic', 'utf8')), + ('uz_UZ.UTF-8@cyrillic', ('uz_UZ@cyrillic', 'UTF-8')), + ('uz_UZ.UTF-8@cyrillic', ('uz_UZ@cyrillic', None)), + ]) + def test_setlocale_with_modifier(self, localename, localetuple): + try: + locale.setlocale(locale.LC_CTYPE, localename) + except locale.Error as exc: + self.skipTest(str(exc)) + loc = locale.setlocale(locale.LC_CTYPE, localetuple) + self.assertEqual(loc, localename) + + loctuple = locale.getlocale(locale.LC_CTYPE) + loc = locale.setlocale(locale.LC_CTYPE, loctuple) + self.assertEqual(loc, localename) + + @support.subTests('localename,localetuple', [ + ('fr_FR.iso885915@euro', ('fr_FR@euro', 'ISO8859-15')), + ('fr_FR.ISO8859-15@euro', ('fr_FR@euro', 'ISO8859-15')), + ('fr_FR@euro', ('fr_FR@euro', 'ISO8859-15')), + ('de_DE.iso885915@euro', ('de_DE@euro', 'ISO8859-15')), + ('de_DE.ISO8859-15@euro', ('de_DE@euro', 'ISO8859-15')), + ('de_DE@euro', ('de_DE@euro', 'ISO8859-15')), + ('el_GR.iso88597@euro', ('el_GR@euro', 'ISO8859-7')), + ('el_GR.ISO8859-7@euro', ('el_GR@euro', 'ISO8859-7')), + ('el_GR@euro', ('el_GR@euro', 'ISO8859-7')), + ('ca_ES.iso885915@euro', ('ca_ES@euro', 'ISO8859-15')), + ('ca_ES.ISO8859-15@euro', ('ca_ES@euro', 'ISO8859-15')), + ('ca_ES@euro', ('ca_ES@euro', 'ISO8859-15')), + ('ca_ES.utf8@valencia', ('ca_ES@valencia', 'UTF-8')), + ('ca_ES.UTF-8@valencia', ('ca_ES@valencia', 'UTF-8')), + ('ca_ES@valencia', ('ca_ES@valencia', 'UTF-8')), + ('ks_IN.utf8@devanagari', ('ks_IN@devanagari', 'UTF-8')), + ('ks_IN.UTF-8@devanagari', ('ks_IN@devanagari', 'UTF-8')), + ('ks_IN@devanagari', ('ks_IN@devanagari', 'UTF-8')), + ('sd_IN.utf8@devanagari', ('sd_IN@devanagari', 'UTF-8')), + ('sd_IN.UTF-8@devanagari', ('sd_IN@devanagari', 'UTF-8')), + ('sd_IN@devanagari', ('sd_IN@devanagari', 'UTF-8')), + ('be_BY.utf8@latin', ('be_BY@latin', 'UTF-8')), + ('be_BY.UTF-8@latin', ('be_BY@latin', 'UTF-8')), + ('be_BY@latin', ('be_BY@latin', 'UTF-8')), + ('sr_RS.utf8@latin', ('sr_RS@latin', 'UTF-8')), + ('sr_RS.UTF-8@latin', ('sr_RS@latin', 'UTF-8')), + ('sr_RS@latin', ('sr_RS@latin', 'UTF-8')), + ('ug_CN.utf8@latin', ('ug_CN@latin', 'UTF-8')), + ('ug_CN.UTF-8@latin', ('ug_CN@latin', 'UTF-8')), + ('ug_CN@latin', ('ug_CN@latin', 'UTF-8')), + ('uz_UZ.utf8@cyrillic', ('uz_UZ@cyrillic', 'UTF-8')), + ('uz_UZ.UTF-8@cyrillic', ('uz_UZ@cyrillic', 'UTF-8')), + ('uz_UZ@cyrillic', ('uz_UZ@cyrillic', 'UTF-8')), + ]) + def test_getlocale_with_modifier(self, localename, localetuple): + try: + locale.setlocale(locale.LC_CTYPE, localename) + except locale.Error as exc: + self.skipTest(str(exc)) + loctuple = locale.getlocale(locale.LC_CTYPE) + self.assertEqual(loctuple, localetuple) + + locale.setlocale(locale.LC_CTYPE, loctuple) + self.assertEqual(locale.getlocale(locale.LC_CTYPE), localetuple) + + class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is @@ -548,27 +700,6 @@ def test_setlocale_category(self): # crasher from bug #7419 self.assertRaises(locale.Error, locale.setlocale, 12345) - def test_getsetlocale_issue1813(self): - # Issue #1813: setting and getting the locale under a Turkish locale - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') - loc = locale.getlocale(locale.LC_CTYPE) - if verbose: - print('testing with %a' % (loc,), end=' ', flush=True) - try: - locale.setlocale(locale.LC_CTYPE, loc) - except locale.Error as exc: - # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) - # and the tr_TR locale on Windows. getlocale() builds a locale - # which is not recognize by setlocale(). - self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") - self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) - def test_invalid_locale_format_in_localetuple(self): with self.assertRaises(TypeError): locale.setlocale(locale.LC_ALL, b'fi_FI') diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 3f113ec1be4..8815426fc99 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -61,7 +61,7 @@ import weakref from http.server import HTTPServer, BaseHTTPRequestHandler -from unittest.mock import patch +from unittest.mock import call, Mock, patch from urllib.parse import urlparse, parse_qs from socketserver import (ThreadingUDPServer, DatagramRequestHandler, ThreadingTCPServer, StreamRequestHandler) @@ -730,6 +730,7 @@ def remove_loop(fname, tries): # based on os.fork existing because that is what users and this test use. # This helps ensure that when fork exists (the important concept) that the # register_at_fork mechanism is also present and used. + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork @@ -1036,7 +1037,7 @@ class TestTCPServer(ControlMixin, ThreadingTCPServer): """ allow_reuse_address = True - allow_reuse_port = True + allow_reuse_port = False def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): @@ -4045,6 +4046,7 @@ def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_man self._apply_simple_queue_listener_configuration(qspec) manager.assert_not_called() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @skip_if_tsan_fork @support.requires_subprocess() @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" @@ -4067,6 +4069,7 @@ def test_config_reject_simple_queue_handler_multiprocessing_context(self): with self.assertRaises(ValueError): self._apply_simple_queue_listener_configuration(qspec) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @skip_if_tsan_fork @support.requires_subprocess() @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" @@ -4107,6 +4110,7 @@ def _mpinit_issue121723(qspec, message_to_log): # log a message (this creates a record put in the queue) logging.getLogger().info(message_to_log) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @skip_if_tsan_fork @support.requires_subprocess() def test_multiprocessing_queues(self): @@ -5337,6 +5341,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): else: return results + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @skip_if_tsan_fork def test_multiprocessing(self): support.skip_if_broken_multiprocessing_synchronize() @@ -5421,7 +5426,7 @@ def test_taskName_with_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @support.requires_working_socket() def test_taskName_without_asyncio_imported(self): @@ -5433,7 +5438,7 @@ def test_taskName_without_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class BasicConfigTest(unittest.TestCase): @@ -5572,12 +5577,19 @@ def test_incompatible(self): assertRaises = self.assertRaises handlers = [logging.StreamHandler()] stream = sys.stderr + formatter = logging.Formatter() assertRaises(ValueError, logging.basicConfig, filename='test.log', stream=stream) assertRaises(ValueError, logging.basicConfig, filename='test.log', handlers=handlers) assertRaises(ValueError, logging.basicConfig, stream=stream, handlers=handlers) + assertRaises(ValueError, logging.basicConfig, formatter=formatter, + format='%(message)s') + assertRaises(ValueError, logging.basicConfig, formatter=formatter, + datefmt='%H:%M:%S') + assertRaises(ValueError, logging.basicConfig, formatter=formatter, + style='%') # Issue 23207: test for invalid kwargs assertRaises(ValueError, logging.basicConfig, loglevel=logging.INFO) # Should pop both filename and filemode even if filename is None @@ -5712,6 +5724,20 @@ def dummy_handle_error(record): # didn't write anything due to the encoding error self.assertEqual(data, r'') + def test_formatter_given(self): + mock_formatter = Mock() + mock_handler = Mock(formatter=None) + with patch("logging.Formatter") as mock_formatter_init: + logging.basicConfig(formatter=mock_formatter, handlers=[mock_handler]) + self.assertEqual(mock_handler.setFormatter.call_args_list, [call(mock_formatter)]) + self.assertEqual(mock_formatter_init.call_count, 0) + + def test_formatter_not_given(self): + mock_handler = Mock(formatter=None) + with patch("logging.Formatter") as mock_formatter_init: + logging.basicConfig(handlers=[mock_handler]) + self.assertEqual(mock_formatter_init.call_count, 1) + @support.requires_working_socket() def test_log_taskName(self): async def log_record(): @@ -5737,7 +5763,7 @@ async def log_record(): data = f.read().strip() self.assertRegex(data, r'Task-\d+ - hello world') finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) if handler: handler.close() @@ -5800,7 +5826,7 @@ def cleanup(): self.addCleanup(cleanup) self.addCleanup(logging.shutdown) - self.adapter = logging.LoggerAdapter(logger=self.logger, extra=None) + self.adapter = logging.LoggerAdapter(logger=self.logger) def test_exception(self): msg = 'testing exception: %r' @@ -5971,6 +5997,18 @@ def test_extra_merged(self): self.assertEqual(record.foo, '1') self.assertEqual(record.bar, '2') + self.adapter.critical('no extra') # should not fail + self.assertEqual(len(self.recording.records), 2) + record = self.recording.records[-1] + self.assertEqual(record.foo, '1') + self.assertNotHasAttr(record, 'bar') + + self.adapter.critical('none extra', extra=None) # should not fail + self.assertEqual(len(self.recording.records), 3) + record = self.recording.records[-1] + self.assertEqual(record.foo, '1') + self.assertNotHasAttr(record, 'bar') + def test_extra_merged_log_call_has_precedence(self): self.adapter = logging.LoggerAdapter(logger=self.logger, extra={'foo': '1'}, @@ -5982,6 +6020,25 @@ def test_extra_merged_log_call_has_precedence(self): self.assertHasAttr(record, 'foo') self.assertEqual(record.foo, '2') + def test_extra_merged_without_extra(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + merge_extra=True) + + self.adapter.critical('foo should be here', extra={'foo': '1'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[-1] + self.assertEqual(record.foo, '1') + + self.adapter.critical('no extra') # should not fail + self.assertEqual(len(self.recording.records), 2) + record = self.recording.records[-1] + self.assertNotHasAttr(record, 'foo') + + self.adapter.critical('none extra', extra=None) # should not fail + self.assertEqual(len(self.recording.records), 3) + record = self.recording.records[-1] + self.assertNotHasAttr(record, 'foo') + class PrefixAdapter(logging.LoggerAdapter): prefix = 'Adapter' @@ -7210,6 +7267,16 @@ def test__all__(self): support.check__all__(self, logging, not_exported=not_exported) +class TestModule(unittest.TestCase): + def test_deprecated__version__and__date__(self): + msg = "is deprecated and slated for removal in Python 3.20" + for attr in ("__version__", "__date__"): + with self.subTest(attr=attr): + with self.assertWarnsRegex(DeprecationWarning, msg) as cm: + getattr(logging, attr) + self.assertEqual(cm.filename, __file__) + + # Set the locale to the platform-dependent default. I have no idea # why the test does this, but in any case we save the current locale # first and restore it at the end. diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index f336d49fa4f..b48a8812a1a 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1374,17 +1374,22 @@ def equivalent_python(n, length, byteorder, signed=False): check(tests4, 'little', signed=False) self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=False) - self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=True) self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=False) - self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (128).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (128).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (-129).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (-129).to_bytes, 1, 'little', signed=True) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False) self.assertEqual((0).to_bytes(0, 'big'), b'') + self.assertEqual((0).to_bytes(0, 'big', signed=True), b'') self.assertEqual((1).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x01') self.assertEqual((0).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x00') self.assertEqual((-1).to_bytes(5, 'big', signed=True), b'\xff\xff\xff\xff\xff') self.assertRaises(OverflowError, (1).to_bytes, 0, 'big') + self.assertRaises(OverflowError, (-1).to_bytes, 0, 'big', signed=True) + self.assertRaises(OverflowError, (-1).to_bytes, 0, 'little', signed=True) # gh-98783 class SubStr(str): @@ -1693,5 +1698,21 @@ class MyInt(int): # GH-117195 -- This shouldn't crash object.__sizeof__(1) + def test_hash(self): + # gh-136599 + self.assertEqual(hash(-1), -2) + self.assertEqual(hash(0), 0) + self.assertEqual(hash(10), 10) + + self.assertEqual(hash(sys.hash_info.modulus - 2), sys.hash_info.modulus - 2) + self.assertEqual(hash(sys.hash_info.modulus - 1), sys.hash_info.modulus - 1) + self.assertEqual(hash(sys.hash_info.modulus), 0) + self.assertEqual(hash(sys.hash_info.modulus + 1), 1) + + self.assertEqual(hash(-sys.hash_info.modulus - 2), -2) + self.assertEqual(hash(-sys.hash_info.modulus - 1), -2) + self.assertEqual(hash(-sys.hash_info.modulus), 0) + self.assertEqual(hash(-sys.hash_info.modulus + 1), -sys.hash_info.modulus + 1) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 9ffb93e797d..e93c3c37354 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -1025,12 +1025,12 @@ def test_peek(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: result = f.peek() self.assertGreater(len(result), 0) - self.assertTrue(INPUT.startswith(result)) + self.assertStartsWith(INPUT, result) self.assertEqual(f.read(), INPUT) with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: result = f.peek(10) self.assertGreater(len(result), 0) - self.assertTrue(INPUT.startswith(result)) + self.assertStartsWith(INPUT, result) self.assertEqual(f.read(), INPUT) def test_peek_bad_args(self): diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 0169948e453..288b2c4496f 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -8,7 +8,7 @@ import io import tempfile from test import support -from test.support import import_helper +from test.support import import_helper, warnings_helper from test.support import os_helper from test.support import refleak_helper from test.support import socket_helper @@ -1212,6 +1212,7 @@ def test_add_and_close(self): self.assertEqual(contents, f.read()) self._box = self._factory(self._path) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_fork() @unittest.skipUnless(hasattr(socket, 'socketpair'), "Test needs socketpair().") def test_lock_conflict(self): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 8b1fb0eba1f..662bdfccc79 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -43,6 +43,11 @@ def test_ints(self): for expected in (-n, n): self.helper(expected) n = n >> 1 + n = 1 << 100 + while n: + for expected in (-n, -n+1, n-1, n): + self.helper(expected) + n = n >> 1 def test_int64(self): # Simulate int marshaling with TYPE_INT64. diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 913a60bf9e0..68f41a2e620 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -17,6 +17,7 @@ eps = 1E-05 NAN = float('nan') +NNAN = float('-nan') INF = float('inf') NINF = float('-inf') FLOAT_MAX = sys.float_info.max @@ -54,56 +55,6 @@ def to_ulps(x): return n -# Here's a pure Python version of the math.factorial algorithm, for -# documentation and comparison purposes. -# -# Formula: -# -# factorial(n) = factorial_odd_part(n) << (n - count_set_bits(n)) -# -# where -# -# factorial_odd_part(n) = product_{i >= 0} product_{0 < j <= n >> i; j odd} j -# -# The outer product above is an infinite product, but once i >= n.bit_length, -# (n >> i) < 1 and the corresponding term of the product is empty. So only the -# finitely many terms for 0 <= i < n.bit_length() contribute anything. -# -# We iterate downwards from i == n.bit_length() - 1 to i == 0. The inner -# product in the formula above starts at 1 for i == n.bit_length(); for each i -# < n.bit_length() we get the inner product for i from that for i + 1 by -# multiplying by all j in {n >> i+1 < j <= n >> i; j odd}. In Python terms, -# this set is range((n >> i+1) + 1 | 1, (n >> i) + 1 | 1, 2). - -def count_set_bits(n): - """Number of '1' bits in binary expansion of a nonnnegative integer.""" - return 1 + count_set_bits(n & n - 1) if n else 0 - -def partial_product(start, stop): - """Product of integers in range(start, stop, 2), computed recursively. - start and stop should both be odd, with start <= stop. - - """ - numfactors = (stop - start) >> 1 - if not numfactors: - return 1 - elif numfactors == 1: - return start - else: - mid = (start + numfactors) | 1 - return partial_product(start, mid) * partial_product(mid, stop) - -def py_factorial(n): - """Factorial of nonnegative integer n, via "Binary Split Factorial Formula" - described at http://www.luschny.de/math/factorial/binarysplitfact.html - - """ - inner = outer = 1 - for i in reversed(range(n.bit_length())): - inner *= partial_product((n >> i + 1) + 1 | 1, (n >> i) + 1 | 1) - outer *= inner - return outer << (n - count_set_bits(n)) - def ulp_abs_check(expected, got, ulp_tol, abs_tol): """Given finite floats `expected` and `got`, check that they're approximately equal to within the given number of ulps or the @@ -238,6 +189,22 @@ def __init__(self, value): def __index__(self): return self.value +class IndexableFloatLike: + def __init__(self, float_value, index_value): + self.float_value = float_value + self.index_value = index_value + + def __float__(self): + if isinstance(self.float_value, BaseException): + raise self.float_value + return self.float_value + + def __index__(self): + if isinstance(self.index_value, BaseException): + raise self.index_value + return self.index_value + + class BadDescr: def __get__(self, obj, objtype=None): raise ValueError @@ -475,6 +442,19 @@ def testCopysign(self): # similarly, copysign(2., NAN) could be 2. or -2. self.assertEqual(abs(math.copysign(2., NAN)), 2.) + def test_signbit(self): + self.assertRaises(TypeError, math.signbit) + self.assertRaises(TypeError, math.signbit, '1.0') + + # C11, §7.12.3.6 requires signbit() to return a nonzero value + # if and only if the sign of its argument value is negative, + # but in practice, we are only interested in a boolean value. + self.assertIsInstance(math.signbit(1.0), bool) + + for arg in [0., 1., INF, NAN]: + self.assertFalse(math.signbit(arg)) + self.assertTrue(math.signbit(-arg)) + def testCos(self): self.assertRaises(TypeError, math.cos) self.ftest('cos(-pi/2)', math.cos(-math.pi/2), 0, abs_tol=math.ulp(1)) @@ -533,33 +513,6 @@ def testFabs(self): self.ftest('fabs(0)', math.fabs(0), 0) self.ftest('fabs(1)', math.fabs(1), 1) - def testFactorial(self): - self.assertEqual(math.factorial(0), 1) - total = 1 - for i in range(1, 1000): - total *= i - self.assertEqual(math.factorial(i), total) - self.assertEqual(math.factorial(i), py_factorial(i)) - self.assertRaises(ValueError, math.factorial, -1) - self.assertRaises(ValueError, math.factorial, -10**100) - - def testFactorialNonIntegers(self): - self.assertRaises(TypeError, math.factorial, 5.0) - self.assertRaises(TypeError, math.factorial, 5.2) - self.assertRaises(TypeError, math.factorial, -1.0) - self.assertRaises(TypeError, math.factorial, -1e100) - self.assertRaises(TypeError, math.factorial, decimal.Decimal('5')) - self.assertRaises(TypeError, math.factorial, decimal.Decimal('5.2')) - self.assertRaises(TypeError, math.factorial, "5") - - # Other implementations may place different upper bounds. - @support.cpython_only - def testFactorialHugeInputs(self): - # Currently raises OverflowError for inputs that are too large - # to fit into a C long. - self.assertRaises(OverflowError, math.factorial, 10**100) - self.assertRaises(TypeError, math.factorial, 1e100) - def testFloor(self): self.assertRaises(TypeError, math.floor) self.assertEqual(int, type(math.floor(0.5))) @@ -623,6 +576,92 @@ def testFmod(self): self.assertEqual(math.fmod(0.0, NINF), 0.0) self.assertRaises(ValueError, math.fmod, INF, INF) + def test_fmax(self): + self.assertRaises(TypeError, math.fmax) + self.assertRaises(TypeError, math.fmax, 'x', 'y') + + self.assertEqual(math.fmax(0., 0.), 0.) + self.assertEqual(math.fmax(1., 2.), 2.) + self.assertEqual(math.fmax(2., 1.), 2.) + + self.assertEqual(math.fmax(+1., +0.), 1.) + self.assertEqual(math.fmax(+0., +1.), 1.) + self.assertEqual(math.fmax(+1., -0.), 1.) + self.assertEqual(math.fmax(-0., +1.), 1.) + + self.assertEqual(math.fmax(-1., +0.), 0.) + self.assertEqual(math.fmax(+0., -1.), 0.) + self.assertEqual(math.fmax(-1., -0.), 0.) + self.assertEqual(math.fmax(-0., -1.), 0.) + + for x in [NINF, -1., -0., 0., 1., INF]: + self.assertFalse(math.isnan(x)) + + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertEqual(math.fmax(INF, x), INF) + self.assertEqual(math.fmax(x, INF), INF) + self.assertEqual(math.fmax(NINF, x), x) + self.assertEqual(math.fmax(x, NINF), x) + + @requires_IEEE_754 + def test_fmax_nans(self): + # When exactly one operand is NaN, the other is returned. + for x in [NINF, -1., -0., 0., 1., INF]: + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertFalse(math.isnan(math.fmax(NAN, x))) + self.assertFalse(math.isnan(math.fmax(x, NAN))) + self.assertFalse(math.isnan(math.fmax(NNAN, x))) + self.assertFalse(math.isnan(math.fmax(x, NNAN))) + # When both operands are NaNs, fmax() returns NaN (see C11, F.10.9.2) + # whose sign is implementation-defined (see C11, F.10.0.3). + self.assertTrue(math.isnan(math.fmax(NAN, NAN))) + self.assertTrue(math.isnan(math.fmax(NNAN, NNAN))) + self.assertTrue(math.isnan(math.fmax(NAN, NNAN))) + self.assertTrue(math.isnan(math.fmax(NNAN, NAN))) + + def test_fmin(self): + self.assertRaises(TypeError, math.fmin) + self.assertRaises(TypeError, math.fmin, 'x', 'y') + + self.assertEqual(math.fmin(0., 0.), 0.) + self.assertEqual(math.fmin(1., 2.), 1.) + self.assertEqual(math.fmin(2., 1.), 1.) + + self.assertEqual(math.fmin(+1., +0.), 0.) + self.assertEqual(math.fmin(+0., +1.), 0.) + self.assertEqual(math.fmin(+1., -0.), 0.) + self.assertEqual(math.fmin(-0., +1.), 0.) + + self.assertEqual(math.fmin(-1., +0.), -1.) + self.assertEqual(math.fmin(+0., -1.), -1.) + self.assertEqual(math.fmin(-1., -0.), -1.) + self.assertEqual(math.fmin(-0., -1.), -1.) + + for x in [NINF, -1., -0., 0., 1., INF]: + self.assertFalse(math.isnan(x)) + + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertEqual(math.fmin(INF, x), x) + self.assertEqual(math.fmin(x, INF), x) + self.assertEqual(math.fmin(NINF, x), NINF) + self.assertEqual(math.fmin(x, NINF), NINF) + + @requires_IEEE_754 + def test_fmin_nans(self): + # When exactly one operand is NaN, the other is returned. + for x in [NINF, -1., -0., 0., 1., INF]: + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertFalse(math.isnan(math.fmin(NAN, x))) + self.assertFalse(math.isnan(math.fmin(x, NAN))) + self.assertFalse(math.isnan(math.fmin(NNAN, x))) + self.assertFalse(math.isnan(math.fmin(x, NNAN))) + # When both operands are NaNs, fmin() returns NaN (see C11, F.10.9.3) + # whose sign is implementation-defined (see C11, F.10.0.3). + self.assertTrue(math.isnan(math.fmin(NAN, NAN))) + self.assertTrue(math.isnan(math.fmin(NNAN, NNAN))) + self.assertTrue(math.isnan(math.fmin(NAN, NNAN))) + self.assertTrue(math.isnan(math.fmin(NNAN, NAN))) + def testFrexp(self): self.assertRaises(TypeError, math.frexp) @@ -1075,68 +1114,6 @@ def test_math_dist_leak(self): with self.assertRaises(ValueError): math.dist([1, 2], [3, 4, 5]) - def testIsqrt(self): - # Test a variety of inputs, large and small. - test_values = ( - list(range(1000)) - + list(range(10**6 - 1000, 10**6 + 1000)) - + [2**e + i for e in range(60, 200) for i in range(-40, 40)] - + [3**9999, 10**5001] - ) - - for value in test_values: - with self.subTest(value=value): - s = math.isqrt(value) - self.assertIs(type(s), int) - self.assertLessEqual(s*s, value) - self.assertLess(value, (s+1)*(s+1)) - - # Negative values - with self.assertRaises(ValueError): - math.isqrt(-1) - - # Integer-like things - s = math.isqrt(True) - self.assertIs(type(s), int) - self.assertEqual(s, 1) - - s = math.isqrt(False) - self.assertIs(type(s), int) - self.assertEqual(s, 0) - - class IntegerLike(object): - def __init__(self, value): - self.value = value - - def __index__(self): - return self.value - - s = math.isqrt(IntegerLike(1729)) - self.assertIs(type(s), int) - self.assertEqual(s, 41) - - with self.assertRaises(ValueError): - math.isqrt(IntegerLike(-3)) - - # Non-integer-like things - bad_values = [ - 3.5, "a string", decimal.Decimal("3.5"), 3.5j, - 100.0, -4.0, - ] - for value in bad_values: - with self.subTest(value=value): - with self.assertRaises(TypeError): - math.isqrt(value) - - @support.bigmemtest(2**32, memuse=0.85) - def test_isqrt_huge(self, size): - if size & 1: - size += 1 - v = 1 << size - w = math.isqrt(v) - self.assertEqual(w.bit_length(), size // 2 + 1) - self.assertEqual(w.bit_count(), 1) - def test_lcm(self): lcm = math.lcm self.assertEqual(lcm(0, 0), 0) @@ -1214,6 +1191,12 @@ def testLdexp(self): self.assertEqual(math.ldexp(NINF, n), NINF) self.assertTrue(math.isnan(math.ldexp(NAN, n))) + @requires_IEEE_754 + def testLdexp_denormal(self): + # Denormal output incorrectly rounded (truncated) + # on some Windows. + self.assertEqual(math.ldexp(6993274598585239, -1126), 1e-323) + def testLog(self): self.assertRaises(TypeError, math.log) self.assertRaises(TypeError, math.log, 1, 2, 3) @@ -1225,13 +1208,32 @@ def testLog(self): self.ftest('log(10**40, 10**20)', math.log(10**40, 10**20), 2) self.ftest('log(10**1000)', math.log(10**1000), 2302.5850929940457) + self.ftest('log(10**2000, 10**1000)', math.log(10**2000, 10**1000), 2) + self.ftest('log(MyIndexable(32), MyIndexable(2))', + math.log(MyIndexable(32), MyIndexable(2)), 5) + self.ftest('log(MyIndexable(10**1000))', + math.log(MyIndexable(10**1000)), + 2302.5850929940457) + self.ftest('log(MyIndexable(10**2000), MyIndexable(10**1000))', + math.log(MyIndexable(10**2000), MyIndexable(10**1000)), + 2) + self.assertRaises(ValueError, math.log, 0.0) + self.assertRaises(ValueError, math.log, 0) + self.assertRaises(ValueError, math.log, MyIndexable(0)) self.assertRaises(ValueError, math.log, -1.5) + self.assertRaises(ValueError, math.log, -1) + self.assertRaises(ValueError, math.log, MyIndexable(-1)) self.assertRaises(ValueError, math.log, -10**1000) + self.assertRaises(ValueError, math.log, MyIndexable(-10**1000)) self.assertRaises(ValueError, math.log, 10, -10) self.assertRaises(ValueError, math.log, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) + self.assertEqual(math.log(IndexableFloatLike(math.e, 10**1000)), 1.0) + self.assertAlmostEqual(math.log(IndexableFloatLike(OverflowError(), 10**1000)), + 2302.5850929940457) + def testLog1p(self): self.assertRaises(TypeError, math.log1p) for n in [2, 2**90, 2**300]: @@ -1247,16 +1249,28 @@ def testLog2(self): self.assertEqual(math.log2(1), 0.0) self.assertEqual(math.log2(2), 1.0) self.assertEqual(math.log2(4), 2.0) + self.assertEqual(math.log2(MyIndexable(4)), 2.0) # Large integer values self.assertEqual(math.log2(2**1023), 1023.0) self.assertEqual(math.log2(2**1024), 1024.0) self.assertEqual(math.log2(2**2000), 2000.0) + self.assertEqual(math.log2(MyIndexable(2**2000)), 2000.0) + self.assertRaises(ValueError, math.log2, 0.0) + self.assertRaises(ValueError, math.log2, 0) + self.assertRaises(ValueError, math.log2, MyIndexable(0)) self.assertRaises(ValueError, math.log2, -1.5) + self.assertRaises(ValueError, math.log2, -1) + self.assertRaises(ValueError, math.log2, MyIndexable(-1)) + self.assertRaises(ValueError, math.log2, -2**2000) + self.assertRaises(ValueError, math.log2, MyIndexable(-2**2000)) self.assertRaises(ValueError, math.log2, NINF) self.assertTrue(math.isnan(math.log2(NAN))) + self.assertEqual(math.log2(IndexableFloatLike(8.0, 2**2000)), 3.0) + self.assertEqual(math.log2(IndexableFloatLike(OverflowError(), 2**2000)), 2000.0) + @requires_IEEE_754 # log2() is not accurate enough on Mac OS X Tiger (10.4) @support.requires_mac_ver(10, 5) @@ -1272,12 +1286,24 @@ def testLog10(self): self.ftest('log10(1)', math.log10(1), 0) self.ftest('log10(10)', math.log10(10), 1) self.ftest('log10(10**1000)', math.log10(10**1000), 1000.0) + self.ftest('log10(MyIndexable(10))', math.log10(MyIndexable(10)), 1) + self.ftest('log10(MyIndexable(10**1000))', + math.log10(MyIndexable(10**1000)), 1000.0) + self.assertRaises(ValueError, math.log10, 0.0) + self.assertRaises(ValueError, math.log10, 0) + self.assertRaises(ValueError, math.log10, MyIndexable(0)) self.assertRaises(ValueError, math.log10, -1.5) + self.assertRaises(ValueError, math.log10, -1) + self.assertRaises(ValueError, math.log10, MyIndexable(-1)) self.assertRaises(ValueError, math.log10, -10**1000) + self.assertRaises(ValueError, math.log10, MyIndexable(-10**1000)) self.assertRaises(ValueError, math.log10, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + self.assertEqual(math.log10(IndexableFloatLike(100.0, 10**1000)), 2.0) + self.assertEqual(math.log10(IndexableFloatLike(OverflowError(), 10**1000)), 1000.0) + @support.bigmemtest(2**32, memuse=0.2) def test_log_huge_integer(self, size): v = 1 << size @@ -1381,7 +1407,6 @@ def __rmul__(self, other): args = ((-5, -5, 10), (1.5, 4611686018427387904, 2305843009213693952)) self.assertEqual(sumprod(*args), 0.0) - @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "sumprod() accuracy not guaranteed on machines with double rounding") @@ -1967,6 +1992,28 @@ def testIsfinite(self): self.assertFalse(math.isfinite(float("inf"))) self.assertFalse(math.isfinite(float("-inf"))) + def testIsnormal(self): + self.assertTrue(math.isnormal(1.25)) + self.assertTrue(math.isnormal(-1.0)) + self.assertFalse(math.isnormal(0.0)) + self.assertFalse(math.isnormal(-0.0)) + self.assertFalse(math.isnormal(INF)) + self.assertFalse(math.isnormal(NINF)) + self.assertFalse(math.isnormal(NAN)) + self.assertFalse(math.isnormal(FLOAT_MIN/2)) + self.assertFalse(math.isnormal(-FLOAT_MIN/2)) + + def testIssubnormal(self): + self.assertFalse(math.issubnormal(1.25)) + self.assertFalse(math.issubnormal(-1.0)) + self.assertFalse(math.issubnormal(0.0)) + self.assertFalse(math.issubnormal(-0.0)) + self.assertFalse(math.issubnormal(INF)) + self.assertFalse(math.issubnormal(NINF)) + self.assertFalse(math.issubnormal(NAN)) + self.assertTrue(math.issubnormal(FLOAT_MIN/2)) + self.assertTrue(math.issubnormal(-FLOAT_MIN/2)) + def testIsnan(self): self.assertTrue(math.isnan(float("nan"))) self.assertTrue(math.isnan(float("-nan"))) @@ -2265,140 +2312,6 @@ def _naive_prod(iterable, start=1): self.assertEqual(type(prod([1, decimal.Decimal(2.0), 3, 4, 5, 6])), decimal.Decimal) - def testPerm(self): - perm = math.perm - factorial = math.factorial - # Test if factorial definition is satisfied - for n in range(500): - for k in (range(n + 1) if n < 100 else range(30) if n < 200 else range(10)): - self.assertEqual(perm(n, k), - factorial(n) // factorial(n - k)) - - # Test for Pascal's identity - for n in range(1, 100): - for k in range(1, n): - self.assertEqual(perm(n, k), perm(n - 1, k - 1) * k + perm(n - 1, k)) - - # Test corner cases - for n in range(1, 100): - self.assertEqual(perm(n, 0), 1) - self.assertEqual(perm(n, 1), n) - self.assertEqual(perm(n, n), factorial(n)) - - # Test one argument form - for n in range(20): - self.assertEqual(perm(n), factorial(n)) - self.assertEqual(perm(n, None), factorial(n)) - - # Raises TypeError if any argument is non-integer or argument count is - # not 1 or 2 - self.assertRaises(TypeError, perm, 10, 1.0) - self.assertRaises(TypeError, perm, 10, decimal.Decimal(1.0)) - self.assertRaises(TypeError, perm, 10, "1") - self.assertRaises(TypeError, perm, 10.0, 1) - self.assertRaises(TypeError, perm, decimal.Decimal(10.0), 1) - self.assertRaises(TypeError, perm, "10", 1) - - self.assertRaises(TypeError, perm) - self.assertRaises(TypeError, perm, 10, 1, 3) - self.assertRaises(TypeError, perm) - - # Raises Value error if not k or n are negative numbers - self.assertRaises(ValueError, perm, -1, 1) - self.assertRaises(ValueError, perm, -2**1000, 1) - self.assertRaises(ValueError, perm, 1, -1) - self.assertRaises(ValueError, perm, 1, -2**1000) - - # Returns zero if k is greater than n - self.assertEqual(perm(1, 2), 0) - self.assertEqual(perm(1, 2**1000), 0) - - n = 2**1000 - self.assertEqual(perm(n, 0), 1) - self.assertEqual(perm(n, 1), n) - self.assertEqual(perm(n, 2), n * (n-1)) - if support.check_impl_detail(cpython=True): - self.assertRaises(OverflowError, perm, n, n) - - for n, k in (True, True), (True, False), (False, False): - self.assertEqual(perm(n, k), 1) - self.assertIs(type(perm(n, k)), int) - self.assertEqual(perm(IntSubclass(5), IntSubclass(2)), 20) - self.assertEqual(perm(MyIndexable(5), MyIndexable(2)), 20) - for k in range(3): - self.assertIs(type(perm(IntSubclass(5), IntSubclass(k))), int) - self.assertIs(type(perm(MyIndexable(5), MyIndexable(k))), int) - - def testComb(self): - comb = math.comb - factorial = math.factorial - # Test if factorial definition is satisfied - for n in range(500): - for k in (range(n + 1) if n < 100 else range(30) if n < 200 else range(10)): - self.assertEqual(comb(n, k), factorial(n) - // (factorial(k) * factorial(n - k))) - - # Test for Pascal's identity - for n in range(1, 100): - for k in range(1, n): - self.assertEqual(comb(n, k), comb(n - 1, k - 1) + comb(n - 1, k)) - - # Test corner cases - for n in range(100): - self.assertEqual(comb(n, 0), 1) - self.assertEqual(comb(n, n), 1) - - for n in range(1, 100): - self.assertEqual(comb(n, 1), n) - self.assertEqual(comb(n, n - 1), n) - - # Test Symmetry - for n in range(100): - for k in range(n // 2): - self.assertEqual(comb(n, k), comb(n, n - k)) - - # Raises TypeError if any argument is non-integer or argument count is - # not 2 - self.assertRaises(TypeError, comb, 10, 1.0) - self.assertRaises(TypeError, comb, 10, decimal.Decimal(1.0)) - self.assertRaises(TypeError, comb, 10, "1") - self.assertRaises(TypeError, comb, 10.0, 1) - self.assertRaises(TypeError, comb, decimal.Decimal(10.0), 1) - self.assertRaises(TypeError, comb, "10", 1) - - self.assertRaises(TypeError, comb, 10) - self.assertRaises(TypeError, comb, 10, 1, 3) - self.assertRaises(TypeError, comb) - - # Raises Value error if not k or n are negative numbers - self.assertRaises(ValueError, comb, -1, 1) - self.assertRaises(ValueError, comb, -2**1000, 1) - self.assertRaises(ValueError, comb, 1, -1) - self.assertRaises(ValueError, comb, 1, -2**1000) - - # Returns zero if k is greater than n - self.assertEqual(comb(1, 2), 0) - self.assertEqual(comb(1, 2**1000), 0) - - n = 2**1000 - self.assertEqual(comb(n, 0), 1) - self.assertEqual(comb(n, 1), n) - self.assertEqual(comb(n, 2), n * (n-1) // 2) - self.assertEqual(comb(n, n), 1) - self.assertEqual(comb(n, n-1), n) - self.assertEqual(comb(n, n-2), n * (n-1) // 2) - if support.check_impl_detail(cpython=True): - self.assertRaises(OverflowError, comb, n, n//2) - - for n, k in (True, True), (True, False), (False, False): - self.assertEqual(comb(n, k), 1) - self.assertIs(type(comb(n, k)), int) - self.assertEqual(comb(IntSubclass(5), IntSubclass(2)), 10) - self.assertEqual(comb(MyIndexable(5), MyIndexable(2)), 10) - for k in range(3): - self.assertIs(type(comb(IntSubclass(5), IntSubclass(k))), int) - self.assertIs(type(comb(MyIndexable(5), MyIndexable(k))), int) - @requires_IEEE_754 def test_nextafter(self): # around 2^52 and 2^63 @@ -2458,7 +2371,6 @@ def test_nextafter(self): with self.assertRaises(ValueError): math.nextafter(1.0, INF, steps=-1) - @requires_IEEE_754 def test_ulp(self): self.assertEqual(math.ulp(1.0), sys.float_info.epsilon) diff --git a/Lib/test/test_math_integer.py b/Lib/test/test_math_integer.py new file mode 100644 index 00000000000..09a98d93bd6 --- /dev/null +++ b/Lib/test/test_math_integer.py @@ -0,0 +1,403 @@ +from decimal import Decimal +from fractions import Fraction +import unittest +from test import support + + +class IntSubclass(int): + pass + +# Class providing an __index__ method. +class MyIndexable(object): + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + +# Here's a pure Python version of the math.integer.factorial algorithm, for +# documentation and comparison purposes. +# +# Formula: +# +# factorial(n) = factorial_odd_part(n) << (n - count_set_bits(n)) +# +# where +# +# factorial_odd_part(n) = product_{i >= 0} product_{0 < j <= n >> i; j odd} j +# +# The outer product above is an infinite product, but once i >= n.bit_length, +# (n >> i) < 1 and the corresponding term of the product is empty. So only the +# finitely many terms for 0 <= i < n.bit_length() contribute anything. +# +# We iterate downwards from i == n.bit_length() - 1 to i == 0. The inner +# product in the formula above starts at 1 for i == n.bit_length(); for each i +# < n.bit_length() we get the inner product for i from that for i + 1 by +# multiplying by all j in {n >> i+1 < j <= n >> i; j odd}. In Python terms, +# this set is range((n >> i+1) + 1 | 1, (n >> i) + 1 | 1, 2). + +def count_set_bits(n): + """Number of '1' bits in binary expansion of a nonnnegative integer.""" + return 1 + count_set_bits(n & n - 1) if n else 0 + +def partial_product(start, stop): + """Product of integers in range(start, stop, 2), computed recursively. + start and stop should both be odd, with start <= stop. + + """ + numfactors = (stop - start) >> 1 + if not numfactors: + return 1 + elif numfactors == 1: + return start + else: + mid = (start + numfactors) | 1 + return partial_product(start, mid) * partial_product(mid, stop) + +def py_factorial(n): + """Factorial of nonnegative integer n, via "Binary Split Factorial Formula" + described at http://www.luschny.de/math/factorial/binarysplitfact.html + + """ + inner = outer = 1 + for i in reversed(range(n.bit_length())): + inner *= partial_product((n >> i + 1) + 1 | 1, (n >> i) + 1 | 1) + outer *= inner + return outer << (n - count_set_bits(n)) + + +class IntMathTests(unittest.TestCase): + import math.integer as module + + def assertIntEqual(self, actual, expected): + self.assertEqual(actual, expected) + self.assertIs(type(actual), int) + + def test_factorial(self): + factorial = self.module.factorial + self.assertEqual(factorial(0), 1) + total = 1 + for i in range(1, 1000): + total *= i + self.assertEqual(factorial(i), total) + self.assertEqual(factorial(i), py_factorial(i)) + + self.assertIntEqual(factorial(False), 1) + self.assertIntEqual(factorial(True), 1) + for i in range(3): + expected = factorial(i) + self.assertIntEqual(factorial(IntSubclass(i)), expected) + self.assertIntEqual(factorial(MyIndexable(i)), expected) + + self.assertRaises(ValueError, factorial, -1) + self.assertRaises(ValueError, factorial, -10**1000) + + def test_factorial_non_integers(self): + factorial = self.module.factorial + self.assertRaises(TypeError, factorial, 5.0) + self.assertRaises(TypeError, factorial, 5.2) + self.assertRaises(TypeError, factorial, -1.0) + self.assertRaises(TypeError, factorial, -1e100) + self.assertRaises(TypeError, factorial, Decimal('5')) + self.assertRaises(TypeError, factorial, Decimal('5.2')) + self.assertRaises(TypeError, factorial, Fraction(5, 1)) + self.assertRaises(TypeError, factorial, "5") + + # Other implementations may place different upper bounds. + @support.cpython_only + def test_factorial_huge_inputs(self): + factorial = self.module.factorial + # Currently raises OverflowError for inputs that are too large + # to fit into a C long. + self.assertRaises(OverflowError, factorial, 10**100) + self.assertRaises(TypeError, factorial, 1e100) + + def test_gcd(self): + gcd = self.module.gcd + self.assertEqual(gcd(0, 0), 0) + self.assertEqual(gcd(1, 0), 1) + self.assertEqual(gcd(-1, 0), 1) + self.assertEqual(gcd(0, 1), 1) + self.assertEqual(gcd(0, -1), 1) + self.assertEqual(gcd(7, 1), 1) + self.assertEqual(gcd(7, -1), 1) + self.assertEqual(gcd(-23, 15), 1) + self.assertEqual(gcd(120, 84), 12) + self.assertEqual(gcd(84, -120), 12) + self.assertEqual(gcd(1216342683557601535506311712, + 436522681849110124616458784), 32) + c = 652560 + x = 434610456570399902378880679233098819019853229470286994367836600566 + y = 1064502245825115327754847244914921553977 + a = x * c + b = y * c + self.assertEqual(gcd(a, b), c) + self.assertEqual(gcd(b, a), c) + self.assertEqual(gcd(-a, b), c) + self.assertEqual(gcd(b, -a), c) + self.assertEqual(gcd(a, -b), c) + self.assertEqual(gcd(-b, a), c) + self.assertEqual(gcd(-a, -b), c) + self.assertEqual(gcd(-b, -a), c) + c = 576559230871654959816130551884856912003141446781646602790216406874 + a = x * c + b = y * c + self.assertEqual(gcd(a, b), c) + self.assertEqual(gcd(b, a), c) + self.assertEqual(gcd(-a, b), c) + self.assertEqual(gcd(b, -a), c) + self.assertEqual(gcd(a, -b), c) + self.assertEqual(gcd(-b, a), c) + self.assertEqual(gcd(-a, -b), c) + self.assertEqual(gcd(-b, -a), c) + + self.assertRaises(TypeError, gcd, 120.0, 84) + self.assertRaises(TypeError, gcd, 120, 84.0) + self.assertIntEqual(gcd(IntSubclass(120), IntSubclass(84)), 12) + self.assertIntEqual(gcd(MyIndexable(120), MyIndexable(84)), 12) + + def test_lcm(self): + lcm = self.module.lcm + self.assertEqual(lcm(0, 0), 0) + self.assertEqual(lcm(1, 0), 0) + self.assertEqual(lcm(-1, 0), 0) + self.assertEqual(lcm(0, 1), 0) + self.assertEqual(lcm(0, -1), 0) + self.assertEqual(lcm(7, 1), 7) + self.assertEqual(lcm(7, -1), 7) + self.assertEqual(lcm(-23, 15), 345) + self.assertEqual(lcm(120, 84), 840) + self.assertEqual(lcm(84, -120), 840) + self.assertEqual(lcm(1216342683557601535506311712, + 436522681849110124616458784), + 16592536571065866494401400422922201534178938447014944) + + x = 43461045657039990237 + y = 10645022458251153277 + for c in (652560, + 57655923087165495981): + a = x * c + b = y * c + d = x * y * c + self.assertEqual(lcm(a, b), d) + self.assertEqual(lcm(b, a), d) + self.assertEqual(lcm(-a, b), d) + self.assertEqual(lcm(b, -a), d) + self.assertEqual(lcm(a, -b), d) + self.assertEqual(lcm(-b, a), d) + self.assertEqual(lcm(-a, -b), d) + self.assertEqual(lcm(-b, -a), d) + + self.assertEqual(lcm(), 1) + self.assertEqual(lcm(120), 120) + self.assertEqual(lcm(-120), 120) + self.assertEqual(lcm(120, 84, 102), 14280) + self.assertEqual(lcm(120, 0, 84), 0) + + self.assertRaises(TypeError, lcm, 120.0) + self.assertRaises(TypeError, lcm, 120.0, 84) + self.assertRaises(TypeError, lcm, 120, 84.0) + self.assertRaises(TypeError, lcm, 120, 0, 84.0) + self.assertEqual(lcm(MyIndexable(120), MyIndexable(84)), 840) + + def test_isqrt(self): + isqrt = self.module.isqrt + # Test a variety of inputs, large and small. + test_values = ( + list(range(1000)) + + list(range(10**6 - 1000, 10**6 + 1000)) + + [2**e + i for e in range(60, 200) for i in range(-40, 40)] + + [3**9999, 10**5001] + ) + + for value in test_values: + with self.subTest(value=value): + s = isqrt(value) + self.assertIs(type(s), int) + self.assertLessEqual(s*s, value) + self.assertLess(value, (s+1)*(s+1)) + + # Negative values + with self.assertRaises(ValueError): + isqrt(-1) + + # Integer-like things + self.assertIntEqual(isqrt(True), 1) + self.assertIntEqual(isqrt(False), 0) + self.assertIntEqual(isqrt(MyIndexable(1729)), 41) + + with self.assertRaises(ValueError): + isqrt(MyIndexable(-3)) + + # Non-integer-like things + bad_values = [ + 3.5, "a string", Decimal("3.5"), 3.5j, + 100.0, -4.0, + ] + for value in bad_values: + with self.subTest(value=value): + with self.assertRaises(TypeError): + isqrt(value) + + @support.bigmemtest(2**32, memuse=0.85) + def test_isqrt_huge(self, size): + isqrt = self.module.isqrt + if size & 1: + size += 1 + v = 1 << size + w = isqrt(v) + self.assertEqual(w.bit_length(), size // 2 + 1) + self.assertEqual(w.bit_count(), 1) + + def test_perm(self): + perm = self.module.perm + factorial = self.module.factorial + # Test if factorial definition is satisfied + for n in range(500): + for k in (range(n + 1) if n < 100 else range(30) if n < 200 else range(10)): + self.assertEqual(perm(n, k), + factorial(n) // factorial(n - k)) + + # Test for Pascal's identity + for n in range(1, 100): + for k in range(1, n): + self.assertEqual(perm(n, k), perm(n - 1, k - 1) * k + perm(n - 1, k)) + + # Test corner cases + for n in range(1, 100): + self.assertEqual(perm(n, 0), 1) + self.assertEqual(perm(n, 1), n) + self.assertEqual(perm(n, n), factorial(n)) + + # Test one argument form + for n in range(20): + self.assertEqual(perm(n), factorial(n)) + self.assertEqual(perm(n, None), factorial(n)) + + # Raises TypeError if any argument is non-integer or argument count is + # not 1 or 2 + self.assertRaises(TypeError, perm, 10, 1.0) + self.assertRaises(TypeError, perm, 10, Decimal(1.0)) + self.assertRaises(TypeError, perm, 10, Fraction(1, 1)) + self.assertRaises(TypeError, perm, 10, "1") + self.assertRaises(TypeError, perm, 10.0, 1) + self.assertRaises(TypeError, perm, Decimal(10.0), 1) + self.assertRaises(TypeError, perm, Fraction(10, 1), 1) + self.assertRaises(TypeError, perm, "10", 1) + + self.assertRaises(TypeError, perm) + self.assertRaises(TypeError, perm, 10, 1, 3) + self.assertRaises(TypeError, perm) + + # Raises Value error if not k or n are negative numbers + self.assertRaises(ValueError, perm, -1, 1) + self.assertRaises(ValueError, perm, -2**1000, 1) + self.assertRaises(ValueError, perm, 1, -1) + self.assertRaises(ValueError, perm, 1, -2**1000) + + # Returns zero if k is greater than n + self.assertEqual(perm(1, 2), 0) + self.assertEqual(perm(1, 2**1000), 0) + + n = 2**1000 + self.assertEqual(perm(n, 0), 1) + self.assertEqual(perm(n, 1), n) + self.assertEqual(perm(n, 2), n * (n-1)) + if support.check_impl_detail(cpython=True): + self.assertRaises(OverflowError, perm, n, n) + + for n, k in (True, True), (True, False), (False, False): + self.assertIntEqual(perm(n, k), 1) + self.assertEqual(perm(IntSubclass(5), IntSubclass(2)), 20) + self.assertEqual(perm(MyIndexable(5), MyIndexable(2)), 20) + for k in range(3): + self.assertIs(type(perm(IntSubclass(5), IntSubclass(k))), int) + self.assertIs(type(perm(MyIndexable(5), MyIndexable(k))), int) + + def test_comb(self): + comb = self.module.comb + factorial = self.module.factorial + # Test if factorial definition is satisfied + for n in range(500): + for k in (range(n + 1) if n < 100 else range(30) if n < 200 else range(10)): + self.assertEqual(comb(n, k), factorial(n) + // (factorial(k) * factorial(n - k))) + + # Test for Pascal's identity + for n in range(1, 100): + for k in range(1, n): + self.assertEqual(comb(n, k), comb(n - 1, k - 1) + comb(n - 1, k)) + + # Test corner cases + for n in range(100): + self.assertEqual(comb(n, 0), 1) + self.assertEqual(comb(n, n), 1) + + for n in range(1, 100): + self.assertEqual(comb(n, 1), n) + self.assertEqual(comb(n, n - 1), n) + + # Test Symmetry + for n in range(100): + for k in range(n // 2): + self.assertEqual(comb(n, k), comb(n, n - k)) + + # Raises TypeError if any argument is non-integer or argument count is + # not 2 + self.assertRaises(TypeError, comb, 10, 1.0) + self.assertRaises(TypeError, comb, 10, Decimal(1.0)) + self.assertRaises(TypeError, comb, 10, "1") + self.assertRaises(TypeError, comb, 10.0, 1) + self.assertRaises(TypeError, comb, Decimal(10.0), 1) + self.assertRaises(TypeError, comb, "10", 1) + + self.assertRaises(TypeError, comb, 10) + self.assertRaises(TypeError, comb, 10, 1, 3) + self.assertRaises(TypeError, comb) + + # Raises Value error if not k or n are negative numbers + self.assertRaises(ValueError, comb, -1, 1) + self.assertRaises(ValueError, comb, -2**1000, 1) + self.assertRaises(ValueError, comb, 1, -1) + self.assertRaises(ValueError, comb, 1, -2**1000) + + # Returns zero if k is greater than n + self.assertEqual(comb(1, 2), 0) + self.assertEqual(comb(1, 2**1000), 0) + + n = 2**1000 + self.assertEqual(comb(n, 0), 1) + self.assertEqual(comb(n, 1), n) + self.assertEqual(comb(n, 2), n * (n-1) // 2) + self.assertEqual(comb(n, n), 1) + self.assertEqual(comb(n, n-1), n) + self.assertEqual(comb(n, n-2), n * (n-1) // 2) + if support.check_impl_detail(cpython=True): + self.assertRaises(OverflowError, comb, n, n//2) + + for n, k in (True, True), (True, False), (False, False): + self.assertIntEqual(comb(n, k), 1) + self.assertEqual(comb(IntSubclass(5), IntSubclass(2)), 10) + self.assertEqual(comb(MyIndexable(5), MyIndexable(2)), 10) + for k in range(3): + self.assertIs(type(comb(IntSubclass(5), IntSubclass(k))), int) + self.assertIs(type(comb(MyIndexable(5), MyIndexable(k))), int) + + +class MathTests(IntMathTests): + import math as module + + +class MiscTests(unittest.TestCase): + + def test_module_name(self): + import math.integer + self.assertEqual(math.integer.__name__, 'math.integer') + for name in dir(math.integer): + if not name.startswith('_'): + obj = getattr(math.integer, name) + self.assertEqual(obj.__module__, 'math.integer') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 61b068c630c..1bd58eb6408 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -600,6 +600,25 @@ def test_memoryview_hex(self): m2 = m1[::-1] self.assertEqual(m2.hex(), '30' * 200000) + def test_memoryview_hex_separator(self): + x = bytes(range(97, 102)) + m1 = memoryview(x) + m2 = m1[::-1] + self.assertEqual(m2.hex(':'), '65:64:63:62:61') + self.assertEqual(m2.hex(':', 2), '65:6463:6261') + self.assertEqual(m2.hex(':', -2), '6564:6362:61') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=2), '65:6463:6261') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=-2), '6564:6362:61') + for bytes_per_sep in 5, -5, 2**31-1, -(2**31-1): + with self.subTest(bytes_per_sep=bytes_per_sep): + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + with self.subTest(bytes_per_sep=bytes_per_sep): + try: + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + except OverflowError: + pass + def test_copy(self): m = memoryview(b'abc') with self.assertRaises(TypeError): @@ -743,19 +762,21 @@ def test_racing_getbuf_and_releasebuf(self): from multiprocessing.managers import SharedMemoryManager except ImportError: self.skipTest("Test requires multiprocessing") - from threading import Thread + from threading import Thread, Event - n = 100 + start = Event() with SharedMemoryManager() as smm: obj = smm.ShareableList(range(100)) - threads = [] - for _ in range(n): - # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` - # counter of `memoryview`, this issue has no direct relation with `ShareableList`. - threads.append(Thread(target=obj.count, args=(1,))) - + def test(): + # Issue gh-127085, the `ShareableList.count` is just a + # convenient way to mess the `exports` counter of `memoryview`, + # this issue has no direct relation with `ShareableList`. + start.wait(support.SHORT_TIMEOUT) + for i in range(10): + obj.count(1) + threads = [Thread(target=test) for _ in range(10)] with threading_helper.start_threads(threads): - pass + start.set() del obj diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index fb57d5e5544..0f29640bc1c 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -112,13 +112,12 @@ def test_non_standard_types(self): eq = self.assertEqual # First try strict eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') - eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') + eq(self.db.guess_extension('image/jpeg', strict=False), '.jpg') + eq(self.db.guess_extension('image/JPEG', strict=False), '.jpg') def test_filename_with_url_delimiters(self): # bpo-38449: URL delimiters cases should be handled also. @@ -179,8 +178,8 @@ def test_guess_all_types(self): self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) self.assertEqual(len(set(all)), len(all)) # no duplicates # And now non-strict - all = self.db.guess_all_extensions('image/jpg', strict=False) - self.assertEqual(all, ['.jpg']) + all = self.db.guess_all_extensions('image/jpeg', strict=False) + self.assertEqual(all, ['.jpg', '.jpe', '.jpeg']) # And now for no hits all = self.db.guess_all_extensions('image/jpg', strict=True) self.assertEqual(all, []) @@ -229,7 +228,11 @@ def check_extensions(): ("application/octet-stream", ".bin"), ("application/gzip", ".gz"), ("application/ogg", ".ogx"), + ("application/pdf", ".pdf"), ("application/postscript", ".ps"), + ("application/rtf", ".rtf"), + ("application/texinfo", ".texi"), + ("application/toml", ".toml"), ("application/vnd.apple.mpegurl", ".m3u"), ("application/vnd.ms-excel", ".xls"), ("application/vnd.ms-fontobject", ".eot"), @@ -246,7 +249,6 @@ def check_extensions(): ("application/x-debian-package", ".deb"), ("application/x-httpd-php", ".php"), ("application/x-rpm", ".rpm"), - ("application/x-texinfo", ".texi"), ("application/x-troff", ".roff"), ("application/xml", ".xsl"), ("application/yaml", ".yaml"), @@ -279,7 +281,6 @@ def check_extensions(): ("model/stl", ".stl"), ("text/html", ".html"), ("text/plain", ".txt"), - ("text/rtf", ".rtf"), ("text/x-rst", ".rst"), ("video/matroska", ".mkv"), ("video/matroska-3d", ".mk3d"), @@ -370,9 +371,7 @@ def test_keywords_args_api(self): self.assertEqual(self.db.guess_type( url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( - type='image/jpg', strict=True), []) - self.assertEqual(self.db.guess_extension( - type='image/jpg', strict=False), '.jpg') + type='image/jpeg', strict=True), ['.jpg', '.jpe', '.jpeg']) def test_added_types_are_used(self): mimetypes.add_type('testing/default-type', '') @@ -450,15 +449,15 @@ def test_parse_args(self): args, help_text = mimetypes._parse_args("--invalid") self.assertTrue(help_text.startswith("usage: ")) - args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpg")) + args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpeg")) self.assertTrue(args.extension) self.assertTrue(args.lenient) - self.assertEqual(args.type, ["image/jpg"]) + self.assertEqual(args.type, ["image/jpeg"]) - args, _ = mimetypes._parse_args(shlex.split("-e image/jpg")) + args, _ = mimetypes._parse_args(shlex.split("-e image/jpeg")) self.assertTrue(args.extension) self.assertFalse(args.lenient) - self.assertEqual(args.type, ["image/jpg"]) + self.assertEqual(args.type, ["image/jpeg"]) args, _ = mimetypes._parse_args(shlex.split("-l foo.webp")) self.assertFalse(args.extension) @@ -470,13 +469,30 @@ def test_parse_args(self): self.assertFalse(args.lenient) self.assertEqual(args.type, ["foo.pic"]) + def test_multiple_inputs(self): + result = "\n".join(mimetypes._main(shlex.split("foo.pdf foo.png"))) + self.assertEqual( + result, + "type: application/pdf encoding: None\n" + "type: image/png encoding: None" + ) + + def test_multiple_inputs_error(self): + result = "\n".join(mimetypes._main(shlex.split("foo.pdf foo.bar_ext"))) + self.assertEqual( + result, + "type: application/pdf encoding: None\n" + "error: media type unknown for foo.bar_ext" + ) + + def test_invocation(self): for command, expected in [ - ("-l -e image/jpg", ".jpg"), ("-e image/jpeg", ".jpg"), ("-l foo.webp", "type: image/webp encoding: None"), ]: - self.assertEqual(mimetypes._main(shlex.split(command)), expected) + result = "\n".join(mimetypes._main(shlex.split(command))) + self.assertEqual(result, expected) def test_invocation_error(self): for command, expected in [ @@ -484,8 +500,8 @@ def test_invocation_error(self): ("foo.bar_ext", "error: media type unknown for foo.bar_ext"), ]: with self.subTest(command=command): - with self.assertRaisesRegex(SystemExit, expected): - mimetypes._main(shlex.split(command)) + result = "\n".join(mimetypes._main(shlex.split(command))) + self.assertEqual(result, expected) if __name__ == "__main__": diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 4f25e9c2a03..4fa5a4e6768 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -2,6 +2,7 @@ import copy import pickle +import time import io from test import support import unittest @@ -173,6 +174,23 @@ def testAppendChild(self): self.assertEqual(dom.documentElement.childNodes[-1].data, "Hello") dom.unlink() + def testAppendChildNoQuadraticComplexity(self): + impl = getDOMImplementation() + + newdoc = impl.createDocument(None, "some_tag", None) + top_element = newdoc.documentElement + children = [newdoc.createElement(f"child-{i}") for i in range(1, 2 ** 15 + 1)] + element = top_element + + start = time.time() + for child in children: + element.appendChild(child) + element = child + end = time.time() + + # This example used to take at least 30 seconds. + self.assertLess(end - start, 1) + def testAppendChildFragment(self): dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() dom.documentElement.appendChild(frag) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index fd4197b7086..368af0cf89c 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,3 +1,4 @@ +from test import support from test.support import ( requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple, in_systemd_nspawn_sync_suppressed, @@ -6,7 +7,6 @@ from test.support.os_helper import TESTFN, unlink from test.support.script_helper import assert_python_ok import unittest -import errno import os import re import itertools @@ -57,6 +57,7 @@ def test_basic(self): f.write(b'\0'* (PAGESIZE-3) ) f.flush() m = mmap.mmap(f.fileno(), 2 * PAGESIZE) + self.addCleanup(m.close) finally: f.close() @@ -114,31 +115,28 @@ def test_basic(self): # Try to seek to negative position... self.assertRaises(ValueError, m.seek, -len(m)-1, 2) + @unittest.skipUnless(hasattr(mmap.mmap, 'resize'), 'requires mmap.resize') + def test_resize(self): + # Create a file to be mmap'ed. + with open(TESTFN, 'bw+') as f: + # Write 2 pages worth of data to the file + f.write(b'\0'* 2 * PAGESIZE) + f.flush() + m = mmap.mmap(f.fileno(), 2 * PAGESIZE) + self.addCleanup(m.close) + # Try resizing map - try: - m.resize(512) - except SystemError: - # resize() not supported - # No messages are printed, since the output of this test suite - # would then be different across platforms. - pass - else: - # resize() is supported - self.assertEqual(len(m), 512) - # Check that we can no longer seek beyond the new size. - self.assertRaises(ValueError, m.seek, 513, 0) + m.resize(512) + self.assertEqual(len(m), 512) + # Check that we can no longer seek beyond the new size. + self.assertRaises(ValueError, m.seek, 513, 0) - # Check that the underlying file is truncated too - # (bug #728515) - f = open(TESTFN, 'rb') - try: - f.seek(0, 2) - self.assertEqual(f.tell(), 512) - finally: - f.close() - self.assertEqual(m.size(), 512) - - m.close() + # Check that the underlying file is truncated too + # (bug #728515) + with open(TESTFN, 'rb') as f: + f.seek(0, 2) + self.assertEqual(f.tell(), 512) + self.assertEqual(m.size(), 512) def test_access_parameter(self): # Test for "access" keyword parameter @@ -183,15 +181,10 @@ def test_access_parameter(self): else: self.fail("Able to write to readonly memory map") - # Ensuring that readonly mmap can't be resized - try: - m.resize(2*mapsize) - except SystemError: # resize is not universally supported - pass - except TypeError: - pass - else: - self.fail("Able to resize readonly memory map") + if hasattr(m, 'resize'): + # Ensuring that readonly mmap can't be resized + with self.assertRaises(TypeError): + m.resize(2 * mapsize) with open(TESTFN, "rb") as fp: self.assertEqual(fp.read(), b'a'*mapsize, "Readonly memory map data file was modified") @@ -242,8 +235,9 @@ def test_access_parameter(self): with open(TESTFN, "rb") as fp: self.assertEqual(fp.read(), b'c'*mapsize, "Copy-on-write test data file should not be modified.") - # Ensuring copy-on-write maps cannot be resized - self.assertRaises(TypeError, m.resize, 2*mapsize) + if hasattr(m, 'resize'): + # Ensuring copy-on-write maps cannot be resized + self.assertRaises(TypeError, m.resize, 2 * mapsize) m.close() # Ensuring invalid access parameter raises exception @@ -270,62 +264,57 @@ def test_access_parameter(self): self.assertRaises(TypeError, m.write_byte, 0) m.close() - @unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows') - def test_trackfd_parameter(self): + @support.subTests('close_original_fd', (True, False)) + def test_trackfd_parameter(self, close_original_fd): size = 64 with open(TESTFN, "wb") as f: f.write(b"a"*size) - for close_original_fd in True, False: - with self.subTest(close_original_fd=close_original_fd): - with open(TESTFN, "r+b") as f: - with mmap.mmap(f.fileno(), size, trackfd=False) as m: - if close_original_fd: - f.close() - self.assertEqual(len(m), size) - with self.assertRaises(OSError) as err_cm: - m.size() - self.assertEqual(err_cm.exception.errno, errno.EBADF) - with self.assertRaises(ValueError): - m.resize(size * 2) - with self.assertRaises(ValueError): - m.resize(size // 2) - self.assertEqual(m.closed, False) + with open(TESTFN, "r+b") as f: + with mmap.mmap(f.fileno(), size, trackfd=False) as m: + if close_original_fd: + f.close() + self.assertEqual(len(m), size) + with self.assertRaises(ValueError): + m.size() + if hasattr(m, 'resize'): + with self.assertRaises(ValueError): + m.resize(size * 2) + with self.assertRaises(ValueError): + m.resize(size // 2) + self.assertIs(m.closed, False) - # Smoke-test other API - m.write_byte(ord('X')) - m[2] = ord('Y') - m.flush() - with open(TESTFN, "rb") as f: - self.assertEqual(f.read(4), b'XaYa') - self.assertEqual(m.tell(), 1) - m.seek(0) - self.assertEqual(m.tell(), 0) - self.assertEqual(m.read_byte(), ord('X')) + # Smoke-test other API + m.write_byte(ord('X')) + m[2] = ord('Y') + m.flush() + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(4), b'XaYa') + self.assertEqual(m.tell(), 1) + m.seek(0) + self.assertEqual(m.tell(), 0) + self.assertEqual(m.read_byte(), ord('X')) - self.assertEqual(m.closed, True) - self.assertEqual(os.stat(TESTFN).st_size, size) + if os.name == 'nt' and not close_original_fd: + self.assertRaises(PermissionError, os.rename, TESTFN, TESTFN+'1') + else: + os.rename(TESTFN, TESTFN+'1') + os.rename(TESTFN+'1', TESTFN) + + self.assertIs(m.closed, True) + self.assertEqual(os.stat(TESTFN).st_size, size) - @unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows') def test_trackfd_neg1(self): size = 64 with mmap.mmap(-1, size, trackfd=False) as m: - with self.assertRaises(OSError): - m.size() with self.assertRaises(ValueError): - m.resize(size // 2) + m.size() + if hasattr(m, 'resize'): + with self.assertRaises(ValueError): + m.resize(size // 2) self.assertEqual(len(m), size) m[0] = ord('a') assert m[0] == ord('a') - @unittest.skipIf(os.name != 'nt', 'trackfd only fails on Windows') - def test_no_trackfd_parameter_on_windows(self): - # 'trackffd' is an invalid keyword argument for this function - size = 64 - with self.assertRaises(TypeError): - mmap.mmap(-1, size, trackfd=True) - with self.assertRaises(TypeError): - mmap.mmap(-1, size, trackfd=False) - def test_bad_file_desc(self): # Try opening a bad file descriptor... self.assertRaises(OSError, mmap.mmap, -2, 4096) @@ -505,6 +494,7 @@ def test_anonymous(self): b = x & 0xff m[x] = b self.assertEqual(m[x], b) + self.assertEqual(m.size(), PAGESIZE) def test_read_all(self): m = mmap.mmap(-1, 16) @@ -614,13 +604,9 @@ def test_offset (self): self.assertEqual(m[0:3], b'foo') f.close() - # Try resizing map - try: + if hasattr(m, 'resize'): + # Try resizing map m.resize(512) - except SystemError: - pass - else: - # resize() is supported self.assertEqual(len(m), 512) # Check that we can no longer seek beyond the new size. self.assertRaises(ValueError, m.seek, 513, 0) @@ -732,7 +718,7 @@ def test_tagname(self): m2.close() m1.close() - with self.assertRaisesRegex(TypeError, 'must be str or None'): + with self.assertRaisesRegex(TypeError, 'tagname'): mmap.mmap(-1, 8, tagname=1) @cpython_only @@ -812,14 +798,12 @@ def test_write_returning_the_number_of_bytes_written(self): self.assertEqual(mm.write(b"yz"), 2) self.assertEqual(mm.write(b"python"), 6) + @unittest.skipUnless(hasattr(mmap.mmap, 'resize'), 'requires mmap.resize') def test_resize_past_pos(self): m = mmap.mmap(-1, 8192) self.addCleanup(m.close) m.read(5000) - try: - m.resize(4096) - except SystemError: - self.skipTest("resizing not supported") + m.resize(4096) self.assertEqual(m.read(14), b'') self.assertRaises(ValueError, m.read_byte) self.assertRaises(ValueError, m.write_byte, 42) @@ -887,6 +871,10 @@ def test_madvise(self): size = 2 * PAGESIZE m = mmap.mmap(-1, size) + class Number: + def __index__(self): + return 2 + with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): m.madvise(mmap.MADV_NORMAL, size) with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): @@ -895,41 +883,70 @@ def test_madvise(self): m.madvise(mmap.MADV_NORMAL, 0, -1) with self.assertRaisesRegex(OverflowError, "madvise length too large"): m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize) + with self.assertRaisesRegex( + TypeError, "'str' object cannot be interpreted as an integer"): + m.madvise(mmap.MADV_NORMAL, PAGESIZE, "Not a Number") self.assertEqual(m.madvise(mmap.MADV_NORMAL), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None) + self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, Number()), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None) - @unittest.skipUnless(os.name == 'nt', 'requires Windows') - def test_resize_up_when_mapped_to_pagefile(self): + @unittest.skipUnless(hasattr(mmap.mmap, 'resize'), 'requires mmap.resize') + def test_resize_up_anonymous_mapping(self): """If the mmap is backed by the pagefile ensure a resize up can happen and that the original data is still in place """ start_size = PAGESIZE new_size = 2 * start_size - data = bytes(random.getrandbits(8) for _ in range(start_size)) + data = random.randbytes(start_size) - m = mmap.mmap(-1, start_size) - m[:] = data - m.resize(new_size) - self.assertEqual(len(m), new_size) - self.assertEqual(m[:start_size], data[:start_size]) + with mmap.mmap(-1, start_size) as m: + m[:] = data + if sys.platform.startswith(('linux', 'android')): + # Can't expand a shared anonymous mapping on Linux. + # See https://bugzilla.kernel.org/show_bug.cgi?id=8691 + with self.assertRaises(ValueError): + m.resize(new_size) + else: + m.resize(new_size) + self.assertEqual(len(m), new_size) + self.assertEqual(m[:start_size], data) + self.assertEqual(m[start_size:], b'\0' * (new_size - start_size)) - @unittest.skipUnless(os.name == 'nt', 'requires Windows') - def test_resize_down_when_mapped_to_pagefile(self): + @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipUnless(hasattr(mmap.mmap, 'resize'), 'requires mmap.resize') + def test_resize_up_private_anonymous_mapping(self): + start_size = PAGESIZE + new_size = 2 * start_size + data = random.randbytes(start_size) + + with mmap.mmap(-1, start_size, flags=mmap.MAP_PRIVATE) as m: + m[:] = data + m.resize(new_size) + self.assertEqual(len(m), new_size) + self.assertEqual(m[:start_size], data) + self.assertEqual(m[start_size:], b'\0' * (new_size - start_size)) + + @unittest.skipUnless(hasattr(mmap.mmap, 'resize'), 'requires mmap.resize') + def test_resize_down_anonymous_mapping(self): """If the mmap is backed by the pagefile ensure a resize down up can happen and that a truncated form of the original data is still in place """ - start_size = PAGESIZE + start_size = 2 * PAGESIZE new_size = start_size // 2 - data = bytes(random.getrandbits(8) for _ in range(start_size)) + data = random.randbytes(start_size) - m = mmap.mmap(-1, start_size) - m[:] = data - m.resize(new_size) - self.assertEqual(len(m), new_size) - self.assertEqual(m[:new_size], data[:new_size]) + with mmap.mmap(-1, start_size) as m: + m[:] = data + m.resize(new_size) + self.assertEqual(len(m), new_size) + self.assertEqual(m[:], data[:new_size]) + if sys.platform.startswith(('linux', 'android')): + # Can't expand to its original size. + with self.assertRaises(ValueError): + m.resize(start_size) @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_resize_fails_if_mapping_held_elsewhere(self): @@ -1136,6 +1153,18 @@ def test_access_violations(self): self.assertEqual(stdout.strip(), b'') self.assertEqual(stderr.strip(), b'') + def test_flush_parameters(self): + with open(TESTFN, 'wb+') as f: + f.write(b'x' * PAGESIZE * 3) + f.flush() + + m = mmap.mmap(f.fileno(), PAGESIZE * 3) + self.addCleanup(m.close) + + m.flush() + m.flush(PAGESIZE) + m.flush(PAGESIZE, PAGESIZE) + class LargeMmapTests(unittest.TestCase): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 263e4e6f394..83bf510ceea 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -3,6 +3,7 @@ import collections import dis import functools +import inspect import math import operator import sys @@ -11,10 +12,10 @@ import unittest import test.support -from test.support import requires_specialization_ft, script_helper +from test.support import import_helper, requires_specialization_ft, script_helper -_testcapi = test.support.import_helper.import_module("_testcapi") -_testinternalcapi = test.support.import_helper.import_module("_testinternalcapi") +_testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") PAIR = (0,1) @@ -1078,6 +1079,25 @@ def f(): self.assertEqual(events, expected) + # gh-140373 + def test_gen_unwind(self): + def gen(): + yield 1 + + def f(): + g = gen() + next(g) + g.close() + + recorders = ( + UnwindRecorder, + ) + events = self.get_events(f, TEST_TOOL, recorders) + expected = [ + ("unwind", GeneratorExit, "gen"), + ] + self.assertEqual(events, expected) + class LineRecorder: event_type = E.LINE @@ -1709,6 +1729,27 @@ def func(v=1): ('branch right', 'func', 6, 8), ('branch right', 'func', 2, 10)]) + def test_callback_set_frame_lineno(self): + def func(s: str) -> int: + if s.startswith("t"): + return 1 + else: + return 0 + + def callback(code, from_, to): + # try set frame.f_lineno + frame = inspect.currentframe() + while frame and frame.f_code is not code: + frame = frame.f_back + + self.assertIsNotNone(frame) + frame.f_lineno = frame.f_lineno + 1 # run next instruction + + sys.monitoring.set_local_events(TEST_TOOL, func.__code__, E.BRANCH_LEFT) + sys.monitoring.register_callback(TEST_TOOL, E.BRANCH_LEFT, callback) + + self.assertEqual(func("true"), 1) + class TestBranchConsistency(MonitoringTestBase, unittest.TestCase): @@ -2157,6 +2198,21 @@ def callback(code, instruction_offset): sys.monitoring.restart_events() sys.monitoring.set_events(0, 0) + def test_134879(self): + # gh-134789 + # Specialized FOR_ITER not incrementing index + def foo(): + t = 0 + for i in [1,2,3,4]: + t += i + self.assertEqual(t, 10) + + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_local_events(0, foo.__code__, E.BRANCH_LEFT | E.BRANCH_RIGHT) + foo() + sys.monitoring.set_local_events(0, foo.__code__, 0) + class TestOptimizer(MonitoringTestBase, unittest.TestCase): diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 81e11a293cc..9d720f62710 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,11 +1,7 @@ import netrc, os, unittest, sys, textwrap +from test import support from test.support import os_helper -try: - import pwd -except ImportError: - pwd = None - temp_filename = os_helper.TESTFN class NetrcTestCase(unittest.TestCase): @@ -269,9 +265,14 @@ def test_comment_at_end_of_machine_line_pass_has_hash(self): machine bar.domain.com login foo password pass """, '#pass') + @unittest.skipUnless(support.is_wasi, 'WASI only test') + def test_security_on_WASI(self): + self.assertFalse(netrc._can_security_check()) + self.assertEqual(netrc._getpwuid(0), 'uid 0') + self.assertEqual(netrc._getpwuid(123456), 'uid 123456') @unittest.skipUnless(os.name == 'posix', 'POSIX only test') - @unittest.skipIf(pwd is None, 'security check requires pwd module') + @unittest.skipUnless(hasattr(os, 'getuid'), "os.getuid is required") @os_helper.skip_unless_working_chmod def test_security(self): # This test is incomplete since we are normally not run as root and diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index c10387b58e3..3a3c60dea13 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1,3 +1,4 @@ +import errno import inspect import ntpath import os @@ -6,8 +7,9 @@ import sys import unittest import warnings -from test.support import cpython_only, os_helper -from test.support import TestFailed, is_emscripten +from ntpath import ALL_BUT_LAST, ALLOW_MISSING +from test import support +from test.support import os_helper from test.support.os_helper import FakePath from test import test_genericpath from tempfile import TemporaryFile @@ -57,7 +59,7 @@ def tester(fn, wantResult): fn = fn.replace("\\", "\\\\") gotResult = eval(fn) if wantResult != gotResult and _norm(wantResult) != _norm(gotResult): - raise TestFailed("%s should return: %s but returned: %s" \ + raise support.TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), str(gotResult))) # then with bytes @@ -73,10 +75,14 @@ def tester(fn, wantResult): warnings.simplefilter("ignore", DeprecationWarning) gotResult = eval(fn) if _norm(wantResult) != _norm(gotResult): - raise TestFailed("%s should return: %s but returned: %s" \ + raise support.TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), repr(gotResult))) +def _parameterize(*parameters): + return support.subTests('kwargs', parameters, _do_cleanups=True) + + class NtpathTestCase(unittest.TestCase): def assertPathEqual(self, path1, path2): if path1 == path2 or _norm(path1) == _norm(path2): @@ -124,6 +130,22 @@ def test_splitdrive(self): tester('ntpath.splitdrive("//?/UNC/server/share/dir")', ("//?/UNC/server/share", "/dir")) + def test_splitdrive_invalid_paths(self): + splitdrive = ntpath.splitdrive + self.assertEqual(splitdrive('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\di\x00r')) + self.assertEqual(splitdrive(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\di\x00r')) + self.assertEqual(splitdrive("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitdrive(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\\xfd')) + def test_splitroot(self): tester("ntpath.splitroot('')", ('', '', '')) tester("ntpath.splitroot('foo')", ('', '', 'foo')) @@ -214,6 +236,22 @@ def test_splitroot(self): tester('ntpath.splitroot(" :/foo")', (" :", "/", "foo")) tester('ntpath.splitroot("/:/foo")', ("", "/", ":/foo")) + def test_splitroot_invalid_paths(self): + splitroot = ntpath.splitroot + self.assertEqual(splitroot('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\', 'di\x00r')) + self.assertEqual(splitroot(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\', b'di\x00r')) + self.assertEqual(splitroot("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\', '\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitroot(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\', b'\xfd')) + def test_split(self): tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar')) tester('ntpath.split("\\\\conky\\mountpoint\\foo\\bar")', @@ -226,6 +264,21 @@ def test_split(self): tester('ntpath.split("c:/")', ('c:/', '')) tester('ntpath.split("//conky/mountpoint/")', ('//conky/mountpoint/', '')) + def test_split_invalid_paths(self): + split = ntpath.split + self.assertEqual(split('c:\\fo\x00o\\ba\x00r'), + ('c:\\fo\x00o', 'ba\x00r')) + self.assertEqual(split(b'c:\\fo\x00o\\ba\x00r'), + (b'c:\\fo\x00o', b'ba\x00r')) + self.assertEqual(split('c:\\\udfff\\\udffe'), + ('c:\\\udfff', '\udffe')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, split, b'c:\\\xff\\bar') + self.assertRaises(UnicodeDecodeError, split, b'c:\\foo\\\xff') + else: + self.assertEqual(split(b'c:\\\xff\\\xfe'), + (b'c:\\\xff', b'\xfe')) + def test_isabs(self): tester('ntpath.isabs("foo\\bar")', 0) tester('ntpath.isabs("foo/bar")', 0) @@ -333,6 +386,30 @@ def test_join(self): tester("ntpath.join('D:a', './c:b')", 'D:a\\.\\c:b') tester("ntpath.join('D:/a', './c:b')", 'D:\\a\\.\\c:b') + def test_normcase(self): + normcase = ntpath.normcase + self.assertEqual(normcase(''), '') + self.assertEqual(normcase(b''), b'') + self.assertEqual(normcase('ABC'), 'abc') + self.assertEqual(normcase(b'ABC'), b'abc') + self.assertEqual(normcase('\xc4\u0141\u03a8'), '\xe4\u0142\u03c8') + expected = '\u03c9\u2126' if sys.platform == 'win32' else '\u03c9\u03c9' + self.assertEqual(normcase('\u03a9\u2126'), expected) + if sys.platform == 'win32' or sys.getfilesystemencoding() == 'utf-8': + self.assertEqual(normcase('\xc4\u0141\u03a8'.encode()), + '\xe4\u0142\u03c8'.encode()) + self.assertEqual(normcase('\u03a9\u2126'.encode()), + expected.encode()) + + def test_normcase_invalid_paths(self): + normcase = ntpath.normcase + self.assertEqual(normcase('abc\x00def'), 'abc\x00def') + self.assertEqual(normcase(b'abc\x00def'), b'abc\x00def') + self.assertEqual(normcase('\udfff'), '\udfff') + if sys.platform == 'win32': + path = b'ABC' + bytes(range(128, 256)) + self.assertEqual(normcase(path), path.lower()) + def test_normpath(self): tester("ntpath.normpath('A//////././//.//B')", r'A\B') tester("ntpath.normpath('A/./B')", r'A\B') @@ -381,6 +458,21 @@ def test_normpath(self): tester("ntpath.normpath('\\\\')", '\\\\') tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') + def test_normpath_invalid_paths(self): + normpath = ntpath.normpath + self.assertEqual(normpath('fo\x00o'), 'fo\x00o') + self.assertEqual(normpath(b'fo\x00o'), b'fo\x00o') + self.assertEqual(normpath('fo\x00o\\..\\bar'), 'bar') + self.assertEqual(normpath(b'fo\x00o\\..\\bar'), b'bar') + self.assertEqual(normpath('\udfff'), '\udfff') + self.assertEqual(normpath('\udfff\\..\\foo'), 'foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, normpath, b'\xff') + self.assertRaises(UnicodeDecodeError, normpath, b'\xff\\..\\foo') + else: + self.assertEqual(normpath(b'\xff'), b'\xff') + self.assertEqual(normpath(b'\xff\\..\\foo'), b'foo') + def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) tester("ntpath.realpath('.')", expected) @@ -389,6 +481,27 @@ def test_realpath_curdir(self): tester("ntpath.realpath('.\\.')", expected) tester("ntpath.realpath('\\'.join(['.'] * 100))", expected) + def test_realpath_curdir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=True)", expected) + tester("ntpath.realpath('./.', strict=True)", expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=True)", expected) + tester("ntpath.realpath('.\\.', strict=True)", expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=True)", expected) + + def test_realpath_curdir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('./.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('.\\.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + def test_realpath_pardir(self): expected = ntpath.normpath(os.getcwd()) tester("ntpath.realpath('..')", ntpath.dirname(expected)) @@ -401,28 +514,59 @@ def test_realpath_pardir(self): tester("ntpath.realpath('\\'.join(['..'] * 50))", ntpath.splitdrive(expected)[0] + '\\') + def test_realpath_pardir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=True)", ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + + def test_realpath_pardir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=ALLOW_MISSING)", + ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_basic(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ABSTFN + "1") - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) - self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")), + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) + self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1"), **kwargs), os.fsencode(ABSTFN)) # gh-88013: call ntpath.realpath with binary drive name may raise a # TypeError. The drive should not exist to reproduce the bug. drives = {f"{c}:\\" for c in string.ascii_uppercase} - set(os.listdrives()) d = drives.pop().encode() - self.assertEqual(ntpath.realpath(d), d) + self.assertEqual(ntpath.realpath(d, strict=False), d) # gh-106242: Embedded nulls and non-strict fallback to abspath - self.assertEqual(ABSTFN + "\0spam", - ntpath.realpath(os_helper.TESTFN + "\0spam", strict=False)) + if kwargs: + with self.assertRaises(OSError): + ntpath.realpath(os_helper.TESTFN + "\0spam", + **kwargs) + else: + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", **kwargs)) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -434,19 +578,81 @@ def test_realpath_strict(self): self.addCleanup(os_helper.unlink, ABSTFN) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_invalid_paths(self): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFN + '\x00' + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(realpath(path, strict=False), path) # gh-106242: Embedded nulls should raise OSError (not ValueError) - self.assertRaises(OSError, ntpath.realpath, ABSTFN + "\0spam", strict=True) + self.assertRaises(OSError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFNb + b'\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFN + '\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFNb + b'\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFN + '\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwd()) + self.assertEqual(realpath(path, strict=ALL_BUT_LAST), os.getcwd()) + self.assertEqual(realpath(path, strict=True), os.getcwd()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwd()) + path = ABSTFNb + b'\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwdb()) + self.assertEqual(realpath(path, strict=ALL_BUT_LAST), os.getcwdb()) + self.assertEqual(realpath(path, strict=True), os.getcwdb()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwdb()) + path = ABSTFN + '\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFN + '\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFN + '\\nonexistent') + path = ABSTFNb + b'\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFNb + b'\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFNb + b'\\nonexistent') + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_invalid_unicode_paths(self, kwargs): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFNb + b'\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\\nonexistent\\\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\\nonexistent\\\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_relative(self): + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ntpath.relpath(ABSTFN + "1")) - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -565,40 +771,114 @@ def test_realpath_symlink_loops_strict(self): self.addCleanup(os_helper.unlink, ABSTFN + "a") os.symlink(ABSTFN, ABSTFN) + self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=ALL_BUT_LAST) self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=True) os.symlink(ABSTFN + "1", ABSTFN + "2") os.symlink(ABSTFN + "2", ABSTFN + "1") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1", strict=ALL_BUT_LAST) self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1", strict=True) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2", strict=ALL_BUT_LAST) self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2", strict=True) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x", strict=ALL_BUT_LAST) self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x", strict=True) # Windows eliminates '..' components before resolving links, so the # following call is not expected to raise. + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..", strict=ALL_BUT_LAST), + ntpath.dirname(ABSTFN)) self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..", strict=True), ntpath.dirname(ABSTFN)) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x", strict=ALL_BUT_LAST), + ntpath.dirname(ABSTFN) + "\\x") self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\..\\x", strict=True) os.symlink(ABSTFN + "x", ABSTFN + "y") + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\" + + ntpath.basename(ABSTFN) + "y", + strict=ALL_BUT_LAST), + ABSTFN + "x") self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "y", strict=True) + self.assertRaises(OSError, ntpath.realpath, + ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1", + strict=ALL_BUT_LAST) self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1", strict=True) os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a", strict=ALL_BUT_LAST) self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a", strict=True) os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN)) + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c", strict=ALL_BUT_LAST) self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c", strict=True) # Test using relative path as well. + self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN), + strict=ALL_BUT_LAST) self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN), strict=True) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_symlink_prefix(self): + def test_realpath_symlink_loops_raise(self): + # Symlink loops raise OSError in ALLOW_MISSING mode + ABSTFN = ntpath.abspath(os_helper.TESTFN) + self.addCleanup(os_helper.unlink, ABSTFN) + self.addCleanup(os_helper.unlink, ABSTFN + "1") + self.addCleanup(os_helper.unlink, ABSTFN + "2") + self.addCleanup(os_helper.unlink, ABSTFN + "y") + self.addCleanup(os_helper.unlink, ABSTFN + "c") + self.addCleanup(os_helper.unlink, ABSTFN + "a") + self.addCleanup(os_helper.unlink, ABSTFN + "x") + + os.symlink(ABSTFN, ABSTFN) + self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=ALLOW_MISSING) + + os.symlink(ABSTFN + "1", ABSTFN + "2") + os.symlink(ABSTFN + "2", ABSTFN + "1") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x", + strict=ALLOW_MISSING) + + # Windows eliminates '..' components before resolving links; + # realpath is not expected to raise if this removes the loop. + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."), + ntpath.dirname(ABSTFN)) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"), + ntpath.dirname(ABSTFN) + "\\x") + + os.symlink(ABSTFN + "x", ABSTFN + "y") + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\" + + ntpath.basename(ABSTFN) + "y"), + ABSTFN + "x") + self.assertRaises( + OSError, ntpath.realpath, + ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1", + strict=ALLOW_MISSING) + + os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a", + strict=ALLOW_MISSING) + + os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN)) + + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c", + strict=ALLOW_MISSING) + + # Test using relative path as well. + self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN), + strict=ALLOW_MISSING) + + @os_helper.skip_unless_symlink + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_prefix(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) self.addCleanup(os_helper.unlink, ABSTFN + "3") self.addCleanup(os_helper.unlink, "\\\\?\\" + ABSTFN + "3.") @@ -613,9 +893,9 @@ def test_realpath_symlink_prefix(self): f.write(b'1') os.symlink("\\\\?\\" + ABSTFN + "3.", ABSTFN + "3.link") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3link", **kwargs), ABSTFN + "3") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") # Resolved paths should be usable to open target files @@ -625,14 +905,18 @@ def test_realpath_symlink_prefix(self): self.assertEqual(f.read(), b'1') # When the prefix is included, it is not stripped - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link", **kwargs), "\\\\?\\" + ABSTFN + "3") - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_nul(self): tester("ntpath.realpath('NUL')", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=False)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=True)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=ALL_BUT_LAST)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=ALLOW_MISSING)", r'\\.\NUL') @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @unittest.skipUnless(HAVE_GETSHORTPATHNAME, 'need _getshortpathname') @@ -656,12 +940,20 @@ def test_realpath_cwd(self): self.assertPathEqual(test_file_long, ntpath.realpath(test_file_short)) - with os_helper.change_cwd(test_dir_long): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_long.lower()): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_short): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) + for kwargs in {}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}: + with self.subTest(**kwargs): + with os_helper.change_cwd(test_dir_long): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_long.lower()): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_short): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_permission(self): @@ -682,12 +974,15 @@ def test_realpath_permission(self): # Automatic generation of short names may be disabled on # NTFS volumes for the sake of performance. # They're not supported at all on ReFS and exFAT. - subprocess.run( + p = subprocess.run( # Try to set the short name manually. ['fsutil.exe', 'file', 'setShortName', test_file, 'LONGFI~1.TXT'], creationflags=subprocess.DETACHED_PROCESS ) + if p.returncode: + raise unittest.SkipTest('failed to set short name') + try: self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) except AssertionError: @@ -705,6 +1000,93 @@ def test_realpath_permission(self): self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) + @os_helper.skip_unless_symlink + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_mode(self): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + self.addCleanup(os_helper.rmdir, ABSTFN) + self.addCleanup(os_helper.rmdir, ABSTFN + "/dir") + self.addCleanup(os_helper.unlink, ABSTFN + "/file") + self.addCleanup(os_helper.unlink, ABSTFN + "/dir/file2") + self.addCleanup(os_helper.unlink, ABSTFN + "/link") + self.addCleanup(os_helper.unlink, ABSTFN + "/link2") + self.addCleanup(os_helper.unlink, ABSTFN + "/broken") + self.addCleanup(os_helper.unlink, ABSTFN + "/cycle") + + os.mkdir(ABSTFN) + os.mkdir(ABSTFN + "\\dir") + open(ABSTFN + "\\file", "wb").close() + open(ABSTFN + "\\dir\\file2", "wb").close() + os.symlink("file", ABSTFN + "\\link") + os.symlink("dir", ABSTFN + "\\link2") + os.symlink("nonexistent", ABSTFN + "\\broken") + os.symlink("cycle", ABSTFN + "\\cycle") + def check(path, modes, expected, errno=None): + path = path.replace('/', '\\') + if isinstance(expected, str): + assert errno is None + expected = expected.replace('/', os.sep) + for mode in modes: + with self.subTest(mode=mode): + self.assertEqual(realpath(path, strict=mode), + ABSTFN + expected) + else: + for mode in modes: + with self.subTest(mode=mode): + with self.assertRaises(expected) as cm: + realpath(path, strict=mode) + if errno is not None: + self.assertEqual(cm.exception.errno, errno) + + self.enterContext(os_helper.change_cwd(ABSTFN)) + all_modes = [False, ALLOW_MISSING, ALL_BUT_LAST, True] + check("file", all_modes, "/file") + check("file/", all_modes, "/file") + check("file/file2", [False, ALLOW_MISSING], "/file/file2") + check("file/file2", [ALL_BUT_LAST, True], FileNotFoundError) + check("file/.", all_modes, "/file") + check("file/../link2", all_modes, "/dir") + + check("dir", all_modes, "/dir") + check("dir/", all_modes, "/dir") + check("dir/file2", all_modes, "/dir/file2") + + check("link", all_modes, "/file") + check("link/", all_modes, "/file") + check("link/file2", [False, ALLOW_MISSING], "/file/file2") + check("link/file2", [ALL_BUT_LAST, True], FileNotFoundError) + check("link/.", all_modes, "/file") + check("link/../link", all_modes, "/file") + + check("link2", all_modes, "/dir") + check("link2/", all_modes, "/dir") + check("link2/file2", all_modes, "/dir/file2") + + check("nonexistent", [False, ALLOW_MISSING, ALL_BUT_LAST], "/nonexistent") + check("nonexistent", [True], FileNotFoundError) + check("nonexistent/", [False, ALLOW_MISSING, ALL_BUT_LAST], "/nonexistent") + check("nonexistent/", [True], FileNotFoundError) + check("nonexistent/file", [False, ALLOW_MISSING], "/nonexistent/file") + check("nonexistent/file", [ALL_BUT_LAST, True], FileNotFoundError) + check("nonexistent/../link", all_modes, "/file") + + check("broken", [False, ALLOW_MISSING, ALL_BUT_LAST], "/nonexistent") + check("broken", [True], FileNotFoundError) + check("broken/", [False, ALLOW_MISSING, ALL_BUT_LAST], "/nonexistent") + check("broken/", [True], FileNotFoundError) + check("broken/file", [False, ALLOW_MISSING], "/nonexistent/file") + check("broken/file", [ALL_BUT_LAST, True], FileNotFoundError) + check("broken/../link", all_modes, "/file") + + check("cycle", [False], "/cycle") + check("cycle", [ALLOW_MISSING, ALL_BUT_LAST, True], OSError, errno.EINVAL) + check("cycle/", [False], "/cycle") + check("cycle/", [ALLOW_MISSING, ALL_BUT_LAST, True], OSError, errno.EINVAL) + check("cycle/file", [False], "/cycle/file") + check("cycle/file", [ALLOW_MISSING, ALL_BUT_LAST, True], OSError, errno.EINVAL) + check("cycle/../link", all_modes, "/file") + def test_expandvars(self): with os_helper.EnvironmentVarGuard() as env: env.clear() @@ -751,6 +1133,19 @@ def check(value, expected): check('%spam%bar', '%sbar' % nonascii) check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = ntpath.expandvars + with os_helper.EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('%A%'*n), 'B'*n) + self.assertEqual(expandvars('%A%A'*n), 'BA'*n) + self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%') + self.assertEqual(expandvars("%%"*n), "%"*n) + self.assertEqual(expandvars("$$"*n), "$"*n) + def test_expanduser(self): tester('ntpath.expanduser("test")', 'test') @@ -812,8 +1207,6 @@ def test_abspath(self): tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) - self.assertEqual(ntpath.abspath("C:\x00"), ntpath.join(ntpath.abspath("C:"), "\x00")) - self.assertEqual(ntpath.abspath("\x00:spam"), "\x00:\\spam") tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") @@ -847,6 +1240,26 @@ def test_abspath(self): drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") + def test_abspath_invalid_paths(self): + abspath = ntpath.abspath + if sys.platform == 'win32': + self.assertEqual(abspath("C:\x00"), ntpath.join(abspath("C:"), "\x00")) + self.assertEqual(abspath(b"C:\x00"), ntpath.join(abspath(b"C:"), b"\x00")) + self.assertEqual(abspath("\x00:spam"), "\x00:\\spam") + self.assertEqual(abspath(b"\x00:spam"), b"\x00:\\spam") + self.assertEqual(abspath('c:\\fo\x00o'), 'c:\\fo\x00o') + self.assertEqual(abspath(b'c:\\fo\x00o'), b'c:\\fo\x00o') + self.assertEqual(abspath('c:\\fo\x00o\\..\\bar'), 'c:\\bar') + self.assertEqual(abspath(b'c:\\fo\x00o\\..\\bar'), b'c:\\bar') + self.assertEqual(abspath('c:\\\udfff'), 'c:\\\udfff') + self.assertEqual(abspath('c:\\\udfff\\..\\foo'), 'c:\\foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff') + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff\\..\\foo') + else: + self.assertEqual(abspath(b'c:\\\xff'), b'c:\\\xff') + self.assertEqual(abspath(b'c:\\\xff\\..\\foo'), b'c:\\foo') + def test_relpath(self): tester('ntpath.relpath("a")', 'a') tester('ntpath.relpath(ntpath.abspath("a"))', 'a') @@ -989,6 +1402,18 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) + def test_ismount_invalid_paths(self): + ismount = ntpath.ismount + self.assertFalse(ismount("c:\\\udfff")) + if sys.platform == 'win32': + self.assertRaises(ValueError, ismount, "c:\\\x00") + self.assertRaises(ValueError, ismount, b"c:\\\x00") + self.assertRaises(UnicodeDecodeError, ismount, b"c:\\\xff") + else: + self.assertFalse(ismount("c:\\\x00")) + self.assertFalse(ismount(b"c:\\\x00")) + self.assertFalse(ismount(b"c:\\\xff")) + def test_isreserved(self): self.assertFalse(ntpath.isreserved('')) self.assertFalse(ntpath.isreserved('.')) @@ -1095,6 +1520,13 @@ def test_isjunction(self): self.assertFalse(ntpath.isjunction('tmpdir')) self.assertPathEqual(ntpath.realpath('testjunc'), ntpath.realpath('tmpdir')) + def test_isfile_invalid_paths(self): + isfile = ntpath.isfile + self.assertIs(isfile('/tmp\udfffabcds'), False) + self.assertIs(isfile(b'/tmp\xffabcds'), False) + self.assertIs(isfile('/tmp\x00abcds'), False) + self.assertIs(isfile(b'/tmp\x00abcds'), False) + @unittest.skipIf(sys.platform != 'win32', "drive letters are a windows concept") def test_isfile_driveletter(self): drive = os.environ.get('SystemDrive') @@ -1131,7 +1563,7 @@ def test_con_device(self): self.assertTrue(os.path.exists(r"\\.\CON")) @unittest.skipIf(sys.platform != 'win32', "Fast paths are only for win32") - @cpython_only + @support.cpython_only def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in @@ -1195,9 +1627,6 @@ def _check_function(self, func): def test_path_normcase(self): self._check_function(self.path.normcase) - if sys.platform == 'win32': - self.assertEqual(ntpath.normcase('\u03a9\u2126'), 'ωΩ') - self.assertEqual(ntpath.normcase('abc\x00def'), 'abc\x00def') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index a45aafc63fa..4113b79ef5c 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -567,6 +567,14 @@ def test(default=None): with self.assertRaises(RecursionError): test() + def test_dont_specialize_custom_vectorcall(self): + def f(): + raise Exception("no way") + + _testinternalcapi.set_vectorcall_nop(f) + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f() + def make_deferred_ref_count_obj(): """Create an object that uses deferred reference counting. @@ -582,7 +590,7 @@ def make_deferred_ref_count_obj(): class TestRacesDoNotCrash(TestBase): # Careful with these. Bigger numbers have a higher chance of catching bugs, # but you can also burn through a *ton* of type/dict/function versions: - ITEMS = 1000 + ITEMS = 1400 LOOPS = 4 WRITERS = 2 @@ -1333,6 +1341,21 @@ def binary_op_add_int(): self.assert_specialized(binary_op_add_int, "BINARY_OP_ADD_INT") self.assert_no_opcode(binary_op_add_int, "BINARY_OP") + def binary_op_int_non_compact(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 10000000000, 1 + c = a + b + self.assertEqual(c, 10000000001) + c = a - b + self.assertEqual(c, 9999999999) + c = a * b + self.assertEqual(c, 10000000000) + + binary_op_int_non_compact() + self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_ADD_INT") + self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_SUBTRACT_INT") + self.assert_no_opcode(binary_op_int_non_compact, "BINARY_OP_MULTIPLY_INT") + def binary_op_add_unicode(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = "foo", "bar" @@ -1810,20 +1833,6 @@ def compare_op_str(): self.assert_specialized(compare_op_str, "COMPARE_OP_STR") self.assert_no_opcode(compare_op_str, "COMPARE_OP") - @cpython_only - @requires_specialization_ft - def test_load_const(self): - def load_const(): - def unused(): pass - # Currently, the empty tuple is immortal, and the otherwise - # unused nested function's code object is mortal. This test will - # have to use different values if either of that changes. - return () - - load_const() - self.assert_specialized(load_const, "LOAD_CONST_IMMORTAL") - self.assert_specialized(load_const, "LOAD_CONST_MORTAL") - self.assert_no_opcode(load_const, "LOAD_CONST") @cpython_only @requires_specialization_ft @@ -1863,6 +1872,33 @@ def for_iter_generator(): self.assert_specialized(for_iter_generator, "FOR_ITER_GEN") self.assert_no_opcode(for_iter_generator, "FOR_ITER") + @cpython_only + @requires_specialization_ft + def test_call_list_append(self): + # gh-141367: only exact lists should use + # CALL_LIST_APPEND instruction after specialization. + + r = range(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + def list_append(l): + for _ in r: + l.append(1) + + list_append([]) + self.assert_specialized(list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(list_append, "CALL") + + def my_list_append(l): + for _ in r: + l.append(1) + + class MyList(list): pass + my_list_append(MyList()) + self.assert_specialized(my_list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(my_list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(my_list_append, "CALL") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index e6ffd2b0ffe..fc8ef9520b3 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -615,9 +615,9 @@ def test_float_default(self): self.parser.add_option( "-p", "--prob", help="blow up with probability PROB [default: %default]") - self.parser.set_defaults(prob=0.43) + self.parser.set_defaults(prob=0.25) expected_help = self.help_prefix + \ - " -p PROB, --prob=PROB blow up with probability PROB [default: 0.43]\n" + " -p PROB, --prob=PROB blow up with probability PROB [default: 0.25]\n" self.assertHelp(self.parser, expected_help) def test_alt_expand(self): @@ -1666,6 +1666,16 @@ def test_translations(self): self.assertMsgidsEqual(optparse) +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(optparse, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == '__main__': # To regenerate translation snapshots if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 9f131a9110d..4204a6a47d2 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -147,7 +147,7 @@ def test_fromkeys(self): def test_abc(self): OrderedDict = self.OrderedDict self.assertIsInstance(OrderedDict(), MutableMapping) - self.assertTrue(issubclass(OrderedDict, MutableMapping)) + self.assertIsSubclass(OrderedDict, MutableMapping) def test_clear(self): OrderedDict = self.OrderedDict @@ -314,14 +314,14 @@ def check(dup): check(dup) self.assertIs(dup.x, od.x) self.assertIs(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') dup = copy.deepcopy(od) check(dup) self.assertEqual(dup.x, od.x) self.assertIsNot(dup.x, od.x) self.assertEqual(dup.z, od.z) self.assertIsNot(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') # pickle directly pulls the module, so we have to fake it with replaced_module('collections', self.module): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -330,7 +330,7 @@ def check(dup): check(dup) self.assertEqual(dup.x, od.x) self.assertEqual(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') check(eval(repr(od))) update_test = OrderedDict() update_test.update(od) diff --git a/Lib/test/test_os/__init__.py b/Lib/test/test_os/__init__.py new file mode 100644 index 00000000000..bc502ef32d2 --- /dev/null +++ b/Lib/test/test_os/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_os.py b/Lib/test/test_os/test_os.py similarity index 88% rename from Lib/test/test_os.py rename to Lib/test/test_os/test_os.py index 333179a71e3..ddb8a63095b 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -7,13 +7,11 @@ import contextlib import decimal import errno -import fnmatch import fractions import itertools import locale import os import pickle -import platform import select import selectors import shutil @@ -32,12 +30,12 @@ import uuid import warnings from test import support -from test.support import import_helper from test.support import os_helper from test.support import socket_helper from test.support import infinite_recursion from test.support import warnings_helper from platform import win32_is_iot +from .utils import create_file try: import resource @@ -47,10 +45,6 @@ import fcntl except ImportError: fcntl = None -try: - import _winapi -except ImportError: - _winapi = None try: import pwd all_users = [u.pw_uid for u in pwd.getpwall()] @@ -94,18 +88,13 @@ def requires_os_func(name): return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) -def create_file(filename, content=b'content'): - with open(filename, "xb", 0) as fp: - fp.write(content) - - # bpo-41625: On AIX, splice() only works with a socket, not with a pipe. requires_splice_pipe = unittest.skipIf(sys.platform.startswith("aix"), 'on AIX, splice() only accepts sockets') def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MiscTests(unittest.TestCase): @@ -175,6 +164,20 @@ def test_getcwdb(self): self.assertIsInstance(cwd, bytes) self.assertEqual(os.fsdecode(cwd), os.getcwd()) + def test_type_fqdn(self): + def fqdn(obj): + return (obj.__module__, obj.__qualname__) + + native = os.name + self.assertEqual(fqdn(os.stat_result), ("os", "stat_result")) + self.assertEqual(fqdn(os.times_result), (native, "times_result")) + if hasattr(os, "statvfs_result"): + self.assertEqual(fqdn(os.statvfs_result), ("os", "statvfs_result")) + if hasattr(os, "sched_param"): + self.assertEqual(fqdn(os.sched_param), (native, "sched_param")) + if hasattr(os, "waitid_result"): + self.assertEqual(fqdn(os.waitid_result), (native, "waitid_result")) + # Tests creating TESTFN class FileTests(unittest.TestCase): @@ -641,6 +644,15 @@ def setUp(self): self.addCleanup(os_helper.unlink, self.fname) create_file(self.fname, b"ABC") + def check_timestamp_agreement(self, result, names): + # Make sure that the st_?time and st_?time_ns fields roughly agree + # (they should always agree up to around tens-of-microseconds) + for name in names: + with self.subTest(name=name): + floaty = int(getattr(result, name) * 100_000) + nanosecondy = getattr(result, name + "_ns") // 10_000 + self.assertAlmostEqual(floaty, nanosecondy, delta=2, msg=name) + def check_stat_attributes(self, fname): result = os.stat(fname) @@ -661,21 +673,15 @@ def trunc(x): return x result[getattr(stat, name)]) self.assertIn(attr, members) - # Make sure that the st_?time and st_?time_ns fields roughly agree - # (they should always agree up to around tens-of-microseconds) - for name in 'st_atime st_mtime st_ctime'.split(): - floaty = int(getattr(result, name) * 100000) - nanosecondy = getattr(result, name + "_ns") // 10000 - self.assertAlmostEqual(floaty, nanosecondy, delta=2) - - # Ensure both birthtime and birthtime_ns roughly agree, if present + time_attributes = ['st_atime', 'st_mtime', 'st_ctime'] try: - floaty = int(result.st_birthtime * 100000) - nanosecondy = result.st_birthtime_ns // 10000 + result.st_birthtime + result.st_birthtime_ns except AttributeError: pass else: - self.assertAlmostEqual(floaty, nanosecondy, delta=2) + time_attributes.append('st_birthtime') + self.check_timestamp_agreement(result, time_attributes) try: result[200] @@ -736,6 +742,160 @@ def test_stat_result_pickle(self): unpickled = pickle.loads(p) self.assertEqual(result, unpickled) + def check_statx_attributes(self, filename): + maximal_mask = 0 + for name in dir(os): + if name.startswith('STATX_'): + maximal_mask |= getattr(os, name) + result = os.statx(filename, maximal_mask) + stat_result = os.stat(filename) + + time_attributes = ('stx_atime', 'stx_btime', 'stx_ctime', 'stx_mtime') + # gh-83714: stx_btime can be None on tmpfs even if STATX_BTIME mask + # is used + time_attributes = [name for name in time_attributes + if getattr(result, name) is not None] + self.check_timestamp_agreement(result, time_attributes) + + def getmask(name): + return getattr(os, name, 0) + + requirements = ( + ('stx_atime', os.STATX_ATIME), + ('stx_atime_ns', os.STATX_ATIME), + ('stx_atomic_write_segments_max', getmask('STATX_WRITE_ATOMIC')), + ('stx_atomic_write_unit_max', getmask('STATX_WRITE_ATOMIC')), + ('stx_atomic_write_unit_max_opt', getmask('STATX_WRITE_ATOMIC')), + ('stx_atomic_write_unit_min', getmask('STATX_WRITE_ATOMIC')), + ('stx_attributes', 0), + ('stx_attributes_mask', 0), + ('stx_blksize', 0), + ('stx_blocks', os.STATX_BLOCKS), + ('stx_btime', os.STATX_BTIME), + ('stx_btime_ns', os.STATX_BTIME), + ('stx_ctime', os.STATX_CTIME), + ('stx_ctime_ns', os.STATX_CTIME), + ('stx_dev', 0), + ('stx_dev_major', 0), + ('stx_dev_minor', 0), + ('stx_dio_mem_align', getmask('STATX_DIOALIGN')), + ('stx_dio_offset_align', getmask('STATX_DIOALIGN')), + ('stx_dio_read_offset_align', getmask('STATX_DIO_READ_ALIGN')), + ('stx_gid', os.STATX_GID), + ('stx_ino', os.STATX_INO), + ('stx_mask', 0), + ('stx_mnt_id', getmask('STATX_MNT_ID')), + ('stx_mode', os.STATX_TYPE | os.STATX_MODE), + ('stx_mtime', os.STATX_MTIME), + ('stx_mtime_ns', os.STATX_MTIME), + ('stx_nlink', os.STATX_NLINK), + ('stx_rdev', 0), + ('stx_rdev_major', 0), + ('stx_rdev_minor', 0), + ('stx_size', os.STATX_SIZE), + ('stx_subvol', getmask('STATX_SUBVOL')), + ('stx_uid', os.STATX_UID), + ) + optional_members = { + 'stx_atomic_write_segments_max', + 'stx_atomic_write_unit_max', + 'stx_atomic_write_unit_max_opt', + 'stx_atomic_write_unit_min', + 'stx_dio_mem_align', + 'stx_dio_offset_align', + 'stx_dio_read_offset_align', + 'stx_mnt_id', + 'stx_subvol', + } + float_type = { + 'stx_atime', + 'stx_btime', + 'stx_ctime', + 'stx_mtime', + } + + members = set(name for name in dir(result) + if name.startswith('stx_')) + tested = set(name for name, mask in requirements) + if members - tested: + raise ValueError(f"statx members not tested: {members - tested}") + + for name, mask in requirements: + with self.subTest(name=name): + try: + x = getattr(result, name) + except AttributeError: + if name in optional_members: + continue + else: + raise + + if not(result.stx_mask & mask == mask): + self.assertIsNone(x) + continue + + if name in float_type: + self.assertIsInstance(x, float) + else: + self.assertIsInstance(x, int) + + # Compare with stat_result + try: + b = getattr(stat_result, "st_" + name[4:]) + except AttributeError: + pass + else: + self.assertEqual(type(x), type(b)) + if isinstance(x, float): + self.assertAlmostEqual(x, b) + else: + self.assertEqual(x, b) + + self.assertEqual(result.stx_rdev_major, os.major(result.stx_rdev)) + self.assertEqual(result.stx_rdev_minor, os.minor(result.stx_rdev)) + self.assertEqual(result.stx_dev_major, os.major(result.stx_dev)) + self.assertEqual(result.stx_dev_minor, os.minor(result.stx_dev)) + + self.assertEqual(result.stx_attributes & result.stx_attributes_mask, + result.stx_attributes) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes(self): + self.check_statx_attributes(self.fname) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes_bytes(self): + try: + fname = self.fname.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + self.skipTest("cannot encode %a for the filesystem" % self.fname) + self.check_statx_attributes(fname) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes_pathlike(self): + self.check_statx_attributes(FakePath(self.fname)) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_result(self): + result = os.statx(self.fname, os.STATX_BASIC_STATS) + + # Check that attributes are read-only + members = [name for name in dir(result) + if name.startswith('stx_')] + for name in members: + try: + setattr(result, name, 1) + except AttributeError: + pass + else: + self.fail("No exception raised") + + # statx_result is not a tuple or tuple-like object. + with self.assertRaisesRegex(TypeError, 'not subscriptable'): + result[0] + with self.assertRaisesRegex(TypeError, 'cannot unpack'): + _, _ = result + @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') def test_statvfs_attributes(self): result = os.statvfs(self.fname) @@ -818,7 +978,7 @@ def test_15261(self): self.assertEqual(ctx.exception.errno, errno.EBADF) def check_file_attributes(self, result): - self.assertTrue(hasattr(result, 'st_file_attributes')) + self.assertHasAttr(result, 'st_file_attributes') self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) @@ -949,6 +1109,20 @@ def ns_to_sec(ns): # issue, os.utime() rounds towards minus infinity. return (ns * 1e-9) + 0.5e-9 + @staticmethod + def ns_to_sec_decimal(ns): + # Convert a number of nanosecond (int) to a number of seconds (Decimal). + # Round towards infinity by adding 0.5 nanosecond to avoid rounding + # issue, os.utime() rounds towards minus infinity. + return decimal.Decimal('1e-9') * ns + decimal.Decimal('0.5e-9') + + @staticmethod + def ns_to_sec_fraction(ns): + # Convert a number of nanosecond (int) to a number of seconds (Fraction). + # Round towards infinity by adding 0.5 nanosecond to avoid rounding + # issue, os.utime() rounds towards minus infinity. + return fractions.Fraction(ns, 10**9) + fractions.Fraction(1, 2*10**9) + def test_utime_by_indexed(self): # pass times as floating-point seconds as the second indexed parameter def set_time(filename, ns): @@ -969,6 +1143,24 @@ def set_time(filename, ns): os.utime(filename, times=(atime, mtime)) self._test_utime(set_time) + def test_utime_decimal(self): + # pass times as Decimal seconds + def set_time(filename, ns): + atime_ns, mtime_ns = ns + atime = self.ns_to_sec_decimal(atime_ns) + mtime = self.ns_to_sec_decimal(mtime_ns) + os.utime(filename, (atime, mtime)) + self._test_utime(set_time) + + def test_utime_fraction(self): + # pass times as Fraction seconds + def set_time(filename, ns): + atime_ns, mtime_ns = ns + atime = self.ns_to_sec_fraction(atime_ns) + mtime = self.ns_to_sec_fraction(mtime_ns) + os.utime(filename, (atime, mtime)) + self._test_utime(set_time) + @unittest.skipUnless(os.utime in os.supports_follow_symlinks, "follow_symlinks support for utime required " "for this test.") @@ -1065,9 +1257,15 @@ def test_large_time(self): if self.get_file_system(self.dirname) != "NTFS": self.skipTest("requires NTFS") - large = 5000000000 # some day in 2128 - os.utime(self.fname, (large, large)) - self.assertEqual(os.stat(self.fname).st_mtime, large) + times = ( + 5000000000, # some day in 2128 + # boundaries of the fast path cutoff in posixmodule.c:fill_time + -9223372037, -9223372036, 9223372035, 9223372036, + ) + for large in times: + with self.subTest(large=large): + os.utime(self.fname, (large, large)) + self.assertEqual(os.stat(self.fname).st_mtime, large) def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive @@ -1467,6 +1665,14 @@ def test_reload_environ(self): self.assertNotIn(b'test_env', os.environb) self.assertNotIn('test_env', os.environ) + def test_clearenv(self): + os.environ['REMOVEME'] = '1' + os.environ.clear() + self.assertEqual(os.environ, {}) + + self.assertRaises(TypeError, os.environ.clear, None) + + class WalkTests(unittest.TestCase): """Tests for os.walk().""" is_fwalk = False @@ -1920,6 +2126,8 @@ def test_makedir(self): "WASI's umask is a stub." ) def test_mode(self): + # Note: in some cases, the umask might already be 2 in which case this + # will pass even if os.umask is actually broken. with os_helper.temp_umask(0o002): base = os_helper.TESTFN parent = os.path.join(base, 'dir1') @@ -2181,7 +2389,7 @@ def test_getrandom0(self): self.assertEqual(empty, b'') def test_getrandom_random(self): - self.assertTrue(hasattr(os, 'GRND_RANDOM')) + self.assertHasAttr(os, 'GRND_RANDOM') # Don't test os.getrandom(1, os.GRND_RANDOM) to not consume the rare # resource /dev/random @@ -2427,42 +2635,6 @@ def test_execve_with_empty_path(self): self.fail('No OSError raised') -@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") -class Win32ErrorTests(unittest.TestCase): - def setUp(self): - try: - os.stat(os_helper.TESTFN) - except FileNotFoundError: - exists = False - except OSError as exc: - exists = True - self.fail("file %s must not exist; os.stat failed with %s" - % (os_helper.TESTFN, exc)) - else: - self.fail("file %s must not exist" % os_helper.TESTFN) - - def test_rename(self): - self.assertRaises(OSError, os.rename, os_helper.TESTFN, os_helper.TESTFN+".bak") - - def test_remove(self): - self.assertRaises(OSError, os.remove, os_helper.TESTFN) - - def test_chdir(self): - self.assertRaises(OSError, os.chdir, os_helper.TESTFN) - - def test_mkdir(self): - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - - with open(os_helper.TESTFN, "x") as f: - self.assertRaises(OSError, os.mkdir, os_helper.TESTFN) - - def test_utime(self): - self.assertRaises(OSError, os.utime, os_helper.TESTFN, None) - - def test_chmod(self): - self.assertRaises(OSError, os.chmod, os_helper.TESTFN, 0) - - @unittest.skipIf(support.is_wasi, "Cannot create invalid FD on WASI.") class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fstat", "fstatvfs", "tcgetpgrp", "ttyname"] @@ -2797,224 +2969,6 @@ def test_stat(self): for fn in self.unicodefn: os.stat(os.path.join(self.dir, fn)) -@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") -class Win32KillTests(unittest.TestCase): - def _kill(self, sig): - # Start sys.executable as a subprocess and communicate from the - # subprocess to the parent that the interpreter is ready. When it - # becomes ready, send *sig* via os.kill to the subprocess and check - # that the return code is equal to *sig*. - import ctypes - from ctypes import wintypes - import msvcrt - - # Since we can't access the contents of the process' stdout until the - # process has exited, use PeekNamedPipe to see what's inside stdout - # without waiting. This is done so we can tell that the interpreter - # is started and running at a point where it could handle a signal. - PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe - PeekNamedPipe.restype = wintypes.BOOL - PeekNamedPipe.argtypes = (wintypes.HANDLE, # Pipe handle - ctypes.POINTER(ctypes.c_char), # stdout buf - wintypes.DWORD, # Buffer size - ctypes.POINTER(wintypes.DWORD), # bytes read - ctypes.POINTER(wintypes.DWORD), # bytes avail - ctypes.POINTER(wintypes.DWORD)) # bytes left - msg = "running" - proc = subprocess.Popen([sys.executable, "-c", - "import sys;" - "sys.stdout.write('{}');" - "sys.stdout.flush();" - "input()".format(msg)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE) - self.addCleanup(proc.stdout.close) - self.addCleanup(proc.stderr.close) - self.addCleanup(proc.stdin.close) - - count, max = 0, 100 - while count < max and proc.poll() is None: - # Create a string buffer to store the result of stdout from the pipe - buf = ctypes.create_string_buffer(len(msg)) - # Obtain the text currently in proc.stdout - # Bytes read/avail/left are left as NULL and unused - rslt = PeekNamedPipe(msvcrt.get_osfhandle(proc.stdout.fileno()), - buf, ctypes.sizeof(buf), None, None, None) - self.assertNotEqual(rslt, 0, "PeekNamedPipe failed") - if buf.value: - self.assertEqual(msg, buf.value.decode()) - break - time.sleep(0.1) - count += 1 - else: - self.fail("Did not receive communication from the subprocess") - - os.kill(proc.pid, sig) - self.assertEqual(proc.wait(), sig) - - def test_kill_sigterm(self): - # SIGTERM doesn't mean anything special, but make sure it works - self._kill(signal.SIGTERM) - - def test_kill_int(self): - # os.kill on Windows can take an int which gets set as the exit code - self._kill(100) - - @unittest.skipIf(mmap is None, "requires mmap") - def _kill_with_event(self, event, name): - tagname = "test_os_%s" % uuid.uuid1() - m = mmap.mmap(-1, 1, tagname) - m[0] = 0 - - # Run a script which has console control handling enabled. - script = os.path.join(os.path.dirname(__file__), - "win_console_handler.py") - cmd = [sys.executable, script, tagname] - proc = subprocess.Popen(cmd, - creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) - - with proc: - # Let the interpreter startup before we send signals. See #3137. - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if proc.poll() is None: - break - else: - # Forcefully kill the process if we weren't able to signal it. - proc.kill() - self.fail("Subprocess didn't finish initialization") - - os.kill(proc.pid, event) - - try: - # proc.send_signal(event) could also be done here. - # Allow time for the signal to be passed and the process to exit. - proc.wait(timeout=support.SHORT_TIMEOUT) - except subprocess.TimeoutExpired: - # Forcefully kill the process if we weren't able to signal it. - proc.kill() - self.fail("subprocess did not stop on {}".format(name)) - - @unittest.skip("subprocesses aren't inheriting Ctrl+C property") - @support.requires_subprocess() - def test_CTRL_C_EVENT(self): - from ctypes import wintypes - import ctypes - - # Make a NULL value by creating a pointer with no argument. - NULL = ctypes.POINTER(ctypes.c_int)() - SetConsoleCtrlHandler = ctypes.windll.kernel32.SetConsoleCtrlHandler - SetConsoleCtrlHandler.argtypes = (ctypes.POINTER(ctypes.c_int), - wintypes.BOOL) - SetConsoleCtrlHandler.restype = wintypes.BOOL - - # Calling this with NULL and FALSE causes the calling process to - # handle Ctrl+C, rather than ignore it. This property is inherited - # by subprocesses. - SetConsoleCtrlHandler(NULL, 0) - - self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - - @support.requires_subprocess() - def test_CTRL_BREAK_EVENT(self): - self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") - - -@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") -class Win32ListdirTests(unittest.TestCase): - """Test listdir on Windows.""" - - def setUp(self): - self.created_paths = [] - for i in range(2): - dir_name = 'SUB%d' % i - dir_path = os.path.join(os_helper.TESTFN, dir_name) - file_name = 'FILE%d' % i - file_path = os.path.join(os_helper.TESTFN, file_name) - os.makedirs(dir_path) - with open(file_path, 'w', encoding='utf-8') as f: - f.write("I'm %s and proud of it. Blame test_os.\n" % file_path) - self.created_paths.extend([dir_name, file_name]) - self.created_paths.sort() - - def tearDown(self): - shutil.rmtree(os_helper.TESTFN) - - def test_listdir_no_extended_path(self): - """Test when the path is not an "extended" path.""" - # unicode - self.assertEqual( - sorted(os.listdir(os_helper.TESTFN)), - self.created_paths) - - # bytes - self.assertEqual( - sorted(os.listdir(os.fsencode(os_helper.TESTFN))), - [os.fsencode(path) for path in self.created_paths]) - - def test_listdir_extended_path(self): - """Test when the path starts with '\\\\?\\'.""" - # See: http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath - # unicode - path = '\\\\?\\' + os.path.abspath(os_helper.TESTFN) - self.assertEqual( - sorted(os.listdir(path)), - self.created_paths) - - # bytes - path = b'\\\\?\\' + os.fsencode(os.path.abspath(os_helper.TESTFN)) - self.assertEqual( - sorted(os.listdir(path)), - [os.fsencode(path) for path in self.created_paths]) - - -@unittest.skipUnless(os.name == "nt", "NT specific tests") -class Win32ListdriveTests(unittest.TestCase): - """Test listdrive, listmounts and listvolume on Windows.""" - - def setUp(self): - # Get drives and volumes from fsutil - out = subprocess.check_output( - ["fsutil.exe", "volume", "list"], - cwd=os.path.join(os.getenv("SystemRoot", "\\Windows"), "System32"), - encoding="mbcs", - errors="ignore", - ) - lines = out.splitlines() - self.known_volumes = {l for l in lines if l.startswith('\\\\?\\')} - self.known_drives = {l for l in lines if l[1:] == ':\\'} - self.known_mounts = {l for l in lines if l[1:3] == ':\\'} - - def test_listdrives(self): - drives = os.listdrives() - self.assertIsInstance(drives, list) - self.assertSetEqual( - self.known_drives, - self.known_drives & set(drives), - ) - - def test_listvolumes(self): - volumes = os.listvolumes() - self.assertIsInstance(volumes, list) - self.assertSetEqual( - self.known_volumes, - self.known_volumes & set(volumes), - ) - - def test_listmounts(self): - for volume in os.listvolumes(): - try: - mounts = os.listmounts(volume) - except OSError as ex: - if support.verbose: - print("Skipping", volume, "because of", ex) - else: - self.assertIsInstance(mounts, list) - self.assertSetEqual( - set(mounts), - self.known_mounts & set(mounts), - ) - @unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') class ReadlinkTests(unittest.TestCase): @@ -3077,370 +3031,6 @@ def test_bytes(self): self.assertIsInstance(path, bytes) -@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") -@os_helper.skip_unless_symlink -class Win32SymlinkTests(unittest.TestCase): - filelink = 'filelinktest' - filelink_target = os.path.abspath(__file__) - dirlink = 'dirlinktest' - dirlink_target = os.path.dirname(filelink_target) - missing_link = 'missing link' - - def setUp(self): - assert os.path.exists(self.dirlink_target) - assert os.path.exists(self.filelink_target) - assert not os.path.exists(self.dirlink) - assert not os.path.exists(self.filelink) - assert not os.path.exists(self.missing_link) - - def tearDown(self): - if os.path.exists(self.filelink): - os.remove(self.filelink) - if os.path.exists(self.dirlink): - os.rmdir(self.dirlink) - if os.path.lexists(self.missing_link): - os.remove(self.missing_link) - - def test_directory_link(self): - os.symlink(self.dirlink_target, self.dirlink) - self.assertTrue(os.path.exists(self.dirlink)) - self.assertTrue(os.path.isdir(self.dirlink)) - self.assertTrue(os.path.islink(self.dirlink)) - self.check_stat(self.dirlink, self.dirlink_target) - - def test_file_link(self): - os.symlink(self.filelink_target, self.filelink) - self.assertTrue(os.path.exists(self.filelink)) - self.assertTrue(os.path.isfile(self.filelink)) - self.assertTrue(os.path.islink(self.filelink)) - self.check_stat(self.filelink, self.filelink_target) - - def _create_missing_dir_link(self): - 'Create a "directory" link to a non-existent target' - linkname = self.missing_link - if os.path.lexists(linkname): - os.remove(linkname) - target = r'c:\\target does not exist.29r3c740' - assert not os.path.exists(target) - target_is_dir = True - os.symlink(target, linkname, target_is_dir) - - def test_remove_directory_link_to_missing_target(self): - self._create_missing_dir_link() - # For compatibility with Unix, os.remove will check the - # directory status and call RemoveDirectory if the symlink - # was created with target_is_dir==True. - os.remove(self.missing_link) - - def test_isdir_on_directory_link_to_missing_target(self): - self._create_missing_dir_link() - self.assertFalse(os.path.isdir(self.missing_link)) - - def test_rmdir_on_directory_link_to_missing_target(self): - self._create_missing_dir_link() - os.rmdir(self.missing_link) - - def check_stat(self, link, target): - self.assertEqual(os.stat(link), os.stat(target)) - self.assertNotEqual(os.lstat(link), os.stat(link)) - - bytes_link = os.fsencode(link) - self.assertEqual(os.stat(bytes_link), os.stat(target)) - self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) - - def test_12084(self): - level1 = os.path.abspath(os_helper.TESTFN) - level2 = os.path.join(level1, "level2") - level3 = os.path.join(level2, "level3") - self.addCleanup(os_helper.rmtree, level1) - - os.mkdir(level1) - os.mkdir(level2) - os.mkdir(level3) - - file1 = os.path.abspath(os.path.join(level1, "file1")) - create_file(file1) - - orig_dir = os.getcwd() - try: - os.chdir(level2) - link = os.path.join(level2, "link") - os.symlink(os.path.relpath(file1), "link") - self.assertIn("link", os.listdir(os.getcwd())) - - # Check os.stat calls from the same dir as the link - self.assertEqual(os.stat(file1), os.stat("link")) - - # Check os.stat calls from a dir below the link - os.chdir(level1) - self.assertEqual(os.stat(file1), - os.stat(os.path.relpath(link))) - - # Check os.stat calls from a dir above the link - os.chdir(level3) - self.assertEqual(os.stat(file1), - os.stat(os.path.relpath(link))) - finally: - os.chdir(orig_dir) - - @unittest.skipUnless(os.path.lexists(r'C:\Users\All Users') - and os.path.exists(r'C:\ProgramData'), - 'Test directories not found') - def test_29248(self): - # os.symlink() calls CreateSymbolicLink, which creates - # the reparse data buffer with the print name stored - # first, so the offset is always 0. CreateSymbolicLink - # stores the "PrintName" DOS path (e.g. "C:\") first, - # with an offset of 0, followed by the "SubstituteName" - # NT path (e.g. "\??\C:\"). The "All Users" link, on - # the other hand, seems to have been created manually - # with an inverted order. - target = os.readlink(r'C:\Users\All Users') - self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) - - def test_buffer_overflow(self): - # Older versions would have a buffer overflow when detecting - # whether a link source was a directory. This test ensures we - # no longer crash, but does not otherwise validate the behavior - segment = 'X' * 27 - path = os.path.join(*[segment] * 10) - test_cases = [ - # overflow with absolute src - ('\\' + path, segment), - # overflow dest with relative src - (segment, path), - # overflow when joining src - (path[:180], path[:180]), - ] - for src, dest in test_cases: - try: - os.symlink(src, dest) - except FileNotFoundError: - pass - else: - try: - os.remove(dest) - except OSError: - pass - # Also test with bytes, since that is a separate code path. - try: - os.symlink(os.fsencode(src), os.fsencode(dest)) - except FileNotFoundError: - pass - else: - try: - os.remove(dest) - except OSError: - pass - - def test_appexeclink(self): - root = os.path.expandvars(r'%LOCALAPPDATA%\Microsoft\WindowsApps') - if not os.path.isdir(root): - self.skipTest("test requires a WindowsApps directory") - - aliases = [os.path.join(root, a) - for a in fnmatch.filter(os.listdir(root), '*.exe')] - - for alias in aliases: - if support.verbose: - print() - print("Testing with", alias) - st = os.lstat(alias) - self.assertEqual(st, os.stat(alias)) - self.assertFalse(stat.S_ISLNK(st.st_mode)) - self.assertEqual(st.st_reparse_tag, stat.IO_REPARSE_TAG_APPEXECLINK) - self.assertTrue(os.path.isfile(alias)) - # testing the first one we see is sufficient - break - else: - self.skipTest("test requires an app execution alias") - -@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") -class Win32JunctionTests(unittest.TestCase): - junction = 'junctiontest' - junction_target = os.path.dirname(os.path.abspath(__file__)) - - def setUp(self): - assert os.path.exists(self.junction_target) - assert not os.path.lexists(self.junction) - - def tearDown(self): - if os.path.lexists(self.junction): - os.unlink(self.junction) - - def test_create_junction(self): - _winapi.CreateJunction(self.junction_target, self.junction) - self.assertTrue(os.path.lexists(self.junction)) - self.assertTrue(os.path.exists(self.junction)) - self.assertTrue(os.path.isdir(self.junction)) - self.assertNotEqual(os.stat(self.junction), os.lstat(self.junction)) - self.assertEqual(os.stat(self.junction), os.stat(self.junction_target)) - - # bpo-37834: Junctions are not recognized as links. - self.assertFalse(os.path.islink(self.junction)) - self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), - os.path.normcase(os.readlink(self.junction))) - - def test_unlink_removes_junction(self): - _winapi.CreateJunction(self.junction_target, self.junction) - self.assertTrue(os.path.exists(self.junction)) - self.assertTrue(os.path.lexists(self.junction)) - - os.unlink(self.junction) - self.assertFalse(os.path.exists(self.junction)) - -@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") -class Win32NtTests(unittest.TestCase): - def test_getfinalpathname_handles(self): - nt = import_helper.import_module('nt') - ctypes = import_helper.import_module('ctypes') - # Ruff false positive -- it thinks we're redefining `ctypes` here - import ctypes.wintypes # noqa: F811 - - kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) - kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE - - kernel.GetProcessHandleCount.restype = ctypes.wintypes.BOOL - kernel.GetProcessHandleCount.argtypes = (ctypes.wintypes.HANDLE, - ctypes.wintypes.LPDWORD) - - # This is a pseudo-handle that doesn't need to be closed - hproc = kernel.GetCurrentProcess() - - handle_count = ctypes.wintypes.DWORD() - ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) - self.assertEqual(1, ok) - - before_count = handle_count.value - - # The first two test the error path, __file__ tests the success path - filenames = [ - r'\\?\C:', - r'\\?\NUL', - r'\\?\CONIN', - __file__, - ] - - for _ in range(10): - for name in filenames: - try: - nt._getfinalpathname(name) - except Exception: - # Failure is expected - pass - try: - os.stat(name) - except Exception: - pass - - ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) - self.assertEqual(1, ok) - - handle_delta = handle_count.value - before_count - - self.assertEqual(0, handle_delta) - - @support.requires_subprocess() - def test_stat_unlink_race(self): - # bpo-46785: the implementation of os.stat() falls back to reading - # the parent directory if CreateFileW() fails with a permission - # error. If reading the parent directory fails because the file or - # directory are subsequently unlinked, or because the volume or - # share are no longer available, then the original permission error - # should not be restored. - filename = os_helper.TESTFN - self.addCleanup(os_helper.unlink, filename) - deadline = time.time() + 5 - command = textwrap.dedent("""\ - import os - import sys - import time - - filename = sys.argv[1] - deadline = float(sys.argv[2]) - - while time.time() < deadline: - try: - with open(filename, "w") as f: - pass - except OSError: - pass - try: - os.remove(filename) - except OSError: - pass - """) - - with subprocess.Popen([sys.executable, '-c', command, filename, str(deadline)]) as proc: - while time.time() < deadline: - try: - os.stat(filename) - except FileNotFoundError as e: - assert e.winerror == 2 # ERROR_FILE_NOT_FOUND - try: - proc.wait(1) - except subprocess.TimeoutExpired: - proc.terminate() - - @support.requires_subprocess() - def test_stat_inaccessible_file(self): - filename = os_helper.TESTFN - ICACLS = os.path.expandvars(r"%SystemRoot%\System32\icacls.exe") - - with open(filename, "wb") as f: - f.write(b'Test data') - - stat1 = os.stat(filename) - - try: - # Remove all permissions from the file - subprocess.check_output([ICACLS, filename, "/inheritance:r"], - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as ex: - if support.verbose: - print(ICACLS, filename, "/inheritance:r", "failed.") - print(ex.stdout.decode("oem", "replace").rstrip()) - try: - os.unlink(filename) - except OSError: - pass - self.skipTest("Unable to create inaccessible file") - - def cleanup(): - # Give delete permission to the owner (us) - subprocess.check_output([ICACLS, filename, "/grant", "*WD:(D)"], - stderr=subprocess.STDOUT) - os.unlink(filename) - - self.addCleanup(cleanup) - - if support.verbose: - print("File:", filename) - print("stat with access:", stat1) - - # First test - we shouldn't raise here, because we still have access to - # the directory and can extract enough information from its metadata. - stat2 = os.stat(filename) - - if support.verbose: - print(" without access:", stat2) - - # We may not get st_dev/st_ino, so ensure those are 0 or match - self.assertIn(stat2.st_dev, (0, stat1.st_dev)) - self.assertIn(stat2.st_ino, (0, stat1.st_ino)) - - # st_mode and st_size should match (for a normal file, at least) - self.assertEqual(stat1.st_mode, stat2.st_mode) - self.assertEqual(stat1.st_size, stat2.st_size) - - # st_ctime and st_mtime should be the same - self.assertEqual(stat1.st_ctime, stat2.st_ctime) - self.assertEqual(stat1.st_mtime, stat2.st_mtime) - - # st_atime should be the same or later - self.assertGreaterEqual(stat1.st_atime, stat2.st_atime) - - @os_helper.skip_unless_symlink class NonLocalSymlinkTests(unittest.TestCase): @@ -3517,6 +3107,7 @@ def test_getppid(self): self.assertEqual(error, b'') self.assertEqual(int(stdout), os.getpid()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def check_waitpid(self, code, exitcode, callback=None): if sys.platform == 'win32': # On Windows, os.spawnv() simply joins arguments with spaces: @@ -3619,30 +3210,35 @@ def create_args(self, *, with_env=False, use_bytes=False): return program, args + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnl') def test_spawnl(self): program, args = self.create_args() exitcode = os.spawnl(os.P_WAIT, program, *args) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnle') def test_spawnle(self): program, args = self.create_args(with_env=True) exitcode = os.spawnle(os.P_WAIT, program, *args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnlp') def test_spawnlp(self): program, args = self.create_args() exitcode = os.spawnlp(os.P_WAIT, program, *args) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnlpe') def test_spawnlpe(self): program, args = self.create_args(with_env=True) exitcode = os.spawnlpe(os.P_WAIT, program, *args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnv') def test_spawnv(self): program, args = self.create_args() @@ -3653,30 +3249,35 @@ def test_spawnv(self): exitcode = os.spawnv(os.P_WAIT, FakePath(program), args) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnve') def test_spawnve(self): program, args = self.create_args(with_env=True) exitcode = os.spawnve(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnvp') def test_spawnvp(self): program, args = self.create_args() exitcode = os.spawnvp(os.P_WAIT, program, args) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnvpe') def test_spawnvpe(self): program, args = self.create_args(with_env=True) exitcode = os.spawnvpe(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnv') def test_nowait(self): program, args = self.create_args() pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -3684,18 +3285,21 @@ def test_spawnve_bytes(self): exitcode = os.spawnve(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnl') def test_spawnl_noargs(self): program, __ = self.create_args() self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, program) self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, program, '') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnle') def test_spawnle_noargs(self): program, __ = self.create_args() self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, program, {}) self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, program, '', {}) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnv') def test_spawnv_noargs(self): program, __ = self.create_args() @@ -3704,6 +3308,7 @@ def test_spawnv_noargs(self): self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ('',)) self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ['']) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnve') def test_spawnve_noargs(self): program, __ = self.create_args() @@ -3760,22 +3365,28 @@ def _test_invalid_env(self, spawn): exitcode = spawn(os.P_WAIT, program, args, newenv) self.assertEqual(exitcode, 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnve') def test_spawnve_invalid_env(self): self._test_invalid_env(os.spawnve) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_os_func('spawnvpe') def test_spawnvpe_invalid_env(self): self._test_invalid_env(os.spawnvpe) -# The introduction of this TestCase caused at least two different errors on -# *nix buildbots. Temporarily skip this to let the buildbots move along. -@unittest.skip("Skip due to platform/environment differences on *NIX buildbots") @unittest.skipUnless(hasattr(os, 'getlogin'), "test needs os.getlogin") class LoginTests(unittest.TestCase): def test_getlogin(self): - user_name = os.getlogin() + try: + user_name = os.getlogin() + except OSError as exc: + # See https://man7.org/linux/man-pages/man3/getlogin.3.html#ERRORS. + if exc.errno in (errno.ENXIO, errno.ENOENT, errno.ENOTTY): + self.skipTest(str(exc)) + else: + raise self.assertNotEqual(len(user_name), 0) @@ -3939,6 +3550,11 @@ async def test_invalid_offset(self): await self.async_sendfile(self.sockno, self.fileno, -1, 4096) self.assertEqual(cm.exception.errno, errno.EINVAL) + async def test_invalid_count(self): + with self.assertRaises(ValueError, msg="count cannot be negative"): + await self.sendfile_wrapper(self.sockno, self.fileno, offset=0, + count=-1) + async def test_keywords(self): # Keyword arguments should be supported await self.async_sendfile(out_fd=self.sockno, in_fd=self.fileno, @@ -4291,13 +3907,8 @@ def test_eventfd_select(self): @unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # 1 ms accuracy is reliably achievable on every platform except Android - # emulators, where we allow 10 ms (gh-108277). - if sys.platform == "android" and platform.android_ver().is_emulator: - CLOCK_RES_PLACES = 2 - else: - CLOCK_RES_PLACES = 3 - + # gh-126112: Use 10 ms to tolerate slow buildbots + CLOCK_RES_PLACES = 2 # 10 ms CLOCK_RES = 10 ** -CLOCK_RES_PLACES CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) @@ -4652,6 +4263,7 @@ def test_oserror_filename(self): (self.filenames, os.listdir,), (self.filenames, os.rename, "dst"), (self.filenames, os.replace, "dst"), + (self.filenames, os.utime, None), ] if os_helper.can_chmod(): funcs.append((self.filenames, os.chmod, 0o777)) @@ -4692,6 +4304,19 @@ def test_oserror_filename(self): else: self.fail(f"No exception thrown by {func}") + def test_mkdir(self): + filename = os_helper.TESTFN + subdir = os.path.join(filename, 'subdir') + self.assertRaises(FileNotFoundError, os.mkdir, subdir) + + self.addCleanup(os_helper.unlink, filename) + create_file(filename) + self.assertRaises(FileExistsError, os.mkdir, filename) + + self.assertRaises((NotADirectoryError, FileNotFoundError), + os.mkdir, subdir) + + class CPUCountTests(unittest.TestCase): def check_cpu_count(self, cpus): if cpus is None: @@ -4885,6 +4510,7 @@ def test_posix_pty_functions(self): self.addCleanup(os.close, son_fd) self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") @support.requires_subprocess() def test_pipe_spawnl(self): @@ -5431,8 +5057,8 @@ def test_fsencode_fsdecode(self): def test_pathlike(self): self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) - self.assertTrue(issubclass(FakePath, os.PathLike)) - self.assertTrue(isinstance(FakePath('x'), os.PathLike)) + self.assertIsSubclass(FakePath, os.PathLike) + self.assertIsInstance(FakePath('x'), os.PathLike) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -5458,8 +5084,8 @@ def test_pathlike_subclasshook(self): # true on abstract implementation. class A(os.PathLike): pass - self.assertFalse(issubclass(FakePath, A)) - self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertNotIsSubclass(FakePath, A) + self.assertIsSubclass(FakePath, os.PathLike) def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) @@ -5469,7 +5095,7 @@ class A(os.PathLike): __slots__ = () def __fspath__(self): return '' - self.assertFalse(hasattr(A(), '__dict__')) + self.assertNotHasAttr(A(), '__dict__') def test_fspath_set_to_None(self): class Foo: diff --git a/Lib/test/test_posix.py b/Lib/test/test_os/test_posix.py similarity index 94% rename from Lib/test/test_posix.py rename to Lib/test/test_os/test_posix.py index 0817d0a87a3..37da293a441 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_os/test_posix.py @@ -668,22 +668,65 @@ def test_fstat(self): finally: fp.close() - def test_stat(self): - self.assertTrue(posix.stat(os_helper.TESTFN)) - self.assertTrue(posix.stat(os.fsencode(os_helper.TESTFN))) + def check_statlike_path(self, func): + self.assertTrue(func(os_helper.TESTFN)) + self.assertTrue(func(os.fsencode(os_helper.TESTFN))) + self.assertTrue(func(os_helper.FakePath(os_helper.TESTFN))) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, bytearray(os.fsencode(os_helper.TESTFN))) + func, bytearray(os.fsencode(os_helper.TESTFN))) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, None) + func, None) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, list(os_helper.TESTFN)) + func, list(os_helper.TESTFN)) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, list(os.fsencode(os_helper.TESTFN))) + func, list(os.fsencode(os_helper.TESTFN))) + + def test_stat(self): + self.check_statlike_path(posix.stat) + + @unittest.skipUnless(hasattr(posix, 'statx'), 'test needs posix.statx()') + def test_statx(self): + def func(path, **kwargs): + return posix.statx(path, posix.STATX_BASIC_STATS, **kwargs) + self.check_statlike_path(func) + + @unittest.skipUnless(hasattr(posix, 'statx'), 'test needs posix.statx()') + def test_statx_flags(self): + # glibc's fallback implementation of statx via the stat family fails + # with EINVAL on the (nonzero) sync flags. If you see this failure, + # update your kernel and/or seccomp syscall filter. + valid_flag_names = ('AT_NO_AUTOMOUNT', 'AT_STATX_SYNC_AS_STAT', + 'AT_STATX_FORCE_SYNC', 'AT_STATX_DONT_SYNC') + for flag_name in valid_flag_names: + flag = getattr(posix, flag_name) + with self.subTest(msg=flag_name, flags=flag): + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flag) + + # These flags are not exposed to Python because their functionality is + # implemented via kwargs instead. + kwarg_equivalent_flags = ( + (0x0100, 'AT_SYMLINK_NOFOLLOW', 'follow_symlinks'), + (0x0400, 'AT_SYMLINK_FOLLOW', 'follow_symlinks'), + (0x1000, 'AT_EMPTY_PATH', 'dir_fd'), + ) + for flag, flag_name, kwarg_name in kwarg_equivalent_flags: + with self.subTest(msg=flag_name, flags=flag): + with self.assertRaisesRegex(ValueError, kwarg_name): + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flag) + + with self.subTest(msg="AT_STATX_FORCE_SYNC | AT_STATX_DONT_SYNC"): + with self.assertRaises(OSError) as ctx: + flags = posix.AT_STATX_FORCE_SYNC | posix.AT_STATX_DONT_SYNC + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flags) + self.assertEqual(ctx.exception.errno, errno.EINVAL) @unittest.skipUnless(hasattr(posix, 'mkfifo'), "don't have mkfifo()") def test_mkfifo(self): @@ -757,12 +800,30 @@ def test_makedev(self): self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) - if sys.platform == 'linux': - NODEV = -1 + # The following tests are needed to test functions accepting or + # returning the special value NODEV (if it is defined). major(), minor() + # and makefile() are the only easily reproducible examples, but that + # behavior is platform specific -- on some platforms their code has + # a special case for NODEV, on others this is just an implementation + # artifact. + if (hasattr(posix, 'NODEV') and + sys.platform.startswith(('linux', 'macos', 'freebsd', 'dragonfly', + 'sunos'))): + NODEV = posix.NODEV self.assertEqual(posix.major(NODEV), NODEV) self.assertEqual(posix.minor(NODEV), NODEV) self.assertEqual(posix.makedev(NODEV, NODEV), NODEV) + def test_nodev(self): + # NODEV is not a part of Posix, but is defined on many systems. + if (not hasattr(posix, 'NODEV') + and (not sys.platform.startswith(('linux', 'macos', 'freebsd', + 'dragonfly', 'netbsd', 'openbsd', + 'sunos')) + or support.linked_to_musl())): + self.skipTest('not defined on this platform') + self.assertHasAttr(posix, 'NODEV') + def _test_all_chown_common(self, chown_func, first_param, stat_func): """Common code for chown, fchown and lchown tests.""" def check_stat(uid, gid): @@ -1107,7 +1168,7 @@ def test_lchmod_dir_symlink(self): def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs): st = os.stat(target_file) - self.assertTrue(hasattr(st, 'st_flags')) + self.assertHasAttr(st, 'st_flags') # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE. flags = st.st_flags | stat.UF_IMMUTABLE @@ -1143,7 +1204,7 @@ def test_lchflags_regular_file(self): def test_lchflags_symlink(self): testfn_st = os.stat(os_helper.TESTFN) - self.assertTrue(hasattr(testfn_st, 'st_flags')) + self.assertHasAttr(testfn_st, 'st_flags') self.addCleanup(os_helper.unlink, _DUMMY_SYMLINK) os.symlink(os_helper.TESTFN, _DUMMY_SYMLINK) @@ -1366,6 +1427,14 @@ def test_sched_param(self): self.assertNotEqual(newparam, param) self.assertEqual(newparam.sched_priority, 0) + @requires_sched + def test_bug_140634(self): + sched_priority = float('inf') # any new reference + param = posix.sched_param(sched_priority) + param.__reduce__() + del sched_priority, param # should not crash + support.gc_collect() # just to be sure + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: @@ -1611,33 +1680,47 @@ def test_chown_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): posix.chown(name, os.getuid(), os.getgid(), dir_fd=dir_fd) - @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") - def test_stat_dir_fd(self): + def check_statlike_dir_fd(self, func, prefix): with self.prepare() as (dir_fd, name, fullname): with open(fullname, 'w') as outfile: outfile.write("testline\n") self.addCleanup(posix.unlink, fullname) - s1 = posix.stat(fullname) - s2 = posix.stat(name, dir_fd=dir_fd) - self.assertEqual(s1, s2) - s2 = posix.stat(fullname, dir_fd=None) - self.assertEqual(s1, s2) + def get(result, attr): + return getattr(result, prefix + attr) + + s1 = func(fullname) + s2 = func(name, dir_fd=dir_fd) + self.assertEqual((get(s1, "dev"), get(s1, "ino")), + (get(s2, "dev"), get(s2, "ino"))) + s2 = func(fullname, dir_fd=None) + self.assertEqual((get(s1, "dev"), get(s1, "ino")), + (get(s2, "dev"), get(s2, "ino"))) self.assertRaisesRegex(TypeError, 'should be integer or None, not', - posix.stat, name, dir_fd=posix.getcwd()) + func, name, dir_fd=posix.getcwd()) self.assertRaisesRegex(TypeError, 'should be integer or None, not', - posix.stat, name, dir_fd=float(dir_fd)) + func, name, dir_fd=float(dir_fd)) self.assertRaises(OverflowError, - posix.stat, name, dir_fd=10**20) + func, name, dir_fd=10**20) for fd in False, True: with self.assertWarnsRegex(RuntimeWarning, 'bool is used as a file descriptor') as cm: with self.assertRaises(OSError): - posix.stat('nonexisting', dir_fd=fd) + func('nonexisting', dir_fd=fd) self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") + def test_stat_dir_fd(self): + self.check_statlike_dir_fd(posix.stat, prefix="st_") + + @unittest.skipUnless(hasattr(posix, 'statx'), "test needs os.statx()") + def test_statx_dir_fd(self): + def func(path, **kwargs): + return posix.statx(path, os.STATX_INO, **kwargs) + self.check_statlike_dir_fd(func, prefix="stx_") + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): @@ -2036,6 +2119,12 @@ def test_setscheduler_only_param(self): @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") + @unittest.skipIf(platform.libc_ver()[0] == 'glibc' and + os.sched_getscheduler(0) in [ + os.SCHED_BATCH, + os.SCHED_IDLE, + os.SCHED_DEADLINE], + "Skip test due to glibc posix_spawn policy") def test_setscheduler_with_policy(self): policy = os.sched_getscheduler(0) priority = os.sched_get_priority_min(policy) @@ -2218,12 +2307,12 @@ def _verify_available(self, name): def test_pwritev(self): self._verify_available("HAVE_PWRITEV") if self.mac_ver >= (10, 16): - self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") - self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + self.assertHasAttr(os, "pwritev") + self.assertHasAttr(os, "preadv") else: - self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") - self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + self.assertNotHasAttr(os, "pwritev") + self.assertNotHasAttr(os, "preadv") def test_stat(self): self._verify_available("HAVE_FSTATAT") diff --git a/Lib/test/test_os/test_windows.py b/Lib/test/test_os/test_windows.py new file mode 100644 index 00000000000..f1c6283f60d --- /dev/null +++ b/Lib/test/test_os/test_windows.py @@ -0,0 +1,605 @@ +import sys +import unittest + +if sys.platform != "win32": + raise unittest.SkipTest("Win32 specific tests") + +import _winapi +import fnmatch +import mmap +import os +import shutil +import signal +import stat +import subprocess +import textwrap +import time +import uuid +from test import support +from test.support import import_helper +from test.support import os_helper +from .utils import create_file + + +class Win32KillTests(unittest.TestCase): + def _kill(self, sig): + # Start sys.executable as a subprocess and communicate from the + # subprocess to the parent that the interpreter is ready. When it + # becomes ready, send *sig* via os.kill to the subprocess and check + # that the return code is equal to *sig*. + import ctypes + from ctypes import wintypes + import msvcrt + + # Since we can't access the contents of the process' stdout until the + # process has exited, use PeekNamedPipe to see what's inside stdout + # without waiting. This is done so we can tell that the interpreter + # is started and running at a point where it could handle a signal. + PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe + PeekNamedPipe.restype = wintypes.BOOL + PeekNamedPipe.argtypes = (wintypes.HANDLE, # Pipe handle + ctypes.POINTER(ctypes.c_char), # stdout buf + wintypes.DWORD, # Buffer size + ctypes.POINTER(wintypes.DWORD), # bytes read + ctypes.POINTER(wintypes.DWORD), # bytes avail + ctypes.POINTER(wintypes.DWORD)) # bytes left + msg = "running" + proc = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write('{}');" + "sys.stdout.flush();" + "input()".format(msg)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + self.addCleanup(proc.stdout.close) + self.addCleanup(proc.stderr.close) + self.addCleanup(proc.stdin.close) + + count, max = 0, 100 + while count < max and proc.poll() is None: + # Create a string buffer to store the result of stdout from the pipe + buf = ctypes.create_string_buffer(len(msg)) + # Obtain the text currently in proc.stdout + # Bytes read/avail/left are left as NULL and unused + rslt = PeekNamedPipe(msvcrt.get_osfhandle(proc.stdout.fileno()), + buf, ctypes.sizeof(buf), None, None, None) + self.assertNotEqual(rslt, 0, "PeekNamedPipe failed") + if buf.value: + self.assertEqual(msg, buf.value.decode()) + break + time.sleep(0.1) + count += 1 + else: + self.fail("Did not receive communication from the subprocess") + + os.kill(proc.pid, sig) + self.assertEqual(proc.wait(), sig) + + def test_kill_sigterm(self): + # SIGTERM doesn't mean anything special, but make sure it works + self._kill(signal.SIGTERM) + + def test_kill_int(self): + # os.kill on Windows can take an int which gets set as the exit code + self._kill(100) + + @unittest.skipIf(mmap is None, "requires mmap") + def _kill_with_event(self, event, name): + tagname = "test_os_%s" % uuid.uuid1() + m = mmap.mmap(-1, 1, tagname) + m[0] = 0 + + # Run a script which has console control handling enabled. + script = os.path.join(os.path.dirname(__file__), + "win_console_handler.py") + cmd = [sys.executable, script, tagname] + proc = subprocess.Popen(cmd, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + + with proc: + # Let the interpreter startup before we send signals. See #3137. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if proc.poll() is None: + break + else: + # Forcefully kill the process if we weren't able to signal it. + proc.kill() + self.fail("Subprocess didn't finish initialization") + + os.kill(proc.pid, event) + + try: + # proc.send_signal(event) could also be done here. + # Allow time for the signal to be passed and the process to exit. + proc.wait(timeout=support.SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + # Forcefully kill the process if we weren't able to signal it. + proc.kill() + self.fail("subprocess did not stop on {}".format(name)) + + @unittest.skip("subprocesses aren't inheriting Ctrl+C property") + @support.requires_subprocess() + def test_CTRL_C_EVENT(self): + from ctypes import wintypes + import ctypes + + # Make a NULL value by creating a pointer with no argument. + NULL = ctypes.POINTER(ctypes.c_int)() + SetConsoleCtrlHandler = ctypes.windll.kernel32.SetConsoleCtrlHandler + SetConsoleCtrlHandler.argtypes = (ctypes.POINTER(ctypes.c_int), + wintypes.BOOL) + SetConsoleCtrlHandler.restype = wintypes.BOOL + + # Calling this with NULL and FALSE causes the calling process to + # handle Ctrl+C, rather than ignore it. This property is inherited + # by subprocesses. + SetConsoleCtrlHandler(NULL, 0) + + self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") + + @support.requires_subprocess() + def test_CTRL_BREAK_EVENT(self): + self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") + + +class Win32ListdirTests(unittest.TestCase): + """Test listdir on Windows.""" + + def setUp(self): + self.created_paths = [] + for i in range(2): + dir_name = 'SUB%d' % i + dir_path = os.path.join(os_helper.TESTFN, dir_name) + file_name = 'FILE%d' % i + file_path = os.path.join(os_helper.TESTFN, file_name) + os.makedirs(dir_path) + with open(file_path, 'w', encoding='utf-8') as f: + f.write("I'm %s and proud of it. Blame test_os.\n" % file_path) + self.created_paths.extend([dir_name, file_name]) + self.created_paths.sort() + + def tearDown(self): + shutil.rmtree(os_helper.TESTFN) + + def test_listdir_no_extended_path(self): + """Test when the path is not an "extended" path.""" + # unicode + self.assertEqual( + sorted(os.listdir(os_helper.TESTFN)), + self.created_paths) + + # bytes + self.assertEqual( + sorted(os.listdir(os.fsencode(os_helper.TESTFN))), + [os.fsencode(path) for path in self.created_paths]) + + def test_listdir_extended_path(self): + """Test when the path starts with '\\\\?\\'.""" + # See: http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath + # unicode + path = '\\\\?\\' + os.path.abspath(os_helper.TESTFN) + self.assertEqual( + sorted(os.listdir(path)), + self.created_paths) + + # bytes + path = b'\\\\?\\' + os.fsencode(os.path.abspath(os_helper.TESTFN)) + self.assertEqual( + sorted(os.listdir(path)), + [os.fsencode(path) for path in self.created_paths]) + + +@unittest.skipUnless(os.name == "nt", "NT specific tests") +class Win32ListdriveTests(unittest.TestCase): + """Test listdrive, listmounts and listvolume on Windows.""" + + def setUp(self): + # Get drives and volumes from fsutil + out = subprocess.check_output( + ["fsutil.exe", "volume", "list"], + cwd=os.path.join(os.getenv("SystemRoot", "\\Windows"), "System32"), + encoding="mbcs", + errors="ignore", + ) + lines = out.splitlines() + self.known_volumes = {l for l in lines if l.startswith('\\\\?\\')} + self.known_drives = {l for l in lines if l[1:] == ':\\'} + self.known_mounts = {l for l in lines if l[1:3] == ':\\'} + + def test_listdrives(self): + drives = os.listdrives() + self.assertIsInstance(drives, list) + self.assertSetEqual( + self.known_drives, + self.known_drives & set(drives), + ) + + def test_listvolumes(self): + volumes = os.listvolumes() + self.assertIsInstance(volumes, list) + self.assertSetEqual( + self.known_volumes, + self.known_volumes & set(volumes), + ) + + def test_listmounts(self): + for volume in os.listvolumes(): + try: + mounts = os.listmounts(volume) + except OSError as ex: + if support.verbose: + print("Skipping", volume, "because of", ex) + else: + self.assertIsInstance(mounts, list) + self.assertSetEqual( + set(mounts), + self.known_mounts & set(mounts), + ) + + +@os_helper.skip_unless_symlink +class Win32SymlinkTests(unittest.TestCase): + filelink = 'filelinktest' + filelink_target = os.path.abspath(__file__) + dirlink = 'dirlinktest' + dirlink_target = os.path.dirname(filelink_target) + missing_link = 'missing link' + + def setUp(self): + assert os.path.exists(self.dirlink_target) + assert os.path.exists(self.filelink_target) + assert not os.path.exists(self.dirlink) + assert not os.path.exists(self.filelink) + assert not os.path.exists(self.missing_link) + + def tearDown(self): + if os.path.exists(self.filelink): + os.remove(self.filelink) + if os.path.exists(self.dirlink): + os.rmdir(self.dirlink) + if os.path.lexists(self.missing_link): + os.remove(self.missing_link) + + def test_directory_link(self): + os.symlink(self.dirlink_target, self.dirlink) + self.assertTrue(os.path.exists(self.dirlink)) + self.assertTrue(os.path.isdir(self.dirlink)) + self.assertTrue(os.path.islink(self.dirlink)) + self.check_stat(self.dirlink, self.dirlink_target) + + def test_file_link(self): + os.symlink(self.filelink_target, self.filelink) + self.assertTrue(os.path.exists(self.filelink)) + self.assertTrue(os.path.isfile(self.filelink)) + self.assertTrue(os.path.islink(self.filelink)) + self.check_stat(self.filelink, self.filelink_target) + + def _create_missing_dir_link(self): + 'Create a "directory" link to a non-existent target' + linkname = self.missing_link + if os.path.lexists(linkname): + os.remove(linkname) + target = r'c:\\target does not exist.29r3c740' + assert not os.path.exists(target) + target_is_dir = True + os.symlink(target, linkname, target_is_dir) + + def test_remove_directory_link_to_missing_target(self): + self._create_missing_dir_link() + # For compatibility with Unix, os.remove will check the + # directory status and call RemoveDirectory if the symlink + # was created with target_is_dir==True. + os.remove(self.missing_link) + + def test_isdir_on_directory_link_to_missing_target(self): + self._create_missing_dir_link() + self.assertFalse(os.path.isdir(self.missing_link)) + + def test_rmdir_on_directory_link_to_missing_target(self): + self._create_missing_dir_link() + os.rmdir(self.missing_link) + + def check_stat(self, link, target): + self.assertEqual(os.stat(link), os.stat(target)) + self.assertNotEqual(os.lstat(link), os.stat(link)) + + bytes_link = os.fsencode(link) + self.assertEqual(os.stat(bytes_link), os.stat(target)) + self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) + + def test_12084(self): + level1 = os.path.abspath(os_helper.TESTFN) + level2 = os.path.join(level1, "level2") + level3 = os.path.join(level2, "level3") + self.addCleanup(os_helper.rmtree, level1) + + os.mkdir(level1) + os.mkdir(level2) + os.mkdir(level3) + + file1 = os.path.abspath(os.path.join(level1, "file1")) + create_file(file1) + + orig_dir = os.getcwd() + try: + os.chdir(level2) + link = os.path.join(level2, "link") + os.symlink(os.path.relpath(file1), "link") + self.assertIn("link", os.listdir(os.getcwd())) + + # Check os.stat calls from the same dir as the link + self.assertEqual(os.stat(file1), os.stat("link")) + + # Check os.stat calls from a dir below the link + os.chdir(level1) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) + + # Check os.stat calls from a dir above the link + os.chdir(level3) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) + finally: + os.chdir(orig_dir) + + @unittest.skipUnless(os.path.lexists(r'C:\Users\All Users') + and os.path.exists(r'C:\ProgramData'), + 'Test directories not found') + def test_29248(self): + # os.symlink() calls CreateSymbolicLink, which creates + # the reparse data buffer with the print name stored + # first, so the offset is always 0. CreateSymbolicLink + # stores the "PrintName" DOS path (e.g. "C:\") first, + # with an offset of 0, followed by the "SubstituteName" + # NT path (e.g. "\??\C:\"). The "All Users" link, on + # the other hand, seems to have been created manually + # with an inverted order. + target = os.readlink(r'C:\Users\All Users') + self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) + + def test_buffer_overflow(self): + # Older versions would have a buffer overflow when detecting + # whether a link source was a directory. This test ensures we + # no longer crash, but does not otherwise validate the behavior + segment = 'X' * 27 + path = os.path.join(*[segment] * 10) + test_cases = [ + # overflow with absolute src + ('\\' + path, segment), + # overflow dest with relative src + (segment, path), + # overflow when joining src + (path[:180], path[:180]), + ] + for src, dest in test_cases: + try: + os.symlink(src, dest) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + # Also test with bytes, since that is a separate code path. + try: + os.symlink(os.fsencode(src), os.fsencode(dest)) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + + def test_appexeclink(self): + root = os.path.expandvars(r'%LOCALAPPDATA%\Microsoft\WindowsApps') + if not os.path.isdir(root): + self.skipTest("test requires a WindowsApps directory") + + aliases = [os.path.join(root, a) + for a in fnmatch.filter(os.listdir(root), '*.exe')] + + for alias in aliases: + if support.verbose: + print() + print("Testing with", alias) + st = os.lstat(alias) + self.assertEqual(st, os.stat(alias)) + self.assertFalse(stat.S_ISLNK(st.st_mode)) + self.assertEqual(st.st_reparse_tag, stat.IO_REPARSE_TAG_APPEXECLINK) + self.assertTrue(os.path.isfile(alias)) + # testing the first one we see is sufficient + break + else: + self.skipTest("test requires an app execution alias") + + +class Win32JunctionTests(unittest.TestCase): + junction = 'junctiontest' + junction_target = os.path.dirname(os.path.abspath(__file__)) + + def setUp(self): + assert os.path.exists(self.junction_target) + assert not os.path.lexists(self.junction) + + def tearDown(self): + if os.path.lexists(self.junction): + os.unlink(self.junction) + + def test_create_junction(self): + _winapi.CreateJunction(self.junction_target, self.junction) + self.assertTrue(os.path.lexists(self.junction)) + self.assertTrue(os.path.exists(self.junction)) + self.assertTrue(os.path.isdir(self.junction)) + self.assertNotEqual(os.stat(self.junction), os.lstat(self.junction)) + self.assertEqual(os.stat(self.junction), os.stat(self.junction_target)) + + # bpo-37834: Junctions are not recognized as links. + self.assertFalse(os.path.islink(self.junction)) + self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), + os.path.normcase(os.readlink(self.junction))) + + def test_unlink_removes_junction(self): + _winapi.CreateJunction(self.junction_target, self.junction) + self.assertTrue(os.path.exists(self.junction)) + self.assertTrue(os.path.lexists(self.junction)) + + os.unlink(self.junction) + self.assertFalse(os.path.exists(self.junction)) + + +class Win32NtTests(unittest.TestCase): + def test_getfinalpathname_handles(self): + nt = import_helper.import_module('nt') + ctypes = import_helper.import_module('ctypes') + # Ruff false positive -- it thinks we're redefining `ctypes` here + import ctypes.wintypes # noqa: F811 + + kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) + kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE + + kernel.GetProcessHandleCount.restype = ctypes.wintypes.BOOL + kernel.GetProcessHandleCount.argtypes = (ctypes.wintypes.HANDLE, + ctypes.wintypes.LPDWORD) + + # This is a pseudo-handle that doesn't need to be closed + hproc = kernel.GetCurrentProcess() + + handle_count = ctypes.wintypes.DWORD() + ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) + self.assertEqual(1, ok) + + before_count = handle_count.value + + # The first two test the error path, __file__ tests the success path + filenames = [ + r'\\?\C:', + r'\\?\NUL', + r'\\?\CONIN', + __file__, + ] + + for _ in range(10): + for name in filenames: + try: + nt._getfinalpathname(name) + except Exception: + # Failure is expected + pass + try: + os.stat(name) + except Exception: + pass + + ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) + self.assertEqual(1, ok) + + handle_delta = handle_count.value - before_count + + self.assertEqual(0, handle_delta) + + @support.requires_subprocess() + def test_stat_unlink_race(self): + # bpo-46785: the implementation of os.stat() falls back to reading + # the parent directory if CreateFileW() fails with a permission + # error. If reading the parent directory fails because the file or + # directory are subsequently unlinked, or because the volume or + # share are no longer available, then the original permission error + # should not be restored. + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + deadline = time.time() + 5 + command = textwrap.dedent("""\ + import os + import sys + import time + + filename = sys.argv[1] + deadline = float(sys.argv[2]) + + while time.time() < deadline: + try: + with open(filename, "w") as f: + pass + except OSError: + pass + try: + os.remove(filename) + except OSError: + pass + """) + + with subprocess.Popen([sys.executable, '-c', command, filename, str(deadline)]) as proc: + while time.time() < deadline: + try: + os.stat(filename) + except FileNotFoundError as e: + assert e.winerror == 2 # ERROR_FILE_NOT_FOUND + try: + proc.wait(1) + except subprocess.TimeoutExpired: + proc.terminate() + + @support.requires_subprocess() + def test_stat_inaccessible_file(self): + filename = os_helper.TESTFN + ICACLS = os.path.expandvars(r"%SystemRoot%\System32\icacls.exe") + + with open(filename, "wb") as f: + f.write(b'Test data') + + stat1 = os.stat(filename) + + try: + # Remove all permissions from the file + subprocess.check_output([ICACLS, filename, "/inheritance:r"], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as ex: + if support.verbose: + print(ICACLS, filename, "/inheritance:r", "failed.") + print(ex.stdout.decode("oem", "replace").rstrip()) + try: + os.unlink(filename) + except OSError: + pass + self.skipTest("Unable to create inaccessible file") + + def cleanup(): + # Give delete permission to the owner (us) + subprocess.check_output([ICACLS, filename, "/grant", "*WD:(D)"], + stderr=subprocess.STDOUT) + os.unlink(filename) + + self.addCleanup(cleanup) + + if support.verbose: + print("File:", filename) + print("stat with access:", stat1) + + # First test - we shouldn't raise here, because we still have access to + # the directory and can extract enough information from its metadata. + stat2 = os.stat(filename) + + if support.verbose: + print(" without access:", stat2) + + # We may not get st_dev/st_ino, so ensure those are 0 or match + self.assertIn(stat2.st_dev, (0, stat1.st_dev)) + self.assertIn(stat2.st_ino, (0, stat1.st_ino)) + + # st_mode and st_size should match (for a normal file, at least) + self.assertEqual(stat1.st_mode, stat2.st_mode) + self.assertEqual(stat1.st_size, stat2.st_size) + + # st_ctime and st_mtime should be the same + self.assertEqual(stat1.st_ctime, stat2.st_ctime) + self.assertEqual(stat1.st_mtime, stat2.st_mtime) + + # st_atime should be the same or later + self.assertGreaterEqual(stat1.st_atime, stat2.st_atime) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_os/utils.py b/Lib/test/test_os/utils.py new file mode 100644 index 00000000000..e0c39598c31 --- /dev/null +++ b/Lib/test/test_os/utils.py @@ -0,0 +1,3 @@ +def create_file(filename, content=b'content'): + with open(filename, "xb", 0) as fp: + fp.write(content) diff --git a/Lib/test/test_pathlib/support/lexical_path.py b/Lib/test/test_pathlib/support/lexical_path.py index f29a521af9b..fd7fbf283a6 100644 --- a/Lib/test/test_pathlib/support/lexical_path.py +++ b/Lib/test/test_pathlib/support/lexical_path.py @@ -9,9 +9,10 @@ from . import is_pypi if is_pypi: - from pathlib_abc import _JoinablePath + from pathlib_abc import vfspath, _JoinablePath else: from pathlib.types import _JoinablePath + from pathlib._os import vfspath class LexicalPath(_JoinablePath): @@ -22,20 +23,20 @@ def __init__(self, *pathsegments): self._segments = pathsegments def __hash__(self): - return hash(str(self)) + return hash(vfspath(self)) def __eq__(self, other): if not isinstance(other, LexicalPath): return NotImplemented - return str(self) == str(other) + return vfspath(self) == vfspath(other) - def __str__(self): + def __vfspath__(self): if not self._segments: return '' return self.parser.join(*self._segments) def __repr__(self): - return f'{type(self).__name__}({str(self)!r})' + return f'{type(self).__name__}({vfspath(self)!r})' def with_segments(self, *pathsegments): return type(self)(*pathsegments) diff --git a/Lib/test/test_pathlib/support/local_path.py b/Lib/test/test_pathlib/support/local_path.py index d481fd45ead..ddfd6fd4195 100644 --- a/Lib/test/test_pathlib/support/local_path.py +++ b/Lib/test/test_pathlib/support/local_path.py @@ -97,7 +97,7 @@ class LocalPathInfo(PathInfo): __slots__ = ('_path', '_exists', '_is_dir', '_is_file', '_is_symlink') def __init__(self, path): - self._path = str(path) + self._path = os.fspath(path) self._exists = None self._is_dir = None self._is_file = None @@ -139,15 +139,13 @@ class ReadableLocalPath(_ReadablePath, LexicalPath): Simple implementation of a ReadablePath class for local filesystem paths. """ __slots__ = ('info',) + __fspath__ = LexicalPath.__vfspath__ def __init__(self, *pathsegments): super().__init__(*pathsegments) self.info = LocalPathInfo(self) - def __fspath__(self): - return str(self) - - def __open_rb__(self, buffering=-1): + def __open_reader__(self): return open(self, 'rb') def iterdir(self): @@ -163,12 +161,10 @@ class WritableLocalPath(_WritablePath, LexicalPath): """ __slots__ = () + __fspath__ = LexicalPath.__vfspath__ - def __fspath__(self): - return str(self) - - def __open_wb__(self, buffering=-1): - return open(self, 'wb') + def __open_writer__(self, mode): + return open(self, f'{mode}b') def mkdir(self, mode=0o777): os.mkdir(self, mode) diff --git a/Lib/test/test_pathlib/support/zip_path.py b/Lib/test/test_pathlib/support/zip_path.py index 2905260c9df..90b939b6a59 100644 --- a/Lib/test/test_pathlib/support/zip_path.py +++ b/Lib/test/test_pathlib/support/zip_path.py @@ -16,9 +16,10 @@ from . import is_pypi if is_pypi: - from pathlib_abc import PathInfo, _ReadablePath, _WritablePath + from pathlib_abc import vfspath, PathInfo, _ReadablePath, _WritablePath else: from pathlib.types import PathInfo, _ReadablePath, _WritablePath + from pathlib._os import vfspath class ZipPathGround: @@ -34,16 +35,16 @@ def teardown(self, root): root.zip_file.close() def create_file(self, path, data=b''): - path.zip_file.writestr(str(path), data) + path.zip_file.writestr(vfspath(path), data) def create_dir(self, path): - zip_info = zipfile.ZipInfo(str(path) + '/') + zip_info = zipfile.ZipInfo(vfspath(path) + '/') zip_info.external_attr |= stat.S_IFDIR << 16 zip_info.external_attr |= stat.FILE_ATTRIBUTE_DIRECTORY path.zip_file.writestr(zip_info, '') def create_symlink(self, path, target): - zip_info = zipfile.ZipInfo(str(path)) + zip_info = zipfile.ZipInfo(vfspath(path)) zip_info.external_attr = stat.S_IFLNK << 16 path.zip_file.writestr(zip_info, target.encode()) @@ -62,28 +63,28 @@ def create_hierarchy(self, p): self.create_symlink(p.joinpath('brokenLinkLoop'), 'brokenLinkLoop') def readtext(self, p): - with p.zip_file.open(str(p), 'r') as f: + with p.zip_file.open(vfspath(p), 'r') as f: f = io.TextIOWrapper(f, encoding='utf-8') return f.read() def readbytes(self, p): - with p.zip_file.open(str(p), 'r') as f: + with p.zip_file.open(vfspath(p), 'r') as f: return f.read() readlink = readtext def isdir(self, p): - path_str = str(p) + "/" + path_str = vfspath(p) + "/" return path_str in p.zip_file.NameToInfo def isfile(self, p): - info = p.zip_file.NameToInfo.get(str(p)) + info = p.zip_file.NameToInfo.get(vfspath(p)) if info is None: return False return not stat.S_ISLNK(info.external_attr >> 16) def islink(self, p): - info = p.zip_file.NameToInfo.get(str(p)) + info = p.zip_file.NameToInfo.get(vfspath(p)) if info is None: return False return stat.S_ISLNK(info.external_attr >> 16) @@ -240,20 +241,20 @@ def __init__(self, *pathsegments, zip_file): zip_file.filelist = ZipFileList(zip_file) def __hash__(self): - return hash((str(self), self.zip_file)) + return hash((vfspath(self), self.zip_file)) def __eq__(self, other): if not isinstance(other, ReadableZipPath): return NotImplemented - return str(self) == str(other) and self.zip_file is other.zip_file + return vfspath(self) == vfspath(other) and self.zip_file is other.zip_file - def __str__(self): + def __vfspath__(self): if not self._segments: return '' return self.parser.join(*self._segments) def __repr__(self): - return f'{type(self).__name__}({str(self)!r}, zip_file={self.zip_file!r})' + return f'{type(self).__name__}({vfspath(self)!r}, zip_file={self.zip_file!r})' def with_segments(self, *pathsegments): return type(self)(*pathsegments, zip_file=self.zip_file) @@ -261,15 +262,15 @@ def with_segments(self, *pathsegments): @property def info(self): tree = self.zip_file.filelist.tree - return tree.resolve(str(self), follow_symlinks=False) + return tree.resolve(vfspath(self), follow_symlinks=False) - def __open_rb__(self, buffering=-1): + def __open_reader__(self): info = self.info.resolve() if not info.exists(): raise FileNotFoundError(errno.ENOENT, "File not found", self) elif info.is_dir(): raise IsADirectoryError(errno.EISDIR, "Is a directory", self) - return self.zip_file.open(info.zip_info, 'r') + return self.zip_file.open(info.zip_info) def iterdir(self): info = self.info.resolve() @@ -301,36 +302,36 @@ def __init__(self, *pathsegments, zip_file): self.zip_file = zip_file def __hash__(self): - return hash((str(self), self.zip_file)) + return hash((vfspath(self), self.zip_file)) def __eq__(self, other): if not isinstance(other, WritableZipPath): return NotImplemented - return str(self) == str(other) and self.zip_file is other.zip_file + return vfspath(self) == vfspath(other) and self.zip_file is other.zip_file - def __str__(self): + def __vfspath__(self): if not self._segments: return '' return self.parser.join(*self._segments) def __repr__(self): - return f'{type(self).__name__}({str(self)!r}, zip_file={self.zip_file!r})' + return f'{type(self).__name__}({vfspath(self)!r}, zip_file={self.zip_file!r})' def with_segments(self, *pathsegments): return type(self)(*pathsegments, zip_file=self.zip_file) - def __open_wb__(self, buffering=-1): - return self.zip_file.open(str(self), 'w') + def __open_writer__(self, mode): + return self.zip_file.open(vfspath(self), mode) def mkdir(self, mode=0o777): - zinfo = zipfile.ZipInfo(str(self) + '/') + zinfo = zipfile.ZipInfo(vfspath(self) + '/') zinfo.external_attr |= stat.S_IFDIR << 16 zinfo.external_attr |= stat.FILE_ATTRIBUTE_DIRECTORY self.zip_file.writestr(zinfo, '') def symlink_to(self, target, target_is_directory=False): - zinfo = zipfile.ZipInfo(str(self)) + zinfo = zipfile.ZipInfo(vfspath(self)) zinfo.external_attr = stat.S_IFLNK << 16 if target_is_directory: zinfo.external_attr |= 0x10 - self.zip_file.writestr(zinfo, str(target)) + self.zip_file.writestr(zinfo, target) diff --git a/Lib/test/test_pathlib/test_join.py b/Lib/test/test_pathlib/test_join.py index f1a24204b4c..4630210e492 100644 --- a/Lib/test/test_pathlib/test_join.py +++ b/Lib/test/test_pathlib/test_join.py @@ -3,6 +3,8 @@ """ import unittest +import threading +from test.support import threading_helper from .support import is_pypi from .support.lexical_path import LexicalPath @@ -158,6 +160,26 @@ def test_parts(self): parts = p.parts self.assertEqual(parts, (sep, 'a', 'b')) + @threading_helper.requires_working_threading() + def test_parts_multithreaded(self): + P = self.cls + + NUM_THREADS = 10 + NUM_ITERS = 10 + + for _ in range(NUM_ITERS): + b = threading.Barrier(NUM_THREADS) + path = P('a') / 'b' / 'c' / 'd' / 'e' + expected = ('a', 'b', 'c', 'd', 'e') + + def check_parts(): + b.wait() + self.assertEqual(path.parts, expected) + + threads = [threading.Thread(target=check_parts) for _ in range(NUM_THREADS)] + with threading_helper.start_threads(threads): + pass + def test_parent(self): # Relative P = self.cls @@ -354,6 +376,61 @@ def test_with_suffix(self): self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') self.assertRaises(TypeError, P('a/b').with_suffix, None) + def test_relative_to(self): + P = self.cls + p = P('a/b') + self.assertEqual(p.relative_to(P('')), P('a', 'b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a', 'b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('..', 'b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('..', 'a', 'b')) + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P('../a')) + self.assertRaises(ValueError, p.relative_to, P('a/..')) + self.assertRaises(ValueError, p.relative_to, P('/a/..')) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('../a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a/..'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a/..'), walk_up=True) + class Q(self.cls): + __eq__ = object.__eq__ + __hash__ = object.__hash__ + q = Q('a/b') + self.assertTrue(q.relative_to(q)) + self.assertRaises(ValueError, q.relative_to, Q('')) + self.assertRaises(ValueError, q.relative_to, Q('a')) + self.assertRaises(ValueError, q.relative_to, Q('a'), walk_up=True) + self.assertRaises(ValueError, q.relative_to, Q('a/b')) + self.assertRaises(ValueError, q.relative_to, Q('c')) + + def test_is_relative_to(self): + P = self.cls + p = P('a/b') + self.assertTrue(p.is_relative_to(P(''))) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + class Q(self.cls): + __eq__ = object.__eq__ + __hash__ = object.__hash__ + q = Q('a/b') + self.assertTrue(q.is_relative_to(q)) + self.assertFalse(q.is_relative_to(Q(''))) + self.assertFalse(q.is_relative_to(Q('a'))) + self.assertFalse(q.is_relative_to(Q('a/b'))) + self.assertFalse(q.is_relative_to(Q('c'))) + class LexicalPathJoinTest(JoinTestBase, unittest.TestCase): cls = LexicalPath diff --git a/Lib/test/test_pathlib/test_join_windows.py b/Lib/test/test_pathlib/test_join_windows.py index 2cc634f25ef..f30c80605f7 100644 --- a/Lib/test/test_pathlib/test_join_windows.py +++ b/Lib/test/test_pathlib/test_join_windows.py @@ -8,6 +8,11 @@ from .support import is_pypi from .support.lexical_path import LexicalWindowsPath +if is_pypi: + from pathlib_abc import vfspath +else: + from pathlib._os import vfspath + class JoinTestBase: def test_join(self): @@ -70,17 +75,17 @@ def test_div(self): self.assertEqual(p / './dd:s', P(r'C:/a/b\./dd:s')) self.assertEqual(p / 'E:d:s', P('E:d:s')) - def test_str(self): + def test_vfspath(self): p = self.cls(r'a\b\c') - self.assertEqual(str(p), 'a\\b\\c') + self.assertEqual(vfspath(p), 'a\\b\\c') p = self.cls(r'c:\a\b\c') - self.assertEqual(str(p), 'c:\\a\\b\\c') + self.assertEqual(vfspath(p), 'c:\\a\\b\\c') p = self.cls('\\\\a\\b\\') - self.assertEqual(str(p), '\\\\a\\b\\') + self.assertEqual(vfspath(p), '\\\\a\\b\\') p = self.cls(r'\\a\b\c') - self.assertEqual(str(p), '\\\\a\\b\\c') + self.assertEqual(vfspath(p), '\\\\a\\b\\c') p = self.cls(r'\\a\b\c\d') - self.assertEqual(str(p), '\\\\a\\b\\c\\d') + self.assertEqual(vfspath(p), '\\\\a\\b\\c\\d') def test_parts(self): P = self.cls diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 8a313cc4292..ef9ea0d11d0 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -17,10 +17,10 @@ from test.support import import_helper from test.support import cpython_only -from test.support import is_emscripten, is_wasi +from test.support import is_emscripten, is_wasi, is_wasm32 from test.support import infinite_recursion from test.support import os_helper -from test.support.os_helper import TESTFN, FakePath +from test.support.os_helper import TESTFN, FS_NONASCII, FakePath try: import fcntl except ImportError: @@ -77,8 +77,8 @@ def needs_symlinks(fn): class UnsupportedOperationTest(unittest.TestCase): def test_is_notimplemented(self): - self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + self.assertIsSubclass(pathlib.UnsupportedOperation, NotImplementedError) + self.assertIsInstance(pathlib.UnsupportedOperation(), NotImplementedError) class LazyImportTest(unittest.TestCase): @@ -293,6 +293,12 @@ def test_pickling_common(self): self.assertEqual(hash(pp), hash(p)) self.assertEqual(str(pp), str(p)) + def test_unpicking_3_13(self): + data = (b"\x80\x04\x95'\x00\x00\x00\x00\x00\x00\x00\x8c\x0e" + b"pathlib._local\x94\x8c\rPurePosixPath\x94\x93\x94)R\x94.") + p = pickle.loads(data) + self.assertIsInstance(p, pathlib.PurePosixPath) + def test_repr_common(self): for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): with self.subTest(pathstr=pathstr): @@ -300,8 +306,8 @@ def test_repr_common(self): clsname = p.__class__.__name__ r = repr(p) # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) + self.assertStartsWith(r, clsname + '(') + self.assertEndsWith(r, ')') inner = r[len(clsname) + 1 : -1] self.assertEqual(eval(inner), p.as_posix()) @@ -539,12 +545,6 @@ def test_with_stem_empty(self): self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - def test_is_reserved_deprecated(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.is_reserved() - def test_full_match_case_sensitive(self): P = self.cls self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) @@ -770,12 +770,16 @@ def test_as_uri_windows(self): self.assertEqual(self.make_uri(P('c:/')), 'file:///c:/') self.assertEqual(self.make_uri(P('c:/a/b.c')), 'file:///c:/a/b.c') self.assertEqual(self.make_uri(P('c:/a/b%#c')), 'file:///c:/a/b%25%23c') - self.assertEqual(self.make_uri(P('c:/a/b\xe9')), 'file:///c:/a/b%C3%A9') self.assertEqual(self.make_uri(P('//some/share/')), 'file://some/share/') self.assertEqual(self.make_uri(P('//some/share/a/b.c')), 'file://some/share/a/b.c') - self.assertEqual(self.make_uri(P('//some/share/a/b%#c\xe9')), - 'file://some/share/a/b%25%23c%C3%A9') + + from urllib.parse import quote_from_bytes + QUOTED_FS_NONASCII = quote_from_bytes(os.fsencode(FS_NONASCII)) + self.assertEqual(self.make_uri(P('c:/a/b' + FS_NONASCII)), + 'file:///c:/a/b' + QUOTED_FS_NONASCII) + self.assertEqual(self.make_uri(P('//some/share/a/b%#c' + FS_NONASCII)), + 'file://some/share/a/b%25%23c' + QUOTED_FS_NONASCII) @needs_windows def test_ordering_windows(self): @@ -2950,7 +2954,13 @@ def test_glob_dotdot(self): else: # ".." segments are normalized first on Windows, so this path is stat()able. self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + if sys.platform == "emscripten": + # Emscripten will return ELOOP if there are 49 or more ..'s. + # Can remove when https://github.com/emscripten-core/emscripten/pull/24591 is merged. + NDOTDOTS = 48 + else: + NDOTDOTS = 50 + self.assertEqual(set(p.glob("/".join([".."] * NDOTDOTS))), { P(self.base, *[".."] * NDOTDOTS)}) def test_glob_inaccessible(self): P = self.cls @@ -3154,7 +3164,7 @@ def test_absolute_posix(self): self.assertEqual(str(P('//a/b').absolute()), '//a/b') @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) @needs_posix @@ -3185,7 +3195,7 @@ def test_resolve_root(self): os.chdir(current_directory) @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) @needs_posix diff --git a/Lib/test/test_pathlib/test_read.py b/Lib/test/test_pathlib/test_read.py index 482203c290a..16fb555b2ae 100644 --- a/Lib/test/test_pathlib/test_read.py +++ b/Lib/test/test_pathlib/test_read.py @@ -13,10 +13,10 @@ if is_pypi: from pathlib_abc import PathInfo, _ReadablePath - from pathlib_abc._os import magic_open + from pathlib_abc._os import vfsopen else: from pathlib.types import PathInfo, _ReadablePath - from pathlib._os import magic_open + from pathlib._os import vfsopen class ReadTestBase: @@ -32,10 +32,16 @@ def test_is_readable(self): def test_open_r(self): p = self.root / 'fileA' - with magic_open(p, 'r', encoding='utf-8') as f: + with vfsopen(p, 'r', encoding='utf-8') as f: self.assertIsInstance(f, io.TextIOBase) self.assertEqual(f.read(), 'this is file A\n') + def test_open_r_buffering_error(self): + p = self.root / 'fileA' + self.assertRaises(ValueError, vfsopen, p, 'r', buffering=0) + self.assertRaises(ValueError, vfsopen, p, 'r', buffering=1) + self.assertRaises(ValueError, vfsopen, p, 'r', buffering=1024) + @unittest.skipIf( not getattr(sys.flags, 'warn_default_encoding', 0), "Requires warn_default_encoding", @@ -43,17 +49,17 @@ def test_open_r(self): def test_open_r_encoding_warning(self): p = self.root / 'fileA' with self.assertWarns(EncodingWarning) as wc: - with magic_open(p, 'r'): + with vfsopen(p, 'r'): pass self.assertEqual(wc.filename, __file__) def test_open_rb(self): p = self.root / 'fileA' - with magic_open(p, 'rb') as f: + with vfsopen(p, 'rb') as f: self.assertEqual(f.read(), b'this is file A\n') - self.assertRaises(ValueError, magic_open, p, 'rb', encoding='utf8') - self.assertRaises(ValueError, magic_open, p, 'rb', errors='strict') - self.assertRaises(ValueError, magic_open, p, 'rb', newline='') + self.assertRaises(ValueError, vfsopen, p, 'rb', encoding='utf8') + self.assertRaises(ValueError, vfsopen, p, 'rb', errors='strict') + self.assertRaises(ValueError, vfsopen, p, 'rb', newline='') def test_read_bytes(self): p = self.root / 'fileA' diff --git a/Lib/test/test_pathlib/test_write.py b/Lib/test/test_pathlib/test_write.py index b958490d0a8..c9c1d64656c 100644 --- a/Lib/test/test_pathlib/test_write.py +++ b/Lib/test/test_pathlib/test_write.py @@ -13,10 +13,10 @@ if is_pypi: from pathlib_abc import _WritablePath - from pathlib_abc._os import magic_open + from pathlib_abc._os import vfsopen else: from pathlib.types import _WritablePath - from pathlib._os import magic_open + from pathlib._os import vfsopen class WriteTestBase: @@ -31,11 +31,17 @@ def test_is_writable(self): def test_open_w(self): p = self.root / 'fileA' - with magic_open(p, 'w', encoding='utf-8') as f: + with vfsopen(p, 'w', encoding='utf-8') as f: self.assertIsInstance(f, io.TextIOBase) f.write('this is file A\n') self.assertEqual(self.ground.readtext(p), 'this is file A\n') + def test_open_w_buffering_error(self): + p = self.root / 'fileA' + self.assertRaises(ValueError, vfsopen, p, 'w', buffering=0) + self.assertRaises(ValueError, vfsopen, p, 'w', buffering=1) + self.assertRaises(ValueError, vfsopen, p, 'w', buffering=1024) + @unittest.skipIf( not getattr(sys.flags, 'warn_default_encoding', 0), "Requires warn_default_encoding", @@ -43,19 +49,19 @@ def test_open_w(self): def test_open_w_encoding_warning(self): p = self.root / 'fileA' with self.assertWarns(EncodingWarning) as wc: - with magic_open(p, 'w'): + with vfsopen(p, 'w'): pass self.assertEqual(wc.filename, __file__) def test_open_wb(self): p = self.root / 'fileA' - with magic_open(p, 'wb') as f: + with vfsopen(p, 'wb') as f: #self.assertIsInstance(f, io.BufferedWriter) f.write(b'this is file A\n') self.assertEqual(self.ground.readbytes(p), b'this is file A\n') - self.assertRaises(ValueError, magic_open, p, 'wb', encoding='utf8') - self.assertRaises(ValueError, magic_open, p, 'wb', errors='strict') - self.assertRaises(ValueError, magic_open, p, 'wb', newline='') + self.assertRaises(ValueError, vfsopen, p, 'wb', encoding='utf8') + self.assertRaises(ValueError, vfsopen, p, 'wb', errors='strict') + self.assertRaises(ValueError, vfsopen, p, 'wb', newline='') def test_write_bytes(self): p = self.root / 'fileA' diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 54797d7898f..c097808e7fd 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3232,6 +3232,37 @@ def test_pdb_issue_gh_127321(): """ +def test_pdb_issue_gh_136057(): + """See GH-136057 + "step" and "next" commands should be able to get over list comprehensions + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... lst = [i for i in range(10)] + ... for i in lst: pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'next', + ... 'next', + ... 'step', + ... 'continue', + ... ]): + ... test_function() + > <doctest test.test_pdb.test_pdb_issue_gh_136057[0]>(2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) next + > <doctest test.test_pdb.test_pdb_issue_gh_136057[0]>(3)test_function() + -> lst = [i for i in range(10)] + (Pdb) next + > <doctest test.test_pdb.test_pdb_issue_gh_136057[0]>(4)test_function() + -> for i in lst: pass + (Pdb) step + --Return-- + > <doctest test.test_pdb.test_pdb_issue_gh_136057[0]>(4)test_function()->None + -> for i in lst: pass + (Pdb) continue + """ + + def test_pdb_issue_gh_80731(): """See GH-80731 @@ -3549,6 +3580,20 @@ def quux(): ('bœr', 5), ) + def test_print_stack_entry_uses_dynamic_line_prefix(self): + """Test that pdb.line_prefix binding is dynamic (gh-141781).""" + stdout = io.StringIO() + p = pdb.Pdb(stdout=stdout) + + # Get the current frame to use for printing + frame = sys._getframe() + + with support.swap_attr(pdb, 'line_prefix', 'CUSTOM_PREFIX> '): + p.print_stack_entry((frame, frame.f_lineno)) + + # Check if the custom prefix appeared in the output + self.assertIn('CUSTOM_PREFIX> ', stdout.getvalue()) + def test_find_function_found_with_encoding_cookie(self): self._assert_find_function( """\ @@ -3974,7 +4019,10 @@ def test_run_module_with_args(self): commands = """ continue """ - self._run_pdb(["calendar", "-m"], commands, expected_returncode=2) + self._run_pdb(["calendar", "-m"], commands, expected_returncode=1) + + _, stderr = self._run_pdb(["-m", "calendar", "-p", "1"], commands) + self.assertIn("unrecognized arguments: -p", stderr) stdout, _ = self._run_pdb(["-m", "calendar", "1"], commands) self.assertIn("December", stdout) @@ -4539,6 +4587,41 @@ def bar(): ])) self.assertIn('break in bar', stdout) + @unittest.skipIf(SKIP_CORO_TESTS, "Coroutine tests are skipped") + def test_async_break(self): + script = """ + import asyncio + + async def main(): + pass + + asyncio.run(main()) + """ + commands = """ + break main + continue + quit + """ + stdout, stderr = self.run_pdb_script(script, commands) + self.assertRegex(stdout, r"Breakpoint 1 at .*main\.py:5") + self.assertIn("pass", stdout) + + def test_issue_59000(self): + script = """ + def foo(): + pass + + class C: + def foo(self): + pass + """ + commands = """ + break C.foo + quit + """ + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("The specified object 'C.foo' is not a function", stdout) + class ChecklineTests(unittest.TestCase): def setUp(self): @@ -4688,6 +4771,41 @@ def foo(): stdout, _ = self._run_script(script, commands) self.assertIn("42", stdout) + def test_readline_not_imported(self): + """GH-138860 + Directly or indirectly importing readline might deadlock a subprocess + if it's launched with process_group=0 or preexec_fn=setpgrp + + It's also a pattern that readline is never imported with just import pdb. + + This test is to ensure that readline is not imported for import pdb. + It's possible that we have a good reason to do that in the future. + """ + + script = textwrap.dedent(""" + import sys + import pdb + if "readline" in sys.modules: + print("readline imported") + """) + commands = "" + stdout, stderr = self._run_script(script, commands) + self.assertNotIn("readline imported", stdout) + self.assertEqual(stderr, "") + + def test_alternate_stdin(self): + script = textwrap.dedent(""" + import pdb + import io + + input_data = io.StringIO("p 40 + 2\\nc\\n") + pdb.Pdb(stdin=input_data).set_trace() + """) + commands = "" + stdout, stderr = self._run_script(script, commands) + self.assertIn("42", stdout) + self.assertEqual(stderr, "") + @support.force_colorized_test_class class PdbTestColorize(unittest.TestCase): @@ -4749,7 +4867,9 @@ def test_return_from_inline_mode_to_REPL(self): @support.force_not_colorized_test_class @support.requires_subprocess() class PdbTestReadline(unittest.TestCase): - def setUpClass(): + + @classmethod + def setUpClass(cls): # Ensure that the readline module is loaded # If this fails, the test is skipped because SkipTest will be raised readline = import_module('readline') @@ -4848,6 +4968,8 @@ def f(): self.assertIn(b'I love Python', output) + @unittest.skipIf(sys.platform.startswith('freebsd'), + '\\x08 is not interpreted as backspace on FreeBSD') def test_multiline_auto_indent(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() @@ -4886,6 +5008,8 @@ def test_multiline_completion(self): self.assertIn(b'42', output) + @unittest.skipIf(sys.platform.startswith('freebsd'), + '\\x08 is not interpreted as backspace on FreeBSD') def test_multiline_indent_completion(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 47f51f1979f..1caf6de4a14 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -12,7 +12,7 @@ from test import support from test.support.bytecode_helper import ( - BytecodeTestCase, CfgOptimizationTestCase, CompilationStepTestCase) + BytecodeTestCase, CfgOptimizationTestCase) def compile_pattern_with_fast_locals(pattern): @@ -292,6 +292,7 @@ def test_constant_folding_unaryop(self): ('---x', 'UNARY_NEGATIVE', None, False, None, None), ('~~~x', 'UNARY_INVERT', None, False, None, None), ('+++x', 'CALL_INTRINSIC_1', intrinsic_positive, False, None, None), + ('~True', 'UNARY_INVERT', None, False, None, None), ] for ( @@ -316,7 +317,7 @@ def negzero(): return -(1.0-1.0) for instr in dis.get_instructions(negzero): - self.assertFalse(instr.opname.startswith('UNARY_')) + self.assertNotStartsWith(instr.opname, 'UNARY_') self.check_lnotab(negzero) def test_constant_folding_binop(self): @@ -718,9 +719,9 @@ def format(fmt, *values): self.assertEqual(format('x = %d!', 1234), 'x = 1234!') self.assertEqual(format('x = %x!', 1234), 'x = 4d2!') self.assertEqual(format('x = %f!', 1234), 'x = 1234.000000!') - self.assertEqual(format('x = %s!', 1234.5678901), 'x = 1234.5678901!') - self.assertEqual(format('x = %f!', 1234.5678901), 'x = 1234.567890!') - self.assertEqual(format('x = %d!', 1234.5678901), 'x = 1234!') + self.assertEqual(format('x = %s!', 1234.0000625), 'x = 1234.0000625!') + self.assertEqual(format('x = %f!', 1234.0000625), 'x = 1234.000063!') + self.assertEqual(format('x = %d!', 1234.0000625), 'x = 1234!') self.assertEqual(format('x = %s%% %%%%', 1234), 'x = 1234% %%') self.assertEqual(format('x = %s!', '%% %s'), 'x = %% %s!') self.assertEqual(format('x = %s, y = %d', 12, 34), 'x = 12, y = 34') @@ -1115,6 +1116,13 @@ def trace(frame, event, arg): self.assertInBytecode(f, "LOAD_FAST_BORROW") self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + def test_import_from_doesnt_clobber_load_fast_borrow(self): + def f(self): + if x: pass + self.x + from shutil import ExecError + print(ExecError) + self.assertInBytecode(f, "LOAD_FAST_BORROW", "self") class DirectCfgOptimizerTests(CfgOptimizationTestCase): @@ -2614,6 +2622,90 @@ def test_send(self): ] self.cfg_optimization_test(insts, expected, consts=[None]) + def test_format_simple(self): + # FORMAT_SIMPLE will leave its operand on the stack if it's a unicode + # object. We treat it conservatively and assume that it always leaves + # its operand on the stack. + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, expected) + + def test_set_function_attribute(self): + # SET_FUNCTION_ATTRIBUTE leaves the function on the stack + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("STORE_FAST", 1, 4), + ("LOAD_CONST", 0, 5), + ("RETURN_VALUE", None, 6) + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + expected = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST_BORROW", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + self.cfg_optimization_test(insts, expected, consts=[None]) + + def test_get_yield_from_iter(self): + # GET_YIELD_FROM_ITER may leave its operand on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_YIELD_FROM_ITER", None, 2), + ("LOAD_CONST", 0, 3), + send := self.Label(), + ("SEND", end := self.Label(), 5), + ("YIELD_VALUE", 1, 6), + ("RESUME", 2, 7), + ("JUMP", send, 8), + end, + ("END_SEND", None, 9), + ("LOAD_CONST", 0, 10), + ("RETURN_VALUE", None, 11), + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + def test_push_exc_info(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("PUSH_EXC_INFO", None, 2), + ] + self.check(insts, insts) + + def test_load_special(self): + # LOAD_SPECIAL may leave self on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_SPECIAL", 0, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + def test_del_in_finally(self): # This loads `obj` onto the stack, executes `del obj`, then returns the # `obj` from the stack. See gh-133371 for more details. @@ -2630,6 +2722,14 @@ def create_obj(): gc.collect() self.assertEqual(obj, [42]) + def test_format_simple_unicode(self): + # Repro from gh-134889 + def f(): + var = f"{1}" + var = f"{var}" + return var + self.assertEqual(f(), "1") + if __name__ == "__main__": diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 1095e7303c1..395f15b9a62 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -100,11 +100,15 @@ def setUpClass(cls): with contextlib.ExitStack() as stack: python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools("venv")) - sitepackages = subprocess.check_output( + platlib_path = subprocess.check_output( [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], text=True, ).strip() - stack.enter_context(import_helper.DirsOnSysPath(sitepackages)) + purelib_path = subprocess.check_output( + [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], + text=True, + ).strip() + stack.enter_context(import_helper.DirsOnSysPath(platlib_path, purelib_path)) cls.addClassCleanup(stack.pop_all().close) @support.requires_venv_with_pip() @@ -387,10 +391,10 @@ def test_with_stmt_with_paren(self) -> None: test_source = """ stmt = "with (\\n a as b,\\n c as d\\n): pass" the_ast = parse.parse_string(stmt, mode=1) - self.assertTrue(ast_dump(the_ast).startswith( + self.assertStartsWith(ast_dump(the_ast), "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), " "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]" - )) + ) """ self.run_test(grammar_source, test_source) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index d8606521345..d03ba07975a 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -91,10 +91,8 @@ def test_gather(self) -> None: """ rules = parse_string(grammar, GrammarParser).rules self.assertEqual(str(rules["start"]), "start: ','.thing+ NEWLINE") - self.assertTrue( - repr(rules["start"]).startswith( - "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" - ) + self.assertStartsWith(repr(rules["start"]), + "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" ) self.assertEqual(str(rules["thing"]), "thing: NUMBER") parser_class = make_parser(grammar) @@ -1108,3 +1106,53 @@ def test_deep_nested_rule(self) -> None: ) self.assertEqual(output, expected_output) + + def test_rule_flags(self) -> None: + """Test the new rule flags syntax that accepts arbitrary lists of flags.""" + # Test grammar with various flag combinations + grammar_source = """ + start: simple_rule + + simple_rule (memo): + | "hello" + + multi_flag_rule (memo, custom, test): + | "world" + + single_custom_flag (custom): + | "test" + + no_flags_rule: + | "plain" + """ + + grammar: Grammar = parse_string(grammar_source, GrammarParser) + rules = grammar.rules + + # Test memo-only rule + simple_rule = rules['simple_rule'] + self.assertTrue('memo' in simple_rule.flags, + "simple_rule should have memo") + self.assertEqual(simple_rule.flags, frozenset(['memo']), + f"simple_rule flags should be {'memo'}, got {simple_rule.flags}") + + # Test multi-flag rule + multi_flag_rule = rules['multi_flag_rule'] + self.assertTrue('memo' in simple_rule.flags, + "multi_flag_rule should have memo") + self.assertEqual(multi_flag_rule.flags, frozenset({'memo', 'custom', 'test'}), + f"multi_flag_rule flags should contain memo, custom, test, got {multi_flag_rule.flags}") + + # Test single custom flag rule + single_custom_rule = rules['single_custom_flag'] + self.assertFalse('memo' not in simple_rule.flags, + "single_custom_flag should not have memo") + self.assertEqual(single_custom_rule.flags, frozenset(['custom']), + f"single_custom_flag flags should be {'custom'}, got {single_custom_rule.flags}") + + # Test no flags rule + no_flags_rule = rules['no_flags_rule'] + self.assertFalse('memo' not in simple_rule.flags, + "no_flags_rule should not have memo") + self.assertEqual(no_flags_rule.flags, frozenset(), + f"no_flags_rule flags should be the empty set, got {no_flags_rule.flags}") diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index c176e505155..66348619073 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -93,9 +93,7 @@ def baz(): perf_line, f"Could not find {expected_symbol} in perf file" ) perf_addr = perf_line.split(" ")[0] - self.assertFalse( - perf_addr.startswith("0x"), "Address should not be prefixed with 0x" - ) + self.assertNotStartsWith(perf_addr, "0x") self.assertTrue( set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters", @@ -162,50 +160,67 @@ def baz(): self.assertIn(f"py::bar_fork:{script}", child_perf_file_contents) self.assertIn(f"py::baz_fork:{script}", child_perf_file_contents) + # The parent's map should not contain the child's symbols. + self.assertNotIn(f"py::foo_fork:{script}", perf_file_contents) + self.assertNotIn(f"py::bar_fork:{script}", perf_file_contents) + self.assertNotIn(f"py::baz_fork:{script}", perf_file_contents) + + # The child's map should not contain the parent's symbols. + self.assertNotIn(f"py::foo:{script}", child_perf_file_contents) + self.assertNotIn(f"py::bar:{script}", child_perf_file_contents) + self.assertNotIn(f"py::baz:{script}", child_perf_file_contents) + @unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") def test_sys_api(self): - code = """if 1: - import sys - def foo(): - pass + for define_eval_hook in (False, True): + code = """if 1: + import sys + def foo(): + pass - def spam(): - pass + def spam(): + pass + + def bar(): + sys.deactivate_stack_trampoline() + foo() + sys.activate_stack_trampoline("perf") + spam() + + def baz(): + bar() - def bar(): - sys.deactivate_stack_trampoline() - foo() sys.activate_stack_trampoline("perf") - spam() + baz() + """ + if define_eval_hook: + set_eval_hook = """if 1: + import _testinternalcapi + _testinternalcapi.set_eval_frame_record([]) +""" + code = set_eval_hook + code + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} + with subprocess.Popen( + [sys.executable, script], + text=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=env, + ) as process: + stdout, stderr = process.communicate() - def baz(): - bar() + self.assertEqual(stderr, "") + self.assertEqual(stdout, "") - sys.activate_stack_trampoline("perf") - baz() - """ - with temp_dir() as script_dir: - script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} - with subprocess.Popen( - [sys.executable, script], - text=True, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - env=env, - ) as process: - stdout, stderr = process.communicate() - - self.assertEqual(stderr, "") - self.assertEqual(stdout, "") - - perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") - self.assertTrue(perf_file.exists()) - perf_file_contents = perf_file.read_text() - self.assertNotIn(f"py::foo:{script}", perf_file_contents) - self.assertIn(f"py::spam:{script}", perf_file_contents) - self.assertIn(f"py::bar:{script}", perf_file_contents) - self.assertIn(f"py::baz:{script}", perf_file_contents) + perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") + self.assertTrue(perf_file.exists()) + perf_file_contents = perf_file.read_text() + self.assertNotIn(f"py::foo:{script}", perf_file_contents) + self.assertIn(f"py::spam:{script}", perf_file_contents) + self.assertIn(f"py::bar:{script}", perf_file_contents) + self.assertIn(f"py::baz:{script}", perf_file_contents) def test_sys_api_with_existing_trampoline(self): code = """if 1: @@ -233,6 +248,24 @@ def test_sys_api_get_status(self): """ assert_python_ok("-c", code, PYTHON_JIT="0") + def test_sys_api_perf_jit_backend(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + assert sys.is_stack_trampoline_active() is True + sys.deactivate_stack_trampoline() + assert sys.is_stack_trampoline_active() is False + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + + def test_sys_api_with_existing_perf_jit_trampoline(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + sys.activate_stack_trampoline("perf_jit") + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") @@ -320,6 +353,7 @@ def run_perf(cwd, *args, use_jit=False, **env_vars): stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, + text=True, ) if proc.returncode: print(proc.stderr, file=sys.stderr) @@ -329,10 +363,10 @@ def run_perf(cwd, *args, use_jit=False, **env_vars): jit_output_file = cwd + "/jit_output.dump" command = ("perf", "inject", "-j", "-i", output_file, "-o", jit_output_file) proc = subprocess.run( - command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env + command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env, text=True ) if proc.returncode: - print(proc.stderr) + print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") # Copy the jit_output_file to the output_file os.rename(jit_output_file, output_file) @@ -344,10 +378,9 @@ def run_perf(cwd, *args, use_jit=False, **env_vars): stderr=subprocess.PIPE, env=env, check=True, + text=True, ) - return proc.stdout.decode("utf-8", "replace"), proc.stderr.decode( - "utf-8", "replace" - ) + return proc.stdout, proc.stderr class TestPerfProfilerMixin: @@ -508,9 +541,12 @@ def _is_perf_version_at_least(major, minor): # The output of perf --version looks like "perf version 6.7-3" but # it can also be perf version "perf version 5.15.143", or even include # a commit hash in the version string, like "6.12.9.g242e6068fd5c" + # + # PermissionError is raised if perf does not exist on the Windows Subsystem + # for Linux, see #134987 try: output = subprocess.check_output(["perf", "--version"], text=True) - except (subprocess.CalledProcessError, FileNotFoundError): + except (subprocess.CalledProcessError, FileNotFoundError, PermissionError): return False version = output.split()[2] version = version.split("-")[0] diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index d4c6fe0124a..647c32656ab 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -1,5 +1,5 @@ import os -import sys +import sysconfig import unittest try: @@ -7,10 +7,14 @@ except ImportError: raise unittest.SkipTest("requires _testinternalcapi") +def supports_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 -if sys.platform != 'linux': - raise unittest.SkipTest('Linux only') - +if not supports_trampoline_profiling(): + raise unittest.SkipTest("perf trampoline profiling not supported") class TestPerfMapWriting(unittest.TestCase): def test_write_perf_map_entry(self): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 742ca8de1be..22c70327fb0 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -59,6 +59,8 @@ class PyUnpicklerTests(AbstractUnpickleTests, unittest.TestCase): truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) + truncated_data_error = (EOFError, '') + size_overflow_error = (pickle.UnpicklingError, 'exceeds') def loads(self, buf, **kwds): f = io.BytesIO(buf) @@ -103,6 +105,8 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) + truncated_data_error = ((pickle.UnpicklingError, EOFError), '') + size_overflow_error = ((OverflowError, pickle.UnpicklingError), 'exceeds') def dumps(self, arg, protocol=None, **kwargs): return pickle.dumps(arg, protocol, **kwargs) @@ -375,6 +379,8 @@ class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) truncated_errors = (pickle.UnpicklingError,) + truncated_data_error = (pickle.UnpicklingError, 'truncated') + size_overflow_error = (OverflowError, 'exceeds') class CPicklingErrorTests(PyPicklingErrorTests): pickler = _pickle.Pickler @@ -478,7 +484,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i') + basesize = support.calcobjsize('2P2n3P 2P2n2i5P 2P3n8P2n2i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. @@ -611,10 +617,10 @@ def test_name_mapping(self): with self.subTest(((module3, name3), (module2, name2))): if (module2, name2) == ('exceptions', 'OSError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, OSError)) + self.assertIsSubclass(attr, OSError) elif (module2, name2) == ('exceptions', 'ImportError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, ImportError)) + self.assertIsSubclass(attr, ImportError) else: module, name = mapping(module2, name2) if module3[:1] != '_': diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index a178d3353ee..57285ddf6eb 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -1,7 +1,11 @@ import io +import itertools import pickle import pickletools +import tempfile +import textwrap from test import support +from test.support import os_helper from test.pickletester import AbstractPickleTests import doctest import unittest @@ -384,13 +388,13 @@ def test_string_without_quotes(self): self.check_dis_error(b'Sabc"\n.', '', "no string quotes around b'abc\"'") self.check_dis_error(b"S'abc\n.", '', - '''strinq quote b"'" not found at both ends of b"'abc"''') + '''string quote b"'" not found at both ends of b"'abc"''') self.check_dis_error(b'S"abc\n.', '', - r"""strinq quote b'"' not found at both ends of b'"abc'""") + r"""string quote b'"' not found at both ends of b'"abc'""") self.check_dis_error(b"S'abc\"\n.", '', - r"""strinq quote b"'" not found at both ends of b'\\'abc"'""") + r"""string quote b"'" not found at both ends of b'\\'abc"'""") self.check_dis_error(b"S\"abc'\n.", '', - r"""strinq quote b'"' not found at both ends of b'"abc\\''""") + r"""string quote b'"' not found at both ends of b'"abc\\''""") def test_binstring(self): self.check_dis(b"T\x03\x00\x00\x00abc.", '''\ @@ -514,6 +518,170 @@ def test__all__(self): support.check__all__(self, pickletools, not_exported=not_exported) +class CommandLineTest(unittest.TestCase): + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + return textwrap.dedent(string).strip() + + def set_pickle_data(self, data): + with open(self.filename, 'wb') as f: + pickle.dump(data, f) + + def invoke_pickletools(self, *flags): + with ( + support.captured_stdout() as stdout, + support.captured_stderr() as stderr, + ): + pickletools._main(args=[*flags, self.filename]) + self.assertEqual(stderr.getvalue(), '') + return self.text_normalize(stdout.getvalue()) + + def check_output(self, data, expect, *flags): + with self.subTest(data=data, flags=flags): + self.set_pickle_data(data) + res = self.invoke_pickletools(*flags) + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + def test_invocation(self): + # test various combinations of parameters + output_file = tempfile.mktemp() + self.addCleanup(os_helper.unlink, output_file) + base_flags = [ + (f'-o={output_file}', f'--output={output_file}'), + ('-m', '--memo'), + ('-l=2', '--indentlevel=2'), + ('-a', '--annotate'), + ('-p="Another:"', '--preamble="Another:"'), + ] + data = { 'a', 'b', 'c' } + + self.set_pickle_data(data) + + for r in range(1, len(base_flags) + 1): + for choices in itertools.combinations(base_flags, r=r): + for args in itertools.product(*choices): + with self.subTest(args=args[1:]): + self.invoke_pickletools(*args) + + def test_unknown_flag(self): + with self.assertRaises(SystemExit): + with support.captured_stderr() as stderr: + pickletools._main(args=['--unknown']) + self.assertStartsWith(stderr.getvalue(), 'usage: ') + + def test_output_flag(self): + # test 'python -m pickletools -o/--output' + output_file = tempfile.mktemp() + self.addCleanup(os_helper.unlink, output_file) + data = ('fake_data',) + expect = r''' + 0: \x80 PROTO 5 + 2: \x95 FRAME 15 + 11: \x8c SHORT_BINUNICODE 'fake_data' + 22: \x94 MEMOIZE (as 0) + 23: \x85 TUPLE1 + 24: \x94 MEMOIZE (as 1) + 25: . STOP + highest protocol among opcodes = 4 + ''' + for flag in [f'-o={output_file}', f'--output={output_file}']: + with self.subTest(data=data, flags=flag): + self.set_pickle_data(data) + res = self.invoke_pickletools(flag) + with open(output_file, 'r') as f: + res_from_file = self.text_normalize(f.read()) + expect = self.text_normalize(expect) + + self.assertListEqual(res.splitlines(), []) + self.assertListEqual(res_from_file.splitlines(), + expect.splitlines()) + + def test_memo_flag(self): + # test 'python -m pickletools -m/--memo' + data = ('fake_data',) + expect = r''' + 0: \x80 PROTO 5 + 2: \x95 FRAME 15 + 11: \x8c SHORT_BINUNICODE 'fake_data' + 22: \x94 MEMOIZE (as 0) + 23: \x85 TUPLE1 + 24: \x94 MEMOIZE (as 1) + 25: . STOP + highest protocol among opcodes = 4 + ''' + for flag in ['-m', '--memo']: + self.check_output(data, expect, flag) + + def test_indentlevel_flag(self): + # test 'python -m pickletools -l/--indentlevel' + data = ('fake_data',) + expect = r''' + 0: \x80 PROTO 5 + 2: \x95 FRAME 15 + 11: \x8c SHORT_BINUNICODE 'fake_data' + 22: \x94 MEMOIZE (as 0) + 23: \x85 TUPLE1 + 24: \x94 MEMOIZE (as 1) + 25: . STOP + highest protocol among opcodes = 4 + ''' + for flag in ['-l=2', '--indentlevel=2']: + self.check_output(data, expect, flag) + + def test_annotate_flag(self): + # test 'python -m pickletools -a/--annotate' + data = ('fake_data',) + expect = r''' + 0: \x80 PROTO 5 Protocol version indicator. + 2: \x95 FRAME 15 Indicate the beginning of a new frame. + 11: \x8c SHORT_BINUNICODE 'fake_data' Push a Python Unicode string object. + 22: \x94 MEMOIZE (as 0) Store the stack top into the memo. The stack is not popped. + 23: \x85 TUPLE1 Build a one-tuple out of the topmost item on the stack. + 24: \x94 MEMOIZE (as 1) Store the stack top into the memo. The stack is not popped. + 25: . STOP Stop the unpickling machine. + highest protocol among opcodes = 4 + ''' + for flag in ['-a', '--annotate']: + self.check_output(data, expect, flag) + + def test_preamble_flag(self): + # test 'python -m pickletools -p/--preamble' + data = ('fake_data',) + expect = r''' + Another: + 0: \x80 PROTO 5 + 2: \x95 FRAME 15 + 11: \x8c SHORT_BINUNICODE 'fake_data' + 22: \x94 MEMOIZE (as 0) + 23: \x85 TUPLE1 + 24: \x94 MEMOIZE (as 1) + 25: . STOP + highest protocol among opcodes = 4 + Another: + 0: \x80 PROTO 5 + 2: \x95 FRAME 15 + 11: \x8c SHORT_BINUNICODE 'fake_data' + 22: \x94 MEMOIZE (as 0) + 23: \x85 TUPLE1 + 24: \x94 MEMOIZE (as 1) + 25: . STOP + highest protocol among opcodes = 4 + ''' + for flag in ['-p=Another:', '--preamble=Another:']: + with self.subTest(data=data, flags=flag): + self.set_pickle_data(data) + with support.captured_stdout() as stdout: + pickletools._main(args=[flag, self.filename, self.filename]) + res = self.text_normalize(stdout.getvalue()) + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(pickletools)) return tests diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 818e807dd3a..9ee97b922ad 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -11,7 +11,7 @@ from unittest import mock from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper try: # Some of the iOS tests need ctypes to operate. @@ -133,6 +133,22 @@ def test_platform(self): for terse in (False, True): res = platform.platform(aliased, terse) + def test__platform(self): + for src, res in [ + ('foo bar', 'foo_bar'), + ( + '1/2\\3:4;5"6(7)8(7)6"5;4:3\\2/1', + '1-2-3-4-5-6-7-8-7-6-5-4-3-2-1' + ), + ('--', ''), + ('-f', '-f'), + ('-foo----', '-foo'), + ('--foo---', '-foo'), + ('---foo--', '-foo'), + ]: + with self.subTest(src=src): + self.assertEqual(platform._platform(src), res) + def test_system(self): res = platform.system() @@ -383,15 +399,6 @@ def raises_oserror(*a): finally: platform._uname_cache = None - def test_java_ver(self): - import re - msg = re.escape( - "'java_ver' is deprecated and slated for removal in Python 3.15" - ) - with self.assertWarnsRegex(DeprecationWarning, msg): - res = platform.java_ver() - self.assertEqual(len(res), 4) - @unittest.skipUnless(support.MS_WINDOWS, 'This test only makes sense on Windows') def test_win32_ver(self): release1, version1, csd1, ptype1 = 'a', 'b', 'c', 'd' @@ -410,7 +417,7 @@ def test_win32_ver(self): for v in version.split('.'): int(v) # should not fail if csd: - self.assertTrue(csd.startswith('SP'), msg=csd) + self.assertStartsWith(csd, 'SP') if ptype: if os.cpu_count() > 1: self.assertIn('Multiprocessor', ptype) @@ -458,7 +465,7 @@ def test_mac_ver(self): else: self.assertEqual(res[2], 'PowerPC') - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(sys.platform == 'darwin', "OSX only test") def test_mac_ver_with_fork(self): # Issue7895: platform.mac_ver() crashes when using fork without exec @@ -525,8 +532,10 @@ def test_ios_ver(self): self.assertEqual(override.model, "Whiz") self.assertTrue(override.is_simulator) - @unittest.skipIf(support.is_emscripten, "Does not apply to Emscripten") def test_libc_ver(self): + if support.is_emscripten: + assert platform.libc_ver() == ("emscripten", "4.0.12") + return # check that libc_ver(executable) doesn't raise an exception if os.path.isdir(sys.executable) and \ os.path.exists(sys.executable+'.exe'): @@ -558,6 +567,10 @@ def test_libc_ver(self): # musl uses semver, but we accept some variations anyway: (b'/aports/main/musl/src/musl-12.5', ('musl', '12.5')), (b'/aports/main/musl/src/musl-1.2.5.7', ('musl', '1.2.5.7')), + (b'libc.musl.so.1', ('musl', '1')), + (b'libc.musl-x86_64.so.1.2.5', ('musl', '1.2.5')), + (b'ld-musl.so.1', ('musl', '1')), + (b'ld-musl-x86_64.so.1.2.5', ('musl', '1.2.5')), (b'', ('', '')), ): with open(filename, 'wb') as fp: @@ -576,6 +589,14 @@ def test_libc_ver(self): (b'GLIBC_1.23.4\0GLIBC_1.9\0GLIBC_1.21\0', ('glibc', '1.23.4')), (b'libc.so.2.4\0libc.so.9\0libc.so.23.1\0', ('libc', '23.1')), (b'musl-1.4.1\0musl-2.1.1\0musl-2.0.1\0', ('musl', '2.1.1')), + ( + b'libc.musl-x86_64.so.1.4.1\0libc.musl-x86_64.so.2.1.1\0libc.musl-x86_64.so.2.0.1', + ('musl', '2.1.1'), + ), + ( + b'ld-musl-x86_64.so.1.4.1\0ld-musl-x86_64.so.2.1.1\0ld-musl-x86_64.so.2.0.1', + ('musl', '2.1.1'), + ), (b'no match here, so defaults are used', ('test', '100.1.0')), ): with open(filename, 'wb') as f: @@ -755,13 +776,14 @@ def invoke_platform(self, *flags): platform._main(args=flags) return output.getvalue() + @support.force_not_colorized def test_unknown_flag(self): + output = io.StringIO() with self.assertRaises(SystemExit): - output = io.StringIO() # suppress argparse error message with contextlib.redirect_stderr(output): _ = self.invoke_platform('--unknown') - self.assertStartsWith(output, "usage: ") + self.assertStartsWith(output.getvalue(), "usage: ") def test_invocation(self): flags = ( diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index a0c76e5dec5..de2a2fd1fc3 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -903,8 +903,7 @@ def test_dump_naive_datetime_with_aware_datetime_option(self): class TestBinaryPlistlib(unittest.TestCase): - @staticmethod - def decode(*objects, offset_size=1, ref_size=1): + def build(self, *objects, offset_size=1, ref_size=1): data = [b'bplist00'] offset = 8 offsets = [] @@ -916,7 +915,11 @@ def decode(*objects, offset_size=1, ref_size=1): len(objects), 0, offset) data.extend(offsets) data.append(tail) - return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY) + return b''.join(data) + + def decode(self, *objects, offset_size=1, ref_size=1): + data = self.build(*objects, offset_size=offset_size, ref_size=ref_size) + return plistlib.loads(data, fmt=plistlib.FMT_BINARY) def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers @@ -1024,6 +1027,34 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_truncated_large_data(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + def check(data): + with open(os_helper.TESTFN, 'wb') as f: + f.write(data) + # buffered file + with open(os_helper.TESTFN, 'rb') as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + # unbuffered file + with open(os_helper.TESTFN, 'rb', buffering=0) as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + for w in range(20, 64): + s = 1 << w + # data + check(self.build(b'\x4f\x13' + s.to_bytes(8, 'big'))) + # ascii string + check(self.build(b'\x5f\x13' + s.to_bytes(8, 'big'))) + # unicode string + check(self.build(b'\x6f\x13' + s.to_bytes(8, 'big'))) + # array + check(self.build(b'\xaf\x13' + s.to_bytes(8, 'big'))) + # dict + check(self.build(b'\xdf\x13' + s.to_bytes(8, 'big'))) + # number of objects + check(b'bplist00' + struct.pack('>6xBBQQQ', 1, 1, s, 0, 8)) + def test_load_aware_datetime(self): data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index eea0625012d..e412cb1d58d 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -37,8 +37,8 @@ def test_invalid_syntax_errors(self): check_syntax_error(self, "def f(/): pass") check_syntax_error(self, "def f(*, a, /): pass") check_syntax_error(self, "def f(*, /, a): pass") - check_syntax_error(self, "def f(a, /, a): pass", "duplicate argument 'a' in function definition") - check_syntax_error(self, "def f(a, /, *, a): pass", "duplicate argument 'a' in function definition") + check_syntax_error(self, "def f(a, /, a): pass", "duplicate parameter 'a' in function definition") + check_syntax_error(self, "def f(a, /, *, a): pass", "duplicate parameter 'a' in function definition") check_syntax_error(self, "def f(a, b/2, c): pass") check_syntax_error(self, "def f(a, /, c, /): pass") check_syntax_error(self, "def f(a, /, c, /, d): pass") @@ -59,8 +59,8 @@ def test_invalid_syntax_errors_async(self): check_syntax_error(self, "async def f(/): pass") check_syntax_error(self, "async def f(*, a, /): pass") check_syntax_error(self, "async def f(*, /, a): pass") - check_syntax_error(self, "async def f(a, /, a): pass", "duplicate argument 'a' in function definition") - check_syntax_error(self, "async def f(a, /, *, a): pass", "duplicate argument 'a' in function definition") + check_syntax_error(self, "async def f(a, /, a): pass", "duplicate parameter 'a' in function definition") + check_syntax_error(self, "async def f(a, /, *, a): pass", "duplicate parameter 'a' in function definition") check_syntax_error(self, "async def f(a, b/2, c): pass") check_syntax_error(self, "async def f(a, /, c, /): pass") check_syntax_error(self, "async def f(a, /, c, /, d): pass") @@ -247,8 +247,8 @@ def test_invalid_syntax_lambda(self): check_syntax_error(self, "lambda /: None") check_syntax_error(self, "lambda *, a, /: None") check_syntax_error(self, "lambda *, /, a: None") - check_syntax_error(self, "lambda a, /, a: None", "duplicate argument 'a' in function definition") - check_syntax_error(self, "lambda a, /, *, a: None", "duplicate argument 'a' in function definition") + check_syntax_error(self, "lambda a, /, a: None", "duplicate parameter 'a' in function definition") + check_syntax_error(self, "lambda a, /, *, a: None", "duplicate parameter 'a' in function definition") check_syntax_error(self, "lambda a, /, b, /: None") check_syntax_error(self, "lambda a, /, b, /, c: None") check_syntax_error(self, "lambda a, /, b, /, c, *, d: None") diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index fa19d549c26..a9975b75f7c 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,15 +1,17 @@ +import errno import inspect import os import posixpath import random import sys import unittest -from posixpath import realpath, abspath, dirname, basename +from functools import partial +from posixpath import realpath, abspath, dirname, basename, ALL_BUT_LAST, ALLOW_MISSING from test import support from test import test_genericpath from test.support import import_helper from test.support import os_helper -from test.support.os_helper import FakePath +from test.support.os_helper import FakePath, TESTFN from unittest import mock try: @@ -21,7 +23,7 @@ # An absolute path to a temporary filename for testing. We can't rely on TESTFN # being an absolute path, so we need this. -ABSTFN = abspath(os_helper.TESTFN) +ABSTFN = abspath(TESTFN) def skip_if_ABSTFN_contains_backslash(test): """ @@ -33,21 +35,16 @@ def skip_if_ABSTFN_contains_backslash(test): msg = "ABSTFN is not a posix path - tests fail" return [test, unittest.skip(msg)(test)][found_backslash] -def safe_rmdir(dirname): - try: - os.rmdir(dirname) - except OSError: - pass + +def _parameterize(*parameters): + return support.subTests('kwargs', parameters) + class PosixPathTest(unittest.TestCase): def setUp(self): - self.tearDown() - - def tearDown(self): for suffix in ["", "1", "2"]: - os_helper.unlink(os_helper.TESTFN + suffix) - safe_rmdir(os_helper.TESTFN + suffix) + self.assertFalse(posixpath.lexists(ABSTFN + suffix)) def test_join(self): fn = posixpath.join @@ -194,25 +191,28 @@ def test_dirname(self): self.assertEqual(posixpath.dirname(b"//foo//bar"), b"//foo") def test_islink(self): - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), False) - with open(os_helper.TESTFN + "1", "wb") as f: + self.addCleanup(os_helper.unlink, TESTFN + "1") + with open(TESTFN + "1", "wb") as f: f.write(b"foo") - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) if os_helper.can_symlink(): - os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN + "2") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - os.remove(os_helper.TESTFN + "1") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - self.assertIs(posixpath.exists(os_helper.TESTFN + "2"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), True) + self.addCleanup(os_helper.unlink, TESTFN + "2") + os.symlink(TESTFN + "1", TESTFN + "2") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + os.remove(TESTFN + "1") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + self.assertIs(posixpath.exists(TESTFN + "2"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), True) - self.assertIs(posixpath.islink(os_helper.TESTFN + "\udfff"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\xff"), False) - self.assertIs(posixpath.islink(os_helper.TESTFN + "\x00"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\x00"), False) + def test_islink_invalid_paths(self): + self.assertIs(posixpath.islink(TESTFN + "\udfff"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\xff"), False) + self.assertIs(posixpath.islink(TESTFN + "\x00"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\x00"), False) def test_ismount(self): self.assertIs(posixpath.ismount("/"), True) @@ -227,8 +227,9 @@ def test_ismount_non_existent(self): os.mkdir(ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) + def test_ismount_invalid_paths(self): self.assertIs(posixpath.ismount('/\udfff'), False) self.assertIs(posixpath.ismount(b'/\xff'), False) self.assertIs(posixpath.ismount('/\x00'), False) @@ -241,7 +242,7 @@ def test_ismount_symlinks(self): os.symlink("/", ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - os.unlink(ABSTFN) + os_helper.unlink(ABSTFN) @unittest.skipIf(posix is None, "Test requires posix module") def test_ismount_different_device(self): @@ -448,32 +449,35 @@ def test_normpath(self): self.assertEqual(result, expected) @skip_if_ABSTFN_contains_backslash - def test_realpath_curdir(self): - self.assertEqual(realpath('.'), os.getcwd()) - self.assertEqual(realpath('./.'), os.getcwd()) - self.assertEqual(realpath('/'.join(['.'] * 100)), os.getcwd()) + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_curdir(self, kwargs): + self.assertEqual(realpath('.', **kwargs), os.getcwd()) + self.assertEqual(realpath('./.', **kwargs), os.getcwd()) + self.assertEqual(realpath('/'.join(['.'] * 100), **kwargs), os.getcwd()) - self.assertEqual(realpath(b'.'), os.getcwdb()) - self.assertEqual(realpath(b'./.'), os.getcwdb()) - self.assertEqual(realpath(b'/'.join([b'.'] * 100)), os.getcwdb()) + self.assertEqual(realpath(b'.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'./.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'/'.join([b'.'] * 100), **kwargs), os.getcwdb()) @skip_if_ABSTFN_contains_backslash - def test_realpath_pardir(self): - self.assertEqual(realpath('..'), dirname(os.getcwd())) - self.assertEqual(realpath('../..'), dirname(dirname(os.getcwd()))) - self.assertEqual(realpath('/'.join(['..'] * 100)), '/') + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_pardir(self, kwargs): + self.assertEqual(realpath('..', **kwargs), dirname(os.getcwd())) + self.assertEqual(realpath('../..', **kwargs), dirname(dirname(os.getcwd()))) + self.assertEqual(realpath('/'.join(['..'] * 100), **kwargs), '/') - self.assertEqual(realpath(b'..'), dirname(os.getcwdb())) - self.assertEqual(realpath(b'../..'), dirname(dirname(os.getcwdb()))) - self.assertEqual(realpath(b'/'.join([b'..'] * 100)), b'/') + self.assertEqual(realpath(b'..', **kwargs), dirname(os.getcwdb())) + self.assertEqual(realpath(b'../..', **kwargs), dirname(dirname(os.getcwdb()))) + self.assertEqual(realpath(b'/'.join([b'..'] * 100), **kwargs), b'/') @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_basic(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): # Basic operation. try: os.symlink(ABSTFN+"1", ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) @@ -489,12 +493,124 @@ def test_realpath_strict(self): finally: os_helper.unlink(ABSTFN) + def test_realpath_invalid_paths(self): + path = '/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = '/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/') + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/nonexistent') + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/nonexistent') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + path = b'/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=ALL_BUT_LAST), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + path = b'/nonexistent/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_relative(self): + @_parameterize({}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): try: os.symlink(posixpath.relpath(ABSTFN+"1"), ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) @@ -502,10 +618,14 @@ def test_realpath_relative(self): @skip_if_ABSTFN_contains_backslash def test_realpath_missing_pardir(self): try: - os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN) - self.assertEqual(realpath("nonexistent/../" + os_helper.TESTFN), ABSTFN + "1") + os.symlink(TESTFN + "1", TESTFN) + path = "nonexistent/../" + TESTFN + self.assertEqual(realpath(path), ABSTFN + "1") + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFN + "1") + self.assertRaises(FileNotFoundError, realpath, path, strict=ALL_BUT_LAST) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) finally: - os_helper.unlink(os_helper.TESTFN) + os_helper.unlink(TESTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -550,37 +670,38 @@ def test_realpath_symlink_loops(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_symlink_loops_strict(self): + @_parameterize({'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_loops_strict(self, kwargs): # Bug #43757, raise OSError if we get into an infinite symlink loop in - # strict mode. + # the strict modes. try: os.symlink(ABSTFN, ABSTFN) - self.assertRaises(OSError, realpath, ABSTFN, strict=True) + self.assertRaises(OSError, realpath, ABSTFN, **kwargs) os.symlink(ABSTFN+"1", ABSTFN+"2") os.symlink(ABSTFN+"2", ABSTFN+"1") - self.assertRaises(OSError, realpath, ABSTFN+"1", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"2", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"2", **kwargs) - self.assertRaises(OSError, realpath, ABSTFN+"1/x", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/..", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/../x", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1/x", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/..", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/../x", **kwargs) os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "y", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "y", **kwargs) self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "1", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "1", **kwargs) os.symlink(basename(ABSTFN) + "a/b", ABSTFN+"a") - self.assertRaises(OSError, realpath, ABSTFN+"a", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"a", **kwargs) os.symlink("../" + basename(dirname(ABSTFN)) + "/" + basename(ABSTFN) + "c", ABSTFN+"c") - self.assertRaises(OSError, realpath, ABSTFN+"c", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"c", **kwargs) # Test using relative path as well. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertRaises(OSError, realpath, basename(ABSTFN), strict=True) + self.assertRaises(OSError, realpath, basename(ABSTFN), **kwargs) finally: os_helper.unlink(ABSTFN) os_helper.unlink(ABSTFN+"1") @@ -591,28 +712,30 @@ def test_realpath_symlink_loops_strict(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_repeated_indirect_symlinks(self): + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_repeated_indirect_symlinks(self, kwargs): # Issue #6975. try: os.mkdir(ABSTFN) os.symlink('../' + basename(ABSTFN), ABSTFN + '/self') os.symlink('self/self/self', ABSTFN + '/link') - self.assertEqual(realpath(ABSTFN + '/link'), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/link', **kwargs), ABSTFN) finally: os_helper.unlink(ABSTFN + '/self') os_helper.unlink(ABSTFN + '/link') - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_deep_recursion(self): + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_deep_recursion(self, kwargs): depth = 10 try: os.mkdir(ABSTFN) for i in range(depth): os.symlink('/'.join(['%d' % i] * 10), ABSTFN + '/%d' % (i + 1)) os.symlink('.', ABSTFN + '/0') - self.assertEqual(realpath(ABSTFN + '/%d' % depth), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/%d' % depth, **kwargs), ABSTFN) # Test using relative path as well. with os_helper.change_cwd(ABSTFN): @@ -620,11 +743,12 @@ def test_realpath_deep_recursion(self): finally: for i in range(depth + 1): os_helper.unlink(ABSTFN + '/%d' % i) - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_parents(self): + @_parameterize({}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_parents(self, kwargs): # We also need to resolve any symlinks in the parents of a relative # path passed to realpath. E.g.: current working directory is # /usr/doc with 'doc' being a symlink to /usr/share/doc. We call @@ -635,15 +759,17 @@ def test_realpath_resolve_parents(self): os.symlink(ABSTFN + "/y", ABSTFN + "/k") with os_helper.change_cwd(ABSTFN + "/k"): - self.assertEqual(realpath("a"), ABSTFN + "/y/a") + self.assertEqual(realpath("a", **kwargs), + ABSTFN + "/y/a") finally: os_helper.unlink(ABSTFN + "/k") - safe_rmdir(ABSTFN + "/y") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/y") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_before_normalizing(self): + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_before_normalizing(self, kwargs): # Bug #990669: Symbolic links should be resolved before we # normalize the path. E.g.: if we have directories 'a', 'k' and 'y' # in the following hierarchy: @@ -658,20 +784,21 @@ def test_realpath_resolve_before_normalizing(self): os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y") # Absolute path. - self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k") + self.assertEqual(realpath(ABSTFN + "/link-y/..", **kwargs), ABSTFN + "/k") # Relative path. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."), + self.assertEqual(realpath(basename(ABSTFN) + "/link-y/..", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "/link-y") - safe_rmdir(ABSTFN + "/k/y") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k/y") + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_first(self): + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_first(self, kwargs): # Bug #1213894: The first component of the path, if not absolute, # must be resolved too. @@ -681,12 +808,12 @@ def test_realpath_resolve_first(self): os.symlink(ABSTFN, ABSTFN + "link") with os_helper.change_cwd(dirname(ABSTFN)): base = basename(ABSTFN) - self.assertEqual(realpath(base + "link"), ABSTFN) - self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k") + self.assertEqual(realpath(base + "link", **kwargs), ABSTFN) + self.assertEqual(realpath(base + "link/k", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "link") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -700,27 +827,105 @@ def test_realpath_unreadable_symlink(self): self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + @_parameterize({'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_realpath_unreadable_symlink_strict(self, kwargs): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) with self.assertRaises(PermissionError): - realpath(ABSTFN, strict=True) + realpath(ABSTFN, **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo', **kwargs), + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/../foo', **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo/..', **kwargs) finally: os.chmod(ABSTFN, 0o755, follow_symlinks=False) os.unlink(ABSTFN) + @skip_if_ABSTFN_contains_backslash + @os_helper.skip_unless_symlink + def test_realpath_unreadable_directory(self): + try: + os.mkdir(ABSTFN) + os.mkdir(ABSTFN + '/k') + os.chmod(ABSTFN, 0o000) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALL_BUT_LAST), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + + try: + os.stat(ABSTFN) + except PermissionError: + pass + else: + self.skipTest('Cannot block permissions') + + self.assertEqual(realpath(ABSTFN + '/k', strict=False), + ABSTFN + '/k') + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=ALL_BUT_LAST) + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + '/missing', strict=False), + ABSTFN + '/missing') + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=ALL_BUT_LAST) + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=ALLOW_MISSING) + finally: + os.chmod(ABSTFN, 0o755) + os_helper.rmdir(ABSTFN + '/k') + os_helper.rmdir(ABSTFN) + @skip_if_ABSTFN_contains_backslash def test_realpath_nonterminal_file(self): try: with open(ABSTFN, 'w') as f: f.write('test_posixpath wuz ere') self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALL_BUT_LAST), ABSTFN) self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) @@ -732,17 +937,36 @@ def test_realpath_nonterminal_symlink_to_file(self): f.write('test_posixpath wuz ere') os.symlink(ABSTFN + "1", ABSTFN) self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=ALL_BUT_LAST), ABSTFN + "1") self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "1/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -753,17 +977,129 @@ def test_realpath_nonterminal_symlink_to_symlinks_to_file(self): os.symlink(ABSTFN + "2", ABSTFN + "1") os.symlink(ABSTFN + "1", ABSTFN) self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=ALL_BUT_LAST), ABSTFN + "2") self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "2/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=ALL_BUT_LAST) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") + os_helper.unlink(ABSTFN + "2") + + @os_helper.skip_unless_symlink + def test_realpath_mode(self): + self.addCleanup(os_helper.rmdir, ABSTFN) + self.addCleanup(os_helper.rmdir, ABSTFN + "/dir") + self.addCleanup(os_helper.unlink, ABSTFN + "/file") + self.addCleanup(os_helper.unlink, ABSTFN + "/dir/file2") + self.addCleanup(os_helper.unlink, ABSTFN + "/link") + self.addCleanup(os_helper.unlink, ABSTFN + "/link2") + self.addCleanup(os_helper.unlink, ABSTFN + "/broken") + self.addCleanup(os_helper.unlink, ABSTFN + "/cycle") + + os.mkdir(ABSTFN) + os.mkdir(ABSTFN + "/dir") + open(ABSTFN + "/file", "wb").close() + open(ABSTFN + "/dir/file2", "wb").close() + os.symlink("file", ABSTFN + "/link") + os.symlink("dir", ABSTFN + "/link2") + os.symlink("nonexistent", ABSTFN + "/broken") + os.symlink("cycle", ABSTFN + "/cycle") + def check(path, modes, expected, errno=None): + if isinstance(expected, str): + assert errno is None + expected = expected.replace('/', os.sep) + for mode in modes: + with self.subTest(mode=mode): + self.assertEqual(realpath(path, strict=mode).replace('/', os.sep), + ABSTFN.replace('/', os.sep) + expected) + else: + for mode in modes: + with self.subTest(mode=mode): + with self.assertRaises(expected) as cm: + realpath(path, strict=mode) + if errno is not None: + self.assertEqual(cm.exception.errno, errno) + + self.enterContext(os_helper.change_cwd(ABSTFN)) + all_modes = [False, ALLOW_MISSING, ALL_BUT_LAST, True] + check("file", all_modes, "/file") + check("file/", [False], "/file") + check("file/", [ALLOW_MISSING, ALL_BUT_LAST, True], NotADirectoryError) + check("file/file2", [False], "/file/file2") + check("file/file2", [ALLOW_MISSING, ALL_BUT_LAST, True], NotADirectoryError) + check("file/.", [False], "/file") + check("file/.", [ALLOW_MISSING, ALL_BUT_LAST, True], NotADirectoryError) + check("file/../link2", [False], "/dir") + check("file/../link2", [ALLOW_MISSING, ALL_BUT_LAST, True], NotADirectoryError) + + check("dir", all_modes, "/dir") + check("dir/", all_modes, "/dir") + check("dir/file2", all_modes, "/dir/file2") + + check("link", all_modes, "/file") + check("link/", [False], "/file") + check("link/", [ALLOW_MISSING, ALL_BUT_LAST, True], NotADirectoryError) + check("link/file2", [False], "/file/file2") + check("link/file2", [ALLOW_MISSING, ALL_BUT_LAST, True], NotADirectoryError) + check("link/.", [False], "/file") + check("link/.", [ALLOW_MISSING, ALL_BUT_LAST, True], NotADirectoryError) + check("link/../link", [False], "/file") + check("link/../link", [ALLOW_MISSING, ALL_BUT_LAST, True], NotADirectoryError) + + check("link2", all_modes, "/dir") + check("link2/", all_modes, "/dir") + check("link2/file2", all_modes, "/dir/file2") + + check("nonexistent", [False, ALLOW_MISSING, ALL_BUT_LAST], "/nonexistent") + check("nonexistent", [True], FileNotFoundError) + check("nonexistent/", [False, ALLOW_MISSING, ALL_BUT_LAST], "/nonexistent") + check("nonexistent/", [True], FileNotFoundError) + check("nonexistent/file", [False, ALLOW_MISSING], "/nonexistent/file") + check("nonexistent/file", [ALL_BUT_LAST, True], FileNotFoundError) + check("nonexistent/../link", [False, ALLOW_MISSING], "/file") + check("nonexistent/../link", [ALL_BUT_LAST, True], FileNotFoundError) + + check("broken", [False, ALLOW_MISSING, ALL_BUT_LAST], "/nonexistent") + check("broken", [True], FileNotFoundError) + check("broken/", [False, ALLOW_MISSING, ALL_BUT_LAST], "/nonexistent") + check("broken/", [True], FileNotFoundError) + check("broken/file", [False, ALLOW_MISSING], "/nonexistent/file") + check("broken/file", [ALL_BUT_LAST, True], FileNotFoundError) + check("broken/../link", [False, ALLOW_MISSING], "/file") + check("broken/../link", [ALL_BUT_LAST, True], FileNotFoundError) + + check("cycle", [False], "/cycle") + check("cycle", [ALLOW_MISSING, ALL_BUT_LAST, True], OSError, errno.ELOOP) + check("cycle/", [False], "/cycle") + check("cycle/", [ALLOW_MISSING, ALL_BUT_LAST, True], OSError, errno.ELOOP) + check("cycle/file", [False], "/cycle/file") + check("cycle/file", [ALLOW_MISSING, ALL_BUT_LAST, True], OSError, errno.ELOOP) + check("cycle/../link", [False], "/file") + check("cycle/../link", [ALLOW_MISSING, ALL_BUT_LAST, True], OSError, errno.ELOOP) def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") @@ -889,8 +1225,8 @@ class PathLikeTests(unittest.TestCase): path = posixpath def setUp(self): - self.file_name = os_helper.TESTFN - self.file_path = FakePath(os_helper.TESTFN) + self.file_name = TESTFN + self.file_path = FakePath(TESTFN) self.addCleanup(os_helper.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -947,9 +1283,12 @@ def test_path_normpath(self): def test_path_abspath(self): self.assertPathEqual(self.path.abspath) - def test_path_realpath(self): + @_parameterize({}, {'strict': True}, {'strict': ALL_BUT_LAST}, {'strict': ALLOW_MISSING}) + def test_path_realpath(self, kwargs): self.assertPathEqual(self.path.realpath) + self.assertPathEqual(partial(self.path.realpath, **kwargs)) + def test_path_relpath(self): self.assertPathEqual(self.path.relpath) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index f68996f72b1..41c337ade7e 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -10,6 +10,7 @@ import re import types import unittest +from collections.abc import ItemsView, KeysView, Mapping, MappingView, ValuesView from test.support import cpython_only from test.support.import_helper import ensure_lazy_imports @@ -70,6 +71,14 @@ class dict_custom_repr(dict): def __repr__(self): return '*'*len(dict.__repr__(self)) +class mappingview_custom_repr(MappingView): + def __repr__(self): + return '*'*len(MappingView.__repr__(self)) + +class keysview_custom_repr(KeysView): + def __repr__(self): + return '*'*len(KeysView.__repr__(self)) + @dataclasses.dataclass class dataclass1: field1: str @@ -180,10 +189,17 @@ def test_knotted(self): # Messy dict. self.d = {} self.d[0] = self.d[1] = self.d[2] = self.d + self.e = {} + self.v = ValuesView(self.e) + self.m = MappingView(self.e) + self.dv = self.e.values() + self.e["v"] = self.v + self.e["m"] = self.m + self.e["dv"] = self.dv pp = pprint.PrettyPrinter() - for icky in self.a, self.b, self.d, (self.d, self.d): + for icky in self.a, self.b, self.d, (self.d, self.d), self.e, self.v, self.m, self.dv: self.assertTrue(pprint.isrecursive(icky), "expected isrecursive") self.assertFalse(pprint.isreadable(icky), "expected not isreadable") self.assertTrue(pp.isrecursive(icky), "expected isrecursive") @@ -191,10 +207,11 @@ def test_knotted(self): # Break the cycles. self.d.clear() + self.e.clear() del self.a[:] del self.b[:] - for safe in self.a, self.b, self.d, (self.d, self.d): + for safe in self.a, self.b, self.d, (self.d, self.d), self.e, self.v, self.m, self.dv: # module-level convenience functions self.assertFalse(pprint.isrecursive(safe), "expected not isrecursive for %r" % (safe,)) @@ -237,6 +254,8 @@ def test_same_as_repr(self): set(), set2(), set3(), frozenset(), frozenset2(), frozenset3(), {}, dict2(), dict3(), + {}.keys(), {}.values(), {}.items(), + MappingView({}), KeysView({}), ItemsView({}), ValuesView({}), self.assertTrue, pprint, -6, -6, -6-6j, -1.5, "x", b"x", bytearray(b"x"), (3,), [3], {3: 6}, @@ -246,6 +265,9 @@ def test_same_as_repr(self): set({7}), set2({7}), set3({7}), frozenset({8}), frozenset2({8}), frozenset3({8}), dict2({5: 6}), dict3({5: 6}), + {5: 6}.keys(), {5: 6}.values(), {5: 6}.items(), + MappingView({5: 6}), KeysView({5: 6}), + ItemsView({5: 6}), ValuesView({5: 6}), range(10, -11, -1), True, False, None, ..., ): @@ -275,6 +297,12 @@ def test_container_repr_override_called(self): dict_custom_repr(), dict_custom_repr({5: 6}), dict_custom_repr(zip(range(N),range(N))), + mappingview_custom_repr({}), + mappingview_custom_repr({5: 6}), + mappingview_custom_repr(dict(zip(range(N),range(N)))), + keysview_custom_repr({}), + keysview_custom_repr({5: 6}), + keysview_custom_repr(dict(zip(range(N),range(N)))), ): native = repr(cont) expected = '*' * len(native) @@ -302,6 +330,56 @@ def test_basic_line_wrap(self): for type in [dict, dict2]: self.assertEqual(pprint.pformat(type(o)), exp) + o = range(100) + exp = 'dict_keys([%s])' % ',\n '.join(map(str, o)) + keys = dict.fromkeys(o).keys() + self.assertEqual(pprint.pformat(keys), exp) + + o = range(100) + exp = 'dict_values([%s])' % ',\n '.join(map(str, o)) + values = {v: v for v in o}.values() + self.assertEqual(pprint.pformat(values), exp) + + o = range(100) + exp = 'dict_items([%s])' % ',\n '.join("(%s, %s)" % (i, i) for i in o) + items = {v: v for v in o}.items() + self.assertEqual(pprint.pformat(items), exp) + + o = range(100) + exp = 'odict_keys([%s])' % ',\n '.join(map(str, o)) + keys = collections.OrderedDict.fromkeys(o).keys() + self.assertEqual(pprint.pformat(keys), exp) + + o = range(100) + exp = 'odict_values([%s])' % ',\n '.join(map(str, o)) + values = collections.OrderedDict({v: v for v in o}).values() + self.assertEqual(pprint.pformat(values), exp) + + o = range(100) + exp = 'odict_items([%s])' % ',\n '.join("(%s, %s)" % (i, i) for i in o) + items = collections.OrderedDict({v: v for v in o}).items() + self.assertEqual(pprint.pformat(items), exp) + + o = range(100) + exp = 'KeysView({%s})' % (': None,\n '.join(map(str, o)) + ': None') + keys_view = KeysView(dict.fromkeys(o)) + self.assertEqual(pprint.pformat(keys_view), exp) + + o = range(100) + exp = 'ItemsView({%s})' % (': None,\n '.join(map(str, o)) + ': None') + items_view = ItemsView(dict.fromkeys(o)) + self.assertEqual(pprint.pformat(items_view), exp) + + o = range(100) + exp = 'MappingView({%s})' % (': None,\n '.join(map(str, o)) + ': None') + mapping_view = MappingView(dict.fromkeys(o)) + self.assertEqual(pprint.pformat(mapping_view), exp) + + o = range(100) + exp = 'ValuesView({%s})' % (': None,\n '.join(map(str, o)) + ': None') + values_view = ValuesView(dict.fromkeys(o)) + self.assertEqual(pprint.pformat(values_view), exp) + o = range(100) exp = '[%s]' % ',\n '.join(map(str, o)) for type in [list, list2]: @@ -380,7 +458,7 @@ def __new__(cls, celsius_degrees): return super().__new__(Temperature, celsius_degrees) def __repr__(self): kelvin_degrees = self + 273.15 - return f"{kelvin_degrees}°K" + return f"{kelvin_degrees:.2f}°K" self.assertEqual(pprint.pformat(Temperature(1000)), '1273.15°K') def test_sorted_dict(self): @@ -425,6 +503,30 @@ def test_ordered_dict(self): ('a', 6), ('lazy', 7), ('dog', 8)])""") + self.assertEqual(pprint.pformat(d.keys(), sort_dicts=False), +"""\ +odict_keys(['the', + 'quick', + 'brown', + 'fox', + 'jumped', + 'over', + 'a', + 'lazy', + 'dog'])""") + self.assertEqual(pprint.pformat(d.items(), sort_dicts=False), +"""\ +odict_items([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)])""") + self.assertEqual(pprint.pformat(d.values(), sort_dicts=False), + "odict_values([0, 1, 2, 3, 4, 5, 6, 7, 8])") def test_mapping_proxy(self): words = 'the quick brown fox jumped over a lazy dog'.split() @@ -453,6 +555,152 @@ def test_mapping_proxy(self): ('lazy', 7), ('dog', 8)]))""") + def test_dict_views(self): + for dict_class in (dict, collections.OrderedDict, collections.Counter): + empty = dict_class({}) + short = dict_class(dict(zip('edcba', 'edcba'))) + long = dict_class(dict((chr(x), chr(x)) for x in range(90, 64, -1))) + lengths = {"empty": empty, "short": short, "long": long} + prefix = "odict" if dict_class is collections.OrderedDict else "dict" + for name, d in lengths.items(): + with self.subTest(length=name, prefix=prefix): + is_short = len(d) < 6 + joiner = ", " if is_short else ",\n " + k = d.keys() + v = d.values() + i = d.items() + self.assertEqual(pprint.pformat(k, sort_dicts=True), + prefix + "_keys([%s])" % + joiner.join(repr(key) for key in sorted(k))) + self.assertEqual(pprint.pformat(v, sort_dicts=True), + prefix + "_values([%s])" % + joiner.join(repr(val) for val in sorted(v))) + self.assertEqual(pprint.pformat(i, sort_dicts=True), + prefix + "_items([%s])" % + joiner.join(repr(item) for item in sorted(i))) + self.assertEqual(pprint.pformat(k, sort_dicts=False), + prefix + "_keys([%s])" % + joiner.join(repr(key) for key in k)) + self.assertEqual(pprint.pformat(v, sort_dicts=False), + prefix + "_values([%s])" % + joiner.join(repr(val) for val in v)) + self.assertEqual(pprint.pformat(i, sort_dicts=False), + prefix + "_items([%s])" % + joiner.join(repr(item) for item in i)) + + def test_abc_views(self): + empty = {} + short = dict(zip('edcba', 'edcba')) + long = dict((chr(x), chr(x)) for x in range(90, 64, -1)) + lengths = {"empty": empty, "short": short, "long": long} + # Test that a subclass that doesn't replace __repr__ works with different lengths + class MV(MappingView): pass + + for name, d in lengths.items(): + with self.subTest(length=name, name="Views"): + is_short = len(d) < 6 + joiner = ", " if is_short else ",\n " + i = d.items() + s = sorted(i) + joined_items = "({%s})" % joiner.join(["%r: %r" % (k, v) for (k, v) in i]) + sorted_items = "({%s})" % joiner.join(["%r: %r" % (k, v) for (k, v) in s]) + self.assertEqual(pprint.pformat(KeysView(d), sort_dicts=True), + KeysView.__name__ + sorted_items) + self.assertEqual(pprint.pformat(ItemsView(d), sort_dicts=True), + ItemsView.__name__ + sorted_items) + self.assertEqual(pprint.pformat(MappingView(d), sort_dicts=True), + MappingView.__name__ + sorted_items) + self.assertEqual(pprint.pformat(MV(d), sort_dicts=True), + MV.__name__ + sorted_items) + self.assertEqual(pprint.pformat(ValuesView(d), sort_dicts=True), + ValuesView.__name__ + sorted_items) + self.assertEqual(pprint.pformat(KeysView(d), sort_dicts=False), + KeysView.__name__ + joined_items) + self.assertEqual(pprint.pformat(ItemsView(d), sort_dicts=False), + ItemsView.__name__ + joined_items) + self.assertEqual(pprint.pformat(MappingView(d), sort_dicts=False), + MappingView.__name__ + joined_items) + self.assertEqual(pprint.pformat(MV(d), sort_dicts=False), + MV.__name__ + joined_items) + self.assertEqual(pprint.pformat(ValuesView(d), sort_dicts=False), + ValuesView.__name__ + joined_items) + + def test_nested_views(self): + d = {1: MappingView({1: MappingView({1: MappingView({1: 2})})})} + self.assertEqual(repr(d), + "{1: MappingView({1: MappingView({1: MappingView({1: 2})})})}") + self.assertEqual(pprint.pformat(d), + "{1: MappingView({1: MappingView({1: MappingView({1: 2})})})}") + self.assertEqual(pprint.pformat(d, depth=2), + "{1: MappingView({1: {...}})}") + d = {} + d1 = {1: d.values()} + d2 = {1: d1.values()} + d3 = {1: d2.values()} + self.assertEqual(pprint.pformat(d3), + "{1: dict_values([dict_values([dict_values([])])])}") + self.assertEqual(pprint.pformat(d3, depth=2), + "{1: dict_values([{...}])}") + + def test_unorderable_items_views(self): + """Check that views with unorderable items have stable sorting.""" + d = dict((((3+1j), 3), ((1+1j), (1+0j)), (1j, 0j), (500, None), (499, None))) + iv = ItemsView(d) + self.assertEqual(pprint.pformat(iv), + pprint.pformat(iv)) + self.assertTrue(pprint.pformat(iv).endswith(", 499: None, 500: None})"), + pprint.pformat(iv)) + self.assertEqual(pprint.pformat(d.items()), # Won't be equal unless _safe_tuple + pprint.pformat(d.items())) # is used in _safe_repr + self.assertTrue(pprint.pformat(d.items()).endswith(", (499, None), (500, None)])")) + + def test_mapping_view_subclass_no_mapping(self): + class BMV(MappingView): + def __init__(self, d): + super().__init__(d) + self.mapping = self._mapping + del self._mapping + + self.assertRaises(AttributeError, pprint.pformat, BMV({})) + + def test_mapping_subclass_repr(self): + """Test that mapping ABC views use their ._mapping's __repr__.""" + class MyMapping(Mapping): + def __init__(self, keys=None): + self._keys = {} if keys is None else dict.fromkeys(keys) + + def __getitem__(self, item): + return self._keys[item] + + def __len__(self): + return len(self._keys) + + def __iter__(self): + return iter(self._keys) + + def __repr__(self): + return f"{self.__class__.__name__}([{', '.join(map(repr, self._keys.keys()))}])" + + m = MyMapping(["test", 1]) + self.assertEqual(repr(m), "MyMapping(['test', 1])") + short_view_repr = "%s(MyMapping(['test', 1]))" + self.assertEqual(repr(m.keys()), short_view_repr % "KeysView") + self.assertEqual(pprint.pformat(m.items()), short_view_repr % "ItemsView") + self.assertEqual(pprint.pformat(m.keys()), short_view_repr % "KeysView") + self.assertEqual(pprint.pformat(MappingView(m)), short_view_repr % "MappingView") + self.assertEqual(pprint.pformat(m.values()), short_view_repr % "ValuesView") + + alpha = "abcdefghijklmnopqrstuvwxyz" + m = MyMapping(alpha) + alpha_repr = ", ".join(map(repr, list(alpha))) + long_view_repr = "%%s(MyMapping([%s]))" % alpha_repr + self.assertEqual(repr(m), "MyMapping([%s])" % alpha_repr) + self.assertEqual(repr(m.keys()), long_view_repr % "KeysView") + self.assertEqual(pprint.pformat(m.items()), long_view_repr % "ItemsView") + self.assertEqual(pprint.pformat(m.keys()), long_view_repr % "KeysView") + self.assertEqual(pprint.pformat(MappingView(m)), long_view_repr % "MappingView") + self.assertEqual(pprint.pformat(m.values()), long_view_repr % "ValuesView") + def test_empty_simple_namespace(self): ns = types.SimpleNamespace() formatted = pprint.pformat(ns) @@ -768,6 +1016,10 @@ def test_sort_unorderable_values(self): 'frozenset({' + ','.join(map(repr, skeys)) + '})') self.assertEqual(clean(pprint.pformat(dict.fromkeys(keys))), '{' + ','.join('%r:None' % k for k in skeys) + '}') + self.assertEqual(clean(pprint.pformat(dict.fromkeys(keys).keys())), + 'dict_keys([' + ','.join('%r' % k for k in skeys) + '])') + self.assertEqual(clean(pprint.pformat(dict.fromkeys(keys).items())), + 'dict_items([' + ','.join('(%r,None)' % k for k in skeys) + '])') # Issue 10017: TypeError on user-defined types as dict keys. self.assertEqual(pprint.pformat({Unorderable: 0, 1: 0}), @@ -1049,6 +1301,66 @@ def test_chainmap(self): ('a', 6), ('lazy', 7), ('dog', 8)]))""") + self.assertEqual(pprint.pformat(d.keys()), +"""\ +KeysView(ChainMap({'a': 6, + 'brown': 2, + 'dog': 8, + 'fox': 3, + 'jumped': 4, + 'lazy': 7, + 'over': 5, + 'quick': 1, + 'the': 0}, + OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)])))""") + self.assertEqual(pprint.pformat(d.items()), + """\ +ItemsView(ChainMap({'a': 6, + 'brown': 2, + 'dog': 8, + 'fox': 3, + 'jumped': 4, + 'lazy': 7, + 'over': 5, + 'quick': 1, + 'the': 0}, + OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)])))""") + self.assertEqual(pprint.pformat(d.values()), + """\ +ValuesView(ChainMap({'a': 6, + 'brown': 2, + 'dog': 8, + 'fox': 3, + 'jumped': 4, + 'lazy': 7, + 'over': 5, + 'quick': 1, + 'the': 0}, + OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)])))""") def test_deque(self): d = collections.deque() @@ -1096,6 +1408,36 @@ def test_user_dict(self): 'over': 5, 'quick': 1, 'the': 0}""") + self.assertEqual(pprint.pformat(d.keys()), """\ +KeysView({'a': 6, + 'brown': 2, + 'dog': 8, + 'fox': 3, + 'jumped': 4, + 'lazy': 7, + 'over': 5, + 'quick': 1, + 'the': 0})""") + self.assertEqual(pprint.pformat(d.items()), """\ +ItemsView({'a': 6, + 'brown': 2, + 'dog': 8, + 'fox': 3, + 'jumped': 4, + 'lazy': 7, + 'over': 5, + 'quick': 1, + 'the': 0})""") + self.assertEqual(pprint.pformat(d.values()), """\ +ValuesView({'a': 6, + 'brown': 2, + 'dog': 8, + 'fox': 3, + 'jumped': 4, + 'lazy': 7, + 'over': 5, + 'quick': 1, + 'the': 0})""") def test_user_list(self): d = collections.UserList() diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index 0f16b923349..c4b2d7ca05c 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -4,12 +4,16 @@ import pstats import unittest import os +import warnings from difflib import unified_diff from io import StringIO from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd from contextlib import contextmanager, redirect_stdout -import profile +# Suppress deprecation warning for profile module (PEP 799) +with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import profile from test.profilee import testfunc, timer from test.support.script_helper import assert_python_failure, assert_python_ok diff --git a/Lib/test/test_profiling/__init__.py b/Lib/test/test_profiling/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_profiling/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_profiling/__main__.py b/Lib/test/test_profiling/__main__.py new file mode 100644 index 00000000000..40a23a297ec --- /dev/null +++ b/Lib/test/test_profiling/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py new file mode 100644 index 00000000000..24bf3d21c2f --- /dev/null +++ b/Lib/test/test_profiling/test_heatmap.py @@ -0,0 +1,647 @@ +"""Tests for the heatmap collector (profiling.sampling).""" + +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +from profiling.sampling.heatmap_collector import ( + HeatmapCollector, + get_python_path_info, + extract_module_name, +) + +from test.support import captured_stdout, captured_stderr + + +# ============================================================================= +# Unit Tests for Public Helper Functions +# ============================================================================= + +class TestPathInfoFunctions(unittest.TestCase): + """Test public helper functions for path information.""" + + def test_get_python_path_info_returns_dict(self): + """Test that get_python_path_info returns a dictionary with expected keys.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info, dict) + self.assertIn('stdlib', path_info) + self.assertIn('site_packages', path_info) + self.assertIn('sys_path', path_info) + + def test_get_python_path_info_stdlib_is_path_or_none(self): + """Test that stdlib is either a Path object or None.""" + path_info = get_python_path_info() + + if path_info['stdlib'] is not None: + self.assertIsInstance(path_info['stdlib'], Path) + + def test_get_python_path_info_site_packages_is_list(self): + """Test that site_packages is a list.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info['site_packages'], list) + for item in path_info['site_packages']: + self.assertIsInstance(item, Path) + + def test_get_python_path_info_sys_path_is_list(self): + """Test that sys_path is a list of Path objects.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info['sys_path'], list) + for item in path_info['sys_path']: + self.assertIsInstance(item, Path) + + def test_extract_module_name_with_none(self): + """Test extract_module_name with None filename.""" + path_info = get_python_path_info() + module_name, module_type = extract_module_name(None, path_info) + + self.assertEqual(module_name, 'unknown') + self.assertEqual(module_type, 'other') + + def test_extract_module_name_with_empty_string(self): + """Test extract_module_name with empty filename.""" + path_info = get_python_path_info() + module_name, module_type = extract_module_name('', path_info) + + self.assertEqual(module_name, 'unknown') + self.assertEqual(module_type, 'other') + + def test_extract_module_name_with_stdlib_file(self): + """Test extract_module_name with a standard library file.""" + path_info = get_python_path_info() + + # Use os module as a known stdlib file + if path_info['stdlib']: + stdlib_file = str(path_info['stdlib'] / 'os.py') + module_name, module_type = extract_module_name(stdlib_file, path_info) + + self.assertEqual(module_type, 'stdlib') + self.assertIn('os', module_name) + + def test_extract_module_name_with_project_file(self): + """Test extract_module_name with a project file.""" + path_info = get_python_path_info() + + # Create a mock project file path + if path_info['sys_path']: + # Use current directory as project path + project_file = '/some/project/path/mymodule.py' + module_name, module_type = extract_module_name(project_file, path_info) + + # Should classify as 'other' if not in sys.path + self.assertIn(module_type, ['project', 'other']) + + def test_extract_module_name_removes_py_extension(self): + """Test that .py extension is removed from module names.""" + path_info = get_python_path_info() + + # Test with a simple .py file + module_name, module_type = extract_module_name('/path/to/test.py', path_info) + + # Module name should not contain .py + self.assertNotIn('.py', module_name) + + def test_extract_module_name_with_special_files(self): + """Test extract_module_name with special filenames like <string>.""" + path_info = get_python_path_info() + + special_files = ['<string>', '<stdin>', '[eval]'] + for special_file in special_files: + module_name, module_type = extract_module_name(special_file, path_info) + self.assertEqual(module_type, 'other') + + +# ============================================================================= +# Unit Tests for HeatmapCollector Public API +# ============================================================================= + +class TestHeatmapCollectorInit(unittest.TestCase): + """Test HeatmapCollector initialization.""" + + def test_init_creates_empty_data_structures(self): + """Test that __init__ creates empty data structures.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Check that data structures are initialized + self.assertIsInstance(collector.line_samples, dict) + self.assertIsInstance(collector.file_samples, dict) + self.assertIsInstance(collector.line_self_samples, dict) + self.assertIsInstance(collector.file_self_samples, dict) + self.assertIsInstance(collector.call_graph, dict) + self.assertIsInstance(collector.callers_graph, dict) + self.assertIsInstance(collector.function_definitions, dict) + self.assertIsInstance(collector.edge_samples, dict) + + # Check that they're empty + self.assertEqual(len(collector.line_samples), 0) + self.assertEqual(len(collector.file_samples), 0) + self.assertEqual(len(collector.line_self_samples), 0) + self.assertEqual(len(collector.file_self_samples), 0) + + def test_init_sets_total_samples_to_zero(self): + """Test that total samples starts at zero.""" + collector = HeatmapCollector(sample_interval_usec=100) + self.assertEqual(collector._total_samples, 0) + + def test_init_gets_path_info(self): + """Test that path info is retrieved during init.""" + collector = HeatmapCollector(sample_interval_usec=100) + self.assertIsNotNone(collector._path_info) + self.assertIn('stdlib', collector._path_info) + + +class TestHeatmapCollectorSetStats(unittest.TestCase): + """Test HeatmapCollector.set_stats() method.""" + + def test_set_stats_stores_all_parameters(self): + """Test that set_stats stores all provided parameters.""" + collector = HeatmapCollector(sample_interval_usec=100) + + collector.set_stats( + sample_interval_usec=500, + duration_sec=10.5, + sample_rate=99.5, + error_rate=0.5 + ) + + self.assertEqual(collector.stats['sample_interval_usec'], 500) + self.assertEqual(collector.stats['duration_sec'], 10.5) + self.assertEqual(collector.stats['sample_rate'], 99.5) + self.assertEqual(collector.stats['error_rate'], 0.5) + + def test_set_stats_includes_system_info(self): + """Test that set_stats includes Python and platform info.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + self.assertIn('python_version', collector.stats) + self.assertIn('python_implementation', collector.stats) + self.assertIn('platform', collector.stats) + + def test_set_stats_accepts_kwargs(self): + """Test that set_stats accepts additional kwargs.""" + collector = HeatmapCollector(sample_interval_usec=100) + + collector.set_stats( + sample_interval_usec=100, + duration_sec=1.0, + sample_rate=100.0, + custom_key='custom_value', + another_key=42 + ) + + self.assertEqual(collector.stats['custom_key'], 'custom_value') + self.assertEqual(collector.stats['another_key'], 42) + + def test_set_stats_with_none_error_rate(self): + """Test set_stats with error_rate=None.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + self.assertIn('error_rate', collector.stats) + self.assertIsNone(collector.stats['error_rate']) + + +class TestHeatmapCollectorProcessFrames(unittest.TestCase): + """Test HeatmapCollector.process_frames() method.""" + + def test_process_frames_increments_total_samples(self): + """Test that process_frames increments total samples count.""" + collector = HeatmapCollector(sample_interval_usec=100) + + initial_count = collector._total_samples + frames = [('file.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + self.assertEqual(collector._total_samples, initial_count + 1) + + def test_process_frames_records_line_samples(self): + """Test that process_frames records line samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 5, 'test_func')] + collector.process_frames(frames, thread_id=1) + + # Check that line was recorded + self.assertIn(('test.py', 5), collector.line_samples) + self.assertEqual(collector.line_samples[('test.py', 5)], 1) + + def test_process_frames_records_multiple_lines_in_stack(self): + """Test that process_frames records all lines in a stack.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('file1.py', 10, 'func1'), + ('file2.py', 20, 'func2'), + ('file3.py', 30, 'func3') + ] + collector.process_frames(frames, thread_id=1) + + # All frames should be recorded + self.assertIn(('file1.py', 10), collector.line_samples) + self.assertIn(('file2.py', 20), collector.line_samples) + self.assertIn(('file3.py', 30), collector.line_samples) + + def test_process_frames_distinguishes_self_samples(self): + """Test that process_frames distinguishes self (leaf) samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('leaf.py', 5, 'leaf_func'), # This is the leaf (top of stack) + ('caller.py', 10, 'caller_func') + ] + collector.process_frames(frames, thread_id=1) + + # Leaf should have self sample + self.assertIn(('leaf.py', 5), collector.line_self_samples) + self.assertEqual(collector.line_self_samples[('leaf.py', 5)], 1) + + # Caller should NOT have self sample + self.assertNotIn(('caller.py', 10), collector.line_self_samples) + + def test_process_frames_accumulates_samples(self): + """Test that multiple calls accumulate samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('file.py', 10, 'func')] + + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + + self.assertEqual(collector.line_samples[('file.py', 10)], 3) + self.assertEqual(collector._total_samples, 3) + + def test_process_frames_ignores_invalid_frames(self): + """Test that process_frames ignores invalid frames.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # These should be ignored + invalid_frames = [ + ('<string>', 1, 'test'), + ('[eval]', 1, 'test'), + ('', 1, 'test'), + (None, 1, 'test'), + ('__init__', 0, 'test'), # Special invalid frame + ] + + for frame in invalid_frames: + collector.process_frames([frame], thread_id=1) + + # Should not record these invalid frames + for frame in invalid_frames: + if frame[0]: + self.assertNotIn((frame[0], frame[1]), collector.line_samples) + + def test_process_frames_builds_call_graph(self): + """Test that process_frames builds call graph relationships.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', 5, 'callee_func'), + ('caller.py', 10, 'caller_func') + ] + collector.process_frames(frames, thread_id=1) + + # Check that call relationship was recorded + caller_key = ('caller.py', 10) + self.assertIn(caller_key, collector.call_graph) + + # Check callers graph + callee_key = ('callee.py', 5) + self.assertIn(callee_key, collector.callers_graph) + + def test_process_frames_records_function_definitions(self): + """Test that process_frames records function definition locations.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('module.py', 42, 'my_function')] + collector.process_frames(frames, thread_id=1) + + self.assertIn(('module.py', 'my_function'), collector.function_definitions) + self.assertEqual(collector.function_definitions[('module.py', 'my_function')], 42) + + def test_process_frames_tracks_edge_samples(self): + """Test that process_frames tracks edge sample counts.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', 5, 'callee'), + ('caller.py', 10, 'caller') + ] + + # Process same call stack multiple times + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + + # Check that edge count is tracked + self.assertGreater(len(collector.edge_samples), 0) + + def test_process_frames_handles_empty_frames(self): + """Test that process_frames handles empty frame list.""" + collector = HeatmapCollector(sample_interval_usec=100) + + initial_count = collector._total_samples + collector.process_frames([], thread_id=1) + + # Should still increment total samples + self.assertEqual(collector._total_samples, initial_count + 1) + + def test_process_frames_with_file_samples_dict(self): + """Test that file_samples dict is properly populated.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + self.assertIn('test.py', collector.file_samples) + self.assertIn(10, collector.file_samples['test.py']) + self.assertEqual(collector.file_samples['test.py'][10], 1) + + +class TestHeatmapCollectorExport(unittest.TestCase): + """Test HeatmapCollector.export() method.""" + + def setUp(self): + """Set up test directory.""" + self.test_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.test_dir) + + def test_export_creates_output_directory(self): + """Test that export creates the output directory.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Add some data + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + self.assertTrue(os.path.exists(output_path)) + self.assertTrue(os.path.isdir(output_path)) + + def test_export_creates_index_html(self): + """Test that export creates index.html.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + index_path = os.path.join(output_path, 'index.html') + self.assertTrue(os.path.exists(index_path)) + + def test_export_creates_file_htmls(self): + """Test that export creates individual file HTMLs.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Check for file_XXXX.html files + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + self.assertGreater(len(html_files), 0) + + def test_export_with_empty_data(self): + """Test export with no data collected.""" + collector = HeatmapCollector(sample_interval_usec=100) + + output_path = os.path.join(self.test_dir, 'empty_output') + + # Should handle empty data gracefully + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + def test_export_handles_html_suffix(self): + """Test that export handles .html suffix in output path.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + # Path with .html suffix should be stripped + output_path = os.path.join(self.test_dir, 'output.html') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Should create directory without .html + expected_dir = os.path.join(self.test_dir, 'output') + self.assertTrue(os.path.exists(expected_dir)) + + def test_export_with_multiple_files(self): + """Test export with multiple files.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Add samples for multiple files + collector.process_frames([('file1.py', 10, 'func1')], thread_id=1) + collector.process_frames([('file2.py', 20, 'func2')], thread_id=1) + collector.process_frames([('file3.py', 30, 'func3')], thread_id=1) + + output_path = os.path.join(self.test_dir, 'multi_file') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Should create HTML for each file + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + self.assertGreaterEqual(len(html_files), 3) + + def test_export_index_contains_file_references(self): + """Test that index.html contains references to profiled files.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + frames = [('mytest.py', 10, 'my_func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'test_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + index_path = os.path.join(output_path, 'index.html') + with open(index_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Should contain reference to the file + self.assertIn('mytest', content) + + def test_export_file_html_has_line_numbers(self): + """Test that exported file HTML contains line numbers.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Create a temporary Python file + temp_file = os.path.join(self.test_dir, 'temp_source.py') + with open(temp_file, 'w') as f: + f.write('def test():\n pass\n') + + frames = [(temp_file, 1, 'test')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'line_test') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Find the generated file HTML + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + + if html_files: + with open(os.path.join(output_path, html_files[0]), 'r', encoding='utf-8') as f: + content = f.read() + + # Should have line-related content + self.assertIn('line-', content) + + +class MockFrameInfo: + """Mock FrameInfo for testing since the real one isn't accessible.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +class MockThreadInfo: + """Mock ThreadInfo for testing since the real one isn't accessible.""" + + def __init__(self, thread_id, frame_info): + self.thread_id = thread_id + self.frame_info = frame_info + + def __repr__(self): + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info})" + + +class MockInterpreterInfo: + """Mock InterpreterInfo for testing since the real one isn't accessible.""" + + def __init__(self, interpreter_id, threads): + self.interpreter_id = interpreter_id + self.threads = threads + + def __repr__(self): + return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" + + +class TestHeatmapCollector(unittest.TestCase): + """Tests for HeatmapCollector functionality.""" + + def test_heatmap_collector_basic(self): + """Test basic HeatmapCollector functionality.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Test empty state + self.assertEqual(len(collector.file_samples), 0) + self.assertEqual(len(collector.line_samples), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo( + 1, + [("file.py", 10, "func1"), ("file.py", 20, "func2")], + )] + ) + ] + collector.collect(test_frames) + + # Should have recorded samples for the file + self.assertGreater(len(collector.line_samples), 0) + self.assertIn("file.py", collector.file_samples) + + # Check that line samples were recorded + file_data = collector.file_samples["file.py"] + self.assertGreater(len(file_data), 0) + + def test_heatmap_collector_export(self): + """Test heatmap HTML export functionality.""" + heatmap_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, heatmap_dir) + + collector = HeatmapCollector(sample_interval_usec=100) + + # Create test data with multiple files + test_frames1 = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])]) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + # Export heatmap + with (captured_stdout(), captured_stderr()): + collector.export(heatmap_dir) + + # Verify index.html was created + index_path = os.path.join(heatmap_dir, "index.html") + self.assertTrue(os.path.exists(index_path)) + self.assertGreater(os.path.getsize(index_path), 0) + + # Check index contains HTML content + with open(index_path, "r", encoding="utf-8") as f: + content = f.read() + + # Should be valid HTML + self.assertIn("<!doctype html>", content.lower()) + self.assertIn("<html", content) + self.assertIn("Tachyon Profiler", content) + + # Should contain file references + self.assertIn("file.py", content) + self.assertIn("other.py", content) + + # Verify individual file HTMLs were created + file_htmls = [f for f in os.listdir(heatmap_dir) if f.startswith("file_") and f.endswith(".html")] + self.assertGreater(len(file_htmls), 0) + + # Check one of the file HTMLs + file_html_path = os.path.join(heatmap_dir, file_htmls[0]) + with open(file_html_path, "r", encoding="utf-8") as f: + file_content = f.read() + + # Should contain heatmap styling and JavaScript + self.assertIn("line-sample", file_content) + self.assertIn("nav-btn", file_content) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/__init__.py b/Lib/test/test_profiling/test_sampling_profiler/__init__.py new file mode 100644 index 00000000000..616ae5b49f0 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/__init__.py @@ -0,0 +1,9 @@ +"""Tests for the sampling profiler (profiling.sampling).""" + +import os +from test.support import load_package_tests + + +def load_tests(*args): + """Load all tests from this subpackage.""" + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_profiling/test_sampling_profiler/_live_collector_helpers.py b/Lib/test/test_profiling/test_sampling_profiler/_live_collector_helpers.py new file mode 100644 index 00000000000..4bb6877f16f --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/_live_collector_helpers.py @@ -0,0 +1,41 @@ +"""Common test helpers and mocks for live collector tests.""" + +from profiling.sampling.constants import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, +) + + +class MockFrameInfo: + """Mock FrameInfo for testing.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +class MockThreadInfo: + """Mock ThreadInfo for testing.""" + + def __init__(self, thread_id, frame_info, status=THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU): + self.thread_id = thread_id + self.frame_info = frame_info + self.status = status + + def __repr__(self): + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" + + +class MockInterpreterInfo: + """Mock InterpreterInfo for testing.""" + + def __init__(self, interpreter_id, threads): + self.interpreter_id = interpreter_id + self.threads = threads + + def __repr__(self): + return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" diff --git a/Lib/test/test_profiling/test_sampling_profiler/helpers.py b/Lib/test/test_profiling/test_sampling_profiler/helpers.py new file mode 100644 index 00000000000..0e32d8dd9ea --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/helpers.py @@ -0,0 +1,176 @@ +"""Helper utilities for sampling profiler tests.""" + +import contextlib +import socket +import subprocess +import sys +import unittest +from collections import namedtuple + +from test.support import SHORT_TIMEOUT +from test.support.socket_helper import find_unused_port +from test.support.os_helper import unlink + + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED # noqa: F401 + import _remote_debugging # noqa: F401 +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) +else: + import profiling.sampling # noqa: F401 + from profiling.sampling.sample import SampleProfiler # noqa: F401 + + +skip_if_not_supported = unittest.skipIf( + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), + "Test only runs on Linux, Windows and MacOS", +) + +SubprocessInfo = namedtuple("SubprocessInfo", ["process", "socket"]) + + +def _wait_for_signal(sock, expected_signals, timeout=SHORT_TIMEOUT): + """ + Wait for expected signal(s) from a socket with proper timeout and EOF handling. + + Args: + sock: Connected socket to read from + expected_signals: Single bytes object or list of bytes objects to wait for + timeout: Socket timeout in seconds + + Returns: + bytes: Complete accumulated response buffer + + Raises: + RuntimeError: If connection closed before signal received or timeout + """ + if isinstance(expected_signals, bytes): + expected_signals = [expected_signals] + + sock.settimeout(timeout) + buffer = b"" + + while True: + # Check if all expected signals are in buffer + if all(sig in buffer for sig in expected_signals): + return buffer + + try: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError( + f"Connection closed before receiving expected signals. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) + buffer += chunk + except socket.timeout: + raise RuntimeError( + f"Timeout waiting for signals. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) from None + except OSError as e: + raise RuntimeError( + f"Socket error while waiting for signals: {e}. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) from None + + +def _cleanup_sockets(*sockets): + """Safely close multiple sockets, ignoring errors.""" + for sock in sockets: + if sock is not None: + try: + sock.close() + except OSError: + pass + + +def _cleanup_process(proc, timeout=SHORT_TIMEOUT): + """Terminate a process gracefully, escalating to kill if needed.""" + if proc.poll() is not None: + return + proc.terminate() + try: + proc.wait(timeout=timeout) + return + except subprocess.TimeoutExpired: + pass + proc.kill() + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + pass # Process refuses to die, nothing more we can do + + +@contextlib.contextmanager +def test_subprocess(script, wait_for_working=False): + """Context manager to create a test subprocess with socket synchronization. + + Args: + script: Python code to execute in the subprocess. If wait_for_working + is True, script should send b"working" after starting work. + wait_for_working: If True, wait for both "ready" and "working" signals. + Default False for backward compatibility. + + Yields: + SubprocessInfo: Named tuple with process and socket objects + """ + # Find an unused port for socket communication + port = find_unused_port() + + # Inject socket connection code at the beginning of the script + socket_code = f""" +import socket +_test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +_test_sock.connect(('localhost', {port})) +_test_sock.sendall(b"ready") +""" + + # Combine socket code with user script + full_script = socket_code + script + + # Create server socket to wait for process to be ready + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + proc = subprocess.Popen( + [sys.executable, "-c", full_script], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + client_socket = None + try: + # Wait for process to connect and send ready signal + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + # Wait for ready signal, and optionally working signal + if wait_for_working: + _wait_for_signal(client_socket, [b"ready", b"working"]) + else: + _wait_for_signal(client_socket, b"ready") + + yield SubprocessInfo(proc, client_socket) + finally: + _cleanup_sockets(client_socket, server_socket) + _cleanup_process(proc) + + +def close_and_unlink(file): + """Close a file and unlink it from the filesystem.""" + file.close() + unlink(file.name) diff --git a/Lib/test/test_profiling/test_sampling_profiler/mocks.py b/Lib/test/test_profiling/test_sampling_profiler/mocks.py new file mode 100644 index 00000000000..7083362c771 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/mocks.py @@ -0,0 +1,73 @@ +"""Mock classes for sampling profiler tests.""" + + +class MockFrameInfo: + """Mock FrameInfo for testing since the real one isn't accessible.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +class MockThreadInfo: + """Mock ThreadInfo for testing since the real one isn't accessible.""" + + def __init__( + self, thread_id, frame_info, status=0 + ): # Default to THREAD_STATE_RUNNING (0) + self.thread_id = thread_id + self.frame_info = frame_info + self.status = status + + def __repr__(self): + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" + + +class MockInterpreterInfo: + """Mock InterpreterInfo for testing since the real one isn't accessible.""" + + def __init__(self, interpreter_id, threads): + self.interpreter_id = interpreter_id + self.threads = threads + + def __repr__(self): + return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" + + +class MockCoroInfo: + """Mock CoroInfo for testing async tasks.""" + + def __init__(self, task_name, call_stack): + self.task_name = task_name # In reality, this is the parent task ID + self.call_stack = call_stack + + def __repr__(self): + return f"MockCoroInfo(task_name={self.task_name}, call_stack={self.call_stack})" + + +class MockTaskInfo: + """Mock TaskInfo for testing async tasks.""" + + def __init__(self, task_id, task_name, coroutine_stack, awaited_by=None): + self.task_id = task_id + self.task_name = task_name + self.coroutine_stack = coroutine_stack # List of CoroInfo objects + self.awaited_by = awaited_by or [] # List of CoroInfo objects (parents) + + def __repr__(self): + return f"MockTaskInfo(task_id={self.task_id}, task_name={self.task_name})" + + +class MockAwaitedInfo: + """Mock AwaitedInfo for testing async tasks.""" + + def __init__(self, thread_id, awaited_by): + self.thread_id = thread_id + self.awaited_by = awaited_by # List of TaskInfo objects + + def __repr__(self): + return f"MockAwaitedInfo(thread_id={self.thread_id}, awaited_by={len(self.awaited_by)} tasks)" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py new file mode 100644 index 00000000000..843fb3b7416 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py @@ -0,0 +1,258 @@ +"""Tests for advanced sampling profiler features (GC tracking, native frames, ProcessPoolExecutor support).""" + +import io +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 + import profiling.sampling + import profiling.sampling.sample +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import ( + SHORT_TIMEOUT, + SuppressCrashReport, + os_helper, + requires_subprocess, + script_helper, +) + +from .helpers import close_and_unlink, skip_if_not_supported, test_subprocess + + +@requires_subprocess() +@skip_if_not_supported +class TestGCFrameTracking(unittest.TestCase): + """Tests for GC frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with GC frames and CPU-intensive work.""" + cls.gc_test_script = ''' +import gc + +class ExpensiveGarbage: + def __init__(self): + self.cycle = self + + def __del__(self): + result = 0 + for i in range(100000): + result += i * i + if i % 1000 == 0: + result = result % 1000000 + +_test_sock.sendall(b"working") +while True: + ExpensiveGarbage() + gc.collect() +''' + + def test_gc_frames_enabled(self): + """Test that GC frames appear when gc tracking is enabled.""" + with ( + test_subprocess(self.gc_test_script, wait_for_working=True) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + from profiling.sampling.pstats_collector import PstatsCollector + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=1, + native=False, + gc=True, + ) + collector.print_stats(show_summary=False) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should be present + self.assertIn("<GC>", output) + + def test_gc_frames_disabled(self): + """Test that GC frames do not appear when gc tracking is disabled.""" + with ( + test_subprocess(self.gc_test_script, wait_for_working=True) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + from profiling.sampling.pstats_collector import PstatsCollector + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=1, + native=False, + gc=False, + ) + collector.print_stats(show_summary=False) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should NOT be present + self.assertNotIn("<GC>", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestNativeFrameTracking(unittest.TestCase): + """Tests for native frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with native frames and CPU-intensive work.""" + cls.native_test_script = """ +import operator + +def inner(): + for _ in range(1_000_0000): + pass + +_test_sock.sendall(b"working") +while True: + operator.call(inner) +""" + + def test_native_frames_enabled(self): + """Test that native frames appear when native tracking is enabled.""" + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with test_subprocess(self.native_test_script, wait_for_working=True) as subproc: + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + from profiling.sampling.stack_collector import CollapsedStackCollector + collector = CollapsedStackCollector(1000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=1, + native=True, + ) + collector.export(collapsed_file.name) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + stacks = [line.rsplit(" ", 1)[0] for line in lines] + + # Most samples should have native code in the middle of the stack: + self.assertTrue(any(";<native>;" in stack for stack in stacks)) + + # No samples should have native code at the top of the stack: + self.assertFalse(any(stack.endswith(";<native>") for stack in stacks)) + + def test_native_frames_disabled(self): + """Test that native frames do not appear when native tracking is disabled.""" + with ( + test_subprocess(self.native_test_script, wait_for_working=True) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + from profiling.sampling.pstats_collector import PstatsCollector + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=1, + ) + collector.print_stats(show_summary=False) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + output = captured_output.getvalue() + # Native frames should NOT be present: + self.assertNotIn("<native>", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestProcessPoolExecutorSupport(unittest.TestCase): + """ + Test that ProcessPoolExecutor works correctly with profiling.sampling. + """ + + def test_process_pool_executor_pickle(self): + # gh-140729: test use ProcessPoolExecutor.map() can sampling + test_script = """ +import concurrent.futures + +def worker(x): + return x * 2 + +if __name__ == "__main__": + with concurrent.futures.ProcessPoolExecutor() as executor: + results = list(executor.map(worker, [1, 2, 3])) + print(f"Results: {results}") +""" + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, "test_process_pool_executor_pickle", test_script + ) + with SuppressCrashReport(): + with script_helper.spawn_python( + "-m", + "profiling.sampling", + "run", + "-d", + "5", + "-i", + "100000", + script, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as proc: + try: + stdout, stderr = proc.communicate( + timeout=SHORT_TIMEOUT + ) + except subprocess.TimeoutExpired: + proc.kill() + stdout, stderr = proc.communicate() + + if "Permission Error" in stderr: + self.skipTest("Insufficient permissions for remote profiling") + + self.assertIn("Results: [2, 4, 6]", stdout) + self.assertNotIn("Can't pickle", stderr) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_async.py b/Lib/test/test_profiling/test_sampling_profiler/test_async.py new file mode 100644 index 00000000000..d8ca86c996b --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_async.py @@ -0,0 +1,799 @@ +"""Tests for async stack reconstruction in the sampling profiler. + +Each test covers a distinct algorithm path or edge case: +1. Graph building: _build_task_graph() +2. Leaf identification: _find_leaf_tasks() +3. Stack traversal: _build_linear_stacks() with BFS +""" + +import unittest + +try: + import _remote_debugging # noqa: F401 + from profiling.sampling.pstats_collector import PstatsCollector +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from .mocks import MockFrameInfo, MockCoroInfo, MockTaskInfo, MockAwaitedInfo + + +class TestAsyncStackReconstruction(unittest.TestCase): + """Test async task tree linear stack reconstruction algorithm.""" + + def test_empty_input(self): + """Test _build_task_graph with empty awaited_info_list.""" + collector = PstatsCollector(sample_interval_usec=1000) + stacks = list(collector._iter_async_frames([])) + self.assertEqual(len(stacks), 0) + + def test_single_root_task(self): + """Test _find_leaf_tasks: root task with no parents is its own leaf.""" + collector = PstatsCollector(sample_interval_usec=1000) + + root = MockTaskInfo( + task_id=123, + task_name="Task-1", + coroutine_stack=[ + MockCoroInfo( + task_name="Task-1", + call_stack=[MockFrameInfo("main.py", 10, "main")] + ) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=100, awaited_by=[root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Single root is both leaf and root + self.assertEqual(len(stacks), 1) + frames, thread_id, leaf_id = stacks[0] + self.assertEqual(leaf_id, 123) + self.assertEqual(thread_id, 100) + + def test_parent_child_chain(self): + """Test _build_linear_stacks: BFS follows parent links from leaf to root. + + Task graph: + + Parent (id=1) + | + Child (id=2) + """ + collector = PstatsCollector(sample_interval_usec=1000) + + child = MockTaskInfo( + task_id=2, + task_name="Child", + coroutine_stack=[ + MockCoroInfo(task_name="Child", call_stack=[MockFrameInfo("c.py", 5, "child_fn")]) + ], + awaited_by=[ + MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("p.py", 10, "parent_await")]) + ] + ) + + parent = MockTaskInfo( + task_id=1, + task_name="Parent", + coroutine_stack=[ + MockCoroInfo(task_name="Parent", call_stack=[MockFrameInfo("p.py", 15, "parent_fn")]) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=200, awaited_by=[child, parent])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Leaf is child, traverses to parent + self.assertEqual(len(stacks), 1) + frames, thread_id, leaf_id = stacks[0] + self.assertEqual(leaf_id, 2) + + # Verify both child and parent frames present + func_names = [f.funcname for f in frames] + self.assertIn("child_fn", func_names) + self.assertIn("parent_fn", func_names) + + def test_multiple_leaf_tasks(self): + """Test _find_leaf_tasks: identifies multiple leaves correctly. + + Task graph (fan-out from root): + + Root (id=1) + / \ + Leaf1 (id=10) Leaf2 (id=20) + + Expected: 2 stacks (one for each leaf). + """ + collector = PstatsCollector(sample_interval_usec=1000) + leaf1 = MockTaskInfo( + task_id=10, + task_name="Leaf1", + coroutine_stack=[MockCoroInfo(task_name="Leaf1", call_stack=[MockFrameInfo("l1.py", 1, "f1")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("r.py", 5, "root")])] + ) + + leaf2 = MockTaskInfo( + task_id=20, + task_name="Leaf2", + coroutine_stack=[MockCoroInfo(task_name="Leaf2", call_stack=[MockFrameInfo("l2.py", 2, "f2")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("r.py", 5, "root")])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="Root", + coroutine_stack=[MockCoroInfo(task_name="Root", call_stack=[MockFrameInfo("r.py", 10, "main")])], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=300, awaited_by=[leaf1, leaf2, root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Two leaves = two stacks + self.assertEqual(len(stacks), 2) + leaf_ids = {leaf_id for _, _, leaf_id in stacks} + self.assertEqual(leaf_ids, {10, 20}) + + def test_cycle_detection(self): + """Test _build_linear_stacks: cycle detection prevents infinite loops. + + Task graph (cyclic dependency): + + A (id=1) <---> B (id=2) + + Neither task is a leaf (both have parents), so no stacks are produced. + """ + collector = PstatsCollector(sample_interval_usec=1000) + task_a = MockTaskInfo( + task_id=1, + task_name="A", + coroutine_stack=[MockCoroInfo(task_name="A", call_stack=[MockFrameInfo("a.py", 1, "a")])], + awaited_by=[MockCoroInfo(task_name=2, call_stack=[MockFrameInfo("b.py", 5, "b")])] + ) + + task_b = MockTaskInfo( + task_id=2, + task_name="B", + coroutine_stack=[MockCoroInfo(task_name="B", call_stack=[MockFrameInfo("b.py", 10, "b")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("a.py", 15, "a")])] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=400, awaited_by=[task_a, task_b])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # No leaves (both have parents), should return empty + self.assertEqual(len(stacks), 0) + + def test_orphaned_parent_reference(self): + """Test _build_linear_stacks: handles parent ID not in task_map.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Task references non-existent parent + orphan = MockTaskInfo( + task_id=5, + task_name="Orphan", + coroutine_stack=[MockCoroInfo(task_name="Orphan", call_stack=[MockFrameInfo("o.py", 1, "orphan")])], + awaited_by=[MockCoroInfo(task_name=999, call_stack=[])] # 999 doesn't exist + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=500, awaited_by=[orphan])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Stops at missing parent, yields what it has + self.assertEqual(len(stacks), 1) + frames, _, leaf_id = stacks[0] + self.assertEqual(leaf_id, 5) + + def test_multiple_coroutines_per_task(self): + """Test _build_linear_stacks: collects frames from all coroutines in task.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Task with multiple coroutines (e.g., nested async generators) + task = MockTaskInfo( + task_id=7, + task_name="Multi", + coroutine_stack=[ + MockCoroInfo(task_name="Multi", call_stack=[MockFrameInfo("g.py", 5, "gen1")]), + MockCoroInfo(task_name="Multi", call_stack=[MockFrameInfo("g.py", 10, "gen2")]), + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=600, awaited_by=[task])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + self.assertEqual(len(stacks), 1) + frames, _, _ = stacks[0] + + # Both coroutine frames should be present + func_names = [f.funcname for f in frames] + self.assertIn("gen1", func_names) + self.assertIn("gen2", func_names) + + def test_multiple_threads(self): + """Test _build_task_graph: handles multiple AwaitedInfo (different threads).""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Two threads with separate task trees + thread1_task = MockTaskInfo( + task_id=100, + task_name="T1", + coroutine_stack=[MockCoroInfo(task_name="T1", call_stack=[MockFrameInfo("t1.py", 1, "t1")])], + awaited_by=[] + ) + + thread2_task = MockTaskInfo( + task_id=200, + task_name="T2", + coroutine_stack=[MockCoroInfo(task_name="T2", call_stack=[MockFrameInfo("t2.py", 1, "t2")])], + awaited_by=[] + ) + + awaited_info_list = [ + MockAwaitedInfo(thread_id=1, awaited_by=[thread1_task]), + MockAwaitedInfo(thread_id=2, awaited_by=[thread2_task]), + ] + + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Two threads = two stacks + self.assertEqual(len(stacks), 2) + + # Verify thread IDs preserved + thread_ids = {thread_id for _, thread_id, _ in stacks} + self.assertEqual(thread_ids, {1, 2}) + + def test_collect_public_interface(self): + """Test collect() method correctly routes to async frame processing.""" + collector = PstatsCollector(sample_interval_usec=1000) + + child = MockTaskInfo( + task_id=50, + task_name="Child", + coroutine_stack=[MockCoroInfo(task_name="Child", call_stack=[MockFrameInfo("c.py", 1, "child")])], + awaited_by=[MockCoroInfo(task_name=51, call_stack=[])] + ) + + parent = MockTaskInfo( + task_id=51, + task_name="Parent", + coroutine_stack=[MockCoroInfo(task_name="Parent", call_stack=[MockFrameInfo("p.py", 1, "parent")])], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=999, awaited_by=[child, parent])] + + # Public interface: collect() + collector.collect(awaited_info_list) + + # Verify stats collected + self.assertGreater(len(collector.result), 0) + func_names = [loc[2] for loc in collector.result.keys()] + self.assertIn("child", func_names) + self.assertIn("parent", func_names) + + def test_diamond_pattern_multiple_parents(self): + """Test _build_linear_stacks: task with 2+ parents picks one deterministically. + + CRITICAL: Tests that when a task has multiple parents, we pick one parent + deterministically (sorted, first one) and annotate the task name with parent count. + """ + collector = PstatsCollector(sample_interval_usec=1000) + + # Diamond pattern: Root spawns A and B, both await Child + # + # Root (id=1) + # / \ + # A (id=2) B (id=3) + # \ / + # Child (id=4) + # + + child = MockTaskInfo( + task_id=4, + task_name="Child", + coroutine_stack=[MockCoroInfo(task_name="Child", call_stack=[MockFrameInfo("c.py", 1, "child_work")])], + awaited_by=[ + MockCoroInfo(task_name=2, call_stack=[MockFrameInfo("a.py", 5, "a_await")]), # Parent A + MockCoroInfo(task_name=3, call_stack=[MockFrameInfo("b.py", 5, "b_await")]), # Parent B + ] + ) + + parent_a = MockTaskInfo( + task_id=2, + task_name="A", + coroutine_stack=[MockCoroInfo(task_name="A", call_stack=[MockFrameInfo("a.py", 10, "a_work")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("root.py", 5, "root_spawn")])] + ) + + parent_b = MockTaskInfo( + task_id=3, + task_name="B", + coroutine_stack=[MockCoroInfo(task_name="B", call_stack=[MockFrameInfo("b.py", 10, "b_work")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("root.py", 5, "root_spawn")])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="Root", + coroutine_stack=[MockCoroInfo(task_name="Root", call_stack=[MockFrameInfo("root.py", 20, "main")])], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=777, awaited_by=[child, parent_a, parent_b, root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Should get 1 stack: Child->A->Root (picks parent with lowest ID: 2) + self.assertEqual(len(stacks), 1, "Diamond should create only 1 path, picking first sorted parent") + + # Verify the single stack + frames, thread_id, leaf_id = stacks[0] + self.assertEqual(leaf_id, 4) + self.assertEqual(thread_id, 777) + + func_names = [f.funcname for f in frames] + # Stack should contain child, parent A (id=2, first when sorted), and root + self.assertIn("child_work", func_names) + self.assertIn("a_work", func_names, "Should use parent A (id=2, first when sorted)") + self.assertNotIn("b_work", func_names, "Should not include parent B") + self.assertIn("main", func_names) + + # Verify Child task is annotated with parent count + self.assertIn("Child (2 parents)", func_names, "Child task should be annotated with parent count") + + def test_empty_coroutine_stack(self): + """Test _build_linear_stacks: handles empty coroutine_stack (line 109 condition false).""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Task with no coroutine_stack + task = MockTaskInfo( + task_id=99, + task_name="EmptyStack", + coroutine_stack=[], # Empty! + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=111, awaited_by=[task])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + self.assertEqual(len(stacks), 1) + frames, _, _ = stacks[0] + + # Should only have task marker, no function frames + func_names = [f.funcname for f in frames] + self.assertEqual(len(func_names), 1, "Should only have task marker") + self.assertIn("EmptyStack", func_names) + + def test_orphaned_parent_with_no_frames_collected(self): + """Test _build_linear_stacks: orphaned parent at start with empty frames (line 94-96).""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Leaf that doesn't exist in task_map (should not happen normally, but test robustness) + # We'll create a scenario where the leaf_id is present but empty + + # Task references non-existent parent, and has no coroutine_stack + orphan = MockTaskInfo( + task_id=88, + task_name="Orphan", + coroutine_stack=[], # No frames + awaited_by=[MockCoroInfo(task_name=999, call_stack=[])] # Parent doesn't exist + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=222, awaited_by=[orphan])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Should yield because we have the task marker even with no function frames + self.assertEqual(len(stacks), 1) + frames, _, leaf_id = stacks[0] + self.assertEqual(leaf_id, 88) + # Has task marker but no function frames + self.assertGreater(len(frames), 0, "Should have at least task marker") + + def test_frame_ordering(self): + """Test _build_linear_stacks: frames are collected in correct order (leaf->root). + + Task graph (3-level chain): + + Root (id=1) <- root_bottom, root_top + | + Middle (id=2) <- mid_bottom, mid_top + | + Leaf (id=3) <- leaf_bottom, leaf_top + + Expected frame order: leaf_bottom, leaf_top, mid_bottom, mid_top, root_bottom, root_top + (stack is built bottom-up: leaf frames first, then parent frames). + """ + collector = PstatsCollector(sample_interval_usec=1000) + leaf = MockTaskInfo( + task_id=3, + task_name="Leaf", + coroutine_stack=[ + MockCoroInfo(task_name="Leaf", call_stack=[ + MockFrameInfo("leaf.py", 1, "leaf_bottom"), + MockFrameInfo("leaf.py", 2, "leaf_top"), + ]) + ], + awaited_by=[MockCoroInfo(task_name=2, call_stack=[])] + ) + + middle = MockTaskInfo( + task_id=2, + task_name="Middle", + coroutine_stack=[ + MockCoroInfo(task_name="Middle", call_stack=[ + MockFrameInfo("mid.py", 1, "mid_bottom"), + MockFrameInfo("mid.py", 2, "mid_top"), + ]) + ], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="Root", + coroutine_stack=[ + MockCoroInfo(task_name="Root", call_stack=[ + MockFrameInfo("root.py", 1, "root_bottom"), + MockFrameInfo("root.py", 2, "root_top"), + ]) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=333, awaited_by=[leaf, middle, root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + self.assertEqual(len(stacks), 1) + frames, _, _ = stacks[0] + + func_names = [f.funcname for f in frames] + + # Order should be: leaf frames, leaf marker, middle frames, middle marker, root frames, root marker + leaf_bottom_idx = func_names.index("leaf_bottom") + leaf_top_idx = func_names.index("leaf_top") + mid_bottom_idx = func_names.index("mid_bottom") + root_bottom_idx = func_names.index("root_bottom") + + # Verify leaf comes before middle comes before root + self.assertLess(leaf_bottom_idx, leaf_top_idx, "Leaf frames in order") + self.assertLess(leaf_top_idx, mid_bottom_idx, "Leaf before middle") + self.assertLess(mid_bottom_idx, root_bottom_idx, "Middle before root") + + def test_complex_multi_parent_convergence(self): + """Test _build_linear_stacks: multiple leaves with same parents pick deterministically. + + Tests that when multiple leaves have multiple parents, each leaf picks the same + parent (sorted, first one) and all leaves are annotated with parent count. + + Task graph structure (both leaves awaited by both A and B):: + + Root (id=1) + / \\ + A (id=2) B (id=3) + | \\ / | + | \\ / | + | \\/ | + | /\\ | + | / \\ | + LeafX (id=4) LeafY (id=5) + + Expected behavior: Both leaves pick parent A (lowest id=2) for their stack path. + Result: 2 stacks, both going through A -> Root (B is skipped). + """ + collector = PstatsCollector(sample_interval_usec=1000) + + leaf_x = MockTaskInfo( + task_id=4, + task_name="LeafX", + coroutine_stack=[MockCoroInfo(task_name="LeafX", call_stack=[MockFrameInfo("x.py", 1, "x")])], + awaited_by=[ + MockCoroInfo(task_name=2, call_stack=[]), + MockCoroInfo(task_name=3, call_stack=[]), + ] + ) + + leaf_y = MockTaskInfo( + task_id=5, + task_name="LeafY", + coroutine_stack=[MockCoroInfo(task_name="LeafY", call_stack=[MockFrameInfo("y.py", 1, "y")])], + awaited_by=[ + MockCoroInfo(task_name=2, call_stack=[]), + MockCoroInfo(task_name=3, call_stack=[]), + ] + ) + + parent_a = MockTaskInfo( + task_id=2, + task_name="A", + coroutine_stack=[MockCoroInfo(task_name="A", call_stack=[MockFrameInfo("a.py", 1, "a")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[])] + ) + + parent_b = MockTaskInfo( + task_id=3, + task_name="B", + coroutine_stack=[MockCoroInfo(task_name="B", call_stack=[MockFrameInfo("b.py", 1, "b")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="Root", + coroutine_stack=[MockCoroInfo(task_name="Root", call_stack=[MockFrameInfo("r.py", 1, "root")])], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=444, awaited_by=[leaf_x, leaf_y, parent_a, parent_b, root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # 2 leaves, each picks same parent (A, id=2) = 2 paths + self.assertEqual(len(stacks), 2, "Should create 2 paths: X->A->Root, Y->A->Root") + + # Verify both leaves pick parent A (id=2, first when sorted) + leaf_ids_seen = set() + for frames, _, leaf_id in stacks: + leaf_ids_seen.add(leaf_id) + func_names = [f.funcname for f in frames] + + # Both stacks should go through parent A only + self.assertIn("a", func_names, "Should use parent A (id=2, first when sorted)") + self.assertNotIn("b", func_names, "Should not include parent B") + self.assertIn("root", func_names, "Should reach root") + + # Check for parent count annotation on the leaf + if leaf_id == 4: + self.assertIn("x", func_names) + self.assertIn("LeafX (2 parents)", func_names, "LeafX should be annotated with parent count") + elif leaf_id == 5: + self.assertIn("y", func_names) + self.assertIn("LeafY (2 parents)", func_names, "LeafY should be annotated with parent count") + + # Both leaves should be represented + self.assertEqual(leaf_ids_seen, {4, 5}, "Both LeafX and LeafY should have paths") + + +class TestFlamegraphCollectorAsync(unittest.TestCase): + """Test FlamegraphCollector with async frames.""" + + def test_flamegraph_with_async_frames(self): + """Test FlamegraphCollector correctly processes async task frames.""" + from profiling.sampling.stack_collector import FlamegraphCollector + + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Build async task tree: Root -> Child + child = MockTaskInfo( + task_id=2, + task_name="ChildTask", + coroutine_stack=[ + MockCoroInfo( + task_name="ChildTask", + call_stack=[MockFrameInfo("child.py", 10, "child_work")] + ) + ], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="RootTask", + coroutine_stack=[ + MockCoroInfo( + task_name="RootTask", + call_stack=[MockFrameInfo("root.py", 20, "root_work")] + ) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=100, awaited_by=[child, root])] + + # Collect async frames + collector.collect(awaited_info_list) + + # Verify samples were collected + self.assertGreater(collector._total_samples, 0) + + # Verify the flamegraph tree structure contains our functions + root_node = collector._root + self.assertGreater(root_node["samples"], 0) + + # Check that thread ID was tracked + self.assertIn(100, collector._all_threads) + + def test_flamegraph_with_task_markers(self): + """Test FlamegraphCollector includes <task> boundary markers.""" + from profiling.sampling.stack_collector import FlamegraphCollector + + collector = FlamegraphCollector(sample_interval_usec=1000) + + task = MockTaskInfo( + task_id=42, + task_name="MyTask", + coroutine_stack=[ + MockCoroInfo( + task_name="MyTask", + call_stack=[MockFrameInfo("work.py", 5, "do_work")] + ) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=200, awaited_by=[task])] + collector.collect(awaited_info_list) + + # Find <task> marker in the tree + def find_task_marker(node, depth=0): + for func, child in node.get("children", {}).items(): + if func[0] == "<task>": + return func + result = find_task_marker(child, depth + 1) + if result: + return result + return None + + task_marker = find_task_marker(collector._root) + self.assertIsNotNone(task_marker, "Should have <task> marker in tree") + self.assertEqual(task_marker[0], "<task>") + self.assertIn("MyTask", task_marker[2]) + + def test_flamegraph_multiple_async_samples(self): + """Test FlamegraphCollector aggregates multiple async samples correctly.""" + from profiling.sampling.stack_collector import FlamegraphCollector + + collector = FlamegraphCollector(sample_interval_usec=1000) + + task = MockTaskInfo( + task_id=1, + task_name="Task", + coroutine_stack=[ + MockCoroInfo( + task_name="Task", + call_stack=[MockFrameInfo("work.py", 10, "work")] + ) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=300, awaited_by=[task])] + + # Collect multiple samples + for _ in range(5): + collector.collect(awaited_info_list) + + # Verify sample count + self.assertEqual(collector._sample_count, 5) + self.assertEqual(collector._total_samples, 5) + + +class TestAsyncAwareParameterFlow(unittest.TestCase): + """Integration tests for async_aware parameter flow from CLI to unwinder.""" + + def test_sample_function_accepts_async_aware(self): + """Test that sample() function accepts async_aware parameter.""" + from profiling.sampling.sample import sample + import inspect + + sig = inspect.signature(sample) + self.assertIn("async_aware", sig.parameters) + + def test_sample_live_function_accepts_async_aware(self): + """Test that sample_live() function accepts async_aware parameter.""" + from profiling.sampling.sample import sample_live + import inspect + + sig = inspect.signature(sample_live) + self.assertIn("async_aware", sig.parameters) + + def test_sample_profiler_sample_accepts_async_aware(self): + """Test that SampleProfiler.sample() accepts async_aware parameter.""" + from profiling.sampling.sample import SampleProfiler + import inspect + + sig = inspect.signature(SampleProfiler.sample) + self.assertIn("async_aware", sig.parameters) + + def test_async_aware_all_sees_sleeping_and_running_tasks(self): + """Test async_aware='all' captures both sleeping and CPU-running tasks.""" + # Sleeping task (awaiting) + sleeping_task = MockTaskInfo( + task_id=1, + task_name="SleepingTask", + coroutine_stack=[ + MockCoroInfo( + task_name="SleepingTask", + call_stack=[MockFrameInfo("sleeper.py", 10, "sleep_work")] + ) + ], + awaited_by=[] + ) + + # CPU-running task (active) + running_task = MockTaskInfo( + task_id=2, + task_name="RunningTask", + coroutine_stack=[ + MockCoroInfo( + task_name="RunningTask", + call_stack=[MockFrameInfo("runner.py", 20, "cpu_work")] + ) + ], + awaited_by=[] + ) + + # Both tasks returned by get_all_awaited_by + awaited_info_list = [MockAwaitedInfo(thread_id=100, awaited_by=[sleeping_task, running_task])] + + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(awaited_info_list) + collector.create_stats() + + # Both tasks should be visible + sleeping_key = ("sleeper.py", 10, "sleep_work") + running_key = ("runner.py", 20, "cpu_work") + + self.assertIn(sleeping_key, collector.stats) + self.assertIn(running_key, collector.stats) + + # Task markers should also be present + task_keys = [k for k in collector.stats if k[0] == "<task>"] + self.assertGreater(len(task_keys), 0, "Should have <task> markers in stats") + + # Verify task names are in the markers + task_names = [k[2] for k in task_keys] + self.assertTrue( + any("SleepingTask" in name for name in task_names), + "SleepingTask should be in task markers" + ) + self.assertTrue( + any("RunningTask" in name for name in task_names), + "RunningTask should be in task markers" + ) + + def test_async_aware_running_sees_only_running_task(self): + """Test async_aware='running' only shows the currently running task stack.""" + # Only the running task's stack is returned by get_async_stack_trace + running_task = MockTaskInfo( + task_id=2, + task_name="RunningTask", + coroutine_stack=[ + MockCoroInfo( + task_name="RunningTask", + call_stack=[MockFrameInfo("runner.py", 20, "cpu_work")] + ) + ], + awaited_by=[] + ) + + # get_async_stack_trace only returns the running task + awaited_info_list = [MockAwaitedInfo(thread_id=100, awaited_by=[running_task])] + + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(awaited_info_list) + collector.create_stats() + + # Only running task should be visible + running_key = ("runner.py", 20, "cpu_work") + self.assertIn(running_key, collector.stats) + + # Verify we don't see the sleeping task (it wasn't in the input) + sleeping_key = ("sleeper.py", 10, "sleep_work") + self.assertNotIn(sleeping_key, collector.stats) + + # Task marker for running task should be present + task_keys = [k for k in collector.stats if k[0] == "<task>"] + self.assertGreater(len(task_keys), 0, "Should have <task> markers in stats") + + task_names = [k[2] for k in task_keys] + self.assertTrue( + any("RunningTask" in name for name in task_names), + "RunningTask should be in task markers" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py new file mode 100644 index 00000000000..e1892ec9155 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -0,0 +1,711 @@ +"""Tests for sampling profiler CLI argument parsing and functionality.""" + +import io +import subprocess +import sys +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import is_emscripten + + +class TestSampleProfilerCLI(unittest.TestCase): + def _setup_sync_mocks(self, mock_socket, mock_popen): + """Helper to set up socket and process mocks for coordinator tests.""" + # Mock the sync socket with context manager support + mock_sock_instance = mock.MagicMock() + mock_sock_instance.getsockname.return_value = ("127.0.0.1", 12345) + + # Mock the connection with context manager support + mock_conn = mock.MagicMock() + mock_conn.recv.return_value = b"ready" + mock_conn.__enter__.return_value = mock_conn + mock_conn.__exit__.return_value = None + + # Mock accept() to return (connection, address) and support indexing + mock_accept_result = mock.MagicMock() + mock_accept_result.__getitem__.return_value = ( + mock_conn # [0] returns the connection + ) + mock_sock_instance.accept.return_value = mock_accept_result + + # Mock socket with context manager support + mock_sock_instance.__enter__.return_value = mock_sock_instance + mock_sock_instance.__exit__.return_value = None + mock_socket.return_value = mock_sock_instance + + # Mock the subprocess + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.poll.return_value = None + mock_popen.return_value = mock_process + return mock_process + + def _verify_coordinator_command(self, mock_popen, expected_target_args): + """Helper to verify the coordinator command was called correctly.""" + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual( + coordinator_cmd[2], "profiling.sampling._sync_coordinator" + ) + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual(coordinator_cmd[5:], expected_target_args) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_argument_parsing(self): + test_args = ["profiling.sampling.cli", "run", "-m", "mymodule"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + from profiling.sampling.cli import main + self._setup_sync_mocks(mock_socket, mock_popen) + main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + # Verify sample was called once (exact arguments will vary with the new API) + mock_sample.assert_called_once() + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_with_arguments(self): + test_args = [ + "profiling.sampling.cli", + "run", + "-m", + "mymodule", + "arg1", + "arg2", + "--flag", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + from profiling.sampling.cli import main + main() + + self._verify_coordinator_command( + mock_popen, ("-m", "mymodule", "arg1", "arg2", "--flag") + ) + mock_sample.assert_called_once() + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_argument_parsing(self): + test_args = ["profiling.sampling.cli", "run", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + from profiling.sampling.cli import main + main() + + self._verify_coordinator_command(mock_popen, ("myscript.py",)) + mock_sample.assert_called_once() + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_with_arguments(self): + test_args = [ + "profiling.sampling.cli", + "run", + "myscript.py", + "arg1", + "arg2", + "--flag", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + # Use the helper to set up mocks consistently + mock_process = self._setup_sync_mocks(mock_socket, mock_popen) + # Override specific behavior for this test + mock_process.wait.side_effect = [ + subprocess.TimeoutExpired(test_args, 0.1), + None, + ] + + from profiling.sampling.cli import main + main() + + # Verify the coordinator command was called + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual( + coordinator_cmd[2], "profiling.sampling._sync_coordinator" + ) + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual( + coordinator_cmd[5:], ("myscript.py", "arg1", "arg2", "--flag") + ) + + def test_cli_mutually_exclusive_pid_module(self): + # In new CLI, attach and run are separate subcommands, so this test + # verifies that mixing them causes an error + test_args = [ + "profiling.sampling.cli", + "attach", # attach subcommand uses PID + "12345", + "-m", # -m is only for run subcommand + "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("unrecognized arguments", error_msg) + + def test_cli_mutually_exclusive_pid_script(self): + # In new CLI, you can't mix attach (PID) with run (script) + # This would be caught by providing a PID to run subcommand + test_args = ["profiling.sampling.cli", "run", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + self.assertRaises(FileNotFoundError) as cm, # Expect FileNotFoundError, not SystemExit + ): + self._setup_sync_mocks(mock_socket, mock_popen) + # Override to raise FileNotFoundError for non-existent script + mock_popen.side_effect = FileNotFoundError("12345") + from profiling.sampling.cli import main + main() + + # Verify the error is about the non-existent script + self.assertIn("12345", str(cm.exception)) + + def test_cli_no_target_specified(self): + # In new CLI, must specify a subcommand + test_args = ["profiling.sampling.cli", "-d", "5"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("invalid choice", error_msg) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_with_profiler_options(self): + test_args = [ + "profiling.sampling.cli", + "run", + "-i", + "1000", + "-d", + "30", + "-a", + "--sort", + "tottime", + "-l", + "20", + "-m", + "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + from profiling.sampling.cli import main + main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once() + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_with_profiler_options(self): + """Test script with various profiler options.""" + test_args = [ + "profiling.sampling.cli", + "run", + "-i", + "2000", + "-d", + "60", + "--collapsed", + "-o", + "output.txt", + "myscript.py", + "scriptarg", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + from profiling.sampling.cli import main + main() + + self._verify_coordinator_command( + mock_popen, ("myscript.py", "scriptarg") + ) + # Verify profiler was called + mock_sample.assert_called_once() + + def test_cli_empty_module_name(self): + test_args = ["profiling.sampling.cli", "run", "-m"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("required: target", error_msg) # argparse error for missing positional arg + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_long_module_option(self): + test_args = [ + "profiling.sampling.cli", + "run", + "-m", + "mymodule", + "arg1", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + from profiling.sampling.cli import main + main() + + self._verify_coordinator_command( + mock_popen, ("-m", "mymodule", "arg1") + ) + + def test_cli_complex_script_arguments(self): + test_args = [ + "profiling.sampling.cli", + "run", + "script.py", + "--input", + "file.txt", + "-v", + "--output=/tmp/out", + "positional", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + mock.patch( + "profiling.sampling.cli._run_with_sync" + ) as mock_run_with_sync, + ): + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.wait.side_effect = [ + subprocess.TimeoutExpired(test_args, 0.1), + None, + ] + mock_process.poll.return_value = None + mock_run_with_sync.return_value = mock_process + + from profiling.sampling.cli import main + main() + + mock_run_with_sync.assert_called_once_with( + ( + sys.executable, + "script.py", + "--input", + "file.txt", + "-v", + "--output=/tmp/out", + "positional", + ), + suppress_output=False + ) + + def test_cli_collapsed_format_validation(self): + """Test that CLI properly validates incompatible options with collapsed format.""" + test_cases = [ + # Test sort option is invalid with collapsed + ( + [ + "profiling.sampling.cli", + "attach", + "12345", + "--collapsed", + "--sort", + "tottime", # Changed from nsamples (default) to trigger validation + ], + "sort", + ), + # Test limit option is invalid with collapsed + ( + [ + "profiling.sampling.cli", + "attach", + "12345", + "--collapsed", + "-l", + "20", + ], + "limit", + ), + # Test no-summary option is invalid with collapsed + ( + [ + "profiling.sampling.cli", + "attach", + "12345", + "--collapsed", + "--no-summary", + ], + "summary", + ), + ] + + from profiling.sampling.cli import main + + for test_args, expected_error_keyword in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + mock.patch("profiling.sampling.cli.sample"), # Prevent actual profiling + self.assertRaises(SystemExit) as cm, + ): + main() + + self.assertEqual(cm.exception.code, 2) # argparse error code + error_msg = mock_stderr.getvalue() + self.assertIn("error:", error_msg) + self.assertIn("only valid with --pstats", error_msg) + + def test_cli_default_collapsed_filename(self): + """Test that collapsed format gets a default filename when not specified.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--collapsed"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + # Check that sample was called (exact filename depends on implementation) + mock_sample.assert_called_once() + + def test_cli_custom_output_filenames(self): + """Test custom output filenames for both formats.""" + test_cases = [ + ( + [ + "profiling.sampling.cli", + "attach", + "12345", + "--pstats", + "-o", + "custom.pstats", + ], + "custom.pstats", + "pstats", + ), + ( + [ + "profiling.sampling.cli", + "attach", + "12345", + "--collapsed", + "-o", + "custom.txt", + ], + "custom.txt", + "collapsed", + ), + ] + + from profiling.sampling.cli import main + + for test_args, expected_filename, expected_format in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + main() + + mock_sample.assert_called_once() + + def test_cli_missing_required_arguments(self): + """Test that CLI requires subcommand.""" + with ( + mock.patch("sys.argv", ["profiling.sampling.cli"]), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + from profiling.sampling.cli import main + main() + + def test_cli_mutually_exclusive_format_options(self): + """Test that pstats and collapsed options are mutually exclusive.""" + with ( + mock.patch( + "sys.argv", + [ + "profiling.sampling.cli", + "attach", + "12345", + "--pstats", + "--collapsed", + ], + ), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + from profiling.sampling.cli import main + main() + + def test_argument_parsing_basic(self): + test_args = ["profiling.sampling.cli", "attach", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + mock_sample.assert_called_once() + + def test_sort_options(self): + from profiling.sampling.cli import main + + sort_options = [ + ("nsamples", 0), + ("tottime", 1), + ("cumtime", 2), + ("sample-pct", 3), + ("cumul-pct", 4), + ("name", -1), + ] + + for option, expected_sort_value in sort_options: + test_args = ["profiling.sampling.cli", "attach", "12345", "--sort", option] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + mock_sample.assert_called_once() + mock_sample.reset_mock() + + def test_async_aware_flag_defaults_to_running(self): + """Test --async-aware flag enables async profiling with default 'running' mode.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + mock_sample.assert_called_once() + # Verify async_aware was passed with default "running" mode + call_kwargs = mock_sample.call_args[1] + self.assertEqual(call_kwargs.get("async_aware"), "running") + + def test_async_aware_with_async_mode_all(self): + """Test --async-aware with --async-mode all.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--async-mode", "all"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + mock_sample.assert_called_once() + call_kwargs = mock_sample.call_args[1] + self.assertEqual(call_kwargs.get("async_aware"), "all") + + def test_async_aware_default_is_none(self): + """Test async_aware defaults to None when --async-aware not specified.""" + test_args = ["profiling.sampling.cli", "attach", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + mock_sample.assert_called_once() + call_kwargs = mock_sample.call_args[1] + self.assertIsNone(call_kwargs.get("async_aware")) + + def test_async_mode_invalid_choice(self): + """Test --async-mode with invalid choice raises error.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--async-mode", "invalid"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()), + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + + def test_async_mode_requires_async_aware(self): + """Test --async-mode without --async-aware raises error.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-mode", "all"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--async-mode requires --async-aware", error_msg) + + def test_async_aware_incompatible_with_native(self): + """Test --async-aware is incompatible with --native.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--native"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--native", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) + + def test_async_aware_incompatible_with_no_gc(self): + """Test --async-aware is incompatible with --no-gc.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--no-gc"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--no-gc", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) + + def test_async_aware_incompatible_with_both_native_and_no_gc(self): + """Test --async-aware is incompatible with both --native and --no-gc.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--native", "--no-gc"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--native", error_msg) + self.assertIn("--no-gc", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) + + def test_async_aware_incompatible_with_mode(self): + """Test --async-aware is incompatible with --mode (non-wall).""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--mode", "cpu"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--mode=cpu", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) + + def test_async_aware_incompatible_with_all_threads(self): + """Test --async-aware is incompatible with --all-threads.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--all-threads"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--all-threads", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py new file mode 100644 index 00000000000..e8c12c22215 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -0,0 +1,1203 @@ +"""Tests for sampling profiler collector components.""" + +import json +import marshal +import os +import tempfile +import unittest + +try: + import _remote_debugging # noqa: F401 + from profiling.sampling.pstats_collector import PstatsCollector + from profiling.sampling.stack_collector import ( + CollapsedStackCollector, + FlamegraphCollector, + ) + from profiling.sampling.gecko_collector import GeckoCollector + from profiling.sampling.constants import ( + PROFILING_MODE_WALL, + PROFILING_MODE_CPU, + ) + from _remote_debugging import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + THREAD_STATUS_GIL_REQUESTED, + ) +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import captured_stdout, captured_stderr + +from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo +from .helpers import close_and_unlink + + +class TestSampleProfilerComponents(unittest.TestCase): + """Unit tests for individual profiler components.""" + + def test_mock_frame_info_with_empty_and_unicode_values(self): + """Test MockFrameInfo handles empty strings, unicode characters, and very long names correctly.""" + # Test with empty strings + frame = MockFrameInfo("", 0, "") + self.assertEqual(frame.filename, "") + self.assertEqual(frame.lineno, 0) + self.assertEqual(frame.funcname, "") + self.assertIn("filename=''", repr(frame)) + + # Test with unicode characters + frame = MockFrameInfo("文件.py", 42, "函数名") + self.assertEqual(frame.filename, "文件.py") + self.assertEqual(frame.funcname, "函数名") + + # Test with very long names + long_filename = "x" * 1000 + ".py" + long_funcname = "func_" + "x" * 1000 + frame = MockFrameInfo(long_filename, 999999, long_funcname) + self.assertEqual(frame.filename, long_filename) + self.assertEqual(frame.lineno, 999999) + self.assertEqual(frame.funcname, long_funcname) + + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): + """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" + # Test with zero interval + collector = PstatsCollector(sample_interval_usec=0) + self.assertEqual(collector.sample_interval_usec, 0) + + # Test with very large interval + collector = PstatsCollector(sample_interval_usec=1000000000) + self.assertEqual(collector.sample_interval_usec, 1000000000) + + # Test collecting empty frames list + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect([]) + self.assertEqual(len(collector.result), 0) + + # Test collecting frames with None thread id + test_frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(None, [MockFrameInfo("file.py", 10, "func")])], + ) + ] + collector.collect(test_frames) + # Should still process the frames + self.assertEqual(len(collector.result), 1) + + # Test collecting duplicate frames in same sample + test_frames = [ + MockInterpreterInfo( + 0, # interpreter_id + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 10, "func1"), # Duplicate + ], + ) + ], + ) + ] + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(test_frames) + # Should count both occurrences + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 + ) + + def test_pstats_collector_single_frame_stacks(self): + """Test PstatsCollector with single-frame call stacks to trigger len(frames) <= 1 branch.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test with exactly one frame (should trigger the <= 1 condition) + single_frame = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("single.py", 10, "single_func")] + ) + ], + ) + ] + collector.collect(single_frame) + + # Should record the single frame with inline call + self.assertEqual(len(collector.result), 1) + single_key = ("single.py", 10, "single_func") + self.assertIn(single_key, collector.result) + self.assertEqual(collector.result[single_key]["direct_calls"], 1) + self.assertEqual(collector.result[single_key]["cumulative_calls"], 1) + + # Test with empty frames (should also trigger <= 1 condition) + empty_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [])])] + collector.collect(empty_frames) + + # Should not add any new entries + self.assertEqual( + len(collector.result), 1 + ) # Still just the single frame + + # Test mixed single and multi-frame stacks + mixed_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("single2.py", 20, "single_func2")], + ), # Single frame + MockThreadInfo( + 2, + [ # Multi-frame stack + MockFrameInfo("multi.py", 30, "multi_func1"), + MockFrameInfo("multi.py", 40, "multi_func2"), + ], + ), + ], + ), + ] + collector.collect(mixed_frames) + + # Should have recorded all functions + self.assertEqual( + len(collector.result), 4 + ) # single + single2 + multi1 + multi2 + + # Verify single frame handling + single2_key = ("single2.py", 20, "single_func2") + self.assertIn(single2_key, collector.result) + self.assertEqual(collector.result[single2_key]["direct_calls"], 1) + self.assertEqual(collector.result[single2_key]["cumulative_calls"], 1) + + # Verify multi-frame handling still works + multi1_key = ("multi.py", 30, "multi_func1") + multi2_key = ("multi.py", 40, "multi_func2") + self.assertIn(multi1_key, collector.result) + self.assertIn(multi2_key, collector.result) + self.assertEqual(collector.result[multi1_key]["direct_calls"], 1) + self.assertEqual( + collector.result[multi2_key]["cumulative_calls"], 1 + ) # Called from multi1 + + def test_collapsed_stack_collector_with_empty_and_deep_stacks(self): + """Test CollapsedStackCollector handles empty frames, single-frame stacks, and very deep call stacks.""" + collector = CollapsedStackCollector(1000) + + # Test with empty frames + collector.collect([]) + self.assertEqual(len(collector.stack_counter), 0) + + # Test with single frame stack + test_frames = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("file.py", 10, "func")])] + ) + ] + collector.collect(test_frames) + self.assertEqual(len(collector.stack_counter), 1) + (((path, thread_id), count),) = collector.stack_counter.items() + self.assertEqual(path, (("file.py", 10, "func"),)) + self.assertEqual(thread_id, 1) + self.assertEqual(count, 1) + + # Test with very deep stack + deep_stack = [(f"file{i}.py", i, f"func{i}") for i in range(100)] + test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, deep_stack)])] + collector = CollapsedStackCollector(1000) + collector.collect(test_frames) + # One aggregated path with 100 frames (reversed) + (((path_tuple, thread_id),),) = (collector.stack_counter.keys(),) + self.assertEqual(len(path_tuple), 100) + self.assertEqual(path_tuple[0], ("file99.py", 99, "func99")) + self.assertEqual(path_tuple[-1], ("file0.py", 0, "func0")) + self.assertEqual(thread_id, 1) + + def test_pstats_collector_basic(self): + """Test basic PstatsCollector functionality.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test empty state + self.assertEqual(len(collector.result), 0) + self.assertEqual(len(collector.stats), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + collector.collect(test_frames) + + # Should have recorded calls for both functions + self.assertEqual(len(collector.result), 2) + self.assertIn(("file.py", 10, "func1"), collector.result) + self.assertIn(("file.py", 20, "func2"), collector.result) + + # Top-level function should have direct call + self.assertEqual( + collector.result[("file.py", 10, "func1")]["direct_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 + ) + + # Calling function should have cumulative call but no direct calls + self.assertEqual( + collector.result[("file.py", 20, "func2")]["cumulative_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 20, "func2")]["direct_calls"], 0 + ) + + def test_pstats_collector_create_stats(self): + """Test PstatsCollector stats creation.""" + collector = PstatsCollector( + sample_interval_usec=1000000 + ) # 1 second intervals + + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + collector.collect(test_frames) + collector.collect(test_frames) # Collect twice + + collector.create_stats() + + # Check stats format: (direct_calls, cumulative_calls, tt, ct, callers) + func1_stats = collector.stats[("file.py", 10, "func1")] + self.assertEqual(func1_stats[0], 2) # direct_calls (top of stack) + self.assertEqual(func1_stats[1], 2) # cumulative_calls + self.assertEqual( + func1_stats[2], 2.0 + ) # tt (total time - 2 samples * 1 sec) + self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + func2_stats = collector.stats[("file.py", 20, "func2")] + self.assertEqual( + func2_stats[0], 0 + ) # direct_calls (never top of stack) + self.assertEqual( + func2_stats[1], 2 + ) # cumulative_calls (appears in stack) + self.assertEqual(func2_stats[2], 0.0) # tt (no direct calls) + self.assertEqual(func2_stats[3], 2.0) # ct (cumulative time) + + def test_collapsed_stack_collector_basic(self): + collector = CollapsedStackCollector(1000) + + # Test empty state + self.assertEqual(len(collector.stack_counter), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + collector.collect(test_frames) + + # Should store one reversed path + self.assertEqual(len(collector.stack_counter), 1) + (((path, thread_id), count),) = collector.stack_counter.items() + expected_tree = (("file.py", 20, "func2"), ("file.py", 10, "func1")) + self.assertEqual(path, expected_tree) + self.assertEqual(thread_id, 1) + self.assertEqual(count, 1) + + def test_collapsed_stack_collector_export(self): + collapsed_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, collapsed_out) + + collector = CollapsedStackCollector(1000) + + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + with captured_stdout(), captured_stderr(): + collector.export(collapsed_out.name) + # Check file contents + with open(collapsed_out.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertEqual(len(lines), 2) # Two unique stacks + + # Check collapsed format: tid:X;file:func:line;file:func:line count + stack1_expected = "tid:1;file.py:func2:20;file.py:func1:10 2" + stack2_expected = "tid:1;other.py:other_func:5 1" + + self.assertIn(stack1_expected, lines) + self.assertIn(stack2_expected, lines) + + def test_flamegraph_collector_basic(self): + """Test basic FlamegraphCollector functionality.""" + collector = FlamegraphCollector(1000) + + # Empty collector should produce 'No Data' + data = collector._convert_to_flamegraph_format() + # With string table, name is now an index - resolve it using the strings array + strings = data.get("strings", []) + name_index = data.get("name", 0) + resolved_name = ( + strings[name_index] + if isinstance(name_index, int) and 0 <= name_index < len(strings) + else str(name_index) + ) + self.assertIn(resolved_name, ("No Data", "No significant data")) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + collector.collect(test_frames) + + # Convert and verify structure: func2 -> func1 with counts = 1 + data = collector._convert_to_flamegraph_format() + # Expect promotion: root is the single child (func2), with func1 as its only child + strings = data.get("strings", []) + name_index = data.get("name", 0) + name = ( + strings[name_index] + if isinstance(name_index, int) and 0 <= name_index < len(strings) + else str(name_index) + ) + self.assertIsInstance(name, str) + self.assertTrue(name.startswith("Program Root: ")) + self.assertIn("func2 (file.py:20)", name) # formatted name + children = data.get("children", []) + self.assertEqual(len(children), 1) + child = children[0] + child_name_index = child.get("name", 0) + child_name = ( + strings[child_name_index] + if isinstance(child_name_index, int) + and 0 <= child_name_index < len(strings) + else str(child_name_index) + ) + self.assertIn("func1 (file.py:10)", child_name) # formatted name + self.assertEqual(child["value"], 1) + + def test_flamegraph_collector_export(self): + """Test flamegraph HTML export functionality.""" + flamegraph_out = tempfile.NamedTemporaryFile( + suffix=".html", delete=False + ) + self.addCleanup(close_and_unlink, flamegraph_out) + + collector = FlamegraphCollector(1000) + + # Create some test data (use Interpreter/Thread objects like runtime) + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + # Export flamegraph + with captured_stdout(), captured_stderr(): + collector.export(flamegraph_out.name) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(flamegraph_out.name)) + self.assertGreater(os.path.getsize(flamegraph_out.name), 0) + + # Check file contains HTML content + with open(flamegraph_out.name, "r", encoding="utf-8") as f: + content = f.read() + + # Should be valid HTML + self.assertIn("<!doctype html>", content.lower()) + self.assertIn("<html", content) + self.assertIn("Tachyon Profiler - Flamegraph", content) + self.assertIn("d3-flame-graph", content) + + # Should contain the data + self.assertIn('"name":', content) + self.assertIn('"value":', content) + self.assertIn('"children":', content) + + def test_gecko_collector_basic(self): + """Test basic GeckoCollector functionality.""" + collector = GeckoCollector(1000) + + # Test empty state + self.assertEqual(len(collector.threads), 0) + self.assertEqual(collector.sample_count, 0) + self.assertEqual(len(collector.global_strings), 1) # "(root)" + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [("file.py", 10, "func1"), ("file.py", 20, "func2")], + ) + ], + ) + ] + collector.collect(test_frames) + + # Should have recorded one thread and one sample + self.assertEqual(len(collector.threads), 1) + self.assertEqual(collector.sample_count, 1) + self.assertIn(1, collector.threads) + + profile_data = collector._build_profile() + + # Verify profile structure + self.assertIn("meta", profile_data) + self.assertIn("threads", profile_data) + self.assertIn("shared", profile_data) + + # Check shared string table + shared = profile_data["shared"] + self.assertIn("stringArray", shared) + string_array = shared["stringArray"] + self.assertGreater(len(string_array), 0) + + # Should contain our functions in the string array + self.assertIn("func1", string_array) + self.assertIn("func2", string_array) + + # Check thread data structure + threads = profile_data["threads"] + self.assertEqual(len(threads), 1) + thread_data = threads[0] + + # Verify thread structure + self.assertIn("samples", thread_data) + self.assertIn("funcTable", thread_data) + self.assertIn("frameTable", thread_data) + self.assertIn("stackTable", thread_data) + + # Verify samples + samples = thread_data["samples"] + self.assertEqual(len(samples["stack"]), 1) + self.assertEqual(len(samples["time"]), 1) + self.assertEqual(samples["length"], 1) + + # Verify function table structure and content + func_table = thread_data["funcTable"] + self.assertIn("name", func_table) + self.assertIn("fileName", func_table) + self.assertIn("lineNumber", func_table) + self.assertEqual(func_table["length"], 2) # Should have 2 functions + + # Verify actual function content through string array indices + func_names = [] + for idx in func_table["name"]: + func_name = ( + string_array[idx] + if isinstance(idx, int) and 0 <= idx < len(string_array) + else str(idx) + ) + func_names.append(func_name) + + self.assertIn("func1", func_names, f"func1 not found in {func_names}") + self.assertIn("func2", func_names, f"func2 not found in {func_names}") + + # Verify frame table + frame_table = thread_data["frameTable"] + self.assertEqual( + frame_table["length"], 2 + ) # Should have frames for both functions + self.assertEqual(len(frame_table["func"]), 2) + + # Verify stack structure + stack_table = thread_data["stackTable"] + self.assertGreater(stack_table["length"], 0) + self.assertGreater(len(stack_table["frame"]), 0) + + def test_gecko_collector_export(self): + """Test Gecko profile export functionality.""" + gecko_out = tempfile.NamedTemporaryFile(suffix=".json", delete=False) + self.addCleanup(close_and_unlink, gecko_out) + + collector = GeckoCollector(1000) + + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + # Export gecko profile + with captured_stdout(), captured_stderr(): + collector.export(gecko_out.name) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(gecko_out.name)) + self.assertGreater(os.path.getsize(gecko_out.name), 0) + + # Check file contains valid JSON + with open(gecko_out.name, "r") as f: + profile_data = json.load(f) + + # Should be valid Gecko profile format + self.assertIn("meta", profile_data) + self.assertIn("threads", profile_data) + self.assertIn("shared", profile_data) + + # Check meta information + self.assertIn("categories", profile_data["meta"]) + self.assertIn("interval", profile_data["meta"]) + + # Check shared string table + self.assertIn("stringArray", profile_data["shared"]) + self.assertGreater(len(profile_data["shared"]["stringArray"]), 0) + + # Should contain our functions + string_array = profile_data["shared"]["stringArray"] + self.assertIn("func1", string_array) + self.assertIn("func2", string_array) + self.assertIn("other_func", string_array) + + def test_gecko_collector_markers(self): + """Test Gecko profile markers for GIL and CPU state tracking.""" + collector = GeckoCollector(1000) + + # Status combinations for different thread states + HAS_GIL_ON_CPU = ( + THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU + ) # Running Python code + NO_GIL_ON_CPU = THREAD_STATUS_ON_CPU # Running native code + WAITING_FOR_GIL = THREAD_STATUS_GIL_REQUESTED # Waiting for GIL + + # Simulate thread state transitions + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [("test.py", 10, "python_func")], + status=HAS_GIL_ON_CPU, + ) + ], + ) + ] + ) + + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [("test.py", 15, "wait_func")], + status=WAITING_FOR_GIL, + ) + ], + ) + ] + ) + + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [("test.py", 20, "python_func2")], + status=HAS_GIL_ON_CPU, + ) + ], + ) + ] + ) + + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [("native.c", 100, "native_func")], + status=NO_GIL_ON_CPU, + ) + ], + ) + ] + ) + + profile_data = collector._build_profile() + + # Verify we have threads with markers + self.assertIn("threads", profile_data) + self.assertEqual(len(profile_data["threads"]), 1) + thread_data = profile_data["threads"][0] + + # Check markers exist + self.assertIn("markers", thread_data) + markers = thread_data["markers"] + + # Should have marker arrays + self.assertIn("name", markers) + self.assertIn("startTime", markers) + self.assertIn("endTime", markers) + self.assertIn("category", markers) + self.assertGreater( + markers["length"], 0, "Should have generated markers" + ) + + # Get marker names from string table + string_array = profile_data["shared"]["stringArray"] + marker_names = [string_array[idx] for idx in markers["name"]] + + # Verify we have different marker types + marker_name_set = set(marker_names) + + # Should have "Has GIL" markers (when thread had GIL) + self.assertIn( + "Has GIL", marker_name_set, "Should have 'Has GIL' markers" + ) + + # Should have "No GIL" markers (when thread didn't have GIL) + self.assertIn( + "No GIL", marker_name_set, "Should have 'No GIL' markers" + ) + + # Should have "On CPU" markers (when thread was on CPU) + self.assertIn( + "On CPU", marker_name_set, "Should have 'On CPU' markers" + ) + + # Should have "Waiting for GIL" markers (when thread was waiting) + self.assertIn( + "Waiting for GIL", + marker_name_set, + "Should have 'Waiting for GIL' markers", + ) + + # Verify marker structure + for i in range(markers["length"]): + # All markers should be interval markers (phase = 1) + self.assertEqual( + markers["phase"][i], 1, f"Marker {i} should be interval marker" + ) + + # All markers should have valid time range + start_time = markers["startTime"][i] + end_time = markers["endTime"][i] + self.assertLessEqual( + start_time, + end_time, + f"Marker {i} should have valid time range", + ) + + # All markers should have valid category + self.assertGreaterEqual( + markers["category"][i], + 0, + f"Marker {i} should have valid category", + ) + + def test_pstats_collector_export(self): + collector = PstatsCollector( + sample_interval_usec=1000000 + ) # 1 second intervals + + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("other.py", 5, "other_func")] + ) + ], + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + pstats_out = tempfile.NamedTemporaryFile( + suffix=".pstats", delete=False + ) + self.addCleanup(close_and_unlink, pstats_out) + collector.export(pstats_out.name) + + # Check file can be loaded with marshal + with open(pstats_out.name, "rb") as f: + stats_data = marshal.load(f) + + # Should be a dictionary with the sampled marker + self.assertIsInstance(stats_data, dict) + self.assertIn(("__sampled__",), stats_data) + self.assertTrue(stats_data[("__sampled__",)]) + + # Should have function data + function_entries = [ + k for k in stats_data.keys() if k != ("__sampled__",) + ] + self.assertGreater(len(function_entries), 0) + + # Check specific function stats format: (cc, nc, tt, ct, callers) + func1_key = ("file.py", 10, "func1") + func2_key = ("file.py", 20, "func2") + other_key = ("other.py", 5, "other_func") + + self.assertIn(func1_key, stats_data) + self.assertIn(func2_key, stats_data) + self.assertIn(other_key, stats_data) + + # Check func1 stats (should have 2 samples) + func1_stats = stats_data[func1_key] + self.assertEqual(func1_stats[0], 2) # total_calls + self.assertEqual(func1_stats[1], 2) # nc (non-recursive calls) + self.assertEqual(func1_stats[2], 2.0) # tt (total time) + self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + def test_flamegraph_collector_stats_accumulation(self): + """Test that FlamegraphCollector accumulates stats across samples.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # First sample + stack_frames_1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_1) + self.assertEqual(collector.thread_status_counts["has_gil"], 1) + self.assertEqual(collector.thread_status_counts["on_cpu"], 1) + self.assertEqual(collector.thread_status_counts["total"], 2) + + # Second sample + stack_frames_2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_GIL_REQUESTED), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(3, [("c.py", 3, "func_c")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_2) + + # Should accumulate + self.assertEqual(collector.thread_status_counts["has_gil"], 2) # 1 + 1 + self.assertEqual(collector.thread_status_counts["on_cpu"], 2) # 1 + 1 + self.assertEqual(collector.thread_status_counts["gil_requested"], 1) # 0 + 1 + self.assertEqual(collector.thread_status_counts["total"], 5) # 2 + 3 + + # Test GC sample tracking + stack_frames_gc = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("~", 0, "<GC>")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames_gc) + self.assertEqual(collector.samples_with_gc_frames, 1) + + # Another sample without GC + collector.collect(stack_frames_1) + self.assertEqual(collector.samples_with_gc_frames, 1) # Still 1 + + # Another GC sample + collector.collect(stack_frames_gc) + self.assertEqual(collector.samples_with_gc_frames, 2) + + def test_flamegraph_collector_per_thread_stats(self): + """Test per-thread statistics tracking in FlamegraphCollector.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Multiple threads with different states + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + MockThreadInfo(3, [("c.py", 3, "func_c")], status=THREAD_STATUS_GIL_REQUESTED), + ], + ) + ] + collector.collect(stack_frames) + + # Check per-thread stats + self.assertIn(1, collector.per_thread_stats) + self.assertIn(2, collector.per_thread_stats) + self.assertIn(3, collector.per_thread_stats) + + # Thread 1: has GIL + self.assertEqual(collector.per_thread_stats[1]["has_gil"], 1) + self.assertEqual(collector.per_thread_stats[1]["on_cpu"], 0) + self.assertEqual(collector.per_thread_stats[1]["total"], 1) + + # Thread 2: on CPU + self.assertEqual(collector.per_thread_stats[2]["has_gil"], 0) + self.assertEqual(collector.per_thread_stats[2]["on_cpu"], 1) + self.assertEqual(collector.per_thread_stats[2]["total"], 1) + + # Thread 3: waiting + self.assertEqual(collector.per_thread_stats[3]["gil_requested"], 1) + self.assertEqual(collector.per_thread_stats[3]["total"], 1) + + # Test accumulation across samples + stack_frames_2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_2) + + self.assertEqual(collector.per_thread_stats[1]["has_gil"], 1) + self.assertEqual(collector.per_thread_stats[1]["on_cpu"], 1) + self.assertEqual(collector.per_thread_stats[1]["total"], 2) + + def test_flamegraph_collector_percentage_calculations(self): + """Test that percentage calculations are correct in exported data.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Create scenario: 60% GIL held, 40% not held + for i in range(6): + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + for i in range(4): + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Export to get calculated percentages + data = collector._convert_to_flamegraph_format() + thread_stats = data["stats"]["thread_stats"] + + self.assertAlmostEqual(thread_stats["has_gil_pct"], 60.0, places=1) + self.assertAlmostEqual(thread_stats["on_cpu_pct"], 40.0, places=1) + self.assertEqual(thread_stats["total"], 10) + + def test_flamegraph_collector_mode_handling(self): + """Test that profiling mode is correctly passed through to exported data.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Collect some data + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats with mode + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_CPU + ) + + data = collector._convert_to_flamegraph_format() + self.assertEqual(data["stats"]["mode"], PROFILING_MODE_CPU) + + def test_flamegraph_collector_zero_samples_edge_case(self): + """Test that collector handles zero samples gracefully.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Export without collecting any samples + data = collector._convert_to_flamegraph_format() + + # Should return a valid structure with no data + self.assertIn("name", data) + self.assertEqual(data["value"], 0) + self.assertIn("children", data) + self.assertEqual(len(data["children"]), 0) + + def test_flamegraph_collector_json_structure_includes_stats(self): + """Test that exported JSON includes thread_stats and per_thread_stats.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Collect some data with multiple threads + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_WALL + ) + + # Export and verify structure + data = collector._convert_to_flamegraph_format() + + # Check that stats object exists and contains expected fields + self.assertIn("stats", data) + stats = data["stats"] + + # Verify thread_stats exists and has expected structure + self.assertIn("thread_stats", stats) + thread_stats = stats["thread_stats"] + self.assertIn("has_gil_pct", thread_stats) + self.assertIn("on_cpu_pct", thread_stats) + self.assertIn("gil_requested_pct", thread_stats) + self.assertIn("gc_pct", thread_stats) + self.assertIn("total", thread_stats) + + # Verify per_thread_stats exists and has data for both threads + self.assertIn("per_thread_stats", stats) + per_thread_stats = stats["per_thread_stats"] + self.assertIn(1, per_thread_stats) + self.assertIn(2, per_thread_stats) + + # Check per-thread structure + for thread_id in [1, 2]: + thread_data = per_thread_stats[thread_id] + self.assertIn("has_gil_pct", thread_data) + self.assertIn("on_cpu_pct", thread_data) + self.assertIn("gil_requested_pct", thread_data) + self.assertIn("gc_pct", thread_data) + self.assertIn("total", thread_data) + + def test_flamegraph_collector_per_thread_gc_percentage(self): + """Test that per-thread GC percentage uses total samples as denominator.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Create 10 samples total: + # - Thread 1 appears in all 10 samples, has GC in 2 of them + # - Thread 2 appears in only 5 samples, has GC in 1 of them + + # First 5 samples: both threads, thread 1 has GC in 2 + for i in range(5): + has_gc = i < 2 # First 2 samples have GC for thread 1 + frames_1 = [("~", 0, "<GC>")] if has_gc else [("a.py", 1, "func_a")] + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, frames_1, status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Next 5 samples: only thread 1, thread 2 appears in first of these with GC + for i in range(5): + if i == 0: + # Thread 2 appears in this sample with GC + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("~", 0, "<GC>")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + else: + # Only thread 1 + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats and export + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_WALL + ) + + data = collector._convert_to_flamegraph_format() + per_thread_stats = data["stats"]["per_thread_stats"] + + # Thread 1: appeared in 10 samples, had GC in 2 + # GC percentage should be 2/10 = 20% (using total samples, not thread appearances) + self.assertEqual(collector.per_thread_stats[1]["gc_samples"], 2) + self.assertEqual(collector.per_thread_stats[1]["total"], 10) + self.assertAlmostEqual(per_thread_stats[1]["gc_pct"], 20.0, places=1) + + # Thread 2: appeared in 6 samples, had GC in 1 + # GC percentage should be 1/10 = 10% (using total samples, not thread appearances) + self.assertEqual(collector.per_thread_stats[2]["gc_samples"], 1) + self.assertEqual(collector.per_thread_stats[2]["total"], 6) + self.assertAlmostEqual(per_thread_stats[2]["gc_pct"], 10.0, places=1) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py new file mode 100644 index 00000000000..e92b3f45fbc --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -0,0 +1,926 @@ +"""Tests for sampling profiler integration and error handling.""" + +import contextlib +import io +import marshal +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +try: + import _remote_debugging + import profiling.sampling + import profiling.sampling.sample + from profiling.sampling.pstats_collector import PstatsCollector + from profiling.sampling.stack_collector import CollapsedStackCollector + from profiling.sampling.sample import SampleProfiler +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import ( + requires_subprocess, + SHORT_TIMEOUT, +) + +from .helpers import ( + test_subprocess, + close_and_unlink, + skip_if_not_supported, + PROCESS_VM_READV_SUPPORTED, +) +from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo + +# Duration for profiling tests - long enough for process to complete naturally +PROFILING_TIMEOUT = str(int(SHORT_TIMEOUT)) + +# Duration for profiling in tests - short enough to complete quickly +PROFILING_DURATION_SEC = 2 + + +@skip_if_not_supported +@unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", +) +class TestRecursiveFunctionProfiling(unittest.TestCase): + """Test profiling of recursive functions and complex call patterns.""" + + def test_recursive_function_call_counting(self): + """Test that recursive function calls are counted correctly.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Simulate a recursive call pattern: fibonacci(5) calling itself + recursive_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ # First sample: deep in recursion + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo( + "fib.py", 10, "fibonacci" + ), # recursive call + MockFrameInfo( + "fib.py", 10, "fibonacci" + ), # deeper recursion + MockFrameInfo( + "fib.py", 10, "fibonacci" + ), # even deeper + MockFrameInfo("main.py", 5, "main"), # main caller + ], + ) + ], + ), + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ # Second sample: different recursion depth + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo( + "fib.py", 10, "fibonacci" + ), # recursive call + MockFrameInfo("main.py", 5, "main"), # main caller + ], + ) + ], + ), + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ # Third sample: back to deeper recursion + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo("main.py", 5, "main"), + ], + ) + ], + ), + ] + + for frames in recursive_frames: + collector.collect([frames]) + + collector.create_stats() + + # Check that recursive calls are counted properly + fib_key = ("fib.py", 10, "fibonacci") + main_key = ("main.py", 5, "main") + + self.assertIn(fib_key, collector.stats) + self.assertIn(main_key, collector.stats) + + # Fibonacci should have many calls due to recursion + fib_stats = collector.stats[fib_key] + direct_calls, cumulative_calls, tt, ct, callers = fib_stats + + # Should have recorded multiple calls (9 total appearances in samples) + self.assertEqual(cumulative_calls, 9) + self.assertGreater(tt, 0) # Should have some total time + self.assertGreater(ct, 0) # Should have some cumulative time + + # Main should have fewer calls + main_stats = collector.stats[main_key] + main_direct_calls, main_cumulative_calls = main_stats[0], main_stats[1] + self.assertEqual(main_direct_calls, 0) # Never directly executing + self.assertEqual(main_cumulative_calls, 3) # Appears in all 3 samples + + def test_nested_function_hierarchy(self): + """Test profiling of deeply nested function calls.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Simulate a deep call hierarchy + deep_call_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("level1.py", 10, "level1_func"), + MockFrameInfo("level2.py", 20, "level2_func"), + MockFrameInfo("level3.py", 30, "level3_func"), + MockFrameInfo("level4.py", 40, "level4_func"), + MockFrameInfo("level5.py", 50, "level5_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ) + ], + ), + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ # Same hierarchy sampled again + MockFrameInfo("level1.py", 10, "level1_func"), + MockFrameInfo("level2.py", 20, "level2_func"), + MockFrameInfo("level3.py", 30, "level3_func"), + MockFrameInfo("level4.py", 40, "level4_func"), + MockFrameInfo("level5.py", 50, "level5_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ) + ], + ), + ] + + for frames in deep_call_frames: + collector.collect([frames]) + + collector.create_stats() + + # All levels should be recorded + for level in range(1, 6): + key = (f"level{level}.py", level * 10, f"level{level}_func") + self.assertIn(key, collector.stats) + + stats = collector.stats[key] + direct_calls, cumulative_calls, tt, ct, callers = stats + + # Each level should appear in stack twice (2 samples) + self.assertEqual(cumulative_calls, 2) + + # Only level1 (deepest) should have direct calls + if level == 1: + self.assertEqual(direct_calls, 2) + else: + self.assertEqual(direct_calls, 0) + + # Deeper levels should have lower cumulative time than higher levels + # (since they don't include time from functions they call) + if level == 1: # Deepest level with most time + self.assertGreater(ct, 0) + + def test_alternating_call_patterns(self): + """Test profiling with alternating call patterns.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Simulate alternating execution paths + pattern_frames = [ + # Pattern A: path through func_a + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("module.py", 10, "func_a"), + MockFrameInfo("module.py", 30, "shared_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ) + ], + ), + # Pattern B: path through func_b + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("module.py", 20, "func_b"), + MockFrameInfo("module.py", 30, "shared_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ) + ], + ), + # Pattern A again + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("module.py", 10, "func_a"), + MockFrameInfo("module.py", 30, "shared_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ) + ], + ), + # Pattern B again + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("module.py", 20, "func_b"), + MockFrameInfo("module.py", 30, "shared_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ) + ], + ), + ] + + for frames in pattern_frames: + collector.collect([frames]) + + collector.create_stats() + + # Check that both paths are recorded equally + func_a_key = ("module.py", 10, "func_a") + func_b_key = ("module.py", 20, "func_b") + shared_key = ("module.py", 30, "shared_func") + main_key = ("main.py", 5, "main") + + # func_a and func_b should each be directly executing twice + self.assertEqual(collector.stats[func_a_key][0], 2) # direct_calls + self.assertEqual(collector.stats[func_a_key][1], 2) # cumulative_calls + self.assertEqual(collector.stats[func_b_key][0], 2) # direct_calls + self.assertEqual(collector.stats[func_b_key][1], 2) # cumulative_calls + + # shared_func should appear in all samples (4 times) but never directly executing + self.assertEqual(collector.stats[shared_key][0], 0) # direct_calls + self.assertEqual(collector.stats[shared_key][1], 4) # cumulative_calls + + # main should appear in all samples but never directly executing + self.assertEqual(collector.stats[main_key][0], 0) # direct_calls + self.assertEqual(collector.stats[main_key][1], 4) # cumulative_calls + + def test_collapsed_stack_with_recursion(self): + """Test collapsed stack collector with recursive patterns.""" + collector = CollapsedStackCollector(1000) + + # Recursive call pattern + recursive_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + ("factorial.py", 10, "factorial"), + ("factorial.py", 10, "factorial"), # recursive + ("factorial.py", 10, "factorial"), # deeper + ("main.py", 5, "main"), + ], + ) + ], + ), + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + ("factorial.py", 10, "factorial"), + ( + "factorial.py", + 10, + "factorial", + ), # different depth + ("main.py", 5, "main"), + ], + ) + ], + ), + ] + + for frames in recursive_frames: + collector.collect([frames]) + + # Should capture both call paths + self.assertEqual(len(collector.stack_counter), 2) + + # First path should be longer (deeper recursion) than the second + path_tuples = list(collector.stack_counter.keys()) + paths = [p[0] for p in path_tuples] # Extract just the call paths + lengths = [len(p) for p in paths] + self.assertNotEqual(lengths[0], lengths[1]) + + # Both should contain factorial calls + self.assertTrue( + any(any(f[2] == "factorial" for f in p) for p in paths) + ) + + # Verify total occurrences via aggregation + factorial_key = ("factorial.py", 10, "factorial") + main_key = ("main.py", 5, "main") + + def total_occurrences(func): + total = 0 + for (path, thread_id), count in collector.stack_counter.items(): + total += sum(1 for f in path if f == func) * count + return total + + self.assertEqual(total_occurrences(factorial_key), 5) + self.assertEqual(total_occurrences(main_key), 2) + + +# Shared workload functions for test scripts +_WORKLOAD_FUNCTIONS = ''' +def slow_fibonacci(n): + if n <= 1: + return n + return slow_fibonacci(n-1) + slow_fibonacci(n-2) + +def cpu_intensive_work(): + result = 0 + for i in range(10000): + result += i * i + if i % 100 == 0: + result = result % 1000000 + return result + +def do_work(): + iteration = 0 + while True: + if iteration % 2 == 0: + slow_fibonacci(15) + else: + cpu_intensive_work() + iteration += 1 +''' + + +@requires_subprocess() +@skip_if_not_supported +class TestSampleProfilerIntegration(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Test script for use with test_subprocess() - signals when work starts + cls.test_script = _WORKLOAD_FUNCTIONS + ''' +_test_sock.sendall(b"working") +do_work() +''' + # CLI test script - runs for fixed duration (no socket sync) + cls.cli_test_script = ''' +import time +''' + _WORKLOAD_FUNCTIONS.replace( + 'while True:', 'end_time = time.time() + 30\n while time.time() < end_time:' +) + ''' +do_work() +''' + + def test_sampling_basic_functionality(self): + with ( + test_subprocess(self.test_script, wait_for_working=True) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=1000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=PROFILING_DURATION_SEC, + ) + collector.print_stats(show_summary=False) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Basic checks on output + self.assertIn("Captured", output) + self.assertIn("samples", output) + self.assertIn("Profile Stats", output) + + # Should see some of our test functions + self.assertIn("slow_fibonacci", output) + + def test_sampling_with_pstats_export(self): + pstats_out = tempfile.NamedTemporaryFile( + suffix=".pstats", delete=False + ) + self.addCleanup(close_and_unlink, pstats_out) + + with test_subprocess(self.test_script, wait_for_working=True) as subproc: + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=10000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=PROFILING_DURATION_SEC, + ) + collector.export(pstats_out.name) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(pstats_out.name)) + self.assertGreater(os.path.getsize(pstats_out.name), 0) + + # Try to load the stats file + with open(pstats_out.name, "rb") as f: + stats_data = marshal.load(f) + + # Should be a dictionary with the sampled marker + self.assertIsInstance(stats_data, dict) + self.assertIn(("__sampled__",), stats_data) + self.assertTrue(stats_data[("__sampled__",)]) + + # Should have some function data + function_entries = [ + k for k in stats_data.keys() if k != ("__sampled__",) + ] + self.assertGreater(len(function_entries), 0) + + def test_sampling_with_collapsed_export(self): + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with ( + test_subprocess(self.test_script, wait_for_working=True) as subproc, + ): + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = CollapsedStackCollector(1000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=PROFILING_DURATION_SEC, + ) + collector.export(collapsed_file.name) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + # Each line should have format: stack_trace count + for line in lines: + parts = line.rsplit(" ", 1) + self.assertEqual(len(parts), 2) + + stack_trace, count_str = parts + self.assertGreater(len(stack_trace), 0) + self.assertTrue(count_str.isdigit()) + self.assertGreater(int(count_str), 0) + + # Stack trace should contain semicolon-separated entries + if ";" in stack_trace: + stack_parts = stack_trace.split(";") + for part in stack_parts: + # Each part should be file:function:line + self.assertIn(":", part) + + def test_sampling_all_threads(self): + with ( + test_subprocess(self.test_script, wait_for_working=True) as subproc, + # Suppress profiler output + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=10000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=PROFILING_DURATION_SEC, + all_threads=True, + ) + collector.print_stats(show_summary=False) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + # Just verify that sampling completed without error + # We're not testing output format here + + def test_sample_target_script(self): + script_file = tempfile.NamedTemporaryFile(delete=False) + script_file.write(self.cli_test_script.encode("utf-8")) + script_file.flush() + self.addCleanup(close_and_unlink, script_file) + + # Sample for PROFILING_DURATION_SEC seconds + test_args = [ + "profiling.sampling.sample", "run", + "-d", str(PROFILING_DURATION_SEC), + script_file.name + ] + + with ( + mock.patch("sys.argv", test_args), + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + from profiling.sampling.cli import main + main() + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Basic checks on output + self.assertIn("Captured", output) + self.assertIn("samples", output) + self.assertIn("Profile Stats", output) + + # Should see some of our test functions + self.assertIn("slow_fibonacci", output) + + def test_sample_target_module(self): + tempdir = tempfile.TemporaryDirectory(delete=False) + self.addCleanup(lambda x: shutil.rmtree(x), tempdir.name) + + module_path = os.path.join(tempdir.name, "test_module.py") + + with open(module_path, "w") as f: + f.write(self.cli_test_script) + + test_args = [ + "profiling.sampling.cli", + "run", + "-d", + str(PROFILING_DURATION_SEC), + "-m", + "test_module", + ] + + with ( + mock.patch("sys.argv", test_args), + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + # Change to temp directory so subprocess can find the module + contextlib.chdir(tempdir.name), + ): + try: + from profiling.sampling.cli import main + main() + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Basic checks on output + self.assertIn("Captured", output) + self.assertIn("samples", output) + self.assertIn("Profile Stats", output) + + # Should see some of our test functions + self.assertIn("slow_fibonacci", output) + + +@skip_if_not_supported +@unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", +) +class TestSampleProfilerErrorHandling(unittest.TestCase): + def test_invalid_pid(self): + with self.assertRaises((OSError, RuntimeError)): + collector = PstatsCollector(sample_interval_usec=100, skip_idle=False) + profiling.sampling.sample.sample(-1, collector, duration_sec=1) + + def test_process_dies_during_sampling(self): + # Use wait_for_working=False since this simple script doesn't send "working" + with test_subprocess( + "import time; time.sleep(0.5); exit()", + wait_for_working=False + ) as subproc: + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=50000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=2, # Longer than process lifetime + ) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + output = captured_output.getvalue() + + self.assertIn("Error rate", output) + + def test_is_process_running(self): + # Use wait_for_working=False since this simple script doesn't send "working" + with test_subprocess( + "import time; time.sleep(1000)", + wait_for_working=False + ) as subproc: + try: + profiler = SampleProfiler( + pid=subproc.process.pid, + sample_interval_usec=1000, + all_threads=False, + ) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + self.assertTrue(profiler._is_process_running()) + self.assertIsNotNone(profiler.unwinder.get_stack_trace()) + subproc.process.kill() + subproc.process.wait() + self.assertRaises( + ProcessLookupError, profiler.unwinder.get_stack_trace + ) + + # Exit the context manager to ensure the process is terminated + self.assertFalse(profiler._is_process_running()) + self.assertRaises( + ProcessLookupError, profiler.unwinder.get_stack_trace + ) + + @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") + def test_esrch_signal_handling(self): + # Use wait_for_working=False since this simple script doesn't send "working" + with test_subprocess( + "import time; time.sleep(1000)", + wait_for_working=False + ) as subproc: + try: + unwinder = _remote_debugging.RemoteUnwinder( + subproc.process.pid + ) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + initial_trace = unwinder.get_stack_trace() + self.assertIsNotNone(initial_trace) + + subproc.process.kill() + + # Wait for the process to die and try to get another trace + subproc.process.wait() + + with self.assertRaises(ProcessLookupError): + unwinder.get_stack_trace() + + def test_script_error_treatment(self): + script_file = tempfile.NamedTemporaryFile( + "w", delete=False, suffix=".py" + ) + script_file.write("open('nonexistent_file.txt')\n") + script_file.close() + self.addCleanup(os.unlink, script_file.name) + + result = subprocess.run( + [ + sys.executable, + "-m", + "profiling.sampling.cli", + "run", + "-d", + "1", + script_file.name, + ], + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + + if "PermissionError" in output: + self.skipTest("Insufficient permissions for remote profiling") + self.assertNotIn("Script file not found", output) + self.assertIn( + "No such file or directory: 'nonexistent_file.txt'", output + ) + + def test_live_incompatible_with_pstats_options(self): + """Test that --live is incompatible with individual pstats options.""" + test_cases = [ + (["--sort", "tottime"], "--sort"), + (["--limit", "30"], "--limit"), + (["--no-summary"], "--no-summary"), + ] + + for args, expected_flag in test_cases: + with self.subTest(args=args): + test_args = ["profiling.sampling.cli", "run", "--live"] + args + ["test.py"] + with mock.patch("sys.argv", test_args): + with self.assertRaises(SystemExit) as cm: + from profiling.sampling.cli import main + main() + self.assertNotEqual(cm.exception.code, 0) + + def test_live_incompatible_with_multiple_pstats_options(self): + """Test that --live is incompatible with multiple pstats options.""" + test_args = [ + "profiling.sampling.cli", "run", "--live", + "--sort", "cumtime", "--limit", "25", "--no-summary", "test.py" + ] + + with mock.patch("sys.argv", test_args): + with self.assertRaises(SystemExit) as cm: + from profiling.sampling.cli import main + main() + self.assertNotEqual(cm.exception.code, 0) + + def test_live_incompatible_with_pstats_default_values(self): + """Test that --live blocks pstats options even with default values.""" + # Test with --sort=nsamples (the default value) + test_args = ["profiling.sampling.cli", "run", "--live", "--sort=nsamples", "test.py"] + + with mock.patch("sys.argv", test_args): + with self.assertRaises(SystemExit) as cm: + from profiling.sampling.cli import main + main() + self.assertNotEqual(cm.exception.code, 0) + + # Test with --limit=15 (the default value) + test_args = ["profiling.sampling.cli", "run", "--live", "--limit=15", "test.py"] + + with mock.patch("sys.argv", test_args): + with self.assertRaises(SystemExit) as cm: + from profiling.sampling.cli import main + main() + self.assertNotEqual(cm.exception.code, 0) + + +@requires_subprocess() +@skip_if_not_supported +@unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", +) +class TestAsyncAwareProfilingIntegration(unittest.TestCase): + """Integration tests for async-aware profiling mode.""" + + @classmethod + def setUpClass(cls): + # Async test script that runs indefinitely until killed. + # Sends "working" signal AFTER tasks are created and scheduled. + cls.async_script = ''' +import asyncio + +async def sleeping_leaf(): + while True: + await asyncio.sleep(0.02) + +async def cpu_leaf(): + total = 0 + while True: + for i in range(10000): + total += i * i + await asyncio.sleep(0) + +async def supervisor(): + tasks = [ + asyncio.create_task(sleeping_leaf(), name="Sleeper-0"), + asyncio.create_task(sleeping_leaf(), name="Sleeper-1"), + asyncio.create_task(sleeping_leaf(), name="Sleeper-2"), + asyncio.create_task(cpu_leaf(), name="Worker"), + ] + await asyncio.sleep(0) # Let tasks get scheduled + _test_sock.sendall(b"working") + await asyncio.gather(*tasks) + +asyncio.run(supervisor()) +''' + + def _collect_async_samples(self, async_aware_mode): + """Helper to collect samples and count function occurrences. + + Returns a dict mapping function names to their sample counts. + """ + with test_subprocess(self.async_script, wait_for_working=True) as subproc: + try: + collector = CollapsedStackCollector(1000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=PROFILING_DURATION_SEC, + async_aware=async_aware_mode, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + # Count samples per function from collapsed stacks + # stack_counter keys are (call_tree, thread_id) where call_tree + # is a tuple of (file, line, func) tuples + func_samples = {} + total = 0 + for (call_tree, _thread_id), count in collector.stack_counter.items(): + total += count + for _file, _line, func in call_tree: + func_samples[func] = func_samples.get(func, 0) + count + + func_samples["_total"] = total + return func_samples + + def test_async_aware_all_sees_sleeping_and_running_tasks(self): + """Test that async_aware='all' captures both sleeping and CPU-running tasks. + + Task tree structure: + main + └── supervisor + ├── Sleeper-0 (sleeping_leaf) + ├── Sleeper-1 (sleeping_leaf) + ├── Sleeper-2 (sleeping_leaf) + └── Worker (cpu_leaf) + + async_aware='all' should see ALL 4 leaf tasks in the output. + """ + samples = self._collect_async_samples("all") + + self.assertGreater(samples["_total"], 0, "Should have collected samples") + self.assertIn("sleeping_leaf", samples) + self.assertIn("cpu_leaf", samples) + self.assertIn("supervisor", samples) + + def test_async_aware_running_sees_only_cpu_task(self): + """Test that async_aware='running' only captures the actively running task. + + Task tree structure: + main + └── supervisor + ├── Sleeper-0 (sleeping_leaf) - NOT visible in 'running' + ├── Sleeper-1 (sleeping_leaf) - NOT visible in 'running' + ├── Sleeper-2 (sleeping_leaf) - NOT visible in 'running' + └── Worker (cpu_leaf) - VISIBLE in 'running' + + async_aware='running' should only see the Worker task doing CPU work. + """ + samples = self._collect_async_samples("running") + + total = samples["_total"] + cpu_leaf_samples = samples.get("cpu_leaf", 0) + + self.assertGreater(total, 0, "Should have collected some samples") + self.assertGreater(cpu_leaf_samples, 0, "cpu_leaf should appear in samples") + + # cpu_leaf should have at least 90% of samples (typically 99%+) + # sleeping_leaf may occasionally appear with very few samples (< 1%) + # when tasks briefly wake up to check sleep timers + cpu_percentage = (cpu_leaf_samples / total) * 100 + self.assertGreater(cpu_percentage, 90.0, + f"cpu_leaf should dominate samples in 'running' mode, " + f"got {cpu_percentage:.1f}% ({cpu_leaf_samples}/{total})") diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_core.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_core.py new file mode 100644 index 00000000000..04e6cd2f1fc --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_core.py @@ -0,0 +1,477 @@ +"""Core functionality tests for LiveStatsCollector. + +Tests for path simplification, frame processing, collect method, +statistics building, sorting, and formatting. +""" + +import os +import unittest +from test.support import requires +from test.support.import_helper import import_module + +# Only run these tests if curses is available +requires("curses") +curses = import_module("curses") + +from profiling.sampling.live_collector import LiveStatsCollector, MockDisplay +from profiling.sampling.constants import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, +) +from ._live_collector_helpers import ( + MockFrameInfo, + MockThreadInfo, + MockInterpreterInfo, +) + + +class TestLiveStatsCollectorPathSimplification(unittest.TestCase): + """Tests for path simplification functionality.""" + + def test_simplify_stdlib_path(self): + """Test simplification of standard library paths.""" + collector = LiveStatsCollector(1000) + # Get actual os module path + os_file = os.__file__ + if os_file: + stdlib_dir = os.path.dirname(os.path.abspath(os_file)) + test_path = os.path.join(stdlib_dir, "json", "decoder.py") + simplified = collector.simplify_path(test_path) + # Should remove the stdlib prefix + self.assertNotIn(stdlib_dir, simplified) + self.assertIn("json", simplified) + + def test_simplify_unknown_path(self): + """Test that unknown paths are returned unchanged.""" + collector = LiveStatsCollector(1000) + test_path = "/some/unknown/path/file.py" + simplified = collector.simplify_path(test_path) + self.assertEqual(simplified, test_path) + + +class TestLiveStatsCollectorFrameProcessing(unittest.TestCase): + """Tests for frame processing functionality.""" + + def test_process_single_frame(self): + """Test processing a single frame.""" + collector = LiveStatsCollector(1000) + frames = [MockFrameInfo("test.py", 10, "test_func")] + collector.process_frames(frames) + + location = ("test.py", 10, "test_func") + self.assertEqual(collector.result[location]["direct_calls"], 1) + self.assertEqual(collector.result[location]["cumulative_calls"], 1) + + def test_process_multiple_frames(self): + """Test processing a stack of multiple frames.""" + collector = LiveStatsCollector(1000) + frames = [ + MockFrameInfo("test.py", 10, "inner_func"), + MockFrameInfo("test.py", 20, "middle_func"), + MockFrameInfo("test.py", 30, "outer_func"), + ] + collector.process_frames(frames) + + # Top frame (inner_func) should have both direct and cumulative + inner_loc = ("test.py", 10, "inner_func") + self.assertEqual(collector.result[inner_loc]["direct_calls"], 1) + self.assertEqual(collector.result[inner_loc]["cumulative_calls"], 1) + + # Other frames should only have cumulative + middle_loc = ("test.py", 20, "middle_func") + self.assertEqual(collector.result[middle_loc]["direct_calls"], 0) + self.assertEqual(collector.result[middle_loc]["cumulative_calls"], 1) + + outer_loc = ("test.py", 30, "outer_func") + self.assertEqual(collector.result[outer_loc]["direct_calls"], 0) + self.assertEqual(collector.result[outer_loc]["cumulative_calls"], 1) + + def test_process_empty_frames(self): + """Test processing empty frames list.""" + collector = LiveStatsCollector(1000) + collector.process_frames([]) + # Should not raise an error and result should remain empty + self.assertEqual(len(collector.result), 0) + + def test_process_frames_accumulation(self): + """Test that multiple calls accumulate correctly.""" + collector = LiveStatsCollector(1000) + frames = [MockFrameInfo("test.py", 10, "test_func")] + + collector.process_frames(frames) + collector.process_frames(frames) + collector.process_frames(frames) + + location = ("test.py", 10, "test_func") + self.assertEqual(collector.result[location]["direct_calls"], 3) + self.assertEqual(collector.result[location]["cumulative_calls"], 3) + + def test_process_frames_with_thread_id(self): + """Test processing frames with per-thread tracking.""" + collector = LiveStatsCollector(1000) + frames = [MockFrameInfo("test.py", 10, "test_func")] + + # Process frames with thread_id + collector.process_frames(frames, thread_id=123) + + # Check aggregated result + location = ("test.py", 10, "test_func") + self.assertEqual(collector.result[location]["direct_calls"], 1) + self.assertEqual(collector.result[location]["cumulative_calls"], 1) + + # Check per-thread result + self.assertIn(123, collector.per_thread_data) + self.assertEqual( + collector.per_thread_data[123].result[location]["direct_calls"], 1 + ) + self.assertEqual( + collector.per_thread_data[123].result[location]["cumulative_calls"], 1 + ) + + def test_process_frames_multiple_threads(self): + """Test processing frames from multiple threads.""" + collector = LiveStatsCollector(1000) + frames1 = [MockFrameInfo("test.py", 10, "test_func")] + frames2 = [MockFrameInfo("test.py", 20, "other_func")] + + # Process frames from different threads + collector.process_frames(frames1, thread_id=123) + collector.process_frames(frames2, thread_id=456) + + # Check that both threads have their own data + self.assertIn(123, collector.per_thread_data) + self.assertIn(456, collector.per_thread_data) + + loc1 = ("test.py", 10, "test_func") + loc2 = ("test.py", 20, "other_func") + + # Thread 123 should only have func1 + self.assertEqual( + collector.per_thread_data[123].result[loc1]["direct_calls"], 1 + ) + self.assertNotIn(loc2, collector.per_thread_data[123].result) + + # Thread 456 should only have func2 + self.assertEqual( + collector.per_thread_data[456].result[loc2]["direct_calls"], 1 + ) + self.assertNotIn(loc1, collector.per_thread_data[456].result) + + +class TestLiveStatsCollectorCollect(unittest.TestCase): + """Tests for the collect method.""" + + def test_collect_initializes_start_time(self): + """Test that collect initializes start_time on first call.""" + collector = LiveStatsCollector(1000) + self.assertIsNone(collector.start_time) + + # Create mock stack frames + thread_info = MockThreadInfo(123, []) + interpreter_info = MockInterpreterInfo(0, [thread_info]) + stack_frames = [interpreter_info] + + collector.collect(stack_frames) + self.assertIsNotNone(collector.start_time) + + def test_collect_increments_sample_count(self): + """Test that collect increments total_samples.""" + collector = LiveStatsCollector(1000) + thread_info = MockThreadInfo(123, []) + interpreter_info = MockInterpreterInfo(0, [thread_info]) + stack_frames = [interpreter_info] + + self.assertEqual(collector.total_samples, 0) + collector.collect(stack_frames) + self.assertEqual(collector.total_samples, 1) + collector.collect(stack_frames) + self.assertEqual(collector.total_samples, 2) + + def test_collect_with_frames(self): + """Test collect with actual frame data.""" + collector = LiveStatsCollector(1000) + frames = [MockFrameInfo("test.py", 10, "test_func")] + thread_info = MockThreadInfo(123, frames) + interpreter_info = MockInterpreterInfo(0, [thread_info]) + stack_frames = [interpreter_info] + + collector.collect(stack_frames) + + location = ("test.py", 10, "test_func") + self.assertEqual(collector.result[location]["direct_calls"], 1) + self.assertEqual(collector.successful_samples, 1) + self.assertEqual(collector.failed_samples, 0) + + def test_collect_with_empty_frames(self): + """Test collect with empty frames.""" + collector = LiveStatsCollector(1000) + thread_info = MockThreadInfo(123, []) + interpreter_info = MockInterpreterInfo(0, [thread_info]) + stack_frames = [interpreter_info] + + collector.collect(stack_frames) + + # Empty frames still count as successful since collect() was called successfully + self.assertEqual(collector.successful_samples, 1) + self.assertEqual(collector.failed_samples, 0) + + def test_collect_skip_idle_threads(self): + """Test that idle threads are skipped when skip_idle=True.""" + collector = LiveStatsCollector(1000, skip_idle=True) + + frames = [MockFrameInfo("test.py", 10, "test_func")] + running_thread = MockThreadInfo( + 123, frames, status=THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU + ) + idle_thread = MockThreadInfo(124, frames, status=0) # No flags = idle + interpreter_info = MockInterpreterInfo( + 0, [running_thread, idle_thread] + ) + stack_frames = [interpreter_info] + + collector.collect(stack_frames) + + # Only one thread should be processed + location = ("test.py", 10, "test_func") + self.assertEqual(collector.result[location]["direct_calls"], 1) + + def test_collect_multiple_threads(self): + """Test collect with multiple threads.""" + collector = LiveStatsCollector(1000) + + frames1 = [MockFrameInfo("test1.py", 10, "func1")] + frames2 = [MockFrameInfo("test2.py", 20, "func2")] + thread1 = MockThreadInfo(123, frames1) + thread2 = MockThreadInfo(124, frames2) + interpreter_info = MockInterpreterInfo(0, [thread1, thread2]) + stack_frames = [interpreter_info] + + collector.collect(stack_frames) + + loc1 = ("test1.py", 10, "func1") + loc2 = ("test2.py", 20, "func2") + self.assertEqual(collector.result[loc1]["direct_calls"], 1) + self.assertEqual(collector.result[loc2]["direct_calls"], 1) + + # Check thread IDs are tracked + self.assertIn(123, collector.thread_ids) + self.assertIn(124, collector.thread_ids) + + +class TestLiveStatsCollectorStatisticsBuilding(unittest.TestCase): + """Tests for statistics building and sorting.""" + + def setUp(self): + """Set up test fixtures.""" + self.collector = LiveStatsCollector(1000) + # Add some test data + self.collector.result[("file1.py", 10, "func1")] = { + "direct_calls": 100, + "cumulative_calls": 150, + "total_rec_calls": 0, + } + self.collector.result[("file2.py", 20, "func2")] = { + "direct_calls": 50, + "cumulative_calls": 200, + "total_rec_calls": 0, + } + self.collector.result[("file3.py", 30, "func3")] = { + "direct_calls": 75, + "cumulative_calls": 75, + "total_rec_calls": 0, + } + self.collector.total_samples = 300 + + def test_build_stats_list(self): + """Test that stats list is built correctly.""" + stats_list = self.collector.build_stats_list() + self.assertEqual(len(stats_list), 3) + + # Check that all expected keys are present + for stat in stats_list: + self.assertIn("func", stat) + self.assertIn("direct_calls", stat) + self.assertIn("cumulative_calls", stat) + self.assertIn("total_time", stat) + self.assertIn("cumulative_time", stat) + + def test_sort_by_nsamples(self): + """Test sorting by number of samples.""" + self.collector.sort_by = "nsamples" + stats_list = self.collector.build_stats_list() + + # Should be sorted by direct_calls descending + self.assertEqual(stats_list[0]["func"][2], "func1") # 100 samples + self.assertEqual(stats_list[1]["func"][2], "func3") # 75 samples + self.assertEqual(stats_list[2]["func"][2], "func2") # 50 samples + + def test_sort_by_tottime(self): + """Test sorting by total time.""" + self.collector.sort_by = "tottime" + stats_list = self.collector.build_stats_list() + + # Should be sorted by total_time descending + # total_time = direct_calls * sample_interval_sec + self.assertEqual(stats_list[0]["func"][2], "func1") + self.assertEqual(stats_list[1]["func"][2], "func3") + self.assertEqual(stats_list[2]["func"][2], "func2") + + def test_sort_by_cumtime(self): + """Test sorting by cumulative time.""" + self.collector.sort_by = "cumtime" + stats_list = self.collector.build_stats_list() + + # Should be sorted by cumulative_time descending + self.assertEqual(stats_list[0]["func"][2], "func2") # 200 cumulative + self.assertEqual(stats_list[1]["func"][2], "func1") # 150 cumulative + self.assertEqual(stats_list[2]["func"][2], "func3") # 75 cumulative + + def test_sort_by_sample_pct(self): + """Test sorting by sample percentage.""" + self.collector.sort_by = "sample_pct" + stats_list = self.collector.build_stats_list() + + # Should be sorted by percentage of direct_calls + self.assertEqual(stats_list[0]["func"][2], "func1") # 33.3% + self.assertEqual(stats_list[1]["func"][2], "func3") # 25% + self.assertEqual(stats_list[2]["func"][2], "func2") # 16.7% + + def test_sort_by_cumul_pct(self): + """Test sorting by cumulative percentage.""" + self.collector.sort_by = "cumul_pct" + stats_list = self.collector.build_stats_list() + + # Should be sorted by percentage of cumulative_calls + self.assertEqual(stats_list[0]["func"][2], "func2") # 66.7% + self.assertEqual(stats_list[1]["func"][2], "func1") # 50% + self.assertEqual(stats_list[2]["func"][2], "func3") # 25% + + +class TestLiveStatsCollectorSortCycle(unittest.TestCase): + """Tests for sort mode cycling.""" + + def test_cycle_sort_from_nsamples(self): + """Test cycling from nsamples.""" + collector = LiveStatsCollector(1000, sort_by="nsamples") + collector._cycle_sort() + self.assertEqual(collector.sort_by, "sample_pct") + + def test_cycle_sort_from_sample_pct(self): + """Test cycling from sample_pct.""" + collector = LiveStatsCollector(1000, sort_by="sample_pct") + collector._cycle_sort() + self.assertEqual(collector.sort_by, "tottime") + + def test_cycle_sort_from_tottime(self): + """Test cycling from tottime.""" + collector = LiveStatsCollector(1000, sort_by="tottime") + collector._cycle_sort() + self.assertEqual(collector.sort_by, "cumul_pct") + + def test_cycle_sort_from_cumul_pct(self): + """Test cycling from cumul_pct.""" + collector = LiveStatsCollector(1000, sort_by="cumul_pct") + collector._cycle_sort() + self.assertEqual(collector.sort_by, "cumtime") + + def test_cycle_sort_from_cumtime(self): + """Test cycling from cumtime back to nsamples.""" + collector = LiveStatsCollector(1000, sort_by="cumtime") + collector._cycle_sort() + self.assertEqual(collector.sort_by, "nsamples") + + def test_cycle_sort_invalid_mode(self): + """Test cycling from invalid mode resets to nsamples.""" + collector = LiveStatsCollector(1000) + collector.sort_by = "invalid_mode" + collector._cycle_sort() + self.assertEqual(collector.sort_by, "nsamples") + + def test_cycle_sort_backward_from_nsamples(self): + """Test cycling backward from nsamples goes to cumtime.""" + collector = LiveStatsCollector(1000, sort_by="nsamples") + collector._cycle_sort(reverse=True) + self.assertEqual(collector.sort_by, "cumtime") + + def test_cycle_sort_backward_from_cumtime(self): + """Test cycling backward from cumtime goes to cumul_pct.""" + collector = LiveStatsCollector(1000, sort_by="cumtime") + collector._cycle_sort(reverse=True) + self.assertEqual(collector.sort_by, "cumul_pct") + + def test_cycle_sort_backward_from_sample_pct(self): + """Test cycling backward from sample_pct goes to nsamples.""" + collector = LiveStatsCollector(1000, sort_by="sample_pct") + collector._cycle_sort(reverse=True) + self.assertEqual(collector.sort_by, "nsamples") + + def test_input_lowercase_s_cycles_forward(self): + """Test that lowercase 's' cycles forward.""" + display = MockDisplay() + collector = LiveStatsCollector( + 1000, sort_by="nsamples", display=display + ) + + display.simulate_input(ord("s")) + collector._handle_input() + + self.assertEqual(collector.sort_by, "sample_pct") + + def test_input_uppercase_s_cycles_backward(self): + """Test that uppercase 'S' cycles backward.""" + display = MockDisplay() + collector = LiveStatsCollector( + 1000, sort_by="nsamples", display=display + ) + + display.simulate_input(ord("S")) + collector._handle_input() + + self.assertEqual(collector.sort_by, "cumtime") + + +class TestLiveStatsCollectorFormatting(unittest.TestCase): + """Tests for formatting methods.""" + + def test_format_uptime_seconds(self): + """Test uptime formatting for seconds only.""" + collector = LiveStatsCollector(1000, display=MockDisplay()) + colors = collector._setup_colors() + collector._initialize_widgets(colors) + self.assertEqual(collector.header_widget.format_uptime(45), "0m45s") + + def test_format_uptime_minutes(self): + """Test uptime formatting for minutes.""" + collector = LiveStatsCollector(1000, display=MockDisplay()) + colors = collector._setup_colors() + collector._initialize_widgets(colors) + self.assertEqual(collector.header_widget.format_uptime(125), "2m05s") + + def test_format_uptime_hours(self): + """Test uptime formatting for hours.""" + collector = LiveStatsCollector(1000, display=MockDisplay()) + colors = collector._setup_colors() + collector._initialize_widgets(colors) + self.assertEqual( + collector.header_widget.format_uptime(3661), "1h01m01s" + ) + + def test_format_uptime_large_values(self): + """Test uptime formatting for large time values.""" + collector = LiveStatsCollector(1000, display=MockDisplay()) + colors = collector._setup_colors() + collector._initialize_widgets(colors) + self.assertEqual( + collector.header_widget.format_uptime(86400), "24h00m00s" + ) + + def test_format_uptime_zero(self): + """Test uptime formatting for zero.""" + collector = LiveStatsCollector(1000, display=MockDisplay()) + colors = collector._setup_colors() + collector._initialize_widgets(colors) + self.assertEqual(collector.header_widget.format_uptime(0), "0m00s") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py new file mode 100644 index 00000000000..a5870366552 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py @@ -0,0 +1,1238 @@ +"""Interactive controls tests for LiveStatsCollector. + +Tests for interactive controls, filtering, filter input, and thread navigation. +""" + +import time +import unittest +from test.support import requires +from test.support.import_helper import import_module + +# Only run these tests if curses is available +requires("curses") +curses = import_module("curses") + +from profiling.sampling.live_collector import LiveStatsCollector, MockDisplay +from profiling.sampling.constants import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, +) +from ._live_collector_helpers import ( + MockFrameInfo, + MockThreadInfo, + MockInterpreterInfo, +) + + +class TestLiveCollectorInteractiveControls(unittest.TestCase): + """Tests for interactive control features.""" + + def setUp(self): + """Set up collector with mock display.""" + self.display = MockDisplay(height=40, width=160) + self.collector = LiveStatsCollector( + 1000, pid=12345, display=self.display + ) + self.collector.start_time = time.perf_counter() + # Set a consistent display update interval for tests + self.collector.display_update_interval = 0.1 + + def tearDown(self): + """Clean up after test.""" + pass + + def test_pause_functionality(self): + """Test pause/resume functionality.""" + self.assertFalse(self.collector.paused) + + # Simulate 'p' key press + self.display.simulate_input(ord("p")) + self.collector._handle_input() + + self.assertTrue(self.collector.paused) + + # Press 'p' again to resume + self.display.simulate_input(ord("p")) + self.collector._handle_input() + + self.assertFalse(self.collector.paused) + + def test_pause_stops_ui_updates(self): + """Test that pausing stops UI updates but profiling continues.""" + # Add some data + self.collector.total_samples = 10 + self.collector.result[("test.py", 1, "func")] = { + "direct_calls": 5, + "cumulative_calls": 10, + "total_rec_calls": 0, + } + + # Pause + self.collector.paused = True + + # Simulate a collect call (profiling continues) + thread_info = MockThreadInfo(123, []) + interpreter_info = MockInterpreterInfo(0, [thread_info]) + stack_frames = [interpreter_info] + + initial_samples = self.collector.total_samples + self.collector.collect(stack_frames) + + # Samples should still increment + self.assertEqual(self.collector.total_samples, initial_samples + 1) + + # But display should not have been updated (buffer stays clear) + self.display.cleared = False + self.collector.collect(stack_frames) + self.assertFalse( + self.display.cleared, "Display should not update when paused" + ) + + def test_reset_stats(self): + """Test reset statistics functionality.""" + # Add some stats + self.collector.total_samples = 100 + self.collector.successful_samples = 90 + self.collector.failed_samples = 10 + self.collector.result[("test.py", 1, "func")] = { + "direct_calls": 50, + "cumulative_calls": 75, + "total_rec_calls": 0, + } + + # Reset + self.collector.reset_stats() + + self.assertEqual(self.collector.total_samples, 0) + self.assertEqual(self.collector.successful_samples, 0) + self.assertEqual(self.collector.failed_samples, 0) + self.assertEqual(len(self.collector.result), 0) + + def test_increase_refresh_rate(self): + """Test increasing refresh rate (faster updates).""" + initial_interval = self.collector.display_update_interval + + # Simulate '+' key press (faster = smaller interval) + self.display.simulate_input(ord("+")) + self.collector._handle_input() + + self.assertLess(self.collector.display_update_interval, initial_interval) + + def test_decrease_refresh_rate(self): + """Test decreasing refresh rate (slower updates).""" + initial_interval = self.collector.display_update_interval + + # Simulate '-' key press (slower = larger interval) + self.display.simulate_input(ord("-")) + self.collector._handle_input() + + self.assertGreater(self.collector.display_update_interval, initial_interval) + + def test_refresh_rate_minimum(self): + """Test that refresh rate has a minimum (max speed).""" + self.collector.display_update_interval = 0.05 # Set to minimum + + # Try to go faster + self.display.simulate_input(ord("+")) + self.collector._handle_input() + + # Should stay at minimum + self.assertEqual(self.collector.display_update_interval, 0.05) + + def test_refresh_rate_maximum(self): + """Test that refresh rate has a maximum (min speed).""" + self.collector.display_update_interval = 1.0 # Set to maximum + + # Try to go slower + self.display.simulate_input(ord("-")) + self.collector._handle_input() + + # Should stay at maximum + self.assertEqual(self.collector.display_update_interval, 1.0) + + def test_help_toggle(self): + """Test help screen toggle.""" + self.assertFalse(self.collector.show_help) + + # Show help + self.display.simulate_input(ord("h")) + self.collector._handle_input() + + self.assertTrue(self.collector.show_help) + + # Pressing any key closes help + self.display.simulate_input(ord("x")) + self.collector._handle_input() + + self.assertFalse(self.collector.show_help) + + def test_help_with_question_mark(self): + """Test help screen with '?' key.""" + self.display.simulate_input(ord("?")) + self.collector._handle_input() + + self.assertTrue(self.collector.show_help) + + def test_help_dismiss_with_q_does_not_quit(self): + """Test that pressing 'q' while help is shown only closes help, not quit""" + self.assertFalse(self.collector.show_help) + self.display.simulate_input(ord("h")) + self.collector._handle_input() + self.assertTrue(self.collector.show_help) + + self.display.simulate_input(ord("q")) + self.collector._handle_input() + + self.assertFalse(self.collector.show_help) + self.assertTrue(self.collector.running) + + def test_filter_clear(self): + """Test clearing filter.""" + self.collector.filter_pattern = "test" + + # Clear filter + self.display.simulate_input(ord("c")) + self.collector._handle_input() + + self.assertIsNone(self.collector.filter_pattern) + + def test_filter_clear_when_none(self): + """Test clearing filter when no filter is set.""" + self.assertIsNone(self.collector.filter_pattern) + + # Should not crash + self.display.simulate_input(ord("c")) + self.collector._handle_input() + + self.assertIsNone(self.collector.filter_pattern) + + def test_paused_status_in_footer(self): + """Test that paused status appears in footer.""" + self.collector.total_samples = 10 + self.collector.paused = True + + self.collector._update_display() + + # Check that PAUSED appears in display + self.assertTrue(self.display.contains_text("PAUSED")) + + def test_filter_status_in_footer(self): + """Test that filter status appears in footer.""" + self.collector.total_samples = 10 + self.collector.filter_pattern = "mytest" + + self.collector._update_display() + + # Check that filter info appears + self.assertTrue(self.display.contains_text("Filter")) + + def test_help_screen_display(self): + """Test that help screen is displayed.""" + self.collector.show_help = True + + self.collector._update_display() + + # Check for help content + self.assertTrue(self.display.contains_text("Interactive Commands")) + + def test_pause_uppercase(self): + """Test pause with uppercase 'P' key.""" + self.assertFalse(self.collector.paused) + + self.display.simulate_input(ord("P")) + self.collector._handle_input() + + self.assertTrue(self.collector.paused) + + def test_help_uppercase(self): + """Test help with uppercase 'H' key.""" + self.assertFalse(self.collector.show_help) + + self.display.simulate_input(ord("H")) + self.collector._handle_input() + + self.assertTrue(self.collector.show_help) + + def test_reset_lowercase(self): + """Test reset with lowercase 'r' key.""" + # Add some stats + self.collector.total_samples = 100 + self.collector.result[("test.py", 1, "func")] = { + "direct_calls": 50, + "cumulative_calls": 75, + "total_rec_calls": 0, + } + + self.display.simulate_input(ord("r")) + self.collector._handle_input() + + self.assertEqual(self.collector.total_samples, 0) + self.assertEqual(len(self.collector.result), 0) + + def test_reset_uppercase(self): + """Test reset with uppercase 'R' key.""" + self.collector.total_samples = 100 + + self.display.simulate_input(ord("R")) + self.collector._handle_input() + + self.assertEqual(self.collector.total_samples, 0) + + def test_filter_clear_uppercase(self): + """Test clearing filter with uppercase 'C' key.""" + self.collector.filter_pattern = "test" + + self.display.simulate_input(ord("C")) + self.collector._handle_input() + + self.assertIsNone(self.collector.filter_pattern) + + def test_increase_refresh_rate_with_equals(self): + """Test increasing refresh rate with '=' key.""" + initial_interval = self.collector.display_update_interval + + # Simulate '=' key press (alternative to '+') + self.display.simulate_input(ord("=")) + self.collector._handle_input() + + self.assertLess(self.collector.display_update_interval, initial_interval) + + def test_decrease_refresh_rate_with_underscore(self): + """Test decreasing refresh rate with '_' key.""" + initial_interval = self.collector.display_update_interval + + # Simulate '_' key press (alternative to '-') + self.display.simulate_input(ord("_")) + self.collector._handle_input() + + self.assertGreater(self.collector.display_update_interval, initial_interval) + + def test_finished_state_displays_banner(self): + """Test that finished state shows prominent banner.""" + # Add some sample data + thread_info = MockThreadInfo( + 123, + [ + MockFrameInfo("test.py", 10, "work"), + MockFrameInfo("test.py", 20, "main"), + ], + ) + interpreter_info = MockInterpreterInfo(0, [thread_info]) + stack_frames = [interpreter_info] + self.collector.collect(stack_frames) + + # Mark as finished + self.collector.mark_finished() + + # Check that finished flag is set + self.assertTrue(self.collector.finished) + + # Check that the banner message is displayed + self.assertTrue(self.display.contains_text("PROFILING COMPLETE")) + self.assertTrue(self.display.contains_text("Press 'q' to Quit")) + + def test_finished_state_allows_ui_controls(self): + """Test that finished state allows UI controls but prioritizes quit.""" + self.collector.finished = True + self.collector.running = True + + # Try pressing 's' (sort) - should work and trigger display update + original_sort = self.collector.sort_by + self.display.simulate_input(ord("s")) + self.collector._handle_input() + self.assertTrue(self.collector.running) # Still running + self.assertNotEqual(self.collector.sort_by, original_sort) # Sort changed + + # Try pressing 'p' (pause) - should work + self.display.simulate_input(ord("p")) + self.collector._handle_input() + self.assertTrue(self.collector.running) # Still running + self.assertTrue(self.collector.paused) # Now paused + + # Try pressing 'r' (reset) - should be ignored when finished + self.collector.total_samples = 100 + self.display.simulate_input(ord("r")) + self.collector._handle_input() + self.assertTrue(self.collector.running) # Still running + self.assertEqual(self.collector.total_samples, 100) # NOT reset when finished + + # Press 'q' - should stop + self.display.simulate_input(ord("q")) + self.collector._handle_input() + self.assertFalse(self.collector.running) # Stopped + + def test_finished_state_footer_message(self): + """Test that footer shows appropriate message when finished.""" + # Add some sample data + thread_info = MockThreadInfo( + 123, + [ + MockFrameInfo("test.py", 10, "work"), + MockFrameInfo("test.py", 20, "main"), + ], + ) + interpreter_info = MockInterpreterInfo(0, [thread_info]) + stack_frames = [interpreter_info] + self.collector.collect(stack_frames) + + # Mark as finished + self.collector.mark_finished() + + # Check that footer contains finished message + self.assertTrue(self.display.contains_text("PROFILING FINISHED")) + + def test_finished_state_freezes_time(self): + """Test that time displays are frozen when finished.""" + import time as time_module + + # Set up collector with known start time + self.collector.start_time = time_module.perf_counter() - 10.0 # 10 seconds ago + + # Mark as finished - this should freeze the time + self.collector.mark_finished() + + # Get the frozen elapsed time + frozen_elapsed = self.collector.elapsed_time + frozen_time_display = self.collector.current_time_display + + # Wait a bit to ensure time would advance + time_module.sleep(0.1) + + # Time should remain frozen + self.assertEqual(self.collector.elapsed_time, frozen_elapsed) + self.assertEqual(self.collector.current_time_display, frozen_time_display) + + # Verify finish timestamp was set + self.assertIsNotNone(self.collector.finish_timestamp) + + # Reset should clear the frozen state + self.collector.reset_stats() + self.assertFalse(self.collector.finished) + self.assertIsNone(self.collector.finish_timestamp) + + +class TestLiveCollectorFiltering(unittest.TestCase): + """Tests for filtering functionality.""" + + def setUp(self): + """Set up collector with test data.""" + self.display = MockDisplay(height=40, width=160) + self.collector = LiveStatsCollector( + 1000, pid=12345, display=self.display + ) + self.collector.start_time = time.perf_counter() + self.collector.total_samples = 100 + + # Add test data + self.collector.result[("app/models.py", 10, "save")] = { + "direct_calls": 50, + "cumulative_calls": 75, + "total_rec_calls": 0, + } + self.collector.result[("app/views.py", 20, "render")] = { + "direct_calls": 30, + "cumulative_calls": 40, + "total_rec_calls": 0, + } + self.collector.result[("lib/utils.py", 30, "helper")] = { + "direct_calls": 20, + "cumulative_calls": 25, + "total_rec_calls": 0, + } + + def test_filter_by_filename(self): + """Test filtering by filename pattern.""" + self.collector.filter_pattern = "models" + + stats_list = self.collector.build_stats_list() + + # Only models.py should be included + self.assertEqual(len(stats_list), 1) + self.assertIn("models.py", stats_list[0]["func"][0]) + + def test_filter_by_function_name(self): + """Test filtering by function name.""" + self.collector.filter_pattern = "render" + + stats_list = self.collector.build_stats_list() + + self.assertEqual(len(stats_list), 1) + self.assertEqual(stats_list[0]["func"][2], "render") + + def test_filter_case_insensitive(self): + """Test that filtering is case-insensitive.""" + self.collector.filter_pattern = "MODELS" + + stats_list = self.collector.build_stats_list() + + # Should still match models.py + self.assertEqual(len(stats_list), 1) + + def test_filter_substring_matching(self): + """Test substring filtering.""" + self.collector.filter_pattern = "app/" + + stats_list = self.collector.build_stats_list() + + # Should match both app files + self.assertEqual(len(stats_list), 2) + + def test_no_filter(self): + """Test with no filter applied.""" + self.collector.filter_pattern = None + + stats_list = self.collector.build_stats_list() + + # All items should be included + self.assertEqual(len(stats_list), 3) + + def test_filter_partial_function_name(self): + """Test filtering by partial function name.""" + self.collector.filter_pattern = "save" + + stats_list = self.collector.build_stats_list() + + self.assertEqual(len(stats_list), 1) + self.assertEqual(stats_list[0]["func"][2], "save") + + def test_filter_combined_filename_funcname(self): + """Test filtering matches filename:funcname pattern.""" + self.collector.filter_pattern = "views.py:render" + + stats_list = self.collector.build_stats_list() + + # Should match the combined pattern + self.assertEqual(len(stats_list), 1) + self.assertEqual(stats_list[0]["func"][2], "render") + + def test_filter_no_matches(self): + """Test filter that matches nothing.""" + self.collector.filter_pattern = "nonexistent" + + stats_list = self.collector.build_stats_list() + + self.assertEqual(len(stats_list), 0) + + +class TestLiveCollectorFilterInput(unittest.TestCase): + """Tests for filter input mode.""" + + def setUp(self): + """Set up collector with mock display.""" + self.display = MockDisplay(height=40, width=160) + self.collector = LiveStatsCollector( + 1000, pid=12345, display=self.display + ) + self.collector.start_time = time.perf_counter() + + def test_enter_filter_mode(self): + """Test entering filter input mode.""" + self.assertFalse(self.collector.filter_input_mode) + + # Press '/' to enter filter mode + self.display.simulate_input(ord("/")) + self.collector._handle_input() + + self.assertTrue(self.collector.filter_input_mode) + + def test_filter_input_typing(self): + """Test typing characters in filter input mode.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "" + + # Type 't', 'e', 's', 't' + for ch in "test": + self.display.simulate_input(ord(ch)) + self.collector._handle_input() + + self.assertEqual(self.collector.filter_input_buffer, "test") + + def test_filter_input_backspace(self): + """Test backspace in filter input mode.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "test" + + # Press backspace (127) + self.display.simulate_input(127) + self.collector._handle_input() + + self.assertEqual(self.collector.filter_input_buffer, "tes") + + def test_filter_input_backspace_alt(self): + """Test alternative backspace key (263) in filter input mode.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "test" + + # Press backspace (263) + self.display.simulate_input(263) + self.collector._handle_input() + + self.assertEqual(self.collector.filter_input_buffer, "tes") + + def test_filter_input_backspace_empty(self): + """Test backspace on empty buffer.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "" + + # Press backspace - should not crash + self.display.simulate_input(127) + self.collector._handle_input() + + self.assertEqual(self.collector.filter_input_buffer, "") + + def test_filter_input_enter_applies_filter(self): + """Test pressing Enter applies the filter.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "myfilter" + + # Press Enter (10) + self.display.simulate_input(10) + self.collector._handle_input() + + self.assertFalse(self.collector.filter_input_mode) + self.assertEqual(self.collector.filter_pattern, "myfilter") + self.assertEqual(self.collector.filter_input_buffer, "") + + def test_filter_input_enter_alt(self): + """Test alternative Enter key (13) applies filter.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "myfilter" + + # Press Enter (13) + self.display.simulate_input(13) + self.collector._handle_input() + + self.assertFalse(self.collector.filter_input_mode) + self.assertEqual(self.collector.filter_pattern, "myfilter") + + def test_filter_input_enter_empty_clears_filter(self): + """Test pressing Enter with empty buffer clears filter.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "" + self.collector.filter_pattern = "oldfilter" + + # Press Enter + self.display.simulate_input(10) + self.collector._handle_input() + + self.assertFalse(self.collector.filter_input_mode) + self.assertIsNone(self.collector.filter_pattern) + + def test_filter_input_escape_cancels(self): + """Test pressing ESC cancels filter input.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "newfilter" + self.collector.filter_pattern = "oldfilter" + + # Press ESC (27) + self.display.simulate_input(27) + self.collector._handle_input() + + self.assertFalse(self.collector.filter_input_mode) + self.assertEqual( + self.collector.filter_pattern, "oldfilter" + ) # Unchanged + self.assertEqual(self.collector.filter_input_buffer, "") + + def test_filter_input_start_with_existing_filter(self): + """Test entering filter mode with existing filter pre-fills buffer.""" + self.collector.filter_pattern = "existing" + + # Enter filter mode + self.display.simulate_input(ord("/")) + self.collector._handle_input() + + # Buffer should be pre-filled with existing pattern + self.assertEqual(self.collector.filter_input_buffer, "existing") + + def test_filter_input_start_without_filter(self): + """Test entering filter mode with no existing filter.""" + self.collector.filter_pattern = None + + # Enter filter mode + self.display.simulate_input(ord("/")) + self.collector._handle_input() + + # Buffer should be empty + self.assertEqual(self.collector.filter_input_buffer, "") + + def test_filter_input_mode_blocks_other_commands(self): + """Test that filter input mode blocks other commands.""" + self.collector.filter_input_mode = True + initial_sort = self.collector.sort_by + + # Try to press 's' (sort) - should be captured as input + self.display.simulate_input(ord("s")) + self.collector._handle_input() + + # Sort should not change, 's' should be in buffer + self.assertEqual(self.collector.sort_by, initial_sort) + self.assertEqual(self.collector.filter_input_buffer, "s") + + def test_filter_input_non_printable_ignored(self): + """Test that non-printable characters are ignored.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "test" + + # Try to input a control character (< 32) + self.display.simulate_input(1) # Ctrl-A + self.collector._handle_input() + + # Buffer should be unchanged + self.assertEqual(self.collector.filter_input_buffer, "test") + + def test_filter_input_high_ascii_ignored(self): + """Test that high ASCII characters (>= 127, except backspace) are ignored.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "test" + + # Try to input high ASCII (128) + self.display.simulate_input(128) + self.collector._handle_input() + + # Buffer should be unchanged + self.assertEqual(self.collector.filter_input_buffer, "test") + + def test_filter_prompt_displayed(self): + """Test that filter prompt is displayed when in input mode.""" + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "myfilter" + self.collector.total_samples = 10 + + self.collector._update_display() + + # Should show the filter prompt + self.assertTrue(self.display.contains_text("Function filter")) + self.assertTrue(self.display.contains_text("myfilter")) + + +if __name__ == "__main__": + unittest.main() + + +class TestLiveCollectorThreadNavigation(unittest.TestCase): + """Tests for thread navigation functionality.""" + + def setUp(self): + """Set up collector with mock display and multiple threads.""" + self.mock_display = MockDisplay(height=40, width=160) + self.collector = LiveStatsCollector( + 1000, pid=12345, display=self.mock_display + ) + self.collector.start_time = time.perf_counter() + + # Simulate data from multiple threads + frames1 = [MockFrameInfo("file1.py", 10, "func1")] + frames2 = [MockFrameInfo("file2.py", 20, "func2")] + frames3 = [MockFrameInfo("file3.py", 30, "func3")] + + thread1 = MockThreadInfo(111, frames1) + thread2 = MockThreadInfo(222, frames2) + thread3 = MockThreadInfo(333, frames3) + + interpreter_info = MockInterpreterInfo(0, [thread1, thread2, thread3]) + stack_frames = [interpreter_info] + + # Collect data to populate thread IDs + self.collector.collect(stack_frames) + + def test_initial_view_mode_is_all(self): + """Test that collector starts in ALL mode.""" + self.assertEqual(self.collector.view_mode, "ALL") + self.assertEqual(self.collector.current_thread_index, 0) + + def test_thread_ids_are_tracked(self): + """Test that thread IDs are tracked during collection.""" + self.assertIn(111, self.collector.thread_ids) + self.assertIn(222, self.collector.thread_ids) + self.assertIn(333, self.collector.thread_ids) + self.assertEqual(len(self.collector.thread_ids), 3) + + def test_toggle_to_per_thread_mode(self): + """Test toggling from ALL to PER_THREAD mode with 't' key.""" + self.assertEqual(self.collector.view_mode, "ALL") + + self.mock_display.simulate_input(ord("t")) + self.collector._handle_input() + + self.assertEqual(self.collector.view_mode, "PER_THREAD") + self.assertEqual(self.collector.current_thread_index, 0) + + def test_toggle_back_to_all_mode(self): + """Test toggling back from PER_THREAD to ALL mode.""" + # Switch to PER_THREAD + self.mock_display.simulate_input(ord("t")) + self.collector._handle_input() + self.assertEqual(self.collector.view_mode, "PER_THREAD") + + # Switch back to ALL + self.mock_display.simulate_input(ord("T")) + self.collector._handle_input() + self.assertEqual(self.collector.view_mode, "ALL") + + def test_arrow_right_navigates_threads_in_per_thread_mode(self): + """Test that arrow keys navigate threads in PER_THREAD mode.""" + # Switch to PER_THREAD mode + self.mock_display.simulate_input(ord("t")) + self.collector._handle_input() + + # Navigate forward + self.assertEqual(self.collector.current_thread_index, 0) + + self.mock_display.simulate_input(curses.KEY_RIGHT) + self.collector._handle_input() + self.assertEqual(self.collector.current_thread_index, 1) + + self.mock_display.simulate_input(curses.KEY_RIGHT) + self.collector._handle_input() + self.assertEqual(self.collector.current_thread_index, 2) + + def test_arrow_left_navigates_threads_backward(self): + """Test that left arrow navigates threads backward.""" + # Switch to PER_THREAD mode + self.mock_display.simulate_input(ord("t")) + self.collector._handle_input() + + # Navigate backward (should wrap around) + self.mock_display.simulate_input(curses.KEY_LEFT) + self.collector._handle_input() + self.assertEqual( + self.collector.current_thread_index, 2 + ) # Wrapped to last + + self.mock_display.simulate_input(curses.KEY_LEFT) + self.collector._handle_input() + self.assertEqual(self.collector.current_thread_index, 1) + + def test_arrow_down_navigates_like_right(self): + """Test that down arrow works like right arrow.""" + # Switch to PER_THREAD mode + self.mock_display.simulate_input(ord("t")) + self.collector._handle_input() + + self.mock_display.simulate_input(curses.KEY_DOWN) + self.collector._handle_input() + self.assertEqual(self.collector.current_thread_index, 1) + + def test_arrow_up_navigates_like_left(self): + """Test that up arrow works like left arrow.""" + # Switch to PER_THREAD mode + self.mock_display.simulate_input(ord("t")) + self.collector._handle_input() + + self.mock_display.simulate_input(curses.KEY_UP) + self.collector._handle_input() + self.assertEqual(self.collector.current_thread_index, 2) # Wrapped + + def test_arrow_keys_switch_to_per_thread_mode(self): + """Test that arrow keys switch from ALL mode to PER_THREAD mode.""" + self.assertEqual(self.collector.view_mode, "ALL") + + self.mock_display.simulate_input(curses.KEY_RIGHT) + self.collector._handle_input() + self.assertEqual(self.collector.view_mode, "PER_THREAD") + self.assertEqual(self.collector.current_thread_index, 0) + + def test_stats_list_in_all_mode(self): + """Test that stats list uses aggregated data in ALL mode.""" + stats_list = self.collector.build_stats_list() + + # Should have all 3 functions + self.assertEqual(len(stats_list), 3) + func_names = {stat["func"][2] for stat in stats_list} + self.assertEqual(func_names, {"func1", "func2", "func3"}) + + def test_stats_list_in_per_thread_mode(self): + """Test that stats list filters by thread in PER_THREAD mode.""" + # Switch to PER_THREAD mode + self.collector.view_mode = "PER_THREAD" + self.collector.current_thread_index = 0 # First thread (111) + + stats_list = self.collector.build_stats_list() + + # Should only have func1 from thread 111 + self.assertEqual(len(stats_list), 1) + self.assertEqual(stats_list[0]["func"][2], "func1") + + def test_stats_list_switches_with_thread_navigation(self): + """Test that stats list updates when navigating threads.""" + self.collector.view_mode = "PER_THREAD" + + # Thread 0 (111) -> func1 + self.collector.current_thread_index = 0 + stats_list = self.collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + self.assertEqual(stats_list[0]["func"][2], "func1") + + # Thread 1 (222) -> func2 + self.collector.current_thread_index = 1 + stats_list = self.collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + self.assertEqual(stats_list[0]["func"][2], "func2") + + # Thread 2 (333) -> func3 + self.collector.current_thread_index = 2 + stats_list = self.collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + self.assertEqual(stats_list[0]["func"][2], "func3") + + def test_reset_stats_clears_thread_data(self): + """Test that reset_stats clears thread tracking data.""" + self.assertGreater(len(self.collector.thread_ids), 0) + self.assertGreater(len(self.collector.per_thread_data), 0) + + self.collector.reset_stats() + + self.assertEqual(len(self.collector.thread_ids), 0) + self.assertEqual(len(self.collector.per_thread_data), 0) + self.assertEqual(self.collector.view_mode, "ALL") + self.assertEqual(self.collector.current_thread_index, 0) + + def test_toggle_with_no_threads_stays_in_all_mode(self): + """Test that toggle does nothing when no threads exist.""" + collector = LiveStatsCollector(1000, display=MockDisplay()) + self.assertEqual(len(collector.thread_ids), 0) + + collector.display.simulate_input(ord("t")) + collector._handle_input() + + # Should remain in ALL mode since no threads + self.assertEqual(collector.view_mode, "ALL") + + def test_per_thread_data_isolation(self): + """Test that per-thread data is properly isolated.""" + # Check that each thread has its own isolated data + self.assertIn(111, self.collector.per_thread_data) + self.assertIn(222, self.collector.per_thread_data) + self.assertIn(333, self.collector.per_thread_data) + + # Thread 111 should only have func1 + thread1_funcs = list(self.collector.per_thread_data[111].result.keys()) + self.assertEqual(len(thread1_funcs), 1) + self.assertEqual(thread1_funcs[0][2], "func1") + + # Thread 222 should only have func2 + thread2_funcs = list(self.collector.per_thread_data[222].result.keys()) + self.assertEqual(len(thread2_funcs), 1) + self.assertEqual(thread2_funcs[0][2], "func2") + + def test_aggregated_data_sums_all_threads(self): + """Test that ALL mode shows aggregated data from all threads.""" + # All three functions should be in the aggregated result + self.assertEqual(len(self.collector.result), 3) + + # Each function should have 1 direct call + for func_location, counts in self.collector.result.items(): + self.assertEqual(counts["direct_calls"], 1) + + def test_per_thread_status_tracking(self): + """Test that per-thread status statistics are tracked.""" + # Each thread should have status counts + self.assertIn(111, self.collector.per_thread_data) + self.assertIn(222, self.collector.per_thread_data) + self.assertIn(333, self.collector.per_thread_data) + + # Each thread should have the expected attributes + for thread_id in [111, 222, 333]: + thread_data = self.collector.per_thread_data[thread_id] + self.assertIsNotNone(thread_data.has_gil) + self.assertIsNotNone(thread_data.on_cpu) + self.assertIsNotNone(thread_data.gil_requested) + self.assertIsNotNone(thread_data.unknown) + self.assertIsNotNone(thread_data.total) + # Each thread was sampled once + self.assertEqual(thread_data.total, 1) + + def test_reset_stats_clears_thread_status(self): + """Test that reset_stats clears per-thread status data.""" + self.assertGreater(len(self.collector.per_thread_data), 0) + + self.collector.reset_stats() + + self.assertEqual(len(self.collector.per_thread_data), 0) + + def test_per_thread_sample_counts(self): + """Test that per-thread sample counts are tracked correctly.""" + # Each thread should have exactly 1 sample (we collected once) + for thread_id in [111, 222, 333]: + self.assertIn(thread_id, self.collector.per_thread_data) + self.assertEqual(self.collector.per_thread_data[thread_id].sample_count, 1) + + def test_per_thread_gc_samples(self): + """Test that per-thread GC samples are tracked correctly.""" + # Initially no threads have GC frames + for thread_id in [111, 222, 333]: + self.assertIn(thread_id, self.collector.per_thread_data) + self.assertEqual( + self.collector.per_thread_data[thread_id].gc_frame_samples, 0 + ) + + # Now collect a sample with a GC frame in thread 222 + gc_frames = [MockFrameInfo("gc.py", 100, "gc_collect")] + thread_with_gc = MockThreadInfo(222, gc_frames) + interpreter_info = MockInterpreterInfo(0, [thread_with_gc]) + stack_frames = [interpreter_info] + + self.collector.collect(stack_frames) + + # Thread 222 should now have 1 GC sample + self.assertEqual(self.collector.per_thread_data[222].gc_frame_samples, 1) + # Other threads should still have 0 + self.assertEqual(self.collector.per_thread_data[111].gc_frame_samples, 0) + self.assertEqual(self.collector.per_thread_data[333].gc_frame_samples, 0) + + def test_only_threads_with_frames_are_tracked(self): + """Test that only threads with actual frame data are added to thread_ids.""" + # Create a new collector + collector = LiveStatsCollector(1000, display=MockDisplay()) + + # Create threads: one with frames, one without + frames = [MockFrameInfo("test.py", 10, "test_func")] + thread_with_frames = MockThreadInfo(111, frames) + thread_without_frames = MockThreadInfo(222, None) # No frames + interpreter_info = MockInterpreterInfo( + 0, [thread_with_frames, thread_without_frames] + ) + stack_frames = [interpreter_info] + + collector.collect(stack_frames) + + # Only thread 111 should be tracked (it has frames) + self.assertIn(111, collector.thread_ids) + self.assertNotIn(222, collector.thread_ids) + + def test_per_thread_status_isolation(self): + """Test that per-thread status counts are isolated per thread.""" + # Create threads with different status flags + + frames1 = [MockFrameInfo("file1.py", 10, "func1")] + frames2 = [MockFrameInfo("file2.py", 20, "func2")] + + # Thread 444: has GIL but not on CPU + thread1 = MockThreadInfo(444, frames1, status=THREAD_STATUS_HAS_GIL) + # Thread 555: on CPU but not has GIL + thread2 = MockThreadInfo(555, frames2, status=THREAD_STATUS_ON_CPU) + + interpreter_info = MockInterpreterInfo(0, [thread1, thread2]) + stack_frames = [interpreter_info] + + collector = LiveStatsCollector(1000, display=MockDisplay()) + collector.collect(stack_frames) + + # Check thread 444 status + self.assertEqual(collector.per_thread_data[444].has_gil, 1) + self.assertEqual(collector.per_thread_data[444].on_cpu, 0) + + # Check thread 555 status + self.assertEqual(collector.per_thread_data[555].has_gil, 0) + self.assertEqual(collector.per_thread_data[555].on_cpu, 1) + + def test_display_uses_per_thread_stats_in_per_thread_mode(self): + """Test that display widget uses per-thread stats when in PER_THREAD mode.""" + + # Create collector with mock display + collector = LiveStatsCollector(1000, display=MockDisplay()) + collector.start_time = time.perf_counter() + + # Create 2 threads with different characteristics + # Thread 111: always has GIL (10 samples) + # Thread 222: never has GIL (10 samples) + for _ in range(10): + frames1 = [MockFrameInfo("file1.py", 10, "func1")] + frames2 = [MockFrameInfo("file2.py", 20, "func2")] + thread1 = MockThreadInfo( + 111, frames1, status=THREAD_STATUS_HAS_GIL + ) + thread2 = MockThreadInfo(222, frames2, status=0) # No flags + interpreter_info = MockInterpreterInfo(0, [thread1, thread2]) + collector.collect([interpreter_info]) + + # In ALL mode, should show mixed stats (50% on GIL, 50% off GIL) + self.assertEqual(collector.view_mode, "ALL") + total_has_gil = collector.thread_status_counts["has_gil"] + total_threads = collector.thread_status_counts["total"] + self.assertEqual(total_has_gil, 10) # Only thread 111 has GIL + self.assertEqual(total_threads, 20) # 10 samples * 2 threads + + # Switch to PER_THREAD mode and select thread 111 + collector.view_mode = "PER_THREAD" + collector.current_thread_index = 0 # Thread 111 + + # Thread 111 should show 100% on GIL + thread_111_data = collector.per_thread_data[111] + self.assertEqual(thread_111_data.has_gil, 10) + self.assertEqual(thread_111_data.total, 10) + + # Switch to thread 222 + collector.current_thread_index = 1 # Thread 222 + + # Thread 222 should show 0% on GIL + thread_222_data = collector.per_thread_data[222] + self.assertEqual(thread_222_data.has_gil, 0) + self.assertEqual(thread_222_data.total, 10) + + def test_display_uses_per_thread_gc_stats_in_per_thread_mode(self): + """Test that GC percentage uses per-thread data in PER_THREAD mode.""" + # Create collector with mock display + collector = LiveStatsCollector(1000, display=MockDisplay()) + collector.start_time = time.perf_counter() + + # Thread 111: 5 samples, 2 with GC + # Thread 222: 5 samples, 0 with GC + for i in range(5): + if i < 2: + # First 2 samples for thread 111 have GC + frames1 = [MockFrameInfo("gc.py", 100, "gc_collect")] + else: + frames1 = [MockFrameInfo("file1.py", 10, "func1")] + + frames2 = [MockFrameInfo("file2.py", 20, "func2")] # No GC + + thread1 = MockThreadInfo(111, frames1) + thread2 = MockThreadInfo(222, frames2) + interpreter_info = MockInterpreterInfo(0, [thread1, thread2]) + collector.collect([interpreter_info]) + + # Check aggregated GC stats (ALL mode) + # 2 GC samples out of 10 total = 20% + self.assertEqual(collector.gc_frame_samples, 2) + self.assertEqual(collector.total_samples, 5) # 5 collect() calls + + # Check per-thread GC stats + # Thread 111: 2 GC samples out of 5 = 40% + self.assertEqual(collector.per_thread_data[111].gc_frame_samples, 2) + self.assertEqual(collector.per_thread_data[111].sample_count, 5) + + # Thread 222: 0 GC samples out of 5 = 0% + self.assertEqual(collector.per_thread_data[222].gc_frame_samples, 0) + self.assertEqual(collector.per_thread_data[222].sample_count, 5) + + # Now verify the display would use the correct stats + collector.view_mode = "PER_THREAD" + + # For thread 111 + collector.current_thread_index = 0 + thread_id = collector.thread_ids[0] + self.assertEqual(thread_id, 111) + thread_gc_pct = ( + collector.per_thread_data[111].gc_frame_samples + / collector.per_thread_data[111].sample_count + ) * 100 + self.assertEqual(thread_gc_pct, 40.0) + + # For thread 222 + collector.current_thread_index = 1 + thread_id = collector.thread_ids[1] + self.assertEqual(thread_id, 222) + thread_gc_pct = ( + collector.per_thread_data[222].gc_frame_samples + / collector.per_thread_data[222].sample_count + ) * 100 + self.assertEqual(thread_gc_pct, 0.0) + + def test_function_counts_are_per_thread_in_per_thread_mode(self): + """Test that function counts (total/exec/stack) are per-thread in PER_THREAD mode.""" + # Create collector with mock display + collector = LiveStatsCollector(1000, display=MockDisplay()) + collector.start_time = time.perf_counter() + + # Thread 111: calls func1, func2, func3 (3 functions) + # Thread 222: calls func4, func5 (2 functions) + frames1 = [ + MockFrameInfo("file1.py", 10, "func1"), + MockFrameInfo("file1.py", 20, "func2"), + MockFrameInfo("file1.py", 30, "func3"), + ] + frames2 = [ + MockFrameInfo("file2.py", 40, "func4"), + MockFrameInfo("file2.py", 50, "func5"), + ] + + thread1 = MockThreadInfo(111, frames1) + thread2 = MockThreadInfo(222, frames2) + interpreter_info = MockInterpreterInfo(0, [thread1, thread2]) + collector.collect([interpreter_info]) + + # In ALL mode, should have 5 total functions + self.assertEqual(len(collector.result), 5) + + # In PER_THREAD mode for thread 111, should have 3 functions + collector.view_mode = "PER_THREAD" + collector.current_thread_index = 0 # Thread 111 + thread_111_result = collector.per_thread_data[111].result + self.assertEqual(len(thread_111_result), 3) + + # Verify the functions are the right ones + thread_111_funcs = {loc[2] for loc in thread_111_result.keys()} + self.assertEqual(thread_111_funcs, {"func1", "func2", "func3"}) + + # In PER_THREAD mode for thread 222, should have 2 functions + collector.current_thread_index = 1 # Thread 222 + thread_222_result = collector.per_thread_data[222].result + self.assertEqual(len(thread_222_result), 2) + + # Verify the functions are the right ones + thread_222_funcs = {loc[2] for loc in thread_222_result.keys()} + self.assertEqual(thread_222_funcs, {"func4", "func5"}) + + +class TestLiveCollectorNewFeatures(unittest.TestCase): + """Tests for new features added to live collector.""" + + def setUp(self): + """Set up test fixtures.""" + self.display = MockDisplay() + self.collector = LiveStatsCollector(1000, display=self.display) + self.collector.start_time = time.perf_counter() + + def test_filter_input_takes_precedence_over_commands(self): + """Test that filter input mode blocks command keys like 'h' and 'p'.""" + # Enter filter input mode + self.collector.filter_input_mode = True + self.collector.filter_input_buffer = "" + + # Press 'h' - should add to filter buffer, not show help + self.display.simulate_input(ord("h")) + self.collector._handle_input() + + self.assertFalse(self.collector.show_help) # Help not triggered + self.assertEqual(self.collector.filter_input_buffer, "h") # Added to filter + self.assertTrue(self.collector.filter_input_mode) # Still in filter mode + + def test_reset_blocked_when_finished(self): + """Test that reset command is blocked when profiling is finished.""" + # Set up some sample data and mark as finished + self.collector.total_samples = 100 + self.collector.finished = True + + # Press 'r' for reset + self.display.simulate_input(ord("r")) + self.collector._handle_input() + + # Should NOT have been reset + self.assertEqual(self.collector.total_samples, 100) + self.assertTrue(self.collector.finished) + + def test_time_display_fix_when_finished(self): + """Test that time display shows correct frozen time when finished.""" + import time as time_module + + # Mark as finished to freeze time + self.collector.mark_finished() + + # Should have set both timestamps correctly + self.assertIsNotNone(self.collector.finish_timestamp) + self.assertIsNotNone(self.collector.finish_wall_time) + + # Get the frozen time display + frozen_time = self.collector.current_time_display + + # Wait a bit + time_module.sleep(0.1) + + # Should still show the same frozen time (not jump to wrong time) + self.assertEqual(self.collector.current_time_display, frozen_time) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py new file mode 100644 index 00000000000..b5a387fa3a3 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -0,0 +1,819 @@ +"""UI and display tests for LiveStatsCollector. + +Tests for MockDisplay, curses integration, display methods, +edge cases, update display, and display helpers. +""" + +import sys +import time +import unittest +from unittest import mock +from test.support import requires +from test.support.import_helper import import_module + +# Only run these tests if curses is available +requires("curses") +curses = import_module("curses") + +from profiling.sampling.live_collector import LiveStatsCollector, MockDisplay +from ._live_collector_helpers import ( + MockThreadInfo, + MockInterpreterInfo, +) + + +class TestLiveStatsCollectorWithMockDisplay(unittest.TestCase): + """Tests for display functionality using MockDisplay.""" + + def setUp(self): + """Set up collector with mock display.""" + self.mock_display = MockDisplay(height=40, width=160) + self.collector = LiveStatsCollector( + 1000, pid=12345, display=self.mock_display + ) + self.collector.start_time = time.perf_counter() + + def test_update_display_with_mock(self): + """Test that update_display works with MockDisplay.""" + self.collector.total_samples = 100 + self.collector.result[("test.py", 10, "test_func")] = { + "direct_calls": 50, + "cumulative_calls": 75, + "total_rec_calls": 0, + } + + self.collector._update_display() + + # Verify display operations were called + self.assertTrue(self.mock_display.cleared) + self.assertTrue(self.mock_display.refreshed) + self.assertTrue(self.mock_display.redrawn) + + # Verify some content was written + self.assertGreater(len(self.mock_display.buffer), 0) + + def test_handle_input_quit(self): + """Test that 'q' input stops the collector.""" + self.mock_display.simulate_input(ord("q")) + self.collector._handle_input() + self.assertFalse(self.collector.running) + + def test_handle_input_sort_cycle(self): + """Test that 's' input cycles sort mode.""" + self.collector.sort_by = "tottime" + self.mock_display.simulate_input(ord("s")) + self.collector._handle_input() + self.assertEqual(self.collector.sort_by, "cumul_pct") + + def test_draw_methods_with_mock_display(self): + """Test that draw methods write to mock display.""" + self.collector.total_samples = 500 + self.collector.successful_samples = 450 + self.collector.failed_samples = 50 + + colors = self.collector._setup_colors() + self.collector._initialize_widgets(colors) + + # Test individual widget methods + line = self.collector.header_widget.draw_header_info(0, 160, 100.5) + self.assertEqual(line, 2) # Title + header info line + self.assertGreater(len(self.mock_display.buffer), 0) + + # Clear buffer and test next method + self.mock_display.buffer.clear() + line = self.collector.header_widget.draw_sample_stats(0, 160, 10.0) + self.assertEqual(line, 1) + self.assertGreater(len(self.mock_display.buffer), 0) + + def test_terminal_too_small_message(self): + """Test terminal too small warning.""" + small_display = MockDisplay(height=10, width=50) + self.collector.display = small_display + + self.collector._show_terminal_too_small(10, 50) + + # Should have written warning message + text = small_display.get_text_at(3, 15) # Approximate center + self.assertIsNotNone(text) + + def test_full_display_rendering_with_data(self): + """Test complete display rendering with realistic data.""" + # Add multiple functions with different call counts + self.collector.total_samples = 1000 + self.collector.successful_samples = 950 + self.collector.failed_samples = 50 + + self.collector.result[("app.py", 10, "main")] = { + "direct_calls": 100, + "cumulative_calls": 500, + "total_rec_calls": 0, + } + self.collector.result[("utils.py", 20, "helper")] = { + "direct_calls": 300, + "cumulative_calls": 400, + "total_rec_calls": 0, + } + self.collector.result[("db.py", 30, "query")] = { + "direct_calls": 50, + "cumulative_calls": 100, + "total_rec_calls": 0, + } + + self.collector._update_display() + + # Verify the display has content + self.assertGreater(len(self.mock_display.buffer), 10) + + # Verify PID is shown + found_pid = False + for (line, col), (text, attr) in self.mock_display.buffer.items(): + if "12345" in text: + found_pid = True + break + self.assertTrue(found_pid, "PID should be displayed") + + def test_efficiency_bar_visualization(self): + """Test that efficiency bar shows correct proportions.""" + self.collector.total_samples = 100 + self.collector.successful_samples = 75 + self.collector.failed_samples = 25 + + colors = self.collector._setup_colors() + self.collector._initialize_widgets(colors) + self.collector.header_widget.draw_efficiency_bar(0, 160) + + # Check that something was drawn to the display + self.assertGreater(len(self.mock_display.buffer), 0) + + def test_stats_display_with_different_sort_modes(self): + """Test that stats are displayed correctly with different sort modes.""" + self.collector.total_samples = 100 + self.collector.result[("a.py", 1, "func_a")] = { + "direct_calls": 10, + "cumulative_calls": 20, + "total_rec_calls": 0, + } + self.collector.result[("b.py", 2, "func_b")] = { + "direct_calls": 30, + "cumulative_calls": 40, + "total_rec_calls": 0, + } + + # Test each sort mode + for sort_mode in [ + "nsamples", + "tottime", + "cumtime", + "sample_pct", + "cumul_pct", + ]: + self.mock_display.buffer.clear() + self.collector.sort_by = sort_mode + + stats_list = self.collector.build_stats_list() + self.assertEqual(len(stats_list), 2) + + # Verify sorting worked (func_b should be first for most modes) + if sort_mode in ["nsamples", "tottime", "sample_pct"]: + self.assertEqual(stats_list[0]["func"][2], "func_b") + + def test_narrow_terminal_column_hiding(self): + """Test that columns are hidden on narrow terminals.""" + narrow_display = MockDisplay(height=40, width=70) + collector = LiveStatsCollector(1000, pid=12345, display=narrow_display) + collector.start_time = time.perf_counter() + + colors = collector._setup_colors() + collector._initialize_widgets(colors) + line, show_sample_pct, show_tottime, show_cumul_pct, show_cumtime = ( + collector.table_widget.draw_column_headers(0, 70) + ) + + # On narrow terminal, some columns should be hidden + self.assertFalse( + show_cumul_pct or show_cumtime, + "Some columns should be hidden on narrow terminal", + ) + + def test_very_narrow_terminal_minimal_columns(self): + """Test minimal display on very narrow terminal.""" + very_narrow = MockDisplay(height=40, width=60) + collector = LiveStatsCollector(1000, pid=12345, display=very_narrow) + collector.start_time = time.perf_counter() + + colors = collector._setup_colors() + collector._initialize_widgets(colors) + line, show_sample_pct, show_tottime, show_cumul_pct, show_cumtime = ( + collector.table_widget.draw_column_headers(0, 60) + ) + + # Very narrow should hide even more columns + self.assertFalse( + show_sample_pct, + "Sample % should be hidden on very narrow terminal", + ) + + def test_display_updates_only_at_interval(self): + """Test that display updates respect the update interval.""" + # Create collector with display + collector = LiveStatsCollector(1000, display=self.mock_display) + + # Simulate multiple rapid collections + thread_info = MockThreadInfo(123, []) + interpreter_info = MockInterpreterInfo(0, [thread_info]) + stack_frames = [interpreter_info] + + # First collect should update display + collector.collect(stack_frames) + first_cleared = self.mock_display.cleared + + # Reset flags + self.mock_display.cleared = False + self.mock_display.refreshed = False + + # Immediate second collect should NOT update display (too soon) + collector.collect(stack_frames) + self.assertFalse( + self.mock_display.cleared, + "Display should not update too frequently", + ) + + def test_top_functions_display(self): + """Test that top functions are highlighted correctly.""" + self.collector.total_samples = 1000 + + # Create functions with different sample counts + for i in range(10): + self.collector.result[(f"file{i}.py", i * 10, f"func{i}")] = { + "direct_calls": (10 - i) * 10, # Decreasing counts + "cumulative_calls": (10 - i) * 20, + "total_rec_calls": 0, + } + + colors = self.collector._setup_colors() + self.collector._initialize_widgets(colors) + stats_list = self.collector.build_stats_list() + + self.collector.header_widget.draw_top_functions(0, 160, stats_list) + + # Top functions section should have written something + self.assertGreater(len(self.mock_display.buffer), 0) + + +class TestLiveStatsCollectorCursesIntegration(unittest.TestCase): + """Tests for curses-related functionality using mocks.""" + + def setUp(self): + """Set up mock curses screen.""" + self.mock_stdscr = mock.MagicMock() + self.mock_stdscr.getmaxyx.return_value = (40, 160) # height, width + self.mock_stdscr.getch.return_value = -1 # No input + # Save original stdout/stderr + self._orig_stdout = sys.stdout + self._orig_stderr = sys.stderr + + def tearDown(self): + """Restore stdout/stderr if changed.""" + sys.stdout = self._orig_stdout + sys.stderr = self._orig_stderr + + def test_init_curses(self): + """Test curses initialization.""" + collector = LiveStatsCollector(1000) + + with ( + mock.patch("curses.curs_set"), + mock.patch("curses.has_colors", return_value=True), + mock.patch("curses.start_color"), + mock.patch("curses.use_default_colors"), + mock.patch("builtins.open", mock.mock_open()) as mock_open_func, + ): + collector.init_curses(self.mock_stdscr) + + self.assertIsNotNone(collector.stdscr) + self.mock_stdscr.nodelay.assert_called_with(True) + self.mock_stdscr.scrollok.assert_called_with(False) + + # Clean up properly + if collector._devnull: + collector._devnull.close() + collector._saved_stdout = None + collector._saved_stderr = None + + def test_cleanup_curses(self): + """Test curses cleanup.""" + mock_display = MockDisplay() + collector = LiveStatsCollector(1000, display=mock_display) + collector.stdscr = self.mock_stdscr + + # Mock devnull file to avoid resource warnings + mock_devnull = mock.MagicMock() + mock_saved_stdout = mock.MagicMock() + mock_saved_stderr = mock.MagicMock() + + collector._devnull = mock_devnull + collector._saved_stdout = mock_saved_stdout + collector._saved_stderr = mock_saved_stderr + + with mock.patch("curses.curs_set"): + collector.cleanup_curses() + + mock_devnull.close.assert_called_once() + # Verify stdout/stderr were set back to the saved values + self.assertEqual(sys.stdout, mock_saved_stdout) + self.assertEqual(sys.stderr, mock_saved_stderr) + # Verify the saved values were cleared + self.assertIsNone(collector._saved_stdout) + self.assertIsNone(collector._saved_stderr) + self.assertIsNone(collector._devnull) + + def test_add_str_with_mock_display(self): + """Test safe_addstr with MockDisplay.""" + mock_display = MockDisplay(height=40, width=160) + collector = LiveStatsCollector(1000, display=mock_display) + colors = collector._setup_colors() + collector._initialize_widgets(colors) + + collector.header_widget.add_str(5, 10, "Test", 0) + # Verify it was added to the buffer + self.assertIn((5, 10), mock_display.buffer) + + def test_setup_colors_with_color_support(self): + """Test color setup when colors are supported.""" + mock_display = MockDisplay(height=40, width=160) + mock_display.colors_supported = True + collector = LiveStatsCollector(1000, display=mock_display) + + colors = collector._setup_colors() + + self.assertIn("header", colors) + self.assertIn("cyan", colors) + self.assertIn("yellow", colors) + self.assertIn("green", colors) + self.assertIn("magenta", colors) + self.assertIn("red", colors) + + def test_setup_colors_without_color_support(self): + """Test color setup when colors are not supported.""" + mock_display = MockDisplay(height=40, width=160) + mock_display.colors_supported = False + collector = LiveStatsCollector(1000, display=mock_display) + + colors = collector._setup_colors() + + # Should still have all keys but with fallback values + self.assertIn("header", colors) + self.assertIn("cyan", colors) + + def test_handle_input_quit(self): + """Test handling 'q' key to quit.""" + mock_display = MockDisplay() + mock_display.simulate_input(ord("q")) + collector = LiveStatsCollector(1000, display=mock_display) + + self.assertTrue(collector.running) + collector._handle_input() + self.assertFalse(collector.running) + + def test_handle_input_quit_uppercase(self): + """Test handling 'Q' key to quit.""" + mock_display = MockDisplay() + mock_display.simulate_input(ord("Q")) + collector = LiveStatsCollector(1000, display=mock_display) + + self.assertTrue(collector.running) + collector._handle_input() + self.assertFalse(collector.running) + + def test_handle_input_cycle_sort(self): + """Test handling 's' key to cycle sort.""" + mock_display = MockDisplay() + mock_display.simulate_input(ord("s")) + collector = LiveStatsCollector( + 1000, sort_by="nsamples", display=mock_display + ) + + collector._handle_input() + self.assertEqual(collector.sort_by, "sample_pct") + + def test_handle_input_cycle_sort_uppercase(self): + """Test handling 'S' key to cycle sort backward.""" + mock_display = MockDisplay() + mock_display.simulate_input(ord("S")) + collector = LiveStatsCollector( + 1000, sort_by="nsamples", display=mock_display + ) + + collector._handle_input() + self.assertEqual(collector.sort_by, "cumtime") + + def test_handle_input_no_key(self): + """Test handling when no key is pressed.""" + mock_display = MockDisplay() + collector = LiveStatsCollector(1000, display=mock_display) + + collector._handle_input() + # Should not change state + self.assertTrue(collector.running) + + +class TestLiveStatsCollectorDisplayMethods(unittest.TestCase): + """Tests for display-related methods.""" + + def setUp(self): + """Set up collector with mock display.""" + self.mock_display = MockDisplay(height=40, width=160) + self.collector = LiveStatsCollector( + 1000, pid=12345, display=self.mock_display + ) + self.collector.start_time = time.perf_counter() + + def test_show_terminal_too_small(self): + """Test terminal too small message display.""" + self.collector._show_terminal_too_small(10, 50) + # Should have written some content to the display buffer + self.assertGreater(len(self.mock_display.buffer), 0) + + def test_draw_header_info(self): + """Test drawing header information.""" + colors = { + "cyan": curses.A_BOLD, + "green": curses.A_BOLD, + "yellow": curses.A_BOLD, + "magenta": curses.A_BOLD, + } + self.collector._initialize_widgets(colors) + + line = self.collector.header_widget.draw_header_info(0, 160, 100.5) + self.assertEqual(line, 2) # Title + header info line + + def test_draw_sample_stats(self): + """Test drawing sample statistics.""" + self.collector.total_samples = 1000 + colors = {"cyan": curses.A_BOLD, "green": curses.A_BOLD} + self.collector._initialize_widgets(colors) + + line = self.collector.header_widget.draw_sample_stats(0, 160, 10.0) + self.assertEqual(line, 1) + self.assertGreater(self.collector.max_sample_rate, 0) + + def test_progress_bar_uses_target_rate(self): + """Test that progress bar uses target rate instead of max rate.""" + # Set up collector with specific sampling interval + collector = LiveStatsCollector( + 10000, pid=12345, display=self.mock_display + ) # 10ms = 100Hz target + collector.start_time = time.perf_counter() + collector.total_samples = 500 + collector.max_sample_rate = ( + 150 # Higher than target to test we don't use this + ) + + colors = {"cyan": curses.A_BOLD, "green": curses.A_BOLD} + collector._initialize_widgets(colors) + + # Clear the display buffer to capture only our progress bar content + self.mock_display.buffer.clear() + + # Draw sample stats with a known elapsed time that gives us a specific sample rate + elapsed = 10.0 # 500 samples in 10 seconds = 50 samples/second + line = collector.header_widget.draw_sample_stats(0, 160, elapsed) + + # Verify display was updated + self.assertEqual(line, 1) + self.assertGreater(len(self.mock_display.buffer), 0) + + # Verify the label shows current/target format with units instead of "max" + found_current_target_label = False + found_max_label = False + for (line_num, col), (text, attr) in self.mock_display.buffer.items(): + # Should show "50.0Hz/100.0Hz (50.0%)" since we're at 50% of target (50/100) + if "50.0Hz/100.0Hz" in text and "50.0%" in text: + found_current_target_label = True + if "max:" in text: + found_max_label = True + + self.assertTrue( + found_current_target_label, + "Should display current/target rate with percentage", + ) + self.assertFalse(found_max_label, "Should not display max rate label") + + def test_progress_bar_different_intervals(self): + """Test that progress bar adapts to different sampling intervals.""" + test_cases = [ + ( + 1000, + "1.0KHz", + "100.0Hz", + ), # 1ms interval -> 1000Hz target (1.0KHz), 100Hz current + ( + 5000, + "200.0Hz", + "100.0Hz", + ), # 5ms interval -> 200Hz target, 100Hz current + ( + 20000, + "50.0Hz", + "100.0Hz", + ), # 20ms interval -> 50Hz target, 100Hz current + ( + 100000, + "10.0Hz", + "100.0Hz", + ), # 100ms interval -> 10Hz target, 100Hz current + ] + + for ( + interval_usec, + expected_target_formatted, + expected_current_formatted, + ) in test_cases: + with self.subTest(interval=interval_usec): + collector = LiveStatsCollector( + interval_usec, display=MockDisplay() + ) + collector.start_time = time.perf_counter() + collector.total_samples = 100 + + colors = {"cyan": curses.A_BOLD, "green": curses.A_BOLD} + collector._initialize_widgets(colors) + + # Clear buffer + collector.display.buffer.clear() + + # Draw with 1 second elapsed time (gives us current rate of 100Hz) + collector.header_widget.draw_sample_stats(0, 160, 1.0) + + # Check that the current/target format appears in the display with proper units + found_current_target_format = False + for (line_num, col), ( + text, + attr, + ) in collector.display.buffer.items(): + # Looking for format like "100.0Hz/1.0KHz" or "100.0Hz/200.0Hz" + expected_format = f"{expected_current_formatted}/{expected_target_formatted}" + if expected_format in text and "%" in text: + found_current_target_format = True + break + + self.assertTrue( + found_current_target_format, + f"Should display current/target rate format with units for {interval_usec}µs interval", + ) + + def test_draw_efficiency_bar(self): + """Test drawing efficiency bar.""" + self.collector.successful_samples = 900 + self.collector.failed_samples = 100 + self.collector.total_samples = 1000 + colors = {"green": curses.A_BOLD, "red": curses.A_BOLD} + self.collector._initialize_widgets(colors) + + line = self.collector.header_widget.draw_efficiency_bar(0, 160) + self.assertEqual(line, 1) + + def test_draw_function_stats(self): + """Test drawing function statistics.""" + self.collector.result[("test.py", 10, "func1")] = { + "direct_calls": 100, + "cumulative_calls": 150, + "total_rec_calls": 0, + } + self.collector.result[("test.py", 20, "func2")] = { + "direct_calls": 0, + "cumulative_calls": 50, + "total_rec_calls": 0, + } + + stats_list = self.collector.build_stats_list() + colors = { + "cyan": curses.A_BOLD, + "green": curses.A_BOLD, + "yellow": curses.A_BOLD, + "magenta": curses.A_BOLD, + } + self.collector._initialize_widgets(colors) + + line = self.collector.header_widget.draw_function_stats( + 0, 160, stats_list + ) + self.assertEqual(line, 1) + + def test_draw_top_functions(self): + """Test drawing top functions.""" + self.collector.total_samples = 300 + self.collector.result[("test.py", 10, "hot_func")] = { + "direct_calls": 100, + "cumulative_calls": 150, + "total_rec_calls": 0, + } + + stats_list = self.collector.build_stats_list() + colors = { + "red": curses.A_BOLD, + "yellow": curses.A_BOLD, + "green": curses.A_BOLD, + } + self.collector._initialize_widgets(colors) + + line = self.collector.header_widget.draw_top_functions( + 0, 160, stats_list + ) + self.assertEqual(line, 1) + + def test_draw_column_headers(self): + """Test drawing column headers.""" + colors = { + "sorted_header": curses.A_BOLD, + "normal_header": curses.A_NORMAL, + } + self.collector._initialize_widgets(colors) + + ( + line, + show_sample_pct, + show_tottime, + show_cumul_pct, + show_cumtime, + ) = self.collector.table_widget.draw_column_headers(0, 160) + self.assertEqual(line, 1) + self.assertTrue(show_sample_pct) + self.assertTrue(show_tottime) + self.assertTrue(show_cumul_pct) + self.assertTrue(show_cumtime) + + def test_draw_column_headers_narrow_terminal(self): + """Test column headers adapt to narrow terminal.""" + colors = { + "sorted_header": curses.A_BOLD, + "normal_header": curses.A_NORMAL, + } + self.collector._initialize_widgets(colors) + + ( + line, + show_sample_pct, + show_tottime, + show_cumul_pct, + show_cumtime, + ) = self.collector.table_widget.draw_column_headers(0, 70) + self.assertEqual(line, 1) + # Some columns should be hidden on narrow terminal + self.assertFalse(show_cumul_pct) + + def test_draw_footer(self): + """Test drawing footer.""" + colors = self.collector._setup_colors() + self.collector._initialize_widgets(colors) + self.collector.footer_widget.render(38, 160) + # Should have written some content to the display buffer + self.assertGreater(len(self.mock_display.buffer), 0) + + def test_draw_progress_bar(self): + """Test progress bar drawing.""" + colors = self.collector._setup_colors() + self.collector._initialize_widgets(colors) + bar, length = self.collector.header_widget.progress_bar.render_bar( + 50, 100, 30 + ) + + self.assertIn("[", bar) + self.assertIn("]", bar) + self.assertGreater(length, 0) + # Should be roughly 50% filled + self.assertIn("█", bar) + self.assertIn("░", bar) + + +class TestLiveStatsCollectorEdgeCases(unittest.TestCase): + """Tests for edge cases and error handling.""" + + def test_very_long_function_name(self): + """Test handling of very long function names.""" + collector = LiveStatsCollector(1000) + long_name = "x" * 200 + collector.result[("test.py", 10, long_name)] = { + "direct_calls": 10, + "cumulative_calls": 20, + "total_rec_calls": 0, + } + + stats_list = collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + self.assertEqual(stats_list[0]["func"][2], long_name) + + +class TestLiveStatsCollectorUpdateDisplay(unittest.TestCase): + """Tests for the _update_display method.""" + + def setUp(self): + """Set up collector with mock display.""" + self.mock_display = MockDisplay(height=40, width=160) + self.collector = LiveStatsCollector( + 1000, pid=12345, display=self.mock_display + ) + self.collector.start_time = time.perf_counter() + + def test_update_display_terminal_too_small(self): + """Test update_display when terminal is too small.""" + small_display = MockDisplay(height=10, width=50) + self.collector.display = small_display + + with mock.patch.object( + self.collector, "_show_terminal_too_small" + ) as mock_show: + self.collector._update_display() + mock_show.assert_called_once() + + def test_update_display_normal(self): + """Test normal update_display operation.""" + self.collector.total_samples = 100 + self.collector.successful_samples = 90 + self.collector.failed_samples = 10 + self.collector.result[("test.py", 10, "func")] = { + "direct_calls": 50, + "cumulative_calls": 75, + "total_rec_calls": 0, + } + + self.collector._update_display() + + self.assertTrue(self.mock_display.cleared) + self.assertTrue(self.mock_display.refreshed) + + def test_update_display_handles_exception(self): + """Test that update_display handles exceptions gracefully.""" + # Make one of the methods raise an exception + with mock.patch.object( + self.collector, + "_prepare_display_data", + side_effect=Exception("Test error"), + ): + # Should not raise an exception (it catches and logs via trace_exception) + try: + self.collector._update_display() + except Exception: + self.fail( + "_update_display should handle exceptions gracefully" + ) + + +class TestLiveCollectorWithMockDisplayHelpers(unittest.TestCase): + """Tests using the new MockDisplay helper methods.""" + + def test_verify_pid_display_with_contains(self): + """Test verifying PID is displayed using contains_text helper.""" + display = MockDisplay(height=40, width=160) + collector = LiveStatsCollector(1000, pid=99999, display=display) + collector.start_time = time.perf_counter() + collector.total_samples = 10 + + collector._update_display() + + # Use the helper method + self.assertTrue( + display.contains_text("99999"), "PID should be visible in display" + ) + + def test_verify_function_names_displayed(self): + """Test verifying function names appear in display.""" + display = MockDisplay(height=40, width=160) + collector = LiveStatsCollector(1000, pid=12345, display=display) + collector.start_time = time.perf_counter() + + collector.total_samples = 100 + collector.result[("mymodule.py", 42, "my_special_function")] = { + "direct_calls": 50, + "cumulative_calls": 75, + "total_rec_calls": 0, + } + + collector._update_display() + + # Verify function name appears + self.assertTrue( + display.contains_text("my_special_function"), + "Function name should be visible", + ) + + def test_get_all_lines_full_display(self): + """Test getting all lines from a full display render.""" + display = MockDisplay(height=40, width=160) + collector = LiveStatsCollector(1000, pid=12345, display=display) + collector.start_time = time.perf_counter() + collector.total_samples = 100 + + collector._update_display() + + lines = display.get_all_lines() + + # Should have multiple lines of content + self.assertGreater(len(lines), 5) + + # Should have header content + self.assertTrue(any("PID" in line for line in lines)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_modes.py b/Lib/test/test_profiling/test_sampling_profiler/test_modes.py new file mode 100644 index 00000000000..c0457ee7eb8 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_modes.py @@ -0,0 +1,433 @@ +"""Tests for sampling profiler mode filtering (CPU and GIL modes).""" + +import io +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 + import profiling.sampling + import profiling.sampling.sample + from profiling.sampling.pstats_collector import PstatsCollector +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import requires_subprocess + +from .helpers import test_subprocess +from .mocks import MockFrameInfo, MockInterpreterInfo + + +class TestCpuModeFiltering(unittest.TestCase): + """Test CPU mode filtering functionality (--mode=cpu).""" + + def test_mode_validation(self): + """Test that CLI validates mode choices correctly.""" + # Invalid mode choice should raise SystemExit + test_args = [ + "profiling.sampling.cli", + "attach", + "12345", + "--mode", + "invalid", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("invalid choice", error_msg) + + def test_frames_filtered_with_skip_idle(self): + """Test that frames are actually filtered when skip_idle=True.""" + # Import thread status flags + try: + from _remote_debugging import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + ) + except ImportError: + THREAD_STATUS_HAS_GIL = 1 << 0 + THREAD_STATUS_ON_CPU = 1 << 1 + + # Create mock frames with different thread statuses + class MockThreadInfoWithStatus: + def __init__(self, thread_id, frame_info, status): + self.thread_id = thread_id + self.frame_info = frame_info + self.status = status + + # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread + ACTIVE_STATUS = ( + THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU + ) # Has GIL and on CPU + IDLE_STATUS = 0 # Neither has GIL nor on CPU + + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfoWithStatus( + 1, + [MockFrameInfo("active1.py", 10, "active_func1")], + ACTIVE_STATUS, + ), + MockThreadInfoWithStatus( + 2, + [MockFrameInfo("idle.py", 20, "idle_func")], + IDLE_STATUS, + ), + MockThreadInfoWithStatus( + 3, + [MockFrameInfo("active2.py", 30, "active_func2")], + ACTIVE_STATUS, + ), + ], + ) + ] + + # Test with skip_idle=True - should only process running threads + collector_skip = PstatsCollector( + sample_interval_usec=1000, skip_idle=True + ) + collector_skip.collect(test_frames) + + # Should only have functions from running threads (status 0) + active1_key = ("active1.py", 10, "active_func1") + active2_key = ("active2.py", 30, "active_func2") + idle_key = ("idle.py", 20, "idle_func") + + self.assertIn(active1_key, collector_skip.result) + self.assertIn(active2_key, collector_skip.result) + self.assertNotIn( + idle_key, collector_skip.result + ) # Idle thread should be filtered out + + # Test with skip_idle=False - should process all threads + collector_no_skip = PstatsCollector( + sample_interval_usec=1000, skip_idle=False + ) + collector_no_skip.collect(test_frames) + + # Should have functions from all threads + self.assertIn(active1_key, collector_no_skip.result) + self.assertIn(active2_key, collector_no_skip.result) + self.assertIn( + idle_key, collector_no_skip.result + ) # Idle thread should be included + + @requires_subprocess() + def test_cpu_mode_integration_filtering(self): + """Integration test: CPU mode should only capture active threads, not idle ones.""" + # Script with one mostly-idle thread and one CPU-active thread + cpu_vs_idle_script = """ +import time +import threading + +cpu_ready = threading.Event() + +def idle_worker(): + time.sleep(999999) + +def cpu_active_worker(): + cpu_ready.set() + x = 1 + while True: + x += 1 + +idle_thread = threading.Thread(target=idle_worker) +cpu_thread = threading.Thread(target=cpu_active_worker) +idle_thread.start() +cpu_thread.start() +cpu_ready.wait() +_test_sock.sendall(b"working") +idle_thread.join() +cpu_thread.join() +""" + with test_subprocess(cpu_vs_idle_script, wait_for_working=True) as subproc: + + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=True) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=2.0, + mode=1, # CPU mode + all_threads=True, + ) + collector.print_stats(show_summary=False, mode=1) + except (PermissionError, RuntimeError) as e: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + cpu_mode_output = captured_output.getvalue() + + # Test wall-clock mode (mode=0) - should capture both functions + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=2.0, + mode=0, # Wall-clock mode + all_threads=True, + ) + collector.print_stats(show_summary=False) + except (PermissionError, RuntimeError) as e: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + wall_mode_output = captured_output.getvalue() + + # Verify both modes captured samples + self.assertIn("Captured", cpu_mode_output) + self.assertIn("samples", cpu_mode_output) + self.assertIn("Captured", wall_mode_output) + self.assertIn("samples", wall_mode_output) + + # CPU mode should strongly favor cpu_active_worker over mostly_idle_worker + self.assertIn("cpu_active_worker", cpu_mode_output) + self.assertNotIn("idle_worker", cpu_mode_output) + + # Wall-clock mode should capture both types of work + self.assertIn("cpu_active_worker", wall_mode_output) + self.assertIn("idle_worker", wall_mode_output) + + def test_cpu_mode_with_no_samples(self): + """Test that CPU mode handles no samples gracefully when no samples are collected.""" + # Mock a collector that returns empty stats + mock_collector = PstatsCollector(sample_interval_usec=5000, skip_idle=True) + mock_collector.stats = {} + + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + mock.patch( + "profiling.sampling.sample.SampleProfiler" + ) as mock_profiler_class, + ): + mock_profiler = mock.MagicMock() + mock_profiler_class.return_value = mock_profiler + + profiling.sampling.sample.sample( + 12345, # dummy PID + mock_collector, + duration_sec=0.5, + mode=1, # CPU mode + all_threads=True, + ) + + mock_collector.print_stats(show_summary=False, mode=1) + + output = captured_output.getvalue() + + # Should see the "No samples were collected" message + self.assertIn("No samples were collected", output) + self.assertIn("CPU mode", output) + + +class TestGilModeFiltering(unittest.TestCase): + """Test GIL mode filtering functionality (--mode=gil).""" + + def test_gil_mode_validation(self): + """Test that CLI accepts gil mode choice correctly.""" + from profiling.sampling.cli import main + + test_args = [ + "profiling.sampling.cli", + "attach", + "12345", + "--mode", + "gil", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + try: + main() + except (SystemExit, OSError, RuntimeError): + pass # Expected due to invalid PID + + # Should have attempted to call sample with mode=2 (GIL mode) + mock_sample.assert_called_once() + call_args = mock_sample.call_args + # Check the mode parameter (should be in kwargs) + self.assertEqual(call_args.kwargs.get("mode"), 2) # PROFILING_MODE_GIL + + def test_gil_mode_sample_function_call(self): + """Test that sample() function correctly uses GIL mode.""" + with ( + mock.patch( + "profiling.sampling.sample.SampleProfiler" + ) as mock_profiler, + ): + # Mock the profiler instance + mock_instance = mock.Mock() + mock_profiler.return_value = mock_instance + + # Create a real collector instance + collector = PstatsCollector(sample_interval_usec=1000, skip_idle=True) + + # Call sample with GIL mode + profiling.sampling.sample.sample( + 12345, + collector, + mode=2, # PROFILING_MODE_GIL + duration_sec=1, + ) + + # Verify SampleProfiler was created with correct mode + mock_profiler.assert_called_once() + call_args = mock_profiler.call_args + self.assertEqual(call_args[1]["mode"], 2) # mode parameter + + # Verify profiler.sample was called + mock_instance.sample.assert_called_once() + + def test_gil_mode_cli_argument_parsing(self): + """Test CLI argument parsing for GIL mode with various options.""" + from profiling.sampling.cli import main + + test_args = [ + "profiling.sampling.cli", + "attach", + "12345", + "--mode", + "gil", + "-i", + "500", + "-d", + "5", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + try: + main() + except (SystemExit, OSError, RuntimeError): + pass # Expected due to invalid PID + + # Verify all arguments were parsed correctly + mock_sample.assert_called_once() + call_args = mock_sample.call_args + self.assertEqual(call_args.kwargs.get("mode"), 2) # GIL mode + self.assertEqual(call_args.kwargs.get("duration_sec"), 5) + + @requires_subprocess() + def test_gil_mode_integration_behavior(self): + """Integration test: GIL mode should capture GIL-holding threads.""" + # Create a test script with GIL-releasing operations + gil_test_script = """ +import time +import threading + +gil_ready = threading.Event() + +def gil_releasing_work(): + time.sleep(999999) + +def gil_holding_work(): + gil_ready.set() + x = 1 + while True: + x += 1 + +idle_thread = threading.Thread(target=gil_releasing_work) +cpu_thread = threading.Thread(target=gil_holding_work) +idle_thread.start() +cpu_thread.start() +gil_ready.wait() +_test_sock.sendall(b"working") +idle_thread.join() +cpu_thread.join() +""" + with test_subprocess(gil_test_script, wait_for_working=True) as subproc: + + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=True) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=2.0, + mode=2, # GIL mode + all_threads=True, + ) + collector.print_stats(show_summary=False) + except (PermissionError, RuntimeError) as e: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + gil_mode_output = captured_output.getvalue() + + # Test wall-clock mode for comparison + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=0.5, + mode=0, # Wall-clock mode + all_threads=True, + ) + collector.print_stats(show_summary=False) + except (PermissionError, RuntimeError) as e: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + wall_mode_output = captured_output.getvalue() + + # GIL mode should primarily capture GIL-holding work + # (Note: actual behavior depends on threading implementation) + self.assertIn("gil_holding_work", gil_mode_output) + + # Wall-clock mode should capture both types of work + self.assertIn("gil_holding_work", wall_mode_output) + + def test_mode_constants_are_defined(self): + """Test that all profiling mode constants are properly defined.""" + self.assertEqual(profiling.sampling.sample.PROFILING_MODE_WALL, 0) + self.assertEqual(profiling.sampling.sample.PROFILING_MODE_CPU, 1) + self.assertEqual(profiling.sampling.sample.PROFILING_MODE_GIL, 2) + + def test_parse_mode_function(self): + """Test the _parse_mode function with all valid modes.""" + from profiling.sampling.cli import _parse_mode + self.assertEqual(_parse_mode("wall"), 0) + self.assertEqual(_parse_mode("cpu"), 1) + self.assertEqual(_parse_mode("gil"), 2) + + # Test invalid mode raises KeyError + with self.assertRaises(KeyError): + _parse_mode("invalid") diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py new file mode 100644 index 00000000000..822f559561e --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -0,0 +1,714 @@ +"""Tests for sampling profiler core functionality.""" + +import io +from unittest import mock +import unittest + +try: + import _remote_debugging # noqa: F401 + from profiling.sampling.sample import SampleProfiler + from profiling.sampling.pstats_collector import PstatsCollector +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import force_not_colorized_test_class + + +def print_sampled_stats(stats, sort=-1, limit=None, show_summary=True, sample_interval_usec=100): + """Helper function to maintain compatibility with old test API. + + This wraps the new PstatsCollector.print_stats() API to work with the + existing test infrastructure. + """ + # Create a mock collector that populates stats correctly + collector = PstatsCollector(sample_interval_usec=sample_interval_usec) + + # Override create_stats to populate self.stats with the provided stats + def mock_create_stats(): + collector.stats = stats.stats + collector.create_stats = mock_create_stats + + # Call the new print_stats method + collector.print_stats(sort=sort, limit=limit, show_summary=show_summary) + + +class TestSampleProfiler(unittest.TestCase): + """Test the SampleProfiler class.""" + + def test_sample_profiler_initialization(self): + """Test SampleProfiler initialization with various parameters.""" + + # Mock RemoteUnwinder to avoid permission issues + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock.MagicMock() + + # Test basic initialization + profiler = SampleProfiler( + pid=12345, sample_interval_usec=1000, all_threads=False + ) + self.assertEqual(profiler.pid, 12345) + self.assertEqual(profiler.sample_interval_usec, 1000) + self.assertEqual(profiler.all_threads, False) + + # Test with all_threads=True + profiler = SampleProfiler( + pid=54321, sample_interval_usec=5000, all_threads=True + ) + self.assertEqual(profiler.pid, 54321) + self.assertEqual(profiler.sample_interval_usec, 5000) + self.assertEqual(profiler.all_threads, True) + + def test_sample_profiler_sample_method_timing(self): + """Test that the sample method respects duration and handles timing correctly.""" + + # Mock the unwinder to avoid needing a real process + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.return_value = [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=100000, all_threads=False + ) # 100ms interval + + # Mock collector + mock_collector = mock.MagicMock() + + # Mock time to control the sampling loop + start_time = 1000.0 + times = [ + start_time + i * 0.1 for i in range(12) + ] # 0, 0.1, 0.2, ..., 1.1 seconds + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=1) + + result = output.getvalue() + + # Should have captured approximately 10 samples (1 second / 0.1 second interval) + self.assertIn("Captured", result) + self.assertIn("samples", result) + + # Verify collector was called multiple times + self.assertGreaterEqual(mock_collector.collect.call_count, 5) + self.assertLessEqual(mock_collector.collect.call_count, 11) + + def test_sample_profiler_error_handling(self): + """Test that the sample method handles errors gracefully.""" + + # Mock unwinder that raises errors + mock_unwinder = mock.MagicMock() + error_sequence = [ + RuntimeError("Process died"), + [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ], + UnicodeDecodeError("utf-8", b"", 0, 1, "invalid"), + [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", + lineno=20, + funcname="test_func2", + ) + ], + ) + ], + OSError("Permission denied"), + ] + mock_unwinder.get_stack_trace.side_effect = error_sequence + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + mock_collector = mock.MagicMock() + + # Control timing to run exactly 5 samples + times = [0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06] + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.05) + + result = output.getvalue() + + # Should report error rate + self.assertIn("Error rate:", result) + self.assertIn("%", result) + + # Collector should have been called only for successful samples (should be > 0) + self.assertGreater(mock_collector.collect.call_count, 0) + self.assertLessEqual(mock_collector.collect.call_count, 3) + + def test_sample_profiler_missed_samples_warning(self): + """Test that the profiler warns about missed samples when sampling is too slow.""" + + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.return_value = [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + # Use very short interval that we'll miss + profiler = SampleProfiler( + pid=12345, sample_interval_usec=1000, all_threads=False + ) # 1ms interval + + mock_collector = mock.MagicMock() + + # Simulate slow sampling where we miss many samples + times = [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + ] # Extra time points to avoid StopIteration + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.5) + + result = output.getvalue() + + # Should warn about missed samples + self.assertIn("Warning: missed", result) + self.assertIn("samples from the expected total", result) + + def test_sample_profiler_keyboard_interrupt(self): + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.side_effect = [ + [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ], + KeyboardInterrupt(), + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + mock_collector = mock.MagicMock() + times = [0.0, 0.01, 0.02, 0.03, 0.04] + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + try: + profiler.sample(mock_collector, duration_sec=1.0) + except KeyboardInterrupt: + self.fail( + "KeyboardInterrupt was not handled by the profiler" + ) + result = output.getvalue() + self.assertIn("Interrupted by user.", result) + self.assertIn("Captured", result) + self.assertIn("samples", result) + self.assertNotIn("Warning: missed", result) + + +@force_not_colorized_test_class +class TestPrintSampledStats(unittest.TestCase): + """Test the print_sampled_stats function.""" + + def setUp(self): + """Set up test data.""" + # Mock stats data + self.mock_stats = mock.MagicMock() + self.mock_stats.stats = { + ("file1.py", 10, "func1"): ( + 100, + 100, + 0.5, + 0.5, + {}, + ), # cc, nc, tt, ct, callers + ("file2.py", 20, "func2"): (50, 50, 0.25, 0.3, {}), + ("file3.py", 30, "func3"): (200, 200, 1.5, 2.0, {}), + ("file4.py", 40, "func4"): ( + 10, + 10, + 0.001, + 0.001, + {}, + ), # millisecond range + ("file5.py", 50, "func5"): ( + 5, + 5, + 0.000001, + 0.000002, + {}, + ), # microsecond range + } + + def test_print_sampled_stats_basic(self): + """Test basic print_sampled_stats functionality.""" + + # Capture output + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Check header is present + self.assertIn("Profile Stats:", result) + self.assertIn("nsamples", result) + self.assertIn("tottime", result) + self.assertIn("cumtime", result) + + # Check functions are present + self.assertIn("func1", result) + self.assertIn("func2", result) + self.assertIn("func3", result) + + def test_print_sampled_stats_sorting(self): + """Test different sorting options.""" + + # Test sort by calls + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=0, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header) + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (200 calls) + self.assertIn("func3", data_lines[0]) + + # Test sort by time + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=1, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (1.5s time) + self.assertIn("func3", data_lines[0]) + + def test_print_sampled_stats_limit(self): + """Test limiting output rows.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, limit=2, sample_interval_usec=100 + ) + + result = output.getvalue() + + # Count function entries in the main stats section (not in summary) + lines = result.split("\n") + # Find where the main stats section ends (before summary) + main_section_lines = [] + for line in lines: + if "Summary of Interesting Functions:" in line: + break + main_section_lines.append(line) + + # Count function entries only in main section + func_count = sum( + 1 + for line in main_section_lines + if "func" in line and ".py" in line + ) + self.assertEqual(func_count, 2) + + def test_print_sampled_stats_time_units(self): + """Test proper time unit selection.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use seconds for the header since max time is > 1s + self.assertIn("tottime (s)", result) + self.assertIn("cumtime (s)", result) + + # Test with only microsecond-range times + micro_stats = mock.MagicMock() + micro_stats.stats = { + ("file1.py", 10, "func1"): (100, 100, 0.000005, 0.000010, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(micro_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use microseconds + self.assertIn("tottime (μs)", result) + self.assertIn("cumtime (μs)", result) + + def test_print_sampled_stats_summary(self): + """Test summary section generation.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Check summary sections are present + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn( + "Functions with Highest Direct/Cumulative Ratio (Hot Spots):", + result, + ) + self.assertIn( + "Functions with Highest Call Frequency (Indirect Calls):", result + ) + self.assertIn( + "Functions with Highest Call Magnification (Cumulative/Direct):", + result, + ) + + def test_print_sampled_stats_no_summary(self): + """Test disabling summary output.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=False, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Summary should not be present + self.assertNotIn("Summary of Interesting Functions:", result) + + def test_print_sampled_stats_empty_stats(self): + """Test with empty stats.""" + + empty_stats = mock.MagicMock() + empty_stats.stats = {} + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(empty_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should print message about no samples + self.assertIn("No samples were collected.", result) + + def test_print_sampled_stats_sample_percentage_sorting(self): + """Test sample percentage sorting options.""" + + # Add a function with high sample percentage (more direct calls than func3's 200) + self.mock_stats.stats[("expensive.py", 60, "expensive_func")] = ( + 300, # direct calls (higher than func3's 200) + 300, # cumulative calls + 1.0, # total time + 1.0, # cumulative time + {}, + ) + + # Test sort by sample percentage + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=3, sample_interval_usec=100 + ) # sample percentage + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if ".py" in l and "func" in l] + # expensive_func should be first (highest sample percentage) + self.assertIn("expensive_func", data_lines[0]) + + def test_print_sampled_stats_with_recursive_calls(self): + """Test print_sampled_stats with recursive calls where nc != cc.""" + + # Create stats with recursive calls (nc != cc) + recursive_stats = mock.MagicMock() + recursive_stats.stats = { + # (direct_calls, cumulative_calls, tt, ct, callers) - recursive function + ("recursive.py", 10, "factorial"): ( + 5, # direct_calls + 10, # cumulative_calls (appears more times in stack due to recursion) + 0.5, + 0.6, + {}, + ), + ("normal.py", 20, "normal_func"): ( + 3, # direct_calls + 3, # cumulative_calls (same as direct for non-recursive) + 0.2, + 0.2, + {}, + ), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(recursive_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should display recursive calls as "5/10" format + self.assertIn("5/10", result) # nc/cc format for recursive calls + self.assertIn("3", result) # just nc for non-recursive calls + self.assertIn("factorial", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_zero_call_counts(self): + """Test print_sampled_stats with zero call counts to trigger division protection.""" + + # Create stats with zero call counts + zero_stats = mock.MagicMock() + zero_stats.stats = { + ("file.py", 10, "zero_calls"): (0, 0, 0.0, 0.0, {}), # Zero calls + ("file.py", 20, "normal_func"): ( + 5, + 5, + 0.1, + 0.1, + {}, + ), # Normal function + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(zero_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should handle zero call counts gracefully + self.assertIn("zero_calls", result) + self.assertIn("zero_calls", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_sort_by_name(self): + """Test sort by function name option.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=-1, sample_interval_usec=100 + ) # sort by name + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header and summary) + # Data lines start with whitespace and numbers, and contain filename:lineno(function) + data_lines = [] + for line in lines: + # Skip header lines and summary sections + if ( + line.startswith(" ") + and "(" in line + and ")" in line + and not line.startswith( + " 1." + ) # Skip summary lines that start with times + and not line.startswith( + " 0." + ) # Skip summary lines that start with times + and not "per call" in line # Skip summary lines + and not "calls" in line # Skip summary lines + and not "total time" in line # Skip summary lines + and not "cumulative time" in line + ): # Skip summary lines + data_lines.append(line) + + # Extract just the function names for comparison + func_names = [] + import re + + for line in data_lines: + # Function name is between the last ( and ), accounting for ANSI color codes + match = re.search(r"\(([^)]+)\)$", line) + if match: + func_name = match.group(1) + # Remove ANSI color codes + func_name = re.sub(r"\x1b\[[0-9;]*m", "", func_name) + func_names.append(func_name) + + # Verify we extracted function names and they are sorted + self.assertGreater( + len(func_names), 0, "Should have extracted some function names" + ) + self.assertEqual( + func_names, + sorted(func_names), + f"Function names {func_names} should be sorted alphabetically", + ) + + def test_print_sampled_stats_with_zero_time_functions(self): + """Test summary sections with functions that have zero time.""" + + # Create stats with zero-time functions + zero_time_stats = mock.MagicMock() + zero_time_stats.stats = { + ("file1.py", 10, "zero_time_func"): ( + 5, + 5, + 0.0, + 0.0, + {}, + ), # Zero time + ("file2.py", 20, "normal_func"): ( + 3, + 3, + 0.1, + 0.1, + {}, + ), # Normal time + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + zero_time_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle zero-time functions gracefully in summary + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn("zero_time_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_malformed_qualified_names(self): + """Test summary generation with function names that don't contain colons.""" + + # Create stats with function names that would create malformed qualified names + malformed_stats = mock.MagicMock() + malformed_stats.stats = { + # Function name without clear module separation + ("no_colon_func", 10, "func"): (3, 3, 0.1, 0.1, {}), + ("", 20, "empty_filename_func"): (2, 2, 0.05, 0.05, {}), + ("normal.py", 30, "normal_func"): (5, 5, 0.2, 0.2, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + malformed_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle malformed names gracefully in summary aggregation + self.assertIn("Summary of Interesting Functions:", result) + # All function names should appear somewhere in the output + self.assertIn("func", result) + self.assertIn("empty_filename_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_recursive_call_stats_creation(self): + """Test create_stats with recursive call data to trigger total_rec_calls branch.""" + collector = PstatsCollector(sample_interval_usec=1000000) # 1 second + + # Simulate recursive function data where total_rec_calls would be set + # We need to manually manipulate the collector result to test this branch + collector.result = { + ("recursive.py", 10, "factorial"): { + "total_rec_calls": 3, # Non-zero recursive calls + "direct_calls": 5, + "cumulative_calls": 10, + }, + ("normal.py", 20, "normal_func"): { + "total_rec_calls": 0, # Zero recursive calls + "direct_calls": 2, + "cumulative_calls": 5, + }, + } + + collector.create_stats() + + # Check that recursive calls are handled differently from non-recursive + factorial_stats = collector.stats[("recursive.py", 10, "factorial")] + normal_stats = collector.stats[("normal.py", 20, "normal_func")] + + # factorial should use cumulative_calls (10) as nc + self.assertEqual( + factorial_stats[1], 10 + ) # nc should be cumulative_calls + self.assertEqual(factorial_stats[0], 5) # cc should be direct_calls + + # normal_func should use cumulative_calls as nc + self.assertEqual(normal_stats[1], 5) # nc should be cumulative_calls + self.assertEqual(normal_stats[0], 2) # cc should be direct_calls diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_trend_tracker.py b/Lib/test/test_profiling/test_sampling_profiler/test_trend_tracker.py new file mode 100644 index 00000000000..43b23be0fe3 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_trend_tracker.py @@ -0,0 +1,97 @@ +"""Simple unit tests for TrendTracker.""" + +import unittest +from test.support import requires +from test.support.import_helper import import_module + +# Only run these tests if curses is available +requires("curses") +curses = import_module("curses") + +from profiling.sampling.live_collector.trend_tracker import TrendTracker + + +class TestTrendTracker(unittest.TestCase): + """Tests for TrendTracker class.""" + + def setUp(self): + """Set up test fixtures.""" + self.colors = { + "trend_up": curses.A_BOLD, + "trend_down": curses.A_REVERSE, + "trend_stable": curses.A_NORMAL, + } + + def test_basic_trend_detection(self): + """Test basic up/down/stable trend detection.""" + tracker = TrendTracker(self.colors, enabled=True) + + # First value is always stable + self.assertEqual(tracker.update("func1", "nsamples", 10), "stable") + + # Increasing value + self.assertEqual(tracker.update("func1", "nsamples", 20), "up") + + # Decreasing value + self.assertEqual(tracker.update("func1", "nsamples", 15), "down") + + # Small change (within threshold) is stable + self.assertEqual(tracker.update("func1", "nsamples", 15.0001), "stable") + + def test_multiple_metrics(self): + """Test tracking multiple metrics simultaneously.""" + tracker = TrendTracker(self.colors, enabled=True) + + trends = tracker.update_metrics("func1", { + "nsamples": 10, + "tottime": 5.0, + }) + + self.assertEqual(trends["nsamples"], "stable") + self.assertEqual(trends["tottime"], "stable") + + # Update with changes + trends = tracker.update_metrics("func1", { + "nsamples": 15, + "tottime": 3.0, + }) + + self.assertEqual(trends["nsamples"], "up") + self.assertEqual(trends["tottime"], "down") + + def test_toggle_enabled(self): + """Test enable/disable toggle.""" + tracker = TrendTracker(self.colors, enabled=True) + self.assertTrue(tracker.enabled) + + tracker.toggle() + self.assertFalse(tracker.enabled) + + # When disabled, should return A_NORMAL + self.assertEqual(tracker.get_color("up"), curses.A_NORMAL) + + def test_get_color(self): + """Test color selection for trends.""" + tracker = TrendTracker(self.colors, enabled=True) + + self.assertEqual(tracker.get_color("up"), curses.A_BOLD) + self.assertEqual(tracker.get_color("down"), curses.A_REVERSE) + self.assertEqual(tracker.get_color("stable"), curses.A_NORMAL) + + def test_clear(self): + """Test clearing tracked values.""" + tracker = TrendTracker(self.colors, enabled=True) + + # Add some data + tracker.update("func1", "nsamples", 10) + tracker.update("func1", "nsamples", 20) + + # Clear + tracker.clear() + + # After clear, first update should be stable + self.assertEqual(tracker.update("func1", "nsamples", 30), "stable") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_profiling/test_tracing_profiler.py similarity index 95% rename from Lib/test/test_cprofile.py rename to Lib/test/test_profiling/test_tracing_profiler.py index 192c8eab26e..d09ca441d4a 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_profiling/test_tracing_profiler.py @@ -4,7 +4,7 @@ import unittest # rip off all interesting stuff from test_profile -import cProfile +import profiling.tracing as cProfile import tempfile import textwrap from test.test_profile import ProfileTest, regenerate_expected_output @@ -125,21 +125,22 @@ def test_throw(self): """ gh-106152 generator.throw() should trigger a call in cProfile - In the any() call below, there should be two entries for the generator: - * one for the call to __next__ which gets a True and terminates any - * one when the generator is garbage collected which will effectively - do a throw. """ + + def gen(): + yield + pr = self.profilerclass() pr.enable() - any(a == 1 for a in (1, 2)) + g = gen() + try: + g.throw(SyntaxError) + except SyntaxError: + pass pr.disable() pr.create_stats() - for func, (cc, nc, _, _, _) in pr.stats.items(): - if func[2] == "<genexpr>": - self.assertEqual(cc, 1) - self.assertEqual(nc, 1) + self.assertTrue(any("throw" in func[2] for func in pr.stats.keys())), def test_bad_descriptor(self): # gh-132250 diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index cea241b0f20..26aefdbf042 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -87,8 +87,8 @@ def test_property_decorator_baseclass(self): self.assertEqual(base.spam, 10) self.assertEqual(base._spam, 10) delattr(base, "spam") - self.assertTrue(not hasattr(base, "spam")) - self.assertTrue(not hasattr(base, "_spam")) + self.assertNotHasAttr(base, "spam") + self.assertNotHasAttr(base, "_spam") base.spam = 20 self.assertEqual(base.spam, 20) self.assertEqual(base._spam, 20) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index c1728f5019d..7e4f4828ce0 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,14 +1,13 @@ import unittest from test.support import ( - is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose + is_android, is_apple_mobile, is_wasm32, reap_children, verbose, warnings_helper ) from test.support.import_helper import import_module -from test.support.os_helper import TESTFN, unlink # Skip these tests if termios is not available import_module('termios') -if is_android or is_apple_mobile or is_emscripten or is_wasi: +if is_android or is_apple_mobile or is_wasm32: raise unittest.SkipTest("pty is not available on this platform") import errno @@ -20,7 +19,6 @@ import signal import socket import io # readline -import warnings TEST_STRING_1 = b"I wish to buy a fish license.\n" TEST_STRING_2 = b"For my pet fish, Eric.\n" @@ -195,6 +193,7 @@ def test_openpty(self): s2 = _readline(master_fd) self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_fork(self): debug("calling pty.fork()") pid, master_fd = pty.fork() @@ -230,6 +229,7 @@ def test_fork(self): os._exit(2) os._exit(4) else: + self.assertFalse(os.get_inheritable(master_fd)) debug("Waiting for child (%d) to finish." % pid) # In verbose mode, we have to consume the debug output from the # child or the child will block, causing this test to hang in the @@ -296,27 +296,29 @@ def test_master_read(self): self.assertEqual(data, b"") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_spawn_doesnt_hang(self): - self.addCleanup(unlink, TESTFN) - with open(TESTFN, 'wb') as f: - STDOUT_FILENO = 1 - dup_stdout = os.dup(STDOUT_FILENO) - os.dup2(f.fileno(), STDOUT_FILENO) - buf = b'' - def master_read(fd): - nonlocal buf - data = os.read(fd, 1024) - buf += data - return data + # gh-140482: Do the test in a pty.fork() child to avoid messing + # with the interactive test runner's terminal settings. + pid, fd = pty.fork() + if pid == pty.CHILD: + pty.spawn([sys.executable, '-c', 'print("hi there")']) + os._exit(0) + + try: + buf = bytearray() try: - pty.spawn([sys.executable, '-c', 'print("hi there")'], - master_read) - finally: - os.dup2(dup_stdout, STDOUT_FILENO) - os.close(dup_stdout) - self.assertEqual(buf, b'hi there\r\n') - with open(TESTFN, 'rb') as f: - self.assertEqual(f.read(), b'hi there\r\n') + while (data := os.read(fd, 1024)) != b'': + buf.extend(data) + except OSError as e: + if e.errno != errno.EIO: + raise + + (pid, status) = os.waitpid(pid, 0) + self.assertEqual(status, 0) + self.assertEqual(buf.take_bytes(), b"hi there\r\n") + finally: + os.close(fd) class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py index 6dc51e4371d..3c8ed251aca 100644 --- a/Lib/test/test_pulldom.py +++ b/Lib/test/test_pulldom.py @@ -46,7 +46,7 @@ def test_parse_semantics(self): items = pulldom.parseString(SMALL_SAMPLE) evt, node = next(items) # Just check the node is a Document: - self.assertTrue(hasattr(node, "createElement")) + self.assertHasAttr(node, "createElement") self.assertEqual(pulldom.START_DOCUMENT, evt) evt, node = next(items) self.assertEqual(pulldom.START_ELEMENT, evt) @@ -192,7 +192,7 @@ def _test_thorough(self, pd, before_root=True): evt, node = next(pd) self.assertEqual(pulldom.START_DOCUMENT, evt) # Just check the node is a Document: - self.assertTrue(hasattr(node, "createElement")) + self.assertHasAttr(node, "createElement") if before_root: evt, node = next(pd) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index df05cd07d7e..79ef178f380 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -11,7 +11,6 @@ import pyclbr from unittest import TestCase, main as unittest_main from test.test_importlib import util as test_importlib_util -import warnings StaticMethodType = type(staticmethod(lambda: None)) @@ -103,7 +102,7 @@ def ismethod(oclass, obj, name): for name, value in dict.items(): if name in ignore: continue - self.assertHasAttr(module, name, ignore) + self.assertHasAttr(module, name) py_item = getattr(module, name) if isinstance(value, pyclbr.Function): self.assertIsInstance(py_item, (FunctionType, BuiltinFunctionType)) @@ -246,15 +245,12 @@ def test_others(self): # These were once some of the longest modules. cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator cm('pickle', ignore=('partial', 'PickleBuffer')) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - cm('sre_parse', ignore=('dump', 'groups', 'pos')) # from sre_constants import *; property with temporary_main_spec(): cm( 'pdb', # pyclbr does not handle elegantly `typing` or properties ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget', 'curframe_locals', - '_InteractState'), + '_InteractState', 'rlcompleter'), ) cm('pydoc', ignore=('input', 'output',)) # properties diff --git a/Lib/test/test_pydoc/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py index 3cc2d5bd57f..412aa3743e4 100644 --- a/Lib/test/test_pydoc/pydocfodder.py +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -87,6 +87,8 @@ def B_classmethod(cls, x): object_repr = object.__repr__ get = {}.get # same name dict_get = {}.get + from math import sin + B.B_classmethod_ref = B.B_classmethod @@ -186,3 +188,4 @@ def __call__(self, inst): object_repr = object.__repr__ get = {}.get # same name dict_get = {}.get +from math import sin # noqa: F401 diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index ac88b3c6f13..c640416327e 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -11,7 +11,6 @@ import _pickle import pkgutil import re -import stat import tempfile import test.support import time @@ -33,7 +32,7 @@ assert_python_failure, spawn_python) from test.support import threading_helper from test.support import (reap_children, captured_stdout, - captured_stderr, is_emscripten, is_wasi, + captured_stderr, is_wasm32, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) from test.test_pydoc import pydoc_mod @@ -553,7 +552,7 @@ class object # of the known subclasses of object. (doc.docclass() used to # fail if HeapType was imported before running this test, like # when running tests sequentially.) - from _testcapi import HeapType + from _testcapi import HeapType # noqa: F401 except ImportError: pass text = doc.docclass(object) @@ -1300,7 +1299,6 @@ def test_apropos_with_unreadable_dir(self): self.assertEqual(out.getvalue(), '') self.assertEqual(err.getvalue(), '') - @os_helper.skip_unless_working_chmod def test_apropos_empty_doc(self): pkgdir = os.path.join(TESTFN, 'walkpkg') os.mkdir(pkgdir) @@ -1308,14 +1306,9 @@ def test_apropos_empty_doc(self): init_path = os.path.join(pkgdir, '__init__.py') with open(init_path, 'w') as fobj: fobj.write("foo = 1") - current_mode = stat.S_IMODE(os.stat(pkgdir).st_mode) - try: - os.chmod(pkgdir, current_mode & ~stat.S_IEXEC) - with self.restrict_walk_packages(path=[TESTFN]), captured_stdout() as stdout: - pydoc.apropos('') - self.assertIn('walkpkg', stdout.getvalue()) - finally: - os.chmod(pkgdir, current_mode) + with self.restrict_walk_packages(path=[TESTFN]), captured_stdout() as stdout: + pydoc.apropos('') + self.assertIn('walkpkg', stdout.getvalue()) def test_url_search_package_error(self): # URL handler search should cope with packages that raise exceptions @@ -1341,47 +1334,6 @@ def test_url_search_package_error(self): finally: sys.path[:] = saved_paths - @unittest.skip('causes undesirable side-effects (#20128)') - def test_modules(self): - # See Helper.listmodules(). - num_header_lines = 2 - num_module_lines_min = 5 # Playing it safe. - num_footer_lines = 3 - expected = num_header_lines + num_module_lines_min + num_footer_lines - - output = StringIO() - helper = pydoc.Helper(output=output) - helper('modules') - result = output.getvalue().strip() - num_lines = len(result.splitlines()) - - self.assertGreaterEqual(num_lines, expected) - - @unittest.skip('causes undesirable side-effects (#20128)') - def test_modules_search(self): - # See Helper.listmodules(). - expected = 'pydoc - ' - - output = StringIO() - helper = pydoc.Helper(output=output) - with captured_stdout() as help_io: - helper('modules pydoc') - result = help_io.getvalue() - - self.assertIn(expected, result) - - @unittest.skip('some buildbots are not cooperating (#20128)') - def test_modules_search_builtin(self): - expected = 'gc - ' - - output = StringIO() - helper = pydoc.Helper(output=output) - with captured_stdout() as help_io: - helper('modules garbage') - result = help_io.getvalue() - - self.assertTrue(result.startswith(expected)) - def test_importfile(self): try: loaded_pydoc = pydoc.importfile(pydoc.__file__) @@ -1461,7 +1413,7 @@ def test_special_form(self): self.assertIn('NoReturn = typing.NoReturn', doc) self.assertIn(typing.NoReturn.__doc__.strip().splitlines()[0], doc) else: - self.assertIn('NoReturn = class _SpecialForm(_Final)', doc) + self.assertIn('NoReturn = class _SpecialForm(_Final, _NotIterable)', doc) def test_typing_pydoc(self): def foo(data: typing.List[typing.Any], @@ -1946,9 +1898,11 @@ def test_text_doc_routines_in_class(self, cls=pydocfodder.B): if not support.MISSING_C_DOCSTRINGS: self.assertIn(' | get(key, default=None, /) method of builtins.dict instance', lines) self.assertIn(' | dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | sin(x, /)', lines) else: self.assertIn(' | get(...) method of builtins.dict instance', lines) self.assertIn(' | dict_get = get(...) method of builtins.dict instance', lines) + self.assertIn(' | sin(object, /)', lines) lines = self.getsection(result, f' | Class methods {where}:', ' | ' + '-'*70) self.assertIn(' | B_classmethod(x)', lines) @@ -2034,6 +1988,11 @@ def test_text_doc_routines_in_module(self): self.assertIn(' __repr__(...) unbound builtins.object method', lines) self.assertIn(' object_repr = __repr__(...) unbound builtins.object method', lines) + # builtin functions + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' sin(x, /)', lines) + else: + self.assertIn(' sin(object, /)', lines) def test_html_doc_routines_in_module(self): doc = pydoc.HTMLDoc() @@ -2074,9 +2033,15 @@ def test_html_doc_routines_in_module(self): self.assertIn(' __repr__(...) unbound builtins.object method', lines) self.assertIn(' object_repr = __repr__(...) unbound builtins.object method', lines) + # builtin functions + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' sin(x, /)', lines) + else: + self.assertIn(' sin(object, /)', lines) + @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Socket server not available on Emscripten/WASI." ) class PydocServerTest(unittest.TestCase): diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 1d56ccd71cf..74a75458289 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -1,20 +1,23 @@ # XXX TypeErrors on calling handlers, or on bad return values from a # handler, are obscure and unhelpful. +import abc +import functools import os +import re import sys import sysconfig +import textwrap import unittest import traceback from io import BytesIO from test import support -from test.support import os_helper - +from test.support import import_helper, os_helper +from test.support import sortdict +from unittest import mock from xml.parsers import expat from xml.parsers.expat import errors -from test.support import sortdict - class SetAttributeTest(unittest.TestCase): def setUp(self): @@ -436,6 +439,19 @@ def test7(self): "<!--abc-->", "4", "<!--def-->", "5", "</a>"], "buffered text not properly split") + def test_change_character_data_handler_in_callback(self): + # Test that xmlparse_handler_setter() properly handles + # the special case "parser.CharacterDataHandler = None". + def handler(*args): + parser.CharacterDataHandler = None + + handler_wrapper = mock.Mock(wraps=handler) + parser = expat.ParserCreate() + parser.CharacterDataHandler = handler_wrapper + parser.Parse(b"<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", True) + handler_wrapper.assert_called_once() + self.assertIsNone(parser.CharacterDataHandler) + # Test handling of exception from callback: class HandlerExceptionTest(unittest.TestCase): @@ -595,7 +611,7 @@ def test_unchanged_size(self): def test_disabling_buffer(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>" + b'a' * 512 xml2 = b'b' * 1024 - xml3 = b'c' * 1024 + b'</a>'; + xml3 = b'c' * 1024 + b'</a>' parser = expat.ParserCreate() parser.CharacterDataHandler = self.counting_handler parser.buffer_text = 1 @@ -668,6 +684,23 @@ def test_change_size_2(self): parser.Parse(xml2, True) self.assertEqual(self.n, 4) +class ElementDeclHandlerTest(unittest.TestCase): + def test_trigger_leak(self): + # Unfixed, this test would leak the memory of the so-called + # "content model" in function ``my_ElementDeclHandler`` of pyexpat. + # See https://github.com/python/cpython/issues/140593. + data = textwrap.dedent('''\ + <!DOCTYPE quotations SYSTEM "quotations.dtd" [ + <!ELEMENT root ANY> + ]> + <root/> + ''').encode('UTF-8') + + parser = expat.ParserCreate() + parser.NotStandaloneHandler = lambda: 1.234 # arbitrary float + parser.ElementDeclHandler = lambda _1, _2: None + self.assertRaises(TypeError, parser.Parse, data, True) + class MalformedInputTest(unittest.TestCase): def test1(self): xml = b"\0\r\n" @@ -755,6 +788,42 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ParentParserLifetimeTest(unittest.TestCase): + """ + Subparsers make use of their parent XML_Parser inside of Expat. + As a result, parent parsers need to outlive subparsers. + + See https://github.com/python/cpython/issues/139400. + """ + + def test_parent_parser_outlives_its_subparsers__single(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + def test_parent_parser_outlives_its_subparsers__multiple(self): + parser = expat.ParserCreate() + subparser_one = parser.ExternalEntityParserCreate(None) + subparser_two = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + def test_parent_parser_outlives_its_subparsers__chain(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + subsubparser = subparser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parsers + # while they are still being referenced by a related subparser. + del parser + del subparser + + class ReparseDeferralTest(unittest.TestCase): def test_getter_setter_round_trip(self): parser = expat.ParserCreate() @@ -809,5 +878,257 @@ def start_element(name, _): self.assertEqual(started, ['doc']) +class AttackProtectionTestBase(abc.ABC): + """ + Base class for testing protections against XML payloads with + disproportionate amplification. + + The protections being tested should detect and prevent attacks + that leverage disproportionate amplification from small inputs. + """ + + @staticmethod + def exponential_expansion_payload(*, nrows, ncols, text='.'): + """Create a billion laughs attack payload. + + Be careful: the number of total items is pow(n, k), thereby + requiring at least pow(ncols, nrows) * sizeof(text) memory! + """ + template = textwrap.dedent(f"""\ + <?xml version="1.0"?> + <!DOCTYPE doc [ + <!ENTITY row0 "{text}"> + <!ELEMENT doc (#PCDATA)> + {{body}} + ]> + <doc>&row{nrows};</doc> + """).rstrip() + + body = '\n'.join( + f'<!ENTITY row{i + 1} "{f"&row{i};" * ncols}">' + for i in range(nrows) + ) + body = textwrap.indent(body, ' ' * 4) + return template.format(body=body) + + def test_payload_generation(self): + # self-test for exponential_expansion_payload() + payload = self.exponential_expansion_payload(nrows=2, ncols=3) + self.assertEqual(payload, textwrap.dedent("""\ + <?xml version="1.0"?> + <!DOCTYPE doc [ + <!ENTITY row0 "."> + <!ELEMENT doc (#PCDATA)> + <!ENTITY row1 "&row0;&row0;&row0;"> + <!ENTITY row2 "&row1;&row1;&row1;"> + ]> + <doc>&row2;</doc> + """).rstrip()) + + def assert_root_parser_failure(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) is invalid for a sub-parser.""" + msg = "parser must be a root parser" + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + @abc.abstractmethod + def assert_rejected(self, func, /, *args, **kwargs): + """Assert that func(*args, **kwargs) triggers the attack protection. + + Note: this method must ensure that the attack protection being tested + is the one that is actually triggered at runtime, e.g., by matching + the exact error message. + """ + + @abc.abstractmethod + def set_activation_threshold(self, parser, threshold): + """Set the activation threshold for the tested protection.""" + + @abc.abstractmethod + def set_maximum_amplification(self, parser, max_factor): + """Set the maximum amplification factor for the tested protection.""" + + @abc.abstractmethod + def test_set_activation_threshold__threshold_reached(self): + """Test when the activation threshold is exceeded.""" + + @abc.abstractmethod + def test_set_activation_threshold__threshold_not_reached(self): + """Test when the activation threshold is not exceeded.""" + + def test_set_activation_threshold__invalid_threshold_type(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_activation_threshold, parser) + + self.assertRaises(TypeError, setter, 1.0) + self.assertRaises(TypeError, setter, -1.5) + self.assertRaises(ValueError, setter, -5) + + def test_set_activation_threshold__invalid_threshold_range(self): + _testcapi = import_helper.import_module("_testcapi") + parser = expat.ParserCreate() + setter = functools.partial(self.set_activation_threshold, parser) + + self.assertRaises(OverflowError, setter, _testcapi.ULLONG_MAX + 1) + + def test_set_activation_threshold__fail_for_subparser(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + setter = functools.partial(self.set_activation_threshold, subparser) + self.assert_root_parser_failure(setter, 12345) + + @abc.abstractmethod + def test_set_maximum_amplification__amplification_exceeded(self): + """Test when the amplification factor is exceeded.""" + + @abc.abstractmethod + def test_set_maximum_amplification__amplification_not_exceeded(self): + """Test when the amplification factor is not exceeded.""" + + def test_set_maximum_amplification__infinity(self): + inf = float('inf') # an 'inf' threshold is allowed by Expat + parser = expat.ParserCreate() + self.assertIsNone(self.set_maximum_amplification(parser, inf)) + + def test_set_maximum_amplification__invalid_max_factor_type(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_maximum_amplification, parser) + + self.assertRaises(TypeError, setter, None) + self.assertRaises(TypeError, setter, 'abc') + + def test_set_maximum_amplification__invalid_max_factor_range(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_maximum_amplification, parser) + + msg = re.escape("'max_factor' must be at least 1.0") + self.assertRaisesRegex(expat.ExpatError, msg, setter, float('nan')) + self.assertRaisesRegex(expat.ExpatError, msg, setter, 0.99) + + def test_set_maximum_amplification__fail_for_subparser(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + setter = functools.partial(self.set_maximum_amplification, subparser) + self.assert_root_parser_failure(setter, 123.45) + + +@unittest.skipIf(expat.version_info < (2, 4, 0), "requires Expat >= 2.4.0") +class ExpansionProtectionTest(AttackProtectionTestBase, unittest.TestCase): + + def assert_rejected(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) hits the allocation limit.""" + msg = ( + r"limit on input amplification factor \(from DTD and entities\) " + r"breached: line \d+, column \d+" + ) + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + def set_activation_threshold(self, parser, threshold): + return parser.SetBillionLaughsAttackProtectionActivationThreshold(threshold) + + def set_maximum_amplification(self, parser, max_factor): + return parser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor) + + def test_set_activation_threshold__threshold_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be always reached. + self.set_activation_threshold(parser, 3) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_activation_threshold__threshold_not_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be never reached. + self.set_activation_threshold(parser, pow(10, 5)) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assertIsNotNone(parser.Parse(payload, True)) + + def test_set_maximum_amplification__amplification_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to always be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + # Craft a payload for which the peak amplification factor is > 1.0. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_maximum_amplification__amplification_not_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to never be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1e4)) + # Craft a payload for which the peak amplification factor is < 1e4. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assertIsNotNone(parser.Parse(payload, True)) + + +@unittest.skipIf(expat.version_info < (2, 7, 2), "requires Expat >= 2.7.2") +class MemoryProtectionTest(AttackProtectionTestBase, unittest.TestCase): + + # NOTE: with the default Expat configuration, the billion laughs protection + # may hit before the allocation limiter if exponential_expansion_payload() + # is not carefully parametrized. As such, the payloads should be chosen so + # that either the allocation limiter is hit before other protections are + # triggered or no protection at all is triggered. + + def assert_rejected(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) hits the allocation limit.""" + msg = r"out of memory: line \d+, column \d+" + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + def set_activation_threshold(self, parser, threshold): + return parser.SetAllocTrackerActivationThreshold(threshold) + + def set_maximum_amplification(self, parser, max_factor): + return parser.SetAllocTrackerMaximumAmplification(max_factor) + + def test_set_activation_threshold__threshold_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be always reached. + self.set_activation_threshold(parser, 3) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_activation_threshold__threshold_not_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be never reached. + self.set_activation_threshold(parser, pow(10, 5)) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assertIsNotNone(parser.Parse(payload, True)) + + def test_set_maximum_amplification__amplification_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to always be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + # Craft a payload for which the peak amplification factor is > 1.0. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_maximum_amplification__amplification_not_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to never be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1e4)) + # Craft a payload for which the peak amplification factor is < 1e4. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assertIsNotNone(parser.Parse(payload, True)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py index 8359d984462..2f37bff6df8 100644 --- a/Lib/test/test_pyrepl/__init__.py +++ b/Lib/test/test_pyrepl/__init__.py @@ -1,14 +1,10 @@ import os import sys -from test.support import requires, load_package_tests -from test.support.import_helper import import_module +from test.support import import_helper, load_package_tests + if sys.platform != "win32": - # On non-Windows platforms, testing pyrepl currently requires that the - # 'curses' resource be given on the regrtest command line using the -u - # option. Additionally, we need to attempt to import curses and readline. - requires("curses") - curses = import_module("curses") + import_helper.import_module("termios") def load_tests(*args): diff --git a/Lib/test/test_pyrepl/eio_test_script.py b/Lib/test/test_pyrepl/eio_test_script.py new file mode 100644 index 00000000000..e3ea6caef58 --- /dev/null +++ b/Lib/test/test_pyrepl/eio_test_script.py @@ -0,0 +1,94 @@ +import errno +import fcntl +import os +import pty +import signal +import sys +import termios + + +def handler(sig, f): + pass + + +def create_eio_condition(): + # SIGINT handler used to produce an EIO. + # See https://github.com/python/cpython/issues/135329. + try: + master_fd, slave_fd = pty.openpty() + child_pid = os.fork() + if child_pid == 0: + try: + os.setsid() + fcntl.ioctl(slave_fd, termios.TIOCSCTTY, 0) + child_process_group_id = os.getpgrp() + grandchild_pid = os.fork() + if grandchild_pid == 0: + os.setpgid(0, 0) # set process group for grandchild + os.dup2(slave_fd, 0) # redirect stdin + if slave_fd > 2: + os.close(slave_fd) + # Fork grandchild for terminal control manipulation + if os.fork() == 0: + sys.exit(0) # exit the child process that was just obtained + else: + try: + os.tcsetpgrp(0, child_process_group_id) + except OSError: + pass + sys.exit(0) + else: + # Back to child + try: + os.setpgid(grandchild_pid, grandchild_pid) + except ProcessLookupError: + pass + os.tcsetpgrp(slave_fd, grandchild_pid) + if slave_fd > 2: + os.close(slave_fd) + os.waitpid(grandchild_pid, 0) + # Manipulate terminal control to create EIO condition + os.tcsetpgrp(master_fd, child_process_group_id) + # Now try to read from master - this might cause EIO + try: + os.read(master_fd, 1) + except OSError as e: + if e.errno == errno.EIO: + print(f"Setup created EIO condition: {e}", file=sys.stderr) + sys.exit(0) + except Exception as setup_e: + print(f"Setup error: {setup_e}", file=sys.stderr) + sys.exit(1) + else: + # Parent process + os.close(slave_fd) + os.waitpid(child_pid, 0) + # Now replace stdin with master_fd and try to read + os.dup2(master_fd, 0) + os.close(master_fd) + # This should now trigger EIO + print(f"Unexpectedly got input: {input()!r}", file=sys.stderr) + sys.exit(0) + except OSError as e: + if e.errno == errno.EIO: + print(f"Got EIO: {e}", file=sys.stderr) + sys.exit(1) + elif e.errno == errno.ENXIO: + print(f"Got ENXIO (no such device): {e}", file=sys.stderr) + sys.exit(1) # Treat ENXIO as success too + else: + print(f"Got other OSError: errno={e.errno} {e}", file=sys.stderr) + sys.exit(2) + except EOFError as e: + print(f"Got EOFError: {e}", file=sys.stderr) + sys.exit(3) + except Exception as e: + print(f"Got unexpected error: {type(e).__name__}: {e}", file=sys.stderr) + sys.exit(4) + + +if __name__ == "__main__": + # Set up signal handler for coordination + signal.signal(signal.SIGUSR1, lambda *a: create_eio_condition()) + print("READY", flush=True) + signal.pause() diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index edfe6ac4748..69d9612b70d 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -3,6 +3,8 @@ from unittest.mock import patch from test import support +from _pyrepl import terminfo + try: from _pyrepl.console import Event from _pyrepl import base_eventqueue @@ -172,17 +174,22 @@ def _push(keys): self.assertEqual(eq.get(), _event("key", "a")) +class EmptyTermInfo(terminfo.TermInfo): + def get(self, cap: str) -> bytes: + return b"" + + @unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") class TestUnixEventQueue(EventQueueTestBase, unittest.TestCase): def setUp(self): - self.enterContext(patch("_pyrepl.curses.tigetstr", lambda x: b"")) self.file = tempfile.TemporaryFile() def tearDown(self) -> None: self.file.close() def make_eventqueue(self) -> base_eventqueue.BaseEventQueue: - return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8") + ti = EmptyTermInfo("ansi") + return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8", ti) @unittest.skipUnless(support.MS_WINDOWS, "No Windows event queue on Unix") diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index a20719033fc..fd4530ebc00 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -1,7 +1,7 @@ import contextlib import io -import unittest import warnings +import unittest from unittest.mock import patch from textwrap import dedent @@ -113,7 +113,7 @@ def test_runsource_show_syntax_error_location(self): r = """ def f(x, x): ... ^ -SyntaxError: duplicate argument 'x' in function definition""" +SyntaxError: duplicate parameter 'x' in function definition""" self.assertIn(r, f.getvalue()) def test_runsource_shows_syntax_error_for_failed_compilation(self): @@ -293,7 +293,7 @@ def f(): """) with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("default") + warnings.simplefilter("always") console.runsource(code) count = sum("'return' in a 'finally' block" in str(w.message) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 93029ab6e08..e298b2add52 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1,3 +1,4 @@ +import importlib import io import itertools import os @@ -8,10 +9,11 @@ import subprocess import sys import tempfile -from unittest import TestCase, skipUnless, skipIf +from pkgutil import ModuleInfo +from unittest import TestCase, skipUnless, skipIf, SkipTest from unittest.mock import patch -from test.support import force_not_colorized, make_clean_env -from test.support import SHORT_TIMEOUT, STDLIB_DIR +from test.support import force_not_colorized, make_clean_env, Py_DEBUG +from test.support import has_subprocess_support, SHORT_TIMEOUT, STDLIB_DIR from test.support.import_helper import import_module from test.support.os_helper import EnvironmentVarGuard, unlink @@ -25,9 +27,16 @@ code_to_events, ) from _pyrepl.console import Event -from _pyrepl._module_completer import ImportParser, ModuleCompleter -from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig, - _ReadlineWrapper) +from _pyrepl._module_completer import ( + ImportParser, + ModuleCompleter, + HARDCODED_SUBMODULES, +) +from _pyrepl.readline import ( + ReadlineAlikeReader, + ReadlineConfig, + _ReadlineWrapper, +) from _pyrepl.readline import multiline_input as readline_multiline_input try: @@ -37,6 +46,10 @@ class ReplTestCase(TestCase): + def setUp(self): + if not has_subprocess_support: + raise SkipTest("test module requires subprocess") + def run_repl( self, repl_input: str | list[str], @@ -46,6 +59,7 @@ def run_repl( cwd: str | None = None, skip: bool = False, timeout: float = SHORT_TIMEOUT, + exit_on_output: str | None = None, ) -> tuple[str, int]: temp_dir = None if cwd is None: @@ -59,6 +73,7 @@ def run_repl( cwd=cwd, skip=skip, timeout=timeout, + exit_on_output=exit_on_output, ) finally: if temp_dir is not None: @@ -73,6 +88,7 @@ def _run_repl( cwd: str, skip: bool, timeout: float, + exit_on_output: str | None, ) -> tuple[str, int]: assert pty master_fd, slave_fd = pty.openpty() @@ -118,6 +134,11 @@ def _run_repl( except OSError: break output.append(data) + if exit_on_output is not None: + output = ["".join(output)] + if exit_on_output in output[0]: + process.kill() + break else: os.close(master_fd) process.kill() @@ -452,6 +473,11 @@ def test_auto_indent_default(self): ) # fmt: on + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + def test_auto_indent_continuation(self): # auto indenting according to previous user indentation # fmt: off @@ -912,7 +938,13 @@ def test_func(self): class TestPyReplModuleCompleter(TestCase): def setUp(self): + # Make iter_modules() search only the standard library. + # This makes the test more reliable in case there are + # other user packages/scripts on PYTHONPATH which can + # interfere with the completions. + lib_path = os.path.dirname(importlib.__path__[0]) self._saved_sys_path = sys.path + sys.path = [lib_path] def tearDown(self): sys.path = self._saved_sys_path @@ -920,19 +952,12 @@ def tearDown(self): def prepare_reader(self, events, namespace): console = FakeConsole(events) config = ReadlineConfig() + config.module_completer = ModuleCompleter(namespace) config.readline_completer = rlcompleter.Completer(namespace).complete reader = ReadlineAlikeReader(console=console, config=config) return reader def test_import_completions(self): - import importlib - # Make iter_modules() search only the standard library. - # This makes the test more reliable in case there are - # other user packages/scripts on PYTHONPATH which can - # intefere with the completions. - lib_path = os.path.dirname(importlib.__path__[0]) - sys.path = [lib_path] - cases = ( ("import path\t\n", "import pathlib"), ("import importlib.\t\tres\t\n", "import importlib.resources"), @@ -954,10 +979,17 @@ def test_import_completions(self): output = reader.readline() self.assertEqual(output, expected) - def test_relative_import_completions(self): + @patch("pkgutil.iter_modules", lambda: [ModuleInfo(None, "public", True), + ModuleInfo(None, "_private", True)]) + @patch("sys.builtin_module_names", ()) + def test_private_completions(self): cases = ( - ("from .readl\t\n", "from .readline"), - ("from . import readl\t\n", "from . import readline"), + # Return public methods by default + ("import \t\n", "import public"), + ("from \t\n", "from public"), + # Return private methods if explicitly specified + ("import _\t\n", "import _private"), + ("from _\t\n", "from _private"), ) for code, expected in cases: with self.subTest(code=code): @@ -966,8 +998,55 @@ def test_relative_import_completions(self): output = reader.readline() self.assertEqual(output, expected) - @patch("pkgutil.iter_modules", lambda: [(None, 'valid_name', None), - (None, 'invalid-name', None)]) + @patch( + "_pyrepl._module_completer.ModuleCompleter.iter_submodules", + lambda *_: [ + ModuleInfo(None, "public", True), + ModuleInfo(None, "_private", True), + ], + ) + def test_sub_module_private_completions(self): + cases = ( + # Return public methods by default + ("from foo import \t\n", "from foo import public"), + # Return private methods if explicitly specified + ("from foo import _\t\n", "from foo import _private"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_builtin_completion_top_level(self): + cases = ( + ("import bui\t\n", "import builtins"), + ("from bui\t\n", "from builtins"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_relative_import_completions(self): + cases = ( + (None, "from .readl\t\n", "from .readl"), + (None, "from . import readl\t\n", "from . import readl"), + ("_pyrepl", "from .readl\t\n", "from .readline"), + ("_pyrepl", "from . import readl\t\n", "from . import readline"), + ) + for package, code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={"__package__": package}) + output = reader.readline() + self.assertEqual(output, expected) + + @patch("pkgutil.iter_modules", lambda: [ModuleInfo(None, "valid_name", True), + ModuleInfo(None, "invalid-name", True)]) def test_invalid_identifiers(self): # Make sure modules which are not valid identifiers # are not suggested as those cannot be imported via 'import'. @@ -983,6 +1062,45 @@ def test_invalid_identifiers(self): output = reader.readline() self.assertEqual(output, expected) + def test_no_fallback_on_regular_completion(self): + cases = ( + ("import pri\t\n", "import pri"), + ("from pri\t\n", "from pri"), + ("from typing import Na\t\n", "from typing import Na"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_hardcoded_stdlib_submodules(self): + cases = ( + ("import collections.\t\n", "import collections.abc"), + ("from os import \t\n", "from os import path"), + ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"), + ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_hardcoded_stdlib_submodules_not_proposed_if_local_import(self): + with tempfile.TemporaryDirectory() as _dir: + dir = pathlib.Path(_dir) + (dir / "collections").mkdir() + (dir / "collections" / "__init__.py").touch() + (dir / "collections" / "foo.py").touch() + with patch.object(sys, "path", [dir, *sys.path]): + events = code_to_events("import collections.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import collections.foo") + def test_get_path_and_prefix(self): cases = ( ('', ('', '')), @@ -1050,11 +1168,15 @@ def test_parse(self): self.assertEqual(actual, parsed) # The parser should not get tripped up by any # other preceding statements - code = f'import xyz\n{code}' - with self.subTest(code=code): + _code = f'import xyz\n{code}' + parser = ImportParser(_code) + actual = parser.parse() + with self.subTest(code=_code): self.assertEqual(actual, parsed) - code = f'import xyz;{code}' - with self.subTest(code=code): + _code = f'import xyz;{code}' + parser = ImportParser(_code) + actual = parser.parse() + with self.subTest(code=_code): self.assertEqual(actual, parsed) def test_parse_error(self): @@ -1107,6 +1229,19 @@ def test_parse_error(self): with self.subTest(code=code): self.assertEqual(actual, None) + +class TestHardcodedSubmodules(TestCase): + def test_hardcoded_stdlib_submodules_are_importable(self): + for parent_path, submodules in HARDCODED_SUBMODULES.items(): + for module_name in submodules: + path = f"{parent_path}.{module_name}" + with self.subTest(path=path): + # We can't use importlib.util.find_spec here, + # since some hardcoded submodules parents are + # not proper packages + importlib.import_module(path) + + class TestPasteEvent(TestCase): def prepare_reader(self, events): console = FakeConsole(events) @@ -1271,6 +1406,9 @@ class TestDumbTerminal(ReplTestCase): def test_dumb_terminal_exits_cleanly(self): env = os.environ.copy() env.pop('PYTHON_BASIC_REPL', None) + # Ignore PYTHONSTARTUP to not pollute the output + # with an unrelated traceback. See GH-137568. + env.pop('PYTHONSTARTUP', None) env.update({"TERM": "dumb"}) output, exit_code = self.run_repl("exit()\n", env=env) self.assertEqual(exit_code, 0) @@ -1286,6 +1424,7 @@ def setUp(self): # Cleanup from PYTHON* variables to isolate from local # user settings, see #121359. Such variables should be # added later in test methods to patched os.environ. + super().setUp() patcher = patch('os.environ', new=make_clean_env()) self.addCleanup(patcher.stop) patcher.start() @@ -1327,7 +1466,7 @@ def _assertMatchOK( ) @force_not_colorized - def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False): + def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False, pythonstartup=False): clean_env = make_clean_env() clean_env["NO_COLOR"] = "1" # force_not_colorized doesn't touch subprocesses @@ -1336,9 +1475,13 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False blue.mkdir() mod = blue / "calx.py" mod.write_text("FOO = 42", encoding="utf-8") + startup = blue / "startup.py" + startup.write_text("BAR = 64", encoding="utf-8") commands = [ "print(f'^{" + var + "=}')" for var in expectations ] + ["exit()"] + if pythonstartup: + clean_env["PYTHONSTARTUP"] = str(startup) if as_file and as_module: self.fail("as_file and as_module are mutually exclusive") elif as_file: @@ -1357,7 +1500,13 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False skip=True, ) else: - self.fail("Choose one of as_file or as_module") + output, exit_code = self.run_repl( + commands, + cmdline_args=[], + env=clean_env, + cwd=td, + skip=True, + ) self.assertEqual(exit_code, 0) for var, expected in expectations.items(): @@ -1370,6 +1519,23 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) + def test_globals_initialized_as_default(self): + expectations = { + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations) + + def test_globals_initialized_from_pythonstartup(self): + expectations = { + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, pythonstartup=True) + def test_inspect_keeps_globals_from_inspected_file(self): expectations = { "FOO": "42", @@ -1379,6 +1545,16 @@ def test_inspect_keeps_globals_from_inspected_file(self): } self._run_repl_globals_test(expectations, as_file=True) + def test_inspect_keeps_globals_from_inspected_file_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True, pythonstartup=True) + def test_inspect_keeps_globals_from_inspected_module(self): expectations = { "FOO": "42", @@ -1388,26 +1564,32 @@ def test_inspect_keeps_globals_from_inspected_module(self): } self._run_repl_globals_test(expectations, as_module=True) + def test_inspect_keeps_globals_from_inspected_module_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True, pythonstartup=True) + @force_not_colorized def test_python_basic_repl(self): env = os.environ.copy() - commands = ("from test.support import initialized_with_pyrepl\n" - "initialized_with_pyrepl()\n" - "exit()\n") - + pyrepl_commands = "clear\nexit()\n" env.pop("PYTHON_BASIC_REPL", None) - output, exit_code = self.run_repl(commands, env=env, skip=True) + output, exit_code = self.run_repl(pyrepl_commands, env=env, skip=True) self.assertEqual(exit_code, 0) - self.assertIn("True", output) - self.assertNotIn("False", output) self.assertNotIn("Exception", output) + self.assertNotIn("NameError", output) self.assertNotIn("Traceback", output) + basic_commands = "help\nexit()\n" env["PYTHON_BASIC_REPL"] = "1" - output, exit_code = self.run_repl(commands, env=env) + output, exit_code = self.run_repl(basic_commands, env=env) self.assertEqual(exit_code, 0) - self.assertIn("False", output) - self.assertNotIn("True", output) + self.assertIn("Type help() for interactive help", output) self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) @@ -1544,6 +1726,17 @@ def test_null_byte(self): self.assertEqual(exit_code, 0) self.assertNotIn("TypeError", output) + @force_not_colorized + def test_non_string_suggestion_candidates(self): + commands = ("import runpy\n" + "runpy._run_module_code('blech', {0: '', 'bluch': ''}, '')\n" + "exit()\n") + + output, exit_code = self.run_repl(commands) + self.assertEqual(exit_code, 0) + self.assertNotIn("all elements in 'candidates' must be strings", output) + self.assertIn("bluch", output) + def test_readline_history_file(self): # skip, if readline module is not available readline = import_module('readline') @@ -1574,18 +1767,24 @@ def test_history_survive_crash(self): commands = "1\n2\n3\nexit()\n" output, exit_code = self.run_repl(commands, env=env, skip=True) + self.assertEqual(exit_code, 0) - commands = "spam\nimport time\ntime.sleep(1000)\nquit\n" - try: - self.run_repl(commands, env=env, timeout=3) - except AssertionError: - pass + # Run until "0xcafe" is printed (as "51966") and then kill the + # process to simulate a crash. Note that the output also includes + # the echoed input commands. + commands = "spam\nimport time\n0xcafe\ntime.sleep(1000)\nquit\n" + output, exit_code = self.run_repl(commands, env=env, + exit_on_output="51966") + self.assertNotEqual(exit_code, 0) history = pathlib.Path(hfile.name).read_text() self.assertIn("2", history) self.assertIn("exit()", history) self.assertIn("spam", history) self.assertIn("import time", history) + # History is written after each command's output is printed to the + # console, so depending on how quickly the process is killed, + # the last command may or may not be written to the history file. self.assertNotIn("sleep", history) self.assertNotIn("quit", history) @@ -1605,3 +1804,110 @@ def test_prompt_after_help(self): # Extra stuff (newline and `exit` rewrites) are necessary # because of how run_repl works. self.assertNotIn(">>> \n>>> >>>", cleaned_output) + + @skipUnless(Py_DEBUG, '-X showrefcount requires a Python debug build') + def test_showrefcount(self): + env = os.environ.copy() + env.pop("PYTHON_BASIC_REPL", "") + output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) + matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) + self.assertEqual(len(matches), 3) + + env["PYTHON_BASIC_REPL"] = "1" + output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) + matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) + self.assertEqual(len(matches), 3) + + def test_detect_pip_usage_in_repl(self): + for pip_cmd in ("pip", "pip3", "python -m pip", "python3 -m pip"): + with self.subTest(pip_cmd=pip_cmd): + output, exit_code = self.run_repl([f"{pip_cmd} install sampleproject", "exit"]) + self.assertIn("SyntaxError", output) + hint = ( + "The Python package manager (pip) can only be used" + " outside of the Python REPL" + ) + self.assertIn(hint, output) + +class TestPyReplCtrlD(TestCase): + """Test Ctrl+D behavior in _pyrepl to match old pre-3.13 REPL behavior. + + Ctrl+D should: + - Exit on empty buffer (raises EOFError) + - Delete character when cursor is in middle of line + - Perform no operation when cursor is at end of line without newline + - Exit multiline mode when cursor is at end with trailing newline + - Run code up to that point when pressed on blank line with preceding lines + """ + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_ctrl_d_empty_line(self): + """Test that pressing Ctrl+D on empty line exits the program""" + events = [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")), # Ctrl+D + ] + reader = self.prepare_reader(events) + with self.assertRaises(EOFError): + multiline_input(reader) + + def test_ctrl_d_multiline_with_new_line(self): + """Test that pressing Ctrl+D in multiline mode with trailing newline exits multiline mode""" + events = itertools.chain( + code_to_events("def f():\n pass\n"), # Enter multiline mode with trailing newline + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")), # Ctrl+D + ], + ) + reader, _ = handle_all_events(events) + self.assertTrue(reader.finished) + self.assertEqual("def f():\n pass\n", "".join(reader.buffer)) + + def test_ctrl_d_multiline_middle_of_line(self): + """Test that pressing Ctrl+D in multiline mode with cursor in middle deletes character""" + events = itertools.chain( + code_to_events("def f():\n hello world"), # Enter multiline mode + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")) + ] * 5, # move cursor to 'w' in "world" + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")) + ], # Ctrl+D should delete 'w' + ) + reader, _ = handle_all_events(events) + self.assertFalse(reader.finished) + self.assertEqual("def f():\n hello orld", "".join(reader.buffer)) + + def test_ctrl_d_multiline_end_of_line_no_newline(self): + """Test that pressing Ctrl+D at end of line without newline performs no operation""" + events = itertools.chain( + code_to_events("def f():\n hello"), # Enter multiline mode, no trailing newline + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")) + ], # Ctrl+D should be no-op + ) + reader, _ = handle_all_events(events) + self.assertFalse(reader.finished) + self.assertEqual("def f():\n hello", "".join(reader.buffer)) + + def test_ctrl_d_single_line_middle_of_line(self): + """Test that pressing Ctrl+D in single line mode deletes current character""" + events = itertools.chain( + code_to_events("hello"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], # move left + [Event(evt="key", data="\x04", raw=bytearray(b"\x04"))], # Ctrl+D + ) + reader, _ = handle_all_events(events) + self.assertEqual("hell", "".join(reader.buffer)) + + def test_ctrl_d_single_line_end_no_newline(self): + """Test that pressing Ctrl+D at end of single line without newline does nothing""" + events = itertools.chain( + code_to_events("hello"), # cursor at end of line + [Event(evt="key", data="\x04", raw=bytearray(b"\x04"))], # Ctrl+D + ) + reader, _ = handle_all_events(events) + self.assertEqual("hello", "".join(reader.buffer)) diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 4ee320a5a4d..b1b6ae16a1e 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -375,8 +375,10 @@ def funct(case: str = sys.platform) -> None: ) match case: case "emscripten": print("on the web") - case "ios" | "android": print("on the phone") + case "ios" | "android": + print("on the phone") case _: print('arms around', match.group(1)) + type type = type[type] """ ) expected = dedent( @@ -393,8 +395,10 @@ def funct(case: str = sys.platform) -> None: {o}){z} {K}match{z} case{o}:{z} {K}case{z} {s}"emscripten"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the web"{z}{o}){z} - {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} + {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} + {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} {K}case{z} {K}_{z}{o}:{z} {b}print{z}{o}({z}{s}'arms around'{z}{o},{z} match{o}.{z}group{o}({z}{n}1{z}{o}){z}{o}){z} + {K}type{z} {b}type{z} {o}={z} {b}type{z}{o}[{z}{b}type{z}{o}]{z} """ ) expected_sync = expected.format(a="", **colors) @@ -402,14 +406,14 @@ def funct(case: str = sys.platform) -> None: reader, _ = handle_all_events(events) self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected_sync) - self.assertEqual(reader.pos, 2**7 + 2**8) - self.assertEqual(reader.cxy, (0, 14)) + self.assertEqual(reader.pos, 419) + self.assertEqual(reader.cxy, (0, 16)) async_msg = "{k}async{z} ".format(**colors) expected_async = expected.format(a=async_msg, **colors) more_events = itertools.chain( code_to_events(code), - [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 13, + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 15, code_to_events("async "), ) reader, _ = handle_all_events(more_events) @@ -497,6 +501,57 @@ def unfinished_function(): self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected) + def test_syntax_highlighting_indentation_error(self): + code = dedent( + """\ + def unfinished_function(): + var = 1 + oops + """ + ) + expected = dedent( + """\ + {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z} + var {o}={z} {n}1{z} + oops + """ + ).format(**colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected) + + def test_syntax_highlighting_literal_brace_in_fstring_or_tstring(self): + code = dedent( + """\ + f"{{" + f"}}" + f"a{{b" + f"a}}b" + f"a{{b}}c" + t"a{{b}}c" + f"{{{0}}}" + f"{ {0} }" + """ + ) + expected = dedent( + """\ + {s}f"{z}{s}<<{z}{s}"{z} + {s}f"{z}{s}>>{z}{s}"{z} + {s}f"{z}{s}a<<{z}{s}b{z}{s}"{z} + {s}f"{z}{s}a>>{z}{s}b{z}{s}"{z} + {s}f"{z}{s}a<<{z}{s}b>>{z}{s}c{z}{s}"{z} + {s}t"{z}{s}a<<{z}{s}b>>{z}{s}c{z}{s}"{z} + {s}f"{z}{s}<<{z}{o}<{z}{n}0{z}{o}>{z}{s}>>{z}{s}"{z} + {s}f"{z}{o}<{z} {o}<{z}{n}0{z}{o}>{z} {o}>{z}{s}"{z} + """ + ).format(**colors).replace("<", "{").replace(">", "}") + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.maxDiff=None + self.assert_screen_equal(reader, expected) + def test_control_characters(self): code = 'flag = "🏳️‍🌈"' events = code_to_events(code) diff --git a/Lib/test/test_pyrepl/test_terminfo.py b/Lib/test/test_pyrepl/test_terminfo.py new file mode 100644 index 00000000000..562cf5c905b --- /dev/null +++ b/Lib/test/test_pyrepl/test_terminfo.py @@ -0,0 +1,651 @@ +"""Tests comparing PyREPL's pure Python curses implementation with the standard curses module.""" + +import json +import os +import subprocess +import sys +import unittest +from test.support import requires, has_subprocess_support +from textwrap import dedent + +# Only run these tests if curses is available +requires("curses") + +try: + import _curses +except ImportError: + try: + import curses as _curses + except ImportError: + _curses = None + +from _pyrepl import terminfo + + +ABSENT_STRING = terminfo.ABSENT_STRING +CANCELLED_STRING = terminfo.CANCELLED_STRING + + +class TestCursesCompatibility(unittest.TestCase): + """Test that PyREPL's curses implementation matches the standard curses behavior. + + Python's `curses` doesn't allow calling `setupterm()` again with a different + $TERM in the same process, so we subprocess all `curses` tests to get correctly + set up terminfo.""" + + @classmethod + def setUpClass(cls): + if _curses is None: + raise unittest.SkipTest( + "`curses` capability provided to regrtest but `_curses` not importable" + ) + + if not has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + # we need to ensure there's a terminfo database on the system and that + # `infocmp` works + cls.infocmp("dumb") + + def setUp(self): + self.original_term = os.environ.get("TERM", None) + + def tearDown(self): + if self.original_term is not None: + os.environ["TERM"] = self.original_term + elif "TERM" in os.environ: + del os.environ["TERM"] + + @classmethod + def infocmp(cls, term) -> list[str]: + all_caps = [] + try: + result = subprocess.run( + ["infocmp", "-l1", term], + capture_output=True, + text=True, + check=True, + ) + except Exception: + raise unittest.SkipTest("calling `infocmp` failed on the system") + + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("#"): + if "terminfo" not in line and "termcap" in line: + # PyREPL terminfo doesn't parse termcap databases + raise unittest.SkipTest( + "curses using termcap.db: no terminfo database on" + " the system" + ) + elif "=" in line: + cap_name = line.split("=")[0] + all_caps.append(cap_name) + + return all_caps + + def test_setupterm_basic(self): + """Test basic setupterm functionality.""" + # Test with explicit terminal type + test_terms = ["xterm", "xterm-256color", "vt100", "ansi"] + + for term in test_terms: + with self.subTest(term=term): + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(term)}, 1) + print(json.dumps({{"success": True}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + # Set up with PyREPL curses + try: + terminfo.TermInfo(term, fallback=False) + pyrepl_success = True + except Exception as e: + pyrepl_success = False + pyrepl_error = e + + # Both should succeed or both should fail + if std_success: + self.assertTrue( + pyrepl_success, + f"Standard curses succeeded but PyREPL failed for {term}", + ) + else: + # If standard curses failed, PyREPL might still succeed with fallback + # This is acceptable as PyREPL has hardcoded fallbacks + pass + + def test_setupterm_none(self): + """Test setupterm with None (uses TERM from environment).""" + # Test with current TERM + ncurses_code = dedent( + """ + import _curses + import json + try: + _curses.setupterm(None, 1) + print(json.dumps({"success": True})) + except Exception as e: + print(json.dumps({"success": False, "error": str(e)})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + try: + terminfo.TermInfo(None, fallback=False) + pyrepl_success = True + except Exception: + pyrepl_success = False + + # Both should have same result + if std_success: + self.assertTrue( + pyrepl_success, + "Standard curses succeeded but PyREPL failed for None", + ) + + def test_tigetstr_common_capabilities(self): + """Test tigetstr for common terminal capabilities.""" + # Test with a known terminal type + term = "xterm" + + # Get ALL capabilities from infocmp + all_caps = self.infocmp(term) + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + + ncurses_data = json.loads(result.stdout) + + ti = terminfo.TermInfo(term, fallback=False) + + # Test every single capability + for cap in all_caps: + if cap not in ncurses_data or ncurses_data[cap] == "error": + continue + + with self.subTest(capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap}: ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_tigetstr_input_types(self): + """Test tigetstr with different input types.""" + term = "xterm" + cap = "cup" + + # Test standard curses behavior with string in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with string input + try: + std_str_result = _curses.tigetstr({repr(cap)}) + std_accepts_str = True + if std_str_result is None: + std_str_val = None + elif std_str_result == -1: + std_str_val = -1 + else: + std_str_val = list(std_str_result) + except TypeError: + std_accepts_str = False + std_str_val = None + + print(json.dumps({{ + "accepts_str": std_accepts_str, + "str_result": std_str_val + }})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # PyREPL behavior with string + try: + pyrepl_str_result = ti.get(cap) + pyrepl_accepts_str = True + except TypeError: + pyrepl_accepts_str = False + + # PyREPL should also only accept strings for compatibility + with self.assertRaises(TypeError): + ti.get(cap.encode("ascii")) + + # Both should accept string input + self.assertEqual( + pyrepl_accepts_str, + ncurses_data["accepts_str"], + "PyREPL and standard curses should have same string handling", + ) + self.assertTrue( + pyrepl_accepts_str, "PyREPL should accept string input" + ) + + def test_tparm_basic(self): + """Test basic tparm functionality.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test cursor positioning (cup) + cup = ti.get("cup") + if cup and cup not in {ABSENT_STRING, CANCELLED_STRING}: + # Test various parameter combinations + test_cases = [ + (0, 0), # Top-left + (5, 10), # Arbitrary position + (23, 79), # Bottom-right of standard terminal + (999, 999), # Large values + ] + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Get cup capability + cup = _curses.tigetstr('cup') + results = {{}} + + for row, col in {repr(test_cases)}: + try: + result = _curses.tparm(cup, row, col) + results[f"{{row}},{{col}}"] = list(result) + except Exception as e: + results[f"{{row}},{{col}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for row, col in test_cases: + with self.subTest(row=row, col=col): + # Standard curses tparm from subprocess + key = f"{row},{col}" + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + # PyREPL curses tparm + pyrepl_result = terminfo.tparm(cup, row, col) + + # Results should be identical + self.assertEqual( + pyrepl_result, + std_result, + f"tparm(cup, {row}, {col}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + else: + raise unittest.SkipTest( + "test_tparm_basic() requires the `cup` capability" + ) + + def test_tparm_multiple_params(self): + """Test tparm with capabilities using multiple parameters.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test capabilities that take parameters + param_caps = { + "cub": 1, # cursor_left with count + "cuf": 1, # cursor_right with count + "cuu": 1, # cursor_up with count + "cud": 1, # cursor_down with count + "dch": 1, # delete_character with count + "ich": 1, # insert_character with count + } + + # Get all capabilities from PyREPL first + pyrepl_caps = {} + for cap in param_caps: + cap_value = ti.get(cap) + if cap_value and cap_value not in { + ABSENT_STRING, + CANCELLED_STRING, + }: + pyrepl_caps[cap] = cap_value + + if not pyrepl_caps: + self.skipTest("No parametrized capabilities found") + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + param_caps = {repr(param_caps)} + test_values = [1, 5, 10, 99] + results = {{}} + + for cap in param_caps: + cap_value = _curses.tigetstr(cap) + if cap_value and cap_value != -1: + for value in test_values: + try: + result = _curses.tparm(cap_value, value) + results[f"{{cap}},{{value}}"] = list(result) + except Exception as e: + results[f"{{cap}},{{value}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for cap, cap_value in pyrepl_caps.items(): + with self.subTest(capability=cap): + # Test with different parameter values + for value in [1, 5, 10, 99]: + key = f"{cap},{value}" + if key in ncurses_data: + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + pyrepl_result = terminfo.tparm(cap_value, value) + self.assertEqual( + pyrepl_result, + std_result, + f"tparm({cap}, {value}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + + def test_tparm_null_handling(self): + """Test tparm with None/null input.""" + term = "xterm" + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with None + try: + _curses.tparm(None) + raises_typeerror = False + except TypeError: + raises_typeerror = True + except Exception as e: + raises_typeerror = False + error_type = type(e).__name__ + + print(json.dumps({{"raises_typeerror": raises_typeerror}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # Test with None - both should raise TypeError + if ncurses_data["raises_typeerror"]: + with self.assertRaises(TypeError): + terminfo.tparm(None) + else: + # If ncurses doesn't raise TypeError, PyREPL shouldn't either + try: + terminfo.tparm(None) + except TypeError: + self.fail("PyREPL raised TypeError but ncurses did not") + + def test_special_terminals(self): + """Test with special terminal types.""" + special_terms = [ + "dumb", # Minimal terminal + "unknown", # Should fall back to defaults + "linux", # Linux console + "screen", # GNU Screen + "tmux", # tmux + ] + + # Get all string capabilities from ncurses + for term in special_terms: + with self.subTest(term=term): + all_caps = self.infocmp(term) + ncurses_code = dedent( + f""" + import _curses + import json + import sys + + try: + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + # Convert bytes to list of ints for JSON + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + except Exception as e: + print(json.dumps({{"error": str(e)}})) + """ + ) + + # Get ncurses results + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.fail( + f"Failed to get ncurses data for {term}: {result.stderr}" + ) + + try: + ncurses_data = json.loads(result.stdout) + except json.JSONDecodeError: + self.fail( + f"Failed to parse ncurses output for {term}: {result.stdout}" + ) + + if "error" in ncurses_data and len(ncurses_data) == 1: + # ncurses failed to setup this terminal + # PyREPL should still work with fallback + ti = terminfo.TermInfo(term, fallback=True) + continue + + ti = terminfo.TermInfo(term, fallback=False) + + # Compare all capabilities + for cap in all_caps: + if cap not in ncurses_data: + continue + + with self.subTest(term=term, capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + # Convert back to bytes + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + # Both should return the same value + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap} for {term}: " + f"ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_terminfo_fallback(self): + """Test that PyREPL falls back gracefully when terminfo is not found.""" + # Use a non-existent terminal type + fake_term = "nonexistent-terminal-type-12345" + + # Check if standard curses can setup this terminal in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(fake_term)}, 1) + print(json.dumps({{"success": True}})) + except _curses.error: + print(json.dumps({{"success": False, "error": "curses.error"}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + if ncurses_data["success"]: + # If it succeeded, skip this test as we can't test fallback + self.skipTest( + f"System unexpectedly has terminfo for '{fake_term}'" + ) + + # PyREPL should succeed with fallback + try: + ti = terminfo.TermInfo(fake_term, fallback=True) + pyrepl_ok = True + except Exception: + pyrepl_ok = False + + self.assertTrue( + pyrepl_ok, "PyREPL should fall back for unknown terminals" + ) + + # Should still be able to get basic capabilities + bel = ti.get("bel") + self.assertIsNotNone( + bel, "PyREPL should provide basic capabilities after fallback" + ) + + def test_invalid_terminal_names(self): + cases = [ + (42, TypeError), + ("", ValueError), + ("w\x00t", ValueError), + (f"..{os.sep}name", ValueError), + ] + + for term, exc in cases: + with self.subTest(term=term): + with self.assertRaises(exc): + terminfo._validate_terminal_name_or_raise(term) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index c447b310c49..f4fb9237ffd 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -1,12 +1,18 @@ +import errno import itertools import os +import signal +import subprocess import sys +import threading import unittest from functools import partial +from test import support from test.support import os_helper, force_not_colorized_test_class +from test.support import script_helper, threading_helper from unittest import TestCase -from unittest.mock import MagicMock, call, patch, ANY +from unittest.mock import MagicMock, call, patch, ANY, Mock from .support import handle_all_events, code_to_events @@ -16,10 +22,15 @@ except ImportError: pass +from _pyrepl.terminfo import _TERMINAL_CAPABILITIES + +TERM_CAPABILITIES = _TERMINAL_CAPABILITIES["ansi"] + def unix_console(events, **kwargs): - console = UnixConsole() + console = UnixConsole(term="xterm") console.get_event = MagicMock(side_effect=events) + console.getpending = MagicMock(return_value=Event("key", "")) height = kwargs.get("height", 25) width = kwargs.get("width", 80) @@ -49,41 +60,11 @@ def unix_console(events, **kwargs): ) -TERM_CAPABILITIES = { - "bel": b"\x07", - "civis": b"\x1b[?25l", - "clear": b"\x1b[H\x1b[2J", - "cnorm": b"\x1b[?12l\x1b[?25h", - "cub": b"\x1b[%p1%dD", - "cub1": b"\x08", - "cud": b"\x1b[%p1%dB", - "cud1": b"\n", - "cuf": b"\x1b[%p1%dC", - "cuf1": b"\x1b[C", - "cup": b"\x1b[%i%p1%d;%p2%dH", - "cuu": b"\x1b[%p1%dA", - "cuu1": b"\x1b[A", - "dch1": b"\x1b[P", - "dch": b"\x1b[%p1%dP", - "el": b"\x1b[K", - "hpa": b"\x1b[%i%p1%dG", - "ich": b"\x1b[%p1%d@", - "ich1": None, - "ind": b"\n", - "pad": None, - "ri": b"\x1bM", - "rmkx": b"\x1b[?1l\x1b>", - "smkx": b"\x1b[?1h\x1b=", -} - - @unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") -@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) @patch( - "_pyrepl.curses.tparm", + "_pyrepl.terminfo.tparm", lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), ) -@patch("_pyrepl.curses.setupterm", lambda a, b: None) @patch( "termios.tcgetattr", lambda _: [ @@ -320,7 +301,7 @@ def same_console(events): def test_getheightwidth_with_invalid_environ(self, _os_write): # gh-128636 - console = UnixConsole() + console = UnixConsole(term="xterm") with os_helper.EnvironmentVarGuard() as env: env["LINES"] = "" self.assertIsInstance(console.getheightwidth(), tuple) @@ -328,3 +309,80 @@ def test_getheightwidth_with_invalid_environ(self, _os_write): self.assertIsInstance(console.getheightwidth(), tuple) os.environ = [] self.assertIsInstance(console.getheightwidth(), tuple) + + @unittest.skipUnless(sys.platform == "darwin", "requires macOS") + def test_restore_with_invalid_environ_on_macos(self, _os_write): + # gh-128636 for macOS + console = UnixConsole(term="xterm") + with os_helper.EnvironmentVarGuard(): + os.environ = [] + console.prepare() # needed to call restore() + console.restore() # this should succeed + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_restore_in_thread(self, _os_write): + # gh-139391: ensure that console.restore() silently suppresses + # exceptions when calling signal.signal() from a non-main thread. + console = unix_console([]) + console.old_sigwinch = signal.SIG_DFL + thread = threading.Thread(target=console.restore) + thread.start() + thread.join() # this should not raise + + +@unittest.skipIf(sys.platform == "win32", "No Unix console on Windows") +class TestUnixConsoleEIOHandling(TestCase): + + @patch('_pyrepl.unix_console.tcsetattr') + @patch('_pyrepl.unix_console.tcgetattr') + def test_eio_error_handling_in_restore(self, mock_tcgetattr, mock_tcsetattr): + + import termios + mock_termios = Mock() + mock_termios.iflag = 0 + mock_termios.oflag = 0 + mock_termios.cflag = 0 + mock_termios.lflag = 0 + mock_termios.cc = [0] * 32 + mock_termios.copy.return_value = mock_termios + mock_tcgetattr.return_value = mock_termios + + console = UnixConsole(term="xterm") + console.prepare() + + mock_tcsetattr.side_effect = termios.error(errno.EIO, "Input/output error") + + # EIO error should be handled gracefully in restore() + console.restore() + + @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") + def test_repl_eio(self): + # Use the pty-based approach to simulate EIO error + script_path = os.path.join(os.path.dirname(__file__), "eio_test_script.py") + + proc = script_helper.spawn_python( + "-S", script_path, + stderr=subprocess.PIPE, + text=True + ) + + ready_line = proc.stdout.readline().strip() + if ready_line != "READY" or proc.poll() is not None: + self.fail("Child process failed to start properly") + + os.kill(proc.pid, signal.SIGUSR1) + # sleep for pty to settle + _, err = proc.communicate(timeout=support.LONG_TIMEOUT) + self.assertEqual( + proc.returncode, + 1, + f"Expected EIO/ENXIO error, got return code {proc.returncode}", + ) + self.assertTrue( + ( + "Got EIO:" in err + or "Got ENXIO:" in err + ), + f"Expected EIO/ENXIO error message in stderr: {err}", + ) diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 8ce1e537138..656a1e441e0 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -1,14 +1,33 @@ from unittest import TestCase -from _pyrepl.utils import str_width, wlen, prev_next_window +from _pyrepl.utils import str_width, wlen, prev_next_window, gen_colors class TestUtils(TestCase): def test_str_width(self): - characters = ['a', '1', '_', '!', '\x1a', '\u263A', '\uffb9'] + characters = [ + 'a', + '1', + '_', + '!', + '\x1a', + '\u263A', + '\uffb9', + '\N{LATIN SMALL LETTER E WITH ACUTE}', # é + '\N{LATIN SMALL LETTER E WITH CEDILLA}', # ȩ + '\u00ad', + ] for c in characters: self.assertEqual(str_width(c), 1) + zero_width_characters = [ + '\N{COMBINING ACUTE ACCENT}', + '\N{ZERO WIDTH JOINER}', + ] + for c in zero_width_characters: + with self.subTest(character=c): + self.assertEqual(str_width(c), 0) + characters = [chr(99989), chr(99999)] for c in characters: self.assertEqual(str_width(c), 2) @@ -25,6 +44,8 @@ def test_wlen(self): self.assertEqual(wlen('hello'), 5) self.assertEqual(wlen('hello' + '\x1a'), 7) + self.assertEqual(wlen('e\N{COMBINING ACUTE ACCENT}'), 1) + self.assertEqual(wlen('a\N{ZERO WIDTH JOINER}b'), 2) def test_prev_next_window(self): def gen_normal(): @@ -60,3 +81,25 @@ def gen_raise(): self.assertEqual(next(pnw), (3, 4, None)) with self.assertRaises(ZeroDivisionError): next(pnw) + + def test_gen_colors_keyword_highlighting(self): + cases = [ + # no highlights + ("a.set", [(".", "op")]), + ("obj.list", [(".", "op")]), + ("obj.match", [(".", "op")]), + ("b. \\\n format", [(".", "op")]), + # highlights + ("set", [("set", "builtin")]), + ("list", [("list", "builtin")]), + (" \n dict", [("dict", "builtin")]), + ] + for code, expected_highlights in cases: + with self.subTest(code=code): + colors = list(gen_colors(code)) + # Extract (text, tag) pairs for comparison + actual_highlights = [] + for color in colors: + span_text = code[color.span.start:color.span.end + 1] + actual_highlights.append((span_text, color.tag)) + self.assertEqual(actual_highlights, expected_highlights) diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py index e7bab226b31..f9607e02c60 100644 --- a/Lib/test/test_pyrepl/test_windows_console.py +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -35,6 +35,7 @@ class WindowsConsoleTests(TestCase): def console(self, events, **kwargs) -> Console: console = WindowsConsole() console.get_event = MagicMock(side_effect=events) + console.getpending = MagicMock(return_value=Event("key", "")) console.wait = MagicMock() console._scroll = MagicMock() console._hide_cursor = MagicMock() @@ -385,6 +386,7 @@ def get_event(self, input_records, **kwargs) -> Console: self.console._read_input = self.mock self.console._WindowsConsole__vt_support = kwargs.get("vt_support", False) + self.console.wait = MagicMock(return_value=True) event = self.console.get_event(block=False) return event diff --git a/Lib/test/test_pystats.py b/Lib/test/test_pystats.py new file mode 100644 index 00000000000..c50cecfcfdd --- /dev/null +++ b/Lib/test/test_pystats.py @@ -0,0 +1,215 @@ +import sys +import textwrap +import unittest +from test.support import script_helper + +# This function is available for the --enable-pystats config. +HAVE_PYSTATS = hasattr(sys, '_stats_on') + +TEST_TEMPLATE = """ + import sys + import threading + import time + + THREADS = 2 + + class A: + pass + + class B: + pass + + def modify_class(): + # This is used as a rare event we can assume doesn't happen unless we do it. + # It increments the "Rare event (set_class)" count. + a = A() + a.__class__ = B + + TURNED_ON = False + def stats_on(): + global TURNED_ON + sys._stats_on() + TURNED_ON = True + + TURNED_OFF = False + def stats_off(): + global TURNED_OFF + sys._stats_off() + TURNED_OFF = True + + CLEARED = False + def stats_clear(): + global CLEARED + sys._stats_clear() + CLEARED = True + + def func_start(): + pass + + def func_end(): + pass + + def func_test(thread_id): + pass + + _TEST_CODE_ + + func_start() + threads = [] + for i in range(THREADS): + t = threading.Thread(target=func_test, args=(i,)) + threads.append(t) + t.start() + for t in threads: + t.join() + func_end() + """ + + +def run_test_code( + test_code, + args=[], + env_vars=None, +): + """Run test code and return the value of the "set_class" stats counter. + """ + code = textwrap.dedent(TEST_TEMPLATE) + code = code.replace('_TEST_CODE_', textwrap.dedent(test_code)) + script_args = args + ['-c', code] + env_vars = env_vars or {} + res, _ = script_helper.run_python_until_end(*script_args, **env_vars) + stderr = res.err.decode("ascii", "backslashreplace") + for line in stderr.split('\n'): + if 'Rare event (set_class)' in line: + label, _, value = line.partition(':') + return value.strip() + return '' + + +@unittest.skipUnless(HAVE_PYSTATS, "requires pystats build option") +class TestPyStats(unittest.TestCase): + """Tests for pystats functionality (requires --enable-pystats build + option). + """ + + def test_stats_toggle_on(self): + """Check the toggle on functionality. + """ + code = """ + def func_start(): + modify_class() + """ + + # If turned on with command line flag, should get one count. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '1') + + # If turned on with env var, should get one count. + stat_count = run_test_code(code, env_vars={'PYTHONSTATS': '1'}) + self.assertEqual(stat_count, '1') + + # If not turned on, should be no counts. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '') + + code = """ + def func_start(): + modify_class() + sys._stats_on() + modify_class() + """ + # Not initially turned on but enabled by sys._stats_on(), should get + # one count. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '1') + + def test_stats_toggle_on_thread(self): + """Check the toggle on functionality when threads are used. + """ + code = """ + def func_test(thread_id): + if thread_id == 0: + modify_class() + stats_on() + modify_class() + else: + while not TURNED_ON: + pass + modify_class() + """ + # Turning on in one thread will count in other thread. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '2') + + code = """ + def func_test(thread_id): + if thread_id == 0: + modify_class() + stats_off() + modify_class() + else: + while not TURNED_OFF: + pass + modify_class() + """ + # Turning off in one thread will not count in other threads. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '1') + + def test_thread_exit_merge(self): + """Check that per-thread stats (when free-threading enabled) are merged. + """ + code = """ + def func_test(thread_id): + modify_class() + if thread_id == 0: + raise SystemExit + """ + # Stats from a thread exiting early should still be counted. + stat_count = run_test_code(code, args=['-X', 'pystats']) + self.assertEqual(stat_count, '2') + + def test_stats_dump(self): + """Check that sys._stats_dump() works. + """ + code = """ + def func_test(thread_id): + if thread_id == 0: + stats_on() + else: + while not TURNED_ON: + pass + modify_class() + sys._stats_dump() + stats_off() + """ + # Stats from a thread exiting early should still be counted. + stat_count = run_test_code(code) + self.assertEqual(stat_count, '1') + + def test_stats_clear(self): + """Check that sys._stats_clear() works. + """ + code = """ + ready = False + def func_test(thread_id): + global ready + if thread_id == 0: + stats_on() + modify_class() + while not ready: + pass # wait until other thread has called modify_class() + stats_clear() # clears stats for all threads + else: + while not TURNED_ON: + pass + modify_class() + ready = True + """ + # Clearing stats will clear for all threads + stat_count = run_test_code(code) + self.assertEqual(stat_count, '0') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 7f4fe357034..c855fb8fe2b 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -6,7 +6,7 @@ import time import unittest import weakref -from test.support import gc_collect +from test.support import gc_collect, bigmemtest from test.support import import_helper from test.support import threading_helper @@ -963,33 +963,33 @@ def test_order(self): # One producer, one consumer => results appended in well-defined order self.assertEqual(results, inputs) - def test_many_threads(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads(self, size): # Test multiple concurrent put() and get() - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, self.feed, self.consume) + results = self.run_threads(size, q, inputs, self.feed, self.consume) # Multiple consumers without synchronization append the # results in random order self.assertEqual(sorted(results), inputs) - def test_many_threads_nonblock(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_nonblock(self, size): # Test multiple concurrent put() and get(block=False) - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_nonblock) self.assertEqual(sorted(results), inputs) - def test_many_threads_timeout(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_timeout(self, size): # Test multiple concurrent put() and get(timeout=...) - N = 50 q = self.q inputs = list(range(1000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_timeout) self.assertEqual(sorted(results), inputs) diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index dcf0753bc82..645ef291a58 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -186,18 +186,14 @@ def test_class_cause(self): self.fail("No exception raised") def test_class_cause_nonexception_result(self): - class ConstructsNone(BaseException): - @classmethod + # See https://github.com/python/cpython/issues/140530. + class ConstructMortal(BaseException): def __new__(*args, **kwargs): - return None - try: - raise IndexError from ConstructsNone - except TypeError as e: - self.assertIn("should have returned an instance of BaseException", str(e)) - except IndexError: - self.fail("Wrong kind of exception raised") - else: - self.fail("No exception raised") + return ["mortal value"] + + msg = ".*should have returned an instance of BaseException.*" + with self.assertRaisesRegex(TypeError, msg): + raise IndexError from ConstructMortal def test_instance_cause(self): cause = KeyError() diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 43957f525f1..1e57b9244b4 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -14,6 +14,17 @@ from fractions import Fraction from collections import abc, Counter +from test.support import warnings_helper + + +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + class TestBasicOps: # Superclass with tests common to all generators. # Subclasses must arrange for self.gen to retrieve the Random instance @@ -142,6 +153,7 @@ def test_sample(self): # Exception raised if size of sample exceeds that of population self.assertRaises(ValueError, self.gen.sample, population, N+1) self.assertRaises(ValueError, self.gen.sample, [], -1) + self.assertRaises(TypeError, self.gen.sample, population, 1.0) def test_sample_distribution(self): # For the entire allowable range of 0 <= k <= N, validate that @@ -259,6 +271,7 @@ def test_choices(self): choices(data, range(4), k=5), choices(k=5, population=data, weights=range(4)), choices(k=5, population=data, cum_weights=range(4)), + choices(data, k=MyIndex(5)), ]: self.assertEqual(len(sample), 5) self.assertEqual(type(sample), list) @@ -369,118 +382,40 @@ def test_gauss(self): self.assertEqual(x1, x2) self.assertEqual(y1, y2) + @support.requires_IEEE_754 + def test_53_bits_per_float(self): + span = 2 ** 53 + cum = 0 + for i in range(100): + cum |= int(self.gen.random() * span) + self.assertEqual(cum, span-1) + def test_getrandbits(self): + getrandbits = self.gen.getrandbits # Verify ranges for k in range(1, 1000): - self.assertTrue(0 <= self.gen.getrandbits(k) < 2**k) - self.assertEqual(self.gen.getrandbits(0), 0) + self.assertTrue(0 <= getrandbits(k) < 2**k) + self.assertEqual(getrandbits(0), 0) # Verify all bits active - getbits = self.gen.getrandbits for span in [1, 2, 3, 4, 31, 32, 32, 52, 53, 54, 119, 127, 128, 129]: all_bits = 2**span-1 cum = 0 cpl_cum = 0 for i in range(100): - v = getbits(span) + v = getrandbits(span) cum |= v cpl_cum |= all_bits ^ v self.assertEqual(cum, all_bits) self.assertEqual(cpl_cum, all_bits) # Verify argument checking - self.assertRaises(TypeError, self.gen.getrandbits) - self.assertRaises(TypeError, self.gen.getrandbits, 1, 2) - self.assertRaises(ValueError, self.gen.getrandbits, -1) - self.assertRaises(TypeError, self.gen.getrandbits, 10.1) - - def test_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - state = pickle.dumps(self.gen, proto) - origseq = [self.gen.random() for i in range(10)] - newgen = pickle.loads(state) - restoredseq = [newgen.random() for i in range(10)] - self.assertEqual(origseq, restoredseq) - - def test_bug_1727780(self): - # verify that version-2-pickles can be loaded - # fine, whether they are created on 32-bit or 64-bit - # platforms, and that version-3-pickles load fine. - files = [("randv2_32.pck", 780), - ("randv2_64.pck", 866), - ("randv3.pck", 343)] - for file, value in files: - with open(support.findfile(file),"rb") as f: - r = pickle.load(f) - self.assertEqual(int(r.random()*1000), value) - - def test_bug_9025(self): - # Had problem with an uneven distribution in int(n*random()) - # Verify the fix by checking that distributions fall within expectations. - n = 100000 - randrange = self.gen.randrange - k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) - self.assertTrue(0.30 < k/n < .37, (k/n)) - - def test_randbytes(self): - # Verify ranges - for n in range(1, 10): - data = self.gen.randbytes(n) - self.assertEqual(type(data), bytes) - self.assertEqual(len(data), n) - - self.assertEqual(self.gen.randbytes(0), b'') - - # Verify argument checking - self.assertRaises(TypeError, self.gen.randbytes) - self.assertRaises(TypeError, self.gen.randbytes, 1, 2) - self.assertRaises(ValueError, self.gen.randbytes, -1) - self.assertRaises(TypeError, self.gen.randbytes, 1.0) - - def test_mu_sigma_default_args(self): - self.assertIsInstance(self.gen.normalvariate(), float) - self.assertIsInstance(self.gen.gauss(), float) - - -try: - random.SystemRandom().random() -except NotImplementedError: - SystemRandom_available = False -else: - SystemRandom_available = True - -@unittest.skipUnless(SystemRandom_available, "random.SystemRandom not available") -class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): - gen = random.SystemRandom() - - def test_autoseed(self): - # Doesn't need to do anything except not fail - self.gen.seed() - - def test_saverestore(self): - self.assertRaises(NotImplementedError, self.gen.getstate) - self.assertRaises(NotImplementedError, self.gen.setstate, None) - - def test_seedargs(self): - # Doesn't need to do anything except not fail - self.gen.seed(100) - - def test_gauss(self): - self.gen.gauss_next = None - self.gen.seed(100) - self.assertEqual(self.gen.gauss_next, None) - - def test_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertRaises(NotImplementedError, pickle.dumps, self.gen, proto) - - def test_53_bits_per_float(self): - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) + self.assertRaises(TypeError, getrandbits) + self.assertRaises(TypeError, getrandbits, 1, 2) + self.assertRaises(ValueError, getrandbits, -1) + self.assertRaises(OverflowError, getrandbits, 1<<1000) + self.assertRaises(ValueError, getrandbits, -1<<1000) + self.assertRaises(TypeError, getrandbits, 10.1) def test_bigrand(self): # The randrange routine should build-up the required number of bits @@ -559,6 +494,10 @@ def test_randrange_step(self): randrange(1000, step=100) with self.assertRaises(TypeError): randrange(1000, None, step=100) + with self.assertRaises(TypeError): + randrange(1000, step=MyIndex(1)) + with self.assertRaises(TypeError): + randrange(1000, None, step=MyIndex(1)) def test_randbelow_logic(self, _log=log, int=int): # check bitcount transition points: 2**i and 2**(i+1)-1 @@ -581,6 +520,116 @@ def test_randbelow_logic(self, _log=log, int=int): self.assertEqual(k, numbits) # note the stronger assertion self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion + def test_randrange_index(self): + randrange = self.gen.randrange + self.assertIn(randrange(MyIndex(5)), range(5)) + self.assertIn(randrange(MyIndex(2), MyIndex(7)), range(2, 7)) + self.assertIn(randrange(MyIndex(5), MyIndex(15), MyIndex(2)), range(5, 15, 2)) + + def test_randint(self): + randint = self.gen.randint + self.assertIn(randint(2, 5), (2, 3, 4, 5)) + self.assertEqual(randint(2, 2), 2) + self.assertIn(randint(MyIndex(2), MyIndex(5)), (2, 3, 4, 5)) + self.assertEqual(randint(MyIndex(2), MyIndex(2)), 2) + + self.assertRaises(ValueError, randint, 5, 2) + self.assertRaises(TypeError, randint) + self.assertRaises(TypeError, randint, 2) + self.assertRaises(TypeError, randint, 2, 5, 1) + self.assertRaises(TypeError, randint, 2.0, 5) + self.assertRaises(TypeError, randint, 2, 5.0) + + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + state = pickle.dumps(self.gen, proto) + origseq = [self.gen.random() for i in range(10)] + newgen = pickle.loads(state) + restoredseq = [newgen.random() for i in range(10)] + self.assertEqual(origseq, restoredseq) + + def test_bug_1727780(self): + # verify that version-2-pickles can be loaded + # fine, whether they are created on 32-bit or 64-bit + # platforms, and that version-3-pickles load fine. + files = [("randv2_32.pck", 780), + ("randv2_64.pck", 866), + ("randv3.pck", 343)] + for file, value in files: + with open(support.findfile(file),"rb") as f: + r = pickle.load(f) + self.assertEqual(int(r.random()*1000), value) + + def test_bug_9025(self): + # Had problem with an uneven distribution in int(n*random()) + # Verify the fix by checking that distributions fall within expectations. + n = 100000 + randrange = self.gen.randrange + k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) + self.assertTrue(0.30 < k/n < .37, (k/n)) + + def test_randrange_bug_1590891(self): + start = 1000000000000 + stop = -100000000000000000000 + step = -200 + x = self.gen.randrange(start, stop, step) + self.assertTrue(stop < x <= start) + self.assertEqual((x+stop)%step, 0) + + def test_randbytes(self): + # Verify ranges + for n in range(1, 10): + data = self.gen.randbytes(n) + self.assertEqual(type(data), bytes) + self.assertEqual(len(data), n) + + self.assertEqual(self.gen.randbytes(0), b'') + + # Verify argument checking + self.assertRaises(TypeError, self.gen.randbytes) + self.assertRaises(TypeError, self.gen.randbytes, 1, 2) + self.assertRaises(ValueError, self.gen.randbytes, -1) + self.assertRaises(OverflowError, self.gen.randbytes, 1<<1000) + self.assertRaises((ValueError, OverflowError), self.gen.randbytes, -1<<1000) + self.assertRaises(TypeError, self.gen.randbytes, 1.0) + + def test_mu_sigma_default_args(self): + self.assertIsInstance(self.gen.normalvariate(), float) + self.assertIsInstance(self.gen.gauss(), float) + + +try: + random.SystemRandom().random() +except NotImplementedError: + SystemRandom_available = False +else: + SystemRandom_available = True + +@unittest.skipUnless(SystemRandom_available, "random.SystemRandom not available") +class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): + gen = random.SystemRandom() + + def test_autoseed(self): + # Doesn't need to do anything except not fail + self.gen.seed() + + def test_saverestore(self): + self.assertRaises(NotImplementedError, self.gen.getstate) + self.assertRaises(NotImplementedError, self.gen.setstate, None) + + def test_seedargs(self): + # Doesn't need to do anything except not fail + self.gen.seed(100) + + def test_gauss(self): + self.gen.gauss_next = None + self.gen.seed(100) + self.assertEqual(self.gen.gauss_next, None) + + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(NotImplementedError, pickle.dumps, self.gen, proto) + class TestRawMersenneTwister(unittest.TestCase): @test.support.cpython_only @@ -766,38 +815,6 @@ def test_long_seed(self): seed = (1 << (10000 * 8)) - 1 # about 10K bytes self.gen.seed(seed) - def test_53_bits_per_float(self): - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) - - def test_bigrand(self): - # The randrange routine should build-up the required number of bits - # in stages so that all bit positions are active. - span = 2 ** 500 - cum = 0 - for i in range(100): - r = self.gen.randrange(span) - self.assertTrue(0 <= r < span) - cum |= r - self.assertEqual(cum, span-1) - - def test_bigrand_ranges(self): - for i in [40,80, 160, 200, 211, 250, 375, 512, 550]: - start = self.gen.randrange(2 ** (i-2)) - stop = self.gen.randrange(2 ** i) - if stop <= start: - continue - self.assertTrue(start <= self.gen.randrange(start, stop) < stop) - - def test_rangelimits(self): - for start, stop in [(-2,0), (-(2**60)-2,-(2**60)), (2**60,2**60+2)]: - self.assertEqual(set(range(start,stop)), - set([self.gen.randrange(start,stop) for i in range(100)])) - def test_getrandbits(self): super().test_getrandbits() @@ -805,6 +822,25 @@ def test_getrandbits(self): self.gen.seed(1234567) self.assertEqual(self.gen.getrandbits(100), 97904845777343510404718956115) + self.gen.seed(1234567) + self.assertEqual(self.gen.getrandbits(MyIndex(100)), + 97904845777343510404718956115) + + def test_getrandbits_2G_bits(self): + size = 2**31 + self.gen.seed(1234567) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) + self.assertEqual(x & (2**100-1), 890186470919986886340158459475) + self.assertEqual(x >> (size-100), 1226514312032729439655761284440) + + @support.bigmemtest(size=2**32, memuse=1/8+2/15, dry_run=False) + def test_getrandbits_4G_bits(self, size): + self.gen.seed(1234568) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) + self.assertEqual(x & (2**100-1), 287241425661104632871036099814) + self.assertEqual(x >> (size-100), 739728759900339699429794460738) def test_randrange_uses_getrandbits(self): # Verify use of getrandbits by randrange @@ -816,27 +852,6 @@ def test_randrange_uses_getrandbits(self): self.assertEqual(self.gen.randrange(2**99), 97904845777343510404718956115) - def test_randbelow_logic(self, _log=log, int=int): - # check bitcount transition points: 2**i and 2**(i+1)-1 - # show that: k = int(1.001 + _log(n, 2)) - # is equal to or one greater than the number of bits in n - for i in range(1, 1000): - n = 1 << i # check an exact power of two - numbits = i+1 - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) - self.assertEqual(n, 2**(k-1)) - - n += n - 1 # check 1 below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertIn(k, [numbits, numbits+1]) - self.assertTrue(2**k > n > 2**(k-2)) - - n -= n >> 15 # check a little farther below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) # note the stronger assertion - self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion - def test_randbelow_without_getrandbits(self): # Random._randbelow() can only use random() when the built-in one # has been overridden but no new getrandbits() method was supplied. @@ -871,14 +886,6 @@ def test_randbelow_without_getrandbits(self): self.gen._randbelow_without_getrandbits(n, maxsize=maxsize) self.assertEqual(random_mock.call_count, 2) - def test_randrange_bug_1590891(self): - start = 1000000000000 - stop = -100000000000000000000 - step = -200 - x = self.gen.randrange(start, stop, step) - self.assertTrue(stop < x <= start) - self.assertEqual((x+stop)%step, 0) - def test_choices_algorithms(self): # The various ways of specifying weights should produce the same results choices = self.gen.choices @@ -962,6 +969,14 @@ def test_randbytes_getrandbits(self): self.assertEqual(self.gen.randbytes(n), gen2.getrandbits(n * 8).to_bytes(n, 'little')) + @support.bigmemtest(size=2**29, memuse=1+16/15, dry_run=False) + def test_randbytes_256M(self, size): + self.gen.seed(2849427419) + x = self.gen.randbytes(size) + self.assertEqual(len(x), size) + self.assertEqual(x[:12].hex(), 'f6fd9ae63855ab91ea238b4f') + self.assertEqual(x[-12:].hex(), '0e7af69a84ee99bf4a11becc') + def test_sample_counts_equivalence(self): # Test the documented strong equivalence to a sample with repeated elements. # We run this test on random.Random() which makes deterministic selections @@ -1386,6 +1401,7 @@ def test__all__(self): # tests validity but not completeness of the __all__ list self.assertTrue(set(random.__all__) <= set(dir(random))) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @test.support.requires_fork() def test_after_fork(self): # Test the global Random instance gets reseeded in child @@ -1415,27 +1431,27 @@ class CommandLineTest(unittest.TestCase): def test_parse_args(self): args, help_text = random._parse_args(shlex.split("--choice a b c")) self.assertEqual(args.choice, ["a", "b", "c"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("--integer 5")) self.assertEqual(args.integer, 5) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("--float 2.5")) self.assertEqual(args.float, 2.5) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("a b c")) self.assertEqual(args.input, ["a", "b", "c"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("5")) self.assertEqual(args.input, ["5"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("2.5")) self.assertEqual(args.input, ["2.5"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") def test_main(self): for command, expected in [ diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 3870b153688..2c9c290e890 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -470,6 +470,16 @@ def test_iterator_setstate(self): it.__setstate__(2**64 - 7) self.assertEqual(list(it), [12, 10]) + def test_iterator_invalid_setstate(self): + for invalid_value in (1.0, ""): + ranges = (('rangeiter', range(10, 100, 2)), + ('longrangeiter', range(10, 2**65, 2))) + for rng_name, rng in ranges: + with self.subTest(invalid_value=invalid_value, range=rng_name): + it = iter(rng) + with self.assertRaises(TypeError): + it.__setstate__(invalid_value) + def test_odd_bug(self): # This used to raise a "SystemError: NULL result without error" # because the range validation step was eating the exception diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index f79a6149078..9f6f04bf6b8 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1639,6 +1639,24 @@ def s_int(scanner, token): return int(token) (['sum', 'op=', 3, 'op*', 'foo', 'op+', 312.5, 'op+', 'bar'], '')) + def test_bug_gh140797(self): + # gh140797: Capturing groups are not allowed in re.Scanner + + msg = r"Cannot use capturing groups in re\.Scanner" + # Capturing group throws an error + with self.assertRaisesRegex(ValueError, msg): + Scanner([("(a)b", None)]) + + # Named Group + with self.assertRaisesRegex(ValueError, msg): + Scanner([("(?P<name>a)", None)]) + + # Non-capturing groups should pass normally + s = Scanner([("(?:a)b", lambda scanner, token: token)]) + result, rem = s.scan("ab") + self.assertEqual(result,['ab']) + self.assertEqual(rem,'') + def test_bug_448951(self): # bug 448951 (similar to 429357, but with single char match) # (Also test greedy matches.) @@ -2178,6 +2196,8 @@ def test_bug_20998(self): self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3)) @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("sunos"), + "test doesn't work on Solaris, gh-91214") def test_locale_caching(self): # Issue #22410 oldlocale = locale.setlocale(locale.LC_CTYPE) @@ -2215,6 +2235,8 @@ def check_en_US_utf8(self): self.assertIsNone(re.match(b'(?Li)\xe5', b'\xc5')) @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("sunos"), + "test doesn't work on Solaris, gh-91214") def test_locale_compiled(self): oldlocale = locale.setlocale(locale.LC_CTYPE) self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) @@ -2868,11 +2890,11 @@ def test_long_pattern(self): pattern = 'Very %spattern' % ('long ' * 1000) r = repr(re.compile(pattern)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") + self.assertStartsWith(r, "re.compile('Very long long lon") r = repr(re.compile(pattern, re.I)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") - self.assertEqual(r[-16:], ", re.IGNORECASE)") + self.assertStartsWith(r, "re.compile('Very long long lon") + self.assertEndsWith(r, ", re.IGNORECASE)") def test_flags_repr(self): self.assertEqual(repr(re.I), "re.IGNORECASE") @@ -2927,33 +2949,6 @@ def test_disallow_instantiation(self): pat = re.compile("") check_disallow_instantiation(self, type(pat.scanner(""))) - def test_deprecated_modules(self): - deprecated = { - 'sre_compile': ['compile', 'error', - 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', - '_compile_info'], - 'sre_constants': ['error', 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', - '_NamedIntConstant'], - 'sre_parse': ['SubPattern', 'parse', - 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', - '_parse_sub'], - } - for name in deprecated: - with self.subTest(module=name): - sys.modules.pop(name, None) - with self.assertWarns(DeprecationWarning) as w: - __import__(name) - self.assertEqual(str(w.warning), - f"module {name!r} is deprecated") - self.assertEqual(w.filename, __file__) - self.assertIn(name, sys.modules) - mod = sys.modules[name] - self.assertEqual(mod.__name__, name) - self.assertEqual(mod.__package__, '') - for attr in deprecated[name]: - self.assertTrue(hasattr(mod, attr)) - del sys.modules[name] - @cpython_only def test_case_helpers(self): import _sre @@ -3139,5 +3134,15 @@ def test_re_tests(self): self.assertTrue(obj.search(s)) +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(re, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index b9d082b3597..3982686dd10 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -1,6 +1,7 @@ """ Very minimal unittests for parts of the readline module. """ +import codecs import locale import os import sys @@ -231,6 +232,13 @@ def test_nonascii(self): # writing and reading non-ASCII bytes into/from a TTY works, but # readline or ncurses ignores non-ASCII bytes on read. self.skipTest(f"the LC_CTYPE locale is {loc!r}") + if sys.flags.utf8_mode: + encoding = locale.getencoding() + encoding = codecs.lookup(encoding).name # normalize the name + if encoding != "utf-8": + # gh-133711: The Python UTF-8 Mode ignores the LC_CTYPE locale + # and always use the UTF-8 encoding. + self.skipTest(f"the LC_CTYPE encoding is {encoding!r}") try: readline.add_history("\xEB\xEF") @@ -405,6 +413,24 @@ def test_write_read_limited_history(self): # So, we've only tested that the read did not fail. # See TestHistoryManipulation for the full test. + @unittest.skipUnless(hasattr(readline, "get_pre_input_hook"), + "get_pre_input_hook not available") + def test_get_pre_input_hook(self): + # Save and restore the original hook to avoid side effects + original_hook = readline.get_pre_input_hook() + self.addCleanup(readline.set_pre_input_hook, original_hook) + + # Test that get_pre_input_hook returns None when no hook is set + readline.set_pre_input_hook(None) + self.assertIsNone(readline.get_pre_input_hook()) + + # Set a hook and verify we can retrieve it + def my_hook(): + pass + + readline.set_pre_input_hook(my_hook) + self.assertIs(readline.get_pre_input_hook(), my_hook) + @unittest.skipUnless(support.Py_GIL_DISABLED, 'these tests can only possibly fail with GIL disabled') class FreeThreadingTest(unittest.TestCase): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 7e317d5ab94..c27b3c86292 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -182,6 +182,22 @@ def test_randomize(self): self.assertTrue(regrtest.randomize) self.assertIsInstance(regrtest.random_seed, int) + def test_no_randomize(self): + ns = self.parse_args([]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize"]) + self.assertIs(ns.randomize, True) + + ns = self.parse_args(["--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize", "--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--no-randomize", "--randomize"]) + self.assertIs(ns.randomize, False) + def test_randseed(self): ns = self.parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) @@ -189,6 +205,10 @@ def test_randseed(self): self.checkError(['--randseed'], 'expected one argument') self.checkError(['--randseed', 'foo'], 'invalid int value') + ns = self.parse_args(['--randseed', '12345', '--no-randomize']) + self.assertEqual(ns.random_seed, 12345) + self.assertFalse(ns.randomize) + def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): @@ -428,15 +448,17 @@ def create_regrtest(self, args): return regrtest - def check_ci_mode(self, args, use_resources, rerun=True): + def check_ci_mode(self, args, use_resources, + *, rerun=True, randomize=True, output_on_failure=True): regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) - self.assertTrue(regrtest.randomize) + self.assertEqual(regrtest.fail_rerun, False) + self.assertEqual(regrtest.randomize, randomize) self.assertIsInstance(regrtest.random_seed, int) self.assertTrue(regrtest.fail_env_changed) self.assertTrue(regrtest.print_slowest) - self.assertTrue(regrtest.output_on_failure) + self.assertEqual(regrtest.output_on_failure, output_on_failure) self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources)) return regrtest @@ -463,12 +485,29 @@ def test_fast_ci_resource(self): use_resources.remove('network') self.check_ci_mode(args, use_resources) + def test_fast_ci_verbose(self): + args = ['--fast-ci', '--verbose'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources, + output_on_failure=False) + self.assertEqual(regrtest.verbose, True) + def test_slow_ci(self): args = ['--slow-ci'] use_resources = sorted(cmdline.ALL_RESOURCES) regrtest = self.check_ci_mode(args, use_resources) self.assertEqual(regrtest.timeout, 20 * 60) + def test_ci_no_randomize(self): + all_resources = set(cmdline.ALL_RESOURCES) + self.check_ci_mode( + ["--slow-ci", "--no-randomize"], all_resources, randomize=False + ) + self.check_ci_mode( + ["--fast-ci", "--no-randomize"], all_resources - {'cpu'}, randomize=False + ) + def test_dont_add_python_opts(self): args = ['--dont-add-python-opts'] ns = cmdline._parse_args(args) @@ -768,13 +807,16 @@ def run_command(self, args, input=None, exitcode=0, **kw): self.fail(msg) return proc - def run_python(self, args, **kw): + def run_python(self, args, isolated=True, **kw): extraargs = [] if 'uops' in sys._xoptions: # Pass -X uops along extraargs.extend(['-X', 'uops']) - args = [sys.executable, *extraargs, '-X', 'faulthandler', '-I', *args] - proc = self.run_command(args, **kw) + cmd = [sys.executable, *extraargs, '-X', 'faulthandler'] + if isolated: + cmd.append('-I') + cmd.extend(args) + proc = self.run_command(cmd, **kw) return proc.stdout @@ -831,8 +873,8 @@ def check_output(self, output): self.check_executed_tests(output, self.tests, randomize=True, stats=len(self.tests)) - def run_tests(self, args, env=None): - output = self.run_python(args, env=env) + def run_tests(self, args, env=None, isolated=True): + output = self.run_python(args, env=env, isolated=isolated) self.check_output(output) def test_script_regrtest(self): @@ -874,7 +916,10 @@ def test_script_autotest(self): self.run_tests(args) def run_batch(self, *args): - proc = self.run_command(args) + proc = self.run_command(args, + # gh-133711: cmd.exe uses the OEM code page + # to display the non-ASCII current directory + errors="backslashreplace") self.check_output(proc.stdout) @unittest.skipUnless(sysconfig.is_python_build(), @@ -2064,7 +2109,7 @@ def load_tests(loader, tests, pattern): self.check_executed_tests(output, [testname], failed=[testname], parallel=True, - stats=TestStats(1, 1, 0)) + stats=TestStats(1, 2, 1)) def _check_random_seed(self, run_workers: bool): # gh-109276: When -r/--randomize is used, random.seed() is called @@ -2273,7 +2318,6 @@ def test_pass(self): def test_xml(self): code = textwrap.dedent(r""" import unittest - from test import support class VerboseTests(unittest.TestCase): def test_failed(self): @@ -2308,6 +2352,50 @@ def test_failed(self): for out in testcase.iter('system-out'): self.assertEqual(out.text, r"abc \x1b def") + def test_nonascii(self): + code = textwrap.dedent(r""" + import unittest + + class NonASCIITests(unittest.TestCase): + def test_docstring(self): + '''docstring:\u20ac''' + + def test_subtest(self): + with self.subTest(param='subtest:\u20ac'): + pass + + def test_skip(self): + self.skipTest('skipped:\u20ac') + """) + testname = self.create_test(code=code) + + env = dict(os.environ) + env['PYTHONIOENCODING'] = 'ascii' + + def check(output): + self.check_executed_tests(output, testname, stats=TestStats(3, 0, 1)) + self.assertIn(r'docstring:\u20ac', output) + self.assertIn(r'skipped:\u20ac', output) + + # Run sequentially + output = self.run_tests('-v', testname, env=env, isolated=False) + check(output) + + # Run in parallel + output = self.run_tests('-j1', '-v', testname, env=env, isolated=False) + check(output) + + def test_pgo_exclude(self): + # Get PGO tests + output = self.run_tests('--pgo', '--list-tests') + pgo_tests = output.strip().split() + + # Exclude test_re + output = self.run_tests('--pgo', '--list-tests', '-x', 'test_re') + tests = output.strip().split() + self.assertNotIn('test_re', tests) + self.assertEqual(len(tests), len(pgo_tests) - 1) + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index aef8a6b0129..ede99de9819 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -1,5 +1,4 @@ import io -import time import itertools import json import os @@ -8,16 +7,13 @@ import socket import subprocess import sys -import tempfile import textwrap -import threading import unittest import unittest.mock from contextlib import closing, contextmanager, redirect_stdout, redirect_stderr, ExitStack -from pathlib import Path -from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT -from test.support.os_helper import temp_dir, TESTFN, unlink -from typing import Dict, List, Optional, Tuple, Union, Any +from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT, subTests +from test.support.os_helper import TESTFN, unlink +from typing import List import pdb from pdb import _PdbServer, _PdbClient @@ -283,37 +279,50 @@ def test_handling_other_message(self): expected_stdout="Some message.\n", ) - def test_handling_help_for_command(self): - """Test handling a request to display help for a command.""" + @unittest.skipIf(sys.flags.optimize >= 2, "Help not available for -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "Usage: ll | longlist"), + # a request to display a help overview + ({"help": ""}, "type help <topic>"), + # a request to display the full PDB manual + ({"help": "pdb"}, ">>> import pdb"), + ], + ) + def test_handling_help_when_available(self, help_request, expected_substring): + """Test handling help requests when help is available.""" incoming = [ - ("server", {"help": "ll"}), + ("server", help_request), ] self.do_test( incoming=incoming, expected_outgoing=[], - expected_stdout_substring="Usage: ll | longlist", + expected_stdout_substring=expected_substring, ) - def test_handling_help_without_a_specific_topic(self): - """Test handling a request to display a help overview.""" + @unittest.skipIf(sys.flags.optimize < 2, "Needs -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "No help for 'll'"), + # a request to display a help overview + ({"help": ""}, "Undocumented commands"), + # a request to display the full PDB manual + ({"help": "pdb"}, "No help for 'pdb'"), + ], + ) + def test_handling_help_when_not_available(self, help_request, expected_substring): + """Test handling help requests when help is not available.""" incoming = [ - ("server", {"help": ""}), + ("server", help_request), ] self.do_test( incoming=incoming, expected_outgoing=[], - expected_stdout_substring="type help <topic>", - ) - - def test_handling_help_pdb(self): - """Test handling a request to display the full PDB manual.""" - incoming = [ - ("server", {"help": "pdb"}), - ] - self.do_test( - incoming=incoming, - expected_outgoing=[], - expected_stdout_substring=">>> import pdb", + expected_stdout_substring=expected_substring, ) def test_handling_pdb_prompts(self): @@ -1434,7 +1443,6 @@ def test_multi_line_commands(self): def _supports_remote_attaching(): - from contextlib import suppress PROCESS_VM_READV_SUPPORTED = False try: @@ -1531,6 +1539,9 @@ def do_integration_test(self, client_stdin): redirect_stdout(client_stdout), redirect_stderr(client_stderr), unittest.mock.patch("sys.argv", ["pdb", "-p", str(process.pid)]), + unittest.mock.patch( + "pdb.exit_with_permission_help_text", side_effect=PermissionError + ), ): try: pdb.main() @@ -1579,5 +1590,17 @@ def test_attach_to_process_with_colors(self): self.assertNotIn("while x == 1", output["client"]["stdout"]) self.assertIn("while x == 1", re.sub("\x1b[^m]*m", "", output["client"]["stdout"])) + def test_attach_to_non_existent_process(self): + with force_color(False): + result = subprocess.run([sys.executable, "-m", "pdb", "-p", "999999"], text=True, capture_output=True) + self.assertNotEqual(result.returncode, 0) + if sys.platform == "darwin": + # On MacOS, attaching to a non-existent process gives PermissionError + error = "The specified process cannot be attached to due to insufficient permissions" + else: + error = "Cannot attach to pid 999999, please make sure that the process exists" + self.assertIn(error, result.stdout) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 228b326699e..042aa84b35d 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -5,6 +5,7 @@ import subprocess import sys import unittest +from functools import partial from textwrap import dedent from test import support from test.support import ( @@ -27,7 +28,7 @@ raise unittest.SkipTest("test module requires subprocess") -def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): +def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, custom=False, **kw): """Run the Python REPL with the given arguments. kw is extra keyword args to pass to subprocess.Popen. Returns a Popen @@ -38,10 +39,14 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): # line option '-i' and the process name set to '<stdin>'. # The directory of argv[0] must match the directory of the Python # executable for the Popen() call to python to succeed as the directory - # path may be used by Py_GetPath() to build the default module search - # path. + # path may be used by PyConfig_Get("module_search_paths") to build the + # default module search path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "<stdin>") - cmd_line = [stdin_fname, '-I', '-i'] + cmd_line = [stdin_fname, '-I'] + # Don't re-run the built-in REPL from interactive mode + # if we're testing a custom REPL (such as the asyncio REPL). + if not custom: + cmd_line.append('-i') cmd_line.extend(args) # Set TERM=vt100, for the rationale see the comments in spawn_python() of @@ -55,6 +60,10 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): stdout=stdout, stderr=stderr, **kw) + +spawn_asyncio_repl = partial(spawn_repl, "-m", "asyncio", custom=True) + + def run_on_interactive_mode(source): """Spawn a new Python interpreter, pass the given input source code from the stdin and return the @@ -188,6 +197,68 @@ def foo(x): ] self.assertEqual(traceback_lines, expected_lines) + def test_pythonstartup_error_reporting(self): + # errors based on https://github.com/python/cpython/issues/137576 + + def make_repl(env): + return subprocess.Popen( + [os.path.join(os.path.dirname(sys.executable), '<stdin>'), "-i"], + executable=sys.executable, + text=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + ) + + # case 1: error in user input, but PYTHONSTARTUP is fine + with os_helper.temp_dir() as tmpdir: + script = os.path.join(tmpdir, "pythonstartup.py") + with open(script, "w") as f: + f.write("print('from pythonstartup')" + os.linesep) + + env = os.environ.copy() + env['PYTHONSTARTUP'] = script + env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".pythonhist") + p = make_repl(env) + p.stdin.write("1/0") + output = kill_python(p) + expected = dedent(""" + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + 1/0 + ~^~ + ZeroDivisionError: division by zero + """) + self.assertIn("from pythonstartup", output) + self.assertIn(expected, output) + + # case 2: error in PYTHONSTARTUP triggered by user input + with os_helper.temp_dir() as tmpdir: + script = os.path.join(tmpdir, "pythonstartup.py") + with open(script, "w") as f: + f.write("def foo():\n 1/0\n") + + env = os.environ.copy() + env['PYTHONSTARTUP'] = script + env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".pythonhist") + p = make_repl(env) + p.stdin.write('foo()') + output = kill_python(p) + expected = dedent(""" + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + foo() + ~~~^^ + File "%s", line 2, in foo + 1/0 + ~^~ + ZeroDivisionError: division by zero + """) % script + self.assertIn(expected, output) + + + def test_runsource_show_syntax_error_location(self): user_input = dedent("""def f(x, x): ... """) @@ -197,7 +268,7 @@ def test_runsource_show_syntax_error_location(self): expected_lines = [ ' def f(x, x): ...', ' ^', - "SyntaxError: duplicate argument 'x' in function definition" + "SyntaxError: duplicate parameter 'x' in function definition" ] self.assertEqual(output.splitlines()[4:-1], expected_lines) @@ -297,7 +368,7 @@ def f(): class TestAsyncioREPL(unittest.TestCase): def test_multiple_statements_fail_early(self): user_input = "1 / 0; print(f'afterwards: {1+1}')" - p = spawn_repl("-m", "asyncio") + p = spawn_asyncio_repl() p.stdin.write(user_input) output = kill_python(p) self.assertIn("ZeroDivisionError", output) @@ -309,7 +380,7 @@ def test_toplevel_contextvars_sync(self): var = ContextVar("var", default="failed") var.set("ok") """) - p = spawn_repl("-m", "asyncio") + p = spawn_asyncio_repl() p.stdin.write(user_input) user_input2 = dedent(""" print(f"toplevel contextvar test: {var.get()}") @@ -325,7 +396,7 @@ def test_toplevel_contextvars_async(self): from contextvars import ContextVar var = ContextVar('var', default='failed') """) - p = spawn_repl("-m", "asyncio") + p = spawn_asyncio_repl() p.stdin.write(user_input) user_input2 = "async def set_var(): var.set('ok')\n" p.stdin.write(user_input2) diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index ffad35092f9..22a55b57c07 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -151,14 +151,38 @@ def test_frozenset(self): eq(r(frozenset({1, 2, 3, 4, 5, 6, 7})), "frozenset({1, 2, 3, 4, 5, 6, ...})") def test_numbers(self): - eq = self.assertEqual - eq(r(123), repr(123)) - eq(r(123), repr(123)) - eq(r(1.0/3), repr(1.0/3)) + for x in [123, 1.0 / 3]: + self.assertEqual(r(x), repr(x)) - n = 10**100 - expected = repr(n)[:18] + "..." + repr(n)[-19:] - eq(r(n), expected) + max_digits = sys.get_int_max_str_digits() + for k in [100, max_digits - 1]: + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + expected = repr(n)[:18] + "..." + repr(n)[-19:] + self.assertEqual(r(n), expected) + + def re_msg(n, d): + return (rf'<{n.__class__.__name__} instance with roughly {d} ' + rf'digits \(limit at {max_digits}\) at 0x[a-f0-9]+>') + + k = max_digits + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + + for k in [max_digits + 1, 2 * max_digits]: + self.assertGreater(k, 100) + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + with self.subTest(f'10 ** {k} - 1', k=k): + n = 10 ** k - 1 + # Here, since math.log10(n) == math.log10(n-1), + # the number of digits of n - 1 is overestimated. + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) def test_instance(self): eq = self.assertEqual @@ -173,13 +197,13 @@ def test_instance(self): eq(r(i3), ("<ClassWithFailingRepr instance at %#x>"%id(i3))) s = r(ClassWithFailingRepr) - self.assertTrue(s.startswith("<class ")) - self.assertTrue(s.endswith(">")) + self.assertStartsWith(s, "<class ") + self.assertEndsWith(s, ">") self.assertIn(s.find("..."), [12, 13]) def test_lambda(self): r = repr(lambda x: x) - self.assertTrue(r.startswith("<function ReprTests.test_lambda.<locals>.<lambda"), r) + self.assertStartsWith(r, "<function ReprTests.test_lambda.<locals>.<lambda") # XXX anonymous functions? see func_repr def test_builtin_function(self): @@ -187,8 +211,8 @@ def test_builtin_function(self): # Functions eq(repr(hash), '<built-in function hash>') # Methods - self.assertTrue(repr(''.split).startswith( - '<built-in method split of str object at 0x')) + self.assertStartsWith(repr(''.split), + '<built-in method split of str object at 0x') def test_range(self): eq = self.assertEqual @@ -373,20 +397,20 @@ def test_valid_indent(self): 'object': { 1: 'two', b'three': [ - (4.5, 6.7), + (4.5, 6.25), [set((8, 9)), frozenset((10, 11))], ], }, 'tests': ( (dict(indent=None), '''\ - {1: 'two', b'three': [(4.5, 6.7), [{8, 9}, frozenset({10, 11})]]}'''), + {1: 'two', b'three': [(4.5, 6.25), [{8, 9}, frozenset({10, 11})]]}'''), (dict(indent=False), '''\ { 1: 'two', b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -406,7 +430,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -426,7 +450,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -446,7 +470,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -466,7 +490,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -494,7 +518,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -514,7 +538,7 @@ def test_valid_indent(self): -->b'three': [ -->-->( -->-->-->4.5, - -->-->-->6.7, + -->-->-->6.25, -->-->), -->-->[ -->-->-->{ @@ -534,7 +558,7 @@ def test_valid_indent(self): ....b'three': [ ........( ............4.5, - ............6.7, + ............6.25, ........), ........[ ............{ @@ -730,8 +754,8 @@ class baz: importlib.invalidate_caches() from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import baz ibaz = baz.baz() - self.assertTrue(repr(ibaz).startswith( - "<%s.baz object at 0x" % baz.__name__)) + self.assertStartsWith(repr(ibaz), + "<%s.baz object at 0x" % baz.__name__) def test_method(self): self._check_path_limitations('qux') @@ -744,13 +768,13 @@ def amethod(self): pass from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import qux # Unbound methods first r = repr(qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod) - self.assertTrue(r.startswith('<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod'), r) + self.assertStartsWith(r, '<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod') # Bound method next iqux = qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa() r = repr(iqux.amethod) - self.assertTrue(r.startswith( + self.assertStartsWith(r, '<bound method aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod of <%s.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa object at 0x' \ - % (qux.__name__,) ), r) + % (qux.__name__,) ) @unittest.skip('needs a built-in function with a really long name') def test_builtin_function(self): diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index d23d3623235..5fd076bee38 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -14,89 +14,161 @@ class ResourceTest(unittest.TestCase): def test_args(self): self.assertRaises(TypeError, resource.getrlimit) - self.assertRaises(TypeError, resource.getrlimit, 42, 42) + self.assertRaises(TypeError, resource.getrlimit, 0, 42) + self.assertRaises(OverflowError, resource.getrlimit, 2**1000) + self.assertRaises(OverflowError, resource.getrlimit, -2**1000) + self.assertRaises(TypeError, resource.getrlimit, '0') self.assertRaises(TypeError, resource.setrlimit) - self.assertRaises(TypeError, resource.setrlimit, 42, 42, 42) + self.assertRaises(TypeError, resource.setrlimit, 0) + self.assertRaises(TypeError, resource.setrlimit, 0, 42) + self.assertRaises(TypeError, resource.setrlimit, 0, 42, 42) + self.assertRaises(OverflowError, resource.setrlimit, 2**1000, (42, 42)) + self.assertRaises(OverflowError, resource.setrlimit, -2**1000, (42, 42)) + self.assertRaises(ValueError, resource.setrlimit, 0, (42,)) + self.assertRaises(ValueError, resource.setrlimit, 0, (42, 42, 42)) + self.assertRaises(TypeError, resource.setrlimit, '0', (42, 42)) + self.assertRaises(TypeError, resource.setrlimit, 0, ('42', 42)) + self.assertRaises(TypeError, resource.setrlimit, 0, (42, '42')) @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_ismax(self): - try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: - pass - else: - # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really big - # number on a platform with large file support. On these platforms, - # we need to test that the get/setrlimit functions properly convert - # the number to a C long long and that the conversion doesn't raise - # an error. - self.assertEqual(resource.RLIM_INFINITY, max) - resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really big + # number on a platform with large file support. On these platforms, + # we need to test that the get/setrlimit functions properly convert + # the number to a C long long and that the conversion doesn't raise + # an error. + self.assertGreater(resource.RLIM_INFINITY, 0) + self.assertEqual(resource.RLIM_INFINITY, max) + self.assertLessEqual(cur, max) + resource.setrlimit(resource.RLIMIT_FSIZE, (max, max)) + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_enforced(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + # Check to see what happens when the RLIMIT_FSIZE is small. Some + # versions of Python were terminated by an uncaught SIGXFSZ, but + # pythonrun.c has been fixed to ignore that exception. If so, the + # write() should return EFBIG when the limit is exceeded. + + # At least one platform has an unlimited RLIMIT_FSIZE and attempts + # to change it raise ValueError instead. try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: - pass - else: - # Check to see what happens when the RLIMIT_FSIZE is small. Some - # versions of Python were terminated by an uncaught SIGXFSZ, but - # pythonrun.c has been fixed to ignore that exception. If so, the - # write() should return EFBIG when the limit is exceeded. - - # At least one platform has an unlimited RLIMIT_FSIZE and attempts - # to change it raise ValueError instead. try: + resource.setrlimit(resource.RLIMIT_FSIZE, (1024, max)) + limit_set = True + except ValueError: + limit_set = False + f = open(os_helper.TESTFN, "wb") + try: + f.write(b"X" * 1024) try: - resource.setrlimit(resource.RLIMIT_FSIZE, (1024, max)) - limit_set = True - except ValueError: - limit_set = False - f = open(os_helper.TESTFN, "wb") - try: - f.write(b"X" * 1024) - try: - f.write(b"Y") + f.write(b"Y") + f.flush() + # On some systems (e.g., Ubuntu on hppa) the flush() + # doesn't always cause the exception, but the close() + # does eventually. Try flushing several times in + # an attempt to ensure the file is really synced and + # the exception raised. + for i in range(5): + time.sleep(.1) f.flush() - # On some systems (e.g., Ubuntu on hppa) the flush() - # doesn't always cause the exception, but the close() - # does eventually. Try flushing several times in - # an attempt to ensure the file is really synced and - # the exception raised. - for i in range(5): - time.sleep(.1) - f.flush() - except OSError: - if not limit_set: - raise - if limit_set: - # Close will attempt to flush the byte we wrote - # Restore limit first to avoid getting a spurious error - resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - finally: - f.close() - finally: + except OSError: + if not limit_set: + raise if limit_set: + # Close will attempt to flush the byte we wrote + # Restore limit first to avoid getting a spurious error resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - os_helper.unlink(os_helper.TESTFN) + finally: + f.close() + finally: + if limit_set: + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + os_helper.unlink(os_helper.TESTFN) - def test_fsize_toobig(self): + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_too_big(self): # Be sure that setrlimit is checking for really large values too_big = 10**50 + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: + resource.setrlimit(resource.RLIMIT_FSIZE, (too_big, max)) + except (OverflowError, ValueError): + pass + try: + resource.setrlimit(resource.RLIMIT_FSIZE, (max, too_big)) + except (OverflowError, ValueError): + pass + + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_not_too_big(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + self.addCleanup(resource.setrlimit, resource.RLIMIT_FSIZE, (cur, max)) + + def expected(cur): + # The glibc wrapper functions use a 64-bit rlim_t data type, + # even on 32-bit platforms. If a program tried to set a resource + # limit to a value larger than can be represented in a 32-bit + # unsigned long, then the glibc setrlimit() wrapper function + # silently converted the limit value to RLIM_INFINITY. + if sys.maxsize < 2**32 <= cur <= resource.RLIM_INFINITY: + return [(resource.RLIM_INFINITY, max), (cur, max)] + return [(min(cur, resource.RLIM_INFINITY), max)] + + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31-5, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**31-5, max)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**31)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**32-5)) + + try: + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32, max)) + except OverflowError: pass else: + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**32)) + + resource.setrlimit(resource.RLIMIT_FSIZE, (2**63-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**63-5)) try: - resource.setrlimit(resource.RLIMIT_FSIZE, (too_big, max)) - except (OverflowError, ValueError): - pass - try: - resource.setrlimit(resource.RLIMIT_FSIZE, (max, too_big)) - except (OverflowError, ValueError): + resource.setrlimit(resource.RLIMIT_FSIZE, (2**63, max)) + except ValueError: + # There is a hard limit on macOS. pass + else: + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**63)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**64-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**64-5)) + + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_negative(self): + self.assertGreater(resource.RLIM_INFINITY, 0) + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + for value in -5, -2**31, -2**32-5, -2**63, -2**64-5, -2**1000: + with self.subTest(value=value): + self.assertRaises(ValueError, resource.setrlimit, resource.RLIMIT_FSIZE, (value, max)) + self.assertRaises(ValueError, resource.setrlimit, resource.RLIMIT_FSIZE, (cur, value)) + + if resource.RLIM_INFINITY in (2**32-3, 2**32-1, 2**64-3, 2**64-1): + value = (resource.RLIM_INFINITY & 0xffff) - 0x10000 + with self.assertWarnsRegex(DeprecationWarning, "RLIM_INFINITY"): + resource.setrlimit(resource.RLIMIT_FSIZE, (value, max)) + with self.assertWarnsRegex(DeprecationWarning, "RLIM_INFINITY"): + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, value)) + @unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage") def test_getrusage(self): @@ -117,27 +189,33 @@ def test_getrusage(self): # Issue 6083: Reference counting bug @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_CPU is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_CPU'), 'requires resource.RLIMIT_CPU') def test_setrusage_refcount(self): - try: - limits = resource.getrlimit(resource.RLIMIT_CPU) - except AttributeError: - pass - else: - class BadSequence: - def __len__(self): - return 2 - def __getitem__(self, key): - if key in (0, 1): - return len(tuple(range(1000000))) - raise IndexError + limits = resource.getrlimit(resource.RLIMIT_CPU) + class BadSequence: + def __len__(self): + return 2 + def __getitem__(self, key): + if key in (0, 1): + return len(tuple(range(1000000))) + raise IndexError - resource.setrlimit(resource.RLIMIT_CPU, BadSequence()) + resource.setrlimit(resource.RLIMIT_CPU, BadSequence()) def test_pagesize(self): pagesize = resource.getpagesize() self.assertIsInstance(pagesize, int) self.assertGreaterEqual(pagesize, 0) + def test_contants(self): + self.assertIsInstance(resource.RLIM_INFINITY, int) + if sys.platform.startswith(('freebsd', 'solaris', 'sunos', 'aix')): + self.assertHasAttr(resource, 'RLIM_SAVED_CUR') + self.assertHasAttr(resource, 'RLIM_SAVED_MAX') + if hasattr(resource, 'RLIM_SAVED_CUR'): + self.assertIsInstance(resource.RLIM_SAVED_CUR, int) + self.assertIsInstance(resource.RLIM_SAVED_MAX, int) + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux only') def test_linux_constants(self): for attr in ['MSGQUEUE', 'NICE', 'RTPRIO', 'RTTIME', 'SIGPENDING']: @@ -145,7 +223,7 @@ def test_linux_constants(self): self.assertIsInstance(getattr(resource, 'RLIMIT_' + attr), int) def test_freebsd_contants(self): - for attr in ['SWAP', 'SBSIZE', 'NPTS']: + for attr in ['SWAP', 'SBSIZE', 'NPTS', 'UMTXP', 'VMEM', 'PIPEBUF']: with contextlib.suppress(AttributeError): self.assertIsInstance(getattr(resource, 'RLIMIT_' + attr), int) @@ -168,7 +246,8 @@ class BadSeq: def __len__(self): return 2 def __getitem__(self, key): - return limits[key] - 1 # new reference + lim = limits[key] + return lim - 1 if lim > 0 else lim + sys.maxsize*2 # new reference limits = resource.getrlimit(resource.RLIMIT_AS) self.assertEqual(resource.prlimit(0, resource.RLIMIT_AS, BadSeq()), diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d403a0fe96b..a8914953ce9 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -88,7 +88,7 @@ def create_expected_for_none(): ['CompleteMe._ham']) matches = self.completer.attr_matches('CompleteMe.__') for x in matches: - self.assertTrue(x.startswith('CompleteMe.__'), x) + self.assertStartsWith(x, 'CompleteMe.__') self.assertIn('CompleteMe.__name__', matches) self.assertIn('CompleteMe.__new__(', matches) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 8d89e2a8224..e33723cc70c 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -16,6 +16,14 @@ class BaseRobotTest: bad = [] site_maps = None + def __init_subclass__(cls): + super().__init_subclass__() + # Remove tests that do nothing. + if not cls.good: + cls.test_good_urls = None + if not cls.bad: + cls.test_bad_urls = None + def setUp(self): lines = io.StringIO(self.robots_txt).readlines() self.parser = urllib.robotparser.RobotFileParser() @@ -231,9 +239,16 @@ class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * Disallow: /some/path?name=value +Disallow: /another/path? +Disallow: /yet/one/path?name=value&more """ - good = ['/some/path'] - bad = ['/some/path?name=value'] + good = ['/some/path', '/some/path?', + '/some/path%3Fname=value', '/some/path?name%3Dvalue', + '/another/path', '/another/path%3F', + '/yet/one/path?name=value%26more'] + bad = ['/some/path?name=value' + '/another/path?', '/another/path?name=value', + '/yet/one/path?name=value&more'] class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): @@ -249,15 +264,79 @@ class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): bad = ['/some/path'] -class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase): - # normalize the URL first (#17403) +class PercentEncodingTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * -Allow: /some/path? -Disallow: /another/path? - """ - good = ['/some/path?'] - bad = ['/another/path?'] +Disallow: /a1/Z-._~ # unreserved characters +Disallow: /a2/%5A%2D%2E%5F%7E # percent-encoded unreserved characters +Disallow: /u1/%F0%9F%90%8D # percent-encoded ASCII Unicode character +Disallow: /u2/%f0%9f%90%8d +Disallow: /u3/\U0001f40d # raw non-ASCII Unicode character +Disallow: /v1/%F0 # percent-encoded non-ASCII octet +Disallow: /v2/%f0 +Disallow: /v3/\udcf0 # raw non-ASCII octet +Disallow: /p1%xy # raw percent +Disallow: /p2% +Disallow: /p3%25xy # percent-encoded percent +Disallow: /p4%2525xy # double percent-encoded percent +Disallow: /john%20smith # space +Disallow: /john doe +Disallow: /trailingspace%20 +Disallow: /question%3Fq=v # not query +Disallow: /hash%23f # not fragment +Disallow: /dollar%24 +Disallow: /asterisk%2A +Disallow: /sub/dir +Disallow: /slash%2F +Disallow: /query/question?q=%3F +Disallow: /query/raw/question?q=? +Disallow: /query/eq?q%3Dv +Disallow: /query/amp?q=v%26a +""" + good = [ + '/u1/%F0', '/u1/%f0', + '/u2/%F0', '/u2/%f0', + '/u3/%F0', '/u3/%f0', + '/p1%2525xy', '/p2%f0', '/p3%2525xy', '/p4%xy', '/p4%25xy', + '/question?q=v', + '/dollar', '/asterisk', + '/query/eq?q=v', + '/query/amp?q=v&a', + ] + bad = [ + '/a1/Z-._~', '/a1/%5A%2D%2E%5F%7E', + '/a2/Z-._~', '/a2/%5A%2D%2E%5F%7E', + '/u1/%F0%9F%90%8D', '/u1/%f0%9f%90%8d', '/u1/\U0001f40d', + '/u2/%F0%9F%90%8D', '/u2/%f0%9f%90%8d', '/u2/\U0001f40d', + '/u3/%F0%9F%90%8D', '/u3/%f0%9f%90%8d', '/u3/\U0001f40d', + '/v1/%F0', '/v1/%f0', '/v1/\udcf0', '/v1/\U0001f40d', + '/v2/%F0', '/v2/%f0', '/v2/\udcf0', '/v2/\U0001f40d', + '/v3/%F0', '/v3/%f0', '/v3/\udcf0', '/v3/\U0001f40d', + '/p1%xy', '/p1%25xy', + '/p2%', '/p2%25', '/p2%2525', '/p2%xy', + '/p3%xy', '/p3%25xy', + '/p4%2525xy', + '/john%20smith', '/john smith', + '/john%20doe', '/john doe', + '/trailingspace%20', '/trailingspace ', + '/question%3Fq=v', + '/hash#f', '/hash%23f', + '/dollar$', '/dollar%24', + '/asterisk*', '/asterisk%2A', + '/sub/dir', '/sub%2Fdir', + '/slash%2F', '/slash/', + '/query/question?q=?', '/query/question?q=%3F', + '/query/raw/question?q=?', '/query/raw/question?q=%3F', + '/query/eq?q%3Dv', + '/query/amp?q=v%26a', + ] + # other reserved characters + for c in ":/#[]@!$&'()*+,;=": + robots_txt += f'Disallow: /raw{c}\nDisallow: /pc%{ord(c):02X}\n' + bad.append(f'/raw{c}') + bad.append(f'/raw%{ord(c):02X}') + bad.append(f'/pc{c}') + bad.append(f'/pc%{ord(c):02X}') class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): @@ -299,26 +378,17 @@ def test_string_formatting(self): self.assertEqual(str(self.parser), self.expected_output) -class RobotHandler(BaseHTTPRequestHandler): - - def do_GET(self): - self.send_error(403, "Forbidden access") - - def log_message(self, format, *args): - pass - - @unittest.skipUnless( support.has_socket_support, "Socket server requires working socket." ) -class PasswordProtectedSiteTestCase(unittest.TestCase): +class BaseLocalNetworkTestCase: def setUp(self): # clear _opener global variable self.addCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), RobotHandler) + self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) self.t = threading.Thread( name='HTTPServer serving', @@ -335,6 +405,57 @@ def tearDown(self): self.t.join() self.server.server_close() + +SAMPLE_ROBOTS_TXT = b'''\ +User-agent: test_robotparser +Disallow: /utf8/\xf0\x9f\x90\x8d +Disallow: /non-utf8/\xf0 +Disallow: //[spam]/path +''' + + +class LocalNetworkTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(SAMPLE_ROBOTS_TXT) + + def log_message(self, format, *args): + pass + + @threading_helper.reap_threads + def testRead(self): + # Test that reading a weird robots.txt doesn't fail. + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + '/robots.txt' + parser = urllib.robotparser.RobotFileParser() + parser.set_url(robots_url) + parser.read() + # And it can even interpret the weird paths in some reasonable way. + agent = 'test_robotparser' + self.assertTrue(parser.can_fetch(agent, robots_url)) + self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) + self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) + self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) + self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) + self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) + + +class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_error(403, "Forbidden access") + + def log_message(self, format, *args): + pass + @threading_helper.reap_threads def testPasswordProtectedSite(self): addr = self.server.server_address diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index ada78ec8e6b..cc76b72b963 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -20,9 +20,11 @@ requires_subprocess, verbose, ) +from test import support from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script +from test.test_importlib.util import temporary_pycache_prefix import runpy @@ -763,6 +765,47 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") + def test_run_module_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with (temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + ns = run_module('test.test_import.data.syntax_warnings') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = ns['__file__'] + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + def test_run_path_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'<run_path>\z') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.script\z') + warnings.filterwarnings('error', module='<run_path>') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename, run_name='package.script') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + @force_not_colorized_test_class class TestExit(unittest.TestCase): @@ -796,7 +839,7 @@ def assertSigInt(self, cmd, *args, **kwargs): # Use -E to ignore PYTHONSAFEPATH cmd = [sys.executable, '-E', *cmd] proc = subprocess.run(cmd, *args, **kwargs, text=True, stderr=subprocess.PIPE) - self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n"), proc.stderr) + self.assertEndsWith(proc.stderr, "\nKeyboardInterrupt\n") self.assertEqual(proc.returncode, self.EXPECTED_CODE) def test_pymain_run_file(self): diff --git a/Lib/test/test_samply_profiler.py b/Lib/test/test_samply_profiler.py new file mode 100644 index 00000000000..ec0ed37ffd0 --- /dev/null +++ b/Lib/test/test_samply_profiler.py @@ -0,0 +1,244 @@ +import unittest +import subprocess +import sys +import sysconfig +import os +import pathlib +from test import support +from test.support.script_helper import ( + make_script, +) +from test.support.os_helper import temp_dir + + +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + +if support.check_sanitizer(address=True, memory=True, ub=True, function=True): + # gh-109580: Skip the test because it does crash randomly if Python is + # built with ASAN. + raise unittest.SkipTest("test crash randomly on ASAN/MSAN/UBSAN build") + + +def supports_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 + + +if not supports_trampoline_profiling(): + raise unittest.SkipTest("perf trampoline profiling not supported") + + +def samply_command_works(): + try: + cmd = ["samply", "--help"] + except (subprocess.SubprocessError, OSError): + return False + + # Check that we can run a simple samply run + with temp_dir() as script_dir: + try: + output_file = script_dir + "/profile.json.gz" + cmd = ( + "samply", + "record", + "--save-only", + "--output", + output_file, + sys.executable, + "-c", + 'print("hello")', + ) + env = {**os.environ, "PYTHON_JIT": "0"} + stdout = subprocess.check_output( + cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env + ) + except (subprocess.SubprocessError, OSError): + return False + + if "hello" not in stdout: + return False + + return True + + +def run_samply(cwd, *args, **env_vars): + env = os.environ.copy() + if env_vars: + env.update(env_vars) + env["PYTHON_JIT"] = "0" + output_file = cwd + "/profile.json.gz" + base_cmd = ( + "samply", + "record", + "--save-only", + "-o", output_file, + ) + proc = subprocess.run( + base_cmd + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + if proc.returncode: + print(proc.stderr, file=sys.stderr) + raise ValueError(f"Samply failed with return code {proc.returncode}") + + import gzip + with gzip.open(output_file, mode="rt", encoding="utf-8") as f: + return f.read() + + +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfilerMixin: + def run_samply(self, script_dir, perf_mode, script): + raise NotImplementedError() + + def test_python_calls_appear_in_the_stack_if_perf_activated(self): + with temp_dir() as script_dir: + code = """if 1: + def foo(n): + x = 0 + for i in range(n): + x += i + + def bar(n): + foo(n) + + def baz(n): + bar(n) + + baz(10000000) + """ + script = make_script(script_dir, "perftest", code) + output = self.run_samply(script_dir, script) + + self.assertIn(f"py::foo:{script}", output) + self.assertIn(f"py::bar:{script}", output) + self.assertIn(f"py::baz:{script}", output) + + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): + with temp_dir() as script_dir: + code = """if 1: + def foo(n): + x = 0 + for i in range(n): + x += i + + def bar(n): + foo(n) + + def baz(n): + bar(n) + + baz(10000000) + """ + script = make_script(script_dir, "perftest", code) + output = self.run_samply( + script_dir, script, activate_trampoline=False + ) + + self.assertNotIn(f"py::foo:{script}", output) + self.assertNotIn(f"py::bar:{script}", output) + self.assertNotIn(f"py::baz:{script}", output) + + +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfiler(unittest.TestCase, TestSamplyProfilerMixin): + def run_samply(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_samply(script_dir, sys.executable, "-Xperf", script) + return run_samply(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + + def test_pre_fork_compile(self): + code = """if 1: + import sys + import os + import sysconfig + from _testinternalcapi import ( + compile_perf_trampoline_entry, + perf_trampoline_set_persist_after_fork, + ) + + def foo_fork(): + pass + + def bar_fork(): + foo_fork() + + def foo(): + import time; time.sleep(1) + + def bar(): + foo() + + def compile_trampolines_for_all_functions(): + perf_trampoline_set_persist_after_fork(1) + for _, obj in globals().items(): + if callable(obj) and hasattr(obj, '__code__'): + compile_perf_trampoline_entry(obj.__code__) + + if __name__ == "__main__": + compile_trampolines_for_all_functions() + pid = os.fork() + if pid == 0: + print(os.getpid()) + bar_fork() + else: + bar() + """ + + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} + with subprocess.Popen( + [sys.executable, "-Xperf", script], + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=env, + ) as process: + stdout, stderr = process.communicate() + + self.assertEqual(process.returncode, 0) + self.assertNotIn("Error:", stderr) + child_pid = int(stdout.strip()) + perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") + perf_child_file = pathlib.Path(f"/tmp/perf-{child_pid}.map") + self.assertTrue(perf_file.exists()) + self.assertTrue(perf_child_file.exists()) + + perf_file_contents = perf_file.read_text() + self.assertIn(f"py::foo:{script}", perf_file_contents) + self.assertIn(f"py::bar:{script}", perf_file_contents) + self.assertIn(f"py::foo_fork:{script}", perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", perf_file_contents) + + child_perf_file_contents = perf_child_file.read_text() + self.assertIn(f"py::foo_fork:{script}", child_perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", child_perf_file_contents) + + # Pre-compiled perf-map entries of a forked process must be + # identical in both the parent and child perf-map files. + perf_file_lines = perf_file_contents.split("\n") + for line in perf_file_lines: + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: + self.assertIn(line, child_perf_file_contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 24a366efc6c..520fbc1b662 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -778,7 +778,7 @@ class X: class X: locals()["x"] = 43 del x - self.assertFalse(hasattr(X, "x")) + self.assertNotHasAttr(X, "x") self.assertEqual(x, 42) @cpython_only diff --git a/Lib/test/test_script_helper.py b/Lib/test/test_script_helper.py index f7871fd3b77..eeea6c4842b 100644 --- a/Lib/test/test_script_helper.py +++ b/Lib/test/test_script_helper.py @@ -74,8 +74,7 @@ class TestScriptHelperEnvironment(unittest.TestCase): """Code coverage for interpreter_requires_environment().""" def setUp(self): - self.assertTrue( - hasattr(script_helper, '__cached_interp_requires_environment')) + self.assertHasAttr(script_helper, '__cached_interp_requires_environment') # Reset the private cached state. script_helper.__dict__['__cached_interp_requires_environment'] = None diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index c01e323553d..c0df9507bd7 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -237,7 +237,7 @@ def test_pickling(self): if type(self.s) not in (set, frozenset): self.assertEqual(self.s.x, dup.x) self.assertEqual(self.s.z, dup.z) - self.assertFalse(hasattr(self.s, 'y')) + self.assertNotHasAttr(self.s, 'y') del self.s.x, self.s.z def test_iterator_pickling(self): @@ -876,8 +876,8 @@ def test_repr(self): def check_repr_against_values(self): text = repr(self.set) - self.assertTrue(text.startswith('{')) - self.assertTrue(text.endswith('}')) + self.assertStartsWith(text, '{') + self.assertEndsWith(text, '}') result = text[1:-1].split(', ') result.sort() diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index 0bb02ef11f6..6fc5bb74036 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -154,7 +154,7 @@ class SetComprehensionTest(unittest.TestCase): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 08c6562f2a2..64609ab9dd9 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,10 +1,11 @@ +import array import unittest import dbm import shelve import pickle import os -from test.support import os_helper +from test.support import import_helper, os_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator @@ -165,6 +166,239 @@ def test_default_protocol(self): with shelve.Shelf({}) as s: self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) + def test_custom_serializer_and_deserializer(self): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol): + if isinstance(obj, (bytes, bytearray, str)): + if protocol == 5: + return obj + return type(obj).__name__ + elif isinstance(obj, array.array): + return obj.tobytes() + raise TypeError(f"Unsupported type for serialization: {type(obj)}") + + def deserializer(data): + if isinstance(data, (bytes, bytearray, str)): + return data.decode("utf-8") + elif isinstance(data, array.array): + return array.array("b", data) + raise TypeError( + f"Unsupported type for deserialization: {type(data)}" + ) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto), shelve.open( + self.fn, + protocol=proto, + serializer=serializer, + deserializer=deserializer + ) as s: + bar = "bar" + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + + if proto == 5: + self.assertEqual(s["foo"], str(bar)) + self.assertEqual(s["bytes_data"], "Hello, world!") + self.assertEqual( + s["bytearray_data"], bytearray_data.decode() + ) + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + else: + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + + def test_custom_incomplete_serializer_and_deserializer(self): + dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with self.assertRaises(dbm_sqlite3.error): + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return data.decode("utf-8") + + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + + def serializer(obj, protocol=None): + return type(obj).__name__.encode("utf-8") + + def deserializer(data): + pass + + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertIsNone(s["foo"]) + + def test_custom_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol=None): + data = obj.__class__.__name__ + if protocol == 5: + data = str(len(data)) + return data.encode("utf-8") + + def deserializer(data): + return data.decode("utf-8") + + def type_name_len(obj): + return f"{(len(type(obj).__name__))}" + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto), shelve.BsdDbShelf( + berkeleydb.btopen(self.fn), + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + bar = "bar" + bytes_obj = b"Hello, world!" + bytearray_obj = bytearray(b"\x00\x01\x02\x03\x04") + arr_obj = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_obj + s["bytearray_data"] = bytearray_obj + s["array_data"] = arr_obj + + if proto == 5: + self.assertEqual(s["foo"], type_name_len(bar)) + self.assertEqual(s["bytes_data"], type_name_len(bytes_obj)) + self.assertEqual(s["bytearray_data"], + type_name_len(bytearray_obj)) + self.assertEqual(s["array_data"], type_name_len(arr_obj)) + + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, type_name_len(bar)) + + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) + + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) + else: + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, "str") + + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") + + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") + + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") + + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") + + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") + + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") + + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual(s["array_data"], "array") + + def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol=None): + return type(obj).__name__.encode("utf-8") + + def deserializer(data): + pass + + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertIsNone(s["foo"]) + self.assertNotEqual(s["foo"], "bar") + + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return data.decode("utf-8") + + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertEqual(s["foo"], "") + self.assertNotEqual(s["foo"], "bar") + + def test_missing_custom_deserializer(self): + def serializer(obj, protocol=None): + pass + + kwargs = dict(protocol=2, writeback=False, serializer=serializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) + + def test_missing_custom_serializer(self): + def deserializer(data): + pass + + kwargs = dict(protocol=2, writeback=False, deserializer=deserializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) + class TestShelveBase: type2test = shelve.Shelf diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index a13ddcb76b7..2a355abdeeb 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -330,6 +330,7 @@ def testQuote(self): unsafe = '"`$\\!' + unicode_sample self.assertEqual(shlex.quote(''), "''") + self.assertEqual(shlex.quote(None), "''") self.assertEqual(shlex.quote(safeunquoted), safeunquoted) self.assertEqual(shlex.quote('test file name'), "'test file name'") for u in unsafe: @@ -338,6 +339,8 @@ def testQuote(self): for u in unsafe: self.assertEqual(shlex.quote("test%s'name'" % u), "'test%s'\"'\"'name'\"'\"''" % u) + self.assertRaises(TypeError, shlex.quote, 42) + self.assertRaises(TypeError, shlex.quote, b"abc") def testJoin(self): for split_command, command in [ diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 87991fbda4c..ebb6cf88336 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -427,12 +427,12 @@ def check_args_to_onerror(self, func, arg, exc): else: self.assertIs(func, os.listdir) self.assertIn(arg, [TESTFN, self.child_dir_path]) - self.assertTrue(issubclass(exc[0], OSError)) + self.assertIsSubclass(exc[0], OSError) self.errorState += 1 else: self.assertEqual(func, os.rmdir) self.assertEqual(arg, TESTFN) - self.assertTrue(issubclass(exc[0], OSError)) + self.assertIsSubclass(exc[0], OSError) self.errorState = 3 @unittest.skipIf(sys.platform[:6] == 'cygwin', @@ -3479,7 +3479,7 @@ class PublicAPITests(unittest.TestCase): """Ensures that the correct values are exposed in the public API.""" def test_module_all_attribute(self): - self.assertTrue(hasattr(shutil, '__all__')) + self.assertHasAttr(shutil, '__all__') target_api = ['copyfileobj', 'copyfile', 'copymode', 'copystat', 'copy', 'copy2', 'copytree', 'move', 'rmtree', 'Error', 'SpecialFileError', 'make_archive', @@ -3492,7 +3492,7 @@ def test_module_all_attribute(self): target_api.append('disk_usage') self.assertEqual(set(shutil.__all__), set(target_api)) with self.assertWarns(DeprecationWarning): - from shutil import ExecError + from shutil import ExecError # noqa: F401 if __name__ == '__main__': diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 6d62d611925..d6cc22558ec 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -14,7 +14,7 @@ import unittest from test import support from test.support import ( - is_apple, is_apple_mobile, os_helper, threading_helper + force_not_colorized, is_apple, is_apple_mobile, os_helper, threading_helper ) from test.support.script_helper import assert_python_ok, spawn_python try: @@ -353,6 +353,7 @@ def check_signum(signals): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @force_not_colorized def test_wakeup_write_error(self): # Issue #16105: write() errors in the C signal handler should not # pass silently. diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index a7e9241f44d..27ae3539b55 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -13,6 +13,7 @@ from test.support import socket_helper from test.support import captured_stderr from test.support.os_helper import TESTFN, EnvironmentVarGuard +from test.support.script_helper import spawn_python, kill_python import ast import builtins import glob @@ -25,6 +26,7 @@ import sys import sysconfig import tempfile +from textwrap import dedent import urllib.error import urllib.request from unittest import mock @@ -307,8 +309,7 @@ def test_getuserbase(self): with EnvironmentVarGuard() as environ: environ['PYTHONUSERBASE'] = 'xoxo' - self.assertTrue(site.getuserbase().startswith('xoxo'), - site.getuserbase()) + self.assertStartsWith(site.getuserbase(), 'xoxo') @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_getusersitepackages(self): @@ -318,7 +319,7 @@ def test_getusersitepackages(self): # the call sets USER_BASE *and* USER_SITE self.assertEqual(site.USER_SITE, user_site) - self.assertTrue(user_site.startswith(site.USER_BASE), user_site) + self.assertStartsWith(user_site, site.USER_BASE) self.assertEqual(site.USER_BASE, site.getuserbase()) def test_getsitepackages(self): @@ -359,11 +360,10 @@ def test_no_home_directory(self): environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() - self.assertTrue(user_base.startswith('~' + os.sep), - user_base) + self.assertStartsWith(user_base, '~' + os.sep) user_site = site.getusersitepackages() - self.assertTrue(user_site.startswith(user_base), user_site) + self.assertStartsWith(user_site, user_base) with mock.patch('os.path.isdir', return_value=False) as mock_isdir, \ mock.patch.object(site, 'addsitedir') as mock_addsitedir, \ @@ -495,18 +495,18 @@ def test_add_build_dir(self): def test_setting_quit(self): # 'quit' and 'exit' should be injected into builtins - self.assertTrue(hasattr(builtins, "quit")) - self.assertTrue(hasattr(builtins, "exit")) + self.assertHasAttr(builtins, "quit") + self.assertHasAttr(builtins, "exit") def test_setting_copyright(self): # 'copyright', 'credits', and 'license' should be in builtins - self.assertTrue(hasattr(builtins, "copyright")) - self.assertTrue(hasattr(builtins, "credits")) - self.assertTrue(hasattr(builtins, "license")) + self.assertHasAttr(builtins, "copyright") + self.assertHasAttr(builtins, "credits") + self.assertHasAttr(builtins, "license") def test_setting_help(self): # 'help' should be set in builtins - self.assertTrue(hasattr(builtins, "help")) + self.assertHasAttr(builtins, "help") def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. @@ -804,6 +804,120 @@ def test_underpth_dll_file(self): )], env=env) self.assertTrue(rc, "sys.path is incorrect") + @support.requires_subprocess() + def test_underpth_no_user_site(self): + pth_lines = [test.support.STDLIB_DIR, 'import site'] + exe_file = self._create_underpth_exe(pth_lines) + p = subprocess.run([exe_file, '-X', 'utf8', '-c', + 'import sys; ' + 'sys.exit(not sys.flags.no_user_site)']) + self.assertEqual(p.returncode, 0, "sys.flags.no_user_site was 0") + + +class CommandLineTests(unittest.TestCase): + def exists(self, path): + if path is not None and os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + + def get_excepted_output(self, *args): + if len(args) == 0: + user_base = site.getuserbase() + user_site = site.getusersitepackages() + output = io.StringIO() + output.write("sys.path = [\n") + for dir in sys.path: + output.write(" %r,\n" % (dir,)) + output.write("]\n") + output.write(f"USER_BASE: {user_base} ({self.exists(user_base)})\n") + output.write(f"USER_SITE: {user_site} ({self.exists(user_site)})\n") + output.write(f"ENABLE_USER_SITE: {site.ENABLE_USER_SITE}\n") + return 0, dedent(output.getvalue()).strip() + + buffer = [] + if '--user-base' in args: + buffer.append(site.getuserbase()) + if '--user-site' in args: + buffer.append(site.getusersitepackages()) + + if buffer: + return_code = 3 + if site.ENABLE_USER_SITE: + return_code = 0 + elif site.ENABLE_USER_SITE is False: + return_code = 1 + elif site.ENABLE_USER_SITE is None: + return_code = 2 + output = os.pathsep.join(buffer) + return return_code, os.path.normpath(dedent(output).strip()) + else: + return 10, None + + def invoke_command_line(self, *args): + cmd_args = [] + if sys.flags.no_user_site: + cmd_args.append("-s") + cmd_args.extend(["-m", "site", *args]) + + with EnvironmentVarGuard() as env: + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + proc = spawn_python(*cmd_args, text=True, env=env, + encoding='utf-8', errors='replace') + + output = kill_python(proc) + return_code = proc.returncode + return return_code, os.path.normpath(dedent(output).strip()) + + @support.requires_subprocess() + def test_no_args(self): + return_code, output = self.invoke_command_line() + excepted_return_code, _ = self.get_excepted_output() + self.assertEqual(return_code, excepted_return_code) + lines = output.splitlines() + self.assertEqual(lines[0], "sys.path = [") + self.assertEqual(lines[-4], "]") + excepted_base = f"USER_BASE: '{site.getuserbase()}'" +\ + f" ({self.exists(site.getuserbase())})" + self.assertEqual(lines[-3], excepted_base) + excepted_site = f"USER_SITE: '{site.getusersitepackages()}'" +\ + f" ({self.exists(site.getusersitepackages())})" + self.assertEqual(lines[-2], excepted_site) + self.assertEqual(lines[-1], f"ENABLE_USER_SITE: {site.ENABLE_USER_SITE}") + + @support.requires_subprocess() + def test_unknown_args(self): + return_code, output = self.invoke_command_line("--unknown-arg") + excepted_return_code, _ = self.get_excepted_output("--unknown-arg") + self.assertEqual(return_code, excepted_return_code) + self.assertIn('[--user-base] [--user-site]', output) + + @support.requires_subprocess() + def test_base_arg(self): + return_code, output = self.invoke_command_line("--user-base") + excepted = self.get_excepted_output("--user-base") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + @support.requires_subprocess() + def test_site_arg(self): + return_code, output = self.invoke_command_line("--user-site") + excepted = self.get_excepted_output("--user-site") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + @support.requires_subprocess() + def test_both_args(self): + return_code, output = self.invoke_command_line("--user-base", + "--user-site") + excepted = self.get_excepted_output("--user-base", "--user-site") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 4c9fc14bd43..b8aac8c2020 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -17,6 +17,7 @@ import threading import unittest +import unittest.mock as mock from test import support, mock_socket from test.support import hashlib_helper from test.support import socket_helper @@ -926,11 +927,14 @@ def _auth_cram_md5(self, arg=None): except ValueError as e: self.push('535 Splitting response {!r} into user and password ' 'failed: {}'.format(logpass, e)) - return False - valid_hashed_pass = hmac.HMAC( - sim_auth[1].encode('ascii'), - self._decode_base64(sim_cram_md5_challenge).encode('ascii'), - 'md5').hexdigest() + return + pwd = sim_auth[1].encode('ascii') + msg = self._decode_base64(sim_cram_md5_challenge).encode('ascii') + try: + valid_hashed_pass = hmac.HMAC(pwd, msg, 'md5').hexdigest() + except ValueError: + self.push('504 CRAM-MD5 is not supported') + return self._authenticated(user, hashed_pass == valid_hashed_pass) # end AUTH related stuff. @@ -1181,6 +1185,39 @@ def testAUTH_CRAM_MD5(self): self.assertEqual(resp, (235, b'Authentication Succeeded')) smtp.close() + @hashlib_helper.block_algorithm('md5') + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked(self): + # CRAM-MD5 is the only "known" method by the server, + # but it is not supported by the client. In particular, + # no challenge will ever be sent. + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + msg = re.escape("No suitable authentication method found.") + with self.assertRaisesRegex(smtplib.SMTPException, msg): + smtp.login(sim_auth[0], sim_auth[1]) + + @hashlib_helper.block_algorithm('md5') + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked_and_fallback(self): + # Test that PLAIN is tried after CRAM-MD5 failed + self.serv.add_feature("AUTH CRAM-MD5 PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with ( + mock.patch.object(smtp, "auth_cram_md5") as smtp_auth_cram_md5, + mock.patch.object( + smtp, "auth_plain", wraps=smtp.auth_plain + ) as smtp_auth_plain + ): + resp = smtp.login(sim_auth[0], sim_auth[1]) + smtp_auth_plain.assert_called_once() + smtp_auth_cram_md5.assert_not_called() + self.assertEqual(resp, (235, b'Authentication Succeeded')) + @hashlib_helper.requires_hashdigest('md5', openssl=True) def testAUTH_multiple(self): # Test that multiple authentication methods are tried. diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 03c54151a22..934b7137096 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -8,7 +8,9 @@ import _thread as thread import array import contextlib +import decimal import errno +import fractions import gc import io import itertools @@ -52,6 +54,7 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" +SOLARIS = sys.platform.startswith("sunos") WSL = "microsoft-standard-WSL" in platform.release() try: @@ -1085,9 +1088,7 @@ def test3542SocketOptions(self): 'IPV6_USE_MIN_MTU', } for opt in opts: - self.assertTrue( - hasattr(socket, opt), f"Missing RFC3542 socket option '{opt}'" - ) + self.assertHasAttr(socket, opt) def testHostnameRes(self): # Testing hostname resolution mechanisms @@ -1175,7 +1176,10 @@ def testInterfaceNameIndex(self): 'socket.if_indextoname() not available.') @support.skip_android_selinux('if_indextoname') def testInvalidInterfaceIndexToName(self): - self.assertRaises(OSError, socket.if_indextoname, 0) + with self.assertRaises(OSError) as cm: + socket.if_indextoname(0) + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(ValueError, socket.if_indextoname, -1) self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') @@ -1195,8 +1199,11 @@ def testInvalidInterfaceIndexToName(self): 'socket.if_nametoindex() not available.') @support.skip_android_selinux('if_nametoindex') def testInvalidInterfaceNameToIndex(self): + with self.assertRaises(OSError) as cm: + socket.if_nametoindex("_DEADBEEF") + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(TypeError, socket.if_nametoindex, 0) - self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') @unittest.skipUnless(hasattr(sys, 'getrefcount'), 'test needs sys.getrefcount()') @@ -1311,10 +1318,20 @@ def testDefaultTimeout(self): self.assertEqual(s.gettimeout(), None) # Set the default timeout to 10, and see if it propagates - with socket_setdefaulttimeout(10): - self.assertEqual(socket.getdefaulttimeout(), 10) + with socket_setdefaulttimeout(10.125): + self.assertEqual(socket.getdefaulttimeout(), 10.125) with socket.socket() as sock: - self.assertEqual(sock.gettimeout(), 10) + self.assertEqual(sock.gettimeout(), 10.125) + + socket.setdefaulttimeout(decimal.Decimal('11.125')) + self.assertEqual(socket.getdefaulttimeout(), 11.125) + with socket.socket() as sock: + self.assertEqual(sock.gettimeout(), 11.125) + + socket.setdefaulttimeout(fractions.Fraction(97, 8)) + self.assertEqual(socket.getdefaulttimeout(), 12.125) + with socket.socket() as sock: + self.assertEqual(sock.gettimeout(), 12.125) # Reset the default timeout to None, and see if it propagates socket.setdefaulttimeout(None) @@ -1540,6 +1557,40 @@ def testSetSockOpt(self): reuse = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) self.assertFalse(reuse == 0, "failed to set reuse mode") + def test_setsockopt_errors(self): + # See issue #107546. + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # No error expected. + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 2 ** 100) + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, - 2 ** 100) + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, 2 ** 100, 1) + + with self.assertRaises(OverflowError): + sock.setsockopt(2 ** 100, socket.SO_REUSEADDR, 1) + + with self.assertRaisesRegex(TypeError, "socket option should be int, bytes-like object or None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, dict()) + + with self.assertRaisesRegex(TypeError, "requires 4 arguments when the third argument is None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, None) + + with self.assertRaisesRegex(TypeError, "only takes 4 arguments when the third argument is None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1, 2) + + with self.assertRaisesRegex(TypeError, "takes at least 3 arguments"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) + + with self.assertRaisesRegex(TypeError, "takes at most 4 arguments"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1, 2, 3) + def testSendAfterClose(self): # testing send() after close() with timeout with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: @@ -1593,11 +1644,11 @@ def test_getsockaddrarg(self): @unittest.skipUnless(os.name == "nt", "Windows specific") def test_sock_ioctl(self): - self.assertTrue(hasattr(socket.socket, 'ioctl')) - self.assertTrue(hasattr(socket, 'SIO_RCVALL')) - self.assertTrue(hasattr(socket, 'RCVALL_ON')) - self.assertTrue(hasattr(socket, 'RCVALL_OFF')) - self.assertTrue(hasattr(socket, 'SIO_KEEPALIVE_VALS')) + self.assertHasAttr(socket.socket, 'ioctl') + self.assertHasAttr(socket, 'SIO_RCVALL') + self.assertHasAttr(socket, 'RCVALL_ON') + self.assertHasAttr(socket, 'RCVALL_OFF') + self.assertHasAttr(socket, 'SIO_KEEPALIVE_VALS') s = socket.socket() self.addCleanup(s.close) self.assertRaises(ValueError, s.ioctl, -1, None) @@ -2360,6 +2411,45 @@ def testCrucialConstants(self): socket.CAN_ISOTP socket.SOCK_DGRAM + @unittest.skipUnless(hasattr(socket, "SOL_CAN_ISOTP"), + "missing <linux/can/isotp.h>") + def testISOTP(self): + socket.SOL_CAN_ISOTP + + socket.CAN_ISOTP_OPTS + socket.CAN_ISOTP_RECV_FC + + socket.CAN_ISOTP_TX_STMIN + socket.CAN_ISOTP_RX_STMIN + socket.CAN_ISOTP_LL_OPTS + + socket.CAN_ISOTP_LISTEN_MODE + socket.CAN_ISOTP_EXTEND_ADDR + socket.CAN_ISOTP_TX_PADDING + socket.CAN_ISOTP_RX_PADDING + socket.CAN_ISOTP_CHK_PAD_LEN + socket.CAN_ISOTP_CHK_PAD_DATA + socket.CAN_ISOTP_HALF_DUPLEX + socket.CAN_ISOTP_FORCE_TXSTMIN + socket.CAN_ISOTP_FORCE_RXSTMIN + socket.CAN_ISOTP_RX_EXT_ADDR + socket.CAN_ISOTP_WAIT_TX_DONE + # This constant is not always available + # socket.CAN_ISOTP_SF_BROADCAST + + socket.CAN_ISOTP_DEFAULT_FLAGS + socket.CAN_ISOTP_DEFAULT_EXT_ADDRESS + socket.CAN_ISOTP_DEFAULT_PAD_CONTENT + socket.CAN_ISOTP_DEFAULT_FRAME_TXTIME + socket.CAN_ISOTP_DEFAULT_RECV_BS + socket.CAN_ISOTP_DEFAULT_EXT_ADDRESS + socket.CAN_ISOTP_DEFAULT_RECV_STMIN + socket.CAN_ISOTP_DEFAULT_RECV_WFTMAX + + socket.CAN_ISOTP_DEFAULT_LL_MTU + socket.CAN_ISOTP_DEFAULT_LL_TX_DL + socket.CAN_ISOTP_DEFAULT_LL_TX_FLAGS + def testCreateSocket(self): with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: pass @@ -3890,6 +3980,10 @@ def testCMSG_SPACE(self): # Test CMSG_SPACE() with various valid and invalid values, # checking the assumptions used by sendmsg(). toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + if SOLARIS and platform.processor() == "sparc": + # On Solaris SPARC, number of bytes returned by socket.CMSG_SPACE + # increases at different lengths; see gh-91214. + toobig -= 3 values = list(range(257)) + list(range(toobig - 257, toobig)) last = socket.CMSG_SPACE(0) @@ -4036,6 +4130,7 @@ def _testFDPassCMSG_LEN(self): self.createAndSendFDs(1) @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): @@ -4047,6 +4142,7 @@ def testFDPassSeparate(self): @testFDPassSeparate.client_skip @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) @@ -4060,6 +4156,7 @@ def _testFDPassSeparate(self): len(MSG)) @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): @@ -4074,6 +4171,7 @@ def testFDPassSeparateMinSpace(self): @testFDPassSeparateMinSpace.client_skip @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) @@ -6082,10 +6180,10 @@ def testTimeoutZero(self): class TestExceptions(unittest.TestCase): def testExceptionTree(self): - self.assertTrue(issubclass(OSError, Exception)) - self.assertTrue(issubclass(socket.herror, OSError)) - self.assertTrue(issubclass(socket.gaierror, OSError)) - self.assertTrue(issubclass(socket.timeout, OSError)) + self.assertIsSubclass(OSError, Exception) + self.assertIsSubclass(socket.herror, OSError) + self.assertIsSubclass(socket.gaierror, OSError) + self.assertIsSubclass(socket.timeout, OSError) self.assertIs(socket.error, OSError) self.assertIs(socket.timeout, TimeoutError) @@ -7047,8 +7145,14 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 9) # see issue29324 + @support.requires_linux_version(4, 9) # see gh-73510 def test_aead_aes_gcm(self): + kernel_version = support._get_kernel_version("Linux") + if kernel_version is not None: + if kernel_version >= (6, 16) and kernel_version < (6, 18): + # See https://github.com/python/cpython/issues/139310. + self.skipTest("upstream Linux kernel issue") + key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 0f62f9eb200..ca33a9a6dac 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -17,6 +17,7 @@ from test.support import os_helper from test.support import socket_helper from test.support import threading_helper +from test.support import warnings_helper test.support.requires("network") @@ -43,6 +44,7 @@ def receive(sock, n, timeout=test.support.SHORT_TIMEOUT): raise RuntimeError("timed out on %r" % (sock,)) +@warnings_helper.ignore_fork_in_thread_deprecation_warnings() @test.support.requires_fork() @contextlib.contextmanager def simple_subprocess(testcase): @@ -173,6 +175,7 @@ def test_ThreadingTCPServer(self): socketserver.StreamRequestHandler, self.stream_examine) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_forking def test_ForkingTCPServer(self): with simple_subprocess(self): @@ -192,6 +195,7 @@ def test_ThreadingUnixStreamServer(self): socketserver.StreamRequestHandler, self.stream_examine) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_unix_sockets @requires_forking def test_ForkingUnixStreamServer(self): @@ -210,6 +214,7 @@ def test_ThreadingUDPServer(self): socketserver.DatagramRequestHandler, self.dgram_examine) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_forking def test_ForkingUDPServer(self): with simple_subprocess(self): @@ -218,17 +223,22 @@ def test_ForkingUDPServer(self): self.dgram_examine) @requires_unix_sockets + @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, + "gh-140702: Test fails regularly on iOS simulator on GitHub Actions") def test_UnixDatagramServer(self): self.run_server(socketserver.UnixDatagramServer, socketserver.DatagramRequestHandler, self.dgram_examine) @requires_unix_sockets + @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, + "gh-140702: Test fails regularly on iOS simulator on GitHub Actions") def test_ThreadingUnixDatagramServer(self): self.run_server(socketserver.ThreadingUnixDatagramServer, socketserver.DatagramRequestHandler, self.dgram_examine) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_unix_sockets @requires_forking def test_ForkingUnixDatagramServer(self): @@ -314,11 +324,13 @@ def test_threading_not_handled(self): self.assertIs(cm.exc_type, SystemExit) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_forking def test_forking_handled(self): ForkingErrorTestServer(ValueError) self.check_result(handled=True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @requires_forking def test_forking_not_handled(self): ForkingErrorTestServer(SystemExit) @@ -503,5 +515,15 @@ class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer): server.server_close() +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(socketserver, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 61b00778f83..46b291192df 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import unittest -from test.support import script_helper, captured_stdout, requires_subprocess, requires_resource +from test import support +from test.support import script_helper from test.support.os_helper import TESTFN, unlink, rmtree from test.support.import_helper import unload import importlib @@ -64,7 +65,7 @@ def test_issue7820(self): # two bytes in common with the UTF-8 BOM self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20') - @requires_subprocess() + @support.requires_subprocess() def test_20731(self): sub = subprocess.Popen([sys.executable, os.path.join(os.path.dirname(__file__), @@ -145,8 +146,7 @@ def test_error_from_string(self): compile(input, "<string>", "exec") expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \ "ordinal not in range(128)" - self.assertTrue(c.exception.args[0].startswith(expected), - msg=c.exception.args[0]) + self.assertStartsWith(c.exception.args[0], expected) def test_file_parse_error_multiline(self): # gh96611: @@ -173,6 +173,8 @@ def test_tokenizer_fstring_warning_in_first_line(self): os.unlink(TESTFN) +BUFSIZ = 2**13 + class AbstractSourceEncodingTest: def test_default_coding(self): @@ -185,14 +187,20 @@ def test_first_coding_line(self): self.check_script_output(src, br"'\xc3\u20ac'") def test_second_coding_line(self): - src = (b'#\n' + src = (b'#!/usr/bin/python\n' + b'#coding:iso8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_second_coding_line_empty_first_line(self): + src = (b'\n' b'#coding:iso8859-15\n' b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") def test_third_coding_line(self): # Only first two lines are tested for a magic comment. - src = (b'#\n' + src = (b'#!/usr/bin/python\n' b'#\n' b'#coding:iso8859-15\n' b'print(ascii("\xc3\xa4"))\n') @@ -210,48 +218,197 @@ def test_double_coding_same_line(self): b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") + def test_double_coding_utf8(self): + src = (b'#coding:utf-8\n' + b'#coding:latin1\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xe4'") + + def test_long_first_coding_line(self): + src = (b'#' + b' '*BUFSIZ + b'coding:iso8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_long_second_coding_line(self): + src = (b'#!/usr/bin/python\n' + b'#' + b' '*BUFSIZ + b'coding:iso8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_long_coding_line(self): + src = (b'#coding:iso-8859-15' + b' '*BUFSIZ + b'\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_long_coding_name(self): + src = (b'#coding:iso-8859-1-' + b'x'*BUFSIZ + b'\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\xa4'") + + def test_long_first_utf8_line(self): + src = b'#' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n' + self.check_script_output(src, b'') + src = b'# ' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n' + self.check_script_output(src, b'') + + def test_long_second_utf8_line(self): + src = b'\n#' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n' + self.check_script_output(src, b'') + src = b'\n# ' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n' + self.check_script_output(src, b'') + def test_first_non_utf8_coding_line(self): src = (b'#coding:iso-8859-15 \xa4\n' b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") def test_second_non_utf8_coding_line(self): - src = (b'\n' + src = (b'#!/usr/bin/python\n' b'#coding:iso-8859-15 \xa4\n' b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") + def test_first_utf8_coding_line_error(self): + src = (b'#coding:ascii \xc3\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + + def test_second_utf8_coding_line_error(self): + src = (b'#!/usr/bin/python\n' + b'#coding:ascii \xc3\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + def test_utf8_bom(self): src = (b'\xef\xbb\xbfprint(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xe4'") + def test_utf8_bom_utf8_comments(self): + src = (b'\xef\xbb\xbf#\xc3\xa4\n' + b'#\xc3\xa4\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xe4'") + def test_utf8_bom_and_utf8_coding_line(self): src = (b'\xef\xbb\xbf#coding:utf-8\n' b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xe4'") + def test_utf8_bom_and_non_utf8_first_coding_line(self): + src = (b'\xef\xbb\xbf#coding:iso-8859-15\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"encoding problem: iso-8859-15 with BOM", + lineno=1) + + def test_utf8_bom_and_non_utf8_second_coding_line(self): + src = (b'\xef\xbb\xbf#first\n' + b'#coding:iso-8859-15\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"encoding problem: iso-8859-15 with BOM", + lineno=2) + + def test_non_utf8_shebang(self): + src = (b'#!/home/\xa4/bin/python\n' + b'#coding:iso-8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_utf8_shebang_error(self): + src = (b'#!/home/\xc3\xa4/bin/python\n' + b'#coding:ascii\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + + def test_non_utf8_shebang_error(self): + src = (b'#!/home/\xa4/bin/python\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"Non-UTF-8 code starting with .* on line 1", + lineno=1) + + def test_non_utf8_second_line_error(self): + src = (b'#first\n' + b'#second\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 2", + lineno=2) + + def test_non_utf8_third_line_error(self): + src = (b'#first\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3", + lineno=3) + + def test_utf8_bom_non_utf8_third_line_error(self): + src = (b'\xef\xbb\xbf#first\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3|" + br"'utf-8' codec can't decode byte", + lineno=3) + + def test_utf_8_non_utf8_third_line_error(self): + src = (b'#coding: utf-8\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3|" + br"'utf-8' codec can't decode byte", + lineno=3) + + def test_utf8_non_utf8_third_line_error(self): + src = (b'#coding: utf8\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"'utf-8' codec can't decode byte|" + br"encoding problem: utf8") + def test_crlf(self): src = (b'print(ascii("""\r\n"""))\n') - out = self.check_script_output(src, br"'\n'") + self.check_script_output(src, br"'\n'") def test_crcrlf(self): src = (b'print(ascii("""\r\r\n"""))\n') - out = self.check_script_output(src, br"'\n\n'") + self.check_script_output(src, br"'\n\n'") def test_crcrcrlf(self): src = (b'print(ascii("""\r\r\r\n"""))\n') - out = self.check_script_output(src, br"'\n\n\n'") + self.check_script_output(src, br"'\n\n\n'") def test_crcrcrlf2(self): src = (b'#coding:iso-8859-1\n' b'print(ascii("""\r\r\r\n"""))\n') - out = self.check_script_output(src, br"'\n\n\n'") + self.check_script_output(src, br"'\n\n\n'") + + def test_nul_in_first_coding_line(self): + src = (b'#coding:iso8859-15\x00\n' + b'\n' + b'\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"source code (string )?cannot contain null bytes") + + def test_nul_in_second_coding_line(self): + src = (b'#!/usr/bin/python\n' + b'#coding:iso8859-15\x00\n' + b'\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"source code (string )?cannot contain null bytes") class UTF8ValidatorTest(unittest.TestCase): @unittest.skipIf(not sys.platform.startswith("linux"), "Too slow to run on non-Linux platforms") - @requires_resource('cpu') + @support.requires_resource('cpu') def test_invalid_utf8(self): # This is a port of test_utf8_decode_invalid_sequences in # test_unicode.py to exercise the separate utf8 validator in @@ -317,15 +474,29 @@ def check(content): check(b'\xF4'+cb+b'\xBF\xBF') +@support.force_not_colorized_test_class class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase): def check_script_output(self, src, expected): - with captured_stdout() as stdout: + with support.captured_stdout() as stdout: exec(src) out = stdout.getvalue().encode('latin1') self.assertEqual(out.rstrip(), expected) + def check_script_error(self, src, expected, lineno=...): + with self.assertRaises(SyntaxError) as cm: + exec(src) + exc = cm.exception + self.assertRegex(str(exc), expected.decode()) + if lineno is not ...: + self.assertEqual(exc.lineno, lineno) + line = src.splitlines()[lineno-1].decode(errors='replace') + if lineno == 1: + line = line.removeprefix('\ufeff') + self.assertEqual(line, exc.text) + +@support.force_not_colorized_test_class class FileSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase): def check_script_output(self, src, expected): @@ -336,6 +507,23 @@ def check_script_output(self, src, expected): res = script_helper.assert_python_ok(fn) self.assertEqual(res.out.rstrip(), expected) + def check_script_error(self, src, expected, lineno=...): + with tempfile.TemporaryDirectory() as tmpd: + fn = os.path.join(tmpd, 'test.py') + with open(fn, 'wb') as fp: + fp.write(src) + res = script_helper.assert_python_failure(fn) + err = res.err.rstrip() + self.assertRegex(err.splitlines()[-1], b'SyntaxError: ' + expected) + if lineno is not ...: + self.assertIn(f', line {lineno}\n'.encode(), + err.replace(os.linesep.encode(), b'\n')) + line = src.splitlines()[lineno-1].decode(errors='replace') + if lineno == 1: + line = line.removeprefix('\ufeff') + self.assertIn(line.encode(), err) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/__init__.py b/Lib/test/test_sqlite3/__init__.py index d777fca82da..78a1e2078a5 100644 --- a/Lib/test/test_sqlite3/__init__.py +++ b/Lib/test/test_sqlite3/__init__.py @@ -8,8 +8,7 @@ # Implement the unittest "load tests" protocol. def load_tests(*args): + if verbose: + print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") pkg_dir = os.path.dirname(__file__) return load_package_tests(pkg_dir, *args) - -if verbose: - print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index ad0dcb3cccb..98aadaa829a 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -1,17 +1,26 @@ """sqlite3 CLI tests.""" import sqlite3 +import sys +import textwrap import unittest +import unittest.mock +import os from sqlite3.__main__ import main as cli +from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink +from test.support.pty_helper import run_pty from test.support import ( captured_stdout, captured_stderr, captured_stdin, - force_not_colorized, + force_not_colorized_test_class, + requires_subprocess, + verbose, ) +@force_not_colorized_test_class class CommandLineInterface(unittest.TestCase): def _do_test(self, *args, expect_success=True): @@ -37,7 +46,6 @@ def expect_failure(self, *args): self.assertEqual(out, "") return err - @force_not_colorized def test_cli_help(self): out = self.expect_success("-h") self.assertIn("usage: ", out) @@ -69,6 +77,7 @@ def test_cli_on_disk_db(self): self.assertIn("(0,)", out) +@force_not_colorized_test_class class InteractiveSession(unittest.TestCase): MEMORY_DB_MSG = "Connected to a transient in-memory database" PS1 = "sqlite> " @@ -116,6 +125,38 @@ def test_interact_version(self): self.assertEqual(out.count(self.PS2), 0) self.assertIn(sqlite3.sqlite_version, out) + def test_interact_empty_source(self): + out, err = self.run_cli(commands=("", " ")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 3) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_dot_commands_unknown(self): + out, err = self.run_cli(commands=(".unknown_command", )) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + self.assertIn('Error: unknown command: "', err) + # test "unknown_command" is pointed out in the error message + self.assertIn("unknown_command", err) + + def test_interact_dot_commands_empty(self): + out, err = self.run_cli(commands=(".")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_dot_commands_with_whitespaces(self): + out, err = self.run_cli(commands=(".version ", ". version")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEqual(out.count(sqlite3.sqlite_version + "\n"), 2) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 3) + self.assertEqual(out.count(self.PS2), 0) + def test_interact_valid_sql(self): out, err = self.run_cli(commands=("SELECT 1;",)) self.assertIn(self.MEMORY_DB_MSG, err) @@ -158,6 +199,240 @@ def test_interact_on_disk_file(self): out, _ = self.run_cli(TESTFN, commands=("SELECT count(t) FROM t;",)) self.assertIn("(0,)\n", out) + def test_color(self): + with unittest.mock.patch("_colorize.can_colorize", return_value=True): + out, err = self.run_cli(commands="TEXT\n") + self.assertIn("\x1b[1;35msqlite> \x1b[0m", out) + self.assertIn("\x1b[1;35m ... \x1b[0m\x1b", out) + out, err = self.run_cli(commands=("sel;",)) + self.assertIn('\x1b[1;35mOperationalError (SQLITE_ERROR)\x1b[0m: ' + '\x1b[35mnear "sel": syntax error\x1b[0m', err) + + +@requires_subprocess() +@force_not_colorized_test_class +class Completion(unittest.TestCase): + PS1 = "sqlite> " + + @classmethod + def setUpClass(cls): + readline = import_module("readline") + if readline.backend == "editline": + raise unittest.SkipTest("libedit readline is not supported") + + def write_input(self, input_, env=None): + script = textwrap.dedent(""" + import readline + from sqlite3.__main__ import main + + # Configure readline to ...: + # - hide control sequences surrounding each candidate + # - hide "Display all xxx possibilities? (y or n)" + # - show candidates one per line + readline.parse_and_bind("set colored-completion-prefix off") + readline.parse_and_bind("set completion-query-items 0") + readline.parse_and_bind("set page-completions off") + readline.parse_and_bind("set completion-display-width 0") + + main() + """) + return run_pty(script, input_, env) + + def test_complete_sql_keywords(self): + _sqlite3 = import_module("_sqlite3") + if not hasattr(_sqlite3, "SQLITE_KEYWORDS"): + raise unittest.SkipTest("unable to determine SQLite keywords") + + # List candidates starting with 'S', there should be multiple matches. + input_ = b"S\t\tEL\t 1;\n.quit\n" + output = self.write_input(input_) + self.assertIn(b"SELECT", output) + self.assertIn(b"SET", output) + self.assertIn(b"SAVEPOINT", output) + self.assertIn(b"(1,)", output) + + # Keywords are completed in upper case for even lower case user input. + input_ = b"sel\t\t 1;\n.quit\n" + output = self.write_input(input_) + self.assertIn(b"SELECT", output) + self.assertIn(b"(1,)", output) + + # .commands are completed without changing case + input_ = b".ver\t\n.quit\n" + output = self.write_input(input_) + self.assertIn(b".version", output) + + def test_complete_table_indexes_triggers_views(self): + input_ = textwrap.dedent("""\ + CREATE TABLE _Table (id); + CREATE INDEX _Index ON _table (id); + CREATE TRIGGER _Trigger BEFORE INSERT + ON _Table BEGIN SELECT 1; END; + CREATE VIEW _View AS SELECT 1; + + CREATE TEMP TABLE _Temp_table (id); + CREATE INDEX temp._Temp_index ON _Temp_table (id); + CREATE TEMP TRIGGER _Temp_trigger BEFORE INSERT + ON _Table BEGIN SELECT 1; END; + CREATE TEMP VIEW _Temp_view AS SELECT 1; + + ATTACH ':memory:' AS attached; + CREATE TABLE attached._Attached_table (id); + CREATE INDEX attached._Attached_index ON _Attached_table (id); + CREATE TRIGGER attached._Attached_trigger BEFORE INSERT + ON _Attached_table BEGIN SELECT 1; END; + CREATE VIEW attached._Attached_view AS SELECT 1; + + SELECT id FROM _\t\tta\t; + .quit\n""").encode() + output = self.write_input(input_) + lines = output.decode().splitlines() + indices = [i for i, line in enumerate(lines) + if line.startswith(self.PS1)] + start, end = indices[-3], indices[-2] + candidates = [l.strip() for l in lines[start+1:end]] + self.assertEqual(candidates, + [ + "_Attached_index", + "_Attached_table", + "_Attached_trigger", + "_Attached_view", + "_Index", + "_Table", + "_Temp_index", + "_Temp_table", + "_Temp_trigger", + "_Temp_view", + "_Trigger", + "_View", + ], + ) + start, end = indices[-2], indices[-1] + # direct match with '_Table' completed, no candidates displayed + candidates = [l.strip() for l in lines[start+1:end]] + self.assertEqual(len(candidates), 0) + + @unittest.skipIf(sqlite3.sqlite_version_info < (3, 16, 0), + "PRAGMA table-valued function is not available until " + "SQLite 3.16.0") + def test_complete_columns(self): + input_ = textwrap.dedent("""\ + CREATE TABLE _table (_col_table); + CREATE TEMP TABLE _temp_table (_col_temp); + ATTACH ':memory:' AS attached; + CREATE TABLE attached._attached_table (_col_attached); + + SELECT _col_\t\tta\tFROM _table; + .quit\n""").encode() + output = self.write_input(input_) + lines = output.decode().splitlines() + indices = [ + i for i, line in enumerate(lines) if line.startswith(self.PS1) + ] + start, end = indices[-3], indices[-2] + candidates = [l.strip() for l in lines[start+1:end]] + + self.assertEqual( + candidates, ["_col_attached", "_col_table", "_col_temp"] + ) + + @unittest.skipIf(sqlite3.sqlite_version_info < (3, 30, 0), + "PRAGMA function_list is not available until " + "SQLite 3.30.0") + def test_complete_functions(self): + input_ = b"SELECT AV\t1);\n.quit\n" + output = self.write_input(input_) + self.assertIn(b"AVG(1);", output) + self.assertIn(b"(1.0,)", output) + + # Functions are completed in upper case for even lower case user input. + input_ = b"SELECT av\t1);\n.quit\n" + output = self.write_input(input_) + self.assertIn(b"AVG(1);", output) + self.assertIn(b"(1.0,)", output) + + def test_complete_schemata(self): + input_ = textwrap.dedent("""\ + ATTACH ':memory:' AS MixedCase; + -- Test '_' is escaped in Like pattern filtering + ATTACH ':memory:' AS _underscore; + -- Let database_list pragma have a 'temp' schema entry + CREATE TEMP TABLE _table (id); + + SELECT * FROM \t\tmIX\t.sqlite_master; + SELECT * FROM _und\t.sqlite_master; + .quit\n""").encode() + output = self.write_input(input_) + lines = output.decode().splitlines() + indices = [ + i for i, line in enumerate(lines) if line.startswith(self.PS1) + ] + start, end = indices[-4], indices[-3] + candidates = [l.strip() for l in lines[start+1:end]] + self.assertIn("MixedCase", candidates) + self.assertIn("_underscore", candidates) + self.assertIn("main", candidates) + self.assertIn("temp", candidates) + + @unittest.skipIf(sys.platform.startswith("freebsd"), + "Two actual tabs are inserted when there are no matching" + " completions in the pseudo-terminal opened by run_pty()" + " on FreeBSD") + def test_complete_no_match(self): + input_ = b"xyzzy\t\t\b\b\b\b\b\b\b.quit\n" + # Set NO_COLOR to disable coloring for self.PS1. + output = self.write_input(input_, env={**os.environ, "NO_COLOR": "1"}) + lines = output.decode().splitlines() + indices = ( + i for i, line in enumerate(lines, 1) + if line.startswith(f"{self.PS1}xyzzy") + ) + line_num = next(indices, -1) + self.assertNotEqual(line_num, -1) + # Completions occupy lines, assert no extra lines when there is nothing + # to complete. + self.assertEqual(line_num, len(lines)) + + def test_complete_no_input(self): + script = textwrap.dedent(""" + import readline + from sqlite3.__main__ import main + + # Configure readline to ...: + # - hide control sequences surrounding each candidate + # - hide "Display all xxx possibilities? (y or n)" + # - hide "--More--" + # - show candidates one per line + readline.parse_and_bind("set colored-completion-prefix off") + readline.parse_and_bind("set colored-stats off") + readline.parse_and_bind("set completion-query-items 0") + readline.parse_and_bind("set page-completions off") + readline.parse_and_bind("set completion-display-width 0") + readline.parse_and_bind("set show-all-if-ambiguous off") + readline.parse_and_bind("set show-all-if-unmodified off") + + main() + """) + input_ = b"\t\t.quit\n" + output = run_pty(script, input_, env={**os.environ, "NO_COLOR": "1"}) + try: + lines = output.decode().splitlines() + indices = [ + i for i, line in enumerate(lines) + if line.startswith(self.PS1) + ] + self.assertEqual(len(indices), 2) + start, end = indices + candidates = [l.strip() for l in lines[start+1:end]] + self.assertEqual(candidates, sorted(candidates)) + except: + if verbose: + print(' PTY output: '.center(30, '-')) + print(output.decode(errors='replace')) + print(' end PTY output '.center(30, '-')) + raise + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index c3aa3bf2d7b..20e39f61e4d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -21,6 +21,7 @@ # 3. This notice may not be removed or altered from any source distribution. import contextlib +import functools import os import sqlite3 as sqlite import subprocess @@ -31,8 +32,7 @@ import warnings from test.support import ( - SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, - is_apple, is_emscripten, is_wasi + SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess ) from test.support import gc_collect from test.support import threading_helper, import_helper @@ -550,17 +550,9 @@ def test_connection_config(self): cx.execute("insert into u values(0)") def test_connect_positional_arguments(self): - regex = ( - r"Passing more than 1 positional argument to sqlite3.connect\(\)" - " is deprecated. Parameters 'timeout', 'detect_types', " - "'isolation_level', 'check_same_thread', 'factory', " - "'cached_statements' and 'uri' will become keyword-only " - "parameters in Python 3.15." - ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - cx = sqlite.connect(":memory:", 1.0) - cx.close() - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(TypeError, + r'connect\(\) takes at most 1 positional arguments'): + sqlite.connect(":memory:", 1.0) def test_connection_resource_warning(self): with self.assertWarns(ResourceWarning): @@ -639,6 +631,14 @@ def test_deserialize_corrupt_database(self): class OpenTests(unittest.TestCase): _sql = "create table test(id integer)" + def test_open_with_bytes_path(self): + path = os.fsencode(TESTFN) + self.addCleanup(unlink, path) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(path)) as cx: + self.assertTrue(os.path.exists(path)) + cx.execute(self._sql) + def test_open_with_path_like_object(self): """ Checks that we can successfully connect to a database using an object that is PathLike, i.e. has __fspath__(). """ @@ -649,14 +649,21 @@ def test_open_with_path_like_object(self): self.assertTrue(os.path.exists(path)) cx.execute(self._sql) - @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(is_apple, "skipped on Apple platforms") - @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") - def test_open_with_undecodable_path(self): + def get_undecodable_path(self): path = TESTFN_UNDECODABLE + if not path: + self.skipTest("only works if there are undecodable paths") + try: + open(path, 'wb').close() + except OSError: + self.skipTest(f"can't create file with undecodable path {path!r}") + unlink(path) + return path + + @unittest.skipIf(sys.platform == "win32", "skipped on Windows") + def test_open_with_undecodable_path(self): + path = self.get_undecodable_path() self.addCleanup(unlink, path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(path)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) @@ -696,14 +703,10 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(is_apple, "skipped on Apple platforms") - @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) uri = "file:" + urllib.parse.quote(path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) @@ -1058,7 +1061,7 @@ def test_array_size(self): # now set to 2 self.cu.arraysize = 2 - # now make the query return 3 rows + # now make the query return 2 rows from a table of 3 rows self.cu.execute("delete from test") self.cu.execute("insert into test(name) values ('A')") self.cu.execute("insert into test(name) values ('B')") @@ -1068,13 +1071,50 @@ def test_array_size(self): self.assertEqual(len(res), 2) + def test_invalid_array_size(self): + UINT32_MAX = (1 << 32) - 1 + setter = functools.partial(setattr, self.cu, 'arraysize') + + self.assertRaises(TypeError, setter, 1.0) + self.assertRaises(ValueError, setter, -3) + self.assertRaises(OverflowError, setter, UINT32_MAX + 1) + def test_fetchmany(self): + # no active SQL statement + res = self.cu.fetchmany() + self.assertEqual(res, []) + res = self.cu.fetchmany(1000) + self.assertEqual(res, []) + + # test default parameter + self.cu.execute("select name from test") + res = self.cu.fetchmany() + self.assertEqual(len(res), 1) + + # test when the number of requested rows exceeds the actual count self.cu.execute("select name from test") res = self.cu.fetchmany(100) self.assertEqual(len(res), 1) res = self.cu.fetchmany(100) self.assertEqual(res, []) + # test when size = 0 + self.cu.execute("select name from test") + res = self.cu.fetchmany(0) + self.assertEqual(res, []) + res = self.cu.fetchmany(100) + self.assertEqual(len(res), 1) + res = self.cu.fetchmany(100) + self.assertEqual(res, []) + + def test_invalid_fetchmany(self): + UINT32_MAX = (1 << 32) - 1 + fetchmany = self.cu.fetchmany + + self.assertRaises(TypeError, fetchmany, 1.0) + self.assertRaises(ValueError, fetchmany, -3) + self.assertRaises(OverflowError, fetchmany, UINT32_MAX + 1) + def test_fetchmany_kw_arg(self): """Checks if fetchmany works with keyword arguments""" self.cu.execute("select name from test") diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index cc9f1ec5c4b..776659e3b16 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -71,18 +71,9 @@ class Factory(sqlite.Connection): def __init__(self, *args, **kwargs): super(Factory, self).__init__(*args, **kwargs) - regex = ( - r"Passing more than 1 positional argument to _sqlite3.Connection\(\) " - r"is deprecated. Parameters 'timeout', 'detect_types', " - r"'isolation_level', 'check_same_thread', 'factory', " - r"'cached_statements' and 'uri' will become keyword-only " - r"parameters in Python 3.15." - ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - with memory_database(5.0, 0, None, True, Factory) as con: - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(TypeError, + r'connect\(\) takes at most 1 positional arguments'): + memory_database(5.0, 0, None, True, Factory) class CursorFactoryTests(MemoryDatabaseMixin, unittest.TestCase): diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 53b8a39bf29..2b907e35131 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -220,16 +220,9 @@ def bad_progress(): """) def test_progress_handler_keyword_args(self): - regex = ( - r"Passing keyword argument 'progress_handler' to " - r"_sqlite3.Connection.set_progress_handler\(\) is deprecated. " - r"Parameter 'progress_handler' will become positional-only in " - r"Python 3.15." - ) - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + with self.assertRaisesRegex(TypeError, + 'takes at least 1 positional argument'): self.con.set_progress_handler(progress_handler=lambda: None, n=1) - self.assertEqual(cm.filename, __file__) class TraceCallbackTests(MemoryDatabaseMixin, unittest.TestCase): @@ -353,16 +346,9 @@ def test_trace_bad_handler(self): cx.execute("select 1") def test_trace_keyword_args(self): - regex = ( - r"Passing keyword argument 'trace_callback' to " - r"_sqlite3.Connection.set_trace_callback\(\) is deprecated. " - r"Parameter 'trace_callback' will become positional-only in " - r"Python 3.15." - ) - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + with self.assertRaisesRegex(TypeError, + 'takes exactly 1 positional argument'): self.con.set_trace_callback(trace_callback=lambda: None) - self.assertEqual(cm.filename, __file__) if __name__ == "__main__": diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index 3abc43a3b1a..11cf877a011 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -422,27 +422,9 @@ def test_func_return_illegal_value(self): self.con.execute, "select badreturn()") def test_func_keyword_args(self): - regex = ( - r"Passing keyword arguments 'name', 'narg' and 'func' to " - r"_sqlite3.Connection.create_function\(\) is deprecated. " - r"Parameters 'name', 'narg' and 'func' will become " - r"positional-only in Python 3.15." - ) - - def noop(): - return None - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - self.con.create_function("noop", 0, func=noop) - self.assertEqual(cm.filename, __file__) - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - self.con.create_function("noop", narg=0, func=noop) - self.assertEqual(cm.filename, __file__) - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - self.con.create_function(name="noop", narg=0, func=noop) - self.assertEqual(cm.filename, __file__) + with self.assertRaisesRegex(TypeError, + 'takes exactly 3 positional arguments'): + self.con.create_function("noop", 0, func=lambda: None) class WindowSumInt: @@ -737,25 +719,9 @@ def test_aggr_text(self): self.assertEqual(val, txt) def test_agg_keyword_args(self): - regex = ( - r"Passing keyword arguments 'name', 'n_arg' and 'aggregate_class' to " - r"_sqlite3.Connection.create_aggregate\(\) is deprecated. " - r"Parameters 'name', 'n_arg' and 'aggregate_class' will become " - r"positional-only in Python 3.15." - ) - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + with self.assertRaisesRegex(TypeError, + 'takes exactly 3 positional arguments'): self.con.create_aggregate("test", 1, aggregate_class=AggrText) - self.assertEqual(cm.filename, __file__) - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - self.con.create_aggregate("test", n_arg=1, aggregate_class=AggrText) - self.assertEqual(cm.filename, __file__) - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - self.con.create_aggregate(name="test", n_arg=0, - aggregate_class=AggrText) - self.assertEqual(cm.filename, __file__) class AuthorizerTests(unittest.TestCase): @@ -800,16 +766,9 @@ def test_clear_authorizer(self): self.con.execute("select c2 from t1") def test_authorizer_keyword_args(self): - regex = ( - r"Passing keyword argument 'authorizer_callback' to " - r"_sqlite3.Connection.set_authorizer\(\) is deprecated. " - r"Parameter 'authorizer_callback' will become positional-only in " - r"Python 3.15." - ) - - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + with self.assertRaisesRegex(TypeError, + 'takes exactly 1 positional argument'): self.con.set_authorizer(authorizer_callback=lambda: None) - self.assertEqual(cm.filename, __file__) class AuthorizerRaiseExceptionTests(AuthorizerTests): diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 395b2ef88ab..ebdf5455163 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -31,6 +31,7 @@ import platform import sysconfig import functools +from contextlib import nullcontext try: import ctypes except ImportError: @@ -47,6 +48,13 @@ PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = socket_helper.HOST IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) +CAN_GET_SELECTED_OPENSSL_GROUP = ssl.OPENSSL_VERSION_INFO >= (3, 2) +CAN_IGNORE_UNKNOWN_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 3) +CAN_GET_AVAILABLE_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 5) +CAN_GET_AVAILABLE_OPENSSL_SIGALGS = ssl.OPENSSL_VERSION_INFO >= (3, 4) +CAN_SET_CLIENT_SIGALGS = "AWS-LC" not in ssl.OPENSSL_VERSION +CAN_IGNORE_UNKNOWN_OPENSSL_SIGALGS = ssl.OPENSSL_VERSION_INFO >= (3, 3) +CAN_GET_SELECTED_OPENSSL_SIGALG = ssl.OPENSSL_VERSION_INFO >= (3, 5) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') PROTOCOL_TO_TLS_VERSION = {} @@ -259,7 +267,9 @@ def utc_offset(): #NOTE: ignore issues like #1647654 def test_wrap_socket(sock, *, cert_reqs=ssl.CERT_NONE, ca_certs=None, - ciphers=None, certfile=None, keyfile=None, + ciphers=None, ciphersuites=None, + min_version=None, max_version=None, + certfile=None, keyfile=None, **kwargs): if not kwargs.get("server_side"): kwargs["server_hostname"] = SIGNED_CERTFILE_HOSTNAME @@ -276,13 +286,20 @@ def test_wrap_socket(sock, *, context.load_cert_chain(certfile, keyfile) if ciphers is not None: context.set_ciphers(ciphers) + if ciphersuites is not None: + context.set_ciphersuites(ciphersuites) + if min_version is not None: + context.minimum_version = min_version + if max_version is not None: + context.maximum_version = max_version return context.wrap_socket(sock, **kwargs) USE_SAME_TEST_CONTEXT = False _TEST_CONTEXT = None -def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): +def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True, + client_cert=None): """Create context client_context, server_context, hostname = testing_context() @@ -309,6 +326,10 @@ def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): if server_chain: server_context.load_verify_locations(SIGNING_CA) + if client_cert: + client_context.load_cert_chain(client_cert) + server_context.verify_mode = ssl.CERT_REQUIRED + if USE_SAME_TEST_CONTEXT: if _TEST_CONTEXT is not None: _TEST_CONTEXT = client_context, server_context, hostname @@ -539,9 +560,9 @@ def test_openssl_version(self): openssl_ver = f"OpenSSL {major:d}.{minor:d}.{patch:d}" else: openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" - self.assertTrue( - s.startswith((openssl_ver, libressl_ver, "AWS-LC")), - (s, t, hex(n)) + self.assertStartsWith( + s, (openssl_ver, libressl_ver, "AWS-LC"), + (t, hex(n)) ) @support.cpython_only @@ -959,6 +980,56 @@ def test_get_ciphers(self): len(intersection), 2, f"\ngot: {sorted(names)}\nexpected: {sorted(expected)}" ) + def test_set_groups(self): + ctx = ssl.create_default_context() + # We use P-256 and P-384 (FIPS 186-4) that are alloed by OpenSSL + # even if FIPS module is enabled. Ignoring unknown groups is only + # supported since OpenSSL 3.3. + self.assertIsNone(ctx.set_groups('P-256:P-384')) + + self.assertRaises(ssl.SSLError, ctx.set_groups, 'P-256:foo') + if CAN_IGNORE_UNKNOWN_OPENSSL_GROUPS: + self.assertIsNone(ctx.set_groups('P-256:?foo')) + + @unittest.skipUnless(CAN_GET_AVAILABLE_OPENSSL_GROUPS, + "OpenSSL version doesn't support getting groups") + def test_get_groups(self): + ctx = ssl.create_default_context() + # By default, only return official IANA names. + self.assertNotIn('P-256', ctx.get_groups()) + self.assertIn('P-256', ctx.get_groups(include_aliases=True)) + + @unittest.skipUnless(CAN_GET_AVAILABLE_OPENSSL_SIGALGS, + "SSL library doesn't support getting sigalgs") + def test_get_sigalgs(self): + self.assertIn('rsa_pss_rsae_sha256', ssl.get_sigalgs()) + + @unittest.skipUnless(CAN_SET_CLIENT_SIGALGS, + "SSL library doesn't support setting client sigalgs") + def test_set_client_sigalgs(self): + ctx = ssl.create_default_context() + + self.assertIsNone(ctx.set_client_sigalgs('rsa_pss_rsae_sha256')) + + self.assertRaises(ssl.SSLError, ctx.set_client_sigalgs, + 'rsa_pss_rsae_sha256:foo') + + # Ignoring unknown sigalgs is only supported since OpenSSL 3.3. + if CAN_IGNORE_UNKNOWN_OPENSSL_SIGALGS: + self.assertIsNone(ctx.set_client_sigalgs('rsa_pss_rsae_sha256:?foo')) + + def test_set_server_sigalgs(self): + ctx = ssl.create_default_context() + + self.assertIsNone(ctx.set_server_sigalgs('rsa_pss_rsae_sha256')) + + self.assertRaises(ssl.SSLError, ctx.set_server_sigalgs, + 'rsa_pss_rsae_sha256:foo') + + # Ignoring unknown sigalgs is only supported since OpenSSL 3.3. + if CAN_IGNORE_UNKNOWN_OPENSSL_SIGALGS: + self.assertIsNone(ctx.set_server_sigalgs('rsa_pss_rsae_sha256:?foo')) + def test_options(self): # Test default SSLContext options ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) @@ -1238,6 +1309,25 @@ def getpass(self): # Make sure the password function isn't called if it isn't needed ctx.load_cert_chain(CERTFILE, password=getpass_exception) + @threading_helper.requires_working_threading() + def test_load_cert_chain_thread_safety(self): + # gh-134698: _ssl detaches the thread state (and as such, + # releases the GIL and critical sections) around expensive + # OpenSSL calls. Unfortunately, OpenSSL structures aren't + # thread-safe, so executing these calls concurrently led + # to crashes. + ctx = ssl.create_default_context() + + def race(): + ctx.load_cert_chain(CERTFILE) + + threads = [threading.Thread(target=race) for _ in range(8)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(threads): + pass + + self.assertIsNone(cm.exc_value) + def test_load_verify_locations(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_verify_locations(CERTFILE) @@ -1668,7 +1758,7 @@ def test_lib_reason(self): regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" self.assertRegex(cm.exception.reason, regex) s = str(cm.exception) - self.assertTrue("NO_START_LINE" in s, s) + self.assertIn("NO_START_LINE", s) def test_subclass(self): # Check that the appropriate SSLError subclass is raised @@ -1683,7 +1773,7 @@ def test_subclass(self): with self.assertRaises(ssl.SSLWantReadError) as cm: c.do_handshake() s = str(cm.exception) - self.assertTrue(s.startswith("The operation did not complete (read)"), s) + self.assertStartsWith(s, "The operation did not complete (read)") # For compatibility self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) @@ -2196,6 +2286,68 @@ def test_transport_eof(self): self.assertRaises(ssl.SSLEOFError, sslobj.read) +@unittest.skipUnless(has_tls_version('TLSv1_3'), "TLS 1.3 is not available") +class SimpleBackgroundTestsTLS_1_3(unittest.TestCase): + """Tests that connect to a simple server running in the background.""" + + def setUp(self): + server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ciphers = [cipher['name'] for cipher in server_ctx.get_ciphers() + if cipher['protocol'] == 'TLSv1.3'] + + if not ciphers: + self.skipTest("No cipher supports TLSv1.3") + + self.matching_cipher = ciphers[0] + # Some tests need at least two ciphers, and are responsible + # to skip themselves if matching_cipher == mismatched_cipher. + self.mismatched_cipher = ciphers[-1] + + server_ctx.set_ciphersuites(self.matching_cipher) + server_ctx.load_cert_chain(SIGNED_CERTFILE) + server = ThreadedEchoServer(context=server_ctx) + self.enterContext(server) + self.server_addr = (HOST, server.port) + + def test_ciphersuites(self): + # Test unrecognized TLS 1.3 cipher suite name + with ( + socket.socket(socket.AF_INET) as sock, + self.assertRaisesRegex(ssl.SSLError, + "No cipher suite can be selected") + ): + test_wrap_socket(sock, cert_reqs=ssl.CERT_NONE, + ciphersuites="XXX", + min_version=ssl.TLSVersion.TLSv1_3) + + # Test successful TLS 1.3 handshake + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, + ciphersuites=self.matching_cipher, + min_version=ssl.TLSVersion.TLSv1_3) as s: + s.connect(self.server_addr) + self.assertEqual(s.cipher()[0], self.matching_cipher) + + def test_ciphersuite_downgrade(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, + ciphersuites=self.matching_cipher, + min_version=ssl.TLSVersion.TLSv1_2, + max_version=ssl.TLSVersion.TLSv1_2) as s: + s.connect(self.server_addr) + self.assertEqual(s.cipher()[1], 'TLSv1.2') + + def test_ciphersuite_mismatch(self): + if self.matching_cipher == self.mismatched_cipher: + self.skipTest("Multiple TLS 1.3 ciphers are not available") + + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, + ciphersuites=self.mismatched_cipher, + min_version=ssl.TLSVersion.TLSv1_3) as s: + self.assertRaises(ssl.SSLError, s.connect, self.server_addr) + + @support.requires_resource('network') class NetworkedTests(unittest.TestCase): @@ -2700,6 +2852,11 @@ def server_params_test(client_context, server_context, indata=b"FOO\n", 'session_reused': s.session_reused, 'session': s.session, }) + if CAN_GET_SELECTED_OPENSSL_GROUP: + stats.update({'group': s.group()}) + if CAN_GET_SELECTED_OPENSSL_SIGALG: + stats.update({'client_sigalg': s.client_sigalg()}) + stats.update({'server_sigalg': s.server_sigalg()}) s.close() stats['server_alpn_protocols'] = server.selected_alpn_protocols stats['server_shared_ciphers'] = server.shared_ciphers @@ -2843,6 +3000,7 @@ def test_ssl_in_multiple_threads(self): # See GH-124984: OpenSSL is not thread safe. threads = [] + warnings_filters = sys.flags.context_aware_warnings global USE_SAME_TEST_CONTEXT USE_SAME_TEST_CONTEXT = True try: @@ -2851,7 +3009,10 @@ def test_ssl_in_multiple_threads(self): self.test_alpn_protocols, self.test_getpeercert, self.test_crl_check, - self.test_check_hostname_idn, + functools.partial( + self.test_check_hostname_idn, + warnings_filters=warnings_filters, + ), self.test_wrong_cert_tls12, self.test_wrong_cert_tls13, ): @@ -3097,7 +3258,7 @@ def test_dual_rsa_ecc(self): cipher = s.cipher()[0].split('-') self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) - def test_check_hostname_idn(self): + def test_check_hostname_idn(self, warnings_filters=True): if support.verbose: sys.stdout.write("\n") @@ -3152,16 +3313,30 @@ def test_check_hostname_idn(self): server_hostname="python.example.org") as s: with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) - with ThreadedEchoServer(context=server_context, chatty=True) as server: - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(UnicodeError): - context.wrap_socket(socket.socket(), - server_hostname='.pythontest.net') - with ThreadedEchoServer(context=server_context, chatty=True) as server: - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(UnicodeDecodeError): - context.wrap_socket(socket.socket(), - server_hostname=b'k\xf6nig.idn.pythontest.net') + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeError), + ): + context.wrap_socket(socket.socket(), server_hostname='.pythontest.net') + + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeDecodeError), + ): + context.wrap_socket( + socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net', + ) def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate @@ -3832,6 +4007,8 @@ def test_no_shared_ciphers(self): with self.assertRaises(OSError): s.connect((HOST, server.port)) self.assertIn("NO_SHARED_CIPHER", server.conn_errors[0]) + self.assertIsNone(s.cipher()) + self.assertIsNone(s.group()) def test_version_basic(self): """ @@ -4107,6 +4284,109 @@ def test_ecdh_curve(self): chatty=True, connectionchatty=True, sni_name=hostname) + def test_groups(self): + # server secp384r1, client auto + client_context, server_context, hostname = testing_context() + + server_context.set_groups("secp384r1") + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if CAN_GET_SELECTED_OPENSSL_GROUP: + self.assertEqual(stats['group'], "secp384r1") + + # server auto, client secp384r1 + client_context, server_context, hostname = testing_context() + client_context.set_groups("secp384r1") + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if CAN_GET_SELECTED_OPENSSL_GROUP: + self.assertEqual(stats['group'], "secp384r1") + + # server / client curve mismatch + client_context, server_context, hostname = testing_context() + client_context.set_groups("prime256v1") + server_context.set_groups("secp384r1") + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + with self.assertRaises(ssl.SSLError): + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + @unittest.skipUnless(CAN_SET_CLIENT_SIGALGS, + "SSL library doesn't support setting client sigalgs") + def test_client_sigalgs(self): + # no mutual auth, so cient_sigalg should be None + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if CAN_GET_SELECTED_OPENSSL_SIGALG: + self.assertIsNone(stats['client_sigalg']) + + # server auto, client rsa_pss_rsae_sha384 + sigalg = "rsa_pss_rsae_sha384" + client_context, server_context, hostname = \ + testing_context(client_cert=SIGNED_CERTFILE) + client_context.set_client_sigalgs(sigalg) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if CAN_GET_SELECTED_OPENSSL_SIGALG: + self.assertEqual(stats['client_sigalg'], sigalg) + + @unittest.skipUnless(CAN_SET_CLIENT_SIGALGS, + "SSL library doesn't support setting client sigalgs") + def test_client_sigalgs_mismatch(self): + client_context, server_context, hostname = \ + testing_context(client_cert=SIGNED_CERTFILE) + client_context.set_client_sigalgs("rsa_pss_rsae_sha256") + server_context.set_client_sigalgs("rsa_pss_rsae_sha384") + + with self.assertRaises(( + ssl.SSLError, + # On handshake failures, some systems raise a ConnectionResetError. + ConnectionResetError, + # On handshake failures, macOS may raise a BrokenPipeError. + # See https://github.com/python/cpython/issues/139504. + BrokenPipeError, + )): + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_server_sigalgs(self): + # server rsa_pss_rsae_sha384, client auto + sigalg = "rsa_pss_rsae_sha384" + client_context, server_context, hostname = testing_context() + server_context.set_server_sigalgs(sigalg) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if CAN_GET_SELECTED_OPENSSL_SIGALG: + self.assertEqual(stats['server_sigalg'], sigalg) + + # server auto, client rsa_pss_rsae_sha384 + client_context, server_context, hostname = testing_context() + client_context.set_server_sigalgs(sigalg) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if CAN_GET_SELECTED_OPENSSL_SIGALG: + self.assertEqual(stats['server_sigalg'], sigalg) + + def test_server_sigalgs_mismatch(self): + client_context, server_context, hostname = testing_context() + client_context.set_server_sigalgs("rsa_pss_rsae_sha256") + server_context.set_server_sigalgs("rsa_pss_rsae_sha384") + with self.assertRaises(ssl.SSLError): + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + def test_selected_alpn_protocol(self): # selected_alpn_protocol() is None unless ALPN is used. client_context, server_context, hostname = testing_context() @@ -4297,19 +4577,30 @@ def test_read_write_after_close_raises_valuerror(self): self.assertRaises(ValueError, s.write, b'hello') def test_sendfile(self): + """Try to send a file using kTLS if possible.""" TEST_DATA = b"x" * 512 with open(os_helper.TESTFN, 'wb') as f: f.write(TEST_DATA) self.addCleanup(os_helper.unlink, os_helper.TESTFN) client_context, server_context, hostname = testing_context() + client_context.options |= getattr(ssl, 'OP_ENABLE_KTLS', 0) server = ThreadedEchoServer(context=server_context, chatty=False) - with server: - with client_context.wrap_socket(socket.socket(), - server_hostname=hostname) as s: - s.connect((HOST, server.port)) + # kTLS seems to work only with a connection created before + # wrapping `sock` by the SSL context in contrast to calling + # `sock.connect()` after the wrapping. + with server, socket.create_connection((HOST, server.port)) as sock: + with client_context.wrap_socket( + sock, server_hostname=hostname + ) as ssock: + if support.verbose: + ktls_used = ssock._sslobj.uses_ktls_for_send() + print( + 'kTLS is', + 'available' if ktls_used else 'unavailable', + ) with open(os_helper.TESTFN, 'rb') as file: - s.sendfile(file) - self.assertEqual(s.recv(1024), TEST_DATA) + ssock.sendfile(file) + self.assertEqual(ssock.recv(1024), TEST_DATA) def test_session(self): client_context, server_context, hostname = testing_context() @@ -4488,6 +4779,7 @@ def server_callback(identity): @requires_tls_version('TLSv1_3') @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') + @unittest.skipUnless(ssl.HAS_PSK_TLS13, 'TLS 1.3 PSK disabled on this OpenSSL build') def test_psk_tls1_3(self): psk = bytes.fromhex('deadbeef') identity_hint = 'identity-hint' @@ -4519,6 +4811,42 @@ def server_callback(identity): with client_context.wrap_socket(socket.socket()) as s: s.connect((HOST, server.port)) + def test_thread_recv_while_main_thread_sends(self): + # GH-137583: Locking was added to calls to send() and recv() on SSL + # socket objects. This seemed fine at the surface level because those + # calls weren't re-entrant, but recv() calls would implicitly mimick + # holding a lock by blocking until it received data. This means that + # if a thread started to infinitely block until data was received, calls + # to send() would deadlock, because it would wait forever on the lock + # that the recv() call held. + data = b"1" * 1024 + event = threading.Event() + def background(sock): + event.set() + received = sock.recv(len(data)) + self.assertEqual(received, data) + + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as sock: + sock.connect((HOST, server.port)) + sock.settimeout(1) + sock.setblocking(1) + # Ensure that the server is ready to accept requests + sock.sendall(b"123") + self.assertEqual(sock.recv(3), b"123") + with threading_helper.catch_threading_exception() as cm: + thread = threading.Thread(target=background, + args=(sock,), daemon=True) + thread.start() + event.wait() + sock.sendall(data) + thread.join() + if cm.exc_value is not None: + raise cm.exc_value + @unittest.skipUnless(has_tls_version('TLSv1_3') and ssl.HAS_PHA, "Test needs TLS 1.3 PHA") @@ -5285,7 +5613,7 @@ def test_tlsalerttype(self): class Checked_TLSAlertType(enum.IntEnum): """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 1e6f69d49e9..2e93ac08f82 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -45,6 +45,7 @@ class TestStableABIAvailability(unittest.TestCase): SYMBOL_NAMES = ( + "PyABIInfo_Check", "PyAIter_Check", "PyArg_Parse", "PyArg_ParseTuple", @@ -164,6 +165,7 @@ SYMBOL_NAMES = ( "PyDict_MergeFromSeq2", "PyDict_New", "PyDict_Next", + "PyDict_SetDefaultRef", "PyDict_SetItem", "PyDict_SetItemString", "PyDict_Size", @@ -468,8 +470,10 @@ SYMBOL_NAMES = ( "PyModule_AddStringConstant", "PyModule_AddType", "PyModule_Create2", + "PyModule_Exec", "PyModule_ExecDef", "PyModule_FromDefAndSpec2", + "PyModule_FromSlotsAndSpec", "PyModule_GetDef", "PyModule_GetDict", "PyModule_GetFilename", @@ -477,6 +481,8 @@ SYMBOL_NAMES = ( "PyModule_GetName", "PyModule_GetNameObject", "PyModule_GetState", + "PyModule_GetStateSize", + "PyModule_GetToken", "PyModule_New", "PyModule_NewObject", "PyModule_SetDocString", @@ -658,7 +664,11 @@ SYMBOL_NAMES = ( "PySys_AuditTuple", "PySys_FormatStderr", "PySys_FormatStdout", + "PySys_GetAttr", + "PySys_GetAttrString", "PySys_GetObject", + "PySys_GetOptionalAttr", + "PySys_GetOptionalAttrString", "PySys_GetXOptions", "PySys_HasWarnOptions", "PySys_ResetWarnOptions", @@ -728,6 +738,7 @@ SYMBOL_NAMES = ( "PyType_GetFullyQualifiedName", "PyType_GetModule", "PyType_GetModuleByDef", + "PyType_GetModuleByToken", "PyType_GetModuleName", "PyType_GetModuleState", "PyType_GetName", @@ -890,6 +901,7 @@ SYMBOL_NAMES = ( "Py_GetRecursionLimit", "Py_GetVersion", "Py_HasFileSystemDefaultEncoding", + "Py_IS_TYPE", "Py_IncRef", "Py_Initialize", "Py_InitializeEx", @@ -909,6 +921,8 @@ SYMBOL_NAMES = ( "Py_REFCNT", "Py_ReprEnter", "Py_ReprLeave", + "Py_SET_SIZE", + "Py_SIZE", "Py_SetPath", "Py_SetProgramName", "Py_SetPythonHome", diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 49013a4bcd8..5fd25d5012c 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -157,7 +157,7 @@ def test_mode(self): os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() - self.assertEqual(modestr[:3], '-rw') + self.assertStartsWith(modestr, '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) @@ -256,7 +256,7 @@ def test_flags_consistent(self): "FILE_ATTRIBUTE_* constants are Win32 specific") def test_file_attribute_constants(self): for key, value in sorted(self.file_attributes.items()): - self.assertTrue(hasattr(self.statmod, key), key) + self.assertHasAttr(self.statmod, key) modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) @@ -314,7 +314,7 @@ def test_macosx_attribute_values(self): self.assertEqual(self.statmod.S_ISGID, 0o002000) self.assertEqual(self.statmod.S_ISVTX, 0o001000) - self.assertFalse(hasattr(self.statmod, "S_ISTXT")) + self.assertNotHasAttr(self.statmod, "S_ISTXT") self.assertEqual(self.statmod.S_IREAD, self.statmod.S_IRUSR) self.assertEqual(self.statmod.S_IWRITE, self.statmod.S_IWUSR) self.assertEqual(self.statmod.S_IEXEC, self.statmod.S_IXUSR) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index c69baa4bf4d..677a87b51b9 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -645,7 +645,7 @@ def do_test(self, args): def test_numerictestcase_is_testcase(self): # Ensure that NumericTestCase actually is a TestCase. - self.assertTrue(issubclass(NumericTestCase, unittest.TestCase)) + self.assertIsSubclass(NumericTestCase, unittest.TestCase) def test_error_msg_numeric(self): # Test the error message generated for numeric comparisons. @@ -683,32 +683,23 @@ class GlobalsTest(unittest.TestCase): def test_meta(self): # Test for the existence of metadata. for meta in self.expected_metadata: - self.assertTrue(hasattr(self.module, meta), - "%s not present" % meta) + self.assertHasAttr(self.module, meta) def test_check_all(self): # Check everything in __all__ exists and is public. module = self.module for name in module.__all__: # No private names in __all__: - self.assertFalse(name.startswith("_"), + self.assertNotStartsWith(name, "_", 'private name "%s" in __all__' % name) # And anything in __all__ must exist: - self.assertTrue(hasattr(module, name), - 'missing name "%s" in __all__' % name) + self.assertHasAttr(module, name) class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): - errmsg = ( - "Expected StatisticsError to be a ValueError, but got a" - " subclass of %r instead." - ) - self.assertTrue(hasattr(statistics, 'StatisticsError')) - self.assertTrue( - issubclass(statistics.StatisticsError, ValueError), - errmsg % statistics.StatisticsError.__base__ - ) + self.assertHasAttr(statistics, 'StatisticsError') + self.assertIsSubclass(statistics.StatisticsError, ValueError) # === Tests for private utility functions === @@ -2014,7 +2005,6 @@ def test_iter_list_same(self): expected = self.func(data) self.assertEqual(self.func(iter(data)), expected) - class TestPVariance(VarianceStdevMixin, NumericTestCase, UnivariateTypeMixin): # Tests for population variance. def setUp(self): @@ -2122,6 +2112,14 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) + def test_gh_140938(self): + # Inputs with inf/nan should raise a ValueError + with self.assertRaises(ValueError): + self.func([1.0, math.inf]) + with self.assertRaises(ValueError): + self.func([1.0, math.nan]) + + class TestSqrtHelpers(unittest.TestCase): def test_integer_sqrt_of_frac_rto(self): @@ -2355,6 +2353,7 @@ def test_mixed_int_and_float(self): class TestKDE(unittest.TestCase): + @support.requires_resource('cpu') def test_kde(self): kde = statistics.kde StatisticsError = statistics.StatisticsError @@ -3327,7 +3326,8 @@ def tearDown(self): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) - tests.addTests(doctest.DocTestSuite(statistics)) + if sys.float_repr_style == 'short': + tests.addTests(doctest.DocTestSuite(statistics)) return tests diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index d6a7bd0da59..2584fbf72d3 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -1231,10 +1231,10 @@ def __repr__(self): self.assertEqual('{0:\x00^6}'.format(3), '\x00\x003\x00\x00\x00') self.assertEqual('{0:<6}'.format(3), '3 ') - self.assertEqual('{0:\x00<6}'.format(3.14), '3.14\x00\x00') - self.assertEqual('{0:\x01<6}'.format(3.14), '3.14\x01\x01') - self.assertEqual('{0:\x00^6}'.format(3.14), '\x003.14\x00') - self.assertEqual('{0:^6}'.format(3.14), ' 3.14 ') + self.assertEqual('{0:\x00<6}'.format(3.25), '3.25\x00\x00') + self.assertEqual('{0:\x01<6}'.format(3.25), '3.25\x01\x01') + self.assertEqual('{0:\x00^6}'.format(3.25), '\x003.25\x00') + self.assertEqual('{0:^6}'.format(3.25), ' 3.25 ') self.assertEqual('{0:\x00<12}'.format(3+2.0j), '(3+2j)\x00\x00\x00\x00\x00\x00') self.assertEqual('{0:\x01<12}'.format(3+2.0j), '(3+2j)\x01\x01\x01\x01\x01\x01') diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index 752e31359cf..375f6aaedd8 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -39,7 +39,21 @@ def _update_variables(self, now): if now[3] < 12: self.ampm='(AM|am)' else: self.ampm='(PM|pm)' - self.jan1 = time.localtime(time.mktime((now[0], 1, 1, 0, 0, 0, 0, 1, 0))) + jan1 = time.struct_time( + ( + now.tm_year, # Year + 1, # Month (January) + 1, # Day (1st) + 0, # Hour (0) + 0, # Minute (0) + 0, # Second (0) + -1, # tm_wday (will be determined) + 1, # tm_yday (day 1 of the year) + -1, # tm_isdst (let the system determine) + ) + ) + # use mktime to get the correct tm_wday and tm_isdst values + self.jan1 = time.localtime(time.mktime(jan1)) try: if now[8]: self.tz = time.tzname[1] diff --git a/Lib/test/test_string/_support.py b/Lib/test/test_string/_support.py index eaa3354a559..e1d7f6f6500 100644 --- a/Lib/test/test_string/_support.py +++ b/Lib/test/test_string/_support.py @@ -1,35 +1,46 @@ -import unittest from string.templatelib import Interpolation class TStringBaseCase: + def assertInterpolationEqual(self, i, exp): + """Test Interpolation equality. + + The *i* argument must be an Interpolation instance. + + The *exp* argument must be a tuple of the form + (value, expression, conversion, format_spec) where the final three + items may be omitted and are assumed to be '', None and '' respectively. + """ + if len(exp) == 4: + actual = (i.value, i.expression, i.conversion, i.format_spec) + self.assertEqual(actual, exp) + elif len(exp) == 3: + self.assertEqual((i.value, i.expression, i.conversion), exp) + self.assertEqual(i.format_spec, "") + elif len(exp) == 2: + self.assertEqual((i.value, i.expression), exp) + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + elif len(exp) == 1: + self.assertEqual((i.value,), exp) + self.assertEqual(i.expression, "") + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + def assertTStringEqual(self, t, strings, interpolations): """Test template string literal equality. The *strings* argument must be a tuple of strings equal to *t.strings*. The *interpolations* argument must be a sequence of tuples which are - compared against *t.interpolations*. Each tuple consists of - (value, expression, conversion, format_spec), though the final two - items may be omitted, and are assumed to be None and '' respectively. + compared against *t.interpolations*. Each tuple must match the form + described in the `assertInterpolationEqual` method. """ self.assertEqual(t.strings, strings) self.assertEqual(len(t.interpolations), len(interpolations)) for i, exp in zip(t.interpolations, interpolations, strict=True): - if len(exp) == 4: - actual = (i.value, i.expression, i.conversion, i.format_spec) - self.assertEqual(actual, exp) - continue - - if len(exp) == 3: - self.assertEqual((i.value, i.expression, i.conversion), exp) - self.assertEqual(i.format_spec, '') - continue - - self.assertEqual((i.value, i.expression), exp) - self.assertEqual(i.format_spec, '') - self.assertIsNone(i.conversion) + self.assertInterpolationEqual(i, exp) def convert(value, conversion): diff --git a/Lib/test/test_string/test_templatelib.py b/Lib/test/test_string/test_templatelib.py index 5b9490c2be6..1c86717155f 100644 --- a/Lib/test/test_string/test_templatelib.py +++ b/Lib/test/test_string/test_templatelib.py @@ -1,7 +1,7 @@ import pickle import unittest from collections.abc import Iterator, Iterable -from string.templatelib import Template, Interpolation +from string.templatelib import Template, Interpolation, convert from test.test_string._support import TStringBaseCase, fstring @@ -45,6 +45,19 @@ def test_basic_creation(self): self.assertEqual(len(t.interpolations), 0) self.assertEqual(fstring(t), 'Hello,\nworld') + def test_interpolation_creation(self): + i = Interpolation('Maria', 'name', 'a', 'fmt') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a', 'fmt')) + + i = Interpolation('Maria', 'name', 'a') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a')) + + i = Interpolation('Maria', 'name') + self.assertInterpolationEqual(i, ('Maria', 'name')) + + i = Interpolation('Maria') + self.assertInterpolationEqual(i, ('Maria',)) + def test_creation_interleaving(self): # Should add strings on either side t = Template(Interpolation('Maria', 'name', None, '')) @@ -148,6 +161,33 @@ def test_iter(self): self.assertEqual(res[1].format_spec, '') self.assertEqual(res[2], ' yz') + def test_exhausted(self): + # See https://github.com/python/cpython/issues/134119. + template_iter = iter(t"{1}") + self.assertIsInstance(next(template_iter), Interpolation) + self.assertRaises(StopIteration, next, template_iter) + self.assertRaises(StopIteration, next, template_iter) + + +class TestFunctions(unittest.TestCase): + def test_convert(self): + from fractions import Fraction + + for obj in ('Café', None, 3.14, Fraction(1, 2)): + with self.subTest(f'{obj=}'): + self.assertEqual(convert(obj, None), obj) + self.assertEqual(convert(obj, 's'), str(obj)) + self.assertEqual(convert(obj, 'r'), repr(obj)) + self.assertEqual(convert(obj, 'a'), ascii(obj)) + + # Invalid conversion specifier + with self.assertRaises(ValueError): + convert(obj, 'z') + with self.assertRaises(ValueError): + convert(obj, 1) + with self.assertRaises(ValueError): + convert(obj, object()) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 268230f6da7..40e114aada6 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -221,14 +221,16 @@ def test_ValueError(self): self.assertRaises(ValueError, _strptime._strptime_time, data_string="%d", format="%A") for bad_format in ("%", "% ", "%\n"): - with self.assertRaisesRegex(ValueError, "stray % in format "): + with (self.subTest(format=bad_format), + self.assertRaisesRegex(ValueError, "stray % in format ")): _strptime._strptime_time("2005", bad_format) - for bad_format in ("%e", "%Oe", "%O", "%O ", "%Ee", "%E", "%E ", - "%.", "%+", "%_", "%~", "%\\", + for bad_format in ("%i", "%Oi", "%O", "%O ", "%Ee", "%E", "%E ", + "%.", "%+", "%~", "%\\", "%O.", "%O+", "%O_", "%O~", "%O\\"): directive = bad_format[1:].rstrip() - with self.assertRaisesRegex(ValueError, - f"'{re.escape(directive)}' is a bad directive in format "): + with (self.subTest(format=bad_format), + self.assertRaisesRegex(ValueError, + f"'{re.escape(directive)}' is a bad directive in format ")): _strptime._strptime_time("2005", bad_format) msg_week_no_year_or_weekday = r"ISO week directive '%V' must be used with " \ @@ -335,6 +337,15 @@ def test_month_locale(self): self.roundtrip('%B', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) self.roundtrip('%b', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) + @run_with_locales('LC_TIME', 'az_AZ', 'ber_DZ', 'ber_MA', 'crh_UA') + def test_month_locale2(self): + # Test for month directives + # Month name contains 'İ' ('\u0130') + self.roundtrip('%B', 1, (2025, 6, 1, 0, 0, 0, 6, 152, 0)) + self.roundtrip('%b', 1, (2025, 6, 1, 0, 0, 0, 6, 152, 0)) + self.roundtrip('%B', 1, (2025, 7, 1, 0, 0, 0, 1, 182, 0)) + self.roundtrip('%b', 1, (2025, 7, 1, 0, 0, 0, 1, 182, 0)) + def test_day(self): # Test for day directives self.roundtrip('%d %Y', 2) @@ -395,37 +406,50 @@ def test_offset(self): (*_, offset), _, offset_fraction = _strptime._strptime("-013030.000001", "%z") self.assertEqual(offset, -(one_hour + half_hour + half_minute)) self.assertEqual(offset_fraction, -1) - (*_, offset), _, offset_fraction = _strptime._strptime("+01:00", "%z") - self.assertEqual(offset, one_hour) - self.assertEqual(offset_fraction, 0) - (*_, offset), _, offset_fraction = _strptime._strptime("-01:30", "%z") - self.assertEqual(offset, -(one_hour + half_hour)) - self.assertEqual(offset_fraction, 0) - (*_, offset), _, offset_fraction = _strptime._strptime("-01:30:30", "%z") - self.assertEqual(offset, -(one_hour + half_hour + half_minute)) - self.assertEqual(offset_fraction, 0) - (*_, offset), _, offset_fraction = _strptime._strptime("-01:30:30.000001", "%z") - self.assertEqual(offset, -(one_hour + half_hour + half_minute)) - self.assertEqual(offset_fraction, -1) - (*_, offset), _, offset_fraction = _strptime._strptime("+01:30:30.001", "%z") - self.assertEqual(offset, one_hour + half_hour + half_minute) - self.assertEqual(offset_fraction, 1000) - (*_, offset), _, offset_fraction = _strptime._strptime("Z", "%z") - self.assertEqual(offset, 0) - self.assertEqual(offset_fraction, 0) + + cases = [ + ("+01:00", one_hour, 0), + ("-01:30", -(one_hour + half_hour), 0), + ("-01:30:30", -(one_hour + half_hour + half_minute), 0), + ("-01:30:30.000001", -(one_hour + half_hour + half_minute), -1), + ("+01:30:30.001", +(one_hour + half_hour + half_minute), 1000), + ("Z", 0, 0), + ] + for directive in ("%z", "%:z"): + for offset_str, expected_offset, expected_fraction in cases: + with self.subTest(offset_str=offset_str, directive=directive): + (*_, offset), _, offset_fraction = _strptime._strptime( + offset_str, directive + ) + self.assertEqual(offset, expected_offset) + self.assertEqual(offset_fraction, expected_fraction) def test_bad_offset(self): - with self.assertRaises(ValueError): - _strptime._strptime("-01:30:30.", "%z") - with self.assertRaises(ValueError): - _strptime._strptime("-0130:30", "%z") - with self.assertRaises(ValueError): - _strptime._strptime("-01:30:30.1234567", "%z") - with self.assertRaises(ValueError): - _strptime._strptime("-01:30:30:123456", "%z") + error_cases_any_z = [ + "-01:30:30.", # Decimal point not followed with digits + "-01:30:30.1234567", # Too many digits after decimal point + "-01:30:30:123456", # Colon as decimal separator + "-0130:30", # Incorrect use of colons + ] + for directive in ("%z", "%:z"): + for timestr in error_cases_any_z: + with self.subTest(timestr=timestr, directive=directive): + with self.assertRaises(ValueError): + _strptime._strptime(timestr, directive) + + required_colons_cases = ["-013030", "+0130", "-01:3030.123456"] + for timestr in required_colons_cases: + with self.subTest(timestr=timestr): + with self.assertRaises(ValueError): + _strptime._strptime(timestr, "%:z") + with self.assertRaises(ValueError) as err: _strptime._strptime("-01:3030", "%z") self.assertEqual("Inconsistent use of : in -01:3030", str(err.exception)) + with self.assertRaises(ValueError) as err: + _strptime._strptime("-01:3030", "%:z") + self.assertEqual("Missing colon in %:z before '30', got '-01:3030'", + str(err.exception)) @skip_if_buggy_ucrt_strfptime def test_timezone(self): @@ -480,13 +504,11 @@ def test_bad_timezone(self): # * Year is not included: ha_NG. # * Use non-Gregorian calendar: lo_LA, thai, th_TH. # On Windows: ar_IN, ar_SA, fa_IR, ps_AF. - # - # BUG: Generates regexp that does not match the current date and time - # for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG', 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', - 'my_MM', 'or_IN', 'shn_MM', 'az_IR') + 'my_MM', 'or_IN', 'shn_MM', 'az_IR', + 'byn_ER', 'wal_ET', 'lzh_TW') def test_date_time_locale(self): # Test %c directive loc = locale.getlocale(locale.LC_TIME)[0] @@ -525,11 +547,9 @@ def test_date_time_locale2(self): # NB: Does not roundtrip because use non-Gregorian calendar: # lo_LA, thai, th_TH. On Windows: ar_IN, ar_SA, fa_IR, ps_AF. - # BUG: Generates regexp that does not match the current date - # for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', 'eu_ES', 'ar_AE', - 'az_IR', 'my_MM', 'or_IN', 'shn_MM') + 'az_IR', 'my_MM', 'or_IN', 'shn_MM', 'lzh_TW') def test_date_locale(self): # Test %x directive now = time.time() @@ -546,11 +566,11 @@ def test_date_locale(self): # NB: Dates before 1969 do not roundtrip on many locales, including C. @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', - 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM') + 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM', 'lzh_TW') def test_date_locale2(self): # Test %x directive loc = locale.getlocale(locale.LC_TIME)[0] - if sys.platform.startswith('sunos'): + if sys.platform.startswith(('sunos', 'aix')): if loc in ('en_US', 'de_DE', 'ar_AE'): self.skipTest(f'locale {loc!r} may not work on this platform') self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) @@ -562,11 +582,11 @@ def test_date_locale2(self): # norwegian, nynorsk. # * Hours are in 12-hour notation without AM/PM indication: hy_AM, # ms_MY, sm_WS. - # BUG: Generates regexp that does not match the current time for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'aa_ET', 'am_ET', 'az_IR', 'byn_ER', 'fa_IR', 'gez_ET', 'my_MM', 'om_ET', 'or_IN', 'shn_MM', 'sid_ET', 'so_SO', - 'ti_ET', 'tig_ER', 'wal_ET') + 'ti_ET', 'tig_ER', 'wal_ET', 'lzh_TW', + 'ar_SA', 'bg_BG') def test_time_locale(self): # Test %X directive loc = locale.getlocale(locale.LC_TIME)[0] diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 7df01f28f09..cceecdd526c 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -5,6 +5,7 @@ import math import operator import unittest +import platform import struct import sys import weakref @@ -799,6 +800,23 @@ def test_c_complex_round_trip(self): round_trip = struct.unpack(f, struct.pack(f, z))[0] self.assertComplexesAreIdentical(z, round_trip) + @unittest.skipIf( + support.is_android or support.is_apple_mobile, + "Subinterpreters are not supported on Android and iOS" + ) + def test_endian_table_init_subinterpreters(self): + # Verify that the _struct extension module can be initialized + # concurrently in subinterpreters (gh-140260). + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + raise unittest.SkipTest("InterpreterPoolExecutor not available") + + code = "import struct" + with InterpreterPoolExecutor(max_workers=5) as executor: + results = executor.map(exec, [code] * 5) + self.assertListEqual(list(results), [None] * 5) + class UnpackIteratorTest(unittest.TestCase): """ @@ -917,10 +935,17 @@ def test_half_float(self): # Check that packing produces a bit pattern representing a quiet NaN: # all exponent bits and the msb of the fraction should all be 1. + if platform.machine().startswith('parisc'): + # HP PA RISC uses 0 for quiet, see: + # https://en.wikipedia.org/wiki/NaN#Encoding + expected = 0x7c + else: + expected = 0x7e + packed = struct.pack('<e', math.nan) - self.assertEqual(packed[1] & 0x7e, 0x7e) + self.assertEqual(packed[1] & 0x7e, expected) packed = struct.pack('<e', -math.nan) - self.assertEqual(packed[1] & 0x7e, 0x7e) + self.assertEqual(packed[1] & 0x7e, expected) # Checks for round-to-even behavior format_bits_float__rounding_list = [ diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index d0bc0bd7b61..9622151143c 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -42,7 +42,7 @@ def test_repr(self): # os.stat() gives a complicated struct sequence. st = os.stat(__file__) rep = repr(st) - self.assertTrue(rep.startswith("os.stat_result")) + self.assertStartsWith(rep, "os.stat_result") self.assertIn("st_mode=", rep) self.assertIn("st_ino=", rep) self.assertIn("st_dev=", rep) @@ -307,7 +307,7 @@ def test_copy_replace_with_invisible_fields(self): self.assertEqual(t5.tm_mon, 2) # named invisible fields - self.assertTrue(hasattr(t, 'tm_zone'), f"{t} has no attribute 'tm_zone'") + self.assertHasAttr(t, 'tm_zone') with self.assertRaisesRegex(AttributeError, 'readonly attribute'): t.tm_zone = 'some other zone' self.assertEqual(t2.tm_zone, t.tm_zone) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ca35804fb36..806a1e3fa30 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -957,6 +957,48 @@ def test_communicate(self): self.assertEqual(stdout, b"banana") self.assertEqual(stderr, b"pineapple") + def test_communicate_memoryview_input(self): + # Test memoryview input with byte elements + test_data = b"Hello, memoryview!" + mv = memoryview(test_data) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(mv) + self.assertEqual(stdout, test_data) + self.assertIsNone(stderr) + + def test_communicate_memoryview_input_nonbyte(self): + # Test memoryview input with non-byte elements (e.g., int32) + # This tests the fix for gh-134453 where non-byte memoryviews + # had incorrect length tracking on POSIX + import array + # Create an array of 32-bit integers that's large enough to trigger + # the chunked writing behavior (> PIPE_BUF) + pipe_buf = getattr(select, 'PIPE_BUF', 512) + # Each 'i' element is 4 bytes, so we need more than pipe_buf/4 elements + # Add some extra to ensure we exceed the buffer size + num_elements = pipe_buf + 1 + test_array = array.array('i', [0x64306f66 for _ in range(num_elements)]) + expected_bytes = test_array.tobytes() + mv = memoryview(test_array) + + p = subprocess.Popen([sys.executable, "-c", + 'import sys; ' + 'data = sys.stdin.buffer.read(); ' + 'sys.stdout.buffer.write(data)'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(mv) + self.assertEqual(stdout, expected_bytes, + msg=f"{len(stdout)=} =? {len(expected_bytes)=}") + self.assertIsNone(stderr) + def test_communicate_timeout(self): p = subprocess.Popen([sys.executable, "-c", 'import sys,os,time;' @@ -992,6 +1034,62 @@ def test_communicate_timeout_large_output(self): (stdout, _) = p.communicate() self.assertEqual(len(stdout), 4 * 64 * 1024) + def test_communicate_timeout_large_input(self): + # Test that timeout is enforced when writing large input to a + # slow-to-read subprocess, and that partial input is preserved + # for continuation after timeout (gh-141473). + # + # This is a regression test for Windows matching POSIX behavior. + # On POSIX, select() is used to multiplex I/O with timeout checking. + # On Windows, stdin writing must also honor the timeout rather than + # blocking indefinitely when the pipe buffer fills. + + # Input larger than typical pipe buffer (4-64KB on Windows) + input_data = b"x" * (128 * 1024) + + p = subprocess.Popen( + [sys.executable, "-c", + "import sys, time; " + "time.sleep(30); " # Don't read stdin for a long time + "sys.stdout.buffer.write(sys.stdin.buffer.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + try: + timeout = 0.2 + start = time.monotonic() + try: + p.communicate(input_data, timeout=timeout) + # If we get here without TimeoutExpired, the timeout was ignored + elapsed = time.monotonic() - start + self.fail( + f"TimeoutExpired not raised. communicate() completed in " + f"{elapsed:.2f}s, but subprocess sleeps for 30s. " + "Stdin writing blocked without enforcing timeout.") + except subprocess.TimeoutExpired: + elapsed = time.monotonic() - start + + # Timeout should occur close to the specified timeout value, + # not after waiting for the subprocess to finish sleeping. + # Allow generous margin for slow CI, but must be well under + # the subprocess sleep time. + self.assertLess(elapsed, 5.0, + f"TimeoutExpired raised after {elapsed:.2f}s; expected ~{timeout}s. " + "Stdin writing blocked without checking timeout.") + + # After timeout, continue communication. The remaining input + # should be sent and we should receive all data back. + stdout, stderr = p.communicate() + + # Verify all input was eventually received by the subprocess + self.assertEqual(len(stdout), len(input_data), + f"Expected {len(input_data)} bytes output but got {len(stdout)}") + self.assertEqual(stdout, input_data) + finally: + p.kill() + p.wait() + # Test for the fd leak reported in http://bugs.python.org/issue2791. def test_communicate_pipe_fd_leak(self): for stdin_pipe in (False, True): @@ -1062,6 +1160,19 @@ def test_writes_before_communicate(self): self.assertEqual(stdout, b"bananasplit") self.assertEqual(stderr, b"") + def test_communicate_stdin_closed_before_call(self): + # gh-70560, gh-74389: stdin.close() before communicate() + # should not raise ValueError from stdin.flush() + with subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(0)'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + p.stdin.close() # Close stdin before communicate + # This should not raise ValueError + (stdout, stderr) = p.communicate() + self.assertEqual(p.returncode, 0) + def test_universal_newlines_and_text(self): args = [ sys.executable, "-c", @@ -1178,7 +1289,7 @@ def test_universal_newlines_communicate_stdin_stdout_stderr(self): self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) # Python debug build push something like "[42442 refs]\n" # to stderr at exit of subprocess. - self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + self.assertStartsWith(stderr, "eline2\neline6\neline7\n") def test_universal_newlines_communicate_encodings(self): # Check that universal newlines mode works for various encodings, @@ -1510,7 +1621,7 @@ def test_issue8780(self): "[sys.executable, '-c', 'print(\"Hello World!\")'])", 'assert retcode == 0')) output = subprocess.check_output([sys.executable, '-c', code]) - self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + self.assertStartsWith(output, b'Hello World!') def test_handles_closed_on_exception(self): # If CreateProcess exits with an error, ensure the @@ -1643,6 +1754,40 @@ def test_wait_negative_timeout(self): self.assertEqual(proc.wait(), 0) + def test_post_timeout_communicate_sends_input(self): + """GH-141473 regression test; the stdin pipe must close""" + with subprocess.Popen( + [sys.executable, "-uc", """\ +import sys +while c := sys.stdin.read(512): + sys.stdout.write(c) +print() +"""], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as proc: + try: + data = f"spam{'#'*4096}beans" + proc.communicate( + input=data, + timeout=0, + ) + except subprocess.TimeoutExpired as exc: + pass + # Prior to the bugfix, this would hang as the stdin + # pipe to the child had not been closed. + try: + stdout, stderr = proc.communicate(timeout=15) + except subprocess.TimeoutExpired as exc: + self.fail("communicate() hung waiting on child process that should have seen its stdin pipe close and exit") + self.assertEqual( + proc.returncode, 0, + msg=f"STDERR:\n{stderr}\nSTDOUT:\n{stdout}") + self.assertStartsWith(stdout, "spam") + self.assertIn("beans", stdout) + class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): @@ -1835,8 +1980,8 @@ def test_encoding_warning(self): capture_output=True) lines = cp.stderr.splitlines() self.assertEqual(len(lines), 2, lines) - self.assertTrue(lines[0].startswith(b"<string>:2: EncodingWarning: ")) - self.assertTrue(lines[1].startswith(b"<string>:3: EncodingWarning: ")) + self.assertStartsWith(lines[0], b"<string>:2: EncodingWarning: ") + self.assertStartsWith(lines[1], b"<string>:3: EncodingWarning: ") def _get_test_grp_name(): diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 5cef612a340..193c8b7d7f3 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -547,11 +547,11 @@ def test_special_methods(self): self.assertEqual(s.__reduce__, e.__reduce__) self.assertEqual(s.__reduce_ex__, e.__reduce_ex__) self.assertEqual(s.__getstate__, e.__getstate__) - self.assertFalse(hasattr(s, '__getnewargs__')) - self.assertFalse(hasattr(s, '__getnewargs_ex__')) - self.assertFalse(hasattr(s, '__setstate__')) - self.assertFalse(hasattr(s, '__copy__')) - self.assertFalse(hasattr(s, '__deepcopy__')) + self.assertNotHasAttr(s, '__getnewargs__') + self.assertNotHasAttr(s, '__getnewargs_ex__') + self.assertNotHasAttr(s, '__setstate__') + self.assertNotHasAttr(s, '__copy__') + self.assertNotHasAttr(s, '__deepcopy__') def test_pickling(self): e = E() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 8446da03e36..667fcc81d8e 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -1,6 +1,8 @@ import contextlib import errno import importlib +import itertools +import inspect import io import logging import os @@ -17,6 +19,7 @@ import warnings from test import support +from test.support import hashlib_helper from test.support import import_helper from test.support import os_helper from test.support import script_helper @@ -93,9 +96,15 @@ def test_get_attribute(self): self.test_get_attribute) self.assertRaises(unittest.SkipTest, support.get_attribute, self, "foo") - @unittest.skip("failing buildbots") + @unittest.skipIf(support.is_android or support.is_apple_mobile, + 'Mobile platforms redirect stdout to system log') def test_get_original_stdout(self): - self.assertEqual(support.get_original_stdout(), sys.stdout) + if isinstance(sys.stdout, io.StringIO): + # gh-55258: When --junit-xml is used, stdout is a StringIO: + # use sys.__stdout__ in this case. + self.assertEqual(support.get_original_stdout(), sys.__stdout__) + else: + self.assertEqual(support.get_original_stdout(), sys.stdout) def test_unload(self): import sched # noqa: F401 @@ -407,10 +416,10 @@ class Obj: with support.swap_attr(obj, "y", 5) as y: self.assertEqual(obj.y, 5) self.assertIsNone(y) - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') with support.swap_attr(obj, "y", 5): del obj.y - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') def test_swap_item(self): D = {"x":1} @@ -482,6 +491,7 @@ def test_check__all__(self): self.assertRaises(AssertionError, support.check__all__, self, unittest) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @unittest.skipUnless(hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG'), 'need os.waitpid() and os.WNOHANG') @support.requires_fork() @@ -784,14 +794,14 @@ def test_get_signal_name(self): def test_linked_to_musl(self): linked = support.linked_to_musl() self.assertIsNotNone(linked) - if support.is_wasi or support.is_emscripten: + if support.is_wasm32: self.assertTrue(linked) # The value is cached, so make sure it returns the same value again. self.assertIs(linked, support.linked_to_musl()) - # The unlike libc, the musl version is a triple. + # The musl version is either triple or just a major version number. if linked: self.assertIsInstance(linked, tuple) - self.assertEqual(3, len(linked)) + self.assertIn(len(linked), (1, 3)) for v in linked: self.assertIsInstance(v, int) @@ -818,5 +828,238 @@ def test_linked_to_musl(self): # SuppressCrashReport +@hashlib_helper.requires_builtin_hashes() +class TestHashlibSupport(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.hashlib = import_helper.import_module("hashlib") + cls.hmac = import_helper.import_module("hmac") + + # All C extension modules must be present since blocking + # the built-in implementation while allowing OpenSSL or vice-versa + # may result in failures depending on the exposed built-in hashes. + cls._hashlib = import_helper.import_module("_hashlib") + cls._hmac = import_helper.import_module("_hmac") + cls._md5 = import_helper.import_module("_md5") + + def skip_if_fips_mode(self): + if self._hashlib.get_fips_mode(): + self.skipTest("disabled in FIPS mode") + + def skip_if_not_fips_mode(self): + if not self._hashlib.get_fips_mode(): + self.skipTest("requires FIPS mode") + + def check_context(self, disabled=True): + if disabled: + return self.assertRaises(ValueError) + return contextlib.nullcontext() + + def try_import_attribute(self, fullname, default=None): + if fullname is None: + return default + assert fullname.count('.') == 1, fullname + module_name, attribute = fullname.split('.', maxsplit=1) + try: + module = importlib.import_module(module_name) + except ImportError: + return default + try: + return getattr(module, attribute, default) + except TypeError: + return default + + def fetch_hash_function(self, name, implementation): + info = hashlib_helper.get_hash_func_info(name) + match hashlib_helper.Implementation(implementation): + case hashlib_helper.Implementation.hashlib: + method_name = info.hashlib.member_name + assert isinstance(method_name, str), method_name + return getattr(self.hashlib, method_name) + case hashlib_helper.Implementation.openssl: + method_name = info.openssl.member_name + assert isinstance(method_name, str | None), method_name + return getattr(self._hashlib, method_name or "", None) + fullname = info[implementation].fullname + return self.try_import_attribute(fullname) + + def fetch_hmac_function(self, name): + target = hashlib_helper.get_hmac_item_info(name) + return target.import_member() + + def check_openssl_hash(self, name, *, disabled=True): + """Check that OpenSSL HASH interface is enabled/disabled.""" + with self.check_context(disabled): + _ = self._hashlib.new(name) + if do_hash := self.fetch_hash_function(name, "openssl"): + self.assertStartsWith(do_hash.__name__, 'openssl_') + with self.check_context(disabled): + _ = do_hash(b"") + + def check_openssl_hmac(self, name, *, disabled=True): + """Check that OpenSSL HMAC interface is enabled/disabled.""" + if name in hashlib_helper.NON_HMAC_DIGEST_NAMES: + # HMAC-BLAKE and HMAC-SHAKE raise a ValueError as they are not + # supported at all (they do not make any sense in practice). + with self.assertRaises(ValueError): + self._hashlib.hmac_digest(b"", b"", name) + else: + with self.check_context(disabled): + _ = self._hashlib.hmac_digest(b"", b"", name) + # OpenSSL does not provide one-shot explicit HMAC functions + + def check_builtin_hash(self, name, *, disabled=True): + """Check that HACL* HASH interface is enabled/disabled.""" + if do_hash := self.fetch_hash_function(name, "builtin"): + self.assertEqual(do_hash.__name__, name) + with self.check_context(disabled): + _ = do_hash(b"") + + def check_builtin_hmac(self, name, *, disabled=True): + """Check that HACL* HMAC interface is enabled/disabled.""" + if name in hashlib_helper.NON_HMAC_DIGEST_NAMES: + # HMAC-BLAKE and HMAC-SHAKE raise a ValueError as they are not + # supported at all (they do not make any sense in practice). + with self.assertRaises(ValueError): + self._hmac.compute_digest(b"", b"", name) + else: + with self.check_context(disabled): + _ = self._hmac.compute_digest(b"", b"", name) + + with self.check_context(disabled): + _ = self._hmac.new(b"", b"", name) + + if do_hmac := self.fetch_hmac_function(name): + self.assertStartsWith(do_hmac.__name__, 'compute_') + with self.check_context(disabled): + _ = do_hmac(b"", b"") + else: + self.assertIn(name, hashlib_helper.NON_HMAC_DIGEST_NAMES) + + @support.subTests( + ('name', 'allow_openssl', 'allow_builtin'), + itertools.product( + hashlib_helper.CANONICAL_DIGEST_NAMES, + [True, False], + [True, False], + ) + ) + def test_disable_hash(self, name, allow_openssl, allow_builtin): + # In FIPS mode, the function may be available but would still need + # to raise a ValueError, so we will test the helper separately. + self.skip_if_fips_mode() + flags = dict(allow_openssl=allow_openssl, allow_builtin=allow_builtin) + is_fully_disabled = not allow_builtin and not allow_openssl + + with hashlib_helper.block_algorithm(name, **flags): + # OpenSSL's blake2s and blake2b are unknown names + # when only the OpenSSL interface is available. + if allow_openssl and not allow_builtin: + aliases = {'blake2s': 'blake2s256', 'blake2b': 'blake2b512'} + name_for_hashlib_new = aliases.get(name, name) + else: + name_for_hashlib_new = name + + with self.check_context(is_fully_disabled): + _ = self.hashlib.new(name_for_hashlib_new) + + # Since _hashlib is present, explicit blake2b/blake2s constructors + # use the built-in implementation, while others (since we are not + # in FIPS mode and since _hashlib exists) use the OpenSSL function. + with self.check_context(is_fully_disabled): + _ = getattr(self.hashlib, name)() + + self.check_openssl_hash(name, disabled=not allow_openssl) + self.check_builtin_hash(name, disabled=not allow_builtin) + + if name not in hashlib_helper.NON_HMAC_DIGEST_NAMES: + with self.check_context(is_fully_disabled): + _ = self.hmac.new(b"", b"", name) + with self.check_context(is_fully_disabled): + _ = self.hmac.HMAC(b"", b"", name) + with self.check_context(is_fully_disabled): + _ = self.hmac.digest(b"", b"", name) + + self.check_openssl_hmac(name, disabled=not allow_openssl) + self.check_builtin_hmac(name, disabled=not allow_builtin) + + @hashlib_helper.block_algorithm("md5") + def test_disable_hash_md5_in_fips_mode(self): + self.skip_if_not_fips_mode() + + self.assertRaises(ValueError, self.hashlib.new, "md5") + self.assertRaises(ValueError, self._hashlib.new, "md5") + self.assertRaises(ValueError, self.hashlib.md5) + self.assertRaises(ValueError, self._hashlib.openssl_md5) + + kwargs = dict(usedforsecurity=True) + self.assertRaises(ValueError, self.hashlib.new, "md5", **kwargs) + self.assertRaises(ValueError, self._hashlib.new, "md5", **kwargs) + self.assertRaises(ValueError, self.hashlib.md5, **kwargs) + self.assertRaises(ValueError, self._hashlib.openssl_md5, **kwargs) + + @hashlib_helper.block_algorithm("md5", allow_openssl=True) + def test_disable_hash_md5_in_fips_mode_allow_openssl(self): + self.skip_if_not_fips_mode() + # Allow the OpenSSL interface to be used but not the HACL* one. + # hashlib.new("md5") is dispatched to hashlib.openssl_md5() + self.assertRaises(ValueError, self.hashlib.new, "md5") + # dispatched to hashlib.openssl_md5() in FIPS mode + h2 = self.hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h2, self._hashlib.HASH) + + # block_algorithm() does not mock hashlib.md5 and _hashlib.openssl_md5 + self.assertNotHasAttr(self.hashlib.md5, "__wrapped__") + self.assertNotHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + hashlib_md5 = inspect.unwrap(self.hashlib.md5) + self.assertIs(hashlib_md5, self._hashlib.openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + # allow MD5 to be used in FIPS mode if usedforsecurity=False + h3 = self.hashlib.md5(usedforsecurity=False) + self.assertIsInstance(h3, self._hashlib.HASH) + + @hashlib_helper.block_algorithm("md5", allow_builtin=True) + def test_disable_hash_md5_in_fips_mode_allow_builtin(self): + self.skip_if_not_fips_mode() + # Allow the HACL* interface to be used but not the OpenSSL one. + h1 = self.hashlib.new("md5") # dispatched to _md5.md5() + self.assertNotIsInstance(h1, self._hashlib.HASH) + h2 = self.hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h2, type(h1)) + + # block_algorithm() mocks hashlib.md5 and _hashlib.openssl_md5 + self.assertHasAttr(self.hashlib.md5, "__wrapped__") + self.assertHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + hashlib_md5 = inspect.unwrap(self.hashlib.md5) + openssl_md5 = inspect.unwrap(self._hashlib.openssl_md5) + self.assertIs(hashlib_md5, openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + self.assertRaises(ValueError, self.hashlib.md5, + usedforsecurity=False) + + @hashlib_helper.block_algorithm("md5", + allow_openssl=True, + allow_builtin=True) + def test_disable_hash_md5_in_fips_mode_allow_all(self): + self.skip_if_not_fips_mode() + # hashlib.new() isn't blocked as it falls back to _md5.md5 + self.assertIsInstance(self.hashlib.new("md5"), self._md5.MD5Type) + self.assertRaises(ValueError, self._hashlib.new, "md5") + h = self._hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h, self._hashlib.HASH) + + self.assertNotHasAttr(self.hashlib.md5, "__wrapped__") + self.assertNotHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + self.assertIs(self.hashlib.md5, self._hashlib.openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + h = self.hashlib.md5(usedforsecurity=False) + self.assertIsInstance(h, self._hashlib.HASH) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 24d89b09d94..094ab8f573e 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -5,6 +5,7 @@ import re import textwrap import symtable +import warnings import unittest from test import support @@ -527,6 +528,89 @@ def test_symtable_entry_repr(self): expected = f"<symtable entry top({self.top.get_id()}), line {self.top.get_lineno()}>" self.assertEqual(repr(self.top._table), expected) + def test_lambda(self): + st = symtable.symtable("lambda x: x", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<lambda>") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["x"]) + self.assertEqual(st.get_children(), []) + + def test_nested_lambda(self): + st = symtable.symtable("lambda x: lambda y=x: y", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<lambda>") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["x"]) + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<lambda>") + self.assertTrue(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["y"]) + self.assertEqual(st.get_children(), []) + + def test_genexpr(self): + st = symtable.symtable("(x for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<genexpr>") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + self.assertEqual(st.get_children(), []) + + def test_nested_genexpr(self): + st = symtable.symtable("((y for y in x) for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<genexpr>") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<genexpr>") + self.assertTrue(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "y"]) + self.assertEqual(st.get_children(), []) + + def test__symtable_refleak(self): + # Regression test for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'compile_type' AC conversion failed + self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1) + + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + symtable.symtable(source, filename, 'exec') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + symtable.symtable(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 0ee17849e28..ade8f273a1e 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -370,18 +370,55 @@ Traceback (most recent call last): SyntaxError: invalid syntax ->>> match ...: -... case {**rest, "key": value}: -... ... -Traceback (most recent call last): -SyntaxError: invalid syntax - >>> match ...: ... case {**_}: ... ... Traceback (most recent call last): SyntaxError: invalid syntax +# Check incorrect "case" placement with specialized error messages + +>>> case "pattern": ... +Traceback (most recent call last): +SyntaxError: case statement must be inside match statement + +>>> case 1 | 2: ... +Traceback (most recent call last): +SyntaxError: case statement must be inside match statement + +>>> case klass(attr=1) | {}: ... +Traceback (most recent call last): +SyntaxError: case statement must be inside match statement + +>>> case [] if x > 1: ... +Traceback (most recent call last): +SyntaxError: case statement must be inside match statement + +>>> case match: ... +Traceback (most recent call last): +SyntaxError: case statement must be inside match statement + +>>> case case: ... +Traceback (most recent call last): +SyntaxError: case statement must be inside match statement + +>>> if some: +... case 1: ... +Traceback (most recent call last): +SyntaxError: case statement must be inside match statement + +>>> case some: +... case 1: ... +Traceback (most recent call last): +SyntaxError: case statement must be inside match statement + +# But prefixes of soft keywords should +# still raise specialized errors + +>>> (mat x) +Traceback (most recent call last): +SyntaxError: invalid syntax. Perhaps you forgot a comma? + From compiler_complex_args(): >>> def f(None=1): @@ -419,7 +456,7 @@ >>> def foo(/,a,b=,c): ... pass Traceback (most recent call last): -SyntaxError: at least one argument must precede / +SyntaxError: at least one parameter must precede / >>> def foo(a,/,/,b,c): ... pass @@ -454,67 +491,67 @@ >>> def foo(a,*b=3,c): ... pass Traceback (most recent call last): -SyntaxError: var-positional argument cannot have default value +SyntaxError: var-positional parameter cannot have default value >>> def foo(a,*b: int=,c): ... pass Traceback (most recent call last): -SyntaxError: var-positional argument cannot have default value +SyntaxError: var-positional parameter cannot have default value >>> def foo(a,**b=3): ... pass Traceback (most recent call last): -SyntaxError: var-keyword argument cannot have default value +SyntaxError: var-keyword parameter cannot have default value >>> def foo(a,**b: int=3): ... pass Traceback (most recent call last): -SyntaxError: var-keyword argument cannot have default value +SyntaxError: var-keyword parameter cannot have default value >>> def foo(a,*a, b, **c, d): ... pass Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> def foo(a,*a, b, **c, d=4): ... pass Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> def foo(a,*a, b, **c, *d): ... pass Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> def foo(a,*a, b, **c, **d): ... pass Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> def foo(a=1,/,**b,/,c): ... pass Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> def foo(*b,*d): ... pass Traceback (most recent call last): -SyntaxError: * argument may appear only once +SyntaxError: * may appear only once >>> def foo(a,*b,c,*d,*e,c): ... pass Traceback (most recent call last): -SyntaxError: * argument may appear only once +SyntaxError: * may appear only once >>> def foo(a,b,/,c,*b,c,*d,*e,c): ... pass Traceback (most recent call last): -SyntaxError: * argument may appear only once +SyntaxError: * may appear only once >>> def foo(a,b,/,c,*b,c,*d,**e): ... pass Traceback (most recent call last): -SyntaxError: * argument may appear only once +SyntaxError: * may appear only once >>> def foo(a=1,/*,b,c): ... pass @@ -538,7 +575,7 @@ >>> lambda /,a,b,c: None Traceback (most recent call last): -SyntaxError: at least one argument must precede / +SyntaxError: at least one parameter must precede / >>> lambda a,/,/,b,c: None Traceback (most recent call last): @@ -570,47 +607,47 @@ >>> lambda a,*b=3,c: None Traceback (most recent call last): -SyntaxError: var-positional argument cannot have default value +SyntaxError: var-positional parameter cannot have default value >>> lambda a,**b=3: None Traceback (most recent call last): -SyntaxError: var-keyword argument cannot have default value +SyntaxError: var-keyword parameter cannot have default value >>> lambda a, *a, b, **c, d: None Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> lambda a,*a, b, **c, d=4: None Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> lambda a,*a, b, **c, *d: None Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> lambda a,*a, b, **c, **d: None Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> lambda a=1,/,**b,/,c: None Traceback (most recent call last): -SyntaxError: arguments cannot follow var-keyword argument +SyntaxError: parameters cannot follow var-keyword parameter >>> lambda *b,*d: None Traceback (most recent call last): -SyntaxError: * argument may appear only once +SyntaxError: * may appear only once >>> lambda a,*b,c,*d,*e,c: None Traceback (most recent call last): -SyntaxError: * argument may appear only once +SyntaxError: * may appear only once >>> lambda a,b,/,c,*b,c,*d,*e,c: None Traceback (most recent call last): -SyntaxError: * argument may appear only once +SyntaxError: * may appear only once >>> lambda a,b,/,c,*b,c,*d,**e: None Traceback (most recent call last): -SyntaxError: * argument may appear only once +SyntaxError: * may appear only once >>> lambda a=1,d=,c: None Traceback (most recent call last): @@ -1304,7 +1341,7 @@ Traceback (most recent call last): SyntaxError: expected '(' -Parenthesized arguments in function definitions +Parenthesized parameters in function definitions >>> def f(x, (y, z), w): ... pass @@ -1431,6 +1468,23 @@ Traceback (most recent call last): SyntaxError: cannot use except* statement with literal +Regression tests for gh-133999: + + >>> try: pass + ... except TypeError as name: raise from None + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + + >>> try: pass + ... except* TypeError as name: raise from None + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + + >>> match 1: + ... case 1 | 2 as abc: raise from None + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + Ensure that early = are not matched by the parser as invalid comparisons >>> f(2, 4, x=34); 1 $ 2 Traceback (most recent call last): @@ -1678,6 +1732,28 @@ ... SyntaxError: invalid syntax +Better errors for `raise` statement: + + >>> raise ValueError from + Traceback (most recent call last): + SyntaxError: did you forget an expression after 'from'? + + >>> raise mod.ValueError() from + Traceback (most recent call last): + SyntaxError: did you forget an expression after 'from'? + + >>> raise from exc + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + + >>> raise from None + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + + >>> raise from + Traceback (most recent call last): + SyntaxError: did you forget an expression between 'raise' and 'from'? + Check that an multiple exception types with missing parentheses raise a custom exception only when using 'as' @@ -2087,6 +2163,25 @@ Traceback (most recent call last): SyntaxError: cannot use subscript as import target +# Check that we don't raise a "cannot use name as import target" error +# if there is an error in an unrelated statement after ';' + +>>> import a as b; None = 1 +Traceback (most recent call last): +SyntaxError: cannot assign to None + +>>> import a, b as c; d = 1; None = 1 +Traceback (most recent call last): +SyntaxError: cannot assign to None + +>>> from a import b as c; None = 1 +Traceback (most recent call last): +SyntaxError: cannot assign to None + +>>> from a import b, c as d; e = 1; None = 1 +Traceback (most recent call last): +SyntaxError: cannot assign to None + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. @@ -2178,7 +2273,7 @@ >>> with (lambda *:0): pass Traceback (most recent call last): - SyntaxError: named arguments must follow bare * + SyntaxError: named parameters must follow bare * Corner-cases that used to crash: @@ -2194,7 +2289,7 @@ Traceback (most recent call last): SyntaxError: invalid character '£' (U+00A3) - Invalid pattern matching constructs: +Invalid pattern matching constructs: >>> match ...: ... case 42 as _: @@ -2256,6 +2351,24 @@ Traceback (most recent call last): SyntaxError: positional patterns follow keyword patterns + >>> match ...: + ... case {**double_star, "spam": "eggs"}: + ... ... + Traceback (most recent call last): + SyntaxError: double star pattern must be the last (right-most) subpattern in the mapping pattern + + >>> match ...: + ... case {"foo": 1, **double_star, "spam": "eggs"}: + ... ... + Traceback (most recent call last): + SyntaxError: double star pattern must be the last (right-most) subpattern in the mapping pattern + + >>> match ...: + ... case {"spam": "eggs", "b": {**d, "ham": "bacon"}}: + ... ... + Traceback (most recent call last): + SyntaxError: double star pattern must be the last (right-most) subpattern in the mapping pattern + Uses of the star operator which should fail: A[:*b] @@ -2640,6 +2753,76 @@ def f(x: *b) >>> f(x = 5, *:) Traceback (most recent call last): SyntaxError: Invalid star expression + +Asserts: + + >>> assert (a := 1) # ok + >>> assert 1, (a := 1) # ok + + >>> assert a := 1 + Traceback (most recent call last): + SyntaxError: cannot use named expression without parentheses here + + >>> assert 1, a := 1 + Traceback (most recent call last): + SyntaxError: cannot use named expression without parentheses here + + >>> assert 1 = 2 = 3 + Traceback (most recent call last): + SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='? + + >>> assert 1 = 2 + Traceback (most recent call last): + SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='? + + >>> assert (1 = 2) + Traceback (most recent call last): + SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='? + + >>> assert 'a' = a + Traceback (most recent call last): + SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='? + + >>> assert x[0] = 1 + Traceback (most recent call last): + SyntaxError: cannot assign to subscript here. Maybe you meant '==' instead of '='? + + >>> assert (yield a) = 2 + Traceback (most recent call last): + SyntaxError: cannot assign to yield expression here. Maybe you meant '==' instead of '='? + + >>> assert a = 2 + Traceback (most recent call last): + SyntaxError: cannot assign to name here. Maybe you meant '==' instead of '='? + + >>> assert (a = 2) + Traceback (most recent call last): + SyntaxError: invalid syntax. Maybe you meant '==' or ':=' instead of '='? + + >>> assert a = b + Traceback (most recent call last): + SyntaxError: cannot assign to name here. Maybe you meant '==' instead of '='? + + >>> assert 1, 1 = b + Traceback (most recent call last): + SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='? + + >>> assert 1, (1 = b) + Traceback (most recent call last): + SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='? + + >>> assert 1, a = 1 + Traceback (most recent call last): + SyntaxError: cannot assign to name here. Maybe you meant '==' instead of '='? + + >>> assert 1, (a = 1) + Traceback (most recent call last): + SyntaxError: invalid syntax. Maybe you meant '==' or ':=' instead of '='? + + >>> assert 1 = a, a = 1 + Traceback (most recent call last): + SyntaxError: cannot assign to literal here. Maybe you meant '==' instead of '='? + """ import re @@ -2826,6 +3009,13 @@ def error2(): """ self._check_error(source, "parameter and nonlocal", lineno=3) + def test_raise_from_error_message(self): + source = """if 1: + raise AssertionError() from None + print(1,,2) + """ + self._check_error(source, "invalid syntax", lineno=3) + def test_yield_outside_function(self): self._check_error("if 0: yield", "outside function") self._check_error("if 0: yield\nelse: x=1", "outside function") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 59ef5c99309..04018e9603f 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -24,7 +24,7 @@ from test.support import force_not_colorized from test.support import SHORT_TIMEOUT try: - from test.support import interpreters + from concurrent import interpreters except ImportError: interpreters = None import textwrap @@ -57,7 +57,7 @@ def test_original_displayhook(self): dh(None) self.assertEqual(out.getvalue(), "") - self.assertTrue(not hasattr(builtins, "_")) + self.assertNotHasAttr(builtins, "_") # sys.displayhook() requires arguments self.assertRaises(TypeError, dh) @@ -172,7 +172,7 @@ def test_original_excepthook(self): with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) - self.assertTrue(err.getvalue().endswith("ValueError: 42\n")) + self.assertEndsWith(err.getvalue(), "ValueError: 42\n") self.assertRaises(TypeError, sys.__excepthook__) @@ -192,7 +192,7 @@ def test_excepthook_bytes_filename(self): err = err.getvalue() self.assertIn(""" File "b'bytes_filename'", line 123\n""", err) self.assertIn(""" text\n""", err) - self.assertTrue(err.endswith("SyntaxError: msg\n")) + self.assertEndsWith(err, "SyntaxError: msg\n") def test_excepthook(self): with test.support.captured_output("stderr") as stderr: @@ -269,8 +269,7 @@ def check_exit_message(code, expected, **env_vars): rc, out, err = assert_python_failure('-c', code, **env_vars) self.assertEqual(rc, 1) self.assertEqual(out, b'') - self.assertTrue(err.startswith(expected), - "%s doesn't start with %s" % (ascii(err), ascii(expected))) + self.assertStartsWith(err, expected) # test that stderr buffer is flushed before the exit message is written # into stderr @@ -437,7 +436,7 @@ def test_call_tracing(self): @unittest.skipUnless(hasattr(sys, "setdlopenflags"), 'test needs sys.setdlopenflags()') def test_dlopenflags(self): - self.assertTrue(hasattr(sys, "getdlopenflags")) + self.assertHasAttr(sys, "getdlopenflags") self.assertRaises(TypeError, sys.getdlopenflags, 42) oldflags = sys.getdlopenflags() self.assertRaises(TypeError, sys.setdlopenflags) @@ -623,8 +622,7 @@ def g456(): # And the next record must be for g456(). filename, lineno, funcname, sourceline = stack[i+1] self.assertEqual(funcname, "g456") - self.assertTrue((sourceline.startswith("if leave_g.wait(") or - sourceline.startswith("g_raised.set()"))) + self.assertStartsWith(sourceline, ("if leave_g.wait(", "g_raised.set()")) finally: # Reap the spawned thread. leave_g.set() @@ -731,15 +729,25 @@ def test_thread_info(self): info = sys.thread_info self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) - self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) - if sys.platform.startswith(("linux", "android", "freebsd")): + self.assertIn(info.lock, ('pymutex', None)) + if sys.platform.startswith(("linux", "android", "freebsd", "wasi")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": self.assertEqual(info.name, "nt") elif sys.platform == "emscripten": self.assertIn(info.name, {"pthread", "pthread-stubs"}) - elif sys.platform == "wasi": - self.assertEqual(info.name, "pthread-stubs") + + def test_abi_info(self): + info = sys.abi_info + info_keys = {'pointer_bits', 'free_threaded', 'debug', 'byteorder'} + self.assertEqual(set(vars(info)), info_keys) + pointer_bits = 64 if sys.maxsize > 2**32 else 32 + self.assertEqual(info.pointer_bits, pointer_bits) + self.assertEqual(info.free_threaded, + bool(sysconfig.get_config_var('Py_GIL_DISABLED'))) + self.assertEqual(info.debug, + bool(sysconfig.get_config_var('Py_DEBUG'))) + self.assertEqual(info.byteorder, sys.byteorder) @unittest.skipUnless(support.is_emscripten, "only available on Emscripten") def test_emscripten_info(self): @@ -860,7 +868,7 @@ def test_sys_flags(self): "hash_randomization", "isolated", "dev_mode", "utf8_mode", "warn_default_encoding", "safe_path", "int_max_str_digits") for attr in attrs: - self.assertTrue(hasattr(sys.flags, attr), attr) + self.assertHasAttr(sys.flags, attr) attr_type = bool if attr in ("dev_mode", "safe_path") else int self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) self.assertTrue(repr(sys.flags)) @@ -871,12 +879,7 @@ def test_sys_flags(self): def assert_raise_on_new_sys_type(self, sys_attr): # Users are intentionally prevented from creating new instances of # sys.flags, sys.version_info, and sys.getwindowsversion. - arg = sys_attr - attr_type = type(sys_attr) - with self.assertRaises(TypeError): - attr_type(arg) - with self.assertRaises(TypeError): - attr_type.__new__(attr_type, arg) + support.check_disallow_instantiation(self, type(sys_attr), sys_attr) def test_sys_flags_no_instantiation(self): self.assert_raise_on_new_sys_type(sys.flags) @@ -1072,10 +1075,11 @@ def test_implementation(self): levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF} - self.assertTrue(hasattr(sys.implementation, 'name')) - self.assertTrue(hasattr(sys.implementation, 'version')) - self.assertTrue(hasattr(sys.implementation, 'hexversion')) - self.assertTrue(hasattr(sys.implementation, 'cache_tag')) + self.assertHasAttr(sys.implementation, 'name') + self.assertHasAttr(sys.implementation, 'version') + self.assertHasAttr(sys.implementation, 'hexversion') + self.assertHasAttr(sys.implementation, 'cache_tag') + self.assertHasAttr(sys.implementation, 'supports_isolated_interpreters') version = sys.implementation.version self.assertEqual(version[:2], (version.major, version.minor)) @@ -1089,6 +1093,15 @@ def test_implementation(self): self.assertEqual(sys.implementation.name, sys.implementation.name.lower()) + # https://peps.python.org/pep-0734 + sii = sys.implementation.supports_isolated_interpreters + self.assertIsInstance(sii, bool) + if test.support.check_impl_detail(cpython=True): + if test.support.is_emscripten or test.support.is_wasi: + self.assertFalse(sii) + else: + self.assertTrue(sii) + @test.support.cpython_only def test_debugmallocstats(self): # Test sys._debugmallocstats() @@ -1137,23 +1150,12 @@ def test_getallocatedblocks(self): b = sys.getallocatedblocks() self.assertLessEqual(b, a) try: - # While we could imagine a Python session where the number of - # multiple buffer objects would exceed the sharing of references, - # it is unlikely to happen in a normal test run. - # - # In free-threaded builds each code object owns an array of - # pointers to copies of the bytecode. When the number of - # code objects is a large fraction of the total number of - # references, this can cause the total number of allocated - # blocks to exceed the total number of references. - # - # For some reason, iOS seems to trigger the "unlikely to happen" - # case reliably under CI conditions. It's not clear why; but as - # this test is checking the behavior of getallocatedblock() - # under garbage collection, we can skip this pre-condition check - # for now. See GH-130384. - if not support.Py_GIL_DISABLED and not support.is_apple_mobile: - self.assertLess(a, sys.gettotalrefcount()) + # The reported blocks will include immortalized strings, but the + # total ref count will not. This will sanity check that among all + # other objects (those eligible for garbage collection) there + # are more references being tracked than allocated blocks. + interned_immortal = sys.getunicodeinternedsize(_only_immortal=True) + self.assertLess(a - interned_immortal, sys.gettotalrefcount()) except AttributeError: # gettotalrefcount() not available pass @@ -1301,6 +1303,7 @@ def test_module_names(self): for name in sys.stdlib_module_names: self.assertIsInstance(name, str) + @unittest.skipUnless(hasattr(sys, '_stdlib_dir'), 'need sys._stdlib_dir') def test_stdlib_dir(self): os = import_helper.import_fresh_module('os') marker = getattr(os, '__file__', None) @@ -1347,6 +1350,7 @@ def test_disable_gil_abi(self): @test.support.cpython_only +@force_not_colorized class UnraisableHookTest(unittest.TestCase): def test_original_unraisablehook(self): _testcapi = import_helper.import_module('_testcapi') @@ -1419,7 +1423,7 @@ def __del__(self): else: self.assertIn("ValueError", report) self.assertIn("del is broken", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") def test_original_unraisablehook_exception_qualname(self): # See bpo-41031, bpo-45083. @@ -1577,7 +1581,7 @@ def test_objecttypes(self): samples = [b'', b'u'*100000] for sample in samples: x = bytearray(sample) - check(x, vsize('n2Pi') + x.__alloc__()) + check(x, vsize('n2PiP') + x.__alloc__()) # bytearray_iterator check(iter(bytearray()), size('nP')) # bytes @@ -1719,9 +1723,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module if support.Py_GIL_DISABLED: - check(unittest, size('PPPPPP')) + md_gil = '?' else: - check(unittest, size('PPPPP')) + md_gil = '' + check(unittest, size('PPPP?' + md_gil + 'NPPPPP')) # None check(None, size('')) # NotImplementedType @@ -1955,33 +1960,19 @@ def write(self, s): self.assertEqual(out, b"") self.assertEqual(err, b"") - -def _supports_remote_attaching(): - PROCESS_VM_READV_SUPPORTED = False - - try: - from _remote_debugging import PROCESS_VM_READV_SUPPORTED - except ImportError: - pass - - return PROCESS_VM_READV_SUPPORTED - -@unittest.skipIf(not sys.is_remote_debug_enabled(), "Remote debugging is not enabled") -@unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32", - "Test only runs on Linux, Windows and MacOS") -@unittest.skipIf(sys.platform == "linux" and not _supports_remote_attaching(), - "Test only runs on Linux with process_vm_readv support") +@test.support.support_remote_exec_only @test.support.cpython_only class TestRemoteExec(unittest.TestCase): def tearDown(self): test.support.reap_children() - def _run_remote_exec_test(self, script_code, python_args=None, env=None, prologue=''): + def _run_remote_exec_test(self, script_code, python_args=None, env=None, + prologue='', + script_path=os_helper.TESTFN + '_remote.py'): # Create the script that will be remotely executed - script = os_helper.TESTFN + '_remote.py' - self.addCleanup(os_helper.unlink, script) + self.addCleanup(os_helper.unlink, script_path) - with open(script, 'w') as f: + with open(script_path, 'w') as f: f.write(script_code) # Create and run the target process @@ -2050,7 +2041,7 @@ def _run_remote_exec_test(self, script_code, python_args=None, env=None, prologu self.assertEqual(response, b"ready") # Try remote exec on the target process - sys.remote_exec(proc.pid, script) + sys.remote_exec(proc.pid, script_path) # Signal script to continue client_socket.sendall(b"continue") @@ -2073,14 +2064,32 @@ def _run_remote_exec_test(self, script_code, python_args=None, env=None, prologu def test_remote_exec(self): """Test basic remote exec functionality""" - script = ''' -print("Remote script executed successfully!") -''' + script = 'print("Remote script executed successfully!")' returncode, stdout, stderr = self._run_remote_exec_test(script) # self.assertEqual(returncode, 0) self.assertIn(b"Remote script executed successfully!", stdout) self.assertEqual(stderr, b"") + def test_remote_exec_bytes(self): + script = 'print("Remote script executed successfully!")' + script_path = os.fsencode(os_helper.TESTFN) + b'_bytes_remote.py' + returncode, stdout, stderr = self._run_remote_exec_test(script, + script_path=script_path) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, 'requires undecodable path') + @unittest.skipIf(sys.platform == 'darwin', + 'undecodable paths are not supported on macOS') + def test_remote_exec_undecodable(self): + script = 'print("Remote script executed successfully!")' + script_path = os_helper.TESTFN_UNDECODABLE + b'_undecodable_remote.py' + for script_path in [script_path, os.fsdecode(script_path)]: + returncode, stdout, stderr = self._run_remote_exec_test(script, + script_path=script_path) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + def test_remote_exec_with_self_process(self): """Test remote exec with the target process being the same as the test process""" @@ -2110,7 +2119,7 @@ def audit_hook(event, arg): returncode, stdout, stderr = self._run_remote_exec_test(script, prologue=prologue) self.assertEqual(returncode, 0) self.assertIn(b"Remote script executed successfully!", stdout) - self.assertIn(b"Audit event: remote_debugger_script, arg: ", stdout) + self.assertIn(b"Audit event: cpython.remote_debugger_script, arg: ", stdout) self.assertEqual(stderr, b"") def test_remote_exec_with_exception(self): @@ -2157,6 +2166,13 @@ def test_remote_exec_invalid_pid(self): with self.assertRaises(OSError): sys.remote_exec(99999, "print('should not run')") + def test_remote_exec_invalid_script(self): + """Test remote exec with invalid script type""" + with self.assertRaises(TypeError): + sys.remote_exec(0, None) + with self.assertRaises(TypeError): + sys.remote_exec(0, 123) + def test_remote_exec_syntax_error(self): """Test remote exec with syntax error in script""" script = ''' @@ -2235,9 +2251,10 @@ def frame_2_jit(expected: bool) -> None: def frame_3_jit() -> None: # JITs just before the last loop: - for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + # 1 extra iteration for tracing. + for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i == _testinternalcapi.TIER2_THRESHOLD + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index 345c022bd23..a22b728b569 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -272,6 +272,8 @@ def g(p): self.check_events(g, [(1, 'call', g_ident, None), (2, 'call', f_ident, None), (2, 'return', f_ident, 0), + (2, 'call', f_ident, None), + (2, 'return', f_ident, None), (1, 'return', g_ident, None), ], check_args=True) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index b3685a91c57..199a9087dfe 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -360,6 +360,8 @@ class TraceTestCase(unittest.TestCase): # Disable gc collection when tracing, otherwise the # deallocators may be traced as well. def setUp(self): + if os.environ.get('PYTHON_UOPS_OPTIMIZE') == '0': + self.skipTest("Line tracing behavior differs when JIT optimizer is disabled") self.using_gc = gc.isenabled() gc.disable() self.addCleanup(sys.settrace, sys.gettrace()) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 53e55383bf9..502103ce629 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -20,7 +20,7 @@ ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, - change_cwd) + change_cwd, EnvironmentVarGuard) from test.support.venv import VirtualEnvironmentMixin import sysconfig @@ -32,7 +32,6 @@ from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name import _imp import _osx_support -import _sysconfig HAS_USER_BASE = sysconfig._HAS_USER_BASE @@ -186,7 +185,7 @@ def test_posix_venv_scheme(self): # The include directory on POSIX isn't exactly the same as before, # but it is "within" sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) - self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) + self.assertStartsWith(sysconfig_includedir, incpath + os.sep) def test_nt_venv_scheme(self): # The following directories were hardcoded in the venv module @@ -353,6 +352,13 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'macosx-10.4-%s' % arch) + for macver in range(11, 16): + _osx_support._remove_original_values(get_config_vars()) + get_config_vars()['CFLAGS'] = ('-fno-strict-overflow -Wsign-compare -Wunreachable-code' + '-arch arm64 -fno-common -dynamic -DNDEBUG -g -O3 -Wall') + get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = f"{macver}.0" + self.assertEqual(get_platform(), 'macosx-%d.0-arm64' % macver) + # linux debian sarge os.name = 'posix' sys.version = ('2.3.5 (#1, Jul 4 2007, 17:28:59) ' @@ -531,13 +537,10 @@ def test_srcdir(self): Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) # <srcdir>/PC/pyconfig.h.in always exists even if unused - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') - self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) if os.name == 'nt': - # <executable dir>/pyconfig.h exists on Windows in a build tree - pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) @@ -572,8 +575,7 @@ def test_linux_ext_suffix(self): expected_suffixes = 'i386-linux-gnu.so', 'x86_64-linux-gnux32.so', 'i386-linux-musl.so' else: # 8 byte pointer size expected_suffixes = 'x86_64-linux-gnu.so', 'x86_64-linux-musl.so' - self.assertTrue(suffix.endswith(expected_suffixes), - f'unexpected suffix {suffix!r}') + self.assertEndsWith(suffix, expected_suffixes) @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') def test_android_ext_suffix(self): @@ -585,13 +587,12 @@ def test_android_ext_suffix(self): "aarch64": "aarch64-linux-android", "armv7l": "arm-linux-androideabi", }[machine] - self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), - f"{machine=}, {suffix=}") + self.assertEndsWith(suffix, f"-{expected_triplet}.so") @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') - self.assertTrue(suffix.endswith('-darwin.so'), suffix) + self.assertEndsWith(suffix, '-darwin.so') def test_always_set_py_debug(self): self.assertIn('Py_DEBUG', sysconfig.get_config_vars()) @@ -714,11 +715,11 @@ def test_sysconfigdata_json(self): ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) if sysconfig._installation_is_relocated(): - ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase', 'srcdir'} for key in ignore_keys: - json_config_vars.pop(key) - system_config_vars.pop(key) + json_config_vars.pop(key, None) + system_config_vars.pop(key, None) self.assertEqual(system_config_vars, json_config_vars) @@ -763,8 +764,13 @@ def test_parse_makefile(self): print("var3=42", file=makefile) print("var4=$/invalid", file=makefile) print("var5=dollar$$5", file=makefile) - print("var6=${var3}/lib/python3.5/config-$(VAR2)$(var5)" + print("var6=${var7}/lib/python3.5/config-$(VAR2)$(var5)" "-x86_64-linux-gnu", file=makefile) + print("var7=${var3}", file=makefile) + print("var8=$$(var3)", file=makefile) + print("var9=$(var10)(var3)", file=makefile) + print("var10=$$", file=makefile) + print("var11=$${ORIGIN}${var5}", file=makefile) vars = _parse_makefile(TESTFN) self.assertEqual(vars, { 'var1': 'ab42', @@ -773,6 +779,74 @@ def test_parse_makefile(self): 'var4': '$/invalid', 'var5': 'dollar$5', 'var6': '42/lib/python3.5/config-b42dollar$5-x86_64-linux-gnu', + 'var7': 42, + 'var8': '$(var3)', + 'var9': '$(var3)', + 'var10': '$', + 'var11': '${ORIGIN}dollar$5', + }) + + def _test_parse_makefile_recursion(self): + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as makefile: + print("var1=var1=$(var1)", file=makefile) + print("var2=var3=$(var3)", file=makefile) + print("var3=var2=$(var2)", file=makefile) + vars = _parse_makefile(TESTFN) + self.assertEqual(vars, { + 'var1': 'var1=', + 'var2': 'var3=var2=', + 'var3': 'var2=', + }) + + def test_parse_makefile_renamed_vars(self): + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as makefile: + print("var1=$(CFLAGS)", file=makefile) + print("PY_CFLAGS=-Wall $(CPPFLAGS)", file=makefile) + print("PY_LDFLAGS=-lm", file=makefile) + print("var2=$(LDFLAGS)", file=makefile) + print("var3=$(CPPFLAGS)", file=makefile) + with EnvironmentVarGuard() as env: + env.clear() + vars = _parse_makefile(TESTFN) + self.assertEqual(vars, { + 'var1': '-Wall', + 'CFLAGS': '-Wall', + 'PY_CFLAGS': '-Wall', + 'LDFLAGS': '-lm', + 'PY_LDFLAGS': '-lm', + 'var2': '-lm', + 'var3': '', + }) + + def test_parse_makefile_keep_unresolved(self): + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as makefile: + print("var1=value", file=makefile) + print("var2=$/", file=makefile) + print("var3=$/$(var1)", file=makefile) + print("var4=var5=$(var5)", file=makefile) + print("var5=$/$(var1)", file=makefile) + print("var6=$(var1)$/", file=makefile) + print("var7=var8=$(var8)", file=makefile) + print("var8=$(var1)$/", file=makefile) + vars = _parse_makefile(TESTFN) + self.assertEqual(vars, { + 'var1': 'value', + 'var2': '$/', + 'var3': '$/value', + 'var4': 'var5=$/value', + 'var5': '$/value', + 'var6': 'value$/', + 'var7': 'var8=value$/', + 'var8': 'value$/', + }) + vars = _parse_makefile(TESTFN, keep_unresolved=False) + self.assertEqual(vars, { + 'var1': 'value', + 'var4': 'var5=', + 'var7': 'var8=', }) diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index 30dcb3e3c4f..e575aac037e 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -3,7 +3,8 @@ Glossary: * errored : Whitespace related problems present in file. """ -from unittest import TestCase, mock + +from unittest import TestCase, main, mock import errno import os import tabnanny @@ -352,3 +353,17 @@ def test_double_verbose_mode(self): "offending line: '\\tprint(\"world\")'" ).strip() self.validate_cmd("-vv", path, stdout=stdout, partial=True) + + +class TestModule(TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(tabnanny, "__version__") + self.assertEqual(cm.filename, __file__) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2d9649237a9..9892005787c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -55,6 +55,7 @@ def sha256sum(data): zstname = os.path.join(TEMPDIR, "testtar.tar.zst") tmpname = os.path.join(TEMPDIR, "tmp.tar") dotlessname = os.path.join(TEMPDIR, "testtar") +SPACE = b" " sha256_regtype = ( "e09e4bc8b3c9d9177e77256353b36c159f5f040531bbd4b024a8f9b9196c71ce" @@ -840,6 +841,57 @@ def test_next_on_empty_tarfile(self): with tarfile.open(fileobj=fd, mode="r") as tf: self.assertEqual(tf.next(), None) + def _setup_symlink_to_target(self, temp_dirpath): + target_filepath = os.path.join(temp_dirpath, "target") + ustar_dirpath = os.path.join(temp_dirpath, "ustar") + hardlink_filepath = os.path.join(ustar_dirpath, "lnktype") + with open(target_filepath, "wb") as f: + f.write(b"target") + os.makedirs(ustar_dirpath) + os.symlink(target_filepath, hardlink_filepath) + return target_filepath, hardlink_filepath + + def _assert_on_file_content(self, filepath, digest): + with open(filepath, "rb") as f: + data = f.read() + self.assertEqual(sha256sum(data), digest) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extract_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", DIR, filter="data") + tar.extract("ustar/lnktype", DIR, filter="data") + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extractall_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extractall( + DIR, members=["ustar/regtype", "ustar/lnktype"], filter="data", + ) + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None @@ -1650,7 +1702,7 @@ def test_cwd(self): try: for t in tar: if t.name != ".": - self.assertTrue(t.name.startswith("./"), t.name) + self.assertStartsWith(t.name, "./") finally: tar.close() @@ -1736,6 +1788,16 @@ def test_file_mode(self): finally: os.umask(original_umask) + def test_pathlike_name(self): + expected_name = os.path.abspath(tmpname) + tarpath = os_helper.FakePath(tmpname) + + for func in (tarfile.open, tarfile.TarFile.open): + with self.subTest(): + with func(tarpath, self.mode) as tar: + self.assertEqual(tar.name, expected_name) + os_helper.unlink(tmpname) + class GzipStreamWriteTest(GzipTest, StreamWriteTest): def test_source_directory_not_leaked(self): @@ -2715,6 +2777,31 @@ def test_useful_error_message_when_modules_missing(self): str(excinfo.exception), ) + @os_helper.skip_unless_symlink + @unittest.skipUnless(hasattr(os, 'chmod'), "missing os.chmod") + @unittest.mock.patch('os.chmod') + def test_deferred_directory_attributes_update(self, mock_chmod): + # Regression test for gh-127987: setting attributes on arbitrary files + tempdir = os.path.join(TEMPDIR, 'test127987') + def mock_chmod_side_effect(path, mode, **kwargs): + target_path = os.path.realpath(path) + if os.path.commonpath([target_path, tempdir]) != tempdir: + raise Exception("should not try to chmod anything outside the destination", target_path) + mock_chmod.side_effect = mock_chmod_side_effect + + outside_tree_dir = os.path.join(TEMPDIR, 'outside_tree_dir') + with ArchiveMaker() as arc: + arc.add('x', symlink_to='.') + arc.add('x', type=tarfile.DIRTYPE, mode='?rwsrwsrwt') + arc.add('x', symlink_to=outside_tree_dir) + + os.makedirs(outside_tree_dir) + try: + arc.open().extractall(path=tempdir, filter='tar') + finally: + os_helper.rmtree(outside_tree_dir) + os_helper.rmtree(tempdir) + class CommandLineTest(unittest.TestCase): @@ -3275,6 +3362,10 @@ def check_files_present(self, directory): got_paths = set( p.relative_to(directory) for p in pathlib.Path(directory).glob('**/*')) + if self.extraction_filter in (None, 'data'): + # The 'data' filter is expected to reject special files + for path in 'ustar/fifotype', 'ustar/blktype', 'ustar/chrtype': + got_paths.discard(pathlib.Path(path)) self.assertEqual(self.control_paths, got_paths) @contextmanager @@ -3490,11 +3581,12 @@ class ArchiveMaker: with t.open() as tar: ... # `tar` is now a TarFile with 'filename' in it! """ - def __init__(self): + def __init__(self, **kwargs): self.bio = io.BytesIO() + self.tar_kwargs = dict(kwargs) def __enter__(self): - self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio) + self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio, **self.tar_kwargs) return self def __exit__(self, *exc): @@ -3503,12 +3595,28 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, size=None, **kwargs): - """Add a member to the test archive. Call within `with`.""" + mode=None, size=None, content=None, **kwargs): + """Add a member to the test archive. Call within `with`. + + Provides many shortcuts: + - default `type` is based on symlink_to, hardlink_to, and trailing `/` + in name (which is stripped) + - size & content defaults are based on each other + - content can be str or bytes + - mode should be textual ('-rwxrwxrwx') + + (add more! this is unstable internal test-only API) + """ name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if content is not None: + if isinstance(content, str): + content = content.encode() + size = len(content) if size is not None: tarinfo.size = size + if content is None: + content = bytes(tarinfo.size) if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3522,7 +3630,7 @@ def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, if type is not None: tarinfo.type = type if tarinfo.isreg(): - fileobj = io.BytesIO(bytes(tarinfo.size)) + fileobj = io.BytesIO(content) else: fileobj = None self.tar_w.addfile(tarinfo, fileobj) @@ -3555,8 +3663,41 @@ class TestExtractionFilters(unittest.TestCase): # The destination for the extraction, within `outerdir` destdir = outerdir / 'dest' + @classmethod + def setUpClass(cls): + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + cls.dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + cls.dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + else: + cls.dotdot_resolves_early = False + @contextmanager - def check_context(self, tar, filter): + def check_context(self, tar, filter, *, check_flag=True): """Extracts `tar` to `self.destdir` and allows checking the result If an error occurs, it must be checked using `expect_exception` @@ -3565,27 +3706,40 @@ def check_context(self, tar, filter): except the destination directory itself and parent directories of other files. When checking directories, do so before their contents. + + A file called 'flag' is made in outerdir (i.e. outside destdir) + before extraction; it should not be altered nor should its contents + be read/copied. """ with os_helper.temp_dir(self.outerdir): + flag_path = self.outerdir / 'flag' + flag_path.write_text('capture me') try: tar.extractall(self.destdir, filter=filter) except Exception as exc: self.raised_exception = exc + self.reraise_exception = True self.expected_paths = set() else: self.raised_exception = None + self.reraise_exception = False self.expected_paths = set(self.outerdir.glob('**/*')) self.expected_paths.discard(self.destdir) + self.expected_paths.discard(flag_path) try: - yield + yield self finally: tar.close() - if self.raised_exception: + if self.reraise_exception: raise self.raised_exception self.assertEqual(self.expected_paths, set()) + if check_flag: + self.assertEqual(flag_path.read_text(), 'capture me') + else: + assert filter == 'fully_trusted' def expect_file(self, name, type=None, symlink_to=None, mode=None, - size=None): + size=None, content=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3604,26 +3758,45 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None, # The symlink might be the same (textually) as what we expect, # but some systems change the link to an equivalent path, so # we fall back to samefile(). - if expected != got: - self.assertTrue(got.samefile(expected)) + try: + if expected != got: + self.assertTrue(got.samefile(expected)) + except Exception as e: + # attach a note, so it's shown even if `samefile` fails + e.add_note(f'{expected=}, {got=}') + raise elif type == tarfile.REGTYPE or type is None: self.assertTrue(path.is_file()) elif type == tarfile.DIRTYPE: self.assertTrue(path.is_dir()) elif type == tarfile.FIFOTYPE: self.assertTrue(path.is_fifo()) + elif type == tarfile.SYMTYPE: + self.assertTrue(path.is_symlink()) else: raise NotImplementedError(type) if size is not None: self.assertEqual(path.stat().st_size, size) + if content is not None: + self.assertEqual(path.read_text(), content) for parent in path.parents: self.expected_paths.discard(parent) + def expect_any_tree(self, name): + """Check a directory; forget about its contents.""" + tree_path = (self.destdir / name).resolve() + self.expect_file(tree_path, type=tarfile.DIRTYPE) + self.expected_paths = { + p for p in self.expected_paths + if tree_path not in p.parents + } + def expect_exception(self, exc_type, message_re='.'): with self.assertRaisesRegex(exc_type, message_re): if self.raised_exception is not None: raise self.raised_exception - self.raised_exception = None + self.reraise_exception = False + return self.raised_exception def test_benign_file(self): with ArchiveMaker() as arc: @@ -3669,23 +3842,21 @@ def test_parent_symlink(self): arc.add('current', symlink_to='.') # effectively points to ./../ - arc.add('parent', symlink_to='current/..') + if self.dotdot_resolves_early and os_helper.can_symlink(): + arc.add('parent', symlink_to='current/../..') + else: + arc.add('parent', symlink_to='current/..') arc.add('parent/evil') if os_helper.can_symlink(): with self.check_context(arc.open(), 'fully_trusted'): - if self.raised_exception is not None: - # Windows will refuse to create a file that's a symlink to itself - # (and tarfile doesn't swallow that exception) - self.expect_exception(FileExistsError) - # The other cases will fail with this error too. - # Skip the rest of this test. - return + self.expect_file('current', symlink_to='.') + if self.dotdot_resolves_early: + self.expect_file('parent', symlink_to='current/../..') else: - self.expect_file('current', symlink_to='.') self.expect_file('parent', symlink_to='current/..') - self.expect_file('../evil') + self.expect_file('../evil') with self.check_context(arc.open(), 'tar'): self.expect_exception( @@ -3708,40 +3879,85 @@ def test_parent_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_file('parent/evil') + @symlink_test + @os_helper.skip_unless_symlink + def test_realpath_limit_attack(self): + # (CVE-2025-4517) + + with ArchiveMaker() as arc: + # populate the symlinks and dirs that expand in os.path.realpath() + # The component length is chosen so that in common cases, the unexpanded + # path fits in PATH_MAX, but it overflows when the final symlink + # is expanded + steps = "abcdefghijklmnop" + if sys.platform == 'win32': + component = 'd' * 25 + elif 'PC_PATH_MAX' in os.pathconf_names: + max_path_len = os.pathconf(self.outerdir.parent, "PC_PATH_MAX") + path_sep_len = 1 + dest_len = len(str(self.destdir)) + path_sep_len + component_len = (max_path_len - dest_len) // (len(steps) + path_sep_len) + component = 'd' * component_len + else: + raise NotImplementedError("Need to guess component length for {sys.platform}") + path = "" + step_path = "" + for i in steps: + arc.add(os.path.join(path, component), type=tarfile.DIRTYPE, + mode='drwxrwxrwx') + arc.add(os.path.join(path, i), symlink_to=component) + path = os.path.join(path, component) + step_path = os.path.join(step_path, i) + # create the final symlink that exceeds PATH_MAX and simply points + # to the top dir. + # this link will never be expanded by + # os.path.realpath(strict=False), nor anything after it. + linkpath = os.path.join(*steps, "l"*254) + parent_segments = [".."] * len(steps) + arc.add(linkpath, symlink_to=os.path.join(*parent_segments)) + # make a symlink outside to keep the tar command happy + arc.add("escape", symlink_to=os.path.join(linkpath, "..")) + # use the symlinks above, that are not checked, to create a hardlink + # to a file outside of the destination path + arc.add("flaglink", hardlink_to=os.path.join("escape", "flag")) + # now that we have the hardlink we can overwrite the file + arc.add("flaglink", content='overwrite') + # we can also create new files as well! + arc.add("escape/newfile", content='new') + + with (self.subTest('fully_trusted'), + self.check_context(arc.open(), filter='fully_trusted', + check_flag=False)): + if sys.platform == 'win32': + self.expect_exception((FileNotFoundError, FileExistsError)) + elif self.raised_exception: + # Cannot symlink/hardlink: tarfile falls back to getmember() + self.expect_exception(KeyError) + # Otherwise, this block should never enter. + else: + self.expect_any_tree(component) + self.expect_file('flaglink', content='overwrite') + self.expect_file('../newfile', content='new') + self.expect_file('escape', type=tarfile.SYMTYPE) + self.expect_file('a', symlink_to=component) + + for filter in 'tar', 'data': + with self.subTest(filter), self.check_context(arc.open(), filter=filter): + exc = self.expect_exception((OSError, KeyError)) + if isinstance(exc, OSError): + if sys.platform == 'win32': + # 3: ERROR_PATH_NOT_FOUND + # 5: ERROR_ACCESS_DENIED + # 206: ERROR_FILENAME_EXCED_RANGE + self.assertIn(exc.winerror, (3, 5, 206)) + else: + self.assertEqual(exc.errno, errno.ENAMETOOLONG) + @symlink_test def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives - # Posix and Windows have different pathname resolution: - # either symlink or a '..' component resolve first. - # Let's see which we are on. - if os_helper.can_symlink(): - testpath = os.path.join(TEMPDIR, 'resolution_test') - os.mkdir(testpath) - - # testpath/current links to `.` which is all of: - # - `testpath` - # - `testpath/current` - # - `testpath/current/current` - # - etc. - os.symlink('.', os.path.join(testpath, 'current')) - - # we'll test where `testpath/current/../file` ends up - with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): - pass - - if os.path.exists(os.path.join(testpath, 'file')): - # Windows collapses 'current\..' to '.' first, leaving - # 'testpath\file' - dotdot_resolves_early = True - elif os.path.exists(os.path.join(testpath, '..', 'file')): - # Posix resolves 'current' to '.' first, leaving - # 'testpath/../file' - dotdot_resolves_early = False - else: - raise AssertionError('Could not determine link resolution') - with ArchiveMaker() as arc: # `current` links to `.` which is both the destination directory @@ -3777,7 +3993,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'data'): if os_helper.can_symlink(): - if dotdot_resolves_early: + if self.dotdot_resolves_early: # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, @@ -3825,6 +4041,21 @@ def test_absolute_symlink(self): tarfile.AbsoluteLinkError, "'parent' is a link to an absolute path") + @symlink_test + @os_helper.skip_unless_symlink + def test_symlink_target_seperator_rewrite_on_windows(self): + with ArchiveMaker() as arc: + arc.add('link', symlink_to="relative/test/path") + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_file('link', type=tarfile.SYMTYPE) + link_path = os.path.normpath(self.destdir / "link") + link_target = os.readlink(link_path) + if os.name == "nt": + self.assertEqual(link_target, "relative\\test\\path") + else: + self.assertEqual(link_target, "relative/test/path") + def test_absolute_hardlink(self): # Test hardlink to an absolute path # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives @@ -3930,8 +4161,8 @@ def test_chains(self): arc.add('symlink2', symlink_to=os.path.join( 'linkdir', 'hardlink2')) arc.add('targetdir/target', size=3) - arc.add('linkdir/hardlink', hardlink_to='targetdir/target') - arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + arc.add('linkdir/hardlink', hardlink_to=os.path.join('targetdir', 'target')) + arc.add('linkdir/hardlink2', hardlink_to=os.path.join('linkdir', 'symlink')) for filter in 'tar', 'data', 'fully_trusted': with self.check_context(arc.open(), filter): @@ -3947,6 +4178,129 @@ def test_chains(self): self.expect_file('linkdir/symlink', size=3) self.expect_file('symlink2', size=3) + @symlink_test + def test_sneaky_hardlink_fallback(self): + # (CVE-2025-4330) + # Test that when hardlink extraction falls back to extracting members + # from the archive, the extracted member is (re-)filtered. + with ArchiveMaker() as arc: + # Create a directory structure so the c/escape symlink stays + # inside the path + arc.add("a/t/dummy") + # Create b/ directory + arc.add("b/") + # Point "c" to the bottom of the tree in "a" + arc.add("c", symlink_to=os.path.join("a", "t")) + # link to non-existant location under "a" + arc.add("c/escape", symlink_to=os.path.join("..", "..", + "link_here")) + # Move "c" to point to "b" ("c/escape" no longer exists) + arc.add("c", symlink_to="b") + # Attempt to create a hard link to "c/escape". Since it doesn't + # exist it will attempt to extract "cescape" but at "boom". + arc.add("boom", hardlink_to=os.path.join("c", "escape")) + + with self.check_context(arc.open(), 'data'): + if not os_helper.can_symlink(): + # When 'c/escape' is extracted, 'c' is a regular + # directory, and 'c/escape' *would* point outside + # the destination if symlinks were allowed. + self.expect_exception( + tarfile.LinkOutsideDestinationError) + elif sys.platform == "win32": + # On Windows, 'c/escape' points outside the destination + self.expect_exception(tarfile.LinkOutsideDestinationError) + else: + e = self.expect_exception( + tarfile.LinkFallbackError, + "link 'boom' would be extracted as a copy of " + + "'c/escape', which was rejected") + self.assertIsInstance(e.__cause__, + tarfile.LinkOutsideDestinationError) + for filter in 'tar', 'fully_trusted': + with self.subTest(filter), self.check_context(arc.open(), filter): + if not os_helper.can_symlink(): + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("c/") + else: + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("a/t/escape", symlink_to='../../link_here') + self.expect_file("boom", symlink_to='../../link_here') + self.expect_file("c", symlink_to='b') + + @symlink_test + def test_exfiltration_via_symlink(self): + # (CVE-2025-4138) + # Test changing symlinks that result in a symlink pointing outside + # the extraction directory, unless prevented by 'data' filter's + # normalization. + with ArchiveMaker() as arc: + arc.add("escape", symlink_to=os.path.join('link', 'link', '..', '..', 'link-here')) + arc.add("link", symlink_to='./') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + if os_helper.can_symlink(): + self.expect_file("link", symlink_to='./') + if filter == 'data': + self.expect_file("escape", symlink_to='link-here') + else: + self.expect_file("escape", + symlink_to='link/link/../../link-here') + else: + # Nothing is extracted. + pass + + @symlink_test + def test_chmod_outside_dir(self): + # (CVE-2024-12718) + # Test that members used for delayed updates of directory metadata + # are (re-)filtered. + with ArchiveMaker() as arc: + # "pwn" is a veeeery innocent symlink: + arc.add("a/pwn", symlink_to='.') + # But now "pwn" is also a directory, so it's scheduled to have its + # metadata updated later: + arc.add("a/pwn/", mode='drwxrwxrwx') + # Oops, "pwn" is not so innocent any more: + arc.add("a/pwn", symlink_to='x/../') + # Newly created symlink points to the dest dir, + # so it's OK for the "data" filter. + arc.add('a/x', symlink_to=('../')) + # But now "pwn" points outside the dest dir + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + if not os_helper.can_symlink(): + self.expect_file("a/pwn/") + elif filter == 'data': + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='.') + else: + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='x/../') + if sys.platform != "win32": + st_mode = cc.outerdir.stat().st_mode + self.assertNotEqual(st_mode & 0o777, 0o777) + + def test_link_fallback_normalizes(self): + # Make sure hardlink fallbacks work for non-normalized paths for all + # filters + with ArchiveMaker() as arc: + arc.add("dir/") + arc.add("dir/../afile") + arc.add("link1", hardlink_to='dir/../afile') + arc.add("link2", hardlink_to='dir/../dir/../afile') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + self.expect_file("dir/") + self.expect_file("afile") + self.expect_file("link1") + self.expect_file("link2") + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) @@ -4071,24 +4425,64 @@ def test_tar_filter(self): # The 'tar' filter returns TarInfo objects with the same name/type. # (It can also fail for particularly "evil" input, but we don't have # that in the test archive.) - with tarfile.TarFile.open(tarname) as tar: + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: for tarinfo in tar.getmembers(): - filtered = tarfile.tar_filter(tarinfo, '') + try: + filtered = tarfile.tar_filter(tarinfo, '') + except UnicodeEncodeError: + continue self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) def test_data_filter(self): # The 'data' filter either raises, or returns TarInfo with the same # name/type. - with tarfile.TarFile.open(tarname) as tar: + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: for tarinfo in tar.getmembers(): try: filtered = tarfile.data_filter(tarinfo, '') - except tarfile.FilterError: + except (tarfile.FilterError, UnicodeEncodeError): continue self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) + @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths') + def test_filter_unencodable(self): + # Sanity check using a valid path. + tarinfo = tarfile.TarInfo(os_helper.TESTFN) + filtered = tarfile.tar_filter(tarinfo, '') + self.assertIs(filtered.name, tarinfo.name) + filtered = tarfile.data_filter(tarinfo, '') + self.assertIs(filtered.name, tarinfo.name) + + tarinfo = tarfile.TarInfo('test\x00') + self.assertRaises(ValueError, tarfile.tar_filter, tarinfo, '') + self.assertRaises(ValueError, tarfile.data_filter, tarinfo, '') + tarinfo = tarfile.TarInfo('\ud800') + self.assertRaises(UnicodeEncodeError, tarfile.tar_filter, tarinfo, '') + self.assertRaises(UnicodeEncodeError, tarfile.data_filter, tarinfo, '') + + @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths') + def test_extract_unencodable(self): + # Create a member with name \xed\xa0\x80 which is UTF-8 encoded + # lone surrogate \ud800. + with ArchiveMaker(encoding='ascii', errors='surrogateescape') as arc: + arc.add('\udced\udca0\udc80') + with os_helper.temp_cwd() as tmp: + tar = arc.open(encoding='utf-8', errors='surrogatepass', + errorlevel=1) + self.assertEqual(tar.getnames(), ['\ud800']) + with self.assertRaises(UnicodeEncodeError): + tar.extractall() + self.assertEqual(os.listdir(), []) + + tar = arc.open(encoding='utf-8', errors='surrogatepass', + errorlevel=0, debug=1) + with support.captured_stderr() as stderr: + tar.extractall() + self.assertEqual(os.listdir(), []) + self.assertIn('tarfile: UnicodeEncodeError ', stderr.getvalue()) + def test_change_default_filter_on_instance(self): tar = tarfile.TarFile(tarname, 'r') def strict_filter(tarinfo, path): @@ -4201,13 +4595,13 @@ def valueerror_filter(tarinfo, path): # If errorlevel is 0, errors affected by errorlevel are ignored with self.check_context(arc.open(errorlevel=0), extracterror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), filtererror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), oserror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), tarerror_filter): self.expect_exception(tarfile.TarError) @@ -4218,7 +4612,7 @@ def valueerror_filter(tarinfo, path): # If 1, all fatal errors are raised with self.check_context(arc.open(errorlevel=1), extracterror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=1), filtererror_filter): self.expect_exception(tarfile.FilterError) @@ -4287,6 +4681,161 @@ def extractall(self, ar): ar.extractall(self.testdir, filter='fully_trusted') +class OffsetValidationTests(unittest.TestCase): + tarname = tmpname + invalid_posix_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, space, null terminator: 8 bytes + + b"000755" + SPACE + tarfile.NUL + # uid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0011407" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # magic: 6 bytes, version: 2 bytes + + tarfile.POSIX_MAGIC + # uname: 32 bytes + + tarfile.NUL * 32 + # gname: 32 bytes + + tarfile.NUL * 32 + # devmajor, space, null terminator: 8 bytes + + tarfile.NUL * 6 + SPACE + tarfile.NUL + # devminor, space, null terminator: 8 bytes + + tarfile.NUL * 6 + SPACE + tarfile.NUL + # prefix: 155 bytes + + tarfile.NUL * tarfile.LENGTH_PREFIX + # padding: 12 bytes + + tarfile.NUL * 12 + ) + invalid_gnu_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, null terminator: 8 bytes + + b"0000755" + tarfile.NUL + # uid, null terminator: 8 bytes + + b"0000001" + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"0000001" + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0011327" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # magic: 8 bytes + + tarfile.GNU_MAGIC + # uname: 32 bytes + + tarfile.NUL * 32 + # gname: 32 bytes + + tarfile.NUL * 32 + # devmajor, null terminator: 8 bytes + + tarfile.NUL * 8 + # devminor, null terminator: 8 bytes + + tarfile.NUL * 8 + # padding: 167 bytes + + tarfile.NUL * 167 + ) + invalid_v7_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, space, null terminator: 8 bytes + + b"000755" + SPACE + tarfile.NUL + # uid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0010070" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # padding: 255 bytes + + tarfile.NUL * 255 + ) + valid_gnu_header = tarfile.TarInfo("filename").tobuf(tarfile.GNU_FORMAT) + data_block = b"\xff" * tarfile.BLOCKSIZE + + def _write_buffer(self, buffer): + with open(self.tarname, "wb") as f: + f.write(buffer) + + def _get_members(self, ignore_zeros=None): + with open(self.tarname, "rb") as f: + with tarfile.open( + mode="r", fileobj=f, ignore_zeros=ignore_zeros + ) as tar: + return tar.getmembers() + + def _assert_raises_read_error_exception(self): + with self.assertRaisesRegex( + tarfile.ReadError, "file could not be opened successfully" + ): + self._get_members() + + def test_invalid_offset_header_validations(self): + for tar_format, invalid_header in ( + ("posix", self.invalid_posix_header), + ("gnu", self.invalid_gnu_header), + ("v7", self.invalid_v7_header), + ): + with self.subTest(format=tar_format): + self._write_buffer(invalid_header) + self._assert_raises_read_error_exception() + + def test_early_stop_at_invalid_offset_header(self): + buffer = self.valid_gnu_header + self.invalid_gnu_header + self.valid_gnu_header + self._write_buffer(buffer) + members = self._get_members() + self.assertEqual(len(members), 1) + self.assertEqual(members[0].name, "filename") + self.assertEqual(members[0].offset, 0) + + def test_ignore_invalid_archive(self): + # 3 invalid headers with their respective data + buffer = (self.invalid_gnu_header + self.data_block) * 3 + self._write_buffer(buffer) + members = self._get_members(ignore_zeros=True) + self.assertEqual(len(members), 0) + + def test_ignore_invalid_offset_headers(self): + for first_block, second_block, expected_offset in ( + ( + (self.valid_gnu_header), + (self.invalid_gnu_header + self.data_block), + 0, + ), + ( + (self.invalid_gnu_header + self.data_block), + (self.valid_gnu_header), + 1024, + ), + ): + self._write_buffer(first_block + second_block) + members = self._get_members(ignore_zeros=True) + self.assertEqual(len(members), 1) + self.assertEqual(members[0].name, "filename") + self.assertEqual(members[0].offset, expected_offset) + + def setUpModule(): os_helper.unlink(TEMPDIR) os.makedirs(TEMPDIR) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index d46d3c0f040..7eec34f2f29 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -516,11 +516,11 @@ def test_collision_with_existing_file(self): _mock_candidate_names('aaa', 'aaa', 'bbb'): (fd1, name1) = self.make_temp() os.close(fd1) - self.assertTrue(name1.endswith('aaa')) + self.assertEndsWith(name1, 'aaa') (fd2, name2) = self.make_temp() os.close(fd2) - self.assertTrue(name2.endswith('bbb')) + self.assertEndsWith(name2, 'bbb') def test_collision_with_existing_directory(self): # _mkstemp_inner tries another name when a directory with @@ -528,11 +528,11 @@ def test_collision_with_existing_directory(self): with _inside_empty_temp_dir(), \ _mock_candidate_names('aaa', 'aaa', 'bbb'): dir = tempfile.mkdtemp() - self.assertTrue(dir.endswith('aaa')) + self.assertEndsWith(dir, 'aaa') (fd, name) = self.make_temp() os.close(fd) - self.assertTrue(name.endswith('bbb')) + self.assertEndsWith(name, 'bbb') class TestGetTempPrefix(BaseTestCase): @@ -828,9 +828,9 @@ def test_collision_with_existing_file(self): _mock_candidate_names('aaa', 'aaa', 'bbb'): file = tempfile.NamedTemporaryFile(delete=False) file.close() - self.assertTrue(file.name.endswith('aaa')) + self.assertEndsWith(file.name, 'aaa') dir = tempfile.mkdtemp() - self.assertTrue(dir.endswith('bbb')) + self.assertEndsWith(dir, 'bbb') def test_collision_with_existing_directory(self): # mkdtemp tries another name when a directory with @@ -838,9 +838,9 @@ def test_collision_with_existing_directory(self): with _inside_empty_temp_dir(), \ _mock_candidate_names('aaa', 'aaa', 'bbb'): dir1 = tempfile.mkdtemp() - self.assertTrue(dir1.endswith('aaa')) + self.assertEndsWith(dir1, 'aaa') dir2 = tempfile.mkdtemp() - self.assertTrue(dir2.endswith('bbb')) + self.assertEndsWith(dir2, 'bbb') def test_for_tempdir_is_bytes_issue40701_api_warts(self): orig_tempdir = tempfile.tempdir @@ -1386,7 +1386,7 @@ def test_properties(self): f.write(b'x') self.assertTrue(f._rolled) - self.assertEqual(f.mode, 'rb+') + self.assertEqual(f.mode, 'wb+') self.assertIsNotNone(f.name) with self.assertRaises(AttributeError): f.newlines diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index e5d11cf84d2..ce8392a6ccd 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -290,8 +290,8 @@ def test_ioctl_constants(self): self.assertGreaterEqual(value, 0) def test_exception(self): - self.assertTrue(issubclass(termios.error, Exception)) - self.assertFalse(issubclass(termios.error, OSError)) + self.assertIsSubclass(termios.error, Exception) + self.assertNotIsSubclass(termios.error, OSError) if __name__ == '__main__': diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index cbd383ea4e2..aca1f427656 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -605,7 +605,7 @@ def test_break_long(self): # bug 1146. Prevent a long word to be wrongly wrapped when the # preceding word is exactly one character shorter than the width self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', 'ialidocious?', @@ -633,7 +633,7 @@ def test_nobreak_long(self): def test_max_lines_long(self): self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', '[...]'], diff --git a/Lib/test/test_threadedtempfile.py b/Lib/test/test_threadedtempfile.py index 420fc6ec8be..acb427b0c78 100644 --- a/Lib/test/test_threadedtempfile.py +++ b/Lib/test/test_threadedtempfile.py @@ -15,6 +15,7 @@ import tempfile +from test import support from test.support import threading_helper import unittest import io @@ -49,7 +50,8 @@ def run(self): class ThreadedTempFileTest(unittest.TestCase): - def test_main(self): + @support.bigmemtest(size=NUM_THREADS, memuse=60*2**20, dry_run=False) + def test_main(self, size): threads = [TempFileGreedy() for i in range(NUM_THREADS)] with threading_helper.start_threads(threads, startEvent.set): pass diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 4ab38c2598b..efd69a1f4fe 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -6,7 +6,7 @@ from test.support import threading_helper, requires_subprocess, requires_gil_enabled from test.support import verbose, cpython_only, os_helper from test.support.import_helper import ensure_lazy_imports, import_module -from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.script_helper import assert_python_ok, assert_python_failure, spawn_python from test.support import force_not_colorized import random @@ -28,7 +28,7 @@ from test import support try: - from test.support import interpreters + from concurrent import interpreters except ImportError: interpreters = None @@ -530,7 +530,8 @@ def test_enumerate_after_join(self): finally: sys.setswitchinterval(old_interval) - def test_join_from_multiple_threads(self): + @support.bigmemtest(size=20, memuse=72*2**20, dry_run=False) + def test_join_from_multiple_threads(self, size): # Thread.join() should be thread-safe errors = [] @@ -1246,13 +1247,68 @@ def __del__(self): self.assertEqual(err, b"") self.assertIn(b"all clear", out) + @support.subTests('lock_class_name', ['Lock', 'RLock']) + def test_acquire_daemon_thread_lock_in_finalization(self, lock_class_name): + # gh-123940: Py_Finalize() prevents other threads from running Python + # code (and so, releasing locks), so acquiring a locked lock can not + # succeed. + # We raise an exception rather than hang. + code = textwrap.dedent(f""" + import threading + import time + + thread_started_event = threading.Event() + + lock = threading.{lock_class_name}() + def loop(): + if {lock_class_name!r} == 'RLock': + lock.acquire() + with lock: + thread_started_event.set() + while True: + time.sleep(1) + + uncontested_lock = threading.{lock_class_name}() + + class Cycle: + def __init__(self): + self.self_ref = self + self.thr = threading.Thread( + target=loop, daemon=True) + self.thr.start() + thread_started_event.wait() + + def __del__(self): + assert self.thr.is_alive() + + # We *can* acquire an unlocked lock + uncontested_lock.acquire() + if {lock_class_name!r} == 'RLock': + uncontested_lock.acquire() + + # Acquiring a locked one fails + try: + lock.acquire() + except PythonFinalizationError: + assert self.thr.is_alive() + print('got the correct exception!') + + # Cycle holds a reference to itself, which ensures it is + # cleaned up during the GC that runs after daemon threads + # have been forced to exit during finalization. + Cycle() + """) + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(err, b"") + self.assertIn(b"got the correct exception", out) + def test_start_new_thread_failed(self): # gh-109746: if Python fails to start newly created thread # due to failure of underlying PyThread_start_new_thread() call, # its state should be removed from interpreter' thread states list # to avoid its double cleanup try: - from resource import setrlimit, RLIMIT_NPROC + from resource import setrlimit, RLIMIT_NPROC # noqa: F401 except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD code = """if 1: @@ -1283,12 +1339,6 @@ def f(): @cpython_only def test_finalize_daemon_thread_hang(self): - if support.check_sanitizer(thread=True, memory=True): - # the thread running `time.sleep(100)` below will still be alive - # at process exit - self.skipTest( - "https://github.com/python/cpython/issues/124878 - Known" - " race condition that TSAN identifies.") # gh-87135: tests that daemon threads hang during finalization script = textwrap.dedent(''' import os @@ -1351,6 +1401,62 @@ def do_flush(*args, **kwargs): ''') assert_python_ok("-c", script) + @skip_unless_reliable_fork + @unittest.skipUnless(hasattr(threading, 'get_native_id'), "test needs threading.get_native_id()") + def test_native_id_after_fork(self): + script = """if True: + import threading + import os + from test import support + + parent_thread_native_id = threading.current_thread().native_id + print(parent_thread_native_id, flush=True) + assert parent_thread_native_id == threading.get_native_id() + childpid = os.fork() + if childpid == 0: + print(threading.current_thread().native_id, flush=True) + assert threading.current_thread().native_id == threading.get_native_id() + else: + try: + assert parent_thread_native_id == threading.current_thread().native_id + assert parent_thread_native_id == threading.get_native_id() + finally: + support.wait_process(childpid, exitcode=0) + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(rc, 0) + self.assertEqual(err, b"") + native_ids = out.strip().splitlines() + self.assertEqual(len(native_ids), 2) + self.assertNotEqual(native_ids[0], native_ids[1]) + + def test_stop_the_world_during_finalization(self): + # gh-137433: Test functions that trigger a stop-the-world in the free + # threading build concurrent with interpreter finalization. + script = """if True: + import gc + import sys + import threading + NUM_THREADS = 5 + b = threading.Barrier(NUM_THREADS + 1) + def run_in_bg(): + b.wait() + while True: + sys.setprofile(None) + gc.collect() + + for _ in range(NUM_THREADS): + t = threading.Thread(target=run_in_bg, daemon=True) + t.start() + + b.wait() + print("Exiting...") + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(rc, 0) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"Exiting...") + class ThreadJoinOnShutdown(BaseTestCase): def _run_and_join(self, script): @@ -1431,7 +1537,8 @@ def worker(): self._run_and_join(script) @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_daemon_threads(self): + @support.bigmemtest(size=40, memuse=70*2**20, dry_run=False) + def test_4_daemon_threads(self, size): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. @@ -1669,6 +1776,7 @@ def task(): self.assertEqual(os.read(r_interp, 1), DONE) @cpython_only + @support.skip_if_sanitizer(thread=True, memory=True) def test_daemon_threads_fatal_error(self): import_module("_testcapi") subinterp_code = f"""if 1: @@ -1687,10 +1795,7 @@ def f(): _testcapi.run_in_subinterp(%r) """ % (subinterp_code,) - with test.support.SuppressCrashReport(): - rc, out, err = assert_python_failure("-c", script) - self.assertIn("Fatal Python error: Py_EndInterpreter: " - "not the last thread", err.decode()) + assert_python_ok("-c", script) def _check_allowed(self, before_start='', *, allowed=True, @@ -1962,6 +2067,49 @@ def modify_file(): t.start() t.join() + def test_dummy_thread_on_interpreter_shutdown(self): + # GH-130522: When `threading` held a reference to itself and then a + # _DummyThread() object was created, destruction of the dummy thread + # would emit an unraisable exception at shutdown, due to a lock being + # destroyed. + code = """if True: + import sys + import threading + + threading.x = sys.modules[__name__] + x = threading._DummyThread() + """ + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(out, b"") + self.assertEqual(err, b"") + + @requires_subprocess() + @unittest.skipIf(os.name == 'nt', "signals don't work well on windows") + def test_keyboard_interrupt_during_threading_shutdown(self): + import subprocess + source = f""" + from threading import Thread + import time + import os + + + def test(): + print('a', flush=True, end='') + time.sleep(10) + + + for _ in range(3): + Thread(target=test).start() + """ + + with spawn_python("-c", source, stderr=subprocess.PIPE) as proc: + self.assertEqual(proc.stdout.read(3), b'aaa') + proc.send_signal(signal.SIGINT) + proc.stderr.flush() + error = proc.stderr.read() + self.assertIn(b"KeyboardInterrupt", error) + class ThreadRunFail(threading.Thread): def run(self): @@ -2135,8 +2283,7 @@ def test_signature(self): # gh-102029 ] for args, kwargs in arg_types: with self.subTest(args=args, kwargs=kwargs): - with self.assertWarns(DeprecationWarning): - threading.RLock(*args, **kwargs) + self.assertRaises(TypeError, threading.RLock, *args, **kwargs) # Subtypes with custom `__init__` are allowed (but, not recommended): class CustomRLock(self.locktype): @@ -2154,6 +2301,9 @@ class ConditionAsRLockTests(lock_tests.RLockTests): # Condition uses an RLock by default and exports its API. locktype = staticmethod(threading.Condition) + def test_constructor_noargs(self): + self.skipTest("Condition allows positional arguments") + def test_recursion_count(self): self.skipTest("Condition does not expose _recursion_count()") @@ -2182,6 +2332,9 @@ def test__all__(self): @unittest.skipUnless(hasattr(_thread, 'set_name'), "missing _thread.set_name") @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") def test_set_name(self): + # Ensure main thread name is restored after test + self.addCleanup(_thread.set_name, _thread._get_name()) + # set_name() limit in bytes truncate = getattr(_thread, "_NAME_MAXLEN", None) limit = truncate or 100 @@ -2221,7 +2374,8 @@ def test_set_name(self): tests.append(os_helper.TESTFN_UNENCODABLE) if sys.platform.startswith("sunos"): - encoding = "utf-8" + # Use ASCII encoding on Solaris/Illumos/OpenIndiana + encoding = "ascii" else: encoding = sys.getfilesystemencoding() @@ -2237,7 +2391,7 @@ def work(): if truncate is not None: encoded = encoded[:truncate] if sys.platform.startswith("sunos"): - expected = encoded.decode("utf-8", "surrogateescape") + expected = encoded.decode("ascii", "surrogateescape") else: expected = os.fsdecode(encoded) else: @@ -2256,7 +2410,11 @@ def work(): if '\0' in expected: expected = expected.split('\0', 1)[0] - with self.subTest(name=name, expected=expected): + with self.subTest(name=name, expected=expected, thread="main"): + _thread.set_name(name) + self.assertEqual(_thread._get_name(), expected) + + with self.subTest(name=name, expected=expected, thread="worker"): work_name = None thread = threading.Thread(target=work, name=name) thread.start() @@ -2395,6 +2553,7 @@ def test_atexit_called_once(self): self.assertFalse(err) + @force_not_colorized def test_atexit_after_shutdown(self): # The only way to do this is by registering an atexit within # an atexit, which is intended to raise an exception. diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index d06f65270ef..c7e81fff6f7 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -2,6 +2,7 @@ from test.support import warnings_helper import decimal import enum +import fractions import math import platform import sys @@ -170,10 +171,12 @@ def test_sleep_exceptions(self): # Improved exception #81267 with self.assertRaises(TypeError) as errmsg: time.sleep([]) - self.assertIn("integer or float", str(errmsg.exception)) + self.assertIn("real number", str(errmsg.exception)) def test_sleep(self): - for value in [-0.0, 0, 0.0, 1e-100, 1e-9, 1e-6, 1, 1.2]: + for value in [-0.0, 0, 0.0, 1e-100, 1e-9, 1e-6, 1, 1.2, + decimal.Decimal('0.02'), + fractions.Fraction(1, 50)]: with self.subTest(value=value): time.sleep(value) @@ -756,22 +759,21 @@ class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): @skip_if_buggy_ucrt_strfptime - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. lt = time.localtime() - self.assertTrue(hasattr(lt, "tm_gmtoff")) - self.assertTrue(hasattr(lt, "tm_zone")) + self.assertHasAttr(lt, "tm_gmtoff") + self.assertHasAttr(lt, "tm_zone") # See if the offset and zone are similar to the module # attributes. if lt.tm_gmtoff is None: - self.assertTrue(not hasattr(time, "timezone")) + self.assertNotHasAttr(time, "timezone") else: self.assertEqual(lt.tm_gmtoff, -[time.timezone, time.altzone][lt.tm_isdst]) if lt.tm_zone is None: - self.assertTrue(not hasattr(time, "tzname")) + self.assertNotHasAttr(time, "tzname") else: self.assertEqual(lt.tm_zone, time.tzname[lt.tm_isdst]) @@ -791,14 +793,12 @@ def test_localtime_timezone(self): self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) self.assertEqual(new_lt9.tm_zone, lt.tm_zone) - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_strptime_timezone(self): t = time.strptime("UTC", "%Z") self.assertEqual(t.tm_zone, 'UTC') t = time.strptime("+0500", "%z") self.assertEqual(t.tm_gmtoff, 5 * 3600) - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_short_times(self): import pickle @@ -1184,11 +1184,11 @@ def test_clock_functions(self): if mac_ver >= (10, 12): for name in clock_names: - self.assertTrue(hasattr(time, name), f"time.{name} is not available") + self.assertHasAttr(time, name) else: for name in clock_names: - self.assertFalse(hasattr(time, name), f"time.{name} is available") + self.assertNotHasAttr(time, name) if __name__ == "__main__": diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index f5ae0a84eb3..f8bc306b455 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -4,8 +4,9 @@ import io from textwrap import dedent -from test.support import captured_stdout -from test.support import captured_stderr +from test.support import ( + captured_stdout, captured_stderr, force_not_colorized, +) # timeit's default number of iterations. DEFAULT_NUMBER = 1000000 @@ -222,8 +223,8 @@ def test_repeat_function_zero_iters(self): def assert_exc_string(self, exc_string, expected_exc_name): exc_lines = exc_string.splitlines() self.assertGreater(len(exc_lines), 2) - self.assertTrue(exc_lines[0].startswith('Traceback')) - self.assertTrue(exc_lines[-1].startswith(expected_exc_name)) + self.assertStartsWith(exc_lines[0], 'Traceback') + self.assertStartsWith(exc_lines[-1], expected_exc_name) def test_print_exc(self): s = io.StringIO() @@ -351,11 +352,13 @@ def test_main_with_time_unit(self): self.assertEqual(error_stringio.getvalue(), "Unrecognized unit. Please select nsec, usec, msec, or sec.\n") + @force_not_colorized def test_main_exception(self): with captured_stderr() as error_stringio: s = self.run_main(switches=['1/0']) self.assert_exc_string(error_stringio.getvalue(), 'ZeroDivisionError') + @force_not_colorized def test_main_exception_fixed_reps(self): with captured_stderr() as error_stringio: s = self.run_main(switches=['-n1', '1/0']) diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index ebb9e00ff91..46b01e6f131 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -58,7 +58,7 @@ def _test_widget(self, constructor): destroy_default_root() tkinter.NoDefaultRoot() self.assertRaises(RuntimeError, constructor) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') def destroy_default_root(): diff --git a/Lib/test/test_tkinter/test_font.py b/Lib/test/test_tkinter/test_font.py index 563707ddd2f..fc50f9fdbb5 100644 --- a/Lib/test/test_tkinter/test_font.py +++ b/Lib/test/test_tkinter/test_font.py @@ -1,3 +1,4 @@ +import collections.abc import unittest import tkinter from tkinter import font @@ -118,6 +119,16 @@ def test_repr(self): repr(self.font), f'<tkinter.font.Font object {fontname!r}>' ) + def test_iterable_protocol(self): + self.assertNotIsSubclass(font.Font, collections.abc.Iterable) + self.assertNotIsSubclass(font.Font, collections.abc.Container) + self.assertNotIsInstance(self.font, collections.abc.Iterable) + self.assertNotIsInstance(self.font, collections.abc.Container) + with self.assertRaisesRegex(TypeError, 'is not iterable'): + iter(self.font) + with self.assertRaisesRegex(TypeError, 'is not a container or iterable'): + self.font in self.font + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): @@ -159,5 +170,15 @@ def test_nametofont(self): self.assertRaises(RuntimeError, font.nametofont, fontname) +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(font, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index 38371fe00d6..358a18beee2 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -1,3 +1,4 @@ +import collections.abc import unittest import tkinter from test import support @@ -61,7 +62,33 @@ def test_image_create_photo(self): self.assertRaises(RuntimeError, tkinter.PhotoImage) -class BitmapImageTest(AbstractTkTest, unittest.TestCase): +class BaseImageTest: + def create(self): + return self.image_class('::img::test', master=self.root, + file=self.testfile) + + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + self.image_class('::img::test', master=self.root, spam=print) + image = self.image_class('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + + def test_iterable_protocol(self): + image = self.create() + self.assertNotIsSubclass(self.image_class, collections.abc.Iterable) + self.assertNotIsSubclass(self.image_class, collections.abc.Container) + self.assertNotIsInstance(image, collections.abc.Iterable) + self.assertNotIsInstance(image, collections.abc.Container) + with self.assertRaisesRegex(TypeError, 'is not iterable'): + iter(image) + with self.assertRaisesRegex(TypeError, 'is not a container or iterable'): + image in image + + +class BitmapImageTest(BaseImageTest, AbstractTkTest, unittest.TestCase): + image_class = tkinter.BitmapImage @classmethod def setUpClass(cls): @@ -144,26 +171,15 @@ def test_configure_foreground(self): self.assertEqual(image['foreground'], '-foreground {} {} #000000 yellow') - def test_bug_100814(self): - # gh-100814: Passing a callable option value causes AttributeError. - with self.assertRaises(tkinter.TclError): - tkinter.BitmapImage('::img::test', master=self.root, spam=print) - image = tkinter.BitmapImage('::img::test', master=self.root) - with self.assertRaises(tkinter.TclError): - image.configure(spam=print) - -class PhotoImageTest(AbstractTkTest, unittest.TestCase): +class PhotoImageTest(BaseImageTest, AbstractTkTest, unittest.TestCase): + image_class = tkinter.PhotoImage @classmethod def setUpClass(cls): AbstractTkTest.setUpClass.__func__(cls) cls.testfile = support.findfile('python.gif', subdir='tkinterdata') - def create(self): - return tkinter.PhotoImage('::img::test', master=self.root, - file=self.testfile) - def colorlist(self, *args): if tkinter.TkVersion >= 8.6 and self.wantobjects: return args @@ -282,14 +298,6 @@ def test_configure_palette(self): image.configure(palette='3/4/2') self.assertEqual(image['palette'], '3/4/2') - def test_bug_100814(self): - # gh-100814: Passing a callable option value causes AttributeError. - with self.assertRaises(tkinter.TclError): - tkinter.PhotoImage('::img::test', master=self.root, spam=print) - image = tkinter.PhotoImage('::img::test', master=self.root) - with self.assertRaises(tkinter.TclError): - image.configure(spam=print) - def test_blank(self): image = self.create() image.blank() diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 96ea3f0117c..32e2329506e 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -1,3 +1,4 @@ +import collections.abc import functools import unittest import tkinter @@ -497,7 +498,7 @@ def test_info_patchlevel(self): self.assertEqual(vi.serial, 0) else: self.assertEqual(vi.micro, 0) - self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) + self.assertStartsWith(str(vi), f'{vi.major}.{vi.minor}') def test_embedded_null(self): widget = tkinter.Entry(self.root) @@ -508,6 +509,17 @@ def test_embedded_null(self): widget.selection_range(0, 'end') self.assertEqual(widget.selection_get(), '\u20ac\0abc\x00def') + def test_iterable_protocol(self): + widget = tkinter.Entry(self.root) + self.assertNotIsSubclass(tkinter.Entry, collections.abc.Iterable) + self.assertNotIsSubclass(tkinter.Entry, collections.abc.Container) + self.assertNotIsInstance(widget, collections.abc.Iterable) + self.assertNotIsInstance(widget, collections.abc.Container) + with self.assertRaisesRegex(TypeError, 'is not iterable'): + iter(widget) + with self.assertRaisesRegex(TypeError, 'is not a container or iterable'): + widget in widget + class WmTest(AbstractTkTest, unittest.TestCase): @@ -609,7 +621,7 @@ def test_focus(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, '??') self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, '??') self.assertEqual(e.char, '??') @@ -642,7 +654,7 @@ def test_configure(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, '??') self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, '??') self.assertEqual(e.char, '??') @@ -676,7 +688,7 @@ def test_event_generate_key_press(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertIsInstance(e.state, int) self.assertNotEqual(e.state, 0) @@ -747,7 +759,7 @@ def test_event_generate_button_press(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, 1) self.assertEqual(e.state, 0) self.assertEqual(e.char, '??') @@ -781,7 +793,7 @@ def test_event_generate_motion(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, 0x100) self.assertEqual(e.char, '??') @@ -814,7 +826,7 @@ def test_event_generate_mouse_wheel(self): self.assertIs(e.widget, f) self.assertIsInstance(e.serial, int) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.time, 0) self.assertEqual(e.num, '??') self.assertEqual(e.state, 0) @@ -849,7 +861,7 @@ def test_generate_event_virtual_event(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, 0) self.assertEqual(e.char, '??') @@ -1308,17 +1320,17 @@ def test_no_default_root(self): self.assertIs(tkinter._default_root, root) tkinter.NoDefaultRoot() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') # repeated call is no-op tkinter.NoDefaultRoot() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root.destroy() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root = tkinter.Tk() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root.destroy() def test_getboolean(self): diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index b26956930d3..d579cca95ee 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -34,12 +34,116 @@ def test_search(self): # Invalid text index. self.assertRaises(tkinter.TclError, text.search, '', 0) + self.assertRaises(tkinter.TclError, text.search, '', '') + self.assertRaises(tkinter.TclError, text.search, '', 'invalid') + self.assertRaises(tkinter.TclError, text.search, '', '1.0', 0) + self.assertRaises(tkinter.TclError, text.search, '', '1.0', '') + self.assertRaises(tkinter.TclError, text.search, '', '1.0', 'invalid') - # Check if we are getting the indices as strings -- you are likely - # to get Tcl_Obj under Tk 8.5 if Tkinter doesn't convert it. - text.insert('1.0', 'hi-test') - self.assertEqual(text.search('-test', '1.0', 'end'), '1.2') - self.assertEqual(text.search('test', '1.0', 'end'), '1.3') + text.insert('1.0', + 'This is a test. This is only a test.\n' + 'Another line.\n' + 'Yet another line.\n' + '64-bit') + + self.assertEqual(text.search('test', '1.0'), '1.10') + self.assertEqual(text.search('test', '1.0', 'end'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.10'), '') + self.assertEqual(text.search('test', '1.11'), '1.31') + self.assertEqual(text.search('test', '1.32', 'end'), '') + self.assertEqual(text.search('test', '1.32'), '1.10') + + self.assertEqual(text.search('', '1.0'), '1.0') # empty pattern + self.assertEqual(text.search('nonexistent', '1.0'), '') + self.assertEqual(text.search('-bit', '1.0'), '4.2') # starts with a hyphen + + self.assertEqual(text.search('line', '3.0'), '3.12') + self.assertEqual(text.search('line', '3.0', forwards=True), '3.12') + self.assertEqual(text.search('line', '3.0', backwards=True), '2.8') + self.assertEqual(text.search('line', '3.0', forwards=True, backwards=True), '2.8') + + self.assertEqual(text.search('t.', '1.0'), '1.13') + self.assertEqual(text.search('t.', '1.0', exact=True), '1.13') + self.assertEqual(text.search('t.', '1.0', regexp=True), '1.10') + self.assertEqual(text.search('t.', '1.0', exact=True, regexp=True), '1.10') + + self.assertEqual(text.search('TEST', '1.0'), '') + self.assertEqual(text.search('TEST', '1.0', nocase=True), '1.10') + + self.assertEqual(text.search('.*line', '1.0', regexp=True), '2.0') + self.assertEqual(text.search('.*line', '1.0', regexp=True, nolinestop=True), '1.0') + + self.assertEqual(text.search('test', '1.0', '1.13'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.13', strictlimits=True), '') + self.assertEqual(text.search('test', '1.0', '1.14', strictlimits=True), '1.10') + + var = tkinter.Variable(self.root) + self.assertEqual(text.search('test', '1.0', count=var), '1.10') + self.assertEqual(var.get(), 4 if self.wantobjects else '4') + + # TODO: Add test for elide=True + + def test_search_all(self): + text = self.text + + # pattern and index are obligatory arguments. + self.assertRaises(tkinter.TclError, text.search_all, None, '1.0') + self.assertRaises(tkinter.TclError, text.search_all, 'a', None) + self.assertRaises(tkinter.TclError, text.search_all, None, None) + + # Keyword-only arguments + self.assertRaises(TypeError, text.search_all, 'a', '1.0', 'end', None) + + # Invalid text index. + self.assertRaises(tkinter.TclError, text.search_all, '', 0) + self.assertRaises(tkinter.TclError, text.search_all, '', '') + self.assertRaises(tkinter.TclError, text.search_all, '', 'invalid') + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 0) + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', '') + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 'invalid') + + def eq(res, expected): + self.assertIsInstance(res, tuple) + self.assertEqual([str(i) for i in res], expected) + + text.insert('1.0', 'ababa\naba\n64-bit') + + eq(text.search_all('aba', '1.0'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.0', 'end'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.1', 'end'), ['1.2', '2.0']) + eq(text.search_all('aba', '1.1'), ['1.2', '2.0', '1.0']) + + res = text.search_all('', '1.0') # empty pattern + eq(res[:5], ['1.0', '1.1', '1.2', '1.3', '1.4']) + eq(res[-5:], ['3.2', '3.3', '3.4', '3.5', '3.6']) + eq(text.search_all('nonexistent', '1.0'), []) + eq(text.search_all('-bit', '1.0'), ['3.2']) # starts with a hyphen + + eq(text.search_all('aba', '1.0', 'end', forwards=True), ['1.0', '2.0']) + eq(text.search_all('aba', 'end', '1.0', backwards=True), ['2.0', '1.2']) + + eq(text.search_all('aba', '1.0', overlap=True), ['1.0', '1.2', '2.0']) + eq(text.search_all('aba', 'end', '1.0', overlap=True, backwards=True), ['2.0', '1.2', '1.0']) + + eq(text.search_all('aba', '1.0', exact=True), ['1.0', '2.0']) + eq(text.search_all('a.a', '1.0', exact=True), []) + eq(text.search_all('a.a', '1.0', regexp=True), ['1.0', '2.0']) + + eq(text.search_all('ABA', '1.0'), []) + eq(text.search_all('ABA', '1.0', nocase=True), ['1.0', '2.0']) + + eq(text.search_all('a.a', '1.0', regexp=True), ['1.0', '2.0']) + eq(text.search_all('a.a', '1.0', regexp=True, nolinestop=True), ['1.0', '1.4']) + + eq(text.search_all('aba', '1.0', '2.2'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.0', '2.2', strictlimits=True), ['1.0']) + eq(text.search_all('aba', '1.0', '2.3', strictlimits=True), ['1.0', '2.0']) + + var = tkinter.Variable(self.root) + eq(text.search_all('aba', '1.0', count=var), ['1.0', '2.0']) + self.assertEqual(var.get(), (3, 3) if self.wantobjects else '3 3') + + # TODO: Add test for elide=True def test_count(self): text = self.text diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index ff3f92e9b5e..20e385ad0b6 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -315,7 +315,10 @@ def test_configure_direction(self): def test_configure_height(self): widget = self.create() - self.checkIntegerParam(widget, 'height', 100, -100, 0, conv=str) + if tk_version < (9, 0): + self.checkIntegerParam(widget, 'height', 100, -100, 0, conv=str) + else: + self.checkIntegerParam(widget, 'height', 0, -100, 0) def test_configure_image(self): widget = self.create() @@ -342,7 +345,10 @@ def test_configure_menu(self): def test_configure_width(self): widget = self.create() - self.checkIntegerParam(widget, 'width', 402, -402, 0, conv=str) + if tk_version < (9, 0): + self.checkIntegerParam(widget, 'width', 402, -402, 0, conv=str) + else: + self.checkIntegerParam(widget, 'width', 402, 0, 0) class OptionMenuTest(MenubuttonTest, unittest.TestCase): @@ -391,8 +397,12 @@ def test_configure_disabledbackground(self): def test_configure_insertborderwidth(self): widget = self.create(insertwidth=100) - self.checkPixelsParam(widget, 'insertborderwidth', - 0, 1.3, 2.6, 6, '10p') + if tk_version < (9, 0): + self.checkPixelsParam(widget, 'insertborderwidth', + 0, 1.3, 2.6, 6, '10p') + else: + self.checkPixelsParam(widget, 'insertborderwidth', + 0, 1.3, 3, 6, '10p') self.checkParam(widget, 'insertborderwidth', -2) # insertborderwidth is bounded above by a half of insertwidth. expected = 100 // 2 if tk_version < (9, 0) else 60 @@ -551,11 +561,22 @@ def test_configure_values(self): # XXX widget = self.create() self.assertEqual(widget['values'], '') - self.checkParam(widget, 'values', 'mon tue wed thur') + if tk_version < (9, 0): + expected = 'mon tue wed thur' + else: + expected = ('mon', 'tue', 'wed', 'thur') + self.checkParam(widget, 'values', 'mon tue wed thur', + expected=expected) self.checkParam(widget, 'values', ('mon', 'tue', 'wed', 'thur'), - expected='mon tue wed thur') + expected=expected) + + if tk_version < (9, 0): + expected = '42 3.14 {} {any string}' + else: + expected = (42, 3.14, '', 'any string') self.checkParam(widget, 'values', (42, 3.14, '', 'any string'), - expected='42 3.14 {} {any string}') + expected=expected) + self.checkParam(widget, 'values', '') def test_configure_wrap(self): @@ -649,10 +670,9 @@ def test_configure_endline(self): def test_configure_height(self): widget = self.create() self.checkPixelsParam(widget, 'height', 100, 101.2, 102.6, '3c') - self.checkParam(widget, 'height', -100, - expected=1 if tk_version < (9, 0) else -100) - self.checkParam(widget, 'height', 0, - expected=1 if tk_version < (9, 0) else 0 ) + expected = 1 if tk_version < (9, 0) else 0 + self.checkParam(widget, 'height', -100, expected=expected) + self.checkParam(widget, 'height', 0, expected=expected) def test_configure_maxundo(self): widget = self.create() @@ -670,8 +690,9 @@ def test_configure_insertunfocussed(self): def test_configure_selectborderwidth(self): widget = self.create() + value = -2 if tk_version < (9, 0) else 0 self.checkPixelsParam(widget, 'selectborderwidth', - 1.3, 2.6, -2, '10p', conv=False) + 1.3, 2.6, value, '10p', conv=False) def test_configure_spacing1(self): widget = self.create() diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index f518925e994..dd2d7c4da45 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -247,7 +247,11 @@ def test_configure_borderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'borderwidth', 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'borderwidth', -2) + if tk_version < (9, 0): + self.checkParam(widget, 'borderwidth', -2) + else: + self.checkParam(widget, 'borderwidth', 0) + if 'bd' in self.OPTIONS: self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, '10p') self.checkParam(widget, 'bd', -2, expected=expected) @@ -260,27 +264,46 @@ def test_configure_highlightthickness(self): def test_configure_insertborderwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'insertborderwidth', - 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'insertborderwidth', -2) + if tk_version < (9, 0): + values = (0, 1.3, 2.6, 6, -2, '10p') + value = -2 + else: + values = (0, 1, 3, 6, 13) + value = 0 + self.checkPixelsParam(widget, 'insertborderwidth', *values) + self.checkParam(widget, 'insertborderwidth', value) def test_configure_insertwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'insertwidth', 1.3, 2.6, -2, '10p') + if tk_version < (9, 0): + self.checkPixelsParam(widget, 'insertwidth', 1.3, 2.6, -2, '10p') + else: + self.checkPixelsParam(widget, 'insertwidth', 1, 3, 0, 13) def test_configure_padx(self): widget = self.create() self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'padx', -2) + if tk_version < (9, 0): + self.checkParam(widget, 'padx', -2) + else: + self.checkParam(widget, 'padx', 0) def test_configure_pady(self): widget = self.create() self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'pady', -2) + if tk_version < (9, 0): + self.checkParam(widget, 'pady', -2) + else: + self.checkParam(widget, 'pady', 0) def test_configure_selectborderwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'selectborderwidth', 1.3, 2.6, -2, '10p') + if tk_version < (9, 0): + values = (1.3, 2.6, -2, '10p') + else: + values = (1, 3, 0, 13) + self.checkPixelsParam(widget, 'selectborderwidth', *values) + class StandardOptionsTests(PixelOptionsTests): @@ -546,22 +569,34 @@ class IntegerSizeTests: """ Tests widgets which only accept integral width and height.""" def test_configure_height(self): widget = self.create() - self.checkIntegerParam(widget, 'height', 100, -100, 0) + if tk_version < (9, 0): + self.checkIntegerParam(widget, 'height', 100, -100, 0) + else: + self.checkIntegerParam(widget, 'height', 100, 0, 0) def test_configure_width(self): widget = self.create() - self.checkIntegerParam(widget, 'width', 402, -402, 0) + if tk_version < (9, 0): + self.checkIntegerParam(widget, 'width', 402, -402, 0) + else: + self.checkIntegerParam(widget, 'width', 402, 0, 0) class PixelSizeTests: """ Tests widgets which accept screen distances for width and height.""" def test_configure_height(self): widget = self.create() - self.checkPixelsParam(widget, 'height', 100, 101.2, 102.6, -100, 0, '3c') + value = -100 if tk_version < (9, 0) else 0 + self.checkPixelsParam( + widget, 'height', 100, 101.2, 102.6, value, 0, '3c' + ) def test_configure_width(self): widget = self.create() - self.checkPixelsParam(widget, 'width', 402, 403.4, 404.6, -402, 0, '5i') + value = -402 if tk_version < (9, 0) else 0 + self.checkPixelsParam( + widget, 'width', 402, 403.4, 404.6, value, 0, '5i' + ) def add_configure_tests(*source_classes): diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 2d41a5e5ac0..ca67e381958 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1,6 +1,8 @@ import contextlib +import itertools import os import re +import string import tempfile import token import tokenize @@ -1214,6 +1216,23 @@ def test_multiline_non_ascii_fstring_with_expr(self): FSTRING_END "\'\'\'" (3, 1) (3, 4) """) + # gh-139516, the '\n' is explicit to ensure no trailing whitespace which would invalidate the test + self.check_tokenize('''f"{f(a=lambda: 'à'\n)}"''', """\ + FSTRING_START \'f"\' (1, 0) (1, 2) + OP '{' (1, 2) (1, 3) + NAME 'f' (1, 3) (1, 4) + OP '(' (1, 4) (1, 5) + NAME 'a' (1, 5) (1, 6) + OP '=' (1, 6) (1, 7) + NAME 'lambda' (1, 7) (1, 13) + OP ':' (1, 13) (1, 14) + STRING "\'à\'" (1, 15) (1, 18) + NL '\\n' (1, 18) (1, 19) + OP ')' (2, 0) (2, 1) + OP '}' (2, 1) (2, 2) + FSTRING_END \'"\' (2, 2) (2, 3) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. @@ -1344,7 +1363,8 @@ def readline(): def test_no_bom_no_encoding_cookie(self): lines = ( - b'# something\n', + b'#!/home/\xc3\xa4/bin/python\n', + b'# something \xe2\x82\xac\n', b'print(something)\n', b'do_something(else)\n' ) @@ -1352,16 +1372,54 @@ def test_no_bom_no_encoding_cookie(self): self.assertEqual(encoding, 'utf-8') self.assertEqual(consumed_lines, list(lines[:2])) + def test_no_bom_no_encoding_cookie_first_line_error(self): + lines = ( + b'#!/home/\xa4/bin/python\n\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_no_bom_no_encoding_cookie_second_line_error(self): + lines = ( + b'#!/usr/bin/python\n', + b'# something \xe2\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + def test_bom_no_cookie(self): lines = ( - b'\xef\xbb\xbf# something\n', + b'\xef\xbb\xbf#!/home/\xc3\xa4/bin/python\n', b'print(something)\n', b'do_something(else)\n' ) encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'utf-8-sig') self.assertEqual(consumed_lines, - [b'# something\n', b'print(something)\n']) + [b'#!/home/\xc3\xa4/bin/python\n', b'print(something)\n']) + + def test_bom_no_cookie_first_line_error(self): + lines = ( + b'\xef\xbb\xbf#!/home/\xa4/bin/python\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_bom_no_cookie_second_line_error(self): + lines = ( + b'\xef\xbb\xbf#!/usr/bin/python\n', + b'# something \xe2\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) def test_cookie_first_line_no_bom(self): lines = ( @@ -1437,16 +1495,60 @@ def test_cookie_second_line_noncommented_first_line(self): expected = [b"print('\xc2\xa3')\n"] self.assertEqual(consumed_lines, expected) - def test_cookie_second_line_commented_first_line(self): + def test_first_non_utf8_coding_line(self): lines = ( - b"#print('\xc2\xa3')\n", - b'# vim: set fileencoding=iso8859-15 :\n', - b"print('\xe2\x82\xac')\n" + b'#coding:iso-8859-15 \xa4\n', + b'print(something)\n' ) encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'iso8859-15') - expected = [b"#print('\xc2\xa3')\n", b'# vim: set fileencoding=iso8859-15 :\n'] - self.assertEqual(consumed_lines, expected) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_first_utf8_coding_line_error(self): + lines = ( + b'#coding:ascii \xc3\xa4\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_second_non_utf8_coding_line(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:iso-8859-15 \xa4\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_second_utf8_coding_line_error(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:ascii \xc3\xa4\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_non_utf8_shebang(self): + lines = ( + b'#!/home/\xa4/bin/python\n', + b'#coding:iso-8859-15\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_utf8_shebang_error(self): + lines = ( + b'#!/home/\xc3\xa4/bin/python\n', + b'#coding:ascii\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) def test_cookie_second_line_empty_first_line(self): lines = ( @@ -1459,6 +1561,70 @@ def test_cookie_second_line_empty_first_line(self): expected = [b'\n', b'# vim: set fileencoding=iso8859-15 :\n'] self.assertEqual(consumed_lines, expected) + def test_cookie_third_line(self): + lines = ( + b'#!/home/\xc3\xa4/bin/python\n', + b'# something\n', + b'# vim: set fileencoding=ascii :\n', + b'print(something)\n', + b'do_something(else)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'utf-8') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_double_coding_line(self): + # If the first line matches the second line is ignored. + lines = ( + b'#coding:iso8859-15\n', + b'#coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_double_coding_same_line(self): + lines = ( + b'#coding:iso8859-15 coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_double_coding_utf8(self): + lines = ( + b'#coding:utf-8\n', + b'#coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'utf-8') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_nul_in_first_coding_line(self): + lines = ( + b'#coding:iso8859-15\x00\n', + b'\n', + b'\n', + b'print(something)\n' + ) + with self.assertRaisesRegex(SyntaxError, + "source code cannot contain null bytes"): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_nul_in_second_coding_line(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:iso8859-15\x00\n', + b'\n', + b'print(something)\n' + ) + with self.assertRaisesRegex(SyntaxError, + "source code cannot contain null bytes"): + tokenize.detect_encoding(self.get_readline(lines)) + def test_latin1_normalization(self): # See get_normal_name() in Parser/tokenizer/helpers.c. encodings = ("latin-1", "iso-8859-1", "iso-latin-1", "latin-1-unix", @@ -1483,7 +1649,6 @@ def test_syntaxerror_latin1(self): readline = self.get_readline(lines) self.assertRaises(SyntaxError, tokenize.detect_encoding, readline) - def test_utf8_normalization(self): # See get_normal_name() in Parser/tokenizer/helpers.c. encodings = ("utf-8", "utf-8-mac", "utf-8-unix") @@ -1975,6 +2140,10 @@ def test_roundtrip(self): for case in cases: self.check_roundtrip(case) + self.check_roundtrip(r"t'{ {}}'") + self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'") + self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'") + def test_continuation(self): # Balancing continuation @@ -3014,6 +3183,7 @@ def get_tokens(string): f'__{ x:d }__'""", + " a\n\x00", ]: with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) @@ -3234,5 +3404,77 @@ def test_exact_flag(self): self.check_output(source, expect, flag) +class StringPrefixTest(unittest.TestCase): + @staticmethod + def determine_valid_prefixes(): + # Try all lengths until we find a length that has zero valid + # prefixes. This will miss the case where for example there + # are no valid 3 character prefixes, but there are valid 4 + # character prefixes. That seems unlikely. + + single_char_valid_prefixes = set() + + # Find all of the single character string prefixes. Just get + # the lowercase version, we'll deal with combinations of upper + # and lower case later. I'm using this logic just in case + # some uppercase-only prefix is added. + for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase): + try: + eval(f'{letter}""') + single_char_valid_prefixes.add(letter.lower()) + except SyntaxError: + pass + + # This logic assumes that all combinations of valid prefixes only use + # the characters that are valid single character prefixes. That seems + # like a valid assumption, but if it ever changes this will need + # adjusting. + valid_prefixes = set() + for length in itertools.count(): + num_at_this_length = 0 + for prefix in ( + "".join(l) + for l in itertools.combinations(single_char_valid_prefixes, length) + ): + for t in itertools.permutations(prefix): + for u in itertools.product(*[(c, c.upper()) for c in t]): + p = "".join(u) + if p == "not": + # 'not' can never be a string prefix, + # because it's a valid expression: not "" + continue + try: + eval(f'{p}""') + + # No syntax error, so p is a valid string + # prefix. + + valid_prefixes.add(p) + num_at_this_length += 1 + except SyntaxError: + pass + if num_at_this_length == 0: + return valid_prefixes + + + def test_prefixes(self): + # Get the list of defined string prefixes. I don't see an + # obvious documented way of doing this, but probably the best + # thing is to split apart tokenize.StringPrefix. + + # Make sure StringPrefix begins and ends in parens. We're + # assuming it's of the form "(a|b|ab)", if a, b, and cd are + # valid string prefixes. + self.assertEqual(tokenize.StringPrefix[0], '(') + self.assertEqual(tokenize.StringPrefix[-1], ')') + + # Then split apart everything else by '|'. + defined_prefixes = set(tokenize.StringPrefix[1:-1].split('|')) + + # Now compute the actual allowed string prefixes and compare + # to what is defined in the tokenize module. + self.assertEqual(defined_prefixes, self.determine_valid_prefixes()) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py index 151a55a4b56..14559a632da 100644 --- a/Lib/test/test_tools/i18n_data/docstrings.py +++ b/Lib/test/test_tools/i18n_data/docstrings.py @@ -1,7 +1,7 @@ """Module docstring""" # Test docstring extraction -from gettext import gettext as _ +from gettext import gettext as _ # noqa: F401 # Empty docstring diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index bf54c999537..19eee19bdea 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -142,6 +142,8 @@ def test_traced_func_linear(self): self.assertEqual(self.tracer.results().counts, expected) + @unittest.skipIf(os.environ.get('PYTHON_UOPS_OPTIMIZE') == '0', + "Line counts differ when JIT optimizer is disabled") def test_traced_func_loop(self): self.tracer.runfunc(traced_func_loop, 2, 3) @@ -166,6 +168,8 @@ def test_traced_func_importing(self): self.assertEqual(self.tracer.results().counts, expected) + @unittest.skipIf(os.environ.get('PYTHON_UOPS_OPTIMIZE') == '0', + "Line counts differ when JIT optimizer is disabled") def test_trace_func_generator(self): self.tracer.runfunc(traced_func_calling_generator) @@ -236,6 +240,8 @@ def setUp(self): self.my_py_filename = fix_ext_py(__file__) self.addCleanup(sys.settrace, sys.gettrace()) + @unittest.skipIf(os.environ.get('PYTHON_UOPS_OPTIMIZE') == '0', + "Line counts differ when JIT optimizer is disabled") def test_exec_counts(self): self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) code = r'''traced_func_loop(2, 5)''' diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b9be87f357f..d107ad92594 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -19,7 +19,7 @@ requires_debug_ranges, has_no_debug_ranges, requires_subprocess) from test.support.os_helper import TESTFN, unlink -from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.script_helper import assert_python_ok, assert_python_failure, make_script from test.support.import_helper import forget from test.support import force_not_colorized, force_not_colorized_test_class @@ -1740,6 +1740,66 @@ def f(): ] self.assertEqual(result_lines, expected) +class TestKeywordTypoSuggestions(unittest.TestCase): + TYPO_CASES = [ + ("with block ad something:\n pass", "and"), + ("fur a in b:\n pass", "for"), + ("for a in b:\n pass\nelso:\n pass", "else"), + ("whille True:\n pass", "while"), + ("iff x > 5:\n pass", "if"), + ("if x:\n pass\nelseif y:\n pass", "elif"), + ("tyo:\n pass\nexcept y:\n pass", "try"), + ("classe MyClass:\n pass", "class"), + ("impor math", "import"), + ("form x import y", "from"), + ("defn calculate_sum(a, b):\n return a + b", "def"), + ("def foo():\n returm result", "return"), + ("lamda x: x ** 2", "lambda"), + ("def foo():\n yeld i", "yield"), + ("def foo():\n globel counter", "global"), + ("frum math import sqrt", "from"), + ("asynch def fetch_data():\n pass", "async"), + ("async def foo():\n awaid fetch_data()", "await"), + ('raisee ValueError("Error")', "raise"), + ("[x for x\nin range(3)\nof x]", "if"), + ("[123 fur x\nin range(3)\nif x]", "for"), + ("for x im n:\n pass", "in"), + ] + + def test_keyword_suggestions_from_file(self): + with tempfile.TemporaryDirectory() as script_dir: + for i, (code, expected_kw) in enumerate(self.TYPO_CASES): + with self.subTest(typo=expected_kw): + source = textwrap.dedent(code).strip() + script_name = make_script(script_dir, f"script_{i}", source) + rc, stdout, stderr = assert_python_failure(script_name) + stderr_text = stderr.decode('utf-8') + self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + + def test_keyword_suggestions_from_command_string(self): + for code, expected_kw in self.TYPO_CASES: + with self.subTest(typo=expected_kw): + source = textwrap.dedent(code).strip() + rc, stdout, stderr = assert_python_failure('-c', source) + stderr_text = stderr.decode('utf-8') + self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + + def test_no_keyword_suggestion_for_comma_errors(self): + # When the parser identifies a missing comma, don't suggest + # bogus keyword replacements like 'print' -> 'not' + code = '''\ +import sys +print( + "line1" + "line2" + file=sys.stderr +) +''' + source = textwrap.dedent(code).strip() + rc, stdout, stderr = assert_python_failure('-c', source) + stderr_text = stderr.decode('utf-8') + self.assertIn("Perhaps you forgot a comma", stderr_text) + self.assertNotIn("Did you mean", stderr_text) @requires_debug_ranges() @force_not_colorized_test_class @@ -4021,11 +4081,13 @@ def test_dont_swallow_subexceptions_of_falsey_exceptiongroup(self): global_for_suggestions = None -class SuggestionFormattingTestBase: +class SuggestionFormattingTestMixin: + attr_function = getattr + def get_suggestion(self, obj, attr_name=None): if attr_name is not None: def callable(): - getattr(obj, attr_name) + self.attr_function(obj, attr_name) else: callable = obj @@ -4034,7 +4096,9 @@ def callable(): ) return result_lines[0] - def test_getattr_suggestions(self): + +class BaseSuggestionTests(SuggestionFormattingTestMixin): + def test_suggestions(self): class Substitution: noise = more_noise = a = bc = None blech = None @@ -4077,7 +4141,7 @@ class CaseChangeOverSubstitution: actual = self.get_suggestion(cls(), 'bluch') self.assertIn(suggestion, actual) - def test_getattr_suggestions_underscored(self): + def test_suggestions_underscored(self): class A: bluch = None @@ -4085,10 +4149,11 @@ class A: self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + attr_function = self.attr_function class B: _bluch = None def method(self, name): - getattr(self, name) + attr_function(self, name) self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) @@ -4098,20 +4163,21 @@ def method(self, name): self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) - def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): + + def test_do_not_trigger_for_long_attributes(self): class A: blech = None actual = self.get_suggestion(A(), 'somethingverywrong') self.assertNotIn("blech", actual) - def test_getattr_error_bad_suggestions_do_not_trigger_for_small_names(self): + def test_do_not_trigger_for_small_names(self): class MyClass: vvv = mom = w = id = pytho = None for name in ("b", "v", "m", "py"): with self.subTest(name=name): - actual = self.get_suggestion(MyClass, name) + actual = self.get_suggestion(MyClass(), name) self.assertNotIn("Did you mean", actual) self.assertNotIn("'vvv", actual) self.assertNotIn("'mom'", actual) @@ -4119,7 +4185,7 @@ class MyClass: self.assertNotIn("'w'", actual) self.assertNotIn("'pytho'", actual) - def test_getattr_suggestions_do_not_trigger_for_big_dicts(self): + def test_do_not_trigger_for_big_dicts(self): class A: blech = None # A class with a very big __dict__ will not be considered @@ -4130,7 +4196,16 @@ class A: actual = self.get_suggestion(A(), 'bluch') self.assertNotIn("blech", actual) - def test_getattr_suggestions_no_args(self): + def test_suggestions_for_same_name(self): + class A: + def __dir__(self): + return ['blech'] + actual = self.get_suggestion(A(), 'blech') + self.assertNotIn("Did you mean", actual) + + +class GetattrSuggestionTests(BaseSuggestionTests): + def test_suggestions_no_args(self): class A: blech = None def __getattr__(self, attr): @@ -4147,7 +4222,7 @@ def __getattr__(self, attr): actual = self.get_suggestion(A(), 'bluch') self.assertIn("blech", actual) - def test_getattr_suggestions_invalid_args(self): + def test_suggestions_invalid_args(self): class NonStringifyClass: __str__ = None __repr__ = None @@ -4171,13 +4246,12 @@ def __getattr__(self, attr): actual = self.get_suggestion(cls(), 'bluch') self.assertIn("blech", actual) - def test_getattr_suggestions_for_same_name(self): - class A: - def __dir__(self): - return ['blech'] - actual = self.get_suggestion(A(), 'blech') - self.assertNotIn("Did you mean", actual) +class DelattrSuggestionTests(BaseSuggestionTests): + attr_function = delattr + + +class SuggestionFormattingTestBase(SuggestionFormattingTestMixin): def test_attribute_error_with_failing_dict(self): class T: bluch = 1 @@ -4188,6 +4262,15 @@ def __dir__(self): self.assertNotIn("blech", actual) self.assertNotIn("oh no!", actual) + def test_attribute_error_with_non_string_candidates(self): + class T: + bluch = 1 + + instance = T() + instance.__dict__[0] = 1 + actual = self.get_suggestion(instance, 'blich') + self.assertIn("bluch", actual) + def test_attribute_error_with_bad_name(self): def raise_attribute_error_with_bad_name(): raise AttributeError(name=12, obj=23) @@ -4210,6 +4293,184 @@ def __getattribute__(self, attr): self.assertIn("Did you mean", actual) self.assertIn("bluch", actual) + def test_getattr_nested_attribute_suggestions(self): + # Test that nested attributes are suggested when no direct match + class Inner: + def __init__(self): + self.value = 42 + self.data = "test" + + class Outer: + def __init__(self): + self.inner = Inner() + + # Should suggest 'inner.value' + actual = self.get_suggestion(Outer(), 'value') + self.assertIn("Did you mean: 'inner.value'", actual) + + # Should suggest 'inner.data' + actual = self.get_suggestion(Outer(), 'data') + self.assertIn("Did you mean: 'inner.data'", actual) + + def test_getattr_nested_prioritizes_direct_matches(self): + # Test that direct attribute matches are prioritized over nested ones + class Inner: + def __init__(self): + self.foo = 42 + + class Outer: + def __init__(self): + self.inner = Inner() + self.fooo = 100 # Similar to 'foo' + + # Should suggest 'fooo' (direct) not 'inner.foo' (nested) + actual = self.get_suggestion(Outer(), 'foo') + self.assertIn("Did you mean: 'fooo'", actual) + self.assertNotIn("inner.foo", actual) + + def test_getattr_nested_with_property(self): + # Test that descriptors (including properties) are suggested in nested attributes + class Inner: + @property + def computed(self): + return 42 + + class Outer: + def __init__(self): + self.inner = Inner() + + actual = self.get_suggestion(Outer(), 'computed') + # Descriptors should not be suggested to avoid executing arbitrary code + self.assertIn("inner.computed", actual) + + def test_getattr_nested_no_suggestion_for_deep_nesting(self): + # Test that deeply nested attributes (2+ levels) are not suggested + class Deep: + def __init__(self): + self.value = 42 + + class Middle: + def __init__(self): + self.deep = Deep() + + class Outer: + def __init__(self): + self.middle = Middle() + + # Should not suggest 'middle.deep.value' (too deep) + actual = self.get_suggestion(Outer(), 'value') + self.assertNotIn("Did you mean", actual) + + def test_getattr_nested_ignores_private_attributes(self): + # Test that nested suggestions ignore private attributes + class Inner: + def __init__(self): + self.public_value = 42 + + class Outer: + def __init__(self): + self._private_inner = Inner() + + # Should not suggest '_private_inner.public_value' + actual = self.get_suggestion(Outer(), 'public_value') + self.assertNotIn("Did you mean", actual) + + def test_getattr_nested_limits_attribute_checks(self): + # Test that nested suggestions are limited to checking first 20 non-private attributes + class Inner: + def __init__(self): + self.target_value = 42 + + class Outer: + def __init__(self): + # Add many attributes before 'inner' + for i in range(25): + setattr(self, f'attr_{i:02d}', i) + # Add the inner object after 20+ attributes + self.inner = Inner() + + obj = Outer() + # Verify that 'inner' is indeed present but after position 20 + attrs = [x for x in sorted(dir(obj)) if not x.startswith('_')] + inner_position = attrs.index('inner') + self.assertGreater(inner_position, 19, "inner should be after position 20 in sorted attributes") + + # Should not suggest 'inner.target_value' because inner is beyond the first 20 attributes checked + actual = self.get_suggestion(obj, 'target_value') + self.assertNotIn("inner.target_value", actual) + + def test_getattr_nested_returns_first_match_only(self): + # Test that only the first nested match is returned (not multiple) + class Inner1: + def __init__(self): + self.value = 1 + + class Inner2: + def __init__(self): + self.value = 2 + + class Inner3: + def __init__(self): + self.value = 3 + + class Outer: + def __init__(self): + # Multiple inner objects with same attribute + self.a_inner = Inner1() + self.b_inner = Inner2() + self.c_inner = Inner3() + + # Should suggest only the first match (alphabetically) + actual = self.get_suggestion(Outer(), 'value') + self.assertIn("'a_inner.value'", actual) + # Verify it's a single suggestion, not multiple + self.assertEqual(actual.count("Did you mean"), 1) + + def test_getattr_nested_handles_attribute_access_exceptions(self): + # Test that exceptions raised when accessing attributes don't crash the suggestion system + class ExplodingProperty: + @property + def exploding_attr(self): + raise RuntimeError("BOOM! This property always explodes") + + def __repr__(self): + raise RuntimeError("repr also explodes") + + class SafeInner: + def __init__(self): + self.target = 42 + + class Outer: + def __init__(self): + self.exploder = ExplodingProperty() # Accessing attributes will raise + self.safe_inner = SafeInner() + + # Should still suggest 'safe_inner.target' without crashing + # even though accessing exploder.target would raise an exception + actual = self.get_suggestion(Outer(), 'target') + self.assertIn("'safe_inner.target'", actual) + + def test_getattr_nested_handles_hasattr_exceptions(self): + # Test that exceptions in hasattr don't crash the system + class WeirdObject: + def __getattr__(self, name): + if name == 'target': + raise RuntimeError("Can't check for target attribute") + raise AttributeError(f"No attribute {name}") + + class NormalInner: + def __init__(self): + self.target = 100 + + class Outer: + def __init__(self): + self.weird = WeirdObject() # hasattr will raise for 'target' + self.normal = NormalInner() + + # Should still find 'normal.target' even though weird.target check fails + actual = self.get_suggestion(Outer(), 'target') + self.assertIn("'normal.target'", actual) + def make_module(self, code): tmpdir = Path(tempfile.mkdtemp()) self.addCleanup(shutil.rmtree, tmpdir) @@ -4223,8 +4484,8 @@ def make_module(self, code): return mod_name - def get_import_from_suggestion(self, mod_dict, name): - modname = self.make_module(mod_dict) + def get_import_from_suggestion(self, code, name): + modname = self.make_module(code) def callable(): try: @@ -4301,6 +4562,13 @@ def test_import_from_suggestions_underscored(self): self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + def test_import_from_suggestions_non_string(self): + modWithNonStringAttr = textwrap.dedent("""\ + globals()[0] = 1 + bluch = 1 + """) + self.assertIn("'bluch'", self.get_import_from_suggestion(modWithNonStringAttr, 'blech')) + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): code = "blech = None" @@ -4397,6 +4665,15 @@ def func(): actual = self.get_suggestion(func) self.assertIn("'ZeroDivisionError'?", actual) + def test_name_error_suggestions_with_non_string_candidates(self): + def func(): + abc = 1 + custom_globals = globals().copy() + custom_globals[0] = 1 + print(eval("abv", custom_globals, locals())) + actual = self.get_suggestion(func) + self.assertIn("abc", actual) + def test_name_error_suggestions_do_not_trigger_for_long_names(self): def func(): somethingverywronghehehehehehe = None @@ -4630,6 +4907,51 @@ class CPythonSuggestionFormattingTests( """ +class PurePythonGetattrSuggestionFormattingTests( + PurePythonExceptionFormattingMixin, + GetattrSuggestionTests, + unittest.TestCase, +): + """ + Same set of tests (for attribute access) as above using the pure Python + implementation of traceback printing in traceback.py. + """ + + +class PurePythonDelattrSuggestionFormattingTests( + PurePythonExceptionFormattingMixin, + DelattrSuggestionTests, + unittest.TestCase, +): + """ + Same set of tests (for attribute deletion) as above using the pure Python + implementation of traceback printing in traceback.py. + """ + + +@cpython_only +class CPythonGetattrSuggestionFormattingTests( + CAPIExceptionFormattingMixin, + GetattrSuggestionTests, + unittest.TestCase, +): + """ + Same set of tests (for attribute access) as above but with Python's + internal traceback printing. + """ + + +@cpython_only +class CPythonDelattrSuggestionFormattingTests( + CAPIExceptionFormattingMixin, + DelattrSuggestionTests, + unittest.TestCase, +): + """ + Same set of tests (for attribute deletion) as above but with Python's + internal traceback printing. + """ + class MiscTest(unittest.TestCase): def test_all(self): @@ -4723,7 +5045,59 @@ class MyList(list): with self.assertRaises(TypeError): _suggestions._generate_suggestions(MyList(), "") + def test_no_site_package_flavour(self): + code = """import boo""" + _, _, stderr = assert_python_failure('-S', '-c', code) + self.assertIn( + (b"Site initialization is disabled, did you forget to " + b"add the site-packages directory to sys.path " + b"or to enable your virtual environment?"), stderr + ) + + code = """ + import sys + sys.stdlib_module_names = sys.stdlib_module_names + ("boo",) + import boo + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertNotIn( + (b"Site initialization is disabled, did you forget to " + b"add the site-packages directory to sys.path " + b"or to enable your virtual environment?"), stderr + ) + + def test_missing_stdlib_module(self): + code = """ + import sys + sys.stdlib_module_names |= {'spam'} + import spam + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertIn(b"Standard library module 'spam' was not found", stderr) + + code = """ + import sys + import traceback + traceback._MISSING_STDLIB_MODULE_MESSAGES = {'spam': "Install 'spam4life' for 'spam'"} + sys.stdlib_module_names |= {'spam'} + import spam + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertIn(b"Install 'spam4life' for 'spam'", stderr) + + @unittest.skipIf(sys.platform == "win32", "Non-Windows test") + def test_windows_only_module_error(self): + try: + import msvcrt # noqa: F401 + except ModuleNotFoundError: + formatted = traceback.format_exc() + self.assertIn("Unsupported platform for Windows-only standard library module 'msvcrt'", formatted) + else: + self.fail("ModuleNotFoundError was not raised") class TestColorizedTraceback(unittest.TestCase): diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 6dc6880ff8c..9d3ff8a620b 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -8,7 +8,7 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, interpreter_requires_environment) from test import support -from test.support import force_not_colorized +from test.support import force_not_colorized, warnings_helper from test.support import os_helper from test.support import threading_helper @@ -354,6 +354,7 @@ def fork_child(self): # everything is fine return 0 + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_fork() def test_fork(self): # check that tracemalloc is still working after fork diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index e72a1ea5417..74653c77c55 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -150,7 +150,6 @@ def test_raw_tstrings(self): t = tr"{path}\Documents" self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")]) - def test_template_concatenation(self): # Test template + template t1 = t"Hello, " @@ -161,9 +160,10 @@ def test_template_concatenation(self): # Test template + string t1 = t"Hello" - combined = t1 + ", world" - self.assertTStringEqual(combined, ("Hello, world",), ()) - self.assertEqual(fstring(combined), "Hello, world") + expected_msg = 'can only concatenate string.templatelib.Template ' \ + '\\(not "str"\\) to string.templatelib.Template' + with self.assertRaisesRegex(TypeError, expected_msg): + t1 + ", world" # Test template + template with interpolation name = "Python" @@ -174,9 +174,10 @@ def test_template_concatenation(self): self.assertEqual(fstring(combined), "Hello, Python") # Test string + template - t = "Hello, " + t"{name}" - self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) - self.assertEqual(fstring(t), "Hello, Python") + expected_msg = 'can only concatenate str ' \ + '\\(not "string.templatelib.Template"\\) to str' + with self.assertRaisesRegex(TypeError, expected_msg): + "Hello, " + t"{name}" def test_nested_templates(self): # Test a template inside another template expression @@ -219,6 +220,7 @@ def test_syntax_errors(self): ("t'{lambda:1}'", "t-string: lambda expressions are not allowed " "without parentheses"), ("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"), + ("t'{1:d\n}'", "t-string: newlines are not allowed in format specifiers") ): with self.subTest(case), self.assertRaisesRegex(SyntaxError, err): eval(case) @@ -240,52 +242,28 @@ def test_literal_concatenation(self): self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) self.assertEqual(fstring(t), "Hello, Python") - # Test concatenation with string literal - name = "Python" - t = t"Hello, {name}" "and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with Unicode literal - name = "Python" - t = t"Hello, {name}" u"and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with f-string literal - tab = '\t' - t = t"Tab: {tab}. " f"f-tab: {tab}." - self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")]) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.") - - # Test concatenation with raw string literal - tab = '\t' - t = t"Tab: {tab}. " r"Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.") - - # Test concatenation with raw f-string literal - tab = '\t' - t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.") - + # Test disallowed mix of t-string and string/f-string (incl. bytes) what = 't' - expected_msg = 'cannot mix bytes and nonbytes literals' + expected_msg = 'cannot mix t-string literals with string or bytes literals' for case in ( + "t'{what}-string literal' 'str literal'", + "t'{what}-string literal' u'unicode literal'", + "t'{what}-string literal' f'f-string literal'", + "t'{what}-string literal' r'raw string literal'", + "t'{what}-string literal' rf'raw f-string literal'", "t'{what}-string literal' b'bytes literal'", "t'{what}-string literal' br'raw bytes literal'", + "'str literal' t'{what}-string literal'", + "u'unicode literal' t'{what}-string literal'", + "f'f-string literal' t'{what}-string literal'", + "r'raw string literal' t'{what}-string literal'", + "rf'raw f-string literal' t'{what}-string literal'", + "b'bytes literal' t'{what}-string literal'", + "br'raw bytes literal' t'{what}-string literal'", ): - with self.assertRaisesRegex(SyntaxError, expected_msg): - eval(case) + with self.subTest(case): + with self.assertRaisesRegex(SyntaxError, expected_msg): + eval(case) def test_triple_quoted(self): # Test triple-quoted t-strings diff --git a/Lib/test/test_ttk/__init__.py b/Lib/test/test_ttk/__init__.py index 7ee7ffbd6d7..4a077a0f966 100644 --- a/Lib/test/test_ttk/__init__.py +++ b/Lib/test/test_ttk/__init__.py @@ -19,6 +19,16 @@ from tkinter import ttk +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(ttk, "__version__") + self.assertEqual(cm.filename, __file__) + + def setUpModule(): root = None try: diff --git a/Lib/test/test_tuple.py b/Lib/test/test_tuple.py index 9ce80c5e8ea..e533392b8ca 100644 --- a/Lib/test/test_tuple.py +++ b/Lib/test/test_tuple.py @@ -290,12 +290,18 @@ def test_repr(self): self.assertEqual(repr(a0), "()") self.assertEqual(repr(a2), "(0, 1, 2)") + # Checks that t is not tracked without any GC collections. + def _not_tracked_instantly(self, t): + self.assertFalse(gc.is_tracked(t), t) + + # Checks that t is not tracked after GC collection. def _not_tracked(self, t): # Nested tuples can take several collections to untrack gc.collect() gc.collect() self.assertFalse(gc.is_tracked(t), t) + # Checks that t continues to be tracked even after GC collection. def _tracked(self, t): self.assertTrue(gc.is_tracked(t), t) gc.collect() @@ -307,13 +313,19 @@ def test_track_literals(self): # Test GC-optimization of tuple literals x, y, z = 1.5, "a", [] - self._not_tracked(()) - self._not_tracked((1,)) - self._not_tracked((1, 2)) - self._not_tracked((1, 2, "a")) - self._not_tracked((1, 2, (None, True, False, ()), int)) - self._not_tracked((object(),)) + # We check that those objects aren't tracked at all. + # It's essential for the GC performance, see gh-139951. + self._not_tracked_instantly(()) + self._not_tracked_instantly((1,)) + self._not_tracked_instantly((1, 2)) + self._not_tracked_instantly((1, 2, "a")) + self._not_tracked_instantly((1, 2) * 5) + self._not_tracked_instantly((12, 10**10, 'a_' * 100)) + self._not_tracked_instantly((object(),)) + self._not_tracked(((1, x), y, (2, 3))) + self._not_tracked((1, 2, (None, True, False, ()), int)) + self._not_tracked((object(), ())) # Tuples with mutable elements are always tracked, even if those # elements are not tracked right now. @@ -343,6 +355,12 @@ def check_track_dynamic(self, tp, always_track): self._tracked(tp(tuple([obj]) for obj in [x, y, z])) self._tracked(tuple(tp([obj]) for obj in [x, y, z])) + t = tp([1, x, y, z]) + self.assertEqual(type(t), tp) + self._tracked(t) + self.assertEqual(type(t[:]), tuple) + self._tracked(t[:]) + @support.cpython_only def test_track_dynamic(self): # Test GC-optimization of dynamically constructed tuples. diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index d02cac284a9..12d2eed8741 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -60,12 +60,25 @@ def patch_screen(): We must patch the _Screen class itself instead of the _Screen instance because instantiating it requires a display. """ + # Create a mock screen that delegates color validation to the real TurtleScreen methods + mock_screen = unittest.mock.MagicMock() + mock_screen.__class__ = turtle._Screen + mock_screen.mode.return_value = "standard" + mock_screen._colormode = 1.0 + + def mock_iscolorstring(color): + valid_colors = {'red', 'green', 'blue', 'black', 'white', 'yellow', + 'orange', 'purple', 'pink', 'brown', 'gray', 'grey', + 'cyan', 'magenta'} + + return color in valid_colors or (isinstance(color, str) and color.startswith('#')) + + mock_screen._iscolorstring = mock_iscolorstring + mock_screen._colorstr = turtle._Screen._colorstr.__get__(mock_screen) + return unittest.mock.patch( "turtle._Screen.__new__", - **{ - "return_value.__class__": turtle._Screen, - "return_value.mode.return_value": "standard", - }, + return_value=mock_screen ) @@ -635,6 +648,28 @@ def test_poly_context_when_creating_poly(self): self.assertTrue(self.turtle._creatingPoly) self.assertFalse(self.turtle._creatingPoly) + def test_dot_signature(self): + self.turtle.dot() + self.turtle.dot(10) + self.turtle.dot(size=10) + self.turtle.dot((0, 0, 0)) + self.turtle.dot(size=(0, 0, 0)) + self.turtle.dot("blue") + self.turtle.dot("") + self.turtle.dot(size="blue") + self.turtle.dot(20, "blue") + self.turtle.dot(20, "blue") + self.turtle.dot(20, (0, 0, 0)) + self.turtle.dot(20, 0, 0, 0) + with self.assertRaises(TypeError): + self.turtle.dot(color="blue") + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, "_not_a_color_") + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, (0, 0, 0, 0)) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, 0, 0, 0, 0) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, (-1, 0, 0)) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, -1, 0, 0) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, (0, 257, 0)) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, 0, 257, 0) class TestModuleLevel(unittest.TestCase): def test_all_signatures(self): diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index ee1791bc1d0..9ceee565764 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -8,6 +8,11 @@ Callable, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, Unpack, get_args, ) +type GlobalTypeAlias = int + +def get_type_alias(): + type TypeAliasInFunc = str + return TypeAliasInFunc class TypeParamsInvalidTest(unittest.TestCase): def test_name_collisions(self): @@ -70,6 +75,8 @@ def inner[B](self): class TypeParamsAliasValueTest(unittest.TestCase): + type TypeAliasInClass = dict + def test_alias_value_01(self): type TA1 = int @@ -142,33 +149,67 @@ def test_subscripting(self): self.assertIs(specialized2.__origin__, VeryGeneric) self.assertEqual(specialized2.__args__, (int, str, float, [bool, range])) + def test___name__(self): + type TypeAliasLocal = GlobalTypeAlias + + self.assertEqual(GlobalTypeAlias.__name__, 'GlobalTypeAlias') + self.assertEqual(get_type_alias().__name__, 'TypeAliasInFunc') + self.assertEqual(self.TypeAliasInClass.__name__, 'TypeAliasInClass') + self.assertEqual(TypeAliasLocal.__name__, 'TypeAliasLocal') + + with self.assertRaisesRegex( + AttributeError, + "readonly attribute", + ): + setattr(TypeAliasLocal, '__name__', 'TA') + + def test___qualname__(self): + type TypeAliasLocal = GlobalTypeAlias + + self.assertEqual(GlobalTypeAlias.__qualname__, + 'GlobalTypeAlias') + self.assertEqual(get_type_alias().__qualname__, + 'get_type_alias.<locals>.TypeAliasInFunc') + self.assertEqual(self.TypeAliasInClass.__qualname__, + 'TypeParamsAliasValueTest.TypeAliasInClass') + self.assertEqual(TypeAliasLocal.__qualname__, + 'TypeParamsAliasValueTest.test___qualname__.<locals>.TypeAliasLocal') + + with self.assertRaisesRegex( + AttributeError, + "readonly attribute", + ): + setattr(TypeAliasLocal, '__qualname__', 'TA') + def test_repr(self): type Simple = int - type VeryGeneric[T, *Ts, **P] = Callable[P, tuple[T, *Ts]] + self.assertEqual(repr(Simple), Simple.__qualname__) - self.assertEqual(repr(Simple), "Simple") - self.assertEqual(repr(VeryGeneric), "VeryGeneric") + type VeryGeneric[T, *Ts, **P] = Callable[P, tuple[T, *Ts]] + self.assertEqual(repr(VeryGeneric), VeryGeneric.__qualname__) + fullname = f"{VeryGeneric.__module__}.{VeryGeneric.__qualname__}" self.assertEqual(repr(VeryGeneric[int, bytes, str, [float, object]]), - "VeryGeneric[int, bytes, str, [float, object]]") + f"{fullname}[int, bytes, str, [float, object]]") self.assertEqual(repr(VeryGeneric[int, []]), - "VeryGeneric[int, []]") + f"{fullname}[int, []]") self.assertEqual(repr(VeryGeneric[int, [VeryGeneric[int], list[str]]]), - "VeryGeneric[int, [VeryGeneric[int], list[str]]]") + f"{fullname}[int, [{fullname}[int], list[str]]]") def test_recursive_repr(self): type Recursive = Recursive - self.assertEqual(repr(Recursive), "Recursive") + self.assertEqual(repr(Recursive), Recursive.__qualname__) type X = list[Y] type Y = list[X] - self.assertEqual(repr(X), "X") - self.assertEqual(repr(Y), "Y") + self.assertEqual(repr(X), X.__qualname__) + self.assertEqual(repr(Y), Y.__qualname__) type GenericRecursive[X] = list[X | GenericRecursive[X]] - self.assertEqual(repr(GenericRecursive), "GenericRecursive") - self.assertEqual(repr(GenericRecursive[int]), "GenericRecursive[int]") + self.assertEqual(repr(GenericRecursive), GenericRecursive.__qualname__) + fullname = f"{GenericRecursive.__module__}.{GenericRecursive.__qualname__}" + self.assertEqual(repr(GenericRecursive[int]), f"{fullname}[int]") self.assertEqual(repr(GenericRecursive[GenericRecursive[int]]), - "GenericRecursive[GenericRecursive[int]]") + f"{fullname}[{fullname}[int]]") def test_raising(self): type MissingName = list[_My_X] @@ -193,15 +234,25 @@ class TypeAliasConstructorTest(unittest.TestCase): def test_basic(self): TA = TypeAliasType("TA", int) self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__qualname__, "TA") self.assertIs(TA.__value__, int) self.assertEqual(TA.__type_params__, ()) self.assertEqual(TA.__module__, __name__) + def test_with_qualname(self): + TA = TypeAliasType("TA", str, qualname="Class.TA") + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__qualname__, "Class.TA") + self.assertIs(TA.__value__, str) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + def test_attributes_with_exec(self): ns = {} exec("type TA = int", ns, ns) TA = ns["TA"] self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__qualname__, "TA") self.assertIs(TA.__value__, int) self.assertEqual(TA.__type_params__, ()) self.assertIs(TA.__module__, None) @@ -210,6 +261,7 @@ def test_generic(self): T = TypeVar("T") TA = TypeAliasType("TA", list[T], type_params=(T,)) self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__qualname__, "TA") self.assertEqual(TA.__value__, list[T]) self.assertEqual(TA.__type_params__, (T,)) self.assertEqual(TA.__module__, __name__) @@ -218,6 +270,7 @@ def test_generic(self): def test_not_generic(self): TA = TypeAliasType("TA", list[int], type_params=()) self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__qualname__, "TA") self.assertEqual(TA.__value__, list[int]) self.assertEqual(TA.__type_params__, ()) self.assertEqual(TA.__module__, __name__) @@ -268,8 +321,9 @@ def test_expects_type_like(self): TypeAliasType("A", int, type_params=(T, 2)) def test_keywords(self): - TA = TypeAliasType(name="TA", value=int) + TA = TypeAliasType(name="TA", value=int, type_params=(), qualname=None) self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__qualname__, "TA") self.assertIs(TA.__value__, int) self.assertEqual(TA.__type_params__, ()) self.assertEqual(TA.__module__, __name__) @@ -283,6 +337,8 @@ def test_errors(self): TypeAliasType("TA", list, ()) with self.assertRaises(TypeError): TypeAliasType("TA", list, type_params=42) + with self.assertRaises(TypeError): + TypeAliasType("TA", list, qualname=range(5)) class TypeAliasTypeTest(unittest.TestCase): diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 2c886bb6d36..d40eb382c53 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -309,7 +309,7 @@ def check_annotations(self, f): print(f.__annotations__) f.__annotate__ = lambda x: 42 - with self.assertRaisesRegex(TypeError, r"__annotate__ returned non-dict of type 'int'"): + with self.assertRaisesRegex(TypeError, r"__annotate__\(\) must return a dict, not int"): print(f.__annotations__) f.__annotate__ = lambda x: {"x": x} @@ -498,6 +498,28 @@ def f(x: int) -> int: pass self.assertEqual(f.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(f.__annotations__, annos) + def test_set_annotations(self): + function_code = textwrap.dedent(""" + def f(x: int): + pass + """) + class_code = textwrap.dedent(""" + class f: + x: int + """) + for future in (False, True): + for label, code in (("function", function_code), ("class", class_code)): + with self.subTest(future=future, label=label): + if future: + code = "from __future__ import annotations\n" + code + ns = run_code(code) + f = ns["f"] + anno = "int" if future else int + self.assertEqual(f.__annotations__, {"x": anno}) + + f.__annotations__ = {"x": str} + self.assertEqual(f.__annotations__, {"x": str}) + def test_name_clash_with_format(self): # this test would fail if __annotate__'s parameter was called "format" # during symbol table construction @@ -813,3 +835,40 @@ def test_complex_comprehension_inlining_exec(self): genexp = annos["unique_name_2"][0] lamb = list(genexp)[0] self.assertEqual(lamb(), 42) + + # gh-138349 + def test_module_level_annotation_plus_listcomp(self): + cases = [ + """ + def report_error(): + pass + try: + [0 for name_2 in unique_name_0 if (lambda: name_2)] + except: + pass + annotated_name: 0 + """, + """ + class Generic: + pass + try: + [0 for name_2 in unique_name_0 if (0 for unique_name_1 in unique_name_2 for unique_name_3 in name_2)] + except: + pass + annotated_name: 0 + """, + """ + class Generic: + pass + annotated_name: 0 + try: + [0 for name_2 in [[0]] for unique_name_1 in unique_name_2 if (lambda: name_2)] + except: + pass + """, + ] + for code in cases: + with self.subTest(code=code): + mod = build_module(code) + annos = mod.__annotations__ + self.assertEqual(annos, {"annotated_name": 0}) diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index ee8939f62d0..c40c45594f4 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -344,7 +344,7 @@ def test_longargs(self): todo = set(t.name[1:]) self.assertEqual(len(t.args.args) + len(t.args.posonlyargs), len(todo) - bool(t.args.vararg) - bool(t.args.kwarg)) - self.assertTrue(t.name.startswith('f'), t.name) + self.assertStartsWith(t.name, 'f') for index, c in enumerate(t.name[1:]): todo.remove(c) if c == 'v': diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3552b6b4ef8..4595e7e5d3e 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2,7 +2,7 @@ from test.support import ( run_with_locale, cpython_only, no_rerun, - MISSING_C_DOCSTRINGS, EqualToForwardRef, + MISSING_C_DOCSTRINGS, EqualToForwardRef, check_disallow_instantiation, ) from test.support.script_helper import assert_python_ok from test.support.import_helper import import_fresh_module @@ -21,6 +21,7 @@ import unittest.mock import weakref import typing +import re c_types = import_fresh_module('types', fresh=['_types']) py_types = import_fresh_module('types', blocked=['_types']) @@ -52,6 +53,7 @@ def test_names(self): 'AsyncGeneratorType', 'BuiltinFunctionType', 'BuiltinMethodType', 'CapsuleType', 'CellType', 'ClassMethodDescriptorType', 'CodeType', 'CoroutineType', 'EllipsisType', 'FrameType', 'FunctionType', + 'FrameLocalsProxyType', 'GeneratorType', 'GenericAlias', 'GetSetDescriptorType', 'LambdaType', 'MappingProxyType', 'MemberDescriptorType', 'MethodDescriptorType', 'MethodType', 'MethodWrapperType', @@ -517,8 +519,8 @@ def test(f, format_spec, result): # and a number after the decimal. This is tricky, because # a totally empty format specifier means something else. # So, just use a sign flag - test(1e200, '+g', '+1e+200') - test(1e200, '+', '+1e+200') + test(1.25e200, '+g', '+1.25e+200') + test(1.25e200, '+', '+1.25e+200') test(1.1e200, '+g', '+1.1e+200') test(1.1e200, '+', '+1.1e+200') @@ -710,6 +712,19 @@ def call(part): """ assert_python_ok("-c", code) + def test_frame_locals_proxy_type(self): + self.assertIsInstance(types.FrameLocalsProxyType, type) + if MISSING_C_DOCSTRINGS: + self.assertIsNone(types.FrameLocalsProxyType.__doc__) + else: + self.assertIsInstance(types.FrameLocalsProxyType.__doc__, str) + self.assertEqual(types.FrameLocalsProxyType.__module__, 'builtins') + self.assertEqual(types.FrameLocalsProxyType.__name__, 'FrameLocalsProxy') + + frame = inspect.currentframe() + self.assertIsNotNone(frame) + self.assertIsInstance(frame.f_locals, types.FrameLocalsProxyType) + class UnionTests(unittest.TestCase): @@ -827,15 +842,15 @@ def test_instancecheck_and_subclasscheck(self): self.assertIsInstance(True, x) self.assertIsInstance('a', x) self.assertNotIsInstance(None, x) - self.assertTrue(issubclass(int, x)) - self.assertTrue(issubclass(bool, x)) - self.assertTrue(issubclass(str, x)) - self.assertFalse(issubclass(type(None), x)) + self.assertIsSubclass(int, x) + self.assertIsSubclass(bool, x) + self.assertIsSubclass(str, x) + self.assertNotIsSubclass(type(None), x) for x in (int | None, typing.Union[int, None]): with self.subTest(x=x): self.assertIsInstance(None, x) - self.assertTrue(issubclass(type(None), x)) + self.assertIsSubclass(type(None), x) for x in ( int | collections.abc.Mapping, @@ -844,8 +859,8 @@ def test_instancecheck_and_subclasscheck(self): with self.subTest(x=x): self.assertIsInstance({}, x) self.assertNotIsInstance((), x) - self.assertTrue(issubclass(dict, x)) - self.assertFalse(issubclass(list, x)) + self.assertIsSubclass(dict, x) + self.assertNotIsSubclass(list, x) def test_instancecheck_and_subclasscheck_order(self): T = typing.TypeVar('T') @@ -857,7 +872,7 @@ def test_instancecheck_and_subclasscheck_order(self): for x in will_resolve: with self.subTest(x=x): self.assertIsInstance(1, x) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) wont_resolve = ( T | int, @@ -890,7 +905,7 @@ class BadMeta(type): def __subclasscheck__(cls, sub): 1/0 x = int | BadMeta('A', (), {}) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) self.assertRaises(ZeroDivisionError, issubclass, list, x) def test_or_type_operator_with_TypeVar(self): @@ -1148,8 +1163,7 @@ def test_or_type_operator_reference_cycle(self): msg='Check for union reference leak.') def test_instantiation(self): - with self.assertRaises(TypeError): - types.UnionType() + check_disallow_instantiation(self, types.UnionType) self.assertIs(int, types.UnionType[int]) self.assertIs(int, types.UnionType[int, int]) self.assertEqual(int | str, types.UnionType[int, str]) @@ -1376,6 +1390,27 @@ def __hash__(self): view = self.mappingproxy(mapping) self.assertEqual(hash(view), hash(mapping)) + def test_richcompare(self): + mp1 = self.mappingproxy({'a': 1}) + mp1_2 = self.mappingproxy({'a': 1}) + mp2 = self.mappingproxy({'a': 2}) + + self.assertTrue(mp1 == mp1_2) + self.assertFalse(mp1 != mp1_2) + self.assertFalse(mp1 == mp2) + self.assertTrue(mp1 != mp2) + + msg = "not supported between instances of 'mappingproxy' and 'mappingproxy'" + + with self.assertRaisesRegex(TypeError, msg): + mp1 > mp2 + with self.assertRaisesRegex(TypeError, msg): + mp1 < mp1_2 + with self.assertRaisesRegex(TypeError, msg): + mp2 >= mp2 + with self.assertRaisesRegex(TypeError, msg): + mp1_2 <= mp1 + class ClassCreationTests(unittest.TestCase): @@ -1399,7 +1434,7 @@ def test_new_class_basics(self): def test_new_class_subclass(self): C = types.new_class("C", (int,)) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) def test_new_class_meta(self): Meta = self.Meta @@ -1444,7 +1479,7 @@ def func(ns): bases=(int,), kwds=dict(metaclass=Meta, z=2), exec_body=func) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) self.assertIsInstance(C, Meta) self.assertEqual(C.x, 0) self.assertEqual(C.y, 1) @@ -2010,6 +2045,24 @@ def test_equal(self): self.assertEqual(ns1, ns2) self.assertNotEqual(ns2, types.SimpleNamespace()) + def test_richcompare_unsupported(self): + ns1 = types.SimpleNamespace(x=1) + ns2 = types.SimpleNamespace(y=2) + + msg = re.escape( + "not supported between instances of " + "'types.SimpleNamespace' and 'types.SimpleNamespace'" + ) + + with self.assertRaisesRegex(TypeError, msg): + ns1 > ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 >= ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 < ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 <= ns2 + def test_nested(self): ns1 = types.SimpleNamespace(a=1, b=2) ns2 = types.SimpleNamespace() @@ -2513,15 +2566,16 @@ class SubinterpreterTests(unittest.TestCase): def setUpClass(cls): global interpreters try: - from test.support import interpreters + from concurrent import interpreters except ModuleNotFoundError: raise unittest.SkipTest('subinterpreters required') - import test.support.interpreters.channels + from test.support import channels # noqa: F401 + cls.create_channel = staticmethod(channels.create) @cpython_only @no_rerun('channels (and queues) might have a refleak; see gh-122199') def test_static_types_inherited_slots(self): - rch, sch = interpreters.channels.create() + rch, sch = self.create_channel() script = textwrap.dedent(""" import test.support @@ -2547,7 +2601,7 @@ def collate_results(raw): main_results = collate_results(raw) interp = interpreters.create() - interp.exec('from test.support import interpreters') + interp.exec('from concurrent import interpreters') interp.prepare_main(sch=sch) interp.exec(script) raw = rch.recv_nowait() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 8c55ba4623e..e896df51844 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -13,6 +13,7 @@ import pickle import re import sys +import warnings from unittest import TestCase, main, skip from unittest.mock import patch from copy import copy, deepcopy @@ -32,9 +33,9 @@ from typing import is_typeddict, is_protocol from typing import reveal_type from typing import dataclass_transform -from typing import no_type_check, no_type_check_decorator +from typing import no_type_check from typing import Type -from typing import NamedTuple, NotRequired, Required, ReadOnly, TypedDict +from typing import NamedTuple, NotRequired, Required, ReadOnly, TypedDict, NoExtraItems from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef @@ -46,11 +47,10 @@ import textwrap import typing import weakref -import warnings import types from test.support import ( - captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper, run_code, + captured_stderr, cpython_only, requires_docstrings, import_helper, run_code, EqualToForwardRef, ) from test.typinganndata import ( @@ -762,6 +762,16 @@ class A(Generic[T, P, U]): ... self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + def test_paramspec_and_typevar_specialization_2(self): + T = TypeVar("T") + P = ParamSpec('P', default=...) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, ...) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, ..., float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + def test_typevartuple_none(self): U = TypeVarTuple('U') U_None = TypeVarTuple('U_None', default=None) @@ -1606,7 +1616,10 @@ def func1(*args: *Ts): pass self.assertEqual(gth(func1), {'args': Unpack[Ts]}) def func2(*args: *tuple[int, str]): pass - self.assertEqual(gth(func2), {'args': Unpack[tuple[int, str]]}) + hint = gth(func2)['args'] + self.assertIsInstance(hint, types.GenericAlias) + self.assertEqual(hint.__args__[0], int) + self.assertIs(hint.__unpacked__, True) class CustomVariadic(Generic[*Ts]): pass @@ -1621,7 +1634,10 @@ def func1(*args: '*Ts'): pass {'args': Unpack[Ts]}) def func2(*args: '*tuple[int, str]'): pass - self.assertEqual(gth(func2), {'args': Unpack[tuple[int, str]]}) + hint = gth(func2)['args'] + self.assertIsInstance(hint, types.GenericAlias) + self.assertEqual(hint.__args__[0], int) + self.assertIs(hint.__unpacked__, True) class CustomVariadic(Generic[*Ts]): pass @@ -2267,6 +2283,15 @@ class Ints(enum.IntEnum): self.assertEqual(Union[Literal[1], Literal[Ints.B], Literal[True]].__args__, (Literal[1], Literal[Ints.B], Literal[True])) + def test_allow_non_types_in_or(self): + # gh-140348: Test that using | with a Union object allows things that are + # not allowed by is_unionable(). + U1 = Union[int, str] + self.assertEqual(U1 | float, Union[int, str, float]) + self.assertEqual(U1 | "float", Union[int, str, "float"]) + self.assertEqual(float | U1, Union[float, int, str]) + self.assertEqual("float" | U1, Union["float", int, str]) + class TupleTests(BaseTestCase): @@ -3953,6 +3978,7 @@ class C: pass def test_defining_generic_protocols(self): T = TypeVar('T') + T2 = TypeVar('T2') S = TypeVar('S') @runtime_checkable @@ -3962,17 +3988,26 @@ def meth(self): pass class P(PR[int, T], Protocol[T]): y = 1 + self.assertEqual(P.__parameters__, (T,)) + with self.assertRaises(TypeError): PR[int] with self.assertRaises(TypeError): P[int, str] + with self.assertRaisesRegex( + TypeError, + re.escape('Some type variables (~S) are not listed in Protocol[~T, ~T2]'), + ): + class ExtraTypeVars(P[S], Protocol[T, T2]): ... class C(PR[int, T]): pass + self.assertEqual(C.__parameters__, (T,)) self.assertIsInstance(C[str](), C) def test_defining_generic_protocols_old_style(self): T = TypeVar('T') + T2 = TypeVar('T2') S = TypeVar('S') @runtime_checkable @@ -3991,9 +4026,19 @@ class P(PR[int, str], Protocol): class P1(Protocol, Generic[T]): def bar(self, x: T) -> str: ... + self.assertEqual(P1.__parameters__, (T,)) + class P2(Generic[T], Protocol): def bar(self, x: T) -> str: ... + self.assertEqual(P2.__parameters__, (T,)) + + msg = re.escape('Some type variables (~S) are not listed in Protocol[~T, ~T2]') + with self.assertRaisesRegex(TypeError, msg): + class ExtraTypeVars(P1[S], Protocol[T, T2]): ... + with self.assertRaisesRegex(TypeError, msg): + class ExtraTypeVars(P2[S], Protocol[T, T2]): ... + @runtime_checkable class PSub(P1[str], Protocol): x = 1 @@ -4006,6 +4051,28 @@ def bar(self, x: str) -> str: self.assertIsInstance(Test(), PSub) + def test_protocol_parameter_order(self): + # https://github.com/python/cpython/issues/137191 + T1 = TypeVar("T1") + T2 = TypeVar("T2", default=object) + + class A(Protocol[T1]): ... + + class B0(A[T2], Generic[T1, T2]): ... + self.assertEqual(B0.__parameters__, (T1, T2)) + + class B1(A[T2], Protocol, Generic[T1, T2]): ... + self.assertEqual(B1.__parameters__, (T1, T2)) + + class B2(A[T2], Protocol[T1, T2]): ... + self.assertEqual(B2.__parameters__, (T1, T2)) + + class B3[T1, T2](A[T2], Protocol): + @staticmethod + def get_typeparams(): + return (T1, T2) + self.assertEqual(B3.__parameters__, B3.get_typeparams()) + def test_pep695_generic_protocol_callable_members(self): @runtime_checkable class Foo[T](Protocol): @@ -4674,6 +4741,34 @@ class D(Generic[T]): pass with self.assertRaises(TypeError): D[()] + def test_generic_init_subclass_not_called_error(self): + notes = ["Note: this exception may have been caused by " + r"'GenericTests.test_generic_init_subclass_not_called_error.<locals>.Base.__init_subclass__' " + "(or the '__init_subclass__' method on a superclass) not calling 'super().__init_subclass__()'"] + + class Base: + def __init_subclass__(cls) -> None: + # Oops, I forgot super().__init_subclass__()! + pass + + with self.subTest(): + class Sub(Base, Generic[T]): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + + with self.subTest(): + class Sub[U](Base): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + def test_generic_subclass_checks(self): for typ in [list[int], List[int], tuple[int, str], Tuple[int, str], @@ -5797,6 +5892,23 @@ class A: with self.assertRaises(TypeError): a[int] + def test_return_non_tuple_while_unpacking(self): + # GH-138497: GenericAlias objects didn't ensure that __typing_subst__ actually + # returned a tuple + class EvilTypeVar: + __typing_is_unpacked_typevartuple__ = True + def __typing_prepare_subst__(*_): + return None # any value + def __typing_subst__(*_): + return 42 # not tuple + + evil = EvilTypeVar() + # Create a dummy TypeAlias that will be given the evil generic from + # above. + type type_alias[*_] = 0 + with self.assertRaisesRegex(TypeError, ".+__typing_subst__.+tuple.+int.*"): + type_alias[evil][0] + class ClassVarTests(BaseTestCase): @@ -6273,62 +6385,8 @@ class F: for clazz in [C, D, E, F]: self.assertEqual(get_type_hints(clazz), expected_result) - def test_meta_no_type_check(self): - depr_msg = ( - "'typing.no_type_check_decorator' is deprecated " - "and slated for removal in Python 3.15" - ) - with self.assertWarnsRegex(DeprecationWarning, depr_msg): - @no_type_check_decorator - def magic_decorator(func): - return func - - self.assertEqual(magic_decorator.__name__, 'magic_decorator') - - @magic_decorator - def foo(a: 'whatevers') -> {}: - pass - - @magic_decorator - class C: - def foo(a: 'whatevers') -> {}: - pass - - self.assertEqual(foo.__name__, 'foo') - th = get_type_hints(foo) - self.assertEqual(th, {}) - cth = get_type_hints(C.foo) - self.assertEqual(cth, {}) - ith = get_type_hints(C().foo) - self.assertEqual(ith, {}) - class InternalsTests(BaseTestCase): - def test_deprecation_for_no_type_params_passed_to__evaluate(self): - with self.assertWarnsRegex( - DeprecationWarning, - ( - "Failing to pass a value to the 'type_params' parameter " - "of 'typing._eval_type' is deprecated" - ) - ) as cm: - self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) - - self.assertEqual(cm.filename, __file__) - - f = ForwardRef("int") - - with self.assertWarnsRegex( - DeprecationWarning, - ( - "Failing to pass a value to the 'type_params' parameter " - "of 'typing.ForwardRef._evaluate' is deprecated" - ) - ) as cm: - self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) - - self.assertEqual(cm.filename, __file__) - def test_collect_parameters(self): typing = import_helper.import_fresh_module("typing") with self.assertWarnsRegex( @@ -6859,12 +6917,10 @@ def test_forward_ref_and_final(self): self.assertEqual(hints, {'value': Final}) def test_top_level_class_var(self): - # https://bugs.python.org/issue45166 - with self.assertRaisesRegex( - TypeError, - r'typing.ClassVar\[int\] is not valid as type argument', - ): - get_type_hints(ann_module6) + # This is not meaningful but we don't raise for it. + # https://github.com/python/cpython/issues/133959 + hints = get_type_hints(ann_module6) + self.assertEqual(hints, {'wrong': ClassVar[int]}) def test_get_type_hints_typeddict(self): self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) @@ -6967,6 +7023,11 @@ def foo(a: 'Callable[..., T]'): self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': Callable[..., T]}) + def test_special_forms_no_forward(self): + def f(x: ClassVar[int]): + pass + self.assertEqual(get_type_hints(f), {'x': ClassVar[int]}) + def test_special_forms_forward(self): class C: @@ -6982,8 +7043,9 @@ class CF: self.assertEqual(get_type_hints(C, globals())['b'], Final[int]) self.assertEqual(get_type_hints(C, globals())['x'], ClassVar) self.assertEqual(get_type_hints(C, globals())['y'], Final) - with self.assertRaises(TypeError): - get_type_hints(CF, globals()), + lfi = get_type_hints(CF, globals())['b'] + self.assertIs(get_origin(lfi), list) + self.assertEqual(get_args(lfi), (Final[int],)) def test_union_forward_recursion(self): ValueList = List['Value'] @@ -7111,6 +7173,37 @@ def add_right(self, node: 'Node[T]' = None): right_hints = get_type_hints(t.add_right, globals(), locals()) self.assertEqual(right_hints['node'], Node[T]) + def test_get_type_hints_preserve_generic_alias_subclasses(self): + # https://github.com/python/cpython/issues/130870 + # A real world example of this is `collections.abc.Callable`. When parameterized, + # the result is a subclass of `types.GenericAlias`. + class MyAlias(types.GenericAlias): + pass + + class MyClass: + def __class_getitem__(cls, args): + return MyAlias(cls, args) + + # Using a forward reference is important, otherwise it works as expected. + # `y` tests that the `GenericAlias` subclass is preserved when stripping `Annotated`. + def func(x: MyClass['int'], y: MyClass[Annotated[int, ...]]): ... + + assert isinstance(get_type_hints(func)['x'], MyAlias) + assert isinstance(get_type_hints(func)['y'], MyAlias) + + def test_stringified_typeddict(self): + ns = run_code( + """ + from __future__ import annotations + from typing import TypedDict + class TD[UniqueT](TypedDict): + a: UniqueT + """ + ) + TD = ns['TD'] + self.assertEqual(TD.__annotations__, {'a': EqualToForwardRef('UniqueT', owner=TD, module=TD.__module__)}) + self.assertEqual(get_type_hints(TD), {'a': TD.__type_params__[0]}) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -7216,33 +7309,119 @@ class C(Generic[T]): pass class EvaluateForwardRefTests(BaseTestCase): def test_evaluate_forward_ref(self): int_ref = ForwardRef('int') - missing = ForwardRef('missing') + self.assertIs(typing.evaluate_forward_ref(int_ref), int) self.assertIs( typing.evaluate_forward_ref(int_ref, type_params=()), int, ) self.assertIs( - typing.evaluate_forward_ref( - int_ref, type_params=(), format=annotationlib.Format.FORWARDREF, - ), + typing.evaluate_forward_ref(int_ref, format=annotationlib.Format.VALUE), int, ) self.assertIs( typing.evaluate_forward_ref( - missing, type_params=(), format=annotationlib.Format.FORWARDREF, + int_ref, format=annotationlib.Format.FORWARDREF, + ), + int, + ) + self.assertEqual( + typing.evaluate_forward_ref( + int_ref, format=annotationlib.Format.STRING, + ), + 'int', + ) + + def test_evaluate_forward_ref_undefined(self): + missing = ForwardRef('missing') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(missing) + self.assertIs( + typing.evaluate_forward_ref( + missing, format=annotationlib.Format.FORWARDREF, ), missing, ) self.assertEqual( typing.evaluate_forward_ref( - int_ref, type_params=(), format=annotationlib.Format.STRING, + missing, format=annotationlib.Format.STRING, ), - 'int', + "missing", ) - def test_evaluate_forward_ref_no_type_params(self): - ref = ForwardRef('int') - self.assertIs(typing.evaluate_forward_ref(ref), int) + def test_evaluate_forward_ref_nested(self): + ref = ForwardRef("int | list['str']") + self.assertEqual( + typing.evaluate_forward_ref(ref), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.STRING), + "int | list['str']", + ) + + why = ForwardRef('"\'str\'"') + self.assertIs(typing.evaluate_forward_ref(why), str) + + def test_evaluate_forward_ref_none(self): + none_ref = ForwardRef('None') + self.assertIs(typing.evaluate_forward_ref(none_ref), None) + + def test_globals(self): + A = "str" + ref = ForwardRef('list[A]') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + self.assertEqual( + typing.evaluate_forward_ref(ref, globals={'A': A}), + list[str], + ) + + def test_owner(self): + ref = ForwardRef("A") + + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + # We default to the globals of `owner`, + # so it no longer raises `NameError` + self.assertIs( + typing.evaluate_forward_ref(ref, owner=Loop), A + ) + + def test_inherited_owner(self): + # owner passed to evaluate_forward_ref + ref = ForwardRef("list['A']") + self.assertEqual( + typing.evaluate_forward_ref(ref, owner=Loop), + list[A], + ) + + # owner set on the ForwardRef + ref = ForwardRef("list['A']", owner=Loop) + self.assertEqual( + typing.evaluate_forward_ref(ref), + list[A], + ) + + def test_partial_evaluation(self): + ref = ForwardRef("list[A]") + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + list[EqualToForwardRef('A')], + ) + + def test_with_module(self): + from test.typinganndata import fwdref_module + + typing.evaluate_forward_ref( + fwdref_module.fw,) class CollectionsAbcTests(BaseTestCase): @@ -7339,6 +7518,25 @@ def test_mutablesequence(self): self.assertIsInstance([], typing.MutableSequence) self.assertNotIsInstance((), typing.MutableSequence) + def test_bytestring(self): + previous_typing_module = sys.modules.pop("typing", None) + self.addCleanup(sys.modules.__setitem__, "typing", previous_typing_module) + + with self.assertWarns(DeprecationWarning): + from typing import ByteString + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(b'', ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(bytearray(b''), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsSubclass(bytes, ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsSubclass(bytearray, ByteString) + with self.assertWarns(DeprecationWarning): + class Foo(ByteString): ... + with self.assertWarns(DeprecationWarning): + class Bar(ByteString, typing.Awaitable): ... + def test_list(self): self.assertIsSubclass(list, typing.List) @@ -8080,78 +8278,13 @@ class Group(NamedTuple): self.assertIs(type(a), Group) self.assertEqual(a, (1, [2])) - def test_namedtuple_keyword_usage(self): - with self.assertWarnsRegex( - DeprecationWarning, - "Creating NamedTuple classes using keyword arguments is deprecated" - ): - LocalEmployee = NamedTuple("LocalEmployee", name=str, age=int) - - nick = LocalEmployee('Nick', 25) - self.assertIsInstance(nick, tuple) - self.assertEqual(nick.name, 'Nick') - self.assertEqual(LocalEmployee.__name__, 'LocalEmployee') - self.assertEqual(LocalEmployee._fields, ('name', 'age')) - self.assertEqual(LocalEmployee.__annotations__, dict(name=str, age=int)) - - with self.assertRaisesRegex( - TypeError, - "Either list of fields or keywords can be provided to NamedTuple, not both" - ): - NamedTuple('Name', [('x', int)], y=str) - - with self.assertRaisesRegex( - TypeError, - "Either list of fields or keywords can be provided to NamedTuple, not both" - ): - NamedTuple('Name', [], y=str) - - with self.assertRaisesRegex( - TypeError, - ( - r"Cannot pass `None` as the 'fields' parameter " - r"and also specify fields using keyword arguments" - ) - ): - NamedTuple('Name', None, x=int) - - def test_namedtuple_special_keyword_names(self): - with self.assertWarnsRegex( - DeprecationWarning, - "Creating NamedTuple classes using keyword arguments is deprecated" - ): - NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list) - - self.assertEqual(NT.__name__, 'NT') - self.assertEqual(NT._fields, ('cls', 'self', 'typename', 'fields')) - a = NT(cls=str, self=42, typename='foo', fields=[('bar', tuple)]) - self.assertEqual(a.cls, str) - self.assertEqual(a.self, 42) - self.assertEqual(a.typename, 'foo') - self.assertEqual(a.fields, [('bar', tuple)]) - def test_empty_namedtuple(self): - expected_warning = re.escape( - "Failing to pass a value for the 'fields' parameter is deprecated " - "and will be disallowed in Python 3.15. " - "To create a NamedTuple class with 0 fields " - "using the functional syntax, " - "pass an empty list, e.g. `NT1 = NamedTuple('NT1', [])`." - ) - with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): - NT1 = NamedTuple('NT1') + with self.assertRaisesRegex(TypeError, "missing.*required.*argument"): + BAD = NamedTuple('BAD') - expected_warning = re.escape( - "Passing `None` as the 'fields' parameter is deprecated " - "and will be disallowed in Python 3.15. " - "To create a NamedTuple class with 0 fields " - "using the functional syntax, " - "pass an empty list, e.g. `NT2 = NamedTuple('NT2', [])`." - ) - with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): - NT2 = NamedTuple('NT2', None) - - NT3 = NamedTuple('NT2', []) + NT1 = NamedTuple('NT1', {}) + NT2 = NamedTuple('NT2', ()) + NT3 = NamedTuple('NT3', []) class CNT(NamedTuple): pass # empty body @@ -8166,16 +8299,18 @@ class CNT(NamedTuple): def test_namedtuple_errors(self): with self.assertRaises(TypeError): NamedTuple.__new__() + with self.assertRaisesRegex(TypeError, "object is not iterable"): + NamedTuple('Name', None) with self.assertRaisesRegex( TypeError, - "missing 1 required positional argument" + "missing 2 required positional arguments" ): NamedTuple() with self.assertRaisesRegex( TypeError, - "takes from 1 to 2 positional arguments but 3 were given" + "takes 2 positional arguments but 3 were given" ): NamedTuple('Emp', [('name', str)], None) @@ -8187,10 +8322,22 @@ def test_namedtuple_errors(self): with self.assertRaisesRegex( TypeError, - "missing 1 required positional argument: 'typename'" + "got some positional-only arguments passed as keyword arguments" ): NamedTuple(typename='Emp', name=str, id=int) + with self.assertRaisesRegex( + TypeError, + "got an unexpected keyword argument" + ): + NamedTuple('Name', [('x', int)], y=str) + + with self.assertRaisesRegex( + TypeError, + "got an unexpected keyword argument" + ): + NamedTuple('Name', [], y=str) + def test_copy_and_pickle(self): global Emp # pickle wants to reference the class by name Emp = NamedTuple('Emp', [('name', str), ('cool', int)]) @@ -8538,6 +8685,36 @@ class Child(Base1, Base2): self.assertEqual(Child.__required_keys__, frozenset(['a'])) self.assertEqual(Child.__optional_keys__, frozenset()) + def test_inheritance_pep563(self): + def _make_td(future, class_name, annos, base, extra_names=None): + lines = [] + if future: + lines.append('from __future__ import annotations') + lines.append('from typing import TypedDict') + lines.append(f'class {class_name}({base}):') + for name, anno in annos.items(): + lines.append(f' {name}: {anno}') + code = '\n'.join(lines) + ns = run_code(code, extra_names) + return ns[class_name] + + for base_future in (True, False): + for child_future in (True, False): + with self.subTest(base_future=base_future, child_future=child_future): + base = _make_td( + base_future, "Base", {"base": "int"}, "TypedDict" + ) + self.assertIsNotNone(base.__annotate__) + child = _make_td( + child_future, "Child", {"child": "int"}, "Base", {"Base": base} + ) + base_anno = ForwardRef("int", module="builtins", owner=base) if base_future else int + child_anno = ForwardRef("int", module="builtins", owner=child) if child_future else int + self.assertEqual(base.__annotations__, {'base': base_anno}) + self.assertEqual( + child.__annotations__, {'child': child_anno, 'base': base_anno} + ) + def test_required_notrequired_keys(self): self.assertEqual(NontotalMovie.__required_keys__, frozenset({"title"})) @@ -8661,6 +8838,32 @@ class ChildWithInlineAndOptional(Untotal, Inline): class Wrong(*bases): pass + def test_closed_values(self): + class Implicit(TypedDict): ... + class ExplicitTrue(TypedDict, closed=True): ... + class ExplicitFalse(TypedDict, closed=False): ... + + self.assertIsNone(Implicit.__closed__) + self.assertIs(ExplicitTrue.__closed__, True) + self.assertIs(ExplicitFalse.__closed__, False) + + def test_extra_items_class_arg(self): + class TD(TypedDict, extra_items=int): + a: str + + self.assertIs(TD.__extra_items__, int) + self.assertEqual(TD.__annotations__, {'a': str}) + self.assertEqual(TD.__required_keys__, frozenset({'a'})) + self.assertEqual(TD.__optional_keys__, frozenset()) + + class NoExtra(TypedDict): + a: str + + self.assertIs(NoExtra.__extra_items__, NoExtraItems) + self.assertEqual(NoExtra.__annotations__, {'a': str}) + self.assertEqual(NoExtra.__required_keys__, frozenset({'a'})) + self.assertEqual(NoExtra.__optional_keys__, frozenset()) + def test_is_typeddict(self): self.assertIs(is_typeddict(Point2D), True) self.assertIs(is_typeddict(Union[str, int]), False) @@ -8904,39 +9107,27 @@ class MultipleGenericBases(GenericParent[int], GenericParent[float]): self.assertEqual(CallTypedDict.__orig_bases__, (TypedDict,)) def test_zero_fields_typeddicts(self): - T1 = TypedDict("T1", {}) + T1a = TypedDict("T1a", {}) + T1b = TypedDict("T1b", []) + T1c = TypedDict("T1c", ()) class T2(TypedDict): pass class T3[tvar](TypedDict): pass S = TypeVar("S") class T4(TypedDict, Generic[S]): pass - expected_warning = re.escape( - "Failing to pass a value for the 'fields' parameter is deprecated " - "and will be disallowed in Python 3.15. " - "To create a TypedDict class with 0 fields " - "using the functional syntax, " - "pass an empty dictionary, e.g. `T5 = TypedDict('T5', {})`." - ) - with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): - T5 = TypedDict('T5') - - expected_warning = re.escape( - "Passing `None` as the 'fields' parameter is deprecated " - "and will be disallowed in Python 3.15. " - "To create a TypedDict class with 0 fields " - "using the functional syntax, " - "pass an empty dictionary, e.g. `T6 = TypedDict('T6', {})`." - ) - with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): - T6 = TypedDict('T6', None) - - for klass in T1, T2, T3, T4, T5, T6: + for klass in T1a, T1b, T1c, T2, T3, T4: with self.subTest(klass=klass.__name__): self.assertEqual(klass.__annotations__, {}) self.assertEqual(klass.__required_keys__, set()) self.assertEqual(klass.__optional_keys__, set()) self.assertIsInstance(klass(), dict) + def test_errors(self): + with self.assertRaisesRegex(TypeError, "missing 1 required.*argument"): + TypedDict('TD') + with self.assertRaisesRegex(TypeError, "object is not iterable"): + TypedDict('TD', None) + def test_readonly_inheritance(self): class Base1(TypedDict): a: ReadOnly[int] @@ -9000,6 +9191,71 @@ class AllTheThings(TypedDict): }, ) + def test_closed_inheritance(self): + class Base(TypedDict, extra_items=ReadOnly[Union[str, None]]): + a: int + + self.assertEqual(Base.__required_keys__, frozenset({"a"})) + self.assertEqual(Base.__optional_keys__, frozenset({})) + self.assertEqual(Base.__readonly_keys__, frozenset({})) + self.assertEqual(Base.__mutable_keys__, frozenset({"a"})) + self.assertEqual(Base.__annotations__, {"a": int}) + self.assertEqual(Base.__extra_items__, ReadOnly[Union[str, None]]) + self.assertIsNone(Base.__closed__) + + class Child(Base, extra_items=int): + a: str + + self.assertEqual(Child.__required_keys__, frozenset({'a'})) + self.assertEqual(Child.__optional_keys__, frozenset({})) + self.assertEqual(Child.__readonly_keys__, frozenset({})) + self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) + self.assertEqual(Child.__annotations__, {"a": str}) + self.assertIs(Child.__extra_items__, int) + self.assertIsNone(Child.__closed__) + + class GrandChild(Child, closed=True): + a: float + + self.assertEqual(GrandChild.__required_keys__, frozenset({'a'})) + self.assertEqual(GrandChild.__optional_keys__, frozenset({})) + self.assertEqual(GrandChild.__readonly_keys__, frozenset({})) + self.assertEqual(GrandChild.__mutable_keys__, frozenset({'a'})) + self.assertEqual(GrandChild.__annotations__, {"a": float}) + self.assertIs(GrandChild.__extra_items__, NoExtraItems) + self.assertIs(GrandChild.__closed__, True) + + class GrandGrandChild(GrandChild): + ... + self.assertEqual(GrandGrandChild.__required_keys__, frozenset({'a'})) + self.assertEqual(GrandGrandChild.__optional_keys__, frozenset({})) + self.assertEqual(GrandGrandChild.__readonly_keys__, frozenset({})) + self.assertEqual(GrandGrandChild.__mutable_keys__, frozenset({'a'})) + self.assertEqual(GrandGrandChild.__annotations__, {"a": float}) + self.assertIs(GrandGrandChild.__extra_items__, NoExtraItems) + self.assertIsNone(GrandGrandChild.__closed__) + + def test_implicit_extra_items(self): + class Base(TypedDict): + a: int + + self.assertIs(Base.__extra_items__, NoExtraItems) + self.assertIsNone(Base.__closed__) + + class ChildA(Base, closed=True): + ... + + self.assertEqual(ChildA.__extra_items__, NoExtraItems) + self.assertIs(ChildA.__closed__, True) + + def test_cannot_combine_closed_and_extra_items(self): + with self.assertRaisesRegex( + TypeError, + "Cannot combine closed=True and extra_items" + ): + class TD(TypedDict, closed=True, extra_items=range): + x: str + def test_annotations(self): # _type_check is applied with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): @@ -9229,6 +9485,12 @@ class A(typing.Match): class B(typing.Pattern): pass + def test_typed_dict_signature(self): + self.assertListEqual( + list(inspect.signature(TypedDict).parameters), + ['typename', 'fields', 'total', 'closed', 'extra_items'] + ) + class AnnotatedTests(BaseTestCase): @@ -9691,6 +9953,19 @@ class B(str): ... self.assertIs(type(field_c2.__metadata__[0]), float) self.assertIs(type(field_c3.__metadata__[0]), bool) + def test_forwardref_partial_evaluation(self): + # Test that Annotated partially evaluates if it contains a ForwardRef + # See: https://github.com/python/cpython/issues/137706 + def f(x: Annotated[undefined, '']): pass + + ann = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + + # Test that the attributes are retrievable from the partially evaluated annotation + x_ann = ann['x'] + self.assertIs(get_origin(x_ann), Annotated) + self.assertEqual(x_ann.__origin__, EqualToForwardRef('undefined', owner=f)) + self.assertEqual(x_ann.__metadata__, ('',)) + class TypeAliasTests(BaseTestCase): def test_canonical_usage_with_variable_annotation(self): @@ -10305,6 +10580,10 @@ def test_no_isinstance(self): class SpecialAttrsTests(BaseTestCase): def test_special_attrs(self): + with warnings.catch_warnings( + action='ignore', category=DeprecationWarning + ): + typing_ByteString = typing.ByteString cls_to_check = { # ABC classes typing.AbstractSet: 'AbstractSet', @@ -10313,6 +10592,7 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', + typing_ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', @@ -10665,7 +10945,8 @@ def test_all_exported_names(self): # there's a few types and metaclasses that aren't exported not k.endswith(('Meta', '_contra', '_co')) and not k.upper() == k and - # but export all things that have __module__ == 'typing' + k not in {"ByteString"} and + # but export all other things that have __module__ == 'typing' getattr(v, '__module__', None) == typing.__name__ ) } @@ -10731,6 +11012,9 @@ def test_eq(self): with self.assertWarns(DeprecationWarning): self.assertNotEqual(int, typing._UnionGenericAlias) + def test_hashable(self): + self.assertEqual(hash(typing._UnionGenericAlias), hash(Union)) + def load_tests(loader, tests, pattern): import doctest diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 8e3fef6b6fe..a3c22a4f27e 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -24,7 +24,7 @@ class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = '9e43ee3929471739680c0e705482b4ae1c4122e4' + expectedchecksum = '8b2615a9fc627676cbc0b6fac0191177df97ef5f' @requires_resource('cpu') def test_method_checksum(self): @@ -77,7 +77,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = '23ab09ed4abdf93db23b97359108ed630dd8311d' + expectedchecksum = '65670ae03a324c5f9e826a4de3e25bae4d73c9b7' @requires_resource('cpu') def test_function_checksum(self): @@ -276,6 +276,33 @@ def test_east_asian_width_9_0_changes(self): self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N') self.assertEqual(self.db.east_asian_width('\u231a'), 'W') + def test_isxidstart(self): + self.assertTrue(self.db.isxidstart('S')) + self.assertTrue(self.db.isxidstart('\u0AD0')) # GUJARATI OM + self.assertTrue(self.db.isxidstart('\u0EC6')) # LAO KO LA + self.assertTrue(self.db.isxidstart('\u17DC')) # KHMER SIGN AVAKRAHASANYA + self.assertTrue(self.db.isxidstart('\uA015')) # YI SYLLABLE WU + self.assertTrue(self.db.isxidstart('\uFE7B')) # ARABIC KASRA MEDIAL FORM + + self.assertFalse(self.db.isxidstart(' ')) + self.assertFalse(self.db.isxidstart('0')) + self.assertRaises(TypeError, self.db.isxidstart) + self.assertRaises(TypeError, self.db.isxidstart, 'xx') + + def test_isxidcontinue(self): + self.assertTrue(self.db.isxidcontinue('S')) + self.assertTrue(self.db.isxidcontinue('_')) + self.assertTrue(self.db.isxidcontinue('0')) + self.assertTrue(self.db.isxidcontinue('\u00BA')) # MASCULINE ORDINAL INDICATOR + self.assertTrue(self.db.isxidcontinue('\u0640')) # ARABIC TATWEEL + self.assertTrue(self.db.isxidcontinue('\u0710')) # SYRIAC LETTER ALAPH + self.assertTrue(self.db.isxidcontinue('\u0B3E')) # ORIYA VOWEL SIGN AA + self.assertTrue(self.db.isxidcontinue('\u17D7')) # KHMER SIGN LEK TOO + + self.assertFalse(self.db.isxidcontinue(' ')) + self.assertRaises(TypeError, self.db.isxidcontinue) + self.assertRaises(TypeError, self.db.isxidcontinue, 'xx') + class UnicodeMiscTest(UnicodeDatabaseTest): @cpython_only diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 993e6bf013c..91d45283eb3 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -12,7 +12,7 @@ class MyException(Exception): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCM: @@ -480,7 +480,7 @@ def test_setup_get_event_loop(self): class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): - asyncio._get_event_loop_policy().get_event_loop() + asyncio.events._get_event_loop_policy().get_event_loop() async def test_demo1(self): pass @@ -490,7 +490,7 @@ async def test_demo1(self): self.assertTrue(result.wasSuccessful()) def test_loop_factory(self): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCase1(unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index a04af55f3fc..cf10e956bf2 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1920,6 +1920,22 @@ def testAssertLogsUnexpectedException(self): with self.assertLogs(): raise ZeroDivisionError("Unexpected") + def testAssertLogsWithFormatter(self): + # Check alternative formats will be respected + format = "[No.1: the larch] %(levelname)s:%(name)s:%(message)s" + formatter = logging.Formatter(format) + with self.assertNoStderr(): + with self.assertLogs() as cm: + log_foo.info("1") + log_foobar.debug("2") + self.assertEqual(cm.output, ["INFO:foo:1"]) + self.assertLogRecords(cm.records, [{'name': 'foo'}]) + with self.assertLogs(formatter=formatter) as cm: + log_foo.info("1") + log_foobar.debug("2") + self.assertEqual(cm.output, ["[No.1: the larch] INFO:foo:1"]) + self.assertLogRecords(cm.records, [{'name': 'foo'}]) + def testAssertNoLogsDefault(self): with self.assertRaises(self.failureException) as cm: with self.assertNoLogs(): @@ -1989,7 +2005,7 @@ def testAssertNoLogsYieldsNone(self): pass self.assertIsNone(value) - def testAssertStartswith(self): + def testAssertStartsWith(self): self.assertStartsWith('ababahalamaha', 'ababa') self.assertStartsWith('ababahalamaha', ('x', 'ababa', 'y')) self.assertStartsWith(UserString('ababahalamaha'), 'ababa') @@ -2034,7 +2050,7 @@ def testAssertStartswith(self): self.assertStartsWith('ababahalamaha', 'amaha', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertNotStartswith(self): + def testAssertNotStartsWith(self): self.assertNotStartsWith('ababahalamaha', 'amaha') self.assertNotStartsWith('ababahalamaha', ('x', 'amaha', 'y')) self.assertNotStartsWith(UserString('ababahalamaha'), 'amaha') @@ -2079,7 +2095,7 @@ def testAssertNotStartswith(self): self.assertNotStartsWith('ababahalamaha', 'ababa', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertEndswith(self): + def testAssertEndsWith(self): self.assertEndsWith('ababahalamaha', 'amaha') self.assertEndsWith('ababahalamaha', ('x', 'amaha', 'y')) self.assertEndsWith(UserString('ababahalamaha'), 'amaha') @@ -2124,7 +2140,7 @@ def testAssertEndswith(self): self.assertEndsWith('ababahalamaha', 'ababa', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertNotEndswith(self): + def testAssertNotEndsWith(self): self.assertNotEndsWith('ababahalamaha', 'ababa') self.assertNotEndsWith('ababahalamaha', ('x', 'ababa', 'y')) self.assertNotEndsWith(UserString('ababahalamaha'), 'ababa') diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 6092ed292d8..8ed92373e5e 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -75,6 +75,14 @@ def testUnexpectedSuccess(self): class Empty(unittest.TestCase): pass + class SetUpClassFailure(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + raise Exception + def testPass(self): + pass + class TestLoader(unittest.TestLoader): """Test loader that returns a suite containing the supplied testcase.""" @@ -191,6 +199,18 @@ def test_ExitEmptySuite(self): out = stream.getvalue() self.assertIn('\nNO TESTS RAN\n', out) + def test_ExitSetUpClassFailureSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["setup_class_failure"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.SetUpClassFailure)) + self.assertEqual(cm.exception.code, 1) + out = stream.getvalue() + self.assertIn("ERROR: setUpClass", out) + self.assertIn("SetUpClassFailure", out) + class InitialisableProgram(unittest.TestProgram): exit = False diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 9ac4c52449c..3f44e617303 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -1282,14 +1282,22 @@ def setUpModule(): suite(result) expected_out = '\nStdout:\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 1) + self.assertEqual(len(result.errors), 2) description = 'tearDownModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + def testBufferSetUpModule_DoModuleCleanups(self): with captured_stdout() as stdout: result = unittest.TestResult() @@ -1313,22 +1321,34 @@ def setUpModule(): suite(result) expected_out = '\nStdout:\nset up module\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 2) + self.assertEqual(len(result.errors), 3) description = 'setUpModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ValueError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn('\nStdout:\nset up module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] self.assertIn(expected_out, formatted_exc) self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ZeroDivisionError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertIn(expected_out, formatted_exc) + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + def testBufferTearDownModule_DoModuleCleanups(self): with captured_stdout() as stdout: result = unittest.TestResult() @@ -1355,21 +1375,32 @@ def tearDownModule(): suite(result) expected_out = '\nStdout:\ntear down module\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 2) + self.assertEqual(len(result.errors), 3) description = 'tearDownModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ValueError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn('\nStdout:\ntear down module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ZeroDivisionError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 4d3cfd60b8d..a47e2ebb59d 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -13,6 +13,7 @@ LoggingResult, ResultWithNoStartTestRunStopTestRun, ) +from test.support.testcase import ExceptionIsLikeMixin def resultFactory(*_): @@ -604,7 +605,7 @@ class EmptyTest(unittest.TestCase): @support.force_not_colorized_test_class -class TestModuleCleanUp(unittest.TestCase): +class TestModuleCleanUp(ExceptionIsLikeMixin, unittest.TestCase): def test_add_and_do_ModuleCleanup(self): module_cleanups = [] @@ -646,11 +647,50 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(CustomError) as e: + with self.assertRaises(Exception) as e: unittest.case.doModuleCleanups() - self.assertEqual(str(e.exception), 'CleanUpExc') + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', + [CustomError('CleanUpExc')])) self.assertEqual(unittest.case._module_cleanups, []) + def test_doModuleCleanup_with_multiple_errors_in_addModuleCleanup(self): + def module_cleanup_bad1(): + raise TypeError('CleanUpExc1') + + def module_cleanup_bad2(): + raise ValueError('CleanUpExc2') + + class Module: + unittest.addModuleCleanup(module_cleanup_bad1) + unittest.addModuleCleanup(module_cleanup_bad2) + with self.assertRaises(ExceptionGroup) as e: + unittest.case.doModuleCleanups() + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ])) + + def test_doModuleCleanup_with_exception_group_in_addModuleCleanup(self): + def module_cleanup_bad(): + raise ExceptionGroup('CleanUpExc', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ]) + + class Module: + unittest.addModuleCleanup(module_cleanup_bad) + with self.assertRaises(ExceptionGroup) as e: + unittest.case.doModuleCleanups() + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', [ + ExceptionGroup('CleanUpExc', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ]), + ])) + def test_addModuleCleanup_arg_errors(self): cleanups = [] def cleanup(*args, **kwargs): @@ -871,9 +911,11 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(CustomError) as cm: + with self.assertRaises(Exception) as cm: suite.debug() - self.assertEqual(str(cm.exception), 'CleanUpExc') + self.assertExceptionIsLike(cm.exception, + ExceptionGroup('module cleanup failed', + [CustomError('CleanUpExc')])) self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) self.assertEqual(unittest.case._module_cleanups, []) diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 0791675b540..dc36ceeb650 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class AsyncClass: diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index d1e48bde982..0e82c723ec3 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -1050,6 +1050,7 @@ def __post_init__(self): create_autospec(WithPostInit()), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithPostInit) self.assertIsInstance(mock.a, int) self.assertIsInstance(mock.b, int) @@ -1072,6 +1073,7 @@ class WithDefault: create_autospec(WithDefault(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithDefault) self.assertIsInstance(mock.a, int) self.assertIsInstance(mock.b, int) @@ -1087,6 +1089,7 @@ def b(self) -> int: create_autospec(WithMethod(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithMethod) self.assertIsInstance(mock.a, int) mock.b.assert_not_called() @@ -1102,11 +1105,29 @@ class WithNonFields: create_autospec(WithNonFields(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithNonFields) with self.assertRaisesRegex(AttributeError, msg): mock.a with self.assertRaisesRegex(AttributeError, msg): mock.b + def test_dataclass_special_attrs(self): + @dataclass + class Description: + name: str + + for mock in [ + create_autospec(Description, instance=True), + create_autospec(Description(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock, Description) + self.assertIs(mock.__class__, Description) + self.assertIsInstance(mock.__dataclass_fields__, MagicMock) + self.assertIsInstance(mock.__dataclass_params__, MagicMock) + self.assertIsInstance(mock.__match_args__, MagicMock) + self.assertIsInstance(mock.__hash__, MagicMock) + class TestCallList(unittest.TestCase): def test_args_list_contains_call_list(self): diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index d3af7a8489e..35e4652a87b 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -206,10 +206,97 @@ def test_tstrings(self): self.check_ast_roundtrip("t'foo'") self.check_ast_roundtrip("t'foo {bar}'") self.check_ast_roundtrip("t'foo {bar!s:.2f}'") - self.check_ast_roundtrip("t'foo {bar}' f'{bar}'") - self.check_ast_roundtrip("f'{bar}' t'foo {bar}'") - self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'") - self.check_ast_roundtrip("t'foo {bar}' u'bar'") + self.check_ast_roundtrip("t'{a + b}'") + self.check_ast_roundtrip("t'{a + b:x}'") + self.check_ast_roundtrip("t'{a + b!s}'") + self.check_ast_roundtrip("t'{ {a}}'") + self.check_ast_roundtrip("t'{ {a}=}'") + self.check_ast_roundtrip("t'{{a}}'") + self.check_ast_roundtrip("t''") + self.check_ast_roundtrip('t""') + self.check_ast_roundtrip("t'{(lambda x: x)}'") + self.check_ast_roundtrip("t'{t'{x}'}'") + + def test_tstring_with_nonsensical_str_field(self): + # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.Name(id="test1", ctx=ast.Load()), str="test2", conversion=-1 + ) + ] + ) + ), + "t'{test2}'", + ) + + def test_tstring_with_none_str_field(self): + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="test1"), str=None, conversion=-1)] + ) + ), + "t'{test1}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ + ast.Interpolation( + value=ast.Lambda( + args=ast.arguments(args=[ast.arg(arg="x")]), + body=ast.Name(id="x"), + ), + str=None, + conversion=-1, + ) + ] + ) + ), + "t'{(lambda x: x)}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + # `str` field kept here + [ast.Interpolation(value=ast.Name(id="x"), str="y", conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{y}'}"''', + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="x"), str=None, conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{x}'}"''', + ) + self.assertEqual( + ast.unparse(ast.TemplateStr( + [ast.Interpolation(value=ast.Constant(value="foo"), str=None, conversion=114)] + )), + '''t"{'foo'!r}"''', + ) def test_strings(self): self.check_ast_roundtrip("u'foo'") diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index c965860fbb1..ae524c5ffba 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -109,7 +109,7 @@ def setUp(self): finally: f.close() self.pathname = os_helper.TESTFN - self.quoted_pathname = urllib.parse.quote(self.pathname) + self.quoted_pathname = urllib.parse.quote(os.fsencode(self.pathname)) self.returned_obj = urllib.request.urlopen("file:%s" % self.quoted_pathname) def tearDown(self): @@ -1526,6 +1526,14 @@ def test_url2pathname(self): self.assertEqual(fn('////foo/bar'), f'{sep}{sep}foo{sep}bar') self.assertEqual(fn('data:blah'), 'data:blah') self.assertEqual(fn('data://blah'), f'data:{sep}{sep}blah') + self.assertEqual(fn('foo?bar'), 'foo') + self.assertEqual(fn('foo#bar'), 'foo') + self.assertEqual(fn('foo?bar=baz'), 'foo') + self.assertEqual(fn('foo?bar#baz'), 'foo') + self.assertEqual(fn('foo%3Fbar'), 'foo?bar') + self.assertEqual(fn('foo%23bar'), 'foo#bar') + self.assertEqual(fn('foo%3Fbar%3Dbaz'), 'foo?bar=baz') + self.assertEqual(fn('foo%3Fbar%23baz'), 'foo?bar#baz') def test_url2pathname_require_scheme(self): sep = os.path.sep @@ -1569,6 +1577,7 @@ def test_url2pathname_require_scheme_errors(self): urllib.request.url2pathname, url, require_scheme=True) + @unittest.skipIf(support.is_emscripten, "Fixed by https://github.com/emscripten-core/emscripten/pull/24593") def test_url2pathname_resolve_host(self): fn = urllib.request.url2pathname sep = os.path.sep @@ -1581,6 +1590,10 @@ def test_url2pathname_resolve_host(self): def test_url2pathname_win(self): fn = urllib.request.url2pathname self.assertEqual(fn('/C:/'), 'C:\\') + self.assertEqual(fn('//C:'), 'C:') + self.assertEqual(fn('//C:/'), 'C:\\') + self.assertEqual(fn('//C:\\'), 'C:\\') + self.assertEqual(fn('//C:80/'), 'C:80\\') self.assertEqual(fn("///C|"), 'C:') self.assertEqual(fn("///C:"), 'C:') self.assertEqual(fn('///C:/'), 'C:\\') diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index e6a18476908..17db686942f 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -1,9 +1,13 @@ +import contextlib import errno +import sysconfig import unittest +from unittest import mock from test import support from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied +from test.support.warnings_helper import check_no_resource_warning import os import socket @@ -143,6 +147,43 @@ def test_ftp(self): ] self._test_urls(urls, self._extra_handlers()) + @support.requires_resource('walltime') + @unittest.skipIf(sysconfig.get_platform() == 'linux-ppc64le', + 'leaks on PPC64LE (gh-140691)') + def test_ftp_no_leak(self): + # gh-140691: When the data connection (but not control connection) + # cannot be made established, we shouldn't leave an open socket object. + + class MockError(OSError): + pass + + orig_create_connection = socket.create_connection + def patched_create_connection(address, *args, **kwargs): + """Simulate REJECTing connections to ports other than 21""" + host, port = address + if port != 21: + raise MockError() + return orig_create_connection(address, *args, **kwargs) + + url = 'ftp://www.pythontest.net/README' + entry = url, None, urllib.error.URLError + no_cache_handlers = [urllib.request.FTPHandler()] + cache_handlers = self._extra_handlers() + with mock.patch('socket.create_connection', patched_create_connection): + with check_no_resource_warning(self): + # Try without CacheFTPHandler + self._test_urls([entry], handlers=no_cache_handlers, + retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (uncached) + self._test_urls([entry], cache_handlers, retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (cached) + self._test_urls([entry], cache_handlers, retry=False) + # Try without the mock: the handler should not use a closed connection + with check_no_resource_warning(self): + self._test_urls([url], cache_handlers, retry=False) + def test_file(self): TESTFN = os_helper.TESTFN f = open(TESTFN, 'w') @@ -218,27 +259,6 @@ def test_custom_headers(self): opener.open(request) self.assertEqual(request.get_header('User-agent'),'Test-Agent') - @unittest.skip('XXX: http://www.imdb.com is gone') - def test_sites_no_connection_close(self): - # Some sites do not send Connection: close header. - # Verify that those work properly. (#issue12576) - - URL = 'http://www.imdb.com' # mangles Connection:close - - with socket_helper.transient_internet(URL): - try: - with urllib.request.urlopen(URL) as res: - pass - except ValueError: - self.fail("urlopen failed for site not sending \ - Connection:close") - else: - self.assertTrue(res) - - req = urllib.request.urlopen(URL) - res = req.read() - self.assertTrue(res) - def _test_urls(self, urls, handlers, retry=True): import time import logging @@ -255,18 +275,16 @@ def _test_urls(self, urls, handlers, retry=True): else: req = expected_err = None + if expected_err: + context = self.assertRaises(expected_err) + else: + context = contextlib.nullcontext() + with socket_helper.transient_internet(url): - try: + f = None + with context: f = urlopen(url, req, support.INTERNET_TIMEOUT) - # urllib.error.URLError is a subclass of OSError - except OSError as err: - if expected_err: - msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % - (expected_err, url, req, type(err), err)) - self.assertIsInstance(err, expected_err, msg) - else: - raise - else: + if f is not None: try: with time_out, \ socket_peer_reset, \ diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index aabc360289a..b2bde5a9b1d 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -2,6 +2,7 @@ import unicodedata import unittest import urllib.parse +from test import support RFC1808_BASE = "http://a/b/c/d;p?q#f" RFC2396_BASE = "http://a/b/c/d;p?q" @@ -156,27 +157,25 @@ def checkRoundtrips(self, url, parsed, split, url2=None): self.assertEqual(result3.hostname, result.hostname) self.assertEqual(result3.port, result.port) - def test_qsl(self): - for orig, expect in parse_qsl_test_cases: - result = urllib.parse.parse_qsl(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = [v for v in expect if len(v[1])] - result = urllib.parse.parse_qsl(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) + @support.subTests('orig,expect', parse_qsl_test_cases) + def test_qsl(self, orig, expect): + result = urllib.parse.parse_qsl(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = [v for v in expect if len(v[1])] + result = urllib.parse.parse_qsl(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) - def test_qs(self): - for orig, expect in parse_qs_test_cases: - result = urllib.parse.parse_qs(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = {v: expect[v] - for v in expect if len(expect[v][0])} - result = urllib.parse.parse_qs(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) + @support.subTests('orig,expect', parse_qs_test_cases) + def test_qs(self, orig, expect): + result = urllib.parse.parse_qs(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = {v: expect[v] + for v in expect if len(expect[v][0])} + result = urllib.parse.parse_qs(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) - def test_roundtrips(self): - str_cases = [ + @support.subTests('bytes', (False, True)) + @support.subTests('url,parsed,split', [ ('path/to/file', ('', '', 'path/to/file', '', '', ''), ('', '', 'path/to/file', '', '')), @@ -263,23 +262,21 @@ def test_roundtrips(self): ('sch_me:path/to/file', ('', '', 'sch_me:path/to/file', '', '', ''), ('', '', 'sch_me:path/to/file', '', '')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - str_cases += [ ('schème:path/to/file', ('', '', 'schème:path/to/file', '', '', ''), ('', '', 'schème:path/to/file', '', '')), - ] - for url, parsed, split in str_cases + bytes_cases: - with self.subTest(url): - self.checkRoundtrips(url, parsed, split) + ]) + def test_roundtrips(self, bytes, url, parsed, split): + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split) - def test_roundtrips_normalization(self): - str_cases = [ + @support.subTests('bytes', (False, True)) + @support.subTests('url,url2,parsed,split', [ ('///path/to/file', '/path/to/file', ('', '', '/path/to/file', '', '', ''), @@ -300,22 +297,18 @@ def test_roundtrips_normalization(self): 'https:///tmp/junk.txt', ('https', '', '/tmp/junk.txt', '', '', ''), ('https', '', '/tmp/junk.txt', '', '')), - ] - def _encode(t): - return (t[0].encode('ascii'), - t[1].encode('ascii'), - tuple(x.encode('ascii') for x in t[2]), - tuple(x.encode('ascii') for x in t[3])) - bytes_cases = [_encode(x) for x in str_cases] - for url, url2, parsed, split in str_cases + bytes_cases: - with self.subTest(url): - self.checkRoundtrips(url, parsed, split, url2) + ]) + def test_roundtrips_normalization(self, bytes, url, url2, parsed, split): + if bytes: + url = str_encode(url) + url2 = str_encode(url2) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split, url2) - def test_http_roundtrips(self): - # urllib.parse.urlsplit treats 'http:' as an optimized special case, - # so we test both 'http:' and 'https:' in all the following. - # Three cheers for white box knowledge! - str_cases = [ + @support.subTests('bytes', (False, True)) + @support.subTests('scheme', ('http', 'https')) + @support.subTests('url,parsed,split', [ ('://www.python.org', ('www.python.org', '', '', '', ''), ('www.python.org', '', '', '')), @@ -331,23 +324,20 @@ def test_http_roundtrips(self): ('://a/b/c/d;p?q#f', ('a', '/b/c/d', 'p', 'q', 'f'), ('a', '/b/c/d;p', 'q', 'f')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - str_schemes = ('http', 'https') - bytes_schemes = (b'http', b'https') - str_tests = str_schemes, str_cases - bytes_tests = bytes_schemes, bytes_cases - for schemes, test_cases in (str_tests, bytes_tests): - for scheme in schemes: - for url, parsed, split in test_cases: - url = scheme + url - parsed = (scheme,) + parsed - split = (scheme,) + split - self.checkRoundtrips(url, parsed, split) + ]) + def test_http_roundtrips(self, bytes, scheme, url, parsed, split): + # urllib.parse.urlsplit treats 'http:' as an optimized special case, + # so we test both 'http:' and 'https:' in all the following. + # Three cheers for white box knowledge! + if bytes: + scheme = str_encode(scheme) + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + url = scheme + url + parsed = (scheme,) + parsed + split = (scheme,) + split + self.checkRoundtrips(url, parsed, split) def checkJoin(self, base, relurl, expected, *, relroundtrip=True): with self.subTest(base=base, relurl=relurl): @@ -363,12 +353,13 @@ def checkJoin(self, base, relurl, expected, *, relroundtrip=True): relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) - def test_unparse_parse(self): - str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] - bytes_cases = [x.encode('ascii') for x in str_cases] - for u in str_cases + bytes_cases: - self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) - self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) + @support.subTests('bytes', (False, True)) + @support.subTests('u', ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]) + def test_unparse_parse(self, bytes, u): + if bytes: + u = str_encode(u) + self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) + self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) def test_RFC1808(self): # "normal" cases from RFC 1808: @@ -695,8 +686,8 @@ def test_urljoins_relative_base(self): self.checkJoin('///b/c', '///w', '///w') self.checkJoin('///b/c', 'w', '///b/w') - def test_RFC2732(self): - str_cases = [ + @support.subTests('bytes', (False, True)) + @support.subTests('url,hostname,port', [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432), ('http://[::1]:5432/foo/', '::1', 5432), @@ -727,26 +718,28 @@ def test_RFC2732(self): ('http://[::12.34.56.78]:/foo/', '::12.34.56.78', None), ('http://[::ffff:12.34.56.78]:/foo/', '::ffff:12.34.56.78', None), - ] - def _encode(t): - return t[0].encode('ascii'), t[1].encode('ascii'), t[2] - bytes_cases = [_encode(x) for x in str_cases] - for url, hostname, port in str_cases + bytes_cases: - urlparsed = urllib.parse.urlparse(url) - self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port)) + ]) + def test_RFC2732(self, bytes, url, hostname, port): + if bytes: + url = str_encode(url) + hostname = str_encode(hostname) + urlparsed = urllib.parse.urlparse(url) + self.assertEqual((urlparsed.hostname, urlparsed.port), (hostname, port)) - str_cases = [ + @support.subTests('bytes', (False, True)) + @support.subTests('invalid_url', [ 'http://::12.34.56.78]/', 'http://[::1/foo/', 'ftp://[::1/foo/bad]/bad', 'http://[::1/foo/bad]/bad', - 'http://[::ffff:12.34.56.78'] - bytes_cases = [x.encode('ascii') for x in str_cases] - for invalid_url in str_cases + bytes_cases: - self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) + 'http://[::ffff:12.34.56.78']) + def test_RFC2732_invalid(self, bytes, invalid_url): + if bytes: + invalid_url = str_encode(invalid_url) + self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) - def test_urldefrag(self): - str_cases = [ + @support.subTests('bytes', (False, True)) + @support.subTests('url,defrag,frag', [ ('http://python.org#frag', 'http://python.org', 'frag'), ('http://python.org', 'http://python.org', ''), ('http://python.org/#frag', 'http://python.org/', 'frag'), @@ -770,18 +763,18 @@ def test_urldefrag(self): ('http:?q#f', 'http:?q', 'f'), ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), - ] - def _encode(t): - return type(t)(x.encode('ascii') for x in t) - bytes_cases = [_encode(x) for x in str_cases] - for url, defrag, frag in str_cases + bytes_cases: - with self.subTest(url): - result = urllib.parse.urldefrag(url) - hash = '#' if isinstance(url, str) else b'#' - self.assertEqual(result.geturl(), url.rstrip(hash)) - self.assertEqual(result, (defrag, frag)) - self.assertEqual(result.url, defrag) - self.assertEqual(result.fragment, frag) + ]) + def test_urldefrag(self, bytes, url, defrag, frag): + if bytes: + url = str_encode(url) + defrag = str_encode(defrag) + frag = str_encode(frag) + result = urllib.parse.urldefrag(url) + hash = '#' if isinstance(url, str) else b'#' + self.assertEqual(result.geturl(), url.rstrip(hash)) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') @@ -981,42 +974,35 @@ def test_urlsplit_strip_url(self): self.assertEqual(p.scheme, "https") self.assertEqual(p.geturl(), "https://www.python.org/") - def test_attributes_bad_port(self): + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('port', ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६")) + def test_attributes_bad_port(self, bytes, parse, port): """Check handling of invalid ports.""" - for bytes in (False, True): - for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for port in ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६"): - with self.subTest(bytes=bytes, parse=parse, port=port): - netloc = "www.example.net:" + port - url = "http://" + netloc + "/" - if bytes: - if netloc.isascii() and port.isascii(): - netloc = netloc.encode("ascii") - url = url.encode("ascii") - else: - continue - p = parse(url) - self.assertEqual(p.netloc, netloc) - with self.assertRaises(ValueError): - p.port + netloc = "www.example.net:" + port + url = "http://" + netloc + "/" + if bytes: + if not (netloc.isascii() and port.isascii()): + self.skipTest('non-ASCII bytes') + netloc = str_encode(netloc) + url = str_encode(url) + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port - def test_attributes_bad_scheme(self): + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('scheme', (".", "+", "-", "0", "http&", "६http")) + def test_attributes_bad_scheme(self, bytes, parse, scheme): """Check handling of invalid schemes.""" - for bytes in (False, True): - for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for scheme in (".", "+", "-", "0", "http&", "६http"): - with self.subTest(bytes=bytes, parse=parse, scheme=scheme): - url = scheme + "://www.example.net" - if bytes: - if url.isascii(): - url = url.encode("ascii") - else: - continue - p = parse(url) - if bytes: - self.assertEqual(p.scheme, b"") - else: - self.assertEqual(p.scheme, "") + url = scheme + "://www.example.net" + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.scheme, b"" if bytes else "") def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it @@ -1128,24 +1114,21 @@ def test_anyscheme(self): self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"), (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'')) - def test_default_scheme(self): + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_default_scheme(self, func): # Exercise the scheme parameter of urlparse() and urlsplit() - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - with self.subTest(function=func): - result = func("http://example.net/", "ftp") - self.assertEqual(result.scheme, "http") - result = func(b"http://example.net/", b"ftp") - self.assertEqual(result.scheme, b"http") - self.assertEqual(func("path", "ftp").scheme, "ftp") - self.assertEqual(func("path", scheme="ftp").scheme, "ftp") - self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") - self.assertEqual(func("path").scheme, "") - self.assertEqual(func(b"path").scheme, b"") - self.assertEqual(func(b"path", "").scheme, b"") + result = func("http://example.net/", "ftp") + self.assertEqual(result.scheme, "http") + result = func(b"http://example.net/", b"ftp") + self.assertEqual(result.scheme, b"http") + self.assertEqual(func("path", "ftp").scheme, "ftp") + self.assertEqual(func("path", scheme="ftp").scheme, "ftp") + self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") + self.assertEqual(func("path").scheme, "") + self.assertEqual(func(b"path").scheme, b"") + self.assertEqual(func(b"path", "").scheme, b"") - def test_parse_fragments(self): - # Exercise the allow_fragments parameter of urlparse() and urlsplit() - tests = ( + @support.subTests('url,attr,expected_frag', ( ("http:#frag", "path", "frag"), ("//example.net#frag", "path", "frag"), ("index.html#frag", "path", "frag"), @@ -1156,24 +1139,24 @@ def test_parse_fragments(self): ("//abc#@frag", "path", "@frag"), ("//abc:80#@frag", "path", "@frag"), ("//abc#@frag:80", "path", "@frag:80"), - ) - for url, attr, expected_frag in tests: - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - if attr == "params" and func is urllib.parse.urlsplit: - attr = "path" - with self.subTest(url=url, function=func): - result = func(url, allow_fragments=False) - self.assertEqual(result.fragment, "") - self.assertEndsWith(getattr(result, attr), - "#" + expected_frag) - self.assertEqual(func(url, "", False).fragment, "") + )) + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_parse_fragments(self, url, attr, expected_frag, func): + # Exercise the allow_fragments parameter of urlparse() and urlsplit() + if attr == "params" and func is urllib.parse.urlsplit: + attr = "path" + result = func(url, allow_fragments=False) + self.assertEqual(result.fragment, "") + self.assertEndsWith(getattr(result, attr), + "#" + expected_frag) + self.assertEqual(func(url, "", False).fragment, "") - result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, expected_frag) - self.assertNotEndsWith(getattr(result, attr), expected_frag) - self.assertEqual(func(url, "", True).fragment, - expected_frag) - self.assertEqual(func(url).fragment, expected_frag) + result = func(url, allow_fragments=True) + self.assertEqual(result.fragment, expected_frag) + self.assertNotEndsWith(getattr(result, attr), expected_frag) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -1199,7 +1182,14 @@ def test_mixed_types_rejected(self): with self.assertRaisesRegex(TypeError, "Cannot mix str"): urllib.parse.urljoin(b"http://python.org", "http://python.org") - def _check_result_type(self, str_type): + @support.subTests('result_type', [ + urllib.parse.DefragResult, + urllib.parse.SplitResult, + urllib.parse.ParseResult, + ]) + def test_result_pairs(self, result_type): + # Check encoding and decoding between result pairs + str_type = result_type num_args = len(str_type._fields) bytes_type = str_type._encoded_counterpart self.assertIs(bytes_type._decoded_counterpart, str_type) @@ -1224,16 +1214,6 @@ def _check_result_type(self, str_type): self.assertEqual(str_result.encode(encoding, errors), bytes_args) self.assertEqual(str_result.encode(encoding, errors), bytes_result) - def test_result_pairs(self): - # Check encoding and decoding between result pairs - result_types = [ - urllib.parse.DefragResult, - urllib.parse.SplitResult, - urllib.parse.ParseResult, - ] - for result_type in result_types: - self._check_result_type(result_type) - def test_parse_qs_encoding(self): result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1") self.assertEqual(result, {'key': ['\u0141\xE9']}) @@ -1265,8 +1245,7 @@ def test_parse_qsl_max_num_fields(self): urllib.parse.parse_qsl('&'.join(['a=a']*11), max_num_fields=10) urllib.parse.parse_qsl('&'.join(['a=a']*10), max_num_fields=10) - def test_parse_qs_separator(self): - parse_qs_semicolon_cases = [ + @support.subTests('orig,expect', [ (";", {}), (";;", {}), (";a=b", {'a': ['b']}), @@ -1277,17 +1256,14 @@ def test_parse_qs_separator(self): (b";a=b", {b'a': [b'b']}), (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1;a=2", {b'a': [b'1', b'2']}), - ] - for orig, expect in parse_qs_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qs(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qs(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) + ]) + def test_parse_qs_separator(self, orig, expect): + result = urllib.parse.parse_qs(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qs(orig, separator=b';') + self.assertEqual(result_bytes, expect) - - def test_parse_qsl_separator(self): - parse_qsl_semicolon_cases = [ + @support.subTests('orig,expect', [ (";", []), (";;", []), (";a=b", [('a', 'b')]), @@ -1298,13 +1274,12 @@ def test_parse_qsl_separator(self): (b";a=b", [(b'a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), - ] - for orig, expect in parse_qsl_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qsl(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qsl(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) + ]) + def test_parse_qsl_separator(self, orig, expect): + result = urllib.parse.parse_qsl(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qsl(orig, separator=b';') + self.assertEqual(result_bytes, expect) def test_parse_qsl_bytes(self): self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) @@ -1695,11 +1670,12 @@ def test_to_bytes(self): self.assertRaises(UnicodeError, urllib.parse._to_bytes, 'http://www.python.org/medi\u00e6val') - def test_unwrap(self): - for wrapped_url in ('<URL:scheme://host/path>', '<scheme://host/path>', - 'URL:scheme://host/path', 'scheme://host/path'): - url = urllib.parse.unwrap(wrapped_url) - self.assertEqual(url, 'scheme://host/path') + @support.subTests('wrapped_url', + ('<URL:scheme://host/path>', '<scheme://host/path>', + 'URL:scheme://host/path', 'scheme://host/path')) + def test_unwrap(self, wrapped_url): + url = urllib.parse.unwrap(wrapped_url) + self.assertEqual(url, 'scheme://host/path') class DeprecationTest(unittest.TestCase): @@ -1780,5 +1756,11 @@ def test_to_bytes_deprecation(self): 'urllib.parse.to_bytes() is deprecated as of 3.8') +def str_encode(s): + return s.encode('ascii') + +def tuple_encode(t): + return tuple(str_encode(x) for x in t) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index ace84ef564d..75de9ea252d 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -166,7 +166,7 @@ def test_update(self): def test_missing(self): # Make sure UserDict doesn't have a __missing__ method - self.assertEqual(hasattr(collections.UserDict, "__missing__"), False) + self.assertNotHasAttr(collections.UserDict, "__missing__") # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index f66881044e1..b8e49440c9f 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -89,8 +89,8 @@ def test_env_var(self): # the UTF-8 mode if not self.posix_locale(): # PYTHONUTF8 should be ignored if -E is used - out = self.get_output('-E', '-c', code, PYTHONUTF8='1') - self.assertEqual(out, '0') + out = self.get_output('-E', '-c', code, PYTHONUTF8='0') + self.assertEqual(out, '1') # invalid mode out = self.get_output('-c', code, PYTHONUTF8='xxx', failure=True) @@ -116,7 +116,7 @@ def test_filesystemencoding(self): # PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode # and has the priority over -X utf8 and PYTHONUTF8 out = self.get_output('-X', 'utf8', '-c', code, - PYTHONUTF8='strict', + PYTHONUTF8='xxx', PYTHONLEGACYWINDOWSFSENCODING='1') self.assertEqual(out, 'mbcs/replace') diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 958be5408ce..5f9ab048cde 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -13,7 +13,8 @@ from unittest import mock from test import support -from test.support import import_helper +from test.support import force_not_colorized_test_class, import_helper, warnings_helper +from test.support.script_helper import assert_python_ok py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) @@ -589,6 +590,7 @@ def test_uuid1_safe(self): # dependent on the underlying platform support. At least it cannot be # unknown (unless I suppose the platform is buggy). self.assertNotEqual(u.is_safe, self.uuid.SafeUUID.unknown) + self.assertEqual(u.version, 1) @contextlib.contextmanager def mock_generate_time_safe(self, safe_value): @@ -611,24 +613,28 @@ def test_uuid1_unknown(self): with self.mock_generate_time_safe(None): u = self.uuid.uuid1() self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) + self.assertEqual(u.version, 1) @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') def test_uuid1_is_safe(self): with self.mock_generate_time_safe(0): u = self.uuid.uuid1() self.assertEqual(u.is_safe, self.uuid.SafeUUID.safe) + self.assertEqual(u.version, 1) @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') def test_uuid1_is_unsafe(self): with self.mock_generate_time_safe(-1): u = self.uuid.uuid1() self.assertEqual(u.is_safe, self.uuid.SafeUUID.unsafe) + self.assertEqual(u.version, 1) @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') def test_uuid1_bogus_return_value(self): with self.mock_generate_time_safe(3): u = self.uuid.uuid1() self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) + self.assertEqual(u.version, 1) def test_uuid1_time(self): with mock.patch.object(self.uuid, '_generate_time_safe', None), \ @@ -1111,6 +1117,7 @@ def test_uuid8_uniqueness(self): versions = {u.version for u in uuids} self.assertSetEqual(versions, {8}) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates @@ -1139,6 +1146,23 @@ def test_uuid_weakref(self): weak = weakref.ref(strong) self.assertIs(strong, weak()) + +class CommandLineTestCases: + uuid = None # to be defined in subclasses + + def do_test_standalone_uuid(self, version): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + output = stdout.getvalue().strip() + u = self.uuid.UUID(output) + self.assertEqual(output, str(u)) + self.assertEqual(u.version, version) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid1"]) + def test_cli_uuid1(self): + self.do_test_standalone_uuid(1) + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns"]) @mock.patch('sys.stderr', new_callable=io.StringIO) def test_cli_namespace_required_for_uuid3(self, mock_err): @@ -1213,14 +1237,55 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 5) + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) + def test_cli_uuid6(self): + self.do_test_standalone_uuid(6) -class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): + @mock.patch.object(sys, "argv", ["", "-u", "uuid7"]) + def test_cli_uuid7(self): + self.do_test_standalone_uuid(7) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid8"]) + def test_cli_uuid8(self): + self.do_test_standalone_uuid(8) + + +@force_not_colorized_test_class +class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid + +@force_not_colorized_test_class @unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid + def check_has_stable_libuuid_extractable_node(self): + if not self.uuid._has_stable_extractable_node: + self.skipTest("libuuid cannot deduce MAC address") + + @unittest.skipUnless(os.name == 'posix', 'POSIX only') + def test_unix_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._unix_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + + @unittest.skipUnless(os.name == 'nt', 'Windows only') + def test_windows_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._windll_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + class BaseTestInternals: _uuid = py_uuid diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index adc86a49b06..68bcf535ead 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -12,7 +12,6 @@ import pathlib import re import shutil -import struct import subprocess import sys import sysconfig @@ -21,7 +20,7 @@ from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_android, is_apple_mobile, - is_emscripten, is_wasi, + is_wasm32, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, @@ -42,7 +41,7 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_android or is_apple_mobile or is_emscripten or is_wasi: +if is_android or is_apple_mobile or is_wasm32: raise unittest.SkipTest("venv is not available on this platform") @requires_subprocess() @@ -138,14 +137,9 @@ def _check_output_of_default_create(self): self.isdir(self.bindir) self.isdir(self.include) self.isdir(*self.lib) - # Issue 21197 p = self.get_env_file('lib64') - conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and - (sys.platform != 'darwin')) - if conditions: - self.assertTrue(os.path.islink(p)) - else: - self.assertFalse(os.path.exists(p)) + if os.path.exists(p): + self.assertFalse(os.path.islink(p)) data = self.get_text_file_contents('pyvenv.cfg') executable = sys._base_executable path = os.path.dirname(executable) @@ -522,6 +516,8 @@ def test_special_chars_bash(self): # gh-124651: test quoted strings @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + @unittest.skipIf(sys.platform.startswith('netbsd'), + "NetBSD csh fails with quoted special chars; see gh-139308") def test_special_chars_csh(self): """ Test that the template strings are quoted properly (csh) @@ -774,7 +770,7 @@ def test_activate_shell_script_has_no_dos_newlines(self): with open(script_path, 'rb') as script: for i, line in enumerate(script, 1): error_message = f"CR LF found in line {i}" - self.assertFalse(line.endswith(b'\r\n'), error_message) + self.assertNotEndsWith(line, b'\r\n', error_message) @requireVenvCreate def test_scm_ignore_files_git(self): @@ -978,7 +974,7 @@ def do_test_with_pip(self, system_site_packages): self.assertEqual(err, "") out = out.decode("latin-1") # Force to text, prevent decoding errors expected_version = "pip {}".format(ensurepip.version()) - self.assertEqual(out[:len(expected_version)], expected_version) + self.assertStartsWith(out, expected_version) env_dir = os.fsencode(self.env_dir).decode("latin-1") self.assertIn(env_dir, out) @@ -1008,7 +1004,7 @@ def do_test_with_pip(self, system_site_packages): err, flags=re.MULTILINE) # Ignore warning about missing optional module: try: - import ssl + import ssl # noqa: F401 except ImportError: err = re.sub( "^WARNING: Disabling truststore since ssl support is missing$", diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 05710c46934..a6af5057cc8 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -102,7 +102,7 @@ class PublicAPITests(BaseTest): """ def test_module_all_attribute(self): - self.assertTrue(hasattr(self.module, '__all__')) + self.assertHasAttr(self.module, '__all__') target_api = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings", "deprecated"] @@ -241,6 +241,96 @@ def test_once(self): 42) self.assertEqual(len(w), 0) + def test_filter_module(self): + MS_WINDOWS = (sys.platform == 'win32') + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'package\.module\z') + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='package.module') + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + self.assertEqual(len(w), 2) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + self.assertEqual(len(w), 3) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module/__init__.py', 42) + self.assertEqual(len(w), 4) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module/__init__', 42) + if MS_WINDOWS: + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PY', 42) + self.assertEqual(len(w), 5) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module\__INIT__.PY', 42) + self.assertEqual(len(w), 6) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PYW', 42) + self.assertEqual(len(w), 7) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module\__INIT__.PYW', 42) + self.assertEqual(len(w), 8) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module='package') + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='package.module') + self.assertEqual(len(w), 1) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='other.package.module') + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/otherpackage/module.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'/path/to/package/module\z') + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + self.assertEqual(len(w), 2) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/PATH/TO/PACKAGE/MODULE', 42) + if MS_WINDOWS: + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.PY', 42) + self.assertEqual(len(w), 3) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module/__init__.py', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.pyw', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'\path\to\package\module', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'/path/to/package/__init__\z') + self.module.warn_explicit('msg', UserWarning, '/path/to/package/__init__.py', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/__init__', 42) + self.assertEqual(len(w), 2) + + if MS_WINDOWS: + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'C:\\path\\to\\package\\module\z') + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.py', 42) + self.assertEqual(len(w), 2) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PY', 42) + self.assertEqual(len(w), 3) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.pyw', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\PATH\TO\PACKAGE\MODULE', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:/path/to/package/module', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module\__init__.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'<unknown>\z') + self.module.warn_explicit('msg', UserWarning, '', 42) + self.assertEqual(len(w), 1) + def test_module_globals(self): with self.module.catch_warnings(record=True) as w: self.module.simplefilter("always", UserWarning) @@ -320,7 +410,7 @@ def test_message_matching(self): def test_mutate_filter_list(self): class X: - def match(self, a): + def match(self, a, start=0): L[:] = [] L = [("default",X(),UserWarning,X(),0) for i in range(2)] @@ -555,13 +645,7 @@ def test_warn_explicit_non_ascii_filename(self): with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("always", category=UserWarning) - filenames = ["nonascii\xe9\u20ac"] - if not support.is_emscripten: - # JavaScript does not like surrogates. - # Invalid UTF-8 leading byte 0x80 encountered when - # deserializing a UTF-8 string in wasm memory to a JS - # string! - filenames.append("surrogate\udc80") + filenames = ["nonascii\xe9\u20ac", "surrogate\udc80"] for filename in filenames: try: os.fsencode(filename) @@ -602,25 +686,19 @@ def test_warning_classes(self): class MyWarningClass(Warning): pass - class NonWarningSubclass: - pass - # passing a non-subclass of Warning should raise a TypeError - with self.assertRaises(TypeError) as cm: + expected = "category must be a Warning subclass, not 'str'" + with self.assertRaisesRegex(TypeError, expected): self.module.warn('bad warning category', '') - self.assertIn('category must be a Warning subclass, not ', - str(cm.exception)) - with self.assertRaises(TypeError) as cm: - self.module.warn('bad warning category', NonWarningSubclass) - self.assertIn('category must be a Warning subclass, not ', - str(cm.exception)) + expected = "category must be a Warning subclass, not class 'int'" + with self.assertRaisesRegex(TypeError, expected): + self.module.warn('bad warning category', int) # check that warning instances also raise a TypeError - with self.assertRaises(TypeError) as cm: + expected = "category must be a Warning subclass, not '.*MyWarningClass'" + with self.assertRaisesRegex(TypeError, expected): self.module.warn('bad warning category', MyWarningClass()) - self.assertIn('category must be a Warning subclass, not ', - str(cm.exception)) with self.module.catch_warnings(): self.module.resetwarnings() @@ -649,7 +727,7 @@ def check_module_globals(self, module_globals): def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): if self.module is py_warnings: - self.check_module_globals(module_globals) + self.check_module_globals_deprecated(module_globals, errmsg) return with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') @@ -660,9 +738,6 @@ def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError) self.assertEqual(len(w), 0) def check_module_globals_deprecated(self, module_globals, msg): - if self.module is py_warnings: - self.check_module_globals(module_globals) - return with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') self.module.warn_explicit( @@ -735,7 +810,7 @@ class CWarnTests(WarnTests, unittest.TestCase): # test.import_helper.import_fresh_module utility function def test_accelerated(self): self.assertIsNot(original_warnings, self.module) - self.assertFalse(hasattr(self.module.warn, '__code__')) + self.assertNotHasAttr(self.module.warn, '__code__') class PyWarnTests(WarnTests, unittest.TestCase): module = py_warnings @@ -744,7 +819,7 @@ class PyWarnTests(WarnTests, unittest.TestCase): # test.import_helper.import_fresh_module utility function def test_pure_python(self): self.assertIsNot(original_warnings, self.module) - self.assertTrue(hasattr(self.module.warn, '__code__')) + self.assertHasAttr(self.module.warn, '__code__') class WCmdLineTests(BaseTest): @@ -767,6 +842,10 @@ def test_improper_input(self): self.module._setoption('ignore::===') with self.assertRaisesRegex(self.module._OptionError, 'Wärning'): self.module._setoption('ignore::Wärning') + with self.assertRaisesRegex(self.module._OptionError, 'message'): + self.module._setoption('ignore:/?/:Warning') + with self.assertRaisesRegex(self.module._OptionError, 'module'): + self.module._setoption('ignore::Warning:/?/') self.module._setoption('error::Warning::0') self.assertRaises(UserWarning, self.module.warn, 'convert to error') @@ -781,6 +860,31 @@ def test_import_from_module(self): with self.assertRaises(TestWarning): self.module.warn('test warning', TestWarning) + def test_message(self): + # Match prefix, case-insensitive. + with self.module.catch_warnings(): + self.module._setoption('error:TEST WARN:UserWarning') + with self.assertRaises(UserWarning): + self.module.warn('Test Warning') + with self.module.catch_warnings(): + self.module._setoption(r'error:/TE.*WARN/:UserWarning') + with self.assertRaises(UserWarning): + self.module.warn('Test Warning') + + def test_module(self): + with self.module.catch_warnings(): + self.module._setoption(f'error::UserWarning:{__name__}') + with self.assertRaises(UserWarning): + self.module.warn('test warning') + # Only full match. + self.module._setoption(f'ignore::UserWarning:{__name__[:-2]}') + with self.assertRaises(UserWarning): + self.module.warn('test warning') + with self.module.catch_warnings(): + self.module._setoption(f'error::UserWarning:/{re.escape(__name__[:-2])}./') + with self.assertRaises(UserWarning): + self.module.warn('test warning') + class CWCmdLineTests(WCmdLineTests, unittest.TestCase): module = c_warnings @@ -1528,12 +1632,12 @@ def test_late_resource_warning(self): # (_warnings will try to import it) code = "f = open(%a)" % __file__ rc, out, err = assert_python_ok("-Wd", "-c", code) - self.assertTrue(err.startswith(expected), ascii(err)) + self.assertStartsWith(err, expected) # import the warnings module code = "import warnings; f = open(%a)" % __file__ rc, out, err = assert_python_ok("-Wd", "-c", code) - self.assertTrue(err.startswith(expected), ascii(err)) + self.assertStartsWith(err, expected) class AsyncTests(BaseTest): @@ -1875,6 +1979,25 @@ class D(C, x=3): self.assertEqual(D.inited, 3) + def test_existing_init_subclass_in_sibling_base(self): + @deprecated("A will go away soon") + class A: + pass + class B: + def __init_subclass__(cls, x): + super().__init_subclass__() + cls.inited = x + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + class C(A, B, x=42): + pass + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + class D(B, A, x=42): + pass + self.assertEqual(D.inited, 42) + def test_init_subclass_has_correct_cls(self): init_subclass_saw = None diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 5e771c8de96..4c21f165537 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,8 +1,11 @@ import unittest from test import audiotests from test import support +from test.support.os_helper import FakePath import io +import os import struct +import tempfile import sys import wave @@ -136,32 +139,6 @@ def test__all__(self): not_exported = {'WAVE_FORMAT_PCM', 'WAVE_FORMAT_EXTENSIBLE', 'KSDATAFORMAT_SUBTYPE_PCM'} support.check__all__(self, wave, not_exported=not_exported) - def test_read_deprecations(self): - filename = support.findfile('pluck-pcm8.wav', subdir='audiodata') - with wave.open(filename) as reader: - with self.assertWarns(DeprecationWarning): - with self.assertRaises(wave.Error): - reader.getmark('mark') - with self.assertWarns(DeprecationWarning): - self.assertIsNone(reader.getmarkers()) - - def test_write_deprecations(self): - with io.BytesIO(b'') as tmpfile: - with wave.open(tmpfile, 'wb') as writer: - writer.setnchannels(1) - writer.setsampwidth(1) - writer.setframerate(1) - writer.setcomptype('NONE', 'not compressed') - - with self.assertWarns(DeprecationWarning): - with self.assertRaises(wave.Error): - writer.setmark(0, 0, 'mark') - with self.assertWarns(DeprecationWarning): - with self.assertRaises(wave.Error): - writer.getmark('mark') - with self.assertWarns(DeprecationWarning): - self.assertIsNone(writer.getmarkers()) - class WaveLowLevelTest(unittest.TestCase): @@ -222,6 +199,34 @@ def test_read_wrong_sample_width(self): with self.assertRaisesRegex(wave.Error, 'bad sample width'): wave.open(io.BytesIO(b)) + def test_open_in_write_raises(self): + # gh-136523: Wave_write.__del__ should not throw + with support.catch_unraisable_exception() as cm: + with self.assertRaises(OSError): + wave.open(os.curdir, "wb") + support.gc_collect() + self.assertIsNone(cm.unraisable) + + +class WaveOpen(unittest.TestCase): + def test_open_pathlike(self): + """It is possible to use `wave.read` and `wave.write` with a path-like object""" + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + cases = ( + FakePath(fp.name), + FakePath(os.fsencode(fp.name)), + os.fsencode(fp.name), + ) + for fake_path in cases: + with self.subTest(fake_path): + with wave.open(fake_path, 'wb') as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(44100) + + with wave.open(fake_path, 'rb') as f: + pass + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4faad6629fe..47f6b46061a 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -432,7 +432,7 @@ def check_proxy(self, o, proxy): self.assertEqual(proxy.foo, 2, "proxy does not reflect attribute modification") del o.foo - self.assertFalse(hasattr(proxy, 'foo'), + self.assertNotHasAttr(proxy, 'foo', "proxy does not reflect attribute removal") proxy.foo = 1 @@ -442,7 +442,7 @@ def check_proxy(self, o, proxy): self.assertEqual(o.foo, 2, "object does not reflect attribute modification via proxy") del proxy.foo - self.assertFalse(hasattr(o, 'foo'), + self.assertNotHasAttr(o, 'foo', "object does not reflect attribute removal via proxy") def test_proxy_deletion(self): @@ -1044,6 +1044,32 @@ def callback(obj): stderr = res.err.decode("ascii", "backslashreplace") self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + def test_clearing_weakrefs_in_gc(self): + # This test checks that when finalizers are called: + # 1. weakrefs with callbacks have been cleared + # 2. weakrefs without callbacks have not been cleared + errors = [] + def test(): + class Class: + def __init__(self): + self._self = self + self.wr1 = weakref.ref(Class, lambda x: None) + self.wr2 = weakref.ref(Class) + + def __del__(self): + # we can't use assert* here, because gc will swallow + # exceptions + if self.wr1() is not None: + errors.append("weakref with callback as cleared") + if self.wr2() is not Class: + errors.append("weakref without callback was cleared") + + Class() + + test() + gc.collect() + self.assertEqual(errors, []) + class SubclassableWeakrefTestCase(TestBase): @@ -1108,7 +1134,7 @@ def meth(self): self.assertEqual(r.slot1, "abc") self.assertEqual(r.slot2, "def") self.assertEqual(r.meth(), "abcdef") - self.assertFalse(hasattr(r, "__dict__")) + self.assertNotHasAttr(r, "__dict__") def test_subclass_refs_with_cycle(self): """Confirm https://bugs.python.org/issue3100 is fixed.""" diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 76e8e5c8ab7..c1e4f9c8366 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -466,7 +466,7 @@ def test_copying(self): self.assertIsNot(dup, s) self.assertIs(dup.x, s.x) self.assertIs(dup.z, s.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') dup = copy.deepcopy(s) self.assertIsInstance(dup, cls) @@ -476,7 +476,7 @@ def test_copying(self): self.assertIsNot(dup.x, s.x) self.assertEqual(dup.z, s.z) self.assertIsNot(dup.z, s.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') if __name__ == "__main__": diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 4c3ea1cd8df..20d347168b3 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -6,8 +6,8 @@ import sys import unittest import webbrowser -from functools import partial from test import support +from test.support import force_not_colorized_test_class from test.support import import_helper from test.support import is_apple_mobile from test.support import os_helper @@ -504,6 +504,7 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) +@force_not_colorized_test_class class CliTest(unittest.TestCase): def test_parse_args(self): for command, url, new_win in [ diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index d9076e77c15..1bae884ed9a 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -17,9 +17,9 @@ class WindowsConsoleIOTests(unittest.TestCase): def test_abc(self): - self.assertTrue(issubclass(ConIO, io.RawIOBase)) - self.assertFalse(issubclass(ConIO, io.BufferedIOBase)) - self.assertFalse(issubclass(ConIO, io.TextIOBase)) + self.assertIsSubclass(ConIO, io.RawIOBase) + self.assertNotIsSubclass(ConIO, io.BufferedIOBase) + self.assertNotIsSubclass(ConIO, io.TextIOBase) def test_open_fd(self): self.assertRaisesRegex(ValueError, diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 924a962781a..733d30b3922 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -209,6 +209,33 @@ def _test_named_args(self, key, sub_key): access=KEY_ALL_ACCESS) as okey: self.assertTrue(okey.handle != 0) + def test_hkey_comparison(self): + """Test HKEY comparison by handle value rather than object identity.""" + key1 = OpenKey(HKEY_CURRENT_USER, None) + key2 = OpenKey(HKEY_CURRENT_USER, None) + key3 = OpenKey(HKEY_LOCAL_MACHINE, None) + + self.addCleanup(CloseKey, key1) + self.addCleanup(CloseKey, key2) + self.addCleanup(CloseKey, key3) + + self.assertEqual(key1.handle, key2.handle) + self.assertTrue(key1 == key2) + self.assertFalse(key1 != key2) + + self.assertTrue(key1 != key3) + self.assertFalse(key1 == key3) + + # Closed keys should be equal (all have handle=0) + CloseKey(key1) + CloseKey(key2) + CloseKey(key3) + + self.assertEqual(key1.handle, 0) + self.assertEqual(key2.handle, 0) + self.assertEqual(key3.handle, 0) + self.assertEqual(key2, key3) + class LocalWinregTests(BaseWinregTests): @@ -517,6 +544,21 @@ def test_exception_numbers(self): with self.assertRaises(FileNotFoundError) as ctx: QueryValue(HKEY_CLASSES_ROOT, 'some_value_that_does_not_exist') + def test_delete_tree(self): + with CreateKey(HKEY_CURRENT_USER, test_key_name) as main_key: + with CreateKey(main_key, "subkey1") as subkey1: + SetValueEx(subkey1, "value1", 0, REG_SZ, "test_value1") + with CreateKey(subkey1, "subsubkey1") as subsubkey1: + SetValueEx(subsubkey1, "value2", 0, REG_DWORD, 42) + + with CreateKey(main_key, "subkey2") as subkey2: + SetValueEx(subkey2, "value3", 0, REG_SZ, "test_value3") + + DeleteTree(HKEY_CURRENT_USER, test_key_name) + + with self.assertRaises(OSError): + OpenKey(HKEY_CURRENT_USER, test_key_name) + if __name__ == "__main__": if not REMOTE_NAME: diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index fd7abd1782e..f16611b29a2 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -679,7 +679,7 @@ def testSingleComplexTarget(self): class C: pass blah = C() with mock_contextmanager_generator() as blah.foo: - self.assertEqual(hasattr(blah, "foo"), True) + self.assertHasAttr(blah, "foo") def testMultipleComplexTargets(self): class C: diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index ac7c9cb3a5a..90eb40439d4 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -70,8 +70,8 @@ def test_wmi_query_overflow(self): def test_wmi_query_multiple_rows(self): # Multiple instances should have an extra null separator r = wmi_exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") - self.assertFalse(r.startswith("\0"), r) - self.assertFalse(r.endswith("\0"), r) + self.assertNotStartsWith(r, "\0") + self.assertNotEndsWith(r, "\0") it = iter(r.split("\0")) try: while True: diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index b047f7b06f8..e04a4d2c221 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -149,9 +149,9 @@ def bad_app(environ,start_response): start_response("200 OK", ('Content-Type','text/plain')) return ["Hello, world!"] out, err = run_amock(validator(bad_app)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual( err.splitlines()[-2], "AssertionError: Headers (('Content-Type', 'text/plain')) must" @@ -174,9 +174,9 @@ def bad_app(environ, start_response): for status, exc_message in tests: with self.subTest(status=status): out, err = run_amock(create_bad_app(status)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual(err.splitlines()[-2], exc_message) def test_wsgi_input(self): @@ -185,9 +185,9 @@ def bad_app(e,s): s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) return [b"data"] out, err = run_amock(validator(bad_app)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual( err.splitlines()[-2], "AssertionError" ) @@ -200,7 +200,7 @@ def app(e, s): ]) return [b"data"] out, err = run_amock(validator(app)) - self.assertTrue(err.endswith('"GET / HTTP/1.0" 200 4\n')) + self.assertEndsWith(err, '"GET / HTTP/1.0" 200 4\n') ver = sys.version.split()[0].encode('ascii') py = python_implementation().encode('ascii') pyver = py + b"/" + ver diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 5fe9d688410..87811199706 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -218,6 +218,33 @@ class ElementTreeTest(unittest.TestCase): def serialize_check(self, elem, expected): self.assertEqual(serialize(elem), expected) + def test_constructor(self): + # Test constructor behavior. + + with self.assertRaises(TypeError): + tree = ET.ElementTree("") + with self.assertRaises(TypeError): + tree = ET.ElementTree(ET.ElementTree()) + + def test_setroot(self): + # Test _setroot behavior. + + tree = ET.ElementTree() + element = ET.Element("tag") + tree._setroot(element) + self.assertEqual(tree.getroot().tag, "tag") + self.assertEqual(tree.getroot(), element) + + # Test behavior with an invalid root element + + tree = ET.ElementTree() + with self.assertRaises(TypeError): + tree._setroot("") + with self.assertRaises(TypeError): + tree._setroot(ET.ElementTree()) + with self.assertRaises(TypeError): + tree._setroot(None) + def test_interface(self): # Test element tree interface. @@ -225,8 +252,7 @@ def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") direlem = dir(element) for attr in 'tag', 'attrib', 'text', 'tail': - self.assertTrue(hasattr(element, attr), - msg='no %s member' % attr) + self.assertHasAttr(element, attr) self.assertIn(attr, direlem, msg='no %s visible by dir' % attr) @@ -251,7 +277,7 @@ def check_element(element): # Make sure all standard element methods exist. def check_method(method): - self.assertTrue(hasattr(method, '__call__'), + self.assertHasAttr(method, '__call__', msg="%s not callable" % method) check_method(element.append) @@ -548,208 +574,6 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - def test_iterparse(self): - # Test iterparse interface. - - iterparse = ET.iterparse - - context = iterparse(SIMPLE_XMLFILE) - self.assertIsNone(context.root) - action, elem = next(context) - self.assertIsNone(context.root) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - context = iterparse(SIMPLE_NS_XMLFILE) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', '{namespace}element'), - ('end', '{namespace}element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ]) - - with open(SIMPLE_XMLFILE, 'rb') as source: - context = iterparse(source) - action, elem = next(context) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - events = () - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = () - context = iterparse(SIMPLE_XMLFILE, events=events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = ("start", "end") - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('start', 'root'), - ('start', 'element'), - ('end', 'element'), - ('start', 'element'), - ('end', 'element'), - ('start', 'empty-element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - - events = ("start", "end", "start-ns", "end-ns") - context = iterparse(SIMPLE_NS_XMLFILE, events) - self.assertEqual([(action, elem.tag) if action in ("start", "end") - else (action, elem) - for action, elem in context], [ - ('start-ns', ('', 'namespace')), - ('start', '{namespace}root'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}empty-element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ('end-ns', None), - ]) - - events = ('start-ns', 'end-ns') - context = iterparse(io.StringIO(r"<root xmlns=''/>"), events) - res = [action for action, elem in context] - self.assertEqual(res, ['start-ns', 'end-ns']) - - events = ("start", "end", "bogus") - with open(SIMPLE_XMLFILE, "rb") as f: - with self.assertRaises(ValueError) as cm: - iterparse(f, events) - self.assertFalse(f.closed) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError) as cm: - iterparse(SIMPLE_XMLFILE, events) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - del cm - - source = io.BytesIO( - b"<?xml version='1.0' encoding='iso-8859-1'?>\n" - b"<body xmlns='http://&#233;ffbot.org/ns'\n" - b" xmlns:cl\xe9='http://effbot.org/ns'>text</body>\n") - events = ("start-ns",) - context = iterparse(source, events) - self.assertEqual([(action, elem) for action, elem in context], [ - ('start-ns', ('', 'http://\xe9ffbot.org/ns')), - ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), - ]) - - source = io.StringIO("<document />junk") - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - - self.addCleanup(os_helper.unlink, TESTFN) - with open(TESTFN, "wb") as f: - f.write(b"<document />junk") - it = iterparse(TESTFN) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - del cm, it - - # Not exhausting the iterator still closes the resource (bpo-43292) - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - it.close() - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - it.close() - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with self.assertRaises(FileNotFoundError): - iterparse("nonexistent") - - def test_iterparse_close(self): - iterparse = ET.iterparse - - it = iterparse(SIMPLE_XMLFILE) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - list(it) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - list(it) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -1473,6 +1297,281 @@ def test_attlist_default(self): {'{http://www.w3.org/XML/1998/namespace}lang': 'eng'}) +class IterparseTest(unittest.TestCase): + # Test iterparse interface. + + def test_basic(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(it.root) + action, elem = next(it) + self.assertIsNone(it.root) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + it.close() + + it = iterparse(SIMPLE_NS_XMLFILE) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', '{namespace}element'), + ('end', '{namespace}element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ]) + it.close() + + def test_external_file(self): + with open(SIMPLE_XMLFILE, 'rb') as source: + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + + def test_events(self): + iterparse = ET.iterparse + + events = () + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = () + it = iterparse(SIMPLE_XMLFILE, events=events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = ("start", "end") + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'root'), + ('start', 'element'), + ('end', 'element'), + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + it.close() + + def test_namespace_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "start-ns", "end-ns") + it = iterparse(SIMPLE_NS_XMLFILE, events) + self.assertEqual([(action, elem.tag) if action in ("start", "end") + else (action, elem) + for action, elem in it], [ + ('start-ns', ('', 'namespace')), + ('start', '{namespace}root'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}empty-element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ('end-ns', None), + ]) + it.close() + + events = ('start-ns', 'end-ns') + it = iterparse(io.BytesIO(br"<root xmlns=''/>"), events) + res = [action for action, elem in it] + self.assertEqual(res, ['start-ns', 'end-ns']) + it.close() + + def test_unknown_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "bogus") + with open(SIMPLE_XMLFILE, "rb") as f: + with self.assertRaises(ValueError) as cm: + iterparse(f, events) + self.assertFalse(f.closed) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError) as cm: + iterparse(SIMPLE_XMLFILE, events) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + del cm + gc_collect() + + def test_non_utf8(self): + source = io.BytesIO( + b"<?xml version='1.0' encoding='iso-8859-1'?>\n" + b"<body xmlns='http://&#233;ffbot.org/ns'\n" + b" xmlns:cl\xe9='http://effbot.org/ns'>text</body>\n") + events = ("start-ns",) + it = ET.iterparse(source, events) + self.assertEqual([(action, elem) for action, elem in it], [ + ('start-ns', ('', 'http://\xe9ffbot.org/ns')), + ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), + ]) + + def test_parsing_error(self): + source = io.BytesIO(b"<document />junk") + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + + def test_nonexistent_file(self): + with self.assertRaises(FileNotFoundError): + ET.iterparse("nonexistent") + + def test_resource_warnings_not_exhausted(self): + # Not exhausting the iterator still closes the underlying file (bpo-43292) + # Not closing before del should emit ResourceWarning + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + it.close() + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: + del it + gc_collect() + # Not 'unclosed file'. + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + it.close() + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + + def test_resource_warnings_failed_iteration(self): + self.addCleanup(os_helper.unlink, TESTFN) + with open(TESTFN, "wb") as f: + f.write(b"<document />junk") + + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + it.close() + del cm, it + gc_collect() + + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertWarns(ResourceWarning) as wm: + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(TESTFN), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + + def test_resource_warnings_exhausted(self): + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + list(it) + it.close() + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: + list(it) + del it + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + + def test_close_not_exhausted(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + def test_close_exhausted(self): + iterparse = ET.iterparse + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + class XMLPullParserTest(unittest.TestCase): def _feed(self, parser, data, chunk_size=None, flush=False): @@ -1723,6 +1822,8 @@ def __next__(self): def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + with self.assertRaisesRegex(ValueError, "unknown event 'bogus'"): + ET.XMLPullParser(events=(x.decode() for x in (b'start', b'end', b'bogus'))) @unittest.skipIf(pyexpat.version_info < (2, 6, 0), f'Expat {pyexpat.version_info} does not ' @@ -2960,6 +3061,50 @@ def element_factory(x, y): del b gc_collect() + def test_deepcopy_clear(self): + # Prevent crashes when __deepcopy__() clears the children list. + # See https://github.com/python/cpython/issues/133009. + class X(ET.Element): + def __deepcopy__(self, memo): + root.clear() + return self + + root = ET.Element('a') + evil = X('x') + root.extend([evil, ET.Element('y')]) + if is_python_implementation(): + # Mutating a list over which we iterate raises an error. + self.assertRaises(RuntimeError, copy.deepcopy, root) + else: + c = copy.deepcopy(root) + # In the C implementation, we can still copy the evil element. + self.assertListEqual(list(c), [evil]) + + def test_deepcopy_grow(self): + # Prevent crashes when __deepcopy__() mutates the children list. + # See https://github.com/python/cpython/issues/133009. + a = ET.Element('a') + b = ET.Element('b') + c = ET.Element('c') + + class X(ET.Element): + def __deepcopy__(self, memo): + root.append(a) + root.append(b) + return self + + root = ET.Element('top') + evil1, evil2 = X('1'), X('2') + root.extend([evil1, c, evil2]) + children = list(copy.deepcopy(root)) + # mock deep copies + self.assertIs(children[0], evil1) + self.assertIs(children[2], evil2) + # true deep copies + self.assertEqual(children[1].tag, c.tag) + self.assertEqual([c.tag for c in children[3:]], + [a.tag, b.tag, a.tag, b.tag]) + class MutationDeleteElementPath(str): def __new__(cls, elem, *args): diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 9ed0f4096a4..270b9d6da8e 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -58,7 +58,7 @@ def test_del_attribute(self): self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) @support.skip_wasi_stack_overflow() - @unittest.skipIf(support.is_emscripten, "segfaults") + @support.skip_emscripten_stack_overflow() def test_trashcan(self): # If this test fails, it will most likely die via segfault. e = root = cET.Element('root') diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py index 6dbfb3f4393..b52e78bc4fb 100644 --- a/Lib/test/test_xxlimited.py +++ b/Lib/test/test_xxlimited.py @@ -31,7 +31,7 @@ def test_foo(self): self.assertEqual(self.module.foo(1, 2), 3) def test_str(self): - self.assertTrue(issubclass(self.module.Str, str)) + self.assertIsSubclass(self.module.Str, str) self.assertIsNot(self.module.Str, str) custom_string = self.module.Str("abcd") diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index d4766c59a10..8fb0a68deba 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -259,7 +259,7 @@ def test_pack_to_fileobj(self): (source / '__main__.py').touch() target = io.BytesIO() zipapp.create_archive(str(source), target, interpreter='python') - self.assertTrue(target.getvalue().startswith(b'#!python\n')) + self.assertStartsWith(target.getvalue(), b'#!python\n') def test_read_shebang(self): # Test that we can read the shebang line correctly. @@ -300,7 +300,7 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(source), str(target), interpreter='python') new_target = io.BytesIO() zipapp.create_archive(str(target), new_target, interpreter='python2.7') - self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) + self.assertStartsWith(new_target.getvalue(), b'#!python2.7\n') def test_read_from_pathlike_obj(self): # Test that we can copy an archive using a path-like object @@ -326,7 +326,7 @@ def test_read_from_fileobj(self): new_target = io.BytesIO() temp_archive.seek(0) zipapp.create_archive(temp_archive, new_target, interpreter='python2.7') - self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) + self.assertStartsWith(new_target.getvalue(), b'#!python2.7\n') def test_remove_shebang(self): # Test that we can remove the shebang from a file. diff --git a/Lib/test/test_zipfile/__main__.py b/Lib/test/test_zipfile/__main__.py index e25ac946edf..90da74ade38 100644 --- a/Lib/test/test_zipfile/__main__.py +++ b/Lib/test/test_zipfile/__main__.py @@ -1,6 +1,6 @@ import unittest -from . import load_tests # noqa: F401 +from . import load_tests if __name__ == "__main__": diff --git a/Lib/test/test_zipfile/_path/_test_params.py b/Lib/test/test_zipfile/_path/_test_params.py index bc95b4ebf4a..00a9eaf2f99 100644 --- a/Lib/test/test_zipfile/_path/_test_params.py +++ b/Lib/test/test_zipfile/_path/_test_params.py @@ -1,5 +1,5 @@ -import types import functools +import types from ._itertools import always_iterable diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index b505dd7c376..7c108fc6ab8 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -8,10 +8,8 @@ from ._functools import compose from ._itertools import consume - from ._support import import_or_skip - big_o = import_or_skip('big_o') pytest = import_or_skip('pytest') diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 0afabc0c668..e7931b6f394 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -1,6 +1,6 @@ +import contextlib import io import itertools -import contextlib import pathlib import pickle import stat @@ -9,12 +9,11 @@ import zipfile import zipfile._path -from test.support.os_helper import temp_dir, FakePath +from test.support.os_helper import FakePath, temp_dir from ._functools import compose from ._itertools import Counter - -from ._test_params import parameterize, Invoked +from ._test_params import Invoked, parameterize class jaraco: @@ -193,10 +192,10 @@ def test_encoding_warnings(self, alpharep): """EncodingWarning must blame the read_text and open calls.""" assert sys.flags.warn_default_encoding root = zipfile.Path(alpharep) - with self.assertWarns(EncodingWarning) as wc: + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) root.joinpath("a.txt").read_text() assert __file__ == wc.filename - with self.assertWarns(EncodingWarning) as wc: + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) root.joinpath("a.txt").open("r").close() assert __file__ == wc.filename @@ -275,7 +274,8 @@ def test_pathlike_construction(self, alpharep): """ zipfile_ondisk = self.zipfile_ondisk(alpharep) pathlike = FakePath(str(zipfile_ondisk)) - zipfile.Path(pathlike) + root = zipfile.Path(pathlike) + root.root.close() @pass_alpharep def test_traverse_pathlike(self, alpharep): @@ -317,7 +317,7 @@ def test_mutability(self, alpharep): HUGE_ZIPFILE_NUM_ENTRIES = 2**13 def huge_zipfile(self): - """Create a read-only zipfile with a huge number of entries entries.""" + """Create a read-only zipfile with a huge number of entries.""" strm = io.BytesIO() zf = zipfile.ZipFile(strm, "w") for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): @@ -364,6 +364,18 @@ def test_root_name(self, alpharep): root = zipfile.Path(alpharep) assert root.name == 'alpharep.zip' == root.filename.name + @pass_alpharep + def test_root_on_disk(self, alpharep): + """ + The name/stem of the root should match the zipfile on disk. + + This condition must hold across platforms. + """ + root = zipfile.Path(self.zipfile_ondisk(alpharep)) + assert root.name == 'alpharep.zip' == root.filename.name + assert root.stem == 'alpharep' == root.filename.stem + root.root.close() + @pass_alpharep def test_suffix(self, alpharep): """ @@ -564,11 +576,13 @@ def test_inheritance(self, alpharep): ) def test_pickle(self, alpharep, path_type, subpath): zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) - - saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) + root = zipfile.Path(zipfile_ondisk, at=subpath) + saved_1 = pickle.dumps(root) + root.root.close() restored_1 = pickle.loads(saved_1) first, *rest = restored_1.iterdir() assert first.read_text(encoding='utf-8').startswith('content of ') + restored_1.root.close() @pass_alpharep def test_extract_orig_with_implied_dirs(self, alpharep): @@ -580,6 +594,7 @@ def test_extract_orig_with_implied_dirs(self, alpharep): # wrap the zipfile for its side effect zipfile.Path(zf) zf.extractall(source_path.parent) + zf.close() @pass_alpharep def test_getinfo_missing(self, alpharep): diff --git a/Lib/test/test_zipfile/_path/write-alpharep.py b/Lib/test/test_zipfile/_path/write-alpharep.py index 48c09b53717..7418391abad 100644 --- a/Lib/test/test_zipfile/_path/write-alpharep.py +++ b/Lib/test/test_zipfile/_path/write-alpharep.py @@ -1,4 +1,3 @@ from . import test_path - __name__ == '__main__' and test_path.build_alpharep_fixture().extractall('alpharep') diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index ae898150658..6887a5e5cc4 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -312,26 +312,26 @@ def test_low_compression(self): self.assertEqual(openobj.read(1), b'2') def test_writestr_compression(self): - zipfp = zipfile.ZipFile(TESTFN2, "w") - zipfp.writestr("b.txt", "hello world", compress_type=self.compression) - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, self.compression) + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.writestr("b.txt", "hello world", compress_type=self.compression) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, self.compression) def test_writestr_compresslevel(self): - zipfp = zipfile.ZipFile(TESTFN2, "w", compresslevel=1) - zipfp.writestr("a.txt", "hello world", compress_type=self.compression) - zipfp.writestr("b.txt", "hello world", compress_type=self.compression, - compresslevel=2) + with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: + zipfp.writestr("a.txt", "hello world", compress_type=self.compression) + zipfp.writestr("b.txt", "hello world", compress_type=self.compression, + compresslevel=2) - # Compression level follows the constructor. - a_info = zipfp.getinfo('a.txt') - self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info.compress_level, 1) + # Compression level follows the constructor. + a_info = zipfp.getinfo('a.txt') + self.assertEqual(a_info.compress_type, self.compression) + self.assertEqual(a_info.compress_level, 1) - # Compression level is overridden. - b_info = zipfp.getinfo('b.txt') - self.assertEqual(b_info.compress_type, self.compression) - self.assertEqual(b_info._compresslevel, 2) + # Compression level is overridden. + b_info = zipfp.getinfo('b.txt') + self.assertEqual(b_info.compress_type, self.compression) + self.assertEqual(b_info._compresslevel, 2) def test_read_return_size(self): # Issue #9837: ZipExtFile.read() shouldn't return more bytes @@ -898,6 +898,8 @@ def make_zip64_file( self, file_size_64_set=False, file_size_extra=False, compress_size_64_set=False, compress_size_extra=False, header_offset_64_set=False, header_offset_extra=False, + extensible_data=b'', + end_of_central_dir_size=None, offset_to_end_of_central_dir=None, ): """Generate bytes sequence for a zip with (incomplete) zip64 data. @@ -951,6 +953,12 @@ def make_zip64_file( central_dir_size = struct.pack('<Q', 58 + 8 * len(central_zip64_fields)) offset_to_central_dir = struct.pack('<Q', 50 + 8 * len(local_zip64_fields)) + if end_of_central_dir_size is None: + end_of_central_dir_size = 44 + len(extensible_data) + if offset_to_end_of_central_dir is None: + offset_to_end_of_central_dir = (108 + + 8 * len(local_zip64_fields) + + 8 * len(central_zip64_fields)) local_extra_length = struct.pack("<H", 4 + 8 * len(local_zip64_fields)) central_extra_length = struct.pack("<H", 4 + 8 * len(central_zip64_fields)) @@ -979,14 +987,17 @@ def make_zip64_file( + filename + central_extra # Zip64 end of central directory - + b"PK\x06\x06,\x00\x00\x00\x00\x00\x00\x00-\x00-" - + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + + b"PK\x06\x06" + + struct.pack('<Q', end_of_central_dir_size) + + b"-\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + b"\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00" + central_dir_size + offset_to_central_dir + + extensible_data # Zip64 end of central directory locator - + b"PK\x06\x07\x00\x00\x00\x00l\x00\x00\x00\x00\x00\x00\x00\x01" - + b"\x00\x00\x00" + + b"PK\x06\x07\x00\x00\x00\x00" + + struct.pack('<Q', offset_to_end_of_central_dir) + + b"\x01\x00\x00\x00" # end of central directory + b"PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00:\x00\x00\x002\x00" + b"\x00\x00\x00\x00" @@ -1017,6 +1028,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_file_size_extra)) self.assertIn('file size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_file_size_extra))) # zip64 file size present, zip64 compress size present, one field in # extra, expecting two, equals missing compress size. @@ -1028,6 +1040,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) self.assertIn('compress size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra))) # zip64 compress size present, no fields in extra, expecting one, # equals missing compress size. @@ -1037,6 +1050,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) self.assertIn('compress size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra))) # zip64 file size present, zip64 compress size present, zip64 header # offset present, two fields in extra, expecting three, equals missing @@ -1051,6 +1065,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 compress size present, zip64 header offset present, one field # in extra, expecting two, equals missing header offset @@ -1063,6 +1078,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 file size present, zip64 header offset present, one field in # extra, expecting two, equals missing header offset @@ -1075,6 +1091,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 header offset present, no fields in extra, expecting one, # equals missing header offset @@ -1086,6 +1103,63 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) + + def test_bad_zip64_end_of_central_dir(self): + zipdata = self.make_zip64_file(end_of_central_dir_size=0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(end_of_central_dir_size=100) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(offset_to_end_of_central_dir=0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(offset_to_end_of_central_dir=1000) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*locator'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + def test_zip64_end_of_central_dir_record_not_found(self): + zipdata = self.make_zip64_file() + zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4) + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file( + extensible_data=b'\xca\xfe\x04\x00\x00\x00data') + zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4) + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + def test_zip64_extensible_data(self): + # These values are what is set in the make_zip64_file method. + expected_file_size = 8 + expected_compress_size = 8 + expected_header_offset = 0 + expected_content = b"test1234" + + zipdata = self.make_zip64_file( + extensible_data=b'\xca\xfe\x04\x00\x00\x00data') + with zipfile.ZipFile(io.BytesIO(zipdata)) as zf: + zinfo = zf.infolist()[0] + self.assertEqual(zinfo.file_size, expected_file_size) + self.assertEqual(zinfo.compress_size, expected_compress_size) + self.assertEqual(zinfo.header_offset, expected_header_offset) + self.assertEqual(zf.read(zinfo), expected_content) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(zipdata))) + + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(b'prepended' + zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(b'prepended' + zipdata))) def test_generated_valid_zip64_extra(self): # These values are what is set in the make_zip64_file method. @@ -1991,6 +2065,25 @@ def test_is_zip_erroneous_file(self): self.assertFalse(zipfile.is_zipfile(fp)) fp.seek(0, 0) self.assertFalse(zipfile.is_zipfile(fp)) + # - passing non-zipfile with ZIP header elements + # data created using pyPNG like so: + # d = [(ord('P'), ord('K'), 5, 6), (ord('P'), ord('K'), 6, 6)] + # w = png.Writer(1,2,alpha=True,compression=0) + # f = open('onepix.png', 'wb') + # w.write(f, d) + # w.close() + data = (b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00" + b"\x00\x02\x08\x06\x00\x00\x00\x99\x81\xb6'\x00\x00\x00\x15I" + b"DATx\x01\x01\n\x00\xf5\xff\x00PK\x05\x06\x00PK\x06\x06\x07" + b"\xac\x01N\xc6|a\r\x00\x00\x00\x00IEND\xaeB`\x82") + # - passing a filename + with open(TESTFN, "wb") as fp: + fp.write(data) + self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a file-like object + fp = io.BytesIO() + fp.write(data) + self.assertFalse(zipfile.is_zipfile(fp)) def test_damaged_zipfile(self): """Check that zipfiles with missing bytes at the end raise BadZipFile.""" @@ -2237,6 +2330,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except zipfile.BadZipFile: self.fail("Unable to create empty ZIP file in 'w' mode") + zipf.close() zipf = zipfile.ZipFile(TESTFN, mode="a") zipf.close() @@ -2244,6 +2338,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except: self.fail("Unable to create empty ZIP file in 'a' mode") + zipf.close() def test_open_empty_file(self): # Issue 1710703: Check that opening a file with less than 22 bytes @@ -2436,6 +2531,10 @@ def test_decompress_without_3rd_party_library(self): @requires_zlib() def test_full_overlap_different_names(self): + # The ZIP file contains two central directory entries with + # different names which refer to the same local header. + # The name of the local header matches the name of the first + # central directory entry. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00b\xed' @@ -2465,6 +2564,10 @@ def test_full_overlap_different_names(self): @requires_zlib() def test_full_overlap_different_names2(self): + # The ZIP file contains two central directory entries with + # different names which refer to the same local header. + # The name of the local header matches the name of the second + # central directory entry. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' @@ -2496,6 +2599,8 @@ def test_full_overlap_different_names2(self): @requires_zlib() def test_full_overlap_same_name(self): + # The ZIP file contains two central directory entries with + # the same name which refer to the same local header. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' @@ -2528,6 +2633,8 @@ def test_full_overlap_same_name(self): @requires_zlib() def test_quoted_overlap(self): + # The ZIP file contains two files. The second local header + # is contained in the range of the first file. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' @@ -2559,6 +2666,7 @@ def test_quoted_overlap(self): @requires_zlib() def test_overlap_with_central_dir(self): + # The local header offset is equal to the central directory offset. data = ( b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' @@ -2573,11 +2681,15 @@ def test_overlap_with_central_dir(self): self.assertEqual(zi.header_offset, 0) self.assertEqual(zi.compress_size, 11) self.assertEqual(zi.file_size, 1033) + # Found central directory signature PK\x01\x02 instead of + # local header signature PK\x03\x04. with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number'): zipf.read('a') @requires_zlib() def test_overlap_with_archive_comment(self): + # The local header is written after the central directory, + # in the archive comment. data = ( b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' @@ -3179,7 +3291,7 @@ def test_write_dir(self): with zipfile.ZipFile(TESTFN, "w") as zipf: zipf.write(dirpath) zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("/x/")) + self.assertEndsWith(zinfo.filename, "/x/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) zipf.write(dirpath, "y") zinfo = zipf.filelist[1] @@ -3187,7 +3299,7 @@ def test_write_dir(self): self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) with zipfile.ZipFile(TESTFN, "r") as zipf: zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("/x/")) + self.assertEndsWith(zinfo.filename, "/x/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) zinfo = zipf.filelist[1] self.assertTrue(zinfo.filename, "y/") @@ -3207,7 +3319,7 @@ def test_writestr_dir(self): self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) with zipfile.ZipFile(TESTFN, "r") as zipf: zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("x/")) + self.assertEndsWith(zinfo.filename, "x/") self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) target = os.path.join(TESTFN2, "target") os.mkdir(target) @@ -3451,60 +3563,6 @@ def test_execute_zip64(self): self.assertIn(b'number in executable: 5', output) -class TestDataOffsetPrependedZip(unittest.TestCase): - """Test .data_offset on reading zip files with an executable prepended.""" - - def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='archivetestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='archivetestdata') - - def _test_data_offset(self, name): - with zipfile.ZipFile(name) as zipfp: - self.assertEqual(zipfp.data_offset, 713) - - def test_data_offset_with_exe_prepended(self): - self._test_data_offset(self.exe_zip) - - def test_data_offset_with_exe_prepended_zip64(self): - self._test_data_offset(self.exe_zip64) - -class TestDataOffsetZipWrite(unittest.TestCase): - """Test .data_offset for ZipFile opened in write mode.""" - - def setUp(self): - os.mkdir(TESTFNDIR) - self.addCleanup(rmtree, TESTFNDIR) - self.test_path = os.path.join(TESTFNDIR, 'testoffset.zip') - - def test_data_offset_write_no_prefix(self): - with io.BytesIO() as fp: - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertEqual(zipfp.data_offset, 0) - - def test_data_offset_write_with_prefix(self): - with io.BytesIO() as fp: - fp.write(b"this is a prefix") - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertEqual(zipfp.data_offset, 16) - - def test_data_offset_append_with_bad_zip(self): - with io.BytesIO() as fp: - fp.write(b"this is a prefix") - with zipfile.ZipFile(fp, "a") as zipfp: - self.assertEqual(zipfp.data_offset, 16) - - def test_data_offset_write_no_tell(self): - # The initializer in ZipFile checks if tell raises AttributeError or - # OSError when creating a file in write mode when deducing the offset - # of the beginning of zip data - class NoTellBytesIO(io.BytesIO): - def tell(self): - raise OSError("Unimplemented!") - with NoTellBytesIO() as fp: - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertIsNone(zipfp.data_offset) - - class EncodedMetadataTests(unittest.TestCase): file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three' file_content = [ @@ -3642,7 +3700,7 @@ def test_cli_with_metadata_encoding_extract(self): except OSError: pass except UnicodeEncodeError: - self.skipTest(f'cannot encode file name {fn!r}') + self.skipTest(f'cannot encode file name {fn!a}') zipfile.main(["--metadata-encoding=shift_jis", "-e", TESTFN, TESTFN2]) listing = os.listdir(TESTFN2) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 1f288c8b45d..dce3e1d9d38 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -9,13 +9,12 @@ import time import unittest import unittest.mock -import warnings from test import support from test.support import import_helper from test.support import os_helper -from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED +from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED, ZIP_ZSTANDARD import zipimport import linecache @@ -194,19 +193,38 @@ def testAFakeZlib(self): # occur in that case (builtin modules are always found first), # so we'll simply skip it then. Bug #765456. # - if "zlib" in sys.builtin_module_names: - self.skipTest('zlib is a builtin module') - if "zlib" in sys.modules: - del sys.modules["zlib"] - files = {"zlib.py": test_src} - try: - self.doTest(".py", files, "zlib") - except ImportError: - if self.compression != ZIP_DEFLATED: - self.fail("expected test to not raise ImportError") + if self.compression == ZIP_DEFLATED: + mod_name = "zlib" + if zipimport._zlib_decompress: # validate attr name + # reset the cached import to avoid test order dependencies + zipimport._zlib_decompress = None # reset cache + elif self.compression == ZIP_ZSTANDARD: + mod_name = "_zstd" + if zipimport._zstd_decompressor_class: # validate attr name + # reset the cached import to avoid test order dependencies + zipimport._zstd_decompressor_class = None else: + mod_name = "zlib" # the ZIP_STORED case below + + if mod_name in sys.builtin_module_names: + self.skipTest(f"{mod_name} is a builtin module") + if mod_name in sys.modules: + del sys.modules[mod_name] + files = {f"{mod_name}.py": test_src} + try: + self.doTest(".py", files, mod_name) + except ImportError: if self.compression != ZIP_STORED: - self.fail("expected test to raise ImportError") + # Expected - fake compression module can't decompress + pass + else: + self.fail("expected test to not raise ImportError for uncompressed") + else: + if self.compression == ZIP_STORED: + # Expected - no compression needed, so fake module works + pass + else: + self.fail("expected test to raise ImportError for compressed zip with fake compression module") def testPy(self): files = {TESTMOD + ".py": test_src} @@ -556,13 +574,6 @@ def testZipImporterMethods(self): self.assertEqual(zi.archive, TEMP_ZIP) self.assertTrue(zi.is_package(TESTPACK)) - # PEP 302 - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - - mod = zi.load_module(TESTPACK) - self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) - # PEP 451 spec = zi.find_spec('spam') self.assertIsNotNone(spec) @@ -577,6 +588,8 @@ def testZipImporterMethods(self): spec.loader.exec_module(mod) self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) + sys.path.insert(0, TEMP_ZIP) + existing_pack_path = importlib.import_module(TESTPACK).__path__[0] expected_path_path = os.path.join(TEMP_ZIP, TESTPACK) self.assertEqual(existing_pack_path, expected_path_path) @@ -675,11 +688,6 @@ def testZipImporterMethodsInSubDirectory(self): self.assertEqual(zi.archive, TEMP_ZIP) self.assertEqual(zi.prefix, packdir) self.assertTrue(zi.is_package(TESTPACK2)) - # PEP 302 - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - mod = zi.load_module(TESTPACK2) - self.assertEqual(zi.get_filename(TESTPACK2), mod.__file__) # PEP 451 spec = zi.find_spec(TESTPACK2) mod = importlib.util.module_from_spec(spec) @@ -702,9 +710,12 @@ def testZipImporterMethodsInSubDirectory(self): self.assertEqual( spec.loader.get_filename(TESTMOD), load_mod.__file__) + sys.path.insert(0, TEMP_ZIP + os.sep + TESTPACK) + mod_path = TESTPACK2 + os.sep + TESTMOD mod_name = module_path_to_dotted_name(mod_path) mod = importlib.import_module(mod_name) + self.assertTrue(mod_name in sys.modules) self.assertIsNone(zi.get_source(TESTPACK2)) self.assertIsNone(zi.get_source(mod_path)) @@ -835,11 +846,11 @@ def doTraceback(self, module): s = io.StringIO() print_tb(tb, 1, s) - self.assertTrue(s.getvalue().endswith( + self.assertEndsWith(s.getvalue(), ' def do_raise(): raise TypeError\n' '' if support.has_no_debug_ranges() else ' ^^^^^^^^^^^^^^^\n' - )) + ) else: raise AssertionError("This ought to be impossible") @@ -1008,10 +1019,15 @@ def assertDataEntry(name): @support.requires_zlib() -class CompressedZipImportTestCase(UncompressedZipImportTestCase): +class DeflateCompressedZipImportTestCase(UncompressedZipImportTestCase): compression = ZIP_DEFLATED +@support.requires_zstd() +class ZStdCompressedZipImportTestCase(UncompressedZipImportTestCase): + compression = ZIP_ZSTANDARD + + class BadFileZipImportTestCase(unittest.TestCase): def assertZipFailure(self, filename): self.assertRaises(zipimport.ZipImportError, @@ -1069,9 +1085,6 @@ def _testBogusZipFile(self): z = zipimport.zipimporter(TESTMOD) try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.assertRaises(TypeError, z.load_module, None) self.assertRaises(TypeError, z.find_module, None) self.assertRaises(TypeError, z.find_spec, None) self.assertRaises(TypeError, z.exec_module, None) @@ -1082,10 +1095,6 @@ def _testBogusZipFile(self): error = zipimport.ZipImportError self.assertIsNone(z.find_spec('abc')) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.assertRaises(error, z.load_module, 'abc') self.assertRaises(error, z.get_code, 'abc') self.assertRaises(OSError, z.get_data, 'abc') self.assertRaises(error, z.get_source, 'abc') diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index ae8a8c99762..2b28f46149b 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -13,9 +13,12 @@ import inspect import linecache import unittest +import warnings +from test import support from test.support import os_helper from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, make_script, make_zip_script) +from test.support import import_helper verbose = test.support.verbose @@ -236,6 +239,26 @@ def f(): # bdb/pdb applies normcase to its filename before displaying self.assertIn(os.path.normcase(run_name.encode('utf-8')), data) + def test_import_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with (os_helper.temp_dir() as tmpdir, + import_helper.DirsOnSysPath()): + zip_name, _ = make_zip_script(tmpdir, "test_zip", + filename, 'test_pkg/test_mod.py') + sys.path.insert(0, zip_name) + import_helper.unload('test_pkg.test_mod') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'test_pkg\.test_mod\z') + warnings.filterwarnings('error', module='test_mod') + import test_pkg.test_mod + self.assertEqual(sorted(wm.lineno for wm in wlog), + sorted([4, 7, 10, 13, 14, 21]*2)) + filename = test_pkg.test_mod.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + def tearDownModule(): test.support.reap_children() diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 4d97fe56f3a..ed9d8540815 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -119,6 +119,114 @@ def test_same_as_binascii_crc32(self): self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam')) +class ChecksumCombineMixin: + """Mixin class for testing checksum combination.""" + + N = 1000 + default_iv: int + + def parse_iv(self, iv): + """Parse an IV value. + + - The default IV is returned if *iv* is None. + - A random IV is returned if *iv* is -1. + - Otherwise, *iv* is returned as is. + """ + if iv is None: + return self.default_iv + if iv == -1: + return random.randint(1, 0x80000000) + return iv + + def checksum(self, data, init=None): + """Compute the checksum of data with a given initial value. + + The *init* value is parsed by ``parse_iv``. + """ + iv = self.parse_iv(init) + return self._checksum(data, iv) + + def _checksum(self, data, init): + raise NotImplementedError + + def combine(self, a, b, blen): + """Combine two checksums together.""" + raise NotImplementedError + + def get_random_data(self, data_len, *, iv=None): + """Get a triplet (data, iv, checksum).""" + data = random.randbytes(data_len) + init = self.parse_iv(iv) + checksum = self.checksum(data, init) + return data, init, checksum + + def test_combine_empty(self): + for _ in range(self.N): + a, iv, checksum = self.get_random_data(32, iv=-1) + res = self.combine(iv, self.checksum(a), len(a)) + self.assertEqual(res, checksum) + + def test_combine_no_iv(self): + for _ in range(self.N): + a, _, chk_a = self.get_random_data(32) + b, _, chk_b = self.get_random_data(64) + res = self.combine(chk_a, chk_b, len(b)) + self.assertEqual(res, self.checksum(a + b)) + + def test_combine_no_iv_invalid_length(self): + a, _, chk_a = self.get_random_data(32) + b, _, chk_b = self.get_random_data(64) + checksum = self.checksum(a + b) + for invalid_len in [1, len(a), 48, len(b) + 1, 191]: + invalid_res = self.combine(chk_a, chk_b, invalid_len) + self.assertNotEqual(invalid_res, checksum) + + self.assertRaises(TypeError, self.combine, 0, 0, "len") + + def test_combine_with_iv(self): + for _ in range(self.N): + a, iv_a, chk_a_with_iv = self.get_random_data(32, iv=-1) + chk_a_no_iv = self.checksum(a) + b, iv_b, chk_b_with_iv = self.get_random_data(64, iv=-1) + chk_b_no_iv = self.checksum(b) + + # We can represent c = COMBINE(CHK(a, iv_a), CHK(b, iv_b)) as: + # + # c = CHK(CHK(b'', iv_a) + CHK(a) + CHK(b'', iv_b) + CHK(b)) + # = COMBINE( + # COMBINE(CHK(b'', iv_a), CHK(a)), + # COMBINE(CHK(b'', iv_b), CHK(b)), + # ) + # = COMBINE(COMBINE(iv_a, CHK(a)), COMBINE(iv_b, CHK(b))) + tmp0 = self.combine(iv_a, chk_a_no_iv, len(a)) + tmp1 = self.combine(iv_b, chk_b_no_iv, len(b)) + expected = self.combine(tmp0, tmp1, len(b)) + checksum = self.combine(chk_a_with_iv, chk_b_with_iv, len(b)) + self.assertEqual(checksum, expected) + + +class CRC32CombineTestCase(ChecksumCombineMixin, unittest.TestCase): + + default_iv = 0 + + def _checksum(self, data, init): + return zlib.crc32(data, init) + + def combine(self, a, b, blen): + return zlib.crc32_combine(a, b, blen) + + +class Adler32CombineTestCase(ChecksumCombineMixin, unittest.TestCase): + + default_iv = 1 + + def _checksum(self, data, init): + return zlib.adler32(data, init) + + def combine(self, a, b, blen): + return zlib.adler32_combine(a, b, blen) + + # Issue #10276 - check that inputs >=4 GiB are handled correctly. class ChecksumBigBufferTestCase(unittest.TestCase): @@ -1114,5 +1222,15 @@ def __index__(self): return 100 +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(zlib, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index d2845495c7f..49b620f5c8a 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -18,10 +18,11 @@ from functools import cached_property from test.support import MISSING_C_DOCSTRINGS -from test.support.os_helper import EnvironmentVarGuard +from test.support.os_helper import EnvironmentVarGuard, FakePath from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import TZPATH_TEST_LOCK, ZoneInfoTestBase from test.support.import_helper import import_module, CleanImport +from test.support.script_helper import assert_python_ok lzma = import_module('lzma') py_zoneinfo, c_zoneinfo = test_support.get_modules() @@ -58,6 +59,10 @@ def tearDownModule(): shutil.rmtree(TEMP_DIR) +class CustomError(Exception): + pass + + class TzPathUserMixin: """ Adds a setUp() and tearDown() to make TZPATH manipulations thread-safe. @@ -404,6 +409,25 @@ def test_time_fixed_offset(self): self.assertEqual(t.utcoffset(), offset.utcoffset) self.assertEqual(t.dst(), offset.dst) + def test_cache_exception(self): + class Incomparable(str): + eq_called = False + def __eq__(self, other): + self.eq_called = True + raise CustomError + __hash__ = str.__hash__ + + key = "America/Los_Angeles" + tz1 = self.klass(key) + key = Incomparable(key) + try: + tz2 = self.klass(key) + except CustomError: + self.assertTrue(key.eq_called) + else: + self.assertFalse(key.eq_called) + self.assertIs(tz2, tz1) + class CZoneInfoTest(ZoneInfoTest): module = c_zoneinfo @@ -1507,6 +1531,26 @@ def test_clear_cache_two_keys(self): self.assertIsNot(dub0, dub1) self.assertIs(tok0, tok1) + def test_clear_cache_refleak(self): + class Stringy(str): + allow_comparisons = True + def __eq__(self, other): + if not self.allow_comparisons: + raise CustomError + return super().__eq__(other) + __hash__ = str.__hash__ + + key = Stringy("America/Los_Angeles") + self.klass(key) + key.allow_comparisons = False + try: + # Note: This is try/except rather than assertRaises because + # there is no guarantee that the key is even still in the cache, + # or that the key for the cache is the original `key` object. + self.klass.clear_cache(only_keys="America/Los_Angeles") + except CustomError: + pass + class CZoneInfoCacheTest(ZoneInfoCacheTest): module = c_zoneinfo @@ -1740,6 +1784,7 @@ def test_reset_tzpath_relative_paths(self): ("/usr/share/zoneinfo", "../relative/path",), ("path/to/somewhere", "../relative/path",), ("/usr/share/zoneinfo", "path/to/somewhere", "../relative/path",), + (FakePath("path/to/somewhere"),) ] for input_paths in bad_values: with self.subTest(input_paths=input_paths): @@ -1751,6 +1796,9 @@ def test_tzpath_type_error(self): "/etc/zoneinfo:/usr/share/zoneinfo", b"/etc/zoneinfo:/usr/share/zoneinfo", 0, + (b"/bytes/path", "/valid/path"), + (FakePath(b"/bytes/path"),), + (0,), ] for bad_value in bad_values: @@ -1761,6 +1809,7 @@ def test_tzpath_type_error(self): def test_tzpath_attribute(self): tzpath_0 = [f"{DRIVE}/one", f"{DRIVE}/two"] tzpath_1 = [f"{DRIVE}/three"] + tzpath_pathlike = (FakePath(f"{DRIVE}/usr/share/zoneinfo"),) with self.tzpath_context(tzpath_0): query_0 = self.module.TZPATH @@ -1768,8 +1817,12 @@ def test_tzpath_attribute(self): with self.tzpath_context(tzpath_1): query_1 = self.module.TZPATH + with self.tzpath_context(tzpath_pathlike): + query_pathlike = self.module.TZPATH + self.assertSequenceEqual(tzpath_0, query_0) self.assertSequenceEqual(tzpath_1, query_1) + self.assertSequenceEqual(tuple([os.fspath(p) for p in tzpath_pathlike]), query_pathlike) class CTzPathTest(TzPathTest): @@ -1898,11 +1951,46 @@ def test_exclude_posixrules(self): actual = self.module.available_timezones() self.assertEqual(actual, expected) + def test_exclude_localtime(self): + expected = { + "America/New_York", + "Europe/London", + } + + tree = list(expected) + ["localtime"] + + with tempfile.TemporaryDirectory() as td: + for key in tree: + self.touch_zone(key, td) + + with self.tzpath_context([td]): + actual = self.module.available_timezones() + self.assertEqual(actual, expected) class CTestModule(TestModule): module = c_zoneinfo +class MiscTests(unittest.TestCase): + def test_pydatetime(self): + # Test that zoneinfo works if the C implementation of datetime + # is not available and the Python implementation of datetime is used. + # The Python implementation of zoneinfo should be used in thet case. + # + # Run the test in a subprocess, as importing _zoneinfo with + # _datettime disabled causes crash in the previously imported + # _zoneinfo. + assert_python_ok('-c', '''if 1: + import sys + sys.modules['_datetime'] = None + import datetime + import zoneinfo + tzinfo = zoneinfo.ZoneInfo('Europe/London') + datetime.datetime(2025, 10, 26, 2, 0, tzinfo=tzinfo) + ''', + PYTHONTZPATH=str(ZONEINFO_DATA.tzpath)) + + class ExtensionBuiltTest(unittest.TestCase): """Smoke test to ensure that the C and Python extensions are both tested. @@ -1915,8 +2003,8 @@ class ExtensionBuiltTest(unittest.TestCase): def test_cache_location(self): # The pure Python version stores caches on attributes, but the C # extension stores them in C globals (at least for now) - self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache")) - self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) + self.assertNotHasAttr(c_zoneinfo.ZoneInfo, "_weak_cache") + self.assertHasAttr(py_zoneinfo.ZoneInfo, "_weak_cache") def test_gc_tracked(self): import gc diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py index feaa77f3e7f..c00815e2fd4 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo_property.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -146,20 +146,22 @@ def setUp(self): @add_key_examples def test_pickle_unpickle_cache(self, key): zi = self.klass(key) - pkl_str = pickle.dumps(zi) - zi_rt = pickle.loads(pkl_str) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIs(zi, zi_rt) + self.assertIs(zi, zi_rt) @hypothesis.given(key=valid_keys()) @add_key_examples def test_pickle_unpickle_no_cache(self, key): zi = self.klass.no_cache(key) - pkl_str = pickle.dumps(zi) - zi_rt = pickle.loads(pkl_str) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIsNot(zi, zi_rt) - self.assertEqual(str(zi), str(zi_rt)) + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) @hypothesis.given(key=valid_keys()) @add_key_examples diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index f4a25376e52..6358cc78739 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -12,7 +12,6 @@ from test.support.import_helper import import_module from test.support import threading_helper from test.support import _1M -from test.support import Py_GIL_DISABLED _zstd = import_module("_zstd") zstd = import_module("compression.zstd") @@ -63,11 +62,18 @@ TRAINED_DICT = None -SUPPORT_MULTITHREADING = False +# Cannot be deferred to setup as it is used to check whether or not to skip +# tests +try: + SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) +except Exception: + SUPPORT_MULTITHREADING = False + +C_INT_MIN = -(2**31) +C_INT_MAX = (2**31) - 1 + def setUpModule(): - global SUPPORT_MULTITHREADING - SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) # uncompressed size 130KB, more than a zstd block. # with a frame epilogue, 4 bytes checksum. global DAT_130K_D @@ -196,14 +202,21 @@ def test_simple_compress_bad_args(self): self.assertRaises(TypeError, ZstdCompressor, zstd_dict=b"abcd1234") self.assertRaises(TypeError, ZstdCompressor, zstd_dict={1: 2, 3: 4}) - with self.assertRaises(ValueError): - ZstdCompressor(2**31) - with self.assertRaises(ValueError): - ZstdCompressor(options={2**31: 100}) + # valid range for compression level is [-(1<<17), 22] + msg = r'illegal compression level {}; the valid range is \[-?\d+, -?\d+\]' + with self.assertRaisesRegex(ValueError, msg.format(C_INT_MAX)): + ZstdCompressor(C_INT_MAX) + with self.assertRaisesRegex(ValueError, msg.format(C_INT_MIN)): + ZstdCompressor(C_INT_MIN) + msg = r'illegal compression level; the valid range is \[-?\d+, -?\d+\]' + with self.assertRaisesRegex(ValueError, msg): + ZstdCompressor(level=-(2**1000)) + with self.assertRaisesRegex(ValueError, msg): + ZstdCompressor(level=2**1000) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.window_log: 100}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={3333: 100}) # Method bad arguments @@ -254,43 +267,57 @@ def test_compress_parameters(self): } ZstdCompressor(options=d) - # larger than signed int, ValueError d1 = d.copy() - d1[CompressionParameter.ldm_bucket_size_log] = 2**31 - self.assertRaises(ValueError, ZstdCompressor, options=d1) + # larger than signed int + d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MAX + with self.assertRaises(ValueError): + ZstdCompressor(options=d1) + # smaller than signed int + d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MIN + with self.assertRaises(ValueError): + ZstdCompressor(options=d1) - # clamp compressionLevel + # out of bounds compression level level_min, level_max = CompressionParameter.compression_level.bounds() - compress(b'', level_max+1) - compress(b'', level_min-1) - - compress(b'', options={CompressionParameter.compression_level:level_max+1}) - compress(b'', options={CompressionParameter.compression_level:level_min-1}) + with self.assertRaises(ValueError): + compress(b'', level_max+1) + with self.assertRaises(ValueError): + compress(b'', level_min-1) + with self.assertRaises(ValueError): + compress(b'', 2**1000) + with self.assertRaises(ValueError): + compress(b'', -(2**1000)) + with self.assertRaises(ValueError): + compress(b'', options={ + CompressionParameter.compression_level: level_max+1}) + with self.assertRaises(ValueError): + compress(b'', options={ + CompressionParameter.compression_level: level_min-1}) # zstd lib doesn't support MT compression if not SUPPORT_MULTITHREADING: - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.nb_workers:4}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.job_size:4}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.overlap_log:4}) # out of bounds error msg option = {CompressionParameter.window_log:100} - with self.assertRaisesRegex(ZstdError, - (r'Error when setting zstd compression parameter "window_log", ' - r'it should \d+ <= value <= \d+, provided value is 100\. ' - r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')): + with self.assertRaisesRegex( + ValueError, + "compression parameter 'window_log' received an illegal value 100; " + r'the valid range is \[-?\d+, -?\d+\]', + ): compress(b'', options=option) def test_unknown_compression_parameter(self): KEY = 100001234 option = {CompressionParameter.compression_level: 10, KEY: 200000000} - pattern = r'Zstd compression parameter.*?"unknown parameter \(key %d\)"' \ - % KEY - with self.assertRaisesRegex(ZstdError, pattern): + pattern = rf"invalid compression parameter 'unknown parameter \(key {KEY}\)'" + with self.assertRaisesRegex(ValueError, pattern): ZstdCompressor(options=option) @unittest.skipIf(not SUPPORT_MULTITHREADING, @@ -371,6 +398,115 @@ def test_compress_empty(self): c = ZstdCompressor() self.assertNotEqual(c.compress(b'', c.FLUSH_FRAME), b'') + def test_set_pledged_input_size(self): + DAT = DECOMPRESSED_100_PLUS_32KB + CHUNK_SIZE = len(DAT) // 3 + + # wrong value + c = ZstdCompressor() + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(-300) + # overflow + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64) + # ZSTD_CONTENTSIZE_ERROR is invalid + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-2) + # ZSTD_CONTENTSIZE_UNKNOWN should use None + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-1) + + # check valid values are settable + c.set_pledged_input_size(2**63) + c.set_pledged_input_size(2**64-3) + + # check that zero means empty frame + c = ZstdCompressor(level=1) + c.set_pledged_input_size(0) + c.compress(b'') + dat = c.flush() + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, 0) + + + # wrong mode + c = ZstdCompressor(level=1) + c.compress(b'123456') + self.assertEqual(c.last_mode, c.CONTINUE) + with self.assertRaisesRegex(ValueError, + r'last_mode == FLUSH_FRAME'): + c.set_pledged_input_size(300) + + # None value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(None) + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, None) + + # correct value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + chunks = [] + posi = 0 + while posi < len(DAT): + dat = c.compress(DAT[posi:posi+CHUNK_SIZE]) + posi += CHUNK_SIZE + chunks.append(dat) + + dat = c.flush() + chunks.append(dat) + chunks = b''.join(chunks) + + ret = get_frame_info(chunks) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(chunks), DAT) + + c.set_pledged_input_size(len(DAT)) # the second frame + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(dat), DAT) + + # not enough data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)+1) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.flush() + + # too much data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.compress(b'extra', ZstdCompressor.FLUSH_FRAME) + + # content size not set if content_size_flag == 0 + c = ZstdCompressor(options={CompressionParameter.content_size_flag: 0}) + c.set_pledged_input_size(10) + dat1 = c.compress(b"hello") + dat2 = c.compress(b"world") + dat3 = c.flush() + frame_data = get_frame_info(dat1 + dat2 + dat3) + self.assertIsNone(frame_data.decompressed_size) + + class DecompressorTestCase(unittest.TestCase): def test_simple_decompress_bad_args(self): @@ -385,12 +521,22 @@ def test_simple_decompress_bad_args(self): self.assertRaises(TypeError, ZstdDecompressor, options=b'abc') with self.assertRaises(ValueError): - ZstdDecompressor(options={2**31 : 100}) + ZstdDecompressor(options={C_INT_MAX: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={C_INT_MIN: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={0: C_INT_MAX}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={2**1000: 100}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={-(2**1000): 100}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={0: -(2**1000)}) - with self.assertRaises(ZstdError): - ZstdDecompressor(options={DecompressionParameter.window_log_max:100}) - with self.assertRaises(ZstdError): - ZstdDecompressor(options={3333 : 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={DecompressionParameter.window_log_max: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={3333: 100}) empty = compress(b'') lzd = ZstdDecompressor() @@ -403,26 +549,52 @@ def test_decompress_parameters(self): d = {DecompressionParameter.window_log_max : 15} ZstdDecompressor(options=d) - # larger than signed int, ValueError d1 = d.copy() - d1[DecompressionParameter.window_log_max] = 2**31 - self.assertRaises(ValueError, ZstdDecompressor, None, d1) + # larger than signed int + d1[DecompressionParameter.window_log_max] = 2**1000 + with self.assertRaises(OverflowError): + ZstdDecompressor(None, d1) + # smaller than signed int + d1[DecompressionParameter.window_log_max] = -(2**1000) + with self.assertRaises(OverflowError): + ZstdDecompressor(None, d1) + + d1[DecompressionParameter.window_log_max] = C_INT_MAX + with self.assertRaises(ValueError): + ZstdDecompressor(None, d1) + d1[DecompressionParameter.window_log_max] = C_INT_MIN + with self.assertRaises(ValueError): + ZstdDecompressor(None, d1) # out of bounds error msg options = {DecompressionParameter.window_log_max:100} - with self.assertRaisesRegex(ZstdError, - (r'Error when setting zstd decompression parameter "window_log_max", ' - r'it should \d+ <= value <= \d+, provided value is 100\. ' - r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')): + with self.assertRaisesRegex( + ValueError, + "decompression parameter 'window_log_max' received an illegal value 100; " + r'the valid range is \[-?\d+, -?\d+\]', + ): + decompress(b'', options=options) + + # out of bounds deecompression parameter + options[DecompressionParameter.window_log_max] = C_INT_MAX + with self.assertRaises(ValueError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = C_INT_MIN + with self.assertRaises(ValueError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = 2**1000 + with self.assertRaises(OverflowError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = -(2**1000) + with self.assertRaises(OverflowError): decompress(b'', options=options) def test_unknown_decompression_parameter(self): KEY = 100001234 options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1], KEY: 200000000} - pattern = r'Zstd decompression parameter.*?"unknown parameter \(key %d\)"' \ - % KEY - with self.assertRaisesRegex(ZstdError, pattern): + pattern = rf"invalid decompression parameter 'unknown parameter \(key {KEY}\)'" + with self.assertRaisesRegex(ValueError, pattern): ZstdDecompressor(options=options) def test_decompress_epilogue_flags(self): @@ -507,7 +679,7 @@ def test_decompress_epilogue_flags(self): self.assertFalse(d.needs_input) def test_decompressor_arg(self): - zd = ZstdDict(b'12345678', True) + zd = ZstdDict(b'12345678', is_raw=True) with self.assertRaises(TypeError): d = ZstdDecompressor(zstd_dict={}) @@ -1021,6 +1193,10 @@ def test_decompressor_skippable(self): class ZstdDictTestCase(unittest.TestCase): def test_is_raw(self): + # must be passed as a keyword argument + with self.assertRaises(TypeError): + ZstdDict(bytes(8), True) + # content < 8 b = b'1234567' with self.assertRaises(ValueError): @@ -1068,35 +1244,49 @@ def test_invalid_dict(self): # corrupted zd = ZstdDict(dict_content, is_raw=False) - with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?corrupted'): + with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'): ZstdCompressor(zstd_dict=zd.as_digested_dict) - with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?corrupted'): + with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'): ZstdDecompressor(zd) # wrong type - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): - ZstdCompressor(zstd_dict=(zd, b'123')) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, 1, 2)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, -1)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, 3)) + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, -2**1000)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): - ZstdDecompressor(zstd_dict=(zd, b'123')) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, 1, 2)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, -1)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, 3)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, -2**1000)) def test_train_dict(self): - - TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1) - ZstdDict(TRAINED_DICT.dict_content, False) + ZstdDict(TRAINED_DICT.dict_content, is_raw=False) self.assertNotEqual(TRAINED_DICT.dict_id, 0) self.assertGreater(len(TRAINED_DICT.dict_content), 0) @@ -1174,43 +1364,91 @@ def test_finalize_dict_arguments(self): def test_train_dict_c(self): # argument wrong type with self.assertRaises(TypeError): - _zstd._train_dict({}, (), 100) + _zstd.train_dict({}, (), 100) with self.assertRaises(TypeError): - _zstd._train_dict(b'', 99, 100) + _zstd.train_dict(bytearray(), (), 100) with self.assertRaises(TypeError): - _zstd._train_dict(b'', (), 100.1) + _zstd.train_dict(b'', 99, 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', [], 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', (), 100.1) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', (99.1,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (4, -1), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (2,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (99,), 100) # size > size_t with self.assertRaises(ValueError): - _zstd._train_dict(b'', (2**64+1,), 100) + _zstd.train_dict(b'', (2**1000,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (-2**1000,), 100) # dict_size <= 0 with self.assertRaises(ValueError): - _zstd._train_dict(b'', (), 0) + _zstd.train_dict(b'', (), 0) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (), -1) + + with self.assertRaises(ZstdError): + _zstd.train_dict(b'', (), 1) def test_finalize_dict_c(self): with self.assertRaises(TypeError): - _zstd._finalize_dict(1, 2, 3, 4, 5) + _zstd.finalize_dict(1, 2, 3, 4, 5) # argument wrong type with self.assertRaises(TypeError): - _zstd._finalize_dict({}, b'', (), 100, 5) + _zstd.finalize_dict({}, b'', (), 100, 5) with self.assertRaises(TypeError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5) + _zstd.finalize_dict(bytearray(TRAINED_DICT.dict_content), b'', (), 100, 5) with self.assertRaises(TypeError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5) with self.assertRaises(TypeError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, bytearray(), (), 100, 5) with self.assertRaises(TypeError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', [], 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1) + + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (4, -1), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (2,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (99,), 100, 5) # size > size_t with self.assertRaises(ValueError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**1000,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (-2**1000,), 100, 5) # dict_size <= 0 with self.assertRaises(ValueError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -1, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 2**1000, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -2**1000, 5) + + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 2**1000) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, -2**1000) + + with self.assertRaises(ZstdError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5) def test_train_buffer_protocol_samples(self): def _nbytes(dat): @@ -1232,25 +1470,25 @@ def _nbytes(dat): # wrong size list with self.assertRaisesRegex(ValueError, "The samples size tuple doesn't match the concatenation's size"): - _zstd._train_dict(concatenation, tuple(wrong_size_lst), 100*_1K) + _zstd.train_dict(concatenation, tuple(wrong_size_lst), 100*_1K) # correct size list - _zstd._train_dict(concatenation, tuple(correct_size_lst), 3*_1K) + _zstd.train_dict(concatenation, tuple(correct_size_lst), 3*_1K) # wrong size list with self.assertRaisesRegex(ValueError, "The samples size tuple doesn't match the concatenation's size"): - _zstd._finalize_dict(TRAINED_DICT.dict_content, + _zstd.finalize_dict(TRAINED_DICT.dict_content, concatenation, tuple(wrong_size_lst), 300*_1K, 5) # correct size list - _zstd._finalize_dict(TRAINED_DICT.dict_content, + _zstd.finalize_dict(TRAINED_DICT.dict_content, concatenation, tuple(correct_size_lst), 300*_1K, 5) def test_as_prefix(self): # V1 V1 = THIS_FILE_BYTES - zd = ZstdDict(V1, True) + zd = ZstdDict(V1, is_raw=True) # V2 mid = len(V1) // 2 @@ -1266,7 +1504,7 @@ def test_as_prefix(self): self.assertEqual(decompress(dat, zd.as_prefix), V2) # use wrong prefix - zd2 = ZstdDict(SAMPLES[0], True) + zd2 = ZstdDict(SAMPLES[0], is_raw=True) try: decompressed = decompress(dat, zd2.as_prefix) except ZstdError: # expected @@ -1420,11 +1658,12 @@ def test_init_bad_mode(self): with self.assertRaises(ValueError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rw") - with self.assertRaisesRegex(TypeError, r"NOT be CompressionParameter"): + with self.assertRaisesRegex(TypeError, + r"not be a CompressionParameter"): ZstdFile(io.BytesIO(), 'rb', options={CompressionParameter.compression_level:5}) with self.assertRaisesRegex(TypeError, - r"NOT be DecompressionParameter"): + r"not be a DecompressionParameter"): ZstdFile(io.BytesIO(), 'wb', options={DecompressionParameter.window_log_max:21}) @@ -1435,19 +1674,19 @@ def test_init_bad_check(self): with self.assertRaises(TypeError): ZstdFile(io.BytesIO(), "w", level='asd') # CHECK_UNKNOWN and anything above CHECK_ID_MAX should be invalid. - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdFile(io.BytesIO(), "w", options={999:9999}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdFile(io.BytesIO(), "w", options={CompressionParameter.window_log:99}) with self.assertRaises(TypeError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r", options=33) - with self.assertRaises(ValueError): + with self.assertRaises(OverflowError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), options={DecompressionParameter.window_log_max:2**31}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), options={444:333}) @@ -1463,7 +1702,7 @@ def test_init_close_fp(self): tmp_f.write(DAT_130K_C) filename = tmp_f.name - with self.assertRaises(ValueError): + with self.assertRaises(TypeError): ZstdFile(filename, options={'a':'b'}) # for PyPy @@ -1682,10 +1921,10 @@ def test_read_incomplete(self): # Trailing data isn't a valid compressed stream with ZstdFile(io.BytesIO(self.FRAME_42 + b'12345')) as f: - self.assertEqual(f.read(), self.DECOMPRESSED_42) + self.assertRaises(ZstdError, f.read) with ZstdFile(io.BytesIO(SKIPPABLE_FRAME + b'12345')) as f: - self.assertEqual(f.read(), b'') + self.assertRaises(ZstdError, f.read) def test_read_truncated(self): # Drop stream epilogue: 4 bytes checksum @@ -2428,15 +2667,18 @@ def test_buffer_protocol(self): class FreeThreadingMethodTests(unittest.TestCase): - @unittest.skipUnless(Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_compress_locking(self): input = b'a'* (16*_1K) num_threads = 8 + # gh-136394: the first output of .compress() includes the frame header + # we run the first .compress() call outside of the threaded portion + # to make the test order-independent + comp = ZstdCompressor() - parts = [] + parts = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] for _ in range(num_threads): res = comp.compress(input, ZstdCompressor.FLUSH_BLOCK) if res: @@ -2445,7 +2687,7 @@ def test_compress_locking(self): expected = b''.join(parts) + rest1 comp = ZstdCompressor() - output = [] + output = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] def run_method(method, input_data, output_data): res = method(input_data, ZstdCompressor.FLUSH_BLOCK) if res: @@ -2465,7 +2707,6 @@ def run_method(method, input_data, output_data): actual = b''.join(output) + rest2 self.assertEqual(expected, actual) - @unittest.skipUnless(Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_decompress_locking(self): @@ -2501,6 +2742,59 @@ def run_method(method, input_data, output_data): actual = b''.join(output) self.assertEqual(expected, actual) + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_compress_shared_dict(self): + num_threads = 8 + + def run_method(b): + level = threading.get_ident() % 4 + # sync threads to increase chance of contention on + # capsule storing dictionary levels + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_decompress_shared_dict(self): + num_threads = 8 + + def run_method(b): + # sync threads to increase chance of contention on + # decompression dictionary + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass if __name__ == "__main__": diff --git a/Lib/test/typinganndata/fwdref_module.py b/Lib/test/typinganndata/fwdref_module.py new file mode 100644 index 00000000000..7347a7a4245 --- /dev/null +++ b/Lib/test/typinganndata/fwdref_module.py @@ -0,0 +1,6 @@ +from typing import ForwardRef + +MyList = list[int] +MyDict = dict[str, 'MyList'] + +fw = ForwardRef('MyDict', module=__name__) diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 5ae439f5cd3..41366fbf443 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -211,7 +211,7 @@ def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. - if self.break_long_words: + if self.break_long_words and space_left > 0: end = space_left chunk = reversed_chunks[-1] if self.break_on_hyphens and len(chunk) > space_left: diff --git a/Lib/threading.py b/Lib/threading.py index 39a1a7f4cdf..4ebceae7029 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -123,7 +123,7 @@ def gettrace(): Lock = _LockType -def RLock(*args, **kwargs): +def RLock(): """Factory function that returns a new reentrant lock. A reentrant lock must be released by the thread that acquired it. Once a @@ -132,16 +132,9 @@ def RLock(*args, **kwargs): acquired it. """ - if args or kwargs: - import warnings - warnings.warn( - 'Passing arguments to RLock is deprecated and will be removed in 3.15', - DeprecationWarning, - stacklevel=2, - ) if _CRLock is None: - return _PyRLock(*args, **kwargs) - return _CRLock(*args, **kwargs) + return _PyRLock() + return _CRLock() class _RLock: """This class implements reentrant lock objects. @@ -165,7 +158,7 @@ def __repr__(self): except KeyError: pass return "<%s %s.%s object owner=%r count=%d at %s>" % ( - "locked" if self._block.locked() else "unlocked", + "locked" if self.locked() else "unlocked", self.__class__.__module__, self.__class__.__qualname__, owner, @@ -244,7 +237,7 @@ def __exit__(self, t, v, tb): def locked(self): """Return whether this object is locked.""" - return self._count > 0 + return self._block.locked() # Internal methods used by condition variables @@ -660,7 +653,8 @@ def wait(self, timeout=None): (or fractions thereof). This method returns the internal flag on exit, so it will always return - True except if a timeout is given and the operation times out. + ``True`` except if a timeout is given and the operation times out, when + it will return ``False``. """ with self._cond: @@ -951,6 +945,8 @@ def _after_fork(self, new_ident=None): # This thread is alive. self._ident = new_ident assert self._os_thread_handle.ident == new_ident + if _HAVE_THREAD_NATIVE_ID: + self._set_native_id() else: # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() # already marked our handle done. @@ -1419,7 +1415,7 @@ def __init__(self, dummy_thread): # the related _DummyThread will be kept forever! _thread_local_info._track_dummy_thread_ref = self - def __del__(self): + def __del__(self, _active_limbo_lock=_active_limbo_lock, _active=_active): with _active_limbo_lock: if _active.get(self._tident) is self._dummy_thread: _active.pop(self._tident, None) @@ -1562,8 +1558,9 @@ def _shutdown(): # normally - that won't happen until the interpreter is nearly dead. So # mark it done here. if _main_thread._os_thread_handle.is_done() and _is_main_interpreter(): - # _shutdown() was already called - return + # _shutdown() was already called, but threads might have started + # in the meantime. + return _thread_shutdown() global _SHUTTING_DOWN _SHUTTING_DOWN = True diff --git a/Lib/timeit.py b/Lib/timeit.py index e767f018782..80791acdeca 100644 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -133,7 +133,7 @@ def __init__(self, stmt="pass", setup="pass", timer=default_timer, exec(code, global_ns, local_ns) self.inner = local_ns["inner"] - def print_exc(self, file=None): + def print_exc(self, file=None, **kwargs): """Helper to print a traceback from the timed code. Typical use: @@ -149,6 +149,11 @@ def print_exc(self, file=None): The optional file argument directs where the traceback is sent; it defaults to sys.stderr. + + The optional colorize keyword argument controls whether the + traceback is colorized; it defaults to False for programmatic + usage. When used from the command line, this is automatically + set based on terminal capabilities. """ import linecache, traceback if self.src is not None: @@ -158,7 +163,8 @@ def print_exc(self, file=None): dummy_src_name) # else the source is already stored somewhere else - traceback.print_exc(file=file) + kwargs['colorize'] = kwargs.get('colorize', False) + traceback.print_exc(file=file, **kwargs) def timeit(self, number=default_number): """Time 'number' executions of the main statement. @@ -257,9 +263,12 @@ def main(args=None, *, _wrap_timer=None): is not None, it must be a callable that accepts a timer function and returns another timer function (used for unit testing). """ + import getopt if args is None: args = sys.argv[1:] - import getopt + import _colorize + colorize = _colorize.can_colorize() + try: opts, args = getopt.getopt(args, "n:u:s:r:pvh", ["number=", "setup=", "repeat=", @@ -326,7 +335,7 @@ def callback(number, time_taken): try: number, _ = t.autorange(callback) except: - t.print_exc() + t.print_exc(colorize=colorize) return 1 if verbose: @@ -335,7 +344,7 @@ def callback(number, time_taken): try: raw_timings = t.repeat(repeat, number) except: - t.print_exc() + t.print_exc(colorize=colorize) return 1 def format_time(dt): diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index a693b04870b..737583a42c6 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -7,25 +7,25 @@ LabelFrame and PanedWindow. Properties of the widgets are specified with keyword arguments. -Keyword arguments have the same name as the corresponding resource +Keyword arguments have the same name as the corresponding options under Tk. Widgets are positioned with one of the geometry managers Place, Pack or Grid. These managers can be called with methods place, pack, grid available in every Widget. -Actions are bound to events by resources (e.g. keyword argument -command) or with the method bind. +Actions are bound to events by options (e.g. the command +keyword argument) or with the bind() method. Example (Hello, World): import tkinter from tkinter.constants import * tk = tkinter.Tk() frame = tkinter.Frame(tk, relief=RIDGE, borderwidth=2) -frame.pack(fill=BOTH,expand=1) +frame.pack(fill=BOTH, expand=1) label = tkinter.Label(frame, text="Hello, World") label.pack(fill=X, expand=1) -button = tkinter.Button(frame,text="Exit",command=tk.destroy) +button = tkinter.Button(frame, text="Exit", command=tk.destroy) button.pack(side=BOTTOM) tk.mainloop() """ @@ -847,7 +847,7 @@ def tk_focusNext(self): The focus order first goes to the next child, then to the children of the child recursively and then to the next sibling which is higher in the stacking order. A - widget is omitted if it has the takefocus resource set + widget is omitted if it has the takefocus option set to 0.""" name = self.tk.call('tk_focusNext', self._w) if not name: return None @@ -1827,27 +1827,34 @@ def _configure(self, cmd, cnf, kw): # These used to be defined in Widget: def configure(self, cnf=None, **kw): - """Configure resources of a widget. + """Query or modify the configuration options of the widget. - The values for resources are specified as keyword - arguments. To get an overview about - the allowed keyword arguments call the method keys. + If no arguments are specified, return a dictionary describing + all of the available options for the widget. + + If an option name is specified, then return a tuple describing + the one named option. + + If one or more keyword arguments are specified or a dictionary + is specified, then modify the widget option(s) to have the given + value(s). """ return self._configure('configure', cnf, kw) config = configure def cget(self, key): - """Return the resource value for a KEY given as string.""" + """Return the current value of the configuration option.""" return self.tk.call(self._w, 'cget', '-' + key) __getitem__ = cget + __iter__ = None # prevent using __getitem__ for iteration def __setitem__(self, key, value): self.configure({key: value}) def keys(self): - """Return a list of all resource names of this widget.""" + """Return a list of all option names of this widget.""" splitlist = self.tk.splitlist return [splitlist(x)[0][1:] for x in splitlist(self.tk.call(self._w, 'configure'))] @@ -1966,7 +1973,7 @@ def _grid_configure(self, command, index, cnf, kw): def grid_columnconfigure(self, index, cnf={}, **kw): """Configure column INDEX of a grid. - Valid resources are minsize (minimum size of the column), + Valid options are minsize (minimum size of the column), weight (how much does additional space propagate to this column) and pad (how much space to let additionally).""" return self._grid_configure('columnconfigure', index, cnf, kw) @@ -1997,7 +2004,7 @@ def grid_propagate(self, flag=_noarg_): def grid_rowconfigure(self, index, cnf={}, **kw): """Configure row INDEX of a grid. - Valid resources are minsize (minimum size of the row), + Valid options are minsize (minimum size of the row), weight (how much does additional space propagate to this row) and pad (how much space to let additionally).""" return self._grid_configure('rowconfigure', index, cnf, kw) @@ -2817,7 +2824,7 @@ class Toplevel(BaseWidget, Wm): def __init__(self, master=None, cnf={}, **kw): """Construct a toplevel widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, class, + Valid option names: background, bd, bg, borderwidth, class, colormap, container, cursor, height, highlightbackground, highlightcolor, highlightthickness, menu, relief, screen, takefocus, use, visual, width.""" @@ -2894,7 +2901,7 @@ class Canvas(Widget, XView, YView): def __init__(self, master=None, cnf={}, **kw): """Construct a canvas widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, closeenough, + Valid option names: background, bd, bg, borderwidth, closeenough, confine, cursor, height, highlightbackground, highlightcolor, highlightthickness, insertbackground, insertborderwidth, insertofftime, insertontime, insertwidth, offset, relief, @@ -3103,16 +3110,14 @@ def insert(self, *args): self.tk.call((self._w, 'insert') + args) def itemcget(self, tagOrId, option): - """Return the resource value for an OPTION for item TAGORID.""" + """Return the value of OPTION for item TAGORID.""" return self.tk.call( (self._w, 'itemcget') + (tagOrId, '-'+option)) def itemconfigure(self, tagOrId, cnf=None, **kw): - """Configure resources of an item TAGORID. + """Query or modify the configuration options of an item TAGORID. - The values for resources are specified as keyword - arguments. To get an overview about - the allowed keyword arguments call the method without arguments. + Similar to configure() except that it applies to the specified item. """ return self._configure(('itemconfigure', tagOrId), cnf, kw) @@ -3204,7 +3209,7 @@ class Checkbutton(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a checkbutton widget with the parent MASTER. - Valid resource names: activebackground, activeforeground, anchor, + Valid option names: activebackground, activeforeground, anchor, background, bd, bg, bitmap, borderwidth, command, cursor, disabledforeground, fg, font, foreground, height, highlightbackground, highlightcolor, highlightthickness, image, @@ -3235,7 +3240,7 @@ def flash(self): self.tk.call(self._w, 'flash') def invoke(self): - """Toggle the button and invoke a command if given as resource.""" + """Toggle the button and invoke a command if given as option.""" return self.tk.call(self._w, 'invoke') def select(self): @@ -3253,7 +3258,7 @@ class Entry(Widget, XView): def __init__(self, master=None, cnf={}, **kw): """Construct an entry widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, cursor, + Valid option names: background, bd, bg, borderwidth, cursor, exportselection, fg, font, foreground, highlightbackground, highlightcolor, highlightthickness, insertbackground, insertborderwidth, insertofftime, insertontime, insertwidth, @@ -3339,7 +3344,7 @@ class Frame(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a frame widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, class, + Valid option names: background, bd, bg, borderwidth, class, colormap, container, cursor, height, highlightbackground, highlightcolor, highlightthickness, relief, takefocus, visual, width.""" cnf = _cnfmerge((cnf, kw)) @@ -3383,7 +3388,7 @@ class Listbox(Widget, XView, YView): def __init__(self, master=None, cnf={}, **kw): """Construct a listbox widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, cursor, + Valid option names: background, bd, bg, borderwidth, cursor, exportselection, fg, font, foreground, height, highlightbackground, highlightcolor, highlightthickness, relief, selectbackground, selectborderwidth, selectforeground, selectmode, setgrid, takefocus, @@ -3476,18 +3481,15 @@ def size(self): return self.tk.getint(self.tk.call(self._w, 'size')) def itemcget(self, index, option): - """Return the resource value for an ITEM and an OPTION.""" + """Return the value of OPTION for item at INDEX.""" return self.tk.call( (self._w, 'itemcget') + (index, '-'+option)) def itemconfigure(self, index, cnf=None, **kw): - """Configure resources of an ITEM. + """Query or modify the configuration options of an item at INDEX. - The values for resources are specified as keyword arguments. - To get an overview about the allowed keyword arguments - call the method without arguments. - Valid resource names: background, bg, foreground, fg, - selectbackground, selectforeground.""" + Similar to configure() except that it applies to the specified item. + """ return self._configure(('itemconfigure', index), cnf, kw) itemconfig = itemconfigure @@ -3499,7 +3501,7 @@ class Menu(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct menu widget with the parent MASTER. - Valid resource names: activebackground, activeborderwidth, + Valid option names: activebackground, activeborderwidth, activeforeground, background, bd, bg, borderwidth, cursor, disabledforeground, fg, font, foreground, postcommand, relief, selectcolor, takefocus, tearoff, tearoffcommand, title, type.""" @@ -3580,11 +3582,15 @@ def delete(self, index1, index2=None): self.tk.call(self._w, 'delete', index1, index2) def entrycget(self, index, option): - """Return the resource value of a menu item for OPTION at INDEX.""" + """Return the value of OPTION for a menu item at INDEX.""" return self.tk.call(self._w, 'entrycget', index, '-' + option) def entryconfigure(self, index, cnf=None, **kw): - """Configure a menu item at INDEX.""" + """Query or modify the configuration options of a menu item at INDEX. + + Similar to configure() except that it applies to the specified + menu item. + """ return self._configure(('entryconfigure', index), cnf, kw) entryconfig = entryconfigure @@ -3642,7 +3648,7 @@ class Radiobutton(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a radiobutton widget with the parent MASTER. - Valid resource names: activebackground, activeforeground, anchor, + Valid option names: activebackground, activeforeground, anchor, background, bd, bg, bitmap, borderwidth, command, cursor, disabledforeground, fg, font, foreground, height, highlightbackground, highlightcolor, highlightthickness, image, @@ -3661,7 +3667,7 @@ def flash(self): self.tk.call(self._w, 'flash') def invoke(self): - """Toggle the button and invoke a command if given as resource.""" + """Toggle the button and invoke a command if given as option.""" return self.tk.call(self._w, 'invoke') def select(self): @@ -3675,7 +3681,7 @@ class Scale(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a scale widget with the parent MASTER. - Valid resource names: activebackground, background, bigincrement, bd, + Valid option names: activebackground, background, bigincrement, bd, bg, borderwidth, command, cursor, digits, fg, font, foreground, from, highlightbackground, highlightcolor, highlightthickness, label, length, orient, relief, repeatdelay, repeatinterval, resolution, @@ -3714,7 +3720,7 @@ class Scrollbar(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a scrollbar widget with the parent MASTER. - Valid resource names: activebackground, activerelief, + Valid option names: activebackground, activerelief, background, bd, bg, borderwidth, command, cursor, elementborderwidth, highlightbackground, highlightcolor, highlightthickness, jump, orient, @@ -3958,7 +3964,11 @@ def image_cget(self, index, option): return self.tk.call(self._w, "image", "cget", index, option) def image_configure(self, index, cnf=None, **kw): - """Configure an embedded image at INDEX.""" + """Query or modify the configuration options of an embedded image at INDEX. + + Similar to configure() except that it applies to the specified + embedded image. + """ return self._configure(('image', 'configure', index), cnf, kw) def image_create(self, index, cnf={}, **kw): @@ -4039,8 +4049,9 @@ def scan_dragto(self, x, y): self.tk.call(self._w, 'scan', 'dragto', x, y) def search(self, pattern, index, stopindex=None, - forwards=None, backwards=None, exact=None, - regexp=None, nocase=None, count=None, elide=None): + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, + elide=None, *, nolinestop=None, strictlimits=None): """Search PATTERN beginning from INDEX until STOPINDEX. Return the index of the first character of a match or an empty string.""" @@ -4052,12 +4063,39 @@ def search(self, pattern, index, stopindex=None, if nocase: args.append('-nocase') if elide: args.append('-elide') if count: args.append('-count'); args.append(count) + if nolinestop: args.append('-nolinestop') + if strictlimits: args.append('-strictlimits') if pattern and pattern[0] == '-': args.append('--') args.append(pattern) args.append(index) - if stopindex: args.append(stopindex) + if stopindex is not None: args.append(stopindex) return str(self.tk.call(tuple(args))) + def search_all(self, pattern, index, stopindex=None, *, + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, + elide=None, nolinestop=None, overlap=None, + strictlimits=None): + """Search all occurrences of PATTERN from INDEX to STOPINDEX. + Return a tuple of indices where matches begin.""" + args = [self._w, 'search', '-all'] + if forwards: args.append('-forwards') + if backwards: args.append('-backwards') + if exact: args.append('-exact') + if regexp: args.append('-regexp') + if nocase: args.append('-nocase') + if elide: args.append('-elide') + if count: args.append('-count'); args.append(count) + if nolinestop: args.append('-nolinestop') + if overlap: args.append('-overlap') + if strictlimits: args.append('-strictlimits') + if pattern and pattern[0] == '-': args.append('--') + args.append(pattern) + args.append(index) + if stopindex is not None: args.append(stopindex) + result = self.tk.call(tuple(args)) + return self.tk.splitlist(result) + def see(self, index): """Scroll such that the character at INDEX is visible.""" self.tk.call(self._w, 'see', index) @@ -4096,7 +4134,10 @@ def tag_cget(self, tagName, option): return self.tk.call(self._w, 'tag', 'cget', tagName, option) def tag_configure(self, tagName, cnf=None, **kw): - """Configure a tag TAGNAME.""" + """Query or modify the configuration options of a tag TAGNAME. + + Similar to configure() except that it applies to the specified tag. + """ return self._configure(('tag', 'configure', tagName), cnf, kw) tag_config = tag_configure @@ -4154,7 +4195,11 @@ def window_cget(self, index, option): return self.tk.call(self._w, 'window', 'cget', index, option) def window_configure(self, index, cnf=None, **kw): - """Configure an embedded window at INDEX.""" + """Query or modify the configuration options of an embedded window at INDEX. + + Similar to configure() except that it applies to the specified + embedded window. + """ return self._configure(('window', 'configure', index), cnf, kw) window_config = window_configure @@ -4194,7 +4239,7 @@ class OptionMenu(Menubutton): def __init__(self, master, variable, value, *values, **kwargs): """Construct an optionmenu widget with the parent MASTER, with - the resource textvariable set to VARIABLE, the initially selected + the option textvariable set to VARIABLE, the initially selected value VALUE, the other menu values VALUES and an additional keyword argument command.""" kw = {"borderwidth": 2, "textvariable": variable, @@ -4264,6 +4309,8 @@ def __setitem__(self, key, value): def __getitem__(self, key): return self.tk.call(self.name, 'configure', '-'+key) + __iter__ = None # prevent using __getitem__ for iteration + def configure(self, **kw): """Configure the image.""" res = () @@ -4296,7 +4343,7 @@ class PhotoImage(Image): def __init__(self, name=None, cnf={}, master=None, **kw): """Create an image with NAME. - Valid resource names: data, format, file, gamma, height, palette, + Valid option names: data, format, file, gamma, height, palette, width.""" Image.__init__(self, 'photo', name, cnf, master, **kw) @@ -4559,7 +4606,7 @@ class BitmapImage(Image): def __init__(self, name=None, cnf={}, master=None, **kw): """Create a bitmap with NAME. - Valid resource names: background, data, file, foreground, maskdata, maskfile.""" + Valid option names: background, data, file, foreground, maskdata, maskfile.""" Image.__init__(self, 'bitmap', name, cnf, master, **kw) @@ -4877,26 +4924,17 @@ def sash_place(self, index, x, y): return self.sash("place", index, x, y) def panecget(self, child, option): - """Query a management option for window. - - Option may be any value allowed by the paneconfigure subcommand - """ + """Return the value of option for a child window.""" return self.tk.call( (self._w, 'panecget') + (child, '-'+option)) def paneconfigure(self, tagOrId, cnf=None, **kw): - """Query or modify the management options for window. + """Query or modify the configuration options for a child window. - If no option is specified, returns a list describing all - of the available options for pathName. If option is - specified with no value, then the command returns a list - describing the one named option (this list will be identical - to the corresponding sublist of the value returned if no - option is specified). If one or more option-value pairs are - specified, then the command modifies the given widget - option(s) to have the given value(s); in this case the - command returns an empty string. The following options - are supported: + Similar to configure() except that it applies to the specified + window. + + The following options are supported: after window Insert the window after the window specified. window diff --git a/Lib/tkinter/font.py b/Lib/tkinter/font.py index 3e24e28ef58..896e910d69f 100644 --- a/Lib/tkinter/font.py +++ b/Lib/tkinter/font.py @@ -6,7 +6,6 @@ import itertools import tkinter -__version__ = "0.9" __all__ = ["NORMAL", "ROMAN", "BOLD", "ITALIC", "nametofont", "Font", "families", "names"] @@ -115,6 +114,8 @@ def __getitem__(self, key): def __setitem__(self, key, value): self.configure(**{key: value}) + __iter__ = None # prevent using __getitem__ for iteration + def __del__(self): try: if self.delete_font: @@ -198,6 +199,15 @@ def names(root=None): return root.tk.splitlist(root.tk.call("font", "names")) +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "0.9" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + # -------------------------------------------------------------------- # test stuff diff --git a/Lib/tkinter/scrolledtext.py b/Lib/tkinter/scrolledtext.py index 4f9a8815b61..8dcead5e319 100644 --- a/Lib/tkinter/scrolledtext.py +++ b/Lib/tkinter/scrolledtext.py @@ -23,7 +23,7 @@ def __init__(self, master=None, **kw): self.vbar = Scrollbar(self.frame) self.vbar.pack(side=RIGHT, fill=Y) - kw.update({'yscrollcommand': self.vbar.set}) + kw['yscrollcommand'] = self.vbar.set Text.__init__(self, self.frame, **kw) self.pack(side=LEFT, fill=BOTH, expand=True) self.vbar['command'] = self.yview diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index c0cf1e787fa..5c5ef11ae05 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -12,8 +12,6 @@ of the widgets appearance lies at Themes. """ -__version__ = "0.3.1" - __author__ = "Guilherme Polo <ggpolo@gmail.com>" __all__ = ["Button", "Checkbutton", "Combobox", "Entry", "Frame", "Label", @@ -1589,7 +1587,7 @@ class OptionMenu(Menubutton): def __init__(self, master, variable, default=None, *values, **kwargs): """Construct a themed OptionMenu widget with master as the parent, - the resource textvariable set to variable, the initially selected + the option textvariable set to variable, the initially selected value specified by the default parameter, the menu values given by *values and additional keywords. @@ -1648,3 +1646,12 @@ def destroy(self): except AttributeError: pass super().destroy() + + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "0.3.1" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 8d01fd7bce4..1f31258ce36 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -36,7 +36,7 @@ from token import EXACT_TOKEN_TYPES import _tokenize -cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +cookie_re = re.compile(br'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) import token @@ -86,7 +86,7 @@ def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. - _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr'] # if we add binary f-strings, add: ['fb', 'fbr'] result = {''} for prefix in _valid_string_prefixes: @@ -274,7 +274,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False - in_fstring = 0 + in_fstring_or_tstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -293,10 +293,10 @@ def compat(self, token, iterable): else: prevstring = False - if toknum == FSTRING_START: - in_fstring += 1 - elif toknum == FSTRING_END: - in_fstring -= 1 + if toknum in {FSTRING_START, TSTRING_START}: + in_fstring_or_tstring += 1 + elif toknum in {FSTRING_END, TSTRING_END}: + in_fstring_or_tstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -311,8 +311,8 @@ def compat(self, token, iterable): elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}: tokval = self.escape_brackets(tokval) - # Insert a space between two consecutive brackets if we are in an f-string - if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + # Insert a space between two consecutive brackets if we are in an f-string or t-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring_or_tstring: tokval = ' ' + tokval # Insert a space between two consecutive f-strings @@ -385,22 +385,23 @@ def read_or_stop(): except StopIteration: return b'' - def find_cookie(line): + def check(line, encoding): + # Check if the line matches the encoding. + if 0 in line: + raise SyntaxError("source code cannot contain null bytes") try: - # Decode as UTF-8. Either the line is an encoding declaration, - # in which case it should be pure ASCII, or it must be UTF-8 - # per default encoding. - line_string = line.decode('utf-8') + line.decode(encoding) except UnicodeDecodeError: msg = "invalid or missing encoding declaration" if filename is not None: msg = '{} for {!r}'.format(msg, filename) raise SyntaxError(msg) - match = cookie_re.match(line_string) + def find_cookie(line): + match = cookie_re.match(line) if not match: return None - encoding = _get_normal_name(match.group(1)) + encoding = _get_normal_name(match.group(1).decode()) try: codec = lookup(encoding) except LookupError: @@ -433,18 +434,23 @@ def find_cookie(line): encoding = find_cookie(first) if encoding: + check(first, encoding) return encoding, [first] if not blank_re.match(first): + check(first, default) return default, [first] second = read_or_stop() if not second: + check(first, default) return default, [first] encoding = find_cookie(second) if encoding: + check(first + second, encoding) return encoding, [first, second] + check(first + second, default) return default, [first, second] diff --git a/Lib/traceback.py b/Lib/traceback.py index 17b082eced6..c1052adeed2 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -14,6 +14,11 @@ from contextlib import suppress +try: + from _missing_stdlib_info import _MISSING_STDLIB_MODULE_MESSAGES +except ImportError: + _MISSING_STDLIB_MODULE_MESSAGES = {} + __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', 'format_tb', 'print_exc', 'format_exc', 'print_exception', @@ -137,8 +142,9 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ BUILTIN_EXCEPTION_LIMIT = object() -def _print_exception_bltin(exc, /): - file = sys.stderr if sys.stderr is not None else sys.__stderr__ +def _print_exception_bltin(exc, file=None, /): + if file is None: + file = sys.stderr if sys.stderr is not None else sys.__stderr__ colorize = _colorize.can_colorize(file=file) return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) @@ -205,9 +211,9 @@ def _safe_string(value, what, func=str): # -- -def print_exc(limit=None, file=None, chain=True): +def print_exc(limit=None, file=None, chain=True, **kwargs): """Shorthand for 'print_exception(sys.exception(), limit=limit, file=file, chain=chain)'.""" - print_exception(sys.exception(), limit=limit, file=file, chain=chain) + print_exception(sys.exception(), limit=limit, file=file, chain=chain, **kwargs) def format_exc(limit=None, chain=True): """Like print_exc() but return a string.""" @@ -287,6 +293,12 @@ class FrameSummary: of code that was running when the frame was captured. - :attr:`locals` Either None if locals were not supplied, or a dict mapping the name to the repr() of the variable. + - :attr:`end_lineno` The last line number of the source code for this frame. + By default, it is set to lineno and indexation starts from 1. + - :attr:`colno` The column number of the source code for this frame. + By default, it is None and indexation starts from 0. + - :attr:`end_colno` The last column number of the source code for this frame. + By default, it is None and indexation starts from 0. """ __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno', @@ -534,7 +546,7 @@ def format_frame_summary(self, frame_summary, **kwargs): colorize = kwargs.get("colorize", False) row = [] filename = frame_summary.filename - if frame_summary.filename.startswith("<stdin>-"): + if frame_summary.filename.startswith("<stdin-") and frame_summary.filename.endswith('>'): filename = "<stdin>" if colorize: theme = _colorize.get_theme(force_color=True).traceback @@ -1100,6 +1112,18 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) if suggestion: self._str += f". Did you mean: '{suggestion}'?" + elif exc_type and issubclass(exc_type, ModuleNotFoundError): + module_name = getattr(exc_value, "name", None) + if module_name in sys.stdlib_module_names: + message = _MISSING_STDLIB_MODULE_MESSAGES.get( + module_name, + f"Standard library module {module_name!r} was not found" + ) + self._str = message + elif sys.flags.no_site: + self._str += (". Site initialization is disabled, did you forget to " + + "add the site-packages directory to sys.path " + + "or to enable your virtual environment?") elif exc_type and issubclass(exc_type, (NameError, AttributeError)) and \ getattr(exc_value, "name", None) is not None: wrong_name = getattr(exc_value, "name", None) @@ -1310,13 +1334,21 @@ def _find_keyword_typos(self): lines = source.splitlines() error_code = lines[line -1 if line > 0 else 0:end_line] - error_code[0] = error_code[0][offset:] error_code = textwrap.dedent('\n'.join(error_code)) # Do not continue if the source is too large if len(error_code) > 1024: return + # If the original code doesn't raise SyntaxError, we can't validate + # that a keyword replacement actually fixes anything + try: + codeop.compile_command(error_code, symbol="exec", flags=codeop.PyCF_ONLY_AST) + except SyntaxError: + pass # Good - the original code has a syntax error we might fix + else: + return # Original code compiles or is incomplete - can't validate fixes + error_lines = error_code.splitlines() tokens = tokenize.generate_tokens(io.StringIO(error_code).readline) tokens_left_to_process = 10 @@ -1326,7 +1358,8 @@ def _find_keyword_typos(self): if token.type != tokenize.NAME: continue # Only consider NAME tokens on the same line as the error - if from_filename and token.start[0]+line != end_line+1: + the_end = end_line if line == 0 else end_line + 1 + if from_filename and token.start[0]+line != the_end: continue wrong_name = token.string if wrong_name in keyword.kwlist: @@ -1589,13 +1622,45 @@ def _substitution_cost(ch_a, ch_b): return _MOVE_COST +def _check_for_nested_attribute(obj, wrong_name, attrs): + """Check if any attribute of obj has the wrong_name as a nested attribute. + + Returns the first nested attribute suggestion found, or None. + Limited to checking 20 attributes. + Only considers non-descriptor attributes to avoid executing arbitrary code. + """ + # Check for nested attributes (only one level deep) + attrs_to_check = [x for x in attrs if not x.startswith('_')][:20] # Limit number of attributes to check + for attr_name in attrs_to_check: + with suppress(Exception): + # Check if attr_name is a descriptor - if so, skip it + attr_from_class = getattr(type(obj), attr_name, None) + if attr_from_class is not None and hasattr(attr_from_class, '__get__'): + continue # Skip descriptors to avoid executing arbitrary code + + # Safe to get the attribute since it's not a descriptor + attr_obj = getattr(obj, attr_name) + + # Check if the nested attribute exists and is not a descriptor + nested_attr_from_class = getattr(type(attr_obj), wrong_name, None) + + if hasattr(attr_obj, wrong_name): + return f"{attr_name}.{wrong_name}" + + return None + + def _compute_suggestion_error(exc_value, tb, wrong_name): if wrong_name is None or not isinstance(wrong_name, str): return None if isinstance(exc_value, AttributeError): obj = exc_value.obj try: - d = dir(obj) + try: + d = dir(obj) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(obj.__class__.__dict__.keys()) + list(obj.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) hide_underscored = (wrong_name[:1] != '_') if hide_underscored and tb is not None: while tb.tb_next is not None: @@ -1610,7 +1675,11 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) - d = dir(mod) + try: + d = dir(mod) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(mod.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) if wrong_name[:1] != '_': d = [x for x in d if x[:1] != '_'] except Exception: @@ -1628,6 +1697,7 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): + list(frame.f_globals) + list(frame.f_builtins) ) + d = [x for x in d if isinstance(x, str)] # Check first if we are in a method and the instance # has the wrong name as attribute @@ -1645,7 +1715,9 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): except ImportError: pass else: - return _suggestions._generate_suggestions(d, wrong_name) + suggestion = _suggestions._generate_suggestions(d, wrong_name) + if suggestion: + return suggestion # Compute closest match @@ -1670,6 +1742,14 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): if not suggestion or current_distance < best_distance: suggestion = possible_name best_distance = current_distance + + # If no direct attribute match found, check for nested attributes + if not suggestion and isinstance(exc_value, AttributeError): + with suppress(Exception): + nested_suggestion = _check_for_nested_attribute(exc_value.obj, wrong_name, d) + if nested_suggestion: + return nested_suggestion + return suggestion diff --git a/Lib/turtle.py b/Lib/turtle.py index e88981d298a..e5ce2c0a03c 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -1214,16 +1214,32 @@ def turtles(self): def bgcolor(self, *args): """Set or return backgroundcolor of the TurtleScreen. - Arguments (if given): a color string or three numbers - in the range 0..colormode or a 3-tuple of such numbers. + Four input formats are allowed: + - bgcolor() + Return the current background color as color specification + string or as a tuple (see example). May be used as input + to another color/pencolor/fillcolor/bgcolor call. + - bgcolor(colorstring) + Set the background color to colorstring, which is a Tk color + specification string, such as "red", "yellow", or "#33cc8c". + - bgcolor((r, g, b)) + Set the background color to the RGB color represented by + the tuple of r, g, and b. Each of r, g, and b must be in + the range 0..colormode, where colormode is either 1.0 or 255 + (see colormode()). + - bgcolor(r, g, b) + Set the background color to the RGB color represented by + r, g, and b. Each of r, g, and b must be in the range + 0..colormode. Example (for a TurtleScreen instance named screen): >>> screen.bgcolor("orange") >>> screen.bgcolor() 'orange' - >>> screen.bgcolor(0.5,0,0.5) + >>> colormode(255) + >>> screen.bgcolor('#800080') >>> screen.bgcolor() - '#800080' + (128.0, 0.0, 128.0) """ if args: color = self._colorstr(args) @@ -1678,7 +1694,7 @@ def forward(self, distance): Example (for a Turtle instance named turtle): >>> turtle.position() - (0.00, 0.00) + (0.00,0.00) >>> turtle.forward(25) >>> turtle.position() (25.00,0.00) @@ -1701,10 +1717,10 @@ def back(self, distance): Example (for a Turtle instance named turtle): >>> turtle.position() - (0.00, 0.00) + (0.00,0.00) >>> turtle.backward(30) >>> turtle.position() - (-30.00, 0.00) + (-30.00,0.00) """ self._go(-distance) @@ -1811,7 +1827,7 @@ def goto(self, x, y=None): Example (for a Turtle instance named turtle): >>> tp = turtle.pos() >>> tp - (0.00, 0.00) + (0.00,0.00) >>> turtle.setpos(60,30) >>> turtle.pos() (60.00,30.00) @@ -1891,7 +1907,7 @@ def distance(self, x, y=None): Example (for a Turtle instance named turtle): >>> turtle.pos() - (0.00, 0.00) + (0.00,0.00) >>> turtle.distance(30,40) 50.0 >>> pen = Turtle() @@ -2230,19 +2246,17 @@ def color(self, *args): Arguments: Several input formats are allowed. - They use 0, 1, 2, or 3 arguments as follows: - - color() - Return the current pencolor and the current fillcolor - as a pair of color specification strings as are returned - by pencolor and fillcolor. - color(colorstring), color((r,g,b)), color(r,g,b) - inputs as in pencolor, set both, fillcolor and pencolor, + They use 0 to 3 arguments as follows: + - color() + Return the current pencolor and the current fillcolor as + a pair of color specification strings or tuples as returned + by pencolor() and fillcolor(). + - color(colorstring), color((r,g,b)), color(r,g,b) + Inputs as in pencolor(), set both, fillcolor and pencolor, to the given value. - color(colorstring1, colorstring2), - color((r1,g1,b1), (r2,g2,b2)) - equivalent to pencolor(colorstring1) and fillcolor(colorstring2) - and analogously, if the other input format is used. + - color(colorstring1, colorstring2), color((r1,g1,b1), (r2,g2,b2)) + Equivalent to pencolor(colorstring1) and fillcolor(colorstring2) + and analogously if the other input format is used. If turtleshape is a polygon, outline and interior of that polygon is drawn with the newly set colors. @@ -2253,9 +2267,9 @@ def color(self, *args): >>> turtle.color() ('red', 'green') >>> colormode(255) - >>> color((40, 80, 120), (160, 200, 240)) + >>> color(('#285078', '#a0c8f0')) >>> color() - ('#285078', '#a0c8f0') + ((40.0, 80.0, 120.0), (160.0, 200.0, 240.0)) """ if args: l = len(args) @@ -2277,28 +2291,32 @@ def pencolor(self, *args): Arguments: Four input formats are allowed: - pencolor() - Return the current pencolor as color specification string, - possibly in hex-number format (see example). - May be used as input to another color/pencolor/fillcolor call. + Return the current pencolor as color specification string or + as a tuple (see example). May be used as input to another + color/pencolor/fillcolor/bgcolor call. - pencolor(colorstring) - s is a Tk color specification string, such as "red" or "yellow" + Set pencolor to colorstring, which is a Tk color + specification string, such as "red", "yellow", or "#33cc8c". - pencolor((r, g, b)) - *a tuple* of r, g, and b, which represent, an RGB color, - and each of r, g, and b are in the range 0..colormode, - where colormode is either 1.0 or 255 + Set pencolor to the RGB color represented by the tuple of + r, g, and b. Each of r, g, and b must be in the range + 0..colormode, where colormode is either 1.0 or 255 (see + colormode()). - pencolor(r, g, b) - r, g, and b represent an RGB color, and each of r, g, and b - are in the range 0..colormode + Set pencolor to the RGB color represented by r, g, and b. + Each of r, g, and b must be in the range 0..colormode. If turtleshape is a polygon, the outline of that polygon is drawn with the newly set pencolor. Example (for a Turtle instance named turtle): >>> turtle.pencolor('brown') - >>> tup = (0.2, 0.8, 0.55) - >>> turtle.pencolor(tup) >>> turtle.pencolor() - '#33cc8c' + 'brown' + >>> colormode(255) + >>> turtle.pencolor('#32c18f') + >>> turtle.pencolor() + (50.0, 193.0, 143.0) """ if args: color = self._colorstr(args) @@ -2315,26 +2333,31 @@ def fillcolor(self, *args): Four input formats are allowed: - fillcolor() Return the current fillcolor as color specification string, - possibly in hex-number format (see example). - May be used as input to another color/pencolor/fillcolor call. + possibly in tuple format (see example). May be used as + input to another color/pencolor/fillcolor/bgcolor call. - fillcolor(colorstring) - s is a Tk color specification string, such as "red" or "yellow" + Set fillcolor to colorstring, which is a Tk color + specification string, such as "red", "yellow", or "#33cc8c". - fillcolor((r, g, b)) - *a tuple* of r, g, and b, which represent, an RGB color, - and each of r, g, and b are in the range 0..colormode, - where colormode is either 1.0 or 255 + Set fillcolor to the RGB color represented by the tuple of + r, g, and b. Each of r, g, and b must be in the range + 0..colormode, where colormode is either 1.0 or 255 (see + colormode()). - fillcolor(r, g, b) - r, g, and b represent an RGB color, and each of r, g, and b - are in the range 0..colormode + Set fillcolor to the RGB color represented by r, g, and b. + Each of r, g, and b must be in the range 0..colormode. If turtleshape is a polygon, the interior of that polygon is drawn with the newly set fillcolor. Example (for a Turtle instance named turtle): >>> turtle.fillcolor('violet') - >>> col = turtle.pencolor() - >>> turtle.fillcolor(col) - >>> turtle.fillcolor(0, .5, 0) + >>> turtle.fillcolor() + 'violet' + >>> colormode(255) + >>> turtle.fillcolor('#ffffff') + >>> turtle.fillcolor() + (255.0, 255.0, 255.0) """ if args: color = self._colorstr(args) diff --git a/Lib/types.py b/Lib/types.py index 6efac339434..f96c75b46da 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -58,7 +58,10 @@ def _m(self): pass raise TypeError except TypeError as exc: TracebackType = type(exc.__traceback__) - FrameType = type(exc.__traceback__.tb_frame) + + _f = (lambda: sys._getframe())() + FrameType = type(_f) + FrameLocalsProxyType = type(_f.f_locals) GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) @@ -250,7 +253,6 @@ def deleter(self, fdel): class _GeneratorWrapper: - # TODO: Implement this in C. def __init__(self, gen): self.__wrapped = gen self.__isgen = gen.__class__ is GeneratorType @@ -305,7 +307,6 @@ def coroutine(func): # Check if 'func' is a generator function. # (0x20 == CO_GENERATOR) if co_flags & 0x20: - # TODO: Implement this in C. co = func.__code__ # 0x100 == CO_ITERABLE_COROUTINE func.__code__ = co.replace(co_flags=co.co_flags | 0x100) diff --git a/Lib/typing.py b/Lib/typing.py index 2baf655256d..eb0519986a8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -139,8 +139,8 @@ 'Never', 'NewType', 'no_type_check', - 'no_type_check_decorator', 'NoDefault', + 'NoExtraItems', 'NoReturn', 'NotRequired', 'overload', @@ -171,16 +171,16 @@ def __getattr__(self, attr): _lazy_annotationlib = _LazyAnnotationLib() -def _type_convert(arg, module=None, *, allow_special_forms=False): +def _type_convert(arg, module=None, *, allow_special_forms=False, owner=None): """For converting None to type(None), and strings to ForwardRef.""" if arg is None: return type(None) if isinstance(arg, str): - return _make_forward_ref(arg, module=module, is_class=allow_special_forms) + return _make_forward_ref(arg, module=module, is_class=allow_special_forms, owner=owner) return arg -def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False): +def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False, owner=None): """Check that the argument is a type, and return it (internal helper). As a special case, accept None and return type(None) instead. Also wrap strings @@ -198,7 +198,7 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if is_argument: invalid_generic_forms += (Final,) - arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms) + arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms, owner=owner) if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") @@ -256,16 +256,27 @@ def _type_repr(obj): return _lazy_annotationlib.type_repr(obj) -def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): +def _collect_type_parameters( + args, + *, + enforce_default_ordering: bool = True, + validate_all: bool = False, +): """Collect all type parameters in args in order of first appearance (lexicographic order). + Having an explicit `Generic` or `Protocol` base class determines + the exact parameter order. + For example:: >>> P = ParamSpec('P') >>> T = TypeVar('T') >>> _collect_type_parameters((T, Callable[P, T])) (~T, ~P) + >>> _collect_type_parameters((list[T], Generic[P, T])) + (~P, ~T) + """ # required type parameter cannot appear after parameter with default default_encountered = False @@ -297,6 +308,17 @@ def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): ' follows type parameter with a default') parameters.append(t) + elif ( + not validate_all + and isinstance(t, _GenericAlias) + and t.__origin__ in (Generic, Protocol) + ): + # If we see explicit `Generic[...]` or `Protocol[...]` base classes, + # we need to just copy them as-is. + # Unless `validate_all` is passed, in this case it means that + # we are doing a validation of `Generic` subclasses, + # then we collect all unique parameters to be able to inspect them. + parameters = t.__parameters__ else: if _is_unpacked_typevartuple(t): type_var_tuple_encountered = True @@ -407,6 +429,17 @@ def inner(*args, **kwds): return decorator +def _rebuild_generic_alias(alias: GenericAlias, args: tuple[object, ...]) -> GenericAlias: + is_unpacked = alias.__unpacked__ + if _should_unflatten_callable_args(alias, args): + t = alias.__origin__[(args[:-1], args[-1])] + else: + t = alias.__origin__[args] + if is_unpacked: + t = Unpack[t] + return t + + def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: import warnings @@ -420,59 +453,54 @@ def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) -class _Sentinel: - __slots__ = () - def __repr__(self): - return '<sentinel>' - - -_sentinel = _Sentinel() - - -def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), - format=None, owner=None): +def _eval_type(t, globalns, localns, type_params, *, recursive_guard=frozenset(), + format=None, owner=None, parent_fwdref=None, prefer_fwd_module=False): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ - if type_params is _sentinel: - _deprecation_warning_for_no_type_params_passed("typing._eval_type") - type_params = () if isinstance(t, _lazy_annotationlib.ForwardRef): # If the forward_ref has __forward_module__ set, evaluate() infers the globals # from the module, and it will probably pick better than the globals we have here. - if t.__forward_module__ is not None: + # We do this only for calls from get_type_hints() (which opts in through the + # prefer_fwd_module flag), so that the default behavior remains more straightforward. + if prefer_fwd_module and t.__forward_module__ is not None: globalns = None + # If there are type params on the owner, we need to add them back, because + # annotationlib won't. + if owner_type_params := getattr(owner, "__type_params__", None): + globalns = getattr( + sys.modules.get(t.__forward_module__, None), "__dict__", None + ) + if globalns is not None: + globalns = dict(globalns) + for type_param in owner_type_params: + globalns[type_param.__name__] = type_param return evaluate_forward_ref(t, globals=globalns, locals=localns, type_params=type_params, owner=owner, _recursive_guard=recursive_guard, format=format) if isinstance(t, (_GenericAlias, GenericAlias, Union)): if isinstance(t, GenericAlias): args = tuple( - _make_forward_ref(arg) if isinstance(arg, str) else arg + _make_forward_ref(arg, parent_fwdref=parent_fwdref) if isinstance(arg, str) else arg for arg in t.__args__ ) - is_unpacked = t.__unpacked__ - if _should_unflatten_callable_args(t, args): - t = t.__origin__[(args[:-1], args[-1])] - else: - t = t.__origin__[args] - if is_unpacked: - t = Unpack[t] + else: + args = t.__args__ ev_args = tuple( _eval_type( a, globalns, localns, type_params, recursive_guard=recursive_guard, - format=format, owner=owner, + format=format, owner=owner, prefer_fwd_module=prefer_fwd_module, ) - for a in t.__args__ + for a in args ) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): - return GenericAlias(t.__origin__, ev_args) + return _rebuild_generic_alias(t, ev_args) if isinstance(t, Union): return functools.reduce(operator.or_, ev_args) else: @@ -936,7 +964,12 @@ def run(arg: Child | Unrelated): return _GenericAlias(self, (item,)) -def _make_forward_ref(code, **kwargs): +def _make_forward_ref(code, *, parent_fwdref=None, **kwargs): + if parent_fwdref is not None: + if parent_fwdref.__forward_module__ is not None: + kwargs['module'] = parent_fwdref.__forward_module__ + if parent_fwdref.__owner__ is not None: + kwargs['owner'] = parent_fwdref.__owner__ forward_ref = _lazy_annotationlib.ForwardRef(code, **kwargs) # For compatibility, eagerly compile the forwardref's code. forward_ref.__forward_code__ @@ -956,12 +989,8 @@ def evaluate_forward_ref( """Evaluate a forward reference as a type hint. This is similar to calling the ForwardRef.evaluate() method, - but unlike that method, evaluate_forward_ref() also: - - * Recursively evaluates forward references nested within the type hint. - * Rejects certain objects that are not valid type hints. - * Replaces type hints that evaluate to None with types.NoneType. - * Supports the *FORWARDREF* and *STRING* formats. + but unlike that method, evaluate_forward_ref() also + recursively evaluates forward references nested within the type hint. *forward_ref* must be an instance of ForwardRef. *owner*, if given, should be the object that holds the annotations that the forward reference @@ -981,35 +1010,39 @@ def evaluate_forward_ref( if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref - try: - value = forward_ref.evaluate(globals=globals, locals=locals, - type_params=type_params, owner=owner) - except NameError: - if format == _lazy_annotationlib.Format.FORWARDREF: - return forward_ref - else: - raise + if format is None: + format = _lazy_annotationlib.Format.VALUE + value = forward_ref.evaluate(globals=globals, locals=locals, + type_params=type_params, owner=owner, format=format) - type_ = _type_check( - value, - "Forward references must evaluate to types.", - is_argument=forward_ref.__forward_is_argument__, - allow_special_forms=forward_ref.__forward_is_class__, - ) + if (isinstance(value, _lazy_annotationlib.ForwardRef) + and format == _lazy_annotationlib.Format.FORWARDREF): + return value + + if isinstance(value, str): + value = _make_forward_ref(value, module=forward_ref.__forward_module__, + owner=owner or forward_ref.__owner__, + is_argument=forward_ref.__forward_is_argument__, + is_class=forward_ref.__forward_is_class__) + if owner is None: + owner = forward_ref.__owner__ return _eval_type( - type_, + value, globals, locals, type_params, recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, format=format, owner=owner, + parent_fwdref=forward_ref, ) def _is_unpacked_typevartuple(x: Any) -> bool: + # Need to check 'is True' here + # See: https://github.com/python/cpython/issues/137706 return ((not isinstance(x, type)) and - getattr(x, '__typing_is_unpacked_typevartuple__', False)) + getattr(x, '__typing_is_unpacked_typevartuple__', False) is True) def _is_typevar_like(x: Any) -> bool: @@ -1079,7 +1112,7 @@ def _paramspec_prepare_subst(self, alias, args): params = alias.__parameters__ i = params.index(self) if i == len(args) and self.has_default(): - args = [*args, self.__default__] + args = (*args, self.__default__) if i >= len(args): raise TypeError(f"Too few arguments for {alias}") # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. @@ -1124,14 +1157,26 @@ def _generic_class_getitem(cls, args): f"Parameters to {cls.__name__}[...] must all be unique") else: # Subscripting a regular Generic subclass. - for param in cls.__parameters__: + try: + parameters = cls.__parameters__ + except AttributeError as e: + init_subclass = getattr(cls, '__init_subclass__', None) + if init_subclass not in {None, Generic.__init_subclass__}: + e.add_note( + f"Note: this exception may have been caused by " + f"{init_subclass.__qualname__!r} (or the " + f"'__init_subclass__' method on a superclass) not " + f"calling 'super().__init_subclass__()'" + ) + raise + for param in parameters: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: args = prepare(cls, args) _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, args): + for param, new_arg in zip(parameters, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: @@ -1153,20 +1198,22 @@ def _generic_init_subclass(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_type_parameters(cls.__orig_bases__) + tvars = _collect_type_parameters(cls.__orig_bases__, validate_all=True) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. # Also check for and reject plain Generic, # and reject multiple Generic[...]. gvars = None + basename = None for base in cls.__orig_bases__: if (isinstance(base, _GenericAlias) and - base.__origin__ is Generic): + base.__origin__ in (Generic, Protocol)): if gvars is not None: raise TypeError( "Cannot inherit from Generic[...] multiple times.") gvars = base.__parameters__ + basename = base.__origin__.__name__ if gvars is not None: tvarset = set(tvars) gvarset = set(gvars) @@ -1174,7 +1221,7 @@ def _generic_init_subclass(cls, *args, **kwargs): s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) s_args = ', '.join(str(g) for g in gvars) raise TypeError(f"Some type variables ({s_vars}) are" - f" not listed in Generic[{s_args}]") + f" not listed in {basename}[{s_args}]") tvars = gvars cls.__parameters__ = tuple(tvars) @@ -1649,6 +1696,9 @@ def __eq__(self, other): return True return NotImplemented + def __hash__(self): + return hash(Union) + class _UnionGenericAlias(metaclass=_UnionGenericAliasMeta): """Compatibility hack. @@ -1860,7 +1910,9 @@ def _allow_reckless_class_checks(depth=2): The abc and functools modules indiscriminately call isinstance() and issubclass() on the whole MRO of a user class, which may contain protocols. """ - return _caller(depth) in {'abc', 'functools', None} + # gh-136047: When `_abc` module is not available, `_py_abc` is required to + # allow `_py_abc.ABCMeta` fallback. + return _caller(depth) in {'abc', '_py_abc', 'functools', None} _PROTO_ALLOWLIST = { @@ -2334,13 +2386,16 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, # *base_globals* first rather than *base_locals*. # This only affects ForwardRefs. base_globals, base_locals = base_locals, base_globals + type_params = base.__type_params__ + base_globals, base_locals = _add_type_params_to_scope( + type_params, base_globals, base_locals, True) for name, value in ann.items(): - if value is None: - value = type(None) if isinstance(value, str): value = _make_forward_ref(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals, base.__type_params__, - format=format, owner=obj) + value = _eval_type(value, base_globals, base_locals, (), + format=format, owner=obj, prefer_fwd_module=True) + if value is None: + value = type(None) hints[name] = value if include_extras or format == Format.STRING: return hints @@ -2373,9 +2428,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, elif localns is None: localns = globalns type_params = getattr(obj, "__type_params__", ()) + globalns, localns = _add_type_params_to_scope(type_params, globalns, localns, False) for name, value in hints.items(): - if value is None: - value = type(None) if isinstance(value, str): # class-level forward refs were handled above, this must be either # a module-level annotation or a function argument annotation @@ -2384,10 +2438,27 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns, type_params, format=format, owner=obj) + value = _eval_type(value, globalns, localns, (), format=format, owner=obj, prefer_fwd_module=True) + if value is None: + value = type(None) + hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} +# Add type parameters to the globals and locals scope. This is needed for +# compatibility. +def _add_type_params_to_scope(type_params, globalns, localns, is_class): + if not type_params: + return globalns, localns + globalns = dict(globalns) + localns = dict(localns) + for param in type_params: + if not is_class or param.__name__ not in globalns: + globalns[param.__name__] = param + localns.pop(param.__name__, None) + return globalns, localns + + def _strip_annotations(t): """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): @@ -2403,7 +2474,7 @@ def _strip_annotations(t): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: return t - return GenericAlias(t.__origin__, stripped_args) + return _rebuild_generic_alias(t, stripped_args) if isinstance(t, Union): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: @@ -2551,23 +2622,6 @@ def no_type_check(arg): return arg -def no_type_check_decorator(decorator): - """Decorator to give another decorator the @no_type_check effect. - - This wraps the decorator with something that wraps the decorated - function in @no_type_check. - """ - import warnings - warnings._deprecated("typing.no_type_check_decorator", remove=(3, 15)) - @functools.wraps(decorator) - def wrapped_decorator(*args, **kwds): - func = decorator(*args, **kwds) - func = no_type_check(func) - return func - - return wrapped_decorator - - def _overload_dummy(*args, **kwds): """Helper for @overload to raise when called.""" raise NotImplementedError( @@ -2968,7 +3022,7 @@ def annotate(format): return nm_tpl -def NamedTuple(typename, fields=_sentinel, /, **kwargs): +def NamedTuple(typename, fields, /): """Typed version of namedtuple. Usage:: @@ -2988,48 +3042,9 @@ class Employee(NamedTuple): Employee = NamedTuple('Employee', [('name', str), ('id', int)]) """ - if fields is _sentinel: - if kwargs: - deprecated_thing = "Creating NamedTuple classes using keyword arguments" - deprecation_msg = ( - "{name} is deprecated and will be disallowed in Python {remove}. " - "Use the class-based or functional syntax instead." - ) - else: - deprecated_thing = "Failing to pass a value for the 'fields' parameter" - example = f"`{typename} = NamedTuple({typename!r}, [])`" - deprecation_msg = ( - "{name} is deprecated and will be disallowed in Python {remove}. " - "To create a NamedTuple class with 0 fields " - "using the functional syntax, " - "pass an empty list, e.g. " - ) + example + "." - elif fields is None: - if kwargs: - raise TypeError( - "Cannot pass `None` as the 'fields' parameter " - "and also specify fields using keyword arguments" - ) - else: - deprecated_thing = "Passing `None` as the 'fields' parameter" - example = f"`{typename} = NamedTuple({typename!r}, [])`" - deprecation_msg = ( - "{name} is deprecated and will be disallowed in Python {remove}. " - "To create a NamedTuple class with 0 fields " - "using the functional syntax, " - "pass an empty list, e.g. " - ) + example + "." - elif kwargs: - raise TypeError("Either list of fields or keywords" - " can be provided to NamedTuple, not both") - if fields is _sentinel or fields is None: - import warnings - warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) - fields = kwargs.items() types = {n: _type_check(t, f"field {n} annotation must be a type") for n, t in fields} field_names = [n for n, _ in fields] - nt = _make_nmtuple(typename, field_names, _make_eager_annotate(types), module=_caller()) nt.__orig_bases__ = (NamedTuple,) return nt @@ -3043,6 +3058,33 @@ def _namedtuple_mro_entries(bases): NamedTuple.__mro_entries__ = _namedtuple_mro_entries +class _SingletonMeta(type): + def __setattr__(cls, attr, value): + # TypeError is consistent with the behavior of NoneType + raise TypeError( + f"cannot set {attr!r} attribute of immutable type {cls.__name__!r}" + ) + + +class _NoExtraItemsType(metaclass=_SingletonMeta): + """The type of the NoExtraItems singleton.""" + + __slots__ = () + + def __new__(cls): + return globals().get("NoExtraItems") or object.__new__(cls) + + def __repr__(self): + return 'typing.NoExtraItems' + + def __reduce__(self): + return 'NoExtraItems' + +NoExtraItems = _NoExtraItemsType() +del _NoExtraItemsType +del _SingletonMeta + + def _get_typeddict_qualifiers(annotation_type): while True: annotation_origin = get_origin(annotation_type) @@ -3066,7 +3108,8 @@ def _get_typeddict_qualifiers(annotation_type): class _TypedDictMeta(type): - def __new__(cls, name, bases, ns, total=True): + def __new__(cls, name, bases, ns, total=True, closed=None, + extra_items=NoExtraItems): """Create a new typed dict class object. This method is called when TypedDict is subclassed, @@ -3078,20 +3121,24 @@ def __new__(cls, name, bases, ns, total=True): if type(base) is not _TypedDictMeta and base is not Generic: raise TypeError('cannot inherit from both a TypedDict type ' 'and a non-TypedDict base class') + if closed is not None and extra_items is not NoExtraItems: + raise TypeError(f"Cannot combine closed={closed!r} and extra_items") if any(issubclass(b, Generic) for b in bases): generic_base = (Generic,) else: generic_base = () + ns_annotations = ns.pop('__annotations__', None) + tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns) if not hasattr(tp_dict, '__orig_bases__'): tp_dict.__orig_bases__ = bases - if "__annotations__" in ns: + if ns_annotations is not None: own_annotate = None - own_annotations = ns["__annotations__"] + own_annotations = ns_annotations elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: own_annotations = _lazy_annotationlib.call_annotate_function( own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict @@ -3101,7 +3148,7 @@ def __new__(cls, name, bases, ns, total=True): own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" own_checked_annotations = { - n: _type_check(tp, msg, module=tp_dict.__module__) + n: _type_check(tp, msg, owner=tp_dict, module=tp_dict.__module__) for n, tp in own_annotations.items() } required_keys = set() @@ -3162,7 +3209,7 @@ def __annotate__(format): if base_annotate is None: continue base_annos = _lazy_annotationlib.call_annotate_function( - base.__annotate__, format, owner=base) + base_annotate, format, owner=base) annos.update(base_annos) if own_annotate is not None: own = _lazy_annotationlib.call_annotate_function( @@ -3187,6 +3234,8 @@ def __annotate__(format): tp_dict.__readonly_keys__ = frozenset(readonly_keys) tp_dict.__mutable_keys__ = frozenset(mutable_keys) tp_dict.__total__ = total + tp_dict.__closed__ = closed + tp_dict.__extra_items__ = extra_items return tp_dict __call__ = dict # static method @@ -3198,7 +3247,8 @@ def __subclasscheck__(cls, other): __instancecheck__ = __subclasscheck__ -def TypedDict(typename, fields=_sentinel, /, *, total=True): +def TypedDict(typename, fields, /, *, total=True, closed=None, + extra_items=NoExtraItems): """A simple typed namespace. At runtime it is equivalent to a plain dict. TypedDict creates a dictionary type such that a type checker will expect all @@ -3252,32 +3302,41 @@ class DatabaseUser(TypedDict): id: ReadOnly[int] # the "id" key must not be modified username: str # the "username" key can be changed + The closed argument controls whether the TypedDict allows additional + non-required items during inheritance and assignability checks. + If closed=True, the TypedDict does not allow additional items:: + + Point2D = TypedDict('Point2D', {'x': int, 'y': int}, closed=True) + class Point3D(Point2D): + z: int # Type checker error + + Passing closed=False explicitly requests TypedDict's default open behavior. + If closed is not provided, the behavior is inherited from the superclass. + A type checker is only expected to support a literal False or True as the + value of the closed argument. + + The extra_items argument can instead be used to specify the assignable type + of unknown non-required keys:: + + Point2D = TypedDict('Point2D', {'x': int, 'y': int}, extra_items=int) + class Point3D(Point2D): + z: int # OK + label: str # Type checker error + + The extra_items argument is also inherited through subclassing. It is unset + by default, and it may not be used with the closed argument at the same + time. + + See PEP 728 for more information about closed and extra_items. """ - if fields is _sentinel or fields is None: - import warnings - - if fields is _sentinel: - deprecated_thing = "Failing to pass a value for the 'fields' parameter" - else: - deprecated_thing = "Passing `None` as the 'fields' parameter" - - example = f"`{typename} = TypedDict({typename!r}, {{{{}}}})`" - deprecation_msg = ( - "{name} is deprecated and will be disallowed in Python {remove}. " - "To create a TypedDict class with 0 fields " - "using the functional syntax, " - "pass an empty dictionary, e.g. " - ) + example + "." - warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) - fields = {} - ns = {'__annotations__': dict(fields)} module = _caller() if module is not None: # Setting correct module is necessary to make typed dict classes pickleable. ns['__module__'] = module - td = _TypedDictMeta(typename, (), ns, total=total) + td = _TypedDictMeta(typename, (), ns, total=total, closed=closed, + extra_items=extra_items) td.__orig_bases__ = (TypedDict,) return td @@ -3776,6 +3835,48 @@ def __getattr__(attr): ) warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) obj = _collect_type_parameters + elif attr == "ByteString": + import warnings + + warnings._deprecated( + "typing.ByteString", + message=( + "{name!r} and 'collections.abc.ByteString' are deprecated " + "and slated for removal in Python {remove}" + ), + remove=(3, 17) + ) + + class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): + def __init__( + self, origin, nparams, *, removal_version, inst=True, name=None + ): + super().__init__(origin, nparams, inst=inst, name=name) + self._removal_version = removal_version + + def __instancecheck__(self, inst): + import warnings + warnings._deprecated( + f"{self.__module__}.{self._name}", remove=self._removal_version + ) + return super().__instancecheck__(inst) + + def __subclasscheck__(self, cls): + import warnings + warnings._deprecated( + f"{self.__module__}.{self._name}", remove=self._removal_version + ) + return super().__subclasscheck__(cls) + + with warnings.catch_warnings( + action="ignore", category=DeprecationWarning + ): + # Not generic + ByteString = globals()["ByteString"] = _DeprecatedGenericAlias( + collections.abc.ByteString, 0, removal_version=(3, 17) + ) + + return ByteString else: raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") globals()[attr] = obj diff --git a/Lib/unittest/_log.py b/Lib/unittest/_log.py index 94868e5bb95..3d69385ea24 100644 --- a/Lib/unittest/_log.py +++ b/Lib/unittest/_log.py @@ -30,7 +30,7 @@ class _AssertLogsContext(_BaseTestCaseContext): LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s" - def __init__(self, test_case, logger_name, level, no_logs): + def __init__(self, test_case, logger_name, level, no_logs, formatter=None): _BaseTestCaseContext.__init__(self, test_case) self.logger_name = logger_name if level: @@ -39,13 +39,14 @@ def __init__(self, test_case, logger_name, level, no_logs): self.level = logging.INFO self.msg = None self.no_logs = no_logs + self.formatter = formatter def __enter__(self): if isinstance(self.logger_name, logging.Logger): logger = self.logger = self.logger_name else: logger = self.logger = logging.getLogger(self.logger_name) - formatter = logging.Formatter(self.LOGGING_FORMAT) + formatter = self.formatter or logging.Formatter(self.LOGGING_FORMAT) handler = _CapturingHandler() handler.setLevel(self.level) handler.setFormatter(formatter) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 884fc1b21f6..eba50839cd3 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -149,9 +149,7 @@ def doModuleCleanups(): except Exception as exc: exceptions.append(exc) if exceptions: - # Swallows all but first exception. If a multi-exception handler - # gets written we should use that here instead. - raise exceptions[0] + raise ExceptionGroup('module cleanup failed', exceptions) def skip(reason): @@ -851,7 +849,7 @@ def _assertNotWarns(self, expected_warning, *args, **kwargs): context = _AssertNotWarnsContext(expected_warning, self) return context.handle('_assertNotWarns', args, kwargs) - def assertLogs(self, logger=None, level=None): + def assertLogs(self, logger=None, level=None, formatter=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to INFO and *logger* defaults to the root logger. @@ -863,6 +861,8 @@ def assertLogs(self, logger=None, level=None): `records` attribute will be a list of the corresponding LogRecord objects. + Optionally supply `formatter` to control how messages are formatted. + Example:: with self.assertLogs('foo', level='INFO') as cm: @@ -873,7 +873,7 @@ def assertLogs(self, logger=None, level=None): """ # Lazy import to avoid importing logging if it is not needed. from ._log import _AssertLogsContext - return _AssertLogsContext(self, logger, level, no_logs=False) + return _AssertLogsContext(self, logger, level, no_logs=False, formatter=formatter) def assertNoLogs(self, logger=None, level=None): """ Fail unless no log messages of level *level* or higher are emitted diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 6fd949581f3..be99d93c78c 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -269,12 +269,12 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - if self.result.testsRun == 0 and len(self.result.skipped) == 0: - sys.exit(_NO_TESTS_EXITCODE) - elif self.result.wasSuccessful(): - sys.exit(0) - else: + if not self.result.wasSuccessful(): sys.exit(1) + elif self.result.testsRun == 0 and len(self.result.skipped) == 0: + sys.exit(_NO_TESTS_EXITCODE) + else: + sys.exit(0) main = TestProgram diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 55cb4b1f6af..0bb67506553 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -569,6 +569,11 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, __dict__['_mock_methods'] = spec __dict__['_spec_asyncs'] = _spec_asyncs + def _mock_extend_spec_methods(self, spec_methods): + methods = self.__dict__.get('_mock_methods') or [] + methods.extend(spec_methods) + self.__dict__['_mock_methods'] = methods + def __get_return_value(self): ret = self._mock_return_value if self._mock_delegate is not None: @@ -695,7 +700,7 @@ def __getattr__(self, name): if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')) or name in _ATTRIB_DENY_LIST: raise AttributeError( f"{name!r} is not a valid assertion. Use a spec " - f"for the mock if {name!r} is meant to be an attribute.") + f"for the mock if {name!r} is meant to be an attribute") with NonCallableMock._lock: result = self._mock_children.get(name) @@ -981,7 +986,7 @@ def _error_message(): def assert_called_once_with(self, /, *args, **kwargs): - """assert that the mock was called exactly once and that that call was + """assert that the mock was called exactly once and that call was with the specified arguments.""" if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times.%s" @@ -2766,14 +2771,16 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, raise InvalidSpecError(f'Cannot autospec a Mock object. ' f'[object={spec!r}]') is_async_func = _is_async_func(spec) + _kwargs = {'spec': spec} entries = [(entry, _missing) for entry in dir(spec)] if is_type and instance and is_dataclass(spec): + is_dataclass_spec = True dataclass_fields = fields(spec) entries.extend((f.name, f.type) for f in dataclass_fields) - _kwargs = {'spec': [f.name for f in dataclass_fields]} + dataclass_spec_list = [f.name for f in dataclass_fields] else: - _kwargs = {'spec': spec} + is_dataclass_spec = False if spec_set: _kwargs = {'spec_set': spec} @@ -2810,6 +2817,8 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name, name=_name, **_kwargs) + if is_dataclass_spec: + mock._mock_extend_spec_methods(dataclass_spec_list) if isinstance(spec, FunctionTypes): # should only happen at the top level because we don't diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py index 6f45b6fe5f6..ae9ca2d615d 100644 --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -223,6 +223,11 @@ def _handleModuleFixture(self, test, result): if result._moduleSetUpFailed: try: case.doModuleCleanups() + except ExceptionGroup as eg: + for e in eg.exceptions: + self._createClassOrModuleLevelException(result, e, + 'setUpModule', + currentModule) except Exception as e: self._createClassOrModuleLevelException(result, e, 'setUpModule', @@ -235,15 +240,15 @@ def _createClassOrModuleLevelException(self, result, exc, method_name, errorName = f'{method_name} ({parent})' self._addClassOrModuleLevelException(result, exc, errorName, info) - def _addClassOrModuleLevelException(self, result, exception, errorName, + def _addClassOrModuleLevelException(self, result, exc, errorName, info=None): error = _ErrorHolder(errorName) addSkip = getattr(result, 'addSkip', None) - if addSkip is not None and isinstance(exception, case.SkipTest): - addSkip(error, str(exception)) + if addSkip is not None and isinstance(exc, case.SkipTest): + addSkip(error, str(exc)) else: if not info: - result.addError(error, sys.exc_info()) + result.addError(error, (type(exc), exc, exc.__traceback__)) else: result.addError(error, info) @@ -273,6 +278,13 @@ def _handleModuleTearDown(self, result): previousModule) try: case.doModuleCleanups() + except ExceptionGroup as eg: + if isinstance(result, _DebugResult): + raise + for e in eg.exceptions: + self._createClassOrModuleLevelException(result, e, + 'tearDownModule', + previousModule) except Exception as e: if isinstance(result, _DebugResult): raise diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index a9cd1ecadd6..beb32d0df48 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -18,7 +18,7 @@ class URLError(OSError): # URLError is a sub-type of OSError, but it doesn't share any of - # the implementation. need to override __init__ and __str__. + # the implementation. It overrides __init__ and __str__. # It sets self.args for compatibility with other OSError # subclasses, but args doesn't have the typical format with errno in # slot 0 and strerror in slot 1. This may be better than nothing. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 67d9bbea0d3..c7946d76253 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -97,11 +97,9 @@ def clear_cache(): _byte_quoter_factory.cache_clear() # Helpers for bytes handling -# For 3.2, we deliberately require applications that -# handle improperly quoted URLs to do their own -# decoding and encoding. If valid use cases are -# presented, we may relax this by using latin-1 -# decoding internally for 3.3 +# We deliberately require applications that +# handle improperly quoted URLs to do their +# own decoding and encoding. _implicit_encoding = 'ascii' _implicit_errors = 'strict' diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 41dc5d7b35d..566b8087aec 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1535,6 +1535,7 @@ def ftp_open(self, req): dirs, file = dirs[:-1], dirs[-1] if dirs and not dirs[0]: dirs = dirs[1:] + fw = None try: fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' @@ -1552,8 +1553,12 @@ def ftp_open(self, req): headers += "Content-length: %d\n" % retrlen headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) - except ftplib.all_errors as exp: - raise URLError(f"ftp error: {exp}") from exp + except Exception as exp: + if fw is not None and not fw.keepalive: + fw.close() + if isinstance(exp, ftplib.all_errors): + raise URLError(f"ftp error: {exp}") from exp + raise def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, @@ -1577,14 +1582,15 @@ def setMaxConns(self, m): def connect_ftp(self, user, passwd, host, port, dirs, timeout): key = user, host, port, '/'.join(dirs), timeout - if key in self.cache: - self.timeout[key] = time.time() + self.delay - else: - self.cache[key] = ftpwrapper(user, passwd, host, port, - dirs, timeout) - self.timeout[key] = time.time() + self.delay + conn = self.cache.get(key) + if conn is None or not conn.keepalive: + if conn is not None: + conn.close() + conn = self.cache[key] = ftpwrapper(user, passwd, host, port, + dirs, timeout) + self.timeout[key] = time.time() + self.delay self.check_cache() - return self.cache[key] + return conn def check_cache(self): # first check for old ones @@ -1654,13 +1660,16 @@ def url2pathname(url, *, require_scheme=False, resolve_host=False): The URL authority may be resolved with gethostbyname() if *resolve_host* is set to true. """ - if require_scheme: - scheme, url = _splittype(url) - if scheme != 'file': - raise URLError("URL is missing a 'file:' scheme") - authority, url = _splithost(url) + if not require_scheme: + url = 'file:' + url + scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment. + if scheme != 'file': + raise URLError("URL is missing a 'file:' scheme") if os.name == 'nt': - if not _is_local_authority(authority, resolve_host): + if authority[1:2] == ':': + # e.g. file://c:/file.txt + url = authority + url + elif not _is_local_authority(authority, resolve_host): # e.g. file://server/share/file.txt url = '//' + authority + url elif url[:3] == '///': diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 409f2b2e48d..4009fd6b58f 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -11,6 +11,7 @@ """ import collections +import re import urllib.error import urllib.parse import urllib.request @@ -20,6 +21,19 @@ RequestRate = collections.namedtuple("RequestRate", "requests seconds") +def normalize(path): + unquoted = urllib.parse.unquote(path, errors='surrogateescape') + return urllib.parse.quote(unquoted, errors='surrogateescape') + +def normalize_path(path): + path, sep, query = path.partition('?') + path = normalize(path) + if sep: + query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + + class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -55,7 +69,7 @@ def modified(self): def set_url(self, url): """Sets the URL referring to a robots.txt file.""" self.url = url - self.host, self.path = urllib.parse.urlparse(url)[1:3] + self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" @@ -69,7 +83,7 @@ def read(self): err.close() else: raw = f.read() - self.parse(raw.decode("utf-8").splitlines()) + self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) def _add_entry(self, entry): if "*" in entry.useragents: @@ -113,7 +127,7 @@ def parse(self, lines): line = line.split(':', 1) if len(line) == 2: line[0] = line[0].strip().lower() - line[1] = urllib.parse.unquote(line[1].strip()) + line[1] = line[1].strip() if line[0] == "user-agent": if state == 2: self._add_entry(entry) @@ -167,10 +181,11 @@ def can_fetch(self, useragent, url): return False # search for given user agent matches # the first match counts - parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url)) - url = urllib.parse.urlunparse(('','',parsed_url.path, - parsed_url.params,parsed_url.query, parsed_url.fragment)) - url = urllib.parse.quote(url) + # TODO: The private API is used in order to preserve an empty query. + # This is temporary until the public API starts supporting this feature. + parsed_url = urllib.parse._urlsplit(url, '') + url = urllib.parse._urlunsplit(None, None, *parsed_url[2:]) + url = normalize_path(url) if not url: url = "/" for entry in self.entries: @@ -213,7 +228,6 @@ def __str__(self): entries = entries + [self.default_entry] return '\n\n'.join(map(str, entries)) - class RuleLine: """A rule line is a single "Allow:" (allowance==True) or "Disallow:" (allowance==False) followed by a path.""" @@ -221,8 +235,7 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True - path = urllib.parse.urlunparse(urllib.parse.urlparse(path)) - self.path = urllib.parse.quote(path) + self.path = normalize_path(path) self.allowance = allowance def applies_to(self, filename): @@ -268,7 +281,7 @@ def applies_to(self, useragent): def allowance(self, filename): """Preconditions: - our agent applies to this entry - - filename is URL decoded""" + - filename is URL encoded""" for line in self.rulelines: if line.applies_to(filename): return line.allowance diff --git a/Lib/uuid.py b/Lib/uuid.py index 036ffebf67a..c0150a59d7c 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -633,39 +633,43 @@ def _netstat_getnode(): try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) + _has_stable_extractable_node = _uuid.has_stable_extractable_node _UuidCreate = getattr(_uuid, "UuidCreate", None) except ImportError: _uuid = None _generate_time_safe = None + _has_stable_extractable_node = False _UuidCreate = None def _unix_getnode(): """Get the hardware address on Unix using the _uuid extension module.""" - if _generate_time_safe: + if _generate_time_safe and _has_stable_extractable_node: uuid_time, _ = _generate_time_safe() return UUID(bytes=uuid_time).node def _windll_getnode(): """Get the hardware address on Windows using the _uuid extension module.""" - if _UuidCreate: + if _UuidCreate and _has_stable_extractable_node: uuid_bytes = _UuidCreate() return UUID(bytes_le=uuid_bytes).node def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or - # pseudo-randomly generated value may be used; see Section 4.5. The - # multicast bit must be set in such addresses, in order that they will - # never conflict with addresses obtained from network cards." + # RFC 9562, §6.10-3 says that + # + # Implementations MAY elect to obtain a 48-bit cryptographic-quality + # random number as per Section 6.9 to use as the Node ID. [...] [and] + # implementations MUST set the least significant bit of the first octet + # of the Node ID to 1. This bit is the unicast or multicast bit, which + # will never be set in IEEE 802 addresses obtained from network cards. # # The "multicast bit" of a MAC address is defined to be "the least # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/w/index.php?title=MAC_address&oldid=1128764812#Universal_vs._local_(U/L_bit) - import random - return random.getrandbits(48) | (1 << 40) + return int.from_bytes(os.urandom(6)) | (1 << 40) # _OS_GETTERS, when known, are targeted for a specific OS or platform. @@ -733,6 +737,7 @@ def uuid1(node=None, clock_seq=None): is_safe = SafeUUID(safely_generated) except ValueError: is_safe = SafeUUID.unknown + # The version field is assumed to be handled by _generate_time_safe(). return UUID(bytes=uuid_time, is_safe=is_safe) global _last_timestamp diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 15e15b7a518..e5addcc393a 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -174,6 +174,7 @@ def create_if_needed(d): context.python_exe = exename binpath = self._venv_path(env_dir, 'scripts') libpath = self._venv_path(env_dir, 'purelib') + platlibpath = self._venv_path(env_dir, 'platlib') # PEP 405 says venvs should create a local include directory. # See https://peps.python.org/pep-0405/#include-files @@ -191,12 +192,8 @@ def create_if_needed(d): create_if_needed(incpath) context.lib_path = libpath create_if_needed(libpath) - # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX - if ((sys.maxsize > 2**32) and (os.name == 'posix') and - (sys.platform != 'darwin')): - link_path = os.path.join(env_dir, 'lib64') - if not os.path.exists(link_path): # Issue #21643 - os.symlink('lib', link_path) + context.platlib_path = platlibpath + create_if_needed(platlibpath) context.bin_path = binpath context.bin_name = os.path.relpath(binpath, env_dir) context.env_exe = os.path.join(binpath, exename) @@ -313,11 +310,8 @@ def setup_python(self, context): copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - - suffixes = ['python', 'python3', f'python3.{sys.version_info[1]}'] - if sys.version_info[:2] == (3, 14): - suffixes.append('𝜋thon') - for suffix in suffixes: + for suffix in ('python', 'python3', + f'python3.{sys.version_info[1]}'): path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if diff --git a/Lib/wave.py b/Lib/wave.py index a34af244c3e..25ca9ef168e 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -20,10 +20,6 @@ compression type ('not compressed' linear samples) getparams() -- returns a namedtuple consisting of all of the above in the above order - getmarkers() -- returns None (for compatibility with the - old aifc module) - getmark(id) -- raises an error since the mark does not - exist (for compatibility with the old aifc module) readframes(n) -- returns at most n frames of audio rewind() -- rewind to the beginning of the audio stream setpos(pos) -- seek to the specified position @@ -73,6 +69,7 @@ from collections import namedtuple import builtins +import os import struct import sys @@ -100,7 +97,7 @@ def _byteswap(data, width): for j in range(width): swapped_data[i + width - 1 - j] = data[i + j] - return bytes(swapped_data) + return swapped_data.take_bytes() class _Chunk: @@ -278,7 +275,7 @@ def initfp(self, file): def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'rb') self._i_opened_the_file = f # else, assume it is an open file object already @@ -341,16 +338,6 @@ def getparams(self): self.getframerate(), self.getnframes(), self.getcomptype(), self.getcompname()) - def getmarkers(self): - import warnings - warnings._deprecated("Wave_read.getmarkers", remove=(3, 15)) - return None - - def getmark(self, id): - import warnings - warnings._deprecated("Wave_read.getmark", remove=(3, 15)) - raise Error('no marks') - def setpos(self, pos): if pos < 0 or pos > self._nframes: raise Error('position not in range') @@ -441,9 +428,11 @@ class Wave_write: _datawritten -- the size of the audio samples actually written """ + _file = None + def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'wb') self._i_opened_the_file = f try: @@ -551,21 +540,6 @@ def getparams(self): return _wave_params(self._nchannels, self._sampwidth, self._framerate, self._nframes, self._comptype, self._compname) - def setmark(self, id, pos, name): - import warnings - warnings._deprecated("Wave_write.setmark", remove=(3, 15)) - raise Error('setmark() not supported') - - def getmark(self, id): - import warnings - warnings._deprecated("Wave_write.getmark", remove=(3, 15)) - raise Error('no marks') - - def getmarkers(self): - import warnings - warnings._deprecated("Wave_write.getmarkers", remove=(3, 15)) - return None - def tell(self): return self._nframeswritten diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py index cafe872c7aa..9353fb67862 100644 --- a/Lib/wsgiref/handlers.py +++ b/Lib/wsgiref/handlers.py @@ -69,7 +69,8 @@ def read_environ(): # Python 3's http.server.CGIHTTPRequestHandler decodes # using the urllib.unquote default of UTF-8, amongst other - # issues. + # issues. While the CGI handler is removed in 3.15, this + # is kept for legacy reasons. elif ( software.startswith('simplehttp/') and 'python/3' in software diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index db51f350ea0..0a2ccc00f18 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -292,13 +292,6 @@ def _append_child(self, node): childNodes.append(node) node.parentNode = self -def _in_document(node): - # return True iff node is part of a document tree - while node is not None: - if node.nodeType == Node.DOCUMENT_NODE: - return True - node = node.parentNode - return False def _write_data(writer, text, attr): "Writes datachars to writer." @@ -1555,7 +1548,7 @@ def _clear_id_cache(node): if node.nodeType == Node.DOCUMENT_NODE: node._id_cache.clear() node._id_search_stack = None - elif _in_document(node): + elif node.ownerDocument: node.ownerDocument._id_cache.clear() node.ownerDocument._id_search_stack= None diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 44ab5d18624..d8c0b1b6216 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -527,7 +527,9 @@ class ElementTree: """ def __init__(self, element=None, file=None): - # assert element is None or iselement(element) + if element is not None and not iselement(element): + raise TypeError('expected an Element, not %s' % + type(element).__name__) self._root = element # first node if file: self.parse(file) @@ -543,7 +545,9 @@ def _setroot(self, element): with the given element. Use with care! """ - # assert iselement(element) + if not iselement(element): + raise TypeError('expected an Element, not %s' + % type(element).__name__) self._root = element def parse(self, source, parser=None): @@ -709,6 +713,8 @@ def write(self, file_or_filename, of start/end tags """ + if self._root is None: + raise TypeError('ElementTree not initialized') if not method: method = "xml" elif method not in _serialize: @@ -1255,16 +1261,20 @@ def iterator(source): gen = iterator(source) class IterParseIterator(collections.abc.Iterator): __next__ = gen.__next__ + def close(self): + nonlocal close_source if close_source: source.close() + close_source = False gen.close() - def __del__(self): - # TODO: Emit a ResourceWarning if it was not explicitly closed. - # (When the close() method will be supported in all maintained Python versions.) + def __del__(self, _warn=warnings.warn): if close_source: - source.close() + try: + _warn(f"unclosed iterparse iterator {source.name!r}", ResourceWarning, stacklevel=2) + finally: + source.close() it = IterParseIterator() it.root = None diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 90a356fbb8e..3e6871157d0 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -239,7 +239,7 @@ def register_multicall_functions(self): see http://www.xmlrpc.com/discuss/msgReader$1208""" - self.funcs.update({'system.multicall' : self.system_multicall}) + self.funcs['system.multicall'] = self.system_multicall def _marshaled_dispatch(self, data, dispatch_method = None, path = None): """Dispatches an XML-RPC method from marshalled (XML) data. @@ -578,7 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True - allow_reuse_port = True + allow_reuse_port = False # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 88356abe8cb..ac2332e5846 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -38,8 +38,8 @@ __all__ = ["BadZipFile", "BadZipfile", "error", "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", - "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile", - "Path"] + "ZIP_ZSTANDARD", "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", + "LargeZipFile", "Path"] class BadZipFile(Exception): pass @@ -234,8 +234,19 @@ def strip(cls, data, xids): def _check_zipfile(fp): try: - if _EndRecData(fp): - return True # file has correct magic number + endrec = _EndRecData(fp) + if endrec: + if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0: + return True # Empty zipfiles are still zipfiles + elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]: + # Central directory is on the same disk + fp.seek(sum(_handle_prepended_data(endrec))) + if endrec[_ECD_SIZE] >= sizeCentralDir: + data = fp.read(sizeCentralDir) # CD is where we expect it to be + if len(data) == sizeCentralDir: + centdir = struct.unpack(structCentralDir, data) # CD is the right size + if centdir[_CD_SIGNATURE] == stringCentralDir: + return True # First central directory entry has correct magic number except OSError: pass return False @@ -254,24 +265,36 @@ def is_zipfile(filename): else: with open(filename, "rb") as fp: result = _check_zipfile(fp) - except OSError: + except (OSError, BadZipFile): pass return result +def _handle_prepended_data(endrec, debug=0): + size_cd = endrec[_ECD_SIZE] # bytes in central directory + offset_cd = endrec[_ECD_OFFSET] # offset of central directory + + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd + + if debug > 2: + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) + + return offset_cd, concat + def _EndRecData64(fpin, offset, endrec): """ Read the ZIP64 end-of-archive records and use that to update endrec """ - try: - fpin.seek(offset - sizeEndCentDir64Locator, 2) - except OSError: - # If the seek fails, the file is not large enough to contain a ZIP64 + offset -= sizeEndCentDir64Locator + if offset < 0: + # The file is not large enough to contain a ZIP64 # end-of-archive record, so just return the end record we were given. return endrec - + fpin.seek(offset) data = fpin.read(sizeEndCentDir64Locator) if len(data) != sizeEndCentDir64Locator: - return endrec + raise OSError("Unknown I/O error") sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) if sig != stringEndArchive64Locator: return endrec @@ -279,16 +302,33 @@ def _EndRecData64(fpin, offset, endrec): if diskno != 0 or disks > 1: raise BadZipFile("zipfiles that span multiple disks are not supported") - # Assume no 'zip64 extensible data' - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) + offset -= sizeEndCentDir64 + if reloff > offset: + raise BadZipFile("Corrupt zip64 end of central directory locator") + # First, check the assumption that there is no prepended data. + fpin.seek(reloff) + extrasz = offset - reloff data = fpin.read(sizeEndCentDir64) if len(data) != sizeEndCentDir64: - return endrec + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64) and reloff != offset: + # Since we already have seen the Zip64 EOCD Locator, it's + # possible we got here because there is prepended data. + # Assume no 'zip64 extensible data' + fpin.seek(offset) + extrasz = 0 + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64): + raise BadZipFile("Zip64 end of central directory record not found") + sig, sz, create_version, read_version, disk_num, disk_dir, \ dircount, dircount2, dirsize, diroffset = \ struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec + if (diroffset + dirsize != reloff or + sz + 12 != sizeEndCentDir64 + extrasz): + raise BadZipFile("Corrupt zip64 end of central directory record") # Update the original endrec using data from the ZIP64 record endrec[_ECD_SIGNATURE] = sig @@ -298,6 +338,7 @@ def _EndRecData64(fpin, offset, endrec): endrec[_ECD_ENTRIES_TOTAL] = dircount2 endrec[_ECD_SIZE] = dirsize endrec[_ECD_OFFSET] = diroffset + endrec[_ECD_LOCATION] = offset - extrasz return endrec @@ -331,7 +372,7 @@ def _EndRecData(fpin): endrec.append(filesize - sizeEndCentDir) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) + return _EndRecData64(fpin, filesize - sizeEndCentDir, endrec) # Either this is not a ZIP file, or it is a ZIP file with an archive # comment. Search the end of the file for the "end of central directory" @@ -355,8 +396,7 @@ def _EndRecData(fpin): endrec.append(maxCommentStart + start) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) + return _EndRecData64(fpin, maxCommentStart + start, endrec) # Unable to find a valid end of central directory structure return None @@ -812,11 +852,11 @@ def _get_compressor(compress_type, compresslevel=None): if compresslevel is not None: return bz2.BZ2Compressor(compresslevel) return bz2.BZ2Compressor() - # compresslevel is ignored for ZIP_LZMA and ZIP_ZSTANDARD + # compresslevel is ignored for ZIP_LZMA elif compress_type == ZIP_LZMA: return LZMACompressor() elif compress_type == ZIP_ZSTANDARD: - return zstd.ZstdCompressor() + return zstd.ZstdCompressor(level=compresslevel) else: return None @@ -1352,7 +1392,8 @@ class ZipFile: mode: The mode can be either read 'r', write 'w', exclusive create 'x', or append 'a'. compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), - ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma). + ZIP_BZIP2 (requires bz2), ZIP_LZMA (requires lzma), or + ZIP_ZSTANDARD (requires compression.zstd). allowZip64: if True ZipFile will create files with ZIP64 extensions when needed, otherwise it will raise an exception when this would be necessary. @@ -1361,6 +1402,9 @@ class ZipFile: When using ZIP_STORED or ZIP_LZMA this keyword has no effect. When using ZIP_DEFLATED integers 0 through 9 are accepted. When using ZIP_BZIP2 integers 1 through 9 are accepted. + When using ZIP_ZSTANDARD integers -7 though 22 are common, + see the CompressionParameter enum in compression.zstd for + details. """ @@ -1421,7 +1465,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self._lock = threading.RLock() self._seekable = True self._writing = False - self._data_offset = None try: if mode == 'r': @@ -1432,7 +1475,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self._didModify = True try: self.start_dir = self.fp.tell() - self._data_offset = self.start_dir except (AttributeError, OSError): self.fp = _Tellable(self.fp) self.start_dir = 0 @@ -1457,7 +1499,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, # even if no files are added to the archive self._didModify = True self.start_dir = self.fp.tell() - self._data_offset = self.start_dir else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: @@ -1497,28 +1538,17 @@ def _RealGetContents(self): raise BadZipFile("File is not a zip file") if self.debug > 1: print(endrec) - size_cd = endrec[_ECD_SIZE] # bytes in central directory - offset_cd = endrec[_ECD_OFFSET] # offset of central directory self._comment = endrec[_ECD_COMMENT] # archive comment - # "concat" is zero, unless zip was concatenated to another file - concat = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) + offset_cd, concat = _handle_prepended_data(endrec, self.debug) - # store the offset to the beginning of data for the - # .data_offset property - self._data_offset = concat - - if self.debug > 2: - inferred = concat + offset_cd - print("given, inferred, offset", offset_cd, inferred, concat) # self.start_dir: Position of start of central directory self.start_dir = offset_cd + concat + if self.start_dir < 0: raise BadZipFile("Bad offset for central directory") fp.seek(self.start_dir, 0) + size_cd = endrec[_ECD_SIZE] data = fp.read(size_cd) fp = io.BytesIO(data) total = 0 @@ -1575,12 +1605,6 @@ def _RealGetContents(self): zinfo._end_offset = end_offset end_offset = zinfo.header_offset - @property - def data_offset(self): - """The offset to the start of zip data in the file or None if - unavailable.""" - return self._data_offset - def namelist(self): """Return a list of file names in the archive.""" return [data.filename for data in self.filelist] @@ -2093,6 +2117,8 @@ def _write_end_record(self): min_version = max(BZIP2_VERSION, min_version) elif zinfo.compress_type == ZIP_LZMA: min_version = max(LZMA_VERSION, min_version) + elif zinfo.compress_type == ZIP_ZSTANDARD: + min_version = max(ZSTANDARD_VERSION, min_version) extract_version = max(min_version, zinfo.extract_version) create_version = max(min_version, zinfo.create_version) @@ -2129,7 +2155,7 @@ def _write_end_record(self): " would require ZIP64 extensions") zip64endrec = struct.pack( structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, + sizeEndCentDir64 - 12, 45, 45, 0, 0, centDirCount, centDirCount, centDirSize, centDirOffset) self.fp.write(zip64endrec) diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 5ae16ec970d..faae4c84cae 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -7,19 +7,19 @@ for more detail. """ +import functools import io -import posixpath -import zipfile import itertools -import contextlib import pathlib +import posixpath import re import stat import sys +import zipfile +from ._functools import save_method_args from .glob import Translator - __all__ = ['Path'] @@ -86,13 +86,12 @@ class InitializedState: Mix-in to save the initialization state for pickling. """ + @save_method_args def __init__(self, *args, **kwargs): - self.__args = args - self.__kwargs = kwargs super().__init__(*args, **kwargs) def __getstate__(self): - return self.__args, self.__kwargs + return self._saved___init__.args, self._saved___init__.kwargs def __setstate__(self, state): args, kwargs = state @@ -181,22 +180,27 @@ class FastLookup(CompleteDirs): """ def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super().namelist() - return self.__names + return self._namelist + + @functools.cached_property + def _namelist(self): + return super().namelist() def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super()._name_set() - return self.__lookup + return self._name_set_prop + + @functools.cached_property + def _name_set_prop(self): + return super()._name_set() def _extract_text_encoding(encoding=None, *args, **kwargs): # compute stack level so that the caller of the caller sees any warning. is_pypy = sys.implementation.name == 'pypy' - stack_level = 3 + is_pypy + # PyPy no longer special cased after 7.3.19 (or maybe 7.3.18) + # See jaraco/zipp#143 + is_old_pypi = is_pypy and sys.pypy_version_info < (7, 3, 19) + stack_level = 3 + is_old_pypi return io.text_encoding(encoding, stack_level), args, kwargs @@ -351,7 +355,7 @@ def open(self, mode='r', *args, pwd=None, **kwargs): return io.TextIOWrapper(stream, encoding, *args, **kwargs) def _base(self): - return pathlib.PurePosixPath(self.at or self.root.filename) + return pathlib.PurePosixPath(self.at) if self.at else self.filename @property def name(self): diff --git a/Lib/zipfile/_path/_functools.py b/Lib/zipfile/_path/_functools.py new file mode 100644 index 00000000000..7390be21873 --- /dev/null +++ b/Lib/zipfile/_path/_functools.py @@ -0,0 +1,20 @@ +import collections +import functools + + +# from jaraco.functools 4.0.2 +def save_method_args(method): + """ + Wrap a method such that when it is called, the args and kwargs are + saved on the method. + """ + args_and_kwargs = collections.namedtuple('args_and_kwargs', 'args kwargs') # noqa: PYI024 + + @functools.wraps(method) + def wrapper(self, /, *args, **kwargs): + attr_name = '_saved_' + method.__name__ + attr = args_and_kwargs(args, kwargs) + setattr(self, attr_name, attr) + return method(self, *args, **kwargs) + + return wrapper diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py index d7fe45a4947..bd2839304b7 100644 --- a/Lib/zipfile/_path/glob.py +++ b/Lib/zipfile/_path/glob.py @@ -1,7 +1,6 @@ import os import re - _default_seps = os.sep + str(os.altsep) * bool(os.altsep) diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 444c9dd11d8..19279d1c2be 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -10,15 +10,12 @@ to Zip archives. """ -#from importlib import _bootstrap_external -#from importlib import _bootstrap # for _verbose_message import _frozen_importlib_external as _bootstrap_external from _frozen_importlib_external import _unpack_uint16, _unpack_uint32, _unpack_uint64 import _frozen_importlib as _bootstrap # for _verbose_message import _imp # for check_hash_based_pycs import _io # for open import marshal # for loads -import sys # for modules import time # for mktime __all__ = ['ZipImportError', 'zipimporter'] @@ -34,8 +31,6 @@ class ZipImportError(ImportError): # _read_directory() cache _zip_directory_cache = {} -_module_type = type(sys) - END_CENTRAL_DIR_SIZE = 22 END_CENTRAL_DIR_SIZE_64 = 56 END_CENTRAL_DIR_LOCATOR_SIZE_64 = 20 @@ -210,52 +205,6 @@ def is_package(self, fullname): return mi - # Load and return the module named by 'fullname'. - def load_module(self, fullname): - """load_module(fullname) -> module. - - Load the module specified by 'fullname'. 'fullname' must be the - fully qualified (dotted) module name. It returns the imported - module, or raises ZipImportError if it could not be imported. - - Deprecated since Python 3.10. Use exec_module() instead. - """ - import warnings - warnings._deprecated("zipimport.zipimporter.load_module", - f"{warnings._DEPRECATED_MSG}; " - "use zipimport.zipimporter.exec_module() instead", - remove=(3, 15)) - code, ispackage, modpath = _get_module_code(self, fullname) - mod = sys.modules.get(fullname) - if mod is None or not isinstance(mod, _module_type): - mod = _module_type(fullname) - sys.modules[fullname] = mod - mod.__loader__ = self - - try: - if ispackage: - # add __path__ to the module *before* the code gets - # executed - path = _get_module_path(self, fullname) - fullpath = _bootstrap_external._path_join(self.archive, path) - mod.__path__ = [fullpath] - - if not hasattr(mod, '__builtins__'): - mod.__builtins__ = __builtins__ - _bootstrap_external._fix_up_module(mod.__dict__, fullname, modpath) - exec(code, mod.__dict__) - except: - del sys.modules[fullname] - raise - - try: - mod = sys.modules[fullname] - except KeyError: - raise ImportError(f'Loaded module {fullname!r} not found in sys.modules') - _bootstrap._verbose_message('import {} # loaded from Zip {}', fullname, modpath) - return mod - - def get_resource_reader(self, fullname): """Return the ResourceReader for a module in a zip file.""" from importlib.readers import ZipReader @@ -603,11 +552,16 @@ def _read_directory(archive): ) _importing_zlib = False +_zlib_decompress = None # Return the zlib.decompress function object, or NULL if zlib couldn't # be imported. The function is cached when found, so subsequent calls # don't import zlib again. -def _get_decompress_func(): +def _get_zlib_decompress_func(): + global _zlib_decompress + if _zlib_decompress: + return _zlib_decompress + global _importing_zlib if _importing_zlib: # Someone has a zlib.py[co] in their Zip file @@ -617,7 +571,7 @@ def _get_decompress_func(): _importing_zlib = True try: - from zlib import decompress + from zlib import decompress as _zlib_decompress except Exception: _bootstrap._verbose_message('zipimport: zlib UNAVAILABLE') raise ZipImportError("can't decompress data; zlib not available") @@ -625,7 +579,54 @@ def _get_decompress_func(): _importing_zlib = False _bootstrap._verbose_message('zipimport: zlib available') - return decompress + return _zlib_decompress + + +_importing_zstd = False +_zstd_decompressor_class = None + +# Return the _zstd.ZstdDecompressor function object, or NULL if _zstd couldn't +# be imported. The result is cached when found. +def _get_zstd_decompressor_class(): + global _zstd_decompressor_class + if _zstd_decompressor_class: + return _zstd_decompressor_class + + global _importing_zstd + if _importing_zstd: + # Someone has a _zstd.py[co] in their Zip file + # let's avoid a stack overflow. + _bootstrap._verbose_message("zipimport: zstd UNAVAILABLE") + raise ZipImportError("can't decompress data; zstd not available") + + _importing_zstd = True + try: + from _zstd import ZstdDecompressor as _zstd_decompressor_class + except Exception: + _bootstrap._verbose_message("zipimport: zstd UNAVAILABLE") + raise ZipImportError("can't decompress data; zstd not available") + finally: + _importing_zstd = False + + _bootstrap._verbose_message("zipimport: zstd available") + return _zstd_decompressor_class + + +def _zstd_decompress(data): + # A simple version of compression.zstd.decompress() as we cannot import + # that here as the stdlib itself could be being zipimported. + results = [] + while True: + decomp = _get_zstd_decompressor_class()() + results.append(decomp.decompress(data)) + if not decomp.eof: + raise ZipImportError("zipimport: zstd compressed data ended before " + "the end-of-stream marker") + data = decomp.unused_data + if not data: + break + return b"".join(results) + # Given a path to a Zip file and a toc_entry, return the (uncompressed) data. def _get_data(archive, toc_entry): @@ -659,16 +660,23 @@ def _get_data(archive, toc_entry): if len(raw_data) != data_size: raise OSError("zipimport: can't read data") - if compress == 0: - # data is not compressed - return raw_data - - # Decompress with zlib - try: - decompress = _get_decompress_func() - except Exception: - raise ZipImportError("can't decompress data; zlib not available") - return decompress(raw_data, -15) + match compress: + case 0: # stored + return raw_data + case 8: # deflate aka zlib + try: + decompress = _get_zlib_decompress_func() + except Exception: + raise ZipImportError("can't decompress data; zlib not available") + return decompress(raw_data, -15) + case 93: # zstd + try: + return _zstd_decompress(raw_data) + except Exception: + raise ZipImportError("could not decompress zstd data") + # bz2 and lzma could be added, but are largely obsolete. + case _: + raise ZipImportError(f"zipimport: unsupported compression {compress}") # Lenient date/time comparison function. The precision of the mtime @@ -734,9 +742,9 @@ def _normalize_line_endings(source): # Given a string buffer containing Python source code, compile it # and return a code object. -def _compile_source(pathname, source): +def _compile_source(pathname, source, module): source = _normalize_line_endings(source) - return compile(source, pathname, 'exec', dont_inherit=True) + return compile(source, pathname, 'exec', dont_inherit=True, module=module) # Convert the date/time values found in the Zip archive to a value # that's compatible with the time stamp stored in .pyc files. @@ -807,7 +815,7 @@ def _get_module_code(self, fullname): except ImportError as exc: import_error = exc else: - code = _compile_source(modpath, data) + code = _compile_source(modpath, data, fullname) if code is None: # bad magic number or non-matching mtime # in byte code, try next diff --git a/Lib/zoneinfo/__init__.py b/Lib/zoneinfo/__init__.py index f5510ee0497..df2ae909f53 100644 --- a/Lib/zoneinfo/__init__.py +++ b/Lib/zoneinfo/__init__.py @@ -12,7 +12,10 @@ try: from _zoneinfo import ZoneInfo -except ImportError: # pragma: nocover +except (ImportError, AttributeError): # pragma: nocover + # AttributeError: module 'datetime' has no attribute 'datetime_CAPI'. + # This happens when the '_datetime' module is not available and the + # pure Python implementation is used instead. from ._zoneinfo import ZoneInfo reset_tzpath = _tzpath.reset_tzpath diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 6e05abc3239..03cc42149f9 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -9,9 +9,13 @@ def load_tzdata(key): resource_name = components[-1] try: - return resources.files(package_name).joinpath(resource_name).open("rb") + path = resources.files(package_name).joinpath(resource_name) + # gh-85702: Prevent PermissionError on Windows + if path.is_dir(): + raise IsADirectoryError + return path.open("rb") except (ImportError, FileNotFoundError, UnicodeEncodeError, IsADirectoryError): - # There are three types of exception that can be raised that all amount + # There are four types of exception that can be raised that all amount # to "we cannot find this key": # # ImportError: If package_name doesn't exist (e.g. if tzdata is not diff --git a/Lib/zoneinfo/_tzpath.py b/Lib/zoneinfo/_tzpath.py index 5db17bea045..3661c837daa 100644 --- a/Lib/zoneinfo/_tzpath.py +++ b/Lib/zoneinfo/_tzpath.py @@ -13,6 +13,13 @@ def _reset_tzpath(to=None, stacklevel=4): + f"not {type(tzpaths)}: {tzpaths!r}" ) + tzpaths = [os.fspath(p) for p in tzpaths] + if not all(isinstance(p, str) for p in tzpaths): + raise TypeError( + "All elements of a tzpath sequence must be strings or " + "os.PathLike objects which convert to strings." + ) + if not all(map(os.path.isabs, tzpaths)): raise ValueError(_get_invalid_paths_message(tzpaths)) base_tzpath = tzpaths @@ -124,7 +131,8 @@ def available_timezones(): # Start with loading from the tzdata package if it exists: this has a # pre-assembled list of zones that only requires opening one file. try: - with resources.files("tzdata").joinpath("zones").open("r") as f: + zones_file = resources.files("tzdata").joinpath("zones") + with zones_file.open("r", encoding="utf-8") as f: for zone in f: zone = zone.strip() if zone: @@ -169,6 +177,10 @@ def valid_key(fpath): # posixrules is a special symlink-only time zone where it exists, it # should not be included in the output valid_zones.remove("posixrules") + if "localtime" in valid_zones: + # localtime is a special symlink-only time zone where it exists, it + # should not be included in the output + valid_zones.remove("localtime") return valid_zones diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py index b77dc0ed391..3ffdb4c8371 100644 --- a/Lib/zoneinfo/_zoneinfo.py +++ b/Lib/zoneinfo/_zoneinfo.py @@ -75,12 +75,12 @@ def _new_instance(cls, key): return obj @classmethod - def from_file(cls, fobj, /, key=None): + def from_file(cls, file_obj, /, key=None): obj = super().__new__(cls) obj._key = key obj._file_path = None - obj._load_file(fobj) - obj._file_repr = repr(fobj) + obj._load_file(file_obj) + obj._file_repr = repr(file_obj) # Disable pickling for objects created from files obj.__reduce__ = obj._file_reduce diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index b31cb766a46..1852397ed6f 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.16", - url="https://github.com/openssl/openssl/releases/download/openssl-3.0.16/openssl-3.0.16.tar.gz", - checksum='57e03c50feab5d31b152af2b764f10379aecd8ee92f16c985983ce4a99f7ef86', + name="OpenSSL 3.5.4", + url="https://github.com/openssl/openssl/releases/download/openssl-3.5.4/openssl-3.5.4.tar.gz", + checksum="967311f84955316969bdb1d8d4b983718ef42338639c621ec4c34fddef355e99", buildrecipe=build_universal_openssl, configure=None, install=None, @@ -264,10 +264,10 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.16' - tcl_checksum='91cb8fa61771c63c262efb553059b7c7ad6757afa5857af6265e4b0bdc2a14a5' + tcl_tk_ver='9.0.2' + tcl_checksum='e074c6a8d9ba2cddf914ba97b6677a552d7a52a3ca102924389a05ccb249b520' - tk_checksum='be9f94d3575d4b3099d84bc3c10de8994df2d7aa405208173c709cc404a7e5fe' + tk_checksum='76fb852b2f167592fe8b41aa6549ce4e486dbf3b259a269646600e3894517c76' tk_patches = [] @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.49.1", - url="https://sqlite.org/2025/sqlite-autoconf-3490100.tar.gz", - checksum="106642d8ccb36c5f7323b64e4152e9b719f7c0215acf5bfeac3d5e7f97b59254", + name="SQLite 3.50.4", + url="https://www.sqlite.org/2025/sqlite-autoconf-3500400.tar.gz", + checksum="a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -378,9 +378,9 @@ def library_recipes(): install=f"make && ranlib libsqlite3.a && make install DESTDIR={shellQuote(os.path.join(WORKDIR, 'libraries'))}", ), dict( - name="libmpdec 4.0.0", - url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", - checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", + name="libmpdec 4.0.1", + url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.1.tar.gz", + checksum="96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", configure_pre=[ "--disable-cxx", "MACHINE=universal", @@ -1747,7 +1747,7 @@ def main(): fn = os.path.join(folder, "ReadMe.rtf") patchFile("resources/ReadMe.rtf", fn) fn = os.path.join(folder, "Update Shell Profile.command") - patchScript("scripts/postflight.patch-profile", fn) + patchScript("resources/update_shell_profile.command", fn) fn = os.path.join(folder, "Install Certificates.command") patchScript("resources/install_certificates.command", fn) os.chmod(folder, STAT_0o755) diff --git a/Mac/BuildScript/resources/update_shell_profile.command b/Mac/BuildScript/resources/update_shell_profile.command new file mode 100755 index 00000000000..3cf4d74de9f --- /dev/null +++ b/Mac/BuildScript/resources/update_shell_profile.command @@ -0,0 +1,116 @@ +#!/bin/sh + +echo "This script will update your shell profile when the 'bin' directory" +echo "of python is not early enough of the PATH of your shell." +echo "These changes will be effective only in shell windows that you open" +echo "after running this script." + +PYVER=@PYVER@ +PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions/@PYVER@" + +if [ `id -ur` = 0 ]; then + # Run from the installer, do some trickery to fetch the information + # we need. + theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" + +else + theShell="${SHELL}" +fi + +# Make sure the directory ${PYTHON_ROOT}/bin is on the users PATH. +BSH="`basename "${theShell}"`" +case "${BSH}" in +bash|ksh|sh|*csh|zsh|fish) + if [ `id -ur` = 0 ]; then + P=`su - ${USER} -c 'echo A-X-4-X@@$PATH@@X-4-X-A' | grep 'A-X-4-X@@.*@@X-4-X-A' | sed -e 's/^A-X-4-X@@//g' -e 's/@@X-4-X-A$//g'` + else + P="`(exec -l ${theShell} -c 'echo $PATH')`" + fi + ;; +*) + echo "Sorry, I don't know how to patch $BSH shells" + exit 0 + ;; +esac + +# Now ensure that our bin directory is on $P and before /usr/bin at that +for elem in `echo $P | tr ':' ' '` +do + if [ "${elem}" = "${PYTHON_ROOT}/bin" ]; then + echo "All right, you're a python lover already" + exit 0 + elif [ "${elem}" = "/usr/bin" ]; then + break + fi +done + +echo "${PYTHON_ROOT}/bin is not on your PATH or at least not early enough" +case "${BSH}" in +*csh) + if [ -f "${HOME}/.tcshrc" ]; then + RC="${HOME}/.tcshrc" + else + RC="${HOME}/.cshrc" + fi + # Create backup copy before patching + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" >> "${RC}" + echo "# The original version is saved in .cshrc.pysave" >> "${RC}" + echo "set path=(${PYTHON_ROOT}/bin "'$path'")" >> "${RC}" + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; +bash) + if [ -e "${HOME}/.bash_profile" ]; then + PR="${HOME}/.bash_profile" + elif [ -e "${HOME}/.bash_login" ]; then + PR="${HOME}/.bash_login" + elif [ -e "${HOME}/.profile" ]; then + PR="${HOME}/.profile" + else + PR="${HOME}/.bash_profile" + fi + ;; +fish) + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" + mkdir -p "$CONFIG_DIR" + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; +zsh) + PR="${HOME}/.zprofile" + ;; +*sh) + PR="${HOME}/.profile" + ;; +esac + +# Create backup copy before patching +if [ -f "${PR}" ]; then + cp -fp "${PR}" "${PR}.pysave" +fi +echo "" >> "${PR}" +echo "# Setting PATH for Python ${PYVER}" >> "${PR}" +echo "# The original version is saved in `basename ${PR}`.pysave" >> "${PR}" +echo 'PATH="'"${PYTHON_ROOT}/bin"':${PATH}"' >> "${PR}" +echo 'export PATH' >> "${PR}" +if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${PR}" +fi +exit 0 diff --git a/Mac/BuildScript/scripts/postflight.patch-profile b/Mac/BuildScript/scripts/postflight.patch-profile index 9caf62211dd..ce8720f895d 100755 --- a/Mac/BuildScript/scripts/postflight.patch-profile +++ b/Mac/BuildScript/scripts/postflight.patch-profile @@ -1,116 +1,104 @@ #!/bin/sh -echo "This script will update your shell profile when the 'bin' directory" -echo "of python is not early enough of the PATH of your shell." -echo "These changes will be effective only in shell windows that you open" -echo "after running this script." - PYVER=@PYVER@ PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions/@PYVER@" -if [ `id -ur` = 0 ]; then - # Run from the installer, do some trickery to fetch the information - # we need. - theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" -else - theShell="${SHELL}" -fi +# Run from the installer, do some trickery to fetch the information +# we need. +theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" # Make sure the directory ${PYTHON_ROOT}/bin is on the users PATH. BSH="`basename "${theShell}"`" case "${BSH}" in bash|ksh|sh|*csh|zsh|fish) - if [ `id -ur` = 0 ]; then - P=`su - ${USER} -c 'echo A-X-4-X@@$PATH@@X-4-X-A' | grep 'A-X-4-X@@.*@@X-4-X-A' | sed -e 's/^A-X-4-X@@//g' -e 's/@@X-4-X-A$//g'` - else - P="`(exec -l ${theShell} -c 'echo $PATH')`" - fi - ;; + true + ;; *) - echo "Sorry, I don't know how to patch $BSH shells" - exit 0 - ;; + exit 0 + ;; esac -# Now ensure that our bin directory is on $P and before /usr/bin at that -for elem in `echo $P | tr ':' ' '` -do - if [ "${elem}" = "${PYTHON_ROOT}/bin" ]; then - echo "All right, you're a python lover already" - exit 0 - elif [ "${elem}" = "/usr/bin" ]; then - break - fi -done - -echo "${PYTHON_ROOT}/bin is not on your PATH or at least not early enough" case "${BSH}" in *csh) - if [ -f "${HOME}/.tcshrc" ]; then - RC="${HOME}/.tcshrc" - else - RC="${HOME}/.cshrc" - fi - # Create backup copy before patching - if [ -f "${RC}" ]; then - cp -fp "${RC}" "${RC}.pysave" - fi - echo "" >> "${RC}" - echo "# Setting PATH for Python ${PYVER}" >> "${RC}" - echo "# The original version is saved in .cshrc.pysave" >> "${RC}" - echo "set path=(${PYTHON_ROOT}/bin "'$path'")" >> "${RC}" - if [ `id -ur` = 0 ]; then - chown "${USER}" "${RC}" - fi - exit 0 - ;; + if [ -f "${HOME}/.tcshrc" ]; then + RC="${HOME}/.tcshrc" + else + RC="${HOME}/.cshrc" + fi + + # Drop privileges while writing files. + su -m ${USER} <<EOFC + # Create backup copy before patching + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" >> "${RC}" + echo "# The original version is saved in .cshrc.pysave" >> "${RC}" + echo "set path=(${PYTHON_ROOT}/bin "'\$path'")" >> "${RC}" +EOFC + + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; bash) - if [ -e "${HOME}/.bash_profile" ]; then - PR="${HOME}/.bash_profile" - elif [ -e "${HOME}/.bash_login" ]; then - PR="${HOME}/.bash_login" - elif [ -e "${HOME}/.profile" ]; then - PR="${HOME}/.profile" - else - PR="${HOME}/.bash_profile" - fi - ;; + if [ -e "${HOME}/.bash_profile" ]; then + PR="${HOME}/.bash_profile" + elif [ -e "${HOME}/.bash_login" ]; then + PR="${HOME}/.bash_login" + elif [ -e "${HOME}/.profile" ]; then + PR="${HOME}/.profile" + else + PR="${HOME}/.bash_profile" + fi + ;; fish) - CONFIG_DIR="${HOME}/.config/fish/conf.d/" - RC="${CONFIG_DIR}/python-${PYVER}.fish" - mkdir -p "$CONFIG_DIR" - if [ -f "${RC}" ]; then - cp -fp "${RC}" "${RC}.pysave" - fi - echo "# Setting PATH for Python ${PYVER}" > "${RC}" - if [ -f "${RC}.pysave" ]; then - echo "# The original version is saved in ${RC}.pysave" >> "${RC}" - fi - echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" - if [ `id -ur` = 0 ]; then - chown "${USER}" "${RC}" - fi - exit 0 - ;; + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" + + # Drop privileges while writing files. + su -m ${USER} <<EOFF + mkdir -p "$CONFIG_DIR" + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" +EOFF + + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; zsh) - PR="${HOME}/.zprofile" - ;; + PR="${HOME}/.zprofile" + ;; *sh) - PR="${HOME}/.profile" - ;; + PR="${HOME}/.profile" + ;; esac +# Drop privileges while writing files. +su -m ${USER} <<EOFS # Create backup copy before patching if [ -f "${PR}" ]; then - cp -fp "${PR}" "${PR}.pysave" + cp -fp "${PR}" "${PR}.pysave" fi echo "" >> "${PR}" echo "# Setting PATH for Python ${PYVER}" >> "${PR}" echo "# The original version is saved in `basename ${PR}`.pysave" >> "${PR}" -echo 'PATH="'"${PYTHON_ROOT}/bin"':${PATH}"' >> "${PR}" +echo 'PATH="'"${PYTHON_ROOT}/bin"':\${PATH}"' >> "${PR}" echo 'export PATH' >> "${PR}" +EOFS + if [ `id -ur` = 0 ]; then - chown "${USER}" "${PR}" + chown -h "${USER}" "${PR}" fi exit 0 diff --git a/Makefile.pre.in b/Makefile.pre.in index 17e0c9904cc..f3086ec1462 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -483,6 +483,7 @@ PYTHON_OBJS= \ Python/pylifecycle.o \ Python/pymath.o \ Python/pystate.o \ + Python/pystats.o \ Python/pythonrun.o \ Python/pytime.o \ Python/qsbr.o \ @@ -501,7 +502,6 @@ PYTHON_OBJS= \ Python/pystrtod.o \ Python/pystrhex.o \ Python/dtoa.o \ - Python/formatter_unicode.o \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ @@ -558,8 +558,11 @@ OBJECT_OBJS= \ Objects/tupleobject.o \ Objects/typeobject.o \ Objects/typevarobject.o \ - Objects/unicodeobject.o \ + Objects/unicode_format.o \ + Objects/unicode_formatter.o \ + Objects/unicode_writer.o \ Objects/unicodectype.o \ + Objects/unicodeobject.o \ Objects/unionobject.o \ Objects/weakrefobject.o \ @PERF_TRAMPOLINE_OBJ@ @@ -801,10 +804,10 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ - python-config checksharedmods + python-config checksharedmods build-details.json .PHONY: build_emscripten -build_emscripten: build_wasm web_example +build_emscripten: build_wasm web_example web_example_pyrepl_jspi # Check that the source is clean when building out of source. .PHONY: check-clean-src @@ -1012,7 +1015,12 @@ $(LIBRARY): $(LIBRARY_OBJS) $(AR) $(ARFLAGS) $@ $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + # AIX Linker don't support "-h" option + if test "$(MACHDEP)" != "aix"; then \ + $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + else \ + $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + fi if test $(INSTSONAME) != $@; then \ $(LN) -f $(INSTSONAME) $@; \ fi @@ -1090,8 +1098,17 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) # wasm32-emscripten browser web example -WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ -web_example/python.html: $(WEBEX_DIR)/python.html +EMSCRIPTEN_DIR=$(srcdir)/Tools/wasm/emscripten +WEBEX_DIR=$(EMSCRIPTEN_DIR)/web_example/ + +ZIP_STDLIB=python$(VERSION)$(ABI_THREAD).zip +$(ZIP_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(EMSCRIPTEN_DIR)/wasm_assets.py \ + Makefile pybuilddir.txt Modules/Setup.local + $(PYTHON_FOR_BUILD) $(EMSCRIPTEN_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ + +web_example/index.html: $(WEBEX_DIR)/index.html @mkdir -p web_example @cp $< $@ @@ -1103,12 +1120,9 @@ web_example/server.py: $(WEBEX_DIR)/server.py @mkdir -p web_example @cp $< $@ -WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip -$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(WEBEX_DIR)/wasm_assets.py \ - Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ - --buildroot . --prefix $(prefix) -o $@ +web_example/$(ZIP_STDLIB): $(ZIP_STDLIB) + @mkdir -p web_example + @cp $< $@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ @@ -1119,7 +1133,35 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) cp python.wasm web_example/python.wasm .PHONY: web_example -web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/index.html web_example/server.py web_example/$(ZIP_STDLIB) + +WEBEX2=web_example_pyrepl_jspi +WEBEX2_DIR=$(EMSCRIPTEN_DIR)/$(WEBEX2)/ + +$(WEBEX2)/python.mjs $(WEBEX2)/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + @mkdir -p $(WEBEX2) + @cp python.mjs $(WEBEX2)/python.mjs + @cp python.wasm $(WEBEX2)/python.wasm + +$(WEBEX2)/index.html: $(WEBEX2_DIR)/index.html + @mkdir -p $(WEBEX2) + @cp $< $@ + +$(WEBEX2)/src.mjs: $(WEBEX2_DIR)/src.mjs + @mkdir -p $(WEBEX2) + @cp $< $@ + +$(WEBEX2)/$(ZIP_STDLIB): $(ZIP_STDLIB) + @mkdir -p $(WEBEX2) + @cp $< $@ + +.PHONY: web_example_pyrepl_jspi +web_example_pyrepl_jspi: $(WEBEX2)/python.mjs $(WEBEX2)/index.html $(WEBEX2)/src.mjs $(WEBEX2)/$(ZIP_STDLIB) + ############################################################################ # Header files @@ -1152,14 +1194,12 @@ PYTHON_HEADERS= \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/lock.h \ $(srcdir)/Include/longobject.h \ $(srcdir)/Include/marshal.h \ $(srcdir)/Include/memoryobject.h \ $(srcdir)/Include/methodobject.h \ $(srcdir)/Include/modsupport.h \ $(srcdir)/Include/moduleobject.h \ - $(srcdir)/Include/monitoring.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ @@ -1201,6 +1241,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/unicodeobject.h \ $(srcdir)/Include/warnings.h \ $(srcdir)/Include/weakrefobject.h \ + $(srcdir)/Python/remote_debug.h \ \ pyconfig.h \ $(PARSER_HEADERS) \ @@ -1228,9 +1269,10 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/listobject.h \ - $(srcdir)/Include/cpython/lock.h \ + $(srcdir)/Include/cpython/pylock.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ + $(srcdir)/Include/cpython/marshal.h \ $(srcdir)/Include/cpython/memoryobject.h \ $(srcdir)/Include/cpython/methodobject.h \ $(srcdir)/Include/cpython/modsupport.h \ @@ -1256,6 +1298,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/pythonrun.h \ $(srcdir)/Include/cpython/pythread.h \ $(srcdir)/Include/cpython/setobject.h \ + $(srcdir)/Include/cpython/sliceobject.h \ + $(srcdir)/Include/cpython/structseq.h \ $(srcdir)/Include/cpython/traceback.h \ $(srcdir)/Include/cpython/tracemalloc.h \ $(srcdir)/Include/cpython/tupleobject.h \ @@ -1391,10 +1435,12 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \ $(srcdir)/Include/internal/pycore_ucnhash.h \ + $(srcdir)/Include/internal/pycore_unicodectype.h \ $(srcdir)/Include/internal/pycore_unicodeobject.h \ $(srcdir)/Include/internal/pycore_unicodeobject_generated.h \ $(srcdir)/Include/internal/pycore_unionobject.h \ $(srcdir)/Include/internal/pycore_uniqueid.h \ + $(srcdir)/Include/internal/pycore_uop.h \ $(srcdir)/Include/internal/pycore_uop_ids.h \ $(srcdir)/Include/internal/pycore_uop_metadata.h \ $(srcdir)/Include/internal/pycore_warnings.h \ @@ -1558,6 +1604,11 @@ sharedmods: $(SHAREDMODS) pybuilddir.txt # dependency on BUILDPYTHON ensures that the target is run last .PHONY: checksharedmods checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) + @if [ -n "@MISSING_STDLIB_CONFIG@" ]; then \ + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info --with-missing-stdlib-config="@MISSING_STDLIB_CONFIG@"; \ + else \ + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info; \ + fi @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py .PHONY: rundsymutil @@ -2049,7 +2100,6 @@ UNICODE_DEPS = \ $(srcdir)/Objects/stringlib/fastsearch.h \ $(srcdir)/Objects/stringlib/find.h \ $(srcdir)/Objects/stringlib/find_max_char.h \ - $(srcdir)/Objects/stringlib/localeutil.h \ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/replace.h \ $(srcdir)/Objects/stringlib/repr.h \ @@ -2064,6 +2114,7 @@ Objects/bytes_methods.o: $(srcdir)/Objects/bytes_methods.c $(BYTESTR_DEPS) Objects/bytesobject.o: $(srcdir)/Objects/bytesobject.c $(BYTESTR_DEPS) Objects/bytearrayobject.o: $(srcdir)/Objects/bytearrayobject.c $(BYTESTR_DEPS) +Objects/unicode_format.o: $(srcdir)/Objects/unicode_format.c $(UNICODE_DEPS) Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS) Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h @@ -2190,7 +2241,7 @@ Python/frozen.o: $(FROZEN_FILES_OUT) # an include guard, so we can't use a pipeline to transform its output. Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d $(MKDIR_P) Include - CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -h -s $< + CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -h -s $(srcdir)/Include/pydtrace.d : sed in-place edit with POSIX-only tools sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp mv $@.tmp $@ @@ -2200,7 +2251,7 @@ Python/gc.o: $(srcdir)/Include/pydtrace.h Python/import.o: $(srcdir)/Include/pydtrace.h Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) - CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) + CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) Objects/typeobject.o: Objects/typeslots.inc @@ -2279,7 +2330,7 @@ testios: fi # Clone the testbed project into the XCFOLDER - $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" + $(PYTHON_FOR_BUILD) $(srcdir)/Apple/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" # Run the testbed project $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run --verbose -- test -uall --single-process --rerun -W @@ -2507,9 +2558,8 @@ maninstall: altmaninstall XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax LIBSUBDIRS= asyncio \ collections \ - compression compression/bz2 compression/gzip compression/zstd \ - compression/lzma compression/zlib compression/_common \ - concurrent concurrent/futures \ + compression compression/_common compression/zstd \ + concurrent concurrent/futures concurrent/interpreters \ csv \ ctypes ctypes/macholib \ curses \ @@ -2525,6 +2575,15 @@ LIBSUBDIRS= asyncio \ logging \ multiprocessing multiprocessing/dummy \ pathlib \ + profile \ + profiling profiling/sampling profiling/tracing \ + profiling/sampling/_assets \ + profiling/sampling/_heatmap_assets \ + profiling/sampling/_flamegraph_assets \ + profiling/sampling/_shared_assets \ + profiling/sampling/live_collector \ + profiling/sampling/_vendor/d3/7.8.5 \ + profiling/sampling/_vendor/d3-flame-graph/4.1.3 \ pydoc_data \ re \ site-packages \ @@ -2549,6 +2608,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_ast \ test/test_ast/data \ test/archivetestdata \ + test/audit_test_data \ test/audiodata \ test/certdata \ test/certdata/capath \ @@ -2568,7 +2628,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/subprocessdata \ test/support \ test/support/_hypothesis_stubs \ - test/support/interpreters \ test/test_asyncio \ test/test_capi \ test/test_cext \ @@ -2608,6 +2667,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/namespace_pkgs \ test/test_importlib/namespace_pkgs/both_portions \ test/test_importlib/namespace_pkgs/both_portions/foo \ + test/test_importlib/namespace_pkgs/foo \ test/test_importlib/namespace_pkgs/module_and_namespace_package \ test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test \ test/test_importlib/namespace_pkgs/not_a_namespace_pkg \ @@ -2630,14 +2690,18 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/source \ test/test_inspect \ test/test_interpreters \ + test/test_io \ test/test_json \ test/test_module \ test/test_multiprocessing_fork \ test/test_multiprocessing_forkserver \ test/test_multiprocessing_spawn \ + test/test_os \ test/test_pathlib \ test/test_pathlib/support \ test/test_peg_generator \ + test/test_profiling \ + test/test_profiling/test_sampling_profiler \ test/test_pydoc \ test/test_pyrepl \ test/test_string \ @@ -2764,6 +2828,7 @@ libinstall: all $(srcdir)/Modules/xxmodule.c $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_missing_stdlib_info.py $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against @@ -2995,6 +3060,9 @@ frameworkinstallunversionedstructure: $(LDLIBRARY) $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(LIBDIR) + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib" + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib" $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ @@ -3067,6 +3135,12 @@ config.status: $(srcdir)/configure Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< +Python/emscripten_trampoline_inner.wasm: $(srcdir)/Python/emscripten_trampoline_inner.c + # emcc has a path that ends with emsdk/upstream/emscripten/emcc, we're looking for emsdk/upstream/bin/clang. + $$(dirname $$(dirname $(CC)))/bin/clang -o $@ $< -mgc -O2 -Wl,--no-entry -Wl,--import-table -Wl,--import-memory -target wasm32-unknown-unknown -nostdlib + +Python/emscripten_trampoline_wasm.c: Python/emscripten_trampoline_inner.wasm + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/wasm/emscripten/prepare_external_wasm.py $< $@ getWasmTrampolineModule JIT_DEPS = \ $(srcdir)/Tools/jit/*.c \ @@ -3074,7 +3148,7 @@ JIT_DEPS = \ $(srcdir)/Python/executor_cases.c.h \ pyconfig.h -jit_stencils.h: $(JIT_DEPS) +jit_stencils.h @JIT_STENCILS_H@: $(JIT_DEPS) @REGEN_JIT_COMMAND@ Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ @@ -3175,14 +3249,14 @@ clean-retain-profile: pycremoval -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST - -rm -f jit_stencils.h + -rm -f jit_stencils*.h -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp - -rm -rf iOS/testbed/Python.xcframework/ios-*/bin - -rm -rf iOS/testbed/Python.xcframework/ios-*/lib - -rm -rf iOS/testbed/Python.xcframework/ios-*/include - -rm -rf iOS/testbed/Python.xcframework/ios-*/Python.framework + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/bin + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/lib + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/include + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/Python.framework .PHONY: profile-removal profile-removal: @@ -3208,7 +3282,7 @@ clobber: clean config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) - -rm -rf iOS/Frameworks + -rm -rf Apple/iOS/Frameworks -rm -rf iOSTestbed.* -rm -f python-config.py python-config -rm -rf cross-build @@ -3248,6 +3322,11 @@ check-c-globals: --format summary \ --traceback +# Check for undocumented C APIs. +.PHONY: check-c-api-docs +check-c-api-docs: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/check-c-api-docs/main.py + # Find files with funny names .PHONY: funny funny: @@ -3316,10 +3395,11 @@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/uni MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ -MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ +MODULE__DECIMAL_DEPS=@LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h +MODULE__REMOTE_DEBUGGING_DEPS=$(srcdir)/Modules/_remote_debugging/_remote_debugging.h # HACL*-based cryptographic primitives MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) @@ -3341,7 +3421,7 @@ MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_tes MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h -MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h +MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h MODULE__CODECS_CN_DEPS=$(srcdir)/Modules/cjkcodecs/mappings_cn.h $(CODECS_COMMON_HEADERS) diff --git a/Misc/ACKS b/Misc/ACKS index 610dcf9f423..e3927ff0b33 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -43,6 +43,7 @@ Ray Allen Billy G. Allie Jamiel Almeida Kevin Altis +Nazım Can Altınova Samy Lahfa Skyler Leigh Amador Joe Amenta @@ -209,6 +210,7 @@ Médéric Boquien Matias Bordese Jonas Borgström Jurjen Bos +Jeffrey Bosboom Peter Bosch Dan Boswell Eric Bouck @@ -478,9 +480,11 @@ Dean Draayer Fred L. Drake, Jr. Mehdi Drissi Derk Drukker +Weilin Du John DuBois Paul Dubois Jacques Ducasse +Jadon Duff Andrei Dorian Duma Graham Dumpleton Quinn Dunkan @@ -619,6 +623,7 @@ Soumendra Ganguly (गङ्गोपाध्याय) Fred Gansevles Paul Ganssle Tian Gao +Katie Gardner Lars Marius Garshol Jake Garver Dan Gass @@ -658,6 +663,7 @@ Michael Goderbauer Karan Goel Jeroen Van Goey Christoph Gohlke +Daniel Golding Tim Golden Yonatan Goldschmidt Mark Gollahon @@ -763,6 +769,7 @@ Chris Herborth Ivan Herman Jürgen Hermann Joshua Jay Herman +Kevin Hernandez Gary Herron Ernie Hershey Thomas Herve @@ -795,6 +802,7 @@ Albert Hofkamp Chris Hogan Tomas Hoger Jonathan Hogg +John Keith Hohm Vladyslav Hoi Kamilla Holanda Steve Holden @@ -900,6 +908,7 @@ Jim Jewett Pedro Diaz Jimenez Orjan Johansen Fredrik Johansson +Benjamin K. Johnson Gregory K. Johnson Kent Johnson Michael Johnson @@ -918,6 +927,7 @@ Kristján Valur Jónsson Jens B. Jorgensen John Jorgensen Sijin Joseph +Paresh Joshi Andreas Jung Tattoo Mabonzo K. Sarah K. @@ -928,6 +938,7 @@ Bob Kahn Kurt B. Kaiser Tamito Kajiyama Jan Kaliszewski +Ajay Kamdar Peter van Kampen Jan Kanis Rafe Kaplan @@ -940,6 +951,7 @@ Anton Kasyanov Lou Kates Makoto Kato Irit Katriel +Kattni Hiroaki Kawai Dmitry Kazakov Brian Kearns @@ -1050,6 +1062,7 @@ Alexander Lakeev David Lam Thomas Lamb Valerie Lambert +Kliment Lamonov Peter Lamut Jean-Baptiste "Jiba" Lamy Ronan Lamy @@ -1288,6 +1301,7 @@ Paul Moore Ross Moore Ben Morgan Emily Morehouse +Semyon Moroz Derek Morr James A Morrison Martin Morrison @@ -1362,6 +1376,7 @@ Milan Oberkirch Pascal Oberndoerfer Géry Ogam Seonkyo Ok +Andrea Oliveri Jeffrey Ollie Adam Olsen Bryan Olson @@ -1475,6 +1490,7 @@ Jean-François Piéronne Oleg Plakhotnyuk Anatoliy Platonov Marcel Plch +Stefan Pochmann Kirill Podoprigora Remi Pointel Jon Poler @@ -1666,6 +1682,7 @@ David Scherer Wolfgang Scherer Felix Scherz Hynek Schlawack +Jakob Schluse Bob Schmertz Gregor Schmid Ralf Schmitt @@ -1734,6 +1751,7 @@ Joel Shprentz Yue Shuaijie Jaysinh Shukla Terrel Shumway +Richard Si Eric Siegerman Reilly Tucker Siemens Paul Sijben @@ -1864,6 +1882,7 @@ Neil Tallim Geoff Talvola Anish Tambe Musashi Tamura +Long Tan William Tanksley Christian Tanzer Steven Taschuk @@ -1889,6 +1908,7 @@ Nicolas M. Thiéry James Thomas Reuben Thomas Robin Thomas +Douglas Thor Brian Thorne Christopher Thorne Stephen Thorne @@ -1902,6 +1922,7 @@ Tim Tisdall Jason Tishler Christian Tismer Jim Tittsler +Abhishek Tiwari Frank J. Tobin James Tocknell Bennett Todd @@ -1942,6 +1963,7 @@ Adnan Umer Utkarsh Upadhyay Roger Upole Daniel Urban +Alexander Enrique Urieles Nieto Matthias Urlichs Michael Urman Hector Urtubia @@ -1977,6 +1999,7 @@ Olivier Vielpeau Kannan Vijayan Kurt Vile Norman Vine +Tom Viner Pauli Virtanen Frank Visser Long Vo @@ -2096,6 +2119,7 @@ Xiang Zhang Robert Xiao Florent Xicluna Yanbo, Xie +Kaisheng Xu Xinhang Xu Arnon Yaari Alakshendra Yadav diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index f09842f1e77..473e7c7ac0f 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -3275,8 +3275,8 @@ Types created with :c:func:`PyType_FromSpec` now make any signature in their .. nonce: u6Xfr2 .. section: C API -Fix bug in PyOS_mystrnicmp and PyOS_mystricmp that incremented pointers -beyond the end of a string. +Fix bug in :c:func:`PyOS_mystrnicmp` and :c:func:`PyOS_mystricmp` that +incremented pointers beyond the end of a string. .. diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 33c3e14b7a4..6cf3db3eb43 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -394,7 +394,7 @@ Removed the ``formatter`` module, which was deprecated in Python 3.4. It is somewhat obsolete, little used, and not tested. It was originally scheduled to be removed in Python 3.6, but such removals were delayed until after Python 2.7 EOL. Existing users should copy whatever classes they use into -their code. Patch by Donghee Na and and Terry J. Reedy. +their code. Patch by Donghee Na and Terry J. Reedy. .. @@ -1395,9 +1395,9 @@ but now can get the condition by calling the new private .. nonce: -Br3Co .. section: C API -:c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, -:c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and -:c:func:`Py_GetProgramName` functions now return ``NULL`` if called before +:c:func:`!Py_GetPath`, :c:func:`!Py_GetPrefix`, :c:func:`!Py_GetExecPrefix`, +:c:func:`!Py_GetProgramFullPath`, :c:func:`!Py_GetPythonHome` and +:c:func:`!Py_GetProgramName` functions now return ``NULL`` if called before :c:func:`Py_Initialize` (before Python is initialized). Use the new :ref:`Python Initialization Configuration API <init-config>` to get the :ref:`Python Path Configuration. <init-path-config>`. Patch by Victor diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 406a5d7853e..5bc78b9007a 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -402,8 +402,8 @@ the heap. Should speed up dispatch in the interpreter. .. nonce: eUn4p5 .. section: Core and Builtins -Static methods (:func:`@staticmethod <staticmethod>`) and class methods -(:func:`@classmethod <classmethod>`) now inherit the method attributes +Static methods (:deco:`staticmethod`) and class methods +(:deco:`classmethod`) now inherit the method attributes (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, ``__annotations__``) and have a new ``__wrapped__`` attribute. Patch by Victor Stinner. @@ -454,7 +454,7 @@ file locations. .. nonce: VSF3vg .. section: Core and Builtins -Static methods (:func:`@staticmethod <staticmethod>`) are now callable as +Static methods (:deco:`staticmethod`) are now callable as regular functions. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 0b49c2a7877..10ef2968db2 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -2741,7 +2741,7 @@ Fix deprecation of :data:`ssl.OP_NO_TLSv1_3` .. nonce: TMWh1i .. section: Library -:meth:`pathlib.PureWindowsPath.is_reserved` now identifies a greater range +:meth:`!pathlib.PureWindowsPath.is_reserved` now identifies a greater range of reserved filenames, including those with trailing spaces or colons. .. @@ -2953,7 +2953,7 @@ support for Metadata 2.2. .. nonce: xTUyyX .. section: Library -Remove the :func:`@asyncio.coroutine <asyncio.coroutine>` :term:`decorator` +Remove the :deco:`asyncio.coroutine` :term:`decorator` enabling legacy generator-based coroutines to be compatible with async/await code; remove :class:`asyncio.coroutines.CoroWrapper` used for wrapping legacy coroutine objects in the debug mode. The decorator has been diff --git a/Misc/NEWS.d/3.11.0a2.rst b/Misc/NEWS.d/3.11.0a2.rst index 48cf2c1e428..12e03b46db0 100644 --- a/Misc/NEWS.d/3.11.0a2.rst +++ b/Misc/NEWS.d/3.11.0a2.rst @@ -1188,7 +1188,7 @@ context objects can now be disabled. .. nonce: Z0Zk_m .. section: C API -Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never +Exclude :c:func:`!PyWeakref_GET_OBJECT` from the limited C API. It never worked since the :c:type:`!PyWeakReference` structure is opaque in the limited C API. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index a2d36202045..47cbf33c3bb 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -1161,7 +1161,7 @@ no-op now. .. nonce: Lq2_gR .. section: C API -Replaced deprecated usage of :c:func:`PyImport_ImportModuleNoBlock` with +Replaced deprecated usage of :c:func:`!PyImport_ImportModuleNoBlock` with :c:func:`PyImport_ImportModule` in stdlib modules. Patch by Kumar Aditya. .. diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index c3a1942b881..7b8b983ebf9 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -664,7 +664,7 @@ for :func:`os.fcopyfile` available in macOs. .. nonce: l1p7CJ .. section: Library -For :func:`@dataclass <dataclasses.dataclass>`, add *weakref_slot*. +For :deco:`~dataclasses.dataclass`, add *weakref_slot*. The new parameter defaults to ``False``. If true, and if ``slots=True``, add a slot named ``"__weakref__"``, which will allow instances to be weakref'd. Contributed by Eric V. Smith diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index f2668e99a62..0da7cdde1b2 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -4330,7 +4330,7 @@ and ``sendfile`` inside ``IocpProactor``. .. nonce: GsBL9- .. section: Library -Fixed :meth:`collections.UserDict.get` to not call :meth:`__missing__` when +Fixed :meth:`collections.UserDict.get` to not call :meth:`~object.__missing__` when a value is not found. This matches the behavior of :class:`dict`. Patch by Bar Harel. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index bc028f30636..20e27c0d92f 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -35,11 +35,11 @@ Update bundled libexpat to 2.5.0 .. nonce: ik4iOv .. section: Core and Builtins -The docs clearly say that ``PyImport_Inittab``, +The docs clearly say that :c:data:`PyImport_Inittab`, :c:func:`PyImport_AppendInittab`, and :c:func:`PyImport_ExtendInittab` should not be used after :c:func:`Py_Initialize` has been called. We now enforce this for the two functions. Additionally, the runtime now uses an -internal copy of ``PyImport_Inittab``, to guard against modification. +internal copy of :c:data:`PyImport_Inittab`, to guard against modification. .. diff --git a/Misc/NEWS.d/3.12.0a5.rst b/Misc/NEWS.d/3.12.0a5.rst index 5dc443bb55b..b73bbfbfdc4 100644 --- a/Misc/NEWS.d/3.12.0a5.rst +++ b/Misc/NEWS.d/3.12.0a5.rst @@ -253,7 +253,7 @@ Adapt the ``_elementtree`` extension module to multi-phase init .. section: Library Avoid potential unexpected ``freeaddrinfo`` call (double free) in -:mod:`socket` when when a libc ``getaddrinfo()`` implementation leaves +:mod:`socket` when a libc ``getaddrinfo()`` implementation leaves garbage in an output pointer when returning an error. Original patch by Sergey G. Brester. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 91e9fee7e37..1391d670575 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -153,7 +153,7 @@ about a 10% improvement. .. section: Core and Builtins Guard ``assert(tstate->thread_id > 0)`` with ``#ifndef HAVE_PTHREAD_STUBS``. -This allows for for pydebug builds to work under WASI which (currently) +This allows for pydebug builds to work under WASI which (currently) lacks thread support. .. @@ -2294,7 +2294,7 @@ superclass. Patch by James Hilton-Balfe .. nonce: VksX1D .. section: Library -:class:`http.server.CGIHTTPRequestHandler` has been deprecated for removal +:class:`!http.server.CGIHTTPRequestHandler` has been deprecated for removal in 3.15. Its design is old and the web world has long since moved beyond CGI. @@ -3426,7 +3426,7 @@ This bug was introduced in Python 3.12.0 beta 1. .. nonce: hSlB17 .. section: Library -Deprecate :func:`typing.no_type_check_decorator`. No major type checker ever +Deprecate :func:`!typing.no_type_check_decorator`. No major type checker ever added support for this decorator. Patch by Alex Waygood. .. @@ -6458,8 +6458,8 @@ Victor Stinner. .. nonce: GRxZtI .. section: C API -Deprecate the :c:func:`PyWeakref_GetObject` and -:c:func:`PyWeakref_GET_OBJECT` functions: use the new +Deprecate the :c:func:`!PyWeakref_GetObject` and +:c:func:`!PyWeakref_GET_OBJECT` functions: use the new :c:func:`PyWeakref_GetRef` function instead. Patch by Victor Stinner. .. @@ -6470,7 +6470,7 @@ Deprecate the :c:func:`PyWeakref_GetObject` and .. section: C API Add :c:func:`PyWeakref_GetRef` function: similar to -:c:func:`PyWeakref_GetObject` but returns a :term:`strong reference`, or +:c:func:`!PyWeakref_GetObject` but returns a :term:`strong reference`, or ``NULL`` if the referent is no longer live. Patch by Victor Stinner. .. @@ -6538,7 +6538,7 @@ to hide implementation details. Patch by Victor Stinner. .. nonce: FQJG5B .. section: C API -Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just +Deprecate the :c:func:`!PyImport_ImportModuleNoBlock` function which is just an alias to :c:func:`PyImport_ImportModule` since Python 3.3. Patch by Victor Stinner. @@ -6592,13 +6592,13 @@ functions, deprecated in Python 3.9. Patch by Victor Stinner. Deprecate old Python initialization functions: -* :c:func:`PySys_ResetWarnOptions` -* :c:func:`Py_GetExecPrefix` -* :c:func:`Py_GetPath` -* :c:func:`Py_GetPrefix` -* :c:func:`Py_GetProgramFullPath` -* :c:func:`Py_GetProgramName` -* :c:func:`Py_GetPythonHome` +* :c:func:`!PySys_ResetWarnOptions` +* :c:func:`!Py_GetExecPrefix` +* :c:func:`!Py_GetPath` +* :c:func:`!Py_GetPrefix` +* :c:func:`!Py_GetProgramFullPath` +* :c:func:`!Py_GetProgramName` +* :c:func:`!Py_GetPythonHome` Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.13.0a4.rst b/Misc/NEWS.d/3.13.0a4.rst index 1b971113173..8afbe1b77f7 100644 --- a/Misc/NEWS.d/3.13.0a4.rst +++ b/Misc/NEWS.d/3.13.0a4.rst @@ -1096,7 +1096,7 @@ Also changed its name and daemonic status, it can be now joined. Add :func:`os.path.isreserved`, which identifies reserved pathnames such as "NUL", "AUX" and "CON". This function is only available on Windows. -Deprecate :meth:`pathlib.PurePath.is_reserved`. +Deprecate :meth:`!pathlib.PurePath.is_reserved`. .. diff --git a/Misc/NEWS.d/3.13.0a5.rst b/Misc/NEWS.d/3.13.0a5.rst index d56b1542b01..19ba16bc8c8 100644 --- a/Misc/NEWS.d/3.13.0a5.rst +++ b/Misc/NEWS.d/3.13.0a5.rst @@ -742,8 +742,8 @@ Add ``windows_31j`` to aliases for ``cp932`` codec .. nonce: fv35wU .. section: Library -:func:`functools.partial`s of :func:`repr` has been improved to include the -:term:`module` name. Patched by Furkan Onder and Anilyka Barry. +Always include the :term:`module` name in the :func:`repr` of +:func:`functools.partial` objects. Patch by Furkan Onder and Anilyka Barry. .. diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst index 2740b4f0d96..ad6622d23bf 100644 --- a/Misc/NEWS.d/3.13.0a6.rst +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -264,7 +264,8 @@ Improve performance of :func:`os.path.join` and :func:`os.path.expanduser`. .. nonce: hqk9Hn .. section: Library -Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath`. +Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath +<os.path.relpath>`. .. @@ -273,7 +274,8 @@ Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath`. .. nonce: l6rWlj .. section: Library -Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush`. +Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush +<mailbox.Mailbox.flush>`. Patch by Tony Mountifield. .. diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst index 98639f0d350..1938976fa42 100644 --- a/Misc/NEWS.d/3.14.0a1.rst +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -1033,7 +1033,7 @@ retrieve the spec information. .. nonce: s3vKql .. section: Library -:mod:`argparse` vim supports abbreviated single-dash long options separated +:mod:`argparse` supports abbreviated single-dash long options separated by ``=`` from its value. .. @@ -1999,7 +1999,7 @@ with an escape character. .. nonce: vi2bP- .. section: Library -:func:`@warnings.deprecated <warnings.deprecated>` now copies the coroutine +:deco:`warnings.deprecated` now copies the coroutine status of functions and methods so that :func:`inspect.iscoroutinefunction` returns the correct result. @@ -4913,11 +4913,11 @@ Allow tuples of length 20 in the freelist to be reused. .. nonce: lYKYYP .. section: Core and Builtins -:exc:`ValueError` messages for :meth:`!list.index`, :meth:`!range.index`, +:exc:`ValueError` messages for :meth:`list.index`, :meth:`range.index`, :meth:`!deque.index`, :meth:`!deque.remove` and :meth:`!ShareableList.index` no longer contain the repr of the searched value (which can be arbitrary -large) and are consistent with error messages for other :meth:`!index` and -:meth:`!remove` methods. +large) and are consistent with error messages for other :meth:`~sequence.index` and +:meth:`~sequence.remove` methods. .. @@ -5604,7 +5604,7 @@ Using :data:`NotImplemented` in a boolean context now raises .. nonce: wNMKVd .. section: Core and Builtins -Fix race condition in free-threaded build where :meth:`!list.extend` could +Fix race condition in free-threaded build where :meth:`list.extend` could expose uninitialised memory to concurrent readers. .. @@ -6092,7 +6092,7 @@ Patch by Victor Stinner. .. nonce: qOr9GF .. section: C API -Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +Soft deprecate the :c:macro:`Py_MEMCPY` macro: use directly ``memcpy()`` instead. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.14.0a2.rst b/Misc/NEWS.d/3.14.0a2.rst index 7405a1344a9..4a64bcc7520 100644 --- a/Misc/NEWS.d/3.14.0a2.rst +++ b/Misc/NEWS.d/3.14.0a2.rst @@ -1400,7 +1400,7 @@ The :class:`memoryview` type now supports subscription, making it a .. nonce: KlCdgD .. section: Core and Builtins -Adds :opcode:`LOAD_SMALL_INT` and :opcode:`LOAD_CONST_IMMORTAL` +Adds :opcode:`LOAD_SMALL_INT` and :opcode:`!LOAD_CONST_IMMORTAL` instructions. ``LOAD_SMALL_INT`` pushes a small integer equal to the ``oparg`` to the stack. ``LOAD_CONST_IMMORTAL`` does the same as ``LOAD_CONST`` but is more efficient for immortal objects. Removes diff --git a/Misc/NEWS.d/3.14.0a6.rst b/Misc/NEWS.d/3.14.0a6.rst index bafd8845de6..9064402bcf7 100644 --- a/Misc/NEWS.d/3.14.0a6.rst +++ b/Misc/NEWS.d/3.14.0a6.rst @@ -758,7 +758,7 @@ Patch by Semyon Moroz. .. nonce: wDLTay .. section: Library -Delay deprecated :meth:`zipimport.zipimporter.load_module` removal time to +Delay deprecated :meth:`!zipimport.zipimporter.load_module` removal time to 3.15. Use :meth:`zipimport.zipimporter.exec_module` instead. .. @@ -1325,7 +1325,7 @@ variable. .. nonce: d75n8U .. section: Core and Builtins -Adapt :func:`reversed` for use in the free-theading build. The +Adapt :func:`reversed` for use in the free-threading build. The :func:`reversed` is still not thread-safe in the sense that concurrent iterations may see the same object, but they will not corrupt the interpreter state. diff --git a/Misc/NEWS.d/3.14.0a7.rst b/Misc/NEWS.d/3.14.0a7.rst index 35b96d33da4..946ca27046a 100644 --- a/Misc/NEWS.d/3.14.0a7.rst +++ b/Misc/NEWS.d/3.14.0a7.rst @@ -192,7 +192,7 @@ The :class:`ctypes.py_object` type now supports subscription, making it a .. nonce: cX4yTn .. section: Library -Add the :attr:`zipfile.ZipFile.data_offset` attribute, which stores the +Add the :attr:`!zipfile.ZipFile.data_offset` attribute, which stores the offset to the beginning of ZIP data in a file when available. When the :class:`zipfile.ZipFile` is opened in either mode ``'w'`` or ``'x'`` and the underlying file does not support ``tell()``, the value will be ``None`` @@ -671,7 +671,7 @@ Allow the JIT to remove an extra ``_TO_BOOL_BOOL`` instruction after .. nonce: dNh64H .. section: Core and Builtins -Fix crash when calling :meth:`!list.append` as an unbound method. +Fix crash when calling :meth:`list.append` as an unbound method. .. diff --git a/Misc/NEWS.d/3.14.0b1.rst b/Misc/NEWS.d/3.14.0b1.rst index 5847dea7d5e..5d03d429f9e 100644 --- a/Misc/NEWS.d/3.14.0b1.rst +++ b/Misc/NEWS.d/3.14.0b1.rst @@ -1051,7 +1051,7 @@ warning filtering state if the :data:`sys.flags.context_aware_warnings` flag is set to true. This makes using the context manager thread-safe in multi-threaded programs. The flag is true by default in free-threaded builds and is otherwise false. The value of the flag can be overridden by -the the :option:`-X context_aware_warnings <-X>` command-line option or by +the :option:`-X context_aware_warnings <-X>` command-line option or by the :envvar:`PYTHON_CONTEXT_AWARE_WARNINGS` environment variable. .. @@ -1756,7 +1756,7 @@ Add support for macOS multi-arch builds with the JIT enabled .. nonce: q9fvyM .. section: Core and Builtins -PyREPL now supports syntax highlighing. Contributed by Łukasz Langa. +PyREPL now supports syntax highlighting. Contributed by Łukasz Langa. .. @@ -1797,7 +1797,7 @@ non-``None`` ``closure``. Patch by Bartosz Sławecki. .. nonce: Uj7lyY .. section: Core and Builtins -Fix a bug that was allowing newlines inconsitently in format specifiers for +Fix a bug that was allowing newlines inconsistently in format specifiers for single-quoted f-strings. Patch by Pablo Galindo. .. @@ -2012,7 +2012,7 @@ interpreter. .. nonce: ofI5Fl .. section: C API -Add support of nullable arguments in :c:func:`PyArg_Parse` and similar +[Reverted in :gh:`136991`] Add support of nullable arguments in :c:func:`PyArg_Parse` and similar functions. Adding ``?`` after any format unit makes ``None`` be accepted as a value. diff --git a/Misc/NEWS.d/3.15.0a1.rst b/Misc/NEWS.d/3.15.0a1.rst new file mode 100644 index 00000000000..5331f270162 --- /dev/null +++ b/Misc/NEWS.d/3.15.0a1.rst @@ -0,0 +1,6438 @@ +.. date: 2025-10-14-00-17-48 +.. gh-issue: 115119 +.. nonce: 470I1N +.. release date: 2025-10-14 +.. section: macOS + +Update macOS installer to use libmpdecimal 4.0.1. + +.. + +.. date: 2025-10-14-00-08-16 +.. gh-issue: 124111 +.. nonce: 7-j-DQ +.. section: macOS + +Update macOS installer to use Tcl/Tk 9.0.2. + +.. + +.. date: 2025-10-13-23-46-12 +.. gh-issue: 132339 +.. nonce: kAp603 +.. section: macOS + +Update macOS installer version of OpenSSL to 3.5.4. + +.. + +.. date: 2025-08-06-06-29-12 +.. gh-issue: 137450 +.. nonce: JZypb7 +.. section: macOS + +macOS installer shell path management improvements: separate the installer +``Shell profile updater`` postinstall script from the ``Update Shell +Profile.command`` to enable more robust error handling. + +.. + +.. date: 2025-07-27-02-17-40 +.. gh-issue: 137134 +.. nonce: pjgITs +.. section: macOS + +Update macOS installer to ship with SQLite version 3.50.4. + +.. + +.. date: 2025-10-08-22-54-38 +.. gh-issue: 139810 +.. nonce: LAaemi +.. section: Windows + +Installing with ``py install 3[.x]-dev`` will now select final versions as +well as prereleases. + +.. + +.. date: 2025-10-04-12-18-45 +.. gh-issue: 139573 +.. nonce: EO9kVB +.. section: Windows + +Updated bundled version of OpenSSL to 3.0.18. + +.. + +.. date: 2025-09-15-15-34-29 +.. gh-issue: 138896 +.. nonce: lkiF_7 +.. section: Windows + +Fix error installing C runtime on non-updated Windows machines + +.. + +.. date: 2025-09-03-01-07-44 +.. gh-issue: 138314 +.. nonce: IeWQ2i +.. section: Windows + +Add :func:`winreg.DeleteTree`. + +.. + +.. date: 2025-07-27-14-25-11 +.. gh-issue: 137136 +.. nonce: xNthFT +.. section: Windows + +Suppress build warnings when build on Windows with +``--experimental-jit-interpreter``. + +.. + +.. date: 2025-07-27-02-16-53 +.. gh-issue: 137134 +.. nonce: W0WpDF +.. section: Windows + +Update Windows installer to ship with SQLite 3.50.4. + +.. + +.. date: 2025-06-03-18-26-54 +.. gh-issue: 135099 +.. nonce: Q9usKm +.. section: Windows + +Fix a crash that could occur on Windows when a background thread waits on a +:c:type:`PyMutex` while the main thread is shutting down the interpreter. + +.. + +.. date: 2025-05-20-21-43-20 +.. gh-issue: 130727 +.. nonce: -69t4D +.. section: Windows + +Fix a race in internal calls into WMI that can result in an "invalid handle" +exception under high load. Patch by Chris Eibl. + +.. + +.. date: 2025-05-19-03-02-04 +.. gh-issue: 76023 +.. nonce: vHOf6M +.. section: Windows + +Make :func:`os.path.realpath` ignore Windows error 1005 when in non-strict +mode. + +.. + +.. date: 2025-05-13-13-25-27 +.. gh-issue: 133779 +.. nonce: -YcTBz +.. section: Windows + +Reverts the change to generate different :file:`pyconfig.h` files based on +compiler settings, as it was frequently causing extension builds to break. +In particular, the ``Py_GIL_DISABLED`` preprocessor variable must now always +be defined explicitly when compiling for the experimental free-threaded +runtime. The :func:`sysconfig.get_config_var` function can be used to +determine whether the current runtime was compiled with that flag or not. + +.. + +.. date: 2025-05-08-19-07-26 +.. gh-issue: 133626 +.. nonce: yFTKYK +.. section: Windows + +Ensures packages are not accidentally bundled into the traditional +installer. + +.. + +.. date: 2025-05-07-13-04-22 +.. gh-issue: 133580 +.. nonce: jBMujJ +.. section: Windows + +Fix :func:`sys.getwindowsversion` failing without setting an exception when +called on some WinAPI partitions. + +.. + +.. date: 2025-05-07-11-45-30 +.. gh-issue: 133572 +.. nonce: Xc2zxH +.. section: Windows + +Avoid LsaNtStatus to WinError conversion on unsupported WinAPI partitions. + +.. + +.. date: 2025-05-07-11-25-29 +.. gh-issue: 133568 +.. nonce: oYV0d8 +.. section: Windows + +Fix compile error when using a WinAPI partition that doesn't support the RPC +runtime library. + +.. + +.. date: 2025-05-07-09-02-19 +.. gh-issue: 133562 +.. nonce: lqqNW1 +.. section: Windows + +Disable handling of security descriptors by :func:`os.mkdir` with mode +``0o700`` on WinAPI partitions that do not support it. This only affects +custom builds for specialized targets. + +.. + +.. date: 2025-05-07-08-19-15 +.. gh-issue: 133537 +.. nonce: yzf963 +.. section: Windows + +Avoid using console I/O in WinAPI partitions that don’t support it + +.. + +.. date: 2025-03-31-15-37-57 +.. gh-issue: 131942 +.. nonce: jip_aL +.. section: Windows + +Use the Python-specific :c:macro:`Py_DEBUG` macro rather than +:c:macro:`!_DEBUG` in Windows-related C code. Patch by Xuehai Pan. + +.. + +.. date: 2025-09-25-10-31-02 +.. gh-issue: 139330 +.. nonce: 5WWkY0 +.. section: Tools/Demos + +SBOM generation tool didn't cross-check the version and checksum values +against the ``Modules/expat/refresh.sh`` script, leading to the values +becoming out-of-date during routine updates. + +.. + +.. date: 2025-08-28-06-22-26 +.. gh-issue: 132006 +.. nonce: eZQmc6 +.. section: Tools/Demos + +XCframeworks now include privacy manifests to satisfy Apple App Store +submission requirements. + +.. + +.. date: 2025-08-27-11-14-53 +.. gh-issue: 138171 +.. nonce: Suz8ob +.. section: Tools/Demos + +A script for building an iOS XCframework was added. As part of this change, +the top level ``iOS`` folder has been moved to be a subdirectory of the +``Apple`` folder. + +.. + +.. date: 2025-08-21-14-04-50 +.. gh-issue: 137873 +.. nonce: qxffLt +.. section: Tools/Demos + +The iOS test runner has been simplified, resolving some issues that have +been observed using the runner in GitHub Actions and Azure Pipelines test +environments. + +.. + +.. date: 2025-08-06-11-54-55 +.. gh-issue: 137484 +.. nonce: 8iFAQs +.. section: Tools/Demos + +Have ``Tools/wasm/wasi`` put the build Python into a directory named after +the build triple instead of "build". + +.. + +.. date: 2025-08-01-13-27-43 +.. gh-issue: 137025 +.. nonce: ubuhQC +.. section: Tools/Demos + +The ``wasm_build.py`` script has been removed. ``Tools/wasm/emscripten`` +and ``Tools/wasm/wasi`` should be used instead, as described in the `Dev +Guide <https://devguide.python.org/contrib/workflows/compile/>`__. + +.. + +.. date: 2025-07-30-11-15-47 +.. gh-issue: 137248 +.. nonce: 8IxwY3 +.. section: Tools/Demos + +Add a ``--logdir`` option to ``Tools/wasm/wasi`` for specifying where to +write log files. + +.. + +.. date: 2025-07-30-10-28-35 +.. gh-issue: 137243 +.. nonce: NkdUqH +.. section: Tools/Demos + +Have Tools/wasm/wasi detect a WASI SDK install in /opt when it was directly +extracted from a release tarball. + +.. + +.. date: 2025-07-05-15-10-42 +.. gh-issue: 136251 +.. nonce: GRM6o8 +.. section: Tools/Demos + +Fixes and usability improvements for ``Tools/wasm/emscripten/web_example`` + +.. + +.. date: 2025-06-26-15-58-13 +.. gh-issue: 135968 +.. nonce: C4v_-W +.. section: Tools/Demos + +Stubs for ``strip`` are now provided as part of an iOS install. + +.. + +.. date: 2025-06-11-12-14-06 +.. gh-issue: 135379 +.. nonce: 25ttXq +.. section: Tools/Demos + +The cases generator no longer accepts type annotations on stack items. +Conversions to non-default types are now done explicitly in bytecodes.c and +optimizer_bytecodes.c. This will simplify code generation for top-of-stack +caching and other future features. + +.. + +.. date: 2025-05-19-14-57-46 +.. gh-issue: 134215 +.. nonce: sbdDK6 +.. section: Tools/Demos + +:term:`REPL` import autocomplete only suggests private modules when +explicitly specified. + +.. + +.. date: 2025-09-22-15-40-09 +.. gh-issue: 139208 +.. nonce: Tc13dl +.. section: Tests + +Fix regrtest ``--fast-ci --verbose``: don't ignore the ``--verbose`` option +anymore. Patch by Victor Stinner. + +.. + +.. date: 2025-09-21-16-00-30 +.. gh-issue: 138313 +.. nonce: lBx2en +.. section: Tests + +Restore skipped test and add janky workaround to prevent select buildbots +from failing with a ResourceWarning. + +.. + +.. date: 2025-06-26-15-15-35 +.. gh-issue: 135966 +.. nonce: EBpF8Y +.. section: Tests + +The iOS testbed now handles the ``app_packages`` folder as a site directory. + +.. + +.. date: 2025-06-19-15-29-38 +.. gh-issue: 135494 +.. nonce: FVl9a0 +.. section: Tests + +Fix regrtest to support excluding tests from ``--pgo`` tests. Patch by +Victor Stinner. + +.. + +.. date: 2025-06-17-08-48-08 +.. gh-issue: 132815 +.. nonce: CY1Esu +.. section: Tests + +Fix test__opcode: add ``JUMP_BACKWARD`` to specialization stats. + +.. + +.. date: 2025-06-14-13-20-17 +.. gh-issue: 135489 +.. nonce: Uh0yVO +.. section: Tests + +Show verbose output for failing tests during PGO profiling step with +--enable-optimizations. + +.. + +.. date: 2025-06-11-16-52-49 +.. gh-issue: 135401 +.. nonce: ccMXmL +.. section: Tests + +Add a new GitHub CI job to test the :mod:`ssl` module with `AWS-LC +<https://github.com/aws/aws-lc>`_ as the backing cryptography and TLS +library. + +.. + +.. date: 2025-06-04-13-07-44 +.. gh-issue: 135120 +.. nonce: NapnZT +.. section: Tests + +Add :func:`!test.support.subTests`. + +.. + +.. date: 2025-05-23-09-19-52 +.. gh-issue: 134567 +.. nonce: hwEIMb +.. section: Tests + +Expose log formatter to users in TestCase.assertLogs. +:func:`unittest.TestCase.assertLogs` will now optionally accept a formatter +that will be used to format the strings in output if provided. + +.. + +.. date: 2025-05-09-14-54-48 +.. gh-issue: 133744 +.. nonce: LCquu0 +.. section: Tests + +Fix multiprocessing interrupt test. Add an event to synchronize the parent +process with the child process: wait until the child process starts +sleeping. Patch by Victor Stinner. + +.. + +.. date: 2025-05-09-04-11-06 +.. gh-issue: 133682 +.. nonce: -_lwo3 +.. section: Tests + +Fixed test case ``test.test_annotationlib.TestStringFormat.test_displays`` +which ensures proper handling of complex data structures (lists, sets, +dictionaries, and tuples) in string annotations. + +.. + +.. date: 2025-05-08-15-06-01 +.. gh-issue: 133639 +.. nonce: 50-kbV +.. section: Tests + +Fix ``TestPyReplAutoindent.test_auto_indent_default()`` doesn't run +``input_code``. + +.. + +.. date: 2025-10-07-19-31-34 +.. gh-issue: 139700 +.. nonce: vNHU1O +.. section: Security + +Check consistency of the zip64 end of central directory record. Support +records with "zip64 extensible data" if there are no bytes prepended to the +ZIP file. + +.. + +.. date: 2025-09-29-00-01-28 +.. gh-issue: 139400 +.. nonce: X2T-jO +.. section: Security + +:mod:`xml.parsers.expat`: Make sure that parent Expat parsers are only +garbage-collected once they are no longer referenced by subparsers created +by :meth:`~xml.parsers.expat.xmlparser.ExternalEntityParserCreate`. Patch by +Sebastian Pipping. + +.. + +.. date: 2025-09-24-13-39-56 +.. gh-issue: 139283 +.. nonce: jODz_q +.. section: Security + +:mod:`sqlite3`: correctly handle maximum number of rows to fetch in +:meth:`Cursor.fetchmany <sqlite3.Cursor.fetchmany>` and reject negative +values for :attr:`Cursor.arraysize <sqlite3.Cursor.arraysize>`. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-06-27-21-23-19 +.. gh-issue: 136053 +.. nonce: QZxcee +.. section: Security + +:mod:`marshal`: fix a possible crash when deserializing :class:`slice` +objects. + +.. + +.. date: 2025-06-25-14-13-39 +.. gh-issue: 135661 +.. nonce: idjQ0B +.. section: Security + +Fix parsing start and end tags in :class:`html.parser.HTMLParser` according +to the HTML5 standard. + +* Whitespaces no longer accepted between ``</`` and the tag name. + E.g. ``</ script>`` does not end the script section. + +* Vertical tabulation (``\v``) and non-ASCII whitespaces no longer recognized + as whitespaces. The only whitespaces are ``\t\n\r\f`` and space. + +* Null character (U+0000) no longer ends the tag name. + +* Attributes and slashes after the tag name in end tags are now ignored, + instead of terminating after the first ``>`` in quoted attribute value. + E.g. ``</script/foo=">"/>``. + +* Multiple slashes and whitespaces between the last attribute and closing ``>`` + are now ignored in both start and end tags. E.g. ``<a foo=bar/ //>``. + +* Multiple ``=`` between attribute name and value are no longer collapsed. + E.g. ``<a foo==bar>`` produces attribute "foo" with value "=bar". + +.. + +.. date: 2025-06-18-13-34-55 +.. gh-issue: 135661 +.. nonce: NZlpWf +.. section: Security + +Fix CDATA section parsing in :class:`html.parser.HTMLParser` according to +the HTML5 standard: ``] ]>`` and ``]] >`` no longer end the CDATA section. +Add private method ``_set_support_cdata()`` which can be used to specify how +to parse ``<[CDATA[`` --- as a CDATA section in foreign content (SVG or +MathML) or as a bogus comment in the HTML namespace. + +.. + +.. date: 2025-06-18-13-28-08 +.. gh-issue: 102555 +.. nonce: nADrzJ +.. section: Security + +Fix comment parsing in :class:`html.parser.HTMLParser` according to the +HTML5 standard. ``--!>`` now ends the comment. ``-- >`` no longer ends the +comment. Support abnormally ended empty comments ``<-->`` and ``<--->``. + +.. + +.. date: 2025-06-13-15-55-22 +.. gh-issue: 135462 +.. nonce: KBeJpc +.. section: Security + +Fix quadratic complexity in processing specially crafted input in +:class:`html.parser.HTMLParser`. End-of-file errors are now handled +according to the HTML5 specs -- comments and declarations are automatically +closed, tags are ignored. + +.. + +.. date: 2025-06-09-20-38-25 +.. gh-issue: 118350 +.. nonce: KgWCcP +.. section: Security + +Fix support of escapable raw text mode (elements "textarea" and "title") in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-06-02-11-32-23 +.. gh-issue: 135034 +.. nonce: RLGjbp +.. section: Security + +Fixes multiple issues that allowed ``tarfile`` extraction filters +(``filter="data"`` and ``filter="tar"``) to be bypassed using crafted +symlinks and hard links. + +Addresses :cve:`2024-12718`, :cve:`2025-4138`, :cve:`2025-4330`, and +:cve:`2025-4517`. + +.. + +.. date: 2025-05-09-20-22-54 +.. gh-issue: 133767 +.. nonce: kN2i3Q +.. section: Security + +Fix use-after-free in the "unicode-escape" decoder with a non-"strict" error +handler. + +.. + +.. date: 2025-05-07-22-49-27 +.. gh-issue: 133623 +.. nonce: fgWkBm +.. section: Security + +Indicate through :data:`ssl.HAS_PSK_TLS13` whether the :mod:`ssl` module +supports "External PSKs" in TLSv1.3, as described in RFC 9258. Patch by Will +Childs-Klein. + +.. + +.. date: 2025-01-14-11-19-07 +.. gh-issue: 128840 +.. nonce: M1doZW +.. section: Security + +Short-circuit the processing of long IPv6 addresses early in +:mod:`ipaddress` to prevent excessive memory consumption and a minor +denial-of-service. + +.. + +.. date: 2025-10-11-20-03-13 +.. gh-issue: 139482 +.. nonce: du2Stg +.. section: Library + +Optimize :data:`os.environ.clear() <os.environ>` by calling +:manpage:`clearenv(3)` when this function is available. Patch by Victor +Stinner. + +.. + +.. date: 2025-10-11-17-41-26 +.. gh-issue: 139958 +.. nonce: AnCakj +.. section: Library + +The ``application/toml`` mime type is now supported by :mod:`mimetypes`. +Patch by Gil Forcada. + +.. + +.. date: 2025-10-11-14-37-42 +.. gh-issue: 139823 +.. nonce: uGF4oh +.. section: Library + +:mod:`ensurepip` now fails with a nicer error message when the :mod:`zlib` +module is not available. + +.. + +.. date: 2025-10-11-10-02-56 +.. gh-issue: 139905 +.. nonce: UyJIR_ +.. section: Library + +Add suggestion to error message for :class:`typing.Generic` subclasses when +``cls.__parameters__`` is missing due to a parent class failing to call +:meth:`super().__init_subclass__() <object.__init_subclass__>` in its +``__init_subclass__``. + +.. + +.. date: 2025-10-10-11-22-50 +.. gh-issue: 139894 +.. nonce: ECAXqj +.. section: Library + +Fix incorrect sharing of current task with the child process while forking +in :mod:`asyncio`. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-09-21-37-20 +.. gh-issue: 139845 +.. nonce: dzx5UP +.. section: Library + +Fix to not print KeyboardInterrupt twice in default asyncio REPL. + +.. + +.. date: 2025-10-09-13-48-28 +.. gh-issue: 139783 +.. nonce: __NUgo +.. section: Library + +Fix :func:`inspect.getsourcelines` for the case when a decorator is followed +by a comment or an empty line. + +.. + +.. date: 2025-10-09-03-06-19 +.. gh-issue: 139809 +.. nonce: lzHJNu +.. section: Library + +Prevent premature colorization of subparser ``prog`` in +:meth:`argparse.ArgumentParser.add_subparsers` to respect color environment +variable changes after parser creation. + +.. + +.. date: 2025-10-08-00-06-30 +.. gh-issue: 139736 +.. nonce: baPeBd +.. section: Library + +Fix excessive indentation in the default :mod:`argparse` +:class:`!HelpFormatter`. Patch by Alexander Edland. + +.. + +.. date: 2025-10-02-17-40-10 +.. gh-issue: 70765 +.. nonce: zVlLZn +.. section: Library + +:mod:`http.server`: fix default handling of HTTP/0.9 requests in +:class:`~http.server.BaseHTTPRequestHandler`. Previously, +:meth:`!BaseHTTPRequestHandler.parse_request` incorrectly waited for headers +in the request although those are not supported in HTTP/0.9. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-10-02-15-45-08 +.. gh-issue: 139322 +.. nonce: rouPGj +.. section: Library + +Fix :func:`os.getlogin` error handling: fix the error number. Patch by +Victor Stinner. + +.. + +.. date: 2025-10-01-20-30-03 +.. gh-issue: 135953 +.. nonce: NAofJl +.. section: Library + +Add a Gecko format output to the tachyon profiler via ``--gecko``. + +.. + +.. date: 2025-09-29-14-15-20 +.. gh-issue: 139184 +.. nonce: dNl9O4 +.. section: Library + +:func:`os.forkpty` does now make the returned file descriptor +non-inheritable. + +.. + +.. date: 2025-09-28-16-34-11 +.. gh-issue: 139391 +.. nonce: nRFnmx +.. section: Library + +Fix an issue when, on non-Windows platforms, it was not possible to +gracefully exit a ``python -m asyncio`` process suspended by Ctrl+Z and +later resumed by :manpage:`fg` other than with :manpage:`kill`. + +.. + +.. date: 2025-09-27-08-26-31 +.. gh-issue: 139374 +.. nonce: hfh-dl +.. section: Library + +:mod:`timeit`: Add color to error tracebacks. + +.. + +.. date: 2025-09-26-18-04-28 +.. gh-issue: 90949 +.. nonce: YHjSzX +.. section: Library + +Add +:meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold` +and +:meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification` +to :ref:`xmlparser <xmlparser-objects>` objects to tune protections against +`billion laughs <https://en.wikipedia.org/wiki/Billion_laughs_attack>`_ +attacks. Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-25-07-33-43 +.. gh-issue: 139312 +.. nonce: ygE8AC +.. section: Library + +Upgrade bundled libexpat to 2.7.3 + +.. + +.. date: 2025-09-24-14-17-34 +.. gh-issue: 139289 +.. nonce: Vmk25k +.. section: Library + +Do a real lazy-import on :mod:`rlcompleter` in :mod:`pdb` and restore the +existing completer after importing :mod:`rlcompleter`. + +.. + +.. date: 2025-09-22-14-40-11 +.. gh-issue: 90949 +.. nonce: UM35nb +.. section: Library + +Add :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerActivationThreshold` +and :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerMaximumAmplification` +to :ref:`xmlparser <xmlparser-objects>` objects to tune protections against +disproportional amounts of dynamic memory usage from within an Expat parser. +Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-22-11-30-45 +.. gh-issue: 67795 +.. nonce: fROoZt +.. section: Library + +Functions that take timestamp or timeout arguments now accept any real +numbers (such as :class:`~decimal.Decimal` and +:class:`~fractions.Fraction`), not only integers or floats, although this +does not improve precision. + +.. + +.. date: 2025-09-22-11-19-05 +.. gh-issue: 95953 +.. nonce: 7oLoag +.. section: Library + +A CSS class, ``diff_changed``, was added to the changed lines in the +``make_table`` output of :class:`difflib.HtmlDiff`. Patch by Katie Gardner. + +.. + +.. date: 2025-09-21-15-58-57 +.. gh-issue: 139210 +.. nonce: HGbMvz +.. section: Library + +Fix use-after-free when reporting unknown event in +:func:`xml.etree.ElementTree.iterparse`. Patch by Ken Jin. + +.. + +.. date: 2025-09-20-17-50-31 +.. gh-issue: 138860 +.. nonce: Y9JXap +.. section: Library + +Lazy import :mod:`rlcompleter` in :mod:`pdb` to avoid deadlock in +subprocess. + +.. + +.. date: 2025-09-19-09-36-42 +.. gh-issue: 112729 +.. nonce: mmty0_ +.. section: Library + +Fix crash when calling :func:`concurrent.interpreters.create` when the +process is out of memory. + +.. + +.. date: 2025-09-19-07-41-52 +.. gh-issue: 126016 +.. nonce: Uz9W6h +.. section: Library + +Fix an assertion failure when sending :exc:`KeyboardInterrupt` to a Python +process running a subinterpreter in a separate thread. + +.. + +.. date: 2025-09-18-14-21-57 +.. gh-issue: 118803 +.. nonce: 2JPbto +.. section: Library + +:class:`collections.abc.ByteString` has been removed from +``collections.abc.__all__``, and :class:`typing.ByteString` has been removed +from ``typing.__all__``. The former has been deprecated since Python 3.12, +and the latter has been deprecated since Python 3.9. Both classes are +scheduled for removal in Python 3.17. + +Additionally, the following statements now cause ``DeprecationWarning``\ s +to be emitted at runtime: ``from collections.abc import ByteString``, ``from +typing import ByteString``, ``import collections.abc; +collections.abc.ByteString`` and ``import typing; typing.ByteString``. Both +classes already caused ``DeprecationWarning``\ s to be emitted if they were +subclassed or used as the second argument to ``isinstance()`` or +``issubclass()``, but they did not previously lead to +``DeprecationWarning``\ s if they were merely imported or accessed from +their respective modules. + +.. + +.. date: 2025-09-18-05-32-18 +.. gh-issue: 135729 +.. nonce: 8AmMza +.. section: Library + +Fix unraisable exception during finalization when using +:mod:`concurrent.interpreters` in the REPL. + +.. + +.. date: 2025-09-17-21-54-53 +.. gh-issue: 139076 +.. nonce: 2eX9lG +.. section: Library + +Fix a bug in the :mod:`pydoc` module that was hiding functions in a Python +module if they were implemented in an extension module and the module did +not have ``__all__``. + +.. + +.. date: 2025-09-17-21-52-30 +.. gh-issue: 139090 +.. nonce: W7vbhF +.. section: Library + +Add :data:`os.RWF_DONTCACHE` constant for Linux 6.14+. + +.. + +.. date: 2025-09-17-19-08-34 +.. gh-issue: 139065 +.. nonce: Hu8fM5 +.. section: Library + +Fix trailing space before a wrapped long word if the line length is exactly +*width* in :mod:`textwrap`. + +.. + +.. date: 2025-09-17-12-07-21 +.. gh-issue: 139001 +.. nonce: O6tseN +.. section: Library + +Fix race condition in :class:`pathlib.Path` on the internal ``_raw_paths`` +field. + +.. + +.. date: 2025-09-17-08-32-43 +.. gh-issue: 138813 +.. nonce: LHkHjX +.. section: Library + +:class:`!multiprocessing.BaseProcess` defaults ``kwargs`` to ``None`` +instead of a shared dictionary. + +.. + +.. date: 2025-09-16-19-05-29 +.. gh-issue: 138998 +.. nonce: URl0Y_ +.. section: Library + +Update bundled libexpat to 2.7.2 + +.. + +.. date: 2025-09-16-16-46-58 +.. gh-issue: 138993 +.. nonce: -8s8_T +.. section: Library + +Dedent :data:`credits` text. + +.. + +.. date: 2025-09-16-15-56-29 +.. gh-issue: 118803 +.. nonce: aOPtmL +.. section: Library + +Add back :class:`collections.abc.ByteString` and :class:`typing.ByteString`. +Both had been removed in prior alpha, beta and release candidates for Python +3.14, but their removal has now been postponed to Python 3.17. + +.. + +.. date: 2025-09-15-19-29-12 +.. gh-issue: 130567 +.. nonce: shDEnT +.. section: Library + +Fix possible crash in :func:`locale.strxfrm` due to a platform bug on macOS. + +.. + +.. date: 2025-09-15-13-09-19 +.. gh-issue: 137226 +.. nonce: HH3_ik +.. section: Library + +Fix :func:`typing.get_type_hints` calls on generic :class:`typing.TypedDict` +classes defined with string annotations. + +.. + +.. date: 2025-09-15-08-57-39 +.. gh-issue: 138899 +.. nonce: Uh6fvY +.. section: Library + +Executing ``quit`` command in :mod:`pdb` will raise :exc:`bdb.BdbQuit` when +:mod:`pdb` is started from an asyncio console using :func:`breakpoint` or +:func:`pdb.set_trace`. + +.. + +.. date: 2025-09-12-01-01-05 +.. gh-issue: 138804 +.. nonce: 46ZukT +.. section: Library + +Raise :exc:`TypeError` instead of :exc:`AttributeError` when an argument of +incorrect type is passed to :func:`shlex.quote`. This restores the behavior +of the function prior to 3.14. + +.. + +.. date: 2025-09-11-11-09-28 +.. gh-issue: 138779 +.. nonce: TNZnLr +.. section: Library + +Support device numbers larger than ``2**63-1`` for the +:attr:`~os.stat_result.st_rdev` field of the :class:`os.stat_result` +structure. + +.. + +.. date: 2025-09-10-13-32-25 +.. gh-issue: 138682 +.. nonce: iExqx1 +.. section: Library + +Added symmetric difference support to :class:`collections.Counter` objects. + +.. + +.. date: 2025-09-10-10-11-59 +.. gh-issue: 138712 +.. nonce: avrPG5 +.. section: Library + +Add :const:`os.NODEV`. + +.. + +.. date: 2025-09-10-10-02-59 +.. gh-issue: 128636 +.. nonce: ldRKGZ +.. section: Library + +Fix crash in PyREPL when os.environ is overwritten with an invalid value for +mac + +.. + +.. date: 2025-09-09-17-57-49 +.. gh-issue: 138720 +.. nonce: hAtsm- +.. section: Library + +Fix an issue where :class:`io.BufferedWriter` and :class:`io.BufferedRandom` +had different definitions of "closed" for :meth:`~io.IOBase.close` and +:meth:`~io.IOBase.flush` which resulted in an exception when close called +flush but flush thought the file was already closed. + +.. + +.. date: 2025-09-09-10-48-26 +.. gh-issue: 138706 +.. nonce: xB--LX +.. section: Library + +Update :mod:`unicodedata` database to Unicode 17.0.0. + +.. + +.. date: 2025-09-08-17-32-02 +.. gh-issue: 76007 +.. nonce: peEgcr +.. section: Library + +Deprecate ``__version__`` from a number of standard library modules. Patch +by Hugo van Kemenade. + +.. + +.. date: 2025-09-06-20-09-32 +.. gh-issue: 138535 +.. nonce: mlntEe +.. section: Library + +Speed up :func:`os.stat` for files with reasonable timestamps. Contributed +by Jeffrey Bosboom. + +.. + +.. date: 2025-09-06-14-56-40 +.. gh-issue: 116946 +.. nonce: GGIeyO +.. section: Library + +:mod:`curses.panel`: the type of :func:`curses.panel.new_panel` is now +immutable. Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-06-14-54-01 +.. gh-issue: 116946 +.. nonce: hzQEWI +.. section: Library + +:mod:`zlib`: the types of :func:`zlib.compressobj` and +:func:`zlib.decompressobj` are now immutable. Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-06-14-53-19 +.. gh-issue: 116946 +.. nonce: c-npxd +.. section: Library + +:mod:`os`: the :class:`os.DirEntry` type and the type of :func:`os.scandir` +are now immutable. Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-06-14-47-23 +.. gh-issue: 116946 +.. nonce: hj_u1t +.. section: Library + +:mod:`tkinter`: the types :class:`!_tkinter.Tcl_Obj` (wrapper for Tcl +objects), :class:`!_tkinter.tktimertoken` (obtained by calling +``createtimerhandler()`` on a :attr:`Tk <tkinter.Tk.tk>` application) and +:class:`!_tkinter.tkapp` (the runtime type of Tk applications) are now +immutable. Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-06-11-26-21 +.. gh-issue: 138514 +.. nonce: 66ltOb +.. section: Library + +Raise :exc:`ValueError` when a multi-character string is passed to the +*echo_char* parameter of :func:`getpass.getpass`. Patch by Benjamin Johnson. + +.. + +.. date: 2025-09-05-21-10-24 +.. gh-issue: 137706 +.. nonce: 0EztiJ +.. section: Library + +Fix the partial evaluation of annotations that use ``typing.Annotated[T, +x]`` where ``T`` is a forward reference. + +.. + +.. date: 2025-09-05-15-35-59 +.. gh-issue: 88375 +.. nonce: dC491a +.. section: Library + +Fix normalization of the ``robots.txt`` rules and URLs in the +:mod:`urllib.robotparser` module. No longer ignore trailing ``?``. +Distinguish raw special characters ``?``, ``=`` and ``&`` from the +percent-encoded ones. + +.. + +.. date: 2025-09-05-07-50-18 +.. gh-issue: 138515 +.. nonce: E3M-pu +.. section: Library + +:mod:`email` is added to Emscripten build. + +.. + +.. date: 2025-09-05-05-53-43 +.. gh-issue: 99948 +.. nonce: KMSlG6 +.. section: Library + +:func:`ctypes.util.find_library` now works in Emscripten build. + +.. + +.. date: 2025-09-04-15-18-11 +.. gh-issue: 111788 +.. nonce: tuTEM5 +.. section: Library + +Fix parsing errors in the :mod:`urllib.robotparser` module. Don't fail +trying to parse weird paths. Don't fail trying to decode non-UTF-8 +``robots.txt`` files. + +.. + +.. date: 2025-09-03-15-20-10 +.. gh-issue: 138432 +.. nonce: RMc7UX +.. section: Library + +:meth:`zoneinfo.reset_tzpath` will now convert any :class:`os.PathLike` +objects it receives into strings before adding them to ``TZPATH``. It will +raise ``TypeError`` if anything other than a string is found after this +conversion. If given an :class:`os.PathLike` object that represents a +relative path, it will now raise ``ValueError`` instead of ``TypeError``, +and present a more informative error message. + +.. + +.. date: 2025-09-03-09-03-11 +.. gh-issue: 132657 +.. nonce: cbAIDh +.. section: Library + +Improve the scaling of :func:`copy.copy` and :func:`copy.deepcopy` in the +free-threading build. + +.. + +.. date: 2025-09-02-10-27-21 +.. gh-issue: 116946 +.. nonce: VxXNGD +.. section: Library + +The types of :func:`select.poll` and :func:`select.epoll` objects are now +immutable. Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-02-10-23-09 +.. gh-issue: 116946 +.. nonce: U6RpwK +.. section: Library + +The :class:`!_random.Random` C type is now immutable. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-08-31-22-10-22 +.. gh-issue: 57911 +.. nonce: N_Ixtv +.. section: Library + +When extracting tar files on Windows, slashes in symlink targets will be +replaced by backslashes to prevent corrupted links. + +.. + +.. date: 2025-08-31-12-34-02 +.. gh-issue: 138205 +.. nonce: iHXb1z +.. section: Library + +Removed the :meth:`~mmap.mmap.resize` method on platforms that don't support +the underlying syscall, instead of raising a :exc:`SystemError`. + +.. + +.. date: 2025-08-31-09-06-49 +.. gh-issue: 138008 +.. nonce: heOvsU +.. section: Library + +Fix segmentation faults in the :mod:`ctypes` module due to invalid +:attr:`~ctypes._CFuncPtr.argtypes`. Patch by Dung Nguyen. + +.. + +.. date: 2025-08-30-17-58-04 +.. gh-issue: 138252 +.. nonce: CDiEby +.. section: Library + +:mod:`ssl`: :class:`~ssl.SSLContext` objects can now set client and server +TLS signature algorithms. If Python has been built with OpenSSL 3.5 or +later, :class:`~ssl.SSLSocket` objects can return the signature algorithms +selected on a connection. + +.. + +.. date: 2025-08-30-10-58-15 +.. gh-issue: 138253 +.. nonce: 9Ehj-N +.. section: Library + +Add the *block* parameter in the :meth:`!put` and :meth:`!get` methods of +the :mod:`concurrent.interpreters` queues for compatibility with the +:class:`queue.Queue` interface. + +.. + +.. date: 2025-08-30-10-04-28 +.. gh-issue: 60462 +.. nonce: yh_vDc +.. section: Library + +Fix :func:`locale.strxfrm` on Solaris (and possibly other platforms). + +.. + +.. date: 2025-08-29-12-56-55 +.. gh-issue: 138239 +.. nonce: uthZFI +.. section: Library + +The REPL now highlights :keyword:`type` as a soft keyword in :ref:`type +statements <type>`. + +.. + +.. date: 2025-08-29-12-05-33 +.. gh-issue: 78502 +.. nonce: VpIMxg +.. section: Library + +:class:`mmap.mmap` now has a *trackfd* parameter on Windows; if it is +``False``, the file handle corresponding to *fileno* will not be duplicated. + +.. + +.. date: 2025-08-28-13-20-09 +.. gh-issue: 138204 +.. nonce: 8oLOud +.. section: Library + +Forbid expansion of shared anonymous :mod:`memory maps <mmap>` on Linux, +which caused a bus error. + +.. + +.. date: 2025-08-27-17-05-36 +.. gh-issue: 138010 +.. nonce: ZZJmPL +.. section: Library + +Fix an issue where defining a class with a +:deco:`warnings.deprecated`-decorated base class may not invoke the correct +:meth:`~object.__init_subclass__` method in cases involving multiple +inheritance. Patch by Brian Schubert. + +.. + +.. date: 2025-08-25-22-38-03 +.. gh-issue: 134716 +.. nonce: kyYKeX +.. section: Library + +Add support of regular expressions in the :option:`-W` option and the +:envvar:`PYTHONWARNINGS` environment variable. + +.. + +.. date: 2025-08-25-18-06-04 +.. gh-issue: 138133 +.. nonce: Zh9rGo +.. section: Library + +Prevent infinite traceback loop when sending CTRL^C to Python through +``strace``. + +.. + +.. date: 2025-08-25-16-22-32 +.. gh-issue: 138122 +.. nonce: eMNDZ1 +.. section: Library + +Implement :pep:`799` -- A dedicated profiling package for organizing Python +profiling tools. Patch by Pablo Galindo. + +.. + +.. date: 2025-08-24-02-04-32 +.. gh-issue: 138092 +.. nonce: V4-wTO +.. section: Library + +Fixed a bug in :meth:`mmap.mmap.flush` where calling with only an offset +parameter would fail. + +.. + +.. date: 2025-08-22-12-48-14 +.. gh-issue: 138044 +.. nonce: lEQULC +.. section: Library + +Remove compatibility shim for deprecated parameter *package* in +:func:`importlib.resources.files`. Patch by Semyon Moroz. + +.. + +.. date: 2025-08-22-09-53-45 +.. gh-issue: 86819 +.. nonce: ECxvwx +.. section: Library + +:mod:`socket`: Add missing constants for ISO-TP sockets. + +.. + +.. date: 2025-08-19-00-12-57 +.. gh-issue: 137884 +.. nonce: 4faCA_ +.. section: Library + +Add :func:`threading.get_native_id` support for Illumos/Solaris. Patch by +Yüce Tekol. + +.. + +.. date: 2025-08-18-16-02-51 +.. gh-issue: 134869 +.. nonce: GnAjnU +.. section: Library + +Fix an issue where pressing Ctrl+C during tab completion in the REPL would +leave the autocompletion menu in a corrupted state. + +.. + +.. date: 2025-08-18-07-10-55 +.. gh-issue: 137840 +.. nonce: 9b7AnG +.. section: Library + +:class:`typing.TypedDict` now supports the ``closed`` and ``extra_items`` +keyword arguments (as described in :pep:`728`) to control whether additional +non-required keys are allowed and to specify their value type. + +.. + +.. date: 2025-08-17-10-22-31 +.. gh-issue: 132947 +.. nonce: XR4MJ8 +.. section: Library + +Applied changes to ``importlib.metadata`` from `importlib_metadata 8.7 +<https://importlib-metadata.readthedocs.io/en/latest/history.html#v8-7-0>`_, +including ``dist`` now disallowed for ``EntryPoints.select``; deferred +imports for faster import times; added support for metadata with newlines +(python/cpython#119650); and ``metadata()`` function now returns ``None`` +when a metadata directory is present but no metadata is present. + +.. + +.. date: 2025-08-16-18-11-41 +.. gh-issue: 90548 +.. nonce: q3aJUK +.. section: Library + +Fix ``musl`` detection for :func:`platform.libc_ver` on Alpine Linux if +compiled with --strip-all. + +.. + +.. date: 2025-08-16-16-04-15 +.. gh-issue: 137317 +.. nonce: Dl13B5 +.. section: Library + +:func:`inspect.signature` now correctly handles classes that use a +descriptor on a wrapped :meth:`!__init__` or :meth:`!__new__` method. +Contributed by Yongyu Yan. + +.. + +.. date: 2025-08-16-09-02-11 +.. gh-issue: 137754 +.. nonce: mCev1Y +.. section: Library + +Fix import of the :mod:`zoneinfo` module if the C implementation of the +:mod:`datetime` module is not available. + +.. + +.. date: 2025-08-14-10-27-07 +.. gh-issue: 125854 +.. nonce: vDzFcZ +.. section: Library + +Improve error messages for invalid category in :func:`warnings.warn`. + +.. + +.. date: 2025-08-14-00-00-12 +.. gh-issue: 137729 +.. nonce: i9NSKP +.. section: Library + +:func:`locale.setlocale` now supports language codes with ``@``-modifiers. +``@``-modifier are no longer silently removed in :func:`locale.getlocale`, +but included in the language code. + +.. + +.. date: 2025-08-13-10-50-22 +.. gh-issue: 73487 +.. nonce: DUHbBq +.. section: Library + +Speedup processing arguments (up to 1.5x) in the :mod:`decimal` module +methods, that now using :c:macro:`METH_FASTCALL` calling convention. Patch +by Sergey B Kirpichev. + +.. + +.. date: 2025-08-11-14-18-32 +.. gh-issue: 137634 +.. nonce: M7iBG6 +.. section: Library + +Calendar pages generated by the :class:`calendar.HTMLCalendar` class now +support dark mode and have been migrated to the HTML5 standard for improved +accessibility. + +.. + +.. date: 2025-08-11-05-05-08 +.. gh-issue: 137630 +.. nonce: 9lmqyc +.. section: Library + +The :mod:`!_interpreters` module now uses Argument Clinic to parse +arguments. Patch by Adam Turner. + +.. + +.. date: 2025-08-09-08-53-32 +.. gh-issue: 137583 +.. nonce: s6OZud +.. section: Library + +Fix a deadlock introduced in 3.13.6 when a call to :meth:`ssl.SSLSocket.recv +<socket.socket.recv>` was blocked in one thread, and then another method on +the object (such as :meth:`ssl.SSLSocket.send <socket.socket.send>`) was +subsequently called in another thread. + +.. + +.. date: 2025-08-08-21-20-14 +.. gh-issue: 92936 +.. nonce: rOgG1S +.. section: Library + +Update regex used by ``http.cookies.SimpleCookie`` to handle values +containing double quotes. + +.. + +.. date: 2025-08-08-15-00-38 +.. gh-issue: 137426 +.. nonce: lW-Rk2 +.. section: Library + +Remove the code deprecation of ``importlib.abc.ResourceLoader``. It is +documented as deprecated, but left for backwards compatibility with other +classes in ``importlib.abc``. + +.. + +.. date: 2025-08-07-17-18-57 +.. gh-issue: 137490 +.. nonce: s89ieZ +.. section: Library + +Handle :data:`~errno.ECANCELED` in the same way as :data:`~errno.EINTR` in +:func:`signal.sigwaitinfo` on NetBSD. + +.. + +.. date: 2025-08-07-15-07-44 +.. gh-issue: 137512 +.. nonce: j2or5h +.. section: Library + +Add new constants in the :mod:`resource` module: +:data:`~resource.RLIMIT_NTHR`, :data:`~resource.RLIMIT_UMTXP`, +:data:`~resource.RLIMIT_PIPEBUF`, :data:`~resource.RLIMIT_THREADS`, +:data:`~resource.RLIM_SAVED_CUR`, and :data:`~resource.RLIM_SAVED_MAX`. + +.. + +.. date: 2025-08-07-12-32-23 +.. gh-issue: 137044 +.. nonce: abNoIy +.. section: Library + +:data:`resource.RLIM_INFINITY` is now always a positive integer. On all +supported platforms, it is larger than any limited resource value, which +simplifies comparison of the resource values. Previously, it could be +negative, such as -1 or -3, depending on platform. + +.. + +.. date: 2025-08-06-23-16-42 +.. gh-issue: 137477 +.. nonce: bk6BDV +.. section: Library + +Fix :func:`!inspect.getblock`, :func:`inspect.getsourcelines` and +:func:`inspect.getsource` for generator expressions. + +.. + +.. date: 2025-08-06-16-54-22 +.. gh-issue: 137481 +.. nonce: eSTkK0 +.. section: Library + +Calendar uses the lengths of the locale's weekdays to decide if the width +requires abbreviation. + +.. + +.. date: 2025-08-06-16-13-47 +.. gh-issue: 137466 +.. nonce: Whv0-A +.. section: Library + +Remove undocumented :func:`!glob.glob0` and :func:`!glob.glob1` functions, +which have been deprecated since Python 3.13. Use :func:`glob.glob` and pass +a directory to its *root_dir* argument instead. + +.. + +.. date: 2025-08-03-13-16-39 +.. gh-issue: 137044 +.. nonce: 0hPVL_ +.. section: Library + +Return large limit values as positive integers instead of negative integers +in :func:`resource.getrlimit`. Accept large values and reject negative +values (except :data:`~resource.RLIM_INFINITY`) for limits in +:func:`resource.setrlimit`. + +.. + +.. date: 2025-08-03-00-36-57 +.. gh-issue: 115766 +.. nonce: nJCFkW +.. section: Library + +Fix :attr:`!ipaddress.IPv4Interface.is_unspecified`. + +.. + +.. date: 2025-08-01-23-52-49 +.. gh-issue: 75989 +.. nonce: 5aYXNJ +.. section: Library + +:func:`tarfile.TarFile.extractall` and :func:`tarfile.TarFile.extract` now +overwrite symlinks when extracting hardlinks. (Contributed by Alexander +Enrique Urieles Nieto in :gh:`75989`.) + +.. + +.. date: 2025-08-01-23-11-25 +.. gh-issue: 137017 +.. nonce: 0yGcNc +.. section: Library + +Fix :obj:`threading.Thread.is_alive` to remain ``True`` until the underlying +OS thread is fully cleaned up. This avoids false negatives in edge cases +involving thread monitoring or premature :obj:`threading.Thread.is_alive` +calls. + +.. + +.. date: 2025-08-01-15-07-59 +.. gh-issue: 137273 +.. nonce: 4V8Xmv +.. section: Library + +Fix debug assertion failure in :func:`locale.setlocale` on Windows. + +.. + +.. date: 2025-07-31-16-43-16 +.. gh-issue: 137191 +.. nonce: FIogE8 +.. section: Library + +Fix how type parameters are collected, when :class:`typing.Protocol` are +specified with explicit parameters. Now, :class:`typing.Generic` and +:class:`typing.Protocol` always dictate the parameter number and parameter +ordering of types. Previous behavior was a bug. + +.. + +.. date: 2025-07-31-10-31-56 +.. gh-issue: 137282 +.. nonce: GOCwIC +.. section: Library + +Fix tab completion and :func:`dir` on :mod:`concurrent.futures`. + +.. + +.. date: 2025-07-30-18-07-33 +.. gh-issue: 137257 +.. nonce: XBtzf2 +.. section: Library + +Bump the version of pip bundled in ensurepip to version 25.2 + +.. + +.. date: 2025-07-30-17-42-36 +.. gh-issue: 137239 +.. nonce: qSpj32 +.. section: Library + +:mod:`heapq`: Update :data:`!heapq.__all__` with ``*_max`` functions. + +.. + +.. date: 2025-07-30-11-12-22 +.. gh-issue: 124503 +.. nonce: d4hc7b +.. section: Library + +:func:`ast.literal_eval` is 10-20% faster for small inputs. + +.. + +.. date: 2025-07-29-21-18-31 +.. gh-issue: 137226 +.. nonce: B_4lpu +.. section: Library + +Fix behavior of :meth:`annotationlib.ForwardRef.evaluate` when the +*type_params* parameter is passed and the name of a type param is also +present in an enclosing scope. + +.. + +.. date: 2025-07-29-05-12-50 +.. gh-issue: 137197 +.. nonce: bMK3sO +.. section: Library + +:class:`~ssl.SSLContext` objects can now set TLS 1.3 cipher suites via +:meth:`~ssl.SSLContext.set_ciphersuites`. + +.. + +.. date: 2025-07-28-23-11-29 +.. gh-issue: 81325 +.. nonce: jMJFBe +.. section: Library + +:class:`tarfile.TarFile` now accepts a :term:`path-like <path-like object>` +when working on a tar archive. (Contributed by Alexander Enrique Urieles +Nieto in :gh:`81325`.) + +.. + +.. date: 2025-07-28-20-48-32 +.. gh-issue: 137185 +.. nonce: fgI7-B +.. section: Library + +Fix a potential async-signal-safety issue in :mod:`faulthandler` when +printing C stack traces. + +.. + +.. date: 2025-07-27-17-03-17 +.. gh-issue: 133951 +.. nonce: 7kwt78 +.. section: Library + +Remove lib64-lib symlink creation when creating new virtual environments in +:mod:`venv` module + +.. + +.. date: 2025-07-25-09-21-56 +.. gh-issue: 130522 +.. nonce: Crwq68 +.. section: Library + +Fix unraisable :exc:`TypeError` raised during :term:`interpreter shutdown` +in the :mod:`threading` module. + +.. + +.. date: 2025-07-24-00-38-07 +.. gh-issue: 137059 +.. nonce: fr64oW +.. section: Library + +Fix handling of file URLs with a Windows drive letter in the URL authority +by :func:`urllib.request.url2pathname`. This fixes a regression in earlier +pre-releases of Python 3.14. + +.. + +.. date: 2025-07-23-11-59-48 +.. gh-issue: 136980 +.. nonce: BIJzkB +.. section: Library + +Remove unused C tracing code in bdb for event type ``c_call``, ``c_return`` +and ``c_exception`` + +.. + +.. date: 2025-07-23-00-35-29 +.. gh-issue: 130577 +.. nonce: c7EITy +.. section: Library + +:mod:`tarfile` now validates archives to ensure member offsets are +non-negative. (Contributed by Alexander Enrique Urieles Nieto in +:gh:`130577`.) + +.. + +.. date: 2025-07-21-22-35-50 +.. gh-issue: 136170 +.. nonce: QUlc78 +.. section: Library + +Removed the unreleased ``zipfile.ZipFile.data_offset`` property added in +3.14.0a7 as it wasn't fully clear which behavior it should have in some +situations so the result was not always what a user might expect. + +.. + +.. date: 2025-07-21-20-00-42 +.. gh-issue: 121237 +.. nonce: DyxNqo +.. section: Library + +Support ``%:z`` directive for :meth:`datetime.datetime.strptime`, +:meth:`datetime.time.strptime` and :func:`time.strptime`. Patch by Lucas +Esposito and Semyon Moroz. + +.. + +.. date: 2025-07-21-16-13-20 +.. gh-issue: 136929 +.. nonce: obKZ2S +.. section: Library + +Ensure that hash functions guaranteed to be always *available* exist as +attributes of :mod:`hashlib` even if they will not work at runtime due to +missing backend implementations. For instance, ``hashlib.md5`` will no +longer raise :exc:`AttributeError` if OpenSSL is not available and Python +has been built without MD5 support. Patch by Bénédikt Tran. + +.. + +.. date: 2025-07-21-16-10-24 +.. gh-issue: 124621 +.. nonce: wyoWc1 +.. section: Library + +pyrepl now works in Emscripten. + +.. + +.. date: 2025-07-21-15-40-00 +.. gh-issue: 136914 +.. nonce: -GNG-d +.. section: Library + +Fix retrieval of :attr:`doctest.DocTest.lineno` for objects decorated with +:func:`functools.cache` or :class:`functools.cached_property`. + +.. + +.. date: 2025-07-21-11-56-47 +.. gh-issue: 136912 +.. nonce: zWosAL +.. section: Library + +:func:`hmac.digest` now properly handles large keys and messages by falling +back to the pure Python implementation when necessary. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-07-21-01-16-32 +.. gh-issue: 83424 +.. nonce: Y3tEV4 +.. section: Library + +Allows creating a :class:`ctypes.CDLL` without name when passing a handle as +an argument. + +.. + +.. date: 2025-07-20-16-56-55 +.. gh-issue: 135228 +.. nonce: n_XIao +.. section: Library + +When :mod:`dataclasses` replaces a class with a slotted dataclass, the +original class can now be garbage collected again. Earlier changes in Python +3.14 caused this class to always remain in existence together with the +replacement class synthesized by :mod:`dataclasses`. + +.. + +.. date: 2025-07-20-16-02-00 +.. gh-issue: 136874 +.. nonce: cLC3o1 +.. section: Library + +Discard URL query and fragment in :func:`urllib.request.url2pathname`. + +.. + +.. date: 2025-07-20-10-21-49 +.. gh-issue: 136787 +.. nonce: _0Rbp_ +.. section: Library + +:mod:`hashlib`: improve exception messages when a hash algorithm is not +recognized, blocked by the current security policy or incompatible with the +desired operation (for instance, using HMAC with SHAKE). Patch by Bénédikt +Tran. + +.. + +.. date: 2025-07-19-16-20-54 +.. gh-issue: 130645 +.. nonce: O-dYcN +.. section: Library + +Enable color help by default in :mod:`argparse`. + +.. + +.. date: 2025-07-19-15-40-47 +.. gh-issue: 131724 +.. nonce: LS59nA +.. section: Library + +In :mod:`http.client`, a new *max_response_headers* keyword-only parameter +has been added to :class:`~http.client.HTTPConnection` and +:class:`~http.client.HTTPSConnection` constructors. This parameter sets the +maximum number of allowed response headers, helping to prevent +denial-of-service attacks. + +.. + +.. date: 2025-07-19-11-53-19 +.. gh-issue: 135427 +.. nonce: iJM_X2 +.. section: Library + +With :option:`-Werror <-W>`, the DeprecationWarning emitted by +:py:func:`os.fork` and :py:func:`os.forkpty` in mutli-threaded processes is +now raised as an exception. Previously it was silently ignored. Patch by +Rani Pinchuk. + +.. + +.. date: 2025-07-17-16-12-23 +.. gh-issue: 136234 +.. nonce: VmTxtj +.. section: Library + +Fix :meth:`asyncio.WriteTransport.writelines` to be robust to connection +failure, by using the same behavior as +:meth:`~asyncio.WriteTransport.write`. + +.. + +.. date: 2025-07-16-09-45-58 +.. gh-issue: 53144 +.. nonce: mrKwMW +.. section: Library + +:mod:`!encodings.aliases`: Add ``latin_N`` aliases + +.. + +.. date: 2025-07-15-16-37-34 +.. gh-issue: 136669 +.. nonce: Yexwah +.. section: Library + +:mod:`!_asyncio` is now statically linked for improved performance. + +.. + +.. date: 2025-07-13-13-31-22 +.. gh-issue: 136134 +.. nonce: mh6VjS +.. section: Library + +:meth:`!SMTP.auth_cram_md5` now raises an :exc:`~smtplib.SMTPException` +instead of a :exc:`ValueError` if Python has been built without MD5 support. +In particular, :class:`~smtplib.SMTP` clients will not attempt to use this +method even if the remote server is assumed to support it. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-07-13-11-20-05 +.. gh-issue: 136134 +.. nonce: xhh0Kq +.. section: Library + +:meth:`IMAP4.login_cram_md5 <imaplib.IMAP4.login_cram_md5>` now raises an +:exc:`IMAP4.error <imaplib.IMAP4.error>` if CRAM-MD5 authentication is not +supported. Patch by Bénédikt Tran. + +.. + +.. date: 2025-07-12-18-05-37 +.. gh-issue: 136591 +.. nonce: ujXmSN +.. section: Library + +:mod:`!_hashlib`: avoid using deprecated functions +:manpage:`ERR_func_error_string` and :manpage:`EVP_MD_CTX_md` when using +OpenSSL 3.0 and later. Patch by Bénédikt Tran. + +.. + +.. date: 2025-07-12-14-15-47 +.. gh-issue: 136571 +.. nonce: muHmBv +.. section: Library + +:meth:`datetime.date.fromisocalendar` can now raise OverflowError for out of +range arguments. + +.. + +.. date: 2025-07-11-23-04-39 +.. gh-issue: 136549 +.. nonce: oAi8u4 +.. section: Library + +Fix signature of :func:`threading.excepthook`. + +.. + +.. date: 2025-07-11-10-23-44 +.. gh-issue: 136492 +.. nonce: BVi5h0 +.. section: Library + +Expose :pep:`667`'s :data:`~types.FrameLocalsProxyType` in the :mod:`types` +module. + +.. + +.. date: 2025-07-11-08-15-17 +.. gh-issue: 83336 +.. nonce: ptpmq7 +.. section: Library + +``utf8_sig`` is now aliased to :mod:`encodings.utf_8_sig` + +.. + +.. date: 2025-07-11-03-39-15 +.. gh-issue: 136523 +.. nonce: s7caKL +.. section: Library + +Fix :class:`wave.Wave_write` emitting an unraisable when open raises. + +.. + +.. date: 2025-07-10-21-02-43 +.. gh-issue: 136507 +.. nonce: pnEuGS +.. section: Library + +Fix mimetypes CLI to handle multiple file parameters. + +.. + +.. date: 2025-07-10-10-18-19 +.. gh-issue: 52876 +.. nonce: 9Vjrd8 +.. section: Library + +Add missing ``keepends`` (default ``True``) parameter to +:meth:`!codecs.StreamReaderWriter.readline` and +:meth:`!codecs.StreamReaderWriter.readlines`. + +.. + +.. date: 2025-07-10-00-47-37 +.. gh-issue: 136470 +.. nonce: KlUEUG +.. section: Library + +Correct :class:`concurrent.futures.InterpreterPoolExecutor`'s default thread +name. + +.. + +.. date: 2025-07-09-20-29-30 +.. gh-issue: 136476 +.. nonce: HyLLzh +.. section: Library + +Fix a bug that was causing the ``get_async_stack_trace`` function to miss +some frames in the stack trace. + +.. + +.. date: 2025-07-08-20-58-01 +.. gh-issue: 136434 +.. nonce: uuJsjS +.. section: Library + +Fix docs generation of ``UnboundItem`` in :mod:`concurrent.interpreters` +when running with :option:`-OO`. + +.. + +.. date: 2025-07-07-22-12-32 +.. gh-issue: 136380 +.. nonce: 1b_nXl +.. section: Library + +Raises :exc:`AttributeError` when accessing +:class:`concurrent.futures.InterpreterPoolExecutor` and subinterpreters are +not available. + +.. + +.. date: 2025-07-07-16-46-55 +.. gh-issue: 72327 +.. nonce: wLvRuj +.. section: Library + +Suggest using the system command prompt when ``pip install`` is typed into +the REPL. Patch by Tom Viner, Richard Si, and Brian Schubert. + +.. + +.. date: 2025-07-06-18-38-10 +.. gh-issue: 135953 +.. nonce: Z29DCz +.. section: Library + +Implement a new high-frequency runtime profiler that leverages the existing +remote debugging functionality to collect detailed execution statistics from +running Python processes. This tool is exposed in the ``profile.sample`` +module and enables non-intrusive observation of production applications by +attaching to already-running processes without requiring any code +modifications, restarts, or special startup flags. The observer can perform +extremely high-frequency sampling of stack traces and interpreter state, +providing detailed runtime execution analysis of live applications. + +.. + +.. date: 2025-07-06-10-18-48 +.. gh-issue: 136021 +.. nonce: f-FJYT +.. section: Library + +Make ``type_params`` parameter required in :func:`!typing._eval_type` after +a deprecation period for not providing this parameter. Also remove the +:exc:`DeprecationWarning` for the old behavior. + +.. + +.. date: 2025-07-05-09-45-04 +.. gh-issue: 136286 +.. nonce: N67Amr +.. section: Library + +Fix pickling failures for protocols 0 and 1 for many objects related to +subinterpreters. + +.. + +.. date: 2025-07-05-06-59-46 +.. gh-issue: 136047 +.. nonce: qWvycf +.. section: Library + +Fix issues with :mod:`typing` when the C implementation of :mod:`abc` is not +available. + +.. + +.. date: 2025-07-05-06-56-16 +.. gh-issue: 136316 +.. nonce: 3zj_Do +.. section: Library + +Improve support for evaluating nested forward references in +:func:`typing.evaluate_forward_ref`. + +.. + +.. date: 2025-07-04-23-45-00 +.. gh-issue: 136306 +.. nonce: O1YLIU +.. section: Library + +:mod:`ssl` can now get and set groups used for key agreement. + +.. + +.. date: 2025-07-04-12-53-02 +.. gh-issue: 136156 +.. nonce: OYlXoz +.. section: Library + +:func:`tempfile.TemporaryFile` no longer uses :data:`os.O_EXCL` with +:data:`os.O_TMPFILE`, so it's possible to use ``linkat()`` on the file +descriptor. Patch by Victor Stinner. + +.. + +.. date: 2025-07-02-18-41-45 +.. gh-issue: 133982 +.. nonce: 7qqAn6 +.. section: Library + +Update Python implementation of :class:`io.BytesIO` to be thread safe. + +.. + +.. date: 2025-07-02-10-48-21 +.. gh-issue: 136193 +.. nonce: xfvras +.. section: Library + +Improve :exc:`TypeError` error message, when richcomparing two +:class:`types.SimpleNamespace` objects. + +.. + +.. date: 2025-07-01-14-44-03 +.. gh-issue: 136097 +.. nonce: bI1n14 +.. section: Library + +Fix potential infinite recursion and KeyError in ``sysconfig +--generate-posix-vars``. + +.. + +.. date: 2025-06-30-11-12-24 +.. gh-issue: 85702 +.. nonce: 0Lrbwu +.. section: Library + +If ``zoneinfo._common.load_tzdata`` is given a package without a resource a +:exc:`zoneinfo.ZoneInfoNotFoundError` is raised rather than a +:exc:`PermissionError`. Patch by Victor Stinner. + +.. + +.. date: 2025-06-29-15-22-13 +.. gh-issue: 90733 +.. nonce: NiquaA +.. section: Library + +Improve error messages when reporting invalid parameters in +:func:`hashlib.scrypt`. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-28-11-32-57 +.. gh-issue: 134759 +.. nonce: AjjKcG +.. section: Library + +Fix :exc:`UnboundLocalError` in :func:`email.message.Message.get_payload` +when the payload to decode is a :class:`bytes` object. Patch by Kliment +Lamonov. + +.. + +.. date: 2025-06-27-13-34-28 +.. gh-issue: 136028 +.. nonce: RY727g +.. section: Library + +Fix parsing month names containing "İ" (U+0130, LATIN CAPITAL LETTER I WITH +DOT ABOVE) in :func:`time.strptime`. This affects locales az_AZ, ber_DZ, +ber_MA and crh_UA. + +.. + +.. date: 2025-06-27-09-26-04 +.. gh-issue: 87135 +.. nonce: 33z0UW +.. section: Library + +Acquiring a :class:`threading.Lock` or :class:`threading.RLock` at +interpreter shutdown will raise :exc:`PythonFinalizationError` if Python can +determine that it would otherwise deadlock. + +.. + +.. date: 2025-06-26-17-28-49 +.. gh-issue: 135995 +.. nonce: pPrDCt +.. section: Library + +In the palmos encoding, make byte ``0x9b`` decode to ``›`` (U+203A - SINGLE +RIGHT-POINTING ANGLE QUOTATION MARK). + +.. + +.. date: 2025-06-26-17-19-36 +.. gh-issue: 105456 +.. nonce: eR9oHB +.. section: Library + +Removed :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` +modules. + +.. + +.. date: 2025-06-26-11-52-40 +.. gh-issue: 53203 +.. nonce: TMigBr +.. section: Library + +Fix :func:`time.strptime` for ``%c`` and ``%x`` formats on locales byn_ER, +wal_ET and lzh_TW, and for ``%X`` format on locales ar_SA, bg_BG and lzh_TW. + +.. + +.. date: 2025-06-24-14-43-24 +.. gh-issue: 135878 +.. nonce: Db4roX +.. section: Library + +Fixes a crash of :class:`types.SimpleNamespace` on :term:`free threading` +builds, when several threads were calling its :meth:`~object.__repr__` +method at the same time. + +.. + +.. date: 2025-06-24-13-30-47 +.. gh-issue: 135853 +.. nonce: 7ejTvK +.. section: Library + +Add :func:`math.fmax` and :func:`math.fmin` to get the larger and smaller of +two floating-point values. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-24-10-52-35 +.. gh-issue: 135836 +.. nonce: s37351 +.. section: Library + +Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could +occur when non-\ :exc:`OSError` exception is raised during connection and +socket's ``close()`` raises :exc:`!OSError`. + +.. + +.. date: 2025-06-24-10-23-37 +.. gh-issue: 135853 +.. nonce: 6xDNOG +.. section: Library + +:mod:`math`: expose C99 :func:`~math.signbit` function to determine whether +the sign bit of a floating-point value is set. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-23-13-02-08 +.. gh-issue: 134531 +.. nonce: yUmj07 +.. section: Library + +:mod:`hmac`: use the :manpage:`EVP_MAC(3ssl)` interface for HMAC when Python +is built with OpenSSL 3.0 and later instead of the deprecated +:manpage:`HMAC_CTX(3ssl) <hmac(3)>` interface. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-23-11-04-25 +.. gh-issue: 135836 +.. nonce: -C-c4v +.. section: Library + +Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could +occur when the Happy Eyeballs algorithm resulted in an empty exceptions list +during connection attempts. + +.. + +.. date: 2025-06-23-10-19-11 +.. gh-issue: 135855 +.. nonce: -J0AGF +.. section: Library + +Raise :exc:`TypeError` instead of :exc:`SystemError` when +:func:`!_interpreters.set___main___attrs` is passed a non-dict object. Patch +by Brian Schubert. + +.. + +.. date: 2025-06-22-22-03-06 +.. gh-issue: 135823 +.. nonce: iDBg97 +.. section: Library + +:mod:`netrc`: improve the error message when the security check for the +ownership of the default configuration file ``~/.netrc`` fails. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-06-22-16-23-44 +.. gh-issue: 135815 +.. nonce: 0DandH +.. section: Library + +:mod:`netrc`: skip security checks if :func:`os.getuid` is missing. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-06-22-02-16-17 +.. gh-issue: 135640 +.. nonce: FXyFL6 +.. section: Library + +Address bug where it was possible to call +:func:`xml.etree.ElementTree.ElementTree.write` on an ElementTree object +with an invalid root element. This behavior blanked the file passed to +``write`` if it already existed. + +.. + +.. date: 2025-06-20-17-06-59 +.. gh-issue: 90117 +.. nonce: GYWVrn +.. section: Library + +Speed up :mod:`pprint` for :class:`list` and :class:`tuple`. + +.. + +.. date: 2025-06-20-16-28-47 +.. gh-issue: 135759 +.. nonce: jne0Zi +.. section: Library + +:mod:`hashlib`: reject negative digest lengths in OpenSSL-based SHAKE +objects by raising a :exc:`ValueError`. Previously, negative lengths were +implicitly rejected by raising a :exc:`MemoryError` or a :exc:`SystemError`. +Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-18-19-25-32 +.. gh-issue: 123471 +.. nonce: lx1Xbt +.. section: Library + +Make concurrent iterations over :class:`itertools.chain` safe under +:term:`free threading`. + +.. + +.. date: 2025-06-18-13-58-13 +.. gh-issue: 135645 +.. nonce: 109nff +.. section: Library + +Added ``supports_isolated_interpreters`` field to +:data:`sys.implementation`. + +.. + +.. date: 2025-06-18-11-43-17 +.. gh-issue: 135646 +.. nonce: r7ekEn +.. section: Library + +Raise consistent :exc:`NameError` exceptions in +:func:`annotationlib.ForwardRef.evaluate` + +.. + +.. date: 2025-06-17-23-13-56 +.. gh-issue: 135557 +.. nonce: Bfcy4v +.. section: Library + +Fix races on :mod:`heapq` updates and :class:`list` reads on the :term:`free +threaded <free threading>` build. + +.. + +.. date: 2025-06-17-22-44-19 +.. gh-issue: 119180 +.. nonce: Ogv8Nj +.. section: Library + +Only fetch globals and locals if necessary in +:func:`annotationlib.get_annotations` + +.. + +.. date: 2025-06-16-15-03-03 +.. gh-issue: 135561 +.. nonce: mJCN8D +.. section: Library + +Fix a crash on DEBUG builds when an HACL* HMAC routine fails. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-06-16-15-00-13 +.. gh-issue: 135386 +.. nonce: lNrxLc +.. section: Library + +Fix opening a :mod:`dbm.sqlite3` database for reading from read-only file or +directory. + +.. + +.. date: 2025-06-16-12-37-02 +.. gh-issue: 135444 +.. nonce: An2eeA +.. section: Library + +Fix :meth:`asyncio.DatagramTransport.sendto` to account for datagram header +size when data cannot be sent. + +.. + +.. date: 2025-06-15-03-03-22 +.. gh-issue: 65697 +.. nonce: COdwZd +.. section: Library + +:class:`configparser`'s error message when attempting to write an invalid +key is now more helpful. + +.. + +.. date: 2025-06-14-14-19-13 +.. gh-issue: 135497 +.. nonce: 1pzwdA +.. section: Library + +Fix :func:`os.getlogin` failing for longer usernames on BSD-based platforms. + +.. + +.. date: 2025-06-14-12-06-55 +.. gh-issue: 135487 +.. nonce: KdVFff +.. section: Library + +Fix :meth:`!reprlib.Repr.repr_int` when given integers with more than +:func:`sys.get_int_max_str_digits` digits. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-12-18-15-31 +.. gh-issue: 135429 +.. nonce: mch75_ +.. section: Library + +Fix the argument mismatch in ``_lsprof`` for ``PY_THROW`` event. + +.. + +.. date: 2025-06-12-10-45-02 +.. gh-issue: 135368 +.. nonce: OjWVHL +.. section: Library + +Fix :class:`unittest.mock.Mock` generation on :func:`dataclasses.dataclass` +objects. Now all special attributes are set as it was before :gh:`124429`. + +.. + +.. date: 2025-06-11-15-08-02 +.. gh-issue: 135336 +.. nonce: 6Gq6MI +.. section: Library + +:mod:`json` now encodes strings up to 2.2x faster if they consist solely of +characters that don’t require escaping. + +.. + +.. date: 2025-06-10-21-42-04 +.. gh-issue: 135335 +.. nonce: WnUqb_ +.. section: Library + +:mod:`multiprocessing`: Flush ``stdout`` and ``stderr`` after preloading +modules in the ``forkserver``. + +.. + +.. date: 2025-06-10-21-00-48 +.. gh-issue: 126631 +.. nonce: eITVJd +.. section: Library + +Fix :mod:`multiprocessing` ``forkserver`` bug which prevented ``__main__`` +from being preloaded. + +.. + +.. date: 2025-06-10-16-11-00 +.. gh-issue: 133967 +.. nonce: P0c24q +.. section: Library + +Do not normalize :mod:`locale` name 'C.UTF-8' to 'en_US.UTF-8'. + +.. + +.. date: 2025-06-10-10-22-18 +.. gh-issue: 130870 +.. nonce: JipqbO +.. section: Library + +Preserve :class:`types.GenericAlias` subclasses in +:func:`typing.get_type_hints` + +.. + +.. date: 2025-06-10-00-42-30 +.. gh-issue: 135321 +.. nonce: UHh9jT +.. section: Library + +Raise a correct exception for values greater than 0x7fffffff for the +``BINSTRING`` opcode in the C implementation of :mod:`pickle`. + +.. + +.. date: 2025-06-09-10-16-55 +.. gh-issue: 121914 +.. nonce: G6Avkq +.. section: Library + +Changed the names of the symbol tables for lambda expressions and generator +expressions to "<lambda>" and "<genexpr>" respectively to avoid conflicts +with user-defined names. + +.. + +.. date: 2025-06-08-14-50-34 +.. gh-issue: 135276 +.. nonce: ZLUhV1 +.. section: Library + +Synchronized zipfile.Path with zipp 3.23, including improved performance of +:meth:`zipfile.Path.open` for non-reading modes, rely on +:func:`functools.cached_property` to cache values on the instance. Rely on +``save_method_args`` to save the initialization method arguments. Fixed +``.name``, ``.stem`` and other basename-based properties on Windows when +working with a zipfile on disk. + +.. + +.. date: 2025-06-08-11-11-07 +.. gh-issue: 135234 +.. nonce: wJCdh0 +.. section: Library + +:mod:`hashlib`: improve exception messages when an OpenSSL function failed. +When memory allocation fails on OpenSSL's side, a :exc:`MemoryError` is +raised instead of a :exc:`ValueError`. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-08-10-22-22 +.. gh-issue: 135244 +.. nonce: Y2SOTJ +.. section: Library + +:mod:`uuid`: when the MAC address cannot be determined, the 48-bit node ID +is now generated with a cryptographically-secure pseudo-random number +generator (CSPRNG) as per :rfc:`RFC 9562, §6.10.3 <9562#section-6.10-3>`. +This affects :func:`~uuid.uuid1` and :func:`~uuid.uuid6`. + +.. + +.. date: 2025-06-08-01-10-34 +.. gh-issue: 135241 +.. nonce: 5j18IW +.. section: Library + +The :code:`INT` opcode of the C accelerator :mod:`!_pickle` module was +updated to look only for "00" and "01" to push booleans onto the stack, +aligning with the Python :mod:`pickle` module. + +.. + +.. date: 2025-06-06-17-34-18 +.. gh-issue: 133934 +.. nonce: yT1r68 +.. section: Library + +Improve :mod:`sqlite3` CLI's ``.help`` message. + +.. + +.. date: 2025-06-03-12-59-17 +.. gh-issue: 135069 +.. nonce: xop30V +.. section: Library + +Fix the "Invalid error handling" exception in +:class:`!encodings.idna.IncrementalDecoder` to correctly replace the +'errors' parameter. + +.. + +.. date: 2025-06-02-14-36-28 +.. gh-issue: 130662 +.. nonce: Gpr2GB +.. section: Library + ++Accept leading zeros in precision and width fields for ++:class:`~decimal.Decimal` formatting, for example ``format(Decimal(1.25), +'.016f')``. + +.. + +.. date: 2025-06-02-14-28-30 +.. gh-issue: 130662 +.. nonce: EIgIR8 +.. section: Library + +Accept leading zeros in precision and width fields for +:class:`~fractions.Fraction` formatting, for example ``format(Fraction(1, +3), '.016f')``. + +.. + +.. date: 2025-06-01-14-18-48 +.. gh-issue: 135004 +.. nonce: cq3-fp +.. section: Library + +Rewrite and cleanup the internal :mod:`!_blake2` module. Some exception +messages were changed but their types were left untouched. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-06-01-11-14-00 +.. gh-issue: 134953 +.. nonce: ashdfs +.. section: Library + +Expand ``_colorize`` theme with ``keyword_constant`` and implement in +:term:`repl`. + +.. + +.. date: 2025-05-31-15-49-46 +.. gh-issue: 134978 +.. nonce: mXXuvW +.. section: Library + +:mod:`hashlib`: Supporting the ``string`` keyword parameter in hash function +constructors such as :func:`~hashlib.new` or the direct hash-named +constructors such as :func:`~hashlib.md5` and :func:`~hashlib.sha256` is now +deprecated and slated for removal in Python 3.19. Prefer passing the initial +data as a positional argument for maximum backwards compatibility. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-05-31-12-08-12 +.. gh-issue: 134970 +.. nonce: lgSaxq +.. section: Library + +Fix the "unknown action" exception in +:meth:`argparse.ArgumentParser.add_argument_group` to correctly replace the +action class. + +.. + +.. date: 2025-05-30-18-13-48 +.. gh-issue: 134718 +.. nonce: 5FEspx +.. section: Library + +By default, omit optional ``Load()`` values in :func:`ast.dump`. + +.. + +.. date: 2025-05-30-13-07-29 +.. gh-issue: 134718 +.. nonce: 9Qvhxn +.. section: Library + +:func:`ast.dump` now only omits ``None`` and ``[]`` values if they are +default values. + +.. + +.. date: 2025-05-30-09-46-21 +.. gh-issue: 134939 +.. nonce: Pu3nnm +.. section: Library + +Add the :mod:`concurrent.interpreters` module. See :pep:`734`. + +.. + +.. date: 2025-05-29-17-39-13 +.. gh-issue: 108885 +.. nonce: MegCRA +.. section: Library + +Run each example as a subtest in unit tests synthesized by +:func:`doctest.DocFileSuite` and :func:`doctest.DocTestSuite`. Add the +:meth:`doctest.DocTestRunner.report_skip` method. + +.. + +.. date: 2025-05-29-06-53-40 +.. gh-issue: 134885 +.. nonce: -_L22o +.. section: Library + +Fix possible crash in the :mod:`compression.zstd` module related to setting +parameter types. Patch by Jelle Zijlstra. + +.. + +.. date: 2025-05-28-20-49-29 +.. gh-issue: 134857 +.. nonce: dVYXVO +.. section: Library + +Improve error report for :mod:`doctest`\ s run with :mod:`unittest`. Remove +:mod:`!doctest` module frames from tracebacks and redundant newline +character from a failure message. + +.. + +.. date: 2025-05-28-15-53-27 +.. gh-issue: 128840 +.. nonce: Nur2pB +.. section: Library + +Fix parsing long IPv6 addresses with embedded IPv4 address. + +.. + +.. date: 2025-05-27-11-24-38 +.. gh-issue: 133579 +.. nonce: WGPUC1 +.. section: Library + +:mod:`curses`: Consistently report failures of curses C API calls in +module-level methods by raising a :exc:`curses.error`. This affects +:func:`~curses.assume_default_colors`, :func:`~curses.baudrate`, +:func:`~curses.cbreak`, :func:`~curses.echo`, :func:`~curses.longname`, +:func:`~curses.initscr`, :func:`~curses.nl`, :func:`~curses.raw`, +:func:`~curses.termattrs`, :func:`~curses.termname` and +:func:`~curses.unctrl`. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-27-11-18-13 +.. gh-issue: 133579 +.. nonce: ohtgdC +.. section: Library + +:meth:`curses.window.refresh` and :meth:`curses.window.noutrefresh` now +raise a :exc:`TypeError` instead of :exc:`curses.error` when called with an +incorrect number of arguments for :ref:`pads <windows-and-pads>`. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-05-27-11-13-51 +.. gh-issue: 133579 +.. nonce: KY9M6S +.. section: Library + +:ref:`curses.window <curses-window-objects>`: Consistently report failures +of curses C API calls in Window methods by raising a :exc:`curses.error`. +This affects :meth:`~curses.window.addch`, :meth:`~curses.window.addnstr`, +:meth:`~curses.window.addstr`, :meth:`~curses.window.border`, +:meth:`~curses.window.box`, :meth:`~curses.window.chgat`, +:meth:`~curses.window.getbkgd`, :meth:`~curses.window.inch`, +:meth:`~curses.window.insstr` and :meth:`~curses.window.insnstr`. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-05-26-22-18-32 +.. gh-issue: 134771 +.. nonce: RKXpLT +.. section: Library + +The ``time_clockid_converter()`` function now selects correct type for +``clockid_t`` on Cygwin which fixes a build error. + +.. + +.. date: 2025-05-26-17-06-40 +.. gh-issue: 134637 +.. nonce: 9-3zRL +.. section: Library + +Fix performance regression in calling a :mod:`ctypes` function pointer in +:term:`free threading`. + +.. + +.. date: 2025-05-26-14-04-39 +.. gh-issue: 134696 +.. nonce: P04xUa +.. section: Library + +Built-in HACL* and OpenSSL implementations of hash function constructors now +correctly accept the same *documented* named arguments. For instance, +:func:`~hashlib.md5` could be previously invoked as ``md5(data=data)`` or +``md5(string=string)`` depending on the underlying implementation but these +calls were not compatible. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-26-12-31-08 +.. gh-issue: 132710 +.. nonce: ApU3TZ +.. section: Library + +If possible, ensure that :func:`uuid.getnode` returns the same result even +across different processes. Previously, the result was constant only within +the same process. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-26-11-01-54 +.. gh-issue: 134531 +.. nonce: my1Fzt +.. section: Library + +:mod:`!_hashlib`: Rename internal C functions for :class:`!_hashlib.HASH` +and :class:`!_hashlib.HASHXOF` objects. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-26-10-52-27 +.. gh-issue: 134698 +.. nonce: aJ1mZ1 +.. section: Library + +Fix a crash when calling methods of :class:`ssl.SSLContext` or +:class:`ssl.SSLSocket` across multiple threads. + +.. + +.. date: 2025-05-25-23-23-05 +.. gh-issue: 134151 +.. nonce: 13Wwsb +.. section: Library + +:mod:`email`: Fix :exc:`TypeError` in :func:`email.utils.decode_params` when +sorting :rfc:`2231` continuations that contain an unnumbered section. + +.. + +.. date: 2025-05-25-13-46-37 +.. gh-issue: 134635 +.. nonce: ZlPrlX +.. section: Library + +:mod:`zlib`: Allow to combine Adler-32 and CRC-32 checksums via +:func:`~zlib.adler32_combine` and :func:`~zlib.crc32_combine`. Patch by +Callum Attryde and Bénédikt Tran. + +.. + +.. date: 2025-05-25-11-02-05 +.. gh-issue: 134657 +.. nonce: 3YFhR9 +.. section: Library + +:mod:`asyncio`: Remove some private names from ``asyncio.__all__``. + +.. + +.. date: 2025-05-24-13-10-35 +.. gh-issue: 134210 +.. nonce: 0IuMY2 +.. section: Library + +:func:`curses.window.getch` now correctly handles signals. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-05-24-03-10-36 +.. gh-issue: 80334 +.. nonce: z21cMa +.. section: Library + +:func:`multiprocessing.freeze_support` now checks for work on any "spawn" +start method platform rather than only on Windows. + +.. + +.. date: 2025-05-23-23-43-39 +.. gh-issue: 134582 +.. nonce: 9POq3l +.. section: Library + +Fix tokenize.untokenize() round-trip errors related to t-strings braces +escaping + +.. + +.. date: 2025-05-23-20-01-52 +.. gh-issue: 134580 +.. nonce: xnaJ70 +.. section: Library + +Improved the styling of HTML diff pages generated by the +:class:`difflib.HtmlDiff` class, and migrated the output to the HTML5 +standard. + +.. + +.. date: 2025-05-23-10-15-36 +.. gh-issue: 134565 +.. nonce: zmb66C +.. section: Library + +:func:`unittest.doModuleCleanups` no longer swallows all but first exception +raised in the cleanup code, but raises a :exc:`ExceptionGroup` if multiple +errors occurred. + +.. + +.. date: 2025-05-22-18-14-13 +.. gh-issue: 134546 +.. nonce: fjLVzK +.. section: Library + +Ensure :mod:`pdb` remote debugging script is readable by remote Python +process. + +.. + +.. date: 2025-05-22-14-12-53 +.. gh-issue: 134451 +.. nonce: M1rD-j +.. section: Library + +Converted ``asyncio.tools.CycleFoundException`` from dataclass to a regular +exception type. + +.. + +.. date: 2025-05-22-13-10-32 +.. gh-issue: 114177 +.. nonce: 3TYUJ3 +.. section: Library + +Fix :mod:`asyncio` to not close subprocess pipes which would otherwise error +out when the event loop is already closed. + +.. + +.. date: 2025-05-20-21-45-58 +.. gh-issue: 90871 +.. nonce: Gkvtp6 +.. section: Library + +Fixed an off by one error concerning the backlog parameter in +:meth:`~asyncio.loop.create_unix_server`. Contributed by Christian Harries. + +.. + +.. date: 2025-05-20-19-16-30 +.. gh-issue: 134323 +.. nonce: ZQZGvw +.. section: Library + +Fix the :meth:`threading.RLock.locked` method. + +.. + +.. date: 2025-05-20-15-13-43 +.. gh-issue: 86802 +.. nonce: trF7TM +.. section: Library + +Fixed asyncio memory leak in cancelled shield tasks. For shielded tasks +where the shield was cancelled, log potential exceptions through the +exception handler. Contributed by Christian Harries. + +.. + +.. date: 2025-05-20-11-51-17 +.. gh-issue: 71189 +.. nonce: 0LpTB1 +.. section: Library + +Add support of the all-but-last mode in :func:`os.path.realpath`. + +.. + +.. date: 2025-05-20-11-35-08 +.. gh-issue: 72902 +.. nonce: jzEI-E +.. section: Library + +Improve speed (x1.1-1.8) of the :class:`~fractions.Fraction` constructor for +typical inputs (:class:`float`'s, :class:`~decimal.Decimal`'s or strings). + +.. + +.. date: 2025-05-19-20-59-06 +.. gh-issue: 134209 +.. nonce: anhTcF +.. section: Library + +:mod:`curses`: The :meth:`curses.window.instr` and +:meth:`curses.window.getstr` methods now allocate their internal buffer on +the heap instead of the stack; in addition, the max buffer size is increased +from 1023 to 2047. + +.. + +.. date: 2025-05-19-18-12-42 +.. gh-issue: 88994 +.. nonce: 7avvVu +.. section: Library + +Change :func:`datetime.datetime.now` to half-even rounding for consistency +with :func:`datetime.datetime.fromtimestamp`. Patch by John Keith Hohm. + +.. + +.. date: 2025-05-19-17-27-21 +.. gh-issue: 80184 +.. nonce: LOkbaw +.. section: Library + +The default queue size is now ``socket.SOMAXCONN`` for +:class:`socketserver.TCPServer`. + +.. + +.. date: 2025-05-19-15-30-00 +.. gh-issue: 132983 +.. nonce: asdsfs +.. section: Library + +Add :mod:`!compression.zstd` version information to ``test.pythoninfo``. + +.. + +.. date: 2025-05-19-15-05-24 +.. gh-issue: 134235 +.. nonce: pz9PwV +.. section: Library + +Updated tab completion on REPL to include builtin modules. Contributed by +Tom Wang, Hunter Young + +.. + +.. date: 2025-05-19-10-32-11 +.. gh-issue: 134152 +.. nonce: INJC2j +.. section: Library + +Fixed :exc:`UnboundLocalError` that could occur during :mod:`email` header +parsing if an expected trailing delimiter is missing in some contexts. + +.. + +.. date: 2025-05-18-23-46-21 +.. gh-issue: 134152 +.. nonce: 30HwbX +.. section: Library + +:mod:`email`: Fix parsing of email message ID with invalid domain. + +.. + +.. date: 2025-05-18-13-23-29 +.. gh-issue: 134168 +.. nonce: hgx3Xg +.. section: Library + +:mod:`http.server`: Fix IPv6 address binding and :option:`--directory +<http.server --directory>` handling when using HTTPS. + +.. + +.. date: 2025-05-18-12-48-39 +.. gh-issue: 62184 +.. nonce: y11l10 +.. section: Library + +Remove import of C implementation of :class:`io.FileIO` from Python +implementation which has its own implementation + +.. + +.. date: 2025-05-18-12-23-07 +.. gh-issue: 134087 +.. nonce: HilZWl +.. section: Library + +Remove support for arbitrary positional or keyword arguments in the C +implementation of :class:`threading.RLock` objects. This was deprecated +since Python 3.14. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-18-07-25-15 +.. gh-issue: 134173 +.. nonce: 53oOoF +.. section: Library + +Speed up :mod:`asyncio` performance of transferring state from thread pool +:class:`concurrent.futures.Future` by up to 4.4x. Patch by J. Nick Koston. + +.. + +.. date: 2025-05-17-20-23-57 +.. gh-issue: 133982 +.. nonce: smS7au +.. section: Library + +Emit :exc:`RuntimeWarning` in the Python implementation of :mod:`io` when +the :term:`file-like object <file object>` is not closed explicitly in the +presence of multiple I/O layers. + +.. + +.. date: 2025-05-17-18-08-35 +.. gh-issue: 133890 +.. nonce: onn9_X +.. section: Library + +The :mod:`tarfile` module now handles :exc:`UnicodeEncodeError` in the same +way as :exc:`OSError` when cannot extract a member. + +.. + +.. date: 2025-05-17-13-46-20 +.. gh-issue: 134097 +.. nonce: fgkjE1 +.. section: Library + +Fix interaction of the new :term:`REPL` and :option:`-X showrefcount <-X>` +command line option. + +.. + +.. date: 2025-05-17-12-40-12 +.. gh-issue: 133889 +.. nonce: Eh-zO4 +.. section: Library + +The generated directory listing page in +:class:`http.server.SimpleHTTPRequestHandler` now only shows the decoded +path component of the requested URL, and not the query and fragment. + +.. + +.. date: 2025-05-16-20-10-25 +.. gh-issue: 134098 +.. nonce: YyTkKr +.. section: Library + +Fix handling paths that end with a percent-encoded slash (``%2f`` or +``%2F``) in :class:`http.server.SimpleHTTPRequestHandler`. + +.. + +.. date: 2025-05-16-12-40-37 +.. gh-issue: 132124 +.. nonce: T_5Odx +.. section: Library + +On POSIX-compliant systems, :func:`!multiprocessing.util.get_temp_dir` now +ignores :envvar:`TMPDIR` (and similar environment variables) if the path +length of ``AF_UNIX`` socket files exceeds the platform-specific maximum +length when using the :ref:`forkserver +<multiprocessing-start-method-forkserver>` start method. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-05-15-14-27-01 +.. gh-issue: 134062 +.. nonce: fRbJet +.. section: Library + +:mod:`ipaddress`: fix collisions in :meth:`~object.__hash__` for +:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` objects. + +.. + +.. date: 2025-05-15-00-27-09 +.. gh-issue: 134004 +.. nonce: e8k4-R +.. section: Library + +:mod:`shelve` as well as underlying :mod:`!dbm.dumb` and :mod:`!dbm.sqlite` +now have :meth:`!reorganize` methods to recover unused free space previously +occupied by deleted entries. + +.. + +.. date: 2025-05-13-18-54-56 +.. gh-issue: 133970 +.. nonce: 6G-Oi6 +.. section: Library + +Make :class:`!string.templatelib.Template` and +:class:`!string.templatelib.Interpolation` generic. + +.. + +.. date: 2025-05-13-18-21-59 +.. gh-issue: 71253 +.. nonce: -3Sf_K +.. section: Library + +Raise :exc:`ValueError` in :func:`open` if *opener* returns a negative +file-descriptor in the Python implementation of :mod:`io` to match the C +implementation. + +.. + +.. date: 2025-05-12-20-38-57 +.. gh-issue: 133960 +.. nonce: Aee79f +.. section: Library + +Simplify and improve :func:`typing.evaluate_forward_ref`. It now no longer +raises errors on certain invalid types. In several situations, it is now +able to evaluate forward references that were previously unsupported. + +.. + +.. date: 2025-05-12-06-52-10 +.. gh-issue: 133925 +.. nonce: elInBY +.. section: Library + +Make the private class ``typing._UnionGenericAlias`` hashable. + +.. + +.. date: 2025-05-11-12-56-52 +.. gh-issue: 133604 +.. nonce: kFxhc8 +.. section: Library + +Remove :func:`!platform.java_ver` which was deprecated since Python 3.13. + +.. + +.. date: 2025-05-11-11-39-05 +.. gh-issue: 133875 +.. nonce: pUar3l +.. section: Library + +Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. Use +:func:`os.path.isreserved` to detect reserved paths on Windows. + +.. + +.. date: 2025-05-11-10-28-11 +.. gh-issue: 133873 +.. nonce: H03nov +.. section: Library + +Remove the deprecated ``getmark()``, ``setmark()`` and ``getmarkers()`` +methods of the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` +classes, which were deprecated since Python 3.13. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-11-10-01-48 +.. gh-issue: 133866 +.. nonce: g3dHP_ +.. section: Library + +Remove the undocumented function :func:`!ctypes.SetPointerType`, which has +been deprecated since Python 3.13. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-11-08-48-55 +.. gh-issue: 133823 +.. nonce: F8udQy +.. section: Library + +Remove support for ``TD = TypedDict("TD")`` and ``TD = TypedDict("TD", +None)`` calls for constructing :class:`typing.TypedDict` objects with zero +field. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-10-17-42-03 +.. gh-issue: 125996 +.. nonce: vaQp0- +.. section: Library + +Fix thread safety of :class:`collections.OrderedDict`. Patch by Kumar +Aditya. + +.. + +.. date: 2025-05-10-12-07-54 +.. gh-issue: 133817 +.. nonce: 4GMtKV +.. section: Library + +Remove support for creating :class:`~typing.NamedTuple` classes via the +undocumented keyword argument syntax. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-10-12-06-55 +.. gh-issue: 133653 +.. nonce: Gb2aG4 +.. section: Library + +Fix :class:`argparse.ArgumentParser` with the *formatter_class* argument. +Fix TypeError when *formatter_class* is a custom subclass of +:class:`!HelpFormatter`. Fix TypeError when *formatter_class* is not a +subclass of :class:`!HelpFormatter` and non-standard *prefix_char* is used. +Fix support of colorizing when *formatter_class* is not a subclass of +:class:`!HelpFormatter`. + +.. + +.. date: 2025-05-10-11-04-47 +.. gh-issue: 133810 +.. nonce: 03WhnK +.. section: Library + +Remove :class:`!http.server.CGIHTTPRequestHandler` and ``--cgi`` flag from +the :program:`python -m http.server` command-line interface. They were +deprecated in Python 3.13. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-09-20-59-24 +.. gh-issue: 132641 +.. nonce: 3qTw44 +.. section: Library + +Fixed a race in :func:`functools.lru_cache` under free-threading. + +.. + +.. date: 2025-05-09-19-05-24 +.. gh-issue: 133783 +.. nonce: 1voCnR +.. section: Library + +Fix bug with applying :func:`copy.replace` to :mod:`ast` objects. Attributes +that default to ``None`` were incorrectly treated as required for manually +created AST nodes. + +.. + +.. date: 2025-05-09-18-29-25 +.. gh-issue: 133684 +.. nonce: Y1DFSt +.. section: Library + +Fix bug where :func:`annotationlib.get_annotations` would return the wrong +result for certain classes that are part of a class hierarchy where ``from +__future__ import annotations`` is used. + +.. + +.. date: 2025-05-09-15-50-00 +.. gh-issue: 77057 +.. nonce: fV8SU- +.. section: Library + +Fix handling of invalid markup declarations in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-05-09-09-10-34 +.. gh-issue: 130328 +.. nonce: s9h4By +.. section: Library + +Speedup pasting in ``PyREPL`` on Windows in a legacy console. Patch by Chris +Eibl. + +.. + +.. date: 2025-05-09-08-49-03 +.. gh-issue: 133701 +.. nonce: KI8tGz +.. section: Library + +Fix bug where :class:`typing.TypedDict` classes defined under ``from +__future__ import annotations`` and inheriting from another ``TypedDict`` +had an incorrect ``__annotations__`` attribute. + +.. + +.. date: 2025-05-08-20-45-35 +.. gh-issue: 133656 +.. nonce: cxZODA +.. section: Library + +Remove deprecated :meth:`!zipimport.zipimporter.load_module`. Use +:meth:`zipimport.zipimporter.exec_module` instead. + +.. + +.. date: 2025-05-08-20-03-20 +.. gh-issue: 133722 +.. nonce: 1-B82a +.. section: Library + +Added a *color* option to :func:`difflib.unified_diff` that colors output +similar to :program:`git diff`. + +.. + +.. date: 2025-05-08-13-43-19 +.. gh-issue: 133489 +.. nonce: 9eGS1Z +.. section: Library + +:func:`random.getrandbits` can now generate more that 2\ :sup:`31` bits. +:func:`random.randbytes` can now generate more that 256 MiB. + +.. + +.. date: 2025-05-07-22-15-15 +.. gh-issue: 133595 +.. nonce: c3U88r +.. section: Library + +Clean up :class:`sqlite3.Connection` APIs. All parameters of +:func:`sqlite3.connect` except *database* are now keyword-only. The first +three parameters of methods :meth:`~sqlite3.Connection.create_function` and +:meth:`~sqlite3.Connection.create_aggregate` are now positional-only. The +first parameter of methods :meth:`~sqlite3.Connection.set_authorizer`, +:meth:`~sqlite3.Connection.set_progress_handler` and +:meth:`~sqlite3.Connection.set_trace_callback` is now positional-only. + +.. + +.. date: 2025-05-07-19-16-41 +.. gh-issue: 133581 +.. nonce: kERUCJ +.. section: Library + +Improve unparsing of t-strings in :func:`ast.unparse` and ``from __future__ +import annotations``. Empty t-strings now round-trip correctly and +formatting in interpolations is preserved. Patch by Jelle Zijlstra. + +.. + +.. date: 2025-05-07-14-36-30 +.. gh-issue: 133577 +.. nonce: BggPk9 +.. section: Library + +Add parameter ``formatter`` to :func:`logging.basicConfig`. + +.. + +.. date: 2025-05-07-13-31-06 +.. gh-issue: 92897 +.. nonce: ubeqGE +.. section: Library + +Removed the ``check_home`` parameter from :func:`sysconfig.is_python_build`, +deprecated since Python 3.12. + +.. + +.. date: 2025-05-06-22-54-37 +.. gh-issue: 133551 +.. nonce: rfy1tJ +.. section: Library + +Support t-strings (:pep:`750`) in :mod:`annotationlib`. Patch by Jelle +Zijlstra. + +.. + +.. date: 2025-05-06-14-44-55 +.. gh-issue: 133517 +.. nonce: Ca6NgW +.. section: Library + +Remove :func:`os.listdrives`, :func:`os.listvolumes` and +:func:`os.listmounts` in non Windows desktop builds since the underlying +functionality is missing. + +.. + +.. date: 2025-05-05-22-11-24 +.. gh-issue: 133439 +.. nonce: LpmyFz +.. section: Library + +Fix dot commands with trailing spaces are mistaken for multi-line SQL +statements in the sqlite3 command-line interface. + +.. + +.. date: 2025-05-05-18-50-00 +.. gh-issue: 133447 +.. nonce: ajshdb +.. section: Library + +Add basic color to :mod:`sqlite3` CLI interface. + +.. + +.. date: 2025-05-05-10-41-41 +.. gh-issue: 133253 +.. nonce: J5-xDD +.. section: Library + +Fix thread-safety issues in :mod:`linecache`. + +.. + +.. date: 2025-05-05-03-14-08 +.. gh-issue: 133390 +.. nonce: AuTggn +.. section: Library + +Support keyword completion in the :mod:`sqlite3` command-line interface and +add :data:`sqlite3.SQLITE_KEYWORDS` constant. + +.. + +.. date: 2025-05-04-17-04-55 +.. gh-issue: 132493 +.. nonce: huirKi +.. section: Library + +Avoid accessing ``__annotations__`` unnecessarily in +:func:`inspect.signature`. + +.. + +.. date: 2025-05-01-16-03-11 +.. gh-issue: 133017 +.. nonce: k7RLQp +.. section: Library + +Improve the error message of :func:`multiprocessing.sharedctypes.Array`, +:func:`multiprocessing.sharedctypes.RawArray`, +:func:`multiprocessing.sharedctypes.Value` and +:func:`multiprocessing.sharedctypes.RawValue` when an invalid typecode is +passed. Patch by Tomas Roun + +.. + +.. date: 2025-05-01-10-56-44 +.. gh-issue: 132813 +.. nonce: rKurvp +.. section: Library + +Improve error messages for incorrect types and values of +:class:`csv.Dialect` attributes. + +.. + +.. date: 2025-04-30-19-32-18 +.. gh-issue: 132969 +.. nonce: EagQ3G +.. section: Library + +Prevent the :class:`~concurrent.futures.ProcessPoolExecutor` executor +thread, which remains running when :meth:`shutdown(wait=False) +<concurrent.futures.Executor.shutdown>`, from attempting to adjust the +pool's worker processes after the object state has already been reset during +shutdown. A combination of conditions, including a worker process having +terminated abormally, resulted in an exception and a potential hang when the +still-running executor thread attempted to replace dead workers within the +pool. + +.. + +.. date: 2025-04-29-11-48-46 +.. gh-issue: 132876 +.. nonce: lyTQGZ +.. section: Library + +``ldexp()`` on Windows doesn't round subnormal results before Windows 11, +but should. Python's :func:`math.ldexp` wrapper now does round them, so +results may change slightly, in rare cases of very small results, on Windows +versions before 11. + +.. + +.. date: 2025-04-26-15-50-12 +.. gh-issue: 133009 +.. nonce: etBuz5 +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +<object.__deepcopy__>` when the element is concurrently mutated. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-04-25-16-06-53 +.. gh-issue: 132908 +.. nonce: wV5rja +.. section: Library + +Add :func:`math.isnormal` and :func:`math.issubnormal` functions. Patch by +Sergey B Kirpichev. + +.. + +.. date: 2025-04-25-11-53-37 +.. gh-issue: 95380 +.. nonce: 7dvPe- +.. section: Library + +:func:`fcntl.fcntl` and :func:`fcntl.ioctl`: Remove the 1024 bytes limit on +the size of not mutated bytes-like argument. + +.. + +.. date: 2025-04-25-11-48-00 +.. gh-issue: 122781 +.. nonce: ajsdns +.. section: Library + +Fix ``%z`` directive in :func:`datetime.datetime.strptime` to allow for no +provided offset as was documented. + +.. + +.. date: 2025-04-22-21-00-23 +.. gh-issue: 123471 +.. nonce: asOLA2 +.. section: Library + +Make concurrent iterations over :class:`itertools.combinations` and +:class:`itertools.product` safe under free-threading. + +.. + +.. date: 2025-04-21-01-05-14 +.. gh-issue: 127081 +.. nonce: Egrpq7 +.. section: Library + +Fix libc thread safety issues with :mod:`dbm` by performing stateful +operations in critical sections. + +.. + +.. date: 2025-04-21-01-03-15 +.. gh-issue: 127081 +.. nonce: WXRliX +.. section: Library + +Fix libc thread safety issues with :mod:`os` by replacing ``getlogin`` with +``getlogin_r`` re-entrant version. + +.. + +.. date: 2025-04-21-00-58-04 +.. gh-issue: 127081 +.. nonce: 3DCl92 +.. section: Library + +Fix libc thread safety issues with :mod:`pwd` by locking access to +``getpwall``. + +.. + +.. date: 2025-04-16-21-02-57 +.. gh-issue: 132551 +.. nonce: Psa7pL +.. section: Library + +Make :class:`io.BytesIO` safe in :term:`free-threaded <free threading>` +build. + +.. + +.. date: 2025-04-08-07-25-10 +.. gh-issue: 107583 +.. nonce: JGfbhq +.. section: Library + +Fix :class:`!Flag` inversion when flag set has missing values +(:class:`!IntFlag` still flips all bits); fix negative assigned values +during flag creation (both :class:`!Flag` and :class:`!IntFlag` ignore +missing values). + +.. + +.. date: 2025-04-07-10-20-16 +.. gh-issue: 87790 +.. nonce: X2SjJe +.. section: Library + +Support underscore and comma as thousands separators in the fractional part +for :class:`~fractions.Fraction`'s formatting. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-04-07-09-53-54 +.. gh-issue: 87790 +.. nonce: 6nj3zQ +.. section: Library + +Support underscore and comma as thousands separators in the fractional part +for :class:`~decimal.Decimal`'s formatting. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-04-07-06-41-54 +.. gh-issue: 131884 +.. nonce: ym9BJN +.. section: Library + +Fix formatting issues in :func:`json.dump` when both *indent* and *skipkeys* +are used. + +.. + +.. date: 2025-03-27-08-13-32 +.. gh-issue: 131788 +.. nonce: 0RWiFc +.. section: Library + +Make ``ResourceTracker.send`` from :mod:`multiprocessing` re-entrant safe + +.. + +.. date: 2025-03-19-12-41-42 +.. gh-issue: 91349 +.. nonce: 8eTOCP +.. section: Library + +Adjust default ``compressionlevel=`` to 6 (down from 9) in :mod:`gzip` and +:mod:`tarfile`. It is the default level used by most compression tools and a +better tradeoff between speed and performance. + +.. + +.. date: 2025-03-17-21-21-06 +.. gh-issue: 131146 +.. nonce: A5Obgv +.. section: Library + +Fix :class:`calendar.TextCalendar`, :class:`calendar.HTMLCalendar`, and the +:mod:`calendar` CLI to display month names in the nominative case by adding +:data:`calendar.standalone_month_name` and +:data:`calendar.standalone_month_abbr`, which provide month names and +abbreviations in the grammatical form used when a month name stands by +itself, if the locale supports it. + +.. + +.. date: 2025-03-13-20-48-58 +.. gh-issue: 123471 +.. nonce: cM4w4f +.. section: Library + +Make concurrent iterations over :class:`itertools.cycle` safe under +free-threading. + +.. + +.. date: 2025-03-11-05-24-14 +.. gh-issue: 130664 +.. nonce: g0yNMm +.. section: Library + +Handle corner-case for :class:`~fractions.Fraction`'s formatting: treat +zero-padding (preceding the width field by a zero (``'0'``) character) as an +equivalent to a fill character of ``'0'`` with an alignment type of ``'='``, +just as in case of :class:`float`'s. + +.. + +.. date: 2025-03-09-03-13-41 +.. gh-issue: 130999 +.. nonce: tBRBVB +.. section: Library + +Avoid exiting the new REPL and offer suggestions even if there are +non-string candidates when errors occur. + +.. + +.. date: 2025-03-08-17-07-00 +.. gh-issue: 88473 +.. nonce: qg23g8 +.. section: Library + +Implement a fast path for :class:`datetime.date` objects in +:func:`datetime.date.today` which results in a 5x performance gain while +proper subclasses retain their previous performance. + +.. + +.. date: 2024-11-25-10-22-08 +.. gh-issue: 126883 +.. nonce: MAEF7g +.. section: Library + +Add check that timezone fields are in range for +:meth:`datetime.datetime.fromisoformat` and +:meth:`datetime.time.fromisoformat`. Patch by Semyon Moroz. + +.. + +.. date: 2024-10-28-06-54-22 +.. gh-issue: 125028 +.. nonce: GEY8Ws +.. section: Library + +:data:`functools.Placeholder` cannot be passed to :func:`functools.partial` +as a keyword argument. + +.. + +.. date: 2024-10-22-16-21-55 +.. gh-issue: 125843 +.. nonce: 2ttzYo +.. section: Library + +If possible, indicate which :mod:`curses` C function or macro is responsible +for raising a :exc:`curses.error` exception. Patch by Bénédikt Tran. + +.. + +.. date: 2024-10-17-01-12-22 +.. gh-issue: 119109 +.. nonce: u4hcvb +.. section: Library + +:func:`functools.partial` calls are now faster when keyword arguments are +used. + +.. + +.. date: 2024-09-13-09-48-25 +.. gh-issue: 124033 +.. nonce: WNudS0 +.. section: Library + +``SimplePath`` is now presented in ``importlib.metadata.__all__``. + +.. + +.. date: 2024-09-13-09-46-47 +.. gh-issue: 91216 +.. nonce: LuOsF4 +.. section: Library + +``importlib.metadata`` now raises a ``KeyError`` instead of returning +``None`` when a key is missing from the metadata. + +.. + +.. date: 2024-09-13-09-43-15 +.. gh-issue: 120492 +.. nonce: Mm6CJ6 +.. section: Library + +``importlib.metadata`` now prioritizes valid dists to invalid dists when +retrieving by name. + +.. + +.. date: 2024-07-16-00-01-04 +.. gh-issue: 99631 +.. nonce: GWD4fD +.. section: Library + +The :mod:`shelve` module now accepts custom serialization and +deserialization functions. + +.. + +.. date: 2024-07-06-14-32-30 +.. gh-issue: 119186 +.. nonce: E5B1HQ +.. section: Library + +Slightly speed up :func:`os.walk` by calling :func:`os.path.join` less +often. + +.. + +.. date: 2024-06-06-17-49-07 +.. gh-issue: 120170 +.. nonce: DUxhmT +.. section: Library + +Fix an issue in the :mod:`!_pickle` extension module in which importing +:mod:`multiprocessing` could change how pickle identifies which module an +object belongs to, potentially breaking the unpickling of those objects. + +.. + +.. date: 2024-05-13-09-50-31 +.. gh-issue: 118981 +.. nonce: zgOQPv +.. section: Library + +Fix potential hang in ``multiprocessing.popen_spawn_posix`` that can happen +when the child proc dies early by closing the child fds right away. + +.. + +.. date: 2023-07-05-14-34-10 +.. gh-issue: 105497 +.. nonce: HU5u89 +.. section: Library + +Fix flag mask inversion when unnamed flags exist. + +.. + +.. date: 2023-03-13-22-51-40 +.. gh-issue: 99813 +.. nonce: 40TV02 +.. section: Library + +:mod:`ssl` now uses ``SSL_sendfile`` internally when it is possible (see +:data:`~ssl.OP_ENABLE_KTLS`). The function sends a file more efficiently +because it performs TLS encryption in the kernel to avoid additional context +switches. Patch by Illia Volochii. + +.. + +.. date: 2023-02-13-21-56-38 +.. gh-issue: 62824 +.. nonce: CBZzX3 +.. section: Library + +Fix aliases for ``iso8859_8`` encoding. Patch by Dave Goncalves. + +.. + +.. date: 2023-02-13-21-41-34 +.. gh-issue: 86155 +.. nonce: ppIGSC +.. section: Library + +:meth:`html.parser.HTMLParser.close` no longer loses data when the +``<script>`` tag is not closed. Patch by Waylan Limberg. + +.. + +.. date: 2023-02-13-20-34-52 +.. gh-issue: 78319 +.. nonce: V1zzed +.. section: Library + +UTF8 support for the IMAP APPEND command has been made RFC compliant. + +.. + +.. date: 2022-10-08-14-56-07 +.. gh-issue: 93334 +.. nonce: 0KUm8d +.. section: Library + +Reraise :exc:`KeyError` as :exc:`ModuleNotFoundError` when +:meth:`importlib.machinery.PathFinder.find_spec` is called on a submodule +without importing the parent (and without a ``path`` argument). + +.. + +.. date: 2022-07-24-20-56-32 +.. gh-issue: 69426 +.. nonce: unccw7 +.. section: Library + +Fix :class:`html.parser.HTMLParser` to not unescape character entities in +attribute values if they are followed by an ASCII alphanumeric or an equals +sign. + +.. + +.. bpo: 38735 +.. date: 2022-01-07-16-56-57 +.. nonce: NFfJX6 +.. section: Library + +Fix failure when importing a module from the root directory on unix-like +platforms with sys.pycache_prefix set. + +.. + +.. bpo: 45959 +.. date: 2021-12-18-12-46-20 +.. nonce: vPlr3P +.. section: Library + +:mod:`pprint` can now pretty-print dict views. + +.. + +.. date: 2021-09-21-17-17-29 +.. gh-issue: 84683 +.. nonce: wDSRsG +.. section: Library + +:mod:`zoneinfo`: Check in ``<prefix>/share/zoneinfo`` for data files on +Windows + +.. + +.. bpo: 43429 +.. date: 2021-03-07-16-31-36 +.. nonce: Koa0mf +.. section: Library + +The :meth:`~mmap.mmap.size` method of the :class:`mmap.mmap` class now +returns the size of an anonymous mapping on both Unix and Windows. +Previously, the size would be returned on Windows and an :exc:`OSError` +would be raised on Unix. :exc:`ValueError` is now raised instead of +:exc:`OSError` when ``trackfd=False``. + +.. + +.. bpo: 41839 +.. date: 2020-09-23-11-54-17 +.. nonce: kU5Ywl +.. section: Library + +Allow negative priority values from :func:`os.sched_get_priority_min` and +:func:`os.sched_get_priority_max` functions. + +.. + +.. bpo: 28494 +.. date: 2017-12-30-18-21-00 +.. nonce: Dt_Wks +.. section: Library + +Improve Zip file validation false positive rate in +:func:`zipfile.is_zipfile`. + +.. + +.. date: 2025-10-09-12-53-47 +.. gh-issue: 96491 +.. nonce: 4YKxvy +.. section: IDLE + +Deduplicate version number in IDLE shell title bar after saving to a file. + +.. + +.. date: 2025-10-08-08-35-50 +.. gh-issue: 139742 +.. nonce: B3fZLg +.. section: IDLE + +Colorize t-string prefixes for template strings in IDLE, as done for +f-string prefixes. + +.. + +.. date: 2025-07-01-23-00-58 +.. gh-issue: 136155 +.. nonce: 4siQQO +.. section: Documentation + +We are now checking for fatal errors in EPUB builds in CI. + +.. + +.. date: 2025-06-10-17-02-06 +.. gh-issue: 135171 +.. nonce: quHvts +.. section: Documentation + +Document that the :term:`iterator` for the leftmost :keyword:`!for` clause +in the generator expression is created immediately. + +.. + +.. bpo: 45210 +.. date: 2021-09-15-13-07-25 +.. nonce: RtGk7i +.. section: Documentation + +Document that error indicator may be set in tp_dealloc, and how to avoid +clobbering it. + +.. + +.. date: 2025-10-13-17-56-23 +.. gh-issue: 140000 +.. nonce: tLhn3e +.. section: Core and Builtins + +Fix potential memory leak when a reference cycle exists between an instance +of :class:`typing.TypeAliasType`, :class:`typing.TypeVar`, +:class:`typing.ParamSpec`, or :class:`typing.TypeVarTuple` and its +``__name__`` attribute. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-12-18-54-06 +.. gh-issue: 140009 +.. nonce: -MbFh_ +.. section: Core and Builtins + +Improve performance of list extension by dictionary items. + +.. + +.. date: 2025-10-12-11-00-06 +.. gh-issue: 139988 +.. nonce: 4wi51t +.. section: Core and Builtins + +Fix a memory leak when failing to create a :class:`~typing.Union` type. +Patch by Bénédikt Tran. + +.. + +.. date: 2025-10-08-13-52-00 +.. gh-issue: 139748 +.. nonce: jq0yFJ +.. section: Core and Builtins + +Fix reference leaks in error branches of functions accepting path strings or +bytes such as :func:`compile` and :func:`os.system`. Patch by Bénédikt Tran. + +.. + +.. date: 2025-10-06-13-15-26 +.. gh-issue: 139516 +.. nonce: d9Pkur +.. section: Core and Builtins + +Fix lambda colon erroneously start format spec in f-string in tokenizer. + +.. + +.. date: 2025-10-01-18-21-19 +.. gh-issue: 63161 +.. nonce: ef1S6N +.. section: Core and Builtins + +Support non-UTF-8 shebang and comments in Python source files if non-UTF-8 +encoding is specified. Detect decoding error in comments for default (UTF-8) +encoding. Show the line and position of decoding error for default encoding +in a traceback. Show the line containing the coding cookie when it conflicts +with the BOM in a traceback. + +.. + +.. date: 2025-09-30-14-57-19 +.. gh-issue: 139116 +.. nonce: nlVf40 +.. section: Core and Builtins + +Prevent a deadlock when multiple threads start, stop and use +:mod:`tracemalloc` simultaneously. + +.. + +.. date: 2025-09-24-17-32-52 +.. gh-issue: 139275 +.. nonce: novrqf +.. section: Core and Builtins + +Fix compilation problems in ``_remote_debugging_module.c`` when the system +doesn't have ``process_vm_readv``. Patch by Pablo Galindo + +.. + +.. date: 2025-09-24-17-08-42 +.. gh-issue: 133059 +.. nonce: EXvxb7 +.. section: Core and Builtins + +Increased the number of cached small positive integers from 256 to 1024. + +.. + +.. date: 2025-09-22-15-21-49 +.. gh-issue: 74857 +.. nonce: 5XRQaA +.. section: Core and Builtins + +:pep:`538`: Coerce the POSIX locale to a UTF-8 based locale. Patch by Victor +Stinner. + +.. + +.. date: 2025-09-21-14-33-17 +.. gh-issue: 116738 +.. nonce: vNaI4h +.. section: Core and Builtins + +Make :mod:`mmap` thread-safe on the :term:`free threaded <free threading>` +build. + +.. + +.. date: 2025-09-17-17-17-21 +.. gh-issue: 138558 +.. nonce: 0VbzCH +.. section: Core and Builtins + +Fix handling of unusual t-string annotations in annotationlib. Patch by Dave +Peck. + +.. + +.. date: 2025-09-15-14-04-56 +.. gh-issue: 134466 +.. nonce: yR4fYW +.. section: Core and Builtins + +Don't run PyREPL in a degraded environment where setting termios attributes +is not allowed. + +.. + +.. date: 2025-09-11-15-56-18 +.. gh-issue: 138794 +.. nonce: nrOn1K +.. section: Core and Builtins + +When a new tracing function is registered with +:c:func:`PyRefTracer_SetTracer`, replacing the current a call to the trace +function will be made with the object set to **NULL** and **event** set to +:c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new +function is registered. Patch by Pablo Galindo + +.. + +.. date: 2025-09-10-14-53-59 +.. gh-issue: 71810 +.. nonce: ppf0J- +.. section: Core and Builtins + +Raise :exc:`OverflowError` for ``(-1).to_bytes()`` for signed conversions +when bytes count is zero. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-09-09-23-59-13 +.. gh-issue: 138716 +.. nonce: UawDY0 +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for :keyword:`assert` in cases like +``assert a := b``. + +.. + +.. date: 2025-09-06-13-53-33 +.. gh-issue: 105487 +.. nonce: a43YaY +.. section: Core and Builtins + +Remove non-existent :meth:`~object.__copy__`, :meth:`~object.__deepcopy__`, +and :attr:`~type.__bases__` from the :meth:`~object.__dir__` entries of +:class:`types.GenericAlias`. + +.. + +.. date: 2025-09-05-01-19-04 +.. gh-issue: 138192 +.. nonce: erluq5 +.. section: Core and Builtins + +Fix :mod:`contextvars` initialization so that all subinterpreters are +assigned the :attr:`~contextvars.Token.MISSING` value. + +.. + +.. date: 2025-09-03-17-00-30 +.. gh-issue: 138479 +.. nonce: qUxgWs +.. section: Core and Builtins + +Fix a crash when a generic object's ``__typing_subst__`` returns an object +that isn't a :class:`tuple`. + +.. + +.. date: 2025-09-03-15-35-34 +.. gh-issue: 138431 +.. nonce: EUsrtA +.. section: Core and Builtins + +Fix a bug in the JIT optimizer when round-tripping strings and tuples. + +.. + +.. date: 2025-09-03-10-16-09 +.. gh-issue: 138378 +.. nonce: r6BQxV +.. section: Core and Builtins + +Move the globals-to-const JIT optimizer pass into to the main JIT optimizer +pass + +.. + +.. date: 2025-09-02-22-17-55 +.. gh-issue: 138401 +.. nonce: uTRvue +.. section: Core and Builtins + +Add missing validation of argument ``count`` in :func:`os.sendfile` to be +non-negative. + +.. + +.. date: 2025-09-02-09-10-06 +.. gh-issue: 138372 +.. nonce: h1Xk4- +.. section: Core and Builtins + +Fix :exc:`SyntaxWarning` emitted for erroneous subscript expressions +involving :ref:`template string literals <t-strings>`. Patch by Brian +Schubert. + +.. + +.. date: 2025-09-01-21-52-54 +.. gh-issue: 138302 +.. nonce: -ez47B +.. section: Core and Builtins + +``BINARY_OP`` now specializes to ``BINARY_OP_ADD_INT``, +``BINARY_OP_SUBTRACT_INT`` or ``BINARY_OP_MULTIPLY_INT`` if operands are +compact ints. + +.. + +.. date: 2025-09-01-16-09-02 +.. gh-issue: 138318 +.. nonce: t-WEN5 +.. section: Core and Builtins + +The default REPL now avoids highlighting built-in names (for instance +:class:`set` or :func:`format`) when they are used as attribute names (for +instance in ``value.set`` or ``text.format``). + +.. + +.. date: 2025-09-01-13-54-43 +.. gh-issue: 138349 +.. nonce: 0fGmAi +.. section: Core and Builtins + +Fix crash in certain cases where a module contains both a module-level +annotation and a comprehension. + +.. + +.. date: 2025-08-30-17-15-05 +.. gh-issue: 69605 +.. nonce: KjBk99 +.. section: Core and Builtins + +Fix some standard library submodules missing from the :term:`REPL` +auto-completion of imports. + +.. + +.. date: 2025-08-30-00-55-35 +.. gh-issue: 61206 +.. nonce: HeFLvl +.. section: Core and Builtins + +:mod:`zipimport` now supports zstandard compressed zip file entries. + +.. + +.. date: 2025-08-28-09-29-46 +.. gh-issue: 116738 +.. nonce: yLZJpV +.. section: Core and Builtins + +Make :mod:`cProfile` thread-safe on the :term:`free threaded <free +threading>` build. + +.. + +.. date: 2025-08-27-17-51-38 +.. gh-issue: 137838 +.. nonce: lK6T0j +.. section: Core and Builtins + +Fix JIT trace buffer overrun by increasing possible exit stubs. Patch by +Donghee Na. + +.. + +.. date: 2025-08-27-13-11-47 +.. gh-issue: 71679 +.. nonce: V0yFeT +.. section: Core and Builtins + +Use the same quoting algorithm for the repr of bytearrays as for bytes +objects and strings -- use double quotes for quoting if the bytearray +contains single quotes and does not contain double quotes. + +.. + +.. date: 2025-08-22-11-39-40 +.. gh-issue: 137384 +.. nonce: j4b_in +.. section: Core and Builtins + +Fix a crash when using the :mod:`warnings` module in a finalizer at +shutdown. Patch by Kumar Aditya. + +.. + +.. date: 2025-08-21-06-31-42 +.. gh-issue: 138004 +.. nonce: FH2Hre +.. section: Core and Builtins + +On Solaris/Illumos platforms, thread names are now encoded as ASCII to avoid +errors on systems (e.g. OpenIndiana) that don't support non-ASCII names. + +.. + +.. date: 2025-08-21-01-46-39 +.. gh-issue: 137976 +.. nonce: p4sb4x +.. section: Core and Builtins + +Removed ``localtime`` from the list of reported system timezones. + +.. + +.. date: 2025-08-20-14-17-47 +.. gh-issue: 137992 +.. nonce: fcL3SK +.. section: Core and Builtins + +Ensure that :c:func:`PyRefTracer_SetTracer` sync with all existing threads +when called to avoid races in the free threaded build. Patch by Pablo +Galindo + +.. + +.. date: 2025-08-19-18-52-22 +.. gh-issue: 137967 +.. nonce: uw67Ys +.. section: Core and Builtins + +Show error suggestions on nested attribute access. Patch by Pablo Galindo + +.. + +.. date: 2025-08-19-16-07-07 +.. gh-issue: 137959 +.. nonce: EWj0RZ +.. section: Core and Builtins + +Replace the shim code added to every piece of jitted code with a single +trampoline function. + +.. + +.. date: 2025-08-17-13-36-53 +.. gh-issue: 137883 +.. nonce: 55VDCN +.. section: Core and Builtins + +Fix runaway recursion when calling a function with keyword arguments. + +.. + +.. date: 2025-08-15-15-45-26 +.. gh-issue: 137079 +.. nonce: YEow69 +.. section: Core and Builtins + +Fix keyword typo recognition when parsing files. Patch by Pablo Galindo. + +.. + +.. date: 2025-08-14-14-18-29 +.. gh-issue: 137728 +.. nonce: HdYS9R +.. section: Core and Builtins + +Fix the JIT's handling of many local variables. This previously caused a +segfault. + +.. + +.. date: 2025-08-13-16-58-58 +.. gh-issue: 137716 +.. nonce: ZcZSyi +.. section: Core and Builtins + +Fix double period in :exc:`AttributeError` message for invalid mock +assertions + +.. + +.. date: 2025-08-13-13-39-02 +.. gh-issue: 137433 +.. nonce: g6Atfz +.. section: Core and Builtins + +Fix a potential deadlock in the :term:`free threading` build when daemon +threads enable or disable profiling or tracing while the main thread is +shutting down the interpreter. + +.. + +.. date: 2025-08-10-21-34-12 +.. gh-issue: 137576 +.. nonce: 0ZicS- +.. section: Core and Builtins + +Fix for incorrect source code being shown in tracebacks from the Basic REPL +when :envvar:`PYTHONSTARTUP` is given. Patch by Adam Hartz. + +.. + +.. date: 2025-08-09-11-38-37 +.. gh-issue: 37817 +.. nonce: Y5Fhde +.. section: Core and Builtins + +Allow assignment to :attr:`~type.__bases__` of direct subclasses of builtin +classes. + +.. + +.. date: 2025-08-09-04-07-05 +.. gh-issue: 132732 +.. nonce: 8BiIVJ +.. section: Core and Builtins + +Optimize ``_COMPARE_OP``, ``_CONTAINS_OP``, ``_UNARY_NEGATIVE``, +``_UNARY_NOT``, and ``_UNARY_INVERT`` in JIT builds with constant-loading +uops (``_POP_TWO_LOAD_CONST_INLINE_BORROW`` and +``_POP_TOP_LOAD_CONST_INLINE_BORROW``), and then remove both to reduce +instruction count. + +.. + +.. date: 2025-08-07-09-52-19 +.. gh-issue: 137400 +.. nonce: AK1dy- +.. section: Core and Builtins + +Fix a crash in the :term:`free threading` build when disabling profiling or +tracing across all threads with :c:func:`PyEval_SetProfileAllThreads` or +:c:func:`PyEval_SetTraceAllThreads` or their Python equivalents +:func:`threading.settrace_all_threads` and +:func:`threading.setprofile_all_threads`. + +.. + +.. date: 2025-08-06-16-55-44 +.. gh-issue: 133143 +.. nonce: l7CI9v +.. section: Core and Builtins + +Add :data:`sys.abi_info` object to make ABI information more easily +accessible. + +.. + +.. date: 2025-08-06-15-39-54 +.. gh-issue: 137400 +.. nonce: xIw0zs +.. section: Core and Builtins + +Fix a crash in the :term:`free threading` build when disabling profiling or +tracing across all threads with :c:func:`PyEval_SetProfileAllThreads` or +:c:func:`PyEval_SetTraceAllThreads` or their Python equivalents +:func:`threading.settrace_all_threads` and +:func:`threading.setprofile_all_threads`. + +.. + +.. date: 2025-08-05-20-24-12 +.. gh-issue: 120037 +.. nonce: MB7MmI +.. section: Core and Builtins + +Disable user site packages directory when a ``._pth`` file is used, even if +it contains ``import site``. + +.. + +.. date: 2025-08-05-17-22-24 +.. gh-issue: 58124 +.. nonce: q1__53 +.. section: Core and Builtins + +Fix name of the Python encoding in Unicode errors of the code page codec: +use "cp65000" and "cp65001" instead of "CP_UTF7" and "CP_UTF8" which are not +valid Python code names. Patch by Victor Stinner. + +.. + +.. date: 2025-08-05-10-22-15 +.. gh-issue: 136966 +.. nonce: J5lrE0 +.. section: Core and Builtins + +The :attr:`object.__dict__` and :attr:`!__weakref__` descriptors now use a +single descriptor instance per interpreter, shared across all types that +need them. This speeds up class creation, and helps avoid reference cycles. + +.. + +.. date: 2025-08-02-23-04-57 +.. gh-issue: 137314 +.. nonce: wjEdzD +.. section: Core and Builtins + +Fixed a regression where raw f-strings incorrectly interpreted escape +sequences in format specifications. Raw f-strings now properly preserve +literal backslashes in format specs, matching the behavior from Python 3.11. +For example, ``rf"{obj:\xFF}"`` now correctly produces ``'\\xFF'`` instead +of ``'ÿ'``. Patch by Pablo Galindo. + +.. + +.. date: 2025-08-02-10-27-53 +.. gh-issue: 137308 +.. nonce: at05p_ +.. section: Core and Builtins + +A standalone docstring in a node body is optimized as a :keyword:`pass` +statement to ensure that the node's body is never empty. There was a +:exc:`ValueError` in :func:`compile` otherwise. + +.. + +.. date: 2025-08-01-18-54-31 +.. gh-issue: 137288 +.. nonce: FhE7ku +.. section: Core and Builtins + +Fix bug where some bytecode instructions of a boolean expression are not +associated with the correct exception handler. + +.. + +.. date: 2025-07-31-23-02-02 +.. gh-issue: 137291 +.. nonce: kIxVZd +.. section: Core and Builtins + +The perf profiler can now be used if a previous frame evaluation API has +been provided. + +.. + +.. date: 2025-07-28-19-11-34 +.. gh-issue: 134291 +.. nonce: IiB9Id +.. section: Core and Builtins + +Remove some newer macOS API usage from the JIT compiler in order to restore +compatibility with older OSX 10.15 deployment targets. + +.. + +.. date: 2025-07-28-17-01-05 +.. gh-issue: 88886 +.. nonce: g4XFPb +.. section: Core and Builtins + +The codecs lookup function now again performs only minimal normalization of +the encoding name before passing it to the search functions: all ASCII +letters are converted to lower case, spaces are replaced with hyphens. This +restores the pre-Python 3.9 behavior. + +.. + +.. date: 2025-07-25-22-31-52 +.. gh-issue: 131338 +.. nonce: zJDCMp +.. section: Core and Builtins + +Disable computed stack limit checks on non-glibc linux platforms to fix +crashes on deep recursion. + +.. + +.. date: 2025-07-24-17-30-58 +.. gh-issue: 136870 +.. nonce: ncx82J +.. section: Core and Builtins + +Fix data races while de-instrumenting bytecode of code objects running +concurrently in threads. + +.. + +.. date: 2025-07-24-02-13-59 +.. gh-issue: 132732 +.. nonce: p77xkb +.. section: Core and Builtins + +Optimize constant comparison for ``_COMPARE_OP_INT``, ``_COMPARE_OP_FLOAT`` +and ``_COMPARE_OP_STR`` in JIT builds + +.. + +.. date: 2025-07-19-17-08-09 +.. gh-issue: 127598 +.. nonce: Mx8S-y +.. section: Core and Builtins + +Improve :exc:`ModuleNotFoundError` by adding flavour text to the exception +when the :option:`-S` option is passed. Patch by Andrea Mattei. + +.. + +.. date: 2025-07-19-12-37-05 +.. gh-issue: 136801 +.. nonce: XU_tF2 +.. section: Core and Builtins + +Fix PyREPL syntax highlighting on match cases after multi-line case. +Contributed by Olga Matoula. + +.. + +.. date: 2025-07-19-10-35-31 +.. gh-issue: 74185 +.. nonce: 7hPCA5 +.. section: Core and Builtins + +The :meth:`~object.__repr__` of :class:`ImportError` and +:class:`ModuleNotFoundError` now shows "name" and "path" as ``name=<name>`` +and ``path=<path>`` if they were given as keyword arguments at construction +time. Patch by Serhiy Storchaka, Oleg Iarygin, and Yoav Nir + +.. + +.. date: 2025-07-18-08-43-35 +.. gh-issue: 116738 +.. nonce: i0HWtP +.. section: Core and Builtins + +Make functions in :mod:`syslog` thread-safe on the :term:`free threaded +<free threading>` build. + +.. + +.. date: 2025-07-15-10-03-57 +.. gh-issue: 116738 +.. nonce: oFttKl +.. section: Core and Builtins + +Make functions in :mod:`pwd` thread-safe on the :term:`free threaded <free +threading>` build. + +.. + +.. date: 2025-07-14-17-01-23 +.. gh-issue: 136616 +.. nonce: FQjXE_ +.. section: Core and Builtins + +Improve :exc:`SyntaxError` error messages for invalid :keyword:`assert` +usages. + +.. + +.. date: 2025-07-13-21-21-17 +.. gh-issue: 136599 +.. nonce: sLhm2O +.. section: Core and Builtins + +Improve performance of :class:`int` hash calculations. + +.. + +.. date: 2025-07-12-09-59-14 +.. gh-issue: 136421 +.. nonce: ZD1rNj +.. section: Core and Builtins + +Fix crash when initializing :mod:`datetime` concurrently. + +.. + +.. date: 2025-07-11-13-45-48 +.. gh-issue: 136541 +.. nonce: uZ_-Ju +.. section: Core and Builtins + +Fix some issues with the perf trampolines on x86-64 and aarch64. The +trampolines were not being generated correctly for some cases, which could +lead to the perf integration not working correctly. Patch by Pablo Galindo. + +.. + +.. date: 2025-07-11-12-29-09 +.. gh-issue: 107545 +.. nonce: ipfl7U +.. section: Core and Builtins + +Improve the error messages that may be raised by +:meth:`~socket.socket.setsockopt`. + +.. + +.. date: 2025-07-10-23-23-50 +.. gh-issue: 136517 +.. nonce: _NHJyv +.. section: Core and Builtins + +Fixed a typo that prevented printing of uncollectable objects when the +:const:`gc.DEBUG_UNCOLLECTABLE` mode was set. + +.. + +.. date: 2025-07-10-15-53-16 +.. gh-issue: 136525 +.. nonce: xAko0e +.. section: Core and Builtins + +Fix issue where per-thread bytecode was not instrumented for newly created +threads. + +.. + +.. date: 2025-07-09-21-27-14 +.. gh-issue: 132657 +.. nonce: kSA8R3 +.. section: Core and Builtins + +Improve performance of :class:`frozenset` by removing locks in the +free-threading build. + +.. + +.. date: 2025-07-09-11-15-42 +.. gh-issue: 136459 +.. nonce: m4Udh8 +.. section: Core and Builtins + +Add support for perf trampoline on macOS, to allow profilers wit JIT map +support to read Python calls. While profiling, ``PYTHONPERFSUPPORT=1`` can +be appended to enable the trampoline. + +.. + +.. date: 2025-07-08-23-53-51 +.. gh-issue: 132661 +.. nonce: B84iYt +.. section: Core and Builtins + +``Interpolation.expression`` now has a default, the empty string. + +.. + +.. date: 2025-07-08-23-22-08 +.. gh-issue: 132661 +.. nonce: 34ftJl +.. section: Core and Builtins + +Reflect recent :pep:`750` change. + +Disallow concatenation of ``string.templatelib.Template`` and :class:`str`. +Also, disallow implicit concatenation of t-string literals with string or +f-string literals. + +.. + +.. date: 2025-07-07-17-26-06 +.. gh-issue: 91636 +.. nonce: GyHU72 +.. section: Core and Builtins + +While performing garbage collection, clear weakrefs to unreachable objects +that are created during running of finalizers. If those weakrefs were are +not cleared, they could reveal unreachable objects. + +.. + +.. date: 2025-07-07-12-24-00 +.. gh-issue: 136355 +.. nonce: MTcA8j +.. section: Core and Builtins + +Deprecate :option:`-b` and :option:`!-bb` command line options and schedule +them to become no-op in Python 3.17. + +.. + +.. date: 2025-07-06-14-53-19 +.. gh-issue: 109700 +.. nonce: KVNQQi +.. section: Core and Builtins + +Fix memory error handling in :c:func:`PyDict_SetDefault`. + +.. + +.. date: 2025-07-03-06-04-42 +.. gh-issue: 135552 +.. nonce: CbBQof +.. section: Core and Builtins + +Fix a bug caused by the garbage collector clearing weakrefs too early. The +weakrefs in the ``tp_subclasses`` dictionary are needed in order to +correctly invalidate type caches (for example, by calling +``PyType_Modified()``). Clearing weakrefs before calling finalizers causes +the caches to not be correctly invalidated. That can cause crashes since +the caches can refer to invalid objects. Defer the clearing of weakrefs +without callbacks until after finalizers are executed. + +.. + +.. date: 2025-07-02-15-18-41 +.. gh-issue: 136203 +.. nonce: Y934sC +.. section: Core and Builtins + +Improve :exc:`TypeError` error message, when richcomparing two +:class:`types.MappingProxyType` objects. + +.. + +.. date: 2025-06-26-18-44-34 +.. gh-issue: 136003 +.. nonce: sln51d +.. section: Core and Builtins + +Fix :class:`threading.Thread` objects becoming incorrectly daemon when +created from an :mod:`atexit` callback or a pending call +(:c:func:`Py_AddPendingCall`). + +.. + +.. date: 2025-06-26-15-25-51 +.. gh-issue: 78465 +.. nonce: MbDN8X +.. section: Core and Builtins + +Fix error message for ``cls.__new__(cls, ...)`` where ``cls`` is not +instantiable builtin or extension type (with ``tp_new`` set to ``NULL``). + +.. + +.. date: 2025-06-24-16-46-34 +.. gh-issue: 135904 +.. nonce: 78xfon +.. section: Core and Builtins + +Perform more aggressive control-flow optimizations on the machine code +templates emitted by the experimental JIT compiler. + +.. + +.. date: 2025-06-24-06-41-47 +.. gh-issue: 129958 +.. nonce: EaJuS0 +.. section: Core and Builtins + +Differentiate between t-strings and f-strings in syntax error for newlines +in format specifiers of single-quoted interpolated strings. + +.. + +.. date: 2025-06-23-18-08-32 +.. gh-issue: 135871 +.. nonce: 50C528 +.. section: Core and Builtins + +Non-blocking mutex lock attempts now return immediately when the lock is +busy instead of briefly spinning in the :term:`free threading` build. + +.. + +.. date: 2025-06-20-14-50-44 +.. gh-issue: 134584 +.. nonce: 3CJdAI +.. section: Core and Builtins + +Specialize :opcode:`POP_TOP` in the JIT compiler by specializing for +reference lifetime and type. This will also enable easier top of stack +caching in the JIT compiler. + +.. + +.. date: 2025-06-18-16-45-36 +.. gh-issue: 135106 +.. nonce: cpl6Aq +.. section: Core and Builtins + +Restrict the trashcan mechanism to GC'ed objects and untrack them while in +the trashcan to prevent the GC and trashcan mechanisms conflicting. + +.. + +.. date: 2025-06-18-12-19-13 +.. gh-issue: 135379 +.. nonce: TCvGpj +.. section: Core and Builtins + +Changes specialization of ``BINARY_OP`` for ints to only specialize for +"compact" ints. This streamlines the fast path at the cost of fewer +specializations when very large integers are used. + +.. + +.. date: 2025-06-17-22-34-58 +.. gh-issue: 135607 +.. nonce: ucsLVu +.. section: Core and Builtins + +Fix potential :mod:`weakref` races in an object's destructor on the +:term:`free threaded <free threading>` build. + +.. + +.. date: 2025-06-17-12-50-48 +.. gh-issue: 135608 +.. nonce: PnHckD +.. section: Core and Builtins + +Fix a crash in the JIT involving attributes of modules. + +.. + +.. date: 2025-06-17-08-37-45 +.. gh-issue: 82088 +.. nonce: TgPvLg +.. section: Core and Builtins + +Improve performance of ``PyLongObject`` conversion functions +``PyLong_AsLongAndOverflow()``, ``PyLong_AsSsize_t()``, +``PyLong_AsUnsignedLong()``, ``PyLong_AsSize_t()``, +``PyLong_AsUnsignedLongMask()``, ``PyLong_AsUnsignedLongLongMask()``, +``PyLong_AsLongLongAndOverflow()`` for integers larger than 2**30 up to 30%. + +.. + +.. date: 2025-06-16-03-56-15 +.. gh-issue: 135551 +.. nonce: hRTQO- +.. section: Core and Builtins + +Sorting randomly ordered lists will often run a bit faster, thanks to a new +scheme for picking minimum run lengths from Stefan Pochmann, which arranges +for the merge tree to be as evenly balanced as is possible. + +.. + +.. date: 2025-06-16-02-31-42 +.. gh-issue: 135543 +.. nonce: 6b0HOF +.. section: Core and Builtins + +Emit ``sys.remote_exec`` audit event when :func:`sys.remote_exec` is called +and migrate ``remote_debugger_script`` to +``cpython.remote_debugger_script``. + +.. + +.. date: 2025-06-14-01-01-14 +.. gh-issue: 135496 +.. nonce: ER0Me3 +.. section: Core and Builtins + +Fix typo in the f-string conversion type error ("exclamanation" -> +"exclamation"). + +.. + +.. date: 2025-06-13-16-05-24 +.. gh-issue: 135474 +.. nonce: 67nOl3 +.. section: Core and Builtins + +Specialize integer operations only on compact integers. This is a CPython +internal change. + +.. + +.. date: 2025-06-12-18-12-42 +.. gh-issue: 135371 +.. nonce: R_YUtR +.. section: Core and Builtins + +Fixed :mod:`asyncio` debugging tools to properly display internal coroutine +call stacks alongside external task dependencies. The ``python -m asyncio +ps`` and ``python -m asyncio pstree`` commands now show complete execution +context. Patch by Pablo Galindo. + +.. + +.. date: 2025-06-12-11-19-52 +.. gh-issue: 135422 +.. nonce: F6yQi6 +.. section: Core and Builtins + +Fix regression in :exc:`SyntaxError` messages after :gh:`134036`. + +.. + +.. date: 2025-06-12-00-03-34 +.. gh-issue: 116738 +.. nonce: iBBAdo +.. section: Core and Builtins + +Make functions in :mod:`grp` thread-safe on the :term:`free threaded <free +threading>` build. + +.. + +.. date: 2025-06-11-15-08-10 +.. gh-issue: 127319 +.. nonce: OVGFSZ +.. section: Core and Builtins + +Set the ``allow_reuse_port`` class variable to ``False`` on the XMLRPC, +logging, and HTTP servers. This matches the behavior in prior Python +releases, which is to not allow port reuse. + +.. + +.. date: 2025-06-09-23-57-37 +.. gh-issue: 130077 +.. nonce: MHknDB +.. section: Core and Builtins + +Properly raise custom syntax errors when incorrect syntax containing names +that are prefixes of soft keywords is encountered. Patch by Pablo Galindo. + +.. + +.. date: 2025-06-08-14-24-29 +.. gh-issue: 131798 +.. nonce: qfw91T +.. section: Core and Builtins + +Optimize _CALL_LEN in the JIT when the length is known. Patch by Tomas Roun + +.. + +.. date: 2025-06-06-19-17-22 +.. gh-issue: 131798 +.. nonce: XoV8Eb +.. section: Core and Builtins + +Optimize ``_UNARY_NEGATIVE`` in JIT-compiled code. + +.. + +.. date: 2025-06-06-02-24-42 +.. gh-issue: 135148 +.. nonce: r-t2sC +.. section: Core and Builtins + +Fixed a bug where f-string debug expressions (using =) would incorrectly +strip out parts of strings containing escaped quotes and # characters. Patch +by Pablo Galindo. + +.. + +.. date: 2025-06-06-01-09-44 +.. gh-issue: 131798 +.. nonce: 1SuxO9 +.. section: Core and Builtins + +Optimize ``_UNARY_INVERT`` in JIT-compiled code. + +.. + +.. date: 2025-06-05-21-58-30 +.. gh-issue: 131798 +.. nonce: nt5Ab7 +.. section: Core and Builtins + +Optimize away ``_CALL_TYPE_1`` in the JIT when the return type is known. +Patch by Tomas Roun + +.. + +.. date: 2025-06-03-21-06-22 +.. gh-issue: 133136 +.. nonce: Usnvri +.. section: Core and Builtins + +Limit excess memory usage in the :term:`free threading` build when a large +dictionary or list is resized and accessed by multiple threads. + +.. + +.. date: 2025-06-02-20-13-37 +.. gh-issue: 131798 +.. nonce: JQRFvR +.. section: Core and Builtins + +Optimize ``_CHECK_METHOD_VERSION`` into ``_CHECK_FUNCTION_VERSION_INLINE`` +in JIT-compiled code. + +.. + +.. date: 2025-06-02-13-57-40 +.. gh-issue: 116738 +.. nonce: ycJsL8 +.. section: Core and Builtins + +Make methods in :mod:`heapq` thread-safe on the :term:`free threaded <free +threading>` build. + +.. + +.. date: 2025-05-31-19-24-54 +.. gh-issue: 134280 +.. nonce: NDVbzY +.. section: Core and Builtins + +Disable constant folding for ``~`` with a boolean argument. This moves the +deprecation warning from compile time to runtime. + +.. + +.. date: 2025-05-31-10-26-46 +.. gh-issue: 134876 +.. nonce: 8mBGJI +.. section: Core and Builtins + +Add support to :pep:`768` remote debugging for Linux kernels which don't +have CONFIG_CROSS_MEMORY_ATTACH configured. + +.. + +.. date: 2025-05-30-18-09-54 +.. gh-issue: 134889 +.. nonce: Ic9UM- +.. section: Core and Builtins + +Fix handling of a few opcodes that leave operands on the stack when +optimizing ``LOAD_FAST``. + +.. + +.. date: 2025-05-30-15-56-19 +.. gh-issue: 134908 +.. nonce: 3a7PxM +.. section: Core and Builtins + +Fix crash when iterating over lines in a text file on the :term:`free +threaded <free threading>` build. + +.. + +.. date: 2025-05-28-23-58-50 +.. gh-issue: 117852 +.. nonce: BO9g7z +.. section: Core and Builtins + +Fix argument checking of :meth:`~agen.athrow`. + +.. + +.. date: 2025-05-27-20-29-00 +.. gh-issue: 132617 +.. nonce: EmUfQQ +.. section: Core and Builtins + +Fix :meth:`dict.update` modification check that could incorrectly raise a +"dict mutated during update" error when a different dictionary was modified +that happens to share the same underlying keys object. + +.. + +.. date: 2025-05-27-20-21-34 +.. gh-issue: 131798 +.. nonce: b32zkl +.. section: Core and Builtins + +Allow the JIT to remove unnecessary ``_ITER_CHECK_TUPLE`` ops. + +.. + +.. date: 2025-05-27-18-59-54 +.. gh-issue: 134679 +.. nonce: FWPBu6 +.. section: Core and Builtins + +Fix crash in the :term:`free threading` build's QSBR code that could occur +when changing an object's ``__dict__`` attribute. + +.. + +.. date: 2025-05-26-15-55-50 +.. gh-issue: 133912 +.. nonce: -xAguL +.. section: Core and Builtins + +Fix the C API function ``PyObject_GenericSetDict`` to handle extension +classes with inline values. + +.. + +.. date: 2025-05-25-19-32-15 +.. gh-issue: 131798 +.. nonce: f5h8aI +.. section: Core and Builtins + +Make the JIT optimizer understand that slicing a string/list/tuple returns +the same type. + +.. + +.. date: 2025-05-23-14-54-07 +.. gh-issue: 134584 +.. nonce: y-WDjf +.. section: Core and Builtins + +Add a reference count elimination pass to the JIT compiler. Patch by Ken +Jin. + +.. + +.. date: 2025-05-22-17-49-39 +.. gh-issue: 131798 +.. nonce: U6ZmFm +.. section: Core and Builtins + +Optimize ``_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW``. + +.. + +.. date: 2025-05-22-14-48-19 +.. gh-issue: 134381 +.. nonce: 2BXhth +.. section: Core and Builtins + +Fix :exc:`RuntimeError` when using a not-started :class:`threading.Thread` +after calling :func:`os.fork` + +.. + +.. date: 2025-05-21-18-02-56 +.. gh-issue: 127960 +.. nonce: W3J_2X +.. section: Core and Builtins + +PyREPL interactive shell no longer starts with ``__package__`` and +``__file__`` global names set to ``_pyrepl`` package internals. Contributed +by Yuichiro Tachibana. + +.. + +.. date: 2025-05-21-15-14-32 +.. gh-issue: 130397 +.. nonce: aG6EON +.. section: Core and Builtins + +Remove special-casing for C stack depth limits for WASI. Due to +WebAssembly's built-in stack protection this does not pose a security +concern. + +.. + +.. date: 2025-05-21-13-57-26 +.. gh-issue: 131798 +.. nonce: QwS5Bb +.. section: Core and Builtins + +JIT: replace ``_LOAD_SMALL_INT`` with ``_LOAD_CONST_INLINE_BORROW`` + +.. + +.. date: 2025-05-20-23-32-11 +.. gh-issue: 131798 +.. nonce: G9ZQZw +.. section: Core and Builtins + +Improve the JIT's ability to optimize away cached class attribute and method +loads. + +.. + +.. date: 2025-05-20-14-41-50 +.. gh-issue: 128066 +.. nonce: qzzGfv +.. section: Core and Builtins + +Fixes an edge case where PyREPL improperly threw an error when Python is +invoked on a read only filesystem while trying to write history file +entries. + +.. + +.. date: 2025-05-20-13-58-18 +.. gh-issue: 131798 +.. nonce: hG8xBw +.. section: Core and Builtins + +Improve the JIT's ability to narrow unknown classes to constant values. + +.. + +.. date: 2025-05-19-20-52-53 +.. gh-issue: 134268 +.. nonce: HPKX1e +.. section: Core and Builtins + +Add ``_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW`` and use it to further +optimize ``CALL_ISINSTANCE``. + +.. + +.. date: 2025-05-19-15-15-58 +.. gh-issue: 131798 +.. nonce: PCP71j +.. section: Core and Builtins + +Split ``CALL_LIST_APPEND`` into several uops. Patch by Diego Russo. + +.. + +.. date: 2025-05-18-14-33-23 +.. gh-issue: 69605 +.. nonce: ZMO49F +.. section: Core and Builtins + +When auto-completing an import in the :term:`REPL`, finding no candidates +now issues no suggestion, rather than suggestions from the current +namespace. + +.. + +.. date: 2025-05-18-10-50-46 +.. gh-issue: 134170 +.. nonce: J0Hvmi +.. section: Core and Builtins + +Add colorization to :func:`sys.unraisablehook` by default. + +.. + +.. date: 2025-05-17-20-56-05 +.. gh-issue: 91153 +.. nonce: afgtG2 +.. section: Core and Builtins + +Fix a crash when a :class:`bytearray` is concurrently mutated during item +assignment. + +.. + +.. date: 2025-05-17-20-44-51 +.. gh-issue: 134158 +.. nonce: ewLNLp +.. section: Core and Builtins + +Fix coloring of double braces in f-strings and t-strings in the +:term:`REPL`. + +.. + +.. date: 2025-05-16-20-59-12 +.. gh-issue: 134119 +.. nonce: w8expI +.. section: Core and Builtins + +Fix crash when calling :func:`next` on an exhausted template string +iterator. Patch by Jelle Zijlstra. + +.. + +.. date: 2025-05-16-17-25-52 +.. gh-issue: 134100 +.. nonce: 5-FbLK +.. section: Core and Builtins + +Fix a use-after-free bug that occurs when an imported module isn't in +:data:`sys.modules` after its initial import. Patch by Nico-Posada. + +.. + +.. date: 2025-05-16-09-06-38 +.. gh-issue: 134036 +.. nonce: st2e-B +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message when using invalid :keyword:`raise` +statements. + +.. + +.. date: 2025-05-15-11-38-16 +.. gh-issue: 133999 +.. nonce: uBZ8uS +.. section: Core and Builtins + +Fix :exc:`SyntaxError` regression in :keyword:`except` parsing after +:gh:`123440`. + +.. + +.. date: 2025-05-11-13-40-42 +.. gh-issue: 133886 +.. nonce: ryBAyo +.. section: Core and Builtins + +Fix :func:`sys.remote_exec` for non-ASCII paths in non-UTF-8 locales and +non-UTF-8 paths in UTF-8 locales. + +.. + +.. date: 2025-05-11-09-40-19 +.. gh-issue: 133400 +.. nonce: zkWla8 +.. section: Core and Builtins + +Fixed Ctrl+D (^D) behavior in _pyrepl module to match old pre-3.13 REPL +behavior. + +.. + +.. date: 2025-05-10-17-12-27 +.. gh-issue: 133703 +.. nonce: bVM-re +.. section: Core and Builtins + +Fix hashtable in dict can be bigger than intended in some situations. + +.. + +.. date: 2025-05-09-18-11-21 +.. gh-issue: 133778 +.. nonce: pWEV3t +.. section: Core and Builtins + +Fix bug where assigning to the :attr:`~type.__annotations__` attributes of +classes defined under ``from __future__ import annotations`` had no effect. + +.. + +.. date: 2025-05-08-22-19-10 +.. gh-issue: 133711 +.. nonce: e91wUy +.. section: Core and Builtins + +Implement :pep:`686`: Enable :ref:`Python UTF-8 Mode <utf8-mode>` by +default. Patch by Adam Turner. + +.. + +.. date: 2025-05-08-13-48-02 +.. gh-issue: 132762 +.. nonce: tKbygC +.. section: Core and Builtins + +:meth:`~dict.fromkeys` no longer loops forever when adding a small set of +keys to a large base dict. Patch by Angela Liss. + +.. + +.. date: 2025-05-07-23-26-53 +.. gh-issue: 133541 +.. nonce: bHIC55 +.. section: Core and Builtins + +Inconsistent indentation in user input crashed the new REPL when syntax +highlighting was active. This is now fixed. + +.. + +.. date: 2025-05-06-15-01-41 +.. gh-issue: 133516 +.. nonce: RqWVf2 +.. section: Core and Builtins + +Raise :exc:`ValueError` when constants ``True``, ``False`` or ``None`` are +used as an identifier after NFKC normalization. + +.. + +.. date: 2025-05-03-22-31-53 +.. gh-issue: 131798 +.. nonce: fQ0ato +.. section: Core and Builtins + +Allow the JIT to remove int guards after ``_GET_LEN`` by setting the return +type to int. + +.. + +.. date: 2025-05-03-13-36-01 +.. gh-issue: 131798 +.. nonce: U4_QEJ +.. section: Core and Builtins + +Split ``CALL_ISINSTANCE`` into several uops, allowing the JIT to remove some +of them. + +.. + +.. date: 2025-04-30-14-13-01 +.. gh-issue: 132554 +.. nonce: GqQaUp +.. section: Core and Builtins + +Change iteration to use "virtual iterators" for sequences. Instead of +creating an iterator, a tagged integer representing the next index is pushed +to the stack above the iterable. For non-sequence iterators, ``NULL`` is +pushed. + +.. + +.. date: 2025-04-28-18-59-11 +.. gh-issue: 130821 +.. nonce: B11LU1 +.. section: Core and Builtins + +Enhance wrong type error messages and make them more consistent. Patch by +Semyon Moroz. + +.. + +.. date: 2025-04-26-17-50-01 +.. gh-issue: 131798 +.. nonce: XiOgw5 +.. section: Core and Builtins + +Narrow the return type and constant-evaluate ``CALL_ISINSTANCE`` for a +subset of known values in the JIT. Patch by Tomas Roun + +.. + +.. date: 2025-04-19-17-16-46 +.. gh-issue: 132542 +.. nonce: 7T_TY_ +.. section: Core and Builtins + +Update :attr:`Thread.native_id <threading.Thread.native_id>` after +:manpage:`fork(2)` to ensure accuracy. Patch by Noam Cohen. + +.. + +.. date: 2025-04-19-16-22-47 +.. gh-issue: 132732 +.. nonce: jgqhlF +.. section: Core and Builtins + +Automatically constant evaluate bytecode operations marked as pure in the +JIT optimizer. + +.. + +.. date: 2025-04-16-12-01-13 +.. gh-issue: 127971 +.. nonce: pMDOQ0 +.. section: Core and Builtins + +Fix off-by-one read beyond the end of a string in string search. + +.. + +.. date: 2025-04-10-01-52-42 +.. gh-issue: 132042 +.. nonce: fePwlj +.. section: Core and Builtins + +Improve class creation times by up to 12% by pre-computing type slots just +once. Patch by Sergey Miryanov. + +.. + +.. date: 2025-04-04-16-41-00 +.. gh-issue: 133379 +.. nonce: asdjhjdf +.. section: Core and Builtins + +Correct usage of *arguments* in error messages. + +.. + +.. date: 2025-03-14-13-08-20 +.. gh-issue: 127266 +.. nonce: _tyfBp +.. section: Core and Builtins + +In the free-threaded build, avoid data races caused by updating type slots +or type flags after the type was initially created. For those (typically +rare) cases, use the stop-the-world mechanism. Remove the use of atomics +when reading or writing type flags. The use of atomics is not sufficient to +avoid races (since flags are sometimes read without a lock and without +atomics) and are no longer required. + +.. + +.. date: 2025-02-22-01-23-23 +.. gh-issue: 130425 +.. nonce: x5SNQ8 +.. section: Core and Builtins + +Add ``"Did you mean: 'attr'?"`` suggestion when using ``del obj.attr`` if +``attr`` does not exist. + +.. + +.. date: 2025-01-08-12-52-47 +.. gh-issue: 128640 +.. nonce: 9nbh9z +.. section: Core and Builtins + +Fix a crash when using threads inside of a subinterpreter. + +.. + +.. date: 2024-06-04-20-26-21 +.. gh-issue: 116738 +.. nonce: q_hPYq +.. section: Core and Builtins + +Make the module :mod:`json` safe to use under the free-threading build. + +.. + +.. date: 2024-05-24-07-02-47 +.. gh-issue: 119494 +.. nonce: x3KUMC +.. section: Core and Builtins + +Exception text when trying to delete attributes of types was clarified. + +.. + +.. date: 2025-10-10-20-59-07 +.. gh-issue: 139924 +.. nonce: ALByCb +.. section: C API + +Function watchers can now receive a PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME +event when a watched functions qualname is changed. + +.. + +.. date: 2025-10-07-12-51-32 +.. gh-issue: 111489 +.. nonce: LCKKlg +.. section: C API + +Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. +Patch by Victor Stinner. + +.. + +.. date: 2025-09-14-14-44-24 +.. gh-issue: 136355 +.. nonce: LCaYyC +.. section: C API + +Deprecate :c:member:`PyConfig.bytes_warning` field and schedule its removal +in 3.17. + +.. + +.. date: 2025-09-14-13-09-47 +.. gh-issue: 138886 +.. nonce: dlcTXL +.. section: C API + +Remove deprecated :c:func:`!PySys_ResetWarnOptions` C-API function. + +.. + +.. date: 2025-09-12-13-05-20 +.. gh-issue: 129813 +.. nonce: dJZpME +.. section: C API + +Implement :pep:`782`, the :c:type:`PyBytesWriter` API. Add functions: + +* :c:func:`PyBytesWriter_Create` +* :c:func:`PyBytesWriter_Discard` +* :c:func:`PyBytesWriter_FinishWithPointer` +* :c:func:`PyBytesWriter_FinishWithSize` +* :c:func:`PyBytesWriter_Finish` +* :c:func:`PyBytesWriter_Format` +* :c:func:`PyBytesWriter_GetData` +* :c:func:`PyBytesWriter_GetSize` +* :c:func:`PyBytesWriter_GrowAndUpdatePointer` +* :c:func:`PyBytesWriter_Grow` +* :c:func:`PyBytesWriter_Resize` +* :c:func:`PyBytesWriter_WriteBytes` + +Patch by Victor Stinner. + +.. + +.. date: 2025-08-19-15-31-36 +.. gh-issue: 137956 +.. nonce: P4TK1d +.. section: C API + +Display and raise an exception if an extension compiled for +non-free-threaded Python is loaded in a free-threaded interpreter. + +.. + +.. date: 2025-08-13-13-41-04 +.. gh-issue: 137573 +.. nonce: r6uwRf +.. section: C API + +Mark ``_PyOptimizer_Optimize`` as :c:macro:`Py_NO_INLINE` to prevent stack +overflow crashes on macOS. + +.. + +.. date: 2025-07-31-04-30-42 +.. gh-issue: 128813 +.. nonce: opL-Pv +.. section: C API + +Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, +:c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and +previously undocumented :c:func:`_Py_c_abs` are :term:`soft deprecated`. +Deprecate also :c:member:`~PyComplexObject.cval` field of the +:c:type:`PyComplexObject` type. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-07-29-18-00-22 +.. gh-issue: 137210 +.. nonce: DD4VEm +.. section: C API + +Add API for checking an extension module's ABI compatibility: +:c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, :c:macro:`PyABIInfo_VAR` +and :c:data:`Py_mod_abi`. + +.. + +.. date: 2025-07-23-22-30-23 +.. gh-issue: 136759 +.. nonce: ffB4wO +.. section: C API + +Rename ``lock.h`` to ``pylock.h`` to avoid potential include conflicts. + +.. + +.. date: 2025-07-22-15-18-08 +.. gh-issue: 112068 +.. nonce: 4WvT-8 +.. section: C API + +Revert support of nullable arguments in :c:func:`PyArg_Parse`. + +.. + +.. date: 2025-07-08-22-07-54 +.. gh-issue: 136006 +.. nonce: XRU5w4 +.. section: C API + +On Solaris, the :c:macro:`!Py_NAN` macro now expands to a :c:type:`!double` +instead of a function address. Patch by Bénédikt Tran. + +.. + +.. date: 2025-07-01-16-22-39 +.. gh-issue: 135075 +.. nonce: angu3J +.. section: C API + +Make :c:func:`PyObject_SetAttr` and :c:func:`PyObject_SetAttrString` fail if +called with ``NULL`` value and an exception set. Patch by Victor Stinner. + +.. + +.. date: 2025-06-25-01-03-10 +.. gh-issue: 135906 +.. nonce: UBrCWq +.. section: C API + +Fix compilation errors when compiling the internal headers with a C++ +compiler. + +.. + +.. date: 2025-06-24-11-10-01 +.. gh-issue: 133296 +.. nonce: lIEuVJ +.. section: C API + +New variants for the critical section API that accept one or two +:c:type:`PyMutex` pointers rather than :c:type:`PyObject` instances are now +public in the non-limited C API. + +.. + +.. date: 2025-06-19-12-47-18 +.. gh-issue: 133157 +.. nonce: 1WA85f +.. section: C API + +Remove the private, undocumented macro +:c:macro:`!_Py_NO_SANITIZE_UNDEFINED`. + +.. + +.. date: 2025-06-05-11-06-07 +.. gh-issue: 134989 +.. nonce: 74p4ud +.. section: C API + +Fix ``Py_RETURN_NONE``, ``Py_RETURN_TRUE`` and ``Py_RETURN_FALSE`` macros in +the limited C API 3.11 and older: don't treat ``Py_None``, ``Py_True`` and +``Py_False`` as immortal. Patch by Victor Stinner. + +.. + +.. date: 2025-06-02-13-19-22 +.. gh-issue: 134989 +.. nonce: sDDyBN +.. section: C API + +Implement :c:func:`PyObject_DelAttr` and :c:func:`PyObject_DelAttrString` as +macros in the limited C API 3.12 and older. Patch by Victor Stinner. + +.. + +.. date: 2025-05-30-11-33-17 +.. gh-issue: 134745 +.. nonce: GN-zk2 +.. section: C API + +Change :c:func:`!PyThread_allocate_lock` implementation to ``PyMutex``. On +Windows, :c:func:`!PyThread_acquire_lock_timed` now supports the *intr_flag* +parameter: it can be interrupted. Patch by Victor Stinner. + +.. + +.. date: 2025-05-29-16-56-23 +.. gh-issue: 134891 +.. nonce: 7eKO8U +.. section: C API + +Add :c:type:`PyUnstable_Unicode_GET_CACHED_HASH` to get the cached hash of a +string. + +.. + +.. date: 2025-05-20-17-13-51 +.. gh-issue: 134009 +.. nonce: CpCmry +.. section: C API + +Expose :c:func:`PyMutex_IsLocked` as part of the public C API. + +.. + +.. date: 2025-05-17-14-41-21 +.. gh-issue: 134144 +.. nonce: xVpZik +.. section: C API + +Fix crash when calling :c:func:`Py_EndInterpreter` with a :term:`thread +state` that isn't the initial thread for the interpreter. + +.. + +.. date: 2025-05-13-16-06-46 +.. gh-issue: 133968 +.. nonce: 6alWst +.. section: C API + +Add :c:func:`PyUnicodeWriter_WriteASCII` function to write an ASCII string +into a :c:type:`PyUnicodeWriter`. The function is faster than +:c:func:`PyUnicodeWriter_WriteUTF8`, but has an undefined behavior if the +input string contains non-ASCII characters. Patch by Victor Stinner. + +.. + +.. date: 2025-05-08-13-14-45 +.. gh-issue: 133644 +.. nonce: J8_KZ2 +.. section: C API + +Remove deprecated Python initialization getter functions ``Py_Get*``. Patch +by Bénédikt Tran. + +.. + +.. date: 2025-05-08-12-40-59 +.. gh-issue: 133644 +.. nonce: FNexLJ +.. section: C API + +Remove deprecated function :c:func:`!PyWeakref_GetObject` and macro +:c:macro:`!PyWeakref_GET_OBJECT`. Use :c:func:`PyWeakref_GetRef` instead. +Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-08-12-25-47 +.. gh-issue: 133644 +.. nonce: Yb86Rm +.. section: C API + +Remove deprecated alias :c:func:`!PyImport_ImportModuleNoBlock` of +:c:func:`PyImport_ImportModule`. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-07-21-18-00 +.. gh-issue: 133610 +.. nonce: asdfjs +.. section: C API + +Remove deprecated functions :c:func:`!PyUnicode_AsDecodedObject`, +:c:func:`!PyUnicode_AsDecodedUnicode`, :c:func:`!PyUnicode_AsEncodedObject`, +and :c:func:`!PyUnicode_AsEncodedUnicode`. + +.. + +.. date: 2025-04-17-12-37-27 +.. gh-issue: 132629 +.. nonce: 01ArwX +.. section: C API + +For unsigned integer formats in :c:func:`PyArg_ParseTuple`, accepting Python +integers with value that is larger than the maximal value for the C type or +less than the minimal value for the corresponding signed integer type of the +same size is now deprecated. + +.. + +.. date: 2025-04-14-07-41-28 +.. gh-issue: 131185 +.. nonce: ZCjMHD +.. section: C API + +:c:func:`PyGILState_Ensure` no longer crashes when called after interpreter +finalization. + +.. + +.. date: 2023-10-18-14-36-35 +.. gh-issue: 108512 +.. nonce: fMZLfr +.. section: C API + +Add functions :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, +:c:func:`PySys_GetOptionalAttr` and :c:func:`PySys_GetOptionalAttrString`. + +.. + +.. date: 2025-09-24-13-59-26 +.. gh-issue: 138489 +.. nonce: 1AcuZM +.. section: Build + +When cross-compiling for WASI by ``build_wasm`` or ``build_emscripten``, the +``build-details.json`` step is now included in the build process, just like +with native builds. + +This fixes the ``libinstall`` task which requires the ``build-details.json`` +file during the process. + +.. + +.. date: 2025-09-04-12-16-31 +.. gh-issue: 138497 +.. nonce: Y_5YXh +.. section: Build + +The LLVM version used by the JIT at build time can now be modified using the +``LLVM_VERSION`` environment variable. Use this at your own risk, as there +is only one officially supported LLVM version. For more information, please +check ``Tools/jit/README.md``. + +.. + +.. date: 2025-08-27-11-32-02 +.. gh-issue: 95952 +.. nonce: KSymc7 +.. section: Build + +When cross-compiling for WASI, require that the HOSTRUNNER environment +variable be explicitly set. + +This was needed as macOS lacks the appropriate CLI tools to set a reasonable +default. + +.. + +.. date: 2025-08-27-09-52-45 +.. gh-issue: 138061 +.. nonce: fMVS9w +.. section: Build + +Ensure reproducible builds by making JIT stencil header generation +deterministic. + +.. + +.. date: 2025-08-26-21-18-32 +.. gh-issue: 128042 +.. nonce: 5voC8H +.. section: Build + +``./configure`` now warns when ``--enable-optimizations`` and ``CFLAGS=-O0`` +are both set, suggesting removing ``-O0`` from ``CFLAGS`` for optimal +performance. Patch by Taegyun Kim. + +.. + +.. date: 2025-08-13-12-10-12 +.. gh-issue: 132339 +.. nonce: 3Czz5y +.. section: Build + +Add support for OpenSSL 3.5. + +.. + +.. date: 2025-07-18-17-15-00 +.. gh-issue: 135621 +.. nonce: 9cyCNb +.. section: Build + +PyREPL no longer depends on the :mod:`curses` standard library. Contributed +by Łukasz Langa. + +.. + +.. date: 2025-06-25-13-27-14 +.. gh-issue: 135927 +.. nonce: iCNPQc +.. section: Build + +Fix building with MSVC when passing option ``/std:clatest``. + +.. + +.. date: 2025-06-16-07-20-28 +.. gh-issue: 119132 +.. nonce: fcI8s7 +.. section: Build + +Remove "experimental" tag from the CPython free-threading build. + +.. + +.. date: 2025-06-14-10-32-11 +.. gh-issue: 135497 +.. nonce: ajlV4F +.. section: Build + +Fix the detection of ``MAXLOGNAME`` in the ``configure.ac`` script. + +.. + +.. date: 2025-05-30-11-02-30 +.. gh-issue: 134923 +.. nonce: gBkRg4 +.. section: Build + +Windows builds with profile-guided optimization enabled now use +``/GENPROFILE`` and ``/USEPROFILE`` instead of deprecated ``/LTCG:`` +options. + +.. + +.. date: 2025-05-24-16-59-20 +.. gh-issue: 134632 +.. nonce: i0W2hc +.. section: Build + +Fixed ``build-details.json`` generation to use ``INCLUDEPY``, in order to +reference the ``pythonX.Y`` subdirectory of the include directory, as +required in :pep:`739`, instead of the top-level include directory. + +.. + +.. date: 2025-05-21-22-13-30 +.. gh-issue: 134486 +.. nonce: yvdL6f +.. section: Build + +The :mod:`ctypes` module now performs a more portable test for the +definition of :manpage:`alloca(3)`, fixing a compilation failure on NetBSD. + +.. + +.. date: 2025-05-21-19-46-28 +.. gh-issue: 134455 +.. nonce: vdwlrq +.. section: Build + +Fixed ``build-details.json`` generation to use the correct ``c_api.headers`` +as defined in :pep:`739`, instead of ``c_api.include``. + +.. + +.. date: 2025-05-19-18-09-20 +.. gh-issue: 134273 +.. nonce: ZAliyy +.. section: Build + +Add support for configuring compiler flags for the JIT with ``CFLAGS_JIT`` + +.. + +.. date: 2025-05-16-07-46-06 +.. gh-issue: 115119 +.. nonce: ALBgS_ +.. section: Build + +Removed implicit fallback to the bundled copy of the ``libmpdec`` library. +Now this should be explicitly enabled via :option:`--with-system-libmpdec` +set to ``no`` or :option:`!--without-system-libmpdec`. Patch by Sergey B +Kirpichev. + +.. + +.. date: 2025-05-14-09-43-48 +.. gh-issue: 131769 +.. nonce: H0oy5x +.. section: Build + +Fix detecting when the build Python in a cross-build is a pydebug build. + +.. + +.. date: 2025-04-16-09-38-48 +.. gh-issue: 117088 +.. nonce: EFt_5c +.. section: Build + +AIX linker don't support -h option, so avoid it through platform check + +.. + +.. date: 2025-01-03-13-02-06 +.. gh-issue: 123681 +.. nonce: gQ67nK +.. section: Build + +Check the ``strftime()`` behavior at runtime instead of at the compile time +to support cross-compiling. Remove the internal macro +``_Py_NORMALIZE_CENTURY``. + +.. + +.. date: 2024-12-04-10-00-35 +.. gh-issue: 127545 +.. nonce: t0THjE +.. section: Build + +Fix crash when building on Linux/m68k. diff --git a/Misc/NEWS.d/3.15.0a2.rst b/Misc/NEWS.d/3.15.0a2.rst new file mode 100644 index 00000000000..4e3a62b0f4a --- /dev/null +++ b/Misc/NEWS.d/3.15.0a2.rst @@ -0,0 +1,1746 @@ +.. date: 2025-11-04-19-20-05 +.. gh-issue: 140849 +.. nonce: YjB2ZZ +.. release date: 2025-11-18 +.. section: Windows + +Update bundled liblzma to version 5.8.1. + +.. + +.. date: 2025-11-12-12-54-28 +.. gh-issue: 141442 +.. nonce: 50dS3P +.. section: Tools/Demos + +The iOS testbed now correctly handles test arguments that contain spaces. + +.. + +.. date: 2025-10-29-15-20-19 +.. gh-issue: 140702 +.. nonce: ZXtW8h +.. section: Tools/Demos + +The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment +variable to iOS apps being tested. + +.. + +.. date: 2025-09-21-10-30-08 +.. gh-issue: 139198 +.. nonce: Fm7NfU +.. section: Tools/Demos + +Remove ``Tools/scripts/checkpip.py`` script. + +.. + +.. date: 2025-09-20-20-31-54 +.. gh-issue: 139188 +.. nonce: zfcxkW +.. section: Tools/Demos + +Remove ``Tools/tz/zdump.py`` script. + +.. + +.. date: 2025-10-23-16-39-49 +.. gh-issue: 140482 +.. nonce: ZMtyeD +.. section: Tests + +Preserve and restore the state of ``stty echo`` as part of the test +environment. + +.. + +.. date: 2025-10-15-00-52-12 +.. gh-issue: 140082 +.. nonce: fpET50 +.. section: Tests + +Update ``python -m test`` to set ``FORCE_COLOR=1`` when being run with color +enabled so that :mod:`unittest` which is run by it with redirected output +will output in color. + +.. + +.. date: 2025-07-09-21-45-51 +.. gh-issue: 136442 +.. nonce: jlbklP +.. section: Tests + +Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` +raises an exception + +.. + +.. date: 2025-08-15-23-08-44 +.. gh-issue: 137836 +.. nonce: b55rhh +.. section: Security + +Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe", +"noembed" and "noframes", and optionally RAWTEXT element "noscript" in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-06-28-13-23-53 +.. gh-issue: 136063 +.. nonce: aGk0Jv +.. section: Security + +:mod:`email.message`: ensure linear complexity for legacy HTTP parameters +parsing. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-30-22-33-27 +.. gh-issue: 136065 +.. nonce: bu337o +.. section: Security + +Fix quadratic complexity in :func:`os.path.expandvars`. + +.. + +.. date: 2025-11-14-16-24-20 +.. gh-issue: 141497 +.. nonce: L_CxDJ +.. section: Library + +:mod:`ipaddress`: ensure that the methods :meth:`IPv4Network.hosts() +<ipaddress.IPv4Network.hosts>` and :meth:`IPv6Network.hosts() +<ipaddress.IPv6Network.hosts>` always return an iterator. + +.. + +.. date: 2025-11-13-14-51-30 +.. gh-issue: 140938 +.. nonce: kXsHHv +.. section: Library + +The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now +raise a :exc:`ValueError` when the input contains an infinity or a NaN. + +.. + +.. date: 2025-11-12-15-42-47 +.. gh-issue: 124111 +.. nonce: hTw4OE +.. section: Library + +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. + +.. + +.. date: 2025-11-12-01-49-03 +.. gh-issue: 137109 +.. nonce: D6sq2B +.. section: Library + +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. + +.. + +.. date: 2025-11-10-01-47-18 +.. gh-issue: 141314 +.. nonce: baaa28 +.. section: Library + +Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files +with standalone carriage return (``\r``) line endings. + +.. + +.. date: 2025-11-09-18-55-13 +.. gh-issue: 141311 +.. nonce: qZ3swc +.. section: Library + +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. + +.. + +.. date: 2025-11-08-13-03-10 +.. gh-issue: 87710 +.. nonce: XJeZlP +.. section: Library + +:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. + +.. + +.. date: 2025-11-07-12-25-46 +.. gh-issue: 85524 +.. nonce: 9SWFIC +.. section: Library + +Update ``io.FileIO.readall``, an implementation of +:meth:`io.RawIOBase.readall`, to follow :class:`io.IOBase` guidelines and +raise :exc:`io.UnsupportedOperation` when a file is in "w" mode rather than +:exc:`OSError` + +.. + +.. date: 2025-11-06-15-11-50 +.. gh-issue: 141141 +.. nonce: tgIfgH +.. section: Library + +Fix a thread safety issue with :func:`base64.b85decode`. Contributed by +Benel Tayar. + +.. + +.. date: 2025-11-04-20-08-41 +.. gh-issue: 141018 +.. nonce: d_oyOI +.. section: Library + +:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when +``strict=False``) ``.jpg`` to their correct IANA mime type. + +.. + +.. date: 2025-11-04-15-40-35 +.. gh-issue: 137969 +.. nonce: 9VZQVt +.. section: Library + +Fix :meth:`annotationlib.ForwardRef.evaluate` returning +:class:`~annotationlib.ForwardRef` objects which don't update with new +globals. + +.. + +.. date: 2025-11-04-12-16-13 +.. gh-issue: 75593 +.. nonce: EFVhKR +.. section: Library + +Add support of :term:`path-like objects <path-like object>` and +:term:`bytes-like objects <bytes-like object>` in :func:`wave.open`. + +.. + +.. date: 2025-11-03-16-23-54 +.. gh-issue: 140797 +.. nonce: DuFEeR +.. section: Library + +The undocumented :class:`!re.Scanner` class now forbids regular expressions +containing capturing groups in its lexicon patterns. Patterns using +capturing groups could previously lead to crashes with segmentation fault. +Use non-capturing groups (?:...) instead. + +.. + +.. date: 2025-11-03-05-38-31 +.. gh-issue: 125115 +.. nonce: jGS8MN +.. section: Library + +Refactor the :mod:`pdb` parsing issue so positional arguments can pass +through intuitively. + +.. + +.. date: 2025-11-02-19-23-32 +.. gh-issue: 140815 +.. nonce: McEG-T +.. section: Library + +:mod:`faulthandler` now detects if a frame or a code object is invalid or +freed. Patch by Victor Stinner. + +.. + +.. date: 2025-11-02-11-46-00 +.. gh-issue: 100218 +.. nonce: 9Ezfdq +.. section: Library + +Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or +:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-11-02-09-37-22 +.. gh-issue: 140734 +.. nonce: f8gST9 +.. section: Library + +:mod:`multiprocessing`: fix off-by-one error when checking the length of a +temporary socket file path. Patch by Bénédikt Tran. + +.. + +.. date: 2025-11-01-14-44-09 +.. gh-issue: 140873 +.. nonce: kfuc9B +.. section: Library + +Add support of non-:term:`descriptor` callables in +:func:`functools.singledispatchmethod`. + +.. + +.. date: 2025-11-01-00-36-14 +.. gh-issue: 140874 +.. nonce: eAWt3K +.. section: Library + +Bump the version of pip bundled in ensurepip to version 25.3 + +.. + +.. date: 2025-11-01-00-34-53 +.. gh-issue: 140826 +.. nonce: JEDd7U +.. section: Library + +Now :class:`!winreg.HKEYType` objects are compared by their underlying +Windows registry handle value instead of their object identity. + +.. + +.. date: 2025-10-31-16-25-13 +.. gh-issue: 140808 +.. nonce: XBiQ4j +.. section: Library + +The internal class ``mailbox._ProxyFile`` is no longer a parameterized +generic. + +.. + +.. date: 2025-10-31-15-06-26 +.. gh-issue: 140691 +.. nonce: JzHGtg +.. section: Library + +In :mod:`urllib.request`, when opening a FTP URL fails because a data +connection cannot be made, the control connection's socket is now closed to +avoid a :exc:`ResourceWarning`. + +.. + +.. date: 2025-10-31-13-57-55 +.. gh-issue: 103847 +.. nonce: VM7TnW +.. section: Library + +Fix hang when cancelling process created by +:func:`asyncio.create_subprocess_exec` or +:func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-30-15-33-07 +.. gh-issue: 137821 +.. nonce: 8_Iavt +.. section: Library + +Convert ``_json`` module to use Argument Clinic. Patched by Yoonho Hann. + +.. + +.. date: 2025-10-30-12-36-19 +.. gh-issue: 140790 +.. nonce: _3T6-N +.. section: Library + +Initialize all Pdb's instance variables in ``__init__``, remove some +hasattr/getattr + +.. + +.. date: 2025-10-29-16-53-00 +.. gh-issue: 140766 +.. nonce: CNagKF +.. section: Library + +Add :func:`enum.show_flag_values` and ``enum.bin`` to ``enum.__all__``. + +.. + +.. date: 2025-10-29-16-12-41 +.. gh-issue: 120057 +.. nonce: qGj5Dl +.. section: Library + +Add :func:`os.reload_environ` to ``os.__all__``. + +.. + +.. date: 2025-10-29-09-40-10 +.. gh-issue: 140741 +.. nonce: L13UCV +.. section: Library + +Fix ``profiling.sampling.sample()`` incorrectly handling a +:exc:`FileNotFoundError` or :exc:`PermissionError`. + +.. + +.. date: 2025-10-28-17-43-51 +.. gh-issue: 140228 +.. nonce: 8kfHhO +.. section: Library + +Avoid making unnecessary filesystem calls for frozen modules in +:mod:`linecache` when the global module cache is not present. + +.. + +.. date: 2025-10-28-02-46-56 +.. gh-issue: 139946 +.. nonce: aN3_uY +.. section: Library + +Error and warning keywords in ``argparse.ArgumentParser`` messages are now +colorized when color output is enabled, fixing a visual inconsistency in +which they remained plain text while other output was colorized. + +.. + +.. date: 2025-10-27-18-29-42 +.. gh-issue: 140590 +.. nonce: LT9HHn +.. section: Library + +Fix arguments checking for the :meth:`!functools.partial.__setstate__` that +may lead to internal state corruption and crash. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-27-16-01-41 +.. gh-issue: 125434 +.. nonce: qy0uRA +.. section: Library + +Display thread name in :mod:`faulthandler` on Windows. Patch by Victor +Stinner. + +.. + +.. date: 2025-10-27-13-49-31 +.. gh-issue: 140634 +.. nonce: ULng9G +.. section: Library + +Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. + +.. + +.. date: 2025-10-27-00-40-49 +.. gh-issue: 140650 +.. nonce: DYJPJ9 +.. section: Library + +Fix an issue where closing :class:`io.BufferedWriter` could crash if the +closed attribute raised an exception on access or could not be converted to +a boolean. + +.. + +.. date: 2025-10-26-16-24-12 +.. gh-issue: 140633 +.. nonce: ioayC1 +.. section: Library + +Ignore :exc:`AttributeError` when setting a module's ``__file__`` attribute +when loading an extension module packaged as Apple Framework. + +.. + +.. date: 2025-10-25-22-55-07 +.. gh-issue: 140601 +.. nonce: In3MlS +.. section: Library + +:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` +when the iterator is not explicitly closed and was opened with a filename. +This helps developers identify and fix resource leaks. Patch by Osama +Abdelkader. + +.. + +.. date: 2025-10-25-21-26-16 +.. gh-issue: 140593 +.. nonce: OxlLc9 +.. section: Library + +:mod:`xml.parsers.expat`: Fix a memory leak that could affect users with +:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler` set to a custom +element declaration handler. Patch by Sebastian Pipping. + +.. + +.. date: 2025-10-25-21-04-00 +.. gh-issue: 140607 +.. nonce: oOZGxS +.. section: Library + +Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned +by :meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). + +.. + +.. date: 2025-10-23-19-39-16 +.. gh-issue: 138162 +.. nonce: Znw5DN +.. section: Library + +Fix :class:`logging.LoggerAdapter` with ``merge_extra=True`` and without the +*extra* argument. + +.. + +.. date: 2025-10-23-13-42-15 +.. gh-issue: 140481 +.. nonce: XKxWpq +.. section: Library + +Improve error message when trying to iterate a Tk widget, image or font. + +.. + +.. date: 2025-10-23-12-12-22 +.. gh-issue: 138774 +.. nonce: mnh2gU +.. section: Library + +:func:`ast.unparse` now generates full source code when handling +:class:`ast.Interpolation` nodes that do not have a specified source. + +.. + +.. date: 2025-10-22-20-52-13 +.. gh-issue: 140474 +.. nonce: xIWlip +.. section: Library + +Fix memory leak in :class:`array.array` when creating arrays from an empty +:class:`str` and the ``u`` type code. + +.. + +.. date: 2025-10-22-12-56-57 +.. gh-issue: 140448 +.. nonce: GsEkXD +.. section: Library + +Change the default of ``suggest_on_error`` to ``True`` in +``argparse.ArgumentParser``. + +.. + +.. date: 2025-10-21-15-54-13 +.. gh-issue: 137530 +.. nonce: ZyIVUH +.. section: Library + +:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by +replacing the annotations that were in-line in the generated source code +with ``__annotate__`` functions attached to the methods. + +.. + +.. date: 2025-10-20-12-33-49 +.. gh-issue: 140348 +.. nonce: SAKnQZ +.. section: Library + +Fix regression in Python 3.14.0 where using the ``|`` operator on a +:class:`typing.Union` object combined with an object that is not a type +would raise an error. + +.. + +.. date: 2025-10-18-15-20-25 +.. gh-issue: 76007 +.. nonce: SNUzRq +.. section: Library + +:mod:`decimal`: Deprecate ``__version__`` and replace with +:data:`decimal.SPEC_VERSION`. + +.. + +.. date: 2025-10-18-14-30-21 +.. gh-issue: 76007 +.. nonce: peEgcr +.. section: Library + +Deprecate ``__version__`` from a :mod:`imaplib`. Patch by Hugo van Kemenade. + +.. + +.. date: 2025-10-17-23-58-11 +.. gh-issue: 140272 +.. nonce: lhY8uS +.. section: Library + +Fix memory leak in the :meth:`!clear` method of the :mod:`dbm.gnu` database. + +.. + +.. date: 2025-10-17-20-42-38 +.. gh-issue: 129117 +.. nonce: X9jr4p +.. section: Library + +:mod:`unicodedata`: Add :func:`~unicodedata.isxidstart` and +:func:`~unicodedata.isxidcontinue` functions to check whether a character +can start or continue a `Unicode Standard Annex #31 +<https://www.unicode.org/reports/tr31/>`_ identifier. + +.. + +.. date: 2025-10-17-12-33-01 +.. gh-issue: 140251 +.. nonce: esM-OX +.. section: Library + +Colorize the default import statement ``import asyncio`` in asyncio REPL. + +.. + +.. date: 2025-10-16-22-49-16 +.. gh-issue: 140212 +.. nonce: llBNd0 +.. section: Library + +Calendar's HTML formatting now accepts year and month as options. +Previously, running ``python -m calendar -t html 2025 10`` would result in +an error message. It now generates an HTML document displaying the calendar +for the specified month. Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-16-17-17-20 +.. gh-issue: 135801 +.. nonce: faH3fa +.. section: Library + +Improve filtering by module in :func:`warnings.warn_explicit` if no *module* +argument is passed. It now tests the module regular expression in the +warnings filter not only against the filename with ``.py`` stripped, but +also against module names constructed starting from different parent +directories of the filename (with ``/__init__.py``, ``.py`` and, on Windows, +``.pyw`` stripped). + +.. + +.. date: 2025-10-16-16-10-11 +.. gh-issue: 139707 +.. nonce: zR6Qtn +.. section: Library + +Improve :exc:`ModuleNotFoundError` error message when a :term:`standard +library` module is missing. + +.. + +.. date: 2025-10-15-21-42-13 +.. gh-issue: 140041 +.. nonce: _Fka2j +.. section: Library + +Fix import of :mod:`ctypes` on Android and Cygwin when ABI flags are +present. + +.. + +.. date: 2025-10-15-20-47-04 +.. gh-issue: 140120 +.. nonce: 3gffZq +.. section: Library + +Fixed a memory leak in :mod:`hmac` when it was using the hacl-star backend. +Discovered by ``@ashm-dev`` using AddressSanitizer. + +.. + +.. date: 2025-10-15-17-23-51 +.. gh-issue: 140141 +.. nonce: j2mUDB +.. section: Library + +The :py:class:`importlib.metadata.PackageNotFoundError` traceback raised +when ``importlib.metadata.Distribution.from_name`` cannot discover a +distribution no longer includes a transient :exc:`StopIteration` exception +trace. + +Contributed by Bartosz Sławecki in :gh:`140142`. + +.. + +.. date: 2025-10-15-15-10-34 +.. gh-issue: 140166 +.. nonce: NtxRez +.. section: Library + +:mod:`mimetypes`: Per the `IANA assignment +<https://www.iana.org/assignments/media-types/application/texinfo>`_, update +the MIME type for the ``.texi`` and ``.texinfo`` file formats to +``application/texinfo``, instead of ``application/x-texinfo``. + +.. + +.. date: 2025-10-15-02-26-50 +.. gh-issue: 140135 +.. nonce: 54JYfM +.. section: Library + +Speed up :meth:`io.RawIOBase.readall` by using PyBytesWriter API (about 4x +faster) + +.. + +.. date: 2025-10-14-20-27-06 +.. gh-issue: 76007 +.. nonce: 2NcUbo +.. section: Library + +:mod:`zlib`: Deprecate ``__version__`` and schedule for removal in Python +3.20. + +.. + +.. date: 2025-10-13-11-25-41 +.. gh-issue: 136702 +.. nonce: uvLGK1 +.. section: Library + +:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to +:func:`encodings.normalize_encoding` and schedule removal of support for +Python 3.17. + +.. + +.. date: 2025-10-11-09-07-06 +.. gh-issue: 139940 +.. nonce: g54efZ +.. section: Library + +Print clearer error message when using ``pdb`` to attach to a non-existing +process. + +.. + +.. date: 2025-10-02-22-29-00 +.. gh-issue: 139462 +.. nonce: VZXUHe +.. section: Library + +When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` +terminates abruptly, the resulting traceback will now tell you the PID and +exit code of the terminated process. Contributed by Jonathan Berg. + +.. + +.. date: 2025-09-30-12-52-54 +.. gh-issue: 63161 +.. nonce: mECM1A +.. section: Library + +Fix :func:`tokenize.detect_encoding`. Support non-UTF-8 shebang and comments +if non-UTF-8 encoding is specified. Detect decoding error for non-UTF-8 +encoding. Detect null bytes in source code. + +.. + +.. date: 2025-09-25-20-16-10 +.. gh-issue: 101828 +.. nonce: yTxJlJ +.. section: Library + +Fix ``'shift_jisx0213'``, ``'shift_jis_2004'``, ``'euc_jisx0213'`` and +``'euc_jis_2004'`` codecs truncating null chars as they were treated as part +of multi-character sequences. + +.. + +.. date: 2025-09-23-09-46-46 +.. gh-issue: 139246 +.. nonce: pzfM-w +.. section: Library + +fix: paste zero-width in default repl width is wrong. + +.. + +.. date: 2025-09-18-21-25-41 +.. gh-issue: 83714 +.. nonce: TQjDWZ +.. section: Library + +Implement :func:`os.statx` on Linux kernel versions 4.11 and later with +glibc versions 2.28 and later. Contributed by Jeffrey Bosboom and Victor +Stinner. + +.. + +.. date: 2025-09-15-21-03-11 +.. gh-issue: 138891 +.. nonce: oZFdtR +.. section: Library + +Fix ``SyntaxError`` when ``inspect.get_annotations(f, eval_str=True)`` is +called on a function annotated with a :pep:`646` ``star_expression`` + +.. + +.. date: 2025-09-13-12-19-17 +.. gh-issue: 138859 +.. nonce: PxjIoN +.. section: Library + +Fix generic type parameterization raising a :exc:`TypeError` when omitting a +:class:`ParamSpec` that has a default which is not a list of types. + +.. + +.. date: 2025-09-12-09-34-37 +.. gh-issue: 138764 +.. nonce: mokHoY +.. section: Library + +Prevent :func:`annotationlib.call_annotate_function` from calling +``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in +a fake globals namespace with empty globals. + +Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` +annotations in the case that neither their own format, nor +``VALUE_WITH_FAKE_GLOBALS`` are supported. + +.. + +.. date: 2025-09-11-15-03-37 +.. gh-issue: 138775 +.. nonce: w7rnSx +.. section: Library + +Use of ``python -m`` with :mod:`base64` has been fixed to detect input from +a terminal so that it properly notices EOF. + +.. + +.. date: 2025-09-03-20-18-39 +.. gh-issue: 98896 +.. nonce: tjez89 +.. section: Library + +Fix a failure in multiprocessing resource_tracker when SharedMemory names +contain colons. Patch by Rani Pinchuk. + +.. + +.. date: 2025-09-03-18-26-07 +.. gh-issue: 138425 +.. nonce: cVE9Ho +.. section: Library + +Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which +rely on names defined as globals. + +.. + +.. date: 2025-08-26-08-17-56 +.. gh-issue: 138151 +.. nonce: I6CdAk +.. section: Library + +In :mod:`annotationlib`, improve evaluation of forward references to +nonlocal variables that are not yet defined when the annotations are +initially evaluated. + +.. + +.. date: 2025-08-15-20-35-30 +.. gh-issue: 69528 +.. nonce: qc-Eh_ +.. section: Library + +The :attr:`~io.FileIO.mode` attribute of files opened in the ``'wb+'`` mode +is now ``'wb+'`` instead of ``'rb+'``. + +.. + +.. date: 2025-08-11-04-52-18 +.. gh-issue: 137627 +.. nonce: Ku5Yi2 +.. section: Library + +Speed up :meth:`csv.Sniffer.sniff` delimiter detection by up to 1.6x. + +.. + +.. date: 2025-07-14-09-33-17 +.. gh-issue: 55531 +.. nonce: Gt2e12 +.. section: Library + +:mod:`encodings`: Improve :func:`~encodings.normalize_encoding` performance +by implementing the function in C using the private +``_Py_normalize_encoding`` which has been modified to make lowercase +conversion optional. + +.. + +.. date: 2025-07-01-04-57-57 +.. gh-issue: 136057 +.. nonce: 4-t596 +.. section: Library + +Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't +go over the line if a loop exists in the line. + +.. + +.. date: 2025-06-29-22-01-00 +.. gh-issue: 133390 +.. nonce: I1DW_3 +.. section: Library + +Support table, index, trigger, view, column, function, and schema completion +for :mod:`sqlite3`'s :ref:`command-line interface <sqlite3-cli>`. + +.. + +.. date: 2025-06-10-18-02-29 +.. gh-issue: 135307 +.. nonce: fXGrcK +.. section: Library + +:mod:`email`: Fix exception in ``set_content()`` when encoding text and +max_line_length is set to ``0`` or ``None`` (unlimited). + +.. + +.. date: 2025-05-10-15-10-54 +.. gh-issue: 133789 +.. nonce: I-ZlUX +.. section: Library + +Fix unpickling of :mod:`pathlib` objects that were pickled in Python 3.13. + +.. + +.. date: 2025-05-07-22-09-28 +.. gh-issue: 133601 +.. nonce: 9kUL3P +.. section: Library + +Remove deprecated :func:`!typing.no_type_check_decorator`. + +.. + +.. date: 2025-04-18-18-08-05 +.. gh-issue: 132686 +.. nonce: 6kV_Gs +.. section: Library + +Add parameters *inherit_class_doc* and *fallback_to_class_doc* for +:func:`inspect.getdoc`. + +.. + +.. date: 2025-03-12-18-57-10 +.. gh-issue: 131116 +.. nonce: uTpwXZ +.. section: Library + +:func:`inspect.getdoc` now correctly returns an inherited docstring on +:class:`~functools.cached_property` objects if none is given in a subclass. + +.. + +.. date: 2025-03-04-17-19-26 +.. gh-issue: 130693 +.. nonce: Kv01r8 +.. section: Library + +Add support for ``-nolinestop``, and ``-strictlimits`` options to +:meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` +method for ``-all`` and ``-overlap`` options. + +.. + +.. date: 2024-08-08-12-39-36 +.. gh-issue: 122255 +.. nonce: J_gU8Y +.. section: Library + +In the :mod:`linecache` module and in the Python implementation of the +:mod:`warnings` module, a ``DeprecationWarning`` is issued when +``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C +implementation of the :mod:`!warnings` module). + +.. + +.. date: 2024-06-26-16-16-43 +.. gh-issue: 121011 +.. nonce: qW54eh +.. section: Library + +:func:`math.log` now supports arbitrary large integer-like arguments in the +same way as arbitrary large integer arguments. + +.. + +.. date: 2024-05-28-17-14-30 +.. gh-issue: 119668 +.. nonce: RrIGpn +.. section: Library + +Publicly expose and document :class:`importlib.machinery.NamespacePath`. + +.. + +.. date: 2023-03-21-10-59-40 +.. gh-issue: 102431 +.. nonce: eUDnf4 +.. section: Library + +Clarify constraints for "logical" arguments in methods of +:class:`decimal.Context`. + +.. + +.. date: 2019-06-02-13-56-16 +.. gh-issue: 81313 +.. nonce: axawSH +.. section: Library + +Add the :mod:`math.integer` module (:pep:`791`). + +.. + +.. date: 2025-11-15-01-21-00 +.. gh-issue: 141579 +.. nonce: aB7cD9 +.. section: Core and Builtins + +Fix :func:`sys.activate_stack_trampoline` to properly support the +``perf_jit`` backend. Patch by Pablo Galindo. + +.. + +.. date: 2025-11-14-16-25-15 +.. gh-issue: 114203 +.. nonce: n3tlQO +.. section: Core and Builtins + +Skip locking if object is already locked by two-mutex critical section. + +.. + +.. date: 2025-11-14-00-19-45 +.. gh-issue: 141528 +.. nonce: VWdax1 +.. section: Core and Builtins + +Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of +the private ``_interpreters.destroy`` function when warning about remaining +subinterpreters. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-11-13-40-45 +.. gh-issue: 141367 +.. nonce: I5KY7F +.. section: Core and Builtins + +Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list +subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. + +.. + +.. date: 2025-11-10-23-07-06 +.. gh-issue: 141312 +.. nonce: H-58GB +.. section: Core and Builtins + +Fix the assertion failure in the ``__setstate__`` method of the range +iterator when a non-integer argument is passed. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-05-19-50-37 +.. gh-issue: 140643 +.. nonce: QCEOqG +.. section: Core and Builtins + +Add support for ``<GC>`` and ``<native>`` frames to +:mod:`!profiling.sampling` output to denote active garbage collection and +calls to native code. + +.. + +.. date: 2025-11-04-12-18-06 +.. gh-issue: 140942 +.. nonce: GYns6n +.. section: Core and Builtins + +Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of +``application/node``. + +.. + +.. date: 2025-11-04-04-57-24 +.. gh-issue: 140479 +.. nonce: lwQ2v2 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 21 at build time. + +.. + +.. date: 2025-11-03-17-21-38 +.. gh-issue: 140939 +.. nonce: FVboAw +.. section: Core and Builtins + +Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with +the ``%*b`` format with a large width that results in a :exc:`MemoryError`. + +.. + +.. date: 2025-11-02-15-28-33 +.. gh-issue: 140260 +.. nonce: JNzlGz +.. section: Core and Builtins + +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. + +.. + +.. date: 2025-11-02-12-47-38 +.. gh-issue: 140530 +.. nonce: S934bp +.. section: Core and Builtins + +Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-10-31-14-03-42 +.. gh-issue: 90344 +.. nonce: gvZigO +.. section: Core and Builtins + +Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline +decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was +being called once. + +.. + +.. date: 2025-10-29-20-59-10 +.. gh-issue: 140373 +.. nonce: -uoaPP +.. section: Core and Builtins + +Correctly emit ``PY_UNWIND`` event when generator object is closed. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-29-11-31-59 +.. gh-issue: 140729 +.. nonce: t9JsNt +.. section: Core and Builtins + +Fix pickling error in the sampling profiler when using +``concurrent.futures.ProcessPoolExecutor`` script can not be properly +pickled and executed in worker processes. + +.. + +.. date: 2025-10-25-21-31-43 +.. gh-issue: 131527 +.. nonce: V-JVNP +.. section: Core and Builtins + +Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` +mode. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-25-17-36-46 +.. gh-issue: 140576 +.. nonce: kj0SCY +.. section: Core and Builtins + +Fixed crash in :func:`tokenize.generate_tokens` in case of specific +incorrect input. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-25-07-25-52 +.. gh-issue: 140544 +.. nonce: lwjtQe +.. section: Core and Builtins + +Speed up accessing interpreter state by caching it in a thread local +variable. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-24-20-42-33 +.. gh-issue: 140551 +.. nonce: -9swrl +.. section: Core and Builtins + +Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup +stage. Patch by Mikhail Efimov and Inada Naoki. + +.. + +.. date: 2025-10-24-20-16-42 +.. gh-issue: 140517 +.. nonce: cqun-K +.. section: Core and Builtins + +Fixed a reference leak when iterating over the result of :func:`map` with +``strict=True`` when the input iterables have different lengths. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-24-14-29-12 +.. gh-issue: 133467 +.. nonce: A5d6TM +.. section: Core and Builtins + +Fix race when updating :attr:`!type.__bases__` that could allow a read of +:attr:`!type.__base__` to observe an inconsistent value on the free threaded +build. + +.. + +.. date: 2025-10-23-16-05-50 +.. gh-issue: 140471 +.. nonce: Ax_aXn +.. section: Core and Builtins + +Fix potential buffer overflow in :class:`ast.AST` node initialization when +encountering malformed :attr:`~ast.AST._fields` containing non-:class:`str`. + +.. + +.. date: 2025-10-22-23-26-37 +.. gh-issue: 140443 +.. nonce: wT5i1A +.. section: Core and Builtins + +The logarithm functions (such as :func:`math.log10` and :func:`math.log`) +may now produce slightly different results for extremely large integers that +cannot be converted to floats without overflow. These results are generally +more accurate, with reduced worst-case error and a tighter overall error +distribution. + +.. + +.. date: 2025-10-22-17-22-22 +.. gh-issue: 140431 +.. nonce: m8D_A- +.. section: Core and Builtins + +Fix a crash in Python's :term:`garbage collector <garbage collection>` due +to partially initialized :term:`coroutine` objects when coroutine origin +tracking depth is enabled (:func:`sys.set_coroutine_origin_tracking_depth`). + +.. + +.. date: 2025-10-22-12-48-05 +.. gh-issue: 140476 +.. nonce: F3-d1P +.. section: Core and Builtins + +Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded +<free threading>` build. + +.. + +.. date: 2025-10-22-11-30-16 +.. gh-issue: 135904 +.. nonce: 3WE5oW +.. section: Core and Builtins + +Add special labels to the assembly created during stencil creation to +support relocations that the native object file format does not support. +Specifically, 19 bit branches for AArch64 in Mach-O object files. + +.. + +.. date: 2025-10-21-09-20-03 +.. gh-issue: 140398 +.. nonce: SoABwJ +.. section: Core and Builtins + +Fix memory leaks in :mod:`readline` functions +:func:`~readline.read_init_file`, :func:`~readline.read_history_file`, +:func:`~readline.write_history_file`, and +:func:`~readline.append_history_file` when :c:func:`PySys_Audit` fails. + +.. + +.. date: 2025-10-21-06-51-50 +.. gh-issue: 140406 +.. nonce: 0gJs8M +.. section: Core and Builtins + +Fix memory leak when an object's :meth:`~object.__hash__` method returns an +object that isn't an :class:`int`. + +.. + +.. date: 2025-10-20-11-24-36 +.. gh-issue: 140358 +.. nonce: UQuKdV +.. section: Core and Builtins + +Restore elapsed time and unreachable object count in GC debug output. These +were inadvertently removed during a refactor of ``gc.c``. The debug log now +again reports elapsed collection time and the number of unreachable objects. +Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-19-10-32-28 +.. gh-issue: 136895 +.. nonce: HfsEh0 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 20 at build time. + +.. + +.. date: 2025-10-18-21-50-44 +.. gh-issue: 139109 +.. nonce: 9QQOzN +.. section: Core and Builtins + +A new tracing frontend for the JIT compiler has been implemented. Patch by +Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. + +.. + +.. date: 2025-10-18-21-29-45 +.. gh-issue: 140306 +.. nonce: xS5CcS +.. section: Core and Builtins + +Fix memory leaks in cross-interpreter channel operations and shared +namespace handling. + +.. + +.. date: 2025-10-18-19-52-20 +.. gh-issue: 116738 +.. nonce: NLJW0L +.. section: Core and Builtins + +Make _suggestions module thread-safe on the :term:`free threaded <free +threading>` build. + +.. + +.. date: 2025-10-18-18-08-36 +.. gh-issue: 140301 +.. nonce: m-2HxC +.. section: Core and Builtins + +Fix memory leak of ``PyConfig`` in subinterpreters. + +.. + +.. date: 2025-10-17-20-23-19 +.. gh-issue: 140257 +.. nonce: 8Txmem +.. section: Core and Builtins + +Fix data race between interpreter_clear() and take_gil() on eval_breaker +during finalization with daemon threads. + +.. + +.. date: 2025-10-17-18-03-12 +.. gh-issue: 139951 +.. nonce: IdwM2O +.. section: Core and Builtins + +Fixes a regression in GC performance for a growing heap composed mostly of +small tuples. + +* Counts number of actually tracked objects, instead of trackable objects. + This ensures that untracking tuples has the desired effect of reducing GC overhead. +* Does not track most untrackable tuples during creation. + This prevents large numbers of small tuples causing excessive GCs. + +.. + +.. date: 2025-10-17-14-38-10 +.. gh-issue: 140253 +.. nonce: gCqFaL +.. section: Core and Builtins + +Wrong placement of a double-star pattern inside a mapping pattern now throws +a specialized syntax error. Contributed by Bartosz Sławecki in :gh:`140253`. + +.. + +.. date: 2025-10-16-21-47-00 +.. gh-issue: 140104 +.. nonce: A8SQIm +.. section: Core and Builtins + +Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported +by Daniel Diniz. + +.. + +.. date: 2025-10-15-17-12-32 +.. gh-issue: 140149 +.. nonce: cy1m3d +.. section: Core and Builtins + +Speed up parsing bytes literals concatenation by using PyBytesWriter API and +a single memory allocation (about 3x faster). + +.. + +.. date: 2025-10-15-00-21-40 +.. gh-issue: 140061 +.. nonce: J0XeDV +.. section: Core and Builtins + +Fixing the checking of whether an object is uniquely referenced to ensure +free-threaded compatibility. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-14-20-18-31 +.. gh-issue: 140080 +.. nonce: 8ROjxW +.. section: Core and Builtins + +Fix hang during finalization when attempting to call :mod:`atexit` handlers +under no memory. + +.. + +.. date: 2025-10-14-18-24-16 +.. gh-issue: 139871 +.. nonce: SWtuUz +.. section: Core and Builtins + +Update :class:`bytearray` to use a :class:`bytes` under the hood as its +buffer and add :meth:`bytearray.take_bytes` to take it out. + +.. + +.. date: 2025-10-14-17-07-37 +.. gh-issue: 140067 +.. nonce: ID2gOm +.. section: Core and Builtins + +Fix memory leak in sub-interpreter creation. + +.. + +.. date: 2025-10-13-13-54-19 +.. gh-issue: 139914 +.. nonce: M-y_3E +.. section: Core and Builtins + +Restore support for HP PA-RISC, which has an upwards-growing stack. + +.. + +.. date: 2025-10-12-01-12-12 +.. gh-issue: 139817 +.. nonce: PAn-8Z +.. section: Core and Builtins + +Attribute ``__qualname__`` is added to :class:`typing.TypeAliasType`. Patch +by Mikhail Efimov. + +.. + +.. date: 2025-10-06-14-19-47 +.. gh-issue: 135801 +.. nonce: OhxEZS +.. section: Core and Builtins + +Many functions related to compiling or parsing Python code, such as +:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and +:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify the +module name. It is needed to unambiguous :ref:`filter <warning-filter>` +syntax warnings by module name. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb2 +.. section: Core and Builtins + +:func:`ast.parse` no longer emits syntax warnings for +``return``/``break``/``continue`` in ``finally`` (see :pep:`765`) -- they +are only emitted during compilation. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb +.. section: Core and Builtins + +Fix swallowing some syntax warnings in different modules if they +accidentally have the same message and are emitted from the same line. Fix +duplicated warnings in the ``finally`` block. + +.. + +.. date: 2025-10-03-17-51-43 +.. gh-issue: 139475 +.. nonce: _684ED +.. section: Core and Builtins + +Changes in stackref debugging mode when ``Py_STACKREF_DEBUG`` is set. We use +the same pattern of refcounting for stackrefs as in production build. + +.. + +.. date: 2025-09-23-21-01-12 +.. gh-issue: 139269 +.. nonce: 1rIaxy +.. section: Core and Builtins + +Fix undefined behavior when using unaligned store in JIT's ``patch_*`` +functions. + +.. + +.. date: 2025-09-15-13-06-11 +.. gh-issue: 138944 +.. nonce: PeCgLb +.. section: Core and Builtins + +Fix :exc:`SyntaxError` message when invalid syntax appears on the same line +as a valid ``import ... as ...`` or ``from ... import ... as ...`` +statement. Patch by Brian Schubert. + +.. + +.. date: 2025-09-13-01-23-25 +.. gh-issue: 138857 +.. nonce: YQ5gdc +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for ``case`` keyword placed outside +:keyword:`match` body. + +.. + +.. date: 2025-07-29-17-51-14 +.. gh-issue: 131253 +.. nonce: GpRjWy +.. section: Core and Builtins + +Support the ``--enable-pystats`` build option for the free-threaded build. + +.. + +.. date: 2025-07-08-00-41-46 +.. gh-issue: 136327 +.. nonce: 7AiTb_ +.. section: Core and Builtins + +Errors when calling functions with invalid values after ``*`` and ``**`` now +do not include the function name. Patch by Ilia Solin. + +.. + +.. date: 2025-06-24-13-12-58 +.. gh-issue: 134786 +.. nonce: MF0VVk +.. section: Core and Builtins + +If :c:macro:`Py_TPFLAGS_MANAGED_DICT` and +:c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` are used, then +:c:macro:`Py_TPFLAGS_HAVE_GC` must be used as well. + +.. + +.. date: 2025-11-10-11-26-26 +.. gh-issue: 141341 +.. nonce: OsO6-y +.. section: C API + +On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name +conflicts. Patch by Victor Stinner. + +.. + +.. date: 2025-11-08-10-51-50 +.. gh-issue: 116146 +.. nonce: pCmx6L +.. section: C API + +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a *spec* and *initfunc*. Patch by Itamar Oren. + +.. + +.. date: 2025-11-06-06-28-14 +.. gh-issue: 141042 +.. nonce: brOioJ +.. section: C API + +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-11-05-05-45-49 +.. gh-issue: 141004 +.. nonce: N9Ooh9 +.. section: C API + +:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. + +.. + +.. date: 2025-11-05-04-38-16 +.. gh-issue: 141004 +.. nonce: rJL43P +.. section: C API + +The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. + +.. + +.. date: 2025-10-26-16-45-28 +.. gh-issue: 140556 +.. nonce: s__Dae +.. section: C API + +:pep:`793`: Add a new entry point for C extension modules, +``PyModExport_<modulename>``. + +.. + +.. date: 2025-10-26-16-45-06 +.. gh-issue: 140487 +.. nonce: fGOqss +.. section: C API + +Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: +don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. + +.. + +.. date: 2025-10-15-15-59-59 +.. gh-issue: 140153 +.. nonce: BO7sH4 +.. section: C API + +Fix :c:func:`Py_REFCNT` definition on limited C API 3.11-3.13. Patch by +Victor Stinner. + +.. + +.. date: 2025-10-06-22-17-47 +.. gh-issue: 139653 +.. nonce: 6-1MOd +.. section: C API + +Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and +:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the +stack protection base address and stack protection size of a Python thread +state. Patch by Victor Stinner. + +.. + +.. date: 2025-10-31-13-20-16 +.. gh-issue: 140454 +.. nonce: gF6dCe +.. section: Build + +When building the JIT, match the jit_stencils filename expectations in +Makefile with the generator script. This avoid needless JIT recompilation +during ``make install``. + +.. + +.. date: 2025-10-29-12-30-38 +.. gh-issue: 140768 +.. nonce: ITYrzw +.. section: Build + +Warn when the WASI SDK version doesn't match what's supported. + +.. + +.. date: 2025-10-25-08-07-06 +.. gh-issue: 140513 +.. nonce: 6OhLTs +.. section: Build + +Generate a clear compilation error when ``_Py_TAIL_CALL_INTERP`` is enabled +but either ``preserve_none`` or ``musttail`` is not supported. + +.. + +.. date: 2025-10-22-12-44-07 +.. gh-issue: 140475 +.. nonce: OhzQbR +.. section: Build + +Support WASI SDK 25. + +.. + +.. date: 2025-10-17-11-33-45 +.. gh-issue: 140239 +.. nonce: _k-GgW +.. section: Build + +Check ``statx`` availability only on Linux (including Android). + +.. + +.. date: 2025-10-16-11-30-53 +.. gh-issue: 140189 +.. nonce: YCrUyt +.. section: Build + +iOS builds were added to CI. + +.. + +.. date: 2025-08-10-22-28-06 +.. gh-issue: 137618 +.. nonce: FdNvIE +.. section: Build + +``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. Patch by Adam +Turner. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index d75132b0aac..cc24bae5881 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -5536,8 +5536,8 @@ Tyler Kieft. .. section: C API :c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full -path (:c:func:`Py_GetProgramFullPath`) rather than to the program name -(:c:func:`Py_GetProgramName`). +path (:c:func:`!Py_GetProgramFullPath`) rather than to the program name +(:c:func:`!Py_GetProgramName`). .. diff --git a/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst b/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst new file mode 100644 index 00000000000..d9870d26704 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst @@ -0,0 +1,4 @@ +Add configure option :option:`--with-missing-stdlib-config=FILE` allows +which distributors to pass a `JSON <https://www.json.org/json-en.html>`_ +configuration file containing custom error messages for missing +:term:`standard library` modules. diff --git a/Misc/NEWS.d/next/Build/2025-11-19-09-21-17.gh-issue-141172.cYWc4x.rst b/Misc/NEWS.d/next/Build/2025-11-19-09-21-17.gh-issue-141172.cYWc4x.rst new file mode 100644 index 00000000000..7cade1ebd41 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-19-09-21-17.gh-issue-141172.cYWc4x.rst @@ -0,0 +1 @@ +Update to WASI SDK 29. diff --git a/Misc/NEWS.d/next/Build/2025-11-20-17-01-05.gh-issue-141784.LkYI2n.rst b/Misc/NEWS.d/next/Build/2025-11-20-17-01-05.gh-issue-141784.LkYI2n.rst new file mode 100644 index 00000000000..f20d8409416 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-20-17-01-05.gh-issue-141784.LkYI2n.rst @@ -0,0 +1,4 @@ +Fix ``_remote_debugging_module.c`` compilation on 32-bit Linux. Include +Python.h before system headers to make sure that +``_remote_debugging_module.c`` uses the same types (ABI) than Python. Patch +by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst b/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst new file mode 100644 index 00000000000..dab79ba5cf9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst @@ -0,0 +1,4 @@ +``RUNSHARED`` is no longer cleared when cross-compiling. Previously, +``RUNSHARED`` was cleared when cross-compiling, which breaks PGO when using +``--enabled-shared`` on systems where the cross-compiled CPython is otherwise +executable (e.g., via transparent emulation). diff --git a/Misc/NEWS.d/next/Build/2025-11-28-19-49-01.gh-issue-141808.cV5K12.rst b/Misc/NEWS.d/next/Build/2025-11-28-19-49-01.gh-issue-141808.cV5K12.rst new file mode 100644 index 00000000000..3162c7c4141 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-28-19-49-01.gh-issue-141808.cV5K12.rst @@ -0,0 +1 @@ +Do not generate the jit stencils twice in case of PGO builds on Windows. diff --git a/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst b/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst new file mode 100644 index 00000000000..8917d5df76e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst @@ -0,0 +1 @@ +Fixed a bug where JIT stencils produced on Windows contained debug data. Patch by Chris Eibl. diff --git a/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst b/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst new file mode 100644 index 00000000000..a586512fc0a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst @@ -0,0 +1,3 @@ +Allow ``--enable-wasm-dynamic-linking`` for WASI. While CPython doesn't +directly support it so external/downstream users do not have to patch in +support for the flag. diff --git a/Misc/NEWS.d/next/C_API/2025-09-22-16-32-00.gh-issue-139165.6Czn7S.rst b/Misc/NEWS.d/next/C_API/2025-09-22-16-32-00.gh-issue-139165.6Czn7S.rst new file mode 100644 index 00000000000..039c25b8ce2 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-09-22-16-32-00.gh-issue-139165.6Czn7S.rst @@ -0,0 +1,2 @@ +Expose the functions :c:func:`Py_SIZE`, :c:func:`Py_IS_TYPE` and +:c:func:`Py_SET_SIZE` in the Stable ABI. diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst b/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst new file mode 100644 index 00000000000..39cfcf73404 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. It should +only be used for debugging. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-11-18-04-16-09.gh-issue-140042.S1C7id.rst b/Misc/NEWS.d/next/C_API/2025-11-18-04-16-09.gh-issue-140042.S1C7id.rst new file mode 100644 index 00000000000..608e806b431 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-18-04-16-09.gh-issue-140042.S1C7id.rst @@ -0,0 +1 @@ +Removed the sqlite3_shutdown call that could cause closing connections for sqlite when used with multiple sub interpreters. diff --git a/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst b/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst new file mode 100644 index 00000000000..3fdad5c6b3e --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst @@ -0,0 +1 @@ +Add :c:func:`PyDict_SetDefaultRef` to the Stable ABI. diff --git a/Misc/NEWS.d/next/C_API/2025-11-21-10-34-00.gh-issue-137422.tzZKLi.rst b/Misc/NEWS.d/next/C_API/2025-11-21-10-34-00.gh-issue-137422.tzZKLi.rst new file mode 100644 index 00000000000..656289663cf --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-21-10-34-00.gh-issue-137422.tzZKLi.rst @@ -0,0 +1,4 @@ +Fix :term:`free threading` race condition in +:c:func:`PyImport_AddModuleRef`. It was previously possible for two calls to +the function return two different objects, only one of which was stored in +:data:`sys.modules`. diff --git a/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst b/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst new file mode 100644 index 00000000000..5edcfd81992 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst @@ -0,0 +1,2 @@ +Fix the ``HAVE_THREAD_LOCAL`` macro being defined without the +``Py_BUILD_CORE`` macro set after including :file:`Python.h`. diff --git a/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst b/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst new file mode 100644 index 00000000000..1eaf5b713d9 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst @@ -0,0 +1 @@ +Fixed the :c:macro:`PyABIInfo_VAR` macro. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-00-14-20.gh-issue-116738.IxliC_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-00-14-20.gh-issue-116738.IxliC_.rst new file mode 100644 index 00000000000..8b08bccafd7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-00-14-20.gh-issue-116738.IxliC_.rst @@ -0,0 +1,2 @@ +Make csv module thread-safe on the :term:`free threaded <free threading>` +build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-14-04-35.gh-issue-141589.VfdMDD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-14-04-35.gh-issue-141589.VfdMDD.rst new file mode 100644 index 00000000000..5eb0e0c8b89 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-14-04-35.gh-issue-141589.VfdMDD.rst @@ -0,0 +1,3 @@ +Change ``backoff counter`` to use prime numbers instead of powers of 2. +Use only 3 bits for ``counter`` and 13 bits for ``value``. +This allows to support values up to 8191. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst new file mode 100644 index 00000000000..c038dc742cc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst @@ -0,0 +1 @@ +Improve multithreaded scaling of dataclasses on the free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-16-21-14-48.gh-issue-41779.rXIj5h.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-16-21-14-48.gh-issue-41779.rXIj5h.rst new file mode 100644 index 00000000000..8ba3ca8df86 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-16-21-14-48.gh-issue-41779.rXIj5h.rst @@ -0,0 +1,2 @@ +Allowed defining the *__dict__* and *__weakref__* :ref:`__slots__ <slots>` +for any class. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst new file mode 100644 index 00000000000..c3ae0e8adab --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst @@ -0,0 +1,4 @@ +Only raise a ``RecursionError`` or trigger a fatal error if the stack +pointer is both below the limit pointer *and* above the stack base. If +outside of these bounds assume that it is OK. This prevents false positives +when user-space threads swap stacks. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-18-07-45-37.gh-issue-140638.i06qxD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-18-07-45-37.gh-issue-140638.i06qxD.rst new file mode 100644 index 00000000000..891e24d7644 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-18-07-45-37.gh-issue-140638.i06qxD.rst @@ -0,0 +1,2 @@ +Expose a ``"duration"`` stat in :func:`gc.get_stats` and +:data:`gc.callbacks`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-19-16-40-24.gh-issue-141732.PTetqp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-19-16-40-24.gh-issue-141732.PTetqp.rst new file mode 100644 index 00000000000..08420fd5f4d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-19-16-40-24.gh-issue-141732.PTetqp.rst @@ -0,0 +1,2 @@ +Ensure the :meth:`~object.__repr__` for :exc:`ExceptionGroup` and :exc:`BaseExceptionGroup` does +not change when the exception sequence that was original passed in to its constructor is subsequently mutated. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-20-13-18-57.gh-issue-141780.xDrVNr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-20-13-18-57.gh-issue-141780.xDrVNr.rst new file mode 100644 index 00000000000..8700ac14824 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-20-13-18-57.gh-issue-141780.xDrVNr.rst @@ -0,0 +1,2 @@ +Fix :c:macro:`Py_mod_gil` with API added in :pep:`793`: +:c:func:`!PyModule_FromSlotsAndSpec` and ``PyModExport`` hooks diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-20-22-09-22.gh-issue-140638.f6btj0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-20-22-09-22.gh-issue-140638.f6btj0.rst new file mode 100644 index 00000000000..e3af941523c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-20-22-09-22.gh-issue-140638.f6btj0.rst @@ -0,0 +1,2 @@ +Expose a ``"candidates"`` stat in :func:`gc.get_stats` and +:data:`gc.callbacks`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-22-10-43-26.gh-issue-120158.41_rXd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-22-10-43-26.gh-issue-120158.41_rXd.rst new file mode 100644 index 00000000000..b3b5f252ac0 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-22-10-43-26.gh-issue-120158.41_rXd.rst @@ -0,0 +1,2 @@ +Fix inconsistent state when enabling or disabling monitoring events too many +times. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-16-07-57.gh-issue-138122.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-16-07-57.gh-issue-138122.m3EF9E.rst new file mode 100644 index 00000000000..a4a29e40027 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-16-07-57.gh-issue-138122.m3EF9E.rst @@ -0,0 +1,6 @@ +Add incomplete sample detection to prevent corrupted profiling data. Each +thread state now contains an embedded base frame (sentinel at the bottom of +the frame stack) with owner type ``FRAME_OWNED_BY_INTERPRETER``. The profiler +validates that stack unwinding terminates at this sentinel frame. Samples that +fail to reach the base frame (due to race conditions, memory corruption, or +other errors) are now rejected rather than being included as spurious data. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst new file mode 100644 index 00000000000..06a12f98224 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst @@ -0,0 +1,2 @@ +When importing a module, use Python's regular file object to ensure that +writes to ``.pyc`` files are complete or an appropriate error is raised. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-25-02-23-31.gh-issue-141861.QcMdcM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-25-02-23-31.gh-issue-141861.QcMdcM.rst new file mode 100644 index 00000000000..4a115669998 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-25-02-23-31.gh-issue-141861.QcMdcM.rst @@ -0,0 +1 @@ +Fix invalid memory read in the ``ENTER_EXECUTOR`` instruction. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-25-13-13-34.gh-issue-116738.MnZRdV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-25-13-13-34.gh-issue-116738.MnZRdV.rst new file mode 100644 index 00000000000..151f8968292 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-25-13-13-34.gh-issue-116738.MnZRdV.rst @@ -0,0 +1,2 @@ +Fix thread safety issue with :mod:`re` scanner objects in free-threaded +builds. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst new file mode 100644 index 00000000000..d6d9d02b017 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst @@ -0,0 +1,2 @@ +Fix :mod:`cmath` data race when initializing trigonometric tables with +subinterpreters. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst new file mode 100644 index 00000000000..1400dae13ff --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst @@ -0,0 +1,2 @@ +Fix quadratically increasing garbage collection delays in free-threaded +build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst new file mode 100644 index 00000000000..f77315b7c37 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst @@ -0,0 +1 @@ +Check against abstract stack overflow in the JIT optimizer. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst new file mode 100644 index 00000000000..a8ce0fc6526 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst @@ -0,0 +1,2 @@ +Fix crash when inserting into a split table dictionary with a non +:class:`str` key that matches an existing key. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-16-43.gh-issue-142236.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-16-43.gh-issue-142236.m3EF9E.rst new file mode 100644 index 00000000000..b5c6a27fd6a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-16-43.gh-issue-142236.m3EF9E.rst @@ -0,0 +1,4 @@ +Fix incorrect keyword suggestions for syntax errors in :mod:`traceback`. The +keyword typo suggestion mechanism would incorrectly suggest replacements when +the extracted source code was incomplete rather than containing an actual typo. +Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Documentation/2025-11-26-23-30-09.gh-issue-141994.arBEG6.rst b/Misc/NEWS.d/next/Documentation/2025-11-26-23-30-09.gh-issue-141994.arBEG6.rst new file mode 100644 index 00000000000..c370e8a86e1 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-11-26-23-30-09.gh-issue-141994.arBEG6.rst @@ -0,0 +1,4 @@ +:mod:`xml.sax.handler`: Make Documentation of +:data:`xml.sax.handler.feature_external_ges` warn of opening up to `external +entity attacks <https://en.wikipedia.org/wiki/XML_external_entity_attack>`_. +Patch by Sebastian Pipping. diff --git a/Misc/NEWS.d/next/Library/2024-05-20-12-35-52.gh-issue-115952.J6n_Kf.rst b/Misc/NEWS.d/next/Library/2024-05-20-12-35-52.gh-issue-115952.J6n_Kf.rst new file mode 100644 index 00000000000..4c4c65d45d7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-20-12-35-52.gh-issue-115952.J6n_Kf.rst @@ -0,0 +1,7 @@ +Fix a potential memory denial of service in the :mod:`pickle` module. +When reading a pickled data received from untrusted source, it could cause +an arbitrary amount of memory to be allocated, even if the code that is +allowed to execute is restricted by overriding the +:meth:`~pickle.Unpickler.find_class` method. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. diff --git a/Misc/NEWS.d/next/Library/2025-05-30-18-37-44.gh-issue-134453.kxkA-o.rst b/Misc/NEWS.d/next/Library/2025-05-30-18-37-44.gh-issue-134453.kxkA-o.rst new file mode 100644 index 00000000000..fee975ea702 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-30-18-37-44.gh-issue-134453.kxkA-o.rst @@ -0,0 +1,4 @@ +Fixed :func:`subprocess.Popen.communicate` ``input=`` handling of :class:`memoryview` +instances that were non-byte shaped on POSIX platforms. Those are now properly +cast to a byte shaped view instead of truncating the input. Windows platforms +did not have this bug. diff --git a/Misc/NEWS.d/next/Library/2025-09-09-10-13-24.gh-issue-138525.hDTaAM.rst b/Misc/NEWS.d/next/Library/2025-09-09-10-13-24.gh-issue-138525.hDTaAM.rst new file mode 100644 index 00000000000..c4cea4b74b8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-09-10-13-24.gh-issue-138525.hDTaAM.rst @@ -0,0 +1,2 @@ +Add support for single-dash long options and alternate prefix characters in +:class:`argparse.BooleanOptionalAction`. diff --git a/Misc/NEWS.d/next/Library/2025-09-09-13-00-42.gh-issue-138697.QVwJw_.rst b/Misc/NEWS.d/next/Library/2025-09-09-13-00-42.gh-issue-138697.QVwJw_.rst new file mode 100644 index 00000000000..35aaa7cf70f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-09-13-00-42.gh-issue-138697.QVwJw_.rst @@ -0,0 +1,4 @@ +Fix inferring *dest* from a single-dash long option in :mod:`argparse`. If a +short option and a single-dash long option are passed to +:meth:`!add_argument`, *dest* is now inferred from the single-dash long +option. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-06-38-35.gh-issue-139946.HZa5hu.rst b/Misc/NEWS.d/next/Library/2025-10-23-06-38-35.gh-issue-139946.HZa5hu.rst new file mode 100644 index 00000000000..fb479317284 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-23-06-38-35.gh-issue-139946.HZa5hu.rst @@ -0,0 +1 @@ +Distinguish stdout and stderr when colorizing output in argparse module. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-17-00-11.gh-issue-140677.hM9pTq.rst b/Misc/NEWS.d/next/Library/2025-10-27-17-00-11.gh-issue-140677.hM9pTq.rst new file mode 100644 index 00000000000..2daa15e13ad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-27-17-00-11.gh-issue-140677.hM9pTq.rst @@ -0,0 +1,4 @@ +Add heatmap visualization mode to the Tachyon sampling profiler. The new +``--heatmap`` output format provides a line-by-line view showing execution +intensity with color-coded samples, inline statistics, and interactive call +graph navigation between callers and callees. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-10-44-23.gh-issue-140875.wt6B37.rst b/Misc/NEWS.d/next/Library/2025-11-02-10-44-23.gh-issue-140875.wt6B37.rst new file mode 100644 index 00000000000..c08a8966d53 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-10-44-23.gh-issue-140875.wt6B37.rst @@ -0,0 +1,3 @@ +Fix handling of unclosed character references (named and numerical) +followed by the end of file in :class:`html.parser.HTMLParser` with +``convert_charrefs=False``. diff --git a/Misc/NEWS.d/next/Library/2025-11-03-17-13-00.gh-issue-140911.7KFvSQ.rst b/Misc/NEWS.d/next/Library/2025-11-03-17-13-00.gh-issue-140911.7KFvSQ.rst new file mode 100644 index 00000000000..b0b6e461192 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-03-17-13-00.gh-issue-140911.7KFvSQ.rst @@ -0,0 +1,3 @@ +:mod:`collections`: Ensure that the methods ``UserString.rindex()`` and +``UserString.index()`` accept :class:`collections.UserString` instances as the +sub argument. diff --git a/Misc/NEWS.d/next/Library/2025-11-13-13-11-02.gh-issue-60107.LZq3QF.rst b/Misc/NEWS.d/next/Library/2025-11-13-13-11-02.gh-issue-60107.LZq3QF.rst new file mode 100644 index 00000000000..c5103fb8005 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-13-13-11-02.gh-issue-60107.LZq3QF.rst @@ -0,0 +1,2 @@ +Remove a copy from :meth:`io.RawIOBase.read`. If the underlying I/O class +keeps a reference to the mutable memory, raise a :exc:`BufferError`. diff --git a/Misc/NEWS.d/next/Library/2025-11-14-18-00-41.gh-issue-141565.Ap2bhJ.rst b/Misc/NEWS.d/next/Library/2025-11-14-18-00-41.gh-issue-141565.Ap2bhJ.rst new file mode 100644 index 00000000000..628f1e0af03 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-14-18-00-41.gh-issue-141565.Ap2bhJ.rst @@ -0,0 +1 @@ +Add async-aware profiling to the Tachyon sampling profiler. The profiler now reconstructs and displays async task hierarchies in flamegraphs, making the output more actionable for users. Patch by Savannah Ostrowski and Pablo Galindo Salgado. diff --git a/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst b/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst new file mode 100644 index 00000000000..37b91196658 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst @@ -0,0 +1,3 @@ +Add :func:`readline.get_pre_input_hook` function to retrieve the current +pre-input hook. This allows applications to save and restore the hook +without overwriting user settings. Patch by Sanyam Khurana. diff --git a/Misc/NEWS.d/next/Library/2025-11-15-14-58-12.gh-issue-141600.XY2BXg.rst b/Misc/NEWS.d/next/Library/2025-11-15-14-58-12.gh-issue-141600.XY2BXg.rst new file mode 100644 index 00000000000..8071246f130 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-15-14-58-12.gh-issue-141600.XY2BXg.rst @@ -0,0 +1 @@ +Fix musl version detection on Void Linux. diff --git a/Misc/NEWS.d/next/Library/2025-11-16-04-40-06.gh-issue-69113.Xy7Fmn.rst b/Misc/NEWS.d/next/Library/2025-11-16-04-40-06.gh-issue-69113.Xy7Fmn.rst new file mode 100644 index 00000000000..cd76ae9b11e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-16-04-40-06.gh-issue-69113.Xy7Fmn.rst @@ -0,0 +1 @@ +Fix :mod:`doctest` to correctly report line numbers for doctests in ``__test__`` dictionary when formatted as triple-quoted strings by finding unique lines in the string and matching them in the source file. diff --git a/Misc/NEWS.d/next/Library/2025-11-16-06-08-46.gh-issue-141615.--6EK3.rst b/Misc/NEWS.d/next/Library/2025-11-16-06-08-46.gh-issue-141615.--6EK3.rst new file mode 100644 index 00000000000..bb54e683987 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-16-06-08-46.gh-issue-141615.--6EK3.rst @@ -0,0 +1 @@ +Check ``stdin`` instead of ``stdout`` for ``use_rawinput`` in :mod:`pdb`. diff --git a/Misc/NEWS.d/next/Library/2025-11-17-00-53-51.gh-issue-141645.TC3TL3.rst b/Misc/NEWS.d/next/Library/2025-11-17-00-53-51.gh-issue-141645.TC3TL3.rst new file mode 100644 index 00000000000..d3f3bfddf6e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-17-00-53-51.gh-issue-141645.TC3TL3.rst @@ -0,0 +1,4 @@ +Add a new ``--live`` mode to the tachyon profiler in +:mod:`!profiling.sampling` module. This mode consist of a live TUI that +displays real-time profiling statistics as the target application runs, +similar to ``top``. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst b/Misc/NEWS.d/next/Library/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst new file mode 100644 index 00000000000..eeb055c6012 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst @@ -0,0 +1 @@ +Fix bad file descriptor errors from ``_posixsubprocess`` on AIX. diff --git a/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst b/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst new file mode 100644 index 00000000000..87e9cb8d69b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst @@ -0,0 +1,2 @@ +Break reference cycles created by each call to :func:`json.dump` or +:meth:`json.JSONEncoder.iterencode`. diff --git a/Misc/NEWS.d/next/Library/2025-11-17-21-41-58.gh-issue-141679.fs7zLJ.rst b/Misc/NEWS.d/next/Library/2025-11-17-21-41-58.gh-issue-141679.fs7zLJ.rst new file mode 100644 index 00000000000..b8203ca5a4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-17-21-41-58.gh-issue-141679.fs7zLJ.rst @@ -0,0 +1 @@ +Add colour to defaults in :mod:`argparse` help. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2025-11-18-14-39-31.gh-issue-141570.q3n984.rst b/Misc/NEWS.d/next/Library/2025-11-18-14-39-31.gh-issue-141570.q3n984.rst new file mode 100644 index 00000000000..8f4641ce4cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-18-14-39-31.gh-issue-141570.q3n984.rst @@ -0,0 +1,2 @@ +Support :term:`file-like object` raising :exc:`OSError` from :meth:`~io.IOBase.fileno` in color +detection (``_colorize.can_colorize()``). This can occur when ``sys.stdout`` is redirected. diff --git a/Misc/NEWS.d/next/Library/2025-11-18-15-48-13.gh-issue-105836.sbUw24.rst b/Misc/NEWS.d/next/Library/2025-11-18-15-48-13.gh-issue-105836.sbUw24.rst new file mode 100644 index 00000000000..d2edc5b2cb7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-18-15-48-13.gh-issue-105836.sbUw24.rst @@ -0,0 +1,2 @@ +Fix :meth:`asyncio.run_coroutine_threadsafe` leaving underlying cancelled +asyncio task running. diff --git a/Misc/NEWS.d/next/Library/2025-11-21-21-14-10.gh-issue-141817._v5LdB.rst b/Misc/NEWS.d/next/Library/2025-11-21-21-14-10.gh-issue-141817._v5LdB.rst new file mode 100644 index 00000000000..774e13b2129 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-21-21-14-10.gh-issue-141817._v5LdB.rst @@ -0,0 +1 @@ +Add :data:`!socket.IPV6_HDRINCL` constant. diff --git a/Misc/NEWS.d/next/Library/2025-11-22-16-33-48.gh-issue-141863.4PLhnv.rst b/Misc/NEWS.d/next/Library/2025-11-22-16-33-48.gh-issue-141863.4PLhnv.rst new file mode 100644 index 00000000000..585774ba2cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-22-16-33-48.gh-issue-141863.4PLhnv.rst @@ -0,0 +1,2 @@ +Update :ref:`asyncio-streams` to use :meth:`bytearray.take_bytes` for a over +10% performance improvement on pyperformance asyncio_tcp benchmark. diff --git a/Misc/NEWS.d/next/Library/2025-11-24-06-44-45.gh-issue-141781.MsK27r.rst b/Misc/NEWS.d/next/Library/2025-11-24-06-44-45.gh-issue-141781.MsK27r.rst new file mode 100644 index 00000000000..21db97025a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-24-06-44-45.gh-issue-141781.MsK27r.rst @@ -0,0 +1 @@ +Fixed an issue where pdb.line_prefix assignment was ignored if assigned after the module was imported. diff --git a/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst b/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst new file mode 100644 index 00000000000..5742beeb85c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst @@ -0,0 +1,5 @@ +The ``profiling.sampling`` flamegraph profiler now displays thread status +statistics showing the percentage of time threads spend holding the GIL, +running without the GIL, waiting for the GIL, and performing garbage +collection. These statistics help identify GIL contention and thread behavior +patterns. When filtering by thread, the display shows per-thread metrics. diff --git a/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst b/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst new file mode 100644 index 00000000000..33ab8a0659e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst @@ -0,0 +1 @@ +Fix :mod:`pdb` breakpoint resolution for class methods when the module defining the class is not imported. diff --git a/Misc/NEWS.d/next/Library/2025-11-25-22-54-07.gh-issue-141968.vg3AMJ.rst b/Misc/NEWS.d/next/Library/2025-11-25-22-54-07.gh-issue-141968.vg3AMJ.rst new file mode 100644 index 00000000000..4c89902813f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-22-54-07.gh-issue-141968.vg3AMJ.rst @@ -0,0 +1,2 @@ +Remove a data copy from :func:`base64.b32decode` and +:func:`base64.b32encode` by using :meth:`bytearray.take_bytes`. diff --git a/Misc/NEWS.d/next/Library/2025-11-25-23-22-46.gh-issue-141968.R1sHnJ.rst b/Misc/NEWS.d/next/Library/2025-11-25-23-22-46.gh-issue-141968.R1sHnJ.rst new file mode 100644 index 00000000000..eca0ac4c8e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-23-22-46.gh-issue-141968.R1sHnJ.rst @@ -0,0 +1,2 @@ +Remove data copy from :func:`wave.Wave_read.readframes` and +:func:`wave.Wave_write.writeframes` by using :meth:`bytearray.take_bytes`. diff --git a/Misc/NEWS.d/next/Library/2025-11-25-23-29-08.gh-issue-141968.0JnjXf.rst b/Misc/NEWS.d/next/Library/2025-11-25-23-29-08.gh-issue-141968.0JnjXf.rst new file mode 100644 index 00000000000..0cefeed30ed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-23-29-08.gh-issue-141968.0JnjXf.rst @@ -0,0 +1,2 @@ +Remove data copy from :mod:`codecs` ``punycode`` encoding by using +:meth:`bytearray.take_bytes`. diff --git a/Misc/NEWS.d/next/Library/2025-11-25-23-35-07.gh-issue-141968.b3Gscp.rst b/Misc/NEWS.d/next/Library/2025-11-25-23-35-07.gh-issue-141968.b3Gscp.rst new file mode 100644 index 00000000000..50124e8b196 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-23-35-07.gh-issue-141968.b3Gscp.rst @@ -0,0 +1,2 @@ +Remove data copy from :mod:`encodings.idna` :meth:`~codecs.Codec.encode` and +:meth:`~codecs.IncrementalEncoder.encode` by using :meth:`bytearray.take_bytes`. diff --git a/Misc/NEWS.d/next/Library/2025-11-26-14-20-10.gh-issue-141968.W139Pv.rst b/Misc/NEWS.d/next/Library/2025-11-26-14-20-10.gh-issue-141968.W139Pv.rst new file mode 100644 index 00000000000..c5375707814 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-26-14-20-10.gh-issue-141968.W139Pv.rst @@ -0,0 +1,2 @@ +Remove data copy from :mod:`re` compilation of regexes with large charsets +by using :meth:`bytearray.take_bytes`. diff --git a/Misc/NEWS.d/next/Library/2025-11-27-10-49-13.gh-issue-142006.nzJDG5.rst b/Misc/NEWS.d/next/Library/2025-11-27-10-49-13.gh-issue-142006.nzJDG5.rst new file mode 100644 index 00000000000..49643892ff9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-27-10-49-13.gh-issue-142006.nzJDG5.rst @@ -0,0 +1 @@ +Fix a bug in the :mod:`email.policy.default` folding algorithm which incorrectly resulted in a doubled newline when a line ending at exactly max_line_length was followed by an unfoldable token. diff --git a/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst b/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst new file mode 100644 index 00000000000..3b54a831b54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst @@ -0,0 +1,2 @@ +Correctly allow :exc:`KeyboardInterrupt` to stop the process when using +:mod:`!profiling.sampling`. diff --git a/Misc/NEWS.d/next/Library/2025-11-27-20-16-38.gh-issue-141473.Wq4xVN.rst b/Misc/NEWS.d/next/Library/2025-11-27-20-16-38.gh-issue-141473.Wq4xVN.rst new file mode 100644 index 00000000000..f6aa592cefd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-27-20-16-38.gh-issue-141473.Wq4xVN.rst @@ -0,0 +1,4 @@ +When :meth:`subprocess.Popen.communicate` was called with *input* and a +*timeout* and is called for a second time after a +:exc:`~subprocess.TimeoutExpired` exception before the process has died, it +should no longer hang. diff --git a/Misc/NEWS.d/next/Library/2025-11-29-03-02-45.gh-issue-87512.bn4xbm.rst b/Misc/NEWS.d/next/Library/2025-11-29-03-02-45.gh-issue-87512.bn4xbm.rst new file mode 100644 index 00000000000..091350108ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-29-03-02-45.gh-issue-87512.bn4xbm.rst @@ -0,0 +1,5 @@ +Fix :func:`subprocess.Popen.communicate` timeout handling on Windows +when writing large input. Previously, the timeout was ignored during +stdin writing, causing the method to block indefinitely if the child +process did not consume input quickly. The stdin write is now performed +in a background thread, allowing the timeout to be properly enforced. diff --git a/Misc/NEWS.d/next/Library/2025-11-29-04-20-44.gh-issue-74389.pW3URj.rst b/Misc/NEWS.d/next/Library/2025-11-29-04-20-44.gh-issue-74389.pW3URj.rst new file mode 100644 index 00000000000..a9bf5f80d80 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-29-04-20-44.gh-issue-74389.pW3URj.rst @@ -0,0 +1,3 @@ +When the stdin being used by a :class:`subprocess.Popen` instance is closed, +this is now ignored in :meth:`subprocess.Popen.communicate` instead of +leaving the class in an inconsistent state. diff --git a/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst b/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst new file mode 100644 index 00000000000..e5ec593dd6e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst @@ -0,0 +1 @@ +Allow :mod:`pdb` to set breakpoints on async functions with function names. diff --git a/Misc/NEWS.d/next/Library/2025-12-01-14-43-58.gh-issue-138122.nRm3ic.rst b/Misc/NEWS.d/next/Library/2025-12-01-14-43-58.gh-issue-138122.nRm3ic.rst new file mode 100644 index 00000000000..e24fea416ff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-01-14-43-58.gh-issue-138122.nRm3ic.rst @@ -0,0 +1,5 @@ +The ``_remote_debugging`` module now implements frame caching in the +``RemoteUnwinder`` class to reduce memory reads when profiling remote +processes. When ``cache_frames=True``, unchanged portions of the call stack +are reused from previous samples, significantly improving profiling +performance for deep call stacks. diff --git a/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst b/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst new file mode 100644 index 00000000000..b87430ec1a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst @@ -0,0 +1,12 @@ +Fix two regressions in :mod:`dataclasses` in Python 3.14.1 related to +annotations. + +* An exception is no longer raised if ``slots=True`` is used and the + ``__init__`` method does not have an ``__annotate__`` attribute + (likely because ``init=False`` was used). + +* An exception is no longer raised if annotations are requested on the + ``__init__`` method and one of the fields is not present in the class + annotations. This can occur in certain dynamic scenarios. + +Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst b/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst new file mode 100644 index 00000000000..90e4dd96985 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst @@ -0,0 +1,4 @@ +The resource tracker in the :mod:`multiprocessing` module can now understand +messages from older versions of itself. This avoids issues with upgrading +Python while it is running. (Note that such 'in-place' upgrades are not +tested.) diff --git a/Misc/NEWS.d/next/Library/2025-12-04-09-22-31.gh-issue-68552.I_v-xB.rst b/Misc/NEWS.d/next/Library/2025-12-04-09-22-31.gh-issue-68552.I_v-xB.rst new file mode 100644 index 00000000000..bd3e53c9f81 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-04-09-22-31.gh-issue-68552.I_v-xB.rst @@ -0,0 +1 @@ +``MisplacedEnvelopeHeaderDefect`` and ``Missing header name`` defects are now correctly passed to the ``handle_defect`` method of ``policy`` in :class:`~email.parser.FeedParser`. diff --git a/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst b/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst new file mode 100644 index 00000000000..2bee8881a75 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst @@ -0,0 +1 @@ +Remove ``color`` parameter from :class:`!argparse.HelpFormatter` constructor. Color is controlled by :class:`~argparse.ArgumentParser`. diff --git a/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst b/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst new file mode 100644 index 00000000000..f46e82105fc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst @@ -0,0 +1 @@ +Improve :mod:`argparse` performance by caching the formatter used for argument validation. diff --git a/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst b/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst new file mode 100644 index 00000000000..5ca3fc05b98 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst @@ -0,0 +1 @@ +Fix :mod:`argparse` to preserve ``|`` separators in mutually exclusive groups when the usage line wraps due to length. diff --git a/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst b/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst new file mode 100644 index 00000000000..8710ebfb1a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst @@ -0,0 +1,2 @@ +Fix typing ``'q'`` at the help of the interactive tachyon profiler exiting +the profiler. diff --git a/Misc/NEWS.d/next/Library/2025-12-06-13-02-13.gh-issue-142332.PNvXCV.rst b/Misc/NEWS.d/next/Library/2025-12-06-13-02-13.gh-issue-142332.PNvXCV.rst new file mode 100644 index 00000000000..ee2d5e1d491 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-13-02-13.gh-issue-142332.PNvXCV.rst @@ -0,0 +1,2 @@ +Fix usage formatting for positional arguments in mutually exclusive groups in :mod:`argparse`. +in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2025-12-06-13-19-43.gh-issue-142207.x_X9oH.rst b/Misc/NEWS.d/next/Library/2025-12-06-13-19-43.gh-issue-142207.x_X9oH.rst new file mode 100644 index 00000000000..69ca8c41ac9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-13-19-43.gh-issue-142207.x_X9oH.rst @@ -0,0 +1,2 @@ +Fix: profiling.sampling may cause assertion ``!(has_gil && +gil_requested)`` diff --git a/Misc/NEWS.d/next/Library/2025-12-06-16-45-34.gh-issue-64532.4OXZpF.rst b/Misc/NEWS.d/next/Library/2025-12-06-16-45-34.gh-issue-64532.4OXZpF.rst new file mode 100644 index 00000000000..3bd950050ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-16-45-34.gh-issue-64532.4OXZpF.rst @@ -0,0 +1 @@ +Subparser help now includes required optional arguments from the parent parser in the usage, making it clearer what arguments are needed to run a subcommand. Patch by Savannah Ostrowski. diff --git a/Misc/NEWS.d/next/Library/2025-12-07-17-30-05.gh-issue-142346.okcAAp.rst b/Misc/NEWS.d/next/Library/2025-12-07-17-30-05.gh-issue-142346.okcAAp.rst new file mode 100644 index 00000000000..cf570f314c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-07-17-30-05.gh-issue-142346.okcAAp.rst @@ -0,0 +1,3 @@ +Fix usage formatting for mutually exclusive groups in :mod:`argparse` +when they are preceded by positional arguments or followed or intermixed +with other optional arguments. diff --git a/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst new file mode 100644 index 00000000000..04fd8faca4c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`plistlib` module. +When reading a Plist file received from untrusted source, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. diff --git a/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst new file mode 100644 index 00000000000..6d6f25cd2f8 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`http.client` module. +When connecting to a malicious server, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. diff --git a/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst b/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst new file mode 100644 index 00000000000..17dc241aef9 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst @@ -0,0 +1,3 @@ +Fix buffer overflow in ``_Py_wrealpath()`` for paths exceeding ``MAXPATHLEN`` bytes +by using dynamic memory allocation instead of fixed-size buffer. +Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst new file mode 100644 index 00000000000..440bc7794c6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst @@ -0,0 +1 @@ +Remove quadratic behavior in ``xml.minidom`` node ID cache clearing. diff --git a/Misc/NEWS.d/next/Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst b/Misc/NEWS.d/next/Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst new file mode 100644 index 00000000000..f2064bfd377 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-10-16-15-08-58.gh-issue-140210.P9vUP8.rst @@ -0,0 +1,2 @@ +Make ``test_sysconfig.test_parse_makefile_renamed_vars`` less fragile by +clearing the environment variables before parsing the Makefile. diff --git a/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst b/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst new file mode 100644 index 00000000000..568a2b65d7d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst @@ -0,0 +1 @@ +Fix flaky test_profiling tests on i686 and s390x architectures by increasing slow_fibonacci call frequency from every 5th iteration to every 2nd iteration. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst new file mode 100644 index 00000000000..d85c54db364 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst @@ -0,0 +1,3 @@ +Each slice of an iOS XCframework now contains a ``lib`` folder that contains +a symlink to the libpython dylib. This allows binary modules to be compiled +for iOS using dynamic libreary linking, rather than Framework linking. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 69f3beec82e..dba01de8352 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,42 +70,42 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "6bb739ecddbd2cfb6d255eb5898437a9b5739277dee931338d3275bac5d96ba2" + "checksumValue": "9b07560b6c1afa666bd78b8d3aa5c83fdda02149afdf048596d5b0e0dac1ee55" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.16.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.18.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.16:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.18:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.0.16" + "versionInfo": "3.0.18" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "e335aeb44fa36cde60ecbb6a9f8be6f5d449d645ce9b0199ee53a7e6728d19d2" + "checksumValue": "fb5ab81f27612b0a7b4861ba655906c76dc85ee969e7a4905d2075aff931e8d0" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.49.1.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.50.4.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.49.1.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.50.4.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "sqlite", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.49.1.0" + "versionInfo": "3.50.4.0" }, { "SPDXID": "SPDXRef-PACKAGE-tcl-core", @@ -154,21 +154,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "a15c168e39e87d750c3dc766edc7f19bdda57dacf01e509678467eace91ad282" + "checksumValue": "1bfaba0ccacc6681d3ba85335cc7f49c24cf6f9d16f848cbd153b896d8a7d631" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.2.5.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/xz-5.8.1.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tukaani:xz:5.2.5:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tukaani:xz:5.8.1.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "xz", "primaryPackagePurpose": "SOURCE", - "versionInfo": "5.2.5" + "versionInfo": "5.8.1.1" }, { "SPDXID": "SPDXRef-PACKAGE-zlib-ng", diff --git a/Misc/python.man b/Misc/python.man index 15174b62d5f..612e2bbf568 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -529,11 +529,11 @@ See also the \fB\-X frozen_modules\fR option. If this variable is set to 1, the global interpreter lock (GIL) will be forced on. Setting it to 0 forces the GIL off. Only available in builds configured with \fB\-\-disable\-gil\fP. +.IP +This is equivalent to the \fB\-X gil\fR option. .IP PYTHON_HISTORY This environment variable can be used to set the location of a history file (on Unix, it is \fI~/.python_history\fP by default). -.IP -This is equivalent to the \fB\-X gil\fR option. .IP PYTHONNODEBUGRANGES If this variable is set, it disables the inclusion of the tables mapping extra location information (end line, start column offset and end column diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 4a697d047ca..e9554bf7837 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "6984055af7b4e01429d8ebc910fe2be900d8ee9c" + "checksumValue": "a4395dd0589a97aab0904f7a5f5dc5781a086aa2" }, { "algorithm": "SHA256", - "checksumValue": "7c16a5cf0eea844ae579db083b8d75f23a71859cac77e3c4cb7a8fa3b7621685" + "checksumValue": "610b844bbfa3ec955772cc825db4d4db470827d57adcb214ad372d0eaf00e591" } ], "fileName": "Modules/expat/expat.h" @@ -62,11 +62,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9e615c6e5c3ba00670f674a6b071bb855b0b563d" + "checksumValue": "c22196e3d8bee88fcdda715623b3b9d2119d2fb3" }, { "algorithm": "SHA256", - "checksumValue": "3d90a4b65c40a3f848c36100f4d73b933a015c7b7cd85c28e4331a6b845c1ad0" + "checksumValue": "f2c2283ba03b057e92beefc7f81ba901ebb6dfc1a45b036c8a7d65808eb77a84" } ], "fileName": "Modules/expat/expat_external.h" @@ -90,11 +90,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "60b0ee8b4a93ef0276193ed1051c15ecab73c02e" + "checksumValue": "7dce7d98943c5db33ae05e54801dcafb4547b9dd" }, { "algorithm": "SHA256", - "checksumValue": "6af6e8fbf5c83c1431464a2811b10ea2d1ff64c0eabfd9f18b1d4e53bf400c35" + "checksumValue": "6bfe307d52e7e4c71dbc30d3bd902a4905cdd83bbe4226a7e8dfa8e4c462a157" } ], "fileName": "Modules/expat/internal.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "3db0435d69e5eb904c9c88400a5ab073a81049bc" + "checksumValue": "4c81a1f04fc653877c63c834145c18f93cd95f3e" }, { "algorithm": "SHA256", - "checksumValue": "633b272fa893dfbef539edbba35f1b11ecf09a13b89189105b0dfa6c7ecfc3bf" + "checksumValue": "04a379615f476d55f95ca1853107e20627b48ca4afe8d0fd5981ac77188bf0a6" } ], "fileName": "Modules/expat/xmlparse.c" @@ -202,11 +202,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c961fb1a80f7b0601a63e69fba793fe5f6dff157" + "checksumValue": "ac2964cca107f62dd133bfd4736a9a17defbc401" }, { "algorithm": "SHA256", - "checksumValue": "228470eb9181a9a7575b63137edcb61b817ee4e0923faffdbeba29e07c939713" + "checksumValue": "92e41f373b67f6e0dcd7735faef3c3f1e2c17fe59e007e6b74beef6a2e70fa88" } ], "fileName": "Modules/expat/xmlrole.h" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8394790c0199c8f88108542ad78f23095d28a3fe" + "checksumValue": "1e2d35d90a1c269217f83d3bdf3c71cc22cb4c3f" }, { "algorithm": "SHA256", - "checksumValue": "5b16c671ccc42496374762768e4bf48f614aecfd2025a07925b8d94244aec645" + "checksumValue": "98d0fc735041956cc2e7bbbe2fb8f03130859410e0aee5e8015f406a37c02a3c" } ], "fileName": "Modules/expat/xmltok.c" @@ -230,11 +230,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7d2943a0128094455004b1a98007b98734221bae" + "checksumValue": "d126831eaa5158cff187a8c93f4bc1c8118f3b17" }, { "algorithm": "SHA256", - "checksumValue": "6b8919dc951606dc6f2b0175f8955a9ced901ce8bd08db47f291b6c04227ae7f" + "checksumValue": "91bf003a725a675761ea8d92cebc299a76fd28c3a950572f41bc7ce5327ee7b5" } ], "fileName": "Modules/expat/xmltok.h" @@ -300,11 +300,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "de7179fe6970e2b5d281dfed977ed91be635b8d2" + "checksumValue": "a57d53c35aa916ce0cd1567c8f10dcea58270321" }, { "algorithm": "SHA256", - "checksumValue": "c0ba888d87775c7d7f7d8a08dac7b3988fed81e11bb52396d90f762a8e90a7eb" + "checksumValue": "d68209032703137326f9aadd9abac8fe4a5c92d391e2f43b0414fa63087adc6b" } ], "fileName": "Modules/_hacl/Hacl_HMAC.h" @@ -314,11 +314,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4b6e7696e8d84f322fb24b1fbb08ccb9b0e7d51b" + "checksumValue": "808af7ff8a2cb2b4ef3a9ce3dbfef58d90828c9f" }, { "algorithm": "SHA256", - "checksumValue": "50a65a34a7a7569eedf7fa864a7892eeee5840a7fdf6fa8f1e87d42c65f6c877" + "checksumValue": "6a492aa586f2d10b1b300ce8ce4c72c976ff7548fee667aded2253f99969ac87" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.c" @@ -328,11 +328,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7c66ac004a1dcf3fee0ab9aa62d61972f029de3a" + "checksumValue": "4d767388a34e2a2000e0cbd31f06cf36af1ae10d" }, { "algorithm": "SHA256", - "checksumValue": "9a7239a01a4ee8defbe3ebd9f0d12c873a1dd8e0659070380b2eab3ab0177333" + "checksumValue": "fac493e96af252abcf5705f0ab4eec59c1f3bc8244e105d75a57c43552dd1569" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.h" @@ -342,11 +342,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0f75e44a42775247a46acc2beaa6bae8f199a3d9" + "checksumValue": "1bb072f2be9e5d194274fdcc87825cb094e4b32e" }, { "algorithm": "SHA256", - "checksumValue": "03b612c24193464ed6848aeebbf44f9266b78ec6eed2486056211cde8992c49a" + "checksumValue": "9eb22953ce60dde9dc970fec9dfce9d94235f4b7ccd8f0151cad4707dc835d1d" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c" @@ -356,11 +356,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7f273d26942233e5dcdfb4c1a16ff2486b15f899" + "checksumValue": "280fd03ee23e13ecf90711d2be8230035d957343" }, { "algorithm": "SHA256", - "checksumValue": "dbc0dacc68ed52dbf1b7d6fba2c87870317998bc046e65f6deaaa150625432f8" + "checksumValue": "817dcd05d06d804587fce7d8f2f3f42a6fcf6818d2419a3551ef0df70cb7125a" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h" @@ -384,11 +384,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "65bf44140691b046dcfed3ab1576dbf8bbf96dc5" + "checksumValue": "cdd6e9ca86dbede92d1a47b9224d2af70c599a71" }, { "algorithm": "SHA256", - "checksumValue": "0f98959dafffce039ade9d296f7a05bed151c9c512498f48e4b326a5523a240b" + "checksumValue": "f3204f3e60734d811b6630f879b69ce54eaf367f3fca5889c1026e7a1f3ee1e4" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.c" @@ -398,11 +398,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2120c8c467aeebcc7c8b9678c15e79648433b91a" + "checksumValue": "79f47ab5458d88bce0f210a566aa117ce2125049" }, { "algorithm": "SHA256", - "checksumValue": "45735f7fe2dbbad7656d07854e9ec8176ad26c79f90dcc0fec0b9a59a6311ba7" + "checksumValue": "5cc96313f8c066f055c2819e473c79aeff086ba91a1449d54aa569127cd8601e" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.h" @@ -412,11 +412,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0da9782455923aede8d8dce9dfdc38f4fc1de572" + "checksumValue": "4056bb6e3ed184400d1610bdfd4260b3fd05ccbb" }, { "algorithm": "SHA256", - "checksumValue": "2d17ae768fd3d7d6decddd8b4aaf23ce02a809ee62bb98da32c8a7f54acf92d0" + "checksumValue": "c93746df2f219cbb1634ee6fb0ab1c4cbd381d1f36c637114c68346c9935326d" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c" @@ -426,11 +426,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9028e24c9876d9d16b2435ec29240c6b57bfe2a0" + "checksumValue": "c98890b6193baa3dbd472cfb67f22aea3dd0d3e1" }, { "algorithm": "SHA256", - "checksumValue": "062e3b856acac4f929c1e04b8264a754cad21ca6580215f7094a3f0a04edb912" + "checksumValue": "fe3f17bf237166f872ba88c8204333c4f64bdc4bb29c265448581c7bfea4b151" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h" @@ -454,11 +454,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "38e8d96ef1879480780494058a93cec181f8d6d7" + "checksumValue": "61f678cd9234c6eab5d4409ae66f470b068b959b" }, { "algorithm": "SHA256", - "checksumValue": "61e77d2063cf60c96e9ce06af215efe5d42c43026833bffed5732326fe97ed1e" + "checksumValue": "5b91ed0339074e2e546119833398e2cdb7190c33a59c405bf43b2417c789547d" } ], "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" @@ -468,11 +468,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e67a9bc18358c57afaeff3a174893ddfdb52dfc6" + "checksumValue": "2ed70f612a998ef6c04d62f9487a1b2534e6d76a" }, { "algorithm": "SHA256", - "checksumValue": "16e982081f6c2fd03ea751fcc64f5a835c94652841836e231fe562b9e287f4bc" + "checksumValue": "a34534ef36bc8428ac1169ca5f4f1c17a0817507ebae388e3dd825d1a331f28d" } ], "fileName": "Modules/_hacl/Hacl_Hash_MD5.h" @@ -482,11 +482,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "986dd5ba0b34d15f3e5e5c656979aea1b502e8aa" + "checksumValue": "c7fc5c9721caf37c5a24c9beff27b0ac2ed68cc9" }, { "algorithm": "SHA256", - "checksumValue": "38d5f1f2e67a0eb30789f81fc56c07a6e7246e2b1be6c65485bcca1dcd0e0806" + "checksumValue": "ce08721d491f3b8a9bd4cde6c27bfcc8fc01471512ccca4bd3c0b764cb551d29" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" @@ -496,11 +496,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f1ca21f1ee8b15ad9ccfbda72165b9d86912166c" + "checksumValue": "a3f9bdc9e73f80eb4a586dc180246cfb8267ffc0" }, { "algorithm": "SHA256", - "checksumValue": "4b2ad9ea93fdd9c2fdc521fc4e14e02550666c2717a23b85819db2e07ea555f3" + "checksumValue": "2d5cae94382f5473cf6d2654760375ff1ee9cebdb8d4506b63ba33f488de1559" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA1.h" @@ -510,11 +510,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f732a6710fe3e13cd28130f0f20504e347d1c412" + "checksumValue": "fffe8c4f67669ac8ccd87c2e0f95db2427481df1" }, { "algorithm": "SHA256", - "checksumValue": "86cf32e4d1f3ba93a94108271923fdafe2204447792a918acf4a2250f352dbde" + "checksumValue": "f8af382de7e29a73c726f9c70770498ddd99e2c4702489ed6e634f0b68597c95" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" @@ -524,11 +524,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f38cebeeca40a83aeb2cf5dfce578ffefe176d84" + "checksumValue": "68b35d0c573f7301ba4d6919b1680d55675a0d98" }, { "algorithm": "SHA256", - "checksumValue": "ee03bf9368d1a3a3c70cfd4e9391b2485466404db4a60bfc5319630cc314b590" + "checksumValue": "442d8997d2bcda20ddf628665e2e69945400e1ab3019bc14fc7c8e20db20c320" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA2.h" @@ -538,11 +538,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "50f75337b31f509b5bfcc7ebb3d066b82a0f1b33" + "checksumValue": "77d3d879dfa5949030bca0e8ee75d3d369ec54a7" }, { "algorithm": "SHA256", - "checksumValue": "c9e1442899e5b902fa39f413f1a3131f7ab5c2283d5100dc8ac675a7d5ebbdf1" + "checksumValue": "8744f5b6e054c3e5c44f413a60f9154b76dd7230135dcee26fd063755ec64be1" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" @@ -552,11 +552,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "01717207aef77174e328186d48c27517f6644c15" + "checksumValue": "db1008095b64b2f8ee2a4ce72fa7cfc4a622a108" }, { "algorithm": "SHA256", - "checksumValue": "620dded172e94cb3f25f9904b44977d91f2cc9573e41b38f19e929d083ae0308" + "checksumValue": "961a686186b76a1cd6a64bcfd06afdc8657b1f901bac991166f498ed16f9349a" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA3.h" @@ -566,11 +566,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8140310f505bb2619a749777a91487d666237bcf" + "checksumValue": "eb224e26bc0503a48c88c837e28bbc7a9878ba5c" }, { "algorithm": "SHA256", - "checksumValue": "9d95e6a651c22185d9b7c38f363d30159f810e6fcdc2208f29492837ed891e82" + "checksumValue": "29fcf948a3715e09cbbd434cebb6105f276236a6e4947d237b7bf06634bf2432" } ], "fileName": "Modules/_hacl/Hacl_Streaming_HMAC.c" @@ -580,11 +580,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "49523144583a15d96ba1646af02dc292e633bf8f" + "checksumValue": "2d6c600024275c780e8313e0a5ab506e025bb4d6" }, { "algorithm": "SHA256", - "checksumValue": "78345519bf6789264f6792b809ee97a9ecf7cb5829c674c61e2d99bfdfdc36fc" + "checksumValue": "6450aef92f507fb0a8a3086b1d2792e7fca07121580c24fdaedd1b32e9ad0a76" } ], "fileName": "Modules/_hacl/Hacl_Streaming_HMAC.h" @@ -594,11 +594,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "372448599774a98e5c5d083e91f301ed1c4b822e" + "checksumValue": "ed283b95ebb772b05bdf802b70dbb301ece84e91" }, { "algorithm": "SHA256", - "checksumValue": "95d8e70ca4bc6aa98f6d2435ceb6410ead299b1f700fae1f5c603ec3f57ea551" + "checksumValue": "efc7bd11460744768425aedf4e004d3ad3397a5489752b80044ae785f30b78d4" } ], "fileName": "Modules/_hacl/Hacl_Streaming_Types.h" @@ -608,11 +608,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c9651ef21479c4d8a3b04c5baa1902866dbb1cdf" + "checksumValue": "0a0b7f3714167ad45ddf5a6a48d76f525a119c9c" }, { "algorithm": "SHA256", - "checksumValue": "e039c82ba670606ca111573942baad800f75da467abbc74cd7d1fe175ebcdfaf" + "checksumValue": "135d4afb4812468885c963c9c87a55ba5fae9181df4431af5fbad08588dda229" } ], "fileName": "Modules/_hacl/Lib_Memzero0.c" @@ -622,11 +622,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "eaa543c778300238dc23034aafeada0951154af1" + "checksumValue": "ae06c415ae7e0e1e85558863f29d1b9013e46e9b" }, { "algorithm": "SHA256", - "checksumValue": "3fd2552d527a23110d61ad2811c774810efb1eaee008f136c2a0d609daa77f5b" + "checksumValue": "69dc2e78411e9b271100eb0d2a2d8dc39dd2348afc26f8b4cfba14c025102c7f" } ], "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" @@ -636,11 +636,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "41ee1e34ede7ef5b24b87d4ca816fd6d9fac8010" + "checksumValue": "edefe48ced707327fd6cdf3f18b3ca42dda9a9f7" }, { "algorithm": "SHA256", - "checksumValue": "d48ed03e504cb87793a310a9552fb3ba2ebd6fe90127b7d642c8740fba1b9748" + "checksumValue": "f01e3f0892935f3f659019875f580445b9ea23482afbe11ffbe7cdbd22dbd4d2" } ], "fileName": "Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h" @@ -678,11 +678,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e01d7d493fbaceeedc4b1c6451d8240bcb9c903a" + "checksumValue": "7a943fbb8f55729a960f9b426e9ff2794453aca9" }, { "algorithm": "SHA256", - "checksumValue": "c2f0a43884771f24d7cb744b79818b160020d2739b2881b2054cfc97fb2e7b4a" + "checksumValue": "08bb43ef5626a8450f985da0af345b6658ac8bcc4585f77271decd0e41fc02e0" } ], "fileName": "Modules/_hacl/include/krml/internal/target.h" @@ -692,11 +692,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "3f66313d16891f43b21c1a736081c2c6d46bf370" + "checksumValue": "b4e6c5fc5e17864cc2bf452db2688be733d6df72" }, { "algorithm": "SHA256", - "checksumValue": "78e9bff9124968108e1699e1c6388e3d4ec9bd72dd8adff49734a69ab380ee5c" + "checksumValue": "404691cf9b2269ecc754ceca27bb8f8f029f1c8deaa4d967990dcf5ce08cd016" } ], "fileName": "Modules/_hacl/include/krml/internal/types.h" @@ -706,11 +706,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e18efc9239a5df0f222b5f7b0a65f72509d7e304" + "checksumValue": "8c3e09e00459951e060ad469c1fa31eeb9b6b9cb" }, { "algorithm": "SHA256", - "checksumValue": "47dd5a7d21b5302255f9fff28884f65d3056fc3f54471ed62ec85fa1904f8aa5" + "checksumValue": "6d1a350ec272c83761f1789611d8f60947a4d69fed3d23d8eee38709a0ad2c0a" } ], "fileName": "Modules/_hacl/include/krml/lowstar_endianness.h" @@ -720,11 +720,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "aaa656e25a92ba83655e1398a97efa6981f60fc4" + "checksumValue": "c9517c43046de129f02da8b74e454bade4872c00" }, { "algorithm": "SHA256", - "checksumValue": "a59abc6e9b3019cb18976a15e634f5146bd965fc9babf4ccbf2b531164a34f85" + "checksumValue": "96c2a973bc01b1295f7af67c7de3839facfeee36c8848d7d88c62695de98ab21" } ], "fileName": "Modules/_hacl/internal/Hacl_HMAC.h" @@ -734,11 +734,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0741cb8497309d648428be1e7b5944b1fc167187" + "checksumValue": "1c5eeb5d1866acb6bc8b4e5a01a107e3a87f5694" }, { "algorithm": "SHA256", - "checksumValue": "f9b923a566d62de047c753637143d439ca1c25221c08352ddc1738ff4a6ac721" + "checksumValue": "f262738390f56e8e9692acadecd1434c688075047d788283e1fb45d920ea6956" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2b.h" @@ -748,11 +748,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "878ae284c93824b80b1763e8b3e6be3c410777a8" + "checksumValue": "b600c1b5eafc34b29a778186737d8a51e2342d85" }, { "algorithm": "SHA256", - "checksumValue": "49df6223f6403daf503a1af1a3d2f943d30b5889fe7ed20299c3df24c1e3853d" + "checksumValue": "f22e088ad9c2eb739aefc3685ef3ab1ccaab3c5ef2e5d06cc846ad9f8e3d2669" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h" @@ -762,11 +762,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "25552d8cbf8aa345907635b38f284eec9075301e" + "checksumValue": "eb618ecc6ca2830066448f5f6d4df84a5c09f0f4" }, { "algorithm": "SHA256", - "checksumValue": "a3424cf4c5518654908086bbbf5d465715ec3b23625ef0cadc29492d1f90366c" + "checksumValue": "a4728e43deb0a9d8213b8ddcbda68a63bc73a12fb99aad54b7d28b314776a0d4" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2s.h" @@ -776,11 +776,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "54a712fc3ed5a817288351cbac5b7d9afa7e379f" + "checksumValue": "5441b6a97cc053c332b29477238d70fa011c74ac" }, { "algorithm": "SHA256", - "checksumValue": "c6abae648b8a1e9d5631c0a959620cad1f7e92ce522e07c3416199fe51debef6" + "checksumValue": "dad568d256a2ccbbbcdd419fe0543ea7137d8065a713a5f009aa52521c0f7f6a" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h" @@ -790,11 +790,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c15c5f83bbb9f62611c49f0f8f723eaab1a27488" + "checksumValue": "7081b58f28568f600d4624dd5bd6f735191e068b" }, { "algorithm": "SHA256", - "checksumValue": "95cd5d91c4a9217901d0b3395dcd8881e62e2055d723b532ec5176386a636d22" + "checksumValue": "305af1422213ed97e8e5d3d8a9dfee4f21cb2fcd2acf65ee7303ce00b2b4bd2a" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_MD5.h" @@ -804,11 +804,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7b8717e3a24e7e16a34b251d0d02da6f68439695" + "checksumValue": "b6fb6219cb40d039e789666e34b43461e3b5c82a" }, { "algorithm": "SHA256", - "checksumValue": "9473d8bc9506fe0053d7d98c225d4873011329863f1c4a8e93e43fc71bd1f314" + "checksumValue": "63c58363ff95e8146d5628dab2da25bc2fd0e8b590fd4823b512c33f843355bb" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA1.h" @@ -818,11 +818,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e319c949f5a2dd765be2c8c7ff77bfe52ee6c7da" + "checksumValue": "bcc71e702df1070cb0081cb983aec7683e036719" }, { "algorithm": "SHA256", - "checksumValue": "75261448e51c3eb1ba441e973b193e23570b167f67743942ee2ee57417491c9f" + "checksumValue": "709c6272f77e2368f6cc0bf36527e3b16dd5d5f3dc26a2afdef8238200af8770" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" @@ -832,11 +832,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "dbd92415c31606804102b79d5ba3d1752fe03887" + "checksumValue": "e3b5e6add5357760554b032b818992ce9bc211e0" }, { "algorithm": "SHA256", - "checksumValue": "5d74a76a0ac3659a1ae1276c3ca55521f09e83d2f0039f5c519a76f8f3c76a8e" + "checksumValue": "cf49d536a5663379fb4517018b4b3f8a232f8d4c7ddc7edfd60163d13f98a530" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA3.h" @@ -846,11 +846,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ad788265f8e1b078c4d1cb6e90b8c031590e6baf" + "checksumValue": "09a0ea364fb073f5f622a763f1351fbf4bac4627" }, { "algorithm": "SHA256", - "checksumValue": "d8354a9b75e2470085fa7e538493130e81fa23a804a6a69d34da8fdcc941c038" + "checksumValue": "916d54d7217517f0360edea920dc21585fbe6d1c2458eac86826182e12c82ec2" } ], "fileName": "Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h" @@ -860,11 +860,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2048f3cd61dbda2df862a2982ebaf24b6815ed51" + "checksumValue": "b082645b43f8841db5e9e57c0b9a3825fa7e52f5" }, { "algorithm": "SHA256", - "checksumValue": "b0f5a79c98525b0cb1659238e095641328b7da16a94cb57a0793e635d1da3653" + "checksumValue": "059cf0d31427abf84c626797a97c140630a1a6ead578005162f46fad46663389" } ], "fileName": "Modules/_hacl/internal/Hacl_Streaming_HMAC.h" @@ -874,11 +874,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4e6b098e89fd447bd03f47b55208208456b20966" + "checksumValue": "1f85ed69e395829b3ac5af9e2d049af7708cb9cb" }, { "algorithm": "SHA256", - "checksumValue": "d54d947968ca125978d61fea844711b990f0a18ab0fbca87e41029004d9d04b6" + "checksumValue": "76997c7069a347ac78b65d26354a0324d54add4ca7a02e7be36d6d5e1c9702e0" } ], "fileName": "Modules/_hacl/internal/Hacl_Streaming_Types.h" @@ -930,11 +930,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4a0bdb9496d49bbfa3ad50bb7854d8f099e84891" + "checksumValue": "682b069d3888f3e8f8cc90f1a49bac9d1a5903d2" }, { "algorithm": "SHA256", - "checksumValue": "b1a45149239ee7af7de769a3e9339950d47c199bb9eaa10edce8a00fde603b12" + "checksumValue": "8551d2c3fde03b92c6fab5febde00347bd9184ea0085077976863c7836e9669d" } ], "fileName": "Modules/_hacl/python_hacl_namespaces.h" @@ -944,11 +944,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0fbc026a9771d9675e7094790b5b945334d3cb53" + "checksumValue": "0d83ed429ede0a2c6617cd2f24490be16d8393f4" }, { "algorithm": "SHA256", - "checksumValue": "1e77c01eec8f167ed10b754f153c0c743c8e5196ae9c81dffc08f129ab56dbfd" + "checksumValue": "876f4486d08d3eac3581a8681d7fb3cec2fe5b823d1bef27cca15e755510365c" } ], "fileName": "Lib/ctypes/macholib/__init__.py" @@ -1730,14 +1730,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "17aa6cfc5c4c219c09287abfc10bc13f0c06f30bb654b28bfe6f567ca646eb79" + "checksumValue": "821ac9710d2c073eaf13e1b1895a9c9aa66c1157a99635c639fbff65cdbdd732" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_3/expat-2.6.3.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_7_3/expat-2.7.3.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.3:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.7.3:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,21 +1745,21 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.6.3" + "versionInfo": "2.7.3" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "02dfcf0c79d488b120d7f2c2a0f9206301c7927ed5106545e0b6f2aef88da76a" + "checksumValue": "61e48893f37cb2280d106cefacf6fb5afe84edf625fec39572d0ee94e1018f26" } ], - "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/7720f6d4fc0468a99d5ea6120976bcc271e42727.zip", + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/8ba599b2f6c9701b3dc961db895b0856a2210f76.zip", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:7720f6d4fc0468a99d5ea6120976bcc271e42727:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:8ba599b2f6c9701b3dc961db895b0856a2210f76:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1767,7 +1767,7 @@ "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", - "versionInfo": "7720f6d4fc0468a99d5ea6120976bcc271e42727" + "versionInfo": "8ba599b2f6c9701b3dc961db895b0856a2210f76" }, { "SPDXID": "SPDXRef-PACKAGE-macholib", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index d3e1f0db057..31d22e64b84 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -888,6 +888,7 @@ added = '3.2' [function.PyImport_ImportModuleNoBlock] added = '3.2' + abi_only = true [function.PyImport_ReloadModule] added = '3.2' [function.PyInterpreterState_Clear] @@ -1334,6 +1335,7 @@ abi_only = true [function.PySys_ResetWarnOptions] added = '3.2' + abi_only = true [function.PySys_SetArgv] added = '3.2' [function.PySys_SetArgvEx] @@ -1462,14 +1464,18 @@ added = '3.2' [function.PyUnicode_AsDecodedObject] added = '3.2' + abi_only = true [function.PyUnicode_AsDecodedUnicode] added = '3.2' + abi_only = true [function.PyUnicode_AsEncodedObject] added = '3.2' + abi_only = true [function.PyUnicode_AsEncodedString] added = '3.2' [function.PyUnicode_AsEncodedUnicode] added = '3.2' + abi_only = true [function.PyUnicode_AsLatin1String] added = '3.2' [function.PyUnicode_AsRawUnicodeEscapeString] @@ -1594,6 +1600,7 @@ added = '3.2' [function.PyWeakref_GetObject] added = '3.2' + abi_only = true [function.PyWeakref_NewProxy] added = '3.2' [function.PyWeakref_NewRef] @@ -1632,18 +1639,24 @@ added = '3.2' [function.Py_GetExecPrefix] added = '3.2' + abi_only = true [function.Py_GetPath] added = '3.2' + abi_only = true [function.Py_GetPlatform] added = '3.2' [function.Py_GetPrefix] added = '3.2' + abi_only = true [function.Py_GetProgramFullPath] added = '3.2' + abi_only = true [function.Py_GetProgramName] added = '3.2' + abi_only = true [function.Py_GetPythonHome] added = '3.2' + abi_only = true [function.Py_GetRecursionLimit] added = '3.2' [function.Py_GetVersion] @@ -1889,6 +1902,13 @@ added = '3.5' [data.PyModuleDef_Type] added = '3.5' +[const.Py_mod_create] + added = '3.5' +[const.Py_mod_exec] + added = '3.5' +[struct.PyModuleDef_Slot] + added = '3.5' + struct_abi_kind = 'full-abi' # New slots in 3.5: # d51374ed78a3e3145911a16cdf3b9b84b3ba7d15 - Matrix multiplication (PEP 465) @@ -2293,6 +2313,10 @@ added = '3.11' [function.PyMemoryView_FromBuffer] added = '3.11' +[const.Py_bf_getbuffer] + added = '3.11' +[const.Py_bf_releasebuffer] + added = '3.11' # Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) # (they were available with 3.11.0) @@ -2427,6 +2451,9 @@ added = '3.12' [const.Py_TPFLAGS_ITEMS_AT_END] added = '3.12' +[const.Py_mod_multiple_interpreters] + added = '3.12' + [function.PyImport_AddModuleRef] added = '3.13' [function.PyWeakref_GetRef] @@ -2505,10 +2532,14 @@ added = '3.13' [function.PyEval_GetFrameLocals] added = '3.13' +[const.Py_mod_gil] + added = '3.13' [function.Py_TYPE] + # Before 3.14, this was a macro that accessed the PyObject member added = '3.14' [function.Py_REFCNT] + # Before 3.14, this was a macro that accessed the PyObject member added = '3.14' [function.PyIter_NextItem] added = '3.14' @@ -2564,3 +2595,72 @@ added = '3.14' [function.Py_PACK_VERSION] added = '3.14' + +[function.PySys_GetAttr] + added = '3.15' +[function.PySys_GetAttrString] + added = '3.15' +[function.PySys_GetOptionalAttr] + added = '3.15' +[function.PySys_GetOptionalAttrString] + added = '3.15' +[function.PyABIInfo_Check] + added = '3.15' +[macro.PyABIInfo_VAR] + added = '3.15' +[struct.PyABIInfo] + added = '3.15' + struct_abi_kind = 'full-abi' +[const.Py_mod_abi] + added = '3.15' +[const.PyABIInfo_DEFAULT_ABI_VERSION] + added = '3.15' +[const.PyABIInfo_DEFAULT_FLAGS] + added = '3.15' +[const.PyABIInfo_STABLE] + added = '3.15' +[const.PyABIInfo_FREETHREADED] + added = '3.15' +[const.PyABIInfo_GIL] + added = '3.15' +[const.PyABIInfo_FREETHREADING_AGNOSTIC] + added = '3.15' +[function.PyModule_FromSlotsAndSpec] + added = '3.15' +[function.PyModule_Exec] + added = '3.15' +[function.PyModule_GetToken] + added = '3.15' +[function.PyType_GetModuleByToken] + added = '3.15' +[function.PyModule_GetStateSize] + added = '3.15' +[macro.PyMODEXPORT_FUNC] + added = '3.15' +[const.Py_mod_name] + added = '3.15' +[const.Py_mod_doc] + added = '3.15' +[const.Py_mod_state_size] + added = '3.15' +[const.Py_mod_methods] + added = '3.15' +[const.Py_mod_state_traverse] + added = '3.15' +[const.Py_mod_state_clear] + added = '3.15' +[const.Py_mod_state_free] + added = '3.15' +[const.Py_mod_token] + added = '3.15' +[function.PyDict_SetDefaultRef] + added = '3.15' +[function.Py_SIZE] + # Before 3.15, this was a macro that accessed the PyObject member + added = '3.15' +[function.Py_IS_TYPE] + # Before 3.15, this was a macro that accessed the PyObject member + added = '3.15' +[function.Py_SET_SIZE] + # Before 3.15, this was a macro that accessed the PyObject member + added = '3.15' diff --git a/Modules/Setup b/Modules/Setup index a066982df1a..7d816ead843 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -156,6 +156,7 @@ PYTHONPATH=$(COREPYTHONPATH) #binascii binascii.c #cmath cmathmodule.c #math mathmodule.c +#_math_integer mathintegermodule.c #mmap mmapmodule.c #select selectmodule.c #_sysconfig _sysconfig.c @@ -284,7 +285,7 @@ PYTHONPATH=$(COREPYTHONPATH) #*shared* #_ctypes_test _ctypes/_ctypes_test.c -#_remote_debugging _remote_debugging_module.c +#_remote_debugging _remote_debugging/module.c _remote_debugging/object_reading.c _remote_debugging/code_objects.c _remote_debugging/frames.c _remote_debugging/threads.c _remote_debugging/asyncio.c #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 2b2e8cb3e3c..65a1fefe72e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -12,6 +12,8 @@ posix posixmodule.c _signal signalmodule.c _tracemalloc _tracemalloc.c _suggestions _suggestions.c +# needs libm and on some platforms librt +_datetime _datetimemodule.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig _codecs _codecsmodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 3a38a60a152..1be83b45526 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -32,16 +32,16 @@ ############################################################################ # Modules that should always be present (POSIX and Windows): @MODULE_ARRAY_TRUE@array arraymodule.c -@MODULE__ASYNCIO_TRUE@_asyncio _asynciomodule.c @MODULE__BISECT_TRUE@_bisect _bisectmodule.c @MODULE__CSV_TRUE@_csv _csv.c @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c @MODULE__JSON_TRUE@_json _json.c @MODULE__LSPROF_TRUE@_lsprof _lsprof.c rotatingtree.c +@MODULE__MATH_INTEGER_TRUE@_math_integer mathintegermodule.c @MODULE__PICKLE_TRUE@_pickle _pickle.c @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c -@MODULE__REMOTE_DEBUGGING_TRUE@_remote_debugging _remote_debugging_module.c +@MODULE__REMOTE_DEBUGGING_TRUE@_remote_debugging _remote_debugging/module.c _remote_debugging/object_reading.c _remote_debugging/code_objects.c _remote_debugging/frames.c _remote_debugging/frame_cache.c _remote_debugging/threads.c _remote_debugging/asyncio.c @MODULE__STRUCT_TRUE@_struct _struct.c # build supports subinterpreters @@ -56,9 +56,6 @@ @MODULE_CMATH_TRUE@cmath cmathmodule.c @MODULE__STATISTICS_TRUE@_statistics _statisticsmodule.c -# needs libm and on some platforms librt -@MODULE__DATETIME_TRUE@_datetime _datetimemodule.c - # _decimal uses libmpdec # either static libmpdec.a from Modules/_decimal/libmpdec or libmpdec.so # with ./configure --with-system-libmpdec @@ -178,7 +175,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c @@ -193,3 +190,9 @@ # Limited API template modules; must be built as shared modules. @MODULE_XXLIMITED_TRUE@xxlimited xxlimited.c @MODULE_XXLIMITED_35_TRUE@xxlimited_35 xxlimited_35.c + + +# for performance +*static* + +@MODULE__ASYNCIO_TRUE@_asyncio _asynciomodule.c diff --git a/Modules/_abc.c b/Modules/_abc.c index d6a953b3360..f87a5c70294 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -484,6 +484,7 @@ compute_abstract_methods(PyObject *self) #define COLLECTION_FLAGS (Py_TPFLAGS_SEQUENCE | Py_TPFLAGS_MAPPING) /*[clinic input] +@permit_long_summary _abc._abc_init self: object @@ -494,7 +495,7 @@ Internal ABC helper for class set-up. Should be never used outside abc module. static PyObject * _abc__abc_init(PyObject *module, PyObject *self) -/*[clinic end generated code: output=594757375714cda1 input=8d7fe470ff77f029]*/ +/*[clinic end generated code: output=594757375714cda1 input=0b3513f947736d39]*/ { _abcmodule_state *state = get_abc_state(module); PyObject *data; @@ -543,6 +544,7 @@ _abc__abc_init(PyObject *module, PyObject *self) } /*[clinic input] +@permit_long_summary _abc._abc_register self: object @@ -554,7 +556,7 @@ Internal ABC helper for subclasss registration. Should be never used outside abc static PyObject * _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) -/*[clinic end generated code: output=7851e7668c963524 input=ca589f8c3080e67f]*/ +/*[clinic end generated code: output=7851e7668c963524 input=135ab13a581b4414]*/ { if (!PyType_Check(subclass)) { PyErr_SetString(PyExc_TypeError, "Can only register classes"); @@ -606,6 +608,7 @@ _abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass) /*[clinic input] +@permit_long_summary _abc._abc_instancecheck self: object @@ -618,7 +621,7 @@ Internal ABC helper for instance checks. Should be never used outside abc module static PyObject * _abc__abc_instancecheck_impl(PyObject *module, PyObject *self, PyObject *instance) -/*[clinic end generated code: output=b8b5148f63b6b56f input=a4f4525679261084]*/ +/*[clinic end generated code: output=b8b5148f63b6b56f input=0bbc8da0ea346719]*/ { PyObject *subtype, *result = NULL, *subclass = NULL; _abc_data *impl = _get_impl(module, self); @@ -692,6 +695,7 @@ static int subclasscheck_check_registry(_abc_data *impl, PyObject *subclass, PyObject **result); /*[clinic input] +@permit_long_summary _abc._abc_subclasscheck self: object @@ -704,7 +708,7 @@ Internal ABC helper for subclasss checks. Should be never used outside abc modul static PyObject * _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, PyObject *subclass) -/*[clinic end generated code: output=b56c9e4a530e3894 input=1d947243409d10b8]*/ +/*[clinic end generated code: output=b56c9e4a530e3894 input=5bf1ef712f5d3610]*/ { if (!PyType_Check(subclass)) { PyErr_SetString(PyExc_TypeError, "issubclass() arg 1 must be a class"); diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 5f9181395c4..9b2b7011244 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -119,7 +119,7 @@ typedef struct _Py_AsyncioModuleDebugOffsets { } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; -GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _AsyncioDebug) +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _Py_AsyncioDebug) = {.asyncio_task_object = { .size = sizeof(TaskObj), .task_name = offsetof(TaskObj, task_name), @@ -821,7 +821,7 @@ future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, Invariants: * callbacks != NULL: - There are some callbacks in in the list. Just + There are some callbacks in the list. Just add the new callback to it. * callbacks == NULL and callback0 == NULL: @@ -4079,30 +4079,44 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) return NULL; } - PyInterpreterState *interp = PyInterpreterState_Get(); - // Stop the world and traverse the per-thread linked list - // of asyncio tasks for every thread, as well as the - // interpreter's linked list, and add them to `tasks`. - // The interpreter linked list is used for any lingering tasks - // whose thread state has been deallocated while the task was - // still alive. This can happen if a task is referenced by - // a different thread, in which case the task is moved to - // the interpreter's linked list from the thread's linked - // list before deallocation. See PyThreadState_Clear. - // - // The stop-the-world pause is required so that no thread - // modifies its linked list while being iterated here - // in parallel. This design allows for lock-free - // register_task/unregister_task for loops running in parallel - // in different threads (the general case). - _PyEval_StopTheWorld(interp); - int ret = add_tasks_interp(interp, (PyListObject *)tasks); - _PyEval_StartTheWorld(interp); - if (ret < 0) { - // call any escaping calls after starting the world to avoid any deadlocks. - Py_DECREF(tasks); - Py_DECREF(loop); - return NULL; + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (ts->asyncio_running_loop == loop) { + // Fast path for the current running loop of current thread + // no locking or stop the world pause is required + struct llist_node *head = &ts->asyncio_tasks_head; + if (add_tasks_llist(head, (PyListObject *)tasks) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + } + else { + // Slow path for loop running in different thread + PyInterpreterState *interp = ts->base.interp; + // Stop the world and traverse the per-thread linked list + // of asyncio tasks for every thread, as well as the + // interpreter's linked list, and add them to `tasks`. + // The interpreter linked list is used for any lingering tasks + // whose thread state has been deallocated while the task was + // still alive. This can happen if a task is referenced by + // a different thread, in which case the task is moved to + // the interpreter's linked list from the thread's linked + // list before deallocation. See PyThreadState_Clear. + // + // The stop-the-world pause is required so that no thread + // modifies its linked list while being iterated here + // in parallel. This design allows for lock-free + // register_task/unregister_task for loops running in parallel + // in different threads (the general case). + _PyEval_StopTheWorld(interp); + int ret = add_tasks_interp(interp, (PyListObject *)tasks); + _PyEval_StartTheWorld(interp); + if (ret < 0) { + // call any escaping calls after starting the world to avoid any deadlocks. + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } } // All the tasks are now in the list, now filter the tasks which are done @@ -4324,7 +4338,7 @@ module_init(asyncio_state *state) goto fail; } - state->debug_offsets = &_AsyncioDebug; + state->debug_offsets = &_Py_AsyncioDebug; Py_DECREF(module); return 0; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9b146265445..3a1491e5b96 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -57,10 +57,6 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t Py_ssize_t mid; int res; - if (lo < 0) { - PyErr_SetString(PyExc_ValueError, "lo must be non-negative"); - return -1; - } if (hi == -1) { hi = PySequence_Size(list); if (hi < 0) @@ -153,7 +149,7 @@ _bisect.bisect_right -> Py_ssize_t a: object x: object - lo: Py_ssize_t = 0 + lo: Py_ssize_t(allow_negative=False) = 0 hi: Py_ssize_t(c_default='-1', accept={int, NoneType}) = None * key: object = None @@ -173,7 +169,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_right_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=3a4bc09cc7c8a73d input=43071869772dd53a]*/ +/*[clinic end generated code: output=3a4bc09cc7c8a73d input=b476bc45667273ac]*/ { return internal_bisect_right(a, x, lo, hi, key); } @@ -183,7 +179,7 @@ _bisect.insort_right a: object x: object - lo: Py_ssize_t = 0 + lo: Py_ssize_t(allow_negative=False) = 0 hi: Py_ssize_t(c_default='-1', accept={int, NoneType}) = None * key: object = None @@ -201,7 +197,7 @@ A custom key function can be supplied to customize the sort order. static PyObject * _bisect_insort_right_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=ac3bf26d07aedda2 input=f60777d2b6ddb239]*/ +/*[clinic end generated code: output=ac3bf26d07aedda2 input=f2caa8abec0763e8]*/ { PyObject *result, *key_x; Py_ssize_t index; @@ -241,10 +237,6 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h Py_ssize_t mid; int res; - if (lo < 0) { - PyErr_SetString(PyExc_ValueError, "lo must be non-negative"); - return -1; - } if (hi == -1) { hi = PySequence_Size(list); if (hi < 0) @@ -338,7 +330,7 @@ _bisect.bisect_left -> Py_ssize_t a: object x: object - lo: Py_ssize_t = 0 + lo: Py_ssize_t(allow_negative=False) = 0 hi: Py_ssize_t(c_default='-1', accept={int, NoneType}) = None * key: object = None @@ -358,7 +350,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_left_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=70749d6e5cae9284 input=f29c4fe7f9b797c7]*/ +/*[clinic end generated code: output=70749d6e5cae9284 input=9b4d49b5ddecfad7]*/ { return internal_bisect_left(a, x, lo, hi, key); } @@ -369,7 +361,7 @@ _bisect.insort_left a: object x: object - lo: Py_ssize_t = 0 + lo: Py_ssize_t(allow_negative=False) = 0 hi: Py_ssize_t(c_default='-1', accept={int, NoneType}) = None * key: object = None @@ -387,7 +379,7 @@ A custom key function can be supplied to customize the sort order. static PyObject * _bisect_insort_left_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=b1d33e5e7ffff11e input=0a700a82edbd472c]*/ +/*[clinic end generated code: output=b1d33e5e7ffff11e input=ff85a79826e22f31]*/ { PyObject *result, *key_x; Py_ssize_t index; diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 9e85e0de42c..452b88dfed2 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -97,20 +97,11 @@ OutputBuffer_OnError(_BlocksOutputBuffer *buffer) #endif /* ! BZ_CONFIG_ERROR */ -#define ACQUIRE_LOCK(obj) do { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while (0) -#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) - - typedef struct { PyObject_HEAD bz_stream bzs; int flushed; - PyThread_type_lock lock; + PyMutex mutex; } BZ2Compressor; typedef struct { @@ -126,7 +117,7 @@ typedef struct { separately. Conversion and looping is encapsulated in decompress_buf() */ size_t bzs_avail_in_real; - PyThread_type_lock lock; + PyMutex mutex; } BZ2Decompressor; #define _BZ2Compressor_CAST(op) ((BZ2Compressor *)(op)) @@ -190,7 +181,7 @@ static PyObject * compress(BZ2Compressor *c, char *data, size_t len, int action) { PyObject *result; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; if (OutputBuffer_InitAndGrow(&buffer, -1, &c->bzs.next_out, &c->bzs.avail_out) < 0) { goto error; @@ -271,12 +262,12 @@ _bz2_BZ2Compressor_compress_impl(BZ2Compressor *self, Py_buffer *data) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); else result = compress(self, data->buf, data->len, BZ_RUN); - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -296,14 +287,14 @@ _bz2_BZ2Compressor_flush_impl(BZ2Compressor *self) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) PyErr_SetString(PyExc_ValueError, "Repeated call to flush()"); else { self->flushed = 1; result = compress(self, NULL, 0, BZ_FINISH); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -357,13 +348,7 @@ _bz2_BZ2Compressor_impl(PyTypeObject *type, int compresslevel) return NULL; } - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } - + self->mutex = (PyMutex){0}; self->bzs.opaque = NULL; self->bzs.bzalloc = BZ2_Malloc; self->bzs.bzfree = BZ2_Free; @@ -382,22 +367,13 @@ static void BZ2Compressor_dealloc(PyObject *op) { BZ2Compressor *self = _BZ2Compressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); BZ2_bzCompressEnd(&self->bzs); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject *)self); Py_DECREF(tp); } -static int -BZ2Compressor_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static PyMethodDef BZ2Compressor_methods[] = { _BZ2_BZ2COMPRESSOR_COMPRESS_METHODDEF _BZ2_BZ2COMPRESSOR_FLUSH_METHODDEF @@ -409,7 +385,6 @@ static PyType_Slot bz2_compressor_type_slots[] = { {Py_tp_methods, BZ2Compressor_methods}, {Py_tp_new, _bz2_BZ2Compressor}, {Py_tp_doc, (char *)_bz2_BZ2Compressor__doc__}, - {Py_tp_traverse, BZ2Compressor_traverse}, {0, 0} }; @@ -437,7 +412,7 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length) compare against max_length and PyBytes_GET_SIZE we declare it as signed */ PyObject *result; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; bz_stream *bzs = &d->bzs; if (OutputBuffer_InitAndGrow(&buffer, max_length, &bzs->next_out, &bzs->avail_out) < 0) { @@ -598,6 +573,7 @@ decompress(BZ2Decompressor *d, char *data, size_t len, Py_ssize_t max_length) } /*[clinic input] +@permit_long_docstring_body _bz2.BZ2Decompressor.decompress data: Py_buffer @@ -622,16 +598,16 @@ the unused_data attribute. static PyObject * _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=23e41045deb240a3 input=52e1ffc66a8ea624]*/ +/*[clinic end generated code: output=23e41045deb240a3 input=3703e78f91757655]*/ { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->eof) PyErr_SetString(PyExc_EOFError, "End of stream already reached"); else result = decompress(self, data->buf, data->len, max_length); - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -657,20 +633,12 @@ _bz2_BZ2Decompressor_impl(PyTypeObject *type) return NULL; } - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } - + self->mutex = (PyMutex){0}; self->needs_input = 1; self->bzs_avail_in_real = 0; self->input_buffer = NULL; self->input_buffer_size = 0; - self->unused_data = PyBytes_FromStringAndSize(NULL, 0); - if (self->unused_data == NULL) - goto error; + self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); bzerror = BZ2_bzDecompressInit(&self->bzs, 0, 0); if (catch_bz2_error(bzerror)) @@ -687,28 +655,19 @@ static void BZ2Decompressor_dealloc(PyObject *op) { BZ2Decompressor *self = _BZ2Decompressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); if(self->input_buffer != NULL) { PyMem_Free(self->input_buffer); } BZ2_bzDecompressEnd(&self->bzs); Py_CLEAR(self->unused_data); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject *)self); Py_DECREF(tp); } -static int -BZ2Decompressor_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static PyMethodDef BZ2Decompressor_methods[] = { _BZ2_BZ2DECOMPRESSOR_DECOMPRESS_METHODDEF {NULL} @@ -739,7 +698,6 @@ static PyType_Slot bz2_decompressor_type_slots[] = { {Py_tp_doc, (char *)_bz2_BZ2Decompressor__doc__}, {Py_tp_members, BZ2Decompressor_members}, {Py_tp_new, _bz2_BZ2Decompressor}, - {Py_tp_traverse, BZ2Decompressor_traverse}, {0, 0} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 7cf3f152eee..2f2edbb05ab 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -92,6 +92,7 @@ _codecs_unregister(PyObject *module, PyObject *search_function) } /*[clinic input] +@permit_long_summary _codecs.lookup encoding: str / @@ -101,7 +102,7 @@ Looks up a codec tuple in the Python codec registry and returns a CodecInfo obje static PyObject * _codecs_lookup_impl(PyObject *module, const char *encoding) -/*[clinic end generated code: output=9f0afa572080c36d input=3c572c0db3febe9c]*/ +/*[clinic end generated code: output=9f0afa572080c36d input=02227d5429491ab3]*/ { return _PyCodec_Lookup(encoding); } @@ -201,55 +202,50 @@ _codecs_escape_encode_impl(PyObject *module, PyObject *data, const char *errors) /*[clinic end generated code: output=4af1d477834bab34 input=8f4b144799a94245]*/ { - Py_ssize_t size; - Py_ssize_t newsize; - PyObject *v; - - size = PyBytes_GET_SIZE(data); + Py_ssize_t size = PyBytes_GET_SIZE(data); if (size > PY_SSIZE_T_MAX / 4) { PyErr_SetString(PyExc_OverflowError, "string is too large to encode"); return NULL; } - newsize = 4*size; - v = PyBytes_FromStringAndSize(NULL, newsize); + Py_ssize_t newsize = 4*size; - if (v == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(newsize); + if (writer == NULL) { return NULL; } - else { - Py_ssize_t i; - char c; - char *p = PyBytes_AS_STRING(v); + char *p = PyBytesWriter_GetData(writer); - for (i = 0; i < size; i++) { - /* There's at least enough room for a hex escape */ - assert(newsize - (p - PyBytes_AS_STRING(v)) >= 4); - c = PyBytes_AS_STRING(data)[i]; - if (c == '\'' || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = Py_hexdigits[(c & 0xf0) >> 4]; - *p++ = Py_hexdigits[c & 0xf]; - } - else - *p++ = c; + for (Py_ssize_t i = 0; i < size; i++) { + /* There's at least enough room for a hex escape */ + assert(newsize - (p - (char*)PyBytesWriter_GetData(writer)) >= 4); + + char c = PyBytes_AS_STRING(data)[i]; + if (c == '\'' || c == '\\') { + *p++ = '\\'; *p++ = c; } - *p = '\0'; - if (_PyBytes_Resize(&v, (p - PyBytes_AS_STRING(v)))) { - return NULL; + else if (c == '\t') { + *p++ = '\\'; *p++ = 't'; + } + else if (c == '\n') { + *p++ = '\\'; *p++ = 'n'; + } + else if (c == '\r') { + *p++ = '\\'; *p++ = 'r'; + } + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = Py_hexdigits[(c & 0xf0) >> 4]; + *p++ = Py_hexdigits[c & 0xf]; + } + else { + *p++ = c; } } - return codec_tuple(v, size); + PyObject *decoded = PyBytesWriter_FinishWithPointer(writer, p); + return codec_tuple(decoded, size); } /* --- Decoder ------------------------------------------------------------ */ @@ -675,7 +671,7 @@ _codecs_utf_7_encode_impl(PyObject *module, PyObject *str, const char *errors) /*[clinic end generated code: output=0feda21ffc921bc8 input=2546dbbb3fa53114]*/ { - return codec_tuple(_PyUnicode_EncodeUTF7(str, 0, 0, errors), + return codec_tuple(_PyUnicode_EncodeUTF7(str, errors), PyUnicode_GET_LENGTH(str)); } @@ -1022,6 +1018,47 @@ _codecs_lookup_error_impl(PyObject *module, const char *name) return PyCodec_LookupError(name); } +extern int _Py_normalize_encoding(const char *, char *, size_t, int); + +/*[clinic input] +_codecs._normalize_encoding + encoding: unicode + +Normalize an encoding name *encoding*. + +Used for encodings.normalize_encoding. Does not convert to lower case. +[clinic start generated code]*/ + +static PyObject * +_codecs__normalize_encoding_impl(PyObject *module, PyObject *encoding) +/*[clinic end generated code: output=d27465d81e361f8e input=3ff3f4d64995b988]*/ +{ + Py_ssize_t len; + const char *cstr = PyUnicode_AsUTF8AndSize(encoding, &len); + if (cstr == NULL) { + return NULL; + } + + if (len > PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, "encoding is too large"); + return NULL; + } + + char *normalized = PyMem_Malloc(len + 1); + if (normalized == NULL) { + return PyErr_NoMemory(); + } + + if (!_Py_normalize_encoding(cstr, normalized, len + 1, 0)) { + PyMem_Free(normalized); + return NULL; + } + + PyObject *result = PyUnicode_FromString(normalized); + PyMem_Free(normalized); + return result; +} + /* --- Module API --------------------------------------------------------- */ static PyMethodDef _codecs_functions[] = { @@ -1071,6 +1108,7 @@ static PyMethodDef _codecs_functions[] = { _CODECS_REGISTER_ERROR_METHODDEF _CODECS__UNREGISTER_ERROR_METHODDEF _CODECS_LOOKUP_ERROR_METHODDEF + _CODECS__NORMALIZE_ENCODING_METHODDEF {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ad670293ec5..3ba48d5d9d3 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> @@ -1532,9 +1533,7 @@ deque_dealloc(PyObject *self) Py_ssize_t i; PyObject_GC_UnTrack(deque); - if (deque->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, deque->weakreflist); if (deque->leftblock != NULL) { (void)deque_clear(self); assert(deque->leftblock != NULL); diff --git a/Modules/_csv.c b/Modules/_csv.c index e5ae853590b..1f41976e95f 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -237,7 +237,7 @@ _set_int(const char *name, int *target, PyObject *src, int dflt) int value; if (!PyLong_CheckExact(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be an integer", name); + "\"%s\" must be an integer, not %T", name, src); return -1; } value = PyLong_AsInt(src); @@ -255,27 +255,29 @@ _set_char_or_none(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt if (src == NULL) { *target = dflt; } - else { + else if (src == Py_None) { *target = NOT_SET; - if (src != Py_None) { - if (!PyUnicode_Check(src)) { - PyErr_Format(PyExc_TypeError, - "\"%s\" must be string or None, not %.200s", name, - Py_TYPE(src)->tp_name); - return -1; - } - Py_ssize_t len = PyUnicode_GetLength(src); - if (len < 0) { - return -1; - } - if (len != 1) { - PyErr_Format(PyExc_TypeError, - "\"%s\" must be a 1-character string", - name); - return -1; - } - *target = PyUnicode_READ_CHAR(src, 0); + } + else { + // similar to PyArg_Parse("C?") + if (!PyUnicode_Check(src)) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be a unicode character or None, not %T", + name, src); + return -1; } + Py_ssize_t len = PyUnicode_GetLength(src); + if (len < 0) { + return -1; + } + if (len != 1) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be a unicode character or None, " + "not a string of length %zd", + name, len); + return -1; + } + *target = PyUnicode_READ_CHAR(src, 0); } return 0; } @@ -287,11 +289,12 @@ _set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) *target = dflt; } else { + // similar to PyArg_Parse("C") if (!PyUnicode_Check(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be string, not %.200s", name, - Py_TYPE(src)->tp_name); - return -1; + "\"%s\" must be a unicode character, not %T", + name, src); + return -1; } Py_ssize_t len = PyUnicode_GetLength(src); if (len < 0) { @@ -299,8 +302,9 @@ _set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) } if (len != 1) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be a 1-character string", - name); + "\"%s\" must be a unicode character, " + "not a string of length %zd", + name, len); return -1; } *target = PyUnicode_READ_CHAR(src, 0); @@ -314,16 +318,12 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) if (src == NULL) *target = PyUnicode_DecodeASCII(dflt, strlen(dflt), NULL); else { - if (src == Py_None) - *target = NULL; - else if (!PyUnicode_Check(src)) { + if (!PyUnicode_Check(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be a string", name); + "\"%s\" must be a string, not %T", name, src); return -1; } - else { - Py_XSETREF(*target, Py_NewRef(src)); - } + Py_XSETREF(*target, Py_NewRef(src)); } return 0; } @@ -533,11 +533,6 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) /* validate options */ if (dialect_check_quoting(self->quoting)) goto err; - if (self->delimiter == NOT_SET) { - PyErr_SetString(PyExc_TypeError, - "\"delimiter\" must be a 1-character string"); - goto err; - } if (quotechar == Py_None && quoting == NULL) self->quoting = QUOTE_NONE; if (self->quoting != QUOTE_NONE && self->quotechar == NOT_SET) { @@ -545,10 +540,6 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "quotechar must be set if quoting enabled"); goto err; } - if (self->lineterminator == NULL) { - PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); - goto err; - } if (dialect_check_char("delimiter", self->delimiter, self, true) || dialect_check_char("escapechar", self->escapechar, self, !self->skipinitialspace) || @@ -927,7 +918,7 @@ parse_reset(ReaderObj *self) } static PyObject * -Reader_iternext(PyObject *op) +Reader_iternext_lock_held(PyObject *op) { ReaderObj *self = _ReaderObj_CAST(op); @@ -994,6 +985,16 @@ Reader_iternext(PyObject *op) return fields; } +static PyObject * +Reader_iternext(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = Reader_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + static void Reader_dealloc(PyObject *op) { @@ -1312,14 +1313,8 @@ join_append_lineterminator(WriterObj *self) return 1; } -PyDoc_STRVAR(csv_writerow_doc, -"writerow(iterable)\n" -"\n" -"Construct and write a CSV record from an iterable of fields. Non-string\n" -"elements will be converted to string."); - static PyObject * -csv_writerow(PyObject *op, PyObject *seq) +csv_writerow_lock_held(PyObject *op, PyObject *seq) { WriterObj *self = _WriterObj_CAST(op); DialectObj *dialect = self->dialect; @@ -1422,11 +1417,29 @@ csv_writerow(PyObject *op, PyObject *seq) return result; } -PyDoc_STRVAR(csv_writerows_doc, -"writerows(iterable of iterables)\n" +PyDoc_STRVAR(csv_writerow_doc, +"writerow($self, row, /)\n" +"--\n\n" +"Construct and write a CSV record from an iterable of fields.\n" "\n" -"Construct and write a series of iterables to a csv file. Non-string\n" -"elements will be converted to string."); +"Non-string elements will be converted to string."); + +static PyObject * +csv_writerow(PyObject *op, PyObject *seq) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = csv_writerow_lock_held(op, seq); + Py_END_CRITICAL_SECTION(); + return result; +} + +PyDoc_STRVAR(csv_writerows_doc, +"writerows($self, rows, /)\n" +"--\n\n" +"Construct and write a series of iterables to a csv file.\n" +"\n" +"Non-string elements will be converted to string."); static PyObject * csv_writerows(PyObject *self, PyObject *seqseq) @@ -1583,13 +1596,11 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args) _csv.list_dialects Return a list of all known dialect names. - - names = csv.list_dialects() [clinic start generated code]*/ static PyObject * _csv_list_dialects_impl(PyObject *module) -/*[clinic end generated code: output=a5b92b215b006a6d input=8953943eb17d98ab]*/ +/*[clinic end generated code: output=a5b92b215b006a6d input=ec58040aafd6a20a]*/ { return PyDict_Keys(get_csv_state(module)->dialects); } @@ -1626,13 +1637,11 @@ _csv.unregister_dialect name: object Delete the name/dialect mapping associated with a string name. - - csv.unregister_dialect(name) [clinic start generated code]*/ static PyObject * _csv_unregister_dialect_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=0813ebca6c058df4 input=6b5c1557bf60c7e7]*/ +/*[clinic end generated code: output=0813ebca6c058df4 input=e1cf81bfe3ba0f62]*/ { _csvstate *module_state = get_csv_state(module); int rc = PyDict_Pop(module_state->dialects, name, NULL); @@ -1652,13 +1661,11 @@ _csv.get_dialect name: object Return the dialect instance associated with name. - - dialect = csv.get_dialect(name) [clinic start generated code]*/ static PyObject * _csv_get_dialect_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=aa988cd573bebebb input=edf9ddab32e448fb]*/ +/*[clinic end generated code: output=aa988cd573bebebb input=74865c659dcb441f]*/ { return get_dialect_from_registry(name, get_csv_state(module)); } @@ -1670,15 +1677,13 @@ _csv.field_size_limit Sets an upper limit on parsed fields. - csv.field_size_limit([limit]) - Returns old limit. If limit is not given, no new limit is set and the old limit is returned [clinic start generated code]*/ static PyObject * _csv_field_size_limit_impl(PyObject *module, PyObject *new_limit) -/*[clinic end generated code: output=f2799ecd908e250b input=cec70e9226406435]*/ +/*[clinic end generated code: output=f2799ecd908e250b input=77db7485ee3ae90a]*/ { _csvstate *module_state = get_csv_state(module); Py_ssize_t old_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); @@ -1714,14 +1719,13 @@ PyType_Spec error_spec = { PyDoc_STRVAR(csv_module_doc, "CSV parsing and writing.\n"); PyDoc_STRVAR(csv_reader_doc, -" csv_reader = reader(iterable [, dialect='excel']\n" -" [optional keyword args])\n" -" for row in csv_reader:\n" -" process(row)\n" +"reader($module, iterable, /, dialect='excel', **fmtparams)\n" +"--\n\n" +"Return a reader object that will process lines from the given iterable.\n" "\n" "The \"iterable\" argument can be any object that returns a line\n" "of input for each iteration, such as a file object or a list. The\n" -"optional \"dialect\" parameter is discussed below. The function\n" +"optional \"dialect\" argument defines a CSV dialect. The function\n" "also accepts optional keyword arguments which override settings\n" "provided by the dialect.\n" "\n" @@ -1729,22 +1733,24 @@ PyDoc_STRVAR(csv_reader_doc, "of the CSV file (which can span multiple input lines).\n"); PyDoc_STRVAR(csv_writer_doc, -" csv_writer = csv.writer(fileobj [, dialect='excel']\n" -" [optional keyword args])\n" -" for row in sequence:\n" -" csv_writer.writerow(row)\n" +"writer($module, fileobj, /, dialect='excel', **fmtparams)\n" +"--\n\n" +"Return a writer object that will write user data on the given file object.\n" "\n" -" [or]\n" -"\n" -" csv_writer = csv.writer(fileobj [, dialect='excel']\n" -" [optional keyword args])\n" -" csv_writer.writerows(rows)\n" -"\n" -"The \"fileobj\" argument can be any object that supports the file API.\n"); +"The \"fileobj\" argument can be any object that supports the file API.\n" +"The optional \"dialect\" argument defines a CSV dialect. The function\n" +"also accepts optional keyword arguments which override settings\n" +"provided by the dialect.\n"); PyDoc_STRVAR(csv_register_dialect_doc, -"Create a mapping from a string name to a dialect class.\n" -" dialect = csv.register_dialect(name[, dialect[, **fmtparams]])"); +"register_dialect($module, name, /, dialect='excel', **fmtparams)\n" +"--\n\n" +"Create a mapping from a string name to a CVS dialect.\n" +"\n" +"The optional \"dialect\" argument specifies the base dialect instance\n" +"or the name of the registered dialect. The function also accepts\n" +"optional keyword arguments which override settings provided by the\n" +"dialect.\n"); static struct PyMethodDef csv_methods[] = { { "reader", _PyCFunction_CAST(csv_reader), diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 1bb65e0a649..91fd23d413d 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -576,8 +576,15 @@ _ctypes_CType_Type___sizeof___impl(PyObject *self, PyTypeObject *cls) return PyLong_FromSsize_t(size); } +/*[clinic input] +@getter +_ctypes.CType_Type.__pointer_type__ + +[clinic start generated code]*/ + static PyObject * -ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) +_ctypes_CType_Type___pointer_type___get_impl(PyObject *self) +/*[clinic end generated code: output=718c9ff10b2b0012 input=ad12dc835943ceb8]*/ { ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *info; @@ -588,9 +595,12 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) PyErr_Format(PyExc_TypeError, "%R must have storage info", self); return NULL; } - - if (info->pointer_type) { - return Py_NewRef(info->pointer_type); + PyObject *pointer_type; + STGINFO_LOCK(info); + pointer_type = Py_XNewRef(info->pointer_type); + STGINFO_UNLOCK(); + if (pointer_type) { + return pointer_type; } PyErr_Format(PyExc_AttributeError, @@ -599,8 +609,15 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored)) return NULL; } +/*[clinic input] +@setter +_ctypes.CType_Type.__pointer_type__ + +[clinic start generated code]*/ + static int -ctype_set_pointer_type(PyObject *self, PyObject *tp, void *Py_UNUSED(ignored)) +_ctypes_CType_Type___pointer_type___set_impl(PyObject *self, PyObject *value) +/*[clinic end generated code: output=6259be8ea21693fa input=a05055fc7f4714b6]*/ { ctypes_state *st = get_module_state_by_def(Py_TYPE(self)); StgInfo *info; @@ -611,8 +628,9 @@ ctype_set_pointer_type(PyObject *self, PyObject *tp, void *Py_UNUSED(ignored)) PyErr_Format(PyExc_TypeError, "%R must have storage info", self); return -1; } - - Py_XSETREF(info->pointer_type, Py_XNewRef(tp)); + STGINFO_LOCK(info); + Py_XSETREF(info->pointer_type, Py_XNewRef(value)); + STGINFO_UNLOCK(); return 0; } @@ -626,8 +644,7 @@ static PyMethodDef ctype_methods[] = { }; static PyGetSetDef ctype_getsets[] = { - { "__pointer_type__", ctype_get_pointer_type, ctype_set_pointer_type, - "pointer type", NULL }, + _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF { NULL, NULL } }; @@ -842,7 +859,7 @@ _ctypes.CDataType.from_buffer as CDataType_from_buffer type: self cls: defining_class obj: object - offset: Py_ssize_t = 0 + offset: Py_ssize_t(allow_negative=False) = 0 / C.from_buffer(object, offset=0) -> C instance @@ -853,7 +870,7 @@ Create a C instance from a writeable buffer. static PyObject * CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, Py_ssize_t offset) -/*[clinic end generated code: output=57604e99635abd31 input=0f36cedd105ca28d]*/ +/*[clinic end generated code: output=57604e99635abd31 input=8f43e6bc44373180]*/ { PyObject *mv; PyObject *result; @@ -889,13 +906,6 @@ CDataType_from_buffer_impl(PyObject *type, PyTypeObject *cls, PyObject *obj, return NULL; } - if (offset < 0) { - PyErr_SetString(PyExc_ValueError, - "offset cannot be negative"); - Py_DECREF(mv); - return NULL; - } - if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small " @@ -938,7 +948,7 @@ _ctypes.CDataType.from_buffer_copy as CDataType_from_buffer_copy type: self cls: defining_class buffer: Py_buffer - offset: Py_ssize_t = 0 + offset: Py_ssize_t(allow_negative=False) = 0 / C.from_buffer_copy(object, offset=0) -> C instance @@ -949,7 +959,7 @@ Create a C instance from a readable buffer. static PyObject * CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, Py_buffer *buffer, Py_ssize_t offset) -/*[clinic end generated code: output=c8fc62b03e5cc6fa input=2a81e11b765a6253]*/ +/*[clinic end generated code: output=c8fc62b03e5cc6fa input=41f97f512295ceec]*/ { PyObject *result; @@ -963,12 +973,6 @@ CDataType_from_buffer_copy_impl(PyObject *type, PyTypeObject *cls, return NULL; } - if (offset < 0) { - PyErr_SetString(PyExc_ValueError, - "offset cannot be negative"); - return NULL; - } - if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small (%zd instead of at least %zd bytes)", @@ -1254,9 +1258,11 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb return -1; } Py_XSETREF(stginfo->proto, Py_NewRef(proto)); + STGINFO_LOCK(info); if (info->pointer_type == NULL) { Py_XSETREF(info->pointer_type, Py_NewRef(self)); } + STGINFO_UNLOCK(); return 0; } @@ -3591,6 +3597,48 @@ generic_pycdata_new(ctypes_state *st, PyCFuncPtr_Type */ +static inline void +atomic_xsetref(PyObject **field, PyObject *value) +{ +#ifdef Py_GIL_DISABLED + PyObject *old = *field; + _Py_atomic_store_ptr(field, value); + Py_XDECREF(old); +#else + Py_XSETREF(*field, value); +#endif +} +/* + This function atomically loads the reference from *field, and + tries to get a new reference to it. If the incref fails, + it acquires critical section of obj and returns a new reference to the *field. + In the general case, this avoids contention on acquiring the critical section. +*/ +static inline PyObject * +atomic_xgetref(PyObject *obj, PyObject **field) +{ +#ifdef Py_GIL_DISABLED + PyObject *value = _Py_atomic_load_ptr(field); + if (value == NULL) { + return NULL; + } + if (_Py_TryIncrefCompare(field, value)) { + return value; + } + Py_BEGIN_CRITICAL_SECTION(obj); + value = Py_XNewRef(*field); + Py_END_CRITICAL_SECTION(); + return value; +#else + return Py_XNewRef(*field); +#endif +} + +static int +_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags, PyObject *argtypes); + + + /*[clinic input] @critical_section @setter @@ -3607,7 +3655,7 @@ _ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value) return -1; } Py_XINCREF(value); - Py_XSETREF(self->errcheck, value); + atomic_xsetref(&self->errcheck, value); return 0; } @@ -3639,12 +3687,10 @@ static int _ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value) /*[clinic end generated code: output=0be0a086abbabf18 input=683c3bef4562ccc6]*/ { - PyObject *checker, *oldchecker; + PyObject *checker; if (value == NULL) { - oldchecker = self->checker; - self->checker = NULL; - Py_CLEAR(self->restype); - Py_XDECREF(oldchecker); + atomic_xsetref(&self->restype, NULL); + atomic_xsetref(&self->checker, NULL); return 0; } ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); @@ -3660,11 +3706,9 @@ _ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value) if (PyObject_GetOptionalAttr(value, &_Py_ID(_check_retval_), &checker) < 0) { return -1; } - oldchecker = self->checker; - self->checker = checker; Py_INCREF(value); - Py_XSETREF(self->restype, value); - Py_XDECREF(oldchecker); + atomic_xsetref(&self->checker, checker); + atomic_xsetref(&self->restype, value); return 0; } @@ -3706,19 +3750,25 @@ static int _ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value) /*[clinic end generated code: output=596a36e2ae89d7d1 input=c4627573e980aa8b]*/ { - PyObject *converters; - if (value == NULL || value == Py_None) { - Py_CLEAR(self->converters); - Py_CLEAR(self->argtypes); + atomic_xsetref(&self->argtypes, NULL); + atomic_xsetref(&self->converters, NULL); } else { - ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); - converters = converters_from_argtypes(st, value); + PyTypeObject *type = Py_TYPE(self); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + + PyObject *converters = converters_from_argtypes(st, value); if (!converters) return -1; - Py_XSETREF(self->converters, converters); + + /* Verify paramflags again due to constraints with argtypes */ + if (!_validate_paramflags(st, type, self->paramflags, value)) { + Py_DECREF(converters); + return -1; + } + atomic_xsetref(&self->converters, converters); Py_INCREF(value); - Py_XSETREF(self->argtypes, value); + atomic_xsetref(&self->argtypes, value); } return 0; } @@ -3845,10 +3895,9 @@ _check_outarg_type(ctypes_state *st, PyObject *arg, Py_ssize_t index) /* Returns 1 on success, 0 on error */ static int -_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) +_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags, PyObject *argtypes) { Py_ssize_t i, len; - PyObject *argtypes; StgInfo *info; if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { @@ -3859,10 +3908,13 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) "abstract class"); return 0; } - argtypes = info->argtypes; + if (argtypes == NULL) { + argtypes = info->argtypes; + } - if (paramflags == NULL || info->argtypes == NULL) + if (paramflags == NULL || argtypes == NULL) { return 1; + } if (!PyTuple_Check(paramflags)) { PyErr_SetString(PyExc_TypeError, @@ -3871,7 +3923,7 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) } len = PyTuple_GET_SIZE(paramflags); - if (len != PyTuple_GET_SIZE(info->argtypes)) { + if (len != PyTuple_GET_SIZE(argtypes)) { PyErr_SetString(PyExc_ValueError, "paramflags must have the same length as argtypes"); return 0; @@ -3883,7 +3935,9 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) PyObject *name = Py_None; PyObject *defval; PyObject *typ; - if (!PyArg_ParseTuple(item, "i|U?O", &flag, &name, &defval)) { + if (!PyArg_ParseTuple(item, "i|OO", &flag, &name, &defval) || + !(name == Py_None || PyUnicode_Check(name))) + { PyErr_SetString(PyExc_TypeError, "paramflags must be a sequence of (int [,string [,value]]) tuples"); return 0; @@ -3948,8 +4002,10 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) void *handle; PyObject *paramflags = NULL; - if (!PyArg_ParseTuple(args, "O|O?", &ftuple, &paramflags)) + if (!PyArg_ParseTuple(args, "O|O", &ftuple, &paramflags)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ftuple = PySequence_Tuple(ftuple); if (!ftuple) @@ -4043,7 +4099,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #endif #undef USE_DLERROR ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); - if (!_validate_paramflags(st, type, paramflags)) { + if (!_validate_paramflags(st, type, paramflags, NULL)) { Py_DECREF(ftuple); return NULL; } @@ -4081,11 +4137,13 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) GUID *iid = NULL; Py_ssize_t iid_len = 0; - if (!PyArg_ParseTuple(args, "is|O?z#", &index, &name, &paramflags, &iid, &iid_len)) + if (!PyArg_ParseTuple(args, "is|Oz#", &index, &name, &paramflags, &iid, &iid_len)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); - if (!_validate_paramflags(st, type, paramflags)) { + if (!_validate_paramflags(st, type, paramflags, NULL)) { return NULL; } self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); @@ -4514,16 +4572,11 @@ _build_result(PyObject *result, PyObject *callargs, } static PyObject * -PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) +PyCFuncPtr_call(PyObject *op, PyObject *inargs, PyObject *kwds) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - PyObject *restype; - PyObject *converters; - PyObject *checker; - PyObject *argtypes; - PyObject *result; - PyObject *callargs; - PyObject *errcheck; + PyObject *result = NULL; + PyObject *callargs = NULL; + PyObject *ret = NULL; #ifdef MS_WIN32 IUnknown *piunk = NULL; #endif @@ -4541,13 +4594,24 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) } assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ - restype = self->restype ? self->restype : info->restype; - converters = self->converters ? self->converters : info->converters; - checker = self->checker ? self->checker : info->checker; - argtypes = self->argtypes ? self->argtypes : info->argtypes; -/* later, we probably want to have an errcheck field in stginfo */ - errcheck = self->errcheck /* ? self->errcheck : info->errcheck */; - + PyObject *restype = atomic_xgetref(op, &self->restype); + if (restype == NULL) { + restype = Py_XNewRef(info->restype); + } + PyObject *converters = atomic_xgetref(op, &self->converters); + if (converters == NULL) { + converters = Py_XNewRef(info->converters); + } + PyObject *checker = atomic_xgetref(op, &self->checker); + if (checker == NULL) { + checker = Py_XNewRef(info->checker); + } + PyObject *argtypes = atomic_xgetref(op, &self->argtypes); + if (argtypes == NULL) { + argtypes = Py_XNewRef(info->argtypes); + } + /* later, we probably want to have an errcheck field in stginfo */ + PyObject *errcheck = atomic_xgetref(op, &self->errcheck); pProc = *(void **)self->b_ptr; #ifdef MS_WIN32 @@ -4558,25 +4622,25 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) if (!this) { PyErr_SetString(PyExc_ValueError, "native com method call without 'this' parameter"); - return NULL; + goto finally; } if (!CDataObject_Check(st, this)) { PyErr_SetString(PyExc_TypeError, "Expected a COM this pointer as first argument"); - return NULL; + goto finally; } /* there should be more checks? No, in Python */ /* First arg is a pointer to an interface instance */ if (!this->b_ptr || *(void **)this->b_ptr == NULL) { PyErr_SetString(PyExc_ValueError, "NULL COM pointer access"); - return NULL; + goto finally; } piunk = *(IUnknown **)this->b_ptr; if (NULL == piunk->lpVtbl) { PyErr_SetString(PyExc_ValueError, "COM method call without VTable"); - return NULL; + goto finally; } pProc = ((void **)piunk->lpVtbl)[self->index - 0x1000]; } @@ -4584,8 +4648,9 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) callargs = _build_callargs(st, self, argtypes, inargs, kwds, &outmask, &inoutmask, &numretvals); - if (callargs == NULL) - return NULL; + if (callargs == NULL) { + goto finally; + } if (converters) { int required = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(converters), @@ -4604,7 +4669,7 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) required, required == 1 ? "" : "s", actual); - return NULL; + goto finally; } } else if (required != actual) { Py_DECREF(callargs); @@ -4613,7 +4678,7 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) required, required == 1 ? "" : "s", actual); - return NULL; + goto finally; } } @@ -4644,23 +4709,19 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) if (v == NULL || v != callargs) { Py_DECREF(result); Py_DECREF(callargs); - return v; + ret = v; + goto finally; } Py_DECREF(v); } - - return _build_result(result, callargs, - outmask, inoutmask, numretvals); -} - -static PyObject * -PyCFuncPtr_call(PyObject *op, PyObject *inargs, PyObject *kwds) -{ - PyObject *result; - Py_BEGIN_CRITICAL_SECTION(op); - result = PyCFuncPtr_call_lock_held(op, inargs, kwds); - Py_END_CRITICAL_SECTION(); - return result; + ret = _build_result(result, callargs, outmask, inoutmask, numretvals); +finally: + Py_XDECREF(restype); + Py_XDECREF(converters); + Py_XDECREF(checker); + Py_XDECREF(argtypes); + Py_XDECREF(errcheck); + return ret; } static int diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index d28e5708b44..a0c9d8b70fe 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -23,7 +23,7 @@ # define _Py_thread_local __thread #endif -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) # include <complex.h> // csqrt() # undef I // for _ctypes_test_generated.c.h #endif @@ -457,7 +457,7 @@ EXPORT(double) my_sqrt(double a) return sqrt(a); } -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) EXPORT(double complex) my_csqrt(double complex a) { return csqrt(a); @@ -989,13 +989,10 @@ EXPORT(RECT) ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr, { case 0: return ar; - break; case 1: return dr; - break; case 2: return gr; - break; } return ar; diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index ec113e41d16..fd508ae61f2 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -11,18 +11,9 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_runtime.h" // _Py_ID() -#ifdef MS_WIN32 -# include <malloc.h> -#endif - #include <ffi.h> #include "ctypes.h" -#ifdef HAVE_ALLOCA_H -/* AIX needs alloca.h for alloca() */ -#include <alloca.h> -#endif - /**************************************************************/ static int diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 856b0376e5e..a8c16547e4b 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -77,22 +77,14 @@ module _ctypes #include <mach-o/dyld.h> #endif -#ifdef MS_WIN32 -#include <malloc.h> -#endif - #include <ffi.h> #include "ctypes.h" -#ifdef HAVE_ALLOCA_H -/* AIX needs alloca.h for alloca() */ -#include <alloca.h> -#endif #ifdef _Py_MEMORY_SANITIZER #include <sanitizer/msan_interface.h> #endif -#if defined(_DEBUG) || defined(__MINGW32__) +#if defined(Py_DEBUG) || defined(__MINGW32__) /* Don't use structured exception handling on Windows if this is defined. MingW, AFAIK, doesn't support it. */ @@ -1389,7 +1381,7 @@ static PyObject *format_error(PyObject *self, PyObject *args) code = GetLastError(); lpMsgBuf = FormatError(code); if (lpMsgBuf) { - result = PyUnicode_FromWideChar(lpMsgBuf, wcslen(lpMsgBuf)); + result = PyUnicode_FromWideChar(lpMsgBuf, -1); LocalFree(lpMsgBuf); } else { result = PyUnicode_FromString("<no description>"); @@ -1795,6 +1787,7 @@ align_func(PyObject *self, PyObject *obj) /*[clinic input] +@permit_long_summary @critical_section obj _ctypes.byref obj: object(subclass_of="clinic_state()->PyCData_Type") @@ -1806,7 +1799,7 @@ Return a pointer lookalike to a C instance, only usable as function argument. static PyObject * _ctypes_byref_impl(PyObject *module, PyObject *obj, Py_ssize_t offset) -/*[clinic end generated code: output=60dec5ed520c71de input=6ec02d95d15fbd56]*/ +/*[clinic end generated code: output=60dec5ed520c71de input=870076149a2de427]*/ { ctypes_state *st = get_module_state(module); diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 86bcc805360..547e2471a1c 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -759,7 +759,7 @@ d_get(void *ptr, Py_ssize_t size) return PyFloat_FromDouble(val); } -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) /* We don't use _Complex types here, using arrays instead, as the C11+ standard says: "Each complex type has the same representation and alignment @@ -1253,7 +1253,7 @@ Z_get(void *ptr, Py_ssize_t size) wchar_t *p; p = *(wchar_t **)ptr; if (p) { - return PyUnicode_FromWideChar(p, wcslen(p)); + return PyUnicode_FromWideChar(p, -1); } else { Py_RETURN_NONE; } @@ -1599,7 +1599,7 @@ for base_code, base_c_type in [ /////////////////////////////////////////////////////////////////////////// TABLE_ENTRY_SW(d, &ffi_type_double); -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) if (Py_FFI_COMPLEX_AVAILABLE) { TABLE_ENTRY(D, &ffi_type_complex_double); TABLE_ENTRY(F, &ffi_type_complex_float); diff --git a/Modules/_ctypes/clinic/_ctypes.c.h b/Modules/_ctypes/clinic/_ctypes.c.h index 92dfb8f83b7..529872f0f17 100644 --- a/Modules/_ctypes/clinic/_ctypes.c.h +++ b/Modules/_ctypes/clinic/_ctypes.c.h @@ -31,6 +31,48 @@ _ctypes_CType_Type___sizeof__(PyObject *self, PyTypeObject *cls, PyObject *const return _ctypes_CType_Type___sizeof___impl(self, cls); } +#if !defined(_ctypes_CType_Type___pointer_type___DOCSTR) +# define _ctypes_CType_Type___pointer_type___DOCSTR NULL +#endif +#if defined(_CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF) +# undef _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF +# define _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF {"__pointer_type__", (getter)_ctypes_CType_Type___pointer_type___get, (setter)_ctypes_CType_Type___pointer_type___set, _ctypes_CType_Type___pointer_type___DOCSTR}, +#else +# define _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF {"__pointer_type__", (getter)_ctypes_CType_Type___pointer_type___get, NULL, _ctypes_CType_Type___pointer_type___DOCSTR}, +#endif + +static PyObject * +_ctypes_CType_Type___pointer_type___get_impl(PyObject *self); + +static PyObject * +_ctypes_CType_Type___pointer_type___get(PyObject *self, void *Py_UNUSED(context)) +{ + return _ctypes_CType_Type___pointer_type___get_impl(self); +} + +#if !defined(_ctypes_CType_Type___pointer_type___DOCSTR) +# define _ctypes_CType_Type___pointer_type___DOCSTR NULL +#endif +#if defined(_CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF) +# undef _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF +# define _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF {"__pointer_type__", (getter)_ctypes_CType_Type___pointer_type___get, (setter)_ctypes_CType_Type___pointer_type___set, _ctypes_CType_Type___pointer_type___DOCSTR}, +#else +# define _CTYPES_CTYPE_TYPE___POINTER_TYPE___GETSETDEF {"__pointer_type__", NULL, (setter)_ctypes_CType_Type___pointer_type___set, NULL}, +#endif + +static int +_ctypes_CType_Type___pointer_type___set_impl(PyObject *self, PyObject *value); + +static int +_ctypes_CType_Type___pointer_type___set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + return_value = _ctypes_CType_Type___pointer_type___set_impl(self, value); + + return return_value; +} + PyDoc_STRVAR(CDataType_from_address__doc__, "from_address($self, value, /)\n" "--\n" @@ -134,6 +176,11 @@ CDataType_from_buffer(PyObject *type, PyTypeObject *cls, PyObject *const *args, goto exit; } offset = ival; + if (offset < 0) { + PyErr_SetString(PyExc_ValueError, + "offset cannot be negative"); + goto exit; + } } skip_optional_posonly: return_value = CDataType_from_buffer_impl(type, cls, obj, offset); @@ -200,6 +247,11 @@ CDataType_from_buffer_copy(PyObject *type, PyTypeObject *cls, PyObject *const *a goto exit; } offset = ival; + if (offset < 0) { + PyErr_SetString(PyExc_ValueError, + "offset cannot be negative"); + goto exit; + } } skip_optional_posonly: return_value = CDataType_from_buffer_copy_impl(type, cls, &buffer, offset); @@ -1000,4 +1052,4 @@ Simple_from_outparm(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py } return Simple_from_outparm_impl(self, cls); } -/*[clinic end generated code: output=9fb75bf7e9a17df2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=22105663d71237ca input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 2d859ed63e1..9f995bf2408 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -1,5 +1,17 @@ -#if defined (__SVR4) && defined (__sun) +/* Get a definition of alloca(). */ +#if (defined (__SVR4) && defined (__sun)) || defined(HAVE_ALLOCA_H) # include <alloca.h> +#elif defined(MS_WIN32) +# include <malloc.h> +#endif + +/* If the system does not define alloca(), we have to hope for a compiler builtin. */ +#ifndef alloca +# if defined __GNUC__ || (__clang_major__ >= 4) +# define alloca __builtin_alloca +# else +# error "Could not define alloca() on your platform." +# endif #endif #include <stdbool.h> @@ -11,7 +23,7 @@ // Do we support C99 complex types in ffi? // For Apple's libffi, this must be determined at runtime (see gh-128156). -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) # if USING_APPLE_OS_LIBFFI && defined(__has_builtin) # if __has_builtin(__builtin_available) # define Py_FFI_COMPLEX_AVAILABLE __builtin_available(macOS 10.15, *) @@ -419,7 +431,7 @@ typedef struct { visible to other threads before the `dict_final` bit is set. */ -#define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUT(&(stginfo)->mutex) +#define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUTEX(&(stginfo)->mutex) #define STGINFO_UNLOCK() Py_END_CRITICAL_SECTION() static inline uint8_t diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index 80ba96614bf..db405acf872 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -30,11 +30,6 @@ #ifdef Py_GIL_DISABLED static PyMutex malloc_closure_lock; -# define MALLOC_CLOSURE_LOCK() PyMutex_Lock(&malloc_closure_lock) -# define MALLOC_CLOSURE_UNLOCK() PyMutex_Unlock(&malloc_closure_lock) -#else -# define MALLOC_CLOSURE_LOCK() ((void)0) -# define MALLOC_CLOSURE_UNLOCK() ((void)0) #endif typedef union _tagITEM { @@ -120,11 +115,11 @@ void Py_ffi_closure_free(void *p) } #endif #endif - MALLOC_CLOSURE_LOCK(); + FT_MUTEX_LOCK(&malloc_closure_lock); ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); } /* return one item from the free list, allocating more if needed */ @@ -143,13 +138,13 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc) } #endif #endif - MALLOC_CLOSURE_LOCK(); + FT_MUTEX_LOCK(&malloc_closure_lock); ITEM *item; if (!free_list) { more_core(); } if (!free_list) { - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); return NULL; } item = free_list; @@ -160,6 +155,6 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc) #else *codeloc = (void *)item; #endif - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); return (void *)item; } diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f208d2956e4..ab955a0b824 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -484,7 +484,7 @@ error:; /* Replace array elements at stginfo->ffi_type_pointer.elements. - Return -1 if error occured. + Return -1 if error occurred. */ int _replace_array_elements(ctypes_state *st, PyObject *layout_fields, diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index eecf7a1c8a1..3b46fdf838b 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -17,6 +17,7 @@ static const char PyCursesVersion[] = "2.1"; #include "Python.h" +#define CURSES_PANEL_MODULE #include "py_curses.h" #if defined(HAVE_NCURSESW_PANEL_H) @@ -28,10 +29,12 @@ static const char PyCursesVersion[] = "2.1"; #endif typedef struct { - PyObject *PyCursesError; + PyObject *error; PyTypeObject *PyCursesPanel_Type; } _curses_panel_state; +typedef struct PyCursesPanelObject PyCursesPanelObject; + static inline _curses_panel_state * get_curses_panel_state(PyObject *module) { @@ -40,11 +43,30 @@ get_curses_panel_state(PyObject *module) return (_curses_panel_state *)state; } +static inline _curses_panel_state * +get_curses_panel_state_by_panel(PyCursesPanelObject *panel) +{ + /* + * Note: 'state' may be NULL if Py_TYPE(panel) is not a heap + * type associated with this module, but the compiler would + * have likely already complained with an "invalid pointer + * type" at compile-time. + * + * To make it more robust, all functions recovering a module's + * state from an object should expect to return NULL with an + * exception set (in contrast to functions recovering a module's + * state from a module itself). + */ + void *state = PyType_GetModuleState(Py_TYPE(panel)); + assert(state != NULL); + return (_curses_panel_state *)state; +} + static int _curses_panel_clear(PyObject *mod) { _curses_panel_state *state = get_curses_panel_state(mod); - Py_CLEAR(state->PyCursesError); + Py_CLEAR(state->error); Py_CLEAR(state->PyCursesPanel_Type); return 0; } @@ -54,7 +76,7 @@ _curses_panel_traverse(PyObject *mod, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(mod)); _curses_panel_state *state = get_curses_panel_state(mod); - Py_VISIT(state->PyCursesError); + Py_VISIT(state->error); Py_VISIT(state->PyCursesPanel_Type); return 0; } @@ -65,28 +87,149 @@ _curses_panel_free(void *mod) (void)_curses_panel_clear((PyObject *)mod); } +/* Utility Error Procedures + * + * The naming and implementations are identical to those in _cursesmodule.c. + * Functions that are not yet needed (for instance, reporting an ERR value + * from a module-wide function, namely curses_panel_set_error()) are + * omitted and should only be added if needed. + */ + +static void +_curses_panel_format_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname, + const char *return_value, + const char *default_message) +{ + assert(!PyErr_Occurred()); + if (python_funcname == NULL && curses_funcname == NULL) { + PyErr_SetString(state->error, default_message); + } + else if (python_funcname == NULL) { + (void)PyErr_Format(state->error, CURSES_ERROR_FORMAT, + curses_funcname, return_value); + } + else { + assert(python_funcname != NULL); + (void)PyErr_Format(state->error, CURSES_ERROR_VERBOSE_FORMAT, + curses_funcname, python_funcname, return_value); + } +} + +/* + * Format a curses error for a function that returned ERR. + * + * Specify a non-NULL 'python_funcname' when the latter differs from + * 'curses_funcname'. If both names are NULL, uses the 'catchall_ERR' + * message instead. + */ +static void +_curses_panel_set_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_format_error(state, curses_funcname, python_funcname, + "ERR", catchall_ERR); +} + +/* + * Format a curses error for a function that returned NULL. + * + * Specify a non-NULL 'python_funcname' when the latter differs from + * 'curses_funcname'. If both names are NULL, uses the 'catchall_NULL' + * message instead. + */ +static void +_curses_panel_set_null_error(_curses_panel_state *state, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_format_error(state, curses_funcname, python_funcname, + "NULL", catchall_NULL); +} + +/* Same as _curses_panel_set_null_error() for a module object. */ +static void +curses_panel_set_null_error(PyObject *module, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state(module); + _curses_panel_set_null_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_panel_set_error() for a panel object. */ +static void +curses_panel_panel_set_error(PyCursesPanelObject *panel, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state_by_panel(panel); + _curses_panel_set_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_panel_set_null_error() for a panel object. */ +static void +curses_panel_panel_set_null_error(PyCursesPanelObject *panel, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_panel_state *state = get_curses_panel_state_by_panel(panel); + _curses_panel_set_null_error(state, curses_funcname, python_funcname); +} + +/* + * Indicate that a panel object couldn't be found. + * + * Use it for the following constructions: + * + * PROC caller_funcname: + * pan = called_funcname() + * find_po(panel) + * + * PROC caller_funcname: + * find_po(self->pan) +*/ +static void +curses_panel_notfound_error(const char *called_funcname, + const char *caller_funcname) +{ + assert(!(called_funcname == NULL && caller_funcname == NULL)); + if (caller_funcname == NULL) { + (void)PyErr_Format(PyExc_RuntimeError, + "%s(): cannot find panel object", + called_funcname); + } + else { + (void)PyErr_Format(PyExc_RuntimeError, + "%s() (called by %s()): cannot find panel object", + called_funcname, caller_funcname); + } +} + /* Utility Functions */ /* - * Check the return code from a curses function and return None - * or raise an exception as appropriate. + * Check the return code from a curses function, returning None + * on success and setting an exception on error. */ +/* + * Return None if 'code' is different from ERR (implementation-defined). + * Otherwise, set an exception using curses_panel_panel_set_error() and + * the remaining arguments, and return NULL. + */ static PyObject * -PyCursesCheckERR(_curses_panel_state *state, int code, const char *fname) +curses_panel_panel_check_err(PyCursesPanelObject *panel, int code, + const char *curses_funcname, + const char *python_funcname) { if (code != ERR) { Py_RETURN_NONE; } - else { - if (fname == NULL) { - PyErr_SetString(state->PyCursesError, catchall_ERR); - } - else { - PyErr_Format(state->PyCursesError, "%s() returned ERR", fname); - } - return NULL; - } + curses_panel_panel_set_error(panel, curses_funcname, python_funcname); + return NULL; } /***************************************************************************** @@ -95,7 +238,7 @@ PyCursesCheckERR(_curses_panel_state *state, int code, const char *fname) /* Definition of the panel object and panel type */ -typedef struct { +typedef struct PyCursesPanelObject { PyObject_HEAD PANEL *pan; PyCursesWindowObject *wo; /* for reference counts */ @@ -144,8 +287,11 @@ insert_lop(PyCursesPanelObject *po) return 0; } -/* Remove the panel object from lop */ -static void +/* Remove the panel object from lop. + * + * Return -1 on error but do NOT set an exception; otherwise return 0. + */ +static int remove_lop(PyCursesPanelObject *po) { list_of_panels *temp, *n; @@ -154,25 +300,23 @@ remove_lop(PyCursesPanelObject *po) if (temp->po == po) { lop = temp->next; PyMem_Free(temp); - return; + return 0; } while (temp->next == NULL || temp->next->po != po) { if (temp->next == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "remove_lop: can't find Panel Object"); - return; + return -1; } temp = temp->next; } n = temp->next->next; PyMem_Free(temp->next); temp->next = n; - return; + return 0; } /* Return the panel object that corresponds to pan */ static PyCursesPanelObject * -find_po(PANEL *pan) +find_po_impl(PANEL *pan) { list_of_panels *temp; for (temp = lop; temp->po->pan != pan; temp = temp->next) @@ -180,6 +324,17 @@ find_po(PANEL *pan) return temp->po; } +/* Same as find_po_impl() but with caller context information. */ +static PyCursesPanelObject * +find_po(PANEL *pan, const char *called_funcname, const char *caller_funcname) +{ + PyCursesPanelObject *res = find_po_impl(pan); + if (res == NULL) { + curses_panel_notfound_error(called_funcname, caller_funcname); + } + return res; +} + /*[clinic input] module _curses_panel class _curses_panel.panel "PyCursesPanelObject *" "&PyCursesPanel_Type" @@ -193,67 +348,60 @@ class _curses_panel.panel "PyCursesPanelObject *" "&PyCursesPanel_Type" /*[clinic input] _curses_panel.panel.bottom - cls: defining_class - Push the panel to the bottom of the stack. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=8ec7fbbc08554021 input=6b7d2c0578b5a1c4]*/ +_curses_panel_panel_bottom_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=7aa7d14d7e1d1ce6 input=b6c920c071b61e2e]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, bottom_panel(self->pan), "bottom"); + int rtn = bottom_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "bottom_panel", "bottom"); } /*[clinic input] +@permit_long_docstring_body _curses_panel.panel.hide - cls: defining_class - Hide the panel. This does not delete the object, it just makes the window on screen invisible. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc6ab7203cdc1450 input=1bfc741f473e6055]*/ +_curses_panel_panel_hide_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=a7bbbd523e1eab49 input=9071b463a39a1a6a]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, hide_panel(self->pan), "hide"); + int rtn = hide_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "hide_panel", "hide"); } /*[clinic input] _curses_panel.panel.show - cls: defining_class - Display the panel (which might have been hidden). [clinic start generated code]*/ static PyObject * -_curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=dc3421de375f0409 input=8122e80151cb4379]*/ +_curses_panel_panel_show_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=6b4553ab45c97769 input=57b167bbefaa3755]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, show_panel(self->pan), "show"); + int rtn = show_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "show_panel", "show"); } /*[clinic input] _curses_panel.panel.top - cls: defining_class - Push panel to the top of the stack. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=10a072e511e873f7 input=1f372d597dda3379]*/ +_curses_panel_panel_top_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=0f5f2f8cdd2d1777 input=be33975ec3ca0e9a]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, top_panel(self->pan), "top"); + int rtn = top_panel(self->pan); + return curses_panel_panel_check_err(self, rtn, "top_panel", "top"); } /* Allocation and deallocation of Panel Objects */ @@ -262,8 +410,11 @@ static PyObject * PyCursesPanel_New(_curses_panel_state *state, PANEL *pan, PyCursesWindowObject *wo) { - PyCursesPanelObject *po = PyObject_New(PyCursesPanelObject, - state->PyCursesPanel_Type); + assert(state != NULL); + PyTypeObject *type = state->PyCursesPanel_Type; + assert(type != NULL); + assert(type->tp_alloc != NULL); + PyCursesPanelObject *po = (PyCursesPanelObject *)type->tp_alloc(type, 0); if (po == NULL) { return NULL; } @@ -278,27 +429,57 @@ PyCursesPanel_New(_curses_panel_state *state, PANEL *pan, return (PyObject *)po; } +static int +PyCursesPanel_Clear(PyObject *op) +{ + PyCursesPanelObject *self = _PyCursesPanelObject_CAST(op); + PyObject *extra = (PyObject *)panel_userptr(self->pan); + if (extra != NULL) { + Py_DECREF(extra); + if (set_panel_userptr(self->pan, NULL) == ERR) { + curses_panel_panel_set_error(self, "set_panel_userptr", NULL); + return -1; + } + } + // self->wo should not be cleared because an associated WINDOW may exist + return 0; +} + static void PyCursesPanel_Dealloc(PyObject *self) { - PyObject *tp, *obj; - PyCursesPanelObject *po = _PyCursesPanelObject_CAST(self); + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); - tp = (PyObject *) Py_TYPE(po); - obj = (PyObject *) panel_userptr(po->pan); - if (obj) { - (void)set_panel_userptr(po->pan, NULL); - Py_DECREF(obj); + PyCursesPanelObject *po = _PyCursesPanelObject_CAST(self); + if (PyCursesPanel_Clear(self) < 0) { + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); + } + if (del_panel(po->pan) == ERR && !PyErr_Occurred()) { + curses_panel_panel_set_error(po, "del_panel", "__del__"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); } - (void)del_panel(po->pan); if (po->wo != NULL) { Py_DECREF(po->wo); - remove_lop(po); + if (remove_lop(po) < 0) { + PyErr_SetString(PyExc_RuntimeError, "__del__: no panel object to delete"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); + } } - PyObject_Free(po); + tp->tp_free(po); Py_DECREF(tp); } +static int +PyCursesPanel_Traverse(PyObject *op, visitproc visit, void *arg) +{ + PyCursesPanelObject *self = _PyCursesPanelObject_CAST(op); + Py_VISIT(Py_TYPE(op)); + Py_VISIT(panel_userptr(self->pan)); + Py_VISIT(self->wo); + return 0; +} + /* panel_above(NULL) returns the bottom panel in the stack. To get this behaviour we use curses.panel.bottom_panel(). */ /*[clinic input] @@ -315,18 +496,11 @@ _curses_panel_panel_above_impl(PyCursesPanelObject *self) PyCursesPanelObject *po; pan = panel_above(self->pan); - - if (pan == NULL) { /* valid output, it means the calling panel - is on top of the stack */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_above: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_above", "above"); + return Py_XNewRef(po); } /* panel_below(NULL) returns the top panel in the stack. To get @@ -345,18 +519,11 @@ _curses_panel_panel_below_impl(PyCursesPanelObject *self) PyCursesPanelObject *po; pan = panel_below(self->pan); - - if (pan == NULL) { /* valid output, it means the calling panel - is on the bottom of the stack */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_below: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_below", "below"); + return Py_XNewRef(po); } /*[clinic input] @@ -378,7 +545,6 @@ _curses_panel_panel_hidden_impl(PyCursesPanelObject *self) /*[clinic input] _curses_panel.panel.move - cls: defining_class y: int x: int / @@ -387,12 +553,11 @@ Move the panel to the screen coordinates (y, x). [clinic start generated code]*/ static PyObject * -_curses_panel_panel_move_impl(PyCursesPanelObject *self, PyTypeObject *cls, - int y, int x) -/*[clinic end generated code: output=ce546c93e56867da input=60a0e7912ff99849]*/ +_curses_panel_panel_move_impl(PyCursesPanelObject *self, int y, int x) +/*[clinic end generated code: output=d867535a89777415 input=e0b36b78acc03fba]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, move_panel(self->pan, y, x), "move_panel"); + int rtn = move_panel(self->pan, y, x); + return curses_panel_panel_check_err(self, rtn, "move_panel", "move"); } /*[clinic input] @@ -411,7 +576,6 @@ _curses_panel_panel_window_impl(PyCursesPanelObject *self) /*[clinic input] _curses_panel.panel.replace - cls: defining_class win: object(type="PyCursesWindowObject *", subclass_of="&PyCursesWindow_Type") / @@ -420,22 +584,17 @@ Change the window associated with the panel to the window win. static PyObject * _curses_panel_panel_replace_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyCursesWindowObject *win) -/*[clinic end generated code: output=c71f95c212d58ae7 input=dbec7180ece41ff5]*/ +/*[clinic end generated code: output=2253a95f7b287255 input=4b1c4283987d9dfa]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - - PyCursesPanelObject *po = find_po(self->pan); + PyCursesPanelObject *po = find_po(self->pan, "replace", NULL); if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "replace_panel: can't find Panel Object"); return NULL; } int rtn = replace_panel(self->pan, win->win); if (rtn == ERR) { - PyErr_SetString(state->PyCursesError, "replace_panel() returned ERR"); + curses_panel_panel_set_error(self, "replace_panel", "replace"); return NULL; } Py_SETREF(po->wo, (PyCursesWindowObject*)Py_NewRef(win)); @@ -445,7 +604,6 @@ _curses_panel_panel_replace_impl(PyCursesPanelObject *self, /*[clinic input] _curses_panel.panel.set_userptr - cls: defining_class obj: object / @@ -454,8 +612,8 @@ Set the panel's user pointer to obj. static PyObject * _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyObject *obj) -/*[clinic end generated code: output=db74f3db07b28080 input=e3fee2ff7b1b8e48]*/ + PyObject *obj) +/*[clinic end generated code: output=7fa1fd23f69db71e input=d2c6a9dbefabbf39]*/ { PyCursesInitialised; Py_INCREF(obj); @@ -464,34 +622,27 @@ _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, if (rc == ERR) { /* In case of an ncurses error, decref the new object again */ Py_DECREF(obj); + curses_panel_panel_set_error(self, "set_panel_userptr", "set_userptr"); + return NULL; } - else { - Py_XDECREF(oldobj); - } - - _curses_panel_state *state = PyType_GetModuleState(cls); - return PyCursesCheckERR(state, rc, "set_panel_userptr"); + Py_XDECREF(oldobj); + Py_RETURN_NONE; } /*[clinic input] _curses_panel.panel.userptr - cls: defining_class - Return the user pointer for the panel. [clinic start generated code]*/ static PyObject * -_curses_panel_panel_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls) -/*[clinic end generated code: output=eea6e6f39ffc0179 input=f22ca4f115e30a80]*/ +_curses_panel_panel_userptr_impl(PyCursesPanelObject *self) +/*[clinic end generated code: output=e849c307b5dc9237 input=f78b7a47aef0fd50]*/ { - _curses_panel_state *state = PyType_GetModuleState(cls); - PyCursesInitialised; PyObject *obj = (PyObject *) panel_userptr(self->pan); if (obj == NULL) { - PyErr_SetString(state->PyCursesError, "no userptr set"); + curses_panel_panel_set_null_error(self, "panel_userptr", "userptr"); return NULL; } @@ -520,7 +671,9 @@ static PyMethodDef PyCursesPanel_Methods[] = { /* -------------------------------------------------------*/ static PyType_Slot PyCursesPanel_Type_slots[] = { + {Py_tp_clear, PyCursesPanel_Clear}, {Py_tp_dealloc, PyCursesPanel_Dealloc}, + {Py_tp_traverse, PyCursesPanel_Traverse}, {Py_tp_methods, PyCursesPanel_Methods}, {0, 0}, }; @@ -528,7 +681,12 @@ static PyType_Slot PyCursesPanel_Type_slots[] = { static PyType_Spec PyCursesPanel_Type_spec = { .name = "_curses_panel.panel", .basicsize = sizeof(PyCursesPanelObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC + ), .slots = PyCursesPanel_Type_slots }; @@ -552,18 +710,11 @@ _curses_panel_bottom_panel_impl(PyObject *module) PyCursesInitialised; pan = panel_above(NULL); - - if (pan == NULL) { /* valid output, it means - there's no panel at all */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_above: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_above", "bottom_panel"); + return Py_XNewRef(po); } /*[clinic input] @@ -579,14 +730,13 @@ static PyObject * _curses_panel_new_panel_impl(PyObject *module, PyCursesWindowObject *win) /*[clinic end generated code: output=45e948e0176a9bd2 input=74d4754e0ebe4800]*/ { - _curses_panel_state *state = get_curses_panel_state(module); - PANEL *pan = new_panel(win->win); if (pan == NULL) { - PyErr_SetString(state->PyCursesError, catchall_NULL); + curses_panel_set_null_error(module, "new_panel", NULL); return NULL; } - return (PyObject *)PyCursesPanel_New(state, pan, win); + _curses_panel_state *state = get_curses_panel_state(module); + return PyCursesPanel_New(state, pan, win); } @@ -610,18 +760,11 @@ _curses_panel_top_panel_impl(PyObject *module) PyCursesInitialised; pan = panel_below(NULL); - - if (pan == NULL) { /* valid output, it means - there's no panel at all */ + if (pan == NULL) { /* valid output: it means no panel exists yet */ Py_RETURN_NONE; } - po = find_po(pan); - if (po == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "panel_below: can't find Panel Object"); - return NULL; - } - return Py_NewRef(po); + po = find_po(pan, "panel_below", "top_panel"); + return Py_XNewRef(po); } /*[clinic input] @@ -673,10 +816,10 @@ _curses_panel_exec(PyObject *mod) } /* For exception _curses_panel.error */ - state->PyCursesError = PyErr_NewException( + state->error = PyErr_NewException( "_curses_panel.error", NULL, NULL); - if (PyModule_AddObjectRef(mod, "error", state->PyCursesError) < 0) { + if (PyModule_AddObjectRef(mod, "error", state->error) < 0) { return -1; } diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index cd185bc2b02..61464348d6f 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -108,7 +108,6 @@ static const char PyCursesVersion[] = "2.2"; #include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_structseq.h" // _PyStructSequence_NewType() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include "pycore_fileutils.h" // _Py_set_inheritable #ifdef __hpux @@ -211,18 +210,116 @@ static int curses_start_color_called = FALSE; static const char *curses_screen_encoding = NULL; +/* Utility Error Procedures */ + +static void +_curses_format_error(cursesmodule_state *state, + const char *curses_funcname, + const char *python_funcname, + const char *return_value, + const char *default_message) +{ + assert(!PyErr_Occurred()); + if (python_funcname == NULL && curses_funcname == NULL) { + PyErr_SetString(state->error, default_message); + } + else if (python_funcname == NULL) { + (void)PyErr_Format(state->error, CURSES_ERROR_FORMAT, + curses_funcname, return_value); + } + else { + assert(python_funcname != NULL); + (void)PyErr_Format(state->error, CURSES_ERROR_VERBOSE_FORMAT, + curses_funcname, python_funcname, return_value); + } +} + +/* + * Format a curses error for a function that returned ERR. + * + * Specify a non-NULL 'python_funcname' when the latter differs from + * 'curses_funcname'. If both names are NULL, uses the 'catchall_ERR' + * message instead. + */ +static void +_curses_set_error(cursesmodule_state *state, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_format_error(state, curses_funcname, python_funcname, + "ERR", catchall_ERR); +} + +/* + * Format a curses error for a function that returned NULL. + * + * Specify a non-NULL 'python_funcname' when the latter differs from + * 'curses_funcname'. If both names are NULL, uses the 'catchall_NULL' + * message instead. + */ +static inline void +_curses_set_null_error(cursesmodule_state *state, + const char *curses_funcname, + const char *python_funcname) +{ + _curses_format_error(state, curses_funcname, python_funcname, + "NULL", catchall_NULL); +} + +/* Same as _curses_set_error() for a module object. */ +static void +curses_set_error(PyObject *module, + const char *curses_funcname, + const char *python_funcname) +{ + cursesmodule_state *state = get_cursesmodule_state(module); + _curses_set_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_set_null_error() for a module object. */ +static void +curses_set_null_error(PyObject *module, + const char *curses_funcname, + const char *python_funcname) +{ + cursesmodule_state *state = get_cursesmodule_state(module); + _curses_set_null_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_set_error() for a Window object. */ +static void +curses_window_set_error(PyCursesWindowObject *win, + const char *curses_funcname, + const char *python_funcname) +{ + cursesmodule_state *state = get_cursesmodule_state_by_win(win); + _curses_set_error(state, curses_funcname, python_funcname); +} + +/* Same as _curses_set_null_error() for a Window object. */ +static void +curses_window_set_null_error(PyCursesWindowObject *win, + const char *curses_funcname, + const char *python_funcname) +{ + cursesmodule_state *state = get_cursesmodule_state_by_win(win); + _curses_set_null_error(state, curses_funcname, python_funcname); +} + /* Utility Checking Procedures */ /* * Function to check that 'funcname' has been called by testing - * the 'called' boolean. If an error occurs, a PyCursesError is + * the 'called' boolean. If an error occurs, an exception is * set and this returns 0. Otherwise, this returns 1. * * Since this function can be called in functions that do not * have a direct access to the module's state, '_curses.error' * is imported on demand. + * + * Use _PyCursesStatefulCheckFunction() if the module is given. */ -static inline int +static int _PyCursesCheckFunction(int called, const char *funcname) { if (called == TRUE) { @@ -230,7 +327,7 @@ _PyCursesCheckFunction(int called, const char *funcname) } PyObject *exc = PyImport_ImportModuleAttrString("_curses", "error"); if (exc != NULL) { - PyErr_Format(exc, "must call %s() first", funcname); + PyErr_Format(exc, CURSES_ERROR_MUST_CALL_FORMAT, funcname); Py_DECREF(exc); } assert(PyErr_Occurred()); @@ -244,14 +341,15 @@ _PyCursesCheckFunction(int called, const char *funcname) * * The exception type is obtained from the 'module' state. */ -static inline int -_PyCursesStatefulCheckFunction(PyObject *module, int called, const char *funcname) +static int +_PyCursesStatefulCheckFunction(PyObject *module, + int called, const char *funcname) { if (called == TRUE) { return 1; } cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_Format(state->error, "must call %s() first", funcname); + PyErr_Format(state->error, CURSES_ERROR_MUST_CALL_FORMAT, funcname); return 0; } @@ -287,44 +385,39 @@ _PyCursesStatefulCheckFunction(PyObject *module, int called, const char *funcnam /* Utility Functions */ -static inline void -_PyCursesSetError(cursesmodule_state *state, const char *funcname) -{ - if (funcname == NULL) { - PyErr_SetString(state->error, catchall_ERR); - } - else { - PyErr_Format(state->error, "%s() returned ERR", funcname); - } -} - /* - * Check the return code from a curses function and return None - * or raise an exception as appropriate. + * Check the return code from a curses function, returning None + * on success and setting an exception on error. */ +/* + * Return None if 'code' is different from ERR (implementation-defined). + * Otherwise, set an exception using curses_set_error() and the remaining + * arguments, and return NULL. + */ static PyObject * -PyCursesCheckERR(PyObject *module, int code, const char *fname) +curses_check_err(PyObject *module, int code, + const char *curses_funcname, + const char *python_funcname) { if (code != ERR) { Py_RETURN_NONE; - } else { - cursesmodule_state *state = get_cursesmodule_state(module); - _PyCursesSetError(state, fname); - return NULL; } + curses_set_error(module, curses_funcname, python_funcname); + return NULL; } +/* Same as curses_check_err() for a Window object. */ static PyObject * -PyCursesCheckERR_ForWin(PyCursesWindowObject *win, int code, const char *fname) +curses_window_check_err(PyCursesWindowObject *win, int code, + const char *curses_funcname, + const char *python_funcname) { if (code != ERR) { Py_RETURN_NONE; - } else { - cursesmodule_state *state = get_cursesmodule_state_by_win(win); - _PyCursesSetError(state, fname); - return NULL; } + curses_window_set_error(win, curses_funcname, python_funcname); + return NULL; } /* Convert an object to a byte (an integer of type chtype): @@ -650,13 +743,16 @@ class component_converter(CConverter): The Window Object ******************************************************************************/ -/* Function prototype macros for Window object - - X - function name - TYPE - parameter Type - ERGSTR - format string for construction of the return value - PARSESTR - format string for argument parsing -*/ +/* + * Macros for creating a PyCursesWindowObject object's method. + * + * Parameters + * + * X The name of the curses C function or macro to invoke. + * TYPE The function parameter(s) type. + * ERGSTR The format string for construction of the return value. + * PARSESTR The format string for argument parsing. + */ #define Window_NoArgNoReturnFunction(X) \ static PyObject *PyCursesWindow_ ## X \ @@ -664,7 +760,7 @@ class component_converter(CConverter): { \ PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ int code = X(self->win); \ - return PyCursesCheckERR_ForWin(self, code, # X); \ + return curses_window_check_err(self, code, # X, NULL); \ } #define Window_NoArgTrueFalseFunction(X) \ @@ -717,7 +813,7 @@ class component_converter(CConverter): } \ PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ int code = X(self->win, arg1); \ - return PyCursesCheckERR_ForWin(self, code, # X); \ + return curses_window_check_err(self, code, # X, NULL); \ } #define Window_TwoArgNoReturnFunction(X, TYPE, PARSESTR) \ @@ -730,7 +826,7 @@ class component_converter(CConverter): } \ PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ int code = X(self->win, arg1, arg2); \ - return PyCursesCheckERR_ForWin(self, code, # X); \ + return curses_window_check_err(self, code, # X, NULL); \ } /* ------------- WINDOW routines --------------- */ @@ -835,8 +931,10 @@ PyCursesWindow_dealloc(PyObject *self) PyObject_GC_UnTrack(self); PyCursesWindowObject *wo = (PyCursesWindowObject *)self; if (wo->win != stdscr && wo->win != NULL) { - // silently ignore errors in delwin(3) - (void)delwin(wo->win); + if (delwin(wo->win) == ERR) { + curses_window_set_error(wo, "delwin", "__del__"); + PyErr_FormatUnraisable("Exception ignored in delwin()"); + } } if (wo->encoding != NULL) { PyMem_Free(wo->encoding); @@ -903,13 +1001,19 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, #ifdef HAVE_NCURSESW type = PyCurses_ConvertToCchar_t(self, ch, &cch, wstr); if (type == 2) { - funcname = "add_wch"; wstr[1] = L'\0'; - setcchar(&wcval, wstr, attr, PAIR_NUMBER(attr), NULL); - if (coordinates_group) + rtn = setcchar(&wcval, wstr, attr, PAIR_NUMBER(attr), NULL); + if (rtn == ERR) { + curses_window_set_error(self, "setcchar", "addch"); + return NULL; + } + if (coordinates_group) { rtn = mvwadd_wch(self->win,y,x, &wcval); + funcname = "mvwadd_wch"; + } else { rtn = wadd_wch(self->win, &wcval); + funcname = "wadd_wch"; } } else @@ -917,17 +1021,40 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, type = PyCurses_ConvertToCchar_t(self, ch, &cch); #endif if (type == 1) { - funcname = "addch"; - if (coordinates_group) + if (coordinates_group) { rtn = mvwaddch(self->win,y,x, cch | (attr_t) attr); + funcname = "mvwaddch"; + } else { rtn = waddch(self->win, cch | (attr_t) attr); + funcname = "waddch"; } } else { return NULL; } - return PyCursesCheckERR_ForWin(self, rtn, funcname); + return curses_window_check_err(self, rtn, funcname, "addch"); +} + +#ifdef HAVE_NCURSESW +#define curses_release_wstr(STRTYPE, WSTR) \ + do { \ + if ((STRTYPE) == 2) { \ + PyMem_Free((WSTR)); \ + } \ + } while (0) +#else +#define curses_release_wstr(_STRTYPE, _WSTR) +#endif + +static int +curses_wattrset(PyCursesWindowObject *self, long attr, const char *funcname) +{ + if (wattrset(self->win, attr) == ERR) { + curses_window_set_error(self, "wattrset", funcname); + return -1; + } + return 0; } /*[clinic input] @@ -983,31 +1110,46 @@ _curses_window_addstr_impl(PyCursesWindowObject *self, int group_left_1, } if (use_attr) { attr_old = getattrs(self->win); - (void)wattrset(self->win,attr); + if (curses_wattrset(self, attr, "addstr") < 0) { + curses_release_wstr(strtype, wstr); + return NULL; + } } #ifdef HAVE_NCURSESW if (strtype == 2) { - funcname = "addwstr"; - if (use_xy) + if (use_xy) { rtn = mvwaddwstr(self->win,y,x,wstr); - else + funcname = "mvwaddwstr"; + } + else { rtn = waddwstr(self->win,wstr); + funcname = "waddwstr"; + } PyMem_Free(wstr); } else #endif { const char *str = PyBytes_AS_STRING(bytesobj); - funcname = "addstr"; - if (use_xy) + if (use_xy) { rtn = mvwaddstr(self->win,y,x,str); - else + funcname = "mvwaddstr"; + } + else { rtn = waddstr(self->win,str); + funcname = "waddstr"; + } Py_DECREF(bytesobj); } - if (use_attr) - (void)wattrset(self->win,attr_old); - return PyCursesCheckERR_ForWin(self, rtn, funcname); + if (rtn == ERR) { + curses_window_set_error(self, funcname, "addstr"); + return NULL; + } + if (use_attr) { + rtn = wattrset(self->win, attr_old); + return curses_window_check_err(self, rtn, "wattrset", "addstr"); + } + Py_RETURN_NONE; } /*[clinic input] @@ -1066,31 +1208,46 @@ _curses_window_addnstr_impl(PyCursesWindowObject *self, int group_left_1, if (use_attr) { attr_old = getattrs(self->win); - (void)wattrset(self->win,attr); + if (curses_wattrset(self, attr, "addnstr") < 0) { + curses_release_wstr(strtype, wstr); + return NULL; + } } #ifdef HAVE_NCURSESW if (strtype == 2) { - funcname = "addnwstr"; - if (use_xy) + if (use_xy) { rtn = mvwaddnwstr(self->win,y,x,wstr,n); - else + funcname = "mvwaddnwstr"; + } + else { rtn = waddnwstr(self->win,wstr,n); + funcname = "waddnwstr"; + } PyMem_Free(wstr); } else #endif { const char *str = PyBytes_AS_STRING(bytesobj); - funcname = "addnstr"; - if (use_xy) + if (use_xy) { rtn = mvwaddnstr(self->win,y,x,str,n); - else + funcname = "mvwaddnstr"; + } + else { rtn = waddnstr(self->win,str,n); + funcname = "waddnstr"; + } Py_DECREF(bytesobj); } - if (use_attr) - (void)wattrset(self->win,attr_old); - return PyCursesCheckERR_ForWin(self, rtn, funcname); + if (rtn == ERR) { + curses_window_set_error(self, funcname, "addnstr"); + return NULL; + } + if (use_attr) { + rtn = wattrset(self->win, attr_old); + return curses_window_check_err(self, rtn, "wattrset", "addnstr"); + } + Py_RETURN_NONE; } /*[clinic input] @@ -1114,7 +1271,8 @@ _curses_window_bkgd_impl(PyCursesWindowObject *self, PyObject *ch, long attr) if (!PyCurses_ConvertToChtype(self, ch, &bkgd)) return NULL; - return PyCursesCheckERR_ForWin(self, wbkgd(self->win, bkgd | attr), "bkgd"); + int rtn = wbkgd(self->win, bkgd | attr); + return curses_window_check_err(self, rtn, "wbkgd", "bkgd"); } /*[clinic input] @@ -1130,7 +1288,8 @@ static PyObject * _curses_window_attroff_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=8a2fcd4df682fc64 input=786beedf06a7befe]*/ { - return PyCursesCheckERR_ForWin(self, wattroff(self->win, (attr_t)attr), "attroff"); + int rtn = wattroff(self->win, (attr_t)attr); + return curses_window_check_err(self, rtn, "wattroff", "attroff"); } /*[clinic input] @@ -1139,14 +1298,15 @@ _curses.window.attron attr: long / -Add attribute attr from the "background" set. +Add attribute attr to the "background" set. [clinic start generated code]*/ static PyObject * _curses_window_attron_impl(PyCursesWindowObject *self, long attr) -/*[clinic end generated code: output=7afea43b237fa870 input=5a88fba7b1524f32]*/ +/*[clinic end generated code: output=7afea43b237fa870 input=b57f824e1bf58326]*/ { - return PyCursesCheckERR_ForWin(self, wattron(self->win, (attr_t)attr), "attron"); + int rtn = wattron(self->win, (attr_t)attr); + return curses_window_check_err(self, rtn, "wattron", "attron"); } /*[clinic input] @@ -1162,7 +1322,8 @@ static PyObject * _curses_window_attrset_impl(PyCursesWindowObject *self, long attr) /*[clinic end generated code: output=84e379bff20c0433 input=42e400c0d0154ab5]*/ { - return PyCursesCheckERR_ForWin(self, wattrset(self->win, (attr_t)attr), "attrset"); + int rtn = wattrset(self->win, (attr_t)attr); + return curses_window_check_err(self, rtn, "wattrset", "attrset"); } /*[clinic input] @@ -1188,10 +1349,11 @@ _curses_window_bkgdset_impl(PyCursesWindowObject *self, PyObject *ch, return NULL; wbkgdset(self->win, bkgd | attr); - return PyCursesCheckERR_ForWin(self, 0, "bkgdset"); + Py_RETURN_NONE; } /*[clinic input] +@permit_long_docstring_body _curses.window.border ls: object(c_default="NULL") = _curses.ACS_VLINE @@ -1225,10 +1387,10 @@ _curses_window_border_impl(PyCursesWindowObject *self, PyObject *ls, PyObject *rs, PyObject *ts, PyObject *bs, PyObject *tl, PyObject *tr, PyObject *bl, PyObject *br) -/*[clinic end generated code: output=670ef38d3d7c2aa3 input=e015f735d67a240b]*/ +/*[clinic end generated code: output=670ef38d3d7c2aa3 input=adaafca87488ee35]*/ { chtype ch[8]; - int i; + int i, rtn; /* Clear the array of parameters */ for(i=0; i<8; i++) @@ -1249,10 +1411,10 @@ _curses_window_border_impl(PyCursesWindowObject *self, PyObject *ls, #undef CONVERTTOCHTYPE - wborder(self->win, - ch[0], ch[1], ch[2], ch[3], - ch[4], ch[5], ch[6], ch[7]); - Py_RETURN_NONE; + rtn = wborder(self->win, + ch[0], ch[1], ch[2], ch[3], + ch[4], ch[5], ch[6], ch[7]); + return curses_window_check_err(self, rtn, "wborder", "border"); } /*[clinic input] @@ -1286,8 +1448,7 @@ _curses_window_box_impl(PyCursesWindowObject *self, int group_right_1, return NULL; } } - box(self->win,ch1,ch2); - Py_RETURN_NONE; + return curses_window_check_err(self, box(self->win, ch1, ch2), "box", NULL); } #if defined(HAVE_NCURSES_H) || defined(MVWDELCH_IS_EXPRESSION) @@ -1312,38 +1473,34 @@ int py_mvwdelch(WINDOW *w, int y, int x) /* chgat, added by Fabian Kreutz <fabian.kreutz at gmx.net> */ #ifdef HAVE_CURSES_WCHGAT -/*[-clinic input] -_curses.window.chgat - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] +PyDoc_STRVAR(_curses_window_chgat__doc__, +"chgat([y, x,] [n=-1,] attr)\n" +"Set the attributes of characters.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Number of characters.\n" +" attr\n" +" Attributes for characters.\n" +"\n" +"Set the attributes of num characters at the current cursor position, or at\n" +"position (y, x) if supplied. If no value of num is given or num = -1, the\n" +"attribute will be set on all the characters to the end of the line. This\n" +"function does not move the cursor. The changed line will be touched using\n" +"the touchline() method so that the contents will be redisplayed by the next\n" +"window refresh."); - n: int = -1 - Number of characters. - - attr: long - Attributes for characters. - / - -Set the attributes of characters. - -Set the attributes of num characters at the current cursor position, or at -position (y, x) if supplied. If no value of num is given or num = -1, the -attribute will be set on all the characters to the end of the line. This -function does not move the cursor. The changed line will be touched using -the touchline() method so that the contents will be redisplayed by the next -window refresh. -[-clinic start generated code]*/ static PyObject * PyCursesWindow_ChgAt(PyObject *op, PyObject *args) { PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); int rtn; + const char *funcname; int x, y; int num = -1; short color; @@ -1363,19 +1520,20 @@ PyCursesWindow_ChgAt(PyObject *op, PyObject *args) attr = lattr; break; case 3: - if (!PyArg_ParseTuple(args,"iil;int,int,attr", &y, &x, &lattr)) + if (!PyArg_ParseTuple(args,"iil;y,x,attr", &y, &x, &lattr)) return NULL; attr = lattr; use_xy = TRUE; break; case 4: - if (!PyArg_ParseTuple(args,"iiil;int,int,n,attr", &y, &x, &num, &lattr)) + if (!PyArg_ParseTuple(args,"iiil;y,x,n,attr", &y, &x, &num, &lattr)) return NULL; attr = lattr; use_xy = TRUE; break; default: - PyErr_SetString(PyExc_TypeError, "chgat requires 1 to 4 arguments"); + PyErr_SetString(PyExc_TypeError, + "_curses.window.chgat requires 1 to 4 arguments"); return NULL; } @@ -1384,13 +1542,18 @@ PyCursesWindow_ChgAt(PyObject *op, PyObject *args) if (use_xy) { rtn = mvwchgat(self->win,y,x,num,attr,color,NULL); - touchline(self->win,y,1); + funcname = "mvwchgat"; } else { getyx(self->win,y,x); rtn = wchgat(self->win,num,attr,color,NULL); - touchline(self->win,y,1); + funcname = "wchgat"; } - return PyCursesCheckERR_ForWin(self, rtn, "chgat"); + if (rtn == ERR) { + curses_window_set_error(self, funcname, "chgat"); + return NULL; + } + rtn = touchline(self->win,y,1); + return curses_window_check_err(self, rtn, "touchline", "chgat"); } #endif @@ -1413,15 +1576,21 @@ _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) /*[clinic end generated code: output=22e77bb9fa11b461 input=d2f79e630a4fc6d0]*/ { + int rtn; + const char *funcname; if (!group_right_1) { - return PyCursesCheckERR_ForWin(self, wdelch(self->win), "wdelch"); + rtn = wdelch(self->win); + funcname = "wdelch"; } else { - return PyCursesCheckERR_ForWin(self, py_mvwdelch(self->win, y, x), "mvwdelch"); + rtn = py_mvwdelch(self->win, y, x); + funcname = "mvwdelch"; } + return curses_window_check_err(self, rtn, funcname, "delch"); } /*[clinic input] +@permit_long_docstring_body _curses.window.derwin [ @@ -1446,15 +1615,14 @@ screen. static PyObject * _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=7924b112d9f70d6e input=966d9481f7f5022e]*/ +/*[clinic end generated code: output=7924b112d9f70d6e input=ebe95ded1c284c8e]*/ { WINDOW *win; win = derwin(self->win,nlines,ncols,begin_y,begin_x); if (win == NULL) { - cursesmodule_state *state = get_cursesmodule_state_by_win(self); - PyErr_SetString(state->error, catchall_NULL); + curses_window_set_null_error(self, "derwin", NULL); return NULL; } @@ -1485,21 +1653,25 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, if (!PyCurses_ConvertToChtype(self, ch, &ch_)) return NULL; + int rtn; + const char *funcname; #ifdef py_is_pad if (py_is_pad(self->win)) { - return PyCursesCheckERR_ForWin(self, - pechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + rtn = pechochar(self->win, ch_ | (attr_t)attr); + funcname = "pechochar"; } else #endif - return PyCursesCheckERR_ForWin(self, - wechochar(self->win, ch_ | (attr_t)attr), - "echochar"); + { + rtn = wechochar(self->win, ch_ | (attr_t)attr); + funcname = "wechochar"; + } + return curses_window_check_err(self, rtn, funcname, "echochar"); } #ifdef NCURSES_MOUSE_VERSION /*[clinic input] +@permit_long_summary _curses.window.enclose y: int @@ -1513,27 +1685,48 @@ Return True if the screen-relative coordinates are enclosed by the window. static PyObject * _curses_window_enclose_impl(PyCursesWindowObject *self, int y, int x) -/*[clinic end generated code: output=8679beef50502648 input=4fd3355d723f7bc9]*/ +/*[clinic end generated code: output=8679beef50502648 input=9ba7c894cffe5507]*/ { return PyBool_FromLong(wenclose(self->win, y, x)); } #endif /*[clinic input] -_curses.window.getbkgd -> long +_curses.window.getbkgd Return the window's current background character/attribute pair. [clinic start generated code]*/ -static long +static PyObject * _curses_window_getbkgd_impl(PyCursesWindowObject *self) -/*[clinic end generated code: output=c52b25dc16b215c3 input=a69db882fa35426c]*/ +/*[clinic end generated code: output=3ff953412b0e6028 input=7cf1f59a31f89df4]*/ { - return (long) getbkgd(self->win); + chtype rtn = getbkgd(self->win); + if (rtn == (chtype)ERR) { + curses_window_set_error(self, "getbkgd", NULL); + return NULL; + } + return PyLong_FromLong(rtn); +} + +static PyObject * +curses_check_signals_on_input_error(PyCursesWindowObject *self, + const char *curses_funcname, + const char *python_funcname) +{ + assert(!PyErr_Occurred()); + if (PyErr_CheckSignals()) { + return NULL; + } + cursesmodule_state *state = get_cursesmodule_state_by_win(self); + PyErr_Format(state->error, "%s() (called by %s()): no input", + curses_funcname, python_funcname); + return NULL; } /*[clinic input] -_curses.window.getch -> int +@permit_long_docstring_body +_curses.window.getch [ y: int @@ -1550,10 +1743,10 @@ keypad keys and so on return numbers higher than 256. In no-delay mode, -1 is returned if there is no input, else getch() waits until a key is pressed. [clinic start generated code]*/ -static int +static PyObject * _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=980aa6af0c0ca387 input=bb24ebfb379f991f]*/ +/*[clinic end generated code: output=e1639e87d545e676 input=9a053077373e2a30]*/ { int rtn; @@ -1566,10 +1759,21 @@ _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, } Py_END_ALLOW_THREADS - return rtn; + if (rtn == ERR) { + // We suppress ERR returned by wgetch() in nodelay mode + // after we handled possible interruption signals. + if (PyErr_CheckSignals()) { + return NULL; + } + // ERR is an implementation detail, so to be on the safe side, + // we forcibly set the return value to -1 as documented above. + rtn = -1; + } + return PyLong_FromLong(rtn); } /*[clinic input] +@permit_long_docstring_body _curses.window.getkey [ @@ -1590,7 +1794,7 @@ key name. In no-delay mode, an exception is raised if there is no input. static PyObject * _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=8490a182db46b10f input=be2dee34f5cf57f8]*/ +/*[clinic end generated code: output=8490a182db46b10f input=5177f03fb6c31ea6]*/ { int rtn; @@ -1604,13 +1808,9 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, Py_END_ALLOW_THREADS if (rtn == ERR) { - /* getch() returns ERR in nodelay mode */ - PyErr_CheckSignals(); - if (!PyErr_Occurred()) { - cursesmodule_state *state = get_cursesmodule_state_by_win(self); - PyErr_SetString(state->error, "no input"); - } - return NULL; + /* wgetch() returns ERR in nodelay mode */ + const char *funcname = group_right_1 ? "mvwgetch" : "wgetch"; + return curses_check_signals_on_input_error(self, funcname, "getkey"); } else if (rtn <= 255) { #ifdef NCURSES_VERSION_MAJOR #if NCURSES_VERSION_MAJOR*100+NCURSES_VERSION_MINOR <= 507 @@ -1663,13 +1863,9 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, Py_END_ALLOW_THREADS if (ct == ERR) { - if (PyErr_CheckSignals()) - return NULL; - - /* get_wch() returns ERR in nodelay mode */ - cursesmodule_state *state = get_cursesmodule_state_by_win(self); - PyErr_SetString(state->error, "no input"); - return NULL; + /* wget_wch() returns ERR in nodelay mode */ + const char *funcname = group_right_1 ? "mvwget_wch" : "wget_wch"; + return curses_check_signals_on_input_error(self, funcname, "get_wch"); } if (ct == KEY_CODE_YES) return PyLong_FromLong(rtn); @@ -1678,84 +1874,100 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, } #endif -/*[-clinic input] -_curses.window.getstr +/* + * Helper function for parsing parameters from getstr() and instr(). + * This function is necessary because Argument Clinic does not know + * how to handle nested optional groups with default values inside. + * + * Return 1 on success and 0 on failure, similar to PyArg_ParseTuple(). + */ +static int +curses_clinic_parse_optional_xy_n(PyObject *args, + int *y, int *x, unsigned int *n, int *use_xy, + const char *qualname) +{ + switch (PyTuple_GET_SIZE(args)) { + case 0: { + *use_xy = 0; + return 1; + } + case 1: { + *use_xy = 0; + return PyArg_ParseTuple(args, "O&;n", + _PyLong_UnsignedInt_Converter, n); + } + case 2: { + *use_xy = 1; + return PyArg_ParseTuple(args, "ii;y,x", y, x); + } + case 3: { + *use_xy = 1; + return PyArg_ParseTuple(args, "iiO&;y,x,n", y, x, + _PyLong_UnsignedInt_Converter, n); + } + default: { + *use_xy = 0; + PyErr_Format(PyExc_TypeError, "%s requires 0 to 3 arguments", + qualname); + return 0; + } + } +} - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] - n: int = 1023 - Maximal number of characters. - / - -Read a string from the user, with primitive line editing capacity. -[-clinic start generated code]*/ +PyDoc_STRVAR(_curses_window_getstr__doc__, +"getstr([[y, x,] n=2047])\n" +"Read a string from the user, with primitive line editing capacity.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Maximal number of characters."); static PyObject * -PyCursesWindow_GetStr(PyObject *op, PyObject *args) +PyCursesWindow_getstr(PyObject *op, PyObject *args) { PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn, use_xy = 0, y = 0, x = 0; + unsigned int max_buf_size = 2048; + unsigned int n = max_buf_size - 1; - int x, y, n; - char rtn[1024]; /* This should be big enough.. I hope */ - int rtn2; - - switch (PyTuple_Size(args)) { - case 0: - Py_BEGIN_ALLOW_THREADS - rtn2 = wgetnstr(self->win,rtn, 1023); - Py_END_ALLOW_THREADS - break; - case 1: - if (!PyArg_ParseTuple(args,"i;n", &n)) - return NULL; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; - } - Py_BEGIN_ALLOW_THREADS - rtn2 = wgetnstr(self->win, rtn, Py_MIN(n, 1023)); - Py_END_ALLOW_THREADS - break; - case 2: - if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) - return NULL; - Py_BEGIN_ALLOW_THREADS -#ifdef STRICT_SYSV_CURSES - rtn2 = wmove(self->win,y,x)==ERR ? ERR : wgetnstr(self->win, rtn, 1023); -#else - rtn2 = mvwgetnstr(self->win,y,x,rtn, 1023); -#endif - Py_END_ALLOW_THREADS - break; - case 3: - if (!PyArg_ParseTuple(args,"iii;y,x,n", &y, &x, &n)) - return NULL; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; - } -#ifdef STRICT_SYSV_CURSES - Py_BEGIN_ALLOW_THREADS - rtn2 = wmove(self->win,y,x)==ERR ? ERR : - wgetnstr(self->win, rtn, Py_MIN(n, 1023)); - Py_END_ALLOW_THREADS -#else - Py_BEGIN_ALLOW_THREADS - rtn2 = mvwgetnstr(self->win, y, x, rtn, Py_MIN(n, 1023)); - Py_END_ALLOW_THREADS -#endif - break; - default: - PyErr_SetString(PyExc_TypeError, "getstr requires 0 to 3 arguments"); + if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, + "_curses.window.instr")) + { return NULL; } - if (rtn2 == ERR) - rtn[0] = 0; - return PyBytes_FromString(rtn); + + n = Py_MIN(n, max_buf_size - 1); + PyBytesWriter *writer = PyBytesWriter_Create(n + 1); + if (writer == NULL) { + return NULL; + } + char *buf = PyBytesWriter_GetData(writer); + + if (use_xy) { + Py_BEGIN_ALLOW_THREADS +#ifdef STRICT_SYSV_CURSES + rtn = wmove(self->win, y, x) == ERR + ? ERR + : wgetnstr(self->win, buf, n); +#else + rtn = mvwgetnstr(self->win, y, x, buf, n); +#endif + Py_END_ALLOW_THREADS + } + else { + Py_BEGIN_ALLOW_THREADS + rtn = wgetnstr(self->win, buf, n); + Py_END_ALLOW_THREADS + } + + if (rtn == ERR) { + PyBytesWriter_Discard(writer); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + return PyBytesWriter_FinishWithSize(writer, strlen(buf)); } /*[clinic input] @@ -1794,13 +2006,16 @@ _curses_window_hline_impl(PyCursesWindowObject *self, int group_left_1, return NULL; if (group_left_1) { if (wmove(self->win, y, x) == ERR) { - return PyCursesCheckERR_ForWin(self, ERR, "wmove"); + curses_window_set_error(self, "wmove", "hline"); + return NULL; } } - return PyCursesCheckERR_ForWin(self, whline(self->win, ch_ | (attr_t)attr, n), "hline"); + int rtn = whline(self->win, ch_ | (attr_t)attr, n); + return curses_window_check_err(self, rtn, "whline", "hline"); } /*[clinic input] +@permit_long_docstring_body _curses.window.insch [ @@ -1829,7 +2044,7 @@ static PyObject * _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *ch, int group_right_1, long attr) -/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=336342756ee19812]*/ +/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=3f2a230cb09fed5a]*/ { int rtn; chtype ch_ = 0; @@ -1837,18 +2052,22 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, if (!PyCurses_ConvertToChtype(self, ch, &ch_)) return NULL; + const char *funcname; if (!group_left_1) { rtn = winsch(self->win, ch_ | (attr_t)attr); + funcname = "winsch"; } else { rtn = mvwinsch(self->win, y, x, ch_ | (attr_t)attr); + funcname = "mvwwinsch"; } - return PyCursesCheckERR_ForWin(self, rtn, "insch"); + return curses_window_check_err(self, rtn, funcname, "insch"); } /*[clinic input] -_curses.window.inch -> unsigned_long +@permit_long_docstring_body +_curses.window.inch [ y: int @@ -1863,86 +2082,78 @@ Return the character at the given position in the window. The bottom 8 bits are the character proper, and upper bits are the attributes. [clinic start generated code]*/ -static unsigned long +static PyObject * _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=6c4719fe978fe86a input=fac23ee11e3b3a66]*/ +/*[clinic end generated code: output=97ca8581baaafd06 input=a5846f315464dc86]*/ { - unsigned long rtn; + chtype rtn; + const char *funcname; if (!group_right_1) { rtn = winch(self->win); + funcname = "winch"; } else { rtn = mvwinch(self->win, y, x); + funcname = "mvwinch"; } - - return rtn; -} - -/*[-clinic input] -_curses.window.instr - - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] - n: int = 1023 - Maximal number of characters. - / - -Return a string of characters, extracted from the window. - -Return a string of characters, extracted from the window starting at the -current cursor position, or at y, x if specified. Attributes are stripped -from the characters. If n is specified, instr() returns a string at most -n characters long (exclusive of the trailing NUL). -[-clinic start generated code]*/ -static PyObject * -PyCursesWindow_InStr(PyObject *op, PyObject *args) -{ - PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); - - int x, y, n; - char rtn[1024]; /* This should be big enough.. I hope */ - int rtn2; - - switch (PyTuple_Size(args)) { - case 0: - rtn2 = winnstr(self->win,rtn, 1023); - break; - case 1: - if (!PyArg_ParseTuple(args,"i;n", &n)) - return NULL; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; - } - rtn2 = winnstr(self->win, rtn, Py_MIN(n, 1023)); - break; - case 2: - if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) - return NULL; - rtn2 = mvwinnstr(self->win,y,x,rtn,1023); - break; - case 3: - if (!PyArg_ParseTuple(args, "iii;y,x,n", &y, &x, &n)) - return NULL; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; - } - rtn2 = mvwinnstr(self->win, y, x, rtn, Py_MIN(n,1023)); - break; - default: - PyErr_SetString(PyExc_TypeError, "instr requires 0 or 3 arguments"); + if (rtn == (chtype)ERR) { + curses_window_set_error(self, funcname, "inch"); return NULL; } - if (rtn2 == ERR) - rtn[0] = 0; - return PyBytes_FromString(rtn); + return PyLong_FromUnsignedLong(rtn); +} + +PyDoc_STRVAR(_curses_window_instr__doc__, +"instr([y, x,] n=2047)\n" +"Return a string of characters, extracted from the window.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Maximal number of characters.\n" +"\n" +"Return a string of characters, extracted from the window starting at the\n" +"current cursor position, or at y, x if specified. Attributes are stripped\n" +"from the characters. If n is specified, instr() returns a string at most\n" +"n characters long (exclusive of the trailing NUL)."); + +static PyObject * +PyCursesWindow_instr(PyObject *op, PyObject *args) +{ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn, use_xy = 0, y = 0, x = 0; + unsigned int max_buf_size = 2048; + unsigned int n = max_buf_size - 1; + + if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, + "_curses.window.instr")) + { + return NULL; + } + + n = Py_MIN(n, max_buf_size - 1); + PyBytesWriter *writer = PyBytesWriter_Create(n + 1); + if (writer == NULL) { + return NULL; + } + char *buf = PyBytesWriter_GetData(writer); + + if (use_xy) { + rtn = mvwinnstr(self->win, y, x, buf, n); + } + else { + rtn = winnstr(self->win, buf, n); + } + + if (rtn == ERR) { + PyBytesWriter_Discard(writer); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + return PyBytesWriter_FinishWithSize(writer, strlen(buf)); } /*[clinic input] @@ -1999,34 +2210,50 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, if (use_attr) { attr_old = getattrs(self->win); - (void)wattrset(self->win, (attr_t)attr); + if (curses_wattrset(self, attr, "insstr") < 0) { + curses_release_wstr(strtype, wstr); + return NULL; + } } #ifdef HAVE_NCURSESW if (strtype == 2) { - funcname = "inswstr"; - if (use_xy) + if (use_xy) { rtn = mvwins_wstr(self->win,y,x,wstr); - else + funcname = "mvwins_wstr"; + } + else { rtn = wins_wstr(self->win,wstr); + funcname = "wins_wstr"; + } PyMem_Free(wstr); } else #endif { const char *str = PyBytes_AS_STRING(bytesobj); - funcname = "insstr"; - if (use_xy) + if (use_xy) { rtn = mvwinsstr(self->win,y,x,str); - else + funcname = "mvwinsstr"; + } + else { rtn = winsstr(self->win,str); + funcname = "winsstr"; + } Py_DECREF(bytesobj); } - if (use_attr) - (void)wattrset(self->win,attr_old); - return PyCursesCheckERR_ForWin(self, rtn, funcname); + if (rtn == ERR) { + curses_window_set_error(self, funcname, "insstr"); + return NULL; + } + if (use_attr) { + rtn = wattrset(self->win, attr_old); + return curses_window_check_err(self, rtn, "wattrset", "insstr"); + } + Py_RETURN_NONE; } /*[clinic input] +@permit_long_docstring_body _curses.window.insnstr [ @@ -2062,7 +2289,7 @@ static PyObject * _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int n, int group_right_1, long attr) -/*[clinic end generated code: output=971a32ea6328ec8b input=70fa0cd543901a4c]*/ +/*[clinic end generated code: output=971a32ea6328ec8b input=dcdc554102fbcd5d]*/ { int rtn; int strtype; @@ -2084,34 +2311,50 @@ _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, if (use_attr) { attr_old = getattrs(self->win); - (void)wattrset(self->win, (attr_t)attr); + if (curses_wattrset(self, attr, "insnstr") < 0) { + curses_release_wstr(strtype, wstr); + return NULL; + } } #ifdef HAVE_NCURSESW if (strtype == 2) { - funcname = "insn_wstr"; - if (use_xy) + if (use_xy) { rtn = mvwins_nwstr(self->win,y,x,wstr,n); - else + funcname = "mvwins_nwstr"; + } + else { rtn = wins_nwstr(self->win,wstr,n); + funcname = "wins_nwstr"; + } PyMem_Free(wstr); } else #endif { const char *str = PyBytes_AS_STRING(bytesobj); - funcname = "insnstr"; - if (use_xy) + if (use_xy) { rtn = mvwinsnstr(self->win,y,x,str,n); - else + funcname = "mvwinsnstr"; + } + else { rtn = winsnstr(self->win,str,n); + funcname = "winsnstr"; + } Py_DECREF(bytesobj); } - if (use_attr) - (void)wattrset(self->win,attr_old); - return PyCursesCheckERR_ForWin(self, rtn, funcname); + if (rtn == ERR) { + curses_window_set_error(self, funcname, "insnstr"); + return NULL; + } + if (use_attr) { + rtn = wattrset(self->win, attr_old); + return curses_window_check_err(self, rtn, "wattrset", "insnstr"); + } + Py_RETURN_NONE; } /*[clinic input] +@permit_long_docstring_body _curses.window.is_linetouched line: int @@ -2125,13 +2368,12 @@ Raise a curses.error exception if line is not valid for the given window. static PyObject * _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line) -/*[clinic end generated code: output=ad4a4edfee2db08c input=a7be0c189f243914]*/ +/*[clinic end generated code: output=ad4a4edfee2db08c input=af71c040b951c467]*/ { int erg; erg = is_linetouched(self->win, line); if (erg == ERR) { - PyErr_SetString(PyExc_TypeError, - "is_linetouched: line number outside of boundaries"); + curses_window_set_error(self, "is_linetouched", NULL); return NULL; } return PyBool_FromLong(erg); @@ -2139,6 +2381,7 @@ _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line) #ifdef py_is_pad /*[clinic input] +@permit_long_docstring_body _curses.window.noutrefresh [ @@ -2163,9 +2406,10 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=809a1f3c6a03e23e input=3e56898388cd739e]*/ +/*[clinic end generated code: output=809a1f3c6a03e23e input=b39fe8fc79b9980b]*/ #else /*[clinic input] +@permit_long_docstring_body _curses.window.noutrefresh Mark for refresh but wait. @@ -2177,7 +2421,7 @@ that, call doupdate(). static PyObject * _curses_window_noutrefresh_impl(PyCursesWindowObject *self) -/*[clinic end generated code: output=6ef6dec666643fee input=876902e3fa431dbd]*/ +/*[clinic end generated code: output=6ef6dec666643fee input=6a9f59ae5e4c139e]*/ #endif { int rtn; @@ -2185,8 +2429,7 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - cursesmodule_state *state = get_cursesmodule_state_by_win(self); - PyErr_SetString(state->error, + PyErr_SetString(PyExc_TypeError, "noutrefresh() called for a pad " "requires 6 arguments"); return NULL; @@ -2195,7 +2438,8 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) rtn = pnoutrefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR_ForWin(self, rtn, "pnoutrefresh"); + return curses_window_check_err(self, rtn, + "pnoutrefresh", "noutrefresh"); } if (group_right_1) { PyErr_SetString(PyExc_TypeError, @@ -2206,10 +2450,11 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) Py_BEGIN_ALLOW_THREADS rtn = wnoutrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR_ForWin(self, rtn, "wnoutrefresh"); + return curses_window_check_err(self, rtn, "wnoutrefresh", "noutrefresh"); } /*[clinic input] +@permit_long_docstring_body _curses.window.overlay destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") @@ -2241,22 +2486,23 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, PyCursesWindowObject *destwin, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=82bb2c4cb443ca58 input=6e4b32a7c627a356]*/ +/*[clinic end generated code: output=82bb2c4cb443ca58 input=dd6af34deb892a65]*/ { int rtn; if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, TRUE); - return PyCursesCheckERR_ForWin(self, rtn, "copywin"); + return curses_window_check_err(self, rtn, "copywin", "overlay"); } else { rtn = overlay(self->win, destwin->win); - return PyCursesCheckERR_ForWin(self, rtn, "overlay"); + return curses_window_check_err(self, rtn, "overlay", NULL); } } /*[clinic input] +@permit_long_docstring_body _curses.window.overwrite destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") @@ -2289,18 +2535,18 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=12ae007d1681be28 input=d83dd8b24ff2bcc9]*/ +/*[clinic end generated code: output=12ae007d1681be28 input=e84d8ebdf1c09596]*/ { int rtn; if (group_right_1) { rtn = copywin(self->win, destwin->win, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol, FALSE); - return PyCursesCheckERR_ForWin(self, rtn, "copywin"); + return curses_window_check_err(self, rtn, "copywin", "overwrite"); } else { rtn = overwrite(self->win, destwin->win); - return PyCursesCheckERR_ForWin(self, rtn, "overwrite"); + return curses_window_check_err(self, rtn, "overwrite", NULL); } } @@ -2329,7 +2575,7 @@ _curses_window_putwin_impl(PyCursesWindowObject *self, PyObject *file) return PyErr_SetFromErrno(PyExc_OSError); if (_Py_set_inheritable(fileno(fp), 0, NULL) < 0) goto exit; - res = PyCursesCheckERR_ForWin(self, putwin(self->win, fp), "putwin"); + res = curses_window_check_err(self, putwin(self->win, fp), "putwin", NULL); if (res == NULL) goto exit; fseek(fp, 0, 0); @@ -2368,10 +2614,12 @@ static PyObject * _curses_window_redrawln_impl(PyCursesWindowObject *self, int beg, int num) /*[clinic end generated code: output=ea216e334f9ce1b4 input=152155e258a77a7a]*/ { - return PyCursesCheckERR_ForWin(self, wredrawln(self->win,beg,num), "redrawln"); + int rtn = wredrawln(self->win,beg, num); + return curses_window_check_err(self, rtn, "wredrawln", "redrawln"); } /*[clinic input] +@permit_long_docstring_body _curses.window.refresh [ @@ -2403,15 +2651,14 @@ static PyObject * _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=42199543115e6e63 input=95e01cb5ffc635d0]*/ +/*[clinic end generated code: output=42199543115e6e63 input=65405c03290496a6]*/ { int rtn; #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { - cursesmodule_state *state = get_cursesmodule_state_by_win(self); - PyErr_SetString(state->error, + PyErr_SetString(PyExc_TypeError, "refresh() for a pad requires 6 arguments"); return NULL; } @@ -2419,7 +2666,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, rtn = prefresh(self->win, pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol); Py_END_ALLOW_THREADS - return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); + return curses_window_check_err(self, rtn, "prefresh", "refresh"); } #endif if (group_right_1) { @@ -2430,7 +2677,7 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, Py_BEGIN_ALLOW_THREADS rtn = wrefresh(self->win); Py_END_ALLOW_THREADS - return PyCursesCheckERR_ForWin(self, rtn, "prefresh"); + return curses_window_check_err(self, rtn, "wrefresh", "refresh"); } /*[clinic input] @@ -2452,10 +2699,12 @@ _curses_window_setscrreg_impl(PyCursesWindowObject *self, int top, int bottom) /*[clinic end generated code: output=486ab5db218d2b1a input=1b517b986838bf0e]*/ { - return PyCursesCheckERR_ForWin(self, wsetscrreg(self->win, top, bottom), "wsetscrreg"); + int rtn = wsetscrreg(self->win, top, bottom); + return curses_window_check_err(self, rtn, "wsetscrreg", "setscrreg"); } /*[clinic input] +@permit_long_docstring_body _curses.window.subwin [ @@ -2479,22 +2728,26 @@ lower right corner of the window. static PyObject * _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=93e898afc348f59a input=2129fa47fd57721c]*/ +/*[clinic end generated code: output=93e898afc348f59a input=5292cf610e2f3585]*/ { WINDOW *win; + const char *funcname; /* printf("Subwin: %i %i %i %i \n", nlines, ncols, begin_y, begin_x); */ #ifdef py_is_pad if (py_is_pad(self->win)) { win = subpad(self->win, nlines, ncols, begin_y, begin_x); + funcname = "subpad"; } else #endif + { win = subwin(self->win, nlines, ncols, begin_y, begin_x); + funcname = "subwin"; + } if (win == NULL) { - cursesmodule_state *state = get_cursesmodule_state_by_win(self); - PyErr_SetString(state->error, catchall_NULL); + curses_window_set_null_error(self, funcname, "subwin"); return NULL; } @@ -2503,6 +2756,7 @@ _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] +@permit_long_docstring_body _curses.window.scroll [ @@ -2519,17 +2773,23 @@ Scroll upward if the argument is positive and downward if it is negative. static PyObject * _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, int lines) -/*[clinic end generated code: output=4541a8a11852d360 input=c969ca0cfabbdbec]*/ +/*[clinic end generated code: output=4541a8a11852d360 input=386456524c550113]*/ { + int rtn; + const char *funcname; if (!group_right_1) { - return PyCursesCheckERR_ForWin(self, scroll(self->win), "scroll"); + rtn = scroll(self->win); + funcname = "scroll"; } else { - return PyCursesCheckERR_ForWin(self, wscrl(self->win, lines), "scroll"); + rtn = wscrl(self->win, lines); + funcname = "wscrl"; } + return curses_window_check_err(self, rtn, funcname, "scroll"); } /*[clinic input] +@permit_long_docstring_body _curses.window.touchline start: int @@ -2548,14 +2808,19 @@ as having been changed (changed=True) or unchanged (changed=False). static PyObject * _curses_window_touchline_impl(PyCursesWindowObject *self, int start, int count, int group_right_1, int changed) -/*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ +/*[clinic end generated code: output=65d05b3f7438c61d input=36e13b6f5eb591f5]*/ { + int rtn; + const char *funcname; if (!group_right_1) { - return PyCursesCheckERR_ForWin(self, touchline(self->win, start, count), "touchline"); + rtn = touchline(self->win, start, count); + funcname = "touchline"; } else { - return PyCursesCheckERR_ForWin(self, wtouchln(self->win, start, count, changed), "touchline"); + rtn = wtouchln(self->win, start, count, changed); + funcname = "wtouchln"; } + return curses_window_check_err(self, rtn, funcname, "touchline"); } /*[clinic input] @@ -2593,10 +2858,13 @@ _curses_window_vline_impl(PyCursesWindowObject *self, int group_left_1, if (!PyCurses_ConvertToChtype(self, ch, &ch_)) return NULL; if (group_left_1) { - if (wmove(self->win, y, x) == ERR) - return PyCursesCheckERR_ForWin(self, ERR, "wmove"); + if (wmove(self->win, y, x) == ERR) { + curses_window_set_error(self, "wmove", "vline"); + return NULL; + } } - return PyCursesCheckERR_ForWin(self, wvline(self->win, ch_ | (attr_t)attr, n), "vline"); + int rtn = wvline(self->win, ch_ | (attr_t)attr, n); + return curses_window_check_err(self, rtn, "wvline", "vline"); } static PyObject * @@ -2653,7 +2921,10 @@ static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ATTRSET_METHODDEF _CURSES_WINDOW_BKGD_METHODDEF #ifdef HAVE_CURSES_WCHGAT - {"chgat", PyCursesWindow_ChgAt, METH_VARARGS}, + { + "chgat", PyCursesWindow_ChgAt, METH_VARARGS, + _curses_window_chgat__doc__ + }, #endif _CURSES_WINDOW_BKGDSET_METHODDEF _CURSES_WINDOW_BORDER_METHODDEF @@ -2676,7 +2947,10 @@ static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_GET_WCH_METHODDEF {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS}, {"getparyx", PyCursesWindow_getparyx, METH_NOARGS}, - {"getstr", PyCursesWindow_GetStr, METH_VARARGS}, + { + "getstr", PyCursesWindow_getstr, METH_VARARGS, + _curses_window_getstr__doc__ + }, {"getyx", PyCursesWindow_getyx, METH_NOARGS}, _CURSES_WINDOW_HLINE_METHODDEF {"idcok", PyCursesWindow_idcok, METH_VARARGS}, @@ -2690,7 +2964,10 @@ static PyMethodDef PyCursesWindow_methods[] = { {"insertln", PyCursesWindow_winsertln, METH_NOARGS}, _CURSES_WINDOW_INSNSTR_METHODDEF _CURSES_WINDOW_INSSTR_METHODDEF - {"instr", PyCursesWindow_InStr, METH_VARARGS}, + { + "instr", PyCursesWindow_instr, METH_VARARGS, + _curses_window_instr__doc__ + }, _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF {"is_wintouched", PyCursesWindow_is_wintouched, METH_NOARGS}, {"keypad", PyCursesWindow_keypad, METH_VARARGS}, @@ -2761,49 +3038,78 @@ static PyType_Spec PyCursesWindow_Type_spec = { /* -------------------------------------------------------*/ -/* Function Body Macros - They are ugly but very, very useful. ;-) +/* + * Macros for implementing simple module's methods. + * + * Parameters + * + * X The name of the curses C function or macro to invoke. + * FLAG When false, prefixes the function name with 'no' at runtime, + * This parameter is present in the signature and auto-generated + * by Argument Clinic. + * + * These macros should only be used for generating the body of + * the module's methods since they need a module reference. + * + * The Python function name must be the same as the curses function name (X). + */ - X - function name - TYPE - parameter Type - ERGSTR - format string for construction of the return value - PARSESTR - format string for argument parsing - */ - -#define NoArgNoReturnFunctionBody(X) \ -{ \ - PyCursesStatefulInitialised(module); \ - return PyCursesCheckERR(module, X(), # X); } - -#define NoArgOrFlagNoReturnFunctionBody(X, flag) \ -{ \ - PyCursesStatefulInitialised(module); \ - if (flag) \ - return PyCursesCheckERR(module, X(), # X); \ - else \ - return PyCursesCheckERR(module, no ## X(), # X); \ +#define NoArgNoReturnFunctionBody(X) \ +{ \ + PyCursesStatefulInitialised(module); \ + return curses_check_err(module, X(), # X, NULL); \ } -#define NoArgReturnIntFunctionBody(X) \ -{ \ - PyCursesStatefulInitialised(module); \ - return PyLong_FromLong((long) X()); } +#define NoArgOrFlagNoReturnFunctionBody(X, FLAG) \ +{ \ + PyCursesStatefulInitialised(module); \ + int rtn; \ + const char *funcname; \ + if (FLAG) { \ + rtn = X(); \ + funcname = # X; \ + } \ + else { \ + rtn = no ## X(); \ + funcname = "no" # X; \ + } \ + return curses_check_err(module, rtn, funcname, # X); \ +} +#define NoArgReturnIntFunctionBody(X) \ +{ \ + PyCursesStatefulInitialised(module); \ + int rtn = X(); \ + if (rtn == ERR) { \ + curses_set_error(module, # X, NULL); \ + return NULL; \ + } \ + return PyLong_FromLong(rtn); \ +} -#define NoArgReturnStringFunctionBody(X) \ -{ \ - PyCursesStatefulInitialised(module); \ - return PyBytes_FromString(X()); } +#define NoArgReturnStringFunctionBody(X) \ +{ \ + PyCursesStatefulInitialised(module); \ + const char *res = X(); \ + if (res == NULL) { \ + curses_set_null_error(module, # X, NULL); \ + return NULL; \ + } \ + return PyBytes_FromString(res); \ +} -#define NoArgTrueFalseFunctionBody(X) \ -{ \ - PyCursesStatefulInitialised(module); \ - return PyBool_FromLong(X()); } +#define NoArgTrueFalseFunctionBody(X) \ +{ \ + PyCursesStatefulInitialised(module); \ + return PyBool_FromLong(X()); \ +} -#define NoArgNoReturnVoidFunctionBody(X) \ -{ \ - PyCursesStatefulInitialised(module); \ - X(); \ - Py_RETURN_NONE; } +#define NoArgNoReturnVoidFunctionBody(X) \ +{ \ + PyCursesStatefulInitialised(module); \ + X(); \ + Py_RETURN_NONE; \ +} /********************************************************************* Global Functions @@ -2849,6 +3155,7 @@ _curses_beep_impl(PyObject *module) NoArgNoReturnFunctionBody(beep) /*[clinic input] +@permit_long_summary _curses.can_change_color Return True if the programmer can change the colors displayed by the terminal. @@ -2856,7 +3163,7 @@ Return True if the programmer can change the colors displayed by the terminal. static PyObject * _curses_can_change_color_impl(PyObject *module) -/*[clinic end generated code: output=359df8c3c77d8bf1 input=d7718884de0092f2]*/ +/*[clinic end generated code: output=359df8c3c77d8bf1 input=8315c364ba1e5b4c]*/ NoArgTrueFalseFunctionBody(can_change_color) /*[clinic input] @@ -2881,6 +3188,7 @@ _curses_cbreak_impl(PyObject *module, int flag) NoArgOrFlagNoReturnFunctionBody(cbreak, flag) /*[clinic input] +@permit_long_docstring_body _curses.color_content color_number: color @@ -2895,7 +3203,7 @@ which will be between 0 (no component) and 1000 (maximum amount of component). static PyObject * _curses_color_content_impl(PyObject *module, int color_number) -/*[clinic end generated code: output=17b466df7054e0de input=03b5ed0472662aea]*/ +/*[clinic end generated code: output=17b466df7054e0de input=baffe25b351eb916]*/ { _CURSES_COLOR_VAL_TYPE r,g,b; @@ -2903,9 +3211,8 @@ _curses_color_content_impl(PyObject *module, int color_number) PyCursesStatefulInitialisedColor(module); if (_COLOR_CONTENT_FUNC(color_number, &r, &g, &b) == ERR) { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_Format(state->error, "%s() returned ERR", - Py_STRINGIFY(_COLOR_CONTENT_FUNC)); + const char *funcname = Py_STRINGIFY(_COLOR_CONTENT_FUNC); + curses_set_error(module, funcname, "color_content"); return NULL; } @@ -2959,7 +3266,10 @@ _curses_curs_set_impl(PyObject *module, int visibility) PyCursesStatefulInitialised(module); erg = curs_set(visibility); - if (erg == ERR) return PyCursesCheckERR(module, erg, "curs_set"); + if (erg == ERR) { + curses_set_error(module, "curs_set", NULL); + return NULL; + } return PyLong_FromLong((long) erg); } @@ -3010,7 +3320,7 @@ _curses_delay_output_impl(PyObject *module, int ms) { PyCursesStatefulInitialised(module); - return PyCursesCheckERR(module, delay_output(ms), "delay_output"); + return curses_check_err(module, delay_output(ms), "delay_output", NULL); } /*[clinic input] @@ -3072,6 +3382,7 @@ _curses_erasechar_impl(PyObject *module) } /*[clinic input] +@permit_long_docstring_body _curses.flash Flash the screen. @@ -3081,7 +3392,7 @@ That is, change it to reverse-video and then change it back in a short interval. static PyObject * _curses_flash_impl(PyObject *module) -/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=02fdfb06c8fc3171]*/ +/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=dd33d718e6edf436]*/ NoArgNoReturnFunctionBody(flash) /*[clinic input] @@ -3143,8 +3454,7 @@ _curses_getmouse_impl(PyObject *module) rtn = getmouse( &event ); if (rtn == ERR) { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, "getmouse() returned ERR"); + curses_set_error(module, "getmouse", NULL); return NULL; } return Py_BuildValue("(hiiik)", @@ -3182,7 +3492,7 @@ _curses_ungetmouse_impl(PyObject *module, short id, int x, int y, int z, event.y = y; event.z = z; event.bstate = bstate; - return PyCursesCheckERR(module, ungetmouse(&event), "ungetmouse"); + return curses_check_err(module, ungetmouse(&event), "ungetmouse", NULL); } #endif @@ -3238,8 +3548,7 @@ _curses_getwin(PyObject *module, PyObject *file) fseek(fp, 0, 0); win = getwin(fp); if (win == NULL) { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, catchall_NULL); + curses_set_null_error(module, "getwin", NULL); goto error; } cursesmodule_state *state = get_cursesmodule_state(module); @@ -3268,7 +3577,7 @@ _curses_halfdelay_impl(PyObject *module, unsigned char tenths) { PyCursesStatefulInitialised(module); - return PyCursesCheckERR(module, halfdelay(tenths), "halfdelay"); + return curses_check_err(module, halfdelay(tenths), "halfdelay", NULL); } /*[clinic input] @@ -3353,9 +3662,10 @@ _curses_init_color_impl(PyObject *module, int color_number, short r, short g, PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); - return PyCursesCheckERR(module, + return curses_check_err(module, _CURSES_INIT_COLOR_FUNC(color_number, r, g, b), - Py_STRINGIFY(_CURSES_INIT_COLOR_FUNC)); + Py_STRINGIFY(_CURSES_INIT_COLOR_FUNC), + NULL); } /*[clinic input] @@ -3389,9 +3699,8 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) COLOR_PAIRS - 1); } else { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_Format(state->error, "%s() returned ERR", - Py_STRINGIFY(_CURSES_INIT_PAIR_FUNC)); + const char *funcname = Py_STRINGIFY(_CURSES_INIT_PAIR_FUNC); + curses_set_error(module, funcname, "init_pair"); } return NULL; } @@ -3414,16 +3723,19 @@ _curses_initscr_impl(PyObject *module) WINDOW *win; if (curses_initscr_called) { - wrefresh(stdscr); cursesmodule_state *state = get_cursesmodule_state(module); + int code = wrefresh(stdscr); + if (code == ERR) { + _curses_set_null_error(state, "wrefresh", "initscr"); + return NULL; + } return PyCursesWindow_New(state, stdscr, NULL, NULL); } win = initscr(); if (win == NULL) { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, catchall_NULL); + curses_set_null_error(module, "initscr", NULL); return NULL; } @@ -3550,7 +3862,7 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) if (fd == -1) { PyObject* sys_stdout; - if (_PySys_GetOptionalAttrString("stdout", &sys_stdout) < 0) { + if (PySys_GetOptionalAttrString("stdout", &sys_stdout) < 0) { return NULL; } @@ -3628,7 +3940,7 @@ _curses_set_escdelay_impl(PyObject *module, int ms) return NULL; } - return PyCursesCheckERR(module, set_escdelay(ms), "set_escdelay"); + return curses_check_err(module, set_escdelay(ms), "set_escdelay", NULL); } /*[clinic input] @@ -3667,7 +3979,7 @@ _curses_set_tabsize_impl(PyObject *module, int size) return NULL; } - return PyCursesCheckERR(module, set_tabsize(size), "set_tabsize"); + return curses_check_err(module, set_tabsize(size), "set_tabsize", NULL); } #endif @@ -3685,7 +3997,7 @@ _curses_intrflush_impl(PyObject *module, int flag) { PyCursesStatefulInitialised(module); - return PyCursesCheckERR(module, intrflush(NULL, flag), "intrflush"); + return curses_check_err(module, intrflush(NULL, flag), "intrflush", NULL); } /*[clinic input] @@ -3701,6 +4013,7 @@ NoArgTrueFalseFunctionBody(isendwin) #ifdef HAVE_CURSES_IS_TERM_RESIZED /*[clinic input] +@permit_long_summary _curses.is_term_resized nlines: int @@ -3714,7 +4027,7 @@ Return True if resize_term() would modify the window structure, False otherwise. static PyObject * _curses_is_term_resized_impl(PyObject *module, int nlines, int ncols) -/*[clinic end generated code: output=aafe04afe50f1288 input=ca9c0bd0fb8ab444]*/ +/*[clinic end generated code: output=aafe04afe50f1288 input=5792a3f40cecb010]*/ { PyCursesStatefulInitialised(module); @@ -3767,6 +4080,7 @@ _curses_killchar_impl(PyObject *module) } /*[clinic input] +@permit_long_docstring_body _curses.longname Return the terminfo long name field describing the current terminal. @@ -3777,7 +4091,7 @@ only after the call to initscr(). static PyObject * _curses_longname_impl(PyObject *module) -/*[clinic end generated code: output=fdf30433727ef568 input=84c3f20201b1098e]*/ +/*[clinic end generated code: output=fdf30433727ef568 input=5de06852f2230ddb]*/ NoArgReturnStringFunctionBody(longname) /*[clinic input] @@ -3798,7 +4112,7 @@ _curses_meta_impl(PyObject *module, int yes) { PyCursesStatefulInitialised(module); - return PyCursesCheckERR(module, meta(stdscr, yes), "meta"); + return curses_check_err(module, meta(stdscr, yes), "meta", NULL); } #ifdef NCURSES_MOUSE_VERSION @@ -3821,11 +4135,16 @@ _curses_mouseinterval_impl(PyObject *module, int interval) /*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ { PyCursesStatefulInitialised(module); - - return PyCursesCheckERR(module, mouseinterval(interval), "mouseinterval"); + int value = mouseinterval(interval); + if (value == ERR) { + curses_set_error(module, "mouseinterval", NULL); + return NULL; + } + return PyLong_FromLong(value); } /*[clinic input] +@permit_long_summary _curses.mousemask newmask: unsigned_long(bitwise=True) @@ -3841,7 +4160,7 @@ If this function is never called, no mouse events are ever reported. static PyObject * _curses_mousemask_impl(PyObject *module, unsigned long newmask) -/*[clinic end generated code: output=9406cf1b8a36e485 input=bdf76b7568a3c541]*/ +/*[clinic end generated code: output=9406cf1b8a36e485 input=b92ff4fbe5ce61b1]*/ { mmask_t oldmask, availmask; @@ -3898,8 +4217,7 @@ _curses_newpad_impl(PyObject *module, int nlines, int ncols) win = newpad(nlines, ncols); if (win == NULL) { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, catchall_NULL); + curses_set_null_error(module, "newpad", NULL); return NULL; } @@ -3939,8 +4257,7 @@ _curses_newwin_impl(PyObject *module, int nlines, int ncols, win = newwin(nlines,ncols,begin_y,begin_x); if (win == NULL) { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, catchall_NULL); + curses_set_null_error(module, "newwin", NULL); return NULL; } @@ -4059,9 +4376,8 @@ _curses_pair_content_impl(PyObject *module, int pair_number) COLOR_PAIRS - 1); } else { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_Format(state->error, "%s() returned ERR", - Py_STRINGIFY(_CURSES_PAIR_CONTENT_FUNC)); + const char *funcname = Py_STRINGIFY(_CURSES_PAIR_CONTENT_FUNC); + curses_set_error(module, funcname, "pair_content"); } return NULL; } @@ -4105,7 +4421,7 @@ static PyObject * _curses_putp_impl(PyObject *module, const char *string) /*[clinic end generated code: output=e98081d1b8eb5816 input=1601faa828b44cb3]*/ { - return PyCursesCheckERR(module, putp(string), "putp"); + return curses_check_err(module, putp(string), "putp", NULL); } /*[clinic input] @@ -4226,6 +4542,7 @@ _curses_raw_impl(PyObject *module, int flag) NoArgOrFlagNoReturnFunctionBody(raw, flag) /*[clinic input] +@permit_long_summary _curses.reset_prog_mode Restore the terminal to "program" mode, as previously saved by def_prog_mode(). @@ -4233,10 +4550,11 @@ Restore the terminal to "program" mode, as previously saved by def_prog_mode(). static PyObject * _curses_reset_prog_mode_impl(PyObject *module) -/*[clinic end generated code: output=15eb765abf0b6575 input=3d82bea2b3243471]*/ +/*[clinic end generated code: output=15eb765abf0b6575 input=a8b44b5261c8cf3a]*/ NoArgNoReturnFunctionBody(reset_prog_mode) /*[clinic input] +@permit_long_summary _curses.reset_shell_mode Restore the terminal to "shell" mode, as previously saved by def_shell_mode(). @@ -4244,7 +4562,7 @@ Restore the terminal to "shell" mode, as previously saved by def_shell_mode(). static PyObject * _curses_reset_shell_mode_impl(PyObject *module) -/*[clinic end generated code: output=0238de2962090d33 input=1c738fa64bd1a24f]*/ +/*[clinic end generated code: output=0238de2962090d33 input=f5224034a2c95931]*/ NoArgNoReturnFunctionBody(reset_shell_mode) /*[clinic input] @@ -4279,10 +4597,12 @@ _curses_resizeterm_impl(PyObject *module, short nlines, short ncols) /*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/ { PyObject *result; + int code; PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(module, resizeterm(nlines, ncols), "resizeterm"); + code = resizeterm(nlines, ncols); + result = curses_check_err(module, code, "resizeterm", NULL); if (!result) return NULL; if (!update_lines_cols(module)) { @@ -4296,6 +4616,7 @@ _curses_resizeterm_impl(PyObject *module, short nlines, short ncols) #ifdef HAVE_CURSES_RESIZE_TERM /*[clinic input] +@permit_long_docstring_body _curses.resize_term nlines: short @@ -4315,13 +4636,15 @@ without additional interaction with the application. static PyObject * _curses_resize_term_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=46c6d749fa291dbd input=276afa43d8ea7091]*/ +/*[clinic end generated code: output=46c6d749fa291dbd input=ebfa840f6b5f03fa]*/ { PyObject *result; + int code; PyCursesStatefulInitialised(module); - result = PyCursesCheckERR(module, resize_term(nlines, ncols), "resize_term"); + code = resize_term(nlines, ncols); + result = curses_check_err(module, code, "resize_term", NULL); if (!result) return NULL; if (!update_lines_cols(module)) { @@ -4390,8 +4713,7 @@ _curses_start_color_impl(PyObject *module) PyCursesStatefulInitialised(module); if (start_color() == ERR) { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, "start_color() returned ERR"); + curses_set_error(module, "start_color", NULL); return NULL; } @@ -4433,6 +4755,7 @@ _curses_termattrs_impl(PyObject *module) NoArgReturnIntFunctionBody(termattrs) /*[clinic input] +@permit_long_summary _curses.termname Return the value of the environment variable TERM, truncated to 14 characters. @@ -4440,7 +4763,7 @@ Return the value of the environment variable TERM, truncated to 14 characters. static PyObject * _curses_termname_impl(PyObject *module) -/*[clinic end generated code: output=96375577ebbd67fd input=33c08d000944f33f]*/ +/*[clinic end generated code: output=96375577ebbd67fd input=c34f724d8ce8fc4e]*/ NoArgReturnStringFunctionBody(termname) /*[clinic input] @@ -4543,8 +4866,7 @@ _curses_tparm_impl(PyObject *module, const char *str, int i1, int i2, int i3, result = tparm((char *)str,i1,i2,i3,i4,i5,i6,i7,i8,i9); if (!result) { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, "tparm() returned NULL"); + curses_set_null_error(module, "tparm", NULL); return NULL; } @@ -4570,7 +4892,7 @@ _curses_typeahead_impl(PyObject *module, int fd) { PyCursesStatefulInitialised(module); - return PyCursesCheckERR(module, typeahead( fd ), "typeahead"); + return curses_check_err(module, typeahead(fd), "typeahead", NULL); } #endif @@ -4597,7 +4919,12 @@ _curses_unctrl(PyObject *module, PyObject *ch) if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; - return PyBytes_FromString(unctrl(ch_)); + const char *res = unctrl(ch_); + if (res == NULL) { + curses_set_null_error(module, "unctrl", NULL); + return NULL; + } + return PyBytes_FromString(res); } /*[clinic input] @@ -4620,7 +4947,7 @@ _curses_ungetch(PyObject *module, PyObject *ch) if (!PyCurses_ConvertToChtype(NULL, ch, &ch_)) return NULL; - return PyCursesCheckERR(module, ungetch(ch_), "ungetch"); + return curses_check_err(module, ungetch(ch_), "ungetch", NULL); } #ifdef HAVE_NCURSESW @@ -4690,12 +5017,13 @@ _curses_unget_wch(PyObject *module, PyObject *ch) if (!PyCurses_ConvertToWchar_t(ch, &wch)) return NULL; - return PyCursesCheckERR(module, unget_wch(wch), "unget_wch"); + return curses_check_err(module, unget_wch(wch), "unget_wch", NULL); } #endif #ifdef HAVE_CURSES_USE_ENV /*[clinic input] +@permit_long_docstring_body _curses.use_env flag: bool @@ -4715,7 +5043,7 @@ not set). static PyObject * _curses_use_env_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b2c445e435c0b164 input=06ac30948f2d78e4]*/ +/*[clinic end generated code: output=b2c445e435c0b164 input=eaa9047ec73c27d3]*/ { use_env(flag); Py_RETURN_NONE; @@ -4739,13 +5067,7 @@ _curses_use_default_colors_impl(PyObject *module) PyCursesStatefulInitialisedColor(module); code = use_default_colors(); - if (code != ERR) { - Py_RETURN_NONE; - } else { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, "use_default_colors() returned ERR"); - return NULL; - } + return curses_check_err(module, code, "use_default_colors", NULL); } /*[clinic input] @@ -4772,13 +5094,7 @@ _curses_assume_default_colors_impl(PyObject *module, int fg, int bg) PyCursesStatefulInitialisedColor(module); code = assume_default_colors(fg, bg); - if (code != ERR) { - Py_RETURN_NONE; - } else { - cursesmodule_state *state = get_cursesmodule_state(module); - PyErr_SetString(state->error, "assume_default_colors() returned ERR"); - return NULL; - } + return curses_check_err(module, code, "assume_default_colors", NULL); } #endif /* STRICT_SYSV_CURSES */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9bba0e3354b..46c4f57984b 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -14,6 +14,8 @@ #include "pycore_object.h" // _PyObject_Init() #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_pyatomic_ft_wrappers.h" #include "datetime.h" @@ -124,10 +126,9 @@ get_module_state(PyObject *module) #define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) static PyObject * -get_current_module(PyInterpreterState *interp, int *p_reloading) +get_current_module(PyInterpreterState *interp) { PyObject *mod = NULL; - int reloading = 0; PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { @@ -138,7 +139,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) goto error; } if (ref != NULL) { - reloading = 1; if (ref != Py_None) { (void)PyWeakref_GetRef(ref, &mod); if (mod == Py_None) { @@ -147,9 +147,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) Py_DECREF(ref); } } - if (p_reloading != NULL) { - *p_reloading = reloading; - } return mod; error: @@ -163,7 +160,7 @@ static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *mod = get_current_module(interp, NULL); + PyObject *mod = get_current_module(interp); if (mod == NULL) { assert(!PyErr_Occurred()); if (PyErr_Occurred()) { @@ -218,7 +215,7 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - if (ref != NULL) { + if (ref != NULL && ref != Py_None) { PyObject *current = NULL; int rc = PyWeakref_GetRef(ref, &current); /* We only need "current" for pointer comparison. */ @@ -335,8 +332,10 @@ class datetime.datetime "PyDateTime_DateTime *" "get_datetime_state()->datetime_ class datetime.date "PyDateTime_Date *" "get_datetime_state()->date_type" class datetime.time "PyDateTime_Time *" "get_datetime_state()->time_type" class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "get_datetime_state()->isocalendar_date_type" +class datetime.timedelta "PyDateTime_Delta *" "&PyDateTime_DeltaType" +class datetime.timezone "PyDateTime_TimeZone *" "&PyDateTime_TimeZoneType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c8f3d834a860d50a]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c54b9adf60082f0d]*/ #include "clinic/_datetimemodule.c.h" @@ -1088,6 +1087,7 @@ parse_isoformat_time(const char *dtstr, size_t dtlen, int *hour, int *minute, // -3: Failed to parse time component // -4: Failed to parse time separator // -5: Malformed timezone string + // -6: Timezone fields are not in range const char *p = dtstr; const char *p_end = dtstr + dtlen; @@ -1134,6 +1134,11 @@ parse_isoformat_time(const char *dtstr, size_t dtlen, int *hour, int *minute, rv = parse_hh_mm_ss_ff(tzinfo_pos, p_end, &tzhour, &tzminute, &tzsecond, tzmicrosecond); + // Check if timezone fields are in range + if (check_time_args(tzhour, tzminute, tzsecond, *tzmicrosecond, 0) < 0) { + return -6; + } + *tzoffset = tzsign * ((tzhour * 3600) + (tzminute * 60) + tzsecond); *tzmicrosecond *= tzsign; @@ -1169,19 +1174,18 @@ new_datetime_ex(int, int, int, int, int, int, int, PyObject *, PyTypeObject *); /* Create date instance with no range checking, or call subclass constructor */ static PyObject * -new_date_subclass_ex(int year, int month, int day, PyObject *cls) +new_date_subclass_ex(int year, int month, int day, PyTypeObject *cls) { PyObject *result; // We have "fast path" constructors for two subclasses: date and datetime - if ((PyTypeObject *)cls == DATE_TYPE(NO_STATE)) { - result = new_date_ex(year, month, day, (PyTypeObject *)cls); + if (cls == DATE_TYPE(NO_STATE)) { + result = new_date_ex(year, month, day, cls); } - else if ((PyTypeObject *)cls == DATETIME_TYPE(NO_STATE)) { - result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, - (PyTypeObject *)cls); + else if (cls == DATETIME_TYPE(NO_STATE)) { + result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, cls); } else { - result = PyObject_CallFunction(cls, "iii", year, month, day); + result = PyObject_CallFunction((PyObject *)cls, "iii", year, month, day); } return result; @@ -1233,7 +1237,7 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, DATETIME_TYPE(NO_STATE)) static PyObject * -call_subclass_fold(PyObject *cls, int fold, const char *format, ...) +call_subclass_fold(PyTypeObject *cls, int fold, const char *format, ...) { PyObject *kwargs = NULL, *res = NULL; va_list va; @@ -1259,7 +1263,7 @@ call_subclass_fold(PyObject *cls, int fold, const char *format, ...) goto Done; } } - res = PyObject_Call(cls, args, kwargs); + res = PyObject_Call((PyObject *)cls, args, kwargs); Done: Py_DECREF(args); Py_XDECREF(kwargs); @@ -1269,10 +1273,10 @@ call_subclass_fold(PyObject *cls, int fold, const char *format, ...) static PyObject * new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - int fold, PyObject *cls) + int fold, PyTypeObject *cls) { PyObject* dt; - if ((PyTypeObject*)cls == DATETIME_TYPE(NO_STATE)) { + if (cls == DATETIME_TYPE(NO_STATE)) { // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); @@ -1289,7 +1293,7 @@ new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute static PyObject * new_datetime_subclass_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - PyObject *cls) { + PyTypeObject *cls) { return new_datetime_subclass_fold_ex(year, month, day, hour, minute, second, usecond, tzinfo, 0, cls); @@ -1338,10 +1342,10 @@ new_time_ex(int hour, int minute, int second, int usecond, static PyObject * new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, - PyObject *tzinfo, int fold, PyObject *cls) + PyObject *tzinfo, int fold, PyTypeObject *cls) { PyObject *t; - if ((PyTypeObject*)cls == TIME_TYPE(NO_STATE)) { + if (cls == TIME_TYPE(NO_STATE)) { // Use the fast path constructor t = new_time(hour, minute, second, usecond, tzinfo, fold); } @@ -1757,6 +1761,24 @@ format_utcoffset(char *buf, size_t buflen, const char *sep, return 0; } +/* Check whether year with century should be normalized for strftime. */ +inline static int +normalize_century(void) +{ + static int cache = -1; + if (cache < 0) { + char year[5]; + struct tm date = { + .tm_year = -1801, + .tm_mon = 0, + .tm_mday = 1 + }; + cache = (strftime(year, sizeof(year), "%Y", &date) && + strcmp(year, "0099") != 0); + } + return cache; +} + static PyObject * make_somezreplacement(PyObject *object, char *sep, PyObject *tzinfoarg) { @@ -1928,10 +1950,9 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, } replacement = freplacement; } -#ifdef Py_NORMALIZE_CENTURY - else if (ch == 'Y' || ch == 'G' - || ch == 'F' || ch == 'C' - ) { + else if (normalize_century() + && (ch == 'Y' || ch == 'G' || ch == 'F' || ch == 'C')) + { /* 0-pad year with century as necessary */ PyObject *item = PySequence_GetItem(timetuple, 0); if (item == NULL) { @@ -1982,7 +2003,6 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, } continue; } -#endif else { /* percent followed by something else */ continue; @@ -2521,14 +2541,16 @@ static Py_hash_t delta_hash(PyObject *op) { PyDateTime_Delta *self = PyDelta_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *temp = delta_getstate(self); if (temp != NULL) { - self->hashcode = PyObject_Hash(temp); + hash = PyObject_Hash(temp); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp); } } - return self->hashcode; + return hash; } static PyObject * @@ -2766,38 +2788,39 @@ accum(const char* tag, PyObject *sofar, PyObject *num, PyObject *factor, return NULL; } +/*[clinic input] +@classmethod +datetime.timedelta.__new__ as delta_new + + days: object(c_default="NULL") = 0 + seconds: object(c_default="NULL") = 0 + microseconds: object(c_default="NULL") = 0 + milliseconds: object(c_default="NULL") = 0 + minutes: object(c_default="NULL") = 0 + hours: object(c_default="NULL") = 0 + weeks: object(c_default="NULL") = 0 + +Difference between two datetime values. + +All arguments are optional and default to 0. +Arguments may be integers or floats, and may be positive or negative. +[clinic start generated code]*/ + static PyObject * -delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) +delta_new_impl(PyTypeObject *type, PyObject *days, PyObject *seconds, + PyObject *microseconds, PyObject *milliseconds, + PyObject *minutes, PyObject *hours, PyObject *weeks) +/*[clinic end generated code: output=61d7e02a92a97700 input=e8cd54819295d34b]*/ { PyObject *self = NULL; PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); - /* Argument objects. */ - PyObject *day = NULL; - PyObject *second = NULL; - PyObject *us = NULL; - PyObject *ms = NULL; - PyObject *minute = NULL; - PyObject *hour = NULL; - PyObject *week = NULL; - PyObject *x = NULL; /* running sum of microseconds */ PyObject *y = NULL; /* temp sum of microseconds */ double leftover_us = 0.0; - static char *keywords[] = { - "days", "seconds", "microseconds", "milliseconds", - "minutes", "hours", "weeks", NULL - }; - - if (PyArg_ParseTupleAndKeywords(args, kw, "|OOOOOOO:__new__", - keywords, - &day, &second, &us, - &ms, &minute, &hour, &week) == 0) - goto Done; - x = PyLong_FromLong(0); if (x == NULL) goto Done; @@ -2808,32 +2831,32 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (x == NULL) \ goto Done - if (us) { - y = accum("microseconds", x, us, _PyLong_GetOne(), &leftover_us); + if (microseconds) { + y = accum("microseconds", x, microseconds, _PyLong_GetOne(), &leftover_us); CLEANUP; } - if (ms) { - y = accum("milliseconds", x, ms, CONST_US_PER_MS(st), &leftover_us); + if (milliseconds) { + y = accum("milliseconds", x, milliseconds, CONST_US_PER_MS(st), &leftover_us); CLEANUP; } - if (second) { - y = accum("seconds", x, second, CONST_US_PER_SECOND(st), &leftover_us); + if (seconds) { + y = accum("seconds", x, seconds, CONST_US_PER_SECOND(st), &leftover_us); CLEANUP; } - if (minute) { - y = accum("minutes", x, minute, CONST_US_PER_MINUTE(st), &leftover_us); + if (minutes) { + y = accum("minutes", x, minutes, CONST_US_PER_MINUTE(st), &leftover_us); CLEANUP; } - if (hour) { - y = accum("hours", x, hour, CONST_US_PER_HOUR(st), &leftover_us); + if (hours) { + y = accum("hours", x, hours, CONST_US_PER_HOUR(st), &leftover_us); CLEANUP; } - if (day) { - y = accum("days", x, day, CONST_US_PER_DAY(st), &leftover_us); + if (days) { + y = accum("days", x, days, CONST_US_PER_DAY(st), &leftover_us); CLEANUP; } - if (week) { - y = accum("weeks", x, week, CONST_US_PER_WEEK(st), &leftover_us); + if (weeks) { + y = accum("weeks", x, weeks, CONST_US_PER_WEEK(st), &leftover_us); CLEANUP; } if (leftover_us) { @@ -3032,13 +3055,6 @@ static PyMethodDef delta_methods[] = { {NULL, NULL}, }; -static const char delta_doc[] = -PyDoc_STR("Difference between two datetime values.\n\n" - "timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, " - "minutes=0, hours=0, weeks=0)\n\n" - "All arguments are optional and default to 0.\n" - "Arguments may be integers or floats, and may be positive or negative."); - static PyNumberMethods delta_as_number = { delta_add, /* nb_add */ delta_subtract, /* nb_subtract */ @@ -3096,7 +3112,7 @@ static PyTypeObject PyDateTime_DeltaType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - delta_doc, /* tp_doc */ + delta_new__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ delta_richcompare, /* tp_richcompare */ @@ -3172,8 +3188,6 @@ static PyGetSetDef date_getset[] = { /* Constructors. */ -static char *date_kws[] = {"year", "month", "day", NULL}; - static PyObject * date_from_pickle(PyTypeObject *type, PyObject *state) { @@ -3191,11 +3205,6 @@ date_from_pickle(PyTypeObject *type, PyObject *state) static PyObject * date_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int year; - int month; - int day; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) == 1) { PyObject *state = PyTuple_GET_ITEM(args, 0); @@ -3221,22 +3230,36 @@ date_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = date_from_pickle(type, state); + PyObject *self = date_from_pickle(type, state); Py_DECREF(state); return self; } } } - if (PyArg_ParseTupleAndKeywords(args, kw, "iii", date_kws, - &year, &month, &day)) { - self = new_date_ex(year, month, day, type); - } - return self; + return datetime_date(type, args, kw); +} + +/*[clinic input] +@classmethod +datetime.date.__new__ + + year: int + month: int + day: int + +Concrete date type. +[clinic start generated code]*/ + +static PyObject * +datetime_date_impl(PyTypeObject *type, int year, int month, int day) +/*[clinic end generated code: output=6654caa3dea7d518 input=fd1bac0658690455]*/ +{ + return new_date_ex(year, month, day, type); } static PyObject * -date_fromtimestamp(PyObject *cls, PyObject *obj) +date_fromtimestamp(PyTypeObject *cls, PyObject *obj) { struct tm tm; time_t t; @@ -3258,27 +3281,50 @@ date_fromtimestamp(PyObject *cls, PyObject *obj) * only way to be sure of that is to *call* time.time(). That's not * generally the same as calling C's time. */ -static PyObject * -date_today(PyObject *cls, PyObject *Py_UNUSED(dummy)) -{ - PyObject *time; - PyObject *result; - time = time_time(); - if (time == NULL) - return NULL; +/*[clinic input] +@classmethod +datetime.date.today - /* Note well: today() is a class method, so this may not call - * date.fromtimestamp. For example, it may call - * datetime.fromtimestamp. That's why we need all the accuracy - * time.time() delivers; if someone were gonzo about optimization, - * date.today() could get away with plain C time(). +Current date or datetime. + +Equivalent to fromtimestamp(time.time()). +[clinic start generated code]*/ + +static PyObject * +datetime_date_today_impl(PyTypeObject *type) +/*[clinic end generated code: output=d5474697df6b251c input=21688afa289c0a06]*/ +{ + /* Use C implementation to boost performance for date type */ + if (type == &PyDateTime_DateType) { + struct tm tm; + time_t t; + time(&t); + + if (_PyTime_localtime(t, &tm) != 0) { + return NULL; + } + + return new_date_ex(tm.tm_year + 1900, + tm.tm_mon + 1, + tm.tm_mday, + type); + } + + PyObject *time = time_time(); + if (time == NULL) { + return NULL; + } + + /* Note well: since today() is a class method, it may not call + * date.fromtimestamp, e.g., it may call datetime.fromtimestamp. */ - result = PyObject_CallMethodOneArg(cls, &_Py_ID(fromtimestamp), time); + PyObject *result = PyObject_CallMethodOneArg((PyObject*)type, &_Py_ID(fromtimestamp), time); Py_DECREF(time); return result; } /*[clinic input] +@permit_long_docstring_body @classmethod datetime.date.fromtimestamp @@ -3293,9 +3339,9 @@ as local time. static PyObject * datetime_date_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp) -/*[clinic end generated code: output=59def4e32c028fb6 input=eabb3fe7f40491fe]*/ +/*[clinic end generated code: output=59def4e32c028fb6 input=55ff6940f0a8339f]*/ { - return date_fromtimestamp((PyObject *) type, timestamp); + return date_fromtimestamp(type, timestamp); } /* bpo-36025: This is a wrapper for API compatibility with the public C API, @@ -3309,52 +3355,58 @@ datetime_date_fromtimestamp_capi(PyObject *cls, PyObject *args) PyObject *result = NULL; if (PyArg_UnpackTuple(args, "fromtimestamp", 1, 1, &timestamp)) { - result = date_fromtimestamp(cls, timestamp); + result = date_fromtimestamp((PyTypeObject *)cls, timestamp); } return result; } -/* Return new date from proleptic Gregorian ordinal. Raises ValueError if - * the ordinal is out of range. - */ +/*[clinic input] +@classmethod +datetime.date.fromordinal + + ordinal: int + / + +Construct a date from a proleptic Gregorian ordinal. + +January 1 of year 1 is day 1. Only the year, month and day are +non-zero in the result. +[clinic start generated code]*/ + static PyObject * -date_fromordinal(PyObject *cls, PyObject *args) +datetime_date_fromordinal_impl(PyTypeObject *type, int ordinal) +/*[clinic end generated code: output=ea5cc69d86614a6b input=a3a4eedf582f145e]*/ { - PyObject *result = NULL; - int ordinal; + int year; + int month; + int day; - if (PyArg_ParseTuple(args, "i:fromordinal", &ordinal)) { - int year; - int month; - int day; - - if (ordinal < 1) - PyErr_SetString(PyExc_ValueError, "ordinal must be " - ">= 1"); - else { - ord_to_ymd(ordinal, &year, &month, &day); - result = new_date_subclass_ex(year, month, day, cls); - } - } - return result; -} - -/* Return the new date from a string as generated by date.isoformat() */ -static PyObject * -date_fromisoformat(PyObject *cls, PyObject *dtstr) -{ - assert(dtstr != NULL); - - if (!PyUnicode_Check(dtstr)) { - PyErr_SetString(PyExc_TypeError, - "fromisoformat: argument must be str"); + if (ordinal < 1) { + PyErr_SetString(PyExc_ValueError, "ordinal must be >= 1"); return NULL; } + ord_to_ymd(ordinal, &year, &month, &day); + return new_date_subclass_ex(year, month, day, type); +} +/*[clinic input] +@classmethod +datetime.date.fromisoformat + + string: unicode + / + +Construct a date from a string in ISO 8601 format. +[clinic start generated code]*/ + +static PyObject * +datetime_date_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=8b9f9324904fca02 input=73c64216c10bcc8e]*/ +{ Py_ssize_t len; - const char *dt_ptr = PyUnicode_AsUTF8AndSize(dtstr, &len); + const char *dt_ptr = PyUnicode_AsUTF8AndSize(string, &len); if (dt_ptr == NULL) { goto invalid_string_error; } @@ -3373,33 +3425,32 @@ date_fromisoformat(PyObject *cls, PyObject *dtstr) goto invalid_string_error; } - return new_date_subclass_ex(year, month, day, cls); + return new_date_subclass_ex(year, month, day, type); invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); return NULL; } +/*[clinic input] +@classmethod +datetime.date.fromisocalendar + + year: int + week: int + day: int + +Construct a date from the ISO year, week number and weekday. + +This is the inverse of the date.isocalendar() function. +[clinic start generated code]*/ + static PyObject * -date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) +datetime_date_fromisocalendar_impl(PyTypeObject *type, int year, int week, + int day) +/*[clinic end generated code: output=7b26e15115d24df6 input=fbb05b53d6fb51d8]*/ { - static char *keywords[] = { - "year", "week", "day", NULL - }; - - int year, week, day; - if (PyArg_ParseTupleAndKeywords(args, kw, "iii:fromisocalendar", - keywords, - &year, &week, &day) == 0) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - PyErr_Format(PyExc_ValueError, - "ISO calendar component out of range"); - - } - return NULL; - } - int month; int rv = iso_to_ymd(year, week, day, &year, &month, &day); @@ -3420,26 +3471,37 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return NULL; } - return new_date_subclass_ex(year, month, day, cls); + return new_date_subclass_ex(year, month, day, type); } -/* Return new date from _strptime.strptime_datetime_date(). */ -static PyObject * -date_strptime(PyObject *cls, PyObject *args) -{ - PyObject *string, *format, *result; +/*[clinic input] +@classmethod +datetime.date.strptime - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { - return NULL; - } + string: unicode + format: unicode + / + +Parse string according to the given date format (like time.strptime()). + +For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes +[clinic start generated code]*/ + +static PyObject * +datetime_date_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=454d473bee2d5161 input=31d57bb789433e99]*/ +{ + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { return NULL; } result = PyObject_CallMethodObjArgs(module, - &_Py_ID(_strptime_datetime_date), cls, - string, format, NULL); + &_Py_ID(_strptime_datetime_date), + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } @@ -3463,8 +3525,7 @@ add_date_timedelta(PyDateTime_Date *date, PyDateTime_Delta *delta, int negate) int day = GET_DAY(date) + (negate ? -deltadays : deltadays); if (normalize_date(&year, &month, &day) >= 0) - result = new_date_subclass_ex(year, month, day, - (PyObject* )Py_TYPE(date)); + result = new_date_subclass_ex(year, month, day, Py_TYPE(date)); return result; } @@ -3556,20 +3617,29 @@ date_ctime(PyObject *self, PyObject *Py_UNUSED(dummy)) return format_ctime(self, 0, 0, 0); } +/*[clinic input] +datetime.date.strftime + + self: self(type="PyObject *") + format: unicode + +Format using strftime(). + +Example: "%d/%m/%Y, %H:%M:%S". + +For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes +[clinic start generated code]*/ + static PyObject * -date_strftime(PyObject *self, PyObject *args, PyObject *kw) +datetime_date_strftime_impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=6529b70095e16778 input=b6fd4a2ded27b557]*/ { /* This method can be inherited, and needs to call the * timetuple() method appropriate to self's class. */ PyObject *result; PyObject *tuple; - PyObject *format; - static char *keywords[] = {"format", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords, - &format)) - return NULL; tuple = PyObject_CallMethodNoArgs(self, &_Py_ID(timetuple)); if (tuple == NULL) @@ -3579,14 +3649,20 @@ date_strftime(PyObject *self, PyObject *args, PyObject *kw) return result; } +/*[clinic input] +datetime.date.__format__ + + self: self(type="PyObject *") + format: unicode + / + +Formats self with strftime. +[clinic start generated code]*/ + static PyObject * -date_format(PyObject *self, PyObject *args) +datetime_date___format___impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=efa0223d000a93b7 input=e417a7c84e1abaf9]*/ { - PyObject *format; - - if (!PyArg_ParseTuple(args, "U:__format__", &format)) - return NULL; - /* if the format is zero length, return str(self) */ if (PyUnicode_GetLength(format) == 0) return PyObject_Str(self); @@ -3832,7 +3908,7 @@ datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, int day) /*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/ { - return new_date_subclass_ex(year, month, day, (PyObject *)Py_TYPE(self)); + return new_date_subclass_ex(year, month, day, Py_TYPE(self)); } static Py_hash_t @@ -3848,12 +3924,14 @@ static Py_hash_t date_hash(PyObject *op) { PyDateTime_Date *self = PyDate_CAST(op); - if (self->hashcode == -1) { - self->hashcode = generic_hash( + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_DATE_DATASIZE); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); } - return self->hashcode; + return hash; } static PyObject * @@ -3893,38 +3971,19 @@ static PyMethodDef date_methods[] = { /* Class methods: */ DATETIME_DATE_FROMTIMESTAMP_METHODDEF - - {"fromordinal", date_fromordinal, METH_VARARGS | METH_CLASS, - PyDoc_STR("int -> date corresponding to a proleptic Gregorian " - "ordinal.")}, - - {"fromisoformat", date_fromisoformat, METH_O | METH_CLASS, - PyDoc_STR("str -> Construct a date from a string in ISO 8601 format.")}, - - {"fromisocalendar", _PyCFunction_CAST(date_fromisocalendar), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("int, int, int -> Construct a date from the ISO year, week " - "number and weekday.\n\n" - "This is the inverse of the date.isocalendar() function")}, - - {"strptime", date_strptime, METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new date parsed from a string " - "(like time.strptime()).")}, - - {"today", date_today, METH_NOARGS | METH_CLASS, - PyDoc_STR("Current date or datetime: same as " - "self.__class__.fromtimestamp(time.time()).")}, + DATETIME_DATE_FROMORDINAL_METHODDEF + DATETIME_DATE_FROMISOFORMAT_METHODDEF + DATETIME_DATE_FROMISOCALENDAR_METHODDEF + DATETIME_DATE_STRPTIME_METHODDEF + DATETIME_DATE_TODAY_METHODDEF /* Instance methods: */ {"ctime", date_ctime, METH_NOARGS, PyDoc_STR("Return ctime() style string.")}, - {"strftime", _PyCFunction_CAST(date_strftime), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("format -> strftime() style string.")}, - - {"__format__", date_format, METH_VARARGS, - PyDoc_STR("Formats self with strftime.")}, + DATETIME_DATE_STRFTIME_METHODDEF + DATETIME_DATE___FORMAT___METHODDEF {"timetuple", date_timetuple, METH_NOARGS, PyDoc_STR("Return time tuple, compatible with time.localtime().")}, @@ -3959,9 +4018,6 @@ static PyMethodDef date_methods[] = { {NULL, NULL} }; -static const char date_doc[] = -PyDoc_STR("date(year, month, day) --> date object"); - static PyNumberMethods date_as_number = { date_add, /* nb_add */ date_subtract, /* nb_subtract */ @@ -3996,7 +4052,7 @@ static PyTypeObject PyDateTime_DateType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - date_doc, /* tp_doc */ + datetime_date__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ date_richcompare, /* tp_richcompare */ @@ -4199,7 +4255,8 @@ static PyMethodDef tzinfo_methods[] = { }; static const char tzinfo_doc[] = -PyDoc_STR("Abstract base class for time zone info objects."); +PyDoc_STR("Abstract base class for time zone info objects.\n\n" + "Subclasses must override the tzname(), utcoffset() and dst() methods."); static PyTypeObject PyDateTime_TZInfoType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -4243,18 +4300,21 @@ static PyTypeObject PyDateTime_TZInfoType = { 0, /* tp_free */ }; -static char *timezone_kws[] = {"offset", "name", NULL}; +/*[clinic input] +@classmethod +datetime.timezone.__new__ as timezone_new + + offset: object(subclass_of="DELTA_TYPE(NO_STATE)") + name: unicode = NULL + +Fixed offset from UTC implementation of tzinfo. +[clinic start generated code]*/ static PyObject * -timezone_new(PyTypeObject *type, PyObject *args, PyObject *kw) +timezone_new_impl(PyTypeObject *type, PyObject *offset, PyObject *name) +/*[clinic end generated code: output=41a2dda500424187 input=d51255afe60382cd]*/ { - PyObject *offset; - PyObject *name = NULL; - if (PyArg_ParseTupleAndKeywords(args, kw, "O!|U:timezone", timezone_kws, - DELTA_TYPE(NO_STATE), &offset, &name)) - return new_timezone(offset, name); - - return NULL; + return new_timezone(offset, name); } static void @@ -4442,9 +4502,6 @@ static PyMethodDef timezone_methods[] = { {NULL, NULL} }; -static const char timezone_doc[] = -PyDoc_STR("Fixed offset from UTC implementation of tzinfo."); - static PyTypeObject PyDateTime_TimeZoneType = { PyVarObject_HEAD_INIT(NULL, 0) "datetime.timezone", /* tp_name */ @@ -4466,7 +4523,7 @@ static PyTypeObject PyDateTime_TimeZoneType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - timezone_doc, /* tp_doc */ + timezone_new__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ timezone_richcompare, /* tp_richcompare */ @@ -4476,7 +4533,7 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ - 0, /* tp_base; filled in PyInit__datetime */ + &PyDateTime_TZInfoType, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -4568,9 +4625,6 @@ static PyGetSetDef time_getset[] = { * Constructors. */ -static char *time_kws[] = {"hour", "minute", "second", "microsecond", - "tzinfo", "fold", NULL}; - static PyObject * time_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) { @@ -4606,17 +4660,10 @@ time_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) static PyObject * time_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int hour = 0; - int minute = 0; - int second = 0; - int usecond = 0; - PyObject *tzinfo = Py_None; - int fold = 0; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) >= 1 && PyTuple_GET_SIZE(args) <= 2) { PyObject *state = PyTuple_GET_ITEM(args, 0); + PyObject *tzinfo = Py_None; if (PyTuple_GET_SIZE(args) == 2) { tzinfo = PyTuple_GET_ITEM(args, 1); } @@ -4642,40 +4689,70 @@ time_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = time_from_pickle(type, state, tzinfo); + PyObject *self = time_from_pickle(type, state, tzinfo); Py_DECREF(state); return self; } } - tzinfo = Py_None; } - if (PyArg_ParseTupleAndKeywords(args, kw, "|iiiiO$i", time_kws, - &hour, &minute, &second, &usecond, - &tzinfo, &fold)) { - self = new_time_ex2(hour, minute, second, usecond, tzinfo, fold, - type); - } - return self; + return datetime_time(type, args, kw); } -/* Return new time from _strptime.strptime_datetime_time(). */ -static PyObject * -time_strptime(PyObject *cls, PyObject *args) -{ - PyObject *string, *format, *result; +/*[clinic input] +@classmethod +datetime.time.__new__ - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { - return NULL; - } + hour: int = 0 + minute: int = 0 + second: int = 0 + microsecond: int = 0 + tzinfo: object = None + * + fold: int = 0 + +Time with time zone. + +All arguments are optional. tzinfo may be None, or an instance of +a tzinfo subclass. The remaining arguments may be ints. +[clinic start generated code]*/ + +static PyObject * +datetime_time_impl(PyTypeObject *type, int hour, int minute, int second, + int microsecond, PyObject *tzinfo, int fold) +/*[clinic end generated code: output=f06bb4315225e7f6 input=0148df5e8138fe7b]*/ +{ + return new_time_ex2(hour, minute, second, microsecond, tzinfo, fold, type); +} + +/*[clinic input] +@classmethod +datetime.time.strptime + + string: unicode + format: unicode + / + +Parse string according to the given time format (like time.strptime()). + +For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes +[clinic start generated code]*/ + +static PyObject * +datetime_time_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=ae05a9bc0241d3bf input=82ba425ecacc54aa]*/ +{ + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { return NULL; } result = PyObject_CallMethodObjArgs(module, - &_Py_ID(_strptime_datetime_time), cls, - string, format, NULL); + &_Py_ID(_strptime_datetime_time), + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } @@ -4754,17 +4831,30 @@ time_str(PyObject *op) return PyObject_CallMethodNoArgs(op, &_Py_ID(isoformat)); } +/*[clinic input] +datetime.time.isoformat + + timespec: str(c_default="NULL") = 'auto' + +Return the time formatted according to ISO. + +The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional +part is omitted if self.microsecond == 0. + +The optional argument timespec specifies the number of additional +terms of the time to include. Valid options are 'auto', 'hours', +'minutes', 'seconds', 'milliseconds' and 'microseconds'. +[clinic start generated code]*/ + static PyObject * -time_isoformat(PyObject *op, PyObject *args, PyObject *kw) +datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec) +/*[clinic end generated code: output=2bcc7cab65c35545 input=afbbbd953d10ad07]*/ { char buf[100]; - const char *timespec = NULL; - static char *keywords[] = {"timespec", NULL}; - PyDateTime_Time *self = PyTime_CAST(op); PyObject *result; int us = TIME_GET_MICROSECOND(self); - static const char *specs[][2] = { + static const char * const specs[][2] = { {"hours", "%02d"}, {"minutes", "%02d:%02d"}, {"seconds", "%02d:%02d:%02d"}, @@ -4773,9 +4863,6 @@ time_isoformat(PyObject *op, PyObject *args, PyObject *kw) }; size_t given_spec; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|s:isoformat", keywords, &timespec)) - return NULL; - if (timespec == NULL || strcmp(timespec, "auto") == 0) { if (us == 0) { /* seconds */ @@ -4821,18 +4908,26 @@ time_isoformat(PyObject *op, PyObject *args, PyObject *kw) return result; } +/*[clinic input] +@permit_long_docstring_body +datetime.time.strftime + + format: unicode + +Format using strftime(). + +The date part of the timestamp passed to underlying strftime should not be used. + +For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes +[clinic start generated code]*/ + static PyObject * -time_strftime(PyObject *op, PyObject *args, PyObject *kw) +datetime_time_strftime_impl(PyDateTime_Time *self, PyObject *format) +/*[clinic end generated code: output=10f65af20e2a78c7 input=c4a5bbecd798654b]*/ { PyObject *result; PyObject *tuple; - PyObject *format; - static char *keywords[] = {"format", NULL}; - PyDateTime_Time *self = PyTime_CAST(op); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords, - &format)) - return NULL; /* Python's strftime does insane things with the year part of the * timetuple. The year is forced to (the otherwise nonsensical) @@ -4853,6 +4948,27 @@ time_strftime(PyObject *op, PyObject *args, PyObject *kw) return result; } +/*[clinic input] +datetime.time.__format__ + + self: self(type="PyObject *") + format: unicode + / + +Formats self with strftime. +[clinic start generated code]*/ + +static PyObject * +datetime_time___format___impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=4646451f7a5d2156 input=6a858ae787d20230]*/ +{ + /* if the format is zero length, return str(self) */ + if (PyUnicode_GetLength(format) == 0) + return PyObject_Str(self); + + return PyObject_CallMethodOneArg(self, &_Py_ID(strftime), format); +} + /* * Miscellaneous methods. */ @@ -4932,7 +5048,8 @@ static Py_hash_t time_hash(PyObject *op) { PyDateTime_Time *self = PyTime_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *offset, *self0; if (TIME_GET_FOLD(self)) { self0 = new_time_ex2(TIME_GET_HOUR(self), @@ -4954,10 +5071,11 @@ time_hash(PyObject *op) return -1; /* Reduce this to a hash of another object. */ - if (offset == Py_None) - self->hashcode = generic_hash( + if (offset == Py_None) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_TIME_DATASIZE); - else { + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); + } else { PyObject *temp1, *temp2; int seconds, microseconds; assert(HASTZINFO(self)); @@ -4976,12 +5094,13 @@ time_hash(PyObject *op) Py_DECREF(offset); return -1; } - self->hashcode = PyObject_Hash(temp2); + hash = PyObject_Hash(temp2); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp2); } Py_DECREF(offset); } - return self->hashcode; + return hash; } /*[clinic input] @@ -5005,20 +5124,25 @@ datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, /*[clinic end generated code: output=0b89a44c299e4f80 input=abf23656e8df4e97]*/ { return new_time_subclass_fold_ex(hour, minute, second, microsecond, tzinfo, - fold, (PyObject *)Py_TYPE(self)); + fold, Py_TYPE(self)); } +/*[clinic input] +@classmethod +datetime.time.fromisoformat + + string: unicode + / + +Construct a time from a string in ISO 8601 format. +[clinic start generated code]*/ + static PyObject * -time_fromisoformat(PyObject *cls, PyObject *tstr) { - assert(tstr != NULL); - - if (!PyUnicode_Check(tstr)) { - PyErr_SetString(PyExc_TypeError, "fromisoformat: argument must be str"); - return NULL; - } - +datetime_time_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=97c57e896e7f2535 input=bdb4b8abea9cd688]*/ +{ Py_ssize_t len; - const char *p = PyUnicode_AsUTF8AndSize(tstr, &len); + const char *p = PyUnicode_AsUTF8AndSize(string, &len); if (p == NULL) { goto invalid_string_error; @@ -5039,6 +5163,9 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { &tzoffset, &tzimicrosecond); if (rv < 0) { + if (rv == -6) { + goto error; + } goto invalid_string_error; } @@ -5058,10 +5185,10 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } PyObject *t; - if ( (PyTypeObject *)cls == TIME_TYPE(NO_STATE)) { + if (type == TIME_TYPE(NO_STATE)) { t = new_time(hour, minute, second, microsecond, tzinfo, 0); } else { - t = PyObject_CallFunction(cls, "iiiiO", + t = PyObject_CallFunction((PyObject *)type, "iiiiO", hour, minute, second, microsecond, tzinfo); } @@ -5073,7 +5200,10 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { return NULL; invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", tstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); + return NULL; + +error: return NULL; } @@ -5106,21 +5236,28 @@ time_getstate(PyDateTime_Time *self, int proto) return result; } -static PyObject * -time_reduce_ex(PyObject *op, PyObject *args) -{ - int proto; - if (!PyArg_ParseTuple(args, "i:__reduce_ex__", &proto)) - return NULL; +/*[clinic input] +datetime.time.__reduce_ex__ - PyDateTime_Time *self = PyTime_CAST(op); + proto: int + / +[clinic start generated code]*/ + +static PyObject * +datetime_time___reduce_ex___impl(PyDateTime_Time *self, int proto) +/*[clinic end generated code: output=ccfab65f5c320c1b input=4cd06bb3ac3657bb]*/ +{ return Py_BuildValue("(ON)", Py_TYPE(self), time_getstate(self, proto)); } +/*[clinic input] +datetime.time.__reduce__ +[clinic start generated code]*/ + static PyObject * -time_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) +datetime_time___reduce___impl(PyDateTime_Time *self) +/*[clinic end generated code: output=9a2fcc87e64ce300 input=0fb8dd14d275857f]*/ { - PyDateTime_Time *self = PyTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), time_getstate(self, 2)); } @@ -5128,26 +5265,14 @@ static PyMethodDef time_methods[] = { /* Class method: */ - {"strptime", time_strptime, - METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new time parsed from a string " - "(like time.strptime()).")}, + DATETIME_TIME_FROMISOFORMAT_METHODDEF + DATETIME_TIME_STRPTIME_METHODDEF /* Instance methods: */ - {"isoformat", _PyCFunction_CAST(time_isoformat), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]]" - "[+HH:MM].\n\n" - "The optional argument timespec specifies the number " - "of additional terms\nof the time to include. Valid " - "options are 'auto', 'hours', 'minutes',\n'seconds', " - "'milliseconds' and 'microseconds'.\n")}, - - {"strftime", _PyCFunction_CAST(time_strftime), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("format -> strftime() style string.")}, - - {"__format__", date_format, METH_VARARGS, - PyDoc_STR("Formats self with strftime.")}, + DATETIME_TIME_ISOFORMAT_METHODDEF + DATETIME_TIME_STRFTIME_METHODDEF + DATETIME_TIME___FORMAT___METHODDEF {"utcoffset", time_utcoffset, METH_NOARGS, PyDoc_STR("Return self.tzinfo.utcoffset(self).")}, @@ -5163,24 +5288,12 @@ static PyMethodDef time_methods[] = { {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, - {"fromisoformat", time_fromisoformat, METH_O | METH_CLASS, - PyDoc_STR("string -> time from a string in ISO 8601 format")}, - - {"__reduce_ex__", time_reduce_ex, METH_VARARGS, - PyDoc_STR("__reduce_ex__(proto) -> (cls, state)")}, - - {"__reduce__", time_reduce, METH_NOARGS, - PyDoc_STR("__reduce__() -> (cls, state)")}, + DATETIME_TIME___REDUCE_EX___METHODDEF + DATETIME_TIME___REDUCE___METHODDEF {NULL, NULL} }; -static const char time_doc[] = -PyDoc_STR("time([hour[, minute[, second[, microsecond[, tzinfo]]]]]) --> a time object\n\ -\n\ -All arguments are optional. tzinfo may be None, or an instance of\n\ -a tzinfo subclass. The remaining arguments may be ints.\n"); - static PyTypeObject PyDateTime_TimeType = { PyVarObject_HEAD_INIT(NULL, 0) "datetime.time", /* tp_name */ @@ -5201,8 +5314,8 @@ static PyTypeObject PyDateTime_TimeType = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - time_doc, /* tp_doc */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + datetime_time__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ time_richcompare, /* tp_richcompare */ @@ -5288,11 +5401,6 @@ static PyGetSetDef datetime_getset[] = { * Constructors. */ -static char *datetime_kws[] = { - "year", "month", "day", "hour", "minute", "second", - "microsecond", "tzinfo", "fold", NULL -}; - static PyObject * datetime_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) { @@ -5328,20 +5436,10 @@ datetime_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) static PyObject * datetime_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int year; - int month; - int day; - int hour = 0; - int minute = 0; - int second = 0; - int usecond = 0; - int fold = 0; - PyObject *tzinfo = Py_None; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) >= 1 && PyTuple_GET_SIZE(args) <= 2) { PyObject *state = PyTuple_GET_ITEM(args, 0); + PyObject *tzinfo = Py_None; if (PyTuple_GET_SIZE(args) == 2) { tzinfo = PyTuple_GET_ITEM(args, 1); } @@ -5367,22 +5465,46 @@ datetime_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = datetime_from_pickle(type, state, tzinfo); + PyObject *self = datetime_from_pickle(type, state, tzinfo); Py_DECREF(state); return self; } } - tzinfo = Py_None; } - if (PyArg_ParseTupleAndKeywords(args, kw, "iii|iiiiO$i", datetime_kws, - &year, &month, &day, &hour, &minute, - &second, &usecond, &tzinfo, &fold)) { - self = new_datetime_ex2(year, month, day, - hour, minute, second, usecond, - tzinfo, fold, type); - } - return self; + return datetime_datetime(type, args, kw); +} + +/*[clinic input] +@classmethod +datetime.datetime.__new__ + + year: int + month: int + day: int + hour: int = 0 + minute: int = 0 + second: int = 0 + microsecond: int = 0 + tzinfo: object = None + * + fold: int = 0 + +A combination of a date and a time. + +The year, month and day arguments are required. tzinfo may be None, or an +instance of a tzinfo subclass. The remaining arguments may be ints. +[clinic start generated code]*/ + +static PyObject * +datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, + int hour, int minute, int second, int microsecond, + PyObject *tzinfo, int fold) +/*[clinic end generated code: output=47983ddb47d36037 input=2af468d7a9c1e568]*/ +{ + return new_datetime_ex2(year, month, day, + hour, minute, second, microsecond, + tzinfo, fold, type); } /* TM_FUNC is the shared type of _PyTime_localtime() and @@ -5439,7 +5561,7 @@ local(long long u) * Pass localtime or gmtime for f, to control the interpretation of timet. */ static PyObject * -datetime_from_timet_and_us(PyObject *cls, TM_FUNC f, time_t timet, int us, +datetime_from_timet_and_us(PyTypeObject *cls, TM_FUNC f, time_t timet, int us, PyObject *tzinfo) { struct tm tm; @@ -5511,7 +5633,7 @@ datetime_from_timet_and_us(PyObject *cls, TM_FUNC f, time_t timet, int us, * to get that much precision (e.g., C time() isn't good enough). */ static PyObject * -datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, +datetime_from_timestamp(PyTypeObject *cls, TM_FUNC f, PyObject *timestamp, PyObject *tzinfo) { time_t timet; @@ -5529,7 +5651,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, * gmtime for f as appropriate. */ static PyObject * -datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) +datetime_best_possible(PyTypeObject *cls, TM_FUNC f, PyObject *tzinfo) { PyTime_t ts; if (PyTime_Time(&ts) < 0) { @@ -5539,8 +5661,9 @@ datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) time_t secs; int us; - if (_PyTime_AsTimevalTime_t(ts, &secs, &us, _PyTime_ROUND_FLOOR) < 0) + if (_PyTime_AsTimevalTime_t(ts, &secs, &us, _PyTime_ROUND_HALF_EVEN) < 0) { return NULL; + } assert(0 <= us && us <= 999999); return datetime_from_timet_and_us(cls, f, secs, us, tzinfo); @@ -5571,7 +5694,7 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) if (check_tzinfo_subclass(tz) < 0) return NULL; - self = datetime_best_possible((PyObject *)type, + self = datetime_best_possible(type, tz == Py_None ? _PyTime_localtime : _PyTime_gmtime, tz); @@ -5587,8 +5710,16 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) /* Return best possible UTC time -- this isn't constrained by the * precision of a timestamp. */ +/*[clinic input] +@classmethod +datetime.datetime.utcnow + +Return a new datetime representing UTC day and time. +[clinic start generated code]*/ + static PyObject * -datetime_utcnow(PyObject *cls, PyObject *dummy) +datetime_datetime_utcnow_impl(PyTypeObject *type) +/*[clinic end generated code: output=cfcfe71c6c916ba9 input=576eff2b222b80a1]*/ { if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.datetime.utcnow() is deprecated and scheduled for removal in a " @@ -5597,25 +5728,33 @@ datetime_utcnow(PyObject *cls, PyObject *dummy) { return NULL; } - return datetime_best_possible(cls, _PyTime_gmtime, Py_None); + return datetime_best_possible(type, _PyTime_gmtime, Py_None); } -/* Return new local datetime from timestamp (Python timestamp -- a double). */ +/*[clinic input] +@permit_long_docstring_body +@classmethod +datetime.datetime.fromtimestamp + + timestamp: object + tz as tzinfo: object = None + +Create a datetime from a POSIX timestamp. + +The timestamp is a number, e.g. created via time.time(), that is interpreted +as local time. +[clinic start generated code]*/ + static PyObject * -datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) +datetime_datetime_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp, + PyObject *tzinfo) +/*[clinic end generated code: output=9c47ea2b2ebdaded input=d6b5b2095c5a34b2]*/ { PyObject *self; - PyObject *timestamp; - PyObject *tzinfo = Py_None; - static char *keywords[] = {"timestamp", "tz", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kw, "O|O:fromtimestamp", - keywords, &timestamp, &tzinfo)) - return NULL; if (check_tzinfo_subclass(tzinfo) < 0) return NULL; - self = datetime_from_timestamp(cls, + self = datetime_from_timestamp(type, tzinfo == Py_None ? _PyTime_localtime : _PyTime_gmtime, timestamp, @@ -5629,9 +5768,35 @@ datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) return self; } -/* Return new UTC datetime from timestamp (Python timestamp -- a double). */ +/* This is a wrapper for API compatibility with the public C API. */ static PyObject * -datetime_utcfromtimestamp(PyObject *cls, PyObject *args) +datetime_datetime_fromtimestamp_capi(PyObject *cls, PyObject *args, PyObject *kw) +{ + PyObject *timestamp; + PyObject *tzinfo = Py_None; + static char *keywords[] = {"timestamp", "tz", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|O:fromtimestamp", + keywords, &timestamp, &tzinfo)) + return NULL; + return datetime_datetime_fromtimestamp_impl((PyTypeObject *)cls, + timestamp, tzinfo); +} + +/*[clinic input] +@classmethod +datetime.datetime.utcfromtimestamp + + timestamp: object + / + +Create a naive UTC datetime from a POSIX timestamp. +[clinic start generated code]*/ + +static PyObject * +datetime_datetime_utcfromtimestamp_impl(PyTypeObject *type, + PyObject *timestamp) +/*[clinic end generated code: output=66d0b1741d788fd2 input=13fabd4296b1c206]*/ { if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.datetime.utcfromtimestamp() is deprecated and scheduled for removal " @@ -5640,23 +5805,31 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) { return NULL; } - PyObject *timestamp; - PyObject *result = NULL; - if (PyArg_ParseTuple(args, "O:utcfromtimestamp", &timestamp)) - result = datetime_from_timestamp(cls, _PyTime_gmtime, timestamp, - Py_None); - return result; + return datetime_from_timestamp(type, _PyTime_gmtime, timestamp, Py_None); } -/* Return new datetime from _strptime.strptime_datetime_datetime(). */ -static PyObject * -datetime_strptime(PyObject *cls, PyObject *args) -{ - PyObject *string, *format, *result; +/*[clinic input] +@permit_long_summary +@classmethod +datetime.datetime.strptime - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) - return NULL; + string: unicode + format: unicode + / + +Parse string according to the given date and time format (like time.strptime()). + +For a list of supported format codes, see the documentation: + https://docs.python.org/3/library/datetime.html#format-codes +[clinic start generated code]*/ + +static PyObject * +datetime_datetime_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=af2c2d024f3203f5 input=ef7807589f1d50e7]*/ +{ + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { @@ -5664,42 +5837,43 @@ datetime_strptime(PyObject *cls, PyObject *args) } result = PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime_datetime), - cls, string, format, NULL); + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } -/* Return new datetime from date/datetime and time arguments. */ -static PyObject * -datetime_combine(PyObject *cls, PyObject *args, PyObject *kw) -{ - static char *keywords[] = {"date", "time", "tzinfo", NULL}; - PyObject *date; - PyObject *time; - PyObject *tzinfo = NULL; - PyObject *result = NULL; +/*[clinic input] +@classmethod +datetime.datetime.combine - if (PyArg_ParseTupleAndKeywords(args, kw, "O!O!|O:combine", keywords, - DATE_TYPE(NO_STATE), &date, - TIME_TYPE(NO_STATE), &time, &tzinfo)) { - if (tzinfo == NULL) { - if (HASTZINFO(time)) - tzinfo = ((PyDateTime_Time *)time)->tzinfo; - else - tzinfo = Py_None; - } - result = new_datetime_subclass_fold_ex(GET_YEAR(date), - GET_MONTH(date), - GET_DAY(date), - TIME_GET_HOUR(time), - TIME_GET_MINUTE(time), - TIME_GET_SECOND(time), - TIME_GET_MICROSECOND(time), - tzinfo, - TIME_GET_FOLD(time), - cls); + date: object(subclass_of="DATE_TYPE(NO_STATE)") + time: object(subclass_of="TIME_TYPE(NO_STATE)") + tzinfo: object = NULL + +Construct a datetime from a given date and a given time. +[clinic start generated code]*/ + +static PyObject * +datetime_datetime_combine_impl(PyTypeObject *type, PyObject *date, + PyObject *time, PyObject *tzinfo) +/*[clinic end generated code: output=a10f3cbb90f4d0aa input=4fcf0743288d0bab]*/ +{ + if (tzinfo == NULL) { + if (HASTZINFO(time)) + tzinfo = ((PyDateTime_Time *)time)->tzinfo; + else + tzinfo = Py_None; } - return result; + return new_datetime_subclass_fold_ex(GET_YEAR(date), + GET_MONTH(date), + GET_DAY(date), + TIME_GET_HOUR(time), + TIME_GET_MINUTE(time), + TIME_GET_SECOND(time), + TIME_GET_MICROSECOND(time), + tzinfo, + TIME_GET_FOLD(time), + type); } static PyObject * @@ -5857,23 +6031,26 @@ _find_isoformat_datetime_separator(const char *dtstr, Py_ssize_t len) { } } +/*[clinic input] +@classmethod +datetime.datetime.fromisoformat + + string: unicode + / + +Construct a date from a string in ISO 8601 format. +[clinic start generated code]*/ + static PyObject * -datetime_fromisoformat(PyObject *cls, PyObject *dtstr) +datetime_datetime_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=1800a952fcab79d9 input=d517b158209ded42]*/ { - assert(dtstr != NULL); - - if (!PyUnicode_Check(dtstr)) { - PyErr_SetString(PyExc_TypeError, - "fromisoformat: argument must be str"); - return NULL; - } - // We only need to sanitize this string if the separator is a surrogate // character. In the situation where the separator location is ambiguous, // we don't have to sanitize it anything because that can only happen when // the separator is either '-' or a number. This should mostly be a noop // but it makes the reference counting easier if we still sanitize. - PyObject *dtstr_clean = _sanitize_isoformat_str(dtstr); + PyObject *dtstr_clean = _sanitize_isoformat_str(string); if (dtstr_clean == NULL) { goto invalid_string_error; } @@ -5927,6 +6104,9 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) len -= (p - dt_ptr); rv = parse_isoformat_time(p, len, &hour, &minute, &second, &microsecond, &tzoffset, &tzusec); + if (rv == -6) { + goto error; + } } if (rv < 0) { goto invalid_string_error; @@ -5958,7 +6138,7 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) } } PyObject *dt = new_datetime_subclass_ex(year, month, day, hour, minute, - second, microsecond, tzinfo, cls); + second, microsecond, tzinfo, type); Py_DECREF(tzinfo); Py_DECREF(dtstr_clean); @@ -5971,7 +6151,7 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) return NULL; invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); error: Py_XDECREF(dtstr_clean); @@ -6048,7 +6228,7 @@ add_datetime_timedelta(PyDateTime_DateTime *date, PyDateTime_Delta *delta, return new_datetime_subclass_ex(year, month, day, hour, minute, second, microsecond, HASTZINFO(date) ? date->tzinfo : Py_None, - (PyObject *)Py_TYPE(date)); + Py_TYPE(date)); } static PyObject * @@ -6210,18 +6390,38 @@ datetime_str(PyObject *op) return res; } +/*[clinic input] +datetime.datetime.isoformat + + sep: int(accept={str}, c_default="'T'", py_default="'T'") = ord('T') + timespec: str(c_default="NULL") = 'auto' + +Return the time formatted according to ISO. + +The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. +By default, the fractional part is omitted if self.microsecond == 0. + +If self.tzinfo is not None, the UTC offset is also attached, giving +a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. + +Optional argument sep specifies the separator between date and +time, default 'T'. + +The optional argument timespec specifies the number of additional +terms of the time to include. Valid options are 'auto', 'hours', +'minutes', 'seconds', 'milliseconds' and 'microseconds'. +[clinic start generated code]*/ + static PyObject * -datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) +datetime_datetime_isoformat_impl(PyDateTime_DateTime *self, int sep, + const char *timespec) +/*[clinic end generated code: output=9b6ce1383189b0bf input=2fa2512172ccf5d5]*/ { - int sep = 'T'; - char *timespec = NULL; - static char *keywords[] = {"sep", "timespec", NULL}; char buffer[100]; - PyDateTime_DateTime *self = PyDateTime_CAST(op); PyObject *result = NULL; int us = DATE_GET_MICROSECOND(self); - static const char *specs[][2] = { + static const char * const specs[][2] = { {"hours", "%04d-%02d-%02d%c%02d"}, {"minutes", "%04d-%02d-%02d%c%02d:%02d"}, {"seconds", "%04d-%02d-%02d%c%02d:%02d:%02d"}, @@ -6230,9 +6430,6 @@ datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) }; size_t given_spec; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|Cs:isoformat", keywords, &sep, &timespec)) - return NULL; - if (timespec == NULL || strcmp(timespec, "auto") == 0) { if (us == 0) { /* seconds */ @@ -6270,7 +6467,7 @@ datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) return result; /* We need to append the UTC offset. */ - if (format_utcoffset(buffer, sizeof(buffer), ":", self->tzinfo, op) < 0) { + if (format_utcoffset(buffer, sizeof(buffer), ":", self->tzinfo, (PyObject *)self) < 0) { Py_DECREF(result); return NULL; } @@ -6400,8 +6597,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op) PyDateTime_Delta *delta; assert(offset1 != offset2); /* else last "if" handled it */ - delta = (PyDateTime_Delta *)datetime_subtract((PyObject *)self, - other); + delta = (PyDateTime_Delta *)datetime_subtract(self, other); if (delta == NULL) goto done; diff = GET_TD_DAYS(delta); @@ -6439,7 +6635,8 @@ static Py_hash_t datetime_hash(PyObject *op) { PyDateTime_DateTime *self = PyDateTime_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *offset, *self0; if (DATE_GET_FOLD(self)) { self0 = new_datetime_ex2(GET_YEAR(self), @@ -6464,10 +6661,11 @@ datetime_hash(PyObject *op) return -1; /* Reduce this to a hash of another object. */ - if (offset == Py_None) - self->hashcode = generic_hash( + if (offset == Py_None) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_DATETIME_DATASIZE); - else { + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); + } else { PyObject *temp1, *temp2; int days, seconds; @@ -6491,12 +6689,13 @@ datetime_hash(PyObject *op) Py_DECREF(offset); return -1; } - self->hashcode = PyObject_Hash(temp2); + hash = PyObject_Hash(temp2); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp2); } Py_DECREF(offset); } - return self->hashcode; + return hash; } /*[clinic input] @@ -6525,7 +6724,7 @@ datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, { return new_datetime_subclass_fold_ex(year, month, day, hour, minute, second, microsecond, tzinfo, fold, - (PyObject *)Py_TYPE(self)); + Py_TYPE(self)); } static PyObject * @@ -6661,20 +6860,23 @@ local_timezone_from_local(PyDateTime_DateTime *local_dt) return local_timezone_from_timestamp(timestamp); } +/*[clinic input] +datetime.datetime.astimezone + + tz as tzinfo: object = None + +Convert to local time in new timezone tz. +[clinic start generated code]*/ + static PyObject * -datetime_astimezone(PyObject *op, PyObject *args, PyObject *kw) +datetime_datetime_astimezone_impl(PyDateTime_DateTime *self, + PyObject *tzinfo) +/*[clinic end generated code: output=ae2263d04e944537 input=9c675c8595009935]*/ { - PyDateTime_DateTime *self = PyDateTime_CAST(op); PyDateTime_DateTime *result; PyObject *offset; PyObject *temp; PyObject *self_tzinfo; - PyObject *tzinfo = Py_None; - static char *keywords[] = {"tz", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|O:astimezone", keywords, - &tzinfo)) - return NULL; if (check_tzinfo_subclass(tzinfo) == -1) return NULL; @@ -6971,22 +7173,29 @@ datetime_getstate(PyDateTime_DateTime *self, int proto) return result; } -static PyObject * -datetime_reduce_ex(PyObject *op, PyObject *args) -{ - int proto; - if (!PyArg_ParseTuple(args, "i:__reduce_ex__", &proto)) - return NULL; +/*[clinic input] +datetime.datetime.__reduce_ex__ - PyDateTime_DateTime *self = PyDateTime_CAST(op); + proto: int + / +[clinic start generated code]*/ + +static PyObject * +datetime_datetime___reduce_ex___impl(PyDateTime_DateTime *self, int proto) +/*[clinic end generated code: output=53d712ce3e927735 input=bab748e49ffb30c3]*/ +{ return Py_BuildValue("(ON)", Py_TYPE(self), datetime_getstate(self, proto)); } +/*[clinic input] +datetime.datetime.__reduce__ +[clinic start generated code]*/ + static PyObject * -datetime_reduce(PyObject *op, PyObject *Py_UNUSED(arg)) +datetime_datetime___reduce___impl(PyDateTime_DateTime *self) +/*[clinic end generated code: output=6794df9ea75666cf input=cadbbeb3bf3bf94c]*/ { - PyDateTime_DateTime *self = PyDateTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), datetime_getstate(self, 2)); } @@ -6996,31 +7205,12 @@ static PyMethodDef datetime_methods[] = { /* Class methods: */ DATETIME_DATETIME_NOW_METHODDEF - - {"utcnow", datetime_utcnow, - METH_NOARGS | METH_CLASS, - PyDoc_STR("Return a new datetime representing UTC day and time.")}, - - {"fromtimestamp", _PyCFunction_CAST(datetime_fromtimestamp), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("timestamp[, tz] -> tz's local time from POSIX timestamp.")}, - - {"utcfromtimestamp", datetime_utcfromtimestamp, - METH_VARARGS | METH_CLASS, - PyDoc_STR("Construct a naive UTC datetime from a POSIX timestamp.")}, - - {"strptime", datetime_strptime, - METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new datetime parsed from a string " - "(like time.strptime()).")}, - - {"combine", _PyCFunction_CAST(datetime_combine), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("date, time -> datetime with same date and time fields")}, - - {"fromisoformat", datetime_fromisoformat, - METH_O | METH_CLASS, - PyDoc_STR("string -> datetime from a string in most ISO 8601 formats")}, + DATETIME_DATETIME_UTCNOW_METHODDEF + DATETIME_DATETIME_FROMTIMESTAMP_METHODDEF + DATETIME_DATETIME_UTCFROMTIMESTAMP_METHODDEF + DATETIME_DATETIME_STRPTIME_METHODDEF + DATETIME_DATETIME_COMBINE_METHODDEF + DATETIME_DATETIME_FROMISOFORMAT_METHODDEF /* Instance methods: */ @@ -7045,15 +7235,7 @@ static PyMethodDef datetime_methods[] = { {"utctimetuple", datetime_utctimetuple, METH_NOARGS, PyDoc_STR("Return UTC time tuple, compatible with time.localtime().")}, - {"isoformat", _PyCFunction_CAST(datetime_isoformat), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("[sep] -> string in ISO 8601 format, " - "YYYY-MM-DDT[HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\n" - "sep is used to separate the year from the time, and " - "defaults to 'T'.\n" - "The optional argument timespec specifies the number " - "of additional terms\nof the time to include. Valid " - "options are 'auto', 'hours', 'minutes',\n'seconds', " - "'milliseconds' and 'microseconds'.\n")}, + DATETIME_DATETIME_ISOFORMAT_METHODDEF {"utcoffset", datetime_utcoffset, METH_NOARGS, PyDoc_STR("Return self.tzinfo.utcoffset(self).")}, @@ -7069,24 +7251,13 @@ static PyMethodDef datetime_methods[] = { {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, - {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, - - {"__reduce_ex__", datetime_reduce_ex, METH_VARARGS, - PyDoc_STR("__reduce_ex__(proto) -> (cls, state)")}, - - {"__reduce__", datetime_reduce, METH_NOARGS, - PyDoc_STR("__reduce__() -> (cls, state)")}, + DATETIME_DATETIME_ASTIMEZONE_METHODDEF + DATETIME_DATETIME___REDUCE_EX___METHODDEF + DATETIME_DATETIME___REDUCE___METHODDEF {NULL, NULL} }; -static const char datetime_doc[] = -PyDoc_STR("datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])\n\ -\n\ -The year, month and day arguments are required. tzinfo may be None, or an\n\ -instance of a tzinfo subclass. The remaining arguments may be ints.\n"); - static PyNumberMethods datetime_as_number = { datetime_add, /* nb_add */ datetime_subtract, /* nb_subtract */ @@ -7120,8 +7291,8 @@ static PyTypeObject PyDateTime_DateTimeType = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - datetime_doc, /* tp_doc */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + datetime_datetime__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ datetime_richcompare, /* tp_richcompare */ @@ -7131,8 +7302,7 @@ static PyTypeObject PyDateTime_DateTimeType = { datetime_methods, /* tp_methods */ 0, /* tp_members */ datetime_getset, /* tp_getset */ - 0, /* tp_base; filled in - PyInit__datetime */ + &PyDateTime_DateType, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -7179,7 +7349,7 @@ static PyDateTime_CAPI capi = { .Time_FromTime = new_time_ex, .Delta_FromDelta = new_delta_ex, .TimeZone_FromTimeZone = new_timezone, - .DateTime_FromTimestamp = datetime_fromtimestamp, + .DateTime_FromTimestamp = datetime_datetime_fromtimestamp_capi, .Date_FromTimestamp = datetime_date_fromtimestamp_capi, .DateTime_FromDateAndTimeAndFold = new_datetime_ex2, .Time_FromTimeAndFold = new_time_ex2, @@ -7313,29 +7483,82 @@ clear_state(datetime_state *st) } -static int -init_static_types(PyInterpreterState *interp, int reloading) +PyStatus +_PyDateTime_InitTypes(PyInterpreterState *interp) { - if (reloading) { - return 0; - } - - // `&...` is not a constant expression according to a strict reading - // of C standards. Fill tp_base at run-time rather than statically. - // See https://bugs.python.org/issue40777 - PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; - PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - /* Bases classes must be initialized before subclasses, * so capi_types must have the types in the appropriate order. */ for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { PyTypeObject *type = capi_types[i]; if (_PyStaticType_InitForExtension(interp, type) < 0) { - return -1; + return _PyStatus_ERR("could not initialize static types"); } } - return 0; +#define DATETIME_ADD_MACRO(dict, c, value_expr) \ + do { \ + assert(!PyErr_Occurred()); \ + PyObject *value = (value_expr); \ + if (value == NULL) { \ + goto error; \ + } \ + if (PyDict_SetItemString(dict, c, value) < 0) { \ + Py_DECREF(value); \ + goto error; \ + } \ + Py_DECREF(value); \ + } while(0) + + /* timedelta values */ + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); + DATETIME_ADD_MACRO(d, "max", + new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); + + /* date values */ + d = _PyType_GetDict(&PyDateTime_DateType); + DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); + DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); + + /* time values */ + d = _PyType_GetDict(&PyDateTime_TimeType); + DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* datetime values */ + d = _PyType_GetDict(&PyDateTime_DateTimeType); + DATETIME_ADD_MACRO(d, "min", + new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, + 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* timezone values */ + d = _PyType_GetDict(&PyDateTime_TimeZoneType); + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { + goto error; + } + + /* bpo-37642: These attributes are rounded to the nearest minute for backwards + * compatibility, even though the constructor will accept a wider range of + * values. This may change in the future.*/ + + /* -23:59 */ + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); + + /* +23:59 */ + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + +#undef DATETIME_ADD_MACRO + + return _PyStatus_OK(); + +error: + return _PyStatus_NO_MEMORY(); } @@ -7353,20 +7576,15 @@ _datetime_exec(PyObject *module) { int rc = -1; datetime_state *st = get_module_state(module); - int reloading = 0; PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *old_module = get_current_module(interp, &reloading); + PyObject *old_module = get_current_module(interp); if (PyErr_Occurred()) { assert(old_module == NULL); goto error; } /* We actually set the "current" module right before a successful return. */ - if (init_static_types(interp, reloading) < 0) { - goto error; - } - for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { PyTypeObject *type = capi_types[i]; const char *name = _PyType_Name(type); @@ -7380,68 +7598,6 @@ _datetime_exec(PyObject *module) goto error; } -#define DATETIME_ADD_MACRO(dict, c, value_expr) \ - do { \ - assert(!PyErr_Occurred()); \ - PyObject *value = (value_expr); \ - if (value == NULL) { \ - goto error; \ - } \ - if (PyDict_SetItemString(dict, c, value) < 0) { \ - Py_DECREF(value); \ - goto error; \ - } \ - Py_DECREF(value); \ - } while(0) - - if (!reloading) { - /* timedelta values */ - PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); - DATETIME_ADD_MACRO(d, "max", - new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); - - /* date values */ - d = _PyType_GetDict(&PyDateTime_DateType); - DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); - DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); - - /* time values */ - d = _PyType_GetDict(&PyDateTime_TimeType); - DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* datetime values */ - d = _PyType_GetDict(&PyDateTime_DateTimeType); - DATETIME_ADD_MACRO(d, "min", - new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, - 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* timezone values */ - d = _PyType_GetDict(&PyDateTime_TimeZoneType); - if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { - goto error; - } - - /* bpo-37642: These attributes are rounded to the nearest minute for backwards - * compatibility, even though the constructor will accept a wider range of - * values. This may change in the future.*/ - - /* -23:59 */ - DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); - - /* +23:59 */ - DATETIME_ADD_MACRO( - d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); - } - -#undef DATETIME_ADD_MACRO - /* Add module level attributes */ if (PyModule_AddIntMacro(module, MINYEAR) < 0) { goto error; @@ -7498,6 +7654,7 @@ _datetime_exec(PyObject *module) } static PyModuleDef_Slot module_slots[] = { + _Py_INTERNAL_ABI_SLOT, {Py_mod_exec, _datetime_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index cc65cbd98d7..f88861fa244 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -8,6 +8,7 @@ #endif #include "Python.h" +#include "pycore_object.h" // _PyObject_VisitType() #include <sys/types.h> #include <sys/stat.h> @@ -69,6 +70,7 @@ typedef struct { #include "clinic/_dbmmodule.c.h" #define check_dbmobject_open(v, err) \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED((v)) \ if ((v)->di_dbm == NULL) { \ PyErr_SetString(err, "DBM object has already been closed"); \ return NULL; \ @@ -95,12 +97,6 @@ newdbmobject(_dbm_state *state, const char *file, int flags, int mode) } /* Methods */ -static int -dbm_traverse(PyObject *dp, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(dp)); - return 0; -} static void dbm_dealloc(PyObject *self) @@ -116,7 +112,7 @@ dbm_dealloc(PyObject *self) } static Py_ssize_t -dbm_length(PyObject *self) +dbm_length_lock_held(PyObject *self) { dbmobject *dp = dbmobject_CAST(self); _dbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); @@ -138,8 +134,18 @@ dbm_length(PyObject *self) return dp->di_size; } +static Py_ssize_t +dbm_length(PyObject *self) +{ + Py_ssize_t result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_length_lock_held(self); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -dbm_bool(PyObject *self) +dbm_bool_lock_held(PyObject *self) { dbmobject *dp = dbmobject_CAST(self); _dbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); @@ -170,8 +176,18 @@ dbm_bool(PyObject *self) return 1; } +static int +dbm_bool(PyObject *self) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_bool_lock_held(self); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * -dbm_subscript(PyObject *self, PyObject *key) +dbm_subscript_lock_held(PyObject *self, PyObject *key) { datum drec, krec; Py_ssize_t tmp_size; @@ -197,8 +213,18 @@ dbm_subscript(PyObject *self, PyObject *key) return PyBytes_FromStringAndSize(drec.dptr, drec.dsize); } +static PyObject * +dbm_subscript(PyObject *self, PyObject *key) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_subscript_lock_held(self, key); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -dbm_ass_sub(PyObject *self, PyObject *v, PyObject *w) +dbm_ass_sub_lock_held(PyObject *self, PyObject *v, PyObject *w) { datum krec, drec; Py_ssize_t tmp_size; @@ -252,7 +278,18 @@ dbm_ass_sub(PyObject *self, PyObject *v, PyObject *w) return 0; } +static int +dbm_ass_sub(PyObject *self, PyObject *v, PyObject *w) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_ass_sub_lock_held(self, v, w); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] +@critical_section _dbm.dbm.close Close the database. @@ -260,7 +297,7 @@ Close the database. static PyObject * _dbm_dbm_close_impl(dbmobject *self) -/*[clinic end generated code: output=c8dc5b6709600b86 input=046db72377d51be8]*/ +/*[clinic end generated code: output=c8dc5b6709600b86 input=4a94f79facbc28ca]*/ { if (self->di_dbm) { dbm_close(self->di_dbm); @@ -270,6 +307,7 @@ _dbm_dbm_close_impl(dbmobject *self) } /*[clinic input] +@critical_section _dbm.dbm.keys cls: defining_class @@ -279,7 +317,7 @@ Return a list of all keys in the database. static PyObject * _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=f2a593b3038e5996 input=d3706a28fc051097]*/ +/*[clinic end generated code: output=f2a593b3038e5996 input=6ddefeadf2a80156]*/ { PyObject *v, *item; datum key; @@ -310,7 +348,7 @@ _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls) } static int -dbm_contains(PyObject *self, PyObject *arg) +dbm_contains_lock_held(PyObject *self, PyObject *arg) { dbmobject *dp = dbmobject_CAST(self); datum key, val; @@ -343,7 +381,18 @@ dbm_contains(PyObject *self, PyObject *arg) return val.dptr != NULL; } +static int +dbm_contains(PyObject *self, PyObject *arg) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_contains_lock_held(self, arg); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] +@critical_section _dbm.dbm.get cls: defining_class key: str(accept={str, robuffer}, zeroes=True) @@ -356,7 +405,7 @@ Return the value for key if present, otherwise default. static PyObject * _dbm_dbm_get_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value) -/*[clinic end generated code: output=b4e55f8b6d482bc4 input=66b993b8349fa8c1]*/ +/*[clinic end generated code: output=b4e55f8b6d482bc4 input=1d88a22bb5e55202]*/ { datum dbm_key, val; _dbm_state *state = PyType_GetModuleState(cls); @@ -373,6 +422,7 @@ _dbm_dbm_get_impl(dbmobject *self, PyTypeObject *cls, const char *key, } /*[clinic input] +@critical_section _dbm.dbm.setdefault cls: defining_class key: str(accept={str, robuffer}, zeroes=True) @@ -387,7 +437,7 @@ If key is not in the database, it is inserted with default as the value. static PyObject * _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value) -/*[clinic end generated code: output=9c2f6ea6d0fb576c input=126a3ff15c5f8232]*/ +/*[clinic end generated code: output=9c2f6ea6d0fb576c input=c01510ef7571e13b]*/ { datum dbm_key, val; Py_ssize_t tmp_size; @@ -401,10 +451,7 @@ _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, return PyBytes_FromStringAndSize(val.dptr, val.dsize); } if (default_value == NULL) { - default_value = PyBytes_FromStringAndSize(NULL, 0); - if (default_value == NULL) { - return NULL; - } + default_value = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); val.dptr = NULL; val.dsize = 0; } @@ -427,6 +474,7 @@ _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, } /*[clinic input] +@critical_section _dbm.dbm.clear cls: defining_class / @@ -436,7 +484,7 @@ Remove all items from the database. static PyObject * _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=8d126b9e1d01a434 input=43aa6ca1acb7f5f5]*/ +/*[clinic end generated code: output=8d126b9e1d01a434 input=a1aa5d99adfb9656]*/ { _dbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -467,8 +515,12 @@ dbm__enter__(PyObject *self, PyObject *Py_UNUSED(dummy)) static PyObject * dbm__exit__(PyObject *self, PyObject *Py_UNUSED(args)) { + PyObject *result; dbmobject *dp = dbmobject_CAST(self); - return _dbm_dbm_close_impl(dp); + Py_BEGIN_CRITICAL_SECTION(self); + result = _dbm_dbm_close_impl(dp); + Py_END_CRITICAL_SECTION(); + return result; } static PyMethodDef dbm_methods[] = { @@ -484,7 +536,7 @@ static PyMethodDef dbm_methods[] = { static PyType_Slot dbmtype_spec_slots[] = { {Py_tp_dealloc, dbm_dealloc}, - {Py_tp_traverse, dbm_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_methods, dbm_methods}, {Py_sq_contains, dbm_contains}, {Py_mp_length, dbm_length}, diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 602b23cfca8..0484d9896a1 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -30,9 +30,9 @@ #endif #include <Python.h> +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_typeobject.h" -#include "complexobject.h" #include <mpdecimal.h> @@ -50,14 +50,24 @@ #include <ctype.h> // isascii() #include <stdlib.h> -#include "docstrings.h" - #ifdef EXTRA_FUNCTIONALITY #define _PY_DEC_ROUND_GUARD MPD_ROUND_GUARD #else #define _PY_DEC_ROUND_GUARD (MPD_ROUND_GUARD-1) #endif +#include "clinic/_decimal.c.h" + +#define MPD_SPEC_VERSION "1.70" // Highest version of the spec this complies with + // See https://speleotrove.com/decimal/decarith.html + +/*[clinic input] +module _decimal +class _decimal.Decimal "PyObject *" "&dec_spec" +class _decimal.Context "PyObject *" "&context_spec" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a6a6c0bdf4e576ef]*/ + struct PyDecContextObject; struct DecCondMap; @@ -739,13 +749,6 @@ signaldict_setitem(PyObject *self, PyObject *key, PyObject *value) return 0; } -static int -signaldict_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static void signaldict_dealloc(PyObject *self) { @@ -838,7 +841,7 @@ static PyMethodDef signaldict_methods[] = { static PyType_Slot signaldict_slots[] = { {Py_tp_dealloc, signaldict_dealloc}, - {Py_tp_traverse, signaldict_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_repr, signaldict_repr}, {Py_tp_hash, PyObject_HashNotImplemented}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -914,14 +917,34 @@ context_getallcr(PyObject *self, void *Py_UNUSED(closure)) } #endif +/*[clinic input] +_decimal.Context.Etiny + +Return a value equal to Emin - prec + 1. + +This is the minimum exponent value for subnormal results. When +underflow occurs, the exponent is set to Etiny. +[clinic start generated code]*/ + static PyObject * -context_getetiny(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Context_Etiny_impl(PyObject *self) +/*[clinic end generated code: output=c9a4a1a3e3575289 input=1274040f303f2244]*/ { return PyLong_FromSsize_t(mpd_etiny(CTX(self))); } +/*[clinic input] +_decimal.Context.Etop + +Return a value equal to Emax - prec + 1. + +This is the maximum exponent if the _clamp field of the context is set +to 1 (IEEE clamp mode). Etop() must not be negative. +[clinic start generated code]*/ + static PyObject * -context_getetop(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Context_Etop_impl(PyObject *self) +/*[clinic end generated code: output=f0a3f6e1b829074e input=838a4409316ec728]*/ { return PyLong_FromSsize_t(mpd_etop(CTX(self))); } @@ -987,16 +1010,19 @@ context_setemax(PyObject *self, PyObject *value, void *Py_UNUSED(closure)) } #ifdef CONFIG_32 +/*[clinic input] +_decimal.Context._unsafe_setprec + + x: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -context_unsafe_setprec(PyObject *self, PyObject *value) +_decimal_Context__unsafe_setprec_impl(PyObject *self, Py_ssize_t x) +/*[clinic end generated code: output=dd838edf08e12dd9 input=23a1b19ceb1569be]*/ { mpd_context_t *ctx = CTX(self); - mpd_ssize_t x; - - x = PyLong_AsSsize_t(value); - if (x == -1 && PyErr_Occurred()) { - return NULL; - } if (x < 1 || x > 1070000000L) { return value_error_ptr( @@ -1007,16 +1033,19 @@ context_unsafe_setprec(PyObject *self, PyObject *value) Py_RETURN_NONE; } +/*[clinic input] +_decimal.Context._unsafe_setemin + + x: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -context_unsafe_setemin(PyObject *self, PyObject *value) +_decimal_Context__unsafe_setemin_impl(PyObject *self, Py_ssize_t x) +/*[clinic end generated code: output=0c49cafee8a65846 input=652f1ecacca7e0ce]*/ { mpd_context_t *ctx = CTX(self); - mpd_ssize_t x; - - x = PyLong_AsSsize_t(value); - if (x == -1 && PyErr_Occurred()) { - return NULL; - } if (x < -1070000000L || x > 0) { return value_error_ptr( @@ -1027,16 +1056,19 @@ context_unsafe_setemin(PyObject *self, PyObject *value) Py_RETURN_NONE; } +/*[clinic input] +_decimal.Context._unsafe_setemax + + x: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -context_unsafe_setemax(PyObject *self, PyObject *value) +_decimal_Context__unsafe_setemax_impl(PyObject *self, Py_ssize_t x) +/*[clinic end generated code: output=776563e0377a00e8 input=b2a32a9a2750e7a8]*/ { mpd_context_t *ctx = CTX(self); - mpd_ssize_t x; - - x = PyLong_AsSsize_t(value); - if (x == -1 && PyErr_Occurred()) { - return NULL; - } if (x < 0 || x > 1070000000L) { return value_error_ptr( @@ -1361,15 +1393,29 @@ context_setattrs(PyObject *self, PyObject *prec, PyObject *rounding, return 0; } +/*[clinic input] +_decimal.Context.clear_traps + +Set all traps to False. +[clinic start generated code]*/ + static PyObject * -context_clear_traps(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Context_clear_traps_impl(PyObject *self) +/*[clinic end generated code: output=b47cfa6e32407d40 input=3872e80637148035]*/ { CTX(self)->traps = 0; Py_RETURN_NONE; } +/*[clinic input] +_decimal.Context.clear_flags + +Reset all flags to False. +[clinic start generated code]*/ + static PyObject * -context_clear_flags(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Context_clear_flags_impl(PyObject *self) +/*[clinic end generated code: output=c86719a70177d0b6 input=a06055e2f3e7edb1]*/ { CTX(self)->status = 0; Py_RETURN_NONE; @@ -1467,32 +1513,37 @@ context_dealloc(PyObject *self) Py_DECREF(tp); } +/*[clinic input] +_decimal.Context.__init__ as context_init + + prec: object = None + rounding: object = None + Emin as emin: object = None + Emax as emax: object = None + capitals: object = None + clamp: object = None + flags as status: object = None + traps: object = None + +Create context. + +The context affects almost all operations and controls rounding, +Over/Underflow, raising of exceptions and much more. A new context +can be constructed as follows: + + >>> c = Context(prec=28, Emin=-425000000, Emax=425000000, + ... rounding=ROUND_HALF_EVEN, capitals=1, clamp=1, + ... traps=[InvalidOperation, DivisionByZero, Overflow], + ... flags=[]) + >>> +[clinic start generated code]*/ + static int -context_init(PyObject *self, PyObject *args, PyObject *kwds) +context_init_impl(PyObject *self, PyObject *prec, PyObject *rounding, + PyObject *emin, PyObject *emax, PyObject *capitals, + PyObject *clamp, PyObject *status, PyObject *traps) +/*[clinic end generated code: output=8bfdc59fbe862f44 input=45c704b93cd02959]*/ { - static char *kwlist[] = { - "prec", "rounding", "Emin", "Emax", "capitals", "clamp", - "flags", "traps", NULL - }; - PyObject *prec = Py_None; - PyObject *rounding = Py_None; - PyObject *emin = Py_None; - PyObject *emax = Py_None; - PyObject *capitals = Py_None; - PyObject *clamp = Py_None; - PyObject *status = Py_None; - PyObject *traps = Py_None; - - assert(PyTuple_Check(args)); - - if (!PyArg_ParseTupleAndKeywords( - args, kwds, - "|OOOOOOOO", kwlist, - &prec, &rounding, &emin, &emax, &capitals, &clamp, &status, &traps - )) { - return -1; - } - return context_setattrs( self, prec, rounding, emin, emax, capitals, @@ -1558,17 +1609,26 @@ init_extended_context(PyObject *v) } /* Factory function for creating IEEE interchange format contexts */ + +/*[clinic input] +_decimal.IEEEContext + + bits: Py_ssize_t + / + +Return a context, initialized as one of the IEEE interchange formats. + +The argument must be a multiple of 32 and less than +IEEE_CONTEXT_MAX_BITS. +[clinic start generated code]*/ + static PyObject * -ieee_context(PyObject *module, PyObject *v) +_decimal_IEEEContext_impl(PyObject *module, Py_ssize_t bits) +/*[clinic end generated code: output=19a35f320fe19789 input=5cff864d899eb2d7]*/ { PyObject *context; - mpd_ssize_t bits; mpd_context_t ctx; - bits = PyLong_AsSsize_t(v); - if (bits == -1 && PyErr_Occurred()) { - return NULL; - } if (bits <= 0 || bits > INT_MAX) { goto error; } @@ -1594,31 +1654,67 @@ ieee_context(PyObject *module, PyObject *v) } static PyObject * -context_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) +context_copy(decimal_state *state, PyObject *v) { - PyObject *copy; + PyObject *copy = + PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); - decimal_state *state = get_module_state_from_ctx(self); - copy = PyObject_CallObject((PyObject *)state->PyDecContext_Type, NULL); if (copy == NULL) { return NULL; } - *CTX(copy) = *CTX(self); + *CTX(copy) = *CTX(v); CTX(copy)->newtrap = 0; - CtxCaps(copy) = CtxCaps(self); + CtxCaps(copy) = CtxCaps(v); return copy; } +/*[clinic input] +_decimal.Context.copy + + cls: defining_class + +Return a duplicate of the context with all flags cleared. +[clinic start generated code]*/ + static PyObject * -context_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Context_copy_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=31c9c8eeb0c0cf77 input=aef1c0bddabdf8f0]*/ +{ + decimal_state *state = PyType_GetModuleState(cls); + + return context_copy(state, self); +} + +/*[clinic input] +_decimal.Context.__copy__ = _decimal.Context.copy + +[clinic start generated code]*/ + +static PyObject * +_decimal_Context___copy___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=93552486e5fb0ab4 input=4a55dd22f6d31bcc]*/ +{ + decimal_state *state = PyType_GetModuleState(cls); + + return context_copy(state, self); +} + +/*[clinic input] +_decimal.Context.__reduce__ = _decimal.Context.copy + +[clinic start generated code]*/ + +static PyObject * +_decimal_Context___reduce___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=4e77de55efdbb56a input=787683f13d047ce8]*/ { PyObject *flags; PyObject *traps; PyObject *ret; mpd_context_t *ctx; - decimal_state *state = get_module_state_from_ctx(self); + decimal_state *state = PyType_GetModuleState(cls); ctx = CTX(self); @@ -1721,7 +1817,7 @@ current_context_from_dict(decimal_state *modstate) } /* Set up a new thread local context. */ - tl_context = context_copy(modstate->default_context_template, NULL); + tl_context = context_copy(modstate, modstate->default_context_template); if (tl_context == NULL) { return NULL; } @@ -1767,7 +1863,7 @@ current_context(decimal_state *modstate) /* Return a new reference to the current context */ static PyObject * -PyDec_GetCurrentContext(PyObject *self, PyObject *Py_UNUSED(dummy)) +PyDec_GetCurrentContext(PyObject *self) { PyObject *context; decimal_state *state = get_module_state(self); @@ -1797,7 +1893,7 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) if (v == state->default_context_template || v == state->basic_context_template || v == state->extended_context_template) { - v = context_copy(v, NULL); + v = context_copy(state, v); if (v == NULL) { return NULL; } @@ -1820,7 +1916,7 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) static PyObject * init_current_context(decimal_state *state) { - PyObject *tl_context = context_copy(state->default_context_template, NULL); + PyObject *tl_context = context_copy(state, state->default_context_template); if (tl_context == NULL) { return NULL; } @@ -1863,7 +1959,7 @@ current_context(decimal_state *state) /* Return a new reference to the current context */ static PyObject * -PyDec_GetCurrentContext(PyObject *self, PyObject *Py_UNUSED(dummy)) +PyDec_GetCurrentContext(PyObject *self) { decimal_state *state = get_module_state(self); return current_context(state); @@ -1881,7 +1977,7 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) if (v == state->default_context_template || v == state->basic_context_template || v == state->extended_context_template) { - v = context_copy(v, NULL); + v = context_copy(state, v); if (v == NULL) { return NULL; } @@ -1902,36 +1998,73 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) } #endif +/*[clinic input] +_decimal.getcontext + +Get the current default context. +[clinic start generated code]*/ + +static PyObject * +_decimal_getcontext_impl(PyObject *module) +/*[clinic end generated code: output=5982062c4d39e3dd input=7ac316fe42a1b6f5]*/ +{ + return PyDec_GetCurrentContext(module); +} + +/*[clinic input] +_decimal.setcontext + + context: object + / + +Set a new default context. +[clinic start generated code]*/ + +static PyObject * +_decimal_setcontext(PyObject *module, PyObject *context) +/*[clinic end generated code: output=8065f870be2852ce input=b57d7ee786b022a6]*/ +{ + return PyDec_SetCurrentContext(module, context); +} + /* Context manager object for the 'with' statement. The manager * owns one reference to the global (outer) context and one * to the local (inner) context. */ + +/*[clinic input] +@text_signature "($module, /, ctx=None, **kwargs)" +_decimal.localcontext + + ctx as local: object = None + * + prec: object = None + rounding: object = None + Emin: object = None + Emax: object = None + capitals: object = None + clamp: object = None + flags: object = None + traps: object = None + +Return a context manager for a copy of the supplied context. + +That will set the default context to a copy of ctx on entry to the +with-statement and restore the previous default context when exiting +the with-statement. If no context is specified, a copy of the current +default context is used. +[clinic start generated code]*/ + static PyObject * -ctxmanager_new(PyObject *m, PyObject *args, PyObject *kwds) +_decimal_localcontext_impl(PyObject *module, PyObject *local, PyObject *prec, + PyObject *rounding, PyObject *Emin, + PyObject *Emax, PyObject *capitals, + PyObject *clamp, PyObject *flags, PyObject *traps) +/*[clinic end generated code: output=9bf4e47742a809b0 input=490307b9689c3856]*/ { - static char *kwlist[] = { - "ctx", "prec", "rounding", - "Emin", "Emax", "capitals", - "clamp", "flags", "traps", - NULL - }; - PyObject *local = Py_None; PyObject *global; - PyObject *prec = Py_None; - PyObject *rounding = Py_None; - PyObject *Emin = Py_None; - PyObject *Emax = Py_None; - PyObject *capitals = Py_None; - PyObject *clamp = Py_None; - PyObject *flags = Py_None; - PyObject *traps = Py_None; - - decimal_state *state = get_module_state(m); + decimal_state *state = get_module_state(module); CURRENT_CONTEXT(state, global); - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOOOOOOO", kwlist, &local, - &prec, &rounding, &Emin, &Emax, &capitals, &clamp, &flags, &traps)) { - return NULL; - } if (local == Py_None) { local = global; } @@ -1941,7 +2074,7 @@ ctxmanager_new(PyObject *m, PyObject *args, PyObject *kwds) return NULL; } - PyObject *local_copy = context_copy(local, NULL); + PyObject *local_copy = context_copy(state, local); if (local_copy == NULL) { return NULL; } @@ -2089,13 +2222,6 @@ PyDecType_New(decimal_state *state, PyTypeObject *type) } #define dec_alloc(st) PyDecType_New(st, (st)->PyDec_Type) -static int -dec_traverse(PyObject *dec, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(dec)); - return 0; -} - static void dec_dealloc(PyObject *dec) { @@ -2869,18 +2995,43 @@ PyDecType_FromSequenceExact(PyTypeObject *type, PyObject *v, #define PyDec_FromSequenceExact(st, sequence, context) \ PyDecType_FromSequenceExact((st)->PyDec_Type, sequence, context) -/* class method */ +/*[clinic input] +@classmethod +_decimal.Decimal.from_float + + cls: defining_class + f as pyfloat: object + / + +Class method that converts a float to a decimal number, exactly. + +Since 0.1 is not exactly representable in binary floating point, +Decimal.from_float(0.1) is not the same as Decimal('0.1'). + + >>> Decimal.from_float(0.1) + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_float(float('nan')) + Decimal('NaN') + >>> Decimal.from_float(float('inf')) + Decimal('Infinity') + >>> Decimal.from_float(float('-inf')) + Decimal('-Infinity') +[clinic start generated code]*/ + static PyObject * -dec_from_float(PyObject *type, PyObject *pyfloat) +_decimal_Decimal_from_float_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *pyfloat) +/*[clinic end generated code: output=fcb7d55d2f9dc790 input=03bc8dbe963e52ca]*/ { PyObject *context; PyObject *result; - decimal_state *state = get_module_state_by_def((PyTypeObject *)type); + decimal_state *state = PyType_GetModuleState(cls); CURRENT_CONTEXT(state, context); result = PyDecType_FromFloatExact(state->PyDec_Type, pyfloat, context); - if (type != (PyObject *)state->PyDec_Type && result != NULL) { - Py_SETREF(result, PyObject_CallFunctionObjArgs(type, result, NULL)); + if (type != state->PyDec_Type && result != NULL) { + Py_SETREF(result, + PyObject_CallFunctionObjArgs((PyObject *)type, result, NULL)); } return result; @@ -2890,9 +3041,10 @@ dec_from_float(PyObject *type, PyObject *pyfloat) an exact conversion. If the result does not meet the restrictions for an mpd_t, fail with InvalidOperation. */ static PyObject * -PyDecType_FromNumberExact(PyTypeObject *type, PyObject *v, PyObject *context) +PyDecType_FromNumberExact(PyTypeObject *type, PyTypeObject *cls, + PyObject *v, PyObject *context) { - decimal_state *state = get_module_state_by_def(type); + decimal_state *state = PyType_GetModuleState(cls); assert(v != NULL); if (PyDec_Check(state, v)) { return PyDecType_FromDecimalExact(type, v, context); @@ -2914,29 +3066,67 @@ PyDecType_FromNumberExact(PyTypeObject *type, PyObject *v, PyObject *context) } } -/* class method */ +/*[clinic input] +@classmethod +_decimal.Decimal.from_number + + cls: defining_class + number: object + / + +Class method that converts a real number to a decimal number, exactly. + + >>> Decimal.from_number(314) # int + Decimal('314') + >>> Decimal.from_number(0.1) # float + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance + Decimal('3.14') +[clinic start generated code]*/ + static PyObject * -dec_from_number(PyObject *type, PyObject *number) +_decimal_Decimal_from_number_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *number) +/*[clinic end generated code: output=4d3ec722b7acfd8b input=271cb4feb3148804]*/ { PyObject *context; PyObject *result; - decimal_state *state = get_module_state_by_def((PyTypeObject *)type); + decimal_state *state = PyType_GetModuleState(cls); CURRENT_CONTEXT(state, context); - result = PyDecType_FromNumberExact(state->PyDec_Type, number, context); - if (type != (PyObject *)state->PyDec_Type && result != NULL) { - Py_SETREF(result, PyObject_CallFunctionObjArgs(type, result, NULL)); + result = PyDecType_FromNumberExact(state->PyDec_Type, cls, number, context); + if (type != state->PyDec_Type && result != NULL) { + Py_SETREF(result, + PyObject_CallFunctionObjArgs((PyObject *)type, result, NULL)); } return result; } /* create_decimal_from_float */ + +/*[clinic input] +_decimal.Context.create_decimal_from_float + + self as context: self + cls: defining_class + f: object + / + +Create a new Decimal instance from float f. + +Unlike the Decimal.from_float() class method, this function observes +the context limits. +[clinic start generated code]*/ + static PyObject * -ctx_from_float(PyObject *context, PyObject *v) +_decimal_Context_create_decimal_from_float_impl(PyObject *context, + PyTypeObject *cls, + PyObject *f) +/*[clinic end generated code: output=a5548f5140fa0870 input=8c66eeb22b01ddd4]*/ { - decimal_state *state = get_module_state_from_ctx(context); - return PyDec_FromFloat(state, v, context); + decimal_state *state = PyType_GetModuleState(cls); + return PyDec_FromFloat(state, f, context); } /* Apply the context to the input operand. Return a new PyDecObject. */ @@ -3053,33 +3243,49 @@ PyDec_FromObject(PyObject *v, PyObject *context) } } -static PyObject * -dec_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"value", "context", NULL}; - PyObject *v = NULL; - PyObject *context = Py_None; +/*[clinic input] +@classmethod +_decimal.Decimal.__new__ as dec_new - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, - &v, &context)) { - return NULL; - } + value: object(c_default="NULL") = "0" + context: object = None + +Construct a new Decimal object. + +value can be an integer, string, tuple, or another Decimal object. If +no value is given, return Decimal('0'). The context does not affect +the conversion and is only passed to determine if the InvalidOperation +trap is active. +[clinic start generated code]*/ + +static PyObject * +dec_new_impl(PyTypeObject *type, PyObject *value, PyObject *context) +/*[clinic end generated code: output=35f48a40c65625ba input=5f8a0892d3fcef80]*/ +{ decimal_state *state = get_module_state_by_def(type); CONTEXT_CHECK_VA(state, context); - return PyDecType_FromObjectExact(type, v, context); + return PyDecType_FromObjectExact(type, value, context); } +/*[clinic input] +_decimal.Context.create_decimal + + self as context: self + num: object(c_default="NULL") = "0" + / + +Create a new Decimal instance from num, using self as the context. + +Unlike the Decimal constructor, this function observes the context +limits. +[clinic start generated code]*/ + static PyObject * -ctx_create_decimal(PyObject *context, PyObject *args) +_decimal_Context_create_decimal_impl(PyObject *context, PyObject *num) +/*[clinic end generated code: output=85e08ae02f3b34da input=d2c4946cf7804fbe]*/ { - PyObject *v = NULL; - - if (!PyArg_ParseTuple(args, "|O", &v)) { - return NULL; - } - - return PyDec_FromObject(v, context); + return PyDec_FromObject(num, context); } @@ -3520,15 +3726,28 @@ pydec_format(PyObject *dec, PyObject *context, PyObject *fmt, decimal_state *sta } /* Formatted representation of a PyDecObject. */ + +/*[clinic input] +_decimal.Decimal.__format__ + + self as dec: self + cls: defining_class + format_spec as fmtarg: unicode + override: object = NULL + / + +Formats the Decimal according to format_spec. +[clinic start generated code]*/ + static PyObject * -dec_format(PyObject *dec, PyObject *args) +_decimal_Decimal___format___impl(PyObject *dec, PyTypeObject *cls, + PyObject *fmtarg, PyObject *override) +/*[clinic end generated code: output=6d95f91bbb28b3ed input=2dbfaa0cbe243e9e]*/ { PyObject *result = NULL; - PyObject *override = NULL; PyObject *dot = NULL; PyObject *sep = NULL; PyObject *grouping = NULL; - PyObject *fmtarg; PyObject *context; mpd_spec_t spec; char *fmt; @@ -3536,42 +3755,29 @@ dec_format(PyObject *dec, PyObject *args) uint32_t status = 0; int replace_fillchar = 0; Py_ssize_t size; - - - decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); + decimal_state *state = PyType_GetModuleState(cls); CURRENT_CONTEXT(state, context); - if (!PyArg_ParseTuple(args, "O|O", &fmtarg, &override)) { + fmt = (char *)PyUnicode_AsUTF8AndSize(fmtarg, &size); + if (fmt == NULL) { return NULL; } - if (PyUnicode_Check(fmtarg)) { - fmt = (char *)PyUnicode_AsUTF8AndSize(fmtarg, &size); + if (size > 0 && fmt[size-1] == 'N') { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Format specifier 'N' is deprecated", 1) < 0) { + return NULL; + } + } + + if (size > 0 && fmt[0] == '\0') { + /* NUL fill character: must be replaced with a valid UTF-8 char + before calling mpd_parse_fmt_str(). */ + replace_fillchar = 1; + fmt = dec_strdup(fmt, size); if (fmt == NULL) { return NULL; } - - if (size > 0 && fmt[size-1] == 'N') { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Format specifier 'N' is deprecated", 1) < 0) { - return NULL; - } - } - - if (size > 0 && fmt[0] == '\0') { - /* NUL fill character: must be replaced with a valid UTF-8 char - before calling mpd_parse_fmt_str(). */ - replace_fillchar = 1; - fmt = dec_strdup(fmt, size); - if (fmt == NULL) { - return NULL; - } - fmt[0] = '_'; - } - } - else { - PyErr_SetString(PyExc_TypeError, - "format arg must be str"); - return NULL; + fmt[0] = '_'; } if (!mpd_parse_fmt_str(&spec, fmt, CtxCaps(context))) { @@ -3750,9 +3956,20 @@ dec_as_long(PyObject *dec, PyObject *context, int round) return PyLongWriter_Finish(writer); } -/* Convert a Decimal to its exact integer ratio representation. */ +/*[clinic input] +_decimal.Decimal.as_integer_ratio + + cls: defining_class + +Return a pair of integers whose ratio is exactly equal to the original. + +The ratio is in lowest terms and with a positive denominator. +Raise OverflowError on infinities and a ValueError on NaNs. +[clinic start generated code]*/ + static PyObject * -dec_as_integer_ratio(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_as_integer_ratio_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=eb49c512701f844b input=07e33d8852184761]*/ { PyObject *numerator = NULL; PyObject *denominator = NULL; @@ -3775,7 +3992,7 @@ dec_as_integer_ratio(PyObject *self, PyObject *Py_UNUSED(dummy)) return NULL; } - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); CURRENT_CONTEXT(state, context); tmp = dec_alloc(state); @@ -3854,21 +4071,31 @@ dec_as_integer_ratio(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } +/*[clinic input] +_decimal.Decimal.to_integral_value + + cls: defining_class + rounding: object = None + context: object = None + +Round to the nearest integer without signaling Inexact or Rounded. + +The rounding mode is determined by the rounding parameter if given, +else by the given context. If neither parameter is given, then the +rounding mode of the current default context is used. +[clinic start generated code]*/ + static PyObject * -PyDec_ToIntegralValue(PyObject *dec, PyObject *args, PyObject *kwds) +_decimal_Decimal_to_integral_value_impl(PyObject *self, PyTypeObject *cls, + PyObject *rounding, + PyObject *context) +/*[clinic end generated code: output=23047d848ef84db1 input=85aa9499a21ea8d7]*/ { - static char *kwlist[] = {"rounding", "context", NULL}; PyObject *result; - PyObject *rounding = Py_None; - PyObject *context = Py_None; uint32_t status = 0; mpd_context_t workctx; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, - &rounding, &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); + decimal_state *state = PyType_GetModuleState(cls); CONTEXT_CHECK_VA(state, context); workctx = *CTX(context); @@ -3887,7 +4114,7 @@ PyDec_ToIntegralValue(PyObject *dec, PyObject *args, PyObject *kwds) return NULL; } - mpd_qround_to_int(MPD(result), MPD(dec), &workctx, &status); + mpd_qround_to_int(MPD(result), MPD(self), &workctx, &status); if (dec_addstatus(context, status)) { Py_DECREF(result); return NULL; @@ -3896,21 +4123,46 @@ PyDec_ToIntegralValue(PyObject *dec, PyObject *args, PyObject *kwds) return result; } +/*[clinic input] +_decimal.Decimal.to_integral = _decimal.Decimal.to_integral_value + +Identical to the to_integral_value() method. + +The to_integral() name has been kept for compatibility with older +versions. +[clinic start generated code]*/ + static PyObject * -PyDec_ToIntegralExact(PyObject *dec, PyObject *args, PyObject *kwds) +_decimal_Decimal_to_integral_impl(PyObject *self, PyTypeObject *cls, + PyObject *rounding, PyObject *context) +/*[clinic end generated code: output=5dac8f54c2a3ed26 input=709b54618ecd0d8b]*/ +{ + return _decimal_Decimal_to_integral_value_impl(self, cls, rounding, + context); +} + +/*[clinic input] +_decimal.Decimal.to_integral_exact = _decimal.Decimal.to_integral_value + +Round to the nearest integer. + +Decimal.to_integral_exact() signals Inexact or Rounded as appropriate +if rounding occurs. The rounding mode is determined by the rounding +parameter if given, else by the given context. If neither parameter is +given, then the rounding mode of the current default context is used. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_to_integral_exact_impl(PyObject *self, PyTypeObject *cls, + PyObject *rounding, + PyObject *context) +/*[clinic end generated code: output=543a39a02eea9917 input=fabce7a744b8087c]*/ { - static char *kwlist[] = {"rounding", "context", NULL}; PyObject *result; - PyObject *rounding = Py_None; - PyObject *context = Py_None; uint32_t status = 0; mpd_context_t workctx; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, - &rounding, &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); + decimal_state *state = PyType_GetModuleState(cls); CONTEXT_CHECK_VA(state, context); workctx = *CTX(context); @@ -3929,7 +4181,7 @@ PyDec_ToIntegralExact(PyObject *dec, PyObject *args, PyObject *kwds) return NULL; } - mpd_qround_to_intx(MPD(result), MPD(dec), &workctx, &status); + mpd_qround_to_intx(MPD(result), MPD(self), &workctx, &status); if (dec_addstatus(context, status)) { Py_DECREF(result); return NULL; @@ -3970,32 +4222,38 @@ PyDec_AsFloat(PyObject *dec) return f; } +/*[clinic input] +_decimal.Decimal.__round__ + + cls: defining_class + ndigits: object = NULL + / + +Return the Integral closest to self, rounding half toward even. +[clinic start generated code]*/ + static PyObject * -PyDec_Round(PyObject *dec, PyObject *args) +_decimal_Decimal___round___impl(PyObject *self, PyTypeObject *cls, + PyObject *ndigits) +/*[clinic end generated code: output=790c2c6bd57890e6 input=d69e7178a58a66b1]*/ { PyObject *result; - PyObject *x = NULL; uint32_t status = 0; PyObject *context; - - decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); + decimal_state *state = PyType_GetModuleState(cls); CURRENT_CONTEXT(state, context); - if (!PyArg_ParseTuple(args, "|O", &x)) { - return NULL; - } - - if (x) { + if (ndigits) { mpd_uint_t dq[1] = {1}; mpd_t q = {MPD_STATIC|MPD_CONST_DATA,0,1,1,1,dq}; mpd_ssize_t y; - if (!PyLong_Check(x)) { + if (!PyLong_Check(ndigits)) { PyErr_SetString(PyExc_TypeError, "optional arg must be an integer"); return NULL; } - y = PyLong_AsSsize_t(x); + y = PyLong_AsSsize_t(ndigits); if (y == -1 && PyErr_Occurred()) { return NULL; } @@ -4005,7 +4263,7 @@ PyDec_Round(PyObject *dec, PyObject *args) } q.exp = (y == MPD_SSIZE_MIN) ? MPD_SSIZE_MAX : -y; - mpd_qquantize(MPD(result), MPD(dec), &q, CTX(context), &status); + mpd_qquantize(MPD(result), MPD(self), &q, CTX(context), &status); if (dec_addstatus(context, status)) { Py_DECREF(result); return NULL; @@ -4014,13 +4272,21 @@ PyDec_Round(PyObject *dec, PyObject *args) return result; } else { - return dec_as_long(dec, context, MPD_ROUND_HALF_EVEN); + return dec_as_long(self, context, MPD_ROUND_HALF_EVEN); } } -/* Return the DecimalTuple representation of a PyDecObject. */ +/*[clinic input] +_decimal.Decimal.as_tuple + + cls: defining_class + +Return a tuple representation of the number. +[clinic start generated code]*/ + static PyObject * -PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_as_tuple_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=d68b967becee8ab9 input=bfa86d640224d9f5]*/ { PyObject *result = NULL; PyObject *sign = NULL; @@ -4032,13 +4298,13 @@ PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) Py_ssize_t intlen, i; - x = mpd_qncopy(MPD(dec)); + x = mpd_qncopy(MPD(self)); if (x == NULL) { PyErr_NoMemory(); goto out; } - sign = PyLong_FromUnsignedLong(mpd_sign(MPD(dec))); + sign = PyLong_FromUnsignedLong(mpd_sign(MPD(self))); if (sign == NULL) { goto out; } @@ -4059,7 +4325,7 @@ PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) expt = PyUnicode_FromString(mpd_isqnan(x)?"n":"N"); } else { - expt = PyLong_FromSsize_t(MPD(dec)->exp); + expt = PyLong_FromSsize_t(MPD(self)->exp); } if (expt == NULL) { goto out; @@ -4100,7 +4366,7 @@ PyDec_AsTuple(PyObject *dec, PyObject *Py_UNUSED(dummy)) } } - decimal_state *state = get_module_state_by_def(Py_TYPE(dec)); + decimal_state *state = PyType_GetModuleState(cls); result = PyObject_CallFunctionObjArgs((PyObject *)state->DecimalTuple, sign, coeff, expt, NULL); @@ -4152,7 +4418,7 @@ nm_##MPDFUNC(PyObject *self, PyObject *other) \ PyObject *context; \ uint32_t status = 0; \ \ - decimal_state *state = find_state_left_or_right(self, other); \ + decimal_state *state = find_state_left_or_right(self, other); \ CURRENT_CONTEXT(state, context) ; \ CONVERT_BINOP(&a, &b, self, other, context); \ \ @@ -4173,48 +4439,35 @@ nm_##MPDFUNC(PyObject *self, PyObject *other) \ return result; \ } -/* Boolean function without a context arg. */ +/* Boolean function without a context arg. + Argument Clinic provides PyObject *self +*/ #define Dec_BoolFunc(MPDFUNC) \ -static PyObject * \ -dec_##MPDFUNC(PyObject *self, PyObject *Py_UNUSED(dummy)) \ { \ return MPDFUNC(MPD(self)) ? incr_true() : incr_false(); \ } -/* Boolean function with an optional context arg. */ +/* Boolean function with an optional context arg. + Argument Clinic provides PyObject *self, PyTypeObject *cls, + PyObject *context +*/ #define Dec_BoolFuncVA(MPDFUNC) \ -static PyObject * \ -dec_##MPDFUNC(PyObject *self, PyObject *args, PyObject *kwds) \ { \ - static char *kwlist[] = {"context", NULL}; \ - PyObject *context = Py_None; \ - \ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, \ - &context)) { \ - return NULL; \ - } \ - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); \ + decimal_state *state = PyType_GetModuleState(cls); \ CONTEXT_CHECK_VA(state, context); \ \ return MPDFUNC(MPD(self), CTX(context)) ? incr_true() : incr_false(); \ } -/* Unary function with an optional context arg. */ +/* Unary function with an optional context arg. + Argument Clinic provides PyObject *self, PyTypeObject *cls, + PyObject *context +*/ #define Dec_UnaryFuncVA(MPDFUNC) \ -static PyObject * \ -dec_##MPDFUNC(PyObject *self, PyObject *args, PyObject *kwds) \ { \ - static char *kwlist[] = {"context", NULL}; \ PyObject *result; \ - PyObject *context = Py_None; \ uint32_t status = 0; \ - \ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, \ - &context)) { \ - return NULL; \ - } \ - decimal_state *state = \ - get_module_state_by_def(Py_TYPE(self)); \ + decimal_state *state = PyType_GetModuleState(cls); \ CONTEXT_CHECK_VA(state, context); \ \ if ((result = dec_alloc(state)) == NULL) { \ @@ -4230,24 +4483,16 @@ dec_##MPDFUNC(PyObject *self, PyObject *args, PyObject *kwds) \ return result; \ } -/* Binary function with an optional context arg. */ +/* Binary function with an optional context arg. + Argument Clinic provides PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context +*/ #define Dec_BinaryFuncVA(MPDFUNC) \ -static PyObject * \ -dec_##MPDFUNC(PyObject *self, PyObject *args, PyObject *kwds) \ { \ - static char *kwlist[] = {"other", "context", NULL}; \ - PyObject *other; \ PyObject *a, *b; \ PyObject *result; \ - PyObject *context = Py_None; \ uint32_t status = 0; \ - \ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, \ - &other, &context)) { \ - return NULL; \ - } \ - decimal_state *state = \ - get_module_state_by_def(Py_TYPE(self)); \ + decimal_state *state = PyType_GetModuleState(cls); \ CONTEXT_CHECK_VA(state, context); \ CONVERT_BINOP_RAISE(&a, &b, self, other, context); \ \ @@ -4270,23 +4515,16 @@ dec_##MPDFUNC(PyObject *self, PyObject *args, PyObject *kwds) \ /* Binary function with an optional context arg. Actual MPDFUNC does NOT take a context. The context is used to record InvalidOperation - if the second operand cannot be converted exactly. */ + if the second operand cannot be converted exactly. + + Argument Clinic provides PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context +*/ #define Dec_BinaryFuncVA_NO_CTX(MPDFUNC) \ -static PyObject * \ -dec_##MPDFUNC(PyObject *self, PyObject *args, PyObject *kwds) \ { \ - static char *kwlist[] = {"other", "context", NULL}; \ - PyObject *context = Py_None; \ - PyObject *other; \ PyObject *a, *b; \ PyObject *result; \ - \ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, \ - &other, &context)) { \ - return NULL; \ - } \ - decimal_state *state = \ - get_module_state_by_def(Py_TYPE(self)); \ + decimal_state *state = PyType_GetModuleState(cls); \ CONTEXT_CHECK_VA(state, context); \ CONVERT_BINOP_RAISE(&a, &b, self, other, context); \ \ @@ -4303,23 +4541,17 @@ dec_##MPDFUNC(PyObject *self, PyObject *args, PyObject *kwds) \ return result; \ } -/* Ternary function with an optional context arg. */ +/* Ternary function with an optional context arg. + Argument Clinic provides PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *third, + PyObject *context +*/ #define Dec_TernaryFuncVA(MPDFUNC) \ -static PyObject * \ -dec_##MPDFUNC(PyObject *self, PyObject *args, PyObject *kwds) \ { \ - static char *kwlist[] = {"other", "third", "context", NULL}; \ - PyObject *other, *third; \ PyObject *a, *b, *c; \ PyObject *result; \ - PyObject *context = Py_None; \ uint32_t status = 0; \ - \ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O", kwlist, \ - &other, &third, &context)) { \ - return NULL; \ - } \ - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); \ + decimal_state *state = PyType_GetModuleState(cls); \ CONTEXT_CHECK_VA(state, context); \ CONVERT_TERNOP_RAISE(&a, &b, &c, self, other, third, context); \ \ @@ -4468,44 +4700,400 @@ nm_mpd_qpow(PyObject *base, PyObject *exp, PyObject *mod) /******************************************************************************/ /* Unary arithmetic functions, optional context arg */ + +/*[clinic input] +_decimal.Decimal.exp + + cls: defining_class + context: object = None + +Return the value of the (natural) exponential function e**x. + +The function always uses the ROUND_HALF_EVEN mode and the result is +correctly rounded. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_exp_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=40317012aedbaeac input=84919aad3dabda08]*/ Dec_UnaryFuncVA(mpd_qexp) + +/*[clinic input] +_decimal.Decimal.ln = _decimal.Decimal.exp + +Return the natural (base e) logarithm of the operand. + +The function always uses the ROUND_HALF_EVEN mode and the result is +correctly rounded. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_ln_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=e8f9e81cac38e5dc input=d353c51ec00d1cff]*/ Dec_UnaryFuncVA(mpd_qln) + +/*[clinic input] +_decimal.Decimal.log10 = _decimal.Decimal.exp + +Return the base ten logarithm of the operand. + +The function always uses the ROUND_HALF_EVEN mode and the result is +correctly rounded. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_log10_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=00b3255648135c95 input=48a6be60154c0b46]*/ Dec_UnaryFuncVA(mpd_qlog10) + +/*[clinic input] +_decimal.Decimal.next_minus = _decimal.Decimal.exp + +Returns the largest representable number smaller than itself. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_next_minus_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=a187a55e6976b572 input=666b348f71e6c090]*/ Dec_UnaryFuncVA(mpd_qnext_minus) + +/*[clinic input] +_decimal.Decimal.next_plus = _decimal.Decimal.exp + +Returns the smallest representable number larger than itself. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_next_plus_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=13737d41714e320e input=04e105060ad1fa15]*/ Dec_UnaryFuncVA(mpd_qnext_plus) + +/*[clinic input] +_decimal.Decimal.normalize = _decimal.Decimal.exp + +Normalize the number by stripping trailing 0s + +This also change anything equal to 0 to 0e0. Used for producing +canonical values for members of an equivalence class. For example, +Decimal('32.100') and Decimal('0.321000e+2') both normalize to +the equivalent value Decimal('32.1'). +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_normalize_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=32c4c0d13fe33fb9 input=d5ee63acd904d4de]*/ Dec_UnaryFuncVA(mpd_qreduce) + +/*[clinic input] +_decimal.Decimal.sqrt = _decimal.Decimal.exp + +Return the square root of the argument to full precision. + +The result is correctly rounded using the ROUND_HALF_EVEN rounding mode. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_sqrt_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=deb1280077b5e586 input=3a76afbd39dc20b9]*/ Dec_UnaryFuncVA(mpd_qsqrt) /* Binary arithmetic functions, optional context arg */ + +/*[clinic input] +_decimal.Decimal.compare + + cls: defining_class + other: object + context: object = None + +Compare self to other. + +Return a decimal value: + + a or b is a NaN ==> Decimal('NaN') + a < b ==> Decimal('-1') + a == b ==> Decimal('0') + a > b ==> Decimal('1') +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_compare_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=a4a1d383ec192cfa input=d18a02bb8083e92a]*/ Dec_BinaryFuncVA(mpd_qcompare) + +/*[clinic input] +_decimal.Decimal.compare_signal = _decimal.Decimal.compare + +Identical to compare, except that all NaNs signal. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_compare_signal_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=22f757371fd4167b input=a52a39d1c6fc369d]*/ Dec_BinaryFuncVA(mpd_qcompare_signal) + +/*[clinic input] +_decimal.Decimal.max = _decimal.Decimal.compare + +Maximum of self and other. + +If one operand is a quiet NaN and the other is numeric, the numeric +operand is returned. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_max_impl(PyObject *self, PyTypeObject *cls, PyObject *other, + PyObject *context) +/*[clinic end generated code: output=d3d12db9815869e5 input=2ae2582f551296d8]*/ Dec_BinaryFuncVA(mpd_qmax) + +/*[clinic input] +_decimal.Decimal.max_mag = _decimal.Decimal.compare + +As the max() method, but compares the absolute values of the operands. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_max_mag_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=f71f2c27d9bc7cac input=88b105e66cf138c5]*/ Dec_BinaryFuncVA(mpd_qmax_mag) + +/*[clinic input] +_decimal.Decimal.min = _decimal.Decimal.compare + +Minimum of self and other. + +If one operand is a quiet NaN and the other is numeric, the numeric +operand is returned. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_min_impl(PyObject *self, PyTypeObject *cls, PyObject *other, + PyObject *context) +/*[clinic end generated code: output=c5620344ae5f3dd1 input=2a70f2c087c418c9]*/ Dec_BinaryFuncVA(mpd_qmin) + +/*[clinic input] +_decimal.Decimal.min_mag = _decimal.Decimal.compare + +As the min() method, but compares the absolute values of the operands. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_min_mag_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=018562ad1c22aae3 input=351fa3c0e592746a]*/ Dec_BinaryFuncVA(mpd_qmin_mag) + +/*[clinic input] +_decimal.Decimal.next_toward = _decimal.Decimal.compare + +Returns the number closest to self, in the direction towards other. + +If the two operands are unequal, return the number closest to the first +operand in the direction of the second operand. If both operands are +numerically equal, return a copy of the first operand with the sign set +to be the same as the sign of the second operand. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_next_toward_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=71d879bca8bc1019 input=fdf0091ea6e9e416]*/ Dec_BinaryFuncVA(mpd_qnext_toward) + +/*[clinic input] +_decimal.Decimal.remainder_near = _decimal.Decimal.compare + +Return the remainder from dividing self by other. + +This differs from self % other in that the sign of the remainder is +chosen so as to minimize its absolute value. More precisely, the return +value is self - n * other where n is the integer nearest to the exact +value of self / other, and if two integers are equally near then the +even one is chosen. + +If the result is zero then its sign will be the sign of self. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_remainder_near_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=d3fbb4985f2077fa input=eb5a8dfe3470b794]*/ Dec_BinaryFuncVA(mpd_qrem_near) /* Ternary arithmetic functions, optional context arg */ + +/*[clinic input] +_decimal.Decimal.fma + + cls: defining_class + other: object + third: object + context: object = None + +Fused multiply-add. + +Return self*other+third with no rounding of the intermediate product +self*other. + + >>> Decimal(2).fma(3, 5) + Decimal('11') +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_fma_impl(PyObject *self, PyTypeObject *cls, PyObject *other, + PyObject *third, PyObject *context) +/*[clinic end generated code: output=db49a777e85b71e4 input=2104c001f6077c35]*/ Dec_TernaryFuncVA(mpd_qfma) /* Boolean functions, no context arg */ + +/*[clinic input] +_decimal.Decimal.is_canonical + +Return True if the argument is canonical and False otherwise. + +Currently, a Decimal instance is always canonical, so this operation +always returns True. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_canonical_impl(PyObject *self) +/*[clinic end generated code: output=b29668684f45443e input=b3b3e6878ccf40b8]*/ Dec_BoolFunc(mpd_iscanonical) + +/*[clinic input] +_decimal.Decimal.is_finite + +Return True if the argument is a finite number, and False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_finite_impl(PyObject *self) +/*[clinic end generated code: output=537306fbfc9131f8 input=e9b8b5866704bae6]*/ Dec_BoolFunc(mpd_isfinite) + +/*[clinic input] +_decimal.Decimal.is_infinite + +Return True if the argument is infinite, and False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_infinite_impl(PyObject *self) +/*[clinic end generated code: output=31b775ff28f05ce2 input=8f3937a790ee4ec2]*/ Dec_BoolFunc(mpd_isinfinite) + +/*[clinic input] +_decimal.Decimal.is_nan + +Return True if the argument is a (quiet or signaling) NaN, else False. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_nan_impl(PyObject *self) +/*[clinic end generated code: output=b704e8b49a164388 input=795e5dac85976994]*/ Dec_BoolFunc(mpd_isnan) + +/*[clinic input] +_decimal.Decimal.is_qnan + +Return True if the argument is a quiet NaN, and False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_qnan_impl(PyObject *self) +/*[clinic end generated code: output=85b5241f43798376 input=00485f3c3cfae0af]*/ Dec_BoolFunc(mpd_isqnan) + +/*[clinic input] +_decimal.Decimal.is_snan + +Return True if the argument is a signaling NaN and False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_snan_impl(PyObject *self) +/*[clinic end generated code: output=50de9ec6507e4a4f input=f3b0f8592c921879]*/ Dec_BoolFunc(mpd_issnan) + +/*[clinic input] +_decimal.Decimal.is_signed + +Return True if the argument has a negative sign and False otherwise. + +Note that both zeros and NaNs can carry signs. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_signed_impl(PyObject *self) +/*[clinic end generated code: output=8ec7bc85d8e755e4 input=97c3437ab5dffecc]*/ Dec_BoolFunc(mpd_issigned) + +/*[clinic input] +_decimal.Decimal.is_zero + +Return True if the argument is a zero and False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_zero_impl(PyObject *self) +/*[clinic end generated code: output=2d87ea1b15879112 input=ae616674cd050a51]*/ Dec_BoolFunc(mpd_iszero) /* Boolean functions, optional context arg */ + +/*[clinic input] +_decimal.Decimal.is_normal = _decimal.Decimal.exp + +Return True if the argument is a normal number and False otherwise. + +Normal number is a finite nonzero number, which is not subnormal. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_normal_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=92a3878e293758d4 input=9afe43b9db9f4818]*/ Dec_BoolFuncVA(mpd_isnormal) + +/*[clinic input] +_decimal.Decimal.is_subnormal = _decimal.Decimal.exp + +Return True if the argument is subnormal, and False otherwise. + +A number is subnormal if it is non-zero, finite, and has an adjusted +exponent less than Emin. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_is_subnormal_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=1404c04d980ebc07 input=11839c122c185b8b]*/ Dec_BoolFuncVA(mpd_issubnormal) /* Unary functions, no context arg */ + +/*[clinic input] +_decimal.Decimal.adjusted + +Return the adjusted exponent (exp + digits - 1) of the number. +[clinic start generated code]*/ + static PyObject * -dec_mpd_adjexp(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_adjusted_impl(PyObject *self) +/*[clinic end generated code: output=21ea2c9f23994c52 input=8ba2029d8d906b18]*/ { mpd_ssize_t retval; @@ -4519,14 +5107,31 @@ dec_mpd_adjexp(PyObject *self, PyObject *Py_UNUSED(dummy)) return PyLong_FromSsize_t(retval); } +/*[clinic input] +_decimal.Decimal.canonical + +Return the canonical encoding of the argument. + +Currently, the encoding of a Decimal instance is always canonical, +so this operation returns its argument unchanged. +[clinic start generated code]*/ + static PyObject * -dec_canonical(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_canonical_impl(PyObject *self) +/*[clinic end generated code: output=3cbeb47d91e6da2d input=8a4719d14c52d521]*/ { return Py_NewRef(self); } +/*[clinic input] +_decimal.Decimal.conjugate + +Return self. +[clinic start generated code]*/ + static PyObject * -dec_conjugate(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_conjugate_impl(PyObject *self) +/*[clinic end generated code: output=9a37bf633f25a291 input=c7179975ef74fd84]*/ { return Py_NewRef(self); } @@ -4545,20 +5150,44 @@ _dec_mpd_radix(decimal_state *state) return result; } +/*[clinic input] +_decimal.Decimal.radix + + cls: defining_class + +Return Decimal(10). + +This is the radix (base) in which the Decimal class does +all its arithmetic. Included for compatibility with the specification. +[clinic start generated code]*/ + static PyObject * -dec_mpd_radix(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_radix_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=40a3bc7ec3d99228 input=b0d4cb9f870bbac1]*/ { - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); return _dec_mpd_radix(state); } +/*[clinic input] +_decimal.Decimal.copy_abs + + cls: defining_class + +Return the absolute value of the argument. + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. +[clinic start generated code]*/ + static PyObject * -dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_copy_abs_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=081cb7fb4230676e input=676d7c62b1795512]*/ { PyObject *result; uint32_t status = 0; + decimal_state *state = PyType_GetModuleState(cls); - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); if ((result = dec_alloc(state)) == NULL) { return NULL; } @@ -4573,13 +5202,23 @@ dec_mpd_qcopy_abs(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } +/*[clinic input] +_decimal.Decimal.copy_negate = _decimal.Decimal.copy_abs + +Return the negation of the argument. + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. +[clinic start generated code]*/ + static PyObject * -dec_mpd_qcopy_negate(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal_copy_negate_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=04fed82c17d4e28b input=23f41ee8899f3891]*/ { PyObject *result; uint32_t status = 0; + decimal_state *state = PyType_GetModuleState(cls); - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); if ((result = dec_alloc(state)) == NULL) { return NULL; } @@ -4595,41 +5234,98 @@ dec_mpd_qcopy_negate(PyObject *self, PyObject *Py_UNUSED(dummy)) } /* Unary functions, optional context arg */ -Dec_UnaryFuncVA(mpd_qinvert) -Dec_UnaryFuncVA(mpd_qlogb) + +/*[clinic input] +_decimal.Decimal.logical_invert = _decimal.Decimal.exp + +Invert all its digits. + +The self must be logical number. +[clinic start generated code]*/ static PyObject * -dec_mpd_class(PyObject *self, PyObject *args, PyObject *kwds) +_decimal_Decimal_logical_invert_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=c626ed4b104a97b7 input=7158d5b525417955]*/ +Dec_UnaryFuncVA(mpd_qinvert) + +/*[clinic input] +_decimal.Decimal.logb = _decimal.Decimal.exp + +Return the adjusted exponent of the operand as a Decimal instance. + +If the operand is a zero, then Decimal('-Infinity') is returned and the +DivisionByZero condition is raised. If the operand is an infinity then +Decimal('Infinity') is returned. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_logb_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=36b0bda09e934245 input=a8df027d1b8a2b17]*/ +Dec_UnaryFuncVA(mpd_qlogb) + +/*[clinic input] +_decimal.Decimal.number_class = _decimal.Decimal.exp + +Return a string describing the class of the operand. + +The returned value is one of the following ten strings: + + * '-Infinity', indicating that the operand is negative infinity. + * '-Normal', indicating that the operand is a negative normal + number. + * '-Subnormal', indicating that the operand is negative and + subnormal. + * '-Zero', indicating that the operand is a negative zero. + * '+Zero', indicating that the operand is a positive zero. + * '+Subnormal', indicating that the operand is positive and + subnormal. + * '+Normal', indicating that the operand is a positive normal + number. + * '+Infinity', indicating that the operand is positive infinity. + * 'NaN', indicating that the operand is a quiet NaN (Not a Number). + * 'sNaN', indicating that the operand is a signaling NaN. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_number_class_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=1ac82412e0849c52 input=447095d2677fa0ca]*/ { - static char *kwlist[] = {"context", NULL}; - PyObject *context = Py_None; const char *cp; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, - &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); CONTEXT_CHECK_VA(state, context); cp = mpd_class(MPD(self), CTX(context)); return PyUnicode_FromString(cp); } +/*[clinic input] +_decimal.Decimal.to_eng_string = _decimal.Decimal.exp + +Convert to an engineering-type string. + +Engineering notation has an exponent which is a multiple of 3, so there +are up to 3 digits left of the decimal place. For example, +Decimal('123E+1') is converted to Decimal('1.23E+3'). + +The value of context.capitals determines whether the exponent sign is +lower or upper case. Otherwise, the context does not affect the +operation. +[clinic start generated code]*/ + static PyObject * -dec_mpd_to_eng(PyObject *self, PyObject *args, PyObject *kwds) +_decimal_Decimal_to_eng_string_impl(PyObject *self, PyTypeObject *cls, + PyObject *context) +/*[clinic end generated code: output=901f128d437ae5c0 input=b2cb7e01e268e45d]*/ { - static char *kwlist[] = {"context", NULL}; PyObject *result; - PyObject *context = Py_None; mpd_ssize_t size; char *s; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, - &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); CONTEXT_CHECK_VA(state, context); size = mpd_to_eng_size(&s, MPD(self), CtxCaps(context)); @@ -4645,24 +5341,85 @@ dec_mpd_to_eng(PyObject *self, PyObject *args, PyObject *kwds) } /* Binary functions, optional context arg for conversion errors */ -Dec_BinaryFuncVA_NO_CTX(mpd_compare_total) -Dec_BinaryFuncVA_NO_CTX(mpd_compare_total_mag) + +/*[clinic input] +_decimal.Decimal.compare_total = _decimal.Decimal.compare + +Compare two operands using their abstract representation. + +Similar to the compare() method, but the result +gives a total ordering on Decimal instances. Two Decimal instances with +the same numeric value but different representations compare unequal +in this ordering: + + >>> Decimal('12.0').compare_total(Decimal('12')) + Decimal('-1') + +Quiet and signaling NaNs are also included in the total ordering. The +result of this function is Decimal('0') if both operands have the same +representation, Decimal('-1') if the first operand is lower in the +total order than the second, and Decimal('1') if the first operand is +higher in the total order than the second operand. See the +specification for details of the total order. + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. As an exception, the C version +may raise InvalidOperation if the second operand cannot be converted +exactly. +[clinic start generated code]*/ static PyObject * -dec_mpd_qcopy_sign(PyObject *self, PyObject *args, PyObject *kwds) +_decimal_Decimal_compare_total_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=83649010bad7815f input=6f3111ec5fdbf3c1]*/ +Dec_BinaryFuncVA_NO_CTX(mpd_compare_total) + +/*[clinic input] +_decimal.Decimal.compare_total_mag = _decimal.Decimal.compare + +As compare_total(), but ignores the sign of each operand. + +x.compare_total_mag(y) is equivalent to +x.copy_abs().compare_total(y.copy_abs()). + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. As an exception, the C version +may raise InvalidOperation if the second operand cannot be converted +exactly. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_compare_total_mag_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=b99c924cafb5f0e3 input=eba17c4c24eb2833]*/ +Dec_BinaryFuncVA_NO_CTX(mpd_compare_total_mag) + +/*[clinic input] +_decimal.Decimal.copy_sign = _decimal.Decimal.compare + +Return a copy of *self* with the sign of *other*. + +For example: + + >>> Decimal('2.3').copy_sign(Decimal('-1.5')) + Decimal('-2.3') + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. As an exception, the C version +may raise InvalidOperation if the second operand cannot be converted +exactly. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_copy_sign_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=e4c8f884f4d75801 input=51ed9e4691e2249e]*/ { - static char *kwlist[] = {"other", "context", NULL}; - PyObject *other; PyObject *a, *b; PyObject *result; - PyObject *context = Py_None; uint32_t status = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, - &other, &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); CONTEXT_CHECK_VA(state, context); CONVERT_BINOP_RAISE(&a, &b, self, other, context); @@ -4684,20 +5441,26 @@ dec_mpd_qcopy_sign(PyObject *self, PyObject *args, PyObject *kwds) return result; } +/*[clinic input] +_decimal.Decimal.same_quantum = _decimal.Decimal.compare + +Test whether self and other have the same exponent or both are NaN. + +This operation is unaffected by context and is quiet: no flags are +changed and no rounding is performed. As an exception, the C version +may raise InvalidOperation if the second operand cannot be converted +exactly. +[clinic start generated code]*/ + static PyObject * -dec_mpd_same_quantum(PyObject *self, PyObject *args, PyObject *kwds) +_decimal_Decimal_same_quantum_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=7c757edb0c263721 input=8339415fa359e7df]*/ { - static char *kwlist[] = {"other", "context", NULL}; - PyObject *other; PyObject *a, *b; PyObject *result; - PyObject *context = Py_None; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, - &other, &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); CONTEXT_CHECK_VA(state, context); CONVERT_BINOP_RAISE(&a, &b, self, other, context); @@ -4709,30 +5472,146 @@ dec_mpd_same_quantum(PyObject *self, PyObject *args, PyObject *kwds) } /* Binary functions, optional context arg */ -Dec_BinaryFuncVA(mpd_qand) -Dec_BinaryFuncVA(mpd_qor) -Dec_BinaryFuncVA(mpd_qxor) -Dec_BinaryFuncVA(mpd_qrotate) -Dec_BinaryFuncVA(mpd_qscaleb) -Dec_BinaryFuncVA(mpd_qshift) +/*[clinic input] +_decimal.Decimal.logical_and = _decimal.Decimal.compare + +Applies an 'and' operation between self and other's digits. + +Both self and other must be logical numbers. +[clinic start generated code]*/ static PyObject * -dec_mpd_qquantize(PyObject *v, PyObject *args, PyObject *kwds) +_decimal_Decimal_logical_and_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=9a4cbb74c180b0bb input=f22460f1285782d2]*/ +Dec_BinaryFuncVA(mpd_qand) + +/*[clinic input] +_decimal.Decimal.logical_or = _decimal.Decimal.compare + +Applies an 'or' operation between self and other's digits. + +Both self and other must be logical numbers. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_logical_or_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=063c4de18dc41ecb input=b5afa1e1fdebdfce]*/ +Dec_BinaryFuncVA(mpd_qor) + +/*[clinic input] +_decimal.Decimal.logical_xor = _decimal.Decimal.compare + +Applies an 'xor' operation between self and other's digits. + +Both self and other must be logical numbers. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_logical_xor_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=829b09cb49926ad7 input=84d722ada08a2da7]*/ +Dec_BinaryFuncVA(mpd_qxor) + +/*[clinic input] +_decimal.Decimal.rotate = _decimal.Decimal.compare + +Returns a rotated copy of self's digits, value-of-other times. + +The second operand must be an integer in the range -precision through +precision. The absolute value of the second operand gives the number of +places to rotate. If the second operand is positive then rotation is to +the left; otherwise rotation is to the right. The coefficient of the +first operand is padded on the left with zeros to length precision if +necessary. The sign and exponent of the first operand are unchanged. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_rotate_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=09f2737082882b83 input=cde7b032eac43f0b]*/ +Dec_BinaryFuncVA(mpd_qrotate) + +/*[clinic input] +_decimal.Decimal.scaleb = _decimal.Decimal.compare + +Return the first operand with the exponent adjusted the second. + +Equivalently, return the first operand multiplied by 10**other. The +second operand must be an integer. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_scaleb_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=ae8730536c9f2d30 input=7f29f83278d05f83]*/ +Dec_BinaryFuncVA(mpd_qscaleb) + +/*[clinic input] +_decimal.Decimal.shift = _decimal.Decimal.compare + +Returns a shifted copy of self's digits, value-of-other times. + +The second operand must be an integer in the range -precision through +precision. The absolute value of the second operand gives the number +of places to shift. If the second operand is positive, then the shift +is to the left; otherwise the shift is to the right. Digits shifted +into the coefficient are zeros. The sign and exponent of the first +operand are unchanged. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_shift_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context) +/*[clinic end generated code: output=82e061a0d9ecc4f5 input=501759c2522cb78e]*/ +Dec_BinaryFuncVA(mpd_qshift) + +/*[clinic input] +_decimal.Decimal.quantize + + cls: defining_class + exp as w: object + rounding: object = None + context: object = None + +Quantize *self* so its exponent is the same as that of *exp*. + +Return a value equal to *self* after rounding, with the exponent +of *exp*. + + >>> Decimal('1.41421356').quantize(Decimal('1.000')) + Decimal('1.414') + +Unlike other operations, if the length of the coefficient after the +quantize operation would be greater than precision, then an +InvalidOperation is signaled. This guarantees that, unless there +is an error condition, the quantized exponent is always equal to +that of the right-hand operand. + +Also unlike other operations, quantize never signals Underflow, even +if the result is subnormal and inexact. + +If the exponent of the second operand is larger than that of the first, +then rounding may be necessary. In this case, the rounding mode is +determined by the rounding argument if given, else by the given context +argument; if neither argument is given, the rounding mode of the +current thread's context is used. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal_quantize_impl(PyObject *self, PyTypeObject *cls, + PyObject *w, PyObject *rounding, + PyObject *context) +/*[clinic end generated code: output=fc51edf458559913 input=1166e6311e047b74]*/ { - static char *kwlist[] = {"exp", "rounding", "context", NULL}; - PyObject *rounding = Py_None; - PyObject *context = Py_None; - PyObject *w, *a, *b; + PyObject *a, *b; PyObject *result; uint32_t status = 0; mpd_context_t workctx; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO", kwlist, - &w, &rounding, &context)) { - return NULL; - } - decimal_state *state = get_module_state_by_def(Py_TYPE(v)); + decimal_state *state = PyType_GetModuleState(cls); CONTEXT_CHECK_VA(state, context); workctx = *CTX(context); @@ -4746,7 +5625,7 @@ dec_mpd_qquantize(PyObject *v, PyObject *args, PyObject *kwds) } } - CONVERT_BINOP_RAISE(&a, &b, v, w, context); + CONVERT_BINOP_RAISE(&a, &b, self, w, context); result = dec_alloc(state); if (result == NULL) { @@ -4826,20 +5705,34 @@ dec_richcompare(PyObject *v, PyObject *w, int op) return PyBool_FromLong(r); } -/* __ceil__ */ +/*[clinic input] +_decimal.Decimal.__ceil__ + + cls: defining_class + +Return the ceiling as an Integral. +[clinic start generated code]*/ + static PyObject * -dec_ceil(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal___ceil___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=d986ebf9aadbf9fe input=a8e0b87897706816]*/ { PyObject *context; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); CURRENT_CONTEXT(state, context); return dec_as_long(self, context, MPD_ROUND_CEILING); } -/* __complex__ */ +/*[clinic input] +_decimal.Decimal.__complex__ + +Convert this value to exact type complex. +[clinic start generated code]*/ + static PyObject * -dec_complex(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal___complex___impl(PyObject *self) +/*[clinic end generated code: output=c9b5b4a9fdebc912 input=6b11c6f20af7061a]*/ { PyObject *f; double x; @@ -4858,20 +5751,46 @@ dec_complex(PyObject *self, PyObject *Py_UNUSED(dummy)) return PyComplex_FromDoubles(x, 0); } -/* __copy__ (METH_NOARGS) and __deepcopy__ (METH_O) */ +/*[clinic input] +_decimal.Decimal.__copy__ + +[clinic start generated code]*/ + static PyObject * -dec_copy(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal___copy___impl(PyObject *self) +/*[clinic end generated code: output=8eb3656c0250762b input=3dfd30a3e1493c01]*/ { return Py_NewRef(self); } -/* __floor__ */ +/*[clinic input] +_decimal.Decimal.__deepcopy__ + + memo: object + / + +[clinic start generated code]*/ + static PyObject * -dec_floor(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal___deepcopy__(PyObject *self, PyObject *memo) +/*[clinic end generated code: output=988fb34e0136b376 input=f95598c6f43233aa]*/ +{ + return Py_NewRef(self); +} + +/*[clinic input] +_decimal.Decimal.__floor__ = _decimal.Decimal.__ceil__ + +Return the floor as an Integral. +[clinic start generated code]*/ + +static PyObject * +_decimal_Decimal___floor___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=e239a2f7f6514c12 input=dcc37aeceb0efb8d]*/ { PyObject *context; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); CURRENT_CONTEXT(state, context); return dec_as_long(self, context, MPD_ROUND_FLOOR); } @@ -4880,7 +5799,7 @@ dec_floor(PyObject *self, PyObject *Py_UNUSED(dummy)) static Py_hash_t _dec_hash(PyDecObject *v) { -#if defined(CONFIG_64) && _PyHASH_BITS == 61 +#if defined(CONFIG_64) && PyHASH_BITS == 61 /* 2**61 - 1 */ mpd_uint_t p_data[1] = {2305843009213693951ULL}; mpd_t p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 19, 1, 1, p_data}; @@ -4888,7 +5807,7 @@ _dec_hash(PyDecObject *v) mpd_uint_t inv10_p_data[1] = {2075258708292324556ULL}; mpd_t inv10_p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 19, 1, 1, inv10_p_data}; -#elif defined(CONFIG_32) && _PyHASH_BITS == 31 +#elif defined(CONFIG_32) && PyHASH_BITS == 31 /* 2**31 - 1 */ mpd_uint_t p_data[2] = {147483647UL, 2}; mpd_t p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 10, 2, 2, p_data}; @@ -4897,7 +5816,7 @@ _dec_hash(PyDecObject *v) mpd_t inv10_p = {MPD_POS|MPD_STATIC|MPD_CONST_DATA, 0, 10, 2, 2, inv10_p_data}; #else - #error "No valid combination of CONFIG_64, CONFIG_32 and _PyHASH_BITS" + #error "No valid combination of CONFIG_64, CONFIG_32 and PyHASH_BITS" #endif const Py_hash_t py_hash_inf = 314159; mpd_uint_t ten_data[1] = {10}; @@ -5003,9 +5922,15 @@ dec_hash(PyObject *op) return self->hash; } -/* __reduce__ */ +/*[clinic input] +_decimal.Decimal.__reduce__ + +Return state information for pickling. +[clinic start generated code]*/ + static PyObject * -dec_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal___reduce___impl(PyObject *self) +/*[clinic end generated code: output=84fa6648a496a8d2 input=0345ea951d9b986f]*/ { PyObject *result, *str; @@ -5020,9 +5945,17 @@ dec_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) return result; } -/* __sizeof__ */ +/*[clinic input] +_decimal.Decimal.__sizeof__ + + self as v: self + +Returns size in memory, in bytes +[clinic start generated code]*/ + static PyObject * -dec_sizeof(PyObject *v, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal___sizeof___impl(PyObject *v) +/*[clinic end generated code: output=f16de05097c62b79 input=a557db538cfddbb7]*/ { size_t res = _PyObject_SIZE(Py_TYPE(v)); if (mpd_isdynamic_data(MPD(v))) { @@ -5031,13 +5964,19 @@ dec_sizeof(PyObject *v, PyObject *Py_UNUSED(dummy)) return PyLong_FromSize_t(res); } -/* __trunc__ */ +/*[clinic input] +_decimal.Decimal.__trunc__ = _decimal.Decimal.__ceil__ + +Return the Integral closest to x between 0 and x. +[clinic start generated code]*/ + static PyObject * -dec_trunc(PyObject *self, PyObject *Py_UNUSED(dummy)) +_decimal_Decimal___trunc___impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=7b3decc4b636ce32 input=9b3a3a85f63b0515]*/ { PyObject *context; - decimal_state *state = get_module_state_by_def(Py_TYPE(self)); + decimal_state *state = PyType_GetModuleState(cls); CURRENT_CONTEXT(state, context); return dec_as_long(self, context, MPD_ROUND_DOWN); } @@ -5075,92 +6014,92 @@ static PyGetSetDef dec_getsets [] = static PyMethodDef dec_methods [] = { /* Unary arithmetic functions, optional context arg */ - { "exp", _PyCFunction_CAST(dec_mpd_qexp), METH_VARARGS|METH_KEYWORDS, doc_exp }, - { "ln", _PyCFunction_CAST(dec_mpd_qln), METH_VARARGS|METH_KEYWORDS, doc_ln }, - { "log10", _PyCFunction_CAST(dec_mpd_qlog10), METH_VARARGS|METH_KEYWORDS, doc_log10 }, - { "next_minus", _PyCFunction_CAST(dec_mpd_qnext_minus), METH_VARARGS|METH_KEYWORDS, doc_next_minus }, - { "next_plus", _PyCFunction_CAST(dec_mpd_qnext_plus), METH_VARARGS|METH_KEYWORDS, doc_next_plus }, - { "normalize", _PyCFunction_CAST(dec_mpd_qreduce), METH_VARARGS|METH_KEYWORDS, doc_normalize }, - { "to_integral", _PyCFunction_CAST(PyDec_ToIntegralValue), METH_VARARGS|METH_KEYWORDS, doc_to_integral }, - { "to_integral_exact", _PyCFunction_CAST(PyDec_ToIntegralExact), METH_VARARGS|METH_KEYWORDS, doc_to_integral_exact }, - { "to_integral_value", _PyCFunction_CAST(PyDec_ToIntegralValue), METH_VARARGS|METH_KEYWORDS, doc_to_integral_value }, - { "sqrt", _PyCFunction_CAST(dec_mpd_qsqrt), METH_VARARGS|METH_KEYWORDS, doc_sqrt }, + _DECIMAL_DECIMAL_EXP_METHODDEF + _DECIMAL_DECIMAL_LN_METHODDEF + _DECIMAL_DECIMAL_LOG10_METHODDEF + _DECIMAL_DECIMAL_NEXT_MINUS_METHODDEF + _DECIMAL_DECIMAL_NEXT_PLUS_METHODDEF + _DECIMAL_DECIMAL_NORMALIZE_METHODDEF + _DECIMAL_DECIMAL_TO_INTEGRAL_METHODDEF + _DECIMAL_DECIMAL_TO_INTEGRAL_EXACT_METHODDEF + _DECIMAL_DECIMAL_TO_INTEGRAL_VALUE_METHODDEF + _DECIMAL_DECIMAL_SQRT_METHODDEF /* Binary arithmetic functions, optional context arg */ - { "compare", _PyCFunction_CAST(dec_mpd_qcompare), METH_VARARGS|METH_KEYWORDS, doc_compare }, - { "compare_signal", _PyCFunction_CAST(dec_mpd_qcompare_signal), METH_VARARGS|METH_KEYWORDS, doc_compare_signal }, - { "max", _PyCFunction_CAST(dec_mpd_qmax), METH_VARARGS|METH_KEYWORDS, doc_max }, - { "max_mag", _PyCFunction_CAST(dec_mpd_qmax_mag), METH_VARARGS|METH_KEYWORDS, doc_max_mag }, - { "min", _PyCFunction_CAST(dec_mpd_qmin), METH_VARARGS|METH_KEYWORDS, doc_min }, - { "min_mag", _PyCFunction_CAST(dec_mpd_qmin_mag), METH_VARARGS|METH_KEYWORDS, doc_min_mag }, - { "next_toward", _PyCFunction_CAST(dec_mpd_qnext_toward), METH_VARARGS|METH_KEYWORDS, doc_next_toward }, - { "quantize", _PyCFunction_CAST(dec_mpd_qquantize), METH_VARARGS|METH_KEYWORDS, doc_quantize }, - { "remainder_near", _PyCFunction_CAST(dec_mpd_qrem_near), METH_VARARGS|METH_KEYWORDS, doc_remainder_near }, + _DECIMAL_DECIMAL_COMPARE_METHODDEF + _DECIMAL_DECIMAL_COMPARE_SIGNAL_METHODDEF + _DECIMAL_DECIMAL_MAX_METHODDEF + _DECIMAL_DECIMAL_MAX_MAG_METHODDEF + _DECIMAL_DECIMAL_MIN_METHODDEF + _DECIMAL_DECIMAL_MIN_MAG_METHODDEF + _DECIMAL_DECIMAL_NEXT_TOWARD_METHODDEF + _DECIMAL_DECIMAL_QUANTIZE_METHODDEF + _DECIMAL_DECIMAL_REMAINDER_NEAR_METHODDEF /* Ternary arithmetic functions, optional context arg */ - { "fma", _PyCFunction_CAST(dec_mpd_qfma), METH_VARARGS|METH_KEYWORDS, doc_fma }, + _DECIMAL_DECIMAL_FMA_METHODDEF /* Boolean functions, no context arg */ - { "is_canonical", dec_mpd_iscanonical, METH_NOARGS, doc_is_canonical }, - { "is_finite", dec_mpd_isfinite, METH_NOARGS, doc_is_finite }, - { "is_infinite", dec_mpd_isinfinite, METH_NOARGS, doc_is_infinite }, - { "is_nan", dec_mpd_isnan, METH_NOARGS, doc_is_nan }, - { "is_qnan", dec_mpd_isqnan, METH_NOARGS, doc_is_qnan }, - { "is_snan", dec_mpd_issnan, METH_NOARGS, doc_is_snan }, - { "is_signed", dec_mpd_issigned, METH_NOARGS, doc_is_signed }, - { "is_zero", dec_mpd_iszero, METH_NOARGS, doc_is_zero }, + _DECIMAL_DECIMAL_IS_CANONICAL_METHODDEF + _DECIMAL_DECIMAL_IS_FINITE_METHODDEF + _DECIMAL_DECIMAL_IS_INFINITE_METHODDEF + _DECIMAL_DECIMAL_IS_NAN_METHODDEF + _DECIMAL_DECIMAL_IS_QNAN_METHODDEF + _DECIMAL_DECIMAL_IS_SNAN_METHODDEF + _DECIMAL_DECIMAL_IS_SIGNED_METHODDEF + _DECIMAL_DECIMAL_IS_ZERO_METHODDEF /* Boolean functions, optional context arg */ - { "is_normal", _PyCFunction_CAST(dec_mpd_isnormal), METH_VARARGS|METH_KEYWORDS, doc_is_normal }, - { "is_subnormal", _PyCFunction_CAST(dec_mpd_issubnormal), METH_VARARGS|METH_KEYWORDS, doc_is_subnormal }, + _DECIMAL_DECIMAL_IS_NORMAL_METHODDEF + _DECIMAL_DECIMAL_IS_SUBNORMAL_METHODDEF /* Unary functions, no context arg */ - { "adjusted", dec_mpd_adjexp, METH_NOARGS, doc_adjusted }, - { "canonical", dec_canonical, METH_NOARGS, doc_canonical }, - { "conjugate", dec_conjugate, METH_NOARGS, doc_conjugate }, - { "radix", dec_mpd_radix, METH_NOARGS, doc_radix }, + _DECIMAL_DECIMAL_ADJUSTED_METHODDEF + _DECIMAL_DECIMAL_CANONICAL_METHODDEF + _DECIMAL_DECIMAL_CONJUGATE_METHODDEF + _DECIMAL_DECIMAL_RADIX_METHODDEF /* Unary functions, optional context arg for conversion errors */ - { "copy_abs", dec_mpd_qcopy_abs, METH_NOARGS, doc_copy_abs }, - { "copy_negate", dec_mpd_qcopy_negate, METH_NOARGS, doc_copy_negate }, + _DECIMAL_DECIMAL_COPY_ABS_METHODDEF + _DECIMAL_DECIMAL_COPY_NEGATE_METHODDEF /* Unary functions, optional context arg */ - { "logb", _PyCFunction_CAST(dec_mpd_qlogb), METH_VARARGS|METH_KEYWORDS, doc_logb }, - { "logical_invert", _PyCFunction_CAST(dec_mpd_qinvert), METH_VARARGS|METH_KEYWORDS, doc_logical_invert }, - { "number_class", _PyCFunction_CAST(dec_mpd_class), METH_VARARGS|METH_KEYWORDS, doc_number_class }, - { "to_eng_string", _PyCFunction_CAST(dec_mpd_to_eng), METH_VARARGS|METH_KEYWORDS, doc_to_eng_string }, + _DECIMAL_DECIMAL_LOGB_METHODDEF + _DECIMAL_DECIMAL_LOGICAL_INVERT_METHODDEF + _DECIMAL_DECIMAL_NUMBER_CLASS_METHODDEF + _DECIMAL_DECIMAL_TO_ENG_STRING_METHODDEF /* Binary functions, optional context arg for conversion errors */ - { "compare_total", _PyCFunction_CAST(dec_mpd_compare_total), METH_VARARGS|METH_KEYWORDS, doc_compare_total }, - { "compare_total_mag", _PyCFunction_CAST(dec_mpd_compare_total_mag), METH_VARARGS|METH_KEYWORDS, doc_compare_total_mag }, - { "copy_sign", _PyCFunction_CAST(dec_mpd_qcopy_sign), METH_VARARGS|METH_KEYWORDS, doc_copy_sign }, - { "same_quantum", _PyCFunction_CAST(dec_mpd_same_quantum), METH_VARARGS|METH_KEYWORDS, doc_same_quantum }, + _DECIMAL_DECIMAL_COMPARE_TOTAL_METHODDEF + _DECIMAL_DECIMAL_COMPARE_TOTAL_MAG_METHODDEF + _DECIMAL_DECIMAL_COPY_SIGN_METHODDEF + _DECIMAL_DECIMAL_SAME_QUANTUM_METHODDEF /* Binary functions, optional context arg */ - { "logical_and", _PyCFunction_CAST(dec_mpd_qand), METH_VARARGS|METH_KEYWORDS, doc_logical_and }, - { "logical_or", _PyCFunction_CAST(dec_mpd_qor), METH_VARARGS|METH_KEYWORDS, doc_logical_or }, - { "logical_xor", _PyCFunction_CAST(dec_mpd_qxor), METH_VARARGS|METH_KEYWORDS, doc_logical_xor }, - { "rotate", _PyCFunction_CAST(dec_mpd_qrotate), METH_VARARGS|METH_KEYWORDS, doc_rotate }, - { "scaleb", _PyCFunction_CAST(dec_mpd_qscaleb), METH_VARARGS|METH_KEYWORDS, doc_scaleb }, - { "shift", _PyCFunction_CAST(dec_mpd_qshift), METH_VARARGS|METH_KEYWORDS, doc_shift }, + _DECIMAL_DECIMAL_LOGICAL_AND_METHODDEF + _DECIMAL_DECIMAL_LOGICAL_OR_METHODDEF + _DECIMAL_DECIMAL_LOGICAL_XOR_METHODDEF + _DECIMAL_DECIMAL_ROTATE_METHODDEF + _DECIMAL_DECIMAL_SCALEB_METHODDEF + _DECIMAL_DECIMAL_SHIFT_METHODDEF /* Miscellaneous */ - { "from_float", dec_from_float, METH_O|METH_CLASS, doc_from_float }, - { "from_number", dec_from_number, METH_O|METH_CLASS, doc_from_number }, - { "as_tuple", PyDec_AsTuple, METH_NOARGS, doc_as_tuple }, - { "as_integer_ratio", dec_as_integer_ratio, METH_NOARGS, doc_as_integer_ratio }, + _DECIMAL_DECIMAL_FROM_FLOAT_METHODDEF + _DECIMAL_DECIMAL_FROM_NUMBER_METHODDEF + _DECIMAL_DECIMAL_AS_TUPLE_METHODDEF + _DECIMAL_DECIMAL_AS_INTEGER_RATIO_METHODDEF /* Special methods */ - { "__copy__", dec_copy, METH_NOARGS, NULL }, - { "__deepcopy__", dec_copy, METH_O, NULL }, - { "__format__", dec_format, METH_VARARGS, NULL }, - { "__reduce__", dec_reduce, METH_NOARGS, NULL }, - { "__round__", PyDec_Round, METH_VARARGS, NULL }, - { "__ceil__", dec_ceil, METH_NOARGS, NULL }, - { "__floor__", dec_floor, METH_NOARGS, NULL }, - { "__trunc__", dec_trunc, METH_NOARGS, NULL }, - { "__complex__", dec_complex, METH_NOARGS, NULL }, - { "__sizeof__", dec_sizeof, METH_NOARGS, NULL }, + _DECIMAL_DECIMAL___COPY___METHODDEF + _DECIMAL_DECIMAL___DEEPCOPY___METHODDEF + _DECIMAL_DECIMAL___FORMAT___METHODDEF + _DECIMAL_DECIMAL___REDUCE___METHODDEF + _DECIMAL_DECIMAL___ROUND___METHODDEF + _DECIMAL_DECIMAL___CEIL___METHODDEF + _DECIMAL_DECIMAL___FLOOR___METHODDEF + _DECIMAL_DECIMAL___TRUNC___METHODDEF + _DECIMAL_DECIMAL___COMPLEX___METHODDEF + _DECIMAL_DECIMAL___SIZEOF___METHODDEF { NULL, NULL, 1 } }; @@ -5169,11 +6108,11 @@ static PyType_Slot dec_slots[] = { {Py_tp_token, Py_TP_USE_SPEC}, {Py_tp_dealloc, dec_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_traverse, dec_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_repr, dec_repr}, {Py_tp_hash, dec_hash}, {Py_tp_str, dec_str}, - {Py_tp_doc, (void *)doc_decimal}, + {Py_tp_doc, (void *)dec_new__doc__}, {Py_tp_richcompare, dec_richcompare}, {Py_tp_methods, dec_methods}, {Py_tp_getset, dec_getsets}, @@ -5216,47 +6155,47 @@ static PyType_Spec dec_spec = { /* Macros for converting mpdecimal functions to Context methods */ /************************************************************************/ -/* Boolean context method. */ +/* Boolean context method. + Argument Clinic provides PyObject *context, PyObject *x +*/ #define DecCtx_BoolFunc(MPDFUNC) \ -static PyObject * \ -ctx_##MPDFUNC(PyObject *context, PyObject *v) \ { \ PyObject *ret; \ PyObject *a; \ \ - CONVERT_OP_RAISE(&a, v, context); \ + CONVERT_OP_RAISE(&a, x, context); \ \ ret = MPDFUNC(MPD(a), CTX(context)) ? incr_true() : incr_false(); \ Py_DECREF(a); \ return ret; \ } -/* Boolean context method. MPDFUNC does NOT use a context. */ +/* Boolean context method. MPDFUNC does NOT use a context. + Argument Clinic provides PyObject *context, PyObject *x +*/ #define DecCtx_BoolFunc_NO_CTX(MPDFUNC) \ -static PyObject * \ -ctx_##MPDFUNC(PyObject *context, PyObject *v) \ { \ PyObject *ret; \ PyObject *a; \ \ - CONVERT_OP_RAISE(&a, v, context); \ + CONVERT_OP_RAISE(&a, x, context); \ \ ret = MPDFUNC(MPD(a)) ? incr_true() : incr_false(); \ Py_DECREF(a); \ return ret; \ } -/* Unary context method. */ +/* Unary context method. + Argument Clinic provides PyObject *context, + PyTypeObject *cls, PyObject *x +*/ #define DecCtx_UnaryFunc(MPDFUNC) \ -static PyObject * \ -ctx_##MPDFUNC(PyObject *context, PyObject *v) \ { \ PyObject *result, *a; \ uint32_t status = 0; \ \ - CONVERT_OP_RAISE(&a, v, context); \ - decimal_state *state = \ - get_module_state_from_ctx(context); \ + CONVERT_OP_RAISE(&a, x, context); \ + decimal_state *state = PyType_GetModuleState(cls); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ return NULL; \ @@ -5272,23 +6211,18 @@ ctx_##MPDFUNC(PyObject *context, PyObject *v) \ return result; \ } -/* Binary context method. */ +/* Binary context method. + Argument Clinic provides PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y +*/ #define DecCtx_BinaryFunc(MPDFUNC) \ -static PyObject * \ -ctx_##MPDFUNC(PyObject *context, PyObject *args) \ { \ - PyObject *v, *w; \ PyObject *a, *b; \ PyObject *result; \ uint32_t status = 0; \ \ - if (!PyArg_ParseTuple(args, "OO", &v, &w)) { \ - return NULL; \ - } \ - \ - CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ - decimal_state *state = \ - get_module_state_from_ctx(context); \ + CONVERT_BINOP_RAISE(&a, &b, x, y, context); \ + decimal_state *state = PyType_GetModuleState(cls); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5309,22 +6243,17 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ /* * Binary context method. The context is only used for conversion. * The actual MPDFUNC does NOT take a context arg. + * Argument Clinic provides PyObject *context, PyTypeObject *cls, + * PyObject *x, PyObject *y */ #define DecCtx_BinaryFunc_NO_CTX(MPDFUNC) \ -static PyObject * \ -ctx_##MPDFUNC(PyObject *context, PyObject *args) \ { \ - PyObject *v, *w; \ PyObject *a, *b; \ PyObject *result; \ \ - if (!PyArg_ParseTuple(args, "OO", &v, &w)) { \ - return NULL; \ - } \ - \ - CONVERT_BINOP_RAISE(&a, &b, v, w, context); \ + CONVERT_BINOP_RAISE(&a, &b, x, y, context); \ decimal_state *state = \ - get_module_state_from_ctx(context); \ + PyType_GetModuleState(cls); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5338,22 +6267,18 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ return result; \ } -/* Ternary context method. */ +/* Ternary context method. + Argument Clinic provides PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y, PyObject *z +*/ #define DecCtx_TernaryFunc(MPDFUNC) \ -static PyObject * \ -ctx_##MPDFUNC(PyObject *context, PyObject *args) \ { \ - PyObject *v, *w, *x; \ PyObject *a, *b, *c; \ PyObject *result; \ uint32_t status = 0; \ \ - if (!PyArg_ParseTuple(args, "OOO", &v, &w, &x)) { \ - return NULL; \ - } \ - \ - CONVERT_TERNOP_RAISE(&a, &b, &c, v, w, x, context); \ - decimal_state *state = get_module_state_from_ctx(context); \ + CONVERT_TERNOP_RAISE(&a, &b, &c, x, y, z, context); \ + decimal_state *state = PyType_GetModuleState(cls); \ if ((result = dec_alloc(state)) == NULL) { \ Py_DECREF(a); \ Py_DECREF(b); \ @@ -5375,50 +6300,382 @@ ctx_##MPDFUNC(PyObject *context, PyObject *args) \ /* Unary arithmetic functions */ + +/*[clinic input] +_decimal.Context.abs + + self as context: self + cls: defining_class + x: object + / + +Return the absolute value of x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_abs_impl(PyObject *context, PyTypeObject *cls, PyObject *x) +/*[clinic end generated code: output=fe080467d32e229c input=00a33f9c68463bb0]*/ DecCtx_UnaryFunc(mpd_qabs) + +/*[clinic input] +_decimal.Context.exp = _decimal.Context.abs + +Return e ** x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_exp_impl(PyObject *context, PyTypeObject *cls, PyObject *x) +/*[clinic end generated code: output=c7477a67010ccc5f input=5b443c4ab153dd2e]*/ DecCtx_UnaryFunc(mpd_qexp) + +/*[clinic input] +_decimal.Context.ln = _decimal.Context.abs + +Return the natural (base e) logarithm of x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_ln_impl(PyObject *context, PyTypeObject *cls, PyObject *x) +/*[clinic end generated code: output=63e691b0680bffc7 input=cf43cd98a0fe7425]*/ DecCtx_UnaryFunc(mpd_qln) + +/*[clinic input] +_decimal.Context.log10 = _decimal.Context.abs + +Return the base 10 logarithm of x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_log10_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=e0d9fc928570304d input=309e57faf42c257d]*/ DecCtx_UnaryFunc(mpd_qlog10) + +/*[clinic input] +_decimal.Context.minus = _decimal.Context.abs + +Minus corresponds to unary prefix minus in Python. + +This operation applies the context to the result. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_minus_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=f06c409b6aef1aad input=63be4c419d1d554b]*/ DecCtx_UnaryFunc(mpd_qminus) + +/*[clinic input] +_decimal.Context.next_minus = _decimal.Context.abs + +Return the largest representable number smaller than x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_next_minus_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=8dd168f08bec9547 input=969f4d24dfcd5e85]*/ DecCtx_UnaryFunc(mpd_qnext_minus) + +/*[clinic input] +_decimal.Context.next_plus = _decimal.Context.abs + +Return the smallest representable number larger than x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_next_plus_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=2a50586ad2f7c108 input=af1a85ee59b56a3c]*/ DecCtx_UnaryFunc(mpd_qnext_plus) -DecCtx_UnaryFunc(mpd_qplus) + +/*[clinic input] +_decimal.Context.normalize = _decimal.Context.abs + +Reduce x to its simplest form. Alias for reduce(x). +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_normalize_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=9a9510f442ba2852 input=a65bc39c81a654a9]*/ DecCtx_UnaryFunc(mpd_qreduce) + +/*[clinic input] +_decimal.Context.plus = _decimal.Context.abs + +Plus corresponds to the unary prefix plus operator in Python. + +This operation applies the context to the result. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_plus_impl(PyObject *context, PyTypeObject *cls, PyObject *x) +/*[clinic end generated code: output=c37d29f58a47f93a input=5d8a75702d20e2f9]*/ +DecCtx_UnaryFunc(mpd_qplus) + +/*[clinic input] +_decimal.Context.to_integral_value = _decimal.Context.abs + +Round to an integer. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_to_integral_value_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=e3d9ad000bc06036 input=3103e147cb9de9ed]*/ DecCtx_UnaryFunc(mpd_qround_to_int) + +/*[clinic input] +_decimal.Context.to_integral_exact = _decimal.Context.abs + +Round to an integer. Signal if the result is rounded or inexact. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_to_integral_exact_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=680b796dfae8e2ef input=677dc4b915907b68]*/ DecCtx_UnaryFunc(mpd_qround_to_intx) + +/*[clinic input] +_decimal.Context.to_integral = _decimal.Context.abs + +Identical to to_integral_value(x). +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_to_integral_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=09f4823b90b2cf17 input=89d4a4b15495b8c9]*/ +DecCtx_UnaryFunc(mpd_qround_to_int) + +/*[clinic input] +_decimal.Context.sqrt = _decimal.Context.abs + +Square root of a non-negative number to context precision. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_sqrt_impl(PyObject *context, PyTypeObject *cls, PyObject *x) +/*[clinic end generated code: output=2b9c16c6f5ceead0 input=90bd954b0b8076fb]*/ DecCtx_UnaryFunc(mpd_qsqrt) /* Binary arithmetic functions */ -DecCtx_BinaryFunc(mpd_qadd) -DecCtx_BinaryFunc(mpd_qcompare) -DecCtx_BinaryFunc(mpd_qcompare_signal) -DecCtx_BinaryFunc(mpd_qdiv) -DecCtx_BinaryFunc(mpd_qdivint) -DecCtx_BinaryFunc(mpd_qmax) -DecCtx_BinaryFunc(mpd_qmax_mag) -DecCtx_BinaryFunc(mpd_qmin) -DecCtx_BinaryFunc(mpd_qmin_mag) -DecCtx_BinaryFunc(mpd_qmul) -DecCtx_BinaryFunc(mpd_qnext_toward) -DecCtx_BinaryFunc(mpd_qquantize) -DecCtx_BinaryFunc(mpd_qrem) -DecCtx_BinaryFunc(mpd_qrem_near) -DecCtx_BinaryFunc(mpd_qsub) + +/*[clinic input] +_decimal.Context.add + + self as context: self + cls: defining_class + x: object + y: object + / + +Return the sum of x and y. +[clinic start generated code]*/ static PyObject * -ctx_mpd_qdivmod(PyObject *context, PyObject *args) +_decimal_Context_add_impl(PyObject *context, PyTypeObject *cls, PyObject *x, + PyObject *y) +/*[clinic end generated code: output=ab4f0fb841e6a867 input=f2c74f6a845f62e9]*/ +DecCtx_BinaryFunc(mpd_qadd) + +/*[clinic input] +_decimal.Context.compare = _decimal.Context.add + +Compare x and y numerically. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_compare_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=56efd1faf653f1d7 input=f701cb179c966ec1]*/ +DecCtx_BinaryFunc(mpd_qcompare) + +/*[clinic input] +_decimal.Context.compare_signal = _decimal.Context.add + +Compare x and y numerically. All NaNs signal. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_compare_signal_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=7c1a9a9f6ae4e5cd input=32a1bcef7bbc5179]*/ +DecCtx_BinaryFunc(mpd_qcompare_signal) + +/*[clinic input] +_decimal.Context.divide = _decimal.Context.add + +Return x divided by y. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_divide_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=1a7924b20e24a528 input=00cd9bc2ba2a1786]*/ +DecCtx_BinaryFunc(mpd_qdiv) + +/*[clinic input] +_decimal.Context.divide_int = _decimal.Context.add + +Return x divided by y, truncated to an integer. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_divide_int_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=7a1d8948625105f0 input=e80ada2f50d9719d]*/ +DecCtx_BinaryFunc(mpd_qdivint) + +/*[clinic input] +_decimal.Context.max = _decimal.Context.add + +Compare the values numerically and return the maximum. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_max_impl(PyObject *context, PyTypeObject *cls, PyObject *x, + PyObject *y) +/*[clinic end generated code: output=cd54af10a51c11fc input=22008ab898c86a8b]*/ +DecCtx_BinaryFunc(mpd_qmax) + +/*[clinic input] +_decimal.Context.max_mag = _decimal.Context.add + +Compare the values numerically with their sign ignored. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_max_mag_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=1c812e73bcb7827f input=f7ce42ef82a7c52e]*/ +DecCtx_BinaryFunc(mpd_qmax_mag) + +/*[clinic input] +_decimal.Context.min = _decimal.Context.add + +Compare the values numerically and return the minimum. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_min_impl(PyObject *context, PyTypeObject *cls, PyObject *x, + PyObject *y) +/*[clinic end generated code: output=aa494e95b88107b3 input=2aeec1167638c5ef]*/ +DecCtx_BinaryFunc(mpd_qmin) + +/*[clinic input] +_decimal.Context.min_mag = _decimal.Context.add + +Compare the values numerically with their sign ignored. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_min_mag_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=ee0b69c1d9a14185 input=19d158c29e4fc140]*/ +DecCtx_BinaryFunc(mpd_qmin_mag) + +/*[clinic input] +_decimal.Context.multiply = _decimal.Context.add + +Return the product of x and y. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_multiply_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=45f33b805afa01a8 input=2fdd01acdbeef8ba]*/ +DecCtx_BinaryFunc(mpd_qmul) + +/*[clinic input] +_decimal.Context.next_toward = _decimal.Context.add + +Return the number closest to x, in the direction towards y. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_next_toward_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=436afff6f43edec2 input=aac775298e02b68c]*/ +DecCtx_BinaryFunc(mpd_qnext_toward) + +/*[clinic input] +_decimal.Context.quantize = _decimal.Context.add + +Return a value equal to x (rounded), having the exponent of y. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_quantize_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=fcf8cd32b7d628c9 input=43d67a696ab6d895]*/ +DecCtx_BinaryFunc(mpd_qquantize) + +/*[clinic input] +_decimal.Context.remainder = _decimal.Context.add + +Return the remainder from integer division. + +The sign of the result, if non-zero, is the same as that of the +original dividend. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_remainder_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=e0f96c834abbfbd2 input=36d0eb2b392c1215]*/ +DecCtx_BinaryFunc(mpd_qrem) + +/*[clinic input] +_decimal.Context.remainder_near = _decimal.Context.add + +Return x - y * n. + +Here n is the integer nearest the exact value of x / y (if the result +is 0 then its sign will be the sign of x). +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_remainder_near_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=7f18c535a12cf8ac input=bafb6327bb314c5c]*/ +DecCtx_BinaryFunc(mpd_qrem_near) + +/*[clinic input] +_decimal.Context.subtract = _decimal.Context.add + +Return the difference between x and y. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_subtract_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=3d764a8a87e79401 input=6767683ec68f7a1a]*/ +DecCtx_BinaryFunc(mpd_qsub) + +/*[clinic input] +_decimal.Context.divmod + + self as context: self + x: object + y: object + / + +Return quotient and remainder of the division x / y. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_divmod_impl(PyObject *context, PyObject *x, PyObject *y) +/*[clinic end generated code: output=5dbf5410e3f302af input=4d8eee07823c752a]*/ { - PyObject *v, *w; PyObject *a, *b; PyObject *q, *r; uint32_t status = 0; PyObject *ret; - if (!PyArg_ParseTuple(args, "OO", &v, &w)) { - return NULL; - } - - CONVERT_BINOP_RAISE(&a, &b, v, w, context); + CONVERT_BINOP_RAISE(&a, &b, x, y, context); decimal_state *state = get_module_state_from_ctx(context); q = dec_alloc(state); if (q == NULL) { @@ -5450,20 +6707,42 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) } /* Binary or ternary arithmetic functions */ + +/*[clinic input] +_decimal.Context.power + + self as context: self + cls: defining_class + a as base: object + b as exp: object + modulo as mod: object = None + +Compute a**b. + +If 'a' is negative, then 'b' must be integral. The result will be +inexact unless 'a' is integral and the result is finite and can be +expressed exactly in 'precision' digits. In the Python version the +result is always correctly rounded, in the C version the result is +almost always correctly rounded. + +If modulo is given, compute (a**b) % modulo. The following +restrictions hold: + + * all three arguments must be integral + * 'b' must be nonnegative + * at least one of 'a' or 'b' must be nonzero + * modulo must be nonzero and less than 10**prec in absolute value +[clinic start generated code]*/ + static PyObject * -ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) +_decimal_Context_power_impl(PyObject *context, PyTypeObject *cls, + PyObject *base, PyObject *exp, PyObject *mod) +/*[clinic end generated code: output=d06d40c37cdd69dc input=2a70edd03317c666]*/ { - static char *kwlist[] = {"a", "b", "modulo", NULL}; - PyObject *base, *exp, *mod = Py_None; PyObject *a, *b, *c = NULL; PyObject *result; uint32_t status = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O", kwlist, - &base, &exp, &mod)) { - return NULL; - } - CONVERT_BINOP_RAISE(&a, &b, base, exp, context); if (mod != Py_None) { @@ -5474,7 +6753,7 @@ ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) } } - decimal_state *state = get_module_state_from_ctx(context); + decimal_state *state = PyType_GetModuleState(cls); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5503,74 +6782,255 @@ ctx_mpd_qpow(PyObject *context, PyObject *args, PyObject *kwds) } /* Ternary arithmetic functions */ + +/*[clinic input] +_decimal.Context.fma + + self as context: self + cls: defining_class + x: object + y: object + z: object + / + +Return x multiplied by y, plus z. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_fma_impl(PyObject *context, PyTypeObject *cls, PyObject *x, + PyObject *y, PyObject *z) +/*[clinic end generated code: output=08ec3cefc59d71a9 input=da3963b1a1da83b9]*/ DecCtx_TernaryFunc(mpd_qfma) /* No argument */ + +/*[clinic input] +_decimal.Context.radix + + self as context: self + cls: defining_class + +Return 10. +[clinic start generated code]*/ + static PyObject * -ctx_mpd_radix(PyObject *context, PyObject *dummy) +_decimal_Context_radix_impl(PyObject *context, PyTypeObject *cls) +/*[clinic end generated code: output=674b88b7cd0c264d input=e1e4f8c0abf86825]*/ { - decimal_state *state = get_module_state_from_ctx(context); + decimal_state *state = PyType_GetModuleState(cls); return _dec_mpd_radix(state); } /* Boolean functions: single decimal argument */ -DecCtx_BoolFunc(mpd_isnormal) -DecCtx_BoolFunc(mpd_issubnormal) -DecCtx_BoolFunc_NO_CTX(mpd_isfinite) -DecCtx_BoolFunc_NO_CTX(mpd_isinfinite) -DecCtx_BoolFunc_NO_CTX(mpd_isnan) -DecCtx_BoolFunc_NO_CTX(mpd_isqnan) -DecCtx_BoolFunc_NO_CTX(mpd_issigned) -DecCtx_BoolFunc_NO_CTX(mpd_issnan) -DecCtx_BoolFunc_NO_CTX(mpd_iszero) + +/*[clinic input] +_decimal.Context.is_normal + + self as context: self + cls: defining_class + x: object + / + +Return True if x is a normal number, False otherwise. +[clinic start generated code]*/ static PyObject * -ctx_iscanonical(PyObject *context, PyObject *v) +_decimal_Context_is_normal_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=089c5609db60bf57 input=7c90b825a517ef7e]*/ +DecCtx_BoolFunc(mpd_isnormal) + +/*[clinic input] +_decimal.Context.is_subnormal = _decimal.Context.is_normal + +Return True if x is subnormal, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_subnormal_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=f58c45a288aadeda input=73f1bd9367b913a4]*/ +DecCtx_BoolFunc(mpd_issubnormal) + +/*[clinic input] +_decimal.Context.is_finite = _decimal.Context.is_normal + +Return True if x is finite, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_finite_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=dfb00f1b5589b9f0 input=abff92a8a6bb85e6]*/ +DecCtx_BoolFunc_NO_CTX(mpd_isfinite) + +/*[clinic input] +_decimal.Context.is_infinite = _decimal.Context.is_normal + +Return True if x is infinite, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_infinite_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=1c28517500811d01 input=591242ae9a1e60e6]*/ +DecCtx_BoolFunc_NO_CTX(mpd_isinfinite) + +/*[clinic input] +_decimal.Context.is_nan = _decimal.Context.is_normal + +Return True if x is a qNaN or sNaN, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_nan_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=9dc15463ee19864a input=520218376d5eec5e]*/ +DecCtx_BoolFunc_NO_CTX(mpd_isnan) + +/*[clinic input] +_decimal.Context.is_qnan = _decimal.Context.is_normal + +Return True if x is a quiet NaN, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_qnan_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=4caa672e03703b6d input=97d06a14ab3360d1]*/ +DecCtx_BoolFunc_NO_CTX(mpd_isqnan) + +/*[clinic input] +_decimal.Context.is_snan = _decimal.Context.is_normal + +Return True if x is a signaling NaN, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_snan_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=a8caa929d9f82ecd input=0059fe4e9c3b25a8]*/ +DecCtx_BoolFunc_NO_CTX(mpd_issnan) + +/*[clinic input] +_decimal.Context.is_signed = _decimal.Context.is_normal + +Return True if x is negative, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_signed_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=42c450c99d4fe7db input=b950cd697721ab8b]*/ +DecCtx_BoolFunc_NO_CTX(mpd_issigned) + +/*[clinic input] +_decimal.Context.is_zero = _decimal.Context.is_normal + +Return True if x is a zero, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_zero_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=e6c55359b7241d9e input=bf08197d142a8027]*/ +DecCtx_BoolFunc_NO_CTX(mpd_iszero) + +/*[clinic input] +_decimal.Context.is_canonical = _decimal.Context.is_normal + +Return True if x is canonical, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_is_canonical_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=18ee249d9aec957c input=1bf2129808e55eb9]*/ { - decimal_state *state = get_module_state_from_ctx(context); - if (!PyDec_Check(state, v)) { + decimal_state *state = PyType_GetModuleState(cls); + if (!PyDec_Check(state, x)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); return NULL; } - return mpd_iscanonical(MPD(v)) ? incr_true() : incr_false(); + return mpd_iscanonical(MPD(x)) ? incr_true() : incr_false(); } /* Functions with a single decimal argument */ +/*[clinic input] +_decimal.Context._apply = _decimal.Context.is_normal + +Apply self to Decimal x. +[clinic start generated code]*/ + static PyObject * -PyDecContext_Apply(PyObject *context, PyObject *v) +_decimal_Context__apply_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=c6b542f4e8114b97 input=12b34468ca4a4c30]*/ { PyObject *result, *a; - CONVERT_OP_RAISE(&a, v, context); + CONVERT_OP_RAISE(&a, x, context); result = dec_apply(a, context); Py_DECREF(a); return result; } +#ifdef EXTRA_FUNCTIONALITY +/*[clinic input] +_decimal.Context.apply = _decimal.Context._apply + +Apply self to Decimal x. +[clinic start generated code]*/ + static PyObject * -ctx_canonical(PyObject *context, PyObject *v) +_decimal_Context_apply_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=f8a7142d47ad4ff3 input=388e66ca82733516]*/ { - decimal_state *state = get_module_state_from_ctx(context); - if (!PyDec_Check(state, v)) { + return _decimal_Context__apply(context, v); +} +#endif + +/*[clinic input] +_decimal.Context.canonical = _decimal.Context.is_normal + +Return a new instance of x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_canonical_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=f213e433e2032e5e input=025ecb106ac15bff]*/ +{ + decimal_state *state = PyType_GetModuleState(cls); + if (!PyDec_Check(state, x)) { PyErr_SetString(PyExc_TypeError, "argument must be a Decimal"); return NULL; } - return Py_NewRef(v); + return Py_NewRef(x); } +/*[clinic input] +_decimal.Context.copy_abs = _decimal.Context.is_normal + +Return a copy of x with the sign set to 0. +[clinic start generated code]*/ + static PyObject * -ctx_mpd_qcopy_abs(PyObject *context, PyObject *v) +_decimal_Context_copy_abs_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=a141ad4b9afe2deb input=4aa2f612625f0f73]*/ { PyObject *result, *a; uint32_t status = 0; - CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_from_ctx(context); + CONVERT_OP_RAISE(&a, x, context); + decimal_state *state = PyType_GetModuleState(cls); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5587,23 +7047,39 @@ ctx_mpd_qcopy_abs(PyObject *context, PyObject *v) return result; } +/*[clinic input] +_decimal.Context.copy_decimal = _decimal.Context.is_normal + +Return a copy of Decimal x. +[clinic start generated code]*/ + static PyObject * -ctx_copy_decimal(PyObject *context, PyObject *v) +_decimal_Context_copy_decimal_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=639a82e1193d31f6 input=4db4f942f45fb7c9]*/ { PyObject *result; - CONVERT_OP_RAISE(&result, v, context); + CONVERT_OP_RAISE(&result, x, context); return result; } +/*[clinic input] +_decimal.Context.copy_negate = _decimal.Context.is_normal + +Return a copy of x with the sign inverted. +[clinic start generated code]*/ + static PyObject * -ctx_mpd_qcopy_negate(PyObject *context, PyObject *v) +_decimal_Context_copy_negate_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=e49d013489dc252b input=2e6e213e2ed0efda]*/ { PyObject *result, *a; uint32_t status = 0; - CONVERT_OP_RAISE(&a, v, context); - decimal_state *state = get_module_state_from_ctx(context); + CONVERT_OP_RAISE(&a, x, context); + decimal_state *state = PyType_GetModuleState(cls); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5620,16 +7096,57 @@ ctx_mpd_qcopy_negate(PyObject *context, PyObject *v) return result; } -DecCtx_UnaryFunc(mpd_qlogb) -DecCtx_UnaryFunc(mpd_qinvert) +/*[clinic input] +_decimal.Context.logb = _decimal.Context.abs + +Return the exponent of the magnitude of the operand's MSD. +[clinic start generated code]*/ static PyObject * -ctx_mpd_class(PyObject *context, PyObject *v) +_decimal_Context_logb_impl(PyObject *context, PyTypeObject *cls, PyObject *x) +/*[clinic end generated code: output=9b9697e1eb68093f input=28d1cd1a8a906b9a]*/ +DecCtx_UnaryFunc(mpd_qlogb) + +/*[clinic input] +_decimal.Context.logical_invert = _decimal.Context.abs + +Invert all the digits in the operand. + +The operand must be a logical number. + + >>> ExtendedContext.logical_invert(Decimal('0')) + Decimal('111111111') + >>> ExtendedContext.logical_invert(Decimal('1')) + Decimal('111111110') + >>> ExtendedContext.logical_invert(Decimal('111111111')) + Decimal('0') + >>> ExtendedContext.logical_invert(Decimal('101010101')) + Decimal('10101010') + >>> ExtendedContext.logical_invert(1101) + Decimal('111110010') +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_logical_invert_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=97760277a958e2b0 input=8e568f4c745ab596]*/ +DecCtx_UnaryFunc(mpd_qinvert) + +/*[clinic input] +_decimal.Context.number_class = _decimal.Context.is_normal + +Return an indication of the class of x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_number_class_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=c1592a23e25ba5ee input=1ead8462f1800e4e]*/ { PyObject *a; const char *cp; - CONVERT_OP_RAISE(&a, v, context); + CONVERT_OP_RAISE(&a, x, context); cp = mpd_class(MPD(a), CTX(context)); Py_DECREF(a); @@ -5637,15 +7154,23 @@ ctx_mpd_class(PyObject *context, PyObject *v) return PyUnicode_FromString(cp); } +/*[clinic input] +_decimal.Context.to_sci_string = _decimal.Context.is_normal + +Convert a number to a string using scientific notation. +[clinic start generated code]*/ + static PyObject * -ctx_mpd_to_sci(PyObject *context, PyObject *v) +_decimal_Context_to_sci_string_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=092dcdef999d72da input=ed442677c66d342d]*/ { PyObject *result; PyObject *a; mpd_ssize_t size; char *s; - CONVERT_OP_RAISE(&a, v, context); + CONVERT_OP_RAISE(&a, x, context); size = mpd_to_sci_size(&s, MPD(a), CtxCaps(context)); Py_DECREF(a); @@ -5660,15 +7185,23 @@ ctx_mpd_to_sci(PyObject *context, PyObject *v) return result; } +/*[clinic input] +_decimal.Context.to_eng_string = _decimal.Context.is_normal + +Convert a number to a string, using engineering notation. +[clinic start generated code]*/ + static PyObject * -ctx_mpd_to_eng(PyObject *context, PyObject *v) +_decimal_Context_to_eng_string_impl(PyObject *context, PyTypeObject *cls, + PyObject *x) +/*[clinic end generated code: output=7fc53216c208f487 input=a574385e2e3e3bc0]*/ { PyObject *result; PyObject *a; mpd_ssize_t size; char *s; - CONVERT_OP_RAISE(&a, v, context); + CONVERT_OP_RAISE(&a, x, context); size = mpd_to_eng_size(&s, MPD(a), CtxCaps(context)); Py_DECREF(a); @@ -5684,23 +7217,54 @@ ctx_mpd_to_eng(PyObject *context, PyObject *v) } /* Functions with two decimal arguments */ -DecCtx_BinaryFunc_NO_CTX(mpd_compare_total) -DecCtx_BinaryFunc_NO_CTX(mpd_compare_total_mag) + +/*[clinic input] +_decimal.Context.compare_total + + self as context: self + cls: defining_class + x: object + y: object + / + +Compare x and y using their abstract representation. +[clinic start generated code]*/ static PyObject * -ctx_mpd_qcopy_sign(PyObject *context, PyObject *args) +_decimal_Context_compare_total_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=f79177b27fe930e3 input=2bfc677a841e297a]*/ +DecCtx_BinaryFunc_NO_CTX(mpd_compare_total) + +/*[clinic input] +_decimal.Context.compare_total_mag = _decimal.Context.compare_total + +Compare x and y using their abstract representation, ignoring sign. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_compare_total_mag_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=2528c669ccd6d6ff input=2b982e69f932dcb2]*/ +DecCtx_BinaryFunc_NO_CTX(mpd_compare_total_mag) + +/*[clinic input] +_decimal.Context.copy_sign = _decimal.Context.compare_total + +Copy the sign from y to x. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_copy_sign_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=77d23b6f4e42120c input=c0682aeaffc7cfdf]*/ { - PyObject *v, *w; PyObject *a, *b; PyObject *result; uint32_t status = 0; - if (!PyArg_ParseTuple(args, "OO", &v, &w)) { - return NULL; - } - - CONVERT_BINOP_RAISE(&a, &b, v, w, context); - decimal_state *state = get_module_state_from_ctx(context); + CONVERT_BINOP_RAISE(&a, &b, x, y, context); + decimal_state *state = PyType_GetModuleState(cls); result = dec_alloc(state); if (result == NULL) { Py_DECREF(a); @@ -5719,26 +7283,156 @@ ctx_mpd_qcopy_sign(PyObject *context, PyObject *args) return result; } -DecCtx_BinaryFunc(mpd_qand) -DecCtx_BinaryFunc(mpd_qor) -DecCtx_BinaryFunc(mpd_qxor) +/*[clinic input] +_decimal.Context.logical_and = _decimal.Context.add -DecCtx_BinaryFunc(mpd_qrotate) -DecCtx_BinaryFunc(mpd_qscaleb) -DecCtx_BinaryFunc(mpd_qshift) +Applies the logical operation 'and' between each operand's digits. + +The operands must be both logical numbers. + + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010')) + Decimal('1000') + >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10')) + Decimal('10') + >>> ExtendedContext.logical_and(110, 1101) + Decimal('100') + >>> ExtendedContext.logical_and(Decimal(110), 1101) + Decimal('100') + >>> ExtendedContext.logical_and(110, Decimal(1101)) + Decimal('100') +[clinic start generated code]*/ static PyObject * -ctx_mpd_same_quantum(PyObject *context, PyObject *args) +_decimal_Context_logical_and_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=009dfa08ecaa2ac8 input=bcb7d3d6ab7530de]*/ +DecCtx_BinaryFunc(mpd_qand) + +/*[clinic input] +_decimal.Context.logical_or = _decimal.Context.add + +Applies the logical operation 'or' between each operand's digits. + +The operands must be both logical numbers. + + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010')) + Decimal('1110') + >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10')) + Decimal('1110') + >>> ExtendedContext.logical_or(110, 1101) + Decimal('1111') + >>> ExtendedContext.logical_or(Decimal(110), 1101) + Decimal('1111') + >>> ExtendedContext.logical_or(110, Decimal(1101)) + Decimal('1111') +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_logical_or_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=eb38617e8d31bf12 input=47b45d296fb90846]*/ +DecCtx_BinaryFunc(mpd_qor) + +/*[clinic input] +_decimal.Context.logical_xor = _decimal.Context.add + +Applies the logical operation 'xor' between each operand's digits. + +The operands must be both logical numbers. + + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0')) + Decimal('1') + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1')) + Decimal('0') + >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010')) + Decimal('110') + >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10')) + Decimal('1101') + >>> ExtendedContext.logical_xor(110, 1101) + Decimal('1011') + >>> ExtendedContext.logical_xor(Decimal(110), 1101) + Decimal('1011') + >>> ExtendedContext.logical_xor(110, Decimal(1101)) + Decimal('1011') +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_logical_xor_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=23cd81fdcd865d5a input=fcaaf828c1d2d089]*/ +DecCtx_BinaryFunc(mpd_qxor) + +/*[clinic input] +_decimal.Context.rotate = _decimal.Context.add + +Return a copy of x, rotated by y places. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_rotate_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=3d5b3cfcb4659432 input=7ad91845c909eb0a]*/ +DecCtx_BinaryFunc(mpd_qrotate) + +/*[clinic input] +_decimal.Context.scaleb = _decimal.Context.add + +Return the first operand after adding the second value to its exp. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_scaleb_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=795ac61bcbe61c67 input=c5d2ee7a57f65f8c]*/ +DecCtx_BinaryFunc(mpd_qscaleb) + +/*[clinic input] +_decimal.Context.shift = _decimal.Context.add + +Return a copy of x, shifted by y places. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_shift_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=43d69615f0271c81 input=1ab44ff0854420ce]*/ +DecCtx_BinaryFunc(mpd_qshift) + +/*[clinic input] +_decimal.Context.same_quantum = _decimal.Context.add + +Return True if the two operands have the same exponent. +[clinic start generated code]*/ + +static PyObject * +_decimal_Context_same_quantum_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y) +/*[clinic end generated code: output=91a4d8325f98d9e9 input=194cd156e398eaf9]*/ { - PyObject *v, *w; PyObject *a, *b; PyObject *result; - if (!PyArg_ParseTuple(args, "OO", &v, &w)) { - return NULL; - } - - CONVERT_BINOP_RAISE(&a, &b, v, w, context); + CONVERT_BINOP_RAISE(&a, &b, x, y, context); result = mpd_same_quantum(MPD(a), MPD(b)) ? incr_true() : incr_false(); Py_DECREF(a); @@ -5751,105 +7445,101 @@ ctx_mpd_same_quantum(PyObject *context, PyObject *args) static PyMethodDef context_methods [] = { /* Unary arithmetic functions */ - { "abs", ctx_mpd_qabs, METH_O, doc_ctx_abs }, - { "exp", ctx_mpd_qexp, METH_O, doc_ctx_exp }, - { "ln", ctx_mpd_qln, METH_O, doc_ctx_ln }, - { "log10", ctx_mpd_qlog10, METH_O, doc_ctx_log10 }, - { "minus", ctx_mpd_qminus, METH_O, doc_ctx_minus }, - { "next_minus", ctx_mpd_qnext_minus, METH_O, doc_ctx_next_minus }, - { "next_plus", ctx_mpd_qnext_plus, METH_O, doc_ctx_next_plus }, - { "normalize", ctx_mpd_qreduce, METH_O, doc_ctx_normalize }, - { "plus", ctx_mpd_qplus, METH_O, doc_ctx_plus }, - { "to_integral", ctx_mpd_qround_to_int, METH_O, doc_ctx_to_integral }, - { "to_integral_exact", ctx_mpd_qround_to_intx, METH_O, doc_ctx_to_integral_exact }, - { "to_integral_value", ctx_mpd_qround_to_int, METH_O, doc_ctx_to_integral_value }, - { "sqrt", ctx_mpd_qsqrt, METH_O, doc_ctx_sqrt }, + _DECIMAL_CONTEXT_ABS_METHODDEF + _DECIMAL_CONTEXT_EXP_METHODDEF + _DECIMAL_CONTEXT_LN_METHODDEF + _DECIMAL_CONTEXT_LOG10_METHODDEF + _DECIMAL_CONTEXT_MINUS_METHODDEF + _DECIMAL_CONTEXT_NEXT_MINUS_METHODDEF + _DECIMAL_CONTEXT_NEXT_PLUS_METHODDEF + _DECIMAL_CONTEXT_NORMALIZE_METHODDEF + _DECIMAL_CONTEXT_PLUS_METHODDEF + _DECIMAL_CONTEXT_TO_INTEGRAL_METHODDEF + _DECIMAL_CONTEXT_TO_INTEGRAL_EXACT_METHODDEF + _DECIMAL_CONTEXT_TO_INTEGRAL_VALUE_METHODDEF + _DECIMAL_CONTEXT_SQRT_METHODDEF /* Binary arithmetic functions */ - { "add", ctx_mpd_qadd, METH_VARARGS, doc_ctx_add }, - { "compare", ctx_mpd_qcompare, METH_VARARGS, doc_ctx_compare }, - { "compare_signal", ctx_mpd_qcompare_signal, METH_VARARGS, doc_ctx_compare_signal }, - { "divide", ctx_mpd_qdiv, METH_VARARGS, doc_ctx_divide }, - { "divide_int", ctx_mpd_qdivint, METH_VARARGS, doc_ctx_divide_int }, - { "divmod", ctx_mpd_qdivmod, METH_VARARGS, doc_ctx_divmod }, - { "max", ctx_mpd_qmax, METH_VARARGS, doc_ctx_max }, - { "max_mag", ctx_mpd_qmax_mag, METH_VARARGS, doc_ctx_max_mag }, - { "min", ctx_mpd_qmin, METH_VARARGS, doc_ctx_min }, - { "min_mag", ctx_mpd_qmin_mag, METH_VARARGS, doc_ctx_min_mag }, - { "multiply", ctx_mpd_qmul, METH_VARARGS, doc_ctx_multiply }, - { "next_toward", ctx_mpd_qnext_toward, METH_VARARGS, doc_ctx_next_toward }, - { "quantize", ctx_mpd_qquantize, METH_VARARGS, doc_ctx_quantize }, - { "remainder", ctx_mpd_qrem, METH_VARARGS, doc_ctx_remainder }, - { "remainder_near", ctx_mpd_qrem_near, METH_VARARGS, doc_ctx_remainder_near }, - { "subtract", ctx_mpd_qsub, METH_VARARGS, doc_ctx_subtract }, + _DECIMAL_CONTEXT_ADD_METHODDEF + _DECIMAL_CONTEXT_COMPARE_METHODDEF + _DECIMAL_CONTEXT_COMPARE_SIGNAL_METHODDEF + _DECIMAL_CONTEXT_DIVIDE_METHODDEF + _DECIMAL_CONTEXT_DIVIDE_INT_METHODDEF + _DECIMAL_CONTEXT_DIVMOD_METHODDEF + _DECIMAL_CONTEXT_MAX_METHODDEF + _DECIMAL_CONTEXT_MAX_MAG_METHODDEF + _DECIMAL_CONTEXT_MIN_METHODDEF + _DECIMAL_CONTEXT_MIN_MAG_METHODDEF + _DECIMAL_CONTEXT_MULTIPLY_METHODDEF + _DECIMAL_CONTEXT_NEXT_TOWARD_METHODDEF + _DECIMAL_CONTEXT_QUANTIZE_METHODDEF + _DECIMAL_CONTEXT_REMAINDER_METHODDEF + _DECIMAL_CONTEXT_REMAINDER_NEAR_METHODDEF + _DECIMAL_CONTEXT_SUBTRACT_METHODDEF /* Binary or ternary arithmetic functions */ - { "power", _PyCFunction_CAST(ctx_mpd_qpow), METH_VARARGS|METH_KEYWORDS, doc_ctx_power }, + _DECIMAL_CONTEXT_POWER_METHODDEF /* Ternary arithmetic functions */ - { "fma", ctx_mpd_qfma, METH_VARARGS, doc_ctx_fma }, + _DECIMAL_CONTEXT_FMA_METHODDEF /* No argument */ - { "Etiny", context_getetiny, METH_NOARGS, doc_ctx_Etiny }, - { "Etop", context_getetop, METH_NOARGS, doc_ctx_Etop }, - { "radix", ctx_mpd_radix, METH_NOARGS, doc_ctx_radix }, + _DECIMAL_CONTEXT_ETINY_METHODDEF + _DECIMAL_CONTEXT_ETOP_METHODDEF + _DECIMAL_CONTEXT_RADIX_METHODDEF /* Boolean functions */ - { "is_canonical", ctx_iscanonical, METH_O, doc_ctx_is_canonical }, - { "is_finite", ctx_mpd_isfinite, METH_O, doc_ctx_is_finite }, - { "is_infinite", ctx_mpd_isinfinite, METH_O, doc_ctx_is_infinite }, - { "is_nan", ctx_mpd_isnan, METH_O, doc_ctx_is_nan }, - { "is_normal", ctx_mpd_isnormal, METH_O, doc_ctx_is_normal }, - { "is_qnan", ctx_mpd_isqnan, METH_O, doc_ctx_is_qnan }, - { "is_signed", ctx_mpd_issigned, METH_O, doc_ctx_is_signed }, - { "is_snan", ctx_mpd_issnan, METH_O, doc_ctx_is_snan }, - { "is_subnormal", ctx_mpd_issubnormal, METH_O, doc_ctx_is_subnormal }, - { "is_zero", ctx_mpd_iszero, METH_O, doc_ctx_is_zero }, + _DECIMAL_CONTEXT_IS_CANONICAL_METHODDEF + _DECIMAL_CONTEXT_IS_FINITE_METHODDEF + _DECIMAL_CONTEXT_IS_INFINITE_METHODDEF + _DECIMAL_CONTEXT_IS_NAN_METHODDEF + _DECIMAL_CONTEXT_IS_NORMAL_METHODDEF + _DECIMAL_CONTEXT_IS_QNAN_METHODDEF + _DECIMAL_CONTEXT_IS_SIGNED_METHODDEF + _DECIMAL_CONTEXT_IS_SNAN_METHODDEF + _DECIMAL_CONTEXT_IS_SUBNORMAL_METHODDEF + _DECIMAL_CONTEXT_IS_ZERO_METHODDEF /* Functions with a single decimal argument */ - { "_apply", PyDecContext_Apply, METH_O, NULL }, /* alias for apply */ -#ifdef EXTRA_FUNCTIONALITY - { "apply", PyDecContext_Apply, METH_O, doc_ctx_apply }, -#endif - { "canonical", ctx_canonical, METH_O, doc_ctx_canonical }, - { "copy_abs", ctx_mpd_qcopy_abs, METH_O, doc_ctx_copy_abs }, - { "copy_decimal", ctx_copy_decimal, METH_O, doc_ctx_copy_decimal }, - { "copy_negate", ctx_mpd_qcopy_negate, METH_O, doc_ctx_copy_negate }, - { "logb", ctx_mpd_qlogb, METH_O, doc_ctx_logb }, - { "logical_invert", ctx_mpd_qinvert, METH_O, doc_ctx_logical_invert }, - { "number_class", ctx_mpd_class, METH_O, doc_ctx_number_class }, - { "to_sci_string", ctx_mpd_to_sci, METH_O, doc_ctx_to_sci_string }, - { "to_eng_string", ctx_mpd_to_eng, METH_O, doc_ctx_to_eng_string }, + _DECIMAL_CONTEXT__APPLY_METHODDEF + _DECIMAL_CONTEXT_APPLY_METHODDEF + _DECIMAL_CONTEXT_CANONICAL_METHODDEF + _DECIMAL_CONTEXT_COPY_ABS_METHODDEF + _DECIMAL_CONTEXT_COPY_DECIMAL_METHODDEF + _DECIMAL_CONTEXT_COPY_NEGATE_METHODDEF + _DECIMAL_CONTEXT_LOGB_METHODDEF + _DECIMAL_CONTEXT_LOGICAL_INVERT_METHODDEF + _DECIMAL_CONTEXT_NUMBER_CLASS_METHODDEF + _DECIMAL_CONTEXT_TO_SCI_STRING_METHODDEF + _DECIMAL_CONTEXT_TO_ENG_STRING_METHODDEF /* Functions with two decimal arguments */ - { "compare_total", ctx_mpd_compare_total, METH_VARARGS, doc_ctx_compare_total }, - { "compare_total_mag", ctx_mpd_compare_total_mag, METH_VARARGS, doc_ctx_compare_total_mag }, - { "copy_sign", ctx_mpd_qcopy_sign, METH_VARARGS, doc_ctx_copy_sign }, - { "logical_and", ctx_mpd_qand, METH_VARARGS, doc_ctx_logical_and }, - { "logical_or", ctx_mpd_qor, METH_VARARGS, doc_ctx_logical_or }, - { "logical_xor", ctx_mpd_qxor, METH_VARARGS, doc_ctx_logical_xor }, - { "rotate", ctx_mpd_qrotate, METH_VARARGS, doc_ctx_rotate }, - { "same_quantum", ctx_mpd_same_quantum, METH_VARARGS, doc_ctx_same_quantum }, - { "scaleb", ctx_mpd_qscaleb, METH_VARARGS, doc_ctx_scaleb }, - { "shift", ctx_mpd_qshift, METH_VARARGS, doc_ctx_shift }, + _DECIMAL_CONTEXT_COMPARE_TOTAL_METHODDEF + _DECIMAL_CONTEXT_COMPARE_TOTAL_MAG_METHODDEF + _DECIMAL_CONTEXT_COPY_SIGN_METHODDEF + _DECIMAL_CONTEXT_LOGICAL_AND_METHODDEF + _DECIMAL_CONTEXT_LOGICAL_OR_METHODDEF + _DECIMAL_CONTEXT_LOGICAL_XOR_METHODDEF + _DECIMAL_CONTEXT_ROTATE_METHODDEF + _DECIMAL_CONTEXT_SAME_QUANTUM_METHODDEF + _DECIMAL_CONTEXT_SCALEB_METHODDEF + _DECIMAL_CONTEXT_SHIFT_METHODDEF /* Set context values */ - { "clear_flags", context_clear_flags, METH_NOARGS, doc_ctx_clear_flags }, - { "clear_traps", context_clear_traps, METH_NOARGS, doc_ctx_clear_traps }, + _DECIMAL_CONTEXT_CLEAR_FLAGS_METHODDEF + _DECIMAL_CONTEXT_CLEAR_TRAPS_METHODDEF -#ifdef CONFIG_32 /* Unsafe set functions with relaxed range checks */ - { "_unsafe_setprec", context_unsafe_setprec, METH_O, NULL }, - { "_unsafe_setemin", context_unsafe_setemin, METH_O, NULL }, - { "_unsafe_setemax", context_unsafe_setemax, METH_O, NULL }, -#endif + _DECIMAL_CONTEXT__UNSAFE_SETPREC_METHODDEF + _DECIMAL_CONTEXT__UNSAFE_SETEMIN_METHODDEF + _DECIMAL_CONTEXT__UNSAFE_SETEMAX_METHODDEF /* Miscellaneous */ - { "__copy__", context_copy, METH_NOARGS, NULL }, - { "__reduce__", context_reduce, METH_NOARGS, NULL }, - { "copy", context_copy, METH_NOARGS, doc_ctx_copy }, - { "create_decimal", ctx_create_decimal, METH_VARARGS, doc_ctx_create_decimal }, - { "create_decimal_from_float", ctx_from_float, METH_O, doc_ctx_create_decimal_from_float }, + _DECIMAL_CONTEXT___COPY___METHODDEF + _DECIMAL_CONTEXT___REDUCE___METHODDEF + _DECIMAL_CONTEXT_COPY_METHODDEF + _DECIMAL_CONTEXT_CREATE_DECIMAL_METHODDEF + _DECIMAL_CONTEXT_CREATE_DECIMAL_FROM_FLOAT_METHODDEF { NULL, NULL, 1 } }; @@ -5862,7 +7552,7 @@ static PyType_Slot context_slots[] = { {Py_tp_repr, context_repr}, {Py_tp_getattro, context_getattr}, {Py_tp_setattro, context_setattr}, - {Py_tp_doc, (void *)doc_context}, + {Py_tp_doc, (void *)context_init__doc__}, {Py_tp_methods, context_methods}, {Py_tp_getset, context_getsets}, {Py_tp_init, context_init}, @@ -5879,12 +7569,35 @@ static PyType_Spec context_spec = { }; +static PyObject * +decimal_getattr(PyObject *self, PyObject *args) +{ + PyObject *name; + if (!PyArg_UnpackTuple(args, "__getattr__", 1, 1, &name)) { + return NULL; + } + + if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "__version__")) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + 1) < 0) { + return NULL; + } + return PyUnicode_FromString(MPD_SPEC_VERSION); + } + + PyErr_Format(PyExc_AttributeError, "module 'decimal' has no attribute %R", name); + return NULL; +} + + static PyMethodDef _decimal_methods [] = { - { "getcontext", PyDec_GetCurrentContext, METH_NOARGS, doc_getcontext}, - { "setcontext", PyDec_SetCurrentContext, METH_O, doc_setcontext}, - { "localcontext", _PyCFunction_CAST(ctxmanager_new), METH_VARARGS|METH_KEYWORDS, doc_localcontext}, - { "IEEEContext", ieee_context, METH_O, doc_ieee_context}, + _DECIMAL_GETCONTEXT_METHODDEF + _DECIMAL_SETCONTEXT_METHODDEF + _DECIMAL_LOCALCONTEXT_METHODDEF + _DECIMAL_IEEECONTEXT_METHODDEF + {"__getattr__", decimal_getattr, METH_VARARGS, "Module __getattr__"}, { NULL, NULL, 1, NULL } }; @@ -6204,7 +7917,7 @@ _decimal_exec(PyObject *m) } /* Add specification version number */ - CHECK_INT(PyModule_AddStringConstant(m, "__version__", "1.70")); + CHECK_INT(PyModule_AddStringConstant(m, "SPEC_VERSION", MPD_SPEC_VERSION)); CHECK_INT(PyModule_AddStringConstant(m, "__libmpdec_version__", mpd_version())); return 0; @@ -6319,7 +8032,7 @@ static struct PyModuleDef_Slot _decimal_slots[] = { static struct PyModuleDef _decimal_module = { PyModuleDef_HEAD_INIT, .m_name = "decimal", - .m_doc = doc__decimal, + .m_doc = "C decimal arithmetic module", .m_size = sizeof(decimal_state), .m_methods = _decimal_methods, .m_slots = _decimal_slots, diff --git a/Modules/_decimal/clinic/_decimal.c.h b/Modules/_decimal/clinic/_decimal.c.h new file mode 100644 index 00000000000..b09200845d1 --- /dev/null +++ b/Modules/_decimal/clinic/_decimal.c.h @@ -0,0 +1,6983 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_decimal_Context_Etiny__doc__, +"Etiny($self, /)\n" +"--\n" +"\n" +"Return a value equal to Emin - prec + 1.\n" +"\n" +"This is the minimum exponent value for subnormal results. When\n" +"underflow occurs, the exponent is set to Etiny."); + +#define _DECIMAL_CONTEXT_ETINY_METHODDEF \ + {"Etiny", (PyCFunction)_decimal_Context_Etiny, METH_NOARGS, _decimal_Context_Etiny__doc__}, + +static PyObject * +_decimal_Context_Etiny_impl(PyObject *self); + +static PyObject * +_decimal_Context_Etiny(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Context_Etiny_impl(self); +} + +PyDoc_STRVAR(_decimal_Context_Etop__doc__, +"Etop($self, /)\n" +"--\n" +"\n" +"Return a value equal to Emax - prec + 1.\n" +"\n" +"This is the maximum exponent if the _clamp field of the context is set\n" +"to 1 (IEEE clamp mode). Etop() must not be negative."); + +#define _DECIMAL_CONTEXT_ETOP_METHODDEF \ + {"Etop", (PyCFunction)_decimal_Context_Etop, METH_NOARGS, _decimal_Context_Etop__doc__}, + +static PyObject * +_decimal_Context_Etop_impl(PyObject *self); + +static PyObject * +_decimal_Context_Etop(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Context_Etop_impl(self); +} + +#if defined(CONFIG_32) + +PyDoc_STRVAR(_decimal_Context__unsafe_setprec__doc__, +"_unsafe_setprec($self, x, /)\n" +"--\n" +"\n"); + +#define _DECIMAL_CONTEXT__UNSAFE_SETPREC_METHODDEF \ + {"_unsafe_setprec", (PyCFunction)_decimal_Context__unsafe_setprec, METH_O, _decimal_Context__unsafe_setprec__doc__}, + +static PyObject * +_decimal_Context__unsafe_setprec_impl(PyObject *self, Py_ssize_t x); + +static PyObject * +_decimal_Context__unsafe_setprec(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t x; + + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + x = ival; + } + return_value = _decimal_Context__unsafe_setprec_impl(self, x); + +exit: + return return_value; +} + +#endif /* defined(CONFIG_32) */ + +#if defined(CONFIG_32) + +PyDoc_STRVAR(_decimal_Context__unsafe_setemin__doc__, +"_unsafe_setemin($self, x, /)\n" +"--\n" +"\n"); + +#define _DECIMAL_CONTEXT__UNSAFE_SETEMIN_METHODDEF \ + {"_unsafe_setemin", (PyCFunction)_decimal_Context__unsafe_setemin, METH_O, _decimal_Context__unsafe_setemin__doc__}, + +static PyObject * +_decimal_Context__unsafe_setemin_impl(PyObject *self, Py_ssize_t x); + +static PyObject * +_decimal_Context__unsafe_setemin(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t x; + + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + x = ival; + } + return_value = _decimal_Context__unsafe_setemin_impl(self, x); + +exit: + return return_value; +} + +#endif /* defined(CONFIG_32) */ + +#if defined(CONFIG_32) + +PyDoc_STRVAR(_decimal_Context__unsafe_setemax__doc__, +"_unsafe_setemax($self, x, /)\n" +"--\n" +"\n"); + +#define _DECIMAL_CONTEXT__UNSAFE_SETEMAX_METHODDEF \ + {"_unsafe_setemax", (PyCFunction)_decimal_Context__unsafe_setemax, METH_O, _decimal_Context__unsafe_setemax__doc__}, + +static PyObject * +_decimal_Context__unsafe_setemax_impl(PyObject *self, Py_ssize_t x); + +static PyObject * +_decimal_Context__unsafe_setemax(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t x; + + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + x = ival; + } + return_value = _decimal_Context__unsafe_setemax_impl(self, x); + +exit: + return return_value; +} + +#endif /* defined(CONFIG_32) */ + +PyDoc_STRVAR(_decimal_Context_clear_traps__doc__, +"clear_traps($self, /)\n" +"--\n" +"\n" +"Set all traps to False."); + +#define _DECIMAL_CONTEXT_CLEAR_TRAPS_METHODDEF \ + {"clear_traps", (PyCFunction)_decimal_Context_clear_traps, METH_NOARGS, _decimal_Context_clear_traps__doc__}, + +static PyObject * +_decimal_Context_clear_traps_impl(PyObject *self); + +static PyObject * +_decimal_Context_clear_traps(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Context_clear_traps_impl(self); +} + +PyDoc_STRVAR(_decimal_Context_clear_flags__doc__, +"clear_flags($self, /)\n" +"--\n" +"\n" +"Reset all flags to False."); + +#define _DECIMAL_CONTEXT_CLEAR_FLAGS_METHODDEF \ + {"clear_flags", (PyCFunction)_decimal_Context_clear_flags, METH_NOARGS, _decimal_Context_clear_flags__doc__}, + +static PyObject * +_decimal_Context_clear_flags_impl(PyObject *self); + +static PyObject * +_decimal_Context_clear_flags(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Context_clear_flags_impl(self); +} + +PyDoc_STRVAR(context_init__doc__, +"Context(prec=None, rounding=None, Emin=None, Emax=None, capitals=None,\n" +" clamp=None, flags=None, traps=None)\n" +"--\n" +"\n" +"Create context.\n" +"\n" +"The context affects almost all operations and controls rounding,\n" +"Over/Underflow, raising of exceptions and much more. A new context\n" +"can be constructed as follows:\n" +"\n" +" >>> c = Context(prec=28, Emin=-425000000, Emax=425000000,\n" +" ... rounding=ROUND_HALF_EVEN, capitals=1, clamp=1,\n" +" ... traps=[InvalidOperation, DivisionByZero, Overflow],\n" +" ... flags=[])\n" +" >>>"); + +static int +context_init_impl(PyObject *self, PyObject *prec, PyObject *rounding, + PyObject *emin, PyObject *emax, PyObject *capitals, + PyObject *clamp, PyObject *status, PyObject *traps); + +static int +context_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 8 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(prec), &_Py_ID(rounding), &_Py_ID(Emin), &_Py_ID(Emax), &_Py_ID(capitals), &_Py_ID(clamp), &_Py_ID(flags), &_Py_ID(traps), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"prec", "rounding", "Emin", "Emax", "capitals", "clamp", "flags", "traps", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "Context", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[8]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *prec = Py_None; + PyObject *rounding = Py_None; + PyObject *emin = Py_None; + PyObject *emax = Py_None; + PyObject *capitals = Py_None; + PyObject *clamp = Py_None; + PyObject *status = Py_None; + PyObject *traps = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + prec = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + rounding = fastargs[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + emin = fastargs[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[3]) { + emax = fastargs[3]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + capitals = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[5]) { + clamp = fastargs[5]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[6]) { + status = fastargs[6]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + traps = fastargs[7]; +skip_optional_pos: + return_value = context_init_impl(self, prec, rounding, emin, emax, capitals, clamp, status, traps); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_IEEEContext__doc__, +"IEEEContext($module, bits, /)\n" +"--\n" +"\n" +"Return a context, initialized as one of the IEEE interchange formats.\n" +"\n" +"The argument must be a multiple of 32 and less than\n" +"IEEE_CONTEXT_MAX_BITS."); + +#define _DECIMAL_IEEECONTEXT_METHODDEF \ + {"IEEEContext", (PyCFunction)_decimal_IEEEContext, METH_O, _decimal_IEEEContext__doc__}, + +static PyObject * +_decimal_IEEEContext_impl(PyObject *module, Py_ssize_t bits); + +static PyObject * +_decimal_IEEEContext(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t bits; + + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + bits = ival; + } + return_value = _decimal_IEEEContext_impl(module, bits); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"Return a duplicate of the context with all flags cleared."); + +#define _DECIMAL_CONTEXT_COPY_METHODDEF \ + {"copy", _PyCFunction_CAST(_decimal_Context_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_copy__doc__}, + +static PyObject * +_decimal_Context_copy_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Context_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); + return NULL; + } + return _decimal_Context_copy_impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Context___copy____doc__, +"__copy__($self, /)\n" +"--\n" +"\n"); + +#define _DECIMAL_CONTEXT___COPY___METHODDEF \ + {"__copy__", _PyCFunction_CAST(_decimal_Context___copy__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context___copy____doc__}, + +static PyObject * +_decimal_Context___copy___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Context___copy__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__copy__() takes no arguments"); + return NULL; + } + return _decimal_Context___copy___impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Context___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define _DECIMAL_CONTEXT___REDUCE___METHODDEF \ + {"__reduce__", _PyCFunction_CAST(_decimal_Context___reduce__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context___reduce____doc__}, + +static PyObject * +_decimal_Context___reduce___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Context___reduce__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__reduce__() takes no arguments"); + return NULL; + } + return _decimal_Context___reduce___impl(self, cls); +} + +PyDoc_STRVAR(_decimal_getcontext__doc__, +"getcontext($module, /)\n" +"--\n" +"\n" +"Get the current default context."); + +#define _DECIMAL_GETCONTEXT_METHODDEF \ + {"getcontext", (PyCFunction)_decimal_getcontext, METH_NOARGS, _decimal_getcontext__doc__}, + +static PyObject * +_decimal_getcontext_impl(PyObject *module); + +static PyObject * +_decimal_getcontext(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_getcontext_impl(module); +} + +PyDoc_STRVAR(_decimal_setcontext__doc__, +"setcontext($module, context, /)\n" +"--\n" +"\n" +"Set a new default context."); + +#define _DECIMAL_SETCONTEXT_METHODDEF \ + {"setcontext", (PyCFunction)_decimal_setcontext, METH_O, _decimal_setcontext__doc__}, + +PyDoc_STRVAR(_decimal_localcontext__doc__, +"localcontext($module, /, ctx=None, **kwargs)\n" +"--\n" +"\n" +"Return a context manager for a copy of the supplied context.\n" +"\n" +"That will set the default context to a copy of ctx on entry to the\n" +"with-statement and restore the previous default context when exiting\n" +"the with-statement. If no context is specified, a copy of the current\n" +"default context is used."); + +#define _DECIMAL_LOCALCONTEXT_METHODDEF \ + {"localcontext", _PyCFunction_CAST(_decimal_localcontext), METH_FASTCALL|METH_KEYWORDS, _decimal_localcontext__doc__}, + +static PyObject * +_decimal_localcontext_impl(PyObject *module, PyObject *local, PyObject *prec, + PyObject *rounding, PyObject *Emin, + PyObject *Emax, PyObject *capitals, + PyObject *clamp, PyObject *flags, PyObject *traps); + +static PyObject * +_decimal_localcontext(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(ctx), &_Py_ID(prec), &_Py_ID(rounding), &_Py_ID(Emin), &_Py_ID(Emax), &_Py_ID(capitals), &_Py_ID(clamp), &_Py_ID(flags), &_Py_ID(traps), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"ctx", "prec", "rounding", "Emin", "Emax", "capitals", "clamp", "flags", "traps", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "localcontext", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *local = Py_None; + PyObject *prec = Py_None; + PyObject *rounding = Py_None; + PyObject *Emin = Py_None; + PyObject *Emax = Py_None; + PyObject *capitals = Py_None; + PyObject *clamp = Py_None; + PyObject *flags = Py_None; + PyObject *traps = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + local = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + prec = args[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[2]) { + rounding = args[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[3]) { + Emin = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[4]) { + Emax = args[4]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[5]) { + capitals = args[5]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[6]) { + clamp = args[6]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[7]) { + flags = args[7]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + traps = args[8]; +skip_optional_kwonly: + return_value = _decimal_localcontext_impl(module, local, prec, rounding, Emin, Emax, capitals, clamp, flags, traps); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_from_float__doc__, +"from_float($type, f, /)\n" +"--\n" +"\n" +"Class method that converts a float to a decimal number, exactly.\n" +"\n" +"Since 0.1 is not exactly representable in binary floating point,\n" +"Decimal.from_float(0.1) is not the same as Decimal(\'0.1\').\n" +"\n" +" >>> Decimal.from_float(0.1)\n" +" Decimal(\'0.1000000000000000055511151231257827021181583404541015625\')\n" +" >>> Decimal.from_float(float(\'nan\'))\n" +" Decimal(\'NaN\')\n" +" >>> Decimal.from_float(float(\'inf\'))\n" +" Decimal(\'Infinity\')\n" +" >>> Decimal.from_float(float(\'-inf\'))\n" +" Decimal(\'-Infinity\')"); + +#define _DECIMAL_DECIMAL_FROM_FLOAT_METHODDEF \ + {"from_float", _PyCFunction_CAST(_decimal_Decimal_from_float), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, _decimal_Decimal_from_float__doc__}, + +static PyObject * +_decimal_Decimal_from_float_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *pyfloat); + +static PyObject * +_decimal_Decimal_from_float(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_float", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *pyfloat; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + pyfloat = args[0]; + return_value = _decimal_Decimal_from_float_impl((PyTypeObject *)type, cls, pyfloat); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_from_number__doc__, +"from_number($type, number, /)\n" +"--\n" +"\n" +"Class method that converts a real number to a decimal number, exactly.\n" +"\n" +" >>> Decimal.from_number(314) # int\n" +" Decimal(\'314\')\n" +" >>> Decimal.from_number(0.1) # float\n" +" Decimal(\'0.1000000000000000055511151231257827021181583404541015625\')\n" +" >>> Decimal.from_number(Decimal(\'3.14\')) # another decimal instance\n" +" Decimal(\'3.14\')"); + +#define _DECIMAL_DECIMAL_FROM_NUMBER_METHODDEF \ + {"from_number", _PyCFunction_CAST(_decimal_Decimal_from_number), METH_METHOD|METH_FASTCALL|METH_KEYWORDS|METH_CLASS, _decimal_Decimal_from_number__doc__}, + +static PyObject * +_decimal_Decimal_from_number_impl(PyTypeObject *type, PyTypeObject *cls, + PyObject *number); + +static PyObject * +_decimal_Decimal_from_number(PyObject *type, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "from_number", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *number; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + number = args[0]; + return_value = _decimal_Decimal_from_number_impl((PyTypeObject *)type, cls, number); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_create_decimal_from_float__doc__, +"create_decimal_from_float($self, f, /)\n" +"--\n" +"\n" +"Create a new Decimal instance from float f.\n" +"\n" +"Unlike the Decimal.from_float() class method, this function observes\n" +"the context limits."); + +#define _DECIMAL_CONTEXT_CREATE_DECIMAL_FROM_FLOAT_METHODDEF \ + {"create_decimal_from_float", _PyCFunction_CAST(_decimal_Context_create_decimal_from_float), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_create_decimal_from_float__doc__}, + +static PyObject * +_decimal_Context_create_decimal_from_float_impl(PyObject *context, + PyTypeObject *cls, + PyObject *f); + +static PyObject * +_decimal_Context_create_decimal_from_float(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "create_decimal_from_float", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *f; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + f = args[0]; + return_value = _decimal_Context_create_decimal_from_float_impl(context, cls, f); + +exit: + return return_value; +} + +PyDoc_STRVAR(dec_new__doc__, +"Decimal(value=\'0\', context=None)\n" +"--\n" +"\n" +"Construct a new Decimal object.\n" +"\n" +"value can be an integer, string, tuple, or another Decimal object. If\n" +"no value is given, return Decimal(\'0\'). The context does not affect\n" +"the conversion and is only passed to determine if the InvalidOperation\n" +"trap is active."); + +static PyObject * +dec_new_impl(PyTypeObject *type, PyObject *value, PyObject *context); + +static PyObject * +dec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(value), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"value", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "Decimal", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *value = NULL; + PyObject *context = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + value = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = fastargs[1]; +skip_optional_pos: + return_value = dec_new_impl(type, value, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_create_decimal__doc__, +"create_decimal($self, num=\'0\', /)\n" +"--\n" +"\n" +"Create a new Decimal instance from num, using self as the context.\n" +"\n" +"Unlike the Decimal constructor, this function observes the context\n" +"limits."); + +#define _DECIMAL_CONTEXT_CREATE_DECIMAL_METHODDEF \ + {"create_decimal", _PyCFunction_CAST(_decimal_Context_create_decimal), METH_FASTCALL, _decimal_Context_create_decimal__doc__}, + +static PyObject * +_decimal_Context_create_decimal_impl(PyObject *context, PyObject *num); + +static PyObject * +_decimal_Context_create_decimal(PyObject *context, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *num = NULL; + + if (!_PyArg_CheckPositional("create_decimal", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + num = args[0]; +skip_optional: + return_value = _decimal_Context_create_decimal_impl(context, num); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal___format____doc__, +"__format__($self, format_spec, override=<unrepresentable>, /)\n" +"--\n" +"\n" +"Formats the Decimal according to format_spec."); + +#define _DECIMAL_DECIMAL___FORMAT___METHODDEF \ + {"__format__", _PyCFunction_CAST(_decimal_Decimal___format__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal___format____doc__}, + +static PyObject * +_decimal_Decimal___format___impl(PyObject *dec, PyTypeObject *cls, + PyObject *fmtarg, PyObject *override); + +static PyObject * +_decimal_Decimal___format__(PyObject *dec, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "__format__", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *fmtarg; + PyObject *override = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("__format__", "argument 1", "str", args[0]); + goto exit; + } + fmtarg = args[0]; + if (nargs < 2) { + goto skip_optional_posonly; + } + override = args[1]; +skip_optional_posonly: + return_value = _decimal_Decimal___format___impl(dec, cls, fmtarg, override); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_as_integer_ratio__doc__, +"as_integer_ratio($self, /)\n" +"--\n" +"\n" +"Return a pair of integers whose ratio is exactly equal to the original.\n" +"\n" +"The ratio is in lowest terms and with a positive denominator.\n" +"Raise OverflowError on infinities and a ValueError on NaNs."); + +#define _DECIMAL_DECIMAL_AS_INTEGER_RATIO_METHODDEF \ + {"as_integer_ratio", _PyCFunction_CAST(_decimal_Decimal_as_integer_ratio), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_as_integer_ratio__doc__}, + +static PyObject * +_decimal_Decimal_as_integer_ratio_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Decimal_as_integer_ratio(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "as_integer_ratio() takes no arguments"); + return NULL; + } + return _decimal_Decimal_as_integer_ratio_impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Decimal_to_integral_value__doc__, +"to_integral_value($self, /, rounding=None, context=None)\n" +"--\n" +"\n" +"Round to the nearest integer without signaling Inexact or Rounded.\n" +"\n" +"The rounding mode is determined by the rounding parameter if given,\n" +"else by the given context. If neither parameter is given, then the\n" +"rounding mode of the current default context is used."); + +#define _DECIMAL_DECIMAL_TO_INTEGRAL_VALUE_METHODDEF \ + {"to_integral_value", _PyCFunction_CAST(_decimal_Decimal_to_integral_value), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_integral_value__doc__}, + +static PyObject * +_decimal_Decimal_to_integral_value_impl(PyObject *self, PyTypeObject *cls, + PyObject *rounding, + PyObject *context); + +static PyObject * +_decimal_Decimal_to_integral_value(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(rounding), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"rounding", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral_value", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *rounding = Py_None; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + rounding = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_to_integral_value_impl(self, cls, rounding, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_to_integral__doc__, +"to_integral($self, /, rounding=None, context=None)\n" +"--\n" +"\n" +"Identical to the to_integral_value() method.\n" +"\n" +"The to_integral() name has been kept for compatibility with older\n" +"versions."); + +#define _DECIMAL_DECIMAL_TO_INTEGRAL_METHODDEF \ + {"to_integral", _PyCFunction_CAST(_decimal_Decimal_to_integral), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_integral__doc__}, + +static PyObject * +_decimal_Decimal_to_integral_impl(PyObject *self, PyTypeObject *cls, + PyObject *rounding, PyObject *context); + +static PyObject * +_decimal_Decimal_to_integral(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(rounding), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"rounding", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *rounding = Py_None; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + rounding = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_to_integral_impl(self, cls, rounding, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_to_integral_exact__doc__, +"to_integral_exact($self, /, rounding=None, context=None)\n" +"--\n" +"\n" +"Round to the nearest integer.\n" +"\n" +"Decimal.to_integral_exact() signals Inexact or Rounded as appropriate\n" +"if rounding occurs. The rounding mode is determined by the rounding\n" +"parameter if given, else by the given context. If neither parameter is\n" +"given, then the rounding mode of the current default context is used."); + +#define _DECIMAL_DECIMAL_TO_INTEGRAL_EXACT_METHODDEF \ + {"to_integral_exact", _PyCFunction_CAST(_decimal_Decimal_to_integral_exact), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_integral_exact__doc__}, + +static PyObject * +_decimal_Decimal_to_integral_exact_impl(PyObject *self, PyTypeObject *cls, + PyObject *rounding, + PyObject *context); + +static PyObject * +_decimal_Decimal_to_integral_exact(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(rounding), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"rounding", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral_exact", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *rounding = Py_None; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + rounding = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_to_integral_exact_impl(self, cls, rounding, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal___round____doc__, +"__round__($self, ndigits=<unrepresentable>, /)\n" +"--\n" +"\n" +"Return the Integral closest to self, rounding half toward even."); + +#define _DECIMAL_DECIMAL___ROUND___METHODDEF \ + {"__round__", _PyCFunction_CAST(_decimal_Decimal___round__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal___round____doc__}, + +static PyObject * +_decimal_Decimal___round___impl(PyObject *self, PyTypeObject *cls, + PyObject *ndigits); + +static PyObject * +_decimal_Decimal___round__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "__round__", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *ndigits = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 1) { + goto skip_optional_posonly; + } + ndigits = args[0]; +skip_optional_posonly: + return_value = _decimal_Decimal___round___impl(self, cls, ndigits); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_as_tuple__doc__, +"as_tuple($self, /)\n" +"--\n" +"\n" +"Return a tuple representation of the number."); + +#define _DECIMAL_DECIMAL_AS_TUPLE_METHODDEF \ + {"as_tuple", _PyCFunction_CAST(_decimal_Decimal_as_tuple), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_as_tuple__doc__}, + +static PyObject * +_decimal_Decimal_as_tuple_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Decimal_as_tuple(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "as_tuple() takes no arguments"); + return NULL; + } + return _decimal_Decimal_as_tuple_impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Decimal_exp__doc__, +"exp($self, /, context=None)\n" +"--\n" +"\n" +"Return the value of the (natural) exponential function e**x.\n" +"\n" +"The function always uses the ROUND_HALF_EVEN mode and the result is\n" +"correctly rounded."); + +#define _DECIMAL_DECIMAL_EXP_METHODDEF \ + {"exp", _PyCFunction_CAST(_decimal_Decimal_exp), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_exp__doc__}, + +static PyObject * +_decimal_Decimal_exp_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_exp(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "exp", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_exp_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_ln__doc__, +"ln($self, /, context=None)\n" +"--\n" +"\n" +"Return the natural (base e) logarithm of the operand.\n" +"\n" +"The function always uses the ROUND_HALF_EVEN mode and the result is\n" +"correctly rounded."); + +#define _DECIMAL_DECIMAL_LN_METHODDEF \ + {"ln", _PyCFunction_CAST(_decimal_Decimal_ln), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_ln__doc__}, + +static PyObject * +_decimal_Decimal_ln_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_ln(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "ln", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_ln_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_log10__doc__, +"log10($self, /, context=None)\n" +"--\n" +"\n" +"Return the base ten logarithm of the operand.\n" +"\n" +"The function always uses the ROUND_HALF_EVEN mode and the result is\n" +"correctly rounded."); + +#define _DECIMAL_DECIMAL_LOG10_METHODDEF \ + {"log10", _PyCFunction_CAST(_decimal_Decimal_log10), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_log10__doc__}, + +static PyObject * +_decimal_Decimal_log10_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_log10(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "log10", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_log10_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_next_minus__doc__, +"next_minus($self, /, context=None)\n" +"--\n" +"\n" +"Returns the largest representable number smaller than itself."); + +#define _DECIMAL_DECIMAL_NEXT_MINUS_METHODDEF \ + {"next_minus", _PyCFunction_CAST(_decimal_Decimal_next_minus), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_next_minus__doc__}, + +static PyObject * +_decimal_Decimal_next_minus_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_next_minus(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "next_minus", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_next_minus_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_next_plus__doc__, +"next_plus($self, /, context=None)\n" +"--\n" +"\n" +"Returns the smallest representable number larger than itself."); + +#define _DECIMAL_DECIMAL_NEXT_PLUS_METHODDEF \ + {"next_plus", _PyCFunction_CAST(_decimal_Decimal_next_plus), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_next_plus__doc__}, + +static PyObject * +_decimal_Decimal_next_plus_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_next_plus(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "next_plus", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_next_plus_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_normalize__doc__, +"normalize($self, /, context=None)\n" +"--\n" +"\n" +"Normalize the number by stripping trailing 0s\n" +"\n" +"This also change anything equal to 0 to 0e0. Used for producing\n" +"canonical values for members of an equivalence class. For example,\n" +"Decimal(\'32.100\') and Decimal(\'0.321000e+2\') both normalize to\n" +"the equivalent value Decimal(\'32.1\')."); + +#define _DECIMAL_DECIMAL_NORMALIZE_METHODDEF \ + {"normalize", _PyCFunction_CAST(_decimal_Decimal_normalize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_normalize__doc__}, + +static PyObject * +_decimal_Decimal_normalize_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_normalize(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "normalize", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_normalize_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_sqrt__doc__, +"sqrt($self, /, context=None)\n" +"--\n" +"\n" +"Return the square root of the argument to full precision.\n" +"\n" +"The result is correctly rounded using the ROUND_HALF_EVEN rounding mode."); + +#define _DECIMAL_DECIMAL_SQRT_METHODDEF \ + {"sqrt", _PyCFunction_CAST(_decimal_Decimal_sqrt), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_sqrt__doc__}, + +static PyObject * +_decimal_Decimal_sqrt_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_sqrt(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sqrt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_sqrt_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_compare__doc__, +"compare($self, /, other, context=None)\n" +"--\n" +"\n" +"Compare self to other.\n" +"\n" +"Return a decimal value:\n" +"\n" +" a or b is a NaN ==> Decimal(\'NaN\')\n" +" a < b ==> Decimal(\'-1\')\n" +" a == b ==> Decimal(\'0\')\n" +" a > b ==> Decimal(\'1\')"); + +#define _DECIMAL_DECIMAL_COMPARE_METHODDEF \ + {"compare", _PyCFunction_CAST(_decimal_Decimal_compare), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_compare__doc__}, + +static PyObject * +_decimal_Decimal_compare_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_compare(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compare", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_compare_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_compare_signal__doc__, +"compare_signal($self, /, other, context=None)\n" +"--\n" +"\n" +"Identical to compare, except that all NaNs signal."); + +#define _DECIMAL_DECIMAL_COMPARE_SIGNAL_METHODDEF \ + {"compare_signal", _PyCFunction_CAST(_decimal_Decimal_compare_signal), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_compare_signal__doc__}, + +static PyObject * +_decimal_Decimal_compare_signal_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_compare_signal(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compare_signal", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_compare_signal_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_max__doc__, +"max($self, /, other, context=None)\n" +"--\n" +"\n" +"Maximum of self and other.\n" +"\n" +"If one operand is a quiet NaN and the other is numeric, the numeric\n" +"operand is returned."); + +#define _DECIMAL_DECIMAL_MAX_METHODDEF \ + {"max", _PyCFunction_CAST(_decimal_Decimal_max), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_max__doc__}, + +static PyObject * +_decimal_Decimal_max_impl(PyObject *self, PyTypeObject *cls, PyObject *other, + PyObject *context); + +static PyObject * +_decimal_Decimal_max(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "max", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_max_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_max_mag__doc__, +"max_mag($self, /, other, context=None)\n" +"--\n" +"\n" +"As the max() method, but compares the absolute values of the operands."); + +#define _DECIMAL_DECIMAL_MAX_MAG_METHODDEF \ + {"max_mag", _PyCFunction_CAST(_decimal_Decimal_max_mag), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_max_mag__doc__}, + +static PyObject * +_decimal_Decimal_max_mag_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_max_mag(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "max_mag", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_max_mag_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_min__doc__, +"min($self, /, other, context=None)\n" +"--\n" +"\n" +"Minimum of self and other.\n" +"\n" +"If one operand is a quiet NaN and the other is numeric, the numeric\n" +"operand is returned."); + +#define _DECIMAL_DECIMAL_MIN_METHODDEF \ + {"min", _PyCFunction_CAST(_decimal_Decimal_min), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_min__doc__}, + +static PyObject * +_decimal_Decimal_min_impl(PyObject *self, PyTypeObject *cls, PyObject *other, + PyObject *context); + +static PyObject * +_decimal_Decimal_min(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "min", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_min_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_min_mag__doc__, +"min_mag($self, /, other, context=None)\n" +"--\n" +"\n" +"As the min() method, but compares the absolute values of the operands."); + +#define _DECIMAL_DECIMAL_MIN_MAG_METHODDEF \ + {"min_mag", _PyCFunction_CAST(_decimal_Decimal_min_mag), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_min_mag__doc__}, + +static PyObject * +_decimal_Decimal_min_mag_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_min_mag(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "min_mag", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_min_mag_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_next_toward__doc__, +"next_toward($self, /, other, context=None)\n" +"--\n" +"\n" +"Returns the number closest to self, in the direction towards other.\n" +"\n" +"If the two operands are unequal, return the number closest to the first\n" +"operand in the direction of the second operand. If both operands are\n" +"numerically equal, return a copy of the first operand with the sign set\n" +"to be the same as the sign of the second operand."); + +#define _DECIMAL_DECIMAL_NEXT_TOWARD_METHODDEF \ + {"next_toward", _PyCFunction_CAST(_decimal_Decimal_next_toward), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_next_toward__doc__}, + +static PyObject * +_decimal_Decimal_next_toward_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_next_toward(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "next_toward", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_next_toward_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_remainder_near__doc__, +"remainder_near($self, /, other, context=None)\n" +"--\n" +"\n" +"Return the remainder from dividing self by other.\n" +"\n" +"This differs from self % other in that the sign of the remainder is\n" +"chosen so as to minimize its absolute value. More precisely, the return\n" +"value is self - n * other where n is the integer nearest to the exact\n" +"value of self / other, and if two integers are equally near then the\n" +"even one is chosen.\n" +"\n" +"If the result is zero then its sign will be the sign of self."); + +#define _DECIMAL_DECIMAL_REMAINDER_NEAR_METHODDEF \ + {"remainder_near", _PyCFunction_CAST(_decimal_Decimal_remainder_near), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_remainder_near__doc__}, + +static PyObject * +_decimal_Decimal_remainder_near_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_remainder_near(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "remainder_near", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_remainder_near_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_fma__doc__, +"fma($self, /, other, third, context=None)\n" +"--\n" +"\n" +"Fused multiply-add.\n" +"\n" +"Return self*other+third with no rounding of the intermediate product\n" +"self*other.\n" +"\n" +" >>> Decimal(2).fma(3, 5)\n" +" Decimal(\'11\')"); + +#define _DECIMAL_DECIMAL_FMA_METHODDEF \ + {"fma", _PyCFunction_CAST(_decimal_Decimal_fma), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_fma__doc__}, + +static PyObject * +_decimal_Decimal_fma_impl(PyObject *self, PyTypeObject *cls, PyObject *other, + PyObject *third, PyObject *context); + +static PyObject * +_decimal_Decimal_fma(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(third), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "third", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fma", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *other; + PyObject *third; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + third = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[2]; +skip_optional_pos: + return_value = _decimal_Decimal_fma_impl(self, cls, other, third, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_is_canonical__doc__, +"is_canonical($self, /)\n" +"--\n" +"\n" +"Return True if the argument is canonical and False otherwise.\n" +"\n" +"Currently, a Decimal instance is always canonical, so this operation\n" +"always returns True."); + +#define _DECIMAL_DECIMAL_IS_CANONICAL_METHODDEF \ + {"is_canonical", (PyCFunction)_decimal_Decimal_is_canonical, METH_NOARGS, _decimal_Decimal_is_canonical__doc__}, + +static PyObject * +_decimal_Decimal_is_canonical_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_is_canonical(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_is_canonical_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_is_finite__doc__, +"is_finite($self, /)\n" +"--\n" +"\n" +"Return True if the argument is a finite number, and False otherwise."); + +#define _DECIMAL_DECIMAL_IS_FINITE_METHODDEF \ + {"is_finite", (PyCFunction)_decimal_Decimal_is_finite, METH_NOARGS, _decimal_Decimal_is_finite__doc__}, + +static PyObject * +_decimal_Decimal_is_finite_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_is_finite(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_is_finite_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_is_infinite__doc__, +"is_infinite($self, /)\n" +"--\n" +"\n" +"Return True if the argument is infinite, and False otherwise."); + +#define _DECIMAL_DECIMAL_IS_INFINITE_METHODDEF \ + {"is_infinite", (PyCFunction)_decimal_Decimal_is_infinite, METH_NOARGS, _decimal_Decimal_is_infinite__doc__}, + +static PyObject * +_decimal_Decimal_is_infinite_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_is_infinite(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_is_infinite_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_is_nan__doc__, +"is_nan($self, /)\n" +"--\n" +"\n" +"Return True if the argument is a (quiet or signaling) NaN, else False."); + +#define _DECIMAL_DECIMAL_IS_NAN_METHODDEF \ + {"is_nan", (PyCFunction)_decimal_Decimal_is_nan, METH_NOARGS, _decimal_Decimal_is_nan__doc__}, + +static PyObject * +_decimal_Decimal_is_nan_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_is_nan(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_is_nan_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_is_qnan__doc__, +"is_qnan($self, /)\n" +"--\n" +"\n" +"Return True if the argument is a quiet NaN, and False otherwise."); + +#define _DECIMAL_DECIMAL_IS_QNAN_METHODDEF \ + {"is_qnan", (PyCFunction)_decimal_Decimal_is_qnan, METH_NOARGS, _decimal_Decimal_is_qnan__doc__}, + +static PyObject * +_decimal_Decimal_is_qnan_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_is_qnan(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_is_qnan_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_is_snan__doc__, +"is_snan($self, /)\n" +"--\n" +"\n" +"Return True if the argument is a signaling NaN and False otherwise."); + +#define _DECIMAL_DECIMAL_IS_SNAN_METHODDEF \ + {"is_snan", (PyCFunction)_decimal_Decimal_is_snan, METH_NOARGS, _decimal_Decimal_is_snan__doc__}, + +static PyObject * +_decimal_Decimal_is_snan_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_is_snan(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_is_snan_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_is_signed__doc__, +"is_signed($self, /)\n" +"--\n" +"\n" +"Return True if the argument has a negative sign and False otherwise.\n" +"\n" +"Note that both zeros and NaNs can carry signs."); + +#define _DECIMAL_DECIMAL_IS_SIGNED_METHODDEF \ + {"is_signed", (PyCFunction)_decimal_Decimal_is_signed, METH_NOARGS, _decimal_Decimal_is_signed__doc__}, + +static PyObject * +_decimal_Decimal_is_signed_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_is_signed(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_is_signed_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_is_zero__doc__, +"is_zero($self, /)\n" +"--\n" +"\n" +"Return True if the argument is a zero and False otherwise."); + +#define _DECIMAL_DECIMAL_IS_ZERO_METHODDEF \ + {"is_zero", (PyCFunction)_decimal_Decimal_is_zero, METH_NOARGS, _decimal_Decimal_is_zero__doc__}, + +static PyObject * +_decimal_Decimal_is_zero_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_is_zero(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_is_zero_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_is_normal__doc__, +"is_normal($self, /, context=None)\n" +"--\n" +"\n" +"Return True if the argument is a normal number and False otherwise.\n" +"\n" +"Normal number is a finite nonzero number, which is not subnormal."); + +#define _DECIMAL_DECIMAL_IS_NORMAL_METHODDEF \ + {"is_normal", _PyCFunction_CAST(_decimal_Decimal_is_normal), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_is_normal__doc__}, + +static PyObject * +_decimal_Decimal_is_normal_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_is_normal(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_normal", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_is_normal_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_is_subnormal__doc__, +"is_subnormal($self, /, context=None)\n" +"--\n" +"\n" +"Return True if the argument is subnormal, and False otherwise.\n" +"\n" +"A number is subnormal if it is non-zero, finite, and has an adjusted\n" +"exponent less than Emin."); + +#define _DECIMAL_DECIMAL_IS_SUBNORMAL_METHODDEF \ + {"is_subnormal", _PyCFunction_CAST(_decimal_Decimal_is_subnormal), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_is_subnormal__doc__}, + +static PyObject * +_decimal_Decimal_is_subnormal_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_is_subnormal(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_subnormal", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_is_subnormal_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_adjusted__doc__, +"adjusted($self, /)\n" +"--\n" +"\n" +"Return the adjusted exponent (exp + digits - 1) of the number."); + +#define _DECIMAL_DECIMAL_ADJUSTED_METHODDEF \ + {"adjusted", (PyCFunction)_decimal_Decimal_adjusted, METH_NOARGS, _decimal_Decimal_adjusted__doc__}, + +static PyObject * +_decimal_Decimal_adjusted_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_adjusted(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_adjusted_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_canonical__doc__, +"canonical($self, /)\n" +"--\n" +"\n" +"Return the canonical encoding of the argument.\n" +"\n" +"Currently, the encoding of a Decimal instance is always canonical,\n" +"so this operation returns its argument unchanged."); + +#define _DECIMAL_DECIMAL_CANONICAL_METHODDEF \ + {"canonical", (PyCFunction)_decimal_Decimal_canonical, METH_NOARGS, _decimal_Decimal_canonical__doc__}, + +static PyObject * +_decimal_Decimal_canonical_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_canonical(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_canonical_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_conjugate__doc__, +"conjugate($self, /)\n" +"--\n" +"\n" +"Return self."); + +#define _DECIMAL_DECIMAL_CONJUGATE_METHODDEF \ + {"conjugate", (PyCFunction)_decimal_Decimal_conjugate, METH_NOARGS, _decimal_Decimal_conjugate__doc__}, + +static PyObject * +_decimal_Decimal_conjugate_impl(PyObject *self); + +static PyObject * +_decimal_Decimal_conjugate(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal_conjugate_impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal_radix__doc__, +"radix($self, /)\n" +"--\n" +"\n" +"Return Decimal(10).\n" +"\n" +"This is the radix (base) in which the Decimal class does\n" +"all its arithmetic. Included for compatibility with the specification."); + +#define _DECIMAL_DECIMAL_RADIX_METHODDEF \ + {"radix", _PyCFunction_CAST(_decimal_Decimal_radix), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_radix__doc__}, + +static PyObject * +_decimal_Decimal_radix_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Decimal_radix(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "radix() takes no arguments"); + return NULL; + } + return _decimal_Decimal_radix_impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Decimal_copy_abs__doc__, +"copy_abs($self, /)\n" +"--\n" +"\n" +"Return the absolute value of the argument.\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed."); + +#define _DECIMAL_DECIMAL_COPY_ABS_METHODDEF \ + {"copy_abs", _PyCFunction_CAST(_decimal_Decimal_copy_abs), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_copy_abs__doc__}, + +static PyObject * +_decimal_Decimal_copy_abs_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Decimal_copy_abs(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "copy_abs() takes no arguments"); + return NULL; + } + return _decimal_Decimal_copy_abs_impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Decimal_copy_negate__doc__, +"copy_negate($self, /)\n" +"--\n" +"\n" +"Return the negation of the argument.\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed."); + +#define _DECIMAL_DECIMAL_COPY_NEGATE_METHODDEF \ + {"copy_negate", _PyCFunction_CAST(_decimal_Decimal_copy_negate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_copy_negate__doc__}, + +static PyObject * +_decimal_Decimal_copy_negate_impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Decimal_copy_negate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "copy_negate() takes no arguments"); + return NULL; + } + return _decimal_Decimal_copy_negate_impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Decimal_logical_invert__doc__, +"logical_invert($self, /, context=None)\n" +"--\n" +"\n" +"Invert all its digits.\n" +"\n" +"The self must be logical number."); + +#define _DECIMAL_DECIMAL_LOGICAL_INVERT_METHODDEF \ + {"logical_invert", _PyCFunction_CAST(_decimal_Decimal_logical_invert), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_logical_invert__doc__}, + +static PyObject * +_decimal_Decimal_logical_invert_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_logical_invert(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logical_invert", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_logical_invert_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_logb__doc__, +"logb($self, /, context=None)\n" +"--\n" +"\n" +"Return the adjusted exponent of the operand as a Decimal instance.\n" +"\n" +"If the operand is a zero, then Decimal(\'-Infinity\') is returned and the\n" +"DivisionByZero condition is raised. If the operand is an infinity then\n" +"Decimal(\'Infinity\') is returned."); + +#define _DECIMAL_DECIMAL_LOGB_METHODDEF \ + {"logb", _PyCFunction_CAST(_decimal_Decimal_logb), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_logb__doc__}, + +static PyObject * +_decimal_Decimal_logb_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_logb(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logb", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_logb_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_number_class__doc__, +"number_class($self, /, context=None)\n" +"--\n" +"\n" +"Return a string describing the class of the operand.\n" +"\n" +"The returned value is one of the following ten strings:\n" +"\n" +" * \'-Infinity\', indicating that the operand is negative infinity.\n" +" * \'-Normal\', indicating that the operand is a negative normal\n" +" number.\n" +" * \'-Subnormal\', indicating that the operand is negative and\n" +" subnormal.\n" +" * \'-Zero\', indicating that the operand is a negative zero.\n" +" * \'+Zero\', indicating that the operand is a positive zero.\n" +" * \'+Subnormal\', indicating that the operand is positive and\n" +" subnormal.\n" +" * \'+Normal\', indicating that the operand is a positive normal\n" +" number.\n" +" * \'+Infinity\', indicating that the operand is positive infinity.\n" +" * \'NaN\', indicating that the operand is a quiet NaN (Not a Number).\n" +" * \'sNaN\', indicating that the operand is a signaling NaN."); + +#define _DECIMAL_DECIMAL_NUMBER_CLASS_METHODDEF \ + {"number_class", _PyCFunction_CAST(_decimal_Decimal_number_class), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_number_class__doc__}, + +static PyObject * +_decimal_Decimal_number_class_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_number_class(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "number_class", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_number_class_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_to_eng_string__doc__, +"to_eng_string($self, /, context=None)\n" +"--\n" +"\n" +"Convert to an engineering-type string.\n" +"\n" +"Engineering notation has an exponent which is a multiple of 3, so there\n" +"are up to 3 digits left of the decimal place. For example,\n" +"Decimal(\'123E+1\') is converted to Decimal(\'1.23E+3\').\n" +"\n" +"The value of context.capitals determines whether the exponent sign is\n" +"lower or upper case. Otherwise, the context does not affect the\n" +"operation."); + +#define _DECIMAL_DECIMAL_TO_ENG_STRING_METHODDEF \ + {"to_eng_string", _PyCFunction_CAST(_decimal_Decimal_to_eng_string), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_eng_string__doc__}, + +static PyObject * +_decimal_Decimal_to_eng_string_impl(PyObject *self, PyTypeObject *cls, + PyObject *context); + +static PyObject * +_decimal_Decimal_to_eng_string(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_eng_string", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + context = args[0]; +skip_optional_pos: + return_value = _decimal_Decimal_to_eng_string_impl(self, cls, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_compare_total__doc__, +"compare_total($self, /, other, context=None)\n" +"--\n" +"\n" +"Compare two operands using their abstract representation.\n" +"\n" +"Similar to the compare() method, but the result\n" +"gives a total ordering on Decimal instances. Two Decimal instances with\n" +"the same numeric value but different representations compare unequal\n" +"in this ordering:\n" +"\n" +" >>> Decimal(\'12.0\').compare_total(Decimal(\'12\'))\n" +" Decimal(\'-1\')\n" +"\n" +"Quiet and signaling NaNs are also included in the total ordering. The\n" +"result of this function is Decimal(\'0\') if both operands have the same\n" +"representation, Decimal(\'-1\') if the first operand is lower in the\n" +"total order than the second, and Decimal(\'1\') if the first operand is\n" +"higher in the total order than the second operand. See the\n" +"specification for details of the total order.\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed. As an exception, the C version\n" +"may raise InvalidOperation if the second operand cannot be converted\n" +"exactly."); + +#define _DECIMAL_DECIMAL_COMPARE_TOTAL_METHODDEF \ + {"compare_total", _PyCFunction_CAST(_decimal_Decimal_compare_total), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_compare_total__doc__}, + +static PyObject * +_decimal_Decimal_compare_total_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_compare_total(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compare_total", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_compare_total_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_compare_total_mag__doc__, +"compare_total_mag($self, /, other, context=None)\n" +"--\n" +"\n" +"As compare_total(), but ignores the sign of each operand.\n" +"\n" +"x.compare_total_mag(y) is equivalent to\n" +"x.copy_abs().compare_total(y.copy_abs()).\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed. As an exception, the C version\n" +"may raise InvalidOperation if the second operand cannot be converted\n" +"exactly."); + +#define _DECIMAL_DECIMAL_COMPARE_TOTAL_MAG_METHODDEF \ + {"compare_total_mag", _PyCFunction_CAST(_decimal_Decimal_compare_total_mag), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_compare_total_mag__doc__}, + +static PyObject * +_decimal_Decimal_compare_total_mag_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_compare_total_mag(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compare_total_mag", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_compare_total_mag_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_copy_sign__doc__, +"copy_sign($self, /, other, context=None)\n" +"--\n" +"\n" +"Return a copy of *self* with the sign of *other*.\n" +"\n" +"For example:\n" +"\n" +" >>> Decimal(\'2.3\').copy_sign(Decimal(\'-1.5\'))\n" +" Decimal(\'-2.3\')\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed. As an exception, the C version\n" +"may raise InvalidOperation if the second operand cannot be converted\n" +"exactly."); + +#define _DECIMAL_DECIMAL_COPY_SIGN_METHODDEF \ + {"copy_sign", _PyCFunction_CAST(_decimal_Decimal_copy_sign), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_copy_sign__doc__}, + +static PyObject * +_decimal_Decimal_copy_sign_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_copy_sign(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "copy_sign", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_copy_sign_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_same_quantum__doc__, +"same_quantum($self, /, other, context=None)\n" +"--\n" +"\n" +"Test whether self and other have the same exponent or both are NaN.\n" +"\n" +"This operation is unaffected by context and is quiet: no flags are\n" +"changed and no rounding is performed. As an exception, the C version\n" +"may raise InvalidOperation if the second operand cannot be converted\n" +"exactly."); + +#define _DECIMAL_DECIMAL_SAME_QUANTUM_METHODDEF \ + {"same_quantum", _PyCFunction_CAST(_decimal_Decimal_same_quantum), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_same_quantum__doc__}, + +static PyObject * +_decimal_Decimal_same_quantum_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_same_quantum(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "same_quantum", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_same_quantum_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_logical_and__doc__, +"logical_and($self, /, other, context=None)\n" +"--\n" +"\n" +"Applies an \'and\' operation between self and other\'s digits.\n" +"\n" +"Both self and other must be logical numbers."); + +#define _DECIMAL_DECIMAL_LOGICAL_AND_METHODDEF \ + {"logical_and", _PyCFunction_CAST(_decimal_Decimal_logical_and), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_logical_and__doc__}, + +static PyObject * +_decimal_Decimal_logical_and_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_logical_and(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logical_and", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_logical_and_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_logical_or__doc__, +"logical_or($self, /, other, context=None)\n" +"--\n" +"\n" +"Applies an \'or\' operation between self and other\'s digits.\n" +"\n" +"Both self and other must be logical numbers."); + +#define _DECIMAL_DECIMAL_LOGICAL_OR_METHODDEF \ + {"logical_or", _PyCFunction_CAST(_decimal_Decimal_logical_or), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_logical_or__doc__}, + +static PyObject * +_decimal_Decimal_logical_or_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_logical_or(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logical_or", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_logical_or_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_logical_xor__doc__, +"logical_xor($self, /, other, context=None)\n" +"--\n" +"\n" +"Applies an \'xor\' operation between self and other\'s digits.\n" +"\n" +"Both self and other must be logical numbers."); + +#define _DECIMAL_DECIMAL_LOGICAL_XOR_METHODDEF \ + {"logical_xor", _PyCFunction_CAST(_decimal_Decimal_logical_xor), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_logical_xor__doc__}, + +static PyObject * +_decimal_Decimal_logical_xor_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_logical_xor(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logical_xor", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_logical_xor_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_rotate__doc__, +"rotate($self, /, other, context=None)\n" +"--\n" +"\n" +"Returns a rotated copy of self\'s digits, value-of-other times.\n" +"\n" +"The second operand must be an integer in the range -precision through\n" +"precision. The absolute value of the second operand gives the number of\n" +"places to rotate. If the second operand is positive then rotation is to\n" +"the left; otherwise rotation is to the right. The coefficient of the\n" +"first operand is padded on the left with zeros to length precision if\n" +"necessary. The sign and exponent of the first operand are unchanged."); + +#define _DECIMAL_DECIMAL_ROTATE_METHODDEF \ + {"rotate", _PyCFunction_CAST(_decimal_Decimal_rotate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_rotate__doc__}, + +static PyObject * +_decimal_Decimal_rotate_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_rotate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "rotate", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_rotate_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_scaleb__doc__, +"scaleb($self, /, other, context=None)\n" +"--\n" +"\n" +"Return the first operand with the exponent adjusted the second.\n" +"\n" +"Equivalently, return the first operand multiplied by 10**other. The\n" +"second operand must be an integer."); + +#define _DECIMAL_DECIMAL_SCALEB_METHODDEF \ + {"scaleb", _PyCFunction_CAST(_decimal_Decimal_scaleb), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_scaleb__doc__}, + +static PyObject * +_decimal_Decimal_scaleb_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_scaleb(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "scaleb", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_scaleb_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_shift__doc__, +"shift($self, /, other, context=None)\n" +"--\n" +"\n" +"Returns a shifted copy of self\'s digits, value-of-other times.\n" +"\n" +"The second operand must be an integer in the range -precision through\n" +"precision. The absolute value of the second operand gives the number\n" +"of places to shift. If the second operand is positive, then the shift\n" +"is to the left; otherwise the shift is to the right. Digits shifted\n" +"into the coefficient are zeros. The sign and exponent of the first\n" +"operand are unchanged."); + +#define _DECIMAL_DECIMAL_SHIFT_METHODDEF \ + {"shift", _PyCFunction_CAST(_decimal_Decimal_shift), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_shift__doc__}, + +static PyObject * +_decimal_Decimal_shift_impl(PyObject *self, PyTypeObject *cls, + PyObject *other, PyObject *context); + +static PyObject * +_decimal_Decimal_shift(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(other), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"other", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "shift", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *other; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + other = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + context = args[1]; +skip_optional_pos: + return_value = _decimal_Decimal_shift_impl(self, cls, other, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal_quantize__doc__, +"quantize($self, /, exp, rounding=None, context=None)\n" +"--\n" +"\n" +"Quantize *self* so its exponent is the same as that of *exp*.\n" +"\n" +"Return a value equal to *self* after rounding, with the exponent\n" +"of *exp*.\n" +"\n" +" >>> Decimal(\'1.41421356\').quantize(Decimal(\'1.000\'))\n" +" Decimal(\'1.414\')\n" +"\n" +"Unlike other operations, if the length of the coefficient after the\n" +"quantize operation would be greater than precision, then an\n" +"InvalidOperation is signaled. This guarantees that, unless there\n" +"is an error condition, the quantized exponent is always equal to\n" +"that of the right-hand operand.\n" +"\n" +"Also unlike other operations, quantize never signals Underflow, even\n" +"if the result is subnormal and inexact.\n" +"\n" +"If the exponent of the second operand is larger than that of the first,\n" +"then rounding may be necessary. In this case, the rounding mode is\n" +"determined by the rounding argument if given, else by the given context\n" +"argument; if neither argument is given, the rounding mode of the\n" +"current thread\'s context is used."); + +#define _DECIMAL_DECIMAL_QUANTIZE_METHODDEF \ + {"quantize", _PyCFunction_CAST(_decimal_Decimal_quantize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_quantize__doc__}, + +static PyObject * +_decimal_Decimal_quantize_impl(PyObject *self, PyTypeObject *cls, + PyObject *w, PyObject *rounding, + PyObject *context); + +static PyObject * +_decimal_Decimal_quantize(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(exp), &_Py_ID(rounding), &_Py_ID(context), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"exp", "rounding", "context", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "quantize", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *w; + PyObject *rounding = Py_None; + PyObject *context = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + w = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + rounding = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + context = args[2]; +skip_optional_pos: + return_value = _decimal_Decimal_quantize_impl(self, cls, w, rounding, context); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Decimal___ceil____doc__, +"__ceil__($self, /)\n" +"--\n" +"\n" +"Return the ceiling as an Integral."); + +#define _DECIMAL_DECIMAL___CEIL___METHODDEF \ + {"__ceil__", _PyCFunction_CAST(_decimal_Decimal___ceil__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal___ceil____doc__}, + +static PyObject * +_decimal_Decimal___ceil___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Decimal___ceil__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__ceil__() takes no arguments"); + return NULL; + } + return _decimal_Decimal___ceil___impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Decimal___complex____doc__, +"__complex__($self, /)\n" +"--\n" +"\n" +"Convert this value to exact type complex."); + +#define _DECIMAL_DECIMAL___COMPLEX___METHODDEF \ + {"__complex__", (PyCFunction)_decimal_Decimal___complex__, METH_NOARGS, _decimal_Decimal___complex____doc__}, + +static PyObject * +_decimal_Decimal___complex___impl(PyObject *self); + +static PyObject * +_decimal_Decimal___complex__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal___complex___impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal___copy____doc__, +"__copy__($self, /)\n" +"--\n" +"\n"); + +#define _DECIMAL_DECIMAL___COPY___METHODDEF \ + {"__copy__", (PyCFunction)_decimal_Decimal___copy__, METH_NOARGS, _decimal_Decimal___copy____doc__}, + +static PyObject * +_decimal_Decimal___copy___impl(PyObject *self); + +static PyObject * +_decimal_Decimal___copy__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal___copy___impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal___deepcopy____doc__, +"__deepcopy__($self, memo, /)\n" +"--\n" +"\n"); + +#define _DECIMAL_DECIMAL___DEEPCOPY___METHODDEF \ + {"__deepcopy__", (PyCFunction)_decimal_Decimal___deepcopy__, METH_O, _decimal_Decimal___deepcopy____doc__}, + +PyDoc_STRVAR(_decimal_Decimal___floor____doc__, +"__floor__($self, /)\n" +"--\n" +"\n" +"Return the floor as an Integral."); + +#define _DECIMAL_DECIMAL___FLOOR___METHODDEF \ + {"__floor__", _PyCFunction_CAST(_decimal_Decimal___floor__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal___floor____doc__}, + +static PyObject * +_decimal_Decimal___floor___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Decimal___floor__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__floor__() takes no arguments"); + return NULL; + } + return _decimal_Decimal___floor___impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Decimal___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling."); + +#define _DECIMAL_DECIMAL___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)_decimal_Decimal___reduce__, METH_NOARGS, _decimal_Decimal___reduce____doc__}, + +static PyObject * +_decimal_Decimal___reduce___impl(PyObject *self); + +static PyObject * +_decimal_Decimal___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal___reduce___impl(self); +} + +PyDoc_STRVAR(_decimal_Decimal___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n" +"Returns size in memory, in bytes"); + +#define _DECIMAL_DECIMAL___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)_decimal_Decimal___sizeof__, METH_NOARGS, _decimal_Decimal___sizeof____doc__}, + +static PyObject * +_decimal_Decimal___sizeof___impl(PyObject *v); + +static PyObject * +_decimal_Decimal___sizeof__(PyObject *v, PyObject *Py_UNUSED(ignored)) +{ + return _decimal_Decimal___sizeof___impl(v); +} + +PyDoc_STRVAR(_decimal_Decimal___trunc____doc__, +"__trunc__($self, /)\n" +"--\n" +"\n" +"Return the Integral closest to x between 0 and x."); + +#define _DECIMAL_DECIMAL___TRUNC___METHODDEF \ + {"__trunc__", _PyCFunction_CAST(_decimal_Decimal___trunc__), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal___trunc____doc__}, + +static PyObject * +_decimal_Decimal___trunc___impl(PyObject *self, PyTypeObject *cls); + +static PyObject * +_decimal_Decimal___trunc__(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "__trunc__() takes no arguments"); + return NULL; + } + return _decimal_Decimal___trunc___impl(self, cls); +} + +PyDoc_STRVAR(_decimal_Context_abs__doc__, +"abs($self, x, /)\n" +"--\n" +"\n" +"Return the absolute value of x."); + +#define _DECIMAL_CONTEXT_ABS_METHODDEF \ + {"abs", _PyCFunction_CAST(_decimal_Context_abs), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_abs__doc__}, + +static PyObject * +_decimal_Context_abs_impl(PyObject *context, PyTypeObject *cls, PyObject *x); + +static PyObject * +_decimal_Context_abs(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "abs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_abs_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_exp__doc__, +"exp($self, x, /)\n" +"--\n" +"\n" +"Return e ** x."); + +#define _DECIMAL_CONTEXT_EXP_METHODDEF \ + {"exp", _PyCFunction_CAST(_decimal_Context_exp), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_exp__doc__}, + +static PyObject * +_decimal_Context_exp_impl(PyObject *context, PyTypeObject *cls, PyObject *x); + +static PyObject * +_decimal_Context_exp(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "exp", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_exp_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_ln__doc__, +"ln($self, x, /)\n" +"--\n" +"\n" +"Return the natural (base e) logarithm of x."); + +#define _DECIMAL_CONTEXT_LN_METHODDEF \ + {"ln", _PyCFunction_CAST(_decimal_Context_ln), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_ln__doc__}, + +static PyObject * +_decimal_Context_ln_impl(PyObject *context, PyTypeObject *cls, PyObject *x); + +static PyObject * +_decimal_Context_ln(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "ln", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_ln_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_log10__doc__, +"log10($self, x, /)\n" +"--\n" +"\n" +"Return the base 10 logarithm of x."); + +#define _DECIMAL_CONTEXT_LOG10_METHODDEF \ + {"log10", _PyCFunction_CAST(_decimal_Context_log10), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_log10__doc__}, + +static PyObject * +_decimal_Context_log10_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_log10(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "log10", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_log10_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_minus__doc__, +"minus($self, x, /)\n" +"--\n" +"\n" +"Minus corresponds to unary prefix minus in Python.\n" +"\n" +"This operation applies the context to the result."); + +#define _DECIMAL_CONTEXT_MINUS_METHODDEF \ + {"minus", _PyCFunction_CAST(_decimal_Context_minus), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_minus__doc__}, + +static PyObject * +_decimal_Context_minus_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_minus(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "minus", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_minus_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_next_minus__doc__, +"next_minus($self, x, /)\n" +"--\n" +"\n" +"Return the largest representable number smaller than x."); + +#define _DECIMAL_CONTEXT_NEXT_MINUS_METHODDEF \ + {"next_minus", _PyCFunction_CAST(_decimal_Context_next_minus), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_next_minus__doc__}, + +static PyObject * +_decimal_Context_next_minus_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_next_minus(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "next_minus", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_next_minus_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_next_plus__doc__, +"next_plus($self, x, /)\n" +"--\n" +"\n" +"Return the smallest representable number larger than x."); + +#define _DECIMAL_CONTEXT_NEXT_PLUS_METHODDEF \ + {"next_plus", _PyCFunction_CAST(_decimal_Context_next_plus), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_next_plus__doc__}, + +static PyObject * +_decimal_Context_next_plus_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_next_plus(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "next_plus", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_next_plus_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_normalize__doc__, +"normalize($self, x, /)\n" +"--\n" +"\n" +"Reduce x to its simplest form. Alias for reduce(x)."); + +#define _DECIMAL_CONTEXT_NORMALIZE_METHODDEF \ + {"normalize", _PyCFunction_CAST(_decimal_Context_normalize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_normalize__doc__}, + +static PyObject * +_decimal_Context_normalize_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_normalize(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "normalize", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_normalize_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_plus__doc__, +"plus($self, x, /)\n" +"--\n" +"\n" +"Plus corresponds to the unary prefix plus operator in Python.\n" +"\n" +"This operation applies the context to the result."); + +#define _DECIMAL_CONTEXT_PLUS_METHODDEF \ + {"plus", _PyCFunction_CAST(_decimal_Context_plus), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_plus__doc__}, + +static PyObject * +_decimal_Context_plus_impl(PyObject *context, PyTypeObject *cls, PyObject *x); + +static PyObject * +_decimal_Context_plus(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "plus", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_plus_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_to_integral_value__doc__, +"to_integral_value($self, x, /)\n" +"--\n" +"\n" +"Round to an integer."); + +#define _DECIMAL_CONTEXT_TO_INTEGRAL_VALUE_METHODDEF \ + {"to_integral_value", _PyCFunction_CAST(_decimal_Context_to_integral_value), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_to_integral_value__doc__}, + +static PyObject * +_decimal_Context_to_integral_value_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_to_integral_value(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral_value", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_to_integral_value_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_to_integral_exact__doc__, +"to_integral_exact($self, x, /)\n" +"--\n" +"\n" +"Round to an integer. Signal if the result is rounded or inexact."); + +#define _DECIMAL_CONTEXT_TO_INTEGRAL_EXACT_METHODDEF \ + {"to_integral_exact", _PyCFunction_CAST(_decimal_Context_to_integral_exact), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_to_integral_exact__doc__}, + +static PyObject * +_decimal_Context_to_integral_exact_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_to_integral_exact(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral_exact", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_to_integral_exact_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_to_integral__doc__, +"to_integral($self, x, /)\n" +"--\n" +"\n" +"Identical to to_integral_value(x)."); + +#define _DECIMAL_CONTEXT_TO_INTEGRAL_METHODDEF \ + {"to_integral", _PyCFunction_CAST(_decimal_Context_to_integral), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_to_integral__doc__}, + +static PyObject * +_decimal_Context_to_integral_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_to_integral(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_integral", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_to_integral_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_sqrt__doc__, +"sqrt($self, x, /)\n" +"--\n" +"\n" +"Square root of a non-negative number to context precision."); + +#define _DECIMAL_CONTEXT_SQRT_METHODDEF \ + {"sqrt", _PyCFunction_CAST(_decimal_Context_sqrt), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_sqrt__doc__}, + +static PyObject * +_decimal_Context_sqrt_impl(PyObject *context, PyTypeObject *cls, PyObject *x); + +static PyObject * +_decimal_Context_sqrt(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sqrt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_sqrt_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_add__doc__, +"add($self, x, y, /)\n" +"--\n" +"\n" +"Return the sum of x and y."); + +#define _DECIMAL_CONTEXT_ADD_METHODDEF \ + {"add", _PyCFunction_CAST(_decimal_Context_add), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_add__doc__}, + +static PyObject * +_decimal_Context_add_impl(PyObject *context, PyTypeObject *cls, PyObject *x, + PyObject *y); + +static PyObject * +_decimal_Context_add(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "add", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_add_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_compare__doc__, +"compare($self, x, y, /)\n" +"--\n" +"\n" +"Compare x and y numerically."); + +#define _DECIMAL_CONTEXT_COMPARE_METHODDEF \ + {"compare", _PyCFunction_CAST(_decimal_Context_compare), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_compare__doc__}, + +static PyObject * +_decimal_Context_compare_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_compare(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compare", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_compare_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_compare_signal__doc__, +"compare_signal($self, x, y, /)\n" +"--\n" +"\n" +"Compare x and y numerically. All NaNs signal."); + +#define _DECIMAL_CONTEXT_COMPARE_SIGNAL_METHODDEF \ + {"compare_signal", _PyCFunction_CAST(_decimal_Context_compare_signal), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_compare_signal__doc__}, + +static PyObject * +_decimal_Context_compare_signal_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_compare_signal(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compare_signal", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_compare_signal_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_divide__doc__, +"divide($self, x, y, /)\n" +"--\n" +"\n" +"Return x divided by y."); + +#define _DECIMAL_CONTEXT_DIVIDE_METHODDEF \ + {"divide", _PyCFunction_CAST(_decimal_Context_divide), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_divide__doc__}, + +static PyObject * +_decimal_Context_divide_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_divide(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "divide", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_divide_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_divide_int__doc__, +"divide_int($self, x, y, /)\n" +"--\n" +"\n" +"Return x divided by y, truncated to an integer."); + +#define _DECIMAL_CONTEXT_DIVIDE_INT_METHODDEF \ + {"divide_int", _PyCFunction_CAST(_decimal_Context_divide_int), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_divide_int__doc__}, + +static PyObject * +_decimal_Context_divide_int_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_divide_int(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "divide_int", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_divide_int_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_max__doc__, +"max($self, x, y, /)\n" +"--\n" +"\n" +"Compare the values numerically and return the maximum."); + +#define _DECIMAL_CONTEXT_MAX_METHODDEF \ + {"max", _PyCFunction_CAST(_decimal_Context_max), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_max__doc__}, + +static PyObject * +_decimal_Context_max_impl(PyObject *context, PyTypeObject *cls, PyObject *x, + PyObject *y); + +static PyObject * +_decimal_Context_max(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "max", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_max_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_max_mag__doc__, +"max_mag($self, x, y, /)\n" +"--\n" +"\n" +"Compare the values numerically with their sign ignored."); + +#define _DECIMAL_CONTEXT_MAX_MAG_METHODDEF \ + {"max_mag", _PyCFunction_CAST(_decimal_Context_max_mag), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_max_mag__doc__}, + +static PyObject * +_decimal_Context_max_mag_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_max_mag(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "max_mag", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_max_mag_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_min__doc__, +"min($self, x, y, /)\n" +"--\n" +"\n" +"Compare the values numerically and return the minimum."); + +#define _DECIMAL_CONTEXT_MIN_METHODDEF \ + {"min", _PyCFunction_CAST(_decimal_Context_min), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_min__doc__}, + +static PyObject * +_decimal_Context_min_impl(PyObject *context, PyTypeObject *cls, PyObject *x, + PyObject *y); + +static PyObject * +_decimal_Context_min(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "min", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_min_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_min_mag__doc__, +"min_mag($self, x, y, /)\n" +"--\n" +"\n" +"Compare the values numerically with their sign ignored."); + +#define _DECIMAL_CONTEXT_MIN_MAG_METHODDEF \ + {"min_mag", _PyCFunction_CAST(_decimal_Context_min_mag), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_min_mag__doc__}, + +static PyObject * +_decimal_Context_min_mag_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_min_mag(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "min_mag", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_min_mag_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_multiply__doc__, +"multiply($self, x, y, /)\n" +"--\n" +"\n" +"Return the product of x and y."); + +#define _DECIMAL_CONTEXT_MULTIPLY_METHODDEF \ + {"multiply", _PyCFunction_CAST(_decimal_Context_multiply), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_multiply__doc__}, + +static PyObject * +_decimal_Context_multiply_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_multiply(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "multiply", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_multiply_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_next_toward__doc__, +"next_toward($self, x, y, /)\n" +"--\n" +"\n" +"Return the number closest to x, in the direction towards y."); + +#define _DECIMAL_CONTEXT_NEXT_TOWARD_METHODDEF \ + {"next_toward", _PyCFunction_CAST(_decimal_Context_next_toward), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_next_toward__doc__}, + +static PyObject * +_decimal_Context_next_toward_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_next_toward(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "next_toward", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_next_toward_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_quantize__doc__, +"quantize($self, x, y, /)\n" +"--\n" +"\n" +"Return a value equal to x (rounded), having the exponent of y."); + +#define _DECIMAL_CONTEXT_QUANTIZE_METHODDEF \ + {"quantize", _PyCFunction_CAST(_decimal_Context_quantize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_quantize__doc__}, + +static PyObject * +_decimal_Context_quantize_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_quantize(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "quantize", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_quantize_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_remainder__doc__, +"remainder($self, x, y, /)\n" +"--\n" +"\n" +"Return the remainder from integer division.\n" +"\n" +"The sign of the result, if non-zero, is the same as that of the\n" +"original dividend."); + +#define _DECIMAL_CONTEXT_REMAINDER_METHODDEF \ + {"remainder", _PyCFunction_CAST(_decimal_Context_remainder), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_remainder__doc__}, + +static PyObject * +_decimal_Context_remainder_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_remainder(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "remainder", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_remainder_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_remainder_near__doc__, +"remainder_near($self, x, y, /)\n" +"--\n" +"\n" +"Return x - y * n.\n" +"\n" +"Here n is the integer nearest the exact value of x / y (if the result\n" +"is 0 then its sign will be the sign of x)."); + +#define _DECIMAL_CONTEXT_REMAINDER_NEAR_METHODDEF \ + {"remainder_near", _PyCFunction_CAST(_decimal_Context_remainder_near), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_remainder_near__doc__}, + +static PyObject * +_decimal_Context_remainder_near_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_remainder_near(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "remainder_near", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_remainder_near_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_subtract__doc__, +"subtract($self, x, y, /)\n" +"--\n" +"\n" +"Return the difference between x and y."); + +#define _DECIMAL_CONTEXT_SUBTRACT_METHODDEF \ + {"subtract", _PyCFunction_CAST(_decimal_Context_subtract), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_subtract__doc__}, + +static PyObject * +_decimal_Context_subtract_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_subtract(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "subtract", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_subtract_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_divmod__doc__, +"divmod($self, x, y, /)\n" +"--\n" +"\n" +"Return quotient and remainder of the division x / y."); + +#define _DECIMAL_CONTEXT_DIVMOD_METHODDEF \ + {"divmod", _PyCFunction_CAST(_decimal_Context_divmod), METH_FASTCALL, _decimal_Context_divmod__doc__}, + +static PyObject * +_decimal_Context_divmod_impl(PyObject *context, PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_divmod(PyObject *context, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *x; + PyObject *y; + + if (!_PyArg_CheckPositional("divmod", nargs, 2, 2)) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_divmod_impl(context, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_power__doc__, +"power($self, /, a, b, modulo=None)\n" +"--\n" +"\n" +"Compute a**b.\n" +"\n" +"If \'a\' is negative, then \'b\' must be integral. The result will be\n" +"inexact unless \'a\' is integral and the result is finite and can be\n" +"expressed exactly in \'precision\' digits. In the Python version the\n" +"result is always correctly rounded, in the C version the result is\n" +"almost always correctly rounded.\n" +"\n" +"If modulo is given, compute (a**b) % modulo. The following\n" +"restrictions hold:\n" +"\n" +" * all three arguments must be integral\n" +" * \'b\' must be nonnegative\n" +" * at least one of \'a\' or \'b\' must be nonzero\n" +" * modulo must be nonzero and less than 10**prec in absolute value"); + +#define _DECIMAL_CONTEXT_POWER_METHODDEF \ + {"power", _PyCFunction_CAST(_decimal_Context_power), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_power__doc__}, + +static PyObject * +_decimal_Context_power_impl(PyObject *context, PyTypeObject *cls, + PyObject *base, PyObject *exp, PyObject *mod); + +static PyObject * +_decimal_Context_power(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(modulo), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "modulo", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "power", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *base; + PyObject *exp; + PyObject *mod = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + base = args[0]; + exp = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + mod = args[2]; +skip_optional_pos: + return_value = _decimal_Context_power_impl(context, cls, base, exp, mod); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_fma__doc__, +"fma($self, x, y, z, /)\n" +"--\n" +"\n" +"Return x multiplied by y, plus z."); + +#define _DECIMAL_CONTEXT_FMA_METHODDEF \ + {"fma", _PyCFunction_CAST(_decimal_Context_fma), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_fma__doc__}, + +static PyObject * +_decimal_Context_fma_impl(PyObject *context, PyTypeObject *cls, PyObject *x, + PyObject *y, PyObject *z); + +static PyObject * +_decimal_Context_fma(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fma", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *x; + PyObject *y; + PyObject *z; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + z = args[2]; + return_value = _decimal_Context_fma_impl(context, cls, x, y, z); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_radix__doc__, +"radix($self, /)\n" +"--\n" +"\n" +"Return 10."); + +#define _DECIMAL_CONTEXT_RADIX_METHODDEF \ + {"radix", _PyCFunction_CAST(_decimal_Context_radix), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_radix__doc__}, + +static PyObject * +_decimal_Context_radix_impl(PyObject *context, PyTypeObject *cls); + +static PyObject * +_decimal_Context_radix(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "radix() takes no arguments"); + return NULL; + } + return _decimal_Context_radix_impl(context, cls); +} + +PyDoc_STRVAR(_decimal_Context_is_normal__doc__, +"is_normal($self, x, /)\n" +"--\n" +"\n" +"Return True if x is a normal number, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_NORMAL_METHODDEF \ + {"is_normal", _PyCFunction_CAST(_decimal_Context_is_normal), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_normal__doc__}, + +static PyObject * +_decimal_Context_is_normal_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_normal(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_normal", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_normal_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_subnormal__doc__, +"is_subnormal($self, x, /)\n" +"--\n" +"\n" +"Return True if x is subnormal, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_SUBNORMAL_METHODDEF \ + {"is_subnormal", _PyCFunction_CAST(_decimal_Context_is_subnormal), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_subnormal__doc__}, + +static PyObject * +_decimal_Context_is_subnormal_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_subnormal(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_subnormal", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_subnormal_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_finite__doc__, +"is_finite($self, x, /)\n" +"--\n" +"\n" +"Return True if x is finite, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_FINITE_METHODDEF \ + {"is_finite", _PyCFunction_CAST(_decimal_Context_is_finite), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_finite__doc__}, + +static PyObject * +_decimal_Context_is_finite_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_finite(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_finite", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_finite_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_infinite__doc__, +"is_infinite($self, x, /)\n" +"--\n" +"\n" +"Return True if x is infinite, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_INFINITE_METHODDEF \ + {"is_infinite", _PyCFunction_CAST(_decimal_Context_is_infinite), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_infinite__doc__}, + +static PyObject * +_decimal_Context_is_infinite_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_infinite(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_infinite", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_infinite_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_nan__doc__, +"is_nan($self, x, /)\n" +"--\n" +"\n" +"Return True if x is a qNaN or sNaN, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_NAN_METHODDEF \ + {"is_nan", _PyCFunction_CAST(_decimal_Context_is_nan), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_nan__doc__}, + +static PyObject * +_decimal_Context_is_nan_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_nan(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_nan", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_nan_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_qnan__doc__, +"is_qnan($self, x, /)\n" +"--\n" +"\n" +"Return True if x is a quiet NaN, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_QNAN_METHODDEF \ + {"is_qnan", _PyCFunction_CAST(_decimal_Context_is_qnan), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_qnan__doc__}, + +static PyObject * +_decimal_Context_is_qnan_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_qnan(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_qnan", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_qnan_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_snan__doc__, +"is_snan($self, x, /)\n" +"--\n" +"\n" +"Return True if x is a signaling NaN, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_SNAN_METHODDEF \ + {"is_snan", _PyCFunction_CAST(_decimal_Context_is_snan), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_snan__doc__}, + +static PyObject * +_decimal_Context_is_snan_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_snan(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_snan", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_snan_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_signed__doc__, +"is_signed($self, x, /)\n" +"--\n" +"\n" +"Return True if x is negative, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_SIGNED_METHODDEF \ + {"is_signed", _PyCFunction_CAST(_decimal_Context_is_signed), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_signed__doc__}, + +static PyObject * +_decimal_Context_is_signed_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_signed(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_signed", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_signed_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_zero__doc__, +"is_zero($self, x, /)\n" +"--\n" +"\n" +"Return True if x is a zero, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_ZERO_METHODDEF \ + {"is_zero", _PyCFunction_CAST(_decimal_Context_is_zero), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_zero__doc__}, + +static PyObject * +_decimal_Context_is_zero_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_zero(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_zero", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_zero_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_is_canonical__doc__, +"is_canonical($self, x, /)\n" +"--\n" +"\n" +"Return True if x is canonical, False otherwise."); + +#define _DECIMAL_CONTEXT_IS_CANONICAL_METHODDEF \ + {"is_canonical", _PyCFunction_CAST(_decimal_Context_is_canonical), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_is_canonical__doc__}, + +static PyObject * +_decimal_Context_is_canonical_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_is_canonical(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_canonical", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_is_canonical_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context__apply__doc__, +"_apply($self, x, /)\n" +"--\n" +"\n" +"Apply self to Decimal x."); + +#define _DECIMAL_CONTEXT__APPLY_METHODDEF \ + {"_apply", _PyCFunction_CAST(_decimal_Context__apply), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context__apply__doc__}, + +static PyObject * +_decimal_Context__apply_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context__apply(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_apply", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context__apply_impl(context, cls, x); + +exit: + return return_value; +} + +#if defined(EXTRA_FUNCTIONALITY) + +PyDoc_STRVAR(_decimal_Context_apply__doc__, +"apply($self, x, /)\n" +"--\n" +"\n" +"Apply self to Decimal x."); + +#define _DECIMAL_CONTEXT_APPLY_METHODDEF \ + {"apply", _PyCFunction_CAST(_decimal_Context_apply), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_apply__doc__}, + +static PyObject * +_decimal_Context_apply_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_apply(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "apply", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_apply_impl(context, cls, x); + +exit: + return return_value; +} + +#endif /* defined(EXTRA_FUNCTIONALITY) */ + +PyDoc_STRVAR(_decimal_Context_canonical__doc__, +"canonical($self, x, /)\n" +"--\n" +"\n" +"Return a new instance of x."); + +#define _DECIMAL_CONTEXT_CANONICAL_METHODDEF \ + {"canonical", _PyCFunction_CAST(_decimal_Context_canonical), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_canonical__doc__}, + +static PyObject * +_decimal_Context_canonical_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_canonical(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "canonical", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_canonical_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_copy_abs__doc__, +"copy_abs($self, x, /)\n" +"--\n" +"\n" +"Return a copy of x with the sign set to 0."); + +#define _DECIMAL_CONTEXT_COPY_ABS_METHODDEF \ + {"copy_abs", _PyCFunction_CAST(_decimal_Context_copy_abs), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_copy_abs__doc__}, + +static PyObject * +_decimal_Context_copy_abs_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_copy_abs(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "copy_abs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_copy_abs_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_copy_decimal__doc__, +"copy_decimal($self, x, /)\n" +"--\n" +"\n" +"Return a copy of Decimal x."); + +#define _DECIMAL_CONTEXT_COPY_DECIMAL_METHODDEF \ + {"copy_decimal", _PyCFunction_CAST(_decimal_Context_copy_decimal), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_copy_decimal__doc__}, + +static PyObject * +_decimal_Context_copy_decimal_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_copy_decimal(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "copy_decimal", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_copy_decimal_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_copy_negate__doc__, +"copy_negate($self, x, /)\n" +"--\n" +"\n" +"Return a copy of x with the sign inverted."); + +#define _DECIMAL_CONTEXT_COPY_NEGATE_METHODDEF \ + {"copy_negate", _PyCFunction_CAST(_decimal_Context_copy_negate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_copy_negate__doc__}, + +static PyObject * +_decimal_Context_copy_negate_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_copy_negate(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "copy_negate", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_copy_negate_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_logb__doc__, +"logb($self, x, /)\n" +"--\n" +"\n" +"Return the exponent of the magnitude of the operand\'s MSD."); + +#define _DECIMAL_CONTEXT_LOGB_METHODDEF \ + {"logb", _PyCFunction_CAST(_decimal_Context_logb), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_logb__doc__}, + +static PyObject * +_decimal_Context_logb_impl(PyObject *context, PyTypeObject *cls, PyObject *x); + +static PyObject * +_decimal_Context_logb(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logb", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_logb_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_logical_invert__doc__, +"logical_invert($self, x, /)\n" +"--\n" +"\n" +"Invert all the digits in the operand.\n" +"\n" +"The operand must be a logical number.\n" +"\n" +" >>> ExtendedContext.logical_invert(Decimal(\'0\'))\n" +" Decimal(\'111111111\')\n" +" >>> ExtendedContext.logical_invert(Decimal(\'1\'))\n" +" Decimal(\'111111110\')\n" +" >>> ExtendedContext.logical_invert(Decimal(\'111111111\'))\n" +" Decimal(\'0\')\n" +" >>> ExtendedContext.logical_invert(Decimal(\'101010101\'))\n" +" Decimal(\'10101010\')\n" +" >>> ExtendedContext.logical_invert(1101)\n" +" Decimal(\'111110010\')"); + +#define _DECIMAL_CONTEXT_LOGICAL_INVERT_METHODDEF \ + {"logical_invert", _PyCFunction_CAST(_decimal_Context_logical_invert), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_logical_invert__doc__}, + +static PyObject * +_decimal_Context_logical_invert_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_logical_invert(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logical_invert", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_logical_invert_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_number_class__doc__, +"number_class($self, x, /)\n" +"--\n" +"\n" +"Return an indication of the class of x."); + +#define _DECIMAL_CONTEXT_NUMBER_CLASS_METHODDEF \ + {"number_class", _PyCFunction_CAST(_decimal_Context_number_class), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_number_class__doc__}, + +static PyObject * +_decimal_Context_number_class_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_number_class(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "number_class", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_number_class_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_to_sci_string__doc__, +"to_sci_string($self, x, /)\n" +"--\n" +"\n" +"Convert a number to a string using scientific notation."); + +#define _DECIMAL_CONTEXT_TO_SCI_STRING_METHODDEF \ + {"to_sci_string", _PyCFunction_CAST(_decimal_Context_to_sci_string), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_to_sci_string__doc__}, + +static PyObject * +_decimal_Context_to_sci_string_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_to_sci_string(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_sci_string", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_to_sci_string_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_to_eng_string__doc__, +"to_eng_string($self, x, /)\n" +"--\n" +"\n" +"Convert a number to a string, using engineering notation."); + +#define _DECIMAL_CONTEXT_TO_ENG_STRING_METHODDEF \ + {"to_eng_string", _PyCFunction_CAST(_decimal_Context_to_eng_string), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_to_eng_string__doc__}, + +static PyObject * +_decimal_Context_to_eng_string_impl(PyObject *context, PyTypeObject *cls, + PyObject *x); + +static PyObject * +_decimal_Context_to_eng_string(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "to_eng_string", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *x; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + return_value = _decimal_Context_to_eng_string_impl(context, cls, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_compare_total__doc__, +"compare_total($self, x, y, /)\n" +"--\n" +"\n" +"Compare x and y using their abstract representation."); + +#define _DECIMAL_CONTEXT_COMPARE_TOTAL_METHODDEF \ + {"compare_total", _PyCFunction_CAST(_decimal_Context_compare_total), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_compare_total__doc__}, + +static PyObject * +_decimal_Context_compare_total_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_compare_total(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compare_total", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_compare_total_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_compare_total_mag__doc__, +"compare_total_mag($self, x, y, /)\n" +"--\n" +"\n" +"Compare x and y using their abstract representation, ignoring sign."); + +#define _DECIMAL_CONTEXT_COMPARE_TOTAL_MAG_METHODDEF \ + {"compare_total_mag", _PyCFunction_CAST(_decimal_Context_compare_total_mag), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_compare_total_mag__doc__}, + +static PyObject * +_decimal_Context_compare_total_mag_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_compare_total_mag(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compare_total_mag", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_compare_total_mag_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_copy_sign__doc__, +"copy_sign($self, x, y, /)\n" +"--\n" +"\n" +"Copy the sign from y to x."); + +#define _DECIMAL_CONTEXT_COPY_SIGN_METHODDEF \ + {"copy_sign", _PyCFunction_CAST(_decimal_Context_copy_sign), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_copy_sign__doc__}, + +static PyObject * +_decimal_Context_copy_sign_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_copy_sign(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "copy_sign", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_copy_sign_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_logical_and__doc__, +"logical_and($self, x, y, /)\n" +"--\n" +"\n" +"Applies the logical operation \'and\' between each operand\'s digits.\n" +"\n" +"The operands must be both logical numbers.\n" +"\n" +" >>> ExtendedContext.logical_and(Decimal(\'0\'), Decimal(\'0\'))\n" +" Decimal(\'0\')\n" +" >>> ExtendedContext.logical_and(Decimal(\'0\'), Decimal(\'1\'))\n" +" Decimal(\'0\')\n" +" >>> ExtendedContext.logical_and(Decimal(\'1\'), Decimal(\'0\'))\n" +" Decimal(\'0\')\n" +" >>> ExtendedContext.logical_and(Decimal(\'1\'), Decimal(\'1\'))\n" +" Decimal(\'1\')\n" +" >>> ExtendedContext.logical_and(Decimal(\'1100\'), Decimal(\'1010\'))\n" +" Decimal(\'1000\')\n" +" >>> ExtendedContext.logical_and(Decimal(\'1111\'), Decimal(\'10\'))\n" +" Decimal(\'10\')\n" +" >>> ExtendedContext.logical_and(110, 1101)\n" +" Decimal(\'100\')\n" +" >>> ExtendedContext.logical_and(Decimal(110), 1101)\n" +" Decimal(\'100\')\n" +" >>> ExtendedContext.logical_and(110, Decimal(1101))\n" +" Decimal(\'100\')"); + +#define _DECIMAL_CONTEXT_LOGICAL_AND_METHODDEF \ + {"logical_and", _PyCFunction_CAST(_decimal_Context_logical_and), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_logical_and__doc__}, + +static PyObject * +_decimal_Context_logical_and_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_logical_and(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logical_and", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_logical_and_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_logical_or__doc__, +"logical_or($self, x, y, /)\n" +"--\n" +"\n" +"Applies the logical operation \'or\' between each operand\'s digits.\n" +"\n" +"The operands must be both logical numbers.\n" +"\n" +" >>> ExtendedContext.logical_or(Decimal(\'0\'), Decimal(\'0\'))\n" +" Decimal(\'0\')\n" +" >>> ExtendedContext.logical_or(Decimal(\'0\'), Decimal(\'1\'))\n" +" Decimal(\'1\')\n" +" >>> ExtendedContext.logical_or(Decimal(\'1\'), Decimal(\'0\'))\n" +" Decimal(\'1\')\n" +" >>> ExtendedContext.logical_or(Decimal(\'1\'), Decimal(\'1\'))\n" +" Decimal(\'1\')\n" +" >>> ExtendedContext.logical_or(Decimal(\'1100\'), Decimal(\'1010\'))\n" +" Decimal(\'1110\')\n" +" >>> ExtendedContext.logical_or(Decimal(\'1110\'), Decimal(\'10\'))\n" +" Decimal(\'1110\')\n" +" >>> ExtendedContext.logical_or(110, 1101)\n" +" Decimal(\'1111\')\n" +" >>> ExtendedContext.logical_or(Decimal(110), 1101)\n" +" Decimal(\'1111\')\n" +" >>> ExtendedContext.logical_or(110, Decimal(1101))\n" +" Decimal(\'1111\')"); + +#define _DECIMAL_CONTEXT_LOGICAL_OR_METHODDEF \ + {"logical_or", _PyCFunction_CAST(_decimal_Context_logical_or), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_logical_or__doc__}, + +static PyObject * +_decimal_Context_logical_or_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_logical_or(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logical_or", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_logical_or_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_logical_xor__doc__, +"logical_xor($self, x, y, /)\n" +"--\n" +"\n" +"Applies the logical operation \'xor\' between each operand\'s digits.\n" +"\n" +"The operands must be both logical numbers.\n" +"\n" +" >>> ExtendedContext.logical_xor(Decimal(\'0\'), Decimal(\'0\'))\n" +" Decimal(\'0\')\n" +" >>> ExtendedContext.logical_xor(Decimal(\'0\'), Decimal(\'1\'))\n" +" Decimal(\'1\')\n" +" >>> ExtendedContext.logical_xor(Decimal(\'1\'), Decimal(\'0\'))\n" +" Decimal(\'1\')\n" +" >>> ExtendedContext.logical_xor(Decimal(\'1\'), Decimal(\'1\'))\n" +" Decimal(\'0\')\n" +" >>> ExtendedContext.logical_xor(Decimal(\'1100\'), Decimal(\'1010\'))\n" +" Decimal(\'110\')\n" +" >>> ExtendedContext.logical_xor(Decimal(\'1111\'), Decimal(\'10\'))\n" +" Decimal(\'1101\')\n" +" >>> ExtendedContext.logical_xor(110, 1101)\n" +" Decimal(\'1011\')\n" +" >>> ExtendedContext.logical_xor(Decimal(110), 1101)\n" +" Decimal(\'1011\')\n" +" >>> ExtendedContext.logical_xor(110, Decimal(1101))\n" +" Decimal(\'1011\')"); + +#define _DECIMAL_CONTEXT_LOGICAL_XOR_METHODDEF \ + {"logical_xor", _PyCFunction_CAST(_decimal_Context_logical_xor), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_logical_xor__doc__}, + +static PyObject * +_decimal_Context_logical_xor_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_logical_xor(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "logical_xor", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_logical_xor_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_rotate__doc__, +"rotate($self, x, y, /)\n" +"--\n" +"\n" +"Return a copy of x, rotated by y places."); + +#define _DECIMAL_CONTEXT_ROTATE_METHODDEF \ + {"rotate", _PyCFunction_CAST(_decimal_Context_rotate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_rotate__doc__}, + +static PyObject * +_decimal_Context_rotate_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_rotate(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "rotate", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_rotate_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_scaleb__doc__, +"scaleb($self, x, y, /)\n" +"--\n" +"\n" +"Return the first operand after adding the second value to its exp."); + +#define _DECIMAL_CONTEXT_SCALEB_METHODDEF \ + {"scaleb", _PyCFunction_CAST(_decimal_Context_scaleb), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_scaleb__doc__}, + +static PyObject * +_decimal_Context_scaleb_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_scaleb(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "scaleb", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_scaleb_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_shift__doc__, +"shift($self, x, y, /)\n" +"--\n" +"\n" +"Return a copy of x, shifted by y places."); + +#define _DECIMAL_CONTEXT_SHIFT_METHODDEF \ + {"shift", _PyCFunction_CAST(_decimal_Context_shift), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_shift__doc__}, + +static PyObject * +_decimal_Context_shift_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_shift(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "shift", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_shift_impl(context, cls, x, y); + +exit: + return return_value; +} + +PyDoc_STRVAR(_decimal_Context_same_quantum__doc__, +"same_quantum($self, x, y, /)\n" +"--\n" +"\n" +"Return True if the two operands have the same exponent."); + +#define _DECIMAL_CONTEXT_SAME_QUANTUM_METHODDEF \ + {"same_quantum", _PyCFunction_CAST(_decimal_Context_same_quantum), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_same_quantum__doc__}, + +static PyObject * +_decimal_Context_same_quantum_impl(PyObject *context, PyTypeObject *cls, + PyObject *x, PyObject *y); + +static PyObject * +_decimal_Context_same_quantum(PyObject *context, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "same_quantum", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *x; + PyObject *y; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + x = args[0]; + y = args[1]; + return_value = _decimal_Context_same_quantum_impl(context, cls, x, y); + +exit: + return return_value; +} + +#ifndef _DECIMAL_CONTEXT__UNSAFE_SETPREC_METHODDEF + #define _DECIMAL_CONTEXT__UNSAFE_SETPREC_METHODDEF +#endif /* !defined(_DECIMAL_CONTEXT__UNSAFE_SETPREC_METHODDEF) */ + +#ifndef _DECIMAL_CONTEXT__UNSAFE_SETEMIN_METHODDEF + #define _DECIMAL_CONTEXT__UNSAFE_SETEMIN_METHODDEF +#endif /* !defined(_DECIMAL_CONTEXT__UNSAFE_SETEMIN_METHODDEF) */ + +#ifndef _DECIMAL_CONTEXT__UNSAFE_SETEMAX_METHODDEF + #define _DECIMAL_CONTEXT__UNSAFE_SETEMAX_METHODDEF +#endif /* !defined(_DECIMAL_CONTEXT__UNSAFE_SETEMAX_METHODDEF) */ + +#ifndef _DECIMAL_CONTEXT_APPLY_METHODDEF + #define _DECIMAL_CONTEXT_APPLY_METHODDEF +#endif /* !defined(_DECIMAL_CONTEXT_APPLY_METHODDEF) */ +/*[clinic end generated code: output=b288181c82fdc9f1 input=a9049054013a1b77]*/ diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h deleted file mode 100644 index 77017a92252..00000000000 --- a/Modules/_decimal/docstrings.h +++ /dev/null @@ -1,894 +0,0 @@ -/* - * Copyright (c) 2001 Python Software Foundation. All Rights Reserved. - * Modified and extended by Stefan Krah. - */ - - -#ifndef DOCSTRINGS_H -#define DOCSTRINGS_H - - -#include "pymacro.h" - - -/******************************************************************************/ -/* Module */ -/******************************************************************************/ - - -PyDoc_STRVAR(doc__decimal, -"C decimal arithmetic module"); - -PyDoc_STRVAR(doc_getcontext, -"getcontext($module, /)\n--\n\n\ -Get the current default context.\n\ -\n"); - -PyDoc_STRVAR(doc_setcontext, -"setcontext($module, context, /)\n--\n\n\ -Set a new default context.\n\ -\n"); - -PyDoc_STRVAR(doc_localcontext, -"localcontext($module, /, ctx=None, **kwargs)\n--\n\n\ -Return a context manager that will set the default context to a copy of ctx\n\ -on entry to the with-statement and restore the previous default context when\n\ -exiting the with-statement. If no context is specified, a copy of the current\n\ -default context is used.\n\ -\n"); - -PyDoc_STRVAR(doc_ieee_context, -"IEEEContext($module, bits, /)\n--\n\n\ -Return a context object initialized to the proper values for one of the\n\ -IEEE interchange formats. The argument must be a multiple of 32 and less\n\ -than IEEE_CONTEXT_MAX_BITS.\n\ -\n"); - - -/******************************************************************************/ -/* Decimal Object and Methods */ -/******************************************************************************/ - -PyDoc_STRVAR(doc_decimal, -"Decimal(value=\"0\", context=None)\n--\n\n\ -Construct a new Decimal object. 'value' can be an integer, string, tuple,\n\ -or another Decimal object. If no value is given, return Decimal('0'). The\n\ -context does not affect the conversion and is only passed to determine if\n\ -the InvalidOperation trap is active.\n\ -\n"); - -PyDoc_STRVAR(doc_adjusted, -"adjusted($self, /)\n--\n\n\ -Return the adjusted exponent of the number. Defined as exp + digits - 1.\n\ -\n"); - -PyDoc_STRVAR(doc_as_tuple, -"as_tuple($self, /)\n--\n\n\ -Return a tuple representation of the number.\n\ -\n"); - -PyDoc_STRVAR(doc_as_integer_ratio, -"as_integer_ratio($self, /)\n--\n\n\ -Decimal.as_integer_ratio() -> (int, int)\n\ -\n\ -Return a pair of integers, whose ratio is exactly equal to the original\n\ -Decimal and with a positive denominator. The ratio is in lowest terms.\n\ -Raise OverflowError on infinities and a ValueError on NaNs.\n\ -\n"); - -PyDoc_STRVAR(doc_canonical, -"canonical($self, /)\n--\n\n\ -Return the canonical encoding of the argument. Currently, the encoding\n\ -of a Decimal instance is always canonical, so this operation returns its\n\ -argument unchanged.\n\ -\n"); - -PyDoc_STRVAR(doc_compare, -"compare($self, /, other, context=None)\n--\n\n\ -Compare self to other. Return a decimal value:\n\ -\n\ - a or b is a NaN ==> Decimal('NaN')\n\ - a < b ==> Decimal('-1')\n\ - a == b ==> Decimal('0')\n\ - a > b ==> Decimal('1')\n\ -\n"); - -PyDoc_STRVAR(doc_compare_signal, -"compare_signal($self, /, other, context=None)\n--\n\n\ -Identical to compare, except that all NaNs signal.\n\ -\n"); - -PyDoc_STRVAR(doc_compare_total, -"compare_total($self, /, other, context=None)\n--\n\n\ -Compare two operands using their abstract representation rather than\n\ -their numerical value. Similar to the compare() method, but the result\n\ -gives a total ordering on Decimal instances. Two Decimal instances with\n\ -the same numeric value but different representations compare unequal\n\ -in this ordering:\n\ -\n\ - >>> Decimal('12.0').compare_total(Decimal('12'))\n\ - Decimal('-1')\n\ -\n\ -Quiet and signaling NaNs are also included in the total ordering. The result\n\ -of this function is Decimal('0') if both operands have the same representation,\n\ -Decimal('-1') if the first operand is lower in the total order than the second,\n\ -and Decimal('1') if the first operand is higher in the total order than the\n\ -second operand. See the specification for details of the total order.\n\ -\n\ -This operation is unaffected by context and is quiet: no flags are changed\n\ -and no rounding is performed. As an exception, the C version may raise\n\ -InvalidOperation if the second operand cannot be converted exactly.\n\ -\n"); - -PyDoc_STRVAR(doc_compare_total_mag, -"compare_total_mag($self, /, other, context=None)\n--\n\n\ -Compare two operands using their abstract representation rather than their\n\ -value as in compare_total(), but ignoring the sign of each operand.\n\ -\n\ -x.compare_total_mag(y) is equivalent to x.copy_abs().compare_total(y.copy_abs()).\n\ -\n\ -This operation is unaffected by context and is quiet: no flags are changed\n\ -and no rounding is performed. As an exception, the C version may raise\n\ -InvalidOperation if the second operand cannot be converted exactly.\n\ -\n"); - -PyDoc_STRVAR(doc_conjugate, -"conjugate($self, /)\n--\n\n\ -Return self.\n\ -\n"); - -PyDoc_STRVAR(doc_copy_abs, -"copy_abs($self, /)\n--\n\n\ -Return the absolute value of the argument. This operation is unaffected by\n\ -context and is quiet: no flags are changed and no rounding is performed.\n\ -\n"); - -PyDoc_STRVAR(doc_copy_negate, -"copy_negate($self, /)\n--\n\n\ -Return the negation of the argument. This operation is unaffected by context\n\ -and is quiet: no flags are changed and no rounding is performed.\n\ -\n"); - -PyDoc_STRVAR(doc_copy_sign, -"copy_sign($self, /, other, context=None)\n--\n\n\ -Return a copy of the first operand with the sign set to be the same as the\n\ -sign of the second operand. For example:\n\ -\n\ - >>> Decimal('2.3').copy_sign(Decimal('-1.5'))\n\ - Decimal('-2.3')\n\ -\n\ -This operation is unaffected by context and is quiet: no flags are changed\n\ -and no rounding is performed. As an exception, the C version may raise\n\ -InvalidOperation if the second operand cannot be converted exactly.\n\ -\n"); - -PyDoc_STRVAR(doc_exp, -"exp($self, /, context=None)\n--\n\n\ -Return the value of the (natural) exponential function e**x at the given\n\ -number. The function always uses the ROUND_HALF_EVEN mode and the result\n\ -is correctly rounded.\n\ -\n"); - -PyDoc_STRVAR(doc_from_float, -"from_float($type, f, /)\n--\n\n\ -Class method that converts a float to a decimal number, exactly.\n\ -Since 0.1 is not exactly representable in binary floating point,\n\ -Decimal.from_float(0.1) is not the same as Decimal('0.1').\n\ -\n\ - >>> Decimal.from_float(0.1)\n\ - Decimal('0.1000000000000000055511151231257827021181583404541015625')\n\ - >>> Decimal.from_float(float('nan'))\n\ - Decimal('NaN')\n\ - >>> Decimal.from_float(float('inf'))\n\ - Decimal('Infinity')\n\ - >>> Decimal.from_float(float('-inf'))\n\ - Decimal('-Infinity')\n\ -\n\ -\n"); - -PyDoc_STRVAR(doc_from_number, -"from_number($type, number, /)\n--\n\n\ -Class method that converts a real number to a decimal number, exactly.\n\ -\n\ - >>> Decimal.from_number(314) # int\n\ - Decimal('314')\n\ - >>> Decimal.from_number(0.1) # float\n\ - Decimal('0.1000000000000000055511151231257827021181583404541015625')\n\ - >>> Decimal.from_number(Decimal('3.14')) # another decimal instance\n\ - Decimal('3.14')\n\ -\n\ -\n"); - -PyDoc_STRVAR(doc_fma, -"fma($self, /, other, third, context=None)\n--\n\n\ -Fused multiply-add. Return self*other+third with no rounding of the\n\ -intermediate product self*other.\n\ -\n\ - >>> Decimal(2).fma(3, 5)\n\ - Decimal('11')\n\ -\n\ -\n"); - -PyDoc_STRVAR(doc_is_canonical, -"is_canonical($self, /)\n--\n\n\ -Return True if the argument is canonical and False otherwise. Currently,\n\ -a Decimal instance is always canonical, so this operation always returns\n\ -True.\n\ -\n"); - -PyDoc_STRVAR(doc_is_finite, -"is_finite($self, /)\n--\n\n\ -Return True if the argument is a finite number, and False if the argument\n\ -is infinite or a NaN.\n\ -\n"); - -PyDoc_STRVAR(doc_is_infinite, -"is_infinite($self, /)\n--\n\n\ -Return True if the argument is either positive or negative infinity and\n\ -False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_is_nan, -"is_nan($self, /)\n--\n\n\ -Return True if the argument is a (quiet or signaling) NaN and False\n\ -otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_is_normal, -"is_normal($self, /, context=None)\n--\n\n\ -Return True if the argument is a normal finite non-zero number with an\n\ -adjusted exponent greater than or equal to Emin. Return False if the\n\ -argument is zero, subnormal, infinite or a NaN.\n\ -\n"); - -PyDoc_STRVAR(doc_is_qnan, -"is_qnan($self, /)\n--\n\n\ -Return True if the argument is a quiet NaN, and False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_is_signed, -"is_signed($self, /)\n--\n\n\ -Return True if the argument has a negative sign and False otherwise.\n\ -Note that both zeros and NaNs can carry signs.\n\ -\n"); - -PyDoc_STRVAR(doc_is_snan, -"is_snan($self, /)\n--\n\n\ -Return True if the argument is a signaling NaN and False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_is_subnormal, -"is_subnormal($self, /, context=None)\n--\n\n\ -Return True if the argument is subnormal, and False otherwise. A number is\n\ -subnormal if it is non-zero, finite, and has an adjusted exponent less\n\ -than Emin.\n\ -\n"); - -PyDoc_STRVAR(doc_is_zero, -"is_zero($self, /)\n--\n\n\ -Return True if the argument is a (positive or negative) zero and False\n\ -otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ln, -"ln($self, /, context=None)\n--\n\n\ -Return the natural (base e) logarithm of the operand. The function always\n\ -uses the ROUND_HALF_EVEN mode and the result is correctly rounded.\n\ -\n"); - -PyDoc_STRVAR(doc_log10, -"log10($self, /, context=None)\n--\n\n\ -Return the base ten logarithm of the operand. The function always uses the\n\ -ROUND_HALF_EVEN mode and the result is correctly rounded.\n\ -\n"); - -PyDoc_STRVAR(doc_logb, -"logb($self, /, context=None)\n--\n\n\ -For a non-zero number, return the adjusted exponent of the operand as a\n\ -Decimal instance. If the operand is a zero, then Decimal('-Infinity') is\n\ -returned and the DivisionByZero condition is raised. If the operand is\n\ -an infinity then Decimal('Infinity') is returned.\n\ -\n"); - -PyDoc_STRVAR(doc_logical_and, -"logical_and($self, /, other, context=None)\n--\n\n\ -Return the digit-wise 'and' of the two (logical) operands.\n\ -\n"); - -PyDoc_STRVAR(doc_logical_invert, -"logical_invert($self, /, context=None)\n--\n\n\ -Return the digit-wise inversion of the (logical) operand.\n\ -\n"); - -PyDoc_STRVAR(doc_logical_or, -"logical_or($self, /, other, context=None)\n--\n\n\ -Return the digit-wise 'or' of the two (logical) operands.\n\ -\n"); - -PyDoc_STRVAR(doc_logical_xor, -"logical_xor($self, /, other, context=None)\n--\n\n\ -Return the digit-wise 'exclusive or' of the two (logical) operands.\n\ -\n"); - -PyDoc_STRVAR(doc_max, -"max($self, /, other, context=None)\n--\n\n\ -Maximum of self and other. If one operand is a quiet NaN and the other is\n\ -numeric, the numeric operand is returned.\n\ -\n"); - -PyDoc_STRVAR(doc_max_mag, -"max_mag($self, /, other, context=None)\n--\n\n\ -Similar to the max() method, but the comparison is done using the absolute\n\ -values of the operands.\n\ -\n"); - -PyDoc_STRVAR(doc_min, -"min($self, /, other, context=None)\n--\n\n\ -Minimum of self and other. If one operand is a quiet NaN and the other is\n\ -numeric, the numeric operand is returned.\n\ -\n"); - -PyDoc_STRVAR(doc_min_mag, -"min_mag($self, /, other, context=None)\n--\n\n\ -Similar to the min() method, but the comparison is done using the absolute\n\ -values of the operands.\n\ -\n"); - -PyDoc_STRVAR(doc_next_minus, -"next_minus($self, /, context=None)\n--\n\n\ -Return the largest number representable in the given context (or in the\n\ -current default context if no context is given) that is smaller than the\n\ -given operand.\n\ -\n"); - -PyDoc_STRVAR(doc_next_plus, -"next_plus($self, /, context=None)\n--\n\n\ -Return the smallest number representable in the given context (or in the\n\ -current default context if no context is given) that is larger than the\n\ -given operand.\n\ -\n"); - -PyDoc_STRVAR(doc_next_toward, -"next_toward($self, /, other, context=None)\n--\n\n\ -If the two operands are unequal, return the number closest to the first\n\ -operand in the direction of the second operand. If both operands are\n\ -numerically equal, return a copy of the first operand with the sign set\n\ -to be the same as the sign of the second operand.\n\ -\n"); - -PyDoc_STRVAR(doc_normalize, -"normalize($self, /, context=None)\n--\n\n\ -Normalize the number by stripping the rightmost trailing zeros and\n\ -converting any result equal to Decimal('0') to Decimal('0e0'). Used\n\ -for producing canonical values for members of an equivalence class.\n\ -For example, Decimal('32.100') and Decimal('0.321000e+2') both normalize\n\ -to the equivalent value Decimal('32.1').\n\ -\n"); - -PyDoc_STRVAR(doc_number_class, -"number_class($self, /, context=None)\n--\n\n\ -Return a string describing the class of the operand. The returned value\n\ -is one of the following ten strings:\n\ -\n\ - * '-Infinity', indicating that the operand is negative infinity.\n\ - * '-Normal', indicating that the operand is a negative normal number.\n\ - * '-Subnormal', indicating that the operand is negative and subnormal.\n\ - * '-Zero', indicating that the operand is a negative zero.\n\ - * '+Zero', indicating that the operand is a positive zero.\n\ - * '+Subnormal', indicating that the operand is positive and subnormal.\n\ - * '+Normal', indicating that the operand is a positive normal number.\n\ - * '+Infinity', indicating that the operand is positive infinity.\n\ - * 'NaN', indicating that the operand is a quiet NaN (Not a Number).\n\ - * 'sNaN', indicating that the operand is a signaling NaN.\n\ -\n\ -\n"); - -PyDoc_STRVAR(doc_quantize, -"quantize($self, /, exp, rounding=None, context=None)\n--\n\n\ -Return a value equal to the first operand after rounding and having the\n\ -exponent of the second operand.\n\ -\n\ - >>> Decimal('1.41421356').quantize(Decimal('1.000'))\n\ - Decimal('1.414')\n\ -\n\ -Unlike other operations, if the length of the coefficient after the quantize\n\ -operation would be greater than precision, then an InvalidOperation is signaled.\n\ -This guarantees that, unless there is an error condition, the quantized exponent\n\ -is always equal to that of the right-hand operand.\n\ -\n\ -Also unlike other operations, quantize never signals Underflow, even if the\n\ -result is subnormal and inexact.\n\ -\n\ -If the exponent of the second operand is larger than that of the first, then\n\ -rounding may be necessary. In this case, the rounding mode is determined by the\n\ -rounding argument if given, else by the given context argument; if neither\n\ -argument is given, the rounding mode of the current thread's context is used.\n\ -\n"); - -PyDoc_STRVAR(doc_radix, -"radix($self, /)\n--\n\n\ -Return Decimal(10), the radix (base) in which the Decimal class does\n\ -all its arithmetic. Included for compatibility with the specification.\n\ -\n"); - -PyDoc_STRVAR(doc_remainder_near, -"remainder_near($self, /, other, context=None)\n--\n\n\ -Return the remainder from dividing self by other. This differs from\n\ -self % other in that the sign of the remainder is chosen so as to minimize\n\ -its absolute value. More precisely, the return value is self - n * other\n\ -where n is the integer nearest to the exact value of self / other, and\n\ -if two integers are equally near then the even one is chosen.\n\ -\n\ -If the result is zero then its sign will be the sign of self.\n\ -\n"); - -PyDoc_STRVAR(doc_rotate, -"rotate($self, /, other, context=None)\n--\n\n\ -Return the result of rotating the digits of the first operand by an amount\n\ -specified by the second operand. The second operand must be an integer in\n\ -the range -precision through precision. The absolute value of the second\n\ -operand gives the number of places to rotate. If the second operand is\n\ -positive then rotation is to the left; otherwise rotation is to the right.\n\ -The coefficient of the first operand is padded on the left with zeros to\n\ -length precision if necessary. The sign and exponent of the first operand are\n\ -unchanged.\n\ -\n"); - -PyDoc_STRVAR(doc_same_quantum, -"same_quantum($self, /, other, context=None)\n--\n\n\ -Test whether self and other have the same exponent or whether both are NaN.\n\ -\n\ -This operation is unaffected by context and is quiet: no flags are changed\n\ -and no rounding is performed. As an exception, the C version may raise\n\ -InvalidOperation if the second operand cannot be converted exactly.\n\ -\n"); - -PyDoc_STRVAR(doc_scaleb, -"scaleb($self, /, other, context=None)\n--\n\n\ -Return the first operand with the exponent adjusted the second. Equivalently,\n\ -return the first operand multiplied by 10**other. The second operand must be\n\ -an integer.\n\ -\n"); - -PyDoc_STRVAR(doc_shift, -"shift($self, /, other, context=None)\n--\n\n\ -Return the result of shifting the digits of the first operand by an amount\n\ -specified by the second operand. The second operand must be an integer in\n\ -the range -precision through precision. The absolute value of the second\n\ -operand gives the number of places to shift. If the second operand is\n\ -positive, then the shift is to the left; otherwise the shift is to the\n\ -right. Digits shifted into the coefficient are zeros. The sign and exponent\n\ -of the first operand are unchanged.\n\ -\n"); - -PyDoc_STRVAR(doc_sqrt, -"sqrt($self, /, context=None)\n--\n\n\ -Return the square root of the argument to full precision. The result is\n\ -correctly rounded using the ROUND_HALF_EVEN rounding mode.\n\ -\n"); - -PyDoc_STRVAR(doc_to_eng_string, -"to_eng_string($self, /, context=None)\n--\n\n\ -Convert to an engineering-type string. Engineering notation has an exponent\n\ -which is a multiple of 3, so there are up to 3 digits left of the decimal\n\ -place. For example, Decimal('123E+1') is converted to Decimal('1.23E+3').\n\ -\n\ -The value of context.capitals determines whether the exponent sign is lower\n\ -or upper case. Otherwise, the context does not affect the operation.\n\ -\n"); - -PyDoc_STRVAR(doc_to_integral, -"to_integral($self, /, rounding=None, context=None)\n--\n\n\ -Identical to the to_integral_value() method. The to_integral() name has been\n\ -kept for compatibility with older versions.\n\ -\n"); - -PyDoc_STRVAR(doc_to_integral_exact, -"to_integral_exact($self, /, rounding=None, context=None)\n--\n\n\ -Round to the nearest integer, signaling Inexact or Rounded as appropriate if\n\ -rounding occurs. The rounding mode is determined by the rounding parameter\n\ -if given, else by the given context. If neither parameter is given, then the\n\ -rounding mode of the current default context is used.\n\ -\n"); - -PyDoc_STRVAR(doc_to_integral_value, -"to_integral_value($self, /, rounding=None, context=None)\n--\n\n\ -Round to the nearest integer without signaling Inexact or Rounded. The\n\ -rounding mode is determined by the rounding parameter if given, else by\n\ -the given context. If neither parameter is given, then the rounding mode\n\ -of the current default context is used.\n\ -\n"); - - -/******************************************************************************/ -/* Context Object and Methods */ -/******************************************************************************/ - -PyDoc_STRVAR(doc_context, -"Context(prec=None, rounding=None, Emin=None, Emax=None, capitals=None, clamp=None, flags=None, traps=None)\n--\n\n\ -The context affects almost all operations and controls rounding,\n\ -Over/Underflow, raising of exceptions and much more. A new context\n\ -can be constructed as follows:\n\ -\n\ - >>> c = Context(prec=28, Emin=-425000000, Emax=425000000,\n\ - ... rounding=ROUND_HALF_EVEN, capitals=1, clamp=1,\n\ - ... traps=[InvalidOperation, DivisionByZero, Overflow],\n\ - ... flags=[])\n\ - >>>\n\ -\n\ -\n"); - -#ifdef EXTRA_FUNCTIONALITY -PyDoc_STRVAR(doc_ctx_apply, -"apply($self, x, /)\n--\n\n\ -Apply self to Decimal x.\n\ -\n"); -#endif - -PyDoc_STRVAR(doc_ctx_clear_flags, -"clear_flags($self, /)\n--\n\n\ -Reset all flags to False.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_clear_traps, -"clear_traps($self, /)\n--\n\n\ -Set all traps to False.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_copy, -"copy($self, /)\n--\n\n\ -Return a duplicate of the context with all flags cleared.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_copy_decimal, -"copy_decimal($self, x, /)\n--\n\n\ -Return a copy of Decimal x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_create_decimal, -"create_decimal($self, num=\"0\", /)\n--\n\n\ -Create a new Decimal instance from num, using self as the context. Unlike the\n\ -Decimal constructor, this function observes the context limits.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_create_decimal_from_float, -"create_decimal_from_float($self, f, /)\n--\n\n\ -Create a new Decimal instance from float f. Unlike the Decimal.from_float()\n\ -class method, this function observes the context limits.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_Etiny, -"Etiny($self, /)\n--\n\n\ -Return a value equal to Emin - prec + 1, which is the minimum exponent value\n\ -for subnormal results. When underflow occurs, the exponent is set to Etiny.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_Etop, -"Etop($self, /)\n--\n\n\ -Return a value equal to Emax - prec + 1. This is the maximum exponent\n\ -if the _clamp field of the context is set to 1 (IEEE clamp mode). Etop()\n\ -must not be negative.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_abs, -"abs($self, x, /)\n--\n\n\ -Return the absolute value of x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_add, -"add($self, x, y, /)\n--\n\n\ -Return the sum of x and y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_canonical, -"canonical($self, x, /)\n--\n\n\ -Return a new instance of x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_compare, -"compare($self, x, y, /)\n--\n\n\ -Compare x and y numerically.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_compare_signal, -"compare_signal($self, x, y, /)\n--\n\n\ -Compare x and y numerically. All NaNs signal.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_compare_total, -"compare_total($self, x, y, /)\n--\n\n\ -Compare x and y using their abstract representation.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_compare_total_mag, -"compare_total_mag($self, x, y, /)\n--\n\n\ -Compare x and y using their abstract representation, ignoring sign.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_copy_abs, -"copy_abs($self, x, /)\n--\n\n\ -Return a copy of x with the sign set to 0.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_copy_negate, -"copy_negate($self, x, /)\n--\n\n\ -Return a copy of x with the sign inverted.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_copy_sign, -"copy_sign($self, x, y, /)\n--\n\n\ -Copy the sign from y to x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_divide, -"divide($self, x, y, /)\n--\n\n\ -Return x divided by y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_divide_int, -"divide_int($self, x, y, /)\n--\n\n\ -Return x divided by y, truncated to an integer.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_divmod, -"divmod($self, x, y, /)\n--\n\n\ -Return quotient and remainder of the division x / y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_exp, -"exp($self, x, /)\n--\n\n\ -Return e ** x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_fma, -"fma($self, x, y, z, /)\n--\n\n\ -Return x multiplied by y, plus z.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_canonical, -"is_canonical($self, x, /)\n--\n\n\ -Return True if x is canonical, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_finite, -"is_finite($self, x, /)\n--\n\n\ -Return True if x is finite, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_infinite, -"is_infinite($self, x, /)\n--\n\n\ -Return True if x is infinite, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_nan, -"is_nan($self, x, /)\n--\n\n\ -Return True if x is a qNaN or sNaN, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_normal, -"is_normal($self, x, /)\n--\n\n\ -Return True if x is a normal number, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_qnan, -"is_qnan($self, x, /)\n--\n\n\ -Return True if x is a quiet NaN, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_signed, -"is_signed($self, x, /)\n--\n\n\ -Return True if x is negative, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_snan, -"is_snan($self, x, /)\n--\n\n\ -Return True if x is a signaling NaN, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_subnormal, -"is_subnormal($self, x, /)\n--\n\n\ -Return True if x is subnormal, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_is_zero, -"is_zero($self, x, /)\n--\n\n\ -Return True if x is a zero, False otherwise.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_ln, -"ln($self, x, /)\n--\n\n\ -Return the natural (base e) logarithm of x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_log10, -"log10($self, x, /)\n--\n\n\ -Return the base 10 logarithm of x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_logb, -"logb($self, x, /)\n--\n\n\ -Return the exponent of the magnitude of the operand's MSD.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_logical_and, -"logical_and($self, x, y, /)\n--\n\n\ -Digit-wise and of x and y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_logical_invert, -"logical_invert($self, x, /)\n--\n\n\ -Invert all digits of x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_logical_or, -"logical_or($self, x, y, /)\n--\n\n\ -Digit-wise or of x and y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_logical_xor, -"logical_xor($self, x, y, /)\n--\n\n\ -Digit-wise xor of x and y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_max, -"max($self, x, y, /)\n--\n\n\ -Compare the values numerically and return the maximum.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_max_mag, -"max_mag($self, x, y, /)\n--\n\n\ -Compare the values numerically with their sign ignored.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_min, -"min($self, x, y, /)\n--\n\n\ -Compare the values numerically and return the minimum.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_min_mag, -"min_mag($self, x, y, /)\n--\n\n\ -Compare the values numerically with their sign ignored.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_minus, -"minus($self, x, /)\n--\n\n\ -Minus corresponds to the unary prefix minus operator in Python, but applies\n\ -the context to the result.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_multiply, -"multiply($self, x, y, /)\n--\n\n\ -Return the product of x and y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_next_minus, -"next_minus($self, x, /)\n--\n\n\ -Return the largest representable number smaller than x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_next_plus, -"next_plus($self, x, /)\n--\n\n\ -Return the smallest representable number larger than x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_next_toward, -"next_toward($self, x, y, /)\n--\n\n\ -Return the number closest to x, in the direction towards y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_normalize, -"normalize($self, x, /)\n--\n\n\ -Reduce x to its simplest form. Alias for reduce(x).\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_number_class, -"number_class($self, x, /)\n--\n\n\ -Return an indication of the class of x.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_plus, -"plus($self, x, /)\n--\n\n\ -Plus corresponds to the unary prefix plus operator in Python, but applies\n\ -the context to the result.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_power, -"power($self, /, a, b, modulo=None)\n--\n\n\ -Compute a**b. If 'a' is negative, then 'b' must be integral. The result\n\ -will be inexact unless 'a' is integral and the result is finite and can\n\ -be expressed exactly in 'precision' digits. In the Python version the\n\ -result is always correctly rounded, in the C version the result is almost\n\ -always correctly rounded.\n\ -\n\ -If modulo is given, compute (a**b) % modulo. The following restrictions\n\ -hold:\n\ -\n\ - * all three arguments must be integral\n\ - * 'b' must be nonnegative\n\ - * at least one of 'a' or 'b' must be nonzero\n\ - * modulo must be nonzero and less than 10**prec in absolute value\n\ -\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_quantize, -"quantize($self, x, y, /)\n--\n\n\ -Return a value equal to x (rounded), having the exponent of y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_radix, -"radix($self, /)\n--\n\n\ -Return 10.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_remainder, -"remainder($self, x, y, /)\n--\n\n\ -Return the remainder from integer division. The sign of the result,\n\ -if non-zero, is the same as that of the original dividend.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_remainder_near, -"remainder_near($self, x, y, /)\n--\n\n\ -Return x - y * n, where n is the integer nearest the exact value of x / y\n\ -(if the result is 0 then its sign will be the sign of x).\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_rotate, -"rotate($self, x, y, /)\n--\n\n\ -Return a copy of x, rotated by y places.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_same_quantum, -"same_quantum($self, x, y, /)\n--\n\n\ -Return True if the two operands have the same exponent.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_scaleb, -"scaleb($self, x, y, /)\n--\n\n\ -Return the first operand after adding the second value to its exp.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_shift, -"shift($self, x, y, /)\n--\n\n\ -Return a copy of x, shifted by y places.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_sqrt, -"sqrt($self, x, /)\n--\n\n\ -Square root of a non-negative number to context precision.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_subtract, -"subtract($self, x, y, /)\n--\n\n\ -Return the difference between x and y.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_to_eng_string, -"to_eng_string($self, x, /)\n--\n\n\ -Convert a number to a string, using engineering notation.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_to_integral, -"to_integral($self, x, /)\n--\n\n\ -Identical to to_integral_value(x).\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_to_integral_exact, -"to_integral_exact($self, x, /)\n--\n\n\ -Round to an integer. Signal if the result is rounded or inexact.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_to_integral_value, -"to_integral_value($self, x, /)\n--\n\n\ -Round to an integer.\n\ -\n"); - -PyDoc_STRVAR(doc_ctx_to_sci_string, -"to_sci_string($self, x, /)\n--\n\n\ -Convert a number to a string using scientific notation.\n\ -\n"); - - -#endif /* DOCSTRINGS_H */ - - - diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 8c3efa36353..3173b52afb3 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -17,6 +17,7 @@ #include "Python.h" #include "pycore_pyhash.h" // _Py_HashSecret +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include "expat.h" @@ -690,8 +691,7 @@ element_dealloc(PyObject *op) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); /* element_gc_clear clears all references and deallocates extra */ @@ -811,6 +811,8 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) PyTypeObject *tp = Py_TYPE(self); elementtreestate *st = get_elementtree_state_by_type(tp); + // The deepcopy() helper takes care of incrementing the refcount + // of the object to copy so to avoid use-after-frees. tag = deepcopy(st, self->tag, memo); if (!tag) return NULL; @@ -845,11 +847,13 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) assert(!element->extra || !element->extra->length); if (self->extra) { - if (element_resize(element, self->extra->length) < 0) + Py_ssize_t expected_count = self->extra->length; + if (element_resize(element, expected_count) < 0) { + assert(!element->extra->length); goto error; + } - // TODO(picnixz): check for an evil child's __deepcopy__ on 'self' - for (i = 0; i < self->extra->length; i++) { + for (i = 0; self->extra && i < self->extra->length; i++) { PyObject* child = deepcopy(st, self->extra->children[i], memo); if (!child || !Element_Check(st, child)) { if (child) { @@ -859,11 +863,24 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) element->extra->length = i; goto error; } + if (self->extra && expected_count != self->extra->length) { + // 'self->extra' got mutated and 'element' may not have + // sufficient space to hold the next iteration's item. + expected_count = self->extra->length; + if (element_resize(element, expected_count) < 0) { + Py_DECREF(child); + element->extra->length = i; + goto error; + } + } element->extra->children[i] = child; } assert(!element->extra->length); - element->extra->length = self->extra->length; + // The original 'self->extra' may be gone at this point if deepcopy() + // has side-effects. However, 'i' is the number of copied items that + // we were able to successfully copy. + element->extra->length = i; } /* add object to memo dictionary (so deepcopy won't visit it again) */ @@ -895,7 +912,7 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) return Py_NewRef(object); } - if (Py_REFCNT(object) == 1) { + if (_PyObject_IsUniquelyReferenced(object)) { if (PyDict_CheckExact(object)) { PyObject *key, *value; Py_ssize_t pos = 0; @@ -906,13 +923,20 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) break; } } - if (simple) + if (simple) { return PyDict_Copy(object); + } /* Fall through to general case */ } else if (Element_CheckExact(st, object)) { - return _elementtree_Element___deepcopy___impl( + // The __deepcopy__() call may call arbitrary code even if the + // object to copy is a built-in XML element (one of its children + // any of its parents in its own __deepcopy__() implementation). + Py_INCREF(object); + PyObject *res = _elementtree_Element___deepcopy___impl( (ElementObject *)object, memo); + Py_DECREF(object); + return res; } } @@ -923,8 +947,11 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) return NULL; } + Py_INCREF(object); PyObject *args[2] = {object, memo}; - return PyObject_Vectorcall(st->deepcopy_obj, args, 2, NULL); + PyObject *res = PyObject_Vectorcall(st->deepcopy_obj, args, 2, NULL); + Py_DECREF(object); + return res; } @@ -2767,8 +2794,9 @@ treebuilder_handle_data(TreeBuilderObject* self, PyObject* data) self->data = Py_NewRef(data); } else { /* more than one item; use a list to collect items */ - if (PyBytes_CheckExact(self->data) && Py_REFCNT(self->data) == 1 && - PyBytes_CheckExact(data) && PyBytes_GET_SIZE(data) == 1) { + if (PyBytes_CheckExact(self->data) + && _PyObject_IsUniquelyReferenced(self->data) + && PyBytes_CheckExact(data) && PyBytes_GET_SIZE(data) == 1) { /* XXX this code path unused in Python 3? */ /* expat often generates single character data sections; handle the most common case by resizing the existing string... */ @@ -4187,8 +4215,8 @@ _elementtree_XMLParser__setevents_impl(XMLParserObject *self, (XML_ProcessingInstructionHandler) expat_pi_handler ); } else { - Py_DECREF(events_seq); PyErr_Format(PyExc_ValueError, "unknown event '%s'", event_name); + Py_DECREF(events_seq); return NULL; } } diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index e6c454faf4b..5773083ff68 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -7,6 +7,7 @@ #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/_functoolsmodule.c.h" @@ -107,20 +108,13 @@ placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return placeholder; } -static int -placeholder_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static PyType_Slot placeholder_type_slots[] = { {Py_tp_dealloc, placeholder_dealloc}, {Py_tp_repr, placeholder_repr}, {Py_tp_doc, (void *)placeholder_doc}, {Py_tp_methods, placeholder_methods}, {Py_tp_new, placeholder_new}, - {Py_tp_traverse, placeholder_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0, 0} }; @@ -196,6 +190,19 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } + /* keyword Placeholder prohibition */ + if (kw != NULL) { + PyObject *key, *val; + Py_ssize_t pos = 0; + while (PyDict_Next(kw, &pos, &key, &val)) { + if (val == phold) { + PyErr_SetString(PyExc_TypeError, + "Placeholder cannot be passed as a keyword argument"); + return NULL; + } + } + } + /* check wrapped function / object */ pto_args = pto_kw = NULL; int res = PyObject_TypeCheck(func, state->partial_type); @@ -284,7 +291,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (kw == NULL) { pto->kw = PyDict_New(); } - else if (Py_REFCNT(kw) == 1) { + else if (_PyObject_IsUniquelyReferenced(kw)) { pto->kw = Py_NewRef(kw); } else { @@ -338,9 +345,7 @@ partial_dealloc(PyObject *self) PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - if (partialobject_CAST(self)->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, partialobject_CAST(self)->weakreflist); (void)partial_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -355,19 +360,6 @@ partial_descr_get(PyObject *self, PyObject *obj, PyObject *type) return PyMethod_New(self, obj); } -/* Merging keyword arguments using the vectorcall convention is messy, so - * if we would need to do that, we stop using vectorcall and fall back - * to using partial_call() instead. */ -Py_NO_INLINE static PyObject * -partial_vectorcall_fallback(PyThreadState *tstate, partialobject *pto, - PyObject *const *args, size_t nargsf, - PyObject *kwnames) -{ - pto->vectorcall = NULL; - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall(tstate, (PyObject *)pto, args, nargs, kwnames); -} - static PyObject * partial_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) @@ -376,10 +368,7 @@ partial_vectorcall(PyObject *self, PyObject *const *args, PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - /* pto->kw is mutable, so need to check every time */ - if (PyDict_GET_SIZE(pto->kw)) { - return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames); - } + /* Placeholder check */ Py_ssize_t pto_phcount = pto->phcount; if (nargs < pto_phcount) { PyErr_Format(PyExc_TypeError, @@ -388,50 +377,143 @@ partial_vectorcall(PyObject *self, PyObject *const *args, return NULL; } - Py_ssize_t nargskw = nargs; - if (kwnames != NULL) { - nargskw += PyTuple_GET_SIZE(kwnames); - } - PyObject **pto_args = _PyTuple_ITEMS(pto->args); Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); + Py_ssize_t pto_nkwds = PyDict_GET_SIZE(pto->kw); + Py_ssize_t nkwds = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); + Py_ssize_t nargskw = nargs + nkwds; - /* Fast path if we're called without arguments */ - if (nargskw == 0) { - return _PyObject_VectorcallTstate(tstate, pto->fn, - pto_args, pto_nargs, NULL); - } + /* Special cases */ + if (!pto_nkwds) { + /* Fast path if we're called without arguments */ + if (nargskw == 0) { + return _PyObject_VectorcallTstate(tstate, pto->fn, pto_args, + pto_nargs, NULL); + } - /* Fast path using PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single - * positional argument */ - if (pto_nargs == 1 && (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET)) { - PyObject **newargs = (PyObject **)args - 1; - PyObject *tmp = newargs[0]; - newargs[0] = pto_args[0]; - PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, - newargs, nargs + 1, kwnames); - newargs[0] = tmp; - return ret; - } - - PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; - PyObject **stack; - - Py_ssize_t tot_nargskw = pto_nargs + nargskw - pto_phcount; - if (tot_nargskw <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { - stack = small_stack; - } - else { - stack = PyMem_Malloc(tot_nargskw * sizeof(PyObject *)); - if (stack == NULL) { - PyErr_NoMemory(); - return NULL; + /* Use PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single + * positional argument. */ + if (pto_nargs == 1 && (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET)) { + PyObject **newargs = (PyObject **)args - 1; + PyObject *tmp = newargs[0]; + newargs[0] = pto_args[0]; + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, newargs, + nargs + 1, kwnames); + newargs[0] = tmp; + return ret; } } - Py_ssize_t tot_nargs; + /* Total sizes */ + Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount; + Py_ssize_t tot_nkwds = pto_nkwds + nkwds; + Py_ssize_t tot_nargskw = tot_nargs + tot_nkwds; + + PyObject *pto_kw_merged = NULL; // pto_kw with duplicates merged (if any) + PyObject *tot_kwnames; + + /* Allocate Stack + * Note, _PY_FASTCALL_SMALL_STACK is optimal for positional only + * This case might have keyword arguments + * furthermore, it might use extra stack space for temporary key storage + * thus, double small_stack size is used, which is 10 * 8 = 80 bytes */ + PyObject *small_stack[_PY_FASTCALL_SMALL_STACK * 2]; + PyObject **tmp_stack, **stack; + Py_ssize_t init_stack_size = tot_nargskw; + if (pto_nkwds) { + // If pto_nkwds, allocate additional space for temporary new keys + init_stack_size += nkwds; + } + if (init_stack_size <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + stack = small_stack; + } + else { + stack = PyMem_Malloc(init_stack_size * sizeof(PyObject *)); + if (stack == NULL) { + return PyErr_NoMemory(); + } + } + + /* Copy keywords to stack */ + if (!pto_nkwds) { + tot_kwnames = kwnames; + if (nkwds) { + /* if !pto_nkwds & nkwds, then simply append kw */ + memcpy(stack + tot_nargs, args + nargs, nkwds * sizeof(PyObject*)); + } + } + else { + /* stack is now [<positionals>, <pto_kwds>, <kwds>, <kwds_keys>] + * Will resize later to [<positionals>, <merged_kwds>] */ + PyObject *key, *val; + + /* Merge kw to pto_kw or add to tail (if not duplicate) */ + Py_ssize_t n_tail = 0; + for (Py_ssize_t i = 0; i < nkwds; ++i) { + key = PyTuple_GET_ITEM(kwnames, i); + val = args[nargs + i]; + if (PyDict_Contains(pto->kw, key)) { + if (pto_kw_merged == NULL) { + pto_kw_merged = PyDict_Copy(pto->kw); + if (pto_kw_merged == NULL) { + goto error; + } + } + if (PyDict_SetItem(pto_kw_merged, key, val) < 0) { + Py_DECREF(pto_kw_merged); + goto error; + } + } + else { + /* Copy keyword tail to stack */ + stack[tot_nargs + pto_nkwds + n_tail] = val; + stack[tot_nargskw + n_tail] = key; + n_tail++; + } + } + Py_ssize_t n_merges = nkwds - n_tail; + + /* Create total kwnames */ + tot_kwnames = PyTuple_New(tot_nkwds - n_merges); + if (tot_kwnames == NULL) { + Py_XDECREF(pto_kw_merged); + goto error; + } + for (Py_ssize_t i = 0; i < n_tail; ++i) { + key = Py_NewRef(stack[tot_nargskw + i]); + PyTuple_SET_ITEM(tot_kwnames, pto_nkwds + i, key); + } + + /* Copy pto_keywords with overlapping call keywords merged + * Note, tail is already coppied. */ + Py_ssize_t pos = 0, i = 0; + while (PyDict_Next(n_merges ? pto_kw_merged : pto->kw, &pos, &key, &val)) { + assert(i < pto_nkwds); + PyTuple_SET_ITEM(tot_kwnames, i, Py_NewRef(key)); + stack[tot_nargs + i] = val; + i++; + } + assert(i == pto_nkwds); + Py_XDECREF(pto_kw_merged); + + /* Resize Stack if the removing overallocation saves some noticable memory + * NOTE: This whole block can be removed without breaking anything */ + Py_ssize_t noveralloc = n_merges + nkwds; + if (stack != small_stack && noveralloc > 6 && noveralloc > init_stack_size / 10) { + tmp_stack = PyMem_Realloc(stack, (tot_nargskw - n_merges) * sizeof(PyObject *)); + if (tmp_stack == NULL) { + Py_DECREF(tot_kwnames); + if (stack != small_stack) { + PyMem_Free(stack); + } + return PyErr_NoMemory(); + } + stack = tmp_stack; + } + } + + /* Copy Positionals to stack */ if (pto_phcount) { - tot_nargs = pto_nargs + nargs - pto_phcount; Py_ssize_t j = 0; // New args index for (Py_ssize_t i = 0; i < pto_nargs; i++) { if (pto_args[i] == pto->placeholder) { @@ -443,22 +525,31 @@ partial_vectorcall(PyObject *self, PyObject *const *args, } } assert(j == pto_phcount); - if (nargskw > pto_phcount) { - memcpy(stack + pto_nargs, args + j, (nargskw - j) * sizeof(PyObject*)); + /* Add remaining args from new_args */ + if (nargs > pto_phcount) { + memcpy(stack + pto_nargs, args + j, (nargs - j) * sizeof(PyObject*)); } } else { - tot_nargs = pto_nargs + nargs; - /* Copy to new stack, using borrowed references */ memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); - memcpy(stack + pto_nargs, args, nargskw * sizeof(PyObject*)); + memcpy(stack + pto_nargs, args, nargs * sizeof(PyObject*)); } - PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, - stack, tot_nargs, kwnames); + + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, stack, + tot_nargs, tot_kwnames); if (stack != small_stack) { PyMem_Free(stack); } + if (pto_nkwds) { + Py_DECREF(tot_kwnames); + } return ret; + + error: + if (stack != small_stack) { + PyMem_Free(stack); + } + return NULL; } /* Set pto->vectorcall depending on the parameters of the partial object */ @@ -687,7 +778,8 @@ partial_setstate(PyObject *self, PyObject *state) if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || - (kw != Py_None && !PyDict_Check(kw))) + (kw != Py_None && !PyDict_Check(kw)) || + (dict != Py_None && !PyDict_Check(dict))) { PyErr_SetString(PyExc_TypeError, "invalid partial state"); return NULL; @@ -941,6 +1033,8 @@ _functools_cmp_to_key_impl(PyObject *module, PyObject *mycmp) /* reduce (used to be a builtin) ********************************************/ /*[clinic input] +@permit_long_summary +@permit_long_docstring_body _functools.reduce function as func: object @@ -961,7 +1055,7 @@ calculates ((((1 + 2) + 3) + 4) + 5). static PyObject * _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, PyObject *result) -/*[clinic end generated code: output=30d898fe1267c79d input=1511e9a8c38581ac]*/ +/*[clinic end generated code: output=30d898fe1267c79d input=4ccfb74548ce5170]*/ { PyObject *args, *it; @@ -983,7 +1077,7 @@ _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, for (;;) { PyObject *op2; - if (Py_REFCNT(args) > 1) { + if (!_PyObject_IsUniquelyReferenced(args)) { Py_DECREF(args); if ((args = PyTuple_New(2)) == NULL) goto Fail; @@ -1370,8 +1464,8 @@ bounded_lru_cache_update_lock_held(lru_cache_object *self, this same key, then this setitem call will update the cache dict with this new link, leaving the old link as an orphan (i.e. not having a cache dict entry that refers to it). */ - if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, - hash) < 0) { + if (_PyDict_SetItem_KnownHash_LockHeld((PyDictObject *)self->cache, key, + (PyObject *)link, hash) < 0) { Py_DECREF(link); return NULL; } @@ -1440,8 +1534,8 @@ bounded_lru_cache_update_lock_held(lru_cache_object *self, for successful insertion in the cache dict before adding the link to the linked list. Otherwise, the potentially reentrant __eq__ call could cause the then orphan link to be visited. */ - if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, - hash) < 0) { + if (_PyDict_SetItem_KnownHash_LockHeld((PyDictObject *)self->cache, key, + (PyObject *)link, hash) < 0) { /* Somehow the cache dict update failed. We no longer can restore the old link. Let the error propagate upward and leave the cache short one link. */ @@ -1608,9 +1702,7 @@ lru_cache_dealloc(PyObject *op) PyTypeObject *tp = Py_TYPE(obj); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(obj); - if (obj->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, obj->weakreflist); (void)lru_cache_tp_clear(op); tp->tp_free(obj); @@ -1676,7 +1768,13 @@ _functools__lru_cache_wrapper_cache_clear_impl(PyObject *self) lru_list_elem *list = lru_cache_unlink_list(_self); FT_ATOMIC_STORE_SSIZE_RELAXED(_self->hits, 0); FT_ATOMIC_STORE_SSIZE_RELAXED(_self->misses, 0); - PyDict_Clear(_self->cache); + if (_self->wrapper == bounded_lru_cache_wrapper) { + /* The critical section on the lru cache itself protects the dictionary + for bounded_lru_cache instances. */ + _PyDict_Clear_LockHeld(_self->cache); + } else { + PyDict_Clear(_self->cache); + } lru_cache_clear_list(list); Py_RETURN_NONE; } diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index ab2ebdba924..72f568ceb06 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -8,11 +8,12 @@ #endif #include "Python.h" -#include "pycore_pyerrors.h" // _PyErr_SetLocaleString() +#include "pycore_object.h" // _PyObject_VisitType() +#include "pycore_pyerrors.h" // _PyErr_SetLocaleString() #include "gdbm.h" #include <fcntl.h> -#include <stdlib.h> // free() +#include <stdlib.h> // free() #include <sys/stat.h> #include <sys/types.h> @@ -81,6 +82,7 @@ typedef struct { #include "clinic/_gdbmmodule.c.h" #define check_gdbmobject_open(v, err) \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED((v)) \ if ((v)->di_dbm == NULL) { \ PyErr_SetString(err, "GDBM object has already been closed"); \ return NULL; \ @@ -121,13 +123,6 @@ newgdbmobject(_gdbm_state *state, const char *file, int flags, int mode) } /* Methods */ -static int -gdbm_traverse(PyObject *op, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(op)); - return 0; -} - static void gdbm_dealloc(PyObject *op) { @@ -142,7 +137,7 @@ gdbm_dealloc(PyObject *op) } static Py_ssize_t -gdbm_length(PyObject *op) +gdbm_length_lock_held(PyObject *op) { gdbmobject *dp = _gdbmobject_CAST(op); _gdbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); @@ -188,8 +183,18 @@ gdbm_length(PyObject *op) return dp->di_size; } +static Py_ssize_t +gdbm_length(PyObject *op) +{ + Py_ssize_t result; + Py_BEGIN_CRITICAL_SECTION(op); + result = gdbm_length_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -gdbm_bool(PyObject *op) +gdbm_bool_lock_held(PyObject *op) { gdbmobject *dp = _gdbmobject_CAST(op); _gdbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); @@ -218,6 +223,16 @@ gdbm_bool(PyObject *op) return 1; } +static int +gdbm_bool(PyObject *op) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = gdbm_bool_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + // Wrapper function for PyArg_Parse(o, "s#", &d.dptr, &d.size). // This function is needed to support PY_SSIZE_T_CLEAN. // Return 1 on success, same to PyArg_Parse(). @@ -240,7 +255,7 @@ parse_datum(PyObject *o, datum *d, const char *failmsg) } static PyObject * -gdbm_subscript(PyObject *op, PyObject *key) +gdbm_subscript_lock_held(PyObject *op, PyObject *key) { PyObject *v; datum drec, krec; @@ -265,6 +280,16 @@ gdbm_subscript(PyObject *op, PyObject *key) return v; } +static PyObject * +gdbm_subscript(PyObject *op, PyObject *key) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = gdbm_subscript_lock_held(op, key); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] _gdbm.gdbm.get @@ -290,7 +315,7 @@ _gdbm_gdbm_get_impl(gdbmobject *self, PyObject *key, PyObject *default_value) } static int -gdbm_ass_sub(PyObject *op, PyObject *v, PyObject *w) +gdbm_ass_sub_lock_held(PyObject *op, PyObject *v, PyObject *w) { datum krec, drec; const char *failmsg = "gdbm mappings have bytes or string indices only"; @@ -335,7 +360,19 @@ gdbm_ass_sub(PyObject *op, PyObject *v, PyObject *w) return 0; } +static int +gdbm_ass_sub(PyObject *op, PyObject *v, PyObject *w) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = gdbm_ass_sub_lock_held(op, v, w); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] +@permit_long_summary +@critical_section _gdbm.gdbm.setdefault key: object @@ -348,7 +385,7 @@ Get value for key, or set it to default and return default if not present. static PyObject * _gdbm_gdbm_setdefault_impl(gdbmobject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=f3246e880509f142 input=0db46b69e9680171]*/ +/*[clinic end generated code: output=f3246e880509f142 input=f4008b358165bbb8]*/ { PyObject *res; @@ -363,6 +400,7 @@ _gdbm_gdbm_setdefault_impl(gdbmobject *self, PyObject *key, } /*[clinic input] +@critical_section _gdbm.gdbm.close Close the database. @@ -370,7 +408,7 @@ Close the database. static PyObject * _gdbm_gdbm_close_impl(gdbmobject *self) -/*[clinic end generated code: output=f5abb4d6bb9e52d5 input=0a203447379b45fd]*/ +/*[clinic end generated code: output=f5abb4d6bb9e52d5 input=56b604f4e77f533d]*/ { if (self->di_dbm) { gdbm_close(self->di_dbm); @@ -381,6 +419,7 @@ _gdbm_gdbm_close_impl(gdbmobject *self) /* XXX Should return a set or a set view */ /*[clinic input] +@critical_section _gdbm.gdbm.keys cls: defining_class @@ -390,7 +429,7 @@ Get a list of all keys in the database. static PyObject * _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=c24b824e81404755 input=1428b7c79703d7d5]*/ +/*[clinic end generated code: output=c24b824e81404755 input=785988b1ea8f77e0]*/ { PyObject *v, *item; datum key, nextkey; @@ -432,7 +471,7 @@ _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls) } static int -gdbm_contains(PyObject *self, PyObject *arg) +gdbm_contains_lock_held(PyObject *self, PyObject *arg) { gdbmobject *dp = (gdbmobject *)self; datum key; @@ -463,7 +502,18 @@ gdbm_contains(PyObject *self, PyObject *arg) return gdbm_exists(dp->di_dbm, key); } +static int +gdbm_contains(PyObject *self, PyObject *arg) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(self); + result = gdbm_contains_lock_held(self, arg); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] +@critical_section _gdbm.gdbm.firstkey cls: defining_class @@ -477,7 +527,7 @@ hash values, and won't be sorted by the key values. static PyObject * _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=139275e9c8b60827 input=ed8782a029a5d299]*/ +/*[clinic end generated code: output=139275e9c8b60827 input=aad5a7c886c542f5]*/ { PyObject *v; datum key; @@ -497,6 +547,7 @@ _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _gdbm.gdbm.nextkey cls: defining_class @@ -517,7 +568,7 @@ to create a list in memory that contains them all: static PyObject * _gdbm_gdbm_nextkey_impl(gdbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length) -/*[clinic end generated code: output=c81a69300ef41766 input=365e297bc0b3db48]*/ +/*[clinic end generated code: output=c81a69300ef41766 input=181f1130d5bfeb1e]*/ { PyObject *v; datum dbm_key, nextkey; @@ -539,6 +590,7 @@ _gdbm_gdbm_nextkey_impl(gdbmobject *self, PyTypeObject *cls, const char *key, } /*[clinic input] +@critical_section _gdbm.gdbm.reorganize cls: defining_class @@ -554,7 +606,7 @@ kept and reused as new (key,value) pairs are added. static PyObject * _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=d77c69e8e3dd644a input=e1359faeef844e46]*/ +/*[clinic end generated code: output=d77c69e8e3dd644a input=3e3ca0d2ea787861]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -573,6 +625,7 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _gdbm.gdbm.sync cls: defining_class @@ -585,7 +638,7 @@ any unwritten data to be written to the disk. static PyObject * _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=bb680a2035c3f592 input=3d749235f79b6f2a]*/ +/*[clinic end generated code: output=bb680a2035c3f592 input=6054385b071d238a]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -595,6 +648,7 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _gdbm.gdbm.clear cls: defining_class / @@ -604,7 +658,7 @@ Remove all items from the database. static PyObject * _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=673577c573318661 input=34136d52fcdd4210]*/ +/*[clinic end generated code: output=673577c573318661 input=b17467adfe62f23d]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -619,8 +673,10 @@ _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls) } if (gdbm_delete(self->di_dbm, key) < 0) { PyErr_SetString(state->gdbm_error, "cannot delete item from database"); + free(key.dptr); return NULL; } + free(key.dptr); } Py_RETURN_NONE; } @@ -634,7 +690,11 @@ gdbm__enter__(PyObject *self, PyObject *args) static PyObject * gdbm__exit__(PyObject *self, PyObject *args) { - return _gdbm_gdbm_close_impl((gdbmobject *)self); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(self); + result = _gdbm_gdbm_close_impl((gdbmobject *)self); + Py_END_CRITICAL_SECTION(); + return result; } static PyMethodDef gdbm_methods[] = { @@ -654,7 +714,7 @@ static PyMethodDef gdbm_methods[] = { static PyType_Slot gdbmtype_spec_slots[] = { {Py_tp_dealloc, gdbm_dealloc}, - {Py_tp_traverse, gdbm_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_methods, gdbm_methods}, {Py_sq_contains, gdbm_contains}, {Py_mp_length, gdbm_length}, diff --git a/Modules/_hacl/Hacl_HMAC.h b/Modules/_hacl/Hacl_HMAC.h index 10ff15183f2..7dca53c9254 100644 --- a/Modules/_hacl/Hacl_HMAC.h +++ b/Modules/_hacl/Hacl_HMAC.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_HMAC_H -#define __Hacl_HMAC_H +#ifndef Hacl_HMAC_H +#define Hacl_HMAC_H #if defined(__cplusplus) extern "C" { @@ -220,5 +220,5 @@ Hacl_HMAC_compute_blake2b_32( } #endif -#define __Hacl_HMAC_H_DEFINED -#endif +#define Hacl_HMAC_H_DEFINED +#endif /* Hacl_HMAC_H */ diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.c b/Modules/_hacl/Hacl_Hash_Blake2b.c index 21ab2b88c79..a5b75d61798 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b.c +++ b/Modules/_hacl/Hacl_Hash_Blake2b.c @@ -544,11 +544,9 @@ void Hacl_Hash_Blake2b_init(uint64_t *hash, uint32_t kk, uint32_t nn) uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk1 - << 8U - ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk1 << 8U ^ + ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); tmp[1U] = p.node_offset; tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; tmp[3U] = 0ULL; @@ -860,14 +858,10 @@ static Hacl_Hash_Blake2b_state_t uint64_t x = r4; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ - ((uint64_t)pv.fanout - << 16U - ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ + ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1059,11 +1053,9 @@ static void reset_raw(Hacl_Hash_Blake2b_state_t *state, Hacl_Hash_Blake2b_params uint64_t x = r; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1200,8 +1192,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1265,8 +1256,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1296,8 +1286,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state10, @@ -1359,8 +1348,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1690,14 +1678,10 @@ Hacl_Hash_Blake2b_hash_with_key_and_params( uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn - ^ - ((uint64_t)kk - << 8U - ^ - ((uint64_t)params.fanout - << 16U - ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + (uint64_t)nn ^ + ((uint64_t)kk << 8U ^ + ((uint64_t)params.fanout << 16U ^ + ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); tmp[1U] = params.node_offset; tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; tmp[3U] = 0ULL; diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.h b/Modules/_hacl/Hacl_Hash_Blake2b.h index 3a73f358c98..b07893ba339 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b.h +++ b/Modules/_hacl/Hacl_Hash_Blake2b.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_Blake2b_H -#define __Hacl_Hash_Blake2b_H +#ifndef Hacl_Hash_Blake2b_H +#define Hacl_Hash_Blake2b_H #if defined(__cplusplus) extern "C" { @@ -220,5 +220,5 @@ Hacl_Hash_Blake2b_hash_with_key_and_params( } #endif -#define __Hacl_Hash_Blake2b_H_DEFINED -#endif +#define Hacl_Hash_Blake2b_H_DEFINED +#endif /* Hacl_Hash_Blake2b_H */ diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c index c4d9b4a689d..f955f1c6115 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c @@ -274,11 +274,9 @@ Hacl_Hash_Blake2b_Simd256_init(Lib_IntVector_Intrinsics_vec256 *hash, uint32_t k uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk1 - << 8U - ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk1 << 8U ^ + ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); tmp[1U] = p.node_offset; tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; tmp[3U] = 0ULL; @@ -746,14 +744,10 @@ static Hacl_Hash_Blake2b_Simd256_state_t uint64_t x = r4; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ - ((uint64_t)pv.fanout - << 16U - ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ + ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -936,11 +930,9 @@ reset_raw(Hacl_Hash_Blake2b_Simd256_state_t *state, Hacl_Hash_Blake2b_params_and uint64_t x = r; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1075,8 +1067,7 @@ Hacl_Hash_Blake2b_Simd256_update( uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1140,8 +1131,7 @@ Hacl_Hash_Blake2b_Simd256_update( nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1171,8 +1161,7 @@ Hacl_Hash_Blake2b_Simd256_update( uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state10, @@ -1234,8 +1223,7 @@ Hacl_Hash_Blake2b_Simd256_update( nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1578,14 +1566,10 @@ Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn - ^ - ((uint64_t)kk - << 8U - ^ - ((uint64_t)params.fanout - << 16U - ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + (uint64_t)nn ^ + ((uint64_t)kk << 8U ^ + ((uint64_t)params.fanout << 16U ^ + ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); tmp[1U] = params.node_offset; tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; tmp[3U] = 0ULL; diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h index 0271ab8e024..8be8f32db62 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_Blake2b_Simd256_H -#define __Hacl_Hash_Blake2b_Simd256_H +#ifndef Hacl_Hash_Blake2b_Simd256_H +#define Hacl_Hash_Blake2b_Simd256_H #if defined(__cplusplus) extern "C" { @@ -206,5 +206,5 @@ Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( } #endif -#define __Hacl_Hash_Blake2b_Simd256_H_DEFINED -#endif +#define Hacl_Hash_Blake2b_Simd256_H_DEFINED +#endif /* Hacl_Hash_Blake2b_Simd256_H */ diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.c b/Modules/_hacl/Hacl_Hash_Blake2s.c index 730ba135afb..0d4fcc7f395 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s.c +++ b/Modules/_hacl/Hacl_Hash_Blake2s.c @@ -543,13 +543,13 @@ void Hacl_Hash_Blake2s_init(uint32_t *hash, uint32_t kk, uint32_t nn) uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)(uint8_t)nn - ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + (uint32_t)(uint8_t)nn ^ + ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); tmp[1U] = p.leaf_length; tmp[2U] = (uint32_t)p.node_offset; tmp[3U] = - (uint32_t)(p.node_offset >> 32U) - ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + (uint32_t)(p.node_offset >> 32U) ^ + ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -846,16 +846,14 @@ static Hacl_Hash_Blake2s_state_t uint32_t x = r4; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ - ((uint32_t)pv.key_length - << 8U - ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ + ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1042,13 +1040,13 @@ static void reset_raw(Hacl_Hash_Blake2s_state_t *state, Hacl_Hash_Blake2b_params uint32_t x = r; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1182,8 +1180,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1237,8 +1234,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1268,8 +1264,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state10, @@ -1321,8 +1316,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1639,16 +1633,14 @@ Hacl_Hash_Blake2s_hash_with_key_and_params( uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)params.digest_length - ^ - ((uint32_t)params.key_length - << 8U - ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + (uint32_t)params.digest_length ^ + ((uint32_t)params.key_length << 8U ^ + ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); tmp[1U] = params.leaf_length; tmp[2U] = (uint32_t)params.node_offset; tmp[3U] = - (uint32_t)(params.node_offset >> 32U) - ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + (uint32_t)(params.node_offset >> 32U) ^ + ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.h b/Modules/_hacl/Hacl_Hash_Blake2s.h index fbf8cff5cd1..2582a3d34e3 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s.h +++ b/Modules/_hacl/Hacl_Hash_Blake2s.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_Blake2s_H -#define __Hacl_Hash_Blake2s_H +#ifndef Hacl_Hash_Blake2s_H +#define Hacl_Hash_Blake2s_H #if defined(__cplusplus) extern "C" { @@ -198,5 +198,5 @@ Hacl_Hash_Blake2s_hash_with_key_and_params( } #endif -#define __Hacl_Hash_Blake2s_H_DEFINED -#endif +#define Hacl_Hash_Blake2s_H_DEFINED +#endif /* Hacl_Hash_Blake2s_H */ diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c index 7e9cd79544f..fa46be045f3 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c @@ -271,13 +271,13 @@ Hacl_Hash_Blake2s_Simd128_init(Lib_IntVector_Intrinsics_vec128 *hash, uint32_t k uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)(uint8_t)nn - ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + (uint32_t)(uint8_t)nn ^ + ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); tmp[1U] = p.leaf_length; tmp[2U] = (uint32_t)p.node_offset; tmp[3U] = - (uint32_t)(p.node_offset >> 32U) - ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + (uint32_t)(p.node_offset >> 32U) ^ + ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -736,16 +736,14 @@ static Hacl_Hash_Blake2s_Simd128_state_t uint32_t x = r4; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ - ((uint32_t)pv.key_length - << 8U - ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ + ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -923,13 +921,13 @@ reset_raw(Hacl_Hash_Blake2s_Simd128_state_t *state, Hacl_Hash_Blake2b_params_and uint32_t x = r; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1061,8 +1059,7 @@ Hacl_Hash_Blake2s_Simd128_update( uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1116,8 +1113,7 @@ Hacl_Hash_Blake2s_Simd128_update( Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1147,8 +1143,7 @@ Hacl_Hash_Blake2s_Simd128_update( uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state10, @@ -1200,8 +1195,7 @@ Hacl_Hash_Blake2s_Simd128_update( Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1531,16 +1525,14 @@ Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)params.digest_length - ^ - ((uint32_t)params.key_length - << 8U - ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + (uint32_t)params.digest_length ^ + ((uint32_t)params.key_length << 8U ^ + ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); tmp[1U] = params.leaf_length; tmp[2U] = (uint32_t)params.node_offset; tmp[3U] = - (uint32_t)(params.node_offset >> 32U) - ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + (uint32_t)(params.node_offset >> 32U) ^ + ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h index 56456e6fbb3..4b3b4e4fbb5 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_Blake2s_Simd128_H -#define __Hacl_Hash_Blake2s_Simd128_H +#ifndef Hacl_Hash_Blake2s_Simd128_H +#define Hacl_Hash_Blake2s_Simd128_H #if defined(__cplusplus) extern "C" { @@ -206,5 +206,5 @@ Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( } #endif -#define __Hacl_Hash_Blake2s_Simd128_H_DEFINED -#endif +#define Hacl_Hash_Blake2s_Simd128_H_DEFINED +#endif /* Hacl_Hash_Blake2s_Simd128_H */ diff --git a/Modules/_hacl/Hacl_Hash_MD5.c b/Modules/_hacl/Hacl_Hash_MD5.c index 75ce8d2926e..305c75483c0 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.c +++ b/Modules/_hacl/Hacl_Hash_MD5.c @@ -66,11 +66,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti0 = _t[0U]; uint32_t v = - vb0 - + - ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) - << 7U - | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); + vb0 + + ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) << 7U | + (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); abcd[0U] = v; uint32_t va0 = abcd[3U]; uint32_t vb1 = abcd[0U]; @@ -82,11 +80,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti1 = _t[1U]; uint32_t v0 = - vb1 - + - ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) - << 12U - | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); + vb1 + + ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) << 12U | + (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); abcd[3U] = v0; uint32_t va1 = abcd[2U]; uint32_t vb2 = abcd[3U]; @@ -98,11 +94,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti2 = _t[2U]; uint32_t v1 = - vb2 - + - ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) - << 17U - | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); + vb2 + + ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) << 17U | + (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); abcd[2U] = v1; uint32_t va2 = abcd[1U]; uint32_t vb3 = abcd[2U]; @@ -114,11 +108,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti3 = _t[3U]; uint32_t v2 = - vb3 - + - ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) - << 22U - | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); + vb3 + + ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) << 22U | + (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); abcd[1U] = v2; uint32_t va3 = abcd[0U]; uint32_t vb4 = abcd[1U]; @@ -130,11 +122,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti4 = _t[4U]; uint32_t v3 = - vb4 - + - ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) - << 7U - | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); + vb4 + + ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) << 7U | + (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); abcd[0U] = v3; uint32_t va4 = abcd[3U]; uint32_t vb5 = abcd[0U]; @@ -146,11 +136,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti5 = _t[5U]; uint32_t v4 = - vb5 - + - ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) - << 12U - | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); + vb5 + + ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) << 12U | + (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); abcd[3U] = v4; uint32_t va5 = abcd[2U]; uint32_t vb6 = abcd[3U]; @@ -162,11 +150,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti6 = _t[6U]; uint32_t v5 = - vb6 - + - ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) - << 17U - | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); + vb6 + + ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) << 17U | + (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); abcd[2U] = v5; uint32_t va6 = abcd[1U]; uint32_t vb7 = abcd[2U]; @@ -178,11 +164,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti7 = _t[7U]; uint32_t v6 = - vb7 - + - ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) - << 22U - | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); + vb7 + + ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) << 22U | + (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); abcd[1U] = v6; uint32_t va7 = abcd[0U]; uint32_t vb8 = abcd[1U]; @@ -194,11 +178,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti8 = _t[8U]; uint32_t v7 = - vb8 - + - ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) - << 7U - | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); + vb8 + + ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) << 7U | + (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); abcd[0U] = v7; uint32_t va8 = abcd[3U]; uint32_t vb9 = abcd[0U]; @@ -210,11 +192,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti9 = _t[9U]; uint32_t v8 = - vb9 - + - ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) - << 12U - | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); + vb9 + + ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) << 12U | + (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); abcd[3U] = v8; uint32_t va9 = abcd[2U]; uint32_t vb10 = abcd[3U]; @@ -226,11 +206,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti10 = _t[10U]; uint32_t v9 = - vb10 - + - ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) - << 17U - | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); + vb10 + + ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) << 17U | + (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); abcd[2U] = v9; uint32_t va10 = abcd[1U]; uint32_t vb11 = abcd[2U]; @@ -242,11 +220,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti11 = _t[11U]; uint32_t v10 = - vb11 - + - ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) - << 22U - | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); + vb11 + + ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) << 22U | + (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); abcd[1U] = v10; uint32_t va11 = abcd[0U]; uint32_t vb12 = abcd[1U]; @@ -258,11 +234,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti12 = _t[12U]; uint32_t v11 = - vb12 - + - ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) - << 7U - | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); + vb12 + + ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) << 7U | + (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); abcd[0U] = v11; uint32_t va12 = abcd[3U]; uint32_t vb13 = abcd[0U]; @@ -274,11 +248,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti13 = _t[13U]; uint32_t v12 = - vb13 - + - ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) - << 12U - | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); + vb13 + + ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) << 12U | + (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); abcd[3U] = v12; uint32_t va13 = abcd[2U]; uint32_t vb14 = abcd[3U]; @@ -290,11 +262,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti14 = _t[14U]; uint32_t v13 = - vb14 - + - ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) - << 17U - | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); + vb14 + + ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) << 17U | + (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); abcd[2U] = v13; uint32_t va14 = abcd[1U]; uint32_t vb15 = abcd[2U]; @@ -306,11 +276,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti15 = _t[15U]; uint32_t v14 = - vb15 - + - ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) - << 22U - | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); + vb15 + + ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) << 22U | + (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); abcd[1U] = v14; uint32_t va15 = abcd[0U]; uint32_t vb16 = abcd[1U]; @@ -322,11 +290,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti16 = _t[16U]; uint32_t v15 = - vb16 - + - ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) - << 5U - | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); + vb16 + + ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) << 5U | + (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); abcd[0U] = v15; uint32_t va16 = abcd[3U]; uint32_t vb17 = abcd[0U]; @@ -338,11 +304,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti17 = _t[17U]; uint32_t v16 = - vb17 - + - ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) - << 9U - | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); + vb17 + + ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) << 9U | + (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); abcd[3U] = v16; uint32_t va17 = abcd[2U]; uint32_t vb18 = abcd[3U]; @@ -354,11 +318,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti18 = _t[18U]; uint32_t v17 = - vb18 - + - ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) - << 14U - | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); + vb18 + + ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) << 14U | + (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); abcd[2U] = v17; uint32_t va18 = abcd[1U]; uint32_t vb19 = abcd[2U]; @@ -370,11 +332,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti19 = _t[19U]; uint32_t v18 = - vb19 - + - ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) - << 20U - | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); + vb19 + + ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) << 20U | + (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); abcd[1U] = v18; uint32_t va19 = abcd[0U]; uint32_t vb20 = abcd[1U]; @@ -386,11 +346,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti20 = _t[20U]; uint32_t v19 = - vb20 - + - ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) - << 5U - | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); + vb20 + + ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) << 5U | + (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); abcd[0U] = v19; uint32_t va20 = abcd[3U]; uint32_t vb21 = abcd[0U]; @@ -402,11 +360,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti21 = _t[21U]; uint32_t v20 = - vb21 - + - ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) - << 9U - | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); + vb21 + + ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) << 9U | + (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); abcd[3U] = v20; uint32_t va21 = abcd[2U]; uint32_t vb22 = abcd[3U]; @@ -418,11 +374,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti22 = _t[22U]; uint32_t v21 = - vb22 - + - ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) - << 14U - | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); + vb22 + + ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) << 14U | + (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); abcd[2U] = v21; uint32_t va22 = abcd[1U]; uint32_t vb23 = abcd[2U]; @@ -434,11 +388,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti23 = _t[23U]; uint32_t v22 = - vb23 - + - ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) - << 20U - | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); + vb23 + + ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) << 20U | + (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); abcd[1U] = v22; uint32_t va23 = abcd[0U]; uint32_t vb24 = abcd[1U]; @@ -450,11 +402,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti24 = _t[24U]; uint32_t v23 = - vb24 - + - ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) - << 5U - | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); + vb24 + + ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) << 5U | + (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); abcd[0U] = v23; uint32_t va24 = abcd[3U]; uint32_t vb25 = abcd[0U]; @@ -466,11 +416,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti25 = _t[25U]; uint32_t v24 = - vb25 - + - ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) - << 9U - | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); + vb25 + + ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) << 9U | + (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); abcd[3U] = v24; uint32_t va25 = abcd[2U]; uint32_t vb26 = abcd[3U]; @@ -482,11 +430,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti26 = _t[26U]; uint32_t v25 = - vb26 - + - ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) - << 14U - | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); + vb26 + + ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) << 14U | + (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); abcd[2U] = v25; uint32_t va26 = abcd[1U]; uint32_t vb27 = abcd[2U]; @@ -498,11 +444,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti27 = _t[27U]; uint32_t v26 = - vb27 - + - ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) - << 20U - | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); + vb27 + + ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) << 20U | + (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); abcd[1U] = v26; uint32_t va27 = abcd[0U]; uint32_t vb28 = abcd[1U]; @@ -514,11 +458,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti28 = _t[28U]; uint32_t v27 = - vb28 - + - ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) - << 5U - | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); + vb28 + + ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) << 5U | + (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); abcd[0U] = v27; uint32_t va28 = abcd[3U]; uint32_t vb29 = abcd[0U]; @@ -530,11 +472,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti29 = _t[29U]; uint32_t v28 = - vb29 - + - ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) - << 9U - | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); + vb29 + + ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) << 9U | + (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); abcd[3U] = v28; uint32_t va29 = abcd[2U]; uint32_t vb30 = abcd[3U]; @@ -546,11 +486,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti30 = _t[30U]; uint32_t v29 = - vb30 - + - ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) - << 14U - | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); + vb30 + + ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) << 14U | + (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); abcd[2U] = v29; uint32_t va30 = abcd[1U]; uint32_t vb31 = abcd[2U]; @@ -562,11 +500,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti31 = _t[31U]; uint32_t v30 = - vb31 - + - ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) - << 20U - | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); + vb31 + + ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) << 20U | + (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); abcd[1U] = v30; uint32_t va31 = abcd[0U]; uint32_t vb32 = abcd[1U]; @@ -578,11 +514,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti32 = _t[32U]; uint32_t v31 = - vb32 - + - ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) - << 4U - | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); + vb32 + + ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) << 4U | + (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); abcd[0U] = v31; uint32_t va32 = abcd[3U]; uint32_t vb33 = abcd[0U]; @@ -594,11 +528,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti33 = _t[33U]; uint32_t v32 = - vb33 - + - ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) - << 11U - | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); + vb33 + + ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) << 11U | + (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); abcd[3U] = v32; uint32_t va33 = abcd[2U]; uint32_t vb34 = abcd[3U]; @@ -610,11 +542,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti34 = _t[34U]; uint32_t v33 = - vb34 - + - ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) - << 16U - | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); + vb34 + + ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) << 16U | + (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); abcd[2U] = v33; uint32_t va34 = abcd[1U]; uint32_t vb35 = abcd[2U]; @@ -626,11 +556,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti35 = _t[35U]; uint32_t v34 = - vb35 - + - ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) - << 23U - | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); + vb35 + + ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) << 23U | + (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); abcd[1U] = v34; uint32_t va35 = abcd[0U]; uint32_t vb36 = abcd[1U]; @@ -642,11 +570,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti36 = _t[36U]; uint32_t v35 = - vb36 - + - ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) - << 4U - | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); + vb36 + + ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) << 4U | + (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); abcd[0U] = v35; uint32_t va36 = abcd[3U]; uint32_t vb37 = abcd[0U]; @@ -658,11 +584,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti37 = _t[37U]; uint32_t v36 = - vb37 - + - ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) - << 11U - | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); + vb37 + + ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) << 11U | + (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); abcd[3U] = v36; uint32_t va37 = abcd[2U]; uint32_t vb38 = abcd[3U]; @@ -674,11 +598,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti38 = _t[38U]; uint32_t v37 = - vb38 - + - ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) - << 16U - | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); + vb38 + + ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) << 16U | + (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); abcd[2U] = v37; uint32_t va38 = abcd[1U]; uint32_t vb39 = abcd[2U]; @@ -690,11 +612,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti39 = _t[39U]; uint32_t v38 = - vb39 - + - ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) - << 23U - | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); + vb39 + + ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) << 23U | + (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); abcd[1U] = v38; uint32_t va39 = abcd[0U]; uint32_t vb40 = abcd[1U]; @@ -706,11 +626,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti40 = _t[40U]; uint32_t v39 = - vb40 - + - ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) - << 4U - | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); + vb40 + + ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) << 4U | + (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); abcd[0U] = v39; uint32_t va40 = abcd[3U]; uint32_t vb41 = abcd[0U]; @@ -722,11 +640,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti41 = _t[41U]; uint32_t v40 = - vb41 - + - ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) - << 11U - | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); + vb41 + + ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) << 11U | + (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); abcd[3U] = v40; uint32_t va41 = abcd[2U]; uint32_t vb42 = abcd[3U]; @@ -738,11 +654,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti42 = _t[42U]; uint32_t v41 = - vb42 - + - ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) - << 16U - | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); + vb42 + + ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) << 16U | + (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); abcd[2U] = v41; uint32_t va42 = abcd[1U]; uint32_t vb43 = abcd[2U]; @@ -754,11 +668,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti43 = _t[43U]; uint32_t v42 = - vb43 - + - ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) - << 23U - | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); + vb43 + + ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) << 23U | + (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); abcd[1U] = v42; uint32_t va43 = abcd[0U]; uint32_t vb44 = abcd[1U]; @@ -770,11 +682,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti44 = _t[44U]; uint32_t v43 = - vb44 - + - ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) - << 4U - | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); + vb44 + + ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) << 4U | + (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); abcd[0U] = v43; uint32_t va44 = abcd[3U]; uint32_t vb45 = abcd[0U]; @@ -786,11 +696,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti45 = _t[45U]; uint32_t v44 = - vb45 - + - ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) - << 11U - | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); + vb45 + + ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) << 11U | + (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); abcd[3U] = v44; uint32_t va45 = abcd[2U]; uint32_t vb46 = abcd[3U]; @@ -802,11 +710,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti46 = _t[46U]; uint32_t v45 = - vb46 - + - ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) - << 16U - | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); + vb46 + + ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) << 16U | + (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); abcd[2U] = v45; uint32_t va46 = abcd[1U]; uint32_t vb47 = abcd[2U]; @@ -818,11 +724,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti47 = _t[47U]; uint32_t v46 = - vb47 - + - ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) - << 23U - | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); + vb47 + + ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) << 23U | + (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); abcd[1U] = v46; uint32_t va47 = abcd[0U]; uint32_t vb48 = abcd[1U]; @@ -834,11 +738,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti48 = _t[48U]; uint32_t v47 = - vb48 - + - ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) - << 6U - | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); + vb48 + + ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) << 6U | + (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); abcd[0U] = v47; uint32_t va48 = abcd[3U]; uint32_t vb49 = abcd[0U]; @@ -850,11 +752,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti49 = _t[49U]; uint32_t v48 = - vb49 - + - ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) - << 10U - | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); + vb49 + + ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) << 10U | + (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); abcd[3U] = v48; uint32_t va49 = abcd[2U]; uint32_t vb50 = abcd[3U]; @@ -866,11 +766,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti50 = _t[50U]; uint32_t v49 = - vb50 - + - ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) - << 15U - | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); + vb50 + + ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) << 15U | + (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); abcd[2U] = v49; uint32_t va50 = abcd[1U]; uint32_t vb51 = abcd[2U]; @@ -882,11 +780,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti51 = _t[51U]; uint32_t v50 = - vb51 - + - ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) - << 21U - | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); + vb51 + + ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) << 21U | + (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); abcd[1U] = v50; uint32_t va51 = abcd[0U]; uint32_t vb52 = abcd[1U]; @@ -898,11 +794,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti52 = _t[52U]; uint32_t v51 = - vb52 - + - ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) - << 6U - | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); + vb52 + + ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) << 6U | + (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); abcd[0U] = v51; uint32_t va52 = abcd[3U]; uint32_t vb53 = abcd[0U]; @@ -914,11 +808,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti53 = _t[53U]; uint32_t v52 = - vb53 - + - ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) - << 10U - | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); + vb53 + + ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) << 10U | + (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); abcd[3U] = v52; uint32_t va53 = abcd[2U]; uint32_t vb54 = abcd[3U]; @@ -930,11 +822,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti54 = _t[54U]; uint32_t v53 = - vb54 - + - ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) - << 15U - | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); + vb54 + + ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) << 15U | + (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); abcd[2U] = v53; uint32_t va54 = abcd[1U]; uint32_t vb55 = abcd[2U]; @@ -946,11 +836,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti55 = _t[55U]; uint32_t v54 = - vb55 - + - ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) - << 21U - | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); + vb55 + + ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) << 21U | + (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); abcd[1U] = v54; uint32_t va55 = abcd[0U]; uint32_t vb56 = abcd[1U]; @@ -962,11 +850,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti56 = _t[56U]; uint32_t v55 = - vb56 - + - ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) - << 6U - | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); + vb56 + + ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) << 6U | + (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); abcd[0U] = v55; uint32_t va56 = abcd[3U]; uint32_t vb57 = abcd[0U]; @@ -978,11 +864,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti57 = _t[57U]; uint32_t v56 = - vb57 - + - ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) - << 10U - | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); + vb57 + + ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) << 10U | + (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); abcd[3U] = v56; uint32_t va57 = abcd[2U]; uint32_t vb58 = abcd[3U]; @@ -994,11 +878,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti58 = _t[58U]; uint32_t v57 = - vb58 - + - ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) - << 15U - | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); + vb58 + + ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) << 15U | + (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); abcd[2U] = v57; uint32_t va58 = abcd[1U]; uint32_t vb59 = abcd[2U]; @@ -1010,11 +892,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti59 = _t[59U]; uint32_t v58 = - vb59 - + - ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) - << 21U - | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); + vb59 + + ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) << 21U | + (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); abcd[1U] = v58; uint32_t va59 = abcd[0U]; uint32_t vb60 = abcd[1U]; @@ -1026,11 +906,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti60 = _t[60U]; uint32_t v59 = - vb60 - + - ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) - << 6U - | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); + vb60 + + ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) << 6U | + (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); abcd[0U] = v59; uint32_t va60 = abcd[3U]; uint32_t vb61 = abcd[0U]; @@ -1042,11 +920,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti61 = _t[61U]; uint32_t v60 = - vb61 - + - ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) - << 10U - | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); + vb61 + + ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) << 10U | + (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); abcd[3U] = v60; uint32_t va61 = abcd[2U]; uint32_t vb62 = abcd[3U]; @@ -1058,11 +934,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti62 = _t[62U]; uint32_t v61 = - vb62 - + - ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) - << 15U - | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); + vb62 + + ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) << 15U | + (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); abcd[2U] = v61; uint32_t va62 = abcd[1U]; uint32_t vb = abcd[2U]; @@ -1074,11 +948,8 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti = _t[63U]; uint32_t v62 = - vb - + - ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) - << 21U - | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); + vb + + ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) << 21U | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); abcd[1U] = v62; uint32_t a = abcd[0U]; uint32_t b = abcd[1U]; @@ -1282,8 +1153,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1328,8 +1198,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1359,8 +1228,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -1403,8 +1271,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_MD5.h b/Modules/_hacl/Hacl_Hash_MD5.h index 521c2addc50..b220bbf6890 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.h +++ b/Modules/_hacl/Hacl_Hash_MD5.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_MD5_H -#define __Hacl_Hash_MD5_H +#ifndef Hacl_Hash_MD5_H +#define Hacl_Hash_MD5_H #if defined(__cplusplus) extern "C" { @@ -62,5 +62,5 @@ void Hacl_Hash_MD5_hash(uint8_t *output, uint8_t *input, uint32_t input_len); } #endif -#define __Hacl_Hash_MD5_H_DEFINED -#endif +#define Hacl_Hash_MD5_H_DEFINED +#endif /* Hacl_Hash_MD5_H */ diff --git a/Modules/_hacl/Hacl_Hash_SHA1.c b/Modules/_hacl/Hacl_Hash_SHA1.c index 508e447bf27..97bd4f2204c 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.c +++ b/Modules/_hacl/Hacl_Hash_SHA1.c @@ -315,8 +315,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -361,8 +360,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -392,8 +390,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -436,8 +433,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_SHA1.h b/Modules/_hacl/Hacl_Hash_SHA1.h index 63ac83f9dce..7fc8c8cfd20 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/Hacl_Hash_SHA1.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_SHA1_H -#define __Hacl_Hash_SHA1_H +#ifndef Hacl_Hash_SHA1_H +#define Hacl_Hash_SHA1_H #if defined(__cplusplus) extern "C" { @@ -62,5 +62,5 @@ void Hacl_Hash_SHA1_hash(uint8_t *output, uint8_t *input, uint32_t input_len); } #endif -#define __Hacl_Hash_SHA1_H_DEFINED -#endif +#define Hacl_Hash_SHA1_H_DEFINED +#endif /* Hacl_Hash_SHA1_H */ diff --git a/Modules/_hacl/Hacl_Hash_SHA2.c b/Modules/_hacl/Hacl_Hash_SHA2.c index d612bafa72c..d2ee0c9ef51 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.c +++ b/Modules/_hacl/Hacl_Hash_SHA2.c @@ -100,15 +100,14 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) uint32_t k_e_t = k_t; uint32_t t1 = - h02 - + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) - + ((e0 & f0) ^ (~e0 & g0)) + h02 + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) + + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint32_t t2 = - ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) - + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); + ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) + + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint32_t a1 = t1 + t2; uint32_t b1 = a0; uint32_t c1 = b0; @@ -301,15 +300,14 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) uint64_t k_e_t = k_t; uint64_t t1 = - h02 - + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) - + ((e0 & f0) ^ (~e0 & g0)) + h02 + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) + + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint64_t t2 = - ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) - + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); + ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) + + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint64_t a1 = t1 + t2; uint64_t b1 = a0; uint64_t c1 = b0; @@ -639,8 +637,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -685,8 +682,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -716,8 +712,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -760,8 +755,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1205,8 +1199,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, @@ -1251,8 +1244,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, @@ -1282,8 +1274,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state10, @@ -1326,8 +1317,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_SHA2.h b/Modules/_hacl/Hacl_Hash_SHA2.h index a93138fb7ee..dd168b8aa1a 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/Hacl_Hash_SHA2.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_SHA2_H -#define __Hacl_Hash_SHA2_H +#ifndef Hacl_Hash_SHA2_H +#define Hacl_Hash_SHA2_H #if defined(__cplusplus) extern "C" { @@ -199,5 +199,5 @@ void Hacl_Hash_SHA2_hash_384(uint8_t *output, uint8_t *input, uint32_t input_len } #endif -#define __Hacl_Hash_SHA2_H_DEFINED -#endif +#define Hacl_Hash_SHA2_H_DEFINED +#endif /* Hacl_Hash_SHA2_H */ diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index 87638df9549..466d2b96c0c 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -866,8 +866,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ((Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, .total_len = total_len2 }); } else if (sz == 0U) @@ -910,8 +909,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, @@ -941,8 +939,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state10, @@ -972,10 +969,8 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint32_t ite; if ( - (uint64_t)(chunk_len - diff) - % (uint64_t)block_len(i) - == 0ULL - && (uint64_t)(chunk_len - diff) > 0ULL + (uint64_t)(chunk_len - diff) % (uint64_t)block_len(i) == 0ULL && + (uint64_t)(chunk_len - diff) > 0ULL ) { ite = block_len(i); @@ -994,8 +989,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, @@ -2422,9 +2416,7 @@ Hacl_Hash_SHA3_shake128_squeeze_nblocks( 5U, 1U, _C[i] = - state[i - + 0U] - ^ (state[i + 5U] ^ (state[i + 10U] ^ (state[i + 15U] ^ state[i + 20U])));); + state[i + 0U] ^ (state[i + 5U] ^ (state[i + 10U] ^ (state[i + 15U] ^ state[i + 20U])));); KRML_MAYBE_FOR5(i2, 0U, 5U, diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h index 65ec99ee3a3..df6ebe439e9 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_SHA3_H -#define __Hacl_Hash_SHA3_H +#ifndef Hacl_Hash_SHA3_H +#define Hacl_Hash_SHA3_H #if defined(__cplusplus) extern "C" { @@ -155,5 +155,5 @@ Hacl_Hash_SHA3_shake128_squeeze_nblocks( } #endif -#define __Hacl_Hash_SHA3_H_DEFINED -#endif +#define Hacl_Hash_SHA3_H_DEFINED +#endif /* Hacl_Hash_SHA3_H */ diff --git a/Modules/_hacl/Hacl_Streaming_HMAC.c b/Modules/_hacl/Hacl_Streaming_HMAC.c index d28b39792af..f89c00ee1ae 100644 --- a/Modules/_hacl/Hacl_Streaming_HMAC.c +++ b/Modules/_hacl/Hacl_Streaming_HMAC.c @@ -198,8 +198,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_MD5_a, { .case_MD5_a = s1 } }); + st[0U] = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_MD5_a, { .case_MD5_a = s1 } }); } if (st == NULL) { @@ -220,8 +219,8 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA1_a, { .case_SHA1_a = s1 } }); + st[0U] = + ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA1_a, { .case_SHA1_a = s1 } }); } if (st == NULL) { @@ -242,8 +241,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_224_a, @@ -270,8 +268,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_256_a, @@ -298,8 +295,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_384_a, @@ -326,8 +322,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_512_a, @@ -354,8 +349,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_224_a, @@ -382,8 +376,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_256_a, @@ -410,8 +403,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_384_a, @@ -438,8 +430,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_512_a, @@ -466,8 +457,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2S_a, @@ -495,8 +485,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2S_128_a, @@ -531,8 +520,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2B_a, @@ -560,8 +548,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2B_256_a, @@ -2059,8 +2046,8 @@ Hacl_Streaming_HMAC_update( Hacl_Streaming_HMAC_Definitions_index i1 = Hacl_Streaming_HMAC_index_of_state(block_state); if ( - (uint64_t)chunk_len - > max_input_len64(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - total_len + (uint64_t)chunk_len > + max_input_len64(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - total_len ) { return Hacl_Streaming_Types_MaximumLengthExceeded; @@ -2068,9 +2055,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz; if ( - total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len > 0ULL ) { @@ -2079,8 +2064,8 @@ Hacl_Streaming_HMAC_update( else { sz = - (uint32_t)(total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (chunk_len <= block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - sz) { @@ -2091,9 +2076,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2102,14 +2085,13 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2127,9 +2109,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2138,8 +2118,8 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (!(sz1 == 0U)) { @@ -2153,8 +2133,8 @@ Hacl_Streaming_HMAC_update( uint32_t ite; if ( - (uint64_t)chunk_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) + (uint64_t)chunk_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && (uint64_t)chunk_len > 0ULL ) @@ -2164,8 +2144,8 @@ Hacl_Streaming_HMAC_update( else { ite = - (uint32_t)((uint64_t)chunk_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)((uint64_t)chunk_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint32_t n_blocks = (chunk_len - ite) / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); @@ -2178,8 +2158,7 @@ Hacl_Streaming_HMAC_update( update_multi(s11, total_len1, data1, data1_len); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2200,9 +2179,8 @@ Hacl_Streaming_HMAC_update( uint32_t sz10; if ( - total_len10 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len10 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == + 0ULL && total_len10 > 0ULL ) { @@ -2211,14 +2189,13 @@ Hacl_Streaming_HMAC_update( else { sz10 = - (uint32_t)(total_len10 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len10 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state10, @@ -2233,9 +2210,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2244,8 +2219,8 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (!(sz1 == 0U)) { @@ -2259,8 +2234,8 @@ Hacl_Streaming_HMAC_update( uint32_t ite; if ( - (uint64_t)(chunk_len - diff) - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) + (uint64_t)(chunk_len - diff) % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL ) @@ -2270,13 +2245,12 @@ Hacl_Streaming_HMAC_update( else { ite = - (uint32_t)((uint64_t)(chunk_len - diff) - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)((uint64_t)(chunk_len - diff) % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint32_t n_blocks = - (chunk_len - diff - ite) - / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); + (chunk_len - diff - ite) / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); uint32_t data1_len = n_blocks * block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); uint32_t data2_len = chunk_len - diff - data1_len; @@ -2286,8 +2260,7 @@ Hacl_Streaming_HMAC_update( update_multi(s11, total_len1, data1, data1_len); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2324,9 +2297,7 @@ Hacl_Streaming_HMAC_digest( uint32_t r; if ( - total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len > 0ULL ) { @@ -2335,8 +2306,8 @@ Hacl_Streaming_HMAC_digest( else { r = - (uint32_t)(total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf_1 = buf_; Hacl_Agile_Hash_state_s *s110 = malloc_(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)); @@ -2404,9 +2375,13 @@ Hacl_Streaming_HMAC_digest( Hacl_Agile_Hash_state_s *s112 = tmp_block_state1.snd; update_multi(s112, prev_len, buf_multi, 0U); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_Agile_Hash_state_s *s11 = tmp_block_state1.snd; - update_last(s11, prev_len_last, buf_last, r); + Hacl_Agile_Hash_state_s *s113 = tmp_block_state1.snd; + update_last(s113, prev_len_last, buf_last, r); finish0(tmp_block_state1, output); + Hacl_Agile_Hash_state_s *s210 = tmp_block_state1.thd; + Hacl_Agile_Hash_state_s *s11 = tmp_block_state1.snd; + free_(s11); + free_(s210); return Hacl_Streaming_Types_Success; } KRML_HOST_EPRINTF("KaRaMeL abort at %s:%d\n%s\n", diff --git a/Modules/_hacl/Hacl_Streaming_HMAC.h b/Modules/_hacl/Hacl_Streaming_HMAC.h index a0806c02d0b..6c302b1f92d 100644 --- a/Modules/_hacl/Hacl_Streaming_HMAC.h +++ b/Modules/_hacl/Hacl_Streaming_HMAC.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Streaming_HMAC_H -#define __Hacl_Streaming_HMAC_H +#ifndef Hacl_Streaming_HMAC_H +#define Hacl_Streaming_HMAC_H #if defined(__cplusplus) extern "C" { @@ -130,5 +130,5 @@ Hacl_Streaming_HMAC_agile_state } #endif -#define __Hacl_Streaming_HMAC_H_DEFINED -#endif +#define Hacl_Streaming_HMAC_H_DEFINED +#endif /* Hacl_Streaming_HMAC_H */ diff --git a/Modules/_hacl/Hacl_Streaming_Types.h b/Modules/_hacl/Hacl_Streaming_Types.h index 5c497750793..cce25eb7946 100644 --- a/Modules/_hacl/Hacl_Streaming_Types.h +++ b/Modules/_hacl/Hacl_Streaming_Types.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Streaming_Types_H -#define __Hacl_Streaming_Types_H +#ifndef Hacl_Streaming_Types_H +#define Hacl_Streaming_Types_H #if defined(__cplusplus) extern "C" { @@ -68,5 +68,5 @@ typedef struct Hacl_Streaming_MD_state_64_s Hacl_Streaming_MD_state_64; } #endif -#define __Hacl_Streaming_Types_H_DEFINED -#endif +#define Hacl_Streaming_Types_H_DEFINED +#endif /* Hacl_Streaming_Types_H */ diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c index 28abd1aa4e2..f94e0e2254a 100644 --- a/Modules/_hacl/Lib_Memzero0.c +++ b/Modules/_hacl/Lib_Memzero0.c @@ -11,18 +11,18 @@ #if defined(__APPLE__) && defined(__MACH__) #include <AvailabilityMacros.h> // memset_s is available from macOS 10.9, iOS 7, watchOS 2, and on all tvOS and visionOS versions. -# if (defined(MAC_OS_X_VERSION_MIN_REQUIRED) && (MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_9)) -# define APPLE_HAS_MEMSET_S 1 -# elif (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && (__IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_7_0)) -# define APPLE_HAS_MEMSET_S 1 +# if (defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_X_VERSION_10_9) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_9)) +# define APPLE_HAS_MEMSET_S +# elif (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && defined(__IPHONE_7_0) && (__IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_7_0)) +# define APPLE_HAS_MEMSET_S # elif (defined(TARGET_OS_TV) && TARGET_OS_TV) -# define APPLE_HAS_MEMSET_S 1 -# elif (defined(__WATCH_OS_VERSION_MIN_REQUIRED) && (__WATCH_OS_VERSION_MIN_REQUIRED >= __WATCHOS_2_0)) -# define APPLE_HAS_MEMSET_S 1 +# define APPLE_HAS_MEMSET_S +# elif (defined(__WATCH_OS_VERSION_MIN_REQUIRED) && defined(__WATCHOS_2_0) && (__WATCH_OS_VERSION_MIN_REQUIRED >= __WATCHOS_2_0)) +# define APPLE_HAS_MEMSET_S # elif (defined(TARGET_OS_VISION) && TARGET_OS_VISION) -# define APPLE_HAS_MEMSET_S 1 +# define APPLE_HAS_MEMSET_S # else -# define APPLE_HAS_MEMSET_S 0 +# undef APPLE_HAS_MEMSET_S # endif #endif @@ -55,7 +55,7 @@ void Lib_Memzero0_memzero0(void *dst, uint64_t len) { #ifdef _WIN32 SecureZeroMemory(dst, len_); - #elif defined(__APPLE__) && defined(__MACH__) && APPLE_HAS_MEMSET_S + #elif defined(__APPLE__) && defined(__MACH__) && defined(APPLE_HAS_MEMSET_S) memset_s(dst, len_, 0, len_); #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__FreeBSD__) || defined(__OpenBSD__) explicit_bzero(dst, len_); diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h index d4a90220bea..7aa2a6d83f9 100644 --- a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -4,8 +4,8 @@ */ -#ifndef __FStar_UInt128_Verified_H -#define __FStar_UInt128_Verified_H +#ifndef FStar_UInt128_Verified_H +#define FStar_UInt128_Verified_H #include "FStar_UInt_8_16_32_64.h" #include <inttypes.h> @@ -257,11 +257,11 @@ FStar_UInt128_gte_mask(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) { FStar_UInt128_uint128 lit; lit.low = - (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) - | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) | + (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); lit.high = - (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) - | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) | + (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); return lit; } @@ -294,14 +294,12 @@ static inline FStar_UInt128_uint128 FStar_UInt128_mul32(uint64_t x, uint32_t y) { FStar_UInt128_uint128 lit; lit.low = - FStar_UInt128_u32_combine((x >> FStar_UInt128_u32_32) - * (uint64_t)y - + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32), + FStar_UInt128_u32_combine((x >> FStar_UInt128_u32_32) * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32), FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * (uint64_t)y)); lit.high = - ((x >> FStar_UInt128_u32_32) - * (uint64_t)y - + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32)) + ((x >> FStar_UInt128_u32_32) * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32)) >> FStar_UInt128_u32_32; return lit; } @@ -315,32 +313,23 @@ static inline FStar_UInt128_uint128 FStar_UInt128_mul_wide(uint64_t x, uint64_t { FStar_UInt128_uint128 lit; lit.low = - FStar_UInt128_u32_combine_(FStar_UInt128_u64_mod_32(x) - * (y >> FStar_UInt128_u32_32) - + - FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)), + FStar_UInt128_u32_combine_(FStar_UInt128_u64_mod_32(x) * (y >> FStar_UInt128_u32_32) + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)), FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y))); lit.high = - (x >> FStar_UInt128_u32_32) - * (y >> FStar_UInt128_u32_32) - + - (((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)) + (x >> FStar_UInt128_u32_32) * (y >> FStar_UInt128_u32_32) + + (((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)) >> FStar_UInt128_u32_32) + - ((FStar_UInt128_u64_mod_32(x) - * (y >> FStar_UInt128_u32_32) - + - FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32))) + ((FStar_UInt128_u64_mod_32(x) * (y >> FStar_UInt128_u32_32) + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32))) >> FStar_UInt128_u32_32); return lit; } -#define __FStar_UInt128_Verified_H_DEFINED -#endif +#define FStar_UInt128_Verified_H_DEFINED +#endif /* FStar_UInt128_Verified_H */ diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index 00be8083657..2720348bbb7 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -4,8 +4,8 @@ */ -#ifndef __FStar_UInt_8_16_32_64_H -#define __FStar_UInt_8_16_32_64_H +#ifndef FStar_UInt_8_16_32_64_H +#define FStar_UInt_8_16_32_64_H #include <inttypes.h> #include <stdbool.h> @@ -30,6 +30,8 @@ extern uint64_t FStar_UInt64_zero; extern uint64_t FStar_UInt64_one; +extern bool FStar_UInt64_ne(uint64_t a, uint64_t b); + extern uint64_t FStar_UInt64_minus(uint64_t a); extern uint32_t FStar_UInt64_n_minus_one; @@ -80,6 +82,8 @@ extern uint32_t FStar_UInt32_zero; extern uint32_t FStar_UInt32_one; +extern bool FStar_UInt32_ne(uint32_t a, uint32_t b); + extern uint32_t FStar_UInt32_minus(uint32_t a); extern uint32_t FStar_UInt32_n_minus_one; @@ -130,6 +134,8 @@ extern uint16_t FStar_UInt16_zero; extern uint16_t FStar_UInt16_one; +extern bool FStar_UInt16_ne(uint16_t a, uint16_t b); + extern uint16_t FStar_UInt16_minus(uint16_t a); extern uint32_t FStar_UInt16_n_minus_one; @@ -180,6 +186,8 @@ extern uint8_t FStar_UInt8_zero; extern uint8_t FStar_UInt8_one; +extern bool FStar_UInt8_ne(uint8_t a, uint8_t b); + extern uint8_t FStar_UInt8_minus(uint8_t a); extern uint32_t FStar_UInt8_n_minus_one; @@ -217,5 +225,5 @@ extern uint8_t FStar_UInt8_of_string(Prims_string uu___); typedef uint8_t FStar_UInt8_byte; -#define __FStar_UInt_8_16_32_64_H_DEFINED -#endif +#define FStar_UInt_8_16_32_64_H_DEFINED +#endif /* FStar_UInt_8_16_32_64_H */ diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index c592214634a..73555ab5ca1 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -1,8 +1,8 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. Licensed under the Apache 2.0 and MIT Licenses. */ -#ifndef __KRML_TARGET_H -#define __KRML_TARGET_H +#ifndef KRML_HEADER_TARGET_H +#define KRML_HEADER_TARGET_H #include <assert.h> #include <inttypes.h> @@ -12,6 +12,9 @@ #include <stdio.h> #include <stdlib.h> +typedef float float32_t; +typedef double float64_t; + /* Since KaRaMeL emits the inline keyword unconditionally, we follow the * guidelines at https://gcc.gnu.org/onlinedocs/gcc/Inline.html and make this * __inline__ to ensure the code compiles with -std=c90 and earlier. */ @@ -425,4 +428,4 @@ inline static int32_t krml_time(void) { #else # define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif -#endif +#endif /* KRML_HEADER_TARGET_H */ diff --git a/Modules/_hacl/include/krml/internal/types.h b/Modules/_hacl/include/krml/internal/types.h index 2280dfad48d..d96ed19fcca 100644 --- a/Modules/_hacl/include/krml/internal/types.h +++ b/Modules/_hacl/include/krml/internal/types.h @@ -92,7 +92,7 @@ typedef FStar_UInt128_uint128 FStar_UInt128_t, uint128_t; /* Avoid a circular loop: if this header is included via FStar_UInt8_16_32_64, * then don't bring the uint128 definitions into scope. */ -#ifndef __FStar_UInt_8_16_32_64_H +#ifndef FStar_UInt_8_16_32_64_H #if !defined(KRML_VERIFIED_UINT128) && defined(IS_MSVC64) #include "fstar_uint128_msvc.h" diff --git a/Modules/_hacl/include/krml/lowstar_endianness.h b/Modules/_hacl/include/krml/lowstar_endianness.h index af6b882cf25..5f706fa51aa 100644 --- a/Modules/_hacl/include/krml/lowstar_endianness.h +++ b/Modules/_hacl/include/krml/lowstar_endianness.h @@ -1,8 +1,8 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. Licensed under the Apache 2.0 and MIT Licenses. */ -#ifndef __LOWSTAR_ENDIANNESS_H -#define __LOWSTAR_ENDIANNESS_H +#ifndef KRML_HEADER_LOWSTAR_ENDIANNESS_H +#define KRML_HEADER_LOWSTAR_ENDIANNESS_H #include <string.h> #include <inttypes.h> @@ -228,4 +228,4 @@ inline static void store64(uint8_t *b, uint64_t i) { #define load128_be0 load128_be #define store128_be0 store128_be -#endif +#endif /* KRML_HEADER_LOWSTAR_ENDIANNESS_H */ diff --git a/Modules/_hacl/internal/Hacl_HMAC.h b/Modules/_hacl/internal/Hacl_HMAC.h index ad29d50760c..ef9f25f5d42 100644 --- a/Modules/_hacl/internal/Hacl_HMAC.h +++ b/Modules/_hacl/internal/Hacl_HMAC.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_HMAC_H -#define __internal_Hacl_HMAC_H +#ifndef internal_Hacl_HMAC_H +#define internal_Hacl_HMAC_H #if defined(__cplusplus) extern "C" { @@ -48,5 +48,5 @@ K___uint32_t_uint32_t; } #endif -#define __internal_Hacl_HMAC_H_DEFINED -#endif +#define internal_Hacl_HMAC_H_DEFINED +#endif /* internal_Hacl_HMAC_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2b.h b/Modules/_hacl/internal/Hacl_Hash_Blake2b.h index e74c320e073..7ca8e10e34c 100644 --- a/Modules/_hacl/internal/Hacl_Hash_Blake2b.h +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2b.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_Blake2b_H -#define __internal_Hacl_Hash_Blake2b_H +#ifndef internal_Hacl_Hash_Blake2b_H +#define internal_Hacl_Hash_Blake2b_H #if defined(__cplusplus) extern "C" { @@ -91,5 +91,5 @@ Hacl_Hash_Blake2b_state_t; } #endif -#define __internal_Hacl_Hash_Blake2b_H_DEFINED -#endif +#define internal_Hacl_Hash_Blake2b_H_DEFINED +#endif /* internal_Hacl_Hash_Blake2b_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h b/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h index 27633f22b24..6507d287922 100644 --- a/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_Blake2b_Simd256_H -#define __internal_Hacl_Hash_Blake2b_Simd256_H +#ifndef internal_Hacl_Hash_Blake2b_Simd256_H +#define internal_Hacl_Hash_Blake2b_Simd256_H #if defined(__cplusplus) extern "C" { @@ -134,5 +134,5 @@ Hacl_Hash_Blake2b_Simd256_state_t; } #endif -#define __internal_Hacl_Hash_Blake2b_Simd256_H_DEFINED -#endif +#define internal_Hacl_Hash_Blake2b_Simd256_H_DEFINED +#endif /* internal_Hacl_Hash_Blake2b_Simd256_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2s.h b/Modules/_hacl/internal/Hacl_Hash_Blake2s.h index 0c5781df8ce..5fa5bb34efc 100644 --- a/Modules/_hacl/internal/Hacl_Hash_Blake2s.h +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2s.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_Blake2s_H -#define __internal_Hacl_Hash_Blake2s_H +#ifndef internal_Hacl_Hash_Blake2s_H +#define internal_Hacl_Hash_Blake2s_H #if defined(__cplusplus) extern "C" { @@ -90,5 +90,5 @@ Hacl_Hash_Blake2s_state_t; } #endif -#define __internal_Hacl_Hash_Blake2s_H_DEFINED -#endif +#define internal_Hacl_Hash_Blake2s_H_DEFINED +#endif /* internal_Hacl_Hash_Blake2s_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h b/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h index 0f5db552d6c..3220cddac30 100644 --- a/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_Blake2s_Simd128_H -#define __internal_Hacl_Hash_Blake2s_Simd128_H +#ifndef internal_Hacl_Hash_Blake2s_Simd128_H +#define internal_Hacl_Hash_Blake2s_Simd128_H #if defined(__cplusplus) extern "C" { @@ -134,5 +134,5 @@ Hacl_Hash_Blake2s_Simd128_state_t; } #endif -#define __internal_Hacl_Hash_Blake2s_Simd128_H_DEFINED -#endif +#define internal_Hacl_Hash_Blake2s_Simd128_H_DEFINED +#endif /* internal_Hacl_Hash_Blake2s_Simd128_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_MD5.h b/Modules/_hacl/internal/Hacl_Hash_MD5.h index 7fe71a49c6d..d8761de99f5 100644 --- a/Modules/_hacl/internal/Hacl_Hash_MD5.h +++ b/Modules/_hacl/internal/Hacl_Hash_MD5.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_MD5_H -#define __internal_Hacl_Hash_MD5_H +#ifndef internal_Hacl_Hash_MD5_H +#define internal_Hacl_Hash_MD5_H #if defined(__cplusplus) extern "C" { @@ -53,5 +53,5 @@ void Hacl_Hash_MD5_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_ } #endif -#define __internal_Hacl_Hash_MD5_H_DEFINED -#endif +#define internal_Hacl_Hash_MD5_H_DEFINED +#endif /* internal_Hacl_Hash_MD5_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA1.h b/Modules/_hacl/internal/Hacl_Hash_SHA1.h index ed53be559fe..0e9f1c911f6 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA1.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_SHA1_H -#define __internal_Hacl_Hash_SHA1_H +#ifndef internal_Hacl_Hash_SHA1_H +#define internal_Hacl_Hash_SHA1_H #if defined(__cplusplus) extern "C" { @@ -52,5 +52,5 @@ void Hacl_Hash_SHA1_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input } #endif -#define __internal_Hacl_Hash_SHA1_H_DEFINED -#endif +#define internal_Hacl_Hash_SHA1_H_DEFINED +#endif /* internal_Hacl_Hash_SHA1_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA2.h b/Modules/_hacl/internal/Hacl_Hash_SHA2.h index 98498ee9376..e6750207980 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA2.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_SHA2_H -#define __internal_Hacl_Hash_SHA2_H +#ifndef internal_Hacl_Hash_SHA2_H +#define internal_Hacl_Hash_SHA2_H #if defined(__cplusplus) extern "C" { @@ -161,5 +161,5 @@ void Hacl_Hash_SHA2_sha384_finish(uint64_t *st, uint8_t *h); } #endif -#define __internal_Hacl_Hash_SHA2_H_DEFINED -#endif +#define internal_Hacl_Hash_SHA2_H_DEFINED +#endif /* internal_Hacl_Hash_SHA2_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h index e653c73b1d0..c08a4e7858b 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_SHA3_H -#define __internal_Hacl_Hash_SHA3_H +#ifndef internal_Hacl_Hash_SHA3_H +#define internal_Hacl_Hash_SHA3_H #if defined(__cplusplus) extern "C" { @@ -81,5 +81,5 @@ Hacl_Hash_SHA3_state_t; } #endif -#define __internal_Hacl_Hash_SHA3_H_DEFINED -#endif +#define internal_Hacl_Hash_SHA3_H_DEFINED +#endif /* internal_Hacl_Hash_SHA3_H */ diff --git a/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h b/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h index fb3a045cd5c..2726cc48478 100644 --- a/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h +++ b/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Impl_Blake2_Constants_H -#define __internal_Hacl_Impl_Blake2_Constants_H +#ifndef internal_Hacl_Impl_Blake2_Constants_H +#define internal_Hacl_Impl_Blake2_Constants_H #if defined(__cplusplus) extern "C" { @@ -69,5 +69,5 @@ Hacl_Hash_Blake2b_ivTable_B[8U] = } #endif -#define __internal_Hacl_Impl_Blake2_Constants_H_DEFINED -#endif +#define internal_Hacl_Impl_Blake2_Constants_H_DEFINED +#endif /* internal_Hacl_Impl_Blake2_Constants_H */ diff --git a/Modules/_hacl/internal/Hacl_Streaming_HMAC.h b/Modules/_hacl/internal/Hacl_Streaming_HMAC.h index acc4f399602..fb44f707463 100644 --- a/Modules/_hacl/internal/Hacl_Streaming_HMAC.h +++ b/Modules/_hacl/internal/Hacl_Streaming_HMAC.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Streaming_HMAC_H -#define __internal_Hacl_Streaming_HMAC_H +#ifndef internal_Hacl_Streaming_HMAC_H +#define internal_Hacl_Streaming_HMAC_H #if defined(__cplusplus) extern "C" { @@ -90,5 +90,5 @@ Hacl_Streaming_HMAC_agile_state; } #endif -#define __internal_Hacl_Streaming_HMAC_H_DEFINED -#endif +#define internal_Hacl_Streaming_HMAC_H_DEFINED +#endif /* internal_Hacl_Streaming_HMAC_H */ diff --git a/Modules/_hacl/internal/Hacl_Streaming_Types.h b/Modules/_hacl/internal/Hacl_Streaming_Types.h index fed3cbd4259..d6c4fc0d625 100644 --- a/Modules/_hacl/internal/Hacl_Streaming_Types.h +++ b/Modules/_hacl/internal/Hacl_Streaming_Types.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Streaming_Types_H -#define __internal_Hacl_Streaming_Types_H +#ifndef internal_Hacl_Streaming_Types_H +#define internal_Hacl_Streaming_Types_H #if defined(__cplusplus) extern "C" { @@ -83,5 +83,5 @@ Hacl_Streaming_MD_state_64; } #endif -#define __internal_Hacl_Streaming_Types_H_DEFINED -#endif +#define internal_Hacl_Streaming_Types_H_DEFINED +#endif /* internal_Hacl_Streaming_Types_H */ diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index 1c2f7fea5c8..d0b4500395e 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -2,119 +2,29 @@ #define _PYTHON_HACL_NAMESPACES_H /* - * C's excuse for namespaces: Use globally unique names to avoid linkage - * conflicts with builds linking or dynamically loading other code potentially - * using HACL* libraries. + * Use globally unique names to avoid linkage conflicts with builds linking + * or dynamically loading other code potentially using HACL* libraries. * - * Something like this to generate new entries for the list: - * - * nm *.o | grep Hacl | cut -c 20- | sort | uniq | grep -v _Py_LibHacl_ | egrep ^_ | sed 's/_\(.*\)/#define \1 _Py_LibHacl_\1/g' + * Assuming that the current working directory is Modules/_hacl, + * use the following command to generate a list of candidates: + + nm -j *.o | grep -i hacl | grep -P '^[a-zA-Z_][a-zA-Z0-9_]+' \ + | sed -e 's/^_Py_LibHacl_//g' \ + | sed 's/\(.*\)/#define \1 _Py_LibHacl_\1/g' \ + | sort -u + + * Compare the entries to add as follows: + + diff -y --suppress-common-lines \ + <(grep -P '^#define (?!_PY.+_H)' python_hacl_namespaces.h | sort -u) \ + <(nm -j *.o | grep -i hacl | grep -P '^[a-zA-Z_][a-zA-Z0-9_]+' \ + | sed -e 's/^_Py_LibHacl_//g' \ + | sed 's/\(.*\)/#define \1 _Py_LibHacl_\1/g' | sort -u) */ +// --- Utils ------------------------------------------------------------------ #define Lib_Memzero0_memzero0 _Py_LibHacl_Lib_Memzero0_memzero0 - -#define Hacl_Hash_SHA2_state_sha2_224_s _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_224_s -#define Hacl_Hash_SHA2_state_sha2_224 _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_224 -#define Hacl_Hash_SHA2_state_sha2_256 _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_256 -#define Hacl_Hash_SHA2_state_sha2_384_s _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_384_s -#define Hacl_Hash_SHA2_state_sha2_384 _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_384 -#define Hacl_Hash_SHA2_state_sha2_512 _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_512 -#define Hacl_Hash_SHA2_malloc_256 _Py_LibHacl_Hacl_Hash_SHA2_malloc_256 -#define Hacl_Hash_SHA2_malloc_224 _Py_LibHacl_Hacl_Hash_SHA2_malloc_224 -#define Hacl_Hash_SHA2_malloc_512 _Py_LibHacl_Hacl_Hash_SHA2_malloc_512 -#define Hacl_Hash_SHA2_malloc_384 _Py_LibHacl_Hacl_Hash_SHA2_malloc_384 -#define Hacl_Hash_SHA2_copy_256 _Py_LibHacl_Hacl_Hash_SHA2_copy_256 -#define Hacl_Hash_SHA2_copy_224 _Py_LibHacl_Hacl_Hash_SHA2_copy_224 -#define Hacl_Hash_SHA2_copy_512 _Py_LibHacl_Hacl_Hash_SHA2_copy_512 -#define Hacl_Hash_SHA2_copy_384 _Py_LibHacl_Hacl_Hash_SHA2_copy_384 -#define Hacl_Hash_SHA2_init_256 _Py_LibHacl_Hacl_Hash_SHA2_init_256 -#define Hacl_Hash_SHA2_init_224 _Py_LibHacl_Hacl_Hash_SHA2_init_224 -#define Hacl_Hash_SHA2_init_512 _Py_LibHacl_Hacl_Hash_SHA2_init_512 -#define Hacl_Hash_SHA2_init_384 _Py_LibHacl_Hacl_Hash_SHA2_init_384 -#define Hacl_SHA2_Scalar32_sha512_init _Py_LibHacl_Hacl_SHA2_Scalar32_sha512_init -#define Hacl_Hash_SHA2_update_256 _Py_LibHacl_Hacl_Hash_SHA2_update_256 -#define Hacl_Hash_SHA2_update_224 _Py_LibHacl_Hacl_Hash_SHA2_update_224 -#define Hacl_Hash_SHA2_update_512 _Py_LibHacl_Hacl_Hash_SHA2_update_512 -#define Hacl_Hash_SHA2_update_384 _Py_LibHacl_Hacl_Hash_SHA2_update_384 -#define Hacl_Hash_SHA2_digest_256 _Py_LibHacl_Hacl_Hash_SHA2_digest_256 -#define Hacl_Hash_SHA2_digest_224 _Py_LibHacl_Hacl_Hash_SHA2_digest_224 -#define Hacl_Hash_SHA2_digest_512 _Py_LibHacl_Hacl_Hash_SHA2_digest_512 -#define Hacl_Hash_SHA2_digest_384 _Py_LibHacl_Hacl_Hash_SHA2_digest_384 -#define Hacl_Hash_SHA2_free_256 _Py_LibHacl_Hacl_Hash_SHA2_free_256 -#define Hacl_Hash_SHA2_free_224 _Py_LibHacl_Hacl_Hash_SHA2_free_224 -#define Hacl_Hash_SHA2_free_512 _Py_LibHacl_Hacl_Hash_SHA2_free_512 -#define Hacl_Hash_SHA2_free_384 _Py_LibHacl_Hacl_Hash_SHA2_free_384 -#define Hacl_Hash_SHA2_sha256 _Py_LibHacl_Hacl_Hash_SHA2_sha256 -#define Hacl_Hash_SHA2_sha224 _Py_LibHacl_Hacl_Hash_SHA2_sha224 -#define Hacl_Hash_SHA2_sha512 _Py_LibHacl_Hacl_Hash_SHA2_sha512 -#define Hacl_Hash_SHA2_sha384 _Py_LibHacl_Hacl_Hash_SHA2_sha384 - -#define Hacl_Hash_MD5_malloc _Py_LibHacl_Hacl_Hash_MD5_malloc -#define Hacl_Hash_MD5_init _Py_LibHacl_Hacl_Hash_MD5_init -#define Hacl_Hash_MD5_update _Py_LibHacl_Hacl_Hash_MD5_update -#define Hacl_Hash_MD5_digest _Py_LibHacl_Hacl_Hash_MD5_digest -#define Hacl_Hash_MD5_free _Py_LibHacl_Hacl_Hash_MD5_free -#define Hacl_Hash_MD5_copy _Py_LibHacl_Hacl_Hash_MD5_copy -#define Hacl_Hash_MD5_hash _Py_LibHacl_Hacl_Hash_MD5_hash - -#define Hacl_Hash_SHA1_malloc _Py_LibHacl_Hacl_Hash_SHA1_malloc -#define Hacl_Hash_SHA1_init _Py_LibHacl_Hacl_Hash_SHA1_init -#define Hacl_Hash_SHA1_update _Py_LibHacl_Hacl_Hash_SHA1_update -#define Hacl_Hash_SHA1_digest _Py_LibHacl_Hacl_Hash_SHA1_digest -#define Hacl_Hash_SHA1_free _Py_LibHacl_Hacl_Hash_SHA1_free -#define Hacl_Hash_SHA1_copy _Py_LibHacl_Hacl_Hash_SHA1_copy -#define Hacl_Hash_SHA1_hash _Py_LibHacl_Hacl_Hash_SHA1_hash - -#define Hacl_Hash_SHA3_update_last_sha3 _Py_LibHacl_Hacl_Hash_SHA3_update_last_sha3 -#define Hacl_Hash_SHA3_update_multi_sha3 _Py_LibHacl_Hacl_Hash_SHA3_update_multi_sha3 -#define Hacl_Impl_SHA3_absorb_inner _Py_LibHacl_Hacl_Impl_SHA3_absorb_inner -#define Hacl_Impl_SHA3_keccak _Py_LibHacl_Hacl_Impl_SHA3_keccak -#define Hacl_Impl_SHA3_loadState _Py_LibHacl_Hacl_Impl_SHA3_loadState -#define Hacl_Impl_SHA3_squeeze _Py_LibHacl_Hacl_Impl_SHA3_squeeze -#define Hacl_Impl_SHA3_state_permute _Py_LibHacl_Hacl_Impl_SHA3_state_permute -#define Hacl_SHA3_sha3_224 _Py_LibHacl_Hacl_SHA3_sha3_224 -#define Hacl_SHA3_sha3_256 _Py_LibHacl_Hacl_SHA3_sha3_256 -#define Hacl_SHA3_sha3_384 _Py_LibHacl_Hacl_SHA3_sha3_384 -#define Hacl_SHA3_sha3_512 _Py_LibHacl_Hacl_SHA3_sha3_512 -#define Hacl_SHA3_shake128_hacl _Py_LibHacl_Hacl_SHA3_shake128_hacl -#define Hacl_SHA3_shake256_hacl _Py_LibHacl_Hacl_SHA3_shake256_hacl -#define Hacl_Hash_SHA3_block_len _Py_LibHacl_Hacl_Hash_SHA3_block_len -#define Hacl_Hash_SHA3_copy _Py_LibHacl_Hacl_Hash_SHA3_copy -#define Hacl_Hash_SHA3_digest _Py_LibHacl_Hacl_Hash_SHA3_digest -#define Hacl_Hash_SHA3_free _Py_LibHacl_Hacl_Hash_SHA3_free -#define Hacl_Hash_SHA3_get_alg _Py_LibHacl_Hacl_Hash_SHA3_get_alg -#define Hacl_Hash_SHA3_hash_len _Py_LibHacl_Hacl_Hash_SHA3_hash_len -#define Hacl_Hash_SHA3_is_shake _Py_LibHacl_Hacl_Hash_SHA3_is_shake -#define Hacl_Hash_SHA3_init_ _Py_LibHacl_Hacl_Hash_SHA3_init_ -#define Hacl_Hash_SHA3_malloc _Py_LibHacl_Hacl_Hash_SHA3_malloc -#define Hacl_Hash_SHA3_reset _Py_LibHacl_Hacl_Hash_SHA3_reset -#define Hacl_Hash_SHA3_update _Py_LibHacl_Hacl_Hash_SHA3_update -#define Hacl_Hash_SHA3_squeeze _Py_LibHacl_Hacl_Hash_SHA3_squeeze - -#define Hacl_Hash_Blake2b_Simd256_copy _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_copy -#define Hacl_Hash_Blake2b_Simd256_copy_internal_state _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_copy_internal_state -#define Hacl_Hash_Blake2b_Simd256_digest _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_digest -#define Hacl_Hash_Blake2b_Simd256_finish _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_finish -#define Hacl_Hash_Blake2b_Simd256_free _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_free -#define Hacl_Hash_Blake2b_Simd256_hash_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_hash_with_key -#define Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params -#define Hacl_Hash_Blake2b_Simd256_info _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_info -#define Hacl_Hash_Blake2b_Simd256_init _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_init -#define Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32 _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32 -#define Hacl_Hash_Blake2b_Simd256_malloc _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc -#define Hacl_Hash_Blake2b_Simd256_malloc_internal_state_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_internal_state_with_key -#define Hacl_Hash_Blake2b_Simd256_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_with_key -#define Hacl_Hash_Blake2b_Simd256_malloc_with_key0 _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_with_key0 -#define Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key -#define Hacl_Hash_Blake2b_Simd256_reset _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_reset -#define Hacl_Hash_Blake2b_Simd256_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_reset_with_key -#define Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params -#define Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32 _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32 -#define Hacl_Hash_Blake2b_Simd256_update _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update -#define Hacl_Hash_Blake2b_Simd256_update_last _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_last -#define Hacl_Hash_Blake2b_Simd256_update_last_no_inline _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_last_no_inline -#define Hacl_Hash_Blake2b_Simd256_update_multi _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_multi -#define Hacl_Hash_Blake2b_Simd256_update_multi_no_inline _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_multi_no_inline +// --- HASH-BLAKE-2b ---------------------------------------------------------- #define Hacl_Hash_Blake2b_copy _Py_LibHacl_Hacl_Hash_Blake2b_copy #define Hacl_Hash_Blake2b_digest _Py_LibHacl_Hacl_Hash_Blake2b_digest #define Hacl_Hash_Blake2b_finish _Py_LibHacl_Hacl_Hash_Blake2b_finish @@ -129,33 +39,33 @@ #define Hacl_Hash_Blake2b_reset _Py_LibHacl_Hacl_Hash_Blake2b_reset #define Hacl_Hash_Blake2b_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2b_reset_with_key #define Hacl_Hash_Blake2b_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_reset_with_key_and_params +#define Hacl_Hash_Blake2b_Simd256_copy _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_copy +#define Hacl_Hash_Blake2b_Simd256_copy_internal_state _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_copy_internal_state +#define Hacl_Hash_Blake2b_Simd256_digest _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_digest +#define Hacl_Hash_Blake2b_Simd256_finish _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_finish +#define Hacl_Hash_Blake2b_Simd256_free _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_free +#define Hacl_Hash_Blake2b_Simd256_hash_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_hash_with_key +#define Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params +#define Hacl_Hash_Blake2b_Simd256_info _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_info +#define Hacl_Hash_Blake2b_Simd256_init _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_init +#define Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32 _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_load_state256b_from_state32 +#define Hacl_Hash_Blake2b_Simd256_malloc _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc +#define Hacl_Hash_Blake2b_Simd256_malloc_internal_state_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_internal_state_with_key +#define Hacl_Hash_Blake2b_Simd256_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_with_key +#define Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key +#define Hacl_Hash_Blake2b_Simd256_reset _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_reset +#define Hacl_Hash_Blake2b_Simd256_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_reset_with_key +#define Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_reset_with_key_and_params +#define Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32 _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_store_state256b_to_state32 +#define Hacl_Hash_Blake2b_Simd256_update _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update +#define Hacl_Hash_Blake2b_Simd256_update_last _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_last +#define Hacl_Hash_Blake2b_Simd256_update_last_no_inline _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_last_no_inline +#define Hacl_Hash_Blake2b_Simd256_update_multi _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_multi +#define Hacl_Hash_Blake2b_Simd256_update_multi_no_inline _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_multi_no_inline #define Hacl_Hash_Blake2b_update _Py_LibHacl_Hacl_Hash_Blake2b_update #define Hacl_Hash_Blake2b_update_last _Py_LibHacl_Hacl_Hash_Blake2b_update_last #define Hacl_Hash_Blake2b_update_multi _Py_LibHacl_Hacl_Hash_Blake2b_update_multi -#define Hacl_Hash_Blake2s_Simd128_copy _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_copy -#define Hacl_Hash_Blake2s_Simd128_copy_internal_state _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_copy_internal_state -#define Hacl_Hash_Blake2s_Simd128_digest _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_digest -#define Hacl_Hash_Blake2s_Simd128_finish _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_finish -#define Hacl_Hash_Blake2s_Simd128_free _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_free -#define Hacl_Hash_Blake2s_Simd128_hash_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_hash_with_key -#define Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params -#define Hacl_Hash_Blake2s_Simd128_info _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_info -#define Hacl_Hash_Blake2s_Simd128_init _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_init -#define Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32 _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32 -#define Hacl_Hash_Blake2s_Simd128_malloc _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc -#define Hacl_Hash_Blake2s_Simd128_malloc_internal_state_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_internal_state_with_key -#define Hacl_Hash_Blake2s_Simd128_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_with_key -#define Hacl_Hash_Blake2s_Simd128_malloc_with_key0 _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_with_key0 -#define Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key -#define Hacl_Hash_Blake2s_Simd128_reset _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_reset -#define Hacl_Hash_Blake2s_Simd128_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_reset_with_key -#define Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params -#define Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32 _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32 -#define Hacl_Hash_Blake2s_Simd128_update _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update -#define Hacl_Hash_Blake2s_Simd128_update_last _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_last -#define Hacl_Hash_Blake2s_Simd128_update_last_no_inline _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_last_no_inline -#define Hacl_Hash_Blake2s_Simd128_update_multi _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_multi -#define Hacl_Hash_Blake2s_Simd128_update_multi_no_inline _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_multi_no_inline +// --- HASH-BLAKE-2s ---------------------------------------------------------- #define Hacl_Hash_Blake2s_copy _Py_LibHacl_Hacl_Hash_Blake2s_copy #define Hacl_Hash_Blake2s_digest _Py_LibHacl_Hacl_Hash_Blake2s_digest #define Hacl_Hash_Blake2s_finish _Py_LibHacl_Hacl_Hash_Blake2s_finish @@ -170,23 +80,77 @@ #define Hacl_Hash_Blake2s_reset _Py_LibHacl_Hacl_Hash_Blake2s_reset #define Hacl_Hash_Blake2s_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2s_reset_with_key #define Hacl_Hash_Blake2s_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_reset_with_key_and_params +#define Hacl_Hash_Blake2s_Simd128_copy _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_copy +#define Hacl_Hash_Blake2s_Simd128_copy_internal_state _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_copy_internal_state +#define Hacl_Hash_Blake2s_Simd128_digest _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_digest +#define Hacl_Hash_Blake2s_Simd128_finish _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_finish +#define Hacl_Hash_Blake2s_Simd128_free _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_free +#define Hacl_Hash_Blake2s_Simd128_hash_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_hash_with_key +#define Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params +#define Hacl_Hash_Blake2s_Simd128_info _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_info +#define Hacl_Hash_Blake2s_Simd128_init _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_init +#define Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32 _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_load_state128s_from_state32 +#define Hacl_Hash_Blake2s_Simd128_malloc _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc +#define Hacl_Hash_Blake2s_Simd128_malloc_internal_state_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_internal_state_with_key +#define Hacl_Hash_Blake2s_Simd128_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_with_key +#define Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key +#define Hacl_Hash_Blake2s_Simd128_reset _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_reset +#define Hacl_Hash_Blake2s_Simd128_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_reset_with_key +#define Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_reset_with_key_and_params +#define Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32 _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_store_state128s_to_state32 +#define Hacl_Hash_Blake2s_Simd128_update _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update +#define Hacl_Hash_Blake2s_Simd128_update_last _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_last +#define Hacl_Hash_Blake2s_Simd128_update_last_no_inline _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_last_no_inline +#define Hacl_Hash_Blake2s_Simd128_update_multi _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_multi +#define Hacl_Hash_Blake2s_Simd128_update_multi_no_inline _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_multi_no_inline #define Hacl_Hash_Blake2s_update _Py_LibHacl_Hacl_Hash_Blake2s_update #define Hacl_Hash_Blake2s_update_last _Py_LibHacl_Hacl_Hash_Blake2s_update_last #define Hacl_Hash_Blake2s_update_multi _Py_LibHacl_Hacl_Hash_Blake2s_update_multi +// --- HASH-MD5 --------------------------------------------------------------- +#define Hacl_Hash_MD5_copy _Py_LibHacl_Hacl_Hash_MD5_copy +#define Hacl_Hash_MD5_digest _Py_LibHacl_Hacl_Hash_MD5_digest #define Hacl_Hash_MD5_finish _Py_LibHacl_Hacl_Hash_MD5_finish +#define Hacl_Hash_MD5_free _Py_LibHacl_Hacl_Hash_MD5_free +#define Hacl_Hash_MD5_hash _Py_LibHacl_Hacl_Hash_MD5_hash #define Hacl_Hash_MD5_hash_oneshot _Py_LibHacl_Hacl_Hash_MD5_hash_oneshot +#define Hacl_Hash_MD5_init _Py_LibHacl_Hacl_Hash_MD5_init +#define Hacl_Hash_MD5_malloc _Py_LibHacl_Hacl_Hash_MD5_malloc #define Hacl_Hash_MD5_reset _Py_LibHacl_Hacl_Hash_MD5_reset +#define Hacl_Hash_MD5_update _Py_LibHacl_Hacl_Hash_MD5_update #define Hacl_Hash_MD5_update_last _Py_LibHacl_Hacl_Hash_MD5_update_last #define Hacl_Hash_MD5_update_multi _Py_LibHacl_Hacl_Hash_MD5_update_multi +// --- HASH-SHA-1 ------------------------------------------------------------- +#define Hacl_Hash_SHA1_copy _Py_LibHacl_Hacl_Hash_SHA1_copy +#define Hacl_Hash_SHA1_digest _Py_LibHacl_Hacl_Hash_SHA1_digest #define Hacl_Hash_SHA1_finish _Py_LibHacl_Hacl_Hash_SHA1_finish +#define Hacl_Hash_SHA1_free _Py_LibHacl_Hacl_Hash_SHA1_free +#define Hacl_Hash_SHA1_hash _Py_LibHacl_Hacl_Hash_SHA1_hash #define Hacl_Hash_SHA1_hash_oneshot _Py_LibHacl_Hacl_Hash_SHA1_hash_oneshot +#define Hacl_Hash_SHA1_init _Py_LibHacl_Hacl_Hash_SHA1_init +#define Hacl_Hash_SHA1_malloc _Py_LibHacl_Hacl_Hash_SHA1_malloc #define Hacl_Hash_SHA1_reset _Py_LibHacl_Hacl_Hash_SHA1_reset +#define Hacl_Hash_SHA1_update _Py_LibHacl_Hacl_Hash_SHA1_update #define Hacl_Hash_SHA1_update_last _Py_LibHacl_Hacl_Hash_SHA1_update_last #define Hacl_Hash_SHA1_update_multi _Py_LibHacl_Hacl_Hash_SHA1_update_multi +// --- HASH-SHA-2 ------------------------------------------------------------- +#define Hacl_Hash_SHA2_copy_256 _Py_LibHacl_Hacl_Hash_SHA2_copy_256 +#define Hacl_Hash_SHA2_copy_512 _Py_LibHacl_Hacl_Hash_SHA2_copy_512 +#define Hacl_Hash_SHA2_digest_224 _Py_LibHacl_Hacl_Hash_SHA2_digest_224 +#define Hacl_Hash_SHA2_digest_256 _Py_LibHacl_Hacl_Hash_SHA2_digest_256 +#define Hacl_Hash_SHA2_digest_384 _Py_LibHacl_Hacl_Hash_SHA2_digest_384 +#define Hacl_Hash_SHA2_digest_512 _Py_LibHacl_Hacl_Hash_SHA2_digest_512 +#define Hacl_Hash_SHA2_free_224 _Py_LibHacl_Hacl_Hash_SHA2_free_224 +#define Hacl_Hash_SHA2_free_256 _Py_LibHacl_Hacl_Hash_SHA2_free_256 +#define Hacl_Hash_SHA2_free_384 _Py_LibHacl_Hacl_Hash_SHA2_free_384 +#define Hacl_Hash_SHA2_free_512 _Py_LibHacl_Hacl_Hash_SHA2_free_512 #define Hacl_Hash_SHA2_hash_224 _Py_LibHacl_Hacl_Hash_SHA2_hash_224 #define Hacl_Hash_SHA2_hash_256 _Py_LibHacl_Hacl_Hash_SHA2_hash_256 #define Hacl_Hash_SHA2_hash_384 _Py_LibHacl_Hacl_Hash_SHA2_hash_384 #define Hacl_Hash_SHA2_hash_512 _Py_LibHacl_Hacl_Hash_SHA2_hash_512 +#define Hacl_Hash_SHA2_malloc_224 _Py_LibHacl_Hacl_Hash_SHA2_malloc_224 +#define Hacl_Hash_SHA2_malloc_256 _Py_LibHacl_Hacl_Hash_SHA2_malloc_256 +#define Hacl_Hash_SHA2_malloc_384 _Py_LibHacl_Hacl_Hash_SHA2_malloc_384 +#define Hacl_Hash_SHA2_malloc_512 _Py_LibHacl_Hacl_Hash_SHA2_malloc_512 #define Hacl_Hash_SHA2_reset_224 _Py_LibHacl_Hacl_Hash_SHA2_reset_224 #define Hacl_Hash_SHA2_reset_256 _Py_LibHacl_Hacl_Hash_SHA2_reset_256 #define Hacl_Hash_SHA2_reset_384 _Py_LibHacl_Hacl_Hash_SHA2_reset_384 @@ -207,10 +171,25 @@ #define Hacl_Hash_SHA2_sha512_init _Py_LibHacl_Hacl_Hash_SHA2_sha512_init #define Hacl_Hash_SHA2_sha512_update_last _Py_LibHacl_Hacl_Hash_SHA2_sha512_update_last #define Hacl_Hash_SHA2_sha512_update_nblocks _Py_LibHacl_Hacl_Hash_SHA2_sha512_update_nblocks +#define Hacl_Hash_SHA2_update_224 _Py_LibHacl_Hacl_Hash_SHA2_update_224 +#define Hacl_Hash_SHA2_update_256 _Py_LibHacl_Hacl_Hash_SHA2_update_256 +#define Hacl_Hash_SHA2_update_384 _Py_LibHacl_Hacl_Hash_SHA2_update_384 +#define Hacl_Hash_SHA2_update_512 _Py_LibHacl_Hacl_Hash_SHA2_update_512 +// --- HASH-SHA-3 ------------------------------------------------------------- #define Hacl_Hash_SHA3_absorb_inner_32 _Py_LibHacl_Hacl_Hash_SHA3_absorb_inner_32 +#define Hacl_Hash_SHA3_block_len _Py_LibHacl_Hacl_Hash_SHA3_block_len +#define Hacl_Hash_SHA3_copy _Py_LibHacl_Hacl_Hash_SHA3_copy +#define Hacl_Hash_SHA3_digest _Py_LibHacl_Hacl_Hash_SHA3_digest +#define Hacl_Hash_SHA3_free _Py_LibHacl_Hacl_Hash_SHA3_free +#define Hacl_Hash_SHA3_get_alg _Py_LibHacl_Hacl_Hash_SHA3_get_alg +#define Hacl_Hash_SHA3_hash_len _Py_LibHacl_Hacl_Hash_SHA3_hash_len +#define Hacl_Hash_SHA3_init_ _Py_LibHacl_Hacl_Hash_SHA3_init_ +#define Hacl_Hash_SHA3_is_shake _Py_LibHacl_Hacl_Hash_SHA3_is_shake #define Hacl_Hash_SHA3_keccak_piln _Py_LibHacl_Hacl_Hash_SHA3_keccak_piln #define Hacl_Hash_SHA3_keccak_rndc _Py_LibHacl_Hacl_Hash_SHA3_keccak_rndc #define Hacl_Hash_SHA3_keccak_rotc _Py_LibHacl_Hacl_Hash_SHA3_keccak_rotc +#define Hacl_Hash_SHA3_malloc _Py_LibHacl_Hacl_Hash_SHA3_malloc +#define Hacl_Hash_SHA3_reset _Py_LibHacl_Hacl_Hash_SHA3_reset #define Hacl_Hash_SHA3_sha3_224 _Py_LibHacl_Hacl_Hash_SHA3_sha3_224 #define Hacl_Hash_SHA3_sha3_256 _Py_LibHacl_Hacl_Hash_SHA3_sha3_256 #define Hacl_Hash_SHA3_sha3_384 _Py_LibHacl_Hacl_Hash_SHA3_sha3_384 @@ -220,37 +199,39 @@ #define Hacl_Hash_SHA3_shake128_absorb_nblocks _Py_LibHacl_Hacl_Hash_SHA3_shake128_absorb_nblocks #define Hacl_Hash_SHA3_shake128_squeeze_nblocks _Py_LibHacl_Hacl_Hash_SHA3_shake128_squeeze_nblocks #define Hacl_Hash_SHA3_shake256 _Py_LibHacl_Hacl_Hash_SHA3_shake256 +#define Hacl_Hash_SHA3_squeeze _Py_LibHacl_Hacl_Hash_SHA3_squeeze #define Hacl_Hash_SHA3_state_free _Py_LibHacl_Hacl_Hash_SHA3_state_free #define Hacl_Hash_SHA3_state_malloc _Py_LibHacl_Hacl_Hash_SHA3_state_malloc - -// Streaming HMAC +#define Hacl_Hash_SHA3_update _Py_LibHacl_Hacl_Hash_SHA3_update +#define Hacl_Hash_SHA3_update_last_sha3 _Py_LibHacl_Hacl_Hash_SHA3_update_last_sha3 +#define Hacl_Hash_SHA3_update_multi_sha3 _Py_LibHacl_Hacl_Hash_SHA3_update_multi_sha3 +// --- STREAMING-MAC ---------------------------------------------------------- +#define Hacl_Streaming_HMAC_copy _Py_LibHacl_Hacl_Streaming_HMAC_copy +#define Hacl_Streaming_HMAC_digest _Py_LibHacl_Hacl_Streaming_HMAC_digest +#define Hacl_Streaming_HMAC_free _Py_LibHacl_Hacl_Streaming_HMAC_free +#define Hacl_Streaming_HMAC_get_impl _Py_LibHacl_Hacl_Streaming_HMAC_get_impl #define Hacl_Streaming_HMAC_index_of_state _Py_LibHacl_Hacl_Streaming_HMAC_index_of_state #define Hacl_Streaming_HMAC_malloc_ _Py_LibHacl_Hacl_Streaming_HMAC_malloc_ -#define Hacl_Streaming_HMAC_get_impl _Py_LibHacl_Hacl_Streaming_HMAC_get_impl #define Hacl_Streaming_HMAC_reset _Py_LibHacl_Hacl_Streaming_HMAC_reset -#define Hacl_Streaming_HMAC_update _Py_LibHacl_Hacl_Streaming_HMAC_update -#define Hacl_Streaming_HMAC_digest _Py_LibHacl_Hacl_Streaming_HMAC_digest -#define Hacl_Streaming_HMAC_copy _Py_LibHacl_Hacl_Streaming_HMAC_copy -#define Hacl_Streaming_HMAC_free _Py_LibHacl_Hacl_Streaming_HMAC_free #define Hacl_Streaming_HMAC_s1 _Py_LibHacl_Hacl_Streaming_HMAC_s1 #define Hacl_Streaming_HMAC_s2 _Py_LibHacl_Hacl_Streaming_HMAC_s2 - -// HMAC-MD5 +#define Hacl_Streaming_HMAC_update _Py_LibHacl_Hacl_Streaming_HMAC_update +// --- HMAC-MD5 --------------------------------------------------------------- #define Hacl_HMAC_compute_md5 _Py_LibHacl_Hacl_HMAC_compute_md5 -// HMAC-SHA-1 +// --- HMAC-SHA-1 ------------------------------------------------------------- #define Hacl_HMAC_compute_sha1 _Py_LibHacl_Hacl_HMAC_compute_sha1 -// HMAC-SHA-2 +// --- HMAC-SHA-2 ------------------------------------------------------------- #define Hacl_HMAC_compute_sha2_224 _Py_LibHacl_Hacl_HMAC_compute_sha2_224 #define Hacl_HMAC_compute_sha2_256 _Py_LibHacl_Hacl_HMAC_compute_sha2_256 #define Hacl_HMAC_compute_sha2_384 _Py_LibHacl_Hacl_HMAC_compute_sha2_384 #define Hacl_HMAC_compute_sha2_512 _Py_LibHacl_Hacl_HMAC_compute_sha2_512 -// HMAC-SHA-3 +// --- HMAC-SHA-3 ------------------------------------------------------------- #define Hacl_HMAC_compute_sha3_224 _Py_LibHacl_Hacl_HMAC_compute_sha3_224 #define Hacl_HMAC_compute_sha3_256 _Py_LibHacl_Hacl_HMAC_compute_sha3_256 #define Hacl_HMAC_compute_sha3_384 _Py_LibHacl_Hacl_HMAC_compute_sha3_384 #define Hacl_HMAC_compute_sha3_512 _Py_LibHacl_Hacl_HMAC_compute_sha3_512 -// HMAC-BLAKE -#define Hacl_HMAC_compute_blake2s_32 _Py_LibHacl_Hacl_HMAC_compute_blake2s_32 +// --- HMAC-BLAKE-2 ----------------------------------------------------------- #define Hacl_HMAC_compute_blake2b_32 _Py_LibHacl_Hacl_HMAC_compute_blake2b_32 +#define Hacl_HMAC_compute_blake2s_32 _Py_LibHacl_Hacl_HMAC_compute_blake2s_32 #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index d91650b44bb..72ceb27b7f7 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=7720f6d4fc0468a99d5ea6120976bcc271e42727 +expected_hacl_star_rev=8ba599b2f6c9701b3dc961db895b0856a2210f76 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 48eed5eac97..19089b009f7 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -30,7 +30,6 @@ /* EVP is the preferred interface to hashing in OpenSSL */ #include <openssl/evp.h> -#include <openssl/hmac.h> #include <openssl/crypto.h> // FIPS_mode() /* We use the object interface to discover what hashes OpenSSL supports. */ #include <openssl/objects.h> @@ -38,13 +37,20 @@ #include <stdbool.h> +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +# define Py_HAS_OPENSSL3_SUPPORT +# include <openssl/core_names.h> // OSSL_MAC_PARAM_DIGEST +# include <openssl/params.h> // OSSL_PARAM_*() +#else +# include <openssl/hmac.h> // HMAC() +#endif + #ifndef OPENSSL_THREADS # error "OPENSSL_THREADS is not defined, Python requires thread-safe OpenSSL" #endif #define MUNCH_SIZE INT_MAX -#define PY_OPENSSL_HAS_SCRYPT 1 #if defined(NID_sha3_224) && defined(NID_sha3_256) && defined(NID_sha3_384) && defined(NID_sha3_512) #define PY_OPENSSL_HAS_SHA3 1 #endif @@ -55,18 +61,42 @@ #define PY_OPENSSL_HAS_BLAKE2 1 #endif -#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#ifdef Py_HAS_OPENSSL3_SUPPORT #define PY_EVP_MD EVP_MD #define PY_EVP_MD_fetch(algorithm, properties) EVP_MD_fetch(NULL, algorithm, properties) #define PY_EVP_MD_up_ref(md) EVP_MD_up_ref(md) #define PY_EVP_MD_free(md) EVP_MD_free(md) + +#define PY_EVP_MD_CTX_md(CTX) EVP_MD_CTX_get0_md(CTX) + +#define PY_HMAC_CTX_TYPE EVP_MAC_CTX +#define PY_HMAC_CTX_free EVP_MAC_CTX_free +#define PY_HMAC_update EVP_MAC_update #else #define PY_EVP_MD const EVP_MD #define PY_EVP_MD_fetch(algorithm, properties) EVP_get_digestbyname(algorithm) #define PY_EVP_MD_up_ref(md) do {} while(0) #define PY_EVP_MD_free(md) do {} while(0) + +#define PY_EVP_MD_CTX_md(CTX) EVP_MD_CTX_md(CTX) + +#define PY_HMAC_CTX_TYPE HMAC_CTX +#define PY_HMAC_CTX_free HMAC_CTX_free +#define PY_HMAC_update HMAC_Update #endif +/* + * Return 1 if *md* is an extendable-output Function (XOF) and 0 otherwise. + * SHAKE128 and SHAKE256 are XOF functions but not BLAKE2B algorithms. + * + * This is a backport of the EVP_MD_xof() helper added in OpenSSL 3.4. + */ +static inline int +PY_EVP_MD_xof(PY_EVP_MD *md) +{ + return md != NULL && ((EVP_MD_flags(md) & EVP_MD_FLAG_XOF) != 0); +} + /* hash alias map and fast lookup * * Map between Python's preferred names and OpenSSL internal names. Maintain @@ -77,12 +107,12 @@ * py_alias as keys. */ -enum Py_hash_type { - Py_ht_evp, // usedforsecurity=True / default - Py_ht_evp_nosecurity, // usedforsecurity=False - Py_ht_mac, // HMAC - Py_ht_pbkdf2, // PKBDF2 -}; +typedef enum Py_hash_type { + Py_ht_evp, // usedforsecurity=True / default + Py_ht_evp_nosecurity, // usedforsecurity=False + Py_ht_mac, // HMAC + Py_ht_pbkdf2, // PKBDF2 +} Py_hash_type; typedef struct { const char *py_name; @@ -251,18 +281,22 @@ py_hashentry_table_new(void) { return NULL; } -/* Module state */ +// --- Module state ----------------------------------------------------------- + static PyModuleDef _hashlibmodule; typedef struct { - PyTypeObject *EVPtype; - PyTypeObject *HMACtype; + PyTypeObject *HASH_type; // based on EVP_MD + PyTypeObject *HMAC_type; #ifdef PY_OPENSSL_HAS_SHAKE - PyTypeObject *EVPXOFtype; + PyTypeObject *HASHXOF_type; // based on EVP_MD #endif PyObject *constructs; PyObject *unsupported_digestmod_error; _Py_hashtable_t *hashtable; +#ifdef Py_HAS_OPENSSL3_SUPPORT + EVP_MAC *evp_hmac; +#endif } _hashlibstate; static inline _hashlibstate* @@ -273,57 +307,119 @@ get_hashlib_state(PyObject *module) return (_hashlibstate *)state; } -typedef struct { - PyObject_HEAD - EVP_MD_CTX *ctx; /* OpenSSL message digest context */ - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; /* OpenSSL context lock */ -} EVPobject; - -#define EVPobject_CAST(op) ((EVPobject *)(op)) +// --- Module objects --------------------------------------------------------- typedef struct { - PyObject_HEAD - HMAC_CTX *ctx; /* OpenSSL hmac context */ - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; /* HMAC context lock */ + HASHLIB_OBJECT_HEAD + EVP_MD_CTX *ctx; /* OpenSSL message digest context */ +} HASHobject; + +#define HASHobject_CAST(op) ((HASHobject *)(op)) + +typedef struct { + HASHLIB_OBJECT_HEAD +#ifdef Py_HAS_OPENSSL3_SUPPORT + EVP_MAC_CTX *ctx; /* OpenSSL HMAC EVP-based context */ + int evp_md_nid; /* needed to find the message digest name */ +#else + HMAC_CTX *ctx; /* OpenSSL HMAC plain context */ +#endif } HMACobject; #define HMACobject_CAST(op) ((HMACobject *)(op)) -#include "clinic/_hashopenssl.c.h" +// --- Module clinic configuration -------------------------------------------- + /*[clinic input] module _hashlib -class _hashlib.HASH "EVPobject *" "((_hashlibstate *)PyModule_GetState(module))->EVPtype" -class _hashlib.HASHXOF "EVPobject *" "((_hashlibstate *)PyModule_GetState(module))->EVPXOFtype" -class _hashlib.HMAC "HMACobject *" "((_hashlibstate *)PyModule_GetState(module))->HMACtype" +class _hashlib.HASH "HASHobject *" "&PyType_Type" +class _hashlib.HASHXOF "HASHobject *" "&PyType_Type" +class _hashlib.HMAC "HMACobject *" "&PyType_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7df1bcf6f75cb8ef]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6b5c9ce5c28bdc58]*/ +#include "clinic/_hashopenssl.c.h" /* LCOV_EXCL_START */ +/* Thin wrapper around ERR_reason_error_string() returning non-NULL text. */ +static const char * +py_wrapper_ERR_reason_error_string(unsigned long errcode) +{ + const char *reason = ERR_reason_error_string(errcode); + return reason ? reason : "no reason"; +} + +#ifdef Py_HAS_OPENSSL3_SUPPORT +/* + * Set an exception with additional information. + * + * This is only useful in OpenSSL 3.0 and later as the default reason + * usually lacks information and function locations are no longer encoded + * in the error code. + */ +static void +set_exception_with_ssl_errinfo(PyObject *exc_type, PyObject *exc_text, + const char *lib, const char *reason) +{ + assert(exc_type != NULL); + assert(exc_text != NULL); + if (lib && reason) { + PyErr_Format(exc_type, "[%s] %U (reason: %s)", lib, exc_text, reason); + } + else if (lib) { + PyErr_Format(exc_type, "[%s] %U", lib, exc_text); + } + else if (reason) { + PyErr_Format(exc_type, "%U (reason: %s)", exc_text, reason); + } + else { + PyErr_SetObject(exc_type, exc_text); + } +} +#endif + /* Set an exception of given type using the given OpenSSL error code. */ static void -set_ssl_exception_from_errcode(PyObject *exc, unsigned long errcode) +set_ssl_exception_from_errcode(PyObject *exc_type, unsigned long errcode) { + assert(exc_type != NULL); assert(errcode != 0); /* ERR_ERROR_STRING(3) ensures that the messages below are ASCII */ const char *lib = ERR_lib_error_string(errcode); +#ifdef Py_HAS_OPENSSL3_SUPPORT + // Since OpenSSL 3.0, ERR_func_error_string() always returns NULL. + const char *func = NULL; +#else const char *func = ERR_func_error_string(errcode); - const char *reason = ERR_reason_error_string(errcode); +#endif + const char *reason = py_wrapper_ERR_reason_error_string(errcode); if (lib && func) { - PyErr_Format(exc, "[%s: %s] %s", lib, func, reason); + PyErr_Format(exc_type, "[%s: %s] %s", lib, func, reason); } else if (lib) { - PyErr_Format(exc, "[%s] %s", lib, reason); + PyErr_Format(exc_type, "[%s] %s", lib, reason); } else { - PyErr_SetString(exc, reason); + PyErr_SetString(exc_type, reason); + } +} + +/* + * Get an appropriate exception type for the given OpenSSL error code. + * + * The exception type depends on the error code reason. + */ +static PyObject * +get_smart_ssl_exception_type(unsigned long errcode, PyObject *default_exc_type) +{ + switch (ERR_GET_REASON(errcode)) { + case ERR_R_MALLOC_FAILURE: + return PyExc_MemoryError; + default: + return default_exc_type; } } @@ -331,83 +427,271 @@ set_ssl_exception_from_errcode(PyObject *exc, unsigned long errcode) * Set an exception of given type. * * By default, the exception's message is constructed by using the last SSL - * error that occurred. If no error occurred, the 'fallback_format' is used - * to create a C-style formatted fallback message. + * error that occurred. If no error occurred, the 'fallback_message' is used + * to create an exception message. */ static void -raise_ssl_error(PyObject *exc, const char *fallback_format, ...) +raise_ssl_error(PyObject *exc_type, const char *fallback_message) +{ + assert(fallback_message != NULL); + unsigned long errcode = ERR_peek_last_error(); + if (errcode) { + ERR_clear_error(); + set_ssl_exception_from_errcode(exc_type, errcode); + } + else { + PyErr_SetString(exc_type, fallback_message); + } +} + +/* Same as raise_ssl_error() but with a C-style formatted message. */ +static void +raise_ssl_error_f(PyObject *exc_type, const char *fallback_format, ...) { assert(fallback_format != NULL); unsigned long errcode = ERR_peek_last_error(); if (errcode) { ERR_clear_error(); - set_ssl_exception_from_errcode(exc, errcode); + set_ssl_exception_from_errcode(exc_type, errcode); } else { va_list vargs; va_start(vargs, fallback_format); - PyErr_FormatV(exc, fallback_format, vargs); + PyErr_FormatV(exc_type, fallback_format, vargs); + va_end(vargs); + } +} + +/* Same as raise_ssl_error_f() with smart exception types. */ +static void +raise_smart_ssl_error_f(PyObject *exc_type, const char *fallback_format, ...) +{ + unsigned long errcode = ERR_peek_last_error(); + if (errcode) { + ERR_clear_error(); + exc_type = get_smart_ssl_exception_type(errcode, exc_type); + set_ssl_exception_from_errcode(exc_type, errcode); + } + else { + va_list vargs; + va_start(vargs, fallback_format); + PyErr_FormatV(exc_type, fallback_format, vargs); va_end(vargs); } } /* - * Set an exception with a generic default message after an error occurred. - * - * It can also be used without previous calls to SSL built-in functions, - * in which case a generic error message is provided. + * Raise a ValueError with a default message after an error occurred. + * It can also be used without previous calls to SSL built-in functions. */ static inline void -notify_ssl_error_occurred(void) +notify_ssl_error_occurred(const char *message) { - raise_ssl_error(PyExc_ValueError, "no reason supplied"); + raise_ssl_error(PyExc_ValueError, message); } -/* LCOV_EXCL_STOP */ -static PyObject* -py_digest_name(const EVP_MD *md) +/* Same as notify_ssl_error_occurred() for failed OpenSSL functions. */ +static inline void +notify_ssl_error_occurred_in(const char *funcname) { - assert(md != NULL); - int nid = EVP_MD_nid(md); - const char *name = NULL; - const py_hashentry_t *h; + raise_ssl_error_f(PyExc_ValueError, + "error in OpenSSL function %s()", funcname); +} - for (h = py_hashes; h->py_name != NULL; h++) { - if (h->ossl_nid == nid) { - name = h->py_name; +/* Same as notify_ssl_error_occurred_in() with smart exception types. */ +static inline void +notify_smart_ssl_error_occurred_in(const char *funcname) +{ + raise_smart_ssl_error_f(PyExc_ValueError, + "error in OpenSSL function %s()", funcname); +} + +#ifdef Py_HAS_OPENSSL3_SUPPORT +static void +raise_unsupported_algorithm_impl(PyObject *exc_type, + const char *fallback_format, + const void *format_arg) +{ + // Since OpenSSL 3.0, if the algorithm is not supported or fetching fails, + // the reason lacks the algorithm name. + int errcode = ERR_peek_last_error(); + switch (ERR_GET_REASON(errcode)) { + case ERR_R_UNSUPPORTED: { + PyObject *text = PyUnicode_FromFormat(fallback_format, format_arg); + if (text != NULL) { + const char *lib = ERR_lib_error_string(errcode); + set_exception_with_ssl_errinfo(exc_type, text, lib, NULL); + Py_DECREF(text); + } break; } + case ERR_R_FETCH_FAILED: { + PyObject *text = PyUnicode_FromFormat(fallback_format, format_arg); + if (text != NULL) { + const char *lib = ERR_lib_error_string(errcode); + const char *reason = ERR_reason_error_string(errcode); + set_exception_with_ssl_errinfo(exc_type, text, lib, reason); + Py_DECREF(text); + } + break; + } + default: + raise_ssl_error_f(exc_type, fallback_format, format_arg); + break; } - if (name == NULL) { - /* Ignore aliased names and only use long, lowercase name. The aliases - * pollute the list and OpenSSL appears to have its own definition of - * alias as the resulting list still contains duplicate and alternate - * names for several algorithms. - */ - name = OBJ_nid2ln(nid); - if (name == NULL) - name = OBJ_nid2sn(nid); - } + assert(PyErr_Occurred()); +} +#else +/* Before OpenSSL 3.0, error messages included enough information. */ +#define raise_unsupported_algorithm_impl raise_ssl_error_f +#endif - return PyUnicode_FromString(name); +static inline void +raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod) +{ + raise_unsupported_algorithm_impl( + state->unsupported_digestmod_error, + HASHLIB_UNSUPPORTED_ALGORITHM, + digestmod + ); } -/* Get EVP_MD by HID and purpose */ -static PY_EVP_MD* -py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) +static inline void +raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name) { - PY_EVP_MD *digest = NULL; - PY_EVP_MD *other_digest = NULL; - _hashlibstate *state = get_hashlib_state(module); - py_hashentry_t *entry = (py_hashentry_t *)_Py_hashtable_get( - state->hashtable, (const void*)name + raise_unsupported_algorithm_impl( + state->unsupported_digestmod_error, + HASHLIB_UNSUPPORTED_STR_ALGORITHM, + name ); +} - if (entry != NULL) { - switch (py_ht) { +#undef raise_unsupported_algorithm_impl +/* LCOV_EXCL_STOP */ + +/* + * OpenSSL provides a way to go from NIDs to digest names for hash functions + * but lacks this granularity for MAC objects where it is not possible to get + * the underlying digest name (only the block size and digest size are allowed + * to be recovered). + * + * In addition, OpenSSL aliases pollute the list of known digest names + * as OpenSSL appears to have its own definition of alias. In particular, + * the resulting list still contains duplicate and alternate names for several + * algorithms. + * + * Therefore, digest names, whether they are used by hash functions or HMAC, + * are handled through EVP_MD objects or directly by using some NID. + */ + +/* Get a cached entry by OpenSSL NID. */ +static const py_hashentry_t * +get_hashentry_by_nid(int nid) +{ + for (const py_hashentry_t *h = py_hashes; h->py_name != NULL; h++) { + if (h->ossl_nid == nid) { + return h; + } + } + return NULL; +} + +/* + * Convert the NID to a string via OBJ_nid2*() functions. + * + * If 'nid' cannot be resolved, set an exception and return NULL. + */ +static const char * +get_asn1_utf8name_by_nid(int nid) +{ + const char *name = OBJ_nid2ln(nid); + if (name == NULL) { + // In OpenSSL 3.0 and later, OBJ_nid*() are thread-safe and may raise. + assert(ERR_peek_last_error() != 0); + if (ERR_GET_REASON(ERR_peek_last_error()) != OBJ_R_UNKNOWN_NID) { + goto error; + } + // fallback to short name and unconditionally propagate errors + name = OBJ_nid2sn(nid); + if (name == NULL) { + goto error; + } + } + return name; + +error: + raise_ssl_error_f(PyExc_ValueError, "cannot resolve NID %d", nid); + return NULL; +} + +/* + * Convert the NID to an OpenSSL digest name. + * + * On error, set an exception and return NULL. + */ +static const char * +get_hashlib_utf8name_by_nid(int nid) +{ + const py_hashentry_t *e = get_hashentry_by_nid(nid); + return e ? e->py_name : get_asn1_utf8name_by_nid(nid); +} + +#ifdef Py_HAS_OPENSSL3_SUPPORT +/* + * Convert the NID to an OpenSSL "canonical" cached, SN_* or LN_* digest name. + * + * On error, set an exception and return NULL. + */ +static const char * +get_openssl_utf8name_by_nid(int nid) +{ + const py_hashentry_t *e = get_hashentry_by_nid(nid); + return e ? e->ossl_name : get_asn1_utf8name_by_nid(nid); +} +#endif + +/* Same as get_hashlib_utf8name_by_nid() but using an EVP_MD object. */ +static const char * +get_hashlib_utf8name_by_evp_md(const EVP_MD *md) +{ + assert(md != NULL); + return get_hashlib_utf8name_by_nid(EVP_MD_nid(md)); +} + +/* + * Return 1 if the property query clause [1] must be "-fips" and 0 otherwise. + * + * [1] https://docs.openssl.org/master/man7/property + */ +static inline int +disable_fips_property(Py_hash_type py_ht) +{ + switch (py_ht) { case Py_ht_evp: case Py_ht_mac: case Py_ht_pbkdf2: + return 0; + case Py_ht_evp_nosecurity: + return 1; + default: + Py_FatalError("unsupported hash type"); + } +} + +/* + * Get a new reference to an EVP_MD object described by name and purpose. + * + * If 'name' is an OpenSSL indexed name, the return value is cached. + */ +static PY_EVP_MD * +get_openssl_evp_md_by_utf8name(_hashlibstate *state, const char *name, + Py_hash_type py_ht) +{ + PY_EVP_MD *digest = NULL, *other_digest = NULL; + py_hashentry_t *entry = _Py_hashtable_get(state->hashtable, name); + + if (entry != NULL) { + if (!disable_fips_property(py_ht)) { digest = FT_ATOMIC_LOAD_PTR_RELAXED(entry->evp); if (digest == NULL) { digest = PY_EVP_MD_fetch(entry->ossl_name, NULL); @@ -418,8 +702,8 @@ py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) entry->evp = digest; #endif } - break; - case Py_ht_evp_nosecurity: + } + else { digest = FT_ATOMIC_LOAD_PTR_RELAXED(entry->evp_nosecurity); if (digest == NULL) { digest = PY_EVP_MD_fetch(entry->ossl_name, "-fips"); @@ -430,86 +714,127 @@ py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) entry->evp_nosecurity = digest; #endif } - break; } // if another thread same thing at same time make sure we got same ptr assert(other_digest == NULL || other_digest == digest); - if (digest != NULL) { - if (other_digest == NULL) { - PY_EVP_MD_up_ref(digest); - } - } - } else { - // Fall back for looking up an unindexed OpenSSL specific name. - switch (py_ht) { - case Py_ht_evp: - case Py_ht_mac: - case Py_ht_pbkdf2: - digest = PY_EVP_MD_fetch(name, NULL); - break; - case Py_ht_evp_nosecurity: - digest = PY_EVP_MD_fetch(name, "-fips"); - break; + if (digest != NULL && other_digest == NULL) { + PY_EVP_MD_up_ref(digest); } } + else { + // Fall back for looking up an unindexed OpenSSL specific name. + const char *props = disable_fips_property(py_ht) ? "-fips" : NULL; + (void)props; // will only be used in OpenSSL 3.0 and later + digest = PY_EVP_MD_fetch(name, props); + } if (digest == NULL) { - raise_ssl_error(state->unsupported_digestmod_error, - "unsupported hash type %s", name); + raise_unsupported_str_algorithm_error(state, name); return NULL; } return digest; } -/* Get digest EVP from object +/* + * Get a new reference to an EVP_MD described by 'digestmod' and purpose. * - * * string - * * _hashopenssl builtin function + * On error, set an exception and return NULL. * - * on error returns NULL with exception set. + * Parameters + * + * digestmod A digest name or a _hashopenssl builtin function + * py_ht The message digest purpose. */ -static PY_EVP_MD* -py_digest_by_digestmod(PyObject *module, PyObject *digestmod, enum Py_hash_type py_ht) { - PyObject *name_obj = NULL; +static PY_EVP_MD * +get_openssl_evp_md(_hashlibstate *state, PyObject *digestmod, Py_hash_type py_ht) +{ const char *name; - if (PyUnicode_Check(digestmod)) { - name_obj = digestmod; - } else { - _hashlibstate *state = get_hashlib_state(module); - // borrowed ref - name_obj = PyDict_GetItemWithError(state->constructs, digestmod); + name = PyUnicode_AsUTF8(digestmod); } - if (name_obj == NULL) { + else { + PyObject *dict = state->constructs; + assert(dict != NULL); + PyObject *borrowed_ref = PyDict_GetItemWithError(dict, digestmod); + name = borrowed_ref == NULL ? NULL : PyUnicode_AsUTF8(borrowed_ref); + } + if (name == NULL) { if (!PyErr_Occurred()) { - _hashlibstate *state = get_hashlib_state(module); - PyErr_Format( - state->unsupported_digestmod_error, - "Unsupported digestmod %R", digestmod); + raise_unsupported_algorithm_error(state, digestmod); } return NULL; } - - name = PyUnicode_AsUTF8(name_obj); - if (name == NULL) { - return NULL; - } - - return py_digest_by_name(module, name, py_ht); + return get_openssl_evp_md_by_utf8name(state, name, py_ht); } -static EVPobject * -newEVPobject(PyTypeObject *type) +#ifdef Py_HAS_OPENSSL3_SUPPORT +/* + * Get the "canonical" name of an EVP_MD described by 'digestmod' and purpose. + * + * On error, set an exception and return NULL. + * + * This function should not be used to construct the exposed Python name, + * but rather to invoke OpenSSL EVP_* functions. + */ +static const char * +get_openssl_digest_name(_hashlibstate *state, + PyObject *digestmod, Py_hash_type py_ht, + EVP_MD **evp_md) { - EVPobject *retval = PyObject_New(EVPobject, type); + PY_EVP_MD *md = get_openssl_evp_md(state, digestmod, py_ht); + if (md == NULL) { + if (evp_md != NULL) { + *evp_md = NULL; + } + return NULL; + } + int nid = EVP_MD_nid(md); + const char *name = get_openssl_utf8name_by_nid(nid); + if (name == NULL) { + if (evp_md != NULL) { + *evp_md = NULL; + } + PY_EVP_MD_free(md); + raise_unsupported_algorithm_error(state, digestmod); + return NULL; + } + if (evp_md != NULL) { + *evp_md = md; + } + else { + PY_EVP_MD_free(md); + } + return name; +} +#endif + +// --- OpenSSL HASH wrappers -------------------------------------------------- + +/* Thin wrapper around EVP_MD_CTX_new() which sets an exception on failure. */ +static EVP_MD_CTX * +py_wrapper_EVP_MD_CTX_new(void) +{ + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + if (ctx == NULL) { + PyErr_NoMemory(); + return NULL; + } + return ctx; +} + +// --- HASH interface --------------------------------------------------------- + +static HASHobject * +new_hash_object(PyTypeObject *type) +{ + HASHobject *retval = PyObject_New(HASHobject, type); if (retval == NULL) { return NULL; } HASHLIB_INIT_MUTEX(retval); - retval->ctx = EVP_MD_CTX_new(); + retval->ctx = py_wrapper_EVP_MD_CTX_new(); if (retval->ctx == NULL) { Py_DECREF(retval); - PyErr_NoMemory(); return NULL; } @@ -517,7 +842,7 @@ newEVPobject(PyTypeObject *type) } static int -EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len) +_hashlib_HASH_hash(HASHobject *self, const void *vp, Py_ssize_t len) { unsigned int process; const unsigned char *cp = (const unsigned char *)vp; @@ -527,7 +852,7 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len) else process = Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int); if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestUpdate)); return -1; } len -= process; @@ -539,9 +864,9 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len) /* Internal methods for a hash object */ static void -EVP_dealloc(PyObject *op) +_hashlib_HASH_dealloc(PyObject *op) { - EVPobject *self = EVPobject_CAST(op); + HASHobject *self = HASHobject_CAST(op); PyTypeObject *tp = Py_TYPE(self); EVP_MD_CTX_free(self->ctx); PyObject_Free(self); @@ -549,120 +874,98 @@ EVP_dealloc(PyObject *op) } static int -locked_EVP_MD_CTX_copy(EVP_MD_CTX *new_ctx_p, EVPobject *self) +_hashlib_HASH_copy_locked(HASHobject *self, EVP_MD_CTX *new_ctx_p) { int result; - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); result = EVP_MD_CTX_copy(new_ctx_p, self->ctx); - LEAVE_HASHLIB(self); - return result; + HASHLIB_RELEASE_LOCK(self); + if (result == 0) { + notify_smart_ssl_error_occurred_in(Py_STRINGIFY(EVP_MD_CTX_copy)); + return -1; + } + return 0; } /* External methods for a hash object */ /*[clinic input] -_hashlib.HASH.copy as EVP_copy +_hashlib.HASH.copy Return a copy of the hash object. [clinic start generated code]*/ static PyObject * -EVP_copy_impl(EVPobject *self) -/*[clinic end generated code: output=b370c21cdb8ca0b4 input=31455b6a3e638069]*/ +_hashlib_HASH_copy_impl(HASHobject *self) +/*[clinic end generated code: output=2545541af18d53d7 input=814b19202cd08a26]*/ { - EVPobject *newobj; + HASHobject *newobj; - if ((newobj = newEVPobject(Py_TYPE(self))) == NULL) + if ((newobj = new_hash_object(Py_TYPE(self))) == NULL) return NULL; - if (!locked_EVP_MD_CTX_copy(newobj->ctx, self)) { + if (_hashlib_HASH_copy_locked(self, newobj->ctx) < 0) { Py_DECREF(newobj); - notify_ssl_error_occurred(); return NULL; } return (PyObject *)newobj; } +static Py_ssize_t +_hashlib_HASH_digest_compute(HASHobject *self, unsigned char *digest) +{ + EVP_MD_CTX *ctx = py_wrapper_EVP_MD_CTX_new(); + if (ctx == NULL) { + return -1; + } + if (_hashlib_HASH_copy_locked(self, ctx) < 0) { + goto error; + } + Py_ssize_t digest_size = EVP_MD_CTX_size(ctx); + if (!EVP_DigestFinal(ctx, digest, NULL)) { + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestFinal)); + goto error; + } + EVP_MD_CTX_free(ctx); + return digest_size; + +error: + EVP_MD_CTX_free(ctx); + return -1; +} + /*[clinic input] -_hashlib.HASH.digest as EVP_digest +_hashlib.HASH.digest Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * -EVP_digest_impl(EVPobject *self) -/*[clinic end generated code: output=0f6a3a0da46dc12d input=03561809a419bf00]*/ +_hashlib_HASH_digest_impl(HASHobject *self) +/*[clinic end generated code: output=3fc6f9671d712850 input=d8d528d6e50af0de]*/ { unsigned char digest[EVP_MAX_MD_SIZE]; - EVP_MD_CTX *temp_ctx; - PyObject *retval; - unsigned int digest_size; - - temp_ctx = EVP_MD_CTX_new(); - if (temp_ctx == NULL) { - PyErr_NoMemory(); - return NULL; - } - - if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) { - goto error; - } - digest_size = EVP_MD_CTX_size(temp_ctx); - if (!EVP_DigestFinal(temp_ctx, digest, NULL)) { - goto error; - } - - retval = PyBytes_FromStringAndSize((const char *)digest, digest_size); - EVP_MD_CTX_free(temp_ctx); - return retval; - -error: - EVP_MD_CTX_free(temp_ctx); - notify_ssl_error_occurred(); - return NULL; + Py_ssize_t n = _hashlib_HASH_digest_compute(self, digest); + return n < 0 ? NULL : PyBytes_FromStringAndSize((const char *)digest, n); } /*[clinic input] -_hashlib.HASH.hexdigest as EVP_hexdigest +_hashlib.HASH.hexdigest Return the digest value as a string of hexadecimal digits. [clinic start generated code]*/ static PyObject * -EVP_hexdigest_impl(EVPobject *self) -/*[clinic end generated code: output=18e6decbaf197296 input=aff9cf0e4c741a9a]*/ +_hashlib_HASH_hexdigest_impl(HASHobject *self) +/*[clinic end generated code: output=1b8e60d9711e7f4d input=ae7553f78f8372d8]*/ { unsigned char digest[EVP_MAX_MD_SIZE]; - EVP_MD_CTX *temp_ctx; - unsigned int digest_size; - - temp_ctx = EVP_MD_CTX_new(); - if (temp_ctx == NULL) { - PyErr_NoMemory(); - return NULL; - } - - /* Get the raw (binary) digest value */ - if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) { - goto error; - } - digest_size = EVP_MD_CTX_size(temp_ctx); - if (!EVP_DigestFinal(temp_ctx, digest, NULL)) { - goto error; - } - - EVP_MD_CTX_free(temp_ctx); - - return _Py_strhex((const char *)digest, (Py_ssize_t)digest_size); - -error: - EVP_MD_CTX_free(temp_ctx); - notify_ssl_error_occurred(); - return NULL; + Py_ssize_t n = _hashlib_HASH_digest_compute(self, digest); + return n < 0 ? NULL : _Py_strhex((const char *)digest, n); } /*[clinic input] -_hashlib.HASH.update as EVP_update +_hashlib.HASH.update obj: object / @@ -671,82 +974,70 @@ Update this hash object's state with the provided string. [clinic start generated code]*/ static PyObject * -EVP_update_impl(EVPobject *self, PyObject *obj) -/*[clinic end generated code: output=d56f91c68348f95f input=9b30ec848f015501]*/ +_hashlib_HASH_update_impl(HASHobject *self, PyObject *obj) +/*[clinic end generated code: output=62ad989754946b86 input=aa1ce20e3f92ceb6]*/ { int result; Py_buffer view; - GET_BUFFER_VIEW_OR_ERROUT(obj, &view); - - if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - result = EVP_hash(self, view.buf, view.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - result = EVP_hash(self, view.buf, view.len); - } - + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, view.len, + result = _hashlib_HASH_hash(self, view.buf, view.len) + ); PyBuffer_Release(&view); - - if (result == -1) - return NULL; - Py_RETURN_NONE; + return result < 0 ? NULL : Py_None; } -static PyMethodDef EVP_methods[] = { - EVP_UPDATE_METHODDEF - EVP_DIGEST_METHODDEF - EVP_HEXDIGEST_METHODDEF - EVP_COPY_METHODDEF +static PyMethodDef HASH_methods[] = { + _HASHLIB_HASH_COPY_METHODDEF + _HASHLIB_HASH_DIGEST_METHODDEF + _HASHLIB_HASH_HEXDIGEST_METHODDEF + _HASHLIB_HASH_UPDATE_METHODDEF {NULL, NULL} /* sentinel */ }; static PyObject * -EVP_get_block_size(PyObject *op, void *Py_UNUSED(closure)) +_hashlib_HASH_get_blocksize(PyObject *op, void *Py_UNUSED(closure)) { - EVPobject *self = EVPobject_CAST(op); + HASHobject *self = HASHobject_CAST(op); long block_size = EVP_MD_CTX_block_size(self->ctx); return PyLong_FromLong(block_size); } static PyObject * -EVP_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) +_hashlib_HASH_get_digestsize(PyObject *op, void *Py_UNUSED(closure)) { - EVPobject *self = EVPobject_CAST(op); + HASHobject *self = HASHobject_CAST(op); long size = EVP_MD_CTX_size(self->ctx); return PyLong_FromLong(size); } static PyObject * -EVP_get_name(PyObject *op, void *Py_UNUSED(closure)) +_hashlib_HASH_get_name(PyObject *op, void *Py_UNUSED(closure)) { - EVPobject *self = EVPobject_CAST(op); - const EVP_MD *md = EVP_MD_CTX_md(self->ctx); + HASHobject *self = HASHobject_CAST(op); + const EVP_MD *md = PY_EVP_MD_CTX_md(self->ctx); if (md == NULL) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred("missing EVP_MD for HASH context"); return NULL; } - return py_digest_name(md); + const char *name = get_hashlib_utf8name_by_evp_md(md); + assert(name != NULL || PyErr_Occurred()); + return name == NULL ? NULL : PyUnicode_FromString(name); } -static PyGetSetDef EVP_getseters[] = { - {"digest_size", EVP_get_digest_size, NULL, NULL, NULL}, - {"block_size", EVP_get_block_size, NULL, NULL, NULL}, - {"name", EVP_get_name, NULL, NULL, PyDoc_STR("algorithm name.")}, +static PyGetSetDef HASH_getsets[] = { + {"digest_size", _hashlib_HASH_get_digestsize, NULL, NULL, NULL}, + {"block_size", _hashlib_HASH_get_blocksize, NULL, NULL, NULL}, + {"name", _hashlib_HASH_get_name, NULL, NULL, PyDoc_STR("algorithm name.")}, {NULL} /* Sentinel */ }; static PyObject * -EVP_repr(PyObject *self) +_hashlib_HASH_repr(PyObject *self) { - PyObject *name = EVP_get_name(self, NULL); + PyObject *name = _hashlib_HASH_get_name(self, NULL); if (name == NULL) { return NULL; } @@ -756,7 +1047,7 @@ EVP_repr(PyObject *self) return repr; } -PyDoc_STRVAR(hashtype_doc, +PyDoc_STRVAR(HASHobject_type_doc, "HASH(name, string=b\'\')\n" "--\n" "\n" @@ -774,105 +1065,116 @@ PyDoc_STRVAR(hashtype_doc, "name -- the hash algorithm being used by this object\n" "digest_size -- number of bytes in this hashes output"); -static PyType_Slot EVPtype_slots[] = { - {Py_tp_dealloc, EVP_dealloc}, - {Py_tp_repr, EVP_repr}, - {Py_tp_doc, (char *)hashtype_doc}, - {Py_tp_methods, EVP_methods}, - {Py_tp_getset, EVP_getseters}, +static PyType_Slot HASHobject_type_slots[] = { + {Py_tp_dealloc, _hashlib_HASH_dealloc}, + {Py_tp_repr, _hashlib_HASH_repr}, + {Py_tp_doc, (char *)HASHobject_type_doc}, + {Py_tp_methods, HASH_methods}, + {Py_tp_getset, HASH_getsets}, {0, 0}, }; -static PyType_Spec EVPtype_spec = { - "_hashlib.HASH", /*tp_name*/ - sizeof(EVPobject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE, - EVPtype_slots +static PyType_Spec HASHobject_type_spec = { + .name = "_hashlib.HASH", + .basicsize = sizeof(HASHobject), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = HASHobject_type_slots }; #ifdef PY_OPENSSL_HAS_SHAKE /*[clinic input] -_hashlib.HASHXOF.digest as EVPXOF_digest +_hashlib.HASHXOF.digest - length: Py_ssize_t + length: Py_ssize_t(allow_negative=False) Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * -EVPXOF_digest_impl(EVPobject *self, Py_ssize_t length) -/*[clinic end generated code: output=ef9320c23280efad input=816a6537cea3d1db]*/ +_hashlib_HASHXOF_digest_impl(HASHobject *self, Py_ssize_t length) +/*[clinic end generated code: output=dcb09335dd2fe908 input=224d047da2c12a42]*/ { EVP_MD_CTX *temp_ctx; - PyObject *retval = PyBytes_FromStringAndSize(NULL, length); - if (retval == NULL) { + if (length == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + + PyBytesWriter *writer = PyBytesWriter_Create(length); + if (writer == NULL) { return NULL; } - temp_ctx = EVP_MD_CTX_new(); + temp_ctx = py_wrapper_EVP_MD_CTX_new(); if (temp_ctx == NULL) { - Py_DECREF(retval); - PyErr_NoMemory(); + PyBytesWriter_Discard(writer); return NULL; } - if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) { + if (_hashlib_HASH_copy_locked(self, temp_ctx) < 0) { goto error; } if (!EVP_DigestFinalXOF(temp_ctx, - (unsigned char*)PyBytes_AS_STRING(retval), + (unsigned char*)PyBytesWriter_GetData(writer), length)) { + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestFinalXOF)); goto error; } EVP_MD_CTX_free(temp_ctx); - return retval; + return PyBytesWriter_Finish(writer); error: - Py_DECREF(retval); + PyBytesWriter_Discard(writer); EVP_MD_CTX_free(temp_ctx); - notify_ssl_error_occurred(); return NULL; } /*[clinic input] -_hashlib.HASHXOF.hexdigest as EVPXOF_hexdigest +_hashlib.HASHXOF.hexdigest - length: Py_ssize_t + length: Py_ssize_t(allow_negative=False) Return the digest value as a string of hexadecimal digits. [clinic start generated code]*/ static PyObject * -EVPXOF_hexdigest_impl(EVPobject *self, Py_ssize_t length) -/*[clinic end generated code: output=eb3e6ee7788bf5b2 input=5f9d6a8f269e34df]*/ +_hashlib_HASHXOF_hexdigest_impl(HASHobject *self, Py_ssize_t length) +/*[clinic end generated code: output=519431cafa014f39 input=4a41b8ab5d3bfee2]*/ { unsigned char *digest; EVP_MD_CTX *temp_ctx; PyObject *retval; + if (length == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } + digest = (unsigned char*)PyMem_Malloc(length); if (digest == NULL) { - PyErr_NoMemory(); + (void)PyErr_NoMemory(); return NULL; } - temp_ctx = EVP_MD_CTX_new(); + temp_ctx = py_wrapper_EVP_MD_CTX_new(); if (temp_ctx == NULL) { PyMem_Free(digest); - PyErr_NoMemory(); return NULL; } /* Get the raw (binary) digest value */ - if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) { + if (_hashlib_HASH_copy_locked(self, temp_ctx) < 0) { goto error; } if (!EVP_DigestFinalXOF(temp_ctx, digest, length)) { + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestFinalXOF)); goto error; } @@ -885,29 +1187,29 @@ EVPXOF_hexdigest_impl(EVPobject *self, Py_ssize_t length) error: PyMem_Free(digest); EVP_MD_CTX_free(temp_ctx); - notify_ssl_error_occurred(); return NULL; } -static PyMethodDef EVPXOF_methods[] = { - EVPXOF_DIGEST_METHODDEF - EVPXOF_HEXDIGEST_METHODDEF +static PyMethodDef HASHXOFobject_methods[] = { + _HASHLIB_HASHXOF_DIGEST_METHODDEF + _HASHLIB_HASHXOF_HEXDIGEST_METHODDEF {NULL, NULL} /* sentinel */ }; static PyObject * -EVPXOF_get_digest_size(PyObject *Py_UNUSED(self), void *Py_UNUSED(closure)) +_hashlib_HASHXOF_digest_size(PyObject *Py_UNUSED(self), + void *Py_UNUSED(closure)) { return PyLong_FromLong(0); } -static PyGetSetDef EVPXOF_getseters[] = { - {"digest_size", EVPXOF_get_digest_size, NULL, NULL, NULL}, +static PyGetSetDef HASHXOFobject_getsets[] = { + {"digest_size", _hashlib_HASHXOF_digest_size, NULL, NULL, NULL}, {NULL} /* Sentinel */ }; -PyDoc_STRVAR(hashxoftype_doc, +PyDoc_STRVAR(HASHXOFobject_type_doc, "HASHXOF(name, string=b\'\')\n" "--\n" "\n" @@ -925,51 +1227,49 @@ PyDoc_STRVAR(hashxoftype_doc, "name -- the hash algorithm being used by this object\n" "digest_size -- number of bytes in this hashes output"); -static PyType_Slot EVPXOFtype_slots[] = { - {Py_tp_doc, (char *)hashxoftype_doc}, - {Py_tp_methods, EVPXOF_methods}, - {Py_tp_getset, EVPXOF_getseters}, +static PyType_Slot HASHXOFobject_type_slots[] = { + {Py_tp_doc, (char *)HASHXOFobject_type_doc}, + {Py_tp_methods, HASHXOFobject_methods}, + {Py_tp_getset, HASHXOFobject_getsets}, {0, 0}, }; -static PyType_Spec EVPXOFtype_spec = { - "_hashlib.HASHXOF", /*tp_name*/ - sizeof(EVPobject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE, - EVPXOFtype_slots +static PyType_Spec HASHXOFobject_type_spec = { + .name = "_hashlib.HASHXOF", + .basicsize = sizeof(HASHobject), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = HASHXOFobject_type_slots }; #endif -static PyObject* -py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, - int usedforsecurity) +static PyObject * +_hashlib_HASH(_hashlibstate *state, const char *digestname, PyObject *data_obj, + int usedforsecurity) { Py_buffer view = { 0 }; PY_EVP_MD *digest = NULL; PyTypeObject *type; - EVPobject *self = NULL; + HASHobject *self = NULL; if (data_obj != NULL) { GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); } - digest = py_digest_by_name( - module, digestname, usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity - ); + Py_hash_type purpose = usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity; + digest = get_openssl_evp_md_by_utf8name(state, digestname, purpose); if (digest == NULL) { goto exit; } - if ((EVP_MD_flags(digest) & EVP_MD_FLAG_XOF) == EVP_MD_FLAG_XOF) { - type = get_hashlib_state(module)->EVPXOFtype; - } else { - type = get_hashlib_state(module)->EVPtype; - } - - self = newEVPobject(type); + type = PY_EVP_MD_xof(digest) ? state->HASHXOF_type : state->HASH_type; + self = new_hash_object(type); if (self == NULL) { goto exit; } @@ -984,21 +1284,18 @@ py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, int result = EVP_DigestInit_ex(self->ctx, digest, NULL); if (!result) { - notify_ssl_error_occurred(); + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_DigestInit_ex)); Py_CLEAR(self); goto exit; } if (view.buf && view.len) { - if (view.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor - * where it is not yet possible to have concurrent access. */ - Py_BEGIN_ALLOW_THREADS - result = EVP_hash(self, view.buf, view.len); - Py_END_ALLOW_THREADS - } else { - result = EVP_hash(self, view.buf, view.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + view.len, + result = _hashlib_HASH_hash(self, view.buf, view.len) + ); if (result == -1) { assert(PyErr_Occurred()); Py_CLEAR(self); @@ -1017,16 +1314,22 @@ py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, return (PyObject *)self; } +// In Python 3.19, we can remove the "STRING" argument and would also be able +// to remove the macro (or keep it as an alias for better naming) since calls +// to _hashlib_HASH_new_impl() would fit on 80 characters. +#define CALL_HASHLIB_NEW(MODULE, NAME, DATA, STRING, USEDFORSECURITY) \ + return _hashlib_HASH_new_impl(MODULE, NAME, DATA, USEDFORSECURITY, STRING) /* The module-level function: new() */ /*[clinic input] -_hashlib.new as EVP_new +_hashlib.new as _hashlib_HASH_new - name as name_obj: object - string as data_obj: object(c_default="NULL") = b'' + name: str + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new hash object using the named algorithm. @@ -1037,136 +1340,142 @@ The MD5 and SHA1 algorithms are always supported. [clinic start generated code]*/ static PyObject * -EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=ddd5053f92dffe90 input=c24554d0337be1b0]*/ +_hashlib_HASH_new_impl(PyObject *module, const char *name, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=b905aaf9840c1bbd input=c34af6c6e696d44e]*/ { - char *name; - if (!PyArg_Parse(name_obj, "s", &name)) { - PyErr_SetString(PyExc_TypeError, "name must be a string"); + PyObject *data_obj; + if (_Py_hashlib_data_argument(&data_obj, data, string) < 0) { return NULL; } - return py_evp_fromname(module, name, data_obj, usedforsecurity); + _hashlibstate *state = get_hashlib_state(module); + return _hashlib_HASH(state, name, data_obj, usedforsecurity); } /*[clinic input] _hashlib.openssl_md5 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a md5 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=87b0186440a44f8c input=990e36d5e689b16e]*/ +_hashlib_openssl_md5_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=ca8cf184d90f7432 input=e7c0adbd6a867db1]*/ { - return py_evp_fromname(module, Py_hash_md5, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_md5, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha1 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha1 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=6813024cf690670d input=948f2f4b6deabc10]*/ +_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=1736fb7b310d64be input=f7e5bb1711e952d8]*/ { - return py_evp_fromname(module, Py_hash_sha1, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha1, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha224 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha224 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=a2dfe7cc4eb14ebb input=f9272821fadca505]*/ +_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=0d6ff57be5e5c140 input=3820fff7ed3a53b8]*/ { - return py_evp_fromname(module, Py_hash_sha224, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha224, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha256 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=1f874a34870f0a68 input=549fad9d2930d4c5]*/ +_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=412ea7111555b6e7 input=9a2f115cf1f7e0eb]*/ { - return py_evp_fromname(module, Py_hash_sha256, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha256, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha384 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha384 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=58529eff9ca457b2 input=48601a6e3bf14ad7]*/ +_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=2e0dc395b59ed726 input=1ea48f6f01e77cfb]*/ { - return py_evp_fromname(module, Py_hash_sha384, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha384, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha512 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha512 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=2c744c9e4a40d5f6 input=c5c46a2a817aa98f]*/ +_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=4bdd760388dbfc0f input=3cf56903e07d1f5c]*/ { - return py_evp_fromname(module, Py_hash_sha512, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha512, data, string, usedforsecurity); } @@ -1175,121 +1484,132 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, /*[clinic input] _hashlib.openssl_sha3_224 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-224 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=144641c1d144b974 input=e3a01b2888916157]*/ +_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=6d8dc2a924f3ba35 input=7f14f16a9f6a3158]*/ { - return py_evp_fromname(module, Py_hash_sha3_224, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_224, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-256 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=c61f1ab772d06668 input=e2908126c1b6deed]*/ +_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=9e520f537b3a4622 input=7987150939d5e352]*/ { - return py_evp_fromname(module, Py_hash_sha3_256, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_256, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_384 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-384 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=f68e4846858cf0ee input=ec0edf5c792f8252]*/ +_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=d239ba0463fd6138 input=fc943401f67e3b81]*/ { - return py_evp_fromname(module, Py_hash_sha3_384, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_384, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_512 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-512 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=2eede478c159354a input=64e2cc0c094d56f4]*/ +_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=17662f21038c2278 input=6601ddd2c6c1516d]*/ { - return py_evp_fromname(module, Py_hash_sha3_512, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_512, data, string, usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHA3 */ #ifdef PY_OPENSSL_HAS_SHAKE /*[clinic input] +@permit_long_summary _hashlib.openssl_shake_128 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a shake-128 variable hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=bc49cdd8ada1fa97 input=6c9d67440eb33ec8]*/ +_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=4e6afed8d18980ad input=0d2803af1158b23c]*/ { - return py_evp_fromname(module, Py_hash_shake_128, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_shake_128, data, string, usedforsecurity); } /*[clinic input] +@permit_long_summary _hashlib.openssl_shake_256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a shake-256 variable hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=358d213be8852df7 input=479cbe9fefd4a9f8]*/ +_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=62481bce4a77d16c input=f27b98d9c749f55d]*/ { - return py_evp_fromname(module, Py_hash_shake_256, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_shake_256, data, string, usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHAKE */ +#undef CALL_HASHLIB_NEW + /*[clinic input] +@permit_long_summary _hashlib.pbkdf2_hmac as pbkdf2_hmac hash_name: str @@ -1305,14 +1625,15 @@ static PyObject * pbkdf2_hmac_impl(PyObject *module, const char *hash_name, Py_buffer *password, Py_buffer *salt, long iterations, PyObject *dklen_obj) -/*[clinic end generated code: output=144b76005416599b input=ed3ab0d2d28b5d5c]*/ +/*[clinic end generated code: output=144b76005416599b input=83417fbd9ec2b8a3]*/ { + _hashlibstate *state = get_hashlib_state(module); PyObject *key_obj = NULL; - char *key; long dklen; int retval; - PY_EVP_MD *digest = py_digest_by_name(module, hash_name, Py_ht_pbkdf2); + PY_EVP_MD *digest = get_openssl_evp_md_by_utf8name(state, hash_name, + Py_ht_pbkdf2); if (digest == NULL) { goto end; } @@ -1360,24 +1681,24 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, goto end; } - key_obj = PyBytes_FromStringAndSize(NULL, dklen); - if (key_obj == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(dklen); + if (writer == NULL) { goto end; } - key = PyBytes_AS_STRING(key_obj); Py_BEGIN_ALLOW_THREADS retval = PKCS5_PBKDF2_HMAC((const char *)password->buf, (int)password->len, (const unsigned char *)salt->buf, (int)salt->len, iterations, digest, dklen, - (unsigned char *)key); + (unsigned char *)PyBytesWriter_GetData(writer)); Py_END_ALLOW_THREADS if (!retval) { - Py_CLEAR(key_obj); - notify_ssl_error_occurred(); + PyBytesWriter_Discard(writer); + notify_ssl_error_occurred_in(Py_STRINGIFY(PKCS5_PBKDF2_HMAC)); goto end; } + key_obj = PyBytesWriter_Finish(writer); end: if (digest != NULL) { @@ -1386,7 +1707,25 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, return key_obj; } -#ifdef PY_OPENSSL_HAS_SCRYPT +// --- PBKDF: scrypt (RFC 7914) ----------------------------------------------- + +/* + * By default, OpenSSL 1.1.0 restricts 'maxmem' in EVP_PBE_scrypt() + * to 32 MiB (1024 * 1024 * 32) but only if 'maxmem = 0' and allows + * for an arbitrary large limit fitting on an uint64_t otherwise. + * + * For legacy reasons, we limited 'maxmem' to be at most INTMAX, + * but if users need a more relaxed value, we will revisit this + * limit in the future. + */ +#define HASHLIB_SCRYPT_MAX_MAXMEM INT_MAX + +/* + * Limit 'dklen' to INT_MAX even if it can be at most (32 * UINT32_MAX). + * + * See https://datatracker.ietf.org/doc/html/rfc7914.html for details. + */ +#define HASHLIB_SCRYPT_MAX_DKLEN INT_MAX /*[clinic input] _hashlib.scrypt @@ -1400,7 +1739,6 @@ _hashlib.scrypt maxmem: long = 0 dklen: long = 64 - scrypt password-based key derivation function. [clinic start generated code]*/ @@ -1408,81 +1746,94 @@ static PyObject * _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, unsigned long n, unsigned long r, unsigned long p, long maxmem, long dklen) -/*[clinic end generated code: output=d424bc3e8c6b9654 input=0c9a84230238fd79]*/ +/*[clinic end generated code: output=d424bc3e8c6b9654 input=bdeac9628d07f7a1]*/ { - PyObject *key_obj = NULL; - char *key; int retval; if (password->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "password is too long."); + PyErr_SetString(PyExc_OverflowError, "password is too long"); return NULL; } if (salt->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "salt is too long."); + PyErr_SetString(PyExc_OverflowError, "salt is too long"); return NULL; } if (n < 2 || n & (n - 1)) { - PyErr_SetString(PyExc_ValueError, - "n must be a power of 2."); + PyErr_SetString(PyExc_ValueError, "n must be a power of 2"); return NULL; } - if (maxmem < 0 || maxmem > INT_MAX) { - /* OpenSSL 1.1.0 restricts maxmem to 32 MiB. It may change in the - future. The maxmem constant is private to OpenSSL. */ + if (maxmem < 0 || maxmem > HASHLIB_SCRYPT_MAX_MAXMEM) { PyErr_Format(PyExc_ValueError, - "maxmem must be positive and smaller than %d", - INT_MAX); + "maxmem must be positive and at most %d", + HASHLIB_SCRYPT_MAX_MAXMEM); return NULL; } - if (dklen < 1 || dklen > INT_MAX) { + if (dklen < 1 || dklen > HASHLIB_SCRYPT_MAX_DKLEN) { PyErr_Format(PyExc_ValueError, - "dklen must be greater than 0 and smaller than %d", - INT_MAX); + "dklen must be at least 1 and at most %d", + HASHLIB_SCRYPT_MAX_DKLEN); return NULL; } /* let OpenSSL validate the rest */ - retval = EVP_PBE_scrypt(NULL, 0, NULL, 0, n, r, p, maxmem, NULL, 0); + retval = EVP_PBE_scrypt(NULL, 0, NULL, 0, n, r, p, + (uint64_t)maxmem, NULL, 0); if (!retval) { - raise_ssl_error(PyExc_ValueError, - "Invalid parameter combination for n, r, p, maxmem."); + notify_ssl_error_occurred("invalid parameter combination for " + "n, r, p, and maxmem"); return NULL; } - key_obj = PyBytes_FromStringAndSize(NULL, dklen); - if (key_obj == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(dklen); + if (writer == NULL) { return NULL; } - key = PyBytes_AS_STRING(key_obj); Py_BEGIN_ALLOW_THREADS retval = EVP_PBE_scrypt( - (const char*)password->buf, (size_t)password->len, + (const char *)password->buf, (size_t)password->len, (const unsigned char *)salt->buf, (size_t)salt->len, - n, r, p, maxmem, - (unsigned char *)key, (size_t)dklen + (uint64_t)n, (uint64_t)r, (uint64_t)p, (uint64_t)maxmem, + (unsigned char *)PyBytesWriter_GetData(writer), (size_t)dklen ); Py_END_ALLOW_THREADS if (!retval) { - Py_CLEAR(key_obj); - notify_ssl_error_occurred(); + PyBytesWriter_Discard(writer); + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_PBE_scrypt)); return NULL; } - return key_obj; + return PyBytesWriter_Finish(writer); } -#endif /* PY_OPENSSL_HAS_SCRYPT */ -/* Fast HMAC for hmac.digest() +#undef HASHLIB_SCRYPT_MAX_DKLEN +#undef HASHLIB_SCRYPT_MAX_MAXMEM + +// --- OpenSSL HMAC interface ------------------------------------------------- + +/* + * Functions prefixed by hashlib_openssl_HMAC_* are wrappers around OpenSSL + * and implement "atomic" operations (e.g., "free"). These functions are used + * by those prefixed by _hashlib_HMAC_* that are methods for HMAC objects, or + * other (local) helper functions prefixed by hashlib_HMAC_*. */ +#ifdef Py_HAS_OPENSSL3_SUPPORT +/* EVP_MAC_CTX array of parameters specifying the "digest" */ +#define HASHLIB_HMAC_OSSL_PARAMS(DIGEST) \ + (const OSSL_PARAM []) { \ + OSSL_PARAM_utf8_string(OSSL_MAC_PARAM_DIGEST, \ + (char *)DIGEST, strlen(DIGEST)), \ + OSSL_PARAM_END \ + } +#endif + +// --- One-shot HMAC interface ------------------------------------------------ + /*[clinic input] _hashlib.hmac_digest as _hashlib_hmac_singleshot @@ -1498,10 +1849,17 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, Py_buffer *msg, PyObject *digest) /*[clinic end generated code: output=82f19965d12706ac input=0a0790cc3db45c2e]*/ { + _hashlibstate *state = get_hashlib_state(module); unsigned char md[EVP_MAX_MD_SIZE] = {0}; +#ifdef Py_HAS_OPENSSL3_SUPPORT + size_t md_len = 0; + const char *digest_name = NULL; +#else unsigned int md_len = 0; - unsigned char *result; - PY_EVP_MD *evp; +#endif + unsigned char *result = NULL; + PY_EVP_MD *evp = NULL; + int is_xof; if (key->len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, @@ -1514,10 +1872,32 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, return NULL; } - evp = py_digest_by_digestmod(module, digest, Py_ht_mac); +#ifdef Py_HAS_OPENSSL3_SUPPORT + digest_name = get_openssl_digest_name(state, digest, Py_ht_mac, &evp); + if (digest_name == NULL) { + assert(evp == NULL); + return NULL; + } + assert(evp != NULL); + is_xof = PY_EVP_MD_xof(evp); + + Py_BEGIN_ALLOW_THREADS + result = EVP_Q_mac( + NULL, OSSL_MAC_NAME_HMAC, NULL, NULL, + HASHLIB_HMAC_OSSL_PARAMS(digest_name), + (const void *)key->buf, (size_t)key->len, + (const unsigned char *)msg->buf, (size_t)msg->len, + md, sizeof(md), &md_len + ); + Py_END_ALLOW_THREADS + PY_EVP_MD_free(evp); + assert(md_len < (size_t)PY_SSIZE_T_MAX); +#else + evp = get_openssl_evp_md(state, digest, Py_ht_mac); if (evp == NULL) { return NULL; } + is_xof = PY_EVP_MD_xof(evp); Py_BEGIN_ALLOW_THREADS result = HMAC( @@ -1528,28 +1908,211 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, ); Py_END_ALLOW_THREADS PY_EVP_MD_free(evp); - +#endif if (result == NULL) { - notify_ssl_error_occurred(); + if (is_xof) { + /* use a better default error message if an XOF is used */ + raise_unsupported_algorithm_error(state, digest); + } + else { +#ifdef Py_HAS_OPENSSL3_SUPPORT + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_Q_mac)); +#else + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC)); +#endif + } return NULL; } return PyBytes_FromStringAndSize((const char*)md, md_len); } -/* OpenSSL-based HMAC implementation - */ +// --- HMAC Object ------------------------------------------------------------ + +#ifndef Py_HAS_OPENSSL3_SUPPORT +/* Thin wrapper around HMAC_CTX_new() which sets an exception on failure. */ +static HMAC_CTX * +py_openssl_wrapper_HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = HMAC_CTX_new(); + if (ctx == NULL) { + PyErr_NoMemory(); + return NULL; + } + return ctx; +} +#endif static int _hmac_update(HMACobject*, PyObject*); +#ifndef Py_HAS_OPENSSL3_SUPPORT static const EVP_MD * _hashlib_hmac_get_md(HMACobject *self) { + assert(self->ctx != NULL); const EVP_MD *md = HMAC_CTX_get_md(self->ctx); if (md == NULL) { - raise_ssl_error(PyExc_ValueError, "missing EVP_MD for HMAC context"); + notify_ssl_error_occurred("missing EVP_MD for HMAC context"); } return md; } +#endif + +static const char * +hashlib_HMAC_get_hashlib_digest_name(HMACobject *self) +{ +#ifdef Py_HAS_OPENSSL3_SUPPORT + return get_hashlib_utf8name_by_nid(self->evp_md_nid); +#else + const EVP_MD *md = _hashlib_hmac_get_md(self); + return md == NULL ? NULL : get_hashlib_utf8name_by_evp_md(md); +#endif +} + +static int +hashlib_openssl_HMAC_update_once(PY_HMAC_CTX_TYPE *ctx, const Py_buffer *v) +{ + if (!PY_HMAC_update(ctx, (const unsigned char *)v->buf, (size_t)v->len)) { + notify_smart_ssl_error_occurred_in(Py_STRINGIFY(PY_HMAC_update)); + return -1; + } + return 0; +} + +/* Thin wrapper around PY_HMAC_CTX_free that allows to pass a NULL 'ctx'. */ +static inline void +hashlib_openssl_HMAC_CTX_free(PY_HMAC_CTX_TYPE *ctx) +{ + /* The NULL check was not present in every OpenSSL versions. */ + if (ctx) { + PY_HMAC_CTX_free(ctx); + } +} + +static PY_HMAC_CTX_TYPE * +hashlib_openssl_HMAC_ctx_copy_with_lock(HMACobject *self) +{ + PY_HMAC_CTX_TYPE *ctx = NULL; +#ifdef Py_HAS_OPENSSL3_SUPPORT + HASHLIB_ACQUIRE_LOCK(self); + ctx = EVP_MAC_CTX_dup(self->ctx); + HASHLIB_RELEASE_LOCK(self); + if (ctx == NULL) { + notify_smart_ssl_error_occurred_in(Py_STRINGIFY(EVP_MAC_CTX_dup)); + goto error; + } +#else + int r; + ctx = py_openssl_wrapper_HMAC_CTX_new(); + if (ctx == NULL) { + return NULL; + } + HASHLIB_ACQUIRE_LOCK(self); + r = HMAC_CTX_copy(ctx, self->ctx); + HASHLIB_RELEASE_LOCK(self); + if (r == 0) { + notify_smart_ssl_error_occurred_in(Py_STRINGIFY(HMAC_CTX_copy)); + goto error; + } +#endif + return ctx; + +error: + hashlib_openssl_HMAC_CTX_free(ctx); + return NULL; +} + +static PY_HMAC_CTX_TYPE * +hashlib_HMAC_CTX_new_from_digestmod(_hashlibstate *state, + Py_buffer *key, PyObject *digestmod, + int *nid) +{ + PY_HMAC_CTX_TYPE *ctx = NULL; + PY_EVP_MD *md = NULL; + int is_xof, r; +#ifdef Py_HAS_OPENSSL3_SUPPORT + const char *digest = NULL; +#endif + +#ifdef Py_HAS_OPENSSL3_SUPPORT + /* + * OpenSSL 3.0 does not provide a way to extract the NID from an EVP_MAC + * object and does not expose the underlying digest name. The reason is + * that OpenSSL 3.0 treats HMAC objects as being the "same", differing + * only by their *context* parameters. While it is *required* to set + * the digest name when constructing EVP_MAC_CTX objects, that name + * is unfortunately not recoverable through EVP_MAC_CTX_get_params(). + * + * On the other hand, the (deprecated) interface based on HMAC_CTX is + * based on EVP_MD, which allows to treat HMAC objects as if they were + * hash functions when querying the digest name. + * + * Since HMAC objects are constructed from DIGESTMOD values and since + * we have a way to map DIGESTMOD to EVP_MD objects, and then to NIDs, + * HMAC objects based on EVP_MAC will store the NID of the EVP_MD we + * used to deduce the digest name to pass to EVP_MAC_CTX_set_params(). + */ + assert(nid != NULL); + digest = get_openssl_digest_name(state, digestmod, Py_ht_mac, &md); + assert((digest == NULL && md == NULL) || (digest != NULL && md != NULL)); + if (digest == NULL) { + *nid = NID_undef; + return NULL; + } + *nid = EVP_MD_nid(md); + is_xof = PY_EVP_MD_xof(md); + PY_EVP_MD_free(md); + + /* + * OpenSSL is responsible for managing the EVP_MAC object's ref. count + * by calling EVP_MAC_up_ref() and EVP_MAC_free() in EVP_MAC_CTX_new() + * and EVP_MAC_CTX_free() respectively. + */ + ctx = EVP_MAC_CTX_new(state->evp_hmac); + if (ctx == NULL) { + /* EVP_MAC_CTX_new() may also set an ERR_R_EVP_LIB error */ + notify_smart_ssl_error_occurred_in(Py_STRINGIFY(EVP_MAC_CTX_new)); + return NULL; + } + + r = EVP_MAC_init( + ctx, + (const unsigned char *)key->buf, + (size_t)key->len, + HASHLIB_HMAC_OSSL_PARAMS(digest) + ); +#else + assert(nid == NULL); + md = get_openssl_evp_md(state, digestmod, Py_ht_mac); + if (md == NULL) { + return NULL; + } + is_xof = PY_EVP_MD_xof(md); + + ctx = py_openssl_wrapper_HMAC_CTX_new(); + if (ctx == NULL) { + PY_EVP_MD_free(md); + return NULL; + } + + r = HMAC_Init_ex(ctx, key->buf, (int)key->len, md, NULL /* impl */); + PY_EVP_MD_free(md); +#endif + if (r == 0) { + if (is_xof) { + /* use a better default error message if an XOF is used */ + raise_unsupported_algorithm_error(state, digestmod); + } + else { +#ifdef Py_HAS_OPENSSL3_SUPPORT + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_MAC_init)); +#else + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC_Init_ex)); +#endif + } + return NULL; + } + return ctx; +} /*[clinic input] _hashlib.hmac_new @@ -1566,10 +2129,12 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, PyObject *digestmod) /*[clinic end generated code: output=c20d9e4d9ed6d219 input=5f4071dcc7f34362]*/ { - PY_EVP_MD *digest; - HMAC_CTX *ctx = NULL; + _hashlibstate *state = get_hashlib_state(module); + PY_HMAC_CTX_TYPE *ctx = NULL; HMACobject *self = NULL; - int r; +#ifdef Py_HAS_OPENSSL3_SUPPORT + int nid; +#endif if (key->len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, @@ -1583,73 +2148,76 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, return NULL; } - digest = py_digest_by_digestmod(module, digestmod, Py_ht_mac); - if (digest == NULL) { +#ifdef Py_HAS_OPENSSL3_SUPPORT + ctx = hashlib_HMAC_CTX_new_from_digestmod(state, key, digestmod, &nid); +#else + ctx = hashlib_HMAC_CTX_new_from_digestmod(state, key, digestmod, NULL); +#endif + + if (ctx == NULL) { + assert(PyErr_Occurred()); return NULL; } - ctx = HMAC_CTX_new(); - if (ctx == NULL) { - PY_EVP_MD_free(digest); - PyErr_NoMemory(); - goto error; - } - - r = HMAC_Init_ex(ctx, key->buf, (int)key->len, digest, NULL /* impl */); - PY_EVP_MD_free(digest); - if (r == 0) { - notify_ssl_error_occurred(); - goto error; - } - - _hashlibstate *state = get_hashlib_state(module); - self = PyObject_New(HMACobject, state->HMACtype); + self = PyObject_New(HMACobject, state->HMAC_type); if (self == NULL) { goto error; } self->ctx = ctx; ctx = NULL; // 'ctx' is now owned by 'self' +#ifdef Py_HAS_OPENSSL3_SUPPORT + assert(nid != NID_undef); + self->evp_md_nid = nid; +#endif HASHLIB_INIT_MUTEX(self); + /* feed initial data */ if ((msg_obj != NULL) && (msg_obj != Py_None)) { - if (!_hmac_update(self, msg_obj)) { + if (_hmac_update(self, msg_obj) < 0) { goto error; } } return (PyObject *)self; error: - if (ctx) HMAC_CTX_free(ctx); + hashlib_openssl_HMAC_CTX_free(ctx); Py_XDECREF(self); return NULL; } /* helper functions */ -static int -locked_HMAC_CTX_copy(HMAC_CTX *new_ctx_p, HMACobject *self) -{ - int result; - ENTER_HASHLIB(self); - result = HMAC_CTX_copy(new_ctx_p, self->ctx); - LEAVE_HASHLIB(self); - return result; -} +#define BAD_DIGEST_SIZE 0 -/* returning 0 means that an error occurred and an exception is set */ +/* + * Return the digest size in bytes. + * + * On error, set an exception and return BAD_DIGEST_SIZE. + */ static unsigned int _hashlib_hmac_digest_size(HMACobject *self) { + assert(EVP_MAX_MD_SIZE < INT_MAX); +#ifdef Py_HAS_OPENSSL3_SUPPORT + assert(self->ctx != NULL); + size_t digest_size = EVP_MAC_CTX_get_mac_size(self->ctx); + assert(digest_size <= (size_t)EVP_MAX_MD_SIZE); +#else const EVP_MD *md = _hashlib_hmac_get_md(self); if (md == NULL) { - return 0; + return BAD_DIGEST_SIZE; } - unsigned int digest_size = EVP_MD_size(md); - assert(digest_size <= EVP_MAX_MD_SIZE); + int digest_size = EVP_MD_size(md); + /* digest_size < 0 iff EVP_MD context is NULL (which is impossible here) */ + assert(digest_size >= 0); + assert(digest_size <= (int)EVP_MAX_MD_SIZE); +#endif + /* digest_size == 0 means that the context is not entirely initialized */ if (digest_size == 0) { - raise_ssl_error(PyExc_ValueError, "invalid digest size"); + raise_ssl_error(PyExc_ValueError, "missing digest size"); + return BAD_DIGEST_SIZE; } - return digest_size; + return (unsigned int)digest_size; } static int @@ -1657,33 +2225,13 @@ _hmac_update(HMACobject *self, PyObject *obj) { int r; Py_buffer view = {0}; - - GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0); - - if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - r = HMAC_Update(self->ctx, - (const unsigned char *)view.buf, - (size_t)view.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - r = HMAC_Update(self->ctx, - (const unsigned char *)view.buf, - (size_t)view.len); - } - + GET_BUFFER_VIEW_OR_ERROR(obj, &view, return -1); + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, view.len, + r = hashlib_openssl_HMAC_update_once(self->ctx, &view) + ); PyBuffer_Release(&view); - - if (r == 0) { - notify_ssl_error_occurred(); - return 0; - } - return 1; + return r; } /*[clinic input] @@ -1697,25 +2245,17 @@ _hashlib_HMAC_copy_impl(HMACobject *self) /*[clinic end generated code: output=29aa28b452833127 input=e2fa6a05db61a4d6]*/ { HMACobject *retval; - - HMAC_CTX *ctx = HMAC_CTX_new(); + PY_HMAC_CTX_TYPE *ctx = hashlib_openssl_HMAC_ctx_copy_with_lock(self); if (ctx == NULL) { - return PyErr_NoMemory(); - } - if (!locked_HMAC_CTX_copy(ctx, self)) { - HMAC_CTX_free(ctx); - notify_ssl_error_occurred(); return NULL; } - retval = PyObject_New(HMACobject, Py_TYPE(self)); if (retval == NULL) { - HMAC_CTX_free(ctx); + PY_HMAC_CTX_free(ctx); return NULL; } retval->ctx = ctx; HASHLIB_INIT_MUTEX(retval); - return (PyObject *)retval; } @@ -1725,7 +2265,7 @@ _hmac_dealloc(PyObject *op) HMACobject *self = HMACobject_CAST(op); PyTypeObject *tp = Py_TYPE(self); if (self->ctx != NULL) { - HMAC_CTX_free(self->ctx); + PY_HMAC_CTX_free(self->ctx); self->ctx = NULL; } PyObject_Free(self); @@ -1736,19 +2276,12 @@ static PyObject * _hmac_repr(PyObject *op) { HMACobject *self = HMACobject_CAST(op); - const EVP_MD *md = _hashlib_hmac_get_md(self); - if (md == NULL) { - return NULL; - } - PyObject *digest_name = py_digest_name(md); + const char *digest_name = hashlib_HMAC_get_hashlib_digest_name(self); if (digest_name == NULL) { + assert(PyErr_Occurred()); return NULL; } - PyObject *repr = PyUnicode_FromFormat( - "<%U HMAC object @ %p>", digest_name, self - ); - Py_DECREF(digest_name); - return repr; + return PyUnicode_FromFormat("<%s HMAC object @ %p>", digest_name, self); } /*[clinic input] @@ -1762,32 +2295,48 @@ static PyObject * _hashlib_HMAC_update_impl(HMACobject *self, PyObject *msg) /*[clinic end generated code: output=f31f0ace8c625b00 input=1829173bb3cfd4e6]*/ { - if (!_hmac_update(self, msg)) { + if (_hmac_update(self, msg) < 0) { return NULL; } Py_RETURN_NONE; } -static int -_hmac_digest(HMACobject *self, unsigned char *buf, unsigned int len) +/* + * Extract the MAC value to 'buf' and return the digest size. + * + * The buffer 'buf' must have at least _hashlib_hmac_digest_size(self) + * bytes. Smaller buffers lead to undefined behaviors. + * + * On error, set an exception and return -1. + */ +static Py_ssize_t +_hmac_digest(HMACobject *self, unsigned char *buf) { - HMAC_CTX *temp_ctx = HMAC_CTX_new(); - if (temp_ctx == NULL) { - (void)PyErr_NoMemory(); - return 0; + unsigned int digest_size = _hashlib_hmac_digest_size(self); + assert(digest_size <= EVP_MAX_MD_SIZE); + if (digest_size == BAD_DIGEST_SIZE) { + assert(PyErr_Occurred()); + return -1; } - if (!locked_HMAC_CTX_copy(temp_ctx, self)) { - HMAC_CTX_free(temp_ctx); - notify_ssl_error_occurred(); - return 0; + PY_HMAC_CTX_TYPE *ctx = hashlib_openssl_HMAC_ctx_copy_with_lock(self); + if (ctx == NULL) { + return -1; } - int r = HMAC_Final(temp_ctx, buf, &len); - HMAC_CTX_free(temp_ctx); +#ifdef Py_HAS_OPENSSL3_SUPPORT + int r = EVP_MAC_final(ctx, buf, NULL, digest_size); +#else + int r = HMAC_Final(ctx, buf, NULL); +#endif + PY_HMAC_CTX_free(ctx); if (r == 0) { - notify_ssl_error_occurred(); - return 0; +#ifdef Py_HAS_OPENSSL3_SUPPORT + notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_MAC_final)); +#else + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC_Final)); +#endif + return -1; } - return 1; + return digest_size; } /*[clinic input] @@ -1799,19 +2348,14 @@ static PyObject * _hashlib_HMAC_digest_impl(HMACobject *self) /*[clinic end generated code: output=1b1424355af7a41e input=bff07f74da318fb4]*/ { - unsigned char digest[EVP_MAX_MD_SIZE]; - unsigned int digest_size = _hashlib_hmac_digest_size(self); - if (digest_size == 0) { - return NULL; - } - int r = _hmac_digest(self, digest, digest_size); - if (r == 0) { - return NULL; - } - return PyBytes_FromStringAndSize((const char *)digest, digest_size); + unsigned char buf[EVP_MAX_MD_SIZE]; + Py_ssize_t n = _hmac_digest(self, buf); + return n < 0 ? NULL : PyBytes_FromStringAndSize((const char *)buf, n); } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body _hashlib.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. @@ -1822,51 +2366,44 @@ environments. static PyObject * _hashlib_HMAC_hexdigest_impl(HMACobject *self) -/*[clinic end generated code: output=80d825be1eaae6a7 input=5abc42702874ddcf]*/ +/*[clinic end generated code: output=80d825be1eaae6a7 input=5e48db83ab1a4d19]*/ { - unsigned char digest[EVP_MAX_MD_SIZE]; - unsigned int digest_size = _hashlib_hmac_digest_size(self); - if (digest_size == 0) { - return NULL; - } - int r = _hmac_digest(self, digest, digest_size); - if (r == 0) { - return NULL; - } - return _Py_strhex((const char *)digest, digest_size); + unsigned char buf[EVP_MAX_MD_SIZE]; + Py_ssize_t n = _hmac_digest(self, buf); + return n < 0 ? NULL : _Py_strhex((const char *)buf, n); } static PyObject * _hashlib_hmac_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) { HMACobject *self = HMACobject_CAST(op); - unsigned int digest_size = _hashlib_hmac_digest_size(self); - return digest_size == 0 ? NULL : PyLong_FromLong(digest_size); + unsigned int size = _hashlib_hmac_digest_size(self); + return size == BAD_DIGEST_SIZE ? NULL : PyLong_FromLong(size); } static PyObject * _hashlib_hmac_get_block_size(PyObject *op, void *Py_UNUSED(closure)) { HMACobject *self = HMACobject_CAST(op); +#ifdef Py_HAS_OPENSSL3_SUPPORT + assert(self->ctx != NULL); + return PyLong_FromSize_t(EVP_MAC_CTX_get_block_size(self->ctx)); +#else const EVP_MD *md = _hashlib_hmac_get_md(self); return md == NULL ? NULL : PyLong_FromLong(EVP_MD_block_size(md)); +#endif } static PyObject * _hashlib_hmac_get_name(PyObject *op, void *Py_UNUSED(closure)) { HMACobject *self = HMACobject_CAST(op); - const EVP_MD *md = _hashlib_hmac_get_md(self); - if (md == NULL) { - return NULL; - } - PyObject *digest_name = py_digest_name(md); + const char *digest_name = hashlib_HMAC_get_hashlib_digest_name(self); if (digest_name == NULL) { + assert(PyErr_Occurred()); return NULL; } - PyObject *name = PyUnicode_FromFormat("hmac-%U", digest_name); - Py_DECREF(digest_name); - return name; + return PyUnicode_FromFormat("hmac-%s", digest_name); } static PyMethodDef HMAC_methods[] = { @@ -1926,7 +2463,7 @@ typedef struct _internal_name_mapper_state { /* A callback function to pass to OpenSSL's OBJ_NAME_do_all(...) */ static void -#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#ifdef Py_HAS_OPENSSL3_SUPPORT _openssl_hash_name_mapper(EVP_MD *md, void *arg) #else _openssl_hash_name_mapper(const EVP_MD *md, const char *from, @@ -1942,7 +2479,9 @@ _openssl_hash_name_mapper(const EVP_MD *md, const char *from, return; } - py_name = py_digest_name(md); + const char *name = get_hashlib_utf8name_by_evp_md(md); + assert(name != NULL || PyErr_Occurred()); + py_name = name == NULL ? NULL : PyUnicode_FromString(name); if (py_name == NULL) { state->error = 1; } else { @@ -1966,7 +2505,7 @@ hashlib_md_meth_names(PyObject *module) return -1; } -#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#ifdef Py_HAS_OPENSSL3_SUPPORT // get algorithms from all activated providers in default context EVP_MD_do_all_provided(NULL, &_openssl_hash_name_mapper, &state); #else @@ -1999,21 +2538,18 @@ _hashlib_get_fips_mode_impl(PyObject *module) /*[clinic end generated code: output=87eece1bab4d3fa9 input=2db61538c41c6fef]*/ { -#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#ifdef Py_HAS_OPENSSL3_SUPPORT return EVP_default_properties_is_fips_enabled(NULL); #else ERR_clear_error(); int result = FIPS_mode(); - if (result == 0) { + if (result == 0 && ERR_peek_last_error()) { // "If the library was built without support of the FIPS Object Module, // then the function will return 0 with an error code of // CRYPTO_R_FIPS_MODE_NOT_SUPPORTED (0x0f06d065)." // But 0 is also a valid result value. - unsigned long errcode = ERR_peek_last_error(); - if (errcode) { - notify_ssl_error_occurred(); - return -1; - } + notify_ssl_error_occurred_in(Py_STRINGIFY(FIPS_mode)); + return -1; } return result; #endif @@ -2134,7 +2670,7 @@ _hashlib_compare_digest_impl(PyObject *module, PyObject *a, PyObject *b) /* List of functions exported by this module */ static struct PyMethodDef EVP_functions[] = { - EVP_NEW_METHODDEF + _HASHLIB_HASH_NEW_METHODDEF PBKDF2_HMAC_METHODDEF _HASHLIB_SCRYPT_METHODDEF _HASHLIB_GET_FIPS_MODE_METHODDEF @@ -2163,10 +2699,10 @@ static int hashlib_traverse(PyObject *m, visitproc visit, void *arg) { _hashlibstate *state = get_hashlib_state(m); - Py_VISIT(state->EVPtype); - Py_VISIT(state->HMACtype); + Py_VISIT(state->HASH_type); + Py_VISIT(state->HMAC_type); #ifdef PY_OPENSSL_HAS_SHAKE - Py_VISIT(state->EVPXOFtype); + Py_VISIT(state->HASHXOF_type); #endif Py_VISIT(state->constructs); Py_VISIT(state->unsupported_digestmod_error); @@ -2177,10 +2713,10 @@ static int hashlib_clear(PyObject *m) { _hashlibstate *state = get_hashlib_state(m); - Py_CLEAR(state->EVPtype); - Py_CLEAR(state->HMACtype); + Py_CLEAR(state->HASH_type); + Py_CLEAR(state->HMAC_type); #ifdef PY_OPENSSL_HAS_SHAKE - Py_CLEAR(state->EVPXOFtype); + Py_CLEAR(state->HASHXOF_type); #endif Py_CLEAR(state->constructs); Py_CLEAR(state->unsupported_digestmod_error); @@ -2189,6 +2725,12 @@ hashlib_clear(PyObject *m) _Py_hashtable_destroy(state->hashtable); state->hashtable = NULL; } +#ifdef Py_HAS_OPENSSL3_SUPPORT + if (state->evp_hmac != NULL) { + EVP_MAC_free(state->evp_hmac); + state->evp_hmac = NULL; + } +#endif return 0; } @@ -2214,37 +2756,37 @@ hashlib_init_hashtable(PyObject *module) } static int -hashlib_init_evptype(PyObject *module) +hashlib_init_HASH_type(PyObject *module) { _hashlibstate *state = get_hashlib_state(module); - state->EVPtype = (PyTypeObject *)PyType_FromSpec(&EVPtype_spec); - if (state->EVPtype == NULL) { + state->HASH_type = (PyTypeObject *)PyType_FromSpec(&HASHobject_type_spec); + if (state->HASH_type == NULL) { return -1; } - if (PyModule_AddType(module, state->EVPtype) < 0) { + if (PyModule_AddType(module, state->HASH_type) < 0) { return -1; } return 0; } static int -hashlib_init_evpxoftype(PyObject *module) +hashlib_init_HASHXOF_type(PyObject *module) { #ifdef PY_OPENSSL_HAS_SHAKE _hashlibstate *state = get_hashlib_state(module); - if (state->EVPtype == NULL) { + if (state->HASH_type == NULL) { return -1; } - state->EVPXOFtype = (PyTypeObject *)PyType_FromSpecWithBases( - &EVPXOFtype_spec, (PyObject *)state->EVPtype + state->HASHXOF_type = (PyTypeObject *)PyType_FromSpecWithBases( + &HASHXOFobject_type_spec, (PyObject *)state->HASH_type ); - if (state->EVPXOFtype == NULL) { + if (state->HASHXOF_type == NULL) { return -1; } - if (PyModule_AddType(module, state->EVPXOFtype) < 0) { + if (PyModule_AddType(module, state->HASHXOF_type) < 0) { return -1; } #endif @@ -2256,13 +2798,22 @@ hashlib_init_hmactype(PyObject *module) { _hashlibstate *state = get_hashlib_state(module); - state->HMACtype = (PyTypeObject *)PyType_FromSpec(&HMACtype_spec); - if (state->HMACtype == NULL) { + state->HMAC_type = (PyTypeObject *)PyType_FromSpec(&HMACtype_spec); + if (state->HMAC_type == NULL) { return -1; } - if (PyModule_AddType(module, state->HMACtype) < 0) { + if (PyModule_AddType(module, state->HMAC_type) < 0) { return -1; } +#ifdef Py_HAS_OPENSSL3_SUPPORT + state->evp_hmac = EVP_MAC_fetch(NULL, "HMAC", NULL); + if (state->evp_hmac == NULL) { + ERR_clear_error(); + PyErr_SetString(PyExc_ImportError, "cannot initialize EVP_MAC HMAC"); + return -1; + } +#endif + return 0; } @@ -2341,8 +2892,8 @@ hashlib_constants(PyObject *module) static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_init_hashtable}, - {Py_mod_exec, hashlib_init_evptype}, - {Py_mod_exec, hashlib_init_evpxoftype}, + {Py_mod_exec, hashlib_init_HASH_type}, + {Py_mod_exec, hashlib_init_HASHXOF_type}, {Py_mod_exec, hashlib_init_hmactype}, {Py_mod_exec, hashlib_md_meth_names}, {Py_mod_exec, hashlib_init_constructors}, diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 095866eec7d..05d01acd771 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -11,7 +11,8 @@ annotated by François Pinard, and converted to C by Raymond Hettinger. #endif #include "Python.h" -#include "pycore_list.h" // _PyList_ITEMS() +#include "pycore_list.h" // _PyList_ITEMS(), _PyList_AppendTakeRef() +#include "pycore_pyatomic_ft_wrappers.h" #include "clinic/_heapqmodule.c.h" @@ -59,8 +60,8 @@ siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos) arr = _PyList_ITEMS(heap); parent = arr[parentpos]; newitem = arr[pos]; - arr[parentpos] = newitem; - arr[pos] = parent; + FT_ATOMIC_STORE_PTR_RELAXED(arr[parentpos], newitem); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], parent); pos = parentpos; } return 0; @@ -108,8 +109,8 @@ siftup(PyListObject *heap, Py_ssize_t pos) /* Move the smaller child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; - arr[childpos] = tmp2; - arr[pos] = tmp1; + FT_ATOMIC_STORE_PTR_RELAXED(arr[childpos], tmp2); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], tmp1); pos = childpos; } /* Bubble it up to its final resting place (by sifting its parents down). */ @@ -117,6 +118,7 @@ siftup(PyListObject *heap, Py_ssize_t pos) } /*[clinic input] +@critical_section heap _heapq.heappush heap: object(subclass_of='&PyList_Type') @@ -128,13 +130,22 @@ Push item onto heap, maintaining the heap invariant. static PyObject * _heapq_heappush_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=912c094f47663935 input=7c69611f3698aceb]*/ +/*[clinic end generated code: output=912c094f47663935 input=f7a4f03ef8d52e67]*/ { - if (PyList_Append(heap, item)) + if (item == NULL) { + PyErr_BadInternalCall(); return NULL; + } - if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) + // In a free-threaded build, the heap is locked at this point. + // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { return NULL; + } + + if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) { + return NULL; + } Py_RETURN_NONE; } @@ -162,8 +173,9 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) if (!n) return lastelt; returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, lastelt); - if (siftup_func((PyListObject *)heap, 0)) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], lastelt); + if (siftup_func(list, 0)) { Py_DECREF(returnitem); return NULL; } @@ -171,6 +183,7 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) } /*[clinic input] +@critical_section heap _heapq.heappop heap: object(subclass_of='&PyList_Type') @@ -181,7 +194,7 @@ Pop the smallest item off the heap, maintaining the heap invariant. static PyObject * _heapq_heappop_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=96dfe82d37d9af76 input=91487987a583c856]*/ +/*[clinic end generated code: output=96dfe82d37d9af76 input=ed396461b153dd51]*/ { return heappop_internal(heap, siftup); } @@ -197,8 +210,9 @@ heapreplace_internal(PyObject *heap, PyObject *item, int siftup_func(PyListObjec } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup_func((PyListObject *)heap, 0)) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup_func(list, 0)) { Py_DECREF(returnitem); return NULL; } @@ -207,6 +221,7 @@ heapreplace_internal(PyObject *heap, PyObject *item, int siftup_func(PyListObjec /*[clinic input] +@critical_section heap _heapq.heapreplace heap: object(subclass_of='&PyList_Type') @@ -226,12 +241,13 @@ this routine unless written as part of a conditional replacement: static PyObject * _heapq_heapreplace_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=82ea55be8fbe24b4 input=719202ac02ba10c8]*/ +/*[clinic end generated code: output=82ea55be8fbe24b4 input=9be1678b817ef1a9]*/ { return heapreplace_internal(heap, item, siftup); } /*[clinic input] +@critical_section heap _heapq.heappushpop heap: object(subclass_of='&PyList_Type') @@ -246,7 +262,7 @@ a separate call to heappop(). static PyObject * _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=67231dc98ed5774f input=5dc701f1eb4a4aa7]*/ +/*[clinic end generated code: output=67231dc98ed5774f input=db05c81b1dd92c44]*/ { PyObject *returnitem; int cmp; @@ -271,8 +287,9 @@ _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup((PyListObject *)heap, 0)) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup(list, 0)) { Py_DECREF(returnitem); return NULL; } @@ -371,6 +388,7 @@ heapify_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) } /*[clinic input] +@critical_section heap _heapq.heapify heap: object(subclass_of='&PyList_Type') @@ -381,7 +399,7 @@ Transform list into a heap, in-place, in O(len(heap)) time. static PyObject * _heapq_heapify_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=e63a636fcf83d6d0 input=53bb7a2166febb73]*/ +/*[clinic end generated code: output=e63a636fcf83d6d0 input=aaaaa028b9b6af08]*/ { return heapify_internal(heap, siftup); } @@ -423,8 +441,8 @@ siftdown_max(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos) arr = _PyList_ITEMS(heap); parent = arr[parentpos]; newitem = arr[pos]; - arr[parentpos] = newitem; - arr[pos] = parent; + FT_ATOMIC_STORE_PTR_RELAXED(arr[parentpos], newitem); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], parent); pos = parentpos; } return 0; @@ -445,11 +463,11 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) return -1; } - /* Bubble up the smaller child until hitting a leaf. */ + /* Bubble up the larger child until hitting a leaf. */ arr = _PyList_ITEMS(heap); limit = endpos >> 1; /* smallest pos that has no child */ while (pos < limit) { - /* Set childpos to index of smaller child. */ + /* Set childpos to index of larger child. */ childpos = 2*pos + 1; /* leftmost child position */ if (childpos + 1 < endpos) { PyObject* a = arr[childpos + 1]; @@ -469,11 +487,11 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) return -1; } } - /* Move the smaller child up. */ + /* Move the larger child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; - arr[childpos] = tmp2; - arr[pos] = tmp1; + FT_ATOMIC_STORE_PTR_RELAXED(arr[childpos], tmp2); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], tmp1); pos = childpos; } /* Bubble it up to its final resting place (by sifting its parents down). */ @@ -481,6 +499,7 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) } /*[clinic input] +@critical_section heap _heapq.heappush_max heap: object(subclass_of='&PyList_Type') @@ -492,9 +511,16 @@ Push item onto max heap, maintaining the heap invariant. static PyObject * _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=c869d5f9deb08277 input=4743d7db137b6e2b]*/ +/*[clinic end generated code: output=c869d5f9deb08277 input=c437e3d1ff8dcb70]*/ { - if (PyList_Append(heap, item)) { + if (item == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + // In a free-threaded build, the heap is locked at this point. + // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { return NULL; } @@ -506,6 +532,7 @@ _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) } /*[clinic input] +@critical_section heap _heapq.heappop_max heap: object(subclass_of='&PyList_Type') @@ -516,12 +543,13 @@ Maxheap variant of heappop. static PyObject * _heapq_heappop_max_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=2f051195ab404b77 input=e62b14016a5a26de]*/ +/*[clinic end generated code: output=2f051195ab404b77 input=5d70c997798aec64]*/ { return heappop_internal(heap, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heapreplace_max heap: object(subclass_of='&PyList_Type') @@ -533,12 +561,13 @@ Maxheap variant of heapreplace. static PyObject * _heapq_heapreplace_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=8770778b5a9cbe9b input=21a3d28d757c881c]*/ +/*[clinic end generated code: output=8770778b5a9cbe9b input=fe70175356e4a649]*/ { return heapreplace_internal(heap, item, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heapify_max heap: object(subclass_of='&PyList_Type') @@ -549,12 +578,13 @@ Maxheap variant of heapify. static PyObject * _heapq_heapify_max_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=8401af3856529807 input=edda4255728c431e]*/ +/*[clinic end generated code: output=8401af3856529807 input=4eee63231e7d1573]*/ { return heapify_internal(heap, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heappushpop_max heap: object(subclass_of='&PyList_Type') @@ -569,7 +599,7 @@ a separate call to heappop_max(). static PyObject * _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=ff0019f0941aca0d input=525a843013cbd6c0]*/ +/*[clinic end generated code: output=ff0019f0941aca0d input=24d0defa6fd6df4a]*/ { PyObject *returnitem; int cmp; @@ -595,8 +625,9 @@ _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup_max((PyListObject *)heap, 0) < 0) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup_max(list, 0) < 0) { Py_DECREF(returnitem); return NULL; } diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 172cebcaa48..ef9cf01ecbe 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -20,9 +20,11 @@ #endif #define REGISTERS_HEAP_TYPES +#define HAS_FALLBACK #define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" #undef HAS_UNBOUND_ITEMS +#undef HAS_FALLBACK #undef REGISTERS_HEAP_TYPES @@ -218,6 +220,22 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) return 0; } +static int +ensure_highlevel_module_loaded(void) +{ + PyObject *highlevel = + PyImport_ImportModule("concurrent.interpreters._channels"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.channels"); + if (highlevel == NULL) { + return -1; + } + } + Py_DECREF(highlevel); + return 0; +} + /* module state *************************************************************/ @@ -252,10 +270,10 @@ _get_current_module_state(void) { PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } module_state *state = get_module_state(mod); Py_DECREF(mod); @@ -493,12 +511,12 @@ _waiting_release(_waiting_t *waiting, int received) assert(!waiting->received); waiting->status = WAITING_RELEASING; - PyThread_release_lock(waiting->mutex); if (waiting->received != received) { assert(received == 1); waiting->received = received; } waiting->status = WAITING_RELEASED; + PyThread_release_lock(waiting->mutex); } static void @@ -523,7 +541,7 @@ typedef struct _channelitem { int64_t interpid; _PyXIData_t *data; _waiting_t *waiting; - int unboundop; + unboundop_t unboundop; struct _channelitem *next; } _channelitem; @@ -536,7 +554,7 @@ _channelitem_ID(_channelitem *item) static void _channelitem_init(_channelitem *item, int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { if (interpid < 0) { interpid = _get_interpid(data); @@ -562,7 +580,7 @@ _channelitem_clear_data(_channelitem *item, int removed) { if (item->data != NULL) { // It was allocated in channel_send(). - (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + (void)_release_xid_data(item->data, XID_IGNORE_EXC | XID_FREE); item->data = NULL; } @@ -583,7 +601,7 @@ _channelitem_clear(_channelitem *item) static _channelitem * _channelitem_new(int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { _channelitem *item = GLOBAL_MALLOC(_channelitem); if (item == NULL) { @@ -694,7 +712,7 @@ _channelqueue_free(_channelqueue *queue) static int _channelqueue_put(_channelqueue *queue, int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { _channelitem *item = _channelitem_new(interpid, data, waiting, unboundop); if (item == NULL) { @@ -798,7 +816,7 @@ _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, } queue->count -= 1; - int unboundop; + unboundop_t unboundop; _channelitem_popped(item, p_data, p_waiting, &unboundop); } @@ -1083,16 +1101,18 @@ typedef struct _channel { PyThread_type_lock mutex; _channelqueue *queue; _channelends *ends; - struct { - int unboundop; + struct _channeldefaults { + unboundop_t unboundop; + xidata_fallback_t fallback; } defaults; int open; struct _channel_closing *closing; } _channel_state; static _channel_state * -_channel_new(PyThread_type_lock mutex, int unboundop) +_channel_new(PyThread_type_lock mutex, struct _channeldefaults defaults) { + assert(check_unbound(defaults.unboundop)); _channel_state *chan = GLOBAL_MALLOC(_channel_state); if (chan == NULL) { return NULL; @@ -1109,7 +1129,7 @@ _channel_new(PyThread_type_lock mutex, int unboundop) GLOBAL_FREE(chan); return NULL; } - chan->defaults.unboundop = unboundop; + chan->defaults = defaults; chan->open = 1; chan->closing = NULL; return chan; @@ -1130,7 +1150,7 @@ _channel_free(_channel_state *chan) static int _channel_add(_channel_state *chan, int64_t interpid, - _PyXIData_t *data, _waiting_t *waiting, int unboundop) + _PyXIData_t *data, _waiting_t *waiting, unboundop_t unboundop) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1611,7 +1631,7 @@ _channels_release_cid_object(_channels *channels, int64_t cid) struct channel_id_and_info { int64_t id; - int unboundop; + struct _channeldefaults defaults; }; static struct channel_id_and_info * @@ -1628,7 +1648,7 @@ _channels_list_all(_channels *channels, int64_t *count) for (int64_t i=0; ref != NULL; ref = ref->next, i++) { ids[i] = (struct channel_id_and_info){ .id = ref->cid, - .unboundop = ref->chan->defaults.unboundop, + .defaults = ref->chan->defaults, }; } *count = channels->numopen; @@ -1714,13 +1734,13 @@ _channel_finish_closing(_channel_state *chan) { // Create a new channel. static int64_t -channel_create(_channels *channels, int unboundop) +channel_create(_channels *channels, struct _channeldefaults defaults) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_CHANNEL_MUTEX_INIT; } - _channel_state *chan = _channel_new(mutex, unboundop); + _channel_state *chan = _channel_new(mutex, defaults); if (chan == NULL) { PyThread_free_lock(mutex); return -1; @@ -1752,7 +1772,7 @@ channel_destroy(_channels *channels, int64_t cid) // Optionally request to be notified when it is received. static int channel_send(_channels *channels, int64_t cid, PyObject *obj, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop, xidata_fallback_t fallback) { PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; @@ -1779,7 +1799,7 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, PyThread_release_lock(mutex); return -1; } - if (_PyObject_GetXIData(tstate, obj, data) != 0) { + if (_PyObject_GetXIData(tstate, obj, fallback, data) != 0) { PyThread_release_lock(mutex); GLOBAL_FREE(data); return -1; @@ -1823,7 +1843,8 @@ channel_clear_sent(_channels *channels, int64_t cid, _waiting_t *waiting) // Like channel_send(), but strictly wait for the object to be received. static int channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, - int unboundop, PY_TIMEOUT_T timeout) + unboundop_t unboundop, PY_TIMEOUT_T timeout, + xidata_fallback_t fallback) { // We use a stack variable here, so we must ensure that &waiting // is not held by any channel item at the point this function exits. @@ -1834,7 +1855,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, } /* Queue up the object. */ - int res = channel_send(channels, cid, obj, &waiting, unboundop); + int res = channel_send(channels, cid, obj, &waiting, unboundop, fallback); if (res < 0) { assert(waiting.status == WAITING_NO_STATUS); goto finally; @@ -2005,6 +2026,20 @@ channel_is_associated(_channels *channels, int64_t cid, int64_t interpid, return (end != NULL && end->open); } +static int +channel_get_defaults(_channels *channels, int64_t cid, struct _channeldefaults *defaults) +{ + PyThread_type_lock mutex = NULL; + _channel_state *channel = NULL; + int err = _channels_lookup(channels, cid, &mutex, &channel); + if (err != 0) { + return err; + } + *defaults = channel->defaults; + PyThread_release_lock(mutex); + return 0; +} + static int _channel_get_count(_channels *channels, int64_t cid, Py_ssize_t *p_count) { @@ -2694,7 +2729,7 @@ add_channelid_type(PyObject *mod) Py_DECREF(cls); return NULL; } - if (ensure_xid_class(cls, _channelid_shared) < 0) { + if (ensure_xid_class(cls, GETDATA(_channelid_shared)) < 0) { Py_DECREF(cls); return NULL; } @@ -2723,15 +2758,9 @@ _get_current_channelend_type(int end) } if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); - if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.channels"); - if (highlevel == NULL) { - return NULL; - } + if (ensure_highlevel_module_loaded() < 0) { + return NULL; } - Py_DECREF(highlevel); if (end == CHANNEL_SEND) { cls = state->send_channel_type; } @@ -2797,12 +2826,12 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) // Add and register the types. state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - if (ensure_xid_class(send, _channelend_shared) < 0) { + if (ensure_xid_class(send, GETDATA(_channelend_shared)) < 0) { Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); return -1; } - if (ensure_xid_class(recv, _channelend_shared) < 0) { + if (ensure_xid_class(recv, GETDATA(_channelend_shared)) < 0) { (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); @@ -2881,20 +2910,27 @@ clear_interpreter(void *data) static PyObject * channelsmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"unboundop", NULL}; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "i:create", kwlist, - &unboundop)) + static char *kwlist[] = {"unboundop", "fallback", NULL}; + int unboundarg = -1; + int fallbackarg = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|ii:create", kwlist, + &unboundarg, &fallbackarg)) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + struct _channeldefaults defaults = {0}; + if (resolve_unboundop(unboundarg, UNBOUND_REPLACE, + &defaults.unboundop) < 0) + { + return NULL; + } + if (resolve_fallback(fallbackarg, _PyXIDATA_FULL_FALLBACK, + &defaults.fallback) < 0) + { return NULL; } - int64_t cid = channel_create(&_globals.channels, unboundop); + int64_t cid = channel_create(&_globals.channels, defaults); if (cid < 0) { (void)handle_channel_error(-1, self, cid); return NULL; @@ -2987,7 +3023,9 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } assert(cidobj != NULL); - PyObject *item = Py_BuildValue("Oi", cidobj, cur->unboundop); + PyObject *item = Py_BuildValue("Oii", cidobj, + cur->defaults.unboundop, + cur->defaults.fallback); Py_DECREF(cidobj); if (item == NULL) { Py_SETREF(ids, NULL); @@ -3075,40 +3113,54 @@ receive end."); static PyObject * channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", - NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "fallback", + "blocking", "timeout", NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; - int unboundop = UNBOUND_REPLACE; + int unboundarg = -1; + int fallbackarg = -1; int blocking = 1; PyObject *timeout_obj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|i$pO:channel_send", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&O|ii$pO:channel_send", kwlist, channel_id_converter, &cid_data, &obj, - &unboundop, &blocking, &timeout_obj)) + &unboundarg, &fallbackarg, + &blocking, &timeout_obj)) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); - return NULL; - } - int64_t cid = cid_data.cid; PY_TIMEOUT_T timeout; if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) { return NULL; } + struct _channeldefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = channel_get_defaults(&_globals.channels, cid, &defaults); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { + return NULL; + } + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { + return NULL; + } /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, obj, unboundop, timeout); + err = channel_send_wait( + &_globals.channels, cid, obj, unboundop, timeout, fallback); } else { - err = channel_send(&_globals.channels, cid, obj, NULL, unboundop); + err = channel_send( + &_globals.channels, cid, obj, NULL, unboundop, fallback); } if (handle_channel_error(err, self, cid)) { return NULL; @@ -3126,32 +3178,44 @@ By default this waits for the object to be received."); static PyObject * channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", - NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "fallback", + "blocking", "timeout", NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; - int unboundop = UNBOUND_REPLACE; - int blocking = 1; + int unboundarg = -1; + int fallbackarg = -1; + int blocking = -1; PyObject *timeout_obj = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&O|i$pO:channel_send_buffer", kwlist, + "O&O|ii$pO:channel_send_buffer", kwlist, channel_id_converter, &cid_data, &obj, - &unboundop, &blocking, &timeout_obj)) { + &unboundarg, &fallbackarg, + &blocking, &timeout_obj)) + { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); - return NULL; - } - int64_t cid = cid_data.cid; PY_TIMEOUT_T timeout; if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) { return NULL; } + struct _channeldefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = channel_get_defaults(&_globals.channels, cid, &defaults); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { + return NULL; + } + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { + return NULL; + } PyObject *tempobj = PyMemoryView_FromObject(obj); if (tempobj == NULL) { @@ -3162,10 +3226,11 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) int err = 0; if (blocking) { err = channel_send_wait( - &_globals.channels, cid, tempobj, unboundop, timeout); + &_globals.channels, cid, tempobj, unboundop, timeout, fallback); } else { - err = channel_send(&_globals.channels, cid, tempobj, NULL, unboundop); + err = channel_send( + &_globals.channels, cid, tempobj, NULL, unboundop, fallback); } Py_DECREF(tempobj); if (handle_channel_error(err, self, cid)) { @@ -3197,7 +3262,7 @@ channelsmod_recv(PyObject *self, PyObject *args, PyObject *kwds) cid = cid_data.cid; PyObject *obj = NULL; - int unboundop = 0; + unboundop_t unboundop = 0; int err = channel_recv(&_globals.channels, cid, &obj, &unboundop); if (err == ERR_CHANNEL_EMPTY && dflt != NULL) { // Use the default. @@ -3388,17 +3453,14 @@ channelsmod_get_channel_defaults(PyObject *self, PyObject *args, PyObject *kwds) } int64_t cid = cid_data.cid; - PyThread_type_lock mutex = NULL; - _channel_state *channel = NULL; - int err = _channels_lookup(&_globals.channels, cid, &mutex, &channel); + struct _channeldefaults defaults = {0}; + int err = channel_get_defaults(&_globals.channels, cid, &defaults); if (handle_channel_error(err, self, cid)) { return NULL; } - int unboundop = channel->defaults.unboundop; - PyThread_release_lock(mutex); - PyObject *defaults = Py_BuildValue("i", unboundop); - return defaults; + PyObject *res = Py_BuildValue("ii", defaults.unboundop, defaults.fallback); + return res; } PyDoc_STRVAR(channelsmod_get_channel_defaults_doc, @@ -3552,8 +3614,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) { module_state *state = get_module_state(mod); assert(state != NULL); - (void)traverse_module_state(state, visit, arg); - return 0; + return traverse_module_state(state, visit, arg); } static int @@ -3563,8 +3624,7 @@ module_clear(PyObject *mod) assert(state != NULL); // Now we clear the module state. - (void)clear_module_state(state); - return 0; + return clear_module_state(state); } static void diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 526249a0e1a..417c5fbcee2 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -9,9 +9,11 @@ #include "pycore_crossinterp.h" // _PyXIData_t #define REGISTERS_HEAP_TYPES +#define HAS_FALLBACK #define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" #undef HAS_UNBOUND_ITEMS +#undef HAS_FALLBACK #undef REGISTERS_HEAP_TYPES @@ -20,6 +22,11 @@ #define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) +/*[clinic input] +module _interpqueues +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=cb1313f77fab132b]*/ + #define GLOBAL_MALLOC(TYPE) \ PyMem_RawMalloc(sizeof(TYPE)) #define GLOBAL_FREE(VAR) \ @@ -134,13 +141,10 @@ idarg_int64_converter(PyObject *arg, void *ptr) static int ensure_highlevel_module_loaded(void) { - PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); + PyObject *highlevel = + PyImport_ImportModule("concurrent.interpreters._queues"); if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.queues"); - if (highlevel == NULL) { - return -1; - } + return -1; } Py_DECREF(highlevel); return 0; @@ -297,7 +301,7 @@ add_QueueError(PyObject *mod) { module_state *state = get_module_state(mod); -#define PREFIX "test.support.interpreters." +#define PREFIX "concurrent.interpreters." #define ADD_EXCTYPE(NAME, BASE, DOC) \ assert(state->NAME == NULL); \ if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ @@ -401,14 +405,13 @@ typedef struct _queueitem { meaning the interpreter has been destroyed. */ int64_t interpid; _PyXIData_t *data; - int fmt; - int unboundop; + unboundop_t unboundop; struct _queueitem *next; } _queueitem; static void _queueitem_init(_queueitem *item, - int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) + int64_t interpid, _PyXIData_t *data, unboundop_t unboundop) { if (interpid < 0) { interpid = _get_interpid(data); @@ -422,7 +425,6 @@ _queueitem_init(_queueitem *item, *item = (_queueitem){ .interpid = interpid, .data = data, - .fmt = fmt, .unboundop = unboundop, }; } @@ -434,7 +436,7 @@ _queueitem_clear_data(_queueitem *item) return; } // It was allocated in queue_put(). - (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + (void)_release_xid_data(item->data, XID_IGNORE_EXC | XID_FREE); item->data = NULL; } @@ -446,14 +448,14 @@ _queueitem_clear(_queueitem *item) } static _queueitem * -_queueitem_new(int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) +_queueitem_new(int64_t interpid, _PyXIData_t *data, int unboundop) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, interpid, data, fmt, unboundop); + _queueitem_init(item, interpid, data, unboundop); return item; } @@ -476,10 +478,9 @@ _queueitem_free_all(_queueitem *item) static void _queueitem_popped(_queueitem *item, - _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) + _PyXIData_t **p_data, unboundop_t *p_unboundop) { *p_data = item->data; - *p_fmt = item->fmt; *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; @@ -527,16 +528,16 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; - struct { - int fmt; + struct _queuedefaults { + xidata_fallback_t fallback; int unboundop; } defaults; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) +_queue_init(_queue *queue, Py_ssize_t maxsize, struct _queuedefaults defaults) { - assert(check_unbound(unboundop)); + assert(check_unbound(defaults.unboundop)); PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_QUEUE_ALLOC; @@ -547,10 +548,7 @@ _queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) .items = { .maxsize = maxsize, }, - .defaults = { - .fmt = fmt, - .unboundop = unboundop, - }, + .defaults = defaults, }; return 0; } @@ -631,8 +629,7 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, - int fmt, int unboundop) +_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, int unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -648,7 +645,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(interpid, data, fmt, unboundop); + _queueitem *item = _queueitem_new(interpid, data, unboundop); if (item == NULL) { _queue_unlock(queue); return -1; @@ -668,8 +665,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, } static int -_queue_next(_queue *queue, - _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) +_queue_next(_queue *queue, _PyXIData_t **p_data, int *p_unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -688,7 +684,7 @@ _queue_next(_queue *queue, } queue->items.count -= 1; - _queueitem_popped(item, p_data, p_fmt, p_unboundop); + _queueitem_popped(item, p_data, p_unboundop); _queue_unlock(queue); return 0; @@ -716,8 +712,11 @@ _queue_is_full(_queue *queue, int *p_is_full) return err; } - assert(queue->items.count <= queue->items.maxsize); - *p_is_full = queue->items.count == queue->items.maxsize; + assert(queue->items.maxsize <= 0 + || queue->items.count <= queue->items.maxsize); + *p_is_full = queue->items.maxsize > 0 + ? queue->items.count == queue->items.maxsize + : 0; _queue_unlock(queue); return 0; @@ -1035,8 +1034,7 @@ _queues_decref(_queues *queues, int64_t qid) struct queue_id_and_info { int64_t id; - int fmt; - int unboundop; + struct _queuedefaults defaults; }; static struct queue_id_and_info * @@ -1053,8 +1051,7 @@ _queues_list_all(_queues *queues, int64_t *p_count) for (int64_t i=0; ref != NULL; ref = ref->next, i++) { ids[i].id = ref->qid; assert(ref->queue != NULL); - ids[i].fmt = ref->queue->defaults.fmt; - ids[i].unboundop = ref->queue->defaults.unboundop; + ids[i].defaults = ref->queue->defaults; } *p_count = queues->count; @@ -1090,13 +1087,14 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize, int fmt, int unboundop) +queue_create(_queues *queues, Py_ssize_t maxsize, + struct _queuedefaults defaults) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize, fmt, unboundop); + int err = _queue_init(queue, maxsize, defaults); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; @@ -1125,7 +1123,8 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) +queue_put(_queues *queues, int64_t qid, PyObject *obj, unboundop_t unboundop, + xidata_fallback_t fallback) { PyThreadState *tstate = PyThreadState_Get(); @@ -1138,27 +1137,27 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) assert(queue != NULL); // Convert the object to cross-interpreter data. - _PyXIData_t *data = _PyXIData_New(); - if (data == NULL) { + _PyXIData_t *xidata = _PyXIData_New(); + if (xidata == NULL) { _queue_unmark_waiter(queue, queues->mutex); return -1; } - if (_PyObject_GetXIData(tstate, obj, data) != 0) { + if (_PyObject_GetXIData(tstate, obj, fallback, xidata) != 0) { _queue_unmark_waiter(queue, queues->mutex); - GLOBAL_FREE(data); + GLOBAL_FREE(xidata); return -1; } - assert(_PyXIData_INTERPID(data) == + assert(_PyXIData_INTERPID(xidata) == PyInterpreterState_GetID(tstate->interp)); // Add the data to the queue. int64_t interpid = -1; // _queueitem_init() will set it. - int res = _queue_add(queue, interpid, data, fmt, unboundop); + int res = _queue_add(queue, interpid, xidata, unboundop); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: - (void)_release_xid_data(data, 0); - GLOBAL_FREE(data); + (void)_release_xid_data(xidata, 0); + GLOBAL_FREE(xidata); return res; } @@ -1169,7 +1168,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) // XXX Support a "wait" mutex? static int queue_get(_queues *queues, int64_t qid, - PyObject **res, int *p_fmt, int *p_unboundop) + PyObject **res, int *p_unboundop) { int err; *res = NULL; @@ -1185,7 +1184,7 @@ queue_get(_queues *queues, int64_t qid, // Pop off the next item from the queue. _PyXIData_t *data = NULL; - err = _queue_next(queue, &data, p_fmt, p_unboundop); + err = _queue_next(queue, &data, p_unboundop); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1216,6 +1215,20 @@ queue_get(_queues *queues, int64_t qid, return 0; } +static int +queue_get_defaults(_queues *queues, int64_t qid, + struct _queuedefaults *p_defaults) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + *p_defaults = queue->defaults; + _queue_unmark_waiter(queue, queues->mutex); + return 0; +} + static int queue_get_maxsize(_queues *queues, int64_t qid, Py_ssize_t *p_maxsize) { @@ -1270,7 +1283,7 @@ set_external_queue_type(module_state *state, PyTypeObject *queue_type) } // Add and register the new type. - if (ensure_xid_class(queue_type, _queueobj_shared) < 0) { + if (ensure_xid_class(queue_type, GETDATA(_queueobj_shared)) < 0) { return -1; } state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); @@ -1348,10 +1361,10 @@ _queueobj_from_xid(_PyXIData_t *data) PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } PyTypeObject *cls = get_external_queue_type(mod); @@ -1458,40 +1471,63 @@ clear_interpreter(void *data) } -typedef struct idarg_int64_converter_data qidarg_converter_data; +/*[python input] + +class qidarg_converter(CConverter): + type = 'int64_t' + converter = 'qidarg_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=c64fbf36771164d6]*/ static int qidarg_converter(PyObject *arg, void *ptr) { - qidarg_converter_data *data = ptr; - if (data->label == NULL) { - data->label = "queue ID"; - } - return idarg_int64_converter(arg, ptr); + int64_t *qid_ptr = ptr; + struct idarg_int64_converter_data data = { + .label = "queue ID", + }; + int res = idarg_int64_converter(arg, &data); + *qid_ptr = data.id; + return res; } +#include "clinic/_interpqueuesmodule.c.h" + + +/*[clinic input] +_interpqueues.create + maxsize: Py_ssize_t + unboundop as unboundarg: int = -1 + fallback as fallbackarg: int = -1 + +Create a new cross-interpreter queue and return its unique generated ID. + +It is a new reference as though bind() had been called on the queue. +The caller is responsible for calling destroy() for the new queue +before the runtime is finalized. +[clinic start generated code]*/ static PyObject * -queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_create_impl(PyObject *module, Py_ssize_t maxsize, + int unboundarg, int fallbackarg) +/*[clinic end generated code: output=9a889b93773251eb input=4f79b710a87360e1]*/ { - static char *kwlist[] = {"maxsize", "fmt", "unboundop", NULL}; - Py_ssize_t maxsize; - int fmt; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "nii:create", kwlist, - &maxsize, &fmt, &unboundop)) + struct _queuedefaults defaults = {0}; + if (resolve_unboundop(unboundarg, UNBOUND_REPLACE, + &defaults.unboundop) < 0) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + if (resolve_fallback(fallbackarg, _PyXIDATA_FULL_FALLBACK, + &defaults.fallback) < 0) + { return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize, fmt, unboundop); + int64_t qid = queue_create(&_globals.queues, maxsize, defaults); if (qid < 0) { - (void)handle_queue_error((int)qid, self, qid); + (void)handle_queue_error((int)qid, module, qid); return NULL; } @@ -1499,7 +1535,7 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) if (qidobj == NULL) { PyObject *exc = PyErr_GetRaisedException(); int err = queue_destroy(&_globals.queues, qid); - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { // XXX issue a warning? PyErr_Clear(); } @@ -1510,41 +1546,37 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) return qidobj; } -PyDoc_STRVAR(queuesmod_create_doc, -"create(maxsize, fmt, unboundop) -> qid\n\ -\n\ -Create a new cross-interpreter queue and return its unique generated ID.\n\ -It is a new reference as though bind() had been called on the queue.\n\ -\n\ -The caller is responsible for calling destroy() for the new queue\n\ -before the runtime is finalized."); +/*[clinic input] +_interpqueues.destroy + qid: qidarg + +Clear and destroy the queue. + +Afterward attempts to use the queue will behave as though it never existed. +[clinic start generated code]*/ static PyObject * -queuesmod_destroy(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_destroy_impl(PyObject *module, int64_t qid) +/*[clinic end generated code: output=46b35623f080cbff input=8632bba87f81e3e9]*/ { - static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:destroy", kwlist, - qidarg_converter, &qidarg)) { - return NULL; - } - int64_t qid = qidarg.id; - int err = queue_destroy(&_globals.queues, qid); - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(queuesmod_destroy_doc, -"destroy(qid)\n\ -\n\ -Clear and destroy the queue. Afterward attempts to use the queue\n\ -will behave as though it never existed."); +/*[clinic input] +_interpqueues.list_all + +Return the list of ID triples for all queues. + +Each ID triple consists of (ID, default unbound op, default fallback). +[clinic start generated code]*/ static PyObject * -queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) +_interpqueues_list_all_impl(PyObject *module) +/*[clinic end generated code: output=974280cb6442afdb input=19495f02cbb38b33]*/ { int64_t count = 0; struct queue_id_and_info *qids = _queues_list_all(&_globals.queues, &count); @@ -1560,8 +1592,9 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } struct queue_id_and_info *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *item = Py_BuildValue("Lii", cur->id, cur->fmt, - cur->unboundop); + PyObject *item = Py_BuildValue("Lii", cur->id, + cur->defaults.unboundop, + cur->defaults.fallback); if (item == NULL) { Py_SETREF(ids, NULL); break; @@ -1574,99 +1607,93 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) return ids; } -PyDoc_STRVAR(queuesmod_list_all_doc, -"list_all() -> [(qid, fmt)]\n\ -\n\ -Return the list of IDs for all queues.\n\ -Each corresponding default format is also included."); +/*[clinic input] +_interpqueues.put + qid: qidarg + obj: object + unboundop as unboundarg: int = -1 + fallback as fallbackarg: int = -1 + +Add the object's data to the queue. +[clinic start generated code]*/ static PyObject * -queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_put_impl(PyObject *module, int64_t qid, PyObject *obj, + int unboundarg, int fallbackarg) +/*[clinic end generated code: output=2e0b31c6eaec29c9 input=4906550ab5c73be3]*/ { - static char *kwlist[] = {"qid", "obj", "fmt", "unboundop", NULL}; - qidarg_converter_data qidarg = {0}; - PyObject *obj; - int fmt; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oii:put", kwlist, - qidarg_converter, &qidarg, &obj, &fmt, - &unboundop)) - { + struct _queuedefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = queue_get_defaults(&_globals.queues, qid, &defaults); + if (handle_queue_error(err, module, qid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { return NULL; } - int64_t qid = qidarg.id; - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { return NULL; } /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj, fmt, unboundop); + int err = queue_put(&_globals.queues, qid, obj, unboundop, fallback); // This is the only place that raises QueueFull. - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj, fmt)\n\ -\n\ -Add the object's data to the queue."); +/*[clinic input] +_interpqueues.get + qid: qidarg + +Return the (object, unbound op) from the front of the queue. + +If there is nothing to receive then raise QueueEmpty. +[clinic start generated code]*/ static PyObject * -queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_get_impl(PyObject *module, int64_t qid) +/*[clinic end generated code: output=b0988a0e29194f05 input=c5bccbc409ad0190]*/ { - static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:get", kwlist, - qidarg_converter, &qidarg)) { - return NULL; - } - int64_t qid = qidarg.id; - PyObject *obj = NULL; - int fmt = 0; int unboundop = 0; - int err = queue_get(&_globals.queues, qid, &obj, &fmt, &unboundop); + int err = queue_get(&_globals.queues, qid, &obj, &unboundop); // This is the only place that raises QueueEmpty. - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { return NULL; } if (obj == NULL) { - return Py_BuildValue("Oii", Py_None, fmt, unboundop); + return Py_BuildValue("Oi", Py_None, unboundop); } - PyObject *res = Py_BuildValue("OiO", obj, fmt, Py_None); + PyObject *res = Py_BuildValue("OO", obj, Py_None); Py_DECREF(obj); return res; } -PyDoc_STRVAR(queuesmod_get_doc, -"get(qid) -> (obj, fmt)\n\ -\n\ -Return a new object from the data at the front of the queue.\n\ -The object's format is also returned.\n\ -\n\ -If there is nothing to receive then raise QueueEmpty."); +/*[clinic input] +_interpqueues.bind + qid: qidarg + +Take a reference to the identified queue. + +The queue is not destroyed until there are no references left. +[clinic start generated code]*/ static PyObject * -queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_bind_impl(PyObject *module, int64_t qid) +/*[clinic end generated code: output=02b515e203c3f926 input=b0efd1a6ce0e576e]*/ { - static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:bind", kwlist, - qidarg_converter, &qidarg)) { - return NULL; - } - int64_t qid = qidarg.id; - // XXX Check module state if bound already. int err = _queues_incref(&_globals.queues, qid); - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { return NULL; } @@ -1675,112 +1702,86 @@ queuesmod_bind(PyObject *self, PyObject *args, PyObject *kwds) Py_RETURN_NONE; } -PyDoc_STRVAR(queuesmod_bind_doc, -"bind(qid)\n\ -\n\ -Take a reference to the identified queue.\n\ -The queue is not destroyed until there are no references left."); +/*[clinic input] +_interpqueues.release + qid: qidarg + +Release a reference to the queue. + +The queue is destroyed once there are no references left. +[clinic start generated code]*/ static PyObject * -queuesmod_release(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_release_impl(PyObject *module, int64_t qid) +/*[clinic end generated code: output=a59545d7c61fc6ee input=664125cf0262ff6f]*/ { // Note that only the current interpreter is affected. - static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&:release", kwlist, - qidarg_converter, &qidarg)) { - return NULL; - } - int64_t qid = qidarg.id; // XXX Check module state if bound already. // XXX Update module state. int err = _queues_decref(&_globals.queues, qid); - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(queuesmod_release_doc, -"release(qid)\n\ -\n\ -Release a reference to the queue.\n\ -The queue is destroyed once there are no references left."); +/*[clinic input] +_interpqueues.get_maxsize + qid: qidarg + +Return the maximum number of items in the queue. +[clinic start generated code]*/ static PyObject * -queuesmod_get_maxsize(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_get_maxsize_impl(PyObject *module, int64_t qid) +/*[clinic end generated code: output=074202b9c6dc37bf input=ef55def3496cc379]*/ { - static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&:get_maxsize", kwlist, - qidarg_converter, &qidarg)) { - return NULL; - } - int64_t qid = qidarg.id; - Py_ssize_t maxsize = -1; int err = queue_get_maxsize(&_globals.queues, qid, &maxsize); - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { return NULL; } return PyLong_FromLongLong(maxsize); } -PyDoc_STRVAR(queuesmod_get_maxsize_doc, -"get_maxsize(qid)\n\ -\n\ -Return the maximum number of items in the queue."); +/*[clinic input] +_interpqueues.get_queue_defaults + qid: qidarg + +Return the queue's default values, set when it was created. +[clinic start generated code]*/ static PyObject * -queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_get_queue_defaults_impl(PyObject *module, int64_t qid) +/*[clinic end generated code: output=b1b8b8103834191a input=3102315a7bff77fc]*/ { - static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&:get_queue_defaults", kwlist, - qidarg_converter, &qidarg)) { + struct _queuedefaults defaults = {0}; + int err = queue_get_defaults(&_globals.queues, qid, &defaults); + if (handle_queue_error(err, module, qid)) { return NULL; } - int64_t qid = qidarg.id; - _queue *queue = NULL; - int err = _queues_lookup(&_globals.queues, qid, &queue); - if (handle_queue_error(err, self, qid)) { - return NULL; - } - int fmt = queue->defaults.fmt; - int unboundop = queue->defaults.unboundop; - _queue_unmark_waiter(queue, _globals.queues.mutex); - - PyObject *defaults = Py_BuildValue("ii", fmt, unboundop); - return defaults; + PyObject *res = Py_BuildValue("ii", defaults.unboundop, defaults.fallback); + return res; } -PyDoc_STRVAR(queuesmod_get_queue_defaults_doc, -"get_queue_defaults(qid)\n\ -\n\ -Return the queue's default values, set when it was created."); +/*[clinic input] +_interpqueues.is_full + qid: qidarg + +Return true if the queue has a maxsize and has reached it. +[clinic start generated code]*/ static PyObject * -queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_is_full_impl(PyObject *module, int64_t qid) +/*[clinic end generated code: output=47a6e18477cddfee input=25d86a327ed3a2e7]*/ { - static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&:is_full", kwlist, - qidarg_converter, &qidarg)) { - return NULL; - } - int64_t qid = qidarg.id; - int is_full = 0; int err = queue_is_full(&_globals.queues, qid, &is_full); - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { return NULL; } if (is_full) { @@ -1789,54 +1790,42 @@ queuesmod_is_full(PyObject *self, PyObject *args, PyObject *kwds) Py_RETURN_FALSE; } -PyDoc_STRVAR(queuesmod_is_full_doc, -"is_full(qid)\n\ -\n\ -Return true if the queue has a maxsize and has reached it."); +/*[clinic input] +_interpqueues.get_count + qid: qidarg + +Return the number of items in the queue. +[clinic start generated code]*/ static PyObject * -queuesmod_get_count(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues_get_count_impl(PyObject *module, int64_t qid) +/*[clinic end generated code: output=fb9e66e829cdd964 input=ce47690e7598884b]*/ { - static char *kwlist[] = {"qid", NULL}; - qidarg_converter_data qidarg = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&:get_count", kwlist, - qidarg_converter, &qidarg)) { - return NULL; - } - int64_t qid = qidarg.id; - Py_ssize_t count = -1; int err = queue_get_count(&_globals.queues, qid, &count); - if (handle_queue_error(err, self, qid)) { + if (handle_queue_error(err, module, qid)) { return NULL; } assert(count >= 0); return PyLong_FromSsize_t(count); } -PyDoc_STRVAR(queuesmod_get_count_doc, -"get_count(qid)\n\ -\n\ -Return the number of items in the queue."); +/*[clinic input] +_interpqueues._register_heap_types + queuetype: object(subclass_of='&PyType_Type', type='PyTypeObject *') + emptyerror: object + fullerror: object + +Return the number of items in the queue. +[clinic start generated code]*/ static PyObject * -queuesmod__register_heap_types(PyObject *self, PyObject *args, PyObject *kwds) +_interpqueues__register_heap_types_impl(PyObject *module, + PyTypeObject *queuetype, + PyObject *emptyerror, + PyObject *fullerror) +/*[clinic end generated code: output=f33f6e8b5af905cd input=57d24ae405eda521]*/ { - static char *kwlist[] = {"queuetype", "emptyerror", "fullerror", NULL}; - PyObject *queuetype; - PyObject *emptyerror; - PyObject *fullerror; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OOO:_register_heap_types", kwlist, - &queuetype, &emptyerror, &fullerror)) { - return NULL; - } - if (!PyType_Check(queuetype)) { - PyErr_SetString(PyExc_TypeError, - "expected a type for 'queuetype'"); - return NULL; - } if (!PyExceptionClass_Check(emptyerror)) { PyErr_SetString(PyExc_TypeError, "expected an exception type for 'emptyerror'"); @@ -1848,9 +1837,9 @@ queuesmod__register_heap_types(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - module_state *state = get_module_state(self); + module_state *state = get_module_state(module); - if (set_external_queue_type(state, (PyTypeObject *)queuetype) < 0) { + if (set_external_queue_type(state, queuetype) < 0) { return NULL; } if (set_external_exc_types(state, emptyerror, fullerror) < 0) { @@ -1861,30 +1850,18 @@ queuesmod__register_heap_types(PyObject *self, PyObject *args, PyObject *kwds) } static PyMethodDef module_functions[] = { - {"create", _PyCFunction_CAST(queuesmod_create), - METH_VARARGS | METH_KEYWORDS, queuesmod_create_doc}, - {"destroy", _PyCFunction_CAST(queuesmod_destroy), - METH_VARARGS | METH_KEYWORDS, queuesmod_destroy_doc}, - {"list_all", queuesmod_list_all, - METH_NOARGS, queuesmod_list_all_doc}, - {"put", _PyCFunction_CAST(queuesmod_put), - METH_VARARGS | METH_KEYWORDS, queuesmod_put_doc}, - {"get", _PyCFunction_CAST(queuesmod_get), - METH_VARARGS | METH_KEYWORDS, queuesmod_get_doc}, - {"bind", _PyCFunction_CAST(queuesmod_bind), - METH_VARARGS | METH_KEYWORDS, queuesmod_bind_doc}, - {"release", _PyCFunction_CAST(queuesmod_release), - METH_VARARGS | METH_KEYWORDS, queuesmod_release_doc}, - {"get_maxsize", _PyCFunction_CAST(queuesmod_get_maxsize), - METH_VARARGS | METH_KEYWORDS, queuesmod_get_maxsize_doc}, - {"get_queue_defaults", _PyCFunction_CAST(queuesmod_get_queue_defaults), - METH_VARARGS | METH_KEYWORDS, queuesmod_get_queue_defaults_doc}, - {"is_full", _PyCFunction_CAST(queuesmod_is_full), - METH_VARARGS | METH_KEYWORDS, queuesmod_is_full_doc}, - {"get_count", _PyCFunction_CAST(queuesmod_get_count), - METH_VARARGS | METH_KEYWORDS, queuesmod_get_count_doc}, - {"_register_heap_types", _PyCFunction_CAST(queuesmod__register_heap_types), - METH_VARARGS | METH_KEYWORDS, NULL}, + _INTERPQUEUES_CREATE_METHODDEF + _INTERPQUEUES_DESTROY_METHODDEF + _INTERPQUEUES_LIST_ALL_METHODDEF + _INTERPQUEUES_PUT_METHODDEF + _INTERPQUEUES_GET_METHODDEF + _INTERPQUEUES_BIND_METHODDEF + _INTERPQUEUES_RELEASE_METHODDEF + _INTERPQUEUES_GET_MAXSIZE_METHODDEF + _INTERPQUEUES_GET_QUEUE_DEFAULTS_METHODDEF + _INTERPQUEUES_IS_FULL_METHODDEF + _INTERPQUEUES_GET_COUNT_METHODDEF + _INTERPQUEUES__REGISTER_HEAP_TYPES_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1931,8 +1908,7 @@ static int module_traverse(PyObject *mod, visitproc visit, void *arg) { module_state *state = get_module_state(mod); - (void)traverse_module_state(state, visit, arg); - return 0; + return traverse_module_state(state, visit, arg); } static int @@ -1941,8 +1917,7 @@ module_clear(PyObject *mod) module_state *state = get_module_state(mod); // Now we clear the module state. - (void)clear_module_state(state); - return 0; + return clear_module_state(state); } static void diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index edd65577284..40fd51d752e 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -5,8 +5,10 @@ _RESOLVE_MODINIT_FUNC_NAME(NAME) +#define GETDATA(FUNC) ((_PyXIData_getdata_t){.basic=FUNC}) + static int -ensure_xid_class(PyTypeObject *cls, xidatafunc getdata) +ensure_xid_class(PyTypeObject *cls, _PyXIData_getdata_t getdata) { PyThreadState *tstate = PyThreadState_Get(); return _PyXIData_RegisterClass(tstate, cls, getdata); @@ -37,10 +39,37 @@ _get_interpid(_PyXIData_t *data) } +#ifdef HAS_FALLBACK +static int +resolve_fallback(int arg, xidata_fallback_t dflt, + xidata_fallback_t *p_fallback) +{ + if (arg < 0) { + *p_fallback = dflt; + return 0; + } + xidata_fallback_t fallback; + if (arg == _PyXIDATA_XIDATA_ONLY) { + fallback =_PyXIDATA_XIDATA_ONLY; + } + else if (arg == _PyXIDATA_FULL_FALLBACK) { + fallback = _PyXIDATA_FULL_FALLBACK; + } + else { + PyErr_Format(PyExc_ValueError, "unsupported fallback %d", arg); + return -1; + } + *p_fallback = fallback; + return 0; +} +#endif + + /* unbound items ************************************************************/ #ifdef HAS_UNBOUND_ITEMS +typedef int unboundop_t; #define UNBOUND_REMOVE 1 #define UNBOUND_ERROR 2 #define UNBOUND_REPLACE 3 @@ -51,6 +80,7 @@ _get_interpid(_PyXIData_t *data) // object is released but the underlying data is copied (with the "raw" // allocator) and used when the item is popped off the queue. +#ifndef NDEBUG static int check_unbound(int unboundop) { @@ -63,5 +93,31 @@ check_unbound(int unboundop) return 0; } } +#endif + +static int +resolve_unboundop(int arg, unboundop_t dflt, unboundop_t *p_unboundop) +{ + if (arg < 0) { + *p_unboundop = dflt; + return 0; + } + unboundop_t op; + if (arg == UNBOUND_REMOVE) { + op = UNBOUND_REMOVE; + } + else if (arg == UNBOUND_ERROR) { + op = UNBOUND_ERROR; + } + else if (arg == UNBOUND_REPLACE) { + op = UNBOUND_REPLACE; + } + else { + PyErr_Format(PyExc_ValueError, "unsupported unboundop %d", arg); + return -1; + } + *p_unboundop = op; + return 0; +} #endif diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 77678f7c126..2aee8b07891 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_code.h" // _PyCode_HAS_EXECUTORS() #include "pycore_crossinterp.h" // _PyXIData_t +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_namespace.h" // _PyNamespace_New() @@ -19,12 +20,18 @@ #include "_interpreters_common.h" +#include "clinic/_interpretersmodule.c.h" #define MODULE_NAME _interpreters #define MODULE_NAME_STR Py_STRINGIFY(MODULE_NAME) #define MODINIT_FUNC_NAME RESOLVE_MODINIT_FUNC_NAME(MODULE_NAME) +/*[clinic input] +module _interpreters +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=bfd967980a0de892]*/ + static PyInterpreterState * _get_current_interp(void) { @@ -71,6 +78,22 @@ is_running_main(PyInterpreterState *interp) } +static inline int +is_notshareable_raised(PyThreadState *tstate) +{ + PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); + return _PyErr_ExceptionMatches(tstate, exctype); +} + +static void +unwrap_not_shareable(PyThreadState *tstate, _PyXI_failure *failure) +{ + if (_PyXI_UnwrapNotShareableError(tstate, failure) < 0) { + _PyErr_Clear(tstate); + } +} + + /* Cross-interpreter Buffer Views *******************************************/ /* When a memoryview object is "shared" between interpreters, @@ -286,7 +309,7 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) *p_state = cls; // Register XID for the builtin memoryview type. - if (ensure_xid_class(&PyMemoryView_Type, _pybuffer_shared) < 0) { + if (ensure_xid_class(&PyMemoryView_Type, GETDATA(_pybuffer_shared)) < 0) { return -1; } // We don't ever bother un-registering memoryview. @@ -319,10 +342,10 @@ _get_current_module_state(void) { PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } module_state *state = get_module_state(mod); Py_DECREF(mod); @@ -359,96 +382,6 @@ _get_current_xibufferview_type(void) } -/* Python code **************************************************************/ - -static const char * -check_code_str(PyUnicodeObject *text) -{ - assert(text != NULL); - if (PyUnicode_GET_LENGTH(text) == 0) { - return "too short"; - } - - // XXX Verify that it parses? - - return NULL; -} - -static const char * -check_code_object(PyCodeObject *code) -{ - assert(code != NULL); - if (code->co_argcount > 0 - || code->co_posonlyargcount > 0 - || code->co_kwonlyargcount > 0 - || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) - { - return "arguments not supported"; - } - if (code->co_ncellvars > 0) { - return "closures not supported"; - } - // We trust that no code objects under co_consts have unbound cell vars. - - if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) { - return "only basic functions are supported"; - } - if (code->_co_monitoring != NULL) { - return "only basic functions are supported"; - } - if (code->co_extra != NULL) { - return "only basic functions are supported"; - } - - return NULL; -} - -#define RUN_TEXT 1 -#define RUN_CODE 2 - -static const char * -get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) -{ - const char *codestr = NULL; - Py_ssize_t len = -1; - PyObject *bytes_obj = NULL; - int flags = 0; - - if (PyUnicode_Check(arg)) { - assert(PyUnicode_Check(arg) - && (check_code_str((PyUnicodeObject *)arg) == NULL)); - codestr = PyUnicode_AsUTF8AndSize(arg, &len); - if (codestr == NULL) { - return NULL; - } - if (strlen(codestr) != (size_t)len) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - return NULL; - } - flags = RUN_TEXT; - } - else { - assert(PyCode_Check(arg) - && (check_code_object((PyCodeObject *)arg) == NULL)); - flags = RUN_CODE; - - // Serialize the code object. - bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); - if (bytes_obj == NULL) { - return NULL; - } - codestr = PyBytes_AS_STRING(bytes_obj); - len = PyBytes_GET_SIZE(bytes_obj); - } - - *flags_p = flags; - *bytes_p = bytes_obj; - *len_p = len; - return codestr; -} - - /* interpreter-specific code ************************************************/ static int @@ -511,73 +444,290 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } -static int -_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) +struct interp_call { + _PyXIData_t *func; + _PyXIData_t *args; + _PyXIData_t *kwargs; + struct { + _PyXIData_t func; + _PyXIData_t args; + _PyXIData_t kwargs; + } _preallocated; +}; + +static void +_interp_call_clear(struct interp_call *call) { - PyObject *result = NULL; - if (flags & RUN_TEXT) { - result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + if (call->func != NULL) { + _PyXIData_Clear(NULL, call->func); } - else if (flags & RUN_CODE) { - PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); - if (code != NULL) { - result = PyEval_EvalCode(code, ns, ns); - Py_DECREF(code); + if (call->args != NULL) { + _PyXIData_Clear(NULL, call->args); + } + if (call->kwargs != NULL) { + _PyXIData_Clear(NULL, call->kwargs); + } + *call = (struct interp_call){0}; +} + +static int +_interp_call_pack(PyThreadState *tstate, struct interp_call *call, + PyObject *func, PyObject *args, PyObject *kwargs) +{ + xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; + assert(call->func == NULL); + assert(call->args == NULL); + assert(call->kwargs == NULL); + // Handle the func. + if (!PyCallable_Check(func)) { + _PyErr_Format(tstate, PyExc_TypeError, + "expected a callable, got %R", func); + return -1; + } + if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + _PyErr_SetRaisedException(tstate, exc); + return -1; + } + Py_DECREF(exc); + } + call->func = &call->_preallocated.func; + // Handle the args. + if (args == NULL || args == Py_None) { + // Leave it empty. + } + else { + assert(PyTuple_Check(args)); + if (PyTuple_GET_SIZE(args) > 0) { + if (_PyObject_GetXIData( + tstate, args, fallback, &call->_preallocated.args) < 0) + { + _interp_call_clear(call); + return -1; + } + call->args = &call->_preallocated.args; + } + } + // Handle the kwargs. + if (kwargs == NULL || kwargs == Py_None) { + // Leave it empty. + } + else { + assert(PyDict_Check(kwargs)); + if (PyDict_GET_SIZE(kwargs) > 0) { + if (_PyObject_GetXIData( + tstate, kwargs, fallback, &call->_preallocated.kwargs) < 0) + { + _interp_call_clear(call); + return -1; + } + call->kwargs = &call->_preallocated.kwargs; + } + } + return 0; +} + +static void +wrap_notshareable(PyThreadState *tstate, const char *label) +{ + if (!is_notshareable_raised(tstate)) { + return; + } + assert(label != NULL && strlen(label) > 0); + PyObject *cause = _PyErr_GetRaisedException(tstate); + _PyXIData_FormatNotShareableError(tstate, "%s not shareable", label); + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyException_SetCause(exc, cause); + _PyErr_SetRaisedException(tstate, exc); +} + +static int +_interp_call_unpack(struct interp_call *call, + PyObject **p_func, PyObject **p_args, PyObject **p_kwargs) +{ + PyThreadState *tstate = PyThreadState_Get(); + + // Unpack the func. + PyObject *func = _PyXIData_NewObject(call->func); + if (func == NULL) { + wrap_notshareable(tstate, "func"); + return -1; + } + // Unpack the args. + PyObject *args; + if (call->args == NULL) { + args = PyTuple_New(0); + if (args == NULL) { + Py_DECREF(func); + return -1; } } else { - Py_UNREACHABLE(); + args = _PyXIData_NewObject(call->args); + if (args == NULL) { + wrap_notshareable(tstate, "args"); + Py_DECREF(func); + return -1; + } + assert(PyTuple_Check(args)); } - if (result == NULL) { - return -1; + // Unpack the kwargs. + PyObject *kwargs = NULL; + if (call->kwargs != NULL) { + kwargs = _PyXIData_NewObject(call->kwargs); + if (kwargs == NULL) { + wrap_notshareable(tstate, "kwargs"); + Py_DECREF(func); + Py_DECREF(args); + return -1; + } + assert(PyDict_Check(kwargs)); } - Py_DECREF(result); // We throw away the result. + *p_func = func; + *p_args = args; + *p_kwargs = kwargs; return 0; } static int -_run_in_interpreter(PyInterpreterState *interp, - const char *codestr, Py_ssize_t codestrlen, - PyObject *shareables, int flags, - PyObject **p_excinfo) +_make_call(struct interp_call *call, + PyObject **p_result, _PyXI_failure *failure) { - assert(!PyErr_Occurred()); - _PyXI_session session = {0}; + assert(call != NULL && call->func != NULL); + PyThreadState *tstate = _PyThreadState_GET(); + + // Get the func and args. + PyObject *func = NULL, *args = NULL, *kwargs = NULL; + if (_interp_call_unpack(call, &func, &args, &kwargs) < 0) { + assert(func == NULL); + assert(args == NULL); + assert(kwargs == NULL); + _PyXI_InitFailure(failure, _PyXI_ERR_OTHER, NULL); + unwrap_not_shareable(tstate, failure); + return -1; + } + assert(!_PyErr_Occurred(tstate)); + + // Make the call. + PyObject *resobj = PyObject_Call(func, args, kwargs); + Py_DECREF(func); + Py_XDECREF(args); + Py_XDECREF(kwargs); + if (resobj == NULL) { + return -1; + } + *p_result = resobj; + return 0; +} + +static int +_run_script(_PyXIData_t *script, PyObject *ns, _PyXI_failure *failure) +{ + PyObject *code = _PyXIData_NewObject(script); + if (code == NULL) { + _PyXI_InitFailure(failure, _PyXI_ERR_NOT_SHAREABLE, NULL); + return -1; + } + PyObject *result = PyEval_EvalCode(code, ns, ns); + Py_DECREF(code); + if (result == NULL) { + _PyXI_InitFailure(failure, _PyXI_ERR_UNCAUGHT_EXCEPTION, NULL); + return -1; + } + assert(result == Py_None); + Py_DECREF(result); // We throw away the result. + return 0; +} + +struct run_result { + PyObject *result; + PyObject *excinfo; +}; + +static void +_run_result_clear(struct run_result *runres) +{ + Py_CLEAR(runres->result); + Py_CLEAR(runres->excinfo); +} + +static int +_run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, + _PyXIData_t *script, struct interp_call *call, + PyObject *shareables, struct run_result *runres) +{ + assert(!_PyErr_Occurred(tstate)); + int res = -1; + _PyXI_failure *failure = _PyXI_NewFailure(); + if (failure == NULL) { + return -1; + } + _PyXI_session *session = _PyXI_NewSession(); + if (session == NULL) { + _PyXI_FreeFailure(failure); + return -1; + } + _PyXI_session_result result = {0}; // Prep and switch interpreters. - if (_PyXI_Enter(&session, interp, shareables) < 0) { - if (PyErr_Occurred()) { - // If an error occured at this step, it means that interp - // was not prepared and switched. - return -1; - } - // Now, apply the error from another interpreter: - PyObject *excinfo = _PyXI_ApplyError(session.error); - if (excinfo != NULL) { - *p_excinfo = excinfo; - } - assert(PyErr_Occurred()); + if (_PyXI_Enter(session, interp, shareables, &result) < 0) { + // If an error occurred at this step, it means that interp + // was not prepared and switched. + _PyXI_FreeSession(session); + _PyXI_FreeFailure(failure); + assert(result.excinfo == NULL); return -1; } - // Run the script. - int res = _run_script(session.main_ns, codestr, codestrlen, flags); - - // Clean up and switch back. - _PyXI_Exit(&session); - - // Propagate any exception out to the caller. - assert(!PyErr_Occurred()); - if (res < 0) { - PyObject *excinfo = _PyXI_ApplyCapturedException(&session); - if (excinfo != NULL) { - *p_excinfo = excinfo; + // Run in the interpreter. + if (script != NULL) { + assert(call == NULL); + PyObject *mainns = _PyXI_GetMainNamespace(session, failure); + if (mainns == NULL) { + goto finally; } + res = _run_script(script, mainns, failure); } else { - assert(!_PyXI_HasCapturedException(&session)); + assert(call != NULL); + PyObject *resobj; + res = _make_call(call, &resobj, failure); + if (res == 0) { + res = _PyXI_Preserve(session, "resobj", resobj, failure); + Py_DECREF(resobj); + if (res < 0) { + goto finally; + } + } } +finally: + // Clean up and switch back. + (void)res; + int exitres = _PyXI_Exit(session, failure, &result); + assert(res == 0 || exitres != 0); + _PyXI_FreeSession(session); + _PyXI_FreeFailure(failure); + + res = exitres; + if (_PyErr_Occurred(tstate)) { + // It's a directly propagated exception. + assert(res < 0); + } + else if (res < 0) { + assert(result.excinfo != NULL); + runres->excinfo = Py_NewRef(result.excinfo); + res = -1; + } + else { + assert(result.excinfo == NULL); + runres->result = _PyXI_GetPreserved(&result, "resobj"); + if (_PyErr_Occurred(tstate)) { + res = -1; + } + } + _PyXI_ClearResult(&result); return res; } @@ -653,12 +803,12 @@ get_summary(PyInterpreterState *interp) } +// Not converted to Argument Clinic because the function uses ``**kwargs``. static PyObject * interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) { const char *name = NULL; - if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", - &name)) + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", &name)) { return NULL; } @@ -686,7 +836,8 @@ interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(new_config_doc, -"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +"new_config($module, name='isolated', /, **overrides)\n\ +--\n\ \n\ Return a representation of a new PyInterpreterConfig.\n\ \n\ @@ -697,17 +848,28 @@ Any keyword arguments are set on the corresponding config fields,\n\ overriding the initial values."); -static PyObject * -interp_create(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"config", "reqrefs", NULL}; - PyObject *configobj = NULL; - int reqrefs = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, - &configobj, &reqrefs)) { - return NULL; - } +/*[clinic input] +_interpreters.create + config as configobj: object(py_default="'isolated'") = NULL + * + reqrefs: bool = False +Create a new interpreter and return a unique generated ID. + +The caller is responsible for destroying the interpreter before exiting, +typically by using _interpreters.destroy(). This can be managed +automatically by passing "reqrefs=True" and then using _incref() and +_decref() appropriately. + +"config" must be a valid interpreter config or the name of a +predefined config ('isolated' or 'legacy'). The default +is 'isolated'. +[clinic start generated code]*/ + +static PyObject * +_interpreters_create_impl(PyObject *module, PyObject *configobj, int reqrefs) +/*[clinic end generated code: output=c1cc6835b1277c16 input=235ce396a23624d5]*/ +{ PyInterpreterConfig config; if (config_from_object(configobj, &config) < 0) { return NULL; @@ -741,34 +903,22 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) } -PyDoc_STRVAR(create_doc, -"create([config], *, reqrefs=False) -> ID\n\ -\n\ -Create a new interpreter and return a unique generated ID.\n\ -\n\ -The caller is responsible for destroying the interpreter before exiting,\n\ -typically by using _interpreters.destroy(). This can be managed \n\ -automatically by passing \"reqrefs=True\" and then using _incref() and\n\ -_decref() appropriately.\n\ -\n\ -\"config\" must be a valid interpreter config or the name of a\n\ -predefined config (\"isolated\" or \"legacy\"). The default\n\ -is \"isolated\"."); +/*[clinic input] +_interpreters.destroy + id: object + * + restrict as restricted: bool = False +Destroy the identified interpreter. + +Attempting to destroy the current interpreter raises InterpreterError. +So does an unrecognized ID. +[clinic start generated code]*/ static PyObject * -interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_destroy_impl(PyObject *module, PyObject *id, int restricted) +/*[clinic end generated code: output=0bc20da8700ab4dd input=561bdd6537639d40]*/ { - static char *kwlist[] = {"id", "restrict", NULL}; - PyObject *id; - int restricted = 0; - // XXX Use "L" for id? - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:destroy", kwlist, &id, &restricted)) - { - return NULL; - } - // Look up the interpreter. int reqready = 0; PyInterpreterState *interp = \ @@ -802,27 +952,19 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) Py_RETURN_NONE; } -PyDoc_STRVAR(destroy_doc, -"destroy(id, *, restrict=False)\n\ -\n\ -Destroy the identified interpreter.\n\ -\n\ -Attempting to destroy the current interpreter raises InterpreterError.\n\ -So does an unrecognized ID."); +/*[clinic input] +_interpreters.list_all + * + require_ready as reqready: bool = False + +Return a list containing the ID of every existing interpreter. +[clinic start generated code]*/ static PyObject * -interp_list_all(PyObject *self, PyObject *args, PyObject *kwargs) +_interpreters_list_all_impl(PyObject *module, int reqready) +/*[clinic end generated code: output=3f21c1a7c78043c0 input=35bae91c381a2cf9]*/ { - static char *kwlist[] = {"require_ready", NULL}; - int reqready = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|$p:" MODULE_NAME_STR ".list_all", - kwlist, &reqready)) - { - return NULL; - } - PyObject *ids = PyList_New(0); if (ids == NULL) { return NULL; @@ -851,14 +993,16 @@ interp_list_all(PyObject *self, PyObject *args, PyObject *kwargs) return ids; } -PyDoc_STRVAR(list_all_doc, -"list_all() -> [(ID, whence)]\n\ -\n\ -Return a list containing the ID of every existing interpreter."); +/*[clinic input] +_interpreters.get_current + +Return (ID, whence) of the current interpreter. +[clinic start generated code]*/ static PyObject * -interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) +_interpreters_get_current_impl(PyObject *module) +/*[clinic end generated code: output=03161c8fcc0136eb input=37fb2c067c14d543]*/ { PyInterpreterState *interp =_get_current_interp(); if (interp == NULL) { @@ -868,39 +1012,38 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) return get_summary(interp); } -PyDoc_STRVAR(get_current_doc, -"get_current() -> (ID, whence)\n\ -\n\ -Return the ID of current interpreter."); +/*[clinic input] +_interpreters.get_main + +Return (ID, whence) of the main interpreter. +[clinic start generated code]*/ static PyObject * -interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) +_interpreters_get_main_impl(PyObject *module) +/*[clinic end generated code: output=9647288aff735557 input=b4ace23ca562146f]*/ { PyInterpreterState *interp = _PyInterpreterState_Main(); assert(_PyInterpreterState_IsReady(interp)); return get_summary(interp); } -PyDoc_STRVAR(get_main_doc, -"get_main() -> (ID, whence)\n\ -\n\ -Return the ID of main interpreter."); +/*[clinic input] +_interpreters.set___main___attrs + id: object + updates: object(subclass_of='&PyDict_Type') + * + restrict as restricted: bool = False + +Bind the given attributes in the interpreter's __main__ module. +[clinic start generated code]*/ static PyObject * -interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) +_interpreters_set___main___attrs_impl(PyObject *module, PyObject *id, + PyObject *updates, int restricted) +/*[clinic end generated code: output=f3803010cb452bf0 input=d16ab8d81371f86a]*/ { - static char *kwlist[] = {"id", "updates", "restrict", NULL}; - PyObject *id, *updates; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "OO|$p:" MODULE_NAME_STR ".set___main___attrs", - kwlist, &id, &updates, &restricted)) - { - return NULL; - } - // Look up the interpreter. int reqready = 1; PyInterpreterState *interp = \ @@ -910,164 +1053,86 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) } // Check the updates. - if (updates != Py_None) { - Py_ssize_t size = PyObject_Size(updates); - if (size < 0) { - return NULL; - } - if (size == 0) { - PyErr_SetString(PyExc_ValueError, - "arg 2 must be a non-empty mapping"); - return NULL; - } + Py_ssize_t size = PyDict_Size(updates); + if (size < 0) { + return NULL; + } + if (size == 0) { + PyErr_SetString(PyExc_ValueError, + "arg 2 must be a non-empty dict"); + return NULL; } - _PyXI_session session = {0}; + _PyXI_session *session = _PyXI_NewSession(); + if (session == NULL) { + return NULL; + } // Prep and switch interpreters, including apply the updates. - if (_PyXI_Enter(&session, interp, updates) < 0) { - if (!PyErr_Occurred()) { - _PyXI_ApplyCapturedException(&session); - assert(PyErr_Occurred()); - } - else { - assert(!_PyXI_HasCapturedException(&session)); - } + if (_PyXI_Enter(session, interp, updates, NULL) < 0) { + _PyXI_FreeSession(session); return NULL; } // Clean up and switch back. - _PyXI_Exit(&session); + assert(!PyErr_Occurred()); + int res = _PyXI_Exit(session, NULL, NULL); + _PyXI_FreeSession(session); + assert(res == 0); + if (res < 0) { + // unreachable + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "unresolved error"); + } + return NULL; + } Py_RETURN_NONE; } -PyDoc_STRVAR(set___main___attrs_doc, -"set___main___attrs(id, ns, *, restrict=False)\n\ -\n\ -Bind the given attributes in the interpreter's __main__ module."); - - -static PyUnicodeObject * -convert_script_arg(PyObject *arg, const char *fname, const char *displayname, - const char *expected) -{ - PyUnicodeObject *str = NULL; - if (PyUnicode_CheckExact(arg)) { - str = (PyUnicodeObject *)Py_NewRef(arg); - } - else if (PyUnicode_Check(arg)) { - // XXX str = PyUnicode_FromObject(arg); - str = (PyUnicodeObject *)Py_NewRef(arg); - } - else { - _PyArg_BadArgument(fname, displayname, expected, arg); - return NULL; - } - - const char *err = check_code_str(str); - if (err != NULL) { - Py_DECREF(str); - PyErr_Format(PyExc_ValueError, - "%.200s(): bad script text (%s)", fname, err); - return NULL; - } - - return str; -} - -static PyCodeObject * -convert_code_arg(PyObject *arg, const char *fname, const char *displayname, - const char *expected) -{ - const char *kind = NULL; - PyCodeObject *code = NULL; - if (PyFunction_Check(arg)) { - if (PyFunction_GetClosure(arg) != NULL) { - PyErr_Format(PyExc_ValueError, - "%.200s(): closures not supported", fname); - return NULL; - } - code = (PyCodeObject *)PyFunction_GetCode(arg); - if (code == NULL) { - if (PyErr_Occurred()) { - // This chains. - PyErr_Format(PyExc_ValueError, - "%.200s(): bad func", fname); - } - else { - PyErr_Format(PyExc_ValueError, - "%.200s(): func.__code__ missing", fname); - } - return NULL; - } - Py_INCREF(code); - kind = "func"; - } - else if (PyCode_Check(arg)) { - code = (PyCodeObject *)Py_NewRef(arg); - kind = "code object"; - } - else { - _PyArg_BadArgument(fname, displayname, expected, arg); - return NULL; - } - - const char *err = check_code_object(code); - if (err != NULL) { - Py_DECREF(code); - PyErr_Format(PyExc_ValueError, - "%.200s(): bad %s (%s)", fname, kind, err); - return NULL; - } - - return code; -} - -static int -_interp_exec(PyObject *self, PyInterpreterState *interp, - PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) -{ - if (shared_arg != NULL && !PyDict_CheckExact(shared_arg)) { - PyErr_SetString(PyExc_TypeError, "expected 'shared' to be a dict"); - return -1; - } - - // Extract code. - Py_ssize_t codestrlen = -1; - PyObject *bytes_obj = NULL; - int flags = 0; - const char *codestr = get_code_str(code_arg, - &codestrlen, &bytes_obj, &flags); - if (codestr == NULL) { - return -1; - } - - // Run the code in the interpreter. - int res = _run_in_interpreter(interp, codestr, codestrlen, - shared_arg, flags, p_excinfo); - Py_XDECREF(bytes_obj); - if (res < 0) { - return -1; - } - - return 0; -} static PyObject * -interp_exec(PyObject *self, PyObject *args, PyObject *kwds) +_handle_script_error(struct run_result *runres) { - static char *kwlist[] = {"id", "code", "shared", "restrict", NULL}; - PyObject *id, *code; - PyObject *shared = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O$p:" MODULE_NAME_STR ".exec", kwlist, - &id, &code, &shared, &restricted)) - { + assert(runres->result == NULL); + if (runres->excinfo == NULL) { + assert(PyErr_Occurred()); return NULL; } + assert(!PyErr_Occurred()); + return runres->excinfo; +} +/*[clinic input] +_interpreters.exec + id: object + code: object + shared: object(subclass_of='&PyDict_Type', c_default='NULL') = {} + * + restrict as restricted: bool = False + +Execute the provided code in the identified interpreter. + +This is equivalent to running the builtin exec() under the target +interpreter, using the __dict__ of its __main__ module as both +globals and locals. + +"code" may be a string containing the text of a Python script. + +Functions (and code objects) are also supported, with some restrictions. +The code/function must not take any arguments or be a closure +(i.e. have cell vars). Methods and other callables are not supported. + +If a function is provided, its code object is used and all its state +is ignored, including its __globals__ dict. +[clinic start generated code]*/ + +static PyObject * +_interpreters_exec_impl(PyObject *module, PyObject *id, PyObject *code, + PyObject *shared, int restricted) +/*[clinic end generated code: output=492057c4f10dc304 input=5a22c1ed0c5dbcf3]*/ +{ + PyThreadState *tstate = _PyThreadState_GET(); int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "exec code for"); @@ -1075,60 +1140,46 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - const char *expected = "a string, a function, or a code object"; - if (PyUnicode_Check(code)) { - code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec", - "argument 2", expected); - } - else { - code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec", - "argument 2", expected); - } - if (code == NULL) { + // We don't need the script to be "pure", which means it can use + // global variables. They will be resolved against __main__. + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { + unwrap_not_shareable(tstate, NULL); return NULL; } - PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, code, shared, &excinfo); - Py_DECREF(code); + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); + _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; } -PyDoc_STRVAR(exec_doc, -"exec(id, code, shared=None, *, restrict=False)\n\ -\n\ -Execute the provided code in the identified interpreter.\n\ -This is equivalent to running the builtin exec() under the target\n\ -interpreter, using the __dict__ of its __main__ module as both\n\ -globals and locals.\n\ -\n\ -\"code\" may be a string containing the text of a Python script.\n\ -\n\ -Functions (and code objects) are also supported, with some restrictions.\n\ -The code/function must not take any arguments or be a closure\n\ -(i.e. have cell vars). Methods and other callables are not supported.\n\ -\n\ -If a function is provided, its code object is used and all its state\n\ -is ignored, including its __globals__ dict."); +/*[clinic input] +_interpreters.run_string + id: object + script: unicode + shared: object(subclass_of='&PyDict_Type', c_default='NULL') = {} + * + restrict as restricted: bool = False + +Execute the provided string in the identified interpreter. + +(See _interpreters.exec().) +[clinic start generated code]*/ static PyObject * -interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_run_string_impl(PyObject *module, PyObject *id, + PyObject *script, PyObject *shared, + int restricted) +/*[clinic end generated code: output=a30a64fb9ad396a2 input=51ce549b9a8dbe21]*/ { - static char *kwlist[] = {"id", "script", "shared", "restrict", NULL}; - PyObject *id, *script; - PyObject *shared = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O$p:" MODULE_NAME_STR ".run_string", - kwlist, &id, &script, &shared, &restricted)) - { - return NULL; - } - +#define FUNCNAME MODULE_NAME_STR ".run_string" + PyThreadState *tstate = _PyThreadState_GET(); int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "run a string in"); @@ -1136,43 +1187,52 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".run_string", - "argument 2", "a string"); - if (script == NULL) { + if (PyFunction_Check(script) || PyCode_Check(script)) { + _PyArg_BadArgument(FUNCNAME, "argument 2", "a string", script); return NULL; } - PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, script, shared, &excinfo); - Py_DECREF(script); - if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) { + unwrap_not_shareable(tstate, NULL); + return NULL; } + + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); + _PyXIData_Release(&xidata); + if (res < 0) { + return _handle_script_error(&runres); + } + assert(runres.result == NULL); Py_RETURN_NONE; +#undef FUNCNAME } -PyDoc_STRVAR(run_string_doc, -"run_string(id, script, shared=None, *, restrict=False)\n\ -\n\ -Execute the provided string in the identified interpreter.\n\ -\n\ -(See " MODULE_NAME_STR ".exec()."); +/*[clinic input] +_interpreters.run_func + id: object + func: object + shared: object(subclass_of='&PyDict_Type', c_default='NULL') = {} + * + restrict as restricted: bool = False + +Execute the body of the provided function in the identified interpreter. + +Code objects are also supported. In both cases, closures and args +are not supported. Methods and other callables are not supported either. + +(See _interpreters.exec().) +[clinic start generated code]*/ static PyObject * -interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func, + PyObject *shared, int restricted) +/*[clinic end generated code: output=131f7202ca4a0c5e input=2d62bb9b9eaf4948]*/ { - static char *kwlist[] = {"id", "func", "shared", "restrict", NULL}; - PyObject *id, *func; - PyObject *shared = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O$p:" MODULE_NAME_STR ".run_func", - kwlist, &id, &func, &shared, &restricted)) - { - return NULL; - } - +#define FUNCNAME MODULE_NAME_STR ".run_func" + PyThreadState *tstate = _PyThreadState_GET(); int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "run a function in"); @@ -1180,49 +1240,60 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec", - "argument 2", - "a function or a code object"); - if (code == NULL) { + // We don't worry about checking globals. They will be resolved + // against __main__. + PyObject *code; + if (PyFunction_Check(func)) { + code = PyFunction_GET_CODE(func); + } + else if (PyCode_Check(func)) { + code = func; + } + else { + _PyArg_BadArgument(FUNCNAME, "argument 2", "a function", func); return NULL; } - PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); - Py_DECREF(code); - if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { + unwrap_not_shareable(tstate, NULL); + return NULL; } + + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); + _PyXIData_Release(&xidata); + if (res < 0) { + return _handle_script_error(&runres); + } + assert(runres.result == NULL); Py_RETURN_NONE; +#undef FUNCNAME } -PyDoc_STRVAR(run_func_doc, -"run_func(id, func, shared=None, *, restrict=False)\n\ -\n\ -Execute the body of the provided function in the identified interpreter.\n\ -Code objects are also supported. In both cases, closures and args\n\ -are not supported. Methods and other callables are not supported either.\n\ -\n\ -(See " MODULE_NAME_STR ".exec()."); +/*[clinic input] +_interpreters.call + id: object + callable: object + args: object(subclass_of='&PyTuple_Type', c_default='NULL') = () + kwargs: object(subclass_of='&PyDict_Type', c_default='NULL') = {} + * + preserve_exc: bool = False + restrict as restricted: bool = False + +Call the provided object in the identified interpreter. + +Pass the given args and kwargs, if possible. +[clinic start generated code]*/ static PyObject * -interp_call(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_call_impl(PyObject *module, PyObject *id, PyObject *callable, + PyObject *args, PyObject *kwargs, int preserve_exc, + int restricted) +/*[clinic end generated code: output=b7a4a27d72df3ebc input=b026d0b212a575e6]*/ { - static char *kwlist[] = {"id", "callable", "args", "kwargs", - "restrict", NULL}; - PyObject *id, *callable; - PyObject *args_obj = NULL; - PyObject *kwargs_obj = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|OO$p:" MODULE_NAME_STR ".call", kwlist, - &id, &callable, &args_obj, &kwargs_obj, - &restricted)) - { - return NULL; - } - + PyThreadState *tstate = _PyThreadState_GET(); int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "make a call in"); @@ -1230,54 +1301,44 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (args_obj != NULL) { - PyErr_SetString(PyExc_ValueError, "got unexpected args"); - return NULL; - } - if (kwargs_obj != NULL) { - PyErr_SetString(PyExc_ValueError, "got unexpected kwargs"); + struct interp_call call = {0}; + if (_interp_call_pack(tstate, &call, callable, args, kwargs) < 0) { return NULL; } - PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call", - "argument 2", "a function"); - if (code == NULL) { - return NULL; + PyObject *res_and_exc = NULL; + struct run_result runres = {0}; + if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) { + if (runres.excinfo == NULL) { + assert(_PyErr_Occurred(tstate)); + goto finally; + } + assert(!_PyErr_Occurred(tstate)); } + assert(runres.result == NULL || runres.excinfo == NULL); + res_and_exc = Py_BuildValue("OO", + (runres.result ? runres.result : Py_None), + (runres.excinfo ? runres.excinfo : Py_None)); - PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, code, NULL, &excinfo); - Py_DECREF(code); - if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; - } - Py_RETURN_NONE; +finally: + _interp_call_clear(&call); + _run_result_clear(&runres); + return res_and_exc; } -PyDoc_STRVAR(call_doc, -"call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ -\n\ -Call the provided object in the identified interpreter.\n\ -Pass the given args and kwargs, if possible.\n\ -\n\ -\"callable\" may be a plain function with no free vars that takes\n\ -no arguments.\n\ -\n\ -The function's code object is used and all its state\n\ -is ignored, including its __globals__ dict."); +/*[clinic input] +@permit_long_summary +_interpreters.is_shareable + obj: object + +Return True if the object's data may be shared between interpreters and False otherwise. +[clinic start generated code]*/ static PyObject * -object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_is_shareable_impl(PyObject *module, PyObject *obj) +/*[clinic end generated code: output=227856926a22940b input=95f888d35a6d4bb3]*/ { - static char *kwlist[] = {"obj", NULL}; - PyObject *obj; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:is_shareable", kwlist, &obj)) { - return NULL; - } - PyThreadState *tstate = _PyThreadState_GET(); if (_PyObject_CheckXIData(tstate, obj) == 0) { Py_RETURN_TRUE; @@ -1286,26 +1347,20 @@ object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) Py_RETURN_FALSE; } -PyDoc_STRVAR(is_shareable_doc, -"is_shareable(obj) -> bool\n\ -\n\ -Return True if the object's data may be shared between interpreters and\n\ -False otherwise."); +/*[clinic input] +_interpreters.is_running + id: object + * + restrict as restricted: bool = False + +Return whether or not the identified interpreter is running. +[clinic start generated code]*/ static PyObject * -interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_is_running_impl(PyObject *module, PyObject *id, int restricted) +/*[clinic end generated code: output=32a6225d5ded9bdb input=3291578d04231125]*/ { - static char *kwlist[] = {"id", "restrict", NULL}; - PyObject *id; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:is_running", kwlist, - &id, &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "check if running for"); @@ -1319,28 +1374,27 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) Py_RETURN_FALSE; } -PyDoc_STRVAR(is_running_doc, -"is_running(id, *, restrict=False) -> bool\n\ -\n\ -Return whether or not the identified interpreter is running."); +/*[clinic input] +_interpreters.get_config + id: object + * + restrict as restricted: bool = False + +Return a representation of the config used to initialize the interpreter. +[clinic start generated code]*/ static PyObject * -interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_get_config_impl(PyObject *module, PyObject *id, int restricted) +/*[clinic end generated code: output=56773353b9b7224a input=59519a01c22d96d1]*/ { - static char *kwlist[] = {"id", "restrict", NULL}; - PyObject *idobj = NULL; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O?|$p:get_config", kwlist, - &idobj, &restricted)) - { - return NULL; + if (id == Py_None) { + id = NULL; } int reqready = 0; PyInterpreterState *interp = \ - resolve_interp(idobj, restricted, reqready, "get the config of"); + resolve_interp(id, restricted, reqready, "get the config of"); if (interp == NULL) { return NULL; } @@ -1359,23 +1413,18 @@ interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) return configobj; } -PyDoc_STRVAR(get_config_doc, -"get_config(id, *, restrict=False) -> types.SimpleNamespace\n\ -\n\ -Return a representation of the config used to initialize the interpreter."); +/*[clinic input] +_interpreters.whence + id: object + +Return an identifier for where the interpreter was created. +[clinic start generated code]*/ static PyObject * -interp_whence(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_whence_impl(PyObject *module, PyObject *id) +/*[clinic end generated code: output=ef2c21ab106c2c20 input=eeede0a2fbfa2968]*/ { - static char *kwlist[] = {"id", NULL}; - PyObject *id; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:whence", kwlist, &id)) - { - return NULL; - } - PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; @@ -1385,26 +1434,21 @@ interp_whence(PyObject *self, PyObject *args, PyObject *kwds) return PyLong_FromLong(whence); } -PyDoc_STRVAR(whence_doc, -"whence(id) -> int\n\ -\n\ -Return an identifier for where the interpreter was created."); +/*[clinic input] +_interpreters.incref + id: object + * + implieslink: bool = False + restrict as restricted: bool = False + +[clinic start generated code]*/ static PyObject * -interp_incref(PyObject *self, PyObject *args, PyObject *kwds) +_interpreters_incref_impl(PyObject *module, PyObject *id, int implieslink, + int restricted) +/*[clinic end generated code: output=eccaa4e03fbe8ee2 input=a0a614748f2e348c]*/ { - static char *kwlist[] = {"id", "implieslink", "restrict", NULL}; - PyObject *id; - int implieslink = 0; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$pp:incref", kwlist, - &id, &implieslink, &restricted)) - { - return NULL; - } - int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "incref"); @@ -1422,18 +1466,18 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) } -static PyObject * -interp_decref(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "restrict", NULL}; - PyObject *id; - int restricted = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:decref", kwlist, &id, &restricted)) - { - return NULL; - } +/*[clinic input] +_interpreters.decref + id: object + * + restrict as restricted: bool = False +[clinic start generated code]*/ + +static PyObject * +_interpreters_decref_impl(PyObject *module, PyObject *id, int restricted) +/*[clinic end generated code: output=5c54db4b22086171 input=c4aa34f09c44e62a]*/ +{ int reqready = 1; PyInterpreterState *interp = \ resolve_interp(id, restricted, reqready, "decref"); @@ -1447,20 +1491,23 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) } -static PyObject * -capture_exception(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"exc", NULL}; - PyObject *exc_arg = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|O?:capture_exception", kwlist, - &exc_arg)) - { - return NULL; - } +/*[clinic input] +@permit_long_docstring_body +_interpreters.capture_exception + exc as exc_arg: object = None +Return a snapshot of an exception. + +If "exc" is None then the current exception, if any, is used (but not cleared). +The returned snapshot is the same as what _interpreters.exec() returns. +[clinic start generated code]*/ + +static PyObject * +_interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg) +/*[clinic end generated code: output=ef3f5393ef9c88a6 input=6c4dcb78fb722217]*/ +{ PyObject *exc = exc_arg; - if (exc == NULL) { + if (exc == NULL || exc == Py_None) { exc = PyErr_GetRaisedException(); if (exc == NULL) { Py_RETURN_NONE; @@ -1472,16 +1519,16 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *captured = NULL; - _PyXI_excinfo info = {0}; - if (_PyXI_InitExcInfo(&info, exc) < 0) { + _PyXI_excinfo *info = _PyXI_NewExcInfo(exc); + if (info == NULL) { goto finally; } - captured = _PyXI_ExcInfoAsObject(&info); + captured = _PyXI_ExcInfoAsObject(info); if (captured == NULL) { goto finally; } - PyObject *formatted = _PyXI_FormatExcInfo(&info); + PyObject *formatted = _PyXI_FormatExcInfo(info); if (formatted == NULL) { Py_CLEAR(captured); goto finally; @@ -1494,7 +1541,7 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) } finally: - _PyXI_ClearExcInfo(&info); + _PyXI_FreeExcInfo(info); if (exc != exc_arg) { if (PyErr_Occurred()) { PyErr_SetRaisedException(exc); @@ -1506,58 +1553,33 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) return captured; } -PyDoc_STRVAR(capture_exception_doc, -"capture_exception(exc=None) -> types.SimpleNamespace\n\ -\n\ -Return a snapshot of an exception. If \"exc\" is None\n\ -then the current exception, if any, is used (but not cleared).\n\ -\n\ -The returned snapshot is the same as what _interpreters.exec() returns."); - static PyMethodDef module_functions[] = { {"new_config", _PyCFunction_CAST(interp_new_config), METH_VARARGS | METH_KEYWORDS, new_config_doc}, - {"create", _PyCFunction_CAST(interp_create), - METH_VARARGS | METH_KEYWORDS, create_doc}, - {"destroy", _PyCFunction_CAST(interp_destroy), - METH_VARARGS | METH_KEYWORDS, destroy_doc}, - {"list_all", _PyCFunction_CAST(interp_list_all), - METH_VARARGS | METH_KEYWORDS, list_all_doc}, - {"get_current", interp_get_current, - METH_NOARGS, get_current_doc}, - {"get_main", interp_get_main, - METH_NOARGS, get_main_doc}, + _INTERPRETERS_CREATE_METHODDEF + _INTERPRETERS_DESTROY_METHODDEF + _INTERPRETERS_LIST_ALL_METHODDEF + _INTERPRETERS_GET_CURRENT_METHODDEF + _INTERPRETERS_GET_MAIN_METHODDEF - {"is_running", _PyCFunction_CAST(interp_is_running), - METH_VARARGS | METH_KEYWORDS, is_running_doc}, - {"get_config", _PyCFunction_CAST(interp_get_config), - METH_VARARGS | METH_KEYWORDS, get_config_doc}, - {"whence", _PyCFunction_CAST(interp_whence), - METH_VARARGS | METH_KEYWORDS, whence_doc}, - {"exec", _PyCFunction_CAST(interp_exec), - METH_VARARGS | METH_KEYWORDS, exec_doc}, - {"call", _PyCFunction_CAST(interp_call), - METH_VARARGS | METH_KEYWORDS, call_doc}, - {"run_string", _PyCFunction_CAST(interp_run_string), - METH_VARARGS | METH_KEYWORDS, run_string_doc}, - {"run_func", _PyCFunction_CAST(interp_run_func), - METH_VARARGS | METH_KEYWORDS, run_func_doc}, + _INTERPRETERS_IS_RUNNING_METHODDEF + _INTERPRETERS_GET_CONFIG_METHODDEF + _INTERPRETERS_WHENCE_METHODDEF + _INTERPRETERS_EXEC_METHODDEF + _INTERPRETERS_CALL_METHODDEF + _INTERPRETERS_RUN_STRING_METHODDEF + _INTERPRETERS_RUN_FUNC_METHODDEF - {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), - METH_VARARGS | METH_KEYWORDS, set___main___attrs_doc}, + _INTERPRETERS_SET___MAIN___ATTRS_METHODDEF - {"incref", _PyCFunction_CAST(interp_incref), - METH_VARARGS | METH_KEYWORDS, NULL}, - {"decref", _PyCFunction_CAST(interp_decref), - METH_VARARGS | METH_KEYWORDS, NULL}, + _INTERPRETERS_INCREF_METHODDEF + _INTERPRETERS_DECREF_METHODDEF - {"is_shareable", _PyCFunction_CAST(object_is_shareable), - METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, + _INTERPRETERS_IS_SHAREABLE_METHODDEF - {"capture_exception", _PyCFunction_CAST(capture_exception), - METH_VARARGS | METH_KEYWORDS, capture_exception_doc}, + _INTERPRETERS_CAPTURE_EXCEPTION_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1623,8 +1645,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) { module_state *state = get_module_state(mod); assert(state != NULL); - (void)traverse_module_state(state, visit, arg); - return 0; + return traverse_module_state(state, visit, arg); } static int @@ -1632,8 +1653,7 @@ module_clear(PyObject *mod) { module_state *state = get_module_state(mod); assert(state != NULL); - (void)clear_module_state(state); - return 0; + return clear_module_state(state); } static void diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 50fe5d50c91..433d68d515c 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -70,6 +70,7 @@ PyDoc_STRVAR(module_doc, /*[clinic input] module _io +@permit_long_docstring_body _io.open file: object mode: str = "r" @@ -199,7 +200,7 @@ static PyObject * _io_open_impl(PyObject *module, PyObject *file, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd, PyObject *opener) -/*[clinic end generated code: output=aefafc4ce2b46dc0 input=28027fdaabb8d744]*/ +/*[clinic end generated code: output=aefafc4ce2b46dc0 input=8629579a442a99e3]*/ { size_t i; @@ -498,6 +499,7 @@ _io_text_encoding_impl(PyObject *module, PyObject *encoding, int stacklevel) /*[clinic input] +@permit_long_docstring_body _io.open_code path : unicode @@ -511,7 +513,7 @@ with calling open(path, 'rb'). static PyObject * _io_open_code_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=f5c18e23f4b2ed9f]*/ +/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=53d38a37d780d034]*/ { return PyFile_OpenCodeObject(path); } @@ -679,40 +681,40 @@ iomodule_exec(PyObject *m) } // Base classes - ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); - ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); - ADD_TYPE(m, state->PyIOBase_Type, &iobase_spec, NULL); + ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &_Py_nldecoder_spec, NULL); + ADD_TYPE(m, state->PyBytesIOBuffer_Type, &_Py_bytesiobuf_spec, NULL); + ADD_TYPE(m, state->PyIOBase_Type, &_Py_iobase_spec, NULL); // PyIOBase_Type subclasses - ADD_TYPE(m, state->PyTextIOBase_Type, &textiobase_spec, + ADD_TYPE(m, state->PyTextIOBase_Type, &_Py_textiobase_spec, state->PyIOBase_Type); - ADD_TYPE(m, state->PyBufferedIOBase_Type, &bufferediobase_spec, + ADD_TYPE(m, state->PyBufferedIOBase_Type, &_Py_bufferediobase_spec, state->PyIOBase_Type); - ADD_TYPE(m, state->PyRawIOBase_Type, &rawiobase_spec, + ADD_TYPE(m, state->PyRawIOBase_Type, &_Py_rawiobase_spec, state->PyIOBase_Type); // PyBufferedIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyBytesIO_Type, &bytesio_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedWriter_Type, &bufferedwriter_spec, + ADD_TYPE(m, state->PyBytesIO_Type, &_Py_bytesio_spec, state->PyBufferedIOBase_Type); + ADD_TYPE(m, state->PyBufferedWriter_Type, &_Py_bufferedwriter_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedReader_Type, &bufferedreader_spec, + ADD_TYPE(m, state->PyBufferedReader_Type, &_Py_bufferedreader_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedRWPair_Type, &bufferedrwpair_spec, + ADD_TYPE(m, state->PyBufferedRWPair_Type, &_Py_bufferedrwpair_spec, state->PyBufferedIOBase_Type); - ADD_TYPE(m, state->PyBufferedRandom_Type, &bufferedrandom_spec, + ADD_TYPE(m, state->PyBufferedRandom_Type, &_Py_bufferedrandom_spec, state->PyBufferedIOBase_Type); // PyRawIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyFileIO_Type, &fileio_spec, state->PyRawIOBase_Type); + ADD_TYPE(m, state->PyFileIO_Type, &_Py_fileio_spec, state->PyRawIOBase_Type); #ifdef HAVE_WINDOWS_CONSOLE_IO - ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &winconsoleio_spec, + ADD_TYPE(m, state->PyWindowsConsoleIO_Type, &_Py_winconsoleio_spec, state->PyRawIOBase_Type); #endif // PyTextIOBase_Type(PyIOBase_Type) subclasses - ADD_TYPE(m, state->PyStringIO_Type, &stringio_spec, state->PyTextIOBase_Type); - ADD_TYPE(m, state->PyTextIOWrapper_Type, &textiowrapper_spec, + ADD_TYPE(m, state->PyStringIO_Type, &_Py_stringio_spec, state->PyTextIOBase_Type); + ADD_TYPE(m, state->PyTextIOWrapper_Type, &_Py_textiowrapper_spec, state->PyTextIOBase_Type); #undef ADD_TYPE diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 18cf20edf26..4ae487c8e2a 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -9,23 +9,23 @@ #include "structmember.h" /* Type specs */ -extern PyType_Spec bufferediobase_spec; -extern PyType_Spec bufferedrandom_spec; -extern PyType_Spec bufferedreader_spec; -extern PyType_Spec bufferedrwpair_spec; -extern PyType_Spec bufferedwriter_spec; -extern PyType_Spec bytesio_spec; -extern PyType_Spec bytesiobuf_spec; -extern PyType_Spec fileio_spec; -extern PyType_Spec iobase_spec; -extern PyType_Spec nldecoder_spec; -extern PyType_Spec rawiobase_spec; -extern PyType_Spec stringio_spec; -extern PyType_Spec textiobase_spec; -extern PyType_Spec textiowrapper_spec; +extern PyType_Spec _Py_bufferediobase_spec; +extern PyType_Spec _Py_bufferedrandom_spec; +extern PyType_Spec _Py_bufferedreader_spec; +extern PyType_Spec _Py_bufferedrwpair_spec; +extern PyType_Spec _Py_bufferedwriter_spec; +extern PyType_Spec _Py_bytesio_spec; +extern PyType_Spec _Py_bytesiobuf_spec; +extern PyType_Spec _Py_fileio_spec; +extern PyType_Spec _Py_iobase_spec; +extern PyType_Spec _Py_nldecoder_spec; +extern PyType_Spec _Py_rawiobase_spec; +extern PyType_Spec _Py_stringio_spec; +extern PyType_Spec _Py_textiobase_spec; +extern PyType_Spec _Py_textiowrapper_spec; #ifdef HAVE_WINDOWS_CONSOLE_IO -extern PyType_Spec winconsoleio_spec; +extern PyType_Spec _Py_winconsoleio_spec; #endif /* These functions are used as METH_NOARGS methods, are normally called diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 4724e97982f..4602f2b42a6 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -13,6 +13,7 @@ #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -171,6 +172,7 @@ _io__BufferedIOBase_read_impl(PyObject *self, PyTypeObject *cls, } /*[clinic input] +@permit_long_summary _io._BufferedIOBase.read1 cls: defining_class @@ -186,7 +188,7 @@ A short result does not imply that EOF is imminent. static PyObject * _io__BufferedIOBase_read1_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(size)) -/*[clinic end generated code: output=2e7fc62972487eaa input=af76380e020fd9e6]*/ +/*[clinic end generated code: output=2e7fc62972487eaa input=1e76df255063afd6]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return bufferediobase_unsupported(state, "read1"); @@ -360,16 +362,24 @@ _enter_buffered_busy(buffered *self) } #define IS_CLOSED(self) \ - (!self->buffer || \ + (!self->buffer ? 1 : \ (self->fast_closed_checks \ ? _PyFileIO_closed(self->raw) \ : buffered_closed(self))) #define CHECK_CLOSED(self, error_msg) \ - if (IS_CLOSED(self) && (Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t) == 0)) { \ - PyErr_SetString(PyExc_ValueError, error_msg); \ - return NULL; \ - } \ + do { \ + int _closed = IS_CLOSED(self); \ + if (_closed < 0) { \ + return NULL; \ + } \ + if (_closed && \ + (Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t) == 0)) \ + { \ + PyErr_SetString(PyExc_ValueError, error_msg); \ + return NULL; \ + } \ + } while (0); #define VALID_READ_BUFFER(self) \ (self->readable && self->read_end != -1) @@ -421,8 +431,7 @@ buffered_dealloc(PyObject *op) return; _PyObject_GC_UNTRACK(self); self->ok = 0; - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); if (self->buffer) { PyMem_Free(self->buffer); self->buffer = NULL; @@ -552,8 +561,8 @@ _io__Buffered_close_impl(buffered *self) if (!ENTER_BUFFERED(self)) { return NULL; } - - r = buffered_closed(self); + /* gh-138720: Use IS_CLOSED to match flush CHECK_CLOSED. */ + r = IS_CLOSED(self); if (r < 0) goto end; if (r > 0) { @@ -1025,9 +1034,6 @@ static PyObject * _io__Buffered_read1_impl(buffered *self, Py_ssize_t n) /*[clinic end generated code: output=bcc4fb4e54d103a3 input=3d0ad241aa52b36c]*/ { - Py_ssize_t have, r; - PyObject *res = NULL; - CHECK_INITIALIZED(self) if (n < 0) { n = self->buffer_size; @@ -1035,48 +1041,53 @@ _io__Buffered_read1_impl(buffered *self, Py_ssize_t n) CHECK_CLOSED(self, "read of closed file") - if (n == 0) - return PyBytes_FromStringAndSize(NULL, 0); + if (n == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } /* Return up to n bytes. If at least one byte is buffered, we only return buffered bytes. Otherwise, we do one raw read. */ - have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + Py_ssize_t have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (have > 0) { n = Py_MIN(have, n); - res = _bufferedreader_read_fast(self, n); + PyObject *res = _bufferedreader_read_fast(self, n); assert(res != Py_None); return res; } - res = PyBytes_FromStringAndSize(NULL, n); - if (res == NULL) - return NULL; + if (!ENTER_BUFFERED(self)) { - Py_DECREF(res); return NULL; } + /* Flush the write buffer if necessary */ if (self->writable) { - PyObject *r = buffered_flush_and_rewind_unlocked(self); - if (r == NULL) { + PyObject *res = buffered_flush_and_rewind_unlocked(self); + if (res == NULL) { LEAVE_BUFFERED(self) - Py_DECREF(res); return NULL; } - Py_DECREF(r); + Py_DECREF(res); } _bufferedreader_reset_buf(self); - r = _bufferedreader_raw_read(self, PyBytes_AS_STRING(res), n); - LEAVE_BUFFERED(self) - if (r == -1) { - Py_DECREF(res); + + PyBytesWriter *writer = PyBytesWriter_Create(n); + if (writer == NULL) { return NULL; } - if (r == -2) + + Py_ssize_t r = _bufferedreader_raw_read(self, + PyBytesWriter_GetData(writer), n); + LEAVE_BUFFERED(self) + if (r == -1) { + PyBytesWriter_Discard(writer); + return NULL; + } + if (r == -2) { r = 0; - if (n > r) - _PyBytes_Resize(&res, r); - return res; + } + + return PyBytesWriter_FinishWithSize(writer, r); } static PyObject * @@ -1786,18 +1797,18 @@ _bufferedreader_read_fast(buffered *self, Py_ssize_t n) static PyObject * _bufferedreader_read_generic(buffered *self, Py_ssize_t n) { - PyObject *res = NULL; Py_ssize_t current_size, remaining, written; - char *out; current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (n <= current_size) return _bufferedreader_read_fast(self, n); - res = PyBytes_FromStringAndSize(NULL, n); - if (res == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(n); + if (writer == NULL) { goto error; - out = PyBytes_AS_STRING(res); + } + char *out = PyBytesWriter_GetData(writer); + remaining = n; written = 0; if (current_size > 0) { @@ -1826,11 +1837,9 @@ _bufferedreader_read_generic(buffered *self, Py_ssize_t n) if (r == 0 || r == -2) { /* EOF occurred or read() would block. */ if (r == 0 || written > 0) { - if (_PyBytes_Resize(&res, written)) - goto error; - return res; + return PyBytesWriter_FinishWithSize(writer, written); } - Py_DECREF(res); + PyBytesWriter_Discard(writer); Py_RETURN_NONE; } remaining -= r; @@ -1850,11 +1859,9 @@ _bufferedreader_read_generic(buffered *self, Py_ssize_t n) if (r == 0 || r == -2) { /* EOF occurred or read() would block. */ if (r == 0 || written > 0) { - if (_PyBytes_Resize(&res, written)) - goto error; - return res; + return PyBytesWriter_FinishWithSize(writer, written); } - Py_DECREF(res); + PyBytesWriter_Discard(writer); Py_RETURN_NONE; } if (remaining > r) { @@ -1873,10 +1880,10 @@ _bufferedreader_read_generic(buffered *self, Py_ssize_t n) break; } - return res; + return PyBytesWriter_Finish(writer); error: - Py_XDECREF(res); + PyBytesWriter_Discard(writer); return NULL; } @@ -2080,6 +2087,7 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) PyObject *res = NULL; Py_ssize_t written, avail, remaining; Py_off_t offset; + int r; CHECK_INITIALIZED(self) @@ -2088,7 +2096,11 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) /* Issue #31976: Check for closed file after acquiring the lock. Another thread could be holding the lock while closing the file. */ - if (IS_CLOSED(self)) { + r = IS_CLOSED(self); + if (r < 0) { + goto error; + } + if (r > 0) { PyErr_SetString(PyExc_ValueError, "write to closed file"); goto error; } @@ -2312,8 +2324,7 @@ bufferedrwpair_dealloc(PyObject *op) rwpair *self = rwpair_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)bufferedrwpair_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -2526,7 +2537,7 @@ static PyType_Slot bufferediobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec bufferediobase_spec = { +PyType_Spec _Py_bufferediobase_spec = { .name = "_io._BufferedIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), @@ -2589,7 +2600,7 @@ static PyType_Slot bufferedreader_slots[] = { {0, NULL}, }; -PyType_Spec bufferedreader_spec = { +PyType_Spec _Py_bufferedreader_spec = { .name = "_io.BufferedReader", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2647,7 +2658,7 @@ static PyType_Slot bufferedwriter_slots[] = { {0, NULL}, }; -PyType_Spec bufferedwriter_spec = { +PyType_Spec _Py_bufferedwriter_spec = { .name = "_io.BufferedWriter", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2697,7 +2708,7 @@ static PyType_Slot bufferedrwpair_slots[] = { {0, NULL}, }; -PyType_Spec bufferedrwpair_spec = { +PyType_Spec _Py_bufferedrwpair_spec = { .name = "_io.BufferedRWPair", .basicsize = sizeof(rwpair), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -2765,7 +2776,7 @@ static PyType_Slot bufferedrandom_slots[] = { {0, NULL}, }; -PyType_Spec bufferedrandom_spec = { +PyType_Spec _Py_bufferedrandom_spec = { .name = "_io.BufferedRandom", .basicsize = sizeof(buffered), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index e45a2d1a16d..96611823ab6 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,8 +1,11 @@ #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_object.h" -#include "pycore_sysmodule.h" // _PySys_GetSizeOf() +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_sysmodule.h" // _PySys_GetSizeOf() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() -#include <stddef.h> // offsetof() +#include <stddef.h> // offsetof() #include "_iomodule.h" /*[clinic input] @@ -50,7 +53,7 @@ check_closed(bytesio *self) static int check_exports(bytesio *self) { - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, "Existing exports of data: object cannot be re-sized"); return 1; @@ -68,15 +71,17 @@ check_exports(bytesio *self) return NULL; \ } -#define SHARED_BUF(self) (Py_REFCNT((self)->buf) > 1) +#define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf)) /* Internal routine to get a line from the buffer of a BytesIO object. Returns the length between the current position to the next newline character. */ static Py_ssize_t -scan_eol(bytesio *self, Py_ssize_t len) +scan_eol_lock_held(bytesio *self, Py_ssize_t len) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + const char *start, *n; Py_ssize_t maxlen; @@ -109,11 +114,13 @@ scan_eol(bytesio *self, Py_ssize_t len) The caller should ensure that the 'size' argument is non-negative and not lesser than self->string_size. Returns 0 on success, -1 otherwise. */ static int -unshare_buffer(bytesio *self, size_t size) +unshare_buffer_lock_held(bytesio *self, size_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + PyObject *new_buf; assert(SHARED_BUF(self)); - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); assert(size >= (size_t)self->string_size); new_buf = PyBytes_FromStringAndSize(NULL, size); if (new_buf == NULL) @@ -128,10 +135,12 @@ unshare_buffer(bytesio *self, size_t size) The caller should ensure that the 'size' argument is non-negative. Returns 0 on success, -1 otherwise. */ static int -resize_buffer(bytesio *self, size_t size) +resize_buffer_lock_held(bytesio *self, size_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + assert(self->buf != NULL); - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ @@ -160,7 +169,7 @@ resize_buffer(bytesio *self, size_t size) } if (SHARED_BUF(self)) { - if (unshare_buffer(self, alloc) < 0) + if (unshare_buffer_lock_held(self, alloc) < 0) return -1; } else { @@ -181,8 +190,10 @@ resize_buffer(bytesio *self, size_t size) Inlining is disabled because it's significantly decreases performance of writelines() in PGO build. */ Py_NO_INLINE static Py_ssize_t -write_bytes(bytesio *self, PyObject *b) +write_bytes_lock_held(bytesio *self, PyObject *b) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + if (check_closed(self)) { return -1; } @@ -202,13 +213,13 @@ write_bytes(bytesio *self, PyObject *b) assert(self->pos >= 0); size_t endpos = (size_t)self->pos + len; if (endpos > (size_t)PyBytes_GET_SIZE(self->buf)) { - if (resize_buffer(self, endpos) < 0) { + if (resize_buffer_lock_held(self, endpos) < 0) { len = -1; goto done; } } else if (SHARED_BUF(self)) { - if (unshare_buffer(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { + if (unshare_buffer_lock_held(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { len = -1; goto done; } @@ -245,16 +256,21 @@ write_bytes(bytesio *self, PyObject *b) static PyObject * bytesio_get_closed(PyObject *op, void *Py_UNUSED(closure)) { + PyObject *ret; bytesio *self = bytesio_CAST(op); + Py_BEGIN_CRITICAL_SECTION(self); if (self->buf == NULL) { - Py_RETURN_TRUE; + ret = Py_True; } else { - Py_RETURN_FALSE; + ret = Py_False; } + Py_END_CRITICAL_SECTION(); + return ret; } /*[clinic input] +@critical_section _io.BytesIO.readable Returns True if the IO object can be read. @@ -262,13 +278,14 @@ Returns True if the IO object can be read. static PyObject * _io_BytesIO_readable_impl(bytesio *self) -/*[clinic end generated code: output=4e93822ad5b62263 input=96c5d0cccfb29f5c]*/ +/*[clinic end generated code: output=4e93822ad5b62263 input=ab7816facef48bfd]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.writable Returns True if the IO object can be written. @@ -276,13 +293,14 @@ Returns True if the IO object can be written. static PyObject * _io_BytesIO_writable_impl(bytesio *self) -/*[clinic end generated code: output=64ff6a254b1150b8 input=700eed808277560a]*/ +/*[clinic end generated code: output=64ff6a254b1150b8 input=4f35d49d26dab024]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.seekable Returns True if the IO object can be seeked. @@ -290,13 +308,14 @@ Returns True if the IO object can be seeked. static PyObject * _io_BytesIO_seekable_impl(bytesio *self) -/*[clinic end generated code: output=6b417f46dcc09b56 input=9421f65627a344dd]*/ +/*[clinic end generated code: output=6b417f46dcc09b56 input=9cc78d15aa1deaa3]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.flush Does nothing. @@ -304,13 +323,14 @@ Does nothing. static PyObject * _io_BytesIO_flush_impl(bytesio *self) -/*[clinic end generated code: output=187e3d781ca134a0 input=561ea490be4581a7]*/ +/*[clinic end generated code: output=187e3d781ca134a0 input=c60842743910b381]*/ { CHECK_CLOSED(self); Py_RETURN_NONE; } /*[clinic input] +@critical_section _io.BytesIO.getbuffer cls: defining_class @@ -321,7 +341,7 @@ Get a read-write view over the contents of the BytesIO object. static PyObject * _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) -/*[clinic end generated code: output=045091d7ce87fe4e input=0668fbb48f95dffa]*/ +/*[clinic end generated code: output=045091d7ce87fe4e input=8295764061be77fd]*/ { _PyIO_State *state = get_io_state_by_cls(cls); PyTypeObject *type = state->PyBytesIOBuffer_Type; @@ -340,6 +360,7 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _io.BytesIO.getvalue Retrieve the entire contents of the BytesIO object. @@ -347,16 +368,16 @@ Retrieve the entire contents of the BytesIO object. static PyObject * _io_BytesIO_getvalue_impl(bytesio *self) -/*[clinic end generated code: output=b3f6a3233c8fd628 input=4b403ac0af3973ed]*/ +/*[clinic end generated code: output=b3f6a3233c8fd628 input=c91bff398df0c352]*/ { CHECK_CLOSED(self); - if (self->string_size <= 1 || self->exports > 0) + if (self->string_size <= 1 || FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) return PyBytes_FromStringAndSize(PyBytes_AS_STRING(self->buf), self->string_size); if (self->string_size != PyBytes_GET_SIZE(self->buf)) { if (SHARED_BUF(self)) { - if (unshare_buffer(self, self->string_size) < 0) + if (unshare_buffer_lock_held(self, self->string_size) < 0) return NULL; } else { @@ -368,6 +389,7 @@ _io_BytesIO_getvalue_impl(bytesio *self) } /*[clinic input] +@critical_section _io.BytesIO.isatty Always returns False. @@ -377,13 +399,14 @@ BytesIO objects are not connected to a TTY-like device. static PyObject * _io_BytesIO_isatty_impl(bytesio *self) -/*[clinic end generated code: output=df67712e669f6c8f input=6f97f0985d13f827]*/ +/*[clinic end generated code: output=df67712e669f6c8f input=50487b74dc5ae8a9]*/ { CHECK_CLOSED(self); Py_RETURN_FALSE; } /*[clinic input] +@critical_section _io.BytesIO.tell Current file position, an integer. @@ -391,32 +414,42 @@ Current file position, an integer. static PyObject * _io_BytesIO_tell_impl(bytesio *self) -/*[clinic end generated code: output=b54b0f93cd0e5e1d input=b106adf099cb3657]*/ +/*[clinic end generated code: output=b54b0f93cd0e5e1d input=2c7b0e8f82e05c4d]*/ { CHECK_CLOSED(self); return PyLong_FromSsize_t(self->pos); } static PyObject * -read_bytes(bytesio *self, Py_ssize_t size) +read_bytes_lock_held(bytesio *self, Py_ssize_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + const char *output; assert(self->buf != NULL); assert(size <= self->string_size); if (size > 1 && self->pos == 0 && size == PyBytes_GET_SIZE(self->buf) && - self->exports == 0) { + FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { self->pos += size; return Py_NewRef(self->buf); } + /* gh-141311: Avoid undefined behavior when self->pos (limit PY_SSIZE_T_MAX) + is beyond the size of self->buf. Assert above validates size is always in + bounds. When self->pos is out of bounds calling code sets size to 0. */ + if (size == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + output = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += size; return PyBytes_FromStringAndSize(output, size); } /*[clinic input] +@critical_section _io.BytesIO.read size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -429,7 +462,7 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9cc025f21c75bdd2 input=74344a39f431c3d7]*/ +/*[clinic end generated code: output=9cc025f21c75bdd2 input=9e2f7ff3075fdd39]*/ { Py_ssize_t n; @@ -443,11 +476,12 @@ _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size) size = 0; } - return read_bytes(self, size); + return read_bytes_lock_held(self, size); } /*[clinic input] +@critical_section _io.BytesIO.read1 size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -460,12 +494,13 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=d0f843285aa95f1c input=440a395bf9129ef5]*/ +/*[clinic end generated code: output=d0f843285aa95f1c input=a08fc9e507ab380c]*/ { return _io_BytesIO_read_impl(self, size); } /*[clinic input] +@critical_section _io.BytesIO.readline size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -479,18 +514,19 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_readline_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=4bff3c251df8ffcd input=e7c3fbd1744e2783]*/ +/*[clinic end generated code: output=4bff3c251df8ffcd input=db09d47e23cf2c9e]*/ { Py_ssize_t n; CHECK_CLOSED(self); - n = scan_eol(self, size); + n = scan_eol_lock_held(self, size); - return read_bytes(self, n); + return read_bytes_lock_held(self, n); } /*[clinic input] +@critical_section _io.BytesIO.readlines size as arg: object = None / @@ -504,7 +540,7 @@ total number of bytes in the lines returned. static PyObject * _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) -/*[clinic end generated code: output=09b8e34c880808ff input=691aa1314f2c2a87]*/ +/*[clinic end generated code: output=09b8e34c880808ff input=5c57d7d78e409985]*/ { Py_ssize_t maxsize, size, n; PyObject *result, *line; @@ -533,7 +569,7 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) return NULL; output = PyBytes_AS_STRING(self->buf) + self->pos; - while ((n = scan_eol(self, -1)) != 0) { + while ((n = scan_eol_lock_held(self, -1)) != 0) { self->pos += n; line = PyBytes_FromStringAndSize(output, n); if (!line) @@ -556,6 +592,7 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) } /*[clinic input] +@critical_section _io.BytesIO.readinto buffer: Py_buffer(accept={rwbuffer}) / @@ -568,7 +605,7 @@ is set not to block and has no data to read. static PyObject * _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) -/*[clinic end generated code: output=a5d407217dcf0639 input=1424d0fdce857919]*/ +/*[clinic end generated code: output=a5d407217dcf0639 input=093a8d330de3fcd1]*/ { Py_ssize_t len, n; @@ -579,21 +616,25 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) n = self->string_size - self->pos; if (len > n) { len = n; - if (len < 0) - len = 0; + if (len < 0) { + /* gh-141311: Avoid undefined behavior when self->pos (limit + PY_SSIZE_T_MAX) points beyond the size of self->buf. */ + return PyLong_FromSsize_t(0); + } } - memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); - assert(self->pos + len < PY_SSIZE_T_MAX); + assert(self->pos + len <= PY_SSIZE_T_MAX); assert(len >= 0); + memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); self->pos += len; return PyLong_FromSsize_t(len); } /*[clinic input] +@critical_section _io.BytesIO.truncate - size: Py_ssize_t(accept={int, NoneType}, c_default="((bytesio *)self)->pos") = None + size: object = None / Truncate the file to at most size bytes. @@ -603,44 +644,68 @@ The current file position is unchanged. Returns the new size. [clinic start generated code]*/ static PyObject * -_io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9ad17650c15fa09b input=dae4295e11c1bbb4]*/ +_io_BytesIO_truncate_impl(bytesio *self, PyObject *size) +/*[clinic end generated code: output=ab42491b4824f384 input=b4acb5f80481c053]*/ { CHECK_CLOSED(self); CHECK_EXPORTS(self); - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "negative size value %zd", size); - return NULL; + Py_ssize_t new_size; + + if (size == Py_None) { + new_size = self->pos; + } + else { + new_size = PyLong_AsLong(size); + if (new_size == -1 && PyErr_Occurred()) { + return NULL; + } + if (new_size < 0) { + PyErr_Format(PyExc_ValueError, + "negative size value %zd", new_size); + return NULL; + } } - if (size < self->string_size) { - self->string_size = size; - if (resize_buffer(self, size) < 0) + if (new_size < self->string_size) { + self->string_size = new_size; + if (resize_buffer_lock_held(self, new_size) < 0) return NULL; } - return PyLong_FromSsize_t(size); + return PyLong_FromSsize_t(new_size); } static PyObject * -bytesio_iternext(PyObject *op) +bytesio_iternext_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + Py_ssize_t n; bytesio *self = bytesio_CAST(op); CHECK_CLOSED(self); - n = scan_eol(self, -1); + n = scan_eol_lock_held(self, -1); if (n == 0) return NULL; - return read_bytes(self, n); + return read_bytes_lock_held(self, n); +} + +static PyObject * +bytesio_iternext(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; } /*[clinic input] +@critical_section _io.BytesIO.seek pos: Py_ssize_t whence: int = 0 @@ -657,7 +722,7 @@ Returns the new absolute position. static PyObject * _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=c26204a68e9190e4 input=1e875e6ebc652948]*/ +/*[clinic end generated code: output=c26204a68e9190e4 input=20f05ddf659255df]*/ { CHECK_CLOSED(self); @@ -700,6 +765,7 @@ _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence) } /*[clinic input] +@critical_section _io.BytesIO.write b: object / @@ -711,13 +777,14 @@ Return the number of bytes written. static PyObject * _io_BytesIO_write_impl(bytesio *self, PyObject *b) -/*[clinic end generated code: output=d3e46bcec8d9e21c input=f5ec7c8c64ed720a]*/ +/*[clinic end generated code: output=d3e46bcec8d9e21c input=46c0c17eac7474a4]*/ { - Py_ssize_t n = write_bytes(self, b); + Py_ssize_t n = write_bytes_lock_held(self, b); return n >= 0 ? PyLong_FromSsize_t(n) : NULL; } /*[clinic input] +@critical_section _io.BytesIO.writelines lines: object / @@ -731,7 +798,7 @@ each element. static PyObject * _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) -/*[clinic end generated code: output=03a43a75773bc397 input=e972539176fc8fc1]*/ +/*[clinic end generated code: output=03a43a75773bc397 input=5d6a616ae39dc9ca]*/ { PyObject *it, *item; @@ -742,7 +809,7 @@ _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) return NULL; while ((item = PyIter_Next(it)) != NULL) { - Py_ssize_t ret = write_bytes(self, item); + Py_ssize_t ret = write_bytes_lock_held(self, item); Py_DECREF(item); if (ret < 0) { Py_DECREF(it); @@ -759,6 +826,7 @@ _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) } /*[clinic input] +@critical_section _io.BytesIO.close Disable all I/O operations. @@ -766,7 +834,7 @@ Disable all I/O operations. static PyObject * _io_BytesIO_close_impl(bytesio *self) -/*[clinic end generated code: output=1471bb9411af84a0 input=37e1f55556e61f60]*/ +/*[clinic end generated code: output=1471bb9411af84a0 input=34ce76d8bd17a23b]*/ { CHECK_EXPORTS(self); Py_CLEAR(self->buf); @@ -788,35 +856,49 @@ _io_BytesIO_close_impl(bytesio *self) function to use the efficient instance representation of PEP 307. */ -static PyObject * -bytesio_getstate(PyObject *op, PyObject *Py_UNUSED(dummy)) -{ - bytesio *self = bytesio_CAST(op); - PyObject *initvalue = _io_BytesIO_getvalue_impl(self); - PyObject *dict; - PyObject *state; + static PyObject * + bytesio_getstate_lock_held(PyObject *op) + { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - if (initvalue == NULL) - return NULL; - if (self->dict == NULL) { - dict = Py_NewRef(Py_None); - } - else { - dict = PyDict_Copy(self->dict); - if (dict == NULL) { - Py_DECREF(initvalue); - return NULL; - } - } + bytesio *self = bytesio_CAST(op); + PyObject *initvalue = _io_BytesIO_getvalue_impl(self); + PyObject *dict; + PyObject *state; - state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); - Py_DECREF(initvalue); - return state; + if (initvalue == NULL) + return NULL; + if (self->dict == NULL) { + dict = Py_NewRef(Py_None); + } + else { + dict = PyDict_Copy(self->dict); + if (dict == NULL) { + Py_DECREF(initvalue); + return NULL; + } + } + + state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); + Py_DECREF(initvalue); + return state; } static PyObject * -bytesio_setstate(PyObject *op, PyObject *state) +bytesio_getstate(PyObject *op, PyObject *Py_UNUSED(dummy)) { + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_getstate_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + +static PyObject * +bytesio_setstate_lock_held(PyObject *op, PyObject *state) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + PyObject *result; PyObject *position_obj; PyObject *dict; @@ -890,21 +972,30 @@ bytesio_setstate(PyObject *op, PyObject *state) Py_RETURN_NONE; } +static PyObject * +bytesio_setstate(PyObject *op, PyObject *state) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_setstate_lock_held(op, state); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void bytesio_dealloc(PyObject *op) { bytesio *self = bytesio_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_SystemError, "deallocated BytesIO object has exported buffers"); PyErr_Print(); } Py_CLEAR(self->buf); Py_CLEAR(self->dict); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } @@ -932,6 +1023,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } /*[clinic input] +@critical_section _io.BytesIO.__init__ initial_bytes as initvalue: object(c_default="NULL") = b'' @@ -940,13 +1032,13 @@ Buffered I/O implementation using an in-memory bytes buffer. static int _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) -/*[clinic end generated code: output=65c0c51e24c5b621 input=aac7f31b67bf0fb6]*/ +/*[clinic end generated code: output=65c0c51e24c5b621 input=3da5a74ee4c4f1ac]*/ { /* In case, __init__ is called multiple times. */ self->string_size = 0; self->pos = 0; - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, "Existing exports of data: object cannot be re-sized"); return -1; @@ -970,8 +1062,10 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) } static PyObject * -bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) +bytesio_sizeof_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + bytesio *self = bytesio_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buf && !SHARED_BUF(self)) { @@ -984,6 +1078,16 @@ bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) return PyLong_FromSize_t(res); } +static PyObject * +bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_sizeof_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int bytesio_traverse(PyObject *op, visitproc visit, void *arg) { @@ -999,7 +1103,7 @@ bytesio_clear(PyObject *op) { bytesio *self = bytesio_CAST(op); Py_CLEAR(self->dict); - if (self->exports == 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { Py_CLEAR(self->buf); } return 0; @@ -1062,7 +1166,7 @@ static PyType_Slot bytesio_slots[] = { {0, NULL}, }; -PyType_Spec bytesio_spec = { +PyType_Spec _Py_bytesio_spec = { .name = "_io.BytesIO", .basicsize = sizeof(bytesio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -1077,18 +1181,15 @@ PyType_Spec bytesio_spec = { */ static int -bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) +bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags) { bytesiobuf *obj = bytesiobuf_CAST(op); bytesio *b = bytesio_CAST(obj->source); - if (view == NULL) { - PyErr_SetString(PyExc_BufferError, - "bytesiobuf_getbuffer: view==NULL argument is obsolete"); - return -1; - } - if (b->exports == 0 && SHARED_BUF(b)) { - if (unshare_buffer(b, b->string_size) < 0) + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(b); + + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0 && SHARED_BUF(b)) { + if (unshare_buffer_lock_held(b, b->string_size) < 0) return -1; } @@ -1096,16 +1197,32 @@ bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) (void)PyBuffer_FillInfo(view, op, PyBytes_AS_STRING(b->buf), b->string_size, 0, flags); - b->exports++; + FT_ATOMIC_ADD_SSIZE(b->exports, 1); return 0; } +static int +bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) +{ + if (view == NULL) { + PyErr_SetString(PyExc_BufferError, + "bytesiobuf_getbuffer: view==NULL argument is obsolete"); + return -1; + } + + int ret; + Py_BEGIN_CRITICAL_SECTION(bytesiobuf_CAST(op)->source); + ret = bytesiobuf_getbuffer_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void bytesiobuf_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) { bytesiobuf *obj = bytesiobuf_CAST(op); bytesio *b = bytesio_CAST(obj->source); - b->exports--; + FT_ATOMIC_ADD_SSIZE(b->exports, -1); } static int @@ -1139,7 +1256,7 @@ static PyType_Slot bytesiobuf_slots[] = { {0, NULL}, }; -PyType_Spec bytesiobuf_spec = { +PyType_Spec _Py_bytesiobuf_spec = { .name = "_io._BytesIOBuffer", .basicsize = sizeof(bytesiobuf), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index aaf4884d173..6595dc937bb 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_io_BytesIO_readable__doc__, @@ -24,7 +25,13 @@ _io_BytesIO_readable_impl(bytesio *self); static PyObject * _io_BytesIO_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_readable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_readable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_writable__doc__, @@ -42,7 +49,13 @@ _io_BytesIO_writable_impl(bytesio *self); static PyObject * _io_BytesIO_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_writable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_writable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_seekable__doc__, @@ -60,7 +73,13 @@ _io_BytesIO_seekable_impl(bytesio *self); static PyObject * _io_BytesIO_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_seekable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_seekable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_flush__doc__, @@ -78,7 +97,13 @@ _io_BytesIO_flush_impl(bytesio *self); static PyObject * _io_BytesIO_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_flush_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_flush_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_getbuffer__doc__, @@ -96,11 +121,18 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * _io_BytesIO_getbuffer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); - return NULL; + goto exit; } - return _io_BytesIO_getbuffer_impl((bytesio *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_getbuffer_impl((bytesio *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_io_BytesIO_getvalue__doc__, @@ -118,7 +150,13 @@ _io_BytesIO_getvalue_impl(bytesio *self); static PyObject * _io_BytesIO_getvalue(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_getvalue_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_getvalue_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_isatty__doc__, @@ -138,7 +176,13 @@ _io_BytesIO_isatty_impl(bytesio *self); static PyObject * _io_BytesIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_isatty_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_isatty_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_tell__doc__, @@ -156,7 +200,13 @@ _io_BytesIO_tell_impl(bytesio *self); static PyObject * _io_BytesIO_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_tell_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_tell_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_read__doc__, @@ -190,7 +240,9 @@ _io_BytesIO_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_read_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -227,7 +279,9 @@ _io_BytesIO_read1(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_read1_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -265,7 +319,9 @@ _io_BytesIO_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readline_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -301,7 +357,9 @@ _io_BytesIO_readlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } arg = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readlines_impl((bytesio *)self, arg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -332,7 +390,9 @@ _io_BytesIO_readinto(PyObject *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readinto_impl((bytesio *)self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -356,13 +416,13 @@ PyDoc_STRVAR(_io_BytesIO_truncate__doc__, {"truncate", _PyCFunction_CAST(_io_BytesIO_truncate), METH_FASTCALL, _io_BytesIO_truncate__doc__}, static PyObject * -_io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size); +_io_BytesIO_truncate_impl(bytesio *self, PyObject *size); static PyObject * _io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t size = ((bytesio *)self)->pos; + PyObject *size = Py_None; if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { goto exit; @@ -370,11 +430,11 @@ _io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (!_Py_convert_optional_to_ssize_t(args[0], &size)) { - goto exit; - } + size = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_truncate_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -428,7 +488,9 @@ _io_BytesIO_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_seek_impl((bytesio *)self, pos, whence); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -453,7 +515,9 @@ _io_BytesIO_write(PyObject *self, PyObject *b) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_write_impl((bytesio *)self, b); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -479,7 +543,9 @@ _io_BytesIO_writelines(PyObject *self, PyObject *lines) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_writelines_impl((bytesio *)self, lines); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -499,7 +565,13 @@ _io_BytesIO_close_impl(bytesio *self); static PyObject * _io_BytesIO_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_close_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_close_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO___init____doc__, @@ -558,9 +630,11 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) } initvalue = fastargs[0]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO___init___impl((bytesio *)self, initvalue); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=6dbfd82f4e9d4ef3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=daa81dfdae5ccc57 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 04870b1c890..96c31ce8d6f 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -277,15 +277,19 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "data is available (EAGAIN is returned before bytes are read) returns None."); #define _IO_FILEIO_READALL_METHODDEF \ - {"readall", (PyCFunction)_io_FileIO_readall, METH_NOARGS, _io_FileIO_readall__doc__}, + {"readall", _PyCFunction_CAST(_io_FileIO_readall), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_readall__doc__}, static PyObject * -_io_FileIO_readall_impl(fileio *self); +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_readall(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_FileIO_readall_impl((fileio *)self); + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "readall() takes no arguments"); + return NULL; + } + return _io_FileIO_readall_impl((fileio *)self, cls); } PyDoc_STRVAR(_io_FileIO_read__doc__, @@ -543,4 +547,4 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1902fac9e39358aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e48f3df2f189170 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index 8e8cd8df9ab..83165e5f7ad 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -149,13 +149,13 @@ PyDoc_STRVAR(_io_StringIO_truncate__doc__, {"truncate", _PyCFunction_CAST(_io_StringIO_truncate), METH_FASTCALL, _io_StringIO_truncate__doc__}, static PyObject * -_io_StringIO_truncate_impl(stringio *self, Py_ssize_t size); +_io_StringIO_truncate_impl(stringio *self, PyObject *pos); static PyObject * _io_StringIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t size = ((stringio *)self)->pos; + PyObject *pos = Py_None; if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { goto exit; @@ -163,12 +163,10 @@ _io_StringIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (!_Py_convert_optional_to_ssize_t(args[0], &size)) { - goto exit; - } + pos = args[0]; skip_optional: Py_BEGIN_CRITICAL_SECTION(self); - return_value = _io_StringIO_truncate_impl((stringio *)self, size); + return_value = _io_StringIO_truncate_impl((stringio *)self, pos); Py_END_CRITICAL_SECTION(); exit: @@ -552,4 +550,4 @@ _io_StringIO_newlines_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=5bfaaab7f41ee6b5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bccc25ef8e6ce9ef input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8fcb27049d6..5d7741fdd83 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -4,6 +4,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> // bool #ifdef HAVE_UNISTD_H @@ -69,6 +70,7 @@ typedef struct { unsigned int writable : 1; unsigned int appending : 1; signed int seekable : 2; /* -1 means unknown */ + unsigned int truncate : 1; unsigned int closefd : 1; char finalizing; /* Stat result which was grabbed at file open, useful for optimizing common @@ -208,6 +210,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->writable = 0; self->appending = 0; self->seekable = -1; + self->truncate = 0; self->stat_atopen = NULL; self->closefd = 1; self->weakreflist = NULL; @@ -340,6 +343,7 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, goto bad_mode; rwa = 1; self->writable = 1; + self->truncate = 1; flags |= O_CREAT | O_TRUNC; break; case 'a': @@ -570,9 +574,7 @@ fileio_dealloc(PyObject *op) PyMem_Free(self->stat_atopen); self->stat_atopen = NULL; } - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)fileio_clear(op); PyTypeObject *tp = Py_TYPE(op); @@ -723,8 +725,12 @@ new_buffersize(fileio *self, size_t currentsize) } /*[clinic input] +@permit_long_docstring_body _io.FileIO.readall + cls: defining_class + / + Read all data from the file, returned as bytes. Reads until either there is an error or read() returns size 0 (indicates EOF). @@ -735,11 +741,11 @@ data is available (EAGAIN is returned before bytes are read) returns None. [clinic start generated code]*/ static PyObject * -_io_FileIO_readall_impl(fileio *self) -/*[clinic end generated code: output=faa0292b213b4022 input=1e19849857f5d0a1]*/ +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) +/*[clinic end generated code: output=d546737ec895c462 input=cecda40bf9961299]*/ { Py_off_t pos, end; - PyObject *result; + PyBytesWriter *writer; Py_ssize_t bytes_read = 0; Py_ssize_t n; size_t bufsize; @@ -747,6 +753,10 @@ _io_FileIO_readall_impl(fileio *self) if (self->fd < 0) { return err_closed(); } + if (!self->readable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "reading"); + } if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) { end = (Py_off_t)self->stat_atopen->st_size; @@ -794,10 +804,10 @@ _io_FileIO_readall_impl(fileio *self) } } - - result = PyBytes_FromStringAndSize(NULL, bufsize); - if (result == NULL) + writer = PyBytesWriter_Create(bufsize); + if (writer == NULL) { return NULL; + } while (1) { if (bytes_read >= (Py_ssize_t)bufsize) { @@ -806,18 +816,18 @@ _io_FileIO_readall_impl(fileio *self) PyErr_SetString(PyExc_OverflowError, "unbounded read returned more bytes " "than a Python bytes object can hold"); - Py_DECREF(result); + PyBytesWriter_Discard(writer); return NULL; } - if (PyBytes_GET_SIZE(result) < (Py_ssize_t)bufsize) { - if (_PyBytes_Resize(&result, bufsize) < 0) + if (PyBytesWriter_GetSize(writer) < (Py_ssize_t)bufsize) { + if (PyBytesWriter_Resize(writer, bufsize) < 0) return NULL; } } n = _Py_read(self->fd, - PyBytes_AS_STRING(result) + bytes_read, + (char*)PyBytesWriter_GetData(writer) + bytes_read, bufsize - bytes_read); if (n == 0) @@ -827,23 +837,20 @@ _io_FileIO_readall_impl(fileio *self) PyErr_Clear(); if (bytes_read > 0) break; - Py_DECREF(result); + PyBytesWriter_Discard(writer); Py_RETURN_NONE; } - Py_DECREF(result); + PyBytesWriter_Discard(writer); return NULL; } bytes_read += n; } - if (PyBytes_GET_SIZE(result) > bytes_read) { - if (_PyBytes_Resize(&result, bytes_read) < 0) - return NULL; - } - return result; + return PyBytesWriter_FinishWithSize(writer, bytes_read); } /*[clinic input] +@permit_long_docstring_body _io.FileIO.read cls: defining_class size: Py_ssize_t(accept={int, NoneType}) = -1 @@ -863,12 +870,8 @@ bytes object at EOF. static PyObject * _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) -/*[clinic end generated code: output=bbd749c7c224143e input=cf21fddef7d38ab6]*/ +/*[clinic end generated code: output=bbd749c7c224143e input=752d1ad3db8564a5]*/ { - char *ptr; - Py_ssize_t n; - PyObject *bytes; - if (self->fd < 0) return err_closed(); if (!self->readable) { @@ -877,22 +880,23 @@ _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) } if (size < 0) - return _io_FileIO_readall_impl(self); + return _io_FileIO_readall_impl(self, cls); if (size > _PY_READ_MAX) { size = _PY_READ_MAX; } - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; - ptr = PyBytes_AS_STRING(bytes); + } + char *ptr = PyBytesWriter_GetData(writer); - n = _Py_read(self->fd, ptr, size); + Py_ssize_t n = _Py_read(self->fd, ptr, size); if (n == -1) { - /* copy errno because Py_DECREF() can indirectly modify it */ + // copy errno because PyBytesWriter_Discard() can indirectly modify it int err = errno; - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); if (err == EAGAIN) { PyErr_Clear(); Py_RETURN_NONE; @@ -900,14 +904,7 @@ _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) return NULL; } - if (n != size) { - if (_PyBytes_Resize(&bytes, n) < 0) { - Py_CLEAR(bytes); - return NULL; - } - } - - return (PyObject *) bytes; + return PyBytesWriter_FinishWithSize(writer, n); } /*[clinic input] @@ -1019,6 +1016,7 @@ portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_er } /*[clinic input] +@permit_long_docstring_body _io.FileIO.seek pos: object whence: int = 0 @@ -1037,7 +1035,7 @@ Note that not all file objects are seekable. static PyObject * _io_FileIO_seek_impl(fileio *self, PyObject *pos, int whence) -/*[clinic end generated code: output=c976acdf054e6655 input=0439194b0774d454]*/ +/*[clinic end generated code: output=c976acdf054e6655 input=f077c492a84c9e62]*/ { if (self->fd < 0) return err_closed(); @@ -1157,10 +1155,17 @@ mode_string(fileio *self) return "ab"; } else if (self->readable) { - if (self->writable) - return "rb+"; - else + if (self->writable) { + if (self->truncate) { + return "wb+"; + } + else { + return "rb+"; + } + } + else { return "rb"; + } } else return "wb"; @@ -1331,7 +1336,7 @@ static PyType_Slot fileio_slots[] = { {0, NULL}, }; -PyType_Spec fileio_spec = { +PyType_Spec _Py_fileio_spec = { .name = "_io.FileIO", .basicsize = sizeof(fileio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index cd4c7e7cead..f036ea503b1 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -14,6 +14,7 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include "_iomodule.h" @@ -92,6 +93,7 @@ iobase_unsupported(_PyIO_State *state, const char *message) /* Positioning */ /*[clinic input] +@permit_long_docstring_body _io._IOBase.seek cls: defining_class offset: int(unused=True) @@ -115,7 +117,7 @@ Return the new absolute position. static PyObject * _io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(offset), int Py_UNUSED(whence)) -/*[clinic end generated code: output=8bd74ea6538ded53 input=74211232b363363e]*/ +/*[clinic end generated code: output=8bd74ea6538ded53 input=a21b5aad416ff6a9]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "seek"); @@ -383,8 +385,7 @@ iobase_dealloc(PyObject *op) } PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); Py_CLEAR(self->dict); tp->tp_free(self); Py_DECREF(tp); @@ -884,7 +885,7 @@ static PyType_Slot iobase_slots[] = { {0, NULL}, }; -PyType_Spec iobase_spec = { +PyType_Spec _Py_iobase_spec = { .name = "_io._IOBase", .basicsize = sizeof(iobase), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -926,26 +927,33 @@ _io__RawIOBase_read_impl(PyObject *self, Py_ssize_t n) return PyObject_CallMethodNoArgs(self, &_Py_ID(readall)); } - /* TODO: allocate a bytes object directly instead and manually construct - a writable memoryview pointing to it. */ b = PyByteArray_FromStringAndSize(NULL, n); - if (b == NULL) + if (b == NULL) { return NULL; + } res = PyObject_CallMethodObjArgs(self, &_Py_ID(readinto), b, NULL); if (res == NULL || res == Py_None) { - Py_DECREF(b); - return res; + goto cleanup; } - n = PyNumber_AsSsize_t(res, PyExc_ValueError); - Py_DECREF(res); - if (n == -1 && PyErr_Occurred()) { - Py_DECREF(b); - return NULL; + Py_ssize_t bytes_filled = PyNumber_AsSsize_t(res, PyExc_ValueError); + Py_CLEAR(res); + if (bytes_filled == -1 && PyErr_Occurred()) { + goto cleanup; } + if (bytes_filled < 0 || bytes_filled > n) { + PyErr_Format(PyExc_ValueError, + "readinto returned %zd outside buffer size %zd", + bytes_filled, n); + goto cleanup; + } + if (PyByteArray_Resize(b, bytes_filled) < 0) { + goto cleanup; + } + res = PyObject_CallMethodNoArgs(b, &_Py_ID(take_bytes)); - res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), n); +cleanup: Py_DECREF(b); return res; } @@ -961,12 +969,10 @@ static PyObject * _io__RawIOBase_readall_impl(PyObject *self) /*[clinic end generated code: output=1987b9ce929425a0 input=688874141213622a]*/ { - int r; - PyObject *chunks = PyList_New(0); - PyObject *result; - - if (chunks == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(0); + if (writer == NULL) { return NULL; + } while (1) { PyObject *data = _PyObject_CallMethod(self, &_Py_ID(read), @@ -977,21 +983,21 @@ _io__RawIOBase_readall_impl(PyObject *self) if (_PyIO_trap_eintr()) { continue; } - Py_DECREF(chunks); + PyBytesWriter_Discard(writer); return NULL; } if (data == Py_None) { - if (PyList_GET_SIZE(chunks) == 0) { - Py_DECREF(chunks); + if (PyBytesWriter_GetSize(writer) == 0) { + PyBytesWriter_Discard(writer); return data; } Py_DECREF(data); break; } if (!PyBytes_Check(data)) { - Py_DECREF(chunks); Py_DECREF(data); PyErr_SetString(PyExc_TypeError, "read() should return bytes"); + PyBytesWriter_Discard(writer); return NULL; } if (PyBytes_GET_SIZE(data) == 0) { @@ -999,16 +1005,16 @@ _io__RawIOBase_readall_impl(PyObject *self) Py_DECREF(data); break; } - r = PyList_Append(chunks, data); - Py_DECREF(data); - if (r < 0) { - Py_DECREF(chunks); + if (PyBytesWriter_WriteBytes(writer, + PyBytes_AS_STRING(data), + PyBytes_GET_SIZE(data)) < 0) { + Py_DECREF(data); + PyBytesWriter_Discard(writer); return NULL; } + Py_DECREF(data); } - result = PyBytes_Join((PyObject *)&_Py_SINGLETON(bytes_empty), chunks); - Py_DECREF(chunks); - return result; + return PyBytesWriter_Finish(writer); } static PyObject * @@ -1040,7 +1046,7 @@ static PyType_Slot rawiobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec rawiobase_spec = { +PyType_Spec _Py_rawiobase_spec = { .name = "_io._RawIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 9d1bfa3ea05..781ca4327f9 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1,6 +1,7 @@ #include "Python.h" #include <stddef.h> // offsetof() #include "pycore_object.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" /* Implementation note: the buffer is always at least one character longer @@ -444,7 +445,7 @@ stringio_iternext(PyObject *op) /*[clinic input] @critical_section _io.StringIO.truncate - pos as size: Py_ssize_t(accept={int, NoneType}, c_default="((stringio *)self)->pos") = None + pos: object = None / Truncate size to pos. @@ -455,16 +456,26 @@ Returns the new absolute position. [clinic start generated code]*/ static PyObject * -_io_StringIO_truncate_impl(stringio *self, Py_ssize_t size) -/*[clinic end generated code: output=eb3aef8e06701365 input=fa8a6c98bb2ba780]*/ +_io_StringIO_truncate_impl(stringio *self, PyObject *pos) +/*[clinic end generated code: output=c76c43b5ecfaf4e2 input=d59fd2ee49757ae6]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "Negative size value %zd", size); - return NULL; + Py_ssize_t size; + if (pos == Py_None) { + size = self->pos; + } + else { + size = PyLong_AsLong(pos); + if (size == -1 && PyErr_Occurred()) { + return NULL; + } + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "negative pos value %zd", size); + return NULL; + } } if (size < self->string_size) { @@ -628,9 +639,7 @@ stringio_dealloc(PyObject *op) } PyUnicodeWriter_Discard(self->writer); (void)stringio_clear(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } @@ -1085,7 +1094,7 @@ static PyType_Slot stringio_slots[] = { {0, NULL}, }; -PyType_Spec stringio_spec = { +PyType_Spec _Py_stringio_spec = { .name = "_io.StringIO", .basicsize = sizeof(stringio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 86328e46a7b..65da300abcf 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -16,6 +16,7 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_unicodeobject.h" // _PyUnicode_AsASCIIString() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -52,6 +53,7 @@ _unsupported(_PyIO_State *state, const char *message) } /*[clinic input] +@permit_long_docstring_body _io._TextIOBase.detach cls: defining_class / @@ -63,7 +65,7 @@ After the underlying buffer has been detached, the TextIO is in an unusable stat static PyObject * _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=50915f40c609eaa4 input=987ca3640d0a3776]*/ +/*[clinic end generated code: output=50915f40c609eaa4 input=8cd0652c17d7f015]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "detach"); @@ -206,7 +208,7 @@ static PyType_Slot textiobase_slots[] = { }; /* Do not set Py_TPFLAGS_HAVE_GC so that tp_traverse and tp_clear are inherited */ -PyType_Spec textiobase_spec = { +PyType_Spec _Py_textiobase_spec = { .name = "_io._TextIOBase", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE), @@ -1469,8 +1471,7 @@ textiowrapper_dealloc(PyObject *op) return; self->ok = 0; _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)textiowrapper_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -1578,6 +1579,8 @@ _io_TextIOWrapper_detach_impl(textio *self) static int _textiowrapper_writeflush(textio *self) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + if (self->pending_bytes == NULL) return 0; @@ -2842,7 +2845,7 @@ _io_TextIOWrapper_tell_impl(textio *self) current pos */ skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip); skip_back = 1; - assert(skip_back <= PyBytes_GET_SIZE(next_input)); + assert(skip_bytes <= PyBytes_GET_SIZE(next_input)); input = PyBytes_AS_STRING(next_input); while (skip_bytes > 0) { /* Decode up to temptative start point */ @@ -3173,8 +3176,9 @@ _io_TextIOWrapper_close_impl(textio *self) } static PyObject * -textiowrapper_iternext(PyObject *op) +textiowrapper_iternext_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyObject *line; textio *self = textio_CAST(op); @@ -3210,6 +3214,16 @@ textiowrapper_iternext(PyObject *op) return line; } +static PyObject * +textiowrapper_iternext(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = textiowrapper_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] @critical_section @getter @@ -3338,7 +3352,7 @@ static PyType_Slot nldecoder_slots[] = { {0, NULL}, }; -PyType_Spec nldecoder_spec = { +PyType_Spec _Py_nldecoder_spec = { .name = "_io.IncrementalNewlineDecoder", .basicsize = sizeof(nldecoder_object), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | @@ -3390,7 +3404,7 @@ static PyGetSetDef textiowrapper_getset[] = { {NULL} }; -PyType_Slot textiowrapper_slots[] = { +static PyType_Slot textiowrapper_slots[] = { {Py_tp_dealloc, textiowrapper_dealloc}, {Py_tp_repr, textiowrapper_repr}, {Py_tp_doc, (void *)_io_TextIOWrapper___init____doc__}, @@ -3404,7 +3418,7 @@ PyType_Slot textiowrapper_slots[] = { {0, NULL}, }; -PyType_Spec textiowrapper_spec = { +PyType_Spec _Py_textiowrapper_spec = { .name = "_io.TextIOWrapper", .basicsize = sizeof(textio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 3e783b9da45..677d7e85d4e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -10,6 +10,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #ifdef HAVE_WINDOWS_CONSOLE_IO @@ -518,8 +519,7 @@ winconsoleio_dealloc(PyObject *op) if (_PyIOBase_finalize(op) < 0) return; _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); Py_CLEAR(self->dict); tp->tp_free(self); Py_DECREF(tp); @@ -1253,7 +1253,7 @@ static PyType_Slot winconsoleio_slots[] = { {0, NULL}, }; -PyType_Spec winconsoleio_spec = { +PyType_Spec _Py_winconsoleio_spec = { .name = "_io._WindowsConsoleIO", .basicsize = sizeof(winconsoleio), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | diff --git a/Modules/_json.c b/Modules/_json.c index 89b0a41dd10..14714d4b346 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -10,6 +10,7 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST() #include "pycore_global_strings.h" // _Py_ID() #include "pycore_pyerrors.h" // _PyErr_FormatNote #include "pycore_runtime.h" // _PyRuntime @@ -17,6 +18,12 @@ #include <stdbool.h> // bool +#include "clinic/_json.c.h" + +/*[clinic input] +module _json +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=549fa53592c925b2]*/ typedef struct _PyScannerObject { PyObject_HEAD @@ -51,7 +58,7 @@ typedef struct _PyEncoderObject { char sort_keys; char skipkeys; int allow_nan; - PyCFunction fast_encode; + int (*fast_encode)(PyUnicodeWriter *, PyObject *); } PyEncoderObject; #define PyEncoderObject_CAST(op) ((PyEncoderObject *)(op)) @@ -102,8 +109,10 @@ static PyObject * _encoded_const(PyObject *obj); static void raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); +static int +_steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen); +static int +encoder_write_string(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj); static PyObject * encoder_encode_float(PyEncoderObject *s, PyObject *obj); @@ -146,22 +155,11 @@ ascii_escape_unichar(Py_UCS4 c, unsigned char *output, Py_ssize_t chars) return chars; } -static PyObject * -ascii_escape_unicode(PyObject *pystr) +static Py_ssize_t +ascii_escape_size(const void *input, int kind, Py_ssize_t input_chars) { - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyUnicode */ Py_ssize_t i; - Py_ssize_t input_chars; Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - const void *input; - Py_UCS1 *output; - int kind; - - input_chars = PyUnicode_GET_LENGTH(pystr); - input = PyUnicode_DATA(pystr); - kind = PyUnicode_KIND(pystr); /* Compute the output size */ for (i = 0, output_size = 2; i < input_chars; i++) { @@ -181,11 +179,22 @@ ascii_escape_unicode(PyObject *pystr) } if (output_size > PY_SSIZE_T_MAX - d) { PyErr_SetString(PyExc_OverflowError, "string is too long to escape"); - return NULL; + return -1; } output_size += d; } + return output_size; +} + +static PyObject * +ascii_escape_unicode_and_size(const void *input, int kind, Py_ssize_t input_chars, Py_ssize_t output_size) +{ + Py_ssize_t i; + Py_ssize_t chars; + PyObject *rval; + Py_UCS1 *output; + rval = PyUnicode_New(output_size, 127); if (rval == NULL) { return NULL; @@ -210,23 +219,62 @@ ascii_escape_unicode(PyObject *pystr) } static PyObject * -escape_unicode(PyObject *pystr) +ascii_escape_unicode(PyObject *pystr) +{ + /* Take a PyUnicode pystr and return a new ASCII-only escaped PyUnicode */ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + + Py_ssize_t output_size = ascii_escape_size(input, kind, input_chars); + if (output_size < 0) { + return NULL; + } + + return ascii_escape_unicode_and_size(input, kind, input_chars, output_size); +} + +static int +write_escaped_ascii(PyUnicodeWriter *writer, PyObject *pystr) { - /* Take a PyUnicode pystr and return a new escaped PyUnicode */ - Py_ssize_t i; Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; const void *input; int kind; - Py_UCS4 maxchar; - maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); input_chars = PyUnicode_GET_LENGTH(pystr); input = PyUnicode_DATA(pystr); kind = PyUnicode_KIND(pystr); + Py_ssize_t output_size = ascii_escape_size(input, kind, input_chars); + if (output_size < 0) { + return -1; + } + + if (output_size == input_chars + 2) { + /* No need to escape anything */ + if (PyUnicodeWriter_WriteChar(writer, '"') < 0) { + return -1; + } + if (PyUnicodeWriter_WriteStr(writer, pystr) < 0) { + return -1; + } + return PyUnicodeWriter_WriteChar(writer, '"'); + } + + PyObject *rval = ascii_escape_unicode_and_size(input, kind, input_chars, output_size); + if (rval == NULL) { + return -1; + } + + return _steal_accumulate(writer, rval); +} + +static Py_ssize_t +escape_size(const void *input, int kind, Py_ssize_t input_chars) +{ + Py_ssize_t i; + Py_ssize_t output_size; + /* Compute the output size */ for (i = 0, output_size = 2; i < input_chars; i++) { Py_UCS4 c = PyUnicode_READ(kind, input, i); @@ -244,11 +292,21 @@ escape_unicode(PyObject *pystr) } if (output_size > PY_SSIZE_T_MAX - d) { PyErr_SetString(PyExc_OverflowError, "string is too long to escape"); - return NULL; + return -1; } output_size += d; } + return output_size; +} + +static PyObject * +escape_unicode_and_size(const void *input, int kind, Py_UCS4 maxchar, Py_ssize_t input_chars, Py_ssize_t output_size) +{ + Py_ssize_t i; + Py_ssize_t chars; + PyObject *rval; + rval = PyUnicode_New(output_size, maxchar); if (rval == NULL) return NULL; @@ -303,6 +361,55 @@ escape_unicode(PyObject *pystr) return rval; } +static PyObject * +escape_unicode(PyObject *pystr) +{ + /* Take a PyUnicode pystr and return a new escaped PyUnicode */ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); + + Py_ssize_t output_size = escape_size(input, kind, input_chars); + if (output_size < 0) { + return NULL; + } + + return escape_unicode_and_size(input, kind, maxchar, input_chars, output_size); +} + +static int +write_escaped_unicode(PyUnicodeWriter *writer, PyObject *pystr) +{ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); + + Py_ssize_t output_size = escape_size(input, kind, input_chars); + if (output_size < 0) { + return -1; + } + + if (output_size == input_chars + 2) { + /* No need to escape anything */ + if (PyUnicodeWriter_WriteChar(writer, '"') < 0) { + return -1; + } + if (PyUnicodeWriter_WriteStr(writer, pystr) < 0) { + return -1; + } + return PyUnicodeWriter_WriteChar(writer, '"'); + } + + PyObject *rval = escape_unicode_and_size(input, kind, maxchar, input_chars, output_size); + if (rval == NULL) { + return -1; + } + + return _steal_accumulate(writer, rval); +} + static void raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end) { @@ -360,13 +467,6 @@ _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { return tpl; } -static inline int -_PyUnicodeWriter_IsEmpty(PyUnicodeWriter *writer_pub) -{ - _PyUnicodeWriter *writer = (_PyUnicodeWriter*)writer_pub; - return (writer->pos == 0); -} - static PyObject * scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) { @@ -385,10 +485,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next const void *buf; int kind; - PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); - if (writer == NULL) { - goto bail; - } + PyUnicodeWriter *writer = NULL; len = PyUnicode_GET_LENGTH(pystr); buf = PyUnicode_DATA(pystr); @@ -419,12 +516,11 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next if (c == '"') { // Fast path for simple case. - if (_PyUnicodeWriter_IsEmpty(writer)) { + if (writer == NULL) { PyObject *ret = PyUnicode_Substring(pystr, end, next); if (ret == NULL) { goto bail; } - PyUnicodeWriter_Discard(writer); *next_end_ptr = next + 1;; return ret; } @@ -432,6 +528,11 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next else if (c != '\\') { raise_errmsg("Unterminated string starting at", pystr, begin); goto bail; + } else if (writer == NULL) { + writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto bail; + } } /* Pick up this chunk if it's not zero length */ @@ -542,89 +643,62 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next return NULL; } -PyDoc_STRVAR(pydoc_scanstring, - "scanstring(string, end, strict=True) -> (string, end)\n" - "\n" - "Scan the string s for a JSON string. End is the index of the\n" - "character in s after the quote that started the JSON string.\n" - "Unescapes all valid JSON string escape sequences and raises ValueError\n" - "on attempt to decode an invalid string. If strict is False then literal\n" - "control characters are allowed in the string.\n" - "\n" - "Returns a tuple of the decoded string and the index of the character in s\n" - "after the end quote." -); +/*[clinic input] +_json.scanstring as py_scanstring + pystr: unicode + end: Py_ssize_t + strict: bool = True + / + +Scan the string s for a JSON string. + +End is the index of the character in s after the quote that started the +JSON string. Unescapes all valid JSON string escape sequences and raises +ValueError on attempt to decode an invalid string. If strict is False +then literal control characters are allowed in the string. + +Returns a tuple of the decoded string and the index of the character in s +after the end quote. +[clinic start generated code]*/ static PyObject * -py_scanstring(PyObject* Py_UNUSED(self), PyObject *args) +py_scanstring_impl(PyObject *module, PyObject *pystr, Py_ssize_t end, + int strict) +/*[clinic end generated code: output=961740cfae07cdb3 input=cff59e47498f4d8e]*/ { - PyObject *pystr; - PyObject *rval; - Py_ssize_t end; Py_ssize_t next_end = -1; - int strict = 1; - if (!PyArg_ParseTuple(args, "On|p:scanstring", &pystr, &end, &strict)) { - return NULL; - } - if (PyUnicode_Check(pystr)) { - rval = scanstring_unicode(pystr, end, strict, &next_end); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } + PyObject *rval = scanstring_unicode(pystr, end, strict, &next_end); return _build_rval_index_tuple(rval, next_end); } -PyDoc_STRVAR(pydoc_encode_basestring_ascii, - "encode_basestring_ascii(string) -> string\n" - "\n" - "Return an ASCII-only JSON representation of a Python string" -); +/*[clinic input] +_json.encode_basestring_ascii as py_encode_basestring_ascii + pystr: unicode + / + +Return an ASCII-only JSON representation of a Python string +[clinic start generated code]*/ static PyObject * -py_encode_basestring_ascii(PyObject* Py_UNUSED(self), PyObject *pystr) +py_encode_basestring_ascii_impl(PyObject *module, PyObject *pystr) +/*[clinic end generated code: output=7b3841287cf211df input=4f3609498aff2de5]*/ { - PyObject *rval; - /* Return an ASCII-only JSON representation of a Python string */ - /* METH_O */ - if (PyUnicode_Check(pystr)) { - rval = ascii_escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return rval; + return ascii_escape_unicode(pystr); } +/*[clinic input] +_json.encode_basestring as py_encode_basestring + pystr: unicode + / -PyDoc_STRVAR(pydoc_encode_basestring, - "encode_basestring(string) -> string\n" - "\n" - "Return a JSON representation of a Python string" -); +Return a JSON representation of a Python string +[clinic start generated code]*/ static PyObject * -py_encode_basestring(PyObject* Py_UNUSED(self), PyObject *pystr) +py_encode_basestring_impl(PyObject *module, PyObject *pystr) +/*[clinic end generated code: output=900950f95df3f1c9 input=d42ef714b2c07386]*/ { - PyObject *rval; - /* Return a JSON representation of a Python string */ - /* METH_O */ - if (PyUnicode_Check(pystr)) { - rval = escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return rval; + return escape_unicode(pystr); } static void @@ -1228,16 +1302,23 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; - PyObject *markers = Py_None, *defaultfn, *encoder, *indent, *key_separator; + PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; PyObject *item_separator; int sort_keys, skipkeys, allow_nan; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!?OOOUUppp:make_encoder", kwlist, - &PyDict_Type, &markers, &defaultfn, &encoder, &indent, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUppp:make_encoder", kwlist, + &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) return NULL; + if (markers != Py_None && !PyDict_Check(markers)) { + PyErr_Format(PyExc_TypeError, + "make_encoder() argument 1 must be dict or None, " + "not %.200s", Py_TYPE(markers)->tp_name); + return NULL; + } + s = (PyEncoderObject *)type->tp_alloc(type, 0); if (s == NULL) return NULL; @@ -1255,8 +1336,11 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyCFunction_Check(s->encoder)) { PyCFunction f = PyCFunction_GetFunction(s->encoder); - if (f == py_encode_basestring_ascii || f == py_encode_basestring) { - s->fast_encode = f; + if (f == py_encode_basestring_ascii) { + s->fast_encode = write_escaped_ascii; + } + else if (f == py_encode_basestring) { + s->fast_encode = write_escaped_unicode; } } @@ -1352,7 +1436,7 @@ write_newline_indent(PyUnicodeWriter *writer, static PyObject * encoder_call(PyObject *op, PyObject *args, PyObject *kwds) { - /* Python callable interface to encode_listencode_obj */ + /* Python callable interface to encoder_listencode_obj */ static char *kwlist[] = {"obj", "_current_indent_level", NULL}; PyObject *obj; Py_ssize_t indent_level; @@ -1437,24 +1521,27 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) return PyFloat_Type.tp_repr(obj); } -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) +static int +encoder_write_string(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj) { /* Return the JSON representation of a string */ PyObject *encoded; if (s->fast_encode) { - return s->fast_encode(NULL, obj); + return s->fast_encode(writer, obj); } encoded = PyObject_CallOneArg(s->encoder, obj); - if (encoded != NULL && !PyUnicode_Check(encoded)) { + if (encoded == NULL) { + return -1; + } + if (!PyUnicode_Check(encoded)) { PyErr_Format(PyExc_TypeError, "encoder() must return a string, not %.80s", Py_TYPE(encoded)->tp_name); Py_DECREF(encoded); - return NULL; + return -1; } - return encoded; + return _steal_accumulate(writer, encoded); } static int @@ -1476,19 +1563,16 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, int rv; if (obj == Py_None) { - return PyUnicodeWriter_WriteUTF8(writer, "null", 4); + return PyUnicodeWriter_WriteASCII(writer, "null", 4); } else if (obj == Py_True) { - return PyUnicodeWriter_WriteUTF8(writer, "true", 4); + return PyUnicodeWriter_WriteASCII(writer, "true", 4); } else if (obj == Py_False) { - return PyUnicodeWriter_WriteUTF8(writer, "false", 5); + return PyUnicodeWriter_WriteASCII(writer, "false", 5); } else if (PyUnicode_Check(obj)) { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded == NULL) - return -1; - return _steal_accumulate(writer, encoded); + return encoder_write_string(s, writer, obj); } else if (PyLong_Check(obj)) { if (PyLong_CheckExact(obj)) { @@ -1577,7 +1661,7 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs PyObject *item_separator) { PyObject *keystr = NULL; - PyObject *encoded; + int rv; if (PyUnicode_Check(key)) { keystr = Py_NewRef(key); @@ -1609,6 +1693,12 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs if (*first) { *first = false; + if (s->indent != Py_None) { + if (write_newline_indent(writer, indent_level, indent_cache) < 0) { + Py_DECREF(keystr); + return -1; + } + } } else { if (PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { @@ -1617,13 +1707,10 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs } } - encoded = encoder_encode_string(s, keystr); + rv = encoder_write_string(s, writer, keystr); Py_DECREF(keystr); - if (encoded == NULL) { - return -1; - } - if (_steal_accumulate(writer, encoded) < 0) { + if (rv < 0) { return -1; } if (PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { @@ -1636,20 +1723,89 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs return 0; } +static inline int +_encoder_iterate_mapping_lock_held(PyEncoderObject *s, PyUnicodeWriter *writer, + bool *first, PyObject *dct, PyObject *items, + Py_ssize_t indent_level, PyObject *indent_cache, + PyObject *separator) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(items); + PyObject *key, *value; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) { + PyObject *item = PyList_GET_ITEM(items, i); +#ifdef Py_GIL_DISABLED + // gh-119438: in the free-threading build the critical section on items can get suspended + Py_INCREF(item); +#endif + if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { + PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); +#ifdef Py_GIL_DISABLED + Py_DECREF(item); +#endif + return -1; + } + + key = PyTuple_GET_ITEM(item, 0); + value = PyTuple_GET_ITEM(item, 1); + if (encoder_encode_key_value(s, writer, first, dct, key, value, + indent_level, indent_cache, + separator) < 0) { +#ifdef Py_GIL_DISABLED + Py_DECREF(item); +#endif + return -1; + } +#ifdef Py_GIL_DISABLED + Py_DECREF(item); +#endif + } + + return 0; +} + +static inline int +_encoder_iterate_dict_lock_held(PyEncoderObject *s, PyUnicodeWriter *writer, + bool *first, PyObject *dct, Py_ssize_t indent_level, + PyObject *indent_cache, PyObject *separator) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dct); + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(dct, &pos, &key, &value)) { +#ifdef Py_GIL_DISABLED + // gh-119438: in the free-threading build the critical section on dct can get suspended + Py_INCREF(key); + Py_INCREF(value); +#endif + if (encoder_encode_key_value(s, writer, first, dct, key, value, + indent_level, indent_cache, + separator) < 0) { +#ifdef Py_GIL_DISABLED + Py_DECREF(key); + Py_DECREF(value); +#endif + return -1; + } +#ifdef Py_GIL_DISABLED + Py_DECREF(key); + Py_DECREF(value); +#endif + } + return 0; +} + static int encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, - PyObject *dct, + PyObject *dct, Py_ssize_t indent_level, PyObject *indent_cache) { /* Encode Python dict dct a JSON term */ PyObject *ident = NULL; - PyObject *items = NULL; - PyObject *key, *value; bool first = true; if (PyDict_GET_SIZE(dct) == 0) { /* Fast path */ - return PyUnicodeWriter_WriteUTF8(writer, "{}", 2); + return PyUnicodeWriter_WriteASCII(writer, "{}", 2); } if (s->markers != Py_None) { @@ -1676,42 +1832,35 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, if (s->indent != Py_None) { indent_level++; separator = get_item_separator(s, indent_level, indent_cache); - if (separator == NULL || - write_newline_indent(writer, indent_level, indent_cache) < 0) - { + if (separator == NULL) goto bail; - } } if (s->sort_keys || !PyDict_CheckExact(dct)) { - items = PyMapping_Items(dct); - if (items == NULL || (s->sort_keys && PyList_Sort(items) < 0)) + PyObject *items = PyMapping_Items(dct); + if (items == NULL || (s->sort_keys && PyList_Sort(items) < 0)) { + Py_XDECREF(items); + goto bail; + } + + int result; + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(items); + result = _encoder_iterate_mapping_lock_held(s, writer, &first, dct, + items, indent_level, indent_cache, separator); + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); + Py_DECREF(items); + if (result < 0) { goto bail; - - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) { - PyObject *item = PyList_GET_ITEM(items, i); - - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - - key = PyTuple_GET_ITEM(item, 0); - value = PyTuple_GET_ITEM(item, 1); - if (encoder_encode_key_value(s, writer, &first, dct, key, value, - indent_level, indent_cache, - separator) < 0) - goto bail; } - Py_CLEAR(items); } else { - Py_ssize_t pos = 0; - while (PyDict_Next(dct, &pos, &key, &value)) { - if (encoder_encode_key_value(s, writer, &first, dct, key, value, - indent_level, indent_cache, - separator) < 0) - goto bail; + int result; + Py_BEGIN_CRITICAL_SECTION(dct); + result = _encoder_iterate_dict_lock_held(s, writer, &first, dct, + indent_level, indent_cache, separator); + Py_END_CRITICAL_SECTION(); + if (result < 0) { + goto bail; } } @@ -1720,7 +1869,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } - if (s->indent != Py_None) { + if (s->indent != Py_None && !first) { indent_level--; if (write_newline_indent(writer, indent_level, indent_cache) < 0) { goto bail; @@ -1733,11 +1882,43 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, return 0; bail: - Py_XDECREF(items); Py_XDECREF(ident); return -1; } +static inline int +_encoder_iterate_fast_seq_lock_held(PyEncoderObject *s, PyUnicodeWriter *writer, + PyObject *seq, PyObject *s_fast, + Py_ssize_t indent_level, PyObject *indent_cache, PyObject *separator) +{ + for (Py_ssize_t i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { + PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); +#ifdef Py_GIL_DISABLED + // gh-119438: in the free-threading build the critical section on s_fast can get suspended + Py_INCREF(obj); +#endif + if (i) { + if (PyUnicodeWriter_WriteStr(writer, separator) < 0) { +#ifdef Py_GIL_DISABLED + Py_DECREF(obj); +#endif + return -1; + } + } + if (encoder_listencode_obj(s, writer, obj, indent_level, indent_cache)) { + _PyErr_FormatNote("when serializing %T item %zd", seq, i); +#ifdef Py_GIL_DISABLED + Py_DECREF(obj); +#endif + return -1; + } +#ifdef Py_GIL_DISABLED + Py_DECREF(obj); +#endif + } + return 0; +} + static int encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *seq, @@ -1745,15 +1926,13 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, { PyObject *ident = NULL; PyObject *s_fast = NULL; - Py_ssize_t i; - ident = NULL; - s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); + s_fast = PySequence_Fast(seq, "encoder_listencode_list needs a sequence"); if (s_fast == NULL) return -1; if (PySequence_Fast_GET_SIZE(s_fast) == 0) { Py_DECREF(s_fast); - return PyUnicodeWriter_WriteUTF8(writer, "[]", 2); + return PyUnicodeWriter_WriteASCII(writer, "[]", 2); } if (s->markers != Py_None) { @@ -1786,16 +1965,13 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, goto bail; } } - for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { - PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); - if (i) { - if (PyUnicodeWriter_WriteStr(writer, separator) < 0) - goto bail; - } - if (encoder_listencode_obj(s, writer, obj, indent_level, indent_cache)) { - _PyErr_FormatNote("when serializing %T item %zd", seq, i); - goto bail; - } + int result; + Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(seq); + result = _encoder_iterate_fast_seq_lock_held(s, writer, seq, s_fast, + indent_level, indent_cache, separator); + Py_END_CRITICAL_SECTION_SEQUENCE_FAST(); + if (result < 0) { + goto bail; } if (ident != NULL) { if (PyDict_DelItem(s->markers, ident)) @@ -1883,18 +2059,9 @@ static PyType_Spec PyEncoderType_spec = { }; static PyMethodDef speedups_methods[] = { - {"encode_basestring_ascii", - py_encode_basestring_ascii, - METH_O, - pydoc_encode_basestring_ascii}, - {"encode_basestring", - py_encode_basestring, - METH_O, - pydoc_encode_basestring}, - {"scanstring", - py_scanstring, - METH_VARARGS, - pydoc_scanstring}, + PY_ENCODE_BASESTRING_ASCII_METHODDEF + PY_ENCODE_BASESTRING_METHODDEF + PY_SCANSTRING_METHODDEF {NULL, NULL, 0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index ad618398d5b..7174eebd0c9 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -87,6 +87,41 @@ copy_grouping(const char* s) return result; } +#if defined(MS_WINDOWS) + +// 16 is the number of elements in the szCodePage field +// of the __crt_locale_strings structure. +#define MAX_CP_LEN 15 + +static int +check_locale_name(const char *locale, const char *end) +{ + size_t len = end ? (size_t)(end - locale) : strlen(locale); + const char *dot = memchr(locale, '.', len); + if (dot && locale + len - dot - 1 > MAX_CP_LEN) { + return -1; + } + return 0; +} + +static int +check_locale_name_all(const char *locale) +{ + const char *start = locale; + while (1) { + const char *end = strchr(start, ';'); + if (check_locale_name(start, end) < 0) { + return -1; + } + if (end == NULL) { + break; + } + start = end + 1; + } + return 0; +} +#endif + /*[clinic input] _locale.setlocale @@ -111,6 +146,18 @@ _locale_setlocale_impl(PyObject *module, int category, const char *locale) "invalid locale category"); return NULL; } + if (locale) { + if ((category == LC_ALL + ? check_locale_name_all(locale) + : check_locale_name(locale, NULL)) < 0) + { + /* Debug assertion failure on Windows. + * _Py_BEGIN_SUPPRESS_IPH/_Py_END_SUPPRESS_IPH do not help. */ + PyErr_SetString(get_locale_state(module)->Error, + "unsupported locale setting"); + return NULL; + } + } #endif if (locale) { @@ -408,35 +455,72 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) goto exit; } - /* assume no change in size, first */ - n1 = n1 + 1; - buf = PyMem_New(wchar_t, n1); - if (!buf) { - PyErr_NoMemory(); - goto exit; - } errno = 0; - n2 = wcsxfrm(buf, s, n1); + n2 = wcsxfrm(NULL, s, 0); if (errno && errno != ERANGE) { PyErr_SetFromErrno(PyExc_OSError); goto exit; } - if (n2 >= (size_t)n1) { - /* more space needed */ - wchar_t * new_buf = PyMem_Realloc(buf, (n2+1)*sizeof(wchar_t)); - if (!new_buf) { - PyErr_NoMemory(); - goto exit; + buf = PyMem_New(wchar_t, n2+1); + if (!buf) { + PyErr_NoMemory(); + goto exit; + } + + errno = 0; + n2 = wcsxfrm(buf, s, n2+1); + if (errno) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + /* The result is just a sequence of integers, they are not necessary + Unicode code points, so PyUnicode_FromWideChar() cannot be used + here. For example, 0xD83D 0xDC0D should not be larger than 0xFF41. + */ +#if SIZEOF_WCHAR_T == 4 + { + /* Some codes can exceed the range of Unicode code points + (0 - 0x10FFFF), so they cannot be directly used in + PyUnicode_FromKindAndData(). They should be first encoded in + a way that preserves the lexicographical order. + + Codes in the range 0-0xFFFF represent themself. + Codes larger than 0xFFFF are encoded as a pair: + * 0x1xxxx -- the highest 16 bits + * 0x0xxxx -- the lowest 16 bits + */ + size_t n3 = 0; + for (size_t i = 0; i < n2; i++) { + if ((Py_UCS4)buf[i] > 0x10000u) { + n3++; + } } - buf = new_buf; - errno = 0; - n2 = wcsxfrm(buf, s, n2+1); - if (errno) { - PyErr_SetFromErrno(PyExc_OSError); + if (n3) { + n3 += n2; // no integer overflow + Py_UCS4 *buf2 = PyMem_New(Py_UCS4, n3); + if (buf2 == NULL) { + PyErr_NoMemory(); + goto exit; + } + size_t j = 0; + for (size_t i = 0; i < n2; i++) { + Py_UCS4 c = (Py_UCS4)buf[i]; + if (c > 0x10000u) { + buf2[j++] = (c >> 16) | 0x10000u; + buf2[j++] = c & 0xFFFFu; + } + else { + buf2[j++] = c; + } + } + assert(j == n3); + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf2, n3); + PyMem_Free(buf2); goto exit; } } - result = PyUnicode_FromWideChar(buf, n2); +#endif + result = PyUnicode_FromKindAndData(sizeof(wchar_t), buf, n2); exit: PyMem_Free(buf); PyMem_Free(s); @@ -692,7 +776,17 @@ _locale_nl_langinfo_impl(PyObject *module, int item) result = result != NULL ? result : ""; char *oldloc = NULL; if (langinfo_constants[i].category != LC_CTYPE - && !is_all_ascii(result) + && *result && ( +#ifdef __GLIBC__ + // gh-133740: Always change the locale for ALT_DIGITS and ERA +# ifdef ALT_DIGITS + item == ALT_DIGITS || +# endif +# ifdef ERA + item == ERA || +# endif +#endif + !is_all_ascii(result)) && change_locale(langinfo_constants[i].category, &oldloc) < 0) { return NULL; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 626c176715b..025a3fac46e 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -6,7 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetProfile() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_FromLong() +#include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_typeobject.h" // _PyType_GetModuleState() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() @@ -111,7 +111,7 @@ static PyTime_t CallExternalTimer(ProfilerObject *pObj) if (pObj->externalTimerUnit > 0.0) { /* interpret the result as an integer that will be scaled in profiler_getstats() */ - err = _PyTime_FromLong(&result, o); + err = PyLong_AsInt64(o, &result); } else { /* interpret the result as a double measured in seconds. @@ -534,6 +534,7 @@ static int statsForEntry(rotating_node_t *node, void *arg) } /*[clinic input] +@critical_section _lsprof.Profiler.getstats cls: defining_class @@ -565,7 +566,7 @@ profiler_subentry objects: static PyObject * _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=1806ef720019ee03 input=445e193ef4522902]*/ +/*[clinic end generated code: output=1806ef720019ee03 input=3dc69eb85ed73d91]*/ { statscollector_t collect; collect.state = _PyType_GetModuleState(cls); @@ -613,6 +614,7 @@ setBuiltins(ProfilerObject *pObj, int nvalue) } /*[clinic input] +@critical_section _lsprof.Profiler._pystart_callback code: object @@ -624,7 +626,7 @@ _lsprof.Profiler._pystart_callback static PyObject * _lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, PyObject *instruction_offset) -/*[clinic end generated code: output=5fec8b7ad5ed25e8 input=b166e6953c579cda]*/ +/*[clinic end generated code: output=5fec8b7ad5ed25e8 input=b61a0e79cf1f8499]*/ { ptrace_enter_call((PyObject*)self, (void *)code, code); @@ -632,6 +634,29 @@ _lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, } /*[clinic input] +@critical_section +_lsprof.Profiler._pythrow_callback + + code: object + instruction_offset: object + exception: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__pythrow_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *exception) +/*[clinic end generated code: output=0a32988919dfb94c input=60c7f272206d3758]*/ +{ + ptrace_enter_call((PyObject*)self, (void *)code, code); + + Py_RETURN_NONE; +} + +/*[clinic input] +@critical_section _lsprof.Profiler._pyreturn_callback code: object @@ -646,7 +671,7 @@ _lsprof_Profiler__pyreturn_callback_impl(ProfilerObject *self, PyObject *code, PyObject *instruction_offset, PyObject *retval) -/*[clinic end generated code: output=9e2f6fc1b882c51e input=667ffaeb2fa6fd1f]*/ +/*[clinic end generated code: output=9e2f6fc1b882c51e input=0ddcc1ec53faa928]*/ { ptrace_leave_call((PyObject*)self, (void *)code); @@ -682,6 +707,7 @@ PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObje } /*[clinic input] +@critical_section _lsprof.Profiler._ccall_callback code: object @@ -696,7 +722,7 @@ static PyObject * _lsprof_Profiler__ccall_callback_impl(ProfilerObject *self, PyObject *code, PyObject *instruction_offset, PyObject *callable, PyObject *self_arg) -/*[clinic end generated code: output=152db83cabd18cad input=0e66687cfb95c001]*/ +/*[clinic end generated code: output=152db83cabd18cad input=2fc1e0630ee5e32b]*/ { if (self->flags & POF_BUILTINS) { PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); @@ -712,6 +738,7 @@ _lsprof_Profiler__ccall_callback_impl(ProfilerObject *self, PyObject *code, } /*[clinic input] +@critical_section _lsprof.Profiler._creturn_callback code: object @@ -727,7 +754,7 @@ _lsprof_Profiler__creturn_callback_impl(ProfilerObject *self, PyObject *code, PyObject *instruction_offset, PyObject *callable, PyObject *self_arg) -/*[clinic end generated code: output=1e886dde8fed8fb0 input=b18afe023746923a]*/ +/*[clinic end generated code: output=1e886dde8fed8fb0 input=bdc246d6b5b8714a]*/ { if (self->flags & POF_BUILTINS) { PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); @@ -747,7 +774,7 @@ static const struct { } callback_table[] = { {PY_MONITORING_EVENT_PY_START, "_pystart_callback"}, {PY_MONITORING_EVENT_PY_RESUME, "_pystart_callback"}, - {PY_MONITORING_EVENT_PY_THROW, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_THROW, "_pythrow_callback"}, {PY_MONITORING_EVENT_PY_RETURN, "_pyreturn_callback"}, {PY_MONITORING_EVENT_PY_YIELD, "_pyreturn_callback"}, {PY_MONITORING_EVENT_PY_UNWIND, "_pyreturn_callback"}, @@ -759,6 +786,7 @@ static const struct { /*[clinic input] +@critical_section _lsprof.Profiler.enable subcalls: bool = True @@ -775,14 +803,14 @@ Start collecting profiling information. static PyObject * _lsprof_Profiler_enable_impl(ProfilerObject *self, int subcalls, int builtins) -/*[clinic end generated code: output=1e747f9dc1edd571 input=9ab81405107ab7f1]*/ +/*[clinic end generated code: output=1e747f9dc1edd571 input=0b88115b1c796173]*/ { int all_events = 0; if (setSubcalls(self, subcalls) < 0 || setBuiltins(self, builtins) < 0) { return NULL; } - PyObject* monitoring = PyImport_ImportModuleAttrString("sys", "monitoring"); + PyObject* monitoring = PySys_GetAttrString("monitoring"); if (!monitoring) { return NULL; } @@ -848,6 +876,7 @@ flush_unmatched(ProfilerObject *pObj) /*[clinic input] +@critical_section _lsprof.Profiler.disable Stop collecting profiling information. @@ -855,7 +884,7 @@ Stop collecting profiling information. static PyObject * _lsprof_Profiler_disable_impl(ProfilerObject *self) -/*[clinic end generated code: output=838cffef7f651870 input=05700b3fc68d1f50]*/ +/*[clinic end generated code: output=838cffef7f651870 input=f7e4787cae20f7f6]*/ { if (self->flags & POF_EXT_TIMER) { PyErr_SetString(PyExc_RuntimeError, @@ -864,7 +893,7 @@ _lsprof_Profiler_disable_impl(ProfilerObject *self) } if (self->flags & POF_ENABLED) { PyObject* result = NULL; - PyObject* monitoring = PyImport_ImportModuleAttrString("sys", "monitoring"); + PyObject* monitoring = PySys_GetAttrString("monitoring"); if (!monitoring) { return NULL; @@ -907,6 +936,7 @@ _lsprof_Profiler_disable_impl(ProfilerObject *self) } /*[clinic input] +@critical_section _lsprof.Profiler.clear Clear all profiling information collected so far. @@ -914,7 +944,7 @@ Clear all profiling information collected so far. static PyObject * _lsprof_Profiler_clear_impl(ProfilerObject *self) -/*[clinic end generated code: output=dd1c668fb84b1335 input=fbe1f88c28be4f98]*/ +/*[clinic end generated code: output=dd1c668fb84b1335 input=4aab219d5d7a9bec]*/ { if (self->flags & POF_EXT_TIMER) { PyErr_SetString(PyExc_RuntimeError, @@ -983,7 +1013,7 @@ profiler_init_impl(ProfilerObject *self, PyObject *timer, double timeunit, Py_XSETREF(self->externalTimer, Py_XNewRef(timer)); self->tool_id = PY_MONITORING_PROFILER_ID; - PyObject* monitoring = PyImport_ImportModuleAttrString("sys", "monitoring"); + PyObject* monitoring = PySys_GetAttrString("monitoring"); if (!monitoring) { return -1; } @@ -1002,6 +1032,7 @@ static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_DISABLE_METHODDEF _LSPROF_PROFILER_CLEAR_METHODDEF _LSPROF_PROFILER__PYSTART_CALLBACK_METHODDEF + _LSPROF_PROFILER__PYTHROW_CALLBACK_METHODDEF _LSPROF_PROFILER__PYRETURN_CALLBACK_METHODDEF _LSPROF_PROFILER__CCALL_CALLBACK_METHODDEF _LSPROF_PROFILER__CRETURN_CALLBACK_METHODDEF diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f9b4c2a170e..58766233998 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -17,6 +17,7 @@ #include <lzma.h> +#include "pycore_long.h" // _PyLong_UInt32_Converter() // Blocks output buffer wrappers #include "pycore_blocks_output_buffer.h" @@ -71,13 +72,6 @@ OutputBuffer_OnError(_BlocksOutputBuffer *buffer) } -#define ACQUIRE_LOCK(obj) do { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while (0) -#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) typedef struct { PyTypeObject *lzma_compressor_type; @@ -110,7 +104,7 @@ typedef struct { lzma_allocator alloc; lzma_stream lzs; int flushed; - PyThread_type_lock lock; + PyMutex mutex; } Compressor; typedef struct { @@ -123,7 +117,7 @@ typedef struct { char needs_input; uint8_t *input_buffer; size_t input_buffer_size; - PyThread_type_lock lock; + PyMutex mutex; } Decompressor; #define Compressor_CAST(op) ((Compressor *)(op)) @@ -223,8 +217,6 @@ FUNCNAME(PyObject *obj, void *ptr) \ return 1; \ } -INT_TYPE_CONVERTER_FUNC(uint32_t, uint32_converter) -INT_TYPE_CONVERTER_FUNC(uint64_t, uint64_converter) INT_TYPE_CONVERTER_FUNC(lzma_vli, lzma_vli_converter) INT_TYPE_CONVERTER_FUNC(lzma_mode, lzma_mode_converter) INT_TYPE_CONVERTER_FUNC(lzma_match_finder, lzma_mf_converter) @@ -254,7 +246,7 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) return NULL; } if (preset_obj != NULL) { - int ok = uint32_converter(preset_obj, &preset); + int ok = _PyLong_UInt32_Converter(preset_obj, &preset); Py_DECREF(preset_obj); if (!ok) { return NULL; @@ -275,14 +267,14 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OOO&O&O&O&O&O&O&O&", optnames, &id, &preset_obj, - uint32_converter, &options->dict_size, - uint32_converter, &options->lc, - uint32_converter, &options->lp, - uint32_converter, &options->pb, + _PyLong_UInt32_Converter, &options->dict_size, + _PyLong_UInt32_Converter, &options->lc, + _PyLong_UInt32_Converter, &options->lp, + _PyLong_UInt32_Converter, &options->pb, lzma_mode_converter, &options->mode, - uint32_converter, &options->nice_len, + _PyLong_UInt32_Converter, &options->nice_len, lzma_mf_converter, &options->mf, - uint32_converter, &options->depth)) { + _PyLong_UInt32_Converter, &options->depth)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for LZMA filter"); PyMem_Free(options); @@ -301,7 +293,7 @@ parse_filter_spec_delta(_lzma_state *state, PyObject *spec) lzma_options_delta *options; if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, - &id, uint32_converter, &dist)) { + &id, _PyLong_UInt32_Converter, &dist)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for delta filter"); return NULL; @@ -325,7 +317,7 @@ parse_filter_spec_bcj(_lzma_state *state, PyObject *spec) lzma_options_bcj *options; if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, - &id, uint32_converter, &start_offset)) { + &id, _PyLong_UInt32_Converter, &start_offset)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for BCJ filter"); return NULL; @@ -555,7 +547,7 @@ static PyObject * compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) { PyObject *result; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; _lzma_state *state = PyType_GetModuleState(Py_TYPE(c)); assert(state != NULL); @@ -618,14 +610,14 @@ _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); } else { result = compress(self, data->buf, data->len, LZMA_RUN); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -645,14 +637,14 @@ _lzma_LZMACompressor_flush_impl(Compressor *self) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Repeated call to flush()"); } else { self->flushed = 1; result = compress(self, NULL, 0, LZMA_FINISH); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -806,7 +798,7 @@ Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } - if (preset_obj != Py_None && !uint32_converter(preset_obj, &preset)) { + if (preset_obj != Py_None && !_PyLong_UInt32_Converter(preset_obj, &preset)) { return NULL; } @@ -821,12 +813,7 @@ Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) self->alloc.free = PyLzma_Free; self->lzs.allocator = &self->alloc; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; self->flushed = 0; switch (format) { @@ -868,10 +855,8 @@ static void Compressor_dealloc(PyObject *op) { Compressor *self = Compressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); lzma_end(&self->lzs); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); @@ -883,13 +868,6 @@ static PyMethodDef Compressor_methods[] = { {NULL} }; -static int -Compressor_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - PyDoc_STRVAR(Compressor_doc, "LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)\n" "\n" @@ -923,7 +901,6 @@ static PyType_Slot lzma_compressor_type_slots[] = { {Py_tp_methods, Compressor_methods}, {Py_tp_new, Compressor_new}, {Py_tp_doc, (char *)Compressor_doc}, - {Py_tp_traverse, Compressor_traverse}, {0, 0} }; @@ -949,7 +926,7 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) { PyObject *result; lzma_stream *lzs = &d->lzs; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; _lzma_state *state = PyType_GetModuleState(Py_TYPE(d)); assert(state != NULL); @@ -1126,6 +1103,7 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) } /*[clinic input] +@permit_long_docstring_body _lzma.LZMADecompressor.decompress data: Py_buffer @@ -1150,16 +1128,16 @@ the unused_data attribute. static PyObject * _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=ef4e20ec7122241d input=60c1f135820e309d]*/ +/*[clinic end generated code: output=ef4e20ec7122241d input=d5cbd45801b4b8b0]*/ { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->eof) PyErr_SetString(PyExc_EOFError, "Already at end of stream"); else result = decompress(self, data->buf, data->len, max_length); - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -1226,7 +1204,7 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, "Cannot specify memory limit with FORMAT_RAW"); return NULL; } - if (!uint64_converter(memlimit, &memlimit_)) { + if (!_PyLong_UInt64_Converter(memlimit, &memlimit_)) { return NULL; } } @@ -1252,21 +1230,13 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, self->lzs.allocator = &self->alloc; self->lzs.next_in = NULL; - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } + self->mutex = (PyMutex){0}; self->check = LZMA_CHECK_UNKNOWN; self->needs_input = 1; self->input_buffer = NULL; self->input_buffer_size = 0; - Py_XSETREF(self->unused_data, PyBytes_FromStringAndSize(NULL, 0)); - if (self->unused_data == NULL) { - goto error; - } + Py_XSETREF(self->unused_data, Py_GetConstant(Py_CONSTANT_EMPTY_BYTES)); switch (format) { case FORMAT_AUTO: @@ -1315,26 +1285,18 @@ static void Decompressor_dealloc(PyObject *op) { Decompressor *self = Decompressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); + if(self->input_buffer != NULL) PyMem_Free(self->input_buffer); lzma_end(&self->lzs); Py_CLEAR(self->unused_data); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); } -static int -Decompressor_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static PyMethodDef Decompressor_methods[] = { _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF {NULL} @@ -1369,7 +1331,6 @@ static PyType_Slot lzma_decompressor_type_slots[] = { {Py_tp_methods, Decompressor_methods}, {Py_tp_new, _lzma_LZMADecompressor}, {Py_tp_doc, (char *)_lzma_LZMADecompressor__doc__}, - {Py_tp_traverse, Decompressor_traverse}, {Py_tp_members, Decompressor_members}, {0, 0} }; @@ -1445,7 +1406,7 @@ _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) { lzma_ret lzret; uint32_t encoded_size; - PyObject *result = NULL; + PyBytesWriter *writer = NULL; _lzma_state *state = get_lzma_state(module); assert(state != NULL); @@ -1453,25 +1414,26 @@ _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) if (catch_lzma_error(state, lzret)) goto error; - result = PyBytes_FromStringAndSize(NULL, encoded_size); - if (result == NULL) + writer = PyBytesWriter_Create(encoded_size); + if (writer == NULL) { goto error; + } - lzret = lzma_properties_encode( - &filter, (uint8_t *)PyBytes_AS_STRING(result)); + lzret = lzma_properties_encode(&filter, PyBytesWriter_GetData(writer)); if (catch_lzma_error(state, lzret)) { goto error; } - return result; + return PyBytesWriter_Finish(writer); error: - Py_XDECREF(result); + PyBytesWriter_Discard(writer); return NULL; } /*[clinic input] +@permit_long_summary _lzma._decode_filter_properties filter_id: lzma_vli encoded_props: Py_buffer @@ -1485,7 +1447,7 @@ The result does not include the filter ID itself, only the options. static PyObject * _lzma__decode_filter_properties_impl(PyObject *module, lzma_vli filter_id, Py_buffer *encoded_props) -/*[clinic end generated code: output=714fd2ef565d5c60 input=246410800782160c]*/ +/*[clinic end generated code: output=714fd2ef565d5c60 input=b9750bf909109bbe]*/ { lzma_filter filter; lzma_ret lzret; diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index cee8cf7b9a8..848784dedc1 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -109,23 +109,22 @@ static PyObject * _multiprocessing_recv_impl(PyObject *module, HANDLE handle, int size) /*[clinic end generated code: output=92322781ba9ff598 input=6a5b0834372cee5b]*/ { - int nread; - PyObject *buf; - - buf = PyBytes_FromStringAndSize(NULL, size); - if (!buf) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (!writer) { return NULL; + } + char *buf = PyBytesWriter_GetData(writer); + Py_ssize_t nread; Py_BEGIN_ALLOW_THREADS - nread = recv((SOCKET) handle, PyBytes_AS_STRING(buf), size, 0); + nread = recv((SOCKET) handle, buf, size, 0); Py_END_ALLOW_THREADS if (nread < 0) { - Py_DECREF(buf); + PyBytesWriter_Discard(writer); return PyErr_SetExcFromWindowsErr(PyExc_OSError, WSAGetLastError()); } - _PyBytes_Resize(&buf, nread); - return buf; + return PyBytesWriter_FinishWithSize(writer, nread); } /*[clinic input] diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index a4a2a866ccb..85cc0ac70a6 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -8,6 +8,7 @@ */ #include "multiprocessing.h" +#include "pycore_object.h" // _PyObject_VisitType() #ifdef HAVE_SYS_TIME_H # include <sys/time.h> // gettimeofday() @@ -720,13 +721,6 @@ _multiprocessing_SemLock___exit___impl(SemLockObject *self, return _multiprocessing_SemLock_release_impl(self); } -static int -semlock_traverse(PyObject *s, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(s)); - return 0; -} - /* * Semaphore methods */ @@ -773,7 +767,7 @@ static PyType_Slot _PyMp_SemLockType_slots[] = { {Py_tp_members, semlock_members}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, _multiprocessing_SemLock}, - {Py_tp_traverse, semlock_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_free, PyObject_GC_Del}, {Py_tp_doc, (void *)PyDoc_STR("Semaphore/Mutex type")}, {0, 0}, diff --git a/Modules/_opcode.c b/Modules/_opcode.c index c295f7b3152..ef271b6ef56 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -5,7 +5,7 @@ #include "Python.h" #include "compile.h" #include "opcode.h" -#include "pycore_ceval.h" +#include "pycore_ceval.h" // SPECIAL_MAX #include "pycore_code.h" #include "pycore_compile.h" #include "pycore_intrinsics.h" diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d260f1a68f8..608598eb5a5 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -155,6 +155,9 @@ enum { /* Prefetch size when unpickling (disabled on unpeekable streams) */ PREFETCH = 8192 * 16, + /* Data larger that this will be read in chunks, to prevent extreme + overallocation. */ + MIN_READ_BUF_SIZE = 1 << 20, FRAME_SIZE_MIN = 4, FRAME_SIZE_TARGET = 64 * 1024, @@ -413,13 +416,6 @@ typedef struct { #define Pdata_CAST(op) ((Pdata *)(op)) -static int -Pdata_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static void Pdata_dealloc(PyObject *op) { @@ -437,7 +433,7 @@ Pdata_dealloc(PyObject *op) static PyType_Slot pdata_slots[] = { {Py_tp_dealloc, Pdata_dealloc}, - {Py_tp_traverse, Pdata_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0, NULL}, }; @@ -654,10 +650,11 @@ typedef struct UnpicklerObject { Pdata *stack; /* Pickle data stack, store unpickled objects. */ /* The unpickler memo is just an array of PyObject *s. Using a dict - is unnecessary, since the keys are contiguous ints. */ + is unnecessary, since the keys usually are contiguous ints. */ PyObject **memo; size_t memo_size; /* Capacity of the memo array */ size_t memo_len; /* Number of objects in the memo */ + PyObject *memo_dict; /* The backup memo dict for non-continuous keys. */ PyObject *persistent_load; /* persistent_load() method, can be NULL. */ PyObject *persistent_load_attr; /* instance attribute, can be NULL. */ @@ -1254,141 +1251,13 @@ _Unpickler_SkipConsumed(UnpicklerObject *self) static const Py_ssize_t READ_WHOLE_LINE = -1; -/* If reading from a file, we need to only pull the bytes we need, since there - may be multiple pickle objects arranged contiguously in the same input - buffer. - - If `n` is READ_WHOLE_LINE, read a whole line. Otherwise, read up to `n` - bytes from the input stream/buffer. - - Update the unpickler's input buffer with the newly-read data. Returns -1 on - failure; on success, returns the number of bytes read from the file. - - On success, self->input_len will be 0; this is intentional so that when - unpickling from a file, the "we've run out of data" code paths will trigger, - causing the Unpickler to go back to the file for more data. Use the returned - size to tell you how much data you can process. */ +/* Don't call it directly: use _Unpickler_ReadInto() */ static Py_ssize_t -_Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) -{ - PyObject *data; - Py_ssize_t read_size; - - assert(self->read != NULL); - - if (_Unpickler_SkipConsumed(self) < 0) - return -1; - - if (n == READ_WHOLE_LINE) { - data = PyObject_CallNoArgs(self->readline); - } - else { - PyObject *len; - /* Prefetch some data without advancing the file pointer, if possible */ - if (self->peek && n < PREFETCH) { - len = PyLong_FromSsize_t(PREFETCH); - if (len == NULL) - return -1; - data = _Pickle_FastCall(self->peek, len); - if (data == NULL) { - if (!PyErr_ExceptionMatches(PyExc_NotImplementedError)) - return -1; - /* peek() is probably not supported by the given file object */ - PyErr_Clear(); - Py_CLEAR(self->peek); - } - else { - read_size = _Unpickler_SetStringInput(self, data); - Py_DECREF(data); - if (read_size < 0) { - return -1; - } - - self->prefetched_idx = 0; - if (n <= read_size) - return n; - } - } - len = PyLong_FromSsize_t(n); - if (len == NULL) - return -1; - data = _Pickle_FastCall(self->read, len); - } - if (data == NULL) - return -1; - - read_size = _Unpickler_SetStringInput(self, data); - Py_DECREF(data); - return read_size; -} - -/* Don't call it directly: use _Unpickler_Read() */ -static Py_ssize_t -_Unpickler_ReadImpl(UnpicklerObject *self, PickleState *st, char **s, Py_ssize_t n) -{ - Py_ssize_t num_read; - - *s = NULL; - if (self->next_read_idx > PY_SSIZE_T_MAX - n) { - PyErr_SetString(st->UnpicklingError, - "read would overflow (invalid bytecode)"); - return -1; - } - - /* This case is handled by the _Unpickler_Read() macro for efficiency */ - assert(self->next_read_idx + n > self->input_len); - - if (!self->read) - return bad_readline(st); - - /* Extend the buffer to satisfy desired size */ - num_read = _Unpickler_ReadFromFile(self, n); - if (num_read < 0) - return -1; - if (num_read < n) - return bad_readline(st); - *s = self->input_buffer; - self->next_read_idx = n; - return n; -} - -/* Read `n` bytes from the unpickler's data source, storing the result in `buf`. - * - * This should only be used for non-small data reads where potentially - * avoiding a copy is beneficial. This method does not try to prefetch - * more data into the input buffer. - * - * _Unpickler_Read() is recommended in most cases. - */ -static Py_ssize_t -_Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, - Py_ssize_t n) +_Unpickler_ReadIntoFromFile(PickleState *state, UnpicklerObject *self, char *buf, + Py_ssize_t n) { assert(n != READ_WHOLE_LINE); - /* Read from available buffer data, if any */ - Py_ssize_t in_buffer = self->input_len - self->next_read_idx; - if (in_buffer > 0) { - Py_ssize_t to_read = Py_MIN(in_buffer, n); - memcpy(buf, self->input_buffer + self->next_read_idx, to_read); - self->next_read_idx += to_read; - buf += to_read; - n -= to_read; - if (n == 0) { - /* Entire read was satisfied from buffer */ - return n; - } - } - - /* Read from file */ - if (!self->read) { - /* We're unpickling memory, this means the input is truncated */ - return bad_readline(state); - } - if (_Unpickler_SkipConsumed(self) < 0) { - return -1; - } - if (!self->readinto) { /* readinto() not supported on file-like object, fall back to read() * and copy into destination buffer (bpo-39681) */ @@ -1442,6 +1311,163 @@ _Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, return n; } +/* If reading from a file, we need to only pull the bytes we need, since there + may be multiple pickle objects arranged contiguously in the same input + buffer. + + If `n` is READ_WHOLE_LINE, read a whole line. Otherwise, read up to `n` + bytes from the input stream/buffer. + + Update the unpickler's input buffer with the newly-read data. Returns -1 on + failure; on success, returns the number of bytes read from the file. + + On success, self->input_len will be 0; this is intentional so that when + unpickling from a file, the "we've run out of data" code paths will trigger, + causing the Unpickler to go back to the file for more data. Use the returned + size to tell you how much data you can process. */ +static Py_ssize_t +_Unpickler_ReadFromFile(PickleState *state, UnpicklerObject *self, Py_ssize_t n) +{ + PyObject *data; + Py_ssize_t read_size; + + assert(self->read != NULL); + + if (_Unpickler_SkipConsumed(self) < 0) + return -1; + + if (n == READ_WHOLE_LINE) { + data = PyObject_CallNoArgs(self->readline); + if (data == NULL) { + return -1; + } + } + else { + PyObject *len; + /* Prefetch some data without advancing the file pointer, if possible */ + if (self->peek && n < PREFETCH) { + len = PyLong_FromSsize_t(PREFETCH); + if (len == NULL) + return -1; + data = _Pickle_FastCall(self->peek, len); + if (data == NULL) { + if (!PyErr_ExceptionMatches(PyExc_NotImplementedError)) + return -1; + /* peek() is probably not supported by the given file object */ + PyErr_Clear(); + Py_CLEAR(self->peek); + } + else { + read_size = _Unpickler_SetStringInput(self, data); + Py_DECREF(data); + if (read_size < 0) { + return -1; + } + + self->prefetched_idx = 0; + if (n <= read_size) + return n; + } + } + Py_ssize_t cursize = Py_MIN(n, MIN_READ_BUF_SIZE); + len = PyLong_FromSsize_t(cursize); + if (len == NULL) + return -1; + data = _Pickle_FastCall(self->read, len); + if (data == NULL) { + return -1; + } + while (cursize < n) { + Py_ssize_t prevsize = cursize; + // geometrically double the chunk size to avoid CPU DoS + cursize += Py_MIN(cursize, n - cursize); + if (_PyBytes_Resize(&data, cursize) < 0) { + return -1; + } + if (_Unpickler_ReadIntoFromFile(state, self, + PyBytes_AS_STRING(data) + prevsize, cursize - prevsize) < 0) + { + Py_DECREF(data); + return -1; + } + } + } + + read_size = _Unpickler_SetStringInput(self, data); + Py_DECREF(data); + return read_size; +} + +/* Don't call it directly: use _Unpickler_Read() */ +static Py_ssize_t +_Unpickler_ReadImpl(UnpicklerObject *self, PickleState *st, char **s, Py_ssize_t n) +{ + Py_ssize_t num_read; + + *s = NULL; + if (self->next_read_idx > PY_SSIZE_T_MAX - n) { + PyErr_SetString(st->UnpicklingError, + "read would overflow (invalid bytecode)"); + return -1; + } + + /* This case is handled by the _Unpickler_Read() macro for efficiency */ + assert(self->next_read_idx + n > self->input_len); + + if (!self->read) + return bad_readline(st); + + /* Extend the buffer to satisfy desired size */ + num_read = _Unpickler_ReadFromFile(st, self, n); + if (num_read < 0) + return -1; + if (num_read < n) + return bad_readline(st); + *s = self->input_buffer; + self->next_read_idx = n; + return n; +} + +/* Read `n` bytes from the unpickler's data source, storing the result in `buf`. + * + * This should only be used for non-small data reads where potentially + * avoiding a copy is beneficial. This method does not try to prefetch + * more data into the input buffer. + * + * _Unpickler_Read() is recommended in most cases. + */ +static Py_ssize_t +_Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, + Py_ssize_t n) +{ + assert(n != READ_WHOLE_LINE); + + /* Read from available buffer data, if any */ + Py_ssize_t in_buffer = self->input_len - self->next_read_idx; + if (in_buffer > 0) { + Py_ssize_t to_read = Py_MIN(in_buffer, n); + memcpy(buf, self->input_buffer + self->next_read_idx, to_read); + self->next_read_idx += to_read; + buf += to_read; + n -= to_read; + if (n == 0) { + /* Entire read was satisfied from buffer */ + return n; + } + } + + /* Read from file */ + if (!self->read) { + /* We're unpickling memory, this means the input is truncated */ + return bad_readline(state); + } + if (_Unpickler_SkipConsumed(self) < 0) { + return -1; + } + + return _Unpickler_ReadIntoFromFile(state, self, buf, n); +} + /* Read `n` bytes from the unpickler's data source, storing the result in `*s`. This should be used for all data reads, rather than accessing the unpickler's @@ -1499,7 +1525,7 @@ _Unpickler_Readline(PickleState *state, UnpicklerObject *self, char **result) if (!self->read) return bad_readline(state); - num_read = _Unpickler_ReadFromFile(self, READ_WHOLE_LINE); + num_read = _Unpickler_ReadFromFile(state, self, READ_WHOLE_LINE); if (num_read < 0) return -1; if (num_read == 0 || self->input_buffer[num_read - 1] != '\n') @@ -1532,12 +1558,35 @@ _Unpickler_ResizeMemoList(UnpicklerObject *self, size_t new_size) /* Returns NULL if idx is out of bounds. */ static PyObject * -_Unpickler_MemoGet(UnpicklerObject *self, size_t idx) +_Unpickler_MemoGet(PickleState *st, UnpicklerObject *self, size_t idx) { - if (idx >= self->memo_size) - return NULL; - - return self->memo[idx]; + PyObject *value; + if (idx < self->memo_size) { + value = self->memo[idx]; + if (value != NULL) { + return value; + } + } + if (self->memo_dict != NULL) { + PyObject *key = PyLong_FromSize_t(idx); + if (key == NULL) { + return NULL; + } + if (idx < self->memo_size) { + (void)PyDict_Pop(self->memo_dict, key, &value); + // Migrate dict entry to array for faster future access + self->memo[idx] = value; + } + else { + value = PyDict_GetItemWithError(self->memo_dict, key); + } + Py_DECREF(key); + if (value != NULL || PyErr_Occurred()) { + return value; + } + } + PyErr_Format(st->UnpicklingError, "Memo value not found at index %zd", idx); + return NULL; } /* Returns -1 (with an exception set) on failure, 0 on success. @@ -1548,6 +1597,27 @@ _Unpickler_MemoPut(UnpicklerObject *self, size_t idx, PyObject *value) PyObject *old_item; if (idx >= self->memo_size) { + if (idx > self->memo_len * 2) { + /* The memo keys are too sparse. Use a dict instead of + * a continuous array for the memo. */ + if (self->memo_dict == NULL) { + self->memo_dict = PyDict_New(); + if (self->memo_dict == NULL) { + return -1; + } + } + PyObject *key = PyLong_FromSize_t(idx); + if (key == NULL) { + return -1; + } + + if (PyDict_SetItem(self->memo_dict, key, value) < 0) { + Py_DECREF(key); + return -1; + } + Py_DECREF(key); + return 0; + } if (_Unpickler_ResizeMemoList(self, idx * 2) < 0) return -1; assert(idx < self->memo_size); @@ -1617,6 +1687,7 @@ _Unpickler_New(PyObject *module) self->memo = memo; self->memo_size = MEMO_SIZE; self->memo_len = 0; + self->memo_dict = NULL; self->persistent_load = NULL; self->persistent_load_attr = NULL; memset(&self->buffer, 0, sizeof(Py_buffer)); @@ -1879,6 +1950,10 @@ _checkmodule(PyObject *module_name, PyObject *module, _PyUnicode_EqualToASCIIString(module_name, "__main__")) { return -1; } + if (PyUnicode_Check(module_name) && + _PyUnicode_EqualToASCIIString(module_name, "__mp_main__")) { + return -1; + } PyObject *candidate = getattribute(module, dotted_path, 0); if (candidate == NULL) { @@ -1911,7 +1986,7 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * __module__ can be None. If it is so, then search sys.modules for the module of global. */ Py_CLEAR(module_name); - modules = _PySys_GetRequiredAttr(&_Py_ID(modules)); + modules = PySys_GetAttr(&_Py_ID(modules)); if (modules == NULL) { return NULL; } @@ -2615,31 +2690,26 @@ save_picklebuffer(PickleState *st, PicklerObject *self, PyObject *obj) static PyObject * raw_unicode_escape(PyObject *obj) { - char *p; - Py_ssize_t i, size; - const void *data; - int kind; - _PyBytesWriter writer; + Py_ssize_t size = PyUnicode_GET_LENGTH(obj); + const void *data = PyUnicode_DATA(obj); + int kind = PyUnicode_KIND(obj); - _PyBytesWriter_Init(&writer); + Py_ssize_t alloc = size; + PyBytesWriter *writer = PyBytesWriter_Create(alloc); + if (writer == NULL) { + return NULL; + } + char *p = PyBytesWriter_GetData(writer); - size = PyUnicode_GET_LENGTH(obj); - data = PyUnicode_DATA(obj); - kind = PyUnicode_KIND(obj); - - p = _PyBytesWriter_Alloc(&writer, size); - if (p == NULL) - goto error; - writer.overallocate = 1; - - for (i=0; i < size; i++) { + for (Py_ssize_t i=0; i < size; i++) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { /* -1: subtract 1 preallocated byte */ - p = _PyBytesWriter_Prepare(&writer, p, 10-1); - if (p == NULL) + p = PyBytesWriter_GrowAndUpdatePointer(writer, 10-1, p); + if (p == NULL) { goto error; + } *p++ = '\\'; *p++ = 'U'; @@ -2658,9 +2728,10 @@ raw_unicode_escape(PyObject *obj) ch == 0x1a) { /* -1: subtract 1 preallocated byte */ - p = _PyBytesWriter_Prepare(&writer, p, 6-1); - if (p == NULL) + p = PyBytesWriter_GrowAndUpdatePointer(writer, 6-1, p); + if (p == NULL) { goto error; + } *p++ = '\\'; *p++ = 'u'; @@ -2674,10 +2745,10 @@ raw_unicode_escape(PyObject *obj) *p++ = (char) ch; } - return _PyBytesWriter_Finish(&writer, p); + return PyBytesWriter_FinishWithPointer(writer, p); error: - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return NULL; } @@ -5251,7 +5322,7 @@ load_int(PickleState *state, UnpicklerObject *self) } } else { - if (len == 3 && (x == 0 || x == 1)) { + if (len == 3 && s[0] == '0' && (s[1] == '0' || s[1] == '1')) { if ((value = PyBool_FromLong(x)) == NULL) return -1; } @@ -5539,17 +5610,16 @@ static int load_counted_binstring(PickleState *st, UnpicklerObject *self, int nbytes) { PyObject *obj; - Py_ssize_t size; + long size; char *s; if (_Unpickler_Read(self, st, &s, nbytes) < 0) return -1; - size = calc_binsize(s, nbytes); + size = calc_binint(s, nbytes); if (size < 0) { - PyErr_Format(st->UnpicklingError, - "BINSTRING exceeds system's maximum size of %zd bytes", - PY_SSIZE_T_MAX); + PyErr_SetString(st->UnpicklingError, + "BINSTRING pickle has negative byte count"); return -1; } @@ -5590,13 +5660,28 @@ load_counted_binbytes(PickleState *state, UnpicklerObject *self, int nbytes) return -1; } - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) - return -1; - if (_Unpickler_ReadInto(state, self, PyBytes_AS_STRING(bytes), size) < 0) { - Py_DECREF(bytes); + Py_ssize_t cursize = Py_MIN(size, MIN_READ_BUF_SIZE); + Py_ssize_t prevsize = 0; + bytes = PyBytes_FromStringAndSize(NULL, cursize); + if (bytes == NULL) { return -1; } + while (1) { + if (_Unpickler_ReadInto(state, self, + PyBytes_AS_STRING(bytes) + prevsize, cursize - prevsize) < 0) + { + Py_DECREF(bytes); + return -1; + } + if (cursize >= size) { + break; + } + prevsize = cursize; + cursize += Py_MIN(cursize, size - cursize); + if (_PyBytes_Resize(&bytes, cursize) < 0) { + return -1; + } + } PDATA_PUSH(self->stack, bytes, -1); return 0; @@ -5621,14 +5706,27 @@ load_counted_bytearray(PickleState *state, UnpicklerObject *self) return -1; } - bytearray = PyByteArray_FromStringAndSize(NULL, size); + Py_ssize_t cursize = Py_MIN(size, MIN_READ_BUF_SIZE); + Py_ssize_t prevsize = 0; + bytearray = PyByteArray_FromStringAndSize(NULL, cursize); if (bytearray == NULL) { return -1; } - char *str = PyByteArray_AS_STRING(bytearray); - if (_Unpickler_ReadInto(state, self, str, size) < 0) { - Py_DECREF(bytearray); - return -1; + while (1) { + if (_Unpickler_ReadInto(state, self, + PyByteArray_AS_STRING(bytearray) + prevsize, + cursize - prevsize) < 0) { + Py_DECREF(bytearray); + return -1; + } + if (cursize >= size) { + break; + } + prevsize = cursize; + cursize += Py_MIN(cursize, size - cursize); + if (PyByteArray_Resize(bytearray, cursize) < 0) { + return -1; + } } PDATA_PUSH(self->stack, bytearray, -1); @@ -6230,20 +6328,15 @@ load_get(PickleState *st, UnpicklerObject *self) if (key == NULL) return -1; idx = PyLong_AsSsize_t(key); + Py_DECREF(key); if (idx == -1 && PyErr_Occurred()) { - Py_DECREF(key); return -1; } - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - } - Py_DECREF(key); return -1; } - Py_DECREF(key); PDATA_APPEND(self->stack, value, -1); return 0; @@ -6261,13 +6354,8 @@ load_binget(PickleState *st, UnpicklerObject *self) idx = Py_CHARMASK(s[0]); - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - PyObject *key = PyLong_FromSsize_t(idx); - if (key != NULL) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - Py_DECREF(key); - } return -1; } @@ -6287,13 +6375,8 @@ load_long_binget(PickleState *st, UnpicklerObject *self) idx = calc_binsize(s, 4); - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - PyObject *key = PyLong_FromSsize_t(idx); - if (key != NULL) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - Py_DECREF(key); - } return -1; } @@ -7258,6 +7341,7 @@ Unpickler_clear(PyObject *op) self->buffer.buf = NULL; } + Py_CLEAR(self->memo_dict); _Unpickler_MemoCleanup(self); PyMem_Free(self->marks); self->marks = NULL; @@ -7294,6 +7378,7 @@ Unpickler_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(self->persistent_load); Py_VISIT(self->persistent_load_attr); Py_VISIT(self->buffers); + Py_VISIT(self->memo_dict); PyObject **memo = self->memo; if (memo) { Py_ssize_t i = self->memo_size; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index b542f86b6fe..6f0a6d1d4e3 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -514,7 +514,13 @@ _close_open_fds_maybe_unsafe(int start_fd, int *fds_to_keep, proc_fd_dir = NULL; else #endif +#if defined(_AIX) + char fd_path[PATH_MAX]; + snprintf(fd_path, sizeof(fd_path), "/proc/%ld/fd", (long)getpid()); + proc_fd_dir = opendir(fd_path); +#else proc_fd_dir = opendir(FD_DIR); +#endif if (!proc_fd_dir) { /* No way to get a list of open fds. */ _close_range_except(start_fd, -1, fds_to_keep, fds_to_keep_len, @@ -630,7 +636,7 @@ reset_signal_handlers(const sigset_t *child_sigmask) * (v)fork to set things up and call exec(). * * All of the code in this function must only use async-signal-safe functions, - * listed at `man 7 signal` or + * listed at `man 7 signal-safety` or * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. * * This restriction is documented at @@ -956,6 +962,7 @@ do_fork_exec(char *const exec_array[], } /*[clinic input] +@permit_long_docstring_body _posixsubprocess.fork_exec as subprocess_fork_exec args as process_args: object executable_list: object @@ -1016,7 +1023,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, PyObject *preexec_fn) -/*[clinic end generated code: output=288464dc56e373c7 input=f311c3bcb5dd55c8]*/ +/*[clinic end generated code: output=288464dc56e373c7 input=58e0db771686f4f6]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 3ee14b61b82..01235c77bd7 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -7,6 +7,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_parking_lot.h" #include "pycore_time.h" // _PyTime_FromSecondsObject() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> #include <stddef.h> // offsetof() @@ -221,9 +222,7 @@ simplequeue_dealloc(PyObject *op) PyObject_GC_UnTrack(self); (void)simplequeue_clear(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index d5bac2f5b78..544e636d18f 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -497,34 +497,32 @@ _random_Random_setstate_impl(RandomObject *self, PyObject *state) _random.Random.getrandbits self: self(type="RandomObject *") - k: int + k: uint64 / getrandbits(k) -> x. Generates an int with k random bits. [clinic start generated code]*/ static PyObject * -_random_Random_getrandbits_impl(RandomObject *self, int k) -/*[clinic end generated code: output=b402f82a2158887f input=87603cd60f79f730]*/ +_random_Random_getrandbits_impl(RandomObject *self, uint64_t k) +/*[clinic end generated code: output=c30ef8435f3433cf input=64226ac13bb4d2a3]*/ { - int i, words; + Py_ssize_t i, words; uint32_t r; uint32_t *wordarray; PyObject *result; - if (k < 0) { - PyErr_SetString(PyExc_ValueError, - "number of bits must be non-negative"); - return NULL; - } - if (k == 0) return PyLong_FromLong(0); if (k <= 32) /* Fast path */ return PyLong_FromUnsignedLong(genrand_uint32(self) >> (32 - k)); - words = (k - 1) / 32 + 1; + if ((k - 1u) / 32u + 1u > PY_SSIZE_T_MAX / 4u) { + PyErr_NoMemory(); + return NULL; + } + words = (Py_ssize_t)((k - 1u) / 32u + 1u); wordarray = (uint32_t *)PyMem_Malloc(words * 4); if (wordarray == NULL) { PyErr_NoMemory(); @@ -597,11 +595,14 @@ static PyType_Slot Random_Type_slots[] = { }; static PyType_Spec Random_Type_spec = { - "_random.Random", - sizeof(RandomObject), - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - Random_Type_slots + .name = "_random.Random", + .basicsize = sizeof(RandomObject), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = Random_Type_slots }; PyDoc_STRVAR(module_doc, diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h new file mode 100644 index 00000000000..7f3c0d363f5 --- /dev/null +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -0,0 +1,574 @@ +/****************************************************************************** + * Python Remote Debugging Module - Shared Header + * + * This header provides common declarations, types, and utilities shared + * across the remote debugging module implementation files. + ******************************************************************************/ + +#ifndef Py_REMOTE_DEBUGGING_H +#define Py_REMOTE_DEBUGGING_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define _GNU_SOURCE + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include <internal/pycore_debug_offsets.h> // _Py_DebugOffsets +#include <internal/pycore_frame.h> // FRAME_SUSPENDED_YIELD_FROM +#include <internal/pycore_interpframe.h> // FRAME_OWNED_BY_INTERPRETER +#include <internal/pycore_llist.h> // struct llist_node +#include <internal/pycore_long.h> // _PyLong_GetZero +#include <internal/pycore_stackref.h> // Py_TAG_BITS +#include "../../Python/remote_debug.h" + +#include <assert.h> +#include <errno.h> +#include <fcntl.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifndef HAVE_PROCESS_VM_READV +# define HAVE_PROCESS_VM_READV 0 +#endif + +#if defined(__APPLE__) && TARGET_OS_OSX +#include <libproc.h> +#include <sys/types.h> +#define MAX_NATIVE_THREADS 4096 +#endif + +#ifdef MS_WINDOWS +#include <windows.h> +#include <winternl.h> +#define STATUS_SUCCESS ((NTSTATUS)0x00000000L) +#define STATUS_INFO_LENGTH_MISMATCH ((NTSTATUS)0xC0000004L) +typedef enum _WIN32_THREADSTATE { + WIN32_THREADSTATE_INITIALIZED = 0, + WIN32_THREADSTATE_READY = 1, + WIN32_THREADSTATE_RUNNING = 2, + WIN32_THREADSTATE_STANDBY = 3, + WIN32_THREADSTATE_TERMINATED = 4, + WIN32_THREADSTATE_WAITING = 5, + WIN32_THREADSTATE_TRANSITION = 6, + WIN32_THREADSTATE_UNKNOWN = 7 +} WIN32_THREADSTATE; +#endif + +/* ============================================================================ + * MACROS AND CONSTANTS + * ============================================================================ */ + +#define GET_MEMBER(type, obj, offset) (*(type*)((char*)(obj) + (offset))) +#define CLEAR_PTR_TAG(ptr) (((uintptr_t)(ptr) & ~Py_TAG_BITS)) +#define GET_MEMBER_NO_TAG(type, obj, offset) (type)(CLEAR_PTR_TAG(*(type*)((char*)(obj) + (offset)))) + +/* Size macros for opaque buffers */ +#define SIZEOF_BYTES_OBJ sizeof(PyBytesObject) +#define SIZEOF_CODE_OBJ sizeof(PyCodeObject) +#define SIZEOF_GEN_OBJ sizeof(PyGenObject) +#define SIZEOF_INTERP_FRAME sizeof(_PyInterpreterFrame) +#define SIZEOF_LLIST_NODE sizeof(struct llist_node) +#define SIZEOF_PAGE_CACHE_ENTRY sizeof(page_cache_entry_t) +#define SIZEOF_PYOBJECT sizeof(PyObject) +#define SIZEOF_SET_OBJ sizeof(PySetObject) +#define SIZEOF_TASK_OBJ 4096 +#define SIZEOF_THREAD_STATE sizeof(PyThreadState) +#define SIZEOF_TYPE_OBJ sizeof(PyTypeObject) +#define SIZEOF_UNICODE_OBJ sizeof(PyUnicodeObject) +#define SIZEOF_LONG_OBJ sizeof(PyLongObject) +#define SIZEOF_GC_RUNTIME_STATE sizeof(struct _gc_runtime_state) +#define SIZEOF_INTERPRETER_STATE sizeof(PyInterpreterState) + +#ifndef MAX +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif + +#ifdef Py_GIL_DISABLED +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)), \ + offsetof(PyInterpreterState, gc.frame) + sizeof(_PyInterpreterFrame *)) +#else +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)), \ + offsetof(PyInterpreterState, gc.frame) + sizeof(_PyInterpreterFrame *)) +#endif +#define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) + +#define MAX_TLBC_SIZE 2048 + +/* Thread status flags */ +#define THREAD_STATUS_HAS_GIL (1 << 0) +#define THREAD_STATUS_ON_CPU (1 << 1) +#define THREAD_STATUS_UNKNOWN (1 << 2) +#define THREAD_STATUS_GIL_REQUESTED (1 << 3) + +/* Exception cause macro */ +#define set_exception_cause(unwinder, exc_type, message) \ + if (unwinder->debug) { \ + _set_debug_exception_cause(exc_type, message); \ + } + +/* ============================================================================ + * TYPE DEFINITIONS + * ============================================================================ */ + +struct _Py_AsyncioModuleDebugOffsets { + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + uint64_t task_node; + } asyncio_task_object; + struct _asyncio_interpreter_state { + uint64_t size; + uint64_t asyncio_tasks_head; + } asyncio_interpreter_state; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + uint64_t asyncio_tasks_head; + } asyncio_thread_state; +}; + +typedef struct { + PyObject *func_name; + PyObject *file_name; + int first_lineno; + PyObject *linetable; // bytes + uintptr_t addr_code_adaptive; +} CachedCodeMetadata; + +/* Frame cache constants and types */ +#define FRAME_CACHE_MAX_THREADS 32 +#define FRAME_CACHE_MAX_FRAMES 1024 + +typedef struct { + uint64_t thread_id; // 0 = empty slot + uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; + Py_ssize_t num_addrs; + PyObject *frame_list; // owned reference, NULL if empty +} FrameCacheEntry; + +/* Statistics for profiling performance analysis */ +typedef struct { + uint64_t total_samples; // Total number of get_stack_trace calls + uint64_t frame_cache_hits; // Full cache hits (entire stack unchanged) + uint64_t frame_cache_misses; // Cache misses requiring full walk + uint64_t frame_cache_partial_hits; // Partial hits (stopped at cached frame) + uint64_t frames_read_from_cache; // Total frames retrieved from cache + uint64_t frames_read_from_memory; // Total frames read from remote memory + uint64_t memory_reads; // Total remote memory read operations + uint64_t memory_bytes_read; // Total bytes read from remote memory + uint64_t code_object_cache_hits; // Code object cache hits + uint64_t code_object_cache_misses; // Code object cache misses + uint64_t stale_cache_invalidations; // Times stale entries were cleared +} UnwinderStats; + +/* Stats tracking macros - no-op when stats collection is disabled */ +#define STATS_INC(unwinder, field) \ + do { if ((unwinder)->collect_stats) (unwinder)->stats.field++; } while(0) + +#define STATS_ADD(unwinder, field, val) \ + do { if ((unwinder)->collect_stats) (unwinder)->stats.field += (val); } while(0) + +typedef struct { + PyTypeObject *RemoteDebugging_Type; + PyTypeObject *TaskInfo_Type; + PyTypeObject *FrameInfo_Type; + PyTypeObject *CoroInfo_Type; + PyTypeObject *ThreadInfo_Type; + PyTypeObject *InterpreterInfo_Type; + PyTypeObject *AwaitedInfo_Type; +} RemoteDebuggingState; + +enum _ThreadState { + THREAD_STATE_RUNNING, + THREAD_STATE_IDLE, + THREAD_STATE_GIL_WAIT, + THREAD_STATE_UNKNOWN +}; + +enum _ProfilingMode { + PROFILING_MODE_WALL = 0, + PROFILING_MODE_CPU = 1, + PROFILING_MODE_GIL = 2, + PROFILING_MODE_ALL = 3 +}; + +typedef struct { + PyObject_HEAD + proc_handle_t handle; + uintptr_t runtime_start_address; + struct _Py_DebugOffsets debug_offsets; + int async_debug_offsets_available; + struct _Py_AsyncioModuleDebugOffsets async_debug_offsets; + uintptr_t interpreter_addr; + uintptr_t tstate_addr; + uint64_t code_object_generation; + _Py_hashtable_t *code_object_cache; + int debug; + int only_active_thread; + int mode; + int skip_non_matching_threads; + int native; + int gc; + int cache_frames; + int collect_stats; // whether to collect statistics + uint32_t stale_invalidation_counter; // counter for throttling frame_cache_invalidate_stale + RemoteDebuggingState *cached_state; + FrameCacheEntry *frame_cache; // preallocated array of FRAME_CACHE_MAX_THREADS entries + UnwinderStats stats; // statistics for performance analysis +#ifdef Py_GIL_DISABLED + uint32_t tlbc_generation; + _Py_hashtable_t *tlbc_cache; +#endif +#ifdef __APPLE__ + uint64_t thread_id_offset; +#endif +#ifdef MS_WINDOWS + PVOID win_process_buffer; + ULONG win_process_buffer_size; +#endif +} RemoteUnwinderObject; + +#define RemoteUnwinder_CAST(op) ((RemoteUnwinderObject *)(op)) + +typedef struct { + int lineno; + int end_lineno; + int column; + int end_column; +} LocationInfo; + +typedef struct { + uintptr_t remote_addr; + size_t size; + void *local_copy; +} StackChunkInfo; + +typedef struct { + StackChunkInfo *chunks; + size_t count; +} StackChunkList; + +/* Function pointer types for iteration callbacks */ +typedef int (*thread_processor_func)( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +); + +typedef int (*set_entry_processor_func)( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +); + +/* ============================================================================ + * STRUCTSEQ DESCRIPTORS (extern declarations) + * ============================================================================ */ + +extern PyStructSequence_Desc TaskInfo_desc; +extern PyStructSequence_Desc FrameInfo_desc; +extern PyStructSequence_Desc CoroInfo_desc; +extern PyStructSequence_Desc ThreadInfo_desc; +extern PyStructSequence_Desc InterpreterInfo_desc; +extern PyStructSequence_Desc AwaitedInfo_desc; + +/* ============================================================================ + * UTILITY FUNCTION DECLARATIONS + * ============================================================================ */ + +/* State access functions */ +extern RemoteDebuggingState *RemoteDebugging_GetState(PyObject *module); +extern RemoteDebuggingState *RemoteDebugging_GetStateFromType(PyTypeObject *type); +extern RemoteDebuggingState *RemoteDebugging_GetStateFromObject(PyObject *obj); +extern int RemoteDebugging_InitState(RemoteDebuggingState *st); + +/* Cache management */ +extern void cached_code_metadata_destroy(void *ptr); + +/* Validation */ +extern int is_prerelease_version(uint64_t version); +extern int validate_debug_offsets(struct _Py_DebugOffsets *debug_offsets); + +/* ============================================================================ + * MEMORY READING FUNCTION DECLARATIONS + * ============================================================================ */ + +extern int read_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *result); +extern int read_Py_ssize_t(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t *result); +extern int read_char(RemoteUnwinderObject *unwinder, uintptr_t address, char *result); +extern int read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr); + +/* Python object reading */ +extern PyObject *read_py_str(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t max_len); +extern PyObject *read_py_bytes(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t max_len); +extern long read_py_long(RemoteUnwinderObject *unwinder, uintptr_t address); + +/* ============================================================================ + * CODE OBJECT FUNCTION DECLARATIONS + * ============================================================================ */ + +extern int parse_code_object( + RemoteUnwinderObject *unwinder, + PyObject **result, + uintptr_t address, + uintptr_t instruction_pointer, + uintptr_t *previous_frame, + int32_t tlbc_index +); + +extern PyObject *make_frame_info( + RemoteUnwinderObject *unwinder, + PyObject *file, + PyObject *line, + PyObject *func +); + +/* Line table parsing */ +extern bool parse_linetable( + const uintptr_t addrq, + const char* linetable, + int firstlineno, + LocationInfo* info +); + +/* TLBC cache (only for Py_GIL_DISABLED) */ +#ifdef Py_GIL_DISABLED +typedef struct { + void *tlbc_array; + Py_ssize_t tlbc_array_size; + uint32_t generation; +} TLBCCacheEntry; + +extern void tlbc_cache_entry_destroy(void *ptr); +extern TLBCCacheEntry *get_tlbc_cache_entry(RemoteUnwinderObject *self, uintptr_t code_addr, uint32_t current_generation); +extern int cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t tlbc_array_addr, uint32_t generation); +#endif + +/* ============================================================================ + * FRAME FUNCTION DECLARATIONS + * ============================================================================ */ + +extern int is_frame_valid( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + uintptr_t code_object_addr +); + +extern int parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* address_of_code_object, + uintptr_t* previous_frame +); + +extern int parse_frame_from_chunks( + RemoteUnwinderObject *unwinder, + PyObject **result, + uintptr_t address, + uintptr_t *previous_frame, + uintptr_t *stackpointer, + StackChunkList *chunks +); + +/* Stack chunk management */ +extern void cleanup_stack_chunks(StackChunkList *chunks); +extern int copy_stack_chunks(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr, StackChunkList *out_chunks); +extern void *find_frame_in_chunks(StackChunkList *chunks, uintptr_t remote_ptr); + +extern int process_frame_chain( + RemoteUnwinderObject *unwinder, + uintptr_t initial_frame_addr, + StackChunkList *chunks, + PyObject *frame_info, + uintptr_t base_frame_addr, + uintptr_t gc_frame, + uintptr_t last_profiled_frame, + int *stopped_at_cached_frame, + uintptr_t *frame_addrs, + Py_ssize_t *num_addrs, + Py_ssize_t max_addrs +); + +/* Frame cache functions */ +extern int frame_cache_init(RemoteUnwinderObject *unwinder); +extern void frame_cache_cleanup(RemoteUnwinderObject *unwinder); +extern FrameCacheEntry *frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id); +extern int clear_last_profiled_frames(RemoteUnwinderObject *unwinder); +extern void frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result); +extern int frame_cache_lookup_and_extend( + RemoteUnwinderObject *unwinder, + uint64_t thread_id, + uintptr_t last_profiled_frame, + PyObject *frame_info, + uintptr_t *frame_addrs, + Py_ssize_t *num_addrs, + Py_ssize_t max_addrs); +// Returns: 1 = stored, 0 = not stored (graceful), -1 = error +extern int frame_cache_store( + RemoteUnwinderObject *unwinder, + uint64_t thread_id, + PyObject *frame_list, + const uintptr_t *addrs, + Py_ssize_t num_addrs); + +extern int collect_frames_with_cache( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + StackChunkList *chunks, + PyObject *frame_info, + uintptr_t gc_frame, + uintptr_t last_profiled_frame, + uint64_t thread_id); + +/* ============================================================================ + * THREAD FUNCTION DECLARATIONS + * ============================================================================ */ + +extern int iterate_threads( + RemoteUnwinderObject *unwinder, + thread_processor_func processor, + void *context +); + +extern int populate_initial_state_data( + int all_threads, + RemoteUnwinderObject *unwinder, + uintptr_t runtime_start_address, + uintptr_t *interpreter_state, + uintptr_t *tstate +); + +extern int find_running_frame( + RemoteUnwinderObject *unwinder, + uintptr_t address_of_thread, + uintptr_t *frame +); + +extern int get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread_id); + +extern PyObject* unwind_stack_for_thread( + RemoteUnwinderObject *unwinder, + uintptr_t *current_tstate, + uintptr_t gil_holder_tstate, + uintptr_t gc_frame +); + +/* ============================================================================ + * ASYNCIO FUNCTION DECLARATIONS + * ============================================================================ */ + +extern uintptr_t _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle); +extern int read_async_debug(RemoteUnwinderObject *unwinder); +extern int ensure_async_debug_offsets(RemoteUnwinderObject *unwinder); + +/* Task parsing */ +extern PyObject *parse_task_name(RemoteUnwinderObject *unwinder, uintptr_t task_address); + +extern int parse_task( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *render_to +); + +extern int parse_coro_chain( + RemoteUnwinderObject *unwinder, + uintptr_t coro_address, + PyObject *render_to +); + +extern int parse_async_frame_chain( + RemoteUnwinderObject *unwinder, + PyObject *calls, + uintptr_t address_of_thread, + uintptr_t running_task_code_obj +); + +/* Set iteration */ +extern int iterate_set_entries( + RemoteUnwinderObject *unwinder, + uintptr_t set_addr, + set_entry_processor_func processor, + void *context +); + +/* Task awaited_by processing */ +extern int process_task_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + set_entry_processor_func processor, + void *context +); + +extern int process_single_task_node( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + PyObject **task_info, + PyObject *result +); + +extern int process_task_and_waiters( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + PyObject *result +); + +extern int find_running_task_in_thread( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + uintptr_t *running_task_addr +); + +extern int get_task_code_object( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + uintptr_t *code_obj_addr +); + +extern int append_awaited_by( + RemoteUnwinderObject *unwinder, + unsigned long tid, + uintptr_t head_addr, + PyObject *result +); + +/* Thread processors for asyncio */ +extern int process_thread_for_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +); + +extern int process_thread_for_async_stack_trace( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +); + +#ifdef __cplusplus +} +#endif + +#endif /* Py_REMOTE_DEBUGGING_H */ diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c new file mode 100644 index 00000000000..7f91f16e3a2 --- /dev/null +++ b/Modules/_remote_debugging/asyncio.c @@ -0,0 +1,1014 @@ +/****************************************************************************** + * Remote Debugging Module - Asyncio Functions + * + * This file contains functions for parsing asyncio tasks, coroutines, + * and awaited_by relationships from remote process memory. + ******************************************************************************/ + +#include "_remote_debugging.h" + +/* ============================================================================ + * ASYNCIO DEBUG ADDRESS FUNCTIONS + * ============================================================================ */ + +uintptr_t +_Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) +{ + uintptr_t address; + +#ifdef MS_WINDOWS + // On Windows, search for asyncio debug in executable or DLL + address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio"); + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#elif defined(__linux__) && HAVE_PROCESS_VM_READV + // On Linux, search for asyncio debug in executable or DLL + address = search_linux_map_for_section(handle, "AsyncioDebug", "python"); + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#elif defined(__APPLE__) && TARGET_OS_OSX + // On macOS, try libpython first, then fall back to python + address = search_map_for_section(handle, "AsyncioDebug", "libpython"); + if (address == 0) { + PyErr_Clear(); + address = search_map_for_section(handle, "AsyncioDebug", "python"); + } + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#else + Py_UNREACHABLE(); +#endif + + return address; +} + +int +read_async_debug(RemoteUnwinderObject *unwinder) +{ + uintptr_t async_debug_addr = _Py_RemoteDebug_GetAsyncioDebugAddress(&unwinder->handle); + if (!async_debug_addr) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to get AsyncioDebug address"); + return -1; + } + + size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); + int result = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, async_debug_addr, size, &unwinder->async_debug_offsets); + if (result < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read AsyncioDebug offsets"); + } + return result; +} + +int +ensure_async_debug_offsets(RemoteUnwinderObject *unwinder) +{ + // If already available, nothing to do + if (unwinder->async_debug_offsets_available) { + return 0; + } + + // Try to load async debug offsets (the target process may have + // loaded asyncio since we last checked) + if (read_async_debug(unwinder) < 0) { + PyErr_Clear(); + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "AsyncioDebug section unavailable - asyncio module may not be loaded in target process"); + return -1; + } + + unwinder->async_debug_offsets_available = 1; + return 0; +} + +/* ============================================================================ + * SET ITERATION FUNCTIONS + * ============================================================================ */ + +int +iterate_set_entries( + RemoteUnwinderObject *unwinder, + uintptr_t set_addr, + set_entry_processor_func processor, + void *context +) { + char set_object[SIZEOF_SET_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, set_addr, + SIZEOF_SET_OBJ, set_object) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set object"); + return -1; + } + + Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); + Py_ssize_t set_len = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask) + 1; + uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + + Py_ssize_t i = 0; + Py_ssize_t els = 0; + while (i < set_len && els < num_els) { + uintptr_t key_addr; + if (read_py_ptr(unwinder, table_ptr, &key_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry key"); + return -1; + } + + if ((void*)key_addr != NULL) { + Py_ssize_t ref_cnt; + if (read_Py_ssize_t(unwinder, table_ptr, &ref_cnt) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry ref count"); + return -1; + } + + if (ref_cnt) { + // Process this valid set entry + if (processor(unwinder, key_addr, context) < 0) { + return -1; + } + els++; + } + } + table_ptr += sizeof(void*) * 2; + i++; + } + + return 0; +} + +/* ============================================================================ + * TASK NAME PARSING + * ============================================================================ */ + +PyObject * +parse_task_name( + RemoteUnwinderObject *unwinder, + uintptr_t task_address +) { + // Read the entire TaskObj at once + char task_obj[SIZEOF_TASK_OBJ]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + task_address, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); + return NULL; + } + + uintptr_t task_name_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_name); + + // The task name can be a long or a string so we need to check the type + char task_name_obj[SIZEOF_PYOBJECT]; + err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + task_name_addr, + SIZEOF_PYOBJECT, + task_name_obj); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task name object"); + return NULL; + } + + // Now read the type object to get the flags + char type_obj[SIZEOF_TYPE_OBJ]; + err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + GET_MEMBER(uintptr_t, task_name_obj, unwinder->debug_offsets.pyobject.ob_type), + SIZEOF_TYPE_OBJ, + type_obj); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task name type object"); + return NULL; + } + + if ((GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_LONG_SUBCLASS)) { + long res = read_py_long(unwinder, task_name_addr); + if (res == -1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Task name PyLong parsing failed"); + return NULL; + } + return PyUnicode_FromFormat("Task-%d", res); + } + + if(!(GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_UNICODE_SUBCLASS)) { + PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Task name object is neither long nor unicode"); + return NULL; + } + + return read_py_str( + unwinder, + task_name_addr, + 255 + ); +} + +/* ============================================================================ + * COROUTINE CHAIN PARSING + * ============================================================================ */ + +static int +handle_yield_from_frame( + RemoteUnwinderObject *unwinder, + uintptr_t gi_iframe_addr, + uintptr_t gen_type_addr, + PyObject *render_to +) { + // Read the entire interpreter frame at once + char iframe[SIZEOF_INTERP_FRAME]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + gi_iframe_addr, + SIZEOF_INTERP_FRAME, + iframe); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame in yield_from handler"); + return -1; + } + + if (GET_MEMBER(char, iframe, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_GENERATOR) { + PyErr_SetString( + PyExc_RuntimeError, + "generator doesn't own its frame \\_o_/"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame ownership mismatch in yield_from"); + return -1; + } + + uintptr_t stackpointer_addr = GET_MEMBER_NO_TAG(uintptr_t, iframe, unwinder->debug_offsets.interpreter_frame.stackpointer); + + if ((void*)stackpointer_addr != NULL) { + uintptr_t gi_await_addr; + err = read_py_ptr( + unwinder, + stackpointer_addr - sizeof(void*), + &gi_await_addr); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await address"); + return -1; + } + + if ((void*)gi_await_addr != NULL) { + uintptr_t gi_await_addr_type_addr; + err = read_ptr( + unwinder, + gi_await_addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type, + &gi_await_addr_type_addr); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await type address"); + return -1; + } + + if (gen_type_addr == gi_await_addr_type_addr) { + /* This needs an explanation. We always start with parsing + native coroutine / generator frames. Ultimately they + are awaiting on something. That something can be + a native coroutine frame or... an iterator. + If it's the latter -- we can't continue building + our chain. So the condition to bail out of this is + to do that when the type of the current coroutine + doesn't match the type of whatever it points to + in its cr_await. + */ + err = parse_coro_chain(unwinder, gi_await_addr, render_to); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse coroutine chain in yield_from"); + return -1; + } + } + } + } + + return 0; +} + +int +parse_coro_chain( + RemoteUnwinderObject *unwinder, + uintptr_t coro_address, + PyObject *render_to +) { + assert((void*)coro_address != NULL); + + // Read the entire generator object at once + char gen_object[SIZEOF_GEN_OBJ]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + coro_address, + SIZEOF_GEN_OBJ, + gen_object); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read generator object in coro chain"); + return -1; + } + + int8_t frame_state = GET_MEMBER(int8_t, gen_object, unwinder->debug_offsets.gen_object.gi_frame_state); + if (frame_state == FRAME_CLEARED) { + return 0; + } + + uintptr_t gen_type_addr = GET_MEMBER(uintptr_t, gen_object, unwinder->debug_offsets.pyobject.ob_type); + + PyObject* name = NULL; + + // Parse the previous frame using the gi_iframe from local copy + uintptr_t prev_frame; + uintptr_t gi_iframe_addr = coro_address + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe; + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); + return -1; + } + + if (!name) { + return 0; + } + + if (PyList_Append(render_to, name)) { + Py_DECREF(name); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to coro chain"); + return -1; + } + Py_DECREF(name); + + if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { + return handle_yield_from_frame(unwinder, gi_iframe_addr, gen_type_addr, render_to); + } + + return 0; +} + +/* ============================================================================ + * TASK PARSING FUNCTIONS + * ============================================================================ */ + +static PyObject* +create_task_result( + RemoteUnwinderObject *unwinder, + uintptr_t task_address +) { + PyObject* result = NULL; + PyObject *call_stack = NULL; + PyObject *tn = NULL; + char task_obj[SIZEOF_TASK_OBJ]; + uintptr_t coro_addr; + + // Create call_stack first since it's the first tuple element + call_stack = PyList_New(0); + if (call_stack == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create call stack list"); + goto error; + } + + // Create task name/address for second tuple element + tn = PyLong_FromUnsignedLongLong(task_address); + if (tn == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task name/address"); + goto error; + } + + // Parse coroutine chain + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object for coro chain"); + goto error; + } + + coro_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_coro); + + if ((void*)coro_addr != NULL) { + if (parse_coro_chain(unwinder, coro_addr, call_stack) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse coroutine chain"); + goto error; + } + + if (PyList_Reverse(call_stack)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to reverse call stack"); + goto error; + } + } + + // Create final CoroInfo result + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result = PyStructSequence_New(state->CoroInfo_Type); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create CoroInfo"); + goto error; + } + + // PyStructSequence_SetItem steals references, so we don't need to DECREF on success + PyStructSequence_SetItem(result, 0, call_stack); // This steals the reference + PyStructSequence_SetItem(result, 1, tn); // This steals the reference + + return result; + +error: + Py_XDECREF(result); + Py_XDECREF(call_stack); + Py_XDECREF(tn); + return NULL; +} + +int +parse_task( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *render_to +) { + char is_task; + PyObject* result = NULL; + int err; + + err = read_char( + unwinder, + task_address + (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + &is_task); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read is_task flag"); + goto error; + } + + if (is_task) { + result = create_task_result(unwinder, task_address); + if (!result) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task result"); + goto error; + } + } else { + // Create an empty CoroInfo for non-task objects + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result = PyStructSequence_New(state->CoroInfo_Type); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create empty CoroInfo"); + goto error; + } + PyObject *empty_list = PyList_New(0); + if (empty_list == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create empty list"); + goto error; + } + PyObject *task_name = PyLong_FromUnsignedLongLong(task_address); + if (task_name == NULL) { + Py_DECREF(empty_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task name"); + goto error; + } + PyStructSequence_SetItem(result, 0, empty_list); // This steals the reference + PyStructSequence_SetItem(result, 1, task_name); // This steals the reference + } + if (PyList_Append(render_to, result)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append task result to render list"); + goto error; + } + + Py_DECREF(result); + return 0; + +error: + Py_XDECREF(result); + return -1; +} + +/* ============================================================================ + * TASK AWAITED_BY PROCESSING + * ============================================================================ */ + +// Forward declaration for mutual recursion +static int process_waiter_task(RemoteUnwinderObject *unwinder, uintptr_t key_addr, void *context); + +// Processor function for parsing tasks in sets +static int +process_task_parser( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +) { + PyObject *awaited_by = (PyObject *)context; + return parse_task(unwinder, key_addr, awaited_by); +} + +static int +parse_task_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *awaited_by +) { + return process_task_awaited_by(unwinder, task_address, process_task_parser, awaited_by); +} + +int +process_task_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + set_entry_processor_func processor, + void *context +) { + // Read the entire TaskObj at once + char task_obj[SIZEOF_TASK_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); + return -1; + } + + uintptr_t task_ab_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by); + if ((void*)task_ab_addr == NULL) { + return 0; // No tasks waiting for this one + } + + char awaited_by_is_a_set = GET_MEMBER(char, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set); + + if (awaited_by_is_a_set) { + return iterate_set_entries(unwinder, task_ab_addr, processor, context); + } else { + // Single task waiting + return processor(unwinder, task_ab_addr, context); + } +} + +int +process_single_task_node( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + PyObject **task_info, + PyObject *result +) { + PyObject *tn = NULL; + PyObject *current_awaited_by = NULL; + PyObject *task_id = NULL; + PyObject *result_item = NULL; + PyObject *coroutine_stack = NULL; + + tn = parse_task_name(unwinder, task_addr); + if (tn == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node"); + goto error; + } + + current_awaited_by = PyList_New(0); + if (current_awaited_by == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by list in single task node"); + goto error; + } + + // Extract the coroutine stack for this task + coroutine_stack = PyList_New(0); + if (coroutine_stack == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create coroutine stack list in single task node"); + goto error; + } + + if (parse_task(unwinder, task_addr, coroutine_stack) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task coroutine stack in single task node"); + goto error; + } + + task_id = PyLong_FromUnsignedLongLong(task_addr); + if (task_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task ID in single task node"); + goto error; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result_item = PyStructSequence_New(state->TaskInfo_Type); + if (result_item == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create TaskInfo in single task node"); + goto error; + } + + PyStructSequence_SetItem(result_item, 0, task_id); // steals ref + PyStructSequence_SetItem(result_item, 1, tn); // steals ref + PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref + PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref + + // References transferred to tuple + task_id = NULL; + tn = NULL; + coroutine_stack = NULL; + current_awaited_by = NULL; + + if (PyList_Append(result, result_item)) { + Py_DECREF(result_item); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append result item in single task node"); + return -1; + } + if (task_info != NULL) { + *task_info = result_item; + } + Py_DECREF(result_item); + + // Get back current_awaited_by reference for parse_task_awaited_by + current_awaited_by = PyStructSequence_GetItem(result_item, 3); + if (parse_task_awaited_by(unwinder, task_addr, current_awaited_by) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse awaited_by in single task node"); + // No cleanup needed here since all references were transferred to result_item + // and result_item was already added to result list and decreffed + return -1; + } + + return 0; + +error: + Py_XDECREF(tn); + Py_XDECREF(current_awaited_by); + Py_XDECREF(task_id); + Py_XDECREF(result_item); + Py_XDECREF(coroutine_stack); + return -1; +} + +int +process_task_and_waiters( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + PyObject *result +) { + // First, add this task to the result + if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { + return -1; + } + + // Now find all tasks that are waiting for this task and process them + return process_task_awaited_by(unwinder, task_addr, process_waiter_task, result); +} + +// Processor function for task waiters +static int +process_waiter_task( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +) { + PyObject *result = (PyObject *)context; + return process_task_and_waiters(unwinder, key_addr, result); +} + +/* ============================================================================ + * RUNNING TASK FUNCTIONS + * ============================================================================ */ + +int +find_running_task_in_thread( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + uintptr_t *running_task_addr +) { + *running_task_addr = (uintptr_t)NULL; + + uintptr_t address_of_running_loop; + int bytes_read = read_py_ptr( + unwinder, + thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, + &address_of_running_loop); + if (bytes_read == -1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running loop address"); + return -1; + } + + // no asyncio loop is now running + if ((void*)address_of_running_loop == NULL) { + return 0; + } + + int err = read_ptr( + unwinder, + thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, + running_task_addr); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task address"); + return -1; + } + + return 0; +} + +int +get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *code_obj_addr) { + uintptr_t running_coro_addr = 0; + + if(read_py_ptr( + unwinder, + task_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_coro, + &running_coro_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro read failed"); + return -1; + } + + if (running_coro_addr == 0) { + PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro address is NULL"); + return -1; + } + + // note: genobject's gi_iframe is an embedded struct so the address to + // the offset leads directly to its first field: f_executable + if (read_py_ptr( + unwinder, + running_coro_addr + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); + return -1; + } + + if (*code_obj_addr == 0) { + PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task code object address is NULL"); + return -1; + } + + return 0; +} + +/* ============================================================================ + * ASYNC FRAME CHAIN PARSING + * ============================================================================ */ + +int +parse_async_frame_chain( + RemoteUnwinderObject *unwinder, + PyObject *calls, + uintptr_t address_of_thread, + uintptr_t running_task_code_obj +) { + uintptr_t address_of_current_frame; + if (find_running_frame(unwinder, address_of_thread, &address_of_current_frame) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Running frame search failed in async chain"); + return -1; + } + + while ((void*)address_of_current_frame != NULL) { + PyObject* frame_info = NULL; + uintptr_t address_of_code_object; + int res = parse_frame_object( + unwinder, + &frame_info, + address_of_current_frame, + &address_of_code_object, + &address_of_current_frame + ); + + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Async frame object parsing failed in chain"); + return -1; + } + + if (!frame_info) { + continue; + } + + if (PyList_Append(calls, frame_info) == -1) { + Py_DECREF(frame_info); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame info to async chain"); + return -1; + } + + Py_DECREF(frame_info); + + if (address_of_code_object == running_task_code_obj) { + break; + } + } + + return 0; +} + +/* ============================================================================ + * AWAITED BY PARSING FUNCTIONS + * ============================================================================ */ + +static int +append_awaited_by_for_thread( + RemoteUnwinderObject *unwinder, + uintptr_t head_addr, + PyObject *result +) { + char task_node[SIZEOF_LLIST_NODE]; + + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, head_addr, + sizeof(task_node), task_node) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task node head"); + return -1; + } + + size_t iteration_count = 0; + const size_t MAX_ITERATIONS = 2 << 15; // A reasonable upper bound + + while (GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) != head_addr) { + if (++iteration_count > MAX_ITERATIONS) { + PyErr_SetString(PyExc_RuntimeError, "Task list appears corrupted"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Task list iteration limit exceeded"); + return -1; + } + + if (GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) == 0) { + PyErr_SetString(PyExc_RuntimeError, + "Invalid linked list structure reading remote memory"); + set_exception_cause(unwinder, PyExc_RuntimeError, "NULL pointer in task linked list"); + return -1; + } + + uintptr_t task_addr = (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) + - (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_node; + + if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process task node in awaited_by"); + return -1; + } + + // Read next node + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next), + sizeof(task_node), + task_node) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read next task node in awaited_by"); + return -1; + } + } + + return 0; +} + +int +append_awaited_by( + RemoteUnwinderObject *unwinder, + unsigned long tid, + uintptr_t head_addr, + PyObject *result) +{ + PyObject *tid_py = PyLong_FromUnsignedLong(tid); + if (tid_py == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID object"); + return -1; + } + + PyObject* awaited_by_for_thread = PyList_New(0); + if (awaited_by_for_thread == NULL) { + Py_DECREF(tid_py); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by thread list"); + return -1; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *result_item = PyStructSequence_New(state->AwaitedInfo_Type); + if (result_item == NULL) { + Py_DECREF(tid_py); + Py_DECREF(awaited_by_for_thread); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create AwaitedInfo"); + return -1; + } + + PyStructSequence_SetItem(result_item, 0, tid_py); // steals ref + PyStructSequence_SetItem(result_item, 1, awaited_by_for_thread); // steals ref + if (PyList_Append(result, result_item)) { + Py_DECREF(result_item); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append awaited_by result item"); + return -1; + } + Py_DECREF(result_item); + + if (append_awaited_by_for_thread(unwinder, head_addr, awaited_by_for_thread)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append awaited_by for thread"); + return -1; + } + + return 0; +} + +/* ============================================================================ + * THREAD PROCESSOR FUNCTIONS FOR ASYNCIO + * ============================================================================ */ + +int +process_thread_for_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +) { + PyObject *result = (PyObject *)context; + uintptr_t head_addr = thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; + return append_awaited_by(unwinder, tid, head_addr, result); +} + +static int +process_running_task_chain( + RemoteUnwinderObject *unwinder, + uintptr_t running_task_addr, + uintptr_t thread_state_addr, + PyObject *result +) { + uintptr_t running_task_code_obj = 0; + if(get_task_code_object(unwinder, running_task_addr, &running_task_code_obj) < 0) { + return -1; + } + + // First, add this task to the result + PyObject *task_info = NULL; + if (process_single_task_node(unwinder, running_task_addr, &task_info, result) < 0) { + return -1; + } + + // Get the chain from the current frame to this task + PyObject *coro_chain = PyStructSequence_GET_ITEM(task_info, 2); + assert(coro_chain != NULL); + if (PyList_GET_SIZE(coro_chain) != 1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Coro chain is not a single item"); + return -1; + } + PyObject *coro_info = PyList_GET_ITEM(coro_chain, 0); + assert(coro_info != NULL); + PyObject *frame_chain = PyStructSequence_GET_ITEM(coro_info, 0); + assert(frame_chain != NULL); + + // Clear the coro_chain + if (PyList_Clear(frame_chain) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to clear coroutine chain"); + return -1; + } + + // Add the chain from the current frame to this task + if (parse_async_frame_chain(unwinder, frame_chain, thread_state_addr, running_task_code_obj) < 0) { + return -1; + } + + // Now find all tasks that are waiting for this task and process them + if (process_task_awaited_by(unwinder, running_task_addr, process_waiter_task, result) < 0) { + return -1; + } + + return 0; +} + +int +process_thread_for_async_stack_trace( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +) { + PyObject *result = (PyObject *)context; + + // Find running task in this thread + uintptr_t running_task_addr; + if (find_running_task_in_thread(unwinder, thread_state_addr, &running_task_addr) < 0) { + return 0; + } + + // If we found a running task, process it and its waiters + if ((void*)running_task_addr != NULL) { + PyObject *task_list = PyList_New(0); + if (task_list == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create task list for thread"); + return -1; + } + + if (process_running_task_chain(unwinder, running_task_addr, thread_state_addr, task_list) < 0) { + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process running task chain"); + return -1; + } + + // Create AwaitedInfo structure for this thread + PyObject *tid_py = PyLong_FromUnsignedLong(tid); + if (tid_py == NULL) { + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + return -1; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *awaited_info = PyStructSequence_New(state->AwaitedInfo_Type); + if (awaited_info == NULL) { + Py_DECREF(tid_py); + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create AwaitedInfo"); + return -1; + } + + PyStructSequence_SetItem(awaited_info, 0, tid_py); // steals ref + PyStructSequence_SetItem(awaited_info, 1, task_list); // steals ref + + if (PyList_Append(result, awaited_info)) { + Py_DECREF(awaited_info); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append AwaitedInfo to result"); + return -1; + } + Py_DECREF(awaited_info); + } + + return 0; +} diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h new file mode 100644 index 00000000000..03127b753cc --- /dev/null +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -0,0 +1,422 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, +"RemoteUnwinder(pid, *, all_threads=False, only_active_thread=False,\n" +" mode=0, debug=False, skip_non_matching_threads=True,\n" +" native=False, gc=False, cache_frames=False, stats=False)\n" +"--\n" +"\n" +"Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" +"\n" +"Args:\n" +" pid: Process ID of the target Python process to debug\n" +" all_threads: If True, initialize state for all threads in the process.\n" +" If False, only initialize for the main thread.\n" +" only_active_thread: If True, only sample the thread holding the GIL.\n" +" mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL (gil-time).\n" +" Cannot be used together with all_threads=True.\n" +" debug: If True, chain exceptions to explain the sequence of events that\n" +" lead to the exception.\n" +" skip_non_matching_threads: If True, skip threads that don\'t match the selected mode.\n" +" If False, include all threads regardless of mode.\n" +" native: If True, include artificial \"<native>\" frames to denote calls to\n" +" non-Python code.\n" +" gc: If True, include artificial \"<GC>\" frames to denote active garbage\n" +" collection.\n" +" cache_frames: If True, enable frame caching optimization to avoid re-reading\n" +" unchanged parent frames between samples.\n" +" stats: If True, collect statistics about cache hits, memory reads, etc.\n" +" Use get_stats() to retrieve the collected statistics.\n" +"\n" +"The RemoteUnwinder provides functionality to inspect and debug a running Python\n" +"process, including examining thread states, stack frames and other runtime data.\n" +"\n" +"Raises:\n" +" PermissionError: If access to the target process is denied\n" +" OSError: If unable to attach to the target process or access its memory\n" +" RuntimeError: If unable to read debug information from the target process\n" +" ValueError: If both all_threads and only_active_thread are True"); + +static int +_remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, + int pid, int all_threads, + int only_active_thread, + int mode, int debug, + int skip_non_matching_threads, + int native, int gc, + int cache_frames, int stats); + +static int +_remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 10 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), &_Py_ID(native), &_Py_ID(gc), &_Py_ID(cache_frames), &_Py_ID(stats), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", "native", "gc", "cache_frames", "stats", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "RemoteUnwinder", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[10]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; + int pid; + int all_threads = 0; + int only_active_thread = 0; + int mode = 0; + int debug = 0; + int skip_non_matching_threads = 1; + int native = 0; + int gc = 0; + int cache_frames = 0; + int stats = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + pid = PyLong_AsInt(fastargs[0]); + if (pid == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (fastargs[1]) { + all_threads = PyObject_IsTrue(fastargs[1]); + if (all_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[2]) { + only_active_thread = PyObject_IsTrue(fastargs[2]); + if (only_active_thread < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + mode = PyLong_AsInt(fastargs[3]); + if (mode == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + debug = PyObject_IsTrue(fastargs[4]); + if (debug < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[5]) { + skip_non_matching_threads = PyObject_IsTrue(fastargs[5]); + if (skip_non_matching_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[6]) { + native = PyObject_IsTrue(fastargs[6]); + if (native < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[7]) { + gc = PyObject_IsTrue(fastargs[7]); + if (gc < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[8]) { + cache_frames = PyObject_IsTrue(fastargs[8]); + if (cache_frames < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + stats = PyObject_IsTrue(fastargs[9]); + if (stats < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads, native, gc, cache_frames, stats); + +exit: + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, +"get_stack_trace($self, /)\n" +"--\n" +"\n" +"Returns stack traces for all interpreters and threads in process.\n" +"\n" +"Each element in the returned list is a tuple of (interpreter_id, thread_list), where:\n" +"- interpreter_id is the interpreter identifier\n" +"- thread_list is a list of tuples (thread_id, frame_list) for threads in that interpreter\n" +" - thread_id is the OS thread identifier\n" +" - frame_list is a list of tuples (function_name, filename, line_number) representing\n" +" the Python stack frames for that thread, ordered from most recent to oldest\n" +"\n" +"The threads returned depend on the initialization parameters:\n" +"- If only_active_thread was True: returns only the thread holding the GIL across all interpreters\n" +"- If all_threads was True: returns all threads across all interpreters\n" +"- Otherwise: returns only the main thread of each interpreter\n" +"\n" +"Example:\n" +" [\n" +" (0, [ # Main interpreter\n" +" (1234, [\n" +" (\'process_data\', \'worker.py\', 127),\n" +" (\'run_worker\', \'worker.py\', 45),\n" +" (\'main\', \'app.py\', 23)\n" +" ]),\n" +" (1235, [\n" +" (\'handle_request\', \'server.py\', 89),\n" +" (\'serve_forever\', \'server.py\', 52)\n" +" ])\n" +" ]),\n" +" (1, [ # Sub-interpreter\n" +" (1236, [\n" +" (\'sub_worker\', \'sub.py\', 15)\n" +" ])\n" +" ])\n" +" ]\n" +"\n" +"Raises:\n" +" RuntimeError: If there is an error copying memory from the target process\n" +" OSError: If there is an error accessing the target process\n" +" PermissionError: If access to the target process is denied\n" +" UnicodeDecodeError: If there is an error decoding strings from the target process"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF \ + {"get_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_stack_trace__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_stack_trace_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, +"get_all_awaited_by($self, /)\n" +"--\n" +"\n" +"Get all tasks and their awaited_by relationships from the remote process.\n" +"\n" +"This provides a tree structure showing which tasks are waiting for other tasks.\n" +"\n" +"For each task, returns:\n" +"1. The call stack frames leading to where the task is currently executing\n" +"2. The name of the task\n" +"3. A list of tasks that this task is waiting for, with their own frames/names/etc\n" +"\n" +"Returns a list of [frames, task_name, subtasks] where:\n" +"- frames: List of (func_name, filename, lineno) showing the call stack\n" +"- task_name: String identifier for the task\n" +"- subtasks: List of tasks being awaited by this task, in same format\n" +"\n" +"Raises:\n" +" RuntimeError: If AsyncioDebug section is not available in the remote process\n" +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output:\n" +"[\n" +" [\n" +" [(\"c5\", \"script.py\", 10), (\"c4\", \"script.py\", 14)],\n" +" \"c2_root\",\n" +" [\n" +" [\n" +" [(\"c1\", \"script.py\", 23)],\n" +" \"sub_main_2\",\n" +" [...]\n" +" ],\n" +" [...]\n" +" ]\n" +" ]\n" +"]"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF \ + {"get_all_awaited_by", (PyCFunction)_remote_debugging_RemoteUnwinder_get_all_awaited_by, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, +"get_async_stack_trace($self, /)\n" +"--\n" +"\n" +"Get the currently running async tasks and their dependency graphs from the remote process.\n" +"\n" +"This returns information about running tasks and all tasks that are waiting for them,\n" +"forming a complete dependency graph for each thread\'s active task.\n" +"\n" +"For each thread with a running task, returns the running task plus all tasks that\n" +"transitively depend on it (tasks waiting for the running task, tasks waiting for\n" +"those tasks, etc.).\n" +"\n" +"Returns a list of per-thread results, where each thread result contains:\n" +"- Thread ID\n" +"- List of task information for the running task and all its waiters\n" +"\n" +"Each task info contains:\n" +"- Task ID (memory address)\n" +"- Task name\n" +"- Call stack frames: List of (func_name, filename, lineno)\n" +"- List of tasks waiting for this task (recursive structure)\n" +"\n" +"Raises:\n" +" RuntimeError: If AsyncioDebug section is not available in the target process\n" +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output (similar structure to get_all_awaited_by but only for running tasks):\n" +"[\n" +" (140234, [\n" +" (4345585712, \'main_task\',\n" +" [(\"run_server\", \"server.py\", 127), (\"main\", \"app.py\", 23)],\n" +" [\n" +" (4345585800, \'worker_1\', [...], [...]),\n" +" (4345585900, \'worker_2\', [...], [...])\n" +" ])\n" +" ])\n" +"]"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF \ + {"get_async_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_async_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stats__doc__, +"get_stats($self, /)\n" +"--\n" +"\n" +"Get collected statistics about profiling performance.\n" +"\n" +"Returns a dictionary containing statistics about cache performance,\n" +"memory reads, and other profiling metrics. Only available if the\n" +"RemoteUnwinder was created with stats=True.\n" +"\n" +"Returns:\n" +" dict: A dictionary containing:\n" +" - total_samples: Total number of get_stack_trace calls\n" +" - frame_cache_hits: Full cache hits (entire stack unchanged)\n" +" - frame_cache_misses: Cache misses requiring full walk\n" +" - frame_cache_partial_hits: Partial hits (stopped at cached frame)\n" +" - frames_read_from_cache: Total frames retrieved from cache\n" +" - frames_read_from_memory: Total frames read from remote memory\n" +" - memory_reads: Total remote memory read operations\n" +" - memory_bytes_read: Total bytes read from remote memory\n" +" - code_object_cache_hits: Code object cache hits\n" +" - code_object_cache_misses: Code object cache misses\n" +" - stale_cache_invalidations: Times stale cache entries were cleared\n" +" - frame_cache_hit_rate: Percentage of samples that hit the cache\n" +" - code_object_cache_hit_rate: Percentage of code object lookups that hit cache\n" +"\n" +"Raises:\n" +" RuntimeError: If stats collection was not enabled (stats=False)"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STATS_METHODDEF \ + {"get_stats", (PyCFunction)_remote_debugging_RemoteUnwinder_get_stats, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_stats__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stats(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_stats_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=f1fd6c1d4c4c7254 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c new file mode 100644 index 00000000000..2cd2505d0f9 --- /dev/null +++ b/Modules/_remote_debugging/code_objects.c @@ -0,0 +1,395 @@ +/****************************************************************************** + * Remote Debugging Module - Code Object Functions + * + * This file contains functions for parsing code objects and line tables + * from remote process memory. + ******************************************************************************/ + +#include "_remote_debugging.h" + +/* ============================================================================ + * TLBC CACHING FUNCTIONS (Py_GIL_DISABLED only) + * ============================================================================ */ + +#ifdef Py_GIL_DISABLED + +void +tlbc_cache_entry_destroy(void *ptr) +{ + TLBCCacheEntry *entry = (TLBCCacheEntry *)ptr; + if (entry->tlbc_array) { + PyMem_RawFree(entry->tlbc_array); + } + PyMem_RawFree(entry); +} + +TLBCCacheEntry * +get_tlbc_cache_entry(RemoteUnwinderObject *self, uintptr_t code_addr, uint32_t current_generation) +{ + void *key = (void *)code_addr; + TLBCCacheEntry *entry = _Py_hashtable_get(self->tlbc_cache, key); + + if (entry && entry->generation != current_generation) { + // Entry is stale, remove it by setting to NULL + _Py_hashtable_set(self->tlbc_cache, key, NULL); + entry = NULL; + } + + return entry; +} + +int +cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t tlbc_array_addr, uint32_t generation) +{ + uintptr_t tlbc_array_ptr; + void *tlbc_array = NULL; + TLBCCacheEntry *entry = NULL; + + // Read the TLBC array pointer + if (read_ptr(unwinder, tlbc_array_addr, &tlbc_array_ptr) != 0) { + PyErr_SetString(PyExc_RuntimeError, "Failed to read TLBC array pointer"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array pointer"); + return 0; // Read error + } + + // Validate TLBC array pointer + if (tlbc_array_ptr == 0) { + PyErr_SetString(PyExc_RuntimeError, "TLBC array pointer is NULL"); + return 0; // No TLBC array + } + + // Read the TLBC array size + Py_ssize_t tlbc_size; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, tlbc_array_ptr, sizeof(tlbc_size), &tlbc_size) != 0) { + PyErr_SetString(PyExc_RuntimeError, "Failed to read TLBC array size"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array size"); + return 0; // Read error + } + + // Validate TLBC array size + if (tlbc_size <= 0) { + PyErr_SetString(PyExc_RuntimeError, "Invalid TLBC array size"); + return 0; // Invalid size + } + + if (tlbc_size > MAX_TLBC_SIZE) { + PyErr_SetString(PyExc_RuntimeError, "TLBC array size exceeds maximum limit"); + return 0; // Invalid size + } + + // Allocate and read the entire TLBC array + size_t array_data_size = tlbc_size * sizeof(void*); + tlbc_array = PyMem_RawMalloc(sizeof(Py_ssize_t) + array_data_size); + if (!tlbc_array) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate TLBC array"); + return 0; // Memory error + } + + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, tlbc_array_ptr, sizeof(Py_ssize_t) + array_data_size, tlbc_array) != 0) { + PyMem_RawFree(tlbc_array); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array data"); + return 0; // Read error + } + + // Create cache entry + entry = PyMem_RawMalloc(sizeof(TLBCCacheEntry)); + if (!entry) { + PyErr_NoMemory(); + PyMem_RawFree(tlbc_array); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate TLBC cache entry"); + return 0; // Memory error + } + + entry->tlbc_array = tlbc_array; + entry->tlbc_array_size = tlbc_size; + entry->generation = generation; + + // Store in cache + void *key = (void *)code_addr; + if (_Py_hashtable_set(unwinder->tlbc_cache, key, entry) < 0) { + tlbc_cache_entry_destroy(entry); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to store TLBC entry in cache"); + return 0; // Cache error + } + + return 1; // Success +} + +#endif + +/* ============================================================================ + * LINE TABLE PARSING FUNCTIONS + * ============================================================================ */ + +static int +scan_varint(const uint8_t **ptr) +{ + unsigned int read = **ptr; + *ptr = *ptr + 1; + unsigned int val = read & 63; + unsigned int shift = 0; + while (read & 64) { + read = **ptr; + *ptr = *ptr + 1; + shift += 6; + val |= (read & 63) << shift; + } + return val; +} + +static int +scan_signed_varint(const uint8_t **ptr) +{ + unsigned int uval = scan_varint(ptr); + if (uval & 1) { + return -(int)(uval >> 1); + } + else { + return uval >> 1; + } +} + +bool +parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, LocationInfo* info) +{ + const uint8_t* ptr = (const uint8_t*)(linetable); + uintptr_t addr = 0; + info->lineno = firstlineno; + + while (*ptr != '\0') { + // See InternalDocs/code_objects.md for where these magic numbers are from + // and for the decoding algorithm. + uint8_t first_byte = *(ptr++); + uint8_t code = (first_byte >> 3) & 15; + size_t length = (first_byte & 7) + 1; + uintptr_t end_addr = addr + length; + switch (code) { + case PY_CODE_LOCATION_INFO_NONE: { + break; + } + case PY_CODE_LOCATION_INFO_LONG: { + int line_delta = scan_signed_varint(&ptr); + info->lineno += line_delta; + info->end_lineno = info->lineno + scan_varint(&ptr); + info->column = scan_varint(&ptr) - 1; + info->end_column = scan_varint(&ptr) - 1; + break; + } + case PY_CODE_LOCATION_INFO_NO_COLUMNS: { + int line_delta = scan_signed_varint(&ptr); + info->lineno += line_delta; + info->column = info->end_column = -1; + break; + } + case PY_CODE_LOCATION_INFO_ONE_LINE0: + case PY_CODE_LOCATION_INFO_ONE_LINE1: + case PY_CODE_LOCATION_INFO_ONE_LINE2: { + int line_delta = code - 10; + info->lineno += line_delta; + info->end_lineno = info->lineno; + info->column = *(ptr++); + info->end_column = *(ptr++); + break; + } + default: { + uint8_t second_byte = *(ptr++); + if ((second_byte & 128) != 0) { + return false; + } + info->column = code << 3 | (second_byte >> 4); + info->end_column = info->column + (second_byte & 15); + break; + } + } + if (addr <= addrq && end_addr > addrq) { + return true; + } + addr = end_addr; + } + return false; +} + +/* ============================================================================ + * CODE OBJECT AND FRAME INFO FUNCTIONS + * ============================================================================ */ + +PyObject * +make_frame_info(RemoteUnwinderObject *unwinder, PyObject *file, PyObject *line, + PyObject *func) +{ + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *info = PyStructSequence_New(state->FrameInfo_Type); + if (info == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create FrameInfo"); + return NULL; + } + Py_INCREF(file); + Py_INCREF(line); + Py_INCREF(func); + PyStructSequence_SetItem(info, 0, file); + PyStructSequence_SetItem(info, 1, line); + PyStructSequence_SetItem(info, 2, func); + return info; +} + +int +parse_code_object(RemoteUnwinderObject *unwinder, + PyObject **result, + uintptr_t address, + uintptr_t instruction_pointer, + uintptr_t *previous_frame, + int32_t tlbc_index) +{ + void *key = (void *)address; + CachedCodeMetadata *meta = NULL; + PyObject *func = NULL; + PyObject *file = NULL; + PyObject *linetable = NULL; + +#ifdef Py_GIL_DISABLED + // In free threading builds, code object addresses might have the low bit set + // as a flag, so we need to mask it off to get the real address + uintptr_t real_address = address & (~1); +#else + uintptr_t real_address = address; +#endif + + if (unwinder && unwinder->code_object_cache != NULL) { + meta = _Py_hashtable_get(unwinder->code_object_cache, key); + if (meta) { + STATS_INC(unwinder, code_object_cache_hits); + } else { + STATS_INC(unwinder, code_object_cache_misses); + } + } + + if (meta == NULL) { + char code_object[SIZEOF_CODE_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, real_address, SIZEOF_CODE_OBJ, code_object) < 0) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read code object"); + goto error; + } + + func = read_py_str(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.qualname), 1024); + if (!func) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read function name from code object"); + goto error; + } + + file = read_py_str(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.filename), 1024); + if (!file) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read filename from code object"); + goto error; + } + + linetable = read_py_bytes(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.linetable), 4096); + if (!linetable) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read linetable from code object"); + goto error; + } + + meta = PyMem_RawMalloc(sizeof(CachedCodeMetadata)); + if (!meta) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate cached code metadata"); + goto error; + } + + meta->func_name = func; + meta->file_name = file; + meta->linetable = linetable; + meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); + meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; + + if (unwinder && unwinder->code_object_cache && _Py_hashtable_set(unwinder->code_object_cache, key, meta) < 0) { + cached_code_metadata_destroy(meta); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to cache code metadata"); + goto error; + } + + // Ownership transferred to meta + func = NULL; + file = NULL; + linetable = NULL; + } + + uintptr_t ip = instruction_pointer; + ptrdiff_t addrq; + +#ifdef Py_GIL_DISABLED + // Handle thread-local bytecode (TLBC) in free threading builds + if (tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) { + // No TLBC or no unwinder - use main bytecode directly + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; + goto done_tlbc; + } + + // Try to get TLBC data from cache (we'll get generation from the caller) + TLBCCacheEntry *tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation); + + if (!tlbc_entry) { + // Cache miss - try to read and cache TLBC array + if (!cache_tlbc_array(unwinder, real_address, real_address + unwinder->debug_offsets.code_object.co_tlbc, unwinder->tlbc_generation)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to cache TLBC array"); + goto error; + } + tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation); + } + + if (tlbc_entry && tlbc_index < tlbc_entry->tlbc_array_size) { + // Use cached TLBC data + uintptr_t *entries = (uintptr_t *)((char *)tlbc_entry->tlbc_array + sizeof(Py_ssize_t)); + uintptr_t tlbc_bytecode_addr = entries[tlbc_index]; + + if (tlbc_bytecode_addr != 0) { + // Calculate offset from TLBC bytecode + addrq = (uint16_t *)ip - (uint16_t *)tlbc_bytecode_addr; + goto done_tlbc; + } + } + + // Fall back to main bytecode + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; + +done_tlbc: +#else + // Non-free-threaded build, always use the main bytecode + (void)tlbc_index; // Suppress unused parameter warning + (void)unwinder; // Suppress unused parameter warning + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; +#endif + ; // Empty statement to avoid C23 extension warning + LocationInfo info = {0}; + bool ok = parse_linetable(addrq, PyBytes_AS_STRING(meta->linetable), + meta->first_lineno, &info); + if (!ok) { + info.lineno = -1; + } + + PyObject *lineno = PyLong_FromLong(info.lineno); + if (!lineno) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create line number object"); + goto error; + } + + PyObject *tuple = make_frame_info(unwinder, meta->file_name, lineno, meta->func_name); + Py_DECREF(lineno); + if (!tuple) { + goto error; + } + + *result = tuple; + return 0; + +error: + Py_XDECREF(func); + Py_XDECREF(file); + Py_XDECREF(linetable); + return -1; +} diff --git a/Modules/_remote_debugging/frame_cache.c b/Modules/_remote_debugging/frame_cache.c new file mode 100644 index 00000000000..4598b9dc353 --- /dev/null +++ b/Modules/_remote_debugging/frame_cache.c @@ -0,0 +1,236 @@ +/****************************************************************************** + * Remote Debugging Module - Frame Cache + * + * This file contains functions for caching frame information to optimize + * repeated stack unwinding for profiling. + ******************************************************************************/ + +#include "_remote_debugging.h" + +/* ============================================================================ + * FRAME CACHE - stores (address, frame_info) pairs per thread + * Uses preallocated fixed-size arrays for efficiency and bounded memory. + * ============================================================================ */ + +int +frame_cache_init(RemoteUnwinderObject *unwinder) +{ + unwinder->frame_cache = PyMem_Calloc(FRAME_CACHE_MAX_THREADS, sizeof(FrameCacheEntry)); + if (!unwinder->frame_cache) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + +void +frame_cache_cleanup(RemoteUnwinderObject *unwinder) +{ + if (!unwinder->frame_cache) { + return; + } + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + Py_CLEAR(unwinder->frame_cache[i].frame_list); + } + PyMem_Free(unwinder->frame_cache); + unwinder->frame_cache = NULL; +} + +// Find cache entry by thread_id +FrameCacheEntry * +frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id) +{ + if (!unwinder->frame_cache || thread_id == 0) { + return NULL; + } + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_id == thread_id) { + return &unwinder->frame_cache[i]; + } + } + return NULL; +} + +// Allocate a cache slot for a thread +// Returns NULL if cache is full (graceful degradation) +static FrameCacheEntry * +frame_cache_alloc_slot(RemoteUnwinderObject *unwinder, uint64_t thread_id) +{ + if (!unwinder->frame_cache || thread_id == 0) { + return NULL; + } + // First check if thread already has an entry + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_id == thread_id) { + return &unwinder->frame_cache[i]; + } + } + // Find empty slot + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_id == 0) { + return &unwinder->frame_cache[i]; + } + } + // Cache full - graceful degradation + return NULL; +} + +// Remove cache entries for threads not seen in the result +// result structure: list of InterpreterInfo, where InterpreterInfo[1] is threads list, +// and ThreadInfo[0] is the thread_id +void +frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result) +{ + if (!unwinder->frame_cache || !result || !PyList_Check(result)) { + return; + } + + // Build array of seen thread IDs from result + uint64_t seen_threads[FRAME_CACHE_MAX_THREADS]; + int num_seen = 0; + + Py_ssize_t num_interps = PyList_GET_SIZE(result); + for (Py_ssize_t i = 0; i < num_interps && num_seen < FRAME_CACHE_MAX_THREADS; i++) { + PyObject *interp_info = PyList_GET_ITEM(result, i); + PyObject *threads = PyStructSequence_GetItem(interp_info, 1); + if (!threads || !PyList_Check(threads)) { + continue; + } + Py_ssize_t num_threads = PyList_GET_SIZE(threads); + for (Py_ssize_t j = 0; j < num_threads && num_seen < FRAME_CACHE_MAX_THREADS; j++) { + PyObject *thread_info = PyList_GET_ITEM(threads, j); + PyObject *tid_obj = PyStructSequence_GetItem(thread_info, 0); + if (tid_obj) { + uint64_t tid = PyLong_AsUnsignedLongLong(tid_obj); + if (!PyErr_Occurred()) { + seen_threads[num_seen++] = tid; + } else { + PyErr_Clear(); + } + } + } + } + + // Invalidate entries not in seen list + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_id == 0) { + continue; + } + int found = 0; + for (int j = 0; j < num_seen; j++) { + if (unwinder->frame_cache[i].thread_id == seen_threads[j]) { + found = 1; + break; + } + } + if (!found) { + // Clear this entry + Py_CLEAR(unwinder->frame_cache[i].frame_list); + unwinder->frame_cache[i].thread_id = 0; + unwinder->frame_cache[i].num_addrs = 0; + STATS_INC(unwinder, stale_cache_invalidations); + } + } +} + +// Find last_profiled_frame in cache and extend frame_info with cached continuation +// If frame_addrs is provided (not NULL), also extends it with cached addresses +int +frame_cache_lookup_and_extend( + RemoteUnwinderObject *unwinder, + uint64_t thread_id, + uintptr_t last_profiled_frame, + PyObject *frame_info, + uintptr_t *frame_addrs, + Py_ssize_t *num_addrs, + Py_ssize_t max_addrs) +{ + if (!unwinder->frame_cache || last_profiled_frame == 0) { + return 0; + } + + FrameCacheEntry *entry = frame_cache_find(unwinder, thread_id); + if (!entry || !entry->frame_list) { + return 0; + } + + // Find the index where last_profiled_frame matches + Py_ssize_t start_idx = -1; + for (Py_ssize_t i = 0; i < entry->num_addrs; i++) { + if (entry->addrs[i] == last_profiled_frame) { + start_idx = i; + break; + } + } + + if (start_idx < 0) { + return 0; // Not found + } + + Py_ssize_t num_frames = PyList_GET_SIZE(entry->frame_list); + + // Extend frame_info with frames from start_idx onwards + PyObject *slice = PyList_GetSlice(entry->frame_list, start_idx, num_frames); + if (!slice) { + return -1; + } + + Py_ssize_t cur_size = PyList_GET_SIZE(frame_info); + int result = PyList_SetSlice(frame_info, cur_size, cur_size, slice); + Py_DECREF(slice); + + if (result < 0) { + return -1; + } + + // Also extend frame_addrs with cached addresses if provided + if (frame_addrs) { + for (Py_ssize_t i = start_idx; i < entry->num_addrs && *num_addrs < max_addrs; i++) { + frame_addrs[(*num_addrs)++] = entry->addrs[i]; + } + } + + return 1; +} + +// Store frame list with addresses in cache +// Returns: 1 = stored successfully, 0 = not stored (graceful degradation), -1 = error +int +frame_cache_store( + RemoteUnwinderObject *unwinder, + uint64_t thread_id, + PyObject *frame_list, + const uintptr_t *addrs, + Py_ssize_t num_addrs) +{ + if (!unwinder->frame_cache || thread_id == 0) { + return 0; + } + + // Clamp to max frames + if (num_addrs > FRAME_CACHE_MAX_FRAMES) { + num_addrs = FRAME_CACHE_MAX_FRAMES; + } + + FrameCacheEntry *entry = frame_cache_alloc_slot(unwinder, thread_id); + if (!entry) { + // Cache full - graceful degradation + return 0; + } + + // Clear old frame_list if replacing + Py_CLEAR(entry->frame_list); + + // Store full frame list (don't truncate to num_addrs - frames beyond the + // address array limit are still valid and needed for full cache hits) + Py_ssize_t num_frames = PyList_GET_SIZE(frame_list); + entry->frame_list = PyList_GetSlice(frame_list, 0, num_frames); + if (!entry->frame_list) { + return -1; + } + entry->thread_id = thread_id; + memcpy(entry->addrs, addrs, num_addrs * sizeof(uintptr_t)); + entry->num_addrs = num_addrs; + + return 1; +} diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c new file mode 100644 index 00000000000..eaf3287c6fe --- /dev/null +++ b/Modules/_remote_debugging/frames.c @@ -0,0 +1,598 @@ +/****************************************************************************** + * Remote Debugging Module - Frame Functions + * + * This file contains functions for parsing interpreter frames and + * managing stack chunks from remote process memory. + ******************************************************************************/ + +#include "_remote_debugging.h" + +/* ============================================================================ + * STACK CHUNK MANAGEMENT FUNCTIONS + * ============================================================================ */ + +void +cleanup_stack_chunks(StackChunkList *chunks) +{ + for (size_t i = 0; i < chunks->count; ++i) { + PyMem_RawFree(chunks->chunks[i].local_copy); + } + PyMem_RawFree(chunks->chunks); +} + +static int +process_single_stack_chunk( + RemoteUnwinderObject *unwinder, + uintptr_t chunk_addr, + StackChunkInfo *chunk_info +) { + // Start with default size assumption + size_t current_size = _PY_DATA_STACK_CHUNK_SIZE; + + char *this_chunk = PyMem_RawMalloc(current_size); + if (!this_chunk) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate stack chunk buffer"); + return -1; + } + + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, chunk_addr, current_size, this_chunk) < 0) { + PyMem_RawFree(this_chunk); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read stack chunk"); + return -1; + } + + // Check actual size and reread if necessary + size_t actual_size = GET_MEMBER(size_t, this_chunk, offsetof(_PyStackChunk, size)); + if (actual_size != current_size) { + this_chunk = PyMem_RawRealloc(this_chunk, actual_size); + if (!this_chunk) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to reallocate stack chunk buffer"); + return -1; + } + + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, chunk_addr, actual_size, this_chunk) < 0) { + PyMem_RawFree(this_chunk); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to reread stack chunk with correct size"); + return -1; + } + current_size = actual_size; + } + + chunk_info->remote_addr = chunk_addr; + chunk_info->size = current_size; + chunk_info->local_copy = this_chunk; + return 0; +} + +int +copy_stack_chunks(RemoteUnwinderObject *unwinder, + uintptr_t tstate_addr, + StackChunkList *out_chunks) +{ + uintptr_t chunk_addr; + StackChunkInfo *chunks = NULL; + size_t count = 0; + size_t max_chunks = 16; + + if (read_ptr(unwinder, tstate_addr + (uintptr_t)unwinder->debug_offsets.thread_state.datastack_chunk, &chunk_addr)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read initial stack chunk address"); + return -1; + } + + chunks = PyMem_RawMalloc(max_chunks * sizeof(StackChunkInfo)); + if (!chunks) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate stack chunks array"); + return -1; + } + + while (chunk_addr != 0) { + // Grow array if needed + if (count >= max_chunks) { + max_chunks *= 2; + StackChunkInfo *new_chunks = PyMem_RawRealloc(chunks, max_chunks * sizeof(StackChunkInfo)); + if (!new_chunks) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to grow stack chunks array"); + goto error; + } + chunks = new_chunks; + } + + // Process this chunk + if (process_single_stack_chunk(unwinder, chunk_addr, &chunks[count]) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process stack chunk"); + goto error; + } + + // Get next chunk address and increment count + chunk_addr = GET_MEMBER(uintptr_t, chunks[count].local_copy, offsetof(_PyStackChunk, previous)); + count++; + } + + out_chunks->chunks = chunks; + out_chunks->count = count; + return 0; + +error: + for (size_t i = 0; i < count; ++i) { + PyMem_RawFree(chunks[i].local_copy); + } + PyMem_RawFree(chunks); + return -1; +} + +void * +find_frame_in_chunks(StackChunkList *chunks, uintptr_t remote_ptr) +{ + for (size_t i = 0; i < chunks->count; ++i) { + uintptr_t base = chunks->chunks[i].remote_addr + offsetof(_PyStackChunk, data); + size_t payload = chunks->chunks[i].size - offsetof(_PyStackChunk, data); + + if (remote_ptr >= base && remote_ptr < base + payload) { + return (char *)chunks->chunks[i].local_copy + (remote_ptr - chunks->chunks[i].remote_addr); + } + } + return NULL; +} + +/* ============================================================================ + * FRAME PARSING FUNCTIONS + * ============================================================================ */ + +int +is_frame_valid( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + uintptr_t code_object_addr +) { + if ((void*)code_object_addr == NULL) { + return 0; + } + + void* frame = (void*)frame_addr; + + char owner = GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner); + if (owner == FRAME_OWNED_BY_INTERPRETER) { + return 0; // C frame or sentinel base frame + } + + if (owner != FRAME_OWNED_BY_GENERATOR && owner != FRAME_OWNED_BY_THREAD) { + PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", owner); + set_exception_cause(unwinder, PyExc_RuntimeError, "Unhandled frame owner type in async frame"); + return -1; + } + return 1; +} + +int +parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* address_of_code_object, + uintptr_t* previous_frame +) { + char frame[SIZEOF_INTERP_FRAME]; + *address_of_code_object = 0; + + Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_INTERP_FRAME, + frame + ); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); + return -1; + } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + + *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); + int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); + if (frame_valid != 1) { + return frame_valid; + } + + uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); + + // Get tlbc_index for free threading builds + int32_t tlbc_index = 0; +#ifdef Py_GIL_DISABLED + if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { + tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); + } +#endif + + *address_of_code_object = code_object; + return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index); +} + +int +parse_frame_from_chunks( + RemoteUnwinderObject *unwinder, + PyObject **result, + uintptr_t address, + uintptr_t *previous_frame, + uintptr_t *stackpointer, + StackChunkList *chunks +) { + void *frame_ptr = find_frame_in_chunks(chunks, address); + if (!frame_ptr) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame not found in stack chunks"); + return -1; + } + + char *frame = (char *)frame_ptr; + *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + *stackpointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.stackpointer); + uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame_ptr, unwinder->debug_offsets.interpreter_frame.executable); + int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); + if (frame_valid != 1) { + return frame_valid; + } + + uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); + + // Get tlbc_index for free threading builds + int32_t tlbc_index = 0; +#ifdef Py_GIL_DISABLED + if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { + tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); + } +#endif + + return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index); +} + +/* ============================================================================ + * FRAME CHAIN PROCESSING + * ============================================================================ */ + +int +process_frame_chain( + RemoteUnwinderObject *unwinder, + uintptr_t initial_frame_addr, + StackChunkList *chunks, + PyObject *frame_info, + uintptr_t base_frame_addr, + uintptr_t gc_frame, + uintptr_t last_profiled_frame, + int *stopped_at_cached_frame, + uintptr_t *frame_addrs, // optional: C array to receive frame addresses + Py_ssize_t *num_addrs, // in/out: current count / updated count + Py_ssize_t max_addrs) // max capacity of frame_addrs array +{ + uintptr_t frame_addr = initial_frame_addr; + uintptr_t prev_frame_addr = 0; + uintptr_t last_frame_addr = 0; // Track last frame visited for validation + const size_t MAX_FRAMES = 1024 + 512; + size_t frame_count = 0; + + // Initialize output flag + if (stopped_at_cached_frame) { + *stopped_at_cached_frame = 0; + } + + // Quick check: if current_frame == last_profiled_frame, entire stack is unchanged + if (last_profiled_frame != 0 && initial_frame_addr == last_profiled_frame) { + if (stopped_at_cached_frame) { + *stopped_at_cached_frame = 1; + } + return 0; + } + + while ((void*)frame_addr != NULL) { + // Check if we've reached the cached frame - if so, stop here + if (last_profiled_frame != 0 && frame_addr == last_profiled_frame) { + if (stopped_at_cached_frame) { + *stopped_at_cached_frame = 1; + } + break; + } + PyObject *frame = NULL; + uintptr_t next_frame_addr = 0; + uintptr_t stackpointer = 0; + last_frame_addr = frame_addr; // Remember this frame address + + if (++frame_count > MAX_FRAMES) { + PyErr_SetString(PyExc_RuntimeError, "Too many stack frames (possible infinite loop)"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain iteration limit exceeded"); + return -1; + } + + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, chunks) < 0) { + PyErr_Clear(); + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object ,&next_frame_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); + return -1; + } + } + if (frame == NULL && PyList_GET_SIZE(frame_info) == 0) { + const char *e = "Failed to parse initial frame in chain"; + PyErr_SetString(PyExc_RuntimeError, e); + return -1; + } + PyObject *extra_frame = NULL; + // This frame kicked off the current GC collection: + if (unwinder->gc && frame_addr == gc_frame) { + _Py_DECLARE_STR(gc, "<GC>"); + extra_frame = &_Py_STR(gc); + } + // Otherwise, check for native frames to insert: + else if (unwinder->native && + // We've reached an interpreter trampoline frame: + frame == NULL && + // Bottommost frame is always native, so skip that one: + next_frame_addr && + // Only suppress native frames if GC tracking is enabled and the next frame will be a GC frame: + !(unwinder->gc && next_frame_addr == gc_frame)) + { + _Py_DECLARE_STR(native, "<native>"); + extra_frame = &_Py_STR(native); + } + if (extra_frame) { + PyObject *extra_frame_info = make_frame_info( + unwinder, _Py_LATIN1_CHR('~'), _PyLong_GetZero(), extra_frame); + if (extra_frame_info == NULL) { + return -1; + } + if (PyList_Append(frame_info, extra_frame_info) < 0) { + Py_DECREF(extra_frame_info); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append extra frame"); + return -1; + } + // Extra frames use 0 as address (they're synthetic) + if (frame_addrs && *num_addrs < max_addrs) { + frame_addrs[(*num_addrs)++] = 0; + } + Py_DECREF(extra_frame_info); + } + if (frame) { + if (prev_frame_addr && frame_addr != prev_frame_addr) { + const char *f = "Broken frame chain: expected frame at 0x%lx, got 0x%lx"; + PyErr_Format(PyExc_RuntimeError, f, prev_frame_addr, frame_addr); + Py_DECREF(frame); + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain consistency check failed"); + return -1; + } + + if (PyList_Append(frame_info, frame) < 0) { + Py_DECREF(frame); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame"); + return -1; + } + // Track the address for this frame + if (frame_addrs && *num_addrs < max_addrs) { + frame_addrs[(*num_addrs)++] = frame_addr; + } + Py_DECREF(frame); + } + + prev_frame_addr = next_frame_addr; + frame_addr = next_frame_addr; + } + + // Validate we reached the base frame (sentinel at bottom of stack) + // Only validate if we walked the full chain (didn't stop at cached frame) + // and base_frame_addr is provided (non-zero) + int stopped_early = stopped_at_cached_frame && *stopped_at_cached_frame; + if (!stopped_early && base_frame_addr != 0 && last_frame_addr != base_frame_addr) { + PyErr_Format(PyExc_RuntimeError, + "Incomplete sample: did not reach base frame (expected 0x%lx, got 0x%lx)", + base_frame_addr, last_frame_addr); + return -1; + } + + return 0; +} + +// Clear last_profiled_frame for all threads in the target process. +// This must be called at the start of profiling to avoid stale values +// from previous profilers causing us to stop frame walking early. +int +clear_last_profiled_frames(RemoteUnwinderObject *unwinder) +{ + uintptr_t current_interp = unwinder->interpreter_addr; + uintptr_t zero = 0; + + while (current_interp != 0) { + // Get first thread in this interpreter + uintptr_t tstate_addr; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + current_interp + unwinder->debug_offsets.interpreter_state.threads_head, + sizeof(void*), + &tstate_addr) < 0) { + // Non-fatal: just skip clearing + PyErr_Clear(); + return 0; + } + + // Iterate all threads in this interpreter + while (tstate_addr != 0) { + // Clear last_profiled_frame + uintptr_t lpf_addr = tstate_addr + unwinder->debug_offsets.thread_state.last_profiled_frame; + if (_Py_RemoteDebug_WriteRemoteMemory(&unwinder->handle, lpf_addr, + sizeof(uintptr_t), &zero) < 0) { + // Non-fatal: just continue + PyErr_Clear(); + } + + // Move to next thread + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + tstate_addr + unwinder->debug_offsets.thread_state.next, + sizeof(void*), + &tstate_addr) < 0) { + PyErr_Clear(); + break; + } + } + + // Move to next interpreter + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + current_interp + unwinder->debug_offsets.interpreter_state.next, + sizeof(void*), + &current_interp) < 0) { + PyErr_Clear(); + break; + } + } + + return 0; +} + +// Fast path: check if we have a full cache hit (parent stack unchanged) +// A "full hit" means current frame == last profiled frame, so we can reuse +// cached parent frames. We always read the current frame from memory to get +// updated line numbers (the line within a frame can change between samples). +// Returns: 1 if full hit (frame_info populated with current frame + cached parents), +// 0 if miss, -1 on error +static int +try_full_cache_hit( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + uintptr_t last_profiled_frame, + uint64_t thread_id, + PyObject *frame_info) +{ + if (!unwinder->frame_cache || last_profiled_frame == 0) { + return 0; + } + // Full hit only if current frame == last profiled frame + if (frame_addr != last_profiled_frame) { + return 0; + } + + FrameCacheEntry *entry = frame_cache_find(unwinder, thread_id); + if (!entry || !entry->frame_list) { + return 0; + } + + // Verify first address matches (sanity check) + if (entry->num_addrs == 0 || entry->addrs[0] != frame_addr) { + return 0; + } + + // Always read the current frame from memory to get updated line number + PyObject *current_frame = NULL; + uintptr_t code_object_addr = 0; + uintptr_t previous_frame = 0; + int parse_result = parse_frame_object(unwinder, &current_frame, frame_addr, + &code_object_addr, &previous_frame); + if (parse_result < 0) { + return -1; + } + + // Get cached parent frames first (before modifying frame_info) + Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); + PyObject *parent_slice = NULL; + if (cached_size > 1) { + parent_slice = PyList_GetSlice(entry->frame_list, 1, cached_size); + if (!parent_slice) { + Py_XDECREF(current_frame); + return -1; + } + } + + // Now safe to modify frame_info - add current frame if valid + if (current_frame != NULL) { + if (PyList_Append(frame_info, current_frame) < 0) { + Py_DECREF(current_frame); + Py_XDECREF(parent_slice); + return -1; + } + Py_DECREF(current_frame); + STATS_ADD(unwinder, frames_read_from_memory, 1); + } + + // Extend with cached parent frames + if (parent_slice) { + Py_ssize_t cur_size = PyList_GET_SIZE(frame_info); + int result = PyList_SetSlice(frame_info, cur_size, cur_size, parent_slice); + Py_DECREF(parent_slice); + if (result < 0) { + return -1; + } + STATS_ADD(unwinder, frames_read_from_cache, cached_size - 1); + } + + STATS_INC(unwinder, frame_cache_hits); + return 1; +} + +// High-level helper: collect frames with cache optimization +// Returns complete frame_info list, handling all cache logic internally +int +collect_frames_with_cache( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + StackChunkList *chunks, + PyObject *frame_info, + uintptr_t gc_frame, + uintptr_t last_profiled_frame, + uint64_t thread_id) +{ + // Fast path: check for full cache hit first (no allocations needed) + int full_hit = try_full_cache_hit(unwinder, frame_addr, last_profiled_frame, + thread_id, frame_info); + if (full_hit != 0) { + return full_hit < 0 ? -1 : 0; // Either error or success + } + + uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; + Py_ssize_t num_addrs = 0; + Py_ssize_t frames_before = PyList_GET_SIZE(frame_info); + + int stopped_at_cached = 0; + if (process_frame_chain(unwinder, frame_addr, chunks, frame_info, 0, gc_frame, + last_profiled_frame, &stopped_at_cached, + addrs, &num_addrs, FRAME_CACHE_MAX_FRAMES) < 0) { + return -1; + } + + // Track frames read from memory (frames added by process_frame_chain) + STATS_ADD(unwinder, frames_read_from_memory, PyList_GET_SIZE(frame_info) - frames_before); + + // If stopped at cached frame, extend with cached continuation (both frames and addresses) + if (stopped_at_cached) { + Py_ssize_t frames_before_cache = PyList_GET_SIZE(frame_info); + int cache_result = frame_cache_lookup_and_extend(unwinder, thread_id, last_profiled_frame, + frame_info, addrs, &num_addrs, + FRAME_CACHE_MAX_FRAMES); + if (cache_result < 0) { + return -1; + } + if (cache_result == 0) { + // Cache miss - continue walking from last_profiled_frame to get the rest + STATS_INC(unwinder, frame_cache_misses); + Py_ssize_t frames_before_walk = PyList_GET_SIZE(frame_info); + if (process_frame_chain(unwinder, last_profiled_frame, chunks, frame_info, 0, gc_frame, + 0, NULL, addrs, &num_addrs, FRAME_CACHE_MAX_FRAMES) < 0) { + return -1; + } + STATS_ADD(unwinder, frames_read_from_memory, PyList_GET_SIZE(frame_info) - frames_before_walk); + } else { + // Partial cache hit + STATS_INC(unwinder, frame_cache_partial_hits); + STATS_ADD(unwinder, frames_read_from_cache, PyList_GET_SIZE(frame_info) - frames_before_cache); + } + } else if (last_profiled_frame == 0) { + // No cache involvement (no last_profiled_frame or cache disabled) + STATS_INC(unwinder, frame_cache_misses); + } + + // Store in cache (frame_cache_store handles truncation if num_addrs > FRAME_CACHE_MAX_FRAMES) + if (frame_cache_store(unwinder, thread_id, frame_info, addrs, num_addrs) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c new file mode 100644 index 00000000000..123e4f5c4d7 --- /dev/null +++ b/Modules/_remote_debugging/module.c @@ -0,0 +1,1108 @@ +/****************************************************************************** + * Remote Debugging Module - Main Module Implementation + * + * This file contains the main module initialization, the RemoteUnwinder + * class implementation, and utility functions. + ******************************************************************************/ + +#include "_remote_debugging.h" +#include "clinic/module.c.h" + +/* ============================================================================ + * STRUCTSEQ TYPE DEFINITIONS + * ============================================================================ */ + +// TaskInfo structseq type +static PyStructSequence_Field TaskInfo_fields[] = { + {"task_id", "Task ID (memory address)"}, + {"task_name", "Task name"}, + {"coroutine_stack", "Coroutine call stack"}, + {"awaited_by", "Tasks awaiting this task"}, + {NULL} +}; + +PyStructSequence_Desc TaskInfo_desc = { + "_remote_debugging.TaskInfo", + "Information about an asyncio task", + TaskInfo_fields, + 4 +}; + +// FrameInfo structseq type +static PyStructSequence_Field FrameInfo_fields[] = { + {"filename", "Source code filename"}, + {"lineno", "Line number"}, + {"funcname", "Function name"}, + {NULL} +}; + +PyStructSequence_Desc FrameInfo_desc = { + "_remote_debugging.FrameInfo", + "Information about a frame", + FrameInfo_fields, + 3 +}; + +// CoroInfo structseq type +static PyStructSequence_Field CoroInfo_fields[] = { + {"call_stack", "Coroutine call stack"}, + {"task_name", "Task name"}, + {NULL} +}; + +PyStructSequence_Desc CoroInfo_desc = { + "_remote_debugging.CoroInfo", + "Information about a coroutine", + CoroInfo_fields, + 2 +}; + +// ThreadInfo structseq type +static PyStructSequence_Field ThreadInfo_fields[] = { + {"thread_id", "Thread ID"}, + {"status", "Thread status (flags: HAS_GIL, ON_CPU, UNKNOWN or legacy enum)"}, + {"frame_info", "Frame information"}, + {NULL} +}; + +PyStructSequence_Desc ThreadInfo_desc = { + "_remote_debugging.ThreadInfo", + "Information about a thread", + ThreadInfo_fields, + 3 +}; + +// InterpreterInfo structseq type +static PyStructSequence_Field InterpreterInfo_fields[] = { + {"interpreter_id", "Interpreter ID"}, + {"threads", "List of threads in this interpreter"}, + {NULL} +}; + +PyStructSequence_Desc InterpreterInfo_desc = { + "_remote_debugging.InterpreterInfo", + "Information about an interpreter", + InterpreterInfo_fields, + 2 +}; + +// AwaitedInfo structseq type +static PyStructSequence_Field AwaitedInfo_fields[] = { + {"thread_id", "Thread ID"}, + {"awaited_by", "List of tasks awaited by this thread"}, + {NULL} +}; + +PyStructSequence_Desc AwaitedInfo_desc = { + "_remote_debugging.AwaitedInfo", + "Information about what a thread is awaiting", + AwaitedInfo_fields, + 2 +}; + +/* ============================================================================ + * UTILITY FUNCTIONS + * ============================================================================ */ + +void +cached_code_metadata_destroy(void *ptr) +{ + CachedCodeMetadata *meta = (CachedCodeMetadata *)ptr; + Py_DECREF(meta->func_name); + Py_DECREF(meta->file_name); + Py_DECREF(meta->linetable); + PyMem_RawFree(meta); +} + +RemoteDebuggingState * +RemoteDebugging_GetState(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (RemoteDebuggingState *)state; +} + +RemoteDebuggingState * +RemoteDebugging_GetStateFromType(PyTypeObject *type) +{ + PyObject *module = PyType_GetModule(type); + assert(module != NULL); + return RemoteDebugging_GetState(module); +} + +RemoteDebuggingState * +RemoteDebugging_GetStateFromObject(PyObject *obj) +{ + RemoteUnwinderObject *unwinder = (RemoteUnwinderObject *)obj; + if (unwinder->cached_state == NULL) { + unwinder->cached_state = RemoteDebugging_GetStateFromType(Py_TYPE(obj)); + } + return unwinder->cached_state; +} + +int +RemoteDebugging_InitState(RemoteDebuggingState *st) +{ + return 0; +} + +int +is_prerelease_version(uint64_t version) +{ + return (version & 0xF0) != 0xF0; +} + +int +validate_debug_offsets(struct _Py_DebugOffsets *debug_offsets) +{ + if (memcmp(debug_offsets->cookie, _Py_Debug_Cookie, sizeof(debug_offsets->cookie)) != 0) { + // The remote is probably running a Python version predating debug offsets. + PyErr_SetString( + PyExc_RuntimeError, + "Can't determine the Python version of the remote process"); + return -1; + } + + // Assume debug offsets could change from one pre-release version to another, + // or one minor version to another, but are stable across patch versions. + if (is_prerelease_version(Py_Version) && Py_Version != debug_offsets->version) { + PyErr_SetString( + PyExc_RuntimeError, + "Can't attach from a pre-release Python interpreter" + " to a process running a different Python version"); + return -1; + } + + if (is_prerelease_version(debug_offsets->version) && Py_Version != debug_offsets->version) { + PyErr_SetString( + PyExc_RuntimeError, + "Can't attach to a pre-release Python interpreter" + " from a process running a different Python version"); + return -1; + } + + unsigned int remote_major = (debug_offsets->version >> 24) & 0xFF; + unsigned int remote_minor = (debug_offsets->version >> 16) & 0xFF; + + if (PY_MAJOR_VERSION != remote_major || PY_MINOR_VERSION != remote_minor) { + PyErr_Format( + PyExc_RuntimeError, + "Can't attach from a Python %d.%d process to a Python %d.%d process", + PY_MAJOR_VERSION, PY_MINOR_VERSION, remote_major, remote_minor); + return -1; + } + + // The debug offsets differ between free threaded and non-free threaded builds. + if (_Py_Debug_Free_Threaded && !debug_offsets->free_threaded) { + PyErr_SetString( + PyExc_RuntimeError, + "Cannot attach from a free-threaded Python process" + " to a process running a non-free-threaded version"); + return -1; + } + + if (!_Py_Debug_Free_Threaded && debug_offsets->free_threaded) { + PyErr_SetString( + PyExc_RuntimeError, + "Cannot attach to a free-threaded Python process" + " from a process running a non-free-threaded version"); + return -1; + } + + return 0; +} + +/* ============================================================================ + * REMOTEUNWINDER CLASS IMPLEMENTATION + * ============================================================================ */ + +/*[clinic input] +module _remote_debugging +class _remote_debugging.RemoteUnwinder "RemoteUnwinderObject *" "&RemoteUnwinder_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=12b4dce200381115]*/ + +/*[clinic input] +@permit_long_summary +@permit_long_docstring_body +_remote_debugging.RemoteUnwinder.__init__ + pid: int + * + all_threads: bool = False + only_active_thread: bool = False + mode: int = 0 + debug: bool = False + skip_non_matching_threads: bool = True + native: bool = False + gc: bool = False + cache_frames: bool = False + stats: bool = False + +Initialize a new RemoteUnwinder object for debugging a remote Python process. + +Args: + pid: Process ID of the target Python process to debug + all_threads: If True, initialize state for all threads in the process. + If False, only initialize for the main thread. + only_active_thread: If True, only sample the thread holding the GIL. + mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL (gil-time). + Cannot be used together with all_threads=True. + debug: If True, chain exceptions to explain the sequence of events that + lead to the exception. + skip_non_matching_threads: If True, skip threads that don't match the selected mode. + If False, include all threads regardless of mode. + native: If True, include artificial "<native>" frames to denote calls to + non-Python code. + gc: If True, include artificial "<GC>" frames to denote active garbage + collection. + cache_frames: If True, enable frame caching optimization to avoid re-reading + unchanged parent frames between samples. + stats: If True, collect statistics about cache hits, memory reads, etc. + Use get_stats() to retrieve the collected statistics. + +The RemoteUnwinder provides functionality to inspect and debug a running Python +process, including examining thread states, stack frames and other runtime data. + +Raises: + PermissionError: If access to the target process is denied + OSError: If unable to attach to the target process or access its memory + RuntimeError: If unable to read debug information from the target process + ValueError: If both all_threads and only_active_thread are True +[clinic start generated code]*/ + +static int +_remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, + int pid, int all_threads, + int only_active_thread, + int mode, int debug, + int skip_non_matching_threads, + int native, int gc, + int cache_frames, int stats) +/*[clinic end generated code: output=b34ef8cce013c975 input=df2221ef114c3d6a]*/ +{ + // Validate that all_threads and only_active_thread are not both True + if (all_threads && only_active_thread) { + PyErr_SetString(PyExc_ValueError, + "all_threads and only_active_thread cannot both be True"); + return -1; + } + +#ifdef Py_GIL_DISABLED + if (only_active_thread) { + PyErr_SetString(PyExc_ValueError, + "only_active_thread is not supported in free-threaded builds"); + return -1; + } +#endif + + self->native = native; + self->gc = gc; + self->cache_frames = cache_frames; + self->collect_stats = stats; + self->stale_invalidation_counter = 0; + self->debug = debug; + self->only_active_thread = only_active_thread; + self->mode = mode; + self->skip_non_matching_threads = skip_non_matching_threads; + self->cached_state = NULL; + self->frame_cache = NULL; + // Initialize stats to zero + memset(&self->stats, 0, sizeof(self->stats)); + if (_Py_RemoteDebug_InitProcHandle(&self->handle, pid) < 0) { + set_exception_cause(self, PyExc_RuntimeError, "Failed to initialize process handle"); + return -1; + } + + self->runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&self->handle); + if (self->runtime_start_address == 0) { + set_exception_cause(self, PyExc_RuntimeError, "Failed to get Python runtime address"); + return -1; + } + + if (_Py_RemoteDebug_ReadDebugOffsets(&self->handle, + &self->runtime_start_address, + &self->debug_offsets) < 0) + { + set_exception_cause(self, PyExc_RuntimeError, "Failed to read debug offsets"); + return -1; + } + + // Validate that the debug offsets are valid + if(validate_debug_offsets(&self->debug_offsets) == -1) { + set_exception_cause(self, PyExc_RuntimeError, "Invalid debug offsets found"); + return -1; + } + + // Try to read async debug offsets, but don't fail if they're not available + self->async_debug_offsets_available = 1; + if (read_async_debug(self) < 0) { + PyErr_Clear(); + memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets)); + self->async_debug_offsets_available = 0; + } + + if (populate_initial_state_data(all_threads, self, self->runtime_start_address, + &self->interpreter_addr ,&self->tstate_addr) < 0) + { + set_exception_cause(self, PyExc_RuntimeError, "Failed to populate initial state data"); + return -1; + } + + self->code_object_cache = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, // keys are stable pointers, don't destroy + cached_code_metadata_destroy, + NULL + ); + if (self->code_object_cache == NULL) { + PyErr_NoMemory(); + set_exception_cause(self, PyExc_MemoryError, "Failed to create code object cache"); + return -1; + } + +#ifdef Py_GIL_DISABLED + // Initialize TLBC cache + self->tlbc_generation = 0; + self->tlbc_cache = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, // keys are stable pointers, don't destroy + tlbc_cache_entry_destroy, + NULL + ); + if (self->tlbc_cache == NULL) { + _Py_hashtable_destroy(self->code_object_cache); + PyErr_NoMemory(); + set_exception_cause(self, PyExc_MemoryError, "Failed to create TLBC cache"); + return -1; + } +#endif + +#if defined(__APPLE__) + self->thread_id_offset = 0; +#endif + +#ifdef MS_WINDOWS + self->win_process_buffer = NULL; + self->win_process_buffer_size = 0; +#endif + + if (cache_frames && frame_cache_init(self) < 0) { + return -1; + } + + // Clear stale last_profiled_frame values from previous profilers + // This prevents us from stopping frame walking early due to stale values + if (cache_frames) { + clear_last_profiled_frames(self); + } + + return 0; +} + +/*[clinic input] +@permit_long_docstring_body +@critical_section +_remote_debugging.RemoteUnwinder.get_stack_trace + +Returns stack traces for all interpreters and threads in process. + +Each element in the returned list is a tuple of (interpreter_id, thread_list), where: +- interpreter_id is the interpreter identifier +- thread_list is a list of tuples (thread_id, frame_list) for threads in that interpreter + - thread_id is the OS thread identifier + - frame_list is a list of tuples (function_name, filename, line_number) representing + the Python stack frames for that thread, ordered from most recent to oldest + +The threads returned depend on the initialization parameters: +- If only_active_thread was True: returns only the thread holding the GIL across all interpreters +- If all_threads was True: returns all threads across all interpreters +- Otherwise: returns only the main thread of each interpreter + +Example: + [ + (0, [ # Main interpreter + (1234, [ + ('process_data', 'worker.py', 127), + ('run_worker', 'worker.py', 45), + ('main', 'app.py', 23) + ]), + (1235, [ + ('handle_request', 'server.py', 89), + ('serve_forever', 'server.py', 52) + ]) + ]), + (1, [ # Sub-interpreter + (1236, [ + ('sub_worker', 'sub.py', 15) + ]) + ]) + ] + +Raises: + RuntimeError: If there is an error copying memory from the target process + OSError: If there is an error accessing the target process + PermissionError: If access to the target process is denied + UnicodeDecodeError: If there is an error decoding strings from the target process + +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=666192b90c69d567 input=bcff01c73cccc1c0]*/ +{ + STATS_INC(self, total_samples); + + PyObject* result = PyList_New(0); + if (!result) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create stack trace result list"); + return NULL; + } + + // Iterate over all interpreters + uintptr_t current_interpreter = self->interpreter_addr; + while (current_interpreter != 0) { + // Read interpreter state to get the interpreter ID + char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &self->handle, + current_interpreter, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer) < 0) { + set_exception_cause(self, PyExc_RuntimeError, "Failed to read interpreter state buffer"); + Py_CLEAR(result); + goto exit; + } + + uintptr_t gc_frame = 0; + if (self->gc) { + gc_frame = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.gc + + self->debug_offsets.gc.frame); + } + + int64_t interpreter_id = GET_MEMBER(int64_t, interp_state_buffer, + self->debug_offsets.interpreter_state.id); + + // Get code object generation from buffer + uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, + self->debug_offsets.interpreter_state.code_object_generation); + + if (code_object_generation != self->code_object_generation) { + self->code_object_generation = code_object_generation; + _Py_hashtable_clear(self->code_object_cache); + } + +#ifdef Py_GIL_DISABLED + // Check TLBC generation and invalidate cache if needed + uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, + self->debug_offsets.interpreter_state.tlbc_generation); + if (current_tlbc_generation != self->tlbc_generation) { + self->tlbc_generation = current_tlbc_generation; + _Py_hashtable_clear(self->tlbc_cache); + } +#endif + + // Create a list to hold threads for this interpreter + PyObject *interpreter_threads = PyList_New(0); + if (!interpreter_threads) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create interpreter threads list"); + Py_CLEAR(result); + goto exit; + } + + // Get the GIL holder for this interpreter (needed for GIL_WAIT logic) + uintptr_t gil_holder_tstate = 0; + int gil_locked = GET_MEMBER(int, interp_state_buffer, + self->debug_offsets.interpreter_state.gil_runtime_state_locked); + if (gil_locked) { + gil_holder_tstate = (uintptr_t)GET_MEMBER(PyThreadState*, interp_state_buffer, + self->debug_offsets.interpreter_state.gil_runtime_state_holder); + } + + uintptr_t current_tstate; + if (self->only_active_thread) { + // Find the GIL holder for THIS interpreter + if (!gil_locked) { + // This interpreter's GIL is not locked, skip it + Py_DECREF(interpreter_threads); + goto next_interpreter; + } + + current_tstate = gil_holder_tstate; + } else if (self->tstate_addr == 0) { + // Get all threads for this interpreter + current_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.threads_head); + } else { + // Target specific thread (only process first interpreter) + current_tstate = self->tstate_addr; + } + + while (current_tstate != 0) { + PyObject* frame_info = unwind_stack_for_thread(self, &current_tstate, + gil_holder_tstate, + gc_frame); + if (!frame_info) { + // Check if this was an intentional skip due to mode-based filtering + if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL) && !PyErr_Occurred()) { + // Thread was skipped due to mode filtering, continue to next thread + continue; + } + // This was an actual error + Py_DECREF(interpreter_threads); + set_exception_cause(self, PyExc_RuntimeError, "Failed to unwind stack for thread"); + Py_CLEAR(result); + goto exit; + } + + if (PyList_Append(interpreter_threads, frame_info) == -1) { + Py_DECREF(frame_info); + Py_DECREF(interpreter_threads); + set_exception_cause(self, PyExc_RuntimeError, "Failed to append thread frame info"); + Py_CLEAR(result); + goto exit; + } + Py_DECREF(frame_info); + + // If targeting specific thread or only active thread, process just one + if (self->tstate_addr || self->only_active_thread) { + break; + } + } + + // Create the InterpreterInfo StructSequence + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)self); + PyObject *interpreter_info = PyStructSequence_New(state->InterpreterInfo_Type); + if (!interpreter_info) { + Py_DECREF(interpreter_threads); + set_exception_cause(self, PyExc_MemoryError, "Failed to create InterpreterInfo"); + Py_CLEAR(result); + goto exit; + } + + PyObject *interp_id = PyLong_FromLongLong(interpreter_id); + if (!interp_id) { + Py_DECREF(interpreter_threads); + Py_DECREF(interpreter_info); + set_exception_cause(self, PyExc_MemoryError, "Failed to create interpreter ID"); + Py_CLEAR(result); + goto exit; + } + + PyStructSequence_SetItem(interpreter_info, 0, interp_id); // steals reference + PyStructSequence_SetItem(interpreter_info, 1, interpreter_threads); // steals reference + + // Add this interpreter to the result list + if (PyList_Append(result, interpreter_info) == -1) { + Py_DECREF(interpreter_info); + set_exception_cause(self, PyExc_RuntimeError, "Failed to append interpreter info"); + Py_CLEAR(result); + goto exit; + } + Py_DECREF(interpreter_info); + +next_interpreter: + + // Get the next interpreter address + current_interpreter = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.next); + + // If we're targeting a specific thread, stop after first interpreter + if (self->tstate_addr != 0) { + break; + } + } + +exit: + // Invalidate cache entries for threads not seen in this sample. + // Only do this every 1024 iterations to avoid performance overhead. + if (self->cache_frames && result) { + if (++self->stale_invalidation_counter >= 1024) { + self->stale_invalidation_counter = 0; + frame_cache_invalidate_stale(self, result); + } + } + _Py_RemoteDebug_ClearCache(&self->handle); + return result; +} + +/*[clinic input] +@permit_long_summary +@permit_long_docstring_body +@critical_section +_remote_debugging.RemoteUnwinder.get_all_awaited_by + +Get all tasks and their awaited_by relationships from the remote process. + +This provides a tree structure showing which tasks are waiting for other tasks. + +For each task, returns: +1. The call stack frames leading to where the task is currently executing +2. The name of the task +3. A list of tasks that this task is waiting for, with their own frames/names/etc + +Returns a list of [frames, task_name, subtasks] where: +- frames: List of (func_name, filename, lineno) showing the call stack +- task_name: String identifier for the task +- subtasks: List of tasks being awaited by this task, in same format + +Raises: + RuntimeError: If AsyncioDebug section is not available in the remote process + MemoryError: If memory allocation fails + OSError: If reading from the remote process fails + +Example output: +[ + # Task c2_root waiting for two subtasks + [ + # Call stack of c2_root + [("c5", "script.py", 10), ("c4", "script.py", 14)], + "c2_root", + [ + # First subtask (sub_main_2) and what it's waiting for + [ + [("c1", "script.py", 23)], + "sub_main_2", + [...] + ], + # Second subtask and its waiters + [...] + ] + ] +] +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=6a49cd345e8aec53 input=307f754cbe38250c]*/ +{ + if (ensure_async_debug_offsets(self) < 0) { + return NULL; + } + + PyObject *result = PyList_New(0); + if (result == NULL) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create awaited_by result list"); + goto result_err; + } + + // Process all threads + if (iterate_threads(self, process_thread_for_awaited_by, result) < 0) { + goto result_err; + } + + uintptr_t head_addr = self->interpreter_addr + + (uintptr_t)self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; + + // On top of a per-thread task lists used by default by asyncio to avoid + // contention, there is also a fallback per-interpreter list of tasks; + // any tasks still pending when a thread is destroyed will be moved to the + // per-interpreter task list. It's unlikely we'll find anything here, but + // interesting for debugging. + if (append_awaited_by(self, 0, head_addr, result)) + { + set_exception_cause(self, PyExc_RuntimeError, "Failed to append interpreter awaited_by in get_all_awaited_by"); + goto result_err; + } + + _Py_RemoteDebug_ClearCache(&self->handle); + return result; + +result_err: + _Py_RemoteDebug_ClearCache(&self->handle); + Py_XDECREF(result); + return NULL; +} + +/*[clinic input] +@permit_long_summary +@permit_long_docstring_body +@critical_section +_remote_debugging.RemoteUnwinder.get_async_stack_trace + +Get the currently running async tasks and their dependency graphs from the remote process. + +This returns information about running tasks and all tasks that are waiting for them, +forming a complete dependency graph for each thread's active task. + +For each thread with a running task, returns the running task plus all tasks that +transitively depend on it (tasks waiting for the running task, tasks waiting for +those tasks, etc.). + +Returns a list of per-thread results, where each thread result contains: +- Thread ID +- List of task information for the running task and all its waiters + +Each task info contains: +- Task ID (memory address) +- Task name +- Call stack frames: List of (func_name, filename, lineno) +- List of tasks waiting for this task (recursive structure) + +Raises: + RuntimeError: If AsyncioDebug section is not available in the target process + MemoryError: If memory allocation fails + OSError: If reading from the remote process fails + +Example output (similar structure to get_all_awaited_by but only for running tasks): +[ + # Thread 140234 results + (140234, [ + # Running task and its complete waiter dependency graph + (4345585712, 'main_task', + [("run_server", "server.py", 127), ("main", "app.py", 23)], + [ + # Tasks waiting for main_task + (4345585800, 'worker_1', [...], [...]), + (4345585900, 'worker_2', [...], [...]) + ]) + ]) +] + +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=6433d52b55e87bbe input=6129b7d509a887c9]*/ +{ + if (ensure_async_debug_offsets(self) < 0) { + return NULL; + } + + PyObject *result = PyList_New(0); + if (result == NULL) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create result list in get_async_stack_trace"); + return NULL; + } + + // Process all threads + if (iterate_threads(self, process_thread_for_async_stack_trace, result) < 0) { + goto result_err; + } + + _Py_RemoteDebug_ClearCache(&self->handle); + return result; +result_err: + _Py_RemoteDebug_ClearCache(&self->handle); + Py_XDECREF(result); + return NULL; +} + +/*[clinic input] +@permit_long_docstring_body +@critical_section +_remote_debugging.RemoteUnwinder.get_stats + +Get collected statistics about profiling performance. + +Returns a dictionary containing statistics about cache performance, +memory reads, and other profiling metrics. Only available if the +RemoteUnwinder was created with stats=True. + +Returns: + dict: A dictionary containing: + - total_samples: Total number of get_stack_trace calls + - frame_cache_hits: Full cache hits (entire stack unchanged) + - frame_cache_misses: Cache misses requiring full walk + - frame_cache_partial_hits: Partial hits (stopped at cached frame) + - frames_read_from_cache: Total frames retrieved from cache + - frames_read_from_memory: Total frames read from remote memory + - memory_reads: Total remote memory read operations + - memory_bytes_read: Total bytes read from remote memory + - code_object_cache_hits: Code object cache hits + - code_object_cache_misses: Code object cache misses + - stale_cache_invalidations: Times stale cache entries were cleared + - frame_cache_hit_rate: Percentage of samples that hit the cache + - code_object_cache_hit_rate: Percentage of code object lookups that hit cache + +Raises: + RuntimeError: If stats collection was not enabled (stats=False) +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=21e36477122be2a0 input=75fef4134c12a8c9]*/ +{ + if (!self->collect_stats) { + PyErr_SetString(PyExc_RuntimeError, + "Statistics collection was not enabled. " + "Create RemoteUnwinder with stats=True to collect statistics."); + return NULL; + } + + PyObject *result = PyDict_New(); + if (!result) { + return NULL; + } + +#define ADD_STAT(name) do { \ + PyObject *val = PyLong_FromUnsignedLongLong(self->stats.name); \ + if (!val || PyDict_SetItemString(result, #name, val) < 0) { \ + Py_XDECREF(val); \ + Py_DECREF(result); \ + return NULL; \ + } \ + Py_DECREF(val); \ +} while(0) + + ADD_STAT(total_samples); + ADD_STAT(frame_cache_hits); + ADD_STAT(frame_cache_misses); + ADD_STAT(frame_cache_partial_hits); + ADD_STAT(frames_read_from_cache); + ADD_STAT(frames_read_from_memory); + ADD_STAT(memory_reads); + ADD_STAT(memory_bytes_read); + ADD_STAT(code_object_cache_hits); + ADD_STAT(code_object_cache_misses); + ADD_STAT(stale_cache_invalidations); + +#undef ADD_STAT + + // Calculate and add derived statistics + // Hit rate is calculated as (hits + partial_hits) / total_cache_lookups + double frame_cache_hit_rate = 0.0; + uint64_t total_cache_lookups = self->stats.frame_cache_hits + self->stats.frame_cache_partial_hits + self->stats.frame_cache_misses; + if (total_cache_lookups > 0) { + frame_cache_hit_rate = 100.0 * (double)(self->stats.frame_cache_hits + self->stats.frame_cache_partial_hits) + / (double)total_cache_lookups; + } + PyObject *hit_rate = PyFloat_FromDouble(frame_cache_hit_rate); + if (!hit_rate || PyDict_SetItemString(result, "frame_cache_hit_rate", hit_rate) < 0) { + Py_XDECREF(hit_rate); + Py_DECREF(result); + return NULL; + } + Py_DECREF(hit_rate); + + double code_object_hit_rate = 0.0; + uint64_t total_code_lookups = self->stats.code_object_cache_hits + self->stats.code_object_cache_misses; + if (total_code_lookups > 0) { + code_object_hit_rate = 100.0 * (double)self->stats.code_object_cache_hits / (double)total_code_lookups; + } + PyObject *code_hit_rate = PyFloat_FromDouble(code_object_hit_rate); + if (!code_hit_rate || PyDict_SetItemString(result, "code_object_cache_hit_rate", code_hit_rate) < 0) { + Py_XDECREF(code_hit_rate); + Py_DECREF(result); + return NULL; + } + Py_DECREF(code_hit_rate); + + return result; +} + +static PyMethodDef RemoteUnwinder_methods[] = { + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STATS_METHODDEF + {NULL, NULL} +}; + +static void +RemoteUnwinder_dealloc(PyObject *op) +{ + RemoteUnwinderObject *self = RemoteUnwinder_CAST(op); + PyTypeObject *tp = Py_TYPE(self); + if (self->code_object_cache) { + _Py_hashtable_destroy(self->code_object_cache); + } +#ifdef MS_WINDOWS + if(self->win_process_buffer != NULL) { + PyMem_Free(self->win_process_buffer); + } +#endif + +#ifdef Py_GIL_DISABLED + if (self->tlbc_cache) { + _Py_hashtable_destroy(self->tlbc_cache); + } +#endif + if (self->handle.pid != 0) { + _Py_RemoteDebug_ClearCache(&self->handle); + _Py_RemoteDebug_CleanupProcHandle(&self->handle); + } + frame_cache_cleanup(self); + PyObject_Del(self); + Py_DECREF(tp); +} + +static PyType_Slot RemoteUnwinder_slots[] = { + {Py_tp_doc, (void *)"RemoteUnwinder(pid): Inspect stack of a remote Python process."}, + {Py_tp_methods, RemoteUnwinder_methods}, + {Py_tp_init, _remote_debugging_RemoteUnwinder___init__}, + {Py_tp_dealloc, RemoteUnwinder_dealloc}, + {0, NULL} +}; + +static PyType_Spec RemoteUnwinder_spec = { + .name = "_remote_debugging.RemoteUnwinder", + .basicsize = sizeof(RemoteUnwinderObject), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = RemoteUnwinder_slots, +}; + +/* ============================================================================ + * MODULE INITIALIZATION + * ============================================================================ */ + +static int +_remote_debugging_exec(PyObject *m) +{ + RemoteDebuggingState *st = RemoteDebugging_GetState(m); +#define CREATE_TYPE(mod, type, spec) \ + do { \ + type = (PyTypeObject *)PyType_FromMetaclass(NULL, mod, spec, NULL); \ + if (type == NULL) { \ + return -1; \ + } \ + } while (0) + + CREATE_TYPE(m, st->RemoteDebugging_Type, &RemoteUnwinder_spec); + + if (PyModule_AddType(m, st->RemoteDebugging_Type) < 0) { + return -1; + } + + // Initialize structseq types + st->TaskInfo_Type = PyStructSequence_NewType(&TaskInfo_desc); + if (st->TaskInfo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, st->TaskInfo_Type) < 0) { + return -1; + } + + st->FrameInfo_Type = PyStructSequence_NewType(&FrameInfo_desc); + if (st->FrameInfo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, st->FrameInfo_Type) < 0) { + return -1; + } + + st->CoroInfo_Type = PyStructSequence_NewType(&CoroInfo_desc); + if (st->CoroInfo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, st->CoroInfo_Type) < 0) { + return -1; + } + + st->ThreadInfo_Type = PyStructSequence_NewType(&ThreadInfo_desc); + if (st->ThreadInfo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, st->ThreadInfo_Type) < 0) { + return -1; + } + + st->InterpreterInfo_Type = PyStructSequence_NewType(&InterpreterInfo_desc); + if (st->InterpreterInfo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, st->InterpreterInfo_Type) < 0) { + return -1; + } + + st->AwaitedInfo_Type = PyStructSequence_NewType(&AwaitedInfo_desc); + if (st->AwaitedInfo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, st->AwaitedInfo_Type) < 0) { + return -1; + } +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + int rc = PyModule_AddIntConstant(m, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + if (rc < 0) { + return -1; + } + + // Add thread status flag constants + if (PyModule_AddIntConstant(m, "THREAD_STATUS_HAS_GIL", THREAD_STATUS_HAS_GIL) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_ON_CPU", THREAD_STATUS_ON_CPU) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_UNKNOWN", THREAD_STATUS_UNKNOWN) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_GIL_REQUESTED", THREAD_STATUS_GIL_REQUESTED) < 0) { + return -1; + } + + if (RemoteDebugging_InitState(st) < 0) { + return -1; + } + return 0; +} + +static int +remote_debugging_traverse(PyObject *mod, visitproc visit, void *arg) +{ + RemoteDebuggingState *state = RemoteDebugging_GetState(mod); + Py_VISIT(state->RemoteDebugging_Type); + Py_VISIT(state->TaskInfo_Type); + Py_VISIT(state->FrameInfo_Type); + Py_VISIT(state->CoroInfo_Type); + Py_VISIT(state->ThreadInfo_Type); + Py_VISIT(state->InterpreterInfo_Type); + Py_VISIT(state->AwaitedInfo_Type); + return 0; +} + +static int +remote_debugging_clear(PyObject *mod) +{ + RemoteDebuggingState *state = RemoteDebugging_GetState(mod); + Py_CLEAR(state->RemoteDebugging_Type); + Py_CLEAR(state->TaskInfo_Type); + Py_CLEAR(state->FrameInfo_Type); + Py_CLEAR(state->CoroInfo_Type); + Py_CLEAR(state->ThreadInfo_Type); + Py_CLEAR(state->InterpreterInfo_Type); + Py_CLEAR(state->AwaitedInfo_Type); + return 0; +} + +static void +remote_debugging_free(void *mod) +{ + (void)remote_debugging_clear((PyObject *)mod); +} + +static PyModuleDef_Slot remote_debugging_slots[] = { + {Py_mod_exec, _remote_debugging_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static PyMethodDef remote_debugging_methods[] = { + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef remote_debugging_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_remote_debugging", + .m_size = sizeof(RemoteDebuggingState), + .m_methods = remote_debugging_methods, + .m_slots = remote_debugging_slots, + .m_traverse = remote_debugging_traverse, + .m_clear = remote_debugging_clear, + .m_free = remote_debugging_free, +}; + +PyMODINIT_FUNC +PyInit__remote_debugging(void) +{ + return PyModuleDef_Init(&remote_debugging_module); +} diff --git a/Modules/_remote_debugging/object_reading.c b/Modules/_remote_debugging/object_reading.c new file mode 100644 index 00000000000..2f465ca0cac --- /dev/null +++ b/Modules/_remote_debugging/object_reading.c @@ -0,0 +1,247 @@ +/****************************************************************************** + * Remote Debugging Module - Object Reading Functions + * + * This file contains functions for reading Python objects from remote + * process memory, including strings, bytes, and integers. + ******************************************************************************/ + +#include "_remote_debugging.h" + +/* ============================================================================ + * MEMORY READING FUNCTIONS + * ============================================================================ */ + +#define DEFINE_MEMORY_READER(type_name, c_type, error_msg) \ +int \ +read_##type_name(RemoteUnwinderObject *unwinder, uintptr_t address, c_type *result) \ +{ \ + int res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(c_type), result); \ + if (res < 0) { \ + set_exception_cause(unwinder, PyExc_RuntimeError, error_msg); \ + return -1; \ + } \ + return 0; \ +} + +DEFINE_MEMORY_READER(ptr, uintptr_t, "Failed to read pointer from remote memory") +DEFINE_MEMORY_READER(Py_ssize_t, Py_ssize_t, "Failed to read Py_ssize_t from remote memory") +DEFINE_MEMORY_READER(char, char, "Failed to read char from remote memory") + +int +read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr) +{ + if (read_ptr(unwinder, address, ptr_addr)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read Python pointer"); + return -1; + } + *ptr_addr &= ~Py_TAG_BITS; + return 0; +} + +/* ============================================================================ + * PYTHON OBJECT READING FUNCTIONS + * ============================================================================ */ + +PyObject * +read_py_str( + RemoteUnwinderObject *unwinder, + uintptr_t address, + Py_ssize_t max_len +) { + PyObject *result = NULL; + char *buf = NULL; + + // Read the entire PyUnicodeObject at once + char unicode_obj[SIZEOF_UNICODE_OBJ]; + int res = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_UNICODE_OBJ, + unicode_obj + ); + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyUnicodeObject"); + goto err; + } + + Py_ssize_t len = GET_MEMBER(Py_ssize_t, unicode_obj, unwinder->debug_offsets.unicode_object.length); + if (len < 0 || len > max_len) { + PyErr_Format(PyExc_RuntimeError, + "Invalid string length (%zd) at 0x%lx", len, address); + set_exception_cause(unwinder, PyExc_RuntimeError, "Invalid string length in remote Unicode object"); + return NULL; + } + + buf = (char *)PyMem_RawMalloc(len+1); + if (buf == NULL) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate buffer for string reading"); + return NULL; + } + + size_t offset = (size_t)unwinder->debug_offsets.unicode_object.asciiobject_size; + res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read string data from remote memory"); + goto err; + } + buf[len] = '\0'; + + result = PyUnicode_FromStringAndSize(buf, len); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create PyUnicode from remote string data"); + goto err; + } + + PyMem_RawFree(buf); + assert(result != NULL); + return result; + +err: + if (buf != NULL) { + PyMem_RawFree(buf); + } + return NULL; +} + +PyObject * +read_py_bytes( + RemoteUnwinderObject *unwinder, + uintptr_t address, + Py_ssize_t max_len +) { + PyObject *result = NULL; + char *buf = NULL; + + // Read the entire PyBytesObject at once + char bytes_obj[SIZEOF_BYTES_OBJ]; + int res = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_BYTES_OBJ, + bytes_obj + ); + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyBytesObject"); + goto err; + } + + Py_ssize_t len = GET_MEMBER(Py_ssize_t, bytes_obj, unwinder->debug_offsets.bytes_object.ob_size); + if (len < 0 || len > max_len) { + PyErr_Format(PyExc_RuntimeError, + "Invalid bytes length (%zd) at 0x%lx", len, address); + set_exception_cause(unwinder, PyExc_RuntimeError, "Invalid bytes length in remote bytes object"); + return NULL; + } + + buf = (char *)PyMem_RawMalloc(len+1); + if (buf == NULL) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate buffer for bytes reading"); + return NULL; + } + + size_t offset = (size_t)unwinder->debug_offsets.bytes_object.ob_sval; + res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read bytes data from remote memory"); + goto err; + } + buf[len] = '\0'; + + result = PyBytes_FromStringAndSize(buf, len); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create PyBytes from remote bytes data"); + goto err; + } + + PyMem_RawFree(buf); + assert(result != NULL); + return result; + +err: + if (buf != NULL) { + PyMem_RawFree(buf); + } + return NULL; +} + +long +read_py_long( + RemoteUnwinderObject *unwinder, + uintptr_t address +) +{ + unsigned int shift = PYLONG_BITS_IN_DIGIT; + + // Read the entire PyLongObject at once + char long_obj[SIZEOF_LONG_OBJ]; + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + (size_t)unwinder->debug_offsets.long_object.size, + long_obj); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyLongObject"); + return -1; + } + + uintptr_t lv_tag = GET_MEMBER(uintptr_t, long_obj, unwinder->debug_offsets.long_object.lv_tag); + int negative = (lv_tag & 3) == 2; + Py_ssize_t size = lv_tag >> 3; + + if (size == 0) { + return 0; + } + + // If the long object has inline digits, use them directly + digit *digits; + if (size <= _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS) { + // For small integers, digits are inline in the long_value.ob_digit array + digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for small PyLong"); + return -1; + } + memcpy(digits, long_obj + unwinder->debug_offsets.long_object.ob_digit, size * sizeof(digit)); + } else { + // For larger integers, we need to read the digits separately + digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for large PyLong"); + return -1; + } + + bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address + (uintptr_t)unwinder->debug_offsets.long_object.ob_digit, + sizeof(digit) * size, + digits + ); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyLong digits from remote memory"); + goto error; + } + } + + long long value = 0; + + // In theory this can overflow, but because of llvm/llvm-project#16778 + // we can't use __builtin_mul_overflow because it fails to link with + // __muloti4 on aarch64. In practice this is fine because all we're + // testing here are task numbers that would fit in a single byte. + for (Py_ssize_t i = 0; i < size; ++i) { + long long factor = digits[i] * (1UL << (Py_ssize_t)(shift * i)); + value += factor; + } + PyMem_RawFree(digits); + if (negative) { + value *= -1; + } + return (long)value; +error: + PyMem_RawFree(digits); + return -1; +} diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c new file mode 100644 index 00000000000..774338f9dc2 --- /dev/null +++ b/Modules/_remote_debugging/threads.c @@ -0,0 +1,457 @@ +/****************************************************************************** + * Remote Debugging Module - Thread Functions + * + * This file contains functions for iterating threads and determining + * thread status in remote process memory. + ******************************************************************************/ + +#include "_remote_debugging.h" + +#ifndef MS_WINDOWS +#include <unistd.h> +#endif + +/* ============================================================================ + * THREAD ITERATION FUNCTIONS + * ============================================================================ */ + +int +iterate_threads( + RemoteUnwinderObject *unwinder, + thread_processor_func processor, + void *context +) { + uintptr_t thread_state_addr; + unsigned long tid = 0; + + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + unwinder->interpreter_addr + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main, + sizeof(void*), + &thread_state_addr)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read main thread state"); + return -1; + } + + while (thread_state_addr != 0) { + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + thread_state_addr + (uintptr_t)unwinder->debug_offsets.thread_state.native_thread_id, + sizeof(tid), + &tid)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread ID"); + return -1; + } + + // Call the processor function for this thread + if (processor(unwinder, thread_state_addr, tid, context) < 0) { + return -1; + } + + // Move to next thread + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + thread_state_addr + (uintptr_t)unwinder->debug_offsets.thread_state.next, + sizeof(void*), + &thread_state_addr)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read next thread state"); + return -1; + } + } + + return 0; +} + +/* ============================================================================ + * INTERPRETER STATE AND THREAD DISCOVERY FUNCTIONS + * ============================================================================ */ + +int +populate_initial_state_data( + int all_threads, + RemoteUnwinderObject *unwinder, + uintptr_t runtime_start_address, + uintptr_t *interpreter_state, + uintptr_t *tstate +) { + uintptr_t interpreter_state_list_head = + (uintptr_t)unwinder->debug_offsets.runtime_state.interpreters_head; + + uintptr_t address_of_interpreter_state; + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + runtime_start_address + interpreter_state_list_head, + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter state address"); + return -1; + } + + if (address_of_interpreter_state == 0) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Interpreter state is NULL"); + return -1; + } + + *interpreter_state = address_of_interpreter_state; + + if (all_threads) { + *tstate = 0; + return 0; + } + + uintptr_t address_of_thread = address_of_interpreter_state + + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main; + + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address_of_thread, + sizeof(void*), + tstate) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read main thread state address"); + return -1; + } + + return 0; +} + +int +find_running_frame( + RemoteUnwinderObject *unwinder, + uintptr_t address_of_thread, + uintptr_t *frame +) { + if ((void*)address_of_thread != NULL) { + int err = read_ptr( + unwinder, + address_of_thread + (uintptr_t)unwinder->debug_offsets.thread_state.current_frame, + frame); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read current frame pointer"); + return -1; + } + return 0; + } + + *frame = (uintptr_t)NULL; + return 0; +} + +/* ============================================================================ + * THREAD STATUS FUNCTIONS + * ============================================================================ */ + +int +get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread_id) { +#if defined(__APPLE__) && TARGET_OS_OSX + if (unwinder->thread_id_offset == 0) { + uint64_t *tids = (uint64_t *)PyMem_Malloc(MAX_NATIVE_THREADS * sizeof(uint64_t)); + if (!tids) { + PyErr_NoMemory(); + return -1; + } + int n = proc_pidinfo(unwinder->handle.pid, PROC_PIDLISTTHREADS, 0, tids, MAX_NATIVE_THREADS * sizeof(uint64_t)) / sizeof(uint64_t); + if (n <= 0) { + PyMem_Free(tids); + return THREAD_STATE_UNKNOWN; + } + uint64_t min_offset = UINT64_MAX; + for (int i = 0; i < n; i++) { + uint64_t offset = tids[i] - pthread_id; + if (offset < min_offset) { + min_offset = offset; + } + } + unwinder->thread_id_offset = min_offset; + PyMem_Free(tids); + } + struct proc_threadinfo ti; + uint64_t tid_with_offset = pthread_id + unwinder->thread_id_offset; + if (proc_pidinfo(unwinder->handle.pid, PROC_PIDTHREADINFO, tid_with_offset, &ti, sizeof(ti)) != sizeof(ti)) { + return THREAD_STATE_UNKNOWN; + } + if (ti.pth_run_state == TH_STATE_RUNNING) { + return THREAD_STATE_RUNNING; + } + return THREAD_STATE_IDLE; +#elif defined(__linux__) + char stat_path[256]; + char buffer[2048] = ""; + + snprintf(stat_path, sizeof(stat_path), "/proc/%d/task/%lu/stat", unwinder->handle.pid, tid); + + int fd = open(stat_path, O_RDONLY); + if (fd == -1) { + return THREAD_STATE_UNKNOWN; + } + + if (read(fd, buffer, 2047) == 0) { + close(fd); + return THREAD_STATE_UNKNOWN; + } + close(fd); + + char *p = strchr(buffer, ')'); + if (!p) { + return THREAD_STATE_UNKNOWN; + } + + p += 2; // Skip ") " + if (*p == ' ') { + p++; + } + + switch (*p) { + case 'R': // Running + return THREAD_STATE_RUNNING; + case 'S': // Interruptible sleep + case 'D': // Uninterruptible sleep + case 'T': // Stopped + case 'Z': // Zombie + case 'I': // Idle kernel thread + return THREAD_STATE_IDLE; + default: + return THREAD_STATE_UNKNOWN; + } +#elif defined(MS_WINDOWS) + ULONG n; + NTSTATUS status = NtQuerySystemInformation( + SystemProcessInformation, + unwinder->win_process_buffer, + unwinder->win_process_buffer_size, + &n + ); + if (status == STATUS_INFO_LENGTH_MISMATCH) { + // Buffer was too small so we reallocate a larger one and try again. + unwinder->win_process_buffer_size = n; + PVOID new_buffer = PyMem_Realloc(unwinder->win_process_buffer, n); + if (!new_buffer) { + return -1; + } + unwinder->win_process_buffer = new_buffer; + return get_thread_status(unwinder, tid, pthread_id); + } + if (status != STATUS_SUCCESS) { + return -1; + } + + SYSTEM_PROCESS_INFORMATION *pi = (SYSTEM_PROCESS_INFORMATION *)unwinder->win_process_buffer; + while ((ULONG)(ULONG_PTR)pi->UniqueProcessId != unwinder->handle.pid) { + if (pi->NextEntryOffset == 0) { + // We didn't find the process + return -1; + } + pi = (SYSTEM_PROCESS_INFORMATION *)(((BYTE *)pi) + pi->NextEntryOffset); + } + + SYSTEM_THREAD_INFORMATION *ti = (SYSTEM_THREAD_INFORMATION *)((char *)pi + sizeof(SYSTEM_PROCESS_INFORMATION)); + for (size_t i = 0; i < pi->NumberOfThreads; i++, ti++) { + if (ti->ClientId.UniqueThread == (HANDLE)tid) { + return ti->ThreadState != WIN32_THREADSTATE_RUNNING ? THREAD_STATE_IDLE : THREAD_STATE_RUNNING; + } + } + + return -1; +#else + return THREAD_STATE_UNKNOWN; +#endif +} + +/* ============================================================================ + * STACK UNWINDING FUNCTIONS + * ============================================================================ */ + +typedef struct { + unsigned int initialized:1; + unsigned int bound:1; + unsigned int unbound:1; + unsigned int bound_gilstate:1; + unsigned int active:1; + unsigned int finalizing:1; + unsigned int cleared:1; + unsigned int finalized:1; + unsigned int :24; +} _thread_status; + +PyObject* +unwind_stack_for_thread( + RemoteUnwinderObject *unwinder, + uintptr_t *current_tstate, + uintptr_t gil_holder_tstate, + uintptr_t gc_frame +) { + PyObject *frame_info = NULL; + PyObject *thread_id = NULL; + PyObject *result = NULL; + StackChunkList chunks = {0}; + + char ts[SIZEOF_THREAD_STATE]; + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, *current_tstate, (size_t)unwinder->debug_offsets.thread_state.size, ts); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.thread_state.size); + + long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); + + // Read GC collecting state from the interpreter (before any skip checks) + uintptr_t interp_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.interp); + + // Read the GC runtime state from the interpreter state + uintptr_t gc_addr = interp_addr + unwinder->debug_offsets.interpreter_state.gc; + char gc_state[SIZEOF_GC_RUNTIME_STATE]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, gc_addr, unwinder->debug_offsets.gc.size, gc_state) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read GC state"); + goto error; + } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.gc.size); + + // Calculate thread status using flags (always) + int status_flags = 0; + + // Check GIL status + int has_gil = 0; + int gil_requested = 0; +#ifdef Py_GIL_DISABLED + int active = GET_MEMBER(_thread_status, ts, unwinder->debug_offsets.thread_state.status).active; + has_gil = active; +#else + // Read holds_gil directly from thread state + has_gil = GET_MEMBER(int, ts, unwinder->debug_offsets.thread_state.holds_gil); + + // Check if thread is actively requesting the GIL + if (unwinder->debug_offsets.thread_state.gil_requested != 0) { + gil_requested = GET_MEMBER(int, ts, unwinder->debug_offsets.thread_state.gil_requested); + } + + // Set GIL_REQUESTED flag if thread is waiting + if (!has_gil && gil_requested) { + status_flags |= THREAD_STATUS_GIL_REQUESTED; + } +#endif + if (has_gil) { + status_flags |= THREAD_STATUS_HAS_GIL; + // gh-142207 for remote debugging. + gil_requested = 0; + } + + // Check CPU status + long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); + + // Optimization: only check CPU status if needed by mode because it's expensive + int cpu_status = -1; + if (unwinder->mode == PROFILING_MODE_CPU || unwinder->mode == PROFILING_MODE_ALL) { + cpu_status = get_thread_status(unwinder, tid, pthread_id); + } + + if (cpu_status == -1) { + status_flags |= THREAD_STATUS_UNKNOWN; + } else if (cpu_status == THREAD_STATE_RUNNING) { + status_flags |= THREAD_STATUS_ON_CPU; + } + + // Check if we should skip this thread based on mode + int should_skip = 0; + if (unwinder->skip_non_matching_threads) { + if (unwinder->mode == PROFILING_MODE_CPU) { + // Skip if not on CPU + should_skip = !(status_flags & THREAD_STATUS_ON_CPU); + } else if (unwinder->mode == PROFILING_MODE_GIL) { + // Skip if doesn't have GIL + should_skip = !(status_flags & THREAD_STATUS_HAS_GIL); + } + // PROFILING_MODE_WALL and PROFILING_MODE_ALL never skip + } + + if (should_skip) { + // Advance to next thread and return NULL to skip processing + *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); + return NULL; + } + + uintptr_t frame_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.current_frame); + uintptr_t base_frame_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.base_frame); + + frame_info = PyList_New(0); + if (!frame_info) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create frame info list"); + goto error; + } + + // In cache mode, copying stack chunks is more expensive than direct memory reads + if (!unwinder->cache_frames) { + if (copy_stack_chunks(unwinder, *current_tstate, &chunks) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to copy stack chunks"); + goto error; + } + } + + if (unwinder->cache_frames) { + // Use cache to avoid re-reading unchanged parent frames + uintptr_t last_profiled_frame = GET_MEMBER(uintptr_t, ts, + unwinder->debug_offsets.thread_state.last_profiled_frame); + if (collect_frames_with_cache(unwinder, frame_addr, &chunks, frame_info, + gc_frame, last_profiled_frame, tid) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to collect frames"); + goto error; + } + // Update last_profiled_frame for next sample + uintptr_t lpf_addr = *current_tstate + unwinder->debug_offsets.thread_state.last_profiled_frame; + if (_Py_RemoteDebug_WriteRemoteMemory(&unwinder->handle, lpf_addr, + sizeof(uintptr_t), &frame_addr) < 0) { + PyErr_Clear(); // Non-fatal + } + } else { + // No caching - process entire frame chain with base_frame validation + if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info, + base_frame_addr, gc_frame, 0, NULL, NULL, NULL, 0) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process frame chain"); + goto error; + } + } + + *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); + + thread_id = PyLong_FromLongLong(tid); + if (thread_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + goto error; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result = PyStructSequence_New(state->ThreadInfo_Type); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create ThreadInfo"); + goto error; + } + + // Always use status_flags + PyObject *py_status = PyLong_FromLong(status_flags); + if (py_status == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread status"); + goto error; + } + + // py_status contains status flags (bitfield) + PyStructSequence_SetItem(result, 0, thread_id); + PyStructSequence_SetItem(result, 1, py_status); // Steals reference + PyStructSequence_SetItem(result, 2, frame_info); // Steals reference + + cleanup_stack_chunks(&chunks); + return result; + +error: + Py_XDECREF(frame_info); + Py_XDECREF(thread_id); + Py_XDECREF(result); + cleanup_stack_chunks(&chunks); + return NULL; +} diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c deleted file mode 100644 index 9314ddd9bed..00000000000 --- a/Modules/_remote_debugging_module.c +++ /dev/null @@ -1,1800 +0,0 @@ -#define _GNU_SOURCE - -#include <errno.h> -#include <fcntl.h> -#include <stddef.h> -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif -#include "Python.h" -#include <internal/pycore_debug_offsets.h> // _Py_DebugOffsets -#include <internal/pycore_frame.h> // FRAME_SUSPENDED_YIELD_FROM -#include <internal/pycore_interpframe.h> // FRAME_OWNED_BY_CSTACK -#include <internal/pycore_llist.h> // struct llist_node -#include <internal/pycore_stackref.h> // Py_TAG_BITS -#include "../Python/remote_debug.h" - -#ifndef HAVE_PROCESS_VM_READV -# define HAVE_PROCESS_VM_READV 0 -#endif - -struct _Py_AsyncioModuleDebugOffsets { - struct _asyncio_task_object { - uint64_t size; - uint64_t task_name; - uint64_t task_awaited_by; - uint64_t task_is_task; - uint64_t task_awaited_by_is_set; - uint64_t task_coro; - uint64_t task_node; - } asyncio_task_object; - struct _asyncio_interpreter_state { - uint64_t size; - uint64_t asyncio_tasks_head; - } asyncio_interpreter_state; - struct _asyncio_thread_state { - uint64_t size; - uint64_t asyncio_running_loop; - uint64_t asyncio_running_task; - uint64_t asyncio_tasks_head; - } asyncio_thread_state; -}; - -// Helper to chain exceptions and avoid repetitions -static void -chain_exceptions(PyObject *exception, const char *string) -{ - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(exception, string); - _PyErr_ChainExceptions1(exc); -} - -// Get the PyAsyncioDebug section address for any platform -static uintptr_t -_Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) -{ - uintptr_t address; - -#ifdef MS_WINDOWS - // On Windows, search for asyncio debug in executable or DLL - address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio"); -#elif defined(__linux__) - // On Linux, search for asyncio debug in executable or DLL - address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); -#elif defined(__APPLE__) && TARGET_OS_OSX - // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); - if (address == 0) { - PyErr_Clear(); - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); - } -#else - Py_UNREACHABLE(); -#endif - - return address; -} - -static inline int -read_ptr(proc_handle_t *handle, uintptr_t address, uintptr_t *ptr_addr) -{ - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(void*), ptr_addr); - if (result < 0) { - return -1; - } - return 0; -} - -static inline int -read_Py_ssize_t(proc_handle_t *handle, uintptr_t address, Py_ssize_t *size) -{ - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(Py_ssize_t), size); - if (result < 0) { - return -1; - } - return 0; -} - -static int -read_py_ptr(proc_handle_t *handle, uintptr_t address, uintptr_t *ptr_addr) -{ - if (read_ptr(handle, address, ptr_addr)) { - return -1; - } - *ptr_addr &= ~Py_TAG_BITS; - return 0; -} - -static int -read_char(proc_handle_t *handle, uintptr_t address, char *result) -{ - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(char), result); - if (res < 0) { - return -1; - } - return 0; -} - -static int -read_sized_int(proc_handle_t *handle, uintptr_t address, void *result, size_t size) -{ - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, size, result); - if (res < 0) { - return -1; - } - return 0; -} - -static int -read_unsigned_long(proc_handle_t *handle, uintptr_t address, unsigned long *result) -{ - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(unsigned long), result); - if (res < 0) { - return -1; - } - return 0; -} - -static int -read_pyobj(proc_handle_t *handle, uintptr_t address, PyObject *ptr_addr) -{ - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(PyObject), ptr_addr); - if (res < 0) { - return -1; - } - return 0; -} - -static PyObject * -read_py_str( - proc_handle_t *handle, - _Py_DebugOffsets* debug_offsets, - uintptr_t address, - Py_ssize_t max_len -) { - PyObject *result = NULL; - char *buf = NULL; - - Py_ssize_t len; - int res = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + debug_offsets->unicode_object.length, - sizeof(Py_ssize_t), - &len - ); - if (res < 0) { - goto err; - } - - buf = (char *)PyMem_RawMalloc(len+1); - if (buf == NULL) { - PyErr_NoMemory(); - return NULL; - } - - size_t offset = debug_offsets->unicode_object.asciiobject_size; - res = _Py_RemoteDebug_ReadRemoteMemory(handle, address + offset, len, buf); - if (res < 0) { - goto err; - } - buf[len] = '\0'; - - result = PyUnicode_FromStringAndSize(buf, len); - if (result == NULL) { - goto err; - } - - PyMem_RawFree(buf); - assert(result != NULL); - return result; - -err: - if (buf != NULL) { - PyMem_RawFree(buf); - } - return NULL; -} - -static PyObject * -read_py_bytes( - proc_handle_t *handle, - _Py_DebugOffsets* debug_offsets, - uintptr_t address -) { - PyObject *result = NULL; - char *buf = NULL; - - Py_ssize_t len; - int res = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + debug_offsets->bytes_object.ob_size, - sizeof(Py_ssize_t), - &len - ); - if (res < 0) { - goto err; - } - - buf = (char *)PyMem_RawMalloc(len+1); - if (buf == NULL) { - PyErr_NoMemory(); - return NULL; - } - - size_t offset = debug_offsets->bytes_object.ob_sval; - res = _Py_RemoteDebug_ReadRemoteMemory(handle, address + offset, len, buf); - if (res < 0) { - goto err; - } - buf[len] = '\0'; - - result = PyBytes_FromStringAndSize(buf, len); - if (result == NULL) { - goto err; - } - - PyMem_RawFree(buf); - assert(result != NULL); - return result; - -err: - if (buf != NULL) { - PyMem_RawFree(buf); - } - return NULL; -} - - - -static long -read_py_long(proc_handle_t *handle, _Py_DebugOffsets* offsets, uintptr_t address) -{ - unsigned int shift = PYLONG_BITS_IN_DIGIT; - - Py_ssize_t size; - uintptr_t lv_tag; - - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, address + offsets->long_object.lv_tag, - sizeof(uintptr_t), - &lv_tag); - if (bytes_read < 0) { - return -1; - } - - int negative = (lv_tag & 3) == 2; - size = lv_tag >> 3; - - if (size == 0) { - return 0; - } - - digit *digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); - if (!digits) { - PyErr_NoMemory(); - return -1; - } - - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->long_object.ob_digit, - sizeof(digit) * size, - digits - ); - if (bytes_read < 0) { - goto error; - } - - long long value = 0; - - // In theory this can overflow, but because of llvm/llvm-project#16778 - // we can't use __builtin_mul_overflow because it fails to link with - // __muloti4 on aarch64. In practice this is fine because all we're - // testing here are task numbers that would fit in a single byte. - for (Py_ssize_t i = 0; i < size; ++i) { - long long factor = digits[i] * (1UL << (Py_ssize_t)(shift * i)); - value += factor; - } - PyMem_RawFree(digits); - if (negative) { - value *= -1; - } - return (long)value; -error: - PyMem_RawFree(digits); - return -1; -} - -static PyObject * -parse_task_name( - proc_handle_t *handle, - _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address -) { - uintptr_t task_name_addr; - int err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_name, - &task_name_addr); - if (err) { - return NULL; - } - - // The task name can be a long or a string so we need to check the type - - PyObject task_name_obj; - err = read_pyobj( - handle, - task_name_addr, - &task_name_obj); - if (err) { - return NULL; - } - - unsigned long flags; - err = read_unsigned_long( - handle, - (uintptr_t)task_name_obj.ob_type + offsets->type_object.tp_flags, - &flags); - if (err) { - return NULL; - } - - if ((flags & Py_TPFLAGS_LONG_SUBCLASS)) { - long res = read_py_long(handle, offsets, task_name_addr); - if (res == -1) { - chain_exceptions(PyExc_RuntimeError, "Failed to get task name"); - return NULL; - } - return PyUnicode_FromFormat("Task-%d", res); - } - - if(!(flags & Py_TPFLAGS_UNICODE_SUBCLASS)) { - PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); - return NULL; - } - - return read_py_str( - handle, - offsets, - task_name_addr, - 255 - ); -} - -static int -parse_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, - uintptr_t address, - uintptr_t* previous_frame -); - -static int -parse_coro_chain( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t coro_address, - PyObject *render_to -) { - assert((void*)coro_address != NULL); - - uintptr_t gen_type_addr; - int err = read_ptr( - handle, - coro_address + offsets->pyobject.ob_type, - &gen_type_addr); - if (err) { - return -1; - } - - PyObject* name = NULL; - uintptr_t prev_frame; - if (parse_frame_object( - handle, - &name, - offsets, - coro_address + offsets->gen_object.gi_iframe, - &prev_frame) - < 0) - { - return -1; - } - - if (PyList_Append(render_to, name)) { - Py_DECREF(name); - return -1; - } - Py_DECREF(name); - - int8_t gi_frame_state; - err = read_sized_int( - handle, - coro_address + offsets->gen_object.gi_frame_state, - &gi_frame_state, - sizeof(int8_t) - ); - if (err) { - return -1; - } - - if (gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { - char owner; - err = read_char( - handle, - coro_address + offsets->gen_object.gi_iframe + - offsets->interpreter_frame.owner, - &owner - ); - if (err) { - return -1; - } - if (owner != FRAME_OWNED_BY_GENERATOR) { - PyErr_SetString( - PyExc_RuntimeError, - "generator doesn't own its frame \\_o_/"); - return -1; - } - - uintptr_t stackpointer_addr; - err = read_py_ptr( - handle, - coro_address + offsets->gen_object.gi_iframe + - offsets->interpreter_frame.stackpointer, - &stackpointer_addr); - if (err) { - return -1; - } - - if ((void*)stackpointer_addr != NULL) { - uintptr_t gi_await_addr; - err = read_py_ptr( - handle, - stackpointer_addr - sizeof(void*), - &gi_await_addr); - if (err) { - return -1; - } - - if ((void*)gi_await_addr != NULL) { - uintptr_t gi_await_addr_type_addr; - int err = read_ptr( - handle, - gi_await_addr + offsets->pyobject.ob_type, - &gi_await_addr_type_addr); - if (err) { - return -1; - } - - if (gen_type_addr == gi_await_addr_type_addr) { - /* This needs an explanation. We always start with parsing - native coroutine / generator frames. Ultimately they - are awaiting on something. That something can be - a native coroutine frame or... an iterator. - If it's the latter -- we can't continue building - our chain. So the condition to bail out of this is - to do that when the type of the current coroutine - doesn't match the type of whatever it points to - in its cr_await. - */ - err = parse_coro_chain( - handle, - offsets, - async_offsets, - gi_await_addr, - render_to - ); - if (err) { - return -1; - } - } - } - } - - } - - return 0; -} - - -static int -parse_task_awaited_by( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address, - PyObject *awaited_by, - int recurse_task -); - - -static int -parse_task( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address, - PyObject *render_to, - int recurse_task -) { - char is_task; - int err = read_char( - handle, - task_address + async_offsets->asyncio_task_object.task_is_task, - &is_task); - if (err) { - return -1; - } - - PyObject* result = PyList_New(0); - if (result == NULL) { - return -1; - } - - PyObject *call_stack = PyList_New(0); - if (call_stack == NULL) { - goto err; - } - if (PyList_Append(result, call_stack)) { - Py_DECREF(call_stack); - goto err; - } - /* we can operate on a borrowed one to simplify cleanup */ - Py_DECREF(call_stack); - - if (is_task) { - PyObject *tn = NULL; - if (recurse_task) { - tn = parse_task_name( - handle, offsets, async_offsets, task_address); - } else { - tn = PyLong_FromUnsignedLongLong(task_address); - } - if (tn == NULL) { - goto err; - } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - goto err; - } - Py_DECREF(tn); - - uintptr_t coro_addr; - err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_coro, - &coro_addr); - if (err) { - goto err; - } - - if ((void*)coro_addr != NULL) { - err = parse_coro_chain( - handle, - offsets, - async_offsets, - coro_addr, - call_stack - ); - if (err) { - goto err; - } - - if (PyList_Reverse(call_stack)) { - goto err; - } - } - } - - if (PyList_Append(render_to, result)) { - goto err; - } - - if (recurse_task) { - PyObject *awaited_by = PyList_New(0); - if (awaited_by == NULL) { - goto err; - } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - goto err; - } - /* we can operate on a borrowed one to simplify cleanup */ - Py_DECREF(awaited_by); - - if (parse_task_awaited_by(handle, offsets, async_offsets, - task_address, awaited_by, 1) - ) { - goto err; - } - } - Py_DECREF(result); - - return 0; - -err: - Py_DECREF(result); - return -1; -} - -static int -parse_tasks_in_set( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t set_addr, - PyObject *awaited_by, - int recurse_task -) { - uintptr_t set_obj; - if (read_py_ptr( - handle, - set_addr, - &set_obj) - ) { - return -1; - } - - Py_ssize_t num_els; - if (read_Py_ssize_t( - handle, - set_obj + offsets->set_object.used, - &num_els) - ) { - return -1; - } - - Py_ssize_t set_len; - if (read_Py_ssize_t( - handle, - set_obj + offsets->set_object.mask, - &set_len) - ) { - return -1; - } - set_len++; // The set contains the `mask+1` element slots. - - uintptr_t table_ptr; - if (read_ptr( - handle, - set_obj + offsets->set_object.table, - &table_ptr) - ) { - return -1; - } - - Py_ssize_t i = 0; - Py_ssize_t els = 0; - while (i < set_len) { - uintptr_t key_addr; - if (read_py_ptr(handle, table_ptr, &key_addr)) { - return -1; - } - - if ((void*)key_addr != NULL) { - Py_ssize_t ref_cnt; - if (read_Py_ssize_t(handle, table_ptr, &ref_cnt)) { - return -1; - } - - if (ref_cnt) { - // if 'ref_cnt=0' it's a set dummy marker - - if (parse_task( - handle, - offsets, - async_offsets, - key_addr, - awaited_by, - recurse_task - ) - ) { - return -1; - } - - if (++els == num_els) { - break; - } - } - } - - table_ptr += sizeof(void*) * 2; - i++; - } - return 0; -} - - -static int -parse_task_awaited_by( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address, - PyObject *awaited_by, - int recurse_task -) { - uintptr_t task_ab_addr; - int err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - &task_ab_addr); - if (err) { - return -1; - } - - if ((void*)task_ab_addr == NULL) { - return 0; - } - - char awaited_by_is_a_set; - err = read_char( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by_is_set, - &awaited_by_is_a_set); - if (err) { - return -1; - } - - if (awaited_by_is_a_set) { - if (parse_tasks_in_set( - handle, - offsets, - async_offsets, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - awaited_by, - recurse_task - ) - ) { - return -1; - } - } else { - uintptr_t sub_task; - if (read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - &sub_task) - ) { - return -1; - } - - if (parse_task( - handle, - offsets, - async_offsets, - sub_task, - awaited_by, - recurse_task - ) - ) { - return -1; - } - } - - return 0; -} - -typedef struct -{ - int lineno; - int end_lineno; - int column; - int end_column; -} LocationInfo; - -static int -scan_varint(const uint8_t **ptr) -{ - unsigned int read = **ptr; - *ptr = *ptr + 1; - unsigned int val = read & 63; - unsigned int shift = 0; - while (read & 64) { - read = **ptr; - *ptr = *ptr + 1; - shift += 6; - val |= (read & 63) << shift; - } - return val; -} - -static int -scan_signed_varint(const uint8_t **ptr) -{ - unsigned int uval = scan_varint(ptr); - if (uval & 1) { - return -(int)(uval >> 1); - } - else { - return uval >> 1; - } -} - - -static bool -parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, LocationInfo* info) -{ - const uint8_t* ptr = (const uint8_t*)(linetable); - uint64_t addr = 0; - info->lineno = firstlineno; - - while (*ptr != '\0') { - // See InternalDocs/code_objects.md for where these magic numbers are from - // and for the decoding algorithm. - uint8_t first_byte = *(ptr++); - uint8_t code = (first_byte >> 3) & 15; - size_t length = (first_byte & 7) + 1; - uintptr_t end_addr = addr + length; - switch (code) { - case PY_CODE_LOCATION_INFO_NONE: { - break; - } - case PY_CODE_LOCATION_INFO_LONG: { - int line_delta = scan_signed_varint(&ptr); - info->lineno += line_delta; - info->end_lineno = info->lineno + scan_varint(&ptr); - info->column = scan_varint(&ptr) - 1; - info->end_column = scan_varint(&ptr) - 1; - break; - } - case PY_CODE_LOCATION_INFO_NO_COLUMNS: { - int line_delta = scan_signed_varint(&ptr); - info->lineno += line_delta; - info->column = info->end_column = -1; - break; - } - case PY_CODE_LOCATION_INFO_ONE_LINE0: - case PY_CODE_LOCATION_INFO_ONE_LINE1: - case PY_CODE_LOCATION_INFO_ONE_LINE2: { - int line_delta = code - 10; - info->lineno += line_delta; - info->end_lineno = info->lineno; - info->column = *(ptr++); - info->end_column = *(ptr++); - break; - } - default: { - uint8_t second_byte = *(ptr++); - assert((second_byte & 128) == 0); - info->column = code << 3 | (second_byte >> 4); - info->end_column = info->column + (second_byte & 15); - break; - } - } - if (addr <= addrq && end_addr > addrq) { - return true; - } - addr = end_addr; - } - return false; -} - -static int -read_remote_pointer(proc_handle_t *handle, uintptr_t address, uintptr_t *out_ptr, const char *error_message) -{ - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(void *), out_ptr); - if (bytes_read < 0) { - return -1; - } - - if ((void *)(*out_ptr) == NULL) { - PyErr_SetString(PyExc_RuntimeError, error_message); - return -1; - } - - return 0; -} - -static int -read_instruction_ptr(proc_handle_t *handle, struct _Py_DebugOffsets *offsets, - uintptr_t current_frame, uintptr_t *instruction_ptr) -{ - return read_remote_pointer( - handle, - current_frame + offsets->interpreter_frame.instr_ptr, - instruction_ptr, - "No instruction ptr found" - ); -} - -static int -parse_code_object(proc_handle_t *handle, - PyObject **result, - struct _Py_DebugOffsets *offsets, - uintptr_t address, - uintptr_t current_frame, - uintptr_t *previous_frame) -{ - uintptr_t addr_func_name, addr_file_name, addr_linetable, instruction_ptr; - - if (read_remote_pointer(handle, address + offsets->code_object.qualname, &addr_func_name, "No function name found") < 0 || - read_remote_pointer(handle, address + offsets->code_object.filename, &addr_file_name, "No file name found") < 0 || - read_remote_pointer(handle, address + offsets->code_object.linetable, &addr_linetable, "No linetable found") < 0 || - read_instruction_ptr(handle, offsets, current_frame, &instruction_ptr) < 0) { - return -1; - } - - int firstlineno; - if (_Py_RemoteDebug_ReadRemoteMemory(handle, - address + offsets->code_object.firstlineno, - sizeof(int), - &firstlineno) < 0) { - return -1; - } - - PyObject *py_linetable = read_py_bytes(handle, offsets, addr_linetable); - if (!py_linetable) { - return -1; - } - - uintptr_t addr_code_adaptive = address + offsets->code_object.co_code_adaptive; - ptrdiff_t addrq = (uint16_t *)instruction_ptr - (uint16_t *)addr_code_adaptive; - - LocationInfo info; - parse_linetable(addrq, PyBytes_AS_STRING(py_linetable), firstlineno, &info); - Py_DECREF(py_linetable); // Done with linetable - - PyObject *py_line = PyLong_FromLong(info.lineno); - if (!py_line) { - return -1; - } - - PyObject *py_func_name = read_py_str(handle, offsets, addr_func_name, 256); - if (!py_func_name) { - Py_DECREF(py_line); - return -1; - } - - PyObject *py_file_name = read_py_str(handle, offsets, addr_file_name, 256); - if (!py_file_name) { - Py_DECREF(py_line); - Py_DECREF(py_func_name); - return -1; - } - - PyObject *result_tuple = PyTuple_New(3); - if (!result_tuple) { - Py_DECREF(py_line); - Py_DECREF(py_func_name); - Py_DECREF(py_file_name); - return -1; - } - - PyTuple_SET_ITEM(result_tuple, 0, py_func_name); // steals ref - PyTuple_SET_ITEM(result_tuple, 1, py_file_name); // steals ref - PyTuple_SET_ITEM(result_tuple, 2, py_line); // steals ref - - *result = result_tuple; - return 0; -} - -static int -parse_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, - uintptr_t address, - uintptr_t* previous_frame -) { - int err; - - Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->interpreter_frame.previous, - sizeof(void*), - previous_frame - ); - if (bytes_read < 0) { - return -1; - } - - char owner; - if (read_char(handle, address + offsets->interpreter_frame.owner, &owner)) { - return -1; - } - - if (owner >= FRAME_OWNED_BY_INTERPRETER) { - return 0; - } - - uintptr_t address_of_code_object; - err = read_py_ptr( - handle, - address + offsets->interpreter_frame.executable, - &address_of_code_object - ); - if (err) { - return -1; - } - - if ((void*)address_of_code_object == NULL) { - return 0; - } - - return parse_code_object( - handle, result, offsets, address_of_code_object, address, previous_frame); -} - -static int -parse_async_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, - uintptr_t address, - uintptr_t* previous_frame, - uintptr_t* code_object -) { - int err; - - Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->interpreter_frame.previous, - sizeof(void*), - previous_frame - ); - if (bytes_read < 0) { - return -1; - } - - char owner; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, address + offsets->interpreter_frame.owner, sizeof(char), &owner); - if (bytes_read < 0) { - return -1; - } - - if (owner == FRAME_OWNED_BY_CSTACK || owner == FRAME_OWNED_BY_INTERPRETER) { - return 0; // C frame - } - - if (owner != FRAME_OWNED_BY_GENERATOR - && owner != FRAME_OWNED_BY_THREAD) { - PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", owner); - return -1; - } - - err = read_py_ptr( - handle, - address + offsets->interpreter_frame.executable, - code_object - ); - if (err) { - return -1; - } - - assert(code_object != NULL); - if ((void*)*code_object == NULL) { - return 0; - } - - if (parse_code_object( - handle, result, offsets, *code_object, address, previous_frame)) { - return -1; - } - - return 1; -} - -static int -read_async_debug( - proc_handle_t *handle, - struct _Py_AsyncioModuleDebugOffsets* async_debug -) { - uintptr_t async_debug_addr = _Py_RemoteDebug_GetAsyncioDebugAddress(handle); - if (!async_debug_addr) { - return -1; - } - - size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, async_debug_addr, size, async_debug); - return result; -} - -static int -find_running_frame( - proc_handle_t *handle, - uintptr_t runtime_start_address, - _Py_DebugOffsets* local_debug_offsets, - uintptr_t *frame -) { - uint64_t interpreter_state_list_head = - local_debug_offsets->runtime_state.interpreters_head; - - uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - runtime_start_address + interpreter_state_list_head, - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read < 0) { - return -1; - } - - if (address_of_interpreter_state == 0) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); - return -1; - } - - uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address_of_interpreter_state + - local_debug_offsets->interpreter_state.threads_main, - sizeof(void*), - &address_of_thread); - if (bytes_read < 0) { - return -1; - } - - // No Python frames are available for us (can happen at tear-down). - if ((void*)address_of_thread != NULL) { - int err = read_ptr( - handle, - address_of_thread + local_debug_offsets->thread_state.current_frame, - frame); - if (err) { - return -1; - } - return 0; - } - - *frame = (uintptr_t)NULL; - return 0; -} - -static int -find_running_task( - proc_handle_t *handle, - uintptr_t runtime_start_address, - _Py_DebugOffsets *local_debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, - uintptr_t *running_task_addr -) { - *running_task_addr = (uintptr_t)NULL; - - uint64_t interpreter_state_list_head = - local_debug_offsets->runtime_state.interpreters_head; - - uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - runtime_start_address + interpreter_state_list_head, - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read < 0) { - return -1; - } - - if (address_of_interpreter_state == 0) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); - return -1; - } - - uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address_of_interpreter_state + - local_debug_offsets->interpreter_state.threads_head, - sizeof(void*), - &address_of_thread); - if (bytes_read < 0) { - return -1; - } - - uintptr_t address_of_running_loop; - // No Python frames are available for us (can happen at tear-down). - if ((void*)address_of_thread == NULL) { - return 0; - } - - bytes_read = read_py_ptr( - handle, - address_of_thread - + async_offsets->asyncio_thread_state.asyncio_running_loop, - &address_of_running_loop); - if (bytes_read == -1) { - return -1; - } - - // no asyncio loop is now running - if ((void*)address_of_running_loop == NULL) { - return 0; - } - - int err = read_ptr( - handle, - address_of_thread - + async_offsets->asyncio_thread_state.asyncio_running_task, - running_task_addr); - if (err) { - return -1; - } - - return 0; -} - -static int -append_awaited_by_for_thread( - proc_handle_t *handle, - uintptr_t head_addr, - struct _Py_DebugOffsets *debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, - PyObject *result -) { - struct llist_node task_node; - - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - head_addr, - sizeof(task_node), - &task_node)) - { - return -1; - } - - size_t iteration_count = 0; - const size_t MAX_ITERATIONS = 2 << 15; // A reasonable upper bound - while ((uintptr_t)task_node.next != head_addr) { - if (++iteration_count > MAX_ITERATIONS) { - PyErr_SetString(PyExc_RuntimeError, "Task list appears corrupted"); - return -1; - } - - if (task_node.next == NULL) { - PyErr_SetString( - PyExc_RuntimeError, - "Invalid linked list structure reading remote memory"); - return -1; - } - - uintptr_t task_addr = (uintptr_t)task_node.next - - async_offsets->asyncio_task_object.task_node; - - PyObject *tn = parse_task_name( - handle, - debug_offsets, - async_offsets, - task_addr); - if (tn == NULL) { - return -1; - } - - PyObject *current_awaited_by = PyList_New(0); - if (current_awaited_by == NULL) { - Py_DECREF(tn); - return -1; - } - - PyObject* task_id = PyLong_FromUnsignedLongLong(task_addr); - if (task_id == NULL) { - Py_DECREF(tn); - Py_DECREF(current_awaited_by); - return -1; - } - - PyObject *result_item = PyTuple_New(3); - if (result_item == NULL) { - Py_DECREF(tn); - Py_DECREF(current_awaited_by); - Py_DECREF(task_id); - return -1; - } - - PyTuple_SET_ITEM(result_item, 0, task_id); // steals ref - PyTuple_SET_ITEM(result_item, 1, tn); // steals ref - PyTuple_SET_ITEM(result_item, 2, current_awaited_by); // steals ref - if (PyList_Append(result, result_item)) { - Py_DECREF(result_item); - return -1; - } - Py_DECREF(result_item); - - if (parse_task_awaited_by(handle, debug_offsets, async_offsets, - task_addr, current_awaited_by, 0)) - { - return -1; - } - - // onto the next one... - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - (uintptr_t)task_node.next, - sizeof(task_node), - &task_node)) - { - return -1; - } - } - - return 0; -} - -static int -append_awaited_by( - proc_handle_t *handle, - unsigned long tid, - uintptr_t head_addr, - struct _Py_DebugOffsets *debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, - PyObject *result) -{ - PyObject *tid_py = PyLong_FromUnsignedLong(tid); - if (tid_py == NULL) { - return -1; - } - - PyObject *result_item = PyTuple_New(2); - if (result_item == NULL) { - Py_DECREF(tid_py); - return -1; - } - - PyObject* awaited_by_for_thread = PyList_New(0); - if (awaited_by_for_thread == NULL) { - Py_DECREF(tid_py); - Py_DECREF(result_item); - return -1; - } - - PyTuple_SET_ITEM(result_item, 0, tid_py); // steals ref - PyTuple_SET_ITEM(result_item, 1, awaited_by_for_thread); // steals ref - if (PyList_Append(result, result_item)) { - Py_DECREF(result_item); - return -1; - } - Py_DECREF(result_item); - - if (append_awaited_by_for_thread( - handle, - head_addr, - debug_offsets, - async_offsets, - awaited_by_for_thread)) - { - return -1; - } - - return 0; -} - -static PyObject* -get_all_awaited_by(PyObject* self, PyObject* args) -{ -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_all_awaited_by is not implemented on this platform"); - return NULL; -#endif - - int pid; - if (!PyArg_ParseTuple(args, "i", &pid)) { - return NULL; - } - - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { - return 0; - } - - PyObject *result = NULL; - - uintptr_t runtime_start_addr = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_addr == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } - goto result_err; - } - struct _Py_DebugOffsets local_debug_offsets; - - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_addr, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; - } - - struct _Py_AsyncioModuleDebugOffsets local_async_debug; - if (read_async_debug(handle, &local_async_debug)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read asyncio debug offsets"); - goto result_err; - } - - result = PyList_New(0); - if (result == NULL) { - goto result_err; - } - - uint64_t interpreter_state_list_head = - local_debug_offsets.runtime_state.interpreters_head; - - uintptr_t interpreter_state_addr; - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - runtime_start_addr + interpreter_state_list_head, - sizeof(void*), - &interpreter_state_addr)) - { - goto result_err; - } - - uintptr_t thread_state_addr; - unsigned long tid = 0; - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - interpreter_state_addr - + local_debug_offsets.interpreter_state.threads_head, - sizeof(void*), - &thread_state_addr)) - { - goto result_err; - } - - uintptr_t head_addr; - while (thread_state_addr != 0) { - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - thread_state_addr - + local_debug_offsets.thread_state.native_thread_id, - sizeof(tid), - &tid)) - { - goto result_err; - } - - head_addr = thread_state_addr - + local_async_debug.asyncio_thread_state.asyncio_tasks_head; - - if (append_awaited_by(handle, tid, head_addr, &local_debug_offsets, - &local_async_debug, result)) - { - goto result_err; - } - - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - thread_state_addr + local_debug_offsets.thread_state.next, - sizeof(void*), - &thread_state_addr)) - { - goto result_err; - } - } - - head_addr = interpreter_state_addr - + local_async_debug.asyncio_interpreter_state.asyncio_tasks_head; - - // On top of a per-thread task lists used by default by asyncio to avoid - // contention, there is also a fallback per-interpreter list of tasks; - // any tasks still pending when a thread is destroyed will be moved to the - // per-interpreter task list. It's unlikely we'll find anything here, but - // interesting for debugging. - if (append_awaited_by(handle, 0, head_addr, &local_debug_offsets, - &local_async_debug, result)) - { - goto result_err; - } - - _Py_RemoteDebug_CleanupProcHandle(handle); - return result; - -result_err: - Py_XDECREF(result); - _Py_RemoteDebug_CleanupProcHandle(handle); - return NULL; -} - -static PyObject* -get_stack_trace(PyObject* self, PyObject* args) -{ -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_stack_trace is not supported on this platform"); - return NULL; -#endif - - int pid; - if (!PyArg_ParseTuple(args, "i", &pid)) { - return NULL; - } - - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { - return 0; - } - - PyObject* result = NULL; - - uintptr_t runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_address == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } - goto result_err; - } - struct _Py_DebugOffsets local_debug_offsets; - - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_address, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; - } - - uintptr_t address_of_current_frame; - if (find_running_frame( - handle, runtime_start_address, &local_debug_offsets, - &address_of_current_frame) - ) { - goto result_err; - } - - result = PyList_New(0); - if (result == NULL) { - goto result_err; - } - - while ((void*)address_of_current_frame != NULL) { - PyObject* frame_info = NULL; - if (parse_frame_object( - handle, - &frame_info, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame) - < 0) - { - Py_DECREF(result); - goto result_err; - } - - if (!frame_info) { - continue; - } - - if (PyList_Append(result, frame_info) == -1) { - Py_DECREF(result); - goto result_err; - } - - Py_DECREF(frame_info); - frame_info = NULL; - - } - -result_err: - _Py_RemoteDebug_CleanupProcHandle(handle); - return result; -} - -static PyObject* -get_async_stack_trace(PyObject* self, PyObject* args) -{ -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_stack_trace is not supported on this platform"); - return NULL; -#endif - int pid; - - if (!PyArg_ParseTuple(args, "i", &pid)) { - return NULL; - } - - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { - return 0; - } - - PyObject *result = NULL; - - uintptr_t runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_address == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } - goto result_err; - } - struct _Py_DebugOffsets local_debug_offsets; - - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_address, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; - } - - struct _Py_AsyncioModuleDebugOffsets local_async_debug; - if (read_async_debug(handle, &local_async_debug)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read asyncio debug offsets"); - goto result_err; - } - - result = PyList_New(1); - if (result == NULL) { - goto result_err; - } - PyObject* calls = PyList_New(0); - if (calls == NULL) { - goto result_err; - } - if (PyList_SetItem(result, 0, calls)) { /* steals ref to 'calls' */ - Py_DECREF(calls); - goto result_err; - } - - uintptr_t running_task_addr = (uintptr_t)NULL; - if (find_running_task( - handle, runtime_start_address, &local_debug_offsets, &local_async_debug, - &running_task_addr) - ) { - chain_exceptions(PyExc_RuntimeError, "Failed to find running task"); - goto result_err; - } - - if ((void*)running_task_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No running task found"); - goto result_err; - } - - uintptr_t running_coro_addr; - if (read_py_ptr( - handle, - running_task_addr + local_async_debug.asyncio_task_object.task_coro, - &running_coro_addr - )) { - chain_exceptions(PyExc_RuntimeError, "Failed to read running task coro"); - goto result_err; - } - - if ((void*)running_coro_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); - goto result_err; - } - - // note: genobject's gi_iframe is an embedded struct so the address to - // the offset leads directly to its first field: f_executable - uintptr_t address_of_running_task_code_obj; - if (read_py_ptr( - handle, - running_coro_addr + local_debug_offsets.gen_object.gi_iframe, - &address_of_running_task_code_obj - )) { - goto result_err; - } - - if ((void*)address_of_running_task_code_obj == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); - goto result_err; - } - - uintptr_t address_of_current_frame; - if (find_running_frame( - handle, runtime_start_address, &local_debug_offsets, - &address_of_current_frame) - ) { - chain_exceptions(PyExc_RuntimeError, "Failed to find running frame"); - goto result_err; - } - - uintptr_t address_of_code_object; - while ((void*)address_of_current_frame != NULL) { - PyObject* frame_info = NULL; - int res = parse_async_frame_object( - handle, - &frame_info, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame, - &address_of_code_object - ); - - if (res < 0) { - chain_exceptions(PyExc_RuntimeError, "Failed to parse async frame object"); - goto result_err; - } - - if (!frame_info) { - continue; - } - - if (PyList_Append(calls, frame_info) == -1) { - Py_DECREF(calls); - goto result_err; - } - - Py_DECREF(frame_info); - frame_info = NULL; - - if (address_of_code_object == address_of_running_task_code_obj) { - break; - } - } - - PyObject *tn = parse_task_name( - handle, &local_debug_offsets, &local_async_debug, running_task_addr); - if (tn == NULL) { - goto result_err; - } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - goto result_err; - } - Py_DECREF(tn); - - PyObject* awaited_by = PyList_New(0); - if (awaited_by == NULL) { - goto result_err; - } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - goto result_err; - } - Py_DECREF(awaited_by); - - if (parse_task_awaited_by( - handle, &local_debug_offsets, &local_async_debug, - running_task_addr, awaited_by, 1) - ) { - goto result_err; - } - - _Py_RemoteDebug_CleanupProcHandle(handle); - return result; - -result_err: - _Py_RemoteDebug_CleanupProcHandle(handle); - Py_XDECREF(result); - return NULL; -} - - -static PyMethodDef methods[] = { - {"get_stack_trace", get_stack_trace, METH_VARARGS, - "Get the Python stack from a given pod"}, - {"get_async_stack_trace", get_async_stack_trace, METH_VARARGS, - "Get the asyncio stack from a given pid"}, - {"get_all_awaited_by", get_all_awaited_by, METH_VARARGS, - "Get all tasks and their awaited_by from a given pid"}, - {NULL, NULL, 0, NULL}, -}; - -static struct PyModuleDef module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_remote_debugging", - .m_size = -1, - .m_methods = methods, -}; - -PyMODINIT_FUNC -PyInit__remote_debugging(void) -{ - PyObject* mod = PyModule_Create(&module); - if (mod == NULL) { - return NULL; - } -#ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); -#endif - int rc = PyModule_AddIntConstant( - mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); - if (rc < 0) { - Py_DECREF(mod); - return NULL; - } - return mod; -} diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 35d090e3ca2..4a213f34888 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -4,6 +4,7 @@ #include "blob.h" #include "util.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" @@ -56,9 +57,7 @@ blob_dealloc(PyObject *op) close_blob(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->in_weakreflist); (void)tp->tp_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -119,7 +118,9 @@ static void blob_seterror(pysqlite_Blob *self, int rc) { assert(self->connection != NULL); - set_error_from_db(self->connection->state, self->connection->db); + assert(rc != SQLITE_OK); + set_error_from_code(self->connection->state, rc); + assert(PyErr_Occurred()); } static PyObject * @@ -144,27 +145,28 @@ read_multiple(pysqlite_Blob *self, Py_ssize_t length, Py_ssize_t offset) assert(length <= sqlite3_blob_bytes(self->blob)); assert(offset < sqlite3_blob_bytes(self->blob)); - PyObject *buffer = PyBytes_FromStringAndSize(NULL, length); - if (buffer == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(length); + if (writer == NULL) { return NULL; } + char *raw_buffer = PyBytesWriter_GetData(writer); - char *raw_buffer = PyBytes_AS_STRING(buffer); int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_read(self->blob, raw_buffer, (int)length, (int)offset); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { - Py_DECREF(buffer); + PyBytesWriter_Discard(writer); blob_seterror(self, rc); return NULL; } - return buffer; + return PyBytesWriter_Finish(writer); } /*[clinic input] +@permit_long_docstring_body _sqlite3.Blob.read as blob_read length: int = -1 @@ -180,7 +182,7 @@ end of the blob. static PyObject * blob_read_impl(pysqlite_Blob *self, int length) -/*[clinic end generated code: output=1fc99b2541360dde input=f2e4aa4378837250]*/ +/*[clinic end generated code: output=1fc99b2541360dde input=e5715bcddbcfca5a]*/ { if (!check_blob(self)) { return NULL; @@ -196,7 +198,7 @@ blob_read_impl(pysqlite_Blob *self, int length) assert(length >= 0); if (length == 0) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } PyObject *buffer = read_multiple(self, length, self->offset); @@ -233,6 +235,7 @@ inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, /*[clinic input] +@permit_long_docstring_body _sqlite3.Blob.write as blob_write data: Py_buffer @@ -246,7 +249,7 @@ blob will result in an exception being raised. static PyObject * blob_write_impl(pysqlite_Blob *self, Py_buffer *data) -/*[clinic end generated code: output=b34cf22601b570b2 input=a84712f24a028e6d]*/ +/*[clinic end generated code: output=b34cf22601b570b2 input=203d3458f244814b]*/ { if (!check_blob(self)) { return NULL; @@ -262,6 +265,7 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) /*[clinic input] +@permit_long_docstring_body _sqlite3.Blob.seek as blob_seek offset: int @@ -277,7 +281,7 @@ and os.SEEK_END (seek relative to the blob's end). static PyObject * blob_seek_impl(pysqlite_Blob *self, int offset, int origin) -/*[clinic end generated code: output=854c5a0e208547a5 input=5da9a07e55fe6bb6]*/ +/*[clinic end generated code: output=854c5a0e208547a5 input=ee4d88e1dc0b1048]*/ { if (!check_blob(self)) { return NULL; @@ -438,20 +442,25 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) if (step == 1) { return read_multiple(self, len, start); } + PyObject *blob = read_multiple(self, stop - start, start); if (blob == NULL) { return NULL; } - PyObject *result = PyBytes_FromStringAndSize(NULL, len); - if (result != NULL) { - char *blob_buf = PyBytes_AS_STRING(blob); - char *res_buf = PyBytes_AS_STRING(result); - for (Py_ssize_t i = 0, j = 0; i < len; i++, j += step) { - res_buf[i] = blob_buf[j]; - } + + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { Py_DECREF(blob); + return NULL; } - return result; + char *res_buf = PyBytesWriter_GetData(writer); + + char *blob_buf = PyBytes_AS_STRING(blob); + for (Py_ssize_t i = 0, j = 0; i < len; i++, j += step) { + res_buf[i] = blob_buf[j]; + } + Py_DECREF(blob); + return PyBytesWriter_Finish(writer); } static PyObject * diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h index 1bcda7702c2..e9d560666c1 100644 --- a/Modules/_sqlite/clinic/_sqlite3.connect.c.h +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -9,23 +9,17 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(pysqlite_connect__doc__, -"connect($module, /, database, timeout=5.0, detect_types=0,\n" +"connect($module, /, database, *, timeout=5.0, detect_types=0,\n" " isolation_level=\'\', check_same_thread=True,\n" -" factory=ConnectionType, cached_statements=128, uri=False, *,\n" +" factory=ConnectionType, cached_statements=128, uri=False,\n" " autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL)\n" "--\n" "\n" "Open a connection to the SQLite database file \'database\'.\n" "\n" "You can use \":memory:\" to open a database connection to a database that\n" -"resides in RAM instead of on disk.\n" -"\n" -"Note: Passing more than 1 positional argument to _sqlite3.connect() is\n" -"deprecated. Parameters \'timeout\', \'detect_types\', \'isolation_level\',\n" -"\'check_same_thread\', \'factory\', \'cached_statements\' and \'uri\' will\n" -"become keyword-only parameters in Python 3.15.\n" -""); +"resides in RAM instead of on disk."); #define PYSQLITE_CONNECT_METHODDEF \ {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, -/*[clinic end generated code: output=69b9b00da71c3c0a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3d83139ba65e0bb5 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index c8e1d0b7a73..abb864eb030 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -16,17 +16,6 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int cache_size, int uri, enum autocommit_mode autocommit); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.__init__'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.__init__'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.__init__'." -# endif -#endif - static int pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -72,25 +61,14 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) int uri = 0; enum autocommit_mode autocommit = LEGACY_TRANSACTION_CONTROL; - if (nargs > 1 && nargs <= 8) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 1 positional argument to _sqlite3.Connection()" - " is deprecated. Parameters 'timeout', 'detect_types', " - "'isolation_level', 'check_same_thread', 'factory', " - "'cached_statements' and 'uri' will become keyword-only " - "parameters in Python 3.15.", 1)) - { - goto exit; - } - } fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 1, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } database = fastargs[0]; if (!noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } if (fastargs[1]) { if (PyFloat_CheckExact(fastargs[1])) { @@ -104,7 +82,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) } } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[2]) { @@ -113,7 +91,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[3]) { @@ -121,7 +99,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[4]) { @@ -130,13 +108,13 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[5]) { factory = fastargs[5]; if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[6]) { @@ -145,7 +123,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } if (fastargs[7]) { @@ -154,13 +132,9 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } if (!--noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } if (!autocommit_converter(fastargs[8], &autocommit)) { goto exit; } @@ -235,7 +209,7 @@ exit: } PyDoc_STRVAR(blobopen__doc__, -"blobopen($self, table, column, row, /, *, readonly=False, name=\'main\')\n" +"blobopen($self, table, column, rowid, /, *, readonly=False, name=\'main\')\n" "--\n" "\n" "Open and return a BLOB object.\n" @@ -244,8 +218,8 @@ PyDoc_STRVAR(blobopen__doc__, " Table name.\n" " column\n" " Column name.\n" -" row\n" -" Row index.\n" +" rowid\n" +" Row id.\n" " readonly\n" " Open the BLOB without write permissions.\n" " name\n" @@ -424,15 +398,10 @@ pysqlite_connection_rollback(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(pysqlite_connection_create_function__doc__, -"create_function($self, /, name, narg, func, *, deterministic=False)\n" +"create_function($self, name, narg, func, /, *, deterministic=False)\n" "--\n" "\n" -"Creates a new function.\n" -"\n" -"Note: Passing keyword arguments \'name\', \'narg\' and \'func\' to\n" -"_sqlite3.Connection.create_function() is deprecated. Parameters\n" -"\'name\', \'narg\' and \'func\' will become positional-only in Python 3.15.\n" -""); +"Creates a new function."); #define PYSQLITE_CONNECTION_CREATE_FUNCTION_METHODDEF \ {"create_function", _PyCFunction_CAST(pysqlite_connection_create_function), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_function__doc__}, @@ -443,24 +412,13 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, int narg, PyObject *func, int deterministic); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.create_function'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.create_function'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.create_function'." -# endif -#endif - static PyObject * pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -469,7 +427,7 @@ pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(narg), &_Py_ID(func), &_Py_ID(deterministic), }, + .ob_item = { &_Py_ID(deterministic), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -478,7 +436,7 @@ pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "narg", "func", "deterministic", NULL}; + static const char * const _keywords[] = {"", "", "", "deterministic", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "create_function", @@ -497,18 +455,8 @@ pysqlite_connection_create_function(PyObject *self, PyTypeObject *cls, PyObject if (!args) { goto exit; } - if (nargs < 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword arguments 'name', 'narg' and 'func' to " - "_sqlite3.Connection.create_function() is deprecated. Parameters " - "'name', 'narg' and 'func' will become positional-only in Python " - "3.15.", 1)) - { - goto exit; - } - } if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("create_function", "argument 'name'", "str", args[0]); + _PyArg_BadArgument("create_function", "argument 1", "str", args[0]); goto exit; } Py_ssize_t name_length; @@ -618,16 +566,10 @@ exit: #endif /* defined(HAVE_WINDOW_FUNCTIONS) */ PyDoc_STRVAR(pysqlite_connection_create_aggregate__doc__, -"create_aggregate($self, /, name, n_arg, aggregate_class)\n" +"create_aggregate($self, name, n_arg, aggregate_class, /)\n" "--\n" "\n" -"Creates a new aggregate.\n" -"\n" -"Note: Passing keyword arguments \'name\', \'n_arg\' and \'aggregate_class\'\n" -"to _sqlite3.Connection.create_aggregate() is deprecated. Parameters\n" -"\'name\', \'n_arg\' and \'aggregate_class\' will become positional-only in\n" -"Python 3.15.\n" -""); +"Creates a new aggregate."); #define PYSQLITE_CONNECTION_CREATE_AGGREGATE_METHODDEF \ {"create_aggregate", _PyCFunction_CAST(pysqlite_connection_create_aggregate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_create_aggregate__doc__}, @@ -638,42 +580,17 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, const char *name, int n_arg, PyObject *aggregate_class); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.create_aggregate'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.create_aggregate'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.create_aggregate'." -# endif -#endif - static PyObject * pysqlite_connection_create_aggregate(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(n_arg), &_Py_ID(aggregate_class), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else # define KWTUPLE NULL - #endif // !Py_BUILD_CORE + #endif - static const char * const _keywords[] = {"name", "n_arg", "aggregate_class", NULL}; + static const char * const _keywords[] = {"", "", "", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "create_aggregate", @@ -690,18 +607,8 @@ pysqlite_connection_create_aggregate(PyObject *self, PyTypeObject *cls, PyObject if (!args) { goto exit; } - if (nargs < 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword arguments 'name', 'n_arg' and 'aggregate_class' " - "to _sqlite3.Connection.create_aggregate() is deprecated. " - "Parameters 'name', 'n_arg' and 'aggregate_class' will become " - "positional-only in Python 3.15.", 1)) - { - goto exit; - } - } if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("create_aggregate", "argument 'name'", "str", args[0]); + _PyArg_BadArgument("create_aggregate", "argument 1", "str", args[0]); goto exit; } Py_ssize_t name_length; @@ -725,15 +632,10 @@ exit: } PyDoc_STRVAR(pysqlite_connection_set_authorizer__doc__, -"set_authorizer($self, /, authorizer_callback)\n" +"set_authorizer($self, authorizer_callback, /)\n" "--\n" "\n" -"Set authorizer callback.\n" -"\n" -"Note: Passing keyword argument \'authorizer_callback\' to\n" -"_sqlite3.Connection.set_authorizer() is deprecated. Parameter\n" -"\'authorizer_callback\' will become positional-only in Python 3.15.\n" -""); +"Set authorizer callback."); #define PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF \ {"set_authorizer", _PyCFunction_CAST(pysqlite_connection_set_authorizer), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_authorizer__doc__}, @@ -743,42 +645,17 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.set_authorizer'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.set_authorizer'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.set_authorizer'." -# endif -#endif - static PyObject * pysqlite_connection_set_authorizer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(authorizer_callback), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else # define KWTUPLE NULL - #endif // !Py_BUILD_CORE + #endif - static const char * const _keywords[] = {"authorizer_callback", NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "set_authorizer", @@ -793,16 +670,6 @@ pysqlite_connection_set_authorizer(PyObject *self, PyTypeObject *cls, PyObject * if (!args) { goto exit; } - if (nargs < 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword argument 'authorizer_callback' to " - "_sqlite3.Connection.set_authorizer() is deprecated. Parameter " - "'authorizer_callback' will become positional-only in Python " - "3.15.", 1)) - { - goto exit; - } - } callable = args[0]; return_value = pysqlite_connection_set_authorizer_impl((pysqlite_Connection *)self, cls, callable); @@ -811,7 +678,7 @@ exit: } PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, -"set_progress_handler($self, /, progress_handler, n)\n" +"set_progress_handler($self, progress_handler, /, n)\n" "--\n" "\n" "Set progress handler callback.\n" @@ -824,12 +691,7 @@ PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, " The number of SQLite virtual machine instructions that are\n" " executed between invocations of \'progress_handler\'.\n" "\n" -"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled.\n" -"\n" -"Note: Passing keyword argument \'progress_handler\' to\n" -"_sqlite3.Connection.set_progress_handler() is deprecated. Parameter\n" -"\'progress_handler\' will become positional-only in Python 3.15.\n" -""); +"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled."); #define PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF \ {"set_progress_handler", _PyCFunction_CAST(pysqlite_connection_set_progress_handler), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_progress_handler__doc__}, @@ -839,24 +701,13 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.set_progress_handler'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.set_progress_handler'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.set_progress_handler'." -# endif -#endif - static PyObject * pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -865,7 +716,7 @@ pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyOb } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(progress_handler), _Py_LATIN1_CHR('n'), }, + .ob_item = { _Py_LATIN1_CHR('n'), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -874,7 +725,7 @@ pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyOb # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"progress_handler", "n", NULL}; + static const char * const _keywords[] = {"", "n", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "set_progress_handler", @@ -890,16 +741,6 @@ pysqlite_connection_set_progress_handler(PyObject *self, PyTypeObject *cls, PyOb if (!args) { goto exit; } - if (nargs < 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword argument 'progress_handler' to " - "_sqlite3.Connection.set_progress_handler() is deprecated. " - "Parameter 'progress_handler' will become positional-only in " - "Python 3.15.", 1)) - { - goto exit; - } - } callable = args[0]; n = PyLong_AsInt(args[1]); if (n == -1 && PyErr_Occurred()) { @@ -912,15 +753,10 @@ exit: } PyDoc_STRVAR(pysqlite_connection_set_trace_callback__doc__, -"set_trace_callback($self, /, trace_callback)\n" +"set_trace_callback($self, trace_callback, /)\n" "--\n" "\n" -"Set a trace callback called for each SQL statement (passed as unicode).\n" -"\n" -"Note: Passing keyword argument \'trace_callback\' to\n" -"_sqlite3.Connection.set_trace_callback() is deprecated. Parameter\n" -"\'trace_callback\' will become positional-only in Python 3.15.\n" -""); +"Set a trace callback called for each SQL statement (passed as unicode)."); #define PYSQLITE_CONNECTION_SET_TRACE_CALLBACK_METHODDEF \ {"set_trace_callback", _PyCFunction_CAST(pysqlite_connection_set_trace_callback), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_trace_callback__doc__}, @@ -930,42 +766,17 @@ pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable); -// Emit compiler warnings when we get to Python 3.15. -#if PY_VERSION_HEX >= 0x030f00C0 -# error "Update the clinic input of '_sqlite3.Connection.set_trace_callback'." -#elif PY_VERSION_HEX >= 0x030f00A0 -# ifdef _MSC_VER -# pragma message ("Update the clinic input of '_sqlite3.Connection.set_trace_callback'.") -# else -# warning "Update the clinic input of '_sqlite3.Connection.set_trace_callback'." -# endif -#endif - static PyObject * pysqlite_connection_set_trace_callback(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(trace_callback), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else # define KWTUPLE NULL - #endif // !Py_BUILD_CORE + #endif - static const char * const _keywords[] = {"trace_callback", NULL}; + static const char * const _keywords[] = {"", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "set_trace_callback", @@ -980,16 +791,6 @@ pysqlite_connection_set_trace_callback(PyObject *self, PyTypeObject *cls, PyObje if (!args) { goto exit; } - if (nargs < 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing keyword argument 'trace_callback' to " - "_sqlite3.Connection.set_trace_callback() is deprecated. " - "Parameter 'trace_callback' will become positional-only in Python" - " 3.15.", 1)) - { - goto exit; - } - } callable = args[0]; return_value = pysqlite_connection_set_trace_callback_impl((pysqlite_Connection *)self, cls, callable); @@ -1921,4 +1722,4 @@ exit: #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=2f325c2444b4bb47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=16d44c1d8a45e622 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index 350577f488d..3cad9f3aef5 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UInt32_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() static int @@ -181,7 +182,7 @@ PyDoc_STRVAR(pysqlite_cursor_fetchmany__doc__, {"fetchmany", _PyCFunction_CAST(pysqlite_cursor_fetchmany), METH_FASTCALL|METH_KEYWORDS, pysqlite_cursor_fetchmany__doc__}, static PyObject * -pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, int maxrows); +pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, uint32_t maxrows); static PyObject * pysqlite_cursor_fetchmany(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -216,7 +217,7 @@ pysqlite_cursor_fetchmany(PyObject *self, PyObject *const *args, Py_ssize_t narg #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int maxrows = ((pysqlite_Cursor *)self)->arraysize; + uint32_t maxrows = ((pysqlite_Cursor *)self)->arraysize; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -226,8 +227,7 @@ pysqlite_cursor_fetchmany(PyObject *self, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_pos; } - maxrows = PyLong_AsInt(args[0]); - if (maxrows == -1 && PyErr_Occurred()) { + if (!_PyLong_UInt32_Converter(args[0], &maxrows)) { goto exit; } skip_optional_pos: @@ -329,4 +329,46 @@ pysqlite_cursor_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_cursor_close_impl((pysqlite_Cursor *)self); } -/*[clinic end generated code: output=d05c7cbbc8bcab26 input=a9049054013a1b77]*/ + +#if !defined(_sqlite3_Cursor_arraysize_DOCSTR) +# define _sqlite3_Cursor_arraysize_DOCSTR NULL +#endif +#if defined(_SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF) +# undef _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF +# define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", (getter)_sqlite3_Cursor_arraysize_get, (setter)_sqlite3_Cursor_arraysize_set, _sqlite3_Cursor_arraysize_DOCSTR}, +#else +# define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", (getter)_sqlite3_Cursor_arraysize_get, NULL, _sqlite3_Cursor_arraysize_DOCSTR}, +#endif + +static PyObject * +_sqlite3_Cursor_arraysize_get_impl(pysqlite_Cursor *self); + +static PyObject * +_sqlite3_Cursor_arraysize_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _sqlite3_Cursor_arraysize_get_impl((pysqlite_Cursor *)self); +} + +#if !defined(_sqlite3_Cursor_arraysize_DOCSTR) +# define _sqlite3_Cursor_arraysize_DOCSTR NULL +#endif +#if defined(_SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF) +# undef _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF +# define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", (getter)_sqlite3_Cursor_arraysize_get, (setter)_sqlite3_Cursor_arraysize_set, _sqlite3_Cursor_arraysize_DOCSTR}, +#else +# define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", NULL, (setter)_sqlite3_Cursor_arraysize_set, NULL}, +#endif + +static int +_sqlite3_Cursor_arraysize_set_impl(pysqlite_Cursor *self, PyObject *value); + +static int +_sqlite3_Cursor_arraysize_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + return_value = _sqlite3_Cursor_arraysize_set_impl((pysqlite_Cursor *)self, value); + + return return_value; +} +/*[clinic end generated code: output=a0e3ebba9e4d0ece input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 2a184f78754..83ff8e60557 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -144,7 +144,7 @@ class _sqlite3.Connection "pysqlite_Connection *" "clinic_state()->ConnectionTyp [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=67369db2faf80891]*/ -static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); +static int _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); static void free_callback_context(callback_context *ctx); static void set_callback_context(callback_context **ctx_pp, callback_context *ctx); @@ -215,7 +215,7 @@ class sqlite3_int64_converter(CConverter): _sqlite3.Connection.__init__ as pysqlite_connection_init database: object - * [from 3.15] + * timeout: double = 5.0 detect_types: int = 0 isolation_level: IsolationLevel = "" @@ -223,7 +223,6 @@ _sqlite3.Connection.__init__ as pysqlite_connection_init factory: object(c_default='(PyObject*)clinic_state()->ConnectionType') = ConnectionType cached_statements as cache_size: int = 128 uri: bool = False - * autocommit: Autocommit(c_default='LEGACY_TRANSACTION_CONTROL') = sqlite3.LEGACY_TRANSACTION_CONTROL [clinic start generated code]*/ @@ -234,7 +233,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int check_same_thread, PyObject *factory, int cache_size, int uri, enum autocommit_mode autocommit) -/*[clinic end generated code: output=cba057313ea7712f input=219c3dbecbae7d99]*/ +/*[clinic end generated code: output=cba057313ea7712f input=5ca4883d8747a49b]*/ { if (PySys_Audit("sqlite3.connect", "O", database) < 0) { return -1; @@ -562,7 +561,10 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return NULL; } - _pysqlite_drop_unused_cursor_references(self); + if (_pysqlite_drop_unused_cursor_references(self) < 0) { + Py_DECREF(cursor); + return NULL; + } if (cursor && self->row_factory != Py_None) { Py_INCREF(self->row_factory); @@ -579,8 +581,8 @@ _sqlite3.Connection.blobopen as blobopen Table name. column as col: str Column name. - row: sqlite3_int64 - Row index. + rowid as row: sqlite3_int64 + Row id. / * readonly: bool = False @@ -594,7 +596,7 @@ Open and return a BLOB object. static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, sqlite3_int64 row, int readonly, const char *name) -/*[clinic end generated code: output=6a02d43efb885d1c input=23576bd1108d8774]*/ +/*[clinic end generated code: output=6a02d43efb885d1c input=cc3d4b47dac08401]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1061,32 +1063,36 @@ final_callback(sqlite3_context *context) PyGILState_Release(threadstate); } -static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) +static int +_pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) { /* we only need to do this once in a while */ if (self->created_cursors++ < 200) { - return; + return 0; } self->created_cursors = 0; PyObject* new_list = PyList_New(0); if (!new_list) { - return; + return -1; } - for (Py_ssize_t i = 0; i < PyList_Size(self->cursors); i++) { - PyObject* weakref = PyList_GetItem(self->cursors, i); + assert(PyList_CheckExact(self->cursors)); + Py_ssize_t imax = PyList_GET_SIZE(self->cursors); + for (Py_ssize_t i = 0; i < imax; i++) { + PyObject* weakref = PyList_GET_ITEM(self->cursors, i); if (_PyWeakref_IsDead(weakref)) { continue; } if (PyList_Append(new_list, weakref) != 0) { Py_DECREF(new_list); - return; + return -1; } } Py_SETREF(self->cursors, new_list); + return 0; } /* Allocate a UDF/callback context structure. In order to ensure that the state @@ -1158,11 +1164,10 @@ check_num_params(pysqlite_Connection *self, const int n, const char *name) _sqlite3.Connection.create_function as pysqlite_connection_create_function cls: defining_class - / name: str narg: int func: object - / [from 3.15] + / * deterministic: bool = False @@ -1174,7 +1179,7 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, PyTypeObject *cls, const char *name, int narg, PyObject *func, int deterministic) -/*[clinic end generated code: output=8a811529287ad240 input=c7c313b0ca8b519e]*/ +/*[clinic end generated code: output=8a811529287ad240 input=a896096ed5390ae1]*/ { int rc; int flags = SQLITE_UTF8; @@ -1366,11 +1371,10 @@ create_window_function_impl(pysqlite_Connection *self, PyTypeObject *cls, _sqlite3.Connection.create_aggregate as pysqlite_connection_create_aggregate cls: defining_class - / name: str n_arg: int aggregate_class: object - / [from 3.15] + / Creates a new aggregate. [clinic start generated code]*/ @@ -1380,7 +1384,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, PyTypeObject *cls, const char *name, int n_arg, PyObject *aggregate_class) -/*[clinic end generated code: output=1b02d0f0aec7ff96 input=8087056db6eae1cf]*/ +/*[clinic end generated code: output=1b02d0f0aec7ff96 input=aa2773f6a42f7e17]*/ { int rc; @@ -1531,7 +1535,7 @@ _sqlite3.Connection.set_authorizer as pysqlite_connection_set_authorizer cls: defining_class authorizer_callback as callable: object - / [from 3.15] + / Set authorizer callback. [clinic start generated code]*/ @@ -1540,7 +1544,7 @@ static PyObject * pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=75fa60114fc971c3 input=a52bd4937c588752]*/ +/*[clinic end generated code: output=75fa60114fc971c3 input=e76469ab0bb1bbcd]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1569,6 +1573,7 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, } /*[clinic input] +@permit_long_docstring_body _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_handler cls: defining_class @@ -1576,7 +1581,7 @@ _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_han A callable that takes no arguments. If the callable returns non-zero, the current query is terminated, and an exception is raised. - / [from 3.15] + / n: int The number of SQLite virtual machine instructions that are executed between invocations of 'progress_handler'. @@ -1590,7 +1595,7 @@ static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n) -/*[clinic end generated code: output=0739957fd8034a50 input=b4d6e2ef8b4d32f9]*/ +/*[clinic end generated code: output=0739957fd8034a50 input=3ecce6c915922ad4]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1617,7 +1622,7 @@ _sqlite3.Connection.set_trace_callback as pysqlite_connection_set_trace_callback cls: defining_class trace_callback as callable: object - / [from 3.15] + / Set a trace callback called for each SQL statement (passed as unicode). [clinic start generated code]*/ @@ -1626,7 +1631,7 @@ static PyObject * pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=d91048c03bfcee05 input=d705d592ec03cf28]*/ +/*[clinic end generated code: output=d91048c03bfcee05 input=f4f59bf2f87f2026]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2218,6 +2223,7 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self, #ifdef PY_SQLITE_HAVE_SERIALIZE /*[clinic input] +@permit_long_docstring_body _sqlite3.Connection.serialize as serialize * @@ -2234,7 +2240,7 @@ were backed up to disk. static PyObject * serialize_impl(pysqlite_Connection *self, const char *name) -/*[clinic end generated code: output=97342b0e55239dd3 input=d2eb5194a65abe2b]*/ +/*[clinic end generated code: output=97342b0e55239dd3 input=963e617cdf75c747]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2268,6 +2274,7 @@ serialize_impl(pysqlite_Connection *self, const char *name) } /*[clinic input] +@permit_long_docstring_body _sqlite3.Connection.deserialize as deserialize data: Py_buffer(accept={buffer, str}) @@ -2290,7 +2297,7 @@ currently in a read transaction or is involved in a backup operation. static PyObject * deserialize_impl(pysqlite_Connection *self, Py_buffer *data, const char *name) -/*[clinic end generated code: output=e394c798b98bad89 input=1be4ca1faacf28f2]*/ +/*[clinic end generated code: output=e394c798b98bad89 input=037e94599aaa5b5c]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2404,6 +2411,7 @@ pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, } /*[clinic input] +@permit_long_docstring_body _sqlite3.Connection.setlimit as setlimit category: int @@ -2422,7 +2430,7 @@ the prior value of the limit is returned. static PyObject * setlimit_impl(pysqlite_Connection *self, int category, int limit) -/*[clinic end generated code: output=0d208213f8d68ccd input=9bd469537e195635]*/ +/*[clinic end generated code: output=0d208213f8d68ccd input=bf06e06a21eb37e2]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 7943bfcca36..4611c9e5e3e 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -31,6 +31,7 @@ #include "util.h" #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() typedef enum { TYPE_LONG, @@ -57,6 +58,41 @@ check_cursor_locked(pysqlite_Cursor *cur) return 1; } +static pysqlite_state * +get_module_state_by_cursor(pysqlite_Cursor *cursor) +{ + if (cursor->connection != NULL && cursor->connection->state != NULL) { + return cursor->connection->state; + } + return pysqlite_get_state_by_type(Py_TYPE(cursor)); +} + +static void +cursor_sqlite3_internal_error(pysqlite_Cursor *cursor, + const char *error_message, + int chain_exceptions) +{ + pysqlite_state *state = get_module_state_by_cursor(cursor); + if (chain_exceptions) { + assert(PyErr_Occurred()); + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(state->InternalError, error_message); + _PyErr_ChainExceptions1(exc); + } + else { + assert(!PyErr_Occurred()); + PyErr_SetString(state->InternalError, error_message); + } +} + +static void +cursor_cannot_reset_stmt_error(pysqlite_Cursor *cursor, int chain_exceptions) +{ + cursor_sqlite3_internal_error(cursor, + "cannot reset statement", + chain_exceptions); +} + /*[clinic input] module _sqlite3 class _sqlite3.Cursor "pysqlite_Cursor *" "clinic_state()->CursorType" @@ -172,8 +208,12 @@ cursor_clear(PyObject *op) Py_CLEAR(self->row_factory); if (self->statement) { /* Reset the statement if the user has not closed the cursor */ - stmt_reset(self->statement); + int rc = stmt_reset(self->statement); Py_CLEAR(self->statement); + if (rc != SQLITE_OK) { + cursor_cannot_reset_stmt_error(self, 0); + PyErr_FormatUnraisable("Exception ignored in cursor_clear()"); + } } return 0; @@ -185,9 +225,7 @@ cursor_dealloc(PyObject *op) pysqlite_Cursor *self = _pysqlite_Cursor_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->in_weakreflist); (void)tp->tp_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -472,6 +510,9 @@ static int check_cursor(pysqlite_Cursor* cur) return 0; } + assert(cur->connection != NULL); + assert(cur->connection->state != NULL); + if (cur->closed) { PyErr_SetString(cur->connection->state->ProgrammingError, "Cannot operate on a closed cursor."); @@ -568,43 +609,40 @@ bind_param(pysqlite_state *state, pysqlite_Statement *self, int pos, switch (paramtype) { case TYPE_LONG: { sqlite_int64 value = _pysqlite_long_as_int64(parameter); - if (value == -1 && PyErr_Occurred()) - rc = -1; - else - rc = sqlite3_bind_int64(self->st, pos, value); + rc = (value == -1 && PyErr_Occurred()) + ? SQLITE_ERROR + : sqlite3_bind_int64(self->st, pos, value); break; } case TYPE_FLOAT: { double value = PyFloat_AsDouble(parameter); - if (value == -1 && PyErr_Occurred()) { - rc = -1; - } - else { - rc = sqlite3_bind_double(self->st, pos, value); - } + rc = (value == -1 && PyErr_Occurred()) + ? SQLITE_ERROR + : sqlite3_bind_double(self->st, pos, value); break; } case TYPE_UNICODE: string = PyUnicode_AsUTF8AndSize(parameter, &buflen); - if (string == NULL) - return -1; + if (string == NULL) { + return SQLITE_ERROR; + } if (buflen > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "string longer than INT_MAX bytes"); - return -1; + return SQLITE_ERROR; } rc = sqlite3_bind_text(self->st, pos, string, (int)buflen, SQLITE_TRANSIENT); break; case TYPE_BUFFER: { Py_buffer view; if (PyObject_GetBuffer(parameter, &view, PyBUF_SIMPLE) != 0) { - return -1; + return SQLITE_ERROR; } if (view.len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "BLOB longer than INT_MAX bytes"); PyBuffer_Release(&view); - return -1; + return SQLITE_ERROR; } rc = sqlite3_bind_blob(self->st, pos, view.buf, (int)view.len, SQLITE_TRANSIENT); PyBuffer_Release(&view); @@ -614,7 +652,7 @@ bind_param(pysqlite_state *state, pysqlite_Statement *self, int pos, PyErr_Format(state->ProgrammingError, "Error binding parameter %d: type '%s' is not supported", pos, Py_TYPE(parameter)->tp_name); - rc = -1; + rc = SQLITE_ERROR; } final: @@ -734,14 +772,17 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, } binding_name++; /* skip first char (the colon) */ - PyObject *current_param; - (void)PyMapping_GetOptionalItemString(parameters, binding_name, &current_param); - if (!current_param) { - if (!PyErr_Occurred() || PyErr_ExceptionMatches(PyExc_LookupError)) { - PyErr_Format(state->ProgrammingError, - "You did not supply a value for binding " - "parameter :%s.", binding_name); - } + PyObject *current_param = NULL; + int found = PyMapping_GetOptionalItemString(parameters, + binding_name, + &current_param); + if (found == -1) { + return; + } + else if (found == 0) { + PyErr_Format(state->ProgrammingError, + "You did not supply a value for binding " + "parameter :%s.", binding_name); return; } @@ -835,7 +876,9 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation if (self->statement) { // Reset pending statements on this cursor. - (void)stmt_reset(self->statement); + if (stmt_reset(self->statement) != SQLITE_OK) { + goto reset_failure; + } } PyObject *stmt = get_statement_from_cache(self, operation); @@ -859,7 +902,9 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation } } - (void)stmt_reset(self->statement); + if (stmt_reset(self->statement) != SQLITE_OK) { + goto reset_failure; + } self->rowcount = self->statement->is_dml ? 0L : -1L; /* We start a transaction implicitly before a DML statement. @@ -941,7 +986,9 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation if (self->statement->is_dml) { self->rowcount += (long)sqlite3_changes(self->connection->db); } - stmt_reset(self->statement); + if (stmt_reset(self->statement) != SQLITE_OK) { + goto reset_failure; + } } Py_XDECREF(parameters); } @@ -966,8 +1013,15 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation if (PyErr_Occurred()) { if (self->statement) { - (void)stmt_reset(self->statement); + sqlite3 *db = sqlite3_db_handle(self->statement->st); + int sqlite3_state = sqlite3_errcode(db); + // stmt_reset() may return a previously set exception, + // either triggered because of Python or sqlite3. + rc = stmt_reset(self->statement); Py_CLEAR(self->statement); + if (sqlite3_state == SQLITE_OK && rc != SQLITE_OK) { + cursor_cannot_reset_stmt_error(self, 1); + } } self->rowcount = -1L; return NULL; @@ -976,6 +1030,20 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation Py_CLEAR(self->statement); } return Py_NewRef((PyObject *)self); + +reset_failure: + /* suite to execute when stmt_reset() failed and no exception is set */ + assert(!PyErr_Occurred()); + + Py_XDECREF(parameters); + Py_XDECREF(parameters_iter); + Py_XDECREF(parameters_list); + + self->locked = 0; + self->rowcount = -1L; + Py_CLEAR(self->statement); + cursor_cannot_reset_stmt_error(self, 0); + return NULL; } /*[clinic input] @@ -1118,14 +1186,20 @@ pysqlite_cursor_iternext(PyObject *op) if (self->statement->is_dml) { self->rowcount = (long)sqlite3_changes(self->connection->db); } - (void)stmt_reset(self->statement); + rc = stmt_reset(self->statement); Py_CLEAR(self->statement); + if (rc != SQLITE_OK) { + goto reset_failure; + } } else if (rc != SQLITE_ROW) { - set_error_from_db(self->connection->state, self->connection->db); - (void)stmt_reset(self->statement); + rc = set_error_from_db(self->connection->state, self->connection->db); + int reset_rc = stmt_reset(self->statement); Py_CLEAR(self->statement); Py_DECREF(row); + if (rc == SQLITE_OK && reset_rc != SQLITE_OK) { + goto reset_failure; + } return NULL; } if (!Py_IsNone(self->row_factory)) { @@ -1135,6 +1209,10 @@ pysqlite_cursor_iternext(PyObject *op) Py_SETREF(row, new_row); } return row; + +reset_failure: + cursor_cannot_reset_stmt_error(self, 0); + return NULL; } /*[clinic input] @@ -1160,35 +1238,31 @@ pysqlite_cursor_fetchone_impl(pysqlite_Cursor *self) /*[clinic input] _sqlite3.Cursor.fetchmany as pysqlite_cursor_fetchmany - size as maxrows: int(c_default='((pysqlite_Cursor *)self)->arraysize') = 1 + size as maxrows: uint32(c_default='((pysqlite_Cursor *)self)->arraysize') = 1 The default value is set by the Cursor.arraysize attribute. Fetches several rows from the resultset. [clinic start generated code]*/ static PyObject * -pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, int maxrows) -/*[clinic end generated code: output=a8ef31fea64d0906 input=035dbe44a1005bf2]*/ +pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, uint32_t maxrows) +/*[clinic end generated code: output=3325f2b477c71baf input=a509c412aa70b27e]*/ { PyObject* row; PyObject* list; - int counter = 0; list = PyList_New(0); if (!list) { return NULL; } - while ((row = pysqlite_cursor_iternext((PyObject *)self))) { - if (PyList_Append(list, row) < 0) { - Py_DECREF(row); - break; - } + while (maxrows > 0 && (row = pysqlite_cursor_iternext((PyObject *)self))) { + int rc = PyList_Append(list, row); Py_DECREF(row); - - if (++counter == maxrows) { + if (rc < 0) { break; } + maxrows--; } if (PyErr_Occurred()) { @@ -1293,8 +1367,15 @@ pysqlite_cursor_close_impl(pysqlite_Cursor *self) } if (self->statement) { - (void)stmt_reset(self->statement); + int rc = stmt_reset(self->statement); + // Force self->statement to be NULL even if stmt_reset() may have + // failed to avoid a possible double-free if someone calls close() + // twice as a leak here would be better than a double-free. Py_CLEAR(self->statement); + if (rc != SQLITE_OK) { + cursor_cannot_reset_stmt_error(self, 0); + return NULL; + } } self->closed = 1; @@ -1302,6 +1383,30 @@ pysqlite_cursor_close_impl(pysqlite_Cursor *self) Py_RETURN_NONE; } +/*[clinic input] +@getter +_sqlite3.Cursor.arraysize +[clinic start generated code]*/ + +static PyObject * +_sqlite3_Cursor_arraysize_get_impl(pysqlite_Cursor *self) +/*[clinic end generated code: output=e0919d97175e6c50 input=3278f8d3ecbd90e3]*/ +{ + return PyLong_FromUInt32(self->arraysize); +} + +/*[clinic input] +@setter +_sqlite3.Cursor.arraysize +[clinic start generated code]*/ + +static int +_sqlite3_Cursor_arraysize_set_impl(pysqlite_Cursor *self, PyObject *value) +/*[clinic end generated code: output=af59a6b09f8cce6e input=ace48cb114e26060]*/ +{ + return PyLong_AsUInt32(value, &self->arraysize); +} + static PyMethodDef cursor_methods[] = { PYSQLITE_CURSOR_CLOSE_METHODDEF PYSQLITE_CURSOR_EXECUTEMANY_METHODDEF @@ -1319,7 +1424,6 @@ static struct PyMemberDef cursor_members[] = { {"connection", _Py_T_OBJECT, offsetof(pysqlite_Cursor, connection), Py_READONLY}, {"description", _Py_T_OBJECT, offsetof(pysqlite_Cursor, description), Py_READONLY}, - {"arraysize", Py_T_INT, offsetof(pysqlite_Cursor, arraysize), 0}, {"lastrowid", _Py_T_OBJECT, offsetof(pysqlite_Cursor, lastrowid), Py_READONLY}, {"rowcount", Py_T_LONG, offsetof(pysqlite_Cursor, rowcount), Py_READONLY}, {"row_factory", _Py_T_OBJECT, offsetof(pysqlite_Cursor, row_factory), 0}, @@ -1327,6 +1431,11 @@ static struct PyMemberDef cursor_members[] = {NULL} }; +static struct PyGetSetDef cursor_getsets[] = { + _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF + {NULL}, +}; + static const char cursor_doc[] = PyDoc_STR("SQLite database cursor class."); @@ -1337,6 +1446,7 @@ static PyType_Slot cursor_slots[] = { {Py_tp_iternext, pysqlite_cursor_iternext}, {Py_tp_methods, cursor_methods}, {Py_tp_members, cursor_members}, + {Py_tp_getset, cursor_getsets}, {Py_tp_init, pysqlite_cursor_init}, {Py_tp_traverse, cursor_traverse}, {Py_tp_clear, cursor_clear}, diff --git a/Modules/_sqlite/cursor.h b/Modules/_sqlite/cursor.h index 42f817af7c5..c840a3d7ed0 100644 --- a/Modules/_sqlite/cursor.h +++ b/Modules/_sqlite/cursor.h @@ -35,7 +35,7 @@ typedef struct pysqlite_Connection* connection; PyObject* description; PyObject* row_cast_map; - int arraysize; + uint32_t arraysize; PyObject* lastrowid; long rowcount; PyObject* row_factory; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 27e8dab92e0..831dd9219f7 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -32,6 +32,7 @@ #include "microprotocols.h" #include "row.h" #include "blob.h" +#include "util.h" #if SQLITE_VERSION_NUMBER < 3015002 #error "SQLite 3.15.2 or higher required" @@ -60,26 +61,16 @@ pysqlite_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargsf, pysqlite_state *state = pysqlite_get_state(module); PyObject *factory = (PyObject *)state->ConnectionType; - static const int FACTORY_POS = 5; Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - if (nargs > 1 && nargs <= 8) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 1 positional argument to sqlite3.connect()" - " is deprecated. Parameters 'timeout', 'detect_types', " - "'isolation_level', 'check_same_thread', 'factory', " - "'cached_statements' and 'uri' will become keyword-only " - "parameters in Python 3.15.", 1)) - { - return NULL; - } + if (nargs > 1) { + PyErr_Format(PyExc_TypeError, + "connect() takes at most 1 positional arguments (%zd given)", nargs); + return NULL; } - if (nargs > FACTORY_POS) { - factory = args[FACTORY_POS]; - } - else if (kwnames != NULL) { + if (kwnames != NULL) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwnames); i++) { PyObject *item = PyTuple_GET_ITEM(kwnames, i); // borrowed ref. - if (PyUnicode_CompareWithASCIIString(item, "factory") == 0) { + if (PyUnicode_EqualToUTF8(item, "factory")) { factory = args[nargs + i]; break; } @@ -414,6 +405,40 @@ pysqlite_error_name(int rc) return NULL; } +static int +add_keyword_tuple(PyObject *module) +{ +#if SQLITE_VERSION_NUMBER >= 3024000 + int count = sqlite3_keyword_count(); + PyObject *keywords = PyTuple_New(count); + if (keywords == NULL) { + return -1; + } + for (int i = 0; i < count; i++) { + const char *keyword; + int size; + int result = sqlite3_keyword_name(i, &keyword, &size); + if (result != SQLITE_OK) { + pysqlite_state *state = pysqlite_get_state(module); + set_error_from_code(state, result); + goto error; + } + PyObject *kwd = PyUnicode_FromStringAndSize(keyword, size); + if (!kwd) { + goto error; + } + PyTuple_SET_ITEM(keywords, i, kwd); + } + return PyModule_Add(module, "SQLITE_KEYWORDS", keywords); + +error: + Py_DECREF(keywords); + return -1; +#else + return 0; +#endif +} + static int add_integer_constants(PyObject *module) { #define ADD_INT(ival) \ @@ -712,6 +737,10 @@ module_exec(PyObject *module) goto error; } + if (add_keyword_tuple(module) < 0) { + goto error; + } + if (PyModule_AddStringConstant(module, "sqlite_version", sqlite3_libversion())) { goto error; } @@ -745,7 +774,6 @@ module_exec(PyObject *module) return 0; error: - sqlite3_shutdown(); return -1; } diff --git a/Modules/_sqlite/prepare_protocol.c b/Modules/_sqlite/prepare_protocol.c index 31092417cb4..0e2812e1e4f 100644 --- a/Modules/_sqlite/prepare_protocol.c +++ b/Modules/_sqlite/prepare_protocol.c @@ -21,21 +21,21 @@ * 3. This notice may not be removed or altered from any source distribution. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "prepare_protocol.h" +#include "pycore_object.h" // _PyObject_VisitType() + + static int pysqlite_prepare_protocol_init(PyObject *self, PyObject *args, PyObject *kwargs) { return 0; } -static int -pysqlite_prepare_protocol_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static void pysqlite_prepare_protocol_dealloc(PyObject *self) { @@ -50,7 +50,7 @@ PyDoc_STRVAR(doc, "PEP 246 style object adaption protocol type."); static PyType_Slot type_slots[] = { {Py_tp_dealloc, pysqlite_prepare_protocol_dealloc}, {Py_tp_init, pysqlite_prepare_protocol_init}, - {Py_tp_traverse, pysqlite_prepare_protocol_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_doc, (void *)doc}, {0, NULL}, }; diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 736e60fd778..f31c699482f 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -21,10 +21,17 @@ * 3. This notice may not be removed or altered from any source distribution. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "connection.h" #include "statement.h" #include "util.h" +#include "pycore_object.h" // _PyObject_VisitType() + + #define _pysqlite_Statement_CAST(op) ((pysqlite_Statement *)(op)) /* prototypes */ @@ -96,7 +103,12 @@ pysqlite_statement_create(pysqlite_Connection *connection, PyObject *sql) return self; error: - (void)sqlite3_finalize(stmt); + assert(PyErr_Occurred()); + if (sqlite3_finalize(stmt) != SQLITE_OK) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(connection->InternalError, "cannot finalize statement"); + _PyErr_ChainExceptions1(exc); + } return NULL; } @@ -107,22 +119,21 @@ stmt_dealloc(PyObject *op) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(op); if (self->st) { + int rc; Py_BEGIN_ALLOW_THREADS - sqlite3_finalize(self->st); + rc = sqlite3_finalize(self->st); Py_END_ALLOW_THREADS - self->st = 0; + self->st = NULL; + if (rc != SQLITE_OK) { + pysqlite_state *state = PyType_GetModuleState(Py_TYPE(op)); + PyErr_SetString(state->InternalError, "cannot finalize statement"); + PyErr_FormatUnraisable("Exception ignored in stmt_dealloc()"); + } } tp->tp_free(self); Py_DECREF(tp); } -static int -stmt_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - /* * Strip leading whitespace and comments from incoming SQL (null terminated C * string) and return a pointer to the first non-whitespace, non-comment @@ -183,7 +194,7 @@ lstrip_sql(const char *sql) static PyType_Slot stmt_slots[] = { {Py_tp_dealloc, stmt_dealloc}, - {Py_tp_traverse, stmt_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0, NULL}, }; diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 103248ff55a..177e0f668a0 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -135,14 +135,14 @@ set_error_from_code(pysqlite_state *state, int code) /** * Checks the SQLite error code and sets the appropriate DB-API exception. */ -void +int set_error_from_db(pysqlite_state *state, sqlite3 *db) { int errorcode = sqlite3_errcode(db); PyObject *exc_class = get_exception_class(state, errorcode); if (exc_class == NULL) { // No new exception need be raised. - return; + return SQLITE_OK; } /* Create and set the exception. */ @@ -150,6 +150,7 @@ set_error_from_db(pysqlite_state *state, sqlite3 *db) // sqlite3_errmsg() always returns an UTF-8 encoded message const char *errmsg = sqlite3_errmsg(db); raise_exception(exc_class, extended_errcode, errmsg); + return errorcode; } #ifdef WORDS_BIGENDIAN diff --git a/Modules/_sqlite/util.h b/Modules/_sqlite/util.h index f8e45baffae..c00369496f9 100644 --- a/Modules/_sqlite/util.h +++ b/Modules/_sqlite/util.h @@ -31,7 +31,7 @@ /** * Checks the SQLite error code and sets the appropriate DB-API exception. */ -void set_error_from_db(pysqlite_state *state, sqlite3 *db); +int set_error_from_db(pysqlite_state *state, sqlite3 *db); void set_error_from_code(pysqlite_state *state, int code); sqlite_int64 _pysqlite_long_as_int64(PyObject * value); diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 602d0ab8588..59ff9078e6c 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -44,6 +44,7 @@ static const char copyright[] = #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_unicodeobject.h" // _PyUnicode_Copy +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "sre.h" // SRE_CODE @@ -736,10 +737,7 @@ pattern_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - PatternObject *obj = _PatternObject_CAST(self); - if (obj->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, _PatternObject_CAST(self)->weakreflist); (void)pattern_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -857,6 +855,7 @@ _sre_SRE_Pattern_fullmatch_impl(PatternObject *self, PyTypeObject *cls, } /*[clinic input] +@permit_long_summary _sre.SRE_Pattern.search cls: defining_class @@ -874,7 +873,7 @@ static PyObject * _sre_SRE_Pattern_search_impl(PatternObject *self, PyTypeObject *cls, PyObject *string, Py_ssize_t pos, Py_ssize_t endpos) -/*[clinic end generated code: output=bd7f2d9d583e1463 input=afa9afb66a74a4b3]*/ +/*[clinic end generated code: output=bd7f2d9d583e1463 input=05e9feee0334c156]*/ { _sremodulestate *module_state = get_sre_module_state_by_class(cls); SRE_STATE state; @@ -999,6 +998,7 @@ _sre_SRE_Pattern_findall_impl(PatternObject *self, PyObject *string, } /*[clinic input] +@permit_long_summary _sre.SRE_Pattern.finditer cls: defining_class @@ -1016,7 +1016,7 @@ static PyObject * _sre_SRE_Pattern_finditer_impl(PatternObject *self, PyTypeObject *cls, PyObject *string, Py_ssize_t pos, Py_ssize_t endpos) -/*[clinic end generated code: output=1791dbf3618ade56 input=812e332a4848cbaf]*/ +/*[clinic end generated code: output=1791dbf3618ade56 input=ee28865796048023]*/ { _sremodulestate *module_state = get_sre_module_state_by_class(cls); PyObject* scanner; @@ -1418,6 +1418,7 @@ pattern_subx(_sremodulestate* module_state, } /*[clinic input] +@permit_long_summary _sre.SRE_Pattern.sub cls: defining_class @@ -1432,7 +1433,7 @@ Return the string obtained by replacing the leftmost non-overlapping occurrences static PyObject * _sre_SRE_Pattern_sub_impl(PatternObject *self, PyTypeObject *cls, PyObject *repl, PyObject *string, Py_ssize_t count) -/*[clinic end generated code: output=4be141ab04bca60d input=d8d1d4ac2311a07c]*/ +/*[clinic end generated code: output=4be141ab04bca60d input=eba511fd1c4908b7]*/ { _sremodulestate *module_state = get_sre_module_state_by_class(cls); @@ -1440,6 +1441,7 @@ _sre_SRE_Pattern_sub_impl(PatternObject *self, PyTypeObject *cls, } /*[clinic input] +@permit_long_summary _sre.SRE_Pattern.subn cls: defining_class @@ -1455,7 +1457,7 @@ static PyObject * _sre_SRE_Pattern_subn_impl(PatternObject *self, PyTypeObject *cls, PyObject *repl, PyObject *string, Py_ssize_t count) -/*[clinic end generated code: output=da02fd85258b1e1f input=8b78a65b8302e58d]*/ +/*[clinic end generated code: output=da02fd85258b1e1f input=6a5bb5b61717abf0]*/ { _sremodulestate *module_state = get_sre_module_state_by_class(cls); @@ -1944,7 +1946,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) sre_match() code is robust even if they don't, and the worst you can get is nonsensical match results. */ GET_ARG; - if (arg > 2 * (size_t)groups + 1) { + if (arg >= 2 * (size_t)groups) { VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups)); FAIL; } @@ -2357,7 +2359,7 @@ match_getindex(MatchObject* self, PyObject* index) } // Check that i*2 cannot overflow to make static analyzers happy - assert(i <= SRE_MAXGROUPS); + assert((size_t)i <= SRE_MAXGROUPS); return i; } @@ -2374,6 +2376,7 @@ match_getslice(MatchObject* self, PyObject* index, PyObject* def) } /*[clinic input] +@permit_long_summary _sre.SRE_Match.expand template: object @@ -2383,7 +2386,7 @@ Return the string obtained by doing backslash substitution on the string templat static PyObject * _sre_SRE_Match_expand_impl(MatchObject *self, PyObject *template) -/*[clinic end generated code: output=931b58ccc323c3a1 input=4bfdb22c2f8b146a]*/ +/*[clinic end generated code: output=931b58ccc323c3a1 input=dc74d81265376ac3]*/ { _sremodulestate *module_state = get_sre_module_state_by_class(Py_TYPE(self)); PyObject *filter = compile_template(module_state, self->pattern, template); @@ -2472,6 +2475,7 @@ _sre_SRE_Match_groups_impl(MatchObject *self, PyObject *default_value) } /*[clinic input] +@permit_long_summary _sre.SRE_Match.groupdict default: object = None @@ -2482,7 +2486,7 @@ Return a dictionary containing all the named subgroups of the match, keyed by th static PyObject * _sre_SRE_Match_groupdict_impl(MatchObject *self, PyObject *default_value) -/*[clinic end generated code: output=29917c9073e41757 input=0ded7960b23780aa]*/ +/*[clinic end generated code: output=29917c9073e41757 input=a8d3a1dc80336872]*/ { PyObject *result; PyObject *key; @@ -2837,20 +2841,25 @@ scanner_dealloc(PyObject *self) static int scanner_begin(ScannerObject* self) { - if (self->executing) { +#ifdef Py_GIL_DISABLED + int was_executing = _Py_atomic_exchange_int(&self->executing, 1); +#else + int was_executing = self->executing; + self->executing = 1; +#endif + if (was_executing) { PyErr_SetString(PyExc_ValueError, "regular expression scanner already executing"); return 0; } - self->executing = 1; return 1; } static void scanner_end(ScannerObject* self) { - assert(self->executing); - self->executing = 0; + assert(FT_ATOMIC_LOAD_INT_RELAXED(self->executing)); + FT_ATOMIC_STORE_INT(self->executing, 0); } /*[clinic input] diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 97a29f4d0e1..25fcea6aaf1 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -43,14 +43,14 @@ /* Redefined below for Windows debug builds after important #includes */ #define _PySSL_FIX_ERRNO -#define PySSL_BEGIN_ALLOW_THREADS_S(save) \ - do { (save) = PyEval_SaveThread(); } while(0) -#define PySSL_END_ALLOW_THREADS_S(save) \ - do { PyEval_RestoreThread(save); _PySSL_FIX_ERRNO; } while(0) -#define PySSL_BEGIN_ALLOW_THREADS { \ +#define PySSL_BEGIN_ALLOW_THREADS_S(save, mutex) \ + do { (save) = PyEval_SaveThread(); PyMutex_Lock(mutex); } while(0) +#define PySSL_END_ALLOW_THREADS_S(save, mutex) \ + do { PyMutex_Unlock(mutex); PyEval_RestoreThread(save); _PySSL_FIX_ERRNO; } while(0) +#define PySSL_BEGIN_ALLOW_THREADS(self) { \ PyThreadState *_save = NULL; \ - PySSL_BEGIN_ALLOW_THREADS_S(_save); -#define PySSL_END_ALLOW_THREADS PySSL_END_ALLOW_THREADS_S(_save); } + PySSL_BEGIN_ALLOW_THREADS_S(_save, &self->tstate_mutex); +#define PySSL_END_ALLOW_THREADS(self) PySSL_END_ALLOW_THREADS_S(_save, &self->tstate_mutex); } #if defined(HAVE_POLL_H) #include <poll.h> @@ -75,6 +75,33 @@ #endif +#ifdef BIO_get_ktls_send +# ifdef MS_WINDOWS +typedef long long Py_off_t; +# else +typedef off_t Py_off_t; +# endif + +static int +Py_off_t_converter(PyObject *arg, void *addr) +{ +#ifdef HAVE_LARGEFILE_SUPPORT + *((Py_off_t *)addr) = PyLong_AsLongLong(arg); +#else + *((Py_off_t *)addr) = PyLong_AsLong(arg); +#endif + return PyErr_Occurred() ? 0 : 1; +} + +/*[python input] + +class Py_off_t_converter(CConverter): + type = 'Py_off_t' + converter = 'Py_off_t_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=3fd9ca8ca6f0cbb8]*/ +#endif /* BIO_get_ktls_send */ struct py_ssl_error_code { const char *mnemonic; @@ -121,9 +148,9 @@ static void _PySSLFixErrno(void) { #endif /* Include generated data (error codes) */ -/* See make_ssl_data.h for notes on adding a new version. */ +/* See Tools/ssl/make_ssl_data.py for notes on adding a new version. */ #if (OPENSSL_VERSION_NUMBER >= 0x30401000L) -#include "_ssl_data_34.h" +#include "_ssl_data_35.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30100000L) #include "_ssl_data_340.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30000000L) @@ -309,6 +336,9 @@ typedef struct { PyObject *psk_client_callback; PyObject *psk_server_callback; #endif + /* Lock to synchronize calls when the thread state is detached. + See also gh-134698. */ + PyMutex tstate_mutex; } PySSLContext; #define PySSLContext_CAST(op) ((PySSLContext *)(op)) @@ -563,7 +593,7 @@ fill_and_set_sslerror(_sslmodulestate *state, goto fail; } } - if (PyUnicodeWriter_WriteUTF8(writer, "] ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "] ", 2) < 0) { goto fail; } } @@ -889,9 +919,9 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, /* Make sure the SSL error state is initialized */ ERR_clear_error(); - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(sslctx) self->ssl = SSL_new(ctx); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(sslctx) if (self->ssl == NULL) { Py_DECREF(self); _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); @@ -907,7 +937,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, } /* bpo43522 and OpenSSL < 1.1.1l: copy hostflags manually */ -#if OPENSSL_VERSION < 0x101010cf +#if OPENSSL_VERSION_NUMBER < 0x101010cf X509_VERIFY_PARAM *ssl_verification_params = SSL_get0_param(self->ssl); X509_VERIFY_PARAM *ssl_ctx_verification_params = SSL_CTX_get0_param(ctx); @@ -960,12 +990,12 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, BIO_set_nbio(SSL_get_wbio(self->ssl), 1); } - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; if (socket_type == PY_SSL_CLIENT) SSL_set_connect_state(self->ssl); else SSL_set_accept_state(self->ssl); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; self->socket_type = socket_type; if (sock != NULL) { @@ -1034,10 +1064,11 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) /* Actually negotiate SSL connection */ /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ do { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ret = SSL_do_handshake(self->ssl); err = _PySSL_errno(ret < 1, self->ssl, ret); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -1406,14 +1437,14 @@ _get_peer_alt_names (_sslmodulestate *state, X509 *certificate) { } PyTuple_SET_ITEM(t, 0, v); - if (name->d.ip->length == 4) { - unsigned char *p = name->d.ip->data; + if (ASN1_STRING_length(name->d.ip) == 4) { + const unsigned char *p = ASN1_STRING_get0_data(name->d.ip); v = PyUnicode_FromFormat( "%d.%d.%d.%d", p[0], p[1], p[2], p[3] ); - } else if (name->d.ip->length == 16) { - unsigned char *p = name->d.ip->data; + } else if (ASN1_STRING_length(name->d.ip) == 16) { + const unsigned char *p = ASN1_STRING_get0_data(name->d.ip); v = PyUnicode_FromFormat( "%X:%X:%X:%X:%X:%X:%X:%X", p[0] << 8 | p[1], @@ -1544,8 +1575,9 @@ _get_aia_uri(X509 *certificate, int nid) { continue; } uri = ad->location->d.uniformResourceIdentifier; - ostr = PyUnicode_FromStringAndSize((char *)uri->data, - uri->length); + ostr = PyUnicode_FromStringAndSize( + (const char *)ASN1_STRING_get0_data(uri), + ASN1_STRING_length(uri)); if (ostr == NULL) { goto fail; } @@ -1611,8 +1643,9 @@ _get_crl_dp(X509 *certificate) { continue; } uri = gn->d.uniformResourceIdentifier; - ouri = PyUnicode_FromStringAndSize((char *)uri->data, - uri->length); + ouri = PyUnicode_FromStringAndSize( + (const char *)ASN1_STRING_get0_data(uri), + ASN1_STRING_length(uri)); if (ouri == NULL) goto done; @@ -1827,14 +1860,14 @@ _certificate_to_der(_sslmodulestate *state, X509 *certificate) /*[clinic input] _ssl._test_decode_cert - path: object(converter="PyUnicode_FSConverter") + path: unicode_fs_encoded / [clinic start generated code]*/ static PyObject * _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=96becb9abb23c091 input=cdeaaf02d4346628]*/ +/*[clinic end generated code: output=96becb9abb23c091 input=cb4988d5e651a4f8]*/ { PyObject *retval = NULL; X509 *x=NULL; @@ -1864,7 +1897,6 @@ _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) X509_free(x); fail0: - Py_DECREF(path); if (cert != NULL) BIO_free(cert); return retval; } @@ -2142,6 +2174,83 @@ _ssl__SSLSocket_cipher_impl(PySSLSocket *self) return cipher_to_tuple(current); } +/*[clinic input] +@critical_section +_ssl._SSLSocket.group +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_group_impl(PySSLSocket *self) +/*[clinic end generated code: output=9c168ee877017b95 input=5f187d8bf0d433b7]*/ +{ +#if OPENSSL_VERSION_NUMBER >= 0x30200000L + const char *group_name; + + if (self->ssl == NULL) { + Py_RETURN_NONE; + } + group_name = SSL_get0_group_name(self->ssl); + if (group_name == NULL) { + Py_RETURN_NONE; + } + return PyUnicode_DecodeFSDefault(group_name); +#else + PyErr_SetString(PyExc_NotImplementedError, + "Getting selected group requires OpenSSL 3.2 or later."); + return NULL; +#endif +} + +static PyObject * +ssl_socket_signame_impl(PySSLSocket *socket, + enum py_ssl_server_or_client self_socket_type) +{ +#if OPENSSL_VERSION_NUMBER >= 0x30500000L + int ret; + const char *sigalg; + + if (socket->ssl == NULL) { + Py_RETURN_NONE; + } + ret = (socket->socket_type == self_socket_type) + ? SSL_get0_signature_name(socket->ssl, &sigalg) + : SSL_get0_peer_signature_name(socket->ssl, &sigalg); + if (ret == 0) { + Py_RETURN_NONE; + } + assert(sigalg != NULL); + return PyUnicode_DecodeFSDefault(sigalg); +#else + PyErr_SetString(PyExc_NotImplementedError, + "Getting sig algorithms requires OpenSSL 3.5 or later."); + return NULL; +#endif +} + +/*[clinic input] +@critical_section +_ssl._SSLSocket.client_sigalg +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_client_sigalg_impl(PySSLSocket *self) +/*[clinic end generated code: output=499dd7fbf021a47b input=a0d9696b5414c627]*/ +{ + return ssl_socket_signame_impl(self, PY_SSL_CLIENT); +} + +/*[clinic input] +@critical_section +_ssl._SSLSocket.server_sigalg +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_server_sigalg_impl(PySSLSocket *self) +/*[clinic end generated code: output=c508a766a8e275dc input=9063e562a1e6b946]*/ +{ + return ssl_socket_signame_impl(self, PY_SSL_SERVER); +} + /*[clinic input] @critical_section _ssl._SSLSocket.version @@ -2211,6 +2320,7 @@ _ssl__SSLSocket_compression_impl(PySSLSocket *self) } /*[clinic input] +@permit_long_docstring_body @critical_section @getter _ssl._SSLSocket.context @@ -2224,12 +2334,13 @@ SSLSocket before the cryptographic exchange handshake messages. static PyObject * _ssl__SSLSocket_context_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=d23e82f72f32e3d7 input=7cbb97407c2ace30]*/ +/*[clinic end generated code: output=d23e82f72f32e3d7 input=0cc8e773a079295e]*/ { return Py_NewRef(self->ctx); } /*[clinic input] +@permit_long_docstring_body @critical_section @setter _ssl._SSLSocket.context @@ -2237,7 +2348,7 @@ _ssl._SSLSocket.context static int _ssl__SSLSocket_context_set_impl(PySSLSocket *self, PyObject *value) -/*[clinic end generated code: output=6b0a6cc5cf33d9fe input=48ece77724fd9dd4]*/ +/*[clinic end generated code: output=6b0a6cc5cf33d9fe input=f7fc1674b660df96]*/ { if (PyObject_TypeCheck(value, self->ctx->state->PySSLContext_Type)) { Py_SETREF(self->ctx, (PySSLContext *)Py_NewRef(value)); @@ -2414,9 +2525,10 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) ms = (int)_PyTime_AsMilliseconds(timeout, _PyTime_ROUND_CEILING); assert(ms <= INT_MAX); - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS rc = poll(&pollfd, 1, (int)ms); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; #else /* Guard against socket too large for select*/ if (!_PyIsSelectable_fd(s->sock_fd)) @@ -2428,13 +2540,14 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) FD_SET(s->sock_fd, &fds); /* Wait until the socket becomes ready */ - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS nfds = Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int); if (writing) rc = select(nfds, NULL, &fds, NULL, &tv); else rc = select(nfds, &fds, NULL, NULL, &tv); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; #endif /* Return SOCKET_TIMED_OUT on timeout, SOCKET_OPERATION_OK otherwise @@ -2442,6 +2555,187 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) return rc == 0 ? SOCKET_HAS_TIMED_OUT : SOCKET_OPERATION_OK; } +/*[clinic input] +@critical_section +_ssl._SSLSocket.uses_ktls_for_send + +Check if the Kernel TLS data-path is used for sending. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self) +/*[clinic end generated code: output=f9d95fbefceb5068 input=8d1ce4a131190e6b]*/ +{ +#ifdef BIO_get_ktls_send + int uses = BIO_get_ktls_send(SSL_get_wbio(self->ssl)); + // BIO_get_ktls_send() returns 1 if kTLS is used and 0 if not. + // Also, it returns -1 for failure before OpenSSL 3.0.4. + return Py_NewRef(uses == 1 ? Py_True : Py_False); +#else + Py_RETURN_FALSE; +#endif +} + +/*[clinic input] +@critical_section +_ssl._SSLSocket.uses_ktls_for_recv + +Check if the Kernel TLS data-path is used for receiving. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self) +/*[clinic end generated code: output=ce38b00317a1f681 input=a13778a924fc7d44]*/ +{ +#ifdef BIO_get_ktls_recv + int uses = BIO_get_ktls_recv(SSL_get_rbio(self->ssl)); + // BIO_get_ktls_recv() returns 1 if kTLS is used and 0 if not. + // Also, it returns -1 for failure before OpenSSL 3.0.4. + return Py_NewRef(uses == 1 ? Py_True : Py_False); +#else + Py_RETURN_FALSE; +#endif +} + +#ifdef BIO_get_ktls_send +/*[clinic input] +@permit_long_summary +@permit_long_docstring_body +@critical_section +_ssl._SSLSocket.sendfile + fd: int + offset: Py_off_t + size: size_t + flags: int = 0 + / + +Write size bytes from offset in the file descriptor fd to the SSL connection. + +This method uses the zero-copy technique and returns the number of bytes +written. It should be called only when Kernel TLS is used for sending data in +the connection. + +The meaning of flags is platform dependent. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, + size_t size, int flags) +/*[clinic end generated code: output=0c6815b0719ca8d5 input=1f193e681bbae664]*/ +{ + Py_ssize_t retval; + int sockstate; + _PySSLError err; + PySocketSockObject *sock = GET_SOCKET(self); + PyTime_t timeout, deadline = 0; + int has_timeout; + + if (sock != NULL) { + if ((PyObject *)sock == Py_None) { + _setSSLError(get_state_sock(self), + "Underlying socket connection gone", + PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); + return NULL; + } + Py_INCREF(sock); + /* just in case the blocking state of the socket has been changed */ + int nonblocking = (sock->sock_timeout >= 0); + BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); + BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + } + + timeout = GET_SOCKET_TIMEOUT(sock); + has_timeout = (timeout > 0); + if (has_timeout) { + deadline = _PyDeadline_Init(timeout); + } + + sockstate = PySSL_select(sock, 1, timeout); + switch (sockstate) { + case SOCKET_HAS_TIMED_OUT: + PyErr_SetString(PyExc_TimeoutError, + "The write operation timed out"); + goto error; + case SOCKET_HAS_BEEN_CLOSED: + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket has been closed."); + goto error; + case SOCKET_TOO_LARGE_FOR_SELECT: + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket too large for select()."); + goto error; + } + + do { + Py_BEGIN_ALLOW_THREADS + retval = SSL_sendfile(self->ssl, fd, (off_t)offset, size, flags); + err = _PySSL_errno(retval < 0, self->ssl, (int)retval); + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; + self->err = err; + + if (PyErr_CheckSignals()) { + goto error; + } + + if (has_timeout) { + timeout = _PyDeadline_Get(deadline); + } + + switch (err.ssl) { + case SSL_ERROR_WANT_READ: + sockstate = PySSL_select(sock, 0, timeout); + break; + case SSL_ERROR_WANT_WRITE: + sockstate = PySSL_select(sock, 1, timeout); + break; + default: + sockstate = SOCKET_OPERATION_OK; + break; + } + + if (sockstate == SOCKET_HAS_TIMED_OUT) { + PyErr_SetString(PyExc_TimeoutError, + "The sendfile operation timed out"); + goto error; + } + else if (sockstate == SOCKET_HAS_BEEN_CLOSED) { + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket has been closed."); + goto error; + } + else if (sockstate == SOCKET_IS_NONBLOCKING) { + break; + } + } while (err.ssl == SSL_ERROR_WANT_READ + || err.ssl == SSL_ERROR_WANT_WRITE); + + if (err.ssl == SSL_ERROR_SSL + && ERR_GET_REASON(ERR_peek_error()) == SSL_R_UNINITIALIZED) + { + /* OpenSSL fails to return SSL_ERROR_SYSCALL if an error + * happens in sendfile(), and returns SSL_ERROR_SSL with + * SSL_R_UNINITIALIZED reason instead. */ + _setSSLError(get_state_sock(self), + "Some I/O error occurred in sendfile()", + PY_SSL_ERROR_SYSCALL, __FILE__, __LINE__); + goto error; + } + Py_XDECREF(sock); + if (retval < 0) { + return PySSL_SetError(self, __FILE__, __LINE__); + } + if (PySSL_ChainExceptions(self) < 0) { + return NULL; + } + return PyLong_FromSize_t(retval); +error: + Py_XDECREF(sock); + (void)PySSL_ChainExceptions(self); + return NULL; +} +#endif /* BIO_get_ktls_send */ + /*[clinic input] @critical_section _ssl._SSLSocket.write @@ -2505,10 +2799,11 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) } do { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; retval = SSL_write_ex(self->ssl, b->buf, (size_t)b->len, &count); err = _PySSL_errno(retval == 0, self->ssl, retval); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -2553,6 +2848,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) } /*[clinic input] +@permit_long_summary @critical_section _ssl._SSLSocket.pending @@ -2561,15 +2857,16 @@ Returns the number of already decrypted bytes available for read, pending on the static PyObject * _ssl__SSLSocket_pending_impl(PySSLSocket *self) -/*[clinic end generated code: output=983d9fecdc308a83 input=32ab982a254e8866]*/ +/*[clinic end generated code: output=983d9fecdc308a83 input=042dcc48bdf3e312]*/ { int count = 0; _PySSLError err; - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; count = SSL_pending(self->ssl); err = _PySSL_errno(count < 0, self->ssl, count); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (count < 0) @@ -2595,7 +2892,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, int group_right_1, Py_buffer *buffer) /*[clinic end generated code: output=49b16e6406023734 input=80ed30436df01a71]*/ { - PyObject *dest = NULL; + PyBytesWriter *writer = NULL; char *mem; size_t count = 0; int retval; @@ -2622,14 +2919,16 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, } if (!group_right_1) { - dest = PyBytes_FromStringAndSize(NULL, len); - if (dest == NULL) - goto error; if (len == 0) { Py_XDECREF(sock); - return dest; + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } - mem = PyBytes_AS_STRING(dest); + + writer = PyBytesWriter_Create(len); + if (writer == NULL) { + goto error; + } + mem = PyBytesWriter_GetData(writer); } else { mem = buffer->buf; @@ -2660,10 +2959,11 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, deadline = _PyDeadline_Init(timeout); do { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; retval = SSL_read_ex(self->ssl, mem, (size_t)len, &count); err = _PySSL_errno(retval == 0, self->ssl, retval); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -2706,8 +3006,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, done: Py_XDECREF(sock); if (!group_right_1) { - _PyBytes_Resize(&dest, count); - return dest; + return PyBytesWriter_FinishWithSize(writer, count); } else { return PyLong_FromSize_t(count); @@ -2716,8 +3015,9 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, error: PySSL_ChainExceptions(self); Py_XDECREF(sock); - if (!group_right_1) - Py_XDECREF(dest); + if (!group_right_1) { + PyBytesWriter_Discard(writer); + } return NULL; } @@ -2762,7 +3062,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } while (1) { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; /* Disable read-ahead so that unwrap can work correctly. * Otherwise OpenSSL might read in too much data, * eating clear text data that happens to be @@ -2775,7 +3075,8 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) SSL_set_read_ahead(self->ssl, 0); ret = SSL_shutdown(self->ssl); err = _PySSL_errno(ret < 0, self->ssl, ret); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; /* If err == 1, a secure shutdown with SSL_shutdown() is complete */ @@ -2842,6 +3143,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } /*[clinic input] +@permit_long_docstring_body @critical_section _ssl._SSLSocket.get_channel_binding cb_type: str = "tls-unique" @@ -2856,7 +3158,7 @@ Only 'tls-unique' channel binding data from RFC 5929 is supported. static PyObject * _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, const char *cb_type) -/*[clinic end generated code: output=34bac9acb6a61d31 input=e008004fc08744db]*/ +/*[clinic end generated code: output=34bac9acb6a61d31 input=26fad522435ecca1]*/ { char buf[PySSL_CB_MAXLEN]; size_t len; @@ -3017,12 +3319,18 @@ static PyGetSetDef ssl_getsetlist[] = { static PyMethodDef PySSLMethods[] = { _SSL__SSLSOCKET_DO_HANDSHAKE_METHODDEF + _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF + _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF + _SSL__SSLSOCKET_SENDFILE_METHODDEF _SSL__SSLSOCKET_WRITE_METHODDEF _SSL__SSLSOCKET_READ_METHODDEF _SSL__SSLSOCKET_PENDING_METHODDEF _SSL__SSLSOCKET_GETPEERCERT_METHODDEF _SSL__SSLSOCKET_GET_CHANNEL_BINDING_METHODDEF _SSL__SSLSOCKET_CIPHER_METHODDEF + _SSL__SSLSOCKET_GROUP_METHODDEF + _SSL__SSLSOCKET_CLIENT_SIGALG_METHODDEF + _SSL__SSLSOCKET_SERVER_SIGALG_METHODDEF _SSL__SSLSOCKET_SHARED_CIPHERS_METHODDEF _SSL__SSLSOCKET_VERSION_METHODDEF _SSL__SSLSOCKET_SELECTED_ALPN_PROTOCOL_METHODDEF @@ -3167,9 +3475,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) // no other thread can be touching this object yet. // (Technically, we can't even lock if we wanted to, as the // lock hasn't been initialized yet.) - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ctx = SSL_CTX_new(method); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (ctx == NULL) { _setSSLError(get_ssl_state(module), NULL, 0, __FILE__, __LINE__); @@ -3194,6 +3503,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) self->psk_client_callback = NULL; self->psk_server_callback = NULL; #endif + self->tstate_mutex = (PyMutex){0}; /* Don't check host name by default */ if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { @@ -3312,9 +3622,10 @@ context_clear(PyObject *op) Py_CLEAR(self->psk_server_callback); #endif if (self->keylog_bio != NULL) { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS BIO_free_all(self->keylog_bio); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; self->keylog_bio = NULL; } return 0; @@ -3358,6 +3669,25 @@ _ssl__SSLContext_set_ciphers_impl(PySSLContext *self, const char *cipherlist) Py_RETURN_NONE; } +/*[clinic input] +@critical_section +_ssl._SSLContext.set_ciphersuites + ciphersuites: str + / +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_set_ciphersuites_impl(PySSLContext *self, + const char *ciphersuites) +/*[clinic end generated code: output=9915bec58e54d76d input=2afcc3693392be41]*/ +{ + if (!SSL_CTX_set_ciphersuites(self->ctx, ciphersuites)) { + _setSSLError(get_state_ctx(self), "No cipher suite can be selected.", 0, __FILE__, __LINE__); + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] @critical_section _ssl._SSLContext.get_ciphers @@ -3402,6 +3732,129 @@ _ssl__SSLContext_get_ciphers_impl(PySSLContext *self) } +/*[clinic input] +@critical_section +_ssl._SSLContext.set_groups + grouplist: str + / +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_set_groups_impl(PySSLContext *self, const char *grouplist) +/*[clinic end generated code: output=0b5d05dfd371ffd0 input=2cc64cef21930741]*/ +{ + if (!SSL_CTX_set1_groups_list(self->ctx, grouplist)) { + _setSSLError(get_state_ctx(self), "unrecognized group", 0, __FILE__, __LINE__); + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +@critical_section +_ssl._SSLContext.get_groups + * + include_aliases: bool = False +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_get_groups_impl(PySSLContext *self, int include_aliases) +/*[clinic end generated code: output=6d6209dd1051529b input=3e8ee5deb277dcc5]*/ +{ +#if OPENSSL_VERSION_NUMBER >= 0x30500000L + STACK_OF(OPENSSL_CSTRING) *groups = NULL; + const char *group; + int i, num; + PyObject *item, *result = NULL; + + // This "groups" object is dynamically allocated, but the strings inside + // it are internal constants which shouldn't ever be modified or freed. + if ((groups = sk_OPENSSL_CSTRING_new_null()) == NULL) { + _setSSLError(get_state_ctx(self), "Can't allocate stack", 0, __FILE__, __LINE__); + goto error; + } + + if (!SSL_CTX_get0_implemented_groups(self->ctx, include_aliases, groups)) { + _setSSLError(get_state_ctx(self), "Can't get groups", 0, __FILE__, __LINE__); + goto error; + } + + num = sk_OPENSSL_CSTRING_num(groups); + result = PyList_New(num); + if (result == NULL) { + goto error; + } + + for (i = 0; i < num; ++i) { + // There's no allocation here, so group won't ever be NULL. + group = sk_OPENSSL_CSTRING_value(groups, i); + assert(group != NULL); + + // Group names are plain ASCII, so there's no chance of a decoding + // error here. However, an allocation failure could occur when + // constructing the Unicode version of the names. + if ((item = PyUnicode_DecodeFSDefault(group)) == NULL) { + goto error; + } + + PyList_SET_ITEM(result, i, item); + } + + sk_OPENSSL_CSTRING_free(groups); + return result; +error: + Py_XDECREF(result); + sk_OPENSSL_CSTRING_free(groups); + return NULL; +#else + PyErr_SetString(PyExc_NotImplementedError, + "Getting implemented groups requires OpenSSL 3.5 or later."); + return NULL; +#endif +} + +/*[clinic input] +@critical_section +_ssl._SSLContext.set_client_sigalgs + sigalgslist: str + / +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_set_client_sigalgs_impl(PySSLContext *self, + const char *sigalgslist) +/*[clinic end generated code: output=f4f5be160a29c7d6 input=500d853ce9fd94ff]*/ +{ +#ifdef OPENSSL_IS_AWSLC + _setSSLError(get_state_ctx(self), "can't set client sigalgs on AWS-LC", 0, __FILE__, __LINE__); + return NULL; +#else + if (!SSL_CTX_set1_client_sigalgs_list(self->ctx, sigalgslist)) { + _setSSLError(get_state_ctx(self), "unrecognized signature algorithm", 0, __FILE__, __LINE__); + return NULL; + } + Py_RETURN_NONE; +#endif +} + +/*[clinic input] +@critical_section +_ssl._SSLContext.set_server_sigalgs + sigalgslist: str + / +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLContext_set_server_sigalgs_impl(PySSLContext *self, + const char *sigalgslist) +/*[clinic end generated code: output=31ecb1d310285644 input=653b752e4f8d801b]*/ +{ + if (!SSL_CTX_set1_sigalgs_list(self->ctx, sigalgslist)) { + _setSSLError(get_state_ctx(self), "unrecognized signature algorithm", 0, __FILE__, __LINE__); + return NULL; + } + Py_RETURN_NONE; +} static int do_protocol_selection(int alpn, unsigned char **out, unsigned char *outlen, @@ -4037,7 +4490,8 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) _PySSLPasswordInfo *pw_info = (_PySSLPasswordInfo*) userdata; PyObject *fn_ret = NULL; - PySSL_END_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); + _PySSL_FIX_ERRNO; if (pw_info->error) { /* already failed previously. OpenSSL 3.0.0-alpha14 invokes the @@ -4067,13 +4521,13 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); memcpy(buf, pw_info->password, pw_info->size); return pw_info->size; error: Py_XDECREF(fn_ret); - PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); pw_info->error = 1; return -1; } @@ -4126,10 +4580,10 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, SSL_CTX_set_default_passwd_cb(self->ctx, _password_callback); SSL_CTX_set_default_passwd_cb_userdata(self->ctx, &pw_info); } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_use_certificate_chain_file(self->ctx, PyBytes_AS_STRING(certfile_bytes)); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); if (r != 1) { if (pw_info.error) { ERR_clear_error(); @@ -4144,11 +4598,11 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, } goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_use_PrivateKey_file(self->ctx, PyBytes_AS_STRING(keyfile ? keyfile_bytes : certfile_bytes), SSL_FILETYPE_PEM); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); Py_CLEAR(keyfile_bytes); Py_CLEAR(certfile_bytes); if (r != 1) { @@ -4165,9 +4619,9 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, } goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_check_private_key(self->ctx); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); if (r != 1) { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); goto error; @@ -4384,9 +4838,9 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, cafile_buf = PyBytes_AS_STRING(cafile_bytes); if (capath) capath_buf = PyBytes_AS_STRING(capath_bytes); - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) r = SSL_CTX_load_verify_locations(self->ctx, cafile_buf, capath_buf); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) if (r != 1) { if (errno != 0) { PyErr_SetFromErrno(PyExc_OSError); @@ -4427,7 +4881,7 @@ _ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) FILE *f; DH *dh; -#if defined(MS_WINDOWS) && defined(_DEBUG) +#if defined(MS_WINDOWS) && defined(Py_DEBUG) PyErr_SetString(PyExc_NotImplementedError, "load_dh_params: unavailable on Windows debug build"); return NULL; @@ -4438,10 +4892,11 @@ _ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) return NULL; errno = 0; - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS dh = PEM_read_DHparams(f, NULL, NULL, NULL); fclose(f); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (dh == NULL) { if (errno != 0) { PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, filepath); @@ -4593,6 +5048,7 @@ _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self) Py_BEGIN_ALLOW_THREADS rc = SSL_CTX_set_default_verify_paths(self->ctx); Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (!rc) { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); return NULL; @@ -4756,6 +5212,8 @@ _servername_callback(SSL *s, int *al, void *args) } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @critical_section @getter _ssl._SSLContext.sni_callback @@ -4770,7 +5228,7 @@ See RFC 6066 for details of the SNI extension. static PyObject * _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) -/*[clinic end generated code: output=961e6575cdfaf036 input=9b2473c5e984cfe6]*/ +/*[clinic end generated code: output=961e6575cdfaf036 input=3aee06696b0874d9]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -4780,6 +5238,8 @@ _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @critical_section @setter _ssl._SSLContext.sni_callback @@ -4787,7 +5247,7 @@ _ssl._SSLContext.sni_callback static int _ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value) -/*[clinic end generated code: output=b32736c6b891f61a input=c3c4ff33540b3c85]*/ +/*[clinic end generated code: output=b32736c6b891f61a input=332def1d8c81d549]*/ { if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { PyErr_SetString(PyExc_ValueError, @@ -5249,6 +5709,10 @@ static struct PyMethodDef context_methods[] = { _SSL__SSLCONTEXT__WRAP_SOCKET_METHODDEF _SSL__SSLCONTEXT__WRAP_BIO_METHODDEF _SSL__SSLCONTEXT_SET_CIPHERS_METHODDEF + _SSL__SSLCONTEXT_SET_CIPHERSUITES_METHODDEF + _SSL__SSLCONTEXT_SET_GROUPS_METHODDEF + _SSL__SSLCONTEXT_SET_CLIENT_SIGALGS_METHODDEF + _SSL__SSLCONTEXT_SET_SERVER_SIGALGS_METHODDEF _SSL__SSLCONTEXT__SET_ALPN_PROTOCOLS_METHODDEF _SSL__SSLCONTEXT_LOAD_CERT_CHAIN_METHODDEF _SSL__SSLCONTEXT_LOAD_DH_PARAMS_METHODDEF @@ -5259,6 +5723,7 @@ static struct PyMethodDef context_methods[] = { _SSL__SSLCONTEXT_CERT_STORE_STATS_METHODDEF _SSL__SSLCONTEXT_GET_CA_CERTS_METHODDEF _SSL__SSLCONTEXT_GET_CIPHERS_METHODDEF + _SSL__SSLCONTEXT_GET_GROUPS_METHODDEF _SSL__SSLCONTEXT_SET_PSK_CLIENT_CALLBACK_METHODDEF _SSL__SSLCONTEXT_SET_PSK_SERVER_CALLBACK_METHODDEF {NULL, NULL} /* sentinel */ @@ -5324,19 +5789,11 @@ _ssl_MemoryBIO_impl(PyTypeObject *type) return (PyObject *) self; } -static int -memory_bio_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static void memory_bio_dealloc(PyObject *op) { PySSLMemoryBIO *self = PySSLMemoryBIO_CAST(op); PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); (void)BIO_free(self->bio); tp->tp_free(self); Py_DECREF(tp); @@ -5393,30 +5850,29 @@ _ssl_MemoryBIO_read_impl(PySSLMemoryBIO *self, int len) /*[clinic end generated code: output=a657aa1e79cd01b3 input=21046f2d7dac3a90]*/ { int avail, nbytes; - PyObject *result; avail = (int)Py_MIN(BIO_ctrl_pending(self->bio), INT_MAX); if ((len < 0) || (len > avail)) len = avail; - result = PyBytes_FromStringAndSize(NULL, len); - if ((result == NULL) || (len == 0)) - return result; + if (len == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } - nbytes = BIO_read(self->bio, PyBytes_AS_STRING(result), len); + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { + return NULL; + } + + nbytes = BIO_read(self->bio, PyBytesWriter_GetData(writer), len); if (nbytes < 0) { _sslmodulestate *state = get_state_mbio(self); - Py_DECREF(result); + PyBytesWriter_Discard(writer); _setSSLError(state, NULL, 0, __FILE__, __LINE__); return NULL; } - /* There should never be any short reads but check anyway. */ - if (nbytes < len) { - _PyBytes_Resize(&result, nbytes); - } - - return result; + return PyBytesWriter_FinishWithSize(writer, nbytes); } /*[clinic input] @@ -5501,15 +5957,13 @@ static PyType_Slot PySSLMemoryBIO_slots[] = { {Py_tp_getset, memory_bio_getsetlist}, {Py_tp_new, _ssl_MemoryBIO}, {Py_tp_dealloc, memory_bio_dealloc}, - {Py_tp_traverse, memory_bio_traverse}, {0, 0}, }; static PyType_Spec PySSLMemoryBIO_spec = { .name = "_ssl.MemoryBIO", .basicsize = sizeof(PySSLMemoryBIO), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | - Py_TPFLAGS_HAVE_GC), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), .slots = PySSLMemoryBIO_slots, }; @@ -5812,6 +6266,7 @@ _ssl_RAND_bytes_impl(PyObject *module, int n) /*[clinic input] +@permit_long_summary @critical_section _ssl.RAND_status @@ -5823,12 +6278,13 @@ using the ssl() function. static PyObject * _ssl_RAND_status_impl(PyObject *module) -/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=636fb5659ea2e727]*/ +/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=aba24a3f3af3b184]*/ { return PyBool_FromLong(RAND_status()); } /*[clinic input] +@permit_long_summary @critical_section _ssl.get_default_verify_paths @@ -5839,7 +6295,7 @@ The values are 'cert_file_env', 'cert_file', 'cert_dir_env', 'cert_dir'. static PyObject * _ssl_get_default_verify_paths_impl(PyObject *module) -/*[clinic end generated code: output=e5b62a466271928b input=c6ae00bc04eb2b6e]*/ +/*[clinic end generated code: output=e5b62a466271928b input=255507e1be890095]*/ { PyObject *ofile_env = NULL; PyObject *ofile = NULL; @@ -5871,6 +6327,39 @@ _ssl_get_default_verify_paths_impl(PyObject *module) return NULL; } +/*[clinic input] +_ssl.get_sigalgs +[clinic start generated code]*/ + +static PyObject * +_ssl_get_sigalgs_impl(PyObject *module) +/*[clinic end generated code: output=ab0791b63856854b input=d96dd6cefec3f86b]*/ +{ +#if OPENSSL_VERSION_NUMBER >= 0x30400000L + const char *sigalgs; + PyObject *sigalgs_str, *sigalgs_list; + + if ((sigalgs = SSL_get1_builtin_sigalgs(NULL)) == NULL) { + PyErr_NoMemory(); + return NULL; + } + + if ((sigalgs_str = PyUnicode_DecodeFSDefault(sigalgs)) == NULL) { + OPENSSL_free((void *)sigalgs); + return NULL; + } + + OPENSSL_free((void *)sigalgs); + sigalgs_list = PyUnicode_Split(sigalgs_str, _Py_LATIN1_CHR(':'), -1); + Py_DECREF(sigalgs_str); + return sigalgs_list; +#else + PyErr_SetString(PyExc_NotImplementedError, + "Getting signature algorithms requires OpenSSL 3.4 or later."); + return NULL; +#endif +} + static PyObject* asn1obj2py(_sslmodulestate *state, ASN1_OBJECT *obj) { @@ -6274,6 +6763,7 @@ static PyMethodDef PySSL_methods[] = { _SSL_RAND_BYTES_METHODDEF _SSL_RAND_STATUS_METHODDEF _SSL_GET_DEFAULT_VERIFY_PATHS_METHODDEF + _SSL_GET_SIGALGS_METHODDEF _SSL_ENUM_CERTIFICATES_METHODDEF _SSL_ENUM_CRLS_METHODDEF _SSL_TXT2OBJ_METHODDEF @@ -6626,6 +7116,12 @@ sslmodule_init_constants(PyObject *m) addbool(m, "HAS_PSK", 1); #endif +#ifdef OPENSSL_NO_EXTERNAL_PSK_TLS13 + addbool(m, "HAS_PSK_TLS13", 0); +#else + addbool(m, "HAS_PSK_TLS13", 1); +#endif + #ifdef SSL_VERIFY_POST_HANDSHAKE addbool(m, "HAS_PHA", 1); #else diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 7c0b4876f43..aee446d0ccb 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -140,13 +140,14 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) * critical debug helper. */ - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS PyThread_acquire_lock(lock, 1); res = BIO_printf(ssl_obj->ctx->keylog_bio, "%s\n", line); e = errno; (void)BIO_flush(ssl_obj->ctx->keylog_bio); PyThread_release_lock(lock); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (res == -1) { errno = e; @@ -175,7 +176,7 @@ _PySSLContext_set_keylog_filename(PyObject *op, PyObject *arg, PySSLContext *self = PySSLContext_CAST(op); FILE *fp; -#if defined(MS_WINDOWS) && defined(_DEBUG) +#if defined(MS_WINDOWS) && defined(Py_DEBUG) PyErr_SetString(PyExc_NotImplementedError, "set_keylog_filename: unavailable on Windows debug build"); return -1; @@ -187,9 +188,10 @@ _PySSLContext_set_keylog_filename(PyObject *op, PyObject *arg, if (self->keylog_bio != NULL) { BIO *bio = self->keylog_bio; self->keylog_bio = NULL; - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS BIO_free_all(bio); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; } if (arg == Py_None) { @@ -211,13 +213,13 @@ _PySSLContext_set_keylog_filename(PyObject *op, PyObject *arg, self->keylog_filename = Py_NewRef(arg); /* Write a header for seekable, empty files (this excludes pipes). */ - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) if (BIO_tell(self->keylog_bio) == 0) { BIO_puts(self->keylog_bio, "# TLS secrets log file, generated by OpenSSL / Python\n"); (void)BIO_flush(self->keylog_bio); } - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) SSL_CTX_set_keylog_callback(self->ctx, _PySSL_keylog_callback); return 0; } diff --git a/Modules/_ssl_data_34.h b/Modules/_ssl_data_35.h similarity index 98% rename from Modules/_ssl_data_34.h rename to Modules/_ssl_data_35.h index 99718c5e605..e4919b550e3 100644 --- a/Modules/_ssl_data_34.h +++ b/Modules/_ssl_data_35.h @@ -1,6 +1,6 @@ /* File generated by Tools/ssl/make_ssl_data.py */ -/* Generated on 2025-03-26T13:47:34.223146+00:00 */ -/* Generated from Git commit openssl-3.4.1-0-ga26d85337d */ +/* Generated on 2025-10-04T17:49:19.148321+00:00 */ +/* Generated from Git commit openssl-3.5.4-0-gc1eeb9406 */ /* generated from args.lib2errnum */ static struct py_ssl_library_code library_codes[] = { @@ -1283,6 +1283,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FAILED_BUILDING_OWN_CHAIN", 58, 164}, #endif + #ifdef CMP_R_FAILED_EXTRACTING_CENTRAL_GEN_KEY + {"FAILED_EXTRACTING_CENTRAL_GEN_KEY", ERR_LIB_CMP, CMP_R_FAILED_EXTRACTING_CENTRAL_GEN_KEY}, + #else + {"FAILED_EXTRACTING_CENTRAL_GEN_KEY", 58, 203}, + #endif #ifdef CMP_R_FAILED_EXTRACTING_PUBKEY {"FAILED_EXTRACTING_PUBKEY", ERR_LIB_CMP, CMP_R_FAILED_EXTRACTING_PUBKEY}, #else @@ -1343,6 +1348,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_ROOTCAKEYUPDATE", 58, 195}, #endif + #ifdef CMP_R_MISSING_CENTRAL_GEN_KEY + {"MISSING_CENTRAL_GEN_KEY", ERR_LIB_CMP, CMP_R_MISSING_CENTRAL_GEN_KEY}, + #else + {"MISSING_CENTRAL_GEN_KEY", 58, 204}, + #endif #ifdef CMP_R_MISSING_CERTID {"MISSING_CERTID", ERR_LIB_CMP, CMP_R_MISSING_CERTID}, #else @@ -1513,6 +1523,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNCLEAN_CTX", 58, 191}, #endif + #ifdef CMP_R_UNEXPECTED_CENTRAL_GEN_KEY + {"UNEXPECTED_CENTRAL_GEN_KEY", ERR_LIB_CMP, CMP_R_UNEXPECTED_CENTRAL_GEN_KEY}, + #else + {"UNEXPECTED_CENTRAL_GEN_KEY", 58, 205}, + #endif #ifdef CMP_R_UNEXPECTED_CERTPROFILE {"UNEXPECTED_CERTPROFILE", ERR_LIB_CMP, CMP_R_UNEXPECTED_CERTPROFILE}, #else @@ -2308,6 +2323,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"BAD_PBM_ITERATIONCOUNT", 56, 100}, #endif + #ifdef CRMF_R_CMS_NOT_SUPPORTED + {"CMS_NOT_SUPPORTED", ERR_LIB_CRMF, CRMF_R_CMS_NOT_SUPPORTED}, + #else + {"CMS_NOT_SUPPORTED", 56, 122}, + #endif #ifdef CRMF_R_CRMFERROR {"CRMFERROR", ERR_LIB_CRMF, CRMF_R_CRMFERROR}, #else @@ -2323,16 +2343,41 @@ static struct py_ssl_error_code error_codes[] = { #else {"ERROR_DECODING_CERTIFICATE", 56, 104}, #endif + #ifdef CRMF_R_ERROR_DECODING_ENCRYPTEDKEY + {"ERROR_DECODING_ENCRYPTEDKEY", ERR_LIB_CRMF, CRMF_R_ERROR_DECODING_ENCRYPTEDKEY}, + #else + {"ERROR_DECODING_ENCRYPTEDKEY", 56, 123}, + #endif #ifdef CRMF_R_ERROR_DECRYPTING_CERTIFICATE {"ERROR_DECRYPTING_CERTIFICATE", ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_CERTIFICATE}, #else {"ERROR_DECRYPTING_CERTIFICATE", 56, 105}, #endif + #ifdef CRMF_R_ERROR_DECRYPTING_ENCRYPTEDKEY + {"ERROR_DECRYPTING_ENCRYPTEDKEY", ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_ENCRYPTEDKEY}, + #else + {"ERROR_DECRYPTING_ENCRYPTEDKEY", 56, 124}, + #endif + #ifdef CRMF_R_ERROR_DECRYPTING_ENCRYPTEDVALUE + {"ERROR_DECRYPTING_ENCRYPTEDVALUE", ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_ENCRYPTEDVALUE}, + #else + {"ERROR_DECRYPTING_ENCRYPTEDVALUE", 56, 125}, + #endif #ifdef CRMF_R_ERROR_DECRYPTING_SYMMETRIC_KEY {"ERROR_DECRYPTING_SYMMETRIC_KEY", ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_SYMMETRIC_KEY}, #else {"ERROR_DECRYPTING_SYMMETRIC_KEY", 56, 106}, #endif + #ifdef CRMF_R_ERROR_SETTING_PURPOSE + {"ERROR_SETTING_PURPOSE", ERR_LIB_CRMF, CRMF_R_ERROR_SETTING_PURPOSE}, + #else + {"ERROR_SETTING_PURPOSE", 56, 126}, + #endif + #ifdef CRMF_R_ERROR_VERIFYING_ENCRYPTEDKEY + {"ERROR_VERIFYING_ENCRYPTEDKEY", ERR_LIB_CRMF, CRMF_R_ERROR_VERIFYING_ENCRYPTEDKEY}, + #else + {"ERROR_VERIFYING_ENCRYPTEDKEY", 56, 127}, + #endif #ifdef CRMF_R_FAILURE_OBTAINING_RANDOM {"FAILURE_OBTAINING_RANDOM", ERR_LIB_CRMF, CRMF_R_FAILURE_OBTAINING_RANDOM}, #else @@ -2358,6 +2403,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"POPOSKINPUT_NOT_SUPPORTED", 56, 113}, #endif + #ifdef CRMF_R_POPO_INCONSISTENT_CENTRAL_KEYGEN + {"POPO_INCONSISTENT_CENTRAL_KEYGEN", ERR_LIB_CRMF, CRMF_R_POPO_INCONSISTENT_CENTRAL_KEYGEN}, + #else + {"POPO_INCONSISTENT_CENTRAL_KEYGEN", 56, 128}, + #endif #ifdef CRMF_R_POPO_INCONSISTENT_PUBLIC_KEY {"POPO_INCONSISTENT_PUBLIC_KEY", ERR_LIB_CRMF, CRMF_R_POPO_INCONSISTENT_PUBLIC_KEY}, #else @@ -3963,6 +4013,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"PBKDF2_ERROR", 6, 181}, #endif + #ifdef EVP_R_PIPELINE_NOT_SUPPORTED + {"PIPELINE_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_PIPELINE_NOT_SUPPORTED}, + #else + {"PIPELINE_NOT_SUPPORTED", 6, 230}, + #endif #ifdef EVP_R_PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED {"PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", ERR_LIB_EVP, EVP_R_PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED}, #else @@ -3978,6 +4033,36 @@ static struct py_ssl_error_code error_codes[] = { #else {"PRIVATE_KEY_ENCODE_ERROR", 6, 146}, #endif + #ifdef EVP_R_PROVIDER_ASYM_CIPHER_FAILURE + {"PROVIDER_ASYM_CIPHER_FAILURE", ERR_LIB_EVP, EVP_R_PROVIDER_ASYM_CIPHER_FAILURE}, + #else + {"PROVIDER_ASYM_CIPHER_FAILURE", 6, 232}, + #endif + #ifdef EVP_R_PROVIDER_ASYM_CIPHER_NOT_SUPPORTED + {"PROVIDER_ASYM_CIPHER_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_PROVIDER_ASYM_CIPHER_NOT_SUPPORTED}, + #else + {"PROVIDER_ASYM_CIPHER_NOT_SUPPORTED", 6, 235}, + #endif + #ifdef EVP_R_PROVIDER_KEYMGMT_FAILURE + {"PROVIDER_KEYMGMT_FAILURE", ERR_LIB_EVP, EVP_R_PROVIDER_KEYMGMT_FAILURE}, + #else + {"PROVIDER_KEYMGMT_FAILURE", 6, 233}, + #endif + #ifdef EVP_R_PROVIDER_KEYMGMT_NOT_SUPPORTED + {"PROVIDER_KEYMGMT_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_PROVIDER_KEYMGMT_NOT_SUPPORTED}, + #else + {"PROVIDER_KEYMGMT_NOT_SUPPORTED", 6, 236}, + #endif + #ifdef EVP_R_PROVIDER_SIGNATURE_FAILURE + {"PROVIDER_SIGNATURE_FAILURE", ERR_LIB_EVP, EVP_R_PROVIDER_SIGNATURE_FAILURE}, + #else + {"PROVIDER_SIGNATURE_FAILURE", 6, 234}, + #endif + #ifdef EVP_R_PROVIDER_SIGNATURE_NOT_SUPPORTED + {"PROVIDER_SIGNATURE_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_PROVIDER_SIGNATURE_NOT_SUPPORTED}, + #else + {"PROVIDER_SIGNATURE_NOT_SUPPORTED", 6, 237}, + #endif #ifdef EVP_R_PUBLIC_KEY_NOT_RSA {"PUBLIC_KEY_NOT_RSA", ERR_LIB_EVP, EVP_R_PUBLIC_KEY_NOT_RSA}, #else @@ -3998,6 +4083,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE", 6, 228}, #endif + #ifdef EVP_R_TOO_MANY_PIPES + {"TOO_MANY_PIPES", ERR_LIB_EVP, EVP_R_TOO_MANY_PIPES}, + #else + {"TOO_MANY_PIPES", 6, 231}, + #endif #ifdef EVP_R_TOO_MANY_RECORDS {"TOO_MANY_RECORDS", ERR_LIB_EVP, EVP_R_TOO_MANY_RECORDS}, #else @@ -4753,6 +4843,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_PUBLIC_KEY_TYPE", 9, 110}, #endif + #ifdef PEM_R_UNSUPPORTED_PVK_KEY_TYPE + {"UNSUPPORTED_PVK_KEY_TYPE", ERR_LIB_PEM, PEM_R_UNSUPPORTED_PVK_KEY_TYPE}, + #else + {"UNSUPPORTED_PVK_KEY_TYPE", 9, 133}, + #endif #ifdef PKCS12_R_CALLBACK_FAILED {"CALLBACK_FAILED", ERR_LIB_PKCS12, PKCS12_R_CALLBACK_FAILED}, #else @@ -5243,6 +5338,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FIPS_MODULE_ENTERING_ERROR_STATE", 57, 224}, #endif + #ifdef PROV_R_FIPS_MODULE_IMPORT_PCT_ERROR + {"FIPS_MODULE_IMPORT_PCT_ERROR", ERR_LIB_PROV, PROV_R_FIPS_MODULE_IMPORT_PCT_ERROR}, + #else + {"FIPS_MODULE_IMPORT_PCT_ERROR", 57, 253}, + #endif #ifdef PROV_R_FIPS_MODULE_IN_ERROR_STATE {"FIPS_MODULE_IN_ERROR_STATE", ERR_LIB_PROV, PROV_R_FIPS_MODULE_IN_ERROR_STATE}, #else @@ -5543,6 +5643,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"MISSING_XCGHASH", 57, 135}, #endif + #ifdef PROV_R_ML_DSA_NO_FORMAT + {"ML_DSA_NO_FORMAT", ERR_LIB_PROV, PROV_R_ML_DSA_NO_FORMAT}, + #else + {"ML_DSA_NO_FORMAT", 57, 245}, + #endif + #ifdef PROV_R_ML_KEM_NO_FORMAT + {"ML_KEM_NO_FORMAT", ERR_LIB_PROV, PROV_R_ML_KEM_NO_FORMAT}, + #else + {"ML_KEM_NO_FORMAT", 57, 246}, + #endif #ifdef PROV_R_MODULE_INTEGRITY_FAILURE {"MODULE_INTEGRITY_FAILURE", ERR_LIB_PROV, PROV_R_MODULE_INTEGRITY_FAILURE}, #else @@ -5593,6 +5703,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_PARAMETERS_SET", 57, 177}, #endif + #ifdef PROV_R_NULL_LENGTH_POINTER + {"NULL_LENGTH_POINTER", ERR_LIB_PROV, PROV_R_NULL_LENGTH_POINTER}, + #else + {"NULL_LENGTH_POINTER", 57, 247}, + #endif + #ifdef PROV_R_NULL_OUTPUT_BUFFER + {"NULL_OUTPUT_BUFFER", ERR_LIB_PROV, PROV_R_NULL_OUTPUT_BUFFER}, + #else + {"NULL_OUTPUT_BUFFER", 57, 248}, + #endif #ifdef PROV_R_ONESHOT_CALL_OUT_OF_ORDER {"ONESHOT_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_ONESHOT_CALL_OUT_OF_ORDER}, #else @@ -5728,6 +5848,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNABLE_TO_RESEED", 57, 204}, #endif + #ifdef PROV_R_UNEXPECTED_KEY_PARAMETERS + {"UNEXPECTED_KEY_PARAMETERS", ERR_LIB_PROV, PROV_R_UNEXPECTED_KEY_PARAMETERS}, + #else + {"UNEXPECTED_KEY_PARAMETERS", 57, 249}, + #endif #ifdef PROV_R_UNSUPPORTED_CEK_ALG {"UNSUPPORTED_CEK_ALG", ERR_LIB_PROV, PROV_R_UNSUPPORTED_CEK_ALG}, #else @@ -5748,6 +5873,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_NUMBER_OF_ROUNDS", 57, 152}, #endif + #ifdef PROV_R_UNSUPPORTED_SELECTION + {"UNSUPPORTED_SELECTION", ERR_LIB_PROV, PROV_R_UNSUPPORTED_SELECTION}, + #else + {"UNSUPPORTED_SELECTION", 57, 250}, + #endif #ifdef PROV_R_UPDATE_CALL_OUT_OF_ORDER {"UPDATE_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_UPDATE_CALL_OUT_OF_ORDER}, #else @@ -5763,6 +5893,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"VALUE_ERROR", 57, 138}, #endif + #ifdef PROV_R_WRONG_CIPHERTEXT_SIZE + {"WRONG_CIPHERTEXT_SIZE", ERR_LIB_PROV, PROV_R_WRONG_CIPHERTEXT_SIZE}, + #else + {"WRONG_CIPHERTEXT_SIZE", 57, 251}, + #endif #ifdef PROV_R_WRONG_FINAL_BLOCK_LENGTH {"WRONG_FINAL_BLOCK_LENGTH", ERR_LIB_PROV, PROV_R_WRONG_FINAL_BLOCK_LENGTH}, #else @@ -5938,6 +6073,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"PRNG_NOT_SEEDED", 36, 100}, #endif + #ifdef RAND_R_RANDOM_POOL_IS_EMPTY + {"RANDOM_POOL_IS_EMPTY", ERR_LIB_RAND, RAND_R_RANDOM_POOL_IS_EMPTY}, + #else + {"RANDOM_POOL_IS_EMPTY", 36, 142}, + #endif #ifdef RAND_R_RANDOM_POOL_OVERFLOW {"RANDOM_POOL_OVERFLOW", ERR_LIB_RAND, RAND_R_RANDOM_POOL_OVERFLOW}, #else @@ -6923,6 +7063,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"DIGEST_CHECK_FAILED", 20, 149}, #endif + #ifdef SSL_R_DOMAIN_USE_ONLY + {"DOMAIN_USE_ONLY", ERR_LIB_SSL, SSL_R_DOMAIN_USE_ONLY}, + #else + {"DOMAIN_USE_ONLY", 20, 422}, + #endif #ifdef SSL_R_DTLS_MESSAGE_TOO_BIG {"DTLS_MESSAGE_TOO_BIG", ERR_LIB_SSL, SSL_R_DTLS_MESSAGE_TOO_BIG}, #else @@ -7213,6 +7358,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"LIBRARY_HAS_NO_CIPHERS", 20, 161}, #endif + #ifdef SSL_R_LISTENER_USE_ONLY + {"LISTENER_USE_ONLY", ERR_LIB_SSL, SSL_R_LISTENER_USE_ONLY}, + #else + {"LISTENER_USE_ONLY", 20, 421}, + #endif #ifdef SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED {"MAXIMUM_ENCRYPTED_PKTS_REACHED", ERR_LIB_SSL, SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED}, #else @@ -7243,6 +7393,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"MISSING_PSK_KEX_MODES_EXTENSION", 20, 310}, #endif + #ifdef SSL_R_MISSING_QUIC_TLS_FUNCTIONS + {"MISSING_QUIC_TLS_FUNCTIONS", ERR_LIB_SSL, SSL_R_MISSING_QUIC_TLS_FUNCTIONS}, + #else + {"MISSING_QUIC_TLS_FUNCTIONS", 20, 423}, + #endif #ifdef SSL_R_MISSING_RSA_CERTIFICATE {"MISSING_RSA_CERTIFICATE", ERR_LIB_SSL, SSL_R_MISSING_RSA_CERTIFICATE}, #else @@ -8983,6 +9138,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", 34, 159}, #endif + #ifdef X509V3_R_PURPOSE_NOT_UNIQUE + {"PURPOSE_NOT_UNIQUE", ERR_LIB_X509V3, X509V3_R_PURPOSE_NOT_UNIQUE}, + #else + {"PURPOSE_NOT_UNIQUE", 34, 173}, + #endif #ifdef X509V3_R_SECTION_NOT_FOUND {"SECTION_NOT_FOUND", ERR_LIB_X509V3, X509V3_R_SECTION_NOT_FOUND}, #else diff --git a/Modules/_stat.c b/Modules/_stat.c index f11ca7d23b4..1dabf2f6d5b 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -57,7 +57,7 @@ typedef unsigned short mode_t; * Only the names are defined by POSIX but not their value. All common file * types seems to have the same numeric value on all platforms, though. * - * pyport.h guarantees S_IFMT, S_IFDIR, S_IFCHR, S_IFREG and S_IFLNK + * fileutils.h guarantees S_IFMT, S_IFDIR, S_IFCHR, S_IFREG and S_IFLNK */ #ifndef S_IFBLK @@ -86,7 +86,7 @@ typedef unsigned short mode_t; /* S_ISXXX() - * pyport.h defines S_ISDIR(), S_ISREG() and S_ISCHR() + * fileutils.h defines S_ISDIR(), S_ISREG() and S_ISCHR() */ #ifndef S_ISBLK diff --git a/Modules/_struct.c b/Modules/_struct.c index c36079f1eb8..2acb3df3a30 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -9,8 +9,10 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_lock.h" // _PyOnceFlag_CallOnce() #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() @@ -1504,6 +1506,53 @@ static formatdef lilendian_table[] = { {0} }; +/* Ensure endian table optimization happens exactly once across all interpreters */ +static _PyOnceFlag endian_tables_init_once = {0}; + +static int +init_endian_tables(void *Py_UNUSED(arg)) +{ + const formatdef *native = native_table; + formatdef *other, *ptr; +#if PY_LITTLE_ENDIAN + other = lilendian_table; +#else + other = bigendian_table; +#endif + /* Scan through the native table, find a matching + entry in the endian table and swap in the + native implementations whenever possible + (64-bit platforms may not have "standard" sizes) */ + while (native->format != '\0' && other->format != '\0') { + ptr = other; + while (ptr->format != '\0') { + if (ptr->format == native->format) { + /* Match faster when formats are + listed in the same order */ + if (ptr == other) + other++; + /* Only use the trick if the + size matches */ + if (ptr->size != native->size) + break; + /* Skip float and double, could be + "unknown" float format */ + if (ptr->format == 'd' || ptr->format == 'f') + break; + /* Skip _Bool, semantics are different for standard size */ + if (ptr->format == '?') + break; + ptr->pack = native->pack; + ptr->unpack = native->unpack; + break; + } + ptr++; + } + native++; + } + return 0; +} + static const formatdef * whichtable(const char **pfmt) @@ -1794,9 +1843,7 @@ s_dealloc(PyObject *op) PyStructObject *s = PyStructObject_CAST(op); PyTypeObject *tp = Py_TYPE(s); PyObject_GC_UnTrack(s); - if (s->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, s->weakreflist); if (s->s_codes != NULL) { PyMem_Free(s->s_codes); } @@ -2190,7 +2237,6 @@ strings."); static PyObject * s_pack(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - char *buf; PyStructObject *soself; _structmodulestate *state = get_struct_state_structinst(self); @@ -2206,21 +2252,19 @@ s_pack(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } /* Allocate a new string */ - _PyBytesWriter writer; - _PyBytesWriter_Init(&writer); - buf = _PyBytesWriter_Alloc(&writer, soself->s_size); - if (buf == NULL) { - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter *writer = PyBytesWriter_Create(soself->s_size); + if (writer == NULL) { return NULL; } + char *buf = PyBytesWriter_GetData(writer); /* Call the guts */ if ( s_pack_internal(soself, args, 0, buf, state) != 0 ) { - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return NULL; } - return _PyBytesWriter_Finish(&writer, buf + soself->s_size); + return PyBytesWriter_FinishWithSize(writer, soself->s_size); } PyDoc_STRVAR(s_pack_into__doc__, @@ -2714,47 +2758,8 @@ _structmodule_exec(PyObject *m) return -1; } - /* Check endian and swap in faster functions */ - { - const formatdef *native = native_table; - formatdef *other, *ptr; -#if PY_LITTLE_ENDIAN - other = lilendian_table; -#else - other = bigendian_table; -#endif - /* Scan through the native table, find a matching - entry in the endian table and swap in the - native implementations whenever possible - (64-bit platforms may not have "standard" sizes) */ - while (native->format != '\0' && other->format != '\0') { - ptr = other; - while (ptr->format != '\0') { - if (ptr->format == native->format) { - /* Match faster when formats are - listed in the same order */ - if (ptr == other) - other++; - /* Only use the trick if the - size matches */ - if (ptr->size != native->size) - break; - /* Skip float and double, could be - "unknown" float format */ - if (ptr->format == 'd' || ptr->format == 'f') - break; - /* Skip _Bool, semantics are different for standard size */ - if (ptr->format == '?') - break; - ptr->pack = native->pack; - ptr->unpack = native->unpack; - break; - } - ptr++; - } - native++; - } - } + /* init cannot fail */ + (void)_PyOnceFlag_CallOnce(&endian_tables_init_once, init_endian_tables, NULL); /* Add some symbolic constants to the module */ state->StructError = PyErr_NewException("struct.error", NULL, NULL); diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c index b8bc6db2477..fb588de7808 100644 --- a/Modules/_suggestions.c +++ b/Modules/_suggestions.c @@ -8,6 +8,7 @@ module _suggestions /*[clinic end generated code: output=da39a3ee5e6b4b0d input=e58d81fafad5637b]*/ /*[clinic input] +@critical_section candidates _suggestions._generate_suggestions candidates: object item: unicode @@ -18,7 +19,7 @@ Returns the candidate in candidates that's closest to item static PyObject * _suggestions__generate_suggestions_impl(PyObject *module, PyObject *candidates, PyObject *item) -/*[clinic end generated code: output=79be7b653ae5e7ca input=ba2a8dddc654e33a]*/ +/*[clinic end generated code: output=79be7b653ae5e7ca input=92861a6c9bd8f667]*/ { // Check if dir is a list if (!PyList_CheckExact(candidates)) { @@ -29,7 +30,7 @@ _suggestions__generate_suggestions_impl(PyObject *module, // Check if all elements in the list are Unicode Py_ssize_t size = PyList_Size(candidates); for (Py_ssize_t i = 0; i < size; ++i) { - PyObject *elem = PyList_GetItem(candidates, i); + PyObject *elem = PyList_GET_ITEM(candidates, i); if (!PyUnicode_Check(elem)) { PyErr_SetString(PyExc_TypeError, "all elements in 'candidates' must be strings"); return NULL; diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index e2d141b5910..345498e670d 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -32,6 +32,7 @@ add_string_value(PyObject *dict, const char *key, const char *str_value) #endif /*[clinic input] +@permit_long_summary _sysconfig.config_vars Returns a dictionary containing build variables intended to be exposed by sysconfig. @@ -39,7 +40,7 @@ Returns a dictionary containing build variables intended to be exposed by syscon static PyObject * _sysconfig_config_vars_impl(PyObject *module) -/*[clinic end generated code: output=9c41cdee63ea9487 input=391ff42f3af57d01]*/ +/*[clinic end generated code: output=9c41cdee63ea9487 input=fdda9cab12ca19fe]*/ { PyObject *config = PyDict_New(); if (config == NULL) { diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 7fc4d61db29..d2e61e9d6ac 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1855,8 +1855,7 @@ ndarray_subscript(PyObject *op, PyObject *key) type_error: PyErr_Format(PyExc_TypeError, - "cannot index memory using \"%.200s\"", - Py_TYPE(key)->tp_name); + "cannot index memory using \"%T\"", key); err_occurred: Py_DECREF(nd); return NULL; diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index d4045afd515..c1f769456ac 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -178,6 +178,42 @@ sequence_fast_get_item(PyObject *self, PyObject *args) } +static PyObject * +object_setattr_null_exc(PyObject *self, PyObject *args) +{ + PyObject *obj, *name, *exc; + if (!PyArg_ParseTuple(args, "OOO", &obj, &name, &exc)) { + return NULL; + } + + PyErr_SetObject((PyObject*)Py_TYPE(exc), exc); + if (PyObject_SetAttr(obj, name, NULL) < 0) { + return NULL; + } + assert(PyErr_Occurred()); + return NULL; +} + + +static PyObject * +object_setattrstring_null_exc(PyObject *self, PyObject *args) +{ + PyObject *obj, *exc; + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &name, &size, &exc)) { + return NULL; + } + + PyErr_SetObject((PyObject*)Py_TYPE(exc), exc); + if (PyObject_SetAttrString(obj, name, NULL) < 0) { + return NULL; + } + assert(PyErr_Occurred()); + return NULL; +} + + static PyMethodDef test_methods[] = { {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, {"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, @@ -191,6 +227,8 @@ static PyMethodDef test_methods[] = { {"sequence_fast_get_size", sequence_fast_get_size, METH_O}, {"sequence_fast_get_item", sequence_fast_get_item, METH_VARARGS}, + {"object_setattr_null_exc", object_setattr_null_exc, METH_VARARGS}, + {"object_setattrstring_null_exc", object_setattrstring_null_exc, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/bytes.c b/Modules/_testcapi/bytes.c index 33903de14ba..f12fc7f5f3a 100644 --- a/Modules/_testcapi/bytes.c +++ b/Modules/_testcapi/bytes.c @@ -1,6 +1,11 @@ +// Use pycore_bytes.h +#define PYTESTCAPI_NEED_INTERNAL_API + #include "parts.h" #include "util.h" +#include "pycore_bytesobject.h" // _PyBytesWriter_CreateByteArray() + /* Test _PyBytes_Resize() */ static PyObject * @@ -51,9 +56,307 @@ bytes_join(PyObject *Py_UNUSED(module), PyObject *args) } +// --- PyBytesWriter type --------------------------------------------------- + +typedef struct { + PyObject_HEAD + PyBytesWriter *writer; +} WriterObject; + + +static PyObject * +writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + WriterObject *self = (WriterObject *)type->tp_alloc(type, 0); + if (!self) { + return NULL; + } + self->writer = NULL; + return (PyObject*)self; +} + + +static int +writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) +{ + if (kwargs && PyDict_GET_SIZE(kwargs)) { + PyErr_Format(PyExc_TypeError, + "PyBytesWriter() takes exactly no keyword arguments"); + return -1; + } + + Py_ssize_t alloc; + char *str; + Py_ssize_t str_size; + int use_bytearray; + if (!PyArg_ParseTuple(args, "ny#i", + &alloc, &str, &str_size, &use_bytearray)) { + return -1; + } + + WriterObject *self = (WriterObject *)self_raw; + if (self->writer) { + PyBytesWriter_Discard(self->writer); + } + if (use_bytearray) { + self->writer = _PyBytesWriter_CreateByteArray(alloc); + } + else { + self->writer = PyBytesWriter_Create(alloc); + } + if (self->writer == NULL) { + return -1; + } + + if (str_size) { + char *buf = PyBytesWriter_GetData(self->writer); + memcpy(buf, str, str_size); + } + + return 0; +} + + +static void +writer_dealloc(PyObject *self_raw) +{ + WriterObject *self = (WriterObject *)self_raw; + PyTypeObject *tp = Py_TYPE(self); + if (self->writer) { + PyBytesWriter_Discard(self->writer); + } + tp->tp_free(self); + Py_DECREF(tp); +} + + +static inline int +writer_check(WriterObject *self) +{ + if (self->writer == NULL) { + PyErr_SetString(PyExc_ValueError, "operation on finished writer"); + return -1; + } + return 0; +} + + +static PyObject* +writer_write_bytes(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + char *bytes; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "yn", &bytes, &size)) { + return NULL; + } + + if (PyBytesWriter_WriteBytes(self->writer, bytes, size) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject* +writer_format_i(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + char *format; + int value; + if (!PyArg_ParseTuple(args, "yi", &format, &value)) { + return NULL; + } + + if (PyBytesWriter_Format(self->writer, format, value) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject* +writer_resize(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + Py_ssize_t size; + char *str; + Py_ssize_t str_size; + if (!PyArg_ParseTuple(args, + "ny#", + &size, &str, &str_size)) { + return NULL; + } + assert(size >= str_size); + + Py_ssize_t pos = PyBytesWriter_GetSize(self->writer); + if (PyBytesWriter_Resize(self->writer, size) < 0) { + return NULL; + } + + char *buf = PyBytesWriter_GetData(self->writer); + memcpy(buf + pos, str, str_size); + + Py_RETURN_NONE; +} + + +static PyObject* +writer_get_size(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + Py_ssize_t alloc = PyBytesWriter_GetSize(self->writer); + return PyLong_FromSsize_t(alloc); +} + + +static PyObject* +writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject *str = PyBytesWriter_Finish(self->writer); + self->writer = NULL; + return str; +} + + +static PyObject* +writer_finish_with_size(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "n", &size)) { + return NULL; + } + + PyObject *str = PyBytesWriter_FinishWithSize(self->writer, size); + self->writer = NULL; + return str; +} + + +static PyMethodDef writer_methods[] = { + {"write_bytes", _PyCFunction_CAST(writer_write_bytes), METH_VARARGS}, + {"format_i", _PyCFunction_CAST(writer_format_i), METH_VARARGS}, + {"resize", _PyCFunction_CAST(writer_resize), METH_VARARGS}, + {"get_size", _PyCFunction_CAST(writer_get_size), METH_NOARGS}, + {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {"finish_with_size", _PyCFunction_CAST(writer_finish_with_size), METH_VARARGS}, + {NULL, NULL} /* sentinel */ +}; + +static PyType_Slot Writer_Type_slots[] = { + {Py_tp_new, writer_new}, + {Py_tp_init, writer_init}, + {Py_tp_dealloc, writer_dealloc}, + {Py_tp_methods, writer_methods}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Writer_spec = { + .name = "_testcapi.PyBytesWriter", + .basicsize = sizeof(WriterObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Writer_Type_slots, +}; + + +static PyObject * +byteswriter_abc(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + PyBytesWriter *writer = PyBytesWriter_Create(3); + if (writer == NULL) { + return NULL; + } + + char *str = PyBytesWriter_GetData(writer); + memcpy(str, "abc", 3); + + return PyBytesWriter_Finish(writer); +} + + +static PyObject * +byteswriter_resize(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // Allocate 10 bytes + PyBytesWriter *writer = PyBytesWriter_Create(10); + if (writer == NULL) { + return NULL; + } + char *buf = PyBytesWriter_GetData(writer); + + // Write some bytes + memcpy(buf, "Hello ", strlen("Hello ")); + buf += strlen("Hello "); + + // Allocate 10 more bytes + buf = PyBytesWriter_GrowAndUpdatePointer(writer, 10, buf); + if (buf == NULL) { + PyBytesWriter_Discard(writer); + return NULL; + } + + // Write more bytes + memcpy(buf, "World", strlen("World")); + buf += strlen("World"); + + // Truncate to the exact size and create a bytes object + return PyBytesWriter_FinishWithPointer(writer, buf); +} + + +static PyObject * +byteswriter_highlevel(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + PyBytesWriter *writer = PyBytesWriter_Create(0); + if (writer == NULL) { + goto error; + } + if (PyBytesWriter_WriteBytes(writer, "Hello", -1) < 0) { + goto error; + } + if (PyBytesWriter_Format(writer, " %s!", "World") < 0) { + goto error; + } + return PyBytesWriter_Finish(writer); + +error: + PyBytesWriter_Discard(writer); + return NULL; +} + + static PyMethodDef test_methods[] = { {"bytes_resize", bytes_resize, METH_VARARGS}, {"bytes_join", bytes_join, METH_VARARGS}, + {"byteswriter_abc", byteswriter_abc, METH_NOARGS}, + {"byteswriter_resize", byteswriter_resize, METH_NOARGS}, + {"byteswriter_highlevel", byteswriter_highlevel, METH_NOARGS}, {NULL}, }; @@ -64,5 +367,15 @@ _PyTestCapi_Init_Bytes(PyObject *m) return -1; } + PyTypeObject *writer_type = (PyTypeObject *)PyType_FromSpec(&Writer_spec); + if (writer_type == NULL) { + return -1; + } + if (PyModule_AddType(m, writer_type) < 0) { + Py_DECREF(writer_type); + return -1; + } + Py_DECREF(writer_type); + return 0; } diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 0604b413e33..c0254e044bc 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -82,6 +82,7 @@ _testcapi_exception_print_impl(PyObject *module, PyObject *exc, int legacy) } /*[clinic input] +@permit_long_summary _testcapi.make_exception_with_doc name: str doc: str = NULL @@ -95,7 +96,7 @@ static PyObject * _testcapi_make_exception_with_doc_impl(PyObject *module, const char *name, const char *doc, PyObject *base, PyObject *dict) -/*[clinic end generated code: output=439f0d963c1ce2c4 input=23a73013f8a8795a]*/ +/*[clinic end generated code: output=439f0d963c1ce2c4 input=508b420b7f9253ed]*/ { return PyErr_NewExceptionWithDoc(name, doc, base, dict); } diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 257e0256655..4fdcc850a33 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -528,6 +528,21 @@ pytype_getmodulebydef(PyObject *self, PyObject *type) return Py_XNewRef(mod); } +static PyObject * +pytype_getmodulebytoken(PyObject *self, PyObject *args) +{ + PyObject *type; + PyObject *py_token; + if (!PyArg_ParseTuple(args, "OO", &type, &py_token)) { + return NULL; + } + void *token = PyLong_AsVoidPtr(py_token); + if ((!token) && PyErr_Occurred()) { + return NULL; + } + return PyType_GetModuleByToken((PyTypeObject *)type, token); +} + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, @@ -546,6 +561,7 @@ static PyMethodDef TestMethods[] = { {"get_tp_token", get_tp_token, METH_O}, {"pytype_getbasebytoken", pytype_getbasebytoken, METH_VARARGS}, {"pytype_getmodulebydef", pytype_getmodulebydef, METH_O}, + {"pytype_getmodulebytoken", pytype_getmodulebytoken, METH_VARARGS}, {NULL}, }; @@ -782,10 +798,7 @@ heapctypesubclasswithfinalizer_finalize(PyObject *self) /* Save the current exception, if any. */ PyObject *exc = PyErr_GetRaisedException(); - if (_testcapimodule == NULL) { - goto cleanup_finalize; - } - PyObject *m = PyState_FindModule(_testcapimodule); + PyObject *m = PyType_GetModule(Py_TYPE(self)); if (m == NULL) { goto cleanup_finalize; } @@ -1402,8 +1415,8 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { if (subclass_with_finalizer_bases == NULL) { return -1; } - PyObject *HeapCTypeSubclassWithFinalizer = PyType_FromSpecWithBases( - &HeapCTypeSubclassWithFinalizer_spec, subclass_with_finalizer_bases); + PyObject *HeapCTypeSubclassWithFinalizer = PyType_FromModuleAndSpec( + m, &HeapCTypeSubclassWithFinalizer_spec, subclass_with_finalizer_bases); Py_DECREF(subclass_with_finalizer_bases); ADD("HeapCTypeSubclassWithFinalizer", HeapCTypeSubclassWithFinalizer); diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c index 0663c3781d4..af510cab655 100644 --- a/Modules/_testcapi/immortal.c +++ b/Modules/_testcapi/immortal.c @@ -28,13 +28,13 @@ test_immortal_builtins(PyObject *self, PyObject *Py_UNUSED(ignored)) static PyObject * test_immortal_small_ints(PyObject *self, PyObject *Py_UNUSED(ignored)) { - for (int i = -5; i <= 256; i++) { + for (int i = -5; i <= 1024; i++) { PyObject *obj = PyLong_FromLong(i); assert(verify_immortality(obj)); int has_int_immortal_bit = ((PyLongObject *)obj)->long_value.lv_tag & IMMORTALITY_BIT_MASK; assert(has_int_immortal_bit); } - for (int i = 257; i <= 260; i++) { + for (int i = 1025; i <= 1030; i++) { PyObject *obj = PyLong_FromLong(i); assert(obj); int has_int_immortal_bit = ((PyLongObject *)obj)->long_value.lv_tag & IMMORTALITY_BIT_MASK; diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 42243023a45..6313abf5485 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -228,7 +228,7 @@ pylongwriter_create(PyObject *module, PyObject *args) goto error; } - if (num < 0 || num >= PyLong_BASE) { + if (num < 0 || num >= (long)PyLong_BASE) { PyErr_SetString(PyExc_ValueError, "digit doesn't fit into digit"); goto error; } diff --git a/Modules/_testcapi/modsupport.c b/Modules/_testcapi/modsupport.c new file mode 100644 index 00000000000..6746eb9eb1e --- /dev/null +++ b/Modules/_testcapi/modsupport.c @@ -0,0 +1,55 @@ +#include "parts.h" + + + +static PyObject * +pyabiinfo_check(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *modname; + unsigned long maj, min, flags, buildver, abiver; + + if (!PyArg_ParseTuple(args, "zkkkkk", + &modname, &maj, &min, &flags, &buildver, &abiver)) + { + return NULL; + } + PyABIInfo info = { + .abiinfo_major_version = (uint8_t)maj, + .abiinfo_minor_version = (uint8_t)min, + .flags = (uint16_t)flags, + .build_version = (uint32_t)buildver, + .abi_version = (uint32_t)abiver}; + if (PyABIInfo_Check(&info, modname) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef TestMethods[] = { + {"pyabiinfo_check", pyabiinfo_check, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Modsupport(PyObject *m) +{ + if (PyModule_AddIntMacro(m, PyABIInfo_STABLE) < 0) { + return -1; + } + if (PyModule_AddIntMacro(m, PyABIInfo_INTERNAL) < 0) { + return -1; + } + if (PyModule_AddIntMacro(m, PyABIInfo_GIL) < 0) { + return -1; + } + if (PyModule_AddIntMacro(m, PyABIInfo_FREETHREADED) < 0) { + return -1; + } + if (PyModule_AddIntMacro(m, PyABIInfo_FREETHREADING_AGNOSTIC) < 0) { + return -1; + } + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/module.c b/Modules/_testcapi/module.c new file mode 100644 index 00000000000..7b5861bc08e --- /dev/null +++ b/Modules/_testcapi/module.c @@ -0,0 +1,401 @@ +#include "parts.h" +#include "util.h" + +// Test PyModule_* API + +/* unittest Cases that use these functions are in: + * Lib/test/test_capi/test_module.py + */ + +static PyObject * +module_from_slots_empty(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_null(PyObject *self, PyObject *spec) +{ + return PyModule_FromSlotsAndSpec(NULL, spec); +} + +static PyObject * +module_from_slots_name(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_name, "currently ignored..."}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_doc(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_doc, "the docstring"}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_size(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_state_size, (void*)123}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + return mod; +} + +static PyObject * +a_method(PyObject *self, PyObject *arg) +{ + return PyTuple_Pack(2, self, arg); +} + +static PyMethodDef a_methoddef_array[] = { + {"a_method", a_method, METH_O}, + {0}, +}; + +static PyObject * +module_from_slots_methods(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_methods, a_methoddef_array}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static int noop_traverse(PyObject *self, visitproc visit, void *arg) { + return 0; +} +static int noop_clear(PyObject *self) { return 0; } +static void noop_free(void *self) { } + +static PyObject * +module_from_slots_gc(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_state_traverse, noop_traverse}, + {Py_mod_state_clear, noop_clear}, + {Py_mod_state_free, noop_free}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + if (PyModule_Add(mod, "traverse", PyLong_FromVoidPtr(&noop_traverse)) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_Add(mod, "clear", PyLong_FromVoidPtr(&noop_clear)) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_Add(mod, "free", PyLong_FromVoidPtr(&noop_free)) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static const char test_token; + +static PyObject * +module_from_slots_token(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_token, (void*)&test_token}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + void *got_token; + if (PyModule_GetToken(mod, &got_token) < 0) { + Py_DECREF(mod); + return NULL; + } + assert(got_token == &test_token); + return mod; +} + +static int +simple_exec(PyObject *module) +{ + return PyModule_AddIntConstant(module, "a_number", 456); +} + +static PyObject * +module_from_slots_exec(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_exec, simple_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); + if (!mod) { + return NULL; + } + int res = PyObject_HasAttrStringWithError(mod, "a_number"); + if (res < 0) { + Py_DECREF(mod); + return NULL; + } + assert(res == 0); + if (PyModule_Exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static PyObject * +create_attr_from_spec(PyObject *spec, PyObject *def) +{ + assert(!def); + return PyObject_GetAttrString(spec, "_gimme_this"); +} + +static PyObject * +module_from_slots_create(PyObject *self, PyObject *spec) +{ + PyModuleDef_Slot slots[] = { + {Py_mod_create, create_attr_from_spec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + + +static int +slot_from_object(PyObject *obj) +{ + PyObject *slot_id_obj = PyObject_GetAttrString(obj, "_test_slot_id"); + if (slot_id_obj == NULL) { + return -1; + } + int slot_id = PyLong_AsInt(slot_id_obj); + if (PyErr_Occurred()) { + return -1; + } + return slot_id; +} + +static PyObject * +module_from_slots_repeat_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, "anything"}, + {slot_id, "anything else"}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_slots_null_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, NULL}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return PyModule_FromSlotsAndSpec(slots, spec); +} + +static PyObject * +module_from_def_slot(PyObject *self, PyObject *spec) +{ + int slot_id = slot_from_object(spec); + if (slot_id < 0) { + return NULL; + } + PyModuleDef_Slot slots[] = { + {slot_id, "anything"}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "currently ignored", + .m_slots = slots, + }; + // PyModuleDef is normally static; the real requirement is that it + // must outlive its module. + // Here, module creation fails, so it's fine on the stack. + PyObject *result = PyModule_FromDefAndSpec(&def, spec); + assert(result == NULL); + return result; +} + +static int +another_exec(PyObject *module) +{ + /* Make sure simple_exec was called */ + assert(PyObject_HasAttrString(module, "a_number")); + + /* Add or negate a global called 'another_number' */ + PyObject *another_number; + if (PyObject_GetOptionalAttrString(module, "another_number", + &another_number) < 0) { + return -1; + } + if (!another_number) { + return PyModule_AddIntConstant(module, "another_number", 789); + } + PyObject *neg_number = PyNumber_Negative(another_number); + Py_DECREF(another_number); + if (!neg_number) { + return -1; + } + int result = PyObject_SetAttrString(module, "another_number", + neg_number); + Py_DECREF(neg_number); + return result; +} + +static PyObject * +module_from_def_multiple_exec(PyObject *self, PyObject *spec) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_exec, simple_exec}, + {Py_mod_exec, another_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + static PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "currently ignored", + .m_slots = slots, + }; + return PyModule_FromDefAndSpec(&def, spec); +} + +static PyObject * +pymodule_exec(PyObject *self, PyObject *module) +{ + if (PyModule_Exec(module) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +pymodule_get_token(PyObject *self, PyObject *module) +{ + void *token; + if (PyModule_GetToken(module, &token) < 0) { + return NULL; + } + return PyLong_FromVoidPtr(token); +} + +static PyObject * +pymodule_get_def(PyObject *self, PyObject *module) +{ + PyModuleDef *def = PyModule_GetDef(module); + if (!def && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromVoidPtr(def); +} + +static PyObject * +pymodule_get_state_size(PyObject *self, PyObject *module) +{ + Py_ssize_t size; + if (PyModule_GetStateSize(module, &size) < 0) { + return NULL; + } + return PyLong_FromSsize_t(size); +} + +static PyMethodDef test_methods[] = { + {"module_from_slots_empty", module_from_slots_empty, METH_O}, + {"module_from_slots_null", module_from_slots_null, METH_O}, + {"module_from_slots_name", module_from_slots_name, METH_O}, + {"module_from_slots_doc", module_from_slots_doc, METH_O}, + {"module_from_slots_size", module_from_slots_size, METH_O}, + {"module_from_slots_methods", module_from_slots_methods, METH_O}, + {"module_from_slots_gc", module_from_slots_gc, METH_O}, + {"module_from_slots_token", module_from_slots_token, METH_O}, + {"module_from_slots_exec", module_from_slots_exec, METH_O}, + {"module_from_slots_create", module_from_slots_create, METH_O}, + {"module_from_slots_repeat_slot", module_from_slots_repeat_slot, METH_O}, + {"module_from_slots_null_slot", module_from_slots_null_slot, METH_O}, + {"module_from_def_multiple_exec", module_from_def_multiple_exec, METH_O}, + {"module_from_def_slot", module_from_def_slot, METH_O}, + {"pymodule_get_token", pymodule_get_token, METH_O}, + {"pymodule_get_def", pymodule_get_def, METH_O}, + {"pymodule_get_state_size", pymodule_get_state_size, METH_O}, + {"pymodule_exec", pymodule_exec, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Module(PyObject *m) +{ +#define ADD_INT_MACRO(C) if (PyModule_AddIntConstant(m, #C, C) < 0) return -1; + ADD_INT_MACRO(Py_mod_create); + ADD_INT_MACRO(Py_mod_exec); + ADD_INT_MACRO(Py_mod_multiple_interpreters); + ADD_INT_MACRO(Py_mod_gil); + ADD_INT_MACRO(Py_mod_name); + ADD_INT_MACRO(Py_mod_doc); + ADD_INT_MACRO(Py_mod_state_size); + ADD_INT_MACRO(Py_mod_methods); + ADD_INT_MACRO(Py_mod_state_traverse); + ADD_INT_MACRO(Py_mod_state_clear); + ADD_INT_MACRO(Py_mod_state_free); + ADD_INT_MACRO(Py_mod_token); +#undef ADD_INT_MACRO + if (PyModule_Add(m, "module_test_token", + PyLong_FromVoidPtr((void*)&test_token)) < 0) + { + return -1; + } + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c index 08a2055c51b..3f99836c1eb 100644 --- a/Modules/_testcapi/monitoring.c +++ b/Modules/_testcapi/monitoring.c @@ -1,8 +1,6 @@ #include "parts.h" #include "util.h" -#include "monitoring.h" - #define Py_BUILD_CORE #include "internal/pycore_instruments.h" @@ -109,7 +107,7 @@ static PyTypeObject PyCodeLike_Type = { }; #define RAISE_UNLESS_CODELIKE(v) if (!Py_IS_TYPE((v), &PyCodeLike_Type)) { \ - PyErr_Format(PyExc_TypeError, "expected a code-like, got %s", Py_TYPE(v)->tp_name); \ + PyErr_Format(PyExc_TypeError, "expected a code-like, got %T", v); \ return NULL; \ } diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 798ef97c495..4c9632c07a9 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -485,6 +485,30 @@ is_uniquely_referenced(PyObject *self, PyObject *op) } +static PyObject * +pyobject_dump(PyObject *self, PyObject *args) +{ + PyObject *op; + int release_gil = 0; + + if (!PyArg_ParseTuple(args, "O|i", &op, &release_gil)) { + return NULL; + } + NULLABLE(op); + + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + PyUnstable_Object_Dump(op); + Py_END_ALLOW_THREADS + + } + else { + PyUnstable_Object_Dump(op); + } + Py_RETURN_NONE; +} + + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, @@ -511,6 +535,7 @@ static PyMethodDef test_methods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, {"is_uniquely_referenced", is_uniquely_referenced, METH_O}, + {"pyobject_dump", pyobject_dump, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index af6400162da..a7feca5bd96 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -58,6 +58,7 @@ int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_Time(PyObject *module); +int _PyTestCapi_Init_Modsupport(PyObject *module); int _PyTestCapi_Init_Monitoring(PyObject *module); int _PyTestCapi_Init_Object(PyObject *module); int _PyTestCapi_Init_Config(PyObject *mod); @@ -65,5 +66,6 @@ int _PyTestCapi_Init_Import(PyObject *mod); int _PyTestCapi_Init_Frame(PyObject *mod); int _PyTestCapi_Init_Type(PyObject *mod); int _PyTestCapi_Init_Function(PyObject *mod); +int _PyTestCapi_Init_Module(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c index 464cf5c3125..4ca6ff587b9 100644 --- a/Modules/_testcapi/time.c +++ b/Modules/_testcapi/time.c @@ -5,8 +5,7 @@ static int pytime_from_nanoseconds(PyTime_t *tp, PyObject *obj) { if (!PyLong_Check(obj)) { - PyErr_Format(PyExc_TypeError, "expect int, got %s", - Py_TYPE(obj)->tp_name); + PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); return -1; } diff --git a/Modules/_testcapi/tuple.c b/Modules/_testcapi/tuple.c index d9c02ba0ff0..5de1c494c0a 100644 --- a/Modules/_testcapi/tuple.c +++ b/Modules/_testcapi/tuple.c @@ -104,12 +104,40 @@ _check_tuple_item_is_NULL(PyObject *Py_UNUSED(module), PyObject *args) } +static PyObject * +tuple_fromarray(PyObject* Py_UNUSED(module), PyObject *args) +{ + PyObject *src; + Py_ssize_t size = UNINITIALIZED_SIZE; + if (!PyArg_ParseTuple(args, "O|n", &src, &size)) { + return NULL; + } + if (src != Py_None && !PyTuple_Check(src)) { + PyErr_SetString(PyExc_TypeError, "expect a tuple"); + return NULL; + } + + PyObject **items; + if (src != Py_None) { + items = &PyTuple_GET_ITEM(src, 0); + if (size == UNINITIALIZED_SIZE) { + size = PyTuple_GET_SIZE(src); + } + } + else { + items = NULL; + } + return PyTuple_FromArray(items, size); +} + + static PyMethodDef test_methods[] = { {"tuple_get_size", tuple_get_size, METH_O}, {"tuple_get_item", tuple_get_item, METH_VARARGS}, {"tuple_set_item", tuple_set_item, METH_VARARGS}, {"_tuple_resize", _tuple_resize, METH_VARARGS}, {"_check_tuple_item_is_NULL", _check_tuple_item_is_NULL, METH_VARARGS}, + {"tuple_fromarray", tuple_fromarray, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index b8ecf53f4f8..203282dd53d 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -220,6 +220,12 @@ unicode_copycharacters(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", to_copy, copied); } +static PyObject* +unicode_GET_CACHED_HASH(PyObject *self, PyObject *arg) +{ + return PyLong_FromSsize_t(PyUnstable_Unicode_GET_CACHED_HASH(arg)); +} + // --- PyUnicodeWriter type ------------------------------------------------- @@ -332,6 +338,27 @@ writer_write_utf8(PyObject *self_raw, PyObject *args) } +static PyObject* +writer_write_ascii(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + char *str; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "yn", &str, &size)) { + return NULL; + } + + if (PyUnicodeWriter_WriteASCII(self->writer, str, size) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + static PyObject* writer_write_widechar(PyObject *self_raw, PyObject *args) { @@ -513,6 +540,7 @@ writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) static PyMethodDef writer_methods[] = { {"write_char", _PyCFunction_CAST(writer_write_char), METH_VARARGS}, {"write_utf8", _PyCFunction_CAST(writer_write_utf8), METH_VARARGS}, + {"write_ascii", _PyCFunction_CAST(writer_write_ascii), METH_VARARGS}, {"write_widechar", _PyCFunction_CAST(writer_write_widechar), METH_VARARGS}, {"write_ucs4", _PyCFunction_CAST(writer_write_ucs4), METH_VARARGS}, {"write_str", _PyCFunction_CAST(writer_write_str), METH_VARARGS}, @@ -548,6 +576,7 @@ static PyMethodDef TestMethods[] = { {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, + {"unicode_GET_CACHED_HASH", unicode_GET_CACHED_HASH, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index 03aaacb328e..a578c6da71b 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -163,6 +163,7 @@ class _testcapi.VectorCallClass "PyObject *" "&PyType_Type" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=95c63c1a47f9a995]*/ /*[clinic input] +@permit_long_summary _testcapi.VectorCallClass.set_vectorcall type: object(subclass_of="&PyType_Type", type="PyTypeObject *") @@ -174,19 +175,19 @@ Set self's vectorcall function for `type` to one that returns "vectorcall" static PyObject * _testcapi_VectorCallClass_set_vectorcall_impl(PyObject *self, PyTypeObject *type) -/*[clinic end generated code: output=b37f0466f15da903 input=840de66182c7d71a]*/ +/*[clinic end generated code: output=b37f0466f15da903 input=170fefc7ee77fd36]*/ { if (!PyObject_TypeCheck(self, type)) { return PyErr_Format( PyExc_TypeError, - "expected %s instance", - PyType_GetName(type)); + "expected %N instance", + type); } if (!type->tp_vectorcall_offset) { return PyErr_Format( PyExc_TypeError, - "type %s has no vectorcall offset", - PyType_GetName(type)); + "type %N has no vectorcall offset", + type); } *(vectorcallfunc*)((char*)self + type->tp_vectorcall_offset) = ( VectorCallClass_vectorcall); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3aa6e4c9e43..c14f925b4e7 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -515,8 +515,7 @@ test_thread_state(PyObject *self, PyObject *args) return NULL; if (!PyCallable_Check(fn)) { - PyErr_Format(PyExc_TypeError, "'%s' object is not callable", - Py_TYPE(fn)->tp_name); + PyErr_Format(PyExc_TypeError, "'%T' object is not callable", fn); return NULL; } @@ -1584,9 +1583,9 @@ getitem_with_error(PyObject *self, PyObject *args) static PyObject * raise_SIGINT_then_send_None(PyObject *self, PyObject *args) { - PyGenObject *gen; + PyObject *gen; - if (!PyArg_ParseTuple(args, "O!", &PyGen_Type, &gen)) + if (!PyArg_ParseTuple(args, "O", &gen)) return NULL; /* This is used in a test to check what happens if a signal arrives just @@ -1600,7 +1599,7 @@ raise_SIGINT_then_send_None(PyObject *self, PyObject *args) because we check for signals before every bytecode operation. */ raise(SIGINT); - return PyObject_CallMethod((PyObject *)gen, "send", "O", Py_None); + return PyObject_CallMethod(gen, "send", "O", Py_None); } @@ -2207,9 +2206,8 @@ test_macros(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject * test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { - // Ignore PyWeakref_GetObject() deprecation, we test it on purpose - _Py_COMP_DIAG_PUSH - _Py_COMP_DIAG_IGNORE_DEPR_DECLS + // Get the function (removed in 3.15) from the stable ABI. + PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *); // Create a new heap type, create an instance of this type, and delete the // type. This object supports weak references. @@ -2250,19 +2248,12 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) ref = PyWeakref_GetObject(weakref); // borrowed ref assert(ref == obj); - // test PyWeakref_GET_OBJECT(), reference is alive - ref = PyWeakref_GET_OBJECT(weakref); // borrowed ref - assert(ref == obj); - // delete the referenced object: clear the weakref assert(Py_REFCNT(obj) == 1); Py_DECREF(obj); assert(PyWeakref_IsDead(weakref)); - // test PyWeakref_GET_OBJECT(), reference is dead - assert(PyWeakref_GET_OBJECT(weakref) == Py_None); - // test PyWeakref_GetRef(), reference is dead ref = UNINITIALIZED_PTR; assert(PyWeakref_GetRef(weakref, &ref) == 0); @@ -2313,13 +2304,12 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_DECREF(weakref); Py_RETURN_NONE; - - _Py_COMP_DIAG_POP } struct simpletracer_data { int create_count; int destroy_count; + int tracker_removed; void* addresses[10]; }; @@ -2327,10 +2317,18 @@ static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { struct simpletracer_data* the_data = (struct simpletracer_data*)data; assert(the_data->create_count + the_data->destroy_count < (int)Py_ARRAY_LENGTH(the_data->addresses)); the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; - if (event == PyRefTracer_CREATE) { - the_data->create_count++; - } else { - the_data->destroy_count++; + switch (event) { + case PyRefTracer_CREATE: + the_data->create_count++; + break; + case PyRefTracer_DESTROY: + the_data->destroy_count++; + break; + case PyRefTracer_TRACKER_REMOVED: + the_data->tracker_removed++; + break; + default: + return -1; } return 0; } @@ -2394,6 +2392,10 @@ test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) PyErr_SetString(PyExc_ValueError, "The object destruction was not correctly traced"); goto failed; } + if (tracer_data.tracker_removed != 1) { + PyErr_SetString(PyExc_ValueError, "The tracker removal was not correctly traced"); + goto failed; + } PyRefTracer_SetTracer(current_tracer, current_data); Py_RETURN_NONE; failed: @@ -2419,12 +2421,22 @@ test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args)) Py_BEGIN_CRITICAL_SECTION2(module, module); Py_END_CRITICAL_SECTION2(); +#ifdef Py_GIL_DISABLED + // avoid unused variable compiler warning on GIL-enabled build + PyMutex mut = {0}; + Py_BEGIN_CRITICAL_SECTION_MUTEX(&mut); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION2_MUTEX(&mut, &mut); + Py_END_CRITICAL_SECTION2(); +#endif + Py_RETURN_NONE; } // Used by `finalize_thread_hang`. -#ifdef _POSIX_THREADS +#if defined(_POSIX_THREADS) && !defined(__wasi__) static void finalize_thread_hang_cleanup_callback(void *Py_UNUSED(arg)) { // Should not reach here. Py_FatalError("pthread thread termination was triggered unexpectedly"); @@ -2524,11 +2536,15 @@ code_offset_to_line(PyObject* self, PyObject* const* args, Py_ssize_t nargsf) static int _reftrace_printer(PyObject *obj, PyRefTracerEvent event, void *counter_data) { - if (event == PyRefTracer_CREATE) { - printf("CREATE %s\n", Py_TYPE(obj)->tp_name); - } - else { // PyRefTracer_DESTROY - printf("DESTROY %s\n", Py_TYPE(obj)->tp_name); + switch (event) { + case PyRefTracer_CREATE: + printf("CREATE %s\n", Py_TYPE(obj)->tp_name); + break; + case PyRefTracer_DESTROY: + printf("DESTROY %s\n", Py_TYPE(obj)->tp_name); + break; + case PyRefTracer_TRACKER_REMOVED: + return 0; } return 0; } @@ -2546,6 +2562,39 @@ toggle_reftrace_printer(PyObject *ob, PyObject *arg) Py_RETURN_NONE; } + +typedef struct { + PyObject_HEAD +} ManagedWeakrefNoGCObject; + +static void +ManagedWeakrefNoGC_dealloc(PyObject *self) +{ + PyObject_ClearWeakRefs(self); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot ManagedWeakrefNoGC_slots[] = { + {Py_tp_dealloc, ManagedWeakrefNoGC_dealloc}, + {0, 0} +}; + +static PyType_Spec ManagedWeakrefNoGC_spec = { + .name = "_testcapi.ManagedWeakrefNoGCType", + .basicsize = sizeof(ManagedWeakrefNoGCObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_MANAGED_WEAKREF), + .slots = ManagedWeakrefNoGC_slots, +}; + +static PyObject * +create_managed_weakref_nogc_type(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return PyType_FromSpec(&ManagedWeakrefNoGC_spec); +} + + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -2640,6 +2689,8 @@ static PyMethodDef TestMethods[] = { {"test_atexit", test_atexit, METH_NOARGS}, {"code_offset_to_line", _PyCFunction_CAST(code_offset_to_line), METH_FASTCALL}, {"toggle_reftrace_printer", toggle_reftrace_printer, METH_O}, + {"create_managed_weakref_nogc_type", + create_managed_weakref_nogc_type, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -3175,75 +3226,98 @@ create_manual_heap_type(void) return (PyObject *)type; } -static struct PyModuleDef _testcapimodule = { - PyModuleDef_HEAD_INIT, - .m_name = "_testcapi", - .m_size = sizeof(testcapistate_t), - .m_methods = TestMethods, +typedef struct { + PyObject_VAR_HEAD +} ManagedDictObject; + +int ManagedDict_traverse(PyObject *self, visitproc visit, void *arg) { + PyObject_VisitManagedDict(self, visit, arg); + Py_VISIT(Py_TYPE(self)); + return 0; +} + +int ManagedDict_clear(PyObject *self) { + PyObject_ClearManagedDict(self); + return 0; +} + +static PyGetSetDef ManagedDict_getset[] = { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {NULL, NULL, NULL, NULL, NULL}, }; -/* Per PEP 489, this module will not be converted to multi-phase initialization - */ +static PyType_Slot ManagedDict_slots[] = { + {Py_tp_new, (void *)PyType_GenericNew}, + {Py_tp_getset, (void *)ManagedDict_getset}, + {Py_tp_traverse, (void *)ManagedDict_traverse}, + {Py_tp_clear, (void *)ManagedDict_clear}, + {0} +}; -PyMODINIT_FUNC -PyInit__testcapi(void) +static PyType_Spec ManagedDict_spec = { + "_testcapi.ManagedDictType", + sizeof(ManagedDictObject), + 0, // itemsize + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_GC, + ManagedDict_slots +}; + +static PyObject * +create_managed_dict_type(void) { - PyObject *m; - - m = PyModule_Create(&_testcapimodule); - if (m == NULL) - return NULL; -#ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif + return PyType_FromSpec(&ManagedDict_spec); +} +static int +_testcapi_exec(PyObject *m) +{ Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); if (PyType_Ready(&_HashInheritanceTester_Type) < 0) { - return NULL; + return -1; } if (PyType_Ready(&matmulType) < 0) - return NULL; + return -1; Py_INCREF(&matmulType); PyModule_AddObject(m, "matmulType", (PyObject *)&matmulType); if (PyType_Ready(&ipowType) < 0) { - return NULL; + return -1; } Py_INCREF(&ipowType); PyModule_AddObject(m, "ipowType", (PyObject *)&ipowType); if (PyType_Ready(&awaitType) < 0) - return NULL; + return -1; Py_INCREF(&awaitType); PyModule_AddObject(m, "awaitType", (PyObject *)&awaitType); MyList_Type.tp_base = &PyList_Type; if (PyType_Ready(&MyList_Type) < 0) - return NULL; + return -1; Py_INCREF(&MyList_Type); PyModule_AddObject(m, "MyList", (PyObject *)&MyList_Type); if (PyType_Ready(&GenericAlias_Type) < 0) - return NULL; + return -1; Py_INCREF(&GenericAlias_Type); PyModule_AddObject(m, "GenericAlias", (PyObject *)&GenericAlias_Type); if (PyType_Ready(&Generic_Type) < 0) - return NULL; + return -1; Py_INCREF(&Generic_Type); PyModule_AddObject(m, "Generic", (PyObject *)&Generic_Type); if (PyType_Ready(&MethInstance_Type) < 0) - return NULL; + return -1; Py_INCREF(&MethInstance_Type); PyModule_AddObject(m, "MethInstance", (PyObject *)&MethInstance_Type); if (PyType_Ready(&MethClass_Type) < 0) - return NULL; + return -1; Py_INCREF(&MethClass_Type); PyModule_AddObject(m, "MethClass", (PyObject *)&MethClass_Type); if (PyType_Ready(&MethStatic_Type) < 0) - return NULL; + return -1; Py_INCREF(&MethStatic_Type); PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type); @@ -3285,14 +3359,18 @@ PyInit__testcapi(void) PyModule_AddObject(m, "INT64_MAX", PyLong_FromInt64(INT64_MAX)); PyModule_AddObject(m, "UINT64_MAX", PyLong_FromUInt64(UINT64_MAX)); + if (PyModule_AddIntMacro(m, _Py_STACK_GROWS_DOWN)) { + return -1; + } + if (PyModule_AddIntMacro(m, Py_single_input)) { - return NULL; + return -1; } if (PyModule_AddIntMacro(m, Py_file_input)) { - return NULL; + return -1; } if (PyModule_AddIntMacro(m, Py_eval_input)) { - return NULL; + return -1; } testcapistate_t *state = get_testcapi_state(m); @@ -3300,135 +3378,168 @@ PyInit__testcapi(void) PyModule_AddObject(m, "error", state->error); if (PyType_Ready(&ContainerNoGC_type) < 0) { - return NULL; + return -1; } Py_INCREF(&ContainerNoGC_type); if (PyModule_AddObject(m, "ContainerNoGC", (PyObject *) &ContainerNoGC_type) < 0) - return NULL; + return -1; PyObject *manual_heap_type = create_manual_heap_type(); if (manual_heap_type == NULL) { - return NULL; + return -1; } if (PyModule_Add(m, "ManualHeapType", manual_heap_type) < 0) { - return NULL; + return -1; } + PyObject *managed_dict_type = create_managed_dict_type(); + if (managed_dict_type == NULL) { + return -1; + } + if (PyModule_Add(m, "ManagedDictType", managed_dict_type) < 0) { + return -1; + } /* Include tests from the _testcapi/ directory */ if (_PyTestCapi_Init_Vectorcall(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Heaptype(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Abstract(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Bytes(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Unicode(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_GetArgs(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_DateTime(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Docstring(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Mem(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Watchers(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Long(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Float(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Complex(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Numbers(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Dict(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Set(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_List(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Tuple(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Structmember(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Exceptions(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Code(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Buffer(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_File(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Codec(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Immortal(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_GC(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_PyAtomic(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Run(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Hash(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Time(m) < 0) { - return NULL; + return -1; + } + if (_PyTestCapi_Init_Modsupport(m) < 0) { + return -1; } if (_PyTestCapi_Init_Monitoring(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Object(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Config(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Import(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Frame(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Type(m) < 0) { - return NULL; + return -1; } if (_PyTestCapi_Init_Function(m) < 0) { - return NULL; + return -1; + } + if (_PyTestCapi_Init_Module(m) < 0) { + return -1; } - PyState_AddModule(m, &_testcapimodule); - return m; + return 0; +} + +static PyModuleDef_Slot _testcapi_slots[] = { + {Py_mod_exec, _testcapi_exec}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL}, +}; + +static struct PyModuleDef _testcapimodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_testcapi", + .m_size = sizeof(testcapistate_t), + .m_methods = TestMethods, + .m_slots = _testcapi_slots +}; + +PyMODINIT_FUNC +PyInit__testcapi(void) +{ + return PyModuleDef_Init(&_testcapimodule); } diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 3e903b6d87d..890f2201b46 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -63,7 +63,7 @@ pack_arguments_2pos_varpos(PyObject *a, PyObject *b, PyObject * const *args, Py_ssize_t args_length) /*[clinic end generated code: output=267032f41bd039cc input=86ee3064b7853e86]*/ { - PyObject *tuple = _PyTuple_FromArray(args, args_length); + PyObject *tuple = PyTuple_FromArray(args, args_length); if (tuple == NULL) { return NULL; } @@ -443,16 +443,21 @@ py_ssize_t_converter a: Py_ssize_t = 12 b: Py_ssize_t(accept={int}) = 34 c: Py_ssize_t(accept={int, NoneType}) = 56 + d: Py_ssize_t(accept={int}, allow_negative=False) = 78 + e: Py_ssize_t(accept={int, NoneType}, allow_negative=False) = 90 + f: Py_ssize_t(accept={int}, allow_negative=False) = -12 + g: Py_ssize_t(accept={int, NoneType}, py_default="-34", allow_negative=False) = -34 / [clinic start generated code]*/ static PyObject * py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, - Py_ssize_t c) -/*[clinic end generated code: output=ce252143e0ed0372 input=76d0f342e9317a1f]*/ + Py_ssize_t c, Py_ssize_t d, Py_ssize_t e, + Py_ssize_t f, Py_ssize_t g) +/*[clinic end generated code: output=ecf8e1a4a9abc95e input=7b7fa954780c1cb0]*/ { - RETURN_PACKED_ARGS(3, PyLong_FromSsize_t, Py_ssize_t, a, b, c); + RETURN_PACKED_ARGS(7, PyLong_FromSsize_t, Py_ssize_t, a, b, c, d, e, f, g); } @@ -658,16 +663,16 @@ str_converter_encoding_impl(PyObject *module, char *a, char *b, char *c, static PyObject * bytes_from_buffer(Py_buffer *buf) { - PyObject *bytes_obj = PyBytes_FromStringAndSize(NULL, buf->len); - if (!bytes_obj) { + PyBytesWriter *writer = PyBytesWriter_Create(buf->len); + if (writer == NULL) { return NULL; } - void *bytes_obj_buf = ((PyBytesObject *)bytes_obj)->ob_sval; - if (PyBuffer_ToContiguous(bytes_obj_buf, buf, buf->len, 'C') < 0) { - Py_DECREF(bytes_obj); + void *data = PyBytesWriter_GetData(writer); + if (PyBuffer_ToContiguous(data, buf, buf->len, 'C') < 0) { + PyBytesWriter_Discard(writer); return NULL; } - return bytes_obj; + return PyBytesWriter_Finish(writer); } /*[clinic input] @@ -1169,7 +1174,7 @@ varpos_array_impl(PyObject *module, PyObject * const *args, Py_ssize_t args_length) /*[clinic end generated code: output=a25f42f39c9b13ad input=97b8bdcf87e019c7]*/ { - return _PyTuple_FromArray(args, args_length); + return PyTuple_FromArray(args, args_length); } @@ -1605,7 +1610,7 @@ _testclinic_TestClass_varpos_array_no_fastcall_impl(PyTypeObject *type, Py_ssize_t args_length) /*[clinic end generated code: output=27c9da663e942617 input=9ba5ae1f1eb58777]*/ { - return _PyTuple_FromArray(args, args_length); + return PyTuple_FromArray(args, args_length); } @@ -2303,6 +2308,88 @@ depr_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, #undef _SAVED_PY_VERSION +/*[clinic input] +output pop +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e7c7c42daced52b0]*/ + + +/*[clinic input] +output push +destination kwarg new file '{dirname}/clinic/_testclinic_kwds.c.h' +output everything kwarg +output docstring_prototype suppress +output parser_prototype suppress +output impl_definition block +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=02965b54b3981cc4]*/ + +#include "clinic/_testclinic_kwds.c.h" + + +/*[clinic input] +lone_kwds + **kwds: dict +[clinic start generated code]*/ + +static PyObject * +lone_kwds_impl(PyObject *module, PyObject *kwds) +/*[clinic end generated code: output=572549c687a0432e input=6ef338b913ecae17]*/ +{ + return pack_arguments_newref(1, kwds); +} + + +/*[clinic input] +kwds_with_pos_only + a: object + b: object + / + **kwds: dict +[clinic start generated code]*/ + +static PyObject * +kwds_with_pos_only_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *kwds) +/*[clinic end generated code: output=573096d3a7efcce5 input=da081a5d9ae8878a]*/ +{ + return pack_arguments_newref(3, a, b, kwds); +} + + +/*[clinic input] +kwds_with_stararg + *args: tuple + **kwds: dict +[clinic start generated code]*/ + +static PyObject * +kwds_with_stararg_impl(PyObject *module, PyObject *args, PyObject *kwds) +/*[clinic end generated code: output=d4b0064626a25208 input=1be404572d685859]*/ +{ + return pack_arguments_newref(2, args, kwds); +} + + +/*[clinic input] +kwds_with_pos_only_and_stararg + a: object + b: object + / + *args: tuple + **kwds: dict +[clinic start generated code]*/ + +static PyObject * +kwds_with_pos_only_and_stararg_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *args, + PyObject *kwds) +/*[clinic end generated code: output=af7df7640c792246 input=2fe330c7981f0829]*/ +{ + return pack_arguments_newref(4, a, b, args, kwds); +} + + /*[clinic input] output pop [clinic start generated code]*/ @@ -2399,6 +2486,12 @@ static PyMethodDef tester_methods[] = { DEPR_KWD_NOINLINE_METHODDEF DEPR_KWD_MULTI_METHODDEF DEPR_MULTI_METHODDEF + + LONE_KWDS_METHODDEF + KWDS_WITH_POS_ONLY_METHODDEF + KWDS_WITH_STARARG_METHODDEF + KWDS_WITH_POS_ONLY_AND_STARARG_METHODDEF + {NULL, NULL} }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ae060c95fd5..89e558b0fe8 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -21,6 +21,7 @@ #include "pycore_fileutils.h" // _Py_normpath() #include "pycore_flowgraph.h" // _PyCompile_OptimizeCfg() #include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_function.h" // _PyFunction_GET_BUILTINS #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_import.h" // _PyImport_ClearExtension() @@ -33,6 +34,7 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pylifecycle.h" // _PyInterpreterConfig_InitFromDict() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime_structs.h" // _PY_NSMALLPOSINTS #include "pycore_unicodeobject.h" // _PyUnicode_TransformDecimalAndSpaceToASCII() #include "clinic/_testinternalcapi.c.h" @@ -120,10 +122,22 @@ get_c_recursion_remaining(PyObject *self, PyObject *Py_UNUSED(args)) PyThreadState *tstate = _PyThreadState_GET(); uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - int remaining = (int)((here_addr - _tstate->c_stack_soft_limit)/PYOS_STACK_MARGIN_BYTES * 50); + int remaining = (int)((here_addr - _tstate->c_stack_soft_limit) / _PyOS_STACK_MARGIN_BYTES * 50); return PyLong_FromLong(remaining); } +static PyObject* +get_stack_pointer(PyObject *self, PyObject *Py_UNUSED(args)) +{ + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + return PyLong_FromSize_t(here_addr); +} + +static PyObject* +get_stack_margin(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return PyLong_FromSize_t(_PyOS_STACK_MARGIN_BYTES); +} static PyObject* test_bswap(PyObject *self, PyObject *Py_UNUSED(args)) @@ -1022,7 +1036,7 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs) globalsns = PyFunction_GET_GLOBALS(codearg); } if (builtinsns == NULL) { - builtinsns = PyFunction_GET_BUILTINS(codearg); + builtinsns = _PyFunction_GET_BUILTINS(codearg); } codearg = PyFunction_GET_CODE(codearg); } @@ -1045,6 +1059,9 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs) #define SET_COUNT(DICT, STRUCT, NAME) \ do { \ PyObject *count = PyLong_FromLong(STRUCT.NAME); \ + if (count == NULL) { \ + goto error; \ + } \ int res = PyDict_SetItemString(DICT, #NAME, count); \ Py_DECREF(count); \ if (res < 0) { \ @@ -1165,6 +1182,47 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs) return NULL; } +static PyObject * +verify_stateless_code(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *codearg; + PyObject *globalnames = NULL; + PyObject *globalsns = NULL; + PyObject *builtinsns = NULL; + static char *kwlist[] = {"code", "globalnames", + "globalsns", "builtinsns", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O|O!O!O!:get_code_var_counts", kwlist, + &codearg, &PySet_Type, &globalnames, + &PyDict_Type, &globalsns, &PyDict_Type, &builtinsns)) + { + return NULL; + } + if (PyFunction_Check(codearg)) { + if (globalsns == NULL) { + globalsns = PyFunction_GET_GLOBALS(codearg); + } + if (builtinsns == NULL) { + builtinsns = _PyFunction_GET_BUILTINS(codearg); + } + codearg = PyFunction_GET_CODE(codearg); + } + else if (!PyCode_Check(codearg)) { + PyErr_SetString(PyExc_TypeError, + "argument must be a code object or a function"); + return NULL; + } + PyCodeObject *code = (PyCodeObject *)codearg; + + if (_PyCode_VerifyStateless( + tstate, code, globalnames, globalsns, builtinsns) < 0) + { + return NULL; + } + Py_RETURN_NONE; +} + #ifdef _Py_TIER2 static PyObject * @@ -1649,11 +1707,12 @@ create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) static PyObject * destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "basic", NULL}; PyObject *idobj = NULL; + int basic = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "O:destroy_interpreter", kwlist, - &idobj)) + "O|p:destroy_interpreter", kwlist, + &idobj, &basic)) { return NULL; } @@ -1663,7 +1722,27 @@ destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } - _PyXI_EndInterpreter(interp, NULL, NULL); + if (basic) + { + // Test the basic Py_EndInterpreter with weird out of order thread states + PyThreadState *t1, *t2; + PyThreadState *prev; + t1 = interp->threads.head; + if (t1 == NULL) { + t1 = PyThreadState_New(interp); + } + t2 = PyThreadState_New(interp); + prev = PyThreadState_Swap(t2); + PyThreadState_Clear(t1); + PyThreadState_Delete(t1); + Py_EndInterpreter(t2); + PyThreadState_Swap(prev); + } + else + { + // use the cross interpreter _PyXI_EndInterpreter normally + _PyXI_EndInterpreter(interp, NULL, NULL); + } Py_RETURN_NONE; } @@ -1723,9 +1802,9 @@ exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) /* To run some code in a sub-interpreter. -Generally you can use test.support.interpreters, +Generally you can use the interpreters module, but we keep this helper as a distinct implementation. -That's especially important for testing test.support.interpreters. +That's especially important for testing the interpreters module. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1929,7 +2008,14 @@ get_crossinterp_data(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } if (strcmp(mode, "xidata") == 0) { - if (_PyObject_GetXIData(tstate, obj, xidata) != 0) { + if (_PyObject_GetXIDataNoFallback(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "fallback") == 0) { + xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; + if (_PyObject_GetXIData(tstate, obj, fallback, xidata) != 0) + { goto error; } } @@ -1948,6 +2034,21 @@ get_crossinterp_data(PyObject *self, PyObject *args, PyObject *kwargs) goto error; } } + else if (strcmp(mode, "func") == 0) { + if (_PyFunction_GetXIData(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "script") == 0) { + if (_PyCode_GetScriptXIData(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "script-pure") == 0) { + if (_PyCode_GetPureScriptXIData(tstate, obj, xidata) != 0) { + goto error; + } + } else { PyErr_Format(PyExc_ValueError, "unsupported mode %R", modeobj); goto error; @@ -2119,8 +2220,7 @@ get_code(PyObject *obj) return (PyCodeObject *)PyFunction_GetCode(obj); } return (PyCodeObject *)PyErr_Format( - PyExc_TypeError, "expected function or code object, got %s", - Py_TYPE(obj)->tp_name); + PyExc_TypeError, "expected function or code object, got %T", obj); } static PyObject * @@ -2258,10 +2358,152 @@ incref_decref_delayed(PyObject *self, PyObject *op) Py_RETURN_NONE; } +#ifdef __EMSCRIPTEN__ +#include "emscripten.h" + +EM_JS(int, emscripten_set_up_async_input_device_js, (void), { + let idx = 0; + const encoder = new TextEncoder(); + const bufs = [ + encoder.encode("ab\n"), + encoder.encode("fi\n"), + encoder.encode("xy\n"), + ]; + function sleep(t) { + return new Promise(res => setTimeout(res, t)); + } + FS.createAsyncInputDevice("/dev", "blah", async () => { + await sleep(5); + return bufs[(idx ++) % 3]; + }); + return !!WebAssembly.promising; +}); + +static PyObject * +emscripten_set_up_async_input_device(PyObject *self, PyObject *Py_UNUSED(ignored)) { + if (emscripten_set_up_async_input_device_js()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} +#endif + +static PyObject * +simple_pending_call(PyObject *self, PyObject *callable) +{ + if (_PyEval_AddPendingCall(_PyInterpreterState_GET(), _pending_callback, Py_NewRef(callable), 0) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +vectorcall_nop(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + Py_RETURN_NONE; +} + +static PyObject * +set_vectorcall_nop(PyObject *self, PyObject *func) +{ + if (!PyFunction_Check(func)) { + PyErr_SetString(PyExc_TypeError, "expected function"); + return NULL; + } + + ((PyFunctionObject*)func)->vectorcall = vectorcall_nop; + Py_RETURN_NONE; +} + +static PyObject * +module_get_gc_hooks(PyObject *self, PyObject *arg) +{ + PyModuleObject *mod = (PyModuleObject *)arg; + PyObject *traverse = NULL; + PyObject *clear = NULL; + PyObject *free = NULL; + PyObject *result = NULL; + traverse = PyLong_FromVoidPtr(mod->md_state_traverse); + if (!traverse) { + goto finally; + } + clear = PyLong_FromVoidPtr(mod->md_state_clear); + if (!clear) { + goto finally; + } + free = PyLong_FromVoidPtr(mod->md_state_free); + if (!free) { + goto finally; + } + result = PyTuple_FromArray((PyObject*[]){ traverse, clear, free }, 3); +finally: + Py_XDECREF(traverse); + Py_XDECREF(clear); + Py_XDECREF(free); + return result; +} + + +static void +check_threadstate_set_stack_protection(PyThreadState *tstate, + void *start, size_t size) +{ + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == 0); + assert(!PyErr_Occurred()); + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_top == (uintptr_t)start + size); + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +} + + +static PyObject * +test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = PyThreadState_GET(); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(!PyErr_Occurred()); + + uintptr_t init_base = ts->c_stack_init_base; + size_t init_top = ts->c_stack_init_top; + + // Test the minimum stack size + size_t size = _PyOS_MIN_STACK_SIZE; + void *start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test a larger size + size = 7654321; + assert(size > _PyOS_MIN_STACK_SIZE); + start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test invalid size (too small) + size = 5; + start = (void*)(_Py_get_machine_stack_pointer() - size); + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == -1); + assert(PyErr_ExceptionMatches(PyExc_ValueError)); + PyErr_Clear(); + + // Test PyUnstable_ThreadState_ResetStackProtection() + PyUnstable_ThreadState_ResetStackProtection(tstate); + assert(ts->c_stack_init_base == init_base); + assert(ts->c_stack_init_top == init_top); + + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS}, + {"get_stack_pointer", get_stack_pointer, METH_NOARGS}, + {"get_stack_margin", get_stack_margin, METH_NOARGS}, {"test_bswap", test_bswap, METH_NOARGS}, {"test_popcount", test_popcount, METH_NOARGS}, {"test_bit_length", test_bit_length, METH_NOARGS}, @@ -2292,6 +2534,8 @@ static PyMethodDef module_functions[] = { {"get_co_localskinds", get_co_localskinds, METH_O, NULL}, {"get_code_var_counts", _PyCFunction_CAST(get_code_var_counts), METH_VARARGS | METH_KEYWORDS, NULL}, + {"verify_stateless_code", _PyCFunction_CAST(verify_stateless_code), + METH_VARARGS | METH_KEYWORDS, NULL}, #ifdef _Py_TIER2 {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, @@ -2358,6 +2602,14 @@ static PyMethodDef module_functions[] = { {"is_static_immortal", is_static_immortal, METH_O}, {"incref_decref_delayed", incref_decref_delayed, METH_O}, GET_NEXT_DICT_KEYS_VERSION_METHODDEF +#ifdef __EMSCRIPTEN__ + {"emscripten_set_up_async_input_device", emscripten_set_up_async_input_device, METH_NOARGS}, +#endif + {"simple_pending_call", simple_pending_call, METH_O}, + {"set_vectorcall_nop", set_vectorcall_nop, METH_O}, + {"module_get_gc_hooks", module_get_gc_hooks, METH_O}, + {"test_threadstate_set_stack_protection", + test_threadstate_set_stack_protection, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -2409,7 +2661,8 @@ module_exec(PyObject *module) } if (PyModule_Add(module, "TIER2_THRESHOLD", - PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 1)) < 0) { + // + 1 more due to one loop spent on tracing. + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 2)) < 0) { return 1; } @@ -2428,6 +2681,10 @@ module_exec(PyObject *module) return 1; } + if (PyModule_AddIntMacro(module, _PY_NSMALLPOSINTS) < 0) { + return 1; + } + return 0; } diff --git a/Modules/_testinternalcapi/pytime.c b/Modules/_testinternalcapi/pytime.c index 2b0a205d158..7fb100c41a1 100644 --- a/Modules/_testinternalcapi/pytime.c +++ b/Modules/_testinternalcapi/pytime.c @@ -17,7 +17,7 @@ test_pytime_fromseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t ts = _PyTime_FromSeconds(seconds); - return _PyTime_AsLong(ts); + return PyLong_FromInt64(ts); } static int @@ -49,7 +49,7 @@ test_pytime_fromsecondsobject(PyObject *self, PyObject *args) if (_PyTime_FromSecondsObject(&ts, obj, round) == -1) { return NULL; } - return _PyTime_AsLong(ts); + return PyLong_FromInt64(ts); } static PyObject * @@ -64,7 +64,7 @@ test_PyTime_AsTimeval(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timeval tv; @@ -91,7 +91,7 @@ test_PyTime_AsTimeval_clamp(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timeval tv; @@ -113,7 +113,7 @@ test_PyTime_AsTimespec(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timespec ts; @@ -131,7 +131,7 @@ test_PyTime_AsTimespec_clamp(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } struct timespec ts; @@ -149,14 +149,14 @@ test_PyTime_AsMilliseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } PyTime_t ms = _PyTime_AsMilliseconds(t, round); - return _PyTime_AsLong(ms); + return PyLong_FromInt64(ms); } static PyObject * @@ -168,14 +168,14 @@ test_PyTime_AsMicroseconds(PyObject *self, PyObject *args) return NULL; } PyTime_t t; - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (check_time_rounding(round) < 0) { return NULL; } PyTime_t us = _PyTime_AsMicroseconds(t, round); - return _PyTime_AsLong(us); + return PyLong_FromInt64(us); } static PyObject * diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index e0ba37abcdd..e3b2fe716d4 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -284,10 +284,111 @@ test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) #endif +#ifdef Py_GIL_DISABLED + +static PyObject * +test_critical_section1_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + + PyCriticalSection cs1, cs2; + // First acquisition of critical section on object locks it + PyCriticalSection_Begin(&cs1, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Attempting to re-acquire critical section on same object which + // is already locked by top-most critical section is a no-op. + PyCriticalSection_Begin(&cs2, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing second critical section is a no-op. + PyCriticalSection_End(&cs2); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing first critical section unlocks the object + PyCriticalSection_End(&cs1); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + + Py_DECREF(a); + Py_RETURN_NONE; +} + +static PyObject * +test_critical_section2_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + PyObject *b = PyDict_New(); + assert(b != NULL); + + PyCriticalSection2 cs; + // First acquisition of critical section on objects locks them + PyCriticalSection2_Begin(&cs, a, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Attempting to re-acquire critical section on either of two + // objects already locked by top-most critical section is a no-op. + + // Check re-acquiring on first object + PyCriticalSection a_cs; + PyCriticalSection_Begin(&a_cs, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&a_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Check re-acquiring on second object + PyCriticalSection b_cs; + PyCriticalSection_Begin(&b_cs, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&b_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Releasing critical section on both objects unlocks them + PyCriticalSection2_End(&cs); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + assert(!PyMutex_IsLocked(&b->ob_mutex)); + + Py_DECREF(a); + Py_DECREF(b); + Py_RETURN_NONE; +} + +#endif // Py_GIL_DISABLED + static PyMethodDef test_methods[] = { {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"test_critical_sections_nest", test_critical_sections_nest, METH_NOARGS}, {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, +#ifdef Py_GIL_DISABLED + {"test_critical_section1_reacquisition", test_critical_section1_reacquisition, METH_NOARGS}, + {"test_critical_section2_reacquisition", test_critical_section2_reacquisition, METH_NOARGS}, +#endif #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 8d678412fe7..ded76ca9fe6 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -57,7 +57,10 @@ lock_thread(void *arg) _Py_atomic_store_int(&test_data->started, 1); PyMutex_Lock(m); - assert(m->_bits == 1); + // gh-135641: in rare cases the lock may still have `_Py_HAS_PARKED` set + // (m->_bits == 3) due to bucket collisions in the parking lot hash table + // between this mutex and the `test_data.done` event. + assert(m->_bits == 1 || m->_bits == 3); PyMutex_Unlock(m); assert(m->_bits == 0); @@ -88,7 +91,8 @@ test_lock_two_threads(PyObject *self, PyObject *obj) } while (v != 3 && iters < 200); // both the "locked" and the "has parked" bits should be set - assert(test_data.m._bits == 3); + v = _Py_atomic_load_uint8_relaxed(&test_data.m._bits); + assert(v == 3); PyMutex_Unlock(&test_data.m); PyEvent_Wait(&test_data.done); diff --git a/Modules/_testlimitedcapi/import.c b/Modules/_testlimitedcapi/import.c index 3707dbedeea..f85daee57d7 100644 --- a/Modules/_testlimitedcapi/import.c +++ b/Modules/_testlimitedcapi/import.c @@ -108,20 +108,19 @@ pyimport_importmodule(PyObject *Py_UNUSED(module), PyObject *args) } -/* Test PyImport_ImportModuleNoBlock() */ +/* Test PyImport_ImportModuleNoBlock() (removed in 3.15) */ static PyObject * pyimport_importmodulenoblock(PyObject *Py_UNUSED(module), PyObject *args) { + // Get the function from the stable ABI. + PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock(const char *name); + const char *name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "z#", &name, &size)) { return NULL; } - - _Py_COMP_DIAG_PUSH - _Py_COMP_DIAG_IGNORE_DEPR_DECLS return PyImport_ImportModuleNoBlock(name); - _Py_COMP_DIAG_POP } diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c index 35da5fa5f00..34ed6b1d60b 100644 --- a/Modules/_testlimitedcapi/set.c +++ b/Modules/_testlimitedcapi/set.c @@ -155,6 +155,51 @@ test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) return NULL; } +static PyObject * +test_set_contains_does_not_convert_unhashable_key(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // See https://docs.python.org/3/c-api/set.html#c.PySet_Contains + PyObject *outer_set = PySet_New(NULL); + + PyObject *needle = PySet_New(NULL); + if (needle == NULL) { + Py_DECREF(outer_set); + return NULL; + } + + PyObject *num = PyLong_FromLong(42); + if (num == NULL) { + Py_DECREF(outer_set); + Py_DECREF(needle); + return NULL; + } + + if (PySet_Add(needle, num) < 0) { + Py_DECREF(outer_set); + Py_DECREF(needle); + Py_DECREF(num); + return NULL; + } + + int result = PySet_Contains(outer_set, needle); + + Py_DECREF(num); + Py_DECREF(needle); + Py_DECREF(outer_set); + + if (result < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + Py_RETURN_NONE; + } + return NULL; + } + + PyErr_SetString(PyExc_AssertionError, + "PySet_Contains should have raised TypeError for unhashable key"); + return NULL; +} + static PyMethodDef test_methods[] = { {"set_check", set_check, METH_O}, {"set_checkexact", set_checkexact, METH_O}, @@ -174,6 +219,8 @@ static PyMethodDef test_methods[] = { {"set_clear", set_clear, METH_O}, {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + {"test_set_contains_does_not_convert_unhashable_key", + test_set_contains_does_not_convert_unhashable_key, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testlimitedcapi/sys.c b/Modules/_testlimitedcapi/sys.c index 7d8b7a8569e..cec7f8ab612 100644 --- a/Modules/_testlimitedcapi/sys.c +++ b/Modules/_testlimitedcapi/sys.c @@ -1,7 +1,76 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +// Need limited C API version 3.15 for PySys_GetAttr() etc +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) +# define Py_LIMITED_API 0x030f0000 +#endif #include "parts.h" #include "util.h" +static PyObject * +sys_getattr(PyObject *Py_UNUSED(module), PyObject *name) +{ + NULLABLE(name); + return PySys_GetAttr(name); +} + +static PyObject * +sys_getattrstring(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + return PySys_GetAttrString(name); +} + +static PyObject * +sys_getoptionalattr(PyObject *Py_UNUSED(module), PyObject *name) +{ + PyObject *value = UNINITIALIZED_PTR; + NULLABLE(name); + + switch (PySys_GetOptionalAttr(name, &value)) { + case -1: + assert(value == NULL); + assert(PyErr_Occurred()); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_AttributeError); + case 1: + return value; + default: + Py_FatalError("PySys_GetOptionalAttr() returned invalid code"); + } +} + +static PyObject * +sys_getoptionalattrstring(PyObject *Py_UNUSED(module), PyObject *arg) +{ + PyObject *value = UNINITIALIZED_PTR; + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + + switch (PySys_GetOptionalAttrString(name, &value)) { + case -1: + assert(value == NULL); + assert(PyErr_Occurred()); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_AttributeError); + case 1: + return value; + default: + Py_FatalError("PySys_GetOptionalAttrString() returned invalid code"); + } +} + static PyObject * sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) { @@ -39,6 +108,10 @@ sys_getxoptions(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(ignored)) static PyMethodDef test_methods[] = { + {"sys_getattr", sys_getattr, METH_O}, + {"sys_getattrstring", sys_getattrstring, METH_O}, + {"sys_getoptionalattr", sys_getoptionalattr, METH_O}, + {"sys_getoptionalattrstring", sys_getoptionalattrstring, METH_O}, {"sys_getobject", sys_getobject, METH_O}, {"sys_setobject", sys_setobject, METH_VARARGS}, {"sys_getxoptions", sys_getxoptions, METH_NOARGS}, diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index bfec0678e2c..e286eaae820 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -850,6 +850,28 @@ PyInit__testmultiphase_exec_unreported_exception(void) return PyModuleDef_Init(&def_exec_unreported_exception); } +static int execfn_a1(PyObject*m) { return PyModule_AddIntConstant(m, "a", 1); } +static int execfn_b2(PyObject*m) { return PyModule_AddIntConstant(m, "b", 2); } +static int execfn_c3(PyObject*m) { return PyModule_AddIntConstant(m, "c", 3); } + +PyMODINIT_FUNC +PyInit__testmultiphase_exec_multiple(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_exec, execfn_a1}, + {Py_mod_exec, execfn_b2}, + {Py_mod_exec, execfn_c3}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0} + }; + static PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name="_testmultiphase_exec_multiple", + .m_slots=slots, + }; + return PyModuleDef_Init(&def); +} + static int meth_state_access_exec(PyObject *m) { @@ -993,3 +1015,188 @@ PyInit__test_no_multiple_interpreter_slot(void) { return PyModuleDef_Init(&no_multiple_interpreter_slot_def); } + + +/* PyModExport_* hooks */ + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport"}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return slots; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_gil_used(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_gil_used"}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_USED}, + {0}, + }; + return slots; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_null(void) +{ + return NULL; +} + +PyMODINIT_FUNC +PyModInit__test_from_modexport_null(void) +{ + // This is not called as fallback for failed PyModExport_* + assert(0); + PyErr_SetString(PyExc_AssertionError, "PyInit_ fallback called"); + return NULL; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_exception(void) +{ + PyErr_SetString(PyExc_ValueError, "failed as requested"); + return NULL; +} + +PyMODINIT_FUNC +PyModInit__test_from_modexport_exception(void) +{ + // This is not called as fallback for failed PyModExport_* + assert(0); + PyErr_SetString(PyExc_AssertionError, "PyInit_ fallback called"); + return NULL; +} + +static PyObject * +modexport_create_string(PyObject *spec, PyModuleDef *def) +{ + assert(def == NULL); + return PyUnicode_FromString("is this \xf0\x9f\xa6\x8b... a module?"); +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_create_nonmodule(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_create_nonmodule"}, + {Py_mod_create, modexport_create_string}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return slots; +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_create_nonmodule_gil_used(void) +{ + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_create_nonmodule"}, + {Py_mod_create, modexport_create_string}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_USED}, + {0}, + }; + return slots; +} + +static PyModuleDef_Slot modexport_empty_slots[] = { + {0}, +}; + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_empty_slots(void) +{ + return modexport_empty_slots; +} + +static int +modexport_smoke_exec(PyObject *mod) +{ + // "magic" values 147 & 258 are expected in the test + if (PyModule_AddIntConstant(mod, "number", 147) < 0) { + return 0; + } + int *state = PyModule_GetState(mod); + if (!state) { + return -1; + } + *state = 258; + + PyObject *tp = PyType_FromModuleAndSpec(mod, &StateAccessType_spec, NULL); + if (PyModule_Add(mod, "Example", tp) < 0) { + return -1; + } + + return 0; +} + +static PyObject * +modexport_smoke_get_state_int(PyObject *mod, PyObject *arg) +{ + int *state = PyModule_GetState(mod); + if (!state) { + return NULL; + } + return PyLong_FromLong(*state); +} + +static const char modexport_smoke_test_token; + +static PyObject * +modexport_smoke_get_test_token(PyObject *mod, PyObject *arg) +{ + return PyLong_FromVoidPtr((void*)&modexport_smoke_test_token); +} + +static PyObject * +modexport_get_empty_slots(PyObject *mod, PyObject *arg) +{ + /* Get the address of modexport_empty_slots. + * This method would be in the `_test_from_modexport_empty_slots` module, + * if it had a methods slot. + */ + return PyLong_FromVoidPtr(&modexport_empty_slots); +} + +static void +modexport_smoke_free(void *op) +{ + PyObject *mod = (PyObject *)op; + int *state = PyModule_GetState(mod); + if (!state) { + PyErr_FormatUnraisable("Exception ignored in module %R free", mod); + } + assert(*state == 258); +} + +PyMODEXPORT_FUNC +PyModExport__test_from_modexport_smoke(void) +{ + static PyMethodDef methods[] = { + {"get_state_int", modexport_smoke_get_state_int, METH_NOARGS}, + {"get_test_token", modexport_smoke_get_test_token, METH_NOARGS}, + {"get_modexport_empty_slots", modexport_get_empty_slots, METH_NOARGS}, + {0}, + }; + static PyModuleDef_Slot slots[] = { + {Py_mod_name, "_test_from_modexport_smoke"}, + {Py_mod_doc, "the expected docstring"}, + {Py_mod_exec, modexport_smoke_exec}, + {Py_mod_state_size, (void*)sizeof(int)}, + {Py_mod_methods, methods}, + {Py_mod_state_free, modexport_smoke_free}, + {Py_mod_token, (void*)&modexport_smoke_test_token}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + return slots; +} diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 2c59085d15b..ee38d61b43a 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -244,6 +244,8 @@ static inline module_state * get_module_state(PyObject *module) { PyModuleDef *def = PyModule_GetDef(module); + assert(def); + if (def->m_size == -1) { return &global_state.module; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 9776a32755d..cc8277c5783 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -10,7 +10,6 @@ #include "pycore_object_deferred.h" // _PyObject_SetDeferredRefcount() #include "pycore_pylifecycle.h" #include "pycore_pystate.h" // _PyThreadState_SetCurrent() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttr() #include "pycore_time.h" // _PyTime_FromSeconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -19,8 +18,6 @@ # include <signal.h> // SIGINT #endif -#include "clinic/_threadmodule.c.h" - // ThreadError is just an alias to PyExc_RuntimeError #define ThreadError PyExc_RuntimeError @@ -31,6 +28,7 @@ static struct PyModuleDef thread_module; typedef struct { PyTypeObject *excepthook_type; PyTypeObject *lock_type; + PyTypeObject *rlock_type; PyTypeObject *local_type; PyTypeObject *local_dummy_type; PyTypeObject *thread_handle_type; @@ -40,6 +38,20 @@ typedef struct { struct llist_node shutdown_handles; } thread_module_state; +typedef struct { + PyObject_HEAD + PyMutex lock; +} lockobject; + +#define lockobject_CAST(op) ((lockobject *)(op)) + +typedef struct { + PyObject_HEAD + _PyRecursiveMutex lock; +} rlockobject; + +#define rlockobject_CAST(op) ((rlockobject *)(op)) + static inline thread_module_state* get_thread_state(PyObject *module) { @@ -48,6 +60,17 @@ get_thread_state(PyObject *module) return (thread_module_state *)state; } +static inline thread_module_state* +get_thread_state_by_cls(PyTypeObject *cls) +{ + // Use PyType_GetModuleByDef() to handle (R)Lock subclasses. + PyObject *module = PyType_GetModuleByDef(cls, &thread_module); + if (module == NULL) { + return NULL; + } + return get_thread_state(module); +} + #ifdef MS_WINDOWS typedef HRESULT (WINAPI *PF_GET_THREAD_DESCRIPTION)(HANDLE, PCWSTR*); @@ -59,9 +82,14 @@ static PF_SET_THREAD_DESCRIPTION pSetThreadDescription = NULL; /*[clinic input] module _thread +class _thread.lock "lockobject *" "clinic_state()->lock_type" +class _thread.RLock "rlockobject *" "clinic_state()->rlock_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=be8dbe5cc4b16df7]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c5a0f8c492a0c263]*/ +#define clinic_state() get_thread_state_by_cls(type) +#include "clinic/_threadmodule.c.h" +#undef clinic_state // _ThreadHandle type @@ -281,6 +309,12 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state) continue; } + // Keep handles for threads that have not been started yet. They are + // safe to start in the child process. + if (handle->state == THREAD_HANDLE_NOT_STARTED) { + continue; + } + // Mark all threads as done. Any attempts to join or detach the // underlying OS thread (if any) could crash. We are the only thread; // it's safe to set this non-atomically. @@ -395,7 +429,7 @@ force_done(void *arg) static int ThreadHandle_start(ThreadHandle *self, PyObject *func, PyObject *args, - PyObject *kwargs) + PyObject *kwargs, int daemon) { // Mark the handle as starting to prevent any other threads from doing so PyMutex_Lock(&self->mutex); @@ -419,7 +453,8 @@ ThreadHandle_start(ThreadHandle *self, PyObject *func, PyObject *args, goto start_failed; } PyInterpreterState *interp = _PyInterpreterState_GET(); - boot->tstate = _PyThreadState_New(interp, _PyThreadState_WHENCE_THREADING); + uint8_t whence = daemon ? _PyThreadState_WHENCE_THREADING_DAEMON : _PyThreadState_WHENCE_THREADING; + boot->tstate = _PyThreadState_New(interp, whence); if (boot->tstate == NULL) { PyMem_RawFree(boot); if (!PyErr_Occurred()) { @@ -621,13 +656,6 @@ PyThreadHandleObject_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)PyThreadHandleObject_new(type); } -static int -PyThreadHandleObject_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static void PyThreadHandleObject_dealloc(PyObject *op) { @@ -661,12 +689,12 @@ PyThreadHandleObject_join(PyObject *op, PyObject *args) PyThreadHandleObject *self = PyThreadHandleObject_CAST(op); PyObject *timeout_obj = NULL; - if (!PyArg_ParseTuple(args, "|O?:join", &timeout_obj)) { + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { return NULL; } PyTime_t timeout_ns = -1; - if (timeout_obj != NULL) { + if (timeout_obj != NULL && timeout_obj != Py_None) { if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { return NULL; @@ -684,6 +712,9 @@ PyThreadHandleObject_is_done(PyObject *op, PyObject *Py_UNUSED(dummy)) { PyThreadHandleObject *self = PyThreadHandleObject_CAST(op); if (_PyEvent_IsSet(&self->handle->thread_is_exiting)) { + if (_PyOnceFlag_CallOnce(&self->handle->once, join_thread, self->handle) == -1) { + return NULL; + } Py_RETURN_TRUE; } else { @@ -717,7 +748,7 @@ static PyType_Slot ThreadHandle_Type_slots[] = { {Py_tp_dealloc, PyThreadHandleObject_dealloc}, {Py_tp_repr, PyThreadHandleObject_repr}, {Py_tp_getset, ThreadHandle_getsetlist}, - {Py_tp_traverse, PyThreadHandleObject_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_methods, ThreadHandle_methods}, {Py_tp_new, PyThreadHandleObject_tp_new}, {0, 0} @@ -733,20 +764,6 @@ static PyType_Spec ThreadHandle_Type_spec = { /* Lock objects */ -typedef struct { - PyObject_HEAD - PyMutex lock; -} lockobject; - -#define lockobject_CAST(op) ((lockobject *)(op)) - -static int -lock_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static void lock_dealloc(PyObject *self) { @@ -759,16 +776,8 @@ lock_dealloc(PyObject *self) static int -lock_acquire_parse_args(PyObject *args, PyObject *kwds, - PyTime_t *timeout) +lock_acquire_parse_timeout(PyObject *timeout_obj, int blocking, PyTime_t *timeout) { - char *kwlist[] = {"blocking", "timeout", NULL}; - int blocking = 1; - PyObject *timeout_obj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pO:acquire", kwlist, - &blocking, &timeout_obj)) - return -1; - // XXX Use PyThread_ParseTimeoutArg(). const PyTime_t unset_timeout = _PyTime_FromSeconds(-1); @@ -803,53 +812,74 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, } return 0; } +/*[clinic input] +_thread.lock.acquire + blocking: bool = True + timeout as timeoutobj: object(py_default="-1") = NULL + +Lock the lock. + +Without argument, this blocks if the lock is already +locked (even by the same thread), waiting for another thread to release +the lock, and return True once the lock is acquired. +With an argument, this will only block if the argument is true, +and the return value reflects whether the lock is acquired. +The blocking operation is interruptible. +[clinic start generated code]*/ static PyObject * -lock_PyThread_acquire_lock(PyObject *op, PyObject *args, PyObject *kwds) +_thread_lock_acquire_impl(lockobject *self, int blocking, + PyObject *timeoutobj) +/*[clinic end generated code: output=569d6b25d508bf6f input=13e999649bc1c798]*/ { - lockobject *self = lockobject_CAST(op); - PyTime_t timeout; - if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { + + if (lock_acquire_parse_timeout(timeoutobj, blocking, &timeout) < 0) { return NULL; } - PyLockStatus r = _PyMutex_LockTimed(&self->lock, timeout, - _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); + PyLockStatus r = _PyMutex_LockTimed( + &self->lock, timeout, + _PY_LOCK_PYTHONLOCK | _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); if (r == PY_LOCK_INTR) { + assert(PyErr_Occurred()); + return NULL; + } + if (r == PY_LOCK_FAILURE && PyErr_Occurred()) { return NULL; } return PyBool_FromLong(r == PY_LOCK_ACQUIRED); } -PyDoc_STRVAR(acquire_doc, -"acquire($self, /, blocking=True, timeout=-1)\n\ ---\n\ -\n\ -Lock the lock. Without argument, this blocks if the lock is already\n\ -locked (even by the same thread), waiting for another thread to release\n\ -the lock, and return True once the lock is acquired.\n\ -With an argument, this will only block if the argument is true,\n\ -and the return value reflects whether the lock is acquired.\n\ -The blocking operation is interruptible."); +/*[clinic input] +_thread.lock.acquire_lock = _thread.lock.acquire -PyDoc_STRVAR(acquire_lock_doc, -"acquire_lock($self, /, blocking=True, timeout=-1)\n\ ---\n\ -\n\ -An obsolete synonym of acquire()."); - -PyDoc_STRVAR(enter_doc, -"__enter__($self, /)\n\ ---\n\ -\n\ -Lock the lock."); +An obsolete synonym of acquire(). +[clinic start generated code]*/ static PyObject * -lock_PyThread_release_lock(PyObject *op, PyObject *Py_UNUSED(dummy)) +_thread_lock_acquire_lock_impl(lockobject *self, int blocking, + PyObject *timeoutobj) +/*[clinic end generated code: output=ea6c87ea13b56694 input=5e65bd56327ebe85]*/ +{ + return _thread_lock_acquire_impl(self, blocking, timeoutobj); +} + +/*[clinic input] +_thread.lock.release + +Release the lock. + +Allows another thread that is blocked waiting for +the lock to acquire the lock. The lock must be in the locked state, +but it needn't be locked by the same thread that unlocks it. +[clinic start generated code]*/ + +static PyObject * +_thread_lock_release_impl(lockobject *self) +/*[clinic end generated code: output=a4ab0d75d6e9fb73 input=dfe48f962dfe99b4]*/ { - lockobject *self = lockobject_CAST(op); /* Sanity check: the lock must be locked */ if (_PyMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(ThreadError, "release unlocked lock"); @@ -859,44 +889,76 @@ lock_PyThread_release_lock(PyObject *op, PyObject *Py_UNUSED(dummy)) Py_RETURN_NONE; } -PyDoc_STRVAR(release_doc, -"release($self, /)\n\ ---\n\ -\n\ -Release the lock, allowing another thread that is blocked waiting for\n\ -the lock to acquire the lock. The lock must be in the locked state,\n\ -but it needn't be locked by the same thread that unlocks it."); +/*[clinic input] +_thread.lock.release_lock -PyDoc_STRVAR(release_lock_doc, -"release_lock($self, /)\n\ ---\n\ -\n\ -An obsolete synonym of release()."); - -PyDoc_STRVAR(lock_exit_doc, -"__exit__($self, /, *exc_info)\n\ ---\n\ -\n\ -Release the lock."); +An obsolete synonym of release(). +[clinic start generated code]*/ static PyObject * -lock_locked_lock(PyObject *op, PyObject *Py_UNUSED(dummy)) +_thread_lock_release_lock_impl(lockobject *self) +/*[clinic end generated code: output=43025044d51789bb input=74d91374fc601433]*/ +{ + return _thread_lock_release_impl(self); +} + +/*[clinic input] +_thread.lock.__enter__ + +Lock the lock. +[clinic start generated code]*/ + +static PyObject * +_thread_lock___enter___impl(lockobject *self) +/*[clinic end generated code: output=f27725de751ae064 input=8f982991608d38e7]*/ +{ + return _thread_lock_acquire_impl(self, 1, NULL); +} + +/*[clinic input] +_thread.lock.__exit__ + exc_type: object + exc_value: object + exc_tb: object + / + +Release the lock. +[clinic start generated code]*/ + +static PyObject * +_thread_lock___exit___impl(lockobject *self, PyObject *exc_type, + PyObject *exc_value, PyObject *exc_tb) +/*[clinic end generated code: output=c9e8eefa69beed07 input=c74d4abe15a6c037]*/ +{ + return _thread_lock_release_impl(self); +} + + +/*[clinic input] +_thread.lock.locked + +Return whether the lock is in the locked state. +[clinic start generated code]*/ + +static PyObject * +_thread_lock_locked_impl(lockobject *self) +/*[clinic end generated code: output=63bb94e5a9efa382 input=d8e3d64861bbce73]*/ { - lockobject *self = lockobject_CAST(op); return PyBool_FromLong(PyMutex_IsLocked(&self->lock)); } -PyDoc_STRVAR(locked_doc, -"locked($self, /)\n\ ---\n\ -\n\ -Return whether the lock is in the locked state."); +/*[clinic input] +_thread.lock.locked_lock -PyDoc_STRVAR(locked_lock_doc, -"locked_lock($self, /)\n\ ---\n\ -\n\ -An obsolete synonym of locked()."); +An obsolete synonym of locked(). +[clinic start generated code]*/ + +static PyObject * +_thread_lock_locked_lock_impl(lockobject *self) +/*[clinic end generated code: output=f747c8329e905f8e input=500b0c3592f9bf84]*/ +{ + return _thread_lock_locked_impl(self); +} static PyObject * lock_repr(PyObject *op) @@ -907,57 +969,48 @@ lock_repr(PyObject *op) } #ifdef HAVE_FORK +/*[clinic input] +_thread.lock._at_fork_reinit +[clinic start generated code]*/ + static PyObject * -lock__at_fork_reinit(PyObject *op, PyObject *Py_UNUSED(dummy)) +_thread_lock__at_fork_reinit_impl(lockobject *self) +/*[clinic end generated code: output=d8609f2d3bfa1fd5 input=a970cb76e2a0a131]*/ { - lockobject *self = lockobject_CAST(op); _PyMutex_at_fork_reinit(&self->lock); Py_RETURN_NONE; } #endif /* HAVE_FORK */ -static lockobject *newlockobject(PyObject *module); +/*[clinic input] +@classmethod +_thread.lock.__new__ as lock_new +[clinic start generated code]*/ static PyObject * -lock_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +lock_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=eab660d5a4c05c8a input=260208a4e277d250]*/ { - // convert to AC? - if (!_PyArg_NoKeywords("lock", kwargs)) { - goto error; + lockobject *self = (lockobject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; } - if (!_PyArg_CheckPositional("lock", PyTuple_GET_SIZE(args), 0, 0)) { - goto error; - } - - PyObject *module = PyType_GetModuleByDef(type, &thread_module); - assert(module != NULL); - return (PyObject *)newlockobject(module); - -error: - return NULL; + self->lock = (PyMutex){0}; + return (PyObject *)self; } static PyMethodDef lock_methods[] = { - {"acquire_lock", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, acquire_lock_doc}, - {"acquire", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, acquire_doc}, - {"release_lock", lock_PyThread_release_lock, - METH_NOARGS, release_lock_doc}, - {"release", lock_PyThread_release_lock, - METH_NOARGS, release_doc}, - {"locked_lock", lock_locked_lock, - METH_NOARGS, locked_lock_doc}, - {"locked", lock_locked_lock, - METH_NOARGS, locked_doc}, - {"__enter__", _PyCFunction_CAST(lock_PyThread_acquire_lock), - METH_VARARGS | METH_KEYWORDS, enter_doc}, - {"__exit__", lock_PyThread_release_lock, - METH_VARARGS, lock_exit_doc}, + _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF + _THREAD_LOCK_ACQUIRE_METHODDEF + _THREAD_LOCK_RELEASE_LOCK_METHODDEF + _THREAD_LOCK_RELEASE_METHODDEF + _THREAD_LOCK_LOCKED_LOCK_METHODDEF + _THREAD_LOCK_LOCKED_METHODDEF + _THREAD_LOCK___ENTER___METHODDEF + _THREAD_LOCK___EXIT___METHODDEF #ifdef HAVE_FORK - {"_at_fork_reinit", lock__at_fork_reinit, - METH_NOARGS, NULL}, + _THREAD_LOCK__AT_FORK_REINIT_METHODDEF #endif {NULL, NULL} /* sentinel */ }; @@ -982,7 +1035,7 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, {Py_tp_methods, lock_methods}, - {Py_tp_traverse, lock_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -997,21 +1050,12 @@ static PyType_Spec lock_type_spec = { /* Recursive lock objects */ -typedef struct { - PyObject_HEAD - _PyRecursiveMutex lock; -} rlockobject; - -#define rlockobject_CAST(op) ((rlockobject *)(op)) - static int -rlock_traverse(PyObject *self, visitproc visit, void *arg) +rlock_locked_impl(rlockobject *self) { - Py_VISIT(Py_TYPE(self)); - return 0; + return PyMutex_IsLocked(&self->lock.mutex); } - static void rlock_dealloc(PyObject *self) { @@ -1022,53 +1066,84 @@ rlock_dealloc(PyObject *self) Py_DECREF(tp); } +/*[clinic input] +_thread.RLock.acquire + blocking: bool = True + timeout as timeoutobj: object(py_default="-1") = NULL + +Lock the lock. + +`blocking` indicates whether we should wait +for the lock to be available or not. If `blocking` is False +and another thread holds the lock, the method will return False +immediately. If `blocking` is True and another thread holds +the lock, the method will wait for the lock to be released, +take it and then return True. +(note: the blocking operation is interruptible.) + +In all other cases, the method will return True immediately. +Precisely, if the current thread already holds the lock, its +internal counter is simply incremented. If nobody holds the lock, +the lock is taken and its internal counter initialized to 1. +[clinic start generated code]*/ static PyObject * -rlock_acquire(PyObject *op, PyObject *args, PyObject *kwds) +_thread_RLock_acquire_impl(rlockobject *self, int blocking, + PyObject *timeoutobj) +/*[clinic end generated code: output=73df5af6f67c1513 input=d55a0f5014522a8d]*/ { - rlockobject *self = rlockobject_CAST(op); PyTime_t timeout; - if (lock_acquire_parse_args(args, kwds, &timeout) < 0) { + if (lock_acquire_parse_timeout(timeoutobj, blocking, &timeout) < 0) { return NULL; } - PyLockStatus r = _PyRecursiveMutex_LockTimed(&self->lock, timeout, - _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); + PyLockStatus r = _PyRecursiveMutex_LockTimed( + &self->lock, timeout, + _PY_LOCK_PYTHONLOCK | _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); if (r == PY_LOCK_INTR) { + assert(PyErr_Occurred()); + return NULL; + } + if (r == PY_LOCK_FAILURE && PyErr_Occurred()) { return NULL; } return PyBool_FromLong(r == PY_LOCK_ACQUIRED); } -PyDoc_STRVAR(rlock_acquire_doc, -"acquire($self, /, blocking=True, timeout=-1)\n\ ---\n\ -\n\ -Lock the lock. `blocking` indicates whether we should wait\n\ -for the lock to be available or not. If `blocking` is False\n\ -and another thread holds the lock, the method will return False\n\ -immediately. If `blocking` is True and another thread holds\n\ -the lock, the method will wait for the lock to be released,\n\ -take it and then return True.\n\ -(note: the blocking operation is interruptible.)\n\ -\n\ -In all other cases, the method will return True immediately.\n\ -Precisely, if the current thread already holds the lock, its\n\ -internal counter is simply incremented. If nobody holds the lock,\n\ -the lock is taken and its internal counter initialized to 1."); +/*[clinic input] +_thread.RLock.__enter__ -PyDoc_STRVAR(rlock_enter_doc, -"__enter__($self, /)\n\ ---\n\ -\n\ -Lock the lock."); +Lock the lock. +[clinic start generated code]*/ static PyObject * -rlock_release(PyObject *op, PyObject *Py_UNUSED(dummy)) +_thread_RLock___enter___impl(rlockobject *self) +/*[clinic end generated code: output=63135898476bf89f input=33be37f459dca390]*/ +{ + return _thread_RLock_acquire_impl(self, 1, NULL); +} + +/*[clinic input] +_thread.RLock.release + +Release the lock. + +Allows another thread that is blocked waiting for +the lock to acquire the lock. The lock must be in the locked state, +and must be locked by the same thread that unlocks it; otherwise a +`RuntimeError` is raised. + +Do note that if the lock was acquire()d several times in a row by the +current thread, release() needs to be called as many times for the lock +to be available for other threads. +[clinic start generated code]*/ + +static PyObject * +_thread_RLock_release_impl(rlockobject *self) +/*[clinic end generated code: output=51f4a013c5fae2c5 input=d425daf1a5782e63]*/ { - rlockobject *self = rlockobject_CAST(op); if (_PyRecursiveMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); @@ -1077,46 +1152,55 @@ rlock_release(PyObject *op, PyObject *Py_UNUSED(dummy)) Py_RETURN_NONE; } -PyDoc_STRVAR(rlock_release_doc, -"release($self, /)\n\ ---\n\ -\n\ -Release the lock, allowing another thread that is blocked waiting for\n\ -the lock to acquire the lock. The lock must be in the locked state,\n\ -and must be locked by the same thread that unlocks it; otherwise a\n\ -`RuntimeError` is raised.\n\ -\n\ -Do note that if the lock was acquire()d several times in a row by the\n\ -current thread, release() needs to be called as many times for the lock\n\ -to be available for other threads."); +/*[clinic input] +_thread.RLock.__exit__ + exc_type: object + exc_value: object + exc_tb: object + / -PyDoc_STRVAR(rlock_exit_doc, -"__exit__($self, /, *exc_info)\n\ ---\n\ -\n\ -Release the lock."); +Release the lock. + +[clinic start generated code]*/ static PyObject * -rlock_locked(PyObject *op, PyObject *Py_UNUSED(ignored)) +_thread_RLock___exit___impl(rlockobject *self, PyObject *exc_type, + PyObject *exc_value, PyObject *exc_tb) +/*[clinic end generated code: output=79bb44d551aedeb5 input=79accf0778d91002]*/ { - rlockobject *self = rlockobject_CAST(op); - int is_locked = _PyRecursiveMutex_IsLockedByCurrentThread(&self->lock); + return _thread_RLock_release_impl(self); +} + +/*[clinic input] +_thread.RLock.locked + +Return a boolean indicating whether this object is locked right now. +[clinic start generated code]*/ + +static PyObject * +_thread_RLock_locked_impl(rlockobject *self) +/*[clinic end generated code: output=e9b6060492b3f94e input=8866d9237ba5391b]*/ +{ + int is_locked = rlock_locked_impl(self); return PyBool_FromLong(is_locked); } -PyDoc_STRVAR(rlock_locked_doc, -"locked()\n\ -\n\ -Return a boolean indicating whether this object is locked right now."); +/*[clinic input] +_thread.RLock._acquire_restore + state: object + / + +For internal use by `threading.Condition`. +[clinic start generated code]*/ static PyObject * -rlock_acquire_restore(PyObject *op, PyObject *args) +_thread_RLock__acquire_restore_impl(rlockobject *self, PyObject *state) +/*[clinic end generated code: output=beb8f2713a35e775 input=c8f2094fde059447]*/ { - rlockobject *self = rlockobject_CAST(op); PyThread_ident_t owner; Py_ssize_t count; - if (!PyArg_ParseTuple(args, "(n" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", + if (!PyArg_Parse(state, "(n" Py_PARSE_THREAD_IDENT_T "):_acquire_restore", &count, &owner)) return NULL; @@ -1126,17 +1210,17 @@ rlock_acquire_restore(PyObject *op, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(rlock_acquire_restore_doc, -"_acquire_restore($self, state, /)\n\ ---\n\ -\n\ -For internal use by `threading.Condition`."); + +/*[clinic input] +_thread.RLock._release_save + +For internal use by `threading.Condition`. +[clinic start generated code]*/ static PyObject * -rlock_release_save(PyObject *op, PyObject *Py_UNUSED(dummy)) +_thread_RLock__release_save_impl(rlockobject *self) +/*[clinic end generated code: output=d2916487315bea93 input=809d227cfc4a112c]*/ { - rlockobject *self = rlockobject_CAST(op); - if (!_PyRecursiveMutex_IsLockedByCurrentThread(&self->lock)) { PyErr_SetString(PyExc_RuntimeError, "cannot release un-acquired lock"); @@ -1150,44 +1234,46 @@ rlock_release_save(PyObject *op, PyObject *Py_UNUSED(dummy)) return Py_BuildValue("n" Py_PARSE_THREAD_IDENT_T, count, owner); } -PyDoc_STRVAR(rlock_release_save_doc, -"_release_save($self, /)\n\ ---\n\ -\n\ -For internal use by `threading.Condition`."); + +/*[clinic input] +_thread.RLock._recursion_count + +For internal use by reentrancy checks. +[clinic start generated code]*/ static PyObject * -rlock_recursion_count(PyObject *op, PyObject *Py_UNUSED(dummy)) +_thread_RLock__recursion_count_impl(rlockobject *self) +/*[clinic end generated code: output=7993fb9695ef2c4d input=7fd1834cd7a4b044]*/ { - rlockobject *self = rlockobject_CAST(op); if (_PyRecursiveMutex_IsLockedByCurrentThread(&self->lock)) { return PyLong_FromSize_t(self->lock.level + 1); } return PyLong_FromLong(0); } -PyDoc_STRVAR(rlock_recursion_count_doc, -"_recursion_count($self, /)\n\ ---\n\ -\n\ -For internal use by reentrancy checks."); + +/*[clinic input] +_thread.RLock._is_owned + +For internal use by `threading.Condition`. +[clinic start generated code]*/ static PyObject * -rlock_is_owned(PyObject *op, PyObject *Py_UNUSED(dummy)) +_thread_RLock__is_owned_impl(rlockobject *self) +/*[clinic end generated code: output=bf14268a3cabbe07 input=fba6535538deb858]*/ { - rlockobject *self = rlockobject_CAST(op); long owned = _PyRecursiveMutex_IsLockedByCurrentThread(&self->lock); return PyBool_FromLong(owned); } -PyDoc_STRVAR(rlock_is_owned_doc, -"_is_owned($self, /)\n\ ---\n\ -\n\ -For internal use by `threading.Condition`."); +/*[clinic input] +@classmethod +_thread.RLock.__new__ as rlock_new +[clinic start generated code]*/ static PyObject * -rlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +rlock_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=bb4fb1edf6818df5 input=013591361bf1ac6e]*/ { rlockobject *self = (rlockobject *) type->tp_alloc(type, 0); if (self == NULL) { @@ -1202,20 +1288,31 @@ rlock_repr(PyObject *op) { rlockobject *self = rlockobject_CAST(op); PyThread_ident_t owner = self->lock.thread; - size_t count = self->lock.level + 1; + int locked = rlock_locked_impl(self); + size_t count; + if (locked) { + count = self->lock.level + 1; + } + else { + count = 0; + } return PyUnicode_FromFormat( "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%zu at %p>", - owner ? "locked" : "unlocked", + locked ? "locked" : "unlocked", Py_TYPE(self)->tp_name, owner, count, self); } #ifdef HAVE_FORK +/*[clinic input] +_thread.RLock._at_fork_reinit +[clinic start generated code]*/ + static PyObject * -rlock__at_fork_reinit(PyObject *op, PyObject *Py_UNUSED(dummy)) +_thread_RLock__at_fork_reinit_impl(rlockobject *self) +/*[clinic end generated code: output=d77a4ce40351817c input=a3b625b026a8df4f]*/ { - rlockobject *self = rlockobject_CAST(op); self->lock = (_PyRecursiveMutex){0}; Py_RETURN_NONE; } @@ -1223,27 +1320,17 @@ rlock__at_fork_reinit(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef rlock_methods[] = { - {"acquire", _PyCFunction_CAST(rlock_acquire), - METH_VARARGS | METH_KEYWORDS, rlock_acquire_doc}, - {"release", rlock_release, - METH_NOARGS, rlock_release_doc}, - {"locked", rlock_locked, - METH_NOARGS, rlock_locked_doc}, - {"_is_owned", rlock_is_owned, - METH_NOARGS, rlock_is_owned_doc}, - {"_acquire_restore", rlock_acquire_restore, - METH_VARARGS, rlock_acquire_restore_doc}, - {"_release_save", rlock_release_save, - METH_NOARGS, rlock_release_save_doc}, - {"_recursion_count", rlock_recursion_count, - METH_NOARGS, rlock_recursion_count_doc}, - {"__enter__", _PyCFunction_CAST(rlock_acquire), - METH_VARARGS | METH_KEYWORDS, rlock_enter_doc}, - {"__exit__", rlock_release, - METH_VARARGS, rlock_exit_doc}, + _THREAD_RLOCK_ACQUIRE_METHODDEF + _THREAD_RLOCK_RELEASE_METHODDEF + _THREAD_RLOCK_LOCKED_METHODDEF + _THREAD_RLOCK__IS_OWNED_METHODDEF + _THREAD_RLOCK__ACQUIRE_RESTORE_METHODDEF + _THREAD_RLOCK__RELEASE_SAVE_METHODDEF + _THREAD_RLOCK__RECURSION_COUNT_METHODDEF + _THREAD_RLOCK___ENTER___METHODDEF + _THREAD_RLOCK___EXIT___METHODDEF #ifdef HAVE_FORK - {"_at_fork_reinit", rlock__at_fork_reinit, - METH_NOARGS, NULL}, + _THREAD_RLOCK__AT_FORK_REINIT_METHODDEF #endif {NULL, NULL} /* sentinel */ }; @@ -1255,7 +1342,7 @@ static PyType_Slot rlock_type_slots[] = { {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, - {Py_tp_traverse, rlock_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1267,20 +1354,6 @@ static PyType_Spec rlock_type_spec = { .slots = rlock_type_slots, }; -static lockobject * -newlockobject(PyObject *module) -{ - thread_module_state *state = get_thread_state(module); - - PyTypeObject *type = state->lock_type; - lockobject *self = (lockobject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - self->lock = (PyMutex){0}; - return self; -} - /* Thread-local objects */ /* Quick overview: @@ -1345,9 +1418,7 @@ static void localdummy_dealloc(PyObject *op) { localdummyobject *self = localdummyobject_CAST(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); @@ -1846,7 +1917,7 @@ do_start_new_thread(thread_module_state *state, PyObject *func, PyObject *args, add_to_shutdown_handles(state, handle); } - if (ThreadHandle_start(handle, func, args, kwargs) < 0) { + if (ThreadHandle_start(handle, func, args, kwargs, daemon) < 0) { if (!daemon) { remove_from_shutdown_handles(handle); } @@ -1929,10 +2000,10 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, PyObject *func = NULL; int daemon = 1; thread_module_state *state = get_thread_state(module); - PyObject *hobj = Py_None; + PyObject *hobj = NULL; if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, - "O|O!?p:start_joinable_thread", keywords, - &func, state->thread_handle_type, &hobj, &daemon)) { + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { return NULL; } @@ -1942,6 +2013,14 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, return NULL; } + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); + return NULL; + } + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, hobj) < 0) { return NULL; @@ -2035,7 +2114,8 @@ Note: the default signal handler for SIGINT raises ``KeyboardInterrupt``." static PyObject * thread_PyThread_allocate_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return (PyObject *) newlockobject(module); + thread_module_state *state = get_thread_state(module); + return lock_new_impl(state->lock_type); } PyDoc_STRVAR(allocate_lock_doc, @@ -2268,7 +2348,7 @@ thread_excepthook(PyObject *module, PyObject *args) PyObject *thread = PyStructSequence_GET_ITEM(args, 3); PyObject *file; - if (_PySys_GetOptionalAttr( &_Py_ID(stderr), &file) < 0) { + if (PySys_GetOptionalAttr( &_Py_ID(stderr), &file) < 0) { return NULL; } if (file == NULL || file == Py_None) { @@ -2301,7 +2381,7 @@ thread_excepthook(PyObject *module, PyObject *args) } PyDoc_STRVAR(excepthook_doc, -"_excepthook($module, (exc_type, exc_value, exc_traceback, thread), /)\n\ +"_excepthook($module, args, /)\n\ --\n\ \n\ Handle uncaught Thread.run() exception."); @@ -2349,10 +2429,8 @@ thread_shutdown(PyObject *self, PyObject *args) // Wait for the thread to finish. If we're interrupted, such // as by a ctrl-c we print the error and exit early. if (ThreadHandle_join(handle, -1) < 0) { - PyErr_FormatUnraisable("Exception ignored while joining a thread " - "in _thread._shutdown()"); ThreadHandle_decref(handle); - Py_RETURN_NONE; + return NULL; } ThreadHandle_decref(handle); @@ -2447,7 +2525,9 @@ _thread__get_name_impl(PyObject *module) } #ifdef __sun - return PyUnicode_DecodeUTF8(name, strlen(name), "surrogateescape"); + // gh-138004: Decode Solaris/Illumos (e.g. OpenIndiana) thread names + // from ASCII, since OpenIndiana only supports ASCII names. + return PyUnicode_DecodeASCII(name, strlen(name), "surrogateescape"); #else return PyUnicode_DecodeFSDefault(name); #endif @@ -2485,8 +2565,9 @@ _thread_set_name_impl(PyObject *module, PyObject *name_obj) { #ifndef MS_WINDOWS #ifdef __sun - // Solaris always uses UTF-8 - const char *encoding = "utf-8"; + // gh-138004: Encode Solaris/Illumos thread names to ASCII, + // since OpenIndiana does not support non-ASCII names. + const char *encoding = "ascii"; #else // Encode the thread name to the filesystem encoding using the "replace" // error handler @@ -2645,15 +2726,13 @@ thread_module_exec(PyObject *module) } // RLock - PyTypeObject *rlock_type = (PyTypeObject *)PyType_FromSpec(&rlock_type_spec); - if (rlock_type == NULL) { + state->rlock_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &rlock_type_spec, NULL); + if (state->rlock_type == NULL) { return -1; } - if (PyModule_AddType(module, rlock_type) < 0) { - Py_DECREF(rlock_type); + if (PyModule_AddType(module, state->rlock_type) < 0) { return -1; } - Py_DECREF(rlock_type); // Local dummy state->local_dummy_type = (PyTypeObject *)PyType_FromSpec(&local_dummy_type_spec); @@ -2740,6 +2819,7 @@ thread_module_traverse(PyObject *module, visitproc visit, void *arg) thread_module_state *state = get_thread_state(module); Py_VISIT(state->excepthook_type); Py_VISIT(state->lock_type); + Py_VISIT(state->rlock_type); Py_VISIT(state->local_type); Py_VISIT(state->local_dummy_type); Py_VISIT(state->thread_handle_type); @@ -2752,6 +2832,7 @@ thread_module_clear(PyObject *module) thread_module_state *state = get_thread_state(module); Py_CLEAR(state->excepthook_type); Py_CLEAR(state->lock_type); + Py_CLEAR(state->rlock_type); Py_CLEAR(state->local_type); Py_CLEAR(state->local_dummy_type); Py_CLEAR(state->thread_handle_type); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 77695401919..8cea7b59fe7 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -31,7 +31,6 @@ Copyright (C) 1994 Steen Lumholt. #endif #include "pycore_long.h" // _PyLong_IsNegative() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8String #ifdef MS_WINDOWS @@ -83,33 +82,10 @@ typedef int Tcl_Size; #ifdef HAVE_CREATEFILEHANDLER -/* This bit is to ensure that TCL_UNIX_FD is defined and doesn't interfere - with the proper calculation of FHANDLETYPE == TCL_UNIX_FD below. */ -#ifndef TCL_UNIX_FD -# ifdef TCL_WIN_SOCKET -# define TCL_UNIX_FD (! TCL_WIN_SOCKET) -# else -# define TCL_UNIX_FD 1 -# endif -#endif - -/* Tcl_CreateFileHandler() changed several times; these macros deal with the - messiness. In Tcl 8.0 and later, it is not available on Windows (and on - Unix, only because Jack added it back); when available on Windows, it only - applies to sockets. */ - -#ifdef MS_WINDOWS -#define FHANDLETYPE TCL_WIN_SOCKET -#else -#define FHANDLETYPE TCL_UNIX_FD -#endif - /* If Tcl can wait for a Unix file descriptor, define the EventHook() routine which uses this to handle Tcl events while the user is typing commands. */ -#if FHANDLETYPE == TCL_UNIX_FD #define WAIT_FOR_STDIN -#endif #endif /* HAVE_CREATEFILEHANDLER */ @@ -146,7 +122,7 @@ _get_tcl_lib_path(void) int stat_return_value; PyObject *prefix; - (void) _PySys_GetOptionalAttrString("base_prefix", &prefix); + (void) PySys_GetOptionalAttrString("base_prefix", &prefix); if (prefix == NULL) { return NULL; } @@ -599,8 +575,12 @@ Tkapp_New(const char *screenName, const char *className, v->interp = Tcl_CreateInterp(); v->wantobjects = wantobjects; +#if TCL_MAJOR_VERSION >= 9 + v->threaded = 1; +#else v->threaded = Tcl_GetVar2Ex(v->interp, "tcl_platform", "threaded", TCL_GLOBAL_ONLY) != NULL; +#endif v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; v->trace = NULL; @@ -907,11 +887,14 @@ static PyType_Slot PyTclObject_Type_slots[] = { }; static PyType_Spec PyTclObject_Type_spec = { - "_tkinter.Tcl_Obj", - sizeof(PyTclObject), - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - PyTclObject_Type_slots, + .name = "_tkinter.Tcl_Obj", + .basicsize = sizeof(PyTclObject), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = PyTclObject_Type_slots, }; @@ -3213,6 +3196,8 @@ _tkinter_create_impl(PyObject *module, const char *screenName, } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body _tkinter.setbusywaitinterval new_val: int @@ -3225,7 +3210,7 @@ It should be set to a divisor of the maximum time between frames in an animation static PyObject * _tkinter_setbusywaitinterval_impl(PyObject *module, int new_val) -/*[clinic end generated code: output=42bf7757dc2d0ab6 input=deca1d6f9e6dae47]*/ +/*[clinic end generated code: output=42bf7757dc2d0ab6 input=07b82a04b56625e1]*/ { if (new_val < 0) { PyErr_SetString(PyExc_ValueError, @@ -3237,6 +3222,7 @@ _tkinter_setbusywaitinterval_impl(PyObject *module, int new_val) } /*[clinic input] +@permit_long_summary _tkinter.getbusywaitinterval -> int Return the current busy-wait interval between successive calls to Tcl_DoOneEvent in a threaded Python interpreter. @@ -3244,7 +3230,7 @@ Return the current busy-wait interval between successive calls to Tcl_DoOneEvent static int _tkinter_getbusywaitinterval_impl(PyObject *module) -/*[clinic end generated code: output=23b72d552001f5c7 input=a695878d2d576a84]*/ +/*[clinic end generated code: output=23b72d552001f5c7 input=62d5b36ddab3976b]*/ { return Tkinter_busywaitinterval; } @@ -3265,11 +3251,14 @@ static PyType_Slot Tktt_Type_slots[] = { }; static PyType_Spec Tktt_Type_spec = { - "_tkinter.tktimertoken", - sizeof(TkttObject), - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - Tktt_Type_slots, + .name = "_tkinter.tktimertoken", + .basicsize = sizeof(TkttObject), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = Tktt_Type_slots, }; @@ -3321,11 +3310,14 @@ static PyType_Slot Tkapp_Type_slots[] = { static PyType_Spec Tkapp_Type_spec = { - "_tkinter.tkapp", - sizeof(TkappObject), - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - Tkapp_Type_slots, + .name = "_tkinter.tkapp", + .basicsize = sizeof(TkappObject), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = Tkapp_Type_slots, }; static PyMethodDef moduleMethods[] = @@ -3547,7 +3539,7 @@ PyInit__tkinter(void) /* This helps the dynamic loader; in Unicode aware Tcl versions it also helps Tcl find its encodings. */ - (void) _PySys_GetOptionalAttrString("executable", &uexe); + (void) PySys_GetOptionalAttrString("executable", &uexe); if (uexe && PyUnicode_Check(uexe)) { // sys.executable can be None cexe = PyUnicode_EncodeFSDefault(uexe); Py_DECREF(uexe); diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index be71fc9fc9c..21baa6ea003 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -168,6 +168,7 @@ _tracemalloc_get_traced_memory_impl(PyObject *module) } /*[clinic input] +@permit_long_summary _tracemalloc.reset_peak Set the peak size of memory blocks traced by tracemalloc to the current size. @@ -178,7 +179,7 @@ Do nothing if the tracemalloc module is not tracing memory allocations. static PyObject * _tracemalloc_reset_peak_impl(PyObject *module) -/*[clinic end generated code: output=140c2870f691dbb2 input=18afd0635066e9ce]*/ +/*[clinic end generated code: output=140c2870f691dbb2 input=4103319210f46286]*/ { _PyTraceMalloc_ResetPeak(); Py_RETURN_NONE; diff --git a/Modules/_typesmodule.c b/Modules/_typesmodule.c index a30a88196e7..df6b4c93cb8 100644 --- a/Modules/_typesmodule.c +++ b/Modules/_typesmodule.c @@ -28,6 +28,7 @@ _types_exec(PyObject *m) EXPORT_STATIC_TYPE("CoroutineType", PyCoro_Type); EXPORT_STATIC_TYPE("EllipsisType", PyEllipsis_Type); EXPORT_STATIC_TYPE("FrameType", PyFrame_Type); + EXPORT_STATIC_TYPE("FrameLocalsProxyType", PyFrameLocalsProxy_Type); EXPORT_STATIC_TYPE("FunctionType", PyFunction_Type); EXPORT_STATIC_TYPE("GeneratorType", PyGen_Type); EXPORT_STATIC_TYPE("GenericAlias", Py_GenericAliasType); diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index c5e78b1510b..c31a7e8fea5 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -78,23 +78,47 @@ py_UuidCreate(PyObject *Py_UNUSED(context), return NULL; } +static int +py_windows_has_stable_node(void) +{ + UUID uuid; + RPC_STATUS res; + Py_BEGIN_ALLOW_THREADS + res = UuidCreateSequential(&uuid); + Py_END_ALLOW_THREADS + return res == RPC_S_OK; +} #endif /* MS_WINDOWS */ static int -uuid_exec(PyObject *module) { +uuid_exec(PyObject *module) +{ +#define ADD_INT(NAME, VALUE) \ + do { \ + if (PyModule_AddIntConstant(module, (NAME), (VALUE)) < 0) { \ + return -1; \ + } \ + } while (0) + assert(sizeof(uuid_t) == 16); #if defined(MS_WINDOWS) - int has_uuid_generate_time_safe = 0; + ADD_INT("has_uuid_generate_time_safe", 0); #elif defined(HAVE_UUID_GENERATE_TIME_SAFE) - int has_uuid_generate_time_safe = 1; + ADD_INT("has_uuid_generate_time_safe", 1); #else - int has_uuid_generate_time_safe = 0; + ADD_INT("has_uuid_generate_time_safe", 0); #endif - if (PyModule_AddIntConstant(module, "has_uuid_generate_time_safe", - has_uuid_generate_time_safe) < 0) { - return -1; - } + +#if defined(MS_WINDOWS) + ADD_INT("has_stable_extractable_node", py_windows_has_stable_node()); +#elif defined(HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC) + ADD_INT("has_stable_extractable_node", 1); +#else + ADD_INT("has_stable_extractable_node", 0); +#endif + +#undef ADD_INT return 0; } diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 02817e09b93..2aebe44c709 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1573,6 +1573,7 @@ static PyObject * _winapi_GetLongPathName_impl(PyObject *module, LPCWSTR path) /*[clinic end generated code: output=c4774b080275a2d0 input=9872e211e3a4a88f]*/ { +#if defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) DWORD cchBuffer; PyObject *result = NULL; @@ -1596,6 +1597,9 @@ _winapi_GetLongPathName_impl(PyObject *module, LPCWSTR path) PyErr_SetFromWindowsErr(0); } return result; +#else + return PyUnicode_FromWideChar(path, -1); +#endif } /*[clinic input] @@ -1629,9 +1633,11 @@ _winapi_GetModuleFileName_impl(PyObject *module, HMODULE module_handle) if (! result) return PyErr_SetFromWindowsErr(GetLastError()); - return PyUnicode_FromWideChar(filename, wcslen(filename)); + return PyUnicode_FromWideChar(filename, -1); } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] _winapi.GetShortPathName @@ -1674,6 +1680,8 @@ _winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path) return result; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] _winapi.GetStdHandle -> HANDLE @@ -1912,7 +1920,6 @@ static PyObject * _winapi_PeekNamedPipe_impl(PyObject *module, HANDLE handle, int size) /*[clinic end generated code: output=d0c3e29e49d323dd input=c7aa53bfbce69d70]*/ { - PyObject *buf = NULL; DWORD nread, navail, nleft; BOOL ret; @@ -1922,20 +1929,26 @@ _winapi_PeekNamedPipe_impl(PyObject *module, HANDLE handle, int size) } if (size) { - buf = PyBytes_FromStringAndSize(NULL, size); - if (!buf) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; + } + char *buf = PyBytesWriter_GetData(writer); + Py_BEGIN_ALLOW_THREADS - ret = PeekNamedPipe(handle, PyBytes_AS_STRING(buf), size, &nread, + ret = PeekNamedPipe(handle, buf, size, &nread, &navail, &nleft); Py_END_ALLOW_THREADS if (!ret) { - Py_DECREF(buf); + PyBytesWriter_Discard(writer); return PyErr_SetExcFromWindowsErr(PyExc_OSError, 0); } - if (_PyBytes_Resize(&buf, nread)) + + PyObject *res = PyBytesWriter_FinishWithSize(writer, nread); + if (res == NULL) { return NULL; - return Py_BuildValue("NII", buf, navail, nleft); + } + return Py_BuildValue("NII", res, navail, nleft); } else { Py_BEGIN_ALLOW_THREADS @@ -2739,6 +2752,19 @@ _winapi_GetACP_impl(PyObject *module) return PyLong_FromUnsignedLong(GetACP()); } +/*[clinic input] +_winapi.GetOEMCP + +Get the current Windows ANSI code page identifier. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetOEMCP_impl(PyObject *module) +/*[clinic end generated code: output=4def5b07a8be1b3b input=e8caf4353a28e28e]*/ +{ + return PyLong_FromUnsignedLong(GetOEMCP()); +} + /*[clinic input] _winapi.GetFileType -> DWORD @@ -2883,6 +2909,7 @@ _winapi_NeedCurrentDirectoryForExePath_impl(PyObject *module, LPCWSTR exe_name) /*[clinic end generated code: output=a65ec879502b58fc input=972aac88a1ec2f00]*/ { +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) BOOL result; Py_BEGIN_ALLOW_THREADS @@ -2890,6 +2917,9 @@ _winapi_NeedCurrentDirectoryForExePath_impl(PyObject *module, Py_END_ALLOW_THREADS return result; +#else + return TRUE; +#endif } @@ -2995,6 +3025,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_WAITFORSINGLEOBJECT_METHODDEF _WINAPI_WRITEFILE_METHODDEF _WINAPI_GETACP_METHODDEF + _WINAPI_GETOEMCP_METHODDEF _WINAPI_GETFILETYPE_METHODDEF _WINAPI__MIMETYPES_READ_WINDOWS_REGISTRY_METHODDEF _WINAPI_NEEDCURRENTDIRECTORYFOREXEPATH_METHODDEF diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index a04f1412eef..0cbe10c79ab 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -10,8 +10,8 @@ See the source code for LLVMFuzzerTestOneInput for details. */ -#ifndef Py_BUILD_CORE -# define Py_BUILD_CORE 1 +#ifndef Py_BUILD_CORE_MODULE +# define Py_BUILD_CORE_MODULE 1 #endif #include <Python.h> diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index abd53436b21..b99be073db5 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -7,6 +7,7 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "datetime.h" // PyDateTime_TZInfo @@ -375,9 +376,7 @@ zoneinfo_dealloc(PyObject *obj_self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(obj_self); - } + FT_CLEAR_WEAKREFS(obj_self, self->weakreflist); if (self->trans_list_utc != NULL) { PyMem_Free(self->trans_list_utc); @@ -549,6 +548,7 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, } /*[clinic input] +@permit_long_summary zoneinfo.ZoneInfo.utcoffset cls: defining_class @@ -561,7 +561,7 @@ Retrieve a timedelta representing the UTC offset in a zone at the given datetime static PyObject * zoneinfo_ZoneInfo_utcoffset_impl(PyObject *self, PyTypeObject *cls, PyObject *dt) -/*[clinic end generated code: output=b71016c319ba1f91 input=2bb6c5364938f19c]*/ +/*[clinic end generated code: output=b71016c319ba1f91 input=8ce0dc2d179f01c5]*/ { zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); _ttinfo *tti = find_ttinfo(state, PyZoneInfo_ZoneInfo_CAST(self), dt); @@ -572,6 +572,7 @@ zoneinfo_ZoneInfo_utcoffset_impl(PyObject *self, PyTypeObject *cls, } /*[clinic input] +@permit_long_summary zoneinfo.ZoneInfo.dst cls: defining_class @@ -583,7 +584,7 @@ Retrieve a timedelta representing the amount of DST applied in a zone at the giv static PyObject * zoneinfo_ZoneInfo_dst_impl(PyObject *self, PyTypeObject *cls, PyObject *dt) -/*[clinic end generated code: output=cb6168d7723a6ae6 input=2167fb80cf8645c6]*/ +/*[clinic end generated code: output=cb6168d7723a6ae6 input=22b2abdf9388423c]*/ { zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); _ttinfo *tti = find_ttinfo(state, PyZoneInfo_ZoneInfo_CAST(self), dt); @@ -594,6 +595,7 @@ zoneinfo_ZoneInfo_dst_impl(PyObject *self, PyTypeObject *cls, PyObject *dt) } /*[clinic input] +@permit_long_summary zoneinfo.ZoneInfo.tzname cls: defining_class @@ -606,7 +608,7 @@ Retrieve a string containing the abbreviation for the time zone that applies in static PyObject * zoneinfo_ZoneInfo_tzname_impl(PyObject *self, PyTypeObject *cls, PyObject *dt) -/*[clinic end generated code: output=3b6ae6c3053ea75a input=15a59a4f92ed1f1f]*/ +/*[clinic end generated code: output=3b6ae6c3053ea75a input=0882926c4e95a1e2]*/ { zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); _ttinfo *tti = find_ttinfo(state, PyZoneInfo_ZoneInfo_CAST(self), dt); diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 4d046859a15..25ededd03a3 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -1,14 +1,16 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include <zstd.h> // ZSTD_*() +#include <zdict.h> // ZDICT_*() + /*[clinic input] module _zstd @@ -17,52 +19,91 @@ module _zstd #include "clinic/_zstdmodule.c.h" +ZstdDict * +_Py_parse_zstd_dict(const _zstd_state *state, PyObject *dict, int *ptype) +{ + if (state == NULL) { + return NULL; + } + + /* Check ZstdDict */ + if (PyObject_TypeCheck(dict, state->ZstdDict_type)) { + return (ZstdDict*)dict; + } + + /* Check (ZstdDict, type) */ + if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2 + && PyObject_TypeCheck(PyTuple_GET_ITEM(dict, 0), state->ZstdDict_type) + && PyLong_Check(PyTuple_GET_ITEM(dict, 1))) + { + int type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); + if (type == -1 && PyErr_Occurred()) { + return NULL; + } + if (type == DICT_TYPE_DIGESTED + || type == DICT_TYPE_UNDIGESTED + || type == DICT_TYPE_PREFIX) + { + *ptype = type; + return (ZstdDict*)PyTuple_GET_ITEM(dict, 0); + } + } + + /* Wrong type */ + PyErr_SetString(PyExc_TypeError, + "zstd_dict argument should be a ZstdDict object."); + return NULL; +} + /* Format error message and set ZstdError. */ void -set_zstd_error(const _zstd_state* const state, - error_type type, size_t zstd_ret) +set_zstd_error(const _zstd_state *state, error_type type, size_t zstd_ret) { - char *msg; + const char *msg; assert(ZSTD_isError(zstd_ret)); - switch (type) - { - case ERR_DECOMPRESS: - msg = "Unable to decompress zstd data: %s"; - break; - case ERR_COMPRESS: - msg = "Unable to compress zstd data: %s"; - break; - case ERR_SET_PLEDGED_INPUT_SIZE: - msg = "Unable to set pledged uncompressed content size: %s"; - break; + if (state == NULL) { + return; + } + switch (type) { + case ERR_DECOMPRESS: + msg = "Unable to decompress Zstandard data: %s"; + break; + case ERR_COMPRESS: + msg = "Unable to compress Zstandard data: %s"; + break; + case ERR_SET_PLEDGED_INPUT_SIZE: + msg = "Unable to set pledged uncompressed content size: %s"; + break; - case ERR_LOAD_D_DICT: - msg = "Unable to load zstd dictionary or prefix for decompression: %s"; - break; - case ERR_LOAD_C_DICT: - msg = "Unable to load zstd dictionary or prefix for compression: %s"; - break; + case ERR_LOAD_D_DICT: + msg = "Unable to load Zstandard dictionary or prefix for " + "decompression: %s"; + break; + case ERR_LOAD_C_DICT: + msg = "Unable to load Zstandard dictionary or prefix for " + "compression: %s"; + break; - case ERR_GET_C_BOUNDS: - msg = "Unable to get zstd compression parameter bounds: %s"; - break; - case ERR_GET_D_BOUNDS: - msg = "Unable to get zstd decompression parameter bounds: %s"; - break; - case ERR_SET_C_LEVEL: - msg = "Unable to set zstd compression level: %s"; - break; + case ERR_GET_C_BOUNDS: + msg = "Unable to get zstd compression parameter bounds: %s"; + break; + case ERR_GET_D_BOUNDS: + msg = "Unable to get zstd decompression parameter bounds: %s"; + break; + case ERR_SET_C_LEVEL: + msg = "Unable to set zstd compression level: %s"; + break; - case ERR_TRAIN_DICT: - msg = "Unable to train zstd dictionary: %s"; - break; - case ERR_FINALIZE_DICT: - msg = "Unable to finalize zstd dictionary: %s"; - break; + case ERR_TRAIN_DICT: + msg = "Unable to train the Zstandard dictionary: %s"; + break; + case ERR_FINALIZE_DICT: + msg = "Unable to finalize the Zstandard dictionary: %s"; + break; - default: - Py_UNREACHABLE(); + default: + Py_UNREACHABLE(); } PyErr_Format(state->ZstdError, msg, ZSTD_getErrorName(zstd_ret)); } @@ -72,8 +113,7 @@ typedef struct { char parameter_name[32]; } ParameterInfo; -static const ParameterInfo cp_list[] = -{ +static const ParameterInfo cp_list[] = { {ZSTD_c_compressionLevel, "compression_level"}, {ZSTD_c_windowLog, "window_log"}, {ZSTD_c_hashLog, "hash_log"}, @@ -98,22 +138,18 @@ static const ParameterInfo cp_list[] = {ZSTD_c_overlapLog, "overlap_log"} }; -static const ParameterInfo dp_list[] = -{ +static const ParameterInfo dp_list[] = { {ZSTD_d_windowLogMax, "window_log_max"} }; void -set_parameter_error(const _zstd_state* const state, int is_compress, - int key_v, int value_v) +set_parameter_error(int is_compress, int key_v, int value_v) { ParameterInfo const *list; int list_size; - char const *name; char *type; ZSTD_bounds bounds; - int i; - char pos_msg[128]; + char pos_msg[64]; if (is_compress) { list = cp_list; @@ -127,8 +163,8 @@ set_parameter_error(const _zstd_state* const state, int is_compress, } /* Find parameter's name */ - name = NULL; - for (i = 0; i < list_size; i++) { + char const *name = NULL; + for (int i = 0; i < list_size; i++) { if (key_v == (list+i)->parameter) { name = (list+i)->parameter_name; break; @@ -150,20 +186,16 @@ set_parameter_error(const _zstd_state* const state, int is_compress, bounds = ZSTD_dParam_getBounds(key_v); } if (ZSTD_isError(bounds.error)) { - PyErr_Format(state->ZstdError, - "Zstd %s parameter \"%s\" is invalid. (zstd v%s)", - type, name, ZSTD_versionString()); + PyErr_Format(PyExc_ValueError, "invalid %s parameter '%s'", + type, name); return; } /* Error message */ - PyErr_Format(state->ZstdError, - "Error when setting zstd %s parameter \"%s\", it " - "should %d <= value <= %d, provided value is %d. " - "(zstd v%s, %d-bit build)", - type, name, - bounds.lowerBound, bounds.upperBound, value_v, - ZSTD_versionString(), 8*(int)sizeof(Py_ssize_t)); + PyErr_Format(PyExc_ValueError, + "%s parameter '%s' received an illegal value %d; " + "the valid range is [%d, %d]", + type, name, value_v, bounds.lowerBound, bounds.upperBound); } static inline _zstd_state* @@ -174,9 +206,57 @@ get_zstd_state(PyObject *module) return (_zstd_state *)state; } +static Py_ssize_t +calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes, + size_t **chunk_sizes) +{ + Py_ssize_t chunks_number; + Py_ssize_t sizes_sum; + Py_ssize_t i; + + chunks_number = PyTuple_GET_SIZE(samples_sizes); + if ((size_t) chunks_number > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, + "The number of samples should be <= %u.", UINT32_MAX); + return -1; + } + + /* Prepare chunk_sizes */ + *chunk_sizes = PyMem_New(size_t, chunks_number); + if (*chunk_sizes == NULL) { + PyErr_NoMemory(); + return -1; + } + + sizes_sum = PyBytes_GET_SIZE(samples_bytes); + for (i = 0; i < chunks_number; i++) { + size_t size = PyLong_AsSize_t(PyTuple_GET_ITEM(samples_sizes, i)); + (*chunk_sizes)[i] = size; + if (size == (size_t)-1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + goto sum_error; + } + return -1; + } + if ((size_t)sizes_sum < size) { + goto sum_error; + } + sizes_sum -= size; + } + + if (sizes_sum != 0) { +sum_error: + PyErr_SetString(PyExc_ValueError, + "The samples size tuple doesn't match the " + "concatenation's size."); + return -1; + } + return chunks_number; +} + /*[clinic input] -_zstd._train_dict +_zstd.train_dict samples_bytes: PyBytesObject Concatenation of samples. @@ -186,59 +266,30 @@ _zstd._train_dict The size of the dictionary. / -Internal function, train a zstd dictionary on sample data. +Train a Zstandard dictionary on sample data. [clinic start generated code]*/ static PyObject * -_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size) -/*[clinic end generated code: output=b5b4f36347c0addd input=2dce5b57d63923e2]*/ +_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size) +/*[clinic end generated code: output=8e87fe43935e8f77 input=d20dedb21c72cb62]*/ { - // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict - // are pretty similar. We should see if we can refactor them to share that code. - Py_ssize_t chunks_number; - size_t *chunk_sizes = NULL; PyObject *dst_dict_bytes = NULL; + size_t *chunk_sizes = NULL; + Py_ssize_t chunks_number; size_t zstd_ret; - Py_ssize_t sizes_sum; - Py_ssize_t i; /* Check arguments */ if (dict_size <= 0) { - PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number."); - return NULL; - } - - chunks_number = Py_SIZE(samples_sizes); - if ((size_t) chunks_number > UINT32_MAX) { - PyErr_Format(PyExc_ValueError, - "The number of samples should be <= %u.", UINT32_MAX); - return NULL; - } - - /* Prepare chunk_sizes */ - chunk_sizes = PyMem_New(size_t, chunks_number); - if (chunk_sizes == NULL) { - PyErr_NoMemory(); - goto error; - } - - sizes_sum = 0; - for (i = 0; i < chunks_number; i++) { - PyObject *size = PyTuple_GetItem(samples_sizes, i); - chunk_sizes[i] = PyLong_AsSize_t(size); - if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "Items in samples_sizes should be an int " - "object, with a value between 0 and %u.", SIZE_MAX); - goto error; - } - sizes_sum += chunk_sizes[i]; - } - - if (sizes_sum != Py_SIZE(samples_bytes)) { PyErr_SetString(PyExc_ValueError, - "The samples size tuple doesn't match the concatenation's size."); + "dict_size argument should be positive number."); + return NULL; + } + + /* Check that the samples are valid and get their sizes */ + chunks_number = calculate_samples_stats(samples_bytes, samples_sizes, + &chunk_sizes); + if (chunks_number < 0) { goto error; } @@ -250,16 +301,16 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, /* Train the dictionary */ char *dst_dict_buffer = PyBytes_AS_STRING(dst_dict_bytes); - char *samples_buffer = PyBytes_AS_STRING(samples_bytes); + const char *samples_buffer = PyBytes_AS_STRING(samples_bytes); Py_BEGIN_ALLOW_THREADS zstd_ret = ZDICT_trainFromBuffer(dst_dict_buffer, dict_size, samples_buffer, chunk_sizes, (uint32_t)chunks_number); Py_END_ALLOW_THREADS - /* Check zstd dict error */ + /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret); goto error; } @@ -280,7 +331,7 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, } /*[clinic input] -_zstd._finalize_dict +_zstd.finalize_dict custom_dict_bytes: PyBytesObject Custom dictionary content. @@ -291,63 +342,36 @@ _zstd._finalize_dict dict_size: Py_ssize_t The size of the dictionary. compression_level: int - Optimize for a specific zstd compression level, 0 means default. + Optimize for a specific Zstandard compression level, 0 means default. / -Internal function, finalize a zstd dictionary. +Finalize a Zstandard dictionary. [clinic start generated code]*/ static PyObject * -_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, - PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size, - int compression_level) -/*[clinic end generated code: output=5dc5b520fddba37f input=8afd42a249078460]*/ +_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, + PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size, + int compression_level) +/*[clinic end generated code: output=f91821ba5ae85bda input=3c7e2480aa08fb56]*/ { Py_ssize_t chunks_number; size_t *chunk_sizes = NULL; PyObject *dst_dict_bytes = NULL; size_t zstd_ret; ZDICT_params_t params; - Py_ssize_t sizes_sum; - Py_ssize_t i; /* Check arguments */ if (dict_size <= 0) { - PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number."); - return NULL; - } - - chunks_number = Py_SIZE(samples_sizes); - if ((size_t) chunks_number > UINT32_MAX) { - PyErr_Format(PyExc_ValueError, - "The number of samples should be <= %u.", UINT32_MAX); - return NULL; - } - - /* Prepare chunk_sizes */ - chunk_sizes = PyMem_New(size_t, chunks_number); - if (chunk_sizes == NULL) { - PyErr_NoMemory(); - goto error; - } - - sizes_sum = 0; - for (i = 0; i < chunks_number; i++) { - PyObject *size = PyTuple_GetItem(samples_sizes, i); - chunk_sizes[i] = PyLong_AsSize_t(size); - if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "Items in samples_sizes should be an int " - "object, with a value between 0 and %u.", SIZE_MAX); - goto error; - } - sizes_sum += chunk_sizes[i]; - } - - if (sizes_sum != Py_SIZE(samples_bytes)) { PyErr_SetString(PyExc_ValueError, - "The samples size tuple doesn't match the concatenation's size."); + "dict_size argument should be positive number."); + return NULL; + } + + /* Check that the samples are valid and get their sizes */ + chunks_number = calculate_samples_stats(samples_bytes, samples_sizes, + &chunk_sizes); + if (chunks_number < 0) { goto error; } @@ -359,7 +383,7 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, /* Parameters */ - /* Optimize for a specific zstd compression level, 0 means default. */ + /* Optimize for a specific Zstandard compression level, 0 means default. */ params.compressionLevel = compression_level; /* Write log to stderr, 0 = none. */ params.notificationLevel = 0; @@ -370,14 +394,15 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, Py_BEGIN_ALLOW_THREADS zstd_ret = ZDICT_finalizeDictionary( PyBytes_AS_STRING(dst_dict_bytes), dict_size, - PyBytes_AS_STRING(custom_dict_bytes), Py_SIZE(custom_dict_bytes), + PyBytes_AS_STRING(custom_dict_bytes), + Py_SIZE(custom_dict_bytes), PyBytes_AS_STRING(samples_bytes), chunk_sizes, (uint32_t)chunks_number, params); Py_END_ALLOW_THREADS - /* Check zstd dict error */ + /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret); goto error; } @@ -399,26 +424,25 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, /*[clinic input] -_zstd._get_param_bounds +_zstd.get_param_bounds parameter: int The parameter to get bounds. is_compress: bool True for CompressionParameter, False for DecompressionParameter. -Internal function, get CompressionParameter/DecompressionParameter bounds. +Get CompressionParameter/DecompressionParameter bounds. [clinic start generated code]*/ static PyObject * -_zstd__get_param_bounds_impl(PyObject *module, int parameter, - int is_compress) -/*[clinic end generated code: output=9892cd822f937e79 input=884cd1a01125267d]*/ +_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress) +/*[clinic end generated code: output=4acf5a876f0620ca input=45742ef0a3531b65]*/ { ZSTD_bounds bound; if (is_compress) { bound = ZSTD_cParam_getBounds(parameter); if (ZSTD_isError(bound.error)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_GET_C_BOUNDS, bound.error); return NULL; } @@ -426,7 +450,7 @@ _zstd__get_param_bounds_impl(PyObject *module, int parameter, else { bound = ZSTD_dParam_getBounds(parameter); if (ZSTD_isError(bound.error)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_GET_D_BOUNDS, bound.error); return NULL; } @@ -436,30 +460,30 @@ _zstd__get_param_bounds_impl(PyObject *module, int parameter, } /*[clinic input] +@permit_long_summary _zstd.get_frame_size frame_buffer: Py_buffer A bytes-like object, it should start from the beginning of a frame, and contains at least one complete frame. -Get the size of a zstd frame, including frame header and 4-byte checksum if it has one. - -It will iterate all blocks' headers within a frame, to accumulate the frame size. +Get the size of a Zstandard frame, including the header and optional checksum. [clinic start generated code]*/ static PyObject * _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=a7384c2f8780f442 input=7d3ad24311893bf3]*/ +/*[clinic end generated code: output=a7384c2f8780f442 input=aac83b33045b5f43]*/ { size_t frame_size; - frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf, frame_buffer->len); + frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf, + frame_buffer->len); if (ZSTD_isError(frame_size)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); PyErr_Format(mod_state->ZstdError, - "Error when finding the compressed size of a zstd frame. " - "Make sure the frame_buffer argument starts from the " - "beginning of a frame, and its length not less than this " + "Error when finding the compressed size of a Zstandard frame. " + "Ensure the frame_buffer argument starts from the " + "beginning of a frame, and its length is not less than this " "complete frame. Zstd error message: %s.", ZSTD_getErrorName(frame_size)); return NULL; @@ -469,17 +493,17 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) } /*[clinic input] -_zstd._get_frame_info +_zstd.get_frame_info frame_buffer: Py_buffer - A bytes-like object, containing the header of a zstd frame. + A bytes-like object, containing the header of a Zstandard frame. -Internal function, get zstd frame infomation from a frame header. +Get Zstandard frame infomation from a frame header. [clinic start generated code]*/ static PyObject * -_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=5462855464ecdf81 input=67f1f8e4b7b89c4d]*/ +_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) +/*[clinic end generated code: output=56e033cf48001929 input=94b240583ae22ca5]*/ { uint64_t decompressed_size; uint32_t dict_id; @@ -491,12 +515,12 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) /* #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) */ if (decompressed_size == ZSTD_CONTENTSIZE_ERROR) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); PyErr_SetString(mod_state->ZstdError, "Error when getting information from the header of " - "a zstd frame. Make sure the frame_buffer argument " + "a Zstandard frame. Ensure the frame_buffer argument " "starts from the beginning of a frame, and its length " - "not less than the frame header (6~18 bytes)."); + "is not less than the frame header (6~18 bytes)."); return NULL; } @@ -511,325 +535,170 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) } /*[clinic input] -_zstd._set_parameter_types +@permit_long_summary +_zstd.set_parameter_types c_parameter_type: object(subclass_of='&PyType_Type') CompressionParameter IntEnum type object d_parameter_type: object(subclass_of='&PyType_Type') DecompressionParameter IntEnum type object -Internal function, set CompressionParameter/DecompressionParameter types for validity check. +Set CompressionParameter and DecompressionParameter types for validity check. [clinic start generated code]*/ static PyObject * -_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, - PyObject *d_parameter_type) -/*[clinic end generated code: output=a13d4890ccbd2873 input=4535545d903853d3]*/ +_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, + PyObject *d_parameter_type) +/*[clinic end generated code: output=f3313b1294f19502 input=0529e918dfe54863]*/ { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); - if (!PyType_Check(c_parameter_type) || !PyType_Check(d_parameter_type)) { - PyErr_SetString(PyExc_ValueError, - "The two arguments should be CompressionParameter and " - "DecompressionParameter types."); - return NULL; - } - - Py_XDECREF(mod_state->CParameter_type); Py_INCREF(c_parameter_type); - mod_state->CParameter_type = (PyTypeObject*)c_parameter_type; - - Py_XDECREF(mod_state->DParameter_type); + Py_XSETREF(mod_state->CParameter_type, (PyTypeObject*)c_parameter_type); Py_INCREF(d_parameter_type); - mod_state->DParameter_type = (PyTypeObject*)d_parameter_type; + Py_XSETREF(mod_state->DParameter_type, (PyTypeObject*)d_parameter_type); Py_RETURN_NONE; } static PyMethodDef _zstd_methods[] = { - _ZSTD__TRAIN_DICT_METHODDEF - _ZSTD__FINALIZE_DICT_METHODDEF - _ZSTD__GET_PARAM_BOUNDS_METHODDEF + _ZSTD_TRAIN_DICT_METHODDEF + _ZSTD_FINALIZE_DICT_METHODDEF + _ZSTD_GET_PARAM_BOUNDS_METHODDEF _ZSTD_GET_FRAME_SIZE_METHODDEF - _ZSTD__GET_FRAME_INFO_METHODDEF - _ZSTD__SET_PARAMETER_TYPES_METHODDEF - - {0} + _ZSTD_GET_FRAME_INFO_METHODDEF + _ZSTD_SET_PARAMETER_TYPES_METHODDEF + {NULL, NULL} }; - -#define ADD_INT_PREFIX_MACRO(module, macro) \ - do { \ - if (PyModule_AddIntConstant(module, "_" #macro, macro) < 0) { \ - return -1; \ - } \ - } while(0) - static int -add_parameters(PyObject *module) +_zstd_exec(PyObject *m) { - /* If add new parameters, please also add to cp_list/dp_list above. */ +#define ADD_TYPE(TYPE, SPEC) \ +do { \ + TYPE = (PyTypeObject *)PyType_FromModuleAndSpec(m, &(SPEC), NULL); \ + if (TYPE == NULL) { \ + return -1; \ + } \ + if (PyModule_AddType(m, TYPE) < 0) { \ + return -1; \ + } \ +} while (0) - /* Compression parameters */ - ADD_INT_PREFIX_MACRO(module, ZSTD_c_compressionLevel); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_windowLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_hashLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_chainLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_searchLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy); +#define ADD_INT_MACRO(MACRO) \ + if (PyModule_AddIntConstant((m), #MACRO, (MACRO)) < 0) { \ + return -1; \ + } - ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmMinMatch); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmBucketSizeLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashRateLog); +#define ADD_INT_CONST_TO_TYPE(TYPE, NAME, VALUE) \ +do { \ + PyObject *v = PyLong_FromLong((VALUE)); \ + if (v == NULL || PyObject_SetAttrString((PyObject *)(TYPE), \ + (NAME), v) < 0) { \ + Py_XDECREF(v); \ + return -1; \ + } \ + Py_DECREF(v); \ +} while (0) - ADD_INT_PREFIX_MACRO(module, ZSTD_c_contentSizeFlag); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_checksumFlag); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_dictIDFlag); + _zstd_state* mod_state = get_zstd_state(m); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_nbWorkers); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_jobSize); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_overlapLog); + /* Reusable objects & variables */ + mod_state->CParameter_type = NULL; + mod_state->DParameter_type = NULL; - /* Decompression parameters */ - ADD_INT_PREFIX_MACRO(module, ZSTD_d_windowLogMax); + /* Create and add heap types */ + ADD_TYPE(mod_state->ZstdDict_type, zstd_dict_type_spec); + ADD_TYPE(mod_state->ZstdCompressor_type, zstd_compressor_type_spec); + ADD_TYPE(mod_state->ZstdDecompressor_type, zstd_decompressor_type_spec); + mod_state->ZstdError = PyErr_NewExceptionWithDoc( + "compression.zstd.ZstdError", + "An error occurred in the zstd library.", + NULL, NULL); + if (mod_state->ZstdError == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject *)mod_state->ZstdError) < 0) { + return -1; + } - /* ZSTD_strategy enum */ - ADD_INT_PREFIX_MACRO(module, ZSTD_fast); - ADD_INT_PREFIX_MACRO(module, ZSTD_dfast); - ADD_INT_PREFIX_MACRO(module, ZSTD_greedy); - ADD_INT_PREFIX_MACRO(module, ZSTD_lazy); - ADD_INT_PREFIX_MACRO(module, ZSTD_lazy2); - ADD_INT_PREFIX_MACRO(module, ZSTD_btlazy2); - ADD_INT_PREFIX_MACRO(module, ZSTD_btopt); - ADD_INT_PREFIX_MACRO(module, ZSTD_btultra); - ADD_INT_PREFIX_MACRO(module, ZSTD_btultra2); + /* Add constants */ + if (PyModule_AddIntConstant(m, "zstd_version_number", + ZSTD_versionNumber()) < 0) { + return -1; + } - return 0; -} - -static inline PyObject * -get_zstd_version_info(void) -{ - uint32_t ver = ZSTD_versionNumber(); - uint32_t major, minor, release; - - major = ver / 10000; - minor = (ver / 100) % 100; - release = ver % 100; - - return Py_BuildValue("III", major, minor, release); -} - -static inline int -add_vars_to_module(PyObject *module) -{ - PyObject *obj; - - /* zstd_version, a str. */ - if (PyModule_AddStringConstant(module, "zstd_version", + if (PyModule_AddStringConstant(m, "zstd_version", ZSTD_versionString()) < 0) { return -1; } - /* zstd_version_info, a tuple. */ - obj = get_zstd_version_info(); - if (PyModule_AddObjectRef(module, "zstd_version_info", obj) < 0) { - Py_XDECREF(obj); +#if ZSTD_VERSION_NUMBER >= 10500 + if (PyModule_AddIntConstant(m, "ZSTD_CLEVEL_DEFAULT", + ZSTD_defaultCLevel()) < 0) { return -1; } - Py_DECREF(obj); - - /* Add zstd parameters */ - if (add_parameters(module) < 0) { - return -1; - } - - /* _compressionLevel_values: (default, min, max) - ZSTD_defaultCLevel() was added in zstd v1.5.0 */ - obj = Py_BuildValue("iii", -#if ZSTD_VERSION_NUMBER < 10500 - ZSTD_CLEVEL_DEFAULT, #else - ZSTD_defaultCLevel(), + ADD_INT_MACRO(ZSTD_CLEVEL_DEFAULT); #endif - ZSTD_minCLevel(), - ZSTD_maxCLevel()); - if (PyModule_AddObjectRef(module, - "_compressionLevel_values", - obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - /* _ZSTD_CStreamSizes */ - obj = Py_BuildValue("II", - (uint32_t)ZSTD_CStreamInSize(), - (uint32_t)ZSTD_CStreamOutSize()); - if (PyModule_AddObjectRef(module, "_ZSTD_CStreamSizes", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_DStreamSizes */ - obj = Py_BuildValue("II", - (uint32_t)ZSTD_DStreamInSize(), - (uint32_t)ZSTD_DStreamOutSize()); - if (PyModule_AddObjectRef(module, "_ZSTD_DStreamSizes", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_CONFIG */ - obj = Py_BuildValue("isOOO", 8*(int)sizeof(Py_ssize_t), "c", - Py_False, - Py_True, -/* User mremap output buffer */ -#if defined(HAVE_MREMAP) - Py_True -#else - Py_False -#endif - ); - if (PyModule_AddObjectRef(module, "_ZSTD_CONFIG", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - return 0; -} - -#define ADD_STR_TO_STATE_MACRO(STR) \ - do { \ - mod_state->str_##STR = PyUnicode_FromString(#STR); \ - if (mod_state->str_##STR == NULL) { \ - return -1; \ - } \ - } while(0) - -static inline int -add_type_to_module(PyObject *module, const char *name, - PyType_Spec *type_spec, PyTypeObject **dest) -{ - PyObject *temp = PyType_FromModuleAndSpec(module, type_spec, NULL); - - if (PyModule_AddObjectRef(module, name, temp) < 0) { - Py_XDECREF(temp); + if (PyModule_Add(m, "ZSTD_DStreamOutSize", + PyLong_FromSize_t(ZSTD_DStreamOutSize())) < 0) { return -1; } - *dest = (PyTypeObject*) temp; + /* Add zstd compression parameters. All should also be in cp_list. */ + ADD_INT_MACRO(ZSTD_c_compressionLevel); + ADD_INT_MACRO(ZSTD_c_windowLog); + ADD_INT_MACRO(ZSTD_c_hashLog); + ADD_INT_MACRO(ZSTD_c_chainLog); + ADD_INT_MACRO(ZSTD_c_searchLog); + ADD_INT_MACRO(ZSTD_c_minMatch); + ADD_INT_MACRO(ZSTD_c_targetLength); + ADD_INT_MACRO(ZSTD_c_strategy); - return 0; -} + ADD_INT_MACRO(ZSTD_c_enableLongDistanceMatching); + ADD_INT_MACRO(ZSTD_c_ldmHashLog); + ADD_INT_MACRO(ZSTD_c_ldmMinMatch); + ADD_INT_MACRO(ZSTD_c_ldmBucketSizeLog); + ADD_INT_MACRO(ZSTD_c_ldmHashRateLog); -static inline int -add_constant_to_type(PyTypeObject *type, const char *name, long value) -{ - PyObject *temp; + ADD_INT_MACRO(ZSTD_c_contentSizeFlag); + ADD_INT_MACRO(ZSTD_c_checksumFlag); + ADD_INT_MACRO(ZSTD_c_dictIDFlag); - temp = PyLong_FromLong(value); - if (temp == NULL) { - return -1; - } + ADD_INT_MACRO(ZSTD_c_nbWorkers); + ADD_INT_MACRO(ZSTD_c_jobSize); + ADD_INT_MACRO(ZSTD_c_overlapLog); - int rc = PyObject_SetAttrString((PyObject*) type, name, temp); - Py_DECREF(temp); - return rc; -} + /* Add zstd decompression parameters. All should also be in dp_list. */ + ADD_INT_MACRO(ZSTD_d_windowLogMax); -static int _zstd_exec(PyObject *module) { - _zstd_state* const mod_state = get_zstd_state(module); + /* Add ZSTD_strategy enum members */ + ADD_INT_MACRO(ZSTD_fast); + ADD_INT_MACRO(ZSTD_dfast); + ADD_INT_MACRO(ZSTD_greedy); + ADD_INT_MACRO(ZSTD_lazy); + ADD_INT_MACRO(ZSTD_lazy2); + ADD_INT_MACRO(ZSTD_btlazy2); + ADD_INT_MACRO(ZSTD_btopt); + ADD_INT_MACRO(ZSTD_btultra); + ADD_INT_MACRO(ZSTD_btultra2); - /* Reusable objects & variables */ - mod_state->empty_bytes = PyBytes_FromStringAndSize(NULL, 0); - if (mod_state->empty_bytes == NULL) { - return -1; - } + /* Add ZSTD_EndDirective enum members to ZstdCompressor */ + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "CONTINUE", ZSTD_e_continue); + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "FLUSH_BLOCK", ZSTD_e_flush); + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "FLUSH_FRAME", ZSTD_e_end); - mod_state->empty_readonly_memoryview = - PyMemoryView_FromMemory((char*)mod_state, 0, PyBUF_READ); - if (mod_state->empty_readonly_memoryview == NULL) { - return -1; - } + /* Make ZstdCompressor immutable (set Py_TPFLAGS_IMMUTABLETYPE) */ + PyType_Freeze(mod_state->ZstdCompressor_type); - /* Add str to module state */ - ADD_STR_TO_STATE_MACRO(read); - ADD_STR_TO_STATE_MACRO(readinto); - ADD_STR_TO_STATE_MACRO(write); - ADD_STR_TO_STATE_MACRO(flush); - - mod_state->CParameter_type = NULL; - mod_state->DParameter_type = NULL; - - /* Add variables to module */ - if (add_vars_to_module(module) < 0) { - return -1; - } - - /* ZstdError */ - mod_state->ZstdError = PyErr_NewExceptionWithDoc( - "_zstd.ZstdError", - "Call to the underlying zstd library failed.", - NULL, NULL); - if (mod_state->ZstdError == NULL) { - return -1; - } - - if (PyModule_AddObjectRef(module, "ZstdError", mod_state->ZstdError) < 0) { - Py_DECREF(mod_state->ZstdError); - return -1; - } - - /* ZstdDict */ - if (add_type_to_module(module, - "ZstdDict", - &zstddict_type_spec, - &mod_state->ZstdDict_type) < 0) { - return -1; - } - - // ZstdCompressor - if (add_type_to_module(module, - "ZstdCompressor", - &zstdcompressor_type_spec, - &mod_state->ZstdCompressor_type) < 0) { - return -1; - } - - // Add EndDirective enum to ZstdCompressor - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "CONTINUE", - ZSTD_e_continue) < 0) { - return -1; - } - - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "FLUSH_BLOCK", - ZSTD_e_flush) < 0) { - return -1; - } - - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "FLUSH_FRAME", - ZSTD_e_end) < 0) { - return -1; - } - - // ZstdDecompressor - if (add_type_to_module(module, - "ZstdDecompressor", - &ZstdDecompressor_type_spec, - &mod_state->ZstdDecompressor_type) < 0) { - return -1; - } +#undef ADD_TYPE +#undef ADD_INT_MACRO +#undef ADD_ZSTD_COMPRESSOR_INT_CONST return 0; } @@ -837,14 +706,7 @@ static int _zstd_exec(PyObject *module) { static int _zstd_traverse(PyObject *module, visitproc visit, void *arg) { - _zstd_state* const mod_state = get_zstd_state(module); - - Py_VISIT(mod_state->empty_bytes); - Py_VISIT(mod_state->empty_readonly_memoryview); - Py_VISIT(mod_state->str_read); - Py_VISIT(mod_state->str_readinto); - Py_VISIT(mod_state->str_write); - Py_VISIT(mod_state->str_flush); + _zstd_state* mod_state = get_zstd_state(module); Py_VISIT(mod_state->ZstdDict_type); Py_VISIT(mod_state->ZstdCompressor_type); @@ -861,14 +723,7 @@ _zstd_traverse(PyObject *module, visitproc visit, void *arg) static int _zstd_clear(PyObject *module) { - _zstd_state* const mod_state = get_zstd_state(module); - - Py_CLEAR(mod_state->empty_bytes); - Py_CLEAR(mod_state->empty_readonly_memoryview); - Py_CLEAR(mod_state->str_read); - Py_CLEAR(mod_state->str_readinto); - Py_CLEAR(mod_state->str_write); - Py_CLEAR(mod_state->str_flush); + _zstd_state* mod_state = get_zstd_state(module); Py_CLEAR(mod_state->ZstdDict_type); Py_CLEAR(mod_state->ZstdCompressor_type); @@ -890,20 +745,21 @@ _zstd_free(void *module) static struct PyModuleDef_Slot _zstd_slots[] = { {Py_mod_exec, _zstd_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - - {0} + {0, NULL}, }; -struct PyModuleDef _zstdmodule = { - PyModuleDef_HEAD_INIT, +static struct PyModuleDef _zstdmodule = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "_zstd", + .m_doc = "Implementation module for Zstandard compression.", .m_size = sizeof(_zstd_state), .m_slots = _zstd_slots, .m_methods = _zstd_methods, .m_traverse = _zstd_traverse, .m_clear = _zstd_clear, - .m_free = _zstd_free + .m_free = _zstd_free, }; PyMODINIT_FUNC diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 9322ee259c5..82226ff8718 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,138 +1,28 @@ -#pragma once -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* Declarations shared between different parts of the _zstd module*/ -#include "Python.h" +#ifndef ZSTD_MODULE_H +#define ZSTD_MODULE_H -#include "zstd.h" -#include "zdict.h" +#include "zstddict.h" +/* Type specs */ +extern PyType_Spec zstd_dict_type_spec; +extern PyType_Spec zstd_compressor_type_spec; +extern PyType_Spec zstd_decompressor_type_spec; -/* Forward declaration of module state */ -typedef struct _zstd_state _zstd_state; - -/* Forward reference of module def */ -extern PyModuleDef _zstdmodule; - -/* For clinic type calculations */ -static inline _zstd_state * -get_zstd_state_from_type(PyTypeObject *type) { - PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule); - if (module == NULL) { - return NULL; - } - void *state = PyModule_GetState(module); - assert(state != NULL); - return (_zstd_state *)state; -} - -extern PyType_Spec zstddict_type_spec; -extern PyType_Spec zstdcompressor_type_spec; -extern PyType_Spec ZstdDecompressor_type_spec; - -struct _zstd_state { - PyObject *empty_bytes; - PyObject *empty_readonly_memoryview; - PyObject *str_read; - PyObject *str_readinto; - PyObject *str_write; - PyObject *str_flush; - +typedef struct { + /* Module heap types. */ PyTypeObject *ZstdDict_type; PyTypeObject *ZstdCompressor_type; PyTypeObject *ZstdDecompressor_type; PyObject *ZstdError; + /* enum types set by set_parameter_types. */ PyTypeObject *CParameter_type; PyTypeObject *DParameter_type; -}; - -typedef struct { - PyObject_HEAD - - /* Reusable compress/decompress dictionary, they are created once and - can be shared by multiple threads concurrently, since its usage is - read-only. - c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ - ZSTD_DDict *d_dict; - PyObject *c_dicts; - - /* Content of the dictionary, bytes object. */ - PyObject *dict_content; - /* Dictionary id */ - uint32_t dict_id; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdDict; - -typedef struct { - PyObject_HEAD - - /* Compression context */ - ZSTD_CCtx *cctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Last mode, initialized to ZSTD_e_end */ - int last_mode; - - /* (nbWorker >= 1) ? 1 : 0 */ - int use_multithread; - - /* Compression level */ - int compression_level; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdCompressor; - -typedef struct { - PyObject_HEAD - - /* Decompression context */ - ZSTD_DCtx *dctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Unconsumed input data */ - char *input_buffer; - size_t input_buffer_size; - size_t in_begin, in_end; - - /* Unused data */ - PyObject *unused_data; - - /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ - char needs_input; - - /* For decompress(), 0 or 1. - 1 when both input and output streams are at a frame edge, means a - frame is completely decoded and fully flushed, or the decompressor - just be initialized. */ - char at_frame_edge; - - /* For ZstdDecompressor, 0 or 1. - 1 means the end of the first frame has been reached. */ - char eof; - - /* Used for fast reset above three variables */ - char _unused_char_for_align; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdDecompressor; - -typedef enum { - TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function -} decompress_type; +} _zstd_state; typedef enum { ERR_DECOMPRESS, @@ -147,7 +37,7 @@ typedef enum { ERR_SET_C_LEVEL, ERR_TRAIN_DICT, - ERR_FINALIZE_DICT + ERR_FINALIZE_DICT, } error_type; typedef enum { @@ -156,41 +46,16 @@ typedef enum { DICT_TYPE_PREFIX = 2 } dictionary_type; -static inline int -mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) { - return in->size == in->pos && out->size != out->pos; -} +extern ZstdDict * +_Py_parse_zstd_dict(const _zstd_state *state, + PyObject *dict, int *type); /* Format error message and set ZstdError. */ extern void -set_zstd_error(const _zstd_state* const state, - const error_type type, size_t zstd_ret); +set_zstd_error(const _zstd_state *state, + error_type type, size_t zstd_ret); extern void -set_parameter_error(const _zstd_state* const state, int is_compress, - int key_v, int value_v); +set_parameter_error(int is_compress, int key_v, int value_v); -static const char init_twice_msg[] = "__init__ method is called twice."; - -extern int -_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict); - -extern int -_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict); - -extern int -_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, - const char *arg_name, const char *arg_type); - -extern int -_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options); - -extern PyObject * -decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, - Py_ssize_t max_length, - Py_ssize_t initial_size, - decompress_type type); - -extern PyObject * -compress_impl(ZstdCompressor *self, Py_buffer *data, - ZSTD_EndDirective end_directive); +#endif // !ZSTD_MODULE_H diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index 319b1214833..807c72c80dd 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -1,11 +1,12 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ + +#ifndef ZSTD_BUFFER_H +#define ZSTD_BUFFER_H -#include "_zstdmodule.h" #include "pycore_blocks_output_buffer.h" +#include <zstd.h> // ZSTD_outBuffer + /* Blocks output buffer wrapper code */ /* Initialize the buffer, and grow the buffer. @@ -15,10 +16,11 @@ static inline int _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, Py_ssize_t max_length) { - /* Ensure .list was set to NULL */ - assert(buffer->list == NULL); + /* Ensure .writer was set to NULL */ + assert(buffer->writer == NULL); - Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length, &ob->dst); + Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length, + &ob->dst); if (res < 0) { return -1; } @@ -33,13 +35,12 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, Return -1 on failure */ static inline int _OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, - Py_ssize_t max_length, - Py_ssize_t init_size) + Py_ssize_t max_length, Py_ssize_t init_size) { Py_ssize_t block_size; - /* Ensure .list was set to NULL */ - assert(buffer->list == NULL); + /* Ensure .writer was set to NULL */ + assert(buffer->writer == NULL); /* Get block size */ if (0 <= max_length && max_length < init_size) { @@ -49,7 +50,8 @@ _OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, block_size = init_size; } - Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size, &ob->dst); + Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size, + &ob->dst); if (res < 0) { return -1; } @@ -102,3 +104,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob) return buffer->allocated == buffer->max_length; } + +#endif // !ZSTD_BUFFER_H diff --git a/Modules/_zstd/clinic/_zstdmodule.c.h b/Modules/_zstd/clinic/_zstdmodule.c.h index 2f8225389b7..766e1cfa776 100644 --- a/Modules/_zstd/clinic/_zstdmodule.c.h +++ b/Modules/_zstd/clinic/_zstdmodule.c.h @@ -9,11 +9,11 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() -PyDoc_STRVAR(_zstd__train_dict__doc__, -"_train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" +PyDoc_STRVAR(_zstd_train_dict__doc__, +"train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" "--\n" "\n" -"Internal function, train a zstd dictionary on sample data.\n" +"Train a Zstandard dictionary on sample data.\n" "\n" " samples_bytes\n" " Concatenation of samples.\n" @@ -22,31 +22,31 @@ PyDoc_STRVAR(_zstd__train_dict__doc__, " dict_size\n" " The size of the dictionary."); -#define _ZSTD__TRAIN_DICT_METHODDEF \ - {"_train_dict", _PyCFunction_CAST(_zstd__train_dict), METH_FASTCALL, _zstd__train_dict__doc__}, +#define _ZSTD_TRAIN_DICT_METHODDEF \ + {"train_dict", _PyCFunction_CAST(_zstd_train_dict), METH_FASTCALL, _zstd_train_dict__doc__}, static PyObject * -_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size); +_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size); static PyObject * -_zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_zstd_train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyBytesObject *samples_bytes; PyObject *samples_sizes; Py_ssize_t dict_size; - if (!_PyArg_CheckPositional("_train_dict", nargs, 3, 3)) { + if (!_PyArg_CheckPositional("train_dict", nargs, 3, 3)) { goto exit; } if (!PyBytes_Check(args[0])) { - _PyArg_BadArgument("_train_dict", "argument 1", "bytes", args[0]); + _PyArg_BadArgument("train_dict", "argument 1", "bytes", args[0]); goto exit; } samples_bytes = (PyBytesObject *)args[0]; if (!PyTuple_Check(args[1])) { - _PyArg_BadArgument("_train_dict", "argument 2", "tuple", args[1]); + _PyArg_BadArgument("train_dict", "argument 2", "tuple", args[1]); goto exit; } samples_sizes = args[1]; @@ -62,18 +62,18 @@ _zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } dict_size = ival; } - return_value = _zstd__train_dict_impl(module, samples_bytes, samples_sizes, dict_size); + return_value = _zstd_train_dict_impl(module, samples_bytes, samples_sizes, dict_size); exit: return return_value; } -PyDoc_STRVAR(_zstd__finalize_dict__doc__, -"_finalize_dict($module, custom_dict_bytes, samples_bytes,\n" -" samples_sizes, dict_size, compression_level, /)\n" +PyDoc_STRVAR(_zstd_finalize_dict__doc__, +"finalize_dict($module, custom_dict_bytes, samples_bytes, samples_sizes,\n" +" dict_size, compression_level, /)\n" "--\n" "\n" -"Internal function, finalize a zstd dictionary.\n" +"Finalize a Zstandard dictionary.\n" "\n" " custom_dict_bytes\n" " Custom dictionary content.\n" @@ -84,19 +84,19 @@ PyDoc_STRVAR(_zstd__finalize_dict__doc__, " dict_size\n" " The size of the dictionary.\n" " compression_level\n" -" Optimize for a specific zstd compression level, 0 means default."); +" Optimize for a specific Zstandard compression level, 0 means default."); -#define _ZSTD__FINALIZE_DICT_METHODDEF \ - {"_finalize_dict", _PyCFunction_CAST(_zstd__finalize_dict), METH_FASTCALL, _zstd__finalize_dict__doc__}, +#define _ZSTD_FINALIZE_DICT_METHODDEF \ + {"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__}, static PyObject * -_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, - PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size, - int compression_level); +_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, + PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size, + int compression_level); static PyObject * -_zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_zstd_finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyBytesObject *custom_dict_bytes; @@ -105,21 +105,21 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_ssize_t dict_size; int compression_level; - if (!_PyArg_CheckPositional("_finalize_dict", nargs, 5, 5)) { + if (!_PyArg_CheckPositional("finalize_dict", nargs, 5, 5)) { goto exit; } if (!PyBytes_Check(args[0])) { - _PyArg_BadArgument("_finalize_dict", "argument 1", "bytes", args[0]); + _PyArg_BadArgument("finalize_dict", "argument 1", "bytes", args[0]); goto exit; } custom_dict_bytes = (PyBytesObject *)args[0]; if (!PyBytes_Check(args[1])) { - _PyArg_BadArgument("_finalize_dict", "argument 2", "bytes", args[1]); + _PyArg_BadArgument("finalize_dict", "argument 2", "bytes", args[1]); goto exit; } samples_bytes = (PyBytesObject *)args[1]; if (!PyTuple_Check(args[2])) { - _PyArg_BadArgument("_finalize_dict", "argument 3", "tuple", args[2]); + _PyArg_BadArgument("finalize_dict", "argument 3", "tuple", args[2]); goto exit; } samples_sizes = args[2]; @@ -139,32 +139,31 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (compression_level == -1 && PyErr_Occurred()) { goto exit; } - return_value = _zstd__finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_sizes, dict_size, compression_level); + return_value = _zstd_finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_sizes, dict_size, compression_level); exit: return return_value; } -PyDoc_STRVAR(_zstd__get_param_bounds__doc__, -"_get_param_bounds($module, /, parameter, is_compress)\n" +PyDoc_STRVAR(_zstd_get_param_bounds__doc__, +"get_param_bounds($module, /, parameter, is_compress)\n" "--\n" "\n" -"Internal function, get CompressionParameter/DecompressionParameter bounds.\n" +"Get CompressionParameter/DecompressionParameter bounds.\n" "\n" " parameter\n" " The parameter to get bounds.\n" " is_compress\n" " True for CompressionParameter, False for DecompressionParameter."); -#define _ZSTD__GET_PARAM_BOUNDS_METHODDEF \ - {"_get_param_bounds", _PyCFunction_CAST(_zstd__get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd__get_param_bounds__doc__}, +#define _ZSTD_GET_PARAM_BOUNDS_METHODDEF \ + {"get_param_bounds", _PyCFunction_CAST(_zstd_get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd_get_param_bounds__doc__}, static PyObject * -_zstd__get_param_bounds_impl(PyObject *module, int parameter, - int is_compress); +_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress); static PyObject * -_zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -190,7 +189,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg static const char * const _keywords[] = {"parameter", "is_compress", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_get_param_bounds", + .fname = "get_param_bounds", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -211,7 +210,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg if (is_compress < 0) { goto exit; } - return_value = _zstd__get_param_bounds_impl(module, parameter, is_compress); + return_value = _zstd_get_param_bounds_impl(module, parameter, is_compress); exit: return return_value; @@ -221,13 +220,11 @@ PyDoc_STRVAR(_zstd_get_frame_size__doc__, "get_frame_size($module, /, frame_buffer)\n" "--\n" "\n" -"Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.\n" +"Get the size of a Zstandard frame, including the header and optional checksum.\n" "\n" " frame_buffer\n" " A bytes-like object, it should start from the beginning of a frame,\n" -" and contains at least one complete frame.\n" -"\n" -"It will iterate all blocks\' headers within a frame, to accumulate the frame size."); +" and contains at least one complete frame."); #define _ZSTD_GET_FRAME_SIZE_METHODDEF \ {"get_frame_size", _PyCFunction_CAST(_zstd_get_frame_size), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_size__doc__}, @@ -288,23 +285,23 @@ exit: return return_value; } -PyDoc_STRVAR(_zstd__get_frame_info__doc__, -"_get_frame_info($module, /, frame_buffer)\n" +PyDoc_STRVAR(_zstd_get_frame_info__doc__, +"get_frame_info($module, /, frame_buffer)\n" "--\n" "\n" -"Internal function, get zstd frame infomation from a frame header.\n" +"Get Zstandard frame infomation from a frame header.\n" "\n" " frame_buffer\n" -" A bytes-like object, containing the header of a zstd frame."); +" A bytes-like object, containing the header of a Zstandard frame."); -#define _ZSTD__GET_FRAME_INFO_METHODDEF \ - {"_get_frame_info", _PyCFunction_CAST(_zstd__get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd__get_frame_info__doc__}, +#define _ZSTD_GET_FRAME_INFO_METHODDEF \ + {"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__}, static PyObject * -_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer); +_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer); static PyObject * -_zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -330,7 +327,7 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static const char * const _keywords[] = {"frame_buffer", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_get_frame_info", + .fname = "get_frame_info", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -345,7 +342,7 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (PyObject_GetBuffer(args[0], &frame_buffer, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _zstd__get_frame_info_impl(module, &frame_buffer); + return_value = _zstd_get_frame_info_impl(module, &frame_buffer); exit: /* Cleanup for frame_buffer */ @@ -356,26 +353,26 @@ exit: return return_value; } -PyDoc_STRVAR(_zstd__set_parameter_types__doc__, -"_set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" +PyDoc_STRVAR(_zstd_set_parameter_types__doc__, +"set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" "--\n" "\n" -"Internal function, set CompressionParameter/DecompressionParameter types for validity check.\n" +"Set CompressionParameter and DecompressionParameter types for validity check.\n" "\n" " c_parameter_type\n" " CompressionParameter IntEnum type object\n" " d_parameter_type\n" " DecompressionParameter IntEnum type object"); -#define _ZSTD__SET_PARAMETER_TYPES_METHODDEF \ - {"_set_parameter_types", _PyCFunction_CAST(_zstd__set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd__set_parameter_types__doc__}, +#define _ZSTD_SET_PARAMETER_TYPES_METHODDEF \ + {"set_parameter_types", _PyCFunction_CAST(_zstd_set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd_set_parameter_types__doc__}, static PyObject * -_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, - PyObject *d_parameter_type); +_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, + PyObject *d_parameter_type); static PyObject * -_zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -401,7 +398,7 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n static const char * const _keywords[] = {"c_parameter_type", "d_parameter_type", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_set_parameter_types", + .fname = "set_parameter_types", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -415,18 +412,18 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n goto exit; } if (!PyObject_TypeCheck(args[0], &PyType_Type)) { - _PyArg_BadArgument("_set_parameter_types", "argument 'c_parameter_type'", (&PyType_Type)->tp_name, args[0]); + _PyArg_BadArgument("set_parameter_types", "argument 'c_parameter_type'", (&PyType_Type)->tp_name, args[0]); goto exit; } c_parameter_type = args[0]; if (!PyObject_TypeCheck(args[1], &PyType_Type)) { - _PyArg_BadArgument("_set_parameter_types", "argument 'd_parameter_type'", (&PyType_Type)->tp_name, args[1]); + _PyArg_BadArgument("set_parameter_types", "argument 'd_parameter_type'", (&PyType_Type)->tp_name, args[1]); goto exit; } d_parameter_type = args[1]; - return_value = _zstd__set_parameter_types_impl(module, c_parameter_type, d_parameter_type); + return_value = _zstd_set_parameter_types_impl(module, c_parameter_type, d_parameter_type); exit: return return_value; } -/*[clinic end generated code: output=189c462236a7096c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=437b084f149e68e5 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index d7909cdf89f..4f8d93fd9e8 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -8,30 +8,30 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__, +PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__, "ZstdCompressor(level=None, options=None, zstd_dict=None)\n" "--\n" "\n" "Create a compressor object for compressing data incrementally.\n" "\n" " level\n" -" The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.\n" +" The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n" " options\n" " A dict object that contains advanced compression parameters.\n" " zstd_dict\n" -" A ZstdDict object, a pre-trained zstd dictionary.\n" +" A ZstdDict object, a pre-trained Zstandard dictionary.\n" "\n" "Thread-safe at method level. For one-shot compression, use the compress()\n" "function instead."); -static int -_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, - PyObject *options, PyObject *zstd_dict); +static PyObject * +_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, + PyObject *options, PyObject *zstd_dict); -static int -_zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 3 @@ -89,7 +89,7 @@ _zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) } zstd_dict = fastargs[2]; skip_optional_pos: - return_value = _zstd_ZstdCompressor___init___impl((ZstdCompressor *)self, level, options, zstd_dict); + return_value = _zstd_ZstdCompressor_new_impl(type, level, options, zstd_dict); exit: return return_value; @@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__, " Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n" " ZstdCompressor.FLUSH_BLOCK\n" "\n" -"Flush any remaining data left in internal buffers. Since zstd data consists\n" -"of one or more independent frames, the compressor object can still be used\n" -"after this method is called."); +"Flush any remaining data left in internal buffers. Since Zstandard data\n" +"consists of one or more independent frames, the compressor object can still\n" +"be used after this method is called."); #define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \ {"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__}, @@ -252,4 +252,43 @@ skip_optional_pos: exit: return return_value; } -/*[clinic end generated code: output=ef69eab155be39f6 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_zstd_ZstdCompressor_set_pledged_input_size__doc__, +"set_pledged_input_size($self, size, /)\n" +"--\n" +"\n" +"Set the uncompressed content size to be written into the frame header.\n" +"\n" +" size\n" +" The size of the uncompressed data to be provided to the compressor.\n" +"\n" +"This method can be used to ensure the header of the frame about to be written\n" +"includes the size of the data, unless the CompressionParameter.content_size_flag\n" +"is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n" +"\n" +"It is important to ensure that the pledged data size matches the actual data\n" +"size. If they do not match the compressed output data may be corrupted and the\n" +"final chunk written may be lost."); + +#define _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF \ + {"set_pledged_input_size", (PyCFunction)_zstd_ZstdCompressor_set_pledged_input_size, METH_O, _zstd_ZstdCompressor_set_pledged_input_size__doc__}, + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self, + unsigned long long size); + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + unsigned long long size; + + if (!zstd_contentsize_converter(arg, &size)) { + goto exit; + } + return_value = _zstd_ZstdCompressor_set_pledged_input_size_impl((ZstdCompressor *)self, size); + +exit: + return return_value; +} +/*[clinic end generated code: output=c1d5c2cf06a8becd input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index 9359c637203..c6fdae74ab0 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -7,31 +7,30 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__, +PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__, "ZstdDecompressor(zstd_dict=None, options=None)\n" "--\n" "\n" "Create a decompressor object for decompressing data incrementally.\n" "\n" " zstd_dict\n" -" A ZstdDict object, a pre-trained zstd dictionary.\n" +" A ZstdDict object, a pre-trained Zstandard dictionary.\n" " options\n" " A dict object that contains advanced decompression parameters.\n" "\n" "Thread-safe at method level. For one-shot decompression, use the decompress()\n" "function instead."); -static int -_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, - PyObject *zstd_dict, PyObject *options); +static PyObject * +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, + PyObject *options); -static int -_zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 2 @@ -82,7 +81,7 @@ _zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs } options = fastargs[1]; skip_optional_pos: - return_value = _zstd_ZstdDecompressor___init___impl((ZstdDecompressor *)self, zstd_dict, options); + return_value = _zstd_ZstdDecompressor_new_impl(type, zstd_dict, options); exit: return return_value; @@ -114,13 +113,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self); static PyObject * _zstd_ZstdDecompressor_unused_data_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self); } PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, @@ -130,7 +123,7 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, "Decompress *data*, returning uncompressed bytes if possible, or b\'\' otherwise.\n" "\n" " data\n" -" A bytes-like object, zstd data to be decompressed.\n" +" A bytes-like object, Zstandard data to be decompressed.\n" " max_length\n" " Maximum size of returned data. When it is negative, the size of\n" " output buffer is unlimited. When it is nonnegative, returns at\n" @@ -227,4 +220,4 @@ exit: return return_value; } -/*[clinic end generated code: output=ae703f0465a2906d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=30c12ef047027ede input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 4e0f7b64172..166d925a542 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -6,38 +6,35 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdDict___init____doc__, -"ZstdDict(dict_content, is_raw=False)\n" +PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, +"ZstdDict(dict_content, /, *, is_raw=False)\n" "--\n" "\n" -"Represents a zstd dictionary, which can be used for compression/decompression.\n" +"Represents a Zstandard dictionary.\n" "\n" " dict_content\n" -" A bytes-like object, dictionary\'s content.\n" +" The content of a Zstandard dictionary as a bytes-like object.\n" " is_raw\n" -" This parameter is for advanced user. True means dict_content\n" -" argument is a \"raw content\" dictionary, free of any format\n" -" restriction. False means dict_content argument is an ordinary\n" -" zstd dictionary, was created by zstd functions, follow a\n" -" specified format.\n" +" If true, perform no checks on *dict_content*, useful for some\n" +" advanced cases. Otherwise, check that the content represents\n" +" a Zstandard dictionary created by the zstd library or CLI.\n" "\n" -"It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n" -"ZstdDecompressor objects."); +"The dictionary can be used for compression or decompression, and can be shared\n" +"by multiple ZstdCompressor or ZstdDecompressor objects."); -static int -_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, - int is_raw); +static PyObject * +_zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, + int is_raw); -static int -_zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -46,7 +43,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(dict_content), &_Py_ID(is_raw), }, + .ob_item = { &_Py_ID(is_raw), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -55,7 +52,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"dict_content", "is_raw", NULL}; + static const char * const _keywords[] = {"", "is_raw", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "ZstdDict", @@ -66,33 +63,68 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; - PyObject *dict_content; + Py_buffer dict_content = {NULL, NULL}; int is_raw = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - dict_content = fastargs[0]; + if (PyObject_GetBuffer(fastargs[0], &dict_content, PyBUF_SIMPLE) != 0) { + goto exit; + } if (!noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } is_raw = PyObject_IsTrue(fastargs[1]); if (is_raw < 0) { goto exit; } -skip_optional_pos: - return_value = _zstd_ZstdDict___init___impl((ZstdDict *)self, dict_content, is_raw); +skip_optional_kwonly: + return_value = _zstd_ZstdDict_new_impl(type, &dict_content, is_raw); exit: + /* Cleanup for dict_content */ + if (dict_content.obj) { + PyBuffer_Release(&dict_content); + } + return return_value; } +PyDoc_STRVAR(_zstd_ZstdDict_dict_content__doc__, +"The content of a Zstandard dictionary, as a bytes object."); +#if defined(_zstd_ZstdDict_dict_content_DOCSTR) +# undef _zstd_ZstdDict_dict_content_DOCSTR +#endif +#define _zstd_ZstdDict_dict_content_DOCSTR _zstd_ZstdDict_dict_content__doc__ + +#if !defined(_zstd_ZstdDict_dict_content_DOCSTR) +# define _zstd_ZstdDict_dict_content_DOCSTR NULL +#endif +#if defined(_ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF) +# undef _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF +# define _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF {"dict_content", (getter)_zstd_ZstdDict_dict_content_get, (setter)_zstd_ZstdDict_dict_content_set, _zstd_ZstdDict_dict_content_DOCSTR}, +#else +# define _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF {"dict_content", (getter)_zstd_ZstdDict_dict_content_get, NULL, _zstd_ZstdDict_dict_content_DOCSTR}, +#endif + +static PyObject * +_zstd_ZstdDict_dict_content_get_impl(ZstdDict *self); + +static PyObject * +_zstd_ZstdDict_dict_content_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _zstd_ZstdDict_dict_content_get_impl((ZstdDict *)self); +} + PyDoc_STRVAR(_zstd_ZstdDict_as_digested_dict__doc__, "Load as a digested dictionary to compressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict)\n" +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_digested_dict)\n" +"\n" "1. Some advanced compression parameters of compressor may be overridden\n" " by parameters of digested dictionary.\n" "2. ZstdDict has a digested dictionaries cache for each compression level.\n" @@ -120,19 +152,15 @@ _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_digested_dict_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self); } PyDoc_STRVAR(_zstd_ZstdDict_as_undigested_dict__doc__, "Load as an undigested dictionary to compressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict)\n" +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_undigested_dict)\n" +"\n" "1. The advanced compression parameters of compressor will not be overridden.\n" "2. Loading an undigested dictionary is costly. If load an undigested dictionary\n" " multiple times, consider reusing a compressor object.\n" @@ -158,23 +186,19 @@ _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_undigested_dict_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self); } PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__, "Load as a prefix to compressor/decompressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix)\n" +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_prefix)\n" +"\n" "1. Prefix is compatible with long distance matching, while dictionary is not.\n" "2. It only works for the first frame, then the compressor/decompressor will\n" " return to no prefix state.\n" -"3. When decompressing, must use the same prefix as when compressing.\""); +"3. When decompressing, must use the same prefix as when compressing."); #if defined(_zstd_ZstdDict_as_prefix_DOCSTR) # undef _zstd_ZstdDict_as_prefix_DOCSTR #endif @@ -196,12 +220,6 @@ _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); } -/*[clinic end generated code: output=59257c053f74eda7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f41d9e2e2cc2928f input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index b735981e747..f90bc9c5ab5 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -1,123 +1,196 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdCompressor class definitions */ /*[clinic input] module _zstd -class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type" +class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/ - +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked #include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_*() +typedef struct { + PyObject_HEAD + + /* Compression context */ + ZSTD_CCtx *cctx; + + /* ZstdDict object in use */ + PyObject *dict; + + /* Last mode, initialized to ZSTD_e_end */ + int last_mode; + + /* (nbWorker >= 1) ? 1 : 0 */ + int use_multithread; + + /* Compression level */ + int compression_level; + + /* Lock to protect the compression context */ + PyMutex lock; +} ZstdCompressor; #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) -int -_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, - const char *arg_name, const char* arg_type) +/*[python input] + +class zstd_contentsize_converter(CConverter): + type = 'unsigned long long' + converter = 'zstd_contentsize_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=0932c350d633c7de]*/ + + +static int +zstd_contentsize_converter(PyObject *size, unsigned long long *p) { - size_t zstd_ret; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + // None means the user indicates the size is unknown. + if (size == Py_None) { + *p = ZSTD_CONTENTSIZE_UNKNOWN; + } + else { + /* ZSTD_CONTENTSIZE_UNKNOWN is 0ULL - 1 + ZSTD_CONTENTSIZE_ERROR is 0ULL - 2 + Users should only pass values < ZSTD_CONTENTSIZE_ERROR */ + unsigned long long pledged_size = PyLong_AsUnsignedLongLong(size); + /* Here we check for (unsigned long long)-1 as a sign of an error in + PyLong_AsUnsignedLongLong */ + if (pledged_size == (unsigned long long)-1 && PyErr_Occurred()) { + *p = ZSTD_CONTENTSIZE_ERROR; + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(PyExc_ValueError, + "size argument should be a positive int less " + "than %ull", ZSTD_CONTENTSIZE_ERROR); + return 0; + } + return 0; + } + if (pledged_size >= ZSTD_CONTENTSIZE_ERROR) { + *p = ZSTD_CONTENTSIZE_ERROR; + PyErr_Format(PyExc_ValueError, + "size argument should be a positive int less " + "than %ull", ZSTD_CONTENTSIZE_ERROR); + return 0; + } + *p = pledged_size; + } + return 1; +} + +#include "clinic/compressor.c.h" + +static int +_zstd_set_c_level(ZstdCompressor *self, int level) +{ + /* Set integer compression level */ + int min_level = ZSTD_minCLevel(); + int max_level = ZSTD_maxCLevel(); + if (level < min_level || level > max_level) { + PyErr_Format(PyExc_ValueError, + "illegal compression level %d; the valid range is [%d, %d]", + level, min_level, max_level); + return -1; + } + + /* Save for generating ZSTD_CDICT */ + self->compression_level = level; + + /* Set compressionLevel to compression context */ + size_t zstd_ret = ZSTD_CCtx_setParameter( + self->cctx, ZSTD_c_compressionLevel, level); + + /* Check error */ + if (ZSTD_isError(zstd_ret)) { + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_SET_C_LEVEL, zstd_ret); + return -1; + } + return 0; +} + +static int +_zstd_set_c_parameters(ZstdCompressor *self, PyObject *options) +{ + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; } - /* Integer compression level */ - if (PyLong_Check(level_or_options)) { - int level = PyLong_AsInt(level_or_options); - if (level == -1 && PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "Compression level should be an int value between %d and %d.", - ZSTD_minCLevel(), ZSTD_maxCLevel()); + if (!PyDict_Check(options)) { + PyErr_Format(PyExc_TypeError, + "ZstdCompressor() argument 'options' must be dict, not %T", + options); + return -1; + } + + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(options, &pos, &key, &value)) { + /* Check key type */ + if (Py_TYPE(key) == mod_state->DParameter_type) { + PyErr_SetString(PyExc_TypeError, + "compression options dictionary key must not be a " + "DecompressionParameter attribute"); return -1; } - /* Save for generating ZSTD_CDICT */ - self->compression_level = level; + Py_INCREF(key); + Py_INCREF(value); + int key_v = PyLong_AsInt(key); + Py_DECREF(key); + if (key_v == -1 && PyErr_Occurred()) { + Py_DECREF(value); + return -1; + } - /* Set compressionLevel to compression context */ - zstd_ret = ZSTD_CCtx_setParameter(self->cctx, - ZSTD_c_compressionLevel, - level); + int value_v = PyLong_AsInt(value); + Py_DECREF(value); + if (value_v == -1 && PyErr_Occurred()) { + return -1; + } + + if (key_v == ZSTD_c_compressionLevel) { + if (_zstd_set_c_level(self, value_v) < 0) { + return -1; + } + continue; + } + if (key_v == ZSTD_c_nbWorkers) { + /* From the zstd library docs: + 1. When nbWorkers >= 1, triggers asynchronous mode when + used with ZSTD_compressStream2(). + 2, Default value is `0`, aka "single-threaded mode" : no + worker is spawned, compression is performed inside + caller's thread, all invocations are blocking. */ + if (value_v != 0) { + self->use_multithread = 1; + } + } + + /* Set parameter to compression context */ + size_t zstd_ret = ZSTD_CCtx_setParameter(self->cctx, key_v, value_v); /* Check error */ if (ZSTD_isError(zstd_ret)) { - set_zstd_error(mod_state, ERR_SET_C_LEVEL, zstd_ret); + set_parameter_error(1, key_v, value_v); return -1; } - return 0; } - - /* Options dict */ - if (PyDict_Check(level_or_options)) { - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(level_or_options, &pos, &key, &value)) { - /* Check key type */ - if (Py_TYPE(key) == mod_state->DParameter_type) { - PyErr_SetString(PyExc_TypeError, - "Key of compression option dict should " - "NOT be DecompressionParameter."); - return -1; - } - - int key_v = PyLong_AsInt(key); - if (key_v == -1 && PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Key of options dict should be a CompressionParameter attribute."); - return -1; - } - - // TODO(emmatyping): check bounds when there is a value error here for better - // error message? - int value_v = PyLong_AsInt(value); - if (value_v == -1 && PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Value of option dict should be an int."); - return -1; - } - - if (key_v == ZSTD_c_compressionLevel) { - /* Save for generating ZSTD_CDICT */ - self->compression_level = value_v; - } - else if (key_v == ZSTD_c_nbWorkers) { - /* From zstd library doc: - 1. When nbWorkers >= 1, triggers asynchronous mode when - used with ZSTD_compressStream2(). - 2, Default value is `0`, aka "single-threaded mode" : no - worker is spawned, compression is performed inside - caller's thread, all invocations are blocking. */ - if (value_v != 0) { - self->use_multithread = 1; - } - } - - /* Set parameter to compression context */ - zstd_ret = ZSTD_CCtx_setParameter(self->cctx, key_v, value_v); - if (ZSTD_isError(zstd_ret)) { - set_parameter_error(mod_state, 1, key_v, value_v); - return -1; - } - } - return 0; - } - PyErr_Format(PyExc_TypeError, "Invalid type for %s. Expected %s", arg_name, arg_type); - return -1; + return 0; } static void @@ -130,12 +203,12 @@ capsule_free_cdict(PyObject *capsule) ZSTD_CDict * _get_CDict(ZstdDict *self, int compressionLevel) { + assert(PyMutex_IsLocked(&self->lock)); PyObject *level = NULL; - PyObject *capsule; + PyObject *capsule = NULL; ZSTD_CDict *cdict; + int ret; - // TODO(emmatyping): refactor critical section code into a lock_held function - Py_BEGIN_CRITICAL_SECTION(self); /* int level object */ level = PyLong_FromLong(compressionLevel); @@ -144,27 +217,23 @@ _get_CDict(ZstdDict *self, int compressionLevel) } /* Get PyCapsule object from self->c_dicts */ - capsule = PyDict_GetItemWithError(self->c_dicts, level); + ret = PyDict_GetItemRef(self->c_dicts, level, &capsule); + if (ret < 0) { + goto error; + } if (capsule == NULL) { - if (PyErr_Occurred()) { - goto error; - } - /* Create ZSTD_CDict instance */ - char *dict_buffer = PyBytes_AS_STRING(self->dict_content); - Py_ssize_t dict_len = Py_SIZE(self->dict_content); Py_BEGIN_ALLOW_THREADS - cdict = ZSTD_createCDict(dict_buffer, - dict_len, + cdict = ZSTD_createCDict(self->dict_buffer, self->dict_len, compressionLevel); Py_END_ALLOW_THREADS if (cdict == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, - "Failed to create ZSTD_CDict instance from zstd " - "dictionary content. Maybe the content is corrupted."); + "Failed to create a ZSTD_CDict instance from " + "Zstandard dictionary content."); } goto error; } @@ -177,11 +246,10 @@ _get_CDict(ZstdDict *self, int compressionLevel) } /* Add PyCapsule object to self->c_dicts */ - if (PyDict_SetItem(self->c_dicts, level, capsule) < 0) { - Py_DECREF(capsule); + ret = PyDict_SetItem(self->c_dicts, level, capsule); + if (ret < 0) { goto error; } - Py_DECREF(capsule); } else { /* ZSTD_CDict instance already exists */ @@ -193,61 +261,15 @@ _get_CDict(ZstdDict *self, int compressionLevel) cdict = NULL; success: Py_XDECREF(level); - Py_END_CRITICAL_SECTION(); + Py_XDECREF(capsule); return cdict; } -int -_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { - +static int +_zstd_load_impl(ZstdCompressor *self, ZstdDict *zd, + _zstd_state *mod_state, int type) +{ size_t zstd_ret; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return -1; - } - ZstdDict *zd; - int type, ret; - - /* Check ZstdDict */ - ret = PyObject_IsInstance(dict, (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* When compressing, use undigested dictionary by default. */ - zd = (ZstdDict*)dict; - type = DICT_TYPE_UNDIGESTED; - goto load; - } - - /* Check (ZstdDict, type) */ - if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2) { - /* Check ZstdDict */ - ret = PyObject_IsInstance(PyTuple_GET_ITEM(dict, 0), - (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* type == -1 may indicate an error. */ - type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); - if (type == DICT_TYPE_DIGESTED || - type == DICT_TYPE_UNDIGESTED || - type == DICT_TYPE_PREFIX) - { - assert(type >= 0); - zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); - goto load; - } - } - } - - /* Wrong type */ - PyErr_SetString(PyExc_TypeError, - "zstd_dict argument should be ZstdDict object."); - return -1; - -load: if (type == DICT_TYPE_DIGESTED) { /* Get ZSTD_CDict */ ZSTD_CDict *c_dict = _get_CDict(zd, self->compression_level); @@ -256,28 +278,18 @@ _PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { } /* Reference a prepared dictionary. It overrides some compression context's parameters. */ - Py_BEGIN_CRITICAL_SECTION(self); zstd_ret = ZSTD_CCtx_refCDict(self->cctx, c_dict); - Py_END_CRITICAL_SECTION(); } else if (type == DICT_TYPE_UNDIGESTED) { /* Load a dictionary. It doesn't override compression context's parameters. */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_CCtx_loadDictionary( - self->cctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); + zstd_ret = ZSTD_CCtx_loadDictionary(self->cctx, zd->dict_buffer, + zd->dict_len); } else if (type == DICT_TYPE_PREFIX) { /* Load a prefix */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_CCtx_refPrefix( - self->cctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); + zstd_ret = ZSTD_CCtx_refPrefix(self->cctx, zd->dict_buffer, + zd->dict_len); } else { Py_UNREACHABLE(); @@ -291,28 +303,57 @@ _PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { return 0; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/compressor.c.h" -#undef clinic_state +static int +_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) +{ + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + /* When compressing, use undigested dictionary by default. */ + int type = DICT_TYPE_UNDIGESTED; + ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type); + if (zd == NULL) { + return -1; + } + int ret; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; +} + +/*[clinic input] +@classmethod +_zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new + level: object = None + The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT. + options: object = None + A dict object that contains advanced compression parameters. + zstd_dict: object = None + A ZstdDict object, a pre-trained Zstandard dictionary. + +Create a compressor object for compressing data incrementally. + +Thread-safe at method level. For one-shot compression, use the compress() +function instead. +[clinic start generated code]*/ static PyObject * -_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) +_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, + PyObject *options, PyObject *zstd_dict) +/*[clinic end generated code: output=cdef61eafecac3d7 input=92de0211ae20ffdc]*/ { - ZstdCompressor *self; - self = PyObject_GC_New(ZstdCompressor, type); + ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { goto error; } - self->inited = 0; - self->dict = NULL; self->use_multithread = 0; - + self->dict = NULL; + self->lock = (PyMutex){0}; /* Compression context */ self->cctx = ZSTD_createCCtx(); if (self->cctx == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, "Unable to create ZSTD_CCtx instance."); @@ -323,12 +364,56 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject /* Last mode */ self->last_mode = ZSTD_e_end; + if (level != Py_None && options != Py_None) { + PyErr_SetString(PyExc_TypeError, + "Only one of level or options should be used."); + goto error; + } + + /* Set compression level */ + if (level != Py_None) { + if (!PyLong_Check(level)) { + PyErr_SetString(PyExc_TypeError, + "invalid type for level, expected int"); + goto error; + } + int level_v = PyLong_AsInt(level); + if (level_v == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(PyExc_ValueError, + "illegal compression level; the valid range is [%d, %d]", + ZSTD_minCLevel(), ZSTD_maxCLevel()); + } + goto error; + } + if (_zstd_set_c_level(self, level_v) < 0) { + goto error; + } + } + + /* Set options dictionary */ + if (options != Py_None) { + if (_zstd_set_c_parameters(self, options) < 0) { + goto error; + } + } + + /* Load Zstandard dictionary to compression context */ + if (zstd_dict != Py_None) { + if (_zstd_load_c_dict(self, zstd_dict) < 0) { + goto error; + } + Py_INCREF(zstd_dict); + self->dict = zstd_dict; + } + + // We can only start GC tracking once self->dict is set. + PyObject_GC_Track(self); + return (PyObject*)self; error: - if (self != NULL) { - PyObject_GC_Del(self); - } + Py_XDECREF(self); return NULL; } @@ -340,7 +425,11 @@ ZstdCompressor_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free compression context */ - ZSTD_freeCCtx(self->cctx); + if (self->cctx) { + ZSTD_freeCCtx(self->cctx); + } + + assert(!PyMutex_IsLocked(&self->lock)); /* Py_XDECREF the dict after free the compression context */ Py_CLEAR(self->dict); @@ -350,75 +439,14 @@ ZstdCompressor_dealloc(PyObject *ob) Py_DECREF(tp); } -/*[clinic input] -_zstd.ZstdCompressor.__init__ - - level: object = None - The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT. - options: object = None - A dict object that contains advanced compression parameters. - zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. - -Create a compressor object for compressing data incrementally. - -Thread-safe at method level. For one-shot compression, use the compress() -function instead. -[clinic start generated code]*/ - -static int -_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, - PyObject *options, PyObject *zstd_dict) -/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/ -{ - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); - return -1; - } - self->inited = 1; - - if (level != Py_None && options != Py_None) { - PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); - return -1; - } - - /* Set compressLevel/options to compression context */ - if (level != Py_None) { - if (_PyZstd_set_c_parameters(self, level, "level", "int") < 0) { - return -1; - } - } - - if (options != Py_None) { - if (_PyZstd_set_c_parameters(self, options, "options", "dict") < 0) { - return -1; - } - } - - /* Load dictionary to compression context */ - if (zstd_dict != Py_None) { - if (_PyZstd_load_c_dict(self, zstd_dict) < 0) { - return -1; - } - - /* Py_INCREF the dict */ - Py_INCREF(zstd_dict); - self->dict = zstd_dict; - } - - // We can only start tracking self with the GC once self->dict is set. - PyObject_GC_Track(self); - return 0; -} - -PyObject * -compress_impl(ZstdCompressor *self, Py_buffer *data, - ZSTD_EndDirective end_directive) +static PyObject * +compress_lock_held(ZstdCompressor *self, Py_buffer *data, + ZSTD_EndDirective end_directive) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; ZSTD_outBuffer out; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; size_t zstd_ret; PyObject *ret; @@ -442,12 +470,12 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, } if (_OutputBuffer_InitWithSize(&buffer, &out, -1, - (Py_ssize_t) output_buffer_size) < 0) { + (Py_ssize_t) output_buffer_size) < 0) { goto error; } - /* zstd stream compress */ + /* Zstandard stream compress */ while (1) { Py_BEGIN_ALLOW_THREADS zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, end_directive); @@ -455,10 +483,8 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); - } + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); goto error; } @@ -487,12 +513,21 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, return NULL; } -static PyObject * -compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) +#ifndef NDEBUG +static inline int +mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) { + return in->size == in->pos && out->size != out->pos; +} +#endif + +static PyObject * +compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data) +{ + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; ZSTD_outBuffer out; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; size_t zstd_ret; PyObject *ret; @@ -505,24 +540,25 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) goto error; } - /* zstd stream compress */ + /* Zstandard stream compress */ while (1) { Py_BEGIN_ALLOW_THREADS do { - zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, ZSTD_e_continue); - } while (out.pos != out.size && in.pos != in.size && !ZSTD_isError(zstd_ret)); + zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, + ZSTD_e_continue); + } while (out.pos != out.size + && in.pos != in.size + && !ZSTD_isError(zstd_ret)); Py_END_ALLOW_THREADS /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); - } + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); goto error; } - /* Like compress_impl(), output as much as possible. */ + /* Like compress_lock_held(), output as much as possible. */ if (out.pos == out.size) { if (_OutputBuffer_Grow(&buffer, &out) < 0) { goto error; @@ -547,6 +583,7 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) } /*[clinic input] +@permit_long_docstring_body _zstd.ZstdCompressor.compress data: Py_buffer @@ -564,7 +601,7 @@ the compression process. static PyObject * _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, int mode) -/*[clinic end generated code: output=ed7982d1cf7b4f98 input=ac2c21d180f579ea]*/ +/*[clinic end generated code: output=ed7982d1cf7b4f98 input=6018ed6cc729cea6]*/ { PyObject *ret; @@ -581,14 +618,14 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, } /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); + PyMutex_Lock(&self->lock); /* Compress */ if (self->use_multithread && mode == ZSTD_e_continue) { - ret = compress_mt_continue_impl(self, data); + ret = compress_mt_continue_lock_held(self, data); } else { - ret = compress_impl(self, data, mode); + ret = compress_lock_held(self, data, mode); } if (ret) { @@ -600,12 +637,13 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, /* Resetting cctx's session never fail */ ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only); } - Py_END_CRITICAL_SECTION(); + PyMutex_Unlock(&self->lock); return ret; } /*[clinic input] +@permit_long_docstring_body _zstd.ZstdCompressor.flush mode: int(c_default="ZSTD_e_end") = ZstdCompressor.FLUSH_FRAME @@ -614,14 +652,14 @@ _zstd.ZstdCompressor.flush Finish the compression process. -Flush any remaining data left in internal buffers. Since zstd data consists -of one or more independent frames, the compressor object can still be used -after this method is called. +Flush any remaining data left in internal buffers. Since Zstandard data +consists of one or more independent frames, the compressor object can still +be used after this method is called. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) -/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a766870301932b85]*/ +/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a9871ec742d79003]*/ { PyObject *ret; @@ -635,8 +673,9 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) } /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); - ret = compress_impl(self, NULL, mode); + PyMutex_Lock(&self->lock); + + ret = compress_lock_held(self, NULL, mode); if (ret) { self->last_mode = mode; @@ -647,28 +686,80 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) /* Resetting cctx's session never fail */ ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only); } - Py_END_CRITICAL_SECTION(); + PyMutex_Unlock(&self->lock); return ret; } + +/*[clinic input] +@permit_long_docstring_body +_zstd.ZstdCompressor.set_pledged_input_size + + size: zstd_contentsize + The size of the uncompressed data to be provided to the compressor. + / + +Set the uncompressed content size to be written into the frame header. + +This method can be used to ensure the header of the frame about to be written +includes the size of the data, unless the CompressionParameter.content_size_flag +is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised. + +It is important to ensure that the pledged data size matches the actual data +size. If they do not match the compressed output data may be corrupted and the +final chunk written may be lost. +[clinic start generated code]*/ + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self, + unsigned long long size) +/*[clinic end generated code: output=3a09e55cc0e3b4f9 input=b4c87bcbd5ce6111]*/ +{ + // Error occurred while converting argument, should be unreachable + assert(size != ZSTD_CONTENTSIZE_ERROR); + + /* Thread-safe code */ + PyMutex_Lock(&self->lock); + + /* Check the current mode */ + if (self->last_mode != ZSTD_e_end) { + PyErr_SetString(PyExc_ValueError, + "set_pledged_input_size() method must be called " + "when last_mode == FLUSH_FRAME"); + PyMutex_Unlock(&self->lock); + return NULL; + } + + /* Set pledged content size */ + size_t zstd_ret = ZSTD_CCtx_setPledgedSrcSize(self->cctx, size); + PyMutex_Unlock(&self->lock); + if (ZSTD_isError(zstd_ret)) { + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_SET_PLEDGED_INPUT_SIZE, zstd_ret); + return NULL; + } + + Py_RETURN_NONE; +} + static PyMethodDef ZstdCompressor_methods[] = { _ZSTD_ZSTDCOMPRESSOR_COMPRESS_METHODDEF _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF - - {0} + _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF + {NULL, NULL} }; PyDoc_STRVAR(ZstdCompressor_last_mode_doc, "The last mode used to this compressor object, its value can be .CONTINUE,\n" ".FLUSH_BLOCK, .FLUSH_FRAME. Initialized to .FLUSH_FRAME.\n\n" -"It can be used to get the current state of a compressor, such as, data flushed,\n" -"a frame ended."); +"It can be used to get the current state of a compressor, such as, data\n" +"flushed, or a frame ended."); static PyMemberDef ZstdCompressor_members[] = { {"last_mode", Py_T_INT, offsetof(ZstdCompressor, last_mode), - Py_READONLY, ZstdCompressor_last_mode_doc}, - {0} + Py_READONLY, ZstdCompressor_last_mode_doc}, + {NULL} }; static int @@ -690,18 +781,20 @@ ZstdCompressor_clear(PyObject *ob) static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, {Py_tp_dealloc, ZstdCompressor_dealloc}, - {Py_tp_init, _zstd_ZstdCompressor___init__}, {Py_tp_methods, ZstdCompressor_methods}, {Py_tp_members, ZstdCompressor_members}, - {Py_tp_doc, (char*)_zstd_ZstdCompressor___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, {Py_tp_clear, ZstdCompressor_clear}, - {0} + {0, 0} }; -PyType_Spec zstdcompressor_type_spec = { - .name = "_zstd.ZstdCompressor", +PyType_Spec zstd_compressor_type_spec = { + .name = "compression.zstd.ZstdCompressor", .basicsize = sizeof(ZstdCompressor), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + // Py_TPFLAGS_IMMUTABLETYPE is not used here as several + // associated constants need to be added to the type. + // PyType_Freeze is called later to set the flag. + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .slots = zstdcompressor_slots, }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index a4be180c008..13071b7a2ba 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -1,176 +1,142 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdDecompressor class definition */ /*[clinic input] module _zstd -class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompressor_type" +class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4e6eae327c0c0c76]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" - #include "buffer.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked +#include <stdbool.h> // bool #include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Decompression context */ + ZSTD_DCtx *dctx; + + /* ZstdDict object in use */ + PyObject *dict; + + /* Unconsumed input data */ + char *input_buffer; + size_t input_buffer_size; + size_t in_begin, in_end; + + /* Unused data */ + PyObject *unused_data; + + /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ + bool needs_input; + + /* For ZstdDecompressor, 0 or 1. + 1 means the end of the first frame has been reached. */ + bool eof; + + /* Lock to protect the decompression context */ + PyMutex lock; +} ZstdDecompressor; #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) +#include "clinic/decompressor.c.h" + static inline ZSTD_DDict * _get_DDict(ZstdDict *self) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_DDict *ret; - /* Already created */ - if (self->d_dict != NULL) { - return self->d_dict; - } - - Py_BEGIN_CRITICAL_SECTION(self); if (self->d_dict == NULL) { /* Create ZSTD_DDict instance from dictionary content */ - char *dict_buffer = PyBytes_AS_STRING(self->dict_content); - Py_ssize_t dict_len = Py_SIZE(self->dict_content); Py_BEGIN_ALLOW_THREADS - self->d_dict = ZSTD_createDDict(dict_buffer, - dict_len); + ret = ZSTD_createDDict(self->dict_buffer, self->dict_len); Py_END_ALLOW_THREADS + self->d_dict = ret; if (self->d_dict == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, - "Failed to create ZSTD_DDict instance from zstd " - "dictionary content. Maybe the content is corrupted."); + "Failed to create a ZSTD_DDict instance from " + "Zstandard dictionary content."); } } } - /* Don't lose any exception */ - ret = self->d_dict; - Py_END_CRITICAL_SECTION(); - - return ret; + return self->d_dict; } -/* Set decompression parameters to decompression context */ -int -_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) +static int +_zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) { - size_t zstd_ret; - PyObject *key, *value; - Py_ssize_t pos; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; } if (!PyDict_Check(options)) { - PyErr_SetString(PyExc_TypeError, - "options argument should be dict object."); + PyErr_Format(PyExc_TypeError, + "ZstdDecompressor() argument 'options' must be dict, not %T", + options); return -1; } - pos = 0; + Py_ssize_t pos = 0; + PyObject *key, *value; while (PyDict_Next(options, &pos, &key, &value)) { /* Check key type */ if (Py_TYPE(key) == mod_state->CParameter_type) { PyErr_SetString(PyExc_TypeError, - "Key of decompression options dict should " - "NOT be CompressionParameter."); + "compression options dictionary key must not be a " + "CompressionParameter attribute"); return -1; } - /* Both key & value should be 32-bit signed int */ + Py_INCREF(key); + Py_INCREF(value); int key_v = PyLong_AsInt(key); + Py_DECREF(key); if (key_v == -1 && PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Key of options dict should be a DecompressionParameter attribute."); return -1; } - // TODO(emmatyping): check bounds when there is a value error here for better - // error message? int value_v = PyLong_AsInt(value); + Py_DECREF(value); if (value_v == -1 && PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Value of options dict should be an int."); return -1; } /* Set parameter to compression context */ - Py_BEGIN_CRITICAL_SECTION(self); - zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v); - Py_END_CRITICAL_SECTION(); + size_t zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v); /* Check error */ if (ZSTD_isError(zstd_ret)) { - set_parameter_error(mod_state, 0, key_v, value_v); + set_parameter_error(0, key_v, value_v); return -1; } } return 0; } -/* Load dictionary or prefix to decompression context */ -int -_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) +static int +_zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd, + _zstd_state *mod_state, int type) { size_t zstd_ret; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return -1; - } - ZstdDict *zd; - int type, ret; - - /* Check ZstdDict */ - ret = PyObject_IsInstance(dict, (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* When decompressing, use digested dictionary by default. */ - zd = (ZstdDict*)dict; - type = DICT_TYPE_DIGESTED; - goto load; - } - - /* Check (ZstdDict, type) */ - if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2) { - /* Check ZstdDict */ - ret = PyObject_IsInstance(PyTuple_GET_ITEM(dict, 0), - (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* type == -1 may indicate an error. */ - type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); - if (type == DICT_TYPE_DIGESTED || - type == DICT_TYPE_UNDIGESTED || - type == DICT_TYPE_PREFIX) - { - assert(type >= 0); - zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); - goto load; - } - } - } - - /* Wrong type */ - PyErr_SetString(PyExc_TypeError, - "zstd_dict argument should be ZstdDict object."); - return -1; - -load: if (type == DICT_TYPE_DIGESTED) { /* Get ZSTD_DDict */ ZSTD_DDict *d_dict = _get_DDict(zd); @@ -178,33 +144,20 @@ _PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) return -1; } /* Reference a prepared dictionary */ - Py_BEGIN_CRITICAL_SECTION(self); zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict); - Py_END_CRITICAL_SECTION(); } else if (type == DICT_TYPE_UNDIGESTED) { /* Load a dictionary */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_DCtx_loadDictionary( - self->dctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); + zstd_ret = ZSTD_DCtx_loadDictionary(self->dctx, zd->dict_buffer, + zd->dict_len); } else if (type == DICT_TYPE_PREFIX) { /* Load a prefix */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_DCtx_refPrefix( - self->dctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); + zstd_ret = ZSTD_DCtx_refPrefix(self->dctx, zd->dict_buffer, + zd->dict_len); } else { - /* Impossible code path */ - PyErr_SetString(PyExc_SystemError, - "load_d_dict() impossible code path"); - return -1; + Py_UNREACHABLE(); } /* Check error */ @@ -215,22 +168,31 @@ _PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) return 0; } - +/* Load dictionary or prefix to decompression context */ +static int +_zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) +{ + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + /* When decompressing, use digested dictionary by default. */ + int type = DICT_TYPE_DIGESTED; + ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type); + if (zd == NULL) { + return -1; + } + int ret; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; +} /* - Given the two types of decompressors (defined in _zstdmodule.h): - - typedef enum { - TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function - } decompress_type; - - Decompress implementation for <D>, <E>, pseudo code: + Decompress implementation in pseudo code: initialize_output_buffer while True: decompress_data - set_object_flag # .eof for <D>, .at_frame_edge for <E>. + set_object_flag # .eof if output_buffer_exhausted: if output_buffer_reached_max_length: @@ -240,76 +202,26 @@ _PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) finish ZSTD_decompressStream()'s size_t return value: - - 0 when a frame is completely decoded and fully flushed, zstd's internal - buffer has no data. + - 0 when a frame is completely decoded and fully flushed, + zstd's internal buffer has no data. - An error code, which can be tested using ZSTD_isError(). - Or any other value > 0, which means there is still some decoding or flushing to do to complete current frame. Note, decompressing "an empty input" in any case will make it > 0. - - <E> supports multiple frames, has an .at_frame_edge flag, it means both the - input and output streams are at a frame edge. The flag can be set by this - statement: - - .at_frame_edge = (zstd_ret == 0) ? 1 : 0 - - But if decompressing "an empty input" at "a frame edge", zstd_ret will be - non-zero, then .at_frame_edge will be wrongly set to false. To solve this - problem, two AFE checks are needed to ensure that: when at "a frame edge", - empty input will not be decompressed. - - // AFE check - if (self->at_frame_edge && in->pos == in->size) { - finish - } - - In <E>, if .at_frame_edge is eventually set to true, but input stream has - unconsumed data (in->pos < in->size), then the outer function - stream_decompress() will set .at_frame_edge to false. In this case, - although the output stream is at a frame edge, for the caller, the input - stream is not at a frame edge, see below diagram. This behavior does not - affect the next AFE check, since (in->pos < in->size). - - input stream: --------------|--- - ^ - output stream: ====================| - ^ */ -PyObject * -decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, - Py_ssize_t max_length, - Py_ssize_t initial_size, - decompress_type type) +static PyObject * +decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in, + Py_ssize_t max_length) { size_t zstd_ret; ZSTD_outBuffer out; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; PyObject *ret; - /* The first AFE check for setting .at_frame_edge flag */ - if (type == TYPE_ENDLESS_DECOMPRESSOR) { - if (self->at_frame_edge && in->pos == in->size) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return NULL; - } - ret = mod_state->empty_bytes; - Py_INCREF(ret); - return ret; - } - } - /* Initialize the output buffer */ - if (initial_size >= 0) { - if (_OutputBuffer_InitWithSize(&buffer, &out, max_length, initial_size) < 0) { - goto error; - } - } - else { - if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) { - goto error; - } + if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) { + goto error; } assert(out.pos == 0); @@ -321,33 +233,20 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret); - } + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret); goto error; } - /* Set .eof/.af_frame_edge flag */ - if (type == TYPE_DECOMPRESSOR) { - /* ZstdDecompressor class stops when a frame is decompressed */ - if (zstd_ret == 0) { - self->eof = 1; - break; - } - } - else if (type == TYPE_ENDLESS_DECOMPRESSOR) { - /* decompress() function supports multiple frames */ - self->at_frame_edge = (zstd_ret == 0) ? 1 : 0; - - /* The second AFE check for setting .at_frame_edge flag */ - if (self->at_frame_edge && in->pos == in->size) { - break; - } + /* Set .eof flag */ + if (zstd_ret == 0) { + /* Stop when a frame is decompressed */ + self->eof = 1; + break; } /* Need to check out before in. Maybe zstd's internal buffer still has - a few bytes can be output, grow the buffer and continue. */ + a few bytes that can be output, grow the buffer and continue. */ if (out.pos == out.size) { /* Output buffer exhausted */ @@ -380,67 +279,40 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, return NULL; } -void -decompressor_reset_session(ZstdDecompressor *self, - decompress_type type) +static void +decompressor_reset_session_lock_held(ZstdDecompressor *self) { - // TODO(emmatyping): use _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED here - // and ensure lock is always held + assert(PyMutex_IsLocked(&self->lock)); /* Reset variables */ self->in_begin = 0; self->in_end = 0; - if (type == TYPE_DECOMPRESSOR) { - Py_CLEAR(self->unused_data); - } + Py_CLEAR(self->unused_data); /* Reset variables in one operation */ self->needs_input = 1; - self->at_frame_edge = 1; self->eof = 0; - self->_unused_char_for_align = 0; - /* Resetting session never fail */ + /* Resetting session is guaranteed to never fail */ ZSTD_DCtx_reset(self->dctx, ZSTD_reset_session_only); } -PyObject * -stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length, - decompress_type type) +static PyObject * +stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data, + Py_ssize_t max_length) { - Py_ssize_t initial_buffer_size = -1; + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; PyObject *ret = NULL; int use_input_buffer; - if (type == TYPE_DECOMPRESSOR) { - /* Check .eof flag */ - if (self->eof) { - PyErr_SetString(PyExc_EOFError, "Already at the end of a zstd frame."); - assert(ret == NULL); - goto success; - } - } - else if (type == TYPE_ENDLESS_DECOMPRESSOR) { - /* Fast path for the first frame */ - if (self->at_frame_edge && self->in_begin == self->in_end) { - /* Read decompressed size */ - uint64_t decompressed_size = ZSTD_getFrameContentSize(data->buf, data->len); - - /* These two zstd constants always > PY_SSIZE_T_MAX: - ZSTD_CONTENTSIZE_UNKNOWN is (0ULL - 1) - ZSTD_CONTENTSIZE_ERROR is (0ULL - 2) - - Use ZSTD_findFrameCompressedSize() to check complete frame, - prevent allocating too much memory for small input chunk. */ - - if (decompressed_size <= (uint64_t) PY_SSIZE_T_MAX && - !ZSTD_isError(ZSTD_findFrameCompressedSize(data->buf, data->len)) ) - { - initial_buffer_size = (Py_ssize_t) decompressed_size; - } - } + /* Check .eof flag */ + if (self->eof) { + PyErr_SetString(PyExc_EOFError, + "Already at the end of a Zstandard frame."); + assert(ret == NULL); + return NULL; } /* Prepare input buffer w/wo unconsumed data */ @@ -527,30 +399,18 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length assert(in.pos == 0); /* Decompress */ - ret = decompress_impl(self, &in, - max_length, initial_buffer_size, - type); + ret = decompress_lock_held(self, &in, max_length); if (ret == NULL) { goto error; } /* Unconsumed input data */ if (in.pos == in.size) { - if (type == TYPE_DECOMPRESSOR) { - if (Py_SIZE(ret) == max_length || self->eof) { - self->needs_input = 0; - } - else { - self->needs_input = 1; - } + if (Py_SIZE(ret) == max_length || self->eof) { + self->needs_input = 0; } - else if (type == TYPE_ENDLESS_DECOMPRESSOR) { - if (Py_SIZE(ret) == max_length && !self->at_frame_edge) { - self->needs_input = 0; - } - else { - self->needs_input = 1; - } + else { + self->needs_input = 1; } if (use_input_buffer) { @@ -564,15 +424,11 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length self->needs_input = 0; - if (type == TYPE_ENDLESS_DECOMPRESSOR) { - self->at_frame_edge = 0; - } - if (!use_input_buffer) { /* Discard buffer if it's too small (resizing it may needlessly copy the current contents) */ - if (self->input_buffer != NULL && - self->input_buffer_size < data_size) + if (self->input_buffer != NULL + && self->input_buffer_size < data_size) { PyMem_Free(self->input_buffer); self->input_buffer = NULL; @@ -600,47 +456,58 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length } } - goto success; + return ret; error: /* Reset decompressor's states/session */ - decompressor_reset_session(self, type); + decompressor_reset_session_lock_held(self); Py_CLEAR(ret); -success: - - return ret; + return NULL; } +/*[clinic input] +@permit_long_docstring_body +@classmethod +_zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new + zstd_dict: object = None + A ZstdDict object, a pre-trained Zstandard dictionary. + options: object = None + A dict object that contains advanced decompression parameters. + +Create a decompressor object for decompressing data incrementally. + +Thread-safe at method level. For one-shot decompression, use the decompress() +function instead. +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, + PyObject *options) +/*[clinic end generated code: output=590ca65c1102ff4a input=ed8891edfd14cdaa]*/ { - ZstdDecompressor *self; - self = PyObject_GC_New(ZstdDecompressor, type); + ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { goto error; } - self->inited = 0; - self->dict = NULL; self->input_buffer = NULL; self->input_buffer_size = 0; self->in_begin = -1; self->in_end = -1; self->unused_data = NULL; self->eof = 0; + self->dict = NULL; + self->lock = (PyMutex){0}; /* needs_input flag */ self->needs_input = 1; - /* at_frame_edge flag */ - self->at_frame_edge = 1; - /* Decompression context */ self->dctx = ZSTD_createDCtx(); if (self->dctx == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, "Unable to create ZSTD_DCtx instance."); @@ -648,12 +515,29 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } + /* Load Zstandard dictionary to decompression context */ + if (zstd_dict != Py_None) { + if (_zstd_load_d_dict(self, zstd_dict) < 0) { + goto error; + } + Py_INCREF(zstd_dict); + self->dict = zstd_dict; + } + + /* Set options dictionary */ + if (options != Py_None) { + if (_zstd_set_d_parameters(self, options) < 0) { + goto error; + } + } + + // We can only start GC tracking once self->dict is set. + PyObject_GC_Track(self); + return (PyObject*)self; error: - if (self != NULL) { - PyObject_GC_Del(self); - } + Py_XDECREF(self); return NULL; } @@ -665,7 +549,11 @@ ZstdDecompressor_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free decompression context */ - ZSTD_freeDCtx(self->dctx); + if (self->dctx) { + ZSTD_freeDCtx(self->dctx); + } + + assert(!PyMutex_IsLocked(&self->lock)); /* Py_CLEAR the dict after free decompression context */ Py_CLEAR(self->dict); @@ -682,56 +570,7 @@ ZstdDecompressor_dealloc(PyObject *ob) } /*[clinic input] -_zstd.ZstdDecompressor.__init__ - - zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. - options: object = None - A dict object that contains advanced decompression parameters. - -Create a decompressor object for decompressing data incrementally. - -Thread-safe at method level. For one-shot decompression, use the decompress() -function instead. -[clinic start generated code]*/ - -static int -_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, - PyObject *zstd_dict, PyObject *options) -/*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/ -{ - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); - return -1; - } - self->inited = 1; - - /* Load dictionary to decompression context */ - if (zstd_dict != Py_None) { - if (_PyZstd_load_d_dict(self, zstd_dict) < 0) { - return -1; - } - - /* Py_INCREF the dict */ - Py_INCREF(zstd_dict); - self->dict = zstd_dict; - } - - /* Set option to decompression context */ - if (options != Py_None) { - if (_PyZstd_set_d_parameters(self, options) < 0) { - return -1; - } - } - - // We can only start tracking self with the GC once self->dict is set. - PyObject_GC_Track(self); - return 0; -} - -/*[clinic input] -@critical_section +@permit_long_docstring_body @getter _zstd.ZstdDecompressor.unused_data @@ -743,20 +582,15 @@ decompressed, unused input data after the frame. Otherwise this will be b''. static PyObject * _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) -/*[clinic end generated code: output=f3a20940f11b6b09 input=5233800bef00df04]*/ +/*[clinic end generated code: output=f3a20940f11b6b09 input=37c2c531ab56f914]*/ { PyObject *ret; - /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); + PyMutex_Lock(&self->lock); if (!self->eof) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return NULL; - } - ret = mod_state->empty_bytes; - Py_INCREF(ret); + PyMutex_Unlock(&self->lock); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else { if (self->unused_data == NULL) { @@ -772,16 +606,17 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) } } - Py_END_CRITICAL_SECTION(); - + PyMutex_Unlock(&self->lock); return ret; } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body _zstd.ZstdDecompressor.decompress data: Py_buffer - A bytes-like object, zstd data to be decompressed. + A bytes-like object, Zstandard data to be decompressed. max_length: Py_ssize_t = -1 Maximum size of returned data. When it is negative, the size of output buffer is unlimited. When it is nonnegative, returns at @@ -807,25 +642,19 @@ static PyObject * _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=a4302b3c940dbec6 input=830e455bc9a50b6e]*/ +/*[clinic end generated code: output=a4302b3c940dbec6 input=e5c905a774df1553]*/ { PyObject *ret; /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); - - ret = stream_decompress(self, data, max_length, TYPE_DECOMPRESSOR); - Py_END_CRITICAL_SECTION(); + PyMutex_Lock(&self->lock); + ret = stream_decompress_lock_held(self, data, max_length); + PyMutex_Unlock(&self->lock); return ret; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/decompressor.c.h" -#undef clinic_state - static PyMethodDef ZstdDecompressor_methods[] = { _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF - - {0} + {NULL, NULL} }; PyDoc_STRVAR(ZstdDecompressor_eof_doc, @@ -833,24 +662,22 @@ PyDoc_STRVAR(ZstdDecompressor_eof_doc, "after that, an EOFError exception will be raised."); PyDoc_STRVAR(ZstdDecompressor_needs_input_doc, -"If the max_length output limit in .decompress() method has been reached, and\n" -"the decompressor has (or may has) unconsumed input data, it will be set to\n" -"False. In this case, pass b'' to .decompress() method may output further data."); +"If the max_length output limit in .decompress() method has been reached,\n" +"and the decompressor has (or may has) unconsumed input data, it will be set\n" +"to False. In this case, passing b'' to the .decompress() method may output\n" +"further data."); static PyMemberDef ZstdDecompressor_members[] = { {"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof), Py_READONLY, ZstdDecompressor_eof_doc}, - {"needs_input", Py_T_BOOL, offsetof(ZstdDecompressor, needs_input), Py_READONLY, ZstdDecompressor_needs_input_doc}, - - {0} + {NULL} }; static PyGetSetDef ZstdDecompressor_getset[] = { _ZSTD_ZSTDDECOMPRESSOR_UNUSED_DATA_GETSETDEF - - {0} + {NULL} }; static int @@ -873,19 +700,19 @@ ZstdDecompressor_clear(PyObject *ob) static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, {Py_tp_dealloc, ZstdDecompressor_dealloc}, - {Py_tp_init, _zstd_ZstdDecompressor___init__}, {Py_tp_methods, ZstdDecompressor_methods}, {Py_tp_members, ZstdDecompressor_members}, {Py_tp_getset, ZstdDecompressor_getset}, - {Py_tp_doc, (char*)_zstd_ZstdDecompressor___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, {Py_tp_clear, ZstdDecompressor_clear}, - {0} + {0, 0} }; -PyType_Spec ZstdDecompressor_type_spec = { - .name = "_zstd.ZstdDecompressor", +PyType_Spec zstd_decompressor_type_spec = { + .name = "compression.zstd.ZstdDecompressor", .basicsize = sizeof(ZstdDecompressor), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC, .slots = ZstdDecompressor_slots, }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index a19224c4a64..b0bfbdc886e 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -1,38 +1,68 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdDict class definitions */ /*[clinic input] module _zstd -class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type" +class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5d1254c497e52ba]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" -#include <stddef.h> // offsetof() +#include "_zstdmodule.h" +#include "clinic/zstddict.c.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked + +#include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict() #define ZstdDict_CAST(op) ((ZstdDict *)op) +/*[clinic input] +@permit_long_docstring_body +@classmethod +_zstd.ZstdDict.__new__ as _zstd_ZstdDict_new + dict_content: Py_buffer + The content of a Zstandard dictionary as a bytes-like object. + / + * + is_raw: bool = False + If true, perform no checks on *dict_content*, useful for some + advanced cases. Otherwise, check that the content represents + a Zstandard dictionary created by the zstd library or CLI. + +Represents a Zstandard dictionary. + +The dictionary can be used for compression or decompression, and can be shared +by multiple ZstdCompressor or ZstdDecompressor objects. +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) +_zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, + int is_raw) +/*[clinic end generated code: output=685b7406a48b0949 input=b132ee40b784c293]*/ { - ZstdDict *self; - self = PyObject_GC_New(ZstdDict, type); - if (self == NULL) { - goto error; + /* All dictionaries must be at least 8 bytes */ + if (dict_content->len < 8) { + PyErr_SetString(PyExc_ValueError, + "Zstandard dictionary content too short " + "(must have at least eight bytes)"); + return NULL; + } + + ZstdDict* self = PyObject_GC_New(ZstdDict, type); + if (self == NULL) { + return NULL; } - self->dict_content = NULL; - self->inited = 0; self->d_dict = NULL; + self->dict_buffer = NULL; + self->dict_id = 0; + self->lock = (PyMutex){0}; /* ZSTD_CDict dict */ self->c_dicts = PyDict_New(); @@ -40,12 +70,29 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U goto error; } - return (PyObject*)self; + self->dict_buffer = PyMem_Malloc(dict_content->len); + if (!self->dict_buffer) { + PyErr_NoMemory(); + goto error; + } + memcpy(self->dict_buffer, dict_content->buf, dict_content->len); + self->dict_len = dict_content->len; + + /* Get dict_id, 0 means "raw content" dictionary. */ + self->dict_id = ZSTD_getDictID_fromDict(self->dict_buffer, self->dict_len); + + /* Check validity for ordinary dictionary */ + if (!is_raw && self->dict_id == 0) { + PyErr_SetString(PyExc_ValueError, "invalid Zstandard dictionary"); + goto error; + } + + PyObject_GC_Track(self); + + return (PyObject *)self; error: - if (self != NULL) { - PyObject_GC_Del(self); - } + Py_XDECREF(self); return NULL; } @@ -57,121 +104,65 @@ ZstdDict_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free ZSTD_DDict instance */ - ZSTD_freeDDict(self->d_dict); + if (self->d_dict) { + ZSTD_freeDDict(self->d_dict); + } - /* Release dict_content after Free ZSTD_CDict/ZSTD_DDict instances */ - Py_CLEAR(self->dict_content); + assert(!PyMutex_IsLocked(&self->lock)); + + /* Release dict_buffer after freeing ZSTD_CDict/ZSTD_DDict instances */ + PyMem_Free(self->dict_buffer); Py_CLEAR(self->c_dicts); PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_Del(ob); + tp->tp_free(self); Py_DECREF(tp); } -/*[clinic input] -_zstd.ZstdDict.__init__ - - dict_content: object - A bytes-like object, dictionary's content. - is_raw: bool = False - This parameter is for advanced user. True means dict_content - argument is a "raw content" dictionary, free of any format - restriction. False means dict_content argument is an ordinary - zstd dictionary, was created by zstd functions, follow a - specified format. - -Represents a zstd dictionary, which can be used for compression/decompression. - -It's thread-safe, and can be shared by multiple ZstdCompressor / -ZstdDecompressor objects. -[clinic start generated code]*/ - -static int -_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, - int is_raw) -/*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/ -{ - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); - return -1; - } - self->inited = 1; - - /* Check dict_content's type */ - self->dict_content = PyBytes_FromObject(dict_content); - if (self->dict_content == NULL) { - PyErr_SetString(PyExc_TypeError, - "dict_content argument should be bytes-like object."); - return -1; - } - - /* Both ordinary dictionary and "raw content" dictionary should - at least 8 bytes */ - if (Py_SIZE(self->dict_content) < 8) { - PyErr_SetString(PyExc_ValueError, - "Zstd dictionary content should at least 8 bytes."); - return -1; - } - - /* Get dict_id, 0 means "raw content" dictionary. */ - self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content), - Py_SIZE(self->dict_content)); - - /* Check validity for ordinary dictionary */ - if (!is_raw && self->dict_id == 0) { - char *msg = "The dict_content argument is not a valid zstd " - "dictionary. The first 4 bytes of a valid zstd dictionary " - "should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n" - "If you are an advanced user, and can be sure that " - "dict_content argument is a \"raw content\" zstd " - "dictionary, set is_raw parameter to True."; - PyErr_SetString(PyExc_ValueError, msg); - return -1; - } - - // Can only track self once self->dict_content is included - PyObject_GC_Track(self); - return 0; -} - -#define clinic_state() (get_zstd_state(type)) -#include "clinic/zstddict.c.h" -#undef clinic_state - PyDoc_STRVAR(ZstdDict_dictid_doc, -"ID of zstd dictionary, a 32-bit unsigned int value.\n\n" -"Non-zero means ordinary dictionary, was created by zstd functions, follow\n" -"a specified format.\n\n" -"0 means a \"raw content\" dictionary, free of any format restriction, used\n" -"for advanced user."); - -PyDoc_STRVAR(ZstdDict_dictcontent_doc, -"The content of zstd dictionary, a bytes object, it's the same as dict_content\n" -"argument in ZstdDict.__init__() method. It can be used with other programs."); +"The Zstandard dictionary, an int between 0 and 2**32.\n\n" +"A non-zero value represents an ordinary Zstandard dictionary,\n" +"conforming to the standardised format.\n\n" +"A value of zero indicates a 'raw content' dictionary,\n" +"without any restrictions on format or content."); static PyObject * -ZstdDict_str(PyObject *ob) +ZstdDict_repr(PyObject *ob) { ZstdDict *dict = ZstdDict_CAST(ob); return PyUnicode_FromFormat("<ZstdDict dict_id=%u dict_size=%zd>", - dict->dict_id, Py_SIZE(dict->dict_content)); + (unsigned int)dict->dict_id, dict->dict_len); } static PyMemberDef ZstdDict_members[] = { {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc}, - {"dict_content", Py_T_OBJECT_EX, offsetof(ZstdDict, dict_content), Py_READONLY, ZstdDict_dictcontent_doc}, - {0} + {NULL} }; /*[clinic input] -@critical_section +@getter +_zstd.ZstdDict.dict_content + +The content of a Zstandard dictionary, as a bytes object. +[clinic start generated code]*/ + +static PyObject * +_zstd_ZstdDict_dict_content_get_impl(ZstdDict *self) +/*[clinic end generated code: output=0d05caa5b550eabb input=4ed526d1c151c596]*/ +{ + return PyBytes_FromStringAndSize(self->dict_buffer, self->dict_len); +} + +/*[clinic input] +@permit_long_docstring_body @getter _zstd.ZstdDict.as_digested_dict Load as a digested dictionary to compressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict) +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_digested_dict) + 1. Some advanced compression parameters of compressor may be overridden by parameters of digested dictionary. 2. ZstdDict has a digested dictionaries cache for each compression level. @@ -182,19 +173,21 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digeste static PyObject * _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=09b086e7a7320dbb input=585448c79f31f74a]*/ +/*[clinic end generated code: output=09b086e7a7320dbb input=8d01ff0b8b043f2e]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED); } /*[clinic input] -@critical_section +@permit_long_docstring_body @getter _zstd.ZstdDict.as_undigested_dict Load as an undigested dictionary to compressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict) +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_undigested_dict) + 1. The advanced compression parameters of compressor will not be overridden. 2. Loading an undigested dictionary is costly. If load an undigested dictionary multiple times, consider reusing a compressor object. @@ -203,48 +196,47 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undiges static PyObject * _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=43c7a989e6d4253a input=022b0829ffb1c220]*/ +/*[clinic end generated code: output=43c7a989e6d4253a input=b1bdb306c3798ad4]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED); } /*[clinic input] -@critical_section +@permit_long_docstring_body @getter _zstd.ZstdDict.as_prefix Load as a prefix to compressor/decompressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix) +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_prefix) + 1. Prefix is compatible with long distance matching, while dictionary is not. 2. It only works for the first frame, then the compressor/decompressor will return to no prefix state. -3. When decompressing, must use the same prefix as when compressing." +3. When decompressing, must use the same prefix as when compressing. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) -/*[clinic end generated code: output=6f7130c356595a16 input=09fb82a6a5407e87]*/ +/*[clinic end generated code: output=6f7130c356595a16 input=77966c012d15e6ab]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX); } static PyGetSetDef ZstdDict_getset[] = { + _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF - _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF - _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF - - {0} + {NULL} }; static Py_ssize_t ZstdDict_length(PyObject *ob) { ZstdDict *self = ZstdDict_CAST(ob); - assert(PyBytes_Check(self->dict_content)); - return Py_SIZE(self->dict_content); + return self->dict_len; } static int @@ -252,7 +244,6 @@ ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdDict *self = ZstdDict_CAST(ob); Py_VISIT(self->c_dicts); - Py_VISIT(self->dict_content); return 0; } @@ -260,7 +251,7 @@ static int ZstdDict_clear(PyObject *ob) { ZstdDict *self = ZstdDict_CAST(ob); - Py_CLEAR(self->dict_content); + Py_CLEAR(self->c_dicts); return 0; } @@ -269,18 +260,18 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_getset, ZstdDict_getset}, {Py_tp_new, _zstd_ZstdDict_new}, {Py_tp_dealloc, ZstdDict_dealloc}, - {Py_tp_init, _zstd_ZstdDict___init__}, - {Py_tp_str, ZstdDict_str}, - {Py_tp_doc, (char*)_zstd_ZstdDict___init____doc__}, + {Py_tp_repr, ZstdDict_repr}, + {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, {Py_tp_clear, ZstdDict_clear}, - {0} + {0, 0} }; -PyType_Spec zstddict_type_spec = { - .name = "_zstd.ZstdDict", +PyType_Spec zstd_dict_type_spec = { + .name = "compression.zstd.ZstdDict", .basicsize = sizeof(ZstdDict), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC, .slots = zstddict_slots, }; diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h new file mode 100644 index 00000000000..e0d3f46b2b1 --- /dev/null +++ b/Modules/_zstd/zstddict.h @@ -0,0 +1,29 @@ +/* Low level interface to the Zstandard algorithm & the zstd library. */ + +#ifndef ZSTD_DICT_H +#define ZSTD_DICT_H + +#include <zstd.h> // ZSTD_DDict + +typedef struct { + PyObject_HEAD + + /* Reusable compress/decompress dictionary, they are created once and + can be shared by multiple threads concurrently, since its usage is + read-only. + c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ + ZSTD_DDict *d_dict; + PyObject *c_dicts; + + /* Dictionary content. */ + char *dict_buffer; + Py_ssize_t dict_len; + + /* Dictionary id */ + uint32_t dict_id; + + /* Lock to protect the digested dictionaries */ + PyMutex lock; +} ZstdDict; + +#endif // !ZSTD_DICT_H diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 401a3a7072b..729e085c19f 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -13,6 +13,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include <stdbool.h> @@ -713,14 +714,6 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) } /* Methods */ - -static int -array_tp_traverse(PyObject *op, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(op)); - return 0; -} - static void array_dealloc(PyObject *op) { @@ -728,9 +721,7 @@ array_dealloc(PyObject *op) PyObject_GC_UnTrack(op); arrayobject *self = arrayobject_CAST(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); if (self->ob_item != NULL) { PyMem_Free(self->ob_item); } @@ -1375,6 +1366,7 @@ array_array_insert_impl(arrayobject *self, Py_ssize_t i, PyObject *v) } /*[clinic input] +@permit_long_summary array.array.buffer_info Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents. @@ -1385,7 +1377,7 @@ the buffer length in bytes. static PyObject * array_array_buffer_info_impl(arrayobject *self) -/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=a58bae5c6e1ac6a6]*/ +/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=63d9ad83ba60cda8]*/ { PyObject *retval = NULL, *v; @@ -1427,6 +1419,7 @@ array_array_append_impl(arrayobject *self, PyObject *v) } /*[clinic input] +@permit_long_docstring_body array.array.byteswap Byteswap all items of the array. @@ -1437,7 +1430,7 @@ raised. static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=6a85591b950a0186]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=9af1d1749000b14f]*/ { char *p; Py_ssize_t i; @@ -1520,11 +1513,12 @@ array_array_reverse_impl(arrayobject *self) } /*[clinic input] +@permit_long_summary array.array.fromfile cls: defining_class f: object - n: Py_ssize_t + n: Py_ssize_t(allow_negative=False) / Read n objects from the file object f and append them to the end of the array. @@ -1533,17 +1527,13 @@ Read n objects from the file object f and append them to the end of the array. static PyObject * array_array_fromfile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f, Py_ssize_t n) -/*[clinic end generated code: output=83a667080b345ebc input=3822e907c1c11f1a]*/ +/*[clinic end generated code: output=83a667080b345ebc input=db46b06ac1b6de87]*/ { PyObject *b, *res; Py_ssize_t itemsize = self->ob_descr->itemsize; Py_ssize_t nbytes; int not_enough_bytes; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "negative count"); - return NULL; - } if (n > PY_SSIZE_T_MAX / itemsize) { PyErr_NoMemory(); return NULL; @@ -1740,6 +1730,7 @@ frombytes(arrayobject *self, Py_buffer *buffer) } /*[clinic input] +@permit_long_summary array.array.frombytes buffer: Py_buffer @@ -1750,12 +1741,13 @@ Appends items from the string, interpreting it as an array of machine values, as static PyObject * array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) -/*[clinic end generated code: output=d9842c8f7510a516 input=378db226dfac949e]*/ +/*[clinic end generated code: output=d9842c8f7510a516 input=6b90ce5895f677a4]*/ { return frombytes(self, buffer); } /*[clinic input] +@permit_long_summary array.array.tobytes Convert the array to an array of machine values and return the bytes representation. @@ -1763,7 +1755,7 @@ Convert the array to an array of machine values and return the bytes representat static PyObject * array_array_tobytes_impl(arrayobject *self) -/*[clinic end generated code: output=87318e4edcdc2bb6 input=90ee495f96de34f5]*/ +/*[clinic end generated code: output=87318e4edcdc2bb6 input=41081256b97f9d7a]*/ { if (Py_SIZE(self) <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { return PyBytes_FromStringAndSize(self->ob_item, @@ -1774,6 +1766,7 @@ array_array_tobytes_impl(arrayobject *self) } /*[clinic input] +@permit_long_docstring_body array.array.fromunicode ustr: unicode @@ -1788,7 +1781,7 @@ some other type. static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) -/*[clinic end generated code: output=24359f5e001a7f2b input=025db1fdade7a4ce]*/ +/*[clinic end generated code: output=24359f5e001a7f2b input=158d47c302f27ca1]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -1836,6 +1829,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) } /*[clinic input] +@permit_long_docstring_body array.array.tounicode Extends this array with data from the unicode string ustr. @@ -1847,7 +1841,7 @@ unicode string from an array of some other type. static PyObject * array_array_tounicode_impl(arrayobject *self) -/*[clinic end generated code: output=08e442378336e1ef input=127242eebe70b66d]*/ +/*[clinic end generated code: output=08e442378336e1ef input=6690997213d219db]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -2839,6 +2833,9 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_SET_SIZE(self, n); self->allocated = n; } + else { + PyMem_Free(ustr); + } } else { // c == 'w' Py_ssize_t n = PyUnicode_GET_LENGTH(initial); @@ -2962,7 +2959,7 @@ static PyType_Slot array_slots[] = { {Py_tp_getset, array_getsets}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, array_new}, - {Py_tp_traverse, array_tp_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, /* as sequence */ {Py_sq_length, array_length}, diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 4b068967a6c..4536b03fbc4 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -112,6 +112,7 @@ atexit_callfuncs(struct atexit_state *state) { PyErr_FormatUnraisable("Exception ignored while " "copying atexit callbacks"); + atexit_cleanup(state); return; } diff --git a/Modules/binascii.c b/Modules/binascii.c index 6bb01d148b6..13e4bc5be03 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -205,11 +205,9 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) /*[clinic end generated code: output=e027f8e0b0598742 input=7cafeaf73df63d1c]*/ { const unsigned char *ascii_data; - unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; Py_ssize_t ascii_len, bin_len; binascii_state *state; @@ -223,9 +221,11 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) ascii_len--; /* Allocate the buffer */ - if ( (rv=PyBytes_FromStringAndSize(NULL, bin_len)) == NULL ) + PyBytesWriter *writer = PyBytesWriter_Create(bin_len); + if (writer == NULL) { return NULL; - bin_data = (unsigned char *)PyBytes_AS_STRING(rv); + } + unsigned char *bin_data = PyBytesWriter_GetData(writer); for( ; bin_len > 0 ; ascii_len--, ascii_data++ ) { /* XXX is it really best to add NULs if there's no more data */ @@ -245,11 +245,10 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) if ( this_ch < ' ' || this_ch > (' ' + 64)) { state = get_binascii_state(module); if (state == NULL) { - return NULL; + goto error; } PyErr_SetString(state->Error, "Illegal char"); - Py_DECREF(rv); - return NULL; + goto error; } this_ch = (this_ch - ' ') & 077; } @@ -277,14 +276,17 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) this_ch != '\n' && this_ch != '\r' ) { state = get_binascii_state(module); if (state == NULL) { - return NULL; + goto error; } PyErr_SetString(state->Error, "Trailing garbage"); - Py_DECREF(rv); - return NULL; + goto error; } } - return rv; + return PyBytesWriter_Finish(writer); + +error: + PyBytesWriter_Discard(writer); + return NULL; } /*[clinic input] @@ -302,16 +304,13 @@ static PyObject * binascii_b2a_uu_impl(PyObject *module, Py_buffer *data, int backtick) /*[clinic end generated code: output=b1b99de62d9bbeb8 input=beb27822241095cd]*/ { - unsigned char *ascii_data; const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; binascii_state *state; - Py_ssize_t bin_len, out_len; - _PyBytesWriter writer; + Py_ssize_t bin_len; - _PyBytesWriter_Init(&writer); bin_data = data->buf; bin_len = data->len; if ( bin_len > 45 ) { @@ -325,10 +324,12 @@ binascii_b2a_uu_impl(PyObject *module, Py_buffer *data, int backtick) } /* We're lazy and allocate to much (fixed up later) */ - out_len = 2 + (bin_len + 2) / 3 * 4; - ascii_data = _PyBytesWriter_Alloc(&writer, out_len); - if (ascii_data == NULL) + Py_ssize_t out_len = 2 + (bin_len + 2) / 3 * 4; + PyBytesWriter *writer = PyBytesWriter_Create(out_len); + if (writer == NULL) { return NULL; + } + unsigned char *ascii_data = PyBytesWriter_GetData(writer); /* Store the length */ if (backtick && !bin_len) @@ -356,10 +357,11 @@ binascii_b2a_uu_impl(PyObject *module, Py_buffer *data, int backtick) } *ascii_data++ = '\n'; /* Append a courtesy newline */ - return _PyBytesWriter_Finish(&writer, ascii_data); + return PyBytesWriter_FinishWithPointer(writer, ascii_data); } /*[clinic input] +@permit_long_docstring_body binascii.a2b_base64 data: ascii_buffer @@ -376,7 +378,7 @@ Decode a line of base64 data. static PyObject * binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) -/*[clinic end generated code: output=5409557788d4f975 input=c0c15fd0f8f9a62d]*/ +/*[clinic end generated code: output=5409557788d4f975 input=13c797187acc9c40]*/ { assert(data->len >= 0); @@ -387,12 +389,11 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) /* Allocate the buffer */ Py_ssize_t bin_len = ((ascii_len+3)/4)*3; /* Upper bound, corrected later */ - _PyBytesWriter writer; - _PyBytesWriter_Init(&writer); - unsigned char *bin_data = _PyBytesWriter_Alloc(&writer, bin_len); - if (bin_data == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(bin_len); + if (writer == NULL) { return NULL; - unsigned char *bin_data_start = bin_data; + } + unsigned char *bin_data = PyBytesWriter_GetData(writer); if (strict_mode && ascii_len > 0 && ascii_data[0] == '=') { state = get_binascii_state(module); @@ -488,12 +489,14 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) state = get_binascii_state(module); if (state == NULL) { /* error already set, from get_binascii_state */ + assert(PyErr_Occurred()); } else if (quad_pos == 1) { /* ** There is exactly one extra valid, non-padding, base64 character. ** This is an invalid length, as there is no possible input that ** could encoded into such a base64 string. */ + unsigned char *bin_data_start = PyBytesWriter_GetData(writer); PyErr_Format(state->Error, "Invalid base64-encoded string: " "number of data characters (%zd) cannot be 1 more " @@ -502,13 +505,15 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) } else { PyErr_SetString(state->Error, "Incorrect padding"); } - error_end: - _PyBytesWriter_Dealloc(&writer); - return NULL; + goto error_end; } done: - return _PyBytesWriter_Finish(&writer, bin_data); + return PyBytesWriter_FinishWithPointer(writer, bin_data); + +error_end: + PyBytesWriter_Discard(writer); + return NULL; } @@ -527,18 +532,15 @@ static PyObject * binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int newline) /*[clinic end generated code: output=4ad62c8e8485d3b3 input=0e20ff59c5f2e3e1]*/ { - unsigned char *ascii_data; const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - Py_ssize_t bin_len, out_len; - _PyBytesWriter writer; + Py_ssize_t bin_len; binascii_state *state; bin_data = data->buf; bin_len = data->len; - _PyBytesWriter_Init(&writer); assert(bin_len >= 0); @@ -554,12 +556,15 @@ binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int newline) /* We're lazy and allocate too much (fixed up later). "+2" leaves room for up to two pad characters. Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ - out_len = bin_len*2 + 2; - if (newline) + Py_ssize_t out_len = bin_len*2 + 2; + if (newline) { out_len++; - ascii_data = _PyBytesWriter_Alloc(&writer, out_len); - if (ascii_data == NULL) + } + PyBytesWriter *writer = PyBytesWriter_Create(out_len); + if (writer == NULL) { return NULL; + } + unsigned char *ascii_data = PyBytesWriter_GetData(writer); for( ; bin_len > 0 ; bin_len--, bin_data++ ) { /* Shift the data into our buffer */ @@ -584,7 +589,7 @@ binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int newline) if (newline) *ascii_data++ = '\n'; /* Append a courtesy newline */ - return _PyBytesWriter_Finish(&writer, ascii_data); + return PyBytesWriter_FinishWithPointer(writer, ascii_data); } @@ -886,8 +891,6 @@ binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr) { const char* argbuf; Py_ssize_t arglen; - PyObject *retval; - char* retbuf; Py_ssize_t i, j; binascii_state *state; @@ -909,10 +912,11 @@ binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr) return NULL; } - retval = PyBytes_FromStringAndSize(NULL, (arglen/2)); - if (!retval) + PyBytesWriter *writer = PyBytesWriter_Create(arglen/2); + if (writer == NULL) { return NULL; - retbuf = PyBytes_AS_STRING(retval); + } + char *retbuf = PyBytesWriter_GetData(writer); for (i=j=0; i < arglen; i += 2) { unsigned int top = _PyLong_DigitValue[Py_CHARMASK(argbuf[i])]; @@ -920,18 +924,18 @@ binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr) if (top >= 16 || bot >= 16) { state = get_binascii_state(module); if (state == NULL) { - return NULL; + goto error; } PyErr_SetString(state->Error, "Non-hexadecimal digit found"); - goto finally; + goto error; } retbuf[j++] = (top << 4) + bot; } - return retval; + return PyBytesWriter_Finish(writer); - finally: - Py_DECREF(retval); +error: + PyBytesWriter_Discard(writer); return NULL; } diff --git a/Modules/blake2module.c b/Modules/blake2module.c index f9acc57f1b2..89b0ebd516f 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -2,6 +2,7 @@ * Written in 2013 by Dmitry Chestnykh <dmitry@codingrobots.com> * Modified for CPython by Christian Heimes <christian@python.org> * Updated to use HACL* by Jonathan Protzenko <jonathan@protzenko.fr> + * Additional work by Bénédikt Tran <10796600+picnixz@users.noreply.github.com> * * To the extent possible under law, the author have dedicated all * copyright and related and neighboring rights to this software to @@ -13,12 +14,12 @@ # define Py_BUILD_CORE_MODULE 1 #endif -#include "pyconfig.h" #include "Python.h" #include "hashlib.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" #include "pycore_moduleobject.h" +#include "pycore_object.h" // _PyObject_VisitType() +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // QUICK CPU AUTODETECTION // @@ -43,104 +44,27 @@ // SIMD256 can't be compiled on macOS ARM64, and performance of SIMD128 isn't // great; but when compiling a universal2 binary, autoconf will set -// HACL_CAN_COMPILE_SIMD128 and HACL_CAN_COMPILE_SIMD256 because they *can* be -// compiled on x86_64. If we're on macOS ARM64, disable these preprocessor -// symbols. +// _Py_HACL_CAN_COMPILE_VEC{128,256} because they *can* be compiled on x86_64. +// If we're on macOS ARM64, we however disable these preprocessor symbols. #if defined(__APPLE__) && defined(__arm64__) -# undef HACL_CAN_COMPILE_SIMD128 -# undef HACL_CAN_COMPILE_SIMD256 +# undef _Py_HACL_CAN_COMPILE_VEC128 +# undef _Py_HACL_CAN_COMPILE_VEC256 #endif -// ECX -#define ECX_SSE3 (1 << 0) -#define ECX_SSSE3 (1 << 9) -#define ECX_SSE4_1 (1 << 19) -#define ECX_SSE4_2 (1 << 20) -#define ECX_AVX (1 << 28) +// HACL* expects HACL_CAN_COMPILE_VEC* macros to be set in order to enable +// the corresponding SIMD instructions so we need to "forward" the values +// we just deduced above. +#define HACL_CAN_COMPILE_VEC128 _Py_HACL_CAN_COMPILE_VEC128 +#define HACL_CAN_COMPILE_VEC256 _Py_HACL_CAN_COMPILE_VEC256 -// EBX -#define EBX_AVX2 (1 << 5) - -// EDX -#define EDX_SSE (1 << 25) -#define EDX_SSE2 (1 << 26) -#define EDX_CMOV (1 << 15) - -// zero-initialized by default -typedef struct { - bool sse, sse2, sse3, sse41, sse42, cmov, avx, avx2; - bool done; -} cpu_flags; - -void detect_cpu_features(cpu_flags *flags) { - if (!flags->done) { - int eax1 = 0, ebx1 = 0, ecx1 = 0, edx1 = 0; - int eax7 = 0, ebx7 = 0, ecx7 = 0, edx7 = 0; -#if defined(__x86_64__) && defined(__GNUC__) - __cpuid_count(1, 0, eax1, ebx1, ecx1, edx1); - __cpuid_count(7, 0, eax7, ebx7, ecx7, edx7); -#elif defined(_M_X64) - int info1[4] = { 0 }; - int info7[4] = { 0 }; - __cpuidex(info1, 1, 0); - __cpuidex(info7, 7, 0); - eax1 = info1[0]; - ebx1 = info1[1]; - ecx1 = info1[2]; - edx1 = info1[3]; - eax7 = info7[0]; - ebx7 = info7[1]; - ecx7 = info7[2]; - edx7 = info7[3]; -#endif - (void) eax1; (void) ebx1; (void) ecx1; (void) edx1; - (void) eax7; (void) ebx7; (void) ecx7; (void) edx7; - - - flags->avx = (ecx1 & ECX_AVX) != 0; - - flags->avx2 = (ebx7 & EBX_AVX2) != 0; - - flags->sse = (edx1 & EDX_SSE) != 0; - flags->sse2 = (edx1 & EDX_SSE2) != 0; - flags->cmov = (edx1 & EDX_CMOV) != 0; - - flags->sse3 = (ecx1 & ECX_SSE3) != 0; - /* ssse3 = (ecx1 & ECX_SSSE3) != 0; */ - flags->sse41 = (ecx1 & ECX_SSE4_1) != 0; - flags->sse42 = (ecx1 & ECX_SSE4_2) != 0; - - flags->done = true; - } -} - -#ifdef HACL_CAN_COMPILE_SIMD128 -static inline bool has_simd128(cpu_flags *flags) { - // For now this is Intel-only, could conceivably be #ifdef'd to something - // else. - return flags->sse && flags->sse2 && flags->sse3 && flags->sse41 && flags->sse42 && flags->cmov; -} -#endif - -#ifdef HACL_CAN_COMPILE_SIMD256 -static inline bool has_simd256(cpu_flags *flags) { - return flags->avx && flags->avx2; -} -#endif - -// Small mismatch between the variable names Python defines as part of configure -// at the ones HACL* expects to be set in order to enable those headers. -#define HACL_CAN_COMPILE_VEC128 HACL_CAN_COMPILE_SIMD128 -#define HACL_CAN_COMPILE_VEC256 HACL_CAN_COMPILE_SIMD256 - -#include "_hacl/Hacl_Hash_Blake2b.h" #include "_hacl/Hacl_Hash_Blake2s.h" -#if HACL_CAN_COMPILE_SIMD256 -#include "_hacl/Hacl_Hash_Blake2b_Simd256.h" -#endif -#if HACL_CAN_COMPILE_SIMD128 +#include "_hacl/Hacl_Hash_Blake2b.h" +#if _Py_HACL_CAN_COMPILE_VEC128 #include "_hacl/Hacl_Hash_Blake2s_Simd128.h" #endif +#if _Py_HACL_CAN_COMPILE_VEC256 +#include "_hacl/Hacl_Hash_Blake2b_Simd256.h" +#endif // MODULE TYPE SLOTS @@ -148,16 +72,16 @@ static PyType_Spec blake2b_type_spec; static PyType_Spec blake2s_type_spec; PyDoc_STRVAR(blake2mod__doc__, -"_blake2b provides BLAKE2b for hashlib\n" -); + "_blake2 provides BLAKE2b and BLAKE2s for hashlib\n"); typedef struct { - PyTypeObject* blake2b_type; - PyTypeObject* blake2s_type; - cpu_flags flags; + PyTypeObject *blake2b_type; + PyTypeObject *blake2s_type; + bool can_run_simd128; + bool can_run_simd256; } Blake2State; -static inline Blake2State* +static inline Blake2State * blake2_get_state(PyObject *module) { void *state = _PyModule_GetState(module); @@ -165,8 +89,8 @@ blake2_get_state(PyObject *module) return (Blake2State *)state; } -#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) -static inline Blake2State* +#if defined(_Py_HACL_CAN_COMPILE_VEC128) || defined(_Py_HACL_CAN_COMPILE_VEC256) +static inline Blake2State * blake2_get_state_from_type(PyTypeObject *module) { void *state = _PyType_GetModuleState(module); @@ -203,31 +127,107 @@ _blake2_free(void *module) (void)_blake2_clear((PyObject *)module); } -#define ADD_INT(d, name, value) do { \ - PyObject *x = PyLong_FromLong(value); \ - if (!x) \ - return -1; \ - if (PyDict_SetItemString(d, name, x) < 0) { \ - Py_DECREF(x); \ - return -1; \ - } \ - Py_DECREF(x); \ -} while(0) +static void +blake2module_init_cpu_features(Blake2State *state) +{ + /* This must be kept in sync with hmacmodule_init_cpu_features() + * in hmacmodule.c */ + int eax1 = 0, ebx1 = 0, ecx1 = 0, edx1 = 0; + int eax7 = 0, ebx7 = 0, ecx7 = 0, edx7 = 0; +#if defined(__x86_64__) && defined(__GNUC__) + __cpuid_count(1, 0, eax1, ebx1, ecx1, edx1); + __cpuid_count(7, 0, eax7, ebx7, ecx7, edx7); +#elif defined(_M_X64) + int info1[4] = {0}; + __cpuidex(info1, 1, 0); + eax1 = info1[0], ebx1 = info1[1], ecx1 = info1[2], edx1 = info1[3]; -#define ADD_INT_CONST(NAME, VALUE) do { \ - if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ - return -1; \ - } \ -} while (0) + int info7[4] = {0}; + __cpuidex(info7, 7, 0); + eax7 = info7[0], ebx7 = info7[1], ecx7 = info7[2], edx7 = info7[3]; +#endif + // fmt: off + (void)eax1; (void)ebx1; (void)ecx1; (void)edx1; + (void)eax7; (void)ebx7; (void)ecx7; (void)edx7; + // fmt: on + +#define EBX_AVX2 (1 << 5) +#define ECX_SSE3 (1 << 0) +#define ECX_SSSE3 (1 << 9) +#define ECX_SSE4_1 (1 << 19) +#define ECX_SSE4_2 (1 << 20) +#define ECX_AVX (1 << 28) +#define EDX_SSE (1 << 25) +#define EDX_SSE2 (1 << 26) +#define EDX_CMOV (1 << 15) + + bool avx = (ecx1 & ECX_AVX) != 0; + bool avx2 = (ebx7 & EBX_AVX2) != 0; + + bool sse = (edx1 & EDX_SSE) != 0; + bool sse2 = (edx1 & EDX_SSE2) != 0; + bool cmov = (edx1 & EDX_CMOV) != 0; + + bool sse3 = (ecx1 & ECX_SSE3) != 0; + bool sse41 = (ecx1 & ECX_SSE4_1) != 0; + bool sse42 = (ecx1 & ECX_SSE4_2) != 0; + +#undef EDX_CMOV +#undef EDX_SSE2 +#undef EDX_SSE +#undef ECX_AVX +#undef ECX_SSE4_2 +#undef ECX_SSE4_1 +#undef ECX_SSSE3 +#undef ECX_SSE3 +#undef EBX_AVX2 + +#if _Py_HACL_CAN_COMPILE_VEC128 + // TODO(picnixz): use py_cpuid_features (gh-125022) to improve detection + state->can_run_simd128 = sse && sse2 && sse3 && sse41 && sse42 && cmov; +#else + // fmt: off + (void)sse; (void)sse2; (void)sse3; (void)sse41; (void)sse42; (void)cmov; + // fmt: on + state->can_run_simd128 = false; +#endif + +#if _Py_HACL_CAN_COMPILE_VEC256 + // TODO(picnixz): use py_cpuid_features (gh-125022) to improve detection + state->can_run_simd256 = state->can_run_simd128 && avx && avx2; +#else + // fmt: off + (void)avx; (void)avx2; + // fmt: on + state->can_run_simd256 = false; +#endif +} static int blake2_exec(PyObject *m) { - Blake2State* st = blake2_get_state(m); + Blake2State *st = blake2_get_state(m); + blake2module_init_cpu_features(st); - // This is called at module initialization-time, and so appears to be as - // good a place as any to probe the CPU flags. - detect_cpu_features(&st->flags); +#define ADD_INT(DICT, NAME, VALUE) \ + do { \ + PyObject *x = PyLong_FromLong(VALUE); \ + if (x == NULL) { \ + return -1; \ + } \ + int rc = PyDict_SetItemString(DICT, NAME, x); \ + Py_DECREF(x); \ + if (rc < 0) { \ + return -1; \ + } \ + } while(0) + +#define ADD_INT_CONST(NAME, VALUE) \ + do { \ + if (PyModule_AddIntConstant(m, NAME, VALUE) < 0) { \ + return -1; \ + } \ + } while (0) ADD_INT_CONST("_GIL_MINSIZE", HASHLIB_GIL_MINSIZE); @@ -237,7 +237,6 @@ blake2_exec(PyObject *m) if (st->blake2b_type == NULL) { return -1; } - /* BLAKE2b */ if (PyModule_AddType(m, st->blake2b_type) < 0) { return -1; } @@ -257,9 +256,9 @@ blake2_exec(PyObject *m) st->blake2s_type = (PyTypeObject *)PyType_FromModuleAndSpec( m, &blake2s_type_spec, NULL); - if (NULL == st->blake2s_type) + if (st->blake2s_type == NULL) { return -1; - + } if (PyModule_AddType(m, st->blake2s_type) < 0) { return -1; } @@ -275,12 +274,11 @@ blake2_exec(PyObject *m) ADD_INT_CONST("BLAKE2S_MAX_KEY_SIZE", HACL_HASH_BLAKE2S_KEY_BYTES); ADD_INT_CONST("BLAKE2S_MAX_DIGEST_SIZE", HACL_HASH_BLAKE2S_OUT_BYTES); +#undef ADD_INT_CONST +#undef ADD_INT return 0; } -#undef ADD_INT -#undef ADD_INT_CONST - static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, @@ -320,65 +318,70 @@ PyInit__blake2(void) // set. typedef enum { Blake2s, Blake2b, Blake2s_128, Blake2b_256 } blake2_impl; -static inline bool is_blake2b(blake2_impl impl) { - return impl == Blake2b || impl == Blake2b_256; +static inline bool +is_blake2b(blake2_impl impl) +{ + return impl == Blake2b || impl == Blake2b_256; } -static inline bool is_blake2s(blake2_impl impl) { - return !is_blake2b(impl); +static inline bool +is_blake2s(blake2_impl impl) +{ + return impl == Blake2s || impl == Blake2s_128; } -static inline blake2_impl type_to_impl(PyTypeObject *type) { -#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) - Blake2State* st = blake2_get_state_from_type(type); +static inline blake2_impl +type_to_impl(PyTypeObject *type) +{ +#if defined(_Py_HACL_CAN_COMPILE_VEC128) || defined(_Py_HACL_CAN_COMPILE_VEC256) + Blake2State *st = blake2_get_state_from_type(type); #endif if (!strcmp(type->tp_name, blake2b_type_spec.name)) { -#ifdef HACL_CAN_COMPILE_SIMD256 - if (has_simd256(&st->flags)) - return Blake2b_256; - else -#endif +#if _Py_HACL_CAN_COMPILE_VEC256 + return st->can_run_simd256 ? Blake2b_256 : Blake2b; +#else return Blake2b; - } else if (!strcmp(type->tp_name, blake2s_type_spec.name)) { -#ifdef HACL_CAN_COMPILE_SIMD128 - if (has_simd128(&st->flags)) - return Blake2s_128; - else #endif - return Blake2s; - } else { - Py_UNREACHABLE(); } + else if (!strcmp(type->tp_name, blake2s_type_spec.name)) { +#if _Py_HACL_CAN_COMPILE_VEC128 + return st->can_run_simd128 ? Blake2s_128 : Blake2s; +#else + return Blake2s; +#endif + } + Py_UNREACHABLE(); } typedef struct { - PyObject_HEAD + HASHLIB_OBJECT_HEAD union { Hacl_Hash_Blake2s_state_t *blake2s_state; Hacl_Hash_Blake2b_state_t *blake2b_state; -#ifdef HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 Hacl_Hash_Blake2s_Simd128_state_t *blake2s_128_state; #endif -#ifdef HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 Hacl_Hash_Blake2b_Simd256_state_t *blake2b_256_state; #endif }; blake2_impl impl; - bool use_mutex; - PyMutex mutex; } Blake2Object; #define _Blake2Object_CAST(op) ((Blake2Object *)(op)) -#include "clinic/blake2module.c.h" +// --- Module clinic configuration -------------------------------------------- /*[clinic input] module _blake2 -class _blake2.blake2b "Blake2Object *" "&PyBlake2_BLAKE2bType" -class _blake2.blake2s "Blake2Object *" "&PyBlake2_BLAKE2sType" +class _blake2.blake2b "Blake2Object *" "&PyType_Type" +class _blake2.blake2s "Blake2Object *" "&PyType_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b7526666bd18af83]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=86b0972b0c41b3d0]*/ +#include "clinic/blake2module.c.h" + +// --- BLAKE-2 object interface ----------------------------------------------- static Blake2Object * new_Blake2Object(PyTypeObject *type) @@ -422,42 +425,127 @@ new_Blake2Object(PyTypeObject *type) } while (0) static void -update(Blake2Object *self, uint8_t *buf, Py_ssize_t len) +blake2_update_unlocked(Blake2Object *self, uint8_t *buf, Py_ssize_t len) { switch (self->impl) { - // These need to be ifdef'd out otherwise it's an unresolved symbol at - // link-time. -#ifdef HACL_CAN_COMPILE_SIMD256 + // blake2b_256_state and blake2s_128_state must be if'd since + // otherwise this results in an unresolved symbol at link-time. +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: - HACL_UPDATE(Hacl_Hash_Blake2b_Simd256_update,self->blake2b_256_state, buf, len); + HACL_UPDATE(Hacl_Hash_Blake2b_Simd256_update, + self->blake2b_256_state, buf, len); return; #endif -#ifdef HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: - HACL_UPDATE(Hacl_Hash_Blake2s_Simd128_update,self->blake2s_128_state, buf, len); + HACL_UPDATE(Hacl_Hash_Blake2s_Simd128_update, + self->blake2s_128_state, buf, len); return; #endif case Blake2b: - HACL_UPDATE(Hacl_Hash_Blake2b_update,self->blake2b_state, buf, len); + HACL_UPDATE(Hacl_Hash_Blake2b_update, + self->blake2b_state, buf, len); return; case Blake2s: - HACL_UPDATE(Hacl_Hash_Blake2s_update,self->blake2s_state, buf, len); + HACL_UPDATE(Hacl_Hash_Blake2s_update, + self->blake2s_state, buf, len); return; default: Py_UNREACHABLE(); } } -static PyObject * -py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, - Py_buffer *key, Py_buffer *salt, Py_buffer *person, - int fanout, int depth, unsigned long leaf_size, - unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) +#define BLAKE2_IMPLNAME(SELF) \ + (is_blake2b((SELF)->impl) ? "blake2b" : "blake2s") +#define GET_BLAKE2_CONST(SELF, NAME) \ + (is_blake2b((SELF)->impl) \ + ? HACL_HASH_BLAKE2B_ ## NAME \ + : HACL_HASH_BLAKE2S_ ## NAME) +#define MAX_OUT_BYTES(SELF) GET_BLAKE2_CONST(SELF, OUT_BYTES) +#define MAX_SALT_LENGTH(SELF) GET_BLAKE2_CONST(SELF, SALT_BYTES) +#define MAX_KEY_BYTES(SELF) GET_BLAKE2_CONST(SELF, KEY_BYTES) +#define MAX_PERSONAL_BYTES(SELF) GET_BLAKE2_CONST(SELF, PERSONAL_BYTES) + +static int +py_blake2_validate_params(Blake2Object *self, + int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size) +{ + /* Validate digest size. */ + if (digest_size <= 0 || (unsigned int)digest_size > MAX_OUT_BYTES(self)) { + PyErr_Format( + PyExc_ValueError, + "digest_size for %s must be between 1 and %d bytes, got %d", + BLAKE2_IMPLNAME(self), MAX_OUT_BYTES(self), digest_size + ); + goto error; + } + +#define CHECK_LENGTH(NAME, VALUE, MAX) \ + do { \ + if ((size_t)(VALUE) > (size_t)(MAX)) { \ + PyErr_Format(PyExc_ValueError, \ + "maximum %s length is %zu bytes, got %zd", \ + (NAME), (size_t)(MAX), (Py_ssize_t)(VALUE)); \ + goto error; \ + } \ + } while (0) + /* Validate key parameter. */ + if (key->obj && key->len) { + CHECK_LENGTH("key", key->len, MAX_KEY_BYTES(self)); + } + /* Validate salt parameter. */ + if (salt->obj && salt->len) { + CHECK_LENGTH("salt", salt->len, MAX_SALT_LENGTH(self)); + } + /* Validate personalization parameter. */ + if (person->obj && person->len) { + CHECK_LENGTH("person", person->len, MAX_PERSONAL_BYTES(self)); + } +#undef CHECK_LENGTH +#define CHECK_TREE(NAME, VALUE, MIN, MAX) \ + do { \ + if ((VALUE) < (MIN) || (size_t)(VALUE) > (size_t)(MAX)) { \ + PyErr_Format(PyExc_ValueError, \ + "'%s' must be between %zu and %zu", \ + (NAME), (size_t)(MIN), (size_t)(MAX)); \ + goto error; \ + } \ + } while (0) + /* Validate tree parameters. */ + CHECK_TREE("fanout", fanout, 0, 255); + CHECK_TREE("depth", depth, 1, 255); + CHECK_TREE("node_depth", node_depth, 0, 255); + CHECK_TREE("inner_size", inner_size, 0, MAX_OUT_BYTES(self)); +#undef CHECK_TREE + if (leaf_size > 0xFFFFFFFFU) { + /* maximum: 2**32 - 1 */ + PyErr_SetString(PyExc_OverflowError, "'leaf_size' is too large"); + goto error; + } + if (is_blake2s(self->impl) && node_offset > 0xFFFFFFFFFFFFULL) { + /* maximum: 2**48 - 1 */ + PyErr_SetString(PyExc_OverflowError, "'node_offset' is too large"); + goto error; + } + return 0; +error: + return -1; +} + + +static PyObject * +py_blake2_new(PyTypeObject *type, PyObject *data, int digest_size, + Py_buffer *key, Py_buffer *salt, Py_buffer *person, + int fanout, int depth, unsigned long leaf_size, + unsigned long long node_offset, int node_depth, + int inner_size, int last_node, int usedforsecurity) { Blake2Object *self = NULL; - Py_buffer buf; self = new_Blake2Object(type); if (self == NULL) { @@ -468,12 +556,12 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, // Ensure that the states are NULL-initialized in case of an error. // See: py_blake2_clear() for more details. switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: self->blake2b_256_state = NULL; break; #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: self->blake2s_128_state = NULL; break; @@ -487,95 +575,30 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, default: Py_UNREACHABLE(); } - // Using Blake2b because we statically know that these are greater than the - // Blake2s sizes -- this avoids a VLA. - uint8_t salt_[HACL_HASH_BLAKE2B_SALT_BYTES] = { 0 }; - uint8_t personal_[HACL_HASH_BLAKE2B_PERSONAL_BYTES] = { 0 }; - - /* Validate digest size. */ - if (digest_size <= 0 || - (unsigned) digest_size > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)) - { - PyErr_Format(PyExc_ValueError, - "digest_size for %s must be between 1 and %d bytes, here it is %d", - is_blake2b(self->impl) ? "Blake2b" : "Blake2s", - is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES, - digest_size); - goto error; - } - - /* Validate salt parameter. */ - if ((salt->obj != NULL) && salt->len) { - if ((size_t)salt->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)) { - PyErr_Format(PyExc_ValueError, - "maximum salt length is %d bytes", - (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_SALT_BYTES : HACL_HASH_BLAKE2S_SALT_BYTES)); - goto error; - } - memcpy(salt_, salt->buf, salt->len); - } - - /* Validate personalization parameter. */ - if ((person->obj != NULL) && person->len) { - if ((size_t)person->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)) { - PyErr_Format(PyExc_ValueError, - "maximum person length is %d bytes", - (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_PERSONAL_BYTES : HACL_HASH_BLAKE2S_PERSONAL_BYTES)); - goto error; - } - memcpy(personal_, person->buf, person->len); - } - - /* Validate tree parameters. */ - if (fanout < 0 || fanout > 255) { - PyErr_SetString(PyExc_ValueError, - "fanout must be between 0 and 255"); - goto error; - } - - if (depth <= 0 || depth > 255) { - PyErr_SetString(PyExc_ValueError, - "depth must be between 1 and 255"); - goto error; - } - - if (leaf_size > 0xFFFFFFFFU) { - PyErr_SetString(PyExc_OverflowError, "leaf_size is too large"); - goto error; - } - - if (is_blake2s(self->impl) && node_offset > 0xFFFFFFFFFFFFULL) { - /* maximum 2**48 - 1 */ - PyErr_SetString(PyExc_OverflowError, "node_offset is too large"); - goto error; - } - - if (node_depth < 0 || node_depth > 255) { - PyErr_SetString(PyExc_ValueError, - "node_depth must be between 0 and 255"); - goto error; - } - - if (inner_size < 0 || - (unsigned) inner_size > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)) { - PyErr_Format(PyExc_ValueError, - "inner_size must be between 0 and is %d", - (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_OUT_BYTES : HACL_HASH_BLAKE2S_OUT_BYTES)); - goto error; - } - - /* Set key length. */ - if ((key->obj != NULL) && key->len) { - if ((size_t)key->len > (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)) { - PyErr_Format(PyExc_ValueError, - "maximum key length is %d bytes", - (is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_KEY_BYTES : HACL_HASH_BLAKE2S_KEY_BYTES)); - goto error; - } - } // Unlike the state types, the parameters share a single (client-friendly) // structure. + if (py_blake2_validate_params(self, + digest_size, + key, salt, person, + fanout, depth, leaf_size, + node_offset, node_depth, inner_size) < 0) + { + goto error; + } + + // Using Blake2b because we statically know that these are greater than the + // Blake2s sizes -- this avoids a VLA. + uint8_t salt_buffer[HACL_HASH_BLAKE2B_SALT_BYTES] = {0}; + uint8_t personal_buffer[HACL_HASH_BLAKE2B_PERSONAL_BYTES] = {0}; + if (salt->obj != NULL) { + assert(salt->buf != NULL); + memcpy(salt_buffer, salt->buf, salt->len); + } + if (person->obj != NULL) { + assert(person->buf != NULL); + memcpy(personal_buffer, person->buf, person->len); + } Hacl_Hash_Blake2b_blake2_params params = { .digest_length = digest_size, @@ -586,63 +609,52 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, .node_offset = node_offset, .node_depth = node_depth, .inner_length = inner_size, - .salt = salt_, - .personal = personal_ + .salt = salt_buffer, + .personal = personal_buffer }; +#define BLAKE2_MALLOC(TYPE, STATE) \ + do { \ + STATE = Hacl_Hash_ ## TYPE ## _malloc_with_params_and_key( \ + &params, last_node, key->buf); \ + if (STATE == NULL) { \ + (void)PyErr_NoMemory(); \ + goto error; \ + } \ + } while (0) + switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 - case Blake2b_256: { - self->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key(&params, last_node, key->buf); - if (self->blake2b_256_state == NULL) { - (void)PyErr_NoMemory(); - goto error; - } +#if _Py_HACL_CAN_COMPILE_VEC256 + case Blake2b_256: + BLAKE2_MALLOC(Blake2b_Simd256, self->blake2b_256_state); break; - } #endif -#if HACL_CAN_COMPILE_SIMD128 - case Blake2s_128: { - self->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key(&params, last_node, key->buf); - if (self->blake2s_128_state == NULL) { - (void)PyErr_NoMemory(); - goto error; - } +#if _Py_HACL_CAN_COMPILE_VEC128 + case Blake2s_128: + BLAKE2_MALLOC(Blake2s_Simd128, self->blake2s_128_state); break; - } #endif - case Blake2b: { - self->blake2b_state = Hacl_Hash_Blake2b_malloc_with_params_and_key(&params, last_node, key->buf); - if (self->blake2b_state == NULL) { - (void)PyErr_NoMemory(); - goto error; - } + case Blake2b: + BLAKE2_MALLOC(Blake2b, self->blake2b_state); break; - } - case Blake2s: { - self->blake2s_state = Hacl_Hash_Blake2s_malloc_with_params_and_key(&params, last_node, key->buf); - if (self->blake2s_state == NULL) { - (void)PyErr_NoMemory(); - goto error; - } + case Blake2s: + BLAKE2_MALLOC(Blake2s, self->blake2s_state); break; - } default: Py_UNREACHABLE(); } +#undef BLAKE2_MALLOC /* Process initial data if any. */ if (data != NULL) { + Py_buffer buf; GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - - if (buf.len >= HASHLIB_GIL_MINSIZE) { - Py_BEGIN_ALLOW_THREADS - update(self, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - update(self, buf.buf, buf.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + buf.len, + blake2_update_unlocked(self, buf.buf, buf.len) + ); PyBuffer_Release(&buf); } @@ -655,8 +667,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, /*[clinic input] @classmethod _blake2.blake2b.__new__ as py_blake2b_new - data: object(c_default="NULL") = b'' - / + data as data_obj: object(c_default="NULL") = b'' * digest_size: int(c_default="HACL_HASH_BLAKE2B_OUT_BYTES") = _blake2.blake2b.MAX_DIGEST_SIZE key: Py_buffer(c_default="NULL", py_default="b''") = None @@ -670,26 +681,33 @@ _blake2.blake2b.__new__ as py_blake2b_new inner_size: int = 0 last_node: bool = False usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new BLAKE2b hash object. [clinic start generated code]*/ static PyObject * -py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2b_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=32bfd8f043c6896f input=8fee2b7b11428b2d]*/ + int inner_size, int last_node, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=de64bd850606b6a0 input=78cf60a2922d2f90]*/ { - return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } + return py_blake2_new(type, data, digest_size, key, salt, person, + fanout, depth, leaf_size, node_offset, node_depth, + inner_size, last_node, usedforsecurity); } /*[clinic input] @classmethod _blake2.blake2s.__new__ as py_blake2s_new - data: object(c_default="NULL") = b'' - / + data as data_obj: object(c_default="NULL") = b'' * digest_size: int(c_default="HACL_HASH_BLAKE2S_OUT_BYTES") = _blake2.blake2s.MAX_DIGEST_SIZE key: Py_buffer(c_default="NULL", py_default="b''") = None @@ -703,61 +721,62 @@ _blake2.blake2s.__new__ as py_blake2s_new inner_size: int = 0 last_node: bool = False usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new BLAKE2s hash object. [clinic start generated code]*/ static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2s_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=556181f73905c686 input=8165a11980eac7f3]*/ + int inner_size, int last_node, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=582a0c4295cc3a3c input=6843d6332eefd295]*/ { - return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } + return py_blake2_new(type, data, digest_size, key, salt, person, + fanout, depth, leaf_size, node_offset, node_depth, + inner_size, last_node, usedforsecurity); } static int -blake2_blake2b_copy_locked(Blake2Object *self, Blake2Object *cpy) +blake2_blake2b_copy_unlocked(Blake2Object *self, Blake2Object *cpy) { assert(cpy != NULL); +#define BLAKE2_COPY(TYPE, STATE_ATTR) \ + do { \ + cpy->STATE_ATTR = Hacl_Hash_ ## TYPE ## _copy(self->STATE_ATTR); \ + if (cpy->STATE_ATTR == NULL) { \ + goto error; \ + } \ + } while (0) + switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 - case Blake2b_256: { - cpy->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_copy(self->blake2b_256_state); - if (cpy->blake2b_256_state == NULL) { - goto error; - } +#if _Py_HACL_CAN_COMPILE_VEC256 + case Blake2b_256: + BLAKE2_COPY(Blake2b_Simd256, blake2b_256_state); break; - } #endif -#if HACL_CAN_COMPILE_SIMD128 - case Blake2s_128: { - cpy->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_copy(self->blake2s_128_state); - if (cpy->blake2s_128_state == NULL) { - goto error; - } +#if _Py_HACL_CAN_COMPILE_VEC128 + case Blake2s_128: + BLAKE2_COPY(Blake2s_Simd128, blake2s_128_state); break; - } #endif - case Blake2b: { - cpy->blake2b_state = Hacl_Hash_Blake2b_copy(self->blake2b_state); - if (cpy->blake2b_state == NULL) { - goto error; - } + case Blake2b: + BLAKE2_COPY(Blake2b, blake2b_state); break; - } - case Blake2s: { - cpy->blake2s_state = Hacl_Hash_Blake2s_copy(self->blake2s_state); - if (cpy->blake2s_state == NULL) { - goto error; - } + case Blake2s: + BLAKE2_COPY(Blake2s, blake2s_state); break; - } default: Py_UNREACHABLE(); } +#undef BLAKE2_COPY cpy->impl = self->impl; return 0; @@ -769,23 +788,25 @@ blake2_blake2b_copy_locked(Blake2Object *self, Blake2Object *cpy) /*[clinic input] _blake2.blake2b.copy + cls: defining_class + Return a copy of the hash object. [clinic start generated code]*/ static PyObject * -_blake2_blake2b_copy_impl(Blake2Object *self) -/*[clinic end generated code: output=622d1c56b91c50d8 input=e383c2d199fd8a2e]*/ +_blake2_blake2b_copy_impl(Blake2Object *self, PyTypeObject *cls) +/*[clinic end generated code: output=5f8ea31c56c52287 input=f38f3475e9aec98d]*/ { int rc; Blake2Object *cpy; - if ((cpy = new_Blake2Object(Py_TYPE(self))) == NULL) { + if ((cpy = new_Blake2Object(cls)) == NULL) { return NULL; } - ENTER_HASHLIB(self); - rc = blake2_blake2b_copy_locked(self, cpy); - LEAVE_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); + rc = blake2_blake2b_copy_unlocked(self, cpy); + HASHLIB_RELEASE_LOCK(self); if (rc < 0) { Py_DECREF(cpy); return NULL; @@ -807,27 +828,38 @@ _blake2_blake2b_update_impl(Blake2Object *self, PyObject *data) /*[clinic end generated code: output=99330230068e8c99 input=ffc4aa6a6a225d31]*/ { Py_buffer buf; - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - update(self, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - update(self, buf.buf, buf.len); - } - + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, buf.len, + blake2_update_unlocked(self, buf.buf, buf.len) + ); PyBuffer_Release(&buf); - Py_RETURN_NONE; } +static uint8_t +blake2_blake2b_compute_digest(Blake2Object *self, uint8_t *digest) +{ + switch (self->impl) { +#if _Py_HACL_CAN_COMPILE_VEC256 + case Blake2b_256: + return Hacl_Hash_Blake2b_Simd256_digest( + self->blake2b_256_state, digest); +#endif +#if _Py_HACL_CAN_COMPILE_VEC128 + case Blake2s_128: + return Hacl_Hash_Blake2s_Simd128_digest( + self->blake2s_128_state, digest); +#endif + case Blake2b: + return Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); + case Blake2s: + return Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); + default: + Py_UNREACHABLE(); + } +} + /*[clinic input] _blake2.blake2b.digest @@ -838,31 +870,10 @@ static PyObject * _blake2_blake2b_digest_impl(Blake2Object *self) /*[clinic end generated code: output=31ab8ad477f4a2f7 input=7d21659e9c5fff02]*/ { - uint8_t digest[HACL_HASH_BLAKE2B_OUT_BYTES]; - - ENTER_HASHLIB(self); - uint8_t digest_length = 0; - switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 - case Blake2b_256: - digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); - break; -#endif -#if HACL_CAN_COMPILE_SIMD128 - case Blake2s_128: - digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); - break; -#endif - case Blake2b: - digest_length = Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); - break; - case Blake2s: - digest_length = Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); - break; - default: - Py_UNREACHABLE(); - } - LEAVE_HASHLIB(self); + uint8_t digest_length = 0, digest[HACL_HASH_BLAKE2B_OUT_BYTES]; + HASHLIB_ACQUIRE_LOCK(self); + digest_length = blake2_blake2b_compute_digest(self, digest); + HASHLIB_RELEASE_LOCK(self); return PyBytes_FromStringAndSize((const char *)digest, digest_length); } @@ -876,31 +887,10 @@ static PyObject * _blake2_blake2b_hexdigest_impl(Blake2Object *self) /*[clinic end generated code: output=5ef54b138db6610a input=76930f6946351f56]*/ { - uint8_t digest[HACL_HASH_BLAKE2B_OUT_BYTES]; - - ENTER_HASHLIB(self); - uint8_t digest_length = 0; - switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 - case Blake2b_256: - digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); - break; -#endif -#if HACL_CAN_COMPILE_SIMD128 - case Blake2s_128: - digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); - break; -#endif - case Blake2b: - digest_length = Hacl_Hash_Blake2b_digest(self->blake2b_state, digest); - break; - case Blake2s: - digest_length = Hacl_Hash_Blake2s_digest(self->blake2s_state, digest); - break; - default: - Py_UNREACHABLE(); - } - LEAVE_HASHLIB(self); + uint8_t digest_length = 0, digest[HACL_HASH_BLAKE2B_OUT_BYTES]; + HASHLIB_ACQUIRE_LOCK(self); + digest_length = blake2_blake2b_compute_digest(self, digest); + HASHLIB_RELEASE_LOCK(self); return _Py_strhex((const char *)digest, digest_length); } @@ -918,40 +908,46 @@ static PyObject * py_blake2b_get_name(PyObject *op, void *Py_UNUSED(closure)) { Blake2Object *self = _Blake2Object_CAST(op); - return PyUnicode_FromString(is_blake2b(self->impl) ? "blake2b" : "blake2s"); + return PyUnicode_FromString(BLAKE2_IMPLNAME(self)); } - static PyObject * py_blake2b_get_block_size(PyObject *op, void *Py_UNUSED(closure)) { Blake2Object *self = _Blake2Object_CAST(op); - return PyLong_FromLong(is_blake2b(self->impl) ? HACL_HASH_BLAKE2B_BLOCK_BYTES : HACL_HASH_BLAKE2S_BLOCK_BYTES); + return PyLong_FromLong(GET_BLAKE2_CONST(self, BLOCK_BYTES)); } +static Hacl_Hash_Blake2b_index +hacl_get_blake2_info(Blake2Object *self) +{ + switch (self->impl) { +#if _Py_HACL_CAN_COMPILE_VEC256 + case Blake2b_256: + return Hacl_Hash_Blake2b_Simd256_info(self->blake2b_256_state); +#endif +#if _Py_HACL_CAN_COMPILE_VEC128 + case Blake2s_128: + return Hacl_Hash_Blake2s_Simd128_info(self->blake2s_128_state); +#endif + case Blake2b: + return Hacl_Hash_Blake2b_info(self->blake2b_state); + case Blake2s: + return Hacl_Hash_Blake2s_info(self->blake2s_state); + default: + Py_UNREACHABLE(); + } +} + static PyObject * py_blake2b_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) { Blake2Object *self = _Blake2Object_CAST(op); - switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 - case Blake2b_256: - return PyLong_FromLong(Hacl_Hash_Blake2b_Simd256_info(self->blake2b_256_state).digest_length); -#endif -#if HACL_CAN_COMPILE_SIMD128 - case Blake2s_128: - return PyLong_FromLong(Hacl_Hash_Blake2s_Simd128_info(self->blake2s_128_state).digest_length); -#endif - case Blake2b: - return PyLong_FromLong(Hacl_Hash_Blake2b_info(self->blake2b_state).digest_length); - case Blake2s: - return PyLong_FromLong(Hacl_Hash_Blake2s_info(self->blake2s_state).digest_length); - default: - Py_UNREACHABLE(); - } + Hacl_Hash_Blake2b_index info = hacl_get_blake2_info(self); + return PyLong_FromLong(info.digest_length); } @@ -971,38 +967,35 @@ py_blake2_clear(PyObject *op) // initializes the HACL* internal state to NULL before allocating // it. If an error occurs in the constructor, we should only free // states that were allocated (i.e. that are not NULL). +#define BLAKE2_FREE(TYPE, STATE) \ + do { \ + if (STATE != NULL) { \ + Hacl_Hash_ ## TYPE ## _free(STATE); \ + STATE = NULL; \ + } \ + } while (0) + switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: - if (self->blake2b_256_state != NULL) { - Hacl_Hash_Blake2b_Simd256_free(self->blake2b_256_state); - self->blake2b_256_state = NULL; - } + BLAKE2_FREE(Blake2b_Simd256, self->blake2b_256_state); break; #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: - if (self->blake2s_128_state != NULL) { - Hacl_Hash_Blake2s_Simd128_free(self->blake2s_128_state); - self->blake2s_128_state = NULL; - } + BLAKE2_FREE(Blake2s_Simd128, self->blake2s_128_state); break; #endif case Blake2b: - if (self->blake2b_state != NULL) { - Hacl_Hash_Blake2b_free(self->blake2b_state); - self->blake2b_state = NULL; - } + BLAKE2_FREE(Blake2b, self->blake2b_state); break; case Blake2s: - if (self->blake2s_state != NULL) { - Hacl_Hash_Blake2s_free(self->blake2s_state); - self->blake2s_state = NULL; - } + BLAKE2_FREE(Blake2s, self->blake2s_state); break; default: Py_UNREACHABLE(); } +#undef BLAKE2_FREE return 0; } @@ -1016,40 +1009,33 @@ py_blake2_dealloc(PyObject *self) Py_DECREF(type); } -static int -py_blake2_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static PyType_Slot blake2b_type_slots[] = { {Py_tp_clear, py_blake2_clear}, {Py_tp_dealloc, py_blake2_dealloc}, - {Py_tp_traverse, py_blake2_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_doc, (char *)py_blake2b_new__doc__}, {Py_tp_methods, py_blake2b_methods}, {Py_tp_getset, py_blake2b_getsetters}, {Py_tp_new, py_blake2b_new}, - {0,0} + {0, 0} }; static PyType_Slot blake2s_type_slots[] = { {Py_tp_clear, py_blake2_clear}, {Py_tp_dealloc, py_blake2_dealloc}, - {Py_tp_traverse, py_blake2_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_doc, (char *)py_blake2s_new__doc__}, {Py_tp_methods, py_blake2b_methods}, {Py_tp_getset, py_blake2b_getsetters}, // only the constructor differs, so that it can receive a clinic-generated // default digest length suitable for blake2s {Py_tp_new, py_blake2s_new}, - {0,0} + {0, 0} }; static PyType_Spec blake2b_type_spec = { .name = "_blake2.blake2b", - .basicsize = sizeof(Blake2Object), + .basicsize = sizeof(Blake2Object), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, .slots = blake2b_type_slots @@ -1057,7 +1043,7 @@ static PyType_Spec blake2b_type_spec = { static PyType_Spec blake2s_type_spec = { .name = "_blake2.blake2s", - .basicsize = sizeof(Blake2Object), + .basicsize = sizeof(Blake2Object), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HEAPTYPE, .slots = blake2s_type_slots diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index ef6faeb7127..b1984df2695 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -802,10 +802,13 @@ jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data, return coded; case 2: /* second character of unicode pair */ - coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1], - jisx0213_pair_encmap, JISX0213_ENCPAIRS); - if (coded != DBCINV) - return coded; + if (data[1] != 0) { /* Don't consume null char as part of pair */ + coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1], + jisx0213_pair_encmap, JISX0213_ENCPAIRS); + if (coded != DBCINV) { + return coded; + } + } _Py_FALLTHROUGH; case -1: /* flush unterminated */ diff --git a/Modules/cjkcodecs/_codecs_jp.c b/Modules/cjkcodecs/_codecs_jp.c index f7127487aa5..cd77888d551 100644 --- a/Modules/cjkcodecs/_codecs_jp.c +++ b/Modules/cjkcodecs/_codecs_jp.c @@ -192,8 +192,11 @@ ENCODER(euc_jis_2004) JISX0213_ENCPAIRS); if (code == DBCINV) return 1; - } else + } + else if (c2 != 0) { + /* Don't consume null char as part of pair */ insize = 2; + } } } } @@ -611,8 +614,10 @@ ENCODER(shift_jis_2004) if (code == DBCINV) return 1; } - else + else if (ch2 != 0) { + /* Don't consume null char as part of pair */ insize = 2; + } } } } diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 08b74740bda..a7fac2380f2 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -590,6 +590,7 @@ multibytecodec_encode(const MultibyteCodec *codec, } /*[clinic input] +@permit_long_docstring_body _multibytecodec.MultibyteCodec.encode input: object @@ -607,7 +608,7 @@ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=2841745b95ed338f]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=0980aede2c564df8]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -655,6 +656,7 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, } /*[clinic input] +@permit_long_docstring_body _multibytecodec.MultibyteCodec.decode input: Py_buffer @@ -672,7 +674,7 @@ static PyObject * _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, Py_buffer *input, const char *errors) -/*[clinic end generated code: output=ff419f65bad6cc77 input=e0c78fc7ab190def]*/ +/*[clinic end generated code: output=ff419f65bad6cc77 input=2c657ef914600c7c]*/ { MultibyteCodec_State state; MultibyteDecodeBuffer buf; diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 314208bc41d..8f3492cd54b 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -93,6 +93,11 @@ _bisect_bisect_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto exit; } lo = ival; + if (lo < 0) { + PyErr_SetString(PyExc_ValueError, + "lo cannot be negative"); + goto exit; + } } if (!--noptargs) { goto skip_optional_pos; @@ -203,6 +208,11 @@ _bisect_insort_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto exit; } lo = ival; + if (lo < 0) { + PyErr_SetString(PyExc_ValueError, + "lo cannot be negative"); + goto exit; + } } if (!--noptargs) { goto skip_optional_pos; @@ -312,6 +322,11 @@ _bisect_bisect_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } lo = ival; + if (lo < 0) { + PyErr_SetString(PyExc_ValueError, + "lo cannot be negative"); + goto exit; + } } if (!--noptargs) { goto skip_optional_pos; @@ -422,6 +437,11 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P goto exit; } lo = ival; + if (lo < 0) { + PyErr_SetString(PyExc_ValueError, + "lo cannot be negative"); + goto exit; + } } if (!--noptargs) { goto skip_optional_pos; @@ -446,4 +466,4 @@ skip_optional_kwonly: exit: return return_value; } -/*[clinic end generated code: output=729385c6a23828ab input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a3c44ed440dd6d81 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index b0310325759..9e2a7950ebd 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -2779,6 +2779,70 @@ exit: return return_value; } +PyDoc_STRVAR(_codecs__normalize_encoding__doc__, +"_normalize_encoding($module, /, encoding)\n" +"--\n" +"\n" +"Normalize an encoding name *encoding*.\n" +"\n" +"Used for encodings.normalize_encoding. Does not convert to lower case."); + +#define _CODECS__NORMALIZE_ENCODING_METHODDEF \ + {"_normalize_encoding", _PyCFunction_CAST(_codecs__normalize_encoding), METH_FASTCALL|METH_KEYWORDS, _codecs__normalize_encoding__doc__}, + +static PyObject * +_codecs__normalize_encoding_impl(PyObject *module, PyObject *encoding); + +static PyObject * +_codecs__normalize_encoding(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(encoding), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"encoding", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_normalize_encoding", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *encoding; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("_normalize_encoding", "argument 'encoding'", "str", args[0]); + goto exit; + } + encoding = args[0]; + return_value = _codecs__normalize_encoding_impl(module, encoding); + +exit: + return return_value; +} + #ifndef _CODECS_MBCS_DECODE_METHODDEF #define _CODECS_MBCS_DECODE_METHODDEF #endif /* !defined(_CODECS_MBCS_DECODE_METHODDEF) */ @@ -2802,4 +2866,4 @@ exit: #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=ed13f20dfb09e306 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a968c493bb28be3e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_csv.c.h b/Modules/clinic/_csv.c.h index 416aeafccf9..b8dd8ac35fa 100644 --- a/Modules/clinic/_csv.c.h +++ b/Modules/clinic/_csv.c.h @@ -12,9 +12,7 @@ PyDoc_STRVAR(_csv_list_dialects__doc__, "list_dialects($module, /)\n" "--\n" "\n" -"Return a list of all known dialect names.\n" -"\n" -" names = csv.list_dialects()"); +"Return a list of all known dialect names."); #define _CSV_LIST_DIALECTS_METHODDEF \ {"list_dialects", (PyCFunction)_csv_list_dialects, METH_NOARGS, _csv_list_dialects__doc__}, @@ -32,9 +30,7 @@ PyDoc_STRVAR(_csv_unregister_dialect__doc__, "unregister_dialect($module, /, name)\n" "--\n" "\n" -"Delete the name/dialect mapping associated with a string name.\n" -"\n" -" csv.unregister_dialect(name)"); +"Delete the name/dialect mapping associated with a string name."); #define _CSV_UNREGISTER_DIALECT_METHODDEF \ {"unregister_dialect", _PyCFunction_CAST(_csv_unregister_dialect), METH_FASTCALL|METH_KEYWORDS, _csv_unregister_dialect__doc__}, @@ -92,9 +88,7 @@ PyDoc_STRVAR(_csv_get_dialect__doc__, "get_dialect($module, /, name)\n" "--\n" "\n" -"Return the dialect instance associated with name.\n" -"\n" -" dialect = csv.get_dialect(name)"); +"Return the dialect instance associated with name."); #define _CSV_GET_DIALECT_METHODDEF \ {"get_dialect", _PyCFunction_CAST(_csv_get_dialect), METH_FASTCALL|METH_KEYWORDS, _csv_get_dialect__doc__}, @@ -154,8 +148,6 @@ PyDoc_STRVAR(_csv_field_size_limit__doc__, "\n" "Sets an upper limit on parsed fields.\n" "\n" -" csv.field_size_limit([limit])\n" -"\n" "Returns old limit. If limit is not given, no new limit is set and\n" "the old limit is returned"); @@ -215,4 +207,4 @@ skip_optional_pos: exit: return return_value; } -/*[clinic end generated code: output=1fb09d5e7667ad0d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ed77cb69fad9f3b4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 6f4966825ec..75cf067c8aa 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -2,10 +2,7 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_runtime.h" // _Py_SINGLETON() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, "bottom($self, /)\n" @@ -14,19 +11,15 @@ PyDoc_STRVAR(_curses_panel_panel_bottom__doc__, "Push the panel to the bottom of the stack."); #define _CURSES_PANEL_PANEL_BOTTOM_METHODDEF \ - {"bottom", _PyCFunction_CAST(_curses_panel_panel_bottom), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_bottom__doc__}, + {"bottom", (PyCFunction)_curses_panel_panel_bottom, METH_NOARGS, _curses_panel_panel_bottom__doc__}, static PyObject * -_curses_panel_panel_bottom_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_bottom_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_bottom(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_bottom(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "bottom() takes no arguments"); - return NULL; - } - return _curses_panel_panel_bottom_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_bottom_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_hide__doc__, @@ -38,19 +31,15 @@ PyDoc_STRVAR(_curses_panel_panel_hide__doc__, "This does not delete the object, it just makes the window on screen invisible."); #define _CURSES_PANEL_PANEL_HIDE_METHODDEF \ - {"hide", _PyCFunction_CAST(_curses_panel_panel_hide), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_hide__doc__}, + {"hide", (PyCFunction)_curses_panel_panel_hide, METH_NOARGS, _curses_panel_panel_hide__doc__}, static PyObject * -_curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_hide_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_hide(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_hide(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "hide() takes no arguments"); - return NULL; - } - return _curses_panel_panel_hide_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_hide_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_show__doc__, @@ -60,19 +49,15 @@ PyDoc_STRVAR(_curses_panel_panel_show__doc__, "Display the panel (which might have been hidden)."); #define _CURSES_PANEL_PANEL_SHOW_METHODDEF \ - {"show", _PyCFunction_CAST(_curses_panel_panel_show), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_show__doc__}, + {"show", (PyCFunction)_curses_panel_panel_show, METH_NOARGS, _curses_panel_panel_show__doc__}, static PyObject * -_curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_show_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_show(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_show(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "show() takes no arguments"); - return NULL; - } - return _curses_panel_panel_show_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_show_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_top__doc__, @@ -82,19 +67,15 @@ PyDoc_STRVAR(_curses_panel_panel_top__doc__, "Push panel to the top of the stack."); #define _CURSES_PANEL_PANEL_TOP_METHODDEF \ - {"top", _PyCFunction_CAST(_curses_panel_panel_top), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_top__doc__}, + {"top", (PyCFunction)_curses_panel_panel_top, METH_NOARGS, _curses_panel_panel_top__doc__}, static PyObject * -_curses_panel_panel_top_impl(PyCursesPanelObject *self, PyTypeObject *cls); +_curses_panel_panel_top_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_top(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_top(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "top() takes no arguments"); - return NULL; - } - return _curses_panel_panel_top_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_top_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_panel_above__doc__, @@ -158,36 +139,19 @@ PyDoc_STRVAR(_curses_panel_panel_move__doc__, "Move the panel to the screen coordinates (y, x)."); #define _CURSES_PANEL_PANEL_MOVE_METHODDEF \ - {"move", _PyCFunction_CAST(_curses_panel_panel_move), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_move__doc__}, + {"move", _PyCFunction_CAST(_curses_panel_panel_move), METH_FASTCALL, _curses_panel_panel_move__doc__}, static PyObject * -_curses_panel_panel_move_impl(PyCursesPanelObject *self, PyTypeObject *cls, - int y, int x); +_curses_panel_panel_move_impl(PyCursesPanelObject *self, int y, int x); static PyObject * -_curses_panel_panel_move(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_move(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - - static const char * const _keywords[] = {"", "", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "move", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; int y; int x; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("move", nargs, 2, 2)) { goto exit; } y = PyLong_AsInt(args[0]); @@ -198,7 +162,7 @@ _curses_panel_panel_move(PyObject *self, PyTypeObject *cls, PyObject *const *arg if (x == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_panel_panel_move_impl((PyCursesPanelObject *)self, cls, y, x); + return_value = _curses_panel_panel_move_impl((PyCursesPanelObject *)self, y, x); exit: return return_value; @@ -229,44 +193,24 @@ PyDoc_STRVAR(_curses_panel_panel_replace__doc__, "Change the window associated with the panel to the window win."); #define _CURSES_PANEL_PANEL_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(_curses_panel_panel_replace), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_replace__doc__}, + {"replace", (PyCFunction)_curses_panel_panel_replace, METH_O, _curses_panel_panel_replace__doc__}, static PyObject * _curses_panel_panel_replace_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyCursesWindowObject *win); static PyObject * -_curses_panel_panel_replace(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_replace(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - - static const char * const _keywords[] = {"", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "replace", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; PyCursesWindowObject *win; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { + if (!PyObject_TypeCheck(arg, &PyCursesWindow_Type)) { + _PyArg_BadArgument("replace", "argument", (&PyCursesWindow_Type)->tp_name, arg); goto exit; } - if (!PyObject_TypeCheck(args[0], &PyCursesWindow_Type)) { - _PyArg_BadArgument("replace", "argument 1", (&PyCursesWindow_Type)->tp_name, args[0]); - goto exit; - } - win = (PyCursesWindowObject *)args[0]; - return_value = _curses_panel_panel_replace_impl((PyCursesPanelObject *)self, cls, win); + win = (PyCursesWindowObject *)arg; + return_value = _curses_panel_panel_replace_impl((PyCursesPanelObject *)self, win); exit: return return_value; @@ -279,41 +223,19 @@ PyDoc_STRVAR(_curses_panel_panel_set_userptr__doc__, "Set the panel\'s user pointer to obj."); #define _CURSES_PANEL_PANEL_SET_USERPTR_METHODDEF \ - {"set_userptr", _PyCFunction_CAST(_curses_panel_panel_set_userptr), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_set_userptr__doc__}, + {"set_userptr", (PyCFunction)_curses_panel_panel_set_userptr, METH_O, _curses_panel_panel_set_userptr__doc__}, static PyObject * _curses_panel_panel_set_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls, PyObject *obj); + PyObject *obj); static PyObject * -_curses_panel_panel_set_userptr(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_set_userptr(PyObject *self, PyObject *obj) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) - #else - # define KWTUPLE NULL - #endif - static const char * const _keywords[] = {"", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "set_userptr", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *obj; + return_value = _curses_panel_panel_set_userptr_impl((PyCursesPanelObject *)self, obj); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - obj = args[0]; - return_value = _curses_panel_panel_set_userptr_impl((PyCursesPanelObject *)self, cls, obj); - -exit: return return_value; } @@ -324,20 +246,15 @@ PyDoc_STRVAR(_curses_panel_panel_userptr__doc__, "Return the user pointer for the panel."); #define _CURSES_PANEL_PANEL_USERPTR_METHODDEF \ - {"userptr", _PyCFunction_CAST(_curses_panel_panel_userptr), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_userptr__doc__}, + {"userptr", (PyCFunction)_curses_panel_panel_userptr, METH_NOARGS, _curses_panel_panel_userptr__doc__}, static PyObject * -_curses_panel_panel_userptr_impl(PyCursesPanelObject *self, - PyTypeObject *cls); +_curses_panel_panel_userptr_impl(PyCursesPanelObject *self); static PyObject * -_curses_panel_panel_userptr(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_curses_panel_panel_userptr(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { - PyErr_SetString(PyExc_TypeError, "userptr() takes no arguments"); - return NULL; - } - return _curses_panel_panel_userptr_impl((PyCursesPanelObject *)self, cls); + return _curses_panel_panel_userptr_impl((PyCursesPanelObject *)self); } PyDoc_STRVAR(_curses_panel_bottom_panel__doc__, @@ -424,4 +341,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=36853ecb4a979814 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db2fe491582784aa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 552360eb80a..e6f9798cdf1 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -301,7 +301,7 @@ PyDoc_STRVAR(_curses_window_attron__doc__, "attron($self, attr, /)\n" "--\n" "\n" -"Add attribute attr from the \"background\" set."); +"Add attribute attr to the \"background\" set."); #define _CURSES_WINDOW_ATTRON_METHODDEF \ {"attron", (PyCFunction)_curses_window_attron, METH_O, _curses_window_attron__doc__}, @@ -733,23 +733,13 @@ PyDoc_STRVAR(_curses_window_getbkgd__doc__, #define _CURSES_WINDOW_GETBKGD_METHODDEF \ {"getbkgd", (PyCFunction)_curses_window_getbkgd, METH_NOARGS, _curses_window_getbkgd__doc__}, -static long +static PyObject * _curses_window_getbkgd_impl(PyCursesWindowObject *self); static PyObject * _curses_window_getbkgd(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *return_value = NULL; - long _return_value; - - _return_value = _curses_window_getbkgd_impl((PyCursesWindowObject *)self); - if ((_return_value == -1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromLong(_return_value); - -exit: - return return_value; + return _curses_window_getbkgd_impl((PyCursesWindowObject *)self); } PyDoc_STRVAR(_curses_window_getch__doc__, @@ -768,7 +758,7 @@ PyDoc_STRVAR(_curses_window_getch__doc__, #define _CURSES_WINDOW_GETCH_METHODDEF \ {"getch", (PyCFunction)_curses_window_getch, METH_VARARGS, _curses_window_getch__doc__}, -static int +static PyObject * _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x); @@ -779,7 +769,6 @@ _curses_window_getch(PyObject *self, PyObject *args) int group_right_1 = 0; int y = 0; int x = 0; - int _return_value; switch (PyTuple_GET_SIZE(args)) { case 0: @@ -794,11 +783,7 @@ _curses_window_getch(PyObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.getch requires 0 to 2 arguments"); goto exit; } - _return_value = _curses_window_getch_impl((PyCursesWindowObject *)self, group_right_1, y, x); - if ((_return_value == -1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromLong((long)_return_value); + return_value = _curses_window_getch_impl((PyCursesWindowObject *)self, group_right_1, y, x); exit: return return_value; @@ -1055,7 +1040,7 @@ PyDoc_STRVAR(_curses_window_inch__doc__, #define _CURSES_WINDOW_INCH_METHODDEF \ {"inch", (PyCFunction)_curses_window_inch, METH_VARARGS, _curses_window_inch__doc__}, -static unsigned long +static PyObject * _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x); @@ -1066,7 +1051,6 @@ _curses_window_inch(PyObject *self, PyObject *args) int group_right_1 = 0; int y = 0; int x = 0; - unsigned long _return_value; switch (PyTuple_GET_SIZE(args)) { case 0: @@ -1081,11 +1065,7 @@ _curses_window_inch(PyObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.inch requires 0 to 2 arguments"); goto exit; } - _return_value = _curses_window_inch_impl((PyCursesWindowObject *)self, group_right_1, y, x); - if ((_return_value == (unsigned long)-1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromUnsignedLong(_return_value); + return_value = _curses_window_inch_impl((PyCursesWindowObject *)self, group_right_1, y, x); exit: return return_value; @@ -2392,7 +2372,22 @@ _curses_ungetmouse(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("ungetmouse", "argument 5", "int", args[4]); goto exit; } - bstate = PyLong_AsUnsignedLongMask(args[4]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[4], &bstate, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } return_value = _curses_ungetmouse_impl(module, id, x, y, z, bstate); exit: @@ -3158,7 +3153,22 @@ _curses_mousemask(PyObject *module, PyObject *arg) _PyArg_BadArgument("mousemask", "argument", "int", arg); goto exit; } - newmask = PyLong_AsUnsignedLongMask(arg); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(arg, &newmask, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } return_value = _curses_mousemask_impl(module, newmask); exit: @@ -4440,4 +4450,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=42b2923d88c8d0f6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=135246e29163510c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 18e6129fad8..ee621c150c3 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -8,6 +8,206 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(delta_new__doc__, +"timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0,\n" +" hours=0, weeks=0)\n" +"--\n" +"\n" +"Difference between two datetime values.\n" +"\n" +"All arguments are optional and default to 0.\n" +"Arguments may be integers or floats, and may be positive or negative."); + +static PyObject * +delta_new_impl(PyTypeObject *type, PyObject *days, PyObject *seconds, + PyObject *microseconds, PyObject *milliseconds, + PyObject *minutes, PyObject *hours, PyObject *weeks); + +static PyObject * +delta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 7 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(days), &_Py_ID(seconds), &_Py_ID(microseconds), &_Py_ID(milliseconds), &_Py_ID(minutes), &_Py_ID(hours), &_Py_ID(weeks), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"days", "seconds", "microseconds", "milliseconds", "minutes", "hours", "weeks", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timedelta", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[7]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *days = NULL; + PyObject *seconds = NULL; + PyObject *microseconds = NULL; + PyObject *milliseconds = NULL; + PyObject *minutes = NULL; + PyObject *hours = NULL; + PyObject *weeks = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 7, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + days = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + seconds = fastargs[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + microseconds = fastargs[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[3]) { + milliseconds = fastargs[3]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + minutes = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[5]) { + hours = fastargs[5]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + weeks = fastargs[6]; +skip_optional_pos: + return_value = delta_new_impl(type, days, seconds, microseconds, milliseconds, minutes, hours, weeks); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date__doc__, +"date(year, month, day)\n" +"--\n" +"\n" +"Concrete date type."); + +static PyObject * +datetime_date_impl(PyTypeObject *type, int year, int month, int day); + +static PyObject * +datetime_date(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "date", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + int year; + int month; + int day; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + year = PyLong_AsInt(fastargs[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + month = PyLong_AsInt(fastargs[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + day = PyLong_AsInt(fastargs[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_date_impl(type, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_today__doc__, +"today($type, /)\n" +"--\n" +"\n" +"Current date or datetime.\n" +"\n" +"Equivalent to fromtimestamp(time.time())."); + +#define DATETIME_DATE_TODAY_METHODDEF \ + {"today", (PyCFunction)datetime_date_today, METH_NOARGS|METH_CLASS, datetime_date_today__doc__}, + +static PyObject * +datetime_date_today_impl(PyTypeObject *type); + +static PyObject * +datetime_date_today(PyObject *type, PyObject *Py_UNUSED(ignored)) +{ + return datetime_date_today_impl((PyTypeObject *)type); +} + PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "fromtimestamp($type, timestamp, /)\n" "--\n" @@ -33,6 +233,278 @@ datetime_date_fromtimestamp(PyObject *type, PyObject *timestamp) return return_value; } +PyDoc_STRVAR(datetime_date_fromordinal__doc__, +"fromordinal($type, ordinal, /)\n" +"--\n" +"\n" +"Construct a date from a proleptic Gregorian ordinal.\n" +"\n" +"January 1 of year 1 is day 1. Only the year, month and day are\n" +"non-zero in the result."); + +#define DATETIME_DATE_FROMORDINAL_METHODDEF \ + {"fromordinal", (PyCFunction)datetime_date_fromordinal, METH_O|METH_CLASS, datetime_date_fromordinal__doc__}, + +static PyObject * +datetime_date_fromordinal_impl(PyTypeObject *type, int ordinal); + +static PyObject * +datetime_date_fromordinal(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + int ordinal; + + ordinal = PyLong_AsInt(arg); + if (ordinal == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_date_fromordinal_impl((PyTypeObject *)type, ordinal); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" +"--\n" +"\n" +"Construct a date from a string in ISO 8601 format."); + +#define DATETIME_DATE_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_date_fromisoformat, METH_O|METH_CLASS, datetime_date_fromisoformat__doc__}, + +static PyObject * +datetime_date_fromisoformat_impl(PyTypeObject *type, PyObject *string); + +static PyObject * +datetime_date_fromisoformat(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); + goto exit; + } + string = arg; + return_value = datetime_date_fromisoformat_impl((PyTypeObject *)type, string); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_fromisocalendar__doc__, +"fromisocalendar($type, /, year, week, day)\n" +"--\n" +"\n" +"Construct a date from the ISO year, week number and weekday.\n" +"\n" +"This is the inverse of the date.isocalendar() function."); + +#define DATETIME_DATE_FROMISOCALENDAR_METHODDEF \ + {"fromisocalendar", _PyCFunction_CAST(datetime_date_fromisocalendar), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_date_fromisocalendar__doc__}, + +static PyObject * +datetime_date_fromisocalendar_impl(PyTypeObject *type, int year, int week, + int day); + +static PyObject * +datetime_date_fromisocalendar(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(week), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "week", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fromisocalendar", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + int year; + int week; + int day; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + week = PyLong_AsInt(args[1]); + if (week == -1 && PyErr_Occurred()) { + goto exit; + } + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_date_fromisocalendar_impl((PyTypeObject *)type, year, week, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_strptime__doc__, +"strptime($type, string, format, /)\n" +"--\n" +"\n" +"Parse string according to the given date format (like time.strptime()).\n" +"\n" +"For a list of supported format codes, see the documentation:\n" +" https://docs.python.org/3/library/datetime.html#format-codes"); + +#define DATETIME_DATE_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_date_strptime), METH_FASTCALL|METH_CLASS, datetime_date_strptime__doc__}, + +static PyObject * +datetime_date_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_date_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_date_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_strftime__doc__, +"strftime($self, /, format)\n" +"--\n" +"\n" +"Format using strftime().\n" +"\n" +"Example: \"%d/%m/%Y, %H:%M:%S\".\n" +"\n" +"For a list of supported format codes, see the documentation:\n" +" https://docs.python.org/3/library/datetime.html#format-codes"); + +#define DATETIME_DATE_STRFTIME_METHODDEF \ + {"strftime", _PyCFunction_CAST(datetime_date_strftime), METH_FASTCALL|METH_KEYWORDS, datetime_date_strftime__doc__}, + +static PyObject * +datetime_date_strftime_impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_date_strftime(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(format), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"format", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "strftime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *format; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strftime", "argument 'format'", "str", args[0]); + goto exit; + } + format = args[0]; + return_value = datetime_date_strftime_impl(self, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date___format____doc__, +"__format__($self, format, /)\n" +"--\n" +"\n" +"Formats self with strftime."); + +#define DATETIME_DATE___FORMAT___METHODDEF \ + {"__format__", (PyCFunction)datetime_date___format__, METH_O, datetime_date___format____doc__}, + +static PyObject * +datetime_date___format___impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_date___format__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *format; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("__format__", "argument", "str", arg); + goto exit; + } + format = arg; + return_value = datetime_date___format___impl(self, format); + +exit: + return return_value; +} + static PyObject * iso_calendar_date_new_impl(PyTypeObject *type, int year, int week, int weekday); @@ -185,6 +657,418 @@ exit: return return_value; } +PyDoc_STRVAR(timezone_new__doc__, +"timezone(offset, name=<unrepresentable>)\n" +"--\n" +"\n" +"Fixed offset from UTC implementation of tzinfo."); + +static PyObject * +timezone_new_impl(PyTypeObject *type, PyObject *offset, PyObject *name); + +static PyObject * +timezone_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(offset), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"offset", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timezone", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; + PyObject *offset; + PyObject *name = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!PyObject_TypeCheck(fastargs[0], DELTA_TYPE(NO_STATE))) { + _PyArg_BadArgument("timezone", "argument 'offset'", (DELTA_TYPE(NO_STATE))->tp_name, fastargs[0]); + goto exit; + } + offset = fastargs[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (!PyUnicode_Check(fastargs[1])) { + _PyArg_BadArgument("timezone", "argument 'name'", "str", fastargs[1]); + goto exit; + } + name = fastargs[1]; +skip_optional_pos: + return_value = timezone_new_impl(type, offset, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time__doc__, +"time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0)\n" +"--\n" +"\n" +"Time with time zone.\n" +"\n" +"All arguments are optional. tzinfo may be None, or an instance of\n" +"a tzinfo subclass. The remaining arguments may be ints."); + +static PyObject * +datetime_time_impl(PyTypeObject *type, int hour, int minute, int second, + int microsecond, PyObject *tzinfo, int fold); + +static PyObject * +datetime_time(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "time", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; + PyObject *tzinfo = Py_None; + int fold = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + hour = PyLong_AsInt(fastargs[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + minute = PyLong_AsInt(fastargs[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + second = PyLong_AsInt(fastargs[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[3]) { + microsecond = PyLong_AsInt(fastargs[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + tzinfo = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(fastargs[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_impl(type, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_strptime__doc__, +"strptime($type, string, format, /)\n" +"--\n" +"\n" +"Parse string according to the given time format (like time.strptime()).\n" +"\n" +"For a list of supported format codes, see the documentation:\n" +" https://docs.python.org/3/library/datetime.html#format-codes"); + +#define DATETIME_TIME_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_time_strptime), METH_FASTCALL|METH_CLASS, datetime_time_strptime__doc__}, + +static PyObject * +datetime_time_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_time_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_time_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_isoformat__doc__, +"isoformat($self, /, timespec=\'auto\')\n" +"--\n" +"\n" +"Return the time formatted according to ISO.\n" +"\n" +"The full format is \'HH:MM:SS.mmmmmm+zz:zz\'. By default, the fractional\n" +"part is omitted if self.microsecond == 0.\n" +"\n" +"The optional argument timespec specifies the number of additional\n" +"terms of the time to include. Valid options are \'auto\', \'hours\',\n" +"\'minutes\', \'seconds\', \'milliseconds\' and \'microseconds\'."); + +#define DATETIME_TIME_ISOFORMAT_METHODDEF \ + {"isoformat", _PyCFunction_CAST(datetime_time_isoformat), METH_FASTCALL|METH_KEYWORDS, datetime_time_isoformat__doc__}, + +static PyObject * +datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec); + +static PyObject * +datetime_time_isoformat(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(timespec), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timespec", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "isoformat", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + const char *timespec = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("isoformat", "argument 'timespec'", "str", args[0]); + goto exit; + } + Py_ssize_t timespec_length; + timespec = PyUnicode_AsUTF8AndSize(args[0], &timespec_length); + if (timespec == NULL) { + goto exit; + } + if (strlen(timespec) != (size_t)timespec_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_pos: + return_value = datetime_time_isoformat_impl((PyDateTime_Time *)self, timespec); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_strftime__doc__, +"strftime($self, /, format)\n" +"--\n" +"\n" +"Format using strftime().\n" +"\n" +"The date part of the timestamp passed to underlying strftime should not be used.\n" +"\n" +"For a list of supported format codes, see the documentation:\n" +" https://docs.python.org/3/library/datetime.html#format-codes"); + +#define DATETIME_TIME_STRFTIME_METHODDEF \ + {"strftime", _PyCFunction_CAST(datetime_time_strftime), METH_FASTCALL|METH_KEYWORDS, datetime_time_strftime__doc__}, + +static PyObject * +datetime_time_strftime_impl(PyDateTime_Time *self, PyObject *format); + +static PyObject * +datetime_time_strftime(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(format), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"format", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "strftime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *format; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strftime", "argument 'format'", "str", args[0]); + goto exit; + } + format = args[0]; + return_value = datetime_time_strftime_impl((PyDateTime_Time *)self, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___format____doc__, +"__format__($self, format, /)\n" +"--\n" +"\n" +"Formats self with strftime."); + +#define DATETIME_TIME___FORMAT___METHODDEF \ + {"__format__", (PyCFunction)datetime_time___format__, METH_O, datetime_time___format____doc__}, + +static PyObject * +datetime_time___format___impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_time___format__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *format; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("__format__", "argument", "str", arg); + goto exit; + } + format = arg; + return_value = datetime_time___format___impl(self, format); + +exit: + return return_value; +} + PyDoc_STRVAR(datetime_time_replace__doc__, "replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" " microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" @@ -305,6 +1189,216 @@ exit: return return_value; } +PyDoc_STRVAR(datetime_time_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" +"--\n" +"\n" +"Construct a time from a string in ISO 8601 format."); + +#define DATETIME_TIME_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_time_fromisoformat, METH_O|METH_CLASS, datetime_time_fromisoformat__doc__}, + +static PyObject * +datetime_time_fromisoformat_impl(PyTypeObject *type, PyObject *string); + +static PyObject * +datetime_time_fromisoformat(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); + goto exit; + } + string = arg; + return_value = datetime_time_fromisoformat_impl((PyTypeObject *)type, string); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___reduce_ex____doc__, +"__reduce_ex__($self, proto, /)\n" +"--\n" +"\n"); + +#define DATETIME_TIME___REDUCE_EX___METHODDEF \ + {"__reduce_ex__", (PyCFunction)datetime_time___reduce_ex__, METH_O, datetime_time___reduce_ex____doc__}, + +static PyObject * +datetime_time___reduce_ex___impl(PyDateTime_Time *self, int proto); + +static PyObject * +datetime_time___reduce_ex__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int proto; + + proto = PyLong_AsInt(arg); + if (proto == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_time___reduce_ex___impl((PyDateTime_Time *)self, proto); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define DATETIME_TIME___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)datetime_time___reduce__, METH_NOARGS, datetime_time___reduce____doc__}, + +static PyObject * +datetime_time___reduce___impl(PyDateTime_Time *self); + +static PyObject * +datetime_time___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return datetime_time___reduce___impl((PyDateTime_Time *)self); +} + +PyDoc_STRVAR(datetime_datetime__doc__, +"datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0,\n" +" tzinfo=None, *, fold=0)\n" +"--\n" +"\n" +"A combination of a date and a time.\n" +"\n" +"The year, month and day arguments are required. tzinfo may be None, or an\n" +"instance of a tzinfo subclass. The remaining arguments may be ints."); + +static PyObject * +datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, + int hour, int minute, int second, int microsecond, + PyObject *tzinfo, int fold); + +static PyObject * +datetime_datetime(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", "hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "datetime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; + int year; + int month; + int day; + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; + PyObject *tzinfo = Py_None; + int fold = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + year = PyLong_AsInt(fastargs[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + month = PyLong_AsInt(fastargs[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + day = PyLong_AsInt(fastargs[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[3]) { + hour = PyLong_AsInt(fastargs[3]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + minute = PyLong_AsInt(fastargs[4]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[5]) { + second = PyLong_AsInt(fastargs[5]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[6]) { + microsecond = PyLong_AsInt(fastargs[6]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[7]) { + tzinfo = fastargs[7]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(fastargs[8]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_datetime_impl(type, year, month, day, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + PyDoc_STRVAR(datetime_datetime_now__doc__, "now($type, /, tz=None)\n" "--\n" @@ -373,6 +1467,370 @@ exit: return return_value; } +PyDoc_STRVAR(datetime_datetime_utcnow__doc__, +"utcnow($type, /)\n" +"--\n" +"\n" +"Return a new datetime representing UTC day and time."); + +#define DATETIME_DATETIME_UTCNOW_METHODDEF \ + {"utcnow", (PyCFunction)datetime_datetime_utcnow, METH_NOARGS|METH_CLASS, datetime_datetime_utcnow__doc__}, + +static PyObject * +datetime_datetime_utcnow_impl(PyTypeObject *type); + +static PyObject * +datetime_datetime_utcnow(PyObject *type, PyObject *Py_UNUSED(ignored)) +{ + return datetime_datetime_utcnow_impl((PyTypeObject *)type); +} + +PyDoc_STRVAR(datetime_datetime_fromtimestamp__doc__, +"fromtimestamp($type, /, timestamp, tz=None)\n" +"--\n" +"\n" +"Create a datetime from a POSIX timestamp.\n" +"\n" +"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" +"as local time."); + +#define DATETIME_DATETIME_FROMTIMESTAMP_METHODDEF \ + {"fromtimestamp", _PyCFunction_CAST(datetime_datetime_fromtimestamp), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_fromtimestamp__doc__}, + +static PyObject * +datetime_datetime_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp, + PyObject *tzinfo); + +static PyObject * +datetime_datetime_fromtimestamp(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(timestamp), &_Py_ID(tz), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timestamp", "tz", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fromtimestamp", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *timestamp; + PyObject *tzinfo = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + timestamp = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[1]; +skip_optional_pos: + return_value = datetime_datetime_fromtimestamp_impl((PyTypeObject *)type, timestamp, tzinfo); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_utcfromtimestamp__doc__, +"utcfromtimestamp($type, timestamp, /)\n" +"--\n" +"\n" +"Create a naive UTC datetime from a POSIX timestamp."); + +#define DATETIME_DATETIME_UTCFROMTIMESTAMP_METHODDEF \ + {"utcfromtimestamp", (PyCFunction)datetime_datetime_utcfromtimestamp, METH_O|METH_CLASS, datetime_datetime_utcfromtimestamp__doc__}, + +static PyObject * +datetime_datetime_utcfromtimestamp_impl(PyTypeObject *type, + PyObject *timestamp); + +static PyObject * +datetime_datetime_utcfromtimestamp(PyObject *type, PyObject *timestamp) +{ + PyObject *return_value = NULL; + + return_value = datetime_datetime_utcfromtimestamp_impl((PyTypeObject *)type, timestamp); + + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_strptime__doc__, +"strptime($type, string, format, /)\n" +"--\n" +"\n" +"Parse string according to the given date and time format (like time.strptime()).\n" +"\n" +"For a list of supported format codes, see the documentation:\n" +" https://docs.python.org/3/library/datetime.html#format-codes"); + +#define DATETIME_DATETIME_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_datetime_strptime), METH_FASTCALL|METH_CLASS, datetime_datetime_strptime__doc__}, + +static PyObject * +datetime_datetime_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_datetime_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_datetime_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_combine__doc__, +"combine($type, /, date, time, tzinfo=<unrepresentable>)\n" +"--\n" +"\n" +"Construct a datetime from a given date and a given time."); + +#define DATETIME_DATETIME_COMBINE_METHODDEF \ + {"combine", _PyCFunction_CAST(datetime_datetime_combine), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_combine__doc__}, + +static PyObject * +datetime_datetime_combine_impl(PyTypeObject *type, PyObject *date, + PyObject *time, PyObject *tzinfo); + +static PyObject * +datetime_datetime_combine(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(date), &_Py_ID(time), &_Py_ID(tzinfo), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"date", "time", "tzinfo", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "combine", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *date; + PyObject *time; + PyObject *tzinfo = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyObject_TypeCheck(args[0], DATE_TYPE(NO_STATE))) { + _PyArg_BadArgument("combine", "argument 'date'", (DATE_TYPE(NO_STATE))->tp_name, args[0]); + goto exit; + } + date = args[0]; + if (!PyObject_TypeCheck(args[1], TIME_TYPE(NO_STATE))) { + _PyArg_BadArgument("combine", "argument 'time'", (TIME_TYPE(NO_STATE))->tp_name, args[1]); + goto exit; + } + time = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[2]; +skip_optional_pos: + return_value = datetime_datetime_combine_impl((PyTypeObject *)type, date, time, tzinfo); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" +"--\n" +"\n" +"Construct a date from a string in ISO 8601 format."); + +#define DATETIME_DATETIME_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_datetime_fromisoformat, METH_O|METH_CLASS, datetime_datetime_fromisoformat__doc__}, + +static PyObject * +datetime_datetime_fromisoformat_impl(PyTypeObject *type, PyObject *string); + +static PyObject * +datetime_datetime_fromisoformat(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); + goto exit; + } + string = arg; + return_value = datetime_datetime_fromisoformat_impl((PyTypeObject *)type, string); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_isoformat__doc__, +"isoformat($self, /, sep=\'T\', timespec=\'auto\')\n" +"--\n" +"\n" +"Return the time formatted according to ISO.\n" +"\n" +"The full format looks like \'YYYY-MM-DD HH:MM:SS.mmmmmm\'.\n" +"By default, the fractional part is omitted if self.microsecond == 0.\n" +"\n" +"If self.tzinfo is not None, the UTC offset is also attached, giving\n" +"a full format of \'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM\'.\n" +"\n" +"Optional argument sep specifies the separator between date and\n" +"time, default \'T\'.\n" +"\n" +"The optional argument timespec specifies the number of additional\n" +"terms of the time to include. Valid options are \'auto\', \'hours\',\n" +"\'minutes\', \'seconds\', \'milliseconds\' and \'microseconds\'."); + +#define DATETIME_DATETIME_ISOFORMAT_METHODDEF \ + {"isoformat", _PyCFunction_CAST(datetime_datetime_isoformat), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_isoformat__doc__}, + +static PyObject * +datetime_datetime_isoformat_impl(PyDateTime_DateTime *self, int sep, + const char *timespec); + +static PyObject * +datetime_datetime_isoformat(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(sep), &_Py_ID(timespec), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"sep", "timespec", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "isoformat", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int sep = 'T'; + const char *timespec = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("isoformat", "argument 'sep'", "a unicode character", args[0]); + goto exit; + } + if (PyUnicode_GET_LENGTH(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "isoformat(): argument 'sep' must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); + goto exit; + } + sep = PyUnicode_READ_CHAR(args[0], 0); + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("isoformat", "argument 'timespec'", "str", args[1]); + goto exit; + } + Py_ssize_t timespec_length; + timespec = PyUnicode_AsUTF8AndSize(args[1], &timespec_length); + if (timespec == NULL) { + goto exit; + } + if (strlen(timespec) != (size_t)timespec_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_pos: + return_value = datetime_datetime_isoformat_impl((PyDateTime_DateTime *)self, sep, timespec); + +exit: + return return_value; +} + PyDoc_STRVAR(datetime_datetime_replace__doc__, "replace($self, /, year=unchanged, month=unchanged, day=unchanged,\n" " hour=unchanged, minute=unchanged, second=unchanged,\n" @@ -524,4 +1982,112 @@ skip_optional_kwonly: exit: return return_value; } -/*[clinic end generated code: output=809640e747529c72 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(datetime_datetime_astimezone__doc__, +"astimezone($self, /, tz=None)\n" +"--\n" +"\n" +"Convert to local time in new timezone tz."); + +#define DATETIME_DATETIME_ASTIMEZONE_METHODDEF \ + {"astimezone", _PyCFunction_CAST(datetime_datetime_astimezone), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_astimezone__doc__}, + +static PyObject * +datetime_datetime_astimezone_impl(PyDateTime_DateTime *self, + PyObject *tzinfo); + +static PyObject * +datetime_datetime_astimezone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(tz), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"tz", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "astimezone", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *tzinfo = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[0]; +skip_optional_pos: + return_value = datetime_datetime_astimezone_impl((PyDateTime_DateTime *)self, tzinfo); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime___reduce_ex____doc__, +"__reduce_ex__($self, proto, /)\n" +"--\n" +"\n"); + +#define DATETIME_DATETIME___REDUCE_EX___METHODDEF \ + {"__reduce_ex__", (PyCFunction)datetime_datetime___reduce_ex__, METH_O, datetime_datetime___reduce_ex____doc__}, + +static PyObject * +datetime_datetime___reduce_ex___impl(PyDateTime_DateTime *self, int proto); + +static PyObject * +datetime_datetime___reduce_ex__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int proto; + + proto = PyLong_AsInt(arg); + if (proto == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_datetime___reduce_ex___impl((PyDateTime_DateTime *)self, proto); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define DATETIME_DATETIME___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)datetime_datetime___reduce__, METH_NOARGS, datetime_datetime___reduce____doc__}, + +static PyObject * +datetime_datetime___reduce___impl(PyDateTime_DateTime *self); + +static PyObject * +datetime_datetime___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return datetime_datetime___reduce___impl((PyDateTime_DateTime *)self); +} +/*[clinic end generated code: output=69658acff6a43ac4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 5e503194408..091ce9edc43 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -5,6 +5,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_runtime.h" // _Py_SINGLETON() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_dbm_dbm_close__doc__, @@ -22,7 +23,13 @@ _dbm_dbm_close_impl(dbmobject *self); static PyObject * _dbm_dbm_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _dbm_dbm_close_impl((dbmobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _dbm_dbm_close_impl((dbmobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_dbm_dbm_keys__doc__, @@ -40,11 +47,18 @@ _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_keys(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); - return NULL; + goto exit; } - return _dbm_dbm_keys_impl((dbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _dbm_dbm_keys_impl((dbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_dbm_dbm_get__doc__, @@ -85,7 +99,9 @@ _dbm_dbm_get(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_ &key, &key_length, &default_value)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _dbm_dbm_get_impl((dbmobject *)self, cls, key, key_length, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -131,7 +147,9 @@ _dbm_dbm_setdefault(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py &key, &key_length, &default_value)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _dbm_dbm_setdefault_impl((dbmobject *)self, cls, key, key_length, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -152,11 +170,18 @@ _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_clear(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); - return NULL; + goto exit; } - return _dbm_dbm_clear_impl((dbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _dbm_dbm_clear_impl((dbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(dbmopen__doc__, @@ -221,4 +246,4 @@ skip_optional: exit: return return_value; } -/*[clinic end generated code: output=3b456118f231b160 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=279511ea7cda38dd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 00950f18e53..6fd6aa3da50 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -5,6 +5,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_runtime.h" // _Py_SINGLETON() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_gdbm_gdbm_get__doc__, @@ -70,7 +71,9 @@ _gdbm_gdbm_setdefault(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _gdbm_gdbm_setdefault_impl((gdbmobject *)self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -91,7 +94,13 @@ _gdbm_gdbm_close_impl(gdbmobject *self); static PyObject * _gdbm_gdbm_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _gdbm_gdbm_close_impl((gdbmobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_close_impl((gdbmobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_keys__doc__, @@ -109,11 +118,18 @@ _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_keys(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_keys_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_keys_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_firstkey__doc__, @@ -135,11 +151,18 @@ _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_firstkey(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "firstkey() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_firstkey_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_firstkey_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_nextkey__doc__, @@ -187,7 +210,9 @@ _gdbm_gdbm_nextkey(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ &key, &key_length)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _gdbm_gdbm_nextkey_impl((gdbmobject *)self, cls, key, key_length); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -214,11 +239,18 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_reorganize(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reorganize() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_reorganize_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_reorganize_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_sync__doc__, @@ -239,11 +271,18 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_sync(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "sync() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_sync_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_sync_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_clear__doc__, @@ -261,11 +300,18 @@ _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_clear(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_clear_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_clear_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(dbmopen__doc__, @@ -343,4 +389,4 @@ skip_optional: exit: return return_value; } -/*[clinic end generated code: output=d974cb39e4ee5d67 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8bca34ce9d4493dd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 59ab46ca3f0..7ae7be185ec 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -10,98 +10,98 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedLong_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(EVP_copy__doc__, +PyDoc_STRVAR(_hashlib_HASH_copy__doc__, "copy($self, /)\n" "--\n" "\n" "Return a copy of the hash object."); -#define EVP_COPY_METHODDEF \ - {"copy", (PyCFunction)EVP_copy, METH_NOARGS, EVP_copy__doc__}, +#define _HASHLIB_HASH_COPY_METHODDEF \ + {"copy", (PyCFunction)_hashlib_HASH_copy, METH_NOARGS, _hashlib_HASH_copy__doc__}, static PyObject * -EVP_copy_impl(EVPobject *self); +_hashlib_HASH_copy_impl(HASHobject *self); static PyObject * -EVP_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +_hashlib_HASH_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return EVP_copy_impl((EVPobject *)self); + return _hashlib_HASH_copy_impl((HASHobject *)self); } -PyDoc_STRVAR(EVP_digest__doc__, +PyDoc_STRVAR(_hashlib_HASH_digest__doc__, "digest($self, /)\n" "--\n" "\n" "Return the digest value as a bytes object."); -#define EVP_DIGEST_METHODDEF \ - {"digest", (PyCFunction)EVP_digest, METH_NOARGS, EVP_digest__doc__}, +#define _HASHLIB_HASH_DIGEST_METHODDEF \ + {"digest", (PyCFunction)_hashlib_HASH_digest, METH_NOARGS, _hashlib_HASH_digest__doc__}, static PyObject * -EVP_digest_impl(EVPobject *self); +_hashlib_HASH_digest_impl(HASHobject *self); static PyObject * -EVP_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) +_hashlib_HASH_digest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return EVP_digest_impl((EVPobject *)self); + return _hashlib_HASH_digest_impl((HASHobject *)self); } -PyDoc_STRVAR(EVP_hexdigest__doc__, +PyDoc_STRVAR(_hashlib_HASH_hexdigest__doc__, "hexdigest($self, /)\n" "--\n" "\n" "Return the digest value as a string of hexadecimal digits."); -#define EVP_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)EVP_hexdigest, METH_NOARGS, EVP_hexdigest__doc__}, +#define _HASHLIB_HASH_HEXDIGEST_METHODDEF \ + {"hexdigest", (PyCFunction)_hashlib_HASH_hexdigest, METH_NOARGS, _hashlib_HASH_hexdigest__doc__}, static PyObject * -EVP_hexdigest_impl(EVPobject *self); +_hashlib_HASH_hexdigest_impl(HASHobject *self); static PyObject * -EVP_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) +_hashlib_HASH_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return EVP_hexdigest_impl((EVPobject *)self); + return _hashlib_HASH_hexdigest_impl((HASHobject *)self); } -PyDoc_STRVAR(EVP_update__doc__, +PyDoc_STRVAR(_hashlib_HASH_update__doc__, "update($self, obj, /)\n" "--\n" "\n" "Update this hash object\'s state with the provided string."); -#define EVP_UPDATE_METHODDEF \ - {"update", (PyCFunction)EVP_update, METH_O, EVP_update__doc__}, +#define _HASHLIB_HASH_UPDATE_METHODDEF \ + {"update", (PyCFunction)_hashlib_HASH_update, METH_O, _hashlib_HASH_update__doc__}, static PyObject * -EVP_update_impl(EVPobject *self, PyObject *obj); +_hashlib_HASH_update_impl(HASHobject *self, PyObject *obj); static PyObject * -EVP_update(PyObject *self, PyObject *obj) +_hashlib_HASH_update(PyObject *self, PyObject *obj) { PyObject *return_value = NULL; - return_value = EVP_update_impl((EVPobject *)self, obj); + return_value = _hashlib_HASH_update_impl((HASHobject *)self, obj); return return_value; } #if defined(PY_OPENSSL_HAS_SHAKE) -PyDoc_STRVAR(EVPXOF_digest__doc__, +PyDoc_STRVAR(_hashlib_HASHXOF_digest__doc__, "digest($self, /, length)\n" "--\n" "\n" "Return the digest value as a bytes object."); -#define EVPXOF_DIGEST_METHODDEF \ - {"digest", _PyCFunction_CAST(EVPXOF_digest), METH_FASTCALL|METH_KEYWORDS, EVPXOF_digest__doc__}, +#define _HASHLIB_HASHXOF_DIGEST_METHODDEF \ + {"digest", _PyCFunction_CAST(_hashlib_HASHXOF_digest), METH_FASTCALL|METH_KEYWORDS, _hashlib_HASHXOF_digest__doc__}, static PyObject * -EVPXOF_digest_impl(EVPobject *self, Py_ssize_t length); +_hashlib_HASHXOF_digest_impl(HASHobject *self, Py_ssize_t length); static PyObject * -EVPXOF_digest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_hashlib_HASHXOF_digest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -150,8 +150,13 @@ EVPXOF_digest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } length = ival; + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length cannot be negative"); + goto exit; + } } - return_value = EVPXOF_digest_impl((EVPobject *)self, length); + return_value = _hashlib_HASHXOF_digest_impl((HASHobject *)self, length); exit: return return_value; @@ -161,20 +166,20 @@ exit: #if defined(PY_OPENSSL_HAS_SHAKE) -PyDoc_STRVAR(EVPXOF_hexdigest__doc__, +PyDoc_STRVAR(_hashlib_HASHXOF_hexdigest__doc__, "hexdigest($self, /, length)\n" "--\n" "\n" "Return the digest value as a string of hexadecimal digits."); -#define EVPXOF_HEXDIGEST_METHODDEF \ - {"hexdigest", _PyCFunction_CAST(EVPXOF_hexdigest), METH_FASTCALL|METH_KEYWORDS, EVPXOF_hexdigest__doc__}, +#define _HASHLIB_HASHXOF_HEXDIGEST_METHODDEF \ + {"hexdigest", _PyCFunction_CAST(_hashlib_HASHXOF_hexdigest), METH_FASTCALL|METH_KEYWORDS, _hashlib_HASHXOF_hexdigest__doc__}, static PyObject * -EVPXOF_hexdigest_impl(EVPobject *self, Py_ssize_t length); +_hashlib_HASHXOF_hexdigest_impl(HASHobject *self, Py_ssize_t length); static PyObject * -EVPXOF_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_hashlib_HASHXOF_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -223,8 +228,13 @@ EVPXOF_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje goto exit; } length = ival; + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length cannot be negative"); + goto exit; + } } - return_value = EVPXOF_hexdigest_impl((EVPobject *)self, length); + return_value = _hashlib_HASHXOF_hexdigest_impl((HASHobject *)self, length); exit: return return_value; @@ -232,8 +242,8 @@ exit: #endif /* defined(PY_OPENSSL_HAS_SHAKE) */ -PyDoc_STRVAR(EVP_new__doc__, -"new($module, /, name, string=b\'\', *, usedforsecurity=True)\n" +PyDoc_STRVAR(_hashlib_HASH_new__doc__, +"new($module, /, name, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new hash object using the named algorithm.\n" @@ -243,15 +253,114 @@ PyDoc_STRVAR(EVP_new__doc__, "\n" "The MD5 and SHA1 algorithms are always supported."); -#define EVP_NEW_METHODDEF \ - {"new", _PyCFunction_CAST(EVP_new), METH_FASTCALL|METH_KEYWORDS, EVP_new__doc__}, +#define _HASHLIB_HASH_NEW_METHODDEF \ + {"new", _PyCFunction_CAST(_hashlib_HASH_new), METH_FASTCALL|METH_KEYWORDS, _hashlib_HASH_new__doc__}, static PyObject * -EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj, - int usedforsecurity); +_hashlib_HASH_new_impl(PyObject *module, const char *name, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * -EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_hashlib_HASH_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(name), &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "data", "usedforsecurity", "string", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "new", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + const char *name; + PyObject *data = NULL; + int usedforsecurity = 1; + PyObject *string = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("new", "argument 'name'", "str", args[0]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + data = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + usedforsecurity = PyObject_IsTrue(args[2]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + string = args[3]; +skip_optional_kwonly: + return_value = _hashlib_HASH_new_impl(module, name, data, usedforsecurity, string); + +exit: + return return_value; +} + +PyDoc_STRVAR(_hashlib_openssl_md5__doc__, +"openssl_md5($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" +"--\n" +"\n" +"Returns a md5 hash object; optionally initialized with a string"); + +#define _HASHLIB_OPENSSL_MD5_METHODDEF \ + {"openssl_md5", _PyCFunction_CAST(_hashlib_openssl_md5), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_md5__doc__}, + +static PyObject * +_hashlib_openssl_md5_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); + +static PyObject * +_hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -265,7 +374,7 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -274,97 +383,18 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "string", "usedforsecurity", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "new", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - PyObject *name_obj; - PyObject *data_obj = NULL; - int usedforsecurity = 1; - - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - name_obj = args[0]; - if (!noptargs) { - goto skip_optional_pos; - } - if (args[1]) { - data_obj = args[1]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - usedforsecurity = PyObject_IsTrue(args[2]); - if (usedforsecurity < 0) { - goto exit; - } -skip_optional_kwonly: - return_value = EVP_new_impl(module, name_obj, data_obj, usedforsecurity); - -exit: - return return_value; -} - -PyDoc_STRVAR(_hashlib_openssl_md5__doc__, -"openssl_md5($module, /, string=b\'\', *, usedforsecurity=True)\n" -"--\n" -"\n" -"Returns a md5 hash object; optionally initialized with a string"); - -#define _HASHLIB_OPENSSL_MD5_METHODDEF \ - {"openssl_md5", _PyCFunction_CAST(_hashlib_openssl_md5), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_md5__doc__}, - -static PyObject * -_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); - -static PyObject * -_hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_md5", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -375,7 +405,7 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -384,19 +414,25 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_md5_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_md5_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha1__doc__, -"openssl_sha1($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha1($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Returns a sha1 hash object; optionally initialized with a string"); @@ -405,8 +441,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha1__doc__, {"openssl_sha1", _PyCFunction_CAST(_hashlib_openssl_sha1), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha1__doc__}, static PyObject * -_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -414,7 +450,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -423,7 +459,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -432,17 +468,18 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha1", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -453,7 +490,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -462,19 +499,26 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha1_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha1_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha224__doc__, -"openssl_sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha224($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha224 hash object; optionally initialized with a string"); @@ -483,8 +527,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha224__doc__, {"openssl_sha224", _PyCFunction_CAST(_hashlib_openssl_sha224), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha224__doc__}, static PyObject * -_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -492,7 +536,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -501,7 +545,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -510,17 +554,18 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -531,7 +576,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -540,19 +585,26 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha224_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha224_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha256__doc__, -"openssl_sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha256 hash object; optionally initialized with a string"); @@ -561,8 +613,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha256__doc__, {"openssl_sha256", _PyCFunction_CAST(_hashlib_openssl_sha256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha256__doc__}, static PyObject * -_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -570,7 +622,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -579,7 +631,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -588,17 +640,18 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -609,7 +662,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -618,19 +671,26 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha256_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha384__doc__, -"openssl_sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha384($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha384 hash object; optionally initialized with a string"); @@ -639,8 +699,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha384__doc__, {"openssl_sha384", _PyCFunction_CAST(_hashlib_openssl_sha384), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha384__doc__}, static PyObject * -_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -648,7 +708,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -657,7 +717,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -666,17 +726,18 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -687,7 +748,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -696,19 +757,26 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha384_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha384_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha512__doc__, -"openssl_sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha512($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha512 hash object; optionally initialized with a string"); @@ -717,8 +785,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha512__doc__, {"openssl_sha512", _PyCFunction_CAST(_hashlib_openssl_sha512), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha512__doc__}, static PyObject * -_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -726,7 +794,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -735,7 +803,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -744,17 +812,18 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -765,7 +834,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -774,12 +843,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha512_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha512_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -788,7 +863,8 @@ exit: #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_224__doc__, -"openssl_sha3_224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_224($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-224 hash object; optionally initialized with a string"); @@ -797,8 +873,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_224__doc__, {"openssl_sha3_224", _PyCFunction_CAST(_hashlib_openssl_sha3_224), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_224__doc__}, static PyObject * -_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -806,7 +882,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -815,7 +891,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -824,17 +900,18 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -845,7 +922,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -854,12 +931,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_224_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_224_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -870,7 +953,8 @@ exit: #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_256__doc__, -"openssl_sha3_256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-256 hash object; optionally initialized with a string"); @@ -879,8 +963,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_256__doc__, {"openssl_sha3_256", _PyCFunction_CAST(_hashlib_openssl_sha3_256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_256__doc__}, static PyObject * -_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -888,7 +972,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -897,7 +981,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -906,17 +990,18 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -927,7 +1012,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -936,12 +1021,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_256_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -952,7 +1043,8 @@ exit: #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_384__doc__, -"openssl_sha3_384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_384($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-384 hash object; optionally initialized with a string"); @@ -961,8 +1053,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_384__doc__, {"openssl_sha3_384", _PyCFunction_CAST(_hashlib_openssl_sha3_384), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_384__doc__}, static PyObject * -_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -970,7 +1062,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -979,7 +1071,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -988,17 +1080,18 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1009,7 +1102,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1018,12 +1111,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_384_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_384_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1034,7 +1133,8 @@ exit: #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_512__doc__, -"openssl_sha3_512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_512($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-512 hash object; optionally initialized with a string"); @@ -1043,8 +1143,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_512__doc__, {"openssl_sha3_512", _PyCFunction_CAST(_hashlib_openssl_sha3_512), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_512__doc__}, static PyObject * -_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1052,7 +1152,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1061,7 +1161,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1070,17 +1170,18 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1091,7 +1192,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1100,12 +1201,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_512_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_512_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1116,7 +1223,8 @@ exit: #if defined(PY_OPENSSL_HAS_SHAKE) PyDoc_STRVAR(_hashlib_openssl_shake_128__doc__, -"openssl_shake_128($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_shake_128($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a shake-128 variable hash object; optionally initialized with a string"); @@ -1125,8 +1233,8 @@ PyDoc_STRVAR(_hashlib_openssl_shake_128__doc__, {"openssl_shake_128", _PyCFunction_CAST(_hashlib_openssl_shake_128), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_shake_128__doc__}, static PyObject * -_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1134,7 +1242,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1143,7 +1251,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1152,17 +1260,18 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_shake_128", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1173,7 +1282,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1182,12 +1291,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_shake_128_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_shake_128_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1198,7 +1313,8 @@ exit: #if defined(PY_OPENSSL_HAS_SHAKE) PyDoc_STRVAR(_hashlib_openssl_shake_256__doc__, -"openssl_shake_256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_shake_256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a shake-256 variable hash object; optionally initialized with a string"); @@ -1207,8 +1323,8 @@ PyDoc_STRVAR(_hashlib_openssl_shake_256__doc__, {"openssl_shake_256", _PyCFunction_CAST(_hashlib_openssl_shake_256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_shake_256__doc__}, static PyObject * -_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1216,7 +1332,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1225,7 +1341,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1234,17 +1350,18 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_shake_256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1255,7 +1372,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1264,12 +1381,18 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_shake_256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_shake_256_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1379,8 +1502,6 @@ exit: return return_value; } -#if defined(PY_OPENSSL_HAS_SCRYPT) - PyDoc_STRVAR(_hashlib_scrypt__doc__, "scrypt($module, /, password, *, salt, n, r, p, maxmem=0, dklen=64)\n" "--\n" @@ -1488,8 +1609,6 @@ exit: return return_value; } -#endif /* defined(PY_OPENSSL_HAS_SCRYPT) */ - PyDoc_STRVAR(_hashlib_hmac_singleshot__doc__, "hmac_digest($module, /, key, msg, digest)\n" "--\n" @@ -1836,13 +1955,13 @@ exit: return return_value; } -#ifndef EVPXOF_DIGEST_METHODDEF - #define EVPXOF_DIGEST_METHODDEF -#endif /* !defined(EVPXOF_DIGEST_METHODDEF) */ +#ifndef _HASHLIB_HASHXOF_DIGEST_METHODDEF + #define _HASHLIB_HASHXOF_DIGEST_METHODDEF +#endif /* !defined(_HASHLIB_HASHXOF_DIGEST_METHODDEF) */ -#ifndef EVPXOF_HEXDIGEST_METHODDEF - #define EVPXOF_HEXDIGEST_METHODDEF -#endif /* !defined(EVPXOF_HEXDIGEST_METHODDEF) */ +#ifndef _HASHLIB_HASHXOF_HEXDIGEST_METHODDEF + #define _HASHLIB_HASHXOF_HEXDIGEST_METHODDEF +#endif /* !defined(_HASHLIB_HASHXOF_HEXDIGEST_METHODDEF) */ #ifndef _HASHLIB_OPENSSL_SHA3_224_METHODDEF #define _HASHLIB_OPENSSL_SHA3_224_METHODDEF @@ -1867,8 +1986,4 @@ exit: #ifndef _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #define _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #endif /* !defined(_HASHLIB_OPENSSL_SHAKE_256_METHODDEF) */ - -#ifndef _HASHLIB_SCRYPT_METHODDEF - #define _HASHLIB_SCRYPT_METHODDEF -#endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=2c78822e38be64a8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9ba35fcc33795b1e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_heapqmodule.c.h b/Modules/clinic/_heapqmodule.c.h index 81d10862726..b43155b6c24 100644 --- a/Modules/clinic/_heapqmodule.c.h +++ b/Modules/clinic/_heapqmodule.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_heapq_heappush__doc__, @@ -32,7 +33,9 @@ _heapq_heappush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappush_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -61,7 +64,9 @@ _heapq_heappop(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappop_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -103,7 +108,9 @@ _heapq_heapreplace(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapreplace_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -140,7 +147,9 @@ _heapq_heappushpop(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappushpop_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -169,7 +178,9 @@ _heapq_heapify(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapify_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -203,7 +214,9 @@ _heapq_heappush_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappush_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -232,7 +245,9 @@ _heapq_heappop_max(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappop_max_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -266,7 +281,9 @@ _heapq_heapreplace_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapreplace_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -295,7 +312,9 @@ _heapq_heapify_max(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapify_max_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -332,9 +351,11 @@ _heapq_heappushpop_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappushpop_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=f55d8595ce150c76 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e83d50002c29a96d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpqueuesmodule.c.h b/Modules/clinic/_interpqueuesmodule.c.h new file mode 100644 index 00000000000..3f08a0cb6d3 --- /dev/null +++ b/Modules/clinic/_interpqueuesmodule.c.h @@ -0,0 +1,765 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_interpqueues_create__doc__, +"create($module, /, maxsize, unboundop=-1, fallback=-1)\n" +"--\n" +"\n" +"Create a new cross-interpreter queue and return its unique generated ID.\n" +"\n" +"It is a new reference as though bind() had been called on the queue.\n" +"The caller is responsible for calling destroy() for the new queue\n" +"before the runtime is finalized."); + +#define _INTERPQUEUES_CREATE_METHODDEF \ + {"create", _PyCFunction_CAST(_interpqueues_create), METH_FASTCALL|METH_KEYWORDS, _interpqueues_create__doc__}, + +static PyObject * +_interpqueues_create_impl(PyObject *module, Py_ssize_t maxsize, + int unboundarg, int fallbackarg); + +static PyObject * +_interpqueues_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(maxsize), &_Py_ID(unboundop), &_Py_ID(fallback), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"maxsize", "unboundop", "fallback", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "create", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + Py_ssize_t maxsize; + int unboundarg = -1; + int fallbackarg = -1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + maxsize = ival; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + unboundarg = PyLong_AsInt(args[1]); + if (unboundarg == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + fallbackarg = PyLong_AsInt(args[2]); + if (fallbackarg == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = _interpqueues_create_impl(module, maxsize, unboundarg, fallbackarg); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_destroy__doc__, +"destroy($module, /, qid)\n" +"--\n" +"\n" +"Clear and destroy the queue.\n" +"\n" +"Afterward attempts to use the queue will behave as though it never existed."); + +#define _INTERPQUEUES_DESTROY_METHODDEF \ + {"destroy", _PyCFunction_CAST(_interpqueues_destroy), METH_FASTCALL|METH_KEYWORDS, _interpqueues_destroy__doc__}, + +static PyObject * +_interpqueues_destroy_impl(PyObject *module, int64_t qid); + +static PyObject * +_interpqueues_destroy(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "destroy", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int64_t qid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + return_value = _interpqueues_destroy_impl(module, qid); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_list_all__doc__, +"list_all($module, /)\n" +"--\n" +"\n" +"Return the list of ID triples for all queues.\n" +"\n" +"Each ID triple consists of (ID, default unbound op, default fallback)."); + +#define _INTERPQUEUES_LIST_ALL_METHODDEF \ + {"list_all", (PyCFunction)_interpqueues_list_all, METH_NOARGS, _interpqueues_list_all__doc__}, + +static PyObject * +_interpqueues_list_all_impl(PyObject *module); + +static PyObject * +_interpqueues_list_all(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _interpqueues_list_all_impl(module); +} + +PyDoc_STRVAR(_interpqueues_put__doc__, +"put($module, /, qid, obj, unboundop=-1, fallback=-1)\n" +"--\n" +"\n" +"Add the object\'s data to the queue."); + +#define _INTERPQUEUES_PUT_METHODDEF \ + {"put", _PyCFunction_CAST(_interpqueues_put), METH_FASTCALL|METH_KEYWORDS, _interpqueues_put__doc__}, + +static PyObject * +_interpqueues_put_impl(PyObject *module, int64_t qid, PyObject *obj, + int unboundarg, int fallbackarg); + +static PyObject * +_interpqueues_put(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), &_Py_ID(obj), &_Py_ID(unboundop), &_Py_ID(fallback), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", "obj", "unboundop", "fallback", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "put", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + int64_t qid; + PyObject *obj; + int unboundarg = -1; + int fallbackarg = -1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + obj = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + unboundarg = PyLong_AsInt(args[2]); + if (unboundarg == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + fallbackarg = PyLong_AsInt(args[3]); + if (fallbackarg == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = _interpqueues_put_impl(module, qid, obj, unboundarg, fallbackarg); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_get__doc__, +"get($module, /, qid)\n" +"--\n" +"\n" +"Return the (object, unbound op) from the front of the queue.\n" +"\n" +"If there is nothing to receive then raise QueueEmpty."); + +#define _INTERPQUEUES_GET_METHODDEF \ + {"get", _PyCFunction_CAST(_interpqueues_get), METH_FASTCALL|METH_KEYWORDS, _interpqueues_get__doc__}, + +static PyObject * +_interpqueues_get_impl(PyObject *module, int64_t qid); + +static PyObject * +_interpqueues_get(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int64_t qid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + return_value = _interpqueues_get_impl(module, qid); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_bind__doc__, +"bind($module, /, qid)\n" +"--\n" +"\n" +"Take a reference to the identified queue.\n" +"\n" +"The queue is not destroyed until there are no references left."); + +#define _INTERPQUEUES_BIND_METHODDEF \ + {"bind", _PyCFunction_CAST(_interpqueues_bind), METH_FASTCALL|METH_KEYWORDS, _interpqueues_bind__doc__}, + +static PyObject * +_interpqueues_bind_impl(PyObject *module, int64_t qid); + +static PyObject * +_interpqueues_bind(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "bind", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int64_t qid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + return_value = _interpqueues_bind_impl(module, qid); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_release__doc__, +"release($module, /, qid)\n" +"--\n" +"\n" +"Release a reference to the queue.\n" +"\n" +"The queue is destroyed once there are no references left."); + +#define _INTERPQUEUES_RELEASE_METHODDEF \ + {"release", _PyCFunction_CAST(_interpqueues_release), METH_FASTCALL|METH_KEYWORDS, _interpqueues_release__doc__}, + +static PyObject * +_interpqueues_release_impl(PyObject *module, int64_t qid); + +static PyObject * +_interpqueues_release(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "release", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int64_t qid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + return_value = _interpqueues_release_impl(module, qid); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_get_maxsize__doc__, +"get_maxsize($module, /, qid)\n" +"--\n" +"\n" +"Return the maximum number of items in the queue."); + +#define _INTERPQUEUES_GET_MAXSIZE_METHODDEF \ + {"get_maxsize", _PyCFunction_CAST(_interpqueues_get_maxsize), METH_FASTCALL|METH_KEYWORDS, _interpqueues_get_maxsize__doc__}, + +static PyObject * +_interpqueues_get_maxsize_impl(PyObject *module, int64_t qid); + +static PyObject * +_interpqueues_get_maxsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_maxsize", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int64_t qid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + return_value = _interpqueues_get_maxsize_impl(module, qid); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_get_queue_defaults__doc__, +"get_queue_defaults($module, /, qid)\n" +"--\n" +"\n" +"Return the queue\'s default values, set when it was created."); + +#define _INTERPQUEUES_GET_QUEUE_DEFAULTS_METHODDEF \ + {"get_queue_defaults", _PyCFunction_CAST(_interpqueues_get_queue_defaults), METH_FASTCALL|METH_KEYWORDS, _interpqueues_get_queue_defaults__doc__}, + +static PyObject * +_interpqueues_get_queue_defaults_impl(PyObject *module, int64_t qid); + +static PyObject * +_interpqueues_get_queue_defaults(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_queue_defaults", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int64_t qid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + return_value = _interpqueues_get_queue_defaults_impl(module, qid); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_is_full__doc__, +"is_full($module, /, qid)\n" +"--\n" +"\n" +"Return true if the queue has a maxsize and has reached it."); + +#define _INTERPQUEUES_IS_FULL_METHODDEF \ + {"is_full", _PyCFunction_CAST(_interpqueues_is_full), METH_FASTCALL|METH_KEYWORDS, _interpqueues_is_full__doc__}, + +static PyObject * +_interpqueues_is_full_impl(PyObject *module, int64_t qid); + +static PyObject * +_interpqueues_is_full(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_full", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int64_t qid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + return_value = _interpqueues_is_full_impl(module, qid); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues_get_count__doc__, +"get_count($module, /, qid)\n" +"--\n" +"\n" +"Return the number of items in the queue."); + +#define _INTERPQUEUES_GET_COUNT_METHODDEF \ + {"get_count", _PyCFunction_CAST(_interpqueues_get_count), METH_FASTCALL|METH_KEYWORDS, _interpqueues_get_count__doc__}, + +static PyObject * +_interpqueues_get_count_impl(PyObject *module, int64_t qid); + +static PyObject * +_interpqueues_get_count(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(qid), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"qid", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_count", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int64_t qid; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!qidarg_converter(args[0], &qid)) { + goto exit; + } + return_value = _interpqueues_get_count_impl(module, qid); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpqueues__register_heap_types__doc__, +"_register_heap_types($module, /, queuetype, emptyerror, fullerror)\n" +"--\n" +"\n" +"Return the number of items in the queue."); + +#define _INTERPQUEUES__REGISTER_HEAP_TYPES_METHODDEF \ + {"_register_heap_types", _PyCFunction_CAST(_interpqueues__register_heap_types), METH_FASTCALL|METH_KEYWORDS, _interpqueues__register_heap_types__doc__}, + +static PyObject * +_interpqueues__register_heap_types_impl(PyObject *module, + PyTypeObject *queuetype, + PyObject *emptyerror, + PyObject *fullerror); + +static PyObject * +_interpqueues__register_heap_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(queuetype), &_Py_ID(emptyerror), &_Py_ID(fullerror), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"queuetype", "emptyerror", "fullerror", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_register_heap_types", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyTypeObject *queuetype; + PyObject *emptyerror; + PyObject *fullerror; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyObject_TypeCheck(args[0], &PyType_Type)) { + _PyArg_BadArgument("_register_heap_types", "argument 'queuetype'", (&PyType_Type)->tp_name, args[0]); + goto exit; + } + queuetype = (PyTypeObject *)args[0]; + emptyerror = args[1]; + fullerror = args[2]; + return_value = _interpqueues__register_heap_types_impl(module, queuetype, emptyerror, fullerror); + +exit: + return return_value; +} +/*[clinic end generated code: output=64cea8e1063429b6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpretersmodule.c.h b/Modules/clinic/_interpretersmodule.c.h new file mode 100644 index 00000000000..d70ffcea527 --- /dev/null +++ b/Modules/clinic/_interpretersmodule.c.h @@ -0,0 +1,1201 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_interpreters_create__doc__, +"create($module, /, config=\'isolated\', *, reqrefs=False)\n" +"--\n" +"\n" +"Create a new interpreter and return a unique generated ID.\n" +"\n" +"The caller is responsible for destroying the interpreter before exiting,\n" +"typically by using _interpreters.destroy(). This can be managed\n" +"automatically by passing \"reqrefs=True\" and then using _incref() and\n" +"_decref() appropriately.\n" +"\n" +"\"config\" must be a valid interpreter config or the name of a\n" +"predefined config (\'isolated\' or \'legacy\'). The default\n" +"is \'isolated\'."); + +#define _INTERPRETERS_CREATE_METHODDEF \ + {"create", _PyCFunction_CAST(_interpreters_create), METH_FASTCALL|METH_KEYWORDS, _interpreters_create__doc__}, + +static PyObject * +_interpreters_create_impl(PyObject *module, PyObject *configobj, int reqrefs); + +static PyObject * +_interpreters_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(config), &_Py_ID(reqrefs), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"config", "reqrefs", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "create", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *configobj = NULL; + int reqrefs = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + configobj = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + reqrefs = PyObject_IsTrue(args[1]); + if (reqrefs < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_create_impl(module, configobj, reqrefs); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_destroy__doc__, +"destroy($module, /, id, *, restrict=False)\n" +"--\n" +"\n" +"Destroy the identified interpreter.\n" +"\n" +"Attempting to destroy the current interpreter raises InterpreterError.\n" +"So does an unrecognized ID."); + +#define _INTERPRETERS_DESTROY_METHODDEF \ + {"destroy", _PyCFunction_CAST(_interpreters_destroy), METH_FASTCALL|METH_KEYWORDS, _interpreters_destroy__doc__}, + +static PyObject * +_interpreters_destroy_impl(PyObject *module, PyObject *id, int restricted); + +static PyObject * +_interpreters_destroy(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "destroy", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[1]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_destroy_impl(module, id, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_list_all__doc__, +"list_all($module, /, *, require_ready=False)\n" +"--\n" +"\n" +"Return a list containing the ID of every existing interpreter."); + +#define _INTERPRETERS_LIST_ALL_METHODDEF \ + {"list_all", _PyCFunction_CAST(_interpreters_list_all), METH_FASTCALL|METH_KEYWORDS, _interpreters_list_all__doc__}, + +static PyObject * +_interpreters_list_all_impl(PyObject *module, int reqready); + +static PyObject * +_interpreters_list_all(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(require_ready), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"require_ready", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "list_all", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int reqready = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + reqready = PyObject_IsTrue(args[0]); + if (reqready < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_list_all_impl(module, reqready); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_get_current__doc__, +"get_current($module, /)\n" +"--\n" +"\n" +"Return (ID, whence) of the current interpreter."); + +#define _INTERPRETERS_GET_CURRENT_METHODDEF \ + {"get_current", (PyCFunction)_interpreters_get_current, METH_NOARGS, _interpreters_get_current__doc__}, + +static PyObject * +_interpreters_get_current_impl(PyObject *module); + +static PyObject * +_interpreters_get_current(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _interpreters_get_current_impl(module); +} + +PyDoc_STRVAR(_interpreters_get_main__doc__, +"get_main($module, /)\n" +"--\n" +"\n" +"Return (ID, whence) of the main interpreter."); + +#define _INTERPRETERS_GET_MAIN_METHODDEF \ + {"get_main", (PyCFunction)_interpreters_get_main, METH_NOARGS, _interpreters_get_main__doc__}, + +static PyObject * +_interpreters_get_main_impl(PyObject *module); + +static PyObject * +_interpreters_get_main(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _interpreters_get_main_impl(module); +} + +PyDoc_STRVAR(_interpreters_set___main___attrs__doc__, +"set___main___attrs($module, /, id, updates, *, restrict=False)\n" +"--\n" +"\n" +"Bind the given attributes in the interpreter\'s __main__ module."); + +#define _INTERPRETERS_SET___MAIN___ATTRS_METHODDEF \ + {"set___main___attrs", _PyCFunction_CAST(_interpreters_set___main___attrs), METH_FASTCALL|METH_KEYWORDS, _interpreters_set___main___attrs__doc__}, + +static PyObject * +_interpreters_set___main___attrs_impl(PyObject *module, PyObject *id, + PyObject *updates, int restricted); + +static PyObject * +_interpreters_set___main___attrs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(updates), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "updates", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set___main___attrs", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *updates; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!PyDict_Check(args[1])) { + _PyArg_BadArgument("set___main___attrs", "argument 'updates'", "dict", args[1]); + goto exit; + } + updates = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[2]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_set___main___attrs_impl(module, id, updates, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_exec__doc__, +"exec($module, /, id, code, shared={}, *, restrict=False)\n" +"--\n" +"\n" +"Execute the provided code in the identified interpreter.\n" +"\n" +"This is equivalent to running the builtin exec() under the target\n" +"interpreter, using the __dict__ of its __main__ module as both\n" +"globals and locals.\n" +"\n" +"\"code\" may be a string containing the text of a Python script.\n" +"\n" +"Functions (and code objects) are also supported, with some restrictions.\n" +"The code/function must not take any arguments or be a closure\n" +"(i.e. have cell vars). Methods and other callables are not supported.\n" +"\n" +"If a function is provided, its code object is used and all its state\n" +"is ignored, including its __globals__ dict."); + +#define _INTERPRETERS_EXEC_METHODDEF \ + {"exec", _PyCFunction_CAST(_interpreters_exec), METH_FASTCALL|METH_KEYWORDS, _interpreters_exec__doc__}, + +static PyObject * +_interpreters_exec_impl(PyObject *module, PyObject *id, PyObject *code, + PyObject *shared, int restricted); + +static PyObject * +_interpreters_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(code), &_Py_ID(shared), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "code", "shared", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "exec", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *code; + PyObject *shared = NULL; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + code = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + if (!PyDict_Check(args[2])) { + _PyArg_BadArgument("exec", "argument 'shared'", "dict", args[2]); + goto exit; + } + shared = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[3]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_exec_impl(module, id, code, shared, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_run_string__doc__, +"run_string($module, /, id, script, shared={}, *, restrict=False)\n" +"--\n" +"\n" +"Execute the provided string in the identified interpreter.\n" +"\n" +"(See _interpreters.exec().)"); + +#define _INTERPRETERS_RUN_STRING_METHODDEF \ + {"run_string", _PyCFunction_CAST(_interpreters_run_string), METH_FASTCALL|METH_KEYWORDS, _interpreters_run_string__doc__}, + +static PyObject * +_interpreters_run_string_impl(PyObject *module, PyObject *id, + PyObject *script, PyObject *shared, + int restricted); + +static PyObject * +_interpreters_run_string(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(script), &_Py_ID(shared), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "script", "shared", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "run_string", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *script; + PyObject *shared = NULL; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("run_string", "argument 'script'", "str", args[1]); + goto exit; + } + script = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + if (!PyDict_Check(args[2])) { + _PyArg_BadArgument("run_string", "argument 'shared'", "dict", args[2]); + goto exit; + } + shared = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[3]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_run_string_impl(module, id, script, shared, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_run_func__doc__, +"run_func($module, /, id, func, shared={}, *, restrict=False)\n" +"--\n" +"\n" +"Execute the body of the provided function in the identified interpreter.\n" +"\n" +"Code objects are also supported. In both cases, closures and args\n" +"are not supported. Methods and other callables are not supported either.\n" +"\n" +"(See _interpreters.exec().)"); + +#define _INTERPRETERS_RUN_FUNC_METHODDEF \ + {"run_func", _PyCFunction_CAST(_interpreters_run_func), METH_FASTCALL|METH_KEYWORDS, _interpreters_run_func__doc__}, + +static PyObject * +_interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func, + PyObject *shared, int restricted); + +static PyObject * +_interpreters_run_func(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(func), &_Py_ID(shared), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "func", "shared", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "run_func", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *func; + PyObject *shared = NULL; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + func = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + if (!PyDict_Check(args[2])) { + _PyArg_BadArgument("run_func", "argument 'shared'", "dict", args[2]); + goto exit; + } + shared = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[3]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_run_func_impl(module, id, func, shared, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_call__doc__, +"call($module, /, id, callable, args=(), kwargs={}, *,\n" +" preserve_exc=False, restrict=False)\n" +"--\n" +"\n" +"Call the provided object in the identified interpreter.\n" +"\n" +"Pass the given args and kwargs, if possible."); + +#define _INTERPRETERS_CALL_METHODDEF \ + {"call", _PyCFunction_CAST(_interpreters_call), METH_FASTCALL|METH_KEYWORDS, _interpreters_call__doc__}, + +static PyObject * +_interpreters_call_impl(PyObject *module, PyObject *id, PyObject *callable, + PyObject *args, PyObject *kwargs, int preserve_exc, + int restricted); + +static PyObject * +_interpreters_call(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(callable), &_Py_ID(args), &_Py_ID(kwargs), &_Py_ID(preserve_exc), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "callable", "args", "kwargs", "preserve_exc", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "call", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *id; + PyObject *callable; + PyObject *__clinic_args = NULL; + PyObject *__clinic_kwargs = NULL; + int preserve_exc = 0; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + callable = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[2]) { + if (!PyTuple_Check(args[2])) { + _PyArg_BadArgument("call", "argument 'args'", "tuple", args[2]); + goto exit; + } + __clinic_args = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + if (!PyDict_Check(args[3])) { + _PyArg_BadArgument("call", "argument 'kwargs'", "dict", args[3]); + goto exit; + } + __clinic_kwargs = args[3]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[4]) { + preserve_exc = PyObject_IsTrue(args[4]); + if (preserve_exc < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + restricted = PyObject_IsTrue(args[5]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_call_impl(module, id, callable, __clinic_args, __clinic_kwargs, preserve_exc, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_is_shareable__doc__, +"is_shareable($module, /, obj)\n" +"--\n" +"\n" +"Return True if the object\'s data may be shared between interpreters and False otherwise."); + +#define _INTERPRETERS_IS_SHAREABLE_METHODDEF \ + {"is_shareable", _PyCFunction_CAST(_interpreters_is_shareable), METH_FASTCALL|METH_KEYWORDS, _interpreters_is_shareable__doc__}, + +static PyObject * +_interpreters_is_shareable_impl(PyObject *module, PyObject *obj); + +static PyObject * +_interpreters_is_shareable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(obj), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"obj", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_shareable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *obj; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + obj = args[0]; + return_value = _interpreters_is_shareable_impl(module, obj); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_is_running__doc__, +"is_running($module, /, id, *, restrict=False)\n" +"--\n" +"\n" +"Return whether or not the identified interpreter is running."); + +#define _INTERPRETERS_IS_RUNNING_METHODDEF \ + {"is_running", _PyCFunction_CAST(_interpreters_is_running), METH_FASTCALL|METH_KEYWORDS, _interpreters_is_running__doc__}, + +static PyObject * +_interpreters_is_running_impl(PyObject *module, PyObject *id, int restricted); + +static PyObject * +_interpreters_is_running(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "is_running", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[1]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_is_running_impl(module, id, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_get_config__doc__, +"get_config($module, /, id, *, restrict=False)\n" +"--\n" +"\n" +"Return a representation of the config used to initialize the interpreter."); + +#define _INTERPRETERS_GET_CONFIG_METHODDEF \ + {"get_config", _PyCFunction_CAST(_interpreters_get_config), METH_FASTCALL|METH_KEYWORDS, _interpreters_get_config__doc__}, + +static PyObject * +_interpreters_get_config_impl(PyObject *module, PyObject *id, int restricted); + +static PyObject * +_interpreters_get_config(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_config", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[1]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_get_config_impl(module, id, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_whence__doc__, +"whence($module, /, id)\n" +"--\n" +"\n" +"Return an identifier for where the interpreter was created."); + +#define _INTERPRETERS_WHENCE_METHODDEF \ + {"whence", _PyCFunction_CAST(_interpreters_whence), METH_FASTCALL|METH_KEYWORDS, _interpreters_whence__doc__}, + +static PyObject * +_interpreters_whence_impl(PyObject *module, PyObject *id); + +static PyObject * +_interpreters_whence(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "whence", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *id; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + return_value = _interpreters_whence_impl(module, id); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_incref__doc__, +"incref($module, /, id, *, implieslink=False, restrict=False)\n" +"--\n" +"\n"); + +#define _INTERPRETERS_INCREF_METHODDEF \ + {"incref", _PyCFunction_CAST(_interpreters_incref), METH_FASTCALL|METH_KEYWORDS, _interpreters_incref__doc__}, + +static PyObject * +_interpreters_incref_impl(PyObject *module, PyObject *id, int implieslink, + int restricted); + +static PyObject * +_interpreters_incref(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(implieslink), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "implieslink", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "incref", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int implieslink = 0; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[1]) { + implieslink = PyObject_IsTrue(args[1]); + if (implieslink < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + restricted = PyObject_IsTrue(args[2]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_incref_impl(module, id, implieslink, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_decref__doc__, +"decref($module, /, id, *, restrict=False)\n" +"--\n" +"\n"); + +#define _INTERPRETERS_DECREF_METHODDEF \ + {"decref", _PyCFunction_CAST(_interpreters_decref), METH_FASTCALL|METH_KEYWORDS, _interpreters_decref__doc__}, + +static PyObject * +_interpreters_decref_impl(PyObject *module, PyObject *id, int restricted); + +static PyObject * +_interpreters_decref(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(id), &_Py_ID(restrict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"id", "restrict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "decref", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *id; + int restricted = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + id = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + restricted = PyObject_IsTrue(args[1]); + if (restricted < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _interpreters_decref_impl(module, id, restricted); + +exit: + return return_value; +} + +PyDoc_STRVAR(_interpreters_capture_exception__doc__, +"capture_exception($module, /, exc=None)\n" +"--\n" +"\n" +"Return a snapshot of an exception.\n" +"\n" +"If \"exc\" is None then the current exception, if any, is used (but not cleared).\n" +"The returned snapshot is the same as what _interpreters.exec() returns."); + +#define _INTERPRETERS_CAPTURE_EXCEPTION_METHODDEF \ + {"capture_exception", _PyCFunction_CAST(_interpreters_capture_exception), METH_FASTCALL|METH_KEYWORDS, _interpreters_capture_exception__doc__}, + +static PyObject * +_interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg); + +static PyObject * +_interpreters_capture_exception(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(exc), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"exc", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "capture_exception", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *exc_arg = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + exc_arg = args[0]; +skip_optional_pos: + return_value = _interpreters_capture_exception_impl(module, exc_arg); + +exit: + return return_value; +} +/*[clinic end generated code: output=c80f73761f860f6c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_json.c.h b/Modules/clinic/_json.c.h new file mode 100644 index 00000000000..cd37a236c76 --- /dev/null +++ b/Modules/clinic/_json.c.h @@ -0,0 +1,128 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(py_scanstring__doc__, +"scanstring($module, pystr, end, strict=True, /)\n" +"--\n" +"\n" +"Scan the string s for a JSON string.\n" +"\n" +"End is the index of the character in s after the quote that started the\n" +"JSON string. Unescapes all valid JSON string escape sequences and raises\n" +"ValueError on attempt to decode an invalid string. If strict is False\n" +"then literal control characters are allowed in the string.\n" +"\n" +"Returns a tuple of the decoded string and the index of the character in s\n" +"after the end quote."); + +#define PY_SCANSTRING_METHODDEF \ + {"scanstring", _PyCFunction_CAST(py_scanstring), METH_FASTCALL, py_scanstring__doc__}, + +static PyObject * +py_scanstring_impl(PyObject *module, PyObject *pystr, Py_ssize_t end, + int strict); + +static PyObject * +py_scanstring(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *pystr; + Py_ssize_t end; + int strict = 1; + + if (!_PyArg_CheckPositional("scanstring", nargs, 2, 3)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("scanstring", "argument 1", "str", args[0]); + goto exit; + } + pystr = args[0]; + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + end = ival; + } + if (nargs < 3) { + goto skip_optional; + } + strict = PyObject_IsTrue(args[2]); + if (strict < 0) { + goto exit; + } +skip_optional: + return_value = py_scanstring_impl(module, pystr, end, strict); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_encode_basestring_ascii__doc__, +"encode_basestring_ascii($module, pystr, /)\n" +"--\n" +"\n" +"Return an ASCII-only JSON representation of a Python string"); + +#define PY_ENCODE_BASESTRING_ASCII_METHODDEF \ + {"encode_basestring_ascii", (PyCFunction)py_encode_basestring_ascii, METH_O, py_encode_basestring_ascii__doc__}, + +static PyObject * +py_encode_basestring_ascii_impl(PyObject *module, PyObject *pystr); + +static PyObject * +py_encode_basestring_ascii(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *pystr; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("encode_basestring_ascii", "argument", "str", arg); + goto exit; + } + pystr = arg; + return_value = py_encode_basestring_ascii_impl(module, pystr); + +exit: + return return_value; +} + +PyDoc_STRVAR(py_encode_basestring__doc__, +"encode_basestring($module, pystr, /)\n" +"--\n" +"\n" +"Return a JSON representation of a Python string"); + +#define PY_ENCODE_BASESTRING_METHODDEF \ + {"encode_basestring", (PyCFunction)py_encode_basestring, METH_O, py_encode_basestring__doc__}, + +static PyObject * +py_encode_basestring_impl(PyObject *module, PyObject *pystr); + +static PyObject * +py_encode_basestring(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *pystr; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("encode_basestring", "argument", "str", arg); + goto exit; + } + pystr = arg; + return_value = py_encode_basestring_impl(module, pystr); + +exit: + return return_value; +} +/*[clinic end generated code: output=5bdd16375c95a4d9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index 2918a6bc7ab..acb4aaf27e3 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_lsprof_Profiler_getstats__doc__, @@ -45,11 +46,18 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls); static PyObject * _lsprof_Profiler_getstats(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getstats() takes no arguments"); - return NULL; + goto exit; } - return _lsprof_Profiler_getstats_impl((ProfilerObject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _lsprof_Profiler_getstats_impl((ProfilerObject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_lsprof_Profiler__pystart_callback__doc__, @@ -76,7 +84,44 @@ _lsprof_Profiler__pystart_callback(PyObject *self, PyObject *const *args, Py_ssi } code = args[0]; instruction_offset = args[1]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler__pystart_callback_impl((ProfilerObject *)self, code, instruction_offset); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler__pythrow_callback__doc__, +"_pythrow_callback($self, code, instruction_offset, exception, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__PYTHROW_CALLBACK_METHODDEF \ + {"_pythrow_callback", _PyCFunction_CAST(_lsprof_Profiler__pythrow_callback), METH_FASTCALL, _lsprof_Profiler__pythrow_callback__doc__}, + +static PyObject * +_lsprof_Profiler__pythrow_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *exception); + +static PyObject * +_lsprof_Profiler__pythrow_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *exception; + + if (!_PyArg_CheckPositional("_pythrow_callback", nargs, 3, 3)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + exception = args[2]; + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _lsprof_Profiler__pythrow_callback_impl((ProfilerObject *)self, code, instruction_offset, exception); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -110,7 +155,9 @@ _lsprof_Profiler__pyreturn_callback(PyObject *self, PyObject *const *args, Py_ss code = args[0]; instruction_offset = args[1]; retval = args[2]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler__pyreturn_callback_impl((ProfilerObject *)self, code, instruction_offset, retval); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -145,7 +192,9 @@ _lsprof_Profiler__ccall_callback(PyObject *self, PyObject *const *args, Py_ssize instruction_offset = args[1]; callable = args[2]; self_arg = args[3]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler__ccall_callback_impl((ProfilerObject *)self, code, instruction_offset, callable, self_arg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -182,7 +231,9 @@ _lsprof_Profiler__creturn_callback(PyObject *self, PyObject *const *args, Py_ssi instruction_offset = args[1]; callable = args[2]; self_arg = args[3]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler__creturn_callback_impl((ProfilerObject *)self, code, instruction_offset, callable, self_arg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -266,7 +317,9 @@ _lsprof_Profiler_enable(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler_enable_impl((ProfilerObject *)self, subcalls, builtins); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -287,7 +340,13 @@ _lsprof_Profiler_disable_impl(ProfilerObject *self); static PyObject * _lsprof_Profiler_disable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _lsprof_Profiler_disable_impl((ProfilerObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _lsprof_Profiler_disable_impl((ProfilerObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_lsprof_Profiler_clear__doc__, @@ -305,7 +364,13 @@ _lsprof_Profiler_clear_impl(ProfilerObject *self); static PyObject * _lsprof_Profiler_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _lsprof_Profiler_clear_impl((ProfilerObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _lsprof_Profiler_clear_impl((ProfilerObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(profiler_init__doc__, @@ -411,4 +476,4 @@ skip_optional_pos: exit: return return_value; } -/*[clinic end generated code: output=fe231309776df7a7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af26a0b0ddcc3351 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 1e989e970c9..2563a16aea0 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_UInt64_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_random_Random_random__doc__, @@ -124,16 +125,15 @@ PyDoc_STRVAR(_random_Random_getrandbits__doc__, {"getrandbits", (PyCFunction)_random_Random_getrandbits, METH_O, _random_Random_getrandbits__doc__}, static PyObject * -_random_Random_getrandbits_impl(RandomObject *self, int k); +_random_Random_getrandbits_impl(RandomObject *self, uint64_t k); static PyObject * _random_Random_getrandbits(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; - int k; + uint64_t k; - k = PyLong_AsInt(arg); - if (k == -1 && PyErr_Occurred()) { + if (!_PyLong_UInt64_Converter(arg, &k)) { goto exit; } Py_BEGIN_CRITICAL_SECTION(self); @@ -143,4 +143,4 @@ _random_Random_getrandbits(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=4458b5a69201ebea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7ce97b2194eecaf7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h new file mode 100644 index 00000000000..60adb357e32 --- /dev/null +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -0,0 +1,350 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, +"RemoteUnwinder(pid, *, all_threads=False, only_active_thread=False,\n" +" mode=0, debug=False, skip_non_matching_threads=True,\n" +" native=False, gc=False)\n" +"--\n" +"\n" +"Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" +"\n" +"Args:\n" +" pid: Process ID of the target Python process to debug\n" +" all_threads: If True, initialize state for all threads in the process.\n" +" If False, only initialize for the main thread.\n" +" only_active_thread: If True, only sample the thread holding the GIL.\n" +" mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL (gil-time).\n" +" Cannot be used together with all_threads=True.\n" +" debug: If True, chain exceptions to explain the sequence of events that\n" +" lead to the exception.\n" +" skip_non_matching_threads: If True, skip threads that don\'t match the selected mode.\n" +" If False, include all threads regardless of mode.\n" +" native: If True, include artificial \"<native>\" frames to denote calls to\n" +" non-Python code.\n" +" gc: If True, include artificial \"<GC>\" frames to denote active garbage\n" +" collection.\n" +"\n" +"The RemoteUnwinder provides functionality to inspect and debug a running Python\n" +"process, including examining thread states, stack frames and other runtime data.\n" +"\n" +"Raises:\n" +" PermissionError: If access to the target process is denied\n" +" OSError: If unable to attach to the target process or access its memory\n" +" RuntimeError: If unable to read debug information from the target process\n" +" ValueError: If both all_threads and only_active_thread are True"); + +static int +_remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, + int pid, int all_threads, + int only_active_thread, + int mode, int debug, + int skip_non_matching_threads, + int native, int gc); + +static int +_remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 8 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), &_Py_ID(native), &_Py_ID(gc), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", "native", "gc", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "RemoteUnwinder", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[8]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; + int pid; + int all_threads = 0; + int only_active_thread = 0; + int mode = 0; + int debug = 0; + int skip_non_matching_threads = 1; + int native = 0; + int gc = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + pid = PyLong_AsInt(fastargs[0]); + if (pid == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (fastargs[1]) { + all_threads = PyObject_IsTrue(fastargs[1]); + if (all_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[2]) { + only_active_thread = PyObject_IsTrue(fastargs[2]); + if (only_active_thread < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + mode = PyLong_AsInt(fastargs[3]); + if (mode == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + debug = PyObject_IsTrue(fastargs[4]); + if (debug < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[5]) { + skip_non_matching_threads = PyObject_IsTrue(fastargs[5]); + if (skip_non_matching_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[6]) { + native = PyObject_IsTrue(fastargs[6]); + if (native < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + gc = PyObject_IsTrue(fastargs[7]); + if (gc < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads, native, gc); + +exit: + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, +"get_stack_trace($self, /)\n" +"--\n" +"\n" +"Returns stack traces for all interpreters and threads in process.\n" +"\n" +"Each element in the returned list is a tuple of (interpreter_id, thread_list), where:\n" +"- interpreter_id is the interpreter identifier\n" +"- thread_list is a list of tuples (thread_id, frame_list) for threads in that interpreter\n" +" - thread_id is the OS thread identifier\n" +" - frame_list is a list of tuples (function_name, filename, line_number) representing\n" +" the Python stack frames for that thread, ordered from most recent to oldest\n" +"\n" +"The threads returned depend on the initialization parameters:\n" +"- If only_active_thread was True: returns only the thread holding the GIL across all interpreters\n" +"- If all_threads was True: returns all threads across all interpreters\n" +"- Otherwise: returns only the main thread of each interpreter\n" +"\n" +"Example:\n" +" [\n" +" (0, [ # Main interpreter\n" +" (1234, [\n" +" (\'process_data\', \'worker.py\', 127),\n" +" (\'run_worker\', \'worker.py\', 45),\n" +" (\'main\', \'app.py\', 23)\n" +" ]),\n" +" (1235, [\n" +" (\'handle_request\', \'server.py\', 89),\n" +" (\'serve_forever\', \'server.py\', 52)\n" +" ])\n" +" ]),\n" +" (1, [ # Sub-interpreter\n" +" (1236, [\n" +" (\'sub_worker\', \'sub.py\', 15)\n" +" ])\n" +" ])\n" +" ]\n" +"\n" +"Raises:\n" +" RuntimeError: If there is an error copying memory from the target process\n" +" OSError: If there is an error accessing the target process\n" +" PermissionError: If access to the target process is denied\n" +" UnicodeDecodeError: If there is an error decoding strings from the target process"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF \ + {"get_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_stack_trace__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_stack_trace_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, +"get_all_awaited_by($self, /)\n" +"--\n" +"\n" +"Get all tasks and their awaited_by relationships from the remote process.\n" +"\n" +"This provides a tree structure showing which tasks are waiting for other tasks.\n" +"\n" +"For each task, returns:\n" +"1. The call stack frames leading to where the task is currently executing\n" +"2. The name of the task\n" +"3. A list of tasks that this task is waiting for, with their own frames/names/etc\n" +"\n" +"Returns a list of [frames, task_name, subtasks] where:\n" +"- frames: List of (func_name, filename, lineno) showing the call stack\n" +"- task_name: String identifier for the task\n" +"- subtasks: List of tasks being awaited by this task, in same format\n" +"\n" +"Raises:\n" +" RuntimeError: If AsyncioDebug section is not available in the remote process\n" +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output:\n" +"[\n" +" [\n" +" [(\"c5\", \"script.py\", 10), (\"c4\", \"script.py\", 14)],\n" +" \"c2_root\",\n" +" [\n" +" [\n" +" [(\"c1\", \"script.py\", 23)],\n" +" \"sub_main_2\",\n" +" [...]\n" +" ],\n" +" [...]\n" +" ]\n" +" ]\n" +"]"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF \ + {"get_all_awaited_by", (PyCFunction)_remote_debugging_RemoteUnwinder_get_all_awaited_by, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, +"get_async_stack_trace($self, /)\n" +"--\n" +"\n" +"Get the currently running async tasks and their dependency graphs from the remote process.\n" +"\n" +"This returns information about running tasks and all tasks that are waiting for them,\n" +"forming a complete dependency graph for each thread\'s active task.\n" +"\n" +"For each thread with a running task, returns the running task plus all tasks that\n" +"transitively depend on it (tasks waiting for the running task, tasks waiting for\n" +"those tasks, etc.).\n" +"\n" +"Returns a list of per-thread results, where each thread result contains:\n" +"- Thread ID\n" +"- List of task information for the running task and all its waiters\n" +"\n" +"Each task info contains:\n" +"- Task ID (memory address)\n" +"- Task name\n" +"- Call stack frames: List of (func_name, filename, lineno)\n" +"- List of tasks waiting for this task (recursive structure)\n" +"\n" +"Raises:\n" +" RuntimeError: If AsyncioDebug section is not available in the target process\n" +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output (similar structure to get_all_awaited_by but only for running tasks):\n" +"[\n" +" (140234, [\n" +" (4345585712, \'main_task\',\n" +" [(\"run_server\", \"server.py\", 127), (\"main\", \"app.py\", 23)],\n" +" [\n" +" (4345585800, \'worker_1\', [...], [...]),\n" +" (4345585900, \'worker_2\', [...], [...])\n" +" ])\n" +" ])\n" +"]"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF \ + {"get_async_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_async_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=99fed5c94cf36881 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index c6e2abd4d93..d1fb024903e 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_Size_t_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, @@ -47,7 +48,7 @@ static PyObject * _ssl__test_decode_cert(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *path; + PyObject *path = NULL; if (!PyUnicode_FSConverter(arg, &path)) { goto exit; @@ -55,6 +56,9 @@ _ssl__test_decode_cert(PyObject *module, PyObject *arg) return_value = _ssl__test_decode_cert_impl(module, path); exit: + /* Cleanup for path */ + Py_XDECREF(path); + return return_value; } @@ -195,6 +199,75 @@ _ssl__SSLSocket_cipher(PyObject *self, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_ssl__SSLSocket_group__doc__, +"group($self, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLSOCKET_GROUP_METHODDEF \ + {"group", (PyCFunction)_ssl__SSLSocket_group, METH_NOARGS, _ssl__SSLSocket_group__doc__}, + +static PyObject * +_ssl__SSLSocket_group_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_group(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_group_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_client_sigalg__doc__, +"client_sigalg($self, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLSOCKET_CLIENT_SIGALG_METHODDEF \ + {"client_sigalg", (PyCFunction)_ssl__SSLSocket_client_sigalg, METH_NOARGS, _ssl__SSLSocket_client_sigalg__doc__}, + +static PyObject * +_ssl__SSLSocket_client_sigalg_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_client_sigalg(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_client_sigalg_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_server_sigalg__doc__, +"server_sigalg($self, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLSOCKET_SERVER_SIGALG_METHODDEF \ + {"server_sigalg", (PyCFunction)_ssl__SSLSocket_server_sigalg, METH_NOARGS, _ssl__SSLSocket_server_sigalg__doc__}, + +static PyObject * +_ssl__SSLSocket_server_sigalg_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_server_sigalg(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_server_sigalg_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(_ssl__SSLSocket_version__doc__, "version($self, /)\n" "--\n" @@ -442,6 +515,115 @@ _ssl__SSLSocket_owner_set(PyObject *self, PyObject *value, void *Py_UNUSED(conte return return_value; } +PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_send__doc__, +"uses_ktls_for_send($self, /)\n" +"--\n" +"\n" +"Check if the Kernel TLS data-path is used for sending."); + +#define _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF \ + {"uses_ktls_for_send", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_send, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_send__doc__}, + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_uses_ktls_for_send_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_recv__doc__, +"uses_ktls_for_recv($self, /)\n" +"--\n" +"\n" +"Check if the Kernel TLS data-path is used for receiving."); + +#define _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF \ + {"uses_ktls_for_recv", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_recv, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_recv__doc__}, + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_uses_ktls_for_recv_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(BIO_get_ktls_send) + +PyDoc_STRVAR(_ssl__SSLSocket_sendfile__doc__, +"sendfile($self, fd, offset, size, flags=0, /)\n" +"--\n" +"\n" +"Write size bytes from offset in the file descriptor fd to the SSL connection.\n" +"\n" +"This method uses the zero-copy technique and returns the number of bytes\n" +"written. It should be called only when Kernel TLS is used for sending data in\n" +"the connection.\n" +"\n" +"The meaning of flags is platform dependent."); + +#define _SSL__SSLSOCKET_SENDFILE_METHODDEF \ + {"sendfile", _PyCFunction_CAST(_ssl__SSLSocket_sendfile), METH_FASTCALL, _ssl__SSLSocket_sendfile__doc__}, + +static PyObject * +_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, + size_t size, int flags); + +static PyObject * +_ssl__SSLSocket_sendfile(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fd; + Py_off_t offset; + size_t size; + int flags = 0; + + if (!_PyArg_CheckPositional("sendfile", nargs, 3, 4)) { + goto exit; + } + fd = PyLong_AsInt(args[0]); + if (fd == -1 && PyErr_Occurred()) { + goto exit; + } + if (!Py_off_t_converter(args[1], &offset)) { + goto exit; + } + if (!_PyLong_Size_t_Converter(args[2], &size)) { + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + flags = PyLong_AsInt(args[3]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_sendfile_impl((PySSLSocket *)self, fd, offset, size, flags); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* defined(BIO_get_ktls_send) */ + PyDoc_STRVAR(_ssl__SSLSocket_write__doc__, "write($self, b, /)\n" "--\n" @@ -836,6 +1018,45 @@ exit: return return_value; } +PyDoc_STRVAR(_ssl__SSLContext_set_ciphersuites__doc__, +"set_ciphersuites($self, ciphersuites, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_CIPHERSUITES_METHODDEF \ + {"set_ciphersuites", (PyCFunction)_ssl__SSLContext_set_ciphersuites, METH_O, _ssl__SSLContext_set_ciphersuites__doc__}, + +static PyObject * +_ssl__SSLContext_set_ciphersuites_impl(PySSLContext *self, + const char *ciphersuites); + +static PyObject * +_ssl__SSLContext_set_ciphersuites(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *ciphersuites; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("set_ciphersuites", "argument", "str", arg); + goto exit; + } + Py_ssize_t ciphersuites_length; + ciphersuites = PyUnicode_AsUTF8AndSize(arg, &ciphersuites_length); + if (ciphersuites == NULL) { + goto exit; + } + if (strlen(ciphersuites) != (size_t)ciphersuites_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_ciphersuites_impl((PySSLContext *)self, ciphersuites); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(_ssl__SSLContext_get_ciphers__doc__, "get_ciphers($self, /)\n" "--\n" @@ -859,6 +1080,189 @@ _ssl__SSLContext_get_ciphers(PyObject *self, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_ssl__SSLContext_set_groups__doc__, +"set_groups($self, grouplist, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_GROUPS_METHODDEF \ + {"set_groups", (PyCFunction)_ssl__SSLContext_set_groups, METH_O, _ssl__SSLContext_set_groups__doc__}, + +static PyObject * +_ssl__SSLContext_set_groups_impl(PySSLContext *self, const char *grouplist); + +static PyObject * +_ssl__SSLContext_set_groups(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *grouplist; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("set_groups", "argument", "str", arg); + goto exit; + } + Py_ssize_t grouplist_length; + grouplist = PyUnicode_AsUTF8AndSize(arg, &grouplist_length); + if (grouplist == NULL) { + goto exit; + } + if (strlen(grouplist) != (size_t)grouplist_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_groups_impl((PySSLContext *)self, grouplist); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_get_groups__doc__, +"get_groups($self, /, *, include_aliases=False)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_GET_GROUPS_METHODDEF \ + {"get_groups", _PyCFunction_CAST(_ssl__SSLContext_get_groups), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_get_groups__doc__}, + +static PyObject * +_ssl__SSLContext_get_groups_impl(PySSLContext *self, int include_aliases); + +static PyObject * +_ssl__SSLContext_get_groups(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(include_aliases), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"include_aliases", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_groups", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int include_aliases = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + include_aliases = PyObject_IsTrue(args[0]); + if (include_aliases < 0) { + goto exit; + } +skip_optional_kwonly: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_get_groups_impl((PySSLContext *)self, include_aliases); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_set_client_sigalgs__doc__, +"set_client_sigalgs($self, sigalgslist, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_CLIENT_SIGALGS_METHODDEF \ + {"set_client_sigalgs", (PyCFunction)_ssl__SSLContext_set_client_sigalgs, METH_O, _ssl__SSLContext_set_client_sigalgs__doc__}, + +static PyObject * +_ssl__SSLContext_set_client_sigalgs_impl(PySSLContext *self, + const char *sigalgslist); + +static PyObject * +_ssl__SSLContext_set_client_sigalgs(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *sigalgslist; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("set_client_sigalgs", "argument", "str", arg); + goto exit; + } + Py_ssize_t sigalgslist_length; + sigalgslist = PyUnicode_AsUTF8AndSize(arg, &sigalgslist_length); + if (sigalgslist == NULL) { + goto exit; + } + if (strlen(sigalgslist) != (size_t)sigalgslist_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_client_sigalgs_impl((PySSLContext *)self, sigalgslist); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLContext_set_server_sigalgs__doc__, +"set_server_sigalgs($self, sigalgslist, /)\n" +"--\n" +"\n"); + +#define _SSL__SSLCONTEXT_SET_SERVER_SIGALGS_METHODDEF \ + {"set_server_sigalgs", (PyCFunction)_ssl__SSLContext_set_server_sigalgs, METH_O, _ssl__SSLContext_set_server_sigalgs__doc__}, + +static PyObject * +_ssl__SSLContext_set_server_sigalgs_impl(PySSLContext *self, + const char *sigalgslist); + +static PyObject * +_ssl__SSLContext_set_server_sigalgs(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *sigalgslist; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("set_server_sigalgs", "argument", "str", arg); + goto exit; + } + Py_ssize_t sigalgslist_length; + sigalgslist = PyUnicode_AsUTF8AndSize(arg, &sigalgslist_length); + if (sigalgslist == NULL) { + goto exit; + } + if (strlen(sigalgslist) != (size_t)sigalgslist_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLContext_set_server_sigalgs_impl((PySSLContext *)self, sigalgslist); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(_ssl__SSLContext__set_alpn_protocols__doc__, "_set_alpn_protocols($self, protos, /)\n" "--\n" @@ -2615,6 +3019,23 @@ _ssl_get_default_verify_paths(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_ssl_get_sigalgs__doc__, +"get_sigalgs($module, /)\n" +"--\n" +"\n"); + +#define _SSL_GET_SIGALGS_METHODDEF \ + {"get_sigalgs", (PyCFunction)_ssl_get_sigalgs, METH_NOARGS, _ssl_get_sigalgs__doc__}, + +static PyObject * +_ssl_get_sigalgs_impl(PyObject *module); + +static PyObject * +_ssl_get_sigalgs(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _ssl_get_sigalgs_impl(module); +} + PyDoc_STRVAR(_ssl_txt2obj__doc__, "txt2obj($module, /, txt, name=False)\n" "--\n" @@ -2893,6 +3314,10 @@ exit: #endif /* defined(_MSC_VER) */ +#ifndef _SSL__SSLSOCKET_SENDFILE_METHODDEF + #define _SSL__SSLSOCKET_SENDFILE_METHODDEF +#endif /* !defined(_SSL__SSLSOCKET_SENDFILE_METHODDEF) */ + #ifndef _SSL_ENUM_CERTIFICATES_METHODDEF #define _SSL_ENUM_CERTIFICATES_METHODDEF #endif /* !defined(_SSL_ENUM_CERTIFICATES_METHODDEF) */ @@ -2900,4 +3325,4 @@ exit: #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=748650909fec8906 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3b6c9cbfc4660ecb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_suggestions.c.h b/Modules/clinic/_suggestions.c.h index 51484b13d5a..3b3ed5056ac 100644 --- a/Modules/clinic/_suggestions.c.h +++ b/Modules/clinic/_suggestions.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_suggestions__generate_suggestions__doc__, @@ -33,9 +34,11 @@ _suggestions__generate_suggestions(PyObject *module, PyObject *const *args, Py_s goto exit; } item = args[1]; + Py_BEGIN_CRITICAL_SECTION(candidates); return_value = _suggestions__generate_suggestions_impl(module, candidates, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=1d8e963cdae30b13 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1690dd15a464d19c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 970528ce9ea..9bcd0eeb008 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -9,7 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_tuple.h" // _PyTuple_ITEMS() PyDoc_STRVAR(test_empty_function__doc__, "test_empty_function($module, /)\n" @@ -746,12 +746,19 @@ unsigned_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional; } { - unsigned long ival = PyLong_AsUnsignedLongMask(args[2]); - if (ival == (unsigned long)-1 && PyErr_Occurred()) { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned char), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { goto exit; } - else { - c = (unsigned char) ival; + if ((size_t)_bytes > sizeof(unsigned char)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } } } skip_optional: @@ -848,9 +855,21 @@ unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - c = (unsigned short)PyLong_AsUnsignedLongMask(args[2]); - if (c == (unsigned short)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned short), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned short)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = unsigned_short_converter_impl(module, a, b, c); @@ -955,9 +974,21 @@ unsigned_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (nargs < 3) { goto skip_optional; } - c = (unsigned int)PyLong_AsUnsignedLongMask(args[2]); - if (c == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = unsigned_int_converter_impl(module, a, b, c); @@ -1042,7 +1073,22 @@ unsigned_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t narg _PyArg_BadArgument("unsigned_long_converter", "argument 3", "int", args[2]); goto exit; } - c = PyLong_AsUnsignedLongMask(args[2]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } skip_optional: return_value = unsigned_long_converter_impl(module, a, b, c); @@ -1126,7 +1172,22 @@ unsigned_long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t _PyArg_BadArgument("unsigned_long_long_converter", "argument 3", "int", args[2]); goto exit; } - c = PyLong_AsUnsignedLongLongMask(args[2]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned long long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } skip_optional: return_value = unsigned_long_long_converter_impl(module, a, b, c); @@ -1135,7 +1196,8 @@ exit: } PyDoc_STRVAR(py_ssize_t_converter__doc__, -"py_ssize_t_converter($module, a=12, b=34, c=56, /)\n" +"py_ssize_t_converter($module, a=12, b=34, c=56, d=78, e=90, f=-12,\n" +" g=-34, /)\n" "--\n" "\n"); @@ -1144,7 +1206,8 @@ PyDoc_STRVAR(py_ssize_t_converter__doc__, static PyObject * py_ssize_t_converter_impl(PyObject *module, Py_ssize_t a, Py_ssize_t b, - Py_ssize_t c); + Py_ssize_t c, Py_ssize_t d, Py_ssize_t e, + Py_ssize_t f, Py_ssize_t g); static PyObject * py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -1153,8 +1216,12 @@ py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_ssize_t a = 12; Py_ssize_t b = 34; Py_ssize_t c = 56; + Py_ssize_t d = 78; + Py_ssize_t e = 90; + Py_ssize_t f = -12; + Py_ssize_t g = -34; - if (!_PyArg_CheckPositional("py_ssize_t_converter", nargs, 0, 3)) { + if (!_PyArg_CheckPositional("py_ssize_t_converter", nargs, 0, 7)) { goto exit; } if (nargs < 1) { @@ -1193,8 +1260,60 @@ py_ssize_t_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_Py_convert_optional_to_ssize_t(args[2], &c)) { goto exit; } + if (nargs < 4) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[3]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + d = ival; + if (d < 0) { + PyErr_SetString(PyExc_ValueError, + "d cannot be negative"); + goto exit; + } + } + if (nargs < 5) { + goto skip_optional; + } + if (!_Py_convert_optional_to_non_negative_ssize_t(args[4], &e)) { + goto exit; + } + if (nargs < 6) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[5]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + f = ival; + if (f < 0) { + PyErr_SetString(PyExc_ValueError, + "f cannot be negative"); + goto exit; + } + } + if (nargs < 7) { + goto skip_optional; + } + if (!_Py_convert_optional_to_non_negative_ssize_t(args[6], &g)) { + goto exit; + } skip_optional: - return_value = py_ssize_t_converter_impl(module, a, b, c); + return_value = py_ssize_t_converter_impl(module, a, b, c, d, e, f, g); exit: return return_value; @@ -2645,7 +2764,7 @@ varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *__clinic_args = NULL; - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -2683,7 +2802,7 @@ posonly_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } a = args[0]; b = args[1]; - __clinic_args = _PyTuple_FromArray(args + 2, nargs - 2); + __clinic_args = PyTuple_FromArray(args + 2, nargs - 2); if (__clinic_args == NULL) { goto exit; } @@ -2726,7 +2845,7 @@ posonly_req_opt_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs b = args[1]; skip_optional: __clinic_args = nargs > 2 - ? _PyTuple_FromArray(args + 2, nargs - 2) + ? PyTuple_FromArray(args + 2, nargs - 2) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -2797,7 +2916,7 @@ posonly_poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, a = fastargs[0]; b = fastargs[1]; __clinic_args = nargs > 2 - ? _PyTuple_FromArray(args + 2, nargs - 2) + ? PyTuple_FromArray(args + 2, nargs - 2) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -2865,7 +2984,7 @@ poskw_varpos(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } a = fastargs[0]; __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -2944,7 +3063,7 @@ poskw_varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg } skip_optional_kwonly: __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -3027,7 +3146,7 @@ poskw_varpos_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar c = fastargs[2]; skip_optional_kwonly: __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -3099,7 +3218,7 @@ varpos_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO } b = fastargs[0]; skip_optional_kwonly: - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -3180,7 +3299,7 @@ varpos_kwonly_req_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } c = fastargs[2]; skip_optional_kwonly: - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -3430,7 +3549,7 @@ gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject kw2 = fastargs[3]; skip_optional_kwonly: varargs = nargs > 2 - ? _PyTuple_FromArray(args + 2, nargs - 2) + ? PyTuple_FromArray(args + 2, nargs - 2) : PyTuple_New(0); if (varargs == NULL) { goto exit; @@ -3507,7 +3626,7 @@ gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb kw = fastargs[1]; skip_optional_kwonly: __clinic_args = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (__clinic_args == NULL) { goto exit; @@ -3539,7 +3658,7 @@ gh_99233_refcount(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *__clinic_args = NULL; - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -3652,7 +3771,7 @@ null_or_tuple_for_varargs(PyObject *module, PyObject *const *args, Py_ssize_t na } skip_optional_kwonly: constraints = nargs > 1 - ? _PyTuple_FromArray(args + 1, nargs - 1) + ? PyTuple_FromArray(args + 1, nargs - 1) : PyTuple_New(0); if (constraints == NULL) { goto exit; @@ -4055,7 +4174,7 @@ _testclinic_TestClass_defclass_varpos(PyObject *self, PyTypeObject *cls, PyObjec if (!fastargs) { goto exit; } - __clinic_args = _PyTuple_FromArray(args, nargs); + __clinic_args = PyTuple_FromArray(args, nargs); if (__clinic_args == NULL) { goto exit; } @@ -4112,7 +4231,7 @@ _testclinic_TestClass_defclass_posonly_varpos(PyObject *self, PyTypeObject *cls, } a = fastargs[0]; b = fastargs[1]; - __clinic_args = _PyTuple_FromArray(args + 2, nargs - 2); + __clinic_args = PyTuple_FromArray(args + 2, nargs - 2); if (__clinic_args == NULL) { goto exit; } @@ -4481,4 +4600,4 @@ _testclinic_TestClass_posonly_poskw_varpos_array_no_fastcall(PyObject *type, PyO exit: return return_value; } -/*[clinic end generated code: output=84ffc31f27215baa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=290d2e346ea7bfa1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index a46d238801b..e2db4fd87ed 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -9,7 +9,7 @@ preserve #include "pycore_long.h" // _PyLong_UnsignedShort_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_tuple.h" // _PyTuple_ITEMS() PyDoc_STRVAR(depr_star_new__doc__, "DeprStarNew(a=None)\n" @@ -2474,4 +2474,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=4e60af44fd6b7b94 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2231bec0ed196830 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_kwds.c.h b/Modules/clinic/_testclinic_kwds.c.h new file mode 100644 index 00000000000..86cad50c56c --- /dev/null +++ b/Modules/clinic/_testclinic_kwds.c.h @@ -0,0 +1,184 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +#endif +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_long.h" // _PyLong_UnsignedShort_Converter() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_runtime.h" // _Py_ID() +#include "pycore_tuple.h" // _PyTuple_ITEMS() + +PyDoc_STRVAR(lone_kwds__doc__, +"lone_kwds($module, /, **kwds)\n" +"--\n" +"\n"); + +#define LONE_KWDS_METHODDEF \ + {"lone_kwds", _PyCFunction_CAST(lone_kwds), METH_VARARGS|METH_KEYWORDS, lone_kwds__doc__}, + +static PyObject * +lone_kwds_impl(PyObject *module, PyObject *kwds); + +static PyObject * +lone_kwds(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_kwds = NULL; + + if (!_PyArg_NoPositional("lone_kwds", args)) { + goto exit; + } + if (kwargs == NULL) { + __clinic_kwds = PyDict_New(); + if (__clinic_kwds == NULL) { + goto exit; + } + } + else { + __clinic_kwds = Py_NewRef(kwargs); + } + return_value = lone_kwds_impl(module, __clinic_kwds); + +exit: + /* Cleanup for kwds */ + Py_XDECREF(__clinic_kwds); + + return return_value; +} + +PyDoc_STRVAR(kwds_with_pos_only__doc__, +"kwds_with_pos_only($module, a, b, /, **kwds)\n" +"--\n" +"\n"); + +#define KWDS_WITH_POS_ONLY_METHODDEF \ + {"kwds_with_pos_only", _PyCFunction_CAST(kwds_with_pos_only), METH_VARARGS|METH_KEYWORDS, kwds_with_pos_only__doc__}, + +static PyObject * +kwds_with_pos_only_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *kwds); + +static PyObject * +kwds_with_pos_only(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b; + PyObject *__clinic_kwds = NULL; + + if (!_PyArg_CheckPositional("kwds_with_pos_only", PyTuple_GET_SIZE(args), 2, 2)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + b = PyTuple_GET_ITEM(args, 1); + if (kwargs == NULL) { + __clinic_kwds = PyDict_New(); + if (__clinic_kwds == NULL) { + goto exit; + } + } + else { + __clinic_kwds = Py_NewRef(kwargs); + } + return_value = kwds_with_pos_only_impl(module, a, b, __clinic_kwds); + +exit: + /* Cleanup for kwds */ + Py_XDECREF(__clinic_kwds); + + return return_value; +} + +PyDoc_STRVAR(kwds_with_stararg__doc__, +"kwds_with_stararg($module, /, *args, **kwds)\n" +"--\n" +"\n"); + +#define KWDS_WITH_STARARG_METHODDEF \ + {"kwds_with_stararg", _PyCFunction_CAST(kwds_with_stararg), METH_VARARGS|METH_KEYWORDS, kwds_with_stararg__doc__}, + +static PyObject * +kwds_with_stararg_impl(PyObject *module, PyObject *args, PyObject *kwds); + +static PyObject * +kwds_with_stararg(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyObject *__clinic_args = NULL; + PyObject *__clinic_kwds = NULL; + + __clinic_args = Py_NewRef(args); + if (kwargs == NULL) { + __clinic_kwds = PyDict_New(); + if (__clinic_kwds == NULL) { + goto exit; + } + } + else { + __clinic_kwds = Py_NewRef(kwargs); + } + return_value = kwds_with_stararg_impl(module, __clinic_args, __clinic_kwds); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + /* Cleanup for kwds */ + Py_XDECREF(__clinic_kwds); + + return return_value; +} + +PyDoc_STRVAR(kwds_with_pos_only_and_stararg__doc__, +"kwds_with_pos_only_and_stararg($module, a, b, /, *args, **kwds)\n" +"--\n" +"\n"); + +#define KWDS_WITH_POS_ONLY_AND_STARARG_METHODDEF \ + {"kwds_with_pos_only_and_stararg", _PyCFunction_CAST(kwds_with_pos_only_and_stararg), METH_VARARGS|METH_KEYWORDS, kwds_with_pos_only_and_stararg__doc__}, + +static PyObject * +kwds_with_pos_only_and_stararg_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *args, + PyObject *kwds); + +static PyObject * +kwds_with_pos_only_and_stararg(PyObject *module, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + PyObject *__clinic_kwds = NULL; + + if (!_PyArg_CheckPositional("kwds_with_pos_only_and_stararg", PyTuple_GET_SIZE(args), 2, PY_SSIZE_T_MAX)) { + goto exit; + } + a = PyTuple_GET_ITEM(args, 0); + b = PyTuple_GET_ITEM(args, 1); + __clinic_args = PyTuple_GetSlice(args, 2, PY_SSIZE_T_MAX); + if (!__clinic_args) { + goto exit; + } + if (kwargs == NULL) { + __clinic_kwds = PyDict_New(); + if (__clinic_kwds == NULL) { + goto exit; + } + } + else { + __clinic_kwds = Py_NewRef(kwargs); + } + return_value = kwds_with_pos_only_and_stararg_impl(module, a, b, __clinic_args, __clinic_kwds); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + /* Cleanup for kwds */ + Py_XDECREF(__clinic_kwds); + + return return_value; +} +/*[clinic end generated code: output=3e5251b10aa44382 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_threadmodule.c.h b/Modules/clinic/_threadmodule.c.h index 8930e54170c..8455f9929ba 100644 --- a/Modules/clinic/_threadmodule.c.h +++ b/Modules/clinic/_threadmodule.c.h @@ -8,6 +8,635 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(_thread_lock_acquire__doc__, +"acquire($self, /, blocking=True, timeout=-1)\n" +"--\n" +"\n" +"Lock the lock.\n" +"\n" +"Without argument, this blocks if the lock is already\n" +"locked (even by the same thread), waiting for another thread to release\n" +"the lock, and return True once the lock is acquired.\n" +"With an argument, this will only block if the argument is true,\n" +"and the return value reflects whether the lock is acquired.\n" +"The blocking operation is interruptible."); + +#define _THREAD_LOCK_ACQUIRE_METHODDEF \ + {"acquire", _PyCFunction_CAST(_thread_lock_acquire), METH_FASTCALL|METH_KEYWORDS, _thread_lock_acquire__doc__}, + +static PyObject * +_thread_lock_acquire_impl(lockobject *self, int blocking, + PyObject *timeoutobj); + +static PyObject * +_thread_lock_acquire(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(blocking), &_Py_ID(timeout), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"blocking", "timeout", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "acquire", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int blocking = 1; + PyObject *timeoutobj = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + timeoutobj = args[1]; +skip_optional_pos: + return_value = _thread_lock_acquire_impl((lockobject *)self, blocking, timeoutobj); + +exit: + return return_value; +} + +PyDoc_STRVAR(_thread_lock_acquire_lock__doc__, +"acquire_lock($self, /, blocking=True, timeout=-1)\n" +"--\n" +"\n" +"An obsolete synonym of acquire()."); + +#define _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF \ + {"acquire_lock", _PyCFunction_CAST(_thread_lock_acquire_lock), METH_FASTCALL|METH_KEYWORDS, _thread_lock_acquire_lock__doc__}, + +static PyObject * +_thread_lock_acquire_lock_impl(lockobject *self, int blocking, + PyObject *timeoutobj); + +static PyObject * +_thread_lock_acquire_lock(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(blocking), &_Py_ID(timeout), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"blocking", "timeout", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "acquire_lock", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int blocking = 1; + PyObject *timeoutobj = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + timeoutobj = args[1]; +skip_optional_pos: + return_value = _thread_lock_acquire_lock_impl((lockobject *)self, blocking, timeoutobj); + +exit: + return return_value; +} + +PyDoc_STRVAR(_thread_lock_release__doc__, +"release($self, /)\n" +"--\n" +"\n" +"Release the lock.\n" +"\n" +"Allows another thread that is blocked waiting for\n" +"the lock to acquire the lock. The lock must be in the locked state,\n" +"but it needn\'t be locked by the same thread that unlocks it."); + +#define _THREAD_LOCK_RELEASE_METHODDEF \ + {"release", (PyCFunction)_thread_lock_release, METH_NOARGS, _thread_lock_release__doc__}, + +static PyObject * +_thread_lock_release_impl(lockobject *self); + +static PyObject * +_thread_lock_release(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_lock_release_impl((lockobject *)self); +} + +PyDoc_STRVAR(_thread_lock_release_lock__doc__, +"release_lock($self, /)\n" +"--\n" +"\n" +"An obsolete synonym of release()."); + +#define _THREAD_LOCK_RELEASE_LOCK_METHODDEF \ + {"release_lock", (PyCFunction)_thread_lock_release_lock, METH_NOARGS, _thread_lock_release_lock__doc__}, + +static PyObject * +_thread_lock_release_lock_impl(lockobject *self); + +static PyObject * +_thread_lock_release_lock(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_lock_release_lock_impl((lockobject *)self); +} + +PyDoc_STRVAR(_thread_lock___enter____doc__, +"__enter__($self, /)\n" +"--\n" +"\n" +"Lock the lock."); + +#define _THREAD_LOCK___ENTER___METHODDEF \ + {"__enter__", (PyCFunction)_thread_lock___enter__, METH_NOARGS, _thread_lock___enter____doc__}, + +static PyObject * +_thread_lock___enter___impl(lockobject *self); + +static PyObject * +_thread_lock___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_lock___enter___impl((lockobject *)self); +} + +PyDoc_STRVAR(_thread_lock___exit____doc__, +"__exit__($self, exc_type, exc_value, exc_tb, /)\n" +"--\n" +"\n" +"Release the lock."); + +#define _THREAD_LOCK___EXIT___METHODDEF \ + {"__exit__", _PyCFunction_CAST(_thread_lock___exit__), METH_FASTCALL, _thread_lock___exit____doc__}, + +static PyObject * +_thread_lock___exit___impl(lockobject *self, PyObject *exc_type, + PyObject *exc_value, PyObject *exc_tb); + +static PyObject * +_thread_lock___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc_type; + PyObject *exc_value; + PyObject *exc_tb; + + if (!_PyArg_CheckPositional("__exit__", nargs, 3, 3)) { + goto exit; + } + exc_type = args[0]; + exc_value = args[1]; + exc_tb = args[2]; + return_value = _thread_lock___exit___impl((lockobject *)self, exc_type, exc_value, exc_tb); + +exit: + return return_value; +} + +PyDoc_STRVAR(_thread_lock_locked__doc__, +"locked($self, /)\n" +"--\n" +"\n" +"Return whether the lock is in the locked state."); + +#define _THREAD_LOCK_LOCKED_METHODDEF \ + {"locked", (PyCFunction)_thread_lock_locked, METH_NOARGS, _thread_lock_locked__doc__}, + +static PyObject * +_thread_lock_locked_impl(lockobject *self); + +static PyObject * +_thread_lock_locked(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_lock_locked_impl((lockobject *)self); +} + +PyDoc_STRVAR(_thread_lock_locked_lock__doc__, +"locked_lock($self, /)\n" +"--\n" +"\n" +"An obsolete synonym of locked()."); + +#define _THREAD_LOCK_LOCKED_LOCK_METHODDEF \ + {"locked_lock", (PyCFunction)_thread_lock_locked_lock, METH_NOARGS, _thread_lock_locked_lock__doc__}, + +static PyObject * +_thread_lock_locked_lock_impl(lockobject *self); + +static PyObject * +_thread_lock_locked_lock(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_lock_locked_lock_impl((lockobject *)self); +} + +#if defined(HAVE_FORK) + +PyDoc_STRVAR(_thread_lock__at_fork_reinit__doc__, +"_at_fork_reinit($self, /)\n" +"--\n" +"\n"); + +#define _THREAD_LOCK__AT_FORK_REINIT_METHODDEF \ + {"_at_fork_reinit", (PyCFunction)_thread_lock__at_fork_reinit, METH_NOARGS, _thread_lock__at_fork_reinit__doc__}, + +static PyObject * +_thread_lock__at_fork_reinit_impl(lockobject *self); + +static PyObject * +_thread_lock__at_fork_reinit(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_lock__at_fork_reinit_impl((lockobject *)self); +} + +#endif /* defined(HAVE_FORK) */ + +static PyObject * +lock_new_impl(PyTypeObject *type); + +static PyObject * +lock_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = clinic_state()->lock_type; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("lock", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("lock", kwargs)) { + goto exit; + } + return_value = lock_new_impl(type); + +exit: + return return_value; +} + +PyDoc_STRVAR(_thread_RLock_acquire__doc__, +"acquire($self, /, blocking=True, timeout=-1)\n" +"--\n" +"\n" +"Lock the lock.\n" +"\n" +"`blocking` indicates whether we should wait\n" +"for the lock to be available or not. If `blocking` is False\n" +"and another thread holds the lock, the method will return False\n" +"immediately. If `blocking` is True and another thread holds\n" +"the lock, the method will wait for the lock to be released,\n" +"take it and then return True.\n" +"(note: the blocking operation is interruptible.)\n" +"\n" +"In all other cases, the method will return True immediately.\n" +"Precisely, if the current thread already holds the lock, its\n" +"internal counter is simply incremented. If nobody holds the lock,\n" +"the lock is taken and its internal counter initialized to 1."); + +#define _THREAD_RLOCK_ACQUIRE_METHODDEF \ + {"acquire", _PyCFunction_CAST(_thread_RLock_acquire), METH_FASTCALL|METH_KEYWORDS, _thread_RLock_acquire__doc__}, + +static PyObject * +_thread_RLock_acquire_impl(rlockobject *self, int blocking, + PyObject *timeoutobj); + +static PyObject * +_thread_RLock_acquire(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(blocking), &_Py_ID(timeout), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"blocking", "timeout", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "acquire", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int blocking = 1; + PyObject *timeoutobj = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + timeoutobj = args[1]; +skip_optional_pos: + return_value = _thread_RLock_acquire_impl((rlockobject *)self, blocking, timeoutobj); + +exit: + return return_value; +} + +PyDoc_STRVAR(_thread_RLock___enter____doc__, +"__enter__($self, /)\n" +"--\n" +"\n" +"Lock the lock."); + +#define _THREAD_RLOCK___ENTER___METHODDEF \ + {"__enter__", (PyCFunction)_thread_RLock___enter__, METH_NOARGS, _thread_RLock___enter____doc__}, + +static PyObject * +_thread_RLock___enter___impl(rlockobject *self); + +static PyObject * +_thread_RLock___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_RLock___enter___impl((rlockobject *)self); +} + +PyDoc_STRVAR(_thread_RLock_release__doc__, +"release($self, /)\n" +"--\n" +"\n" +"Release the lock.\n" +"\n" +"Allows another thread that is blocked waiting for\n" +"the lock to acquire the lock. The lock must be in the locked state,\n" +"and must be locked by the same thread that unlocks it; otherwise a\n" +"`RuntimeError` is raised.\n" +"\n" +"Do note that if the lock was acquire()d several times in a row by the\n" +"current thread, release() needs to be called as many times for the lock\n" +"to be available for other threads."); + +#define _THREAD_RLOCK_RELEASE_METHODDEF \ + {"release", (PyCFunction)_thread_RLock_release, METH_NOARGS, _thread_RLock_release__doc__}, + +static PyObject * +_thread_RLock_release_impl(rlockobject *self); + +static PyObject * +_thread_RLock_release(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_RLock_release_impl((rlockobject *)self); +} + +PyDoc_STRVAR(_thread_RLock___exit____doc__, +"__exit__($self, exc_type, exc_value, exc_tb, /)\n" +"--\n" +"\n" +"Release the lock."); + +#define _THREAD_RLOCK___EXIT___METHODDEF \ + {"__exit__", _PyCFunction_CAST(_thread_RLock___exit__), METH_FASTCALL, _thread_RLock___exit____doc__}, + +static PyObject * +_thread_RLock___exit___impl(rlockobject *self, PyObject *exc_type, + PyObject *exc_value, PyObject *exc_tb); + +static PyObject * +_thread_RLock___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc_type; + PyObject *exc_value; + PyObject *exc_tb; + + if (!_PyArg_CheckPositional("__exit__", nargs, 3, 3)) { + goto exit; + } + exc_type = args[0]; + exc_value = args[1]; + exc_tb = args[2]; + return_value = _thread_RLock___exit___impl((rlockobject *)self, exc_type, exc_value, exc_tb); + +exit: + return return_value; +} + +PyDoc_STRVAR(_thread_RLock_locked__doc__, +"locked($self, /)\n" +"--\n" +"\n" +"Return a boolean indicating whether this object is locked right now."); + +#define _THREAD_RLOCK_LOCKED_METHODDEF \ + {"locked", (PyCFunction)_thread_RLock_locked, METH_NOARGS, _thread_RLock_locked__doc__}, + +static PyObject * +_thread_RLock_locked_impl(rlockobject *self); + +static PyObject * +_thread_RLock_locked(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_RLock_locked_impl((rlockobject *)self); +} + +PyDoc_STRVAR(_thread_RLock__acquire_restore__doc__, +"_acquire_restore($self, state, /)\n" +"--\n" +"\n" +"For internal use by `threading.Condition`."); + +#define _THREAD_RLOCK__ACQUIRE_RESTORE_METHODDEF \ + {"_acquire_restore", (PyCFunction)_thread_RLock__acquire_restore, METH_O, _thread_RLock__acquire_restore__doc__}, + +static PyObject * +_thread_RLock__acquire_restore_impl(rlockobject *self, PyObject *state); + +static PyObject * +_thread_RLock__acquire_restore(PyObject *self, PyObject *state) +{ + PyObject *return_value = NULL; + + return_value = _thread_RLock__acquire_restore_impl((rlockobject *)self, state); + + return return_value; +} + +PyDoc_STRVAR(_thread_RLock__release_save__doc__, +"_release_save($self, /)\n" +"--\n" +"\n" +"For internal use by `threading.Condition`."); + +#define _THREAD_RLOCK__RELEASE_SAVE_METHODDEF \ + {"_release_save", (PyCFunction)_thread_RLock__release_save, METH_NOARGS, _thread_RLock__release_save__doc__}, + +static PyObject * +_thread_RLock__release_save_impl(rlockobject *self); + +static PyObject * +_thread_RLock__release_save(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_RLock__release_save_impl((rlockobject *)self); +} + +PyDoc_STRVAR(_thread_RLock__recursion_count__doc__, +"_recursion_count($self, /)\n" +"--\n" +"\n" +"For internal use by reentrancy checks."); + +#define _THREAD_RLOCK__RECURSION_COUNT_METHODDEF \ + {"_recursion_count", (PyCFunction)_thread_RLock__recursion_count, METH_NOARGS, _thread_RLock__recursion_count__doc__}, + +static PyObject * +_thread_RLock__recursion_count_impl(rlockobject *self); + +static PyObject * +_thread_RLock__recursion_count(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_RLock__recursion_count_impl((rlockobject *)self); +} + +PyDoc_STRVAR(_thread_RLock__is_owned__doc__, +"_is_owned($self, /)\n" +"--\n" +"\n" +"For internal use by `threading.Condition`."); + +#define _THREAD_RLOCK__IS_OWNED_METHODDEF \ + {"_is_owned", (PyCFunction)_thread_RLock__is_owned, METH_NOARGS, _thread_RLock__is_owned__doc__}, + +static PyObject * +_thread_RLock__is_owned_impl(rlockobject *self); + +static PyObject * +_thread_RLock__is_owned(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_RLock__is_owned_impl((rlockobject *)self); +} + +static PyObject * +rlock_new_impl(PyTypeObject *type); + +static PyObject * +rlock_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = clinic_state()->rlock_type; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("RLock", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("RLock", kwargs)) { + goto exit; + } + return_value = rlock_new_impl(type); + +exit: + return return_value; +} + +#if defined(HAVE_FORK) + +PyDoc_STRVAR(_thread_RLock__at_fork_reinit__doc__, +"_at_fork_reinit($self, /)\n" +"--\n" +"\n"); + +#define _THREAD_RLOCK__AT_FORK_REINIT_METHODDEF \ + {"_at_fork_reinit", (PyCFunction)_thread_RLock__at_fork_reinit, METH_NOARGS, _thread_RLock__at_fork_reinit__doc__}, + +static PyObject * +_thread_RLock__at_fork_reinit_impl(rlockobject *self); + +static PyObject * +_thread_RLock__at_fork_reinit(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _thread_RLock__at_fork_reinit_impl((rlockobject *)self); +} + +#endif /* defined(HAVE_FORK) */ + #if (defined(HAVE_PTHREAD_GETNAME_NP) || defined(HAVE_PTHREAD_GET_NAME_NP) || defined(MS_WINDOWS)) PyDoc_STRVAR(_thread__get_name__doc__, @@ -96,6 +725,14 @@ exit: #endif /* (defined(HAVE_PTHREAD_SETNAME_NP) || defined(HAVE_PTHREAD_SET_NAME_NP) || defined(MS_WINDOWS)) */ +#ifndef _THREAD_LOCK__AT_FORK_REINIT_METHODDEF + #define _THREAD_LOCK__AT_FORK_REINIT_METHODDEF +#endif /* !defined(_THREAD_LOCK__AT_FORK_REINIT_METHODDEF) */ + +#ifndef _THREAD_RLOCK__AT_FORK_REINIT_METHODDEF + #define _THREAD_RLOCK__AT_FORK_REINIT_METHODDEF +#endif /* !defined(_THREAD_RLOCK__AT_FORK_REINIT_METHODDEF) */ + #ifndef _THREAD__GET_NAME_METHODDEF #define _THREAD__GET_NAME_METHODDEF #endif /* !defined(_THREAD__GET_NAME_METHODDEF) */ @@ -103,4 +740,4 @@ exit: #ifndef _THREAD_SET_NAME_METHODDEF #define _THREAD_SET_NAME_METHODDEF #endif /* !defined(_THREAD_SET_NAME_METHODDEF) */ -/*[clinic end generated code: output=e978dc4615b9bc35 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1255a1520f43f97a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index f8b623fca08..bd685e75d93 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -857,6 +857,8 @@ exit: return return_value; } +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(_winapi_GetShortPathName__doc__, "GetShortPathName($module, /, path)\n" "--\n" @@ -930,6 +932,8 @@ exit: return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + PyDoc_STRVAR(_winapi_GetStdHandle__doc__, "GetStdHandle($module, std_handle, /)\n" "--\n" @@ -1929,6 +1933,24 @@ _winapi_GetACP(PyObject *module, PyObject *Py_UNUSED(ignored)) return _winapi_GetACP_impl(module); } +PyDoc_STRVAR(_winapi_GetOEMCP__doc__, +"GetOEMCP($module, /)\n" +"--\n" +"\n" +"Get the current Windows ANSI code page identifier."); + +#define _WINAPI_GETOEMCP_METHODDEF \ + {"GetOEMCP", (PyCFunction)_winapi_GetOEMCP, METH_NOARGS, _winapi_GetOEMCP__doc__}, + +static PyObject * +_winapi_GetOEMCP_impl(PyObject *module); + +static PyObject * +_winapi_GetOEMCP(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _winapi_GetOEMCP_impl(module); +} + PyDoc_STRVAR(_winapi_GetFileType__doc__, "GetFileType($module, /, handle)\n" "--\n" @@ -2161,4 +2183,8 @@ exit: return return_value; } -/*[clinic end generated code: output=6cd07628af447d0a input=a9049054013a1b77]*/ + +#ifndef _WINAPI_GETSHORTPATHNAME_METHODDEF + #define _WINAPI_GETSHORTPATHNAME_METHODDEF +#endif /* !defined(_WINAPI_GETSHORTPATHNAME_METHODDEF) */ +/*[clinic end generated code: output=4581fd481c3c6293 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 09ac157cbfd..19564a6c13f 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -434,12 +434,19 @@ zoneinfo_ZoneInfo__unpickle(PyObject *type, PyTypeObject *cls, PyObject *const * } key = args[0]; { - unsigned long ival = PyLong_AsUnsignedLongMask(args[1]); - if (ival == (unsigned long)-1 && PyErr_Occurred()) { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &from_cache, sizeof(unsigned char), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { goto exit; } - else { - from_cache = (unsigned char) ival; + if ((size_t)_bytes > sizeof(unsigned char)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } } } return_value = zoneinfo_ZoneInfo__unpickle_impl((PyTypeObject *)type, cls, key, from_cache); @@ -447,4 +454,4 @@ zoneinfo_ZoneInfo__unpickle(PyObject *type, PyTypeObject *cls, PyObject *const * exit: return return_value; } -/*[clinic end generated code: output=8e9e204f390261b9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6df04d7b400bd7f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 97e5ca771f3..2648583c654 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -419,6 +419,11 @@ array_array_fromfile(PyObject *self, PyTypeObject *cls, PyObject *const *args, P goto exit; } n = ival; + if (n < 0) { + PyErr_SetString(PyExc_ValueError, + "n cannot be negative"); + goto exit; + } } return_value = array_array_fromfile_impl((arrayobject *)self, cls, f, n); @@ -773,4 +778,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=dd49451ac1cc3f39 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c993c3598085840e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 602e42a4c1a..ce29e0d11a4 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -292,9 +292,21 @@ binascii_crc_hqx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - crc = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (crc == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &crc, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } return_value = binascii_crc_hqx_impl(module, &data, crc); @@ -336,9 +348,21 @@ binascii_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - crc = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (crc == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &crc, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: _return_value = binascii_crc32_impl(module, &data, crc); @@ -788,4 +812,4 @@ exit: return return_value; } -/*[clinic end generated code: output=adb855a2797c3cad input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fba6a71e0d7d092f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/blake2module.c.h b/Modules/clinic/blake2module.c.h index bb2e308574a..97d010d03a4 100644 --- a/Modules/clinic/blake2module.c.h +++ b/Modules/clinic/blake2module.c.h @@ -10,20 +10,21 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_blake2b_new__doc__, -"blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" +"blake2b(data=b\'\', *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" " node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" -" usedforsecurity=True)\n" +" usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new BLAKE2b hash object."); static PyObject * -py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2b_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity); + int inner_size, int last_node, int usedforsecurity, + PyObject *string); static PyObject * py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -31,7 +32,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 12 + #define NUM_KEYWORDS 14 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -40,7 +41,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -49,18 +50,18 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "blake2b", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[13]; + PyObject *argsbuf[14]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int digest_size = HACL_HASH_BLAKE2B_OUT_BYTES; Py_buffer key = {NULL, NULL}; Py_buffer salt = {NULL, NULL}; @@ -73,18 +74,23 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int inner_size = 0; int last_node = 0; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -182,12 +188,18 @@ skip_optional_posonly: goto skip_optional_kwonly; } } - usedforsecurity = PyObject_IsTrue(fastargs[12]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[12]) { + usedforsecurity = PyObject_IsTrue(fastargs[12]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[13]; skip_optional_kwonly: - return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + return_value = py_blake2b_new_impl(type, data_obj, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity, string); exit: /* Cleanup for key */ @@ -207,20 +219,21 @@ exit: } PyDoc_STRVAR(py_blake2s_new__doc__, -"blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" +"blake2s(data=b\'\', *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" " node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" -" usedforsecurity=True)\n" +" usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new BLAKE2s hash object."); static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2s_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity); + int inner_size, int last_node, int usedforsecurity, + PyObject *string); static PyObject * py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -228,7 +241,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 12 + #define NUM_KEYWORDS 14 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -237,7 +250,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -246,18 +259,18 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "blake2s", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[13]; + PyObject *argsbuf[14]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int digest_size = HACL_HASH_BLAKE2S_OUT_BYTES; Py_buffer key = {NULL, NULL}; Py_buffer salt = {NULL, NULL}; @@ -270,18 +283,23 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int inner_size = 0; int last_node = 0; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -379,12 +397,18 @@ skip_optional_posonly: goto skip_optional_kwonly; } } - usedforsecurity = PyObject_IsTrue(fastargs[12]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[12]) { + usedforsecurity = PyObject_IsTrue(fastargs[12]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[13]; skip_optional_kwonly: - return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + return_value = py_blake2s_new_impl(type, data_obj, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity, string); exit: /* Cleanup for key */ @@ -410,15 +434,19 @@ PyDoc_STRVAR(_blake2_blake2b_copy__doc__, "Return a copy of the hash object."); #define _BLAKE2_BLAKE2B_COPY_METHODDEF \ - {"copy", (PyCFunction)_blake2_blake2b_copy, METH_NOARGS, _blake2_blake2b_copy__doc__}, + {"copy", _PyCFunction_CAST(_blake2_blake2b_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _blake2_blake2b_copy__doc__}, static PyObject * -_blake2_blake2b_copy_impl(Blake2Object *self); +_blake2_blake2b_copy_impl(Blake2Object *self, PyTypeObject *cls); static PyObject * -_blake2_blake2b_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _blake2_blake2b_copy_impl((Blake2Object *)self); + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); + return NULL; + } + return _blake2_blake2b_copy_impl((Blake2Object *)self, cls); } PyDoc_STRVAR(_blake2_blake2b_update__doc__, @@ -478,4 +506,4 @@ _blake2_blake2b_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl((Blake2Object *)self); } -/*[clinic end generated code: output=d30e8293bd8e2950 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=60a4abbcb8950fe5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/faulthandler.c.h b/Modules/clinic/faulthandler.c.h new file mode 100644 index 00000000000..de8280ce26b --- /dev/null +++ b/Modules/clinic/faulthandler.c.h @@ -0,0 +1,688 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_long.h" // _PyLong_UnsignedInt_Converter() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(faulthandler_dump_traceback_py__doc__, +"dump_traceback($module, /, file=sys.stderr, all_threads=True)\n" +"--\n" +"\n" +"Dump the traceback of the current thread into file.\n" +"\n" +"Dump the traceback of all threads if all_threads is true."); + +#define FAULTHANDLER_DUMP_TRACEBACK_PY_METHODDEF \ + {"dump_traceback", _PyCFunction_CAST(faulthandler_dump_traceback_py), METH_FASTCALL|METH_KEYWORDS, faulthandler_dump_traceback_py__doc__}, + +static PyObject * +faulthandler_dump_traceback_py_impl(PyObject *module, PyObject *file, + int all_threads); + +static PyObject * +faulthandler_dump_traceback_py(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(file), &_Py_ID(all_threads), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"file", "all_threads", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dump_traceback", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *file = NULL; + int all_threads = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + file = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + all_threads = PyObject_IsTrue(args[1]); + if (all_threads < 0) { + goto exit; + } +skip_optional_pos: + return_value = faulthandler_dump_traceback_py_impl(module, file, all_threads); + +exit: + return return_value; +} + +PyDoc_STRVAR(faulthandler_dump_c_stack_py__doc__, +"dump_c_stack($module, /, file=sys.stderr)\n" +"--\n" +"\n" +"Dump the C stack of the current thread."); + +#define FAULTHANDLER_DUMP_C_STACK_PY_METHODDEF \ + {"dump_c_stack", _PyCFunction_CAST(faulthandler_dump_c_stack_py), METH_FASTCALL|METH_KEYWORDS, faulthandler_dump_c_stack_py__doc__}, + +static PyObject * +faulthandler_dump_c_stack_py_impl(PyObject *module, PyObject *file); + +static PyObject * +faulthandler_dump_c_stack_py(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(file), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"file", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dump_c_stack", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *file = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + file = args[0]; +skip_optional_pos: + return_value = faulthandler_dump_c_stack_py_impl(module, file); + +exit: + return return_value; +} + +PyDoc_STRVAR(faulthandler_py_enable__doc__, +"enable($module, /, file=sys.stderr, all_threads=True, c_stack=True)\n" +"--\n" +"\n" +"Enable the fault handler."); + +#define FAULTHANDLER_PY_ENABLE_METHODDEF \ + {"enable", _PyCFunction_CAST(faulthandler_py_enable), METH_FASTCALL|METH_KEYWORDS, faulthandler_py_enable__doc__}, + +static PyObject * +faulthandler_py_enable_impl(PyObject *module, PyObject *file, + int all_threads, int c_stack); + +static PyObject * +faulthandler_py_enable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(file), &_Py_ID(all_threads), &_Py_ID(c_stack), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"file", "all_threads", "c_stack", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "enable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *file = NULL; + int all_threads = 1; + int c_stack = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + file = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + all_threads = PyObject_IsTrue(args[1]); + if (all_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + c_stack = PyObject_IsTrue(args[2]); + if (c_stack < 0) { + goto exit; + } +skip_optional_pos: + return_value = faulthandler_py_enable_impl(module, file, all_threads, c_stack); + +exit: + return return_value; +} + +PyDoc_STRVAR(faulthandler_disable_py__doc__, +"disable($module, /)\n" +"--\n" +"\n" +"Disable the fault handler."); + +#define FAULTHANDLER_DISABLE_PY_METHODDEF \ + {"disable", (PyCFunction)faulthandler_disable_py, METH_NOARGS, faulthandler_disable_py__doc__}, + +static PyObject * +faulthandler_disable_py_impl(PyObject *module); + +static PyObject * +faulthandler_disable_py(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return faulthandler_disable_py_impl(module); +} + +PyDoc_STRVAR(faulthandler_is_enabled__doc__, +"is_enabled($module, /)\n" +"--\n" +"\n" +"Check if the handler is enabled."); + +#define FAULTHANDLER_IS_ENABLED_METHODDEF \ + {"is_enabled", (PyCFunction)faulthandler_is_enabled, METH_NOARGS, faulthandler_is_enabled__doc__}, + +static int +faulthandler_is_enabled_impl(PyObject *module); + +static PyObject * +faulthandler_is_enabled(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = faulthandler_is_enabled_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(faulthandler_dump_traceback_later__doc__, +"dump_traceback_later($module, /, timeout, repeat=False,\n" +" file=sys.stderr, exit=False)\n" +"--\n" +"\n" +"Dump the traceback of all threads in timeout seconds.\n" +"\n" +"If repeat is true, the tracebacks of all threads are dumped every timeout\n" +"seconds. If exit is true, call _exit(1) which is not safe."); + +#define FAULTHANDLER_DUMP_TRACEBACK_LATER_METHODDEF \ + {"dump_traceback_later", _PyCFunction_CAST(faulthandler_dump_traceback_later), METH_FASTCALL|METH_KEYWORDS, faulthandler_dump_traceback_later__doc__}, + +static PyObject * +faulthandler_dump_traceback_later_impl(PyObject *module, + PyObject *timeout_obj, int repeat, + PyObject *file, int exit); + +static PyObject * +faulthandler_dump_traceback_later(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(timeout), &_Py_ID(repeat), &_Py_ID(file), &_Py_ID(exit), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timeout", "repeat", "file", "exit", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dump_traceback_later", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *timeout_obj; + int repeat = 0; + PyObject *file = NULL; + int exit = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + timeout_obj = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + repeat = PyObject_IsTrue(args[1]); + if (repeat < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + file = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + exit = PyObject_IsTrue(args[3]); + if (exit < 0) { + goto exit; + } +skip_optional_pos: + return_value = faulthandler_dump_traceback_later_impl(module, timeout_obj, repeat, file, exit); + +exit: + return return_value; +} + +PyDoc_STRVAR(faulthandler_cancel_dump_traceback_later_py__doc__, +"cancel_dump_traceback_later($module, /)\n" +"--\n" +"\n" +"Cancel the previous call to dump_traceback_later()."); + +#define FAULTHANDLER_CANCEL_DUMP_TRACEBACK_LATER_PY_METHODDEF \ + {"cancel_dump_traceback_later", (PyCFunction)faulthandler_cancel_dump_traceback_later_py, METH_NOARGS, faulthandler_cancel_dump_traceback_later_py__doc__}, + +static PyObject * +faulthandler_cancel_dump_traceback_later_py_impl(PyObject *module); + +static PyObject * +faulthandler_cancel_dump_traceback_later_py(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return faulthandler_cancel_dump_traceback_later_py_impl(module); +} + +#if defined(FAULTHANDLER_USER) + +PyDoc_STRVAR(faulthandler_register_py__doc__, +"register($module, /, signum, file=sys.stderr, all_threads=True,\n" +" chain=False)\n" +"--\n" +"\n" +"Register a handler for the signal \'signum\'.\n" +"\n" +"Dump the traceback of the current thread, or of all threads if\n" +"all_threads is True, into file."); + +#define FAULTHANDLER_REGISTER_PY_METHODDEF \ + {"register", _PyCFunction_CAST(faulthandler_register_py), METH_FASTCALL|METH_KEYWORDS, faulthandler_register_py__doc__}, + +static PyObject * +faulthandler_register_py_impl(PyObject *module, int signum, PyObject *file, + int all_threads, int chain); + +static PyObject * +faulthandler_register_py(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(signum), &_Py_ID(file), &_Py_ID(all_threads), &_Py_ID(chain), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"signum", "file", "all_threads", "chain", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "register", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + int signum; + PyObject *file = NULL; + int all_threads = 1; + int chain = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + signum = PyLong_AsInt(args[0]); + if (signum == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + file = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + all_threads = PyObject_IsTrue(args[2]); + if (all_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + chain = PyObject_IsTrue(args[3]); + if (chain < 0) { + goto exit; + } +skip_optional_pos: + return_value = faulthandler_register_py_impl(module, signum, file, all_threads, chain); + +exit: + return return_value; +} + +#endif /* defined(FAULTHANDLER_USER) */ + +#if defined(FAULTHANDLER_USER) + +PyDoc_STRVAR(faulthandler_unregister_py__doc__, +"unregister($module, signum, /)\n" +"--\n" +"\n" +"Unregister the handler of the signal \'signum\' registered by register()."); + +#define FAULTHANDLER_UNREGISTER_PY_METHODDEF \ + {"unregister", (PyCFunction)faulthandler_unregister_py, METH_O, faulthandler_unregister_py__doc__}, + +static PyObject * +faulthandler_unregister_py_impl(PyObject *module, int signum); + +static PyObject * +faulthandler_unregister_py(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int signum; + + signum = PyLong_AsInt(arg); + if (signum == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = faulthandler_unregister_py_impl(module, signum); + +exit: + return return_value; +} + +#endif /* defined(FAULTHANDLER_USER) */ + +PyDoc_STRVAR(faulthandler__sigsegv__doc__, +"_sigsegv($module, release_gil=False, /)\n" +"--\n" +"\n" +"Raise a SIGSEGV signal."); + +#define FAULTHANDLER__SIGSEGV_METHODDEF \ + {"_sigsegv", _PyCFunction_CAST(faulthandler__sigsegv), METH_FASTCALL, faulthandler__sigsegv__doc__}, + +static PyObject * +faulthandler__sigsegv_impl(PyObject *module, int release_gil); + +static PyObject * +faulthandler__sigsegv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int release_gil = 0; + + if (!_PyArg_CheckPositional("_sigsegv", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + release_gil = PyObject_IsTrue(args[0]); + if (release_gil < 0) { + goto exit; + } +skip_optional: + return_value = faulthandler__sigsegv_impl(module, release_gil); + +exit: + return return_value; +} + +PyDoc_STRVAR(faulthandler__fatal_error_c_thread__doc__, +"_fatal_error_c_thread($module, /)\n" +"--\n" +"\n" +"Call Py_FatalError() in a new C thread."); + +#define FAULTHANDLER__FATAL_ERROR_C_THREAD_METHODDEF \ + {"_fatal_error_c_thread", (PyCFunction)faulthandler__fatal_error_c_thread, METH_NOARGS, faulthandler__fatal_error_c_thread__doc__}, + +static PyObject * +faulthandler__fatal_error_c_thread_impl(PyObject *module); + +static PyObject * +faulthandler__fatal_error_c_thread(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return faulthandler__fatal_error_c_thread_impl(module); +} + +PyDoc_STRVAR(faulthandler__sigfpe__doc__, +"_sigfpe($module, /)\n" +"--\n" +"\n" +"Raise a SIGFPE signal."); + +#define FAULTHANDLER__SIGFPE_METHODDEF \ + {"_sigfpe", (PyCFunction)faulthandler__sigfpe, METH_NOARGS, faulthandler__sigfpe__doc__}, + +static PyObject * +faulthandler__sigfpe_impl(PyObject *module); + +static PyObject * +faulthandler__sigfpe(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return faulthandler__sigfpe_impl(module); +} + +PyDoc_STRVAR(faulthandler__sigabrt__doc__, +"_sigabrt($module, /)\n" +"--\n" +"\n" +"Raise a SIGABRT signal."); + +#define FAULTHANDLER__SIGABRT_METHODDEF \ + {"_sigabrt", (PyCFunction)faulthandler__sigabrt, METH_NOARGS, faulthandler__sigabrt__doc__}, + +static PyObject * +faulthandler__sigabrt_impl(PyObject *module); + +static PyObject * +faulthandler__sigabrt(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return faulthandler__sigabrt_impl(module); +} + +#if defined(FAULTHANDLER_USE_ALT_STACK) + +PyDoc_STRVAR(faulthandler__stack_overflow__doc__, +"_stack_overflow($module, /)\n" +"--\n" +"\n" +"Recursive call to raise a stack overflow."); + +#define FAULTHANDLER__STACK_OVERFLOW_METHODDEF \ + {"_stack_overflow", (PyCFunction)faulthandler__stack_overflow, METH_NOARGS, faulthandler__stack_overflow__doc__}, + +static PyObject * +faulthandler__stack_overflow_impl(PyObject *module); + +static PyObject * +faulthandler__stack_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return faulthandler__stack_overflow_impl(module); +} + +#endif /* defined(FAULTHANDLER_USE_ALT_STACK) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(faulthandler__raise_exception__doc__, +"_raise_exception($module, code, flags=0, /)\n" +"--\n" +"\n" +"Call RaiseException(code, flags)."); + +#define FAULTHANDLER__RAISE_EXCEPTION_METHODDEF \ + {"_raise_exception", _PyCFunction_CAST(faulthandler__raise_exception), METH_FASTCALL, faulthandler__raise_exception__doc__}, + +static PyObject * +faulthandler__raise_exception_impl(PyObject *module, unsigned int code, + unsigned int flags); + +static PyObject * +faulthandler__raise_exception(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int code; + unsigned int flags = 0; + + if (!_PyArg_CheckPositional("_raise_exception", nargs, 1, 2)) { + goto exit; + } + if (!_PyLong_UnsignedInt_Converter(args[0], &code)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (!_PyLong_UnsignedInt_Converter(args[1], &flags)) { + goto exit; + } +skip_optional: + return_value = faulthandler__raise_exception_impl(module, code, flags); + +exit: + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#ifndef FAULTHANDLER_REGISTER_PY_METHODDEF + #define FAULTHANDLER_REGISTER_PY_METHODDEF +#endif /* !defined(FAULTHANDLER_REGISTER_PY_METHODDEF) */ + +#ifndef FAULTHANDLER_UNREGISTER_PY_METHODDEF + #define FAULTHANDLER_UNREGISTER_PY_METHODDEF +#endif /* !defined(FAULTHANDLER_UNREGISTER_PY_METHODDEF) */ + +#ifndef FAULTHANDLER__STACK_OVERFLOW_METHODDEF + #define FAULTHANDLER__STACK_OVERFLOW_METHODDEF +#endif /* !defined(FAULTHANDLER__STACK_OVERFLOW_METHODDEF) */ + +#ifndef FAULTHANDLER__RAISE_EXCEPTION_METHODDEF + #define FAULTHANDLER__RAISE_EXCEPTION_METHODDEF +#endif /* !defined(FAULTHANDLER__RAISE_EXCEPTION_METHODDEF) */ +/*[clinic end generated code: output=31bf0149d0d02ccf input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 00a929064ba..718f80bfe73 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -2,24 +2,31 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" "\n" -"Perform the operation `cmd` on file descriptor fd.\n" +"Perform the operation cmd on file descriptor fd.\n" "\n" -"The values used for `cmd` are operating system dependent, and are available\n" -"as constants in the fcntl module, using the same names as used in\n" -"the relevant C header files. The argument arg is optional, and\n" -"defaults to 0; it may be an int or a string. If arg is given as a string,\n" -"the return value of fcntl is a string of that length, containing the\n" -"resulting value put in the arg buffer by the operating system. The length\n" -"of the arg string is not allowed to exceed 1024 bytes. If the arg given\n" -"is an integer or if none is specified, the result value is an integer\n" -"corresponding to the return value of the fcntl call in the C code."); +"The values used for cmd are operating system dependent, and are\n" +"available as constants in the fcntl module, using the same names as used\n" +"in the relevant C header files. The argument arg is optional, and\n" +"defaults to 0; it may be an integer, a bytes-like object or a string.\n" +"If arg is given as a string, it will be encoded to binary using the\n" +"UTF-8 encoding.\n" +"\n" +"If the arg given is an integer or if none is specified, the result value\n" +"is an integer corresponding to the return value of the fcntl() call in\n" +"the C code.\n" +"\n" +"If arg is given as a bytes-like object, the return value of fcntl() is a\n" +"bytes object of that length, containing the resulting value put in the\n" +"arg buffer by the operating system."); #define FCNTL_FCNTL_METHODDEF \ - {"fcntl", (PyCFunction)(void(*)(void))fcntl_fcntl, METH_FASTCALL, fcntl_fcntl__doc__}, + {"fcntl", _PyCFunction_CAST(fcntl_fcntl), METH_FASTCALL, fcntl_fcntl__doc__}, static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg); @@ -32,12 +39,7 @@ fcntl_fcntl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int code; PyObject *arg = NULL; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "fcntl expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 3) { - PyErr_Format(PyExc_TypeError, "fcntl expected at most 3 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("fcntl", nargs, 2, 3)) { goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -63,37 +65,36 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "ioctl($module, fd, request, arg=0, mutate_flag=True, /)\n" "--\n" "\n" -"Perform the operation `request` on file descriptor `fd`.\n" +"Perform the operation request on file descriptor fd.\n" "\n" -"The values used for `request` are operating system dependent, and are available\n" -"as constants in the fcntl or termios library modules, using the same names as\n" -"used in the relevant C header files.\n" +"The values used for request are operating system dependent, and are\n" +"available as constants in the fcntl or termios library modules, using\n" +"the same names as used in the relevant C header files.\n" "\n" -"The argument `arg` is optional, and defaults to 0; it may be an int or a\n" -"buffer containing character data (most likely a string or an array).\n" +"The argument arg is optional, and defaults to 0; it may be an integer, a\n" +"bytes-like object or a string. If arg is given as a string, it will be\n" +"encoded to binary using the UTF-8 encoding.\n" "\n" -"If the argument is a mutable buffer (such as an array) and if the\n" -"mutate_flag argument (which is only allowed in this case) is true then the\n" -"buffer is (in effect) passed to the operating system and changes made by\n" -"the OS will be reflected in the contents of the buffer after the call has\n" -"returned. The return value is the integer returned by the ioctl system\n" -"call.\n" +"If the arg given is an integer or if none is specified, the result value\n" +"is an integer corresponding to the return value of the ioctl() call in\n" +"the C code.\n" "\n" -"If the argument is a mutable buffer and the mutable_flag argument is false,\n" -"the behavior is as if a string had been passed.\n" +"If the argument is a mutable buffer (such as a bytearray) and the\n" +"mutate_flag argument is true (default) then the buffer is (in effect)\n" +"passed to the operating system and changes made by the OS will be\n" +"reflected in the contents of the buffer after the call has returned.\n" +"The return value is the integer returned by the ioctl() system call.\n" "\n" -"If the argument is an immutable buffer (most likely a string) then a copy\n" -"of the buffer is passed to the operating system and the return value is a\n" -"string of the same length containing whatever the operating system put in\n" -"the buffer. The length of the arg buffer in this case is not allowed to\n" -"exceed 1024 bytes.\n" +"If the argument is a mutable buffer and the mutable_flag argument is\n" +"false, the behavior is as if an immutable buffer had been passed.\n" "\n" -"If the arg given is an integer or if none is specified, the result value is\n" -"an integer corresponding to the return value of the ioctl call in the C\n" -"code."); +"If the argument is an immutable buffer then a copy of the buffer is\n" +"passed to the operating system and the return value is a bytes object of\n" +"the same length containing whatever the operating system put in the\n" +"buffer."); #define FCNTL_IOCTL_METHODDEF \ - {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, + {"ioctl", _PyCFunction_CAST(fcntl_ioctl), METH_FASTCALL, fcntl_ioctl__doc__}, static PyObject * fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, @@ -108,12 +109,7 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *arg = NULL; int mutate_arg = 1; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "ioctl expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 4) { - PyErr_Format(PyExc_TypeError, "ioctl expected at most 4 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("ioctl", nargs, 2, 4)) { goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -121,10 +117,25 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (!PyIndex_Check(args[1])) { - PyErr_Format(PyExc_TypeError, "ioctl() argument 2 must be int, not %T", args[1]); + _PyArg_BadArgument("ioctl", "argument 2", "int", args[1]); goto exit; } - code = PyLong_AsUnsignedLongMask(args[1]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &code, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } if (nargs < 3) { goto skip_optional; } @@ -147,13 +158,13 @@ PyDoc_STRVAR(fcntl_flock__doc__, "flock($module, fd, operation, /)\n" "--\n" "\n" -"Perform the lock operation `operation` on file descriptor `fd`.\n" +"Perform the lock operation on file descriptor fd.\n" "\n" "See the Unix manual page for flock(2) for details (On some systems, this\n" "function is emulated using fcntl())."); #define FCNTL_FLOCK_METHODDEF \ - {"flock", (PyCFunction)(void(*)(void))fcntl_flock, METH_FASTCALL, fcntl_flock__doc__}, + {"flock", _PyCFunction_CAST(fcntl_flock), METH_FASTCALL, fcntl_flock__doc__}, static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code); @@ -165,8 +176,7 @@ fcntl_flock(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int code; - if (nargs != 2) { - PyErr_Format(PyExc_TypeError, "flock expected 2 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("flock", nargs, 2, 2)) { goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -189,29 +199,29 @@ PyDoc_STRVAR(fcntl_lockf__doc__, "\n" "A wrapper around the fcntl() locking calls.\n" "\n" -"`fd` is the file descriptor of the file to lock or unlock, and operation is one\n" -"of the following values:\n" +"fd is the file descriptor of the file to lock or unlock, and operation\n" +"is one of the following values:\n" "\n" " LOCK_UN - unlock\n" " LOCK_SH - acquire a shared lock\n" " LOCK_EX - acquire an exclusive lock\n" "\n" "When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\n" -"LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the\n" -"lock cannot be acquired, an OSError will be raised and the exception will\n" -"have an errno attribute set to EACCES or EAGAIN (depending on the operating\n" -"system -- for portability, check for either value).\n" +"LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and\n" +"the lock cannot be acquired, an OSError will be raised and the exception\n" +"will have an errno attribute set to EACCES or EAGAIN (depending on the\n" +"operating system -- for portability, check for either value).\n" "\n" -"`len` is the number of bytes to lock, with the default meaning to lock to\n" -"EOF. `start` is the byte offset, relative to `whence`, to that the lock\n" -"starts. `whence` is as with fileobj.seek(), specifically:\n" +"len is the number of bytes to lock, with the default meaning to lock to\n" +"EOF. start is the byte offset, relative to whence, to that the lock\n" +"starts. whence is as with fileobj.seek(), specifically:\n" "\n" " 0 - relative to the start of the file (SEEK_SET)\n" " 1 - relative to the current buffer position (SEEK_CUR)\n" " 2 - relative to the end of the file (SEEK_END)"); #define FCNTL_LOCKF_METHODDEF \ - {"lockf", (PyCFunction)(void(*)(void))fcntl_lockf, METH_FASTCALL, fcntl_lockf__doc__}, + {"lockf", _PyCFunction_CAST(fcntl_lockf), METH_FASTCALL, fcntl_lockf__doc__}, static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, @@ -227,12 +237,7 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *startobj = NULL; int whence = 0; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "lockf expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 5) { - PyErr_Format(PyExc_TypeError, "lockf expected at most 5 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("lockf", nargs, 2, 5)) { goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -264,4 +269,4 @@ skip_optional: exit: return return_value; } -/*[clinic end generated code: output=65a16bc64c7b4de4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c782fcf9dd6690e0 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 53ff9e4faf8..08275e35413 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -8,7 +8,6 @@ preserve #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(gc_enable__doc__, "enable($module, /)\n" @@ -324,7 +323,7 @@ gc_get_referrers(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *objs = NULL; - objs = _PyTuple_FromArray(args, nargs); + objs = PyTuple_FromArray(args, nargs); if (objs == NULL) { goto exit; } @@ -355,7 +354,7 @@ gc_get_referents(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *objs = NULL; - objs = _PyTuple_FromArray(args, nargs); + objs = PyTuple_FromArray(args, nargs); if (objs == NULL) { goto exit; } @@ -584,4 +583,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=96d057eac558e6ca input=a9049054013a1b77]*/ +/*[clinic end generated code: output=19738854607938db input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 0af82e7eb05..49816bfcb42 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -345,6 +345,11 @@ itertools_tee(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } n = ival; + if (n < 0) { + PyErr_SetString(PyExc_ValueError, + "n cannot be negative"); + goto exit; + } } skip_optional: return_value = itertools_tee_impl(module, iterable, n); @@ -569,6 +574,11 @@ itertools_combinations(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } r = ival; + if (r < 0) { + PyErr_SetString(PyExc_ValueError, + "r cannot be negative"); + goto exit; + } } return_value = itertools_combinations_impl(type, iterable, r); @@ -643,6 +653,11 @@ itertools_combinations_with_replacement(PyTypeObject *type, PyObject *args, PyOb goto exit; } r = ival; + if (r < 0) { + PyErr_SetString(PyExc_ValueError, + "r cannot be negative"); + goto exit; + } } return_value = itertools_combinations_with_replacement_impl(type, iterable, r); @@ -965,4 +980,4 @@ skip_optional_pos: exit: return return_value; } -/*[clinic end generated code: output=999758202a532e0a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7f385837b13edbeb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathintegermodule.c.h b/Modules/clinic/mathintegermodule.c.h new file mode 100644 index 00000000000..29c2a0ac902 --- /dev/null +++ b/Modules/clinic/mathintegermodule.c.h @@ -0,0 +1,159 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(math_integer_gcd__doc__, +"gcd($module, /, *integers)\n" +"--\n" +"\n" +"Greatest Common Divisor."); + +#define MATH_INTEGER_GCD_METHODDEF \ + {"gcd", _PyCFunction_CAST(math_integer_gcd), METH_FASTCALL, math_integer_gcd__doc__}, + +static PyObject * +math_integer_gcd_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +math_integer_gcd(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = math_integer_gcd_impl(module, __clinic_args, args_length); + + return return_value; +} + +PyDoc_STRVAR(math_integer_lcm__doc__, +"lcm($module, /, *integers)\n" +"--\n" +"\n" +"Least Common Multiple."); + +#define MATH_INTEGER_LCM_METHODDEF \ + {"lcm", _PyCFunction_CAST(math_integer_lcm), METH_FASTCALL, math_integer_lcm__doc__}, + +static PyObject * +math_integer_lcm_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length); + +static PyObject * +math_integer_lcm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject * const *__clinic_args; + Py_ssize_t args_length; + + __clinic_args = args; + args_length = nargs; + return_value = math_integer_lcm_impl(module, __clinic_args, args_length); + + return return_value; +} + +PyDoc_STRVAR(math_integer_isqrt__doc__, +"isqrt($module, n, /)\n" +"--\n" +"\n" +"Return the integer part of the square root of the input."); + +#define MATH_INTEGER_ISQRT_METHODDEF \ + {"isqrt", (PyCFunction)math_integer_isqrt, METH_O, math_integer_isqrt__doc__}, + +PyDoc_STRVAR(math_integer_factorial__doc__, +"factorial($module, n, /)\n" +"--\n" +"\n" +"Find n!."); + +#define MATH_INTEGER_FACTORIAL_METHODDEF \ + {"factorial", (PyCFunction)math_integer_factorial, METH_O, math_integer_factorial__doc__}, + +PyDoc_STRVAR(math_integer_perm__doc__, +"perm($module, n, k=None, /)\n" +"--\n" +"\n" +"Number of ways to choose k items from n items without repetition and with order.\n" +"\n" +"Evaluates to n! / (n - k)! when k <= n and evaluates\n" +"to zero when k > n.\n" +"\n" +"If k is not specified or is None, then k defaults to n\n" +"and the function returns n!.\n" +"\n" +"Raises ValueError if either of the arguments are negative."); + +#define MATH_INTEGER_PERM_METHODDEF \ + {"perm", _PyCFunction_CAST(math_integer_perm), METH_FASTCALL, math_integer_perm__doc__}, + +static PyObject * +math_integer_perm_impl(PyObject *module, PyObject *n, PyObject *k); + +static PyObject * +math_integer_perm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *n; + PyObject *k = Py_None; + + if (!_PyArg_CheckPositional("perm", nargs, 1, 2)) { + goto exit; + } + n = args[0]; + if (nargs < 2) { + goto skip_optional; + } + k = args[1]; +skip_optional: + return_value = math_integer_perm_impl(module, n, k); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_integer_comb__doc__, +"comb($module, n, k, /)\n" +"--\n" +"\n" +"Number of ways to choose k items from n items without repetition and without order.\n" +"\n" +"Evaluates to n! / (k! * (n - k)!) when k <= n and evaluates\n" +"to zero when k > n.\n" +"\n" +"Also called the binomial coefficient because it is equivalent\n" +"to the coefficient of k-th term in polynomial expansion of the\n" +"expression (1 + x)**n.\n" +"\n" +"Raises ValueError if either of the arguments are negative."); + +#define MATH_INTEGER_COMB_METHODDEF \ + {"comb", _PyCFunction_CAST(math_integer_comb), METH_FASTCALL, math_integer_comb__doc__}, + +static PyObject * +math_integer_comb_impl(PyObject *module, PyObject *n, PyObject *k); + +static PyObject * +math_integer_comb(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *n; + PyObject *k; + + if (!_PyArg_CheckPositional("comb", nargs, 2, 2)) { + goto exit; + } + n = args[0]; + k = args[1]; + return_value = math_integer_comb_impl(module, n, k); + +exit: + return return_value; +} +/*[clinic end generated code: output=34697570c923a3af input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 4c2c8acd8f6..b023299dd9c 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -8,60 +8,6 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() -PyDoc_STRVAR(math_gcd__doc__, -"gcd($module, /, *integers)\n" -"--\n" -"\n" -"Greatest Common Divisor."); - -#define MATH_GCD_METHODDEF \ - {"gcd", _PyCFunction_CAST(math_gcd), METH_FASTCALL, math_gcd__doc__}, - -static PyObject * -math_gcd_impl(PyObject *module, PyObject * const *args, - Py_ssize_t args_length); - -static PyObject * -math_gcd(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject * const *__clinic_args; - Py_ssize_t args_length; - - __clinic_args = args; - args_length = nargs; - return_value = math_gcd_impl(module, __clinic_args, args_length); - - return return_value; -} - -PyDoc_STRVAR(math_lcm__doc__, -"lcm($module, /, *integers)\n" -"--\n" -"\n" -"Least Common Multiple."); - -#define MATH_LCM_METHODDEF \ - {"lcm", _PyCFunction_CAST(math_lcm), METH_FASTCALL, math_lcm__doc__}, - -static PyObject * -math_lcm_impl(PyObject *module, PyObject * const *args, - Py_ssize_t args_length); - -static PyObject * -math_lcm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject * const *__clinic_args; - Py_ssize_t args_length; - - __clinic_args = args; - args_length = nargs; - return_value = math_lcm_impl(module, __clinic_args, args_length); - - return return_value; -} - PyDoc_STRVAR(math_ceil__doc__, "ceil($module, x, /)\n" "--\n" @@ -84,6 +30,146 @@ PyDoc_STRVAR(math_floor__doc__, #define MATH_FLOOR_METHODDEF \ {"floor", (PyCFunction)math_floor, METH_O, math_floor__doc__}, +PyDoc_STRVAR(math_fmax__doc__, +"fmax($module, x, y, /)\n" +"--\n" +"\n" +"Return the larger of two floating-point arguments."); + +#define MATH_FMAX_METHODDEF \ + {"fmax", _PyCFunction_CAST(math_fmax), METH_FASTCALL, math_fmax__doc__}, + +static double +math_fmax_impl(PyObject *module, double x, double y); + +static PyObject * +math_fmax(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (!_PyArg_CheckPositional("fmax", nargs, 2, 2)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + _return_value = math_fmax_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_fmin__doc__, +"fmin($module, x, y, /)\n" +"--\n" +"\n" +"Return the smaller of two floating-point arguments."); + +#define MATH_FMIN_METHODDEF \ + {"fmin", _PyCFunction_CAST(math_fmin), METH_FASTCALL, math_fmin__doc__}, + +static double +math_fmin_impl(PyObject *module, double x, double y); + +static PyObject * +math_fmin(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (!_PyArg_CheckPositional("fmin", nargs, 2, 2)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + _return_value = math_fmin_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_signbit__doc__, +"signbit($module, x, /)\n" +"--\n" +"\n" +"Return True if the sign of x is negative and False otherwise."); + +#define MATH_SIGNBIT_METHODDEF \ + {"signbit", (PyCFunction)math_signbit, METH_O, math_signbit__doc__}, + +static PyObject * +math_signbit_impl(PyObject *module, double x); + +static PyObject * +math_signbit(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + double x; + + if (PyFloat_CheckExact(arg)) { + x = PyFloat_AS_DOUBLE(arg); + } + else + { + x = PyFloat_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_signbit_impl(module, x); + +exit: + return return_value; +} + PyDoc_STRVAR(math_fsum__doc__, "fsum($module, seq, /)\n" "--\n" @@ -95,26 +181,6 @@ PyDoc_STRVAR(math_fsum__doc__, #define MATH_FSUM_METHODDEF \ {"fsum", (PyCFunction)math_fsum, METH_O, math_fsum__doc__}, -PyDoc_STRVAR(math_isqrt__doc__, -"isqrt($module, n, /)\n" -"--\n" -"\n" -"Return the integer part of the square root of the input."); - -#define MATH_ISQRT_METHODDEF \ - {"isqrt", (PyCFunction)math_isqrt, METH_O, math_isqrt__doc__}, - -PyDoc_STRVAR(math_factorial__doc__, -"factorial($module, n, /)\n" -"--\n" -"\n" -"Find n!.\n" -"\n" -"Raise a ValueError if x is negative or non-integral."); - -#define MATH_FACTORIAL_METHODDEF \ - {"factorial", (PyCFunction)math_factorial, METH_O, math_factorial__doc__}, - PyDoc_STRVAR(math_trunc__doc__, "trunc($module, x, /)\n" "--\n" @@ -630,6 +696,74 @@ exit: return return_value; } +PyDoc_STRVAR(math_isnormal__doc__, +"isnormal($module, x, /)\n" +"--\n" +"\n" +"Return True if x is normal, and False otherwise."); + +#define MATH_ISNORMAL_METHODDEF \ + {"isnormal", (PyCFunction)math_isnormal, METH_O, math_isnormal__doc__}, + +static PyObject * +math_isnormal_impl(PyObject *module, double x); + +static PyObject * +math_isnormal(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + double x; + + if (PyFloat_CheckExact(arg)) { + x = PyFloat_AS_DOUBLE(arg); + } + else + { + x = PyFloat_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_isnormal_impl(module, x); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_issubnormal__doc__, +"issubnormal($module, x, /)\n" +"--\n" +"\n" +"Return True if x is subnormal, and False otherwise."); + +#define MATH_ISSUBNORMAL_METHODDEF \ + {"issubnormal", (PyCFunction)math_issubnormal, METH_O, math_issubnormal__doc__}, + +static PyObject * +math_issubnormal_impl(PyObject *module, double x); + +static PyObject * +math_issubnormal(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + double x; + + if (PyFloat_CheckExact(arg)) { + x = PyFloat_AS_DOUBLE(arg); + } + else + { + x = PyFloat_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = math_issubnormal_impl(module, x); + +exit: + return return_value; +} + PyDoc_STRVAR(math_isnan__doc__, "isnan($module, x, /)\n" "--\n" @@ -901,89 +1035,6 @@ exit: return return_value; } -PyDoc_STRVAR(math_perm__doc__, -"perm($module, n, k=None, /)\n" -"--\n" -"\n" -"Number of ways to choose k items from n items without repetition and with order.\n" -"\n" -"Evaluates to n! / (n - k)! when k <= n and evaluates\n" -"to zero when k > n.\n" -"\n" -"If k is not specified or is None, then k defaults to n\n" -"and the function returns n!.\n" -"\n" -"Raises TypeError if either of the arguments are not integers.\n" -"Raises ValueError if either of the arguments are negative."); - -#define MATH_PERM_METHODDEF \ - {"perm", _PyCFunction_CAST(math_perm), METH_FASTCALL, math_perm__doc__}, - -static PyObject * -math_perm_impl(PyObject *module, PyObject *n, PyObject *k); - -static PyObject * -math_perm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *n; - PyObject *k = Py_None; - - if (!_PyArg_CheckPositional("perm", nargs, 1, 2)) { - goto exit; - } - n = args[0]; - if (nargs < 2) { - goto skip_optional; - } - k = args[1]; -skip_optional: - return_value = math_perm_impl(module, n, k); - -exit: - return return_value; -} - -PyDoc_STRVAR(math_comb__doc__, -"comb($module, n, k, /)\n" -"--\n" -"\n" -"Number of ways to choose k items from n items without repetition and without order.\n" -"\n" -"Evaluates to n! / (k! * (n - k)!) when k <= n and evaluates\n" -"to zero when k > n.\n" -"\n" -"Also called the binomial coefficient because it is equivalent\n" -"to the coefficient of k-th term in polynomial expansion of the\n" -"expression (1 + x)**n.\n" -"\n" -"Raises TypeError if either of the arguments are not integers.\n" -"Raises ValueError if either of the arguments are negative."); - -#define MATH_COMB_METHODDEF \ - {"comb", _PyCFunction_CAST(math_comb), METH_FASTCALL, math_comb__doc__}, - -static PyObject * -math_comb_impl(PyObject *module, PyObject *n, PyObject *k); - -static PyObject * -math_comb(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *n; - PyObject *k; - - if (!_PyArg_CheckPositional("comb", nargs, 2, 2)) { - goto exit; - } - n = args[0]; - k = args[1]; - return_value = math_comb_impl(module, n, k); - -exit: - return return_value; -} - PyDoc_STRVAR(math_nextafter__doc__, "nextafter($module, x, y, /, *, steps=None)\n" "--\n" @@ -1112,4 +1163,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=634773bd18cd3f93 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=23b2453ba77453e5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index 9ca4f6528ce..f76902586dd 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -89,7 +89,7 @@ MD5Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_md5_md5__doc__, -"md5($module, /, string=b\'\', *, usedforsecurity=True)\n" +"md5($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new MD5 hash object; optionally initialized with a string."); @@ -98,7 +98,8 @@ PyDoc_STRVAR(_md5_md5__doc__, {"md5", _PyCFunction_CAST(_md5_md5), METH_FASTCALL|METH_KEYWORDS, _md5_md5__doc__}, static PyObject * -_md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity); +_md5_md5_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -106,7 +107,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -115,7 +116,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -124,17 +125,18 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "md5", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -145,7 +147,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -154,14 +156,20 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _md5_md5_impl(module, string, usedforsecurity); + return_value = _md5_md5_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=73f4d2034d9fcc63 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=920fe54b9ed06f92 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mmapmodule.c.h b/Modules/clinic/mmapmodule.c.h new file mode 100644 index 00000000000..f7fc172b3af --- /dev/null +++ b/Modules/clinic/mmapmodule.c.h @@ -0,0 +1,799 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(mmap_mmap_close__doc__, +"close($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_CLOSE_METHODDEF \ + {"close", (PyCFunction)mmap_mmap_close, METH_NOARGS, mmap_mmap_close__doc__}, + +static PyObject * +mmap_mmap_close_impl(mmap_object *self); + +static PyObject * +mmap_mmap_close(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_close_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_read_byte__doc__, +"read_byte($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READ_BYTE_METHODDEF \ + {"read_byte", (PyCFunction)mmap_mmap_read_byte, METH_NOARGS, mmap_mmap_read_byte__doc__}, + +static PyObject * +mmap_mmap_read_byte_impl(mmap_object *self); + +static PyObject * +mmap_mmap_read_byte(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_read_byte_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_readline__doc__, +"readline($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READLINE_METHODDEF \ + {"readline", (PyCFunction)mmap_mmap_readline, METH_NOARGS, mmap_mmap_readline__doc__}, + +static PyObject * +mmap_mmap_readline_impl(mmap_object *self); + +static PyObject * +mmap_mmap_readline(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_readline_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_read__doc__, +"read($self, n=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READ_METHODDEF \ + {"read", _PyCFunction_CAST(mmap_mmap_read), METH_FASTCALL, mmap_mmap_read__doc__}, + +static PyObject * +mmap_mmap_read_impl(mmap_object *self, Py_ssize_t num_bytes); + +static PyObject * +mmap_mmap_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t num_bytes = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_Py_convert_optional_to_ssize_t(args[0], &num_bytes)) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_read_impl((mmap_object *)self, num_bytes); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_find__doc__, +"find($self, view, start=None, end=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(mmap_mmap_find), METH_FASTCALL, mmap_mmap_find__doc__}, + +static PyObject * +mmap_mmap_find_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end); + +static PyObject * +mmap_mmap_find(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer view = {NULL, NULL}; + PyObject *start = Py_None; + PyObject *end = Py_None; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + start = args[1]; + if (nargs < 3) { + goto skip_optional; + } + end = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_find_impl((mmap_object *)self, &view, start, end); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for view */ + if (view.obj) { + PyBuffer_Release(&view); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_rfind__doc__, +"rfind($self, view, start=None, end=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(mmap_mmap_rfind), METH_FASTCALL, mmap_mmap_rfind__doc__}, + +static PyObject * +mmap_mmap_rfind_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end); + +static PyObject * +mmap_mmap_rfind(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer view = {NULL, NULL}; + PyObject *start = Py_None; + PyObject *end = Py_None; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + start = args[1]; + if (nargs < 3) { + goto skip_optional; + } + end = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_rfind_impl((mmap_object *)self, &view, start, end); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for view */ + if (view.obj) { + PyBuffer_Release(&view); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_write__doc__, +"write($self, bytes, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_WRITE_METHODDEF \ + {"write", (PyCFunction)mmap_mmap_write, METH_O, mmap_mmap_write__doc__}, + +static PyObject * +mmap_mmap_write_impl(mmap_object *self, Py_buffer *data); + +static PyObject * +mmap_mmap_write(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_write_impl((mmap_object *)self, &data); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_write_byte__doc__, +"write_byte($self, byte, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_WRITE_BYTE_METHODDEF \ + {"write_byte", (PyCFunction)mmap_mmap_write_byte, METH_O, mmap_mmap_write_byte__doc__}, + +static PyObject * +mmap_mmap_write_byte_impl(mmap_object *self, unsigned char value); + +static PyObject * +mmap_mmap_write_byte(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + unsigned char value; + + { + long ival = PyLong_AsLong(arg); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + value = (unsigned char) ival; + } + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_write_byte_impl((mmap_object *)self, value); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_size__doc__, +"size($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SIZE_METHODDEF \ + {"size", (PyCFunction)mmap_mmap_size, METH_NOARGS, mmap_mmap_size__doc__}, + +static PyObject * +mmap_mmap_size_impl(mmap_object *self); + +static PyObject * +mmap_mmap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_size_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if (defined(MS_WINDOWS) || defined(HAVE_MREMAP)) + +PyDoc_STRVAR(mmap_mmap_resize__doc__, +"resize($self, newsize, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_RESIZE_METHODDEF \ + {"resize", (PyCFunction)mmap_mmap_resize, METH_O, mmap_mmap_resize__doc__}, + +static PyObject * +mmap_mmap_resize_impl(mmap_object *self, Py_ssize_t new_size); + +static PyObject * +mmap_mmap_resize(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t new_size; + + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + new_size = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_resize_impl((mmap_object *)self, new_size); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* (defined(MS_WINDOWS) || defined(HAVE_MREMAP)) */ + +PyDoc_STRVAR(mmap_mmap_tell__doc__, +"tell($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_TELL_METHODDEF \ + {"tell", (PyCFunction)mmap_mmap_tell, METH_NOARGS, mmap_mmap_tell__doc__}, + +static PyObject * +mmap_mmap_tell_impl(mmap_object *self); + +static PyObject * +mmap_mmap_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_tell_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_flush__doc__, +"flush($self, offset=0, size=-1, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_FLUSH_METHODDEF \ + {"flush", _PyCFunction_CAST(mmap_mmap_flush), METH_FASTCALL, mmap_mmap_flush__doc__}, + +static PyObject * +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size); + +static PyObject * +mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t offset = 0; + Py_ssize_t size = -1; + + if (!_PyArg_CheckPositional("flush", nargs, 0, 2)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + size = ival; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_flush_impl((mmap_object *)self, offset, size); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_seek__doc__, +"seek($self, pos, whence=0, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SEEK_METHODDEF \ + {"seek", _PyCFunction_CAST(mmap_mmap_seek), METH_FASTCALL, mmap_mmap_seek__doc__}, + +static PyObject * +mmap_mmap_seek_impl(mmap_object *self, Py_ssize_t dist, int how); + +static PyObject * +mmap_mmap_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t dist; + int how = 0; + + if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + dist = ival; + } + if (nargs < 2) { + goto skip_optional; + } + how = PyLong_AsInt(args[1]); + if (how == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_seek_impl((mmap_object *)self, dist, how); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_seekable__doc__, +"seekable($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SEEKABLE_METHODDEF \ + {"seekable", (PyCFunction)mmap_mmap_seekable, METH_NOARGS, mmap_mmap_seekable__doc__}, + +static PyObject * +mmap_mmap_seekable_impl(mmap_object *self); + +static PyObject * +mmap_mmap_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return mmap_mmap_seekable_impl((mmap_object *)self); +} + +PyDoc_STRVAR(mmap_mmap_move__doc__, +"move($self, dest, src, count, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_MOVE_METHODDEF \ + {"move", _PyCFunction_CAST(mmap_mmap_move), METH_FASTCALL, mmap_mmap_move__doc__}, + +static PyObject * +mmap_mmap_move_impl(mmap_object *self, Py_ssize_t dest, Py_ssize_t src, + Py_ssize_t cnt); + +static PyObject * +mmap_mmap_move(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t dest; + Py_ssize_t src; + Py_ssize_t cnt; + + if (!_PyArg_CheckPositional("move", nargs, 3, 3)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + dest = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + src = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + cnt = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_move_impl((mmap_object *)self, dest, src, cnt); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap___enter____doc__, +"__enter__($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___ENTER___METHODDEF \ + {"__enter__", (PyCFunction)mmap_mmap___enter__, METH_NOARGS, mmap_mmap___enter____doc__}, + +static PyObject * +mmap_mmap___enter___impl(mmap_object *self); + +static PyObject * +mmap_mmap___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___enter___impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap___exit____doc__, +"__exit__($self, exc_type, exc_value, traceback, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___EXIT___METHODDEF \ + {"__exit__", _PyCFunction_CAST(mmap_mmap___exit__), METH_FASTCALL, mmap_mmap___exit____doc__}, + +static PyObject * +mmap_mmap___exit___impl(mmap_object *self, PyObject *exc_type, + PyObject *exc_value, PyObject *traceback); + +static PyObject * +mmap_mmap___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc_type; + PyObject *exc_value; + PyObject *traceback; + + if (!_PyArg_CheckPositional("__exit__", nargs, 3, 3)) { + goto exit; + } + exc_type = args[0]; + exc_value = args[1]; + traceback = args[2]; + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___exit___impl((mmap_object *)self, exc_type, exc_value, traceback); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(mmap_mmap___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)mmap_mmap___sizeof__, METH_NOARGS, mmap_mmap___sizeof____doc__}, + +static PyObject * +mmap_mmap___sizeof___impl(mmap_object *self); + +static PyObject * +mmap_mmap___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___sizeof___impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if (defined(MS_WINDOWS) && defined(Py_DEBUG)) + +PyDoc_STRVAR(mmap_mmap__protect__doc__, +"_protect($self, flNewProtect, start, length, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP__PROTECT_METHODDEF \ + {"_protect", _PyCFunction_CAST(mmap_mmap__protect), METH_FASTCALL, mmap_mmap__protect__doc__}, + +static PyObject * +mmap_mmap__protect_impl(mmap_object *self, unsigned int flNewProtect, + Py_ssize_t start, Py_ssize_t length); + +static PyObject * +mmap_mmap__protect(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int flNewProtect; + Py_ssize_t start; + Py_ssize_t length; + + if (!_PyArg_CheckPositional("_protect", nargs, 3, 3)) { + goto exit; + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[0], &flNewProtect, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + start = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + length = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap__protect_impl((mmap_object *)self, flNewProtect, start, length); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* (defined(MS_WINDOWS) && defined(Py_DEBUG)) */ + +#if defined(HAVE_MADVISE) + +PyDoc_STRVAR(mmap_mmap_madvise__doc__, +"madvise($self, option, start=0, length=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_MADVISE_METHODDEF \ + {"madvise", _PyCFunction_CAST(mmap_mmap_madvise), METH_FASTCALL, mmap_mmap_madvise__doc__}, + +static PyObject * +mmap_mmap_madvise_impl(mmap_object *self, int option, Py_ssize_t start, + PyObject *length_obj); + +static PyObject * +mmap_mmap_madvise(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int option; + Py_ssize_t start = 0; + PyObject *length_obj = Py_None; + + if (!_PyArg_CheckPositional("madvise", nargs, 1, 3)) { + goto exit; + } + option = PyLong_AsInt(args[0]); + if (option == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + start = ival; + } + if (nargs < 3) { + goto skip_optional; + } + length_obj = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_madvise_impl((mmap_object *)self, option, start, length_obj); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* defined(HAVE_MADVISE) */ + +#ifndef MMAP_MMAP_RESIZE_METHODDEF + #define MMAP_MMAP_RESIZE_METHODDEF +#endif /* !defined(MMAP_MMAP_RESIZE_METHODDEF) */ + +#ifndef MMAP_MMAP___SIZEOF___METHODDEF + #define MMAP_MMAP___SIZEOF___METHODDEF +#endif /* !defined(MMAP_MMAP___SIZEOF___METHODDEF) */ + +#ifndef MMAP_MMAP__PROTECT_METHODDEF + #define MMAP_MMAP__PROTECT_METHODDEF +#endif /* !defined(MMAP_MMAP__PROTECT_METHODDEF) */ + +#ifndef MMAP_MMAP_MADVISE_METHODDEF + #define MMAP_MMAP_MADVISE_METHODDEF +#endif /* !defined(MMAP_MMAP_MADVISE_METHODDEF) */ +/*[clinic end generated code: output=381f6cf4986ac867 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6b8cc3d07ab..d880fc52bb3 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -186,6 +186,140 @@ exit: return return_value; } +#if defined(HAVE_STATX) + +PyDoc_STRVAR(os_statx__doc__, +"statx($module, /, path, mask, *, flags=0, dir_fd=None,\n" +" follow_symlinks=True)\n" +"--\n" +"\n" +"Perform a statx system call on the given path.\n" +"\n" +" path\n" +" Path to be examined; can be string, bytes, a path-like object or\n" +" open-file-descriptor int.\n" +" mask\n" +" A bitmask of STATX_* constants defining the requested information.\n" +" flags\n" +" A bitmask of AT_NO_AUTOMOUNT and/or AT_STATX_* flags.\n" +" dir_fd\n" +" If not None, it should be a file descriptor open to a directory,\n" +" and path should be a relative string; path will then be relative to\n" +" that directory.\n" +" follow_symlinks\n" +" If False, and the last element of the path is a symbolic link,\n" +" statx will examine the symbolic link itself instead of the file\n" +" the link points to.\n" +"\n" +"It\'s an error to use dir_fd or follow_symlinks when specifying path as\n" +" an open file descriptor."); + +#define OS_STATX_METHODDEF \ + {"statx", _PyCFunction_CAST(os_statx), METH_FASTCALL|METH_KEYWORDS, os_statx__doc__}, + +static PyObject * +os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int flags, + int dir_fd, int follow_symlinks); + +static PyObject * +os_statx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(path), &_Py_ID(mask), &_Py_ID(flags), &_Py_ID(dir_fd), &_Py_ID(follow_symlinks), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", "mask", "flags", "dir_fd", "follow_symlinks", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "statx", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + path_t path = PATH_T_INITIALIZE_P("statx", "path", 0, 0, 0, 1); + unsigned int mask; + int flags = 0; + int dir_fd = DEFAULT_DIR_FD; + int follow_symlinks = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &mask, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + flags = PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[3]) { + if (!dir_fd_converter(args[3], &dir_fd)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + follow_symlinks = PyObject_IsTrue(args[4]); + if (follow_symlinks < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = os_statx_impl(module, &path, mask, flags, dir_fd, follow_symlinks); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(HAVE_STATX) */ + PyDoc_STRVAR(os_access__doc__, "access($module, /, path, mode, *, dir_fd=None, effective_ids=False,\n" " follow_symlinks=True)\n" @@ -215,8 +349,8 @@ PyDoc_STRVAR(os_access__doc__, " NotImplementedError.\n" "\n" "Note that most operations will use the effective uid/gid, therefore this\n" -" routine can be used in a suid/sgid environment to test if the invoking user\n" -" has the specified access to the path."); +" routine can be used in a suid/sgid environment to test if the invoking\n" +" user has the specified access to the path."); #define OS_ACCESS_METHODDEF \ {"access", _PyCFunction_CAST(os_access), METH_FASTCALL|METH_KEYWORDS, os_access__doc__}, @@ -380,7 +514,7 @@ PyDoc_STRVAR(os_chdir__doc__, "\n" "path may always be specified as a string.\n" "On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"If this functionality is unavailable, using it raises an exception."); #define OS_CHDIR_METHODDEF \ {"chdir", _PyCFunction_CAST(os_chdir), METH_FASTCALL|METH_KEYWORDS, os_chdir__doc__}, @@ -844,7 +978,22 @@ os_chflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * _PyArg_BadArgument("chflags", "argument 'flags'", "int", args[1]); goto exit; } - flags = PyLong_AsUnsignedLongMask(args[1]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &flags, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } if (!noptargs) { goto skip_optional_pos; } @@ -928,7 +1077,22 @@ os_lchflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject _PyArg_BadArgument("lchflags", "argument 'flags'", "int", args[1]); goto exit; } - flags = PyLong_AsUnsignedLongMask(args[1]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &flags, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } return_value = os_lchflags_impl(module, &path, flags); exit: @@ -1659,7 +1823,7 @@ exit: return return_value; } -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os_listdrives__doc__, "listdrives($module, /)\n" @@ -1681,9 +1845,9 @@ os_listdrives(PyObject *module, PyObject *Py_UNUSED(ignored)) return os_listdrives_impl(module); } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os_listvolumes__doc__, "listvolumes($module, /)\n" @@ -1705,9 +1869,9 @@ os_listvolumes(PyObject *module, PyObject *Py_UNUSED(ignored)) return os_listvolumes_impl(module); } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os_listmounts__doc__, "listmounts($module, /, volume)\n" @@ -1774,7 +1938,7 @@ exit: return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ #if defined(MS_WINDOWS) @@ -2014,57 +2178,24 @@ exit: #if defined(MS_WINDOWS) PyDoc_STRVAR(os__path_splitroot__doc__, -"_path_splitroot($module, /, path)\n" +"_path_splitroot($module, path, /)\n" "--\n" "\n" "Removes everything after the root on Win32."); #define OS__PATH_SPLITROOT_METHODDEF \ - {"_path_splitroot", _PyCFunction_CAST(os__path_splitroot), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot__doc__}, + {"_path_splitroot", (PyCFunction)os__path_splitroot, METH_O, os__path_splitroot__doc__}, static PyObject * os__path_splitroot_impl(PyObject *module, path_t *path); static PyObject * -os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_splitroot(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(path), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_path_splitroot", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_path_splitroot", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!path_converter(args[0], &path)) { + if (!path_converter(arg, &path)) { goto exit; } return_value = os__path_splitroot_impl(module, &path); @@ -2225,58 +2356,25 @@ exit: #if defined(MS_WINDOWS) PyDoc_STRVAR(os__path_isdir__doc__, -"_path_isdir($module, /, s)\n" +"_path_isdir($module, path, /)\n" "--\n" "\n" "Return true if the pathname refers to an existing directory."); #define OS__PATH_ISDIR_METHODDEF \ - {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, + {"_path_isdir", (PyCFunction)os__path_isdir, METH_O, os__path_isdir__doc__}, static int os__path_isdir_impl(PyObject *module, path_t *path); static PyObject * -os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_isdir(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { _Py_LATIN1_CHR('s'), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"s", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_path_isdir", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_path_isdir", "path", 0, 0, 1, 1); int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!path_converter(args[0], &path)) { + if (!path_converter(arg, &path)) { goto exit; } _return_value = os__path_isdir_impl(module, &path); @@ -2511,7 +2609,7 @@ exit: #endif /* defined(MS_WINDOWS) */ PyDoc_STRVAR(os__path_splitroot_ex__doc__, -"_path_splitroot_ex($module, /, p)\n" +"_path_splitroot_ex($module, path, /)\n" "--\n" "\n" "Split a pathname into drive, root and tail.\n" @@ -2519,51 +2617,18 @@ PyDoc_STRVAR(os__path_splitroot_ex__doc__, "The tail contains anything after the root."); #define OS__PATH_SPLITROOT_EX_METHODDEF \ - {"_path_splitroot_ex", _PyCFunction_CAST(os__path_splitroot_ex), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot_ex__doc__}, + {"_path_splitroot_ex", (PyCFunction)os__path_splitroot_ex, METH_O, os__path_splitroot_ex__doc__}, static PyObject * os__path_splitroot_ex_impl(PyObject *module, path_t *path); static PyObject * -os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_splitroot_ex(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { _Py_LATIN1_CHR('p'), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"p", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_path_splitroot_ex", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE("_path_splitroot_ex", "path", 0, 1, 1, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!path_converter(args[0], &path)) { + if (!path_converter(arg, &path)) { goto exit; } return_value = os__path_splitroot_ex_impl(module, &path); @@ -5104,7 +5169,8 @@ PyDoc_STRVAR(os_forkpty__doc__, "Returns a tuple of (pid, master_fd).\n" "Like fork(), return pid of 0 to the child process,\n" "and pid of child to the parent process.\n" -"To both, return fd of newly opened pseudo-terminal."); +"To both, return fd of newly opened pseudo-terminal.\n" +"The master_fd is non-inheritable."); #define OS_FORKPTY_METHODDEF \ {"forkpty", (PyCFunction)os_forkpty, METH_NOARGS, os_forkpty__doc__}, @@ -7883,6 +7949,7 @@ PyDoc_STRVAR(os_preadv__doc__, "\n" "- RWF_HIPRI\n" "- RWF_NOWAIT\n" +"- RWF_DONTCACHE\n" "\n" "Using non-zero flags requires Linux 4.6 or newer."); @@ -8169,6 +8236,11 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } count = ival; + if (count < 0) { + PyErr_SetString(PyExc_ValueError, + "count cannot be negative"); + goto exit; + } } if (!noptargs) { goto skip_optional_pos; @@ -8275,6 +8347,11 @@ os_sendfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } count = ival; + if (count < 0) { + PyErr_SetString(PyExc_ValueError, + "count cannot be negative"); + goto exit; + } } return_value = os_sendfile_impl(module, out_fd, in_fd, offobj, count); @@ -8614,6 +8691,7 @@ PyDoc_STRVAR(os_pwritev__doc__, "- RWF_DSYNC\n" "- RWF_SYNC\n" "- RWF_APPEND\n" +"- RWF_DONTCACHE\n" "\n" "Using non-zero flags requires Linux 4.7 or newer."); @@ -8758,6 +8836,11 @@ os_copy_file_range(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py goto exit; } count = ival; + if (count < 0) { + PyErr_SetString(PyExc_ValueError, + "count cannot be negative"); + goto exit; + } } if (!noptargs) { goto skip_optional_pos; @@ -8876,6 +8959,11 @@ os_splice(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k goto exit; } count = ival; + if (count < 0) { + PyErr_SetString(PyExc_ValueError, + "count cannot be negative"); + goto exit; + } } if (!noptargs) { goto skip_optional_pos; @@ -9585,6 +9673,27 @@ exit: #endif /* !defined(MS_WINDOWS) */ +#if defined(HAVE_CLEARENV) + +PyDoc_STRVAR(os__clearenv__doc__, +"_clearenv($module, /)\n" +"--\n" +"\n"); + +#define OS__CLEARENV_METHODDEF \ + {"_clearenv", (PyCFunction)os__clearenv, METH_NOARGS, os__clearenv__doc__}, + +static PyObject * +os__clearenv_impl(PyObject *module); + +static PyObject * +os__clearenv(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__clearenv_impl(module); +} + +#endif /* defined(HAVE_CLEARENV) */ + PyDoc_STRVAR(os_strerror__doc__, "strerror($module, code, /)\n" "--\n" @@ -11306,6 +11415,11 @@ os_urandom(PyObject *module, PyObject *arg) goto exit; } size = ival; + if (size < 0) { + PyErr_SetString(PyExc_ValueError, + "size cannot be negative"); + goto exit; + } } return_value = os_urandom_impl(module, size); @@ -11373,9 +11487,21 @@ os_memfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_pos; } - flags = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (flags == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &flags, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional_pos: return_value = os_memfd_create_impl(module, name, flags); @@ -12655,7 +12781,7 @@ PyDoc_STRVAR(os__inputhook__doc__, "_inputhook($module, /)\n" "--\n" "\n" -"Calls PyOS_CallInputHook droppong the GIL first"); +"Calls PyOS_InputHook dropping the GIL first"); #define OS__INPUTHOOK_METHODDEF \ {"_inputhook", (PyCFunction)os__inputhook, METH_NOARGS, os__inputhook__doc__}, @@ -12673,7 +12799,7 @@ PyDoc_STRVAR(os__is_inputhook_installed__doc__, "_is_inputhook_installed($module, /)\n" "--\n" "\n" -"Checks if PyOS_CallInputHook is set"); +"Checks if PyOS_InputHook is set"); #define OS__IS_INPUTHOOK_INSTALLED_METHODDEF \ {"_is_inputhook_installed", (PyCFunction)os__is_inputhook_installed, METH_NOARGS, os__is_inputhook_installed__doc__}, @@ -12727,6 +12853,84 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(__EMSCRIPTEN__) */ +#if defined(__EMSCRIPTEN__) + +PyDoc_STRVAR(os__emscripten_log__doc__, +"_emscripten_log($module, /, arg)\n" +"--\n" +"\n" +"Log something to the JS console. Emscripten only."); + +#define OS__EMSCRIPTEN_LOG_METHODDEF \ + {"_emscripten_log", _PyCFunction_CAST(os__emscripten_log), METH_FASTCALL|METH_KEYWORDS, os__emscripten_log__doc__}, + +static PyObject * +os__emscripten_log_impl(PyObject *module, const char *arg); + +static PyObject * +os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_emscripten_log", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *arg; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("_emscripten_log", "argument 'arg'", "str", args[0]); + goto exit; + } + Py_ssize_t arg_length; + arg = PyUnicode_AsUTF8AndSize(args[0], &arg_length); + if (arg == NULL) { + goto exit; + } + if (strlen(arg) != (size_t)arg_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = os__emscripten_log_impl(module, arg); + +exit: + return return_value; +} + +#endif /* defined(__EMSCRIPTEN__) */ + +#ifndef OS_STATX_METHODDEF + #define OS_STATX_METHODDEF +#endif /* !defined(OS_STATX_METHODDEF) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -13247,6 +13451,10 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS_UNSETENV_METHODDEF #endif /* !defined(OS_UNSETENV_METHODDEF) */ +#ifndef OS__CLEARENV_METHODDEF + #define OS__CLEARENV_METHODDEF +#endif /* !defined(OS__CLEARENV_METHODDEF) */ + #ifndef OS_WCOREDUMP_METHODDEF #define OS_WCOREDUMP_METHODDEF #endif /* !defined(OS_WCOREDUMP_METHODDEF) */ @@ -13398,4 +13606,8 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ -/*[clinic end generated code: output=f7b5635e0b948be4 input=a9049054013a1b77]*/ + +#ifndef OS__EMSCRIPTEN_LOG_METHODDEF + #define OS__EMSCRIPTEN_LOG_METHODDEF +#endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ +/*[clinic end generated code: output=82f60940338c70e4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pwdmodule.c.h b/Modules/clinic/pwdmodule.c.h index 365d99aab1d..43d4825031c 100644 --- a/Modules/clinic/pwdmodule.c.h +++ b/Modules/clinic/pwdmodule.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_BadArgument() + PyDoc_STRVAR(pwd_getpwuid__doc__, "getpwuid($module, uidobj, /)\n" "--\n" @@ -34,7 +36,7 @@ pwd_getpwnam(PyObject *module, PyObject *arg) PyObject *name; if (!PyUnicode_Check(arg)) { - PyErr_Format(PyExc_TypeError, "getpwnam() argument must be str, not %T", arg); + _PyArg_BadArgument("getpwnam", "argument", "str", arg); goto exit; } name = arg; @@ -71,4 +73,4 @@ pwd_getpwall(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef PWD_GETPWALL_METHODDEF #define PWD_GETPWALL_METHODDEF #endif /* !defined(PWD_GETPWALL_METHODDEF) */ -/*[clinic end generated code: output=dac88d500f6d6f49 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5a8fb12939ff4ea3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 13210e3be0f..ff2e28269dc 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_SINGLETON() #endif +#include "pycore_long.h" // _PyLong_UnsignedLongLong_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__, @@ -408,6 +409,267 @@ exit: #endif /* (XML_COMBINED_VERSION >= 19505) */ +#if (XML_COMBINED_VERSION >= 20400) + +PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold__doc__, +"SetBillionLaughsAttackProtectionActivationThreshold($self, threshold, /)\n" +"--\n" +"\n" +"Sets the number of output bytes needed to activate protection against billion laughs attacks.\n" +"\n" +"The number of output bytes includes amplification from entity expansion\n" +"and reading DTD files.\n" +"\n" +"Parser objects usually have a protection activation threshold of 8 MiB,\n" +"but the actual default value depends on the underlying Expat library.\n" +"\n" +"Activation thresholds below 4 MiB are known to break support for DITA 1.3\n" +"payload and are hence not recommended."); + +#define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF \ + {"SetBillionLaughsAttackProtectionActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold__doc__}, + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold); + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetBillionLaughsAttackProtectionActivationThreshold", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + unsigned long long threshold; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &threshold)) { + goto exit; + } + return_value = pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl((xmlparseobject *)self, cls, threshold); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20400) */ + +#if (XML_COMBINED_VERSION >= 20400) + +PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification__doc__, +"SetBillionLaughsAttackProtectionMaximumAmplification($self, max_factor,\n" +" /)\n" +"--\n" +"\n" +"Sets the maximum tolerated amplification factor for protection against billion laughs attacks.\n" +"\n" +"The amplification factor is calculated as \"(direct + indirect) / direct\"\n" +"while parsing, where \"direct\" is the number of bytes read from the primary\n" +"document in parsing and \"indirect\" is the number of bytes added by expanding\n" +"entities and reading external DTD files, combined.\n" +"\n" +"The \'max_factor\' value must be a non-NaN floating point value greater than\n" +"or equal to 1.0. Amplification factors greater than 30,000 can be observed\n" +"in the middle of parsing even with benign files in practice. In particular,\n" +"the activation threshold should be carefully chosen to avoid false positives.\n" +"\n" +"Parser objects usually have a maximum amplification factor of 100,\n" +"but the actual default value depends on the underlying Expat library."); + +#define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF \ + {"SetBillionLaughsAttackProtectionMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification__doc__}, + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor); + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetBillionLaughsAttackProtectionMaximumAmplification", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + float max_factor; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + max_factor = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + max_factor = (float) PyFloat_AsDouble(args[0]); + if (max_factor == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl((xmlparseobject *)self, cls, max_factor); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20400) */ + +#if (XML_COMBINED_VERSION >= 20702) + +PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__, +"SetAllocTrackerActivationThreshold($self, threshold, /)\n" +"--\n" +"\n" +"Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM.\n" +"\n" +"Parser objects usually have an allocation activation threshold of 64 MiB,\n" +"but the actual default value depends on the underlying Expat library."); + +#define PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF \ + {"SetAllocTrackerActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__}, + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold); + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerActivationThreshold(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetAllocTrackerActivationThreshold", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + unsigned long long threshold; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &threshold)) { + goto exit; + } + return_value = pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl((xmlparseobject *)self, cls, threshold); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20702) */ + +#if (XML_COMBINED_VERSION >= 20702) + +PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__, +"SetAllocTrackerMaximumAmplification($self, max_factor, /)\n" +"--\n" +"\n" +"Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated.\n" +"\n" +"The amplification factor is calculated as \"allocated / direct\" while parsing,\n" +"where \"direct\" is the number of bytes read from the primary document in parsing\n" +"and \"allocated\" is the number of bytes of dynamic memory allocated in the parser\n" +"hierarchy.\n" +"\n" +"The \'max_factor\' value must be a non-NaN floating point value greater than\n" +"or equal to 1.0. Amplification factors greater than 100.0 can be observed\n" +"near the start of parsing even with benign files in practice. In particular,\n" +"the activation threshold should be carefully chosen to avoid false positives.\n" +"\n" +"Parser objects usually have a maximum amplification factor of 100,\n" +"but the actual default value depends on the underlying Expat library."); + +#define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF \ + {"SetAllocTrackerMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__}, + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor); + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerMaximumAmplification(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetAllocTrackerMaximumAmplification", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + float max_factor; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + max_factor = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + max_factor = (float) PyFloat_AsDouble(args[0]); + if (max_factor == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl((xmlparseobject *)self, cls, max_factor); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20702) */ + PyDoc_STRVAR(pyexpat_ParserCreate__doc__, "ParserCreate($module, /, encoding=None, namespace_separator=None,\n" " intern=<unrepresentable>)\n" @@ -552,4 +814,20 @@ exit: #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=4dbdc959c67dc2d5 input=a9049054013a1b77]*/ + +#ifndef PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF + #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF) */ + +#ifndef PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF + #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF) */ + +#ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF + #define PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF) */ + +#ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF + #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF) */ +/*[clinic end generated code: output=81101a16a409daf6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/readline.c.h b/Modules/clinic/readline.c.h index 696475f7d00..dc9381e4b97 100644 --- a/Modules/clinic/readline.c.h +++ b/Modules/clinic/readline.c.h @@ -349,6 +349,28 @@ exit: #endif /* defined(HAVE_RL_PRE_INPUT_HOOK) */ +#if defined(HAVE_RL_PRE_INPUT_HOOK) + +PyDoc_STRVAR(readline_get_pre_input_hook__doc__, +"get_pre_input_hook($module, /)\n" +"--\n" +"\n" +"Get the current pre-input hook function."); + +#define READLINE_GET_PRE_INPUT_HOOK_METHODDEF \ + {"get_pre_input_hook", (PyCFunction)readline_get_pre_input_hook, METH_NOARGS, readline_get_pre_input_hook__doc__}, + +static PyObject * +readline_get_pre_input_hook_impl(PyObject *module); + +static PyObject * +readline_get_pre_input_hook(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return readline_get_pre_input_hook_impl(module); +} + +#endif /* defined(HAVE_RL_PRE_INPUT_HOOK) */ + PyDoc_STRVAR(readline_get_completion_type__doc__, "get_completion_type($module, /)\n" "--\n" @@ -794,7 +816,11 @@ readline_redisplay(PyObject *module, PyObject *Py_UNUSED(ignored)) #define READLINE_SET_PRE_INPUT_HOOK_METHODDEF #endif /* !defined(READLINE_SET_PRE_INPUT_HOOK_METHODDEF) */ +#ifndef READLINE_GET_PRE_INPUT_HOOK_METHODDEF + #define READLINE_GET_PRE_INPUT_HOOK_METHODDEF +#endif /* !defined(READLINE_GET_PRE_INPUT_HOOK_METHODDEF) */ + #ifndef READLINE_CLEAR_HISTORY_METHODDEF #define READLINE_CLEAR_HISTORY_METHODDEF #endif /* !defined(READLINE_CLEAR_HISTORY_METHODDEF) */ -/*[clinic end generated code: output=88d9812b6caa2102 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4bd95070973cd0e2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/resource.c.h b/Modules/clinic/resource.c.h index 9eda7de2753..e4ef93900d1 100644 --- a/Modules/clinic/resource.c.h +++ b/Modules/clinic/resource.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + #if defined(HAVE_GETRUSAGE) PyDoc_STRVAR(resource_getrusage__doc__, @@ -66,7 +68,7 @@ PyDoc_STRVAR(resource_setrlimit__doc__, "\n"); #define RESOURCE_SETRLIMIT_METHODDEF \ - {"setrlimit", (PyCFunction)(void(*)(void))resource_setrlimit, METH_FASTCALL, resource_setrlimit__doc__}, + {"setrlimit", _PyCFunction_CAST(resource_setrlimit), METH_FASTCALL, resource_setrlimit__doc__}, static PyObject * resource_setrlimit_impl(PyObject *module, int resource, PyObject *limits); @@ -78,8 +80,7 @@ resource_setrlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits; - if (nargs != 2) { - PyErr_Format(PyExc_TypeError, "setrlimit expected 2 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("setrlimit", nargs, 2, 2)) { goto exit; } resource = PyLong_AsInt(args[0]); @@ -101,7 +102,7 @@ PyDoc_STRVAR(resource_prlimit__doc__, "\n"); #define RESOURCE_PRLIMIT_METHODDEF \ - {"prlimit", (PyCFunction)(void(*)(void))resource_prlimit, METH_FASTCALL, resource_prlimit__doc__}, + {"prlimit", _PyCFunction_CAST(resource_prlimit), METH_FASTCALL, resource_prlimit__doc__}, static PyObject * resource_prlimit_impl(PyObject *module, pid_t pid, int resource, @@ -115,12 +116,7 @@ resource_prlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits = Py_None; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "prlimit expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 3) { - PyErr_Format(PyExc_TypeError, "prlimit expected at most 3 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("prlimit", nargs, 2, 3)) { goto exit; } pid = PyLong_AsPid(args[0]); @@ -178,4 +174,4 @@ exit: #ifndef RESOURCE_PRLIMIT_METHODDEF #define RESOURCE_PRLIMIT_METHODDEF #endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */ -/*[clinic end generated code: output=e45883ace510414a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e905b2f5c35170e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 253ad8c9e78..26ddc6ffba7 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -26,7 +26,7 @@ PyDoc_STRVAR(select_select__doc__, "gotten from a fileno() method call on one of those.\n" "\n" "The optional 4th argument specifies a timeout in seconds; it may be\n" -"a floating-point number to specify fractions of seconds. If it is absent\n" +"a non-integer to specify fractions of seconds. If it is absent\n" "or None, the call will never time out.\n" "\n" "The return value is a tuple of three lists corresponding to the first three\n" @@ -783,9 +783,21 @@ select_epoll_register(PyObject *self, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_pos; } - eventmask = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (eventmask == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &eventmask, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional_pos: return_value = select_epoll_register_impl((pyEpoll_Object *)self, fd, eventmask); @@ -860,9 +872,21 @@ select_epoll_modify(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyO if (fd < 0) { goto exit; } - eventmask = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (eventmask == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &eventmask, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } return_value = select_epoll_modify_impl((pyEpoll_Object *)self, fd, eventmask); @@ -949,7 +973,7 @@ PyDoc_STRVAR(select_epoll_poll__doc__, "Wait for events on the epoll file descriptor.\n" "\n" " timeout\n" -" the maximum time to wait in seconds (as float);\n" +" the maximum time to wait in seconds (with fractions);\n" " a timeout of None or -1 makes poll wait indefinitely\n" " maxevents\n" " the maximum number of events returned; -1 means no limit\n" @@ -1238,7 +1262,7 @@ PyDoc_STRVAR(select_kqueue_control__doc__, " The maximum number of events that the kernel will return.\n" " timeout\n" " The maximum time to wait in seconds, or else None to wait forever.\n" -" This accepts floats for smaller timeouts, too."); +" This accepts non-integers for smaller timeouts, too."); #define SELECT_KQUEUE_CONTROL_METHODDEF \ {"control", _PyCFunction_CAST(select_kqueue_control), METH_FASTCALL, select_kqueue_control__doc__}, @@ -1375,4 +1399,4 @@ exit: #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=6fc20d78802511d1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ae54d65938513132 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index 3e5fd1a41ce..4a58d0cd9b8 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -89,7 +89,7 @@ SHA1Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_sha1_sha1__doc__, -"sha1($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha1($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA1 hash object; optionally initialized with a string."); @@ -98,7 +98,8 @@ PyDoc_STRVAR(_sha1_sha1__doc__, {"sha1", _PyCFunction_CAST(_sha1_sha1), METH_FASTCALL|METH_KEYWORDS, _sha1_sha1__doc__}, static PyObject * -_sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha1_sha1_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -106,7 +107,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -115,7 +116,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -124,17 +125,18 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha1", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -145,7 +147,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -154,14 +156,20 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha1_sha1_impl(module, string, usedforsecurity); + return_value = _sha1_sha1_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=06161e87e2d645d4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fd5a917404b68c4f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index 26612125e75..07be91e4f6c 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -169,7 +169,7 @@ SHA512Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_sha2_sha256__doc__, -"sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha256($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-256 hash object; optionally initialized with a string."); @@ -178,7 +178,8 @@ PyDoc_STRVAR(_sha2_sha256__doc__, {"sha256", _PyCFunction_CAST(_sha2_sha256), METH_FASTCALL|METH_KEYWORDS, _sha2_sha256__doc__}, static PyObject * -_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha256_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -186,7 +187,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -195,7 +196,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -204,17 +205,18 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -225,7 +227,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -234,19 +236,25 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha256_impl(module, string, usedforsecurity); + return_value = _sha2_sha256_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha224__doc__, -"sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha224($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-224 hash object; optionally initialized with a string."); @@ -255,7 +263,8 @@ PyDoc_STRVAR(_sha2_sha224__doc__, {"sha224", _PyCFunction_CAST(_sha2_sha224), METH_FASTCALL|METH_KEYWORDS, _sha2_sha224__doc__}, static PyObject * -_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha224_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -263,7 +272,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -272,7 +281,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -281,17 +290,18 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -302,7 +312,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -311,19 +321,25 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha224_impl(module, string, usedforsecurity); + return_value = _sha2_sha224_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha512__doc__, -"sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha512($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-512 hash object; optionally initialized with a string."); @@ -332,7 +348,8 @@ PyDoc_STRVAR(_sha2_sha512__doc__, {"sha512", _PyCFunction_CAST(_sha2_sha512), METH_FASTCALL|METH_KEYWORDS, _sha2_sha512__doc__}, static PyObject * -_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha512_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -340,7 +357,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -349,7 +366,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -358,17 +375,18 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -379,7 +397,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -388,19 +406,25 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha512_impl(module, string, usedforsecurity); + return_value = _sha2_sha512_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha384__doc__, -"sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha384($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-384 hash object; optionally initialized with a string."); @@ -409,7 +433,8 @@ PyDoc_STRVAR(_sha2_sha384__doc__, {"sha384", _PyCFunction_CAST(_sha2_sha384), METH_FASTCALL|METH_KEYWORDS, _sha2_sha384__doc__}, static PyObject * -_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha384_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -417,7 +442,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -426,7 +451,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -435,17 +460,18 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -456,7 +482,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -465,14 +491,20 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha384_impl(module, string, usedforsecurity); + return_value = _sha2_sha384_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=af11090855b7c85a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=90625b237c774a9f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h index 25f72b74f80..7fdc707626c 100644 --- a/Modules/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -6,17 +6,18 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_long.h" // _PyLong_UnsignedLong_Converter() +#include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_sha3_new__doc__, -"sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" +"sha3_224(data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA3 hash object."); static PyObject * -py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity); +py_sha3_new_impl(PyTypeObject *type, PyObject *data_obj, int usedforsecurity, + PyObject *string); static PyObject * py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -24,7 +25,7 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -33,7 +34,7 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -42,40 +43,51 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha3_224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(fastargs[1]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[1]) { + usedforsecurity = PyObject_IsTrue(fastargs[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[2]; skip_optional_kwonly: - return_value = py_sha3_new_impl(type, data, usedforsecurity); + return_value = py_sha3_new_impl(type, data_obj, usedforsecurity, string); exit: return return_value; @@ -88,15 +100,19 @@ PyDoc_STRVAR(_sha3_sha3_224_copy__doc__, "Return a copy of the hash object."); #define _SHA3_SHA3_224_COPY_METHODDEF \ - {"copy", (PyCFunction)_sha3_sha3_224_copy, METH_NOARGS, _sha3_sha3_224_copy__doc__}, + {"copy", _PyCFunction_CAST(_sha3_sha3_224_copy), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _sha3_sha3_224_copy__doc__}, static PyObject * -_sha3_sha3_224_copy_impl(SHA3object *self); +_sha3_sha3_224_copy_impl(SHA3object *self, PyTypeObject *cls); static PyObject * -_sha3_sha3_224_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +_sha3_sha3_224_copy(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _sha3_sha3_224_copy_impl((SHA3object *)self); + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "copy() takes no arguments"); + return NULL; + } + return _sha3_sha3_224_copy_impl((SHA3object *)self, cls); } PyDoc_STRVAR(_sha3_sha3_224_digest__doc__, @@ -158,26 +174,73 @@ _sha3_sha3_224_update(PyObject *self, PyObject *data) } PyDoc_STRVAR(_sha3_shake_128_digest__doc__, -"digest($self, length, /)\n" +"digest($self, /, length)\n" "--\n" "\n" "Return the digest value as a bytes object."); #define _SHA3_SHAKE_128_DIGEST_METHODDEF \ - {"digest", (PyCFunction)_sha3_shake_128_digest, METH_O, _sha3_shake_128_digest__doc__}, + {"digest", _PyCFunction_CAST(_sha3_shake_128_digest), METH_FASTCALL|METH_KEYWORDS, _sha3_shake_128_digest__doc__}, static PyObject * -_sha3_shake_128_digest_impl(SHA3object *self, unsigned long length); +_sha3_shake_128_digest_impl(SHA3object *self, Py_ssize_t length); static PyObject * -_sha3_shake_128_digest(PyObject *self, PyObject *arg) +_sha3_shake_128_digest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - unsigned long length; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - if (!_PyLong_UnsignedLong_Converter(arg, &length)) { + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(length), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"length", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "digest", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t length; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + length = ival; + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length cannot be negative"); + goto exit; + } + } return_value = _sha3_shake_128_digest_impl((SHA3object *)self, length); exit: @@ -185,29 +248,76 @@ exit: } PyDoc_STRVAR(_sha3_shake_128_hexdigest__doc__, -"hexdigest($self, length, /)\n" +"hexdigest($self, /, length)\n" "--\n" "\n" "Return the digest value as a string of hexadecimal digits."); #define _SHA3_SHAKE_128_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)_sha3_shake_128_hexdigest, METH_O, _sha3_shake_128_hexdigest__doc__}, + {"hexdigest", _PyCFunction_CAST(_sha3_shake_128_hexdigest), METH_FASTCALL|METH_KEYWORDS, _sha3_shake_128_hexdigest__doc__}, static PyObject * -_sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length); +_sha3_shake_128_hexdigest_impl(SHA3object *self, Py_ssize_t length); static PyObject * -_sha3_shake_128_hexdigest(PyObject *self, PyObject *arg) +_sha3_shake_128_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - unsigned long length; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - if (!_PyLong_UnsignedLong_Converter(arg, &length)) { + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(length), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"length", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "hexdigest", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t length; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + length = ival; + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length cannot be negative"); + goto exit; + } + } return_value = _sha3_shake_128_hexdigest_impl((SHA3object *)self, length); exit: return return_value; } -/*[clinic end generated code: output=5b3ac1c06c6899ea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=78284adde71d590c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 955861b4da3..9fd24d15bf2 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -600,7 +600,7 @@ PyDoc_STRVAR(signal_sigtimedwait__doc__, "\n" "Like sigwaitinfo(), but with a timeout.\n" "\n" -"The timeout is specified in seconds, with floating-point numbers allowed."); +"The timeout is specified in seconds, rounded up to nanoseconds."); #define SIGNAL_SIGTIMEDWAIT_METHODDEF \ {"sigtimedwait", _PyCFunction_CAST(signal_sigtimedwait), METH_FASTCALL, signal_sigtimedwait__doc__}, @@ -660,7 +660,22 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("pthread_kill", "argument 1", "int", args[0]); goto exit; } - thread_id = PyLong_AsUnsignedLongMask(args[0]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[0], &thread_id, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } signalnum = PyLong_AsInt(args[1]); if (signalnum == -1 && PyErr_Occurred()) { goto exit; @@ -779,4 +794,4 @@ exit: #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=48bfaffeb25df5d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=42e20d118435d7fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 70ebbaa876b..e0cc1c50dcb 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -6,7 +6,8 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_long.h" // _PyLong_UInt16_Converter() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_socket_socket_close__doc__, "close($self, /)\n" @@ -28,6 +29,170 @@ _socket_socket_close(PyObject *s, PyObject *Py_UNUSED(ignored)) return _socket_socket_close_impl((PySocketSockObject *)s); } +PyDoc_STRVAR(_socket_socket_send__doc__, +"send($self, data, flags=0, /)\n" +"--\n" +"\n" +"Send a data string to the socket.\n" +"\n" +"For the optional flags argument, see the Unix manual.\n" +"Return the number of bytes sent; this may be less than len(data) if the network is busy."); + +#define _SOCKET_SOCKET_SEND_METHODDEF \ + {"send", _PyCFunction_CAST(_socket_socket_send), METH_FASTCALL, _socket_socket_send__doc__}, + +static PyObject * +_socket_socket_send_impl(PySocketSockObject *s, Py_buffer *pbuf, int flags); + +static PyObject * +_socket_socket_send(PyObject *s, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer pbuf = {NULL, NULL}; + int flags = 0; + + if (!_PyArg_CheckPositional("send", nargs, 1, 2)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &pbuf, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + flags = PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = _socket_socket_send_impl((PySocketSockObject *)s, &pbuf, flags); + +exit: + /* Cleanup for pbuf */ + if (pbuf.obj) { + PyBuffer_Release(&pbuf); + } + + return return_value; +} + +PyDoc_STRVAR(_socket_socket_sendall__doc__, +"sendall($self, data, flags=0, /)\n" +"--\n" +"\n" +"Send a data string to the socket.\n" +"\n" +"For the optional flags argument, see the Unix manual.\n" +"This calls send() repeatedly until all data is sent.\n" +"If an error occurs, it\'s impossible to tell how much data has been sent."); + +#define _SOCKET_SOCKET_SENDALL_METHODDEF \ + {"sendall", _PyCFunction_CAST(_socket_socket_sendall), METH_FASTCALL, _socket_socket_sendall__doc__}, + +static PyObject * +_socket_socket_sendall_impl(PySocketSockObject *s, Py_buffer *pbuf, + int flags); + +static PyObject * +_socket_socket_sendall(PyObject *s, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer pbuf = {NULL, NULL}; + int flags = 0; + + if (!_PyArg_CheckPositional("sendall", nargs, 1, 2)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &pbuf, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + flags = PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = _socket_socket_sendall_impl((PySocketSockObject *)s, &pbuf, flags); + +exit: + /* Cleanup for pbuf */ + if (pbuf.obj) { + PyBuffer_Release(&pbuf); + } + + return return_value; +} + +#if defined(CMSG_LEN) + +PyDoc_STRVAR(_socket_socket_sendmsg__doc__, +"sendmsg($self, buffers, ancdata=<unrepresentable>, flags=0,\n" +" address=<unrepresentable>, /)\n" +"--\n" +"\n" +"Send normal and ancillary data to the socket.\n" +"\n" +"It gathering the non-ancillary data from a series of buffers\n" +"and concatenating it into a single message.\n" +"The buffers argument specifies the non-ancillary\n" +"data as an iterable of bytes-like objects (e.g. bytes objects).\n" +"The ancdata argument specifies the ancillary data (control messages)\n" +"as an iterable of zero or more tuples (cmsg_level, cmsg_type,\n" +"cmsg_data), where cmsg_level and cmsg_type are integers specifying the\n" +"protocol level and protocol-specific type respectively, and cmsg_data\n" +"is a bytes-like object holding the associated data. The flags\n" +"argument defaults to 0 and has the same meaning as for send(). If\n" +"address is supplied and not None, it sets a destination address for\n" +"the message. The return value is the number of bytes of non-ancillary\n" +"data sent."); + +#define _SOCKET_SOCKET_SENDMSG_METHODDEF \ + {"sendmsg", _PyCFunction_CAST(_socket_socket_sendmsg), METH_FASTCALL, _socket_socket_sendmsg__doc__}, + +static PyObject * +_socket_socket_sendmsg_impl(PySocketSockObject *s, PyObject *data_arg, + PyObject *cmsg_arg, int flags, + PyObject *addr_arg); + +static PyObject * +_socket_socket_sendmsg(PyObject *s, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *data_arg; + PyObject *cmsg_arg = NULL; + int flags = 0; + PyObject *addr_arg = NULL; + + if (!_PyArg_CheckPositional("sendmsg", nargs, 1, 4)) { + goto exit; + } + data_arg = args[0]; + if (nargs < 2) { + goto skip_optional; + } + cmsg_arg = args[1]; + if (nargs < 3) { + goto skip_optional; + } + flags = PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + addr_arg = args[3]; +skip_optional: + return_value = _socket_socket_sendmsg_impl((PySocketSockObject *)s, data_arg, cmsg_arg, flags, addr_arg); + +exit: + return return_value; +} + +#endif /* defined(CMSG_LEN) */ + static int sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, PyObject *fdobj); @@ -314,7 +479,7 @@ static PyObject * _socket_if_nametoindex(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *oname; + PyObject *oname = NULL; if (!PyUnicode_FSConverter(arg, &oname)) { goto exit; @@ -322,6 +487,9 @@ _socket_if_nametoindex(PyObject *module, PyObject *arg) return_value = _socket_if_nametoindex_impl(module, oname); exit: + /* Cleanup for oname */ + Py_XDECREF(oname); + return return_value; } @@ -358,6 +526,10 @@ exit: #endif /* (defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS)) */ +#ifndef _SOCKET_SOCKET_SENDMSG_METHODDEF + #define _SOCKET_SOCKET_SENDMSG_METHODDEF +#endif /* !defined(_SOCKET_SOCKET_SENDMSG_METHODDEF) */ + #ifndef _SOCKET_INET_NTOA_METHODDEF #define _SOCKET_INET_NTOA_METHODDEF #endif /* !defined(_SOCKET_INET_NTOA_METHODDEF) */ @@ -369,4 +541,4 @@ exit: #ifndef _SOCKET_IF_INDEXTONAME_METHODDEF #define _SOCKET_IF_INDEXTONAME_METHODDEF #endif /* !defined(_SOCKET_IF_INDEXTONAME_METHODDEF) */ -/*[clinic end generated code: output=c971b79d2193b426 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=36051ebf6ad1e6f8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 2ecd3afc00d..65352593f94 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -2,30 +2,67 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_symtable_symtable__doc__, -"symtable($module, source, filename, startstr, /)\n" +"symtable($module, source, filename, startstr, /, *, module=None)\n" "--\n" "\n" "Return symbol and scope dictionaries used internally by compiler."); #define _SYMTABLE_SYMTABLE_METHODDEF \ - {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL, _symtable_symtable__doc__}, + {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL|METH_KEYWORDS, _symtable_symtable__doc__}, static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr); + PyObject *filename, const char *startstr, + PyObject *modname); static PyObject * -_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - PyObject *source; - PyObject *filename; - const char *startstr; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(module), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "", "module", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "symtable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + PyObject *source; + PyObject *filename = NULL; + const char *startstr; + PyObject *modname = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } source = args[0]; @@ -45,9 +82,17 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _symtable_symtable_impl(module, source, filename, startstr); + if (!noptargs) { + goto skip_optional_kwonly; + } + modname = args[3]; +skip_optional_kwonly: + return_value = _symtable_symtable_impl(module, source, filename, startstr, modname); exit: + /* Cleanup for filename */ + Py_XDECREF(filename); + return return_value; } -/*[clinic end generated code: output=931964a76a72f850 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0137be60c487c841 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/unicodedata.c.h b/Modules/clinic/unicodedata.c.h index 345440eeee8..5fcba083c2f 100644 --- a/Modules/clinic/unicodedata.c.h +++ b/Modules/clinic/unicodedata.c.h @@ -518,6 +518,78 @@ exit: return return_value; } +PyDoc_STRVAR(unicodedata_UCD_isxidstart__doc__, +"isxidstart($self, chr, /)\n" +"--\n" +"\n" +"Return True if the character has the XID_Start property, else False."); + +#define UNICODEDATA_UCD_ISXIDSTART_METHODDEF \ + {"isxidstart", (PyCFunction)unicodedata_UCD_isxidstart, METH_O, unicodedata_UCD_isxidstart__doc__}, + +static PyObject * +unicodedata_UCD_isxidstart_impl(PyObject *self, int chr); + +static PyObject * +unicodedata_UCD_isxidstart(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int chr; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("isxidstart", "argument", "a unicode character", arg); + goto exit; + } + if (PyUnicode_GET_LENGTH(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "isxidstart(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); + goto exit; + } + chr = PyUnicode_READ_CHAR(arg, 0); + return_value = unicodedata_UCD_isxidstart_impl(self, chr); + +exit: + return return_value; +} + +PyDoc_STRVAR(unicodedata_UCD_isxidcontinue__doc__, +"isxidcontinue($self, chr, /)\n" +"--\n" +"\n" +"Return True if the character has the XID_Continue property, else False."); + +#define UNICODEDATA_UCD_ISXIDCONTINUE_METHODDEF \ + {"isxidcontinue", (PyCFunction)unicodedata_UCD_isxidcontinue, METH_O, unicodedata_UCD_isxidcontinue__doc__}, + +static PyObject * +unicodedata_UCD_isxidcontinue_impl(PyObject *self, int chr); + +static PyObject * +unicodedata_UCD_isxidcontinue(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int chr; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("isxidcontinue", "argument", "a unicode character", arg); + goto exit; + } + if (PyUnicode_GET_LENGTH(arg) != 1) { + PyErr_Format(PyExc_TypeError, + "isxidcontinue(): argument must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(arg)); + goto exit; + } + chr = PyUnicode_READ_CHAR(arg, 0); + return_value = unicodedata_UCD_isxidcontinue_impl(self, chr); + +exit: + return return_value; +} + PyDoc_STRVAR(unicodedata_UCD_lookup__doc__, "lookup($self, name, /)\n" "--\n" @@ -549,4 +621,4 @@ unicodedata_UCD_lookup(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=8a59d430cee41058 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c5e56c8f6bb80f93 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 2710f65a840..6fba75339b3 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -189,6 +189,11 @@ zlib_decompress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj goto exit; } bufsize = ival; + if (bufsize < 0) { + PyErr_SetString(PyExc_ValueError, + "bufsize cannot be negative"); + goto exit; + } } skip_optional_pos: return_value = zlib_decompress_impl(module, &data, wbits, bufsize); @@ -567,6 +572,11 @@ zlib_Decompress_decompress(PyObject *self, PyTypeObject *cls, PyObject *const *a goto exit; } max_length = ival; + if (max_length < 0) { + PyErr_SetString(PyExc_ValueError, + "max_length cannot be negative"); + goto exit; + } } skip_optional_pos: return_value = zlib_Decompress_decompress_impl((compobject *)self, cls, &data, max_length); @@ -898,7 +908,7 @@ exit: return return_value; } -PyDoc_STRVAR(zlib_ZlibDecompressor_decompress__doc__, +PyDoc_STRVAR(zlib__ZlibDecompressor_decompress__doc__, "decompress($self, /, data, max_length=-1)\n" "--\n" "\n" @@ -917,15 +927,16 @@ PyDoc_STRVAR(zlib_ZlibDecompressor_decompress__doc__, "EOFError. Any data found after the end of the stream is ignored and saved in\n" "the unused_data attribute."); -#define ZLIB_ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF \ - {"decompress", _PyCFunction_CAST(zlib_ZlibDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, zlib_ZlibDecompressor_decompress__doc__}, +#define ZLIB__ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF \ + {"decompress", _PyCFunction_CAST(zlib__ZlibDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, zlib__ZlibDecompressor_decompress__doc__}, static PyObject * -zlib_ZlibDecompressor_decompress_impl(ZlibDecompressor *self, - Py_buffer *data, Py_ssize_t max_length); +zlib__ZlibDecompressor_decompress_impl(ZlibDecompressor *self, + Py_buffer *data, + Py_ssize_t max_length); static PyObject * -zlib_ZlibDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +zlib__ZlibDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -984,7 +995,7 @@ zlib_ZlibDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssize max_length = ival; } skip_optional_pos: - return_value = zlib_ZlibDecompressor_decompress_impl((ZlibDecompressor *)self, &data, max_length); + return_value = zlib__ZlibDecompressor_decompress_impl((ZlibDecompressor *)self, &data, max_length); exit: /* Cleanup for data */ @@ -995,6 +1006,86 @@ exit: return return_value; } +PyDoc_STRVAR(zlib__ZlibDecompressor__doc__, +"_ZlibDecompressor(wbits=MAX_WBITS, zdict=b\'\')\n" +"--\n" +"\n" +"Create a decompressor object for decompressing data incrementally.\n" +"\n" +" zdict\n" +" The predefined compression dictionary. This is a sequence of bytes\n" +" (such as a bytes object) containing subsequences that are expected\n" +" to occur frequently in the data that is to be compressed. Those\n" +" subsequences that are expected to be most common should come at the\n" +" end of the dictionary. This must be the same dictionary as used by the\n" +" compressor that produced the input data."); + +static PyObject * +zlib__ZlibDecompressor_impl(PyTypeObject *type, int wbits, PyObject *zdict); + +static PyObject * +zlib__ZlibDecompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(wbits), &_Py_ID(zdict), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"wbits", "zdict", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_ZlibDecompressor", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + int wbits = MAX_WBITS; + PyObject *zdict = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + wbits = PyLong_AsInt(fastargs[0]); + if (wbits == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + zdict = fastargs[1]; +skip_optional_pos: + return_value = zlib__ZlibDecompressor_impl(type, wbits, zdict); + +exit: + return return_value; +} + PyDoc_STRVAR(zlib_adler32__doc__, "adler32($module, data, value=1, /)\n" "--\n" @@ -1028,9 +1119,21 @@ zlib_adler32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - value = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (value == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &value, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = zlib_adler32_impl(module, &data, value); @@ -1044,6 +1147,89 @@ exit: return return_value; } +PyDoc_STRVAR(zlib_adler32_combine__doc__, +"adler32_combine($module, adler1, adler2, len2, /)\n" +"--\n" +"\n" +"Combine two Adler-32 checksums into one.\n" +"\n" +" adler1\n" +" Adler-32 checksum for sequence A\n" +" adler2\n" +" Adler-32 checksum for sequence B\n" +" len2\n" +" Length of sequence B\n" +"\n" +"Given the Adler-32 checksum \'adler1\' of a sequence A and the\n" +"Adler-32 checksum \'adler2\' of a sequence B of length \'len2\',\n" +"return the Adler-32 checksum of A and B concatenated."); + +#define ZLIB_ADLER32_COMBINE_METHODDEF \ + {"adler32_combine", _PyCFunction_CAST(zlib_adler32_combine), METH_FASTCALL, zlib_adler32_combine__doc__}, + +static unsigned int +zlib_adler32_combine_impl(PyObject *module, unsigned int adler1, + unsigned int adler2, PyObject *len2); + +static PyObject * +zlib_adler32_combine(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int adler1; + unsigned int adler2; + PyObject *len2; + unsigned int _return_value; + + if (!_PyArg_CheckPositional("adler32_combine", nargs, 3, 3)) { + goto exit; + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[0], &adler1, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &adler2, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("adler32_combine", "argument 3", "int", args[2]); + goto exit; + } + len2 = args[2]; + _return_value = zlib_adler32_combine_impl(module, adler1, adler2, len2); + if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(zlib_crc32__doc__, "crc32($module, data, value=0, /)\n" "--\n" @@ -1078,9 +1264,21 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - value = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (value == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &value, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: _return_value = zlib_crc32_impl(module, &data, value); @@ -1098,6 +1296,89 @@ exit: return return_value; } +PyDoc_STRVAR(zlib_crc32_combine__doc__, +"crc32_combine($module, crc1, crc2, len2, /)\n" +"--\n" +"\n" +"Combine two CRC-32 checksums into one.\n" +"\n" +" crc1\n" +" CRC-32 checksum for sequence A\n" +" crc2\n" +" CRC-32 checksum for sequence B\n" +" len2\n" +" Length of sequence B\n" +"\n" +"Given the CRC-32 checksum \'crc1\' of a sequence A and the\n" +"CRC-32 checksum \'crc2\' of a sequence B of length \'len2\',\n" +"return the CRC-32 checksum of A and B concatenated."); + +#define ZLIB_CRC32_COMBINE_METHODDEF \ + {"crc32_combine", _PyCFunction_CAST(zlib_crc32_combine), METH_FASTCALL, zlib_crc32_combine__doc__}, + +static unsigned int +zlib_crc32_combine_impl(PyObject *module, unsigned int crc1, + unsigned int crc2, PyObject *len2); + +static PyObject * +zlib_crc32_combine(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int crc1; + unsigned int crc2; + PyObject *len2; + unsigned int _return_value; + + if (!_PyArg_CheckPositional("crc32_combine", nargs, 3, 3)) { + goto exit; + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[0], &crc1, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &crc2, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + if (!PyLong_Check(args[2])) { + _PyArg_BadArgument("crc32_combine", "argument 3", "int", args[2]); + goto exit; + } + len2 = args[2]; + _return_value = zlib_crc32_combine_impl(module, crc1, crc2, len2); + if ((_return_value == (unsigned int)-1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromUnsignedLong((unsigned long)_return_value); + +exit: + return return_value; +} + #ifndef ZLIB_COMPRESS_COPY_METHODDEF #define ZLIB_COMPRESS_COPY_METHODDEF #endif /* !defined(ZLIB_COMPRESS_COPY_METHODDEF) */ @@ -1121,4 +1402,4 @@ exit: #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=33938c7613a8c1c7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fa5fc356f3090cce input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 81cbf0d554d..65fbcf5cdaa 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -150,7 +150,7 @@ special_type(double d) #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_INFINITY +#define INF INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ @@ -163,8 +163,15 @@ special_type(double d) raised. */ -static Py_complex acos_special_values[7][7]; - +static Py_complex acos_special_values[7][7] = { + { {P34,INF}, {P,INF}, {P,INF}, {P,-INF}, {P,-INF}, {P34,-INF}, {N,INF} }, + { {P12,INF}, {U,U}, {U,U}, {U,U}, {U,U}, {P12,-INF}, {N,N} }, + { {P12,INF}, {U,U}, {P12,0.}, {P12,-0.}, {U,U}, {P12,-INF}, {P12,N} }, + { {P12,INF}, {U,U}, {P12,0.}, {P12,-0.}, {U,U}, {P12,-INF}, {P12,N} }, + { {P12,INF}, {U,U}, {U,U}, {U,U}, {U,U}, {P12,-INF}, {N,N} }, + { {P14,INF}, {0.,INF}, {0.,INF}, {0.,-INF}, {0.,-INF}, {P14,-INF}, {N,INF} }, + { {N,INF}, {N,N}, {N,N}, {N,N}, {N,N}, {N,-INF}, {N,N} } +}; /*[clinic input] cmath.acos -> Py_complex_protected @@ -202,7 +209,15 @@ cmath_acos_impl(PyObject *module, Py_complex z) } -static Py_complex acosh_special_values[7][7]; +static Py_complex acosh_special_values[7][7] = { + { {INF,-P34}, {INF,-P}, {INF,-P}, {INF,P}, {INF,P}, {INF,P34}, {INF,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {0.,-P12}, {0.,P12}, {U,U}, {INF,P12}, {N,P12} }, + { {INF,-P12}, {U,U}, {0.,-P12}, {0.,P12}, {U,U}, {INF,P12}, {N,P12} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,N}, {N,N} } +}; /*[clinic input] cmath.acosh = cmath.acos @@ -257,7 +272,15 @@ cmath_asin_impl(PyObject *module, Py_complex z) } -static Py_complex asinh_special_values[7][7]; +static Py_complex asinh_special_values[7][7] = { + { {-INF,-P14}, {-INF,-0.}, {-INF,-0.}, {-INF,0.}, {-INF,0.}, {-INF,P14}, {-INF,N} }, + { {-INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {-INF,P12}, {N,N} }, + { {-INF,-P12}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {INF,N}, {N,N} } +}; /*[clinic input] cmath.asinh = cmath.acos @@ -318,7 +341,15 @@ cmath_atan_impl(PyObject *module, Py_complex z) } -static Py_complex atanh_special_values[7][7]; +static Py_complex atanh_special_values[7][7] = { + { {-0.,-P12}, {-0.,-P12}, {-0.,-P12}, {-0.,P12}, {-0.,P12}, {-0.,P12}, {-0.,N} }, + { {-0.,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {-0.,P12}, {N,N} }, + { {-0.,-P12}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-0.,P12}, {-0.,N} }, + { {0.,-P12}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,P12}, {0.,N} }, + { {0.,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {0.,P12}, {N,N} }, + { {0.,-P12}, {0.,-P12}, {0.,-P12}, {0.,P12}, {0.,P12}, {0.,P12}, {0.,N} }, + { {0.,-P12}, {N,N}, {N,N}, {N,N}, {N,N}, {0.,P12}, {N,N} } +}; /*[clinic input] cmath.atanh = cmath.acos @@ -391,7 +422,15 @@ cmath_cos_impl(PyObject *module, Py_complex z) /* cosh(infinity + i*y) needs to be dealt with specially */ -static Py_complex cosh_special_values[7][7]; +static Py_complex cosh_special_values[7][7] = { + { {INF,N}, {U,U}, {INF,0.}, {INF,-0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {N,0.}, {U,U}, {1.,0.}, {1.,-0.}, {U,U}, {N,0.}, {N,0.} }, + { {N,0.}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,0.}, {N,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.cosh = cmath.acos @@ -453,7 +492,15 @@ cmath_cosh_impl(PyObject *module, Py_complex z) /* exp(infinity + i*y) and exp(-infinity + i*y) need special treatment for finite y */ -static Py_complex exp_special_values[7][7]; +static Py_complex exp_special_values[7][7] = { + { {0.,0.}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.exp = cmath.acos @@ -512,7 +559,15 @@ cmath_exp_impl(PyObject *module, Py_complex z) return r; } -static Py_complex log_special_values[7][7]; +static Py_complex log_special_values[7][7] = { + { {INF,-P34}, {INF,-P}, {INF,-P}, {INF,P}, {INF,P}, {INF,P34}, {INF,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {-INF,-P}, {-INF,P}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {-INF,-0.}, {-INF,0.}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,N}, {N,N} } +}; static Py_complex c_log(Py_complex z) @@ -628,7 +683,15 @@ cmath_sin_impl(PyObject *module, Py_complex z) /* sinh(infinity + i*y) needs to be dealt with specially */ -static Py_complex sinh_special_values[7][7]; +static Py_complex sinh_special_values[7][7] = { + { {INF,N}, {U,U}, {-INF,-0.}, {-INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {0.,N}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {0.,N}, {0.,N} }, + { {0.,N}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,N}, {0.,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.sinh = cmath.acos @@ -687,7 +750,15 @@ cmath_sinh_impl(PyObject *module, Py_complex z) } -static Py_complex sqrt_special_values[7][7]; +static Py_complex sqrt_special_values[7][7] = { + { {INF,-INF}, {0.,-INF}, {0.,-INF}, {0.,INF}, {0.,INF}, {INF,INF}, {N,INF} }, + { {INF,-INF}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,INF}, {INF,N} }, + { {INF,-INF}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,INF}, {N,N} } +}; /*[clinic input] cmath.sqrt = cmath.acos @@ -786,7 +857,15 @@ cmath_tan_impl(PyObject *module, Py_complex z) /* tanh(infinity + i*y) needs to be dealt with specially */ -static Py_complex tanh_special_values[7][7]; +static Py_complex tanh_special_values[7][7] = { + { {-1.,0.}, {U,U}, {-1.,-0.}, {-1.,0.}, {U,U}, {-1.,0.}, {-1.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {-0.0,N}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-0.0,N}, {-0.,N} }, + { {0.0,N}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.0,N}, {0.,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {1.,0.}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {1.,0.}, {1.,0.} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.tanh = cmath.acos @@ -969,7 +1048,15 @@ cmath_polar_impl(PyObject *module, Py_complex z) */ -static Py_complex rect_special_values[7][7]; +static Py_complex rect_special_values[7][7] = { + { {INF,N}, {U,U}, {-INF,0.}, {-INF,-0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {0.,0.}, {U,U}, {-0.,0.}, {-0.,-0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {0.,0.}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.rect @@ -1035,6 +1122,7 @@ cmath_rect_impl(PyObject *module, double r, double phi) } /*[clinic input] +@permit_long_summary cmath.isfinite = cmath.polar Return True if both the real and imaginary parts of z are finite, else False. @@ -1042,7 +1130,7 @@ Return True if both the real and imaginary parts of z are finite, else False. static PyObject * cmath_isfinite_impl(PyObject *module, Py_complex z) -/*[clinic end generated code: output=ac76611e2c774a36 input=848e7ee701895815]*/ +/*[clinic end generated code: output=ac76611e2c774a36 input=e224f5c36d94f5da]*/ { return PyBool_FromLong(isfinite(z.real) && isfinite(z.imag)); } @@ -1074,6 +1162,7 @@ cmath_isinf_impl(PyObject *module, Py_complex z) } /*[clinic input] +@permit_long_docstring_body cmath.isclose -> bool a: Py_complex @@ -1100,7 +1189,7 @@ not close to anything, even itself. inf and -inf are only close to themselves. static int cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, double rel_tol, double abs_tol) -/*[clinic end generated code: output=8a2486cc6e0014d1 input=df9636d7de1d4ac3]*/ +/*[clinic end generated code: output=8a2486cc6e0014d1 input=0d45feea7c626f47]*/ { double diff; @@ -1184,11 +1273,11 @@ cmath_exec(PyObject *mod) if (PyModule_Add(mod, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(mod, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(mod, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } - Py_complex infj = {0.0, Py_INFINITY}; + Py_complex infj = {0.0, INFINITY}; if (PyModule_Add(mod, "infj", PyComplex_FromCComplex(infj)) < 0) { return -1; } @@ -1200,120 +1289,6 @@ cmath_exec(PyObject *mod) return -1; } - /* initialize special value tables */ - -#define INIT_SPECIAL_VALUES(NAME, BODY) { Py_complex* p = (Py_complex*)NAME; BODY } -#define C(REAL, IMAG) p->real = REAL; p->imag = IMAG; ++p; - - INIT_SPECIAL_VALUES(acos_special_values, { - C(P34,INF) C(P,INF) C(P,INF) C(P,-INF) C(P,-INF) C(P34,-INF) C(N,INF) - C(P12,INF) C(U,U) C(U,U) C(U,U) C(U,U) C(P12,-INF) C(N,N) - C(P12,INF) C(U,U) C(P12,0.) C(P12,-0.) C(U,U) C(P12,-INF) C(P12,N) - C(P12,INF) C(U,U) C(P12,0.) C(P12,-0.) C(U,U) C(P12,-INF) C(P12,N) - C(P12,INF) C(U,U) C(U,U) C(U,U) C(U,U) C(P12,-INF) C(N,N) - C(P14,INF) C(0.,INF) C(0.,INF) C(0.,-INF) C(0.,-INF) C(P14,-INF) C(N,INF) - C(N,INF) C(N,N) C(N,N) C(N,N) C(N,N) C(N,-INF) C(N,N) - }) - - INIT_SPECIAL_VALUES(acosh_special_values, { - C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(asinh_special_values, { - C(-INF,-P14) C(-INF,-0.) C(-INF,-0.) C(-INF,0.) C(-INF,0.) C(-INF,P14) C(-INF,N) - C(-INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(-INF,P12) C(N,N) - C(-INF,-P12) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(atanh_special_values, { - C(-0.,-P12) C(-0.,-P12) C(-0.,-P12) C(-0.,P12) C(-0.,P12) C(-0.,P12) C(-0.,N) - C(-0.,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(-0.,P12) C(N,N) - C(-0.,-P12) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.,P12) C(-0.,N) - C(0.,-P12) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,P12) C(0.,N) - C(0.,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(0.,P12) C(N,N) - C(0.,-P12) C(0.,-P12) C(0.,-P12) C(0.,P12) C(0.,P12) C(0.,P12) C(0.,N) - C(0.,-P12) C(N,N) C(N,N) C(N,N) C(N,N) C(0.,P12) C(N,N) - }) - - INIT_SPECIAL_VALUES(cosh_special_values, { - C(INF,N) C(U,U) C(INF,0.) C(INF,-0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,0.) C(U,U) C(1.,0.) C(1.,-0.) C(U,U) C(N,0.) C(N,0.) - C(N,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,0.) C(N,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(exp_special_values, { - C(0.,0.) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,0.) C(0.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(log_special_values, { - C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(-INF,-P) C(-INF,P) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(-INF,-0.) C(-INF,0.) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(sinh_special_values, { - C(INF,N) C(U,U) C(-INF,-0.) C(-INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(0.,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(0.,N) C(0.,N) - C(0.,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,N) C(0.,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(sqrt_special_values, { - C(INF,-INF) C(0.,-INF) C(0.,-INF) C(0.,INF) C(0.,INF) C(INF,INF) C(N,INF) - C(INF,-INF) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,INF) C(INF,N) - C(INF,-INF) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,INF) C(N,N) - }) - - INIT_SPECIAL_VALUES(tanh_special_values, { - C(-1.,0.) C(U,U) C(-1.,-0.) C(-1.,0.) C(U,U) C(-1.,0.) C(-1.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(-0.0,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.0,N) C(-0.,N) - C(0.0,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.0,N) C(0.,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(1.,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(1.,0.) C(1.,0.) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(rect_special_values, { - C(INF,N) C(U,U) C(-INF,0.) C(-INF,-0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(0.,0.) C(U,U) C(-0.,0.) C(-0.,-0.) C(U,U) C(0.,0.) C(0.,0.) - C(0.,0.) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,0.) C(0.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) return 0; } diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 610e1ddc0e9..290dfeb0f6d 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -19,6 +19,7 @@ Copyright (c) 2023 Hanno Böck <hanno@gentoo.org> Copyright (c) 2023 Sony Corporation / Snild Dolkow <snild@sony.com> Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2025 Matthew Fernandez <matthew.fernandez@gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -42,21 +43,21 @@ */ #ifndef Expat_INCLUDED -#define Expat_INCLUDED 1 +# define Expat_INCLUDED 1 -#include <stdlib.h> -#include "expat_external.h" +# include <stdlib.h> +# include "expat_external.h" -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif +# endif struct XML_ParserStruct; typedef struct XML_ParserStruct *XML_Parser; typedef unsigned char XML_Bool; -#define XML_TRUE ((XML_Bool)1) -#define XML_FALSE ((XML_Bool)0) +# define XML_TRUE ((XML_Bool)1) +# define XML_FALSE ((XML_Bool)0) /* The XML_Status enum gives the possible return values for several API functions. The preprocessor #defines are included so this @@ -73,11 +74,11 @@ typedef unsigned char XML_Bool; */ enum XML_Status { XML_STATUS_ERROR = 0, -#define XML_STATUS_ERROR XML_STATUS_ERROR +# define XML_STATUS_ERROR XML_STATUS_ERROR XML_STATUS_OK = 1, -#define XML_STATUS_OK XML_STATUS_OK +# define XML_STATUS_OK XML_STATUS_OK XML_STATUS_SUSPENDED = 2 -#define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED +# define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED }; enum XML_Error { @@ -276,7 +277,7 @@ XML_ParserCreate_MM(const XML_Char *encoding, /* Prepare a parser object to be reused. This is particularly valuable when memory allocation overhead is disproportionately high, - such as when a large number of small documnents need to be parsed. + such as when a large number of small documents need to be parsed. All handlers are cleared from the parser, except for the unknownEncodingHandler. The parser's external state is re-initialized except for the values of ns and ns_triplets. @@ -680,7 +681,7 @@ XMLPARSEAPI(void) XML_SetUserData(XML_Parser parser, void *userData); /* Returns the last value set by XML_SetUserData or NULL. */ -#define XML_GetUserData(parser) (*(void **)(parser)) +# define XML_GetUserData(parser) (*(void **)(parser)) /* This is equivalent to supplying an encoding argument to XML_ParserCreate. On success XML_SetEncoding returns non-zero, @@ -752,7 +753,7 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); -#ifdef XML_ATTR_INFO +# ifdef XML_ATTR_INFO /* Source file byte offsets for the start and end of attribute names and values. The value indices are exclusive of surrounding quotes; thus in a UTF-8 source file an attribute value of "blah" will yield: @@ -773,7 +774,7 @@ typedef struct { */ XMLPARSEAPI(const XML_AttrInfo *) XML_GetAttributeInfo(XML_Parser parser); -#endif +# endif /* Parses some input. Returns XML_STATUS_ERROR if a fatal error is detected. The last call to XML_Parse must have isFinal true; len @@ -970,9 +971,9 @@ XMLPARSEAPI(const char *) XML_GetInputContext(XML_Parser parser, int *offset, int *size); /* For backwards compatibility with previous versions. */ -#define XML_GetErrorLineNumber XML_GetCurrentLineNumber -#define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber -#define XML_GetErrorByteIndex XML_GetCurrentByteIndex +# define XML_GetErrorLineNumber XML_GetCurrentLineNumber +# define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber +# define XML_GetErrorByteIndex XML_GetCurrentByteIndex /* Frees the content model passed to the element declaration handler */ XMLPARSEAPI(void) @@ -1032,7 +1033,10 @@ enum XML_FeatureEnum { XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, /* Added in Expat 2.6.0. */ - XML_FEATURE_GE + XML_FEATURE_GE, + /* Added in Expat 2.7.2. */ + XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT, /* Additional features must be added to the end of this enum. */ }; @@ -1045,7 +1049,7 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -#if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) +# if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) @@ -1057,7 +1061,17 @@ XML_SetBillionLaughsAttackProtectionMaximumAmplification( XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes); -#endif + +/* Added in Expat 2.7.2. */ +XMLPARSEAPI(XML_Bool) +XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +/* Added in Expat 2.7.2. */ +XMLPARSEAPI(XML_Bool) +XML_SetAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); +# endif /* Added in Expat 2.6.0. */ XMLPARSEAPI(XML_Bool) @@ -1066,12 +1080,12 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); /* Expat follows the semantic versioning convention. See https://semver.org */ -#define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 7 -#define XML_MICRO_VERSION 1 +# define XML_MAJOR_VERSION 2 +# define XML_MINOR_VERSION 7 +# define XML_MICRO_VERSION 3 -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not Expat_INCLUDED */ diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 567872b0983..0f01a05d0e9 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -38,8 +38,7 @@ */ #ifndef Expat_External_INCLUDED -#define Expat_External_INCLUDED 1 - +# define Expat_External_INCLUDED 1 /* Namespace external symbols to allow multiple libexpat version to co-exist. */ #include "pyexpatns.h" @@ -68,12 +67,12 @@ compiled with the cdecl calling convention as the default since system headers may assume the cdecl convention. */ -#ifndef XMLCALL -# if defined(_MSC_VER) -# define XMLCALL __cdecl -# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER) -# define XMLCALL __attribute__((cdecl)) -# else +# ifndef XMLCALL +# if defined(_MSC_VER) +# define XMLCALL __cdecl +# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER) +# define XMLCALL __attribute__((cdecl)) +# else /* For any platform which uses this definition and supports more than one calling convention, we need to extend this definition to declare the convention used on that platform, if it's possible to @@ -84,86 +83,87 @@ pre-processor and how to specify the same calling convention as the platform's malloc() implementation. */ -# define XMLCALL -# endif -#endif /* not defined XMLCALL */ +# define XMLCALL +# endif +# endif /* not defined XMLCALL */ -#if ! defined(XML_STATIC) && ! defined(XMLIMPORT) -# ifndef XML_BUILDING_EXPAT +# if ! defined(XML_STATIC) && ! defined(XMLIMPORT) +# ifndef XML_BUILDING_EXPAT /* using Expat from an application */ -# if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__) -# define XMLIMPORT __declspec(dllimport) +# if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) \ + && ! defined(__CYGWIN__) +# define XMLIMPORT __declspec(dllimport) +# endif + # endif +# endif /* not defined XML_STATIC */ +# ifndef XML_ENABLE_VISIBILITY +# define XML_ENABLE_VISIBILITY 0 # endif -#endif /* not defined XML_STATIC */ -#ifndef XML_ENABLE_VISIBILITY -# define XML_ENABLE_VISIBILITY 0 -#endif - -#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY -# define XMLIMPORT __attribute__((visibility("default"))) -#endif +# if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY +# define XMLIMPORT __attribute__((visibility("default"))) +# endif /* If we didn't define it above, define it away: */ -#ifndef XMLIMPORT -# define XMLIMPORT -#endif +# ifndef XMLIMPORT +# define XMLIMPORT +# endif -#if defined(__GNUC__) \ - && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) -# define XML_ATTR_MALLOC __attribute__((__malloc__)) -#else -# define XML_ATTR_MALLOC -#endif +# if defined(__GNUC__) \ + && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) +# define XML_ATTR_MALLOC __attribute__((__malloc__)) +# else +# define XML_ATTR_MALLOC +# endif -#if defined(__GNUC__) \ - && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) -# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) -#else -# define XML_ATTR_ALLOC_SIZE(x) -#endif +# if defined(__GNUC__) \ + && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) +# else +# define XML_ATTR_ALLOC_SIZE(x) +# endif -#define XMLPARSEAPI(type) XMLIMPORT type XMLCALL +# define XMLPARSEAPI(type) XMLIMPORT type XMLCALL -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif - -#ifdef XML_UNICODE_WCHAR_T -# ifndef XML_UNICODE -# define XML_UNICODE # endif -# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) -# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" -# endif -#endif -#ifdef XML_UNICODE /* Information is UTF-16 encoded. */ # ifdef XML_UNICODE_WCHAR_T +# ifndef XML_UNICODE +# define XML_UNICODE +# endif +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif +# endif + +# ifdef XML_UNICODE /* Information is UTF-16 encoded. */ +# ifdef XML_UNICODE_WCHAR_T typedef wchar_t XML_Char; typedef wchar_t XML_LChar; -# else +# else typedef unsigned short XML_Char; typedef char XML_LChar; -# endif /* XML_UNICODE_WCHAR_T */ -#else /* Information is UTF-8 encoded. */ +# endif /* XML_UNICODE_WCHAR_T */ +# else /* Information is UTF-8 encoded. */ typedef char XML_Char; typedef char XML_LChar; -#endif /* XML_UNICODE */ +# endif /* XML_UNICODE */ -#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ typedef long long XML_Index; typedef unsigned long long XML_Size; -#else +# else typedef long XML_Index; typedef unsigned long XML_Size; -#endif /* XML_LARGE_SIZE */ +# endif /* XML_LARGE_SIZE */ -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not Expat_External_INCLUDED */ diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 6bde6ae6b31..8f5edf48ef7 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -108,6 +108,7 @@ #endif #include <limits.h> // ULONG_MAX +#include <stddef.h> // size_t #if defined(_WIN32) \ && (! defined(__USE_MINGW_ANSI_STDIO) \ @@ -148,6 +149,16 @@ 100.0f #define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ 8388608 // 8 MiB, 2^23 + +#define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f +#define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \ + 67108864 // 64 MiB, 2^26 + +// NOTE: If function expat_alloc was user facing, EXPAT_MALLOC_ALIGNMENT would +// have to take sizeof(long double) into account +#define EXPAT_MALLOC_ALIGNMENT sizeof(long long) // largest parser (sub)member +#define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t)) + /* NOTE END */ #include "expat.h" // so we can use type XML_Parser below @@ -171,6 +182,9 @@ extern #endif XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c #if defined(XML_TESTING) +void *expat_malloc(XML_Parser parser, size_t size, int sourceLine); +void expat_free(XML_Parser parser, void *ptr, int sourceLine); +void *expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine); extern unsigned int g_bytesScanned; // used for testing only #endif diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index 8ee03ef0792..fc6b482d587 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -82,6 +82,8 @@ #define XmlPrologStateInit PyExpat_XmlPrologStateInit #define XmlPrologStateInitExternalEntity PyExpat_XmlPrologStateInitExternalEntity #define XML_ResumeParser PyExpat_XML_ResumeParser +#define XML_SetAllocTrackerActivationThreshold PyExpat_XML_SetAllocTrackerActivationThreshold +#define XML_SetAllocTrackerMaximumAmplification PyExpat_XML_SetAllocTrackerMaximumAmplification #define XML_SetAttlistDeclHandler PyExpat_XML_SetAttlistDeclHandler #define XML_SetBase PyExpat_XML_SetBase #define XML_SetBillionLaughsAttackProtectionActivationThreshold PyExpat_XML_SetBillionLaughsAttackProtectionActivationThreshold diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 3904fc8afd6..d1bf5d19afa 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_7_1" -expected_libexpat_version="2.7.1" -expected_libexpat_sha256="0cce2e6e69b327fc607b8ff264f4b66bdf71ead55a87ffd5f3143f535f15cfa2" +expected_libexpat_tag="R_2_7_3" +expected_libexpat_version="2.7.3" +expected_libexpat_sha256="821ac9710d2c073eaf13e1b1895a9c9aa66c1157a99635c639fbff65cdbdd732" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} @@ -52,7 +52,14 @@ done rm libexpat.tar.gz # Step 3: Add the namespacing include to expat_external.h -sed -i 's/#define Expat_External_INCLUDED 1/&\n\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h +sed -i 's/# define Expat_External_INCLUDED 1/&\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h + +if ! grep -q '#include "pyexpatns\.h"' expat_external.h; then + echo " +Error: namespacing include not found in expat_external.h; +This may be due to source changes and will require updating this script" >&2 + exit 1 +fi echo " Updated! next steps: diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 38a2d9657b6..a187a3a18f1 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* d19ae032c224863c1527ba44d228cc34b99192c3a4c5a27af1f4e054d45ee031 (2.7.1+) +/* 28bcd8b1ba7eb595d82822908257fd9c3589b4243e3c922d0369f35bfcd7b506 (2.7.3+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -41,6 +41,7 @@ Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild@sony.com> Copyright (c) 2024-2025 Berkay Eren Ürün <berkay.ueruen@siemens.com> Copyright (c) 2024 Hanno Böck <hanno@gentoo.org> + Copyright (c) 2025 Matthew Fernandez <matthew.fernandez@gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -97,7 +98,7 @@ #include <stddef.h> #include <string.h> /* memset(), memcpy() */ #include <assert.h> -#include <limits.h> /* UINT_MAX */ +#include <limits.h> /* INT_MAX, UINT_MAX */ #include <stdio.h> /* fprintf */ #include <stdlib.h> /* getenv, rand_s */ #include <stdint.h> /* uintptr_t */ @@ -234,7 +235,7 @@ typedef struct { unsigned char power; size_t size; size_t used; - const XML_Memory_Handling_Suite *mem; + XML_Parser parser; } HASH_TABLE; static size_t keylen(KEY s); @@ -357,7 +358,7 @@ typedef struct { const XML_Char *end; XML_Char *ptr; XML_Char *start; - const XML_Memory_Handling_Suite *mem; + XML_Parser parser; } STRING_POOL; /* The XML_Char before the name is used to determine whether @@ -452,6 +453,14 @@ typedef struct accounting { unsigned long long activationThresholdBytes; } ACCOUNTING; +typedef struct MALLOC_TRACKER { + XmlBigCount bytesAllocated; + XmlBigCount peakBytesAllocated; // updated live only for debug level >=2 + unsigned long debugLevel; + float maximumAmplificationFactor; // >=1.0 + XmlBigCount activationThresholdBytes; +} MALLOC_TRACKER; + typedef struct entity_stats { unsigned int countEverOpened; unsigned int currentDepth; @@ -555,27 +564,24 @@ static XML_Bool setContext(XML_Parser parser, const XML_Char *context); static void FASTCALL normalizePublicId(XML_Char *s); -static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms); +static DTD *dtdCreate(XML_Parser parser); /* do not call if m_parentParser != NULL */ -static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms); -static void dtdDestroy(DTD *p, XML_Bool isDocEntity, - const XML_Memory_Handling_Suite *ms); +static void dtdReset(DTD *p, XML_Parser parser); +static void dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser); static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, - const XML_Memory_Handling_Suite *ms); + XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *table, - const XML_Memory_Handling_Suite *ms); +static void FASTCALL hashTableInit(HASH_TABLE *table, XML_Parser parser); static void FASTCALL hashTableClear(HASH_TABLE *table); static void FASTCALL hashTableDestroy(HASH_TABLE *table); static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table); static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); -static void FASTCALL poolInit(STRING_POOL *pool, - const XML_Memory_Handling_Suite *ms); +static void FASTCALL poolInit(STRING_POOL *pool, XML_Parser parser); static void FASTCALL poolClear(STRING_POOL *pool); static void FASTCALL poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, @@ -595,15 +601,15 @@ static XML_Content *build_model(XML_Parser parser); static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); -static XML_Char *copyString(const XML_Char *s, - const XML_Memory_Handling_Suite *memsuite); +static XML_Char *copyString(const XML_Char *s, XML_Parser parser); static unsigned long generate_hash_secret_salt(XML_Parser parser); static XML_Bool startParsing(XML_Parser parser); static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, - const XML_Char *nameSep, DTD *dtd); + const XML_Char *nameSep, DTD *dtd, + XML_Parser parentParser); static void parserInit(XML_Parser parser, const XML_Char *encodingName); @@ -627,10 +633,10 @@ static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, int sourceLine); static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, int sourceLine); +#endif /* XML_GE == 1 */ static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff); -#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); @@ -773,14 +779,238 @@ struct XML_ParserStruct { unsigned long m_hash_secret_salt; #if XML_GE == 1 ACCOUNTING m_accounting; + MALLOC_TRACKER m_alloc_tracker; ENTITY_STATS m_entity_stats; #endif XML_Bool m_reenter; }; -#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) -#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s))) -#define FREE(parser, p) (parser->m_mem.free_fcn((p))) +#if XML_GE == 1 +# define MALLOC(parser, s) (expat_malloc((parser), (s), __LINE__)) +# define REALLOC(parser, p, s) (expat_realloc((parser), (p), (s), __LINE__)) +# define FREE(parser, p) (expat_free((parser), (p), __LINE__)) +#else +# define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) +# define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s))) +# define FREE(parser, p) (parser->m_mem.free_fcn((p))) +#endif + +#if XML_GE == 1 +static void +expat_heap_stat(XML_Parser rootParser, char operator, XmlBigCount absDiff, + XmlBigCount newTotal, XmlBigCount peakTotal, int sourceLine) { + // NOTE: This can be +infinity or -nan + const float amplification + = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; + fprintf( + stderr, + "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_ULL( + "10") " to " EXPAT_FMT_ULL("10") " (" EXPAT_FMT_ULL("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", + (void *)rootParser, rootParser->m_accounting.countBytesDirect, operator, + absDiff, newTotal, peakTotal, (double)amplification, sourceLine); +} + +static bool +expat_heap_increase_tolerable(XML_Parser rootParser, XmlBigCount increase, + int sourceLine) { + assert(rootParser != NULL); + assert(increase > 0); + + XmlBigCount newTotal = 0; + bool tolerable = true; + + // Detect integer overflow + if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated < increase) { + tolerable = false; + } else { + newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; + + if (newTotal >= rootParser->m_alloc_tracker.activationThresholdBytes) { + assert(newTotal > 0); + // NOTE: This can be +infinity when dividing by zero but not -nan + const float amplification + = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; + if (amplification + > rootParser->m_alloc_tracker.maximumAmplificationFactor) { + tolerable = false; + } + } + } + + if (! tolerable && (rootParser->m_alloc_tracker.debugLevel >= 1)) { + expat_heap_stat(rootParser, '+', increase, newTotal, newTotal, sourceLine); + } + + return tolerable; +} + +# if defined(XML_TESTING) +void * +# else +static void * +# endif +expat_malloc(XML_Parser parser, size_t size, int sourceLine) { + // Detect integer overflow + if (SIZE_MAX - size < sizeof(size_t) + EXPAT_MALLOC_PADDING) { + return NULL; + } + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + + const size_t bytesToAllocate = sizeof(size_t) + EXPAT_MALLOC_PADDING + size; + + if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated + < bytesToAllocate) { + return NULL; // i.e. signal integer overflow as out-of-memory + } + + if (! expat_heap_increase_tolerable(rootParser, bytesToAllocate, + sourceLine)) { + return NULL; // i.e. signal violation as out-of-memory + } + + // Actually allocate + void *const mallocedPtr = parser->m_mem.malloc_fcn(bytesToAllocate); + + if (mallocedPtr == NULL) { + return NULL; + } + + // Update in-block recorded size + *(size_t *)mallocedPtr = size; + + // Update accounting + rootParser->m_alloc_tracker.bytesAllocated += bytesToAllocate; + + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; + } + expat_heap_stat(rootParser, '+', bytesToAllocate, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); + } + + return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING; +} + +# if defined(XML_TESTING) +void +# else +static void +# endif +expat_free(XML_Parser parser, void *ptr, int sourceLine) { + assert(parser != NULL); + + if (ptr == NULL) { + return; + } + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + + // Extract size (to the eyes of malloc_fcn/realloc_fcn) and + // the original pointer returned by malloc/realloc + void *const mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t); + const size_t bytesAllocated + = sizeof(size_t) + EXPAT_MALLOC_PADDING + *(size_t *)mallocedPtr; + + // Update accounting + assert(rootParser->m_alloc_tracker.bytesAllocated >= bytesAllocated); + rootParser->m_alloc_tracker.bytesAllocated -= bytesAllocated; + + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + expat_heap_stat(rootParser, '-', bytesAllocated, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); + } + + // NOTE: This may be freeing rootParser, so freeing has to come last + parser->m_mem.free_fcn(mallocedPtr); +} + +# if defined(XML_TESTING) +void * +# else +static void * +# endif +expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { + assert(parser != NULL); + + if (ptr == NULL) { + return expat_malloc(parser, size, sourceLine); + } + + if (size == 0) { + expat_free(parser, ptr, sourceLine); + return NULL; + } + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + + // Extract original size (to the eyes of the caller) and the original + // pointer returned by malloc/realloc + void *mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t); + const size_t prevSize = *(size_t *)mallocedPtr; + + // Classify upcoming change + const bool isIncrease = (size > prevSize); + const size_t absDiff + = (size > prevSize) ? (size - prevSize) : (prevSize - size); + + // Ask for permission from accounting + if (isIncrease) { + if (! expat_heap_increase_tolerable(rootParser, absDiff, sourceLine)) { + return NULL; // i.e. signal violation as out-of-memory + } + } + + // NOTE: Integer overflow detection has already been done for us + // by expat_heap_increase_tolerable(..) above + assert(SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING >= size); + + // Actually allocate + mallocedPtr = parser->m_mem.realloc_fcn( + mallocedPtr, sizeof(size_t) + EXPAT_MALLOC_PADDING + size); + + if (mallocedPtr == NULL) { + return NULL; + } + + // Update accounting + if (isIncrease) { + assert((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated + >= absDiff); + rootParser->m_alloc_tracker.bytesAllocated += absDiff; + } else { // i.e. decrease + assert(rootParser->m_alloc_tracker.bytesAllocated >= absDiff); + rootParser->m_alloc_tracker.bytesAllocated -= absDiff; + } + + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; + } + expat_heap_stat(rootParser, isIncrease ? '+' : '-', absDiff, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); + } + + // Update in-block recorded size + *(size_t *)mallocedPtr = size; + + return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING; +} +#endif // XML_GE == 1 XML_Parser XMLCALL XML_ParserCreate(const XML_Char *encodingName) { @@ -821,11 +1051,14 @@ writeRandomBytes_getrandom_nonblock(void *target, size_t count) { void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); const size_t bytesToWrite = count - bytesWrittenTotal; + assert(bytesToWrite <= INT_MAX); + const int bytesWrittenMore = # if defined(HAVE_GETRANDOM) - getrandom(currentTarget, bytesToWrite, getrandomFlags); + (int)getrandom(currentTarget, bytesToWrite, getrandomFlags); # else - syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); + (int)syscall(SYS_getrandom, currentTarget, bytesToWrite, + getrandomFlags); # endif if (bytesWrittenMore > 0) { @@ -1012,9 +1245,10 @@ generate_hash_secret_salt(XML_Parser parser) { static unsigned long get_hash_secret_salt(XML_Parser parser) { - if (parser->m_parentParser != NULL) - return get_hash_secret_salt(parser->m_parentParser); - return parser->m_hash_secret_salt; + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + return rootParser->m_hash_secret_salt; } static enum XML_Error @@ -1100,19 +1334,43 @@ XML_Parser XMLCALL XML_ParserCreate_MM(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep) { - return parserCreate(encodingName, memsuite, nameSep, NULL); + return parserCreate(encodingName, memsuite, nameSep, NULL, NULL); } static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep, - DTD *dtd) { - XML_Parser parser; + DTD *dtd, XML_Parser parentParser) { + XML_Parser parser = NULL; + +#if XML_GE == 1 + const size_t increase + = sizeof(size_t) + EXPAT_MALLOC_PADDING + sizeof(struct XML_ParserStruct); + + if (parentParser != NULL) { + const XML_Parser rootParser = getRootParserOf(parentParser, NULL); + if (! expat_heap_increase_tolerable(rootParser, increase, __LINE__)) { + return NULL; + } + } +#else + UNUSED_P(parentParser); +#endif if (memsuite) { XML_Memory_Handling_Suite *mtemp; +#if XML_GE == 1 + void *const sizeAndParser + = memsuite->malloc_fcn(sizeof(size_t) + EXPAT_MALLOC_PADDING + + sizeof(struct XML_ParserStruct)); + if (sizeAndParser != NULL) { + *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct); + parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t) + + EXPAT_MALLOC_PADDING); +#else parser = memsuite->malloc_fcn(sizeof(struct XML_ParserStruct)); if (parser != NULL) { +#endif mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem); mtemp->malloc_fcn = memsuite->malloc_fcn; mtemp->realloc_fcn = memsuite->realloc_fcn; @@ -1120,39 +1378,86 @@ parserCreate(const XML_Char *encodingName, } } else { XML_Memory_Handling_Suite *mtemp; - parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct)); +#if XML_GE == 1 + void *const sizeAndParser = malloc(sizeof(size_t) + EXPAT_MALLOC_PADDING + + sizeof(struct XML_ParserStruct)); + if (sizeAndParser != NULL) { + *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct); + parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t) + + EXPAT_MALLOC_PADDING); +#else + parser = malloc(sizeof(struct XML_ParserStruct)); if (parser != NULL) { +#endif mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem); mtemp->malloc_fcn = malloc; mtemp->realloc_fcn = realloc; mtemp->free_fcn = free; } - } + } // cppcheck-suppress[memleak symbolName=sizeAndParser] // Cppcheck >=2.18.0 if (! parser) return parser; +#if XML_GE == 1 + // Initialize .m_alloc_tracker + memset(&parser->m_alloc_tracker, 0, sizeof(MALLOC_TRACKER)); + if (parentParser == NULL) { + parser->m_alloc_tracker.debugLevel + = getDebugLevel("EXPAT_MALLOC_DEBUG", 0u); + parser->m_alloc_tracker.maximumAmplificationFactor + = EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_alloc_tracker.activationThresholdBytes + = EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT; + + // NOTE: This initialization needs to come this early because these fields + // are read by allocation tracking code + parser->m_parentParser = NULL; + parser->m_accounting.countBytesDirect = 0; + } else { + parser->m_parentParser = parentParser; + } + + // Record XML_ParserStruct allocation we did a few lines up before + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= increase); + rootParser->m_alloc_tracker.bytesAllocated += increase; + + // Report on allocation + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; + } + + expat_heap_stat(rootParser, '+', increase, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, __LINE__); + } +#else + parser->m_parentParser = NULL; +#endif // XML_GE == 1 + parser->m_buffer = NULL; parser->m_bufferLim = NULL; parser->m_attsSize = INIT_ATTS_SIZE; - parser->m_atts - = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE)); + parser->m_atts = MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE)); if (parser->m_atts == NULL) { FREE(parser, parser); return NULL; } #ifdef XML_ATTR_INFO - parser->m_attInfo = (XML_AttrInfo *)MALLOC( - parser, parser->m_attsSize * sizeof(XML_AttrInfo)); + parser->m_attInfo = MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo)); if (parser->m_attInfo == NULL) { FREE(parser, parser->m_atts); FREE(parser, parser); return NULL; } #endif - parser->m_dataBuf - = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); + parser->m_dataBuf = MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); if (parser->m_dataBuf == NULL) { FREE(parser, parser->m_atts); #ifdef XML_ATTR_INFO @@ -1166,7 +1471,7 @@ parserCreate(const XML_Char *encodingName, if (dtd) parser->m_dtd = dtd; else { - parser->m_dtd = dtdCreate(&parser->m_mem); + parser->m_dtd = dtdCreate(parser); if (parser->m_dtd == NULL) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_atts); @@ -1200,8 +1505,8 @@ parserCreate(const XML_Char *encodingName, parser->m_protocolEncodingName = NULL; - poolInit(&parser->m_tempPool, &(parser->m_mem)); - poolInit(&parser->m_temp2Pool, &(parser->m_mem)); + poolInit(&parser->m_tempPool, parser); + poolInit(&parser->m_temp2Pool, parser); parserInit(parser, encodingName); if (encodingName && ! parser->m_protocolEncodingName) { @@ -1233,7 +1538,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_processor = prologInitProcessor; XmlPrologStateInit(&parser->m_prologState); if (encodingName != NULL) { - parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + parser->m_protocolEncodingName = copyString(encodingName, parser); } parser->m_curBase = NULL; XmlInitEncoding(&parser->m_initEncoding, &parser->m_encoding, 0); @@ -1295,7 +1600,6 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_unknownEncodingMem = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; - parser->m_parentParser = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; // Reentry can only be triggered inside m_processor calls parser->m_reenter = XML_FALSE; @@ -1385,7 +1689,7 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; parserInit(parser, encodingName); - dtdReset(parser->m_dtd, &parser->m_mem); + dtdReset(parser->m_dtd, parser); return XML_TRUE; } @@ -1421,7 +1725,7 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { parser->m_protocolEncodingName = NULL; else { /* Copy the new encoding name into allocated memory */ - parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + parser->m_protocolEncodingName = copyString(encodingName, parser); if (! parser->m_protocolEncodingName) return XML_STATUS_ERROR; } @@ -1530,9 +1834,10 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, */ if (parser->m_ns) { XML_Char tmp[2] = {parser->m_namespaceSeparator, 0}; - parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd); + parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd, oldParser); } else { - parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd); + parser + = parserCreate(encodingName, &parser->m_mem, NULL, newDtd, oldParser); } if (! parser) @@ -1576,7 +1881,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_prologState.inEntityValue = oldInEntityValue; if (context) { #endif /* XML_DTD */ - if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem) + if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, parser) || ! setContext(parser, context)) { XML_ParserFree(parser); return NULL; @@ -1688,14 +1993,16 @@ XML_ParserFree(XML_Parser parser) { #else if (parser->m_dtd) #endif /* XML_DTD */ - dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, - &parser->m_mem); - FREE(parser, (void *)parser->m_atts); + dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, parser); + FREE(parser, parser->m_atts); #ifdef XML_ATTR_INFO - FREE(parser, (void *)parser->m_attInfo); + FREE(parser, parser->m_attInfo); #endif FREE(parser, parser->m_groupConnector); - FREE(parser, parser->m_buffer); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); @@ -2014,12 +2321,14 @@ int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { if (parser == NULL) return 0; - if (parser->m_parentParser) - return XML_SetHashSalt(parser->m_parentParser, hash_salt); + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parserBusy(parser)) + if (parserBusy(rootParser)) return 0; - parser->m_hash_secret_salt = hash_salt; + rootParser->m_hash_secret_salt = hash_salt; return 1; } @@ -2287,7 +2596,9 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - newBuf = (char *)MALLOC(parser, bufferSize); + // NOTE: We are avoiding MALLOC(..) here to leave limiting + // the input size to the application using Expat. + newBuf = parser->m_mem.malloc_fcn(bufferSize); if (newBuf == 0) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; @@ -2298,7 +2609,10 @@ XML_GetBuffer(XML_Parser parser, int len) { memcpy(newBuf, &parser->m_bufferPtr[-keep], EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) + keep); - FREE(parser, parser->m_buffer); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); parser->m_buffer = newBuf; parser->m_bufferEnd = parser->m_buffer @@ -2314,7 +2628,10 @@ XML_GetBuffer(XML_Parser parser, int len) { if (parser->m_bufferPtr) { memcpy(newBuf, parser->m_bufferPtr, EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - FREE(parser, parser->m_buffer); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); parser->m_bufferEnd = newBuf + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); @@ -2492,28 +2809,43 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - if (parser != NULL) - FREE(parser, model); + if (parser == NULL) + return; + + // NOTE: We are avoiding FREE(..) here because the content model + // has been created using plain .malloc_fcn(..) rather than MALLOC(..). + parser->m_mem.free_fcn(model); } void *XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { if (parser == NULL) return NULL; - return MALLOC(parser, size); + + // NOTE: We are avoiding MALLOC(..) here to not include + // user allocations with allocation tracking and limiting. + return parser->m_mem.malloc_fcn(size); } void *XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { if (parser == NULL) return NULL; - return REALLOC(parser, ptr, size); + + // NOTE: We are avoiding REALLOC(..) here to not include + // user allocations with allocation tracking and limiting. + return parser->m_mem.realloc_fcn(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - if (parser != NULL) - FREE(parser, ptr); + if (parser == NULL) + return; + + // NOTE: We are avoiding FREE(..) here because XML_MemMalloc and + // XML_MemRealloc are not using MALLOC(..) and REALLOC(..) + // but plain .malloc_fcn(..) and .realloc_fcn(..), internally. + parser->m_mem.free_fcn(ptr); } void XMLCALL @@ -2713,6 +3045,13 @@ XML_GetFeatureList(void) { EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, /* Added in Expat 2.6.0. */ {XML_FEATURE_GE, XML_L("XML_GE"), 0}, + /* Added in Expat 2.7.2. */ + {XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_AT_MAX_AMP"), + (long int)EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_AT_ACT_THRES"), + (long int)EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT}, #endif {XML_FEATURE_END, NULL, 0}}; @@ -2741,6 +3080,29 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold( parser->m_accounting.activationThresholdBytes = activationThresholdBytes; return XML_TRUE; } + +XML_Bool XMLCALL +XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + if ((parser == NULL) || (parser->m_parentParser != NULL) + || isnan(maximumAmplificationFactor) + || (maximumAmplificationFactor < 1.0f)) { + return XML_FALSE; + } + parser->m_alloc_tracker.maximumAmplificationFactor + = maximumAmplificationFactor; + return XML_TRUE; +} + +XML_Bool XMLCALL +XML_SetAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if ((parser == NULL) || (parser->m_parentParser != NULL)) { + return XML_FALSE; + } + parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; + return XML_TRUE; +} #endif /* XML_GE == 1 */ XML_Bool XMLCALL @@ -2761,8 +3123,8 @@ static XML_Bool storeRawNames(XML_Parser parser) { TAG *tag = parser->m_tagStack; while (tag) { - int bufSize; - int nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); + size_t bufSize; + size_t nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); size_t rawNameLen; char *rawNameBuf = tag->buf + nameLen; /* Stop if already stored. Since m_tagStack is a stack, we can stop @@ -2779,9 +3141,9 @@ storeRawNames(XML_Parser parser) { /* Detect and prevent integer overflow. */ if (rawNameLen > (size_t)INT_MAX - nameLen) return XML_FALSE; - bufSize = nameLen + (int)rawNameLen; - if (bufSize > tag->bufEnd - tag->buf) { - char *temp = (char *)REALLOC(parser, tag->buf, bufSize); + bufSize = nameLen + rawNameLen; + if (bufSize > (size_t)(tag->bufEnd - tag->buf)) { + char *temp = REALLOC(parser, tag->buf, bufSize); if (temp == NULL) return XML_FALSE; /* if tag->name.str points to tag->buf (only when namespace @@ -3107,10 +3469,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, tag = parser->m_freeTagList; parser->m_freeTagList = parser->m_freeTagList->parent; } else { - tag = (TAG *)MALLOC(parser, sizeof(TAG)); + tag = MALLOC(parser, sizeof(TAG)); if (! tag) return XML_ERROR_NO_MEMORY; - tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE); + tag->buf = MALLOC(parser, INIT_TAG_BUF_SIZE); if (! tag->buf) { FREE(parser, tag); return XML_ERROR_NO_MEMORY; @@ -3143,7 +3505,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } bufSize = (int)(tag->bufEnd - tag->buf) << 1; { - char *temp = (char *)REALLOC(parser, tag->buf, bufSize); + char *temp = REALLOC(parser, tag->buf, bufSize); if (temp == NULL) return XML_ERROR_NO_MEMORY; tag->buf = temp; @@ -3522,8 +3884,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } #endif - temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, - parser->m_attsSize * sizeof(ATTRIBUTE)); + temp = REALLOC(parser, parser->m_atts, + parser->m_attsSize * sizeof(ATTRIBUTE)); if (temp == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; @@ -3541,8 +3903,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } # endif - temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, - parser->m_attsSize * sizeof(XML_AttrInfo)); + temp2 = REALLOC(parser, parser->m_attInfo, + parser->m_attsSize * sizeof(XML_AttrInfo)); if (temp2 == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; @@ -3677,7 +4039,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, and clear flags that say whether attributes were specified */ i = 0; if (nPrefixes) { - int j; /* hash table index */ + unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; /* Detect and prevent invalid shift */ @@ -3718,8 +4080,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } #endif - temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, - nsAttsSize * sizeof(NS_ATT)); + temp = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); if (! temp) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; @@ -3772,7 +4133,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (! b) return XML_ERROR_UNBOUND_PREFIX; - for (j = 0; j < b->uriLen; j++) { + for (j = 0; j < (unsigned int)b->uriLen; j++) { const XML_Char c = b->uri[j]; if (! poolAppendChar(&parser->m_tempPool, c)) return XML_ERROR_NO_MEMORY; @@ -3866,7 +4227,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, return XML_ERROR_NONE; prefixLen = 0; if (parser->m_ns_triplets && binding->prefix->name) { - for (; binding->prefix->name[prefixLen++];) + while (binding->prefix->name[prefixLen++]) ; /* prefixLen includes null terminator */ } tagNamePtr->localPart = localPart; @@ -3900,7 +4261,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } #endif - uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char)); + uri = MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char)); if (! uri) return XML_ERROR_NO_MEMORY; binding->uriAlloc = n + EXPAND_SPARE; @@ -4146,8 +4507,8 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, } #endif - XML_Char *temp = (XML_Char *)REALLOC( - parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); + XML_Char *temp + = REALLOC(parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (temp == NULL) return XML_ERROR_NO_MEMORY; b->uri = temp; @@ -4155,7 +4516,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, } parser->m_freeBindingList = b->nextTagBinding; } else { - b = (BINDING *)MALLOC(parser, sizeof(BINDING)); + b = MALLOC(parser, sizeof(BINDING)); if (! b) return XML_ERROR_NO_MEMORY; @@ -4173,8 +4534,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, } #endif - b->uri - = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); + b->uri = MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (! b->uri) { FREE(parser, b); return XML_ERROR_NO_MEMORY; @@ -5545,7 +5905,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; } - char *const new_connector = (char *)REALLOC( + char *const new_connector = REALLOC( parser, parser->m_groupConnector, parser->m_groupSize *= 2); if (new_connector == NULL) { parser->m_groupSize /= 2; @@ -5565,15 +5925,14 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } #endif - int *const new_scaff_index = (int *)REALLOC( + int *const new_scaff_index = REALLOC( parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int)); if (new_scaff_index == NULL) return XML_ERROR_NO_MEMORY; dtd->scaffIndex = new_scaff_index; } } else { - parser->m_groupConnector - = (char *)MALLOC(parser, parser->m_groupSize = 32); + parser->m_groupConnector = MALLOC(parser, parser->m_groupSize = 32); if (! parser->m_groupConnector) { parser->m_groupSize = 0; return XML_ERROR_NO_MEMORY; @@ -5730,8 +6089,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_CONTENT_EMPTY: if (dtd->in_eldecl) { if (parser->m_elementDeclHandler) { - XML_Content *content - = (XML_Content *)MALLOC(parser, sizeof(XML_Content)); + // NOTE: We are avoiding MALLOC(..) here to so that + // applications that are not using XML_FreeContentModel but + // plain free(..) or .free_fcn() to free the content model's + // memory are safe. + XML_Content *content = parser->m_mem.malloc_fcn(sizeof(XML_Content)); if (! content) return XML_ERROR_NO_MEMORY; content->quant = XML_CQUANT_NONE; @@ -5787,7 +6149,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, name = el->name; dtd->scaffold[myindex].name = name; nameLen = 0; - for (; name[nameLen++];) + while (name[nameLen++]) ; /* Detect and prevent integer overflow */ @@ -6008,8 +6370,7 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, openEntity = *freeEntityList; *freeEntityList = openEntity->next; } else { - openEntity - = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY)); + openEntity = MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY)); if (! openEntity) return XML_ERROR_NO_MEMORY; } @@ -6087,6 +6448,10 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // process its possible inner entities (which are added to the // m_openInternalEntities during doProlog or doContent calls above) entity->hasMore = XML_FALSE; + if (! entity->is_param + && (openEntity->startTagLevel != parser->m_tagLevel)) { + return XML_ERROR_ASYNC_ENTITY; + } triggerReenter(parser); return result; } // End of entity processing, "if" block will return here @@ -6277,7 +6642,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, case XML_TOK_ENTITY_REF: { const XML_Char *name; ENTITY *entity; - char checkEntityDecl; + bool checkEntityDecl; XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { @@ -6804,8 +7169,8 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (type->nDefaultAtts == type->allocDefaultAtts) { if (type->allocDefaultAtts == 0) { type->allocDefaultAtts = 8; - type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC( - parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); + type->defaultAtts + = MALLOC(parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! type->defaultAtts) { type->allocDefaultAtts = 0; return 0; @@ -6830,8 +7195,8 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, } #endif - temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts, - (count * sizeof(DEFAULT_ATTRIBUTE))); + temp = REALLOC(parser, type->defaultAtts, + (count * sizeof(DEFAULT_ATTRIBUTE))); if (temp == NULL) return 0; type->allocDefaultAtts = count; @@ -7122,19 +7487,19 @@ normalizePublicId(XML_Char *publicId) { } static DTD * -dtdCreate(const XML_Memory_Handling_Suite *ms) { - DTD *p = ms->malloc_fcn(sizeof(DTD)); +dtdCreate(XML_Parser parser) { + DTD *p = MALLOC(parser, sizeof(DTD)); if (p == NULL) return p; - poolInit(&(p->pool), ms); - poolInit(&(p->entityValuePool), ms); - hashTableInit(&(p->generalEntities), ms); - hashTableInit(&(p->elementTypes), ms); - hashTableInit(&(p->attributeIds), ms); - hashTableInit(&(p->prefixes), ms); + poolInit(&(p->pool), parser); + poolInit(&(p->entityValuePool), parser); + hashTableInit(&(p->generalEntities), parser); + hashTableInit(&(p->elementTypes), parser); + hashTableInit(&(p->attributeIds), parser); + hashTableInit(&(p->prefixes), parser); #ifdef XML_DTD p->paramEntityRead = XML_FALSE; - hashTableInit(&(p->paramEntities), ms); + hashTableInit(&(p->paramEntities), parser); #endif /* XML_DTD */ p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; @@ -7154,7 +7519,7 @@ dtdCreate(const XML_Memory_Handling_Suite *ms) { } static void -dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { +dtdReset(DTD *p, XML_Parser parser) { HASH_TABLE_ITER iter; hashTableIterInit(&iter, &(p->elementTypes)); for (;;) { @@ -7162,7 +7527,7 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { if (! e) break; if (e->allocDefaultAtts != 0) - ms->free_fcn(e->defaultAtts); + FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD @@ -7179,9 +7544,9 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { p->in_eldecl = XML_FALSE; - ms->free_fcn(p->scaffIndex); + FREE(parser, p->scaffIndex); p->scaffIndex = NULL; - ms->free_fcn(p->scaffold); + FREE(parser, p->scaffold); p->scaffold = NULL; p->scaffLevel = 0; @@ -7195,7 +7560,7 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { } static void -dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { +dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { HASH_TABLE_ITER iter; hashTableIterInit(&iter, &(p->elementTypes)); for (;;) { @@ -7203,7 +7568,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { if (! e) break; if (e->allocDefaultAtts != 0) - ms->free_fcn(e->defaultAtts); + FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); #ifdef XML_DTD @@ -7215,10 +7580,10 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { poolDestroy(&(p->pool)); poolDestroy(&(p->entityValuePool)); if (isDocEntity) { - ms->free_fcn(p->scaffIndex); - ms->free_fcn(p->scaffold); + FREE(parser, p->scaffIndex); + FREE(parser, p->scaffold); } - ms->free_fcn(p); + FREE(parser, p); } /* Do a deep copy of the DTD. Return 0 for out of memory, non-zero otherwise. @@ -7226,7 +7591,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { */ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, - const XML_Memory_Handling_Suite *ms) { + XML_Parser parser) { HASH_TABLE_ITER iter; /* Copy the prefix table. */ @@ -7307,7 +7672,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } #endif newE->defaultAtts - = ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); + = MALLOC(parser, oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { return 0; } @@ -7469,7 +7834,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { /* table->size is a power of 2 */ table->size = (size_t)1 << INIT_POWER; tsize = table->size * sizeof(NAMED *); - table->v = table->mem->malloc_fcn(tsize); + table->v = MALLOC(table->parser, tsize); if (! table->v) { table->size = 0; return NULL; @@ -7509,7 +7874,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } size_t tsize = newSize * sizeof(NAMED *); - NAMED **newV = table->mem->malloc_fcn(tsize); + NAMED **newV = MALLOC(table->parser, tsize); if (! newV) return NULL; memset(newV, 0, tsize); @@ -7525,7 +7890,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } newV[j] = table->v[i]; } - table->mem->free_fcn(table->v); + FREE(table->parser, table->v); table->v = newV; table->power = newPower; table->size = newSize; @@ -7538,7 +7903,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } - table->v[i] = table->mem->malloc_fcn(createSize); + table->v[i] = MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); @@ -7551,7 +7916,7 @@ static void FASTCALL hashTableClear(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) { - table->mem->free_fcn(table->v[i]); + FREE(table->parser, table->v[i]); table->v[i] = NULL; } table->used = 0; @@ -7561,17 +7926,17 @@ static void FASTCALL hashTableDestroy(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) - table->mem->free_fcn(table->v[i]); - table->mem->free_fcn(table->v); + FREE(table->parser, table->v[i]); + FREE(table->parser, table->v); } static void FASTCALL -hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) { +hashTableInit(HASH_TABLE *p, XML_Parser parser) { p->power = 0; p->size = 0; p->used = 0; p->v = NULL; - p->mem = ms; + p->parser = parser; } static void FASTCALL @@ -7591,13 +7956,13 @@ hashTableIterNext(HASH_TABLE_ITER *iter) { } static void FASTCALL -poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) { +poolInit(STRING_POOL *pool, XML_Parser parser) { pool->blocks = NULL; pool->freeBlocks = NULL; pool->start = NULL; pool->ptr = NULL; pool->end = NULL; - pool->mem = ms; + pool->parser = parser; } static void FASTCALL @@ -7624,13 +7989,13 @@ poolDestroy(STRING_POOL *pool) { BLOCK *p = pool->blocks; while (p) { BLOCK *tem = p->next; - pool->mem->free_fcn(p); + FREE(pool->parser, p); p = tem; } p = pool->freeBlocks; while (p) { BLOCK *tem = p->next; - pool->mem->free_fcn(p); + FREE(pool->parser, p); p = tem; } } @@ -7785,8 +8150,7 @@ poolGrow(STRING_POOL *pool) { if (bytesToAllocate == 0) return XML_FALSE; - temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks, - (unsigned)bytesToAllocate); + temp = REALLOC(pool->parser, pool->blocks, bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; @@ -7826,7 +8190,7 @@ poolGrow(STRING_POOL *pool) { if (bytesToAllocate == 0) return XML_FALSE; - tem = pool->mem->malloc_fcn(bytesToAllocate); + tem = MALLOC(pool->parser, bytesToAllocate); if (! tem) return XML_FALSE; tem->size = blockSize; @@ -7857,12 +8221,17 @@ nextScaffoldPart(XML_Parser parser) { return -1; } #endif - dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int)); + dtd->scaffIndex = MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; dtd->scaffIndex[0] = 0; } + // Will casting to int be safe further down? + if (dtd->scaffCount > INT_MAX) { + return -1; + } + if (dtd->scaffCount >= dtd->scaffSize) { CONTENT_SCAFFOLD *temp; if (dtd->scaffold) { @@ -7880,21 +8249,20 @@ nextScaffoldPart(XML_Parser parser) { } #endif - temp = (CONTENT_SCAFFOLD *)REALLOC( - parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD)); + temp = REALLOC(parser, dtd->scaffold, + dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD)); if (temp == NULL) return -1; dtd->scaffSize *= 2; } else { - temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS - * sizeof(CONTENT_SCAFFOLD)); + temp = MALLOC(parser, INIT_SCAFFOLD_ELEMENTS * sizeof(CONTENT_SCAFFOLD)); if (temp == NULL) return -1; dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS; } dtd->scaffold = temp; } - next = dtd->scaffCount++; + next = (int)dtd->scaffCount++; me = &dtd->scaffold[next]; if (dtd->scaffLevel) { CONTENT_SCAFFOLD *parent @@ -7941,7 +8309,10 @@ build_model(XML_Parser parser) { const size_t allocsize = (dtd->scaffCount * sizeof(XML_Content) + (dtd->contentStringLen * sizeof(XML_Char))); - ret = (XML_Content *)MALLOC(parser, allocsize); + // NOTE: We are avoiding MALLOC(..) here to so that + // applications that are not using XML_FreeContentModel but plain + // free(..) or .free_fcn() to free the content model's memory are safe. + ret = parser->m_mem.malloc_fcn(allocsize); if (! ret) return NULL; @@ -8062,7 +8433,7 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, } static XML_Char * -copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { +copyString(const XML_Char *s, XML_Parser parser) { size_t charsRequired = 0; XML_Char *result; @@ -8074,7 +8445,7 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { charsRequired++; /* Now allocate space for the copy */ - result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char)); + result = MALLOC(parser, charsRequired * sizeof(XML_Char)); if (result == NULL) return NULL; /* Copy the original into place */ @@ -8093,10 +8464,10 @@ accountingGetCurrentAmplification(XML_Parser rootParser) { + rootParser->m_accounting.countBytesIndirect; const float amplificationFactor = rootParser->m_accounting.countBytesDirect - ? (countBytesOutput + ? ((float)countBytesOutput / (float)(rootParser->m_accounting.countBytesDirect)) - : ((lenOfShortestInclude - + rootParser->m_accounting.countBytesIndirect) + : ((float)(lenOfShortestInclude + + rootParser->m_accounting.countBytesIndirect) / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; @@ -8280,6 +8651,8 @@ entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { rootParser->m_entity_stats.currentDepth--; } +#endif /* XML_GE == 1 */ + static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { XML_Parser rootParser = parser; @@ -8295,6 +8668,8 @@ getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { return rootParser; } +#if XML_GE == 1 + const char * unsignedCharToPrintable(unsigned char c) { switch (c) { diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index a7904274c91..9d0d4ff11b7 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper <coopercc@users.sourceforge.net> Copyright (c) 2002 Karl Waclawek <karl@waclawek.net> Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> - Copyright (c) 2017-2024 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2017-2025 Sebastian Pipping <sebastian@pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -34,19 +34,13 @@ */ #ifndef XmlRole_INCLUDED -#define XmlRole_INCLUDED 1 +# define XmlRole_INCLUDED 1 -#ifdef __VMS -/* 0 1 2 3 0 1 2 3 - 1234567890123456789012345678901 1234567890123456789012345678901 */ -# define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt -#endif +# include "xmltok.h" -#include "xmltok.h" - -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif +# endif enum { XML_ROLE_ERROR = -1, @@ -107,11 +101,11 @@ enum { XML_ROLE_CONTENT_ELEMENT_PLUS, XML_ROLE_PI, XML_ROLE_COMMENT, -#ifdef XML_DTD +# ifdef XML_DTD XML_ROLE_TEXT_DECL, XML_ROLE_IGNORE_SECT, XML_ROLE_INNER_PARAM_ENTITY_REF, -#endif /* XML_DTD */ +# endif /* XML_DTD */ XML_ROLE_PARAM_ENTITY_REF }; @@ -120,23 +114,23 @@ typedef struct prolog_state { const char *end, const ENCODING *enc); unsigned level; int role_none; -#ifdef XML_DTD +# ifdef XML_DTD unsigned includeLevel; int documentEntity; int inEntityValue; -#endif /* XML_DTD */ +# endif /* XML_DTD */ } PROLOG_STATE; void XmlPrologStateInit(PROLOG_STATE *state); -#ifdef XML_DTD +# ifdef XML_DTD void XmlPrologStateInitExternalEntity(PROLOG_STATE *state); -#endif /* XML_DTD */ +# endif /* XML_DTD */ -#define XmlTokenRole(state, tok, ptr, end, enc) \ - (((state)->handler)(state, tok, ptr, end, enc)) +# define XmlTokenRole(state, tok, ptr, end, enc) \ + (((state)->handler)(state, tok, ptr, end, enc)) -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not XmlRole_INCLUDED */ diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 29a66d72cee..95d5e84b67f 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -1398,7 +1398,7 @@ unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, } ENCODING * -XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert, +XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert, void *userData) { int i; struct unknown_encoding *e = (struct unknown_encoding *)mem; @@ -1661,7 +1661,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, # undef ns ENCODING * -XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, +XmlInitUnknownEncodingNS(void *mem, const int *table, CONVERTER convert, void *userData) { ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData); if (enc) diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index c51fce1ec15..79a9fb76871 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -35,113 +35,113 @@ */ #ifndef XmlTok_INCLUDED -#define XmlTok_INCLUDED 1 +# define XmlTok_INCLUDED 1 -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif +# endif /* The following token may be returned by XmlContentTok */ -#define XML_TOK_TRAILING_RSQB \ - -5 /* ] or ]] at the end of the scan; might be \ - start of illegal ]]> sequence */ +# define XML_TOK_TRAILING_RSQB \ + -5 /* ] or ]] at the end of the scan; might be \ + start of illegal ]]> sequence */ /* The following tokens may be returned by both XmlPrologTok and XmlContentTok. */ -#define XML_TOK_NONE -4 /* The string to be scanned is empty */ -#define XML_TOK_TRAILING_CR \ - -3 /* A CR at the end of the scan; \ - might be part of CRLF sequence */ -#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */ -#define XML_TOK_PARTIAL -1 /* only part of a token */ -#define XML_TOK_INVALID 0 +# define XML_TOK_NONE -4 /* The string to be scanned is empty */ +# define XML_TOK_TRAILING_CR \ + -3 /* A CR at the end of the scan; \ + might be part of CRLF sequence */ +# define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */ +# define XML_TOK_PARTIAL -1 /* only part of a token */ +# define XML_TOK_INVALID 0 /* The following tokens are returned by XmlContentTok; some are also returned by XmlAttributeValueTok, XmlEntityTok, XmlCdataSectionTok. */ -#define XML_TOK_START_TAG_WITH_ATTS 1 -#define XML_TOK_START_TAG_NO_ATTS 2 -#define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag <e/> */ -#define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4 -#define XML_TOK_END_TAG 5 -#define XML_TOK_DATA_CHARS 6 -#define XML_TOK_DATA_NEWLINE 7 -#define XML_TOK_CDATA_SECT_OPEN 8 -#define XML_TOK_ENTITY_REF 9 -#define XML_TOK_CHAR_REF 10 /* numeric character reference */ +# define XML_TOK_START_TAG_WITH_ATTS 1 +# define XML_TOK_START_TAG_NO_ATTS 2 +# define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag <e/> */ +# define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4 +# define XML_TOK_END_TAG 5 +# define XML_TOK_DATA_CHARS 6 +# define XML_TOK_DATA_NEWLINE 7 +# define XML_TOK_CDATA_SECT_OPEN 8 +# define XML_TOK_ENTITY_REF 9 +# define XML_TOK_CHAR_REF 10 /* numeric character reference */ /* The following tokens may be returned by both XmlPrologTok and XmlContentTok. */ -#define XML_TOK_PI 11 /* processing instruction */ -#define XML_TOK_XML_DECL 12 /* XML decl or text decl */ -#define XML_TOK_COMMENT 13 -#define XML_TOK_BOM 14 /* Byte order mark */ +# define XML_TOK_PI 11 /* processing instruction */ +# define XML_TOK_XML_DECL 12 /* XML decl or text decl */ +# define XML_TOK_COMMENT 13 +# define XML_TOK_BOM 14 /* Byte order mark */ /* The following tokens are returned only by XmlPrologTok */ -#define XML_TOK_PROLOG_S 15 -#define XML_TOK_DECL_OPEN 16 /* <!foo */ -#define XML_TOK_DECL_CLOSE 17 /* > */ -#define XML_TOK_NAME 18 -#define XML_TOK_NMTOKEN 19 -#define XML_TOK_POUND_NAME 20 /* #name */ -#define XML_TOK_OR 21 /* | */ -#define XML_TOK_PERCENT 22 -#define XML_TOK_OPEN_PAREN 23 -#define XML_TOK_CLOSE_PAREN 24 -#define XML_TOK_OPEN_BRACKET 25 -#define XML_TOK_CLOSE_BRACKET 26 -#define XML_TOK_LITERAL 27 -#define XML_TOK_PARAM_ENTITY_REF 28 -#define XML_TOK_INSTANCE_START 29 +# define XML_TOK_PROLOG_S 15 +# define XML_TOK_DECL_OPEN 16 /* <!foo */ +# define XML_TOK_DECL_CLOSE 17 /* > */ +# define XML_TOK_NAME 18 +# define XML_TOK_NMTOKEN 19 +# define XML_TOK_POUND_NAME 20 /* #name */ +# define XML_TOK_OR 21 /* | */ +# define XML_TOK_PERCENT 22 +# define XML_TOK_OPEN_PAREN 23 +# define XML_TOK_CLOSE_PAREN 24 +# define XML_TOK_OPEN_BRACKET 25 +# define XML_TOK_CLOSE_BRACKET 26 +# define XML_TOK_LITERAL 27 +# define XML_TOK_PARAM_ENTITY_REF 28 +# define XML_TOK_INSTANCE_START 29 /* The following occur only in element type declarations */ -#define XML_TOK_NAME_QUESTION 30 /* name? */ -#define XML_TOK_NAME_ASTERISK 31 /* name* */ -#define XML_TOK_NAME_PLUS 32 /* name+ */ -#define XML_TOK_COND_SECT_OPEN 33 /* <![ */ -#define XML_TOK_COND_SECT_CLOSE 34 /* ]]> */ -#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */ -#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */ -#define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */ -#define XML_TOK_COMMA 38 +# define XML_TOK_NAME_QUESTION 30 /* name? */ +# define XML_TOK_NAME_ASTERISK 31 /* name* */ +# define XML_TOK_NAME_PLUS 32 /* name+ */ +# define XML_TOK_COND_SECT_OPEN 33 /* <![ */ +# define XML_TOK_COND_SECT_CLOSE 34 /* ]]> */ +# define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */ +# define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */ +# define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */ +# define XML_TOK_COMMA 38 /* The following token is returned only by XmlAttributeValueTok */ -#define XML_TOK_ATTRIBUTE_VALUE_S 39 +# define XML_TOK_ATTRIBUTE_VALUE_S 39 /* The following token is returned only by XmlCdataSectionTok */ -#define XML_TOK_CDATA_SECT_CLOSE 40 +# define XML_TOK_CDATA_SECT_CLOSE 40 /* With namespace processing this is returned by XmlPrologTok for a name with a colon. */ -#define XML_TOK_PREFIXED_NAME 41 +# define XML_TOK_PREFIXED_NAME 41 -#ifdef XML_DTD -# define XML_TOK_IGNORE_SECT 42 -#endif /* XML_DTD */ +# ifdef XML_DTD +# define XML_TOK_IGNORE_SECT 42 +# endif /* XML_DTD */ -#ifdef XML_DTD -# define XML_N_STATES 4 -#else /* not XML_DTD */ -# define XML_N_STATES 3 -#endif /* not XML_DTD */ +# ifdef XML_DTD +# define XML_N_STATES 4 +# else /* not XML_DTD */ +# define XML_N_STATES 3 +# endif /* not XML_DTD */ -#define XML_PROLOG_STATE 0 -#define XML_CONTENT_STATE 1 -#define XML_CDATA_SECTION_STATE 2 -#ifdef XML_DTD -# define XML_IGNORE_SECTION_STATE 3 -#endif /* XML_DTD */ +# define XML_PROLOG_STATE 0 +# define XML_CONTENT_STATE 1 +# define XML_CDATA_SECTION_STATE 2 +# ifdef XML_DTD +# define XML_IGNORE_SECTION_STATE 3 +# endif /* XML_DTD */ -#define XML_N_LITERAL_TYPES 2 -#define XML_ATTRIBUTE_VALUE_LITERAL 0 -#define XML_ENTITY_VALUE_LITERAL 1 +# define XML_N_LITERAL_TYPES 2 +# define XML_ATTRIBUTE_VALUE_LITERAL 0 +# define XML_ENTITY_VALUE_LITERAL 1 /* The size of the buffer passed to XmlUtf8Encode must be at least this. */ -#define XML_UTF8_ENCODE_MAX 4 +# define XML_UTF8_ENCODE_MAX 4 /* The size of the buffer passed to XmlUtf16Encode must be at least this. */ -#define XML_UTF16_ENCODE_MAX 2 +# define XML_UTF16_ENCODE_MAX 2 typedef struct position { /* first line and first column are 0 not 1 */ @@ -220,63 +220,63 @@ struct encoding { the prolog outside literals, comments and processing instructions. */ -#define XmlTok(enc, state, ptr, end, nextTokPtr) \ - (((enc)->scanners[state])(enc, ptr, end, nextTokPtr)) +# define XmlTok(enc, state, ptr, end, nextTokPtr) \ + (((enc)->scanners[state])(enc, ptr, end, nextTokPtr)) -#define XmlPrologTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr) +# define XmlPrologTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr) -#define XmlContentTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr) +# define XmlContentTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr) -#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr) +# define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr) -#ifdef XML_DTD +# ifdef XML_DTD -# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr) +# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr) -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* This is used for performing a 2nd-level tokenization on the content of a literal that has already been returned by XmlTok. */ -#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \ - (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr)) +# define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \ + (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr)) -#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \ - XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr) +# define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \ + XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr) -#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \ - XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr) +# define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \ + XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr) -#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \ - (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2)) +# define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \ + (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2)) -#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr)) +# define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr)) -#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr)) +# define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr)) -#define XmlGetAttributes(enc, ptr, attsMax, atts) \ - (((enc)->getAtts)(enc, ptr, attsMax, atts)) +# define XmlGetAttributes(enc, ptr, attsMax, atts) \ + (((enc)->getAtts)(enc, ptr, attsMax, atts)) -#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr)) +# define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr)) -#define XmlPredefinedEntityName(enc, ptr, end) \ - (((enc)->predefinedEntityName)(enc, ptr, end)) +# define XmlPredefinedEntityName(enc, ptr, end) \ + (((enc)->predefinedEntityName)(enc, ptr, end)) -#define XmlUpdatePosition(enc, ptr, end, pos) \ - (((enc)->updatePosition)(enc, ptr, end, pos)) +# define XmlUpdatePosition(enc, ptr, end, pos) \ + (((enc)->updatePosition)(enc, ptr, end, pos)) -#define XmlIsPublicId(enc, ptr, end, badPtr) \ - (((enc)->isPublicId)(enc, ptr, end, badPtr)) +# define XmlIsPublicId(enc, ptr, end, badPtr) \ + (((enc)->isPublicId)(enc, ptr, end, badPtr)) -#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \ - (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim)) +# define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \ + (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim)) -#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \ - (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim)) +# define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \ + (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim)) typedef struct { ENCODING initEnc; @@ -299,7 +299,7 @@ int XmlSizeOfUnknownEncoding(void); typedef int(XMLCALL *CONVERTER)(void *userData, const char *p); -ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert, +ENCODING *XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert, void *userData); int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc, @@ -312,10 +312,10 @@ int XmlInitEncodingNS(INIT_ENCODING *p, const ENCODING **encPtr, const char *name); const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); -ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, - void *userData); -#ifdef __cplusplus +ENCODING *XmlInitUnknownEncodingNS(void *mem, const int *table, + CONVERTER convert, void *userData); +# ifdef __cplusplus } -#endif +# endif #endif /* not XmlTok_INCLUDED */ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index d49ce794d88..9b8c77e2b04 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -6,7 +6,6 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() #include "pycore_signal.h" // Py_NSIG -#include "pycore_sysmodule.h" // _PySys_GetRequiredAttr() #include "pycore_time.h" // _PyTime_FromSecondsObject() #include "pycore_traceback.h" // _Py_DumpTracebackThreads #ifdef HAVE_UNISTD_H @@ -31,6 +30,9 @@ #endif +#include "clinic/faulthandler.c.h" + + /* Sentinel to ignore all_threads on free-threading */ #define FT_IGNORE_ALL_THREADS 2 @@ -40,6 +42,12 @@ #define PUTS(fd, str) (void)_Py_write_noraise(fd, str, strlen(str)) +/*[clinic input] +module faulthandler +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c3d4f47c4f3d440f]*/ + + typedef struct { int signum; int enabled; @@ -98,7 +106,7 @@ faulthandler_get_fileno(PyObject **file_ptr) PyObject *file = *file_ptr; if (file == NULL || file == Py_None) { - file = _PySys_GetRequiredAttr(&_Py_ID(stderr)); + file = PySys_GetAttr(&_Py_ID(stderr)); if (file == NULL) { return -1; } @@ -229,22 +237,27 @@ faulthandler_dump_c_stack(int fd) reentrant = 0; } -static PyObject* -faulthandler_dump_traceback_py(PyObject *self, - PyObject *args, PyObject *kwargs) + +/*[clinic input] +faulthandler.dump_traceback as faulthandler_dump_traceback_py + + file: object(py_default="sys.stderr") = NULL + all_threads: bool = True + +Dump the traceback of the current thread into file. + +Dump the traceback of all threads if all_threads is true. +[clinic start generated code]*/ + +static PyObject * +faulthandler_dump_traceback_py_impl(PyObject *module, PyObject *file, + int all_threads) +/*[clinic end generated code: output=34efece0ca18314f input=b832ec55e27a7898]*/ { - static char *kwlist[] = {"file", "all_threads", NULL}; - PyObject *file = NULL; - int all_threads = 1; PyThreadState *tstate; const char *errmsg; int fd; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|Op:dump_traceback", kwlist, - &file, &all_threads)) - return NULL; - fd = faulthandler_get_fileno(&file); if (fd < 0) return NULL; @@ -279,19 +292,19 @@ faulthandler_dump_traceback_py(PyObject *self, Py_RETURN_NONE; } + +/*[clinic input] +faulthandler.dump_c_stack as faulthandler_dump_c_stack_py + + file: object(py_default="sys.stderr") = NULL + +Dump the C stack of the current thread. +[clinic start generated code]*/ + static PyObject * -faulthandler_dump_c_stack_py(PyObject *self, - PyObject *args, PyObject *kwargs) +faulthandler_dump_c_stack_py_impl(PyObject *module, PyObject *file) +/*[clinic end generated code: output=151d6c95e9f8c0f6 input=10f6b6f29b635109]*/ { - static char *kwlist[] = {"file", NULL}; - PyObject *file = NULL; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|O:dump_c_stack", kwlist, - &file)) { - return NULL; - } - int fd = faulthandler_get_fileno(&file); if (fd < 0) { return NULL; @@ -525,6 +538,11 @@ faulthandler_enable(void) } #endif + // gh-137185: Initialize C stack trace dumping outside of the signal + // handler. Specifically, we call backtrace() to ensure that libgcc is + // dynamically loaded outside of the signal handler. + _Py_InitDumpStack(); + for (size_t i=0; i < faulthandler_nsignals; i++) { fault_handler_t *handler; int err; @@ -565,19 +583,24 @@ faulthandler_enable(void) return 0; } -static PyObject* -faulthandler_py_enable(PyObject *self, PyObject *args, PyObject *kwargs) -{ - static char *kwlist[] = {"file", "all_threads", "c_stack", NULL}; - PyObject *file = NULL; - int all_threads = 1; - int fd; - int c_stack = 1; - PyThreadState *tstate; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|Opp:enable", kwlist, &file, &all_threads, &c_stack)) - return NULL; +/*[clinic input] +faulthandler.enable as faulthandler_py_enable + + file: object(py_default="sys.stderr") = NULL + all_threads: bool = True + c_stack: bool = True + +Enable the fault handler. +[clinic start generated code]*/ + +static PyObject * +faulthandler_py_enable_impl(PyObject *module, PyObject *file, + int all_threads, int c_stack) +/*[clinic end generated code: output=580d89b5eb62f1cb input=77277746a88b25ca]*/ +{ + int fd; + PyThreadState *tstate; fd = faulthandler_get_fileno(&file); if (fd < 0) @@ -622,8 +645,16 @@ faulthandler_disable(void) Py_CLEAR(fatal_error.file); } -static PyObject* -faulthandler_disable_py(PyObject *self, PyObject *Py_UNUSED(ignored)) + +/*[clinic input] +faulthandler.disable as faulthandler_disable_py + +Disable the fault handler. +[clinic start generated code]*/ + +static PyObject * +faulthandler_disable_py_impl(PyObject *module) +/*[clinic end generated code: output=e9087a04535af3cb input=6223eac6804550af]*/ { if (!fatal_error.enabled) { Py_RETURN_FALSE; @@ -632,10 +663,18 @@ faulthandler_disable_py(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_TRUE; } -static PyObject* -faulthandler_is_enabled(PyObject *self, PyObject *Py_UNUSED(ignored)) + +/*[clinic input] +faulthandler.is_enabled -> bool + +Check if the handler is enabled. +[clinic start generated code]*/ + +static int +faulthandler_is_enabled_impl(PyObject *module) +/*[clinic end generated code: output=b9f33a3e0f881a23 input=3d5532547eb14bf9]*/ { - return PyBool_FromLong(fatal_error.enabled); + return fatal_error.enabled; } static void @@ -730,26 +769,33 @@ format_timeout(PyTime_t us) return _PyMem_Strdup(buffer); } -static PyObject* -faulthandler_dump_traceback_later(PyObject *self, - PyObject *args, PyObject *kwargs) + +/*[clinic input] +faulthandler.dump_traceback_later + + timeout as timeout_obj: object + repeat: bool = False + file: object(py_default="sys.stderr") = NULL + exit: bool = False + +Dump the traceback of all threads in timeout seconds. + +If repeat is true, the tracebacks of all threads are dumped every timeout +seconds. If exit is true, call _exit(1) which is not safe. +[clinic start generated code]*/ + +static PyObject * +faulthandler_dump_traceback_later_impl(PyObject *module, + PyObject *timeout_obj, int repeat, + PyObject *file, int exit) +/*[clinic end generated code: output=a24d80d694d25ba2 input=fd005625ecc2ba9a]*/ { - static char *kwlist[] = {"timeout", "repeat", "file", "exit", NULL}; - PyObject *timeout_obj; PyTime_t timeout, timeout_us; - int repeat = 0; - PyObject *file = NULL; int fd; - int exit = 0; PyThreadState *tstate; char *header; size_t header_len; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "O|iOi:dump_traceback_later", kwlist, - &timeout_obj, &repeat, &file, &exit)) - return NULL; - if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { return NULL; @@ -832,9 +878,16 @@ faulthandler_dump_traceback_later(PyObject *self, Py_RETURN_NONE; } -static PyObject* -faulthandler_cancel_dump_traceback_later_py(PyObject *self, - PyObject *Py_UNUSED(ignored)) + +/*[clinic input] +faulthandler.cancel_dump_traceback_later as faulthandler_cancel_dump_traceback_later_py + +Cancel the previous call to dump_traceback_later(). +[clinic start generated code]*/ + +static PyObject * +faulthandler_cancel_dump_traceback_later_py_impl(PyObject *module) +/*[clinic end generated code: output=2cf303015d39c926 input=51ad64b6ca8412a4]*/ { cancel_dump_traceback_later(); Py_RETURN_NONE; @@ -934,26 +987,32 @@ check_signum(int signum) return 1; } -static PyObject* -faulthandler_register_py(PyObject *self, - PyObject *args, PyObject *kwargs) + +/*[clinic input] +faulthandler.register as faulthandler_register_py + + signum: int + file: object(py_default="sys.stderr") = NULL + all_threads: bool = True + chain: bool = False + +Register a handler for the signal 'signum'. + +Dump the traceback of the current thread, or of all threads if +all_threads is True, into file. +[clinic start generated code]*/ + +static PyObject * +faulthandler_register_py_impl(PyObject *module, int signum, PyObject *file, + int all_threads, int chain) +/*[clinic end generated code: output=1f770cee150a56cd input=ae9de829e850907b]*/ { - static char *kwlist[] = {"signum", "file", "all_threads", "chain", NULL}; - int signum; - PyObject *file = NULL; - int all_threads = 1; - int chain = 0; int fd; user_signal_t *user; _Py_sighandler_t previous; PyThreadState *tstate; int err; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "i|Opp:register", kwlist, - &signum, &file, &all_threads, &chain)) - return NULL; - if (!check_signum(signum)) return NULL; @@ -1018,16 +1077,23 @@ faulthandler_unregister(user_signal_t *user, int signum) return 1; } -static PyObject* -faulthandler_unregister_py(PyObject *self, PyObject *args) + +/*[clinic input] +faulthandler.unregister as faulthandler_unregister_py + + signum: int + / + +Unregister the handler of the signal 'signum' registered by register(). +[clinic start generated code]*/ + +static PyObject * +faulthandler_unregister_py_impl(PyObject *module, int signum) +/*[clinic end generated code: output=01734423da1155ed input=c016de014495d384]*/ { - int signum; user_signal_t *user; int change; - if (!PyArg_ParseTuple(args, "i:unregister", &signum)) - return NULL; - if (!check_signum(signum)) return NULL; @@ -1069,18 +1135,6 @@ faulthandler_suppress_crash_report(void) #endif } -static PyObject* _Py_NO_SANITIZE_UNDEFINED -faulthandler_read_null(PyObject *self, PyObject *args) -{ - volatile int *x; - volatile int y; - - faulthandler_suppress_crash_report(); - x = NULL; - y = *x; - return PyLong_FromLong(y); - -} static void faulthandler_raise_sigsegv(void) @@ -1105,13 +1159,20 @@ faulthandler_raise_sigsegv(void) #endif } -static PyObject * -faulthandler_sigsegv(PyObject *self, PyObject *args) -{ - int release_gil = 0; - if (!PyArg_ParseTuple(args, "|i:_sigsegv", &release_gil)) - return NULL; +/*[clinic input] +faulthandler._sigsegv + + release_gil: bool = False + / + +Raise a SIGSEGV signal. +[clinic start generated code]*/ + +static PyObject * +faulthandler__sigsegv_impl(PyObject *module, int release_gil) +/*[clinic end generated code: output=96e5a2f215b01b76 input=c6ad893cf2ea2b41]*/ +{ if (release_gil) { Py_BEGIN_ALLOW_THREADS faulthandler_raise_sigsegv(); @@ -1128,8 +1189,16 @@ faulthandler_fatal_error_thread(void *plock) Py_FatalError("in new thread"); } + +/*[clinic input] +faulthandler._fatal_error_c_thread + +Call Py_FatalError() in a new C thread. +[clinic start generated code]*/ + static PyObject * -faulthandler_fatal_error_c_thread(PyObject *self, PyObject *args) +faulthandler__fatal_error_c_thread_impl(PyObject *module) +/*[clinic end generated code: output=101bc8aaf4a5eec1 input=fbdca6fffd639a39]*/ { long tid; PyThread_type_lock lock; @@ -1158,27 +1227,32 @@ faulthandler_fatal_error_c_thread(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject* _Py_NO_SANITIZE_UNDEFINED -faulthandler_sigfpe(PyObject *self, PyObject *Py_UNUSED(dummy)) -{ - faulthandler_suppress_crash_report(); - /* Do an integer division by zero: raise a SIGFPE on Intel CPU, but not on - PowerPC. Use volatile to disable compile-time optimizations. */ - volatile int x = 1, y = 0, z; - z = x / y; +/*[clinic input] +faulthandler._sigfpe - /* If the division by zero didn't raise a SIGFPE (e.g. on PowerPC), - raise it manually. */ - raise(SIGFPE); - - /* This line is never reached, but we pretend to make something with z - to silence a compiler warning. */ - return PyLong_FromLong(z); -} +Raise a SIGFPE signal. +[clinic start generated code]*/ static PyObject * -faulthandler_sigabrt(PyObject *self, PyObject *args) +faulthandler__sigfpe_impl(PyObject *module) +/*[clinic end generated code: output=dec9c98100e986db input=fd608a92d4421d28]*/ +{ + faulthandler_suppress_crash_report(); + raise(SIGFPE); + Py_UNREACHABLE(); +} + + +/*[clinic input] +faulthandler._sigabrt + +Raise a SIGABRT signal. +[clinic start generated code]*/ + +static PyObject * +faulthandler__sigabrt_impl(PyObject *module) +/*[clinic end generated code: output=58c1378a0c166682 input=be3e0ecefb8676b8]*/ { faulthandler_suppress_crash_report(); abort(); @@ -1205,8 +1279,16 @@ stack_overflow(uintptr_t min_sp, uintptr_t max_sp, size_t *depth) return stack_overflow(min_sp, max_sp, depth); } + +/*[clinic input] +faulthandler._stack_overflow + +Recursive call to raise a stack overflow. +[clinic start generated code]*/ + static PyObject * -faulthandler_stack_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) +faulthandler__stack_overflow_impl(PyObject *module) +/*[clinic end generated code: output=efffba4be522d8fb input=4291594a790b6c35]*/ { size_t depth, size; uintptr_t sp = (uintptr_t)&depth; @@ -1257,13 +1339,23 @@ faulthandler_traverse(PyObject *module, visitproc visit, void *arg) return 0; } + #ifdef MS_WINDOWS +/*[clinic input] +faulthandler._raise_exception + + code: unsigned_int + flags: unsigned_int = 0 + / + +Call RaiseException(code, flags). +[clinic start generated code]*/ + static PyObject * -faulthandler_raise_exception(PyObject *self, PyObject *args) +faulthandler__raise_exception_impl(PyObject *module, unsigned int code, + unsigned int flags) +/*[clinic end generated code: output=2346cf318eab10dc input=43a5ba0eb7794504]*/ { - unsigned int code, flags = 0; - if (!PyArg_ParseTuple(args, "I|I:_raise_exception", &code, &flags)) - return NULL; faulthandler_suppress_crash_report(); RaiseException(code, flags, 0, NULL); Py_RETURN_NONE; @@ -1274,73 +1366,26 @@ PyDoc_STRVAR(module_doc, "faulthandler module."); static PyMethodDef module_methods[] = { - {"enable", - _PyCFunction_CAST(faulthandler_py_enable), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("enable($module, /, file=sys.stderr, all_threads=True)\n--\n\n" - "Enable the fault handler.")}, - {"disable", faulthandler_disable_py, METH_NOARGS, - PyDoc_STR("disable($module, /)\n--\n\n" - "Disable the fault handler.")}, - {"is_enabled", faulthandler_is_enabled, METH_NOARGS, - PyDoc_STR("is_enabled($module, /)\n--\n\n" - "Check if the handler is enabled.")}, - {"dump_traceback", - _PyCFunction_CAST(faulthandler_dump_traceback_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_traceback($module, /, file=sys.stderr, all_threads=True)\n--\n\n" - "Dump the traceback of the current thread, or of all threads " - "if all_threads is True, into file.")}, - {"dump_c_stack", - _PyCFunction_CAST(faulthandler_dump_c_stack_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_c_stack($module, /, file=sys.stderr)\n--\n\n" - "Dump the C stack of the current thread.")}, - {"dump_traceback_later", - _PyCFunction_CAST(faulthandler_dump_traceback_later), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("dump_traceback_later($module, /, timeout, repeat=False, file=sys.stderr, exit=False)\n--\n\n" - "Dump the traceback of all threads in timeout seconds,\n" - "or each timeout seconds if repeat is True. If exit is True, " - "call _exit(1) which is not safe.")}, - {"cancel_dump_traceback_later", - faulthandler_cancel_dump_traceback_later_py, METH_NOARGS, - PyDoc_STR("cancel_dump_traceback_later($module, /)\n--\n\n" - "Cancel the previous call to dump_traceback_later().")}, + FAULTHANDLER_PY_ENABLE_METHODDEF + FAULTHANDLER_DISABLE_PY_METHODDEF + FAULTHANDLER_IS_ENABLED_METHODDEF + FAULTHANDLER_DUMP_TRACEBACK_PY_METHODDEF + FAULTHANDLER_DUMP_C_STACK_PY_METHODDEF + FAULTHANDLER_DUMP_TRACEBACK_LATER_METHODDEF + FAULTHANDLER_CANCEL_DUMP_TRACEBACK_LATER_PY_METHODDEF #ifdef FAULTHANDLER_USER - {"register", - _PyCFunction_CAST(faulthandler_register_py), METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("register($module, /, signum, file=sys.stderr, all_threads=True, chain=False)\n--\n\n" - "Register a handler for the signal 'signum': dump the " - "traceback of the current thread, or of all threads if " - "all_threads is True, into file.")}, - {"unregister", - _PyCFunction_CAST(faulthandler_unregister_py), METH_VARARGS, - PyDoc_STR("unregister($module, signum, /)\n--\n\n" - "Unregister the handler of the signal " - "'signum' registered by register().")}, + FAULTHANDLER_REGISTER_PY_METHODDEF + FAULTHANDLER_UNREGISTER_PY_METHODDEF #endif - {"_read_null", faulthandler_read_null, METH_NOARGS, - PyDoc_STR("_read_null($module, /)\n--\n\n" - "Read from NULL, raise " - "a SIGSEGV or SIGBUS signal depending on the platform.")}, - {"_sigsegv", faulthandler_sigsegv, METH_VARARGS, - PyDoc_STR("_sigsegv($module, release_gil=False, /)\n--\n\n" - "Raise a SIGSEGV signal.")}, - {"_fatal_error_c_thread", faulthandler_fatal_error_c_thread, METH_NOARGS, - PyDoc_STR("_fatal_error_c_thread($module, /)\n--\n\n" - "Call Py_FatalError() in a new C thread.")}, - {"_sigabrt", faulthandler_sigabrt, METH_NOARGS, - PyDoc_STR("_sigabrt($module, /)\n--\n\n" - "Raise a SIGABRT signal.")}, - {"_sigfpe", faulthandler_sigfpe, METH_NOARGS, - PyDoc_STR("_sigfpe($module, /)\n--\n\n" - "Raise a SIGFPE signal.")}, + FAULTHANDLER__SIGSEGV_METHODDEF + FAULTHANDLER__FATAL_ERROR_C_THREAD_METHODDEF + FAULTHANDLER__SIGABRT_METHODDEF + FAULTHANDLER__SIGFPE_METHODDEF #ifdef FAULTHANDLER_STACK_OVERFLOW - {"_stack_overflow", faulthandler_stack_overflow, METH_NOARGS, - PyDoc_STR("_stack_overflow($module, /)\n--\n\n" - "Recursive call to raise a stack overflow.")}, + FAULTHANDLER__STACK_OVERFLOW_METHODDEF #endif #ifdef MS_WINDOWS - {"_raise_exception", faulthandler_raise_exception, METH_VARARGS, - PyDoc_STR("_raise_exception($module, code, flags=0, /)\n--\n\n" - "Call RaiseException(code, flags).")}, + FAULTHANDLER__RAISE_EXCEPTION_METHODDEF #endif {NULL, NULL} /* sentinel */ }; @@ -1350,26 +1395,26 @@ PyExec_faulthandler(PyObject *module) { /* Add constants for unit tests */ #ifdef MS_WINDOWS /* RaiseException() codes (prefixed by an underscore) */ - if (PyModule_AddIntConstant(module, "_EXCEPTION_ACCESS_VIOLATION", - EXCEPTION_ACCESS_VIOLATION)) { + if (PyModule_Add(module, "_EXCEPTION_ACCESS_VIOLATION", + PyLong_FromUnsignedLong(EXCEPTION_ACCESS_VIOLATION))) { return -1; } - if (PyModule_AddIntConstant(module, "_EXCEPTION_INT_DIVIDE_BY_ZERO", - EXCEPTION_INT_DIVIDE_BY_ZERO)) { + if (PyModule_Add(module, "_EXCEPTION_INT_DIVIDE_BY_ZERO", + PyLong_FromUnsignedLong(EXCEPTION_INT_DIVIDE_BY_ZERO))) { return -1; } - if (PyModule_AddIntConstant(module, "_EXCEPTION_STACK_OVERFLOW", - EXCEPTION_STACK_OVERFLOW)) { + if (PyModule_Add(module, "_EXCEPTION_STACK_OVERFLOW", + PyLong_FromUnsignedLong(EXCEPTION_STACK_OVERFLOW))) { return -1; } /* RaiseException() flags (prefixed by an underscore) */ - if (PyModule_AddIntConstant(module, "_EXCEPTION_NONCONTINUABLE", - EXCEPTION_NONCONTINUABLE)) { + if (PyModule_Add(module, "_EXCEPTION_NONCONTINUABLE", + PyLong_FromUnsignedLong(EXCEPTION_NONCONTINUABLE))) { return -1; } - if (PyModule_AddIntConstant(module, "_EXCEPTION_NONCONTINUABLE_EXCEPTION", - EXCEPTION_NONCONTINUABLE_EXCEPTION)) { + if (PyModule_Add(module, "_EXCEPTION_NONCONTINUABLE_EXCEPTION", + PyLong_FromUnsignedLong(EXCEPTION_NONCONTINUABLE_EXCEPTION))) { return -1; } #endif diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 220ee9ecdff..e373bf36881 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,9 +1,8 @@ /* fcntl module */ -// Need limited C API version 3.13 for PyLong_AsInt() -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 +// Argument Clinic uses the internal C API +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" @@ -41,22 +40,27 @@ fcntl.fcntl arg: object(c_default='NULL') = 0 / -Perform the operation `cmd` on file descriptor fd. +Perform the operation cmd on file descriptor fd. -The values used for `cmd` are operating system dependent, and are available -as constants in the fcntl module, using the same names as used in -the relevant C header files. The argument arg is optional, and -defaults to 0; it may be an int or a string. If arg is given as a string, -the return value of fcntl is a string of that length, containing the -resulting value put in the arg buffer by the operating system. The length -of the arg string is not allowed to exceed 1024 bytes. If the arg given -is an integer or if none is specified, the result value is an integer -corresponding to the return value of the fcntl call in the C code. +The values used for cmd are operating system dependent, and are +available as constants in the fcntl module, using the same names as used +in the relevant C header files. The argument arg is optional, and +defaults to 0; it may be an integer, a bytes-like object or a string. +If arg is given as a string, it will be encoded to binary using the +UTF-8 encoding. + +If the arg given is an integer or if none is specified, the result value +is an integer corresponding to the return value of the fcntl() call in +the C code. + +If arg is given as a bytes-like object, the return value of fcntl() is a +bytes object of that length, containing the resulting value put in the +arg buffer by the operating system. [clinic start generated code]*/ static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) -/*[clinic end generated code: output=888fc93b51c295bd input=7955340198e5f334]*/ +/*[clinic end generated code: output=888fc93b51c295bd input=77340720f11665da]*/ { int ret; int async_err = 0; @@ -93,29 +97,54 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) return NULL; } Py_ssize_t len = view.len; - if (len > FCNTL_BUFSZ) { - PyErr_SetString(PyExc_ValueError, - "fcntl argument 3 is too long"); + if (len <= FCNTL_BUFSZ) { + memcpy(buf, view.buf, len); + memcpy(buf + len, guard, GUARDSZ); PyBuffer_Release(&view); - return NULL; - } - memcpy(buf, view.buf, len); - memcpy(buf + len, guard, GUARDSZ); - PyBuffer_Release(&view); - do { - Py_BEGIN_ALLOW_THREADS - ret = fcntl(fd, code, buf); - Py_END_ALLOW_THREADS - } while (ret == -1 && errno == EINTR && !(async_err = PyErr_CheckSignals())); - if (ret < 0) { - return !async_err ? PyErr_SetFromErrno(PyExc_OSError) : NULL; + do { + Py_BEGIN_ALLOW_THREADS + ret = fcntl(fd, code, buf); + Py_END_ALLOW_THREADS + } while (ret == -1 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + if (ret < 0) { + return !async_err ? PyErr_SetFromErrno(PyExc_OSError) : NULL; + } + if (memcmp(buf + len, guard, GUARDSZ) != 0) { + PyErr_SetString(PyExc_SystemError, "buffer overflow"); + return NULL; + } + return PyBytes_FromStringAndSize(buf, len); } - if (memcmp(buf + len, guard, GUARDSZ) != 0) { - PyErr_SetString(PyExc_SystemError, "buffer overflow"); - return NULL; + else { + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { + PyBuffer_Release(&view); + return NULL; + } + char *ptr = PyBytesWriter_GetData(writer); + memcpy(ptr, view.buf, len); + PyBuffer_Release(&view); + + do { + Py_BEGIN_ALLOW_THREADS + ret = fcntl(fd, code, ptr); + Py_END_ALLOW_THREADS + } while (ret == -1 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + if (ret < 0) { + if (!async_err) { + PyErr_SetFromErrno(PyExc_OSError); + } + PyBytesWriter_Discard(writer); + return NULL; + } + if (ptr[len] != '\0') { + PyErr_SetString(PyExc_SystemError, "buffer overflow"); + PyBytesWriter_Discard(writer); + return NULL; + } + return PyBytesWriter_Finish(writer); } - return PyBytes_FromStringAndSize(buf, len); #undef FCNTL_BUFSZ } PyErr_Format(PyExc_TypeError, @@ -135,40 +164,39 @@ fcntl.ioctl mutate_flag as mutate_arg: bool = True / -Perform the operation `request` on file descriptor `fd`. +Perform the operation request on file descriptor fd. -The values used for `request` are operating system dependent, and are available -as constants in the fcntl or termios library modules, using the same names as -used in the relevant C header files. +The values used for request are operating system dependent, and are +available as constants in the fcntl or termios library modules, using +the same names as used in the relevant C header files. -The argument `arg` is optional, and defaults to 0; it may be an int or a -buffer containing character data (most likely a string or an array). +The argument arg is optional, and defaults to 0; it may be an integer, a +bytes-like object or a string. If arg is given as a string, it will be +encoded to binary using the UTF-8 encoding. -If the argument is a mutable buffer (such as an array) and if the -mutate_flag argument (which is only allowed in this case) is true then the -buffer is (in effect) passed to the operating system and changes made by -the OS will be reflected in the contents of the buffer after the call has -returned. The return value is the integer returned by the ioctl system -call. +If the arg given is an integer or if none is specified, the result value +is an integer corresponding to the return value of the ioctl() call in +the C code. -If the argument is a mutable buffer and the mutable_flag argument is false, -the behavior is as if a string had been passed. +If the argument is a mutable buffer (such as a bytearray) and the +mutate_flag argument is true (default) then the buffer is (in effect) +passed to the operating system and changes made by the OS will be +reflected in the contents of the buffer after the call has returned. +The return value is the integer returned by the ioctl() system call. -If the argument is an immutable buffer (most likely a string) then a copy -of the buffer is passed to the operating system and the return value is a -string of the same length containing whatever the operating system put in -the buffer. The length of the arg buffer in this case is not allowed to -exceed 1024 bytes. +If the argument is a mutable buffer and the mutable_flag argument is +false, the behavior is as if an immutable buffer had been passed. -If the arg given is an integer or if none is specified, the result value is -an integer corresponding to the return value of the ioctl call in the C -code. +If the argument is an immutable buffer then a copy of the buffer is +passed to the operating system and the return value is a bytes object of +the same length containing whatever the operating system put in the +buffer. [clinic start generated code]*/ static PyObject * fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, int mutate_arg) -/*[clinic end generated code: output=f72baba2454d7a62 input=9c6cca5e2c339622]*/ +/*[clinic end generated code: output=f72baba2454d7a62 input=954fe75c208cc492]*/ { /* We use the unsigned non-checked 'I' format for the 'code' parameter because the system expects it to be a 32bit bit field value @@ -251,29 +279,54 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, return NULL; } Py_ssize_t len = view.len; - if (len > IOCTL_BUFSZ) { - PyErr_SetString(PyExc_ValueError, - "ioctl argument 3 is too long"); + if (len <= IOCTL_BUFSZ) { + memcpy(buf, view.buf, len); + memcpy(buf + len, guard, GUARDSZ); PyBuffer_Release(&view); - return NULL; - } - memcpy(buf, view.buf, len); - memcpy(buf + len, guard, GUARDSZ); - PyBuffer_Release(&view); - do { - Py_BEGIN_ALLOW_THREADS - ret = ioctl(fd, code, buf); - Py_END_ALLOW_THREADS - } while (ret == -1 && errno == EINTR && !(async_err = PyErr_CheckSignals())); - if (ret < 0) { - return !async_err ? PyErr_SetFromErrno(PyExc_OSError) : NULL; + do { + Py_BEGIN_ALLOW_THREADS + ret = ioctl(fd, code, buf); + Py_END_ALLOW_THREADS + } while (ret == -1 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + if (ret < 0) { + return !async_err ? PyErr_SetFromErrno(PyExc_OSError) : NULL; + } + if (memcmp(buf + len, guard, GUARDSZ) != 0) { + PyErr_SetString(PyExc_SystemError, "buffer overflow"); + return NULL; + } + return PyBytes_FromStringAndSize(buf, len); } - if (memcmp(buf + len, guard, GUARDSZ) != 0) { - PyErr_SetString(PyExc_SystemError, "buffer overflow"); - return NULL; + else { + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { + PyBuffer_Release(&view); + return NULL; + } + char *ptr = PyBytesWriter_GetData(writer); + memcpy(ptr, view.buf, len); + PyBuffer_Release(&view); + + do { + Py_BEGIN_ALLOW_THREADS + ret = ioctl(fd, code, ptr); + Py_END_ALLOW_THREADS + } while (ret == -1 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + if (ret < 0) { + if (!async_err) { + PyErr_SetFromErrno(PyExc_OSError); + } + PyBytesWriter_Discard(writer); + return NULL; + } + if (ptr[len] != '\0') { + PyErr_SetString(PyExc_SystemError, "buffer overflow"); + PyBytesWriter_Discard(writer); + return NULL; + } + return PyBytesWriter_Finish(writer); } - return PyBytes_FromStringAndSize(buf, len); #undef IOCTL_BUFSZ } PyErr_Format(PyExc_TypeError, @@ -290,7 +343,7 @@ fcntl.flock operation as code: int / -Perform the lock operation `operation` on file descriptor `fd`. +Perform the lock operation on file descriptor fd. See the Unix manual page for flock(2) for details (On some systems, this function is emulated using fcntl()). @@ -298,7 +351,7 @@ function is emulated using fcntl()). static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code) -/*[clinic end generated code: output=84059e2b37d2fc64 input=0bfc00f795953452]*/ +/*[clinic end generated code: output=84059e2b37d2fc64 input=ade68943e8599f0a]*/ { int ret; int async_err = 0; @@ -361,22 +414,22 @@ fcntl.lockf A wrapper around the fcntl() locking calls. -`fd` is the file descriptor of the file to lock or unlock, and operation is one -of the following values: +fd is the file descriptor of the file to lock or unlock, and operation +is one of the following values: LOCK_UN - unlock LOCK_SH - acquire a shared lock LOCK_EX - acquire an exclusive lock When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with -LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the -lock cannot be acquired, an OSError will be raised and the exception will -have an errno attribute set to EACCES or EAGAIN (depending on the operating -system -- for portability, check for either value). +LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and +the lock cannot be acquired, an OSError will be raised and the exception +will have an errno attribute set to EACCES or EAGAIN (depending on the +operating system -- for portability, check for either value). -`len` is the number of bytes to lock, with the default meaning to lock to -EOF. `start` is the byte offset, relative to `whence`, to that the lock -starts. `whence` is as with fileobj.seek(), specifically: +len is the number of bytes to lock, with the default meaning to lock to +EOF. start is the byte offset, relative to whence, to that the lock +starts. whence is as with fileobj.seek(), specifically: 0 - relative to the start of the file (SEEK_SET) 1 - relative to the current buffer position (SEEK_CUR) @@ -386,7 +439,7 @@ starts. `whence` is as with fileobj.seek(), specifically: static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, PyObject *startobj, int whence) -/*[clinic end generated code: output=4985e7a172e7461a input=5480479fc63a04b8]*/ +/*[clinic end generated code: output=4985e7a172e7461a input=369bef4d7a1c5ff4]*/ { int ret; int async_err = 0; diff --git a/Modules/gc_weakref.txt b/Modules/gc_weakref.txt index f53fb99dd6c..c3b8cc743cc 100644 --- a/Modules/gc_weakref.txt +++ b/Modules/gc_weakref.txt @@ -1,6 +1,16 @@ Intro ===== +************************************************************************** +Note: this document was written long ago, before PEP 442 (safe object +finalization) was implemented. While that has changed some things, this +document is still mostly accurate. Just note that the rules being discussed +here apply to the unreachable set of objects *after* non-legacy finalizers +have been called. Also, the clearing of weakrefs has been changed to happen +later in the collection (after running finalizers but before tp_clear is +called). +************************************************************************** + The basic rule for dealing with weakref callbacks (and __del__ methods too, for that matter) during cyclic gc: diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index ad13496b06d..4c286f5c12c 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -8,7 +8,6 @@ #include "pycore_gc.h" #include "pycore_object.h" // _PyObject_IS_GC() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_tuple.h" // _PyTuple_FromArray() typedef struct _gc_runtime_state GCState; @@ -296,6 +295,7 @@ gc_get_referents_impl(PyObject *module, PyObject *objs) } /*[clinic input] +@permit_long_summary gc.get_objects generation: Py_ssize_t(accept={int, NoneType}, c_default="-1") = None Generation to extract the objects from. @@ -308,7 +308,7 @@ that are in that generation. static PyObject * gc_get_objects_impl(PyObject *module, Py_ssize_t generation) -/*[clinic end generated code: output=48b35fea4ba6cb0e input=ef7da9df9806754c]*/ +/*[clinic end generated code: output=48b35fea4ba6cb0e input=a887f1d9924be7cf]*/ { if (PySys_Audit("gc.get_objects", "n", generation) < 0) { return NULL; @@ -358,10 +358,12 @@ gc_get_stats_impl(PyObject *module) for (i = 0; i < NUM_GENERATIONS; i++) { PyObject *dict; st = &stats[i]; - dict = Py_BuildValue("{snsnsn}", + dict = Py_BuildValue("{snsnsnsnsd}", "collections", st->collections, "collected", st->collected, - "uncollectable", st->uncollectable + "uncollectable", st->uncollectable, + "candidates", st->candidates, + "duration", st->duration ); if (dict == NULL) goto error; @@ -414,6 +416,7 @@ gc_is_finalized_impl(PyObject *module, PyObject *obj) } /*[clinic input] +@permit_long_docstring_body gc.freeze Freeze all current tracked objects and ignore them for future collections. @@ -425,7 +428,7 @@ which can cause copy-on-write. static PyObject * gc_freeze_impl(PyObject *module) -/*[clinic end generated code: output=502159d9cdc4c139 input=b602b16ac5febbe5]*/ +/*[clinic end generated code: output=502159d9cdc4c139 input=11fb59b0a75dcf3d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyGC_Freeze(interp); @@ -476,7 +479,7 @@ PyDoc_STRVAR(gc__doc__, "set_debug() -- Set debugging flags.\n" "get_debug() -- Get debugging flags.\n" "set_threshold() -- Set the collection thresholds.\n" -"get_threshold() -- Return the current the collection thresholds.\n" +"get_threshold() -- Return the current collection thresholds.\n" "get_objects() -- Return a list of all objects tracked by the collector.\n" "is_tracked() -- Returns true if a given object is tracked.\n" "is_finalized() -- Returns true if a given object has been already finalized.\n" diff --git a/Modules/getpath.py b/Modules/getpath.py index be2210345af..ceb605a75c8 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -364,10 +364,9 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None pyvenvcfg = [] - # Search for the 'home' key in pyvenv.cfg. Currently, we don't consider the - # presence of a pyvenv.cfg file without a 'home' key to signify the - # existence of a virtual environment — we quietly ignore them. - # XXX: If we don't find a 'home' key, we don't look for another pyvenv.cfg! + # Search for the 'home' key in pyvenv.cfg. If a home key isn't found, + # then it means a venv is active and home is based on the venv's + # executable (if its a symlink, home is where the symlink points). for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': @@ -412,10 +411,8 @@ def search_up(prefix, *landmarks, test=isfile): if isfile(candidate): base_executable = candidate break + # home key found; stop iterating over lines break - else: - # We didn't find a 'home' key in pyvenv.cfg (no break), reset venv_prefix. - venv_prefix = None # ****************************************************************************** @@ -793,6 +790,7 @@ def search_up(prefix, *landmarks, test=isfile): config['isolated'] = 1 config['use_environment'] = 0 config['site_import'] = 0 + config['user_site_directory'] = 0 config['safe_path'] = 1 pythonpath = [] for line in pth: diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 29da9936b65..652958618a2 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -55,6 +55,11 @@ get_grp_state(PyObject *module) static struct PyModuleDef grpmodule; +/* Mutex to protect calls to getgrgid(), getgrnam(), and getgrent(). + * These functions return pointer to static data structure, which + * may be overwritten by any subsequent calls. */ +static PyMutex group_db_mutex = {0}; + #define DEFAULT_BUFFER_SIZE 1024 static PyObject * @@ -168,9 +173,15 @@ grp_getgrgid_impl(PyObject *module, PyObject *id) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&group_db_mutex); + // The getgrgid() function need not be thread-safe. + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/getgrgid.html p = getgrgid(gid); #endif if (p == NULL) { +#ifndef HAVE_GETGRGID_R + PyMutex_Unlock(&group_db_mutex); +#endif PyMem_RawFree(buf); if (nomem == 1) { return PyErr_NoMemory(); @@ -185,6 +196,8 @@ grp_getgrgid_impl(PyObject *module, PyObject *id) retval = mkgrent(module, p); #ifdef HAVE_GETGRGID_R PyMem_RawFree(buf); +#else + PyMutex_Unlock(&group_db_mutex); #endif return retval; } @@ -249,9 +262,15 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&group_db_mutex); + // The getgrnam() function need not be thread-safe. + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/getgrnam.html p = getgrnam(name_chars); #endif if (p == NULL) { +#ifndef HAVE_GETGRNAM_R + PyMutex_Unlock(&group_db_mutex); +#endif if (nomem == 1) { PyErr_NoMemory(); } @@ -261,6 +280,9 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) goto out; } retval = mkgrent(module, p); +#ifndef HAVE_GETGRNAM_R + PyMutex_Unlock(&group_db_mutex); +#endif out: PyMem_RawFree(buf); Py_DECREF(bytes); @@ -285,8 +307,7 @@ grp_getgrall_impl(PyObject *module) return NULL; } - static PyMutex getgrall_mutex = {0}; - PyMutex_Lock(&getgrall_mutex); + PyMutex_Lock(&group_db_mutex); setgrent(); struct group *p; @@ -306,7 +327,7 @@ grp_getgrall_impl(PyObject *module) done: endgrent(); - PyMutex_Unlock(&getgrall_mutex); + PyMutex_Unlock(&group_db_mutex); return d; } diff --git a/Modules/hashlib.h b/Modules/hashlib.h index 7105e68af7b..5ada4ef4b86 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -3,76 +3,171 @@ #include "pycore_lock.h" // PyMutex /* - * Given a PyObject* obj, fill in the Py_buffer* viewp with the result - * of PyObject_GetBuffer. Sets an exception and issues the erraction - * on any errors, e.g. 'return NULL' or 'goto error'. + * Internal error messages used for reporting an unsupported hash algorithm. + * The algorithm can be given by its name, a callable or a PEP-247 module. + * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() + * and _hmacmodule.c::find_hash_info(). */ -#define GET_BUFFER_VIEW_OR_ERROR(obj, viewp, erraction) do { \ - if (PyUnicode_Check((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "Strings must be encoded before hashing");\ - erraction; \ - } \ - if (!PyObject_CheckBuffer((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "object supporting the buffer API required"); \ - erraction; \ - } \ - if (PyObject_GetBuffer((obj), (viewp), PyBUF_SIMPLE) == -1) { \ - erraction; \ - } \ - if ((viewp)->ndim > 1) { \ - PyErr_SetString(PyExc_BufferError, \ - "Buffer must be single dimension"); \ - PyBuffer_Release((viewp)); \ - erraction; \ - } \ - } while(0) +#define HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" +#define HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" -#define GET_BUFFER_VIEW_OR_ERROUT(obj, viewp) \ - GET_BUFFER_VIEW_OR_ERROR(obj, viewp, return NULL) +/* + * Obtain a buffer view from a buffer-like object 'obj'. + * + * On success, store the result in 'view' and return 0. + * On error, set an exception and return -1. + */ +static inline int +_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view) +{ + if (PyUnicode_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "Strings must be encoded before hashing"); + return -1; + } + if (!PyObject_CheckBuffer(obj)) { + PyErr_SetString(PyExc_TypeError, + "object supporting the buffer API required"); + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) == -1) { + return -1; + } + if (view->ndim > 1) { + PyErr_SetString(PyExc_BufferError, + "Buffer must be single dimension"); + PyBuffer_Release(view); + return -1; + } + return 0; +} + +/* + * Call _Py_hashlib_get_buffer_view() and check if it succeeded. + * + * On error, set an exception and execute the ERRACTION statements. + */ +#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ + do { \ + if (_Py_hashlib_get_buffer_view(OBJ, VIEW) < 0) { \ + assert(PyErr_Occurred()); \ + ERRACTION; \ + } \ + } while (0) + +#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ + GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) /* * Helper code to synchronize access to the hash object when the GIL is - * released around a CPU consuming hashlib operation. All code paths that - * access a mutable part of obj must be enclosed in an ENTER_HASHLIB / - * LEAVE_HASHLIB block or explicitly acquire and release the lock inside - * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for - * an operation. + * released around a CPU consuming hashlib operation. * - * These only drop the GIL if the lock acquisition itself is likely to - * block. Thus the non-blocking acquire gating the GIL release for a - * blocking lock acquisition. The intent of these macros is to surround - * the assumed always "fast" operations that you aren't releasing the - * GIL around. Otherwise use code similar to what you see in hash - * function update() methods. + * Code accessing a mutable part of the hash object must be enclosed in + * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release + * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if + * they wish to release the GIL for an operation. */ -#include "pythread.h" -#define ENTER_HASHLIB(obj) \ - if ((obj)->use_mutex) { \ - PyMutex_Lock(&(obj)->mutex); \ - } -#define LEAVE_HASHLIB(obj) \ - if ((obj)->use_mutex) { \ - PyMutex_Unlock(&(obj)->mutex); \ - } +#define HASHLIB_OBJECT_HEAD \ + PyObject_HEAD \ + /* Guard against race conditions during incremental update(). */ \ + PyMutex mutex; -#ifdef Py_GIL_DISABLED -#define HASHLIB_INIT_MUTEX(obj) \ - do { \ - (obj)->mutex = (PyMutex){0}; \ - (obj)->use_mutex = true; \ +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ } while (0) -#else -#define HASHLIB_INIT_MUTEX(obj) \ - do { \ - (obj)->mutex = (PyMutex){0}; \ - (obj)->use_mutex = false; \ + +#define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) +#define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) + +/* + * Message length above which the GIL is to be released + * when performing hashing operations. + */ +#define HASHLIB_GIL_MINSIZE 2048 + +// Macros for executing code while conditionally holding the GIL. +// +// These only drop the GIL if the lock acquisition itself is likely to +// block. Thus the non-blocking acquire gating the GIL release for a +// blocking lock acquisition. The intent of these macros is to surround +// the assumed always "fast" operations that you aren't releasing the +// GIL around. + +/* + * Execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + STATEMENTS; \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + STATEMENTS; \ + } \ } while (0) -#endif -/* TODO(gpshead): We should make this a module or class attribute - * to allow the user to optimize based on the platform they're using. */ -#define HASHLIB_GIL_MINSIZE 2048 +/* + * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + } \ + } while (0) +static inline int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", 1) < 0) + { + *res = NULL; + return -1; + } + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index c7b49d4dee3..f074f248077 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -31,14 +31,15 @@ #endif #if defined(__APPLE__) && defined(__arm64__) -# undef HACL_CAN_COMPILE_SIMD128 -# undef HACL_CAN_COMPILE_SIMD256 +# undef _Py_HACL_CAN_COMPILE_VEC128 +# undef _Py_HACL_CAN_COMPILE_VEC256 #endif -// Small mismatch between the variable names Python defines as part of configure -// at the ones HACL* expects to be set in order to enable those headers. -#define HACL_CAN_COMPILE_VEC128 HACL_CAN_COMPILE_SIMD128 -#define HACL_CAN_COMPILE_VEC256 HACL_CAN_COMPILE_SIMD256 +// HACL* expects HACL_CAN_COMPILE_VEC* macros to be set in order to enable +// the corresponding SIMD instructions so we need to "forward" the values +// we just deduced above. +#define HACL_CAN_COMPILE_VEC128 _Py_HACL_CAN_COMPILE_VEC128 +#define HACL_CAN_COMPILE_VEC256 _Py_HACL_CAN_COMPILE_VEC256 #include "_hacl/Hacl_HMAC.h" #include "_hacl/Hacl_Streaming_HMAC.h" // Hacl_Agile_Hash_* identifiers @@ -215,105 +216,6 @@ typedef struct py_hmac_hacl_api { #define Py_CHECK_HACL_UINT32_T_LENGTH(LEN) #endif -/* - * Call the HACL* HMAC-HASH update function on the given data. - * - * The magnitude of 'LEN' is not checked and thus 'LEN' must be - * safely convertible to a uint32_t value. - */ -#define Py_HMAC_HACL_UPDATE_CALL(HACL_STATE, BUF, LEN) \ - Hacl_Streaming_HMAC_update(HACL_STATE, BUF, (uint32_t)(LEN)) - -/* - * Call the HACL* HMAC-HASH update function on the given data. - * - * On DEBUG builds, the 'ERRACTION' statements are executed if - * the update() call returned a non-successful HACL* exit code. - * - * The buffer 'BUF' and its length 'LEN' are left untouched. - * - * The formal signature of this macro is: - * - * (HACL_HMAC_state *, uint8_t *, uint32_t, PyObject *, (C statements)) - */ -#ifndef NDEBUG -#define Py_HMAC_HACL_UPDATE_ONCE( \ - HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION \ -) \ - do { \ - Py_CHECK_HACL_UINT32_T_LENGTH(LEN); \ - hacl_errno_t code = Py_HMAC_HACL_UPDATE_CALL(HACL_STATE, BUF, LEN); \ - if (_hacl_convert_errno(code, (ALGORITHM)) < 0) { \ - ERRACTION; \ - } \ - } while (0) -#else -#define Py_HMAC_HACL_UPDATE_ONCE( \ - HACL_STATE, BUF, LEN, \ - _ALGORITHM, _ERRACTION \ -) \ - do { \ - (void)Py_HMAC_HACL_UPDATE_CALL(HACL_STATE, BUF, (LEN)); \ - } while (0) -#endif - -/* - * Repetivively call the HACL* HMAC-HASH update function on the given - * data until the buffer length 'LEN' is strictly less than UINT32_MAX. - * - * On builds with PY_SSIZE_T_MAX <= UINT32_MAX, this is a no-op. - * - * The buffer 'BUF' (resp. 'LEN') is advanced (resp. decremented) - * by UINT32_MAX after each update. On DEBUG builds, each update() - * call is verified and the 'ERRACTION' statements are executed if - * a non-successful HACL* exit code is being returned. - * - * In particular, 'BUF' and 'LEN' must be variable names and not - * expressions on their own. - * - * The formal signature of this macro is: - * - * (HACL_HMAC_state *, uint8_t *, C integer, PyObject *, (C statements)) - */ -#ifdef Py_HMAC_SSIZE_LARGER_THAN_UINT32 -#define Py_HMAC_HACL_UPDATE_LOOP( \ - HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION \ -) \ - do { \ - while ((Py_ssize_t)LEN > UINT32_MAX_AS_SSIZE_T) { \ - Py_HMAC_HACL_UPDATE_ONCE(HACL_STATE, BUF, UINT32_MAX, \ - ALGORITHM, ERRACTION); \ - BUF += UINT32_MAX; \ - LEN -= UINT32_MAX; \ - } \ - } while (0) -#else -#define Py_HMAC_HACL_UPDATE_LOOP( \ - HACL_STATE, BUF, LEN, \ - _ALGORITHM, _ERRACTION \ -) -#endif - -/* - * Perform the HMAC-HASH update() operation in a streaming fashion. - * - * The formal signature of this macro is: - * - * (HACL_HMAC_state *, uint8_t *, C integer, PyObject *, (C statements)) - */ -#define Py_HMAC_HACL_UPDATE( \ - HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION \ -) \ - do { \ - Py_HMAC_HACL_UPDATE_LOOP(HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION); \ - Py_HMAC_HACL_UPDATE_ONCE(HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION); \ - } while (0) - /* * HMAC underlying hash function static information. */ @@ -382,11 +284,7 @@ get_hmacmodule_state_by_cls(PyTypeObject *cls) typedef Hacl_Streaming_HMAC_agile_state HACL_HMAC_state; typedef struct HMACObject { - PyObject_HEAD - - bool use_mutex; - PyMutex mutex; - + HASHLIB_OBJECT_HEAD // Hash function information PyObject *name; // rendered name (exact unicode object) HMAC_Hash_Kind kind; // can be used for runtime dispatch (must be known) @@ -464,7 +362,7 @@ narrow_hmac_hash_kind(hmacmodule_state *state, HMAC_Hash_Kind kind) { switch (kind) { case Py_hmac_kind_hmac_blake2s_32: { -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 if (state->can_run_simd128) { return Py_hmac_kind_hmac_vectorized_blake2s_32; } @@ -472,7 +370,7 @@ narrow_hmac_hash_kind(hmacmodule_state *state, HMAC_Hash_Kind kind) return kind; } case Py_hmac_kind_hmac_blake2b_32: { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 if (state->can_run_simd256) { return Py_hmac_kind_hmac_vectorized_blake2b_32; } @@ -491,38 +389,40 @@ narrow_hmac_hash_kind(hmacmodule_state *state, HMAC_Hash_Kind kind) * Otherwise, this sets an appropriate exception and returns -1. */ static int -_hacl_convert_errno(hacl_errno_t code, PyObject *algorithm) +_hacl_convert_errno(hacl_errno_t code) { + assert(PyGILState_GetThisThreadState() != NULL); + if (code == Hacl_Streaming_Types_Success) { + return 0; + } + + PyGILState_STATE gstate = PyGILState_Ensure(); switch (code) { - case Hacl_Streaming_Types_Success: { - return 0; - } case Hacl_Streaming_Types_InvalidAlgorithm: { - // only makes sense if an algorithm is known at call time - assert(algorithm != NULL); - assert(PyUnicode_CheckExact(algorithm)); - PyErr_Format(PyExc_ValueError, "invalid algorithm: %U", algorithm); - return -1; + PyErr_SetString(PyExc_ValueError, "invalid HACL* algorithm"); + break; } case Hacl_Streaming_Types_InvalidLength: { PyErr_SetString(PyExc_ValueError, "invalid length"); - return -1; + break; } case Hacl_Streaming_Types_MaximumLengthExceeded: { PyErr_SetString(PyExc_OverflowError, "maximum length exceeded"); - return -1; + break; } case Hacl_Streaming_Types_OutOfMemory: { PyErr_NoMemory(); - return -1; + break; } default: { PyErr_Format(PyExc_RuntimeError, - "HACL* internal routine failed with error code: %d", + "HACL* internal routine failed with error code: %u", code); - return -1; + break; } } + PyGILState_Release(gstate); + return -1; } /* @@ -536,7 +436,7 @@ _hacl_hmac_state_new(HMAC_Hash_Kind kind, uint8_t *key, uint32_t len) assert(kind != Py_hmac_kind_hash_unknown); HACL_HMAC_state *state = NULL; hacl_errno_t retcode = Hacl_Streaming_HMAC_malloc_(kind, key, len, &state); - if (_hacl_convert_errno(retcode, NULL) < 0) { + if (_hacl_convert_errno(retcode) < 0) { assert(state == NULL); return NULL; } @@ -554,6 +454,51 @@ _hacl_hmac_state_free(HACL_HMAC_state *state) } } +/* + * Call the HACL* HMAC-HASH update function on the given data. + * + * On DEBUG builds, the update() call is verified. + * + * Return 0 on success; otherwise, set an exception and return -1 on failure. +*/ +static int +_hacl_hmac_state_update_once(HACL_HMAC_state *state, + uint8_t *buf, uint32_t len) +{ +#ifndef NDEBUG + hacl_errno_t code = Hacl_Streaming_HMAC_update(state, buf, len); + return _hacl_convert_errno(code); +#else + (void)Hacl_Streaming_HMAC_update(state, buf, len); + return 0; +#endif +} + +/* + * Perform the HMAC-HASH update() operation in a streaming fashion. + * + * On DEBUG builds, each update() call is verified. + * + * Return 0 on success; otherwise, set an exception and return -1 on failure. + */ +static int +_hacl_hmac_state_update(HACL_HMAC_state *state, uint8_t *buf, Py_ssize_t len) +{ + assert(len >= 0); +#ifdef Py_HMAC_SSIZE_LARGER_THAN_UINT32 + while (len > UINT32_MAX_AS_SSIZE_T) { + if (_hacl_hmac_state_update_once(state, buf, UINT32_MAX) < 0) { + assert(PyErr_Occurred()); + return -1; + } + buf += UINT32_MAX; + len -= UINT32_MAX; + } +#endif + Py_CHECK_HACL_UINT32_T_LENGTH(len); + return _hacl_hmac_state_update_once(state, buf, (uint32_t)len); +} + /* Static information used to construct the hash table. */ static const py_hmac_hinfo py_hmac_static_hinfo[] = { #define Py_HMAC_HINFO_HACL_API(HACL_HID) \ @@ -711,7 +656,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref) } if (rc == 0) { PyErr_Format(state->unknown_hash_error, - "unsupported hash type: %R", hash_info_ref); + HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); return NULL; } assert(info != NULL); @@ -784,45 +729,6 @@ hmac_new_initial_state(HMACObject *self, uint8_t *key, Py_ssize_t len) return self->state == NULL ? -1 : 0; } -/* - * Feed initial data. - * - * This function MUST only be called by the HMAC object constructor - * and after hmac_set_hinfo() and hmac_new_initial_state() have been - * called, lest the behaviour is undefined. - * - * Return 0 on success; otherwise, set an exception and return -1 on failure. - */ -static int -hmac_feed_initial_data(HMACObject *self, uint8_t *msg, Py_ssize_t len) -{ - assert(self->name != NULL); - assert(self->state != NULL); - if (len == 0) { - // do nothing if the buffer is empty - return 0; - } - - if (len < HASHLIB_GIL_MINSIZE) { - Py_HMAC_HACL_UPDATE(self->state, msg, len, self->name, return -1); - return 0; - } - - int res = 0; - Py_BEGIN_ALLOW_THREADS - Py_HMAC_HACL_UPDATE(self->state, msg, len, self->name, goto error); - goto done; -#ifndef NDEBUG -error: - res = -1; -#else - Py_UNREACHABLE(); -#endif -done: - Py_END_ALLOW_THREADS - return res; -} - /*[clinic input] _hmac.new @@ -850,7 +756,7 @@ _hmac_new_impl(PyObject *module, PyObject *keyobj, PyObject *msgobj, return NULL; } - HMACObject *self = PyObject_GC_New(HMACObject, state->hmac_type); + HMACObject *self = PyObject_New(HMACObject, state->hmac_type); if (self == NULL) { return NULL; } @@ -869,7 +775,12 @@ _hmac_new_impl(PyObject *module, PyObject *keyobj, PyObject *msgobj, if (msgobj != NULL && msgobj != Py_None) { Py_buffer msg; GET_BUFFER_VIEW_OR_ERROR(msgobj, &msg, goto error); - rc = hmac_feed_initial_data(self, msg.buf, msg.len); + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + msg.len, + rc = _hacl_hmac_state_update(self->state, msg.buf, msg.len) + ); PyBuffer_Release(&msg); #ifndef NDEBUG if (rc < 0) { @@ -880,7 +791,6 @@ _hmac_new_impl(PyObject *module, PyObject *keyobj, PyObject *msgobj, #endif } assert(rc == 0); - PyObject_GC_Track(self); return (PyObject *)self; error_on_key: @@ -941,17 +851,17 @@ _hmac_HMAC_copy_impl(HMACObject *self, PyTypeObject *cls) /*[clinic end generated code: output=a955bfa55b65b215 input=17b2c0ad0b147e36]*/ { hmacmodule_state *state = get_hmacmodule_state_by_cls(cls); - HMACObject *copy = PyObject_GC_New(HMACObject, state->hmac_type); + HMACObject *copy = PyObject_New(HMACObject, state->hmac_type); if (copy == NULL) { return NULL; } - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); /* copy hash information */ hmac_copy_hinfo(copy, self); /* copy internal state */ int rc = hmac_copy_state(copy, self); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); if (rc < 0) { Py_DECREF(copy); @@ -959,82 +869,9 @@ _hmac_HMAC_copy_impl(HMACObject *self, PyTypeObject *cls) } HASHLIB_INIT_MUTEX(copy); - PyObject_GC_Track(copy); return (PyObject *)copy; } -/* - * Update the HMAC object with the given buffer. - * - * This unconditionally acquires the lock on the HMAC object. - * - * On DEBUG builds, each update() call is verified. - * - * Return 0 on success; otherwise, set an exception and return -1 on failure. - */ -static int -hmac_update_state_with_lock(HMACObject *self, uint8_t *buf, Py_ssize_t len) -{ - int res = 0; - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); // unconditionally acquire a lock - Py_HMAC_HACL_UPDATE(self->state, buf, len, self->name, goto error); - goto done; -#ifndef NDEBUG -error: - res = -1; -#else - Py_UNREACHABLE(); -#endif -done: - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - return res; -} - -/* - * Update the HMAC object with the given buffer. - * - * This conditionally acquires the lock on the HMAC object. - * - * On DEBUG builds, each update() call is verified. - * - * Return 0 on success; otherwise, set an exception and return -1 on failure. - */ -static int -hmac_update_state_cond_lock(HMACObject *self, uint8_t *buf, Py_ssize_t len) -{ - ENTER_HASHLIB(self); // conditionally acquire a lock - Py_HMAC_HACL_UPDATE(self->state, buf, len, self->name, goto error); - LEAVE_HASHLIB(self); - return 0; - -#ifndef NDEBUG -error: - LEAVE_HASHLIB(self); - return -1; -#else - Py_UNREACHABLE(); -#endif -} - -/* - * Update the internal HMAC state with the given buffer. - * - * Return 0 on success; otherwise, set an exception and return -1 on failure. - */ -static inline int -hmac_update_state(HMACObject *self, uint8_t *buf, Py_ssize_t len) -{ - assert(buf != 0); - assert(len >= 0); - return len == 0 - ? 0 /* nothing to do */ - : len < HASHLIB_GIL_MINSIZE - ? hmac_update_state_cond_lock(self, buf, len) - : hmac_update_state_with_lock(self, buf, len); -} - /*[clinic input] _hmac.HMAC.update @@ -1047,9 +884,13 @@ static PyObject * _hmac_HMAC_update_impl(HMACObject *self, PyObject *msgobj) /*[clinic end generated code: output=962134ada5e55985 input=7c0ea830efb03367]*/ { + int rc = 0; Py_buffer msg; GET_BUFFER_VIEW_OR_ERROUT(msgobj, &msg); - int rc = hmac_update_state(self, msg.buf, msg.len); + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, msg.len, + rc = _hacl_hmac_state_update(self->state, msg.buf, msg.len) + ); PyBuffer_Release(&msg); return rc < 0 ? NULL : Py_None; } @@ -1065,18 +906,18 @@ _hmac_HMAC_update_impl(HMACObject *self, PyObject *msgobj) * Note: this function may raise a MemoryError. */ static int -hmac_digest_compute_cond_lock(HMACObject *self, uint8_t *digest) +hmac_digest_compute_locked(HMACObject *self, uint8_t *digest) { assert(digest != NULL); hacl_errno_t rc; - ENTER_HASHLIB(self); // conditionally acquire a lock + HASHLIB_ACQUIRE_LOCK(self); rc = Hacl_Streaming_HMAC_digest(self->state, digest, self->digest_size); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); assert( rc == Hacl_Streaming_Types_Success || rc == Hacl_Streaming_Types_OutOfMemory ); - return _hacl_convert_errno(rc, NULL); + return _hacl_convert_errno(rc); } /*[clinic input] @@ -1093,13 +934,15 @@ _hmac_HMAC_digest_impl(HMACObject *self) { assert(self->digest_size <= Py_hmac_hash_max_digest_size); uint8_t digest[Py_hmac_hash_max_digest_size]; - if (hmac_digest_compute_cond_lock(self, digest) < 0) { + if (hmac_digest_compute_locked(self, digest) < 0) { return NULL; } return PyBytes_FromStringAndSize((const char *)digest, self->digest_size); } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body _hmac.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. @@ -1112,11 +955,11 @@ This method may raise a MemoryError. static PyObject * _hmac_HMAC_hexdigest_impl(HMACObject *self) -/*[clinic end generated code: output=6659807a09ae14ec input=493b2db8013982b9]*/ +/*[clinic end generated code: output=6659807a09ae14ec input=6e0e796e38d82fc8]*/ { assert(self->digest_size <= Py_hmac_hash_max_digest_size); uint8_t digest[Py_hmac_hash_max_digest_size]; - if (hmac_digest_compute_cond_lock(self, digest) < 0) { + if (hmac_digest_compute_locked(self, digest) < 0) { return NULL; } return _Py_strhex((const char *)digest, self->digest_size); @@ -1181,19 +1024,11 @@ static void HMACObject_dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); - PyObject_GC_UnTrack(op); (void)HMACObject_clear(op); type->tp_free(op); Py_DECREF(type); } -static int -HMACObject_traverse(PyObject *op, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(op)); - return 0; -} - static PyMethodDef HMACObject_methods[] = { _HMAC_HMAC_COPY_METHODDEF _HMAC_HMAC_UPDATE_METHODDEF @@ -1213,9 +1048,7 @@ static PyType_Slot HMACObject_Type_slots[] = { {Py_tp_repr, HMACObject_repr}, {Py_tp_methods, HMACObject_methods}, {Py_tp_getset, HMACObject_getsets}, - {Py_tp_clear, HMACObject_clear}, {Py_tp_dealloc, HMACObject_dealloc}, - {Py_tp_traverse, HMACObject_traverse}, {0, NULL} /* sentinel */ }; @@ -1225,8 +1058,7 @@ static PyType_Spec HMAC_Type_spec = { .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HEAPTYPE - | Py_TPFLAGS_IMMUTABLETYPE - | Py_TPFLAGS_HAVE_GC, + | Py_TPFLAGS_IMMUTABLETYPE, .slots = HMACObject_Type_slots, }; @@ -1256,42 +1088,60 @@ _hmac_compute_digest_impl(PyObject *module, PyObject *key, PyObject *msg, } /* - * One-shot HMAC-HASH using the given HACL_HID. + * Obtain a view for 'key' and 'msg', storing it in 'keyview' and 'msgview'. + * + * Return 0 on success; otherwise set an exception and return -1. * * The length of the key and message buffers must not exceed UINT32_MAX, * lest an OverflowError is raised. The Python implementation takes care * of dispatching to the OpenSSL implementation in this case. */ -#define Py_HMAC_HACL_ONESHOT(HACL_HID, KEY, MSG) \ - do { \ - Py_buffer keyview, msgview; \ - GET_BUFFER_VIEW_OR_ERROUT((KEY), &keyview); \ - if (!has_uint32_t_buffer_length(&keyview)) { \ - PyBuffer_Release(&keyview); \ - set_invalid_key_length_error(); \ - return NULL; \ - } \ - GET_BUFFER_VIEW_OR_ERROR((MSG), &msgview, \ - PyBuffer_Release(&keyview); \ - return NULL); \ - if (!has_uint32_t_buffer_length(&msgview)) { \ - PyBuffer_Release(&msgview); \ - PyBuffer_Release(&keyview); \ - set_invalid_msg_length_error(); \ - return NULL; \ - } \ - uint8_t out[Py_hmac_## HACL_HID ##_digest_size]; \ - Py_hmac_## HACL_HID ##_compute_func( \ - out, \ - (uint8_t *)keyview.buf, (uint32_t)keyview.len, \ - (uint8_t *)msgview.buf, (uint32_t)msgview.len \ - ); \ - PyBuffer_Release(&msgview); \ - PyBuffer_Release(&keyview); \ - return PyBytes_FromStringAndSize( \ - (const char *)out, \ - Py_hmac_## HACL_HID ##_digest_size \ - ); \ +static int +hmac_get_buffer_views(PyObject *key, Py_buffer *keyview, + PyObject *msg, Py_buffer *msgview) +{ + if (_Py_hashlib_get_buffer_view(key, keyview) < 0) { + return -1; + } + if (!has_uint32_t_buffer_length(keyview)) { + PyBuffer_Release(keyview); + set_invalid_key_length_error(); + return -1; + } + if (_Py_hashlib_get_buffer_view(msg, msgview) < 0) { + PyBuffer_Release(keyview); + return -1; + } + if (!has_uint32_t_buffer_length(msgview)) { + PyBuffer_Release(msgview); + PyBuffer_Release(keyview); + set_invalid_msg_length_error(); + return -1; + } + return 0; +} + +/* + * One-shot HMAC-HASH using the given HACL_HID. + */ +#define HACL_HMAC_COMPUTE_NAMED_DIGEST(HACL_HID, KEY, MSG) \ + do { \ + Py_buffer keyview, msgview; \ + if (hmac_get_buffer_views(key, &keyview, msg, &msgview) < 0) { \ + return NULL; \ + } \ + uint8_t out[Py_hmac_## HACL_HID ##_digest_size]; \ + Py_hmac_## HACL_HID ##_compute_func( \ + out, \ + (uint8_t *)keyview.buf, (uint32_t)keyview.len, \ + (uint8_t *)msgview.buf, (uint32_t)msgview.len \ + ); \ + PyBuffer_Release(&msgview); \ + PyBuffer_Release(&keyview); \ + return PyBytes_FromStringAndSize( \ + (const char *)out, \ + Py_hmac_## HACL_HID ##_digest_size \ + ); \ } while (0) /*[clinic input] @@ -1307,7 +1157,7 @@ static PyObject * _hmac_compute_md5_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=7837a4ceccbbf636 input=77a4b774c7d61218]*/ { - Py_HMAC_HACL_ONESHOT(md5, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(md5, key, msg); } /*[clinic input] @@ -1323,7 +1173,7 @@ static PyObject * _hmac_compute_sha1_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=79fd7689c83691d8 input=3b64dccc6bdbe4ba]*/ { - Py_HMAC_HACL_ONESHOT(sha1, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha1, key, msg); } /*[clinic input] @@ -1339,7 +1189,7 @@ static PyObject * _hmac_compute_sha2_224_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=7f21f1613e53979e input=a1a75f25f23449af]*/ { - Py_HMAC_HACL_ONESHOT(sha2_224, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha2_224, key, msg); } /*[clinic input] @@ -1355,7 +1205,7 @@ static PyObject * _hmac_compute_sha2_256_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=d4a291f7d9a82459 input=5c9ccf2df048ace3]*/ { - Py_HMAC_HACL_ONESHOT(sha2_256, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha2_256, key, msg); } /*[clinic input] @@ -1371,7 +1221,7 @@ static PyObject * _hmac_compute_sha2_384_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=f211fa26e3700c27 input=2fee2c14766af231]*/ { - Py_HMAC_HACL_ONESHOT(sha2_384, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha2_384, key, msg); } /*[clinic input] @@ -1387,7 +1237,7 @@ static PyObject * _hmac_compute_sha2_512_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=d5c20373762cecca input=3371eaac315c7864]*/ { - Py_HMAC_HACL_ONESHOT(sha2_512, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha2_512, key, msg); } /*[clinic input] @@ -1403,7 +1253,7 @@ static PyObject * _hmac_compute_sha3_224_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=a242ccac9ad9c22b input=d0ab0c7d189c3d87]*/ { - Py_HMAC_HACL_ONESHOT(sha3_224, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha3_224, key, msg); } /*[clinic input] @@ -1419,7 +1269,7 @@ static PyObject * _hmac_compute_sha3_256_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=b539dbb61af2fe0b input=f05d7b6364b35d02]*/ { - Py_HMAC_HACL_ONESHOT(sha3_256, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha3_256, key, msg); } /*[clinic input] @@ -1435,7 +1285,7 @@ static PyObject * _hmac_compute_sha3_384_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=5eb372fb5c4ffd3a input=d842d393e7aa05ae]*/ { - Py_HMAC_HACL_ONESHOT(sha3_384, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha3_384, key, msg); } /*[clinic input] @@ -1451,7 +1301,7 @@ static PyObject * _hmac_compute_sha3_512_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=154bcbf8c2eacac1 input=166fe5baaeaabfde]*/ { - Py_HMAC_HACL_ONESHOT(sha3_512, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha3_512, key, msg); } /*[clinic input] @@ -1467,7 +1317,7 @@ static PyObject * _hmac_compute_blake2s_32_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=cfc730791bc62361 input=d22c36e7fe31a985]*/ { - Py_HMAC_HACL_ONESHOT(blake2s_32, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(blake2s_32, key, msg); } /*[clinic input] @@ -1483,9 +1333,11 @@ static PyObject * _hmac_compute_blake2b_32_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=765c5c4fb9124636 input=4a35ee058d172f4b]*/ { - Py_HMAC_HACL_ONESHOT(blake2b_32, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(blake2b_32, key, msg); } +#undef HACL_HMAC_COMPUTE_NAMED_DIGEST + // --- HMAC module methods ---------------------------------------------------- static PyMethodDef hmacmodule_methods[] = { @@ -1715,11 +1567,11 @@ hmacmodule_init_cpu_features(hmacmodule_state *state) __cpuid_count(1, 0, eax1, ebx1, ecx1, edx1); __cpuid_count(7, 0, eax7, ebx7, ecx7, edx7); #elif defined(_M_X64) - int info1[4] = { 0 }; + int info1[4] = {0}; __cpuidex(info1, 1, 0); eax1 = info1[0], ebx1 = info1[1], ecx1 = info1[2], edx1 = info1[3]; - int info7[4] = { 0 }; + int info7[4] = {0}; __cpuidex(info7, 7, 0); eax7 = info7[0], ebx7 = info7[1], ecx7 = info7[2], edx7 = info7[3]; #endif @@ -1759,7 +1611,7 @@ hmacmodule_init_cpu_features(hmacmodule_state *state) #undef ECX_SSE3 #undef EBX_AVX2 -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 // TODO(picnixz): use py_cpuid_features (gh-125022) to improve detection state->can_run_simd128 = sse && sse2 && sse3 && sse41 && sse42 && cmov; #else @@ -1769,7 +1621,7 @@ hmacmodule_init_cpu_features(hmacmodule_state *state) state->can_run_simd128 = false; #endif -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 // TODO(picnixz): use py_cpuid_features (gh-125022) to improve detection state->can_run_simd256 = state->can_run_simd128 && avx && avx2; #else diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 943c1e8607b..8685eff8be6 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -376,7 +376,7 @@ pairwise_next(PyObject *op) } result = po->result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); PyObject *last_old = PyTuple_GET_ITEM(result, 0); PyObject *last_new = PyTuple_GET_ITEM(result, 1); @@ -802,7 +802,7 @@ teedataobject_traverse(PyObject *op, visitproc visit, void * arg) static void teedataobject_safe_decref(PyObject *obj) { - while (obj && Py_REFCNT(obj) == 1) { + while (obj && _PyObject_IsUniquelyReferenced(obj)) { teedataobject *tmp = teedataobject_CAST(obj); PyObject *nextlink = tmp->nextlink; tmp->nextlink = NULL; @@ -1069,22 +1069,18 @@ static PyType_Spec tee_spec = { /*[clinic input] itertools.tee iterable: object - n: Py_ssize_t = 2 + n: Py_ssize_t(allow_negative=False) = 2 / Returns a tuple of n independent iterators. [clinic start generated code]*/ static PyObject * itertools_tee_impl(PyObject *module, PyObject *iterable, Py_ssize_t n) -/*[clinic end generated code: output=1c64519cd859c2f0 input=c99a1472c425d66d]*/ +/*[clinic end generated code: output=1c64519cd859c2f0 input=0f72d78e655f45cb]*/ { Py_ssize_t i; PyObject *it, *to, *result; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "n must be >= 0"); - return NULL; - } result = PyTuple_New(n); if (result == NULL) return NULL; @@ -1124,12 +1120,12 @@ typedef struct { PyObject *it; PyObject *saved; Py_ssize_t index; - int firstpass; } cycleobject; #define cycleobject_CAST(op) ((cycleobject *)(op)) /*[clinic input] +@permit_long_summary @classmethod itertools.cycle.__new__ iterable: object @@ -1139,7 +1135,7 @@ Return elements from the iterable until it is exhausted. Then repeat the sequenc static PyObject * itertools_cycle_impl(PyTypeObject *type, PyObject *iterable) -/*[clinic end generated code: output=f60e5ec17a45b35c input=9d1d84bcf66e908b]*/ +/*[clinic end generated code: output=f60e5ec17a45b35c input=ead392f4aac7afd8]*/ { PyObject *it; PyObject *saved; @@ -1165,8 +1161,7 @@ itertools_cycle_impl(PyTypeObject *type, PyObject *iterable) } lz->it = it; lz->saved = saved; - lz->index = 0; - lz->firstpass = 0; + lz->index = -1; return (PyObject *)lz; } @@ -1199,11 +1194,11 @@ cycle_next(PyObject *op) cycleobject *lz = cycleobject_CAST(op); PyObject *item; - if (lz->it != NULL) { + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(lz->index); + + if (index < 0) { item = PyIter_Next(lz->it); if (item != NULL) { - if (lz->firstpass) - return item; if (PyList_Append(lz->saved, item)) { Py_DECREF(item); return NULL; @@ -1213,15 +1208,22 @@ cycle_next(PyObject *op) /* Note: StopIteration is already cleared by PyIter_Next() */ if (PyErr_Occurred()) return NULL; + index = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(lz->index, 0); +#ifndef Py_GIL_DISABLED Py_CLEAR(lz->it); +#endif } if (PyList_GET_SIZE(lz->saved) == 0) return NULL; - item = PyList_GET_ITEM(lz->saved, lz->index); - lz->index++; - if (lz->index >= PyList_GET_SIZE(lz->saved)) - lz->index = 0; - return Py_NewRef(item); + item = PyList_GetItemRef(lz->saved, index); + assert(item); + index++; + if (index >= PyList_GET_SIZE(lz->saved)) { + index = 0; + } + FT_ATOMIC_STORE_SSIZE_RELAXED(lz->index, index); + return item; } static PyType_Slot cycle_slots[] = { @@ -1381,6 +1383,7 @@ typedef struct { #define takewhileobject_CAST(op) ((takewhileobject *)(op)) /*[clinic input] +@permit_long_summary @classmethod itertools.takewhile.__new__ predicate as func: object @@ -1391,7 +1394,7 @@ Return successive entries from an iterable as long as the predicate evaluates to static PyObject * itertools_takewhile_impl(PyTypeObject *type, PyObject *func, PyObject *seq) -/*[clinic end generated code: output=bb179ea7864e2ef6 input=ba5255f7519aa119]*/ +/*[clinic end generated code: output=bb179ea7864e2ef6 input=61e42255dd0a7657]*/ { PyObject *it; takewhileobject *lz; @@ -1690,6 +1693,7 @@ typedef struct { #define starmapobject_CAST(op) ((starmapobject *)(op)) /*[clinic input] +@permit_long_summary @classmethod itertools.starmap.__new__ function as func: object @@ -1700,7 +1704,7 @@ Return an iterator whose values are returned from the function evaluated with an static PyObject * itertools_starmap_impl(PyTypeObject *type, PyObject *func, PyObject *seq) -/*[clinic end generated code: output=79eeb81d452c6e8d input=844766df6a0d4dad]*/ +/*[clinic end generated code: output=79eeb81d452c6e8d input=8c9068da0692d6d2]*/ { PyObject *it; starmapobject *lz; @@ -1833,6 +1837,7 @@ chain_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } /*[clinic input] +@permit_long_summary @classmethod itertools.chain.from_iterable iterable as arg: object @@ -1842,7 +1847,7 @@ Alternative chain() constructor taking a single iterable argument that evaluates static PyObject * itertools_chain_from_iterable_impl(PyTypeObject *type, PyObject *arg) -/*[clinic end generated code: output=3d7ea7d46b9e43f5 input=72c39e3a2ca3be85]*/ +/*[clinic end generated code: output=3d7ea7d46b9e43f5 input=a9bf8227221c75b3]*/ { PyObject *source; @@ -1875,8 +1880,8 @@ chain_traverse(PyObject *op, visitproc visit, void *arg) return 0; } -static PyObject * -chain_next(PyObject *op) +static inline PyObject * +chain_next_lock_held(PyObject *op) { chainobject *lz = chainobject_CAST(op); PyObject *item; @@ -1914,6 +1919,16 @@ chain_next(PyObject *op) return NULL; } +static PyObject * +chain_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = chain_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + PyDoc_STRVAR(chain_doc, "chain(*iterables)\n\ --\n\ @@ -2081,7 +2096,7 @@ product_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -product_next(PyObject *op) +product_next_lock_held(PyObject *op) { productobject *lz = productobject_CAST(op); PyObject *pool; @@ -2114,9 +2129,9 @@ product_next(PyObject *op) Py_ssize_t *indices = lz->indices; /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; - result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), npools); + result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), npools); if (result == NULL) goto empty; lz->result = result; @@ -2167,6 +2182,16 @@ product_next(PyObject *op) return NULL; } +static PyObject * +product_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = product_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef product_methods[] = { {"__sizeof__", product_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -2226,7 +2251,7 @@ typedef struct { @classmethod itertools.combinations.__new__ iterable: object - r: Py_ssize_t + r: Py_ssize_t(allow_negative=False) Return successive r-length combinations of elements in the iterable. combinations(range(4), 3) --> (0,1,2), (0,1,3), (0,2,3), (1,2,3) @@ -2235,7 +2260,7 @@ combinations(range(4), 3) --> (0,1,2), (0,1,3), (0,2,3), (1,2,3) static PyObject * itertools_combinations_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t r) -/*[clinic end generated code: output=87a689b39c40039c input=06bede09e3da20f8]*/ +/*[clinic end generated code: output=87a689b39c40039c input=a32f07a15cfa4676]*/ { combinationsobject *co; Py_ssize_t n; @@ -2247,10 +2272,6 @@ itertools_combinations_impl(PyTypeObject *type, PyObject *iterable, if (pool == NULL) goto error; n = PyTuple_GET_SIZE(pool); - if (r < 0) { - PyErr_SetString(PyExc_ValueError, "r must be non-negative"); - goto error; - } indices = PyMem_New(Py_ssize_t, r); if (indices == NULL) { @@ -2314,7 +2335,7 @@ combinations_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -combinations_next(PyObject *op) +combinations_next_lock_held(PyObject *op) { combinationsobject *co = combinationsobject_CAST(op); PyObject *elem; @@ -2343,9 +2364,9 @@ combinations_next(PyObject *op) } } else { /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; - result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); + result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) goto empty; co->result = result; @@ -2399,6 +2420,16 @@ combinations_next(PyObject *op) return NULL; } +static PyObject * +combinations_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = combinations_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef combinations_methods[] = { {"__sizeof__", combinations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -2466,10 +2497,12 @@ typedef struct { #define cwrobject_CAST(op) ((cwrobject *)(op)) /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @classmethod itertools.combinations_with_replacement.__new__ iterable: object - r: Py_ssize_t + r: Py_ssize_t(allow_negative=False) Return successive r-length combinations of elements in the iterable allowing individual elements to have successive repeats. combinations_with_replacement('ABC', 2) --> ('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C') @@ -2479,7 +2512,7 @@ static PyObject * itertools_combinations_with_replacement_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t r) -/*[clinic end generated code: output=48b26856d4e659ca input=1dc58e82a0878fdc]*/ +/*[clinic end generated code: output=48b26856d4e659ca input=828696750169e84f]*/ { cwrobject *co; Py_ssize_t n; @@ -2491,10 +2524,6 @@ itertools_combinations_with_replacement_impl(PyTypeObject *type, if (pool == NULL) goto error; n = PyTuple_GET_SIZE(pool); - if (r < 0) { - PyErr_SetString(PyExc_ValueError, "r must be non-negative"); - goto error; - } indices = PyMem_New(Py_ssize_t, r); if (indices == NULL) { @@ -2589,9 +2618,9 @@ cwr_next(PyObject *op) } } else { /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; - result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); + result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) goto empty; co->result = result; @@ -2850,9 +2879,9 @@ permutations_next(PyObject *op) goto empty; /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; - result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); + result = PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) goto empty; po->result = result; @@ -3818,7 +3847,7 @@ zip_longest_next(PyObject *op) return NULL; if (lz->numactive == 0) return NULL; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); diff --git a/Modules/main.c b/Modules/main.c index ea1239ecc57..74e48c94732 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -128,7 +128,7 @@ pymain_get_importer(const wchar_t *filename, PyObject **importer_p, int *exitcod { PyObject *sys_path0 = NULL, *importer; - sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); + sys_path0 = PyUnicode_FromWideChar(filename, -1); if (sys_path0 == NULL) { goto error; } @@ -269,13 +269,14 @@ pymain_run_command(wchar_t *command) static int -pymain_start_pyrepl_no_main(void) +pymain_start_pyrepl(int pythonstartup) { int res = 0; PyObject *console = NULL; PyObject *empty_tuple = NULL; PyObject *kwargs = NULL; PyObject *console_result = NULL; + PyObject *main_module = NULL; PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main"); if (pyrepl == NULL) { @@ -299,7 +300,13 @@ pymain_start_pyrepl_no_main(void) res = pymain_exit_err_print(); goto done; } - if (!PyDict_SetItemString(kwargs, "pythonstartup", _PyLong_GetOne())) { + main_module = PyImport_AddModuleRef("__main__"); + if (main_module == NULL) { + res = pymain_exit_err_print(); + goto done; + } + if (!PyDict_SetItemString(kwargs, "mainmodule", main_module) + && !PyDict_SetItemString(kwargs, "pythonstartup", pythonstartup ? Py_True : Py_False)) { console_result = PyObject_Call(console, empty_tuple, kwargs); if (console_result == NULL) { res = pymain_exit_err_print(); @@ -311,6 +318,7 @@ pymain_start_pyrepl_no_main(void) Py_XDECREF(empty_tuple); Py_XDECREF(console); Py_XDECREF(pyrepl); + Py_XDECREF(main_module); return res; } @@ -328,7 +336,7 @@ pymain_run_module(const wchar_t *modname, int set_argv0) fprintf(stderr, "Could not import runpy._run_module_as_main\n"); return pymain_exit_err_print(); } - module = PyUnicode_FromWideChar(modname, wcslen(modname)); + module = PyUnicode_FromWideChar(modname, -1); if (module == NULL) { fprintf(stderr, "Could not convert module name to unicode\n"); Py_DECREF(runmodule); @@ -439,7 +447,7 @@ pymain_run_startup(PyConfig *config, int *exitcode) if (env == NULL || env[0] == L'\0') { return 0; } - startup = PyUnicode_FromWideChar(env, wcslen(env)); + startup = PyUnicode_FromWideChar(env, -1); if (startup == NULL) { goto error; } @@ -489,16 +497,13 @@ pymain_run_startup(PyConfig *config, int *exitcode) static int pymain_run_interactive_hook(int *exitcode) { - PyObject *hook = PyImport_ImportModuleAttrString("sys", - "__interactivehook__"); - if (hook == NULL) { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - // no sys.__interactivehook__ attribute - PyErr_Clear(); - return 0; - } + PyObject *hook; + if (PySys_GetOptionalAttrString("__interactivehook__", &hook) < 0) { goto error; } + if (hook == NULL) { + return 0; + } if (PySys_Audit("cpython.run_interactivehook", "O", hook) < 0) { goto error; @@ -562,7 +567,7 @@ pymain_run_stdin(PyConfig *config) int run = PyRun_AnyFileExFlags(stdin, "<stdin>", 0, &cf); return (run != 0); } - return pymain_run_module(L"_pyrepl", 0); + return pymain_start_pyrepl(0); } @@ -595,7 +600,7 @@ pymain_repl(PyConfig *config, int *exitcode) *exitcode = (run != 0); return; } - int run = pymain_start_pyrepl_no_main(); + int run = pymain_start_pyrepl(1); *exitcode = (run != 0); return; } diff --git a/Modules/makesetup b/Modules/makesetup index f6cf695b457..104c824b846 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -90,6 +90,7 @@ NL='\ # Main loop for i in ${*-Setup} do + echo '' # Add a linebreak so we don't choke on files missing EOL. case $i in -n) echo '*noobjects*';; *) echo '*doconfig*'; cat "$i";; diff --git a/Modules/mathintegermodule.c b/Modules/mathintegermodule.c new file mode 100644 index 00000000000..de5f619c9d0 --- /dev/null +++ b/Modules/mathintegermodule.c @@ -0,0 +1,1293 @@ +/* math.integer module -- integer-related mathematical functions */ + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_bitutils.h" // _Py_bit_length() +#include "pycore_long.h" // _PyLong_GetZero() + +#include "clinic/mathintegermodule.c.h" + +/*[clinic input] +module math +module math.integer +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e3d09c1c90de7fa8]*/ + + +/*[clinic input] +math.integer.gcd + + *integers as args: array + +Greatest Common Divisor. +[clinic start generated code]*/ + +static PyObject * +math_integer_gcd_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=8e9c5bab06bea203 input=a90cde2ac5281551]*/ +{ + // Fast-path for the common case: gcd(int, int) + if (args_length == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) + { + return _PyLong_GCD(args[0], args[1]); + } + + if (args_length == 0) { + return PyLong_FromLong(0); + } + + PyObject *res = PyNumber_Index(args[0]); + if (res == NULL) { + return NULL; + } + if (args_length == 1) { + Py_SETREF(res, PyNumber_Absolute(res)); + return res; + } + + PyObject *one = _PyLong_GetOne(); // borrowed ref + for (Py_ssize_t i = 1; i < args_length; i++) { + PyObject *x = _PyNumber_Index(args[i]); + if (x == NULL) { + Py_DECREF(res); + return NULL; + } + if (res == one) { + /* Fast path: just check arguments. + It is okay to use identity comparison here. */ + Py_DECREF(x); + continue; + } + Py_SETREF(res, _PyLong_GCD(res, x)); + Py_DECREF(x); + if (res == NULL) { + return NULL; + } + } + return res; +} + + +static PyObject * +long_lcm(PyObject *a, PyObject *b) +{ + PyObject *g, *m, *f, *ab; + + if (_PyLong_IsZero((PyLongObject *)a) || _PyLong_IsZero((PyLongObject *)b)) { + return PyLong_FromLong(0); + } + g = _PyLong_GCD(a, b); + if (g == NULL) { + return NULL; + } + f = PyNumber_FloorDivide(a, g); + Py_DECREF(g); + if (f == NULL) { + return NULL; + } + m = PyNumber_Multiply(f, b); + Py_DECREF(f); + if (m == NULL) { + return NULL; + } + ab = PyNumber_Absolute(m); + Py_DECREF(m); + return ab; +} + + +/*[clinic input] +math.integer.lcm + + *integers as args: array + +Least Common Multiple. +[clinic start generated code]*/ + +static PyObject * +math_integer_lcm_impl(PyObject *module, PyObject * const *args, + Py_ssize_t args_length) +/*[clinic end generated code: output=3e88889b866ccc28 input=261bddc85a136bdf]*/ +{ + PyObject *res, *x; + Py_ssize_t i; + + if (args_length == 0) { + return PyLong_FromLong(1); + } + res = PyNumber_Index(args[0]); + if (res == NULL) { + return NULL; + } + if (args_length == 1) { + Py_SETREF(res, PyNumber_Absolute(res)); + return res; + } + + PyObject *zero = _PyLong_GetZero(); // borrowed ref + for (i = 1; i < args_length; i++) { + x = PyNumber_Index(args[i]); + if (x == NULL) { + Py_DECREF(res); + return NULL; + } + if (res == zero) { + /* Fast path: just check arguments. + It is okay to use identity comparison here. */ + Py_DECREF(x); + continue; + } + Py_SETREF(res, long_lcm(res, x)); + Py_DECREF(x); + if (res == NULL) { + return NULL; + } + } + return res; +} + + +/* Integer square root + +Given a nonnegative integer `n`, we want to compute the largest integer +`a` for which `a * a <= n`, or equivalently the integer part of the exact +square root of `n`. + +We use an adaptive-precision pure-integer version of Newton's iteration. Given +a positive integer `n`, the algorithm produces at each iteration an integer +approximation `a` to the square root of `n >> s` for some even integer `s`, +with `s` decreasing as the iterations progress. On the final iteration, `s` is +zero and we have an approximation to the square root of `n` itself. + +At every step, the approximation `a` is strictly within 1.0 of the true square +root, so we have + + (a - 1)**2 < (n >> s) < (a + 1)**2 + +After the final iteration, a check-and-correct step is needed to determine +whether `a` or `a - 1` gives the desired integer square root of `n`. + +The algorithm is remarkable in its simplicity. There's no need for a +per-iteration check-and-correct step, and termination is straightforward: the +number of iterations is known in advance (it's exactly `floor(log2(log2(n)))` +for `n > 1`). The only tricky part of the correctness proof is in establishing +that the bound `(a - 1)**2 < (n >> s) < (a + 1)**2` is maintained from one +iteration to the next. A sketch of the proof of this is given below. + +In addition to the proof sketch, a formal, computer-verified proof +of correctness (using Lean) of an equivalent recursive algorithm can be found +here: + + https://github.com/mdickinson/snippets/blob/master/proofs/isqrt/src/isqrt.lean + + +Here's Python code equivalent to the C implementation below: + + def isqrt(n): + """ + Return the integer part of the square root of the input. + """ + n = operator.index(n) + + if n < 0: + raise ValueError("isqrt() argument must be nonnegative") + if n == 0: + return 0 + + c = (n.bit_length() - 1) // 2 + a = 1 + d = 0 + for s in reversed(range(c.bit_length())): + # Loop invariant: (a-1)**2 < (n >> 2*(c - d)) < (a+1)**2 + e = d + d = c >> s + a = (a << d - e - 1) + (n >> 2*c - e - d + 1) // a + + return a - (a*a > n) + + +Sketch of proof of correctness +------------------------------ + +The delicate part of the correctness proof is showing that the loop invariant +is preserved from one iteration to the next. That is, just before the line + + a = (a << d - e - 1) + (n >> 2*c - e - d + 1) // a + +is executed in the above code, we know that + + (1) (a - 1)**2 < (n >> 2*(c - e)) < (a + 1)**2. + +(since `e` is always the value of `d` from the previous iteration). We must +prove that after that line is executed, we have + + (a - 1)**2 < (n >> 2*(c - d)) < (a + 1)**2 + +To facilitate the proof, we make some changes of notation. Write `m` for +`n >> 2*(c-d)`, and write `b` for the new value of `a`, so + + b = (a << d - e - 1) + (n >> 2*c - e - d + 1) // a + +or equivalently: + + (2) b = (a << d - e - 1) + (m >> d - e + 1) // a + +Then we can rewrite (1) as: + + (3) (a - 1)**2 < (m >> 2*(d - e)) < (a + 1)**2 + +and we must show that (b - 1)**2 < m < (b + 1)**2. + +From this point on, we switch to mathematical notation, so `/` means exact +division rather than integer division and `^` is used for exponentiation. We +use the `√` symbol for the exact square root. In (3), we can remove the +implicit floor operation to give: + + (4) (a - 1)^2 < m / 4^(d - e) < (a + 1)^2 + +Taking square roots throughout (4), scaling by `2^(d-e)`, and rearranging gives + + (5) 0 <= | 2^(d-e)a - √m | < 2^(d-e) + +Squaring and dividing through by `2^(d-e+1) a` gives + + (6) 0 <= 2^(d-e-1) a + m / (2^(d-e+1) a) - √m < 2^(d-e-1) / a + +We'll show below that `2^(d-e-1) <= a`. Given that, we can replace the +right-hand side of (6) with `1`, and now replacing the central +term `m / (2^(d-e+1) a)` with its floor in (6) gives + + (7) -1 < 2^(d-e-1) a + m // 2^(d-e+1) a - √m < 1 + +Or equivalently, from (2): + + (7) -1 < b - √m < 1 + +and rearranging gives that `(b-1)^2 < m < (b+1)^2`, which is what we needed +to prove. + +We're not quite done: we still have to prove the inequality `2^(d - e - 1) <= +a` that was used to get line (7) above. From the definition of `c`, we have +`4^c <= n`, which implies + + (8) 4^d <= m + +also, since `e == d >> 1`, `d` is at most `2e + 1`, from which it follows +that `2d - 2e - 1 <= d` and hence that + + (9) 4^(2d - 2e - 1) <= m + +Dividing both sides by `4^(d - e)` gives + + (10) 4^(d - e - 1) <= m / 4^(d - e) + +But we know from (4) that `m / 4^(d-e) < (a + 1)^2`, hence + + (11) 4^(d - e - 1) < (a + 1)^2 + +Now taking square roots of both sides and observing that both `2^(d-e-1)` and +`a` are integers gives `2^(d - e - 1) <= a`, which is what we needed. This +completes the proof sketch. + +*/ + +/* + The _approximate_isqrt_tab table provides approximate square roots for + 16-bit integers. For any n in the range 2**14 <= n < 2**16, the value + + a = _approximate_isqrt_tab[(n >> 8) - 64] + + is an approximate square root of n, satisfying (a - 1)**2 < n < (a + 1)**2. + + The table was computed in Python using the expression: + + [min(round(sqrt(256*n + 128)), 255) for n in range(64, 256)] +*/ + +static const uint8_t _approximate_isqrt_tab[192] = { + 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 144, 145, 146, 147, 148, 149, 150, + 151, 151, 152, 153, 154, 155, 156, 156, 157, 158, 159, 160, + 160, 161, 162, 163, 164, 164, 165, 166, 167, 167, 168, 169, + 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, + 179, 179, 180, 181, 181, 182, 183, 183, 184, 185, 186, 186, + 187, 188, 188, 189, 190, 190, 191, 192, 192, 193, 194, 194, + 195, 196, 196, 197, 198, 198, 199, 200, 200, 201, 201, 202, + 203, 203, 204, 205, 205, 206, 206, 207, 208, 208, 209, 210, + 210, 211, 211, 212, 213, 213, 214, 214, 215, 216, 216, 217, + 217, 218, 219, 219, 220, 220, 221, 221, 222, 223, 223, 224, + 224, 225, 225, 226, 227, 227, 228, 228, 229, 229, 230, 230, + 231, 232, 232, 233, 233, 234, 234, 235, 235, 236, 237, 237, + 238, 238, 239, 239, 240, 240, 241, 241, 242, 242, 243, 243, + 244, 244, 245, 246, 246, 247, 247, 248, 248, 249, 249, 250, + 250, 251, 251, 252, 252, 253, 253, 254, 254, 255, 255, 255, +}; + +/* Approximate square root of a large 64-bit integer. + + Given `n` satisfying `2**62 <= n < 2**64`, return `a` + satisfying `(a - 1)**2 < n < (a + 1)**2`. */ + +static inline uint32_t +_approximate_isqrt(uint64_t n) +{ + uint32_t u = _approximate_isqrt_tab[(n >> 56) - 64]; + u = (u << 7) + (uint32_t)(n >> 41) / u; + return (u << 15) + (uint32_t)((n >> 17) / u); +} + +/*[clinic input] +math.integer.isqrt + + n: object + / + +Return the integer part of the square root of the input. +[clinic start generated code]*/ + +static PyObject * +math_integer_isqrt(PyObject *module, PyObject *n) +/*[clinic end generated code: output=551031e41a0f5d9e input=921ddd9853133d8d]*/ +{ + int a_too_large, c_bit_length; + int64_t c, d; + uint64_t m; + uint32_t u; + PyObject *a = NULL, *b; + + n = _PyNumber_Index(n); + if (n == NULL) { + return NULL; + } + + if (_PyLong_IsNegative((PyLongObject *)n)) { + PyErr_SetString( + PyExc_ValueError, + "isqrt() argument must be nonnegative"); + goto error; + } + if (_PyLong_IsZero((PyLongObject *)n)) { + Py_DECREF(n); + return PyLong_FromLong(0); + } + + /* c = (n.bit_length() - 1) // 2 */ + c = _PyLong_NumBits(n); + assert(c > 0); + assert(!PyErr_Occurred()); + c = (c - 1) / 2; + + /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a + fast, almost branch-free algorithm. */ + if (c <= 31) { + int shift = 31 - (int)c; + m = (uint64_t)PyLong_AsUnsignedLongLong(n); + Py_DECREF(n); + if (m == (uint64_t)(-1) && PyErr_Occurred()) { + return NULL; + } + u = _approximate_isqrt(m << 2*shift) >> shift; + u -= (uint64_t)u * u > m; + return PyLong_FromUnsignedLong(u); + } + + /* Slow path: n >= 2**64. We perform the first five iterations in C integer + arithmetic, then switch to using Python long integers. */ + + /* From n >= 2**64 it follows that c.bit_length() >= 6. */ + c_bit_length = 6; + while ((c >> c_bit_length) > 0) { + ++c_bit_length; + } + + /* Initialise d and a. */ + d = c >> (c_bit_length - 5); + b = _PyLong_Rshift(n, 2*c - 62); + if (b == NULL) { + goto error; + } + m = (uint64_t)PyLong_AsUnsignedLongLong(b); + Py_DECREF(b); + if (m == (uint64_t)(-1) && PyErr_Occurred()) { + goto error; + } + u = _approximate_isqrt(m) >> (31U - d); + a = PyLong_FromUnsignedLong(u); + if (a == NULL) { + goto error; + } + + for (int s = c_bit_length - 6; s >= 0; --s) { + PyObject *q; + int64_t e = d; + + d = c >> s; + + /* q = (n >> 2*c - e - d + 1) // a */ + q = _PyLong_Rshift(n, 2*c - d - e + 1); + if (q == NULL) { + goto error; + } + Py_SETREF(q, PyNumber_FloorDivide(q, a)); + if (q == NULL) { + goto error; + } + + /* a = (a << d - 1 - e) + q */ + Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e)); + if (a == NULL) { + Py_DECREF(q); + goto error; + } + Py_SETREF(a, PyNumber_Add(a, q)); + Py_DECREF(q); + if (a == NULL) { + goto error; + } + } + + /* The correct result is either a or a - 1. Figure out which, and + decrement a if necessary. */ + + /* a_too_large = n < a * a */ + b = PyNumber_Multiply(a, a); + if (b == NULL) { + goto error; + } + a_too_large = PyObject_RichCompareBool(n, b, Py_LT); + Py_DECREF(b); + if (a_too_large == -1) { + goto error; + } + + if (a_too_large) { + Py_SETREF(a, PyNumber_Subtract(a, _PyLong_GetOne())); + } + Py_DECREF(n); + return a; + + error: + Py_XDECREF(a); + Py_DECREF(n); + return NULL; +} + + +static unsigned long +count_set_bits(unsigned long n) +{ + unsigned long count = 0; + while (n != 0) { + ++count; + n &= n - 1; /* clear least significant bit */ + } + return count; +} + + +/* Divide-and-conquer factorial algorithm + * + * Based on the formula and pseudo-code provided at: + * http://www.luschny.de/math/factorial/binarysplitfact.html + * + * Faster algorithms exist, but they're more complicated and depend on + * a fast prime factorization algorithm. + * + * Notes on the algorithm + * ---------------------- + * + * factorial(n) is written in the form 2**k * m, with m odd. k and m are + * computed separately, and then combined using a left shift. + * + * The function factorial_odd_part computes the odd part m (i.e., the greatest + * odd divisor) of factorial(n), using the formula: + * + * factorial_odd_part(n) = + * + * product_{i >= 0} product_{0 < j <= n / 2**i, j odd} j + * + * Example: factorial_odd_part(20) = + * + * (1) * + * (1) * + * (1 * 3 * 5) * + * (1 * 3 * 5 * 7 * 9) * + * (1 * 3 * 5 * 7 * 9 * 11 * 13 * 15 * 17 * 19) + * + * Here i goes from large to small: the first term corresponds to i=4 (any + * larger i gives an empty product), and the last term corresponds to i=0. + * Each term can be computed from the last by multiplying by the extra odd + * numbers required: e.g., to get from the penultimate term to the last one, + * we multiply by (11 * 13 * 15 * 17 * 19). + * + * To see a hint of why this formula works, here are the same numbers as above + * but with the even parts (i.e., the appropriate powers of 2) included. For + * each subterm in the product for i, we multiply that subterm by 2**i: + * + * factorial(20) = + * + * (16) * + * (8) * + * (4 * 12 * 20) * + * (2 * 6 * 10 * 14 * 18) * + * (1 * 3 * 5 * 7 * 9 * 11 * 13 * 15 * 17 * 19) + * + * The factorial_partial_product function computes the product of all odd j in + * range(start, stop) for given start and stop. It's used to compute the + * partial products like (11 * 13 * 15 * 17 * 19) in the example above. It + * operates recursively, repeatedly splitting the range into two roughly equal + * pieces until the subranges are small enough to be computed using only C + * integer arithmetic. + * + * The two-valuation k (i.e., the exponent of the largest power of 2 dividing + * the factorial) is computed independently in the main math_integer_factorial + * function. By standard results, its value is: + * + * two_valuation = n//2 + n//4 + n//8 + .... + * + * It can be shown (e.g., by complete induction on n) that two_valuation is + * equal to n - count_set_bits(n), where count_set_bits(n) gives the number of + * '1'-bits in the binary expansion of n. + */ + +/* factorial_partial_product: Compute product(range(start, stop, 2)) using + * divide and conquer. Assumes start and stop are odd and stop > start. + * max_bits must be >= bit_length(stop - 2). */ + +static PyObject * +factorial_partial_product(unsigned long start, unsigned long stop, + unsigned long max_bits) +{ + unsigned long midpoint, num_operands; + PyObject *left = NULL, *right = NULL, *result = NULL; + + /* If the return value will fit an unsigned long, then we can + * multiply in a tight, fast loop where each multiply is O(1). + * Compute an upper bound on the number of bits required to store + * the answer. + * + * Storing some integer z requires floor(lg(z))+1 bits, which is + * conveniently the value returned by bit_length(z). The + * product x*y will require at most + * bit_length(x) + bit_length(y) bits to store, based + * on the idea that lg product = lg x + lg y. + * + * We know that stop - 2 is the largest number to be multiplied. From + * there, we have: bit_length(answer) <= num_operands * + * bit_length(stop - 2) + */ + + num_operands = (stop - start) / 2; + /* The "num_operands <= 8 * SIZEOF_LONG" check guards against the + * unlikely case of an overflow in num_operands * max_bits. */ + if (num_operands <= 8 * SIZEOF_LONG && + num_operands * max_bits <= 8 * SIZEOF_LONG) { + unsigned long j, total; + for (total = start, j = start + 2; j < stop; j += 2) + total *= j; + return PyLong_FromUnsignedLong(total); + } + + /* find midpoint of range(start, stop), rounded up to next odd number. */ + midpoint = (start + num_operands) | 1; + left = factorial_partial_product(start, midpoint, + _Py_bit_length(midpoint - 2)); + if (left == NULL) + goto error; + right = factorial_partial_product(midpoint, stop, max_bits); + if (right == NULL) + goto error; + result = PyNumber_Multiply(left, right); + + error: + Py_XDECREF(left); + Py_XDECREF(right); + return result; +} + +/* factorial_odd_part: compute the odd part of factorial(n). */ + +static PyObject * +factorial_odd_part(unsigned long n) +{ + long i; + unsigned long v, lower, upper; + PyObject *partial, *tmp, *inner, *outer; + + inner = PyLong_FromLong(1); + if (inner == NULL) + return NULL; + outer = Py_NewRef(inner); + + upper = 3; + for (i = _Py_bit_length(n) - 2; i >= 0; i--) { + v = n >> i; + if (v <= 2) + continue; + lower = upper; + /* (v + 1) | 1 = least odd integer strictly larger than n / 2**i */ + upper = (v + 1) | 1; + /* Here inner is the product of all odd integers j in the range (0, + n/2**(i+1)]. The factorial_partial_product call below gives the + product of all odd integers j in the range (n/2**(i+1), n/2**i]. */ + partial = factorial_partial_product(lower, upper, _Py_bit_length(upper-2)); + /* inner *= partial */ + if (partial == NULL) + goto error; + tmp = PyNumber_Multiply(inner, partial); + Py_DECREF(partial); + if (tmp == NULL) + goto error; + Py_SETREF(inner, tmp); + /* Now inner is the product of all odd integers j in the range (0, + n/2**i], giving the inner product in the formula above. */ + + /* outer *= inner; */ + tmp = PyNumber_Multiply(outer, inner); + if (tmp == NULL) + goto error; + Py_SETREF(outer, tmp); + } + Py_DECREF(inner); + return outer; + + error: + Py_DECREF(outer); + Py_DECREF(inner); + return NULL; +} + + +/* Lookup table for small factorial values */ + +static const unsigned long SmallFactorials[] = { + 1, 1, 2, 6, 24, 120, 720, 5040, 40320, + 362880, 3628800, 39916800, 479001600, +#if SIZEOF_LONG >= 8 + 6227020800, 87178291200, 1307674368000, + 20922789888000, 355687428096000, 6402373705728000, + 121645100408832000, 2432902008176640000 +#endif +}; + +/*[clinic input] +math.integer.factorial + + n as arg: object + / + +Find n!. +[clinic start generated code]*/ + +static PyObject * +math_integer_factorial(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=131c23fd48650414 input=742f4dfa490a1b07]*/ +{ + long x, two_valuation; + int overflow; + PyObject *result, *odd_part; + + x = PyLong_AsLongAndOverflow(arg, &overflow); + if (x == -1 && PyErr_Occurred()) { + return NULL; + } + else if (overflow == 1) { + PyErr_Format(PyExc_OverflowError, + "factorial() argument should not exceed %ld", + LONG_MAX); + return NULL; + } + else if (overflow == -1 || x < 0) { + PyErr_SetString(PyExc_ValueError, + "factorial() not defined for negative values"); + return NULL; + } + + /* use lookup table if x is small */ + if (x < (long)Py_ARRAY_LENGTH(SmallFactorials)) + return PyLong_FromUnsignedLong(SmallFactorials[x]); + + /* else express in the form odd_part * 2**two_valuation, and compute as + odd_part << two_valuation. */ + odd_part = factorial_odd_part(x); + if (odd_part == NULL) + return NULL; + two_valuation = x - count_set_bits(x); + result = _PyLong_Lshift(odd_part, two_valuation); + Py_DECREF(odd_part); + return result; +} + + +/* least significant 64 bits of the odd part of factorial(n), for n in range(128). + +Python code to generate the values: + + import math.integer + + for n in range(128): + fac = math.integer.factorial(n) + fac_odd_part = fac // (fac & -fac) + reduced_fac_odd_part = fac_odd_part % (2**64) + print(f"{reduced_fac_odd_part:#018x}u") +*/ +static const uint64_t reduced_factorial_odd_part[] = { + 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000003u, + 0x0000000000000003u, 0x000000000000000fu, 0x000000000000002du, 0x000000000000013bu, + 0x000000000000013bu, 0x0000000000000b13u, 0x000000000000375fu, 0x0000000000026115u, + 0x000000000007233fu, 0x00000000005cca33u, 0x0000000002898765u, 0x00000000260eeeebu, + 0x00000000260eeeebu, 0x0000000286fddd9bu, 0x00000016beecca73u, 0x000001b02b930689u, + 0x00000870d9df20adu, 0x0000b141df4dae31u, 0x00079dd498567c1bu, 0x00af2e19afc5266du, + 0x020d8a4d0f4f7347u, 0x335281867ec241efu, 0x9b3093d46fdd5923u, 0x5e1f9767cc5866b1u, + 0x92dd23d6966aced7u, 0xa30d0f4f0a196e5bu, 0x8dc3e5a1977d7755u, 0x2ab8ce915831734bu, + 0x2ab8ce915831734bu, 0x81d2a0bc5e5fdcabu, 0x9efcac82445da75bu, 0xbc8b95cf58cde171u, + 0xa0e8444a1f3cecf9u, 0x4191deb683ce3ffdu, 0xddd3878bc84ebfc7u, 0xcb39a64b83ff3751u, + 0xf8203f7993fc1495u, 0xbd2a2a78b35f4bddu, 0x84757be6b6d13921u, 0x3fbbcfc0b524988bu, + 0xbd11ed47c8928df9u, 0x3c26b59e41c2f4c5u, 0x677a5137e883fdb3u, 0xff74e943b03b93ddu, + 0xfe5ebbcb10b2bb97u, 0xb021f1de3235e7e7u, 0x33509eb2e743a58fu, 0x390f9da41279fb7du, + 0xe5cb0154f031c559u, 0x93074695ba4ddb6du, 0x81c471caa636247fu, 0xe1347289b5a1d749u, + 0x286f21c3f76ce2ffu, 0x00be84a2173e8ac7u, 0x1595065ca215b88bu, 0xf95877595b018809u, + 0x9c2efe3c5516f887u, 0x373294604679382bu, 0xaf1ff7a888adcd35u, 0x18ddf279a2c5800bu, + 0x18ddf279a2c5800bu, 0x505a90e2542582cbu, 0x5bacad2cd8d5dc2bu, 0xfe3152bcbff89f41u, + 0xe1467e88bf829351u, 0xb8001adb9e31b4d5u, 0x2803ac06a0cbb91fu, 0x1904b5d698805799u, + 0xe12a648b5c831461u, 0x3516abbd6160cfa9u, 0xac46d25f12fe036du, 0x78bfa1da906b00efu, + 0xf6390338b7f111bdu, 0x0f25f80f538255d9u, 0x4ec8ca55b8db140fu, 0x4ff670740b9b30a1u, + 0x8fd032443a07f325u, 0x80dfe7965c83eeb5u, 0xa3dc1714d1213afdu, 0x205b7bbfcdc62007u, + 0xa78126bbe140a093u, 0x9de1dc61ca7550cfu, 0x84f0046d01b492c5u, 0x2d91810b945de0f3u, + 0xf5408b7f6008aa71u, 0x43707f4863034149u, 0xdac65fb9679279d5u, 0xc48406e7d1114eb7u, + 0xa7dc9ed3c88e1271u, 0xfb25b2efdb9cb30du, 0x1bebda0951c4df63u, 0x5c85e975580ee5bdu, + 0x1591bc60082cb137u, 0x2c38606318ef25d7u, 0x76ca72f7c5c63e27u, 0xf04a75d17baa0915u, + 0x77458175139ae30du, 0x0e6c1330bc1b9421u, 0xdf87d2b5797e8293u, 0xefa5c703e1e68925u, + 0x2b6b1b3278b4f6e1u, 0xceee27b382394249u, 0xd74e3829f5dab91du, 0xfdb17989c26b5f1fu, + 0xc1b7d18781530845u, 0x7b4436b2105a8561u, 0x7ba7c0418372a7d7u, 0x9dbc5c67feb6c639u, + 0x502686d7f6ff6b8fu, 0x6101855406be7a1fu, 0x9956afb5806930e7u, 0xe1f0ee88af40f7c5u, + 0x984b057bda5c1151u, 0x9a49819acc13ea05u, 0x8ef0dead0896ef27u, 0x71f7826efe292b21u, + 0xad80a480e46986efu, 0x01cdc0ebf5e0c6f7u, 0x6e06f839968f68dbu, 0xdd5943ab56e76139u, + 0xcdcf31bf8604c5e7u, 0x7e2b4a847054a1cbu, 0x0ca75697a4d3d0f5u, 0x4703f53ac514a98bu, +}; + +/* inverses of reduced_factorial_odd_part values modulo 2**64. + +Python code to generate the values: + + import math.integer + + for n in range(128): + fac = math.integer.factorial(n) + fac_odd_part = fac // (fac & -fac) + inverted_fac_odd_part = pow(fac_odd_part, -1, 2**64) + print(f"{inverted_fac_odd_part:#018x}u") +*/ +static const uint64_t inverted_factorial_odd_part[] = { + 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000001u, 0xaaaaaaaaaaaaaaabu, + 0xaaaaaaaaaaaaaaabu, 0xeeeeeeeeeeeeeeefu, 0x4fa4fa4fa4fa4fa5u, 0x2ff2ff2ff2ff2ff3u, + 0x2ff2ff2ff2ff2ff3u, 0x938cc70553e3771bu, 0xb71c27cddd93e49fu, 0xb38e3229fcdee63du, + 0xe684bb63544a4cbfu, 0xc2f684917ca340fbu, 0xf747c9cba417526du, 0xbb26eb51d7bd49c3u, + 0xbb26eb51d7bd49c3u, 0xb0a7efb985294093u, 0xbe4b8c69f259eabbu, 0x6854d17ed6dc4fb9u, + 0xe1aa904c915f4325u, 0x3b8206df131cead1u, 0x79c6009fea76fe13u, 0xd8c5d381633cd365u, + 0x4841f12b21144677u, 0x4a91ff68200b0d0fu, 0x8f9513a58c4f9e8bu, 0x2b3e690621a42251u, + 0x4f520f00e03c04e7u, 0x2edf84ee600211d3u, 0xadcaa2764aaacdfdu, 0x161f4f9033f4fe63u, + 0x161f4f9033f4fe63u, 0xbada2932ea4d3e03u, 0xcec189f3efaa30d3u, 0xf7475bb68330bf91u, + 0x37eb7bf7d5b01549u, 0x46b35660a4e91555u, 0xa567c12d81f151f7u, 0x4c724007bb2071b1u, + 0x0f4a0cce58a016bdu, 0xfa21068e66106475u, 0x244ab72b5a318ae1u, 0x366ce67e080d0f23u, + 0xd666fdae5dd2a449u, 0xd740ddd0acc06a0du, 0xb050bbbb28e6f97bu, 0x70b003fe890a5c75u, + 0xd03aabff83037427u, 0x13ec4ca72c783bd7u, 0x90282c06afdbd96fu, 0x4414ddb9db4a95d5u, + 0xa2c68735ae6832e9u, 0xbf72d71455676665u, 0xa8469fab6b759b7fu, 0xc1e55b56e606caf9u, + 0x40455630fc4a1cffu, 0x0120a7b0046d16f7u, 0xa7c3553b08faef23u, 0x9f0bfd1b08d48639u, + 0xa433ffce9a304d37u, 0xa22ad1d53915c683u, 0xcb6cbc723ba5dd1du, 0x547fb1b8ab9d0ba3u, + 0x547fb1b8ab9d0ba3u, 0x8f15a826498852e3u, 0x32e1a03f38880283u, 0x3de4cce63283f0c1u, + 0x5dfe6667e4da95b1u, 0xfda6eeeef479e47du, 0xf14de991cc7882dfu, 0xe68db79247630ca9u, + 0xa7d6db8207ee8fa1u, 0x255e1f0fcf034499u, 0xc9a8990e43dd7e65u, 0x3279b6f289702e0fu, + 0xe7b5905d9b71b195u, 0x03025ba41ff0da69u, 0xb7df3d6d3be55aefu, 0xf89b212ebff2b361u, + 0xfe856d095996f0adu, 0xd6e533e9fdf20f9du, 0xf8c0e84a63da3255u, 0xa677876cd91b4db7u, + 0x07ed4f97780d7d9bu, 0x90a8705f258db62fu, 0xa41bbb2be31b1c0du, 0x6ec28690b038383bu, + 0xdb860c3bb2edd691u, 0x0838286838a980f9u, 0x558417a74b36f77du, 0x71779afc3646ef07u, + 0x743cda377ccb6e91u, 0x7fdf9f3fe89153c5u, 0xdc97d25df49b9a4bu, 0x76321a778eb37d95u, + 0x7cbb5e27da3bd487u, 0x9cff4ade1a009de7u, 0x70eb166d05c15197u, 0xdcf0460b71d5fe3du, + 0x5ac1ee5260b6a3c5u, 0xc922dedfdd78efe1u, 0xe5d381dc3b8eeb9bu, 0xd57e5347bafc6aadu, + 0x86939040983acd21u, 0x395b9d69740a4ff9u, 0x1467299c8e43d135u, 0x5fe440fcad975cdfu, + 0xcaa9a39794a6ca8du, 0xf61dbd640868dea1u, 0xac09d98d74843be7u, 0x2b103b9e1a6b4809u, + 0x2ab92d16960f536fu, 0x6653323d5e3681dfu, 0xefd48c1c0624e2d7u, 0xa496fefe04816f0du, + 0x1754a7b07bbdd7b1u, 0x23353c829a3852cdu, 0xbf831261abd59097u, 0x57a8e656df0618e1u, + 0x16e9206c3100680fu, 0xadad4c6ee921dac7u, 0x635f2b3860265353u, 0xdd6d0059f44b3d09u, + 0xac4dd6b894447dd7u, 0x42ea183eeaa87be3u, 0x15612d1550ee5b5du, 0x226fa19d656cb623u, +}; + +/* exponent of the largest power of 2 dividing factorial(n), for n in range(68) + +Python code to generate the values: + +import math.integer + +for n in range(128): + fac = math.integer.factorial(n) + fac_trailing_zeros = (fac & -fac).bit_length() - 1 + print(fac_trailing_zeros) +*/ + +static const uint8_t factorial_trailing_zeros[] = { + 0, 0, 1, 1, 3, 3, 4, 4, 7, 7, 8, 8, 10, 10, 11, 11, // 0-15 + 15, 15, 16, 16, 18, 18, 19, 19, 22, 22, 23, 23, 25, 25, 26, 26, // 16-31 + 31, 31, 32, 32, 34, 34, 35, 35, 38, 38, 39, 39, 41, 41, 42, 42, // 32-47 + 46, 46, 47, 47, 49, 49, 50, 50, 53, 53, 54, 54, 56, 56, 57, 57, // 48-63 + 63, 63, 64, 64, 66, 66, 67, 67, 70, 70, 71, 71, 73, 73, 74, 74, // 64-79 + 78, 78, 79, 79, 81, 81, 82, 82, 85, 85, 86, 86, 88, 88, 89, 89, // 80-95 + 94, 94, 95, 95, 97, 97, 98, 98, 101, 101, 102, 102, 104, 104, 105, 105, // 96-111 + 109, 109, 110, 110, 112, 112, 113, 113, 116, 116, 117, 117, 119, 119, 120, 120, // 112-127 +}; + +/* Number of permutations and combinations. + * P(n, k) = n! / (n-k)! + * C(n, k) = P(n, k) / k! + */ + +/* Calculate C(n, k) for n in the 63-bit range. */ +static PyObject * +perm_comb_small(unsigned long long n, unsigned long long k, int iscomb) +{ + assert(k != 0); + + /* For small enough n and k the result fits in the 64-bit range and can + * be calculated without allocating intermediate PyLong objects. */ + if (iscomb) { + /* Maps k to the maximal n so that 2*k-1 <= n <= 127 and C(n, k) + * fits into a uint64_t. Exclude k = 1, because the second fast + * path is faster for this case.*/ + static const unsigned char fast_comb_limits1[] = { + 0, 0, 127, 127, 127, 127, 127, 127, // 0-7 + 127, 127, 127, 127, 127, 127, 127, 127, // 8-15 + 116, 105, 97, 91, 86, 82, 78, 76, // 16-23 + 74, 72, 71, 70, 69, 68, 68, 67, // 24-31 + 67, 67, 67, // 32-34 + }; + if (k < Py_ARRAY_LENGTH(fast_comb_limits1) && n <= fast_comb_limits1[k]) { + /* + comb(n, k) fits into a uint64_t. We compute it as + + comb_odd_part << shift + + where 2**shift is the largest power of two dividing comb(n, k) + and comb_odd_part is comb(n, k) >> shift. comb_odd_part can be + calculated efficiently via arithmetic modulo 2**64, using three + lookups and two uint64_t multiplications. + */ + uint64_t comb_odd_part = reduced_factorial_odd_part[n] + * inverted_factorial_odd_part[k] + * inverted_factorial_odd_part[n - k]; + int shift = factorial_trailing_zeros[n] + - factorial_trailing_zeros[k] + - factorial_trailing_zeros[n - k]; + return PyLong_FromUnsignedLongLong(comb_odd_part << shift); + } + + /* Maps k to the maximal n so that 2*k-1 <= n <= 127 and C(n, k)*k + * fits into a long long (which is at least 64 bit). Only contains + * items larger than in fast_comb_limits1. */ + static const unsigned long long fast_comb_limits2[] = { + 0, ULLONG_MAX, 4294967296ULL, 3329022, 102570, 13467, 3612, 1449, // 0-7 + 746, 453, 308, 227, 178, 147, // 8-13 + }; + if (k < Py_ARRAY_LENGTH(fast_comb_limits2) && n <= fast_comb_limits2[k]) { + /* C(n, k) = C(n, k-1) * (n-k+1) / k */ + unsigned long long result = n; + for (unsigned long long i = 1; i < k;) { + result *= --n; + result /= ++i; + } + return PyLong_FromUnsignedLongLong(result); + } + } + else { + /* Maps k to the maximal n so that k <= n and P(n, k) + * fits into a long long (which is at least 64 bit). */ + static const unsigned long long fast_perm_limits[] = { + 0, ULLONG_MAX, 4294967296ULL, 2642246, 65537, 7133, 1627, 568, // 0-7 + 259, 142, 88, 61, 45, 36, 30, 26, // 8-15 + 24, 22, 21, 20, 20, // 16-20 + }; + if (k < Py_ARRAY_LENGTH(fast_perm_limits) && n <= fast_perm_limits[k]) { + if (n <= 127) { + /* P(n, k) fits into a uint64_t. */ + uint64_t perm_odd_part = reduced_factorial_odd_part[n] + * inverted_factorial_odd_part[n - k]; + int shift = factorial_trailing_zeros[n] + - factorial_trailing_zeros[n - k]; + return PyLong_FromUnsignedLongLong(perm_odd_part << shift); + } + + /* P(n, k) = P(n, k-1) * (n-k+1) */ + unsigned long long result = n; + for (unsigned long long i = 1; i < k;) { + result *= --n; + ++i; + } + return PyLong_FromUnsignedLongLong(result); + } + } + + /* For larger n use recursive formulas: + * + * P(n, k) = P(n, j) * P(n-j, k-j) + * C(n, k) = C(n, j) * C(n-j, k-j) // C(k, j) + */ + unsigned long long j = k / 2; + PyObject *a, *b; + a = perm_comb_small(n, j, iscomb); + if (a == NULL) { + return NULL; + } + b = perm_comb_small(n - j, k - j, iscomb); + if (b == NULL) { + goto error; + } + Py_SETREF(a, PyNumber_Multiply(a, b)); + Py_DECREF(b); + if (iscomb && a != NULL) { + b = perm_comb_small(k, j, 1); + if (b == NULL) { + goto error; + } + Py_SETREF(a, PyNumber_FloorDivide(a, b)); + Py_DECREF(b); + } + return a; + +error: + Py_DECREF(a); + return NULL; +} + +/* Calculate P(n, k) or C(n, k) using recursive formulas. + * It is more efficient than sequential multiplication thanks to + * Karatsuba multiplication. + */ +static PyObject * +perm_comb(PyObject *n, unsigned long long k, int iscomb) +{ + if (k == 0) { + return PyLong_FromLong(1); + } + if (k == 1) { + return Py_NewRef(n); + } + + /* P(n, k) = P(n, j) * P(n-j, k-j) */ + /* C(n, k) = C(n, j) * C(n-j, k-j) // C(k, j) */ + unsigned long long j = k / 2; + PyObject *a, *b; + a = perm_comb(n, j, iscomb); + if (a == NULL) { + return NULL; + } + PyObject *t = PyLong_FromUnsignedLongLong(j); + if (t == NULL) { + goto error; + } + n = PyNumber_Subtract(n, t); + Py_DECREF(t); + if (n == NULL) { + goto error; + } + b = perm_comb(n, k - j, iscomb); + Py_DECREF(n); + if (b == NULL) { + goto error; + } + Py_SETREF(a, PyNumber_Multiply(a, b)); + Py_DECREF(b); + if (iscomb && a != NULL) { + b = perm_comb_small(k, j, 1); + if (b == NULL) { + goto error; + } + Py_SETREF(a, PyNumber_FloorDivide(a, b)); + Py_DECREF(b); + } + return a; + +error: + Py_DECREF(a); + return NULL; +} + +/*[clinic input] +@permit_long_summary +math.integer.perm + + n: object + k: object = None + / + +Number of ways to choose k items from n items without repetition and with order. + +Evaluates to n! / (n - k)! when k <= n and evaluates +to zero when k > n. + +If k is not specified or is None, then k defaults to n +and the function returns n!. + +Raises ValueError if either of the arguments are negative. +[clinic start generated code]*/ + +static PyObject * +math_integer_perm_impl(PyObject *module, PyObject *n, PyObject *k) +/*[clinic end generated code: output=9f9b96cd73a94de4 input=fd627e5a09dd5116]*/ +{ + PyObject *result = NULL; + int overflow, cmp; + long long ki, ni; + + if (k == Py_None) { + return math_integer_factorial(module, n); + } + n = PyNumber_Index(n); + if (n == NULL) { + return NULL; + } + k = PyNumber_Index(k); + if (k == NULL) { + Py_DECREF(n); + return NULL; + } + assert(PyLong_CheckExact(n) && PyLong_CheckExact(k)); + + if (_PyLong_IsNegative((PyLongObject *)n)) { + PyErr_SetString(PyExc_ValueError, + "n must be a non-negative integer"); + goto error; + } + if (_PyLong_IsNegative((PyLongObject *)k)) { + PyErr_SetString(PyExc_ValueError, + "k must be a non-negative integer"); + goto error; + } + + cmp = PyObject_RichCompareBool(n, k, Py_LT); + if (cmp != 0) { + if (cmp > 0) { + result = PyLong_FromLong(0); + goto done; + } + goto error; + } + + ki = PyLong_AsLongLongAndOverflow(k, &overflow); + assert(overflow >= 0 && !PyErr_Occurred()); + if (overflow > 0) { + PyErr_Format(PyExc_OverflowError, + "k must not exceed %lld", + LLONG_MAX); + goto error; + } + assert(ki >= 0); + + ni = PyLong_AsLongLongAndOverflow(n, &overflow); + assert(overflow >= 0 && !PyErr_Occurred()); + if (!overflow && ki > 1) { + assert(ni >= 0); + result = perm_comb_small((unsigned long long)ni, + (unsigned long long)ki, 0); + } + else { + result = perm_comb(n, (unsigned long long)ki, 0); + } + +done: + Py_DECREF(n); + Py_DECREF(k); + return result; + +error: + Py_DECREF(n); + Py_DECREF(k); + return NULL; +} + +/*[clinic input] +@permit_long_summary +math.integer.comb + + n: object + k: object + / + +Number of ways to choose k items from n items without repetition and without order. + +Evaluates to n! / (k! * (n - k)!) when k <= n and evaluates +to zero when k > n. + +Also called the binomial coefficient because it is equivalent +to the coefficient of k-th term in polynomial expansion of the +expression (1 + x)**n. + +Raises ValueError if either of the arguments are negative. +[clinic start generated code]*/ + +static PyObject * +math_integer_comb_impl(PyObject *module, PyObject *n, PyObject *k) +/*[clinic end generated code: output=c2c9cdfe0d5dd43f input=8cc12726b682c4a5]*/ +{ + PyObject *result = NULL, *temp; + int overflow, cmp; + long long ki, ni; + + n = PyNumber_Index(n); + if (n == NULL) { + return NULL; + } + k = PyNumber_Index(k); + if (k == NULL) { + Py_DECREF(n); + return NULL; + } + assert(PyLong_CheckExact(n) && PyLong_CheckExact(k)); + + if (_PyLong_IsNegative((PyLongObject *)n)) { + PyErr_SetString(PyExc_ValueError, + "n must be a non-negative integer"); + goto error; + } + if (_PyLong_IsNegative((PyLongObject *)k)) { + PyErr_SetString(PyExc_ValueError, + "k must be a non-negative integer"); + goto error; + } + + ni = PyLong_AsLongLongAndOverflow(n, &overflow); + assert(overflow >= 0 && !PyErr_Occurred()); + if (!overflow) { + assert(ni >= 0); + ki = PyLong_AsLongLongAndOverflow(k, &overflow); + assert(overflow >= 0 && !PyErr_Occurred()); + if (overflow || ki > ni) { + result = PyLong_FromLong(0); + goto done; + } + assert(ki >= 0); + + ki = Py_MIN(ki, ni - ki); + if (ki > 1) { + result = perm_comb_small((unsigned long long)ni, + (unsigned long long)ki, 1); + goto done; + } + /* For k == 1 just return the original n in perm_comb(). */ + } + else { + /* k = min(k, n - k) */ + temp = PyNumber_Subtract(n, k); + if (temp == NULL) { + goto error; + } + assert(PyLong_Check(temp)); + if (_PyLong_IsNegative((PyLongObject *)temp)) { + Py_DECREF(temp); + result = PyLong_FromLong(0); + goto done; + } + cmp = PyObject_RichCompareBool(temp, k, Py_LT); + if (cmp > 0) { + Py_SETREF(k, temp); + } + else { + Py_DECREF(temp); + if (cmp < 0) { + goto error; + } + } + + ki = PyLong_AsLongLongAndOverflow(k, &overflow); + assert(overflow >= 0 && !PyErr_Occurred()); + if (overflow) { + PyErr_Format(PyExc_OverflowError, + "min(n - k, k) must not exceed %lld", + LLONG_MAX); + goto error; + } + assert(ki >= 0); + } + + result = perm_comb(n, (unsigned long long)ki, 1); + +done: + Py_DECREF(n); + Py_DECREF(k); + return result; + +error: + Py_DECREF(n); + Py_DECREF(k); + return NULL; +} + + +static PyMethodDef math_integer_methods[] = { + MATH_INTEGER_COMB_METHODDEF + MATH_INTEGER_FACTORIAL_METHODDEF + MATH_INTEGER_GCD_METHODDEF + MATH_INTEGER_ISQRT_METHODDEF + MATH_INTEGER_LCM_METHODDEF + MATH_INTEGER_PERM_METHODDEF + {NULL, NULL} /* sentinel */ +}; + +static int +math_integer_exec(PyObject *module) +{ + /* Fix the __name__ attribute of the module and the __module__ attribute + * of its functions. + */ + PyObject *name = PyUnicode_FromString("math.integer"); + if (name == NULL) { + return -1; + } + if (PyObject_SetAttrString(module, "__name__", name) < 0) { + Py_DECREF(name); + return -1; + } + for (const PyMethodDef *m = math_integer_methods; m->ml_name; m++) { + PyObject *obj = PyObject_GetAttrString(module, m->ml_name); + if (obj == NULL) { + Py_DECREF(name); + return -1; + } + if (PyObject_SetAttrString(obj, "__module__", name) < 0) { + Py_DECREF(name); + Py_DECREF(obj); + return -1; + } + Py_DECREF(obj); + } + Py_DECREF(name); + return 0; +} + +static PyModuleDef_Slot math_integer_slots[] = { + {Py_mod_exec, math_integer_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} +}; + +PyDoc_STRVAR(module_doc, +"This module provides access to integer related mathematical functions."); + +static struct PyModuleDef math_integer_module = { + PyModuleDef_HEAD_INIT, + .m_name = "math.integer", + .m_doc = module_doc, + .m_size = 0, + .m_methods = math_integer_methods, + .m_slots = math_integer_slots, +}; + +PyMODINIT_FUNC +PyInit__math_integer(void) +{ + return PyModuleDef_Init(&math_integer_module); +} diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 11d9b7418a2..11c46c987e1 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -60,6 +60,7 @@ raised for division by zero and mod by zero. #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_bitutils.h" // _Py_bit_length() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_import.h" // _PyImport_SetModuleString() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyObject_LookupSpecial() @@ -394,7 +395,7 @@ m_tgamma(double x) if (x == 0.0) { errno = EDOM; /* tgamma(+-0.0) = +-inf, divide-by-zero */ - return copysign(Py_INFINITY, x); + return copysign(INFINITY, x); } /* integer arguments */ @@ -425,7 +426,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_INFINITY; + return INFINITY; } } @@ -489,14 +490,14 @@ m_lgamma(double x) if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_INFINITY; /* lgamma(+-inf) = +inf */ + return INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_INFINITY; /* integers n <= 0 */ + return INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -632,7 +633,7 @@ m_log(double x) return log(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log(0) = -inf */ + return -INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } @@ -675,7 +676,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ + return -INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -691,7 +692,7 @@ m_log10(double x) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log10(0) = -inf */ + return -INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } @@ -706,140 +707,6 @@ m_log10(double x) } -/*[clinic input] -math.gcd - - *integers as args: array - -Greatest Common Divisor. -[clinic start generated code]*/ - -static PyObject * -math_gcd_impl(PyObject *module, PyObject * const *args, - Py_ssize_t args_length) -/*[clinic end generated code: output=a26c95907374ffb4 input=ded7f0ea3850c05c]*/ -{ - // Fast-path for the common case: gcd(int, int) - if (args_length == 2 && PyLong_CheckExact(args[0]) && PyLong_CheckExact(args[1])) - { - return _PyLong_GCD(args[0], args[1]); - } - - if (args_length == 0) { - return PyLong_FromLong(0); - } - - PyObject *res = PyNumber_Index(args[0]); - if (res == NULL) { - return NULL; - } - if (args_length == 1) { - Py_SETREF(res, PyNumber_Absolute(res)); - return res; - } - - PyObject *one = _PyLong_GetOne(); // borrowed ref - for (Py_ssize_t i = 1; i < args_length; i++) { - PyObject *x = _PyNumber_Index(args[i]); - if (x == NULL) { - Py_DECREF(res); - return NULL; - } - if (res == one) { - /* Fast path: just check arguments. - It is okay to use identity comparison here. */ - Py_DECREF(x); - continue; - } - Py_SETREF(res, _PyLong_GCD(res, x)); - Py_DECREF(x); - if (res == NULL) { - return NULL; - } - } - return res; -} - - -static PyObject * -long_lcm(PyObject *a, PyObject *b) -{ - PyObject *g, *m, *f, *ab; - - if (_PyLong_IsZero((PyLongObject *)a) || _PyLong_IsZero((PyLongObject *)b)) { - return PyLong_FromLong(0); - } - g = _PyLong_GCD(a, b); - if (g == NULL) { - return NULL; - } - f = PyNumber_FloorDivide(a, g); - Py_DECREF(g); - if (f == NULL) { - return NULL; - } - m = PyNumber_Multiply(f, b); - Py_DECREF(f); - if (m == NULL) { - return NULL; - } - ab = PyNumber_Absolute(m); - Py_DECREF(m); - return ab; -} - - -/*[clinic input] -math.lcm - - *integers as args: array - -Least Common Multiple. -[clinic start generated code]*/ - -static PyObject * -math_lcm_impl(PyObject *module, PyObject * const *args, - Py_ssize_t args_length) -/*[clinic end generated code: output=c8a59a5c2e55c816 input=3e4f4b7cdf948a98]*/ -{ - PyObject *res, *x; - Py_ssize_t i; - - if (args_length == 0) { - return PyLong_FromLong(1); - } - res = PyNumber_Index(args[0]); - if (res == NULL) { - return NULL; - } - if (args_length == 1) { - Py_SETREF(res, PyNumber_Absolute(res)); - return res; - } - - PyObject *zero = _PyLong_GetZero(); // borrowed ref - for (i = 1; i < args_length; i++) { - x = PyNumber_Index(args[i]); - if (x == NULL) { - Py_DECREF(res); - return NULL; - } - if (res == zero) { - /* Fast path: just check arguments. - It is okay to use identity comparison here. */ - Py_DECREF(x); - continue; - } - Py_SETREF(res, long_lcm(res, x)); - Py_DECREF(x); - if (res == NULL) { - return NULL; - } - } - return res; -} - - /* Call is_error when errno != 0, and where x is the result libm * returned. is_error will usually set up an exception and return * true (1), but may return false (0) without setting up an exception. @@ -1214,6 +1081,40 @@ math_floor(PyObject *module, PyObject *number) return PyLong_FromDouble(floor(x)); } +/*[clinic input] +math.fmax -> double + + x: double + y: double + / + +Return the larger of two floating-point arguments. +[clinic start generated code]*/ + +static double +math_fmax_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=00692358d312fee2 input=021596c027336ffe]*/ +{ + return fmax(x, y); +} + +/*[clinic input] +math.fmin -> double + + x: double + y: double + / + +Return the smaller of two floating-point arguments. +[clinic start generated code]*/ + +static double +math_fmin_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=3d5b7826bd292dd9 input=d12e64ccc33f878a]*/ +{ + return fmin(x, y); +} + FUNC1AD(gamma, m_tgamma, "gamma($module, x, /)\n--\n\n" "Gamma function at x.", @@ -1233,6 +1134,23 @@ FUNC2(remainder, m_remainder, "Return x - n*y where n*y is the closest integer multiple of y.\n" "In the case where x is exactly halfway between two multiples of\n" "y, the nearest even value of n is used. The result is always exact.") + +/*[clinic input] +math.signbit + + x: double + / + +Return True if the sign of x is negative and False otherwise. +[clinic start generated code]*/ + +static PyObject * +math_signbit_impl(PyObject *module, double x) +/*[clinic end generated code: output=20c5f20156a9b871 input=3d3493fbcb5bdb3e]*/ +{ + return PyBool_FromLong(signbit(x)); +} + FUNC1D(sin, sin, 0, "sin($module, x, /)\n--\n\n" "Return the sine of x (measured in radians).", @@ -1480,578 +1398,6 @@ math_fsum(PyObject *module, PyObject *seq) #undef NUM_PARTIALS -static unsigned long -count_set_bits(unsigned long n) -{ - unsigned long count = 0; - while (n != 0) { - ++count; - n &= n - 1; /* clear least significant bit */ - } - return count; -} - -/* Integer square root - -Given a nonnegative integer `n`, we want to compute the largest integer -`a` for which `a * a <= n`, or equivalently the integer part of the exact -square root of `n`. - -We use an adaptive-precision pure-integer version of Newton's iteration. Given -a positive integer `n`, the algorithm produces at each iteration an integer -approximation `a` to the square root of `n >> s` for some even integer `s`, -with `s` decreasing as the iterations progress. On the final iteration, `s` is -zero and we have an approximation to the square root of `n` itself. - -At every step, the approximation `a` is strictly within 1.0 of the true square -root, so we have - - (a - 1)**2 < (n >> s) < (a + 1)**2 - -After the final iteration, a check-and-correct step is needed to determine -whether `a` or `a - 1` gives the desired integer square root of `n`. - -The algorithm is remarkable in its simplicity. There's no need for a -per-iteration check-and-correct step, and termination is straightforward: the -number of iterations is known in advance (it's exactly `floor(log2(log2(n)))` -for `n > 1`). The only tricky part of the correctness proof is in establishing -that the bound `(a - 1)**2 < (n >> s) < (a + 1)**2` is maintained from one -iteration to the next. A sketch of the proof of this is given below. - -In addition to the proof sketch, a formal, computer-verified proof -of correctness (using Lean) of an equivalent recursive algorithm can be found -here: - - https://github.com/mdickinson/snippets/blob/master/proofs/isqrt/src/isqrt.lean - - -Here's Python code equivalent to the C implementation below: - - def isqrt(n): - """ - Return the integer part of the square root of the input. - """ - n = operator.index(n) - - if n < 0: - raise ValueError("isqrt() argument must be nonnegative") - if n == 0: - return 0 - - c = (n.bit_length() - 1) // 2 - a = 1 - d = 0 - for s in reversed(range(c.bit_length())): - # Loop invariant: (a-1)**2 < (n >> 2*(c - d)) < (a+1)**2 - e = d - d = c >> s - a = (a << d - e - 1) + (n >> 2*c - e - d + 1) // a - - return a - (a*a > n) - - -Sketch of proof of correctness ------------------------------- - -The delicate part of the correctness proof is showing that the loop invariant -is preserved from one iteration to the next. That is, just before the line - - a = (a << d - e - 1) + (n >> 2*c - e - d + 1) // a - -is executed in the above code, we know that - - (1) (a - 1)**2 < (n >> 2*(c - e)) < (a + 1)**2. - -(since `e` is always the value of `d` from the previous iteration). We must -prove that after that line is executed, we have - - (a - 1)**2 < (n >> 2*(c - d)) < (a + 1)**2 - -To facilitate the proof, we make some changes of notation. Write `m` for -`n >> 2*(c-d)`, and write `b` for the new value of `a`, so - - b = (a << d - e - 1) + (n >> 2*c - e - d + 1) // a - -or equivalently: - - (2) b = (a << d - e - 1) + (m >> d - e + 1) // a - -Then we can rewrite (1) as: - - (3) (a - 1)**2 < (m >> 2*(d - e)) < (a + 1)**2 - -and we must show that (b - 1)**2 < m < (b + 1)**2. - -From this point on, we switch to mathematical notation, so `/` means exact -division rather than integer division and `^` is used for exponentiation. We -use the `√` symbol for the exact square root. In (3), we can remove the -implicit floor operation to give: - - (4) (a - 1)^2 < m / 4^(d - e) < (a + 1)^2 - -Taking square roots throughout (4), scaling by `2^(d-e)`, and rearranging gives - - (5) 0 <= | 2^(d-e)a - √m | < 2^(d-e) - -Squaring and dividing through by `2^(d-e+1) a` gives - - (6) 0 <= 2^(d-e-1) a + m / (2^(d-e+1) a) - √m < 2^(d-e-1) / a - -We'll show below that `2^(d-e-1) <= a`. Given that, we can replace the -right-hand side of (6) with `1`, and now replacing the central -term `m / (2^(d-e+1) a)` with its floor in (6) gives - - (7) -1 < 2^(d-e-1) a + m // 2^(d-e+1) a - √m < 1 - -Or equivalently, from (2): - - (7) -1 < b - √m < 1 - -and rearranging gives that `(b-1)^2 < m < (b+1)^2`, which is what we needed -to prove. - -We're not quite done: we still have to prove the inequality `2^(d - e - 1) <= -a` that was used to get line (7) above. From the definition of `c`, we have -`4^c <= n`, which implies - - (8) 4^d <= m - -also, since `e == d >> 1`, `d` is at most `2e + 1`, from which it follows -that `2d - 2e - 1 <= d` and hence that - - (9) 4^(2d - 2e - 1) <= m - -Dividing both sides by `4^(d - e)` gives - - (10) 4^(d - e - 1) <= m / 4^(d - e) - -But we know from (4) that `m / 4^(d-e) < (a + 1)^2`, hence - - (11) 4^(d - e - 1) < (a + 1)^2 - -Now taking square roots of both sides and observing that both `2^(d-e-1)` and -`a` are integers gives `2^(d - e - 1) <= a`, which is what we needed. This -completes the proof sketch. - -*/ - -/* - The _approximate_isqrt_tab table provides approximate square roots for - 16-bit integers. For any n in the range 2**14 <= n < 2**16, the value - - a = _approximate_isqrt_tab[(n >> 8) - 64] - - is an approximate square root of n, satisfying (a - 1)**2 < n < (a + 1)**2. - - The table was computed in Python using the expression: - - [min(round(sqrt(256*n + 128)), 255) for n in range(64, 256)] -*/ - -static const uint8_t _approximate_isqrt_tab[192] = { - 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, - 140, 141, 142, 143, 144, 144, 145, 146, 147, 148, 149, 150, - 151, 151, 152, 153, 154, 155, 156, 156, 157, 158, 159, 160, - 160, 161, 162, 163, 164, 164, 165, 166, 167, 167, 168, 169, - 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, - 179, 179, 180, 181, 181, 182, 183, 183, 184, 185, 186, 186, - 187, 188, 188, 189, 190, 190, 191, 192, 192, 193, 194, 194, - 195, 196, 196, 197, 198, 198, 199, 200, 200, 201, 201, 202, - 203, 203, 204, 205, 205, 206, 206, 207, 208, 208, 209, 210, - 210, 211, 211, 212, 213, 213, 214, 214, 215, 216, 216, 217, - 217, 218, 219, 219, 220, 220, 221, 221, 222, 223, 223, 224, - 224, 225, 225, 226, 227, 227, 228, 228, 229, 229, 230, 230, - 231, 232, 232, 233, 233, 234, 234, 235, 235, 236, 237, 237, - 238, 238, 239, 239, 240, 240, 241, 241, 242, 242, 243, 243, - 244, 244, 245, 246, 246, 247, 247, 248, 248, 249, 249, 250, - 250, 251, 251, 252, 252, 253, 253, 254, 254, 255, 255, 255, -}; - -/* Approximate square root of a large 64-bit integer. - - Given `n` satisfying `2**62 <= n < 2**64`, return `a` - satisfying `(a - 1)**2 < n < (a + 1)**2`. */ - -static inline uint32_t -_approximate_isqrt(uint64_t n) -{ - uint32_t u = _approximate_isqrt_tab[(n >> 56) - 64]; - u = (u << 7) + (uint32_t)(n >> 41) / u; - return (u << 15) + (uint32_t)((n >> 17) / u); -} - -/*[clinic input] -math.isqrt - - n: object - / - -Return the integer part of the square root of the input. -[clinic start generated code]*/ - -static PyObject * -math_isqrt(PyObject *module, PyObject *n) -/*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/ -{ - int a_too_large, c_bit_length; - int64_t c, d; - uint64_t m; - uint32_t u; - PyObject *a = NULL, *b; - - n = _PyNumber_Index(n); - if (n == NULL) { - return NULL; - } - - if (_PyLong_IsNegative((PyLongObject *)n)) { - PyErr_SetString( - PyExc_ValueError, - "isqrt() argument must be nonnegative"); - goto error; - } - if (_PyLong_IsZero((PyLongObject *)n)) { - Py_DECREF(n); - return PyLong_FromLong(0); - } - - /* c = (n.bit_length() - 1) // 2 */ - c = _PyLong_NumBits(n); - assert(c > 0); - assert(!PyErr_Occurred()); - c = (c - 1) / 2; - - /* Fast path: if c <= 31 then n < 2**64 and we can compute directly with a - fast, almost branch-free algorithm. */ - if (c <= 31) { - int shift = 31 - (int)c; - m = (uint64_t)PyLong_AsUnsignedLongLong(n); - Py_DECREF(n); - if (m == (uint64_t)(-1) && PyErr_Occurred()) { - return NULL; - } - u = _approximate_isqrt(m << 2*shift) >> shift; - u -= (uint64_t)u * u > m; - return PyLong_FromUnsignedLong(u); - } - - /* Slow path: n >= 2**64. We perform the first five iterations in C integer - arithmetic, then switch to using Python long integers. */ - - /* From n >= 2**64 it follows that c.bit_length() >= 6. */ - c_bit_length = 6; - while ((c >> c_bit_length) > 0) { - ++c_bit_length; - } - - /* Initialise d and a. */ - d = c >> (c_bit_length - 5); - b = _PyLong_Rshift(n, 2*c - 62); - if (b == NULL) { - goto error; - } - m = (uint64_t)PyLong_AsUnsignedLongLong(b); - Py_DECREF(b); - if (m == (uint64_t)(-1) && PyErr_Occurred()) { - goto error; - } - u = _approximate_isqrt(m) >> (31U - d); - a = PyLong_FromUnsignedLong(u); - if (a == NULL) { - goto error; - } - - for (int s = c_bit_length - 6; s >= 0; --s) { - PyObject *q; - int64_t e = d; - - d = c >> s; - - /* q = (n >> 2*c - e - d + 1) // a */ - q = _PyLong_Rshift(n, 2*c - d - e + 1); - if (q == NULL) { - goto error; - } - Py_SETREF(q, PyNumber_FloorDivide(q, a)); - if (q == NULL) { - goto error; - } - - /* a = (a << d - 1 - e) + q */ - Py_SETREF(a, _PyLong_Lshift(a, d - 1 - e)); - if (a == NULL) { - Py_DECREF(q); - goto error; - } - Py_SETREF(a, PyNumber_Add(a, q)); - Py_DECREF(q); - if (a == NULL) { - goto error; - } - } - - /* The correct result is either a or a - 1. Figure out which, and - decrement a if necessary. */ - - /* a_too_large = n < a * a */ - b = PyNumber_Multiply(a, a); - if (b == NULL) { - goto error; - } - a_too_large = PyObject_RichCompareBool(n, b, Py_LT); - Py_DECREF(b); - if (a_too_large == -1) { - goto error; - } - - if (a_too_large) { - Py_SETREF(a, PyNumber_Subtract(a, _PyLong_GetOne())); - } - Py_DECREF(n); - return a; - - error: - Py_XDECREF(a); - Py_DECREF(n); - return NULL; -} - -/* Divide-and-conquer factorial algorithm - * - * Based on the formula and pseudo-code provided at: - * http://www.luschny.de/math/factorial/binarysplitfact.html - * - * Faster algorithms exist, but they're more complicated and depend on - * a fast prime factorization algorithm. - * - * Notes on the algorithm - * ---------------------- - * - * factorial(n) is written in the form 2**k * m, with m odd. k and m are - * computed separately, and then combined using a left shift. - * - * The function factorial_odd_part computes the odd part m (i.e., the greatest - * odd divisor) of factorial(n), using the formula: - * - * factorial_odd_part(n) = - * - * product_{i >= 0} product_{0 < j <= n / 2**i, j odd} j - * - * Example: factorial_odd_part(20) = - * - * (1) * - * (1) * - * (1 * 3 * 5) * - * (1 * 3 * 5 * 7 * 9) * - * (1 * 3 * 5 * 7 * 9 * 11 * 13 * 15 * 17 * 19) - * - * Here i goes from large to small: the first term corresponds to i=4 (any - * larger i gives an empty product), and the last term corresponds to i=0. - * Each term can be computed from the last by multiplying by the extra odd - * numbers required: e.g., to get from the penultimate term to the last one, - * we multiply by (11 * 13 * 15 * 17 * 19). - * - * To see a hint of why this formula works, here are the same numbers as above - * but with the even parts (i.e., the appropriate powers of 2) included. For - * each subterm in the product for i, we multiply that subterm by 2**i: - * - * factorial(20) = - * - * (16) * - * (8) * - * (4 * 12 * 20) * - * (2 * 6 * 10 * 14 * 18) * - * (1 * 3 * 5 * 7 * 9 * 11 * 13 * 15 * 17 * 19) - * - * The factorial_partial_product function computes the product of all odd j in - * range(start, stop) for given start and stop. It's used to compute the - * partial products like (11 * 13 * 15 * 17 * 19) in the example above. It - * operates recursively, repeatedly splitting the range into two roughly equal - * pieces until the subranges are small enough to be computed using only C - * integer arithmetic. - * - * The two-valuation k (i.e., the exponent of the largest power of 2 dividing - * the factorial) is computed independently in the main math_factorial - * function. By standard results, its value is: - * - * two_valuation = n//2 + n//4 + n//8 + .... - * - * It can be shown (e.g., by complete induction on n) that two_valuation is - * equal to n - count_set_bits(n), where count_set_bits(n) gives the number of - * '1'-bits in the binary expansion of n. - */ - -/* factorial_partial_product: Compute product(range(start, stop, 2)) using - * divide and conquer. Assumes start and stop are odd and stop > start. - * max_bits must be >= bit_length(stop - 2). */ - -static PyObject * -factorial_partial_product(unsigned long start, unsigned long stop, - unsigned long max_bits) -{ - unsigned long midpoint, num_operands; - PyObject *left = NULL, *right = NULL, *result = NULL; - - /* If the return value will fit an unsigned long, then we can - * multiply in a tight, fast loop where each multiply is O(1). - * Compute an upper bound on the number of bits required to store - * the answer. - * - * Storing some integer z requires floor(lg(z))+1 bits, which is - * conveniently the value returned by bit_length(z). The - * product x*y will require at most - * bit_length(x) + bit_length(y) bits to store, based - * on the idea that lg product = lg x + lg y. - * - * We know that stop - 2 is the largest number to be multiplied. From - * there, we have: bit_length(answer) <= num_operands * - * bit_length(stop - 2) - */ - - num_operands = (stop - start) / 2; - /* The "num_operands <= 8 * SIZEOF_LONG" check guards against the - * unlikely case of an overflow in num_operands * max_bits. */ - if (num_operands <= 8 * SIZEOF_LONG && - num_operands * max_bits <= 8 * SIZEOF_LONG) { - unsigned long j, total; - for (total = start, j = start + 2; j < stop; j += 2) - total *= j; - return PyLong_FromUnsignedLong(total); - } - - /* find midpoint of range(start, stop), rounded up to next odd number. */ - midpoint = (start + num_operands) | 1; - left = factorial_partial_product(start, midpoint, - _Py_bit_length(midpoint - 2)); - if (left == NULL) - goto error; - right = factorial_partial_product(midpoint, stop, max_bits); - if (right == NULL) - goto error; - result = PyNumber_Multiply(left, right); - - error: - Py_XDECREF(left); - Py_XDECREF(right); - return result; -} - -/* factorial_odd_part: compute the odd part of factorial(n). */ - -static PyObject * -factorial_odd_part(unsigned long n) -{ - long i; - unsigned long v, lower, upper; - PyObject *partial, *tmp, *inner, *outer; - - inner = PyLong_FromLong(1); - if (inner == NULL) - return NULL; - outer = Py_NewRef(inner); - - upper = 3; - for (i = _Py_bit_length(n) - 2; i >= 0; i--) { - v = n >> i; - if (v <= 2) - continue; - lower = upper; - /* (v + 1) | 1 = least odd integer strictly larger than n / 2**i */ - upper = (v + 1) | 1; - /* Here inner is the product of all odd integers j in the range (0, - n/2**(i+1)]. The factorial_partial_product call below gives the - product of all odd integers j in the range (n/2**(i+1), n/2**i]. */ - partial = factorial_partial_product(lower, upper, _Py_bit_length(upper-2)); - /* inner *= partial */ - if (partial == NULL) - goto error; - tmp = PyNumber_Multiply(inner, partial); - Py_DECREF(partial); - if (tmp == NULL) - goto error; - Py_SETREF(inner, tmp); - /* Now inner is the product of all odd integers j in the range (0, - n/2**i], giving the inner product in the formula above. */ - - /* outer *= inner; */ - tmp = PyNumber_Multiply(outer, inner); - if (tmp == NULL) - goto error; - Py_SETREF(outer, tmp); - } - Py_DECREF(inner); - return outer; - - error: - Py_DECREF(outer); - Py_DECREF(inner); - return NULL; -} - - -/* Lookup table for small factorial values */ - -static const unsigned long SmallFactorials[] = { - 1, 1, 2, 6, 24, 120, 720, 5040, 40320, - 362880, 3628800, 39916800, 479001600, -#if SIZEOF_LONG >= 8 - 6227020800, 87178291200, 1307674368000, - 20922789888000, 355687428096000, 6402373705728000, - 121645100408832000, 2432902008176640000 -#endif -}; - -/*[clinic input] -math.factorial - - n as arg: object - / - -Find n!. - -Raise a ValueError if x is negative or non-integral. -[clinic start generated code]*/ - -static PyObject * -math_factorial(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=6686f26fae00e9ca input=713fb771677e8c31]*/ -{ - long x, two_valuation; - int overflow; - PyObject *result, *odd_part; - - x = PyLong_AsLongAndOverflow(arg, &overflow); - if (x == -1 && PyErr_Occurred()) { - return NULL; - } - else if (overflow == 1) { - PyErr_Format(PyExc_OverflowError, - "factorial() argument should not exceed %ld", - LONG_MAX); - return NULL; - } - else if (overflow == -1 || x < 0) { - PyErr_SetString(PyExc_ValueError, - "factorial() not defined for negative values"); - return NULL; - } - - /* use lookup table if x is small */ - if (x < (long)Py_ARRAY_LENGTH(SmallFactorials)) - return PyLong_FromUnsignedLong(SmallFactorials[x]); - - /* else express in the form odd_part * 2**two_valuation, and compute as - odd_part << two_valuation. */ - odd_part = factorial_odd_part(x); - if (odd_part == NULL) - return NULL; - two_valuation = x - count_set_bits(x); - result = _PyLong_Lshift(odd_part, two_valuation); - Py_DECREF(odd_part); - return result; -} - - /*[clinic input] math.trunc @@ -2154,7 +1500,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_INFINITY, x); + r = copysign(INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2163,6 +1509,27 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) } else { errno = 0; r = ldexp(x, (int)exp); +#ifdef _MSC_VER + if (DBL_MIN > r && r > -DBL_MIN) { + /* Denormal (or zero) results can be incorrectly rounded here (rather, + truncated). Fixed in newer versions of the C runtime, included + with Windows 11. */ + int original_exp; + frexp(x, &original_exp); + if (original_exp > DBL_MIN_EXP) { + /* Shift down to the smallest normal binade. No bits lost. */ + int shift = DBL_MIN_EXP - original_exp; + x = ldexp(x, shift); + exp -= shift; + } + /* Multiplying by 2**exp finishes the job, and the HW will round as + appropriate. Note: if exp < -DBL_MANT_DIG, all of x is shifted + to be < 0.5ULP of smallest denorm, so should be thrown away. If + exp is so very negative that ldexp underflows to 0, that's fine; + no need to check in advance. */ + r = x*ldexp(1.0, (int)exp); + } +#endif if (isinf(r)) errno = ERANGE; } @@ -2211,44 +1578,63 @@ math_modf_impl(PyObject *module, double x) However, intermediate overflow is possible for an int if the number of bits in that int is larger than PY_SSIZE_T_MAX. */ +static PyObject* +loghelper_int(PyObject* arg, double (*func)(double)) +{ + /* If it is int, do it ourselves. */ + double x, result; + int64_t e; + + /* Negative or zero inputs give a ValueError. */ + if (!_PyLong_IsPositive((PyLongObject *)arg)) { + PyErr_SetString(PyExc_ValueError, + "expected a positive input"); + return NULL; + } + + x = PyLong_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception and continue. */ + PyErr_Clear(); + x = _PyLong_Frexp((PyLongObject *)arg, &e); + assert(!PyErr_Occurred()); + /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ + result = fma(func(2.0), (double)e, func(x)); + } + else + /* Successfully converted x to a double. */ + result = func(x); + return PyFloat_FromDouble(result); +} + static PyObject* loghelper(PyObject* arg, double (*func)(double)) { /* If it is int, do it ourselves. */ if (PyLong_Check(arg)) { - double x, result; - int64_t e; - - /* Negative or zero inputs give a ValueError. */ - if (!_PyLong_IsPositive((PyLongObject *)arg)) { - /* The input can be an arbitrary large integer, so we - don't include it's value in the error message. */ - PyErr_SetString(PyExc_ValueError, - "expected a positive input"); + return loghelper_int(arg, func); + } + /* Else let libm handle it by itself. */ + PyObject *res = math_1(arg, func, 0, "expected a positive input, got %s"); + if (res == NULL && + PyErr_ExceptionMatches(PyExc_OverflowError) && + PyIndex_Check(arg)) + { + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception, convert to + integer and continue. */ + PyErr_Clear(); + arg = _PyNumber_Index(arg); + if (arg == NULL) { return NULL; } - - x = PyLong_AsDouble(arg); - if (x == -1.0 && PyErr_Occurred()) { - if (!PyErr_ExceptionMatches(PyExc_OverflowError)) - return NULL; - /* Here the conversion to double overflowed, but it's possible - to compute the log anyway. Clear the exception and continue. */ - PyErr_Clear(); - x = _PyLong_Frexp((PyLongObject *)arg, &e); - assert(e >= 0); - assert(!PyErr_Occurred()); - /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ - result = func(x) + func(2.0) * e; - } - else - /* Successfully converted x to a double. */ - result = func(x); - return PyFloat_FromDouble(result); + res = loghelper_int(arg, func); + Py_DECREF(arg); } - - /* Else let libm handle it by itself. */ - return math_1(arg, func, 0, "expected a positive input, got %s"); + return res; } @@ -3099,6 +2485,44 @@ math_isfinite_impl(PyObject *module, double x) } +/*[clinic input] +math.isnormal + + x: double + / + +Return True if x is normal, and False otherwise. +[clinic start generated code]*/ + +static PyObject * +math_isnormal_impl(PyObject *module, double x) +/*[clinic end generated code: output=c7b302b5b89c3541 input=fdaa00c58aa7bc17]*/ +{ + return PyBool_FromLong(isnormal(x)); +} + + +/*[clinic input] +math.issubnormal + + x: double + / + +Return True if x is subnormal, and False otherwise. +[clinic start generated code]*/ + +static PyObject * +math_issubnormal_impl(PyObject *module, double x) +/*[clinic end generated code: output=4e76ac98ddcae761 input=9a20aba7107d0d95]*/ +{ +#if !defined(_MSC_VER) && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L + return PyBool_FromLong(issubnormal(x)); +#else + return PyBool_FromLong(isfinite(x) && x && !isnormal(x)); +#endif +} + + /*[clinic input] math.isnan @@ -3406,510 +2830,8 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) } -/* least significant 64 bits of the odd part of factorial(n), for n in range(128). - -Python code to generate the values: - - import math - - for n in range(128): - fac = math.factorial(n) - fac_odd_part = fac // (fac & -fac) - reduced_fac_odd_part = fac_odd_part % (2**64) - print(f"{reduced_fac_odd_part:#018x}u") -*/ -static const uint64_t reduced_factorial_odd_part[] = { - 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000003u, - 0x0000000000000003u, 0x000000000000000fu, 0x000000000000002du, 0x000000000000013bu, - 0x000000000000013bu, 0x0000000000000b13u, 0x000000000000375fu, 0x0000000000026115u, - 0x000000000007233fu, 0x00000000005cca33u, 0x0000000002898765u, 0x00000000260eeeebu, - 0x00000000260eeeebu, 0x0000000286fddd9bu, 0x00000016beecca73u, 0x000001b02b930689u, - 0x00000870d9df20adu, 0x0000b141df4dae31u, 0x00079dd498567c1bu, 0x00af2e19afc5266du, - 0x020d8a4d0f4f7347u, 0x335281867ec241efu, 0x9b3093d46fdd5923u, 0x5e1f9767cc5866b1u, - 0x92dd23d6966aced7u, 0xa30d0f4f0a196e5bu, 0x8dc3e5a1977d7755u, 0x2ab8ce915831734bu, - 0x2ab8ce915831734bu, 0x81d2a0bc5e5fdcabu, 0x9efcac82445da75bu, 0xbc8b95cf58cde171u, - 0xa0e8444a1f3cecf9u, 0x4191deb683ce3ffdu, 0xddd3878bc84ebfc7u, 0xcb39a64b83ff3751u, - 0xf8203f7993fc1495u, 0xbd2a2a78b35f4bddu, 0x84757be6b6d13921u, 0x3fbbcfc0b524988bu, - 0xbd11ed47c8928df9u, 0x3c26b59e41c2f4c5u, 0x677a5137e883fdb3u, 0xff74e943b03b93ddu, - 0xfe5ebbcb10b2bb97u, 0xb021f1de3235e7e7u, 0x33509eb2e743a58fu, 0x390f9da41279fb7du, - 0xe5cb0154f031c559u, 0x93074695ba4ddb6du, 0x81c471caa636247fu, 0xe1347289b5a1d749u, - 0x286f21c3f76ce2ffu, 0x00be84a2173e8ac7u, 0x1595065ca215b88bu, 0xf95877595b018809u, - 0x9c2efe3c5516f887u, 0x373294604679382bu, 0xaf1ff7a888adcd35u, 0x18ddf279a2c5800bu, - 0x18ddf279a2c5800bu, 0x505a90e2542582cbu, 0x5bacad2cd8d5dc2bu, 0xfe3152bcbff89f41u, - 0xe1467e88bf829351u, 0xb8001adb9e31b4d5u, 0x2803ac06a0cbb91fu, 0x1904b5d698805799u, - 0xe12a648b5c831461u, 0x3516abbd6160cfa9u, 0xac46d25f12fe036du, 0x78bfa1da906b00efu, - 0xf6390338b7f111bdu, 0x0f25f80f538255d9u, 0x4ec8ca55b8db140fu, 0x4ff670740b9b30a1u, - 0x8fd032443a07f325u, 0x80dfe7965c83eeb5u, 0xa3dc1714d1213afdu, 0x205b7bbfcdc62007u, - 0xa78126bbe140a093u, 0x9de1dc61ca7550cfu, 0x84f0046d01b492c5u, 0x2d91810b945de0f3u, - 0xf5408b7f6008aa71u, 0x43707f4863034149u, 0xdac65fb9679279d5u, 0xc48406e7d1114eb7u, - 0xa7dc9ed3c88e1271u, 0xfb25b2efdb9cb30du, 0x1bebda0951c4df63u, 0x5c85e975580ee5bdu, - 0x1591bc60082cb137u, 0x2c38606318ef25d7u, 0x76ca72f7c5c63e27u, 0xf04a75d17baa0915u, - 0x77458175139ae30du, 0x0e6c1330bc1b9421u, 0xdf87d2b5797e8293u, 0xefa5c703e1e68925u, - 0x2b6b1b3278b4f6e1u, 0xceee27b382394249u, 0xd74e3829f5dab91du, 0xfdb17989c26b5f1fu, - 0xc1b7d18781530845u, 0x7b4436b2105a8561u, 0x7ba7c0418372a7d7u, 0x9dbc5c67feb6c639u, - 0x502686d7f6ff6b8fu, 0x6101855406be7a1fu, 0x9956afb5806930e7u, 0xe1f0ee88af40f7c5u, - 0x984b057bda5c1151u, 0x9a49819acc13ea05u, 0x8ef0dead0896ef27u, 0x71f7826efe292b21u, - 0xad80a480e46986efu, 0x01cdc0ebf5e0c6f7u, 0x6e06f839968f68dbu, 0xdd5943ab56e76139u, - 0xcdcf31bf8604c5e7u, 0x7e2b4a847054a1cbu, 0x0ca75697a4d3d0f5u, 0x4703f53ac514a98bu, -}; - -/* inverses of reduced_factorial_odd_part values modulo 2**64. - -Python code to generate the values: - - import math - - for n in range(128): - fac = math.factorial(n) - fac_odd_part = fac // (fac & -fac) - inverted_fac_odd_part = pow(fac_odd_part, -1, 2**64) - print(f"{inverted_fac_odd_part:#018x}u") -*/ -static const uint64_t inverted_factorial_odd_part[] = { - 0x0000000000000001u, 0x0000000000000001u, 0x0000000000000001u, 0xaaaaaaaaaaaaaaabu, - 0xaaaaaaaaaaaaaaabu, 0xeeeeeeeeeeeeeeefu, 0x4fa4fa4fa4fa4fa5u, 0x2ff2ff2ff2ff2ff3u, - 0x2ff2ff2ff2ff2ff3u, 0x938cc70553e3771bu, 0xb71c27cddd93e49fu, 0xb38e3229fcdee63du, - 0xe684bb63544a4cbfu, 0xc2f684917ca340fbu, 0xf747c9cba417526du, 0xbb26eb51d7bd49c3u, - 0xbb26eb51d7bd49c3u, 0xb0a7efb985294093u, 0xbe4b8c69f259eabbu, 0x6854d17ed6dc4fb9u, - 0xe1aa904c915f4325u, 0x3b8206df131cead1u, 0x79c6009fea76fe13u, 0xd8c5d381633cd365u, - 0x4841f12b21144677u, 0x4a91ff68200b0d0fu, 0x8f9513a58c4f9e8bu, 0x2b3e690621a42251u, - 0x4f520f00e03c04e7u, 0x2edf84ee600211d3u, 0xadcaa2764aaacdfdu, 0x161f4f9033f4fe63u, - 0x161f4f9033f4fe63u, 0xbada2932ea4d3e03u, 0xcec189f3efaa30d3u, 0xf7475bb68330bf91u, - 0x37eb7bf7d5b01549u, 0x46b35660a4e91555u, 0xa567c12d81f151f7u, 0x4c724007bb2071b1u, - 0x0f4a0cce58a016bdu, 0xfa21068e66106475u, 0x244ab72b5a318ae1u, 0x366ce67e080d0f23u, - 0xd666fdae5dd2a449u, 0xd740ddd0acc06a0du, 0xb050bbbb28e6f97bu, 0x70b003fe890a5c75u, - 0xd03aabff83037427u, 0x13ec4ca72c783bd7u, 0x90282c06afdbd96fu, 0x4414ddb9db4a95d5u, - 0xa2c68735ae6832e9u, 0xbf72d71455676665u, 0xa8469fab6b759b7fu, 0xc1e55b56e606caf9u, - 0x40455630fc4a1cffu, 0x0120a7b0046d16f7u, 0xa7c3553b08faef23u, 0x9f0bfd1b08d48639u, - 0xa433ffce9a304d37u, 0xa22ad1d53915c683u, 0xcb6cbc723ba5dd1du, 0x547fb1b8ab9d0ba3u, - 0x547fb1b8ab9d0ba3u, 0x8f15a826498852e3u, 0x32e1a03f38880283u, 0x3de4cce63283f0c1u, - 0x5dfe6667e4da95b1u, 0xfda6eeeef479e47du, 0xf14de991cc7882dfu, 0xe68db79247630ca9u, - 0xa7d6db8207ee8fa1u, 0x255e1f0fcf034499u, 0xc9a8990e43dd7e65u, 0x3279b6f289702e0fu, - 0xe7b5905d9b71b195u, 0x03025ba41ff0da69u, 0xb7df3d6d3be55aefu, 0xf89b212ebff2b361u, - 0xfe856d095996f0adu, 0xd6e533e9fdf20f9du, 0xf8c0e84a63da3255u, 0xa677876cd91b4db7u, - 0x07ed4f97780d7d9bu, 0x90a8705f258db62fu, 0xa41bbb2be31b1c0du, 0x6ec28690b038383bu, - 0xdb860c3bb2edd691u, 0x0838286838a980f9u, 0x558417a74b36f77du, 0x71779afc3646ef07u, - 0x743cda377ccb6e91u, 0x7fdf9f3fe89153c5u, 0xdc97d25df49b9a4bu, 0x76321a778eb37d95u, - 0x7cbb5e27da3bd487u, 0x9cff4ade1a009de7u, 0x70eb166d05c15197u, 0xdcf0460b71d5fe3du, - 0x5ac1ee5260b6a3c5u, 0xc922dedfdd78efe1u, 0xe5d381dc3b8eeb9bu, 0xd57e5347bafc6aadu, - 0x86939040983acd21u, 0x395b9d69740a4ff9u, 0x1467299c8e43d135u, 0x5fe440fcad975cdfu, - 0xcaa9a39794a6ca8du, 0xf61dbd640868dea1u, 0xac09d98d74843be7u, 0x2b103b9e1a6b4809u, - 0x2ab92d16960f536fu, 0x6653323d5e3681dfu, 0xefd48c1c0624e2d7u, 0xa496fefe04816f0du, - 0x1754a7b07bbdd7b1u, 0x23353c829a3852cdu, 0xbf831261abd59097u, 0x57a8e656df0618e1u, - 0x16e9206c3100680fu, 0xadad4c6ee921dac7u, 0x635f2b3860265353u, 0xdd6d0059f44b3d09u, - 0xac4dd6b894447dd7u, 0x42ea183eeaa87be3u, 0x15612d1550ee5b5du, 0x226fa19d656cb623u, -}; - -/* exponent of the largest power of 2 dividing factorial(n), for n in range(68) - -Python code to generate the values: - -import math - -for n in range(128): - fac = math.factorial(n) - fac_trailing_zeros = (fac & -fac).bit_length() - 1 - print(fac_trailing_zeros) -*/ - -static const uint8_t factorial_trailing_zeros[] = { - 0, 0, 1, 1, 3, 3, 4, 4, 7, 7, 8, 8, 10, 10, 11, 11, // 0-15 - 15, 15, 16, 16, 18, 18, 19, 19, 22, 22, 23, 23, 25, 25, 26, 26, // 16-31 - 31, 31, 32, 32, 34, 34, 35, 35, 38, 38, 39, 39, 41, 41, 42, 42, // 32-47 - 46, 46, 47, 47, 49, 49, 50, 50, 53, 53, 54, 54, 56, 56, 57, 57, // 48-63 - 63, 63, 64, 64, 66, 66, 67, 67, 70, 70, 71, 71, 73, 73, 74, 74, // 64-79 - 78, 78, 79, 79, 81, 81, 82, 82, 85, 85, 86, 86, 88, 88, 89, 89, // 80-95 - 94, 94, 95, 95, 97, 97, 98, 98, 101, 101, 102, 102, 104, 104, 105, 105, // 96-111 - 109, 109, 110, 110, 112, 112, 113, 113, 116, 116, 117, 117, 119, 119, 120, 120, // 112-127 -}; - -/* Number of permutations and combinations. - * P(n, k) = n! / (n-k)! - * C(n, k) = P(n, k) / k! - */ - -/* Calculate C(n, k) for n in the 63-bit range. */ -static PyObject * -perm_comb_small(unsigned long long n, unsigned long long k, int iscomb) -{ - assert(k != 0); - - /* For small enough n and k the result fits in the 64-bit range and can - * be calculated without allocating intermediate PyLong objects. */ - if (iscomb) { - /* Maps k to the maximal n so that 2*k-1 <= n <= 127 and C(n, k) - * fits into a uint64_t. Exclude k = 1, because the second fast - * path is faster for this case.*/ - static const unsigned char fast_comb_limits1[] = { - 0, 0, 127, 127, 127, 127, 127, 127, // 0-7 - 127, 127, 127, 127, 127, 127, 127, 127, // 8-15 - 116, 105, 97, 91, 86, 82, 78, 76, // 16-23 - 74, 72, 71, 70, 69, 68, 68, 67, // 24-31 - 67, 67, 67, // 32-34 - }; - if (k < Py_ARRAY_LENGTH(fast_comb_limits1) && n <= fast_comb_limits1[k]) { - /* - comb(n, k) fits into a uint64_t. We compute it as - - comb_odd_part << shift - - where 2**shift is the largest power of two dividing comb(n, k) - and comb_odd_part is comb(n, k) >> shift. comb_odd_part can be - calculated efficiently via arithmetic modulo 2**64, using three - lookups and two uint64_t multiplications. - */ - uint64_t comb_odd_part = reduced_factorial_odd_part[n] - * inverted_factorial_odd_part[k] - * inverted_factorial_odd_part[n - k]; - int shift = factorial_trailing_zeros[n] - - factorial_trailing_zeros[k] - - factorial_trailing_zeros[n - k]; - return PyLong_FromUnsignedLongLong(comb_odd_part << shift); - } - - /* Maps k to the maximal n so that 2*k-1 <= n <= 127 and C(n, k)*k - * fits into a long long (which is at least 64 bit). Only contains - * items larger than in fast_comb_limits1. */ - static const unsigned long long fast_comb_limits2[] = { - 0, ULLONG_MAX, 4294967296ULL, 3329022, 102570, 13467, 3612, 1449, // 0-7 - 746, 453, 308, 227, 178, 147, // 8-13 - }; - if (k < Py_ARRAY_LENGTH(fast_comb_limits2) && n <= fast_comb_limits2[k]) { - /* C(n, k) = C(n, k-1) * (n-k+1) / k */ - unsigned long long result = n; - for (unsigned long long i = 1; i < k;) { - result *= --n; - result /= ++i; - } - return PyLong_FromUnsignedLongLong(result); - } - } - else { - /* Maps k to the maximal n so that k <= n and P(n, k) - * fits into a long long (which is at least 64 bit). */ - static const unsigned long long fast_perm_limits[] = { - 0, ULLONG_MAX, 4294967296ULL, 2642246, 65537, 7133, 1627, 568, // 0-7 - 259, 142, 88, 61, 45, 36, 30, 26, // 8-15 - 24, 22, 21, 20, 20, // 16-20 - }; - if (k < Py_ARRAY_LENGTH(fast_perm_limits) && n <= fast_perm_limits[k]) { - if (n <= 127) { - /* P(n, k) fits into a uint64_t. */ - uint64_t perm_odd_part = reduced_factorial_odd_part[n] - * inverted_factorial_odd_part[n - k]; - int shift = factorial_trailing_zeros[n] - - factorial_trailing_zeros[n - k]; - return PyLong_FromUnsignedLongLong(perm_odd_part << shift); - } - - /* P(n, k) = P(n, k-1) * (n-k+1) */ - unsigned long long result = n; - for (unsigned long long i = 1; i < k;) { - result *= --n; - ++i; - } - return PyLong_FromUnsignedLongLong(result); - } - } - - /* For larger n use recursive formulas: - * - * P(n, k) = P(n, j) * P(n-j, k-j) - * C(n, k) = C(n, j) * C(n-j, k-j) // C(k, j) - */ - unsigned long long j = k / 2; - PyObject *a, *b; - a = perm_comb_small(n, j, iscomb); - if (a == NULL) { - return NULL; - } - b = perm_comb_small(n - j, k - j, iscomb); - if (b == NULL) { - goto error; - } - Py_SETREF(a, PyNumber_Multiply(a, b)); - Py_DECREF(b); - if (iscomb && a != NULL) { - b = perm_comb_small(k, j, 1); - if (b == NULL) { - goto error; - } - Py_SETREF(a, PyNumber_FloorDivide(a, b)); - Py_DECREF(b); - } - return a; - -error: - Py_DECREF(a); - return NULL; -} - -/* Calculate P(n, k) or C(n, k) using recursive formulas. - * It is more efficient than sequential multiplication thanks to - * Karatsuba multiplication. - */ -static PyObject * -perm_comb(PyObject *n, unsigned long long k, int iscomb) -{ - if (k == 0) { - return PyLong_FromLong(1); - } - if (k == 1) { - return Py_NewRef(n); - } - - /* P(n, k) = P(n, j) * P(n-j, k-j) */ - /* C(n, k) = C(n, j) * C(n-j, k-j) // C(k, j) */ - unsigned long long j = k / 2; - PyObject *a, *b; - a = perm_comb(n, j, iscomb); - if (a == NULL) { - return NULL; - } - PyObject *t = PyLong_FromUnsignedLongLong(j); - if (t == NULL) { - goto error; - } - n = PyNumber_Subtract(n, t); - Py_DECREF(t); - if (n == NULL) { - goto error; - } - b = perm_comb(n, k - j, iscomb); - Py_DECREF(n); - if (b == NULL) { - goto error; - } - Py_SETREF(a, PyNumber_Multiply(a, b)); - Py_DECREF(b); - if (iscomb && a != NULL) { - b = perm_comb_small(k, j, 1); - if (b == NULL) { - goto error; - } - Py_SETREF(a, PyNumber_FloorDivide(a, b)); - Py_DECREF(b); - } - return a; - -error: - Py_DECREF(a); - return NULL; -} - -/*[clinic input] -math.perm - - n: object - k: object = None - / - -Number of ways to choose k items from n items without repetition and with order. - -Evaluates to n! / (n - k)! when k <= n and evaluates -to zero when k > n. - -If k is not specified or is None, then k defaults to n -and the function returns n!. - -Raises TypeError if either of the arguments are not integers. -Raises ValueError if either of the arguments are negative. -[clinic start generated code]*/ - -static PyObject * -math_perm_impl(PyObject *module, PyObject *n, PyObject *k) -/*[clinic end generated code: output=e021a25469653e23 input=5311c5a00f359b53]*/ -{ - PyObject *result = NULL; - int overflow, cmp; - long long ki, ni; - - if (k == Py_None) { - return math_factorial(module, n); - } - n = PyNumber_Index(n); - if (n == NULL) { - return NULL; - } - k = PyNumber_Index(k); - if (k == NULL) { - Py_DECREF(n); - return NULL; - } - assert(PyLong_CheckExact(n) && PyLong_CheckExact(k)); - - if (_PyLong_IsNegative((PyLongObject *)n)) { - PyErr_SetString(PyExc_ValueError, - "n must be a non-negative integer"); - goto error; - } - if (_PyLong_IsNegative((PyLongObject *)k)) { - PyErr_SetString(PyExc_ValueError, - "k must be a non-negative integer"); - goto error; - } - - cmp = PyObject_RichCompareBool(n, k, Py_LT); - if (cmp != 0) { - if (cmp > 0) { - result = PyLong_FromLong(0); - goto done; - } - goto error; - } - - ki = PyLong_AsLongLongAndOverflow(k, &overflow); - assert(overflow >= 0 && !PyErr_Occurred()); - if (overflow > 0) { - PyErr_Format(PyExc_OverflowError, - "k must not exceed %lld", - LLONG_MAX); - goto error; - } - assert(ki >= 0); - - ni = PyLong_AsLongLongAndOverflow(n, &overflow); - assert(overflow >= 0 && !PyErr_Occurred()); - if (!overflow && ki > 1) { - assert(ni >= 0); - result = perm_comb_small((unsigned long long)ni, - (unsigned long long)ki, 0); - } - else { - result = perm_comb(n, (unsigned long long)ki, 0); - } - -done: - Py_DECREF(n); - Py_DECREF(k); - return result; - -error: - Py_DECREF(n); - Py_DECREF(k); - return NULL; -} - -/*[clinic input] -math.comb - - n: object - k: object - / - -Number of ways to choose k items from n items without repetition and without order. - -Evaluates to n! / (k! * (n - k)!) when k <= n and evaluates -to zero when k > n. - -Also called the binomial coefficient because it is equivalent -to the coefficient of k-th term in polynomial expansion of the -expression (1 + x)**n. - -Raises TypeError if either of the arguments are not integers. -Raises ValueError if either of the arguments are negative. - -[clinic start generated code]*/ - -static PyObject * -math_comb_impl(PyObject *module, PyObject *n, PyObject *k) -/*[clinic end generated code: output=bd2cec8d854f3493 input=9a05315af2518709]*/ -{ - PyObject *result = NULL, *temp; - int overflow, cmp; - long long ki, ni; - - n = PyNumber_Index(n); - if (n == NULL) { - return NULL; - } - k = PyNumber_Index(k); - if (k == NULL) { - Py_DECREF(n); - return NULL; - } - assert(PyLong_CheckExact(n) && PyLong_CheckExact(k)); - - if (_PyLong_IsNegative((PyLongObject *)n)) { - PyErr_SetString(PyExc_ValueError, - "n must be a non-negative integer"); - goto error; - } - if (_PyLong_IsNegative((PyLongObject *)k)) { - PyErr_SetString(PyExc_ValueError, - "k must be a non-negative integer"); - goto error; - } - - ni = PyLong_AsLongLongAndOverflow(n, &overflow); - assert(overflow >= 0 && !PyErr_Occurred()); - if (!overflow) { - assert(ni >= 0); - ki = PyLong_AsLongLongAndOverflow(k, &overflow); - assert(overflow >= 0 && !PyErr_Occurred()); - if (overflow || ki > ni) { - result = PyLong_FromLong(0); - goto done; - } - assert(ki >= 0); - - ki = Py_MIN(ki, ni - ki); - if (ki > 1) { - result = perm_comb_small((unsigned long long)ni, - (unsigned long long)ki, 1); - goto done; - } - /* For k == 1 just return the original n in perm_comb(). */ - } - else { - /* k = min(k, n - k) */ - temp = PyNumber_Subtract(n, k); - if (temp == NULL) { - goto error; - } - assert(PyLong_Check(temp)); - if (_PyLong_IsNegative((PyLongObject *)temp)) { - Py_DECREF(temp); - result = PyLong_FromLong(0); - goto done; - } - cmp = PyObject_RichCompareBool(temp, k, Py_LT); - if (cmp > 0) { - Py_SETREF(k, temp); - } - else { - Py_DECREF(temp); - if (cmp < 0) { - goto error; - } - } - - ki = PyLong_AsLongLongAndOverflow(k, &overflow); - assert(overflow >= 0 && !PyErr_Occurred()); - if (overflow) { - PyErr_Format(PyExc_OverflowError, - "min(n - k, k) must not exceed %lld", - LLONG_MAX); - goto error; - } - assert(ki >= 0); - } - - result = perm_comb(n, (unsigned long long)ki, 1); - -done: - Py_DECREF(n); - Py_DECREF(k); - return result; - -error: - Py_DECREF(n); - Py_DECREF(k); - return NULL; -} - - /*[clinic input] +@permit_long_docstring_body math.nextafter x: double @@ -3928,7 +2850,7 @@ Raises ValueError if steps is negative. static PyObject * math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) -/*[clinic end generated code: output=cc6511f02afc099e input=7f2a5842112af2b4]*/ +/*[clinic end generated code: output=cc6511f02afc099e input=cc8f0dad1b27a8a4]*/ { #if defined(_AIX) if (x == y) { @@ -4061,7 +2983,7 @@ math_ulp_impl(PyObject *module, double x) if (isinf(x)) { return x; } - double inf = Py_INFINITY; + double inf = INFINITY; double x2 = nextafter(x, inf); if (isinf(x2)) { /* special case: x is the largest positive representable float */ @@ -4085,12 +3007,38 @@ math_exec(PyObject *module) if (PyModule_Add(module, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(module, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(module, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } if (PyModule_Add(module, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { return -1; } + + PyObject *intmath = PyImport_ImportModule("_math_integer"); + if (!intmath) { + return -1; + } +#define IMPORT_FROM_INTMATH(NAME) do { \ + if (PyModule_Add(module, #NAME, \ + PyObject_GetAttrString(intmath, #NAME)) < 0) { \ + Py_DECREF(intmath); \ + return -1; \ + } \ + } while(0) + + IMPORT_FROM_INTMATH(comb); + IMPORT_FROM_INTMATH(factorial); + IMPORT_FROM_INTMATH(gcd); + IMPORT_FROM_INTMATH(isqrt); + IMPORT_FROM_INTMATH(lcm); + IMPORT_FROM_INTMATH(perm); + if (_PyImport_SetModuleString("math.integer", intmath) < 0) { + Py_DECREF(intmath); + return -1; + } + if (PyModule_Add(module, "integer", intmath) < 0) { + return -1; + } return 0; } @@ -4115,21 +3063,21 @@ static PyMethodDef math_methods[] = { {"exp2", math_exp2, METH_O, math_exp2_doc}, {"expm1", math_expm1, METH_O, math_expm1_doc}, {"fabs", math_fabs, METH_O, math_fabs_doc}, - MATH_FACTORIAL_METHODDEF MATH_FLOOR_METHODDEF MATH_FMA_METHODDEF + MATH_FMAX_METHODDEF MATH_FMOD_METHODDEF + MATH_FMIN_METHODDEF MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF {"gamma", math_gamma, METH_O, math_gamma_doc}, - MATH_GCD_METHODDEF MATH_HYPOT_METHODDEF MATH_ISCLOSE_METHODDEF MATH_ISFINITE_METHODDEF + MATH_ISNORMAL_METHODDEF + MATH_ISSUBNORMAL_METHODDEF MATH_ISINF_METHODDEF MATH_ISNAN_METHODDEF - MATH_ISQRT_METHODDEF - MATH_LCM_METHODDEF MATH_LDEXP_METHODDEF {"lgamma", math_lgamma, METH_O, math_lgamma_doc}, {"log", _PyCFunction_CAST(math_log), METH_FASTCALL, math_log_doc}, @@ -4140,6 +3088,7 @@ static PyMethodDef math_methods[] = { MATH_POW_METHODDEF MATH_RADIANS_METHODDEF {"remainder", _PyCFunction_CAST(math_remainder), METH_FASTCALL, math_remainder_doc}, + MATH_SIGNBIT_METHODDEF {"sin", math_sin, METH_O, math_sin_doc}, {"sinh", math_sinh, METH_O, math_sinh_doc}, {"sqrt", math_sqrt, METH_O, math_sqrt_doc}, @@ -4148,8 +3097,6 @@ static PyMethodDef math_methods[] = { MATH_SUMPROD_METHODDEF MATH_TRUNC_METHODDEF MATH_PROD_METHODDEF - MATH_PERM_METHODDEF - MATH_COMB_METHODDEF MATH_NEXTAFTER_METHODDEF MATH_ULP_METHODDEF {NULL, NULL} /* sentinel */ diff --git a/Modules/md5module.c b/Modules/md5module.c index c36eb41d4d2..56e9faf4c62 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -21,34 +22,28 @@ #endif #include "Python.h" +#include "pycore_object.h" // _PyObject_VisitType() +#include "pycore_strhex.h" // _Py_strhex() + #include "hashlib.h" -/*[clinic input] -module _md5 -class MD5Type "MD5object *" "&PyType_Type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6e5261719957a912]*/ +#include "_hacl/Hacl_Hash_MD5.h" /* The MD5 block size and message digest sizes, in bytes */ #define MD5_BLOCKSIZE 64 #define MD5_DIGESTSIZE 16 -#include "_hacl/Hacl_Hash_MD5.h" - +// --- Module objects --------------------------------------------------------- typedef struct { - PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; + HASHLIB_OBJECT_HEAD Hacl_Hash_MD5_state_t *hash_state; } MD5object; #define _MD5object_CAST(op) ((MD5object *)(op)) -#include "clinic/md5module.c.h" - +// --- Module state ----------------------------------------------------------- typedef struct { PyTypeObject* md5_type; @@ -62,6 +57,18 @@ md5_get_state(PyObject *module) return (MD5State *)state; } +// --- Module clinic configuration -------------------------------------------- + +/*[clinic input] +module _md5 +class MD5Type "MD5object *" "&PyType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6e5261719957a912]*/ + +#include "clinic/md5module.c.h" + +// --- MD5 object interface --------------------------------------------------- + static MD5object * newMD5object(MD5State * st) { @@ -76,13 +83,6 @@ newMD5object(MD5State * st) } /* Internal methods for a hash object */ -static int -MD5_traverse(PyObject *ptr, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(ptr)); - return 0; -} - static void MD5_dealloc(PyObject *op) { @@ -116,11 +116,11 @@ MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) return NULL; } - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); newobj->hash_state = Hacl_Hash_MD5_copy(self->hash_state); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); if (newobj->hash_state == NULL) { - Py_DECREF(self); + Py_DECREF(newobj); return PyErr_NoMemory(); } return (PyObject *)newobj; @@ -136,10 +136,10 @@ static PyObject * MD5Type_digest_impl(MD5object *self) /*[clinic end generated code: output=eb691dc4190a07ec input=bc0c4397c2994be6]*/ { - unsigned char digest[MD5_DIGESTSIZE]; - ENTER_HASHLIB(self); + uint8_t digest[MD5_DIGESTSIZE]; + HASHLIB_ACQUIRE_LOCK(self); Hacl_Hash_MD5_digest(self->hash_state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return PyBytes_FromStringAndSize((const char *)digest, MD5_DIGESTSIZE); } @@ -153,20 +153,11 @@ static PyObject * MD5Type_hexdigest_impl(MD5object *self) /*[clinic end generated code: output=17badced1f3ac932 input=b60b19de644798dd]*/ { - unsigned char digest[MD5_DIGESTSIZE]; - ENTER_HASHLIB(self); + uint8_t digest[MD5_DIGESTSIZE]; + HASHLIB_ACQUIRE_LOCK(self); Hacl_Hash_MD5_digest(self->hash_state, digest); - LEAVE_HASHLIB(self); - - const char *hexdigits = "0123456789abcdef"; - char digest_hex[MD5_DIGESTSIZE * 2]; - char *str = digest_hex; - for (size_t i=0; i < MD5_DIGESTSIZE; i++) { - unsigned char byte = digest[i]; - *str++ = hexdigits[byte >> 4]; - *str++ = hexdigits[byte & 0x0f]; - } - return PyUnicode_FromStringAndSize(digest_hex, sizeof(digest_hex)); + HASHLIB_RELEASE_LOCK(self); + return _Py_strhex((const char *)digest, MD5_DIGESTSIZE); } static void @@ -177,6 +168,7 @@ update(Hacl_Hash_MD5_state_t *state, uint8_t *buf, Py_ssize_t len) * take more than 1 billion years to overflow the maximum admissible length * for MD5 (2^61 - 1). */ + assert(len >= 0); #if PY_SSIZE_T_MAX > UINT32_MAX while (len > UINT32_MAX) { (void)Hacl_Hash_MD5_update(state, buf, UINT32_MAX); @@ -202,22 +194,11 @@ MD5Type_update_impl(MD5object *self, PyObject *obj) /*[clinic end generated code: output=b0fed9a7ce7ad253 input=6e1efcd9ecf17032]*/ { Py_buffer buf; - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - update(self->hash_state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - update(self->hash_state, buf.buf, buf.len); - } - + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, buf.len, + update(self->hash_state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -259,7 +240,7 @@ static PyType_Slot md5_type_slots[] = { {Py_tp_dealloc, MD5_dealloc}, {Py_tp_methods, MD5_methods}, {Py_tp_getset, MD5_getseters}, - {Py_tp_traverse, MD5_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0,0} }; @@ -276,17 +257,24 @@ static PyType_Spec md5_type_spec = { /*[clinic input] _md5.md5 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new MD5 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=587071f76254a4ac input=7a144a1905636985]*/ +_md5_md5_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=d45e187d3d16f3a8 input=7ea5c5366dbb44bf]*/ { + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } + MD5object *new; Py_buffer buf; @@ -312,16 +300,12 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) } if (string) { - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor - * where it is not yet possible to have concurrent access. */ - Py_BEGIN_ALLOW_THREADS - update(new->hash_state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - update(new->hash_state, buf.buf, buf.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + buf.len, + update(new->hash_state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); } diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6a385562845..ac8521f8aa9 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -23,8 +23,10 @@ #endif #include <Python.h> +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #ifndef MS_WINDOWS @@ -89,6 +91,12 @@ my_getpagesize(void) # define MAP_ANONYMOUS MAP_ANON #endif +/*[clinic input] +module mmap +class mmap.mmap "mmap_object *" "" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=82a9f8a529905b9b]*/ + typedef enum { ACCESS_DEFAULT, @@ -117,20 +125,34 @@ typedef struct { #ifdef UNIX int fd; - _Bool trackfd; + int flags; #endif PyObject *weakreflist; access_mode access; + _Bool trackfd; } mmap_object; #define mmap_object_CAST(op) ((mmap_object *)(op)) -static int -mmap_object_traverse(PyObject *op, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(op)); - return 0; +#include "clinic/mmapmodule.c.h" + + +/* Return a Py_ssize_t from the object arg. This conversion logic is similar + to what AC uses for `Py_ssize_t` arguments. + + Returns -1 on error. Use PyErr_Occurred() to disambiguate. +*/ +static Py_ssize_t +_As_Py_ssize_t(PyObject *arg) { + assert(arg != NULL); + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + return ival; } static void @@ -163,17 +185,22 @@ mmap_object_dealloc(PyObject *op) Py_END_ALLOW_THREADS #endif /* UNIX */ - if (m_obj->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, m_obj->weakreflist); tp->tp_free(m_obj); Py_DECREF(tp); } +/*[clinic input] +@critical_section +mmap.mmap.close + +[clinic start generated code]*/ + static PyObject * -mmap_close_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_close_impl(mmap_object *self) +/*[clinic end generated code: output=a1ae0c727546f78d input=25020035f047eae1]*/ { - mmap_object *self = mmap_object_CAST(op); if (self->exports > 0) { PyErr_SetString(PyExc_BufferError, "cannot close "\ "exported pointers exist"); @@ -290,6 +317,24 @@ filter_page_exception_method(mmap_object *self, EXCEPTION_POINTERS *ptrs, } return EXCEPTION_CONTINUE_SEARCH; } + +static void +_PyErr_SetFromNTSTATUS(ULONG status) +{ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + PyErr_SetFromWindowsErr(LsaNtStatusToWinError((NTSTATUS)status)); +#else + if (status & 0x80000000) { + // HRESULT-shaped codes are supported by PyErr_SetFromWindowsErr + PyErr_SetFromWindowsErr((int)status); + } + else { + // No mapping for NTSTATUS values, so just return it for diagnostic purposes + // If we provide it as winerror it could incorrectly change the type of the exception. + PyErr_Format(PyExc_OSError, "Operating system error NTSTATUS=0x%08lX", status); + } +#endif +} #endif #if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) @@ -303,9 +348,7 @@ do { \ assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ - NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ - ULONG code = LsaNtStatusToWinError(status); \ - PyErr_SetFromWindowsErr(code); \ + _PyErr_SetFromNTSTATUS((ULONG)record.ExceptionInformation[2]); \ } \ else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ @@ -332,9 +375,7 @@ do { \ assert(record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR || \ record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION); \ if (record.ExceptionCode == EXCEPTION_IN_PAGE_ERROR) { \ - NTSTATUS status = (NTSTATUS) record.ExceptionInformation[2]; \ - ULONG code = LsaNtStatusToWinError(status); \ - PyErr_SetFromWindowsErr(code); \ + _PyErr_SetFromNTSTATUS((ULONG)record.ExceptionInformation[2]); \ } \ else if (record.ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { \ PyErr_SetFromWindowsErr(ERROR_NOACCESS); \ @@ -439,7 +480,8 @@ _safe_PyBytes_ReverseFind(Py_ssize_t *out, mmap_object *self, } PyObject * -_safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { +_safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) +{ if (num_bytes == 1) { char dest; if (safe_byte_copy(&dest, start) < 0) { @@ -450,21 +492,28 @@ _safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { } } else { - PyObject *result = PyBytes_FromStringAndSize(NULL, num_bytes); - if (result == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(num_bytes); + if (writer == NULL) { return NULL; } - if (safe_memcpy(PyBytes_AS_STRING(result), start, num_bytes) < 0) { - Py_CLEAR(result); + if (safe_memcpy(PyBytesWriter_GetData(writer), start, num_bytes) < 0) { + PyBytesWriter_Discard(writer); + return NULL; } - return result; + return PyBytesWriter_Finish(writer); } } +/*[clinic input] +@critical_section +mmap.mmap.read_byte + +[clinic start generated code]*/ + static PyObject * -mmap_read_byte_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_read_byte_impl(mmap_object *self) +/*[clinic end generated code: output=d931da1319f3869b input=5b8c6a904bdddda9]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); if (self->pos >= self->size) { PyErr_SetString(PyExc_ValueError, "read byte out of range"); @@ -478,12 +527,18 @@ mmap_read_byte_method(PyObject *op, PyObject *Py_UNUSED(ignored)) return PyLong_FromLong((unsigned char) dest); } +/*[clinic input] +@critical_section +mmap.mmap.readline + +[clinic start generated code]*/ + static PyObject * -mmap_read_line_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_readline_impl(mmap_object *self) +/*[clinic end generated code: output=b9d2bf9999283311 input=2c4efd1d06e1cdd1]*/ { Py_ssize_t remaining; char *start, *eol; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -508,15 +563,21 @@ mmap_read_line_method(PyObject *op, PyObject *Py_UNUSED(ignored)) return result; } -static PyObject * -mmap_read_method(PyObject *op, PyObject *args) -{ - Py_ssize_t num_bytes = PY_SSIZE_T_MAX, remaining; - mmap_object *self = mmap_object_CAST(op); +/*[clinic input] +@critical_section +mmap.mmap.read + + n as num_bytes: object(converter='_Py_convert_optional_to_ssize_t', type='Py_ssize_t', c_default='PY_SSIZE_T_MAX') = None + / + +[clinic start generated code]*/ + +static PyObject * +mmap_mmap_read_impl(mmap_object *self, Py_ssize_t num_bytes) +/*[clinic end generated code: output=3b4d4f3704ed0969 input=8f97f361d435e357]*/ +{ + Py_ssize_t remaining; - CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|n?:read", &num_bytes)) - return NULL; CHECK_VALID(NULL); /* silently 'adjust' out-of-range requests */ @@ -533,81 +594,105 @@ mmap_read_method(PyObject *op, PyObject *args) } static PyObject * -mmap_gfind(mmap_object *self, - PyObject *args, - int reverse) +mmap_gfind_lock_held(mmap_object *self, Py_buffer *view, PyObject *start_obj, + PyObject *end_obj, int reverse) { Py_ssize_t start = self->pos; Py_ssize_t end = self->size; - Py_buffer view; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, reverse ? "y*|nn:rfind" : "y*|nn:find", - &view, &start, &end)) { - return NULL; - } - else { - if (start < 0) - start += self->size; - if (start < 0) - start = 0; - else if (start > self->size) - start = self->size; - - if (end < 0) - end += self->size; - if (end < 0) - end = 0; - else if (end > self->size) - end = self->size; - - Py_ssize_t index; - PyObject *result; - CHECK_VALID_OR_RELEASE(NULL, view); - if (end < start) { - result = PyLong_FromSsize_t(-1); + if (start_obj != Py_None) { + start = _As_Py_ssize_t(start_obj); + if (start == -1 && PyErr_Occurred()) { + return NULL; } - else if (reverse) { - assert(0 <= start && start <= end && end <= self->size); - if (_safe_PyBytes_ReverseFind(&index, self, - self->data + start, end - start, - view.buf, view.len, start) < 0) - { - result = NULL; - } - else { - result = PyLong_FromSsize_t(index); + + if (end_obj != Py_None) { + end = _As_Py_ssize_t(end_obj); + if (end == -1 && PyErr_Occurred()) { + return NULL; } } + } + + if (start < 0) + start += self->size; + if (start < 0) + start = 0; + else if (start > self->size) + start = self->size; + + if (end < 0) + end += self->size; + if (end < 0) + end = 0; + else if (end > self->size) + end = self->size; + + Py_ssize_t index; + PyObject *result; + CHECK_VALID(NULL); + if (end < start) { + result = PyLong_FromSsize_t(-1); + } + else if (reverse) { + assert(0 <= start && start <= end && end <= self->size); + if (_safe_PyBytes_ReverseFind(&index, self, + self->data + start, end - start, + view->buf, view->len, start) < 0) + { + result = NULL; + } else { - assert(0 <= start && start <= end && end <= self->size); - if (_safe_PyBytes_Find(&index, self, - self->data + start, end - start, - view.buf, view.len, start) < 0) - { - result = NULL; - } - else { - result = PyLong_FromSsize_t(index); - } + result = PyLong_FromSsize_t(index); } - PyBuffer_Release(&view); - return result; } + else { + assert(0 <= start && start <= end && end <= self->size); + if (_safe_PyBytes_Find(&index, self, + self->data + start, end - start, + view->buf, view->len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } + } + return result; } -static PyObject * -mmap_find_method(PyObject *op, PyObject *args) -{ - mmap_object *self = mmap_object_CAST(op); - return mmap_gfind(self, args, 0); -} +/*[clinic input] +@critical_section +mmap.mmap.find + + view: Py_buffer + start: object = None + end: object = None + / + +[clinic start generated code]*/ static PyObject * -mmap_rfind_method(PyObject *op, PyObject *args) +mmap_mmap_find_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end) +/*[clinic end generated code: output=ef8878a322f00192 input=0135504494b52c2b]*/ { - mmap_object *self = mmap_object_CAST(op); - return mmap_gfind(self, args, 1); + return mmap_gfind_lock_held(self, view, start, end, 0); +} + +/*[clinic input] +@critical_section +mmap.mmap.rfind = mmap.mmap.find + +[clinic start generated code]*/ + +static PyObject * +mmap_mmap_rfind_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end) +/*[clinic end generated code: output=73b918940d67c2b8 input=8aecdd1f70c06c62]*/ +{ + return mmap_gfind_lock_held(self, view, start, end, 1); } static int @@ -619,6 +704,7 @@ is_writable(mmap_object *self) return 0; } +#if defined(MS_WINDOWS) || defined(HAVE_MREMAP) static int is_resizeable(mmap_object *self) { @@ -627,13 +713,11 @@ is_resizeable(mmap_object *self) "mmap can't resize with extant buffers exported."); return 0; } -#ifdef UNIX if (!self->trackfd) { PyErr_SetString(PyExc_ValueError, "mmap can't resize with trackfd=False."); return 0; } -#endif if ((self->access == ACCESS_WRITE) || (self->access == ACCESS_DEFAULT)) return 1; PyErr_Format(PyExc_TypeError, @@ -641,52 +725,58 @@ is_resizeable(mmap_object *self) return 0; } +#endif /* MS_WINDOWS || HAVE_MREMAP */ +/*[clinic input] +@critical_section +mmap.mmap.write + + bytes as data: Py_buffer + / + +[clinic start generated code]*/ + static PyObject * -mmap_write_method(PyObject *op, PyObject *args) +mmap_mmap_write_impl(mmap_object *self, Py_buffer *data) +/*[clinic end generated code: output=9e97063efb6fb27b input=3f16fa79aa89d6f7]*/ { - Py_buffer data; - mmap_object *self = mmap_object_CAST(op); - CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "y*:write", &data)) - return NULL; - if (!is_writable(self)) { - PyBuffer_Release(&data); return NULL; } - if (self->pos > self->size || self->size - self->pos < data.len) { - PyBuffer_Release(&data); + if (self->pos > self->size || self->size - self->pos < data->len) { PyErr_SetString(PyExc_ValueError, "data out of range"); return NULL; } - CHECK_VALID_OR_RELEASE(NULL, data); + CHECK_VALID(NULL); PyObject *result; - if (safe_memcpy(self->data + self->pos, data.buf, data.len) < 0) { + if (safe_memcpy(self->data + self->pos, data->buf, data->len) < 0) { result = NULL; } else { - self->pos += data.len; - result = PyLong_FromSsize_t(data.len); + self->pos += data->len; + result = PyLong_FromSsize_t(data->len); } - PyBuffer_Release(&data); return result; } +/*[clinic input] +@critical_section +mmap.mmap.write_byte + + byte as value: unsigned_char + / + +[clinic start generated code]*/ + static PyObject * -mmap_write_byte_method(PyObject *op, PyObject *args) +mmap_mmap_write_byte_impl(mmap_object *self, unsigned char value) +/*[clinic end generated code: output=aa11adada9b17510 input=32740bfa174f0991]*/ { - char value; - mmap_object *self = mmap_object_CAST(op); - CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "b:write_byte", &value)) - return(NULL); - if (!is_writable(self)) return NULL; @@ -696,17 +786,23 @@ mmap_write_byte_method(PyObject *op, PyObject *args) return NULL; } - if (safe_byte_copy(self->data + self->pos, &value) < 0) { + if (safe_byte_copy(self->data + self->pos, (const char*)&value) < 0) { return NULL; } self->pos++; Py_RETURN_NONE; } +/*[clinic input] +@critical_section +mmap.mmap.size + +[clinic start generated code]*/ + static PyObject * -mmap_size_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_size_impl(mmap_object *self) +/*[clinic end generated code: output=c177e65e83a648ff input=f69c072efd2e1595]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); #ifdef MS_WINDOWS @@ -725,13 +821,11 @@ mmap_size_method(PyObject *op, PyObject *Py_UNUSED(ignored)) return PyLong_FromLong((long)low); size = (((long long)high)<<32) + low; return PyLong_FromLongLong(size); - } else { - return PyLong_FromSsize_t(self->size); } #endif /* MS_WINDOWS */ #ifdef UNIX - { + if (self->fd != -1) { struct _Py_stat_struct status; if (_Py_fstat(self->fd, &status) == -1) return NULL; @@ -742,6 +836,14 @@ mmap_size_method(PyObject *op, PyObject *Py_UNUSED(ignored)) #endif } #endif /* UNIX */ + else if (self->trackfd) { + return PyLong_FromSsize_t(self->size); + } + else { + PyErr_SetString(PyExc_ValueError, + "can't get size with trackfd=False"); + return NULL; + } } /* This assumes that you want the entire file mapped, @@ -753,14 +855,22 @@ mmap_size_method(PyObject *op, PyObject *Py_UNUSED(ignored)) / new size? */ +#if defined(MS_WINDOWS) || defined(HAVE_MREMAP) +/*[clinic input] +@critical_section +mmap.mmap.resize + + newsize as new_size: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_resize_method(PyObject *op, PyObject *args) +mmap_mmap_resize_impl(mmap_object *self, Py_ssize_t new_size) +/*[clinic end generated code: output=6f262537ce9c2dcc input=b6b5dee52a41b79f]*/ { - Py_ssize_t new_size; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "n:resize", &new_size) || - !is_resizeable(self)) { + if (!is_resizeable(self)) { return NULL; } if (new_size < 0 || PY_SSIZE_T_MAX - new_size < self->offset) { @@ -867,13 +977,15 @@ mmap_resize_method(PyObject *op, PyObject *args) #endif /* MS_WINDOWS */ #ifdef UNIX -#ifndef HAVE_MREMAP - PyErr_SetString(PyExc_SystemError, - "mmap: resizing not available--no mremap()"); - return NULL; -#else void *newmap; +#ifdef __linux__ + if (self->fd == -1 && !(self->flags & MAP_PRIVATE) && new_size > self->size) { + PyErr_Format(PyExc_ValueError, + "mmap: can't expand a shared anonymous mapping on Linux"); + return NULL; + } +#endif if (self->fd != -1 && ftruncate(self->fd, self->offset + new_size) == -1) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -896,28 +1008,43 @@ mmap_resize_method(PyObject *op, PyObject *args) self->data = newmap; self->size = new_size; Py_RETURN_NONE; -#endif /* HAVE_MREMAP */ #endif /* UNIX */ } } +#endif /* MS_WINDOWS || HAVE_MREMAP */ + +/*[clinic input] +@critical_section +mmap.mmap.tell + +[clinic start generated code]*/ static PyObject * -mmap_tell_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_tell_impl(mmap_object *self) +/*[clinic end generated code: output=6034958630e1b1d1 input=fd163acacf45c3a5]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); return PyLong_FromSize_t(self->pos); } +/*[clinic input] +@critical_section +mmap.mmap.flush + + offset: Py_ssize_t = 0 + size: Py_ssize_t = -1 + / + +[clinic start generated code]*/ + static PyObject * -mmap_flush_method(PyObject *op, PyObject *args) +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size) +/*[clinic end generated code: output=956ced67466149cf input=c50b893bc69520ec]*/ { - Py_ssize_t offset = 0; - mmap_object *self = mmap_object_CAST(op); - Py_ssize_t size = self->size; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size)) - return NULL; + if (size == -1) { + size = self->size - offset; + } if (size < 0 || offset < 0 || self->size - offset < size) { PyErr_SetString(PyExc_ValueError, "flush values out of range"); return NULL; @@ -945,60 +1072,80 @@ mmap_flush_method(PyObject *op, PyObject *args) #endif } +/*[clinic input] +@critical_section +mmap.mmap.seek + + pos as dist: Py_ssize_t + whence as how: int = 0 + / + +[clinic start generated code]*/ + static PyObject * -mmap_seek_method(PyObject *op, PyObject *args) +mmap_mmap_seek_impl(mmap_object *self, Py_ssize_t dist, int how) +/*[clinic end generated code: output=00310494e8b8c592 input=e2fda5d081c3db22]*/ { - Py_ssize_t dist; - mmap_object *self = mmap_object_CAST(op); - int how=0; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how)) - return NULL; - else { - Py_ssize_t where; - switch (how) { - case 0: /* relative to start */ - where = dist; - break; - case 1: /* relative to current position */ - if (PY_SSIZE_T_MAX - self->pos < dist) - goto onoutofrange; - where = self->pos + dist; - break; - case 2: /* relative to end */ - if (PY_SSIZE_T_MAX - self->size < dist) - goto onoutofrange; - where = self->size + dist; - break; - default: - PyErr_SetString(PyExc_ValueError, "unknown seek type"); - return NULL; - } - if (where > self->size || where < 0) + Py_ssize_t where; + switch (how) { + case 0: /* relative to start */ + where = dist; + break; + case 1: /* relative to current position */ + if (PY_SSIZE_T_MAX - self->pos < dist) goto onoutofrange; - self->pos = where; - return PyLong_FromSsize_t(self->pos); + where = self->pos + dist; + break; + case 2: /* relative to end */ + if (PY_SSIZE_T_MAX - self->size < dist) + goto onoutofrange; + where = self->size + dist; + break; + default: + PyErr_SetString(PyExc_ValueError, "unknown seek type"); + return NULL; } + if (where > self->size || where < 0) + goto onoutofrange; + self->pos = where; + return PyLong_FromSsize_t(self->pos); onoutofrange: PyErr_SetString(PyExc_ValueError, "seek out of range"); return NULL; } +/*[clinic input] +mmap.mmap.seekable + +[clinic start generated code]*/ + static PyObject * -mmap_seekable_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_seekable_impl(mmap_object *self) +/*[clinic end generated code: output=6311dc3ea300fa38 input=5132505f6e259001]*/ { Py_RETURN_TRUE; } +/*[clinic input] +@critical_section +mmap.mmap.move + + dest: Py_ssize_t + src: Py_ssize_t + count as cnt: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_move_method(PyObject *op, PyObject *args) +mmap_mmap_move_impl(mmap_object *self, Py_ssize_t dest, Py_ssize_t src, + Py_ssize_t cnt) +/*[clinic end generated code: output=391f549a44181793 input=cf8cfe10d9f6b448]*/ { - Py_ssize_t dest, src, cnt; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "nnn:move", &dest, &src, &cnt) || - !is_writable(self)) { + if (!is_writable(self)) { return NULL; } else { /* bounds check the values */ @@ -1024,30 +1171,53 @@ static PyObject * mmap_closed_get(PyObject *op, void *Py_UNUSED(closure)) { mmap_object *self = mmap_object_CAST(op); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); #ifdef MS_WINDOWS - return PyBool_FromLong(self->map_handle == NULL ? 1 : 0); + result = PyBool_FromLong(self->map_handle == NULL ? 1 : 0); #elif defined(UNIX) - return PyBool_FromLong(self->data == NULL ? 1 : 0); + result = PyBool_FromLong(self->data == NULL ? 1 : 0); #endif + Py_END_CRITICAL_SECTION(); + return result; } +/*[clinic input] +@critical_section +mmap.mmap.__enter__ + +[clinic start generated code]*/ + static PyObject * -mmap__enter__method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap___enter___impl(mmap_object *self) +/*[clinic end generated code: output=92cfc59f4c4e2d26 input=a446541fbfe0b890]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); return Py_NewRef(self); } +/*[clinic input] +@critical_section +mmap.mmap.__exit__ + + exc_type: object + exc_value: object + traceback: object + / + +[clinic start generated code]*/ + static PyObject * -mmap__exit__method(PyObject *op, PyObject *Py_UNUSED(args)) +mmap_mmap___exit___impl(mmap_object *self, PyObject *exc_type, + PyObject *exc_value, PyObject *traceback) +/*[clinic end generated code: output=bec7e3e319c1f07e input=5f28e91cf752bc64]*/ { - return mmap_close_method(op, NULL); + return mmap_mmap_close_impl(self); } static PyObject * -mmap__repr__method(PyObject *op) +mmap__repr__method_lock_held(PyObject *op) { mmap_object *mobj = mmap_object_CAST(op); @@ -1091,11 +1261,27 @@ mmap__repr__method(PyObject *op) } } -#ifdef MS_WINDOWS static PyObject * -mmap__sizeof__method(PyObject *op, PyObject *Py_UNUSED(dummy)) +mmap__repr__method(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap__repr__method_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + +#ifdef MS_WINDOWS +/*[clinic input] +@critical_section +mmap.mmap.__sizeof__ + +[clinic start generated code]*/ + +static PyObject * +mmap_mmap___sizeof___impl(mmap_object *self) +/*[clinic end generated code: output=1aed30daff807d09 input=8a648868a089553c]*/ { - mmap_object *self = mmap_object_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->tagname) { res += (wcslen(self->tagname) + 1) * sizeof(self->tagname[0]); @@ -1105,18 +1291,26 @@ mmap__sizeof__method(PyObject *op, PyObject *Py_UNUSED(dummy)) #endif #if defined(MS_WINDOWS) && defined(Py_DEBUG) +/*[clinic input] +@critical_section +mmap.mmap._protect + + flNewProtect: unsigned_int(bitwise=True) + start: Py_ssize_t + length: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_protect_method(PyObject *op, PyObject *args) { - DWORD flNewProtect, flOldProtect; - Py_ssize_t start, length; - mmap_object *self = mmap_object_CAST(op); +mmap_mmap__protect_impl(mmap_object *self, unsigned int flNewProtect, + Py_ssize_t start, Py_ssize_t length) +/*[clinic end generated code: output=a87271a34d1ad6cf input=9170498c5e1482da]*/ +{ + DWORD flOldProtect; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "Inn:protect", &flNewProtect, &start, &length)) { - return NULL; - } - if (!VirtualProtect((void *) (self->data + start), length, flNewProtect, &flOldProtect)) { @@ -1129,18 +1323,32 @@ mmap_protect_method(PyObject *op, PyObject *args) { #endif #ifdef HAVE_MADVISE +/*[clinic input] +@critical_section +mmap.mmap.madvise + + option: int + start: Py_ssize_t = 0 + length as length_obj: object = None + / + +[clinic start generated code]*/ + static PyObject * -mmap_madvise_method(PyObject *op, PyObject *args) +mmap_mmap_madvise_impl(mmap_object *self, int option, Py_ssize_t start, + PyObject *length_obj) +/*[clinic end generated code: output=816be656f08c0e3c input=2d37f7a4c87f1053]*/ { - int option; - Py_ssize_t start = 0, length; - mmap_object *self = mmap_object_CAST(op); + Py_ssize_t length; CHECK_VALID(NULL); - length = self->size; - - if (!PyArg_ParseTuple(args, "i|nn:madvise", &option, &start, &length)) { - return NULL; + if (length_obj == Py_None) { + length = self->size; + } else { + length = _As_Py_ssize_t(length_obj); + if (length == -1 && PyErr_Occurred()) { + return NULL; + } } if (start < 0 || start >= self->size) { @@ -1176,32 +1384,26 @@ static struct PyMemberDef mmap_object_members[] = { }; static struct PyMethodDef mmap_object_methods[] = { - {"close", mmap_close_method, METH_NOARGS}, - {"find", mmap_find_method, METH_VARARGS}, - {"rfind", mmap_rfind_method, METH_VARARGS}, - {"flush", mmap_flush_method, METH_VARARGS}, -#ifdef HAVE_MADVISE - {"madvise", mmap_madvise_method, METH_VARARGS}, -#endif - {"move", mmap_move_method, METH_VARARGS}, - {"read", mmap_read_method, METH_VARARGS}, - {"read_byte", mmap_read_byte_method, METH_NOARGS}, - {"readline", mmap_read_line_method, METH_NOARGS}, - {"resize", mmap_resize_method, METH_VARARGS}, - {"seek", mmap_seek_method, METH_VARARGS}, - {"seekable", mmap_seekable_method, METH_NOARGS}, - {"size", mmap_size_method, METH_NOARGS}, - {"tell", mmap_tell_method, METH_NOARGS}, - {"write", mmap_write_method, METH_VARARGS}, - {"write_byte", mmap_write_byte_method, METH_VARARGS}, - {"__enter__", mmap__enter__method, METH_NOARGS}, - {"__exit__", mmap__exit__method, METH_VARARGS}, -#ifdef MS_WINDOWS - {"__sizeof__", mmap__sizeof__method, METH_NOARGS}, -#ifdef Py_DEBUG - {"_protect", mmap_protect_method, METH_VARARGS}, -#endif // Py_DEBUG -#endif // MS_WINDOWS + MMAP_MMAP_CLOSE_METHODDEF + MMAP_MMAP_FIND_METHODDEF + MMAP_MMAP_RFIND_METHODDEF + MMAP_MMAP_FLUSH_METHODDEF + MMAP_MMAP_MADVISE_METHODDEF + MMAP_MMAP_MOVE_METHODDEF + MMAP_MMAP_READ_METHODDEF + MMAP_MMAP_READ_BYTE_METHODDEF + MMAP_MMAP_READLINE_METHODDEF + MMAP_MMAP_RESIZE_METHODDEF + MMAP_MMAP_SEEK_METHODDEF + MMAP_MMAP_SEEKABLE_METHODDEF + MMAP_MMAP_SIZE_METHODDEF + MMAP_MMAP_TELL_METHODDEF + MMAP_MMAP_WRITE_METHODDEF + MMAP_MMAP_WRITE_BYTE_METHODDEF + MMAP_MMAP___ENTER___METHODDEF + MMAP_MMAP___EXIT___METHODDEF + MMAP_MMAP___SIZEOF___METHODDEF + MMAP_MMAP__PROTECT_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1214,7 +1416,7 @@ static PyGetSetDef mmap_object_getset[] = { /* Functions for treating an mmap'ed file as a buffer */ static int -mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +mmap_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); @@ -1225,23 +1427,45 @@ mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) return 0; } +static int +mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_buffer_getbuf_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return result; +} + static void mmap_buffer_releasebuf(PyObject *op, Py_buffer *Py_UNUSED(view)) { mmap_object *self = mmap_object_CAST(op); + Py_BEGIN_CRITICAL_SECTION(self); self->exports--; + Py_END_CRITICAL_SECTION(); } static Py_ssize_t -mmap_length(PyObject *op) +mmap_length_lock_held(PyObject *op) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); return self->size; } +static Py_ssize_t +mmap_length(PyObject *op) +{ + Py_ssize_t result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_length_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * -mmap_item(PyObject *op, Py_ssize_t i) +mmap_item_lock_held(PyObject *op, Py_ssize_t i) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -1258,7 +1482,16 @@ mmap_item(PyObject *op, Py_ssize_t i) } static PyObject * -mmap_subscript(PyObject *op, PyObject *item) +mmap_item(PyObject *op, Py_ssize_t i) { + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_item_lock_held(op, i); + Py_END_CRITICAL_SECTION(); + return result; +} + +static PyObject * +mmap_subscript_lock_held(PyObject *op, PyObject *item) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -1320,8 +1553,18 @@ mmap_subscript(PyObject *op, PyObject *item) } } +static PyObject * +mmap_subscript(PyObject *op, PyObject *item) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_subscript_lock_held(op, item); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) +mmap_ass_item_lock_held(PyObject *op, Py_ssize_t i, PyObject *v) { const char *buf; mmap_object *self = mmap_object_CAST(op); @@ -1352,7 +1595,17 @@ mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) } static int -mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) +mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_ass_item_lock_held(op, i, v); + Py_END_CRITICAL_SECTION(); + return result; +} + +static int +mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); @@ -1448,11 +1701,21 @@ mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) } } +static int +mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_ass_subscript_lock_held(op, item, value); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict); PyDoc_STRVAR(mmap_doc, -"Windows: mmap(fileno, length[, tagname[, access[, offset]]])\n\ +"Windows: mmap(fileno, length[, tagname[, access[, offset[, trackfd]]]])\n\ \n\ Maps length bytes from the file specified by the file handle fileno,\n\ and returns a mmap object. If length is larger than the current size\n\ @@ -1484,7 +1747,7 @@ static PyType_Slot mmap_object_slots[] = { {Py_tp_members, mmap_object_members}, {Py_tp_getset, mmap_object_getset}, {Py_tp_getattro, PyObject_GenericGetAttr}, - {Py_tp_traverse, mmap_object_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, /* as sequence */ {Py_sq_length, mmap_length}, @@ -1670,6 +1933,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) else { m_obj->fd = -1; } + m_obj->flags = flags; Py_BEGIN_ALLOW_THREADS m_obj->data = mmap(NULL, map_size, prot, flags, fd, offset); @@ -1709,19 +1973,20 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ - PyObject *tagname = NULL; + PyObject *tagname = Py_None; DWORD dwErr = 0; int fileno; - HANDLE fh = 0; + HANDLE fh = INVALID_HANDLE_VALUE; int access = (access_mode)ACCESS_DEFAULT; + int trackfd = 1; DWORD flProtect, dwDesiredAccess; static char *keywords[] = { "fileno", "length", "tagname", - "access", "offset", NULL }; + "access", "offset", "trackfd", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|U?iL", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL$p", keywords, &fileno, &map_size, - &tagname, &access, &offset)) { + &tagname, &access, &offset, &trackfd)) { return NULL; } @@ -1788,22 +2053,27 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->map_handle = NULL; m_obj->tagname = NULL; m_obj->offset = offset; + m_obj->trackfd = trackfd; - if (fh) { - /* It is necessary to duplicate the handle, so the - Python code can close it on us */ - if (!DuplicateHandle( - GetCurrentProcess(), /* source process handle */ - fh, /* handle to be duplicated */ - GetCurrentProcess(), /* target proc handle */ - (LPHANDLE)&m_obj->file_handle, /* result */ - 0, /* access - ignored due to options value */ - FALSE, /* inherited by child processes? */ - DUPLICATE_SAME_ACCESS)) { /* options */ - dwErr = GetLastError(); - Py_DECREF(m_obj); - PyErr_SetFromWindowsErr(dwErr); - return NULL; + if (fh != INVALID_HANDLE_VALUE) { + if (trackfd) { + /* It is necessary to duplicate the handle, so the + Python code can close it on us */ + if (!DuplicateHandle( + GetCurrentProcess(), /* source process handle */ + fh, /* handle to be duplicated */ + GetCurrentProcess(), /* target proc handle */ + &fh, /* result */ + 0, /* access - ignored due to options value */ + FALSE, /* inherited by child processes? */ + DUPLICATE_SAME_ACCESS)) /* options */ + { + dwErr = GetLastError(); + Py_DECREF(m_obj); + PyErr_SetFromWindowsErr(dwErr); + return NULL; + } + m_obj->file_handle = fh; } if (!map_size) { DWORD low,high; @@ -1811,7 +2081,8 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) /* low might just happen to have the value INVALID_FILE_SIZE; so we need to check the last error also. */ if (low == INVALID_FILE_SIZE && - (dwErr = GetLastError()) != NO_ERROR) { + (dwErr = GetLastError()) != NO_ERROR) + { Py_DECREF(m_obj); return PyErr_SetFromWindowsErr(dwErr); } @@ -1852,7 +2123,13 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; /* set the tag name */ - if (tagname != NULL) { + if (!Py_IsNone(tagname)) { + if (!PyUnicode_Check(tagname)) { + Py_DECREF(m_obj); + return PyErr_Format(PyExc_TypeError, "expected str or None for " + "'tagname', not %.200s", + Py_TYPE(tagname)->tp_name); + } m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL); if (m_obj->tagname == NULL) { Py_DECREF(m_obj); @@ -1867,7 +2144,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) off_lo = (DWORD)(offset & 0xFFFFFFFF); /* For files, it would be sufficient to pass 0 as size. For anonymous maps, we have to pass the size explicitly. */ - m_obj->map_handle = CreateFileMappingW(m_obj->file_handle, + m_obj->map_handle = CreateFileMappingW(fh, NULL, flProtect, size_hi, diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 922694fa367..fc609b2707c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -40,6 +40,7 @@ // --- System includes ------------------------------------------------------ +#include <stddef.h> // offsetof() #include <stdio.h> // ctermid() #include <stdlib.h> // system() @@ -408,6 +409,31 @@ extern char *ctermid_r(char *); # define STRUCT_STAT struct stat #endif +#ifdef HAVE_STATX +/* until we can assume glibc 2.28 at runtime, we must weakly link */ +# pragma weak statx +static const unsigned int _Py_STATX_KNOWN = (STATX_BASIC_STATS | STATX_BTIME +#ifdef STATX_MNT_ID + | STATX_MNT_ID +#endif +#ifdef STATX_DIOALIGN + | STATX_DIOALIGN +#endif +#ifdef STATX_MNT_ID_UNIQUE + | STATX_MNT_ID_UNIQUE +#endif +#ifdef STATX_SUBVOL + | STATX_SUBVOL +#endif +#ifdef STATX_WRITE_ATOMIC + | STATX_WRITE_ATOMIC +#endif +#ifdef STATX_DIO_READ_ALIGN + | STATX_DIO_READ_ALIGN +#endif + ); +#endif /* HAVE_STATX */ + #if !defined(EX_OK) && defined(EXIT_SUCCESS) # define EX_OK EXIT_SUCCESS @@ -685,7 +711,16 @@ static void reset_remotedebug_data(PyThreadState *tstate) { tstate->remote_debugger_support.debugger_pending_call = 0; - memset(tstate->remote_debugger_support.debugger_script_path, 0, MAX_SCRIPT_PATH_SIZE); + memset(tstate->remote_debugger_support.debugger_script_path, 0, + _Py_MAX_SCRIPT_PATH_SIZE); +} + +static void +reset_asyncio_state(_PyThreadStateImpl *tstate) +{ + llist_init(&tstate->asyncio_tasks_head); + tstate->asyncio_running_loop = NULL; + tstate->asyncio_running_task = NULL; } @@ -724,6 +759,8 @@ PyOS_AfterFork_Child(void) reset_remotedebug_data(tstate); + reset_asyncio_state((_PyThreadStateImpl *)tstate); + // Remove the dead thread states. We "start the world" once we are the only // thread state left to undo the stop the world call in `PyOS_BeforeFork`. // That needs to happen before `_PyThreadState_DeleteList`, because that @@ -1158,6 +1195,9 @@ typedef struct { #endif newfunc statresult_new_orig; PyObject *StatResultType; +#ifdef HAVE_STATX + PyObject *StatxResultType; +#endif PyObject *StatVFSResultType; PyObject *TerminalSizeType; PyObject *TimesResultType; @@ -1782,7 +1822,7 @@ convertenviron(void) return NULL; } #ifdef MS_WINDOWS - v = PyUnicode_FromWideChar(p+1, wcslen(p+1)); + v = PyUnicode_FromWideChar(p+1, -1); #else v = PyBytes_FromStringAndSize(p+1, strlen(p+1)); #endif @@ -2431,7 +2471,7 @@ static PyStructSequence_Field stat_result_fields[] = { #endif static PyStructSequence_Desc stat_result_desc = { - "stat_result", /* name */ + "os.stat_result", /* name; see issue gh-63408 */ stat_result__doc__, /* doc */ stat_result_fields, 10 @@ -2461,7 +2501,7 @@ static PyStructSequence_Field statvfs_result_fields[] = { }; static PyStructSequence_Desc statvfs_result_desc = { - "statvfs_result", /* name */ + "os.statvfs_result", /* name; see issue gh-63408 */ statvfs_result__doc__, /* doc */ statvfs_result_fields, 10 @@ -2486,7 +2526,7 @@ static PyStructSequence_Field waitid_result_fields[] = { }; static PyStructSequence_Desc waitid_result_desc = { - "waitid_result", /* name */ + MODNAME ".waitid_result", /* name */ waitid_result__doc__, /* doc */ waitid_result_fields, 5 @@ -2538,6 +2578,9 @@ _posix_clear(PyObject *module) Py_CLEAR(state->SchedParamType); #endif Py_CLEAR(state->StatResultType); +#ifdef HAVE_STATX + Py_CLEAR(state->StatxResultType); +#endif Py_CLEAR(state->StatVFSResultType); Py_CLEAR(state->TerminalSizeType); Py_CLEAR(state->TimesResultType); @@ -2563,6 +2606,9 @@ _posix_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->SchedParamType); #endif Py_VISIT(state->StatResultType); +#ifdef HAVE_STATX + Py_VISIT(state->StatxResultType); +#endif Py_VISIT(state->StatVFSResultType); Py_VISIT(state->TerminalSizeType); Py_VISIT(state->TimesResultType); @@ -2583,60 +2629,79 @@ _posix_free(void *module) _posix_clear((PyObject *)module); } + +#define SEC_TO_NS (1000000000LL) +static PyObject * +stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) +{ +#if SIZEOF_TIME_T == 4 + return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); +#else + /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ + if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { + return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); + } + else + { + PyObject *ns_total = NULL; + PyObject *s_in_ns = NULL; + PyObject *s = _PyLong_FromTime_t(sec); + PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); + if (s == NULL || ns_fractional == NULL) { + goto exit; + } + + s_in_ns = PyNumber_Multiply(s, state->billion); + if (s_in_ns == NULL) { + goto exit; + } + + ns_total = PyNumber_Add(s_in_ns, ns_fractional); + + exit: + Py_XDECREF(s); + Py_XDECREF(ns_fractional); + Py_XDECREF(s_in_ns); + return ns_total; + } +#endif +} + static int -fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, time_t sec, unsigned long nsec) +fill_time(_posixstate *state, PyObject *v, int s_index, int f_index, + int ns_index, time_t sec, unsigned long nsec) { assert(!PyErr_Occurred()); - - int res = -1; - PyObject *s_in_ns = NULL; - PyObject *ns_total = NULL; - PyObject *float_s = NULL; - - PyObject *s = _PyLong_FromTime_t(sec); - PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); - if (!(s && ns_fractional)) { - goto exit; - } - - s_in_ns = PyNumber_Multiply(s, get_posix_state(module)->billion); - if (!s_in_ns) { - goto exit; - } - - ns_total = PyNumber_Add(s_in_ns, ns_fractional); - if (!ns_total) - goto exit; - - float_s = PyFloat_FromDouble(sec + 1e-9*nsec); - if (!float_s) { - goto exit; - } + assert(nsec < SEC_TO_NS); if (s_index >= 0) { + PyObject *s = _PyLong_FromTime_t(sec); + if (s == NULL) { + return -1; + } PyStructSequence_SET_ITEM(v, s_index, s); - s = NULL; } + if (f_index >= 0) { + PyObject *float_s = PyFloat_FromDouble((double)sec + 1e-9 * nsec); + if (float_s == NULL) { + return -1; + } PyStructSequence_SET_ITEM(v, f_index, float_s); - float_s = NULL; } + if (ns_index >= 0) { + PyObject *ns_total = stat_nanosecond_timestamp(state, sec, nsec); + if (ns_total == NULL) { + return -1; + } PyStructSequence_SET_ITEM(v, ns_index, ns_total); - ns_total = NULL; } assert(!PyErr_Occurred()); - res = 0; - -exit: - Py_XDECREF(s); - Py_XDECREF(ns_fractional); - Py_XDECREF(s_in_ns); - Py_XDECREF(ns_total); - Py_XDECREF(float_s); - return res; + return 0; } +#undef SEC_TO_NS #ifdef MS_WINDOWS static PyObject* @@ -2672,7 +2737,8 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) { assert(!PyErr_Occurred()); - PyObject *StatResultType = get_posix_state(module)->StatResultType; + _posixstate *state = get_posix_state(module); + PyObject *StatResultType = state->StatResultType; PyObject *v = PyStructSequence_New((PyTypeObject *)StatResultType); if (v == NULL) { return NULL; @@ -2726,13 +2792,13 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) #else ansec = mnsec = cnsec = 0; #endif - if (fill_time(module, v, 7, 10, 13, st->st_atime, ansec) < 0) { + if (fill_time(state, v, 7, 10, 13, st->st_atime, ansec) < 0) { goto error; } - if (fill_time(module, v, 8, 11, 14, st->st_mtime, mnsec) < 0) { + if (fill_time(state, v, 8, 11, 14, st->st_mtime, mnsec) < 0) { goto error; } - if (fill_time(module, v, 9, 12, 15, st->st_ctime, cnsec) < 0) { + if (fill_time(state, v, 9, 12, 15, st->st_ctime, cnsec) < 0) { goto error; } @@ -2743,7 +2809,7 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) SET_ITEM(ST_BLOCKS_IDX, PyLong_FromLong((long)st->st_blocks)); #endif #ifdef HAVE_STRUCT_STAT_ST_RDEV - SET_ITEM(ST_RDEV_IDX, PyLong_FromLong((long)st->st_rdev)); + SET_ITEM(ST_RDEV_IDX, _PyLong_FromDev(st->st_rdev)); #endif #ifdef HAVE_STRUCT_STAT_ST_GEN SET_ITEM(ST_GEN_IDX, PyLong_FromLong((long)st->st_gen)); @@ -2760,7 +2826,7 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) SET_ITEM(ST_BIRTHTIME_IDX, PyFloat_FromDouble(bsec + bnsec * 1e-9)); } #elif defined(MS_WINDOWS) - if (fill_time(module, v, -1, ST_BIRTHTIME_IDX, ST_BIRTHTIME_NS_IDX, + if (fill_time(state, v, -1, ST_BIRTHTIME_IDX, ST_BIRTHTIME_NS_IDX, st->st_birthtime, st->st_birthtime_nsec) < 0) { goto error; } @@ -3121,17 +3187,6 @@ class dev_t_return_converter(unsigned_long_return_converter): conversion_fn = '_PyLong_FromDev' unsigned_cast = '(dev_t)' -class FSConverter_converter(CConverter): - type = 'PyObject *' - converter = 'PyUnicode_FSConverter' - def converter_init(self): - if self.default is not unspecified: - fail("FSConverter_converter does not support default values") - self.c_default = 'NULL' - - def cleanup(self): - return "Py_XDECREF(" + self.name + ");\n" - class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' @@ -3195,7 +3250,7 @@ class confname_converter(CConverter): """, argname=argname, converter=self.converter, table=self.table) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=8189d5ae78244626]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=d2759f2332cd39b3]*/ /*[clinic input] @@ -3237,6 +3292,7 @@ os_stat_impl(PyObject *module, path_t *path, int dir_fd, int follow_symlinks) /*[clinic input] +@permit_long_summary os.lstat path : path_t @@ -3253,13 +3309,392 @@ Equivalent to stat(path, follow_symlinks=False). static PyObject * os_lstat_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=ef82a5d35ce8ab37 input=0b7474765927b925]*/ +/*[clinic end generated code: output=ef82a5d35ce8ab37 input=024102124f88e743]*/ { int follow_symlinks = 0; return posix_do_stat(module, "lstat", path, dir_fd, follow_symlinks); } +#ifdef HAVE_STATX +typedef struct { + PyObject_HEAD + dev_t rdev, dev; + struct statx stx; +} Py_statx_result; + +#define Py_statx_result_CAST(op) _Py_CAST(Py_statx_result*, (op)) + +#define M(attr, type, offset, doc) \ + {attr, type, offset, Py_READONLY, PyDoc_STR(doc)} +#define MM(attr, type, member, doc) \ + M(#attr, type, offsetof(Py_statx_result, stx.stx_##member), doc) +#define MX(attr, type, member, doc) \ + M(#attr, type, offsetof(Py_statx_result, member), doc) + +static PyMemberDef pystatx_result_members[] = { + MM(stx_mask, Py_T_UINT, mask, "member validity mask"), + MM(stx_blksize, Py_T_UINT, blksize, "blocksize for filesystem I/O"), + MM(stx_attributes, Py_T_ULONGLONG, attributes, "Linux inode attribute bits"), + MM(stx_attributes_mask, Py_T_ULONGLONG, attributes_mask, + "Mask of supported bits in stx_attributes"), + MM(stx_rdev_major, Py_T_UINT, rdev_major, "represented device major number"), + MM(stx_rdev_minor, Py_T_UINT, rdev_minor, "represented device minor number"), + MX(stx_rdev, Py_T_ULONGLONG, rdev, "device type (if inode device)"), + MM(stx_dev_major, Py_T_UINT, dev_major, "containing device major number"), + MM(stx_dev_minor, Py_T_UINT, dev_minor, "containing device minor number"), + MX(stx_dev, Py_T_ULONGLONG, dev, "device"), + {NULL}, +}; + +#undef MX +#undef MM +#undef M + + +#define STATX_GET_UINT(ATTR, MASK) \ + static PyObject* \ + pystatx_result_get_##ATTR(PyObject *op, void *Py_UNUSED(context)) \ + { \ + Py_statx_result *self = Py_statx_result_CAST(op); \ + if (!(self->stx.stx_mask & MASK)) { \ + Py_RETURN_NONE; \ + } \ + unsigned long value = self->stx.ATTR; \ + return PyLong_FromUnsignedLong(value); \ + } + +STATX_GET_UINT(stx_uid, STATX_UID) +STATX_GET_UINT(stx_gid, STATX_GID) +STATX_GET_UINT(stx_nlink, STATX_NLINK) +#ifdef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN +STATX_GET_UINT(stx_dio_mem_align, STATX_DIOALIGN) +STATX_GET_UINT(stx_dio_offset_align, STATX_DIOALIGN) +#endif +#ifdef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN +STATX_GET_UINT(stx_dio_read_offset_align, STATX_DIO_READ_ALIGN) +#endif +#ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN +STATX_GET_UINT(stx_atomic_write_unit_min, STATX_WRITE_ATOMIC) +STATX_GET_UINT(stx_atomic_write_unit_max, STATX_WRITE_ATOMIC) +STATX_GET_UINT(stx_atomic_write_segments_max, STATX_WRITE_ATOMIC) +#endif +#ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MAX_OPT +STATX_GET_UINT(stx_atomic_write_unit_max_opt, STATX_WRITE_ATOMIC) +#endif + + +static PyObject* +pystatx_result_get_stx_mode(PyObject *op, void *Py_UNUSED(context)) +{ + Py_statx_result *self = Py_statx_result_CAST(op); + if (!(self->stx.stx_mask & (STATX_TYPE | STATX_MODE))) { + Py_RETURN_NONE; + } + return PyLong_FromUnsignedLong(self->stx.stx_mode); +} + + +#define STATX_GET_ULONGLONG(ATTR, MASK) \ + static PyObject* \ + pystatx_result_get_##ATTR(PyObject *op, void *Py_UNUSED(context)) \ + { \ + Py_statx_result *self = Py_statx_result_CAST(op); \ + if (!(self->stx.stx_mask & MASK)) { \ + Py_RETURN_NONE; \ + } \ + unsigned long long value = self->stx.ATTR; \ + return PyLong_FromUnsignedLongLong(value); \ + } + +STATX_GET_ULONGLONG(stx_blocks, STATX_BLOCKS) +STATX_GET_ULONGLONG(stx_ino, STATX_INO) +STATX_GET_ULONGLONG(stx_size, STATX_SIZE) +#if defined(STATX_MNT_ID) && defined(HAVE_STRUCT_STATX_STX_MNT_ID) +STATX_GET_ULONGLONG(stx_mnt_id, STATX_MNT_ID) +#endif +#if defined(STATX_SUBVOL) && defined(HAVE_STRUCT_STATX_STX_SUBVOL) +STATX_GET_ULONGLONG(stx_subvol, STATX_SUBVOL) +#endif + + +#define STATX_GET_DOUBLE(ATTR, MASK) \ + static PyObject* \ + pystatx_result_get_##ATTR(PyObject *op, void *Py_UNUSED(context)) \ + { \ + Py_statx_result *self = Py_statx_result_CAST(op); \ + if (!(self->stx.stx_mask & MASK)) { \ + Py_RETURN_NONE; \ + } \ + struct statx_timestamp *ts = &self->stx.ATTR; \ + double sec = ((double)ts->tv_sec + ts->tv_nsec * 1e-9); \ + return PyFloat_FromDouble(sec); \ + } + +STATX_GET_DOUBLE(stx_atime, STATX_ATIME) +STATX_GET_DOUBLE(stx_btime, STATX_BTIME) +STATX_GET_DOUBLE(stx_ctime, STATX_CTIME) +STATX_GET_DOUBLE(stx_mtime, STATX_MTIME) + +#define STATX_GET_NSEC(ATTR, MEMBER, MASK) \ + static PyObject* \ + pystatx_result_get_##ATTR(PyObject *op, void *context) \ + { \ + Py_statx_result *self = Py_statx_result_CAST(op); \ + if (!(self->stx.stx_mask & MASK)) { \ + Py_RETURN_NONE; \ + } \ + struct statx_timestamp *ts = &self->stx.MEMBER; \ + _posixstate *state = PyType_GetModuleState(Py_TYPE(op)); \ + assert(state != NULL); \ + return stat_nanosecond_timestamp(state, ts->tv_sec, ts->tv_nsec); \ + } + +STATX_GET_NSEC(stx_atime_ns, stx_atime, STATX_ATIME) +STATX_GET_NSEC(stx_btime_ns, stx_btime, STATX_BTIME) +STATX_GET_NSEC(stx_ctime_ns, stx_ctime, STATX_CTIME) +STATX_GET_NSEC(stx_mtime_ns, stx_mtime, STATX_MTIME) + +#define G(attr, doc) \ + {#attr, pystatx_result_get_##attr, NULL, PyDoc_STR(doc), NULL} + +static PyGetSetDef pystatx_result_getset[] = { + G(stx_mode, "protection bits"), + G(stx_nlink, "number of hard links"), + G(stx_uid, "user ID of owner"), + G(stx_gid, "group ID of owner"), + G(stx_ino, "inode"), + G(stx_size, "total size, in bytes"), + G(stx_blocks, "number of blocks allocated"), + G(stx_atime, "time of last access"), + G(stx_atime_ns, "time of last access in nanoseconds"), + G(stx_btime, "time of creation"), + G(stx_btime_ns, "time of creation in nanoseconds"), + G(stx_ctime, "time of last change"), + G(stx_ctime_ns, "time of last change in nanoseconds"), + G(stx_mtime, "time of last modification"), + G(stx_mtime_ns, "time of last modification in nanoseconds"), +#if defined(STATX_MNT_ID) && defined(HAVE_STRUCT_STATX_STX_MNT_ID) + G(stx_mnt_id, "mount ID"), +#endif +#ifdef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN + G(stx_dio_mem_align, "direct I/O memory buffer alignment"), + G(stx_dio_offset_align, "direct I/O file offset alignment"), +#endif +#if defined(STATX_SUBVOL) && defined(HAVE_STRUCT_STATX_STX_SUBVOL) + G(stx_subvol, "subvolume ID"), +#endif +#ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN + G(stx_atomic_write_unit_min, + "minimum size for direct I/O with torn-write protection"), + G(stx_atomic_write_unit_max, + "maximum size for direct I/O with torn-write protection"), + G(stx_atomic_write_segments_max, + "maximum iovecs for direct I/O with torn-write protection"), +#endif +#ifdef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN + G(stx_dio_read_offset_align, "direct I/O file offset alignment for reads"), +#endif +#ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MAX_OPT + G(stx_atomic_write_unit_max_opt, + "maximum optimized size for direct I/O with torn-write protection"), +#endif + {NULL}, +}; + +#undef G + +static PyObject * +pystatx_result_repr(PyObject *op) +{ + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } +#define WRITE_ASCII(s) \ + do { \ + if (PyUnicodeWriter_WriteASCII(writer, s, strlen(s)) < 0) { \ + goto error; \ + } \ + } while (0) + + WRITE_ASCII("os.statx_result("); + + for (size_t i = 0; i < Py_ARRAY_LENGTH(pystatx_result_members) - 1; ++i) { + if (i > 0) { + WRITE_ASCII(", "); + } + + PyMemberDef *d = &pystatx_result_members[i]; + WRITE_ASCII(d->name); + WRITE_ASCII("="); + + PyObject *o = PyMember_GetOne((const char *)op, d); + if (o == NULL) { + goto error; + } + if (PyUnicodeWriter_WriteRepr(writer, o) < 0) { + Py_DECREF(o); + goto error; + } + Py_DECREF(o); + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(pystatx_result_getset) - 1; ++i) { + PyGetSetDef *d = &pystatx_result_getset[i]; + PyObject *o = d->get(op, d->closure); + if (o == NULL) { + goto error; + } + if (o == Py_None) { + continue; + } + + WRITE_ASCII(", "); + WRITE_ASCII(d->name); + WRITE_ASCII("="); + if (PyUnicodeWriter_WriteRepr(writer, o) < 0) { + Py_DECREF(o); + goto error; + } + Py_DECREF(o); + } + + WRITE_ASCII(")"); + return PyUnicodeWriter_Finish(writer); +#undef WRITE_ASCII + +error: + PyUnicodeWriter_Discard(writer); + return NULL; +} + +static int +pystatx_result_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +pystatx_result_dealloc(PyObject *op) +{ + Py_statx_result *self = (Py_statx_result *) op; + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot pystatx_result_slots[] = { + {Py_tp_repr, pystatx_result_repr}, + {Py_tp_traverse, pystatx_result_traverse}, + {Py_tp_dealloc, pystatx_result_dealloc}, + {Py_tp_members, pystatx_result_members}, + {Py_tp_getset, pystatx_result_getset}, + {0, NULL}, +}; + +static PyType_Spec pystatx_result_spec = { + .name = "os.statx_result", + .basicsize = sizeof(Py_statx_result), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .slots = pystatx_result_slots, +}; + +/*[clinic input] + +os.statx + + path : path_t(allow_fd=True) + Path to be examined; can be string, bytes, a path-like object or + open-file-descriptor int. + + mask: unsigned_int(bitwise=True) + A bitmask of STATX_* constants defining the requested information. + + * + + flags: int = 0 + A bitmask of AT_NO_AUTOMOUNT and/or AT_STATX_* flags. + + dir_fd : dir_fd = None + If not None, it should be a file descriptor open to a directory, + and path should be a relative string; path will then be relative to + that directory. + + follow_symlinks: bool = True + If False, and the last element of the path is a symbolic link, + statx will examine the symbolic link itself instead of the file + the link points to. + +Perform a statx system call on the given path. + +It's an error to use dir_fd or follow_symlinks when specifying path as + an open file descriptor. + +[clinic start generated code]*/ + +static PyObject * +os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int flags, + int dir_fd, int follow_symlinks) +/*[clinic end generated code: output=e3765979ac6fe15b input=f0116380c5dc4f2f]*/ +{ + if (path_and_dir_fd_invalid("statx", path, dir_fd) || + dir_fd_and_fd_invalid("statx", dir_fd, path->fd) || + fd_and_follow_symlinks_invalid("statx", path->fd, follow_symlinks)) { + return NULL; + } + + /* reject flags covered by kwargs, but allow unknown flags that may be + future AT_STATX_* extensions */ + if (flags & (AT_SYMLINK_NOFOLLOW | AT_SYMLINK_FOLLOW)) { + PyErr_Format(PyExc_ValueError, + "use follow_symlinks kwarg instead of AT_SYMLINK_* flag"); + return NULL; + } + if (flags & AT_EMPTY_PATH) { + PyErr_Format(PyExc_ValueError, + "use dir_fd kwarg instead of AT_EMPTY_PATH flag"); + return NULL; + } + + /* Future bits may refer to members beyond the current size of struct + statx, so we need to mask them off to prevent memory corruption. */ + mask &= _Py_STATX_KNOWN; + + _posixstate *state = get_posix_state(module); + PyTypeObject *tp = (PyTypeObject *)state->StatxResultType; + Py_statx_result *v = (Py_statx_result *)tp->tp_alloc(tp, 0); + if (v == NULL) { + return NULL; + } + + int result; + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + result = statx(path->fd, "", flags | AT_EMPTY_PATH, mask, &v->stx); + } + else { + result = statx(dir_fd, path->narrow, flags, mask, &v->stx); + } + Py_END_ALLOW_THREADS + + if (result != 0) { + Py_DECREF(v); + return path_error(path); + } + + v->rdev = makedev(v->stx.stx_rdev_major, v->stx.stx_rdev_minor); + v->dev = makedev(v->stx.stx_dev_major, v->stx.stx_dev_minor); + + assert(!PyErr_Occurred()); + return (PyObject *)v; +} +#endif /* HAVE_STATX */ + + /*[clinic input] os.access -> bool @@ -3294,15 +3729,15 @@ dir_fd, effective_ids, and follow_symlinks may not be implemented NotImplementedError. Note that most operations will use the effective uid/gid, therefore this - routine can be used in a suid/sgid environment to test if the invoking user - has the specified access to the path. + routine can be used in a suid/sgid environment to test if the invoking + user has the specified access to the path. [clinic start generated code]*/ static int os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, int effective_ids, int follow_symlinks) -/*[clinic end generated code: output=cf84158bc90b1a77 input=3ffe4e650ee3bf20]*/ +/*[clinic end generated code: output=cf84158bc90b1a77 input=c33565f7584b99e4]*/ { int return_value; @@ -3476,12 +3911,12 @@ Change the current working directory to the specified path. path may always be specified as a string. On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +If this functionality is unavailable, using it raises an exception. [clinic start generated code]*/ static PyObject * os_chdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=3be6400eee26eaae input=1a4a15b4d12cb15d]*/ +/*[clinic end generated code: output=3be6400eee26eaae input=a74ceab5d72adf74]*/ { int result; @@ -4343,6 +4778,7 @@ os_getcwdb_impl(PyObject *module) #ifdef HAVE_LINK /*[clinic input] +@permit_long_docstring_body os.link src : path_t @@ -4368,7 +4804,7 @@ src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your static PyObject * os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int follow_symlinks) -/*[clinic end generated code: output=7f00f6007fd5269a input=1d5e602d115fed7b]*/ +/*[clinic end generated code: output=7f00f6007fd5269a input=e2a50a6497050e44]*/ { #ifdef MS_WINDOWS BOOL result = FALSE; @@ -4662,6 +5098,7 @@ _posix_listdir(path_t *path, PyObject *list) /*[clinic input] +@permit_long_docstring_body os.listdir path : path_t(nullable=True, allow_fd='PATH_HAVE_FDOPENDIR') = None @@ -4684,7 +5121,7 @@ entries '.' and '..' even if they are present in the directory. static PyObject * os_listdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=293045673fcd1a75 input=e3f58030f538295d]*/ +/*[clinic end generated code: output=293045673fcd1a75 input=0bd1728387391b9a]*/ { if (PySys_Audit("os.listdir", "O", path->object ? path->object : Py_None) < 0) { @@ -4698,7 +5135,7 @@ os_listdir_impl(PyObject *module, path_t *path) } -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) /*[clinic input] os.listdrives @@ -4747,6 +5184,10 @@ os_listdrives_impl(PyObject *module) return result; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + +#if defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] os.listvolumes @@ -4808,6 +5249,9 @@ os_listvolumes_impl(PyObject *module) return result; } +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ + +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) /*[clinic input] os.listmounts @@ -4888,6 +5332,9 @@ os_listmounts_impl(PyObject *module, path_t *volume) return result; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + +#ifdef MS_WINDOWS /*[clinic input] os._path_isdevdrive @@ -5042,7 +5489,7 @@ os__getfullpathname_impl(PyObject *module, path_t *path) return PyErr_NoMemory(); } - PyObject *str = PyUnicode_FromWideChar(abspath, wcslen(abspath)); + PyObject *str = PyUnicode_FromWideChar(abspath, -1); PyMem_RawFree(abspath); if (str == NULL) { return NULL; @@ -5158,7 +5605,7 @@ os__findfirstfile_impl(PyObject *module, path_t *path) } wRealFileName = wFileData.cFileName; - result = PyUnicode_FromWideChar(wRealFileName, wcslen(wRealFileName)); + result = PyUnicode_FromWideChar(wRealFileName, -1); FindClose(hFindFile); return result; } @@ -5202,7 +5649,7 @@ os__getvolumepathname_impl(PyObject *module, path_t *path) result = win32_error_object("_getvolumepathname", path->object); goto exit; } - result = PyUnicode_FromWideChar(mountpath, wcslen(mountpath)); + result = PyUnicode_FromWideChar(mountpath, -1); if (PyBytes_Check(path->object)) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); @@ -5215,14 +5662,15 @@ os__getvolumepathname_impl(PyObject *module, path_t *path) /*[clinic input] os._path_splitroot - path: path_t + path: path_t, + / Removes everything after the root on Win32. [clinic start generated code]*/ static PyObject * os__path_splitroot_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=ab7f1a88b654581c input=dc93b1d3984cffb6]*/ +/*[clinic end generated code: output=ab7f1a88b654581c input=42831e41f8458f6d]*/ { wchar_t *buffer; wchar_t *end; @@ -5524,7 +5972,8 @@ os__path_lexists_impl(PyObject *module, path_t *path) /*[clinic input] os._path_isdir -> bool - s as path: path_t(allow_fd=True, suppress_value_error=True) + path: path_t(allow_fd=True, suppress_value_error=True), + / Return true if the pathname refers to an existing directory. @@ -5532,7 +5981,7 @@ Return true if the pathname refers to an existing directory. static int os__path_isdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=d5786196f9e2fa7a input=132a3b5301aecf79]*/ +/*[clinic end generated code: output=d5786196f9e2fa7a input=0d3fd790564d244b]*/ { return _testFileType(path, PY_IFDIR); } @@ -5601,7 +6050,8 @@ os__path_isjunction_impl(PyObject *module, path_t *path) /*[clinic input] os._path_splitroot_ex - p as path: path_t(make_wide=True, nonstrict=True) + path: path_t(make_wide=True, nonstrict=True), + / Split a pathname into drive, root and tail. @@ -5610,7 +6060,7 @@ The tail contains anything after the root. static PyObject * os__path_splitroot_ex_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=4b0072b6cdf4b611 input=4556b615c7cc13f2]*/ +/*[clinic end generated code: output=4b0072b6cdf4b611 input=4ac47b394d68bd21]*/ { Py_ssize_t drvsize, rootsize; PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; @@ -5726,6 +6176,9 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS + // For API sets that don't support these APIs, we have no choice + // but to silently create a directory with default ACL. +#if defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) if (mode == 0700 /* 0o700 */) { ULONG sdSize; pSecAttr = &secAttr; @@ -5741,6 +6194,7 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) error = GetLastError(); } } +#endif if (!error) { result = CreateDirectoryW(path->wide, pSecAttr); if (secAttr.lpSecurityDescriptor && @@ -5957,6 +6411,7 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is /*[clinic input] +@permit_long_docstring_body os.rename src : path_t @@ -5977,13 +6432,14 @@ src_dir_fd and dst_dir_fd, may not be implemented on your platform. static PyObject * os_rename_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=59e803072cf41230 input=faa61c847912c850]*/ +/*[clinic end generated code: output=59e803072cf41230 input=11aae8c091162766]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 0); } /*[clinic input] +@permit_long_docstring_body os.replace = os.rename Rename a file or directory, overwriting the destination. @@ -5998,7 +6454,7 @@ src_dir_fd and dst_dir_fd, may not be implemented on your platform. static PyObject * os_replace_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=1968c02e7857422b input=c003f0def43378ef]*/ +/*[clinic end generated code: output=1968c02e7857422b input=78d6c8087e90994c]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 1); } @@ -6097,14 +6553,14 @@ os_system_impl(PyObject *module, const wchar_t *command) /*[clinic input] os.system -> long - command: FSConverter + command: unicode_fs_encoded Execute the command in a subshell. [clinic start generated code]*/ static long os_system_impl(PyObject *module, PyObject *command) -/*[clinic end generated code: output=290fc437dd4f33a0 input=86a58554ba6094af]*/ +/*[clinic end generated code: output=290fc437dd4f33a0 input=47c6f24b6dc92881]*/ { long result; const char *bytes = PyBytes_AsString(command); @@ -6640,7 +7096,7 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, if (!PyTuple_CheckExact(times) || (PyTuple_Size(times) != 2)) { PyErr_SetString(PyExc_TypeError, "utime: 'times' must be either" - " a tuple of two ints or None"); + " a tuple of two numbers or None"); return NULL; } utime.now = 0; @@ -7975,15 +8431,76 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This should only be called from the parent process after +// This MUST only be called from the parent process after // PyOS_AfterFork_Parent(). -static void -warn_about_fork_with_threads(const char* name) +static int +warn_about_fork_with_threads( + const char* name, // Name of the API to use in the warning message. + const Py_ssize_t num_os_threads // Only trusted when >= 1. +) { // It's not safe to issue the warning while the world is stopped, because // other threads might be holding locks that we need, which would deadlock. assert(!_PyRuntime.stoptheworld.world_stopped); + Py_ssize_t num_python_threads = num_os_threads; + if (num_python_threads <= 0) { + // Fall back to just the number our threading module knows about. + // An incomplete view of the world, but better than nothing. + PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); + if (!threading) { + PyErr_Clear(); + return 0; + } + PyObject *threading_active = + PyObject_GetAttr(threading, &_Py_ID(_active)); + if (!threading_active) { + PyErr_Clear(); + Py_DECREF(threading); + return 0; + } + PyObject *threading_limbo = + PyObject_GetAttr(threading, &_Py_ID(_limbo)); + if (!threading_limbo) { + PyErr_Clear(); + Py_DECREF(threading); + Py_DECREF(threading_active); + return 0; + } + Py_DECREF(threading); + // Duplicating what threading.active_count() does but without holding + // threading._active_limbo_lock so our count could be inaccurate if + // these dicts are mid-update from another thread. Not a big deal. + // Worst case if someone replaced threading._active or threading._limbo + // with non-dicts, we get -1 from *Length() below and undercount. + // Nobody should, but we're best effort so we clear errors and move on. + num_python_threads = (PyMapping_Length(threading_active) + + PyMapping_Length(threading_limbo)); + PyErr_Clear(); + Py_DECREF(threading_active); + Py_DECREF(threading_limbo); + } + if (num_python_threads > 1) { + return PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, +#ifdef HAVE_GETPID + "This process (pid=%d) is multi-threaded, " +#else + "This process is multi-threaded, " +#endif + "use of %s() may lead to deadlocks in the child.", +#ifdef HAVE_GETPID + getpid(), +#endif + name); + } + return 0; +} + +// If this returns <= 0, we were unable to successfully use any OS APIs. +// Returns a positive number of threads otherwise. +static Py_ssize_t get_number_of_os_threads(void) +{ // TODO: Consider making an `os` module API to return the current number // of threads in the process. That'd presumably use this platform code but // raise an error rather than using the inaccurate fallback. @@ -8022,57 +8539,7 @@ warn_about_fork_with_threads(const char* name) } } #endif - if (num_python_threads <= 0) { - // Fall back to just the number our threading module knows about. - // An incomplete view of the world, but better than nothing. - PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); - if (!threading) { - PyErr_Clear(); - return; - } - PyObject *threading_active = - PyObject_GetAttr(threading, &_Py_ID(_active)); - if (!threading_active) { - PyErr_Clear(); - Py_DECREF(threading); - return; - } - PyObject *threading_limbo = - PyObject_GetAttr(threading, &_Py_ID(_limbo)); - if (!threading_limbo) { - PyErr_Clear(); - Py_DECREF(threading); - Py_DECREF(threading_active); - return; - } - Py_DECREF(threading); - // Duplicating what threading.active_count() does but without holding - // threading._active_limbo_lock so our count could be inaccurate if - // these dicts are mid-update from another thread. Not a big deal. - // Worst case if someone replaced threading._active or threading._limbo - // with non-dicts, we get -1 from *Length() below and undercount. - // Nobody should, but we're best effort so we clear errors and move on. - num_python_threads = (PyMapping_Length(threading_active) - + PyMapping_Length(threading_limbo)); - PyErr_Clear(); - Py_DECREF(threading_active); - Py_DECREF(threading_limbo); - } - if (num_python_threads > 1) { - PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, -#ifdef HAVE_GETPID - "This process (pid=%d) is multi-threaded, " -#else - "This process is multi-threaded, " -#endif - "use of %s() may lead to deadlocks in the child.", -#ifdef HAVE_GETPID - getpid(), -#endif - name); - PyErr_Clear(); - } + return num_python_threads; } #endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK @@ -8108,10 +8575,14 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("fork1"); + if (warn_about_fork_with_threads("fork1", num_os_threads) < 0) { + return NULL; + } } if (pid == -1) { errno = saved_errno; @@ -8157,10 +8628,13 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("fork"); + if (warn_about_fork_with_threads("fork", num_os_threads) < 0) + return NULL; } if (pid == -1) { errno = saved_errno; @@ -8185,10 +8659,10 @@ static PyObject * os_sched_get_priority_max_impl(PyObject *module, int policy) /*[clinic end generated code: output=9e465c6e43130521 input=2097b7998eca6874]*/ { - int max; - - max = sched_get_priority_max(policy); - if (max < 0) + /* make sure that errno is cleared before the call */ + errno = 0; + int max = sched_get_priority_max(policy); + if (max == -1 && errno) return posix_error(); return PyLong_FromLong(max); } @@ -8206,8 +8680,10 @@ static PyObject * os_sched_get_priority_min_impl(PyObject *module, int policy) /*[clinic end generated code: output=7595c1138cc47a6d input=21bc8fa0d70983bf]*/ { + /* make sure that errno is cleared before the call */ + errno = 0; int min = sched_get_priority_min(policy); - if (min < 0) + if (min == -1 && errno) return posix_error(); return PyLong_FromLong(min); } @@ -8268,7 +8744,7 @@ os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority) static PyObject * os_sched_param_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) { - return Py_BuildValue("(O(N))", Py_TYPE(self), PyStructSequence_GetItem(self, 0)); + return Py_BuildValue("(O(O))", Py_TYPE(self), PyStructSequence_GetItem(self, 0)); } static PyMethodDef os_sched_param_reduce_method = { @@ -8283,7 +8759,7 @@ static PyStructSequence_Field sched_param_fields[] = { }; static PyStructSequence_Desc sched_param_desc = { - "sched_param", /* name */ + MODNAME ".sched_param", /* name */ os_sched_param__doc__, /* doc */ sched_param_fields, 1 @@ -8414,6 +8890,7 @@ os_sched_setparam_impl(PyObject *module, pid_t pid, PyObject *param_obj) #ifdef HAVE_SCHED_RR_GET_INTERVAL /*[clinic input] +@permit_long_summary os.sched_rr_get_interval -> double pid: pid_t / @@ -8425,7 +8902,7 @@ Value returned is a float. static double os_sched_rr_get_interval_impl(PyObject *module, pid_t pid) -/*[clinic end generated code: output=7e2d935833ab47dc input=2a973da15cca6fae]*/ +/*[clinic end generated code: output=7e2d935833ab47dc input=cab0b83586776b10]*/ { struct timespec interval; if (sched_rr_get_interval(pid, &interval)) { @@ -8565,6 +9042,7 @@ os_sched_setaffinity_impl(PyObject *module, pid_t pid, PyObject *mask) /*[clinic input] +@permit_long_summary os.sched_getaffinity pid: pid_t / @@ -8576,7 +9054,7 @@ The affinity is returned as a set of CPU identifiers. static PyObject * os_sched_getaffinity_impl(PyObject *module, pid_t pid) -/*[clinic end generated code: output=f726f2c193c17a4f input=983ce7cb4a565980]*/ +/*[clinic end generated code: output=f726f2c193c17a4f input=cb79ff13579ef091]*/ { int ncpus = NCPUS_START; size_t setsize; @@ -8806,14 +9284,14 @@ os_ptsname_impl(PyObject *module, int fd) #if defined(HAVE_OPENPTY) || defined(HAVE_FORKPTY) || defined(HAVE_LOGIN_TTY) || defined(HAVE_DEV_PTMX) #ifdef HAVE_PTY_H #include <pty.h> -#ifdef HAVE_UTMP_H -#include <utmp.h> -#endif /* HAVE_UTMP_H */ #elif defined(HAVE_LIBUTIL_H) #include <libutil.h> #elif defined(HAVE_UTIL_H) #include <util.h> #endif /* HAVE_PTY_H */ +#ifdef HAVE_UTMP_H +#include <utmp.h> +#endif /* HAVE_UTMP_H */ #ifdef HAVE_STROPTS_H #include <stropts.h> #endif @@ -8984,11 +9462,12 @@ Returns a tuple of (pid, master_fd). Like fork(), return pid of 0 to the child process, and pid of child to the parent process. To both, return fd of newly opened pseudo-terminal. +The master_fd is non-inheritable. [clinic start generated code]*/ static PyObject * os_forkpty_impl(PyObject *module) -/*[clinic end generated code: output=60d0a5c7512e4087 input=f1f7f4bae3966010]*/ +/*[clinic end generated code: output=60d0a5c7512e4087 input=24765e0f33275b3b]*/ { int master_fd = -1; pid_t pid; @@ -9012,14 +9491,24 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + /* set O_CLOEXEC on master_fd */ + if (_Py_set_inheritable(master_fd, 0, NULL) < 0) { + PyErr_FormatUnraisable("Exception ignored when setting master_fd " + "non-inheritable in forkpty()"); + } + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("forkpty"); + if (warn_about_fork_with_threads("forkpty", num_os_threads) < 0) + return NULL; } if (pid == -1) { return posix_error(); } + return Py_BuildValue("(Ni)", PyLong_FromPid(pid), master_fd); } #endif /* HAVE_FORKPTY */ @@ -9282,7 +9771,7 @@ os_getgroups_impl(PyObject *module) /*[clinic input] os.initgroups - username as oname: FSConverter + username as oname: unicode_fs_encoded gid: int / @@ -9295,12 +9784,12 @@ group id. static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, int gid) -/*[clinic end generated code: output=7f074d30a425fd3a input=df3d54331b0af204]*/ +/*[clinic end generated code: output=7f074d30a425fd3a input=984e60c7fed88cb4]*/ #else /*[clinic input] os.initgroups - username as oname: FSConverter + username as oname: unicode_fs_encoded gid: gid_t / @@ -9313,7 +9802,7 @@ group id. static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, gid_t gid) -/*[clinic end generated code: output=59341244521a9e3f input=0cb91bdc59a4c564]*/ +/*[clinic end generated code: output=59341244521a9e3f input=17d8fbe2dea42ca4]*/ #endif { const char *username = PyBytes_AS_STRING(oname); @@ -9548,6 +10037,24 @@ os_getlogin_impl(PyObject *module) } else result = PyErr_SetFromWindowsErr(GetLastError()); +#elif defined (HAVE_GETLOGIN_R) +# if defined (HAVE_MAXLOGNAME) + char name[MAXLOGNAME + 1]; +# elif defined (HAVE_UT_NAMESIZE) + char name[UT_NAMESIZE + 1]; +# else + char name[256]; +# endif + int err = getlogin_r(name, sizeof(name)); + if (err) { + int old_errno = errno; + errno = err; + posix_error(); + errno = old_errno; + } + else { + result = PyUnicode_DecodeFSDefault(name); + } #else char *name; int old_errno = errno; @@ -10648,7 +11155,7 @@ and elapsed.\n\ See os.times for more information."); static PyStructSequence_Desc times_result_desc = { - "times_result", /* name */ + MODNAME ".times_result", /* name */ times_result__doc__, /* doc */ times_result_fields, 5 @@ -10916,6 +11423,7 @@ os_timerfd_settime_ns_impl(PyObject *module, int fd, int flags, } /*[clinic input] +@permit_long_summary os.timerfd_gettime fd: fildes @@ -10927,7 +11435,7 @@ Return a tuple of a timer file descriptor's (interval, next expiration) in float static PyObject * os_timerfd_gettime_impl(PyObject *module, int fd) -/*[clinic end generated code: output=ec5a94a66cfe6ab4 input=8148e3430870da1c]*/ +/*[clinic end generated code: output=ec5a94a66cfe6ab4 input=05f7d568a4820dc6]*/ { struct itimerspec curr_value; int result; @@ -10942,6 +11450,7 @@ os_timerfd_gettime_impl(PyObject *module, int fd) /*[clinic input] +@permit_long_summary os.timerfd_gettime_ns fd: fildes @@ -10953,7 +11462,7 @@ Return a tuple of a timer file descriptor's (interval, next expiration) in nanos static PyObject * os_timerfd_gettime_ns_impl(PyObject *module, int fd) -/*[clinic end generated code: output=580633a4465f39fe input=a825443e4c6b40ac]*/ +/*[clinic end generated code: output=580633a4465f39fe input=d0de95b9782179c5]*/ { struct itimerspec curr_value; int result; @@ -11378,6 +11887,7 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) /*[clinic input] +@permit_long_docstring_body os.lseek -> Py_off_t fd: int @@ -11398,7 +11908,7 @@ The return value is the number of bytes relative to the beginning of the file. static Py_off_t os_lseek_impl(PyObject *module, int fd, Py_off_t position, int how) -/*[clinic end generated code: output=971e1efb6b30bd2f input=f096e754c5367504]*/ +/*[clinic end generated code: output=971e1efb6b30bd2f input=4a3de549f07e1c40]*/ { Py_off_t result; @@ -11440,9 +11950,6 @@ static PyObject * os_read_impl(PyObject *module, int fd, Py_ssize_t length) /*[clinic end generated code: output=dafbe9a5cddb987b input=1df2eaa27c0bf1d3]*/ { - Py_ssize_t n; - PyObject *buffer; - if (length < 0) { errno = EINVAL; return posix_error(); @@ -11450,23 +11957,22 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) length = Py_MIN(length, _PY_READ_MAX); - buffer = PyBytes_FromStringAndSize((char *)NULL, length); - if (buffer == NULL) - return NULL; - - n = _Py_read(fd, PyBytes_AS_STRING(buffer), length); - if (n == -1) { - Py_DECREF(buffer); + PyBytesWriter *writer = PyBytesWriter_Create(length); + if (writer == NULL) { return NULL; } - if (n != length) - _PyBytes_Resize(&buffer, n); + Py_ssize_t n = _Py_read(fd, PyBytesWriter_GetData(writer), length); + if (n == -1) { + PyBytesWriter_Discard(writer); + return NULL; + } - return buffer; + return PyBytesWriter_FinishWithSize(writer, n); } /*[clinic input] +@permit_long_docstring_body os.readinto -> Py_ssize_t fd: int buffer: Py_buffer(accept={rwbuffer}) @@ -11487,7 +11993,7 @@ negative. static Py_ssize_t os_readinto_impl(PyObject *module, int fd, Py_buffer *buffer) -/*[clinic end generated code: output=8091a3513c683a80 input=d40074d0a68de575]*/ +/*[clinic end generated code: output=8091a3513c683a80 input=a770382bd3d32f9a]*/ { assert(buffer->len >= 0); Py_ssize_t result = _Py_read(fd, buffer->buf, buffer->len); @@ -11620,6 +12126,7 @@ os_readv_impl(PyObject *module, int fd, PyObject *buffers) #ifdef HAVE_PREAD /*[clinic input] +@permit_long_summary os.pread fd: int @@ -11635,24 +12142,24 @@ the beginning of the file. The file offset remains unchanged. static PyObject * os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset) -/*[clinic end generated code: output=3f875c1eef82e32f input=85cb4a5589627144]*/ +/*[clinic end generated code: output=3f875c1eef82e32f input=5943beb009d3da04]*/ { Py_ssize_t n; int async_err = 0; - PyObject *buffer; if (length < 0) { errno = EINVAL; return posix_error(); } - buffer = PyBytes_FromStringAndSize((char *)NULL, length); - if (buffer == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(length); + if (writer == NULL) { return NULL; + } do { Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - n = pread(fd, PyBytes_AS_STRING(buffer), length, offset); + n = pread(fd, PyBytesWriter_GetData(writer), length, offset); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); @@ -11661,17 +12168,16 @@ os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset) if (!async_err) { posix_error(); } - Py_DECREF(buffer); + PyBytesWriter_Discard(writer); return NULL; } - if (n != length) - _PyBytes_Resize(&buffer, n); - return buffer; + return PyBytesWriter_FinishWithSize(writer, n); } #endif /* HAVE_PREAD */ #if defined(HAVE_PREADV) || defined (HAVE_PREADV2) /*[clinic input] +@permit_long_docstring_body os.preadv -> Py_ssize_t fd: int @@ -11693,6 +12199,7 @@ The flags argument contains a bitwise OR of zero or more of the following flags: - RWF_HIPRI - RWF_NOWAIT +- RWF_DONTCACHE Using non-zero flags requires Linux 4.6 or newer. [clinic start generated code]*/ @@ -11700,7 +12207,7 @@ Using non-zero flags requires Linux 4.6 or newer. static Py_ssize_t os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=26fc9c6e58e7ada5 input=4173919dc1f7ed99]*/ +/*[clinic end generated code: output=26fc9c6e58e7ada5 input=34fb3b9ca06f7ba7]*/ { Py_ssize_t cnt, n; int async_err = 0; @@ -11819,7 +12326,7 @@ os.sendfile out_fd: int in_fd: int offset: Py_off_t - count: Py_ssize_t + count: Py_ssize_t(allow_negative=False) headers: object(c_default="NULL") = () trailers: object(c_default="NULL") = () flags: int = 0 @@ -11831,7 +12338,7 @@ static PyObject * os_sendfile_impl(PyObject *module, int out_fd, int in_fd, Py_off_t offset, Py_ssize_t count, PyObject *headers, PyObject *trailers, int flags) -/*[clinic end generated code: output=329ea009bdd55afc input=338adb8ff84ae8cd]*/ +/*[clinic end generated code: output=329ea009bdd55afc input=dcb026b94effa922]*/ #else /*[clinic input] os.sendfile @@ -11839,7 +12346,7 @@ os.sendfile out_fd: int in_fd: int offset as offobj: object - count: Py_ssize_t + count: Py_ssize_t(allow_negative=False) Copy count bytes from file descriptor in_fd to file descriptor out_fd. [clinic start generated code]*/ @@ -11847,12 +12354,22 @@ Copy count bytes from file descriptor in_fd to file descriptor out_fd. static PyObject * os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, Py_ssize_t count) -/*[clinic end generated code: output=ae81216e40f167d8 input=76d64058c74477ba]*/ +/*[clinic end generated code: output=ae81216e40f167d8 input=424df0949059ea5b]*/ #endif { Py_ssize_t ret; int async_err = 0; +#ifdef __APPLE__ + if(sbytes < 0) { + PyErr_SetString(PyExc_ValueError, + "count cannot be negative"); + return NULL; + } +#else + assert(count >= 0); +#endif + #if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) #ifndef __APPLE__ off_t sbytes; @@ -12316,6 +12833,8 @@ os_pwrite_impl(PyObject *module, int fd, Py_buffer *buffer, Py_off_t offset) #if defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2) /*[clinic input] +@permit_long_summary +@permit_long_docstring_body os.pwritev -> Py_ssize_t fd: int @@ -12338,6 +12857,7 @@ The flags argument contains a bitwise OR of zero or more of the following flags: - RWF_DSYNC - RWF_SYNC - RWF_APPEND +- RWF_DONTCACHE Using non-zero flags requires Linux 4.7 or newer. [clinic start generated code]*/ @@ -12345,7 +12865,7 @@ Using non-zero flags requires Linux 4.7 or newer. static Py_ssize_t os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=35358c327e1a2a8e]*/ +/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=664a67626d485665]*/ { Py_ssize_t cnt; Py_ssize_t result; @@ -12426,7 +12946,7 @@ os.copy_file_range Source file descriptor. dst: int Destination file descriptor. - count: Py_ssize_t + count: Py_ssize_t(allow_negative=False) Number of bytes to copy. offset_src: object = None Starting offset in src. @@ -12442,7 +12962,7 @@ respectively for offset_dst. static PyObject * os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, PyObject *offset_src, PyObject *offset_dst) -/*[clinic end generated code: output=1a91713a1d99fc7a input=42fdce72681b25a9]*/ +/*[clinic end generated code: output=1a91713a1d99fc7a input=08dacb760869b87c]*/ { off_t offset_src_val, offset_dst_val; off_t *p_offset_src = NULL; @@ -12454,11 +12974,6 @@ os_copy_file_range_impl(PyObject *module, int src, int dst, Py_ssize_t count, int flags = 0; - if (count < 0) { - PyErr_SetString(PyExc_ValueError, "negative value for 'count' not allowed"); - return NULL; - } - if (offset_src != Py_None) { if (!Py_off_t_converter(offset_src, &offset_src_val)) { return NULL; @@ -12495,7 +13010,7 @@ os.splice Source file descriptor. dst: int Destination file descriptor. - count: Py_ssize_t + count: Py_ssize_t(allow_negative=False) Number of bytes to copy. offset_src: object = None Starting offset in src. @@ -12515,7 +13030,7 @@ static PyObject * os_splice_impl(PyObject *module, int src, int dst, Py_ssize_t count, PyObject *offset_src, PyObject *offset_dst, unsigned int flags) -/*[clinic end generated code: output=d0386f25a8519dc5 input=047527c66c6d2e0a]*/ +/*[clinic end generated code: output=d0386f25a8519dc5 input=034852a7b2e7af35]*/ { off_t offset_src_val, offset_dst_val; off_t *p_offset_src = NULL; @@ -12523,10 +13038,6 @@ os_splice_impl(PyObject *module, int src, int dst, Py_ssize_t count, Py_ssize_t ret; int async_err = 0; - if (count < 0) { - PyErr_SetString(PyExc_ValueError, "negative value for 'count' not allowed"); - return NULL; - } if (offset_src != Py_None) { if (!Py_off_t_converter(offset_src, &offset_src_val)) { @@ -12618,6 +13129,7 @@ os_mkfifo_impl(PyObject *module, path_t *path, int mode, int dir_fd) #if defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV) /*[clinic input] +@permit_long_docstring_body os.mknod path: path_t @@ -12644,7 +13156,7 @@ dir_fd may not be implemented on your platform. static PyObject * os_mknod_impl(PyObject *module, path_t *path, int mode, dev_t device, int dir_fd) -/*[clinic end generated code: output=92e55d3ca8917461 input=ee44531551a4d83b]*/ +/*[clinic end generated code: output=92e55d3ca8917461 input=7121c4723d22545b]*/ { int result; int async_err = 0; @@ -13046,8 +13558,8 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) /*[clinic input] os.putenv - name: FSConverter - value: FSConverter + name: unicode_fs_encoded + value: unicode_fs_encoded / Change or add an environment variable. @@ -13055,7 +13567,7 @@ Change or add an environment variable. static PyObject * os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) -/*[clinic end generated code: output=d29a567d6b2327d2 input=a97bc6152f688d31]*/ +/*[clinic end generated code: output=d29a567d6b2327d2 input=84fcd30f873c8c45]*/ { const char *name_string = PyBytes_AS_STRING(name); const char *value_string = PyBytes_AS_STRING(value); @@ -13098,7 +13610,7 @@ os_unsetenv_impl(PyObject *module, PyObject *name) #else /*[clinic input] os.unsetenv - name: FSConverter + name: unicode_fs_encoded / Delete an environment variable. @@ -13106,7 +13618,7 @@ Delete an environment variable. static PyObject * os_unsetenv_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=54c4137ab1834f02 input=2bb5288a599c7107]*/ +/*[clinic end generated code: output=54c4137ab1834f02 input=78ff12e505ade80a]*/ { if (PySys_Audit("os.unsetenv", "(O)", name) < 0) { return NULL; @@ -13125,6 +13637,25 @@ os_unsetenv_impl(PyObject *module, PyObject *name) #endif /* !MS_WINDOWS */ +#ifdef HAVE_CLEARENV +/*[clinic input] +os._clearenv +[clinic start generated code]*/ + +static PyObject * +os__clearenv_impl(PyObject *module) +/*[clinic end generated code: output=2d6705d62c014b51 input=47d2fa7f323c43ca]*/ +{ + errno = 0; + int err = clearenv(); + if (err) { + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + + /*[clinic input] os.strerror @@ -13235,6 +13766,7 @@ os_WIFSIGNALED_impl(PyObject *module, int status) #ifdef WIFEXITED /*[clinic input] +@permit_long_summary os.WIFEXITED -> bool status: int @@ -13244,7 +13776,7 @@ Return True if the process returning status exited via the exit() system call. static int os_WIFEXITED_impl(PyObject *module, int status) -/*[clinic end generated code: output=01c09d6ebfeea397 input=d63775a6791586c0]*/ +/*[clinic end generated code: output=01c09d6ebfeea397 input=8c24a82148709b30]*/ { WAIT_TYPE wait_status; WAIT_STATUS_INT(wait_status) = status; @@ -13275,6 +13807,7 @@ os_WEXITSTATUS_impl(PyObject *module, int status) #ifdef WTERMSIG /*[clinic input] +@permit_long_summary os.WTERMSIG -> int status: int @@ -13284,7 +13817,7 @@ Return the signal that terminated the process that provided the status value. static int os_WTERMSIG_impl(PyObject *module, int status) -/*[clinic end generated code: output=172f7dfc8dcfc3ad input=727fd7f84ec3f243]*/ +/*[clinic end generated code: output=172f7dfc8dcfc3ad input=89072f6cbf3f8050]*/ { WAIT_TYPE wait_status; WAIT_STATUS_INT(wait_status) = status; @@ -14826,6 +15359,7 @@ os_getresuid_impl(PyObject *module) #ifdef HAVE_GETRESGID /*[clinic input] +@permit_long_summary os.getresgid Return a tuple of the current process's real, effective, and saved group ids. @@ -14833,7 +15367,7 @@ Return a tuple of the current process's real, effective, and saved group ids. static PyObject * os_getresgid_impl(PyObject *module) -/*[clinic end generated code: output=2719c4bfcf27fb9f input=517e68db9ca32df6]*/ +/*[clinic end generated code: output=2719c4bfcf27fb9f input=ad9adadc86fbdb17]*/ { gid_t rgid, egid, sgid; if (getresgid(&rgid, &egid, &sgid) < 0) @@ -14868,9 +15402,6 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, int follow_symlinks) /*[clinic end generated code: output=5f2f44200a43cff2 input=025789491708f7eb]*/ { - Py_ssize_t i; - PyObject *buffer = NULL; - if (fd_and_follow_symlinks_invalid("getxattr", path->fd, follow_symlinks)) return NULL; @@ -14878,8 +15409,7 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, return NULL; } - for (i = 0; ; i++) { - void *ptr; + for (Py_ssize_t i = 0; ; i++) { ssize_t result; static const Py_ssize_t buffer_sizes[] = {128, XATTR_SIZE_MAX, 0}; Py_ssize_t buffer_size = buffer_sizes[i]; @@ -14887,10 +15417,11 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, path_error(path); return NULL; } - buffer = PyBytes_FromStringAndSize(NULL, buffer_size); - if (!buffer) + PyBytesWriter *writer = PyBytesWriter_Create(buffer_size); + if (writer == NULL) { return NULL; - ptr = PyBytes_AS_STRING(buffer); + } + void *ptr = PyBytesWriter_GetData(writer); Py_BEGIN_ALLOW_THREADS; if (path->fd >= 0) @@ -14902,27 +15433,21 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, Py_END_ALLOW_THREADS; if (result < 0) { + PyBytesWriter_Discard(writer); if (errno == ERANGE) { - Py_DECREF(buffer); continue; } path_error(path); - Py_DECREF(buffer); return NULL; } - if (result != buffer_size) { - /* Can only shrink. */ - _PyBytes_Resize(&buffer, result); - } - break; + return PyBytesWriter_FinishWithSize(writer, result); } - - return buffer; } /*[clinic input] +@permit_long_docstring_body os.setxattr path: path_t(allow_fd=True) @@ -14944,7 +15469,7 @@ If follow_symlinks is False, and the last element of the path is a symbolic static PyObject * os_setxattr_impl(PyObject *module, path_t *path, path_t *attribute, Py_buffer *value, int flags, int follow_symlinks) -/*[clinic end generated code: output=98b83f63fdde26bb input=c17c0103009042f0]*/ +/*[clinic end generated code: output=98b83f63fdde26bb input=4098e6f68699f3d7]*/ { ssize_t result; @@ -15026,6 +15551,7 @@ os_removexattr_impl(PyObject *module, path_t *path, path_t *attribute, /*[clinic input] +@permit_long_docstring_body os.listxattr path: path_t(allow_fd=True, nullable=True) = None @@ -15043,7 +15569,7 @@ If follow_symlinks is False, and the last element of the path is a symbolic static PyObject * os_listxattr_impl(PyObject *module, path_t *path, int follow_symlinks) -/*[clinic end generated code: output=bebdb4e2ad0ce435 input=9826edf9fdb90869]*/ +/*[clinic end generated code: output=bebdb4e2ad0ce435 input=48aa9ac8be47dea1]*/ { Py_ssize_t i; PyObject *result = NULL; @@ -15130,9 +15656,10 @@ os_listxattr_impl(PyObject *module, path_t *path, int follow_symlinks) /*[clinic input] +@permit_long_summary os.urandom - size: Py_ssize_t + size: Py_ssize_t(allow_negative=False) / Return a bytes object containing random bytes suitable for cryptographic use. @@ -15140,38 +15667,38 @@ Return a bytes object containing random bytes suitable for cryptographic use. static PyObject * os_urandom_impl(PyObject *module, Py_ssize_t size) -/*[clinic end generated code: output=42c5cca9d18068e9 input=4067cdb1b6776c29]*/ +/*[clinic end generated code: output=42c5cca9d18068e9 input=58a0def87dbc2c22]*/ { - PyObject *bytes; - int result; - - if (size < 0) + if (size < 0) { return PyErr_Format(PyExc_ValueError, "negative argument not allowed"); - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) - return NULL; + } - result = _PyOS_URandom(PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes)); - if (result == -1) { - Py_DECREF(bytes); + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; } - return bytes; + + int result = _PyOS_URandom(PyBytesWriter_GetData(writer), size); + if (result == -1) { + PyBytesWriter_Discard(writer); + return NULL; + } + return PyBytesWriter_Finish(writer); } #ifdef HAVE_MEMFD_CREATE /*[clinic input] os.memfd_create - name: FSConverter + name: unicode_fs_encoded flags: unsigned_int(bitwise=True, c_default="MFD_CLOEXEC") = MFD_CLOEXEC [clinic start generated code]*/ static PyObject * os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) -/*[clinic end generated code: output=6681ede983bdb9a6 input=a42cfc199bcd56e9]*/ +/*[clinic end generated code: output=6681ede983bdb9a6 input=cd0eb092cfac474b]*/ { int fd; const char *bytes = PyBytes_AS_STRING(name); @@ -15973,11 +16500,14 @@ static PyType_Slot DirEntryType_slots[] = { }; static PyType_Spec DirEntryType_spec = { - MODNAME ".DirEntry", - sizeof(DirEntry), - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - DirEntryType_slots + .name = MODNAME ".DirEntry", + .basicsize = sizeof(DirEntry), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = DirEntryType_slots }; @@ -16415,14 +16945,17 @@ static PyType_Slot ScandirIteratorType_slots[] = { }; static PyType_Spec ScandirIteratorType_spec = { - MODNAME ".ScandirIterator", - sizeof(ScandirIterator), - 0, + .name = MODNAME ".ScandirIterator", + .basicsize = sizeof(ScandirIterator), // bpo-40549: Py_TPFLAGS_BASETYPE should not be used, since // PyType_GetModule(Py_TYPE(self)) doesn't work on a subclass instance. - (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_FINALIZE - | Py_TPFLAGS_DISALLOW_INSTANTIATION), - ScandirIteratorType_slots + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_HAVE_FINALIZE + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + ), + .slots = ScandirIteratorType_slots }; /*[clinic input] @@ -16588,6 +17121,7 @@ PyOS_FSPath(PyObject *path) } /*[clinic input] +@permit_long_docstring_body os.fspath path: object @@ -16601,7 +17135,7 @@ types raise a TypeError. static PyObject * os_fspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=c3c3b78ecff2914f input=e357165f7b22490f]*/ +/*[clinic end generated code: output=c3c3b78ecff2914f input=f608743e60a3211e]*/ { return PyOS_FSPath(path); } @@ -16620,25 +17154,20 @@ static PyObject * os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) /*[clinic end generated code: output=b3a618196a61409c input=59bafac39c594947]*/ { - PyObject *bytes; - Py_ssize_t n; - if (size < 0) { errno = EINVAL; return posix_error(); } - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) { - PyErr_NoMemory(); + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; } + void *data = PyBytesWriter_GetData(writer); + Py_ssize_t n; while (1) { - n = syscall(SYS_getrandom, - PyBytes_AS_STRING(bytes), - PyBytes_GET_SIZE(bytes), - flags); + n = syscall(SYS_getrandom, data, size, flags); if (n < 0 && errno == EINTR) { if (PyErr_CheckSignals() < 0) { goto error; @@ -16655,14 +17184,10 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) goto error; } - if (n != size) { - _PyBytes_Resize(&bytes, n); - } - - return bytes; + return PyBytesWriter_FinishWithSize(writer, n); error: - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); return NULL; } #endif /* HAVE_GETRANDOM_SYSCALL */ @@ -16865,24 +17390,28 @@ static PyObject * os__supports_virtual_terminal_impl(PyObject *module) /*[clinic end generated code: output=bd0556a6d9d99fe6 input=0752c98e5d321542]*/ { +#ifdef HAVE_WINDOWS_CONSOLE_IO DWORD mode = 0; HANDLE handle = GetStdHandle(STD_ERROR_HANDLE); if (!GetConsoleMode(handle, &mode)) { Py_RETURN_FALSE; } return PyBool_FromLong(mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING); +#else + Py_RETURN_FALSE; +#endif /* HAVE_WINDOWS_CONSOLE_IO */ } #endif /*[clinic input] os._inputhook -Calls PyOS_CallInputHook droppong the GIL first +Calls PyOS_InputHook dropping the GIL first [clinic start generated code]*/ static PyObject * os__inputhook_impl(PyObject *module) -/*[clinic end generated code: output=525aca4ef3c6149f input=fc531701930d064f]*/ +/*[clinic end generated code: output=525aca4ef3c6149f input=b5018fa1ec3aa440]*/ { int result = 0; if (PyOS_InputHook) { @@ -16896,12 +17425,12 @@ os__inputhook_impl(PyObject *module) /*[clinic input] os._is_inputhook_installed -Checks if PyOS_CallInputHook is set +Checks if PyOS_InputHook is set [clinic start generated code]*/ static PyObject * os__is_inputhook_installed_impl(PyObject *module) -/*[clinic end generated code: output=3b3eab4f672c689a input=ff177c9938dd76d8]*/ +/*[clinic end generated code: output=3b3eab4f672c689a input=757820f79f48820c]*/ { return PyBool_FromLong(PyOS_InputHook != NULL); } @@ -16934,11 +17463,31 @@ os__emscripten_debugger_impl(PyObject *module) emscripten_debugger(); Py_RETURN_NONE; } + +EM_JS(void, emscripten_log_impl_js, (const char* arg), { + console.warn(UTF8ToString(arg)); +}); + +/*[clinic input] +os._emscripten_log + arg: str + +Log something to the JS console. Emscripten only. +[clinic start generated code]*/ + +static PyObject * +os__emscripten_log_impl(PyObject *module, const char *arg) +/*[clinic end generated code: output=9749e5e293c42784 input=350aa1f70bc1e905]*/ +{ + emscripten_log_impl_js(arg); + Py_RETURN_NONE; +} #endif /* __EMSCRIPTEN__ */ static PyMethodDef posix_methods[] = { OS_STAT_METHODDEF + OS_STATX_METHODDEF OS_ACCESS_METHODDEF OS_TTYNAME_METHODDEF OS_CHDIR_METHODDEF @@ -17074,6 +17623,7 @@ static PyMethodDef posix_methods[] = { OS_POSIX_FADVISE_METHODDEF OS_PUTENV_METHODDEF OS_UNSETENV_METHODDEF + OS__CLEARENV_METHODDEF OS_STRERROR_METHODDEF OS_FCHDIR_METHODDEF OS_FSYNC_METHODDEF @@ -17153,6 +17703,7 @@ static PyMethodDef posix_methods[] = { OS__IS_INPUTHOOK_INSTALLED_METHODDEF OS__CREATE_ENVIRON_METHODDEF OS__EMSCRIPTEN_DEBUGGER_METHODDEF + OS__EMSCRIPTEN_LOG_METHODDEF {NULL, NULL} /* Sentinel */ }; @@ -17561,6 +18112,9 @@ all_ins(PyObject *m) #ifdef RWF_NOWAIT if (PyModule_AddIntConstant(m, "RWF_NOWAIT", RWF_NOWAIT)) return -1; #endif +#ifdef RWF_DONTCACHE + if (PyModule_AddIntConstant(m, "RWF_DONTCACHE", RWF_DONTCACHE)) return -1; +#endif #ifdef RWF_APPEND if (PyModule_AddIntConstant(m, "RWF_APPEND", RWF_APPEND)) return -1; #endif @@ -17776,6 +18330,53 @@ all_ins(PyObject *m) #endif #endif /* HAVE_EVENTFD && EFD_CLOEXEC */ +#ifdef NODEV + if (PyModule_Add(m, "NODEV", _PyLong_FromDev(NODEV))) return -1; +#endif + +#ifdef AT_NO_AUTOMOUNT + if (PyModule_AddIntMacro(m, AT_NO_AUTOMOUNT)) return -1; +#endif + +#ifdef HAVE_STATX + if (PyModule_AddIntMacro(m, STATX_TYPE)) return -1; + if (PyModule_AddIntMacro(m, STATX_MODE)) return -1; + if (PyModule_AddIntMacro(m, STATX_NLINK)) return -1; + if (PyModule_AddIntMacro(m, STATX_UID)) return -1; + if (PyModule_AddIntMacro(m, STATX_GID)) return -1; + if (PyModule_AddIntMacro(m, STATX_ATIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_MTIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_CTIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_INO)) return -1; + if (PyModule_AddIntMacro(m, STATX_SIZE)) return -1; + if (PyModule_AddIntMacro(m, STATX_BLOCKS)) return -1; + if (PyModule_AddIntMacro(m, STATX_BASIC_STATS)) return -1; + if (PyModule_AddIntMacro(m, STATX_BTIME)) return -1; +#ifdef STATX_MNT_ID + if (PyModule_AddIntMacro(m, STATX_MNT_ID)) return -1; +#endif +#ifdef STATX_DIOALIGN + if (PyModule_AddIntMacro(m, STATX_DIOALIGN)) return -1; +#endif +#ifdef STATX_MNT_ID_UNIQUE + if (PyModule_AddIntMacro(m, STATX_MNT_ID_UNIQUE)) return -1; +#endif +#ifdef STATX_SUBVOL + if (PyModule_AddIntMacro(m, STATX_SUBVOL)) return -1; +#endif +#ifdef STATX_WRITE_ATOMIC + if (PyModule_AddIntMacro(m, STATX_WRITE_ATOMIC)) return -1; +#endif +#ifdef STATX_DIO_READ_ALIGN + if (PyModule_AddIntMacro(m, STATX_DIO_READ_ALIGN)) return -1; +#endif + /* STATX_ALL intentionally omitted because it is deprecated */ + if (PyModule_AddIntMacro(m, AT_STATX_SYNC_AS_STAT)) return -1; + if (PyModule_AddIntMacro(m, AT_STATX_FORCE_SYNC)) return -1; + if (PyModule_AddIntMacro(m, AT_STATX_DONT_SYNC)) return -1; + /* STATX_ATTR_* constants are in the stat module */ +#endif /* HAVE_STATX */ + #if defined(__APPLE__) if (PyModule_AddIntConstant(m, "_COPYFILE_DATA", COPYFILE_DATA)) return -1; if (PyModule_AddIntConstant(m, "_COPYFILE_STAT", COPYFILE_STAT)) return -1; @@ -18047,6 +18648,24 @@ posixmodule_exec(PyObject *m) } #endif +#ifdef HAVE_STATX + if (statx == NULL) { + PyObject* dct = PyModule_GetDict(m); + if (dct == NULL) { + return -1; + } + if (PyDict_PopString(dct, "statx", NULL) < 0) { + return -1; + } + } + else { + state->StatxResultType = PyType_FromModuleAndSpec(m, &pystatx_result_spec, NULL); + if (PyModule_AddObjectRef(m, "statx_result", state->StatxResultType) < 0) { + return -1; + } + } +#endif + /* Initialize environ dictionary */ if (PyModule_Add(m, "environ", convertenviron()) != 0) { return -1; @@ -18063,14 +18682,12 @@ posixmodule_exec(PyObject *m) } #if defined(HAVE_WAITID) - waitid_result_desc.name = MODNAME ".waitid_result"; state->WaitidResultType = (PyObject *)PyStructSequence_NewType(&waitid_result_desc); if (PyModule_AddObjectRef(m, "waitid_result", state->WaitidResultType) < 0) { return -1; } #endif - stat_result_desc.name = "os.stat_result"; /* see issue #19209 */ stat_result_desc.fields[7].name = PyStructSequence_UnnamedField; stat_result_desc.fields[8].name = PyStructSequence_UnnamedField; stat_result_desc.fields[9].name = PyStructSequence_UnnamedField; @@ -18081,14 +18698,12 @@ posixmodule_exec(PyObject *m) state->statresult_new_orig = ((PyTypeObject *)state->StatResultType)->tp_new; ((PyTypeObject *)state->StatResultType)->tp_new = statresult_new; - statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */ state->StatVFSResultType = (PyObject *)PyStructSequence_NewType(&statvfs_result_desc); if (PyModule_AddObjectRef(m, "statvfs_result", state->StatVFSResultType) < 0) { return -1; } #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) - sched_param_desc.name = MODNAME ".sched_param"; state->SchedParamType = (PyObject *)PyStructSequence_NewType(&sched_param_desc); if (PyModule_AddObjectRef(m, "sched_param", state->SchedParamType) < 0) { return -1; @@ -18120,7 +18735,6 @@ posixmodule_exec(PyObject *m) return -1; } - times_result_desc.name = MODNAME ".times_result"; state->TimesResultType = (PyObject *)PyStructSequence_NewType(&times_result_desc); if (PyModule_AddObjectRef(m, "times_result", state->TimesResultType) < 0) { return -1; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index 2240e2078b2..a18737b24c2 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -1,12 +1,6 @@ /* UNIX password file access module */ -// Need limited C API version 3.13 for PyMem_RawRealloc() -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 -#endif - #include "Python.h" #include "posixmodule.h" @@ -69,6 +63,11 @@ get_pwd_state(PyObject *module) static struct PyModuleDef pwdmodule; +/* Mutex to protect calls to getpwuid(), getpwnam(), and getpwent(). + * These functions return pointer to static data structure, which + * may be overwritten by any subsequent calls. */ +static PyMutex pwd_db_mutex = {0}; + #define DEFAULT_BUFFER_SIZE 1024 static PyObject * @@ -182,9 +181,15 @@ pwd_getpwuid(PyObject *module, PyObject *uidobj) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&pwd_db_mutex); + // The getpwuid() function is not required to be thread-safe. + // https://pubs.opengroup.org/onlinepubs/009604499/functions/getpwuid.html p = getpwuid(uid); #endif if (p == NULL) { +#ifndef HAVE_GETPWUID_R + PyMutex_Unlock(&pwd_db_mutex); +#endif PyMem_RawFree(buf); if (nomem == 1) { return PyErr_NoMemory(); @@ -200,6 +205,8 @@ pwd_getpwuid(PyObject *module, PyObject *uidobj) retval = mkpwent(module, p); #ifdef HAVE_GETPWUID_R PyMem_RawFree(buf); +#else + PyMutex_Unlock(&pwd_db_mutex); #endif return retval; } @@ -265,9 +272,15 @@ pwd_getpwnam_impl(PyObject *module, PyObject *name) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&pwd_db_mutex); + // The getpwnam() function is not required to be thread-safe. + // https://pubs.opengroup.org/onlinepubs/009604599/functions/getpwnam.html p = getpwnam(name_chars); #endif if (p == NULL) { +#ifndef HAVE_GETPWNAM_R + PyMutex_Unlock(&pwd_db_mutex); +#endif if (nomem == 1) { PyErr_NoMemory(); } @@ -278,6 +291,9 @@ pwd_getpwnam_impl(PyObject *module, PyObject *name) goto out; } retval = mkpwent(module, p); +#ifndef HAVE_GETPWNAM_R + PyMutex_Unlock(&pwd_db_mutex); +#endif out: PyMem_RawFree(buf); Py_DECREF(bytes); @@ -286,6 +302,7 @@ pwd_getpwnam_impl(PyObject *module, PyObject *name) #ifdef HAVE_GETPWENT /*[clinic input] +@permit_long_summary pwd.getpwall Return a list of all available password database entries, in arbitrary order. @@ -295,24 +312,37 @@ See help(pwd) for more on password database entries. static PyObject * pwd_getpwall_impl(PyObject *module) -/*[clinic end generated code: output=4853d2f5a0afac8a input=d7ecebfd90219b85]*/ +/*[clinic end generated code: output=4853d2f5a0afac8a input=f8145e0d9a79e32c]*/ { PyObject *d; struct passwd *p; if ((d = PyList_New(0)) == NULL) return NULL; + + PyMutex_Lock(&pwd_db_mutex); + int failure = 0; + PyObject *v = NULL; + // The setpwent(), getpwent() and endpwent() functions are not required to + // be thread-safe. + // https://pubs.opengroup.org/onlinepubs/009696799/functions/setpwent.html setpwent(); while ((p = getpwent()) != NULL) { - PyObject *v = mkpwent(module, p); + v = mkpwent(module, p); if (v == NULL || PyList_Append(d, v) != 0) { - Py_XDECREF(v); - Py_DECREF(d); - endpwent(); - return NULL; + /* NOTE: cannot dec-ref here, while holding the mutex. */ + failure = 1; + goto done; } Py_DECREF(v); } + +done: endpwent(); + PyMutex_Unlock(&pwd_db_mutex); + if (failure) { + Py_XDECREF(v); + Py_CLEAR(d); + } return d; } #endif diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index fa153d86543..e9255038eee 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -9,6 +9,8 @@ #include <stdbool.h> #include <stddef.h> // offsetof() + +#include "expat_config.h" #include "expat.h" #include "pyexpat.h" @@ -74,6 +76,15 @@ typedef struct { PyObject_HEAD XML_Parser itself; + /* + * Strong reference to a parent `xmlparseobject` if this parser + * is a child parser. Set to NULL if this parser is a root parser. + * This is needed to keep the parent parser alive as long as it has + * at least one child parser. + * + * See https://github.com/python/cpython/issues/139400 for details. + */ + PyObject *parent; int ordered_attributes; /* Return attributes as a list. */ int specified_attributes; /* Report only specified attributes. */ int in_callback; /* Is a callback active? */ @@ -98,7 +109,11 @@ typedef struct { #define CHARACTER_DATA_BUFFER_SIZE 8192 -typedef const void *xmlhandler; +// A generic function type for storage. +// To avoid undefined behaviors, a handler must be cast to the correct +// function type before it's called; see SETTER_WRAPPER below. +typedef void (*xmlhandler)(void); + typedef void (*xmlhandlersetter)(XML_Parser self, xmlhandler handler); struct HandlerInfo { @@ -110,9 +125,7 @@ struct HandlerInfo { static struct HandlerInfo handler_info[64]; -// gh-111178: Use _Py_NO_SANITIZE_UNDEFINED, rather than using the exact -// handler API for each handler. -static inline void _Py_NO_SANITIZE_UNDEFINED +static inline void CALL_XML_HANDLER_SETTER(const struct HandlerInfo *handler_info, XML_Parser xml_parser, xmlhandler xml_handler) { @@ -120,50 +133,91 @@ CALL_XML_HANDLER_SETTER(const struct HandlerInfo *handler_info, setter(xml_parser, xml_handler); } +static int +set_xml_error_attr_code(PyObject *err, enum XML_Error code) +{ + PyObject *v = PyLong_FromLong((long)code); + int ok = v != NULL && PyObject_SetAttr(err, &_Py_ID(code), v) != -1; + Py_XDECREF(v); + return ok; +} + /* Set an integer attribute on the error object; return true on success, * false on an exception. */ static int -set_error_attr(PyObject *err, const char *name, int value) +set_xml_error_attr_location(PyObject *err, const char *name, XML_Size value) { - PyObject *v = PyLong_FromLong(value); - - if (v == NULL || PyObject_SetAttrString(err, name, v) == -1) { - Py_XDECREF(v); - return 0; - } - Py_DECREF(v); - return 1; + PyObject *v = PyLong_FromSize_t((size_t)value); + int ok = v != NULL && PyObject_SetAttrString(err, name, v) != -1; + Py_XDECREF(v); + return ok; } + +static PyObject * +set_xml_error(pyexpat_state *state, + enum XML_Error code, XML_Size lineno, XML_Size column, + const char *errmsg) +{ + PyObject *arg; + if (errmsg == NULL) { + arg = PyUnicode_FromFormat( + "%s: line %zu, column %zu", + XML_ErrorString(code), + (size_t)lineno, (size_t)column + ); + } + else { + arg = PyUnicode_FromStringAndSize(errmsg, strlen(errmsg)); + } + if (arg == NULL) { + return NULL; + } + PyObject *res = PyObject_CallOneArg(state->error, arg); + Py_DECREF(arg); + if ( + res != NULL + && set_xml_error_attr_code(res, code) + && set_xml_error_attr_location(res, "lineno", lineno) + && set_xml_error_attr_location(res, "offset", column) + ) { + PyErr_SetObject(state->error, res); + } + Py_XDECREF(res); + return NULL; +} + +#define SET_XML_ERROR(STATE, SELF, CODE, ERRMSG) \ + do { \ + XML_Parser parser = SELF->itself; \ + assert(parser != NULL); \ + XML_Size lineno = XML_GetCurrentLineNumber(parser); \ + XML_Size column = XML_GetCurrentColumnNumber(parser); \ + (void)set_xml_error(state, CODE, lineno, column, ERRMSG); \ + } while (0) + /* Build and set an Expat exception, including positioning * information. Always returns NULL. */ static PyObject * set_error(pyexpat_state *state, xmlparseobject *self, enum XML_Error code) { - PyObject *err; - PyObject *buffer; - XML_Parser parser = self->itself; - int lineno = XML_GetErrorLineNumber(parser); - int column = XML_GetErrorColumnNumber(parser); - - buffer = PyUnicode_FromFormat("%s: line %i, column %i", - XML_ErrorString(code), lineno, column); - if (buffer == NULL) - return NULL; - err = PyObject_CallOneArg(state->error, buffer); - Py_DECREF(buffer); - if ( err != NULL - && set_error_attr(err, "code", code) - && set_error_attr(err, "offset", column) - && set_error_attr(err, "lineno", lineno)) { - PyErr_SetObject(state->error, err); - } - Py_XDECREF(err); + SET_XML_ERROR(state, self, code, NULL); return NULL; } +#if XML_COMBINED_VERSION >= 20400 +static PyObject * +set_invalid_arg(pyexpat_state *state, xmlparseobject *self, const char *errmsg) +{ + SET_XML_ERROR(state, self, XML_ERROR_INVALID_ARGUMENT, errmsg); + return NULL; +} +#endif + +#undef SET_XML_ERROR + static int have_handler(xmlparseobject *self, int type) { @@ -588,7 +642,7 @@ my_ElementDeclHandler(void *userData, PyObject *modelobj, *nameobj; if (PyErr_Occurred()) - return; + goto finally; if (flush_character_buffer(self) < 0) goto finally; @@ -765,6 +819,7 @@ pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, } /*[clinic input] +@permit_long_summary pyexpat.xmlparser.GetReparseDeferralEnabled Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0. @@ -772,7 +827,7 @@ Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0 static PyObject * pyexpat_xmlparser_GetReparseDeferralEnabled_impl(xmlparseobject *self) -/*[clinic end generated code: output=4e91312e88a595a8 input=54b5f11d32b20f3e]*/ +/*[clinic end generated code: output=4e91312e88a595a8 input=ae02d7152ab9e2d0]*/ { return PyBool_FromLong(self->reparse_deferral_enabled); } @@ -966,6 +1021,7 @@ pyexpat_xmlparser_GetBase_impl(xmlparseobject *self) } /*[clinic input] +@permit_long_docstring_body pyexpat.xmlparser.GetInputContext Return the untranslated text of the input that caused the current event. @@ -976,7 +1032,7 @@ for an element with many attributes), not all of the text may be available. static PyObject * pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self) -/*[clinic end generated code: output=a88026d683fc22cc input=034df8712db68379]*/ +/*[clinic end generated code: output=a88026d683fc22cc input=925cea010fdfa682]*/ { if (self->in_callback) { int offset, size; @@ -994,6 +1050,7 @@ pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self) } /*[clinic input] +@permit_long_summary pyexpat.xmlparser.ExternalEntityParserCreate cls: defining_class @@ -1009,7 +1066,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, PyTypeObject *cls, const char *context, const char *encoding) -/*[clinic end generated code: output=01d4472b49cb3f92 input=ec70c6b9e6e9619a]*/ +/*[clinic end generated code: output=01d4472b49cb3f92 input=550479eaff952cc0]*/ { xmlparseobject *new_parser; pyexpat_state *state = PyType_GetModuleState(cls); @@ -1019,6 +1076,11 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, return NULL; } + // The new subparser will make use of the parent XML_Parser inside of Expat. + // So we need to take subparsers into account with the reference counting + // of their parent parser. + Py_INCREF(self); + new_parser->buffer_size = self->buffer_size; new_parser->buffer_used = 0; new_parser->buffer = NULL; @@ -1028,6 +1090,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->ns_prefixes = self->ns_prefixes; new_parser->itself = XML_ExternalEntityParserCreate(self->itself, context, encoding); + new_parser->parent = (PyObject *)self; new_parser->handlers = 0; new_parser->intern = Py_XNewRef(self->intern); @@ -1035,11 +1098,13 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->buffer = PyMem_Malloc(new_parser->buffer_size); if (new_parser->buffer == NULL) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } } if (!new_parser->itself) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } @@ -1053,6 +1118,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->handlers = PyMem_New(PyObject *, i); if (!new_parser->handlers) { Py_DECREF(new_parser); + Py_DECREF(self); return PyErr_NoMemory(); } clear_handlers(new_parser, 1); @@ -1072,6 +1138,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, } /*[clinic input] +@permit_long_summary pyexpat.xmlparser.SetParamEntityParsing flag: int @@ -1087,7 +1154,7 @@ was successful. static PyObject * pyexpat_xmlparser_SetParamEntityParsing_impl(xmlparseobject *self, int flag) -/*[clinic end generated code: output=18668ee8e760d64c input=8aea19b4b15e9af1]*/ +/*[clinic end generated code: output=18668ee8e760d64c input=1c43532fcb405879]*/ { flag = XML_SetParamEntityParsing(self->itself, flag); return PyLong_FromLong(flag); @@ -1096,6 +1163,8 @@ pyexpat_xmlparser_SetParamEntityParsing_impl(xmlparseobject *self, int flag) #if XML_COMBINED_VERSION >= 19505 /*[clinic input] +@permit_long_summary +@permit_long_docstring_body pyexpat.xmlparser.UseForeignDTD cls: defining_class @@ -1112,7 +1181,7 @@ information to the parser. 'flag' defaults to True if not provided. static PyObject * pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, int flag) -/*[clinic end generated code: output=d7d98252bd25a20f input=23440ecb0573fb29]*/ +/*[clinic end generated code: output=d7d98252bd25a20f input=c2264845d8c0029c]*/ { pyexpat_state *state = PyType_GetModuleState(cls); enum XML_Error rc; @@ -1125,6 +1194,192 @@ pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, } #endif +#if XML_COMBINED_VERSION >= 20400 +static PyObject * +set_activation_threshold(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold, + XML_Bool (*setter)(XML_Parser, unsigned long long)) +{ + assert(self->itself != NULL); + if (setter(self->itself, threshold) == XML_TRUE) { + Py_RETURN_NONE; + } + // The setter fails if self->itself is NULL (which is not possible here) + // or is a non-root parser, which currently only happens for parsers + // created by ExternalEntityParserCreate(). + pyexpat_state *state = PyType_GetModuleState(cls); + return set_invalid_arg(state, self, "parser must be a root parser"); +} + +static PyObject * +set_maximum_amplification(xmlparseobject *self, + PyTypeObject *cls, + float max_factor, + XML_Bool (*setter)(XML_Parser, float)) +{ + assert(self->itself != NULL); + if (setter(self->itself, max_factor) == XML_TRUE) { + Py_RETURN_NONE; + } + // The setter fails if self->itself is NULL (which is not possible here), + // is a non-root parser, which currently only happens for parsers created + // by ExternalEntityParserCreate(), or if 'max_factor' is NaN or < 1.0. + pyexpat_state *state = PyType_GetModuleState(cls); + // Note: Expat has no API to determine whether a parser is a root parser, + // and since the Expat functions for defining the various maximum allowed + // amplifcation factors fail when a bad parser or an out-of-range factor + // is given without specifying which check failed, we check whether the + // factor is out-of-range to improve the error message. See also gh-90949. + const char *message = (isnan(max_factor) || max_factor < 1.0f) + ? "'max_factor' must be at least 1.0" + : "parser must be a root parser"; + return set_invalid_arg(state, self, message); +} +#endif + +#if XML_COMBINED_VERSION >= 20400 +/*[clinic input] +@permit_long_summary +@permit_long_docstring_body +pyexpat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold + + cls: defining_class + threshold: unsigned_long_long + / + +Sets the number of output bytes needed to activate protection against billion laughs attacks. + +The number of output bytes includes amplification from entity expansion +and reading DTD files. + +Parser objects usually have a protection activation threshold of 8 MiB, +but the actual default value depends on the underlying Expat library. + +Activation thresholds below 4 MiB are known to break support for DITA 1.3 +payload and are hence not recommended. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold) +/*[clinic end generated code: output=0c082342f1c78114 input=fa2f91f26b62a42a]*/ +{ + return set_activation_threshold( + self, cls, threshold, + XML_SetBillionLaughsAttackProtectionActivationThreshold + ); +} +#endif + +#if XML_COMBINED_VERSION >= 20400 +/*[clinic input] +@permit_long_summary +@permit_long_docstring_body +pyexpat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification + + cls: defining_class + max_factor: float + / + +Sets the maximum tolerated amplification factor for protection against billion laughs attacks. + +The amplification factor is calculated as "(direct + indirect) / direct" +while parsing, where "direct" is the number of bytes read from the primary +document in parsing and "indirect" is the number of bytes added by expanding +entities and reading external DTD files, combined. + +The 'max_factor' value must be a non-NaN floating point value greater than +or equal to 1.0. Amplification factors greater than 30,000 can be observed +in the middle of parsing even with benign files in practice. In particular, +the activation threshold should be carefully chosen to avoid false positives. + +Parser objects usually have a maximum amplification factor of 100, +but the actual default value depends on the underlying Expat library. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor) +/*[clinic end generated code: output=c590439eadf463fa input=cc1e97c1fd2bd950]*/ +{ + return set_maximum_amplification( + self, cls, max_factor, + XML_SetBillionLaughsAttackProtectionMaximumAmplification + ); +} +#endif + +#if XML_COMBINED_VERSION >= 20702 +/*[clinic input] +@permit_long_summary +@permit_long_docstring_body +pyexpat.xmlparser.SetAllocTrackerActivationThreshold + + cls: defining_class + threshold: unsigned_long_long + / + +Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM. + +Parser objects usually have an allocation activation threshold of 64 MiB, +but the actual default value depends on the underlying Expat library. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold) +/*[clinic end generated code: output=bed7e93207ba08c5 input=b7a7a3e3d054286a]*/ +{ + return set_activation_threshold( + self, cls, threshold, + XML_SetAllocTrackerActivationThreshold + ); +} +#endif + +#if XML_COMBINED_VERSION >= 20702 +/*[clinic input] +@permit_long_summary +@permit_long_docstring_body +pyexpat.xmlparser.SetAllocTrackerMaximumAmplification + + cls: defining_class + max_factor: float + / + +Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated. + +The amplification factor is calculated as "allocated / direct" while parsing, +where "direct" is the number of bytes read from the primary document in parsing +and "allocated" is the number of bytes of dynamic memory allocated in the parser +hierarchy. + +The 'max_factor' value must be a non-NaN floating point value greater than +or equal to 1.0. Amplification factors greater than 100.0 can be observed +near the start of parsing even with benign files in practice. In particular, +the activation threshold should be carefully chosen to avoid false positives. + +Parser objects usually have a maximum amplification factor of 100, +but the actual default value depends on the underlying Expat library. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor) +/*[clinic end generated code: output=6e44bd48c9b112a0 input=c6af7ccb76ae5c6b]*/ +{ + return set_maximum_amplification( + self, cls, max_factor, + XML_SetAllocTrackerMaximumAmplification + ); +} +#endif + static struct PyMethodDef xmlparse_methods[] = { PYEXPAT_XMLPARSER_PARSE_METHODDEF PYEXPAT_XMLPARSER_PARSEFILE_METHODDEF @@ -1133,9 +1388,11 @@ static struct PyMethodDef xmlparse_methods[] = { PYEXPAT_XMLPARSER_GETINPUTCONTEXT_METHODDEF PYEXPAT_XMLPARSER_EXTERNALENTITYPARSERCREATE_METHODDEF PYEXPAT_XMLPARSER_SETPARAMENTITYPARSING_METHODDEF -#if XML_COMBINED_VERSION >= 19505 PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF -#endif + PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF + PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF + PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF + PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF {NULL, NULL} /* sentinel */ @@ -1242,6 +1499,7 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, /* namespace_separator is either NULL or contains one char + \0 */ self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler, namespace_separator); + self->parent = NULL; if (self->itself == NULL) { PyErr_SetString(PyExc_RuntimeError, "XML_ParserCreate failed"); @@ -1278,6 +1536,7 @@ xmlparse_traverse(PyObject *op, visitproc visit, void *arg) for (size_t i = 0; handler_info[i].name != NULL; i++) { Py_VISIT(self->handlers[i]); } + Py_VISIT(self->parent); Py_VISIT(Py_TYPE(op)); return 0; } @@ -1288,6 +1547,10 @@ xmlparse_clear(PyObject *op) xmlparseobject *self = xmlparseobject_CAST(op); clear_handlers(self, 0); Py_CLEAR(self->intern); + // NOTE: We cannot call Py_CLEAR(self->parent) prior to calling + // XML_ParserFree(self->itself), or a subparser could lose its parent + // XML_Parser while still making use of it internally. + // https://github.com/python/cpython/issues/139400 return 0; } @@ -1301,6 +1564,7 @@ xmlparse_dealloc(PyObject *op) XML_ParserFree(self->itself); } self->itself = NULL; + Py_CLEAR(self->parent); if (self->handlers != NULL) { PyMem_Free(self->handlers); @@ -1365,7 +1629,7 @@ xmlparse_handler_setter(PyObject *op, PyObject *v, void *closure) elaborate system of handlers and state could remove the C handler more effectively. */ if (handlernum == CharacterData && self->in_callback) { - c_handler = noop_character_data_handler; + c_handler = (xmlhandler)noop_character_data_handler; } v = NULL; } @@ -2141,6 +2405,20 @@ pyexpat_exec(PyObject *mod) #else capi->SetReparseDeferralEnabled = NULL; #endif +#if XML_COMBINED_VERSION >= 20702 + capi->SetAllocTrackerActivationThreshold = XML_SetAllocTrackerActivationThreshold; + capi->SetAllocTrackerMaximumAmplification = XML_SetAllocTrackerMaximumAmplification; +#else + capi->SetAllocTrackerActivationThreshold = NULL; + capi->SetAllocTrackerMaximumAmplification = NULL; +#endif +#if XML_COMBINED_VERSION >= 20400 + capi->SetBillionLaughsAttackProtectionActivationThreshold = XML_SetBillionLaughsAttackProtectionActivationThreshold; + capi->SetBillionLaughsAttackProtectionMaximumAmplification = XML_SetBillionLaughsAttackProtectionMaximumAmplification; +#else + capi->SetAllocTrackerActivationThreshold = NULL; + capi->SetAllocTrackerMaximumAmplification = NULL; +#endif /* export using capsule */ PyObject *capi_object = PyCapsule_New(capi, PyExpat_CAPSULE_NAME, @@ -2222,13 +2500,84 @@ clear_handlers(xmlparseobject *self, int initial) } } +/* To avoid undefined behaviors, a function must be *called* via a function + * pointer of the correct type. + * So, for each `XML_Set*` function, we define a wrapper that calls `XML_Set*` + * with its argument cast to the appropriate type. + */ + +typedef void (*parser_only)(void *); +typedef int (*not_standalone)(void *); +typedef void (*parser_and_data)(void *, const XML_Char *); +typedef void (*parser_and_data_and_int)(void *, const XML_Char *, int); +typedef void (*parser_and_data_and_data)( + void *, const XML_Char *, const XML_Char *); +typedef void (*start_element)(void *, const XML_Char *, const XML_Char **); +typedef void (*element_decl)(void *, const XML_Char *, XML_Content *); +typedef void (*xml_decl)( + void *, const XML_Char *, const XML_Char *, int); +typedef void (*start_doctype_decl)( + void *, const XML_Char *, const XML_Char *, const XML_Char *, int); +typedef void (*notation_decl)( + void *, + const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *); +typedef void (*attlist_decl)( + void *, + const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *, + int); +typedef void (*unparsed_entity_decl)( + void *, + const XML_Char *, const XML_Char *, + const XML_Char *, const XML_Char *, const XML_Char *); +typedef void (*entity_decl)( + void *, + const XML_Char *, int, + const XML_Char *, int, + const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *); +typedef int (*external_entity_ref)( + XML_Parser, + const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *); + +#define SETTER_WRAPPER(NAME, TYPE) \ + static inline void \ + pyexpat_Set ## NAME (XML_Parser parser, xmlhandler handler) \ + { \ + (void)XML_Set ## NAME (parser, (TYPE)handler); \ + } + +SETTER_WRAPPER(StartElementHandler, start_element) +SETTER_WRAPPER(EndElementHandler, parser_and_data) +SETTER_WRAPPER(ProcessingInstructionHandler, parser_and_data_and_data) +SETTER_WRAPPER(CharacterDataHandler, parser_and_data_and_int) +SETTER_WRAPPER(UnparsedEntityDeclHandler, unparsed_entity_decl) +SETTER_WRAPPER(NotationDeclHandler, notation_decl) +SETTER_WRAPPER(StartNamespaceDeclHandler, parser_and_data_and_data) +SETTER_WRAPPER(EndNamespaceDeclHandler, parser_and_data) +SETTER_WRAPPER(CommentHandler, parser_and_data) +SETTER_WRAPPER(StartCdataSectionHandler, parser_only) +SETTER_WRAPPER(EndCdataSectionHandler, parser_only) +SETTER_WRAPPER(DefaultHandler, parser_and_data_and_int) +SETTER_WRAPPER(DefaultHandlerExpand, parser_and_data_and_int) +SETTER_WRAPPER(NotStandaloneHandler, not_standalone) +SETTER_WRAPPER(ExternalEntityRefHandler, external_entity_ref) +SETTER_WRAPPER(StartDoctypeDeclHandler, start_doctype_decl) +SETTER_WRAPPER(EndDoctypeDeclHandler, parser_only) +SETTER_WRAPPER(EntityDeclHandler, entity_decl) +SETTER_WRAPPER(XmlDeclHandler, xml_decl) +SETTER_WRAPPER(ElementDeclHandler, element_decl) +SETTER_WRAPPER(AttlistDeclHandler, attlist_decl) +#if XML_COMBINED_VERSION >= 19504 +SETTER_WRAPPER(SkippedEntityHandler, parser_and_data_and_int) +#endif +#undef SETTER_WRAPPER + static struct HandlerInfo handler_info[] = { // The cast to `xmlhandlersetter` is needed as the signature of XML // handler functions is not compatible with `xmlhandlersetter` since // their second parameter is narrower than a `const void *`. #define HANDLER_INFO(name) \ - {#name, (xmlhandlersetter)XML_Set##name, my_##name}, + {#name, (xmlhandlersetter)pyexpat_Set##name, (xmlhandler)my_##name}, HANDLER_INFO(StartElementHandler) HANDLER_INFO(EndElementHandler) diff --git a/Modules/readline.c b/Modules/readline.c index 0dd99dc66c0..cc84eb6229e 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -255,6 +255,7 @@ readline_read_init_file_impl(PyObject *module, PyObject *filename_obj) if (!PyUnicode_FSConverter(filename_obj, &filename_bytes)) return NULL; if (PySys_Audit("open", "OCi", filename_obj, 'r', 0) < 0) { + Py_DECREF(filename_bytes); return NULL; } errno = rl_read_init_file(PyBytes_AS_STRING(filename_bytes)); @@ -298,6 +299,7 @@ readline_read_history_file_impl(PyObject *module, PyObject *filename_obj) if (!PyUnicode_FSConverter(filename_obj, &filename_bytes)) return NULL; if (PySys_Audit("open", "OCi", filename_obj, 'r', 0) < 0) { + Py_DECREF(filename_bytes); return NULL; } errno = read_history(PyBytes_AS_STRING(filename_bytes)); @@ -343,6 +345,7 @@ readline_write_history_file_impl(PyObject *module, PyObject *filename_obj) return NULL; filename = PyBytes_AS_STRING(filename_bytes); if (PySys_Audit("open", "OCi", filename_obj, 'w', 0) < 0) { + Py_DECREF(filename_bytes); return NULL; } } else { @@ -400,6 +403,7 @@ readline_append_history_file_impl(PyObject *module, int nelements, return NULL; filename = PyBytes_AS_STRING(filename_bytes); if (PySys_Audit("open", "OCi", filename_obj, 'a', 0) < 0) { + Py_DECREF(filename_bytes); return NULL; } } else { @@ -568,6 +572,26 @@ readline_set_pre_input_hook_impl(PyObject *module, PyObject *function) return set_hook("pre_input_hook", &state->pre_input_hook, function); } + +/* Get pre-input hook */ + +/*[clinic input] +readline.get_pre_input_hook + +Get the current pre-input hook function. +[clinic start generated code]*/ + +static PyObject * +readline_get_pre_input_hook_impl(PyObject *module) +/*[clinic end generated code: output=ad56b77a8e8981ca input=fb1e1b1fbd94e4e5]*/ +{ + readlinestate *state = get_readline_state(module); + if (state->pre_input_hook == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(state->pre_input_hook); +} + #endif @@ -1020,6 +1044,7 @@ readline_insert_text_impl(PyObject *module, PyObject *string) /* Redisplay the line buffer */ /*[clinic input] +@permit_long_summary @critical_section readline.redisplay @@ -1028,7 +1053,7 @@ Change what's displayed on the screen to reflect contents of the line buffer. static PyObject * readline_redisplay_impl(PyObject *module) -/*[clinic end generated code: output=a8b9725827c3c34b input=5895fd014615ff58]*/ +/*[clinic end generated code: output=a8b9725827c3c34b input=fb6ce76959c6f0ec]*/ { rl_redisplay(); Py_RETURN_NONE; @@ -1069,6 +1094,7 @@ static struct PyMethodDef readline_methods[] = READLINE_SET_STARTUP_HOOK_METHODDEF #ifdef HAVE_RL_PRE_INPUT_HOOK READLINE_SET_PRE_INPUT_HOOK_METHODDEF + READLINE_GET_PRE_INPUT_HOOK_METHODDEF #endif #ifdef HAVE_RL_COMPLETION_APPEND_CHARACTER READLINE_CLEAR_HISTORY_METHODDEF diff --git a/Modules/resource.c b/Modules/resource.c index 3fe18e7c98e..a463355f424 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,7 +1,5 @@ -// Need limited C API version 3.13 for PySys_Audit() -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" @@ -150,6 +148,42 @@ resource_getrusage_impl(PyObject *module, int who) } #endif +static int +py2rlim(PyObject *obj, rlim_t *out) +{ + obj = PyNumber_Index(obj); + if (obj == NULL) { + return -1; + } + int neg = PyLong_IsNegative(obj); + assert(neg >= 0); + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, out, sizeof(*out), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + Py_DECREF(obj); + if (bytes < 0) { + return -1; + } + else if (neg && *out == RLIM_INFINITY && bytes <= (Py_ssize_t)sizeof(*out)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Use RLIM_INFINITY instead of negative limit value.", 1)) + { + return -1; + } + } + else if (neg) { + PyErr_SetString(PyExc_ValueError, + "Cannot convert negative int"); + return -1; + } + else if (bytes > (Py_ssize_t)sizeof(*out)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C rlim_t"); + return -1; + } + return 0; +} + static int py2rlimit(PyObject *limits, struct rlimit *rl_out) { @@ -166,26 +200,13 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) } curobj = PyTuple_GetItem(limits, 0); // borrowed maxobj = PyTuple_GetItem(limits, 1); // borrowed -#if !defined(HAVE_LARGEFILE_SUPPORT) - rl_out->rlim_cur = PyLong_AsLong(curobj); - if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) + if (py2rlim(curobj, &rl_out->rlim_cur) < 0 || + py2rlim(maxobj, &rl_out->rlim_max) < 0) + { goto error; - rl_out->rlim_max = PyLong_AsLong(maxobj); - if (rl_out->rlim_max == (rlim_t)-1 && PyErr_Occurred()) - goto error; -#else - /* The limits are probably bigger than a long */ - rl_out->rlim_cur = PyLong_AsLongLong(curobj); - if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) - goto error; - rl_out->rlim_max = PyLong_AsLongLong(maxobj); - if (rl_out->rlim_max == (rlim_t)-1 && PyErr_Occurred()) - goto error; -#endif + } Py_DECREF(limits); - rl_out->rlim_cur = rl_out->rlim_cur & RLIM_INFINITY; - rl_out->rlim_max = rl_out->rlim_max & RLIM_INFINITY; return 0; error: @@ -193,15 +214,21 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) return -1; } +static PyObject* +rlim2py(rlim_t value) +{ + return PyLong_FromUnsignedNativeBytes(&value, sizeof(value), -1); +} + static PyObject* rlimit2py(struct rlimit rl) { - if (sizeof(rl.rlim_cur) > sizeof(long)) { - return Py_BuildValue("LL", - (long long) rl.rlim_cur, - (long long) rl.rlim_max); + PyObject *cur = rlim2py(rl.rlim_cur); + if (cur == NULL) { + return NULL; } - return Py_BuildValue("ll", (long) rl.rlim_cur, (long) rl.rlim_max); + PyObject *max = rlim2py(rl.rlim_max); + return Py_BuildValue("NN", cur, max); } /*[clinic input] @@ -495,16 +522,38 @@ resource_exec(PyObject *module) ADD_INT(module, RLIMIT_KQUEUES); #endif - PyObject *v; - if (sizeof(RLIM_INFINITY) > sizeof(long)) { - v = PyLong_FromLongLong((long long) RLIM_INFINITY); - } else - { - v = PyLong_FromLong((long) RLIM_INFINITY); - } - if (PyModule_Add(module, "RLIM_INFINITY", v) < 0) { +#ifdef RLIMIT_NTHR + ADD_INT(module, RLIMIT_NTHR); +#endif + +#ifdef RLIMIT_THREADS + ADD_INT(module, RLIMIT_THREADS); +#endif + +#ifdef RLIMIT_UMTXP + ADD_INT(module, RLIMIT_UMTXP); +#endif + +#ifdef RLIMIT_PIPEBUF + ADD_INT(module, RLIMIT_PIPEBUF); +#endif + + if (PyModule_Add(module, "RLIM_INFINITY", rlim2py(RLIM_INFINITY)) < 0) { return -1; } + +#ifdef RLIM_SAVED_CUR + if (PyModule_Add(module, "RLIM_SAVED_CUR", rlim2py(RLIM_SAVED_CUR)) < 0) { + return -1; + } +#endif + +#ifdef RLIM_SAVED_MAX + if (PyModule_Add(module, "RLIM_SAVED_MAX", rlim2py(RLIM_SAVED_MAX)) < 0) { + return -1; + } +#endif + return 0; #undef ADD_INT diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index d234d504cb5..19fe509ec5e 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -241,6 +241,7 @@ set2list(fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) #endif /* FD_SETSIZE > 1024 */ /*[clinic input] +@permit_long_docstring_body select.select rlist: object @@ -261,7 +262,7 @@ A file descriptor is either a socket or file object, or a small integer gotten from a fileno() method call on one of those. The optional 4th argument specifies a timeout in seconds; it may be -a floating-point number to specify fractions of seconds. If it is absent +a non-integer to specify fractions of seconds. If it is absent or None, the call will never time out. The return value is a tuple of three lists corresponding to the first three @@ -276,7 +277,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=1199d5e101abca4a]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=b0403de75cd11cc1]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -304,8 +305,9 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, - "timeout must be a float or None"); + PyErr_Format(PyExc_TypeError, + "timeout must be a real number or None, not %T", + timeout_obj); } return NULL; } @@ -432,7 +434,7 @@ select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, typedef struct { PyObject_HEAD - PyObject *dict; + PyObject *dict; // cannot create cycles as it only contains exact ints int ufd_uptodate; int ufd_len; struct pollfd *ufds; @@ -631,8 +633,9 @@ select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) if (_PyTime_FromMillisecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, - "timeout must be an integer or None"); + PyErr_Format(PyExc_TypeError, + "timeout must be a real number or None, not %T", + timeout_obj); } return NULL; } @@ -973,8 +976,9 @@ select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) if (_PyTime_FromMillisecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, - "timeout must be an integer or None"); + PyErr_Format(PyExc_TypeError, + "timeout must be a real number or None, not %T", + timeout_obj); } return NULL; } @@ -1199,6 +1203,7 @@ static PyType_Spec devpoll_Type_spec = { /*[clinic input] +@permit_long_docstring_body select.poll Returns a polling object. @@ -1209,7 +1214,7 @@ polling them for I/O events. static PyObject * select_poll_impl(PyObject *module) -/*[clinic end generated code: output=16a665a4e1d228c5 input=3f877909d5696bbf]*/ +/*[clinic end generated code: output=16a665a4e1d228c5 input=5e07eea8ad564e7f]*/ { return (PyObject *)newPollObject(module); } @@ -1217,6 +1222,7 @@ select_poll_impl(PyObject *module) #ifdef HAVE_SYS_DEVPOLL_H /*[clinic input] +@permit_long_docstring_body select.devpoll Returns a polling object. @@ -1227,7 +1233,7 @@ polling them for I/O events. static PyObject * select_devpoll_impl(PyObject *module) -/*[clinic end generated code: output=ea9213cc87fd9581 input=53a1af94564f00a3]*/ +/*[clinic end generated code: output=ea9213cc87fd9581 input=048506faef19d947]*/ { return (PyObject *)newDevPollObject(module); } @@ -1562,7 +1568,7 @@ select_epoll_unregister_impl(pyEpoll_Object *self, int fd) select.epoll.poll timeout as timeout_obj: object = None - the maximum time to wait in seconds (as float); + the maximum time to wait in seconds (with fractions); a timeout of None or -1 makes poll wait indefinitely maxevents: int = -1 the maximum number of events returned; -1 means no limit @@ -1576,7 +1582,7 @@ as a list of (fd, events) 2-tuples. static PyObject * select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int maxevents) -/*[clinic end generated code: output=e02d121a20246c6c input=33d34a5ea430fd5b]*/ +/*[clinic end generated code: output=e02d121a20246c6c input=deafa7f04a60ebe0]*/ { int nfds, i; PyObject *elist = NULL, *etuple = NULL; @@ -1592,8 +1598,9 @@ select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, - "timeout must be an integer or None"); + PyErr_Format(PyExc_TypeError, + "timeout must be a real number or None, not %T", + timeout_obj); } return NULL; } @@ -2288,7 +2295,7 @@ select.kqueue.control The maximum number of events that the kernel will return. timeout as otimeout: object = None The maximum time to wait in seconds, or else None to wait forever. - This accepts floats for smaller timeouts, too. + This accepts non-integers for smaller timeouts, too. / Calls the kernel kevent function. @@ -2297,7 +2304,7 @@ Calls the kernel kevent function. static PyObject * select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, int maxevents, PyObject *otimeout) -/*[clinic end generated code: output=81324ff5130db7ae input=59c4e30811209c47]*/ +/*[clinic end generated code: output=81324ff5130db7ae input=be969d2bc6f84205]*/ { int gotevents = 0; int nchanges = 0; @@ -2328,9 +2335,8 @@ select_kqueue_control_impl(kqueue_queue_Object *self, PyObject *changelist, if (_PyTime_FromSecondsObject(&timeout, otimeout, _PyTime_ROUND_TIMEOUT) < 0) { PyErr_Format(PyExc_TypeError, - "timeout argument must be a number " - "or None, got %.200s", - _PyType_Name(Py_TYPE(otimeout))); + "timeout must be a real number or None, not %T", + otimeout); return NULL; } @@ -2480,7 +2486,11 @@ static PyType_Slot poll_Type_slots[] = { static PyType_Spec poll_Type_spec = { .name = "select.poll", .basicsize = sizeof(pollObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + ), .slots = poll_Type_slots, }; @@ -2526,11 +2536,10 @@ static PyType_Slot pyEpoll_Type_slots[] = { }; static PyType_Spec pyEpoll_Type_spec = { - "select.epoll", - sizeof(pyEpoll_Object), - 0, - Py_TPFLAGS_DEFAULT, - pyEpoll_Type_slots + .name = "select.epoll", + .basicsize = sizeof(pyEpoll_Object), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .slots = pyEpoll_Type_slots }; #endif /* HAVE_EPOLL */ diff --git a/Modules/sha1module.c b/Modules/sha1module.c index f4a00cdb422..89e66240d1d 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -8,48 +8,40 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. */ -/* SHA1 objects */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" #include "hashlib.h" +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() -/*[clinic input] -module _sha1 -class SHA1Type "SHA1object *" "&PyType_Type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dc9a20d1becb759]*/ +#include "_hacl/Hacl_Hash_SHA1.h" /* The SHA1 block size and message digest sizes, in bytes */ #define SHA1_BLOCKSIZE 64 #define SHA1_DIGESTSIZE 20 -#include "_hacl/Hacl_Hash_SHA1.h" +// --- Module objects --------------------------------------------------------- typedef struct { - PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; - PyThread_type_lock lock; + HASHLIB_OBJECT_HEAD Hacl_Hash_SHA1_state_t *hash_state; } SHA1object; #define _SHA1object_CAST(op) ((SHA1object *)(op)) -#include "clinic/sha1module.c.h" - +// --- Module state ----------------------------------------------------------- typedef struct { PyTypeObject* sha1_type; @@ -63,6 +55,18 @@ sha1_get_state(PyObject *module) return (SHA1State *)state; } +// --- Module clinic configuration -------------------------------------------- + +/*[clinic input] +module _sha1 +class SHA1Type "SHA1object *" "&PyType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dc9a20d1becb759]*/ + +#include "clinic/sha1module.c.h" + +// --- SHA-1 object interface configuration ----------------------------------- + static SHA1object * newSHA1object(SHA1State *st) { @@ -78,13 +82,6 @@ newSHA1object(SHA1State *st) /* Internal methods for a hash object */ -static int -SHA1_traverse(PyObject *ptr, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(ptr)); - return 0; -} - static void SHA1_dealloc(PyObject *op) { @@ -121,9 +118,9 @@ SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls) return NULL; } - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); newobj->hash_state = Hacl_Hash_SHA1_copy(self->hash_state); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); if (newobj->hash_state == NULL) { Py_DECREF(newobj); return PyErr_NoMemory(); @@ -142,9 +139,9 @@ SHA1Type_digest_impl(SHA1object *self) /*[clinic end generated code: output=2f05302a7aa2b5cb input=13824b35407444bd]*/ { unsigned char digest[SHA1_DIGESTSIZE]; - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); Hacl_Hash_SHA1_digest(self->hash_state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return PyBytes_FromStringAndSize((const char *)digest, SHA1_DIGESTSIZE); } @@ -159,9 +156,9 @@ SHA1Type_hexdigest_impl(SHA1object *self) /*[clinic end generated code: output=4161fd71e68c6659 input=97691055c0c74ab0]*/ { unsigned char digest[SHA1_DIGESTSIZE]; - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); Hacl_Hash_SHA1_digest(self->hash_state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return _Py_strhex((const char *)digest, SHA1_DIGESTSIZE); } @@ -198,22 +195,11 @@ SHA1Type_update_impl(SHA1object *self, PyObject *obj) /*[clinic end generated code: output=cdc8e0e106dbec5f input=aad8e07812edbba3]*/ { Py_buffer buf; - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - update(self->hash_state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - update(self->hash_state, buf.buf, buf.len); - } - + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, buf.len, + update(self->hash_state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -255,7 +241,7 @@ static PyType_Slot sha1_type_slots[] = { {Py_tp_dealloc, SHA1_dealloc}, {Py_tp_methods, SHA1_methods}, {Py_tp_getset, SHA1_getseters}, - {Py_tp_traverse, SHA1_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0,0} }; @@ -272,19 +258,25 @@ static PyType_Spec sha1_type_spec = { /*[clinic input] _sha1.sha1 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA1 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=6f8b3af05126e18e input=bd54b68e2bf36a8a]*/ +_sha1_sha1_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=0d453775924f88a7 input=807f25264e0ac656]*/ { SHA1object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); @@ -308,16 +300,12 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return PyErr_NoMemory(); } if (string) { - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor - * where it is not yet possible to have concurrent access. */ - Py_BEGIN_ALLOW_THREADS - update(new->hash_state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - update(new->hash_state, buf.buf, buf.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + buf.len, + update(new->hash_state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); } diff --git a/Modules/sha2module.c b/Modules/sha2module.c index e88d7cb2d45..9453b0be512 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -9,32 +9,26 @@ Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) Jonathan Protzenko (jonathan@protzenko.fr) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. */ -/* SHA objects */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" -#include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "hashlib.h" -/*[clinic input] -module _sha2 -class SHA256Type "SHA256object *" "&PyType_Type" -class SHA512Type "SHA512object *" "&PyType_Type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5315a7b611c9afc]*/ - +#include "_hacl/Hacl_Hash_SHA2.h" /* The SHA block sizes and maximum message digest sizes, in bytes */ @@ -43,34 +37,26 @@ class SHA512Type "SHA512object *" "&PyType_Type" #define SHA512_BLOCKSIZE 128 #define SHA512_DIGESTSIZE 64 -/* Our SHA2 implementations defer to the HACL* verified library. */ - -#include "_hacl/Hacl_Hash_SHA2.h" +// --- Module objects --------------------------------------------------------- // TODO: Get rid of int digestsize in favor of Hacl state info? typedef struct { - PyObject_HEAD + HASHLIB_OBJECT_HEAD int digestsize; - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; Hacl_Hash_SHA2_state_t_256 *state; } SHA256object; typedef struct { - PyObject_HEAD + HASHLIB_OBJECT_HEAD int digestsize; - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; Hacl_Hash_SHA2_state_t_512 *state; } SHA512object; #define _SHA256object_CAST(op) ((SHA256object *)(op)) #define _SHA512object_CAST(op) ((SHA512object *)(op)) -#include "clinic/sha2module.c.h" +// --- Module state ----------------------------------------------------------- /* We shall use run-time type information in the remainder of this module to * tell apart SHA2-224 and SHA2-256 */ @@ -89,6 +75,19 @@ sha2_get_state(PyObject *module) return (sha2_state *)state; } +// --- Module clinic configuration -------------------------------------------- + +/*[clinic input] +module _sha2 +class SHA256Type "SHA256object *" "&PyType_Type" +class SHA512Type "SHA512object *" "&PyType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5315a7b611c9afc]*/ + +#include "clinic/sha2module.c.h" + +// --- SHA-2 object interface ------------------------------------------------- + static int SHA256copy(SHA256object *src, SHA256object *dest) { @@ -166,14 +165,6 @@ newSHA512object(sha2_state *state) } /* Internal methods for our hash objects. */ - -static int -SHA2_traverse(PyObject *ptr, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(ptr)); - return 0; -} - static void SHA256_dealloc(PyObject *op) { @@ -272,9 +263,9 @@ SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls) } } - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); rc = SHA256copy(self, newobj); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); if (rc < 0) { Py_DECREF(newobj); return NULL; @@ -309,9 +300,9 @@ SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls) } } - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); rc = SHA512copy(self, newobj); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); if (rc < 0) { Py_DECREF(newobj); return NULL; @@ -331,11 +322,11 @@ SHA256Type_digest_impl(SHA256object *self) { uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. Hacl_Hash_SHA2_digest_256(self->state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -351,11 +342,11 @@ SHA512Type_digest_impl(SHA512object *self) { uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); // HACL* performs copies under the hood so that self->state remains valid // after this call. Hacl_Hash_SHA2_digest_512(self->state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return PyBytes_FromStringAndSize((const char *)digest, self->digestsize); } @@ -371,9 +362,9 @@ SHA256Type_hexdigest_impl(SHA256object *self) { uint8_t digest[SHA256_DIGESTSIZE]; assert(self->digestsize <= SHA256_DIGESTSIZE); - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); Hacl_Hash_SHA2_digest_256(self->state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -389,9 +380,9 @@ SHA512Type_hexdigest_impl(SHA512object *self) { uint8_t digest[SHA512_DIGESTSIZE]; assert(self->digestsize <= SHA512_DIGESTSIZE); - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); Hacl_Hash_SHA2_digest_512(self->state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return _Py_strhex((const char *)digest, self->digestsize); } @@ -409,22 +400,11 @@ SHA256Type_update_impl(SHA256object *self, PyObject *obj) /*[clinic end generated code: output=dc58a580cf8905a5 input=b2d449d5b30f0f5a]*/ { Py_buffer buf; - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - update_256(self->state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - update_256(self->state, buf.buf, buf.len); - } - + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, buf.len, + update_256(self->state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -443,22 +423,11 @@ SHA512Type_update_impl(SHA512object *self, PyObject *obj) /*[clinic end generated code: output=9af211766c0b7365 input=ded2b46656566283]*/ { Py_buffer buf; - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - update_512(self->state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - update_512(self->state, buf.buf, buf.len); - } - + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, buf.len, + update_512(self->state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -543,7 +512,7 @@ static PyType_Slot sha256_types_slots[] = { {Py_tp_dealloc, SHA256_dealloc}, {Py_tp_methods, SHA256_methods}, {Py_tp_getset, SHA256_getseters}, - {Py_tp_traverse, SHA2_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0,0} }; @@ -551,7 +520,7 @@ static PyType_Slot sha512_type_slots[] = { {Py_tp_dealloc, SHA512_dealloc}, {Py_tp_methods, SHA512_methods}, {Py_tp_getset, SHA512_getseters}, - {Py_tp_traverse, SHA2_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0,0} }; @@ -594,18 +563,24 @@ static PyType_Spec sha512_type_spec = { /*[clinic input] _sha2.sha256 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-256 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=243c9dd289931f87 input=6249da1de607280a]*/ +_sha2_sha256_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=49828a7bcd418f45 input=9ce1d70e669abc14]*/ { Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); @@ -632,16 +607,12 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) return PyErr_NoMemory(); } if (string) { - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor - * where it is not yet possible to have concurrent access. */ - Py_BEGIN_ALLOW_THREADS - update_256(new->state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - update_256(new->state, buf.buf, buf.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + buf.len, + update_256(new->state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); } @@ -651,18 +622,25 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha224 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-224 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=68191f232e4a3843 input=c42bcba47fd7d2b7]*/ +_sha2_sha224_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=2163cb03b6cf6157 input=612f7682a889bc2a]*/ { Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } + if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); } @@ -687,16 +665,12 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) return PyErr_NoMemory(); } if (string) { - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor - * where it is not yet possible to have concurrent access. */ - Py_BEGIN_ALLOW_THREADS - update_256(new->state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - update_256(new->state, buf.buf, buf.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + buf.len, + update_256(new->state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); } @@ -706,19 +680,25 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha512 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-512 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=d55c8996eca214d7 input=0576ae2a6ebfad25]*/ +_sha2_sha512_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=cc3fcfce001a4538 input=19c9f2c06d59563a]*/ { SHA512object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } sha2_state *state = sha2_get_state(module); @@ -744,16 +724,12 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) return PyErr_NoMemory(); } if (string) { - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor - * where it is not yet possible to have concurrent access. */ - Py_BEGIN_ALLOW_THREADS - update_512(new->state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - update_512(new->state, buf.buf, buf.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + buf.len, + update_512(new->state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); } @@ -763,19 +739,25 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha384 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-384 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=b29a0d81d51d1368 input=4e9199d8de0d2f9b]*/ +_sha2_sha384_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=b6e3db593b5a0330 input=9fd50c942ad9e0bf]*/ { SHA512object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } sha2_state *state = sha2_get_state(module); @@ -801,16 +783,12 @@ _sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) return PyErr_NoMemory(); } if (string) { - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor - * where it is not yet possible to have concurrent access. */ - Py_BEGIN_ALLOW_THREADS - update_512(new->state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - update_512(new->state, buf.buf, buf.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + buf.len, + update_512(new->state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); } diff --git a/Modules/sha3module.c b/Modules/sha3module.c index a7edf5c66a1..38c9bc0405b 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -9,6 +9,7 @@ * Greg Stein (gstein@lyra.org) * Trevor Perrin (trevp@trevp.net) * Gregory P. Smith (greg@krypto.org) + * Bénédikt Tran (10796600+picnixz@users.noreply.github.com) * * Copyright (C) 2012-2022 Christian Heimes (christian@python.org) * Licensed to PSF under a Contributor Agreement. @@ -20,12 +21,28 @@ #endif #include "Python.h" +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() #include "hashlib.h" +#include "_hacl/Hacl_Hash_SHA3.h" + +/* + * Assert that 'LEN' can be safely casted to uint32_t. + * + * The 'LEN' parameter should be convertible to Py_ssize_t. + */ +#if !defined(NDEBUG) && (PY_SSIZE_T_MAX > UINT32_MAX) +#define CHECK_HACL_UINT32_T_LENGTH(LEN) assert((LEN) < (Py_ssize_t)UINT32_MAX) +#else +#define CHECK_HACL_UINT32_T_LENGTH(LEN) +#endif + #define SHA3_MAX_DIGESTSIZE 64 /* 64 Bytes (512 Bits) for 224 to 512 */ +// --- Module state ----------------------------------------------------------- + typedef struct { PyTypeObject *sha3_224_type; PyTypeObject *sha3_256_type; @@ -43,33 +60,34 @@ sha3_get_state(PyObject *module) return (SHA3State *)state; } -/*[clinic input] -module _sha3 -class _sha3.sha3_224 "SHA3object *" "&SHA3_224typ" -class _sha3.sha3_256 "SHA3object *" "&SHA3_256typ" -class _sha3.sha3_384 "SHA3object *" "&SHA3_384typ" -class _sha3.sha3_512 "SHA3object *" "&SHA3_512typ" -class _sha3.shake_128 "SHA3object *" "&SHAKE128type" -class _sha3.shake_256 "SHA3object *" "&SHAKE256type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b8a53680f370285a]*/ +// --- Module objects --------------------------------------------------------- /* The structure for storing SHA3 info */ -#include "_hacl/Hacl_Hash_SHA3.h" - typedef struct { - PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; + HASHLIB_OBJECT_HEAD Hacl_Hash_SHA3_state_t *hash_state; } SHA3object; #define _SHA3object_CAST(op) ((SHA3object *)(op)) +// --- Module clinic configuration -------------------------------------------- + +/*[clinic input] +module _sha3 +class _sha3.sha3_224 "SHA3object *" "&PyType_Type" +class _sha3.sha3_256 "SHA3object *" "&PyType_Type" +class _sha3.sha3_384 "SHA3object *" "&PyType_Type" +class _sha3.sha3_512 "SHA3object *" "&PyType_Type" +class _sha3.shake_128 "SHA3object *" "&PyType_Type" +class _sha3.shake_256 "SHA3object *" "&PyType_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ccd22550c7fb99bf]*/ + #include "clinic/sha3module.c.h" +// --- SHA-3 object interface ------------------------------------------------- + static SHA3object * newSHA3object(PyTypeObject *type) { @@ -105,18 +123,25 @@ sha3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *buf, Py_ssize_t len) /*[clinic input] @classmethod _sha3.sha3_224.__new__ as py_sha3_new - data: object(c_default="NULL") = b'' - / + + data as data_obj: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new SHA3 hash object. [clinic start generated code]*/ static PyObject * -py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) -/*[clinic end generated code: output=90409addc5d5e8b0 input=637e5f8f6a93982a]*/ +py_sha3_new_impl(PyTypeObject *type, PyObject *data_obj, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=dcec1eca20395f2a input=c106e0b4e2d67d58]*/ { + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } + Py_buffer buf = {NULL, NULL}; SHA3State *state = _PyType_GetModuleState(type); SHA3object *self = newSHA3object(type); @@ -156,16 +181,12 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); - if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor - * where it is not yet possible to have concurrent access. */ - Py_BEGIN_ALLOW_THREADS - sha3_update(self->hash_state, buf.buf, buf.len); - Py_END_ALLOW_THREADS - } - else { - sha3_update(self->hash_state, buf.buf, buf.len); - } + /* Do not use self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ + HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED( + buf.len, + sha3_update(self->hash_state, buf.buf, buf.len) + ); } PyBuffer_Release(&buf); @@ -206,34 +227,28 @@ SHA3_dealloc(PyObject *self) Py_DECREF(tp); } -static int -SHA3_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - /* External methods for a hash object */ /*[clinic input] _sha3.sha3_224.copy + cls: defining_class + Return a copy of the hash object. [clinic start generated code]*/ static PyObject * -_sha3_sha3_224_copy_impl(SHA3object *self) -/*[clinic end generated code: output=6c537411ecdcda4c input=93a44aaebea51ba8]*/ +_sha3_sha3_224_copy_impl(SHA3object *self, PyTypeObject *cls) +/*[clinic end generated code: output=13958b44c244013e input=7134b4dc0a2fbcac]*/ { SHA3object *newobj; - - if ((newobj = newSHA3object(Py_TYPE(self))) == NULL) { + if ((newobj = newSHA3object(cls)) == NULL) { return NULL; } - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); newobj->hash_state = Hacl_Hash_SHA3_copy(self->hash_state); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); if (newobj->hash_state == NULL) { Py_DECREF(newobj); return PyErr_NoMemory(); @@ -255,9 +270,9 @@ _sha3_sha3_224_digest_impl(SHA3object *self) unsigned char digest[SHA3_MAX_DIGESTSIZE]; // This function errors out if the algorithm is SHAKE. Here, we know this // not to be the case, and therefore do not perform error checking. - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); (void)Hacl_Hash_SHA3_digest(self->hash_state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return PyBytes_FromStringAndSize((const char *)digest, Hacl_Hash_SHA3_hash_len(self->hash_state)); } @@ -274,9 +289,9 @@ _sha3_sha3_224_hexdigest_impl(SHA3object *self) /*[clinic end generated code: output=75ad03257906918d input=2d91bb6e0d114ee3]*/ { unsigned char digest[SHA3_MAX_DIGESTSIZE]; - ENTER_HASHLIB(self); + HASHLIB_ACQUIRE_LOCK(self); (void)Hacl_Hash_SHA3_digest(self->hash_state, digest); - LEAVE_HASHLIB(self); + HASHLIB_RELEASE_LOCK(self); return _Py_strhex((const char *)digest, Hacl_Hash_SHA3_hash_len(self->hash_state)); } @@ -296,22 +311,11 @@ _sha3_sha3_224_update_impl(SHA3object *self, PyObject *data) /*[clinic end generated code: output=390b7abf7c9795a5 input=a887f54dcc4ae227]*/ { Py_buffer buf; - GET_BUFFER_VIEW_OR_ERROUT(data, &buf); - - if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) { - self->use_mutex = true; - } - if (self->use_mutex) { - Py_BEGIN_ALLOW_THREADS - PyMutex_Lock(&self->mutex); - sha3_update(self->hash_state, buf.buf, buf.len); - PyMutex_Unlock(&self->mutex); - Py_END_ALLOW_THREADS - } else { - sha3_update(self->hash_state, buf.buf, buf.len); - } - + HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED( + self, buf.len, + sha3_update(self->hash_state, buf.buf, buf.len) + ); PyBuffer_Release(&buf); Py_RETURN_NONE; } @@ -414,7 +418,7 @@ static PyGetSetDef SHA3_getseters[] = { static PyType_Slot type_slots_obj[] = { \ {Py_tp_clear, SHA3_clear}, \ {Py_tp_dealloc, SHA3_dealloc}, \ - {Py_tp_traverse, SHA3_traverse}, \ + {Py_tp_traverse, _PyObject_VisitType}, \ {Py_tp_doc, (char*)type_doc}, \ {Py_tp_methods, type_methods}, \ {Py_tp_getset, type_getseters}, \ @@ -465,71 +469,97 @@ SHA3_TYPE_SPEC(sha3_384_spec, "sha3_384", sha3_384_slots); SHA3_TYPE_SLOTS(sha3_512_slots, sha3_512__doc__, SHA3_methods, SHA3_getseters); SHA3_TYPE_SPEC(sha3_512_spec, "sha3_512", sha3_512_slots); -static PyObject * -_SHAKE_digest(PyObject *op, unsigned long digestlen, int hex) +static int +sha3_shake_check_digest_length(Py_ssize_t length) { - unsigned char *digest = NULL; - PyObject *result = NULL; - SHA3object *self = _SHA3object_CAST(op); - - if (digestlen >= (1 << 29)) { - PyErr_SetString(PyExc_ValueError, "length is too large"); - return NULL; + assert(length >= 0); + if ((size_t)length >= (1 << 29)) { + /* + * Raise OverflowError to match the semantics of OpenSSL SHAKE + * when the digest length exceeds the range of a 'Py_ssize_t'; + * the exception message will however be different in this case. + */ + PyErr_SetString(PyExc_OverflowError, "digest length is too large"); + return -1; } - digest = (unsigned char*)PyMem_Malloc(digestlen); - if (digest == NULL) { - return PyErr_NoMemory(); - } - - /* Get the raw (binary) digest value. The HACL functions errors out if: - * - the algorithm is not shake -- not the case here - * - the output length is zero -- we follow the existing behavior and return - * an empty digest, without raising an error */ - if (digestlen > 0) { - (void)Hacl_Hash_SHA3_squeeze(self->hash_state, digest, digestlen); - } - if (hex) { - result = _Py_strhex((const char *)digest, digestlen); - } - else { - result = PyBytes_FromStringAndSize((const char *)digest, digestlen); - } - PyMem_Free(digest); - return result; + return 0; } /*[clinic input] _sha3.shake_128.digest - length: unsigned_long - / + length: Py_ssize_t(allow_negative=False) Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * -_sha3_shake_128_digest_impl(SHA3object *self, unsigned long length) -/*[clinic end generated code: output=2313605e2f87bb8f input=418ef6a36d2e6082]*/ +_sha3_shake_128_digest_impl(SHA3object *self, Py_ssize_t length) +/*[clinic end generated code: output=6c53fb71a6cff0a0 input=1160c9f86ae0f867]*/ { - return _SHAKE_digest((PyObject *)self, length, 0); + if (sha3_shake_check_digest_length(length) < 0) { + return NULL; + } + + /* + * Hacl_Hash_SHA3_squeeze() fails if the algorithm is not SHAKE, + * or if the length is 0. In the latter case, we follow OpenSSL's + * behavior and return an empty digest, without raising an error. + */ + if (length == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + CHECK_HACL_UINT32_T_LENGTH(length); + + PyBytesWriter *writer = PyBytesWriter_Create(length); + if (writer == NULL) { + return NULL; + } + uint8_t *buffer = (uint8_t *)PyBytesWriter_GetData(writer); + + HASHLIB_ACQUIRE_LOCK(self); + (void)Hacl_Hash_SHA3_squeeze(self->hash_state, buffer, (uint32_t)length); + HASHLIB_RELEASE_LOCK(self); + + return PyBytesWriter_Finish(writer); } /*[clinic input] _sha3.shake_128.hexdigest - length: unsigned_long - / + length: Py_ssize_t(allow_negative=False) Return the digest value as a string of hexadecimal digits. [clinic start generated code]*/ static PyObject * -_sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length) -/*[clinic end generated code: output=bf8e2f1e490944a8 input=69fb29b0926ae321]*/ +_sha3_shake_128_hexdigest_impl(SHA3object *self, Py_ssize_t length) +/*[clinic end generated code: output=a27412d404f64512 input=ff06c9362949d2c8]*/ { - return _SHAKE_digest((PyObject *)self, length, 1); + if (sha3_shake_check_digest_length(length) < 0) { + return NULL; + } + + /* See _sha3_shake_128_digest_impl() for the fast path rationale. */ + if (length == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } + CHECK_HACL_UINT32_T_LENGTH(length); + + uint8_t *buffer = PyMem_Malloc(length); + if (buffer == NULL) { + return PyErr_NoMemory(); + } + + HASHLIB_ACQUIRE_LOCK(self); + (void)Hacl_Hash_SHA3_squeeze(self->hash_state, buffer, (uint32_t)length); + HASHLIB_RELEASE_LOCK(self); + + PyObject *digest = _Py_strhex((const char *)buffer, length); + PyMem_Free(buffer); + return digest; } static PyObject * diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 54bcd3270ef..4d0e224ff75 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -460,6 +460,7 @@ signal_raise_signal_impl(PyObject *module, int signalnum) } /*[clinic input] +@permit_long_docstring_body signal.signal signalnum: int @@ -478,7 +479,7 @@ the first is the signal number, the second is the interrupted stack frame. static PyObject * signal_signal_impl(PyObject *module, int signalnum, PyObject *handler) -/*[clinic end generated code: output=b44cfda43780f3a1 input=deee84af5fa0432c]*/ +/*[clinic end generated code: output=b44cfda43780f3a1 input=7608656f34fa378b]*/ { _signal_module_state *modstate = get_signal_state(module); PyObject *old_handler; @@ -709,6 +710,7 @@ signal_siginterrupt_impl(PyObject *module, int signalnum, int flag) /*[clinic input] +@permit_long_summary signal.set_wakeup_fd fd as fdobj: object @@ -727,7 +729,7 @@ The fd must be non-blocking. static PyObject * signal_set_wakeup_fd_impl(PyObject *module, PyObject *fdobj, int warn_on_full_buffer) -/*[clinic end generated code: output=2280d72dd2a54c4f input=5b545946a28b8339]*/ +/*[clinic end generated code: output=2280d72dd2a54c4f input=1b914d48079e9274]*/ { struct _Py_stat_struct status; #ifdef MS_WINDOWS @@ -847,6 +849,7 @@ PySignal_SetWakeupFd(int fd) #ifdef HAVE_SETITIMER /*[clinic input] +@permit_long_docstring_body signal.setitimer which: int @@ -865,7 +868,7 @@ Returns old values as a tuple: (delay, interval). static PyObject * signal_setitimer_impl(PyObject *module, int which, PyObject *seconds, PyObject *interval) -/*[clinic end generated code: output=65f9dcbddc35527b input=de43daf194e6f66f]*/ +/*[clinic end generated code: output=65f9dcbddc35527b input=ab5bf2b8f5cff3f4]*/ { _signal_module_state *modstate = get_signal_state(module); @@ -1180,7 +1183,13 @@ signal_sigwaitinfo_impl(PyObject *module, sigset_t sigset) err = sigwaitinfo(&sigset, &si); Py_END_ALLOW_THREADS } while (err == -1 - && errno == EINTR && !(async_err = PyErr_CheckSignals())); + && (errno == EINTR +#if defined(__NetBSD__) + /* NetBSD's implementation violates POSIX by setting + * errno to ECANCELED instead of EINTR. */ + || errno == ECANCELED +#endif + ) && !(async_err = PyErr_CheckSignals())); if (err == -1) return (!async_err) ? PyErr_SetFromErrno(PyExc_OSError) : NULL; @@ -1201,13 +1210,13 @@ signal.sigtimedwait Like sigwaitinfo(), but with a timeout. -The timeout is specified in seconds, with floating-point numbers allowed. +The timeout is specified in seconds, rounded up to nanoseconds. [clinic start generated code]*/ static PyObject * signal_sigtimedwait_impl(PyObject *module, sigset_t sigset, PyObject *timeout_obj) -/*[clinic end generated code: output=59c8971e8ae18a64 input=955773219c1596cd]*/ +/*[clinic end generated code: output=59c8971e8ae18a64 input=f89af57d645e48e0]*/ { PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, @@ -1623,7 +1632,7 @@ signal_module_exec(PyObject *m) modstate->ignore_handler = state->ignore_handler; // borrowed ref #ifdef PYHAVE_ITIMER_ERROR - modstate->itimer_error = PyErr_NewException("signal.itimer_error", + modstate->itimer_error = PyErr_NewException("signal.ItimerError", PyExc_OSError, NULL); if (modstate->itimer_error == NULL) { return -1; @@ -1940,7 +1949,7 @@ signal_install_handlers(void) /* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. * * All of the code in this function must only use async-signal-safe functions, - * listed at `man 7 signal` or + * listed at `man 7 signal-safety` or * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. * * If this function is updated, update also _posix_spawn() of subprocess.py. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 47958379263..1ef359cb265 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -109,9 +109,9 @@ Local naming conventions: #include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_fileutils.h" // _Py_set_inheritable() #include "pycore_moduleobject.h" // _PyModule_GetState +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_time.h" // _PyTime_AsMilliseconds() #include "pycore_pystate.h" // _Py_AssertHoldsTstate() -#include "pycore_pyatomic_ft_wrappers.h" #ifdef _Py_MEMORY_SANITIZER # include <sanitizer/msan_interface.h> @@ -565,7 +565,6 @@ static int sock_cloexec_works = -1; static inline void set_sock_fd(PySocketSockObject *s, SOCKET_T fd) { -#ifdef Py_GIL_DISABLED #if SIZEOF_SOCKET_T == SIZEOF_INT _Py_atomic_store_int_relaxed((int *)&s->sock_fd, (int)fd); #elif SIZEOF_SOCKET_T == SIZEOF_LONG @@ -575,15 +574,11 @@ set_sock_fd(PySocketSockObject *s, SOCKET_T fd) #else #error "Unsupported SIZEOF_SOCKET_T" #endif -#else - s->sock_fd = fd; -#endif } static inline SOCKET_T get_sock_fd(PySocketSockObject *s) { -#ifdef Py_GIL_DISABLED #if SIZEOF_SOCKET_T == SIZEOF_INT return (SOCKET_T)_Py_atomic_load_int_relaxed((int *)&s->sock_fd); #elif SIZEOF_SOCKET_T == SIZEOF_LONG @@ -593,9 +588,6 @@ get_sock_fd(PySocketSockObject *s) #else #error "Unsupported SIZEOF_SOCKET_T" #endif -#else - return s->sock_fd; -#endif } #define _PySocketSockObject_CAST(op) ((PySocketSockObject *)(op)) @@ -638,33 +630,22 @@ _PyLong_##NAME##_Converter(PyObject *obj, void *ptr) \ return 1; \ } -UNSIGNED_INT_CONVERTER(UInt16, uint16_t) -UNSIGNED_INT_CONVERTER(UInt32, uint32_t) - #if defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) # ifdef MS_WINDOWS UNSIGNED_INT_CONVERTER(NetIfindex, NET_IFINDEX) # else - UNSIGNED_INT_CONVERTER(NetIfindex, unsigned int) +# define _PyLong_NetIfindex_Converter _PyLong_UnsignedInt_Converter # define NET_IFINDEX unsigned int # endif #endif // defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) /*[python input] -class uint16_converter(CConverter): - type = "uint16_t" - converter = '_PyLong_UInt16_Converter' - -class uint32_converter(CConverter): - type = "uint32_t" - converter = '_PyLong_UInt32_Converter' - class NET_IFINDEX_converter(CConverter): type = "NET_IFINDEX" converter = '_PyLong_NetIfindex_Converter' [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3de2e4a03fbf83b8]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=1cf809c40a407c34]*/ /*[clinic input] module _socket @@ -736,12 +717,6 @@ select_error(void) # define SOCK_INPROGRESS_ERR EINPROGRESS #endif -#ifdef _MSC_VER -# define SUPPRESS_DEPRECATED_CALL __pragma(warning(suppress: 4996)) -#else -# define SUPPRESS_DEPRECATED_CALL -#endif - /* Convenience function to raise an error according to errno and return a NULL pointer from a function. */ @@ -3358,35 +3333,62 @@ sock_setsockopt(PyObject *self, PyObject *args) { PySocketSockObject *s = _PySocketSockObject_CAST(self); + Py_ssize_t arglen; int level; int optname; int res; - Py_buffer optval; + Py_buffer buffer; int flag; unsigned int optlen; - PyObject *none; + PyObject *optval; + + if (!PyArg_ParseTuple(args, "iiO|I:setsockopt", + &level, &optname, &optval, &optlen)) + { + return NULL; + } + + arglen = PyTuple_Size(args); + if (arglen == 3 && optval == Py_None) { + PyErr_Format(PyExc_TypeError, + "setsockopt() requires 4 arguments when the third argument is None", + arglen); + return NULL; + } + if (arglen == 4 && optval != Py_None) { + PyErr_Format(PyExc_TypeError, + "setsockopt() only takes 4 arguments when the third argument is None (got %T)", + optval); + return NULL; + } #ifdef AF_VSOCK if (s->sock_family == AF_VSOCK) { + if (!PyIndex_Check(optval)) { + PyErr_Format(PyExc_TypeError, + "setsockopt() argument 3 for AF_VSOCK must be an int (got %T)", + optval); + } uint64_t vflag; // Must be set width of 64 bits /* setsockopt(level, opt, flag) */ - if (PyArg_ParseTuple(args, "iiK:setsockopt", - &level, &optname, &vflag)) { - // level should always be set to AF_VSOCK - res = setsockopt(get_sock_fd(s), level, optname, - (void*)&vflag, sizeof vflag); - goto done; + if (!PyArg_Parse(optval, "K", &vflag)) { + return NULL; } - return NULL; + // level should always be set to AF_VSOCK + res = setsockopt(get_sock_fd(s), level, optname, + (void*)&vflag, sizeof vflag); + goto done; } #endif /* setsockopt(level, opt, flag) */ - if (PyArg_ParseTuple(args, "iii:setsockopt", - &level, &optname, &flag)) { + if (PyIndex_Check(optval)) { + if (!PyArg_Parse(optval, "i", &flag)) { + return NULL; + } #ifdef MS_WINDOWS if (optname == SIO_TCP_SET_ACK_FREQUENCY) { - int dummy; + DWORD dummy; res = WSAIoctl(get_sock_fd(s), SIO_TCP_SET_ACK_FREQUENCY, &flag, sizeof(flag), NULL, 0, &dummy, NULL, NULL); if (res >= 0) { @@ -3400,36 +3402,40 @@ sock_setsockopt(PyObject *self, PyObject *args) goto done; } - PyErr_Clear(); - /* setsockopt(level, opt, None, flag) */ - if (PyArg_ParseTuple(args, "iiO!I:setsockopt", - &level, &optname, Py_TYPE(Py_None), &none, &optlen)) { + /* setsockopt(level, opt, None, optlen) */ + if (optval == Py_None) { assert(sizeof(socklen_t) >= sizeof(unsigned int)); res = setsockopt(get_sock_fd(s), level, optname, NULL, (socklen_t)optlen); goto done; } - PyErr_Clear(); /* setsockopt(level, opt, buffer) */ - if (!PyArg_ParseTuple(args, "iiy*:setsockopt", - &level, &optname, &optval)) - return NULL; - + if (PyObject_CheckBuffer(optval)) { + if (!PyArg_Parse(optval, "y*", &buffer)) { + return NULL; + } #ifdef MS_WINDOWS - if (optval.len > INT_MAX) { - PyBuffer_Release(&optval); - PyErr_Format(PyExc_OverflowError, - "socket option is larger than %i bytes", - INT_MAX); - return NULL; - } - res = setsockopt(get_sock_fd(s), level, optname, - optval.buf, (int)optval.len); + if (buffer.len > INT_MAX) { + PyBuffer_Release(&buffer); + PyErr_Format(PyExc_OverflowError, + "socket option is larger than %i bytes", + INT_MAX); + return NULL; + } + res = setsockopt(get_sock_fd(s), level, optname, + buffer.buf, (int)buffer.len); #else - res = setsockopt(get_sock_fd(s), level, optname, optval.buf, optval.len); + res = setsockopt(get_sock_fd(s), level, optname, buffer.buf, buffer.len); #endif - PyBuffer_Release(&optval); + PyBuffer_Release(&buffer); + goto done; + } + + PyErr_Format(PyExc_TypeError, + "socket option should be int, bytes-like object or None (got %T)", + optval); + return NULL; done: if (res < 0) { @@ -3462,7 +3468,6 @@ sock_getsockopt(PyObject *self, PyObject *args) int level; int optname; int res; - PyObject *buf; socklen_t buflen = 0; int flag = 0; socklen_t flagsize; @@ -3507,17 +3512,17 @@ sock_getsockopt(PyObject *self, PyObject *args) "getsockopt buflen out of range"); return NULL; } - buf = PyBytes_FromStringAndSize((char *)NULL, buflen); - if (buf == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(buflen); + if (writer == NULL) { return NULL; + } res = getsockopt(get_sock_fd(s), level, optname, - (void *)PyBytes_AS_STRING(buf), &buflen); + PyBytesWriter_GetData(writer), &buflen); if (res < 0) { - Py_DECREF(buf); + PyBytesWriter_Discard(writer); return s->errorhandler(); } - _PyBytes_Resize(&buf, buflen); - return buf; + return PyBytesWriter_FinishWithSize(writer, buflen); } PyDoc_STRVAR(getsockopt_doc, @@ -3974,7 +3979,6 @@ sock_recv(PyObject *self, PyObject *args) Py_ssize_t recvlen, outlen; int flags = 0; - PyObject *buf; if (!PyArg_ParseTuple(args, "n|i:recv", &recvlen, &flags)) return NULL; @@ -3986,25 +3990,21 @@ sock_recv(PyObject *self, PyObject *args) } /* Allocate a new string. */ - buf = PyBytes_FromStringAndSize((char *) 0, recvlen); - if (buf == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(recvlen); + if (writer == NULL) { return NULL; + } /* Call the guts */ - outlen = sock_recv_guts(s, PyBytes_AS_STRING(buf), recvlen, flags); + outlen = sock_recv_guts(s, PyBytesWriter_GetData(writer), recvlen, flags); if (outlen < 0) { /* An error occurred, release the string and return an error. */ - Py_DECREF(buf); + PyBytesWriter_Discard(writer); return NULL; } - if (outlen != recvlen) { - /* We did not read as many bytes as we anticipated, resize the - string if possible and be successful. */ - _PyBytes_Resize(&buf, outlen); - } - return buf; + return PyBytesWriter_FinishWithSize(writer, outlen); } PyDoc_STRVAR(recv_doc, @@ -4160,7 +4160,6 @@ sock_recvfrom(PyObject *self, PyObject *args) { PySocketSockObject *s = _PySocketSockObject_CAST(self); - PyObject *buf = NULL; PyObject *addr = NULL; PyObject *ret = NULL; int flags = 0; @@ -4175,28 +4174,27 @@ sock_recvfrom(PyObject *self, PyObject *args) return NULL; } - buf = PyBytes_FromStringAndSize((char *) 0, recvlen); - if (buf == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(recvlen); + if (writer == NULL) { return NULL; + } - outlen = sock_recvfrom_guts(s, PyBytes_AS_STRING(buf), + outlen = sock_recvfrom_guts(s, PyBytesWriter_GetData(writer), recvlen, flags, &addr); if (outlen < 0) { + PyBytesWriter_Discard(writer); goto finally; } - if (outlen != recvlen) { - /* We did not read as many bytes as we anticipated, resize the - string if possible and be successful. */ - if (_PyBytes_Resize(&buf, outlen) < 0) - /* Oopsy, not so successful after all. */ - goto finally; + PyObject *buf = PyBytesWriter_FinishWithSize(writer, outlen); + if (buf == NULL) { + goto finally; } ret = PyTuple_Pack(2, buf, addr); + Py_DECREF(buf); finally: - Py_XDECREF(buf); Py_XDECREF(addr); return ret; } @@ -4429,11 +4427,10 @@ sock_recvmsg_guts(PySocketSockObject *s, struct iovec *iov, int iovlen, static PyObject * makeval_recvmsg(ssize_t received, void *data) { - PyObject **buf = data; - - if (received < PyBytes_GET_SIZE(*buf)) - _PyBytes_Resize(buf, received); - return Py_XNewRef(*buf); + PyBytesWriter **writer = data; + PyObject *buf = PyBytesWriter_FinishWithSize(*writer, received); + *writer = NULL; + return buf; } /* s.recvmsg(bufsize[, ancbufsize[, flags]]) method */ @@ -4441,13 +4438,8 @@ makeval_recvmsg(ssize_t received, void *data) static PyObject * sock_recvmsg(PyObject *self, PyObject *args) { - PySocketSockObject *s = _PySocketSockObject_CAST(self); - Py_ssize_t bufsize, ancbufsize = 0; int flags = 0; - struct iovec iov; - PyObject *buf = NULL, *retval = NULL; - if (!PyArg_ParseTuple(args, "n|ni:recvmsg", &bufsize, &ancbufsize, &flags)) return NULL; @@ -4455,17 +4447,23 @@ sock_recvmsg(PyObject *self, PyObject *args) PyErr_SetString(PyExc_ValueError, "negative buffer size in recvmsg()"); return NULL; } - if ((buf = PyBytes_FromStringAndSize(NULL, bufsize)) == NULL) + + PyBytesWriter *writer = PyBytesWriter_Create(bufsize); + if (writer == NULL) { return NULL; - iov.iov_base = PyBytes_AS_STRING(buf); + } + struct iovec iov; + iov.iov_base = PyBytesWriter_GetData(writer); iov.iov_len = bufsize; /* Note that we're passing a pointer to *our pointer* to the bytes - object here (&buf); makeval_recvmsg() may incref the object, or - deallocate it and set our pointer to NULL. */ + writer (&writer); makeval_recvmsg() finish it and set our pointer to + NULL. */ + PyObject *retval; + PySocketSockObject *s = _PySocketSockObject_CAST(self); retval = sock_recvmsg_guts(s, &iov, 1, flags, ancbufsize, - &makeval_recvmsg, &buf); - Py_XDECREF(buf); + &makeval_recvmsg, &writer); + PyBytesWriter_Discard(writer); return retval; } @@ -4618,55 +4616,63 @@ sock_send_impl(PySocketSockObject *s, void *data) return (ctx->result >= 0); } -/* s.send(data [,flags]) method */ +/*[clinic input] +@permit_long_docstring_body +_socket.socket.send + self as s: self(type="PySocketSockObject *") + data as pbuf: Py_buffer + flags: int = 0 + / + +Send a data string to the socket. + +For the optional flags argument, see the Unix manual. +Return the number of bytes sent; this may be less than len(data) if the network is busy. +[clinic start generated code]*/ static PyObject * -sock_send(PyObject *self, PyObject *args) -{ - PySocketSockObject *s = _PySocketSockObject_CAST(self); +_socket_socket_send_impl(PySocketSockObject *s, Py_buffer *pbuf, int flags) +/*[clinic end generated code: output=3ddf83f17d0c875b input=e776a48af2e3d615]*/ - int flags = 0; - Py_buffer pbuf; +{ struct sock_send ctx; - if (!PyArg_ParseTuple(args, "y*|i:send", &pbuf, &flags)) - return NULL; - if (!IS_SELECTABLE(s)) { - PyBuffer_Release(&pbuf); return select_error(); } - ctx.buf = pbuf.buf; - ctx.len = pbuf.len; + ctx.buf = pbuf->buf; + ctx.len = pbuf->len; ctx.flags = flags; if (sock_call(s, 1, sock_send_impl, &ctx) < 0) { - PyBuffer_Release(&pbuf); return NULL; } - PyBuffer_Release(&pbuf); return PyLong_FromSsize_t(ctx.result); } -PyDoc_STRVAR(send_doc, -"send(data[, flags]) -> count\n\ -\n\ -Send a data string to the socket. For the optional flags\n\ -argument, see the Unix manual. Return the number of bytes\n\ -sent; this may be less than len(data) if the network is busy."); +/*[clinic input] +_socket.socket.sendall + self as s: self(type="PySocketSockObject *") + data as pbuf: Py_buffer + flags: int = 0 + / -/* s.sendall(data [,flags]) method */ +Send a data string to the socket. + +For the optional flags argument, see the Unix manual. +This calls send() repeatedly until all data is sent. +If an error occurs, it's impossible to tell how much data has been sent. +[clinic start generated code]*/ static PyObject * -sock_sendall(PyObject *self, PyObject *args) -{ - PySocketSockObject *s = _PySocketSockObject_CAST(self); +_socket_socket_sendall_impl(PySocketSockObject *s, Py_buffer *pbuf, + int flags) +/*[clinic end generated code: output=ec92861424d3faa8 input=732b15b9ca64dce6]*/ +{ char *buf; Py_ssize_t len, n; - int flags = 0; - Py_buffer pbuf; struct sock_send ctx; int has_timeout = (s->sock_timeout > 0); PyTime_t timeout = s->sock_timeout; @@ -4674,13 +4680,10 @@ sock_sendall(PyObject *self, PyObject *args) int deadline_initialized = 0; PyObject *res = NULL; - if (!PyArg_ParseTuple(args, "y*|i:sendall", &pbuf, &flags)) - return NULL; - buf = pbuf.buf; - len = pbuf.len; + buf = pbuf->buf; + len = pbuf->len; if (!IS_SELECTABLE(s)) { - PyBuffer_Release(&pbuf); return select_error(); } @@ -4718,23 +4721,13 @@ sock_sendall(PyObject *self, PyObject *args) if (PyErr_CheckSignals()) goto done; } while (len > 0); - PyBuffer_Release(&pbuf); res = Py_NewRef(Py_None); done: - PyBuffer_Release(&pbuf); return res; } -PyDoc_STRVAR(sendall_doc, -"sendall(data[, flags])\n\ -\n\ -Send a data string to the socket. For the optional flags\n\ -argument, see the Unix manual. This calls send() repeatedly\n\ -until all data is sent. If an error occurs, it's impossible\n\ -to tell how much data has been sent."); - #ifdef HAVE_SENDTO struct sock_sendto { @@ -4884,10 +4877,8 @@ sock_sendmsg_iovec(PySocketSockObject *s, PyObject *data_arg, } } for (; ndatabufs < ndataparts; ndatabufs++) { - if (!PyArg_Parse(PySequence_Fast_GET_ITEM(data_fast, ndatabufs), - "y*;sendmsg() argument 1 must be an iterable of " - "bytes-like objects", - &databufs[ndatabufs])) + if (PyObject_GetBuffer(PySequence_Fast_GET_ITEM(data_fast, ndatabufs), + &databufs[ndatabufs], PyBUF_SIMPLE) < 0) goto finally; iovs[ndatabufs].iov_base = databufs[ndatabufs].buf; iovs[ndatabufs].iov_len = databufs[ndatabufs].len; @@ -4909,13 +4900,39 @@ sock_sendmsg_impl(PySocketSockObject *s, void *data) return (ctx->result >= 0); } -/* s.sendmsg(buffers[, ancdata[, flags[, address]]]) method */ +/*[clinic input] +_socket.socket.sendmsg + self as s: self(type="PySocketSockObject *") + buffers as data_arg: object + ancdata as cmsg_arg: object = NULL + flags: int = 0 + address as addr_arg: object = NULL + / + +Send normal and ancillary data to the socket. + +It gathering the non-ancillary data from a series of buffers +and concatenating it into a single message. +The buffers argument specifies the non-ancillary +data as an iterable of bytes-like objects (e.g. bytes objects). +The ancdata argument specifies the ancillary data (control messages) +as an iterable of zero or more tuples (cmsg_level, cmsg_type, +cmsg_data), where cmsg_level and cmsg_type are integers specifying the +protocol level and protocol-specific type respectively, and cmsg_data +is a bytes-like object holding the associated data. The flags +argument defaults to 0 and has the same meaning as for send(). If +address is supplied and not None, it sets a destination address for +the message. The return value is the number of bytes of non-ancillary +data sent. +[clinic start generated code]*/ static PyObject * -sock_sendmsg(PyObject *self, PyObject *args) -{ - PySocketSockObject *s = _PySocketSockObject_CAST(self); +_socket_socket_sendmsg_impl(PySocketSockObject *s, PyObject *data_arg, + PyObject *cmsg_arg, int flags, + PyObject *addr_arg) +/*[clinic end generated code: output=3b4cb1110644ce39 input=479c13d90bd2f88b]*/ +{ Py_ssize_t i, ndatabufs = 0, ncmsgs, ncmsgbufs = 0; Py_buffer *databufs = NULL; sock_addr_t addrbuf; @@ -4927,16 +4944,10 @@ sock_sendmsg(PyObject *self, PyObject *args) } *cmsgs = NULL; void *controlbuf = NULL; size_t controllen, controllen_last; - int addrlen, flags = 0; - PyObject *data_arg, *cmsg_arg = NULL, *addr_arg = NULL, - *cmsg_fast = NULL, *retval = NULL; + int addrlen; + PyObject *cmsg_fast = NULL, *retval = NULL; struct sock_sendmsg ctx; - if (!PyArg_ParseTuple(args, "O|OiO:sendmsg", - &data_arg, &cmsg_arg, &flags, &addr_arg)) { - return NULL; - } - memset(&msg, 0, sizeof(msg)); /* Parse destination address. */ @@ -5098,22 +5109,6 @@ sock_sendmsg(PyObject *self, PyObject *args) return retval; } -PyDoc_STRVAR(sendmsg_doc, -"sendmsg(buffers[, ancdata[, flags[, address]]]) -> count\n\ -\n\ -Send normal and ancillary data to the socket, gathering the\n\ -non-ancillary data from a series of buffers and concatenating it into\n\ -a single message. The buffers argument specifies the non-ancillary\n\ -data as an iterable of bytes-like objects (e.g. bytes objects).\n\ -The ancdata argument specifies the ancillary data (control messages)\n\ -as an iterable of zero or more tuples (cmsg_level, cmsg_type,\n\ -cmsg_data), where cmsg_level and cmsg_type are integers specifying the\n\ -protocol level and protocol-specific type respectively, and cmsg_data\n\ -is a bytes-like object holding the associated data. The flags\n\ -argument defaults to 0 and has the same meaning as for send(). If\n\ -address is supplied and not None, it sets a destination address for\n\ -the message. The return value is the number of bytes of non-ancillary\n\ -data sent."); #endif /* CMSG_LEN */ #ifdef HAVE_SOCKADDR_ALG @@ -5450,8 +5445,8 @@ static PyMethodDef sock_methods[] = { recvfrom_into_doc }, #endif - {"send", sock_send, METH_VARARGS, send_doc}, - {"sendall", sock_sendall, METH_VARARGS, sendall_doc}, + _SOCKET_SOCKET_SEND_METHODDEF + _SOCKET_SOCKET_SENDALL_METHODDEF #ifdef HAVE_SENDTO {"sendto", sock_sendto, METH_VARARGS, sendto_doc}, #endif @@ -5471,7 +5466,7 @@ static PyMethodDef sock_methods[] = { #ifdef CMSG_LEN {"recvmsg", sock_recvmsg, METH_VARARGS, recvmsg_doc}, {"recvmsg_into", sock_recvmsg_into, METH_VARARGS, recvmsg_into_doc}, - {"sendmsg", sock_sendmsg, METH_VARARGS, sendmsg_doc}, + _SOCKET_SOCKET_SENDMSG_METHODDEF #endif #ifdef HAVE_SOCKADDR_ALG { @@ -5536,13 +5531,6 @@ sock_finalize(PyObject *self) PyErr_SetRaisedException(exc); } -static int -sock_traverse(PyObject *s, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(s)); - return 0; -} - static void sock_dealloc(PyObject *s) { @@ -5841,7 +5829,7 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, static PyType_Slot sock_slots[] = { {Py_tp_dealloc, sock_dealloc}, - {Py_tp_traverse, sock_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_repr, sock_repr}, {Py_tp_doc, (void *)sock_doc}, {Py_tp_methods, sock_methods}, @@ -6215,8 +6203,10 @@ socket_gethostbyname_ex(PyObject *self, PyObject *args) #ifdef USE_GETHOSTBYNAME_LOCK PyThread_acquire_lock(netdb_lock, 1); #endif - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS h = gethostbyname(name); + _Py_COMP_DIAG_POP #endif /* HAVE_GETHOSTBYNAME_R */ Py_END_ALLOW_THREADS /* Some C libraries would require addr.__ss_family instead of @@ -6320,8 +6310,10 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) #ifdef USE_GETHOSTBYNAME_LOCK PyThread_acquire_lock(netdb_lock, 1); #endif - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS h = gethostbyaddr(ap, al, af); + _Py_COMP_DIAG_POP #endif /* HAVE_GETHOSTBYNAME_R */ Py_END_ALLOW_THREADS ret = gethost_common(state, h, SAS2SA(&addr), sizeof(addr), af); @@ -6688,6 +6680,7 @@ _socket_htonl_impl(PyObject *module, uint32_t x) /* socket.inet_aton() and socket.inet_ntoa() functions. */ /*[clinic input] +@permit_long_summary _socket.inet_aton ip_addr: str / @@ -6697,7 +6690,7 @@ Convert an IP address in string format (123.45.67.89) to the 32-bit packed binar static PyObject * _socket_inet_aton_impl(PyObject *module, const char *ip_addr) -/*[clinic end generated code: output=f2c2f772eb721b6e input=3a52dec207bf8956]*/ +/*[clinic end generated code: output=f2c2f772eb721b6e input=0bd9e5ee400fafd6]*/ { #ifdef HAVE_INET_ATON struct in_addr buf; @@ -6738,8 +6731,10 @@ _socket_inet_aton_impl(PyObject *module, const char *ip_addr) packed_addr = INADDR_BROADCAST; } else { - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS packed_addr = inet_addr(ip_addr); + _Py_COMP_DIAG_POP if (packed_addr == INADDR_NONE) { /* invalid address */ PyErr_SetString(PyExc_OSError, @@ -6782,8 +6777,10 @@ _socket_inet_ntoa_impl(PyObject *module, Py_buffer *packed_ip) memcpy(&packed_addr, packed_ip->buf, packed_ip->len); PyBuffer_Release(packed_ip); - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS return PyUnicode_FromString(inet_ntoa(packed_addr)); + _Py_COMP_DIAG_POP } #endif // HAVE_INET_NTOA @@ -7186,7 +7183,7 @@ socket_setdefaulttimeout(PyObject *self, PyObject *arg) PyDoc_STRVAR(setdefaulttimeout_doc, "setdefaulttimeout(timeout)\n\ \n\ -Set the default timeout in seconds (float) for new socket objects.\n\ +Set the default timeout in seconds (real number) for new socket objects.\n\ A value of None indicates that new socket objects have no timeout.\n\ When the socket module is first imported, the default is None."); @@ -7281,7 +7278,7 @@ Returns a list of network interface information (index, name) tuples."); /*[clinic input] _socket.if_nametoindex - oname: object(converter="PyUnicode_FSConverter") + oname: unicode_fs_encoded / Returns the interface index corresponding to the interface name if_name. @@ -7289,7 +7286,7 @@ Returns the interface index corresponding to the interface name if_name. static PyObject * _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) -/*[clinic end generated code: output=289a411614f30244 input=01e0f1205307fb77]*/ +/*[clinic end generated code: output=289a411614f30244 input=6125dc20683560cf]*/ { #ifdef MS_WINDOWS NET_IFINDEX index; @@ -7297,11 +7294,10 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) unsigned long index; #endif + errno = ENODEV; // in case 'if_nametoindex' does not set errno index = if_nametoindex(PyBytes_AS_STRING(oname)); - Py_DECREF(oname); if (index == 0) { - /* if_nametoindex() doesn't set errno */ - PyErr_SetString(PyExc_OSError, "no interface with this name"); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -7321,6 +7317,7 @@ static PyObject * _socket_if_indextoname_impl(PyObject *module, NET_IFINDEX index) /*[clinic end generated code: output=e48bc324993052e0 input=c93f753d0cf6d7d1]*/ { + errno = ENXIO; // in case 'if_indextoname' does not set errno char name[IF_NAMESIZE + 1]; if (if_indextoname(index, name) == NULL) { PyErr_SetFromErrno(PyExc_OSError); @@ -8525,6 +8522,43 @@ socket_exec(PyObject *m) ADD_INT_MACRO(m, J1939_FILTER_MAX); #endif +#ifdef HAVE_LINUX_CAN_ISOTP_H + ADD_INT_MACRO(m, SOL_CAN_ISOTP); + + ADD_INT_MACRO(m, CAN_ISOTP_OPTS); + ADD_INT_MACRO(m, CAN_ISOTP_RECV_FC); + + ADD_INT_MACRO(m, CAN_ISOTP_TX_STMIN); + ADD_INT_MACRO(m, CAN_ISOTP_RX_STMIN); + ADD_INT_MACRO(m, CAN_ISOTP_LL_OPTS); + + ADD_INT_MACRO(m, CAN_ISOTP_LISTEN_MODE); + ADD_INT_MACRO(m, CAN_ISOTP_EXTEND_ADDR); + ADD_INT_MACRO(m, CAN_ISOTP_TX_PADDING); + ADD_INT_MACRO(m, CAN_ISOTP_RX_PADDING); + ADD_INT_MACRO(m, CAN_ISOTP_CHK_PAD_LEN); + ADD_INT_MACRO(m, CAN_ISOTP_CHK_PAD_DATA); + ADD_INT_MACRO(m, CAN_ISOTP_HALF_DUPLEX); + ADD_INT_MACRO(m, CAN_ISOTP_FORCE_TXSTMIN); + ADD_INT_MACRO(m, CAN_ISOTP_FORCE_RXSTMIN); + ADD_INT_MACRO(m, CAN_ISOTP_RX_EXT_ADDR); + ADD_INT_MACRO(m, CAN_ISOTP_WAIT_TX_DONE); +#ifdef CAN_ISOTP_SF_BROADCAST + ADD_INT_MACRO(m, CAN_ISOTP_SF_BROADCAST); +#endif + + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_FLAGS); + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_EXT_ADDRESS); + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_PAD_CONTENT); + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_FRAME_TXTIME); + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_RECV_BS); + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_RECV_STMIN); + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_RECV_WFTMAX); + + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_LL_MTU); + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_LL_TX_DL); + ADD_INT_MACRO(m, CAN_ISOTP_DEFAULT_LL_TX_FLAGS); +#endif #ifdef SOL_RDS ADD_INT_MACRO(m, SOL_RDS); #endif @@ -8867,6 +8901,9 @@ socket_exec(PyObject *m) #ifdef IPV6_HOPLIMIT ADD_INT_MACRO(m, IPV6_HOPLIMIT); #endif +#ifdef IPV6_HDRINCL + ADD_INT_MACRO(m, IPV6_HDRINCL); +#endif #ifdef IPV6_HOPOPTS ADD_INT_MACRO(m, IPV6_HOPOPTS); #endif diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 63624d511c3..ac770889ae8 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -18,6 +18,10 @@ */ #ifdef AF_BTH # include <ws2bth.h> +# ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpragma-pack" +# endif # include <pshpack1.h> /* @@ -51,7 +55,10 @@ struct SOCKADDR_BTH_REDEF { }; # include <poppack.h> -#endif +# ifdef __clang__ +# pragma clang diagnostic pop +# endif +#endif /* AF_BTH */ /* Windows 'supports' CMSG_LEN, but does not follow the POSIX standard * interface at all, so there is no point including the code that @@ -158,6 +165,10 @@ typedef int socklen_t; #include <linux/can/bcm.h> #endif +#ifdef HAVE_LINUX_CAN_ISOTP_H +#include <linux/can/isotp.h> +#endif + #ifdef HAVE_LINUX_CAN_J1939_H #include <linux/can/j1939.h> #endif @@ -259,7 +270,7 @@ typedef int SOCKET_T; #endif // AF_HYPERV is only supported on Windows -#if defined(AF_HYPERV) && defined(MS_WINDOWS) +#if defined(AF_HYPERV) && (defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) # define HAVE_AF_HYPERV #endif diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index d0d5223e5ac..a24927a9db6 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -13,17 +13,20 @@ module _symtable _symtable.symtable source: object - filename: object(converter='PyUnicode_FSDecoder') + filename: unicode_fs_decoded startstr: str / + * + module as modname: object = None Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr) -/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/ + PyObject *filename, const char *startstr, + PyObject *modname) +/*[clinic end generated code: output=235ec5a87a9ce178 input=fbf9adaa33c7070d]*/ { struct symtable *st; PyObject *t; @@ -47,12 +50,20 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, else { PyErr_SetString(PyExc_ValueError, "symtable() arg 3 must be 'exec' or 'eval' or 'single'"); - Py_DECREF(filename); Py_XDECREF(source_copy); return NULL; } - st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); - Py_DECREF(filename); + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "symtable() argument 'module' must be str or None, not %T", + modname); + Py_XDECREF(source_copy); + return NULL; + } + st = _Py_SymtableStringObjectFlags(str, filename, start, &cf, modname); Py_XDECREF(source_copy); if (st == NULL) { return NULL; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 9c54af51402..5d7fd20c4e0 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -56,7 +56,6 @@ Revision history: #include "Python.h" #include "osdefs.h" // SEP -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include <syslog.h> @@ -92,7 +91,7 @@ syslog_get_argv(void) Py_ssize_t slash; PyObject *argv; - if (_PySys_GetOptionalAttrString("argv", &argv) <= 0) { + if (PySys_GetOptionalAttrString("argv", &argv) <= 0) { return NULL; } @@ -299,7 +298,13 @@ syslog_setlogmask_impl(PyObject *module, long maskpri) return -1; } - return setlogmask(maskpri); + static PyMutex setlogmask_mutex = {0}; + PyMutex_Lock(&setlogmask_mutex); + // Linux man page (3): setlogmask() is MT-Unsafe race:LogMask. + long previous_mask = setlogmask(maskpri); + PyMutex_Unlock(&setlogmask_mutex); + + return previous_mask; } /*[clinic input] diff --git a/Modules/termios.c b/Modules/termios.c index efb5fcc17fa..b4eb06cf8ae 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -474,6 +474,7 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) } /*[clinic input] +@permit_long_docstring_body termios.tcsetwinsize fd: fildes @@ -488,7 +489,7 @@ is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize(). static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz) -/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=4a06424465b24aee]*/ +/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=9a163c4e06fc4a41]*/ { if (!PySequence_Check(winsz) || PySequence_Size(winsz) != 2) { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 1bfbf3f6a0b..3946d18479e 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -128,7 +128,7 @@ time_time_ns(PyObject *self, PyObject *unused) if (PyTime_Time(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(time_ns_doc, @@ -187,7 +187,7 @@ time_clockid_converter(PyObject *obj, clockid_t *p) { #ifdef _AIX long long clk_id = PyLong_AsLongLong(obj); -#elif defined(__DragonFly__) +#elif defined(__DragonFly__) || defined(__CYGWIN__) long clk_id = PyLong_AsLong(obj); #else int clk_id = PyLong_AsInt(obj); @@ -261,7 +261,7 @@ time_clock_gettime_ns_impl(PyObject *module, clockid_t clk_id) if (_PyTime_FromTimespec(&t, &ts) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } #endif /* HAVE_CLOCK_GETTIME */ @@ -310,7 +310,7 @@ time_clock_settime_ns(PyObject *self, PyObject *args) return NULL; } - if (_PyTime_FromLong(&t, obj) < 0) { + if (PyLong_AsInt64(obj, &t) < 0) { return NULL; } if (_PyTime_AsTimespec(t, &ts) == -1) { @@ -1216,7 +1216,7 @@ time_monotonic_ns(PyObject *self, PyObject *unused) if (PyTime_Monotonic(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(monotonic_ns_doc, @@ -1248,7 +1248,7 @@ time_perf_counter_ns(PyObject *self, PyObject *unused) if (PyTime_PerfCounter(&t) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(perf_counter_ns_doc, @@ -1437,7 +1437,7 @@ time_process_time_ns(PyObject *module, PyObject *unused) if (py_process_time(state, &t, NULL) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(process_time_ns_doc, @@ -1610,7 +1610,7 @@ time_thread_time_ns(PyObject *self, PyObject *unused) if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { return NULL; } - return _PyTime_AsLong(t); + return PyLong_FromInt64(t); } PyDoc_STRVAR(thread_time_ns_doc, diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index ef8cf3d0d27..a6094676d41 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -17,7 +17,9 @@ #endif #include "Python.h" +#include "pycore_object.h" // _PyObject_VisitType() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +#include "pycore_unicodectype.h" // _PyUnicode_IsXidStart() #include <stdbool.h> #include <stddef.h> // offsetof() @@ -305,6 +307,7 @@ unicodedata_UCD_bidirectional_impl(PyObject *self, int chr) } /*[clinic input] +@permit_long_summary unicodedata.UCD.combining -> int self: self @@ -318,7 +321,7 @@ Returns 0 if no combining class is defined. static int unicodedata_UCD_combining_impl(PyObject *self, int chr) -/*[clinic end generated code: output=cad056d0cb6a5920 input=9f2d6b2a95d0a22a]*/ +/*[clinic end generated code: output=cad056d0cb6a5920 input=e05edfbb882ebfed]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -389,6 +392,7 @@ unicodedata_UCD_east_asian_width_impl(PyObject *self, int chr) } /*[clinic input] +@permit_long_summary unicodedata.UCD.decomposition self: self @@ -402,7 +406,7 @@ An empty string is returned in case no such mapping is defined. static PyObject * unicodedata_UCD_decomposition_impl(PyObject *self, int chr) -/*[clinic end generated code: output=7d699f3ec7565d27 input=e4c12459ad68507b]*/ +/*[clinic end generated code: output=7d699f3ec7565d27 input=84d628d1abfd01ec]*/ { char decomp[256]; int code, index, count; @@ -1018,13 +1022,14 @@ is_unified_ideograph(Py_UCS4 code) (0x3400 <= code && code <= 0x4DBF) || /* CJK Ideograph Extension A */ (0x4E00 <= code && code <= 0x9FFF) || /* CJK Ideograph */ (0x20000 <= code && code <= 0x2A6DF) || /* CJK Ideograph Extension B */ - (0x2A700 <= code && code <= 0x2B739) || /* CJK Ideograph Extension C */ + (0x2A700 <= code && code <= 0x2B73F) || /* CJK Ideograph Extension C */ (0x2B740 <= code && code <= 0x2B81D) || /* CJK Ideograph Extension D */ - (0x2B820 <= code && code <= 0x2CEA1) || /* CJK Ideograph Extension E */ + (0x2B820 <= code && code <= 0x2CEAD) || /* CJK Ideograph Extension E */ (0x2CEB0 <= code && code <= 0x2EBE0) || /* CJK Ideograph Extension F */ (0x2EBF0 <= code && code <= 0x2EE5D) || /* CJK Ideograph Extension I */ (0x30000 <= code && code <= 0x3134A) || /* CJK Ideograph Extension G */ - (0x31350 <= code && code <= 0x323AF); /* CJK Ideograph Extension H */ + (0x31350 <= code && code <= 0x323AF) || /* CJK Ideograph Extension H */ + (0x323B0 <= code && code <= 0x33479); /* CJK Ideograph Extension J */ } /* macros used to determine if the given code point is in the PUA range that @@ -1521,6 +1526,58 @@ unicodedata_UCD_name_impl(PyObject *self, int chr, PyObject *default_value) return PyUnicode_FromString(name); } +/*[clinic input] +unicodedata.UCD.isxidstart + + self: self + chr: int(accept={str}) + / + +Return True if the character has the XID_Start property, else False. + +[clinic start generated code]*/ + +static PyObject * +unicodedata_UCD_isxidstart_impl(PyObject *self, int chr) +/*[clinic end generated code: output=944005823c72c3ef input=9353f88d709c21fb]*/ +{ + if (UCD_Check(self)) { + const change_record *old = get_old_record(self, chr); + if (old->category_changed == 0) { + /* unassigned */ + Py_RETURN_FALSE; + } + } + + return PyBool_FromLong(_PyUnicode_IsXidStart(chr)); +} + +/*[clinic input] +unicodedata.UCD.isxidcontinue + + self: self + chr: int(accept={str}) + / + +Return True if the character has the XID_Continue property, else False. + +[clinic start generated code]*/ + +static PyObject * +unicodedata_UCD_isxidcontinue_impl(PyObject *self, int chr) +/*[clinic end generated code: output=9438dcbff5ca3e41 input=bbb8dd3ac0d2d709]*/ +{ + if (UCD_Check(self)) { + const change_record *old = get_old_record(self, chr); + if (old->category_changed == 0) { + /* unassigned */ + Py_RETURN_FALSE; + } + } + + return PyBool_FromLong(_PyUnicode_IsXidContinue(chr)); +} + /*[clinic input] unicodedata.UCD.lookup @@ -1586,19 +1643,14 @@ static PyMethodDef unicodedata_functions[] = { UNICODEDATA_UCD_EAST_ASIAN_WIDTH_METHODDEF UNICODEDATA_UCD_DECOMPOSITION_METHODDEF UNICODEDATA_UCD_NAME_METHODDEF + UNICODEDATA_UCD_ISXIDSTART_METHODDEF + UNICODEDATA_UCD_ISXIDCONTINUE_METHODDEF UNICODEDATA_UCD_LOOKUP_METHODDEF UNICODEDATA_UCD_IS_NORMALIZED_METHODDEF UNICODEDATA_UCD_NORMALIZE_METHODDEF {NULL, NULL} /* sentinel */ }; -static int -ucd_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static void ucd_dealloc(PyObject *self) { @@ -1610,7 +1662,7 @@ ucd_dealloc(PyObject *self) static PyType_Slot ucd_type_slots[] = { {Py_tp_dealloc, ucd_dealloc}, - {Py_tp_traverse, ucd_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_methods, unicodedata_functions}, {Py_tp_members, DB_members}, diff --git a/Modules/unicodedata_db.h b/Modules/unicodedata_db.h index 610ee6b749c..eb0da8a6ff9 100644 --- a/Modules/unicodedata_db.h +++ b/Modules/unicodedata_db.h @@ -1,6 +1,6 @@ /* this file was generated by Tools/unicode/makeunicodedata.py 3.3 */ -#define UNIDATA_VERSION "16.0.0" +#define UNIDATA_VERSION "17.0.0" /* a list of unique database records */ const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = { {0, 0, 0, 0, 0, 0}, @@ -181,8 +181,8 @@ const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = { {8, 0, 1, 0, 0, 0}, {5, 9, 1, 0, 0, 0}, {14, 0, 15, 0, 0, 0}, - {4, 1, 14, 0, 0, 0}, {4, 234, 14, 0, 0, 0}, + {4, 1, 14, 0, 0, 0}, {4, 214, 14, 0, 0, 0}, {4, 202, 14, 0, 0, 0}, {4, 232, 14, 0, 0, 0}, @@ -783,7 +783,7 @@ static const unsigned short index1[] = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 208, 101, 101, 101, 101, 101, 101, 101, 101, 101, 209, 210, 137, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 208, 209, 210, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, @@ -796,9 +796,9 @@ static const unsigned short index1[] = { 216, 216, 216, 218, 219, 220, 78, 221, 222, 223, 224, 225, 226, 137, 227, 228, 229, 230, 231, 232, 233, 234, 78, 78, 78, 78, 235, 236, 137, 137, 137, 137, 137, 137, 137, 137, 237, 137, 238, 239, 240, 137, 137, 241, - 137, 137, 137, 242, 137, 243, 137, 137, 137, 244, 245, 246, 247, 137, - 137, 137, 137, 137, 248, 249, 250, 137, 251, 252, 137, 137, 253, 254, - 255, 256, 257, 137, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, + 137, 137, 137, 242, 137, 243, 137, 244, 137, 245, 246, 247, 248, 137, + 137, 137, 137, 137, 249, 250, 251, 137, 252, 253, 137, 137, 254, 255, + 256, 257, 258, 137, 259, 260, 261, 262, 263, 264, 265, 266, 216, 267, 268, 269, 270, 271, 272, 273, 216, 274, 137, 137, 137, 137, 137, 137, 137, 137, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, @@ -826,54 +826,54 @@ static const unsigned short index1[] = { 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 275, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 276, 101, 277, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 276, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 278, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 277, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 279, 101, 101, - 101, 101, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 121, 121, 121, 121, 282, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 283, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 278, 101, 101, + 101, 101, 279, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 121, 121, 121, 121, 281, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 282, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 284, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 283, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, - 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 285, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 283, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, 101, + 101, 101, 284, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 282, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, @@ -1239,7 +1239,7 @@ static const unsigned short index1[] = { 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 286, 137, 287, 288, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 285, 137, 286, 287, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, @@ -1312,7 +1312,7 @@ static const unsigned short index1[] = { 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 289, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, + 120, 288, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, @@ -1348,7 +1348,7 @@ static const unsigned short index1[] = { 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 120, 120, 120, 120, 289, + 120, 120, 120, 120, 120, 120, 120, 120, 120, 288, }; static const unsigned short index2[] = { @@ -1387,7 +1387,7 @@ static const unsigned short index2[] = { 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 41, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 48, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 48, 48, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 51, 51, 51, 51, 51, 51, 51, 51, 52, 52, 53, 53, 53, 53, 53, 53, 53, 54, 54, 55, 54, 52, 56, 52, 56, 56, 56, 52, 56, 52, 52, 57, 53, 54, 54, 54, @@ -1479,90 +1479,90 @@ static const unsigned short index2[] = { 107, 107, 107, 107, 107, 107, 86, 86, 86, 0, 0, 104, 0, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, - 118, 118, 118, 118, 118, 118, 118, 140, 118, 118, 118, 118, 118, 118, 0, - 108, 108, 0, 0, 0, 0, 0, 81, 81, 86, 86, 86, 81, 81, 81, 81, 118, 118, + 118, 118, 118, 118, 118, 118, 118, 140, 118, 118, 118, 118, 118, 118, + 118, 108, 108, 0, 0, 0, 0, 0, 81, 81, 86, 86, 86, 81, 81, 81, 81, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, - 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 120, 81, 81, 81, - 81, 81, 86, 86, 86, 86, 86, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 108, 86, 81, 81, 86, 81, 81, 86, 81, 81, 81, 86, 86, 86, 121, - 122, 123, 81, 81, 81, 86, 81, 81, 86, 86, 81, 81, 81, 81, 81, 135, 135, - 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 120, 81, 81, + 81, 81, 81, 86, 86, 86, 86, 86, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 108, 86, 81, 81, 86, 81, 81, 86, 81, 81, 81, 86, 86, 86, + 121, 122, 123, 81, 81, 81, 86, 81, 81, 86, 86, 81, 81, 81, 81, 81, 135, + 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 142, 48, 48, - 48, 48, 48, 135, 141, 143, 48, 141, 141, 141, 135, 135, 135, 135, 135, - 135, 135, 135, 141, 141, 141, 141, 144, 141, 141, 48, 81, 86, 81, 81, - 135, 135, 135, 145, 145, 145, 145, 145, 145, 145, 145, 48, 48, 135, 135, - 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 53, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, 141, 0, 48, - 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, - 48, 48, 48, 48, 48, 48, 0, 48, 0, 0, 0, 48, 48, 48, 48, 0, 0, 147, 48, - 148, 141, 141, 135, 135, 135, 135, 0, 0, 141, 141, 0, 0, 149, 149, 144, - 48, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 145, 145, 0, 145, 48, 48, - 135, 135, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, - 85, 85, 150, 150, 150, 150, 150, 150, 80, 85, 48, 83, 81, 0, 0, 135, 135, - 141, 0, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, 48, 0, 0, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 142, + 48, 48, 48, 48, 48, 135, 141, 143, 48, 141, 141, 141, 135, 135, 135, 135, + 135, 135, 135, 135, 141, 141, 141, 141, 144, 141, 141, 48, 81, 86, 81, + 81, 135, 135, 135, 145, 145, 145, 145, 145, 145, 145, 145, 48, 48, 135, + 135, 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 53, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, + 141, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 145, 0, 48, 145, 0, 48, 48, 0, 0, - 147, 0, 141, 141, 141, 135, 135, 0, 0, 0, 0, 135, 135, 0, 0, 135, 135, - 144, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 145, 145, 145, 48, 0, 145, 0, 0, - 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 135, - 135, 48, 48, 48, 135, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 141, 0, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, - 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 0, 0, 147, - 48, 141, 141, 141, 135, 135, 135, 135, 135, 0, 135, 135, 141, 0, 141, - 141, 144, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 135, 135, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 85, - 0, 0, 0, 0, 0, 0, 0, 48, 135, 135, 135, 135, 135, 135, 0, 135, 141, 141, - 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 0, 0, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 0, 0, - 147, 48, 148, 135, 141, 135, 135, 135, 135, 0, 0, 141, 149, 0, 0, 149, - 149, 144, 0, 0, 0, 0, 0, 0, 0, 135, 151, 148, 0, 0, 0, 0, 145, 145, 0, - 48, 48, 48, 135, 135, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 80, 48, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 135, 48, 0, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 0, 48, 48, 142, - 48, 0, 0, 0, 48, 48, 0, 48, 0, 48, 48, 0, 0, 0, 48, 48, 0, 0, 0, 48, 48, - 48, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, - 148, 141, 135, 141, 141, 0, 0, 0, 141, 141, 141, 0, 149, 149, 149, 144, - 0, 0, 48, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 150, 150, 150, 26, - 26, 26, 26, 26, 26, 85, 26, 0, 0, 0, 0, 0, 135, 141, 141, 141, 135, 48, - 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 147, - 48, 135, 135, 135, 141, 141, 141, 141, 0, 135, 135, 152, 0, 135, 135, - 135, 144, 0, 0, 0, 0, 0, 0, 0, 153, 154, 0, 48, 48, 48, 0, 0, 48, 0, 0, + 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 0, 0, 0, 48, 48, 48, 48, 0, 0, + 147, 48, 148, 141, 141, 135, 135, 135, 135, 0, 0, 141, 141, 0, 0, 149, + 149, 144, 48, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 145, 145, 0, 145, 48, 48, 135, 135, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 0, 0, 0, 0, 0, 0, 0, 83, 155, 155, 155, 155, 155, 155, 155, 80, 48, 135, - 141, 141, 83, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 0, 48, 48, + 48, 48, 85, 85, 150, 150, 150, 150, 150, 150, 80, 85, 48, 83, 81, 0, 0, + 135, 135, 141, 0, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, - 48, 0, 0, 147, 48, 141, 156, 149, 141, 148, 141, 141, 0, 156, 149, 149, - 0, 149, 149, 135, 144, 0, 0, 0, 0, 0, 0, 0, 148, 148, 0, 0, 0, 0, 0, 0, - 48, 48, 0, 48, 48, 135, 135, 0, 0, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 0, 48, 48, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, - 135, 141, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 0, 48, + 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 145, 0, 48, 145, 0, 48, + 48, 0, 0, 147, 0, 141, 141, 141, 135, 135, 0, 0, 0, 0, 135, 135, 0, 0, + 135, 135, 144, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 145, 145, 145, 48, 0, + 145, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 135, 135, 48, 48, 48, 135, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, + 135, 141, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 0, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, + 48, 0, 0, 147, 48, 141, 141, 141, 135, 135, 135, 135, 135, 0, 135, 135, + 141, 0, 141, 141, 144, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 48, 48, 135, 135, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 83, 85, 0, 0, 0, 0, 0, 0, 0, 48, 135, 135, 135, 135, 135, 135, + 0, 135, 141, 141, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, + 48, 48, 0, 0, 147, 48, 148, 135, 141, 135, 135, 135, 135, 0, 0, 141, 149, + 0, 0, 149, 149, 144, 0, 0, 0, 0, 0, 0, 0, 135, 151, 148, 0, 0, 0, 0, 145, + 145, 0, 48, 48, 48, 135, 135, 0, 0, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 80, 48, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 135, 48, 0, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 0, 48, + 48, 142, 48, 0, 0, 0, 48, 48, 0, 48, 0, 48, 48, 0, 0, 0, 48, 48, 0, 0, 0, + 48, 48, 48, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, + 0, 0, 0, 148, 141, 135, 141, 141, 0, 0, 0, 141, 141, 141, 0, 149, 149, + 149, 144, 0, 0, 48, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 150, 150, + 150, 26, 26, 26, 26, 26, 26, 85, 26, 0, 0, 0, 0, 0, 135, 141, 141, 141, + 135, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 0, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, + 0, 147, 48, 135, 135, 135, 141, 141, 141, 141, 0, 135, 135, 152, 0, 135, + 135, 135, 144, 0, 0, 0, 0, 0, 0, 0, 153, 154, 0, 48, 48, 48, 0, 48, 48, + 0, 0, 48, 48, 135, 135, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 0, 0, 0, 0, 0, 0, 0, 83, 155, 155, 155, 155, 155, 155, 155, 80, + 48, 135, 141, 141, 83, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, + 48, 48, 48, 0, 0, 147, 48, 141, 156, 149, 141, 148, 141, 141, 0, 156, + 149, 149, 0, 149, 149, 135, 144, 0, 0, 0, 0, 0, 0, 0, 148, 148, 0, 0, 0, + 0, 0, 48, 48, 48, 0, 48, 48, 135, 135, 0, 0, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 0, 48, 48, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 135, 135, 141, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 144, 144, 48, 148, 141, 141, 135, 135, 135, 135, + 0, 141, 141, 141, 0, 149, 149, 149, 144, 48, 80, 0, 0, 0, 0, 48, 48, 48, + 148, 150, 150, 150, 150, 150, 150, 150, 48, 48, 48, 135, 135, 0, 0, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 80, 48, 48, 48, 48, 48, 48, 0, 135, 141, 141, 0, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 157, 0, 0, 0, 0, 148, 141, + 141, 135, 135, 135, 0, 135, 0, 141, 141, 149, 141, 149, 149, 149, 148, 0, + 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, + 141, 141, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 144, 144, 48, 148, 141, 141, 135, 135, 135, 135, 0, 141, - 141, 141, 0, 149, 149, 149, 144, 48, 80, 0, 0, 0, 0, 48, 48, 48, 148, - 150, 150, 150, 150, 150, 150, 150, 48, 48, 48, 135, 135, 0, 0, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 80, 48, 48, 48, 48, 48, 48, 0, 135, 141, 141, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 0, - 0, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 157, 0, 0, 0, 0, 148, 141, 141, - 135, 135, 135, 0, 135, 0, 141, 141, 149, 141, 149, 149, 149, 148, 0, 0, - 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 141, - 141, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 135, 48, 158, 135, 135, 135, 135, 159, 159, 144, 0, - 0, 0, 0, 85, 48, 48, 48, 48, 48, 48, 53, 135, 160, 160, 160, 160, 135, + 48, 48, 48, 48, 48, 48, 135, 48, 158, 135, 135, 135, 135, 159, 159, 144, + 0, 0, 0, 0, 85, 48, 48, 48, 48, 48, 48, 53, 135, 160, 160, 160, 160, 135, 135, 135, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 0, 48, 0, 48, 48, 48, 48, @@ -1729,511 +1729,511 @@ static const unsigned short index2[] = { 146, 146, 146, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 83, 83, 83, 83, 83, 83, 83, 53, 83, 83, 83, 83, 83, 83, 0, 0, 81, 81, 81, 81, 81, 86, 86, 86, 86, 86, 86, 81, 81, 86, - 82, 86, 86, 81, 81, 86, 86, 81, 81, 81, 81, 81, 86, 81, 81, 81, 81, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, - 135, 135, 135, 141, 48, 142, 48, 142, 48, 142, 48, 142, 48, 142, 48, 48, - 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, - 148, 135, 135, 135, 135, 135, 149, 135, 149, 141, 141, 149, 149, 135, - 149, 176, 48, 48, 48, 48, 48, 48, 48, 48, 0, 83, 83, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 83, 83, 83, 83, 83, 83, 83, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 81, 86, 81, 81, 81, 81, 81, 81, 81, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 83, 83, 83, 135, 135, 141, 48, 48, 48, 48, 48, + 82, 86, 86, 81, 81, 86, 86, 81, 81, 81, 81, 81, 86, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 86, 0, 0, 81, 81, 81, + 81, 81, 81, 86, 81, 81, 81, 81, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 141, 48, 142, 48, 142, 48, + 142, 48, 142, 48, 142, 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 141, 135, 135, 135, 135, 141, 141, 135, 135, - 176, 144, 135, 135, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, 135, 135, 141, 141, 141, - 135, 141, 135, 135, 135, 176, 176, 0, 0, 0, 0, 0, 0, 0, 0, 83, 83, 83, - 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 141, 141, 141, 141, 141, 141, 141, 141, 135, 135, 135, 135, 135, 135, - 135, 135, 141, 141, 135, 147, 0, 0, 0, 83, 83, 83, 83, 83, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 48, 48, 48, 146, 146, 146, + 48, 48, 48, 48, 48, 48, 48, 147, 148, 135, 135, 135, 135, 135, 149, 135, + 149, 141, 141, 149, 149, 135, 149, 176, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 83, + 83, 83, 83, 83, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 81, 86, 81, 81, + 81, 81, 81, 81, 81, 80, 80, 80, 80, 80, 80, 80, 80, 80, 83, 83, 83, 135, + 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 135, 135, + 135, 135, 141, 141, 135, 135, 176, 144, 135, 135, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 53, 53, 53, 53, 53, 53, 83, 83, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 44, 47, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 44, 44, - 44, 83, 83, 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 81, - 83, 178, 86, 86, 86, 86, 86, 81, 81, 86, 86, 86, 86, 81, 141, 178, 178, - 178, 178, 178, 178, 178, 48, 48, 48, 48, 86, 48, 48, 48, 48, 48, 48, 81, - 48, 48, 141, 81, 81, 48, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 51, 51, 51, 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 53, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 81, 81, 86, 81, 81, 81, 81, 81, 81, 81, 86, 81, 81, 179, - 180, 86, 181, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 182, 88, 88, 86, 183, 81, 184, 86, 81, 86, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 43, 43, 43, 43, 35, 185, 47, 47, 44, 47, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, - 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 44, 47, 44, 47, 44, - 47, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, - 43, 43, 43, 43, 43, 0, 0, 38, 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, - 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, - 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, 0, 0, 38, - 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 0, 38, 0, 38, - 0, 38, 0, 38, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, - 38, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 43, 186, 0, 0, - 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, - 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, - 43, 43, 43, 43, 43, 43, 43, 43, 187, 187, 187, 187, 187, 187, 187, 187, - 43, 43, 43, 43, 43, 0, 43, 43, 38, 38, 38, 188, 187, 58, 186, 58, 58, 76, - 43, 43, 43, 0, 43, 43, 38, 188, 38, 188, 187, 76, 76, 76, 43, 43, 43, - 186, 0, 0, 43, 43, 38, 38, 38, 188, 0, 76, 76, 76, 43, 43, 43, 186, 43, - 43, 43, 43, 38, 38, 38, 188, 38, 76, 189, 189, 0, 0, 43, 43, 43, 0, 43, - 43, 38, 188, 38, 188, 187, 189, 58, 0, 190, 190, 191, 191, 191, 191, 191, - 191, 191, 191, 191, 177, 177, 177, 192, 193, 194, 195, 84, 194, 194, 194, - 22, 196, 197, 198, 199, 200, 197, 198, 199, 200, 22, 22, 22, 138, 201, - 201, 201, 22, 202, 203, 204, 205, 206, 207, 208, 21, 209, 110, 209, 210, - 211, 22, 196, 196, 138, 28, 36, 22, 196, 138, 201, 212, 212, 138, 138, - 138, 213, 165, 166, 196, 196, 196, 138, 138, 138, 138, 138, 138, 138, - 138, 78, 138, 212, 138, 138, 196, 138, 138, 138, 138, 138, 138, 138, 191, - 177, 177, 177, 177, 177, 0, 214, 215, 216, 217, 177, 177, 177, 177, 177, - 177, 218, 51, 0, 0, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, 222, - 223, 218, 34, 34, 34, 34, 218, 218, 218, 218, 218, 219, 219, 220, 221, - 222, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 85, - 85, 85, 85, 85, 85, 85, 85, 224, 225, 85, 85, 23, 85, 85, 85, 85, 85, 85, - 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 178, 178, 81, 81, 81, 81, 178, 178, - 178, 81, 81, 82, 82, 82, 82, 81, 82, 82, 82, 178, 178, 81, 86, 81, 178, - 178, 86, 86, 86, 86, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 226, 226, 49, 227, 26, 227, 226, 49, 26, 227, 35, 49, 49, 49, 35, 35, 49, - 49, 49, 46, 26, 49, 227, 26, 78, 49, 49, 49, 49, 49, 26, 26, 226, 227, - 227, 26, 49, 26, 228, 26, 49, 26, 188, 228, 49, 49, 229, 35, 49, 49, 44, - 49, 35, 158, 158, 158, 158, 35, 26, 226, 35, 35, 49, 49, 230, 78, 78, 78, - 78, 49, 35, 35, 35, 35, 26, 78, 26, 26, 47, 80, 231, 231, 231, 37, 37, - 231, 231, 231, 231, 231, 231, 37, 37, 37, 37, 231, 232, 232, 232, 232, - 232, 232, 232, 232, 232, 232, 232, 232, 233, 233, 233, 233, 232, 232, - 232, 232, 232, 232, 232, 232, 232, 232, 233, 233, 233, 233, 233, 233, - 175, 175, 175, 44, 47, 175, 175, 175, 175, 37, 26, 26, 0, 0, 0, 0, 40, - 40, 40, 40, 40, 30, 30, 30, 30, 30, 234, 234, 26, 26, 26, 26, 78, 26, 26, - 78, 26, 26, 78, 26, 26, 26, 26, 26, 26, 26, 234, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 235, 234, 234, 26, 26, 40, 26, 40, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 40, 236, 237, 237, 238, 78, 78, 40, 237, 238, 236, 237, - 238, 236, 78, 40, 78, 237, 239, 240, 78, 237, 236, 78, 78, 78, 237, 236, - 236, 237, 40, 237, 237, 236, 236, 40, 238, 40, 238, 40, 40, 40, 40, 237, - 241, 230, 237, 230, 230, 236, 236, 236, 40, 40, 40, 40, 78, 236, 78, 236, - 237, 237, 236, 236, 236, 238, 236, 236, 238, 236, 236, 238, 237, 238, - 236, 236, 237, 78, 78, 78, 78, 78, 237, 236, 236, 236, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 236, 242, 40, 238, 78, 237, 237, 237, 237, 236, 236, - 237, 237, 78, 238, 242, 242, 238, 238, 236, 236, 238, 238, 236, 236, 238, - 238, 236, 236, 236, 236, 236, 236, 238, 238, 237, 237, 238, 238, 237, - 237, 238, 238, 236, 236, 236, 78, 78, 236, 236, 236, 236, 78, 78, 40, 78, - 78, 236, 40, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 78, 40, 236, 236, - 236, 236, 236, 236, 238, 238, 238, 238, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 78, 78, 78, 78, 78, 236, 237, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 236, 236, 236, 236, 236, 78, 78, 236, 236, 78, 78, 78, 78, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 238, 238, 238, 238, 236, - 236, 236, 236, 236, 236, 238, 238, 238, 238, 78, 78, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 26, 26, 26, - 26, 26, 26, 26, 26, 165, 166, 165, 166, 26, 26, 26, 26, 26, 26, 30, 26, - 26, 26, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 236, 236, 26, 26, 26, - 26, 26, 26, 26, 244, 245, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 26, 78, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 80, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 78, 78, 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, 26, 243, 243, - 243, 243, 26, 26, 26, 243, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, - 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, - 37, 37, 37, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 231, 247, - 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, - 247, 247, 247, 247, 247, 247, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, 26, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, 30, 30, 30, 30, 26, 26, 26, - 26, 26, 26, 26, 26, 30, 30, 26, 26, 30, 40, 26, 26, 26, 26, 30, 30, 26, - 26, 30, 40, 26, 26, 26, 26, 30, 30, 30, 26, 26, 30, 26, 26, 30, 30, 30, - 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, - 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, - 26, 26, 26, 78, 78, 78, 78, 78, 248, 248, 78, 26, 26, 26, 26, 26, 30, 30, - 26, 26, 30, 26, 26, 26, 26, 30, 30, 26, 26, 26, 26, 243, 243, 26, 26, 26, - 26, 26, 26, 30, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, - 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, 26, 26, 26, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, 26, 30, 30, 30, 30, 26, 30, 30, - 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 26, - 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 243, 26, - 26, 26, 26, 26, 26, 26, 26, 243, 243, 80, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 30, 26, 26, 26, 26, 243, 243, - 30, 30, 30, 30, 30, 30, 30, 30, 243, 30, 30, 30, 30, 30, 243, 30, 30, 30, - 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 30, 26, 26, 26, 26, 30, 30, - 243, 30, 30, 30, 30, 30, 30, 30, 243, 243, 30, 243, 30, 30, 30, 30, 243, - 30, 30, 243, 30, 30, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 243, 243, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 243, 26, 26, 26, 26, - 243, 243, 243, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, - 166, 165, 166, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 155, - 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, - 155, 155, 155, 155, 155, 26, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 236, 78, 78, - 236, 236, 165, 166, 78, 236, 236, 78, 236, 236, 236, 78, 78, 78, 78, 78, - 236, 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, 236, - 236, 236, 236, 9, 10, 9, 10, 9, 10, 9, 10, 165, 166, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 165, 166, 9, 10, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, - 166, 165, 166, 165, 166, 165, 166, 78, 78, 236, 236, 236, 236, 236, 236, - 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 78, 78, 78, 78, 78, 78, 78, 78, 236, 78, 78, 78, 78, 78, 78, 78, 236, - 236, 236, 236, 236, 236, 78, 78, 78, 236, 78, 78, 78, 78, 236, 236, 236, - 236, 236, 78, 236, 236, 78, 78, 165, 166, 165, 166, 236, 78, 78, 78, 78, - 236, 78, 236, 236, 236, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 236, 236, 236, 236, 236, 236, 78, 78, 165, 166, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 230, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 236, 236, 236, - 236, 78, 78, 236, 78, 236, 78, 78, 236, 78, 236, 236, 236, 236, 78, 78, - 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, - 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, - 236, 78, 78, 78, 78, 236, 236, 236, 236, 78, 236, 236, 78, 78, 236, 230, - 220, 220, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, 78, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 78, 78, - 78, 78, 249, 78, 236, 78, 78, 78, 236, 236, 236, 236, 236, 78, 78, 78, - 78, 78, 236, 236, 236, 78, 78, 78, 78, 236, 78, 78, 78, 236, 236, 236, - 236, 236, 78, 236, 78, 78, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 26, 26, 78, 78, 78, 78, 78, 78, 26, 26, 26, 243, 26, 26, - 26, 26, 243, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 250, 26, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 44, 47, - 44, 44, 44, 47, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 47, 44, 47, - 47, 44, 47, 47, 47, 47, 47, 47, 51, 51, 44, 44, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, - 44, 47, 44, 47, 47, 26, 26, 26, 26, 26, 26, 44, 47, 44, 47, 81, 81, 81, - 44, 47, 0, 0, 0, 0, 0, 138, 138, 138, 138, 155, 138, 138, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 0, - 0, 0, 0, 0, 47, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 51, 83, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, - 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, - 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, - 0, 48, 48, 48, 48, 48, 48, 48, 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 138, 138, 28, 36, 28, 36, 138, 138, 138, 28, 36, 138, 28, - 36, 138, 138, 138, 138, 138, 138, 138, 138, 138, 84, 138, 138, 84, 138, - 28, 36, 138, 138, 28, 36, 165, 166, 165, 166, 165, 166, 165, 166, 138, - 138, 138, 138, 138, 52, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, - 84, 84, 138, 138, 138, 138, 84, 138, 199, 138, 138, 138, 138, 138, 138, - 138, 138, 138, 138, 138, 138, 138, 26, 26, 138, 138, 138, 165, 166, 165, - 166, 165, 166, 165, 166, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 243, 243, 243, 243, 251, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 251, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, - 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 252, 253, - 253, 253, 243, 254, 172, 255, 256, 257, 256, 257, 256, 257, 256, 257, - 256, 257, 243, 243, 256, 257, 256, 257, 256, 257, 256, 257, 258, 259, - 260, 260, 243, 255, 255, 255, 255, 255, 255, 255, 255, 255, 261, 262, - 263, 264, 265, 265, 258, 254, 254, 254, 254, 254, 251, 243, 266, 266, - 266, 254, 172, 253, 243, 26, 0, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, - 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 172, - 267, 172, 267, 172, 267, 172, 172, 172, 172, 172, 172, 267, 267, 172, - 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 267, 172, 172, 0, 0, 268, 268, 269, 269, 254, - 270, 271, 258, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, - 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 172, 267, 172, 267, - 172, 267, 172, 172, 172, 172, 172, 172, 267, 267, 172, 267, 267, 172, - 267, 267, 172, 267, 267, 172, 267, 267, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 267, 172, 172, 267, 267, 267, 267, 253, 254, 254, 270, 271, 0, - 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 0, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, - 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 0, 272, - 272, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 0, 273, 273, - 273, 273, 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 275, 275, 275, 275, - 275, 275, 275, 275, 251, 276, 276, 276, 276, 276, 276, 276, 276, 276, - 276, 276, 276, 276, 276, 276, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 251, 251, 251, 272, 273, 273, 273, 273, - 273, 273, 273, 273, 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, - 276, 276, 276, 276, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 251, 251, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 251, 251, 251, 251, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 251, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 254, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 53, 53, 53, 53, 53, 53, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 53, 138, 138, 138, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 48, 81, 82, 82, 82, 138, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 138, 52, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 51, 51, 81, 81, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 81, 81, 83, - 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 52, 52, - 52, 52, 52, 52, 52, 52, 52, 54, 54, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 47, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 51, 47, 47, - 47, 47, 47, 47, 47, 47, 44, 47, 44, 47, 44, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 52, 277, 277, 44, 47, 44, 47, 48, 44, 47, 44, 47, 47, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 44, 44, 44, 44, 47, 44, 44, 44, 44, 44, 47, 44, 47, 44, 47, 44, - 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 47, 44, 47, 44, 44, - 47, 0, 0, 44, 47, 0, 47, 0, 47, 44, 47, 44, 47, 44, 47, 44, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 51, 51, 44, 47, - 48, 51, 51, 47, 48, 48, 48, 48, 48, 48, 48, 135, 48, 48, 48, 144, 48, 48, - 48, 48, 135, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, 135, 141, 26, 26, 26, 26, - 144, 0, 0, 0, 150, 150, 150, 150, 150, 150, 80, 80, 85, 229, 0, 0, 0, 0, - 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 138, - 138, 138, 138, 0, 0, 0, 0, 0, 0, 0, 0, 141, 141, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, + 141, 135, 135, 141, 141, 141, 135, 141, 135, 135, 135, 176, 176, 0, 0, 0, + 0, 0, 0, 0, 0, 83, 83, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 141, 141, 141, 141, 141, - 141, 141, 141, 141, 141, 141, 141, 141, 144, 135, 0, 0, 0, 0, 0, 0, 0, 0, - 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, - 0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, - 81, 48, 48, 48, 48, 48, 48, 83, 83, 83, 48, 83, 48, 48, 135, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, + 135, 135, 135, 135, 135, 135, 135, 135, 141, 141, 135, 147, 0, 0, 0, 83, + 83, 83, 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, + 0, 48, 48, 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 135, 135, 135, 135, 135, 86, 86, 86, 83, 83, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 53, 53, 53, 53, 53, 83, 83, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 44, 47, 0, 0, 0, 0, 0, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 0, 0, 44, 44, 44, 83, 83, 83, 83, 83, 83, 83, 83, 0, 0, + 0, 0, 0, 0, 0, 0, 81, 81, 81, 83, 179, 86, 86, 86, 86, 86, 81, 81, 86, + 86, 86, 86, 81, 141, 179, 179, 179, 179, 179, 179, 179, 48, 48, 48, 48, + 86, 48, 48, 48, 48, 48, 48, 81, 48, 48, 141, 81, 81, 48, 0, 0, 0, 0, 0, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 51, 51, 51, 53, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 53, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 81, 81, 86, 81, 81, 81, + 81, 81, 81, 81, 86, 81, 81, 178, 180, 86, 181, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 182, 88, 88, 86, 183, + 81, 184, 86, 81, 86, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 43, 43, 43, 43, 35, 185, 47, + 47, 44, 47, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, 43, 38, + 43, 38, 43, 44, 47, 44, 47, 44, 47, 43, 43, 43, 43, 43, 43, 43, 43, 38, + 38, 38, 38, 38, 38, 38, 38, 43, 43, 43, 43, 43, 43, 0, 0, 38, 38, 38, 38, + 38, 38, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, + 38, 43, 43, 43, 43, 43, 43, 43, 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, + 43, 43, 43, 43, 43, 0, 0, 38, 38, 38, 38, 38, 38, 0, 0, 43, 43, 43, 43, + 43, 43, 43, 43, 0, 38, 0, 38, 0, 38, 0, 38, 43, 43, 43, 43, 43, 43, 43, + 43, 38, 38, 38, 38, 38, 38, 38, 38, 43, 186, 43, 186, 43, 186, 43, 186, + 43, 186, 43, 186, 43, 186, 0, 0, 43, 43, 43, 43, 43, 43, 43, 43, 187, + 187, 187, 187, 187, 187, 187, 187, 43, 43, 43, 43, 43, 43, 43, 43, 187, + 187, 187, 187, 187, 187, 187, 187, 43, 43, 43, 43, 43, 43, 43, 43, 187, + 187, 187, 187, 187, 187, 187, 187, 43, 43, 43, 43, 43, 0, 43, 43, 38, 38, + 38, 188, 187, 58, 186, 58, 58, 76, 43, 43, 43, 0, 43, 43, 38, 188, 38, + 188, 187, 76, 76, 76, 43, 43, 43, 186, 0, 0, 43, 43, 38, 38, 38, 188, 0, + 76, 76, 76, 43, 43, 43, 186, 43, 43, 43, 43, 38, 38, 38, 188, 38, 76, + 189, 189, 0, 0, 43, 43, 43, 0, 43, 43, 38, 188, 38, 188, 187, 189, 58, 0, + 190, 190, 191, 191, 191, 191, 191, 191, 191, 191, 191, 177, 177, 177, + 192, 193, 194, 195, 84, 194, 194, 194, 22, 196, 197, 198, 199, 200, 197, + 198, 199, 200, 22, 22, 22, 138, 201, 201, 201, 22, 202, 203, 204, 205, + 206, 207, 208, 21, 209, 110, 209, 210, 211, 22, 196, 196, 138, 28, 36, + 22, 196, 138, 201, 212, 212, 138, 138, 138, 213, 165, 166, 196, 196, 196, + 138, 138, 138, 138, 138, 138, 138, 138, 78, 138, 212, 138, 138, 196, 138, + 138, 138, 138, 138, 138, 138, 191, 177, 177, 177, 177, 177, 0, 214, 215, + 216, 217, 177, 177, 177, 177, 177, 177, 218, 51, 0, 0, 34, 218, 218, 218, + 218, 218, 219, 219, 220, 221, 222, 223, 218, 34, 34, 34, 34, 218, 218, + 218, 218, 218, 219, 219, 220, 221, 222, 0, 51, 51, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 0, 0, 0, 85, 85, 85, 85, 85, 85, 85, 85, 224, + 225, 85, 85, 23, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, + 85, 85, 85, 85, 85, 85, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, + 81, 179, 179, 81, 81, 81, 81, 179, 179, 179, 81, 81, 82, 82, 82, 82, 81, + 82, 82, 82, 179, 179, 81, 86, 81, 179, 179, 86, 86, 86, 86, 81, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 226, 49, 227, 26, 227, 226, 49, + 26, 227, 35, 49, 49, 49, 35, 35, 49, 49, 49, 46, 26, 49, 227, 26, 78, 49, + 49, 49, 49, 49, 26, 26, 226, 227, 227, 26, 49, 26, 228, 26, 49, 26, 188, + 228, 49, 49, 229, 35, 49, 49, 44, 49, 35, 158, 158, 158, 158, 35, 26, + 226, 35, 35, 49, 49, 230, 78, 78, 78, 78, 49, 35, 35, 35, 35, 26, 78, 26, + 26, 47, 80, 231, 231, 231, 37, 37, 231, 231, 231, 231, 231, 231, 37, 37, + 37, 37, 231, 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, + 233, 233, 233, 233, 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, + 233, 233, 233, 233, 233, 233, 175, 175, 175, 44, 47, 175, 175, 175, 175, + 37, 26, 26, 0, 0, 0, 0, 40, 40, 40, 40, 40, 30, 30, 30, 30, 30, 234, 234, + 26, 26, 26, 26, 78, 26, 26, 78, 26, 26, 78, 26, 26, 26, 26, 26, 26, 26, + 234, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 235, 234, 234, 26, + 26, 40, 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 40, 236, 237, 237, 238, 78, + 78, 40, 237, 238, 236, 237, 238, 236, 78, 40, 78, 237, 239, 240, 78, 237, + 236, 78, 78, 78, 237, 236, 236, 237, 40, 237, 237, 236, 236, 40, 238, 40, + 238, 40, 40, 40, 40, 237, 241, 230, 237, 230, 230, 236, 236, 236, 40, 40, + 40, 40, 78, 236, 78, 236, 237, 237, 236, 236, 236, 238, 236, 236, 238, + 236, 236, 238, 237, 238, 236, 236, 237, 78, 78, 78, 78, 78, 237, 236, + 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 242, 40, 238, 78, 237, + 237, 237, 237, 236, 236, 237, 237, 78, 238, 242, 242, 238, 238, 236, 236, + 238, 238, 236, 236, 238, 238, 236, 236, 236, 236, 236, 236, 238, 238, + 237, 237, 238, 238, 237, 237, 238, 238, 236, 236, 236, 78, 78, 236, 236, + 236, 236, 78, 78, 40, 78, 78, 236, 40, 78, 78, 78, 78, 78, 78, 78, 78, + 236, 236, 78, 40, 236, 236, 236, 236, 236, 236, 238, 238, 238, 238, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 78, 78, 78, 78, 78, 236, 237, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 236, 236, 236, 78, 78, 236, + 236, 78, 78, 78, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 238, 238, 238, 238, 236, 236, 236, 236, 236, 236, 238, 238, 238, 238, 78, + 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 26, 26, 26, 26, 26, 26, 26, 26, 165, 166, 165, 166, 26, 26, 26, + 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, + 236, 236, 26, 26, 26, 26, 26, 26, 26, 244, 245, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 26, 78, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 80, 26, 26, 26, 26, 26, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 26, 26, 26, + 26, 26, 26, 26, 243, 243, 243, 243, 26, 26, 26, 243, 26, 26, 243, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 37, 37, 37, 37, 37, 37, 37, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 231, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, + 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 26, + 26, 26, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, 26, 30, 30, 30, 30, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, 30, 30, + 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 26, 30, 40, 26, 26, + 26, 26, 30, 30, 26, 26, 30, 40, 26, 26, 26, 26, 30, 30, 30, 26, 26, 30, + 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 30, 30, 30, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, + 26, 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 248, 248, 78, 26, 26, + 26, 26, 26, 30, 30, 26, 26, 30, 26, 26, 26, 26, 30, 30, 26, 26, 26, 26, + 243, 243, 26, 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, + 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 30, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 30, 30, 26, 30, 30, 30, 26, 30, 30, + 30, 30, 26, 30, 30, 26, 40, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, + 243, 243, 243, 243, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 30, 30, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 80, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 30, + 26, 26, 26, 26, 243, 243, 30, 30, 30, 30, 30, 30, 30, 30, 243, 30, 30, + 30, 30, 30, 243, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 26, + 30, 26, 26, 26, 26, 30, 30, 243, 30, 30, 30, 30, 30, 30, 30, 243, 243, + 30, 243, 30, 30, 30, 30, 243, 30, 30, 243, 30, 30, 26, 26, 26, 26, 26, + 243, 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 26, 243, 26, 26, 26, 26, 243, 243, 243, 26, 243, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 165, 166, 165, 166, 165, 166, + 165, 166, 165, 166, 165, 166, 165, 166, 247, 247, 247, 247, 247, 247, + 247, 247, 247, 247, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 26, 243, 243, 243, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 243, 236, 78, 78, 236, 236, 165, 166, 78, 236, 236, 78, 236, 236, + 236, 78, 78, 78, 78, 78, 236, 236, 236, 236, 78, 78, 78, 78, 78, 236, + 236, 236, 78, 78, 78, 236, 236, 236, 236, 9, 10, 9, 10, 9, 10, 9, 10, + 165, 166, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 165, 166, 9, 10, 165, 166, 165, 166, 165, + 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 165, 166, 78, 78, + 236, 236, 236, 236, 236, 236, 78, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 236, 78, + 78, 78, 78, 78, 78, 78, 236, 236, 236, 236, 236, 236, 78, 78, 78, 236, + 78, 78, 78, 78, 236, 236, 236, 236, 236, 78, 236, 236, 78, 78, 165, 166, + 165, 166, 236, 78, 78, 78, 78, 236, 78, 236, 236, 236, 78, 78, 236, 236, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 236, 236, 236, 236, 78, + 78, 165, 166, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, + 230, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 78, 236, 236, 236, 236, 78, 78, 236, 78, 236, 78, 78, 236, + 78, 236, 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, + 78, 236, 236, 236, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 236, 236, 78, 78, 78, 78, 236, 236, 236, 236, 78, + 236, 236, 78, 78, 236, 230, 220, 220, 78, 78, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 78, 78, 236, 236, 236, + 236, 236, 236, 236, 236, 78, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, 236, + 236, 236, 236, 78, 78, 78, 78, 78, 249, 78, 236, 78, 78, 78, 236, 236, + 236, 236, 236, 78, 78, 78, 78, 78, 236, 236, 236, 78, 78, 78, 78, 236, + 78, 78, 78, 236, 236, 236, 236, 236, 78, 236, 78, 78, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 78, 78, 78, 78, 78, 78, 78, 26, 26, 78, 78, 78, 78, 78, 78, + 26, 26, 26, 243, 26, 26, 26, 26, 243, 30, 30, 30, 30, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 250, 26, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 44, 47, 44, 44, 44, 47, 47, 44, 47, 44, 47, 44, 47, 44, + 44, 44, 44, 47, 44, 47, 47, 44, 47, 47, 47, 47, 47, 47, 51, 51, 44, 44, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 47, 26, 26, 26, 26, 26, 26, 44, + 47, 44, 47, 81, 81, 81, 44, 47, 0, 0, 0, 0, 0, 138, 138, 138, 138, 155, + 138, 138, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 0, 47, 0, 0, 0, 0, 0, 47, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 141, 176, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 83, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, + 0, 51, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, + 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, + 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, + 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 138, 138, 28, 36, 28, 36, 138, + 138, 138, 28, 36, 138, 28, 36, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 84, 138, 138, 84, 138, 28, 36, 138, 138, 28, 36, 165, 166, 165, 166, + 165, 166, 165, 166, 138, 138, 138, 138, 138, 52, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 84, 84, 138, 138, 138, 138, 84, 138, 199, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 26, 26, 138, + 138, 138, 165, 166, 165, 166, 165, 166, 165, 166, 84, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, + 243, 243, 243, 243, 251, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, + 251, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 252, 253, 253, 253, 243, 254, 172, 255, 256, + 257, 256, 257, 256, 257, 256, 257, 256, 257, 243, 243, 256, 257, 256, + 257, 256, 257, 256, 257, 258, 259, 260, 260, 243, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 261, 262, 263, 264, 265, 265, 258, 254, 254, + 254, 254, 254, 251, 243, 266, 266, 266, 254, 172, 253, 243, 26, 0, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, 267, 172, + 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, + 267, 172, 267, 172, 267, 172, 172, 267, 172, 267, 172, 267, 172, 172, + 172, 172, 172, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, + 267, 172, 267, 267, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, + 172, 0, 0, 268, 268, 269, 269, 254, 270, 271, 258, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 267, 172, 267, 172, 267, 172, 267, + 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, 172, 267, + 172, 267, 172, 172, 267, 172, 267, 172, 267, 172, 172, 172, 172, 172, + 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, 267, 172, 267, + 267, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 267, 172, 172, 267, 267, + 267, 267, 253, 254, 254, 270, 271, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 0, 0, 0, 135, 135, 135, 141, 48, 48, 48, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 0, 272, 272, 273, 273, 273, 273, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 243, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 251, 251, 0, 273, 273, 273, 273, 273, 273, 273, 273, + 273, 273, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 275, 275, 275, 275, 275, 275, 275, 275, 251, 276, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 251, 251, 251, 272, 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 276, 276, 276, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 251, 251, 251, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, + 251, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 251, 251, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 251, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 254, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 53, 53, 53, 53, 53, + 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 138, 138, + 138, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 48, 81, 82, 82, 82, 138, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 138, 52, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 51, 51, 81, 81, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 147, 141, 141, 135, 135, 135, 135, 141, - 141, 135, 135, 141, 141, 176, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, - 83, 83, 0, 53, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, - 0, 83, 83, 48, 48, 48, 48, 48, 135, 53, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, - 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 141, 141, 135, 135, - 141, 141, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 135, 48, 48, - 48, 48, 48, 48, 48, 48, 135, 141, 0, 0, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 0, 0, 83, 83, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 53, 48, 48, 48, 48, 48, 48, 80, 80, 80, - 48, 141, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 175, 175, 175, + 175, 175, 175, 175, 175, 175, 175, 81, 81, 83, 83, 83, 83, 83, 83, 0, 0, + 0, 0, 0, 0, 0, 0, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, + 54, 54, 54, 54, 54, 54, 54, 54, 54, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 54, 54, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 47, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 51, 47, 47, 47, 47, 47, 47, 47, 47, 44, + 47, 44, 47, 44, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 52, 277, 277, 44, + 47, 44, 47, 48, 44, 47, 44, 47, 47, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 44, 44, 44, 44, 47, + 44, 44, 44, 44, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 44, 44, 44, 47, 44, 47, 44, 44, 47, 44, 47, 44, 47, 44, 47, + 44, 47, 44, 47, 44, 47, 44, 47, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 51, 51, 51, 51, 44, 47, 48, 51, 51, 47, 48, 48, + 48, 48, 48, 48, 48, 135, 48, 48, 48, 144, 48, 48, 48, 48, 135, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 141, 141, 135, 135, 141, 26, 26, 26, 26, 144, 0, 0, 0, 150, + 150, 150, 150, 150, 150, 80, 80, 85, 229, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 81, 48, 81, 81, 86, 48, 48, 81, 81, 48, 48, 48, 48, 48, 81, 81, 48, - 81, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 48, 48, 53, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 141, 135, 135, 141, 141, 83, 83, 48, 53, 53, 141, 144, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, - 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, - 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 47, 47, 47, 47, 47, 47, 47, 47, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 138, 138, 138, 138, + 0, 0, 0, 0, 0, 0, 0, 0, 141, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 144, 135, 0, 0, 0, 0, 0, 0, 0, 0, 83, 83, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 48, 48, + 48, 48, 48, 48, 83, 83, 83, 48, 83, 48, 48, 135, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, + 135, 135, 135, 86, 86, 86, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 141, 176, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 83, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 0, 0, 0, 135, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 147, 141, 141, 135, 135, 135, 135, 141, 141, 135, 135, + 141, 141, 176, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 0, 53, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 83, 83, 48, + 48, 48, 48, 48, 135, 53, 48, 48, 48, 48, 48, 48, 48, 48, 48, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 48, 48, 48, 48, 48, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 135, 135, 135, 135, 135, 135, 141, 141, 135, 135, 141, 141, + 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 135, 48, 48, 48, 48, 48, + 48, 48, 48, 135, 141, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 0, 0, 83, 83, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 53, 48, 48, 48, 48, 48, 48, 80, 80, 80, 48, 141, 135, + 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 48, 81, + 81, 86, 48, 48, 81, 81, 48, 48, 48, 48, 48, 81, 81, 48, 81, 48, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, + 53, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 135, 135, + 141, 141, 83, 83, 48, 53, 53, 141, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, + 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, 48, + 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, + 48, 48, 48, 48, 48, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 277, - 51, 51, 51, 51, 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 54, 54, 0, 0, 0, - 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 277, 51, 51, 51, 51, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 51, 54, 54, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 135, 141, 141, 135, 141, 141, - 83, 141, 144, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, - 0, 0, 0, 0, 0, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + 48, 48, 48, 48, 141, 141, 135, 141, 141, 135, 141, 141, 83, 141, 144, 0, + 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, @@ -2244,12 +2244,13 @@ static const unsigned short index2[] = { 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, - 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, 48, 48, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 278, 278, 278, 278, + 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, @@ -2258,8 +2259,7 @@ static const unsigned short index2[] = { 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, 279, 279, - 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, @@ -2268,6 +2268,7 @@ static const unsigned short index2[] = { 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, + 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, @@ -2277,39 +2278,39 @@ static const unsigned short index2[] = { 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 172, 172, 280, 172, 280, 172, 172, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 172, 280, 172, 280, 172, 172, 280, 280, 172, - 172, 172, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 172, + 172, 280, 172, 280, 172, 172, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 172, 280, 172, 280, 172, 172, 280, 280, 172, 172, 172, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 281, 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 281, 281, 281, 281, + 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, + 280, 280, 280, 280, 280, 280, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 35, 35, 35, 35, 35, 35, 35, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 35, 35, 35, 35, 35, 0, 0, 0, 0, 0, 282, 283, 282, - 284, 284, 284, 284, 284, 284, 284, 284, 284, 219, 282, 282, 282, 282, - 282, 282, 282, 282, 282, 282, 282, 282, 282, 0, 282, 282, 282, 282, 282, - 0, 282, 0, 282, 282, 0, 282, 282, 0, 282, 282, 282, 282, 282, 282, 282, - 282, 282, 284, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 281, 281, 281, 35, 35, 35, 35, 35, 35, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 35, 35, 35, 35, 35, 0, 0, 0, 0, 0, 282, 283, 282, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 219, 282, 282, 282, 282, 282, 282, 282, + 282, 282, 282, 282, 282, 282, 0, 282, 282, 282, 282, 282, 0, 282, 0, 282, + 282, 0, 282, 282, 0, 282, 282, 282, 282, 282, 282, 282, 282, 282, 284, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, 140, + 140, 140, 140, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, @@ -2331,21 +2332,21 @@ static const unsigned short index2[] = { 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 0, 0, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 26, 26, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, + 131, 131, 131, 131, 131, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 286, 26, 26, - 26, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 287, - 287, 287, 287, 287, 287, 287, 288, 289, 287, 0, 0, 0, 0, 0, 0, 81, 81, - 81, 81, 81, 81, 81, 86, 86, 86, 86, 86, 86, 86, 81, 81, 287, 290, 290, - 291, 291, 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, - 288, 289, 288, 289, 253, 253, 288, 289, 287, 287, 287, 287, 291, 291, - 291, 292, 287, 292, 0, 287, 292, 287, 287, 290, 293, 294, 293, 294, 293, - 294, 295, 287, 287, 296, 297, 298, 298, 299, 0, 287, 300, 295, 287, 0, 0, - 0, 0, 131, 131, 131, 118, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, + 0, 0, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 286, + 26, 26, 26, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 287, 287, 287, 287, 287, 287, 287, 288, 289, 287, 0, 0, 0, 0, 0, 0, + 81, 81, 81, 81, 81, 81, 81, 86, 86, 86, 86, 86, 86, 86, 81, 81, 287, 290, + 290, 291, 291, 288, 289, 288, 289, 288, 289, 288, 289, 288, 289, 288, + 289, 288, 289, 288, 289, 253, 253, 288, 289, 287, 287, 287, 287, 291, + 291, 291, 292, 287, 292, 0, 287, 292, 287, 287, 290, 293, 294, 293, 294, + 293, 294, 295, 287, 287, 296, 297, 298, 298, 299, 0, 287, 300, 295, 287, + 0, 0, 0, 0, 131, 131, 131, 118, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, @@ -2355,140 +2356,141 @@ static const unsigned short index2[] = { 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 0, 0, 177, 0, 301, 301, 302, 303, 302, 301, 301, 304, 305, 301, 306, - 307, 308, 307, 307, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, - 307, 301, 310, 311, 310, 301, 301, 312, 312, 312, 312, 312, 312, 312, + 131, 131, 131, 0, 0, 177, 0, 301, 301, 302, 303, 302, 301, 301, 304, 305, + 301, 306, 307, 308, 307, 307, 309, 309, 309, 309, 309, 309, 309, 309, + 309, 309, 307, 301, 310, 311, 310, 301, 301, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, 312, - 312, 312, 312, 312, 312, 304, 301, 305, 313, 314, 313, 315, 315, 315, + 312, 312, 312, 312, 312, 312, 312, 304, 301, 305, 313, 314, 313, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, - 315, 315, 315, 315, 315, 315, 315, 315, 315, 304, 311, 305, 311, 304, - 305, 316, 317, 318, 316, 316, 319, 319, 319, 319, 319, 319, 319, 319, - 319, 319, 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, + 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 315, 304, 311, 305, + 311, 304, 305, 316, 317, 318, 316, 316, 319, 319, 319, 319, 319, 319, + 319, 319, 319, 319, 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - 319, 319, 319, 319, 319, 319, 320, 320, 319, 319, 319, 319, 319, 319, + 319, 319, 319, 319, 319, 319, 319, 319, 320, 320, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 0, 0, 0, 319, 319, - 319, 319, 319, 319, 0, 0, 319, 319, 319, 319, 319, 319, 0, 0, 319, 319, - 319, 319, 319, 319, 0, 0, 319, 319, 319, 0, 0, 0, 303, 303, 311, 313, - 321, 303, 303, 0, 322, 323, 323, 323, 323, 322, 322, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 324, 324, 324, 26, 30, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, 0, 0, 0, + 319, 319, 319, 319, 319, 319, 0, 0, 319, 319, 319, 319, 319, 319, 0, 0, + 319, 319, 319, 319, 319, 319, 0, 0, 319, 319, 319, 0, 0, 0, 303, 303, + 311, 313, 321, 303, 303, 0, 322, 323, 323, 323, 323, 322, 322, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 324, 324, 324, 26, 30, 0, 0, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, + 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, - 0, 83, 138, 83, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 48, 0, 0, 0, 0, 0, 83, 138, 83, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, - 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 155, - 155, 155, 155, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 155, 155, 26, 80, 80, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, 325, + 325, 325, 155, 155, 155, 155, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 155, 155, 26, 80, 80, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 86, 0, 0, 0, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 86, 326, 326, 326, 326, 326, 326, 326, 326, 326, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, 326, - 326, 326, 326, 326, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 326, 326, 326, 326, 326, 326, 326, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 175, 48, 48, 48, 48, 48, 48, 48, 48, 175, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 175, 48, 48, 48, 48, 48, 48, 48, 48, 175, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 81, - 81, 81, 81, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 83, 175, 175, - 175, 175, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, + 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 83, + 175, 175, 175, 175, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 0, 0, 47, 47, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 0, + 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 0, 0, - 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 47, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, 44, 44, 0, 44, - 44, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, 47, 47, 47, 47, 0, - 47, 47, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, - 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 53, 51, 51, 51, 51, 51, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 0, 0, 0, 83, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 44, 44, 44, 44, 44, + 44, 44, 0, 44, 44, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 0, 47, 47, 47, + 47, 47, 47, 47, 0, 47, 47, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 53, 51, 51, 51, 51, 51, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, + 51, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 0, 0, 107, 0, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 107, - 107, 0, 0, 0, 107, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, - 104, 327, 327, 327, 327, 327, 327, 327, 327, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 328, 328, 327, 327, 327, 327, 327, 327, 327, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 327, 327, 327, 327, 327, 327, 327, 327, - 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 0, 0, 107, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 0, 107, 107, 0, 0, 0, 0, 0, 327, 327, 327, 327, - 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, 327, 327, 327, - 327, 0, 0, 0, 138, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 0, 107, 107, 0, 0, 0, 107, 0, 0, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 0, 104, 327, 327, 327, 327, 327, 327, 327, 327, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 328, 328, 327, 327, 327, 327, 327, + 327, 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 327, 327, 327, 327, 327, + 327, 327, 327, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 0, 107, 107, 0, 0, 0, 0, 0, 327, + 327, 327, 327, 327, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, + 327, 327, 327, 327, 0, 0, 0, 138, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 0, 0, 0, 0, 0, 104, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, @@ -2502,7 +2504,7 @@ static const unsigned short index2[] = { 135, 135, 0, 135, 135, 0, 0, 0, 0, 0, 135, 86, 135, 81, 107, 107, 107, 107, 0, 107, 107, 107, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 0, 0, 81, 178, 86, 0, 0, 0, 0, 144, 327, + 107, 107, 107, 107, 107, 107, 0, 0, 81, 179, 86, 0, 0, 0, 0, 144, 327, 327, 327, 327, 327, 327, 327, 327, 327, 0, 0, 0, 0, 0, 0, 0, 104, 104, 104, 104, 104, 104, 104, 104, 104, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, @@ -2570,97 +2572,97 @@ static const unsigned short index2[] = { 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 0, 81, 81, 102, 0, 0, 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 135, 86, 86, 86, 107, 107, 107, 107, 107, 107, 107, + 120, 118, 118, 0, 0, 0, 0, 0, 0, 0, 0, 138, 26, 26, 26, 26, 26, 26, 26, + 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 86, 135, 86, 86, 86, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, 327, 327, 327, 327, - 327, 327, 327, 327, 107, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 327, 327, + 327, 327, 327, 327, 327, 327, 327, 327, 107, 0, 0, 0, 0, 0, 0, 0, 0, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, - 118, 118, 118, 86, 86, 81, 81, 81, 86, 81, 86, 86, 86, 86, 333, 333, 333, - 333, 113, 113, 113, 113, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 81, 86, 81, 86, 104, 104, 104, - 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 118, 118, 118, 118, 118, 118, 118, 86, 86, 81, 81, 81, 86, 81, 86, 86, + 86, 86, 333, 333, 333, 333, 113, 113, 113, 113, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 81, 86, + 81, 86, 104, 104, 104, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 327, 327, 327, 327, 327, 327, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 327, 327, 327, 327, 327, 327, 327, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 135, 141, 48, 48, 48, 48, 48, 48, + 107, 107, 107, 107, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 144, 83, 83, 83, 83, 83, 83, 83, - 0, 0, 0, 0, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, - 155, 155, 155, 155, 155, 155, 155, 155, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 144, 48, 48, 135, 135, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 144, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 142, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 48, 48, 141, 141, 141, - 135, 135, 135, 135, 141, 141, 144, 143, 83, 83, 192, 83, 83, 83, 83, 135, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, - 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, - 0, 0, 0, 81, 81, 81, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 144, 83, 83, + 83, 83, 83, 83, 83, 0, 0, 0, 0, 155, 155, 155, 155, 155, 155, 155, 155, + 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 155, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 144, 48, 48, 135, 135, 48, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 144, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 142, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 142, 48, 48, 48, + 48, 141, 141, 141, 135, 135, 135, 135, 141, 141, 144, 143, 83, 83, 192, + 83, 83, 83, 83, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 151, 135, 135, 135, 135, 141, 135, 152, 152, 135, - 135, 135, 144, 144, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 83, 83, 83, 83, 48, 141, 141, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, + 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 0, 0, 0, 0, 0, 0, 81, 81, 81, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 147, 83, 83, 48, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 141, 176, 48, 48, 48, 48, 83, 83, 83, 83, 135, 147, 135, 135, 83, - 141, 135, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 48, 83, 48, - 83, 83, 83, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 151, 135, 135, 135, 135, 141, + 135, 152, 152, 135, 135, 135, 144, 144, 0, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 83, 83, 83, 83, 48, 141, 141, 48, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 141, - 141, 135, 176, 147, 135, 83, 83, 83, 83, 83, 83, 135, 48, 48, 135, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 0, - 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 147, 83, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, 141, 141, 135, 135, - 135, 135, 135, 135, 147, 144, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 135, 135, 141, 141, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 0, 0, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, - 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 0, 147, 147, 48, - 148, 141, 135, 141, 141, 141, 141, 0, 0, 141, 141, 0, 0, 149, 149, 176, - 0, 0, 48, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 141, - 141, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 0, 0, 81, 81, 81, 81, 81, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 142, 48, 142, 48, 48, 48, 48, 0, - 48, 0, 0, 142, 0, 48, 142, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 0, 48, 148, 141, 141, 151, 135, 135, 135, - 135, 135, 0, 148, 0, 0, 334, 0, 334, 334, 148, 141, 0, 141, 141, 144, - 176, 144, 48, 135, 48, 83, 83, 0, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 135, - 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 135, 135, 135, 135, 141, - 141, 144, 135, 135, 141, 147, 48, 48, 48, 48, 83, 83, 83, 83, 83, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 0, 83, 81, 48, 48, - 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 141, 176, 48, 48, 48, 48, 83, 83, 83, 83, 135, + 147, 135, 135, 83, 141, 135, 146, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 48, 83, 48, 83, 83, 83, 0, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, - 141, 141, 135, 135, 135, 135, 135, 135, 141, 151, 149, 149, 148, 149, - 135, 135, 141, 144, 147, 48, 48, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, + 135, 135, 141, 141, 135, 176, 147, 135, 83, 83, 83, 83, 83, 83, 135, 48, + 48, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 0, 48, 0, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 141, + 141, 141, 135, 135, 135, 135, 135, 135, 147, 144, 0, 0, 0, 0, 0, 146, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 135, 135, + 141, 141, 0, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 48, 48, 0, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, + 0, 147, 147, 48, 148, 141, 135, 141, 141, 141, 141, 0, 0, 141, 141, 0, 0, + 149, 149, 176, 0, 0, 48, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 0, 48, 48, + 48, 48, 48, 141, 141, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 0, 0, 81, 81, + 81, 81, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 142, 48, 142, + 48, 48, 48, 48, 0, 48, 0, 0, 142, 0, 48, 142, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 148, 141, 141, 151, + 135, 135, 135, 135, 135, 0, 148, 0, 0, 334, 0, 334, 334, 148, 141, 0, + 141, 141, 144, 176, 144, 48, 135, 48, 83, 83, 0, 83, 83, 0, 0, 0, 0, 0, + 0, 0, 0, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 141, 141, 141, 135, 135, 135, 135, 135, 135, + 135, 135, 141, 141, 144, 135, 135, 141, 147, 48, 48, 48, 48, 83, 83, 83, + 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 83, 83, 0, 83, + 81, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 148, 141, 141, 135, 135, 135, 135, 135, 135, 141, 151, 149, 149, + 148, 149, 135, 135, 141, 144, 147, 48, 48, 83, 48, 0, 0, 0, 0, 0, 0, 0, + 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 148, 141, 141, 135, 135, 135, 135, 0, 0, 141, 141, 149, 149, 135, 135, 141, 144, 147, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 48, 48, 48, 48, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2730,224 +2732,227 @@ static const unsigned short index2[] = { 83, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 83, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 141, 135, - 135, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, 135, 135, 141, 335, - 48, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 0, 0, 0, 83, 83, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 0, 141, 135, 135, 135, 135, 135, 135, 135, 141, 135, - 135, 141, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 135, 135, 135, 135, 135, 135, 0, 0, 0, 135, 0, 135, 135, 0, 135, 135, - 135, 147, 135, 144, 144, 48, 135, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, - 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 141, 141, 141, 141, 141, 0, 135, 135, 0, 141, 141, 135, 141, 144, 48, 0, - 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 141, 141, 83, 83, 0, 0, - 0, 0, 0, 0, 0, 135, 135, 48, 141, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 141, 141, 135, 135, 135, 135, 135, 0, 0, 0, 141, 141, 135, 176, - 144, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 146, 146, 146, - 146, 146, 146, 146, 146, 146, 146, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 26, 26, 26, 26, 26, 26, 26, 26, 85, 85, 85, 85, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, - 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 0, - 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 192, 192, 192, 192, 192, 192, 192, 192, 192, - 192, 192, 192, 192, 192, 192, 192, 135, 48, 48, 48, 48, 48, 48, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 151, 151, 151, 336, 336, 336, 336, 336, 336, 336, 336, 151, 141, 141, - 141, 135, 135, 144, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 83, - 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 0, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 0, 0, 178, 178, 178, 178, 178, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 81, 81, 81, 81, 81, - 81, 83, 83, 83, 83, 83, 80, 80, 80, 80, 53, 53, 53, 53, 83, 80, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, - 150, 150, 150, 150, 150, 150, 150, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 53, 53, 53, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 173, 337, 142, 142, 53, 53, 83, 83, 83, 146, - 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, - 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 83, 83, 83, 83, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, + 141, 135, 135, 135, 141, 135, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 83, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 141, 135, 135, 135, 135, 135, 135, 135, 0, 135, 135, 135, 135, + 135, 135, 141, 335, 48, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 0, 0, 0, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 0, 141, 135, 135, 135, 135, 135, + 135, 135, 141, 135, 135, 141, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, + 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 0, 0, 0, 135, 0, 135, + 135, 0, 135, 135, 135, 147, 135, 144, 144, 48, 135, 0, 0, 0, 0, 0, 0, 0, + 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 0, 0, 0, 0, 135, 48, 141, 141, 141, 141, 141, 141, - 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, - 141, 141, 141, 141, 141, 141, 141, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, - 135, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 254, 253, 254, 338, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 339, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, - 254, 254, 254, 0, 254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 0, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 48, + 48, 48, 48, 48, 48, 141, 141, 141, 141, 141, 0, 135, 135, 0, 141, 141, + 135, 141, 144, 48, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 53, 48, 48, 0, 0, + 0, 0, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 135, 135, 141, 141, + 83, 83, 0, 0, 0, 0, 0, 0, 0, 135, 135, 48, 141, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 141, 141, 135, 135, 135, 135, 135, 0, 0, 0, 141, + 141, 135, 176, 144, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, + 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 135, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 26, 26, 26, 26, 26, 26, 26, 26, 85, + 85, 85, 85, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 175, 175, 175, 175, 175, 175, 175, + 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, + 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, + 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, + 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, + 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, + 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, + 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, 175, + 175, 175, 175, 175, 175, 0, 83, 83, 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, - 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 0, 0, 80, 135, 178, 83, 177, 177, 177, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, + 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 135, 48, 48, + 48, 48, 48, 48, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 151, 151, 151, 336, 336, 336, 336, 336, 336, + 336, 336, 151, 141, 141, 141, 135, 135, 144, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 146, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 0, 0, 0, 0, 83, 83, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 0, 0, 179, 179, 179, 179, 179, 83, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, + 81, 81, 81, 81, 81, 81, 83, 83, 83, 83, 83, 80, 80, 80, 80, 53, 53, 53, + 53, 83, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 150, 150, 150, 150, 150, 150, 150, 0, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 53, 53, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 173, 337, 142, 142, + 53, 53, 83, 83, 83, 146, 146, 146, 146, 146, 146, 146, 146, 146, 146, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, + 150, 83, 83, 83, 83, 0, 0, 0, 0, 0, 44, 44, 44, 44, 44, 44, 44, 44, 44, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 0, 0, 47, + 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 135, 48, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 0, 0, 0, 0, 0, 0, 0, 135, 135, + 135, 135, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 254, 253, 254, 338, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 339, 339, 254, 254, 255, 255, 255, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 254, 254, 254, 0, 254, + 254, 254, 254, 254, 254, 254, 0, 254, 254, 0, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 172, 172, 172, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 172, 172, 172, 172, 0, 0, 0, 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 0, 0, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 0, 0, 80, 135, 179, 83, 177, 177, 177, 177, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, @@ -2959,91 +2964,97 @@ static const unsigned short index2[] = { 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 340, 340, 340, 340, 340, 340, + 26, 26, 26, 26, 26, 26, 26, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, - 340, 340, 340, 340, 340, 340, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 340, 340, 340, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 26, 26, + 26, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 135, 135, 135, + 26, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, + 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 80, 80, 80, 80, 80, 80, 80, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 0, 0, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 342, 342, - 342, 342, 342, 342, 342, 343, 343, 178, 178, 178, 80, 80, 80, 344, 343, - 343, 343, 343, 343, 177, 177, 177, 177, 177, 177, 177, 177, 86, 86, 86, - 86, 86, 86, 86, 86, 80, 80, 81, 81, 81, 81, 81, 86, 86, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 81, 81, 81, 81, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 342, 342, 342, 342, 342, 342, 80, 80, 80, 80, + 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 342, 342, 342, 342, 342, 342, 342, 343, 343, 179, 179, 179, 80, 80, 80, + 344, 343, 343, 343, 343, 343, 177, 177, 177, 177, 177, 177, 177, 177, 86, + 86, 86, 86, 86, 86, 86, 86, 80, 80, 81, 81, 81, 81, 81, 86, 86, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 81, 81, 81, 81, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 342, 342, 342, 342, 342, 342, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 81, 81, 81, 26, 0, 0, 0, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 81, 81, 81, 26, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, + 150, 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, - 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, + 150, 150, 150, 150, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, - 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 150, - 150, 0, 0, 0, 0, 0, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 345, 345, 345, 345, 345, 345, 345, + 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, + 345, 150, 150, 0, 0, 0, 0, 0, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, - 35, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, + 35, 35, 35, 35, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 49, 0, 49, 49, 0, 0, 49, 0, 0, 49, 49, 0, 0, 49, 49, 49, 49, 0, 49, 49, - 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 0, 35, 0, 35, 35, 35, 35, 35, 35, - 35, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, + 35, 35, 35, 49, 0, 49, 49, 0, 0, 49, 0, 0, 49, 49, 0, 0, 49, 49, 49, 49, + 0, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 0, 35, 0, 35, 35, 35, + 35, 35, 35, 35, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, + 49, 49, 49, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, + 49, 49, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, - 0, 49, 49, 49, 49, 49, 49, 49, 49, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, + 49, 49, 49, 49, 49, 0, 49, 0, 0, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 49, 49, 0, 49, 49, 49, 49, 0, 49, 49, 49, 49, - 49, 0, 49, 0, 0, 0, 49, 49, 49, 49, 49, 49, 49, 0, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, @@ -3056,16 +3067,7 @@ static const unsigned short index2[] = { 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 0, 0, 49, 49, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 0, 0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, @@ -3078,192 +3080,272 @@ static const unsigned short index2[] = { 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 35, 0, 0, + 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, 49, 49, 49, 49, + 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + 49, 49, 49, 220, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 230, 35, 35, 35, 35, 35, 35, + 49, 35, 0, 0, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 341, 341, 341, 341, 341, 135, 135, 135, 135, 135, 135, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 80, 80, 80, 80, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 80, 80, 80, 80, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 80, 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 83, 83, 83, 83, 83, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, 135, 135, 0, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, - 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 135, 135, 135, 135, 135, 135, 80, 80, 80, 80, 80, 80, 80, 80, 135, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 135, 80, 80, 83, 83, + 83, 83, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 135, 135, + 135, 135, 0, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 135, 135, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 48, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 0, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 48, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, + 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 47, 47, 47, 47, 47, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 81, 81, 81, 81, 81, 81, 81, - 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 0, 0, 81, 81, 81, 81, 81, 81, 81, - 0, 81, 81, 0, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, 51, 51, 51, 51, 51, 51, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 81, 81, 81, 81, 81, 81, 0, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 0, 0, 81, 81, 81, 81, + 81, 81, 81, 0, 81, 81, 0, 81, 81, 81, 81, 81, 0, 0, 0, 0, 0, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 51, 51, 51, 51, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 81, 81, 81, 81, - 81, 81, 81, 53, 53, 53, 53, 53, 53, 53, 0, 0, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 146, 0, 0, 0, 0, 48, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, 0, 0, 81, + 81, 81, 81, 81, 81, 81, 53, 53, 53, 53, 53, 53, 53, 0, 0, 146, 146, 146, + 146, 146, 146, 146, 146, 146, 146, 0, 0, 0, 0, 48, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 81, 81, 81, 81, 146, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 48, 48, 48, 48, 81, 81, 81, 81, 146, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 53, 182, 182, 86, 81, 146, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 81, 86, 48, 146, 146, 146, 146, 146, + 146, 146, 146, 146, 146, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 0, + 48, 48, 48, 81, 48, 48, 81, 48, 48, 48, 48, 48, 48, 48, 81, 81, 48, 48, + 48, 48, 48, 81, 0, 0, 0, 0, 0, 0, 0, 0, 48, 53, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 53, 182, 182, 86, 81, 146, 146, 146, 146, 146, 146, 146, 146, - 146, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 48, 0, 48, 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, + 48, 48, 48, 48, 48, 48, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 107, 107, 107, 107, 0, 0, 327, 327, 327, 327, 327, 327, 327, + 327, 327, 86, 86, 86, 86, 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 48, 48, 48, 48, 81, 86, 48, 146, 146, 146, 146, 146, 146, - 146, 146, 146, 146, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 48, 48, 48, 48, 48, 48, 0, 48, - 48, 48, 48, 0, 48, 48, 0, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, - 48, 48, 48, 0, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 0, 0, 327, 327, 327, 327, 327, 327, 327, 327, 327, - 86, 86, 86, 86, 86, 86, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, + 0, 0, 0, 0, 0, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, - 329, 329, 329, 329, 329, 329, 329, 330, 330, 330, 330, 330, 330, 330, + 329, 329, 329, 329, 329, 329, 329, 329, 329, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, - 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 81, 81, - 81, 81, 81, 81, 147, 137, 0, 0, 0, 0, 136, 136, 136, 136, 136, 136, 136, - 136, 136, 136, 0, 0, 0, 0, 104, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + 330, 81, 81, 81, 81, 81, 81, 147, 137, 0, 0, 0, 0, 136, 136, 136, 136, + 136, 136, 136, 136, 136, 136, 0, 0, 0, 0, 104, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 333, 333, 333, 333, 333, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, - 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 133, 333, 333, 333, - 111, 333, 333, 333, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 133, + 333, 333, 333, 111, 333, 333, 333, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 333, 333, 333, 333, 333, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, - 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 133, 333, 333, 333, - 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 0, 0, 0, 0, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 133, + 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, 333, + 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 131, 131, 131, 0, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, - 0, 131, 0, 0, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 0, 131, 131, 131, 131, 0, 131, 0, 131, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, - 131, 0, 131, 0, 131, 0, 131, 131, 131, 0, 131, 131, 0, 131, 0, 0, 131, 0, - 131, 0, 131, 0, 131, 0, 131, 0, 131, 131, 0, 131, 0, 0, 131, 131, 131, - 131, 0, 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, 131, 131, 0, 131, - 131, 131, 131, 0, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 0, 0, 0, 0, 0, 131, 131, 131, 0, 131, 131, 131, 131, - 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, - 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 0, + 131, 131, 0, 131, 0, 0, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 0, 131, 131, 131, 131, 0, 131, 0, 131, 0, 0, 0, 0, 0, 0, 131, + 0, 0, 0, 0, 131, 0, 131, 0, 131, 0, 131, 131, 131, 0, 131, 131, 0, 131, + 0, 0, 131, 0, 131, 0, 131, 0, 131, 0, 131, 0, 131, 131, 0, 131, 0, 0, + 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, 0, 131, 131, + 131, 131, 0, 131, 131, 131, 131, 0, 131, 0, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 131, 131, 131, 0, 131, + 131, 131, 131, 131, 0, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 78, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 243, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 155, 155, 26, 26, 26, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 340, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 155, 155, 26, 26, 26, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, - 246, 246, 246, 246, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 246, 246, 340, 26, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, 246, + 246, 246, 246, 246, 246, 246, 246, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, - 346, 346, 226, 226, 226, 26, 26, 26, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 226, 226, 226, 26, 26, 26, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, - 346, 346, 346, 346, 346, 346, 346, 346, 346, 272, 346, 246, 272, 272, - 272, 272, 272, 272, 272, 272, 272, 272, 346, 346, 346, 346, 346, 346, - 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 26, 0, 0, 0, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 272, 346, + 246, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, - 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 274, 274, 274, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 274, 274, 274, 274, 274, 274, 274, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, 274, 274, + 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 274, 274, 274, 0, 0, 0, 0, 274, 274, 274, 274, - 274, 274, 274, 274, 274, 0, 0, 0, 0, 0, 0, 0, 274, 274, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, 0, 0, 0, 0, 274, 274, + 274, 274, 274, 274, 274, 274, 274, 0, 0, 0, 0, 0, 0, 0, 274, 274, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, + 26, 26, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 26, 26, 26, 243, 26, 26, 26, 243, 243, 243, 347, 347, + 347, 347, 347, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, - 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 26, 26, 26, 243, 26, 26, 26, 243, 243, 243, 347, 347, 347, - 347, 347, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 26, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 243, 26, 26, + 26, 243, 243, 243, 26, 26, 243, 243, 243, 243, 0, 0, 0, 243, 243, 243, + 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 0, 0, 0, 26, + 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, + 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, + 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 78, 78, 78, 78, 78, + 78, 78, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, @@ -3276,128 +3358,41 @@ static const unsigned short index2[] = { 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 26, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, + 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 243, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 243, 26, 26, 26, 243, 243, - 243, 26, 26, 243, 243, 243, 0, 0, 0, 0, 243, 243, 243, 243, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 243, 243, 0, 0, 0, 26, 26, 26, 26, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 26, 26, 26, 26, 26, 26, + 243, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, + 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, + 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, - 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, - 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 26, 26, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 26, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 243, 243, 0, 0, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 243, 243, 243, 243, - 243, 243, 243, 243, 243, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 341, 341, - 341, 341, 341, 341, 341, 341, 341, 341, 0, 0, 0, 0, 0, 0, 172, 172, 172, + 26, 26, 26, 26, 26, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 26, + 0, 0, 0, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, + 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 172, + 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -3406,7 +3401,21 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, @@ -3415,21 +3424,7 @@ static const unsigned short index2[] = { 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, @@ -3440,9 +3435,10 @@ static const unsigned short index2[] = { 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 280, 280, 280, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, - 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, 281, + 280, 280, 280, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, @@ -3458,34 +3454,33 @@ static const unsigned short index2[] = { 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 0, 0, + 281, 281, 281, 0, 0, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, - 172, 172, 172, 172, 172, 172, 172, 172, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, - 281, 281, 281, 281, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 177, 177, 177, 177, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, 172, + 172, 172, 281, 281, 281, 281, 281, 281, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, - 177, 177, 177, 177, 177, 177, 177, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 177, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, @@ -3498,8 +3493,8 @@ static const unsigned short index2[] = { 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 279, 279, 279, 279, 279, 279, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, @@ -3508,7 +3503,7 @@ static const unsigned short index2[] = { 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, - 279, 279, 279, 279, 279, 279, 279, 0, 0, + 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, 0, 0, }; /* decomposition data */ @@ -3992,85 +3987,85 @@ static const unsigned int decomp_data[] = { 50, 52, 26085, 770, 50, 53, 26085, 770, 50, 54, 26085, 770, 50, 55, 26085, 770, 50, 56, 26085, 770, 50, 57, 26085, 770, 51, 48, 26085, 770, 51, 49, 26085, 778, 103, 97, 108, 259, 1098, 259, 1100, 259, 42863, 259, - 67, 259, 70, 259, 81, 259, 294, 259, 339, 259, 42791, 259, 43831, 259, - 619, 259, 43858, 259, 653, 256, 35912, 256, 26356, 256, 36554, 256, - 36040, 256, 28369, 256, 20018, 256, 21477, 256, 40860, 256, 22865, 256, - 37329, 256, 21895, 256, 22856, 256, 25078, 256, 30313, 256, 32645, 256, - 34367, 256, 34746, 256, 35064, 256, 37007, 256, 27138, 256, 27931, 256, - 28889, 256, 29662, 256, 33853, 256, 37226, 256, 39409, 256, 20098, 256, - 21365, 256, 27396, 256, 29211, 256, 34349, 256, 40478, 256, 23888, 256, - 28651, 256, 34253, 256, 35172, 256, 25289, 256, 33240, 256, 34847, 256, - 24266, 256, 26391, 256, 28010, 256, 29436, 256, 37070, 256, 20358, 256, - 20919, 256, 21214, 256, 25796, 256, 27347, 256, 29200, 256, 30439, 256, - 32769, 256, 34310, 256, 34396, 256, 36335, 256, 38706, 256, 39791, 256, - 40442, 256, 30860, 256, 31103, 256, 32160, 256, 33737, 256, 37636, 256, - 40575, 256, 35542, 256, 22751, 256, 24324, 256, 31840, 256, 32894, 256, - 29282, 256, 30922, 256, 36034, 256, 38647, 256, 22744, 256, 23650, 256, - 27155, 256, 28122, 256, 28431, 256, 32047, 256, 32311, 256, 38475, 256, - 21202, 256, 32907, 256, 20956, 256, 20940, 256, 31260, 256, 32190, 256, - 33777, 256, 38517, 256, 35712, 256, 25295, 256, 35582, 256, 20025, 256, - 23527, 256, 24594, 256, 29575, 256, 30064, 256, 21271, 256, 30971, 256, - 20415, 256, 24489, 256, 19981, 256, 27852, 256, 25976, 256, 32034, 256, - 21443, 256, 22622, 256, 30465, 256, 33865, 256, 35498, 256, 27578, 256, - 36784, 256, 27784, 256, 25342, 256, 33509, 256, 25504, 256, 30053, 256, - 20142, 256, 20841, 256, 20937, 256, 26753, 256, 31975, 256, 33391, 256, - 35538, 256, 37327, 256, 21237, 256, 21570, 256, 22899, 256, 24300, 256, - 26053, 256, 28670, 256, 31018, 256, 38317, 256, 39530, 256, 40599, 256, - 40654, 256, 21147, 256, 26310, 256, 27511, 256, 36706, 256, 24180, 256, - 24976, 256, 25088, 256, 25754, 256, 28451, 256, 29001, 256, 29833, 256, - 31178, 256, 32244, 256, 32879, 256, 36646, 256, 34030, 256, 36899, 256, - 37706, 256, 21015, 256, 21155, 256, 21693, 256, 28872, 256, 35010, 256, - 24265, 256, 24565, 256, 25467, 256, 27566, 256, 31806, 256, 29557, 256, - 20196, 256, 22265, 256, 23994, 256, 24604, 256, 29618, 256, 29801, 256, - 32666, 256, 32838, 256, 37428, 256, 38646, 256, 38728, 256, 38936, 256, - 20363, 256, 31150, 256, 37300, 256, 38584, 256, 24801, 256, 20102, 256, - 20698, 256, 23534, 256, 23615, 256, 26009, 256, 29134, 256, 30274, 256, - 34044, 256, 36988, 256, 40845, 256, 26248, 256, 38446, 256, 21129, 256, - 26491, 256, 26611, 256, 27969, 256, 28316, 256, 29705, 256, 30041, 256, - 30827, 256, 32016, 256, 39006, 256, 20845, 256, 25134, 256, 38520, 256, - 20523, 256, 23833, 256, 28138, 256, 36650, 256, 24459, 256, 24900, 256, - 26647, 256, 38534, 256, 21033, 256, 21519, 256, 23653, 256, 26131, 256, - 26446, 256, 26792, 256, 27877, 256, 29702, 256, 30178, 256, 32633, 256, - 35023, 256, 35041, 256, 37324, 256, 38626, 256, 21311, 256, 28346, 256, - 21533, 256, 29136, 256, 29848, 256, 34298, 256, 38563, 256, 40023, 256, - 40607, 256, 26519, 256, 28107, 256, 33256, 256, 31435, 256, 31520, 256, - 31890, 256, 29376, 256, 28825, 256, 35672, 256, 20160, 256, 33590, 256, - 21050, 256, 20999, 256, 24230, 256, 25299, 256, 31958, 256, 23429, 256, - 27934, 256, 26292, 256, 36667, 256, 34892, 256, 38477, 256, 35211, 256, - 24275, 256, 20800, 256, 21952, 256, 22618, 256, 26228, 256, 20958, 256, - 29482, 256, 30410, 256, 31036, 256, 31070, 256, 31077, 256, 31119, 256, - 38742, 256, 31934, 256, 32701, 256, 34322, 256, 35576, 256, 36920, 256, - 37117, 256, 39151, 256, 39164, 256, 39208, 256, 40372, 256, 37086, 256, - 38583, 256, 20398, 256, 20711, 256, 20813, 256, 21193, 256, 21220, 256, - 21329, 256, 21917, 256, 22022, 256, 22120, 256, 22592, 256, 22696, 256, - 23652, 256, 23662, 256, 24724, 256, 24936, 256, 24974, 256, 25074, 256, - 25935, 256, 26082, 256, 26257, 256, 26757, 256, 28023, 256, 28186, 256, - 28450, 256, 29038, 256, 29227, 256, 29730, 256, 30865, 256, 31038, 256, - 31049, 256, 31048, 256, 31056, 256, 31062, 256, 31069, 256, 31117, 256, - 31118, 256, 31296, 256, 31361, 256, 31680, 256, 32265, 256, 32321, 256, - 32626, 256, 32773, 256, 33261, 256, 33401, 256, 33879, 256, 35088, 256, - 35222, 256, 35585, 256, 35641, 256, 36051, 256, 36104, 256, 36790, 256, - 38627, 256, 38911, 256, 38971, 256, 24693, 256, 148206, 256, 33304, 256, - 20006, 256, 20917, 256, 20840, 256, 20352, 256, 20805, 256, 20864, 256, - 21191, 256, 21242, 256, 21845, 256, 21913, 256, 21986, 256, 22707, 256, - 22852, 256, 22868, 256, 23138, 256, 23336, 256, 24274, 256, 24281, 256, - 24425, 256, 24493, 256, 24792, 256, 24910, 256, 24840, 256, 24928, 256, - 25140, 256, 25540, 256, 25628, 256, 25682, 256, 25942, 256, 26395, 256, - 26454, 256, 27513, 256, 28379, 256, 28363, 256, 28702, 256, 30631, 256, - 29237, 256, 29359, 256, 29809, 256, 29958, 256, 30011, 256, 30237, 256, - 30239, 256, 30427, 256, 30452, 256, 30538, 256, 30528, 256, 30924, 256, - 31409, 256, 31867, 256, 32091, 256, 32574, 256, 33618, 256, 33775, 256, - 34681, 256, 35137, 256, 35206, 256, 35519, 256, 35531, 256, 35565, 256, - 35722, 256, 36664, 256, 36978, 256, 37273, 256, 37494, 256, 38524, 256, - 38875, 256, 38923, 256, 39698, 256, 141386, 256, 141380, 256, 144341, - 256, 15261, 256, 16408, 256, 16441, 256, 152137, 256, 154832, 256, - 163539, 256, 40771, 256, 40846, 514, 102, 102, 514, 102, 105, 514, 102, - 108, 770, 102, 102, 105, 770, 102, 102, 108, 514, 383, 116, 514, 115, - 116, 514, 1396, 1398, 514, 1396, 1381, 514, 1396, 1387, 514, 1406, 1398, - 514, 1396, 1389, 512, 1497, 1460, 512, 1522, 1463, 262, 1506, 262, 1488, - 262, 1491, 262, 1492, 262, 1499, 262, 1500, 262, 1501, 262, 1512, 262, - 1514, 262, 43, 512, 1513, 1473, 512, 1513, 1474, 512, 64329, 1473, 512, - 64329, 1474, 512, 1488, 1463, 512, 1488, 1464, 512, 1488, 1468, 512, + 83, 259, 67, 259, 70, 259, 81, 259, 294, 259, 339, 259, 42791, 259, + 43831, 259, 619, 259, 43858, 259, 653, 256, 35912, 256, 26356, 256, + 36554, 256, 36040, 256, 28369, 256, 20018, 256, 21477, 256, 40860, 256, + 22865, 256, 37329, 256, 21895, 256, 22856, 256, 25078, 256, 30313, 256, + 32645, 256, 34367, 256, 34746, 256, 35064, 256, 37007, 256, 27138, 256, + 27931, 256, 28889, 256, 29662, 256, 33853, 256, 37226, 256, 39409, 256, + 20098, 256, 21365, 256, 27396, 256, 29211, 256, 34349, 256, 40478, 256, + 23888, 256, 28651, 256, 34253, 256, 35172, 256, 25289, 256, 33240, 256, + 34847, 256, 24266, 256, 26391, 256, 28010, 256, 29436, 256, 37070, 256, + 20358, 256, 20919, 256, 21214, 256, 25796, 256, 27347, 256, 29200, 256, + 30439, 256, 32769, 256, 34310, 256, 34396, 256, 36335, 256, 38706, 256, + 39791, 256, 40442, 256, 30860, 256, 31103, 256, 32160, 256, 33737, 256, + 37636, 256, 40575, 256, 35542, 256, 22751, 256, 24324, 256, 31840, 256, + 32894, 256, 29282, 256, 30922, 256, 36034, 256, 38647, 256, 22744, 256, + 23650, 256, 27155, 256, 28122, 256, 28431, 256, 32047, 256, 32311, 256, + 38475, 256, 21202, 256, 32907, 256, 20956, 256, 20940, 256, 31260, 256, + 32190, 256, 33777, 256, 38517, 256, 35712, 256, 25295, 256, 35582, 256, + 20025, 256, 23527, 256, 24594, 256, 29575, 256, 30064, 256, 21271, 256, + 30971, 256, 20415, 256, 24489, 256, 19981, 256, 27852, 256, 25976, 256, + 32034, 256, 21443, 256, 22622, 256, 30465, 256, 33865, 256, 35498, 256, + 27578, 256, 36784, 256, 27784, 256, 25342, 256, 33509, 256, 25504, 256, + 30053, 256, 20142, 256, 20841, 256, 20937, 256, 26753, 256, 31975, 256, + 33391, 256, 35538, 256, 37327, 256, 21237, 256, 21570, 256, 22899, 256, + 24300, 256, 26053, 256, 28670, 256, 31018, 256, 38317, 256, 39530, 256, + 40599, 256, 40654, 256, 21147, 256, 26310, 256, 27511, 256, 36706, 256, + 24180, 256, 24976, 256, 25088, 256, 25754, 256, 28451, 256, 29001, 256, + 29833, 256, 31178, 256, 32244, 256, 32879, 256, 36646, 256, 34030, 256, + 36899, 256, 37706, 256, 21015, 256, 21155, 256, 21693, 256, 28872, 256, + 35010, 256, 24265, 256, 24565, 256, 25467, 256, 27566, 256, 31806, 256, + 29557, 256, 20196, 256, 22265, 256, 23994, 256, 24604, 256, 29618, 256, + 29801, 256, 32666, 256, 32838, 256, 37428, 256, 38646, 256, 38728, 256, + 38936, 256, 20363, 256, 31150, 256, 37300, 256, 38584, 256, 24801, 256, + 20102, 256, 20698, 256, 23534, 256, 23615, 256, 26009, 256, 29134, 256, + 30274, 256, 34044, 256, 36988, 256, 40845, 256, 26248, 256, 38446, 256, + 21129, 256, 26491, 256, 26611, 256, 27969, 256, 28316, 256, 29705, 256, + 30041, 256, 30827, 256, 32016, 256, 39006, 256, 20845, 256, 25134, 256, + 38520, 256, 20523, 256, 23833, 256, 28138, 256, 36650, 256, 24459, 256, + 24900, 256, 26647, 256, 38534, 256, 21033, 256, 21519, 256, 23653, 256, + 26131, 256, 26446, 256, 26792, 256, 27877, 256, 29702, 256, 30178, 256, + 32633, 256, 35023, 256, 35041, 256, 37324, 256, 38626, 256, 21311, 256, + 28346, 256, 21533, 256, 29136, 256, 29848, 256, 34298, 256, 38563, 256, + 40023, 256, 40607, 256, 26519, 256, 28107, 256, 33256, 256, 31435, 256, + 31520, 256, 31890, 256, 29376, 256, 28825, 256, 35672, 256, 20160, 256, + 33590, 256, 21050, 256, 20999, 256, 24230, 256, 25299, 256, 31958, 256, + 23429, 256, 27934, 256, 26292, 256, 36667, 256, 34892, 256, 38477, 256, + 35211, 256, 24275, 256, 20800, 256, 21952, 256, 22618, 256, 26228, 256, + 20958, 256, 29482, 256, 30410, 256, 31036, 256, 31070, 256, 31077, 256, + 31119, 256, 38742, 256, 31934, 256, 32701, 256, 34322, 256, 35576, 256, + 36920, 256, 37117, 256, 39151, 256, 39164, 256, 39208, 256, 40372, 256, + 37086, 256, 38583, 256, 20398, 256, 20711, 256, 20813, 256, 21193, 256, + 21220, 256, 21329, 256, 21917, 256, 22022, 256, 22120, 256, 22592, 256, + 22696, 256, 23652, 256, 23662, 256, 24724, 256, 24936, 256, 24974, 256, + 25074, 256, 25935, 256, 26082, 256, 26257, 256, 26757, 256, 28023, 256, + 28186, 256, 28450, 256, 29038, 256, 29227, 256, 29730, 256, 30865, 256, + 31038, 256, 31049, 256, 31048, 256, 31056, 256, 31062, 256, 31069, 256, + 31117, 256, 31118, 256, 31296, 256, 31361, 256, 31680, 256, 32265, 256, + 32321, 256, 32626, 256, 32773, 256, 33261, 256, 33401, 256, 33879, 256, + 35088, 256, 35222, 256, 35585, 256, 35641, 256, 36051, 256, 36104, 256, + 36790, 256, 38627, 256, 38911, 256, 38971, 256, 24693, 256, 148206, 256, + 33304, 256, 20006, 256, 20917, 256, 20840, 256, 20352, 256, 20805, 256, + 20864, 256, 21191, 256, 21242, 256, 21845, 256, 21913, 256, 21986, 256, + 22707, 256, 22852, 256, 22868, 256, 23138, 256, 23336, 256, 24274, 256, + 24281, 256, 24425, 256, 24493, 256, 24792, 256, 24910, 256, 24840, 256, + 24928, 256, 25140, 256, 25540, 256, 25628, 256, 25682, 256, 25942, 256, + 26395, 256, 26454, 256, 27513, 256, 28379, 256, 28363, 256, 28702, 256, + 30631, 256, 29237, 256, 29359, 256, 29809, 256, 29958, 256, 30011, 256, + 30237, 256, 30239, 256, 30427, 256, 30452, 256, 30538, 256, 30528, 256, + 30924, 256, 31409, 256, 31867, 256, 32091, 256, 32574, 256, 33618, 256, + 33775, 256, 34681, 256, 35137, 256, 35206, 256, 35519, 256, 35531, 256, + 35565, 256, 35722, 256, 36664, 256, 36978, 256, 37273, 256, 37494, 256, + 38524, 256, 38875, 256, 38923, 256, 39698, 256, 141386, 256, 141380, 256, + 144341, 256, 15261, 256, 16408, 256, 16441, 256, 152137, 256, 154832, + 256, 163539, 256, 40771, 256, 40846, 514, 102, 102, 514, 102, 105, 514, + 102, 108, 770, 102, 102, 105, 770, 102, 102, 108, 514, 383, 116, 514, + 115, 116, 514, 1396, 1398, 514, 1396, 1381, 514, 1396, 1387, 514, 1406, + 1398, 514, 1396, 1389, 512, 1497, 1460, 512, 1522, 1463, 262, 1506, 262, + 1488, 262, 1491, 262, 1492, 262, 1499, 262, 1500, 262, 1501, 262, 1512, + 262, 1514, 262, 43, 512, 1513, 1473, 512, 1513, 1474, 512, 64329, 1473, + 512, 64329, 1474, 512, 1488, 1463, 512, 1488, 1464, 512, 1488, 1468, 512, 1489, 1468, 512, 1490, 1468, 512, 1491, 1468, 512, 1492, 1468, 512, 1493, 1468, 512, 1494, 1468, 512, 1496, 1468, 512, 1497, 1468, 512, 1498, 1468, 512, 1499, 1468, 512, 1500, 1468, 512, 1502, 1468, 512, 1504, 1468, 512, @@ -5234,210 +5229,210 @@ static const unsigned short decomp_index2[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6603, 6605, - 6607, 0, 0, 0, 6609, 6611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6603, 6605, 6607, + 6609, 0, 0, 0, 6611, 6613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6613, 6615, 6617, 6619, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 6621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 6623, 6625, 6627, 6629, 6631, 6633, 6635, 6637, 6637, 6639, - 6641, 6643, 6645, 6647, 6649, 6651, 6653, 6655, 6657, 6659, 6661, 6663, - 6665, 6667, 6669, 6671, 6673, 6675, 6677, 6679, 6681, 6683, 6685, 6687, - 6689, 6691, 6693, 6695, 6697, 6699, 6701, 6703, 6705, 6707, 6709, 6711, - 6713, 6715, 6717, 6719, 6721, 6723, 6725, 6727, 6729, 6731, 6733, 6735, - 6737, 6739, 6741, 6743, 6745, 6747, 6749, 6751, 6753, 6755, 6757, 6759, - 6761, 6763, 6765, 6767, 6769, 6771, 6773, 6775, 6777, 6779, 6781, 6783, - 6785, 6787, 6789, 6791, 6793, 6795, 6797, 6799, 6801, 6803, 6661, 6805, - 6807, 6809, 6811, 6813, 6815, 6817, 6819, 6821, 6823, 6825, 6827, 6829, - 6831, 6833, 6835, 6837, 6839, 6841, 6843, 6845, 6847, 6849, 6851, 6853, - 6855, 6857, 6859, 6861, 6863, 6865, 6867, 6869, 6871, 6873, 6875, 6877, - 6879, 6881, 6883, 6885, 6887, 6889, 6891, 6893, 6895, 6897, 6899, 6901, - 6903, 6905, 6907, 6909, 6911, 6913, 6915, 6917, 6919, 6921, 6923, 6925, - 6927, 6929, 6931, 6933, 6935, 6937, 6939, 6841, 6941, 6943, 6945, 6947, - 6949, 6951, 6953, 6955, 6809, 6957, 6959, 6961, 6963, 6965, 6967, 6969, - 6971, 6973, 6975, 6977, 6979, 6981, 6983, 6985, 6987, 6989, 6991, 6993, - 6995, 6661, 6997, 6999, 7001, 7003, 7005, 7007, 7009, 7011, 7013, 7015, - 7017, 7019, 7021, 7023, 7025, 7027, 7029, 7031, 7033, 7035, 7037, 7039, - 7041, 7043, 7045, 7047, 7049, 6813, 7051, 7053, 7055, 7057, 7059, 7061, - 7063, 7065, 7067, 7069, 7071, 7073, 7075, 7077, 7079, 7081, 7083, 7085, - 7087, 7089, 7091, 7093, 7095, 7097, 7099, 7101, 7103, 7105, 7107, 7109, - 7111, 7113, 7115, 7117, 7119, 7121, 7123, 7125, 7127, 7129, 7131, 7133, - 7135, 7137, 7139, 7141, 7143, 7145, 7147, 7149, 0, 0, 7151, 0, 7153, 0, - 0, 7155, 7157, 7159, 7161, 7163, 7165, 7167, 7169, 7171, 7173, 0, 7175, - 0, 7177, 0, 0, 7179, 7181, 0, 0, 0, 7183, 7185, 7187, 7189, 7191, 7193, - 7195, 7197, 7199, 7201, 7203, 7205, 7207, 7209, 7211, 7213, 7215, 7217, - 7219, 7221, 7223, 7225, 7227, 7229, 7231, 7233, 7235, 7237, 7239, 7241, - 7243, 7245, 7247, 7249, 7251, 7253, 7255, 7257, 7259, 7261, 7263, 7265, - 7267, 7269, 7271, 6919, 7273, 7275, 7277, 7279, 7281, 7283, 7283, 7285, - 7287, 7289, 7291, 7293, 7295, 7297, 7299, 7179, 7301, 7303, 7305, 7307, - 7309, 7311, 0, 0, 7313, 7315, 7317, 7319, 7321, 7323, 7325, 7327, 7207, - 7329, 7331, 7333, 7151, 7335, 7337, 7339, 7341, 7343, 7345, 7347, 7349, - 7351, 7353, 7355, 7357, 7225, 7359, 7227, 7361, 7363, 7365, 7367, 7369, - 7153, 6703, 7371, 7373, 7375, 6843, 7017, 7377, 7379, 7241, 7381, 7243, - 7383, 7385, 7387, 7157, 7389, 7391, 7393, 7395, 7397, 7159, 7399, 7401, - 7403, 7405, 7407, 7409, 7271, 7411, 7413, 6919, 7415, 7279, 7417, 7419, - 7421, 7423, 7425, 7289, 7427, 7177, 7429, 7291, 6805, 7431, 7293, 7433, - 7297, 7435, 7437, 7439, 7441, 7443, 7301, 7169, 7445, 7303, 7447, 7305, - 7449, 6637, 7451, 7453, 7455, 7457, 7459, 7461, 7463, 7465, 7467, 7469, - 7471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7473, 7476, 7479, 7482, - 7486, 7490, 7493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7496, 7499, 7502, - 7505, 7508, 0, 0, 0, 0, 0, 7511, 0, 7514, 7517, 7519, 7521, 7523, 7525, - 7527, 7529, 7531, 7533, 7535, 7537, 7540, 7543, 7546, 7549, 7552, 7555, - 7558, 7561, 7564, 7567, 7570, 7573, 0, 7576, 7579, 7582, 7585, 7588, 0, - 7591, 0, 7594, 7597, 0, 7600, 7603, 0, 7606, 7609, 7612, 7615, 7618, - 7621, 7624, 7627, 7630, 7633, 7636, 7638, 7640, 7642, 7644, 7646, 7648, - 7650, 7652, 7654, 7656, 7658, 7660, 7662, 7664, 7666, 7668, 7670, 7672, - 7674, 7676, 7678, 7680, 7682, 7684, 7686, 7688, 7690, 7692, 7694, 7696, - 7698, 7700, 7702, 7704, 7706, 7708, 7710, 7712, 7714, 7716, 7718, 7720, - 7722, 7724, 7726, 7728, 7730, 7732, 7734, 7736, 7738, 7740, 7742, 7744, - 7746, 7748, 7750, 7752, 7754, 7756, 7758, 7760, 7762, 7764, 7766, 7768, - 7770, 7772, 7774, 7776, 7778, 7780, 7782, 7784, 7786, 7788, 7790, 7792, - 7794, 7796, 7798, 7800, 7802, 7804, 7806, 7808, 7810, 7812, 7814, 7816, - 7818, 7820, 7822, 7824, 7826, 7828, 7830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6615, 6617, 6619, 6621, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 6623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 6625, 6627, 6629, 6631, 6633, 6635, 6637, 6639, 6639, 6641, + 6643, 6645, 6647, 6649, 6651, 6653, 6655, 6657, 6659, 6661, 6663, 6665, + 6667, 6669, 6671, 6673, 6675, 6677, 6679, 6681, 6683, 6685, 6687, 6689, + 6691, 6693, 6695, 6697, 6699, 6701, 6703, 6705, 6707, 6709, 6711, 6713, + 6715, 6717, 6719, 6721, 6723, 6725, 6727, 6729, 6731, 6733, 6735, 6737, + 6739, 6741, 6743, 6745, 6747, 6749, 6751, 6753, 6755, 6757, 6759, 6761, + 6763, 6765, 6767, 6769, 6771, 6773, 6775, 6777, 6779, 6781, 6783, 6785, + 6787, 6789, 6791, 6793, 6795, 6797, 6799, 6801, 6803, 6805, 6663, 6807, + 6809, 6811, 6813, 6815, 6817, 6819, 6821, 6823, 6825, 6827, 6829, 6831, + 6833, 6835, 6837, 6839, 6841, 6843, 6845, 6847, 6849, 6851, 6853, 6855, + 6857, 6859, 6861, 6863, 6865, 6867, 6869, 6871, 6873, 6875, 6877, 6879, + 6881, 6883, 6885, 6887, 6889, 6891, 6893, 6895, 6897, 6899, 6901, 6903, + 6905, 6907, 6909, 6911, 6913, 6915, 6917, 6919, 6921, 6923, 6925, 6927, + 6929, 6931, 6933, 6935, 6937, 6939, 6941, 6843, 6943, 6945, 6947, 6949, + 6951, 6953, 6955, 6957, 6811, 6959, 6961, 6963, 6965, 6967, 6969, 6971, + 6973, 6975, 6977, 6979, 6981, 6983, 6985, 6987, 6989, 6991, 6993, 6995, + 6997, 6663, 6999, 7001, 7003, 7005, 7007, 7009, 7011, 7013, 7015, 7017, + 7019, 7021, 7023, 7025, 7027, 7029, 7031, 7033, 7035, 7037, 7039, 7041, + 7043, 7045, 7047, 7049, 7051, 6815, 7053, 7055, 7057, 7059, 7061, 7063, + 7065, 7067, 7069, 7071, 7073, 7075, 7077, 7079, 7081, 7083, 7085, 7087, + 7089, 7091, 7093, 7095, 7097, 7099, 7101, 7103, 7105, 7107, 7109, 7111, + 7113, 7115, 7117, 7119, 7121, 7123, 7125, 7127, 7129, 7131, 7133, 7135, + 7137, 7139, 7141, 7143, 7145, 7147, 7149, 7151, 0, 0, 7153, 0, 7155, 0, + 0, 7157, 7159, 7161, 7163, 7165, 7167, 7169, 7171, 7173, 7175, 0, 7177, + 0, 7179, 0, 0, 7181, 7183, 0, 0, 0, 7185, 7187, 7189, 7191, 7193, 7195, + 7197, 7199, 7201, 7203, 7205, 7207, 7209, 7211, 7213, 7215, 7217, 7219, + 7221, 7223, 7225, 7227, 7229, 7231, 7233, 7235, 7237, 7239, 7241, 7243, + 7245, 7247, 7249, 7251, 7253, 7255, 7257, 7259, 7261, 7263, 7265, 7267, + 7269, 7271, 7273, 6921, 7275, 7277, 7279, 7281, 7283, 7285, 7285, 7287, + 7289, 7291, 7293, 7295, 7297, 7299, 7301, 7181, 7303, 7305, 7307, 7309, + 7311, 7313, 0, 0, 7315, 7317, 7319, 7321, 7323, 7325, 7327, 7329, 7209, + 7331, 7333, 7335, 7153, 7337, 7339, 7341, 7343, 7345, 7347, 7349, 7351, + 7353, 7355, 7357, 7359, 7227, 7361, 7229, 7363, 7365, 7367, 7369, 7371, + 7155, 6705, 7373, 7375, 7377, 6845, 7019, 7379, 7381, 7243, 7383, 7245, + 7385, 7387, 7389, 7159, 7391, 7393, 7395, 7397, 7399, 7161, 7401, 7403, + 7405, 7407, 7409, 7411, 7273, 7413, 7415, 6921, 7417, 7281, 7419, 7421, + 7423, 7425, 7427, 7291, 7429, 7179, 7431, 7293, 6807, 7433, 7295, 7435, + 7299, 7437, 7439, 7441, 7443, 7445, 7303, 7171, 7447, 7305, 7449, 7307, + 7451, 6639, 7453, 7455, 7457, 7459, 7461, 7463, 7465, 7467, 7469, 7471, + 7473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7475, 7478, 7481, 7484, + 7488, 7492, 7495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7498, 7501, 7504, + 7507, 7510, 0, 0, 0, 0, 0, 7513, 0, 7516, 7519, 7521, 7523, 7525, 7527, + 7529, 7531, 7533, 7535, 7537, 7539, 7542, 7545, 7548, 7551, 7554, 7557, + 7560, 7563, 7566, 7569, 7572, 7575, 0, 7578, 7581, 7584, 7587, 7590, 0, + 7593, 0, 7596, 7599, 0, 7602, 7605, 0, 7608, 7611, 7614, 7617, 7620, + 7623, 7626, 7629, 7632, 7635, 7638, 7640, 7642, 7644, 7646, 7648, 7650, + 7652, 7654, 7656, 7658, 7660, 7662, 7664, 7666, 7668, 7670, 7672, 7674, + 7676, 7678, 7680, 7682, 7684, 7686, 7688, 7690, 7692, 7694, 7696, 7698, + 7700, 7702, 7704, 7706, 7708, 7710, 7712, 7714, 7716, 7718, 7720, 7722, + 7724, 7726, 7728, 7730, 7732, 7734, 7736, 7738, 7740, 7742, 7744, 7746, + 7748, 7750, 7752, 7754, 7756, 7758, 7760, 7762, 7764, 7766, 7768, 7770, + 7772, 7774, 7776, 7778, 7780, 7782, 7784, 7786, 7788, 7790, 7792, 7794, + 7796, 7798, 7800, 7802, 7804, 7806, 7808, 7810, 7812, 7814, 7816, 7818, + 7820, 7822, 7824, 7826, 7828, 7830, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 7832, 7834, 7836, 7838, 7840, 7842, 7844, 7846, 7848, 7850, 7852, 7854, - 7856, 7858, 7860, 7862, 7864, 7866, 7868, 7870, 7872, 7874, 7876, 7878, - 7881, 7884, 7887, 7890, 7893, 7896, 7899, 7902, 7905, 7908, 7911, 7914, - 7917, 7920, 7923, 7926, 7929, 7932, 7934, 7936, 7938, 7940, 7943, 7946, - 7923, 7949, 7952, 7955, 7958, 7961, 7964, 7967, 7970, 7973, 7976, 7979, - 7982, 7985, 7988, 7991, 7994, 7997, 8000, 8003, 8006, 8009, 8012, 8015, - 8018, 8021, 8024, 8027, 8030, 8033, 8036, 8039, 8042, 8045, 8048, 8051, - 8054, 8057, 8060, 8063, 8066, 8069, 8072, 8075, 8078, 8081, 8084, 8087, - 8090, 8093, 8096, 8099, 8102, 8105, 8108, 8111, 8114, 8117, 8120, 8123, - 8126, 8129, 8132, 8135, 8138, 8141, 8144, 8147, 8150, 8153, 8156, 8159, - 8162, 8165, 8168, 8171, 8174, 8177, 8180, 8183, 8186, 8189, 8192, 8195, - 8198, 8201, 8204, 8207, 8210, 8213, 8216, 8219, 8223, 8227, 8231, 8235, - 8239, 8243, 8246, 8249, 8252, 7926, 8255, 8258, 8261, 8264, 8267, 8270, - 8273, 8276, 8279, 8282, 8285, 8288, 8291, 8294, 8297, 8300, 8303, 8306, - 8309, 8312, 8315, 8318, 8321, 8324, 8327, 8330, 8333, 8336, 8339, 8342, - 8345, 8348, 8351, 8354, 8357, 8360, 8363, 8366, 8369, 8372, 8375, 8378, - 8381, 8384, 8387, 8390, 8393, 8396, 8399, 8402, 8405, 8408, 8411, 8414, - 8417, 8420, 8423, 8426, 8429, 8432, 8435, 8438, 8441, 8444, 8447, 8450, - 8453, 8456, 8459, 8462, 8465, 8468, 8471, 8474, 8477, 8480, 8483, 8486, - 8489, 8492, 8495, 8498, 8501, 8504, 8507, 8510, 8513, 8516, 8519, 8522, - 8525, 8528, 8531, 8534, 8537, 8540, 8543, 8546, 8549, 8552, 8555, 8558, - 8561, 8564, 8567, 8570, 8573, 8576, 8579, 8582, 8585, 8588, 8591, 8594, - 8597, 8600, 8603, 8606, 8609, 8612, 8615, 8618, 8621, 8624, 8627, 8630, - 8633, 8636, 8639, 8642, 8645, 8648, 8651, 8654, 8657, 8660, 8663, 8666, - 8670, 8674, 8678, 8681, 8684, 8687, 8690, 8693, 8696, 8699, 8702, 8705, - 8708, 8711, 8714, 8717, 8720, 8723, 8726, 8729, 8732, 8735, 8738, 8741, - 8744, 8747, 8750, 8753, 8756, 8759, 8762, 8765, 8768, 8771, 8774, 8777, - 8780, 8783, 8786, 8789, 8792, 8795, 8798, 8801, 8804, 8807, 8810, 8813, - 8816, 8819, 8822, 8825, 8828, 8831, 8834, 8837, 8840, 8843, 8846, 8849, - 8852, 8855, 8858, 8861, 8864, 8867, 8870, 8873, 8876, 8879, 8882, 8885, - 8888, 8891, 8894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8897, 8901, 8905, 8909, 8913, 8917, 8921, 8925, 8929, 8933, 8937, 8941, - 8945, 8949, 8953, 8957, 8961, 8965, 8969, 8973, 8977, 8981, 8985, 8989, - 8993, 8997, 9001, 9005, 9009, 9013, 9017, 9021, 9025, 9029, 9033, 9037, - 9041, 9045, 9049, 9053, 9057, 9061, 9065, 9069, 9073, 9077, 9081, 9085, - 9089, 9093, 9097, 9101, 9105, 9109, 9113, 9117, 9121, 9125, 9129, 9133, - 9137, 9141, 9145, 9149, 0, 0, 9153, 9157, 9161, 9165, 9169, 9173, 9177, - 9181, 9185, 9189, 9193, 9197, 9201, 9205, 9209, 9213, 9217, 9221, 9225, - 9229, 9233, 9237, 9241, 9245, 9249, 9253, 9257, 9261, 9265, 9269, 9273, - 9277, 9281, 9285, 9289, 9293, 9297, 9301, 9305, 9309, 9313, 9317, 9321, - 9325, 9329, 9333, 9337, 9341, 9345, 9349, 9353, 9357, 9361, 9365, 0, 0, + 7834, 7836, 7838, 7840, 7842, 7844, 7846, 7848, 7850, 7852, 7854, 7856, + 7858, 7860, 7862, 7864, 7866, 7868, 7870, 7872, 7874, 7876, 7878, 7880, + 7883, 7886, 7889, 7892, 7895, 7898, 7901, 7904, 7907, 7910, 7913, 7916, + 7919, 7922, 7925, 7928, 7931, 7934, 7936, 7938, 7940, 7942, 7945, 7948, + 7925, 7951, 7954, 7957, 7960, 7963, 7966, 7969, 7972, 7975, 7978, 7981, + 7984, 7987, 7990, 7993, 7996, 7999, 8002, 8005, 8008, 8011, 8014, 8017, + 8020, 8023, 8026, 8029, 8032, 8035, 8038, 8041, 8044, 8047, 8050, 8053, + 8056, 8059, 8062, 8065, 8068, 8071, 8074, 8077, 8080, 8083, 8086, 8089, + 8092, 8095, 8098, 8101, 8104, 8107, 8110, 8113, 8116, 8119, 8122, 8125, + 8128, 8131, 8134, 8137, 8140, 8143, 8146, 8149, 8152, 8155, 8158, 8161, + 8164, 8167, 8170, 8173, 8176, 8179, 8182, 8185, 8188, 8191, 8194, 8197, + 8200, 8203, 8206, 8209, 8212, 8215, 8218, 8221, 8225, 8229, 8233, 8237, + 8241, 8245, 8248, 8251, 8254, 7928, 8257, 8260, 8263, 8266, 8269, 8272, + 8275, 8278, 8281, 8284, 8287, 8290, 8293, 8296, 8299, 8302, 8305, 8308, + 8311, 8314, 8317, 8320, 8323, 8326, 8329, 8332, 8335, 8338, 8341, 8344, + 8347, 8350, 8353, 8356, 8359, 8362, 8365, 8368, 8371, 8374, 8377, 8380, + 8383, 8386, 8389, 8392, 8395, 8398, 8401, 8404, 8407, 8410, 8413, 8416, + 8419, 8422, 8425, 8428, 8431, 8434, 8437, 8440, 8443, 8446, 8449, 8452, + 8455, 8458, 8461, 8464, 8467, 8470, 8473, 8476, 8479, 8482, 8485, 8488, + 8491, 8494, 8497, 8500, 8503, 8506, 8509, 8512, 8515, 8518, 8521, 8524, + 8527, 8530, 8533, 8536, 8539, 8542, 8545, 8548, 8551, 8554, 8557, 8560, + 8563, 8566, 8569, 8572, 8575, 8578, 8581, 8584, 8587, 8590, 8593, 8596, + 8599, 8602, 8605, 8608, 8611, 8614, 8617, 8620, 8623, 8626, 8629, 8632, + 8635, 8638, 8641, 8644, 8647, 8650, 8653, 8656, 8659, 8662, 8665, 8668, + 8672, 8676, 8680, 8683, 8686, 8689, 8692, 8695, 8698, 8701, 8704, 8707, + 8710, 8713, 8716, 8719, 8722, 8725, 8728, 8731, 8734, 8737, 8740, 8743, + 8746, 8749, 8752, 8755, 8758, 8761, 8764, 8767, 8770, 8773, 8776, 8779, + 8782, 8785, 8788, 8791, 8794, 8797, 8800, 8803, 8806, 8809, 8812, 8815, + 8818, 8821, 8824, 8827, 8830, 8833, 8836, 8839, 8842, 8845, 8848, 8851, + 8854, 8857, 8860, 8863, 8866, 8869, 8872, 8875, 8878, 8881, 8884, 8887, + 8890, 8893, 8896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 8899, 8903, 8907, 8911, 8915, 8919, 8923, 8927, 8931, 8935, 8939, 8943, + 8947, 8951, 8955, 8959, 8963, 8967, 8971, 8975, 8979, 8983, 8987, 8991, + 8995, 8999, 9003, 9007, 9011, 9015, 9019, 9023, 9027, 9031, 9035, 9039, + 9043, 9047, 9051, 9055, 9059, 9063, 9067, 9071, 9075, 9079, 9083, 9087, + 9091, 9095, 9099, 9103, 9107, 9111, 9115, 9119, 9123, 9127, 9131, 9135, + 9139, 9143, 9147, 9151, 0, 0, 9155, 9159, 9163, 9167, 9171, 9175, 9179, + 9183, 9187, 9191, 9195, 9199, 9203, 9207, 9211, 9215, 9219, 9223, 9227, + 9231, 9235, 9239, 9243, 9247, 9251, 9255, 9259, 9263, 9267, 9271, 9275, + 9279, 9283, 9287, 9291, 9295, 9299, 9303, 9307, 9311, 9315, 9319, 9323, + 9327, 9331, 9335, 9339, 9343, 9347, 9351, 9355, 9359, 9363, 9367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9369, 9373, 9377, 9382, 9387, - 9392, 9397, 9402, 9407, 9412, 9416, 9435, 9444, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9449, 9451, 9453, 9455, 9457, 9459, - 9461, 9463, 9465, 9467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9469, 9471, 9473, 9475, 9475, 9477, 9479, 9481, 9483, - 9485, 9487, 9489, 9491, 9493, 9495, 9497, 9499, 9501, 9503, 9505, 9507, - 0, 0, 9509, 9511, 9513, 9513, 9513, 9513, 9515, 9515, 9515, 9517, 9519, - 9521, 0, 9523, 9525, 9527, 9529, 9531, 9533, 9535, 9537, 9539, 9541, - 9543, 9545, 9547, 9549, 9551, 9553, 9555, 9557, 9559, 0, 9561, 9563, - 9565, 9567, 0, 0, 0, 0, 9569, 9572, 9575, 0, 9578, 0, 9581, 9584, 9587, - 9590, 9593, 9596, 9599, 9602, 9605, 9608, 9611, 9613, 9615, 9617, 9619, - 9621, 9623, 9625, 9627, 9629, 9631, 9633, 9635, 9637, 9639, 9641, 9643, - 9645, 9647, 9649, 9651, 9653, 9655, 9657, 9659, 9661, 9663, 9665, 9667, - 9669, 9671, 9673, 9675, 9677, 9679, 9681, 9683, 9685, 9687, 9689, 9691, - 9693, 9695, 9697, 9699, 9701, 9703, 9705, 9707, 9709, 9711, 9713, 9715, - 9717, 9719, 9721, 9723, 9725, 9727, 9729, 9731, 9733, 9735, 9737, 9739, - 9741, 9743, 9745, 9747, 9749, 9751, 9753, 9755, 9757, 9759, 9761, 9763, - 9765, 9767, 9769, 9771, 9773, 9775, 9777, 9779, 9781, 9783, 9785, 9787, - 9789, 9791, 9793, 9795, 9797, 9799, 9801, 9803, 9805, 9807, 9809, 9811, - 9813, 9815, 9817, 9819, 9821, 9823, 9825, 9827, 9829, 9831, 9833, 9835, - 9837, 9839, 9841, 9843, 9845, 9848, 9851, 9854, 9857, 9860, 9863, 9866, - 0, 0, 0, 0, 9869, 9871, 9873, 9875, 9877, 9879, 9881, 9883, 9885, 9887, - 9889, 9891, 9893, 9895, 9897, 9899, 9901, 9903, 9905, 9907, 9909, 9911, - 9913, 9915, 9917, 9919, 9921, 9923, 9925, 9927, 9929, 9931, 9933, 9935, - 9937, 9939, 9941, 9943, 9945, 9947, 9949, 9951, 9953, 9955, 9957, 9959, - 9961, 9963, 9965, 9967, 9969, 9971, 9973, 9975, 9977, 9979, 9981, 9983, - 9985, 9987, 9989, 9991, 9993, 9995, 9997, 9999, 10001, 10003, 10005, - 10007, 10009, 10011, 10013, 10015, 10017, 10019, 10021, 10023, 10025, - 10027, 10029, 10031, 10033, 10035, 10037, 10039, 10041, 10043, 10045, - 10047, 10049, 10051, 10053, 10055, 10057, 10059, 10061, 10063, 10065, - 10067, 10069, 10071, 10073, 10075, 10077, 10079, 10081, 10083, 10085, - 10087, 10089, 10091, 10093, 10095, 10097, 10099, 10101, 10103, 10105, - 10107, 10109, 10111, 10113, 10115, 10117, 10119, 10121, 10123, 10125, - 10127, 10129, 10131, 10133, 10135, 10137, 10139, 10141, 10143, 10145, - 10147, 10149, 10151, 10153, 10155, 10157, 10159, 10161, 10163, 10165, - 10167, 10169, 10171, 10173, 10175, 10177, 10179, 10181, 10183, 10185, - 10187, 10189, 10191, 10193, 10195, 10197, 10199, 10201, 10203, 10205, - 10207, 10209, 10211, 10213, 10215, 10217, 10219, 10221, 10223, 10225, - 10227, 10229, 10231, 10233, 10235, 10237, 10239, 10241, 10243, 10245, - 10247, 0, 0, 0, 10249, 10251, 10253, 10255, 10257, 10259, 0, 0, 10261, - 10263, 10265, 10267, 10269, 10271, 0, 0, 10273, 10275, 10277, 10279, - 10281, 10283, 0, 0, 10285, 10287, 10289, 0, 0, 0, 10291, 10293, 10295, - 10297, 10299, 10301, 10303, 0, 10305, 10307, 10309, 10311, 10313, 10315, - 10317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9371, 9375, 9379, 9384, 9389, + 9394, 9399, 9404, 9409, 9414, 9418, 9437, 9446, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9451, 9453, 9455, 9457, 9459, 9461, + 9463, 9465, 9467, 9469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9471, 9473, 9475, 9477, 9477, 9479, 9481, 9483, 9485, + 9487, 9489, 9491, 9493, 9495, 9497, 9499, 9501, 9503, 9505, 9507, 9509, + 0, 0, 9511, 9513, 9515, 9515, 9515, 9515, 9517, 9517, 9517, 9519, 9521, + 9523, 0, 9525, 9527, 9529, 9531, 9533, 9535, 9537, 9539, 9541, 9543, + 9545, 9547, 9549, 9551, 9553, 9555, 9557, 9559, 9561, 0, 9563, 9565, + 9567, 9569, 0, 0, 0, 0, 9571, 9574, 9577, 0, 9580, 0, 9583, 9586, 9589, + 9592, 9595, 9598, 9601, 9604, 9607, 9610, 9613, 9615, 9617, 9619, 9621, + 9623, 9625, 9627, 9629, 9631, 9633, 9635, 9637, 9639, 9641, 9643, 9645, + 9647, 9649, 9651, 9653, 9655, 9657, 9659, 9661, 9663, 9665, 9667, 9669, + 9671, 9673, 9675, 9677, 9679, 9681, 9683, 9685, 9687, 9689, 9691, 9693, + 9695, 9697, 9699, 9701, 9703, 9705, 9707, 9709, 9711, 9713, 9715, 9717, + 9719, 9721, 9723, 9725, 9727, 9729, 9731, 9733, 9735, 9737, 9739, 9741, + 9743, 9745, 9747, 9749, 9751, 9753, 9755, 9757, 9759, 9761, 9763, 9765, + 9767, 9769, 9771, 9773, 9775, 9777, 9779, 9781, 9783, 9785, 9787, 9789, + 9791, 9793, 9795, 9797, 9799, 9801, 9803, 9805, 9807, 9809, 9811, 9813, + 9815, 9817, 9819, 9821, 9823, 9825, 9827, 9829, 9831, 9833, 9835, 9837, + 9839, 9841, 9843, 9845, 9847, 9850, 9853, 9856, 9859, 9862, 9865, 9868, + 0, 0, 0, 0, 9871, 9873, 9875, 9877, 9879, 9881, 9883, 9885, 9887, 9889, + 9891, 9893, 9895, 9897, 9899, 9901, 9903, 9905, 9907, 9909, 9911, 9913, + 9915, 9917, 9919, 9921, 9923, 9925, 9927, 9929, 9931, 9933, 9935, 9937, + 9939, 9941, 9943, 9945, 9947, 9949, 9951, 9953, 9955, 9957, 9959, 9961, + 9963, 9965, 9967, 9969, 9971, 9973, 9975, 9977, 9979, 9981, 9983, 9985, + 9987, 9989, 9991, 9993, 9995, 9997, 9999, 10001, 10003, 10005, 10007, + 10009, 10011, 10013, 10015, 10017, 10019, 10021, 10023, 10025, 10027, + 10029, 10031, 10033, 10035, 10037, 10039, 10041, 10043, 10045, 10047, + 10049, 10051, 10053, 10055, 10057, 10059, 10061, 10063, 10065, 10067, + 10069, 10071, 10073, 10075, 10077, 10079, 10081, 10083, 10085, 10087, + 10089, 10091, 10093, 10095, 10097, 10099, 10101, 10103, 10105, 10107, + 10109, 10111, 10113, 10115, 10117, 10119, 10121, 10123, 10125, 10127, + 10129, 10131, 10133, 10135, 10137, 10139, 10141, 10143, 10145, 10147, + 10149, 10151, 10153, 10155, 10157, 10159, 10161, 10163, 10165, 10167, + 10169, 10171, 10173, 10175, 10177, 10179, 10181, 10183, 10185, 10187, + 10189, 10191, 10193, 10195, 10197, 10199, 10201, 10203, 10205, 10207, + 10209, 10211, 10213, 10215, 10217, 10219, 10221, 10223, 10225, 10227, + 10229, 10231, 10233, 10235, 10237, 10239, 10241, 10243, 10245, 10247, + 10249, 0, 0, 0, 10251, 10253, 10255, 10257, 10259, 10261, 0, 0, 10263, + 10265, 10267, 10269, 10271, 10273, 0, 0, 10275, 10277, 10279, 10281, + 10283, 10285, 0, 0, 10287, 10289, 10291, 0, 0, 0, 10293, 10295, 10297, + 10299, 10301, 10303, 10305, 0, 10307, 10309, 10311, 10313, 10315, 10317, + 10319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10319, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 10322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 10325, 10327, 10329, 10331, 10333, 0, 10335, 10337, - 10339, 10341, 10343, 10345, 10347, 10349, 10351, 10353, 10355, 10357, - 10359, 10361, 10363, 10365, 10367, 10369, 10371, 10373, 10375, 10377, - 10379, 10381, 10383, 10385, 10387, 10389, 10391, 10393, 10395, 10397, - 10399, 10401, 10403, 10405, 10407, 10409, 10411, 10413, 10415, 10417, 0, - 10419, 10421, 10423, 10425, 10427, 10429, 10431, 10433, 10435, 0, 0, 0, + 10324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10327, 10329, 10331, 10333, 10335, 0, 10337, 10339, + 10341, 10343, 10345, 10347, 10349, 10351, 10353, 10355, 10357, 10359, + 10361, 10363, 10365, 10367, 10369, 10371, 10373, 10375, 10377, 10379, + 10381, 10383, 10385, 10387, 10389, 10391, 10393, 10395, 10397, 10399, + 10401, 10403, 10405, 10407, 10409, 10411, 10413, 10415, 10417, 10419, 0, + 10421, 10423, 10425, 10427, 10429, 10431, 10433, 10435, 10437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10437, 0, - 10440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10443, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10439, 0, + 10442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 10446, 10449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 10448, 10451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10452, 10455, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10454, 10457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 10458, 0, 10461, 0, 0, 0, 0, 0, 0, 0, 0, 10464, 0, 0, - 10467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10460, 0, 10463, 0, 0, 0, 0, 0, 0, 0, 0, 10466, 0, 0, + 10469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 10470, 0, 10473, 10476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10472, 0, 10475, 10478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 10479, 10482, 0, 10485, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 10481, 10484, 0, 10487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10488, 10491, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10490, 10493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 10494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10497, 10500, 10503, 10506, 10509, - 10512, 10515, 10518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10499, 10502, 10505, 10508, 10511, + 10514, 10517, 10520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -5445,234 +5440,234 @@ static const unsigned short decomp_index2[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 10521, 10524, 10527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10523, 10526, 10529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, - 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, - 10548, 10550, 10552, 3245, 10554, 10556, 10558, 10560, 10562, 10564, - 10566, 10568, 10570, 10572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10532, 3253, 3192, 3289, 3257, 3259, 10534, 3212, 3218, 10536, 10538, + 3220, 3261, 3224, 10540, 3229, 3231, 3233, 10542, 10544, 10546, 10548, + 10550, 10552, 10554, 3245, 10556, 10558, 10560, 10562, 10564, 10566, + 10568, 10570, 10572, 10574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10574, 10577, 10580, 10583, 10586, - 10589, 10592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10576, 10579, 10582, 10585, 10588, + 10591, 10594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10595, 10598, - 10601, 10604, 10607, 10610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10597, 10600, + 10603, 10606, 10609, 10612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, - 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, - 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, - 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, - 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, - 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, - 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, - 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 0, 3273, 3293, - 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, - 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, - 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, - 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, - 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, - 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, - 10641, 10643, 10645, 10647, 10530, 0, 3192, 3289, 0, 0, 10532, 0, 0, - 10534, 10536, 0, 0, 3224, 10538, 3229, 3231, 0, 10540, 10542, 10544, - 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 0, 10619, 0, - 3214, 3273, 3293, 10621, 3222, 10623, 10625, 0, 10627, 10629, 10631, - 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, - 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, - 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, - 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, - 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, - 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 0, 3289, 3257, - 3259, 10532, 0, 0, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 0, - 10540, 10542, 10544, 10546, 10548, 10550, 10552, 0, 10613, 10615, 10617, - 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, - 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, - 10645, 10647, 10530, 3253, 0, 3289, 3257, 3259, 10532, 0, 3218, 10534, - 10536, 3220, 3261, 0, 10538, 0, 0, 0, 10540, 10542, 10544, 10546, 10548, - 10550, 10552, 0, 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, - 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, - 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, - 3257, 3259, 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, - 3229, 3231, 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, - 10613, 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, - 3222, 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, - 10639, 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, - 10532, 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, - 3233, 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, - 10615, 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, - 10623, 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, - 10641, 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, - 3212, 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, - 10540, 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, - 10617, 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, - 10625, 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, - 10643, 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, - 3218, 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, - 10542, 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, - 3291, 3255, 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, - 3263, 10627, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, - 10645, 10647, 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, - 10534, 10536, 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, - 10544, 10546, 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, - 10619, 3210, 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, - 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, - 10530, 3253, 3192, 3289, 3257, 3259, 10532, 3212, 3218, 10534, 10536, - 3220, 3261, 3224, 10538, 3229, 3231, 3233, 10540, 10542, 10544, 10546, - 10548, 10550, 10552, 3245, 10613, 10615, 10617, 3291, 3255, 10619, 3210, - 3214, 3273, 3293, 10621, 3222, 10623, 10625, 3263, 10627, 10629, 10631, - 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10649, 10651, 0, - 0, 10653, 10655, 3283, 10657, 10659, 10661, 10663, 10665, 10667, 10669, - 10671, 10673, 10675, 10677, 10679, 3285, 10681, 10683, 10685, 10687, - 10689, 10691, 10693, 10695, 10697, 10699, 10701, 10703, 3281, 10705, - 10707, 10709, 10711, 10713, 10715, 10717, 10719, 10721, 10723, 10725, - 10727, 3279, 10729, 10731, 10733, 10735, 10737, 10739, 10741, 10743, - 10745, 10747, 10749, 10751, 10753, 10755, 10757, 10759, 10653, 10655, - 3283, 10657, 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, - 10675, 10677, 10679, 3285, 10681, 10683, 10685, 10687, 10689, 10691, - 10693, 10695, 10697, 10699, 10701, 10703, 3281, 10705, 10707, 10709, - 10711, 10713, 10715, 10717, 10719, 10721, 10723, 10725, 10727, 3279, - 10729, 10731, 10733, 10735, 10737, 10739, 10741, 10743, 10745, 10747, - 10749, 10751, 10753, 10755, 10757, 10759, 10653, 10655, 3283, 10657, - 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, - 10679, 3285, 10681, 10683, 10685, 10687, 10689, 10691, 10693, 10695, - 10697, 10699, 10701, 10703, 3281, 10705, 10707, 10709, 10711, 10713, - 10715, 10717, 10719, 10721, 10723, 10725, 10727, 3279, 10729, 10731, - 10733, 10735, 10737, 10739, 10741, 10743, 10745, 10747, 10749, 10751, - 10753, 10755, 10757, 10759, 10653, 10655, 3283, 10657, 10659, 10661, - 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, 10679, 3285, - 10681, 10683, 10685, 10687, 10689, 10691, 10693, 10695, 10697, 10699, - 10701, 10703, 3281, 10705, 10707, 10709, 10711, 10713, 10715, 10717, - 10719, 10721, 10723, 10725, 10727, 3279, 10729, 10731, 10733, 10735, - 10737, 10739, 10741, 10743, 10745, 10747, 10749, 10751, 10753, 10755, - 10757, 10759, 10653, 10655, 3283, 10657, 10659, 10661, 10663, 10665, - 10667, 10669, 10671, 10673, 10675, 10677, 10679, 3285, 10681, 10683, - 10685, 10687, 10689, 10691, 10693, 10695, 10697, 10699, 10701, 10703, - 3281, 10705, 10707, 10709, 10711, 10713, 10715, 10717, 10719, 10721, - 10723, 10725, 10727, 3279, 10729, 10731, 10733, 10735, 10737, 10739, - 10741, 10743, 10745, 10747, 10749, 10751, 10753, 10755, 10757, 10759, - 10761, 10763, 0, 0, 10554, 10556, 10558, 10560, 10562, 10564, 10566, - 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, - 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, - 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, - 10568, 10570, 10572, 10554, 10556, 10558, 10560, 10562, 10564, 10566, - 10568, 10570, 10572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 10765, 10767, 10769, 10771, 10773, 10775, 10777, - 10779, 10781, 10783, 10785, 10787, 10789, 10791, 10793, 10795, 10797, - 10799, 10801, 10803, 10805, 10807, 10809, 10811, 10813, 10815, 10817, - 10819, 10821, 10823, 10825, 10827, 10829, 10831, 10833, 10835, 10837, - 10839, 10841, 10843, 10845, 10847, 10849, 10851, 10853, 10855, 10857, - 10859, 10861, 10863, 10865, 10867, 10869, 10871, 10873, 10875, 10877, - 10879, 10881, 10883, 10885, 10887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 10889, 10891, 10893, 10895, 0, 10897, 10899, 10901, 10903, - 10905, 10907, 10909, 10911, 10913, 10915, 10917, 10919, 10921, 10923, - 10925, 10927, 10929, 10931, 10933, 10935, 10937, 10939, 10941, 10943, - 10945, 10947, 10949, 0, 10891, 10893, 0, 10951, 0, 0, 10901, 0, 10905, - 10907, 10909, 10911, 10913, 10915, 10917, 10919, 10921, 10923, 0, 10927, - 10929, 10931, 10933, 0, 10937, 0, 10941, 0, 0, 0, 0, 0, 0, 10893, 0, 0, - 0, 0, 10901, 0, 10905, 0, 10909, 0, 10913, 10915, 10917, 0, 10921, 10923, - 0, 10927, 0, 0, 10933, 0, 10937, 0, 10941, 0, 10945, 0, 10949, 0, 10891, - 10893, 0, 10951, 0, 0, 10901, 10903, 10905, 10907, 0, 10911, 10913, - 10915, 10917, 10919, 10921, 10923, 0, 10927, 10929, 10931, 10933, 0, - 10937, 10939, 10941, 10943, 0, 10947, 0, 10889, 10891, 10893, 10895, - 10951, 10897, 10899, 10901, 10903, 10905, 0, 10909, 10911, 10913, 10915, - 10917, 10919, 10921, 10923, 10925, 10927, 10929, 10931, 10933, 10935, - 10937, 10939, 10941, 0, 0, 0, 0, 0, 10891, 10893, 10895, 0, 10897, 10899, - 10901, 10903, 10905, 0, 10909, 10911, 10913, 10915, 10917, 10919, 10921, - 10923, 10925, 10927, 10929, 10931, 10933, 10935, 10937, 10939, 10941, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10953, 10956, - 10959, 10962, 10965, 10968, 10971, 10974, 10977, 10980, 10983, 0, 0, 0, - 0, 0, 10986, 10990, 10994, 10998, 11002, 11006, 11010, 11014, 11018, - 11022, 11026, 11030, 11034, 11038, 11042, 11046, 11050, 11054, 11058, - 11062, 11066, 11070, 11074, 11078, 11082, 11086, 11090, 3926, 3956, - 11094, 11097, 0, 11100, 11102, 11104, 11106, 11108, 11110, 11112, 11114, - 11116, 11118, 11120, 11122, 11124, 11126, 11128, 11130, 11132, 11134, - 11136, 11138, 11140, 11142, 11144, 11146, 11148, 11150, 11152, 6348, - 11155, 11158, 11161, 11165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11168, 11171, 11174, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 11177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11180, - 11183, 11186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11188, 11190, 11192, - 11194, 11196, 11198, 11200, 11202, 11204, 11206, 11208, 11210, 11212, - 11214, 11216, 11218, 11220, 11222, 11224, 11226, 11228, 11230, 11232, - 11234, 11236, 11238, 11240, 11242, 11244, 11246, 11248, 11250, 11252, - 11254, 11256, 11258, 11260, 11262, 11264, 11266, 11268, 11270, 11272, - 11274, 0, 0, 0, 0, 11276, 11280, 11284, 11288, 11292, 11296, 11300, - 11304, 11308, 0, 0, 0, 0, 0, 0, 0, 11312, 11314, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 10554, 10556, 10558, 10560, 10562, 10564, 10566, 10568, - 10570, 10572, 0, 0, 0, 0, 0, 0, 11316, 11318, 11320, 11322, 11324, 7195, - 11326, 11328, 11330, 11332, 7197, 11334, 11336, 11338, 7199, 11340, - 11342, 11344, 11346, 11348, 11350, 11352, 11354, 11356, 11358, 11360, - 11362, 7315, 11364, 11366, 11368, 11370, 11372, 11374, 11376, 11378, - 11380, 7325, 7201, 7203, 7327, 11382, 11384, 6817, 11386, 7205, 11388, - 11390, 11392, 11394, 11394, 11394, 11396, 11398, 11400, 11402, 11404, - 11406, 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424, - 11426, 11428, 11430, 11430, 7331, 11432, 11434, 11436, 11438, 7209, - 11440, 11442, 11444, 7123, 11446, 11448, 11450, 11452, 11454, 11456, - 11458, 11460, 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476, - 11478, 11480, 11482, 11484, 11486, 11488, 11490, 11492, 11494, 11496, - 11496, 11498, 11500, 11502, 6809, 11504, 11506, 11508, 11510, 11512, - 11514, 11516, 11518, 7219, 11520, 11522, 11524, 11526, 11528, 11530, - 11532, 11534, 11536, 11538, 11540, 11542, 11544, 11546, 11548, 11550, - 11552, 11554, 11556, 11558, 11560, 6701, 11562, 11564, 11566, 11566, - 11568, 11570, 11570, 11572, 11574, 11576, 11578, 11580, 11582, 11584, - 11586, 11588, 11590, 11592, 11594, 11596, 7221, 11598, 11600, 11602, - 11604, 7355, 11604, 11606, 7225, 11608, 11610, 11612, 11614, 7227, 6647, - 11616, 11618, 11620, 11622, 11624, 11626, 11628, 11630, 11632, 11634, - 11636, 11638, 11640, 11642, 11644, 11646, 11648, 11650, 11652, 11654, - 11656, 11658, 7229, 11660, 11662, 11664, 11666, 11668, 11670, 7233, - 11672, 11674, 11676, 11678, 11680, 11682, 11684, 11686, 6703, 7371, - 11688, 11690, 11692, 11694, 11696, 11698, 11700, 11702, 7235, 11704, - 11706, 11708, 11710, 7457, 11712, 11714, 11716, 11718, 11720, 11722, - 11724, 11726, 11728, 11730, 11732, 11734, 11736, 6843, 11738, 11740, - 11742, 11744, 11746, 11748, 11750, 11752, 11754, 11756, 11758, 7237, - 7017, 11760, 11762, 11764, 11766, 11768, 11770, 11772, 11774, 7379, - 11776, 11778, 11780, 11782, 11784, 11786, 11788, 11790, 7381, 11792, - 11794, 11796, 11798, 11800, 11802, 11804, 11806, 11808, 11810, 11812, - 11814, 7385, 11816, 11818, 11820, 11822, 11824, 11826, 11828, 11830, - 11832, 11834, 11836, 11836, 11838, 11840, 7389, 11842, 11844, 11846, - 11848, 11850, 11852, 11854, 6815, 11856, 11858, 11860, 11862, 11864, - 11866, 11868, 7401, 11870, 11872, 11874, 11876, 11878, 11880, 11880, - 7403, 7461, 11882, 11884, 11886, 11888, 11890, 6739, 7407, 11892, 11894, - 7259, 11896, 11898, 7167, 11900, 11902, 7267, 11904, 11906, 11908, 11910, - 11910, 11912, 11914, 11916, 11918, 11920, 11922, 11924, 11926, 11928, - 11930, 11932, 11934, 11936, 11938, 11940, 11942, 11944, 11946, 11948, - 11950, 11952, 11954, 11956, 11958, 11960, 11962, 11964, 7279, 11966, - 11968, 11970, 11972, 11974, 11976, 11978, 11980, 11982, 11984, 11986, - 11988, 11990, 11992, 11994, 11996, 11568, 11998, 12000, 12002, 12004, - 12006, 12008, 12010, 12012, 12014, 12016, 12018, 12020, 6851, 12022, - 12024, 12026, 12028, 12030, 12032, 7285, 12034, 12036, 12038, 12040, - 12042, 12044, 12046, 12048, 12050, 12052, 12054, 12056, 12058, 12060, - 12062, 12064, 12066, 12068, 12070, 12072, 6729, 12074, 12076, 12078, - 12080, 12082, 12084, 7421, 12086, 12088, 12090, 12092, 12094, 12096, - 12098, 12100, 12102, 12104, 12106, 12108, 12110, 12112, 12114, 12116, - 12118, 12120, 12122, 12124, 7431, 7433, 12126, 12128, 12130, 12132, - 12134, 12136, 12138, 12140, 12142, 12144, 12146, 12148, 12150, 7435, - 12152, 12154, 12156, 12158, 12160, 12162, 12164, 12166, 12168, 12170, - 12172, 12174, 12176, 12178, 12180, 12182, 12184, 12186, 12188, 12190, - 12192, 12194, 12196, 12198, 12200, 12202, 12204, 12206, 12208, 12210, - 7447, 7447, 12212, 12214, 12216, 12218, 12220, 12222, 12224, 12226, - 12228, 12230, 7449, 12232, 12234, 12236, 12238, 12240, 12242, 12244, - 12246, 12248, 12250, 12252, 12254, 12256, 12258, 12260, 12262, 12264, - 12266, 12268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 10532, 3253, 3192, 3289, 3257, 3259, 10534, 3212, 3218, 10536, 10538, + 3220, 3261, 3224, 10540, 3229, 3231, 3233, 10542, 10544, 10546, 10548, + 10550, 10552, 10554, 3245, 10615, 10617, 10619, 3291, 3255, 10621, 3210, + 3214, 3273, 3293, 10623, 3222, 10625, 10627, 3263, 10629, 10631, 10633, + 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10649, 10532, 3253, + 3192, 3289, 3257, 3259, 10534, 3212, 3218, 10536, 10538, 3220, 3261, + 3224, 10540, 3229, 3231, 3233, 10542, 10544, 10546, 10548, 10550, 10552, + 10554, 3245, 10615, 10617, 10619, 3291, 3255, 10621, 3210, 0, 3273, 3293, + 10623, 3222, 10625, 10627, 3263, 10629, 10631, 10633, 10635, 10637, + 10639, 10641, 10643, 10645, 10647, 10649, 10532, 3253, 3192, 3289, 3257, + 3259, 10534, 3212, 3218, 10536, 10538, 3220, 3261, 3224, 10540, 3229, + 3231, 3233, 10542, 10544, 10546, 10548, 10550, 10552, 10554, 3245, 10615, + 10617, 10619, 3291, 3255, 10621, 3210, 3214, 3273, 3293, 10623, 3222, + 10625, 10627, 3263, 10629, 10631, 10633, 10635, 10637, 10639, 10641, + 10643, 10645, 10647, 10649, 10532, 0, 3192, 3289, 0, 0, 10534, 0, 0, + 10536, 10538, 0, 0, 3224, 10540, 3229, 3231, 0, 10542, 10544, 10546, + 10548, 10550, 10552, 10554, 3245, 10615, 10617, 10619, 3291, 0, 10621, 0, + 3214, 3273, 3293, 10623, 3222, 10625, 10627, 0, 10629, 10631, 10633, + 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10649, 10532, 3253, + 3192, 3289, 3257, 3259, 10534, 3212, 3218, 10536, 10538, 3220, 3261, + 3224, 10540, 3229, 3231, 3233, 10542, 10544, 10546, 10548, 10550, 10552, + 10554, 3245, 10615, 10617, 10619, 3291, 3255, 10621, 3210, 3214, 3273, + 3293, 10623, 3222, 10625, 10627, 3263, 10629, 10631, 10633, 10635, 10637, + 10639, 10641, 10643, 10645, 10647, 10649, 10532, 3253, 0, 3289, 3257, + 3259, 10534, 0, 0, 10536, 10538, 3220, 3261, 3224, 10540, 3229, 3231, 0, + 10542, 10544, 10546, 10548, 10550, 10552, 10554, 0, 10615, 10617, 10619, + 3291, 3255, 10621, 3210, 3214, 3273, 3293, 10623, 3222, 10625, 10627, + 3263, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, 10645, + 10647, 10649, 10532, 3253, 0, 3289, 3257, 3259, 10534, 0, 3218, 10536, + 10538, 3220, 3261, 0, 10540, 0, 0, 0, 10542, 10544, 10546, 10548, 10550, + 10552, 10554, 0, 10615, 10617, 10619, 3291, 3255, 10621, 3210, 3214, + 3273, 3293, 10623, 3222, 10625, 10627, 3263, 10629, 10631, 10633, 10635, + 10637, 10639, 10641, 10643, 10645, 10647, 10649, 10532, 3253, 3192, 3289, + 3257, 3259, 10534, 3212, 3218, 10536, 10538, 3220, 3261, 3224, 10540, + 3229, 3231, 3233, 10542, 10544, 10546, 10548, 10550, 10552, 10554, 3245, + 10615, 10617, 10619, 3291, 3255, 10621, 3210, 3214, 3273, 3293, 10623, + 3222, 10625, 10627, 3263, 10629, 10631, 10633, 10635, 10637, 10639, + 10641, 10643, 10645, 10647, 10649, 10532, 3253, 3192, 3289, 3257, 3259, + 10534, 3212, 3218, 10536, 10538, 3220, 3261, 3224, 10540, 3229, 3231, + 3233, 10542, 10544, 10546, 10548, 10550, 10552, 10554, 3245, 10615, + 10617, 10619, 3291, 3255, 10621, 3210, 3214, 3273, 3293, 10623, 3222, + 10625, 10627, 3263, 10629, 10631, 10633, 10635, 10637, 10639, 10641, + 10643, 10645, 10647, 10649, 10532, 3253, 3192, 3289, 3257, 3259, 10534, + 3212, 3218, 10536, 10538, 3220, 3261, 3224, 10540, 3229, 3231, 3233, + 10542, 10544, 10546, 10548, 10550, 10552, 10554, 3245, 10615, 10617, + 10619, 3291, 3255, 10621, 3210, 3214, 3273, 3293, 10623, 3222, 10625, + 10627, 3263, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, + 10645, 10647, 10649, 10532, 3253, 3192, 3289, 3257, 3259, 10534, 3212, + 3218, 10536, 10538, 3220, 3261, 3224, 10540, 3229, 3231, 3233, 10542, + 10544, 10546, 10548, 10550, 10552, 10554, 3245, 10615, 10617, 10619, + 3291, 3255, 10621, 3210, 3214, 3273, 3293, 10623, 3222, 10625, 10627, + 3263, 10629, 10631, 10633, 10635, 10637, 10639, 10641, 10643, 10645, + 10647, 10649, 10532, 3253, 3192, 3289, 3257, 3259, 10534, 3212, 3218, + 10536, 10538, 3220, 3261, 3224, 10540, 3229, 3231, 3233, 10542, 10544, + 10546, 10548, 10550, 10552, 10554, 3245, 10615, 10617, 10619, 3291, 3255, + 10621, 3210, 3214, 3273, 3293, 10623, 3222, 10625, 10627, 3263, 10629, + 10631, 10633, 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10649, + 10532, 3253, 3192, 3289, 3257, 3259, 10534, 3212, 3218, 10536, 10538, + 3220, 3261, 3224, 10540, 3229, 3231, 3233, 10542, 10544, 10546, 10548, + 10550, 10552, 10554, 3245, 10615, 10617, 10619, 3291, 3255, 10621, 3210, + 3214, 3273, 3293, 10623, 3222, 10625, 10627, 3263, 10629, 10631, 10633, + 10635, 10637, 10639, 10641, 10643, 10645, 10647, 10649, 10651, 10653, 0, + 0, 10655, 10657, 3283, 10659, 10661, 10663, 10665, 10667, 10669, 10671, + 10673, 10675, 10677, 10679, 10681, 3285, 10683, 10685, 10687, 10689, + 10691, 10693, 10695, 10697, 10699, 10701, 10703, 10705, 3281, 10707, + 10709, 10711, 10713, 10715, 10717, 10719, 10721, 10723, 10725, 10727, + 10729, 3279, 10731, 10733, 10735, 10737, 10739, 10741, 10743, 10745, + 10747, 10749, 10751, 10753, 10755, 10757, 10759, 10761, 10655, 10657, + 3283, 10659, 10661, 10663, 10665, 10667, 10669, 10671, 10673, 10675, + 10677, 10679, 10681, 3285, 10683, 10685, 10687, 10689, 10691, 10693, + 10695, 10697, 10699, 10701, 10703, 10705, 3281, 10707, 10709, 10711, + 10713, 10715, 10717, 10719, 10721, 10723, 10725, 10727, 10729, 3279, + 10731, 10733, 10735, 10737, 10739, 10741, 10743, 10745, 10747, 10749, + 10751, 10753, 10755, 10757, 10759, 10761, 10655, 10657, 3283, 10659, + 10661, 10663, 10665, 10667, 10669, 10671, 10673, 10675, 10677, 10679, + 10681, 3285, 10683, 10685, 10687, 10689, 10691, 10693, 10695, 10697, + 10699, 10701, 10703, 10705, 3281, 10707, 10709, 10711, 10713, 10715, + 10717, 10719, 10721, 10723, 10725, 10727, 10729, 3279, 10731, 10733, + 10735, 10737, 10739, 10741, 10743, 10745, 10747, 10749, 10751, 10753, + 10755, 10757, 10759, 10761, 10655, 10657, 3283, 10659, 10661, 10663, + 10665, 10667, 10669, 10671, 10673, 10675, 10677, 10679, 10681, 3285, + 10683, 10685, 10687, 10689, 10691, 10693, 10695, 10697, 10699, 10701, + 10703, 10705, 3281, 10707, 10709, 10711, 10713, 10715, 10717, 10719, + 10721, 10723, 10725, 10727, 10729, 3279, 10731, 10733, 10735, 10737, + 10739, 10741, 10743, 10745, 10747, 10749, 10751, 10753, 10755, 10757, + 10759, 10761, 10655, 10657, 3283, 10659, 10661, 10663, 10665, 10667, + 10669, 10671, 10673, 10675, 10677, 10679, 10681, 3285, 10683, 10685, + 10687, 10689, 10691, 10693, 10695, 10697, 10699, 10701, 10703, 10705, + 3281, 10707, 10709, 10711, 10713, 10715, 10717, 10719, 10721, 10723, + 10725, 10727, 10729, 3279, 10731, 10733, 10735, 10737, 10739, 10741, + 10743, 10745, 10747, 10749, 10751, 10753, 10755, 10757, 10759, 10761, + 10763, 10765, 0, 0, 10556, 10558, 10560, 10562, 10564, 10566, 10568, + 10570, 10572, 10574, 10556, 10558, 10560, 10562, 10564, 10566, 10568, + 10570, 10572, 10574, 10556, 10558, 10560, 10562, 10564, 10566, 10568, + 10570, 10572, 10574, 10556, 10558, 10560, 10562, 10564, 10566, 10568, + 10570, 10572, 10574, 10556, 10558, 10560, 10562, 10564, 10566, 10568, + 10570, 10572, 10574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 10767, 10769, 10771, 10773, 10775, 10777, 10779, + 10781, 10783, 10785, 10787, 10789, 10791, 10793, 10795, 10797, 10799, + 10801, 10803, 10805, 10807, 10809, 10811, 10813, 10815, 10817, 10819, + 10821, 10823, 10825, 10827, 10829, 10831, 10833, 10835, 10837, 10839, + 10841, 10843, 10845, 10847, 10849, 10851, 10853, 10855, 10857, 10859, + 10861, 10863, 10865, 10867, 10869, 10871, 10873, 10875, 10877, 10879, + 10881, 10883, 10885, 10887, 10889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 10891, 10893, 10895, 10897, 0, 10899, 10901, 10903, 10905, + 10907, 10909, 10911, 10913, 10915, 10917, 10919, 10921, 10923, 10925, + 10927, 10929, 10931, 10933, 10935, 10937, 10939, 10941, 10943, 10945, + 10947, 10949, 10951, 0, 10893, 10895, 0, 10953, 0, 0, 10903, 0, 10907, + 10909, 10911, 10913, 10915, 10917, 10919, 10921, 10923, 10925, 0, 10929, + 10931, 10933, 10935, 0, 10939, 0, 10943, 0, 0, 0, 0, 0, 0, 10895, 0, 0, + 0, 0, 10903, 0, 10907, 0, 10911, 0, 10915, 10917, 10919, 0, 10923, 10925, + 0, 10929, 0, 0, 10935, 0, 10939, 0, 10943, 0, 10947, 0, 10951, 0, 10893, + 10895, 0, 10953, 0, 0, 10903, 10905, 10907, 10909, 0, 10913, 10915, + 10917, 10919, 10921, 10923, 10925, 0, 10929, 10931, 10933, 10935, 0, + 10939, 10941, 10943, 10945, 0, 10949, 0, 10891, 10893, 10895, 10897, + 10953, 10899, 10901, 10903, 10905, 10907, 0, 10911, 10913, 10915, 10917, + 10919, 10921, 10923, 10925, 10927, 10929, 10931, 10933, 10935, 10937, + 10939, 10941, 10943, 0, 0, 0, 0, 0, 10893, 10895, 10897, 0, 10899, 10901, + 10903, 10905, 10907, 0, 10911, 10913, 10915, 10917, 10919, 10921, 10923, + 10925, 10927, 10929, 10931, 10933, 10935, 10937, 10939, 10941, 10943, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10955, 10958, + 10961, 10964, 10967, 10970, 10973, 10976, 10979, 10982, 10985, 0, 0, 0, + 0, 0, 10988, 10992, 10996, 11000, 11004, 11008, 11012, 11016, 11020, + 11024, 11028, 11032, 11036, 11040, 11044, 11048, 11052, 11056, 11060, + 11064, 11068, 11072, 11076, 11080, 11084, 11088, 11092, 3926, 3956, + 11096, 11099, 0, 11102, 11104, 11106, 11108, 11110, 11112, 11114, 11116, + 11118, 11120, 11122, 11124, 11126, 11128, 11130, 11132, 11134, 11136, + 11138, 11140, 11142, 11144, 11146, 11148, 11150, 11152, 11154, 6348, + 11157, 11160, 11163, 11167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11170, 11173, 11176, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 11179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11182, + 11185, 11188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11190, 11192, 11194, + 11196, 11198, 11200, 11202, 11204, 11206, 11208, 11210, 11212, 11214, + 11216, 11218, 11220, 11222, 11224, 11226, 11228, 11230, 11232, 11234, + 11236, 11238, 11240, 11242, 11244, 11246, 11248, 11250, 11252, 11254, + 11256, 11258, 11260, 11262, 11264, 11266, 11268, 11270, 11272, 11274, + 11276, 0, 0, 0, 0, 11278, 11282, 11286, 11290, 11294, 11298, 11302, + 11306, 11310, 0, 0, 0, 0, 0, 0, 0, 11314, 11316, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 10556, 10558, 10560, 10562, 10564, 10566, 10568, 10570, + 10572, 10574, 0, 0, 0, 0, 0, 0, 11318, 11320, 11322, 11324, 11326, 7197, + 11328, 11330, 11332, 11334, 7199, 11336, 11338, 11340, 7201, 11342, + 11344, 11346, 11348, 11350, 11352, 11354, 11356, 11358, 11360, 11362, + 11364, 7317, 11366, 11368, 11370, 11372, 11374, 11376, 11378, 11380, + 11382, 7327, 7203, 7205, 7329, 11384, 11386, 6819, 11388, 7207, 11390, + 11392, 11394, 11396, 11396, 11396, 11398, 11400, 11402, 11404, 11406, + 11408, 11410, 11412, 11414, 11416, 11418, 11420, 11422, 11424, 11426, + 11428, 11430, 11432, 11432, 7333, 11434, 11436, 11438, 11440, 7211, + 11442, 11444, 11446, 7125, 11448, 11450, 11452, 11454, 11456, 11458, + 11460, 11462, 11464, 11466, 11468, 11470, 11472, 11474, 11476, 11478, + 11480, 11482, 11484, 11486, 11488, 11490, 11492, 11494, 11496, 11498, + 11498, 11500, 11502, 11504, 6811, 11506, 11508, 11510, 11512, 11514, + 11516, 11518, 11520, 7221, 11522, 11524, 11526, 11528, 11530, 11532, + 11534, 11536, 11538, 11540, 11542, 11544, 11546, 11548, 11550, 11552, + 11554, 11556, 11558, 11560, 11562, 6703, 11564, 11566, 11568, 11568, + 11570, 11572, 11572, 11574, 11576, 11578, 11580, 11582, 11584, 11586, + 11588, 11590, 11592, 11594, 11596, 11598, 7223, 11600, 11602, 11604, + 11606, 7357, 11606, 11608, 7227, 11610, 11612, 11614, 11616, 7229, 6649, + 11618, 11620, 11622, 11624, 11626, 11628, 11630, 11632, 11634, 11636, + 11638, 11640, 11642, 11644, 11646, 11648, 11650, 11652, 11654, 11656, + 11658, 11660, 7231, 11662, 11664, 11666, 11668, 11670, 11672, 7235, + 11674, 11676, 11678, 11680, 11682, 11684, 11686, 11688, 6705, 7373, + 11690, 11692, 11694, 11696, 11698, 11700, 11702, 11704, 7237, 11706, + 11708, 11710, 11712, 7459, 11714, 11716, 11718, 11720, 11722, 11724, + 11726, 11728, 11730, 11732, 11734, 11736, 11738, 6845, 11740, 11742, + 11744, 11746, 11748, 11750, 11752, 11754, 11756, 11758, 11760, 7239, + 7019, 11762, 11764, 11766, 11768, 11770, 11772, 11774, 11776, 7381, + 11778, 11780, 11782, 11784, 11786, 11788, 11790, 11792, 7383, 11794, + 11796, 11798, 11800, 11802, 11804, 11806, 11808, 11810, 11812, 11814, + 11816, 7387, 11818, 11820, 11822, 11824, 11826, 11828, 11830, 11832, + 11834, 11836, 11838, 11838, 11840, 11842, 7391, 11844, 11846, 11848, + 11850, 11852, 11854, 11856, 6817, 11858, 11860, 11862, 11864, 11866, + 11868, 11870, 7403, 11872, 11874, 11876, 11878, 11880, 11882, 11882, + 7405, 7463, 11884, 11886, 11888, 11890, 11892, 6741, 7409, 11894, 11896, + 7261, 11898, 11900, 7169, 11902, 11904, 7269, 11906, 11908, 11910, 11912, + 11912, 11914, 11916, 11918, 11920, 11922, 11924, 11926, 11928, 11930, + 11932, 11934, 11936, 11938, 11940, 11942, 11944, 11946, 11948, 11950, + 11952, 11954, 11956, 11958, 11960, 11962, 11964, 11966, 7281, 11968, + 11970, 11972, 11974, 11976, 11978, 11980, 11982, 11984, 11986, 11988, + 11990, 11992, 11994, 11996, 11998, 11570, 12000, 12002, 12004, 12006, + 12008, 12010, 12012, 12014, 12016, 12018, 12020, 12022, 6853, 12024, + 12026, 12028, 12030, 12032, 12034, 7287, 12036, 12038, 12040, 12042, + 12044, 12046, 12048, 12050, 12052, 12054, 12056, 12058, 12060, 12062, + 12064, 12066, 12068, 12070, 12072, 12074, 6731, 12076, 12078, 12080, + 12082, 12084, 12086, 7423, 12088, 12090, 12092, 12094, 12096, 12098, + 12100, 12102, 12104, 12106, 12108, 12110, 12112, 12114, 12116, 12118, + 12120, 12122, 12124, 12126, 7433, 7435, 12128, 12130, 12132, 12134, + 12136, 12138, 12140, 12142, 12144, 12146, 12148, 12150, 12152, 7437, + 12154, 12156, 12158, 12160, 12162, 12164, 12166, 12168, 12170, 12172, + 12174, 12176, 12178, 12180, 12182, 12184, 12186, 12188, 12190, 12192, + 12194, 12196, 12198, 12200, 12202, 12204, 12206, 12208, 12210, 12212, + 7449, 7449, 12214, 12216, 12218, 12220, 12222, 12224, 12226, 12228, + 12230, 12232, 7451, 12234, 12236, 12238, 12240, 12242, 12244, 12246, + 12248, 12250, 12252, 12254, 12256, 12258, 12260, 12262, 12264, 12266, + 12268, 12270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6196,7 +6191,6 @@ static const change_record change_records_3_2_0[] = { { 9, 255, 255, 255, 255, 0 }, { 255, 255, 255, 1, 255, 0 }, { 255, 20, 255, 255, 255, 0 }, - { 255, 255, 255, 255, 255, 1000000000000.0 }, { 255, 255, 255, 255, 255, 1e+20 }, { 255, 19, 255, 255, 255, -1 }, { 1, 255, 255, 255, 255, 0 }, @@ -6206,42 +6200,43 @@ static const unsigned char changes_3_2_0_index[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 1, 21, 22, 23, 24, 25, 26, 27, 28, 29, 1, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 1, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 1, 1, 1, 50, 1, 1, 51, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 52, 53, 54, 55, 56, 1, - 57, 1, 1, 1, 58, 1, 1, 1, 1, 1, 1, 59, 1, 1, 1, 60, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 61, 1, 55, 1, 1, 1, 1, 1, 1, 62, 1, 1, 63, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 64, 1, 1, 1, 1, 1, 65, 1, 66, 1, 67, 1, - 1, 1, 1, 1, 1, 1, 1, 68, 69, 1, 1, 1, 70, 28, 71, 72, 73, 74, 75, 76, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 52, 53, 54, 55, 1, 1, + 56, 1, 1, 1, 57, 1, 1, 1, 1, 1, 1, 58, 1, 1, 1, 59, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 60, 1, 55, 1, 1, 1, 1, 1, 1, 61, 1, 1, 62, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 63, 1, 1, 1, 1, 1, 64, 1, 65, 1, 66, 1, + 1, 1, 1, 1, 1, 1, 1, 67, 68, 1, 1, 1, 69, 28, 70, 71, 72, 73, 74, 75, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 77, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 76, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 78, 79, 80, 1, 81, 82, 83, 84, 85, 86, 87, 88, 89, 28, 90, - 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, 28, 28, 28, 115, 116, 117, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 118, 28, 28, 28, 28, 119, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 120, 28, 28, 121, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 122, 1, 1, 1, 1, 1, - 1, 28, 28, 123, 124, 1, 125, 126, 127, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 128, 28, 28, - 28, 28, 129, 130, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 131, 28, 132, 133, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 134, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 135, - 28, 136, 137, 1, 138, 139, 140, 141, 1, 142, 143, 28, 28, 144, 1, 1, 1, - 1, 145, 146, 147, 148, 1, 149, 150, 1, 151, 152, 153, 1, 1, 154, 155, - 156, 1, 157, 158, 159, 28, 28, 28, 160, 161, 162, 28, 163, 164, 1, 1, 1, - 1, 165, 67, 1, 1, 1, 1, 1, 1, 1, 166, 167, 168, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 169, 1, 1, 1, 1, 1, 170, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 171, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 77, 78, 79, 1, 80, 81, 82, 83, 84, 85, 86, 87, 88, 28, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, + 106, 107, 108, 109, 110, 111, 112, 113, 28, 28, 28, 114, 115, 116, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 117, 28, 28, 28, 28, 118, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 119, 28, 28, 120, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 121, 1, 1, 1, 1, 1, + 1, 28, 28, 122, 123, 1, 124, 125, 126, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 127, 128, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 129, 28, 130, 131, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 132, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 133, + 28, 134, 135, 1, 136, 137, 138, 139, 1, 140, 141, 28, 28, 142, 1, 1, 1, + 1, 143, 144, 145, 146, 1, 147, 148, 149, 150, 151, 152, 1, 1, 153, 154, + 155, 1, 156, 157, 158, 28, 28, 28, 159, 160, 161, 28, 162, 163, 1, 1, 1, + 1, 164, 66, 1, 1, 1, 1, 1, 1, 1, 165, 166, 167, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 168, 1, 1, 1, 1, 1, 169, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 170, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 172, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 171, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 173, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 174, 175, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 176, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 177, - 28, 28, 43, 1, 1, 1, 1, 1, 1, 1, 1, 1, 169, 1, 1, 1, 1, 1, 1, 1, 28, 28, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 178, - 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 179, 1, 1, 1, + 1, 1, 1, 172, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 173, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 174, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 175, + 28, 28, 43, 1, 1, 1, 1, 1, 1, 1, 1, 1, 168, 1, 1, 1, 1, 1, 1, 1, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 176, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 177, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -6357,7 +6352,7 @@ static const unsigned char changes_3_2_0_index[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 180, 1, 1, 1, 1, 1, + 178, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -6388,8 +6383,7 @@ static const unsigned char changes_3_2_0_index[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }; static const unsigned char changes_3_2_0_data[] = { @@ -6420,7 +6414,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6484,7 +6478,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6525,12 +6519,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, - 0, 0, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 20, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -6670,8 +6664,8 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6723,7 +6717,7 @@ static const unsigned char changes_3_2_0_data[] = { 41, 42, 43, 44, 1, 1, 0, 0, 0, 4, 38, 8, 6, 7, 39, 40, 41, 42, 43, 44, 1, 1, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, @@ -6832,7 +6826,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -6999,23 +6993,13 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, + 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7023,8 +7007,7 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7034,12 +7017,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7048,10 +7031,10 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7067,20 +7050,6 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7089,1109 +7058,12 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 9, 9, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 49, 50, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 21, 21, 21, 21, 21, 0, 0, 0, 1, 1, 21, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 14, 14, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 9, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, - 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 0, 0, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 9, 9, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 0, 9, - 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 0, 9, 9, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 0, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, - 9, 9, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 9, 0, 9, 0, 9, 0, 9, - 9, 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 0, 9, 9, 0, 9, 0, - 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, - 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, - 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, - 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8201,14 +7073,10 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8219,21 +7087,13 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -8254,11 +7114,35 @@ static const unsigned char changes_3_2_0_data[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8266,9 +7150,7 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8278,12 +7160,502 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 9, + 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 19, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 19, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 50, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 21, 21, 21, 21, 21, 21, 0, 0, 0, 1, 1, 21, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 14, 14, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, + 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 0, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, + 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 9, 9, 0, 0, 9, 9, 9, 0, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 0, 0, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8297,28 +7669,426 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 9, + 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 0, 0, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, + 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, + 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, + 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, 9, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 0, 9, 0, 9, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, + 0, 9, 0, 9, 0, 9, 0, 9, 9, 9, 0, 9, 9, 0, 9, 0, 0, 9, 0, 9, 0, 9, 0, 9, + 0, 9, 0, 9, 9, 0, 9, 0, 0, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, + 9, 9, 0, 9, 9, 9, 9, 0, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 0, 9, + 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -8328,9 +8098,212 @@ static const unsigned char changes_3_2_0_data[] = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, + 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 0, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, + 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, + 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, }; static const change_record* get_change_3_2_0(Py_UCS4 n) diff --git a/Modules/unicodename_db.h b/Modules/unicodename_db.h index 0697e259b39..d67e968e7a0 100644 --- a/Modules/unicodename_db.h +++ b/Modules/unicodename_db.h @@ -4,6595 +4,6669 @@ /* name->code dictionary */ static const unsigned char packed_name_dawg[] = { - 238, 255, 4, 254, 2, 65, 218, 130, 3, 66, 130, 154, 3, 67, 174, 245, 4, - 68, 206, 135, 1, 69, 186, 157, 1, 70, 190, 54, 71, 210, 164, 1, 72, 182, - 225, 1, 73, 222, 65, 74, 138, 23, 75, 238, 138, 1, 76, 130, 158, 3, 77, - 250, 200, 3, 78, 242, 107, 79, 138, 131, 1, 80, 128, 159, 1, 2, 81, 85, - 226, 7, 82, 194, 180, 1, 83, 214, 193, 4, 84, 134, 225, 2, 85, 242, 104, - 86, 138, 84, 87, 170, 114, 88, 254, 4, 89, 187, 48, 90, 214, 40, 222, 2, + 136, 135, 5, 254, 2, 65, 246, 145, 3, 66, 206, 160, 3, 67, 146, 251, 4, + 68, 250, 135, 1, 69, 230, 157, 1, 70, 174, 55, 71, 214, 169, 1, 72, 138, + 226, 1, 73, 138, 66, 74, 142, 23, 75, 250, 138, 1, 76, 150, 164, 3, 77, + 198, 201, 3, 78, 246, 107, 79, 174, 131, 1, 80, 212, 159, 1, 2, 81, 85, + 226, 7, 82, 158, 180, 1, 83, 210, 197, 4, 84, 146, 236, 2, 85, 246, 104, + 86, 242, 84, 87, 246, 114, 88, 130, 5, 89, 211, 50, 90, 176, 41, 218, 2, 67, 238, 1, 68, 182, 10, 69, 172, 5, 4, 72, 79, 77, 32, 244, 6, 7, 73, - 82, 80, 76, 65, 78, 69, 82, 76, 146, 36, 77, 198, 1, 78, 154, 26, 80, - 138, 20, 82, 226, 136, 2, 83, 210, 2, 84, 70, 85, 214, 1, 86, 144, 254, - 11, 4, 70, 71, 72, 65, 186, 231, 9, 66, 180, 227, 2, 9, 75, 84, 73, 69, - 83, 69, 76, 83, 75, 196, 198, 2, 2, 81, 85, 199, 196, 10, 88, 18, 132, 1, - 2, 67, 79, 46, 75, 20, 5, 85, 84, 69, 32, 65, 164, 153, 17, 6, 84, 73, - 86, 65, 84, 69, 145, 135, 23, 5, 32, 67, 85, 82, 82, 4, 170, 142, 26, 85, - 173, 133, 14, 2, 82, 68, 5, 151, 168, 34, 78, 4, 154, 199, 35, 67, 143, - 248, 3, 78, 188, 1, 232, 1, 4, 76, 65, 77, 32, 242, 7, 77, 212, 179, 13, - 7, 72, 69, 83, 73, 86, 69, 32, 144, 165, 4, 19, 68, 82, 69, 83, 83, 69, - 68, 32, 84, 79, 32, 84, 72, 69, 32, 83, 85, 66, 74, 146, 233, 13, 85, - 237, 185, 1, 6, 73, 32, 83, 72, 65, 75, 176, 1, 218, 1, 67, 44, 4, 83, + 82, 80, 76, 65, 78, 69, 82, 76, 190, 36, 77, 242, 1, 78, 162, 26, 80, + 162, 20, 82, 186, 150, 2, 83, 162, 3, 84, 70, 85, 214, 1, 86, 252, 197, + 1, 8, 75, 84, 73, 69, 83, 69, 76, 83, 168, 203, 10, 4, 70, 71, 72, 65, + 150, 239, 9, 66, 196, 170, 5, 2, 81, 85, 171, 215, 10, 88, 18, 132, 1, 2, + 67, 79, 46, 75, 20, 5, 85, 84, 69, 32, 65, 160, 188, 17, 6, 84, 73, 86, + 65, 84, 69, 213, 161, 23, 5, 32, 67, 85, 82, 82, 4, 186, 184, 26, 85, + 221, 152, 14, 2, 82, 68, 5, 203, 215, 34, 78, 4, 142, 252, 35, 67, 199, + 254, 3, 78, 188, 1, 232, 1, 4, 76, 65, 77, 32, 242, 7, 77, 252, 207, 13, + 7, 72, 69, 83, 73, 86, 69, 32, 136, 172, 4, 19, 68, 82, 69, 83, 83, 69, + 68, 32, 84, 79, 32, 84, 72, 69, 32, 83, 85, 66, 74, 174, 245, 13, 85, + 233, 185, 1, 6, 73, 32, 83, 72, 65, 75, 176, 1, 218, 1, 67, 44, 4, 83, 77, 65, 76, 168, 4, 7, 71, 69, 77, 73, 78, 65, 84, 116, 8, 73, 78, 73, - 84, 73, 65, 76, 32, 38, 78, 198, 213, 2, 72, 200, 197, 23, 4, 65, 76, 73, - 70, 0, 5, 86, 79, 87, 69, 76, 211, 205, 12, 68, 70, 40, 5, 65, 80, 73, + 84, 73, 65, 76, 32, 38, 78, 146, 228, 2, 72, 140, 225, 23, 4, 65, 76, 73, + 70, 0, 5, 86, 79, 87, 69, 76, 155, 222, 12, 68, 70, 40, 5, 65, 80, 73, 84, 65, 215, 4, 79, 68, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, - 142, 2, 68, 38, 71, 34, 74, 2, 77, 22, 75, 46, 78, 42, 83, 166, 133, 30, - 81, 238, 223, 4, 76, 194, 39, 67, 194, 197, 4, 90, 212, 7, 2, 65, 76, + 142, 2, 68, 38, 71, 34, 74, 2, 77, 22, 75, 46, 78, 42, 83, 130, 176, 30, + 81, 142, 228, 4, 76, 142, 45, 67, 158, 204, 4, 90, 240, 9, 2, 65, 76, 170, 2, 66, 2, 89, 154, 1, 87, 198, 89, 84, 150, 14, 80, 158, 20, 70, 2, - 72, 2, 82, 2, 86, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 170, 218, 5, 65, - 155, 128, 35, 72, 4, 130, 198, 40, 66, 215, 22, 65, 2, 231, 217, 39, 73, - 6, 254, 218, 39, 65, 178, 98, 80, 191, 28, 72, 6, 150, 140, 40, 85, 170, - 77, 72, 3, 89, 4, 212, 185, 2, 6, 73, 78, 78, 89, 73, 73, 195, 159, 38, - 72, 4, 40, 4, 69, 32, 67, 79, 219, 241, 39, 73, 2, 233, 209, 39, 13, 78, - 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 4, 214, 204, 33, 69, 183, - 188, 4, 81, 4, 174, 236, 25, 65, 251, 217, 10, 85, 4, 238, 178, 27, 69, - 189, 146, 12, 12, 73, 83, 83, 73, 79, 78, 32, 84, 73, 67, 75, 69, 116, - 80, 5, 71, 69, 65, 78, 32, 241, 135, 34, 9, 82, 73, 65, 76, 32, 84, 82, + 72, 2, 82, 2, 86, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 150, 240, 5, 65, + 239, 167, 35, 72, 4, 194, 131, 41, 66, 215, 22, 65, 2, 167, 151, 40, 73, + 6, 190, 152, 40, 65, 178, 98, 80, 191, 28, 72, 6, 214, 201, 40, 85, 170, + 77, 72, 3, 89, 4, 164, 200, 2, 6, 73, 78, 78, 89, 73, 73, 179, 206, 38, + 72, 4, 40, 4, 69, 32, 67, 79, 155, 175, 40, 73, 2, 145, 141, 40, 13, 78, + 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 4, 134, 252, 33, 69, 247, + 198, 4, 81, 4, 186, 150, 26, 65, 143, 228, 10, 85, 4, 230, 220, 27, 69, + 245, 177, 12, 12, 73, 83, 83, 73, 79, 78, 32, 84, 73, 67, 75, 69, 116, + 80, 5, 71, 69, 65, 78, 32, 161, 183, 34, 9, 82, 73, 65, 76, 32, 84, 82, 65, 77, 114, 136, 1, 3, 68, 82, 89, 0, 6, 76, 73, 81, 85, 73, 68, 60, 8, - 77, 69, 65, 83, 85, 82, 69, 32, 26, 87, 230, 147, 27, 78, 183, 255, 11, + 77, 69, 65, 83, 85, 82, 69, 32, 26, 87, 146, 190, 27, 78, 183, 144, 12, 67, 2, 153, 2, 11, 32, 77, 69, 65, 83, 85, 82, 69, 32, 70, 73, 4, 246, 1, 83, 31, 84, 14, 100, 6, 69, 73, 71, 72, 84, 32, 241, 1, 14, 79, 82, 68, 32, 83, 69, 80, 65, 82, 65, 84, 79, 82, 32, 10, 54, 70, 66, 83, 30, 84, 65, 5, 66, 65, 83, 69, 32, 4, 38, 73, 89, 5, 79, 85, 82, 84, 72, 2, 85, 3, 82, 83, 84, 2, 49, 4, 69, 67, 79, 78, 2, 21, 3, 72, 73, 82, 2, 11, 68, - 2, 25, 4, 32, 83, 85, 66, 2, 217, 183, 40, 2, 85, 78, 4, 190, 186, 39, - 76, 131, 73, 68, 130, 1, 140, 2, 22, 67, 79, 78, 83, 79, 78, 65, 78, 84, + 2, 25, 4, 32, 83, 85, 66, 2, 153, 245, 40, 2, 85, 78, 4, 250, 245, 39, + 76, 135, 75, 68, 130, 1, 140, 2, 22, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 88, 7, 76, 69, 84, 84, 69, 82, 32, 242, 1, 83, 168, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, - 71, 78, 32, 140, 212, 32, 8, 78, 85, 77, 66, 69, 82, 32, 84, 171, 136, 6, - 68, 6, 26, 76, 243, 206, 40, 82, 4, 204, 134, 40, 7, 73, 71, 65, 84, 73, - 78, 71, 219, 74, 65, 68, 154, 1, 65, 214, 245, 36, 68, 114, 84, 218, 207, - 1, 76, 250, 192, 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, + 71, 78, 32, 196, 131, 33, 8, 78, 85, 77, 66, 69, 82, 32, 84, 203, 147, 6, + 68, 6, 26, 76, 179, 140, 41, 82, 4, 140, 196, 40, 7, 73, 71, 65, 84, 73, + 78, 71, 219, 74, 65, 68, 154, 1, 65, 194, 175, 37, 68, 114, 84, 198, 208, + 1, 76, 226, 195, 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 3, 83, 9, 45, 9, 76, 84, 69, 82, 78, 65, 84, 69, - 32, 6, 230, 204, 40, 66, 2, 71, 3, 84, 10, 60, 4, 73, 71, 78, 32, 153, - 149, 23, 5, 89, 77, 66, 79, 76, 8, 50, 83, 170, 215, 25, 75, 229, 139, - 10, 2, 82, 85, 4, 180, 207, 37, 4, 77, 65, 76, 76, 247, 173, 2, 69, 22, - 66, 65, 234, 249, 36, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 11, - 142, 205, 40, 65, 2, 73, 2, 77, 3, 87, 7, 11, 32, 4, 192, 182, 35, 3, 68, - 69, 80, 145, 197, 4, 5, 65, 82, 82, 73, 86, 150, 2, 216, 1, 20, 67, 72, - 69, 77, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, - 166, 28, 69, 60, 4, 73, 69, 78, 32, 208, 3, 2, 76, 32, 82, 77, 108, 6, - 84, 69, 82, 78, 65, 84, 141, 142, 36, 3, 65, 82, 77, 232, 1, 158, 2, 65, - 246, 2, 66, 210, 1, 67, 138, 3, 68, 98, 71, 38, 72, 132, 1, 4, 73, 82, - 79, 78, 82, 76, 74, 77, 144, 1, 2, 78, 73, 34, 80, 176, 2, 3, 81, 85, 73, - 74, 82, 146, 2, 83, 210, 5, 84, 166, 1, 86, 200, 1, 2, 87, 65, 170, 238, - 34, 85, 154, 215, 1, 69, 158, 181, 1, 79, 139, 31, 70, 30, 194, 1, 76, - 72, 3, 81, 85, 65, 204, 8, 7, 78, 84, 73, 77, 79, 78, 89, 236, 142, 3, 3, - 77, 65, 76, 128, 174, 21, 3, 82, 83, 69, 248, 229, 13, 5, 85, 82, 73, 80, - 73, 210, 134, 1, 83, 171, 91, 73, 8, 224, 24, 2, 69, 77, 212, 171, 26, 4, - 75, 65, 76, 73, 191, 241, 13, 85, 10, 38, 32, 129, 223, 24, 3, 70, 79, - 82, 8, 216, 7, 4, 86, 73, 84, 65, 145, 150, 23, 4, 82, 69, 71, 73, 16, - 148, 1, 7, 65, 84, 72, 32, 79, 70, 32, 204, 6, 6, 73, 83, 77, 85, 84, 72, - 144, 1, 4, 79, 82, 65, 88, 164, 1, 4, 76, 65, 67, 75, 139, 139, 39, 82, - 4, 138, 234, 16, 77, 165, 203, 22, 5, 86, 65, 80, 79, 85, 28, 82, 65, 92, - 6, 79, 80, 80, 69, 82, 32, 34, 82, 185, 160, 39, 4, 73, 78, 78, 65, 6, - 152, 250, 5, 2, 68, 85, 204, 159, 33, 9, 80, 85, 84, 32, 77, 79, 82, 84, - 85, 239, 28, 76, 4, 230, 13, 65, 255, 147, 39, 79, 16, 72, 8, 79, 67, 85, - 83, 32, 79, 70, 32, 53, 6, 85, 67, 73, 66, 76, 69, 6, 228, 16, 5, 67, 79, - 80, 80, 69, 171, 139, 39, 73, 11, 11, 45, 8, 238, 192, 40, 50, 2, 51, 2, - 52, 3, 53, 8, 44, 2, 73, 83, 129, 198, 37, 3, 65, 89, 45, 6, 144, 2, 4, - 83, 79, 76, 86, 255, 197, 37, 84, 4, 218, 161, 30, 79, 179, 141, 10, 85, - 8, 32, 4, 65, 76, 70, 32, 47, 79, 4, 200, 183, 35, 2, 79, 85, 203, 204, - 3, 68, 4, 212, 250, 2, 4, 82, 83, 69, 32, 191, 139, 37, 85, 6, 56, 3, 32, - 79, 82, 73, 7, 45, 67, 79, 80, 80, 69, 82, 4, 235, 187, 26, 69, 4, 48, 3, - 69, 65, 68, 229, 204, 13, 3, 79, 68, 69, 2, 223, 216, 35, 32, 10, 120, - 16, 69, 82, 67, 85, 82, 89, 32, 83, 85, 66, 76, 73, 77, 65, 84, 69, 196, - 183, 16, 4, 65, 82, 67, 65, 203, 130, 22, 79, 7, 179, 187, 40, 45, 4, - 218, 155, 39, 84, 131, 118, 71, 14, 108, 11, 72, 73, 76, 79, 83, 79, 80, - 72, 69, 82, 83, 34, 79, 98, 85, 173, 134, 30, 6, 82, 69, 67, 73, 80, 73, - 2, 209, 9, 4, 32, 83, 85, 76, 6, 52, 4, 87, 68, 69, 82, 205, 136, 36, 3, - 84, 32, 65, 5, 173, 138, 39, 5, 69, 68, 32, 66, 82, 4, 200, 136, 34, 4, - 84, 82, 69, 70, 241, 159, 6, 3, 82, 73, 70, 4, 220, 176, 35, 5, 78, 84, - 69, 83, 83, 237, 240, 3, 4, 67, 75, 32, 76, 24, 58, 69, 173, 182, 26, 8, - 79, 67, 75, 32, 83, 65, 76, 84, 20, 68, 5, 71, 85, 76, 85, 83, 164, 7, 3, - 65, 76, 71, 195, 129, 26, 84, 15, 32, 4, 32, 79, 70, 32, 71, 45, 6, 188, - 181, 26, 8, 65, 78, 84, 73, 77, 79, 78, 89, 247, 220, 12, 73, 6, 142, - 183, 40, 50, 2, 51, 3, 52, 34, 164, 1, 2, 65, 76, 194, 1, 84, 142, 1, 85, - 192, 191, 17, 2, 80, 73, 164, 178, 12, 2, 73, 76, 170, 176, 9, 79, 141, - 20, 11, 67, 69, 80, 84, 69, 82, 32, 79, 70, 32, 74, 8, 58, 84, 141, 141, - 39, 8, 45, 65, 77, 77, 79, 78, 73, 65, 7, 25, 4, 32, 79, 70, 32, 4, 52, - 8, 67, 79, 80, 80, 69, 82, 32, 65, 231, 5, 65, 2, 229, 128, 30, 7, 78, - 84, 73, 77, 79, 78, 73, 6, 236, 3, 8, 65, 82, 82, 69, 68, 32, 84, 82, - 129, 174, 26, 19, 82, 65, 84, 85, 77, 32, 83, 85, 80, 69, 82, 32, 83, 84, - 82, 65, 84, 85, 77, 12, 44, 6, 66, 76, 73, 77, 65, 84, 155, 1, 76, 10, - 44, 5, 69, 32, 79, 70, 32, 175, 226, 39, 73, 8, 68, 8, 83, 65, 76, 84, - 32, 79, 70, 32, 130, 3, 65, 247, 172, 27, 67, 4, 254, 2, 65, 247, 172, - 27, 67, 2, 219, 184, 39, 70, 12, 68, 3, 65, 82, 84, 32, 2, 73, 78, 34, - 82, 157, 195, 38, 2, 85, 84, 4, 11, 65, 4, 179, 174, 26, 82, 4, 182, 154, - 35, 67, 167, 49, 32, 2, 233, 236, 39, 2, 73, 68, 14, 50, 73, 145, 140, - 38, 6, 69, 82, 68, 73, 71, 82, 12, 64, 5, 78, 69, 71, 65, 82, 237, 172, - 26, 5, 84, 82, 73, 79, 76, 9, 44, 5, 32, 79, 70, 32, 65, 223, 173, 40, - 45, 2, 181, 144, 24, 3, 78, 84, 73, 4, 194, 240, 39, 84, 239, 61, 88, 6, - 38, 77, 194, 154, 39, 70, 255, 118, 82, 2, 223, 133, 39, 66, 22, 104, 7, - 77, 79, 78, 83, 84, 69, 82, 138, 1, 83, 241, 204, 35, 10, 67, 82, 65, 66, - 32, 83, 84, 69, 80, 80, 11, 11, 32, 8, 88, 6, 67, 76, 79, 83, 69, 68, 0, - 4, 79, 80, 69, 78, 237, 170, 36, 4, 83, 84, 69, 80, 2, 205, 208, 37, 3, - 32, 74, 65, 8, 60, 6, 80, 73, 68, 69, 82, 32, 57, 5, 81, 85, 73, 68, 32, - 4, 140, 153, 29, 5, 67, 82, 79, 85, 67, 215, 245, 1, 83, 4, 56, 6, 67, - 76, 79, 83, 69, 68, 1, 4, 79, 80, 69, 78, 2, 217, 207, 27, 5, 32, 84, 69, - 78, 84, 4, 190, 214, 33, 69, 209, 174, 4, 11, 65, 82, 79, 85, 78, 68, 45, - 80, 82, 79, 70, 9, 49, 10, 79, 83, 84, 32, 69, 81, 85, 65, 76, 32, 6, 32, - 2, 84, 79, 251, 151, 32, 79, 5, 227, 232, 6, 32, 4, 232, 219, 10, 3, 73, - 86, 69, 253, 163, 28, 5, 69, 32, 79, 78, 69, 10, 162, 1, 80, 200, 165, - 29, 6, 69, 82, 73, 67, 65, 78, 188, 249, 5, 3, 66, 85, 76, 197, 248, 3, - 16, 65, 76, 71, 65, 77, 65, 84, 73, 79, 78, 32, 79, 82, 32, 67, 79, 4, - 210, 215, 36, 69, 139, 94, 72, 194, 9, 92, 3, 65, 84, 79, 174, 20, 71, - 174, 1, 84, 146, 248, 24, 67, 222, 139, 12, 68, 139, 139, 3, 75, 144, 9, - 104, 17, 76, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 32, - 65, 141, 148, 13, 3, 77, 73, 67, 142, 9, 70, 48, 138, 4, 49, 198, 2, 50, - 162, 3, 51, 174, 5, 52, 163, 3, 53, 222, 1, 106, 50, 102, 52, 110, 54, - 102, 57, 182, 7, 51, 210, 219, 11, 49, 206, 134, 23, 48, 242, 1, 53, 2, - 55, 3, 56, 22, 170, 183, 34, 54, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 55, 2, 56, 3, 57, 28, 158, 164, 12, 54, 170, 146, 22, 49, - 2, 53, 134, 236, 5, 48, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 26, - 246, 138, 12, 54, 234, 150, 28, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 55, 2, 56, 3, 57, 24, 246, 180, 34, 55, 2, 56, 134, 236, 5, 48, 2, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 57, 232, 1, 98, 48, 114, 49, 162, 143, - 12, 50, 2, 51, 230, 217, 22, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 42, - 198, 136, 12, 52, 2, 55, 190, 24, 53, 170, 146, 22, 48, 2, 49, 2, 50, - 134, 236, 5, 51, 2, 54, 2, 56, 3, 57, 26, 146, 160, 12, 48, 170, 146, 22, - 53, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, - 222, 1, 102, 48, 110, 49, 102, 57, 210, 1, 56, 206, 253, 11, 50, 2, 54, - 194, 230, 22, 51, 2, 52, 2, 53, 3, 55, 28, 186, 158, 12, 50, 170, 146, - 22, 55, 2, 57, 134, 236, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 3, 56, - 24, 246, 175, 34, 53, 2, 54, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, - 2, 55, 2, 56, 3, 57, 24, 146, 175, 34, 52, 2, 57, 134, 236, 5, 48, 2, 49, - 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 3, 56, 228, 1, 102, 48, 2, 50, 2, 53, - 102, 51, 110, 54, 102, 56, 162, 1, 57, 130, 230, 11, 55, 186, 250, 22, - 49, 3, 52, 22, 194, 173, 34, 57, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 3, 56, 30, 250, 129, 12, 54, 150, 196, 8, 50, - 214, 210, 19, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, - 242, 171, 34, 52, 2, 56, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, - 54, 2, 55, 3, 57, 26, 98, 51, 174, 170, 34, 49, 2, 54, 134, 236, 5, 48, - 2, 50, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 4, 216, 246, 20, 6, 32, 82, 65, - 32, 79, 82, 215, 159, 19, 65, 20, 160, 235, 39, 3, 51, 32, 69, 210, 42, - 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 202, 1, 102, - 49, 210, 1, 53, 202, 205, 20, 57, 202, 143, 14, 48, 2, 50, 2, 51, 2, 52, - 2, 54, 2, 55, 3, 56, 22, 90, 48, 194, 147, 40, 49, 2, 50, 2, 51, 2, 52, - 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 4, 60, 6, 32, 66, 69, 71, 73, 78, 1, - 5, 65, 32, 69, 78, 68, 2, 213, 167, 16, 8, 32, 76, 79, 71, 79, 71, 82, - 65, 24, 198, 166, 34, 48, 2, 55, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 2, 54, 2, 56, 3, 57, 60, 150, 249, 32, 51, 166, 225, 1, 48, 242, 1, - 49, 3, 50, 14, 96, 2, 76, 69, 198, 162, 2, 85, 136, 162, 32, 2, 83, 84, - 242, 232, 3, 69, 241, 165, 1, 2, 82, 89, 7, 33, 6, 32, 87, 73, 84, 72, - 32, 4, 182, 236, 31, 83, 231, 184, 4, 85, 31, 76, 4, 69, 78, 78, 65, 41, - 11, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 5, 241, 230, 36, 5, 32, - 87, 73, 84, 72, 24, 90, 84, 222, 248, 6, 67, 58, 68, 122, 71, 138, 4, 79, - 153, 134, 28, 5, 73, 78, 84, 69, 71, 12, 84, 15, 82, 73, 65, 78, 71, 76, - 69, 45, 72, 69, 65, 68, 69, 68, 32, 155, 253, 6, 79, 10, 112, 3, 76, 69, - 70, 0, 4, 82, 73, 71, 72, 12, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, - 80, 191, 252, 6, 79, 2, 11, 84, 2, 193, 178, 37, 7, 32, 85, 45, 83, 72, - 65, 80, 158, 1, 128, 1, 20, 76, 32, 70, 85, 78, 67, 84, 73, 79, 78, 65, - 76, 32, 83, 89, 77, 66, 79, 76, 32, 206, 15, 79, 38, 80, 179, 251, 39, - 67, 140, 1, 222, 2, 67, 186, 1, 68, 190, 2, 73, 44, 4, 65, 76, 80, 72, 0, - 4, 79, 77, 69, 71, 32, 4, 74, 79, 84, 32, 36, 4, 76, 69, 70, 84, 68, 2, - 81, 85, 218, 3, 82, 54, 83, 92, 4, 69, 80, 83, 73, 32, 6, 66, 65, 67, 75, - 83, 76, 76, 2, 85, 80, 156, 136, 16, 12, 71, 82, 69, 65, 84, 69, 82, 45, - 84, 72, 65, 78, 0, 5, 84, 73, 76, 68, 69, 203, 145, 20, 90, 14, 64, 6, - 73, 82, 67, 76, 69, 32, 169, 230, 38, 4, 79, 77, 77, 65, 12, 80, 2, 83, - 84, 190, 206, 19, 68, 234, 131, 6, 66, 246, 201, 10, 85, 215, 154, 3, 74, - 4, 250, 226, 38, 73, 207, 107, 65, 22, 76, 2, 69, 76, 120, 3, 79, 87, 78, - 129, 244, 31, 6, 73, 65, 77, 79, 78, 68, 10, 30, 32, 53, 3, 84, 65, 32, - 6, 170, 205, 19, 68, 142, 213, 16, 84, 223, 190, 1, 83, 4, 210, 154, 36, - 85, 143, 198, 1, 83, 10, 22, 32, 167, 9, 87, 8, 52, 5, 84, 65, 67, 75, - 32, 182, 1, 83, 227, 6, 67, 4, 222, 153, 36, 85, 215, 154, 3, 74, 6, 40, - 2, 79, 84, 225, 129, 32, 2, 45, 66, 4, 11, 65, 5, 223, 242, 31, 32, 4, - 146, 203, 19, 68, 223, 205, 16, 85, 4, 28, 2, 32, 83, 187, 7, 87, 2, 169, - 222, 37, 4, 72, 79, 69, 32, 46, 44, 2, 65, 68, 133, 3, 4, 79, 84, 69, 32, - 43, 11, 32, 40, 178, 1, 67, 38, 68, 92, 2, 85, 80, 36, 2, 76, 69, 142, - 186, 6, 81, 176, 216, 13, 3, 78, 79, 84, 22, 69, 178, 184, 5, 66, 194, - 158, 7, 83, 170, 132, 5, 71, 162, 114, 82, 195, 79, 74, 4, 178, 175, 38, - 73, 251, 129, 1, 79, 12, 54, 73, 36, 3, 79, 87, 78, 213, 199, 36, 2, 69, - 76, 4, 130, 221, 31, 86, 251, 230, 6, 65, 4, 222, 245, 3, 32, 255, 234, - 34, 87, 4, 162, 238, 37, 83, 255, 113, 70, 4, 238, 153, 12, 81, 231, 250, - 23, 85, 4, 184, 3, 5, 73, 71, 72, 84, 87, 143, 221, 39, 72, 10, 88, 5, - 69, 77, 73, 67, 79, 34, 76, 34, 84, 253, 151, 12, 7, 81, 85, 73, 83, 72, - 32, 81, 2, 237, 236, 31, 3, 76, 79, 78, 2, 185, 220, 38, 3, 65, 83, 72, - 4, 196, 136, 16, 2, 65, 82, 163, 166, 7, 73, 12, 22, 32, 171, 1, 87, 10, - 78, 67, 36, 5, 84, 65, 67, 75, 32, 253, 171, 39, 6, 83, 72, 79, 69, 32, - 74, 2, 229, 173, 23, 4, 65, 82, 69, 84, 6, 202, 195, 19, 68, 198, 138, - 17, 79, 239, 221, 2, 74, 2, 209, 222, 36, 6, 65, 82, 68, 83, 32, 86, 4, - 178, 227, 33, 83, 171, 200, 5, 76, 12, 92, 2, 82, 79, 177, 159, 26, 15, - 76, 73, 67, 65, 84, 73, 79, 78, 32, 80, 82, 79, 71, 82, 65, 10, 112, 9, - 88, 73, 77, 65, 84, 69, 76, 89, 32, 165, 221, 39, 13, 65, 67, 72, 69, 83, - 32, 84, 72, 69, 32, 76, 73, 77, 8, 76, 6, 69, 81, 85, 65, 76, 32, 245, - 242, 24, 7, 66, 85, 84, 32, 78, 79, 84, 6, 32, 2, 84, 79, 135, 232, 31, - 79, 5, 185, 160, 26, 13, 32, 79, 82, 32, 84, 72, 69, 32, 73, 77, 65, 71, - 69, 240, 22, 148, 1, 4, 65, 66, 73, 67, 156, 246, 1, 7, 77, 69, 78, 73, - 65, 78, 32, 224, 12, 3, 82, 79, 87, 228, 3, 2, 84, 73, 174, 220, 36, 73, - 251, 147, 1, 67, 158, 21, 54, 32, 217, 244, 1, 7, 45, 73, 78, 68, 73, 67, - 32, 130, 21, 178, 3, 67, 238, 1, 68, 246, 2, 69, 186, 1, 70, 212, 1, 2, - 72, 65, 104, 5, 75, 65, 83, 82, 65, 86, 76, 248, 167, 1, 2, 77, 65, 204, - 18, 7, 78, 85, 77, 66, 69, 82, 32, 36, 5, 79, 80, 69, 78, 32, 82, 80, - 238, 1, 82, 156, 2, 8, 66, 65, 83, 69, 76, 73, 78, 69, 52, 6, 73, 78, 86, - 69, 82, 84, 98, 83, 150, 33, 84, 202, 4, 86, 232, 184, 17, 9, 87, 65, 86, - 89, 32, 72, 65, 77, 90, 190, 249, 17, 81, 145, 197, 1, 6, 90, 87, 65, 82, - 65, 75, 16, 44, 2, 79, 77, 81, 5, 85, 82, 76, 89, 32, 4, 216, 158, 19, - 11, 66, 73, 78, 73, 78, 71, 32, 65, 76, 69, 70, 207, 209, 20, 77, 12, 72, - 4, 68, 65, 77, 77, 0, 4, 70, 65, 84, 72, 1, 4, 75, 65, 83, 82, 4, 11, 65, - 5, 183, 167, 38, 84, 24, 158, 1, 65, 108, 5, 79, 85, 66, 76, 69, 188, - 206, 23, 5, 69, 67, 73, 77, 65, 209, 192, 9, 16, 73, 83, 80, 85, 84, 69, - 68, 32, 69, 78, 68, 32, 79, 70, 32, 65, 14, 36, 3, 77, 77, 65, 147, 178, - 33, 84, 13, 22, 32, 199, 5, 84, 6, 154, 226, 1, 73, 54, 77, 135, 233, 36, - 87, 6, 190, 4, 68, 165, 193, 1, 18, 32, 82, 73, 71, 72, 84, 32, 65, 82, - 82, 79, 87, 72, 69, 65, 68, 32, 65, 8, 88, 12, 77, 80, 84, 89, 32, 67, - 69, 78, 84, 82, 69, 32, 57, 6, 78, 68, 32, 79, 70, 32, 4, 144, 156, 37, - 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 4, 146, 223, 29, 84, 255, 173, 3, - 65, 22, 84, 4, 65, 84, 72, 65, 238, 209, 1, 79, 156, 223, 4, 3, 73, 86, - 69, 195, 233, 30, 85, 17, 22, 32, 139, 2, 84, 10, 52, 5, 87, 73, 84, 72, - 32, 170, 222, 1, 73, 55, 77, 6, 250, 154, 38, 84, 138, 80, 68, 203, 47, - 82, 6, 72, 13, 76, 70, 32, 77, 65, 68, 68, 65, 32, 79, 86, 69, 82, 235, - 16, 77, 2, 165, 205, 17, 2, 32, 77, 13, 18, 32, 43, 84, 6, 254, 71, 87, - 130, 149, 1, 73, 55, 77, 4, 173, 82, 2, 65, 78, 230, 15, 84, 5, 65, 82, - 71, 69, 32, 146, 1, 69, 149, 90, 8, 73, 71, 65, 84, 85, 82, 69, 32, 8, - 64, 10, 82, 79, 85, 78, 68, 32, 68, 79, 84, 32, 179, 245, 10, 67, 6, 148, - 245, 10, 6, 73, 78, 83, 73, 68, 69, 162, 213, 26, 66, 167, 161, 1, 65, - 130, 8, 80, 5, 84, 84, 69, 82, 32, 185, 150, 7, 9, 70, 84, 32, 65, 82, - 82, 79, 87, 72, 254, 7, 210, 2, 65, 240, 10, 2, 66, 69, 150, 4, 68, 186, - 5, 70, 246, 4, 71, 226, 2, 72, 156, 9, 2, 74, 69, 154, 1, 75, 130, 5, 76, - 222, 2, 77, 154, 1, 78, 140, 3, 3, 80, 69, 72, 176, 1, 3, 81, 65, 70, - 214, 1, 82, 230, 3, 83, 162, 9, 84, 174, 8, 85, 148, 2, 2, 86, 69, 28, 3, - 87, 65, 87, 202, 1, 89, 246, 3, 79, 236, 2, 2, 90, 65, 31, 69, 112, 92, - 7, 70, 82, 73, 67, 65, 78, 32, 92, 2, 73, 78, 144, 2, 3, 76, 69, 70, 131, - 225, 39, 69, 8, 52, 3, 81, 65, 70, 218, 251, 32, 70, 139, 193, 3, 78, 5, - 241, 63, 5, 32, 87, 73, 84, 72, 21, 11, 32, 18, 72, 6, 87, 73, 84, 72, - 32, 84, 150, 160, 1, 70, 238, 43, 73, 199, 9, 77, 10, 60, 10, 72, 82, 69, - 69, 32, 68, 79, 84, 83, 32, 147, 54, 87, 6, 160, 250, 14, 17, 80, 79, 73, - 78, 84, 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 198, 201, 22, 66, - 167, 161, 1, 65, 83, 11, 32, 80, 70, 87, 212, 44, 6, 77, 65, 75, 83, 85, - 82, 182, 113, 70, 255, 52, 73, 70, 48, 4, 73, 84, 72, 32, 161, 44, 3, 65, - 83, 76, 64, 192, 2, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, 220, 1, 19, - 82, 73, 71, 72, 84, 32, 77, 73, 68, 68, 76, 69, 32, 83, 84, 82, 79, 75, - 69, 188, 1, 6, 72, 65, 77, 90, 65, 32, 44, 8, 87, 65, 86, 89, 32, 72, 65, - 77, 134, 70, 69, 204, 1, 4, 77, 65, 68, 68, 136, 165, 36, 9, 76, 69, 70, - 84, 32, 77, 73, 68, 68, 207, 236, 1, 68, 28, 204, 1, 17, 66, 79, 84, 84, - 79, 77, 32, 82, 73, 71, 72, 84, 32, 75, 65, 83, 82, 0, 14, 84, 79, 80, - 32, 82, 73, 71, 72, 84, 32, 70, 65, 84, 72, 94, 82, 56, 3, 76, 69, 70, - 190, 201, 1, 75, 207, 149, 32, 70, 6, 11, 65, 7, 29, 5, 32, 65, 78, 68, - 32, 4, 240, 170, 19, 3, 76, 69, 70, 251, 175, 19, 68, 8, 52, 3, 73, 71, - 72, 253, 214, 1, 4, 79, 85, 78, 68, 4, 17, 2, 84, 32, 4, 166, 176, 1, 82, - 223, 37, 72, 12, 154, 72, 65, 37, 5, 66, 69, 76, 79, 87, 4, 179, 168, 37, - 90, 50, 22, 72, 227, 63, 69, 41, 22, 32, 203, 63, 69, 28, 68, 5, 87, 73, - 84, 72, 32, 194, 150, 1, 70, 238, 43, 73, 199, 9, 77, 20, 116, 6, 83, 77, - 65, 76, 76, 32, 34, 84, 254, 18, 73, 248, 32, 10, 68, 79, 84, 32, 66, 69, - 76, 79, 87, 32, 191, 20, 72, 6, 242, 38, 77, 231, 184, 23, 86, 8, 88, 10, - 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 193, 48, 7, 87, 79, 32, 68, 79, - 84, 83, 6, 144, 1, 22, 80, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, - 65, 82, 68, 83, 32, 66, 69, 76, 79, 87, 161, 54, 8, 72, 79, 82, 73, 90, - 79, 78, 84, 5, 171, 237, 14, 32, 78, 90, 65, 136, 4, 2, 68, 65, 40, 7, - 79, 84, 76, 69, 83, 83, 32, 234, 27, 89, 247, 25, 85, 44, 34, 76, 254, 3, - 72, 235, 45, 68, 27, 11, 32, 24, 56, 5, 87, 73, 84, 72, 32, 222, 145, 1, - 70, 255, 52, 73, 20, 108, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 34, 84, - 168, 1, 3, 68, 79, 84, 250, 246, 11, 70, 207, 137, 27, 82, 4, 238, 14, - 83, 163, 196, 39, 86, 8, 132, 1, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, - 32, 33, 18, 87, 79, 32, 68, 79, 84, 83, 32, 86, 69, 82, 84, 73, 67, 65, - 76, 76, 89, 4, 182, 54, 65, 163, 253, 36, 66, 4, 33, 6, 32, 66, 69, 76, - 79, 87, 5, 157, 154, 15, 11, 32, 65, 78, 68, 32, 83, 77, 65, 76, 76, 32, - 12, 22, 72, 243, 62, 76, 6, 235, 53, 65, 6, 222, 232, 32, 66, 2, 70, 175, - 230, 5, 81, 44, 60, 8, 65, 82, 83, 73, 32, 89, 69, 72, 169, 2, 2, 69, 72, - 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 218, 140, 1, 70, 238, 43, 73, - 199, 9, 77, 12, 144, 1, 28, 69, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, - 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 30, - 84, 207, 39, 73, 6, 250, 8, 70, 255, 49, 84, 4, 150, 145, 7, 72, 179, - 212, 7, 87, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 178, 138, 1, 70, - 238, 43, 73, 199, 9, 77, 12, 32, 4, 68, 79, 84, 32, 51, 84, 6, 174, 40, - 66, 229, 190, 15, 4, 77, 79, 86, 69, 6, 64, 10, 72, 82, 69, 69, 32, 68, - 79, 84, 83, 32, 199, 157, 37, 87, 4, 198, 17, 80, 231, 155, 37, 66, 44, - 72, 2, 65, 70, 160, 1, 4, 72, 65, 73, 78, 222, 20, 85, 167, 179, 38, 82, - 19, 11, 32, 16, 68, 5, 87, 73, 84, 72, 32, 218, 135, 1, 70, 238, 43, 73, - 199, 9, 77, 8, 206, 23, 84, 170, 225, 38, 82, 241, 32, 8, 73, 78, 86, 69, - 82, 84, 69, 68, 15, 11, 32, 12, 68, 5, 87, 73, 84, 72, 32, 186, 134, 1, - 70, 238, 43, 73, 199, 9, 77, 4, 174, 37, 84, 131, 228, 26, 68, 82, 78, - 65, 236, 5, 2, 69, 72, 161, 2, 9, 73, 71, 72, 32, 72, 65, 77, 90, 65, 34, - 34, 72, 249, 47, 3, 77, 90, 65, 31, 11, 32, 28, 68, 5, 87, 73, 84, 72, - 32, 210, 132, 1, 70, 238, 43, 73, 199, 9, 77, 20, 132, 2, 29, 69, 88, 84, - 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, - 32, 68, 73, 71, 73, 84, 32, 70, 30, 73, 92, 24, 83, 77, 65, 76, 76, 32, - 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, - 62, 84, 159, 52, 72, 2, 173, 186, 1, 2, 79, 85, 2, 45, 9, 78, 86, 69, 82, - 84, 69, 68, 32, 83, 2, 217, 150, 37, 6, 77, 65, 76, 76, 32, 86, 6, 26, - 65, 203, 165, 37, 66, 4, 222, 30, 78, 155, 168, 38, 66, 8, 88, 10, 72, - 82, 69, 69, 32, 68, 79, 84, 83, 32, 33, 8, 87, 79, 32, 68, 79, 84, 83, - 32, 4, 230, 8, 80, 139, 189, 38, 65, 4, 180, 197, 38, 8, 86, 69, 82, 84, - 73, 67, 65, 76, 27, 65, 41, 11, 32, 38, 144, 1, 4, 71, 79, 65, 76, 88, 5, - 87, 73, 84, 72, 32, 128, 47, 10, 68, 79, 65, 67, 72, 65, 83, 72, 77, 69, - 230, 78, 70, 238, 43, 73, 199, 9, 77, 13, 11, 32, 10, 180, 49, 6, 87, 73, - 84, 72, 32, 72, 250, 76, 70, 238, 43, 73, 199, 9, 77, 8, 134, 26, 73, - 237, 19, 3, 89, 69, 72, 9, 11, 32, 6, 138, 206, 23, 65, 238, 137, 9, 89, - 179, 233, 5, 87, 22, 28, 2, 69, 77, 167, 45, 72, 17, 11, 32, 14, 68, 6, - 87, 73, 84, 72, 32, 84, 138, 124, 70, 238, 43, 73, 199, 9, 77, 6, 166, - 214, 14, 87, 179, 201, 18, 72, 70, 98, 65, 204, 1, 4, 69, 72, 69, 72, - 176, 2, 7, 73, 82, 71, 72, 73, 90, 32, 197, 31, 2, 72, 65, 24, 46, 70, - 177, 162, 1, 5, 83, 72, 77, 73, 82, 23, 11, 32, 20, 64, 5, 87, 73, 84, - 72, 32, 130, 122, 70, 238, 43, 73, 199, 9, 77, 12, 42, 84, 134, 161, 35, - 68, 243, 201, 3, 82, 6, 222, 27, 87, 251, 241, 36, 72, 25, 11, 32, 22, - 64, 5, 87, 73, 84, 72, 32, 230, 120, 70, 238, 43, 73, 199, 9, 77, 14, 38, - 84, 234, 41, 83, 175, 144, 38, 68, 10, 60, 10, 72, 82, 69, 69, 32, 68, - 79, 84, 83, 32, 139, 26, 87, 6, 42, 80, 230, 155, 37, 66, 167, 161, 1, - 65, 2, 161, 140, 37, 14, 79, 73, 78, 84, 73, 78, 71, 32, 85, 80, 87, 65, - 82, 68, 12, 190, 39, 79, 13, 2, 89, 85, 26, 40, 2, 65, 77, 233, 170, 1, - 2, 79, 87, 25, 11, 32, 22, 64, 5, 87, 73, 84, 72, 32, 234, 117, 70, 238, - 43, 73, 199, 9, 77, 14, 88, 2, 68, 79, 36, 6, 83, 77, 65, 76, 76, 32, - 184, 152, 33, 2, 84, 72, 159, 253, 4, 66, 4, 146, 229, 17, 85, 155, 213, - 20, 84, 4, 196, 27, 16, 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, 69, - 82, 32, 84, 65, 199, 155, 39, 86, 18, 36, 3, 69, 69, 77, 231, 244, 38, - 65, 17, 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, 146, 115, 70, 238, 43, 73, - 199, 9, 77, 6, 134, 18, 84, 187, 136, 35, 68, 60, 38, 71, 26, 89, 17, 3, - 79, 79, 78, 21, 22, 79, 235, 109, 32, 10, 239, 26, 69, 31, 11, 32, 28, - 92, 5, 71, 72, 85, 78, 78, 16, 5, 87, 73, 84, 72, 32, 166, 113, 70, 238, - 43, 73, 199, 9, 77, 6, 251, 33, 65, 14, 116, 6, 83, 77, 65, 76, 76, 32, - 38, 84, 176, 24, 8, 73, 78, 86, 69, 82, 84, 69, 68, 210, 218, 26, 68, - 171, 238, 11, 82, 4, 206, 210, 32, 84, 131, 224, 6, 86, 4, 246, 245, 6, - 72, 191, 142, 30, 87, 25, 22, 32, 143, 24, 69, 12, 88, 11, 87, 73, 84, - 72, 32, 83, 77, 65, 76, 76, 32, 242, 110, 70, 238, 43, 73, 199, 9, 77, 4, - 26, 77, 251, 176, 39, 86, 2, 241, 179, 38, 3, 69, 69, 77, 19, 11, 32, 16, - 64, 5, 87, 73, 84, 72, 32, 230, 109, 70, 238, 43, 73, 199, 9, 77, 8, 36, - 4, 68, 79, 84, 32, 183, 12, 84, 6, 44, 5, 66, 69, 76, 79, 87, 199, 178, - 38, 65, 5, 169, 199, 14, 6, 32, 65, 78, 68, 32, 78, 52, 108, 2, 69, 72, - 184, 28, 3, 82, 69, 72, 252, 2, 4, 78, 79, 79, 78, 205, 118, 7, 79, 72, - 73, 78, 71, 89, 65, 35, 11, 32, 32, 52, 5, 87, 73, 84, 72, 32, 174, 107, - 70, 255, 52, 73, 28, 134, 1, 83, 96, 2, 84, 87, 234, 5, 73, 230, 22, 72, - 206, 180, 11, 70, 144, 133, 7, 5, 68, 79, 84, 32, 66, 234, 39, 76, 215, - 220, 19, 82, 10, 44, 5, 77, 65, 76, 76, 32, 171, 252, 38, 84, 8, 198, 6, - 65, 186, 226, 6, 78, 135, 203, 16, 86, 4, 37, 7, 79, 32, 68, 79, 84, 83, - 32, 4, 166, 8, 86, 175, 166, 38, 65, 60, 184, 1, 2, 65, 68, 120, 3, 69, - 69, 78, 136, 6, 4, 72, 69, 69, 78, 204, 148, 1, 4, 85, 80, 69, 82, 180, - 172, 4, 7, 84, 82, 65, 73, 71, 72, 84, 181, 224, 32, 6, 87, 65, 83, 72, - 32, 75, 17, 11, 32, 14, 68, 6, 87, 73, 84, 72, 32, 84, 238, 102, 70, 238, - 43, 73, 199, 9, 77, 6, 186, 138, 33, 72, 223, 240, 3, 87, 27, 11, 32, 24, - 64, 5, 87, 73, 84, 72, 32, 250, 101, 70, 238, 43, 73, 199, 9, 77, 16, - 232, 1, 3, 68, 79, 84, 50, 73, 48, 7, 83, 77, 65, 76, 76, 32, 65, 110, - 84, 150, 202, 11, 70, 221, 152, 5, 31, 69, 88, 84, 69, 78, 68, 69, 68, - 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, - 84, 32, 70, 79, 85, 2, 149, 209, 18, 7, 32, 66, 69, 76, 79, 87, 32, 2, - 173, 186, 17, 7, 78, 86, 69, 82, 84, 69, 68, 2, 85, 19, 82, 65, 66, 73, - 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 72, 32, 65, 78, 2, 179, 247, - 22, 68, 6, 96, 11, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 66, 105, 9, - 87, 79, 32, 68, 79, 84, 83, 32, 86, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, - 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 84, 2, 247, 230, 6, 72, 2, 237, 198, - 11, 8, 69, 82, 84, 73, 67, 65, 76, 76, 13, 11, 32, 10, 46, 87, 134, 96, - 70, 238, 43, 73, 199, 9, 77, 2, 249, 226, 26, 5, 73, 84, 72, 32, 68, 120, - 92, 2, 65, 72, 132, 2, 4, 67, 72, 69, 72, 124, 2, 69, 72, 150, 3, 72, 61, - 3, 84, 69, 72, 21, 11, 32, 18, 64, 5, 87, 73, 84, 72, 32, 174, 94, 70, - 238, 43, 73, 199, 9, 77, 10, 26, 84, 139, 225, 26, 68, 8, 26, 87, 199, - 129, 33, 72, 4, 37, 7, 79, 32, 68, 79, 84, 83, 32, 4, 48, 6, 86, 69, 82, - 84, 73, 67, 211, 162, 38, 65, 2, 165, 195, 11, 3, 65, 76, 76, 25, 22, 32, - 163, 5, 69, 12, 64, 5, 87, 73, 84, 72, 32, 158, 92, 70, 238, 43, 73, 199, - 9, 77, 4, 198, 13, 83, 175, 144, 38, 68, 37, 22, 32, 167, 4, 69, 24, 62, - 77, 116, 5, 87, 73, 84, 72, 32, 178, 90, 70, 239, 43, 73, 10, 48, 6, 65, - 82, 66, 85, 84, 65, 175, 144, 1, 69, 9, 11, 32, 6, 226, 90, 70, 254, 52, - 73, 233, 131, 37, 2, 71, 79, 8, 104, 6, 83, 77, 65, 76, 76, 32, 56, 12, - 84, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 65, 175, 202, 38, 82, 4, 32, - 2, 84, 69, 199, 155, 39, 86, 2, 191, 158, 38, 72, 2, 161, 233, 27, 4, 66, - 79, 86, 69, 18, 42, 65, 126, 69, 209, 129, 1, 2, 73, 78, 6, 131, 9, 76, - 23, 18, 32, 91, 69, 10, 60, 4, 87, 73, 84, 72, 218, 87, 70, 238, 43, 73, - 199, 9, 77, 2, 129, 9, 2, 32, 83, 10, 131, 11, 72, 15, 158, 1, 32, 169, - 80, 33, 73, 71, 72, 85, 82, 32, 75, 65, 90, 65, 75, 72, 32, 75, 73, 82, - 71, 72, 73, 90, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 8, - 96, 16, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, - 174, 85, 70, 255, 52, 73, 5, 143, 75, 32, 17, 222, 8, 72, 167, 76, 32, - 25, 11, 32, 22, 52, 5, 87, 73, 84, 72, 32, 190, 84, 70, 255, 52, 73, 18, - 80, 4, 68, 79, 84, 32, 162, 2, 69, 182, 1, 72, 170, 170, 14, 84, 143, - 151, 24, 82, 4, 164, 152, 31, 3, 87, 73, 84, 251, 128, 7, 65, 54, 28, 2, - 69, 72, 227, 3, 85, 49, 11, 32, 46, 100, 6, 66, 65, 82, 82, 69, 69, 252, - 2, 5, 87, 73, 84, 72, 32, 170, 79, 70, 238, 43, 73, 199, 9, 77, 17, 11, - 32, 14, 52, 5, 87, 73, 84, 72, 32, 226, 81, 70, 255, 52, 73, 10, 22, 69, - 183, 1, 72, 4, 121, 28, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, - 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 84, 4, 184, - 239, 35, 3, 72, 82, 69, 177, 166, 2, 2, 87, 79, 6, 21, 3, 65, 77, 90, 6, - 11, 65, 6, 17, 2, 32, 65, 6, 21, 3, 66, 79, 86, 6, 11, 69, 7, 159, 79, - 32, 22, 64, 10, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 18, 83, 39, 84, - 10, 167, 2, 69, 2, 165, 165, 17, 4, 77, 65, 76, 76, 10, 108, 18, 87, 79, - 32, 68, 79, 84, 83, 32, 66, 69, 76, 79, 87, 32, 65, 78, 68, 32, 146, 220, - 36, 65, 183, 5, 72, 6, 70, 72, 132, 204, 6, 7, 83, 77, 65, 76, 76, 32, - 78, 207, 194, 31, 68, 2, 169, 236, 33, 3, 65, 77, 90, 18, 26, 72, 17, 2, - 73, 78, 11, 243, 71, 32, 9, 11, 32, 6, 158, 76, 70, 254, 52, 73, 157, 11, - 13, 87, 73, 84, 72, 32, 73, 78, 86, 69, 82, 84, 69, 68, 220, 7, 230, 5, - 65, 158, 5, 66, 176, 2, 9, 68, 65, 68, 32, 87, 73, 84, 72, 32, 80, 9, 70, - 69, 72, 32, 87, 73, 84, 72, 32, 72, 11, 71, 72, 65, 73, 78, 32, 87, 73, - 84, 72, 32, 154, 1, 72, 134, 3, 74, 190, 2, 75, 252, 1, 9, 76, 65, 77, - 32, 87, 73, 84, 72, 32, 226, 4, 77, 152, 4, 10, 78, 79, 79, 78, 32, 87, - 73, 84, 72, 32, 182, 2, 81, 190, 2, 82, 178, 2, 83, 146, 16, 84, 196, 9, - 7, 87, 65, 83, 65, 76, 76, 65, 40, 9, 89, 69, 72, 32, 87, 73, 84, 72, 32, - 216, 3, 53, 85, 73, 71, 72, 85, 82, 32, 75, 73, 82, 71, 72, 73, 90, 32, - 89, 69, 72, 32, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, - 86, 69, 32, 87, 73, 84, 72, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, - 82, 65, 205, 6, 14, 90, 65, 72, 32, 87, 73, 84, 72, 32, 77, 69, 69, 77, - 32, 54, 128, 1, 8, 73, 78, 32, 87, 73, 84, 72, 32, 70, 76, 232, 57, 4, - 75, 66, 65, 82, 129, 197, 36, 8, 90, 90, 65, 32, 87, 65, 32, 74, 28, 152, - 19, 4, 77, 69, 69, 77, 150, 16, 74, 238, 23, 65, 255, 8, 89, 22, 56, 3, - 65, 89, 72, 216, 1, 3, 69, 70, 32, 207, 13, 76, 12, 30, 73, 122, 65, 151, - 56, 69, 8, 52, 9, 32, 65, 83, 45, 83, 65, 76, 65, 65, 47, 77, 4, 80, 4, - 84, 85, 32, 87, 207, 132, 39, 77, 4, 18, 65, 23, 32, 2, 17, 2, 65, 32, 2, - 137, 142, 33, 6, 65, 83, 45, 83, 65, 76, 8, 212, 65, 29, 77, 65, 75, 83, - 85, 82, 65, 32, 87, 73, 84, 72, 32, 83, 85, 80, 69, 82, 83, 67, 82, 73, - 80, 84, 32, 65, 76, 69, 70, 1, 13, 87, 73, 84, 72, 32, 70, 65, 84, 72, - 65, 84, 65, 78, 44, 160, 1, 8, 69, 72, 32, 87, 73, 84, 72, 32, 177, 222, - 25, 25, 73, 83, 77, 73, 76, 76, 65, 72, 32, 65, 82, 45, 82, 65, 72, 77, - 65, 78, 32, 65, 82, 45, 82, 65, 72, 42, 98, 72, 24, 3, 75, 72, 65, 230, - 52, 65, 250, 3, 74, 78, 77, 86, 78, 78, 90, 242, 2, 82, 43, 89, 10, 22, - 65, 171, 56, 69, 6, 131, 59, 72, 36, 222, 6, 72, 158, 7, 75, 218, 38, 65, - 250, 3, 74, 2, 77, 134, 5, 82, 3, 89, 30, 170, 13, 75, 218, 38, 65, 250, + 32, 6, 166, 138, 41, 66, 2, 71, 3, 84, 10, 60, 4, 73, 71, 78, 32, 241, + 190, 23, 5, 89, 77, 66, 79, 76, 8, 50, 83, 182, 129, 26, 75, 197, 150, + 10, 2, 82, 85, 4, 160, 137, 38, 4, 77, 65, 76, 76, 203, 177, 2, 69, 22, + 66, 65, 214, 179, 37, 85, 186, 202, 1, 73, 198, 140, 2, 69, 3, 79, 11, + 206, 138, 41, 65, 2, 73, 2, 77, 3, 87, 7, 11, 32, 4, 180, 235, 35, 3, 68, + 69, 80, 221, 205, 4, 5, 65, 82, 82, 73, 86, 152, 2, 212, 1, 4, 65, 82, + 77, 32, 48, 20, 67, 72, 69, 77, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, + 76, 32, 70, 79, 82, 32, 166, 28, 69, 60, 4, 73, 69, 78, 32, 208, 3, 2, + 76, 32, 82, 77, 109, 6, 84, 69, 82, 78, 65, 84, 4, 136, 140, 14, 3, 66, + 69, 76, 227, 137, 24, 67, 232, 1, 158, 2, 65, 246, 2, 66, 210, 1, 67, + 138, 3, 68, 98, 71, 38, 72, 132, 1, 4, 73, 82, 79, 78, 82, 76, 74, 77, + 144, 1, 2, 78, 73, 34, 80, 176, 2, 3, 81, 85, 73, 74, 82, 146, 2, 83, + 210, 5, 84, 166, 1, 86, 200, 1, 2, 87, 65, 198, 162, 35, 85, 154, 220, 1, + 69, 194, 212, 1, 70, 219, 56, 79, 30, 194, 1, 76, 72, 3, 81, 85, 65, 204, + 8, 7, 78, 84, 73, 77, 79, 78, 89, 224, 157, 3, 3, 77, 65, 76, 240, 200, + 21, 3, 82, 83, 69, 204, 245, 13, 5, 85, 82, 73, 80, 73, 162, 136, 1, 83, + 183, 93, 73, 8, 224, 24, 2, 69, 77, 188, 213, 26, 4, 75, 65, 76, 73, 235, + 132, 14, 85, 10, 38, 32, 233, 136, 25, 3, 70, 79, 82, 8, 216, 7, 4, 86, + 73, 84, 65, 189, 191, 23, 4, 82, 69, 71, 73, 16, 148, 1, 7, 65, 84, 72, + 32, 79, 70, 32, 204, 6, 6, 73, 83, 77, 85, 84, 72, 144, 1, 4, 79, 82, 65, + 88, 164, 1, 4, 76, 65, 67, 75, 139, 198, 39, 82, 4, 218, 140, 17, 77, + 205, 227, 22, 5, 86, 65, 80, 79, 85, 28, 82, 65, 92, 6, 79, 80, 80, 69, + 82, 32, 34, 82, 201, 219, 39, 4, 73, 78, 78, 65, 6, 216, 143, 6, 2, 68, + 85, 140, 197, 33, 9, 80, 85, 84, 32, 77, 79, 82, 84, 85, 231, 28, 76, 4, + 230, 13, 65, 143, 207, 39, 79, 16, 72, 8, 79, 67, 85, 83, 32, 79, 70, 32, + 53, 6, 85, 67, 73, 66, 76, 69, 6, 228, 16, 5, 67, 79, 80, 80, 69, 171, + 198, 39, 73, 11, 11, 45, 8, 130, 254, 40, 50, 2, 51, 2, 52, 3, 53, 8, 44, + 2, 73, 83, 197, 255, 37, 3, 65, 89, 45, 6, 144, 2, 4, 83, 79, 76, 86, + 195, 255, 37, 84, 4, 170, 205, 30, 79, 247, 158, 10, 85, 8, 32, 4, 65, + 76, 70, 32, 47, 79, 4, 144, 236, 35, 2, 79, 85, 131, 211, 3, 68, 4, 200, + 137, 3, 4, 82, 83, 69, 32, 223, 185, 37, 85, 6, 56, 3, 32, 79, 82, 73, 7, + 45, 67, 79, 80, 80, 69, 82, 4, 211, 229, 26, 69, 4, 48, 3, 69, 65, 68, + 209, 233, 13, 3, 79, 68, 69, 2, 187, 141, 36, 32, 10, 120, 16, 69, 82, + 67, 85, 82, 89, 32, 83, 85, 66, 76, 73, 77, 65, 84, 69, 252, 217, 16, 4, + 65, 82, 67, 65, 191, 154, 22, 79, 7, 199, 248, 40, 45, 4, 234, 214, 39, + 84, 135, 120, 71, 14, 108, 11, 72, 73, 76, 79, 83, 79, 80, 72, 69, 82, + 83, 34, 79, 98, 85, 153, 177, 30, 6, 82, 69, 67, 73, 80, 73, 2, 209, 9, + 4, 32, 83, 85, 76, 6, 52, 4, 87, 68, 69, 82, 133, 189, 36, 3, 84, 32, 65, + 5, 173, 197, 39, 5, 69, 68, 32, 66, 82, 4, 252, 182, 34, 4, 84, 82, 69, + 70, 209, 174, 6, 3, 82, 73, 70, 4, 164, 229, 35, 5, 78, 84, 69, 83, 83, + 181, 247, 3, 4, 67, 75, 32, 76, 24, 58, 69, 149, 224, 26, 8, 79, 67, 75, + 32, 83, 65, 76, 84, 20, 68, 5, 71, 85, 76, 85, 83, 164, 7, 3, 65, 76, 71, + 163, 171, 26, 84, 15, 32, 4, 32, 79, 70, 32, 71, 45, 6, 164, 223, 26, 8, + 65, 78, 84, 73, 77, 79, 78, 89, 143, 238, 12, 73, 6, 162, 244, 40, 50, 2, + 51, 3, 52, 34, 164, 1, 2, 65, 76, 194, 1, 84, 142, 1, 85, 180, 226, 17, + 2, 80, 73, 156, 186, 12, 2, 73, 76, 182, 192, 9, 79, 169, 22, 11, 67, 69, + 80, 84, 69, 82, 32, 79, 70, 32, 74, 8, 58, 84, 141, 200, 39, 8, 45, 65, + 77, 77, 79, 78, 73, 65, 7, 25, 4, 32, 79, 70, 32, 4, 52, 8, 67, 79, 80, + 80, 69, 82, 32, 65, 231, 5, 65, 2, 209, 171, 30, 7, 78, 84, 73, 77, 79, + 78, 73, 6, 236, 3, 8, 65, 82, 82, 69, 68, 32, 84, 82, 233, 215, 26, 19, + 82, 65, 84, 85, 77, 32, 83, 85, 80, 69, 82, 32, 83, 84, 82, 65, 84, 85, + 77, 12, 44, 6, 66, 76, 73, 77, 65, 84, 155, 1, 76, 10, 44, 5, 69, 32, 79, + 70, 32, 195, 159, 40, 73, 8, 68, 8, 83, 65, 76, 84, 32, 79, 70, 32, 130, + 3, 65, 231, 214, 27, 67, 4, 254, 2, 65, 231, 214, 27, 67, 2, 239, 245, + 39, 70, 12, 68, 3, 65, 82, 84, 32, 2, 73, 78, 34, 82, 201, 253, 38, 2, + 85, 84, 4, 11, 65, 4, 155, 216, 26, 82, 4, 254, 206, 35, 67, 187, 49, 32, + 2, 253, 169, 40, 2, 73, 68, 14, 50, 73, 209, 197, 38, 6, 69, 82, 68, 73, + 71, 82, 12, 64, 5, 78, 69, 71, 65, 82, 213, 214, 26, 5, 84, 82, 73, 79, + 76, 9, 44, 5, 32, 79, 70, 32, 65, 243, 234, 40, 45, 2, 141, 186, 24, 3, + 78, 84, 73, 4, 214, 173, 40, 84, 239, 61, 88, 6, 38, 77, 186, 213, 39, + 70, 155, 121, 82, 2, 223, 192, 39, 66, 22, 104, 7, 77, 79, 78, 83, 84, + 69, 82, 138, 1, 83, 205, 129, 36, 10, 67, 82, 65, 66, 32, 83, 84, 69, 80, + 80, 11, 11, 32, 8, 88, 6, 67, 76, 79, 83, 69, 68, 0, 4, 79, 80, 69, 78, + 173, 227, 36, 4, 83, 84, 69, 80, 2, 145, 138, 38, 3, 32, 74, 65, 8, 60, + 6, 80, 73, 68, 69, 82, 32, 57, 5, 81, 85, 73, 68, 32, 4, 196, 195, 29, 5, + 67, 82, 79, 85, 67, 175, 250, 1, 83, 4, 56, 6, 67, 76, 79, 83, 69, 68, 1, + 4, 79, 80, 69, 78, 2, 197, 249, 27, 5, 32, 84, 69, 78, 84, 4, 210, 133, + 34, 69, 253, 184, 4, 11, 65, 82, 79, 85, 78, 68, 45, 80, 82, 79, 70, 9, + 49, 10, 79, 83, 84, 32, 69, 81, 85, 65, 76, 32, 6, 32, 2, 84, 79, 247, + 198, 32, 79, 5, 219, 255, 6, 32, 4, 136, 247, 10, 3, 73, 86, 69, 221, + 195, 28, 5, 69, 32, 79, 78, 69, 12, 162, 1, 80, 128, 208, 29, 6, 69, 82, + 73, 67, 65, 78, 204, 131, 6, 3, 66, 85, 76, 245, 254, 3, 16, 65, 76, 71, + 65, 77, 65, 84, 73, 79, 78, 32, 79, 82, 32, 67, 79, 6, 26, 72, 251, 144, + 37, 69, 4, 204, 254, 24, 3, 73, 84, 82, 203, 160, 15, 79, 194, 9, 92, 3, + 65, 84, 79, 182, 20, 71, 174, 1, 84, 190, 161, 25, 67, 190, 155, 12, 68, + 223, 142, 3, 75, 144, 9, 112, 17, 76, 73, 65, 78, 32, 72, 73, 69, 82, 79, + 71, 76, 89, 80, 72, 32, 65, 145, 218, 39, 5, 77, 73, 67, 65, 76, 142, 9, + 70, 48, 138, 4, 49, 198, 2, 50, 162, 3, 51, 174, 5, 52, 163, 3, 53, 222, + 1, 106, 50, 102, 52, 110, 54, 102, 57, 182, 7, 51, 254, 246, 11, 49, 142, + 159, 23, 48, 130, 2, 53, 2, 55, 3, 56, 22, 158, 206, 36, 54, 242, 145, 4, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 28, 202, 191, + 12, 54, 242, 141, 24, 49, 2, 53, 242, 145, 4, 48, 2, 50, 2, 51, 2, 52, 2, + 55, 2, 56, 3, 57, 26, 162, 166, 12, 54, 158, 184, 28, 48, 2, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, 234, 203, 36, 55, 2, 56, + 242, 145, 4, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 57, 232, 1, + 98, 48, 114, 49, 206, 170, 12, 50, 2, 51, 182, 242, 22, 52, 2, 53, 2, 54, + 2, 55, 2, 56, 3, 57, 42, 242, 163, 12, 52, 2, 55, 190, 24, 53, 242, 141, + 24, 48, 2, 49, 2, 50, 242, 145, 4, 51, 2, 54, 2, 56, 3, 57, 26, 190, 187, + 12, 48, 242, 141, 24, 53, 242, 145, 4, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, + 55, 2, 56, 3, 57, 222, 1, 102, 48, 110, 49, 102, 57, 210, 1, 56, 250, + 152, 12, 50, 2, 54, 146, 255, 22, 51, 2, 52, 2, 53, 3, 55, 28, 230, 185, + 12, 50, 242, 141, 24, 55, 2, 57, 242, 145, 4, 48, 2, 49, 2, 51, 2, 52, 2, + 53, 2, 54, 3, 56, 24, 234, 198, 36, 53, 2, 54, 242, 145, 4, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 24, 134, 198, 36, 52, 2, 57, 242, + 145, 4, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 55, 3, 56, 228, 1, 102, + 48, 2, 50, 2, 53, 102, 51, 110, 54, 102, 56, 162, 1, 57, 174, 129, 12, + 55, 138, 147, 23, 49, 3, 52, 22, 182, 196, 36, 57, 242, 145, 4, 48, 2, + 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 30, 166, 157, 12, + 54, 226, 204, 8, 50, 190, 235, 19, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 55, + 2, 56, 3, 57, 24, 230, 194, 36, 52, 2, 56, 242, 145, 4, 48, 2, 49, 2, 50, + 2, 51, 2, 53, 2, 54, 2, 55, 3, 57, 26, 98, 51, 162, 193, 36, 49, 2, 54, + 242, 145, 4, 48, 2, 50, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 4, 196, 154, + 21, 6, 32, 82, 65, 32, 79, 82, 203, 184, 19, 65, 20, 128, 168, 40, 3, 51, + 32, 69, 210, 42, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, + 57, 202, 1, 102, 49, 210, 1, 53, 178, 241, 20, 57, 222, 159, 14, 48, 2, + 50, 2, 51, 2, 52, 2, 54, 2, 55, 3, 56, 22, 90, 48, 162, 208, 40, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 4, 60, 6, 32, 66, + 69, 71, 73, 78, 1, 5, 65, 32, 69, 78, 68, 2, 241, 201, 16, 8, 32, 76, 79, + 71, 79, 71, 82, 65, 24, 186, 189, 36, 48, 2, 55, 242, 145, 4, 49, 2, 50, + 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 60, 226, 167, 33, 51, 198, 230, + 1, 48, 130, 2, 49, 3, 50, 14, 96, 2, 76, 69, 182, 176, 2, 85, 128, 200, + 32, 2, 83, 84, 130, 239, 3, 69, 217, 168, 1, 2, 82, 89, 7, 33, 6, 32, 87, + 73, 84, 72, 32, 4, 254, 154, 32, 83, 135, 195, 4, 85, 31, 76, 4, 69, 78, + 78, 65, 41, 11, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 5, 253, 159, + 37, 5, 32, 87, 73, 84, 72, 24, 90, 84, 162, 143, 7, 67, 58, 68, 122, 71, + 138, 4, 79, 233, 163, 28, 5, 73, 78, 84, 69, 71, 12, 84, 15, 82, 73, 65, + 78, 71, 76, 69, 45, 72, 69, 65, 68, 69, 68, 32, 223, 147, 7, 79, 10, 112, + 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 12, 6, 66, 79, 84, 84, 79, 77, 0, 3, + 84, 79, 80, 131, 147, 7, 79, 2, 11, 84, 2, 209, 235, 37, 7, 32, 85, 45, + 83, 72, 65, 80, 160, 1, 128, 1, 20, 76, 32, 70, 85, 78, 67, 84, 73, 79, + 78, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 206, 15, 79, 38, 80, 147, + 184, 40, 67, 140, 1, 222, 2, 67, 186, 1, 68, 190, 2, 73, 44, 4, 65, 76, + 80, 72, 0, 4, 79, 77, 69, 71, 32, 4, 74, 79, 84, 32, 36, 4, 76, 69, 70, + 84, 68, 2, 81, 85, 218, 3, 82, 54, 83, 92, 4, 69, 80, 83, 73, 32, 6, 66, + 65, 67, 75, 83, 76, 76, 2, 85, 80, 172, 170, 16, 12, 71, 82, 69, 65, 84, + 69, 82, 45, 84, 72, 65, 78, 0, 5, 84, 73, 76, 68, 69, 163, 168, 20, 90, + 14, 64, 6, 73, 82, 67, 76, 69, 32, 133, 161, 39, 4, 79, 77, 77, 65, 12, + 80, 2, 83, 84, 166, 242, 19, 68, 226, 137, 6, 66, 254, 216, 10, 85, 207, + 158, 3, 74, 4, 198, 157, 39, 73, 227, 109, 65, 22, 76, 2, 69, 76, 120, 3, + 79, 87, 78, 201, 162, 32, 6, 73, 65, 77, 79, 78, 68, 10, 30, 32, 53, 3, + 84, 65, 32, 6, 146, 241, 19, 68, 142, 234, 16, 84, 131, 191, 1, 83, 4, + 186, 211, 36, 85, 179, 198, 1, 83, 10, 22, 32, 167, 9, 87, 8, 52, 5, 84, + 65, 67, 75, 32, 182, 1, 83, 227, 6, 67, 4, 198, 210, 36, 85, 207, 158, 3, + 74, 6, 40, 2, 79, 84, 185, 176, 32, 2, 45, 66, 4, 11, 65, 5, 167, 161, + 32, 32, 4, 250, 238, 19, 68, 223, 226, 16, 85, 4, 28, 2, 32, 83, 187, 7, + 87, 2, 181, 151, 38, 4, 72, 79, 69, 32, 46, 44, 2, 65, 68, 133, 3, 4, 79, + 84, 69, 32, 43, 11, 32, 40, 178, 1, 67, 38, 68, 92, 2, 85, 80, 36, 2, 76, + 69, 210, 208, 6, 81, 228, 229, 13, 3, 78, 79, 84, 22, 69, 154, 190, 5, + 66, 174, 163, 7, 83, 226, 142, 5, 71, 250, 115, 82, 199, 81, 74, 4, 246, + 233, 38, 73, 151, 132, 1, 79, 12, 54, 73, 36, 3, 79, 87, 78, 225, 128, + 37, 2, 69, 76, 4, 202, 139, 32, 86, 255, 242, 6, 65, 4, 230, 138, 4, 32, + 211, 144, 35, 87, 4, 166, 167, 38, 83, 215, 115, 70, 4, 154, 181, 12, 81, + 163, 152, 24, 85, 4, 184, 3, 5, 73, 71, 72, 84, 87, 239, 153, 40, 72, 10, + 88, 5, 69, 77, 73, 67, 79, 34, 76, 34, 84, 169, 179, 12, 7, 81, 85, 73, + 83, 72, 32, 81, 2, 181, 155, 32, 3, 76, 79, 78, 2, 149, 151, 39, 3, 65, + 83, 72, 4, 212, 170, 16, 2, 65, 82, 175, 173, 7, 73, 12, 22, 32, 171, 1, + 87, 10, 78, 67, 36, 5, 84, 65, 67, 75, 32, 221, 232, 39, 6, 83, 72, 79, + 69, 32, 74, 2, 129, 215, 23, 4, 65, 82, 69, 84, 6, 178, 231, 19, 68, 234, + 159, 17, 79, 195, 225, 2, 74, 2, 221, 151, 37, 6, 65, 82, 68, 83, 32, 86, + 4, 182, 145, 34, 83, 135, 215, 5, 76, 14, 26, 76, 93, 2, 82, 79, 4, 164, + 201, 26, 14, 73, 67, 65, 84, 73, 79, 78, 32, 80, 82, 79, 71, 82, 65, 139, + 217, 12, 69, 10, 112, 9, 88, 73, 77, 65, 84, 69, 76, 89, 32, 237, 153, + 40, 13, 65, 67, 72, 69, 83, 32, 84, 72, 69, 32, 76, 73, 77, 8, 76, 6, 69, + 81, 85, 65, 76, 32, 137, 156, 25, 7, 66, 85, 84, 32, 78, 79, 84, 6, 32, + 2, 84, 79, 183, 150, 32, 79, 5, 237, 201, 26, 13, 32, 79, 82, 32, 84, 72, + 69, 32, 73, 77, 65, 71, 69, 192, 23, 148, 1, 4, 65, 66, 73, 67, 240, 131, + 2, 7, 77, 69, 78, 73, 65, 78, 32, 224, 12, 3, 82, 79, 87, 228, 3, 2, 84, + 73, 150, 137, 37, 73, 135, 150, 1, 67, 238, 21, 54, 32, 173, 130, 2, 7, + 45, 73, 78, 68, 73, 67, 32, 210, 21, 150, 3, 66, 134, 1, 67, 238, 1, 68, + 138, 3, 69, 186, 1, 70, 212, 1, 2, 72, 65, 104, 5, 75, 65, 83, 82, 65, + 86, 76, 212, 180, 1, 2, 77, 65, 204, 18, 7, 78, 85, 77, 66, 69, 82, 32, + 36, 5, 79, 80, 69, 78, 32, 82, 80, 238, 1, 82, 208, 2, 6, 73, 78, 86, 69, + 82, 84, 98, 83, 146, 33, 84, 202, 4, 86, 228, 206, 17, 9, 87, 65, 86, 89, + 32, 72, 65, 77, 90, 230, 142, 18, 81, 197, 198, 1, 6, 90, 87, 65, 82, 65, + 75, 4, 216, 214, 1, 7, 65, 83, 69, 76, 73, 78, 69, 233, 205, 37, 17, 73, + 66, 76, 73, 67, 65, 76, 32, 69, 78, 68, 32, 79, 70, 32, 86, 69, 16, 44, + 2, 79, 77, 81, 5, 85, 82, 76, 89, 32, 4, 192, 193, 19, 11, 66, 73, 78, + 73, 78, 71, 32, 65, 76, 69, 70, 199, 234, 20, 77, 12, 72, 4, 68, 65, 77, + 77, 0, 4, 70, 65, 84, 72, 1, 4, 75, 65, 83, 82, 4, 11, 65, 5, 251, 224, + 38, 84, 26, 158, 1, 65, 108, 5, 79, 85, 66, 76, 69, 216, 246, 23, 5, 69, + 67, 73, 77, 65, 149, 198, 9, 16, 73, 83, 80, 85, 84, 69, 68, 32, 69, 78, + 68, 32, 79, 70, 32, 65, 14, 36, 3, 77, 77, 65, 231, 223, 33, 84, 13, 22, + 32, 219, 5, 84, 6, 242, 238, 1, 73, 54, 77, 251, 149, 37, 87, 8, 22, 32, + 191, 4, 68, 6, 184, 210, 1, 17, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, + 87, 72, 69, 65, 68, 32, 65, 191, 30, 86, 8, 88, 12, 77, 80, 84, 89, 32, + 67, 69, 78, 84, 82, 69, 32, 57, 6, 78, 68, 32, 79, 70, 32, 4, 140, 212, + 37, 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 4, 210, 139, 30, 84, 139, 175, + 3, 65, 22, 84, 4, 65, 84, 72, 65, 166, 222, 1, 79, 148, 232, 4, 3, 73, + 86, 69, 143, 140, 31, 85, 17, 22, 32, 139, 2, 84, 10, 52, 5, 87, 73, 84, + 72, 32, 238, 234, 1, 73, 55, 77, 6, 170, 212, 38, 84, 166, 82, 68, 203, + 47, 82, 6, 72, 13, 76, 70, 32, 77, 65, 68, 68, 65, 32, 79, 86, 69, 82, + 235, 16, 77, 2, 221, 238, 17, 2, 32, 77, 13, 18, 32, 43, 84, 6, 166, 72, + 87, 158, 161, 1, 73, 55, 77, 4, 253, 82, 2, 65, 78, 174, 16, 84, 5, 65, + 82, 71, 69, 32, 146, 1, 69, 133, 91, 8, 73, 71, 65, 84, 85, 82, 69, 32, + 8, 64, 10, 82, 79, 85, 78, 68, 32, 68, 79, 84, 32, 179, 143, 11, 67, 6, + 148, 143, 11, 6, 73, 78, 83, 73, 68, 69, 146, 243, 26, 66, 131, 165, 1, + 65, 136, 8, 80, 5, 84, 84, 69, 82, 32, 137, 174, 7, 9, 70, 84, 32, 65, + 82, 82, 79, 87, 72, 132, 8, 210, 2, 65, 240, 10, 2, 66, 69, 150, 4, 68, + 186, 5, 70, 246, 4, 71, 226, 2, 72, 156, 9, 2, 74, 69, 158, 1, 75, 138, + 5, 76, 226, 2, 77, 154, 1, 78, 160, 3, 3, 80, 69, 72, 176, 1, 3, 81, 65, + 70, 214, 1, 82, 234, 3, 83, 162, 9, 84, 214, 8, 85, 148, 2, 2, 86, 69, + 28, 3, 87, 65, 87, 202, 1, 89, 246, 3, 79, 140, 3, 2, 90, 65, 31, 69, + 112, 92, 7, 70, 82, 73, 67, 65, 78, 32, 92, 2, 73, 78, 144, 2, 3, 76, 69, + 70, 207, 156, 40, 69, 8, 52, 3, 81, 65, 70, 166, 169, 33, 70, 183, 203, + 3, 78, 5, 153, 64, 5, 32, 87, 73, 84, 72, 21, 11, 32, 18, 72, 6, 87, 73, + 84, 72, 32, 84, 242, 172, 1, 70, 202, 43, 73, 211, 9, 77, 10, 60, 10, 72, + 82, 69, 69, 32, 68, 79, 84, 83, 32, 187, 54, 87, 6, 220, 154, 15, 17, 80, + 79, 73, 78, 84, 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 250, 224, + 22, 66, 131, 165, 1, 65, 83, 11, 32, 80, 70, 87, 228, 44, 6, 77, 65, 75, + 83, 85, 82, 130, 126, 70, 231, 52, 73, 70, 48, 4, 73, 84, 72, 32, 177, + 44, 3, 65, 83, 76, 64, 192, 2, 9, 65, 84, 84, 65, 67, 72, 69, 68, 32, + 220, 1, 19, 82, 73, 71, 72, 84, 32, 77, 73, 68, 68, 76, 69, 32, 83, 84, + 82, 79, 75, 69, 188, 1, 6, 72, 65, 77, 90, 65, 32, 44, 8, 87, 65, 86, 89, + 32, 72, 65, 77, 214, 70, 69, 204, 1, 4, 77, 65, 68, 68, 180, 220, 36, 9, + 76, 69, 70, 84, 32, 77, 73, 68, 68, 159, 240, 1, 68, 28, 204, 1, 17, 66, + 79, 84, 84, 79, 77, 32, 82, 73, 71, 72, 84, 32, 75, 65, 83, 82, 0, 14, + 84, 79, 80, 32, 82, 73, 71, 72, 84, 32, 70, 65, 84, 72, 94, 82, 56, 3, + 76, 69, 70, 130, 214, 1, 75, 143, 182, 32, 70, 6, 11, 65, 7, 29, 5, 32, + 65, 78, 68, 32, 4, 196, 205, 19, 3, 76, 69, 70, 243, 200, 19, 68, 8, 52, + 3, 73, 71, 72, 213, 227, 1, 4, 79, 85, 78, 68, 4, 17, 2, 84, 32, 4, 130, + 189, 1, 82, 219, 37, 72, 12, 234, 72, 65, 37, 5, 66, 69, 76, 79, 87, 4, + 171, 224, 37, 90, 50, 22, 72, 179, 64, 69, 41, 22, 32, 155, 64, 69, 28, + 68, 5, 87, 73, 84, 72, 32, 158, 163, 1, 70, 202, 43, 73, 211, 9, 77, 20, + 116, 6, 83, 77, 65, 76, 76, 32, 34, 84, 254, 18, 73, 160, 33, 10, 68, 79, + 84, 32, 66, 69, 76, 79, 87, 32, 135, 21, 72, 6, 150, 39, 77, 231, 224, + 23, 86, 8, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 233, 48, 7, + 87, 79, 32, 68, 79, 84, 83, 6, 144, 1, 22, 80, 79, 73, 78, 84, 73, 78, + 71, 32, 85, 80, 87, 65, 82, 68, 83, 32, 66, 69, 76, 79, 87, 201, 54, 8, + 72, 79, 82, 73, 90, 79, 78, 84, 5, 231, 141, 15, 32, 78, 90, 65, 136, 4, + 2, 68, 65, 40, 7, 79, 84, 76, 69, 83, 83, 32, 250, 27, 89, 147, 26, 85, + 44, 34, 76, 254, 3, 72, 147, 46, 68, 27, 11, 32, 24, 56, 5, 87, 73, 84, + 72, 32, 186, 158, 1, 70, 231, 52, 73, 20, 108, 9, 73, 78, 86, 69, 82, 84, + 69, 68, 32, 34, 84, 168, 1, 3, 68, 79, 84, 146, 145, 12, 70, 131, 171, + 27, 82, 4, 238, 14, 83, 239, 255, 39, 86, 8, 132, 1, 10, 72, 82, 69, 69, + 32, 68, 79, 84, 83, 32, 33, 18, 87, 79, 32, 68, 79, 84, 83, 32, 86, 69, + 82, 84, 73, 67, 65, 76, 76, 89, 4, 226, 54, 65, 231, 180, 37, 66, 4, 33, + 6, 32, 66, 69, 76, 79, 87, 5, 141, 187, 15, 11, 32, 65, 78, 68, 32, 83, + 77, 65, 76, 76, 32, 12, 22, 72, 195, 63, 76, 6, 151, 54, 65, 6, 170, 150, + 33, 66, 2, 70, 175, 244, 5, 81, 44, 60, 8, 65, 82, 83, 73, 32, 89, 69, + 72, 169, 2, 2, 69, 72, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 182, + 153, 1, 70, 202, 43, 73, 211, 9, 77, 12, 144, 1, 28, 69, 88, 84, 69, 78, + 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, + 73, 71, 73, 84, 32, 30, 84, 247, 39, 73, 6, 250, 8, 70, 207, 50, 84, 4, + 190, 169, 7, 72, 199, 220, 7, 87, 23, 11, 32, 20, 68, 5, 87, 73, 84, 72, + 32, 142, 151, 1, 70, 202, 43, 73, 211, 9, 77, 12, 32, 4, 68, 79, 84, 32, + 51, 84, 6, 214, 40, 66, 197, 223, 15, 4, 77, 79, 86, 69, 6, 64, 10, 72, + 82, 69, 69, 32, 68, 79, 84, 83, 32, 191, 213, 37, 87, 4, 210, 17, 80, + 203, 211, 37, 66, 44, 72, 2, 65, 70, 160, 1, 4, 72, 65, 73, 78, 238, 20, + 85, 227, 238, 38, 82, 19, 11, 32, 16, 68, 5, 87, 73, 84, 72, 32, 182, + 148, 1, 70, 202, 43, 73, 211, 9, 77, 8, 242, 23, 84, 210, 156, 39, 82, + 241, 32, 8, 73, 78, 86, 69, 82, 84, 69, 68, 15, 11, 32, 12, 68, 5, 87, + 73, 84, 72, 32, 150, 147, 1, 70, 202, 43, 73, 211, 9, 77, 4, 214, 37, 84, + 135, 140, 27, 68, 82, 78, 65, 236, 5, 2, 69, 72, 161, 2, 9, 73, 71, 72, + 32, 72, 65, 77, 90, 65, 34, 34, 72, 201, 48, 3, 77, 90, 65, 31, 11, 32, + 28, 68, 5, 87, 73, 84, 72, 32, 174, 145, 1, 70, 202, 43, 73, 211, 9, 77, + 20, 132, 2, 29, 69, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, + 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 70, 30, 73, 92, + 24, 83, 77, 65, 76, 76, 32, 65, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, + 69, 82, 32, 84, 65, 72, 32, 62, 84, 143, 53, 72, 2, 129, 199, 1, 2, 79, + 85, 2, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 83, 2, 209, 206, 37, 6, 77, + 65, 76, 76, 32, 86, 6, 26, 65, 187, 221, 37, 66, 4, 134, 31, 78, 191, + 227, 38, 66, 8, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 33, 8, + 87, 79, 32, 68, 79, 84, 83, 32, 4, 242, 8, 80, 203, 248, 38, 65, 4, 128, + 129, 39, 8, 86, 69, 82, 84, 73, 67, 65, 76, 27, 65, 41, 11, 32, 38, 144, + 1, 4, 71, 79, 65, 76, 88, 5, 87, 73, 84, 72, 32, 240, 47, 10, 68, 79, 65, + 67, 72, 65, 83, 72, 77, 69, 210, 90, 70, 202, 43, 73, 211, 9, 77, 13, 11, + 32, 10, 164, 50, 6, 87, 73, 84, 72, 32, 72, 230, 88, 70, 202, 43, 73, + 211, 9, 77, 8, 174, 26, 73, 149, 20, 3, 89, 69, 72, 9, 11, 32, 6, 174, + 246, 23, 65, 150, 143, 9, 89, 179, 247, 5, 87, 22, 28, 2, 69, 77, 247, + 45, 72, 17, 11, 32, 14, 72, 6, 87, 73, 84, 72, 32, 84, 226, 136, 1, 70, + 202, 43, 73, 211, 9, 77, 6, 222, 246, 14, 87, 227, 213, 18, 72, 70, 98, + 65, 208, 1, 4, 69, 72, 69, 72, 180, 2, 7, 73, 82, 71, 72, 73, 90, 32, + 137, 32, 2, 72, 65, 24, 46, 70, 229, 174, 1, 5, 83, 72, 77, 73, 82, 23, + 11, 32, 20, 68, 5, 87, 73, 84, 72, 32, 214, 134, 1, 70, 202, 43, 73, 211, + 9, 77, 12, 42, 84, 166, 211, 35, 68, 151, 211, 3, 82, 6, 254, 27, 87, + 203, 169, 37, 72, 25, 11, 32, 22, 68, 5, 87, 73, 84, 72, 32, 182, 133, 1, + 70, 202, 43, 73, 211, 9, 77, 14, 38, 84, 206, 42, 83, 139, 203, 38, 68, + 10, 60, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 167, 26, 87, 6, 42, + 80, 202, 211, 37, 66, 131, 165, 1, 65, 2, 141, 196, 37, 14, 79, 73, 78, + 84, 73, 78, 71, 32, 85, 80, 87, 65, 82, 68, 12, 130, 40, 79, 13, 2, 89, + 85, 26, 40, 2, 65, 77, 161, 183, 1, 2, 79, 87, 25, 11, 32, 22, 68, 5, 87, + 73, 84, 72, 32, 182, 130, 1, 70, 202, 43, 73, 211, 9, 77, 14, 88, 2, 68, + 79, 36, 6, 83, 77, 65, 76, 76, 32, 148, 197, 33, 2, 84, 72, 251, 137, 5, + 66, 4, 146, 135, 18, 85, 215, 238, 20, 84, 4, 224, 27, 16, 65, 82, 65, + 66, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 84, 65, 231, 214, 39, 86, 18, + 36, 3, 69, 69, 77, 163, 176, 39, 65, 17, 11, 32, 14, 64, 5, 87, 73, 84, + 72, 32, 222, 127, 70, 202, 43, 73, 211, 9, 77, 6, 158, 18, 84, 187, 186, + 35, 68, 62, 38, 71, 26, 89, 17, 3, 79, 79, 78, 21, 22, 79, 183, 122, 32, + 10, 175, 27, 69, 33, 11, 32, 30, 92, 5, 71, 72, 85, 78, 78, 16, 5, 87, + 73, 84, 72, 32, 242, 125, 70, 202, 43, 73, 211, 9, 77, 6, 187, 34, 65, + 16, 136, 1, 6, 83, 77, 65, 76, 76, 32, 38, 84, 220, 24, 8, 73, 78, 86, + 69, 82, 84, 69, 68, 132, 151, 24, 4, 82, 73, 78, 71, 171, 235, 2, 68, 4, + 246, 255, 32, 84, 131, 238, 6, 86, 4, 250, 141, 7, 72, 143, 174, 30, 87, + 25, 22, 32, 187, 24, 69, 12, 88, 11, 87, 73, 84, 72, 32, 83, 77, 65, 76, + 76, 32, 170, 123, 70, 202, 43, 73, 211, 9, 77, 4, 26, 77, 163, 236, 39, + 86, 2, 153, 239, 38, 3, 69, 69, 77, 19, 11, 32, 16, 64, 5, 87, 73, 84, + 72, 32, 158, 122, 70, 202, 43, 73, 211, 9, 77, 8, 36, 4, 68, 79, 84, 32, + 187, 12, 84, 6, 44, 5, 66, 69, 76, 79, 87, 239, 237, 38, 65, 5, 193, 231, + 14, 6, 32, 65, 78, 68, 32, 78, 52, 112, 2, 69, 72, 224, 28, 3, 82, 69, + 72, 156, 3, 4, 78, 79, 79, 78, 149, 130, 1, 7, 79, 72, 73, 78, 71, 89, + 65, 35, 11, 32, 32, 52, 5, 87, 73, 84, 72, 32, 226, 119, 70, 231, 52, 73, + 28, 134, 1, 83, 96, 2, 84, 87, 234, 5, 73, 174, 23, 72, 246, 205, 11, 70, + 204, 141, 7, 5, 68, 79, 84, 32, 66, 234, 39, 76, 207, 245, 19, 82, 10, + 44, 5, 77, 65, 76, 76, 32, 207, 183, 39, 84, 8, 198, 6, 65, 166, 250, 6, + 78, 151, 219, 16, 86, 4, 37, 7, 79, 32, 68, 79, 84, 83, 32, 4, 166, 8, + 86, 211, 225, 38, 65, 60, 184, 1, 2, 65, 68, 120, 3, 69, 69, 78, 136, 6, + 4, 72, 69, 69, 78, 232, 160, 1, 4, 85, 80, 69, 82, 160, 181, 4, 7, 84, + 82, 65, 73, 71, 72, 84, 209, 134, 33, 6, 87, 65, 83, 72, 32, 75, 17, 11, + 32, 14, 68, 6, 87, 73, 84, 72, 32, 84, 162, 115, 70, 202, 43, 73, 211, 9, + 77, 6, 254, 182, 33, 72, 235, 251, 3, 87, 27, 11, 32, 24, 64, 5, 87, 73, + 84, 72, 32, 174, 114, 70, 202, 43, 73, 211, 9, 77, 16, 232, 1, 3, 68, 79, + 84, 50, 73, 48, 7, 83, 77, 65, 76, 76, 32, 65, 110, 84, 134, 228, 11, 70, + 241, 159, 5, 31, 69, 88, 84, 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, + 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, 32, 70, 79, 85, 2, + 193, 243, 18, 7, 32, 66, 69, 76, 79, 87, 32, 2, 213, 219, 17, 7, 78, 86, + 69, 82, 84, 69, 68, 2, 85, 19, 82, 65, 66, 73, 67, 32, 76, 69, 84, 84, + 69, 82, 32, 84, 65, 72, 32, 65, 78, 2, 147, 159, 23, 68, 6, 96, 11, 72, + 82, 69, 69, 32, 68, 79, 84, 83, 32, 66, 105, 9, 87, 79, 32, 68, 79, 84, + 83, 32, 86, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, 2, 21, 3, 65, 78, 68, 2, + 17, 2, 32, 84, 2, 247, 254, 6, 72, 2, 221, 224, 11, 8, 69, 82, 84, 73, + 67, 65, 76, 76, 13, 11, 32, 10, 46, 87, 186, 108, 70, 202, 43, 73, 211, + 9, 77, 2, 253, 138, 27, 5, 73, 84, 72, 32, 68, 122, 92, 2, 65, 72, 136, + 2, 4, 67, 72, 69, 72, 124, 2, 69, 72, 150, 3, 72, 97, 3, 84, 69, 72, 21, + 11, 32, 18, 64, 5, 87, 73, 84, 72, 32, 226, 106, 70, 202, 43, 73, 211, 9, + 77, 10, 26, 84, 143, 137, 27, 68, 8, 26, 87, 139, 174, 33, 72, 4, 37, 7, + 79, 32, 68, 79, 84, 83, 32, 4, 48, 6, 86, 69, 82, 84, 73, 67, 247, 221, + 38, 65, 2, 197, 169, 37, 4, 65, 76, 76, 89, 25, 22, 32, 199, 5, 69, 12, + 64, 5, 87, 73, 84, 72, 32, 206, 104, 70, 202, 43, 73, 211, 9, 77, 4, 138, + 14, 83, 139, 203, 38, 68, 37, 22, 32, 203, 4, 69, 24, 62, 77, 116, 5, 87, + 73, 84, 72, 32, 226, 102, 70, 203, 43, 73, 10, 48, 6, 65, 82, 66, 85, 84, + 65, 199, 156, 1, 69, 9, 11, 32, 6, 146, 103, 70, 230, 52, 73, 213, 176, + 37, 2, 71, 79, 8, 104, 6, 83, 77, 65, 76, 76, 32, 56, 12, 84, 72, 82, 69, + 69, 32, 68, 79, 84, 83, 32, 65, 207, 133, 39, 82, 4, 32, 2, 84, 69, 231, + 214, 39, 86, 2, 223, 217, 38, 72, 2, 153, 146, 28, 4, 66, 79, 86, 69, 20, + 42, 65, 16, 3, 73, 78, 32, 147, 1, 69, 6, 167, 9, 76, 4, 146, 224, 32, + 89, 183, 203, 3, 78, 23, 18, 32, 91, 69, 10, 60, 4, 87, 73, 84, 72, 230, + 99, 70, 202, 43, 73, 211, 9, 77, 2, 161, 9, 2, 32, 83, 10, 163, 11, 72, + 15, 158, 1, 32, 181, 92, 33, 73, 71, 72, 85, 82, 32, 75, 65, 90, 65, 75, + 72, 32, 75, 73, 82, 71, 72, 73, 90, 32, 65, 76, 69, 70, 32, 77, 65, 75, + 83, 85, 82, 65, 8, 96, 16, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, + 65, 66, 79, 86, 69, 186, 97, 70, 231, 52, 73, 5, 155, 87, 32, 17, 254, 8, + 72, 147, 88, 32, 25, 11, 32, 22, 52, 5, 87, 73, 84, 72, 32, 202, 96, 70, + 231, 52, 73, 18, 80, 4, 68, 79, 84, 32, 162, 2, 69, 182, 1, 72, 150, 202, + 14, 84, 159, 178, 24, 82, 4, 152, 197, 31, 3, 87, 73, 84, 131, 143, 7, + 65, 56, 28, 2, 69, 72, 227, 3, 85, 51, 11, 32, 48, 100, 6, 66, 65, 82, + 82, 69, 69, 252, 2, 5, 87, 73, 84, 72, 32, 182, 91, 70, 202, 43, 73, 211, + 9, 77, 17, 11, 32, 14, 52, 5, 87, 73, 84, 72, 32, 238, 93, 70, 231, 52, + 73, 10, 22, 69, 183, 1, 72, 4, 121, 28, 88, 84, 69, 78, 68, 69, 68, 32, + 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 32, 68, 73, 71, 73, 84, + 32, 84, 4, 224, 166, 36, 3, 72, 82, 69, 133, 170, 2, 2, 87, 79, 6, 21, 3, + 65, 77, 90, 6, 11, 65, 6, 17, 2, 32, 65, 6, 21, 3, 66, 79, 86, 6, 11, 69, + 7, 171, 91, 32, 24, 96, 10, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 18, + 83, 38, 84, 241, 153, 37, 4, 70, 79, 85, 82, 10, 167, 2, 69, 2, 133, 198, + 17, 4, 77, 65, 76, 76, 10, 108, 18, 87, 79, 32, 68, 79, 84, 83, 32, 66, + 69, 76, 79, 87, 32, 65, 78, 68, 32, 206, 152, 37, 72, 167, 82, 65, 6, 70, + 72, 168, 227, 6, 7, 83, 77, 65, 76, 76, 32, 78, 135, 230, 31, 68, 2, 185, + 158, 34, 3, 65, 77, 90, 18, 26, 72, 17, 2, 73, 78, 11, 223, 83, 32, 9, + 11, 32, 6, 138, 88, 70, 230, 52, 73, 177, 11, 13, 87, 73, 84, 72, 32, 73, + 78, 86, 69, 82, 84, 69, 68, 158, 8, 166, 5, 65, 178, 7, 66, 176, 2, 2, + 68, 65, 184, 1, 9, 70, 69, 72, 32, 87, 73, 84, 72, 32, 72, 11, 71, 72, + 65, 73, 78, 32, 87, 73, 84, 72, 32, 154, 1, 72, 150, 4, 74, 234, 2, 75, + 216, 2, 9, 76, 65, 77, 32, 87, 73, 84, 72, 32, 226, 4, 77, 154, 4, 78, + 202, 3, 81, 146, 4, 82, 186, 5, 83, 186, 17, 84, 196, 9, 7, 87, 65, 83, + 65, 76, 76, 65, 40, 9, 89, 69, 72, 32, 87, 73, 84, 72, 32, 216, 3, 53, + 85, 73, 71, 72, 85, 82, 32, 75, 73, 82, 71, 72, 73, 90, 32, 89, 69, 72, + 32, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, + 87, 73, 84, 72, 32, 65, 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 205, + 6, 14, 90, 65, 72, 32, 87, 73, 84, 72, 32, 77, 69, 69, 77, 32, 66, 192, + 1, 8, 73, 78, 32, 87, 73, 84, 72, 32, 68, 13, 74, 74, 65, 76, 32, 65, 76, + 76, 65, 65, 72, 85, 32, 146, 1, 76, 196, 68, 4, 75, 66, 65, 82, 137, 241, + 36, 8, 90, 90, 65, 32, 87, 65, 32, 74, 28, 236, 23, 4, 77, 69, 69, 77, + 130, 22, 74, 154, 25, 65, 255, 8, 89, 4, 54, 70, 1, 9, 84, 65, 65, 65, + 76, 65, 65, 32, 70, 2, 209, 223, 28, 17, 65, 82, 65, 74, 65, 72, 85, 32, + 65, 83, 72, 45, 83, 72, 65, 82, 69, 30, 56, 3, 65, 89, 72, 156, 2, 3, 69, + 70, 32, 207, 16, 76, 20, 30, 73, 102, 65, 135, 67, 69, 14, 24, 2, 32, 65, + 55, 77, 6, 110, 83, 153, 208, 32, 6, 82, 45, 82, 65, 72, 77, 8, 18, 65, + 23, 32, 4, 17, 2, 65, 32, 4, 17, 2, 65, 83, 4, 33, 6, 45, 83, 65, 76, 65, + 65, 4, 236, 184, 33, 10, 84, 85, 32, 87, 65, 83, 45, 83, 65, 76, 135, + 133, 6, 77, 8, 236, 75, 29, 77, 65, 75, 83, 85, 82, 65, 32, 87, 73, 84, + 72, 32, 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, + 1, 13, 87, 73, 84, 72, 32, 70, 65, 84, 72, 65, 84, 65, 78, 44, 160, 1, 8, + 69, 72, 32, 87, 73, 84, 72, 32, 241, 131, 26, 25, 73, 83, 77, 73, 76, 76, + 65, 72, 32, 65, 82, 45, 82, 65, 72, 77, 65, 78, 32, 65, 82, 45, 82, 65, + 72, 42, 98, 72, 24, 3, 75, 72, 65, 254, 62, 65, 250, 3, 74, 78, 77, 86, + 78, 78, 90, 242, 2, 82, 43, 89, 10, 22, 65, 195, 66, 69, 6, 155, 69, 72, + 38, 100, 7, 68, 32, 87, 73, 84, 72, 32, 185, 28, 13, 65, 77, 65, 84, 32, + 66, 65, 82, 65, 75, 65, 65, 84, 36, 154, 8, 72, 250, 7, 75, 242, 45, 65, + 250, 3, 74, 2, 77, 134, 5, 82, 3, 89, 30, 194, 15, 75, 242, 45, 65, 250, 3, 74, 146, 2, 77, 110, 72, 139, 2, 89, 22, 64, 5, 77, 69, 69, 77, 32, - 250, 50, 65, 250, 3, 74, 135, 5, 89, 10, 40, 5, 87, 73, 84, 72, 32, 243, - 103, 73, 6, 174, 21, 77, 170, 25, 65, 203, 12, 89, 40, 80, 8, 65, 72, 32, - 87, 73, 84, 72, 32, 69, 8, 69, 72, 32, 87, 73, 84, 72, 32, 22, 236, 3, 4, - 77, 69, 69, 77, 226, 45, 65, 138, 6, 74, 247, 2, 89, 18, 164, 1, 5, 77, - 69, 69, 77, 32, 196, 11, 6, 65, 76, 69, 70, 32, 77, 150, 2, 89, 134, 38, - 74, 53, 16, 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, - 70, 8, 40, 5, 87, 73, 84, 72, 32, 223, 100, 73, 4, 254, 43, 74, 3, 77, - 30, 104, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 205, 129, 4, 11, 65, 76, - 76, 65, 74, 65, 76, 65, 76, 79, 85, 28, 62, 72, 60, 5, 77, 69, 69, 77, - 32, 186, 45, 65, 255, 8, 89, 8, 17, 2, 65, 72, 8, 11, 32, 8, 178, 29, 87, - 199, 69, 73, 12, 40, 5, 87, 73, 84, 72, 32, 179, 98, 73, 8, 166, 37, 72, - 242, 3, 65, 203, 12, 89, 64, 84, 8, 65, 70, 32, 87, 73, 84, 72, 32, 93, - 9, 72, 65, 72, 32, 87, 73, 84, 72, 32, 46, 168, 46, 2, 65, 76, 218, 1, - 74, 96, 2, 76, 65, 146, 2, 75, 14, 72, 30, 77, 239, 1, 89, 18, 54, 72, - 250, 42, 65, 250, 3, 74, 2, 77, 135, 5, 89, 2, 247, 8, 65, 74, 116, 5, - 65, 76, 69, 70, 32, 210, 1, 72, 124, 5, 74, 69, 69, 77, 32, 78, 75, 28, - 5, 77, 69, 69, 77, 32, 187, 47, 89, 20, 64, 5, 87, 73, 84, 72, 32, 138, - 44, 77, 222, 6, 70, 255, 52, 73, 12, 68, 6, 72, 65, 77, 90, 65, 32, 41, - 7, 77, 65, 68, 68, 65, 32, 65, 8, 38, 65, 249, 45, 4, 66, 69, 76, 79, 4, - 229, 45, 3, 66, 79, 86, 14, 28, 2, 65, 72, 163, 44, 69, 12, 11, 32, 12, - 40, 5, 87, 73, 84, 72, 32, 239, 92, 73, 8, 210, 35, 65, 130, 12, 77, 75, - 89, 14, 40, 5, 87, 73, 84, 72, 32, 159, 92, 73, 10, 130, 47, 74, 2, 77, - 75, 89, 8, 129, 15, 3, 72, 65, 72, 14, 150, 14, 87, 222, 33, 70, 238, 43, - 73, 199, 9, 77, 48, 84, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 193, 36, - 7, 79, 72, 65, 77, 77, 65, 68, 46, 116, 5, 65, 76, 69, 70, 32, 66, 72, 0, - 2, 75, 72, 104, 5, 74, 69, 69, 77, 32, 92, 5, 77, 69, 69, 77, 32, 43, 89, - 4, 22, 77, 219, 45, 70, 2, 173, 35, 6, 65, 75, 83, 85, 82, 65, 10, 21, 3, - 65, 72, 32, 10, 40, 5, 87, 73, 84, 72, 32, 219, 88, 73, 6, 250, 31, 74, - 2, 77, 143, 12, 89, 12, 40, 5, 87, 73, 84, 72, 32, 139, 88, 73, 8, 170, - 31, 77, 194, 7, 75, 14, 72, 195, 4, 89, 8, 246, 40, 87, 246, 2, 70, 239, - 43, 73, 2, 11, 69, 2, 143, 33, 72, 60, 134, 1, 72, 28, 5, 74, 69, 69, 77, - 32, 92, 5, 77, 69, 69, 77, 32, 242, 31, 65, 154, 5, 78, 78, 90, 134, 1, - 75, 238, 1, 82, 43, 89, 14, 146, 27, 65, 155, 9, 69, 16, 40, 5, 87, 73, - 84, 72, 32, 199, 85, 73, 12, 186, 24, 72, 242, 3, 65, 130, 12, 77, 75, - 89, 12, 206, 15, 87, 218, 25, 70, 238, 43, 73, 199, 9, 77, 28, 70, 65, - 209, 138, 32, 11, 85, 68, 68, 73, 83, 65, 32, 83, 73, 82, 82, 26, 64, 7, - 70, 32, 87, 73, 84, 72, 32, 193, 8, 4, 76, 65, 32, 85, 24, 64, 5, 77, 69, - 69, 77, 32, 238, 29, 65, 246, 6, 72, 139, 2, 89, 12, 40, 5, 87, 73, 84, - 72, 32, 231, 82, 73, 8, 34, 77, 186, 21, 72, 187, 16, 89, 2, 129, 38, 3, - 69, 69, 77, 18, 30, 65, 129, 26, 2, 69, 72, 16, 128, 1, 14, 68, 73, 32, - 65, 76, 76, 65, 65, 72, 85, 32, 65, 78, 72, 76, 6, 72, 73, 77, 65, 72, - 85, 161, 26, 4, 83, 79, 85, 76, 11, 26, 85, 231, 228, 38, 65, 6, 26, 77, - 255, 170, 37, 78, 5, 199, 228, 38, 65, 4, 26, 32, 1, 2, 77, 32, 2, 205, - 134, 32, 4, 65, 76, 76, 65, 190, 1, 138, 1, 65, 192, 6, 9, 69, 69, 78, - 32, 87, 73, 84, 72, 32, 250, 3, 72, 233, 5, 13, 85, 66, 72, 65, 65, 78, - 65, 72, 85, 32, 87, 65, 32, 46, 48, 7, 68, 32, 87, 73, 84, 72, 32, 251, - 1, 76, 32, 76, 4, 72, 65, 72, 32, 82, 77, 238, 23, 65, 138, 4, 75, 246, - 4, 82, 3, 89, 10, 22, 87, 203, 77, 73, 6, 25, 4, 73, 84, 72, 32, 6, 162, - 16, 72, 187, 16, 89, 8, 21, 3, 69, 69, 77, 8, 11, 32, 8, 208, 31, 6, 87, - 73, 84, 72, 32, 77, 155, 45, 73, 14, 26, 65, 77, 2, 76, 65, 4, 134, 22, - 77, 129, 218, 36, 11, 65, 77, 85, 72, 85, 32, 65, 76, 65, 89, 78, 10, 34, - 32, 133, 1, 3, 76, 76, 65, 4, 22, 85, 167, 84, 73, 2, 217, 8, 23, 83, 69, - 68, 32, 65, 83, 32, 75, 79, 82, 65, 78, 73, 67, 32, 83, 84, 79, 80, 32, - 83, 73, 71, 6, 98, 72, 237, 222, 38, 18, 65, 72, 85, 32, 65, 76, 65, 89, - 72, 73, 32, 87, 65, 45, 65, 65, 76, 73, 4, 228, 63, 12, 79, 85, 32, 65, - 76, 65, 89, 72, 69, 32, 87, 65, 1, 22, 85, 32, 65, 76, 65, 89, 72, 73, - 32, 87, 65, 65, 65, 76, 73, 72, 69, 69, 32, 87, 65, 45, 60, 130, 1, 72, - 100, 5, 74, 69, 69, 77, 32, 84, 5, 75, 72, 65, 72, 32, 92, 5, 77, 69, 69, - 77, 32, 234, 15, 65, 254, 8, 82, 3, 89, 12, 32, 3, 65, 72, 32, 159, 21, - 69, 8, 156, 14, 6, 87, 73, 84, 72, 32, 74, 222, 56, 73, 199, 9, 77, 10, - 52, 5, 87, 73, 84, 72, 32, 134, 70, 73, 199, 9, 77, 4, 234, 12, 65, 139, - 8, 72, 10, 34, 87, 198, 69, 73, 199, 9, 77, 4, 25, 4, 73, 84, 72, 32, 4, - 142, 12, 65, 203, 12, 89, 16, 52, 5, 87, 73, 84, 72, 32, 214, 68, 73, - 199, 9, 77, 10, 202, 7, 72, 174, 4, 74, 199, 11, 77, 82, 96, 10, 65, 68, - 68, 65, 32, 87, 73, 84, 72, 32, 161, 1, 9, 69, 69, 78, 32, 87, 73, 84, - 72, 32, 18, 96, 4, 68, 65, 77, 77, 0, 4, 75, 65, 83, 82, 138, 11, 83, - 245, 41, 6, 70, 65, 84, 72, 65, 32, 6, 11, 65, 6, 28, 2, 84, 65, 211, 52, - 32, 2, 163, 12, 78, 64, 130, 1, 72, 36, 5, 74, 69, 69, 77, 32, 52, 5, 77, - 69, 69, 77, 32, 170, 11, 65, 228, 4, 4, 75, 72, 65, 72, 154, 4, 82, 3, - 89, 18, 194, 15, 69, 229, 3, 2, 65, 72, 10, 158, 18, 87, 246, 2, 70, 238, - 43, 73, 199, 9, 77, 16, 64, 5, 87, 73, 84, 72, 32, 158, 20, 70, 238, 43, - 73, 199, 9, 77, 8, 252, 2, 2, 75, 72, 243, 15, 77, 122, 66, 65, 176, 2, - 8, 69, 72, 32, 87, 73, 84, 72, 32, 179, 4, 72, 28, 88, 11, 66, 65, 65, - 82, 65, 75, 65, 32, 87, 65, 45, 29, 7, 72, 32, 87, 73, 84, 72, 32, 2, - 245, 133, 28, 2, 84, 65, 26, 64, 5, 77, 69, 69, 77, 32, 194, 8, 65, 246, - 6, 72, 139, 2, 89, 14, 52, 5, 87, 73, 84, 72, 32, 174, 61, 73, 199, 9, - 77, 8, 34, 72, 174, 4, 77, 143, 12, 89, 4, 133, 16, 2, 65, 72, 66, 138, - 1, 72, 108, 3, 75, 72, 65, 12, 4, 74, 69, 69, 77, 92, 5, 77, 69, 69, 77, - 32, 238, 4, 65, 154, 5, 78, 78, 90, 242, 2, 82, 43, 89, 14, 32, 3, 65, - 72, 32, 227, 9, 69, 10, 40, 5, 87, 73, 84, 72, 32, 151, 59, 73, 6, 182, - 2, 77, 199, 11, 74, 10, 11, 72, 10, 11, 32, 10, 40, 5, 87, 73, 84, 72, - 32, 183, 58, 73, 6, 154, 1, 65, 62, 77, 143, 12, 89, 18, 64, 5, 87, 73, - 84, 72, 32, 226, 13, 70, 238, 43, 73, 199, 9, 77, 10, 50, 65, 62, 74, - 194, 7, 75, 14, 72, 195, 4, 89, 2, 217, 12, 11, 76, 69, 70, 32, 77, 65, - 75, 83, 85, 82, 65, 2, 225, 7, 3, 69, 69, 77, 28, 56, 2, 65, 76, 117, 8, - 69, 72, 32, 87, 73, 84, 72, 32, 2, 37, 7, 32, 87, 73, 84, 72, 32, 83, 2, - 197, 1, 15, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, - 26, 108, 3, 74, 69, 69, 126, 65, 198, 4, 77, 86, 78, 78, 90, 242, 2, 82, - 42, 89, 237, 53, 5, 72, 69, 72, 32, 77, 2, 11, 77, 2, 11, 32, 2, 151, 63, - 73, 114, 82, 65, 38, 72, 246, 4, 78, 78, 90, 38, 74, 98, 75, 42, 77, 198, - 1, 82, 43, 89, 4, 217, 2, 5, 76, 69, 70, 32, 77, 76, 22, 65, 139, 3, 69, - 72, 80, 15, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, 73, 84, 72, 32, - 147, 5, 72, 66, 118, 65, 126, 69, 42, 72, 78, 74, 18, 75, 62, 77, 86, 78, - 22, 79, 16, 2, 87, 65, 18, 89, 26, 90, 242, 2, 82, 67, 85, 12, 22, 76, - 247, 6, 69, 8, 21, 3, 69, 70, 32, 8, 34, 77, 222, 6, 70, 255, 52, 73, 4, - 181, 6, 6, 65, 75, 83, 85, 82, 65, 6, 11, 32, 6, 166, 6, 70, 239, 43, 73, - 8, 22, 69, 191, 3, 65, 4, 11, 72, 4, 11, 32, 4, 230, 4, 73, 167, 54, 77, - 4, 131, 49, 69, 2, 11, 72, 2, 11, 65, 2, 11, 72, 2, 149, 4, 2, 32, 73, 8, - 17, 2, 69, 69, 8, 11, 77, 8, 11, 32, 8, 198, 4, 70, 238, 43, 73, 199, 9, - 77, 2, 93, 2, 79, 79, 4, 231, 3, 69, 4, 215, 3, 87, 8, 186, 3, 69, 15, - 85, 2, 17, 2, 65, 73, 2, 239, 2, 78, 6, 21, 3, 69, 69, 77, 6, 11, 32, 6, - 22, 87, 227, 46, 73, 2, 141, 2, 5, 73, 84, 72, 32, 89, 4, 11, 72, 4, 11, - 65, 4, 143, 46, 72, 14, 21, 3, 69, 69, 77, 14, 11, 32, 14, 64, 5, 87, 73, - 84, 72, 32, 194, 1, 70, 238, 43, 73, 199, 9, 77, 6, 18, 77, 75, 89, 4, - 21, 3, 69, 69, 77, 4, 11, 32, 4, 18, 73, 119, 70, 2, 247, 44, 78, 2, 17, - 2, 69, 72, 2, 77, 2, 32, 70, 4, 11, 69, 4, 11, 72, 4, 11, 32, 4, 22, 70, - 255, 52, 73, 2, 209, 53, 2, 73, 78, 6, 210, 43, 73, 199, 9, 77, 166, 2, - 92, 3, 68, 68, 65, 52, 3, 82, 75, 32, 109, 11, 84, 72, 69, 77, 65, 84, - 73, 67, 65, 76, 32, 4, 152, 133, 37, 5, 32, 87, 65, 65, 74, 239, 62, 72, - 4, 58, 78, 1, 10, 83, 73, 68, 69, 87, 65, 89, 83, 32, 78, 2, 173, 137, - 35, 7, 79, 79, 78, 32, 71, 72, 85, 158, 2, 174, 2, 68, 140, 3, 8, 73, 78, - 73, 84, 73, 65, 76, 32, 222, 1, 76, 132, 2, 9, 79, 80, 69, 82, 65, 84, - 79, 82, 32, 166, 1, 83, 142, 3, 84, 190, 162, 24, 65, 102, 75, 110, 90, - 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, 2, 89, 222, 7, 72, - 190, 59, 71, 242, 253, 2, 78, 166, 165, 2, 81, 135, 3, 87, 62, 26, 79, - 235, 186, 37, 65, 58, 88, 6, 84, 76, 69, 83, 83, 32, 61, 12, 85, 66, 76, - 69, 45, 83, 84, 82, 85, 67, 75, 32, 8, 226, 212, 31, 66, 2, 70, 138, 193, - 3, 78, 167, 165, 2, 81, 50, 178, 12, 83, 210, 20, 75, 170, 140, 24, 84, - 74, 90, 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, 2, 89, 222, - 7, 72, 190, 59, 71, 242, 253, 2, 78, 218, 235, 1, 76, 190, 56, 68, 146, - 1, 81, 222, 1, 65, 171, 1, 87, 40, 182, 1, 84, 166, 9, 83, 186, 160, 24, - 72, 30, 75, 242, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 89, 154, 67, - 71, 242, 253, 2, 78, 218, 235, 1, 76, 206, 57, 81, 222, 1, 65, 171, 57, - 68, 4, 150, 209, 31, 72, 207, 230, 6, 69, 56, 48, 6, 79, 79, 80, 69, 68, - 32, 199, 167, 38, 65, 54, 202, 8, 83, 210, 20, 75, 162, 139, 24, 65, 74, - 72, 66, 84, 74, 90, 134, 107, 74, 2, 77, 178, 187, 6, 66, 2, 70, 2, 82, - 2, 89, 154, 67, 71, 242, 253, 2, 78, 218, 235, 1, 76, 190, 56, 68, 146, - 1, 81, 135, 3, 87, 4, 168, 128, 29, 23, 77, 69, 69, 77, 32, 87, 73, 84, - 72, 32, 72, 65, 72, 32, 87, 73, 84, 72, 32, 84, 65, 84, 87, 221, 162, 8, - 9, 72, 65, 72, 32, 87, 73, 84, 72, 32, 52, 84, 9, 84, 82, 69, 84, 67, 72, - 69, 68, 32, 222, 224, 36, 72, 14, 69, 203, 141, 1, 65, 46, 178, 1, 68, - 86, 84, 250, 2, 83, 186, 160, 24, 72, 30, 75, 242, 107, 74, 2, 77, 178, - 187, 6, 66, 2, 70, 2, 89, 222, 7, 90, 190, 59, 71, 242, 253, 2, 78, 166, - 165, 2, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, 83, 32, 199, 236, - 37, 65, 4, 134, 203, 31, 66, 3, 70, 6, 234, 202, 31, 72, 206, 230, 6, 65, - 3, 69, 38, 42, 65, 190, 163, 24, 72, 167, 141, 14, 69, 32, 44, 5, 73, 76, - 69, 68, 32, 195, 177, 38, 72, 30, 146, 1, 68, 94, 83, 210, 20, 75, 246, - 247, 24, 74, 178, 187, 6, 89, 222, 7, 72, 190, 59, 71, 242, 253, 2, 78, - 218, 235, 1, 76, 206, 57, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, 83, - 83, 32, 215, 233, 37, 65, 4, 158, 137, 35, 78, 167, 165, 2, 81, 6, 218, - 219, 36, 72, 14, 69, 203, 141, 1, 65, 4, 242, 197, 18, 77, 203, 226, 18, - 83, 6, 228, 235, 24, 3, 75, 65, 83, 12, 4, 68, 65, 77, 77, 1, 4, 70, 65, - 84, 72, 12, 118, 69, 38, 79, 172, 197, 12, 11, 76, 65, 67, 69, 32, 79, - 70, 32, 83, 65, 74, 165, 254, 5, 6, 73, 65, 83, 84, 82, 69, 4, 182, 157, - 32, 82, 235, 218, 5, 80, 4, 252, 181, 12, 8, 69, 84, 73, 67, 32, 86, 69, - 82, 209, 141, 6, 3, 85, 78, 68, 14, 238, 1, 65, 96, 5, 69, 86, 69, 82, - 83, 36, 15, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 72, 69, 65, 68, 32, - 225, 158, 23, 28, 79, 85, 78, 68, 69, 68, 32, 72, 73, 71, 72, 32, 83, 84, - 79, 80, 32, 87, 73, 84, 72, 32, 70, 73, 76, 76, 69, 68, 4, 40, 4, 73, 83, - 69, 68, 171, 170, 38, 89, 2, 17, 2, 32, 82, 2, 149, 222, 36, 3, 79, 85, - 78, 2, 181, 214, 21, 4, 69, 68, 32, 68, 6, 26, 65, 167, 139, 36, 66, 4, - 157, 133, 37, 3, 66, 79, 86, 202, 1, 238, 1, 69, 244, 2, 5, 72, 65, 68, - 68, 65, 36, 4, 73, 71, 78, 32, 240, 4, 5, 77, 65, 76, 76, 32, 246, 15, - 85, 240, 2, 6, 89, 77, 66, 79, 76, 32, 209, 208, 36, 18, 84, 65, 82, 84, - 32, 79, 70, 32, 82, 85, 66, 32, 69, 76, 32, 72, 73, 90, 20, 52, 7, 81, - 85, 69, 78, 67, 69, 32, 187, 219, 34, 77, 18, 184, 1, 26, 89, 69, 72, 32, - 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, 79, 86, 69, 32, 87, - 73, 84, 72, 32, 209, 189, 31, 13, 78, 79, 79, 78, 32, 87, 73, 84, 72, 32, - 75, 69, 72, 16, 70, 65, 186, 166, 37, 87, 198, 89, 89, 150, 14, 79, 214, - 22, 69, 3, 85, 6, 36, 3, 76, 69, 70, 191, 164, 38, 69, 5, 159, 13, 32, 7, - 11, 32, 4, 246, 22, 73, 55, 77, 22, 132, 1, 2, 82, 65, 158, 1, 83, 188, - 1, 7, 65, 76, 65, 89, 72, 69, 32, 160, 15, 2, 77, 73, 209, 138, 34, 6, - 84, 65, 75, 72, 65, 76, 4, 132, 1, 13, 72, 77, 65, 84, 85, 76, 76, 65, - 72, 32, 65, 76, 65, 133, 180, 33, 13, 68, 73, 32, 65, 76, 76, 65, 72, 79, - 85, 32, 65, 78, 2, 235, 244, 37, 89, 12, 46, 65, 193, 1, 6, 73, 78, 68, - 72, 73, 32, 8, 136, 1, 18, 76, 76, 65, 76, 76, 65, 72, 79, 85, 32, 65, - 76, 65, 89, 72, 69, 32, 87, 170, 192, 31, 78, 194, 229, 4, 70, 217, 102, - 2, 77, 86, 2, 17, 2, 65, 83, 2, 189, 221, 15, 3, 83, 65, 76, 4, 152, 200, - 20, 12, 80, 79, 83, 84, 80, 79, 83, 73, 84, 73, 79, 78, 185, 136, 14, 2, - 65, 77, 106, 132, 1, 2, 70, 65, 32, 5, 72, 73, 71, 72, 32, 240, 11, 4, - 76, 79, 87, 32, 110, 75, 218, 189, 21, 68, 226, 235, 9, 89, 179, 233, 5, - 87, 4, 190, 3, 82, 147, 160, 36, 84, 74, 236, 2, 17, 68, 79, 84, 76, 69, - 83, 83, 32, 72, 69, 65, 68, 32, 79, 70, 32, 75, 22, 70, 102, 76, 142, 3, - 77, 116, 4, 78, 79, 79, 78, 18, 83, 78, 84, 38, 87, 196, 2, 3, 89, 69, - 72, 224, 175, 14, 18, 85, 80, 82, 73, 71, 72, 84, 32, 82, 69, 67, 84, 65, - 78, 71, 85, 76, 65, 236, 213, 8, 7, 82, 79, 85, 78, 68, 69, 68, 234, 126, - 90, 134, 107, 74, 222, 161, 12, 81, 223, 1, 65, 2, 255, 186, 31, 72, 4, - 24, 2, 65, 82, 31, 79, 2, 11, 83, 2, 175, 2, 73, 2, 253, 180, 26, 6, 79, - 84, 78, 79, 84, 69, 10, 60, 8, 73, 71, 65, 84, 85, 82, 69, 32, 213, 11, - 2, 65, 77, 8, 88, 10, 65, 76, 69, 70, 32, 87, 73, 84, 72, 32, 116, 3, 81, - 65, 70, 1, 3, 83, 65, 68, 4, 84, 8, 76, 65, 77, 32, 87, 73, 84, 72, 209, - 168, 36, 7, 89, 69, 72, 32, 66, 65, 82, 2, 181, 176, 31, 2, 32, 89, 2, - 89, 20, 32, 87, 73, 84, 72, 32, 76, 65, 77, 32, 87, 73, 84, 72, 32, 65, - 76, 69, 70, 32, 2, 165, 145, 22, 3, 77, 65, 75, 6, 26, 69, 191, 249, 15, - 65, 4, 17, 2, 69, 77, 4, 17, 2, 32, 73, 4, 22, 78, 135, 9, 83, 2, 193, 9, - 2, 73, 84, 5, 207, 4, 32, 6, 180, 155, 36, 7, 73, 71, 78, 32, 83, 65, 70, - 218, 38, 69, 203, 141, 1, 65, 4, 178, 196, 23, 72, 235, 207, 14, 65, 20, - 40, 4, 79, 82, 68, 32, 191, 150, 37, 65, 18, 74, 65, 172, 1, 2, 83, 65, - 172, 173, 25, 3, 87, 65, 81, 207, 227, 11, 81, 10, 192, 252, 6, 3, 76, - 45, 74, 244, 185, 18, 3, 82, 45, 82, 144, 226, 6, 7, 84, 72, 45, 84, 72, - 65, 76, 164, 250, 4, 5, 78, 45, 78, 73, 83, 161, 92, 5, 83, 45, 83, 65, - 74, 4, 150, 238, 37, 75, 207, 36, 72, 5, 193, 182, 10, 12, 32, 66, 65, - 82, 82, 69, 69, 32, 87, 73, 84, 72, 20, 68, 5, 78, 79, 79, 78, 32, 70, - 87, 234, 237, 24, 77, 155, 207, 11, 83, 2, 33, 6, 87, 73, 84, 72, 32, 75, - 2, 11, 65, 2, 199, 204, 37, 83, 14, 40, 4, 79, 82, 68, 32, 171, 146, 37, - 65, 12, 106, 73, 174, 242, 15, 77, 148, 175, 9, 3, 84, 65, 83, 200, 218, - 8, 2, 83, 65, 249, 218, 3, 3, 81, 65, 83, 4, 156, 148, 13, 2, 77, 65, - 253, 132, 19, 3, 83, 72, 77, 12, 138, 1, 66, 64, 3, 75, 85, 78, 205, 239, - 24, 22, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 32, 77, - 79, 75, 72, 65, 83, 83, 2, 33, 6, 83, 67, 82, 73, 80, 84, 2, 169, 156, - 22, 2, 32, 65, 9, 11, 32, 6, 34, 73, 54, 77, 231, 238, 35, 66, 2, 11, 83, - 2, 213, 204, 31, 5, 79, 76, 65, 84, 69, 2, 11, 69, 2, 11, 68, 2, 11, 73, - 2, 237, 141, 37, 2, 65, 76, 34, 152, 1, 2, 68, 79, 114, 84, 248, 172, 5, - 8, 83, 77, 65, 76, 76, 32, 84, 65, 244, 190, 26, 4, 70, 79, 85, 82, 204, - 123, 4, 87, 65, 83, 76, 239, 209, 4, 82, 6, 88, 16, 85, 66, 76, 69, 32, - 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 135, 236, 35, 84, 2, 247, - 220, 35, 82, 16, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, 121, 8, - 87, 79, 32, 68, 79, 84, 83, 32, 8, 168, 235, 31, 17, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 146, 128, 4, 66, 167, - 161, 1, 65, 8, 152, 234, 35, 10, 86, 69, 82, 84, 73, 67, 65, 76, 76, 89, - 42, 66, 167, 161, 1, 65, 30, 202, 1, 65, 152, 2, 4, 79, 78, 69, 32, 168, - 244, 10, 12, 82, 73, 80, 76, 69, 32, 68, 79, 84, 32, 80, 85, 144, 197, 6, - 8, 85, 82, 78, 69, 68, 32, 68, 65, 217, 143, 14, 8, 72, 79, 85, 83, 65, - 78, 68, 83, 12, 68, 5, 84, 87, 69, 69, 76, 153, 235, 35, 6, 73, 76, 32, - 70, 82, 65, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 112, 11, 79, 86, 69, - 82, 83, 84, 82, 85, 67, 75, 32, 208, 193, 5, 7, 70, 65, 84, 72, 65, 84, - 65, 155, 231, 4, 84, 4, 26, 72, 143, 134, 37, 87, 2, 169, 228, 29, 2, 65, - 77, 12, 68, 3, 79, 78, 69, 214, 133, 30, 84, 201, 223, 5, 4, 76, 79, 79, - 80, 4, 221, 232, 33, 2, 32, 68, 8, 64, 10, 79, 87, 69, 76, 32, 83, 73, - 71, 78, 32, 183, 137, 24, 69, 6, 72, 10, 73, 78, 86, 69, 82, 84, 69, 68, - 32, 83, 2, 83, 255, 194, 25, 68, 2, 25, 4, 77, 65, 76, 76, 2, 133, 133, - 37, 2, 32, 86, 28, 104, 4, 80, 69, 82, 32, 232, 240, 5, 3, 67, 85, 66, - 156, 137, 5, 5, 70, 79, 85, 82, 84, 227, 149, 25, 68, 4, 158, 142, 25, - 77, 31, 84, 188, 1, 222, 1, 65, 34, 67, 138, 3, 69, 60, 7, 83, 77, 65, - 76, 76, 32, 76, 152, 128, 21, 17, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, - 69, 84, 84, 69, 82, 32, 76, 162, 248, 8, 72, 252, 182, 2, 3, 68, 82, 65, - 138, 250, 2, 70, 83, 81, 4, 198, 204, 31, 66, 139, 26, 80, 78, 80, 14, - 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 155, 172, 35, 79, - 76, 250, 1, 84, 36, 2, 89, 73, 150, 3, 67, 38, 82, 34, 69, 42, 71, 34, - 74, 38, 75, 22, 80, 38, 83, 106, 65, 22, 86, 158, 1, 76, 130, 141, 31, - 70, 2, 88, 218, 235, 1, 73, 158, 168, 3, 66, 2, 77, 198, 57, 78, 226, 26, - 90, 210, 96, 72, 190, 28, 68, 171, 1, 79, 4, 254, 167, 36, 73, 151, 212, - 1, 79, 5, 143, 172, 37, 87, 4, 138, 230, 31, 88, 169, 198, 4, 6, 77, 80, - 72, 65, 83, 73, 92, 76, 6, 69, 84, 84, 69, 82, 32, 157, 5, 8, 73, 71, 65, - 84, 85, 82, 69, 32, 80, 242, 1, 67, 38, 82, 34, 69, 42, 71, 34, 74, 38, - 75, 22, 80, 38, 83, 34, 84, 74, 65, 22, 86, 32, 2, 89, 73, 126, 76, 130, - 141, 31, 70, 2, 88, 218, 235, 1, 73, 158, 168, 3, 66, 2, 77, 198, 57, 78, - 226, 26, 90, 210, 96, 72, 190, 28, 68, 171, 1, 79, 8, 34, 72, 186, 248, - 37, 65, 3, 79, 4, 166, 247, 37, 69, 147, 1, 65, 6, 134, 247, 37, 67, 146, - 1, 72, 3, 84, 4, 194, 177, 37, 72, 215, 53, 73, 4, 242, 143, 31, 72, 223, - 231, 6, 65, 4, 211, 212, 33, 69, 4, 194, 244, 3, 73, 199, 129, 34, 69, 4, - 186, 244, 37, 72, 171, 1, 69, 6, 68, 7, 85, 82, 78, 69, 68, 32, 65, 250, - 161, 36, 73, 151, 212, 1, 79, 2, 167, 186, 36, 89, 4, 214, 247, 36, 69, - 163, 126, 79, 7, 146, 168, 21, 32, 231, 253, 15, 87, 12, 84, 5, 69, 67, - 72, 32, 89, 20, 4, 77, 69, 78, 32, 241, 237, 29, 4, 86, 69, 87, 32, 2, - 199, 160, 36, 73, 8, 234, 140, 31, 88, 218, 235, 1, 73, 226, 225, 3, 78, - 255, 119, 69, 14, 108, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 169, - 181, 36, 11, 72, 69, 65, 68, 45, 83, 72, 65, 80, 69, 68, 12, 156, 2, 24, - 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 67, 85, - 82, 86, 73, 78, 71, 32, 48, 16, 85, 80, 87, 65, 82, 68, 83, 32, 84, 72, - 69, 78, 32, 78, 79, 82, 161, 146, 35, 22, 68, 79, 87, 78, 87, 65, 82, 68, - 83, 32, 84, 72, 69, 78, 32, 67, 85, 82, 86, 73, 78, 71, 6, 44, 3, 83, 79, - 85, 198, 209, 26, 68, 35, 85, 2, 149, 147, 29, 4, 84, 72, 32, 87, 4, 168, - 240, 30, 10, 67, 85, 76, 65, 84, 69, 68, 32, 76, 79, 237, 184, 5, 6, 83, - 84, 32, 80, 65, 76, 20, 58, 67, 38, 84, 142, 153, 23, 89, 173, 191, 9, 2, - 83, 69, 4, 206, 168, 8, 69, 231, 180, 11, 73, 12, 48, 4, 69, 82, 73, 83, - 36, 2, 79, 78, 31, 82, 6, 146, 220, 2, 75, 155, 146, 35, 77, 2, 217, 246, - 13, 2, 73, 83, 4, 250, 231, 30, 65, 221, 255, 4, 22, 79, 78, 79, 77, 73, - 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 85, 82, 4, - 132, 171, 19, 6, 72, 76, 69, 84, 73, 67, 193, 174, 17, 2, 79, 77, 10, 68, - 2, 84, 79, 200, 222, 21, 2, 83, 84, 181, 169, 2, 3, 66, 69, 82, 6, 54, - 77, 129, 237, 36, 7, 32, 82, 73, 67, 75, 83, 72, 4, 174, 135, 24, 79, - 189, 197, 5, 12, 65, 84, 69, 68, 32, 84, 69, 76, 76, 69, 82, 32, 112, 56, - 6, 69, 83, 84, 65, 78, 32, 165, 223, 4, 2, 79, 67, 110, 52, 7, 76, 69, - 84, 84, 69, 82, 32, 159, 183, 31, 65, 108, 234, 1, 65, 58, 71, 42, 72, - 34, 78, 50, 88, 42, 83, 42, 89, 34, 84, 178, 146, 34, 85, 202, 141, 1, - 79, 134, 60, 73, 190, 137, 1, 66, 2, 68, 2, 90, 130, 64, 69, 206, 41, 67, - 2, 70, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 86, 17, 230, 155, 35, - 65, 242, 139, 2, 69, 162, 64, 78, 3, 79, 6, 234, 208, 37, 71, 2, 72, 215, - 22, 69, 4, 194, 208, 37, 77, 215, 22, 69, 12, 46, 71, 246, 207, 37, 78, - 2, 89, 215, 22, 69, 6, 242, 207, 37, 86, 2, 89, 215, 22, 69, 8, 38, 72, - 238, 184, 37, 83, 143, 45, 69, 4, 162, 207, 37, 89, 215, 22, 69, 6, 130, - 207, 37, 72, 2, 84, 215, 22, 69, 192, 41, 178, 1, 65, 242, 170, 1, 69, - 252, 15, 9, 72, 65, 73, 75, 83, 85, 75, 73, 32, 202, 6, 73, 130, 3, 76, - 182, 45, 79, 138, 70, 82, 242, 18, 85, 138, 8, 89, 130, 144, 35, 80, 147, - 1, 83, 190, 14, 132, 1, 2, 66, 89, 94, 67, 206, 2, 68, 146, 1, 71, 106, - 76, 192, 29, 4, 77, 85, 77, 32, 238, 114, 78, 178, 1, 82, 98, 83, 223, 6, - 84, 11, 11, 32, 8, 190, 234, 12, 67, 230, 241, 5, 66, 216, 249, 13, 3, - 65, 78, 71, 215, 249, 3, 83, 16, 62, 75, 160, 251, 8, 5, 84, 82, 73, 65, - 78, 211, 150, 28, 79, 12, 42, 32, 46, 83, 153, 222, 10, 2, 45, 84, 4, - 234, 252, 9, 87, 217, 218, 20, 2, 79, 70, 6, 132, 1, 26, 76, 65, 78, 84, - 69, 68, 32, 83, 79, 85, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, - 72, 32, 72, 79, 235, 162, 37, 80, 4, 130, 189, 29, 82, 221, 182, 5, 2, - 79, 75, 4, 144, 241, 34, 27, 77, 73, 78, 84, 79, 78, 32, 82, 65, 67, 81, - 85, 69, 84, 32, 65, 78, 68, 32, 83, 72, 85, 84, 84, 76, 69, 67, 199, 176, - 2, 71, 6, 192, 226, 30, 6, 85, 69, 84, 84, 69, 32, 190, 242, 5, 69, 229, - 6, 8, 71, 65, 71, 69, 32, 67, 76, 65, 156, 2, 44, 6, 73, 78, 69, 83, 69, - 32, 183, 25, 76, 254, 1, 132, 3, 6, 67, 65, 82, 73, 75, 32, 84, 15, 73, - 78, 86, 69, 82, 84, 69, 68, 32, 67, 65, 82, 73, 75, 32, 68, 7, 76, 69, - 84, 84, 69, 82, 32, 224, 8, 15, 77, 85, 83, 73, 67, 65, 76, 32, 83, 89, - 77, 66, 79, 76, 32, 196, 6, 2, 80, 65, 184, 1, 5, 83, 73, 71, 78, 32, - 176, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 224, 164, 14, 5, - 65, 68, 69, 71, 32, 254, 141, 19, 87, 239, 162, 2, 68, 6, 32, 2, 80, 65, - 219, 129, 3, 83, 4, 86, 82, 145, 249, 30, 5, 77, 85, 78, 71, 75, 4, 36, - 3, 80, 65, 82, 131, 129, 3, 83, 2, 165, 133, 36, 2, 69, 82, 110, 166, 2, - 65, 112, 2, 66, 65, 32, 2, 67, 65, 32, 2, 68, 65, 110, 69, 32, 2, 71, 65, - 30, 73, 2, 79, 2, 85, 36, 2, 74, 65, 30, 75, 68, 2, 76, 65, 18, 78, 72, - 2, 80, 65, 36, 2, 82, 65, 16, 2, 83, 65, 54, 84, 128, 1, 2, 86, 69, 0, 3, - 90, 65, 76, 250, 205, 37, 72, 2, 77, 2, 87, 3, 89, 10, 226, 2, 75, 184, - 3, 5, 83, 89, 85, 82, 65, 172, 162, 15, 8, 82, 67, 72, 65, 73, 67, 32, - 74, 167, 249, 3, 73, 5, 165, 244, 18, 3, 32, 75, 69, 5, 129, 248, 30, 3, - 32, 76, 65, 9, 17, 2, 32, 77, 6, 44, 5, 85, 82, 68, 65, 32, 231, 192, 33, - 65, 4, 130, 176, 14, 77, 25, 3, 65, 76, 80, 4, 254, 3, 70, 231, 139, 37, - 75, 5, 137, 227, 34, 2, 32, 71, 4, 11, 75, 4, 161, 15, 2, 65, 82, 5, 193, - 196, 35, 2, 32, 74, 8, 34, 65, 225, 2, 3, 72, 79, 84, 7, 222, 2, 70, 211, - 171, 14, 32, 7, 131, 13, 32, 8, 34, 65, 134, 208, 37, 71, 3, 89, 5, 173, - 252, 26, 4, 32, 82, 65, 77, 5, 213, 200, 36, 4, 32, 75, 65, 80, 7, 167, - 12, 32, 7, 21, 3, 32, 83, 65, 4, 146, 207, 37, 71, 3, 80, 10, 30, 65, 97, - 3, 90, 73, 82, 9, 11, 32, 6, 160, 172, 14, 6, 77, 85, 82, 68, 65, 32, - 208, 7, 3, 76, 65, 84, 255, 171, 18, 84, 2, 193, 168, 14, 2, 32, 83, 56, - 168, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 246, 1, 68, 172, 1, - 10, 76, 69, 70, 84, 45, 72, 65, 78, 68, 32, 117, 11, 82, 73, 71, 72, 84, - 45, 72, 65, 78, 68, 32, 18, 132, 1, 4, 75, 69, 77, 80, 78, 74, 240, 131, - 20, 2, 66, 69, 180, 182, 7, 3, 69, 78, 68, 148, 171, 3, 3, 84, 69, 71, - 159, 164, 4, 71, 8, 32, 2, 76, 73, 1, 2, 85, 76, 5, 37, 7, 32, 87, 73, - 84, 72, 32, 74, 2, 189, 162, 14, 3, 69, 71, 79, 20, 50, 65, 90, 69, 242, - 250, 36, 73, 2, 79, 3, 85, 10, 40, 2, 78, 71, 158, 251, 36, 69, 3, 73, 7, - 11, 32, 4, 218, 4, 83, 155, 244, 10, 71, 4, 238, 250, 36, 85, 167, 80, - 78, 10, 76, 6, 79, 80, 69, 78, 32, 80, 113, 9, 67, 76, 79, 83, 69, 68, - 32, 80, 76, 6, 254, 249, 36, 65, 2, 73, 3, 85, 8, 72, 8, 67, 76, 79, 83, - 69, 68, 32, 84, 29, 6, 79, 80, 69, 78, 32, 68, 4, 238, 198, 37, 65, 3, - 85, 4, 150, 201, 37, 65, 3, 85, 12, 30, 77, 69, 3, 78, 84, 73, 6, 44, 3, - 65, 68, 65, 241, 133, 35, 2, 69, 78, 5, 69, 2, 32, 76, 7, 11, 32, 4, 38, - 76, 245, 146, 35, 3, 66, 65, 87, 2, 225, 239, 36, 3, 65, 78, 84, 12, 106, - 83, 20, 4, 85, 76, 85, 32, 170, 224, 30, 67, 240, 6, 3, 66, 73, 83, 217, - 249, 1, 4, 82, 69, 82, 69, 2, 179, 232, 24, 85, 4, 254, 194, 20, 67, 237, - 241, 16, 3, 82, 73, 67, 30, 120, 3, 76, 65, 32, 32, 3, 82, 65, 32, 12, 4, - 83, 85, 75, 85, 34, 84, 104, 5, 80, 69, 80, 69, 84, 53, 3, 85, 76, 85, 4, - 165, 1, 4, 76, 69, 78, 71, 4, 115, 82, 5, 237, 203, 36, 3, 32, 73, 76, - 10, 36, 5, 65, 76, 73, 78, 71, 99, 69, 9, 11, 32, 6, 18, 82, 55, 84, 4, - 17, 2, 69, 80, 4, 11, 65, 5, 17, 2, 32, 84, 2, 11, 69, 2, 219, 222, 30, - 68, 5, 181, 230, 30, 3, 32, 83, 65, 30, 86, 79, 228, 147, 23, 5, 32, 79, - 70, 32, 89, 177, 155, 13, 6, 69, 84, 32, 83, 72, 79, 26, 32, 2, 79, 78, - 21, 2, 84, 32, 5, 195, 174, 35, 45, 22, 44, 2, 66, 79, 246, 1, 83, 179, - 192, 37, 88, 18, 38, 88, 205, 1, 4, 76, 68, 32, 83, 17, 33, 6, 32, 87, - 73, 84, 72, 32, 14, 82, 66, 86, 83, 168, 211, 12, 4, 76, 73, 71, 72, 246, - 180, 10, 67, 151, 184, 14, 88, 6, 52, 4, 79, 76, 68, 32, 149, 240, 36, 3, - 65, 76, 76, 4, 26, 83, 159, 136, 23, 67, 2, 165, 211, 12, 4, 67, 82, 73, - 80, 164, 10, 120, 2, 67, 79, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 128, - 92, 3, 78, 74, 65, 138, 207, 28, 83, 238, 192, 5, 70, 83, 81, 8, 26, 77, - 135, 239, 36, 76, 6, 72, 12, 66, 73, 78, 73, 78, 71, 32, 77, 65, 82, 75, - 32, 239, 187, 37, 77, 4, 196, 215, 21, 6, 84, 85, 75, 87, 69, 78, 201, - 173, 3, 4, 75, 79, 81, 78, 148, 10, 170, 1, 70, 58, 75, 170, 1, 76, 66, - 77, 102, 78, 234, 1, 80, 214, 103, 82, 102, 83, 134, 1, 84, 54, 89, 178, - 219, 2, 87, 202, 219, 33, 69, 214, 22, 65, 2, 73, 2, 79, 3, 85, 8, 234, - 79, 65, 234, 213, 36, 69, 254, 5, 79, 219, 16, 85, 22, 78, 69, 46, 79, - 190, 203, 35, 89, 130, 237, 1, 80, 186, 2, 65, 2, 73, 3, 85, 6, 250, 174, - 36, 85, 166, 140, 1, 78, 3, 84, 7, 234, 200, 35, 86, 165, 225, 1, 2, 71, - 72, 10, 202, 102, 69, 202, 131, 26, 79, 182, 208, 10, 65, 2, 73, 3, 85, - 19, 62, 69, 174, 101, 66, 158, 212, 36, 65, 2, 73, 2, 79, 3, 85, 4, 142, - 202, 35, 69, 187, 239, 1, 78, 30, 102, 71, 50, 74, 50, 84, 158, 101, 85, - 130, 216, 34, 83, 246, 7, 68, 246, 222, 1, 89, 218, 19, 65, 3, 73, 6, - 186, 209, 32, 75, 202, 228, 4, 71, 187, 2, 65, 6, 242, 205, 26, 85, 198, - 211, 10, 69, 171, 4, 65, 4, 242, 146, 37, 85, 151, 14, 69, 128, 9, 76, 5, - 72, 65, 83, 69, 45, 218, 100, 69, 138, 2, 85, 138, 208, 36, 65, 3, 73, - 244, 8, 116, 2, 65, 32, 168, 18, 2, 66, 32, 136, 13, 2, 67, 32, 160, 20, - 2, 68, 32, 172, 18, 2, 69, 32, 137, 25, 2, 70, 32, 176, 1, 158, 1, 71, - 106, 75, 130, 1, 76, 102, 77, 198, 3, 78, 210, 4, 80, 146, 2, 83, 190, 2, - 84, 154, 1, 85, 184, 198, 30, 2, 70, 73, 150, 238, 4, 86, 215, 222, 1, - 82, 6, 60, 5, 72, 69, 85, 65, 69, 181, 46, 5, 66, 73, 69, 69, 32, 4, 176, - 27, 2, 71, 72, 191, 174, 26, 82, 12, 38, 65, 34, 69, 150, 67, 80, 3, 85, - 4, 254, 176, 37, 70, 187, 2, 81, 4, 240, 214, 33, 5, 85, 75, 69, 85, 84, - 167, 220, 3, 84, 10, 78, 85, 230, 65, 79, 236, 159, 26, 2, 65, 80, 241, - 214, 9, 4, 69, 84, 32, 75, 5, 159, 159, 27, 65, 36, 142, 1, 65, 176, 1, - 2, 66, 65, 38, 79, 76, 6, 86, 69, 85, 65, 69, 78, 244, 70, 8, 69, 85, 78, - 74, 79, 77, 78, 68, 201, 175, 34, 2, 71, 66, 20, 62, 69, 172, 49, 2, 78, - 83, 149, 239, 31, 4, 80, 32, 80, 73, 16, 58, 77, 234, 170, 12, 75, 226, - 168, 7, 78, 211, 200, 17, 83, 11, 186, 24, 66, 14, 71, 194, 44, 86, 239, - 220, 27, 75, 4, 146, 211, 19, 78, 171, 220, 17, 81, 6, 36, 2, 79, 77, - 237, 1, 2, 78, 32, 4, 178, 252, 12, 80, 239, 184, 23, 69, 2, 207, 243, - 35, 71, 48, 122, 65, 32, 2, 68, 65, 54, 71, 98, 75, 32, 2, 83, 72, 38, - 84, 102, 89, 74, 90, 158, 239, 35, 74, 158, 107, 69, 251, 70, 73, 4, 250, - 8, 65, 195, 164, 37, 81, 4, 22, 65, 159, 22, 32, 2, 153, 44, 3, 78, 71, - 71, 8, 52, 3, 75, 85, 69, 222, 188, 32, 65, 239, 155, 3, 71, 4, 250, 7, - 32, 165, 155, 12, 2, 78, 90, 4, 246, 25, 65, 135, 203, 19, 73, 4, 174, - 188, 35, 73, 187, 239, 1, 65, 8, 40, 2, 65, 80, 157, 157, 28, 2, 79, 81, - 7, 11, 32, 4, 188, 184, 35, 2, 77, 70, 1, 2, 78, 84, 6, 26, 73, 155, 142, - 37, 69, 5, 193, 13, 7, 84, 32, 77, 79, 78, 71, 75, 4, 214, 5, 65, 229, - 159, 12, 4, 85, 78, 32, 77, 22, 58, 65, 80, 3, 79, 78, 32, 178, 140, 37, - 69, 163, 28, 85, 10, 42, 65, 154, 18, 32, 250, 17, 77, 15, 83, 4, 230, - 190, 26, 82, 155, 234, 10, 77, 8, 56, 4, 77, 70, 79, 78, 1, 6, 80, 65, - 32, 78, 74, 73, 4, 37, 7, 32, 80, 73, 80, 65, 69, 77, 4, 206, 16, 71, - 243, 148, 37, 66, 22, 70, 72, 194, 1, 79, 192, 66, 2, 69, 85, 170, 190, - 36, 85, 167, 34, 73, 10, 74, 73, 70, 85, 241, 252, 35, 10, 79, 81, 32, - 78, 83, 72, 85, 84, 32, 89, 4, 170, 188, 26, 82, 177, 211, 6, 8, 78, 68, - 65, 32, 80, 65, 32, 78, 4, 188, 11, 4, 69, 78, 83, 72, 195, 154, 37, 77, - 6, 208, 234, 35, 2, 78, 74, 254, 186, 1, 81, 3, 84, 10, 44, 2, 69, 85, - 44, 3, 73, 84, 65, 31, 85, 4, 130, 209, 35, 65, 1, 4, 84, 69, 85, 87, 2, - 11, 32, 2, 223, 30, 77, 4, 186, 26, 32, 251, 246, 26, 65, 4, 228, 3, 5, - 32, 89, 85, 81, 32, 217, 230, 27, 3, 78, 75, 78, 112, 164, 1, 7, 71, 72, - 69, 85, 71, 72, 69, 34, 75, 126, 76, 122, 77, 154, 3, 78, 234, 2, 80, 90, - 83, 178, 1, 84, 106, 89, 190, 22, 87, 204, 46, 2, 70, 69, 131, 201, 11, - 86, 4, 198, 56, 85, 215, 233, 36, 78, 12, 40, 2, 69, 85, 50, 73, 203, - 144, 37, 65, 6, 230, 54, 89, 174, 184, 12, 80, 147, 160, 24, 65, 4, 242, - 142, 37, 69, 175, 18, 81, 8, 46, 65, 240, 69, 2, 79, 77, 183, 190, 36, - 69, 4, 50, 65, 129, 61, 7, 77, 32, 78, 83, 72, 85, 84, 2, 131, 182, 26, - 78, 26, 70, 65, 74, 66, 140, 1, 2, 69, 85, 58, 70, 145, 23, 3, 79, 78, - 84, 7, 21, 3, 32, 78, 74, 4, 178, 212, 18, 85, 229, 142, 17, 3, 69, 85, - 65, 10, 90, 65, 154, 46, 85, 184, 146, 30, 2, 69, 85, 133, 139, 6, 7, 73, - 84, 32, 77, 66, 65, 65, 4, 150, 12, 65, 205, 192, 19, 4, 32, 77, 65, 69, - 4, 252, 141, 32, 5, 84, 32, 78, 71, 71, 239, 143, 5, 81, 4, 48, 4, 79, - 78, 32, 84, 193, 204, 26, 2, 73, 89, 2, 163, 65, 69, 24, 110, 71, 166, 1, - 83, 50, 89, 152, 23, 8, 84, 73, 69, 69, 32, 83, 72, 69, 201, 144, 35, 5, - 68, 85, 32, 78, 74, 12, 70, 71, 164, 162, 36, 8, 75, 73, 78, 68, 73, 32, - 77, 86, 191, 104, 79, 8, 60, 3, 85, 79, 81, 204, 211, 19, 2, 69, 85, 179, - 140, 16, 65, 5, 193, 180, 28, 2, 32, 76, 4, 26, 72, 227, 202, 36, 69, 2, - 167, 229, 36, 85, 4, 248, 45, 2, 65, 69, 211, 17, 73, 8, 200, 195, 10, 2, - 69, 69, 218, 169, 12, 65, 164, 182, 8, 3, 85, 78, 71, 151, 218, 5, 73, - 12, 88, 2, 65, 75, 16, 2, 72, 69, 252, 185, 19, 2, 69, 84, 146, 239, 15, - 73, 231, 216, 1, 85, 2, 211, 25, 69, 4, 128, 200, 26, 4, 84, 32, 78, 74, - 241, 128, 5, 4, 85, 65, 69, 81, 6, 32, 2, 85, 32, 227, 220, 35, 65, 4, - 36, 4, 77, 65, 69, 77, 139, 7, 78, 2, 11, 71, 2, 135, 7, 66, 4, 44, 4, - 65, 70, 85, 32, 229, 4, 2, 69, 85, 2, 221, 165, 32, 6, 76, 69, 69, 82, - 65, 69, 196, 1, 170, 1, 71, 82, 75, 206, 2, 76, 50, 77, 250, 3, 78, 238, - 6, 80, 78, 83, 118, 84, 176, 1, 3, 86, 69, 85, 46, 87, 62, 89, 202, 157, - 30, 66, 182, 218, 2, 70, 163, 230, 3, 82, 6, 40, 2, 72, 65, 185, 184, 19, - 2, 66, 65, 4, 198, 170, 26, 82, 155, 234, 10, 80, 22, 66, 69, 182, 1, 85, - 132, 194, 26, 3, 80, 65, 82, 139, 206, 10, 65, 14, 40, 2, 78, 32, 54, 85, - 155, 147, 37, 84, 4, 180, 136, 33, 4, 70, 65, 84, 73, 235, 140, 3, 76, 8, - 42, 83, 174, 194, 26, 75, 195, 208, 10, 77, 4, 232, 14, 3, 72, 69, 85, - 239, 53, 69, 4, 48, 6, 79, 80, 32, 78, 75, 65, 147, 146, 37, 84, 2, 11, - 65, 2, 235, 167, 26, 82, 8, 202, 47, 65, 242, 145, 26, 73, 183, 208, 10, - 85, 38, 82, 65, 130, 1, 66, 214, 165, 26, 85, 216, 25, 4, 71, 66, 65, 83, - 139, 225, 8, 73, 8, 18, 32, 79, 69, 4, 42, 78, 145, 231, 17, 4, 75, 69, - 85, 65, 2, 11, 83, 2, 231, 160, 35, 73, 4, 226, 226, 36, 77, 211, 25, 83, - 24, 34, 65, 114, 69, 82, 73, 35, 85, 6, 32, 2, 65, 32, 255, 178, 19, 78, - 4, 200, 206, 31, 8, 67, 65, 66, 66, 65, 71, 69, 45, 133, 238, 4, 2, 80, - 73, 8, 50, 85, 142, 164, 26, 82, 173, 218, 5, 2, 69, 75, 4, 162, 142, 37, - 77, 3, 88, 7, 226, 7, 82, 167, 134, 37, 84, 4, 186, 251, 36, 65, 175, 18, - 69, 72, 130, 1, 65, 54, 68, 110, 71, 222, 1, 74, 102, 83, 130, 1, 84, - 102, 90, 237, 7, 12, 89, 73, 82, 32, 77, 75, 80, 65, 82, 65, 81, 32, 4, - 128, 188, 26, 4, 78, 83, 65, 78, 195, 208, 10, 81, 12, 60, 2, 69, 85, - 174, 41, 65, 134, 155, 19, 79, 159, 162, 17, 73, 4, 136, 156, 35, 2, 65, - 69, 199, 239, 1, 84, 20, 50, 71, 98, 75, 222, 185, 26, 65, 223, 191, 10, - 79, 12, 26, 85, 247, 186, 36, 69, 11, 180, 39, 3, 65, 69, 78, 190, 147, - 36, 79, 202, 26, 69, 155, 53, 77, 4, 36, 3, 85, 69, 32, 183, 185, 26, 65, - 2, 241, 179, 19, 3, 77, 65, 69, 10, 34, 65, 34, 69, 183, 214, 12, 85, 4, - 202, 248, 36, 69, 219, 16, 77, 4, 202, 153, 35, 69, 151, 99, 85, 12, 78, - 85, 214, 183, 26, 72, 204, 233, 5, 2, 69, 85, 158, 214, 4, 79, 219, 16, - 65, 4, 224, 244, 36, 4, 79, 84, 32, 78, 179, 19, 78, 8, 54, 69, 244, 234, - 36, 4, 85, 32, 77, 66, 131, 26, 65, 4, 168, 175, 19, 2, 85, 78, 131, 216, - 17, 78, 4, 238, 221, 35, 69, 147, 169, 1, 65, 6, 26, 73, 227, 182, 36, - 69, 4, 26, 82, 167, 134, 37, 78, 2, 175, 178, 35, 73, 10, 30, 69, 50, 72, - 255, 6, 85, 4, 26, 84, 179, 249, 35, 85, 2, 231, 181, 36, 70, 4, 238, - 210, 12, 85, 159, 226, 13, 73, 10, 40, 2, 65, 65, 34, 69, 41, 2, 73, 84, - 2, 11, 83, 2, 203, 154, 26, 72, 4, 228, 25, 2, 85, 84, 235, 234, 36, 84, - 4, 38, 85, 249, 248, 32, 3, 65, 32, 89, 2, 143, 241, 26, 65, 4, 236, 245, - 27, 2, 65, 69, 255, 141, 9, 88, 4, 40, 4, 65, 78, 71, 75, 139, 131, 37, - 85, 2, 143, 19, 85, 10, 42, 85, 158, 208, 12, 69, 135, 176, 24, 65, 6, - 210, 18, 87, 212, 3, 4, 32, 77, 85, 79, 179, 236, 36, 77, 234, 1, 134, 1, - 70, 68, 2, 71, 72, 34, 75, 254, 1, 76, 142, 1, 77, 130, 3, 78, 130, 5, - 80, 122, 82, 90, 83, 190, 1, 84, 158, 1, 87, 39, 89, 4, 36, 3, 69, 85, - 70, 179, 254, 36, 65, 2, 11, 69, 2, 151, 2, 85, 4, 202, 1, 69, 203, 252, - 36, 65, 22, 46, 69, 146, 1, 85, 42, 87, 143, 143, 35, 89, 10, 26, 85, - 227, 255, 36, 84, 8, 72, 3, 65, 69, 84, 20, 5, 79, 84, 32, 77, 66, 130, - 255, 36, 77, 3, 80, 2, 235, 249, 11, 77, 2, 231, 148, 26, 85, 9, 146, - 238, 36, 79, 218, 16, 78, 3, 81, 2, 171, 201, 36, 65, 14, 58, 69, 194, - 173, 26, 79, 254, 224, 8, 73, 227, 222, 1, 85, 8, 42, 85, 146, 142, 35, - 69, 187, 239, 1, 84, 4, 214, 234, 26, 65, 243, 146, 10, 77, 41, 94, 65, - 58, 66, 66, 69, 38, 70, 80, 2, 71, 66, 214, 254, 31, 79, 242, 130, 4, 86, - 151, 121, 85, 4, 164, 211, 17, 2, 76, 69, 133, 244, 18, 3, 69, 78, 74, 6, - 32, 2, 65, 65, 247, 220, 36, 85, 5, 189, 231, 28, 2, 32, 83, 6, 234, 176, - 18, 85, 171, 219, 16, 69, 10, 44, 2, 69, 85, 246, 255, 34, 79, 207, 11, - 73, 4, 162, 228, 36, 65, 215, 22, 84, 6, 158, 139, 35, 73, 168, 70, 2, - 79, 70, 227, 40, 69, 72, 78, 68, 46, 71, 226, 1, 74, 98, 83, 110, 84, 34, - 89, 222, 245, 36, 73, 3, 85, 8, 194, 39, 69, 178, 130, 36, 79, 139, 63, - 65, 24, 18, 71, 99, 75, 12, 54, 65, 202, 42, 69, 162, 231, 31, 87, 147, - 214, 4, 85, 6, 152, 38, 2, 65, 77, 195, 210, 36, 80, 12, 68, 2, 69, 85, - 182, 136, 35, 73, 2, 89, 218, 159, 1, 85, 215, 79, 65, 4, 154, 197, 12, - 65, 251, 158, 24, 82, 12, 60, 2, 69, 85, 230, 15, 73, 214, 180, 12, 85, - 199, 178, 24, 65, 4, 218, 228, 36, 65, 175, 18, 84, 10, 46, 72, 32, 3, - 73, 69, 69, 195, 229, 36, 85, 4, 138, 218, 36, 85, 219, 5, 69, 4, 150, - 246, 36, 80, 3, 84, 6, 130, 14, 69, 147, 195, 36, 85, 8, 174, 217, 36, - 69, 218, 5, 85, 254, 5, 65, 219, 16, 73, 12, 42, 69, 46, 85, 194, 244, - 36, 65, 3, 73, 4, 220, 138, 26, 2, 85, 84, 155, 234, 10, 69, 4, 158, 216, - 36, 85, 175, 28, 81, 8, 48, 3, 69, 78, 32, 162, 224, 36, 73, 175, 1, 65, - 4, 238, 213, 26, 79, 215, 251, 9, 77, 26, 58, 72, 90, 85, 186, 16, 65, - 174, 6, 69, 179, 191, 36, 79, 12, 54, 69, 174, 162, 26, 79, 222, 188, 10, - 73, 219, 19, 85, 6, 214, 7, 85, 139, 235, 36, 69, 6, 234, 219, 36, 65, - 214, 22, 69, 3, 85, 18, 62, 69, 74, 85, 222, 160, 26, 79, 226, 185, 10, - 65, 215, 22, 73, 8, 26, 85, 135, 130, 35, 69, 6, 210, 157, 35, 65, 234, - 211, 1, 78, 3, 84, 5, 227, 212, 36, 79, 4, 134, 138, 32, 85, 235, 230, 4, - 65, 10, 40, 2, 65, 69, 18, 85, 191, 160, 36, 69, 2, 251, 3, 77, 6, 22, - 87, 227, 13, 79, 2, 207, 159, 26, 79, 186, 2, 178, 1, 70, 154, 1, 71, - 202, 1, 75, 170, 1, 76, 158, 1, 77, 134, 2, 78, 190, 7, 80, 166, 2, 82, - 78, 83, 166, 1, 84, 246, 1, 86, 66, 87, 34, 89, 238, 216, 36, 65, 2, 73, - 3, 79, 18, 54, 85, 234, 129, 23, 65, 198, 213, 13, 69, 255, 5, 79, 10, - 26, 32, 135, 142, 31, 69, 6, 132, 229, 23, 4, 82, 69, 77, 69, 146, 129, - 11, 67, 207, 135, 2, 73, 16, 24, 2, 66, 69, 39, 72, 4, 222, 224, 35, 85, - 167, 140, 1, 84, 12, 34, 65, 34, 69, 199, 219, 36, 79, 2, 11, 65, 2, 151, - 130, 26, 77, 8, 26, 85, 131, 236, 36, 84, 6, 170, 213, 36, 65, 214, 22, - 78, 3, 88, 18, 50, 69, 62, 80, 18, 85, 218, 234, 36, 73, 3, 79, 6, 26, - 85, 139, 235, 36, 84, 4, 178, 212, 36, 65, 215, 22, 88, 2, 211, 28, 69, - 6, 170, 206, 36, 69, 162, 28, 79, 15, 84, 18, 50, 65, 40, 2, 69, 85, 22, - 79, 195, 233, 36, 85, 6, 162, 217, 36, 65, 218, 16, 80, 3, 81, 2, 167, - 215, 36, 65, 8, 222, 158, 18, 79, 226, 202, 18, 77, 3, 81, 32, 110, 65, - 44, 2, 66, 69, 34, 70, 20, 2, 71, 66, 34, 73, 142, 253, 25, 85, 186, 154, - 10, 69, 2, 79, 139, 60, 86, 11, 250, 242, 17, 69, 170, 245, 18, 80, 3, - 81, 4, 158, 215, 36, 85, 219, 16, 69, 2, 155, 181, 12, 69, 4, 226, 151, - 36, 69, 227, 79, 65, 5, 207, 208, 36, 69, 80, 114, 68, 170, 1, 71, 138, - 3, 74, 112, 2, 83, 72, 58, 84, 32, 3, 89, 73, 32, 54, 90, 210, 159, 36, - 65, 191, 47, 75, 12, 34, 65, 98, 73, 187, 149, 36, 85, 6, 32, 2, 65, 32, - 215, 229, 36, 80, 4, 240, 164, 18, 3, 77, 89, 32, 137, 177, 13, 3, 83, - 79, 70, 4, 226, 148, 26, 65, 183, 208, 10, 81, 36, 78, 71, 50, 85, 122, - 75, 118, 79, 172, 221, 11, 3, 69, 85, 82, 223, 132, 25, 65, 12, 18, 69, - 51, 85, 6, 26, 85, 191, 244, 34, 69, 4, 255, 144, 24, 65, 6, 64, 6, 65, - 69, 83, 72, 65, 69, 134, 249, 25, 82, 155, 234, 10, 80, 2, 11, 32, 2, - 183, 189, 23, 78, 12, 34, 65, 20, 2, 69, 85, 35, 85, 5, 243, 206, 36, 65, - 4, 150, 208, 36, 65, 175, 18, 88, 4, 162, 226, 36, 77, 3, 80, 4, 134, - 226, 36, 80, 3, 81, 8, 18, 65, 31, 69, 2, 201, 230, 31, 2, 69, 77, 6, 26, - 69, 255, 212, 35, 85, 5, 245, 190, 36, 4, 32, 69, 80, 79, 6, 26, 85, 171, - 241, 34, 73, 4, 210, 224, 36, 79, 15, 69, 4, 234, 207, 36, 85, 207, 16, - 65, 4, 140, 159, 26, 4, 67, 76, 69, 65, 151, 244, 1, 66, 4, 186, 143, 26, - 65, 3, 85, 34, 42, 65, 90, 69, 58, 73, 50, 79, 23, 85, 8, 32, 2, 32, 80, - 223, 233, 17, 65, 4, 196, 148, 28, 2, 69, 79, 241, 192, 7, 2, 76, 85, 6, - 26, 85, 223, 205, 36, 69, 4, 178, 222, 36, 84, 3, 88, 7, 11, 69, 4, 214, - 141, 26, 69, 183, 208, 10, 84, 5, 135, 142, 36, 79, 11, 82, 65, 130, 221, - 36, 69, 3, 77, 8, 46, 65, 238, 14, 69, 149, 246, 18, 2, 73, 77, 4, 254, - 220, 36, 69, 3, 81, 18, 62, 69, 30, 72, 150, 245, 31, 85, 158, 214, 4, - 79, 163, 14, 65, 4, 162, 220, 36, 69, 3, 84, 8, 42, 69, 198, 239, 22, 79, - 231, 155, 3, 73, 2, 137, 169, 12, 2, 85, 65, 28, 34, 65, 94, 69, 50, 79, - 39, 85, 10, 56, 2, 69, 78, 202, 238, 22, 65, 154, 236, 13, 77, 3, 81, 2, - 205, 209, 11, 3, 32, 78, 84, 6, 26, 85, 167, 218, 36, 78, 5, 211, 167, - 12, 65, 6, 138, 223, 34, 79, 135, 251, 1, 81, 6, 170, 7, 77, 239, 187, - 36, 65, 6, 26, 69, 219, 200, 36, 79, 4, 150, 239, 25, 85, 155, 234, 10, - 69, 6, 246, 10, 69, 207, 187, 32, 85, 26, 68, 2, 69, 85, 46, 73, 32, 3, - 79, 81, 32, 54, 85, 155, 215, 36, 65, 8, 174, 133, 24, 65, 246, 210, 12, - 77, 3, 88, 4, 162, 193, 36, 69, 215, 22, 84, 4, 180, 195, 12, 4, 83, 87, - 73, 77, 227, 197, 9, 67, 8, 238, 134, 26, 69, 206, 252, 8, 65, 234, 211, - 1, 78, 3, 81, 108, 162, 1, 75, 82, 76, 46, 77, 98, 78, 190, 1, 80, 66, - 82, 50, 83, 110, 84, 38, 89, 146, 222, 2, 87, 204, 191, 9, 2, 86, 85, - 254, 155, 24, 69, 242, 5, 70, 231, 16, 85, 14, 226, 138, 18, 69, 170, - 219, 16, 89, 130, 237, 1, 80, 186, 2, 65, 2, 79, 3, 85, 6, 190, 132, 26, - 79, 182, 208, 10, 65, 3, 73, 13, 42, 66, 34, 69, 254, 211, 36, 65, 3, 79, - 4, 186, 132, 36, 69, 171, 77, 65, 2, 195, 228, 34, 69, 22, 94, 71, 38, - 74, 38, 85, 130, 216, 34, 83, 246, 7, 68, 150, 3, 84, 226, 219, 1, 89, - 219, 19, 73, 4, 134, 236, 31, 75, 203, 228, 4, 71, 4, 202, 232, 25, 85, - 239, 215, 10, 65, 5, 235, 187, 36, 65, 6, 26, 69, 251, 231, 25, 85, 4, - 234, 197, 35, 85, 167, 140, 1, 69, 12, 186, 2, 69, 150, 134, 9, 73, 159, - 201, 27, 85, 14, 66, 72, 230, 2, 69, 162, 248, 18, 65, 142, 177, 17, 85, - 235, 36, 73, 6, 158, 189, 36, 73, 218, 19, 79, 3, 85, 6, 182, 141, 21, - 65, 239, 166, 15, 69, 4, 246, 255, 25, 79, 183, 208, 10, 65, 4, 178, 185, - 36, 65, 215, 22, 69, 14, 54, 69, 150, 134, 9, 73, 202, 178, 27, 65, 215, - 22, 85, 6, 138, 195, 35, 85, 166, 140, 1, 69, 3, 78, 16, 62, 72, 50, 69, - 162, 248, 18, 65, 142, 177, 17, 85, 235, 36, 73, 8, 46, 69, 190, 186, 36, - 73, 218, 19, 79, 3, 85, 2, 239, 193, 35, 85, 10, 158, 131, 18, 69, 202, - 135, 3, 65, 155, 195, 15, 73, 6, 150, 253, 25, 79, 2, 85, 183, 208, 10, - 65, 14, 42, 75, 242, 144, 35, 65, 147, 157, 1, 74, 11, 49, 10, 78, 79, - 84, 69, 32, 87, 73, 84, 72, 32, 8, 184, 207, 9, 2, 80, 79, 198, 1, 89, - 156, 177, 12, 2, 69, 85, 163, 177, 6, 68, 6, 38, 32, 173, 244, 16, 3, 66, - 69, 82, 4, 134, 202, 29, 67, 213, 240, 5, 5, 79, 70, 32, 83, 79, 78, 72, - 3, 75, 69, 84, 60, 7, 83, 65, 32, 86, 65, 72, 32, 179, 194, 34, 69, 5, - 233, 156, 16, 10, 66, 65, 76, 76, 32, 65, 78, 68, 32, 72, 72, 104, 10, + 170, 60, 65, 250, 3, 74, 135, 5, 89, 10, 40, 5, 87, 73, 84, 72, 32, 255, + 112, 73, 6, 158, 25, 77, 234, 30, 65, 203, 12, 89, 48, 54, 65, 241, 1, 8, + 69, 72, 32, 87, 73, 84, 72, 32, 30, 68, 6, 70, 73, 90, 65, 72, 85, 101, + 7, 72, 32, 87, 73, 84, 72, 32, 8, 22, 77, 247, 28, 32, 4, 30, 32, 1, 3, + 65, 65, 32, 2, 201, 47, 7, 65, 76, 76, 65, 65, 72, 85, 22, 152, 4, 4, 77, + 69, 69, 77, 214, 53, 65, 138, 6, 74, 247, 2, 89, 18, 164, 1, 5, 77, 69, + 69, 77, 32, 204, 12, 6, 65, 76, 69, 70, 32, 77, 150, 2, 89, 158, 45, 74, + 53, 16, 83, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, + 8, 40, 5, 87, 73, 84, 72, 32, 219, 108, 73, 4, 158, 52, 74, 3, 77, 32, + 68, 4, 65, 76, 76, 65, 81, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 4, 168, + 145, 4, 7, 74, 65, 76, 65, 76, 79, 85, 189, 190, 24, 4, 32, 87, 65, 45, + 28, 62, 72, 60, 5, 77, 69, 69, 77, 32, 174, 53, 65, 255, 8, 89, 8, 17, 2, + 65, 72, 8, 11, 32, 8, 150, 37, 87, 179, 69, 73, 12, 40, 5, 87, 73, 84, + 72, 32, 131, 106, 73, 8, 154, 45, 72, 242, 3, 65, 203, 12, 89, 66, 58, + 65, 213, 1, 9, 72, 65, 72, 32, 87, 73, 84, 72, 32, 48, 116, 7, 70, 32, + 87, 73, 84, 72, 32, 221, 99, 17, 82, 82, 65, 77, 65, 32, 65, 76, 76, 65, + 65, 72, 85, 32, 87, 65, 74, 46, 192, 53, 2, 65, 76, 218, 1, 74, 96, 2, + 76, 65, 146, 2, 75, 14, 72, 30, 77, 239, 1, 89, 18, 54, 72, 146, 50, 65, + 250, 3, 74, 2, 77, 135, 5, 89, 2, 247, 8, 65, 74, 116, 5, 65, 76, 69, 70, + 32, 210, 1, 72, 124, 5, 74, 69, 69, 77, 32, 78, 75, 28, 5, 77, 69, 69, + 77, 32, 211, 54, 89, 20, 64, 5, 87, 73, 84, 72, 32, 162, 51, 77, 222, 6, + 70, 231, 52, 73, 12, 68, 6, 72, 65, 77, 90, 65, 32, 41, 7, 77, 65, 68, + 68, 65, 32, 65, 8, 38, 65, 145, 53, 4, 66, 69, 76, 79, 4, 253, 52, 3, 66, + 79, 86, 14, 28, 2, 65, 72, 187, 51, 69, 12, 11, 32, 12, 40, 5, 87, 73, + 84, 72, 32, 227, 99, 73, 8, 234, 42, 65, 130, 12, 77, 75, 89, 14, 40, 5, + 87, 73, 84, 72, 32, 147, 99, 73, 10, 154, 54, 74, 2, 77, 75, 89, 8, 237, + 20, 3, 72, 65, 72, 14, 130, 20, 87, 138, 35, 70, 202, 43, 73, 211, 9, 77, + 48, 84, 9, 69, 69, 77, 32, 87, 73, 84, 72, 32, 217, 43, 7, 79, 72, 65, + 77, 77, 65, 68, 46, 116, 5, 65, 76, 69, 70, 32, 66, 72, 0, 2, 75, 72, + 104, 5, 74, 69, 69, 77, 32, 92, 5, 77, 69, 69, 77, 32, 43, 89, 4, 22, 77, + 243, 52, 70, 2, 197, 42, 6, 65, 75, 83, 85, 82, 65, 10, 21, 3, 65, 72, + 32, 10, 40, 5, 87, 73, 84, 72, 32, 207, 95, 73, 6, 146, 39, 74, 2, 77, + 143, 12, 89, 12, 40, 5, 87, 73, 84, 72, 32, 255, 94, 73, 8, 194, 38, 77, + 194, 7, 75, 14, 72, 195, 4, 89, 8, 142, 48, 87, 246, 2, 70, 203, 43, 73, + 2, 11, 69, 2, 167, 40, 72, 62, 144, 1, 9, 79, 79, 78, 32, 87, 73, 84, 72, + 32, 205, 159, 13, 20, 65, 87, 87, 65, 82, 65, 32, 65, 76, 76, 65, 65, 72, + 85, 32, 77, 65, 82, 81, 65, 60, 134, 1, 72, 28, 5, 74, 69, 69, 77, 32, + 92, 5, 77, 69, 69, 77, 32, 246, 37, 65, 154, 5, 78, 78, 90, 134, 1, 75, + 238, 1, 82, 43, 89, 14, 150, 33, 65, 155, 9, 69, 16, 40, 5, 87, 73, 84, + 72, 32, 167, 91, 73, 12, 190, 30, 72, 242, 3, 65, 130, 12, 77, 75, 89, + 12, 194, 21, 87, 234, 25, 70, 202, 43, 73, 211, 9, 77, 36, 46, 65, 209, + 2, 6, 85, 68, 68, 73, 83, 65, 28, 156, 1, 7, 70, 32, 87, 73, 84, 72, 32, + 212, 12, 4, 76, 65, 32, 85, 233, 163, 32, 18, 68, 68, 65, 83, 65, 32, 65, + 76, 76, 65, 65, 72, 85, 32, 83, 73, 82, 82, 24, 64, 5, 77, 69, 69, 77, + 32, 174, 35, 65, 246, 6, 72, 139, 2, 89, 12, 40, 5, 87, 73, 84, 72, 32, + 131, 88, 73, 8, 34, 77, 250, 26, 72, 187, 16, 89, 2, 193, 43, 3, 69, 69, + 77, 8, 68, 5, 32, 83, 73, 82, 82, 45, 8, 84, 32, 65, 83, 82, 65, 65, 82, + 6, 220, 5, 3, 85, 72, 85, 215, 149, 39, 65, 2, 241, 240, 37, 2, 85, 72, + 44, 30, 65, 177, 30, 2, 69, 72, 42, 92, 11, 68, 73, 32, 65, 76, 76, 65, + 65, 72, 85, 32, 170, 1, 72, 153, 30, 4, 83, 79, 85, 76, 18, 72, 3, 65, + 78, 72, 61, 11, 84, 65, 65, 65, 76, 65, 65, 32, 65, 78, 72, 11, 26, 85, + 223, 151, 39, 65, 6, 186, 3, 77, 195, 216, 37, 78, 9, 142, 3, 85, 175, + 148, 39, 65, 22, 92, 5, 73, 77, 65, 72, 85, 149, 1, 13, 77, 65, 84, 85, + 32, 65, 76, 76, 65, 65, 72, 73, 32, 12, 44, 7, 32, 65, 76, 76, 65, 65, + 72, 19, 77, 5, 215, 20, 32, 8, 30, 32, 1, 3, 65, 65, 32, 4, 33, 6, 65, + 76, 76, 65, 65, 72, 5, 211, 18, 85, 10, 92, 5, 65, 76, 65, 89, 72, 241, + 149, 39, 12, 84, 65, 65, 65, 76, 65, 65, 32, 65, 76, 65, 89, 9, 26, 73, + 175, 148, 39, 65, 4, 11, 77, 5, 159, 148, 39, 65, 194, 1, 134, 1, 65, + 220, 7, 9, 69, 69, 78, 32, 87, 73, 84, 72, 32, 250, 3, 72, 201, 4, 12, + 85, 66, 72, 65, 65, 78, 65, 72, 85, 32, 87, 65, 50, 48, 7, 68, 32, 87, + 73, 84, 72, 32, 251, 1, 76, 32, 76, 4, 72, 65, 72, 32, 82, 77, 154, 25, + 65, 138, 4, 75, 246, 4, 82, 3, 89, 10, 22, 87, 211, 78, 73, 6, 25, 4, 73, + 84, 72, 32, 6, 206, 17, 72, 187, 16, 89, 8, 21, 3, 69, 69, 77, 8, 11, 32, + 8, 252, 32, 6, 87, 73, 84, 72, 32, 77, 247, 44, 73, 18, 26, 65, 77, 2, + 76, 65, 4, 178, 23, 77, 137, 134, 37, 11, 65, 77, 85, 72, 85, 32, 65, 76, + 65, 89, 78, 14, 34, 32, 133, 1, 3, 76, 76, 65, 4, 22, 85, 187, 85, 73, 2, + 245, 9, 23, 83, 69, 68, 32, 65, 83, 32, 75, 79, 82, 65, 78, 73, 67, 32, + 83, 84, 79, 80, 32, 83, 73, 71, 10, 36, 4, 65, 72, 85, 32, 147, 1, 72, 4, + 212, 2, 14, 84, 65, 65, 65, 76, 65, 65, 32, 65, 76, 65, 89, 72, 73, 237, + 140, 39, 14, 65, 76, 65, 89, 72, 73, 32, 87, 65, 45, 65, 65, 76, 73, 6, + 112, 11, 85, 32, 65, 76, 65, 89, 72, 73, 32, 87, 65, 205, 63, 12, 79, 85, + 32, 65, 76, 65, 89, 72, 69, 32, 87, 65, 4, 44, 7, 45, 65, 76, 65, 65, 32, + 65, 3, 65, 2, 33, 6, 65, 76, 73, 72, 69, 69, 2, 245, 62, 4, 32, 87, 65, + 45, 60, 130, 1, 72, 100, 5, 74, 69, 69, 77, 32, 84, 5, 75, 72, 65, 72, + 32, 92, 5, 77, 69, 69, 77, 32, 250, 15, 65, 254, 8, 82, 3, 89, 12, 32, 3, + 65, 72, 32, 175, 21, 69, 8, 172, 14, 6, 87, 73, 84, 72, 32, 74, 186, 56, + 73, 211, 9, 77, 10, 52, 5, 87, 73, 84, 72, 32, 242, 69, 73, 211, 9, 77, + 4, 250, 12, 65, 139, 8, 72, 10, 34, 87, 178, 69, 73, 211, 9, 77, 4, 25, + 4, 73, 84, 72, 32, 4, 158, 12, 65, 203, 12, 89, 16, 52, 5, 87, 73, 84, + 72, 32, 194, 68, 73, 211, 9, 77, 10, 218, 7, 72, 174, 4, 74, 199, 11, 77, + 82, 96, 10, 65, 68, 68, 65, 32, 87, 73, 84, 72, 32, 161, 1, 9, 69, 69, + 78, 32, 87, 73, 84, 72, 32, 18, 96, 4, 68, 65, 77, 77, 0, 4, 75, 65, 83, + 82, 154, 11, 83, 245, 41, 6, 70, 65, 84, 72, 65, 32, 6, 11, 65, 6, 28, 2, + 84, 65, 227, 52, 32, 2, 179, 12, 78, 64, 130, 1, 72, 36, 5, 74, 69, 69, + 77, 32, 52, 5, 77, 69, 69, 77, 32, 186, 11, 65, 228, 4, 4, 75, 72, 65, + 72, 154, 4, 82, 3, 89, 18, 210, 15, 69, 229, 3, 2, 65, 72, 10, 174, 18, + 87, 246, 2, 70, 202, 43, 73, 211, 9, 77, 16, 64, 5, 87, 73, 84, 72, 32, + 174, 20, 70, 202, 43, 73, 211, 9, 77, 8, 140, 3, 2, 75, 72, 243, 15, 77, + 2, 175, 1, 32, 122, 66, 65, 176, 2, 8, 69, 72, 32, 87, 73, 84, 72, 32, + 179, 4, 72, 28, 88, 11, 66, 65, 65, 82, 65, 75, 65, 32, 87, 65, 45, 29, + 7, 72, 32, 87, 73, 84, 72, 32, 2, 129, 162, 28, 2, 84, 65, 26, 64, 5, 77, + 69, 69, 77, 32, 194, 8, 65, 246, 6, 72, 139, 2, 89, 14, 52, 5, 87, 73, + 84, 72, 32, 138, 61, 73, 211, 9, 77, 8, 34, 72, 174, 4, 77, 143, 12, 89, + 4, 133, 16, 2, 65, 72, 66, 138, 1, 72, 108, 3, 75, 72, 65, 12, 4, 74, 69, + 69, 77, 92, 5, 77, 69, 69, 77, 32, 238, 4, 65, 154, 5, 78, 78, 90, 242, + 2, 82, 43, 89, 14, 32, 3, 65, 72, 32, 227, 9, 69, 10, 40, 5, 87, 73, 84, + 72, 32, 243, 58, 73, 6, 182, 2, 77, 199, 11, 74, 10, 11, 72, 10, 11, 32, + 10, 40, 5, 87, 73, 84, 72, 32, 147, 58, 73, 6, 154, 1, 65, 62, 77, 143, + 12, 89, 18, 64, 5, 87, 73, 84, 72, 32, 226, 13, 70, 202, 43, 73, 211, 9, + 77, 10, 50, 65, 62, 74, 194, 7, 75, 14, 72, 195, 4, 89, 2, 217, 12, 11, + 76, 69, 70, 32, 77, 65, 75, 83, 85, 82, 65, 2, 225, 7, 3, 69, 69, 77, 28, + 56, 2, 65, 76, 117, 8, 69, 72, 32, 87, 73, 84, 72, 32, 2, 37, 7, 32, 87, + 73, 84, 72, 32, 83, 2, 197, 1, 15, 85, 80, 69, 82, 83, 67, 82, 73, 80, + 84, 32, 65, 76, 69, 70, 26, 108, 3, 74, 69, 69, 126, 65, 198, 4, 77, 86, + 78, 78, 90, 242, 2, 82, 42, 89, 213, 53, 5, 72, 69, 72, 32, 77, 2, 11, + 77, 2, 11, 32, 2, 255, 62, 73, 114, 82, 65, 38, 72, 246, 4, 78, 78, 90, + 38, 74, 98, 75, 42, 77, 198, 1, 82, 43, 89, 4, 217, 2, 5, 76, 69, 70, 32, + 77, 76, 22, 65, 139, 3, 69, 72, 80, 15, 77, 90, 65, 32, 65, 66, 79, 86, + 69, 32, 87, 73, 84, 72, 32, 147, 5, 72, 66, 118, 65, 126, 69, 42, 72, 78, + 74, 18, 75, 62, 77, 86, 78, 22, 79, 16, 2, 87, 65, 18, 89, 26, 90, 242, + 2, 82, 67, 85, 12, 22, 76, 247, 6, 69, 8, 21, 3, 69, 70, 32, 8, 34, 77, + 222, 6, 70, 231, 52, 73, 4, 181, 6, 6, 65, 75, 83, 85, 82, 65, 6, 11, 32, + 6, 166, 6, 70, 203, 43, 73, 8, 22, 69, 191, 3, 65, 4, 11, 72, 4, 11, 32, + 4, 230, 4, 73, 143, 54, 77, 4, 223, 48, 69, 2, 11, 72, 2, 11, 65, 2, 11, + 72, 2, 149, 4, 2, 32, 73, 8, 17, 2, 69, 69, 8, 11, 77, 8, 11, 32, 8, 198, + 4, 70, 202, 43, 73, 211, 9, 77, 2, 93, 2, 79, 79, 4, 231, 3, 69, 4, 215, + 3, 87, 8, 186, 3, 69, 15, 85, 2, 17, 2, 65, 73, 2, 239, 2, 78, 6, 21, 3, + 69, 69, 77, 6, 11, 32, 6, 22, 87, 191, 46, 73, 2, 141, 2, 5, 73, 84, 72, + 32, 89, 4, 11, 72, 4, 11, 65, 4, 235, 45, 72, 14, 21, 3, 69, 69, 77, 14, + 11, 32, 14, 64, 5, 87, 73, 84, 72, 32, 194, 1, 70, 202, 43, 73, 211, 9, + 77, 6, 18, 77, 75, 89, 4, 21, 3, 69, 69, 77, 4, 11, 32, 4, 18, 73, 119, + 70, 2, 211, 44, 78, 2, 17, 2, 69, 72, 2, 77, 2, 32, 70, 4, 11, 69, 4, 11, + 72, 4, 11, 32, 4, 22, 70, 231, 52, 73, 2, 185, 53, 2, 73, 78, 6, 174, 43, + 73, 211, 9, 77, 166, 2, 92, 3, 68, 68, 65, 52, 3, 82, 75, 32, 109, 11, + 84, 72, 69, 77, 65, 84, 73, 67, 65, 76, 32, 4, 244, 177, 37, 5, 32, 87, + 65, 65, 74, 131, 65, 72, 4, 58, 78, 1, 10, 83, 73, 68, 69, 87, 65, 89, + 83, 32, 78, 2, 201, 180, 35, 7, 79, 79, 78, 32, 71, 72, 85, 158, 2, 174, + 2, 68, 140, 3, 8, 73, 78, 73, 84, 73, 65, 76, 32, 222, 1, 76, 132, 2, 9, + 79, 80, 69, 82, 65, 84, 79, 82, 32, 166, 1, 83, 142, 3, 84, 130, 190, 24, + 65, 102, 75, 110, 90, 234, 106, 74, 2, 77, 250, 192, 6, 66, 2, 70, 2, 82, + 2, 89, 222, 7, 72, 222, 58, 71, 254, 136, 3, 78, 250, 168, 2, 81, 135, 3, + 87, 62, 26, 79, 219, 233, 37, 65, 58, 88, 6, 84, 76, 69, 83, 83, 32, 61, + 12, 85, 66, 76, 69, 45, 83, 84, 82, 85, 67, 75, 32, 8, 210, 245, 31, 66, + 2, 70, 182, 203, 3, 78, 251, 168, 2, 81, 50, 178, 12, 83, 174, 20, 75, + 146, 168, 24, 84, 74, 90, 234, 106, 74, 2, 77, 250, 192, 6, 66, 2, 70, 2, + 82, 2, 89, 222, 7, 72, 222, 58, 71, 254, 136, 3, 78, 154, 237, 1, 76, + 210, 58, 68, 146, 1, 81, 222, 1, 65, 171, 1, 87, 40, 182, 1, 84, 166, 9, + 83, 254, 187, 24, 72, 30, 75, 214, 107, 74, 2, 77, 250, 192, 6, 66, 2, + 70, 2, 89, 186, 66, 71, 254, 136, 3, 78, 154, 237, 1, 76, 226, 59, 81, + 222, 1, 65, 171, 57, 68, 4, 134, 242, 31, 72, 207, 244, 6, 69, 56, 48, 6, + 79, 79, 80, 69, 68, 32, 183, 214, 38, 65, 54, 202, 8, 83, 174, 20, 75, + 138, 167, 24, 65, 74, 72, 66, 84, 74, 90, 234, 106, 74, 2, 77, 250, 192, + 6, 66, 2, 70, 2, 82, 2, 89, 186, 66, 71, 254, 136, 3, 78, 154, 237, 1, + 76, 210, 58, 68, 146, 1, 81, 135, 3, 87, 4, 148, 161, 29, 23, 77, 69, 69, + 77, 32, 87, 73, 84, 72, 32, 72, 65, 72, 32, 87, 73, 84, 72, 32, 84, 65, + 84, 87, 197, 174, 8, 9, 72, 65, 72, 32, 87, 73, 84, 72, 32, 52, 84, 9, + 84, 82, 69, 84, 67, 72, 69, 68, 32, 178, 141, 37, 72, 14, 69, 231, 143, + 1, 65, 46, 178, 1, 68, 86, 84, 250, 2, 83, 254, 187, 24, 72, 30, 75, 214, + 107, 74, 2, 77, 250, 192, 6, 66, 2, 70, 2, 89, 222, 7, 90, 222, 58, 71, + 254, 136, 3, 78, 250, 168, 2, 81, 223, 1, 65, 6, 52, 7, 79, 84, 76, 69, + 83, 83, 32, 183, 155, 38, 65, 4, 246, 235, 31, 66, 3, 70, 6, 218, 235, + 31, 72, 206, 244, 6, 65, 3, 69, 38, 42, 65, 130, 191, 24, 72, 211, 160, + 14, 69, 32, 44, 5, 73, 76, 69, 68, 32, 179, 224, 38, 72, 30, 146, 1, 68, + 94, 83, 174, 20, 75, 194, 147, 25, 74, 250, 192, 6, 89, 222, 7, 72, 222, + 58, 71, 254, 136, 3, 78, 154, 237, 1, 76, 226, 59, 81, 223, 1, 65, 6, 52, + 7, 79, 84, 76, 69, 83, 83, 32, 199, 152, 38, 65, 4, 186, 180, 35, 78, + 251, 168, 2, 81, 6, 174, 136, 37, 72, 14, 69, 231, 143, 1, 65, 4, 250, + 219, 18, 77, 179, 251, 18, 83, 6, 192, 135, 25, 3, 75, 65, 83, 12, 4, 68, + 65, 77, 77, 1, 4, 70, 65, 84, 72, 12, 118, 69, 38, 79, 144, 217, 12, 11, + 76, 65, 67, 69, 32, 79, 70, 32, 83, 65, 74, 201, 128, 6, 6, 73, 65, 83, + 84, 82, 69, 4, 202, 189, 32, 82, 199, 233, 5, 80, 4, 224, 201, 12, 8, 69, + 84, 73, 67, 32, 86, 69, 82, 245, 143, 6, 3, 85, 78, 68, 14, 238, 1, 65, + 96, 5, 69, 86, 69, 82, 83, 36, 15, 73, 71, 72, 84, 32, 65, 82, 82, 79, + 87, 72, 69, 65, 68, 32, 157, 186, 23, 28, 79, 85, 78, 68, 69, 68, 32, 72, + 73, 71, 72, 32, 83, 84, 79, 80, 32, 87, 73, 84, 72, 32, 70, 73, 76, 76, + 69, 68, 4, 40, 4, 73, 83, 69, 68, 155, 217, 38, 89, 2, 17, 2, 32, 82, 2, + 233, 138, 37, 3, 79, 85, 78, 2, 225, 241, 21, 4, 69, 68, 32, 68, 6, 26, + 65, 187, 182, 36, 66, 4, 249, 177, 37, 3, 66, 79, 86, 206, 1, 238, 1, 69, + 244, 2, 5, 72, 65, 68, 68, 65, 36, 4, 73, 71, 78, 32, 240, 4, 5, 77, 65, + 76, 76, 32, 222, 15, 85, 240, 2, 6, 89, 77, 66, 79, 76, 32, 197, 253, 36, + 18, 84, 65, 82, 84, 32, 79, 70, 32, 82, 85, 66, 32, 69, 76, 32, 72, 73, + 90, 20, 52, 7, 81, 85, 69, 78, 67, 69, 32, 215, 134, 35, 77, 18, 184, 1, + 26, 89, 69, 72, 32, 87, 73, 84, 72, 32, 72, 65, 77, 90, 65, 32, 65, 66, + 79, 86, 69, 32, 87, 73, 84, 72, 32, 193, 222, 31, 13, 78, 79, 79, 78, 32, + 87, 73, 84, 72, 32, 75, 69, 72, 16, 70, 65, 170, 213, 37, 87, 198, 89, + 89, 150, 14, 79, 214, 22, 69, 3, 85, 6, 36, 3, 76, 69, 70, 175, 211, 38, + 69, 5, 251, 12, 32, 7, 11, 32, 4, 222, 22, 73, 55, 77, 22, 132, 1, 2, 82, + 65, 158, 1, 83, 188, 1, 7, 65, 76, 65, 89, 72, 69, 32, 136, 15, 2, 77, + 73, 133, 181, 34, 6, 84, 65, 75, 72, 65, 76, 4, 132, 1, 13, 72, 77, 65, + 84, 85, 76, 76, 65, 72, 32, 65, 76, 65, 161, 218, 33, 13, 68, 73, 32, 65, + 76, 76, 65, 72, 79, 85, 32, 65, 78, 2, 219, 163, 38, 89, 12, 46, 65, 193, + 1, 6, 73, 78, 68, 72, 73, 32, 8, 136, 1, 18, 76, 76, 65, 76, 76, 65, 72, + 79, 85, 32, 65, 76, 65, 89, 72, 69, 32, 87, 154, 225, 31, 78, 218, 240, + 4, 70, 249, 115, 2, 77, 86, 2, 17, 2, 65, 83, 2, 141, 242, 15, 3, 83, 65, + 76, 4, 160, 227, 20, 12, 80, 79, 83, 84, 80, 79, 83, 73, 84, 73, 79, 78, + 205, 152, 14, 2, 65, 77, 110, 120, 2, 70, 65, 32, 5, 72, 73, 71, 72, 32, + 134, 11, 89, 88, 4, 76, 79, 87, 32, 118, 75, 158, 217, 21, 68, 215, 232, + 15, 87, 4, 166, 3, 82, 191, 204, 36, 84, 74, 212, 2, 17, 68, 79, 84, 76, + 69, 83, 83, 32, 72, 69, 65, 68, 32, 79, 70, 32, 75, 22, 70, 102, 76, 142, + 3, 77, 118, 83, 78, 84, 38, 87, 198, 2, 89, 142, 1, 78, 180, 195, 14, 18, + 85, 80, 82, 73, 71, 72, 84, 32, 82, 69, 67, 84, 65, 78, 71, 85, 76, 65, + 252, 220, 8, 7, 82, 79, 85, 78, 68, 69, 68, 242, 126, 90, 234, 106, 74, + 166, 181, 12, 81, 223, 1, 65, 2, 147, 220, 31, 72, 4, 24, 2, 65, 82, 31, + 79, 2, 11, 83, 2, 175, 2, 73, 2, 233, 209, 26, 6, 79, 84, 78, 79, 84, 69, + 10, 60, 8, 73, 71, 65, 84, 85, 82, 69, 32, 225, 11, 2, 65, 77, 8, 88, 10, + 65, 76, 69, 70, 32, 87, 73, 84, 72, 32, 116, 3, 81, 65, 70, 1, 3, 83, 65, + 68, 4, 84, 8, 76, 65, 77, 32, 87, 73, 84, 72, 253, 212, 36, 7, 89, 69, + 72, 32, 66, 65, 82, 2, 201, 209, 31, 2, 32, 89, 2, 89, 20, 32, 87, 73, + 84, 72, 32, 76, 65, 77, 32, 87, 73, 84, 72, 32, 65, 76, 69, 70, 32, 2, + 133, 173, 22, 3, 77, 65, 75, 6, 26, 69, 191, 142, 16, 65, 4, 17, 2, 69, + 77, 4, 17, 2, 32, 73, 4, 22, 78, 147, 9, 83, 2, 205, 9, 2, 73, 84, 6, + 240, 199, 36, 7, 73, 71, 78, 32, 83, 65, 70, 166, 39, 69, 231, 143, 1, + 65, 4, 166, 224, 23, 72, 155, 227, 14, 65, 20, 40, 4, 79, 82, 68, 32, + 227, 197, 37, 65, 18, 74, 65, 172, 1, 2, 83, 65, 168, 201, 25, 3, 87, 65, + 81, 247, 246, 11, 81, 10, 244, 137, 7, 3, 76, 45, 74, 188, 200, 18, 3, + 82, 45, 82, 240, 230, 6, 7, 84, 72, 45, 84, 72, 65, 76, 236, 136, 5, 5, + 78, 45, 78, 73, 83, 161, 92, 5, 83, 45, 83, 65, 74, 4, 186, 157, 38, 75, + 207, 36, 72, 4, 17, 2, 69, 72, 5, 157, 196, 10, 12, 32, 66, 65, 82, 82, + 69, 69, 32, 87, 73, 84, 72, 22, 50, 78, 98, 87, 170, 137, 25, 77, 199, + 224, 11, 83, 4, 21, 3, 79, 79, 78, 5, 37, 7, 32, 87, 73, 84, 72, 32, 75, + 2, 11, 65, 2, 207, 251, 37, 83, 14, 40, 4, 79, 82, 68, 32, 179, 193, 37, + 65, 12, 106, 73, 162, 135, 16, 77, 128, 182, 9, 3, 84, 65, 83, 208, 228, + 8, 2, 83, 65, 153, 228, 3, 3, 81, 65, 83, 4, 168, 168, 13, 2, 77, 65, + 181, 145, 19, 3, 83, 72, 77, 12, 138, 1, 66, 64, 3, 75, 85, 78, 141, 139, + 25, 22, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, 76, 69, 70, 32, 77, + 79, 75, 72, 65, 83, 83, 2, 33, 6, 83, 67, 82, 73, 80, 84, 2, 137, 184, + 22, 2, 32, 65, 9, 11, 32, 6, 34, 73, 54, 77, 147, 154, 36, 66, 2, 11, 83, + 2, 209, 237, 31, 5, 79, 76, 65, 84, 69, 2, 11, 69, 2, 11, 68, 2, 11, 73, + 2, 245, 188, 37, 2, 65, 76, 34, 156, 1, 2, 68, 79, 126, 84, 188, 183, 5, + 8, 83, 77, 65, 76, 76, 32, 84, 65, 200, 212, 26, 4, 70, 79, 85, 82, 224, + 129, 1, 4, 87, 65, 83, 76, 187, 218, 4, 82, 6, 48, 6, 85, 66, 76, 69, 32, + 86, 215, 151, 36, 84, 2, 49, 10, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, + 2, 155, 136, 36, 82, 16, 88, 10, 72, 82, 69, 69, 32, 68, 79, 84, 83, 32, + 121, 8, 87, 79, 32, 68, 79, 84, 83, 32, 8, 192, 139, 32, 17, 80, 79, 73, + 78, 84, 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 150, 139, 4, 66, + 131, 165, 1, 65, 8, 128, 186, 10, 9, 86, 69, 82, 84, 73, 67, 65, 76, 76, + 222, 219, 25, 66, 131, 165, 1, 65, 30, 202, 1, 65, 152, 2, 4, 79, 78, 69, + 32, 164, 130, 11, 12, 82, 73, 80, 76, 69, 32, 68, 79, 84, 32, 80, 85, + 144, 205, 6, 8, 85, 82, 78, 69, 68, 32, 68, 65, 197, 154, 14, 8, 72, 79, + 85, 83, 65, 78, 68, 83, 12, 68, 5, 84, 87, 69, 69, 76, 177, 150, 36, 6, + 73, 76, 32, 70, 82, 65, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 112, 11, + 79, 86, 69, 82, 83, 84, 82, 85, 67, 75, 32, 140, 205, 5, 7, 70, 65, 84, + 72, 65, 84, 65, 159, 233, 4, 84, 4, 26, 72, 131, 181, 37, 87, 2, 133, + 133, 30, 2, 65, 77, 12, 68, 3, 79, 78, 69, 194, 166, 30, 84, 245, 233, 5, + 4, 76, 79, 79, 80, 4, 173, 142, 34, 2, 32, 68, 8, 64, 10, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 255, 164, 24, 69, 6, 72, 10, 73, 78, 86, 69, 82, + 84, 69, 68, 32, 83, 2, 83, 211, 222, 25, 68, 2, 25, 4, 77, 65, 76, 76, 2, + 249, 179, 37, 2, 32, 86, 28, 104, 4, 80, 69, 82, 32, 236, 253, 5, 3, 67, + 85, 66, 204, 138, 5, 5, 70, 79, 85, 82, 84, 187, 179, 25, 68, 4, 234, + 169, 25, 77, 31, 84, 188, 1, 222, 1, 65, 34, 67, 138, 3, 69, 60, 7, 83, + 77, 65, 76, 76, 32, 76, 200, 155, 21, 17, 77, 79, 68, 73, 70, 73, 69, 82, + 32, 76, 69, 84, 84, 69, 82, 32, 76, 222, 253, 8, 72, 140, 188, 2, 3, 68, + 82, 65, 178, 255, 2, 70, 83, 81, 4, 218, 236, 31, 66, 143, 26, 80, 78, + 80, 14, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 191, 215, + 35, 79, 76, 250, 1, 84, 36, 2, 89, 73, 150, 3, 67, 38, 82, 34, 69, 42, + 71, 34, 74, 38, 75, 22, 80, 38, 83, 106, 65, 22, 86, 158, 1, 76, 246, + 173, 31, 70, 2, 88, 134, 241, 1, 73, 214, 174, 3, 66, 2, 77, 222, 57, 78, + 230, 28, 90, 210, 96, 72, 190, 28, 68, 171, 1, 79, 4, 214, 212, 36, 73, + 179, 214, 1, 79, 5, 131, 219, 37, 87, 4, 162, 134, 32, 88, 233, 210, 4, + 6, 77, 80, 72, 65, 83, 73, 92, 76, 6, 69, 84, 84, 69, 82, 32, 157, 5, 8, + 73, 71, 65, 84, 85, 82, 69, 32, 80, 242, 1, 67, 38, 82, 34, 69, 42, 71, + 34, 74, 38, 75, 22, 80, 38, 83, 34, 84, 74, 65, 22, 86, 32, 2, 89, 73, + 126, 76, 246, 173, 31, 70, 2, 88, 134, 241, 1, 73, 214, 174, 3, 66, 2, + 77, 222, 57, 78, 230, 28, 90, 210, 96, 72, 190, 28, 68, 171, 1, 79, 8, + 34, 72, 174, 167, 38, 65, 3, 79, 4, 154, 166, 38, 69, 147, 1, 65, 6, 250, + 165, 38, 67, 146, 1, 72, 3, 84, 4, 182, 224, 37, 72, 215, 53, 73, 4, 230, + 176, 31, 72, 223, 245, 6, 65, 4, 223, 250, 33, 69, 4, 214, 251, 3, 73, + 167, 169, 34, 69, 4, 174, 163, 38, 72, 171, 1, 69, 6, 68, 7, 85, 82, 78, + 69, 68, 32, 65, 210, 206, 36, 73, 179, 214, 1, 79, 2, 135, 231, 36, 89, + 4, 202, 166, 37, 69, 163, 126, 79, 7, 194, 195, 21, 32, 171, 145, 16, 87, + 12, 84, 5, 69, 67, 72, 32, 89, 20, 4, 77, 69, 78, 32, 221, 142, 30, 4, + 86, 69, 87, 32, 2, 159, 205, 36, 73, 8, 222, 173, 31, 88, 134, 241, 1, + 73, 178, 232, 3, 78, 131, 122, 69, 14, 108, 10, 32, 80, 79, 73, 78, 84, + 73, 78, 71, 32, 137, 226, 36, 11, 72, 69, 65, 68, 45, 83, 72, 65, 80, 69, + 68, 12, 156, 2, 24, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 84, 72, + 69, 78, 32, 67, 85, 82, 86, 73, 78, 71, 32, 48, 16, 85, 80, 87, 65, 82, + 68, 83, 32, 84, 72, 69, 78, 32, 78, 79, 82, 197, 189, 35, 22, 68, 79, 87, + 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 67, 85, 82, 86, 73, 78, + 71, 6, 44, 3, 83, 79, 85, 222, 237, 26, 68, 35, 85, 2, 153, 173, 34, 4, + 84, 72, 32, 87, 4, 176, 217, 31, 11, 67, 85, 76, 65, 84, 69, 68, 32, 76, + 79, 82, 189, 252, 4, 6, 83, 84, 32, 80, 65, 76, 24, 58, 67, 38, 84, 206, + 180, 23, 89, 145, 202, 9, 2, 83, 69, 4, 242, 181, 8, 69, 167, 194, 11, + 73, 16, 48, 4, 69, 82, 73, 83, 36, 2, 79, 78, 31, 82, 6, 174, 227, 2, 75, + 239, 185, 35, 77, 2, 237, 138, 14, 2, 73, 83, 8, 104, 20, 79, 78, 79, 77, + 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 211, 138, + 14, 65, 4, 164, 227, 25, 13, 65, 83, 84, 69, 82, 79, 73, 68, 32, 80, 82, + 79, 83, 205, 175, 10, 2, 85, 82, 4, 164, 197, 19, 6, 72, 76, 69, 84, 73, + 67, 165, 192, 17, 2, 79, 77, 10, 68, 2, 84, 79, 160, 249, 21, 2, 83, 84, + 209, 169, 2, 3, 66, 69, 82, 6, 54, 77, 161, 155, 37, 7, 32, 82, 73, 67, + 75, 83, 72, 4, 162, 162, 24, 79, 209, 202, 5, 12, 65, 84, 69, 68, 32, 84, + 69, 76, 76, 69, 82, 32, 112, 56, 6, 69, 83, 84, 65, 78, 32, 169, 231, 4, + 2, 79, 67, 110, 52, 7, 76, 69, 84, 84, 69, 82, 32, 223, 214, 31, 65, 108, + 234, 1, 65, 58, 71, 42, 72, 34, 78, 50, 88, 42, 83, 42, 89, 34, 84, 254, + 188, 34, 85, 206, 141, 1, 79, 238, 60, 73, 166, 140, 1, 66, 2, 68, 2, 90, + 130, 64, 69, 206, 41, 67, 2, 70, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, + 82, 3, 86, 17, 182, 198, 35, 65, 194, 143, 2, 69, 162, 64, 78, 3, 79, 6, + 138, 255, 37, 71, 2, 72, 215, 22, 69, 4, 226, 254, 37, 77, 215, 22, 69, + 12, 46, 71, 150, 254, 37, 78, 2, 89, 215, 22, 69, 6, 146, 254, 37, 86, 2, + 89, 215, 22, 69, 8, 38, 72, 142, 231, 37, 83, 143, 45, 69, 4, 194, 253, + 37, 89, 215, 22, 69, 6, 162, 253, 37, 72, 2, 84, 215, 22, 69, 178, 42, + 178, 1, 65, 194, 171, 1, 69, 244, 21, 9, 72, 65, 73, 75, 83, 85, 75, 73, + 32, 202, 6, 73, 130, 3, 76, 182, 45, 79, 138, 70, 82, 246, 18, 85, 138, + 8, 89, 214, 183, 35, 80, 147, 1, 83, 198, 14, 132, 1, 2, 66, 89, 94, 67, + 206, 2, 68, 146, 1, 71, 106, 76, 192, 29, 4, 77, 85, 77, 32, 190, 115, + 78, 178, 1, 82, 98, 83, 223, 6, 84, 11, 11, 32, 8, 226, 253, 12, 67, 222, + 248, 5, 66, 144, 133, 14, 3, 65, 78, 71, 135, 128, 4, 83, 16, 62, 75, + 244, 135, 9, 5, 84, 82, 73, 65, 78, 159, 184, 28, 79, 12, 42, 32, 46, 83, + 145, 236, 10, 2, 45, 84, 4, 214, 137, 10, 87, 253, 237, 20, 2, 79, 70, 6, + 132, 1, 26, 76, 65, 78, 84, 69, 68, 32, 83, 79, 85, 84, 72, 32, 65, 82, + 82, 79, 87, 32, 87, 73, 84, 72, 32, 72, 79, 139, 209, 37, 80, 4, 138, + 221, 29, 82, 165, 193, 5, 2, 79, 75, 4, 224, 155, 35, 27, 77, 73, 78, 84, + 79, 78, 32, 82, 65, 67, 81, 85, 69, 84, 32, 65, 78, 68, 32, 83, 72, 85, + 84, 84, 76, 69, 67, 151, 180, 2, 71, 6, 224, 130, 31, 6, 85, 69, 84, 84, + 69, 32, 162, 254, 5, 69, 129, 9, 8, 71, 65, 71, 69, 32, 67, 76, 65, 156, + 2, 44, 6, 73, 78, 69, 83, 69, 32, 183, 25, 76, 254, 1, 132, 3, 6, 67, 65, + 82, 73, 75, 32, 84, 15, 73, 78, 86, 69, 82, 84, 69, 68, 32, 67, 65, 82, + 73, 75, 32, 68, 7, 76, 69, 84, 84, 69, 82, 32, 224, 8, 15, 77, 85, 83, + 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 196, 6, 2, 80, 65, 184, + 1, 5, 83, 73, 71, 78, 32, 176, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 220, 184, 14, 5, 65, 68, 69, 71, 32, 130, 159, 19, 87, 167, 169, + 2, 68, 6, 32, 2, 80, 65, 167, 136, 3, 83, 4, 86, 82, 177, 153, 31, 5, 77, + 85, 78, 71, 75, 4, 36, 3, 80, 65, 82, 207, 135, 3, 83, 2, 169, 177, 36, + 2, 69, 82, 110, 166, 2, 65, 112, 2, 66, 65, 32, 2, 67, 65, 32, 2, 68, 65, + 110, 69, 32, 2, 71, 65, 30, 73, 2, 79, 2, 85, 36, 2, 74, 65, 30, 75, 68, + 2, 76, 65, 18, 78, 72, 2, 80, 65, 36, 2, 82, 65, 16, 2, 83, 65, 54, 84, + 128, 1, 2, 86, 69, 0, 3, 90, 65, 76, 154, 252, 37, 72, 2, 77, 2, 87, 3, + 89, 10, 226, 2, 75, 184, 3, 5, 83, 89, 85, 82, 65, 184, 182, 15, 8, 82, + 67, 72, 65, 73, 67, 32, 74, 187, 255, 3, 73, 5, 197, 142, 19, 3, 32, 75, + 69, 5, 161, 152, 31, 3, 32, 76, 65, 9, 17, 2, 32, 77, 6, 44, 5, 85, 82, + 68, 65, 32, 231, 229, 33, 65, 4, 254, 195, 14, 77, 25, 3, 65, 76, 80, 4, + 254, 3, 70, 135, 186, 37, 75, 5, 217, 141, 35, 2, 32, 71, 4, 11, 75, 4, + 161, 15, 2, 65, 82, 5, 249, 239, 35, 2, 32, 74, 8, 34, 65, 225, 2, 3, 72, + 79, 84, 7, 222, 2, 70, 207, 191, 14, 32, 7, 131, 13, 32, 8, 34, 65, 166, + 254, 37, 71, 3, 89, 5, 233, 151, 27, 4, 32, 82, 65, 77, 5, 217, 244, 36, + 4, 32, 75, 65, 80, 7, 167, 12, 32, 7, 21, 3, 32, 83, 65, 4, 178, 253, 37, + 71, 3, 80, 10, 30, 65, 97, 3, 90, 73, 82, 9, 11, 32, 6, 156, 192, 14, 6, + 77, 85, 82, 68, 65, 32, 212, 7, 3, 76, 65, 84, 203, 189, 18, 84, 2, 189, + 188, 14, 2, 32, 83, 56, 168, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, + 32, 246, 1, 68, 172, 1, 10, 76, 69, 70, 84, 45, 72, 65, 78, 68, 32, 117, + 11, 82, 73, 71, 72, 84, 45, 72, 65, 78, 68, 32, 18, 132, 1, 4, 75, 69, + 77, 80, 78, 74, 168, 158, 20, 2, 66, 69, 168, 187, 7, 3, 69, 78, 68, 136, + 172, 3, 3, 84, 69, 71, 207, 174, 4, 71, 8, 32, 2, 76, 73, 1, 2, 85, 76, + 5, 37, 7, 32, 87, 73, 84, 72, 32, 74, 2, 185, 182, 14, 3, 69, 71, 79, 20, + 50, 65, 90, 69, 146, 169, 37, 73, 2, 79, 3, 85, 10, 40, 2, 78, 71, 190, + 169, 37, 69, 3, 73, 7, 11, 32, 4, 218, 4, 83, 171, 135, 11, 71, 4, 142, + 169, 37, 85, 167, 80, 78, 10, 76, 6, 79, 80, 69, 78, 32, 80, 113, 9, 67, + 76, 79, 83, 69, 68, 32, 80, 76, 6, 158, 168, 37, 65, 2, 73, 3, 85, 8, 72, + 8, 67, 76, 79, 83, 69, 68, 32, 84, 29, 6, 79, 80, 69, 78, 32, 68, 4, 142, + 245, 37, 65, 3, 85, 4, 182, 247, 37, 65, 3, 85, 12, 30, 77, 69, 3, 78, + 84, 73, 6, 44, 3, 65, 68, 65, 193, 176, 35, 2, 69, 78, 5, 69, 2, 32, 76, + 7, 11, 32, 4, 38, 76, 209, 189, 35, 3, 66, 65, 87, 2, 129, 158, 37, 3, + 65, 78, 84, 12, 106, 83, 20, 4, 85, 76, 85, 32, 202, 128, 31, 67, 240, 6, + 3, 66, 73, 83, 133, 255, 1, 4, 82, 69, 82, 69, 2, 171, 131, 25, 85, 4, + 210, 221, 20, 67, 185, 133, 17, 3, 82, 73, 67, 30, 120, 3, 76, 65, 32, + 32, 3, 82, 65, 32, 12, 4, 83, 85, 75, 85, 34, 84, 104, 5, 80, 69, 80, 69, + 84, 53, 3, 85, 76, 85, 4, 165, 1, 4, 76, 69, 78, 71, 4, 115, 82, 5, 141, + 250, 36, 3, 32, 73, 76, 10, 36, 5, 65, 76, 73, 78, 71, 99, 69, 9, 11, 32, + 6, 18, 82, 55, 84, 4, 17, 2, 69, 80, 4, 11, 65, 5, 17, 2, 32, 84, 2, 11, + 69, 2, 251, 254, 30, 68, 5, 213, 134, 31, 3, 32, 83, 65, 30, 86, 79, 208, + 174, 23, 5, 32, 79, 70, 32, 89, 217, 172, 13, 6, 69, 84, 32, 83, 72, 79, + 26, 32, 2, 79, 78, 21, 2, 84, 32, 5, 135, 217, 35, 45, 22, 44, 2, 66, 79, + 246, 1, 83, 211, 238, 37, 88, 18, 38, 88, 205, 1, 4, 76, 68, 32, 83, 17, + 33, 6, 32, 87, 73, 84, 72, 32, 14, 82, 66, 86, 83, 184, 230, 12, 4, 76, + 73, 71, 72, 134, 189, 10, 67, 151, 203, 14, 88, 6, 52, 4, 79, 76, 68, 32, + 181, 158, 37, 3, 65, 76, 76, 4, 26, 83, 191, 163, 23, 67, 2, 181, 230, + 12, 4, 67, 82, 73, 80, 172, 10, 120, 2, 67, 79, 180, 1, 7, 76, 69, 84, + 84, 69, 82, 32, 208, 92, 3, 78, 74, 65, 194, 238, 28, 83, 182, 203, 5, + 70, 83, 81, 8, 26, 77, 167, 157, 37, 76, 6, 72, 12, 66, 73, 78, 73, 78, + 71, 32, 77, 65, 82, 75, 32, 143, 234, 37, 77, 4, 184, 242, 21, 6, 84, 85, + 75, 87, 69, 78, 217, 173, 3, 4, 75, 79, 81, 78, 156, 10, 170, 1, 70, 58, + 75, 170, 1, 76, 66, 77, 102, 78, 234, 1, 80, 166, 104, 82, 102, 83, 134, + 1, 84, 54, 89, 162, 225, 2, 87, 170, 131, 34, 69, 214, 22, 65, 2, 73, 2, + 79, 3, 85, 8, 170, 80, 65, 202, 131, 37, 69, 254, 5, 79, 219, 16, 85, 22, + 78, 69, 46, 79, 246, 246, 35, 89, 234, 239, 1, 80, 186, 2, 65, 2, 73, 3, + 85, 6, 254, 218, 36, 85, 194, 142, 1, 78, 3, 84, 7, 162, 244, 35, 86, + 141, 228, 1, 2, 71, 72, 10, 154, 103, 69, 182, 158, 26, 79, 154, 227, 10, + 65, 2, 73, 3, 85, 19, 62, 69, 254, 101, 66, 238, 129, 37, 65, 2, 73, 2, + 79, 3, 85, 4, 198, 245, 35, 69, 163, 242, 1, 78, 30, 102, 71, 50, 74, 50, + 84, 238, 101, 85, 234, 130, 35, 83, 246, 7, 68, 222, 225, 1, 89, 218, 19, + 65, 3, 73, 6, 134, 247, 32, 75, 158, 237, 4, 71, 187, 2, 65, 6, 182, 233, + 26, 85, 162, 230, 10, 69, 171, 4, 65, 4, 146, 193, 37, 85, 151, 14, 69, + 136, 9, 76, 5, 72, 65, 83, 69, 45, 170, 101, 69, 138, 2, 85, 218, 253, + 36, 65, 3, 73, 252, 8, 116, 2, 65, 32, 168, 18, 2, 66, 32, 180, 13, 2, + 67, 32, 180, 20, 2, 68, 32, 172, 18, 2, 69, 32, 153, 25, 2, 70, 32, 176, + 1, 158, 1, 71, 106, 75, 130, 1, 76, 102, 77, 198, 3, 78, 210, 4, 80, 146, + 2, 83, 190, 2, 84, 154, 1, 85, 216, 230, 30, 2, 70, 73, 174, 249, 4, 86, + 191, 225, 1, 82, 6, 60, 5, 72, 69, 85, 65, 69, 153, 47, 5, 66, 73, 69, + 69, 32, 4, 220, 27, 2, 71, 72, 215, 201, 26, 82, 12, 38, 65, 34, 69, 214, + 67, 80, 3, 85, 4, 158, 223, 37, 70, 187, 2, 81, 4, 188, 129, 34, 5, 85, + 75, 69, 85, 84, 251, 223, 3, 84, 10, 78, 85, 166, 66, 79, 232, 186, 26, + 2, 65, 80, 213, 233, 9, 4, 69, 84, 32, 75, 5, 203, 190, 27, 65, 36, 142, + 1, 65, 176, 1, 2, 66, 65, 38, 79, 76, 6, 86, 69, 85, 65, 69, 78, 180, 71, + 8, 69, 85, 78, 74, 79, 77, 78, 68, 229, 217, 34, 2, 71, 66, 20, 62, 69, + 236, 49, 2, 78, 83, 169, 148, 32, 4, 80, 32, 80, 73, 16, 58, 77, 142, + 190, 12, 75, 138, 176, 7, 78, 167, 220, 17, 83, 11, 230, 24, 66, 14, 71, + 214, 44, 86, 203, 252, 27, 75, 4, 222, 237, 19, 78, 255, 239, 17, 81, 6, + 36, 2, 79, 77, 237, 1, 2, 78, 32, 4, 242, 143, 13, 80, 207, 211, 23, 69, + 2, 219, 159, 36, 71, 48, 122, 65, 32, 2, 68, 65, 54, 71, 98, 75, 32, 2, + 83, 72, 38, 84, 102, 89, 74, 90, 170, 155, 36, 74, 178, 109, 69, 251, 70, + 73, 4, 250, 8, 65, 227, 210, 37, 81, 4, 22, 65, 203, 22, 32, 2, 217, 44, + 3, 78, 71, 71, 8, 52, 3, 75, 85, 69, 170, 226, 32, 65, 167, 162, 3, 71, + 4, 250, 7, 32, 201, 174, 12, 2, 78, 90, 4, 166, 26, 65, 143, 229, 19, 73, + 4, 230, 231, 35, 73, 163, 242, 1, 65, 8, 40, 2, 65, 80, 185, 189, 28, 2, + 79, 81, 7, 11, 32, 4, 244, 227, 35, 2, 77, 70, 1, 2, 78, 84, 6, 26, 73, + 187, 188, 37, 69, 5, 193, 13, 7, 84, 32, 77, 79, 78, 71, 75, 4, 214, 5, + 65, 137, 179, 12, 4, 85, 78, 32, 77, 22, 58, 65, 80, 3, 79, 78, 32, 210, + 186, 37, 69, 163, 28, 85, 10, 42, 65, 198, 18, 32, 142, 18, 77, 15, 83, + 4, 170, 218, 26, 82, 247, 252, 10, 77, 8, 56, 4, 77, 70, 79, 78, 1, 6, + 80, 65, 32, 78, 74, 73, 4, 37, 7, 32, 80, 73, 80, 65, 69, 77, 4, 250, 16, + 71, 231, 194, 37, 66, 22, 70, 72, 194, 1, 79, 144, 67, 2, 69, 85, 250, + 235, 36, 85, 167, 34, 73, 10, 74, 73, 70, 85, 253, 168, 36, 10, 79, 81, + 32, 78, 83, 72, 85, 84, 32, 89, 4, 238, 215, 26, 82, 237, 220, 6, 8, 78, + 68, 65, 32, 80, 65, 32, 78, 4, 204, 11, 4, 69, 78, 83, 72, 211, 200, 37, + 77, 6, 220, 150, 36, 2, 78, 74, 146, 189, 1, 81, 3, 84, 10, 44, 2, 69, + 85, 44, 3, 73, 84, 65, 31, 85, 4, 134, 253, 35, 65, 1, 4, 84, 69, 85, 87, + 2, 11, 32, 2, 195, 31, 77, 4, 234, 26, 32, 247, 149, 27, 65, 4, 228, 3, + 5, 32, 89, 85, 81, 32, 245, 134, 28, 3, 78, 75, 78, 116, 164, 1, 7, 71, + 72, 69, 85, 71, 72, 69, 34, 75, 126, 76, 122, 77, 154, 3, 78, 250, 2, 80, + 118, 83, 178, 1, 84, 106, 89, 210, 22, 87, 220, 46, 2, 70, 69, 215, 219, + 11, 86, 4, 134, 57, 85, 183, 151, 37, 78, 12, 40, 2, 69, 85, 50, 73, 235, + 190, 37, 65, 6, 166, 55, 89, 174, 203, 12, 80, 243, 186, 24, 65, 4, 146, + 189, 37, 69, 175, 18, 81, 8, 46, 65, 192, 70, 2, 79, 77, 135, 236, 36, + 69, 4, 50, 65, 209, 61, 7, 77, 32, 78, 83, 72, 85, 84, 2, 199, 209, 26, + 78, 26, 70, 65, 74, 66, 140, 1, 2, 69, 85, 58, 70, 193, 23, 3, 79, 78, + 84, 7, 21, 3, 32, 78, 74, 4, 210, 238, 18, 85, 209, 160, 17, 3, 69, 85, + 65, 10, 90, 65, 218, 46, 85, 152, 178, 30, 2, 69, 85, 133, 153, 6, 7, 73, + 84, 32, 77, 66, 65, 65, 4, 198, 12, 65, 213, 218, 19, 4, 32, 77, 65, 69, + 4, 208, 179, 32, 5, 84, 32, 78, 71, 71, 187, 152, 5, 81, 4, 48, 4, 79, + 78, 32, 84, 253, 231, 26, 2, 73, 89, 2, 243, 65, 69, 26, 110, 71, 182, 1, + 83, 50, 89, 200, 23, 8, 84, 73, 69, 69, 32, 83, 72, 69, 193, 187, 35, 5, + 68, 85, 32, 78, 74, 14, 70, 71, 196, 208, 36, 8, 75, 73, 78, 68, 73, 32, + 77, 86, 191, 104, 79, 10, 76, 3, 85, 79, 81, 244, 237, 19, 2, 69, 85, + 134, 158, 16, 65, 187, 172, 1, 79, 5, 205, 212, 28, 2, 32, 76, 4, 26, 72, + 243, 248, 36, 69, 2, 183, 147, 37, 85, 4, 168, 46, 2, 65, 69, 227, 17, + 73, 10, 76, 3, 85, 78, 71, 252, 213, 10, 2, 69, 69, 178, 177, 12, 65, + 243, 163, 14, 73, 4, 194, 194, 31, 71, 247, 199, 4, 65, 12, 88, 2, 65, + 75, 16, 2, 72, 69, 156, 212, 19, 2, 69, 84, 254, 255, 15, 73, 207, 219, + 1, 85, 2, 231, 25, 69, 4, 144, 227, 26, 4, 84, 32, 78, 74, 221, 138, 5, + 4, 85, 65, 69, 81, 6, 32, 2, 85, 32, 195, 136, 36, 65, 4, 36, 4, 77, 65, + 69, 77, 143, 7, 78, 2, 11, 71, 2, 139, 7, 66, 4, 44, 4, 65, 70, 85, 32, + 233, 4, 2, 69, 85, 2, 253, 202, 32, 6, 76, 69, 69, 82, 65, 69, 198, 1, + 174, 1, 71, 82, 75, 206, 2, 76, 50, 77, 250, 3, 78, 238, 6, 80, 78, 83, + 134, 1, 84, 176, 1, 3, 86, 69, 85, 46, 87, 62, 89, 170, 189, 30, 66, 226, + 227, 2, 70, 247, 234, 3, 82, 6, 40, 2, 72, 65, 213, 210, 19, 2, 66, 65, + 4, 218, 197, 26, 82, 247, 252, 10, 80, 22, 66, 69, 182, 1, 85, 144, 221, + 26, 3, 80, 65, 82, 239, 224, 10, 65, 14, 40, 2, 78, 32, 54, 85, 139, 193, + 37, 84, 4, 208, 177, 33, 4, 70, 65, 84, 73, 191, 145, 3, 76, 8, 42, 83, + 186, 221, 26, 75, 167, 227, 10, 77, 4, 248, 14, 3, 72, 69, 85, 255, 53, + 69, 4, 48, 6, 79, 80, 32, 78, 75, 65, 131, 192, 37, 84, 2, 11, 65, 2, + 255, 194, 26, 82, 8, 234, 47, 65, 222, 172, 26, 73, 155, 227, 10, 85, 38, + 82, 65, 130, 1, 66, 234, 192, 26, 85, 208, 25, 4, 71, 66, 65, 83, 135, + 241, 8, 73, 8, 18, 32, 79, 69, 4, 42, 78, 141, 252, 17, 4, 75, 69, 85, + 65, 2, 11, 83, 2, 239, 203, 35, 73, 4, 210, 144, 37, 77, 211, 25, 83, 24, + 34, 65, 114, 69, 82, 73, 35, 85, 6, 32, 2, 65, 32, 155, 205, 19, 78, 4, + 192, 243, 31, 8, 67, 65, 66, 66, 65, 71, 69, 45, 253, 246, 4, 2, 80, 73, + 8, 50, 85, 162, 191, 26, 82, 189, 228, 5, 2, 69, 75, 4, 146, 188, 37, 77, + 3, 88, 7, 226, 7, 82, 151, 180, 37, 84, 4, 170, 169, 37, 65, 175, 18, 69, + 72, 130, 1, 65, 54, 68, 110, 71, 222, 1, 74, 102, 83, 130, 1, 84, 102, + 90, 253, 7, 12, 89, 73, 82, 32, 77, 75, 80, 65, 82, 65, 81, 32, 4, 140, + 215, 26, 4, 78, 83, 65, 78, 167, 227, 10, 81, 12, 60, 2, 69, 85, 206, 41, + 65, 238, 180, 19, 79, 135, 182, 17, 73, 4, 144, 199, 35, 2, 65, 69, 175, + 242, 1, 84, 20, 50, 71, 98, 75, 234, 212, 26, 65, 195, 210, 10, 79, 12, + 26, 85, 231, 232, 36, 69, 11, 212, 39, 3, 65, 69, 78, 142, 193, 36, 79, + 202, 26, 69, 155, 53, 77, 4, 36, 3, 85, 69, 32, 195, 212, 26, 65, 2, 249, + 205, 19, 3, 77, 65, 69, 10, 34, 65, 34, 69, 199, 233, 12, 85, 4, 186, + 166, 37, 69, 219, 16, 77, 4, 210, 196, 35, 69, 227, 99, 85, 12, 78, 85, + 226, 210, 26, 72, 220, 243, 5, 2, 69, 85, 242, 222, 4, 79, 219, 16, 65, + 4, 208, 162, 37, 4, 79, 84, 32, 78, 179, 19, 78, 8, 54, 69, 228, 152, 37, + 4, 85, 32, 77, 66, 131, 26, 65, 4, 176, 201, 19, 2, 85, 78, 235, 235, 17, + 78, 4, 202, 137, 36, 69, 167, 171, 1, 65, 6, 26, 73, 211, 228, 36, 69, 4, + 26, 82, 151, 180, 37, 78, 2, 131, 222, 35, 73, 12, 30, 69, 30, 72, 163, + 7, 85, 4, 78, 84, 211, 164, 36, 85, 6, 48, 2, 69, 84, 226, 229, 12, 85, + 155, 234, 13, 73, 2, 163, 227, 36, 70, 10, 40, 2, 65, 65, 34, 69, 41, 2, + 73, 84, 2, 11, 83, 2, 207, 181, 26, 72, 4, 228, 25, 2, 85, 84, 203, 152, + 37, 84, 4, 38, 85, 133, 162, 33, 3, 65, 32, 89, 2, 251, 143, 27, 65, 4, + 200, 149, 28, 2, 65, 69, 131, 156, 9, 88, 4, 40, 4, 65, 78, 71, 75, 235, + 176, 37, 85, 2, 143, 19, 85, 10, 42, 85, 158, 227, 12, 69, 231, 202, 24, + 65, 6, 210, 18, 87, 212, 3, 4, 32, 77, 85, 79, 147, 154, 37, 77, 234, 1, + 134, 1, 70, 68, 2, 71, 72, 34, 75, 254, 1, 76, 142, 1, 77, 130, 3, 78, + 130, 5, 80, 122, 82, 90, 83, 190, 1, 84, 158, 1, 87, 39, 89, 4, 36, 3, + 69, 85, 70, 147, 172, 37, 65, 2, 11, 69, 2, 151, 2, 85, 4, 202, 1, 69, + 171, 170, 37, 65, 22, 46, 69, 146, 1, 85, 42, 87, 135, 186, 35, 89, 10, + 26, 85, 195, 173, 37, 84, 8, 72, 3, 65, 69, 84, 20, 5, 79, 84, 32, 77, + 66, 226, 172, 37, 77, 3, 80, 2, 207, 140, 12, 77, 2, 235, 175, 26, 85, 9, + 242, 155, 37, 79, 218, 16, 78, 3, 81, 2, 139, 247, 36, 65, 14, 58, 69, + 190, 200, 26, 79, 250, 240, 8, 73, 203, 225, 1, 85, 8, 42, 85, 138, 185, + 35, 69, 163, 242, 1, 84, 4, 194, 137, 27, 65, 231, 161, 10, 77, 41, 94, + 65, 58, 66, 66, 69, 38, 70, 80, 2, 71, 66, 226, 163, 32, 79, 198, 139, 4, + 86, 151, 121, 85, 4, 144, 232, 17, 2, 76, 69, 249, 140, 19, 3, 69, 78, + 74, 6, 32, 2, 65, 65, 215, 138, 37, 85, 5, 133, 135, 29, 2, 32, 83, 6, + 202, 202, 18, 85, 195, 236, 16, 69, 10, 44, 2, 69, 85, 238, 170, 35, 79, + 207, 11, 73, 4, 130, 146, 37, 65, 215, 22, 84, 6, 150, 182, 35, 73, 252, + 70, 2, 79, 70, 247, 42, 69, 72, 78, 68, 46, 71, 226, 1, 74, 98, 83, 110, + 84, 34, 89, 190, 163, 37, 73, 3, 85, 8, 210, 39, 69, 130, 176, 36, 79, + 139, 63, 65, 24, 18, 71, 99, 75, 12, 54, 65, 218, 42, 69, 158, 140, 32, + 87, 231, 222, 4, 85, 6, 168, 38, 2, 65, 77, 147, 128, 37, 80, 12, 68, 2, + 69, 85, 174, 179, 35, 73, 2, 89, 194, 162, 1, 85, 215, 79, 65, 4, 154, + 216, 12, 65, 219, 185, 24, 82, 12, 60, 2, 69, 85, 230, 15, 73, 214, 199, + 12, 85, 167, 205, 24, 65, 4, 186, 146, 37, 65, 175, 18, 84, 10, 46, 72, + 32, 3, 73, 69, 69, 163, 147, 37, 85, 4, 234, 135, 37, 85, 219, 5, 69, 4, + 246, 163, 37, 80, 3, 84, 6, 130, 14, 69, 243, 240, 36, 85, 8, 142, 135, + 37, 69, 218, 5, 85, 254, 5, 65, 219, 16, 73, 12, 42, 69, 46, 85, 162, + 162, 37, 65, 3, 73, 4, 224, 165, 26, 2, 85, 84, 247, 252, 10, 69, 4, 254, + 133, 37, 85, 175, 28, 81, 8, 48, 3, 69, 78, 32, 130, 142, 37, 73, 175, 1, + 65, 4, 138, 242, 26, 79, 155, 141, 10, 77, 26, 58, 72, 90, 85, 202, 16, + 65, 174, 6, 69, 131, 237, 36, 79, 12, 54, 69, 170, 189, 26, 79, 194, 207, + 10, 73, 219, 19, 85, 6, 214, 7, 85, 235, 152, 37, 69, 6, 202, 137, 37, + 65, 214, 22, 69, 3, 85, 18, 62, 69, 74, 85, 218, 187, 26, 79, 198, 204, + 10, 65, 215, 22, 73, 8, 26, 85, 255, 172, 35, 69, 6, 150, 201, 35, 65, + 134, 214, 1, 78, 3, 84, 5, 195, 130, 37, 79, 4, 146, 175, 32, 85, 191, + 239, 4, 65, 10, 40, 2, 65, 69, 18, 85, 159, 206, 36, 69, 2, 251, 3, 77, + 6, 22, 87, 243, 13, 79, 2, 203, 186, 26, 79, 188, 2, 178, 1, 70, 154, 1, + 71, 202, 1, 75, 170, 1, 76, 158, 1, 77, 134, 2, 78, 206, 7, 80, 166, 2, + 82, 78, 83, 166, 1, 84, 246, 1, 86, 66, 87, 34, 89, 190, 134, 37, 65, 2, + 73, 3, 79, 18, 54, 85, 158, 156, 23, 65, 242, 232, 13, 69, 255, 5, 79, + 10, 26, 32, 239, 178, 31, 69, 6, 156, 255, 23, 4, 82, 69, 77, 69, 242, + 145, 11, 67, 183, 138, 2, 73, 16, 24, 2, 66, 69, 39, 72, 4, 162, 140, 36, + 85, 195, 142, 1, 84, 12, 34, 65, 34, 69, 167, 137, 37, 79, 2, 11, 65, 2, + 155, 157, 26, 77, 8, 26, 85, 227, 153, 37, 84, 6, 138, 131, 37, 65, 214, + 22, 78, 3, 88, 18, 50, 69, 62, 80, 18, 85, 186, 152, 37, 73, 3, 79, 6, + 26, 85, 235, 152, 37, 84, 4, 146, 130, 37, 65, 215, 22, 88, 2, 227, 28, + 69, 6, 138, 252, 36, 69, 162, 28, 79, 15, 84, 18, 50, 65, 40, 2, 69, 85, + 22, 79, 163, 151, 37, 85, 6, 130, 135, 37, 65, 218, 16, 80, 3, 81, 2, + 135, 133, 37, 65, 8, 190, 184, 18, 79, 226, 222, 18, 77, 3, 81, 32, 110, + 65, 44, 2, 66, 69, 34, 70, 20, 2, 71, 66, 34, 73, 146, 152, 26, 85, 150, + 173, 10, 69, 2, 79, 139, 60, 86, 11, 218, 140, 18, 69, 170, 137, 19, 80, + 3, 81, 4, 254, 132, 37, 85, 219, 16, 69, 2, 155, 200, 12, 69, 4, 194, + 197, 36, 69, 227, 79, 65, 5, 175, 254, 36, 69, 82, 114, 68, 170, 1, 71, + 154, 3, 74, 112, 2, 83, 72, 58, 84, 32, 3, 89, 73, 32, 54, 90, 162, 205, + 36, 65, 191, 47, 75, 12, 34, 65, 98, 73, 155, 195, 36, 85, 6, 32, 2, 65, + 32, 183, 147, 37, 80, 4, 208, 190, 18, 3, 77, 89, 32, 189, 188, 13, 3, + 83, 79, 70, 4, 222, 175, 26, 65, 155, 227, 10, 81, 38, 78, 71, 66, 85, + 122, 75, 118, 79, 128, 240, 11, 3, 69, 85, 82, 219, 159, 25, 65, 14, 34, + 69, 50, 85, 167, 145, 37, 79, 6, 26, 85, 167, 159, 35, 69, 4, 167, 171, + 24, 65, 6, 64, 6, 65, 69, 83, 72, 65, 69, 250, 147, 26, 82, 247, 252, 10, + 80, 2, 11, 32, 2, 191, 215, 23, 78, 12, 34, 65, 20, 2, 69, 85, 35, 85, 5, + 195, 252, 36, 65, 4, 230, 253, 36, 65, 175, 18, 88, 4, 242, 143, 37, 77, + 3, 80, 4, 214, 143, 37, 80, 3, 81, 8, 18, 65, 31, 69, 2, 197, 139, 32, 2, + 69, 77, 6, 26, 69, 179, 128, 36, 85, 5, 197, 236, 36, 4, 32, 69, 80, 79, + 6, 26, 85, 147, 156, 35, 73, 4, 162, 142, 37, 79, 15, 69, 4, 186, 253, + 36, 85, 207, 16, 65, 4, 180, 186, 26, 4, 67, 76, 69, 65, 167, 248, 1, 66, + 4, 166, 170, 26, 65, 3, 85, 34, 42, 65, 90, 69, 58, 73, 50, 79, 23, 85, + 8, 32, 2, 32, 80, 175, 131, 18, 65, 4, 252, 179, 28, 2, 69, 79, 237, 204, + 7, 2, 76, 85, 6, 26, 85, 175, 251, 36, 69, 4, 130, 140, 37, 84, 3, 88, 7, + 11, 69, 4, 194, 168, 26, 69, 155, 227, 10, 84, 5, 215, 187, 36, 79, 11, + 82, 65, 210, 138, 37, 69, 3, 77, 8, 46, 65, 238, 14, 69, 253, 143, 19, 2, + 73, 77, 4, 206, 138, 37, 69, 3, 81, 18, 62, 69, 30, 72, 146, 154, 32, 85, + 242, 222, 4, 79, 163, 14, 65, 4, 242, 137, 37, 69, 3, 84, 8, 42, 69, 234, + 137, 23, 79, 175, 156, 3, 73, 2, 249, 187, 12, 2, 85, 65, 28, 34, 65, 94, + 69, 50, 79, 39, 85, 10, 56, 2, 69, 78, 238, 136, 23, 65, 198, 255, 13, + 77, 3, 81, 2, 161, 228, 11, 3, 32, 78, 84, 6, 26, 85, 247, 135, 37, 78, + 5, 195, 186, 12, 65, 6, 242, 137, 35, 79, 239, 253, 1, 81, 6, 170, 7, 77, + 191, 233, 36, 65, 6, 26, 69, 171, 246, 36, 79, 4, 138, 138, 26, 85, 247, + 252, 10, 69, 6, 246, 10, 69, 255, 223, 32, 85, 26, 68, 2, 69, 85, 46, 73, + 32, 3, 79, 81, 32, 54, 85, 235, 132, 37, 65, 8, 214, 159, 24, 65, 158, + 230, 12, 77, 3, 88, 4, 242, 238, 36, 69, 215, 22, 84, 4, 168, 214, 12, 4, + 83, 87, 73, 77, 143, 205, 9, 67, 8, 218, 161, 26, 69, 150, 141, 9, 65, + 134, 214, 1, 78, 3, 81, 108, 162, 1, 75, 82, 76, 46, 77, 98, 78, 190, 1, + 80, 66, 82, 50, 83, 110, 84, 38, 89, 130, 228, 2, 87, 204, 204, 9, 2, 86, + 85, 222, 182, 24, 69, 242, 5, 70, 231, 16, 85, 14, 178, 164, 18, 69, 194, + 236, 16, 89, 234, 239, 1, 80, 186, 2, 65, 2, 79, 3, 85, 6, 170, 159, 26, + 79, 154, 227, 10, 65, 3, 73, 13, 42, 66, 34, 69, 206, 129, 37, 65, 3, 79, + 4, 138, 178, 36, 69, 171, 77, 65, 2, 171, 143, 35, 69, 22, 94, 71, 38, + 74, 38, 85, 234, 130, 35, 83, 246, 7, 68, 150, 3, 84, 202, 222, 1, 89, + 219, 19, 73, 4, 130, 145, 32, 75, 159, 237, 4, 71, 4, 190, 131, 26, 85, + 203, 234, 10, 65, 5, 187, 233, 36, 65, 6, 26, 69, 239, 130, 26, 85, 4, + 158, 241, 35, 85, 195, 142, 1, 69, 12, 186, 2, 69, 178, 146, 9, 73, 211, + 234, 27, 85, 14, 66, 72, 230, 2, 69, 138, 146, 19, 65, 246, 196, 17, 85, + 235, 36, 73, 6, 238, 234, 36, 73, 218, 19, 79, 3, 85, 6, 214, 167, 21, + 65, 159, 186, 15, 69, 4, 226, 154, 26, 79, 155, 227, 10, 65, 4, 130, 231, + 36, 65, 215, 22, 69, 14, 54, 69, 178, 146, 9, 73, 254, 211, 27, 65, 215, + 22, 85, 6, 190, 238, 35, 85, 194, 142, 1, 69, 3, 78, 16, 62, 72, 50, 69, + 138, 146, 19, 65, 246, 196, 17, 85, 235, 36, 73, 8, 46, 69, 142, 232, 36, + 73, 218, 19, 79, 3, 85, 2, 163, 237, 35, 85, 10, 238, 156, 18, 69, 154, + 136, 3, 65, 203, 214, 15, 73, 6, 130, 152, 26, 79, 2, 85, 155, 227, 10, + 65, 14, 42, 75, 174, 188, 35, 65, 167, 159, 1, 74, 11, 49, 10, 78, 79, + 84, 69, 32, 87, 73, 84, 72, 32, 8, 224, 220, 9, 2, 80, 79, 198, 1, 89, + 148, 190, 12, 2, 69, 85, 187, 182, 6, 68, 6, 38, 32, 149, 137, 17, 3, 66, + 69, 82, 4, 198, 233, 29, 67, 201, 252, 5, 5, 79, 70, 32, 83, 79, 78, 72, + 3, 75, 69, 84, 60, 7, 83, 65, 32, 86, 65, 72, 32, 155, 237, 34, 69, 5, + 193, 177, 16, 10, 66, 65, 76, 76, 32, 65, 78, 68, 32, 72, 72, 104, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 164, 1, 7, 76, 69, 84, 84, 69, - 82, 32, 151, 246, 33, 70, 10, 52, 4, 72, 73, 71, 72, 44, 3, 76, 79, 87, - 39, 77, 4, 236, 219, 21, 2, 45, 76, 159, 252, 10, 32, 4, 32, 2, 45, 77, - 191, 215, 32, 32, 2, 173, 215, 32, 2, 73, 68, 60, 238, 1, 68, 38, 69, 38, - 71, 34, 75, 38, 85, 20, 2, 87, 65, 22, 89, 212, 228, 29, 2, 72, 87, 230, - 191, 1, 77, 190, 218, 2, 79, 250, 160, 2, 86, 246, 5, 74, 2, 84, 2, 90, + 82, 32, 151, 160, 34, 70, 10, 52, 4, 72, 73, 71, 72, 44, 3, 76, 79, 87, + 39, 77, 4, 136, 246, 21, 2, 45, 76, 255, 138, 11, 32, 4, 32, 2, 45, 77, + 187, 128, 33, 32, 2, 169, 128, 33, 2, 73, 68, 60, 238, 1, 68, 38, 69, 38, + 71, 34, 75, 38, 85, 20, 2, 87, 65, 22, 89, 164, 132, 30, 2, 72, 87, 154, + 197, 1, 77, 186, 223, 2, 79, 202, 164, 2, 86, 246, 5, 74, 2, 84, 2, 90, 162, 8, 67, 2, 83, 158, 20, 66, 2, 70, 2, 80, 186, 2, 65, 3, 73, 4, 138, - 131, 34, 72, 255, 194, 2, 79, 7, 222, 201, 31, 78, 135, 252, 4, 69, 4, - 214, 160, 36, 66, 219, 35, 65, 4, 158, 229, 29, 80, 131, 224, 6, 65, 5, - 147, 160, 36, 87, 5, 227, 159, 36, 68, 4, 254, 196, 35, 69, 131, 105, 73, - 121, 48, 3, 65, 75, 32, 230, 10, 72, 143, 174, 22, 84, 112, 196, 1, 15, + 173, 34, 72, 207, 198, 2, 79, 7, 218, 238, 31, 78, 219, 132, 5, 69, 4, + 166, 206, 36, 66, 219, 35, 65, 4, 238, 132, 30, 80, 131, 238, 6, 65, 5, + 227, 205, 36, 87, 5, 179, 205, 36, 68, 4, 206, 242, 35, 69, 131, 105, 73, + 121, 48, 3, 65, 75, 32, 230, 10, 72, 179, 200, 22, 84, 112, 196, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 248, 4, 3, 80, 65, 78, 50, 83, 141, 2, 11, 86, - 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 4, 226, 193, 36, 78, 87, 72, 76, - 194, 1, 77, 114, 78, 68, 2, 80, 65, 38, 83, 196, 222, 18, 4, 75, 65, 82, - 79, 134, 222, 17, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 76, 2, 82, 2, - 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 10, 26, 65, 135, 190, 36, 66, 9, 45, - 9, 78, 68, 65, 73, 76, 73, 78, 71, 32, 6, 210, 189, 36, 72, 2, 78, 3, 83, - 10, 152, 2, 2, 79, 82, 150, 187, 36, 68, 2, 71, 2, 89, 187, 2, 65, 5, - 189, 244, 20, 4, 75, 80, 65, 75, 24, 80, 10, 73, 77, 65, 76, 85, 78, 71, - 85, 78, 32, 96, 2, 79, 85, 207, 189, 36, 65, 20, 242, 187, 36, 71, 2, 72, - 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 87, 2, 89, 187, 2, 65, 2, 237, 222, - 18, 5, 84, 72, 69, 82, 78, 4, 166, 2, 71, 253, 222, 18, 4, 79, 78, 71, - 79, 10, 96, 12, 89, 77, 66, 79, 76, 32, 66, 73, 78, 68, 85, 32, 249, 179, - 10, 6, 73, 71, 78, 32, 84, 79, 8, 78, 80, 136, 159, 11, 3, 74, 85, 68, - 149, 153, 25, 6, 78, 65, 32, 77, 69, 84, 4, 64, 3, 65, 78, 71, 241, 156, - 23, 7, 73, 78, 65, 82, 66, 79, 82, 2, 151, 228, 25, 79, 18, 122, 85, 252, - 146, 26, 6, 80, 65, 75, 80, 65, 75, 232, 250, 2, 5, 75, 65, 82, 79, 32, - 234, 235, 6, 69, 162, 64, 73, 3, 79, 5, 241, 187, 30, 15, 32, 70, 79, 82, - 32, 83, 73, 77, 65, 76, 85, 78, 71, 85, 78, 5, 195, 219, 23, 84, 254, 1, - 134, 1, 65, 230, 2, 69, 50, 76, 130, 1, 78, 190, 10, 84, 224, 228, 32, 2, - 67, 65, 248, 183, 2, 5, 86, 69, 82, 65, 71, 215, 140, 1, 68, 20, 136, 1, - 4, 77, 69, 68, 32, 170, 1, 82, 136, 169, 3, 10, 67, 72, 32, 87, 73, 84, - 72, 32, 85, 77, 178, 158, 7, 84, 166, 176, 25, 86, 19, 78, 8, 86, 65, 0, - 2, 68, 69, 52, 4, 69, 73, 71, 72, 1, 7, 83, 73, 88, 84, 69, 69, 78, 2, - 197, 164, 20, 8, 83, 67, 69, 78, 68, 73, 78, 71, 2, 189, 164, 20, 2, 84, - 72, 4, 220, 181, 32, 3, 68, 69, 68, 215, 194, 3, 32, 4, 172, 197, 8, 3, - 82, 32, 77, 195, 203, 26, 84, 13, 11, 76, 11, 38, 32, 149, 156, 25, 3, - 72, 79, 80, 6, 238, 185, 12, 80, 152, 160, 16, 6, 87, 73, 84, 72, 32, 67, - 159, 199, 6, 83, 208, 1, 84, 5, 71, 65, 76, 73, 32, 158, 9, 84, 37, 9, - 90, 69, 78, 69, 32, 82, 73, 78, 71, 200, 1, 210, 1, 65, 40, 9, 67, 85, - 82, 82, 69, 78, 67, 89, 32, 148, 2, 7, 76, 69, 84, 84, 69, 82, 32, 204, - 3, 6, 82, 85, 80, 69, 69, 32, 34, 83, 250, 213, 22, 73, 134, 4, 86, 242, - 223, 11, 68, 249, 107, 3, 71, 65, 78, 6, 154, 154, 32, 66, 50, 78, 187, - 57, 85, 12, 120, 10, 78, 85, 77, 69, 82, 65, 84, 79, 82, 32, 209, 170, - 23, 14, 68, 69, 78, 79, 77, 73, 78, 65, 84, 79, 82, 32, 83, 73, 10, 52, - 3, 79, 78, 69, 218, 156, 31, 70, 163, 163, 3, 84, 5, 157, 177, 29, 19, - 32, 76, 69, 83, 83, 32, 84, 72, 65, 78, 32, 84, 72, 69, 32, 68, 69, 78, - 79, 108, 226, 1, 75, 90, 82, 146, 184, 21, 86, 138, 138, 8, 89, 134, 144, - 3, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, - 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 80, 138, 69, 72, 2, 76, 2, 77, 186, - 2, 69, 3, 79, 8, 26, 72, 251, 172, 36, 65, 6, 26, 65, 219, 254, 13, 73, - 5, 253, 205, 18, 3, 78, 68, 65, 10, 34, 65, 226, 169, 36, 72, 3, 82, 7, - 33, 6, 32, 87, 73, 84, 72, 32, 4, 180, 148, 25, 5, 76, 79, 87, 69, 82, 1, - 6, 77, 73, 68, 68, 76, 69, 4, 194, 164, 35, 83, 191, 69, 77, 20, 116, 19, - 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, - 32, 182, 255, 25, 65, 219, 149, 6, 73, 6, 158, 216, 22, 82, 155, 207, 13, - 89, 4, 226, 146, 26, 32, 199, 138, 9, 79, 5, 133, 179, 33, 3, 32, 87, 73, - 4, 182, 213, 34, 87, 219, 64, 32, 194, 1, 184, 2, 7, 76, 69, 84, 84, 69, - 82, 32, 244, 1, 7, 78, 85, 77, 66, 69, 82, 32, 72, 5, 83, 73, 71, 78, 32, - 48, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 142, 245, 25, 68, - 170, 243, 3, 87, 152, 186, 2, 10, 71, 65, 80, 32, 70, 73, 76, 76, 69, 82, - 205, 182, 3, 12, 72, 85, 78, 68, 82, 69, 68, 83, 32, 85, 78, 73, 92, 210, - 1, 86, 218, 202, 32, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, - 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, - 69, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 8, 234, 1, 79, - 143, 163, 36, 65, 36, 142, 126, 69, 38, 70, 66, 78, 26, 83, 138, 173, 21, - 84, 155, 223, 12, 79, 10, 206, 143, 32, 67, 210, 61, 65, 231, 147, 3, 86, - 24, 80, 2, 86, 79, 194, 207, 32, 65, 38, 85, 206, 201, 1, 73, 222, 137, - 2, 69, 3, 79, 6, 33, 6, 67, 65, 76, 73, 67, 32, 6, 158, 208, 32, 82, 203, - 210, 3, 76, 26, 148, 1, 4, 67, 89, 67, 76, 36, 2, 71, 32, 28, 2, 76, 76, - 46, 82, 66, 84, 204, 166, 19, 4, 79, 72, 65, 90, 176, 253, 11, 2, 75, 73, - 155, 172, 4, 83, 4, 174, 183, 32, 73, 255, 233, 3, 69, 4, 178, 139, 34, - 82, 67, 83, 4, 216, 215, 9, 2, 69, 68, 199, 235, 23, 73, 4, 140, 184, 32, - 7, 84, 72, 68, 65, 89, 32, 67, 179, 232, 3, 68, 4, 188, 211, 13, 2, 67, - 79, 137, 203, 22, 4, 73, 78, 71, 32, 192, 7, 42, 65, 174, 33, 79, 165, - 11, 2, 85, 69, 242, 2, 32, 2, 67, 75, 247, 236, 17, 78, 240, 2, 22, 32, - 223, 31, 45, 224, 2, 210, 1, 67, 254, 4, 68, 174, 2, 70, 102, 72, 82, 76, - 186, 4, 77, 250, 2, 78, 38, 80, 46, 82, 150, 4, 83, 154, 4, 84, 82, 85, - 248, 2, 3, 86, 69, 82, 214, 128, 12, 79, 156, 205, 20, 3, 66, 79, 87, - 251, 247, 1, 81, 98, 196, 1, 5, 73, 82, 67, 76, 69, 200, 1, 6, 85, 82, - 86, 69, 68, 32, 224, 251, 25, 12, 82, 79, 83, 83, 32, 79, 78, 32, 83, 72, - 73, 69, 180, 158, 3, 5, 69, 78, 84, 82, 69, 230, 161, 5, 72, 211, 13, 76, - 11, 11, 32, 8, 72, 5, 87, 73, 84, 72, 32, 217, 235, 16, 7, 70, 79, 82, - 32, 82, 69, 67, 6, 140, 69, 8, 87, 72, 73, 84, 69, 32, 68, 79, 198, 242, - 18, 68, 161, 146, 2, 8, 84, 87, 79, 32, 87, 72, 73, 84, 16, 84, 4, 68, - 79, 87, 78, 0, 2, 85, 80, 56, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 4, - 229, 252, 31, 9, 87, 65, 82, 68, 83, 32, 65, 78, 68, 4, 53, 11, 84, 87, - 65, 82, 68, 83, 32, 65, 78, 68, 32, 4, 158, 143, 21, 85, 231, 154, 12, - 68, 30, 64, 6, 73, 65, 77, 79, 78, 68, 152, 1, 3, 79, 87, 78, 43, 82, 13, - 11, 32, 10, 154, 19, 67, 244, 250, 12, 10, 77, 73, 78, 85, 83, 32, 87, - 72, 73, 84, 152, 166, 6, 6, 87, 73, 84, 72, 32, 68, 206, 161, 15, 79, - 135, 13, 83, 12, 214, 19, 32, 62, 45, 247, 167, 31, 87, 6, 212, 182, 34, - 2, 79, 80, 231, 20, 65, 8, 18, 76, 39, 79, 4, 206, 234, 27, 79, 155, 170, - 8, 65, 4, 252, 217, 2, 2, 85, 82, 151, 155, 31, 76, 12, 38, 69, 218, 206, - 34, 65, 251, 2, 79, 6, 160, 208, 34, 2, 65, 82, 247, 11, 88, 40, 64, 5, - 65, 82, 71, 69, 32, 140, 2, 3, 69, 70, 84, 203, 1, 79, 12, 48, 6, 67, 73, - 82, 67, 76, 69, 203, 242, 34, 83, 11, 37, 7, 32, 77, 73, 78, 85, 83, 32, - 8, 54, 76, 36, 4, 82, 73, 71, 72, 13, 3, 85, 80, 80, 4, 32, 2, 69, 70, - 13, 2, 79, 87, 2, 31, 84, 2, 17, 2, 69, 82, 2, 173, 147, 33, 8, 32, 81, - 85, 65, 82, 84, 69, 82, 22, 96, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, - 32, 64, 6, 87, 65, 82, 68, 83, 32, 235, 207, 34, 32, 12, 146, 7, 68, 190, - 8, 73, 190, 195, 34, 80, 206, 24, 83, 51, 84, 4, 242, 164, 33, 69, 255, - 139, 1, 66, 6, 158, 15, 87, 175, 203, 34, 90, 30, 76, 6, 69, 68, 73, 85, - 77, 32, 245, 222, 21, 7, 79, 79, 78, 32, 76, 73, 76, 28, 66, 68, 42, 76, - 36, 4, 82, 73, 71, 72, 12, 2, 85, 80, 111, 83, 6, 84, 3, 79, 87, 78, 163, - 209, 34, 73, 6, 32, 2, 69, 70, 195, 216, 34, 79, 4, 11, 84, 4, 81, 18, - 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 84, 82, 73, 65, 78, 71, 76, 69, - 5, 145, 9, 2, 32, 67, 8, 198, 13, 77, 247, 222, 34, 81, 4, 238, 145, 17, - 69, 155, 191, 17, 73, 8, 238, 135, 25, 85, 158, 201, 9, 65, 87, 69, 34, - 52, 4, 73, 71, 72, 84, 254, 252, 33, 79, 155, 85, 69, 30, 94, 32, 84, 10, - 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 245, 1, 6, 87, 65, 82, 68, 83, - 32, 6, 60, 9, 84, 82, 73, 65, 78, 71, 76, 69, 32, 171, 227, 34, 80, 2, - 231, 204, 6, 67, 16, 90, 68, 88, 8, 84, 82, 73, 65, 78, 71, 76, 69, 230, - 7, 73, 194, 200, 34, 80, 203, 19, 83, 4, 65, 14, 79, 85, 66, 76, 69, 32, - 84, 82, 73, 65, 78, 71, 76, 69, 5, 207, 189, 12, 32, 5, 205, 229, 20, 11, - 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 8, 230, 158, 19, 65, 162, - 254, 13, 69, 255, 139, 1, 66, 40, 158, 2, 77, 148, 1, 5, 81, 85, 65, 82, - 69, 180, 253, 25, 3, 85, 78, 32, 172, 223, 1, 5, 75, 85, 76, 76, 32, 160, - 178, 3, 3, 78, 79, 87, 168, 223, 1, 5, 65, 70, 69, 84, 89, 192, 131, 1, - 13, 76, 73, 71, 72, 84, 76, 89, 32, 83, 77, 65, 76, 76, 254, 91, 67, 50, - 72, 222, 1, 80, 175, 19, 84, 12, 40, 4, 65, 76, 76, 32, 203, 208, 34, 73, - 10, 214, 200, 34, 68, 150, 7, 76, 70, 83, 213, 15, 13, 85, 80, 45, 80, - 79, 73, 78, 84, 73, 78, 71, 32, 67, 9, 11, 32, 6, 54, 67, 208, 178, 33, - 3, 70, 79, 82, 163, 158, 1, 66, 2, 181, 248, 30, 3, 69, 78, 84, 14, 192, - 4, 3, 73, 78, 89, 242, 174, 23, 82, 154, 165, 11, 79, 54, 69, 135, 2, 87, - 18, 38, 80, 229, 213, 32, 3, 78, 73, 86, 16, 46, 32, 62, 45, 170, 1, 80, - 207, 166, 31, 87, 2, 233, 219, 34, 10, 80, 79, 73, 78, 84, 73, 78, 71, - 32, 66, 8, 45, 9, 80, 79, 73, 78, 84, 73, 78, 71, 32, 8, 66, 73, 156, - 216, 34, 5, 68, 79, 85, 66, 76, 238, 3, 83, 51, 84, 2, 213, 218, 33, 8, - 83, 79, 83, 67, 69, 76, 69, 83, 4, 21, 3, 69, 82, 32, 4, 210, 182, 24, - 76, 207, 163, 9, 82, 10, 56, 6, 84, 73, 67, 65, 76, 32, 41, 4, 89, 32, - 83, 77, 4, 152, 198, 34, 2, 82, 69, 239, 22, 69, 6, 21, 3, 65, 76, 76, 6, - 11, 32, 6, 186, 194, 34, 68, 150, 7, 76, 247, 20, 83, 16, 84, 15, 76, 69, - 84, 84, 69, 82, 32, 67, 65, 80, 73, 84, 65, 76, 32, 227, 146, 11, 70, 10, - 154, 254, 35, 67, 2, 72, 2, 73, 2, 82, 3, 90, 200, 4, 52, 3, 67, 75, 32, - 166, 151, 2, 83, 183, 201, 10, 87, 196, 4, 80, 7, 79, 67, 84, 65, 78, 84, - 45, 149, 7, 8, 83, 69, 88, 84, 65, 78, 84, 45, 204, 3, 58, 49, 130, 3, - 50, 114, 53, 50, 54, 18, 51, 215, 1, 52, 234, 1, 74, 50, 246, 1, 51, 174, - 91, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 116, 62, 51, 226, 92, - 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 55, 54, 52, 214, 92, 53, - 38, 54, 30, 55, 187, 157, 35, 56, 22, 50, 53, 166, 2, 54, 190, 90, 55, - 187, 157, 35, 56, 11, 42, 54, 210, 220, 28, 55, 159, 157, 7, 56, 4, 234, - 249, 35, 55, 3, 56, 56, 170, 1, 53, 50, 54, 210, 89, 52, 110, 55, 187, - 157, 35, 56, 118, 62, 52, 254, 89, 51, 98, 53, 38, 54, 30, 55, 187, 157, - 35, 56, 24, 46, 53, 50, 54, 190, 90, 55, 187, 157, 35, 56, 13, 206, 1, - 54, 186, 217, 28, 55, 159, 157, 7, 56, 7, 187, 90, 55, 61, 54, 52, 118, - 53, 230, 88, 54, 30, 55, 187, 157, 35, 56, 31, 46, 53, 170, 89, 54, 30, - 55, 187, 157, 35, 56, 15, 38, 54, 158, 89, 55, 187, 157, 35, 56, 7, 210, - 246, 35, 55, 3, 56, 14, 226, 88, 54, 30, 55, 187, 157, 35, 56, 31, 46, - 54, 234, 87, 53, 66, 55, 187, 157, 35, 56, 6, 166, 88, 55, 187, 157, 35, - 56, 120, 70, 49, 238, 1, 50, 190, 188, 25, 51, 38, 52, 30, 53, 147, 182, - 10, 54, 61, 58, 50, 126, 51, 150, 189, 25, 52, 30, 53, 147, 182, 10, 54, - 31, 50, 51, 222, 189, 25, 52, 30, 53, 147, 182, 10, 54, 15, 42, 52, 206, - 189, 25, 53, 147, 182, 10, 54, 7, 218, 243, 35, 53, 3, 54, 15, 146, 189, - 25, 52, 166, 153, 3, 53, 139, 157, 7, 54, 31, 50, 52, 138, 188, 25, 51, - 66, 53, 147, 182, 10, 54, 7, 199, 188, 25, 53, 6, 190, 153, 22, 32, 237, - 195, 8, 4, 66, 69, 82, 82, 232, 4, 200, 1, 3, 76, 68, 32, 78, 79, 104, 7, - 80, 79, 77, 79, 70, 79, 32, 188, 6, 2, 84, 84, 216, 6, 5, 85, 81, 85, 69, - 84, 54, 87, 198, 1, 88, 206, 215, 5, 77, 178, 168, 3, 89, 170, 167, 26, - 65, 139, 34, 78, 14, 254, 190, 8, 83, 142, 204, 7, 69, 190, 207, 17, 70, - 234, 2, 87, 215, 10, 71, 10, 26, 75, 215, 144, 23, 77, 9, 40, 4, 77, 65, - 82, 75, 175, 239, 35, 83, 5, 133, 171, 23, 3, 32, 84, 65, 150, 1, 96, 13, - 70, 73, 78, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 53, 7, 76, 69, 84, - 84, 69, 82, 32, 10, 162, 238, 35, 71, 2, 72, 2, 75, 2, 80, 3, 84, 140, 1, - 234, 1, 65, 54, 85, 22, 69, 82, 71, 46, 73, 70, 78, 38, 79, 98, 90, 170, - 154, 7, 75, 178, 174, 24, 67, 2, 76, 2, 83, 246, 76, 66, 206, 201, 1, 74, - 222, 137, 2, 68, 2, 70, 2, 72, 2, 77, 2, 80, 2, 81, 2, 82, 2, 84, 2, 86, - 3, 88, 21, 50, 73, 2, 85, 74, 78, 134, 235, 35, 72, 3, 77, 5, 235, 155, - 35, 78, 17, 50, 78, 134, 235, 35, 69, 2, 72, 2, 73, 3, 82, 7, 130, 235, - 35, 71, 3, 78, 11, 230, 234, 35, 72, 2, 78, 2, 85, 3, 87, 15, 180, 239, - 33, 2, 78, 78, 134, 251, 1, 72, 2, 77, 2, 82, 3, 85, 9, 222, 238, 33, 71, - 155, 251, 1, 78, 17, 66, 78, 142, 232, 34, 32, 134, 129, 1, 69, 2, 77, 2, - 79, 3, 85, 4, 142, 233, 35, 71, 3, 78, 9, 242, 232, 35, 72, 2, 73, 3, 89, - 66, 104, 3, 79, 77, 32, 133, 179, 21, 17, 76, 69, 32, 87, 73, 84, 72, 32, - 80, 79, 80, 80, 73, 78, 71, 32, 67, 64, 152, 2, 5, 72, 65, 76, 70, 32, - 232, 1, 5, 76, 69, 70, 84, 32, 88, 6, 82, 73, 71, 72, 84, 32, 88, 14, 83, - 81, 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 206, 213, 15, 65, - 226, 134, 16, 67, 210, 3, 80, 140, 3, 13, 74, 85, 83, 84, 73, 70, 73, 69, - 68, 32, 85, 80, 80, 135, 5, 84, 32, 148, 1, 16, 70, 79, 82, 87, 65, 82, - 68, 45, 70, 65, 67, 73, 78, 71, 32, 82, 130, 225, 31, 76, 22, 82, 252, 2, - 2, 83, 84, 174, 5, 66, 231, 243, 1, 73, 10, 252, 153, 29, 11, 85, 78, 78, - 69, 82, 32, 70, 82, 65, 77, 69, 223, 200, 2, 79, 8, 192, 230, 31, 13, 74, - 85, 83, 84, 73, 70, 73, 69, 68, 32, 85, 80, 80, 126, 67, 51, 72, 8, 230, - 230, 31, 67, 50, 72, 33, 13, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, 85, - 80, 80, 5, 225, 214, 23, 9, 32, 79, 86, 69, 82, 32, 84, 79, 80, 5, 149, - 211, 34, 8, 32, 79, 70, 32, 70, 76, 79, 87, 14, 58, 76, 116, 3, 84, 73, - 69, 141, 134, 33, 3, 32, 65, 78, 6, 26, 32, 247, 143, 35, 73, 4, 148, - 183, 9, 7, 79, 70, 32, 72, 89, 71, 73, 245, 130, 23, 6, 87, 73, 84, 72, - 32, 83, 7, 207, 195, 32, 32, 218, 2, 88, 10, 32, 68, 82, 65, 87, 73, 78, - 71, 83, 32, 193, 226, 34, 6, 73, 78, 71, 32, 71, 76, 216, 2, 176, 1, 2, - 68, 79, 228, 6, 6, 72, 69, 65, 86, 89, 32, 254, 2, 76, 204, 25, 6, 82, - 73, 71, 72, 84, 32, 144, 4, 3, 85, 80, 32, 245, 3, 9, 86, 69, 82, 84, 73, - 67, 65, 76, 32, 66, 52, 5, 85, 66, 76, 69, 32, 165, 3, 3, 87, 78, 32, 30, - 58, 68, 216, 2, 2, 85, 80, 234, 5, 86, 211, 163, 29, 72, 14, 64, 8, 73, - 65, 71, 79, 78, 65, 76, 32, 149, 2, 3, 79, 87, 78, 8, 104, 6, 85, 80, 80, - 69, 82, 32, 205, 14, 15, 76, 79, 87, 69, 82, 32, 76, 69, 70, 84, 32, 84, - 79, 32, 77, 6, 76, 8, 76, 69, 70, 84, 32, 84, 79, 32, 237, 24, 6, 82, 73, - 71, 72, 84, 32, 4, 204, 23, 14, 77, 73, 68, 68, 76, 69, 32, 67, 69, 78, - 84, 82, 69, 32, 159, 173, 23, 76, 6, 199, 26, 32, 36, 128, 1, 10, 72, 69, - 65, 86, 89, 32, 65, 78, 68, 32, 132, 1, 10, 76, 73, 71, 72, 84, 32, 65, - 78, 68, 32, 210, 38, 68, 131, 4, 83, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, - 71, 72, 228, 35, 2, 85, 80, 151, 5, 72, 4, 17, 2, 84, 32, 4, 230, 32, 85, - 239, 140, 35, 76, 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 156, 36, - 2, 85, 80, 235, 4, 72, 4, 17, 2, 84, 32, 4, 198, 32, 85, 135, 9, 72, 46, - 136, 1, 4, 76, 69, 70, 84, 68, 2, 85, 80, 130, 1, 86, 172, 20, 2, 68, 79, - 170, 2, 81, 100, 2, 84, 82, 154, 140, 29, 72, 151, 134, 6, 82, 5, 45, 9, - 32, 65, 78, 68, 32, 76, 73, 71, 72, 2, 195, 132, 34, 84, 11, 29, 5, 32, - 65, 78, 68, 32, 8, 42, 76, 134, 164, 29, 72, 151, 134, 6, 82, 4, 224, - 174, 24, 4, 73, 71, 72, 84, 167, 251, 10, 69, 8, 209, 20, 7, 69, 82, 84, - 73, 67, 65, 76, 156, 1, 56, 4, 69, 70, 84, 32, 169, 2, 5, 73, 71, 72, 84, - 32, 16, 160, 27, 19, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, - 68, 32, 82, 73, 71, 72, 24, 14, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, - 82, 73, 71, 72, 100, 14, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 82, 73, - 71, 72, 97, 17, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, - 73, 71, 72, 140, 1, 248, 1, 4, 65, 82, 67, 32, 30, 68, 248, 15, 10, 72, - 79, 82, 73, 90, 79, 78, 84, 65, 76, 112, 4, 76, 69, 70, 84, 62, 81, 34, - 84, 172, 1, 2, 85, 80, 120, 8, 86, 69, 82, 84, 73, 67, 65, 76, 192, 197, - 13, 7, 66, 79, 84, 84, 79, 77, 32, 143, 203, 21, 82, 8, 230, 202, 17, 68, - 67, 85, 80, 52, 8, 73, 65, 71, 79, 78, 65, 76, 32, 199, 14, 79, 68, 168, - 1, 14, 76, 79, 87, 69, 82, 32, 76, 69, 70, 84, 32, 84, 79, 32, 96, 7, 77, - 73, 68, 68, 76, 69, 32, 136, 3, 6, 85, 80, 80, 69, 82, 32, 162, 136, 34, - 67, 183, 4, 68, 4, 38, 77, 33, 5, 85, 80, 80, 69, 82, 2, 29, 5, 73, 68, - 68, 76, 69, 2, 129, 12, 2, 32, 67, 16, 88, 8, 76, 69, 70, 84, 32, 84, 79, - 32, 213, 1, 9, 82, 73, 71, 72, 84, 32, 84, 79, 32, 10, 156, 1, 6, 76, 79, - 87, 69, 82, 32, 33, 28, 85, 80, 80, 69, 82, 32, 67, 69, 78, 84, 82, 69, - 32, 84, 79, 32, 77, 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, 84, 6, 246, - 3, 67, 255, 155, 35, 82, 5, 135, 191, 20, 32, 6, 232, 4, 15, 85, 80, 80, - 69, 82, 32, 67, 69, 78, 84, 82, 69, 32, 84, 79, 235, 3, 76, 44, 144, 1, - 10, 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 248, 3, 8, 76, 69, 70, 84, - 32, 84, 79, 32, 193, 2, 9, 82, 73, 71, 72, 84, 32, 84, 79, 32, 20, 52, 7, - 77, 73, 68, 68, 76, 69, 32, 171, 176, 23, 76, 16, 56, 4, 76, 69, 70, 84, - 165, 1, 5, 82, 73, 71, 72, 84, 9, 11, 32, 6, 84, 10, 84, 79, 32, 76, 79, - 87, 69, 82, 32, 67, 233, 186, 20, 5, 65, 78, 68, 32, 77, 4, 29, 5, 69, - 78, 84, 82, 69, 5, 133, 196, 32, 3, 32, 84, 79, 9, 11, 32, 6, 88, 3, 65, - 78, 68, 65, 15, 84, 79, 32, 76, 79, 87, 69, 82, 32, 67, 69, 78, 84, 82, - 69, 2, 241, 185, 20, 11, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 5, - 189, 211, 10, 9, 32, 84, 79, 32, 77, 73, 68, 68, 76, 14, 68, 6, 76, 79, - 87, 69, 82, 32, 97, 7, 77, 73, 68, 68, 76, 69, 32, 6, 48, 6, 67, 69, 78, - 84, 82, 69, 227, 152, 35, 82, 5, 11, 32, 2, 233, 4, 4, 84, 79, 32, 85, 8, - 76, 10, 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 33, 5, 82, 73, 71, 72, - 84, 4, 250, 3, 85, 139, 201, 13, 76, 5, 11, 32, 2, 193, 204, 13, 2, 84, - 79, 10, 46, 76, 69, 7, 77, 73, 68, 68, 76, 69, 32, 4, 29, 5, 79, 87, 69, - 82, 32, 4, 198, 191, 32, 67, 235, 214, 2, 76, 6, 34, 67, 37, 4, 76, 69, - 70, 84, 2, 45, 6, 69, 78, 84, 82, 69, 32, 5, 11, 32, 2, 133, 170, 23, 2, - 84, 79, 12, 36, 2, 87, 78, 253, 2, 2, 85, 66, 9, 11, 32, 6, 25, 4, 65, - 78, 68, 32, 6, 210, 142, 29, 72, 250, 133, 6, 76, 31, 82, 9, 11, 32, 6, - 40, 4, 65, 78, 68, 32, 227, 141, 35, 87, 4, 26, 85, 179, 168, 23, 76, 2, - 193, 168, 23, 2, 80, 80, 5, 149, 237, 33, 10, 32, 65, 78, 68, 32, 72, 69, - 65, 86, 89, 4, 109, 5, 85, 65, 68, 82, 85, 6, 66, 82, 225, 199, 13, 10, - 79, 80, 32, 65, 78, 68, 32, 85, 80, 80, 4, 11, 73, 4, 11, 80, 4, 41, 8, - 76, 69, 32, 68, 65, 83, 72, 32, 4, 190, 178, 16, 86, 151, 217, 12, 72, - 11, 29, 5, 32, 65, 78, 68, 32, 8, 34, 72, 230, 144, 35, 76, 31, 82, 4, - 200, 149, 24, 4, 69, 65, 86, 89, 175, 245, 4, 79, 17, 29, 5, 32, 65, 78, - 68, 32, 14, 162, 143, 28, 66, 42, 84, 206, 122, 72, 250, 133, 6, 76, 31, - 82, 16, 148, 2, 18, 68, 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, - 68, 32, 76, 69, 70, 24, 13, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 76, - 69, 70, 100, 13, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 69, 70, 97, - 16, 85, 80, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 2, - 101, 3, 84, 32, 85, 6, 17, 2, 84, 32, 6, 58, 85, 178, 3, 68, 245, 4, 6, - 86, 69, 82, 84, 73, 67, 2, 183, 158, 33, 80, 6, 17, 2, 84, 32, 6, 58, 85, - 134, 4, 68, 205, 4, 6, 86, 69, 82, 84, 73, 67, 2, 239, 8, 80, 2, 185, 2, - 3, 84, 32, 68, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, - 188, 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 186, 2, 68, 131, 4, - 83, 12, 80, 4, 68, 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, - 255, 4, 72, 2, 145, 5, 2, 32, 72, 4, 17, 2, 84, 32, 4, 26, 68, 191, 137, - 35, 76, 2, 133, 155, 33, 3, 79, 87, 78, 12, 80, 4, 68, 79, 87, 78, 24, 3, - 76, 69, 70, 0, 4, 82, 73, 71, 72, 211, 4, 72, 2, 229, 4, 2, 32, 72, 4, - 17, 2, 84, 32, 4, 22, 68, 131, 5, 72, 2, 233, 4, 3, 79, 87, 78, 24, 130, - 1, 68, 188, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 144, 1, 10, - 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 183, 1, 83, 6, 49, 10, 79, 85, - 66, 76, 69, 32, 65, 78, 68, 32, 6, 92, 3, 76, 69, 70, 0, 4, 82, 73, 71, - 72, 13, 10, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 2, 11, 84, 2, 233, - 231, 26, 2, 32, 83, 6, 54, 72, 68, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, - 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 145, 150, 33, 2, 65, 76, 2, 247, - 149, 33, 84, 6, 54, 72, 60, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 37, - 7, 79, 82, 73, 90, 79, 78, 84, 2, 29, 2, 65, 76, 2, 11, 84, 2, 17, 2, 32, - 72, 2, 137, 156, 35, 3, 69, 65, 86, 6, 49, 10, 73, 78, 71, 76, 69, 32, - 65, 78, 68, 32, 6, 96, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 221, 171, 26, - 9, 72, 79, 82, 73, 90, 79, 78, 84, 65, 2, 231, 171, 26, 84, 134, 6, 46, - 65, 178, 14, 69, 150, 1, 73, 175, 1, 79, 232, 5, 36, 4, 72, 77, 73, 32, - 247, 9, 73, 230, 1, 192, 1, 7, 76, 69, 84, 84, 69, 82, 32, 196, 2, 7, 78, - 85, 77, 66, 69, 82, 32, 144, 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, - 79, 78, 32, 84, 5, 83, 73, 71, 78, 32, 122, 86, 239, 244, 24, 68, 108, - 210, 1, 79, 238, 205, 31, 65, 38, 68, 114, 84, 46, 86, 186, 5, 85, 206, - 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, - 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 187, 2, 69, 15, 45, 9, - 76, 68, 32, 84, 65, 77, 73, 76, 32, 12, 158, 255, 28, 76, 206, 149, 2, - 83, 134, 92, 78, 175, 242, 2, 82, 42, 82, 69, 38, 70, 66, 78, 26, 83, - 138, 173, 21, 84, 238, 228, 5, 79, 203, 213, 7, 74, 4, 145, 135, 31, 4, - 73, 71, 72, 84, 8, 26, 79, 227, 147, 21, 73, 4, 142, 185, 33, 82, 159, - 180, 1, 85, 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, - 5, 215, 147, 35, 84, 10, 46, 76, 202, 208, 12, 68, 177, 16, 2, 67, 82, 4, - 198, 190, 19, 79, 235, 204, 14, 73, 12, 246, 143, 31, 67, 152, 62, 9, 79, - 76, 68, 32, 84, 65, 77, 73, 76, 230, 157, 1, 74, 158, 2, 86, 122, 85, - 187, 240, 1, 65, 34, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 167, - 234, 32, 73, 32, 138, 1, 79, 208, 174, 29, 11, 66, 72, 65, 84, 84, 73, - 80, 82, 79, 76, 85, 214, 159, 2, 65, 38, 85, 22, 86, 186, 201, 1, 73, - 223, 137, 2, 69, 7, 253, 142, 31, 10, 76, 68, 32, 84, 65, 77, 73, 76, 32, - 83, 130, 4, 72, 12, 76, 76, 69, 32, 80, 65, 84, 84, 69, 82, 78, 32, 231, - 160, 35, 78, 128, 4, 44, 5, 68, 79, 84, 83, 45, 171, 244, 6, 66, 254, 3, - 74, 49, 74, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, - 129, 2, 66, 50, 66, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, - 129, 1, 58, 51, 54, 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 65, 50, - 52, 46, 53, 38, 54, 30, 55, 187, 157, 35, 56, 33, 42, 53, 38, 54, 30, 55, - 187, 157, 35, 56, 17, 34, 54, 30, 55, 187, 157, 35, 56, 9, 26, 55, 187, - 157, 35, 56, 5, 183, 157, 35, 56, 8, 26, 65, 183, 134, 35, 86, 6, 156, - 185, 20, 11, 75, 32, 80, 69, 82, 77, 73, 84, 84, 69, 68, 196, 171, 8, 6, - 83, 84, 45, 70, 69, 69, 167, 184, 6, 68, 10, 42, 68, 92, 2, 69, 70, 243, - 151, 35, 67, 4, 204, 161, 32, 6, 71, 69, 32, 65, 84, 32, 193, 70, 9, 69, - 32, 87, 73, 84, 72, 32, 86, 69, 4, 190, 245, 31, 67, 203, 165, 3, 83, 12, - 84, 4, 75, 69, 78, 32, 176, 192, 8, 2, 67, 67, 160, 210, 25, 2, 87, 78, - 231, 118, 79, 6, 248, 173, 27, 17, 67, 73, 82, 67, 76, 69, 32, 87, 73, - 84, 72, 32, 78, 79, 82, 84, 72, 218, 202, 6, 66, 147, 26, 72, 134, 1, - 208, 1, 4, 66, 66, 76, 69, 46, 71, 156, 4, 4, 72, 73, 68, 32, 36, 2, 76, - 76, 122, 83, 56, 4, 84, 84, 69, 82, 176, 249, 10, 10, 73, 76, 68, 73, 78, - 71, 32, 67, 79, 78, 184, 213, 16, 2, 82, 82, 247, 239, 6, 67, 4, 204, - 145, 28, 2, 32, 84, 243, 133, 7, 83, 63, 33, 6, 73, 78, 69, 83, 69, 32, - 60, 144, 1, 7, 76, 69, 84, 84, 69, 82, 32, 172, 2, 11, 86, 79, 87, 69, - 76, 32, 83, 73, 71, 78, 32, 190, 176, 16, 69, 225, 241, 13, 4, 80, 65, - 76, 76, 46, 154, 1, 77, 34, 78, 234, 145, 35, 66, 2, 67, 2, 68, 2, 71, 2, - 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, - 2, 65, 4, 134, 146, 35, 80, 187, 2, 65, 12, 46, 71, 34, 89, 154, 145, 35, - 82, 187, 2, 65, 4, 182, 145, 35, 75, 187, 2, 65, 4, 150, 145, 35, 67, - 187, 2, 65, 10, 218, 252, 34, 65, 214, 22, 69, 2, 73, 2, 79, 3, 85, 40, - 142, 152, 10, 76, 135, 241, 18, 86, 10, 56, 2, 69, 84, 20, 4, 72, 79, 82, - 78, 231, 162, 9, 83, 5, 179, 210, 31, 32, 5, 241, 184, 27, 5, 32, 87, 73, - 84, 72, 9, 26, 84, 143, 192, 32, 32, 4, 234, 184, 27, 83, 15, 32, 5, 243, - 237, 34, 70, 240, 3, 140, 1, 23, 90, 65, 78, 84, 73, 78, 69, 32, 77, 85, - 83, 73, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 245, 179, 20, 5, 84, - 69, 32, 79, 82, 238, 3, 154, 2, 65, 128, 7, 2, 67, 72, 170, 1, 68, 158, - 5, 69, 206, 2, 70, 166, 8, 71, 202, 3, 73, 162, 2, 75, 142, 5, 76, 190, - 2, 77, 250, 4, 79, 114, 80, 174, 4, 82, 42, 83, 194, 5, 84, 196, 5, 2, - 86, 65, 250, 1, 89, 178, 160, 32, 78, 245, 142, 2, 9, 88, 73, 82, 79, 78, - 32, 75, 76, 65, 62, 60, 5, 71, 79, 71, 73, 32, 222, 1, 78, 118, 80, 199, - 2, 82, 16, 70, 65, 0, 2, 71, 79, 64, 2, 77, 69, 37, 5, 80, 79, 76, 73, - 32, 4, 17, 2, 82, 71, 4, 160, 227, 15, 2, 79, 84, 159, 169, 19, 73, 4, - 214, 151, 9, 84, 231, 224, 25, 83, 4, 26, 65, 1, 2, 71, 79, 2, 131, 203, - 17, 82, 6, 72, 6, 84, 73, 75, 69, 78, 79, 205, 233, 34, 6, 65, 84, 82, - 73, 67, 72, 4, 168, 32, 2, 75, 89, 147, 232, 34, 77, 22, 52, 5, 69, 83, - 79, 32, 69, 38, 79, 155, 246, 34, 76, 4, 204, 1, 2, 88, 79, 147, 55, 75, - 16, 80, 6, 83, 84, 82, 79, 70, 79, 188, 40, 3, 68, 69, 82, 181, 128, 1, - 2, 84, 72, 10, 24, 2, 73, 32, 79, 83, 4, 56, 9, 83, 89, 78, 68, 69, 83, - 77, 79, 83, 195, 24, 84, 2, 143, 44, 32, 7, 11, 32, 4, 214, 60, 68, 255, - 145, 22, 78, 18, 48, 2, 71, 79, 33, 6, 75, 84, 73, 75, 79, 32, 4, 170, - 21, 83, 171, 242, 34, 78, 14, 182, 190, 27, 86, 138, 170, 7, 90, 162, 8, - 75, 254, 2, 68, 2, 78, 162, 17, 71, 3, 80, 14, 80, 4, 82, 79, 65, 32, - 168, 37, 4, 79, 82, 69, 86, 221, 2, 4, 65, 77, 73, 76, 6, 228, 214, 19, - 2, 83, 80, 220, 191, 3, 3, 90, 89, 71, 185, 188, 10, 3, 75, 76, 73, 42, - 50, 73, 172, 219, 8, 2, 65, 83, 227, 138, 26, 89, 38, 122, 65, 200, 1, 5, - 69, 83, 73, 83, 32, 70, 71, 208, 1, 3, 80, 76, 73, 173, 183, 27, 8, 70, - 84, 79, 71, 71, 79, 83, 32, 10, 48, 6, 83, 84, 79, 76, 73, 32, 179, 203, - 33, 82, 8, 80, 6, 65, 80, 76, 73, 32, 77, 174, 55, 68, 141, 220, 22, 5, - 84, 72, 69, 83, 69, 4, 40, 2, 69, 71, 133, 165, 28, 2, 73, 75, 2, 195, - 135, 30, 65, 12, 38, 84, 246, 50, 65, 42, 68, 63, 77, 6, 194, 10, 69, - 239, 41, 82, 10, 72, 5, 79, 82, 71, 79, 78, 217, 128, 35, 7, 82, 65, 77, - 77, 65, 32, 71, 9, 69, 15, 32, 80, 65, 82, 69, 83, 84, 73, 71, 77, 69, - 78, 79, 78, 32, 6, 222, 38, 68, 137, 10, 8, 65, 82, 73, 83, 84, 69, 82, - 65, 5, 175, 47, 32, 16, 166, 1, 78, 116, 6, 84, 69, 82, 79, 78, 32, 188, - 44, 4, 88, 79, 32, 69, 156, 238, 32, 4, 80, 69, 71, 69, 152, 48, 6, 75, - 83, 84, 82, 69, 80, 237, 13, 3, 76, 65, 70, 4, 224, 22, 3, 68, 79, 70, - 145, 159, 27, 18, 65, 82, 88, 73, 83, 32, 75, 65, 73, 32, 70, 84, 72, 79, - 82, 65, 32, 86, 4, 208, 11, 5, 65, 82, 71, 79, 83, 151, 20, 80, 44, 180, - 1, 9, 65, 78, 69, 82, 79, 83, 73, 83, 32, 52, 6, 84, 72, 79, 82, 65, 32, - 165, 6, 22, 72, 84, 79, 82, 65, 32, 83, 75, 76, 73, 82, 79, 78, 32, 67, - 72, 82, 79, 77, 65, 32, 86, 6, 246, 4, 68, 14, 77, 25, 5, 84, 69, 84, 82, - 65, 36, 220, 2, 8, 65, 82, 67, 72, 65, 73, 79, 78, 80, 10, 68, 73, 65, - 84, 79, 78, 73, 75, 73, 32, 96, 11, 73, 32, 89, 70, 69, 83, 73, 83, 32, - 84, 69, 32, 15, 77, 65, 76, 65, 75, 79, 78, 32, 67, 72, 82, 79, 77, 65, - 32, 74, 78, 40, 8, 83, 75, 76, 73, 82, 79, 78, 32, 141, 209, 18, 18, 69, - 78, 65, 82, 77, 79, 78, 73, 79, 83, 32, 65, 78, 84, 73, 70, 79, 78, 5, - 57, 12, 32, 68, 69, 89, 84, 69, 82, 79, 85, 32, 73, 67, 2, 199, 175, 27, - 72, 14, 62, 78, 254, 216, 34, 90, 162, 8, 75, 254, 2, 68, 163, 17, 80, 6, - 242, 39, 73, 251, 147, 33, 65, 2, 237, 42, 4, 84, 65, 82, 84, 4, 18, 68, - 15, 77, 2, 35, 73, 2, 21, 3, 79, 78, 79, 2, 151, 19, 70, 4, 166, 19, 65, - 213, 219, 32, 2, 69, 78, 6, 84, 7, 67, 72, 82, 79, 77, 65, 32, 193, 149, - 17, 8, 68, 73, 65, 84, 79, 78, 79, 78, 4, 42, 86, 169, 153, 17, 4, 83, - 89, 78, 65, 2, 251, 209, 32, 65, 22, 80, 6, 69, 78, 73, 75, 73, 32, 44, - 2, 79, 82, 185, 26, 5, 82, 79, 78, 84, 72, 4, 192, 242, 30, 2, 68, 73, 1, - 2, 89, 70, 16, 68, 2, 71, 79, 225, 1, 10, 84, 72, 77, 73, 75, 79, 78, 32, - 78, 32, 12, 28, 2, 78, 32, 155, 1, 83, 10, 96, 14, 80, 65, 82, 69, 83, - 84, 73, 71, 77, 69, 78, 79, 78, 32, 242, 33, 65, 113, 3, 78, 69, 79, 4, - 214, 24, 68, 249, 202, 32, 5, 65, 82, 73, 83, 84, 2, 153, 191, 33, 5, 89, - 78, 84, 72, 69, 4, 142, 28, 65, 1, 2, 68, 73, 16, 56, 2, 77, 73, 114, 83, - 225, 240, 33, 4, 67, 72, 65, 68, 8, 34, 70, 169, 28, 3, 68, 73, 65, 6, - 40, 4, 84, 72, 79, 82, 195, 230, 33, 79, 4, 242, 160, 34, 79, 227, 79, - 65, 6, 44, 6, 65, 75, 73, 65, 32, 84, 231, 13, 79, 2, 153, 201, 21, 12, - 69, 76, 79, 85, 83, 32, 73, 67, 72, 73, 77, 65, 54, 108, 2, 65, 84, 96, - 6, 69, 78, 84, 73, 77, 65, 140, 1, 5, 76, 65, 83, 77, 65, 18, 79, 98, 82, - 175, 1, 89, 6, 40, 3, 65, 86, 65, 201, 3, 2, 72, 73, 4, 140, 29, 5, 32, - 84, 82, 79, 77, 135, 176, 34, 83, 18, 24, 2, 84, 65, 15, 32, 11, 11, 32, - 8, 36, 4, 78, 69, 79, 32, 183, 28, 65, 6, 252, 140, 28, 2, 77, 69, 254, - 212, 2, 75, 151, 226, 3, 65, 7, 243, 28, 32, 8, 64, 6, 78, 84, 69, 86, - 77, 65, 202, 212, 8, 82, 151, 246, 25, 85, 5, 181, 166, 14, 2, 32, 65, - 14, 44, 4, 65, 84, 73, 77, 105, 3, 69, 77, 65, 12, 18, 65, 35, 79, 8, - 182, 23, 32, 195, 209, 34, 84, 4, 134, 12, 75, 181, 184, 31, 5, 89, 80, - 79, 82, 82, 2, 227, 140, 27, 83, 2, 159, 201, 34, 76, 14, 22, 69, 147, - 22, 89, 12, 44, 5, 73, 77, 77, 65, 32, 191, 251, 29, 77, 10, 72, 2, 69, - 78, 0, 5, 73, 77, 73, 83, 69, 54, 84, 69, 3, 68, 89, 79, 2, 161, 160, 27, - 8, 79, 83, 32, 67, 72, 82, 79, 78, 4, 44, 5, 69, 83, 83, 65, 82, 1, 2, - 82, 73, 2, 17, 2, 79, 78, 2, 25, 4, 32, 67, 72, 82, 2, 247, 221, 33, 79, - 28, 84, 8, 65, 82, 84, 89, 82, 73, 65, 32, 229, 231, 30, 7, 73, 75, 82, - 79, 78, 32, 73, 26, 72, 5, 65, 76, 76, 73, 32, 38, 68, 38, 80, 134, 1, - 86, 30, 84, 87, 76, 4, 34, 68, 177, 2, 3, 80, 82, 79, 2, 237, 2, 5, 69, - 89, 84, 69, 82, 8, 60, 7, 76, 65, 71, 73, 79, 83, 32, 45, 4, 82, 79, 84, - 79, 4, 200, 1, 5, 84, 69, 84, 65, 82, 111, 73, 4, 22, 86, 227, 1, 83, 2, - 209, 1, 3, 65, 82, 89, 8, 56, 8, 69, 84, 65, 82, 84, 79, 83, 32, 61, 2, - 82, 73, 4, 22, 76, 135, 1, 73, 2, 21, 3, 69, 71, 69, 2, 63, 84, 4, 18, - 70, 35, 84, 2, 209, 197, 21, 3, 79, 78, 73, 2, 11, 79, 2, 11, 83, 2, 17, - 2, 32, 73, 2, 155, 239, 22, 67, 18, 92, 4, 76, 73, 71, 79, 180, 1, 5, 89, - 82, 65, 78, 73, 210, 14, 88, 165, 130, 34, 2, 77, 65, 4, 211, 1, 78, 42, - 60, 2, 65, 82, 100, 2, 73, 65, 90, 69, 169, 1, 2, 83, 73, 12, 40, 2, 65, - 75, 181, 170, 17, 2, 73, 67, 10, 52, 3, 65, 76, 69, 45, 6, 76, 73, 84, - 73, 75, 73, 4, 11, 83, 4, 17, 2, 77, 65, 4, 23, 32, 7, 11, 32, 4, 198, - 15, 65, 155, 151, 22, 78, 12, 72, 3, 84, 65, 83, 136, 3, 6, 76, 65, 83, - 84, 79, 78, 191, 216, 8, 82, 6, 26, 84, 247, 220, 34, 77, 4, 32, 2, 79, - 75, 139, 223, 34, 73, 2, 209, 189, 34, 2, 79, 85, 14, 36, 5, 70, 73, 83, - 84, 79, 67, 76, 10, 46, 80, 214, 1, 78, 170, 8, 76, 187, 1, 83, 2, 135, - 11, 65, 4, 162, 142, 34, 79, 227, 79, 73, 4, 178, 5, 69, 137, 183, 34, 2, - 65, 80, 42, 120, 5, 69, 73, 83, 77, 65, 32, 8, 73, 77, 65, 78, 83, 73, - 83, 32, 182, 1, 84, 154, 1, 89, 245, 240, 1, 3, 65, 88, 73, 5, 11, 32, 2, - 223, 162, 22, 78, 16, 36, 2, 65, 82, 1, 3, 84, 72, 69, 8, 25, 4, 83, 69, - 79, 83, 9, 11, 32, 6, 18, 84, 39, 68, 4, 34, 82, 13, 4, 69, 84, 82, 65, - 2, 11, 73, 2, 141, 146, 27, 3, 83, 73, 77, 8, 68, 5, 65, 86, 82, 79, 83, - 52, 4, 82, 65, 71, 71, 179, 199, 16, 73, 5, 29, 5, 32, 65, 80, 79, 68, 2, - 255, 219, 8, 69, 2, 141, 241, 1, 2, 73, 83, 12, 34, 78, 149, 1, 3, 82, - 77, 65, 8, 36, 5, 65, 71, 77, 65, 32, 91, 69, 6, 154, 8, 65, 154, 151, - 22, 78, 217, 240, 4, 10, 77, 69, 84, 65, 32, 83, 84, 65, 86, 82, 2, 159, - 183, 34, 86, 5, 11, 84, 2, 243, 245, 16, 73, 40, 42, 69, 70, 72, 218, 1, - 82, 227, 2, 73, 6, 136, 12, 3, 84, 82, 65, 242, 161, 8, 76, 201, 154, 24, - 2, 83, 83, 12, 26, 69, 167, 178, 34, 73, 10, 64, 2, 77, 65, 145, 194, 33, - 8, 83, 32, 75, 65, 73, 32, 65, 80, 9, 56, 2, 32, 65, 33, 8, 84, 73, 83, - 77, 79, 83, 32, 69, 2, 189, 213, 33, 3, 80, 76, 79, 4, 218, 182, 34, 83, - 3, 88, 20, 38, 73, 73, 5, 79, 77, 73, 75, 79, 6, 48, 2, 71, 79, 206, 217, - 29, 80, 143, 251, 4, 65, 2, 183, 156, 33, 82, 14, 42, 76, 32, 2, 78, 32, - 62, 80, 95, 83, 2, 11, 89, 2, 227, 178, 34, 71, 6, 26, 65, 139, 154, 22, - 78, 4, 250, 2, 82, 147, 174, 34, 76, 4, 46, 65, 129, 160, 33, 5, 83, 73, - 70, 73, 83, 2, 237, 177, 34, 6, 82, 65, 75, 65, 76, 69, 2, 11, 89, 2, - 145, 192, 16, 2, 78, 65, 10, 26, 82, 219, 176, 34, 84, 8, 21, 3, 69, 73, - 65, 8, 26, 32, 113, 2, 73, 32, 6, 38, 69, 242, 5, 68, 255, 145, 22, 78, - 2, 11, 75, 2, 29, 5, 70, 79, 78, 73, 84, 2, 165, 129, 34, 2, 73, 75, 2, - 11, 65, 2, 11, 82, 2, 153, 128, 34, 3, 67, 72, 65, 22, 28, 2, 70, 69, - 231, 3, 80, 14, 34, 78, 49, 4, 83, 73, 83, 32, 4, 11, 32, 4, 202, 196, - 30, 75, 151, 226, 3, 65, 10, 42, 65, 42, 68, 62, 77, 89, 2, 84, 82, 2, - 133, 2, 6, 80, 76, 73, 32, 68, 89, 2, 233, 1, 11, 73, 71, 82, 65, 77, 77, - 79, 83, 32, 69, 88, 2, 173, 1, 18, 79, 78, 79, 71, 82, 65, 77, 77, 79, - 83, 32, 84, 69, 83, 83, 69, 82, 65, 4, 11, 73, 4, 60, 11, 71, 82, 65, 77, - 77, 79, 83, 32, 79, 75, 84, 59, 84, 2, 11, 79, 2, 181, 174, 2, 6, 32, 68, - 79, 68, 69, 75, 2, 153, 252, 33, 4, 73, 77, 79, 82, 8, 26, 79, 139, 209, - 29, 83, 6, 56, 6, 75, 82, 73, 83, 73, 83, 181, 221, 29, 2, 82, 82, 5, 17, - 2, 32, 68, 2, 11, 73, 2, 183, 208, 29, 80, 172, 82, 182, 1, 65, 134, 72, - 69, 230, 1, 72, 154, 31, 73, 208, 41, 3, 74, 75, 32, 138, 26, 76, 186, - 18, 79, 194, 113, 82, 234, 7, 85, 150, 128, 2, 89, 242, 164, 19, 71, 226, - 216, 10, 83, 203, 18, 67, 198, 13, 110, 68, 66, 76, 50, 77, 90, 78, 174, - 51, 80, 62, 82, 198, 7, 84, 138, 1, 85, 242, 164, 18, 67, 187, 203, 6, - 83, 4, 34, 85, 137, 139, 27, 2, 65, 32, 2, 221, 194, 32, 2, 67, 69, 4, - 152, 207, 9, 3, 76, 32, 77, 247, 200, 19, 69, 6, 36, 3, 69, 82, 65, 231, - 246, 33, 80, 5, 165, 177, 27, 7, 32, 87, 73, 84, 72, 32, 70, 193, 11, - 148, 1, 16, 65, 68, 73, 65, 78, 32, 83, 89, 76, 76, 65, 66, 73, 67, 83, - 32, 148, 49, 2, 67, 69, 94, 68, 216, 227, 23, 3, 78, 69, 68, 131, 154, - 10, 79, 172, 11, 226, 1, 65, 190, 1, 66, 162, 2, 67, 242, 8, 69, 50, 70, - 198, 4, 72, 38, 73, 30, 75, 134, 1, 76, 82, 77, 174, 1, 78, 202, 4, 81, - 178, 1, 79, 194, 1, 80, 70, 82, 142, 1, 83, 194, 4, 84, 246, 3, 87, 254, - 5, 89, 247, 204, 17, 71, 21, 90, 65, 30, 73, 40, 10, 84, 72, 65, 80, 65, - 83, 67, 65, 78, 32, 150, 195, 34, 78, 3, 89, 7, 214, 195, 34, 73, 3, 89, - 5, 169, 221, 23, 5, 86, 73, 76, 73, 75, 4, 146, 195, 34, 77, 3, 83, 42, - 156, 1, 9, 76, 65, 67, 75, 70, 79, 79, 84, 32, 172, 151, 19, 9, 73, 66, - 76, 69, 45, 67, 82, 69, 69, 221, 202, 7, 10, 69, 65, 86, 69, 82, 32, 68, - 69, 78, 69, 36, 82, 87, 230, 228, 11, 75, 2, 78, 162, 220, 22, 65, 2, 69, - 2, 73, 2, 79, 3, 83, 11, 130, 193, 34, 65, 2, 69, 2, 73, 3, 79, 141, 3, - 82, 65, 186, 42, 87, 142, 169, 8, 72, 222, 166, 23, 79, 134, 60, 73, 223, - 137, 2, 69, 241, 2, 48, 6, 82, 82, 73, 69, 82, 32, 239, 181, 32, 65, 234, - 2, 170, 1, 68, 122, 71, 94, 72, 46, 73, 46, 74, 82, 75, 30, 78, 66, 83, - 66, 80, 2, 90, 58, 84, 38, 76, 132, 1, 2, 67, 72, 2, 77, 2, 82, 2, 87, 2, - 89, 207, 161, 34, 69, 32, 46, 69, 202, 5, 76, 2, 90, 163, 184, 34, 73, 6, - 26, 78, 207, 189, 34, 69, 4, 146, 134, 17, 69, 237, 210, 6, 2, 84, 65, - 30, 254, 4, 72, 182, 166, 21, 87, 234, 165, 7, 65, 230, 171, 5, 69, 162, - 64, 73, 2, 79, 3, 85, 19, 162, 4, 87, 206, 161, 34, 69, 215, 22, 73, 5, - 225, 135, 14, 6, 78, 73, 84, 73, 65, 76, 26, 202, 3, 74, 130, 248, 33, - 69, 234, 61, 87, 186, 2, 65, 2, 73, 2, 79, 3, 85, 26, 154, 1, 75, 227, 1, - 72, 14, 222, 250, 33, 69, 162, 64, 65, 2, 71, 2, 73, 2, 79, 3, 85, 26, - 62, 72, 226, 249, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 15, 222, 249, - 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 72, 34, 76, 70, 84, 66, 72, 3, - 83, 24, 130, 1, 72, 130, 248, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, - 24, 62, 83, 130, 248, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 12, 254, - 247, 33, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 7, 236, 29, 4, 65, 83, 84, - 69, 251, 153, 34, 78, 51, 74, 65, 22, 73, 130, 229, 31, 85, 250, 11, 79, - 150, 83, 87, 207, 242, 1, 69, 7, 143, 173, 32, 65, 33, 40, 4, 78, 65, 76, - 32, 175, 182, 34, 73, 28, 160, 1, 2, 68, 79, 134, 1, 82, 78, 83, 166, - 133, 14, 71, 186, 2, 65, 128, 177, 3, 6, 66, 79, 84, 84, 79, 77, 0, 3, - 84, 79, 80, 166, 244, 12, 80, 247, 244, 2, 77, 6, 44, 5, 85, 66, 76, 69, - 32, 131, 160, 24, 87, 4, 252, 198, 6, 12, 83, 72, 79, 82, 84, 32, 86, 69, - 82, 84, 73, 67, 255, 193, 7, 65, 6, 38, 73, 213, 231, 32, 3, 65, 73, 83, - 4, 230, 184, 17, 71, 163, 250, 16, 78, 4, 152, 146, 25, 4, 77, 65, 76, - 76, 213, 252, 6, 4, 72, 79, 82, 84, 4, 150, 176, 26, 89, 223, 130, 8, 75, - 7, 206, 178, 34, 73, 3, 78, 39, 66, 87, 234, 27, 65, 170, 208, 31, 79, - 134, 60, 73, 223, 137, 2, 69, 19, 138, 143, 12, 65, 134, 221, 19, 79, - 134, 60, 73, 223, 137, 2, 69, 49, 142, 7, 72, 154, 20, 65, 66, 87, 234, - 207, 31, 79, 134, 60, 73, 223, 137, 2, 69, 43, 70, 69, 38, 79, 238, 25, - 65, 66, 87, 238, 139, 32, 73, 223, 137, 2, 72, 7, 197, 208, 26, 4, 68, - 73, 65, 76, 7, 11, 79, 5, 249, 234, 32, 8, 83, 69, 45, 67, 82, 69, 69, - 32, 149, 1, 164, 1, 8, 45, 67, 82, 69, 69, 32, 84, 72, 38, 65, 250, 2, - 71, 76, 2, 78, 71, 48, 4, 85, 78, 65, 86, 142, 20, 79, 30, 87, 238, 139, - 32, 73, 222, 137, 2, 69, 3, 72, 6, 170, 164, 32, 73, 223, 137, 2, 69, 63, - 104, 6, 83, 75, 65, 80, 73, 32, 188, 1, 7, 84, 84, 73, 76, 73, 75, 32, - 226, 161, 32, 65, 223, 137, 2, 89, 30, 70, 83, 86, 87, 210, 17, 67, 2, - 75, 2, 77, 2, 78, 2, 84, 3, 89, 14, 168, 192, 28, 2, 75, 87, 254, 122, - 67, 2, 80, 2, 84, 190, 254, 2, 87, 175, 116, 45, 4, 230, 140, 34, 79, - 191, 28, 65, 24, 30, 72, 1, 3, 83, 72, 82, 12, 150, 191, 28, 65, 166, - 166, 3, 79, 135, 60, 73, 19, 38, 65, 230, 228, 31, 79, 135, 60, 73, 9, - 230, 160, 32, 65, 223, 137, 2, 73, 15, 154, 190, 28, 65, 166, 166, 3, 79, - 135, 60, 73, 18, 224, 9, 3, 73, 75, 32, 185, 186, 23, 2, 85, 84, 33, 68, - 7, 74, 73, 66, 87, 65, 89, 32, 246, 168, 34, 78, 2, 79, 3, 89, 24, 74, - 78, 142, 136, 30, 83, 158, 160, 4, 67, 2, 75, 2, 77, 2, 80, 3, 84, 11, - 11, 87, 8, 186, 226, 31, 79, 135, 60, 73, 39, 206, 4, 87, 166, 13, 65, - 38, 79, 138, 140, 32, 73, 223, 137, 2, 69, 39, 104, 7, 45, 67, 82, 69, - 69, 32, 82, 146, 14, 87, 182, 2, 65, 170, 208, 31, 79, 134, 60, 73, 223, - 137, 2, 69, 4, 246, 143, 34, 87, 215, 22, 69, 125, 94, 65, 218, 1, 72, - 138, 1, 79, 238, 2, 87, 226, 195, 11, 80, 198, 210, 20, 73, 223, 137, 2, - 69, 43, 26, 89, 215, 155, 32, 65, 37, 25, 4, 73, 83, 73, 32, 34, 70, 72, - 54, 74, 218, 4, 83, 234, 139, 34, 89, 202, 18, 84, 147, 1, 77, 10, 234, - 222, 31, 79, 226, 197, 2, 65, 2, 69, 3, 73, 6, 238, 208, 30, 85, 171, - 211, 3, 73, 37, 70, 87, 202, 13, 79, 130, 243, 11, 65, 138, 153, 20, 73, - 223, 137, 2, 69, 16, 198, 13, 79, 226, 169, 28, 65, 170, 226, 3, 73, 223, - 137, 2, 69, 15, 80, 12, 85, 84, 72, 45, 83, 76, 65, 86, 69, 89, 32, 75, - 154, 162, 34, 79, 3, 89, 8, 134, 161, 34, 65, 2, 69, 2, 73, 3, 79, 117, - 110, 72, 190, 1, 76, 78, 84, 238, 8, 65, 66, 87, 170, 185, 11, 89, 194, - 150, 20, 79, 134, 60, 73, 223, 137, 2, 69, 39, 108, 7, 45, 67, 82, 69, - 69, 32, 84, 226, 4, 87, 170, 175, 28, 65, 166, 166, 3, 79, 134, 60, 73, - 223, 137, 2, 69, 16, 11, 72, 17, 250, 179, 28, 65, 166, 166, 3, 79, 134, - 60, 73, 223, 137, 2, 69, 12, 11, 72, 12, 210, 217, 31, 79, 142, 175, 2, - 87, 214, 22, 65, 2, 69, 3, 73, 24, 50, 72, 194, 158, 34, 65, 2, 69, 2, - 73, 3, 79, 17, 186, 178, 28, 65, 166, 166, 3, 79, 142, 175, 2, 87, 214, - 22, 69, 3, 73, 224, 1, 54, 69, 218, 3, 79, 130, 247, 11, 65, 139, 153, - 20, 73, 185, 1, 17, 2, 83, 84, 182, 1, 44, 6, 45, 67, 82, 69, 69, 32, - 251, 2, 69, 180, 1, 110, 76, 66, 77, 2, 80, 2, 89, 16, 2, 78, 87, 38, 82, - 62, 83, 26, 67, 2, 75, 18, 84, 26, 70, 199, 4, 87, 27, 178, 6, 87, 198, - 169, 28, 65, 166, 166, 3, 79, 227, 197, 2, 69, 17, 243, 5, 87, 6, 166, - 175, 28, 65, 135, 236, 5, 69, 13, 186, 168, 32, 87, 206, 242, 1, 65, 2, - 69, 2, 73, 3, 79, 28, 22, 72, 239, 4, 87, 14, 235, 4, 87, 16, 22, 72, - 199, 4, 87, 2, 191, 167, 32, 87, 2, 171, 183, 23, 82, 31, 11, 79, 29, 41, - 8, 68, 83, 45, 67, 82, 69, 69, 32, 26, 56, 2, 84, 72, 225, 150, 32, 6, - 70, 73, 78, 65, 76, 32, 25, 50, 87, 190, 152, 34, 65, 2, 69, 2, 73, 3, - 79, 14, 182, 172, 28, 65, 166, 166, 3, 79, 134, 60, 73, 139, 243, 1, 69, - 61, 92, 6, 45, 67, 82, 69, 69, 32, 150, 1, 65, 38, 79, 30, 87, 238, 139, - 32, 73, 223, 137, 2, 69, 24, 110, 80, 134, 207, 31, 67, 2, 75, 2, 76, 2, - 77, 2, 78, 2, 83, 2, 84, 2, 89, 182, 168, 2, 79, 247, 30, 87, 4, 222, - 163, 32, 87, 219, 211, 1, 79, 9, 170, 140, 32, 65, 223, 137, 2, 89, 7, - 226, 149, 34, 79, 3, 89, 14, 194, 169, 28, 65, 166, 166, 3, 79, 134, 60, - 73, 223, 137, 2, 69, 10, 26, 76, 239, 148, 34, 82, 9, 26, 32, 155, 181, - 9, 76, 4, 182, 150, 22, 67, 151, 149, 10, 84, 4, 214, 253, 33, 76, 215, - 22, 89, 4, 216, 171, 15, 3, 73, 84, 85, 253, 132, 8, 3, 82, 73, 67, 124, - 140, 1, 2, 68, 32, 102, 69, 72, 11, 73, 65, 78, 32, 76, 69, 84, 84, 69, - 82, 32, 210, 3, 79, 62, 80, 90, 82, 141, 212, 27, 4, 32, 83, 76, 73, 6, - 52, 5, 73, 78, 68, 69, 88, 169, 133, 33, 2, 70, 73, 5, 165, 131, 33, 6, - 32, 68, 73, 86, 73, 68, 6, 26, 84, 227, 190, 21, 32, 5, 181, 156, 10, 6, - 32, 73, 78, 83, 69, 82, 98, 196, 1, 2, 67, 45, 38, 76, 22, 77, 50, 78, - 38, 83, 46, 84, 22, 85, 186, 213, 3, 65, 2, 68, 2, 69, 2, 71, 2, 75, 2, - 80, 130, 230, 26, 82, 238, 200, 1, 73, 222, 137, 2, 66, 2, 79, 2, 81, 3, - 88, 4, 174, 242, 26, 49, 219, 167, 3, 51, 7, 223, 214, 3, 68, 11, 11, 66, - 9, 134, 143, 34, 50, 2, 51, 3, 52, 9, 226, 142, 34, 68, 2, 71, 3, 78, 13, - 246, 213, 3, 72, 2, 84, 203, 184, 30, 83, 7, 203, 213, 3, 84, 13, 11, 85, - 11, 11, 85, 9, 230, 141, 34, 50, 2, 51, 3, 85, 4, 160, 132, 33, 6, 85, - 83, 69, 76, 32, 72, 163, 137, 1, 78, 4, 224, 189, 27, 6, 32, 83, 84, 82, - 69, 65, 249, 208, 5, 7, 69, 78, 84, 82, 89, 32, 83, 4, 142, 200, 27, 73, - 247, 167, 6, 79, 9, 29, 5, 32, 70, 65, 67, 69, 7, 33, 6, 32, 87, 73, 84, - 72, 32, 4, 208, 235, 6, 2, 84, 69, 129, 251, 24, 6, 87, 82, 89, 32, 83, - 77, 108, 88, 16, 67, 65, 83, 73, 65, 78, 32, 65, 76, 66, 65, 78, 73, 65, - 78, 32, 235, 137, 30, 84, 106, 60, 7, 76, 69, 84, 84, 69, 82, 32, 161, - 250, 27, 2, 67, 73, 104, 174, 2, 65, 34, 67, 146, 1, 68, 78, 69, 34, 71, - 46, 73, 46, 74, 34, 75, 34, 76, 34, 80, 34, 83, 74, 84, 62, 89, 46, 90, - 128, 241, 16, 2, 81, 65, 214, 192, 6, 77, 148, 173, 6, 3, 86, 69, 89, - 178, 232, 2, 70, 128, 19, 3, 78, 79, 87, 150, 20, 82, 242, 21, 88, 154, - 46, 79, 202, 26, 66, 237, 24, 3, 72, 69, 89, 4, 218, 206, 33, 79, 179, - 28, 76, 16, 34, 65, 34, 72, 49, 2, 89, 65, 4, 146, 183, 33, 89, 227, 79, - 82, 8, 242, 166, 28, 65, 154, 206, 5, 79, 203, 17, 73, 4, 162, 134, 34, - 87, 3, 89, 8, 42, 90, 222, 187, 32, 89, 215, 173, 1, 65, 4, 214, 249, 32, - 89, 191, 122, 65, 4, 218, 181, 33, 89, 227, 79, 66, 4, 136, 217, 32, 2, - 72, 69, 187, 155, 1, 73, 6, 138, 201, 32, 82, 134, 108, 87, 135, 77, 78, - 4, 146, 248, 32, 72, 199, 14, 65, 4, 130, 134, 33, 73, 199, 69, 65, 4, - 162, 180, 33, 65, 171, 51, 89, 4, 142, 1, 73, 247, 178, 33, 69, 8, 34, - 72, 137, 128, 34, 2, 69, 89, 6, 182, 151, 20, 65, 163, 218, 13, 79, 6, - 38, 73, 210, 246, 32, 89, 171, 78, 65, 2, 247, 201, 33, 87, 4, 156, 188, - 33, 2, 65, 89, 1, 2, 79, 87, 6, 130, 147, 16, 72, 219, 185, 4, 65, 16, - 72, 2, 68, 73, 32, 2, 78, 84, 212, 156, 32, 3, 76, 84, 73, 187, 80, 82, - 4, 174, 250, 32, 32, 223, 100, 76, 8, 32, 2, 82, 69, 239, 249, 32, 32, 6, - 44, 5, 76, 73, 78, 69, 32, 147, 226, 4, 32, 4, 178, 214, 21, 76, 155, - 194, 10, 79, 252, 5, 102, 65, 170, 17, 69, 134, 8, 73, 150, 1, 79, 244, - 191, 13, 6, 82, 73, 83, 84, 77, 65, 139, 130, 10, 85, 198, 2, 66, 73, 32, - 4, 75, 77, 65, 32, 148, 6, 2, 77, 32, 131, 8, 82, 4, 250, 192, 33, 78, - 223, 61, 82, 142, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 142, 3, 83, - 98, 86, 214, 202, 23, 68, 194, 221, 7, 81, 200, 94, 5, 77, 65, 65, 89, - 89, 136, 177, 1, 2, 65, 85, 3, 79, 76, 202, 1, 68, 54, 78, 54, 84, 54, - 89, 166, 250, 28, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, 2, 80, 190, 141, - 3, 72, 2, 77, 2, 82, 2, 83, 2, 86, 2, 87, 150, 240, 1, 65, 186, 2, 69, 2, - 73, 3, 85, 8, 190, 251, 28, 68, 190, 141, 3, 72, 151, 240, 1, 65, 8, 198, - 136, 32, 71, 2, 78, 2, 89, 151, 240, 1, 65, 8, 214, 250, 28, 84, 190, - 141, 3, 72, 151, 240, 1, 65, 4, 222, 135, 32, 89, 151, 240, 1, 65, 8, 40, - 4, 73, 71, 78, 32, 155, 249, 21, 69, 6, 154, 229, 29, 67, 154, 222, 1, - 86, 179, 241, 1, 65, 26, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 143, 192, 31, 73, 24, 194, 158, 30, 65, 250, 6, 85, 206, 201, 1, 69, 2, - 73, 3, 79, 166, 1, 236, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, - 83, 73, 71, 78, 32, 112, 7, 76, 69, 84, 84, 69, 82, 32, 152, 4, 12, 80, - 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 60, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 207, 255, 31, 68, 14, 72, 6, 70, 73, 78, 65, 76, - 32, 174, 243, 33, 76, 2, 82, 2, 87, 3, 89, 6, 142, 245, 33, 78, 86, 72, - 3, 77, 104, 132, 2, 6, 70, 73, 78, 65, 76, 32, 110, 78, 50, 77, 78, 80, - 226, 215, 11, 66, 138, 135, 6, 68, 206, 134, 12, 83, 194, 130, 2, 65, - 156, 194, 1, 2, 67, 72, 2, 71, 2, 74, 2, 75, 2, 84, 138, 69, 72, 2, 76, - 2, 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 22, 158, 248, 31, - 78, 190, 189, 1, 83, 206, 60, 67, 146, 1, 71, 2, 75, 2, 76, 2, 80, 2, 82, - 2, 84, 3, 89, 14, 46, 71, 34, 72, 170, 219, 33, 85, 215, 22, 65, 4, 198, - 219, 33, 85, 215, 22, 65, 6, 166, 219, 33, 85, 158, 20, 74, 187, 2, 65, - 6, 150, 239, 33, 72, 2, 80, 187, 2, 65, 8, 246, 156, 11, 83, 186, 247, - 18, 68, 45, 4, 84, 82, 73, 80, 20, 158, 157, 30, 65, 242, 201, 1, 73, - 190, 201, 1, 79, 2, 85, 203, 44, 69, 14, 72, 7, 65, 67, 84, 69, 82, 32, - 84, 49, 7, 84, 32, 87, 73, 84, 72, 32, 8, 240, 242, 9, 3, 65, 66, 85, - 147, 230, 23, 73, 6, 128, 1, 13, 85, 80, 87, 65, 82, 68, 83, 32, 84, 82, - 69, 78, 68, 217, 160, 25, 12, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, - 82, 5, 225, 242, 6, 6, 32, 65, 78, 68, 32, 89, 234, 2, 80, 2, 67, 75, 82, - 69, 90, 82, 156, 247, 4, 2, 83, 84, 229, 151, 15, 2, 81, 85, 6, 56, 8, - 69, 82, 32, 66, 79, 65, 82, 68, 175, 171, 33, 32, 5, 211, 244, 30, 32, 4, - 140, 177, 27, 4, 83, 69, 32, 87, 229, 144, 5, 9, 82, 73, 78, 71, 32, 77, - 69, 71, 65, 220, 2, 40, 5, 79, 75, 69, 69, 32, 143, 5, 82, 216, 2, 46, + 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 4, 178, 239, 36, 78, 87, 72, 76, + 194, 1, 77, 114, 78, 68, 2, 80, 65, 38, 83, 192, 248, 18, 4, 75, 65, 82, + 79, 218, 241, 17, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 76, 2, 82, 2, + 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 10, 26, 65, 215, 235, 36, 66, 9, 45, + 9, 78, 68, 65, 73, 76, 73, 78, 71, 32, 6, 162, 235, 36, 72, 2, 78, 3, 83, + 10, 152, 2, 2, 79, 82, 230, 232, 36, 68, 2, 71, 2, 89, 187, 2, 65, 5, + 221, 142, 21, 4, 75, 80, 65, 75, 24, 80, 10, 73, 77, 65, 76, 85, 78, 71, + 85, 78, 32, 96, 2, 79, 85, 159, 235, 36, 65, 20, 194, 233, 36, 71, 2, 72, + 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 87, 2, 89, 187, 2, 65, 2, 233, 248, + 18, 5, 84, 72, 69, 82, 78, 4, 166, 2, 71, 249, 248, 18, 4, 79, 78, 71, + 79, 10, 96, 12, 89, 77, 66, 79, 76, 32, 66, 73, 78, 68, 85, 32, 189, 198, + 10, 6, 73, 71, 78, 32, 84, 79, 8, 78, 80, 220, 177, 11, 3, 74, 85, 68, + 145, 180, 25, 6, 78, 65, 32, 77, 69, 84, 4, 64, 3, 65, 78, 71, 249, 182, + 23, 7, 73, 78, 65, 82, 66, 79, 82, 2, 131, 255, 25, 79, 18, 122, 85, 136, + 175, 26, 6, 80, 65, 75, 80, 65, 75, 152, 254, 2, 5, 75, 65, 82, 79, 32, + 254, 249, 6, 69, 162, 64, 73, 3, 79, 5, 249, 218, 30, 15, 32, 70, 79, 82, + 32, 83, 73, 77, 65, 76, 85, 78, 71, 85, 78, 5, 235, 245, 23, 84, 228, 2, + 182, 1, 65, 230, 2, 69, 50, 76, 146, 1, 78, 188, 10, 9, 82, 73, 65, 32, + 69, 82, 70, 69, 32, 186, 5, 84, 228, 136, 33, 2, 67, 65, 176, 185, 2, 5, + 86, 69, 82, 65, 71, 243, 142, 1, 68, 20, 136, 1, 4, 77, 69, 68, 32, 170, + 1, 82, 140, 176, 3, 10, 67, 72, 32, 87, 73, 84, 72, 32, 85, 77, 194, 169, + 7, 84, 178, 203, 25, 86, 19, 78, 8, 86, 65, 0, 2, 68, 69, 52, 4, 69, 73, + 71, 72, 1, 7, 83, 73, 88, 84, 69, 69, 78, 2, 169, 190, 20, 8, 83, 67, 69, + 78, 68, 73, 78, 71, 2, 161, 190, 20, 2, 84, 72, 4, 168, 222, 32, 3, 68, + 69, 68, 171, 199, 3, 32, 4, 152, 209, 8, 3, 82, 32, 77, 227, 234, 26, 84, + 15, 11, 76, 13, 54, 32, 200, 182, 25, 3, 72, 79, 80, 231, 236, 9, 79, 6, + 162, 204, 12, 80, 236, 172, 16, 6, 87, 73, 84, 72, 32, 67, 139, 211, 6, + 83, 208, 1, 84, 5, 71, 65, 76, 73, 32, 158, 9, 84, 37, 9, 90, 69, 78, 69, + 32, 82, 73, 78, 71, 200, 1, 210, 1, 65, 40, 9, 67, 85, 82, 82, 69, 78, + 67, 89, 32, 148, 2, 7, 76, 69, 84, 84, 69, 82, 32, 204, 3, 6, 82, 85, 80, + 69, 69, 32, 34, 83, 246, 239, 22, 73, 134, 4, 86, 158, 240, 11, 68, 225, + 110, 3, 71, 65, 78, 6, 138, 190, 32, 66, 50, 78, 135, 63, 85, 12, 120, + 10, 78, 85, 77, 69, 82, 65, 84, 79, 82, 32, 153, 196, 23, 14, 68, 69, 78, + 79, 77, 73, 78, 65, 84, 79, 82, 32, 83, 73, 10, 52, 3, 79, 78, 69, 158, + 193, 31, 70, 135, 169, 3, 84, 5, 173, 208, 29, 19, 32, 76, 69, 83, 83, + 32, 84, 72, 65, 78, 32, 84, 72, 69, 32, 68, 69, 78, 79, 108, 226, 1, 75, + 90, 82, 238, 209, 21, 86, 190, 143, 8, 89, 178, 154, 3, 65, 38, 68, 114, + 84, 230, 5, 85, 186, 202, 1, 73, 138, 196, 1, 78, 46, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 80, 138, 69, 72, 2, 76, 2, 77, 186, 2, 69, 3, 79, 8, 26, + 72, 139, 218, 36, 65, 6, 26, 65, 215, 145, 14, 73, 5, 185, 231, 18, 3, + 78, 68, 65, 10, 34, 65, 242, 214, 36, 72, 3, 82, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 232, 174, 25, 5, 76, 79, 87, 69, 82, 1, 6, 77, 73, 68, 68, + 76, 69, 4, 210, 209, 35, 83, 191, 69, 77, 20, 116, 19, 69, 81, 85, 69, + 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 158, 154, 26, + 65, 227, 158, 6, 73, 6, 154, 242, 22, 82, 175, 226, 13, 89, 4, 134, 174, + 26, 32, 151, 154, 9, 79, 5, 197, 220, 33, 3, 32, 87, 73, 100, 56, 6, 67, + 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 50, 45, 9, 76, 32, 76, 69, 84, + 84, 69, 82, 32, 50, 182, 2, 65, 50, 68, 42, 69, 82, 71, 38, 78, 38, 83, + 154, 143, 17, 77, 136, 235, 1, 6, 72, 73, 82, 68, 69, 65, 0, 2, 75, 79, + 172, 199, 13, 6, 84, 65, 84, 65, 83, 79, 196, 209, 2, 5, 66, 65, 83, 73, + 71, 244, 50, 3, 87, 65, 83, 164, 108, 3, 70, 73, 84, 0, 3, 76, 65, 75, + 170, 11, 79, 2, 80, 2, 85, 219, 19, 73, 4, 26, 82, 255, 210, 36, 89, 2, + 215, 157, 23, 75, 4, 160, 223, 31, 3, 65, 82, 66, 3, 74, 6, 40, 4, 82, + 73, 71, 79, 151, 210, 36, 72, 5, 245, 184, 20, 4, 32, 84, 65, 77, 4, 158, + 243, 29, 79, 155, 220, 6, 78, 4, 242, 204, 31, 73, 187, 246, 3, 71, 4, + 194, 175, 26, 72, 155, 182, 3, 69, 4, 242, 250, 34, 87, 219, 64, 32, 194, + 1, 184, 2, 7, 76, 69, 84, 84, 69, 82, 32, 244, 1, 7, 78, 85, 77, 66, 69, + 82, 32, 72, 5, 83, 73, 71, 78, 32, 48, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 190, 138, 26, 68, 198, 247, 3, 87, 208, 195, 2, 10, 71, + 65, 80, 32, 70, 73, 76, 76, 69, 82, 161, 187, 3, 12, 72, 85, 78, 68, 82, + 69, 68, 83, 32, 85, 78, 73, 92, 210, 1, 86, 222, 238, 32, 65, 38, 68, + 114, 84, 230, 5, 85, 186, 202, 1, 73, 138, 196, 1, 78, 46, 83, 82, 66, 2, + 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 89, + 186, 2, 69, 3, 79, 8, 234, 1, 79, 231, 202, 36, 65, 36, 142, 126, 69, 38, + 70, 66, 78, 26, 83, 182, 193, 21, 84, 195, 240, 12, 79, 10, 134, 174, 32, + 67, 158, 67, 65, 187, 151, 3, 86, 24, 80, 2, 86, 79, 198, 243, 32, 65, + 38, 85, 186, 202, 1, 73, 198, 140, 2, 69, 3, 79, 6, 33, 6, 67, 65, 76, + 73, 67, 32, 6, 162, 244, 32, 82, 159, 214, 3, 76, 26, 148, 1, 4, 67, 89, + 67, 76, 36, 2, 71, 32, 28, 2, 76, 76, 46, 82, 66, 84, 224, 186, 19, 4, + 79, 72, 65, 90, 160, 136, 12, 2, 75, 73, 239, 180, 4, 83, 4, 142, 219, + 32, 73, 247, 237, 3, 69, 4, 174, 175, 34, 82, 67, 83, 4, 164, 228, 9, 2, + 69, 68, 131, 131, 24, 73, 4, 236, 219, 32, 7, 84, 72, 68, 65, 89, 32, 67, + 171, 236, 3, 68, 4, 244, 224, 13, 2, 67, 79, 169, 229, 22, 4, 73, 78, 71, + 32, 196, 7, 42, 65, 174, 33, 79, 165, 11, 2, 85, 69, 246, 2, 32, 2, 67, + 75, 207, 128, 18, 78, 244, 2, 22, 32, 223, 31, 45, 228, 2, 210, 1, 67, + 254, 4, 68, 174, 2, 70, 102, 72, 82, 76, 186, 4, 77, 250, 2, 78, 38, 80, + 46, 82, 150, 4, 83, 154, 4, 84, 82, 85, 248, 2, 3, 86, 69, 82, 210, 141, + 12, 79, 164, 228, 20, 3, 66, 79, 87, 187, 249, 1, 81, 102, 196, 1, 5, 73, + 82, 67, 76, 69, 200, 1, 6, 85, 82, 86, 69, 68, 32, 244, 145, 26, 12, 82, + 79, 83, 83, 32, 79, 78, 32, 83, 72, 73, 69, 248, 161, 3, 5, 69, 78, 84, + 82, 69, 254, 172, 5, 72, 159, 14, 76, 11, 11, 32, 8, 72, 5, 87, 73, 84, + 72, 32, 189, 250, 16, 7, 70, 79, 82, 32, 82, 69, 67, 6, 140, 69, 8, 87, + 72, 73, 84, 69, 32, 68, 79, 218, 134, 19, 68, 181, 146, 2, 8, 84, 87, 79, + 32, 87, 72, 73, 84, 16, 84, 4, 68, 79, 87, 78, 0, 2, 85, 80, 56, 3, 76, + 69, 70, 1, 4, 82, 73, 71, 72, 4, 153, 155, 32, 9, 87, 65, 82, 68, 83, 32, + 65, 78, 68, 4, 53, 11, 84, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 4, + 194, 163, 21, 85, 203, 170, 12, 68, 30, 64, 6, 73, 65, 77, 79, 78, 68, + 152, 1, 3, 79, 87, 78, 43, 82, 13, 11, 32, 10, 154, 19, 67, 172, 136, 13, + 10, 77, 73, 78, 85, 83, 32, 87, 72, 73, 84, 244, 172, 6, 6, 87, 73, 84, + 72, 32, 68, 254, 178, 15, 79, 135, 13, 83, 12, 214, 19, 32, 62, 45, 151, + 199, 31, 87, 6, 196, 219, 34, 2, 79, 80, 179, 21, 65, 8, 18, 76, 39, 79, + 4, 142, 132, 28, 79, 179, 184, 8, 65, 4, 184, 219, 2, 2, 85, 82, 223, + 189, 31, 76, 12, 38, 69, 158, 244, 34, 65, 251, 2, 79, 6, 228, 245, 34, + 2, 65, 82, 247, 11, 88, 40, 64, 5, 65, 82, 71, 69, 32, 140, 2, 3, 69, 70, + 84, 203, 1, 79, 12, 48, 6, 67, 73, 82, 67, 76, 69, 159, 152, 35, 83, 11, + 37, 7, 32, 77, 73, 78, 85, 83, 32, 8, 54, 76, 36, 4, 82, 73, 71, 72, 13, + 3, 85, 80, 80, 4, 32, 2, 69, 70, 13, 2, 79, 87, 2, 31, 84, 2, 17, 2, 69, + 82, 2, 177, 183, 33, 8, 32, 81, 85, 65, 82, 84, 69, 82, 22, 96, 10, 45, + 80, 79, 73, 78, 84, 73, 78, 71, 32, 64, 6, 87, 65, 82, 68, 83, 32, 175, + 245, 34, 32, 12, 146, 7, 68, 190, 8, 73, 130, 233, 34, 80, 206, 24, 83, + 51, 84, 4, 250, 200, 33, 69, 231, 140, 1, 66, 6, 158, 15, 87, 243, 240, + 34, 90, 30, 76, 6, 69, 68, 73, 85, 77, 32, 153, 243, 21, 7, 79, 79, 78, + 32, 76, 73, 76, 28, 66, 68, 42, 76, 36, 4, 82, 73, 71, 72, 12, 2, 85, 80, + 111, 83, 6, 84, 3, 79, 87, 78, 231, 246, 34, 73, 6, 32, 2, 69, 70, 135, + 254, 34, 79, 4, 11, 84, 4, 81, 18, 45, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 84, 82, 73, 65, 78, 71, 76, 69, 5, 145, 9, 2, 32, 67, 8, 198, 13, 77, + 203, 132, 35, 81, 4, 194, 165, 17, 69, 139, 209, 17, 73, 8, 234, 156, 25, + 85, 230, 217, 9, 65, 87, 69, 34, 52, 4, 73, 71, 72, 84, 238, 161, 34, 79, + 239, 85, 69, 30, 94, 32, 84, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, + 245, 1, 6, 87, 65, 82, 68, 83, 32, 6, 60, 9, 84, 82, 73, 65, 78, 71, 76, + 69, 32, 239, 136, 35, 80, 2, 243, 210, 6, 67, 16, 90, 68, 88, 8, 84, 82, + 73, 65, 78, 71, 76, 69, 230, 7, 73, 134, 238, 34, 80, 203, 19, 83, 4, 65, + 14, 79, 85, 66, 76, 69, 32, 84, 82, 73, 65, 78, 71, 76, 69, 5, 227, 202, + 12, 32, 5, 241, 249, 20, 11, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, + 8, 250, 178, 19, 65, 150, 142, 14, 69, 231, 140, 1, 66, 40, 158, 2, 77, + 148, 1, 5, 81, 85, 65, 82, 69, 128, 150, 26, 3, 85, 78, 32, 160, 224, 1, + 5, 75, 85, 76, 76, 32, 228, 183, 3, 3, 78, 79, 87, 168, 228, 1, 5, 65, + 70, 69, 84, 89, 184, 131, 1, 13, 76, 73, 71, 72, 84, 76, 89, 32, 83, 77, + 65, 76, 76, 198, 93, 67, 50, 72, 222, 1, 80, 191, 19, 84, 12, 40, 4, 65, + 76, 76, 32, 143, 246, 34, 73, 10, 154, 238, 34, 68, 150, 7, 76, 70, 83, + 213, 15, 13, 85, 80, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 67, 9, 11, + 32, 6, 54, 67, 216, 214, 33, 3, 70, 79, 82, 223, 159, 1, 66, 2, 193, 151, + 31, 3, 69, 78, 84, 14, 192, 4, 3, 73, 78, 89, 162, 195, 23, 82, 174, 182, + 11, 79, 54, 69, 135, 2, 87, 18, 38, 80, 233, 249, 32, 3, 78, 73, 86, 16, + 46, 32, 62, 45, 170, 1, 80, 239, 197, 31, 87, 2, 173, 129, 35, 10, 80, + 79, 73, 78, 84, 73, 78, 71, 32, 66, 8, 45, 9, 80, 79, 73, 78, 84, 73, 78, + 71, 32, 8, 66, 73, 224, 253, 34, 5, 68, 79, 85, 66, 76, 238, 3, 83, 51, + 84, 2, 217, 254, 33, 8, 83, 79, 83, 67, 69, 76, 69, 83, 4, 21, 3, 69, 82, + 32, 4, 130, 204, 24, 76, 163, 178, 9, 82, 10, 56, 6, 84, 73, 67, 65, 76, + 32, 41, 4, 89, 32, 83, 77, 4, 220, 235, 34, 2, 82, 69, 235, 22, 69, 6, + 21, 3, 65, 76, 76, 6, 11, 32, 6, 254, 231, 34, 68, 150, 7, 76, 135, 21, + 83, 16, 84, 15, 76, 69, 84, 84, 69, 82, 32, 67, 65, 80, 73, 84, 65, 76, + 32, 171, 159, 11, 70, 10, 242, 165, 36, 67, 2, 72, 2, 73, 2, 82, 3, 90, + 200, 4, 52, 3, 67, 75, 32, 186, 151, 2, 83, 219, 214, 10, 87, 196, 4, 80, + 7, 79, 67, 84, 65, 78, 84, 45, 149, 7, 8, 83, 69, 88, 84, 65, 78, 84, 45, + 204, 3, 58, 49, 130, 3, 50, 114, 53, 50, 54, 18, 51, 215, 1, 52, 234, 1, + 74, 50, 246, 1, 51, 174, 91, 52, 46, 53, 38, 54, 30, 55, 147, 197, 35, + 56, 116, 62, 51, 226, 92, 52, 46, 53, 38, 54, 30, 55, 147, 197, 35, 56, + 55, 54, 52, 214, 92, 53, 38, 54, 30, 55, 147, 197, 35, 56, 22, 50, 53, + 166, 2, 54, 190, 90, 55, 147, 197, 35, 56, 11, 42, 54, 150, 246, 28, 55, + 179, 171, 7, 56, 4, 194, 161, 36, 55, 3, 56, 56, 170, 1, 53, 50, 54, 210, + 89, 52, 110, 55, 147, 197, 35, 56, 118, 62, 52, 254, 89, 51, 98, 53, 38, + 54, 30, 55, 147, 197, 35, 56, 24, 46, 53, 50, 54, 190, 90, 55, 147, 197, + 35, 56, 13, 206, 1, 54, 254, 242, 28, 55, 179, 171, 7, 56, 7, 187, 90, + 55, 61, 54, 52, 118, 53, 230, 88, 54, 30, 55, 147, 197, 35, 56, 31, 46, + 53, 170, 89, 54, 30, 55, 147, 197, 35, 56, 15, 38, 54, 158, 89, 55, 147, + 197, 35, 56, 7, 170, 158, 36, 55, 3, 56, 14, 226, 88, 54, 30, 55, 147, + 197, 35, 56, 31, 46, 54, 234, 87, 53, 66, 55, 147, 197, 35, 56, 6, 166, + 88, 55, 147, 197, 35, 56, 120, 70, 49, 238, 1, 50, 238, 209, 25, 51, 38, + 52, 30, 53, 187, 200, 10, 54, 61, 58, 50, 126, 51, 198, 210, 25, 52, 30, + 53, 187, 200, 10, 54, 31, 50, 51, 142, 211, 25, 52, 30, 53, 187, 200, 10, + 54, 15, 42, 52, 254, 210, 25, 53, 187, 200, 10, 54, 7, 178, 155, 36, 53, + 3, 54, 15, 194, 210, 25, 52, 186, 157, 3, 53, 159, 171, 7, 54, 31, 50, + 52, 186, 209, 25, 51, 66, 53, 187, 200, 10, 54, 7, 247, 209, 25, 53, 6, + 226, 173, 22, 32, 213, 206, 8, 4, 66, 69, 82, 82, 232, 4, 200, 1, 3, 76, + 68, 32, 78, 79, 104, 7, 80, 79, 77, 79, 70, 79, 32, 188, 6, 2, 84, 84, + 216, 6, 5, 85, 81, 85, 69, 84, 54, 87, 198, 1, 88, 182, 221, 5, 77, 146, + 175, 3, 89, 186, 194, 26, 65, 139, 34, 78, 14, 190, 197, 8, 83, 190, 212, + 7, 69, 202, 228, 17, 70, 234, 2, 87, 203, 11, 71, 10, 26, 75, 135, 165, + 23, 77, 9, 40, 4, 77, 65, 82, 75, 135, 151, 36, 83, 5, 189, 191, 23, 3, + 32, 84, 65, 150, 1, 96, 13, 70, 73, 78, 65, 76, 32, 76, 69, 84, 84, 69, + 82, 32, 53, 7, 76, 69, 84, 84, 69, 82, 32, 10, 250, 149, 36, 71, 2, 72, + 2, 75, 2, 80, 3, 84, 140, 1, 234, 1, 65, 54, 85, 22, 69, 82, 71, 46, 73, + 70, 78, 38, 79, 98, 90, 190, 160, 7, 75, 178, 223, 24, 67, 2, 76, 2, 83, + 230, 57, 66, 186, 202, 1, 74, 198, 140, 2, 68, 2, 70, 2, 72, 2, 77, 2, + 80, 2, 81, 2, 82, 2, 84, 2, 86, 3, 88, 21, 50, 73, 2, 85, 74, 78, 222, + 146, 36, 72, 3, 77, 5, 195, 195, 35, 78, 17, 50, 78, 222, 146, 36, 69, 2, + 72, 2, 73, 3, 82, 7, 218, 146, 36, 71, 3, 78, 11, 190, 146, 36, 72, 2, + 78, 2, 85, 3, 87, 15, 164, 148, 34, 2, 78, 78, 238, 253, 1, 72, 2, 77, 2, + 82, 3, 85, 9, 206, 147, 34, 71, 131, 254, 1, 78, 17, 66, 78, 230, 143, + 35, 32, 134, 129, 1, 69, 2, 77, 2, 79, 3, 85, 4, 230, 144, 36, 71, 3, 78, + 9, 202, 144, 36, 72, 2, 73, 3, 89, 66, 104, 3, 79, 77, 32, 221, 199, 21, + 17, 76, 69, 32, 87, 73, 84, 72, 32, 80, 79, 80, 80, 73, 78, 71, 32, 67, + 64, 152, 2, 5, 72, 65, 76, 70, 32, 232, 1, 5, 76, 69, 70, 84, 32, 88, 6, + 82, 73, 71, 72, 84, 32, 88, 14, 83, 81, 85, 65, 82, 69, 32, 66, 82, 65, + 67, 75, 69, 84, 210, 228, 15, 65, 226, 154, 16, 67, 210, 3, 80, 140, 3, + 13, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, 85, 80, 80, 135, 5, 84, 32, + 148, 1, 16, 70, 79, 82, 87, 65, 82, 68, 45, 70, 65, 67, 73, 78, 71, 32, + 82, 134, 132, 32, 76, 22, 82, 252, 2, 2, 83, 84, 174, 5, 66, 211, 245, 1, + 73, 10, 196, 179, 29, 11, 85, 78, 78, 69, 82, 32, 70, 82, 65, 77, 69, + 155, 210, 2, 79, 8, 196, 137, 32, 13, 74, 85, 83, 84, 73, 70, 73, 69, 68, + 32, 85, 80, 80, 126, 67, 51, 72, 8, 234, 137, 32, 67, 50, 72, 33, 13, 74, + 85, 83, 84, 73, 70, 73, 69, 68, 32, 85, 80, 80, 5, 129, 236, 23, 9, 32, + 79, 86, 69, 82, 32, 84, 79, 80, 5, 209, 248, 34, 8, 32, 79, 70, 32, 70, + 76, 79, 87, 14, 58, 76, 116, 3, 84, 73, 69, 149, 170, 33, 3, 32, 65, 78, + 6, 26, 32, 207, 183, 35, 73, 4, 224, 195, 9, 7, 79, 70, 32, 72, 89, 71, + 73, 173, 154, 23, 6, 87, 73, 84, 72, 32, 83, 7, 211, 231, 32, 32, 218, 2, + 88, 10, 32, 68, 82, 65, 87, 73, 78, 71, 83, 32, 153, 138, 35, 6, 73, 78, + 71, 32, 71, 76, 216, 2, 176, 1, 2, 68, 79, 228, 6, 6, 72, 69, 65, 86, 89, + 32, 254, 2, 76, 204, 25, 6, 82, 73, 71, 72, 84, 32, 144, 4, 3, 85, 80, + 32, 245, 3, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 52, 5, 85, 66, 76, + 69, 32, 165, 3, 3, 87, 78, 32, 30, 58, 68, 216, 2, 2, 85, 80, 234, 5, 86, + 203, 188, 29, 72, 14, 64, 8, 73, 65, 71, 79, 78, 65, 76, 32, 149, 2, 3, + 79, 87, 78, 8, 104, 6, 85, 80, 80, 69, 82, 32, 205, 14, 15, 76, 79, 87, + 69, 82, 32, 76, 69, 70, 84, 32, 84, 79, 32, 77, 6, 76, 8, 76, 69, 70, 84, + 32, 84, 79, 32, 237, 24, 6, 82, 73, 71, 72, 84, 32, 4, 204, 23, 14, 77, + 73, 68, 68, 76, 69, 32, 67, 69, 78, 84, 82, 69, 32, 191, 194, 23, 76, 6, + 199, 26, 32, 36, 128, 1, 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 132, + 1, 10, 76, 73, 71, 72, 84, 32, 65, 78, 68, 32, 210, 38, 68, 131, 4, 83, + 12, 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 228, 35, 2, 85, 80, 151, 5, + 72, 4, 17, 2, 84, 32, 4, 230, 32, 85, 199, 180, 35, 76, 12, 76, 3, 76, + 69, 70, 0, 4, 82, 73, 71, 72, 156, 36, 2, 85, 80, 235, 4, 72, 4, 17, 2, + 84, 32, 4, 198, 32, 85, 135, 9, 72, 46, 136, 1, 4, 76, 69, 70, 84, 68, 2, + 85, 80, 130, 1, 86, 172, 20, 2, 68, 79, 170, 2, 81, 100, 2, 84, 82, 146, + 165, 29, 72, 247, 148, 6, 82, 5, 45, 9, 32, 65, 78, 68, 32, 76, 73, 71, + 72, 2, 255, 169, 34, 84, 11, 29, 5, 32, 65, 78, 68, 32, 8, 42, 76, 254, + 188, 29, 72, 247, 148, 6, 82, 4, 220, 195, 24, 4, 73, 71, 72, 84, 131, + 142, 11, 69, 8, 209, 20, 7, 69, 82, 84, 73, 67, 65, 76, 156, 1, 56, 4, + 69, 70, 84, 32, 169, 2, 5, 73, 71, 72, 84, 32, 16, 160, 27, 19, 68, 79, + 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, 72, 24, + 14, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, 72, 100, 14, 76, + 73, 71, 72, 84, 32, 65, 78, 68, 32, 82, 73, 71, 72, 97, 17, 85, 80, 32, + 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 82, 73, 71, 72, 140, 1, 248, 1, + 4, 65, 82, 67, 32, 30, 68, 248, 15, 10, 72, 79, 82, 73, 90, 79, 78, 84, + 65, 76, 112, 4, 76, 69, 70, 84, 62, 81, 34, 84, 172, 1, 2, 85, 80, 120, + 8, 86, 69, 82, 84, 73, 67, 65, 76, 156, 211, 13, 7, 66, 79, 84, 84, 79, + 77, 32, 139, 229, 21, 82, 8, 166, 222, 17, 68, 67, 85, 80, 52, 8, 73, 65, + 71, 79, 78, 65, 76, 32, 199, 14, 79, 68, 168, 1, 14, 76, 79, 87, 69, 82, + 32, 76, 69, 70, 84, 32, 84, 79, 32, 96, 7, 77, 73, 68, 68, 76, 69, 32, + 136, 3, 6, 85, 80, 80, 69, 82, 32, 230, 173, 34, 67, 183, 4, 68, 4, 38, + 77, 33, 5, 85, 80, 80, 69, 82, 2, 29, 5, 73, 68, 68, 76, 69, 2, 129, 12, + 2, 32, 67, 16, 88, 8, 76, 69, 70, 84, 32, 84, 79, 32, 213, 1, 9, 82, 73, + 71, 72, 84, 32, 84, 79, 32, 10, 156, 1, 6, 76, 79, 87, 69, 82, 32, 33, + 28, 85, 80, 80, 69, 82, 32, 67, 69, 78, 84, 82, 69, 32, 84, 79, 32, 77, + 73, 68, 68, 76, 69, 32, 82, 73, 71, 72, 84, 6, 246, 3, 67, 215, 195, 35, + 82, 5, 171, 211, 20, 32, 6, 232, 4, 15, 85, 80, 80, 69, 82, 32, 67, 69, + 78, 84, 82, 69, 32, 84, 79, 235, 3, 76, 44, 144, 1, 10, 67, 69, 78, 84, + 82, 69, 32, 84, 79, 32, 248, 3, 8, 76, 69, 70, 84, 32, 84, 79, 32, 193, + 2, 9, 82, 73, 71, 72, 84, 32, 84, 79, 32, 20, 52, 7, 77, 73, 68, 68, 76, + 69, 32, 203, 197, 23, 76, 16, 56, 4, 76, 69, 70, 84, 165, 1, 5, 82, 73, + 71, 72, 84, 9, 11, 32, 6, 84, 10, 84, 79, 32, 76, 79, 87, 69, 82, 32, 67, + 141, 207, 20, 5, 65, 78, 68, 32, 77, 4, 29, 5, 69, 78, 84, 82, 69, 5, + 137, 232, 32, 3, 32, 84, 79, 9, 11, 32, 6, 88, 3, 65, 78, 68, 65, 15, 84, + 79, 32, 76, 79, 87, 69, 82, 32, 67, 69, 78, 84, 82, 69, 2, 149, 206, 20, + 11, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 5, 133, 224, 10, 9, 32, + 84, 79, 32, 77, 73, 68, 68, 76, 14, 68, 6, 76, 79, 87, 69, 82, 32, 97, 7, + 77, 73, 68, 68, 76, 69, 32, 6, 48, 6, 67, 69, 78, 84, 82, 69, 187, 192, + 35, 82, 5, 11, 32, 2, 233, 4, 4, 84, 79, 32, 85, 8, 76, 10, 67, 69, 78, + 84, 82, 69, 32, 84, 79, 32, 33, 5, 82, 73, 71, 72, 84, 4, 250, 3, 85, + 231, 214, 13, 76, 5, 11, 32, 2, 157, 218, 13, 2, 84, 79, 10, 46, 76, 69, + 7, 77, 73, 68, 68, 76, 69, 32, 4, 29, 5, 79, 87, 69, 82, 32, 4, 202, 227, + 32, 67, 191, 218, 2, 76, 6, 34, 67, 37, 4, 76, 69, 70, 84, 2, 45, 6, 69, + 78, 84, 82, 69, 32, 5, 11, 32, 2, 165, 191, 23, 2, 84, 79, 12, 36, 2, 87, + 78, 253, 2, 2, 85, 66, 9, 11, 32, 6, 25, 4, 65, 78, 68, 32, 6, 202, 167, + 29, 72, 218, 148, 6, 76, 31, 82, 9, 11, 32, 6, 40, 4, 65, 78, 68, 32, + 187, 181, 35, 87, 4, 26, 85, 211, 189, 23, 76, 2, 225, 189, 23, 2, 80, + 80, 5, 209, 146, 34, 10, 32, 65, 78, 68, 32, 72, 69, 65, 86, 89, 4, 109, + 5, 85, 65, 68, 82, 85, 6, 66, 82, 189, 213, 13, 10, 79, 80, 32, 65, 78, + 68, 32, 85, 80, 80, 4, 11, 73, 4, 11, 80, 4, 41, 8, 76, 69, 32, 68, 65, + 83, 72, 32, 4, 166, 193, 16, 86, 167, 227, 12, 72, 11, 29, 5, 32, 65, 78, + 68, 32, 8, 34, 72, 190, 184, 35, 76, 31, 82, 4, 196, 170, 24, 4, 69, 65, + 86, 89, 171, 249, 4, 79, 17, 29, 5, 32, 65, 78, 68, 32, 14, 230, 168, 28, + 66, 42, 84, 130, 122, 72, 218, 148, 6, 76, 31, 82, 16, 148, 2, 18, 68, + 79, 87, 78, 32, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 24, + 13, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 100, 13, 76, 73, + 71, 72, 84, 32, 65, 78, 68, 32, 76, 69, 70, 97, 16, 85, 80, 32, 72, 69, + 65, 86, 89, 32, 65, 78, 68, 32, 76, 69, 70, 2, 101, 3, 84, 32, 85, 6, 17, + 2, 84, 32, 6, 58, 85, 178, 3, 68, 245, 4, 6, 86, 69, 82, 84, 73, 67, 2, + 179, 194, 33, 80, 6, 17, 2, 84, 32, 6, 58, 85, 134, 4, 68, 205, 4, 6, 86, + 69, 82, 84, 73, 67, 2, 239, 8, 80, 2, 185, 2, 3, 84, 32, 68, 36, 128, 1, + 10, 72, 69, 65, 86, 89, 32, 65, 78, 68, 32, 188, 1, 10, 76, 73, 71, 72, + 84, 32, 65, 78, 68, 32, 186, 2, 68, 131, 4, 83, 12, 80, 4, 68, 79, 87, + 78, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 255, 4, 72, 2, 145, 5, 2, + 32, 72, 4, 17, 2, 84, 32, 4, 26, 68, 151, 177, 35, 76, 2, 129, 191, 33, + 3, 79, 87, 78, 12, 80, 4, 68, 79, 87, 78, 24, 3, 76, 69, 70, 0, 4, 82, + 73, 71, 72, 211, 4, 72, 2, 229, 4, 2, 32, 72, 4, 17, 2, 84, 32, 4, 22, + 68, 131, 5, 72, 2, 233, 4, 3, 79, 87, 78, 24, 130, 1, 68, 188, 1, 10, 72, + 69, 65, 86, 89, 32, 65, 78, 68, 32, 144, 1, 10, 76, 73, 71, 72, 84, 32, + 65, 78, 68, 32, 183, 1, 83, 6, 49, 10, 79, 85, 66, 76, 69, 32, 65, 78, + 68, 32, 6, 92, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 13, 10, 72, 79, 82, + 73, 90, 79, 78, 84, 65, 76, 2, 11, 84, 2, 169, 129, 27, 2, 32, 83, 6, 54, + 72, 68, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 37, 7, 79, 82, 73, 90, + 79, 78, 84, 2, 141, 186, 33, 2, 65, 76, 2, 243, 185, 33, 84, 6, 54, 72, + 60, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 37, 7, 79, 82, 73, 90, 79, + 78, 84, 2, 29, 2, 65, 76, 2, 11, 84, 2, 17, 2, 32, 72, 2, 225, 195, 35, + 3, 69, 65, 86, 6, 49, 10, 73, 78, 71, 76, 69, 32, 65, 78, 68, 32, 6, 96, + 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 177, 197, 26, 9, 72, 79, 82, 73, 90, + 79, 78, 84, 65, 2, 187, 197, 26, 84, 134, 6, 46, 65, 178, 14, 69, 150, 1, + 73, 179, 1, 79, 232, 5, 36, 4, 72, 77, 73, 32, 247, 9, 73, 230, 1, 192, + 1, 7, 76, 69, 84, 84, 69, 82, 32, 196, 2, 7, 78, 85, 77, 66, 69, 82, 32, + 144, 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 84, 5, 83, + 73, 71, 78, 32, 122, 86, 159, 138, 25, 68, 108, 210, 1, 79, 242, 241, 31, + 65, 38, 68, 114, 84, 46, 86, 186, 5, 85, 186, 202, 1, 73, 42, 76, 226, + 195, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, + 72, 2, 77, 2, 82, 2, 89, 187, 2, 69, 15, 45, 9, 76, 68, 32, 84, 65, 77, + 73, 76, 32, 12, 150, 152, 29, 76, 142, 155, 2, 83, 210, 97, 78, 131, 246, + 2, 82, 42, 82, 69, 38, 70, 66, 78, 26, 83, 182, 193, 21, 84, 130, 234, 5, + 79, 227, 227, 7, 74, 4, 129, 189, 31, 4, 73, 71, 72, 84, 8, 26, 79, 143, + 168, 21, 73, 4, 254, 221, 33, 82, 135, 183, 1, 85, 4, 65, 3, 73, 78, 69, + 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 5, 175, 187, 35, 84, 10, 46, 76, + 130, 222, 12, 68, 177, 16, 2, 67, 82, 4, 242, 210, 19, 79, 147, 222, 14, + 73, 12, 174, 174, 31, 67, 228, 67, 9, 79, 76, 68, 32, 84, 65, 77, 73, 76, + 246, 157, 1, 74, 158, 2, 86, 122, 85, 255, 243, 1, 65, 34, 64, 10, 79, + 87, 69, 76, 32, 83, 73, 71, 78, 32, 187, 142, 33, 73, 32, 138, 1, 79, + 228, 199, 29, 11, 66, 72, 65, 84, 84, 73, 80, 82, 79, 76, 85, 198, 170, + 2, 65, 38, 85, 22, 86, 166, 202, 1, 73, 199, 140, 2, 69, 7, 181, 173, 31, + 10, 76, 68, 32, 84, 65, 77, 73, 76, 32, 83, 130, 4, 72, 12, 76, 76, 69, + 32, 80, 65, 84, 84, 69, 82, 78, 32, 191, 200, 35, 78, 128, 4, 44, 5, 68, + 79, 84, 83, 45, 207, 250, 6, 66, 254, 3, 74, 49, 74, 50, 66, 51, 54, 52, + 46, 53, 38, 54, 30, 55, 147, 197, 35, 56, 129, 2, 66, 50, 66, 51, 54, 52, + 46, 53, 38, 54, 30, 55, 147, 197, 35, 56, 129, 1, 58, 51, 54, 52, 46, 53, + 38, 54, 30, 55, 147, 197, 35, 56, 65, 50, 52, 46, 53, 38, 54, 30, 55, + 147, 197, 35, 56, 33, 42, 53, 38, 54, 30, 55, 147, 197, 35, 56, 17, 34, + 54, 30, 55, 147, 197, 35, 56, 9, 26, 55, 147, 197, 35, 56, 5, 143, 197, + 35, 56, 8, 26, 65, 143, 174, 35, 86, 6, 196, 205, 20, 11, 75, 32, 80, 69, + 82, 77, 73, 84, 84, 69, 68, 228, 176, 8, 6, 83, 84, 45, 70, 69, 69, 183, + 198, 6, 68, 10, 42, 68, 96, 2, 69, 70, 199, 191, 35, 67, 4, 212, 197, 32, + 6, 71, 69, 32, 65, 84, 32, 149, 158, 1, 9, 69, 32, 87, 73, 84, 72, 32, + 86, 69, 4, 190, 153, 32, 67, 159, 169, 3, 83, 12, 84, 4, 75, 69, 78, 32, + 244, 204, 8, 2, 67, 67, 176, 237, 25, 2, 87, 78, 231, 118, 79, 6, 196, + 199, 27, 17, 67, 73, 82, 67, 76, 69, 32, 87, 73, 84, 72, 32, 78, 79, 82, + 84, 72, 222, 214, 6, 66, 151, 28, 72, 134, 1, 208, 1, 4, 66, 66, 76, 69, + 46, 71, 156, 4, 4, 72, 73, 68, 32, 36, 2, 76, 76, 122, 83, 56, 4, 84, 84, + 69, 82, 164, 134, 11, 10, 73, 76, 68, 73, 78, 71, 32, 67, 79, 78, 144, + 226, 16, 2, 82, 82, 255, 253, 6, 67, 4, 144, 171, 28, 2, 32, 84, 131, + 148, 7, 83, 63, 33, 6, 73, 78, 69, 83, 69, 32, 60, 144, 1, 7, 76, 69, 84, + 84, 69, 82, 32, 172, 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 146, 196, 16, 69, 141, 253, 13, 4, 80, 65, 76, 76, 46, 154, 1, 77, 34, + 78, 190, 185, 35, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, + 80, 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, 2, 65, 4, 218, 185, 35, 80, + 187, 2, 65, 12, 46, 71, 34, 89, 238, 184, 35, 82, 187, 2, 65, 4, 138, + 185, 35, 75, 187, 2, 65, 4, 234, 184, 35, 67, 187, 2, 65, 10, 174, 164, + 35, 65, 214, 22, 69, 2, 73, 2, 79, 3, 85, 40, 230, 164, 10, 76, 167, 253, + 18, 86, 10, 56, 2, 69, 84, 20, 4, 72, 79, 82, 78, 175, 175, 9, 83, 5, + 179, 246, 31, 32, 5, 189, 210, 27, 5, 32, 87, 73, 84, 72, 9, 26, 84, 147, + 228, 32, 32, 4, 182, 210, 27, 83, 15, 32, 5, 199, 149, 35, 70, 240, 3, + 140, 1, 23, 90, 65, 78, 84, 73, 78, 69, 32, 77, 85, 83, 73, 67, 65, 76, + 32, 83, 89, 77, 66, 79, 76, 32, 153, 200, 20, 5, 84, 69, 32, 79, 82, 238, + 3, 154, 2, 65, 128, 7, 2, 67, 72, 170, 1, 68, 158, 5, 69, 206, 2, 70, + 166, 8, 71, 202, 3, 73, 162, 2, 75, 142, 5, 76, 190, 2, 77, 250, 4, 79, + 114, 80, 174, 4, 82, 42, 83, 194, 5, 84, 196, 5, 2, 86, 65, 250, 1, 89, + 178, 196, 32, 78, 201, 146, 2, 9, 88, 73, 82, 79, 78, 32, 75, 76, 65, 62, + 60, 5, 71, 79, 71, 73, 32, 222, 1, 78, 118, 80, 199, 2, 82, 16, 70, 65, + 0, 2, 71, 79, 64, 2, 77, 69, 37, 5, 80, 79, 76, 73, 32, 4, 17, 2, 82, 71, + 4, 128, 242, 15, 2, 79, 84, 147, 194, 19, 73, 4, 158, 164, 9, 84, 243, + 251, 25, 83, 4, 26, 65, 1, 2, 71, 79, 2, 239, 222, 17, 82, 6, 72, 6, 84, + 73, 75, 69, 78, 79, 161, 145, 35, 6, 65, 84, 82, 73, 67, 72, 4, 168, 32, + 2, 75, 89, 231, 143, 35, 77, 22, 52, 5, 69, 83, 79, 32, 69, 38, 79, 239, + 157, 35, 76, 4, 204, 1, 2, 88, 79, 147, 55, 75, 16, 80, 6, 83, 84, 82, + 79, 70, 79, 188, 40, 3, 68, 69, 82, 237, 129, 1, 2, 84, 72, 10, 24, 2, + 73, 32, 79, 83, 4, 56, 9, 83, 89, 78, 68, 69, 83, 77, 79, 83, 195, 24, + 84, 2, 143, 44, 32, 7, 11, 32, 4, 214, 60, 68, 183, 166, 22, 78, 18, 48, + 2, 71, 79, 33, 6, 75, 84, 73, 75, 79, 32, 4, 170, 21, 83, 255, 153, 35, + 78, 14, 130, 216, 27, 86, 146, 184, 7, 90, 162, 8, 75, 254, 2, 68, 2, 78, + 162, 17, 71, 3, 80, 14, 80, 4, 82, 79, 65, 32, 168, 37, 4, 79, 82, 69, + 86, 221, 2, 4, 65, 77, 73, 76, 6, 132, 235, 19, 2, 83, 80, 232, 192, 3, + 3, 90, 89, 71, 205, 204, 10, 3, 75, 76, 73, 42, 50, 73, 244, 231, 8, 2, + 65, 83, 239, 165, 26, 89, 38, 122, 65, 200, 1, 5, 69, 83, 73, 83, 32, 70, + 71, 208, 1, 3, 80, 76, 73, 249, 208, 27, 8, 70, 84, 79, 71, 71, 79, 83, + 32, 10, 48, 6, 83, 84, 79, 76, 73, 32, 243, 240, 33, 82, 8, 80, 6, 65, + 80, 76, 73, 32, 77, 174, 55, 68, 185, 241, 22, 5, 84, 72, 69, 83, 69, 4, + 40, 2, 69, 71, 217, 190, 28, 2, 73, 75, 2, 195, 166, 30, 65, 12, 38, 84, + 246, 50, 65, 42, 68, 63, 77, 6, 194, 10, 69, 239, 41, 82, 10, 72, 5, 79, + 82, 71, 79, 78, 173, 168, 35, 7, 82, 65, 77, 77, 65, 32, 71, 9, 69, 15, + 32, 80, 65, 82, 69, 83, 84, 73, 71, 77, 69, 78, 79, 78, 32, 6, 222, 38, + 68, 137, 10, 8, 65, 82, 73, 83, 84, 69, 82, 65, 5, 175, 47, 32, 16, 166, + 1, 78, 116, 6, 84, 69, 82, 79, 78, 32, 188, 44, 4, 88, 79, 32, 69, 136, + 147, 33, 4, 80, 69, 71, 69, 236, 48, 6, 75, 83, 84, 82, 69, 80, 237, 13, + 3, 76, 65, 70, 4, 224, 22, 3, 68, 79, 70, 221, 184, 27, 18, 65, 82, 88, + 73, 83, 32, 75, 65, 73, 32, 70, 84, 72, 79, 82, 65, 32, 86, 4, 208, 11, + 5, 65, 82, 71, 79, 83, 151, 20, 80, 44, 180, 1, 9, 65, 78, 69, 82, 79, + 83, 73, 83, 32, 52, 6, 84, 72, 79, 82, 65, 32, 165, 6, 22, 72, 84, 79, + 82, 65, 32, 83, 75, 76, 73, 82, 79, 78, 32, 67, 72, 82, 79, 77, 65, 32, + 86, 6, 246, 4, 68, 14, 77, 25, 5, 84, 69, 84, 82, 65, 36, 220, 2, 8, 65, + 82, 67, 72, 65, 73, 79, 78, 80, 10, 68, 73, 65, 84, 79, 78, 73, 75, 73, + 32, 96, 11, 73, 32, 89, 70, 69, 83, 73, 83, 32, 84, 69, 32, 15, 77, 65, + 76, 65, 75, 79, 78, 32, 67, 72, 82, 79, 77, 65, 32, 74, 78, 40, 8, 83, + 75, 76, 73, 82, 79, 78, 32, 157, 229, 18, 18, 69, 78, 65, 82, 77, 79, 78, + 73, 79, 83, 32, 65, 78, 84, 73, 70, 79, 78, 5, 57, 12, 32, 68, 69, 89, + 84, 69, 82, 79, 85, 32, 73, 67, 2, 147, 201, 27, 72, 14, 62, 78, 210, + 128, 35, 90, 162, 8, 75, 254, 2, 68, 163, 17, 80, 6, 242, 39, 73, 187, + 185, 33, 65, 2, 237, 42, 4, 84, 65, 82, 84, 4, 18, 68, 15, 77, 2, 35, 73, + 2, 21, 3, 79, 78, 79, 2, 151, 19, 70, 4, 166, 19, 65, 193, 128, 33, 2, + 69, 78, 6, 84, 7, 67, 72, 82, 79, 77, 65, 32, 193, 169, 17, 8, 68, 73, + 65, 84, 79, 78, 79, 78, 4, 42, 86, 169, 173, 17, 4, 83, 89, 78, 65, 2, + 251, 245, 32, 65, 22, 80, 6, 69, 78, 73, 75, 73, 32, 44, 2, 79, 82, 185, + 26, 5, 82, 79, 78, 84, 72, 4, 192, 149, 31, 2, 68, 73, 1, 2, 89, 70, 16, + 68, 2, 71, 79, 225, 1, 10, 84, 72, 77, 73, 75, 79, 78, 32, 78, 32, 12, + 28, 2, 78, 32, 155, 1, 83, 10, 96, 14, 80, 65, 82, 69, 83, 84, 73, 71, + 77, 69, 78, 79, 78, 32, 242, 33, 65, 113, 3, 78, 69, 79, 4, 214, 24, 68, + 229, 239, 32, 5, 65, 82, 73, 83, 84, 2, 217, 228, 33, 5, 89, 78, 84, 72, + 69, 4, 142, 28, 65, 1, 2, 68, 73, 16, 56, 2, 77, 73, 114, 83, 181, 152, + 34, 4, 67, 72, 65, 68, 8, 34, 70, 169, 28, 3, 68, 73, 65, 6, 40, 4, 84, + 72, 79, 82, 251, 139, 34, 79, 4, 198, 200, 34, 79, 227, 79, 65, 6, 44, 6, + 65, 75, 73, 65, 32, 84, 231, 13, 79, 2, 165, 221, 21, 12, 69, 76, 79, 85, + 83, 32, 73, 67, 72, 73, 77, 65, 54, 108, 2, 65, 84, 96, 6, 69, 78, 84, + 73, 77, 65, 140, 1, 5, 76, 65, 83, 77, 65, 18, 79, 98, 82, 175, 1, 89, 6, + 40, 3, 65, 86, 65, 201, 3, 2, 72, 73, 4, 140, 29, 5, 32, 84, 82, 79, 77, + 219, 215, 34, 83, 18, 24, 2, 84, 65, 15, 32, 11, 11, 32, 8, 36, 4, 78, + 69, 79, 32, 183, 28, 65, 6, 208, 166, 28, 2, 77, 69, 170, 222, 2, 75, + 235, 230, 3, 65, 7, 243, 28, 32, 8, 64, 6, 78, 84, 69, 86, 77, 65, 146, + 225, 8, 82, 163, 145, 26, 85, 5, 145, 181, 14, 2, 32, 65, 14, 44, 4, 65, + 84, 73, 77, 105, 3, 69, 77, 65, 12, 18, 65, 35, 79, 8, 182, 23, 32, 151, + 249, 34, 84, 4, 134, 12, 75, 181, 220, 31, 5, 89, 80, 79, 82, 82, 2, 175, + 166, 27, 83, 2, 243, 240, 34, 76, 14, 22, 69, 147, 22, 89, 12, 44, 5, 73, + 77, 77, 65, 32, 191, 154, 30, 77, 10, 72, 2, 69, 78, 0, 5, 73, 77, 73, + 83, 69, 54, 84, 69, 3, 68, 89, 79, 2, 237, 185, 27, 8, 79, 83, 32, 67, + 72, 82, 79, 78, 4, 44, 5, 69, 83, 83, 65, 82, 1, 2, 82, 73, 2, 17, 2, 79, + 78, 2, 25, 4, 32, 67, 72, 82, 2, 175, 131, 34, 79, 28, 84, 8, 65, 82, 84, + 89, 82, 73, 65, 32, 229, 138, 31, 7, 73, 75, 82, 79, 78, 32, 73, 26, 72, + 5, 65, 76, 76, 73, 32, 38, 68, 38, 80, 134, 1, 86, 30, 84, 87, 76, 4, 34, + 68, 177, 2, 3, 80, 82, 79, 2, 237, 2, 5, 69, 89, 84, 69, 82, 8, 60, 7, + 76, 65, 71, 73, 79, 83, 32, 45, 4, 82, 79, 84, 79, 4, 200, 1, 5, 84, 69, + 84, 65, 82, 111, 73, 4, 22, 86, 227, 1, 83, 2, 209, 1, 3, 65, 82, 89, 8, + 56, 8, 69, 84, 65, 82, 84, 79, 83, 32, 61, 2, 82, 73, 4, 22, 76, 135, 1, + 73, 2, 21, 3, 69, 71, 69, 2, 63, 84, 4, 18, 70, 35, 84, 2, 221, 217, 21, + 3, 79, 78, 73, 2, 11, 79, 2, 11, 83, 2, 17, 2, 32, 73, 2, 199, 132, 23, + 67, 18, 92, 4, 76, 73, 71, 79, 180, 1, 5, 89, 82, 65, 78, 73, 210, 14, + 88, 249, 169, 34, 2, 77, 65, 4, 211, 1, 78, 42, 60, 2, 65, 82, 100, 2, + 73, 65, 90, 69, 169, 1, 2, 83, 73, 12, 40, 2, 65, 75, 161, 190, 17, 2, + 73, 67, 10, 52, 3, 65, 76, 69, 45, 6, 76, 73, 84, 73, 75, 73, 4, 11, 83, + 4, 17, 2, 77, 65, 4, 23, 32, 7, 11, 32, 4, 198, 15, 65, 211, 171, 22, 78, + 12, 72, 3, 84, 65, 83, 136, 3, 6, 76, 65, 83, 84, 79, 78, 135, 229, 8, + 82, 6, 26, 84, 203, 132, 35, 77, 4, 32, 2, 79, 75, 223, 134, 35, 73, 2, + 165, 229, 34, 2, 79, 85, 14, 36, 5, 70, 73, 83, 84, 79, 67, 76, 10, 46, + 80, 214, 1, 78, 170, 8, 76, 187, 1, 83, 2, 135, 11, 65, 4, 246, 181, 34, + 79, 227, 79, 73, 4, 178, 5, 69, 221, 222, 34, 2, 65, 80, 42, 120, 5, 69, + 73, 83, 77, 65, 32, 8, 73, 77, 65, 78, 83, 73, 83, 32, 182, 1, 84, 154, + 1, 89, 229, 243, 1, 3, 65, 88, 73, 5, 11, 32, 2, 151, 183, 22, 78, 16, + 36, 2, 65, 82, 1, 3, 84, 72, 69, 8, 25, 4, 83, 69, 79, 83, 9, 11, 32, 6, + 18, 84, 39, 68, 4, 34, 82, 13, 4, 69, 84, 82, 65, 2, 11, 73, 2, 217, 171, + 27, 3, 83, 73, 77, 8, 68, 5, 65, 86, 82, 79, 83, 52, 4, 82, 65, 71, 71, + 251, 218, 16, 73, 5, 29, 5, 32, 65, 80, 79, 68, 2, 199, 232, 8, 69, 2, + 253, 243, 1, 2, 73, 83, 12, 34, 78, 149, 1, 3, 82, 77, 65, 8, 36, 5, 65, + 71, 77, 65, 32, 91, 69, 6, 154, 8, 65, 210, 171, 22, 78, 237, 245, 4, 10, + 77, 69, 84, 65, 32, 83, 84, 65, 86, 82, 2, 243, 222, 34, 86, 5, 11, 84, + 2, 243, 137, 17, 73, 40, 42, 69, 70, 72, 218, 1, 82, 227, 2, 73, 6, 136, + 12, 3, 84, 82, 65, 186, 174, 8, 76, 237, 178, 24, 2, 83, 83, 12, 26, 69, + 251, 217, 34, 73, 10, 64, 2, 77, 65, 217, 231, 33, 8, 83, 32, 75, 65, 73, + 32, 65, 80, 9, 56, 2, 32, 65, 33, 8, 84, 73, 83, 77, 79, 83, 32, 69, 2, + 145, 253, 33, 3, 80, 76, 79, 4, 174, 222, 34, 83, 3, 88, 20, 38, 73, 73, + 5, 79, 77, 73, 75, 79, 6, 48, 2, 71, 79, 206, 248, 29, 80, 227, 131, 5, + 65, 2, 247, 193, 33, 82, 14, 42, 76, 32, 2, 78, 32, 62, 80, 95, 83, 2, + 11, 89, 2, 183, 218, 34, 71, 6, 26, 65, 195, 174, 22, 78, 4, 250, 2, 82, + 231, 213, 34, 76, 4, 46, 65, 193, 197, 33, 5, 83, 73, 70, 73, 83, 2, 193, + 217, 34, 6, 82, 65, 75, 65, 76, 69, 2, 11, 89, 2, 217, 211, 16, 2, 78, + 65, 10, 26, 82, 175, 216, 34, 84, 8, 21, 3, 69, 73, 65, 8, 26, 32, 113, + 2, 73, 32, 6, 38, 69, 242, 5, 68, 183, 166, 22, 78, 2, 11, 75, 2, 29, 5, + 70, 79, 78, 73, 84, 2, 249, 168, 34, 2, 73, 75, 2, 11, 65, 2, 11, 82, 2, + 237, 167, 34, 3, 67, 72, 65, 22, 28, 2, 70, 69, 231, 3, 80, 14, 34, 78, + 49, 4, 83, 73, 83, 32, 4, 11, 32, 4, 202, 231, 30, 75, 235, 230, 3, 65, + 10, 42, 65, 42, 68, 62, 77, 89, 2, 84, 82, 2, 133, 2, 6, 80, 76, 73, 32, + 68, 89, 2, 233, 1, 11, 73, 71, 82, 65, 77, 77, 79, 83, 32, 69, 88, 2, + 173, 1, 18, 79, 78, 79, 71, 82, 65, 77, 77, 79, 83, 32, 84, 69, 83, 83, + 69, 82, 65, 4, 11, 73, 4, 60, 11, 71, 82, 65, 77, 77, 79, 83, 32, 79, 75, + 84, 59, 84, 2, 11, 79, 2, 153, 180, 2, 6, 32, 68, 79, 68, 69, 75, 2, 237, + 163, 34, 4, 73, 77, 79, 82, 8, 26, 79, 139, 240, 29, 83, 6, 56, 6, 75, + 82, 73, 83, 73, 83, 181, 252, 29, 2, 82, 82, 5, 17, 2, 32, 68, 2, 11, 73, + 2, 183, 239, 29, 80, 232, 82, 182, 1, 65, 250, 71, 69, 230, 1, 72, 222, + 32, 73, 208, 41, 3, 74, 75, 32, 138, 26, 76, 186, 18, 79, 238, 117, 82, + 234, 7, 85, 150, 128, 2, 89, 134, 180, 19, 71, 190, 235, 10, 83, 203, 18, + 67, 198, 13, 110, 68, 58, 76, 50, 77, 90, 78, 174, 51, 80, 62, 82, 198, + 7, 84, 138, 1, 85, 162, 185, 18, 67, 227, 208, 6, 83, 4, 34, 85, 205, + 164, 27, 2, 65, 32, 2, 147, 229, 7, 67, 4, 248, 219, 9, 3, 76, 32, 77, + 231, 218, 19, 69, 6, 36, 3, 69, 82, 65, 195, 158, 34, 80, 5, 237, 202, + 27, 7, 32, 87, 73, 84, 72, 32, 70, 193, 11, 148, 1, 16, 65, 68, 73, 65, + 78, 32, 83, 89, 76, 76, 65, 66, 73, 67, 83, 32, 148, 49, 2, 67, 69, 94, + 68, 140, 249, 23, 3, 78, 69, 68, 171, 172, 10, 79, 172, 11, 226, 1, 65, + 190, 1, 66, 162, 2, 67, 242, 8, 69, 50, 70, 198, 4, 72, 38, 73, 30, 75, + 134, 1, 76, 82, 77, 174, 1, 78, 202, 4, 81, 178, 1, 79, 194, 1, 80, 70, + 82, 142, 1, 83, 194, 4, 84, 246, 3, 87, 254, 5, 89, 143, 225, 17, 71, 21, + 90, 65, 30, 73, 40, 10, 84, 72, 65, 80, 65, 83, 67, 65, 78, 32, 242, 234, + 34, 78, 3, 89, 7, 178, 235, 34, 73, 3, 89, 5, 169, 242, 23, 5, 86, 73, + 76, 73, 75, 4, 238, 234, 34, 77, 3, 83, 42, 156, 1, 9, 76, 65, 67, 75, + 70, 79, 79, 84, 32, 212, 171, 19, 9, 73, 66, 76, 69, 45, 67, 82, 69, 69, + 137, 208, 7, 10, 69, 65, 86, 69, 82, 32, 68, 69, 78, 69, 36, 82, 87, 162, + 242, 11, 75, 2, 78, 194, 246, 22, 65, 2, 69, 2, 73, 2, 79, 3, 83, 11, + 222, 232, 34, 65, 2, 69, 2, 73, 3, 79, 141, 3, 82, 65, 186, 42, 87, 222, + 181, 8, 72, 154, 190, 23, 79, 238, 60, 73, 199, 140, 2, 69, 241, 2, 48, + 6, 82, 82, 73, 69, 82, 32, 227, 218, 32, 65, 234, 2, 170, 1, 68, 122, 71, + 94, 72, 46, 73, 46, 74, 82, 75, 30, 78, 66, 83, 66, 80, 2, 90, 58, 84, + 38, 76, 132, 1, 2, 67, 72, 2, 77, 2, 82, 2, 87, 2, 89, 171, 201, 34, 69, + 32, 46, 69, 202, 5, 76, 2, 90, 255, 223, 34, 73, 6, 26, 78, 171, 229, 34, + 69, 4, 134, 154, 17, 69, 249, 211, 6, 2, 84, 65, 30, 254, 4, 72, 202, + 186, 21, 87, 198, 147, 9, 65, 210, 209, 3, 69, 162, 64, 73, 2, 79, 3, 85, + 19, 162, 4, 87, 170, 201, 34, 69, 215, 22, 73, 5, 197, 150, 14, 6, 78, + 73, 84, 73, 65, 76, 26, 202, 3, 74, 222, 159, 34, 69, 234, 61, 87, 186, + 2, 65, 2, 73, 2, 79, 3, 85, 26, 154, 1, 75, 227, 1, 72, 14, 186, 162, 34, + 69, 162, 64, 65, 2, 71, 2, 73, 2, 79, 3, 85, 26, 62, 72, 190, 161, 34, + 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 15, 186, 161, 34, 69, 162, 64, 65, + 2, 73, 2, 79, 3, 85, 72, 34, 76, 70, 84, 66, 72, 3, 83, 24, 130, 1, 72, + 222, 159, 34, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 24, 62, 83, 222, 159, + 34, 69, 162, 64, 65, 2, 73, 2, 79, 3, 85, 12, 218, 159, 34, 69, 162, 64, + 65, 2, 73, 2, 79, 3, 85, 7, 236, 29, 4, 65, 83, 84, 69, 215, 193, 34, 78, + 51, 74, 65, 22, 73, 142, 137, 32, 85, 250, 11, 79, 254, 83, 87, 183, 245, + 1, 69, 7, 131, 210, 32, 65, 33, 40, 4, 78, 65, 76, 32, 139, 222, 34, 73, + 28, 160, 1, 2, 68, 79, 134, 1, 82, 78, 83, 138, 148, 14, 71, 186, 2, 65, + 180, 182, 3, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, 80, 150, 131, 13, + 80, 175, 247, 2, 77, 6, 44, 5, 85, 66, 76, 69, 32, 235, 184, 24, 87, 4, + 164, 205, 6, 12, 83, 72, 79, 82, 84, 32, 86, 69, 82, 84, 73, 67, 187, + 202, 7, 65, 6, 38, 73, 149, 141, 33, 3, 65, 73, 83, 4, 254, 204, 17, 71, + 231, 141, 17, 78, 4, 240, 171, 25, 4, 77, 65, 76, 76, 133, 135, 7, 4, 72, + 79, 82, 84, 4, 234, 201, 26, 89, 231, 144, 8, 75, 7, 170, 218, 34, 73, 3, + 78, 39, 66, 87, 234, 27, 65, 182, 244, 31, 79, 238, 60, 73, 199, 140, 2, + 69, 19, 210, 156, 12, 65, 202, 243, 19, 79, 238, 60, 73, 199, 140, 2, 69, + 49, 142, 7, 72, 154, 20, 65, 66, 87, 246, 243, 31, 79, 238, 60, 73, 199, + 140, 2, 69, 43, 70, 69, 38, 79, 238, 25, 65, 66, 87, 226, 176, 32, 73, + 199, 140, 2, 72, 7, 153, 234, 26, 4, 68, 73, 65, 76, 7, 11, 79, 5, 193, + 144, 33, 8, 83, 69, 45, 67, 82, 69, 69, 32, 149, 1, 164, 1, 8, 45, 67, + 82, 69, 69, 32, 84, 72, 38, 65, 250, 2, 71, 76, 2, 78, 71, 48, 4, 85, 78, + 65, 86, 142, 20, 79, 30, 87, 226, 176, 32, 73, 198, 140, 2, 69, 3, 72, 6, + 158, 201, 32, 73, 199, 140, 2, 69, 63, 104, 6, 83, 75, 65, 80, 73, 32, + 188, 1, 7, 84, 84, 73, 76, 73, 75, 32, 214, 198, 32, 65, 199, 140, 2, 89, + 30, 70, 83, 86, 87, 210, 17, 67, 2, 75, 2, 77, 2, 78, 2, 84, 3, 89, 14, + 174, 218, 29, 67, 2, 80, 2, 84, 236, 103, 2, 75, 87, 190, 156, 2, 87, + 151, 119, 45, 4, 194, 180, 34, 79, 191, 28, 65, 24, 30, 72, 1, 3, 83, 72, + 82, 12, 134, 193, 30, 65, 194, 200, 1, 79, 239, 60, 73, 19, 38, 65, 242, + 136, 32, 79, 239, 60, 73, 9, 218, 197, 32, 65, 199, 140, 2, 73, 15, 138, + 192, 30, 65, 194, 200, 1, 79, 239, 60, 73, 18, 224, 9, 3, 73, 75, 32, + 185, 207, 23, 2, 85, 84, 33, 68, 7, 74, 73, 66, 87, 65, 89, 32, 210, 208, + 34, 78, 2, 79, 3, 89, 24, 74, 78, 166, 191, 30, 83, 226, 144, 4, 67, 2, + 75, 2, 77, 2, 80, 3, 84, 11, 11, 87, 8, 198, 134, 32, 79, 239, 60, 73, + 39, 206, 4, 87, 166, 13, 65, 38, 79, 254, 176, 32, 73, 199, 140, 2, 69, + 39, 104, 7, 45, 67, 82, 69, 69, 32, 82, 146, 14, 87, 182, 2, 65, 182, + 244, 31, 79, 238, 60, 73, 199, 140, 2, 69, 4, 210, 183, 34, 87, 215, 22, + 69, 125, 94, 65, 218, 1, 72, 138, 1, 79, 238, 2, 87, 158, 209, 11, 80, + 254, 233, 20, 73, 199, 140, 2, 69, 43, 26, 89, 203, 192, 32, 65, 37, 25, + 4, 73, 83, 73, 32, 34, 70, 72, 54, 74, 218, 4, 83, 198, 179, 34, 89, 202, + 18, 84, 147, 1, 77, 10, 246, 130, 32, 79, 178, 201, 2, 65, 2, 69, 3, 73, + 6, 246, 244, 30, 85, 255, 214, 3, 73, 37, 70, 87, 202, 13, 79, 202, 128, + 12, 65, 182, 176, 20, 73, 199, 140, 2, 69, 16, 198, 13, 79, 210, 171, 30, + 65, 174, 133, 2, 73, 199, 140, 2, 69, 15, 80, 12, 85, 84, 72, 45, 83, 76, + 65, 86, 69, 89, 32, 75, 246, 201, 34, 79, 3, 89, 8, 226, 200, 34, 65, 2, + 69, 2, 73, 3, 79, 117, 110, 72, 190, 1, 76, 78, 84, 238, 8, 65, 66, 87, + 230, 198, 11, 89, 146, 173, 20, 79, 238, 60, 73, 199, 140, 2, 69, 39, + 108, 7, 45, 67, 82, 69, 69, 32, 84, 226, 4, 87, 154, 177, 30, 65, 194, + 200, 1, 79, 238, 60, 73, 199, 140, 2, 69, 16, 11, 72, 17, 234, 181, 30, + 65, 194, 200, 1, 79, 238, 60, 73, 199, 140, 2, 69, 12, 11, 72, 12, 222, + 253, 31, 79, 222, 178, 2, 87, 214, 22, 65, 2, 69, 3, 73, 24, 50, 72, 158, + 198, 34, 65, 2, 69, 2, 73, 3, 79, 17, 170, 180, 30, 65, 194, 200, 1, 79, + 222, 178, 2, 87, 214, 22, 69, 3, 73, 224, 1, 54, 69, 218, 3, 79, 202, + 132, 12, 65, 183, 176, 20, 73, 185, 1, 17, 2, 83, 84, 182, 1, 44, 6, 45, + 67, 82, 69, 69, 32, 251, 2, 69, 180, 1, 110, 76, 66, 77, 2, 80, 2, 89, + 16, 2, 78, 87, 38, 82, 62, 83, 26, 67, 2, 75, 18, 84, 26, 70, 199, 4, 87, + 27, 178, 6, 87, 182, 171, 30, 65, 194, 200, 1, 79, 179, 201, 2, 69, 17, + 243, 5, 87, 6, 150, 177, 30, 65, 243, 145, 4, 69, 13, 174, 205, 32, 87, + 182, 245, 1, 65, 2, 69, 2, 73, 3, 79, 28, 22, 72, 239, 4, 87, 14, 235, 4, + 87, 16, 22, 72, 199, 4, 87, 2, 179, 204, 32, 87, 2, 171, 204, 23, 82, 31, + 11, 79, 29, 41, 8, 68, 83, 45, 67, 82, 69, 69, 32, 26, 56, 2, 84, 72, + 213, 187, 32, 6, 70, 73, 78, 65, 76, 32, 25, 50, 87, 154, 192, 34, 65, 2, + 69, 2, 73, 3, 79, 14, 166, 174, 30, 65, 194, 200, 1, 79, 238, 60, 73, + 243, 245, 1, 69, 61, 92, 6, 45, 67, 82, 69, 69, 32, 150, 1, 65, 38, 79, + 30, 87, 226, 176, 32, 73, 199, 140, 2, 69, 24, 110, 80, 146, 243, 31, 67, + 2, 75, 2, 76, 2, 77, 2, 78, 2, 83, 2, 84, 2, 89, 134, 172, 2, 79, 247, + 30, 87, 4, 210, 200, 32, 87, 195, 214, 1, 79, 9, 158, 177, 32, 65, 199, + 140, 2, 89, 7, 190, 189, 34, 79, 3, 89, 14, 178, 171, 30, 65, 194, 200, + 1, 79, 238, 60, 73, 199, 140, 2, 69, 10, 26, 76, 203, 188, 34, 82, 9, 26, + 32, 231, 193, 9, 76, 4, 218, 171, 22, 67, 231, 164, 10, 84, 4, 178, 165, + 34, 76, 215, 22, 89, 4, 180, 191, 15, 3, 73, 84, 85, 161, 134, 8, 3, 82, + 73, 67, 124, 140, 1, 2, 68, 32, 102, 69, 72, 11, 73, 65, 78, 32, 76, 69, + 84, 84, 69, 82, 32, 210, 3, 79, 62, 80, 90, 82, 217, 237, 27, 4, 32, 83, + 76, 73, 6, 52, 5, 73, 78, 68, 69, 88, 233, 170, 33, 2, 70, 73, 5, 229, + 168, 33, 6, 32, 68, 73, 86, 73, 68, 6, 26, 84, 151, 211, 21, 32, 5, 193, + 169, 10, 6, 32, 73, 78, 83, 69, 82, 98, 196, 1, 2, 67, 45, 38, 76, 22, + 77, 50, 78, 38, 83, 46, 84, 22, 85, 166, 219, 3, 65, 2, 68, 2, 69, 2, 71, + 2, 75, 2, 80, 158, 132, 27, 82, 218, 201, 1, 73, 198, 140, 2, 66, 2, 79, + 2, 81, 3, 88, 4, 246, 139, 27, 49, 155, 177, 3, 51, 7, 203, 220, 3, 68, + 11, 11, 66, 9, 226, 182, 34, 50, 2, 51, 3, 52, 9, 190, 182, 34, 68, 2, + 71, 3, 78, 13, 226, 219, 3, 72, 2, 84, 187, 218, 30, 83, 7, 183, 219, 3, + 84, 13, 11, 85, 11, 11, 85, 9, 194, 181, 34, 50, 2, 51, 3, 85, 4, 224, + 169, 33, 6, 85, 83, 69, 76, 32, 72, 191, 139, 1, 78, 4, 172, 215, 27, 6, + 32, 83, 84, 82, 69, 65, 137, 223, 5, 7, 69, 78, 84, 82, 89, 32, 83, 4, + 218, 225, 27, 73, 135, 182, 6, 79, 9, 29, 5, 32, 70, 65, 67, 69, 7, 33, + 6, 32, 87, 73, 84, 72, 32, 4, 148, 242, 6, 2, 84, 69, 197, 152, 25, 6, + 87, 82, 89, 32, 83, 77, 108, 88, 16, 67, 65, 83, 73, 65, 78, 32, 65, 76, + 66, 65, 78, 73, 65, 78, 32, 243, 172, 30, 84, 106, 60, 7, 76, 69, 84, 84, + 69, 82, 32, 161, 147, 28, 2, 67, 73, 104, 170, 2, 65, 34, 67, 146, 1, 68, + 78, 69, 34, 71, 46, 73, 46, 74, 34, 75, 34, 76, 34, 80, 34, 83, 74, 84, + 62, 89, 46, 90, 148, 133, 17, 2, 81, 65, 190, 193, 6, 77, 252, 132, 9, 3, + 86, 69, 89, 154, 33, 70, 128, 19, 3, 78, 79, 87, 226, 32, 82, 186, 11, + 88, 154, 46, 79, 202, 26, 66, 237, 24, 3, 72, 69, 89, 4, 186, 246, 33, + 79, 179, 28, 76, 16, 34, 65, 34, 72, 49, 2, 89, 65, 4, 242, 222, 33, 89, + 227, 79, 82, 8, 218, 197, 28, 65, 146, 215, 5, 79, 203, 17, 73, 4, 130, + 174, 34, 87, 3, 89, 8, 42, 90, 162, 225, 32, 89, 243, 175, 1, 65, 4, 154, + 159, 33, 89, 219, 124, 65, 4, 186, 221, 33, 89, 227, 79, 66, 4, 212, 254, + 32, 2, 72, 69, 207, 157, 1, 73, 6, 214, 238, 32, 82, 154, 110, 87, 135, + 77, 78, 4, 214, 157, 33, 72, 227, 16, 65, 4, 226, 173, 33, 73, 199, 69, + 65, 4, 130, 220, 33, 65, 171, 51, 89, 4, 142, 1, 73, 215, 218, 33, 69, 8, + 34, 72, 233, 167, 34, 2, 69, 89, 6, 234, 171, 20, 65, 207, 237, 13, 79, + 6, 38, 73, 150, 156, 33, 89, 199, 80, 65, 2, 215, 241, 33, 87, 4, 252, + 227, 33, 2, 65, 89, 1, 2, 79, 87, 6, 242, 166, 16, 72, 131, 186, 4, 65, + 16, 72, 2, 68, 73, 32, 2, 78, 84, 204, 193, 32, 3, 76, 84, 73, 151, 81, + 82, 4, 142, 162, 33, 32, 223, 100, 76, 8, 32, 2, 82, 69, 207, 161, 33, + 32, 6, 44, 5, 76, 73, 78, 69, 32, 167, 232, 4, 32, 4, 246, 234, 21, 76, + 207, 210, 10, 79, 130, 6, 102, 65, 170, 17, 69, 162, 8, 73, 190, 2, 79, + 188, 205, 13, 6, 82, 73, 83, 84, 77, 65, 155, 137, 10, 85, 198, 2, 66, + 73, 32, 4, 75, 77, 65, 32, 148, 6, 2, 77, 32, 131, 8, 82, 4, 218, 232, + 33, 78, 223, 61, 82, 142, 1, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 142, + 3, 83, 98, 86, 142, 224, 23, 68, 154, 236, 7, 81, 176, 95, 5, 77, 65, 65, + 89, 89, 240, 179, 1, 2, 65, 85, 3, 79, 76, 202, 1, 68, 54, 78, 54, 84, + 54, 89, 178, 153, 29, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, 2, 80, 170, + 147, 3, 72, 2, 77, 2, 82, 2, 83, 2, 86, 2, 87, 254, 242, 1, 65, 186, 2, + 69, 2, 73, 3, 85, 8, 202, 154, 29, 68, 170, 147, 3, 72, 255, 242, 1, 65, + 8, 190, 173, 32, 71, 2, 78, 2, 89, 255, 242, 1, 65, 8, 226, 153, 29, 84, + 170, 147, 3, 72, 255, 242, 1, 65, 4, 214, 172, 32, 89, 255, 242, 1, 65, + 8, 40, 4, 73, 71, 78, 32, 195, 142, 22, 69, 6, 218, 131, 30, 67, 246, + 227, 1, 86, 247, 244, 1, 65, 26, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 171, 228, 31, 73, 24, 206, 194, 30, 65, 250, 6, 85, 186, 202, 1, + 69, 2, 73, 3, 79, 166, 1, 236, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, + 32, 83, 73, 71, 78, 32, 112, 7, 76, 69, 84, 84, 69, 82, 32, 152, 4, 12, + 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 60, 11, 86, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 199, 164, 32, 68, 14, 72, 6, 70, 73, 78, 65, + 76, 32, 142, 155, 34, 76, 2, 82, 2, 87, 3, 89, 6, 238, 156, 34, 78, 86, + 72, 3, 77, 104, 132, 2, 6, 70, 73, 78, 65, 76, 32, 110, 78, 50, 77, 78, + 80, 174, 229, 11, 66, 226, 141, 6, 68, 158, 145, 12, 83, 198, 136, 2, 65, + 132, 197, 1, 2, 67, 72, 2, 71, 2, 74, 2, 75, 2, 84, 138, 69, 72, 2, 76, + 2, 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 22, 150, 157, 32, + 78, 166, 192, 1, 83, 206, 60, 67, 146, 1, 71, 2, 75, 2, 76, 2, 80, 2, 82, + 2, 84, 3, 89, 14, 46, 71, 34, 72, 138, 131, 34, 85, 215, 22, 65, 4, 166, + 131, 34, 85, 215, 22, 65, 6, 134, 131, 34, 85, 158, 20, 74, 187, 2, 65, + 6, 246, 150, 34, 72, 2, 80, 187, 2, 65, 8, 182, 170, 11, 83, 134, 142, + 19, 68, 45, 4, 84, 82, 73, 80, 20, 170, 193, 30, 65, 222, 202, 1, 73, + 166, 204, 1, 79, 2, 85, 203, 44, 69, 14, 72, 7, 65, 67, 84, 69, 82, 32, + 84, 49, 7, 84, 32, 87, 73, 84, 72, 32, 8, 244, 255, 9, 3, 65, 66, 85, + 239, 128, 24, 73, 6, 128, 1, 13, 85, 80, 87, 65, 82, 68, 83, 32, 84, 82, + 69, 78, 68, 161, 186, 25, 12, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, + 82, 5, 153, 250, 6, 6, 32, 65, 78, 68, 32, 89, 236, 2, 80, 2, 67, 75, 82, + 69, 90, 82, 176, 253, 4, 2, 83, 84, 133, 166, 15, 2, 81, 85, 6, 56, 8, + 69, 82, 32, 66, 79, 65, 82, 68, 143, 211, 33, 32, 5, 227, 152, 31, 32, 4, + 224, 202, 27, 4, 83, 69, 32, 87, 221, 156, 5, 9, 82, 73, 78, 71, 32, 77, + 69, 71, 65, 222, 2, 40, 5, 79, 75, 69, 69, 32, 143, 5, 82, 216, 2, 46, 76, 1, 7, 83, 77, 65, 76, 76, 32, 76, 172, 1, 33, 6, 69, 84, 84, 69, 82, 32, 172, 1, 170, 1, 68, 74, 72, 74, 78, 70, 83, 62, 84, 54, 71, 2, 76, 2, - 77, 0, 2, 81, 85, 2, 87, 2, 89, 194, 228, 33, 75, 186, 2, 65, 2, 69, 2, - 73, 2, 79, 2, 85, 3, 86, 14, 130, 231, 33, 76, 186, 2, 65, 2, 69, 2, 73, - 2, 79, 2, 85, 3, 86, 14, 186, 230, 33, 78, 186, 2, 65, 2, 69, 2, 73, 2, - 79, 2, 85, 3, 86, 14, 142, 200, 29, 65, 158, 160, 4, 69, 2, 73, 2, 79, 2, - 85, 3, 86, 15, 230, 231, 33, 65, 2, 69, 2, 73, 2, 79, 2, 85, 3, 86, 30, - 50, 76, 2, 83, 250, 230, 33, 65, 2, 69, 3, 73, 12, 246, 230, 33, 65, 2, - 69, 2, 73, 2, 79, 2, 85, 3, 86, 4, 48, 6, 89, 32, 66, 76, 79, 83, 147, - 210, 32, 73, 2, 163, 213, 33, 83, 12, 96, 2, 76, 68, 254, 235, 7, 32, - 204, 246, 2, 2, 80, 77, 224, 174, 21, 2, 67, 75, 255, 131, 1, 82, 5, 237, - 216, 28, 7, 82, 69, 78, 32, 67, 82, 79, 60, 92, 8, 82, 65, 83, 77, 73, - 65, 78, 32, 174, 254, 4, 80, 189, 255, 11, 5, 67, 79, 76, 65, 84, 56, 52, - 7, 76, 69, 84, 84, 69, 82, 32, 247, 194, 21, 78, 42, 224, 1, 6, 67, 85, - 82, 76, 69, 68, 30, 83, 206, 191, 21, 68, 34, 76, 158, 206, 1, 82, 178, - 215, 2, 65, 50, 71, 90, 90, 98, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, - 66, 178, 216, 4, 78, 134, 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 2, 181, - 227, 32, 2, 32, 87, 6, 180, 191, 21, 6, 77, 65, 76, 76, 32, 65, 252, 239, - 5, 2, 65, 77, 195, 178, 5, 72, 232, 4, 66, 78, 20, 2, 82, 67, 201, 40, 7, - 84, 89, 83, 67, 65, 80, 69, 2, 155, 191, 33, 69, 226, 4, 28, 2, 76, 69, - 207, 39, 85, 220, 4, 30, 32, 185, 6, 2, 68, 32, 24, 244, 1, 5, 87, 73, - 84, 72, 32, 185, 186, 18, 49, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, - 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 65, 78, 68, - 32, 84, 79, 80, 32, 72, 65, 76, 70, 32, 68, 73, 86, 73, 68, 69, 68, 32, - 66, 89, 22, 146, 2, 76, 46, 83, 108, 22, 84, 87, 79, 32, 72, 79, 82, 73, - 90, 79, 78, 84, 65, 76, 32, 83, 84, 82, 79, 75, 69, 83, 44, 6, 85, 80, - 80, 69, 82, 32, 44, 17, 65, 76, 76, 32, 66, 85, 84, 32, 85, 80, 80, 69, - 82, 32, 76, 69, 70, 182, 181, 26, 86, 178, 136, 4, 82, 231, 249, 1, 72, - 4, 198, 191, 30, 69, 49, 4, 79, 87, 69, 82, 4, 104, 11, 77, 65, 76, 76, - 32, 67, 73, 82, 67, 76, 69, 221, 5, 10, 85, 80, 69, 82, 73, 77, 80, 79, - 83, 69, 2, 153, 217, 30, 6, 32, 84, 79, 32, 84, 72, 4, 40, 4, 82, 73, 71, - 72, 159, 190, 30, 72, 2, 177, 190, 30, 10, 84, 32, 81, 85, 65, 68, 82, - 65, 78, 84, 196, 4, 230, 2, 65, 186, 1, 66, 58, 67, 230, 1, 68, 246, 1, - 72, 230, 2, 73, 194, 10, 75, 190, 2, 76, 38, 77, 136, 1, 7, 78, 85, 77, - 66, 69, 82, 32, 192, 4, 17, 79, 80, 69, 78, 32, 67, 69, 78, 84, 82, 69, - 32, 69, 73, 71, 72, 84, 54, 80, 114, 82, 50, 84, 78, 87, 232, 248, 9, 4, - 90, 69, 82, 79, 234, 183, 16, 69, 246, 177, 4, 86, 158, 68, 71, 190, 113, - 83, 223, 160, 1, 88, 6, 100, 12, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, - 83, 69, 213, 149, 30, 7, 83, 84, 69, 82, 73, 83, 75, 4, 200, 238, 8, 11, - 45, 82, 79, 84, 65, 84, 69, 68, 32, 68, 73, 179, 204, 23, 32, 4, 32, 2, - 79, 76, 167, 245, 31, 85, 2, 131, 143, 14, 68, 16, 60, 4, 82, 79, 83, 83, - 210, 2, 32, 174, 209, 33, 67, 3, 68, 10, 26, 32, 155, 193, 1, 73, 8, 64, - 6, 70, 79, 82, 77, 69, 69, 229, 227, 31, 4, 80, 79, 77, 77, 7, 33, 6, 32, - 87, 73, 84, 72, 32, 4, 214, 175, 31, 70, 207, 82, 84, 30, 34, 73, 70, 79, - 211, 233, 31, 65, 24, 216, 187, 4, 8, 86, 73, 83, 73, 79, 78, 32, 83, - 211, 165, 27, 71, 4, 64, 10, 76, 76, 65, 82, 32, 83, 73, 71, 78, 32, 191, - 145, 30, 84, 2, 145, 156, 19, 13, 87, 73, 84, 72, 32, 79, 86, 69, 82, 76, - 65, 73, 68, 64, 192, 1, 6, 65, 78, 71, 85, 76, 32, 204, 191, 12, 20, 79, - 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 87, 73, 84, 72, 32, - 78, 136, 210, 9, 4, 85, 77, 65, 78, 145, 1, 4, 69, 65, 86, 89, 58, 110, - 67, 184, 188, 20, 5, 73, 69, 85, 78, 71, 42, 72, 30, 75, 66, 77, 34, 78, - 34, 80, 62, 82, 30, 83, 27, 84, 8, 150, 188, 20, 72, 157, 3, 4, 73, 69, - 85, 67, 116, 220, 1, 9, 68, 69, 79, 71, 82, 65, 80, 72, 32, 196, 8, 27, - 84, 65, 76, 73, 67, 32, 76, 65, 84, 73, 78, 32, 67, 65, 80, 73, 84, 65, - 76, 32, 76, 69, 84, 84, 69, 82, 32, 157, 138, 27, 9, 78, 70, 79, 82, 77, - 65, 84, 73, 79, 110, 174, 1, 65, 110, 67, 90, 69, 86, 70, 62, 72, 38, 75, - 70, 76, 50, 77, 86, 78, 62, 81, 30, 82, 74, 83, 162, 173, 6, 80, 242, - 143, 14, 84, 50, 87, 170, 214, 1, 73, 255, 151, 10, 79, 8, 158, 189, 20, - 76, 184, 133, 1, 3, 67, 67, 69, 204, 239, 6, 3, 84, 84, 69, 249, 17, 5, - 68, 86, 65, 78, 84, 8, 26, 79, 163, 200, 30, 69, 6, 240, 200, 10, 2, 82, - 82, 230, 243, 9, 78, 203, 251, 12, 80, 8, 234, 188, 20, 78, 222, 163, 9, - 65, 188, 165, 3, 5, 88, 67, 69, 76, 76, 231, 24, 73, 10, 246, 188, 20, - 73, 208, 184, 5, 2, 69, 77, 191, 218, 6, 79, 4, 210, 163, 29, 73, 247, - 167, 3, 65, 4, 212, 155, 7, 8, 73, 78, 68, 69, 82, 71, 65, 82, 231, 189, - 22, 79, 6, 134, 188, 20, 65, 142, 141, 12, 79, 195, 83, 69, 8, 38, 69, - 182, 162, 32, 65, 191, 84, 79, 4, 160, 173, 32, 3, 68, 73, 67, 251, 15, - 84, 6, 26, 73, 171, 174, 32, 65, 4, 218, 155, 33, 71, 231, 19, 78, 2, - 153, 179, 28, 2, 85, 69, 8, 26, 69, 251, 154, 33, 73, 6, 142, 187, 20, - 83, 221, 253, 7, 2, 76, 73, 22, 90, 69, 38, 85, 144, 148, 14, 2, 67, 72, - 214, 166, 6, 79, 22, 80, 34, 84, 215, 252, 11, 73, 4, 246, 152, 10, 67, - 215, 215, 21, 86, 6, 226, 187, 20, 80, 166, 255, 1, 73, 139, 137, 11, 78, - 4, 218, 195, 33, 67, 3, 82, 98, 116, 8, 65, 84, 65, 75, 65, 78, 65, 32, - 133, 1, 16, 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, - 32, 94, 170, 229, 10, 72, 2, 75, 2, 77, 2, 78, 2, 82, 2, 83, 2, 84, 126, - 87, 46, 89, 246, 219, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 228, 138, - 16, 3, 74, 85, 69, 221, 151, 17, 4, 67, 72, 65, 77, 106, 158, 173, 27, - 65, 255, 129, 4, 69, 4, 100, 19, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, - 73, 79, 78, 32, 83, 73, 71, 78, 32, 243, 185, 31, 73, 2, 157, 84, 4, 87, - 73, 84, 72, 98, 50, 69, 46, 70, 98, 83, 94, 84, 191, 183, 20, 78, 6, 180, - 1, 3, 73, 71, 72, 199, 169, 25, 76, 30, 28, 3, 73, 70, 84, 35, 79, 6, - 214, 1, 89, 155, 233, 31, 69, 24, 142, 2, 82, 167, 183, 20, 85, 8, 40, 4, - 69, 86, 69, 78, 1, 2, 73, 88, 4, 11, 84, 4, 104, 2, 89, 32, 143, 233, 31, - 69, 52, 56, 2, 69, 78, 32, 4, 72, 73, 82, 84, 29, 2, 87, 69, 5, 11, 32, - 2, 247, 198, 25, 79, 24, 74, 89, 175, 232, 31, 69, 24, 26, 78, 215, 191, - 32, 76, 22, 17, 2, 84, 89, 23, 11, 32, 20, 72, 2, 79, 78, 154, 203, 31, - 70, 30, 83, 42, 84, 170, 86, 78, 235, 110, 69, 4, 210, 197, 25, 32, 235, - 245, 7, 69, 2, 129, 165, 31, 8, 32, 80, 79, 73, 78, 84, 69, 68, 8, 132, - 199, 17, 3, 79, 83, 84, 154, 246, 6, 65, 192, 204, 2, 8, 69, 82, 80, 69, - 78, 68, 73, 67, 139, 171, 4, 76, 4, 128, 250, 29, 3, 73, 78, 71, 171, - 170, 1, 69, 6, 52, 7, 82, 73, 65, 78, 71, 76, 69, 131, 175, 26, 73, 5, - 147, 147, 22, 32, 6, 44, 5, 72, 73, 84, 69, 32, 203, 184, 33, 90, 4, 142, - 217, 31, 66, 139, 25, 83, 6, 218, 160, 27, 77, 196, 178, 4, 6, 76, 65, - 84, 73, 79, 78, 225, 161, 1, 3, 83, 32, 84, 5, 225, 242, 31, 6, 32, 65, - 84, 32, 68, 85, 158, 18, 192, 1, 24, 67, 79, 77, 80, 65, 84, 73, 66, 73, - 76, 73, 84, 89, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 144, 3, 8, - 82, 65, 68, 73, 67, 65, 76, 32, 245, 17, 7, 83, 84, 82, 79, 75, 69, 32, - 236, 15, 24, 2, 50, 70, 75, 70, 188, 8, 34, 65, 162, 246, 10, 56, 3, 57, - 60, 202, 1, 49, 135, 236, 18, 48, 176, 7, 26, 65, 227, 245, 10, 57, 176, - 3, 134, 1, 54, 134, 236, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 139, 146, 9, 68, 28, 198, 179, 33, - 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, - 2, 66, 2, 67, 3, 68, 230, 1, 210, 1, 66, 126, 67, 226, 4, 68, 62, 69, 50, - 70, 34, 71, 50, 72, 86, 74, 154, 1, 76, 62, 77, 130, 1, 80, 46, 82, 78, - 83, 182, 3, 84, 82, 87, 168, 218, 10, 2, 78, 69, 144, 17, 4, 75, 78, 73, - 70, 251, 151, 12, 79, 14, 74, 79, 180, 250, 10, 4, 82, 85, 83, 72, 246, - 233, 4, 65, 251, 192, 13, 76, 6, 186, 161, 10, 76, 154, 248, 22, 78, 215, - 22, 88, 56, 104, 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 68, 32, 162, - 3, 73, 50, 76, 158, 173, 32, 79, 191, 78, 72, 44, 114, 69, 38, 70, 50, - 71, 38, 76, 34, 83, 94, 84, 154, 199, 14, 87, 42, 68, 234, 226, 16, 66, - 222, 120, 72, 135, 2, 67, 4, 174, 218, 31, 86, 191, 183, 1, 65, 6, 210, - 197, 31, 73, 218, 214, 1, 76, 235, 16, 82, 4, 186, 143, 23, 79, 147, 215, - 8, 65, 4, 238, 171, 32, 69, 187, 48, 79, 10, 150, 219, 6, 73, 134, 186, - 17, 65, 198, 194, 1, 72, 248, 178, 7, 3, 80, 69, 69, 215, 11, 69, 6, 214, - 10, 85, 152, 176, 10, 2, 65, 78, 203, 158, 4, 79, 4, 140, 210, 20, 3, 86, - 73, 76, 151, 200, 12, 84, 4, 170, 214, 17, 73, 131, 193, 14, 79, 6, 216, - 148, 24, 2, 73, 86, 186, 241, 2, 69, 179, 164, 6, 79, 10, 196, 6, 2, 65, - 84, 194, 141, 33, 87, 3, 89, 4, 242, 137, 32, 73, 131, 80, 79, 8, 244, 5, - 4, 82, 65, 83, 83, 143, 186, 29, 72, 10, 48, 2, 69, 65, 146, 159, 28, 79, - 163, 252, 3, 65, 6, 186, 8, 82, 239, 160, 33, 68, 10, 72, 12, 45, 83, 73, - 77, 80, 76, 73, 70, 73, 69, 68, 32, 179, 152, 31, 65, 8, 42, 84, 174, - 195, 14, 68, 247, 244, 16, 69, 4, 194, 6, 85, 227, 206, 14, 79, 12, 248, - 241, 10, 3, 79, 78, 71, 197, 245, 15, 3, 65, 77, 69, 14, 18, 69, 35, 79, - 4, 242, 138, 33, 65, 159, 27, 83, 10, 136, 241, 10, 3, 85, 78, 68, 166, - 162, 13, 84, 162, 205, 7, 82, 211, 118, 79, 6, 188, 240, 10, 2, 65, 87, - 175, 182, 18, 69, 8, 34, 65, 169, 213, 28, 2, 69, 80, 6, 130, 214, 32, - 73, 226, 79, 77, 3, 80, 38, 122, 69, 90, 73, 186, 1, 78, 196, 1, 4, 80, - 73, 82, 73, 248, 234, 10, 4, 77, 65, 76, 76, 214, 149, 12, 72, 195, 208, - 9, 85, 8, 40, 4, 67, 79, 78, 68, 227, 154, 32, 65, 6, 11, 32, 6, 166, - 180, 31, 84, 183, 86, 79, 12, 60, 9, 77, 80, 76, 73, 70, 73, 69, 68, 32, - 147, 160, 33, 76, 10, 34, 72, 50, 87, 135, 182, 10, 89, 4, 140, 160, 10, - 3, 65, 76, 70, 223, 248, 17, 79, 4, 234, 208, 6, 65, 175, 129, 22, 72, 6, - 192, 1, 2, 79, 85, 151, 241, 32, 65, 8, 58, 85, 146, 217, 24, 65, 218, - 204, 1, 72, 147, 174, 3, 73, 2, 239, 134, 24, 82, 12, 26, 65, 49, 2, 69, - 83, 8, 164, 235, 10, 2, 76, 75, 1, 3, 84, 69, 82, 4, 247, 234, 10, 84, - 76, 110, 72, 182, 1, 80, 38, 83, 142, 163, 31, 84, 176, 250, 1, 2, 66, - 88, 2, 87, 2, 88, 86, 68, 2, 78, 3, 81, 31, 42, 80, 22, 88, 30, 90, 143, - 159, 33, 71, 5, 231, 158, 33, 87, 4, 210, 158, 33, 87, 87, 71, 19, 50, - 90, 194, 163, 31, 87, 154, 251, 1, 71, 3, 84, 9, 190, 163, 31, 90, 155, - 251, 1, 80, 9, 178, 158, 33, 68, 2, 71, 3, 90, 23, 50, 87, 30, 90, 194, - 157, 33, 71, 2, 80, 3, 84, 7, 218, 157, 33, 71, 3, 90, 9, 234, 156, 33, - 87, 86, 80, 3, 90, 124, 62, 65, 194, 1, 73, 114, 79, 181, 241, 20, 4, 69, - 65, 82, 32, 8, 132, 1, 2, 80, 80, 208, 231, 18, 6, 83, 83, 73, 67, 65, - 76, 169, 137, 13, 14, 77, 83, 72, 69, 76, 76, 32, 77, 79, 66, 73, 76, 69, - 32, 4, 196, 189, 19, 5, 73, 78, 71, 32, 72, 239, 227, 1, 69, 6, 48, 6, - 78, 75, 73, 78, 71, 32, 223, 131, 32, 80, 4, 190, 145, 25, 71, 181, 183, - 3, 6, 66, 69, 69, 82, 32, 77, 108, 72, 2, 67, 75, 140, 10, 2, 83, 69, - 236, 3, 2, 85, 68, 131, 153, 30, 87, 70, 64, 6, 32, 70, 65, 67, 69, 32, - 237, 2, 5, 87, 73, 83, 69, 32, 48, 58, 69, 46, 70, 36, 2, 78, 73, 2, 79, - 18, 83, 51, 84, 8, 120, 2, 76, 69, 125, 4, 73, 71, 72, 84, 8, 178, 1, 73, - 25, 3, 79, 85, 82, 4, 155, 1, 78, 8, 26, 69, 125, 2, 73, 88, 4, 57, 2, - 86, 69, 16, 38, 69, 14, 87, 41, 3, 72, 82, 69, 4, 63, 78, 8, 24, 2, 69, - 76, 27, 79, 4, 11, 86, 4, 11, 69, 4, 180, 252, 28, 2, 32, 79, 137, 173, - 2, 3, 45, 84, 72, 22, 90, 67, 58, 68, 122, 71, 48, 5, 82, 73, 71, 72, 84, - 226, 2, 84, 122, 79, 143, 137, 31, 73, 4, 196, 1, 3, 76, 79, 83, 141, 99, - 4, 79, 78, 84, 79, 2, 141, 3, 26, 79, 87, 78, 87, 65, 82, 68, 83, 32, 65, + 77, 0, 2, 81, 85, 2, 87, 2, 89, 162, 140, 34, 75, 186, 2, 65, 2, 69, 2, + 73, 2, 79, 2, 85, 3, 86, 14, 226, 142, 34, 76, 186, 2, 65, 2, 69, 2, 73, + 2, 79, 2, 85, 3, 86, 14, 154, 142, 34, 78, 186, 2, 65, 2, 69, 2, 73, 2, + 79, 2, 85, 3, 86, 14, 170, 255, 29, 65, 226, 144, 4, 69, 2, 73, 2, 79, 2, + 85, 3, 86, 15, 198, 143, 34, 65, 2, 69, 2, 73, 2, 79, 2, 85, 3, 86, 30, + 50, 76, 2, 83, 218, 142, 34, 65, 2, 69, 3, 73, 12, 214, 142, 34, 65, 2, + 69, 2, 73, 2, 79, 2, 85, 3, 86, 6, 32, 2, 89, 32, 247, 247, 32, 73, 4, + 40, 4, 66, 76, 79, 83, 163, 248, 32, 83, 2, 231, 252, 33, 83, 16, 152, 1, + 2, 76, 68, 48, 11, 78, 69, 83, 69, 32, 83, 77, 65, 76, 76, 32, 206, 247, + 7, 32, 184, 247, 2, 2, 80, 77, 228, 198, 21, 2, 67, 75, 155, 134, 1, 82, + 5, 173, 247, 28, 7, 82, 69, 78, 32, 67, 82, 79, 4, 248, 224, 19, 10, 83, + 73, 77, 80, 76, 73, 70, 73, 69, 68, 1, 11, 84, 82, 65, 68, 73, 84, 73, + 79, 78, 65, 76, 60, 92, 8, 82, 65, 83, 77, 73, 65, 78, 32, 254, 130, 5, + 80, 197, 141, 12, 5, 67, 79, 76, 65, 84, 56, 52, 7, 76, 69, 84, 84, 69, + 82, 32, 199, 214, 21, 78, 42, 224, 1, 6, 67, 85, 82, 76, 69, 68, 30, 83, + 158, 211, 21, 68, 34, 76, 134, 206, 1, 82, 142, 220, 2, 65, 50, 71, 90, + 90, 98, 89, 198, 207, 1, 72, 234, 5, 75, 174, 81, 66, 170, 225, 4, 78, + 134, 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 2, 209, 137, 33, 2, 32, 87, + 6, 132, 211, 21, 6, 77, 65, 76, 76, 32, 65, 232, 243, 5, 2, 65, 77, 163, + 193, 5, 72, 232, 4, 66, 78, 20, 2, 82, 67, 201, 40, 7, 84, 89, 83, 67, + 65, 80, 69, 2, 183, 229, 33, 69, 226, 4, 28, 2, 76, 69, 207, 39, 85, 220, + 4, 30, 32, 185, 6, 2, 68, 32, 24, 244, 1, 5, 87, 73, 84, 72, 32, 161, + 205, 18, 49, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, + 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 65, 78, 68, 32, 84, 79, 80, + 32, 72, 65, 76, 70, 32, 68, 73, 86, 73, 68, 69, 68, 32, 66, 89, 22, 146, + 2, 76, 46, 83, 108, 22, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, + 65, 76, 32, 83, 84, 82, 79, 75, 69, 83, 44, 6, 85, 80, 80, 69, 82, 32, + 44, 17, 65, 76, 76, 32, 66, 85, 84, 32, 85, 80, 80, 69, 82, 32, 76, 69, + 70, 190, 205, 26, 86, 242, 146, 4, 82, 183, 251, 1, 72, 4, 142, 226, 30, + 69, 49, 4, 79, 87, 69, 82, 4, 104, 11, 77, 65, 76, 76, 32, 67, 73, 82, + 67, 76, 69, 221, 5, 10, 85, 80, 69, 82, 73, 77, 80, 79, 83, 69, 2, 225, + 251, 30, 6, 32, 84, 79, 32, 84, 72, 4, 40, 4, 82, 73, 71, 72, 231, 224, + 30, 72, 2, 249, 224, 30, 10, 84, 32, 81, 85, 65, 68, 82, 65, 78, 84, 196, + 4, 230, 2, 65, 186, 1, 66, 58, 67, 230, 1, 68, 246, 1, 72, 230, 2, 73, + 194, 10, 75, 190, 2, 76, 38, 77, 136, 1, 7, 78, 85, 77, 66, 69, 82, 32, + 192, 4, 17, 79, 80, 69, 78, 32, 67, 69, 78, 84, 82, 69, 32, 69, 73, 71, + 72, 84, 54, 80, 114, 82, 50, 84, 78, 87, 192, 132, 10, 4, 90, 69, 82, 79, + 158, 196, 16, 69, 182, 188, 4, 86, 146, 68, 71, 150, 115, 83, 227, 162, + 1, 88, 6, 100, 12, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 157, + 184, 30, 7, 83, 84, 69, 82, 73, 83, 75, 4, 212, 249, 8, 11, 45, 82, 79, + 84, 65, 84, 69, 68, 32, 68, 73, 191, 229, 23, 32, 4, 32, 2, 79, 76, 219, + 152, 32, 85, 2, 223, 221, 14, 68, 16, 60, 4, 82, 79, 83, 83, 210, 2, 32, + 202, 247, 33, 67, 3, 68, 10, 26, 32, 199, 197, 1, 73, 8, 64, 6, 70, 79, + 82, 77, 69, 69, 153, 135, 32, 4, 80, 79, 77, 77, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 158, 210, 31, 70, 135, 84, 84, 30, 34, 73, 70, 79, 135, + 141, 32, 65, 24, 168, 192, 4, 8, 86, 73, 83, 73, 79, 78, 32, 83, 183, + 196, 27, 71, 4, 64, 10, 76, 76, 65, 82, 32, 83, 73, 71, 78, 32, 135, 180, + 30, 84, 2, 173, 175, 19, 13, 87, 73, 84, 72, 32, 79, 86, 69, 82, 76, 65, + 73, 68, 64, 192, 1, 6, 65, 78, 71, 85, 76, 32, 252, 203, 12, 20, 79, 82, + 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, 82, 32, 87, 73, 84, 72, 32, 78, + 204, 217, 9, 4, 85, 77, 65, 78, 145, 1, 4, 69, 65, 86, 89, 58, 110, 67, + 140, 207, 20, 5, 73, 69, 85, 78, 71, 42, 72, 30, 75, 66, 77, 34, 78, 34, + 80, 62, 82, 30, 83, 27, 84, 8, 234, 206, 20, 72, 157, 3, 4, 73, 69, 85, + 67, 116, 220, 1, 9, 68, 69, 79, 71, 82, 65, 80, 72, 32, 196, 8, 27, 84, + 65, 76, 73, 67, 32, 76, 65, 84, 73, 78, 32, 67, 65, 80, 73, 84, 65, 76, + 32, 76, 69, 84, 84, 69, 82, 32, 169, 162, 27, 9, 78, 70, 79, 82, 77, 65, + 84, 73, 79, 110, 174, 1, 65, 110, 67, 90, 69, 86, 70, 62, 72, 38, 75, 70, + 76, 50, 77, 86, 78, 62, 81, 30, 82, 74, 83, 198, 178, 6, 80, 162, 157, + 14, 84, 50, 87, 202, 215, 1, 73, 163, 168, 10, 79, 8, 242, 207, 20, 76, + 200, 134, 1, 3, 67, 67, 69, 184, 249, 6, 3, 84, 84, 69, 249, 17, 5, 68, + 86, 65, 78, 84, 8, 26, 79, 235, 234, 30, 69, 6, 236, 212, 10, 2, 82, 82, + 190, 250, 9, 78, 147, 143, 13, 80, 8, 190, 207, 20, 78, 174, 179, 9, 65, + 180, 169, 3, 5, 88, 67, 69, 76, 76, 231, 24, 73, 10, 202, 207, 20, 73, + 144, 190, 5, 2, 69, 77, 199, 232, 6, 79, 4, 146, 193, 29, 73, 211, 176, + 3, 65, 4, 228, 166, 7, 8, 73, 78, 68, 69, 82, 71, 65, 82, 167, 212, 22, + 79, 6, 218, 206, 20, 65, 214, 160, 12, 79, 195, 83, 69, 8, 38, 69, 190, + 198, 32, 65, 211, 86, 79, 4, 184, 209, 32, 3, 68, 73, 67, 227, 15, 84, 6, + 26, 73, 195, 210, 32, 65, 4, 246, 193, 33, 71, 231, 19, 78, 2, 233, 208, + 28, 2, 85, 69, 8, 26, 69, 151, 193, 33, 73, 6, 226, 205, 20, 83, 217, + 136, 8, 2, 76, 73, 22, 90, 69, 38, 85, 184, 161, 14, 2, 67, 72, 130, 172, + 6, 79, 22, 80, 34, 84, 131, 142, 12, 73, 4, 238, 164, 10, 67, 223, 239, + 21, 86, 6, 182, 206, 20, 80, 146, 128, 2, 73, 231, 155, 11, 78, 4, 246, + 233, 33, 67, 3, 82, 98, 116, 8, 65, 84, 65, 75, 65, 78, 65, 32, 133, 1, + 16, 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, 32, 94, + 166, 241, 10, 72, 2, 75, 2, 77, 2, 78, 2, 82, 2, 83, 2, 84, 126, 87, 46, + 89, 150, 246, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 152, 157, 16, 3, 74, + 85, 69, 197, 171, 17, 4, 67, 72, 65, 77, 106, 222, 196, 27, 65, 255, 140, + 4, 69, 4, 100, 19, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, 73, 79, 78, + 32, 83, 73, 71, 78, 32, 167, 221, 31, 73, 2, 209, 85, 4, 87, 73, 84, 72, + 98, 50, 69, 46, 70, 98, 83, 94, 84, 147, 202, 20, 78, 6, 180, 1, 3, 73, + 71, 72, 203, 193, 25, 76, 30, 28, 3, 73, 70, 84, 35, 79, 6, 214, 1, 89, + 155, 141, 32, 69, 24, 142, 2, 82, 251, 201, 20, 85, 8, 40, 4, 69, 86, 69, + 78, 1, 2, 73, 88, 4, 11, 84, 4, 104, 2, 89, 32, 143, 141, 32, 69, 52, 56, + 2, 69, 78, 32, 4, 72, 73, 82, 84, 29, 2, 87, 69, 5, 11, 32, 2, 139, 223, + 25, 79, 24, 74, 89, 175, 140, 32, 69, 24, 26, 78, 243, 229, 32, 76, 22, + 17, 2, 84, 89, 23, 11, 32, 20, 72, 2, 79, 78, 206, 238, 31, 70, 30, 83, + 42, 84, 142, 87, 78, 239, 112, 69, 4, 230, 221, 25, 32, 243, 131, 8, 69, + 2, 193, 199, 31, 8, 32, 80, 79, 73, 78, 84, 69, 68, 8, 248, 217, 17, 3, + 79, 83, 84, 190, 251, 6, 65, 228, 203, 2, 8, 69, 82, 80, 69, 78, 68, 73, + 67, 131, 183, 4, 76, 4, 200, 156, 30, 3, 73, 78, 71, 163, 170, 1, 69, 6, + 52, 7, 82, 73, 65, 78, 71, 76, 69, 143, 199, 26, 73, 5, 211, 166, 22, 32, + 6, 44, 5, 72, 73, 84, 69, 32, 231, 222, 33, 90, 4, 194, 252, 31, 66, 215, + 25, 83, 6, 154, 184, 27, 77, 184, 190, 4, 6, 76, 65, 84, 73, 79, 78, 201, + 164, 1, 3, 83, 32, 84, 5, 233, 150, 32, 6, 32, 65, 84, 32, 68, 85, 158, + 18, 192, 1, 24, 67, 79, 77, 80, 65, 84, 73, 66, 73, 76, 73, 84, 89, 32, + 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 144, 3, 8, 82, 65, 68, 73, 67, + 65, 76, 32, 245, 17, 7, 83, 84, 82, 79, 75, 69, 32, 236, 15, 24, 2, 50, + 70, 75, 70, 188, 8, 34, 65, 170, 130, 11, 56, 3, 57, 60, 202, 1, 49, 243, + 254, 18, 48, 176, 7, 26, 65, 235, 129, 11, 57, 176, 3, 134, 1, 54, 242, + 254, 18, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 2, 57, 2, + 65, 2, 66, 2, 67, 215, 156, 9, 68, 28, 226, 217, 33, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 3, 68, + 230, 1, 210, 1, 66, 126, 67, 226, 4, 68, 62, 69, 50, 70, 34, 71, 50, 72, + 86, 74, 154, 1, 76, 62, 77, 130, 1, 80, 46, 82, 78, 83, 182, 3, 84, 82, + 87, 164, 230, 10, 2, 78, 69, 156, 17, 4, 75, 78, 73, 70, 203, 160, 12, + 79, 14, 74, 79, 188, 134, 11, 4, 82, 85, 83, 72, 162, 240, 4, 65, 143, + 208, 13, 76, 6, 182, 173, 10, 76, 186, 146, 23, 78, 215, 22, 88, 56, 104, + 12, 45, 83, 73, 77, 80, 76, 73, 70, 73, 69, 68, 32, 162, 3, 73, 50, 76, + 186, 211, 32, 79, 191, 78, 72, 44, 114, 69, 38, 70, 50, 71, 38, 76, 34, + 83, 94, 84, 182, 217, 14, 87, 42, 68, 130, 244, 16, 66, 170, 121, 72, + 163, 4, 67, 4, 174, 254, 31, 86, 219, 185, 1, 65, 6, 134, 233, 31, 73, + 194, 217, 1, 76, 235, 16, 82, 4, 146, 164, 23, 79, 187, 230, 8, 65, 4, + 138, 210, 32, 69, 187, 48, 79, 10, 162, 230, 6, 73, 146, 199, 17, 65, + 194, 194, 1, 72, 128, 193, 7, 3, 80, 69, 69, 215, 11, 69, 6, 214, 10, 85, + 148, 188, 10, 2, 65, 78, 235, 164, 4, 79, 4, 128, 229, 20, 3, 86, 73, 76, + 191, 219, 12, 84, 4, 150, 233, 17, 73, 167, 210, 14, 79, 6, 240, 172, 24, + 2, 73, 86, 222, 240, 2, 69, 147, 179, 6, 79, 10, 196, 6, 2, 65, 84, 222, + 179, 33, 87, 3, 89, 4, 138, 174, 32, 73, 135, 82, 79, 8, 244, 5, 4, 82, + 65, 83, 83, 179, 220, 29, 72, 10, 48, 2, 69, 65, 226, 188, 28, 79, 211, + 130, 4, 65, 6, 186, 8, 82, 139, 199, 33, 68, 10, 72, 12, 45, 83, 73, 77, + 80, 76, 73, 70, 73, 69, 68, 32, 231, 187, 31, 65, 8, 42, 84, 202, 213, + 14, 68, 143, 134, 17, 69, 4, 194, 6, 85, 255, 224, 14, 79, 12, 128, 254, + 10, 3, 79, 78, 71, 205, 129, 16, 3, 65, 77, 69, 14, 18, 69, 35, 79, 4, + 142, 177, 33, 65, 159, 27, 83, 10, 144, 253, 10, 3, 85, 78, 68, 182, 174, + 13, 84, 138, 217, 7, 82, 239, 120, 79, 6, 196, 252, 10, 2, 65, 87, 239, + 203, 18, 69, 8, 34, 65, 197, 194, 32, 2, 69, 80, 6, 158, 252, 32, 73, + 226, 79, 77, 3, 80, 38, 122, 69, 90, 73, 186, 1, 78, 196, 1, 4, 80, 73, + 82, 73, 128, 247, 10, 4, 77, 65, 76, 76, 166, 158, 12, 72, 135, 226, 9, + 85, 8, 40, 4, 67, 79, 78, 68, 227, 190, 32, 65, 6, 11, 32, 6, 218, 215, + 31, 84, 155, 87, 79, 12, 60, 9, 77, 80, 76, 73, 70, 73, 69, 68, 32, 175, + 198, 33, 76, 10, 34, 72, 50, 87, 131, 194, 10, 89, 4, 136, 172, 10, 3, + 65, 76, 70, 179, 138, 18, 79, 4, 246, 219, 6, 65, 191, 227, 25, 72, 6, + 192, 1, 2, 79, 85, 179, 151, 33, 65, 8, 58, 85, 150, 241, 24, 65, 242, + 204, 1, 72, 191, 184, 3, 73, 2, 135, 159, 24, 82, 12, 26, 65, 49, 2, 69, + 83, 8, 172, 247, 10, 2, 76, 75, 1, 3, 84, 69, 82, 4, 255, 246, 10, 84, + 76, 110, 72, 182, 1, 80, 38, 83, 194, 198, 31, 84, 152, 253, 1, 2, 66, + 88, 2, 87, 2, 88, 86, 68, 2, 78, 3, 81, 31, 42, 80, 22, 88, 30, 90, 171, + 197, 33, 71, 5, 131, 197, 33, 87, 4, 238, 196, 33, 87, 87, 71, 19, 50, + 90, 246, 198, 31, 87, 130, 254, 1, 71, 3, 84, 9, 242, 198, 31, 90, 131, + 254, 1, 80, 9, 206, 196, 33, 68, 2, 71, 3, 90, 23, 50, 87, 30, 90, 222, + 195, 33, 71, 2, 80, 3, 84, 7, 246, 195, 33, 71, 3, 90, 9, 134, 195, 33, + 87, 86, 80, 3, 90, 124, 62, 65, 194, 1, 73, 114, 79, 181, 132, 21, 4, 69, + 65, 82, 32, 8, 132, 1, 2, 80, 80, 128, 217, 3, 6, 83, 83, 73, 67, 65, 76, + 129, 188, 28, 14, 77, 83, 72, 69, 76, 76, 32, 77, 79, 66, 73, 76, 69, 32, + 4, 180, 208, 19, 5, 73, 78, 71, 32, 72, 227, 228, 1, 69, 6, 48, 6, 78, + 75, 73, 78, 71, 32, 247, 167, 32, 80, 4, 194, 169, 25, 71, 149, 189, 3, + 6, 66, 69, 69, 82, 32, 77, 108, 72, 2, 67, 75, 140, 10, 2, 83, 69, 236, + 3, 2, 85, 68, 207, 187, 30, 87, 70, 64, 6, 32, 70, 65, 67, 69, 32, 237, + 2, 5, 87, 73, 83, 69, 32, 48, 58, 69, 46, 70, 36, 2, 78, 73, 2, 79, 18, + 83, 51, 84, 8, 120, 2, 76, 69, 125, 4, 73, 71, 72, 84, 8, 178, 1, 73, 25, + 3, 79, 85, 82, 4, 155, 1, 78, 8, 26, 69, 125, 2, 73, 88, 4, 57, 2, 86, + 69, 16, 38, 69, 14, 87, 41, 3, 72, 82, 69, 4, 63, 78, 8, 24, 2, 69, 76, + 27, 79, 4, 11, 86, 4, 11, 69, 4, 176, 153, 29, 2, 32, 79, 193, 179, 2, 3, + 45, 84, 72, 22, 90, 67, 58, 68, 122, 71, 48, 5, 82, 73, 71, 72, 84, 226, + 2, 84, 122, 79, 195, 172, 31, 73, 4, 196, 1, 3, 76, 79, 83, 181, 103, 4, + 79, 78, 84, 79, 2, 141, 3, 26, 79, 87, 78, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 85, 80, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 2, 21, 3, 65, 80, 80, 2, 133, 4, 2, 69, 68, 6, 228, 1, 14, 32, 65, 78, 68, 32, 76, 69, 70, 84, 32, 83, 69, 77, 73, 45, 38, 87, 65, 82, 68, 83, 32, 65, 78, 68, 32, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 80, 69, 78, 32, 67, - 73, 82, 67, 76, 69, 32, 65, 82, 82, 79, 87, 83, 2, 185, 182, 30, 6, 67, - 73, 82, 67, 76, 69, 5, 161, 83, 15, 32, 87, 73, 84, 72, 32, 67, 73, 82, + 73, 82, 67, 76, 69, 32, 65, 82, 82, 79, 87, 83, 2, 133, 217, 30, 6, 67, + 73, 82, 67, 76, 69, 5, 253, 85, 15, 32, 87, 73, 84, 72, 32, 67, 73, 82, 67, 76, 69, 68, 32, 79, 4, 82, 79, 37, 16, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, 65, 68, 69, 68, 32, 79, 2, 69, 6, 80, 32, 83, 69, 77, 73, 2, - 21, 3, 80, 69, 78, 2, 11, 32, 2, 217, 180, 30, 4, 67, 73, 82, 67, 26, 32, - 2, 68, 32, 191, 153, 32, 32, 24, 216, 1, 2, 83, 85, 66, 85, 178, 175, 19, - 77, 180, 3, 9, 76, 79, 67, 75, 32, 87, 73, 84, 72, 222, 134, 12, 66, 249, - 149, 1, 23, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, 78, 32, 87, 73, - 84, 72, 32, 83, 69, 82, 73, 70, 8, 30, 66, 1, 3, 80, 69, 82, 4, 181, 252, - 24, 3, 83, 69, 84, 6, 82, 77, 33, 16, 78, 73, 79, 78, 32, 87, 73, 84, 72, - 32, 83, 69, 82, 73, 70, 83, 2, 145, 223, 24, 3, 66, 82, 69, 5, 237, 252, - 31, 9, 32, 65, 78, 68, 32, 83, 77, 65, 83, 11, 33, 6, 32, 87, 73, 84, 72, - 32, 8, 72, 4, 84, 79, 82, 78, 214, 167, 5, 76, 206, 220, 19, 83, 135, - 218, 6, 82, 2, 203, 217, 25, 65, 226, 9, 158, 1, 67, 82, 76, 98, 77, 162, - 82, 78, 220, 5, 2, 79, 75, 58, 80, 150, 19, 82, 138, 1, 85, 170, 253, 27, - 87, 196, 158, 3, 2, 70, 70, 246, 47, 73, 183, 51, 65, 6, 26, 75, 203, - 148, 4, 79, 4, 142, 233, 8, 82, 189, 136, 23, 4, 84, 65, 73, 76, 8, 44, - 2, 79, 78, 165, 134, 25, 3, 76, 73, 83, 7, 11, 32, 4, 138, 128, 26, 69, - 195, 129, 6, 83, 158, 6, 72, 7, 66, 73, 78, 73, 78, 71, 32, 166, 80, 77, - 94, 80, 151, 154, 32, 69, 142, 6, 214, 2, 65, 174, 2, 66, 134, 1, 67, - 246, 12, 68, 174, 9, 69, 146, 2, 70, 54, 71, 140, 9, 2, 72, 79, 82, 73, - 216, 1, 2, 75, 65, 162, 1, 76, 230, 13, 77, 250, 1, 78, 78, 79, 138, 2, - 80, 174, 1, 82, 244, 4, 5, 90, 73, 71, 90, 65, 166, 2, 83, 242, 3, 84, - 238, 2, 85, 148, 1, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 36, 2, 87, 73, - 183, 1, 88, 24, 148, 1, 4, 67, 85, 84, 69, 86, 78, 252, 131, 25, 6, 83, - 84, 69, 82, 73, 83, 197, 224, 5, 14, 76, 77, 79, 83, 84, 32, 69, 81, 85, - 65, 76, 32, 84, 79, 10, 22, 45, 167, 33, 32, 4, 200, 204, 12, 6, 71, 82, - 65, 86, 69, 45, 199, 205, 3, 77, 6, 244, 2, 4, 84, 73, 67, 76, 189, 179, - 3, 4, 78, 85, 73, 84, 12, 42, 82, 201, 212, 30, 4, 73, 78, 68, 85, 10, - 48, 3, 69, 86, 69, 149, 227, 30, 3, 73, 68, 71, 7, 166, 164, 12, 45, 251, - 175, 18, 32, 142, 1, 142, 1, 65, 34, 76, 98, 79, 116, 8, 89, 82, 73, 76, - 76, 73, 67, 32, 184, 28, 11, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, - 195, 150, 12, 69, 6, 162, 18, 82, 215, 217, 28, 78, 4, 41, 8, 79, 67, 75, - 87, 73, 83, 69, 32, 4, 204, 171, 12, 4, 82, 73, 78, 71, 231, 218, 16, 65, - 10, 76, 4, 77, 77, 65, 32, 189, 16, 10, 78, 74, 79, 73, 78, 73, 78, 71, - 32, 77, 6, 210, 147, 13, 65, 239, 204, 17, 66, 116, 252, 1, 8, 72, 85, + 21, 3, 80, 69, 78, 2, 11, 32, 2, 165, 215, 30, 4, 67, 73, 82, 67, 26, 32, + 2, 68, 32, 219, 191, 32, 32, 24, 216, 1, 2, 83, 85, 66, 85, 162, 194, 19, + 77, 180, 3, 9, 76, 79, 67, 75, 32, 87, 73, 84, 72, 238, 151, 12, 66, 149, + 152, 1, 23, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 79, 78, 32, 87, 73, + 84, 72, 32, 83, 69, 82, 73, 70, 8, 30, 66, 1, 3, 80, 69, 82, 4, 185, 148, + 25, 3, 83, 69, 84, 6, 82, 77, 33, 16, 78, 73, 79, 78, 32, 87, 73, 84, 72, + 32, 83, 69, 82, 73, 70, 83, 2, 149, 247, 24, 3, 66, 82, 69, 5, 237, 160, + 32, 9, 32, 65, 78, 68, 32, 83, 77, 65, 83, 11, 33, 6, 32, 87, 73, 84, 72, + 32, 8, 72, 4, 84, 79, 82, 78, 190, 172, 5, 76, 250, 239, 19, 83, 251, + 229, 6, 82, 2, 211, 241, 25, 65, 152, 10, 158, 1, 67, 82, 76, 98, 77, + 202, 86, 78, 220, 5, 2, 79, 75, 58, 80, 154, 19, 82, 138, 1, 85, 246, + 149, 28, 87, 232, 167, 3, 2, 70, 70, 246, 47, 73, 183, 51, 65, 6, 26, 75, + 155, 153, 4, 79, 4, 202, 244, 8, 82, 153, 161, 23, 4, 84, 65, 73, 76, 8, + 44, 2, 79, 78, 185, 158, 25, 3, 76, 73, 83, 7, 11, 32, 4, 150, 152, 26, + 69, 211, 143, 6, 83, 212, 6, 72, 7, 66, 73, 78, 73, 78, 71, 32, 206, 84, + 77, 94, 80, 139, 188, 32, 69, 196, 6, 210, 2, 65, 186, 2, 66, 118, 67, + 246, 12, 68, 226, 10, 69, 142, 2, 70, 70, 71, 156, 9, 2, 72, 79, 82, 73, + 248, 1, 2, 75, 65, 162, 1, 76, 158, 14, 77, 178, 2, 78, 78, 79, 138, 2, + 80, 182, 1, 82, 248, 4, 5, 90, 73, 71, 90, 65, 138, 2, 83, 162, 4, 84, + 238, 2, 85, 168, 1, 8, 86, 69, 82, 84, 73, 67, 65, 76, 204, 1, 2, 87, 73, + 171, 1, 88, 26, 148, 1, 4, 67, 85, 84, 69, 98, 78, 136, 156, 25, 6, 83, + 84, 69, 82, 73, 83, 241, 234, 5, 14, 76, 77, 79, 83, 84, 32, 69, 81, 85, + 65, 76, 32, 84, 79, 12, 22, 45, 231, 34, 32, 6, 186, 57, 86, 184, 160, + 12, 6, 71, 82, 65, 86, 69, 45, 251, 210, 3, 77, 6, 228, 2, 4, 84, 73, 67, + 76, 237, 183, 3, 4, 78, 85, 73, 84, 12, 42, 82, 137, 247, 30, 4, 73, 78, + 68, 85, 10, 32, 3, 69, 86, 69, 243, 37, 73, 7, 210, 177, 12, 45, 159, + 197, 18, 32, 142, 1, 142, 1, 65, 34, 76, 98, 79, 116, 8, 89, 82, 73, 76, + 76, 73, 67, 32, 252, 29, 11, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, + 171, 162, 12, 69, 6, 222, 19, 82, 159, 245, 28, 78, 4, 41, 8, 79, 67, 75, + 87, 73, 83, 69, 32, 4, 248, 184, 12, 4, 82, 73, 78, 71, 139, 239, 16, 65, + 10, 76, 4, 77, 77, 65, 32, 249, 17, 10, 78, 74, 79, 73, 78, 73, 78, 71, + 32, 77, 6, 142, 161, 13, 65, 251, 225, 17, 66, 116, 252, 1, 8, 72, 85, 78, 68, 82, 69, 68, 32, 32, 7, 76, 69, 84, 84, 69, 82, 32, 166, 5, 80, - 116, 5, 68, 65, 83, 73, 65, 38, 84, 106, 77, 178, 243, 2, 75, 176, 138, - 13, 15, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 66, 89, 229, - 193, 16, 2, 86, 90, 4, 194, 7, 77, 195, 158, 3, 84, 84, 238, 1, 66, 38, - 68, 50, 69, 82, 73, 106, 79, 22, 83, 66, 85, 30, 89, 186, 137, 3, 76, - 140, 5, 5, 77, 79, 78, 79, 71, 150, 18, 72, 238, 226, 12, 84, 190, 243, - 15, 90, 150, 83, 67, 2, 71, 182, 8, 70, 134, 14, 80, 2, 86, 158, 20, 75, - 187, 2, 65, 4, 250, 161, 6, 73, 163, 216, 26, 69, 4, 248, 233, 25, 3, 74, - 69, 82, 255, 143, 7, 69, 14, 58, 83, 142, 249, 32, 70, 2, 76, 2, 77, 2, - 78, 3, 82, 5, 147, 178, 31, 45, 11, 56, 8, 79, 84, 73, 70, 73, 69, 68, - 32, 191, 248, 32, 69, 6, 142, 160, 6, 66, 174, 216, 26, 65, 3, 69, 5, - 159, 193, 25, 77, 6, 26, 72, 171, 133, 16, 79, 4, 158, 253, 30, 67, 195, - 250, 1, 65, 5, 173, 153, 3, 2, 75, 82, 8, 254, 189, 28, 69, 246, 156, 4, - 65, 174, 28, 73, 3, 85, 8, 66, 65, 48, 4, 83, 73, 76, 73, 189, 195, 29, - 4, 79, 75, 82, 89, 4, 142, 246, 2, 89, 133, 228, 12, 3, 76, 65, 84, 2, - 145, 13, 5, 32, 80, 78, 69, 85, 10, 80, 2, 69, 78, 0, 7, 72, 79, 85, 83, - 65, 78, 68, 177, 8, 4, 73, 84, 76, 79, 2, 17, 2, 32, 77, 2, 129, 235, 21, - 5, 73, 76, 76, 73, 79, 110, 62, 69, 244, 1, 8, 73, 65, 69, 82, 69, 83, - 73, 83, 39, 79, 38, 68, 9, 86, 65, 78, 65, 71, 65, 82, 73, 32, 145, 138, - 32, 2, 76, 69, 36, 92, 7, 76, 69, 84, 84, 69, 82, 32, 216, 131, 26, 6, - 83, 73, 71, 78, 32, 65, 227, 253, 4, 68, 14, 254, 222, 32, 86, 162, 17, - 75, 2, 78, 2, 80, 2, 82, 186, 2, 65, 3, 85, 7, 146, 229, 27, 45, 195, - 223, 2, 32, 66, 58, 84, 128, 1, 4, 85, 66, 76, 69, 241, 4, 2, 87, 78, 14, - 38, 32, 173, 139, 16, 3, 84, 69, 68, 10, 64, 5, 65, 66, 79, 86, 69, 197, - 179, 6, 5, 66, 69, 76, 79, 87, 7, 191, 197, 32, 32, 48, 22, 32, 191, 4, - 68, 46, 230, 1, 66, 0, 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 26, + 116, 5, 68, 65, 83, 73, 65, 38, 84, 106, 77, 230, 247, 2, 75, 220, 152, + 13, 15, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 66, 89, 169, + 213, 16, 2, 86, 90, 4, 194, 7, 77, 247, 162, 3, 84, 84, 238, 1, 66, 38, + 68, 50, 69, 82, 73, 106, 79, 22, 83, 66, 85, 30, 89, 238, 141, 3, 76, + 140, 5, 5, 77, 79, 78, 79, 71, 150, 18, 72, 154, 241, 12, 84, 130, 135, + 16, 90, 150, 83, 67, 2, 71, 182, 8, 70, 134, 14, 80, 2, 86, 158, 20, 75, + 187, 2, 65, 4, 142, 173, 6, 73, 179, 243, 26, 69, 4, 136, 130, 26, 3, 74, + 69, 82, 147, 158, 7, 69, 14, 58, 83, 178, 159, 33, 70, 2, 76, 2, 77, 2, + 78, 3, 82, 5, 155, 214, 31, 45, 11, 56, 8, 79, 84, 73, 70, 73, 69, 68, + 32, 227, 158, 33, 69, 6, 162, 171, 6, 66, 190, 243, 26, 65, 3, 69, 5, + 175, 217, 25, 77, 6, 26, 72, 139, 152, 16, 79, 4, 218, 160, 31, 67, 171, + 253, 1, 65, 5, 225, 157, 3, 2, 75, 82, 8, 198, 219, 28, 69, 210, 165, 4, + 65, 174, 28, 73, 3, 85, 8, 66, 65, 48, 4, 83, 73, 76, 73, 141, 230, 29, + 4, 79, 75, 82, 89, 4, 194, 250, 2, 89, 169, 242, 12, 3, 76, 65, 84, 2, + 209, 14, 5, 32, 80, 78, 69, 85, 10, 80, 2, 69, 78, 0, 7, 72, 79, 85, 83, + 65, 78, 68, 237, 9, 4, 73, 84, 76, 79, 2, 17, 2, 32, 77, 2, 201, 254, 21, + 5, 73, 76, 76, 73, 79, 124, 66, 69, 244, 1, 8, 73, 65, 69, 82, 69, 83, + 73, 83, 131, 1, 79, 38, 68, 9, 86, 65, 78, 65, 71, 65, 82, 73, 32, 177, + 176, 32, 2, 76, 69, 36, 92, 7, 76, 69, 84, 84, 69, 82, 32, 248, 155, 26, + 6, 83, 73, 71, 78, 32, 65, 251, 136, 5, 68, 14, 158, 133, 33, 86, 162, + 17, 75, 2, 78, 2, 80, 2, 82, 186, 2, 65, 3, 85, 9, 26, 32, 207, 130, 28, + 45, 4, 162, 246, 30, 66, 189, 142, 1, 16, 87, 73, 84, 72, 32, 82, 65, 73, + 83, 69, 68, 32, 76, 69, 70, 84, 78, 58, 84, 164, 1, 4, 85, 66, 76, 69, + 169, 5, 2, 87, 78, 16, 74, 32, 232, 186, 10, 5, 45, 65, 78, 68, 45, 161, + 226, 5, 3, 84, 69, 68, 10, 64, 5, 65, 66, 79, 86, 69, 217, 189, 6, 5, 66, + 69, 76, 79, 87, 7, 223, 234, 32, 32, 56, 22, 32, 247, 4, 68, 54, 222, 1, + 65, 34, 66, 0, 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 26, 67, 34, 77, 54, 79, 34, 80, 68, 2, 82, 73, 48, 5, 84, 73, 76, 68, 69, 80, 9, 86, - 69, 82, 84, 73, 67, 65, 76, 32, 198, 134, 16, 65, 42, 71, 150, 187, 4, - 76, 203, 172, 11, 67, 4, 229, 16, 2, 82, 69, 4, 21, 3, 65, 67, 82, 4, - 229, 245, 16, 2, 79, 78, 4, 230, 35, 80, 163, 226, 30, 86, 6, 238, 36, - 76, 253, 224, 7, 9, 65, 82, 69, 78, 84, 72, 69, 83, 69, 4, 156, 51, 4, - 71, 72, 84, 87, 207, 248, 9, 78, 7, 11, 32, 4, 44, 3, 76, 69, 70, 1, 4, - 82, 73, 71, 72, 2, 199, 203, 30, 84, 6, 198, 151, 12, 83, 151, 177, 14, - 76, 2, 165, 129, 16, 2, 32, 67, 4, 128, 41, 2, 32, 84, 243, 162, 31, 87, - 16, 76, 9, 78, 67, 76, 79, 83, 73, 78, 71, 32, 205, 152, 26, 4, 81, 85, - 65, 76, 14, 132, 1, 6, 67, 73, 82, 67, 76, 69, 22, 83, 164, 160, 6, 3, - 75, 69, 89, 228, 203, 19, 7, 85, 80, 87, 65, 82, 68, 32, 191, 192, 5, 68, - 5, 175, 180, 18, 32, 4, 214, 216, 25, 67, 131, 240, 5, 81, 4, 32, 2, 69, - 82, 159, 142, 5, 79, 2, 139, 74, 77, 126, 88, 17, 76, 65, 71, 79, 76, 73, - 84, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 143, 3, 82, 76, 238, 1, 68, - 30, 73, 46, 83, 50, 84, 130, 137, 6, 65, 22, 66, 94, 67, 134, 1, 70, 38, - 71, 238, 2, 77, 32, 2, 76, 74, 34, 78, 88, 2, 80, 79, 30, 82, 194, 2, 86, - 22, 89, 90, 90, 226, 132, 19, 72, 174, 128, 2, 85, 170, 168, 5, 79, 235, - 5, 75, 4, 186, 139, 6, 74, 31, 79, 11, 198, 140, 6, 78, 54, 79, 227, 171, - 26, 90, 8, 166, 141, 6, 77, 130, 4, 76, 211, 200, 22, 72, 4, 218, 145, 6, - 86, 219, 191, 26, 83, 50, 38, 65, 173, 3, 4, 69, 69, 75, 32, 36, 84, 5, - 78, 84, 72, 65, 32, 196, 1, 2, 86, 69, 133, 188, 28, 5, 80, 72, 69, 77, - 69, 24, 68, 6, 68, 73, 71, 73, 84, 32, 65, 7, 76, 69, 84, 84, 69, 82, 32, - 14, 250, 229, 16, 83, 130, 141, 14, 70, 70, 84, 62, 90, 251, 85, 79, 10, - 134, 207, 32, 86, 162, 17, 75, 2, 78, 2, 80, 187, 2, 65, 10, 18, 32, 67, - 45, 6, 26, 65, 231, 166, 16, 84, 4, 209, 144, 4, 4, 67, 67, 69, 78, 4, - 212, 132, 12, 6, 65, 67, 85, 84, 69, 45, 215, 243, 3, 77, 14, 148, 1, 8, - 77, 85, 83, 73, 67, 65, 76, 32, 242, 151, 4, 75, 242, 195, 2, 80, 174, 4, - 89, 225, 238, 3, 11, 68, 73, 65, 76, 89, 84, 73, 75, 65, 32, 84, 6, 26, - 84, 247, 186, 15, 80, 4, 206, 187, 15, 69, 35, 82, 6, 166, 246, 12, 79, - 152, 236, 18, 8, 77, 79, 84, 72, 69, 84, 73, 67, 167, 45, 82, 14, 26, 78, - 151, 177, 30, 83, 12, 52, 7, 86, 69, 82, 84, 69, 68, 32, 251, 181, 28, - 70, 10, 64, 2, 66, 82, 45, 10, 68, 79, 85, 66, 76, 69, 32, 65, 82, 67, 6, - 22, 69, 235, 37, 73, 4, 219, 221, 11, 86, 4, 247, 190, 30, 72, 8, 128, 1, - 16, 84, 65, 75, 65, 78, 65, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 229, - 233, 17, 9, 86, 89, 75, 65, 32, 65, 66, 79, 86, 4, 222, 129, 10, 83, 35, - 86, 158, 1, 80, 5, 65, 84, 73, 78, 32, 156, 8, 3, 69, 70, 84, 184, 3, 2, - 73, 71, 83, 79, 106, 156, 1, 21, 76, 69, 84, 84, 69, 82, 32, 83, 77, 65, - 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, 53, 13, 83, 77, 65, 76, 76, - 32, 76, 69, 84, 84, 69, 82, 32, 10, 130, 218, 32, 71, 2, 76, 2, 77, 2, - 78, 3, 82, 96, 226, 1, 65, 70, 67, 34, 69, 30, 70, 78, 73, 86, 76, 106, - 79, 2, 85, 134, 1, 82, 50, 84, 226, 171, 12, 83, 182, 146, 2, 66, 138, - 157, 2, 87, 150, 248, 15, 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 78, 2, 80, - 2, 86, 2, 88, 3, 90, 13, 242, 243, 2, 32, 198, 139, 26, 76, 182, 216, 3, - 69, 2, 79, 3, 86, 5, 241, 139, 12, 3, 32, 67, 69, 7, 246, 213, 32, 83, 3, - 84, 5, 249, 179, 27, 14, 76, 65, 84, 84, 69, 78, 69, 68, 32, 79, 80, 69, - 78, 32, 11, 37, 7, 78, 83, 85, 76, 65, 82, 32, 8, 246, 213, 32, 68, 2, - 71, 2, 82, 3, 84, 7, 164, 134, 16, 14, 32, 87, 73, 84, 72, 32, 68, 79, - 85, 66, 76, 69, 32, 77, 149, 218, 12, 3, 79, 78, 71, 7, 33, 6, 32, 87, - 73, 84, 72, 32, 4, 214, 155, 12, 68, 225, 222, 12, 15, 76, 73, 71, 72, - 84, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 7, 11, 32, 4, 150, 250, 11, - 82, 175, 187, 18, 66, 5, 129, 26, 6, 85, 82, 78, 69, 68, 32, 32, 46, 32, - 213, 2, 6, 87, 65, 82, 68, 83, 32, 28, 154, 1, 65, 112, 12, 80, 65, 82, - 69, 78, 84, 72, 69, 83, 73, 83, 32, 226, 12, 72, 182, 1, 84, 129, 164, - 30, 11, 82, 73, 71, 72, 84, 32, 65, 82, 82, 79, 87, 12, 48, 4, 82, 82, - 79, 87, 185, 178, 30, 2, 78, 71, 8, 26, 72, 195, 178, 30, 32, 4, 177, - 178, 30, 3, 69, 65, 68, 4, 164, 223, 7, 4, 65, 66, 79, 86, 225, 190, 24, - 5, 66, 69, 76, 79, 87, 4, 230, 13, 72, 153, 238, 11, 5, 65, 82, 82, 79, - 87, 10, 52, 6, 65, 84, 85, 82, 69, 32, 169, 17, 2, 72, 84, 8, 238, 1, 76, - 23, 82, 10, 36, 3, 78, 71, 32, 139, 181, 31, 87, 8, 172, 7, 7, 68, 79, - 85, 66, 76, 69, 32, 246, 7, 83, 71, 86, 20, 52, 5, 65, 67, 82, 79, 78, - 221, 251, 25, 2, 73, 78, 19, 18, 32, 127, 45, 10, 34, 76, 22, 82, 135, - 175, 30, 66, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 229, 212, 16, 6, - 84, 32, 72, 65, 76, 70, 6, 186, 158, 12, 71, 186, 2, 65, 135, 169, 3, 66, - 4, 152, 9, 9, 85, 77, 66, 69, 82, 32, 83, 73, 71, 181, 213, 11, 2, 79, - 84, 18, 136, 1, 17, 76, 68, 32, 80, 69, 82, 77, 73, 67, 32, 76, 69, 84, - 84, 69, 82, 32, 82, 80, 140, 156, 12, 4, 71, 79, 78, 69, 151, 198, 18, - 86, 10, 242, 43, 90, 202, 148, 18, 78, 222, 155, 9, 68, 182, 171, 2, 83, - 159, 243, 1, 65, 2, 169, 203, 24, 6, 69, 78, 32, 77, 65, 82, 12, 18, 65, - 107, 76, 8, 236, 7, 9, 82, 69, 78, 84, 72, 69, 83, 69, 83, 225, 194, 24, - 9, 76, 65, 84, 65, 76, 73, 90, 69, 68, 4, 149, 170, 30, 7, 85, 83, 32, - 83, 73, 71, 78, 38, 18, 69, 127, 73, 6, 72, 5, 86, 69, 82, 83, 69, 217, - 200, 24, 7, 84, 82, 79, 70, 76, 69, 88, 4, 22, 32, 227, 12, 68, 2, 141, - 8, 2, 83, 79, 32, 40, 3, 71, 72, 84, 157, 5, 2, 78, 71, 26, 50, 32, 149, - 4, 7, 87, 65, 82, 68, 83, 32, 72, 24, 104, 5, 65, 82, 82, 79, 87, 218, 1, - 72, 124, 12, 80, 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 32, 59, 84, 12, - 44, 5, 72, 69, 65, 68, 32, 235, 166, 30, 32, 8, 26, 65, 235, 166, 30, 66, - 6, 36, 3, 78, 68, 32, 243, 199, 31, 66, 4, 40, 4, 68, 79, 87, 78, 1, 2, - 85, 80, 2, 133, 223, 8, 9, 32, 65, 82, 82, 79, 87, 72, 69, 65, 6, 11, 65, - 6, 48, 6, 76, 70, 32, 82, 73, 78, 21, 2, 82, 80, 4, 243, 164, 30, 71, 2, - 17, 2, 79, 79, 2, 255, 197, 31, 78, 4, 250, 218, 9, 65, 241, 150, 21, 5, - 66, 69, 76, 79, 87, 2, 233, 195, 24, 2, 65, 67, 2, 141, 144, 21, 16, 65, - 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 6, 11, 32, 6, - 202, 237, 11, 79, 226, 181, 18, 66, 167, 161, 1, 65, 18, 188, 1, 5, 72, - 79, 82, 84, 32, 160, 1, 7, 81, 85, 65, 82, 69, 32, 66, 56, 5, 84, 82, 79, - 78, 71, 168, 193, 3, 2, 85, 83, 168, 254, 20, 2, 78, 65, 221, 208, 5, 6, - 69, 65, 71, 85, 76, 76, 6, 18, 83, 71, 86, 4, 26, 79, 243, 234, 11, 84, - 2, 145, 235, 11, 5, 76, 73, 68, 85, 83, 2, 49, 10, 69, 82, 84, 73, 67, - 65, 76, 32, 76, 73, 2, 171, 234, 11, 78, 4, 196, 214, 7, 5, 82, 65, 67, - 75, 69, 223, 201, 22, 69, 2, 181, 191, 24, 17, 32, 67, 69, 78, 84, 82, - 65, 76, 73, 90, 65, 84, 73, 79, 78, 32, 83, 20, 98, 72, 32, 4, 73, 76, - 68, 69, 136, 1, 6, 82, 73, 80, 76, 69, 32, 69, 5, 85, 82, 78, 69, 68, 2, - 185, 212, 7, 3, 82, 69, 69, 11, 11, 32, 8, 76, 3, 76, 69, 70, 0, 4, 82, - 73, 71, 72, 202, 231, 11, 79, 227, 181, 18, 66, 2, 241, 141, 30, 6, 84, - 32, 72, 65, 76, 70, 6, 210, 213, 15, 65, 176, 212, 15, 5, 85, 78, 68, 69, - 82, 211, 64, 68, 2, 161, 176, 15, 2, 32, 67, 10, 34, 80, 134, 189, 31, - 82, 3, 83, 6, 42, 87, 225, 186, 24, 4, 32, 84, 65, 67, 2, 45, 9, 65, 82, - 68, 83, 32, 65, 82, 82, 79, 2, 231, 139, 30, 87, 6, 246, 149, 26, 76, - 199, 191, 2, 84, 6, 52, 3, 68, 69, 32, 153, 229, 11, 4, 71, 71, 76, 89, - 4, 92, 12, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 82, 73, 169, 148, 29, - 5, 66, 82, 73, 68, 71, 2, 197, 185, 24, 2, 68, 71, 6, 198, 236, 11, 45, - 207, 172, 18, 32, 6, 52, 7, 69, 82, 67, 73, 65, 76, 32, 231, 182, 32, 65, - 4, 134, 136, 29, 77, 179, 146, 3, 65, 8, 138, 159, 8, 82, 140, 149, 16, - 3, 79, 83, 73, 190, 239, 2, 76, 159, 252, 3, 65, 38, 214, 1, 70, 84, 10, - 83, 84, 82, 85, 67, 84, 73, 79, 78, 32, 46, 84, 252, 235, 2, 8, 86, 69, - 78, 73, 69, 78, 67, 69, 180, 180, 18, 6, 73, 67, 65, 76, 32, 84, 246, - 173, 9, 74, 241, 99, 7, 71, 82, 85, 69, 78, 84, 32, 6, 228, 147, 27, 4, - 69, 84, 84, 73, 158, 237, 1, 85, 189, 162, 2, 4, 79, 85, 78, 68, 4, 164, - 206, 20, 2, 87, 79, 183, 222, 10, 83, 20, 80, 5, 65, 73, 78, 83, 32, 198, - 1, 79, 28, 4, 82, 79, 76, 32, 143, 146, 3, 73, 12, 48, 3, 65, 83, 32, 93, - 5, 87, 73, 84, 72, 32, 6, 222, 152, 24, 77, 181, 8, 15, 78, 79, 82, 77, - 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 6, 186, 195, 4, 76, 174, 214, - 19, 86, 171, 233, 4, 79, 2, 217, 169, 30, 2, 85, 82, 4, 188, 236, 19, 3, - 75, 78, 79, 165, 129, 4, 8, 83, 69, 81, 85, 69, 78, 67, 69, 6, 26, 73, - 143, 158, 2, 69, 4, 162, 175, 32, 78, 87, 69, 206, 2, 36, 4, 84, 73, 67, - 32, 171, 18, 89, 202, 2, 186, 1, 67, 204, 1, 6, 69, 80, 65, 67, 84, 32, - 86, 70, 36, 11, 79, 76, 68, 32, 78, 85, 66, 73, 65, 78, 32, 110, 83, 181, - 244, 28, 13, 77, 79, 82, 80, 72, 79, 76, 79, 71, 73, 67, 65, 76, 126, 76, - 9, 79, 77, 66, 73, 78, 73, 78, 71, 32, 161, 3, 5, 65, 80, 73, 84, 65, 6, - 68, 9, 83, 80, 73, 82, 73, 84, 85, 83, 32, 213, 175, 31, 2, 78, 73, 4, - 152, 149, 6, 2, 76, 69, 173, 131, 17, 2, 65, 83, 56, 134, 173, 21, 78, - 230, 193, 2, 68, 145, 238, 6, 8, 84, 72, 79, 85, 83, 65, 78, 68, 4, 146, - 173, 18, 82, 231, 172, 11, 85, 8, 58, 68, 0, 3, 73, 78, 68, 142, 15, 86, - 131, 202, 29, 70, 2, 217, 217, 29, 7, 73, 82, 69, 67, 84, 32, 81, 134, 1, - 56, 3, 77, 65, 76, 193, 11, 6, 89, 77, 66, 79, 76, 32, 120, 45, 9, 76, - 32, 76, 69, 84, 84, 69, 82, 32, 120, 138, 2, 65, 48, 6, 66, 79, 72, 65, - 73, 82, 32, 2, 67, 82, 170, 1, 68, 160, 1, 2, 71, 65, 34, 72, 38, 75, 46, - 70, 34, 76, 66, 79, 134, 3, 83, 90, 84, 46, 90, 230, 211, 5, 86, 214, 54, - 80, 190, 210, 3, 73, 238, 162, 22, 82, 198, 3, 69, 218, 7, 77, 2, 78, - 163, 17, 85, 4, 44, 5, 75, 72, 77, 73, 77, 223, 193, 19, 76, 2, 177, 1, - 4, 73, 67, 32, 75, 10, 92, 12, 89, 80, 84, 79, 71, 82, 65, 77, 77, 73, - 67, 32, 53, 7, 79, 83, 83, 69, 68, 32, 83, 8, 50, 83, 226, 4, 71, 210, - 133, 32, 69, 219, 7, 78, 2, 207, 220, 24, 72, 12, 80, 9, 73, 65, 76, 69, - 67, 84, 45, 80, 32, 244, 255, 31, 2, 65, 76, 175, 17, 69, 8, 182, 156, 8, - 72, 250, 150, 8, 65, 244, 183, 12, 2, 75, 65, 255, 165, 3, 78, 4, 190, 3, - 78, 223, 255, 31, 77, 4, 190, 198, 25, 79, 215, 150, 5, 65, 8, 42, 72, - 234, 233, 28, 65, 255, 165, 3, 83, 4, 226, 143, 32, 69, 219, 19, 73, 4, - 176, 191, 19, 7, 45, 83, 72, 65, 80, 69, 68, 155, 241, 7, 65, 35, 36, 3, - 76, 68, 32, 207, 253, 31, 79, 30, 76, 7, 67, 79, 80, 84, 73, 67, 32, 189, - 1, 7, 78, 85, 66, 73, 65, 78, 32, 22, 98, 71, 42, 72, 184, 1, 2, 83, 72, - 154, 222, 14, 68, 142, 248, 9, 79, 222, 224, 5, 69, 207, 104, 65, 2, 17, - 2, 65, 78, 2, 183, 251, 15, 71, 8, 138, 195, 25, 79, 238, 125, 65, 139, - 204, 5, 69, 8, 50, 78, 228, 177, 16, 2, 83, 72, 155, 158, 5, 87, 4, 154, - 140, 32, 71, 3, 89, 10, 54, 72, 138, 151, 6, 65, 178, 227, 25, 79, 219, - 3, 73, 4, 142, 254, 31, 73, 187, 13, 69, 4, 240, 241, 31, 3, 72, 69, 84, - 167, 8, 65, 2, 135, 250, 31, 65, 14, 62, 75, 30, 77, 2, 80, 22, 83, 241, - 234, 27, 3, 84, 65, 85, 4, 26, 72, 143, 138, 32, 65, 2, 131, 235, 27, 73, - 4, 188, 175, 16, 6, 72, 73, 77, 65, 32, 83, 193, 254, 3, 3, 84, 65, 85, - 4, 188, 200, 24, 3, 76, 69, 70, 245, 145, 2, 4, 82, 73, 71, 72, 6, 96, 6, - 78, 73, 83, 72, 32, 86, 204, 200, 25, 8, 82, 69, 83, 80, 79, 78, 68, 83, - 231, 201, 5, 65, 2, 213, 229, 28, 4, 69, 82, 83, 69, 44, 104, 2, 78, 84, - 156, 132, 3, 6, 67, 72, 32, 65, 78, 68, 253, 142, 28, 8, 80, 76, 69, 32, - 87, 73, 84, 72, 40, 56, 2, 69, 82, 37, 8, 73, 78, 71, 32, 82, 79, 68, 32, - 4, 190, 181, 27, 66, 195, 131, 1, 83, 36, 48, 4, 84, 69, 78, 83, 1, 4, - 85, 78, 73, 84, 18, 133, 220, 23, 2, 32, 68, 51, 82, 69, 76, 5, 73, 67, - 75, 69, 84, 34, 79, 198, 5, 85, 50, 89, 143, 214, 30, 65, 4, 196, 168, 8, - 3, 68, 73, 84, 181, 201, 20, 7, 83, 67, 69, 78, 84, 32, 77, 5, 169, 247, - 26, 3, 32, 66, 65, 28, 84, 2, 83, 83, 152, 242, 29, 3, 67, 79, 68, 152, - 155, 1, 3, 73, 83, 83, 255, 57, 87, 22, 46, 32, 184, 2, 3, 69, 68, 32, - 223, 1, 73, 14, 44, 3, 79, 70, 32, 82, 80, 179, 212, 31, 77, 4, 224, 172, - 28, 6, 74, 69, 82, 85, 83, 65, 145, 208, 2, 5, 76, 79, 82, 82, 65, 8, 76, - 10, 65, 84, 84, 89, 32, 87, 73, 84, 72, 32, 41, 5, 79, 77, 77, 69, 69, 4, - 218, 165, 10, 82, 25, 3, 76, 69, 70, 5, 189, 161, 3, 11, 32, 87, 73, 84, - 72, 32, 72, 65, 76, 70, 45, 6, 204, 132, 25, 37, 78, 69, 71, 65, 84, 73, - 86, 69, 32, 83, 81, 85, 65, 82, 69, 68, 32, 76, 65, 84, 73, 78, 32, 67, - 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 192, 221, 2, 2, 70, - 76, 189, 210, 2, 3, 83, 87, 79, 2, 145, 235, 23, 5, 78, 71, 32, 76, 65, - 4, 248, 199, 17, 3, 90, 69, 73, 255, 167, 14, 84, 6, 130, 220, 19, 73, - 189, 149, 7, 4, 83, 84, 65, 76, 206, 19, 154, 1, 66, 20, 8, 78, 69, 73, - 70, 79, 82, 77, 32, 154, 250, 1, 80, 114, 82, 180, 3, 2, 83, 84, 242, - 219, 15, 67, 161, 229, 9, 6, 84, 32, 79, 70, 32, 77, 2, 211, 175, 5, 69, - 170, 19, 176, 1, 13, 78, 85, 77, 69, 82, 73, 67, 32, 83, 73, 71, 78, 32, - 184, 20, 17, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 83, 73, 71, - 78, 32, 233, 1, 5, 83, 73, 71, 78, 32, 222, 1, 78, 69, 166, 2, 70, 152, - 3, 2, 78, 73, 154, 2, 79, 130, 3, 83, 151, 4, 84, 24, 68, 5, 73, 71, 72, - 84, 32, 153, 1, 7, 76, 65, 77, 73, 84, 69, 32, 16, 142, 14, 83, 230, 113, - 85, 238, 50, 71, 136, 56, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, - 77, 32, 85, 83, 83, 85, 214, 193, 26, 68, 219, 248, 1, 65, 8, 232, 140, - 21, 5, 79, 78, 69, 32, 84, 246, 144, 9, 70, 175, 14, 84, 58, 48, 4, 73, - 86, 69, 32, 125, 4, 79, 85, 82, 32, 26, 70, 83, 206, 1, 66, 250, 12, 65, - 82, 71, 138, 110, 85, 203, 172, 27, 68, 6, 182, 15, 72, 173, 144, 19, 5, - 73, 88, 84, 72, 83, 32, 150, 1, 66, 44, 18, 86, 65, 82, 73, 65, 78, 84, - 32, 70, 79, 82, 77, 32, 76, 73, 77, 77, 85, 206, 12, 65, 82, 71, 26, 83, - 242, 109, 85, 203, 172, 27, 68, 6, 204, 123, 3, 65, 78, 50, 163, 212, 26, - 85, 9, 214, 202, 12, 32, 143, 190, 19, 52, 24, 44, 4, 71, 73, 68, 65, 33, - 3, 78, 69, 32, 4, 238, 159, 30, 69, 207, 104, 77, 20, 156, 1, 19, 86, 65, - 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 76, 73, 77, 77, 85, 174, - 7, 83, 230, 113, 85, 238, 50, 71, 222, 249, 26, 68, 219, 248, 1, 65, 9, - 150, 132, 32, 32, 186, 2, 51, 3, 52, 28, 92, 16, 76, 68, 32, 65, 83, 83, - 89, 82, 73, 65, 78, 32, 79, 78, 69, 32, 37, 3, 78, 69, 32, 4, 154, 194, - 18, 83, 175, 192, 11, 81, 24, 166, 1, 69, 52, 8, 81, 85, 65, 82, 84, 69, - 82, 32, 206, 7, 66, 54, 71, 84, 5, 84, 72, 73, 82, 68, 200, 194, 24, 2, - 83, 72, 217, 191, 6, 6, 72, 65, 76, 70, 32, 71, 4, 158, 8, 83, 205, 164, - 1, 5, 73, 71, 72, 84, 72, 4, 166, 155, 30, 65, 207, 111, 71, 38, 144, 1, - 5, 69, 86, 69, 78, 32, 188, 1, 20, 72, 65, 82, 50, 32, 84, 73, 77, 69, - 83, 32, 71, 65, 76, 32, 80, 76, 85, 83, 32, 37, 3, 73, 88, 32, 18, 148, - 1, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 77, 73, - 78, 218, 1, 83, 230, 113, 85, 238, 50, 71, 222, 249, 26, 68, 219, 248, 1, - 65, 6, 238, 194, 12, 32, 143, 190, 19, 51, 4, 222, 159, 28, 68, 167, 225, - 2, 77, 16, 142, 1, 83, 142, 3, 65, 218, 110, 85, 238, 50, 71, 144, 229, - 26, 16, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 65, 83, 72, - 207, 20, 68, 2, 227, 61, 72, 50, 52, 5, 72, 82, 69, 69, 32, 241, 1, 3, - 87, 79, 32, 28, 142, 1, 66, 40, 4, 83, 72, 65, 82, 24, 16, 86, 65, 82, - 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 69, 83, 72, 118, 65, 82, 71, 211, - 154, 28, 68, 6, 136, 112, 3, 85, 82, 85, 199, 10, 65, 8, 226, 111, 50, 3, - 85, 4, 130, 224, 24, 49, 203, 3, 50, 22, 82, 65, 30, 66, 32, 2, 69, 83, - 22, 71, 26, 83, 61, 6, 84, 72, 73, 82, 68, 83, 4, 225, 184, 1, 2, 83, 72, - 4, 254, 120, 65, 223, 201, 26, 85, 2, 247, 226, 24, 72, 4, 53, 3, 69, 83, - 72, 4, 11, 72, 4, 17, 2, 65, 82, 4, 142, 251, 31, 50, 3, 85, 4, 11, 32, - 4, 196, 186, 27, 12, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 171, - 95, 68, 10, 160, 1, 9, 68, 73, 65, 71, 79, 78, 65, 76, 32, 204, 203, 25, - 6, 86, 69, 82, 84, 73, 67, 133, 247, 2, 14, 79, 76, 68, 32, 65, 83, 83, - 89, 82, 73, 65, 78, 32, 87, 6, 192, 173, 28, 2, 84, 82, 160, 251, 2, 4, - 81, 85, 65, 68, 15, 67, 194, 17, 202, 1, 65, 186, 15, 66, 170, 5, 68, - 190, 15, 69, 202, 10, 71, 182, 30, 72, 238, 3, 73, 238, 4, 75, 154, 20, - 76, 134, 37, 77, 134, 6, 78, 226, 17, 80, 210, 3, 82, 42, 83, 238, 19, - 84, 186, 8, 85, 207, 20, 90, 141, 1, 160, 1, 7, 32, 84, 73, 77, 69, 83, - 32, 142, 1, 66, 250, 3, 68, 38, 75, 90, 76, 212, 1, 3, 77, 65, 82, 78, - 78, 134, 2, 82, 42, 83, 154, 235, 30, 80, 215, 127, 50, 16, 144, 173, 1, - 5, 76, 65, 71, 65, 82, 226, 23, 71, 246, 239, 12, 73, 154, 182, 13, 77, - 182, 161, 2, 83, 218, 162, 1, 66, 246, 67, 72, 187, 2, 65, 45, 22, 32, - 219, 2, 50, 28, 48, 6, 84, 73, 77, 69, 83, 32, 139, 203, 1, 71, 26, 180, - 1, 2, 71, 65, 38, 73, 36, 2, 83, 72, 128, 113, 8, 85, 32, 80, 76, 85, 83, - 32, 85, 188, 56, 4, 68, 85, 78, 51, 166, 2, 65, 204, 2, 3, 78, 85, 78, - 242, 27, 76, 143, 165, 30, 72, 4, 194, 210, 1, 78, 231, 159, 30, 76, 4, - 210, 169, 1, 71, 223, 200, 29, 77, 4, 178, 216, 24, 85, 143, 177, 5, 69, - 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, 206, 133, 1, 77, 130, 59, 71, - 194, 9, 83, 234, 10, 84, 140, 179, 28, 2, 66, 65, 147, 233, 1, 65, 5, - 249, 51, 5, 32, 84, 73, 77, 69, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, - 252, 27, 5, 83, 72, 73, 84, 65, 215, 80, 69, 23, 68, 7, 32, 84, 73, 77, - 69, 83, 32, 218, 202, 25, 69, 187, 212, 5, 65, 16, 102, 75, 156, 173, 1, - 2, 68, 73, 246, 223, 26, 71, 218, 248, 1, 85, 238, 94, 65, 170, 92, 83, - 215, 42, 72, 4, 130, 158, 1, 65, 135, 208, 30, 73, 7, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 4, 210, 253, 3, 75, 223, 194, 27, 83, 13, 26, 32, 243, - 191, 31, 83, 8, 128, 1, 10, 80, 76, 85, 83, 32, 78, 65, 71, 65, 32, 128, - 185, 8, 7, 84, 72, 82, 69, 69, 32, 84, 177, 252, 5, 4, 79, 86, 69, 82, 4, - 144, 140, 1, 16, 79, 80, 80, 79, 83, 73, 78, 71, 32, 65, 78, 32, 80, 76, - 85, 83, 251, 175, 23, 83, 6, 200, 49, 2, 65, 68, 239, 151, 18, 75, 18, - 26, 72, 131, 164, 12, 65, 17, 42, 32, 202, 200, 18, 71, 223, 161, 13, 50, - 10, 68, 9, 79, 86, 69, 82, 32, 65, 83, 72, 32, 174, 97, 90, 135, 112, 75, - 6, 176, 1, 8, 79, 86, 69, 82, 32, 65, 83, 72, 157, 143, 1, 29, 84, 85, - 71, 50, 32, 79, 86, 69, 82, 32, 84, 85, 71, 50, 32, 84, 85, 71, 50, 32, - 79, 86, 69, 82, 32, 84, 85, 71, 50, 5, 149, 145, 1, 27, 32, 67, 82, 79, - 83, 83, 73, 78, 71, 32, 65, 83, 72, 32, 79, 86, 69, 82, 32, 65, 83, 72, - 32, 79, 86, 69, 82, 52, 30, 65, 158, 2, 73, 95, 85, 27, 66, 68, 44, 4, - 72, 65, 82, 50, 90, 76, 66, 82, 251, 238, 27, 71, 5, 145, 208, 1, 6, 32, - 84, 73, 77, 69, 83, 9, 37, 7, 32, 84, 73, 77, 69, 83, 32, 6, 238, 164, 1, - 65, 170, 173, 30, 78, 163, 17, 90, 7, 252, 219, 30, 7, 32, 79, 86, 69, - 82, 32, 66, 239, 136, 1, 65, 5, 211, 203, 24, 65, 9, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 6, 134, 156, 1, 73, 134, 167, 29, 71, 187, 161, 1, 65, - 19, 50, 32, 168, 1, 3, 76, 85, 71, 239, 169, 1, 82, 8, 88, 8, 79, 86, 69, - 82, 32, 66, 85, 32, 129, 183, 27, 8, 67, 82, 79, 83, 83, 73, 78, 71, 6, - 164, 198, 12, 7, 84, 73, 77, 69, 83, 32, 78, 246, 224, 17, 65, 135, 108, - 85, 5, 189, 242, 3, 8, 32, 79, 86, 69, 82, 32, 66, 85, 180, 1, 34, 65, - 222, 5, 73, 195, 2, 85, 67, 50, 71, 150, 5, 82, 146, 49, 32, 131, 171, - 31, 77, 55, 26, 32, 139, 225, 31, 51, 50, 76, 13, 75, 73, 83, 73, 77, 53, - 32, 84, 73, 77, 69, 83, 32, 227, 198, 1, 84, 48, 146, 1, 65, 30, 66, 38, - 71, 112, 2, 73, 82, 42, 76, 94, 85, 182, 131, 1, 80, 162, 61, 84, 218, - 230, 27, 75, 242, 158, 2, 78, 254, 2, 83, 163, 17, 72, 4, 110, 32, 255, - 188, 30, 77, 4, 246, 245, 29, 65, 147, 233, 1, 73, 10, 34, 65, 58, 73, - 235, 151, 31, 85, 5, 11, 32, 2, 133, 246, 29, 6, 80, 76, 85, 83, 32, 77, - 5, 219, 196, 24, 82, 5, 229, 255, 13, 5, 32, 80, 76, 85, 83, 8, 26, 85, - 179, 221, 31, 65, 7, 160, 151, 1, 7, 32, 80, 76, 85, 83, 32, 77, 143, - 198, 30, 77, 6, 52, 7, 50, 32, 80, 76, 85, 83, 32, 171, 219, 31, 83, 4, - 128, 27, 2, 71, 73, 139, 217, 29, 77, 7, 191, 193, 12, 65, 25, 54, 77, - 150, 1, 78, 76, 2, 83, 72, 231, 217, 31, 66, 13, 44, 7, 32, 84, 73, 77, - 69, 83, 32, 59, 50, 6, 220, 69, 2, 85, 32, 230, 212, 13, 73, 199, 147, - 17, 83, 5, 221, 152, 13, 6, 32, 84, 73, 77, 69, 83, 5, 181, 239, 18, 14, - 32, 75, 65, 83, 75, 65, 76, 32, 85, 32, 71, 85, 78, 85, 5, 245, 157, 1, - 5, 32, 80, 76, 85, 83, 91, 70, 32, 66, 66, 94, 71, 234, 4, 78, 198, 185, - 24, 82, 175, 153, 7, 72, 6, 154, 176, 1, 71, 170, 3, 83, 153, 197, 12, 4, - 79, 86, 69, 82, 9, 52, 7, 32, 84, 73, 77, 69, 83, 32, 255, 215, 31, 50, - 4, 250, 145, 1, 69, 247, 152, 30, 83, 61, 52, 7, 32, 84, 73, 77, 69, 83, - 32, 131, 145, 31, 85, 56, 122, 65, 58, 68, 30, 71, 74, 75, 90, 76, 110, - 77, 50, 83, 130, 80, 69, 218, 58, 73, 190, 198, 16, 72, 170, 238, 13, 78, - 3, 80, 6, 32, 2, 83, 72, 235, 168, 31, 78, 5, 215, 206, 14, 32, 4, 138, - 134, 31, 73, 3, 85, 8, 26, 73, 183, 213, 31, 65, 7, 140, 141, 1, 2, 82, - 50, 151, 199, 30, 83, 8, 26, 85, 211, 185, 1, 65, 6, 40, 4, 83, 72, 85, - 50, 195, 212, 31, 82, 5, 175, 26, 32, 8, 26, 65, 45, 2, 85, 72, 6, 202, - 26, 77, 205, 158, 24, 3, 75, 45, 48, 2, 225, 57, 5, 32, 80, 76, 85, 83, - 6, 162, 235, 29, 65, 198, 170, 1, 69, 223, 61, 73, 4, 238, 138, 1, 73, - 211, 177, 30, 72, 11, 26, 51, 215, 210, 31, 52, 7, 143, 9, 32, 117, 130, - 1, 32, 90, 50, 210, 1, 78, 202, 1, 82, 60, 3, 83, 72, 50, 52, 3, 90, 69, - 78, 170, 225, 18, 71, 230, 235, 11, 68, 215, 127, 76, 4, 168, 25, 11, 79, - 86, 69, 82, 32, 69, 32, 78, 85, 78, 32, 253, 94, 4, 84, 73, 77, 69, 19, - 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 134, 1, 83, 144, 168, 1, 10, 65, - 32, 80, 76, 85, 83, 32, 72, 65, 32, 242, 133, 29, 71, 206, 16, 80, 154, - 24, 75, 254, 100, 77, 219, 19, 85, 4, 130, 198, 30, 65, 227, 114, 72, 15, - 11, 32, 12, 96, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 36, 6, 84, 73, - 77, 69, 83, 32, 155, 158, 24, 83, 2, 237, 146, 14, 4, 83, 73, 78, 71, 6, - 204, 138, 1, 4, 71, 65, 78, 50, 227, 172, 30, 77, 6, 36, 3, 73, 78, 50, - 203, 253, 30, 69, 5, 131, 193, 30, 32, 5, 229, 16, 9, 32, 67, 82, 79, 83, - 83, 73, 78, 71, 63, 11, 32, 60, 96, 14, 83, 72, 69, 83, 72, 73, 71, 32, - 84, 73, 77, 69, 83, 32, 97, 6, 84, 73, 77, 69, 83, 32, 16, 178, 131, 1, - 73, 194, 224, 2, 77, 222, 255, 25, 65, 244, 107, 2, 76, 65, 198, 87, 83, - 147, 17, 72, 44, 134, 1, 65, 68, 5, 68, 85, 78, 51, 32, 26, 75, 58, 76, - 62, 83, 34, 85, 222, 127, 73, 204, 31, 2, 72, 65, 150, 253, 28, 71, 179, - 101, 66, 9, 244, 86, 9, 32, 80, 76, 85, 83, 32, 76, 65, 76, 147, 243, 30, - 78, 4, 217, 32, 2, 71, 85, 6, 168, 153, 24, 5, 65, 83, 75, 65, 76, 251, - 185, 3, 85, 8, 34, 65, 210, 200, 31, 73, 3, 85, 5, 201, 85, 2, 76, 32, 4, - 226, 177, 31, 72, 215, 22, 85, 4, 150, 200, 31, 50, 3, 68, 160, 2, 42, - 65, 166, 19, 69, 122, 73, 135, 5, 85, 191, 1, 114, 50, 144, 16, 2, 66, - 65, 94, 68, 22, 76, 38, 78, 206, 140, 1, 32, 154, 6, 82, 146, 145, 28, - 83, 191, 145, 2, 77, 153, 1, 11, 32, 150, 1, 68, 6, 84, 73, 77, 69, 83, - 32, 153, 128, 1, 5, 79, 86, 69, 82, 32, 148, 1, 202, 1, 65, 162, 2, 66, - 154, 1, 68, 178, 1, 69, 58, 71, 178, 1, 72, 230, 1, 73, 70, 75, 194, 1, - 76, 62, 77, 50, 78, 130, 1, 83, 142, 1, 85, 222, 154, 1, 84, 240, 130, - 23, 3, 90, 73, 90, 247, 150, 7, 80, 18, 132, 1, 6, 32, 80, 76, 85, 83, - 32, 50, 78, 52, 2, 83, 72, 225, 160, 18, 14, 66, 50, 32, 84, 69, 78, 85, - 32, 80, 76, 85, 83, 32, 84, 6, 202, 58, 68, 134, 200, 13, 73, 155, 190, - 17, 72, 5, 185, 61, 9, 32, 80, 76, 85, 83, 32, 75, 65, 75, 7, 11, 50, 5, - 133, 90, 6, 32, 80, 76, 85, 83, 32, 10, 26, 65, 77, 2, 85, 82, 6, 42, 72, - 44, 2, 82, 32, 151, 193, 31, 68, 2, 11, 65, 2, 135, 168, 24, 82, 5, 11, - 32, 2, 185, 246, 30, 4, 80, 76, 85, 83, 14, 34, 73, 54, 85, 155, 192, 31, - 65, 7, 17, 2, 77, 32, 4, 146, 22, 84, 207, 129, 1, 71, 6, 56, 8, 71, 32, - 84, 73, 77, 69, 83, 32, 223, 191, 31, 66, 4, 158, 119, 73, 151, 45, 75, - 10, 38, 78, 254, 2, 76, 183, 232, 29, 82, 5, 243, 28, 32, 18, 18, 65, 99, - 73, 11, 26, 82, 247, 158, 1, 78, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, - 222, 167, 31, 78, 255, 2, 68, 9, 174, 122, 52, 225, 228, 12, 7, 82, 50, - 32, 80, 76, 85, 83, 12, 62, 65, 154, 124, 85, 205, 226, 12, 6, 73, 32, - 80, 76, 85, 83, 8, 40, 6, 32, 80, 76, 85, 83, 32, 83, 76, 4, 48, 6, 76, - 85, 32, 80, 76, 85, 187, 188, 31, 65, 2, 11, 83, 2, 215, 97, 32, 5, 249, - 221, 13, 5, 32, 80, 76, 85, 83, 4, 176, 100, 10, 83, 72, 32, 80, 76, 85, - 83, 32, 72, 85, 147, 15, 71, 12, 34, 65, 36, 2, 73, 68, 27, 85, 4, 250, - 204, 24, 83, 147, 238, 6, 75, 5, 225, 77, 2, 32, 80, 4, 60, 5, 83, 72, - 85, 50, 32, 197, 127, 5, 51, 32, 80, 76, 85, 2, 205, 158, 1, 3, 80, 76, - 85, 8, 26, 65, 199, 184, 31, 85, 7, 11, 77, 5, 227, 159, 1, 32, 6, 250, - 77, 69, 154, 131, 29, 85, 163, 232, 1, 73, 10, 26, 69, 73, 2, 85, 78, 7, - 33, 6, 32, 80, 76, 85, 83, 32, 4, 150, 159, 24, 69, 215, 133, 7, 71, 5, - 11, 32, 2, 187, 101, 79, 14, 42, 72, 150, 216, 23, 65, 239, 206, 7, 85, - 8, 18, 69, 51, 73, 5, 237, 190, 30, 7, 32, 80, 76, 85, 83, 32, 84, 4, - 146, 183, 31, 68, 3, 77, 7, 11, 68, 5, 209, 214, 13, 5, 32, 80, 76, 85, - 83, 7, 11, 32, 4, 142, 218, 29, 82, 129, 175, 1, 11, 67, 82, 79, 83, 83, - 73, 78, 71, 32, 71, 65, 5, 191, 131, 1, 32, 7, 134, 131, 1, 32, 247, 161, - 30, 65, 11, 11, 50, 9, 11, 32, 6, 80, 8, 67, 82, 79, 83, 83, 73, 78, 71, - 0, 4, 79, 86, 69, 82, 211, 182, 26, 84, 2, 197, 49, 3, 32, 71, 65, 8, 44, - 5, 83, 72, 84, 73, 78, 207, 154, 24, 50, 7, 37, 7, 32, 84, 73, 77, 69, - 83, 32, 4, 250, 186, 30, 75, 215, 120, 85, 49, 70, 32, 94, 52, 114, 82, - 190, 1, 83, 146, 173, 30, 68, 211, 130, 1, 71, 6, 188, 253, 8, 6, 84, 73, - 77, 69, 83, 32, 205, 244, 4, 8, 67, 82, 79, 83, 83, 73, 78, 71, 7, 11, - 32, 4, 64, 8, 67, 82, 79, 83, 83, 73, 78, 71, 1, 4, 79, 86, 69, 82, 2, - 233, 150, 24, 3, 32, 71, 73, 16, 26, 51, 147, 136, 1, 50, 13, 37, 7, 32, - 84, 73, 77, 69, 83, 32, 10, 70, 65, 0, 2, 76, 85, 206, 127, 71, 246, 239, - 12, 73, 155, 190, 17, 80, 2, 189, 239, 13, 7, 32, 80, 76, 85, 83, 32, 73, - 14, 26, 72, 147, 166, 30, 65, 13, 11, 32, 10, 22, 84, 247, 20, 67, 8, 44, - 5, 73, 77, 69, 83, 32, 255, 137, 31, 69, 6, 192, 20, 6, 71, 73, 83, 72, - 32, 67, 146, 126, 84, 243, 213, 29, 66, 43, 110, 50, 174, 1, 68, 218, 1, - 77, 54, 82, 144, 236, 13, 9, 32, 67, 82, 79, 83, 83, 73, 78, 71, 247, - 189, 17, 76, 15, 11, 32, 12, 48, 6, 84, 73, 77, 69, 83, 32, 167, 132, 1, - 71, 10, 252, 24, 3, 75, 65, 75, 194, 75, 73, 152, 20, 10, 83, 65, 76, 32, - 80, 76, 85, 83, 32, 84, 247, 179, 29, 78, 11, 11, 32, 8, 128, 1, 10, 80, - 76, 85, 83, 32, 71, 73, 83, 72, 32, 16, 6, 84, 73, 77, 69, 83, 32, 177, - 66, 8, 79, 86, 69, 82, 32, 71, 85, 68, 2, 135, 121, 84, 4, 172, 145, 1, - 5, 65, 32, 80, 76, 85, 239, 160, 29, 75, 5, 17, 2, 32, 84, 2, 241, 42, 4, - 73, 77, 69, 83, 9, 26, 85, 235, 169, 31, 55, 4, 214, 168, 31, 83, 147, 1, - 78, 50, 30, 65, 82, 73, 215, 1, 85, 11, 26, 32, 143, 169, 31, 76, 6, 32, - 2, 84, 69, 147, 128, 1, 71, 4, 235, 127, 78, 23, 37, 7, 32, 84, 73, 77, - 69, 83, 32, 20, 104, 3, 65, 83, 72, 206, 198, 27, 68, 202, 224, 2, 78, - 94, 75, 170, 57, 66, 2, 71, 162, 25, 83, 143, 45, 85, 7, 224, 55, 8, 32, - 79, 86, 69, 82, 32, 72, 73, 227, 239, 30, 50, 19, 48, 2, 66, 50, 166, - 141, 24, 76, 159, 152, 7, 83, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, - 254, 138, 1, 75, 178, 186, 26, 76, 186, 215, 2, 72, 226, 57, 65, 195, 9, - 85, 49, 104, 3, 68, 73, 77, 94, 71, 214, 1, 76, 62, 77, 206, 159, 31, 32, - 170, 1, 83, 146, 1, 66, 2, 78, 3, 82, 7, 53, 11, 32, 79, 86, 69, 82, 32, - 73, 68, 73, 77, 32, 4, 202, 244, 23, 83, 155, 183, 6, 66, 13, 11, 73, 11, - 11, 32, 8, 162, 123, 71, 220, 230, 11, 31, 79, 86, 69, 82, 32, 73, 71, - 73, 32, 83, 72, 73, 82, 32, 79, 86, 69, 82, 32, 83, 72, 73, 82, 32, 85, - 68, 32, 79, 86, 69, 82, 142, 134, 17, 68, 155, 168, 1, 82, 7, 26, 32, - 167, 162, 31, 50, 2, 169, 72, 4, 84, 73, 77, 69, 13, 26, 32, 139, 210, - 30, 73, 8, 76, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 162, 111, 84, - 143, 130, 23, 83, 2, 197, 158, 30, 5, 83, 73, 78, 71, 32, 224, 1, 78, 65, - 146, 15, 73, 242, 1, 85, 158, 73, 69, 229, 168, 23, 4, 87, 85, 51, 49, - 177, 1, 164, 1, 7, 32, 84, 73, 77, 69, 83, 32, 210, 9, 50, 66, 68, 102, - 75, 50, 76, 98, 77, 28, 4, 83, 75, 65, 76, 248, 192, 15, 6, 80, 32, 69, - 76, 65, 77, 219, 209, 15, 66, 134, 1, 170, 1, 65, 94, 66, 82, 69, 30, 71, - 182, 2, 73, 34, 75, 34, 76, 38, 77, 170, 1, 83, 118, 84, 34, 85, 214, 8, - 72, 246, 51, 78, 222, 215, 16, 80, 154, 221, 13, 82, 147, 17, 90, 13, 46, - 68, 250, 239, 30, 78, 165, 44, 2, 83, 72, 5, 229, 50, 7, 32, 80, 76, 85, - 83, 32, 75, 10, 34, 65, 194, 156, 31, 73, 3, 85, 6, 186, 179, 29, 76, - 134, 233, 1, 68, 3, 82, 4, 138, 25, 82, 151, 61, 83, 26, 30, 65, 98, 73, - 147, 1, 85, 11, 38, 82, 206, 123, 78, 231, 159, 30, 76, 5, 249, 21, 10, - 32, 80, 76, 85, 83, 32, 83, 72, 65, 51, 11, 32, 2, 83, 72, 171, 129, 24, + 69, 82, 84, 73, 67, 65, 76, 32, 146, 152, 16, 71, 191, 187, 4, 76, 6, + 226, 17, 82, 131, 137, 16, 67, 4, 145, 17, 2, 82, 69, 4, 230, 235, 31, + 65, 135, 42, 73, 4, 21, 3, 65, 67, 82, 4, 165, 135, 17, 2, 79, 78, 4, + 138, 37, 80, 255, 130, 31, 86, 6, 146, 38, 76, 177, 233, 7, 9, 65, 82, + 69, 78, 84, 72, 69, 83, 69, 6, 240, 52, 4, 71, 72, 84, 87, 207, 129, 10, + 78, 7, 11, 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 219, 236, + 30, 84, 6, 182, 163, 12, 83, 175, 187, 14, 76, 2, 201, 146, 16, 2, 32, + 67, 6, 138, 51, 32, 207, 187, 31, 87, 18, 72, 9, 78, 67, 76, 79, 83, 73, + 78, 71, 32, 177, 35, 4, 81, 85, 65, 76, 14, 132, 1, 6, 67, 73, 82, 67, + 76, 69, 22, 83, 140, 170, 6, 3, 75, 69, 89, 240, 216, 19, 7, 85, 80, 87, + 65, 82, 68, 32, 171, 204, 5, 68, 5, 163, 198, 18, 32, 4, 182, 239, 25, + 67, 147, 252, 5, 81, 6, 48, 2, 69, 82, 206, 145, 5, 79, 251, 134, 12, 76, + 2, 255, 76, 77, 128, 1, 88, 17, 76, 65, 71, 79, 76, 73, 84, 73, 67, 32, + 76, 69, 84, 84, 69, 82, 32, 143, 3, 82, 76, 238, 1, 68, 30, 73, 46, 83, + 50, 84, 210, 146, 6, 65, 22, 66, 94, 67, 134, 1, 70, 38, 71, 238, 2, 77, + 32, 2, 76, 74, 34, 78, 88, 2, 80, 79, 30, 82, 194, 2, 86, 22, 89, 90, 90, + 234, 145, 19, 72, 190, 133, 2, 85, 162, 177, 5, 79, 235, 5, 75, 4, 138, + 149, 6, 74, 31, 79, 11, 150, 150, 6, 78, 54, 79, 243, 198, 26, 90, 8, + 246, 150, 6, 77, 130, 4, 76, 247, 218, 22, 72, 4, 170, 155, 6, 86, 235, + 218, 26, 83, 52, 38, 65, 185, 3, 4, 69, 69, 75, 32, 38, 84, 5, 78, 84, + 72, 65, 32, 196, 1, 2, 86, 69, 137, 216, 28, 5, 80, 72, 69, 77, 69, 24, + 68, 6, 68, 73, 71, 73, 84, 32, 65, 7, 76, 69, 84, 84, 69, 82, 32, 14, + 170, 247, 16, 83, 202, 157, 14, 70, 70, 84, 62, 90, 223, 86, 79, 10, 230, + 243, 32, 86, 162, 17, 75, 2, 78, 2, 80, 187, 2, 65, 12, 18, 32, 67, 45, + 6, 26, 65, 131, 184, 16, 84, 4, 237, 147, 4, 4, 67, 67, 69, 78, 6, 150, + 22, 86, 168, 250, 11, 6, 65, 67, 85, 84, 69, 45, 139, 249, 3, 77, 14, + 148, 1, 8, 77, 85, 83, 73, 67, 65, 76, 32, 130, 155, 4, 75, 170, 202, 2, + 80, 174, 4, 89, 217, 239, 3, 11, 68, 73, 65, 76, 89, 84, 73, 75, 65, 32, + 84, 6, 26, 84, 255, 203, 15, 80, 4, 214, 204, 15, 69, 35, 82, 6, 146, + 130, 13, 79, 128, 133, 19, 8, 77, 79, 84, 72, 69, 84, 73, 67, 167, 45, + 82, 16, 26, 78, 151, 210, 30, 83, 14, 52, 7, 86, 69, 82, 84, 69, 68, 32, + 243, 209, 28, 70, 12, 60, 2, 66, 82, 69, 9, 68, 79, 85, 66, 76, 69, 32, + 65, 82, 8, 18, 69, 23, 73, 4, 203, 232, 11, 86, 4, 233, 223, 30, 2, 68, + 71, 4, 11, 67, 4, 207, 223, 30, 72, 8, 128, 1, 16, 84, 65, 75, 65, 78, + 65, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 229, 250, 17, 9, 86, 89, 75, + 65, 32, 65, 66, 79, 86, 4, 242, 139, 10, 83, 35, 86, 162, 1, 80, 5, 65, + 84, 73, 78, 32, 164, 8, 3, 69, 70, 84, 232, 3, 2, 73, 71, 83, 79, 106, + 156, 1, 21, 76, 69, 84, 84, 69, 82, 32, 83, 77, 65, 76, 76, 32, 67, 65, + 80, 73, 84, 65, 76, 32, 53, 13, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, + 69, 82, 32, 10, 182, 254, 32, 71, 2, 76, 2, 77, 2, 78, 3, 82, 96, 226, 1, + 65, 70, 67, 34, 69, 30, 70, 78, 73, 86, 76, 106, 79, 2, 85, 134, 1, 82, + 50, 84, 158, 183, 12, 83, 162, 151, 2, 66, 238, 157, 2, 87, 190, 139, 16, + 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 78, 2, 80, 2, 86, 2, 88, 3, 90, 13, + 182, 246, 2, 32, 226, 169, 26, 76, 138, 220, 3, 69, 2, 79, 3, 86, 5, 173, + 151, 12, 3, 32, 67, 69, 7, 170, 250, 32, 83, 3, 84, 5, 225, 207, 27, 14, + 76, 65, 84, 84, 69, 78, 69, 68, 32, 79, 80, 69, 78, 32, 11, 37, 7, 78, + 83, 85, 76, 65, 82, 32, 8, 170, 250, 32, 68, 2, 71, 2, 82, 3, 84, 7, 148, + 151, 16, 14, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 69, 32, 77, 133, + 233, 12, 3, 79, 78, 71, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 146, 167, + 12, 68, 209, 233, 12, 15, 76, 73, 71, 72, 84, 32, 67, 69, 78, 84, 82, 65, + 76, 73, 90, 7, 11, 32, 4, 210, 133, 12, 82, 203, 208, 18, 66, 5, 201, + 198, 30, 7, 85, 82, 78, 69, 68, 32, 87, 36, 46, 32, 133, 3, 6, 87, 65, + 82, 68, 83, 32, 32, 122, 65, 192, 1, 12, 80, 65, 82, 69, 78, 84, 72, 69, + 83, 73, 83, 32, 166, 13, 72, 154, 10, 84, 57, 5, 82, 73, 71, 72, 84, 14, + 52, 5, 78, 71, 76, 69, 32, 77, 4, 82, 82, 79, 87, 6, 140, 253, 3, 6, 67, + 69, 78, 84, 82, 69, 214, 214, 26, 66, 131, 165, 1, 65, 8, 26, 72, 227, + 210, 30, 32, 4, 209, 210, 30, 3, 69, 65, 68, 4, 144, 232, 7, 4, 65, 66, + 79, 86, 241, 217, 24, 5, 66, 69, 76, 79, 87, 4, 142, 14, 72, 245, 248, + 11, 5, 65, 82, 82, 79, 87, 10, 52, 6, 65, 84, 85, 82, 69, 32, 129, 18, 2, + 72, 84, 8, 234, 1, 76, 23, 82, 10, 36, 3, 78, 71, 32, 131, 215, 31, 87, + 8, 236, 7, 7, 68, 79, 85, 66, 76, 69, 32, 242, 7, 83, 71, 86, 26, 48, 5, + 65, 67, 82, 79, 78, 205, 5, 2, 73, 78, 23, 18, 32, 127, 45, 10, 34, 76, + 22, 82, 171, 207, 30, 66, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 189, + 229, 16, 6, 84, 32, 72, 65, 76, 70, 10, 54, 86, 250, 20, 65, 150, 148, + 12, 71, 235, 176, 3, 66, 2, 205, 212, 31, 8, 69, 82, 84, 73, 67, 65, 76, + 45, 4, 164, 9, 9, 85, 77, 66, 69, 82, 32, 83, 73, 71, 129, 223, 11, 2, + 79, 84, 18, 136, 1, 17, 76, 68, 32, 80, 69, 82, 77, 73, 67, 32, 76, 69, + 84, 84, 69, 82, 32, 82, 80, 216, 166, 12, 4, 71, 79, 78, 69, 167, 220, + 18, 86, 10, 198, 45, 90, 142, 163, 18, 78, 182, 166, 9, 68, 186, 176, 2, + 83, 239, 246, 1, 65, 2, 229, 224, 24, 6, 69, 78, 32, 77, 65, 82, 12, 18, + 65, 107, 76, 8, 220, 7, 9, 82, 69, 78, 84, 72, 69, 83, 69, 83, 173, 216, + 24, 9, 76, 65, 84, 65, 76, 73, 90, 69, 68, 4, 11, 85, 4, 241, 201, 30, 6, + 83, 32, 83, 73, 71, 78, 40, 18, 69, 127, 73, 6, 72, 5, 86, 69, 82, 83, + 69, 141, 222, 24, 7, 84, 82, 79, 70, 76, 69, 88, 4, 22, 32, 251, 12, 68, + 2, 137, 8, 2, 83, 79, 34, 40, 3, 71, 72, 84, 133, 5, 2, 78, 71, 28, 50, + 32, 253, 3, 7, 87, 65, 82, 68, 83, 32, 72, 26, 108, 5, 65, 82, 82, 79, + 87, 218, 1, 72, 124, 12, 80, 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 32, + 159, 9, 84, 12, 44, 5, 72, 69, 65, 68, 32, 199, 198, 30, 32, 8, 26, 65, + 199, 198, 30, 66, 6, 36, 3, 78, 68, 32, 171, 235, 31, 66, 4, 40, 4, 68, + 79, 87, 78, 1, 2, 85, 80, 2, 249, 231, 8, 9, 32, 65, 82, 82, 79, 87, 72, + 69, 65, 6, 11, 65, 6, 48, 6, 76, 70, 32, 82, 73, 78, 21, 2, 82, 80, 4, + 207, 196, 30, 71, 2, 17, 2, 79, 79, 2, 183, 233, 31, 78, 4, 146, 228, 9, + 65, 245, 174, 21, 5, 66, 69, 76, 79, 87, 2, 185, 161, 21, 16, 65, 82, 80, + 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 6, 11, 32, 6, 166, + 248, 11, 79, 254, 202, 18, 66, 131, 165, 1, 65, 24, 166, 1, 72, 204, 1, + 6, 81, 85, 65, 82, 69, 32, 84, 5, 84, 82, 79, 78, 71, 128, 195, 3, 2, 85, + 83, 236, 145, 21, 2, 78, 65, 149, 234, 5, 6, 69, 65, 71, 85, 76, 76, 8, + 40, 4, 79, 82, 84, 32, 187, 194, 16, 65, 6, 18, 83, 71, 86, 4, 26, 79, + 187, 245, 11, 84, 2, 217, 245, 11, 5, 76, 73, 68, 85, 83, 2, 49, 10, 69, + 82, 84, 73, 67, 65, 76, 32, 76, 73, 2, 243, 244, 11, 78, 6, 26, 66, 227, + 228, 31, 65, 4, 216, 222, 7, 5, 82, 65, 67, 75, 69, 147, 225, 22, 69, 2, + 209, 212, 24, 17, 32, 67, 69, 78, 84, 82, 65, 76, 73, 90, 65, 84, 73, 79, + 78, 32, 83, 20, 98, 72, 32, 4, 73, 76, 68, 69, 136, 1, 6, 82, 73, 80, 76, + 69, 32, 69, 5, 85, 82, 78, 69, 68, 2, 205, 220, 7, 3, 82, 69, 69, 11, 11, + 32, 8, 76, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 246, 241, 11, 79, 255, + 202, 18, 66, 2, 193, 173, 30, 6, 84, 32, 72, 65, 76, 70, 6, 178, 229, 15, + 65, 216, 229, 15, 5, 85, 78, 68, 69, 82, 239, 66, 68, 2, 249, 191, 15, 2, + 32, 67, 12, 34, 80, 170, 224, 31, 82, 3, 83, 8, 18, 32, 43, 87, 4, 11, + 84, 4, 133, 208, 24, 2, 65, 67, 4, 25, 4, 65, 82, 68, 83, 4, 189, 186, + 30, 6, 32, 65, 82, 82, 79, 87, 16, 42, 32, 37, 6, 45, 76, 73, 78, 69, 45, + 6, 250, 169, 26, 76, 175, 202, 2, 84, 10, 54, 65, 48, 5, 71, 82, 65, 86, + 69, 139, 222, 15, 77, 4, 25, 4, 67, 85, 84, 69, 5, 147, 229, 11, 45, 5, + 143, 139, 12, 45, 6, 52, 3, 68, 69, 32, 137, 238, 11, 4, 71, 71, 76, 89, + 4, 132, 206, 24, 14, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 82, 73, 68, + 71, 149, 229, 4, 5, 66, 82, 73, 68, 71, 6, 194, 245, 11, 45, 235, 193, + 18, 32, 6, 52, 7, 69, 82, 67, 73, 65, 76, 32, 219, 216, 32, 65, 4, 166, + 166, 29, 77, 135, 150, 3, 65, 8, 158, 166, 8, 82, 228, 161, 16, 3, 79, + 83, 73, 250, 244, 2, 76, 231, 130, 4, 65, 38, 214, 1, 70, 84, 10, 83, 84, + 82, 85, 67, 84, 73, 79, 78, 32, 46, 84, 252, 235, 2, 8, 86, 69, 78, 73, + 69, 78, 67, 69, 204, 195, 18, 6, 73, 67, 65, 76, 32, 84, 234, 189, 9, 74, + 217, 102, 7, 71, 82, 85, 69, 78, 84, 32, 6, 140, 173, 27, 4, 69, 84, 84, + 73, 150, 242, 1, 85, 245, 163, 2, 4, 79, 85, 78, 68, 4, 240, 221, 20, 2, + 87, 79, 223, 240, 10, 83, 20, 80, 5, 65, 73, 78, 83, 32, 198, 1, 79, 28, + 4, 82, 79, 76, 32, 183, 146, 3, 73, 12, 48, 3, 65, 83, 32, 93, 5, 87, 73, + 84, 72, 32, 6, 186, 172, 24, 77, 181, 8, 15, 78, 79, 82, 77, 65, 76, 32, + 83, 85, 66, 71, 82, 79, 85, 80, 6, 250, 195, 4, 76, 202, 233, 19, 86, + 239, 243, 4, 79, 2, 229, 200, 30, 2, 85, 82, 4, 144, 251, 19, 3, 75, 78, + 79, 173, 134, 4, 8, 83, 69, 81, 85, 69, 78, 67, 69, 6, 26, 73, 147, 158, + 2, 69, 4, 150, 209, 32, 78, 87, 69, 206, 2, 36, 4, 84, 73, 67, 32, 175, + 18, 89, 202, 2, 186, 1, 67, 204, 1, 6, 69, 80, 65, 67, 84, 32, 86, 70, + 36, 11, 79, 76, 68, 32, 78, 85, 66, 73, 65, 78, 32, 110, 83, 213, 146, + 29, 13, 77, 79, 82, 80, 72, 79, 76, 79, 71, 73, 67, 65, 76, 126, 76, 9, + 79, 77, 66, 73, 78, 73, 78, 71, 32, 161, 3, 5, 65, 80, 73, 84, 65, 6, 68, + 9, 83, 80, 73, 82, 73, 84, 85, 83, 32, 201, 209, 31, 2, 78, 73, 4, 128, + 156, 6, 2, 76, 69, 181, 144, 17, 2, 65, 83, 56, 158, 188, 21, 78, 170, + 198, 2, 68, 141, 250, 6, 8, 84, 72, 79, 85, 83, 65, 78, 68, 4, 218, 187, + 18, 82, 195, 188, 11, 85, 8, 58, 68, 0, 3, 73, 78, 68, 146, 15, 86, 163, + 232, 29, 70, 2, 253, 247, 29, 7, 73, 82, 69, 67, 84, 32, 81, 134, 1, 56, + 3, 77, 65, 76, 197, 11, 6, 89, 77, 66, 79, 76, 32, 120, 45, 9, 76, 32, + 76, 69, 84, 84, 69, 82, 32, 120, 138, 2, 65, 48, 6, 66, 79, 72, 65, 73, + 82, 32, 2, 67, 82, 170, 1, 68, 160, 1, 2, 71, 65, 34, 72, 38, 75, 46, 70, + 34, 76, 66, 79, 138, 3, 83, 90, 84, 46, 90, 202, 218, 5, 86, 214, 54, 80, + 170, 211, 3, 73, 142, 189, 22, 82, 198, 3, 69, 218, 7, 77, 2, 78, 163, + 17, 85, 4, 44, 5, 75, 72, 77, 73, 77, 171, 208, 19, 76, 2, 177, 1, 4, 73, + 67, 32, 75, 10, 92, 12, 89, 80, 84, 79, 71, 82, 65, 77, 77, 73, 67, 32, + 53, 7, 79, 83, 83, 69, 68, 32, 83, 8, 50, 83, 226, 4, 71, 198, 167, 32, + 69, 219, 7, 78, 2, 187, 240, 24, 72, 12, 80, 9, 73, 65, 76, 69, 67, 84, + 45, 80, 32, 232, 161, 32, 2, 65, 76, 175, 17, 69, 8, 206, 163, 8, 72, + 174, 158, 8, 65, 200, 199, 12, 2, 75, 65, 211, 169, 3, 78, 4, 190, 3, 78, + 211, 161, 32, 77, 4, 178, 218, 25, 79, 187, 162, 5, 65, 8, 42, 72, 138, + 136, 29, 65, 211, 169, 3, 83, 4, 214, 177, 32, 69, 219, 19, 73, 4, 252, + 205, 19, 7, 45, 83, 72, 65, 80, 69, 68, 239, 251, 7, 65, 35, 36, 3, 76, + 68, 32, 195, 159, 32, 79, 30, 76, 7, 67, 79, 80, 84, 73, 67, 32, 193, 1, + 7, 78, 85, 66, 73, 65, 78, 32, 22, 98, 71, 42, 72, 188, 1, 2, 83, 72, + 162, 236, 14, 68, 238, 253, 9, 79, 254, 235, 5, 69, 183, 107, 65, 2, 17, + 2, 65, 78, 2, 231, 137, 16, 71, 8, 254, 214, 25, 79, 246, 130, 1, 65, + 131, 213, 5, 69, 8, 50, 78, 172, 192, 16, 2, 83, 72, 223, 158, 5, 87, 4, + 138, 174, 32, 71, 3, 89, 10, 54, 72, 238, 157, 6, 65, 190, 254, 25, 79, + 219, 3, 73, 4, 254, 159, 32, 73, 187, 13, 69, 4, 224, 147, 32, 3, 72, 69, + 84, 167, 8, 65, 2, 247, 155, 32, 65, 14, 62, 75, 30, 77, 2, 80, 22, 83, + 133, 132, 28, 3, 84, 65, 85, 4, 26, 72, 255, 171, 32, 65, 2, 151, 132, + 28, 73, 4, 132, 190, 16, 6, 72, 73, 77, 65, 32, 83, 193, 255, 3, 3, 84, + 65, 85, 4, 164, 220, 24, 3, 76, 69, 70, 133, 151, 2, 4, 82, 73, 71, 72, + 6, 96, 6, 78, 73, 83, 72, 32, 86, 188, 220, 25, 8, 82, 69, 83, 80, 79, + 78, 68, 83, 203, 213, 5, 65, 2, 241, 131, 29, 4, 69, 82, 83, 69, 44, 104, + 2, 78, 84, 192, 132, 3, 6, 67, 72, 32, 65, 78, 68, 201, 176, 28, 8, 80, + 76, 69, 32, 87, 73, 84, 72, 40, 56, 2, 69, 82, 37, 8, 73, 78, 71, 32, 82, + 79, 68, 32, 4, 246, 206, 27, 66, 167, 136, 1, 83, 36, 48, 4, 84, 69, 78, + 83, 1, 4, 85, 78, 73, 84, 18, 221, 239, 23, 2, 32, 68, 51, 82, 69, 76, 5, + 73, 67, 75, 69, 84, 34, 79, 198, 5, 85, 50, 89, 235, 245, 30, 65, 4, 240, + 175, 8, 3, 68, 73, 84, 165, 224, 20, 7, 83, 67, 69, 78, 84, 32, 77, 5, + 205, 144, 27, 3, 32, 66, 65, 28, 84, 2, 83, 83, 180, 144, 30, 3, 67, 79, + 68, 208, 156, 1, 3, 73, 83, 83, 155, 60, 87, 22, 46, 32, 184, 2, 3, 69, + 68, 32, 223, 1, 73, 14, 44, 3, 79, 70, 32, 82, 80, 163, 246, 31, 77, 4, + 216, 202, 28, 6, 74, 69, 82, 85, 83, 65, 133, 210, 2, 5, 76, 79, 82, 82, + 65, 8, 76, 10, 65, 84, 84, 89, 32, 87, 73, 84, 72, 32, 41, 5, 79, 77, 77, + 69, 69, 4, 206, 173, 10, 82, 25, 3, 76, 69, 70, 5, 225, 161, 3, 11, 32, + 87, 73, 84, 72, 32, 72, 65, 76, 70, 45, 6, 168, 152, 25, 37, 78, 69, 71, + 65, 84, 73, 86, 69, 32, 83, 81, 85, 65, 82, 69, 68, 32, 76, 65, 84, 73, + 78, 32, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 248, 226, + 2, 2, 70, 76, 177, 216, 2, 3, 83, 87, 79, 2, 233, 254, 23, 5, 78, 71, 32, + 76, 65, 4, 184, 214, 17, 3, 90, 69, 73, 175, 187, 14, 84, 6, 214, 234, + 19, 73, 141, 160, 7, 4, 83, 84, 65, 76, 206, 19, 154, 1, 66, 20, 8, 78, + 69, 73, 70, 79, 82, 77, 32, 154, 250, 1, 80, 114, 82, 180, 3, 2, 83, 84, + 226, 234, 15, 67, 161, 191, 13, 6, 84, 32, 79, 70, 32, 77, 2, 179, 182, + 5, 69, 170, 19, 176, 1, 13, 78, 85, 77, 69, 82, 73, 67, 32, 83, 73, 71, + 78, 32, 184, 20, 17, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 83, + 73, 71, 78, 32, 233, 1, 5, 83, 73, 71, 78, 32, 222, 1, 78, 69, 166, 2, + 70, 152, 3, 2, 78, 73, 154, 2, 79, 130, 3, 83, 151, 4, 84, 24, 68, 5, 73, + 71, 72, 84, 32, 153, 1, 7, 76, 65, 77, 73, 84, 69, 32, 16, 142, 14, 83, + 230, 113, 85, 238, 50, 71, 136, 56, 17, 86, 65, 82, 73, 65, 78, 84, 32, + 70, 79, 82, 77, 32, 85, 83, 83, 85, 242, 223, 26, 68, 199, 249, 1, 65, 8, + 252, 155, 21, 5, 79, 78, 69, 32, 84, 234, 160, 9, 70, 175, 14, 84, 58, + 48, 4, 73, 86, 69, 32, 125, 4, 79, 85, 82, 32, 26, 70, 83, 206, 1, 66, + 250, 12, 65, 82, 71, 138, 110, 85, 231, 202, 27, 68, 6, 182, 15, 72, 245, + 158, 19, 5, 73, 88, 84, 72, 83, 32, 150, 1, 66, 44, 18, 86, 65, 82, 73, + 65, 78, 84, 32, 70, 79, 82, 77, 32, 76, 73, 77, 77, 85, 206, 12, 65, 82, + 71, 26, 83, 242, 109, 85, 231, 202, 27, 68, 6, 204, 123, 3, 65, 78, 50, + 183, 237, 26, 85, 9, 206, 211, 12, 32, 135, 215, 19, 52, 24, 44, 4, 71, + 73, 68, 65, 33, 3, 78, 69, 32, 4, 246, 190, 30, 69, 183, 107, 77, 20, + 156, 1, 19, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 73, 76, + 73, 77, 77, 85, 174, 7, 83, 230, 113, 85, 238, 50, 71, 250, 151, 27, 68, + 199, 249, 1, 65, 9, 134, 166, 32, 32, 186, 2, 51, 3, 52, 28, 92, 16, 76, + 68, 32, 65, 83, 83, 89, 82, 73, 65, 78, 32, 79, 78, 69, 32, 37, 3, 78, + 69, 32, 4, 246, 208, 18, 83, 219, 208, 11, 81, 24, 166, 1, 69, 52, 8, 81, + 85, 65, 82, 84, 69, 82, 32, 206, 7, 66, 54, 71, 84, 5, 84, 72, 73, 82, + 68, 164, 214, 24, 2, 83, 72, 237, 205, 6, 6, 72, 65, 76, 70, 32, 71, 4, + 158, 8, 83, 205, 164, 1, 5, 73, 71, 72, 84, 72, 4, 174, 186, 30, 65, 183, + 114, 71, 38, 144, 1, 5, 69, 86, 69, 78, 32, 188, 1, 20, 72, 65, 82, 50, + 32, 84, 73, 77, 69, 83, 32, 71, 65, 76, 32, 80, 76, 85, 83, 32, 37, 3, + 73, 88, 32, 18, 148, 1, 17, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, + 77, 32, 73, 77, 73, 78, 218, 1, 83, 230, 113, 85, 238, 50, 71, 250, 151, + 27, 68, 199, 249, 1, 65, 6, 230, 203, 12, 32, 135, 215, 19, 51, 4, 250, + 189, 28, 68, 251, 228, 2, 77, 16, 142, 1, 83, 142, 3, 65, 218, 110, 85, + 238, 50, 71, 172, 130, 27, 16, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, + 82, 77, 32, 65, 83, 72, 207, 21, 68, 2, 227, 61, 72, 50, 52, 5, 72, 82, + 69, 69, 32, 241, 1, 3, 87, 79, 32, 28, 142, 1, 66, 40, 4, 83, 72, 65, 82, + 24, 16, 86, 65, 82, 73, 65, 78, 84, 32, 70, 79, 82, 77, 32, 69, 83, 72, + 118, 65, 82, 71, 239, 184, 28, 68, 6, 136, 112, 3, 85, 82, 85, 199, 10, + 65, 8, 226, 111, 50, 3, 85, 4, 222, 243, 24, 49, 203, 3, 50, 22, 82, 65, + 30, 66, 32, 2, 69, 83, 22, 71, 26, 83, 61, 6, 84, 72, 73, 82, 68, 83, 4, + 225, 184, 1, 2, 83, 72, 4, 254, 120, 65, 243, 226, 26, 85, 2, 211, 246, + 24, 72, 4, 53, 3, 69, 83, 72, 4, 11, 72, 4, 17, 2, 65, 82, 4, 254, 156, + 32, 50, 3, 85, 4, 11, 32, 4, 216, 211, 27, 12, 86, 65, 82, 73, 65, 78, + 84, 32, 70, 79, 82, 77, 179, 100, 68, 10, 160, 1, 9, 68, 73, 65, 71, 79, + 78, 65, 76, 32, 220, 222, 25, 6, 86, 69, 82, 84, 73, 67, 145, 130, 3, 14, + 79, 76, 68, 32, 65, 83, 83, 89, 82, 73, 65, 78, 32, 87, 6, 220, 203, 28, + 2, 84, 82, 244, 254, 2, 4, 81, 85, 65, 68, 15, 67, 194, 17, 202, 1, 65, + 186, 15, 66, 170, 5, 68, 190, 15, 69, 202, 10, 71, 182, 30, 72, 238, 3, + 73, 238, 4, 75, 154, 20, 76, 134, 37, 77, 134, 6, 78, 226, 17, 80, 210, + 3, 82, 42, 83, 238, 19, 84, 186, 8, 85, 207, 20, 90, 141, 1, 160, 1, 7, + 32, 84, 73, 77, 69, 83, 32, 142, 1, 66, 250, 3, 68, 38, 75, 90, 76, 212, + 1, 3, 77, 65, 82, 78, 78, 134, 2, 82, 42, 83, 138, 141, 31, 80, 215, 127, + 50, 16, 144, 173, 1, 5, 76, 65, 71, 65, 82, 226, 23, 71, 254, 253, 12, + 73, 174, 197, 13, 77, 162, 163, 2, 83, 194, 165, 1, 66, 246, 67, 72, 187, + 2, 65, 45, 22, 32, 219, 2, 50, 28, 48, 6, 84, 73, 77, 69, 83, 32, 139, + 203, 1, 71, 26, 180, 1, 2, 71, 65, 38, 73, 36, 2, 83, 72, 128, 113, 8, + 85, 32, 80, 76, 85, 83, 32, 85, 188, 56, 4, 68, 85, 78, 51, 166, 2, 65, + 204, 2, 3, 78, 85, 78, 242, 27, 76, 255, 198, 30, 72, 4, 194, 210, 1, 78, + 215, 193, 30, 76, 4, 210, 169, 1, 71, 207, 234, 29, 77, 4, 142, 236, 24, + 85, 187, 188, 5, 69, 15, 37, 7, 32, 84, 73, 77, 69, 83, 32, 12, 206, 133, + 1, 77, 130, 59, 71, 194, 9, 83, 234, 10, 84, 148, 210, 28, 2, 66, 65, + 251, 235, 1, 65, 5, 249, 51, 5, 32, 84, 73, 77, 69, 7, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 4, 252, 27, 5, 83, 72, 73, 84, 65, 215, 80, 69, 23, 68, + 7, 32, 84, 73, 77, 69, 83, 32, 234, 221, 25, 69, 155, 227, 5, 65, 16, + 102, 75, 156, 173, 1, 2, 68, 73, 146, 254, 26, 71, 198, 249, 1, 85, 186, + 95, 65, 198, 94, 83, 215, 42, 72, 4, 130, 158, 1, 65, 247, 241, 30, 73, + 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 142, 254, 3, 75, 147, 228, 27, + 83, 13, 26, 32, 227, 225, 31, 83, 8, 128, 1, 10, 80, 76, 85, 83, 32, 78, + 65, 71, 65, 32, 172, 192, 8, 7, 84, 72, 82, 69, 69, 32, 84, 141, 131, 6, + 4, 79, 86, 69, 82, 4, 144, 140, 1, 16, 79, 80, 80, 79, 83, 73, 78, 71, + 32, 65, 78, 32, 80, 76, 85, 83, 215, 195, 23, 83, 6, 200, 49, 2, 65, 68, + 151, 166, 18, 75, 18, 26, 72, 139, 173, 12, 65, 17, 42, 32, 242, 214, 18, + 71, 167, 181, 13, 50, 10, 68, 9, 79, 86, 69, 82, 32, 65, 83, 72, 32, 174, + 97, 90, 135, 112, 75, 6, 176, 1, 8, 79, 86, 69, 82, 32, 65, 83, 72, 157, + 143, 1, 29, 84, 85, 71, 50, 32, 79, 86, 69, 82, 32, 84, 85, 71, 50, 32, + 84, 85, 71, 50, 32, 79, 86, 69, 82, 32, 84, 85, 71, 50, 5, 149, 145, 1, + 27, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 65, 83, 72, 32, 79, 86, 69, + 82, 32, 65, 83, 72, 32, 79, 86, 69, 82, 52, 30, 65, 158, 2, 73, 95, 85, + 27, 66, 68, 44, 4, 72, 65, 82, 50, 90, 76, 66, 82, 151, 140, 28, 71, 5, + 145, 208, 1, 6, 32, 84, 73, 77, 69, 83, 9, 37, 7, 32, 84, 73, 77, 69, 83, + 32, 6, 238, 164, 1, 65, 154, 207, 30, 78, 163, 17, 90, 7, 208, 251, 30, + 7, 32, 79, 86, 69, 82, 32, 66, 139, 139, 1, 65, 5, 175, 223, 24, 65, 9, + 37, 7, 32, 84, 73, 77, 69, 83, 32, 6, 134, 156, 1, 73, 242, 198, 29, 71, + 191, 163, 1, 65, 19, 50, 32, 168, 1, 3, 76, 85, 71, 239, 169, 1, 82, 8, + 88, 8, 79, 86, 69, 82, 32, 66, 85, 32, 149, 208, 27, 8, 67, 82, 79, 83, + 83, 73, 78, 71, 6, 160, 207, 12, 7, 84, 73, 77, 69, 83, 32, 78, 214, 247, + 17, 65, 155, 110, 85, 5, 249, 242, 3, 8, 32, 79, 86, 69, 82, 32, 66, 85, + 180, 1, 34, 65, 222, 5, 73, 195, 2, 85, 67, 50, 71, 150, 5, 82, 146, 49, + 32, 243, 204, 31, 77, 55, 26, 32, 251, 130, 32, 51, 50, 76, 13, 75, 73, + 83, 73, 77, 53, 32, 84, 73, 77, 69, 83, 32, 227, 198, 1, 84, 48, 146, 1, + 65, 30, 66, 38, 71, 112, 2, 73, 82, 42, 76, 94, 85, 182, 131, 1, 80, 162, + 61, 84, 134, 133, 28, 75, 182, 162, 2, 78, 254, 2, 83, 163, 17, 72, 4, + 110, 32, 235, 220, 30, 77, 4, 254, 148, 30, 65, 251, 235, 1, 73, 10, 34, + 65, 58, 73, 219, 185, 31, 85, 5, 11, 32, 2, 141, 149, 30, 6, 80, 76, 85, + 83, 32, 77, 5, 183, 216, 24, 82, 5, 129, 142, 14, 5, 32, 80, 76, 85, 83, + 8, 26, 85, 163, 255, 31, 65, 7, 160, 151, 1, 7, 32, 80, 76, 85, 83, 32, + 77, 255, 231, 30, 77, 6, 52, 7, 50, 32, 80, 76, 85, 83, 32, 155, 253, 31, + 83, 4, 128, 27, 2, 71, 73, 147, 248, 29, 77, 7, 187, 202, 12, 65, 25, 54, + 77, 150, 1, 78, 76, 2, 83, 72, 215, 251, 31, 66, 13, 44, 7, 32, 84, 73, + 77, 69, 83, 32, 59, 50, 6, 220, 69, 2, 85, 32, 238, 226, 13, 73, 175, + 167, 17, 83, 5, 205, 166, 13, 6, 32, 84, 73, 77, 69, 83, 5, 253, 253, 18, + 14, 32, 75, 65, 83, 75, 65, 76, 32, 85, 32, 71, 85, 78, 85, 5, 245, 157, + 1, 5, 32, 80, 76, 85, 83, 91, 70, 32, 66, 66, 94, 71, 234, 4, 78, 162, + 205, 24, 82, 195, 167, 7, 72, 6, 154, 176, 1, 71, 170, 3, 83, 181, 211, + 12, 4, 79, 86, 69, 82, 9, 52, 7, 32, 84, 73, 77, 69, 83, 32, 239, 249, + 31, 50, 4, 250, 145, 1, 69, 231, 186, 30, 83, 61, 52, 7, 32, 84, 73, 77, + 69, 83, 32, 243, 178, 31, 85, 56, 122, 65, 58, 68, 30, 71, 74, 75, 90, + 76, 110, 77, 50, 83, 130, 80, 69, 218, 58, 73, 130, 213, 16, 72, 214, + 129, 14, 78, 3, 80, 6, 32, 2, 83, 72, 219, 202, 31, 78, 5, 251, 220, 14, + 32, 4, 250, 167, 31, 73, 3, 85, 8, 26, 73, 167, 247, 31, 65, 7, 140, 141, + 1, 2, 82, 50, 135, 233, 30, 83, 8, 26, 85, 211, 185, 1, 65, 6, 40, 4, 83, + 72, 85, 50, 179, 246, 31, 82, 5, 175, 26, 32, 8, 26, 65, 45, 2, 85, 72, + 6, 202, 26, 77, 169, 178, 24, 3, 75, 45, 48, 2, 225, 57, 5, 32, 80, 76, + 85, 83, 6, 170, 138, 30, 65, 174, 173, 1, 69, 223, 61, 73, 4, 238, 138, + 1, 73, 195, 211, 30, 72, 11, 26, 51, 199, 244, 31, 52, 7, 143, 9, 32, + 117, 130, 1, 32, 90, 50, 210, 1, 78, 202, 1, 82, 60, 3, 83, 72, 50, 52, + 3, 90, 69, 78, 242, 239, 18, 71, 142, 255, 11, 68, 215, 127, 76, 4, 168, + 25, 11, 79, 86, 69, 82, 32, 69, 32, 78, 85, 78, 32, 253, 94, 4, 84, 73, + 77, 69, 19, 37, 7, 32, 84, 73, 77, 69, 83, 32, 16, 134, 1, 83, 144, 168, + 1, 10, 65, 32, 80, 76, 85, 83, 32, 72, 65, 32, 222, 165, 29, 71, 182, 16, + 80, 182, 26, 75, 254, 100, 77, 219, 19, 85, 4, 214, 229, 30, 65, 255, + 116, 72, 15, 11, 32, 12, 96, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 36, + 6, 84, 73, 77, 69, 83, 32, 247, 177, 24, 83, 2, 245, 160, 14, 4, 83, 73, + 78, 71, 6, 204, 138, 1, 4, 71, 65, 78, 50, 211, 206, 30, 77, 6, 36, 3, + 73, 78, 50, 187, 159, 31, 69, 5, 215, 224, 30, 32, 5, 229, 16, 9, 32, 67, + 82, 79, 83, 83, 73, 78, 71, 63, 11, 32, 60, 96, 14, 83, 72, 69, 83, 72, + 73, 71, 32, 84, 73, 77, 69, 83, 32, 97, 6, 84, 73, 77, 69, 83, 32, 16, + 178, 131, 1, 73, 254, 224, 2, 77, 170, 158, 26, 65, 220, 110, 2, 76, 65, + 198, 87, 83, 147, 17, 72, 44, 134, 1, 65, 68, 5, 68, 85, 78, 51, 32, 26, + 75, 58, 76, 62, 83, 34, 85, 222, 127, 73, 204, 31, 2, 72, 65, 242, 156, + 29, 71, 199, 103, 66, 9, 244, 86, 9, 32, 80, 76, 85, 83, 32, 76, 65, 76, + 131, 149, 31, 78, 4, 217, 32, 2, 71, 85, 6, 132, 173, 24, 5, 65, 83, 75, + 65, 76, 187, 195, 3, 85, 8, 34, 65, 194, 234, 31, 73, 3, 85, 5, 201, 85, + 2, 76, 32, 4, 210, 211, 31, 72, 215, 22, 85, 4, 134, 234, 31, 50, 3, 68, + 160, 2, 42, 65, 166, 19, 69, 122, 73, 135, 5, 85, 191, 1, 114, 50, 144, + 16, 2, 66, 65, 94, 68, 22, 76, 38, 78, 206, 140, 1, 32, 154, 6, 82, 166, + 175, 28, 83, 155, 149, 2, 77, 153, 1, 11, 32, 150, 1, 68, 6, 84, 73, 77, + 69, 83, 32, 153, 128, 1, 5, 79, 86, 69, 82, 32, 148, 1, 202, 1, 65, 162, + 2, 66, 154, 1, 68, 178, 1, 69, 58, 71, 178, 1, 72, 230, 1, 73, 70, 75, + 194, 1, 76, 62, 77, 50, 78, 130, 1, 83, 142, 1, 85, 222, 154, 1, 84, 204, + 150, 23, 3, 90, 73, 90, 139, 165, 7, 80, 18, 132, 1, 6, 32, 80, 76, 85, + 83, 32, 50, 78, 52, 2, 83, 72, 137, 175, 18, 14, 66, 50, 32, 84, 69, 78, + 85, 32, 80, 76, 85, 83, 32, 84, 6, 202, 58, 68, 142, 214, 13, 73, 131, + 210, 17, 72, 5, 185, 61, 9, 32, 80, 76, 85, 83, 32, 75, 65, 75, 7, 11, + 50, 5, 133, 90, 6, 32, 80, 76, 85, 83, 32, 10, 26, 65, 77, 2, 85, 82, 6, + 42, 72, 44, 2, 82, 32, 135, 227, 31, 68, 2, 11, 65, 2, 227, 187, 24, 82, + 5, 11, 32, 2, 169, 152, 31, 4, 80, 76, 85, 83, 14, 34, 73, 54, 85, 139, + 226, 31, 65, 7, 17, 2, 77, 32, 4, 146, 22, 84, 207, 129, 1, 71, 6, 56, 8, + 71, 32, 84, 73, 77, 69, 83, 32, 207, 225, 31, 66, 4, 158, 119, 73, 151, + 45, 75, 10, 38, 78, 254, 2, 76, 139, 136, 30, 82, 5, 243, 28, 32, 18, 18, + 65, 99, 73, 11, 26, 82, 247, 158, 1, 78, 7, 33, 6, 32, 80, 76, 85, 83, + 32, 4, 206, 201, 31, 78, 255, 2, 68, 9, 174, 122, 52, 253, 242, 12, 7, + 82, 50, 32, 80, 76, 85, 83, 12, 62, 65, 154, 124, 85, 233, 240, 12, 6, + 73, 32, 80, 76, 85, 83, 8, 40, 6, 32, 80, 76, 85, 83, 32, 83, 76, 4, 48, + 6, 76, 85, 32, 80, 76, 85, 171, 222, 31, 65, 2, 11, 83, 2, 215, 97, 32, + 5, 149, 236, 13, 5, 32, 80, 76, 85, 83, 4, 176, 100, 10, 83, 72, 32, 80, + 76, 85, 83, 32, 72, 85, 147, 15, 71, 12, 34, 65, 36, 2, 73, 68, 27, 85, + 4, 234, 224, 24, 83, 147, 252, 6, 75, 5, 225, 77, 2, 32, 80, 4, 60, 5, + 83, 72, 85, 50, 32, 197, 127, 5, 51, 32, 80, 76, 85, 2, 205, 158, 1, 3, + 80, 76, 85, 8, 26, 65, 183, 218, 31, 85, 7, 11, 77, 5, 227, 159, 1, 32, + 6, 250, 77, 69, 162, 162, 29, 85, 139, 235, 1, 73, 10, 26, 69, 73, 2, 85, + 78, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, 242, 178, 24, 69, 235, 147, 7, + 71, 5, 11, 32, 2, 187, 101, 79, 14, 42, 72, 254, 235, 23, 65, 247, 220, + 7, 85, 8, 18, 69, 51, 73, 5, 221, 224, 30, 7, 32, 80, 76, 85, 83, 32, 84, + 4, 130, 217, 31, 68, 3, 77, 7, 11, 68, 5, 237, 228, 13, 5, 32, 80, 76, + 85, 83, 7, 11, 32, 4, 206, 249, 29, 82, 177, 177, 1, 11, 67, 82, 79, 83, + 83, 73, 78, 71, 32, 71, 65, 5, 191, 131, 1, 32, 7, 134, 131, 1, 32, 231, + 195, 30, 65, 11, 11, 50, 9, 11, 32, 6, 80, 8, 67, 82, 79, 83, 83, 73, 78, + 71, 0, 4, 79, 86, 69, 82, 239, 207, 26, 84, 2, 197, 49, 3, 32, 71, 65, 8, + 44, 5, 83, 72, 84, 73, 78, 171, 174, 24, 50, 7, 37, 7, 32, 84, 73, 77, + 69, 83, 32, 4, 234, 220, 30, 75, 215, 120, 85, 49, 70, 32, 94, 52, 114, + 82, 190, 1, 83, 130, 207, 30, 68, 211, 130, 1, 71, 6, 152, 133, 9, 6, 84, + 73, 77, 69, 83, 32, 249, 250, 4, 8, 67, 82, 79, 83, 83, 73, 78, 71, 7, + 11, 32, 4, 64, 8, 67, 82, 79, 83, 83, 73, 78, 71, 1, 4, 79, 86, 69, 82, + 2, 197, 170, 24, 3, 32, 71, 73, 16, 26, 51, 147, 136, 1, 50, 13, 37, 7, + 32, 84, 73, 77, 69, 83, 32, 10, 70, 65, 0, 2, 76, 85, 206, 127, 71, 254, + 253, 12, 73, 131, 210, 17, 80, 2, 197, 253, 13, 7, 32, 80, 76, 85, 83, + 32, 73, 14, 26, 72, 231, 197, 30, 65, 13, 11, 32, 10, 22, 84, 247, 20, + 67, 8, 44, 5, 73, 77, 69, 83, 32, 239, 171, 31, 69, 6, 192, 20, 6, 71, + 73, 83, 72, 32, 67, 146, 126, 84, 227, 247, 29, 66, 43, 110, 50, 174, 1, + 68, 218, 1, 77, 54, 82, 152, 250, 13, 9, 32, 67, 82, 79, 83, 83, 73, 78, + 71, 223, 209, 17, 76, 15, 11, 32, 12, 48, 6, 84, 73, 77, 69, 83, 32, 167, + 132, 1, 71, 10, 252, 24, 3, 75, 65, 75, 194, 75, 73, 152, 20, 10, 83, 65, + 76, 32, 80, 76, 85, 83, 32, 84, 231, 213, 29, 78, 11, 11, 32, 8, 128, 1, + 10, 80, 76, 85, 83, 32, 71, 73, 83, 72, 32, 16, 6, 84, 73, 77, 69, 83, + 32, 177, 66, 8, 79, 86, 69, 82, 32, 71, 85, 68, 2, 135, 121, 84, 4, 172, + 145, 1, 5, 65, 32, 80, 76, 85, 223, 194, 29, 75, 5, 17, 2, 32, 84, 2, + 241, 42, 4, 73, 77, 69, 83, 9, 26, 85, 219, 203, 31, 55, 4, 198, 202, 31, + 83, 147, 1, 78, 50, 30, 65, 82, 73, 215, 1, 85, 11, 26, 32, 255, 202, 31, + 76, 6, 32, 2, 84, 69, 147, 128, 1, 71, 4, 235, 127, 78, 23, 37, 7, 32, + 84, 73, 77, 69, 83, 32, 20, 104, 3, 65, 83, 72, 234, 228, 27, 68, 158, + 228, 2, 78, 94, 75, 170, 57, 66, 2, 71, 162, 25, 83, 143, 45, 85, 7, 224, + 55, 8, 32, 79, 86, 69, 82, 32, 72, 73, 211, 145, 31, 50, 19, 48, 2, 66, + 50, 130, 161, 24, 76, 179, 166, 7, 83, 13, 37, 7, 32, 84, 73, 77, 69, 83, + 32, 10, 254, 138, 1, 75, 206, 216, 26, 76, 242, 216, 2, 72, 254, 59, 65, + 195, 9, 85, 49, 104, 3, 68, 73, 77, 94, 71, 214, 1, 76, 62, 77, 190, 193, + 31, 32, 170, 1, 83, 146, 1, 66, 2, 78, 3, 82, 7, 53, 11, 32, 79, 86, 69, + 82, 32, 73, 68, 73, 77, 32, 4, 166, 136, 24, 83, 175, 197, 6, 66, 13, 11, + 73, 11, 11, 32, 8, 162, 123, 71, 204, 244, 11, 31, 79, 86, 69, 82, 32, + 73, 71, 73, 32, 83, 72, 73, 82, 32, 79, 86, 69, 82, 32, 83, 72, 73, 82, + 32, 85, 68, 32, 79, 86, 69, 82, 250, 151, 17, 68, 175, 170, 1, 82, 7, 26, + 32, 151, 196, 31, 50, 2, 169, 72, 4, 84, 73, 77, 69, 13, 26, 32, 251, + 243, 30, 73, 8, 76, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 162, 111, + 84, 235, 149, 23, 83, 2, 181, 192, 30, 5, 83, 73, 78, 71, 32, 224, 1, 78, + 65, 146, 15, 73, 242, 1, 85, 158, 73, 69, 193, 188, 23, 4, 87, 85, 51, + 49, 177, 1, 164, 1, 7, 32, 84, 73, 77, 69, 83, 32, 210, 9, 50, 66, 68, + 102, 75, 50, 76, 98, 77, 28, 4, 83, 75, 65, 76, 184, 207, 15, 6, 80, 32, + 69, 76, 65, 77, 139, 229, 15, 66, 134, 1, 170, 1, 65, 94, 66, 82, 69, 30, + 71, 182, 2, 73, 34, 75, 34, 76, 38, 77, 170, 1, 83, 118, 84, 34, 85, 214, + 8, 72, 246, 51, 78, 162, 230, 16, 80, 198, 240, 13, 82, 147, 17, 90, 13, + 46, 68, 234, 145, 31, 78, 165, 44, 2, 83, 72, 5, 229, 50, 7, 32, 80, 76, + 85, 83, 32, 75, 10, 34, 65, 178, 190, 31, 73, 3, 85, 6, 194, 210, 29, 76, + 238, 235, 1, 68, 3, 82, 4, 138, 25, 82, 151, 61, 83, 26, 30, 65, 98, 73, + 147, 1, 85, 11, 38, 82, 206, 123, 78, 215, 193, 30, 76, 5, 249, 21, 10, + 32, 80, 76, 85, 83, 32, 83, 72, 65, 51, 11, 32, 2, 83, 72, 135, 149, 24, 82, 7, 11, 32, 4, 26, 67, 255, 130, 1, 80, 2, 37, 7, 82, 79, 83, 83, 73, - 78, 71, 2, 137, 185, 27, 2, 32, 71, 7, 182, 164, 27, 82, 179, 245, 3, 68, - 4, 234, 133, 31, 71, 219, 19, 77, 8, 250, 8, 73, 195, 164, 17, 65, 6, - 218, 163, 12, 85, 171, 245, 18, 73, 12, 18, 69, 83, 73, 9, 33, 6, 32, 80, - 76, 85, 83, 32, 6, 190, 243, 30, 68, 150, 14, 84, 255, 2, 71, 5, 45, 9, - 32, 80, 76, 85, 83, 32, 78, 85, 78, 2, 159, 244, 26, 85, 18, 62, 72, 178, - 196, 27, 65, 244, 217, 2, 2, 85, 72, 131, 120, 73, 10, 202, 154, 30, 85, - 142, 54, 73, 162, 70, 65, 3, 69, 4, 230, 221, 30, 65, 223, 56, 85, 17, - 110, 32, 238, 92, 82, 180, 142, 26, 9, 77, 85, 77, 32, 84, 73, 77, 69, - 83, 134, 169, 4, 83, 146, 1, 50, 3, 68, 2, 235, 137, 21, 85, 5, 197, 248, - 11, 11, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 75, 10, 42, 53, 182, 148, - 31, 50, 2, 51, 3, 52, 5, 129, 249, 23, 9, 32, 79, 86, 69, 82, 32, 75, 65, - 68, 5, 189, 75, 8, 32, 84, 73, 77, 69, 83, 32, 73, 9, 26, 32, 219, 130, - 31, 65, 4, 166, 105, 84, 249, 187, 23, 9, 67, 82, 79, 83, 83, 73, 78, 71, - 32, 4, 234, 146, 31, 50, 3, 52, 7, 11, 32, 4, 70, 76, 1, 13, 79, 86, 69, + 78, 71, 2, 165, 215, 27, 2, 32, 71, 7, 210, 193, 27, 82, 135, 250, 3, 68, + 4, 218, 167, 31, 71, 219, 19, 77, 8, 250, 8, 73, 135, 179, 17, 65, 6, + 202, 177, 12, 85, 171, 137, 19, 73, 12, 18, 69, 83, 73, 9, 33, 6, 32, 80, + 76, 85, 83, 32, 6, 174, 149, 31, 68, 150, 14, 84, 255, 2, 71, 5, 45, 9, + 32, 80, 76, 85, 83, 32, 78, 85, 78, 2, 147, 219, 29, 85, 18, 62, 72, 206, + 226, 27, 65, 200, 221, 2, 2, 85, 72, 131, 120, 73, 10, 186, 188, 30, 85, + 142, 54, 73, 162, 70, 65, 3, 69, 4, 214, 255, 30, 65, 223, 56, 85, 17, + 110, 32, 238, 92, 82, 200, 167, 26, 9, 77, 85, 77, 32, 84, 73, 77, 69, + 83, 226, 177, 4, 83, 146, 1, 50, 3, 68, 2, 207, 156, 21, 85, 5, 193, 129, + 12, 11, 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 75, 10, 42, 53, 166, 182, + 31, 50, 2, 51, 3, 52, 5, 221, 140, 24, 9, 32, 79, 86, 69, 82, 32, 75, 65, + 68, 5, 189, 75, 8, 32, 84, 73, 77, 69, 83, 32, 73, 9, 26, 32, 203, 164, + 31, 65, 4, 166, 105, 84, 233, 207, 23, 9, 67, 82, 79, 83, 83, 73, 78, 71, + 32, 4, 218, 180, 31, 50, 3, 52, 7, 11, 32, 4, 70, 76, 1, 13, 79, 86, 69, 82, 32, 75, 65, 83, 75, 65, 76, 32, 76, 2, 173, 116, 24, 65, 71, 65, 66, 32, 84, 73, 77, 69, 83, 32, 85, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, - 66, 32, 21, 68, 7, 32, 84, 73, 77, 69, 83, 32, 50, 83, 150, 144, 31, 68, - 3, 78, 6, 26, 85, 255, 201, 30, 66, 5, 167, 144, 31, 68, 8, 52, 3, 73, - 77, 53, 170, 134, 30, 65, 183, 137, 1, 72, 5, 169, 244, 23, 11, 32, 79, + 66, 32, 21, 68, 7, 32, 84, 73, 77, 69, 83, 32, 50, 83, 134, 178, 31, 68, + 3, 78, 6, 26, 85, 239, 235, 30, 66, 5, 151, 178, 31, 68, 8, 52, 3, 73, + 77, 53, 254, 165, 30, 65, 211, 139, 1, 72, 5, 133, 136, 24, 11, 32, 79, 86, 69, 82, 32, 75, 73, 83, 73, 77, 25, 200, 1, 29, 32, 79, 86, 69, 82, 32, 72, 73, 32, 84, 73, 77, 69, 83, 32, 65, 83, 72, 50, 32, 75, 85, 32, - 79, 86, 69, 82, 32, 72, 18, 52, 54, 82, 186, 98, 83, 230, 1, 76, 242, - 168, 30, 51, 2, 55, 3, 78, 2, 155, 71, 73, 5, 221, 142, 30, 8, 32, 86, + 79, 86, 69, 82, 32, 72, 18, 52, 54, 82, 186, 98, 83, 230, 1, 76, 226, + 202, 30, 51, 2, 55, 3, 78, 2, 155, 71, 73, 5, 205, 176, 30, 8, 32, 86, 65, 82, 73, 65, 78, 84, 5, 213, 115, 9, 32, 79, 80, 80, 79, 83, 73, 78, 71, 240, 2, 30, 65, 174, 26, 73, 59, 85, 141, 2, 68, 2, 71, 65, 252, 12, 2, 75, 45, 222, 11, 76, 46, 77, 135, 55, 72, 118, 22, 66, 131, 11, 82, - 109, 11, 32, 106, 48, 6, 84, 73, 77, 69, 83, 32, 167, 219, 23, 83, 104, + 109, 11, 32, 106, 48, 6, 84, 73, 77, 69, 83, 32, 131, 239, 23, 83, 104, 190, 1, 65, 186, 1, 66, 34, 71, 70, 72, 66, 73, 94, 75, 118, 76, 46, 77, - 46, 83, 138, 2, 84, 126, 85, 188, 165, 4, 8, 90, 85, 32, 79, 86, 69, 82, - 32, 214, 186, 25, 68, 218, 81, 69, 143, 57, 78, 15, 80, 6, 32, 80, 76, - 85, 83, 32, 76, 4, 83, 72, 32, 90, 178, 136, 31, 76, 3, 78, 6, 38, 68, - 158, 231, 29, 71, 251, 23, 76, 2, 201, 62, 5, 65, 32, 80, 76, 85, 2, 149, - 112, 2, 73, 68, 4, 246, 193, 30, 65, 163, 70, 73, 10, 48, 2, 85, 68, 254, - 180, 27, 65, 203, 210, 3, 73, 5, 191, 105, 32, 6, 164, 238, 17, 7, 73, - 32, 84, 73, 77, 69, 83, 227, 156, 12, 65, 8, 22, 77, 175, 62, 71, 7, 33, - 6, 32, 80, 76, 85, 83, 32, 4, 206, 225, 30, 76, 179, 34, 72, 10, 26, 85, - 251, 138, 29, 73, 6, 26, 76, 227, 133, 31, 51, 5, 41, 8, 32, 80, 76, 85, - 83, 32, 72, 73, 2, 219, 65, 32, 8, 198, 100, 65, 190, 184, 28, 73, 247, - 107, 85, 6, 26, 69, 191, 156, 29, 85, 5, 175, 25, 32, 12, 26, 72, 219, - 243, 30, 85, 10, 100, 14, 73, 84, 65, 32, 80, 76, 85, 83, 32, 71, 73, 83, + 46, 83, 138, 2, 84, 126, 85, 156, 172, 4, 8, 90, 85, 32, 79, 86, 69, 82, + 32, 226, 211, 25, 68, 222, 83, 69, 143, 57, 78, 15, 80, 6, 32, 80, 76, + 85, 83, 32, 76, 4, 83, 72, 32, 90, 162, 170, 31, 76, 3, 78, 6, 38, 68, + 138, 135, 30, 71, 227, 23, 76, 2, 201, 62, 5, 65, 32, 80, 76, 85, 2, 149, + 112, 2, 73, 68, 4, 230, 227, 30, 65, 163, 70, 73, 10, 48, 2, 85, 68, 154, + 211, 27, 65, 159, 214, 3, 73, 5, 191, 105, 32, 6, 204, 252, 17, 7, 73, + 32, 84, 73, 77, 69, 83, 171, 176, 12, 65, 8, 22, 77, 175, 62, 71, 7, 33, + 6, 32, 80, 76, 85, 83, 32, 4, 190, 131, 31, 76, 179, 34, 72, 10, 26, 85, + 131, 170, 29, 73, 6, 26, 76, 211, 167, 31, 51, 5, 41, 8, 32, 80, 76, 85, + 83, 32, 72, 73, 2, 219, 65, 32, 8, 198, 100, 65, 198, 215, 28, 73, 223, + 110, 85, 6, 26, 69, 199, 187, 29, 85, 5, 175, 25, 32, 12, 26, 72, 203, + 149, 31, 85, 10, 100, 14, 73, 84, 65, 32, 80, 76, 85, 83, 32, 71, 73, 83, 72, 32, 96, 2, 85, 50, 217, 3, 2, 69, 32, 4, 48, 6, 80, 76, 85, 83, 32, - 69, 163, 133, 26, 84, 2, 11, 82, 2, 11, 73, 2, 207, 233, 23, 78, 5, 189, + 69, 191, 158, 26, 84, 2, 11, 82, 2, 11, 73, 2, 171, 253, 23, 78, 5, 189, 73, 5, 32, 80, 76, 85, 83, 6, 86, 65, 189, 30, 16, 69, 32, 80, 76, 85, - 83, 32, 65, 32, 80, 76, 85, 83, 32, 83, 85, 4, 154, 231, 23, 75, 211, - 154, 7, 71, 13, 72, 6, 32, 80, 76, 85, 83, 32, 190, 41, 50, 178, 214, 30, - 83, 147, 1, 68, 4, 26, 85, 227, 128, 31, 65, 2, 255, 40, 32, 11, 11, 32, + 83, 32, 65, 32, 80, 76, 85, 83, 32, 83, 85, 4, 246, 250, 23, 75, 231, + 168, 7, 71, 13, 72, 6, 32, 80, 76, 85, 83, 32, 190, 41, 50, 162, 248, 30, + 83, 147, 1, 68, 4, 26, 85, 211, 162, 31, 65, 2, 255, 40, 32, 11, 11, 32, 8, 68, 4, 71, 85, 78, 85, 97, 9, 84, 73, 77, 69, 83, 32, 83, 72, 69, 5, 73, 16, 32, 79, 86, 69, 82, 32, 76, 65, 71, 65, 82, 32, 71, 85, 78, 85, - 2, 151, 210, 30, 32, 5, 11, 32, 2, 157, 196, 14, 4, 80, 76, 85, 83, 136, + 2, 135, 244, 30, 32, 5, 11, 32, 2, 201, 210, 14, 4, 80, 76, 85, 83, 136, 1, 82, 48, 146, 2, 49, 30, 50, 114, 51, 82, 52, 222, 2, 54, 154, 4, 55, 243, 24, 53, 22, 158, 1, 50, 30, 56, 180, 52, 15, 55, 57, 32, 79, 86, 69, - 82, 32, 76, 65, 75, 45, 48, 55, 57, 246, 245, 10, 53, 134, 185, 12, 54, - 2, 57, 94, 51, 203, 162, 3, 48, 4, 242, 252, 30, 49, 3, 53, 4, 136, 227, - 23, 12, 49, 32, 79, 86, 69, 82, 32, 76, 65, 75, 45, 48, 207, 153, 7, 48, - 4, 214, 226, 23, 52, 95, 51, 16, 46, 50, 38, 54, 222, 219, 23, 49, 159, - 2, 51, 6, 182, 251, 30, 48, 2, 53, 3, 56, 4, 146, 251, 30, 53, 3, 54, 12, - 42, 52, 254, 223, 11, 56, 131, 130, 12, 57, 6, 202, 250, 30, 51, 2, 55, - 3, 56, 30, 62, 52, 214, 1, 53, 30, 57, 170, 223, 23, 55, 203, 162, 3, 56, - 14, 26, 57, 207, 249, 30, 49, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, - 116, 9, 80, 65, 80, 32, 80, 76, 85, 83, 32, 240, 151, 13, 7, 85, 50, 32, - 80, 76, 85, 83, 238, 31, 73, 235, 155, 17, 71, 4, 210, 13, 80, 51, 76, 4, - 146, 248, 30, 48, 3, 55, 8, 246, 247, 30, 48, 2, 50, 2, 51, 3, 53, 46, - 60, 2, 49, 55, 240, 1, 2, 52, 56, 130, 216, 23, 48, 23, 51, 23, 37, 7, + 82, 32, 76, 65, 75, 45, 48, 55, 57, 254, 254, 10, 53, 218, 195, 12, 54, + 2, 57, 94, 51, 139, 172, 3, 48, 4, 226, 158, 31, 49, 3, 53, 4, 228, 246, + 23, 12, 49, 32, 79, 86, 69, 82, 32, 76, 65, 75, 45, 48, 227, 167, 7, 48, + 4, 178, 246, 23, 52, 95, 51, 16, 46, 50, 38, 54, 186, 239, 23, 49, 159, + 2, 51, 6, 166, 157, 31, 48, 2, 53, 3, 56, 4, 130, 157, 31, 53, 3, 54, 12, + 42, 52, 250, 232, 11, 56, 227, 140, 12, 57, 6, 186, 156, 31, 51, 2, 55, + 3, 56, 30, 62, 52, 214, 1, 53, 30, 57, 134, 243, 23, 55, 139, 172, 3, 56, + 14, 26, 57, 191, 155, 31, 49, 13, 37, 7, 32, 84, 73, 77, 69, 83, 32, 10, + 116, 9, 80, 65, 80, 32, 80, 76, 85, 83, 32, 140, 166, 13, 7, 85, 50, 32, + 80, 76, 85, 83, 218, 31, 73, 211, 175, 17, 71, 4, 210, 13, 80, 51, 76, 4, + 130, 154, 31, 48, 3, 55, 8, 230, 153, 31, 48, 2, 50, 2, 51, 3, 53, 46, + 60, 2, 49, 55, 240, 1, 2, 52, 56, 222, 235, 23, 48, 23, 51, 23, 37, 7, 32, 84, 73, 77, 69, 83, 32, 20, 122, 84, 34, 85, 162, 11, 75, 132, 34, 9, - 68, 85, 78, 51, 32, 71, 85, 78, 85, 134, 224, 28, 65, 246, 161, 1, 66, - 247, 67, 76, 4, 138, 189, 30, 65, 223, 56, 69, 6, 178, 199, 14, 82, 150, - 174, 16, 50, 3, 68, 21, 37, 7, 32, 84, 73, 77, 69, 83, 32, 18, 154, 1, - 85, 220, 8, 4, 80, 65, 80, 32, 146, 45, 73, 196, 219, 12, 10, 83, 72, 69, - 83, 72, 32, 80, 76, 85, 83, 210, 132, 5, 68, 130, 202, 12, 78, 163, 17, - 71, 4, 194, 197, 14, 82, 151, 174, 16, 68, 4, 226, 216, 23, 50, 143, 165, + 68, 85, 78, 51, 32, 71, 85, 78, 85, 142, 255, 28, 65, 222, 164, 1, 66, + 247, 67, 76, 4, 250, 222, 30, 65, 223, 56, 69, 6, 222, 213, 14, 82, 218, + 193, 16, 50, 3, 68, 21, 37, 7, 32, 84, 73, 77, 69, 83, 32, 18, 154, 1, + 85, 220, 8, 4, 80, 65, 80, 32, 146, 45, 73, 224, 233, 12, 10, 83, 72, 69, + 83, 72, 32, 80, 76, 85, 83, 254, 132, 5, 68, 170, 221, 12, 78, 163, 17, + 71, 4, 238, 211, 14, 82, 219, 193, 16, 68, 4, 190, 236, 23, 50, 207, 174, 3, 52, 5, 11, 32, 2, 145, 6, 4, 84, 73, 77, 69, 7, 49, 10, 32, 84, 73, - 77, 69, 83, 32, 75, 85, 82, 5, 225, 206, 11, 5, 32, 80, 76, 85, 83, 9, - 204, 206, 11, 2, 77, 77, 170, 162, 19, 83, 147, 1, 76, 93, 90, 50, 212, - 7, 3, 71, 65, 76, 142, 1, 77, 130, 62, 32, 150, 170, 30, 51, 2, 72, 3, + 77, 69, 83, 32, 75, 85, 82, 5, 221, 215, 11, 5, 32, 80, 76, 85, 83, 9, + 200, 215, 11, 2, 77, 77, 158, 187, 19, 83, 147, 1, 76, 93, 90, 50, 212, + 7, 3, 71, 65, 76, 142, 1, 77, 130, 62, 32, 134, 204, 30, 51, 2, 72, 3, 76, 69, 11, 32, 66, 88, 4, 67, 82, 79, 83, 0, 4, 79, 80, 80, 79, 44, 4, - 71, 85, 78, 85, 38, 83, 35, 84, 2, 209, 204, 11, 6, 83, 73, 78, 71, 32, - 76, 2, 193, 24, 5, 32, 84, 73, 77, 69, 6, 250, 68, 72, 227, 250, 22, 81, - 54, 44, 5, 73, 77, 69, 83, 32, 251, 201, 30, 69, 52, 188, 1, 4, 69, 83, + 71, 85, 78, 85, 38, 83, 35, 84, 2, 205, 213, 11, 6, 83, 73, 78, 71, 32, + 76, 2, 193, 24, 5, 32, 84, 73, 77, 69, 6, 250, 68, 72, 191, 142, 23, 81, + 54, 44, 5, 73, 77, 69, 83, 32, 235, 235, 30, 69, 52, 188, 1, 4, 69, 83, 72, 50, 94, 72, 42, 75, 68, 2, 76, 65, 34, 77, 60, 3, 80, 65, 80, 118, - 83, 90, 84, 250, 56, 71, 158, 251, 7, 78, 178, 212, 18, 68, 198, 215, 2, - 65, 138, 67, 66, 215, 53, 73, 7, 11, 32, 4, 26, 80, 147, 239, 25, 84, 2, - 17, 2, 76, 85, 2, 173, 227, 29, 3, 83, 32, 76, 4, 184, 66, 2, 73, 32, - 227, 160, 29, 65, 8, 32, 2, 65, 68, 135, 236, 30, 73, 6, 226, 19, 51, - 163, 216, 30, 50, 4, 174, 20, 32, 219, 181, 17, 71, 2, 11, 69, 2, 11, 32, - 2, 241, 175, 13, 4, 80, 76, 85, 83, 5, 11, 32, 2, 33, 6, 80, 76, 85, 83, - 32, 80, 2, 45, 9, 65, 80, 32, 80, 76, 85, 83, 32, 76, 2, 159, 244, 26, - 85, 6, 26, 73, 147, 197, 30, 72, 4, 194, 18, 32, 181, 171, 26, 7, 75, 50, + 83, 90, 84, 250, 56, 71, 250, 130, 8, 78, 242, 234, 18, 68, 254, 216, 2, + 65, 166, 69, 66, 215, 53, 73, 7, 11, 32, 4, 26, 80, 175, 136, 26, 84, 2, + 17, 2, 76, 85, 2, 129, 131, 30, 3, 83, 32, 76, 4, 184, 66, 2, 73, 32, + 183, 192, 29, 65, 8, 32, 2, 65, 68, 247, 141, 31, 73, 6, 226, 19, 51, + 147, 250, 30, 50, 4, 174, 20, 32, 131, 196, 17, 71, 2, 11, 69, 2, 11, 32, + 2, 249, 189, 13, 4, 80, 76, 85, 83, 5, 11, 32, 2, 33, 6, 80, 76, 85, 83, + 32, 80, 2, 45, 9, 65, 80, 32, 80, 76, 85, 83, 32, 76, 2, 187, 145, 27, + 85, 6, 26, 73, 131, 231, 30, 72, 4, 194, 18, 32, 201, 196, 26, 7, 75, 50, 32, 80, 76, 85, 83, 4, 162, 53, 85, 139, 24, 65, 9, 11, 32, 6, 22, 79, 207, 67, 83, 4, 56, 7, 80, 80, 79, 83, 73, 78, 71, 1, 3, 86, 69, 82, 2, - 21, 3, 32, 76, 85, 2, 231, 222, 29, 71, 7, 45, 9, 32, 79, 86, 69, 82, 32, + 21, 3, 32, 76, 85, 2, 187, 254, 29, 71, 7, 45, 9, 32, 79, 86, 69, 82, 32, 76, 85, 77, 5, 187, 59, 32, 70, 34, 65, 70, 69, 22, 73, 71, 85, 17, 170, - 45, 32, 188, 1, 2, 83, 72, 202, 184, 30, 50, 2, 72, 3, 82, 7, 207, 198, - 26, 83, 7, 164, 241, 26, 8, 32, 80, 76, 85, 83, 32, 90, 65, 179, 245, 3, - 78, 43, 104, 2, 83, 72, 186, 60, 71, 144, 133, 11, 5, 32, 79, 86, 69, 82, - 40, 2, 82, 71, 225, 197, 6, 2, 78, 83, 31, 22, 32, 183, 2, 51, 16, 136, + 45, 32, 188, 1, 2, 83, 72, 186, 218, 30, 50, 2, 72, 3, 82, 7, 251, 247, + 26, 83, 7, 192, 142, 27, 8, 32, 80, 76, 85, 83, 32, 90, 65, 135, 250, 3, + 78, 43, 104, 2, 83, 72, 186, 60, 71, 140, 142, 11, 5, 32, 79, 86, 69, 82, + 40, 2, 82, 71, 173, 203, 6, 2, 78, 83, 31, 22, 32, 183, 2, 51, 16, 136, 1, 9, 79, 86, 69, 82, 32, 77, 85, 83, 72, 124, 6, 84, 73, 77, 69, 83, 32, - 209, 216, 26, 10, 67, 82, 79, 83, 83, 73, 78, 71, 32, 77, 9, 37, 7, 32, - 84, 73, 77, 69, 83, 32, 6, 42, 65, 238, 173, 28, 75, 143, 179, 2, 71, 2, - 241, 188, 11, 5, 32, 80, 76, 85, 83, 6, 178, 234, 29, 75, 158, 118, 90, + 237, 245, 26, 10, 67, 82, 79, 83, 83, 73, 78, 71, 32, 77, 9, 37, 7, 32, + 84, 73, 77, 69, 83, 32, 6, 42, 65, 154, 204, 28, 75, 211, 182, 2, 71, 2, + 237, 197, 11, 5, 32, 80, 76, 85, 83, 6, 162, 140, 30, 75, 158, 118, 90, 187, 2, 65, 13, 11, 32, 10, 44, 6, 84, 73, 77, 69, 83, 32, 203, 57, 71, - 8, 38, 65, 162, 206, 30, 68, 163, 17, 90, 5, 205, 129, 13, 5, 32, 80, 76, + 8, 38, 65, 146, 240, 30, 68, 163, 17, 90, 5, 233, 143, 13, 5, 32, 80, 76, 85, 83, 158, 1, 42, 65, 134, 2, 69, 94, 73, 199, 7, 85, 23, 52, 2, 71, - 65, 166, 1, 77, 198, 223, 30, 50, 3, 52, 11, 26, 32, 207, 224, 30, 82, 6, + 65, 166, 1, 77, 182, 129, 31, 50, 3, 52, 11, 26, 32, 191, 130, 31, 82, 6, 100, 8, 79, 80, 80, 79, 83, 73, 78, 71, 146, 59, 73, 197, 14, 9, 84, 73, - 77, 69, 83, 32, 83, 72, 85, 2, 157, 157, 30, 3, 32, 78, 65, 7, 204, 21, - 2, 32, 78, 247, 201, 30, 50, 9, 11, 32, 6, 44, 6, 84, 73, 77, 69, 83, 32, - 179, 57, 83, 4, 190, 152, 30, 85, 163, 70, 65, 73, 90, 77, 78, 78, 168, - 182, 20, 6, 32, 84, 73, 77, 69, 83, 238, 189, 8, 83, 135, 233, 1, 50, 7, + 77, 69, 83, 32, 83, 72, 85, 2, 141, 191, 30, 3, 32, 78, 65, 7, 204, 21, + 2, 32, 78, 231, 235, 30, 50, 9, 11, 32, 6, 44, 6, 84, 73, 77, 69, 83, 32, + 179, 57, 83, 4, 174, 186, 30, 85, 163, 70, 65, 73, 90, 77, 78, 78, 212, + 198, 20, 6, 32, 84, 73, 77, 69, 83, 202, 204, 8, 83, 239, 235, 1, 50, 7, 45, 9, 32, 84, 73, 77, 69, 83, 32, 71, 65, 4, 158, 3, 82, 179, 58, 78, - 59, 36, 3, 68, 65, 50, 243, 220, 30, 57, 55, 37, 7, 32, 84, 73, 77, 69, + 59, 36, 3, 68, 65, 50, 227, 254, 30, 57, 55, 37, 7, 32, 84, 73, 77, 69, 83, 32, 52, 162, 1, 65, 34, 71, 50, 75, 16, 5, 76, 65, 75, 45, 48, 22, 77, 86, 78, 34, 80, 76, 3, 83, 72, 69, 94, 85, 240, 15, 3, 68, 73, 77, - 230, 190, 29, 66, 235, 117, 72, 6, 246, 2, 83, 175, 216, 30, 78, 8, 26, - 73, 183, 216, 29, 85, 5, 215, 217, 30, 83, 2, 211, 20, 69, 2, 243, 193, - 23, 53, 4, 26, 69, 247, 241, 28, 65, 2, 25, 4, 32, 80, 76, 85, 2, 177, - 41, 3, 83, 32, 71, 4, 250, 137, 30, 85, 227, 79, 69, 2, 33, 6, 65, 80, - 32, 80, 76, 85, 2, 11, 83, 2, 157, 200, 29, 2, 32, 80, 9, 37, 7, 32, 80, - 76, 85, 83, 32, 65, 6, 26, 83, 175, 142, 29, 32, 4, 11, 72, 5, 107, 32, - 11, 50, 32, 34, 50, 174, 169, 14, 82, 135, 173, 16, 83, 2, 173, 142, 14, - 3, 80, 76, 85, 2, 11, 32, 2, 21, 3, 80, 76, 85, 2, 11, 83, 2, 227, 238, - 28, 32, 57, 24, 2, 49, 49, 99, 78, 9, 11, 32, 6, 200, 25, 9, 79, 86, 69, - 82, 32, 78, 85, 49, 49, 150, 191, 25, 84, 207, 161, 3, 82, 47, 30, 32, + 186, 222, 29, 66, 135, 120, 72, 6, 246, 2, 83, 159, 250, 30, 78, 8, 26, + 73, 167, 250, 29, 85, 5, 199, 251, 30, 83, 2, 211, 20, 69, 2, 207, 213, + 23, 53, 4, 26, 69, 255, 144, 29, 65, 2, 25, 4, 32, 80, 76, 85, 2, 177, + 41, 3, 83, 32, 71, 4, 234, 171, 30, 85, 227, 79, 69, 2, 33, 6, 65, 80, + 32, 80, 76, 85, 2, 11, 83, 2, 241, 231, 29, 2, 32, 80, 9, 37, 7, 32, 80, + 76, 85, 83, 32, 65, 6, 26, 83, 131, 174, 29, 32, 4, 11, 72, 5, 107, 32, + 11, 50, 32, 34, 50, 218, 183, 14, 82, 203, 192, 16, 83, 2, 217, 156, 14, + 3, 80, 76, 85, 2, 11, 32, 2, 21, 3, 80, 76, 85, 2, 11, 83, 2, 235, 141, + 29, 32, 57, 24, 2, 49, 49, 99, 78, 9, 11, 32, 6, 200, 25, 9, 79, 86, 69, + 82, 32, 78, 85, 49, 49, 178, 216, 25, 84, 243, 167, 3, 82, 47, 30, 32, 169, 3, 2, 85, 90, 18, 144, 1, 12, 67, 82, 79, 83, 83, 73, 78, 71, 32, 78, 85, 78, 72, 12, 76, 65, 71, 65, 82, 32, 84, 73, 77, 69, 83, 32, 174, - 1, 79, 231, 212, 25, 84, 5, 209, 40, 14, 32, 76, 65, 71, 65, 82, 32, 79, - 86, 69, 82, 32, 76, 65, 10, 56, 3, 83, 65, 76, 158, 235, 28, 77, 14, 85, - 235, 70, 71, 5, 133, 202, 29, 23, 32, 79, 86, 69, 82, 32, 78, 85, 78, 32, - 76, 65, 71, 65, 82, 32, 84, 73, 77, 69, 83, 32, 83, 2, 201, 185, 17, 3, + 1, 79, 131, 238, 25, 84, 5, 209, 40, 14, 32, 76, 65, 71, 65, 82, 32, 79, + 86, 69, 82, 32, 76, 65, 10, 56, 3, 83, 65, 76, 166, 138, 29, 77, 14, 85, + 207, 71, 71, 5, 217, 233, 29, 23, 32, 79, 86, 69, 82, 32, 78, 85, 78, 32, + 76, 65, 71, 65, 82, 32, 84, 73, 77, 69, 83, 32, 83, 2, 241, 199, 17, 3, 86, 69, 82, 27, 11, 32, 24, 120, 10, 65, 66, 50, 32, 84, 73, 77, 69, 83, 32, 205, 37, 15, 75, 73, 83, 73, 77, 53, 32, 84, 73, 77, 69, 83, 32, 66, - 73, 20, 168, 1, 2, 75, 65, 202, 7, 73, 212, 39, 2, 65, 83, 130, 177, 2, - 68, 232, 206, 8, 3, 83, 73, 76, 214, 135, 12, 85, 210, 237, 5, 71, 138, - 149, 1, 78, 254, 2, 66, 163, 17, 76, 2, 251, 217, 26, 68, 46, 42, 65, 36, - 4, 69, 83, 72, 50, 23, 73, 9, 194, 207, 30, 68, 2, 78, 3, 80, 5, 191, - 187, 28, 32, 35, 22, 32, 151, 1, 82, 20, 80, 6, 84, 73, 77, 69, 83, 32, - 201, 131, 18, 8, 67, 82, 79, 83, 83, 73, 78, 71, 18, 214, 21, 85, 150, - 48, 65, 2, 73, 254, 199, 29, 66, 187, 64, 69, 12, 32, 2, 73, 71, 191, - 205, 30, 50, 11, 11, 32, 8, 96, 6, 84, 73, 77, 69, 83, 32, 197, 153, 26, - 12, 79, 80, 80, 79, 83, 73, 78, 71, 32, 80, 73, 82, 6, 138, 195, 29, 75, - 162, 67, 85, 235, 67, 90, 8, 234, 67, 65, 182, 136, 30, 73, 3, 85, 202, + 73, 20, 168, 1, 2, 75, 65, 202, 7, 73, 212, 39, 2, 65, 83, 190, 177, 2, + 68, 168, 215, 8, 3, 83, 73, 76, 182, 146, 12, 85, 210, 249, 5, 71, 158, + 151, 1, 78, 254, 2, 66, 163, 17, 76, 2, 151, 247, 26, 68, 46, 42, 65, 36, + 4, 69, 83, 72, 50, 23, 73, 9, 178, 241, 30, 68, 2, 78, 3, 80, 5, 211, + 217, 28, 32, 35, 22, 32, 151, 1, 82, 20, 80, 6, 84, 73, 77, 69, 83, 32, + 141, 146, 18, 8, 67, 82, 79, 83, 83, 73, 78, 71, 18, 214, 21, 85, 150, + 48, 65, 2, 73, 238, 233, 29, 66, 187, 64, 69, 12, 32, 2, 73, 71, 175, + 239, 30, 50, 11, 11, 32, 8, 96, 6, 84, 73, 77, 69, 83, 32, 217, 178, 26, + 12, 79, 80, 80, 79, 83, 73, 78, 71, 32, 80, 73, 82, 6, 222, 226, 29, 75, + 190, 69, 85, 235, 67, 90, 8, 234, 67, 65, 166, 170, 30, 73, 3, 85, 202, 1, 46, 65, 250, 5, 72, 150, 11, 73, 163, 1, 85, 63, 46, 71, 190, 4, 76, - 122, 78, 227, 197, 30, 82, 53, 11, 32, 50, 92, 4, 71, 85, 78, 85, 54, 78, + 122, 78, 211, 231, 30, 82, 53, 11, 32, 50, 92, 4, 71, 85, 78, 85, 54, 78, 32, 6, 84, 73, 77, 69, 83, 32, 241, 21, 4, 79, 86, 69, 82, 5, 29, 5, 32, - 84, 73, 77, 69, 2, 159, 230, 17, 83, 2, 149, 164, 25, 3, 85, 84, 73, 42, + 84, 73, 77, 69, 2, 231, 244, 17, 83, 2, 185, 189, 25, 3, 85, 84, 73, 42, 150, 1, 73, 42, 75, 34, 83, 64, 2, 84, 65, 38, 85, 220, 62, 2, 68, 85, - 166, 223, 28, 76, 226, 40, 78, 210, 48, 69, 138, 60, 77, 162, 17, 72, - 187, 2, 65, 2, 11, 71, 2, 11, 73, 2, 191, 31, 32, 4, 182, 143, 30, 85, - 199, 53, 65, 6, 26, 72, 167, 190, 29, 65, 4, 162, 192, 13, 69, 151, 193, - 16, 73, 4, 226, 172, 23, 75, 211, 154, 7, 66, 10, 254, 197, 30, 83, 146, + 130, 255, 28, 76, 246, 42, 78, 210, 48, 69, 138, 60, 77, 162, 17, 72, + 187, 2, 65, 2, 11, 71, 2, 11, 73, 2, 191, 31, 32, 4, 166, 177, 30, 85, + 199, 53, 65, 6, 26, 72, 251, 221, 29, 65, 4, 198, 206, 13, 69, 227, 212, + 16, 73, 4, 190, 192, 23, 75, 231, 168, 7, 66, 10, 238, 231, 30, 83, 146, 1, 50, 2, 66, 2, 77, 3, 82, 5, 33, 6, 32, 76, 65, 71, 65, 66, 2, 37, 7, - 32, 84, 73, 77, 69, 83, 32, 2, 11, 65, 2, 11, 83, 2, 199, 172, 23, 72, 2, - 135, 169, 11, 71, 106, 46, 65, 250, 1, 69, 242, 2, 73, 239, 3, 85, 29, - 50, 51, 182, 1, 54, 174, 166, 23, 66, 223, 3, 82, 19, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 16, 90, 85, 146, 25, 83, 222, 201, 26, 71, 166, 232, 2, - 84, 170, 50, 66, 218, 47, 78, 215, 22, 65, 5, 11, 32, 2, 181, 131, 26, 4, - 80, 76, 85, 83, 5, 175, 45, 32, 27, 62, 32, 140, 2, 2, 83, 72, 150, 203, - 26, 71, 199, 245, 3, 78, 14, 84, 8, 79, 86, 69, 82, 32, 83, 72, 69, 88, - 5, 80, 76, 85, 83, 32, 207, 156, 30, 72, 7, 11, 32, 4, 190, 15, 71, 141, + 32, 84, 73, 77, 69, 83, 32, 2, 11, 65, 2, 11, 83, 2, 163, 192, 23, 72, 2, + 131, 178, 11, 71, 106, 46, 65, 250, 1, 69, 242, 2, 73, 239, 3, 85, 29, + 50, 51, 182, 1, 54, 138, 186, 23, 66, 223, 3, 82, 19, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 16, 90, 85, 146, 25, 83, 250, 231, 26, 71, 250, 235, 2, + 84, 170, 50, 66, 218, 47, 78, 215, 22, 65, 5, 11, 32, 2, 201, 156, 26, 4, + 80, 76, 85, 83, 5, 175, 45, 32, 27, 62, 32, 140, 2, 2, 83, 72, 178, 232, + 26, 71, 155, 250, 3, 78, 14, 84, 8, 79, 86, 69, 82, 32, 83, 72, 69, 88, + 5, 80, 76, 85, 83, 32, 191, 190, 30, 72, 7, 11, 32, 4, 190, 15, 71, 141, 6, 12, 84, 65, 66, 32, 79, 86, 69, 82, 32, 84, 65, 66, 6, 48, 2, 72, 85, - 20, 2, 78, 65, 183, 159, 29, 83, 2, 211, 167, 23, 66, 2, 191, 167, 23, - 77, 7, 218, 133, 29, 76, 255, 186, 1, 50, 40, 62, 68, 74, 77, 218, 1, 82, - 206, 138, 26, 78, 211, 176, 4, 84, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, - 4, 242, 174, 30, 73, 219, 16, 65, 25, 37, 7, 32, 84, 73, 77, 69, 83, 32, - 22, 114, 66, 38, 73, 130, 19, 75, 134, 187, 2, 77, 178, 211, 2, 76, 130, - 251, 23, 71, 250, 23, 83, 238, 9, 68, 215, 127, 65, 4, 154, 206, 2, 85, - 195, 230, 26, 65, 4, 249, 20, 2, 71, 73, 7, 11, 32, 4, 60, 9, 79, 86, 69, - 82, 32, 83, 72, 73, 82, 151, 191, 25, 84, 2, 177, 196, 29, 11, 32, 66, + 20, 2, 78, 65, 163, 191, 29, 83, 2, 175, 187, 23, 66, 2, 155, 187, 23, + 77, 7, 182, 165, 29, 76, 147, 189, 1, 50, 40, 62, 68, 74, 77, 218, 1, 82, + 226, 163, 26, 78, 175, 185, 4, 84, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, + 4, 226, 208, 30, 73, 219, 16, 65, 25, 37, 7, 32, 84, 73, 77, 69, 83, 32, + 22, 114, 66, 38, 73, 130, 19, 75, 194, 187, 2, 77, 234, 217, 2, 76, 250, + 147, 24, 71, 226, 23, 83, 138, 12, 68, 215, 127, 65, 4, 214, 206, 2, 85, + 219, 133, 27, 65, 4, 249, 20, 2, 71, 73, 7, 11, 32, 4, 60, 9, 79, 86, 69, + 82, 32, 83, 72, 73, 82, 179, 216, 25, 84, 2, 161, 230, 29, 11, 32, 66, 85, 82, 32, 79, 86, 69, 82, 32, 66, 13, 88, 14, 32, 79, 86, 69, 82, 32, - 73, 78, 86, 69, 82, 84, 69, 68, 34, 50, 251, 194, 29, 66, 2, 11, 32, 2, - 235, 205, 25, 83, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, 88, 7, 69, 50, 32, - 84, 73, 77, 69, 177, 152, 11, 9, 68, 85, 71, 32, 84, 73, 77, 69, 83, 2, - 131, 132, 13, 83, 17, 50, 32, 30, 71, 234, 152, 11, 76, 215, 135, 12, 75, + 73, 78, 86, 69, 82, 84, 69, 68, 34, 50, 235, 228, 29, 66, 2, 11, 32, 2, + 135, 231, 25, 83, 7, 33, 6, 32, 80, 76, 85, 83, 32, 4, 88, 7, 69, 50, 32, + 84, 73, 77, 69, 173, 161, 11, 9, 68, 85, 71, 32, 84, 73, 77, 69, 83, 2, + 139, 146, 13, 83, 17, 50, 32, 30, 71, 230, 161, 11, 76, 183, 146, 12, 75, 4, 138, 8, 84, 163, 9, 71, 7, 11, 52, 5, 49, 10, 32, 79, 86, 69, 82, 32, - 83, 73, 71, 52, 2, 199, 14, 32, 19, 78, 68, 22, 77, 22, 82, 156, 217, 12, - 5, 32, 79, 86, 69, 82, 179, 230, 16, 72, 5, 199, 184, 30, 50, 5, 147, - 208, 28, 65, 5, 159, 184, 30, 57, 72, 46, 65, 150, 4, 73, 250, 1, 85, - 227, 8, 69, 37, 66, 32, 94, 66, 166, 1, 71, 148, 1, 2, 75, 52, 135, 180, - 30, 82, 8, 60, 6, 84, 73, 77, 69, 83, 32, 130, 14, 71, 135, 149, 28, 65, - 4, 254, 162, 30, 72, 3, 77, 7, 11, 32, 4, 180, 203, 17, 29, 79, 86, 69, + 83, 73, 71, 52, 2, 199, 14, 32, 19, 78, 68, 22, 77, 22, 82, 184, 231, 12, + 5, 32, 79, 86, 69, 82, 135, 250, 16, 72, 5, 183, 218, 30, 50, 5, 155, + 239, 28, 65, 5, 143, 218, 30, 57, 72, 46, 65, 150, 4, 73, 250, 1, 85, + 227, 8, 69, 37, 66, 32, 94, 66, 166, 1, 71, 148, 1, 2, 75, 52, 247, 213, + 30, 82, 8, 60, 6, 84, 73, 77, 69, 83, 32, 130, 14, 71, 155, 179, 28, 65, + 4, 238, 196, 30, 72, 3, 77, 7, 11, 32, 4, 252, 217, 17, 29, 79, 86, 69, 82, 32, 84, 65, 66, 32, 78, 73, 32, 79, 86, 69, 82, 32, 78, 73, 32, 68, - 73, 83, 72, 32, 79, 86, 69, 82, 143, 187, 5, 83, 15, 37, 7, 32, 84, 73, - 77, 69, 83, 32, 12, 74, 84, 252, 254, 7, 2, 83, 72, 206, 137, 21, 71, - 190, 101, 85, 203, 50, 66, 2, 11, 85, 2, 235, 154, 23, 71, 5, 29, 5, 32, - 80, 76, 85, 83, 2, 221, 202, 28, 2, 32, 83, 17, 46, 82, 150, 29, 32, 134, - 150, 30, 50, 3, 76, 9, 11, 32, 6, 48, 8, 79, 86, 69, 82, 32, 84, 73, 82, + 73, 83, 72, 32, 79, 86, 69, 82, 163, 192, 5, 83, 15, 37, 7, 32, 84, 73, + 77, 69, 83, 32, 12, 74, 84, 216, 134, 8, 2, 83, 72, 206, 161, 21, 71, + 210, 103, 85, 203, 50, 66, 2, 11, 85, 2, 199, 174, 23, 71, 5, 29, 5, 32, + 80, 76, 85, 83, 2, 229, 233, 28, 2, 32, 83, 17, 46, 82, 150, 29, 32, 246, + 183, 30, 50, 3, 76, 9, 11, 32, 6, 48, 8, 79, 86, 69, 82, 32, 84, 73, 82, 99, 84, 5, 11, 32, 2, 11, 71, 2, 21, 3, 65, 68, 32, 2, 241, 5, 8, 79, 86, 69, 82, 32, 71, 65, 68, 2, 217, 21, 6, 73, 77, 69, 83, 32, 84, 17, 50, - 77, 114, 82, 130, 151, 23, 71, 175, 153, 7, 75, 7, 37, 7, 32, 84, 73, 77, - 69, 83, 32, 4, 46, 71, 205, 197, 17, 5, 84, 72, 82, 69, 69, 2, 221, 16, - 2, 65, 78, 5, 141, 139, 11, 17, 32, 79, 86, 69, 82, 32, 84, 85, 82, 32, + 77, 114, 82, 222, 170, 23, 71, 195, 167, 7, 75, 7, 37, 7, 32, 84, 73, 77, + 69, 83, 32, 4, 46, 71, 149, 212, 17, 5, 84, 72, 82, 69, 69, 2, 221, 16, + 2, 65, 78, 5, 137, 148, 11, 17, 32, 79, 86, 69, 82, 32, 84, 85, 82, 32, 90, 65, 32, 79, 86, 69, 82, 179, 1, 138, 1, 32, 246, 2, 68, 226, 2, 78, - 46, 77, 158, 2, 82, 128, 9, 2, 83, 72, 174, 1, 90, 224, 185, 12, 2, 84, - 85, 158, 226, 17, 50, 3, 66, 12, 64, 7, 79, 86, 69, 82, 32, 85, 32, 158, - 2, 85, 139, 128, 29, 71, 6, 200, 1, 10, 80, 65, 32, 79, 86, 69, 82, 32, - 80, 65, 216, 159, 9, 19, 85, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, - 86, 69, 82, 32, 85, 32, 82, 137, 148, 20, 10, 83, 85, 82, 32, 79, 86, 69, + 46, 77, 158, 2, 82, 128, 9, 2, 83, 72, 174, 1, 90, 252, 199, 12, 2, 84, + 85, 242, 245, 17, 50, 3, 66, 12, 64, 7, 79, 86, 69, 82, 32, 85, 32, 158, + 2, 85, 231, 159, 29, 71, 6, 200, 1, 10, 80, 65, 32, 79, 86, 69, 82, 32, + 80, 65, 220, 167, 9, 19, 85, 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, + 86, 69, 82, 32, 85, 32, 82, 245, 173, 20, 10, 83, 85, 82, 32, 79, 86, 69, 82, 32, 83, 2, 11, 32, 2, 45, 9, 71, 65, 82, 32, 79, 86, 69, 82, 32, 2, - 191, 138, 29, 71, 5, 251, 134, 30, 32, 21, 26, 32, 227, 170, 30, 85, 16, + 171, 170, 29, 71, 5, 235, 168, 30, 32, 21, 26, 32, 211, 204, 30, 85, 16, 70, 75, 44, 2, 83, 72, 100, 6, 84, 73, 77, 69, 83, 32, 135, 1, 71, 2, 11, - 85, 2, 11, 83, 2, 155, 135, 11, 72, 4, 29, 5, 69, 83, 72, 73, 71, 5, 11, - 32, 2, 11, 84, 2, 217, 227, 29, 6, 73, 77, 69, 83, 32, 66, 8, 92, 14, 85, - 32, 80, 76, 85, 83, 32, 85, 32, 80, 76, 85, 83, 32, 210, 226, 29, 66, - 215, 50, 77, 4, 11, 85, 5, 11, 32, 2, 11, 71, 2, 227, 131, 30, 85, 21, - 72, 7, 32, 84, 73, 77, 69, 83, 32, 136, 1, 2, 85, 77, 159, 167, 29, 66, - 10, 50, 76, 16, 2, 77, 69, 50, 83, 135, 167, 30, 85, 2, 231, 6, 65, 5, - 11, 32, 2, 201, 241, 25, 4, 80, 76, 85, 83, 2, 131, 134, 11, 72, 7, 37, - 7, 32, 84, 73, 77, 69, 83, 32, 4, 158, 11, 75, 243, 152, 30, 80, 93, 54, - 32, 102, 50, 194, 2, 73, 22, 85, 187, 162, 30, 52, 4, 62, 83, 221, 172, - 29, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 2, 165, 242, 25, 4, 72, 69, - 83, 72, 23, 11, 32, 20, 42, 73, 37, 6, 84, 73, 77, 69, 83, 32, 2, 165, - 130, 24, 4, 78, 86, 69, 82, 18, 46, 65, 82, 85, 130, 163, 29, 78, 251, - 125, 72, 6, 48, 6, 32, 80, 76, 85, 83, 32, 207, 163, 30, 76, 4, 146, 161, - 30, 72, 3, 78, 8, 26, 50, 151, 163, 30, 68, 7, 33, 6, 32, 80, 76, 85, 83, - 32, 4, 206, 186, 28, 65, 203, 212, 1, 66, 5, 203, 162, 30, 51, 59, 56, 7, + 85, 2, 11, 83, 2, 151, 144, 11, 72, 4, 29, 5, 69, 83, 72, 73, 71, 5, 11, + 32, 2, 11, 84, 2, 201, 133, 30, 6, 73, 77, 69, 83, 32, 66, 8, 92, 14, 85, + 32, 80, 76, 85, 83, 32, 85, 32, 80, 76, 85, 83, 32, 194, 132, 30, 66, + 215, 50, 77, 4, 11, 85, 5, 11, 32, 2, 11, 71, 2, 211, 165, 30, 85, 21, + 72, 7, 32, 84, 73, 77, 69, 83, 32, 136, 1, 2, 85, 77, 143, 201, 29, 66, + 10, 50, 76, 16, 2, 77, 69, 50, 83, 247, 200, 30, 85, 2, 231, 6, 65, 5, + 11, 32, 2, 221, 138, 26, 4, 80, 76, 85, 83, 2, 255, 142, 11, 72, 7, 37, + 7, 32, 84, 73, 77, 69, 83, 32, 4, 158, 11, 75, 227, 186, 30, 80, 93, 54, + 32, 102, 50, 194, 2, 73, 22, 85, 171, 196, 30, 52, 4, 62, 83, 205, 206, + 29, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 2, 185, 139, 26, 4, 72, 69, + 83, 72, 23, 11, 32, 20, 42, 73, 37, 6, 84, 73, 77, 69, 83, 32, 2, 181, + 149, 24, 4, 78, 86, 69, 82, 18, 46, 65, 82, 85, 242, 196, 29, 78, 251, + 125, 72, 6, 48, 6, 32, 80, 76, 85, 83, 32, 191, 197, 30, 76, 4, 130, 195, + 30, 72, 3, 78, 8, 26, 50, 135, 197, 30, 68, 7, 33, 6, 32, 80, 76, 85, 83, + 32, 4, 214, 217, 28, 65, 179, 215, 1, 66, 5, 187, 196, 30, 51, 59, 56, 7, 32, 84, 73, 77, 69, 83, 32, 165, 4, 2, 68, 65, 52, 134, 1, 65, 46, 68, - 38, 71, 82, 73, 46, 76, 78, 83, 46, 85, 250, 252, 28, 66, 230, 33, 77, + 38, 71, 82, 73, 46, 76, 78, 83, 46, 85, 230, 156, 29, 66, 234, 35, 77, 238, 90, 84, 146, 17, 75, 162, 17, 72, 3, 80, 5, 11, 83, 2, 11, 72, 2, - 255, 254, 16, 71, 4, 190, 254, 10, 65, 171, 210, 18, 85, 10, 26, 65, 139, - 160, 30, 85, 9, 34, 78, 230, 159, 30, 76, 3, 82, 2, 211, 9, 50, 6, 250, - 139, 30, 71, 202, 18, 83, 147, 1, 77, 6, 46, 85, 221, 129, 23, 5, 65, 75, - 45, 54, 54, 4, 246, 158, 30, 51, 3, 77, 4, 136, 132, 23, 2, 73, 71, 255, - 131, 7, 72, 6, 42, 32, 242, 239, 13, 82, 151, 174, 16, 68, 2, 165, 242, - 28, 6, 80, 76, 85, 83, 32, 71, 5, 11, 32, 2, 153, 212, 13, 4, 84, 73, 77, - 69, 17, 84, 7, 32, 84, 73, 77, 69, 83, 32, 172, 144, 29, 2, 85, 77, 166, - 140, 1, 50, 3, 88, 8, 50, 84, 212, 201, 26, 2, 75, 85, 203, 210, 3, 65, - 2, 11, 65, 2, 191, 129, 23, 75, 6, 26, 51, 227, 155, 30, 85, 5, 29, 5, - 32, 84, 73, 77, 69, 2, 21, 3, 83, 32, 75, 2, 11, 65, 2, 139, 173, 23, 83, - 42, 50, 65, 190, 1, 73, 146, 1, 85, 223, 254, 22, 69, 13, 50, 32, 254, - 141, 29, 77, 166, 140, 1, 55, 3, 71, 4, 56, 8, 83, 81, 85, 65, 82, 69, - 68, 32, 243, 155, 25, 84, 2, 11, 84, 2, 21, 3, 73, 77, 69, 2, 11, 83, 2, - 221, 160, 29, 2, 32, 75, 15, 86, 66, 220, 229, 22, 6, 32, 79, 86, 69, 82, - 32, 186, 25, 90, 174, 153, 7, 51, 3, 71, 5, 17, 2, 32, 75, 2, 17, 2, 65, + 167, 141, 17, 71, 4, 186, 135, 11, 65, 159, 235, 18, 85, 10, 26, 65, 251, + 193, 30, 85, 9, 34, 78, 214, 193, 30, 76, 3, 82, 2, 211, 9, 50, 6, 234, + 173, 30, 71, 202, 18, 83, 147, 1, 77, 6, 46, 85, 185, 149, 23, 5, 65, 75, + 45, 54, 54, 4, 230, 192, 30, 51, 3, 77, 4, 228, 151, 23, 2, 73, 71, 147, + 146, 7, 72, 6, 42, 32, 158, 254, 13, 82, 219, 193, 16, 68, 2, 129, 146, + 29, 6, 80, 76, 85, 83, 32, 71, 5, 11, 32, 2, 197, 226, 13, 4, 84, 73, 77, + 69, 17, 84, 7, 32, 84, 73, 77, 69, 83, 32, 128, 176, 29, 2, 85, 77, 194, + 142, 1, 50, 3, 88, 8, 50, 84, 240, 231, 26, 2, 75, 85, 159, 214, 3, 65, + 2, 11, 65, 2, 155, 149, 23, 75, 6, 26, 51, 211, 189, 30, 85, 5, 29, 5, + 32, 84, 73, 77, 69, 2, 21, 3, 83, 32, 75, 2, 11, 65, 2, 251, 192, 23, 83, + 42, 50, 65, 190, 1, 73, 146, 1, 85, 187, 146, 23, 69, 13, 50, 32, 210, + 173, 29, 77, 194, 142, 1, 55, 3, 71, 4, 56, 8, 83, 81, 85, 65, 82, 69, + 68, 32, 143, 181, 25, 84, 2, 11, 84, 2, 21, 3, 73, 77, 69, 2, 11, 83, 2, + 205, 194, 29, 2, 32, 75, 15, 86, 66, 184, 249, 22, 6, 32, 79, 86, 69, 82, + 32, 186, 25, 90, 194, 167, 7, 51, 3, 71, 5, 17, 2, 32, 75, 2, 17, 2, 65, 66, 2, 135, 2, 65, 15, 84, 10, 32, 79, 86, 69, 82, 32, 90, 85, 32, 80, - 42, 53, 182, 158, 29, 66, 215, 120, 77, 2, 245, 245, 28, 5, 76, 85, 83, + 42, 53, 166, 192, 29, 66, 215, 120, 77, 2, 225, 149, 29, 5, 76, 85, 83, 32, 83, 7, 37, 7, 32, 84, 73, 77, 69, 83, 32, 4, 44, 5, 84, 72, 82, 69, - 69, 179, 150, 30, 65, 2, 29, 5, 32, 68, 73, 83, 72, 2, 11, 32, 2, 143, - 152, 25, 84, 8, 42, 32, 154, 228, 22, 73, 255, 200, 3, 67, 4, 198, 159, - 22, 79, 209, 247, 6, 8, 87, 73, 84, 72, 32, 83, 84, 82, 18, 134, 1, 76, - 162, 1, 82, 249, 206, 29, 23, 86, 69, 68, 32, 83, 84, 69, 77, 32, 80, 65, - 82, 65, 71, 82, 65, 80, 72, 32, 83, 73, 71, 78, 10, 48, 2, 89, 32, 241, - 162, 3, 4, 73, 78, 71, 32, 8, 68, 2, 76, 79, 193, 243, 27, 9, 66, 82, 65, - 67, 75, 69, 84, 32, 69, 6, 214, 209, 26, 71, 155, 193, 3, 79, 6, 52, 5, - 69, 78, 67, 89, 32, 53, 4, 89, 32, 65, 78, 4, 212, 247, 24, 4, 69, 88, - 67, 72, 227, 147, 4, 83, 2, 217, 235, 19, 3, 68, 32, 82, 4, 218, 131, 3, - 79, 135, 247, 25, 65, 240, 8, 128, 1, 2, 80, 82, 140, 9, 7, 82, 73, 76, - 76, 73, 67, 32, 132, 223, 25, 7, 76, 73, 78, 68, 82, 73, 67, 205, 253, 2, + 69, 163, 184, 30, 65, 2, 29, 5, 32, 68, 73, 83, 72, 2, 11, 32, 2, 171, + 177, 25, 84, 8, 42, 32, 246, 247, 22, 73, 155, 211, 3, 67, 4, 174, 179, + 22, 79, 217, 133, 7, 8, 87, 73, 84, 72, 32, 83, 84, 82, 18, 134, 1, 76, + 162, 1, 82, 233, 240, 29, 23, 86, 69, 68, 32, 83, 84, 69, 77, 32, 80, 65, + 82, 65, 71, 82, 65, 80, 72, 32, 83, 73, 71, 78, 10, 48, 2, 89, 32, 185, + 164, 3, 4, 73, 78, 71, 32, 8, 68, 2, 76, 79, 221, 145, 28, 9, 66, 82, 65, + 67, 75, 69, 84, 32, 69, 6, 242, 239, 26, 71, 239, 196, 3, 79, 6, 52, 5, + 69, 78, 67, 89, 32, 53, 4, 89, 32, 65, 78, 4, 248, 144, 25, 4, 69, 88, + 67, 72, 175, 156, 4, 83, 2, 133, 252, 19, 3, 68, 32, 82, 4, 234, 132, 3, + 79, 227, 149, 26, 65, 240, 8, 128, 1, 2, 80, 82, 140, 9, 7, 82, 73, 76, + 76, 73, 67, 32, 152, 248, 25, 7, 76, 73, 78, 68, 82, 73, 67, 149, 132, 3, 2, 67, 76, 180, 2, 140, 1, 13, 73, 79, 84, 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 169, 1, 16, 79, 45, 77, 73, 78, 79, 65, 78, 32, 83, 73, 71, - 78, 32, 67, 77, 110, 238, 177, 7, 75, 2, 76, 2, 77, 2, 78, 2, 80, 2, 82, - 2, 83, 2, 84, 126, 87, 138, 42, 74, 2, 90, 130, 239, 6, 88, 154, 195, 15, - 65, 2, 69, 2, 73, 2, 79, 3, 85, 198, 1, 46, 48, 146, 5, 49, 153, 135, 26, + 78, 32, 67, 77, 110, 190, 185, 7, 75, 2, 76, 2, 77, 2, 78, 2, 80, 2, 82, + 2, 83, 2, 84, 126, 87, 150, 42, 74, 2, 90, 230, 245, 6, 88, 202, 214, 15, + 65, 2, 69, 2, 73, 2, 79, 3, 85, 198, 1, 46, 48, 146, 5, 49, 181, 164, 26, 2, 51, 48, 170, 1, 102, 48, 78, 49, 74, 50, 78, 51, 78, 52, 62, 53, 86, - 55, 190, 186, 10, 57, 190, 3, 54, 215, 149, 14, 56, 16, 194, 140, 30, 49, - 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 16, 194, 3, 50, 182, - 136, 30, 48, 2, 49, 2, 51, 2, 53, 2, 55, 3, 57, 16, 174, 139, 30, 49, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 16, 226, 138, 30, 48, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 150, 138, 30, 48, 2, - 49, 2, 52, 2, 54, 2, 55, 3, 57, 18, 218, 137, 30, 48, 2, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 20, 82, 53, 182, 136, 30, 48, 2, - 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 56, 3, 57, 5, 179, 136, 30, 66, 24, - 18, 48, 87, 49, 18, 138, 136, 30, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 55, 2, 56, 3, 57, 6, 182, 135, 30, 48, 2, 50, 3, 52, 184, 6, 140, 1, + 55, 198, 195, 10, 57, 190, 3, 54, 219, 165, 14, 56, 16, 178, 174, 30, 49, + 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 16, 194, 3, 50, 166, + 170, 30, 48, 2, 49, 2, 51, 2, 53, 2, 55, 3, 57, 16, 158, 173, 30, 49, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 16, 210, 172, 30, 48, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 134, 172, 30, 48, 2, + 49, 2, 52, 2, 54, 2, 55, 3, 57, 18, 202, 171, 30, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 20, 82, 53, 166, 170, 30, 48, 2, + 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 56, 3, 57, 5, 163, 170, 30, 66, 24, + 18, 48, 87, 49, 18, 250, 169, 30, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 55, 2, 56, 3, 57, 6, 166, 169, 30, 48, 2, 50, 3, 52, 184, 6, 140, 1, 9, 67, 65, 80, 73, 84, 65, 76, 32, 76, 202, 4, 75, 32, 7, 76, 69, 84, 84, 69, 82, 32, 132, 1, 3, 80, 65, 89, 30, 83, 211, 41, 84, 242, 2, 44, 6, 69, 84, 84, 69, 82, 32, 175, 43, 73, 236, 2, 150, 2, 76, 46, 78, 34, 80, 34, 82, 58, 84, 62, 85, 190, 5, 65, 214, 1, 66, 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, 73, 230, 3, 75, 226, 3, 77, - 218, 1, 79, 226, 2, 83, 218, 7, 89, 166, 1, 90, 162, 183, 29, 70, 134, - 14, 74, 2, 86, 2, 87, 159, 20, 81, 6, 222, 25, 73, 254, 210, 29, 74, 159, - 20, 72, 4, 134, 27, 69, 171, 209, 29, 74, 8, 134, 34, 69, 135, 205, 29, - 83, 12, 220, 9, 3, 79, 85, 78, 226, 18, 69, 207, 227, 29, 72, 20, 174, - 33, 69, 106, 83, 234, 178, 29, 67, 186, 22, 74, 3, 87, 13, 198, 34, 32, - 115, 75, 2, 205, 248, 29, 3, 65, 86, 89, 6, 248, 10, 5, 77, 85, 76, 84, - 73, 200, 129, 13, 2, 80, 65, 241, 231, 11, 14, 83, 77, 65, 76, 76, 32, - 67, 65, 80, 73, 84, 65, 76, 32, 2, 177, 214, 29, 2, 69, 82, 186, 3, 136, + 218, 1, 79, 226, 2, 83, 218, 7, 89, 166, 1, 90, 146, 217, 29, 70, 134, + 14, 74, 2, 86, 2, 87, 159, 20, 81, 6, 222, 25, 73, 238, 244, 29, 74, 159, + 20, 72, 4, 134, 27, 69, 155, 243, 29, 74, 8, 134, 34, 69, 247, 238, 29, + 83, 12, 220, 9, 3, 79, 85, 78, 226, 18, 69, 191, 133, 30, 72, 20, 174, + 33, 69, 106, 83, 218, 212, 29, 67, 186, 22, 74, 3, 87, 13, 198, 34, 32, + 115, 75, 2, 189, 154, 30, 3, 65, 86, 89, 6, 248, 10, 5, 77, 85, 76, 84, + 73, 244, 143, 13, 2, 80, 65, 233, 242, 11, 14, 83, 77, 65, 76, 76, 32, + 67, 65, 80, 73, 84, 65, 76, 32, 2, 161, 248, 29, 2, 69, 82, 186, 3, 136, 1, 6, 77, 65, 76, 76, 32, 76, 201, 37, 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 134, 3, 44, 6, 69, 84, 84, 69, 82, 32, 151, 36, 73, 128, 3, 154, 2, 65, 214, 1, 66, 242, 1, 67, 150, 1, 68, 254, 1, 69, 206, 2, 71, 130, 1, 72, 134, 1, 73, 230, 3, 75, 222, 2, 76, 134, 1, 77, 114, 78, 106, 79, 110, 80, 50, 82, - 198, 1, 83, 218, 2, 84, 218, 2, 85, 246, 1, 87, 54, 89, 166, 1, 90, 162, - 183, 29, 70, 134, 14, 74, 2, 86, 159, 20, 81, 17, 108, 6, 32, 87, 73, 84, - 72, 32, 36, 9, 66, 75, 72, 65, 83, 73, 65, 78, 32, 249, 152, 12, 4, 76, - 69, 85, 84, 4, 186, 194, 9, 68, 147, 182, 3, 66, 8, 172, 133, 13, 3, 67, - 72, 69, 158, 213, 6, 68, 255, 157, 10, 72, 18, 110, 65, 78, 73, 32, 3, - 82, 79, 65, 232, 160, 3, 6, 76, 69, 78, 68, 69, 68, 246, 224, 9, 89, 139, - 247, 16, 69, 6, 200, 21, 6, 82, 82, 69, 68, 32, 79, 253, 129, 12, 5, 83, - 72, 75, 73, 82, 4, 230, 2, 78, 163, 158, 3, 71, 2, 171, 179, 10, 68, 14, - 76, 2, 72, 69, 246, 9, 76, 220, 146, 13, 4, 82, 79, 83, 83, 167, 174, 16, - 67, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 150, 29, 68, 255, 169, 29, 86, - 26, 96, 2, 74, 69, 20, 6, 79, 85, 66, 76, 69, 32, 66, 90, 250, 200, 29, - 67, 186, 22, 87, 215, 22, 69, 5, 219, 230, 22, 82, 4, 36, 3, 77, 79, 78, - 159, 246, 29, 79, 2, 153, 13, 2, 79, 67, 12, 46, 69, 182, 245, 28, 90, - 206, 105, 72, 3, 87, 5, 223, 214, 29, 76, 41, 86, 76, 98, 78, 110, 82, - 166, 15, 32, 170, 238, 12, 83, 178, 193, 14, 77, 163, 180, 2, 70, 11, 33, - 6, 32, 87, 73, 84, 72, 32, 8, 166, 20, 77, 130, 235, 12, 68, 190, 193, - 14, 84, 143, 96, 72, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 198, 19, 77, - 130, 235, 12, 68, 202, 52, 76, 246, 140, 14, 84, 143, 96, 72, 5, 133, - 195, 28, 5, 32, 87, 73, 84, 72, 14, 32, 2, 72, 69, 255, 219, 29, 74, 13, - 33, 6, 32, 87, 73, 84, 72, 32, 10, 142, 18, 77, 146, 15, 85, 242, 219, - 12, 68, 155, 31, 83, 12, 26, 65, 135, 219, 29, 87, 11, 48, 6, 32, 87, 73, - 84, 72, 32, 207, 136, 28, 82, 6, 134, 252, 12, 68, 202, 161, 15, 72, 219, - 163, 1, 83, 35, 84, 6, 32, 87, 73, 84, 72, 32, 50, 69, 74, 79, 201, 1, 6, - 90, 72, 73, 84, 83, 65, 6, 174, 183, 9, 68, 214, 10, 71, 243, 196, 3, 77, - 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 174, 193, 9, 71, 191, 171, 3, 66, - 17, 11, 84, 14, 48, 6, 73, 70, 73, 69, 68, 32, 227, 238, 29, 65, 12, 80, - 2, 67, 76, 38, 76, 190, 149, 3, 66, 198, 196, 25, 89, 234, 147, 1, 65, 3, - 69, 2, 33, 6, 79, 83, 69, 68, 32, 76, 2, 151, 4, 73, 5, 197, 136, 13, 14, + 198, 1, 83, 218, 2, 84, 218, 2, 85, 246, 1, 87, 54, 89, 166, 1, 90, 146, + 217, 29, 70, 134, 14, 74, 2, 86, 159, 20, 81, 17, 108, 6, 32, 87, 73, 84, + 72, 32, 36, 9, 66, 75, 72, 65, 83, 73, 65, 78, 32, 149, 167, 12, 4, 76, + 69, 85, 84, 4, 178, 203, 9, 68, 191, 187, 3, 66, 8, 216, 147, 13, 3, 67, + 72, 69, 158, 215, 6, 68, 195, 175, 10, 72, 18, 110, 65, 78, 73, 32, 3, + 82, 79, 65, 200, 167, 3, 6, 76, 69, 78, 68, 69, 68, 194, 232, 9, 89, 207, + 138, 17, 69, 6, 200, 21, 6, 82, 82, 69, 68, 32, 79, 153, 144, 12, 5, 83, + 72, 75, 73, 82, 4, 230, 2, 78, 131, 165, 3, 71, 2, 163, 188, 10, 68, 14, + 76, 2, 72, 69, 246, 9, 76, 136, 161, 13, 4, 82, 79, 83, 83, 235, 193, 16, + 67, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 150, 29, 68, 239, 203, 29, 86, + 26, 96, 2, 74, 69, 20, 6, 79, 85, 66, 76, 69, 32, 66, 90, 234, 234, 29, + 67, 186, 22, 87, 215, 22, 69, 5, 183, 250, 22, 82, 4, 36, 3, 77, 79, 78, + 143, 152, 30, 79, 2, 153, 13, 2, 79, 67, 12, 46, 69, 166, 151, 29, 90, + 206, 105, 72, 3, 87, 5, 207, 248, 29, 76, 41, 86, 76, 98, 78, 110, 82, + 166, 15, 32, 214, 252, 12, 83, 178, 209, 14, 77, 231, 183, 2, 70, 11, 33, + 6, 32, 87, 73, 84, 72, 32, 8, 166, 20, 77, 174, 249, 12, 68, 190, 209, + 14, 84, 183, 97, 72, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 198, 19, 77, + 174, 249, 12, 68, 202, 52, 76, 246, 156, 14, 84, 183, 97, 72, 5, 225, + 226, 28, 5, 32, 87, 73, 84, 72, 14, 32, 2, 72, 69, 239, 253, 29, 74, 13, + 33, 6, 32, 87, 73, 84, 72, 32, 10, 142, 18, 77, 146, 15, 85, 158, 234, + 12, 68, 155, 31, 83, 12, 26, 65, 247, 252, 29, 87, 11, 48, 6, 32, 87, 73, + 84, 72, 32, 215, 167, 28, 82, 6, 178, 138, 13, 68, 242, 178, 15, 72, 247, + 165, 1, 83, 35, 84, 6, 32, 87, 73, 84, 72, 32, 50, 69, 74, 79, 201, 1, 6, + 90, 72, 73, 84, 83, 65, 6, 166, 192, 9, 68, 214, 10, 71, 167, 202, 3, 77, + 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 166, 202, 9, 71, 235, 176, 3, 66, + 17, 11, 84, 14, 48, 6, 73, 70, 73, 69, 68, 32, 211, 144, 30, 65, 12, 80, + 2, 67, 76, 38, 76, 158, 156, 3, 66, 142, 234, 25, 89, 178, 137, 1, 65, 3, + 69, 2, 33, 6, 79, 83, 69, 68, 32, 76, 2, 151, 4, 73, 5, 241, 150, 13, 14, 32, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 69, 32, 71, 34, 102, 65, 98, - 79, 184, 190, 29, 11, 72, 65, 75, 65, 83, 83, 73, 65, 78, 32, 67, 186, - 22, 74, 255, 2, 83, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 226, 246, 12, - 68, 202, 161, 15, 72, 142, 163, 1, 86, 79, 83, 18, 36, 3, 77, 73, 32, - 203, 177, 26, 80, 16, 58, 68, 246, 160, 12, 76, 2, 78, 2, 83, 2, 84, 3, - 90, 6, 242, 160, 12, 90, 154, 179, 17, 74, 215, 22, 69, 8, 94, 73, 228, - 200, 10, 10, 79, 78, 71, 45, 76, 69, 71, 71, 69, 68, 154, 138, 19, 74, - 159, 20, 72, 2, 209, 145, 3, 4, 84, 84, 76, 69, 4, 21, 3, 79, 78, 79, 4, - 18, 67, 39, 71, 2, 145, 132, 19, 4, 85, 76, 65, 82, 2, 245, 10, 4, 82, - 65, 80, 72, 6, 62, 69, 144, 131, 19, 5, 65, 82, 82, 79, 87, 155, 206, 10, - 74, 2, 165, 245, 12, 5, 85, 84, 82, 65, 76, 11, 52, 4, 77, 69, 71, 65, - 166, 3, 32, 251, 227, 29, 84, 5, 173, 190, 29, 8, 32, 87, 73, 84, 72, 32, - 84, 73, 10, 138, 6, 69, 138, 236, 12, 65, 255, 224, 16, 83, 14, 50, 69, - 100, 4, 79, 85, 78, 68, 235, 226, 29, 72, 8, 37, 7, 86, 69, 82, 83, 69, - 68, 32, 8, 166, 197, 19, 68, 170, 151, 9, 84, 166, 100, 89, 151, 14, 90, - 4, 242, 159, 10, 32, 137, 244, 1, 2, 69, 68, 34, 108, 4, 67, 72, 87, 65, - 34, 72, 132, 1, 4, 79, 70, 84, 32, 214, 238, 12, 84, 213, 1, 5, 69, 77, - 73, 83, 79, 5, 11, 32, 2, 187, 238, 5, 87, 16, 92, 4, 79, 82, 84, 32, - 180, 237, 12, 2, 72, 65, 146, 251, 14, 67, 238, 227, 1, 87, 215, 22, 65, - 6, 226, 174, 27, 73, 163, 180, 2, 85, 8, 38, 69, 210, 219, 28, 83, 151, - 112, 68, 4, 182, 226, 29, 76, 3, 77, 28, 140, 1, 4, 65, 76, 76, 32, 50, - 69, 106, 83, 224, 135, 12, 11, 72, 82, 69, 69, 45, 76, 69, 71, 71, 69, - 68, 138, 171, 17, 67, 186, 22, 74, 3, 87, 6, 214, 231, 12, 72, 206, 229, - 15, 89, 151, 125, 84, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 77, 131, - 235, 12, 68, 2, 189, 244, 21, 5, 73, 68, 68, 76, 69, 8, 158, 201, 29, 72, + 79, 180, 219, 17, 10, 72, 65, 75, 65, 83, 83, 73, 65, 78, 32, 174, 155, + 12, 74, 255, 2, 83, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 142, 133, 13, + 68, 242, 178, 15, 72, 170, 165, 1, 86, 79, 83, 18, 36, 3, 77, 73, 32, + 231, 207, 26, 80, 16, 58, 68, 254, 174, 12, 76, 2, 78, 2, 83, 2, 84, 3, + 90, 6, 250, 174, 12, 90, 130, 199, 17, 74, 215, 22, 69, 8, 94, 73, 224, + 209, 10, 10, 79, 78, 71, 45, 76, 69, 71, 71, 69, 68, 142, 163, 19, 74, + 159, 20, 72, 2, 177, 152, 3, 4, 84, 84, 76, 69, 4, 21, 3, 79, 78, 79, 4, + 18, 67, 39, 71, 2, 165, 147, 19, 4, 85, 76, 65, 82, 2, 245, 10, 4, 82, + 65, 80, 72, 6, 62, 69, 164, 146, 19, 5, 65, 82, 82, 79, 87, 247, 224, 10, + 74, 2, 209, 131, 13, 5, 85, 84, 82, 65, 76, 11, 52, 4, 77, 69, 71, 65, + 166, 3, 32, 235, 133, 30, 84, 5, 157, 224, 29, 8, 32, 87, 73, 84, 72, 32, + 84, 73, 10, 138, 6, 69, 182, 250, 12, 65, 195, 244, 16, 83, 14, 50, 69, + 100, 4, 79, 85, 78, 68, 219, 132, 30, 72, 8, 37, 7, 86, 69, 82, 83, 69, + 68, 32, 8, 210, 213, 19, 68, 214, 166, 9, 84, 190, 102, 89, 151, 14, 90, + 4, 234, 168, 10, 32, 153, 249, 1, 2, 69, 68, 34, 108, 4, 67, 72, 87, 65, + 34, 72, 132, 1, 4, 79, 70, 84, 32, 130, 253, 12, 84, 213, 1, 5, 69, 77, + 73, 83, 79, 5, 11, 32, 2, 219, 245, 5, 87, 16, 92, 4, 79, 82, 84, 32, + 224, 251, 12, 2, 72, 65, 238, 139, 15, 67, 214, 230, 1, 87, 215, 22, 65, + 6, 142, 205, 27, 73, 231, 183, 2, 85, 8, 38, 69, 194, 253, 28, 83, 151, + 112, 68, 4, 166, 132, 30, 76, 3, 77, 28, 140, 1, 4, 65, 76, 76, 32, 50, + 69, 106, 83, 232, 149, 12, 11, 72, 82, 69, 69, 45, 76, 69, 71, 71, 69, + 68, 242, 190, 17, 67, 186, 22, 74, 3, 87, 6, 130, 246, 12, 72, 202, 131, + 16, 89, 223, 114, 84, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 77, 175, + 249, 12, 68, 2, 165, 136, 22, 5, 73, 68, 68, 76, 69, 8, 142, 235, 29, 72, 2, 83, 2, 87, 215, 22, 69, 15, 58, 32, 114, 75, 53, 8, 78, 66, 76, 69, - 78, 68, 69, 68, 6, 29, 5, 87, 73, 84, 72, 32, 6, 26, 68, 171, 245, 12, - 77, 4, 212, 167, 9, 5, 79, 85, 66, 76, 69, 187, 8, 73, 5, 11, 82, 2, 213, - 4, 6, 65, 73, 78, 73, 65, 78, 2, 207, 232, 24, 32, 4, 212, 201, 28, 4, - 73, 68, 69, 32, 251, 147, 1, 69, 18, 62, 65, 28, 3, 69, 82, 85, 194, 220, - 29, 73, 2, 78, 3, 85, 7, 218, 220, 29, 69, 3, 84, 7, 33, 6, 32, 87, 73, - 84, 72, 32, 4, 178, 163, 9, 68, 255, 197, 3, 66, 18, 18, 69, 71, 72, 9, - 156, 1, 7, 32, 87, 73, 84, 72, 32, 68, 233, 215, 29, 2, 77, 76, 10, 26, - 69, 179, 196, 29, 87, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 26, 68, 239, - 215, 12, 66, 4, 134, 172, 9, 73, 175, 185, 3, 69, 6, 37, 7, 71, 65, 84, - 85, 82, 69, 32, 6, 66, 65, 188, 198, 15, 2, 84, 69, 237, 229, 13, 4, 69, - 78, 32, 71, 2, 131, 190, 29, 32, 52, 198, 1, 66, 38, 68, 38, 69, 36, 3, - 71, 72, 69, 38, 72, 130, 137, 22, 89, 138, 211, 5, 83, 182, 113, 84, 238, - 8, 90, 130, 64, 73, 150, 19, 67, 186, 22, 80, 2, 86, 158, 20, 75, 186, 2, - 65, 2, 79, 3, 85, 4, 206, 224, 12, 89, 139, 247, 16, 69, 6, 146, 215, 28, - 90, 163, 128, 1, 69, 6, 142, 215, 29, 70, 2, 76, 3, 83, 5, 201, 5, 5, 32, - 87, 73, 84, 72, 4, 11, 65, 5, 227, 237, 27, 82, 2, 153, 248, 15, 4, 72, - 79, 85, 83, 212, 15, 178, 1, 65, 138, 4, 67, 54, 69, 146, 35, 73, 138, - 23, 79, 154, 45, 82, 174, 3, 85, 144, 232, 12, 13, 78, 65, 32, 68, 79, - 85, 66, 76, 69, 32, 72, 69, 76, 134, 183, 15, 86, 207, 47, 76, 30, 116, - 4, 71, 71, 69, 82, 146, 1, 78, 32, 4, 82, 75, 32, 83, 36, 2, 83, 72, 184, - 253, 21, 2, 76, 69, 139, 145, 1, 84, 9, 11, 32, 6, 44, 5, 87, 73, 84, 72, - 32, 219, 210, 6, 75, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 237, - 187, 28, 4, 84, 32, 71, 85, 4, 214, 148, 29, 67, 251, 30, 71, 4, 230, - 200, 21, 85, 211, 146, 5, 72, 10, 30, 32, 105, 3, 69, 68, 32, 4, 60, 9, - 87, 73, 84, 72, 32, 76, 69, 70, 84, 147, 190, 28, 83, 2, 17, 2, 32, 85, - 2, 143, 141, 23, 80, 6, 238, 233, 4, 84, 142, 189, 12, 76, 155, 194, 10, - 79, 10, 194, 208, 29, 49, 2, 50, 2, 51, 2, 52, 3, 83, 180, 4, 226, 1, 67, - 248, 1, 5, 71, 82, 69, 69, 32, 98, 76, 82, 78, 228, 3, 8, 80, 65, 82, 84, - 77, 69, 78, 84, 34, 83, 174, 6, 86, 192, 140, 15, 11, 82, 69, 76, 73, 67, - 84, 32, 72, 79, 85, 83, 168, 180, 10, 2, 65, 70, 143, 199, 3, 69, 8, 50, - 73, 177, 250, 5, 6, 82, 69, 65, 83, 69, 32, 6, 56, 4, 77, 65, 76, 32, - 205, 168, 9, 4, 68, 85, 79, 85, 4, 88, 9, 83, 69, 80, 65, 82, 65, 84, 79, - 82, 149, 248, 21, 7, 69, 88, 80, 79, 78, 69, 78, 2, 21, 3, 32, 75, 69, 2, - 155, 185, 28, 89, 6, 140, 194, 22, 4, 67, 69, 76, 83, 186, 131, 6, 83, - 177, 106, 8, 70, 65, 72, 82, 69, 78, 72, 69, 9, 248, 250, 16, 5, 73, 86, - 69, 82, 89, 248, 252, 5, 2, 84, 65, 231, 140, 5, 69, 34, 104, 20, 84, 73, + 78, 68, 69, 68, 6, 29, 5, 87, 73, 84, 72, 32, 6, 26, 68, 215, 131, 13, + 77, 4, 204, 176, 9, 5, 79, 85, 66, 76, 69, 187, 8, 73, 5, 11, 82, 2, 213, + 4, 6, 65, 73, 78, 73, 65, 78, 2, 235, 129, 25, 32, 4, 184, 233, 28, 4, + 73, 68, 69, 32, 135, 150, 1, 69, 18, 62, 65, 28, 3, 69, 82, 85, 178, 254, + 29, 73, 2, 78, 3, 85, 7, 202, 254, 29, 69, 3, 84, 7, 33, 6, 32, 87, 73, + 84, 72, 32, 4, 170, 172, 9, 68, 179, 203, 3, 66, 18, 18, 69, 71, 72, 9, + 156, 1, 7, 32, 87, 73, 84, 72, 32, 68, 217, 249, 29, 2, 77, 76, 10, 26, + 69, 163, 230, 29, 87, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 26, 68, 147, + 230, 12, 66, 4, 254, 180, 9, 73, 227, 190, 3, 69, 6, 37, 7, 71, 65, 84, + 85, 82, 69, 32, 6, 66, 65, 128, 213, 15, 2, 84, 69, 153, 249, 13, 4, 69, + 78, 32, 71, 2, 243, 223, 29, 32, 52, 198, 1, 66, 38, 68, 38, 69, 36, 3, + 71, 72, 69, 38, 72, 234, 156, 22, 89, 170, 222, 5, 83, 134, 114, 84, 134, + 11, 90, 130, 64, 73, 150, 19, 67, 186, 22, 80, 2, 86, 158, 20, 75, 186, + 2, 65, 2, 79, 3, 85, 4, 250, 238, 12, 89, 207, 138, 17, 69, 6, 130, 249, + 28, 90, 163, 128, 1, 69, 6, 254, 248, 29, 70, 2, 76, 3, 83, 5, 201, 5, 5, + 32, 87, 73, 84, 72, 4, 11, 65, 5, 235, 140, 28, 82, 2, 221, 134, 16, 4, + 72, 79, 85, 83, 214, 15, 178, 1, 65, 138, 4, 67, 54, 69, 154, 35, 73, + 166, 23, 79, 154, 45, 82, 174, 3, 85, 172, 246, 12, 13, 78, 65, 32, 68, + 79, 85, 66, 76, 69, 32, 72, 69, 76, 182, 202, 15, 86, 207, 47, 76, 30, + 116, 4, 71, 71, 69, 82, 146, 1, 78, 32, 4, 82, 75, 32, 83, 36, 2, 83, 72, + 160, 145, 22, 2, 76, 69, 131, 145, 1, 84, 9, 11, 32, 6, 44, 5, 87, 73, + 84, 72, 32, 171, 218, 6, 75, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, + 2, 217, 219, 28, 4, 84, 32, 71, 85, 4, 198, 182, 29, 67, 251, 30, 71, 4, + 190, 220, 21, 85, 155, 157, 5, 72, 10, 30, 32, 105, 3, 69, 68, 32, 4, 60, + 9, 87, 73, 84, 72, 32, 76, 69, 70, 84, 231, 221, 28, 83, 2, 17, 2, 32, + 85, 2, 239, 160, 23, 80, 6, 206, 240, 4, 84, 130, 197, 12, 76, 207, 210, + 10, 79, 10, 178, 242, 29, 49, 2, 50, 2, 51, 2, 52, 3, 83, 180, 4, 222, 1, + 67, 248, 1, 5, 71, 82, 69, 69, 32, 98, 76, 82, 78, 228, 3, 8, 80, 65, 82, + 84, 77, 69, 78, 84, 32, 12, 82, 69, 76, 73, 67, 84, 32, 72, 79, 85, 83, + 69, 42, 83, 174, 6, 86, 224, 221, 25, 2, 65, 70, 227, 203, 3, 69, 8, 50, + 73, 225, 129, 6, 6, 82, 69, 65, 83, 69, 32, 6, 56, 4, 77, 65, 76, 32, + 237, 177, 9, 4, 68, 85, 79, 85, 4, 88, 9, 83, 69, 80, 65, 82, 65, 84, 79, + 82, 129, 140, 22, 7, 69, 88, 80, 79, 78, 69, 78, 2, 21, 3, 32, 75, 69, 2, + 243, 216, 28, 89, 6, 240, 213, 22, 4, 67, 69, 76, 83, 202, 145, 6, 83, + 177, 106, 8, 70, 65, 72, 82, 69, 78, 72, 69, 9, 196, 137, 17, 5, 73, 86, + 69, 82, 89, 160, 130, 6, 2, 84, 65, 203, 152, 5, 69, 34, 104, 20, 84, 73, 83, 84, 82, 89, 32, 83, 89, 77, 66, 79, 76, 32, 76, 73, 71, 72, 84, 32, - 135, 237, 21, 83, 30, 88, 4, 68, 79, 87, 78, 0, 2, 85, 80, 153, 1, 9, 86, + 243, 128, 22, 83, 30, 88, 4, 68, 79, 87, 78, 0, 2, 85, 80, 153, 1, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, 8, 69, 15, 32, 65, 78, 68, 32, 72, 79, - 82, 73, 90, 79, 78, 84, 65, 76, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 150, - 221, 27, 87, 186, 25, 67, 159, 45, 84, 14, 52, 4, 65, 78, 68, 32, 45, 5, - 87, 73, 84, 72, 32, 10, 142, 156, 22, 66, 42, 84, 255, 191, 5, 87, 4, - 190, 245, 27, 67, 159, 45, 84, 2, 193, 226, 24, 3, 32, 83, 84, 170, 1, - 64, 2, 67, 69, 48, 2, 69, 82, 129, 5, 5, 75, 84, 79, 80, 32, 2, 153, 182, - 27, 7, 78, 68, 73, 78, 71, 32, 78, 164, 1, 32, 3, 69, 84, 32, 183, 4, 84, - 160, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 80, 45, 9, - 76, 32, 76, 69, 84, 84, 69, 82, 32, 80, 218, 1, 69, 96, 4, 76, 79, 78, - 71, 0, 5, 83, 72, 79, 82, 84, 74, 79, 30, 84, 2, 90, 158, 231, 11, 67, - 250, 234, 15, 66, 2, 68, 2, 74, 2, 80, 2, 86, 2, 89, 182, 99, 71, 2, 75, - 158, 103, 87, 162, 19, 65, 203, 17, 72, 20, 242, 162, 25, 83, 134, 165, - 2, 78, 138, 250, 1, 84, 146, 1, 70, 2, 76, 2, 77, 2, 82, 3, 87, 12, 11, - 32, 12, 134, 162, 25, 65, 190, 218, 1, 79, 226, 197, 2, 69, 3, 73, 4, - 230, 193, 29, 73, 3, 87, 4, 146, 210, 27, 72, 231, 216, 1, 69, 5, 161, - 179, 28, 4, 32, 73, 83, 76, 4, 158, 198, 26, 67, 193, 224, 1, 4, 87, 73, - 78, 68, 202, 2, 100, 8, 65, 78, 65, 71, 65, 82, 73, 32, 161, 18, 12, 73, - 67, 69, 32, 67, 79, 78, 84, 82, 79, 76, 32, 192, 2, 222, 1, 65, 38, 67, - 22, 71, 36, 4, 72, 69, 65, 68, 96, 7, 76, 69, 84, 84, 69, 82, 32, 198, 5, - 83, 148, 7, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 154, 129, 19, - 68, 212, 129, 6, 4, 74, 65, 73, 78, 171, 156, 4, 79, 4, 210, 216, 12, 67, - 151, 206, 12, 66, 2, 159, 146, 6, 65, 4, 194, 216, 12, 82, 135, 238, 1, - 65, 6, 44, 5, 32, 77, 65, 82, 75, 251, 140, 29, 83, 5, 181, 143, 19, 7, - 32, 87, 73, 84, 72, 32, 72, 158, 1, 158, 2, 65, 54, 67, 62, 68, 50, 71, - 62, 72, 52, 2, 77, 65, 46, 83, 130, 10, 79, 246, 147, 7, 66, 206, 252, 4, - 75, 150, 202, 3, 82, 238, 232, 6, 89, 154, 119, 85, 174, 30, 78, 254, - 198, 1, 74, 150, 52, 76, 70, 84, 46, 86, 134, 207, 1, 73, 158, 194, 1, - 80, 2, 90, 138, 69, 70, 2, 81, 187, 2, 69, 13, 158, 186, 29, 65, 2, 73, - 2, 85, 2, 87, 3, 89, 10, 26, 65, 155, 183, 29, 72, 9, 185, 2, 4, 78, 68, - 82, 65, 12, 178, 223, 25, 68, 198, 215, 3, 72, 187, 2, 65, 10, 130, 238, - 12, 76, 190, 131, 16, 72, 138, 69, 71, 187, 2, 65, 4, 232, 210, 16, 4, - 69, 65, 86, 89, 219, 229, 12, 65, 5, 233, 154, 7, 6, 82, 87, 65, 82, 73, - 32, 12, 38, 72, 134, 181, 29, 83, 187, 2, 65, 8, 36, 3, 79, 82, 84, 151, - 183, 29, 65, 6, 199, 136, 12, 32, 68, 168, 1, 19, 69, 81, 85, 69, 78, 67, - 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 104, 4, 73, 71, 78, - 32, 133, 254, 26, 10, 84, 82, 69, 83, 83, 32, 83, 73, 71, 78, 16, 174, - 217, 3, 71, 2, 75, 228, 225, 23, 3, 68, 68, 68, 2, 82, 230, 247, 1, 89, - 38, 70, 2, 81, 3, 90, 48, 178, 3, 66, 0, 10, 69, 88, 84, 69, 78, 68, 69, - 68, 32, 66, 36, 11, 67, 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 64, 8, - 87, 69, 83, 84, 69, 82, 78, 32, 32, 10, 82, 69, 86, 69, 82, 83, 69, 68, - 32, 78, 226, 200, 6, 83, 190, 190, 12, 73, 142, 149, 6, 77, 46, 78, 242, - 60, 65, 72, 18, 68, 79, 85, 66, 76, 69, 32, 67, 65, 78, 68, 82, 65, 66, - 73, 78, 68, 85, 62, 80, 216, 196, 2, 12, 72, 73, 71, 72, 32, 83, 80, 65, - 67, 73, 78, 71, 139, 78, 86, 4, 241, 222, 12, 4, 72, 65, 76, 69, 11, 11, - 32, 8, 134, 194, 22, 65, 238, 152, 3, 86, 183, 230, 1, 84, 4, 30, 78, 21, - 3, 70, 73, 86, 2, 17, 2, 73, 78, 2, 129, 221, 21, 8, 69, 45, 76, 73, 75, - 69, 32, 66, 50, 150, 1, 65, 52, 7, 67, 65, 78, 68, 82, 65, 32, 62, 79, - 186, 134, 19, 80, 162, 180, 4, 85, 158, 224, 1, 83, 222, 63, 86, 186, - 201, 1, 73, 223, 137, 2, 69, 10, 210, 174, 29, 65, 2, 73, 2, 85, 2, 87, - 3, 89, 6, 172, 135, 19, 4, 76, 79, 78, 71, 242, 166, 10, 69, 3, 79, 7, - 142, 151, 29, 79, 215, 22, 69, 10, 194, 154, 24, 70, 136, 6, 2, 83, 84, - 154, 157, 3, 84, 183, 86, 79, 230, 2, 182, 2, 65, 150, 2, 69, 74, 71, + 82, 73, 90, 79, 78, 84, 65, 76, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 162, + 252, 27, 87, 134, 26, 67, 167, 45, 84, 14, 52, 4, 65, 78, 68, 32, 45, 5, + 87, 73, 84, 72, 32, 10, 238, 175, 22, 66, 42, 84, 171, 203, 5, 87, 4, + 150, 149, 28, 67, 167, 45, 84, 2, 253, 251, 24, 3, 32, 83, 84, 2, 185, + 162, 23, 5, 32, 66, 85, 73, 76, 170, 1, 64, 2, 67, 69, 48, 2, 69, 82, + 129, 5, 5, 75, 84, 79, 80, 32, 2, 253, 212, 27, 7, 78, 68, 73, 78, 71, + 32, 78, 164, 1, 32, 3, 69, 84, 32, 183, 4, 84, 160, 1, 56, 6, 67, 65, 80, + 73, 84, 65, 1, 4, 83, 77, 65, 76, 80, 45, 9, 76, 32, 76, 69, 84, 84, 69, + 82, 32, 80, 218, 1, 69, 96, 4, 76, 79, 78, 71, 0, 5, 83, 72, 79, 82, 84, + 74, 79, 30, 84, 2, 90, 158, 245, 11, 67, 222, 251, 15, 66, 2, 68, 2, 74, + 2, 80, 2, 86, 2, 89, 130, 100, 71, 2, 75, 186, 105, 87, 162, 19, 65, 203, + 17, 72, 20, 250, 211, 25, 83, 226, 146, 2, 78, 242, 252, 1, 84, 146, 1, + 70, 2, 76, 2, 77, 2, 82, 3, 87, 12, 11, 32, 12, 142, 211, 25, 65, 178, + 199, 1, 79, 178, 201, 2, 69, 3, 73, 4, 178, 227, 29, 73, 3, 87, 4, 246, + 240, 27, 72, 207, 219, 1, 69, 5, 209, 210, 28, 4, 32, 73, 83, 76, 4, 154, + 228, 26, 67, 141, 226, 1, 4, 87, 73, 78, 68, 202, 2, 100, 8, 65, 78, 65, + 71, 65, 82, 73, 32, 133, 18, 12, 73, 67, 69, 32, 67, 79, 78, 84, 82, 79, + 76, 32, 192, 2, 222, 1, 65, 38, 67, 22, 71, 36, 4, 72, 69, 65, 68, 96, 7, + 76, 69, 84, 84, 69, 82, 32, 202, 5, 83, 148, 7, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 186, 144, 19, 68, 160, 139, 6, 4, 74, 65, 73, 78, + 135, 165, 4, 79, 4, 218, 230, 12, 67, 187, 216, 12, 66, 2, 199, 153, 6, + 65, 4, 202, 230, 12, 82, 151, 238, 1, 65, 6, 44, 5, 32, 77, 65, 82, 75, + 199, 174, 29, 83, 5, 217, 158, 19, 7, 32, 87, 73, 84, 72, 32, 72, 158, 1, + 162, 2, 65, 54, 67, 62, 68, 50, 71, 62, 72, 52, 2, 77, 65, 46, 83, 170, + 165, 7, 66, 150, 131, 5, 75, 206, 202, 3, 82, 150, 173, 3, 79, 238, 192, + 3, 89, 214, 118, 85, 250, 35, 78, 138, 199, 1, 74, 174, 57, 76, 70, 84, + 46, 86, 242, 207, 1, 73, 134, 197, 1, 80, 2, 90, 138, 69, 70, 2, 81, 187, + 2, 69, 13, 230, 219, 29, 65, 2, 73, 2, 85, 2, 87, 3, 89, 10, 26, 65, 227, + 216, 29, 72, 9, 185, 2, 4, 78, 68, 82, 65, 12, 166, 253, 25, 68, 154, + 219, 3, 72, 187, 2, 65, 10, 134, 252, 12, 76, 130, 151, 16, 72, 138, 69, + 71, 187, 2, 65, 4, 136, 225, 16, 4, 69, 65, 86, 89, 131, 249, 12, 65, 5, + 157, 162, 7, 6, 82, 87, 65, 82, 73, 32, 12, 38, 72, 206, 214, 29, 83, + 187, 2, 65, 8, 36, 3, 79, 82, 84, 223, 216, 29, 65, 6, 167, 150, 12, 32, + 68, 168, 1, 19, 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, + 84, 84, 69, 82, 32, 104, 4, 73, 71, 78, 32, 137, 156, 27, 10, 84, 82, 69, + 83, 83, 32, 83, 73, 71, 78, 16, 210, 223, 3, 71, 2, 75, 160, 250, 23, 3, + 68, 68, 68, 2, 82, 206, 250, 1, 89, 38, 70, 2, 81, 3, 90, 48, 178, 3, 66, + 0, 10, 69, 88, 84, 69, 78, 68, 69, 68, 32, 66, 36, 11, 67, 65, 78, 68, + 82, 65, 66, 73, 78, 68, 85, 64, 8, 87, 69, 83, 84, 69, 82, 78, 32, 32, + 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 78, 138, 208, 6, 83, 182, 198, + 12, 73, 150, 158, 6, 77, 46, 78, 190, 66, 65, 72, 18, 68, 79, 85, 66, 76, + 69, 32, 67, 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 62, 80, 144, 198, 2, + 12, 72, 73, 71, 72, 32, 83, 80, 65, 67, 73, 78, 71, 167, 80, 86, 4, 245, + 236, 12, 4, 72, 65, 76, 69, 11, 11, 32, 8, 206, 213, 22, 65, 154, 163, 3, + 86, 163, 231, 1, 84, 4, 30, 78, 21, 3, 70, 73, 86, 2, 17, 2, 73, 78, 2, + 193, 240, 21, 8, 69, 45, 76, 73, 75, 69, 32, 66, 50, 150, 1, 65, 52, 7, + 67, 65, 78, 68, 82, 65, 32, 218, 150, 19, 79, 34, 80, 162, 183, 4, 85, + 194, 229, 1, 83, 170, 69, 86, 166, 202, 1, 73, 199, 140, 2, 69, 10, 154, + 208, 29, 65, 2, 73, 2, 85, 2, 87, 3, 89, 6, 176, 151, 19, 4, 76, 79, 78, + 71, 182, 184, 10, 69, 3, 79, 10, 222, 179, 24, 70, 136, 6, 2, 83, 84, + 254, 162, 3, 84, 155, 87, 79, 232, 2, 182, 2, 65, 150, 2, 69, 74, 71, 224, 4, 6, 78, 71, 66, 65, 84, 32, 248, 1, 5, 82, 69, 67, 84, 32, 98, 83, - 178, 2, 86, 200, 7, 2, 89, 65, 32, 4, 90, 90, 89, 32, 224, 172, 5, 2, 84, - 84, 184, 156, 15, 10, 70, 70, 69, 82, 69, 78, 67, 69, 32, 66, 173, 205, - 7, 12, 77, 69, 78, 83, 73, 79, 78, 32, 79, 82, 73, 71, 20, 42, 77, 242, - 251, 8, 69, 183, 134, 13, 71, 16, 40, 4, 79, 78, 68, 32, 207, 152, 3, 69, - 14, 128, 1, 5, 87, 73, 84, 72, 32, 186, 164, 18, 84, 164, 224, 2, 12, 83, - 72, 65, 80, 69, 32, 87, 73, 84, 72, 32, 65, 183, 228, 4, 79, 8, 190, 128, - 22, 66, 182, 2, 84, 238, 137, 4, 76, 27, 82, 14, 176, 131, 5, 5, 32, 70, - 65, 67, 69, 133, 161, 17, 4, 83, 69, 76, 32, 78, 64, 3, 73, 84, 32, 149, + 206, 2, 86, 200, 7, 2, 89, 65, 32, 4, 90, 90, 89, 32, 232, 179, 5, 2, 84, + 84, 228, 168, 15, 10, 70, 70, 69, 82, 69, 78, 67, 69, 32, 66, 197, 219, + 7, 12, 77, 69, 78, 83, 73, 79, 78, 32, 79, 82, 73, 71, 20, 42, 77, 226, + 132, 9, 69, 155, 145, 13, 71, 16, 40, 4, 79, 78, 68, 32, 171, 159, 3, 69, + 14, 128, 1, 5, 87, 73, 84, 72, 32, 198, 179, 18, 84, 232, 228, 2, 12, 83, + 72, 65, 80, 69, 32, 87, 73, 84, 72, 32, 65, 251, 238, 4, 79, 8, 146, 148, + 22, 66, 182, 2, 84, 174, 148, 4, 76, 27, 82, 14, 184, 138, 5, 5, 32, 70, + 65, 67, 69, 213, 173, 17, 4, 83, 69, 76, 32, 78, 64, 3, 73, 84, 32, 149, 2, 8, 82, 65, 77, 32, 70, 79, 82, 32, 60, 98, 70, 44, 2, 78, 73, 2, 79, 14, 83, 46, 84, 44, 3, 90, 69, 82, 13, 5, 69, 73, 71, 72, 84, 12, 128, 1, 2, 73, 86, 25, 3, 79, 85, 82, 6, 87, 78, 12, 96, 4, 69, 86, 69, 78, 1, 2, - 73, 88, 12, 28, 3, 72, 82, 69, 15, 87, 6, 23, 69, 6, 11, 79, 7, 143, 247, - 16, 32, 18, 88, 5, 69, 65, 82, 84, 72, 64, 5, 71, 82, 69, 65, 84, 0, 4, - 76, 69, 83, 83, 39, 72, 7, 25, 4, 76, 89, 32, 72, 4, 194, 174, 24, 85, - 147, 141, 1, 69, 4, 169, 132, 13, 4, 69, 82, 32, 89, 4, 188, 148, 5, 7, + 73, 88, 12, 28, 3, 72, 82, 69, 15, 87, 6, 23, 69, 6, 11, 79, 7, 219, 133, + 17, 32, 18, 88, 5, 69, 65, 82, 84, 72, 64, 5, 71, 82, 69, 65, 84, 0, 4, + 76, 69, 83, 83, 39, 72, 7, 25, 4, 76, 89, 32, 72, 4, 214, 199, 24, 85, + 239, 145, 1, 69, 4, 193, 146, 13, 4, 69, 82, 32, 89, 4, 200, 155, 5, 7, 69, 65, 86, 69, 78, 76, 89, 1, 4, 85, 77, 65, 78, 64, 120, 17, 78, 69, 71, 65, 84, 73, 86, 69, 32, 67, 73, 82, 67, 76, 69, 68, 32, 41, 9, 67, - 73, 82, 67, 76, 69, 68, 32, 83, 42, 38, 83, 154, 32, 78, 151, 196, 20, - 68, 22, 49, 10, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 22, 226, 31, 78, - 175, 144, 27, 68, 4, 148, 233, 22, 15, 67, 85, 82, 82, 69, 78, 84, 32, - 83, 89, 77, 66, 79, 76, 32, 195, 155, 6, 72, 10, 70, 65, 144, 1, 5, 67, - 79, 78, 84, 73, 197, 235, 25, 3, 71, 85, 73, 6, 76, 9, 80, 80, 79, 73, - 78, 84, 69, 68, 32, 133, 132, 27, 4, 66, 76, 69, 68, 4, 252, 159, 17, 7, - 66, 85, 84, 32, 82, 69, 76, 147, 194, 11, 70, 2, 53, 11, 78, 85, 79, 85, - 83, 32, 85, 78, 68, 69, 82, 2, 137, 130, 17, 3, 76, 73, 78, 156, 1, 80, - 9, 69, 83, 32, 65, 75, 85, 82, 85, 32, 238, 5, 73, 157, 251, 16, 2, 79, - 82, 144, 1, 142, 2, 68, 36, 7, 76, 69, 84, 84, 69, 82, 32, 236, 1, 5, 83, - 73, 71, 78, 32, 58, 86, 140, 162, 9, 6, 77, 69, 68, 73, 65, 76, 182, 255, - 4, 71, 222, 232, 4, 69, 128, 203, 4, 12, 80, 82, 69, 70, 73, 88, 69, 68, - 32, 78, 65, 83, 217, 248, 4, 7, 73, 78, 73, 84, 73, 65, 76, 22, 174, 190, - 25, 79, 211, 235, 1, 73, 84, 142, 195, 10, 84, 190, 237, 11, 89, 162, - 126, 65, 138, 146, 2, 68, 214, 6, 85, 206, 201, 1, 73, 42, 76, 250, 192, - 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 75, 2, 80, 138, 69, 72, 2, 74, 2, - 77, 2, 82, 2, 86, 2, 90, 186, 2, 69, 3, 79, 8, 174, 217, 24, 72, 150, 43, - 67, 98, 78, 235, 206, 3, 65, 18, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, - 78, 32, 179, 223, 26, 73, 16, 134, 150, 15, 65, 218, 174, 10, 85, 206, - 201, 1, 73, 222, 137, 2, 69, 3, 79, 10, 68, 5, 83, 73, 79, 78, 32, 212, - 241, 1, 2, 78, 71, 171, 145, 26, 68, 6, 26, 83, 227, 227, 5, 84, 4, 170, - 174, 27, 76, 211, 97, 73, 2, 165, 132, 19, 3, 32, 76, 65, 4, 134, 131, - 28, 83, 139, 86, 70, 212, 5, 188, 2, 6, 67, 85, 77, 69, 78, 84, 124, 7, - 69, 83, 32, 78, 79, 84, 32, 194, 3, 71, 202, 3, 76, 36, 10, 77, 73, 78, - 79, 32, 84, 73, 76, 69, 32, 226, 1, 78, 38, 84, 246, 2, 85, 204, 17, 2, - 87, 78, 164, 189, 16, 8, 32, 78, 79, 84, 32, 76, 73, 84, 156, 250, 11, 8, - 86, 69, 32, 79, 70, 32, 80, 69, 174, 4, 79, 235, 25, 68, 9, 33, 6, 32, - 87, 73, 84, 72, 32, 6, 40, 4, 84, 69, 88, 84, 139, 142, 2, 80, 5, 133, - 142, 2, 6, 32, 65, 78, 68, 32, 80, 22, 164, 1, 11, 67, 79, 78, 84, 65, - 73, 78, 32, 65, 83, 32, 92, 6, 68, 73, 86, 73, 68, 69, 112, 2, 80, 82, - 48, 7, 83, 85, 67, 67, 69, 69, 68, 253, 213, 22, 2, 70, 79, 6, 248, 1, - 15, 78, 79, 82, 77, 65, 76, 32, 83, 85, 66, 71, 82, 79, 85, 80, 231, 245, - 20, 77, 5, 217, 249, 21, 23, 32, 87, 73, 84, 72, 32, 82, 69, 86, 69, 82, - 83, 69, 68, 32, 78, 69, 71, 65, 84, 73, 79, 78, 6, 44, 5, 69, 67, 69, 68, - 69, 195, 146, 28, 79, 5, 233, 158, 9, 2, 32, 79, 125, 36, 3, 82, 65, 32, - 191, 209, 28, 32, 120, 120, 7, 76, 69, 84, 84, 69, 82, 32, 208, 1, 11, - 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 170, 161, 23, 65, 187, 2, 83, - 88, 178, 179, 25, 65, 38, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, - 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, - 138, 69, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 22, 194, 140, - 19, 86, 246, 171, 6, 65, 38, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, - 4, 202, 141, 21, 80, 183, 188, 2, 76, 200, 1, 72, 8, 72, 79, 82, 73, 90, - 79, 78, 84, 1, 6, 86, 69, 82, 84, 73, 67, 100, 17, 2, 65, 76, 100, 32, 2, - 45, 48, 155, 251, 23, 32, 98, 58, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 3, 54, 14, 205, 166, 20, 2, 45, 48, 4, 174, 239, 26, 75, 255, 146, 1, 71, - 26, 34, 32, 57, 4, 84, 69, 68, 32, 8, 170, 130, 25, 80, 34, 77, 194, 70, - 79, 239, 194, 2, 65, 18, 214, 1, 67, 34, 83, 220, 145, 16, 3, 76, 73, 78, - 148, 133, 1, 4, 79, 66, 69, 76, 244, 9, 9, 84, 82, 65, 78, 83, 80, 79, - 83, 73, 238, 220, 6, 70, 141, 41, 14, 82, 73, 71, 72, 84, 45, 80, 79, 73, - 78, 84, 73, 78, 71, 4, 226, 180, 27, 73, 231, 16, 82, 4, 134, 243, 26, - 79, 223, 114, 81, 162, 1, 40, 3, 66, 76, 69, 137, 17, 2, 71, 72, 160, 1, - 42, 32, 190, 10, 45, 241, 5, 2, 68, 32, 106, 226, 2, 67, 174, 1, 68, 38, - 72, 32, 4, 73, 78, 84, 69, 38, 76, 144, 1, 7, 78, 69, 83, 84, 69, 68, 32, - 74, 80, 66, 83, 130, 2, 85, 36, 9, 86, 69, 82, 84, 73, 67, 65, 76, 32, - 144, 161, 6, 6, 79, 66, 76, 73, 81, 85, 196, 189, 2, 9, 82, 73, 71, 72, - 84, 32, 65, 82, 67, 246, 183, 3, 65, 242, 215, 9, 69, 182, 188, 4, 81, - 177, 105, 6, 87, 65, 86, 89, 32, 79, 24, 100, 7, 73, 82, 67, 76, 69, 68, - 32, 180, 207, 7, 4, 85, 82, 76, 89, 241, 196, 1, 4, 79, 76, 79, 78, 20, - 26, 78, 151, 196, 20, 68, 2, 157, 213, 2, 5, 85, 77, 66, 69, 82, 4, 190, - 236, 18, 79, 135, 199, 6, 65, 4, 198, 190, 20, 73, 223, 63, 89, 4, 154, - 130, 26, 82, 251, 244, 1, 71, 12, 54, 79, 129, 156, 17, 7, 69, 70, 84, - 32, 65, 82, 67, 10, 26, 87, 199, 190, 25, 71, 6, 26, 45, 139, 230, 27, - 32, 4, 174, 189, 20, 82, 211, 1, 57, 6, 224, 137, 17, 9, 76, 69, 83, 83, - 45, 84, 72, 65, 78, 179, 227, 9, 71, 8, 26, 82, 207, 248, 26, 76, 6, 242, - 157, 1, 69, 207, 235, 15, 73, 18, 80, 6, 81, 85, 65, 82, 69, 32, 30, 84, - 50, 85, 149, 215, 13, 4, 79, 76, 73, 68, 4, 230, 205, 25, 73, 55, 85, 4, - 204, 191, 12, 3, 65, 67, 75, 135, 201, 4, 82, 8, 58, 83, 146, 156, 1, 67, - 222, 136, 21, 80, 151, 236, 4, 66, 2, 169, 147, 28, 4, 80, 69, 78, 83, 4, - 214, 231, 18, 80, 139, 196, 9, 78, 10, 36, 3, 66, 65, 82, 163, 226, 27, - 76, 9, 11, 32, 6, 52, 7, 68, 79, 85, 66, 76, 69, 32, 151, 213, 26, 76, 4, - 146, 213, 26, 76, 51, 82, 48, 112, 5, 76, 73, 78, 69, 32, 220, 1, 7, 83, - 84, 82, 85, 67, 75, 32, 141, 226, 8, 7, 69, 78, 68, 69, 68, 32, 77, 12, - 48, 8, 83, 76, 65, 78, 84, 69, 68, 32, 75, 69, 8, 70, 69, 56, 7, 71, 82, - 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 4, 185, 179, 20, 9, 81, 85, 65, - 76, 32, 84, 79, 32, 79, 2, 221, 165, 14, 5, 45, 84, 72, 65, 78, 34, 160, - 1, 8, 67, 65, 80, 73, 84, 65, 76, 32, 92, 7, 73, 84, 65, 76, 73, 67, 32, - 124, 6, 83, 77, 65, 76, 76, 32, 245, 208, 13, 8, 78, 45, 65, 82, 89, 32, - 83, 85, 18, 198, 163, 12, 71, 226, 201, 14, 80, 222, 137, 2, 67, 2, 72, - 2, 78, 2, 81, 2, 82, 3, 90, 10, 76, 6, 83, 77, 65, 76, 76, 32, 201, 230, - 21, 7, 67, 65, 80, 73, 84, 65, 76, 8, 214, 245, 28, 68, 2, 69, 2, 73, 3, - 74, 4, 238, 161, 12, 71, 231, 191, 16, 80, 6, 218, 161, 10, 70, 26, 77, - 135, 192, 17, 83, 2, 139, 251, 27, 78, 166, 1, 50, 32, 158, 1, 45, 189, - 1, 4, 87, 65, 82, 68, 10, 60, 4, 84, 65, 67, 75, 226, 206, 25, 70, 30, - 82, 191, 79, 65, 5, 29, 5, 32, 87, 73, 84, 72, 2, 11, 32, 2, 11, 67, 2, - 129, 245, 20, 4, 73, 82, 67, 76, 28, 60, 9, 80, 79, 73, 78, 84, 73, 78, - 71, 32, 211, 207, 25, 70, 24, 62, 83, 246, 209, 25, 65, 86, 69, 62, 70, - 46, 82, 207, 1, 84, 4, 142, 180, 17, 84, 205, 160, 8, 6, 77, 65, 76, 76, - 32, 82, 128, 1, 56, 8, 32, 70, 65, 67, 73, 78, 71, 32, 89, 2, 83, 32, 8, - 54, 72, 1, 9, 78, 79, 84, 67, 72, 69, 68, 32, 72, 4, 181, 204, 27, 3, 79, - 79, 75, 120, 178, 1, 65, 200, 2, 6, 66, 76, 65, 67, 75, 32, 38, 84, 172, - 237, 8, 2, 87, 72, 150, 154, 8, 90, 218, 250, 8, 68, 50, 70, 82, 72, 150, - 4, 67, 46, 81, 42, 82, 22, 83, 247, 7, 80, 34, 40, 4, 82, 82, 79, 87, - 179, 253, 25, 78, 33, 11, 32, 30, 144, 1, 5, 87, 73, 84, 72, 32, 160, - 228, 13, 14, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 85, 166, - 153, 12, 65, 178, 10, 70, 179, 5, 84, 22, 184, 183, 5, 6, 67, 79, 82, 78, - 69, 82, 182, 200, 20, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 250, - 12, 84, 139, 58, 72, 6, 178, 130, 21, 65, 175, 255, 4, 67, 36, 36, 2, 82, - 73, 161, 2, 2, 87, 79, 32, 44, 5, 65, 78, 71, 76, 69, 143, 144, 26, 80, - 30, 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 203, 149, 26, 32, 28, 68, 5, - 65, 82, 82, 79, 87, 230, 133, 17, 90, 166, 136, 9, 68, 39, 80, 23, 11, - 32, 20, 184, 137, 26, 15, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, - 32, 85, 80, 98, 84, 23, 87, 4, 134, 143, 26, 32, 105, 15, 45, 72, 69, 65, - 68, 69, 68, 32, 65, 82, 82, 79, 87, 32, 87, 22, 138, 1, 65, 106, 79, 152, - 1, 12, 85, 77, 32, 87, 73, 84, 72, 32, 68, 82, 85, 77, 160, 242, 16, 6, - 73, 86, 69, 32, 83, 76, 231, 220, 10, 69, 8, 238, 202, 2, 67, 200, 147, - 6, 10, 70, 84, 73, 78, 71, 32, 80, 79, 73, 78, 133, 235, 15, 3, 71, 79, - 78, 8, 56, 6, 77, 69, 68, 65, 82, 89, 34, 80, 159, 177, 27, 79, 2, 217, - 234, 20, 3, 32, 67, 65, 4, 204, 181, 18, 6, 32, 79, 70, 32, 66, 76, 171, - 251, 9, 76, 2, 189, 217, 12, 3, 83, 84, 73, 162, 2, 76, 7, 80, 76, 79, - 89, 65, 78, 32, 244, 132, 20, 2, 77, 80, 215, 220, 8, 67, 158, 2, 212, 2, - 6, 65, 70, 70, 73, 88, 32, 160, 7, 7, 76, 69, 84, 84, 69, 82, 32, 148, - 194, 1, 16, 84, 72, 73, 67, 75, 32, 76, 69, 84, 84, 69, 82, 32, 83, 69, - 76, 180, 174, 3, 4, 68, 79, 85, 66, 220, 230, 16, 19, 80, 85, 78, 67, 84, - 85, 65, 84, 73, 79, 78, 32, 67, 72, 73, 78, 79, 79, 75, 149, 194, 5, 11, - 83, 73, 71, 78, 32, 79, 32, 87, 73, 84, 72, 64, 140, 1, 9, 65, 84, 84, - 65, 67, 72, 69, 68, 32, 184, 1, 5, 72, 73, 71, 72, 32, 106, 76, 40, 4, - 82, 73, 71, 72, 173, 2, 4, 77, 73, 68, 32, 14, 112, 2, 84, 65, 232, 4, - 13, 76, 69, 70, 84, 45, 84, 79, 45, 82, 73, 71, 72, 84, 26, 83, 166, 239, - 20, 69, 3, 73, 6, 44, 5, 78, 71, 69, 78, 84, 159, 214, 27, 73, 5, 255, - 243, 20, 32, 20, 174, 2, 76, 50, 84, 38, 86, 166, 174, 8, 71, 186, 2, 65, - 170, 192, 18, 87, 186, 25, 67, 199, 129, 1, 68, 24, 36, 2, 69, 70, 29, 3, - 79, 87, 32, 2, 213, 2, 3, 84, 32, 72, 22, 94, 65, 38, 76, 50, 84, 38, 86, - 166, 174, 8, 71, 226, 194, 18, 87, 186, 25, 67, 199, 129, 1, 68, 4, 186, - 229, 21, 67, 199, 221, 5, 82, 4, 132, 128, 8, 3, 79, 78, 71, 219, 195, - 19, 73, 2, 229, 165, 8, 4, 73, 71, 72, 84, 4, 37, 7, 69, 82, 84, 73, 67, - 65, 76, 5, 131, 1, 32, 4, 42, 72, 41, 6, 86, 69, 82, 84, 73, 67, 2, 37, - 7, 79, 82, 73, 90, 79, 78, 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, - 161, 209, 27, 2, 69, 67, 214, 1, 222, 1, 65, 22, 68, 34, 69, 30, 70, 22, - 71, 22, 74, 162, 1, 75, 66, 76, 50, 77, 62, 78, 134, 1, 79, 50, 80, 86, - 82, 74, 83, 210, 2, 84, 66, 85, 42, 86, 38, 87, 70, 88, 162, 210, 27, 72, - 142, 60, 73, 206, 41, 89, 215, 22, 66, 5, 151, 180, 28, 79, 7, 142, 155, - 28, 32, 223, 61, 72, 7, 202, 216, 28, 69, 3, 85, 5, 207, 136, 28, 32, 5, - 255, 226, 24, 32, 19, 11, 32, 16, 76, 8, 87, 73, 84, 72, 32, 68, 79, 84, - 230, 5, 77, 2, 78, 195, 173, 27, 83, 5, 165, 218, 27, 12, 83, 32, 73, 78, - 83, 73, 68, 69, 32, 65, 78, 68, 9, 26, 32, 207, 214, 28, 75, 4, 186, 225, - 24, 82, 147, 245, 3, 77, 9, 248, 204, 22, 3, 79, 78, 71, 175, 137, 6, 72, - 11, 11, 32, 8, 162, 4, 78, 218, 173, 27, 87, 243, 163, 1, 83, 19, 38, 32, - 49, 5, 65, 83, 65, 76, 32, 8, 202, 3, 77, 218, 173, 27, 87, 243, 163, 1, - 83, 8, 226, 212, 28, 65, 2, 73, 2, 79, 3, 85, 11, 166, 211, 28, 79, 146, - 1, 65, 2, 85, 3, 87, 9, 52, 7, 69, 82, 78, 73, 78, 32, 65, 243, 131, 28, - 32, 4, 206, 211, 28, 77, 3, 78, 11, 248, 201, 22, 6, 79, 77, 65, 78, 73, - 65, 222, 203, 5, 32, 223, 61, 72, 49, 58, 32, 164, 1, 5, 76, 79, 65, 78, - 32, 191, 178, 6, 72, 26, 102, 74, 22, 75, 2, 80, 2, 84, 20, 7, 87, 73, - 84, 72, 32, 68, 79, 162, 209, 28, 77, 2, 78, 3, 83, 5, 235, 147, 28, 32, - 5, 215, 152, 28, 32, 4, 139, 217, 12, 84, 18, 74, 69, 230, 195, 5, 79, - 254, 188, 22, 65, 210, 78, 68, 146, 1, 74, 3, 85, 6, 190, 208, 28, 69, 2, - 72, 3, 78, 9, 26, 32, 131, 208, 28, 72, 4, 238, 218, 24, 82, 147, 245, 3, - 83, 9, 250, 255, 27, 32, 226, 79, 72, 3, 73, 5, 245, 148, 7, 4, 79, 67, - 65, 76, 17, 66, 79, 242, 149, 28, 32, 134, 37, 69, 218, 19, 65, 2, 72, 3, - 73, 5, 203, 206, 28, 87, 188, 91, 234, 2, 65, 188, 3, 10, 68, 73, 84, 79, - 82, 73, 65, 76, 32, 67, 22, 71, 224, 79, 4, 73, 71, 72, 84, 146, 2, 76, - 194, 9, 77, 214, 5, 78, 246, 3, 79, 36, 2, 81, 85, 182, 8, 82, 222, 1, - 83, 118, 84, 242, 31, 85, 198, 1, 88, 128, 5, 2, 89, 69, 200, 80, 4, 45, - 77, 65, 73, 148, 140, 19, 3, 74, 69, 67, 228, 147, 2, 8, 86, 69, 82, 71, - 82, 69, 69, 78, 175, 190, 5, 80, 22, 38, 82, 202, 230, 26, 83, 179, 64, - 71, 19, 30, 32, 137, 1, 2, 84, 72, 6, 88, 3, 79, 70, 32, 233, 229, 4, 13, - 87, 73, 84, 72, 32, 72, 69, 65, 82, 73, 78, 71, 32, 4, 140, 190, 11, 2, - 77, 65, 135, 230, 6, 82, 11, 17, 2, 32, 71, 8, 44, 5, 76, 79, 66, 69, 32, - 203, 148, 25, 82, 6, 70, 65, 181, 235, 21, 11, 69, 85, 82, 79, 80, 69, - 45, 65, 70, 82, 73, 4, 176, 137, 10, 4, 77, 69, 82, 73, 133, 154, 2, 11, - 83, 73, 65, 45, 65, 85, 83, 84, 82, 65, 76, 2, 183, 176, 2, 79, 228, 79, - 92, 17, 89, 80, 84, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, - 72, 151, 199, 28, 71, 226, 79, 30, 32, 197, 74, 2, 45, 49, 172, 17, 146, - 3, 65, 178, 4, 66, 52, 2, 67, 48, 224, 1, 2, 68, 48, 170, 4, 69, 178, 3, - 70, 164, 4, 2, 71, 48, 210, 3, 72, 214, 1, 73, 238, 2, 76, 122, 77, 222, - 8, 78, 210, 5, 79, 140, 5, 2, 80, 48, 120, 2, 82, 48, 232, 1, 2, 83, 48, - 190, 3, 84, 220, 2, 2, 85, 48, 250, 2, 86, 134, 5, 87, 236, 2, 3, 88, 48, - 48, 88, 3, 89, 48, 48, 84, 2, 90, 48, 252, 214, 3, 3, 75, 48, 48, 189, - 146, 15, 3, 81, 48, 48, 228, 1, 30, 48, 133, 3, 2, 65, 48, 160, 1, 86, - 48, 98, 49, 102, 52, 166, 55, 51, 162, 241, 20, 55, 150, 227, 1, 50, 2, - 53, 3, 54, 24, 170, 68, 54, 170, 146, 22, 53, 134, 236, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 55, 2, 56, 3, 57, 24, 242, 213, 22, 52, 2, 55, 134, 236, 5, - 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 56, 3, 57, 28, 142, 213, 22, - 48, 2, 50, 2, 51, 2, 53, 134, 236, 5, 49, 2, 52, 2, 54, 2, 55, 2, 56, 3, - 57, 68, 46, 48, 246, 54, 51, 210, 211, 22, 49, 3, 50, 22, 210, 65, 55, - 174, 254, 27, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 26, - 148, 9, 4, 69, 71, 73, 78, 185, 25, 2, 48, 48, 56, 34, 48, 90, 49, 175, - 244, 8, 50, 24, 230, 39, 50, 234, 150, 28, 49, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 22, 242, 209, 22, 48, 134, 236, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 184, 1, 70, 48, 94, 51, - 102, 52, 102, 53, 106, 54, 194, 29, 50, 195, 230, 22, 49, 20, 194, 208, - 22, 56, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, - 24, 230, 207, 22, 49, 2, 52, 134, 236, 5, 48, 2, 50, 2, 51, 2, 53, 2, 54, - 2, 55, 2, 56, 3, 57, 24, 130, 207, 22, 54, 2, 56, 134, 236, 5, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 57, 42, 214, 60, 48, 202, 145, 22, - 50, 2, 52, 134, 236, 5, 49, 2, 51, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 32, - 194, 60, 55, 250, 252, 27, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, - 94, 30, 48, 189, 2, 2, 78, 68, 88, 38, 48, 94, 50, 102, 51, 215, 7, 49, - 22, 158, 204, 22, 56, 2, 57, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 3, 55, 24, 194, 203, 22, 48, 2, 56, 134, 236, 5, 49, 2, 50, 2, 51, - 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 18, 222, 202, 22, 52, 134, 236, 5, 48, - 2, 49, 2, 50, 2, 51, 2, 54, 2, 55, 3, 56, 6, 11, 32, 6, 136, 180, 5, 6, - 87, 65, 76, 76, 69, 68, 210, 19, 69, 203, 211, 20, 83, 132, 1, 42, 48, - 133, 9, 5, 85, 76, 76, 32, 66, 130, 1, 54, 48, 94, 49, 102, 51, 102, 52, - 102, 53, 215, 1, 50, 20, 202, 200, 22, 49, 134, 236, 5, 50, 2, 51, 2, 52, - 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 22, 238, 199, 22, 51, 134, 236, 5, 48, - 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 26, 138, 199, 22, - 49, 2, 55, 2, 56, 134, 236, 5, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, - 57, 26, 166, 198, 22, 53, 2, 54, 2, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, - 51, 2, 52, 2, 56, 3, 57, 14, 222, 26, 49, 234, 150, 28, 48, 2, 50, 3, 51, - 128, 1, 62, 48, 98, 49, 102, 51, 102, 52, 210, 27, 50, 147, 201, 8, 53, - 24, 166, 50, 55, 170, 146, 22, 54, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, - 2, 53, 2, 56, 3, 57, 22, 238, 195, 22, 49, 134, 236, 5, 48, 2, 50, 2, 51, - 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 138, 195, 22, 54, 2, 55, - 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 56, 3, 57, 24, - 166, 194, 22, 51, 2, 53, 134, 236, 5, 48, 2, 49, 2, 50, 2, 52, 2, 54, 2, - 55, 2, 56, 3, 57, 24, 80, 2, 48, 48, 84, 4, 65, 76, 70, 32, 161, 40, 7, - 79, 82, 73, 90, 79, 78, 84, 18, 238, 192, 22, 54, 134, 236, 5, 49, 2, 50, - 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 4, 22, 66, 255, 42, 76, 2, 223, 238, - 16, 76, 52, 58, 48, 181, 1, 9, 78, 83, 69, 82, 84, 32, 65, 84, 32, 38, - 18, 48, 95, 49, 22, 158, 191, 22, 53, 2, 57, 134, 236, 5, 49, 2, 50, 2, - 51, 2, 52, 2, 54, 2, 55, 3, 56, 16, 194, 190, 22, 48, 2, 49, 134, 236, 5, - 50, 2, 51, 2, 52, 3, 53, 14, 68, 6, 66, 79, 84, 84, 79, 77, 0, 3, 84, 79, - 80, 215, 212, 19, 77, 7, 11, 32, 4, 182, 155, 27, 69, 221, 6, 2, 83, 84, - 22, 32, 2, 48, 48, 203, 221, 15, 79, 20, 222, 188, 22, 50, 2, 54, 134, - 236, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 164, 1, 118, 48, 128, 4, - 15, 79, 68, 73, 70, 73, 69, 82, 32, 68, 65, 77, 65, 71, 69, 68, 205, 228, - 24, 5, 73, 82, 82, 79, 82, 132, 1, 42, 48, 98, 49, 106, 50, 102, 51, 107, - 52, 24, 182, 40, 49, 170, 146, 22, 51, 134, 236, 5, 50, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 44, 138, 41, 50, 246, 144, 22, 48, 2, 53, 2, 54, - 2, 55, 134, 236, 5, 49, 2, 51, 2, 52, 2, 56, 3, 57, 26, 150, 185, 22, 50, - 2, 52, 2, 56, 134, 236, 5, 48, 2, 49, 2, 51, 2, 53, 2, 54, 2, 55, 3, 57, - 26, 138, 38, 51, 170, 146, 22, 49, 134, 236, 5, 48, 2, 50, 2, 52, 2, 53, - 2, 54, 2, 55, 2, 56, 3, 57, 12, 202, 183, 22, 48, 134, 236, 5, 49, 2, 50, - 2, 51, 3, 52, 31, 25, 4, 32, 65, 84, 32, 28, 96, 6, 66, 79, 84, 84, 79, - 77, 120, 5, 83, 84, 65, 82, 84, 68, 3, 84, 79, 80, 227, 146, 27, 69, 11, - 11, 32, 8, 56, 5, 83, 84, 65, 82, 84, 186, 1, 65, 159, 146, 27, 69, 5, - 129, 2, 8, 32, 65, 78, 68, 32, 84, 79, 80, 7, 29, 5, 32, 65, 78, 68, 32, - 4, 206, 181, 24, 66, 155, 242, 2, 84, 11, 11, 32, 8, 54, 65, 20, 5, 83, - 84, 65, 82, 84, 139, 146, 27, 69, 2, 73, 2, 78, 68, 5, 53, 11, 32, 65, - 78, 68, 32, 66, 79, 84, 84, 79, 77, 2, 251, 209, 19, 32, 194, 1, 50, 48, - 128, 2, 2, 76, 48, 229, 1, 2, 85, 48, 98, 58, 49, 98, 51, 194, 14, 50, - 150, 6, 52, 227, 209, 22, 48, 24, 146, 32, 56, 174, 254, 27, 48, 2, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 28, 218, 177, 22, 51, 2, - 52, 2, 53, 2, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, 54, 2, 56, 3, 57, 44, - 34, 48, 94, 49, 175, 131, 21, 50, 20, 210, 176, 22, 53, 134, 236, 5, 49, - 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 22, 246, 175, 22, 55, - 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, - 52, 34, 49, 102, 50, 231, 226, 22, 48, 26, 238, 174, 22, 48, 2, 49, 2, - 56, 134, 236, 5, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 8, 138, - 174, 22, 50, 134, 236, 5, 48, 3, 49, 152, 1, 50, 48, 145, 247, 18, 6, 86, - 69, 82, 76, 65, 89, 150, 1, 66, 48, 154, 1, 49, 138, 1, 50, 102, 51, 106, - 53, 191, 223, 22, 52, 34, 90, 54, 134, 172, 22, 49, 2, 53, 134, 236, 5, - 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, 134, 152, 28, 65, 2, 66, 2, - 67, 2, 68, 2, 69, 3, 70, 28, 98, 48, 230, 170, 22, 57, 134, 236, 5, 49, - 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 9, 230, 150, 28, 65, 2, - 66, 3, 67, 28, 190, 170, 22, 48, 2, 52, 2, 53, 2, 57, 134, 236, 5, 49, 2, - 50, 2, 51, 2, 54, 2, 55, 3, 56, 32, 134, 23, 54, 214, 146, 22, 48, 2, 51, - 134, 236, 5, 49, 2, 50, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 8, 202, 22, - 48, 175, 254, 27, 49, 26, 26, 48, 143, 194, 8, 49, 22, 182, 168, 22, 49, - 2, 51, 134, 236, 5, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 68, 34, - 48, 98, 49, 163, 221, 22, 50, 24, 142, 21, 51, 170, 146, 22, 50, 134, - 236, 5, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 214, 166, 22, - 48, 2, 54, 134, 236, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, - 57, 108, 50, 48, 94, 49, 106, 50, 98, 51, 171, 172, 19, 52, 22, 190, 165, - 22, 50, 2, 54, 134, 236, 5, 49, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, - 26, 186, 18, 52, 170, 146, 22, 55, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, - 2, 53, 2, 54, 2, 56, 3, 57, 24, 210, 17, 54, 174, 254, 27, 48, 2, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 22, 154, 163, 22, 53, 134, - 236, 5, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 90, - 34, 48, 249, 12, 3, 65, 76, 76, 88, 42, 48, 94, 49, 102, 51, 243, 214, - 22, 50, 26, 230, 161, 22, 51, 2, 55, 2, 56, 2, 57, 134, 236, 5, 49, 2, - 50, 2, 52, 2, 53, 3, 54, 24, 138, 161, 22, 49, 2, 54, 134, 236, 5, 48, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 18, 166, 160, 22, 50, 2, - 51, 134, 236, 5, 48, 2, 49, 2, 52, 2, 53, 3, 54, 94, 50, 48, 90, 50, 102, - 51, 102, 52, 211, 211, 22, 49, 22, 254, 12, 54, 174, 254, 27, 49, 2, 50, - 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, 206, 158, 22, 51, 2, 57, - 134, 236, 5, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 22, - 234, 157, 22, 50, 134, 236, 5, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, - 55, 2, 56, 3, 57, 6, 138, 137, 28, 48, 2, 49, 3, 50, 158, 1, 42, 48, 185, - 4, 5, 69, 82, 84, 73, 67, 156, 1, 62, 48, 98, 49, 98, 50, 210, 1, 51, - 225, 152, 22, 2, 52, 48, 42, 198, 9, 55, 98, 49, 202, 145, 22, 50, 134, - 236, 5, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 32, 186, 8, 49, 46, 50, - 174, 254, 27, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 50, - 98, 48, 206, 153, 22, 51, 2, 56, 2, 57, 134, 236, 5, 49, 2, 50, 2, 52, 2, - 53, 2, 54, 3, 55, 27, 206, 133, 28, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, - 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 3, 76, 28, 222, 152, 22, 48, 2, - 49, 2, 51, 2, 55, 134, 236, 5, 50, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 2, - 221, 221, 23, 2, 65, 76, 66, 34, 48, 161, 2, 3, 73, 68, 69, 64, 26, 48, - 94, 49, 103, 50, 22, 158, 151, 22, 51, 2, 57, 134, 236, 5, 49, 2, 50, 2, - 52, 2, 53, 2, 54, 2, 55, 3, 56, 28, 194, 150, 22, 48, 2, 52, 2, 55, 2, - 56, 134, 236, 5, 49, 2, 50, 2, 51, 2, 53, 2, 54, 3, 57, 14, 222, 149, 22, - 52, 134, 236, 5, 48, 2, 49, 2, 50, 2, 51, 3, 53, 2, 17, 2, 32, 76, 2, - 239, 181, 15, 79, 24, 202, 2, 52, 170, 146, 22, 54, 2, 56, 134, 236, 5, - 49, 2, 50, 2, 51, 2, 53, 3, 55, 18, 154, 148, 22, 49, 134, 236, 5, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 82, 22, 48, 167, 1, 49, 34, 90, - 50, 46, 51, 170, 146, 22, 52, 2, 53, 134, 236, 5, 49, 2, 54, 2, 55, 2, - 56, 3, 57, 11, 214, 254, 27, 65, 2, 66, 2, 67, 3, 68, 7, 170, 254, 27, - 65, 3, 66, 48, 66, 53, 86, 54, 250, 252, 27, 48, 2, 49, 2, 50, 2, 51, 3, - 52, 21, 202, 253, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, - 72, 3, 73, 19, 246, 252, 27, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, - 71, 3, 72, 182, 62, 22, 51, 211, 1, 52, 192, 46, 106, 52, 174, 188, 5, + 73, 82, 67, 76, 69, 68, 32, 83, 42, 38, 83, 182, 32, 78, 203, 215, 20, + 68, 22, 49, 10, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 22, 254, 31, 78, + 147, 175, 27, 68, 4, 132, 174, 27, 15, 67, 85, 82, 82, 69, 78, 84, 32, + 83, 89, 77, 66, 79, 76, 32, 187, 248, 1, 72, 12, 98, 65, 144, 1, 5, 67, + 79, 78, 84, 73, 192, 240, 15, 3, 84, 79, 82, 253, 152, 10, 3, 71, 85, 73, + 6, 76, 9, 80, 80, 79, 73, 78, 84, 69, 68, 32, 245, 161, 27, 4, 66, 76, + 69, 68, 4, 144, 175, 17, 7, 66, 85, 84, 32, 82, 69, 76, 203, 212, 11, 70, + 2, 53, 11, 78, 85, 79, 85, 83, 32, 85, 78, 68, 69, 82, 2, 157, 145, 17, + 3, 76, 73, 78, 156, 1, 80, 9, 69, 83, 32, 65, 75, 85, 82, 85, 32, 238, 5, + 73, 177, 138, 17, 2, 79, 82, 144, 1, 142, 2, 68, 36, 7, 76, 69, 84, 84, + 69, 82, 32, 236, 1, 5, 83, 73, 71, 78, 32, 58, 86, 240, 170, 9, 6, 77, + 69, 68, 73, 65, 76, 234, 132, 5, 71, 134, 237, 4, 69, 148, 209, 4, 12, + 80, 82, 69, 70, 73, 88, 69, 68, 32, 78, 65, 83, 209, 129, 5, 7, 73, 78, + 73, 84, 73, 65, 76, 22, 166, 220, 25, 79, 191, 236, 1, 73, 84, 218, 208, + 10, 84, 190, 243, 11, 89, 182, 230, 2, 65, 162, 52, 68, 214, 6, 85, 186, + 202, 1, 73, 42, 76, 226, 195, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 75, + 2, 80, 138, 69, 72, 2, 74, 2, 77, 2, 82, 2, 86, 2, 90, 186, 2, 69, 3, 79, + 8, 158, 242, 24, 72, 210, 42, 67, 98, 78, 139, 216, 3, 65, 18, 64, 10, + 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 187, 253, 26, 73, 16, 166, 164, + 15, 65, 178, 190, 10, 85, 186, 202, 1, 73, 198, 140, 2, 69, 3, 79, 10, + 68, 5, 83, 73, 79, 78, 32, 136, 242, 1, 2, 78, 71, 183, 176, 26, 68, 6, + 26, 83, 235, 234, 5, 84, 4, 142, 205, 27, 76, 187, 100, 73, 2, 253, 150, + 19, 3, 32, 76, 65, 4, 182, 162, 28, 83, 167, 88, 70, 212, 5, 188, 2, 6, + 67, 85, 77, 69, 78, 84, 124, 7, 69, 83, 32, 78, 79, 84, 32, 194, 3, 71, + 202, 3, 76, 36, 10, 77, 73, 78, 79, 32, 84, 73, 76, 69, 32, 226, 1, 78, + 38, 84, 246, 2, 85, 204, 17, 2, 87, 78, 212, 203, 16, 8, 32, 78, 79, 84, + 32, 76, 73, 84, 184, 141, 12, 8, 86, 69, 32, 79, 70, 32, 80, 69, 174, 4, + 79, 235, 25, 68, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 40, 4, 84, 69, 88, + 84, 175, 143, 2, 80, 5, 169, 143, 2, 6, 32, 65, 78, 68, 32, 80, 22, 164, + 1, 11, 67, 79, 78, 84, 65, 73, 78, 32, 65, 83, 32, 92, 6, 68, 73, 86, 73, + 68, 69, 112, 2, 80, 82, 48, 7, 83, 85, 67, 67, 69, 69, 68, 185, 233, 22, + 2, 70, 79, 6, 248, 1, 15, 78, 79, 82, 77, 65, 76, 32, 83, 85, 66, 71, 82, + 79, 85, 80, 155, 137, 21, 77, 5, 145, 141, 22, 23, 32, 87, 73, 84, 72, + 32, 82, 69, 86, 69, 82, 83, 69, 68, 32, 78, 69, 71, 65, 84, 73, 79, 78, + 6, 44, 5, 69, 67, 69, 68, 69, 143, 180, 28, 79, 5, 205, 167, 9, 2, 32, + 79, 125, 36, 3, 82, 65, 32, 139, 243, 28, 32, 120, 120, 7, 76, 69, 84, + 84, 69, 82, 32, 208, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 254, 185, 23, 65, 187, 2, 83, 88, 170, 209, 25, 65, 38, 68, 82, 82, 34, + 84, 230, 5, 85, 186, 202, 1, 73, 138, 196, 1, 78, 46, 83, 82, 66, 2, 67, + 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, + 2, 69, 3, 79, 22, 130, 159, 19, 86, 174, 183, 6, 65, 38, 85, 186, 202, 1, + 73, 198, 140, 2, 69, 3, 79, 4, 142, 161, 21, 80, 199, 193, 2, 76, 200, 1, + 72, 8, 72, 79, 82, 73, 90, 79, 78, 84, 1, 6, 86, 69, 82, 84, 73, 67, 100, + 17, 2, 65, 76, 100, 32, 2, 45, 48, 155, 148, 24, 32, 98, 58, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 14, 149, 186, 20, 2, 45, 48, 4, 158, + 141, 27, 75, 219, 150, 1, 71, 26, 34, 32, 57, 4, 84, 69, 68, 32, 8, 162, + 159, 25, 80, 34, 77, 194, 71, 79, 195, 198, 2, 65, 18, 214, 1, 67, 34, + 83, 128, 160, 16, 3, 76, 73, 78, 148, 134, 1, 4, 79, 66, 69, 76, 244, 9, + 9, 84, 82, 65, 78, 83, 80, 79, 83, 73, 202, 230, 6, 70, 161, 41, 14, 82, + 73, 71, 72, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 4, 146, 212, 27, 73, + 239, 16, 82, 4, 246, 144, 27, 79, 183, 116, 81, 162, 1, 40, 3, 66, 76, + 69, 137, 17, 2, 71, 72, 160, 1, 42, 32, 190, 10, 45, 241, 5, 2, 68, 32, + 106, 226, 2, 67, 174, 1, 68, 38, 72, 32, 4, 73, 78, 84, 69, 38, 76, 144, + 1, 7, 78, 69, 83, 84, 69, 68, 32, 74, 80, 66, 83, 130, 2, 85, 36, 9, 86, + 69, 82, 84, 73, 67, 65, 76, 32, 188, 168, 6, 6, 79, 66, 76, 73, 81, 85, + 144, 191, 2, 9, 82, 73, 71, 72, 84, 32, 65, 82, 67, 134, 189, 3, 65, 166, + 221, 9, 69, 246, 198, 4, 81, 153, 106, 6, 87, 65, 86, 89, 32, 79, 24, + 100, 7, 73, 82, 67, 76, 69, 68, 32, 140, 215, 7, 4, 85, 82, 76, 89, 253, + 197, 1, 4, 79, 76, 79, 78, 20, 26, 78, 203, 215, 20, 68, 2, 221, 219, 2, + 5, 85, 77, 66, 69, 82, 4, 150, 255, 18, 79, 167, 210, 6, 65, 4, 250, 209, + 20, 73, 239, 63, 89, 4, 146, 160, 26, 82, 179, 246, 1, 71, 12, 54, 79, + 165, 171, 17, 7, 69, 70, 84, 32, 65, 82, 67, 10, 26, 87, 191, 220, 25, + 71, 6, 26, 45, 211, 133, 28, 32, 4, 226, 208, 20, 82, 211, 1, 57, 6, 132, + 153, 17, 9, 76, 69, 83, 83, 45, 84, 72, 65, 78, 255, 241, 9, 71, 8, 26, + 82, 179, 151, 27, 76, 6, 138, 158, 1, 69, 219, 250, 15, 73, 18, 80, 6, + 81, 85, 65, 82, 69, 32, 30, 84, 50, 85, 173, 229, 13, 4, 79, 76, 73, 68, + 4, 222, 235, 25, 73, 55, 85, 4, 212, 205, 12, 3, 65, 67, 75, 163, 202, 4, + 82, 8, 58, 83, 170, 156, 1, 67, 146, 156, 21, 80, 175, 247, 4, 66, 2, + 245, 180, 28, 4, 80, 69, 78, 83, 4, 174, 250, 18, 80, 255, 210, 9, 78, + 10, 36, 3, 66, 65, 82, 235, 129, 28, 76, 9, 11, 32, 6, 52, 7, 68, 79, 85, + 66, 76, 69, 32, 143, 243, 26, 76, 4, 138, 243, 26, 76, 51, 82, 48, 112, + 5, 76, 73, 78, 69, 32, 220, 1, 7, 83, 84, 82, 85, 67, 75, 32, 133, 235, + 8, 7, 69, 78, 68, 69, 68, 32, 77, 12, 48, 8, 83, 76, 65, 78, 84, 69, 68, + 32, 75, 69, 8, 70, 69, 56, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, + 83, 83, 4, 237, 198, 20, 9, 81, 85, 65, 76, 32, 84, 79, 32, 79, 2, 249, + 179, 14, 5, 45, 84, 72, 65, 78, 34, 160, 1, 8, 67, 65, 80, 73, 84, 65, + 76, 32, 92, 7, 73, 84, 65, 76, 73, 67, 32, 124, 6, 83, 77, 65, 76, 76, + 32, 141, 223, 13, 8, 78, 45, 65, 82, 89, 32, 83, 85, 18, 206, 177, 12, + 71, 190, 218, 14, 80, 198, 140, 2, 67, 2, 72, 2, 78, 2, 81, 2, 82, 3, 90, + 10, 76, 6, 83, 77, 65, 76, 76, 32, 129, 250, 21, 7, 67, 65, 80, 73, 84, + 65, 76, 8, 162, 151, 29, 68, 2, 69, 2, 73, 3, 74, 4, 246, 175, 12, 71, + 171, 211, 16, 80, 6, 166, 175, 10, 70, 26, 77, 235, 209, 17, 83, 2, 215, + 156, 28, 78, 166, 1, 50, 32, 158, 1, 45, 189, 1, 4, 87, 65, 82, 68, 10, + 60, 4, 84, 65, 67, 75, 218, 236, 25, 70, 30, 82, 195, 79, 65, 5, 29, 5, + 32, 87, 73, 84, 72, 2, 11, 32, 2, 11, 67, 2, 197, 136, 21, 4, 73, 82, 67, + 76, 28, 60, 9, 80, 79, 73, 78, 84, 73, 78, 71, 32, 203, 237, 25, 70, 24, + 62, 83, 238, 239, 25, 65, 86, 69, 62, 70, 46, 82, 207, 1, 84, 4, 178, + 195, 17, 84, 161, 175, 8, 6, 77, 65, 76, 76, 32, 82, 128, 1, 56, 8, 32, + 70, 65, 67, 73, 78, 71, 32, 89, 2, 83, 32, 8, 54, 72, 1, 9, 78, 79, 84, + 67, 72, 69, 68, 32, 72, 4, 237, 235, 27, 3, 79, 79, 75, 120, 178, 1, 65, + 200, 2, 6, 66, 76, 65, 67, 75, 32, 38, 84, 144, 246, 8, 2, 87, 72, 214, + 160, 8, 90, 178, 137, 9, 68, 50, 70, 82, 72, 150, 4, 67, 46, 81, 42, 82, + 22, 83, 247, 7, 80, 34, 40, 4, 82, 82, 79, 87, 175, 155, 26, 78, 33, 11, + 32, 30, 144, 1, 5, 87, 73, 84, 72, 32, 184, 242, 13, 14, 76, 69, 70, 84, + 87, 65, 82, 68, 83, 32, 79, 70, 32, 85, 138, 169, 12, 65, 178, 10, 70, + 179, 5, 84, 22, 192, 190, 5, 6, 67, 79, 82, 78, 69, 82, 170, 223, 20, 68, + 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 250, 12, 84, 135, 58, 72, 6, 246, + 149, 21, 65, 231, 137, 5, 67, 36, 36, 2, 82, 73, 161, 2, 2, 87, 79, 32, + 44, 5, 65, 78, 71, 76, 69, 139, 174, 26, 80, 30, 56, 8, 45, 72, 69, 65, + 68, 69, 68, 32, 199, 179, 26, 32, 28, 68, 5, 65, 82, 82, 79, 87, 138, + 149, 17, 90, 254, 150, 9, 68, 39, 80, 23, 11, 32, 20, 180, 167, 26, 15, + 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 85, 80, 98, 84, 23, + 87, 4, 130, 173, 26, 32, 105, 15, 45, 72, 69, 65, 68, 69, 68, 32, 65, 82, + 82, 79, 87, 32, 87, 22, 138, 1, 65, 106, 79, 152, 1, 12, 85, 77, 32, 87, + 73, 84, 72, 32, 68, 82, 85, 77, 196, 129, 17, 6, 73, 86, 69, 32, 83, 76, + 139, 237, 10, 69, 8, 174, 209, 2, 67, 128, 150, 6, 10, 70, 84, 73, 78, + 71, 32, 80, 79, 73, 78, 181, 250, 15, 3, 71, 79, 78, 8, 56, 6, 77, 69, + 68, 65, 82, 89, 34, 80, 215, 208, 27, 79, 2, 157, 254, 20, 3, 32, 67, 65, + 4, 240, 196, 18, 6, 32, 79, 70, 32, 66, 76, 211, 141, 10, 76, 2, 213, + 231, 12, 3, 83, 84, 73, 162, 2, 76, 7, 80, 76, 79, 89, 65, 78, 32, 188, + 152, 20, 2, 77, 80, 219, 234, 8, 67, 158, 2, 212, 2, 6, 65, 70, 70, 73, + 88, 32, 160, 7, 7, 76, 69, 84, 84, 69, 82, 32, 232, 194, 1, 16, 84, 72, + 73, 67, 75, 32, 76, 69, 84, 84, 69, 82, 32, 83, 69, 76, 232, 180, 3, 4, + 68, 79, 85, 66, 144, 243, 16, 19, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, + 78, 32, 67, 72, 73, 78, 79, 79, 75, 145, 206, 5, 11, 83, 73, 71, 78, 32, + 79, 32, 87, 73, 84, 72, 64, 140, 1, 9, 65, 84, 84, 65, 67, 72, 69, 68, + 32, 184, 1, 5, 72, 73, 71, 72, 32, 106, 76, 40, 4, 82, 73, 71, 72, 173, + 2, 4, 77, 73, 68, 32, 14, 112, 2, 84, 65, 232, 4, 13, 76, 69, 70, 84, 45, + 84, 79, 45, 82, 73, 71, 72, 84, 26, 83, 234, 130, 21, 69, 3, 73, 6, 44, + 5, 78, 71, 69, 78, 84, 207, 245, 27, 73, 5, 195, 135, 21, 32, 20, 174, 2, + 76, 50, 84, 38, 86, 250, 182, 8, 71, 186, 2, 65, 186, 214, 18, 87, 134, + 26, 67, 227, 131, 1, 68, 24, 36, 2, 69, 70, 29, 3, 79, 87, 32, 2, 213, 2, + 3, 84, 32, 72, 22, 94, 65, 38, 76, 50, 84, 38, 86, 250, 182, 8, 71, 242, + 216, 18, 87, 134, 26, 67, 227, 131, 1, 68, 4, 134, 249, 21, 67, 195, 233, + 5, 82, 4, 216, 136, 8, 3, 79, 78, 71, 207, 218, 19, 73, 2, 185, 174, 8, + 4, 73, 71, 72, 84, 4, 37, 7, 69, 82, 84, 73, 67, 65, 76, 5, 131, 1, 32, + 4, 42, 72, 41, 6, 86, 69, 82, 84, 73, 67, 2, 37, 7, 79, 82, 73, 90, 79, + 78, 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, 209, 240, 27, 2, 69, + 67, 214, 1, 222, 1, 65, 22, 68, 34, 69, 30, 70, 22, 71, 22, 74, 170, 1, + 75, 66, 76, 50, 77, 62, 78, 134, 1, 79, 50, 80, 86, 82, 74, 83, 210, 2, + 84, 66, 85, 42, 86, 38, 87, 70, 88, 230, 243, 27, 72, 142, 60, 73, 206, + 41, 89, 215, 22, 66, 5, 227, 213, 28, 79, 7, 218, 188, 28, 32, 223, 61, + 72, 7, 150, 250, 28, 69, 3, 85, 5, 155, 170, 28, 32, 5, 247, 255, 24, 32, + 19, 11, 32, 16, 76, 8, 87, 73, 84, 72, 32, 68, 79, 84, 238, 5, 77, 2, 78, + 243, 204, 27, 83, 5, 53, 11, 83, 32, 73, 78, 83, 73, 68, 69, 32, 65, 78, + 2, 187, 251, 27, 68, 9, 26, 32, 147, 248, 28, 75, 4, 170, 254, 24, 82, + 231, 249, 3, 77, 9, 224, 223, 22, 3, 79, 78, 71, 139, 152, 6, 72, 11, 11, + 32, 8, 162, 4, 78, 138, 205, 27, 87, 135, 166, 1, 83, 19, 38, 32, 49, 5, + 65, 83, 65, 76, 32, 8, 202, 3, 77, 138, 205, 27, 87, 135, 166, 1, 83, 8, + 166, 246, 28, 65, 2, 73, 2, 79, 3, 85, 11, 234, 244, 28, 79, 146, 1, 65, + 2, 85, 3, 87, 9, 52, 7, 69, 82, 78, 73, 78, 32, 65, 183, 165, 28, 32, 4, + 146, 245, 28, 77, 3, 78, 11, 224, 220, 22, 6, 79, 77, 65, 78, 73, 65, + 186, 218, 5, 32, 223, 61, 72, 49, 58, 32, 164, 1, 5, 76, 79, 65, 78, 32, + 239, 185, 6, 72, 26, 102, 74, 22, 75, 2, 80, 2, 84, 20, 7, 87, 73, 84, + 72, 32, 68, 79, 230, 242, 28, 77, 2, 78, 3, 83, 5, 175, 181, 28, 32, 5, + 155, 186, 28, 32, 4, 167, 231, 12, 84, 18, 74, 69, 138, 203, 5, 79, 158, + 215, 22, 65, 210, 78, 68, 146, 1, 74, 3, 85, 6, 130, 242, 28, 69, 2, 72, + 3, 78, 9, 26, 32, 199, 241, 28, 72, 4, 222, 247, 24, 82, 231, 249, 3, 83, + 9, 190, 161, 28, 32, 226, 79, 72, 3, 73, 5, 137, 157, 7, 4, 79, 67, 65, + 76, 17, 66, 79, 182, 183, 28, 32, 134, 37, 69, 218, 19, 65, 2, 72, 3, 73, + 5, 143, 240, 28, 87, 194, 91, 234, 2, 65, 188, 3, 10, 68, 73, 84, 79, 82, + 73, 65, 76, 32, 67, 22, 71, 240, 79, 4, 73, 71, 72, 84, 146, 2, 76, 194, + 9, 77, 214, 5, 78, 246, 3, 79, 36, 2, 81, 85, 182, 8, 82, 222, 1, 83, + 118, 84, 242, 31, 85, 226, 1, 88, 128, 5, 2, 89, 69, 208, 86, 4, 45, 77, + 65, 73, 156, 153, 19, 3, 74, 69, 67, 244, 152, 2, 8, 86, 69, 82, 71, 82, + 69, 69, 78, 167, 199, 5, 80, 22, 38, 82, 166, 133, 27, 83, 135, 65, 71, + 19, 30, 32, 137, 1, 2, 84, 72, 6, 88, 3, 79, 70, 32, 233, 236, 4, 13, 87, + 73, 84, 72, 32, 72, 69, 65, 82, 73, 78, 71, 32, 4, 132, 204, 11, 2, 77, + 65, 143, 232, 6, 82, 11, 17, 2, 32, 71, 8, 44, 5, 76, 79, 66, 69, 32, + 187, 178, 25, 82, 6, 70, 65, 249, 254, 21, 11, 69, 85, 82, 79, 80, 69, + 45, 65, 70, 82, 73, 4, 244, 150, 10, 4, 77, 69, 82, 73, 193, 154, 2, 11, + 83, 73, 65, 45, 65, 85, 83, 84, 82, 65, 76, 2, 239, 182, 2, 79, 230, 79, + 108, 17, 89, 80, 84, 73, 65, 78, 32, 72, 73, 69, 82, 79, 71, 76, 89, 80, + 72, 130, 217, 2, 69, 203, 143, 26, 71, 226, 79, 30, 32, 197, 74, 2, 45, + 49, 172, 17, 146, 3, 65, 178, 4, 66, 52, 2, 67, 48, 224, 1, 2, 68, 48, + 170, 4, 69, 178, 3, 70, 164, 4, 2, 71, 48, 210, 3, 72, 214, 1, 73, 238, + 2, 76, 122, 77, 222, 8, 78, 210, 5, 79, 140, 5, 2, 80, 48, 120, 2, 82, + 48, 232, 1, 2, 83, 48, 190, 3, 84, 220, 2, 2, 85, 48, 250, 2, 86, 134, 5, + 87, 236, 2, 3, 88, 48, 48, 88, 3, 89, 48, 48, 84, 2, 90, 48, 208, 221, 3, + 3, 75, 48, 48, 153, 159, 15, 3, 81, 48, 48, 228, 1, 30, 48, 133, 3, 2, + 65, 48, 160, 1, 86, 48, 98, 49, 102, 52, 166, 55, 51, 194, 132, 21, 55, + 198, 232, 1, 50, 2, 53, 3, 54, 24, 170, 68, 54, 242, 141, 24, 53, 242, + 145, 4, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 24, 186, 209, 24, + 52, 2, 55, 242, 145, 4, 48, 2, 49, 2, 50, 2, 51, 2, 53, 2, 54, 2, 56, 3, + 57, 28, 214, 208, 24, 48, 2, 50, 2, 51, 2, 53, 242, 145, 4, 49, 2, 52, 2, + 54, 2, 55, 2, 56, 3, 57, 68, 46, 48, 246, 54, 51, 162, 236, 22, 49, 3, + 50, 22, 210, 65, 55, 226, 159, 28, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 56, 3, 57, 26, 148, 9, 4, 69, 71, 73, 78, 185, 25, 2, 48, 48, 56, 34, + 48, 90, 49, 251, 252, 8, 50, 24, 230, 39, 50, 158, 184, 28, 49, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 22, 186, 205, 24, 48, 242, 145, 4, + 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 184, 1, 70, + 48, 94, 51, 102, 52, 102, 53, 106, 54, 194, 29, 50, 147, 255, 22, 49, 20, + 138, 204, 24, 56, 242, 145, 4, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 3, 57, 24, 174, 203, 24, 49, 2, 52, 242, 145, 4, 48, 2, 50, 2, 51, 2, + 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 202, 202, 24, 54, 2, 56, 242, 145, 4, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 57, 42, 214, 60, 48, + 146, 141, 24, 50, 2, 52, 242, 145, 4, 49, 2, 51, 2, 53, 2, 54, 2, 55, 2, + 56, 3, 57, 32, 194, 60, 55, 174, 158, 28, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 3, 54, 94, 30, 48, 189, 2, 2, 78, 68, 88, 38, 48, 94, 50, 102, 51, + 215, 7, 49, 22, 230, 199, 24, 56, 2, 57, 242, 145, 4, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 3, 55, 24, 138, 199, 24, 48, 2, 56, 242, 145, 4, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 18, 166, 198, 24, 52, + 242, 145, 4, 48, 2, 49, 2, 50, 2, 51, 2, 54, 2, 55, 3, 56, 6, 11, 32, 6, + 156, 187, 5, 6, 87, 65, 76, 76, 69, 68, 210, 19, 69, 143, 234, 20, 83, + 132, 1, 42, 48, 133, 9, 5, 85, 76, 76, 32, 66, 130, 1, 54, 48, 94, 49, + 102, 51, 102, 52, 102, 53, 215, 1, 50, 20, 146, 196, 24, 49, 242, 145, 4, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 22, 182, 195, 24, + 51, 242, 145, 4, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, + 57, 26, 210, 194, 24, 49, 2, 55, 2, 56, 242, 145, 4, 48, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 3, 57, 26, 238, 193, 24, 53, 2, 54, 2, 55, 242, 145, 4, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 56, 3, 57, 14, 222, 26, 49, 158, 184, + 28, 48, 2, 50, 3, 51, 128, 1, 62, 48, 98, 49, 102, 51, 102, 52, 210, 27, + 50, 223, 209, 8, 53, 24, 166, 50, 55, 242, 141, 24, 54, 242, 145, 4, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 56, 3, 57, 22, 182, 191, 24, 49, 242, 145, + 4, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, 210, + 190, 24, 54, 2, 55, 242, 145, 4, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, + 2, 56, 3, 57, 24, 238, 189, 24, 51, 2, 53, 242, 145, 4, 48, 2, 49, 2, 50, + 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 24, 80, 2, 48, 48, 84, 4, 65, 76, 70, + 32, 161, 40, 7, 79, 82, 73, 90, 79, 78, 84, 18, 182, 188, 24, 54, 242, + 145, 4, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 4, 22, 66, 255, 42, + 76, 2, 235, 253, 16, 76, 52, 58, 48, 181, 1, 9, 78, 83, 69, 82, 84, 32, + 65, 84, 32, 38, 18, 48, 95, 49, 22, 230, 186, 24, 53, 2, 57, 242, 145, 4, + 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 3, 56, 16, 138, 186, 24, 48, 2, + 49, 242, 145, 4, 50, 2, 51, 2, 52, 3, 53, 14, 68, 6, 66, 79, 84, 84, 79, + 77, 0, 3, 84, 79, 80, 243, 231, 19, 77, 7, 11, 32, 4, 206, 186, 27, 69, + 249, 8, 2, 83, 84, 22, 32, 2, 48, 48, 211, 235, 15, 79, 20, 166, 184, 24, + 50, 2, 54, 242, 145, 4, 49, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 164, 1, + 118, 48, 128, 4, 15, 79, 68, 73, 70, 73, 69, 82, 32, 68, 65, 77, 65, 71, + 69, 68, 173, 130, 25, 5, 73, 82, 82, 79, 82, 132, 1, 42, 48, 98, 49, 106, + 50, 102, 51, 107, 52, 24, 182, 40, 49, 242, 141, 24, 51, 242, 145, 4, 50, + 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 44, 138, 41, 50, 190, 140, 24, + 48, 2, 53, 2, 54, 2, 55, 242, 145, 4, 49, 2, 51, 2, 52, 2, 56, 3, 57, 26, + 222, 180, 24, 50, 2, 52, 2, 56, 242, 145, 4, 48, 2, 49, 2, 51, 2, 53, 2, + 54, 2, 55, 3, 57, 26, 138, 38, 51, 242, 141, 24, 49, 242, 145, 4, 48, 2, + 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 146, 179, 24, 48, 242, + 145, 4, 49, 2, 50, 2, 51, 3, 52, 31, 25, 4, 32, 65, 84, 32, 28, 96, 6, + 66, 79, 84, 84, 79, 77, 120, 5, 83, 84, 65, 82, 84, 68, 3, 84, 79, 80, + 251, 177, 27, 69, 11, 11, 32, 8, 56, 5, 83, 84, 65, 82, 84, 186, 1, 65, + 183, 177, 27, 69, 5, 129, 2, 8, 32, 65, 78, 68, 32, 84, 79, 80, 7, 29, 5, + 32, 65, 78, 68, 32, 4, 138, 211, 24, 66, 147, 246, 2, 84, 11, 11, 32, 8, + 54, 65, 20, 5, 83, 84, 65, 82, 84, 163, 177, 27, 69, 2, 73, 2, 78, 68, 5, + 53, 11, 32, 65, 78, 68, 32, 66, 79, 84, 84, 79, 77, 2, 151, 229, 19, 32, + 194, 1, 50, 48, 128, 2, 2, 76, 48, 229, 1, 2, 85, 48, 98, 58, 49, 98, 51, + 194, 14, 50, 150, 6, 52, 163, 234, 22, 48, 24, 146, 32, 56, 226, 159, 28, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 28, 162, 173, + 24, 51, 2, 52, 2, 53, 2, 55, 242, 145, 4, 48, 2, 49, 2, 50, 2, 54, 2, 56, + 3, 57, 44, 34, 48, 94, 49, 207, 150, 21, 50, 20, 154, 172, 24, 53, 242, + 145, 4, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 22, 190, + 171, 24, 55, 242, 145, 4, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 56, 3, 57, 52, 34, 49, 102, 50, 167, 251, 22, 48, 26, 182, 170, 24, + 48, 2, 49, 2, 56, 242, 145, 4, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, + 57, 8, 210, 169, 24, 50, 242, 145, 4, 48, 3, 49, 152, 1, 50, 48, 193, + 138, 19, 6, 86, 69, 82, 76, 65, 89, 150, 1, 66, 48, 154, 1, 49, 138, 1, + 50, 102, 51, 106, 53, 143, 248, 22, 52, 34, 90, 54, 206, 167, 24, 49, 2, + 53, 242, 145, 4, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, 186, 185, 28, + 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 28, 98, 48, 174, 166, 24, 57, 242, + 145, 4, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 9, 154, 184, + 28, 65, 2, 66, 3, 67, 28, 134, 166, 24, 48, 2, 52, 2, 53, 2, 57, 242, + 145, 4, 49, 2, 50, 2, 51, 2, 54, 2, 55, 3, 56, 32, 134, 23, 54, 158, 142, + 24, 48, 2, 51, 242, 145, 4, 49, 2, 50, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, + 8, 202, 22, 48, 227, 159, 28, 49, 26, 26, 48, 219, 202, 8, 49, 22, 254, + 163, 24, 49, 2, 51, 242, 145, 4, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, + 3, 57, 68, 34, 48, 98, 49, 243, 245, 22, 50, 24, 142, 21, 51, 242, 141, + 24, 50, 242, 145, 4, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 24, + 158, 162, 24, 48, 2, 54, 242, 145, 4, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 55, 2, 56, 3, 57, 108, 50, 48, 94, 49, 106, 50, 98, 51, 219, 191, 19, 52, + 22, 134, 161, 24, 50, 2, 54, 242, 145, 4, 49, 2, 51, 2, 52, 2, 53, 2, 55, + 2, 56, 3, 57, 26, 186, 18, 52, 242, 141, 24, 55, 242, 145, 4, 48, 2, 49, + 2, 50, 2, 51, 2, 53, 2, 54, 2, 56, 3, 57, 24, 210, 17, 54, 226, 159, 28, + 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 22, 226, 158, + 24, 53, 242, 145, 4, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, + 3, 57, 90, 34, 48, 249, 12, 3, 65, 76, 76, 88, 42, 48, 94, 49, 102, 51, + 195, 239, 22, 50, 26, 174, 157, 24, 51, 2, 55, 2, 56, 2, 57, 242, 145, 4, + 49, 2, 50, 2, 52, 2, 53, 3, 54, 24, 210, 156, 24, 49, 2, 54, 242, 145, 4, + 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 18, 238, 155, 24, + 50, 2, 51, 242, 145, 4, 48, 2, 49, 2, 52, 2, 53, 3, 54, 94, 50, 48, 90, + 50, 102, 51, 102, 52, 163, 236, 22, 49, 22, 254, 12, 54, 226, 159, 28, + 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, 2, 56, 3, 57, 24, 150, 154, 24, + 51, 2, 57, 242, 145, 4, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 3, + 56, 22, 178, 153, 24, 50, 242, 145, 4, 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 6, 190, 170, 28, 48, 2, 49, 3, 50, 158, 1, 42, + 48, 185, 4, 5, 69, 82, 84, 73, 67, 156, 1, 62, 48, 98, 49, 98, 50, 210, + 1, 51, 169, 148, 24, 2, 52, 48, 42, 198, 9, 55, 98, 49, 146, 141, 24, 50, + 242, 145, 4, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 32, 186, 8, 49, 46, + 50, 226, 159, 28, 48, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, + 50, 98, 48, 150, 149, 24, 51, 2, 56, 2, 57, 242, 145, 4, 49, 2, 50, 2, + 52, 2, 53, 2, 54, 3, 55, 27, 130, 167, 28, 65, 2, 66, 2, 67, 2, 68, 2, + 69, 2, 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 3, 76, 28, 166, 148, 24, + 48, 2, 49, 2, 51, 2, 55, 242, 145, 4, 50, 2, 52, 2, 53, 2, 54, 2, 56, 3, + 57, 2, 181, 246, 23, 2, 65, 76, 66, 34, 48, 161, 2, 3, 73, 68, 69, 64, + 26, 48, 94, 49, 103, 50, 22, 230, 146, 24, 51, 2, 57, 242, 145, 4, 49, 2, + 50, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 28, 138, 146, 24, 48, 2, 52, 2, + 55, 2, 56, 242, 145, 4, 49, 2, 50, 2, 51, 2, 53, 2, 54, 3, 57, 14, 166, + 145, 24, 52, 242, 145, 4, 48, 2, 49, 2, 50, 2, 51, 3, 53, 2, 17, 2, 32, + 76, 2, 247, 195, 15, 79, 24, 202, 2, 52, 242, 141, 24, 54, 2, 56, 242, + 145, 4, 49, 2, 50, 2, 51, 2, 53, 3, 55, 18, 226, 143, 24, 49, 242, 145, + 4, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 82, 22, 48, 167, 1, 49, + 34, 90, 50, 46, 51, 242, 141, 24, 52, 2, 53, 242, 145, 4, 49, 2, 54, 2, + 55, 2, 56, 3, 57, 11, 138, 160, 28, 65, 2, 66, 2, 67, 3, 68, 7, 222, 159, + 28, 65, 3, 66, 48, 66, 53, 86, 54, 174, 158, 28, 48, 2, 49, 2, 50, 2, 51, + 3, 52, 21, 254, 158, 28, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, + 72, 3, 73, 19, 170, 158, 28, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, + 71, 3, 72, 182, 62, 22, 51, 211, 1, 52, 192, 46, 106, 52, 206, 195, 5, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, - 192, 2, 214, 179, 13, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, - 68, 2, 69, 3, 70, 246, 15, 42, 51, 158, 187, 5, 48, 2, 49, 3, 50, 246, 3, - 142, 1, 70, 186, 177, 13, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, - 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 3, 69, 22, 250, 248, 27, + 192, 2, 218, 193, 13, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, + 68, 2, 69, 3, 70, 246, 15, 42, 51, 190, 194, 5, 48, 2, 49, 3, 50, 246, 3, + 142, 1, 70, 190, 191, 13, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 3, 69, 22, 174, 154, 28, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 3, 65, - 18, 26, 32, 239, 226, 15, 72, 16, 70, 80, 124, 5, 82, 65, 89, 83, 32, - 190, 146, 3, 84, 231, 207, 22, 83, 8, 186, 146, 3, 79, 241, 185, 16, 22, + 18, 26, 32, 235, 241, 15, 72, 16, 70, 80, 124, 5, 82, 65, 89, 83, 32, + 226, 152, 3, 84, 155, 231, 22, 83, 8, 222, 152, 3, 79, 233, 198, 16, 22, 69, 84, 65, 76, 76, 69, 68, 32, 79, 85, 84, 76, 73, 78, 69, 68, 32, 66, - 76, 65, 67, 75, 4, 228, 234, 18, 2, 73, 78, 1, 3, 79, 85, 84, 160, 1, + 76, 65, 67, 75, 4, 148, 254, 18, 2, 73, 78, 1, 3, 79, 85, 84, 160, 1, 132, 1, 13, 66, 65, 83, 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 166, 3, - 69, 176, 4, 7, 89, 77, 65, 73, 67, 32, 76, 171, 237, 27, 70, 80, 230, 1, - 71, 78, 76, 34, 78, 50, 82, 158, 232, 25, 69, 190, 137, 1, 67, 2, 68, 2, + 69, 176, 4, 7, 89, 77, 65, 73, 67, 32, 76, 223, 142, 28, 70, 80, 230, 1, + 71, 78, 76, 34, 78, 50, 82, 234, 134, 26, 69, 166, 140, 1, 67, 2, 68, 2, 75, 2, 83, 2, 84, 2, 90, 206, 105, 66, 2, 70, 2, 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 86, 2, 88, 214, 22, 65, 2, 73, 2, 79, 2, 85, 3, 89, 8, 38, 72, - 154, 220, 27, 74, 215, 22, 69, 4, 242, 160, 25, 65, 251, 209, 2, 69, 4, - 242, 219, 27, 76, 215, 22, 69, 8, 210, 219, 27, 68, 2, 74, 214, 22, 65, - 3, 69, 4, 162, 219, 27, 82, 215, 22, 69, 32, 96, 5, 67, 84, 82, 73, 67, - 160, 1, 7, 77, 69, 78, 84, 32, 79, 70, 238, 229, 26, 80, 235, 80, 86, 10, - 26, 32, 179, 130, 24, 65, 8, 98, 80, 144, 205, 17, 2, 84, 79, 128, 231, - 8, 9, 76, 73, 71, 72, 84, 32, 66, 85, 76, 171, 33, 65, 2, 11, 76, 2, 147, - 239, 27, 85, 19, 11, 32, 16, 72, 5, 87, 73, 84, 72, 32, 217, 207, 16, 7, + 206, 253, 27, 74, 215, 22, 69, 4, 214, 190, 25, 65, 203, 213, 2, 69, 4, + 166, 253, 27, 76, 215, 22, 69, 8, 134, 253, 27, 68, 2, 74, 214, 22, 65, + 3, 69, 4, 214, 252, 27, 82, 215, 22, 69, 32, 96, 5, 67, 84, 82, 73, 67, + 160, 1, 7, 77, 69, 78, 84, 32, 79, 70, 134, 133, 27, 80, 135, 83, 86, 10, + 26, 32, 155, 159, 24, 65, 8, 98, 80, 128, 221, 17, 2, 84, 79, 176, 246, + 8, 9, 76, 73, 71, 72, 84, 32, 66, 85, 76, 187, 33, 65, 2, 11, 76, 2, 199, + 144, 28, 85, 19, 11, 32, 16, 72, 5, 87, 73, 84, 72, 32, 177, 222, 16, 7, 79, 80, 69, 78, 73, 78, 71, 12, 130, 1, 76, 32, 12, 84, 87, 79, 32, 72, - 79, 82, 73, 90, 79, 78, 84, 142, 214, 19, 86, 194, 172, 4, 85, 234, 60, - 79, 219, 173, 2, 68, 2, 173, 201, 25, 3, 79, 78, 71, 2, 217, 213, 14, 7, - 65, 76, 32, 83, 84, 82, 79, 46, 254, 168, 4, 69, 129, 202, 15, 15, 73, + 79, 82, 73, 90, 79, 78, 84, 170, 233, 19, 86, 226, 182, 4, 85, 142, 61, + 79, 175, 177, 2, 68, 2, 141, 231, 25, 3, 79, 78, 71, 2, 197, 227, 14, 7, + 65, 76, 32, 83, 84, 82, 79, 46, 238, 175, 4, 69, 189, 214, 15, 15, 73, 71, 65, 84, 85, 82, 69, 32, 90, 65, 89, 73, 78, 45, 89, 53, 48, 4, 79, 74, 73, 32, 218, 2, 80, 163, 3, 32, 18, 164, 1, 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 32, 109, 26, 77, 79, 68, 73, 70, 73, 69, 82, 32, 70, 73, - 84, 90, 80, 65, 84, 82, 73, 67, 75, 32, 84, 89, 80, 69, 45, 8, 128, 255, - 14, 2, 82, 69, 12, 5, 67, 85, 82, 76, 89, 0, 5, 87, 72, 73, 84, 69, 213, - 205, 2, 2, 66, 65, 10, 208, 208, 20, 2, 49, 45, 174, 153, 7, 51, 2, 52, - 2, 53, 3, 54, 26, 44, 3, 84, 89, 32, 177, 244, 3, 2, 72, 65, 24, 82, 78, - 60, 3, 80, 65, 71, 20, 3, 83, 69, 84, 181, 164, 27, 4, 68, 79, 67, 85, 8, - 36, 3, 79, 84, 69, 151, 254, 23, 69, 7, 199, 152, 13, 32, 4, 139, 225, - 25, 69, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 130, 130, 14, 82, 24, 3, - 76, 69, 70, 212, 166, 1, 2, 83, 77, 179, 144, 9, 79, 38, 86, 32, 64, 2, - 68, 32, 214, 1, 81, 20, 6, 86, 69, 76, 79, 80, 69, 227, 220, 15, 84, 6, - 42, 81, 198, 253, 25, 68, 131, 170, 1, 83, 2, 195, 159, 27, 85, 20, 32, - 3, 79, 70, 32, 131, 1, 87, 18, 88, 3, 80, 82, 79, 242, 221, 20, 71, 56, - 2, 83, 69, 162, 71, 77, 30, 84, 195, 165, 5, 76, 4, 178, 222, 20, 84, - 135, 133, 6, 79, 2, 173, 239, 19, 7, 73, 84, 72, 32, 76, 69, 70, 5, 231, - 165, 21, 85, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 42, 76, 209, 233, 23, - 4, 68, 79, 87, 78, 2, 209, 203, 22, 4, 73, 71, 72, 84, 6, 254, 226, 27, - 76, 2, 77, 3, 84, 44, 28, 2, 65, 76, 231, 6, 73, 38, 30, 32, 133, 2, 2, - 83, 32, 12, 56, 3, 84, 79, 32, 177, 141, 13, 5, 65, 78, 68, 32, 80, 10, - 68, 3, 79, 82, 32, 221, 143, 27, 8, 66, 89, 32, 68, 69, 70, 73, 78, 8, - 64, 3, 80, 82, 69, 28, 3, 83, 85, 67, 162, 206, 25, 71, 39, 76, 2, 193, - 156, 15, 2, 67, 69, 2, 233, 128, 26, 3, 67, 69, 69, 26, 72, 4, 83, 73, - 71, 78, 224, 203, 25, 4, 87, 73, 84, 72, 235, 195, 1, 67, 23, 11, 32, 20, + 84, 90, 80, 65, 84, 82, 73, 67, 75, 32, 84, 89, 80, 69, 45, 8, 140, 141, + 15, 2, 82, 69, 12, 5, 67, 85, 82, 76, 89, 0, 5, 87, 72, 73, 84, 69, 185, + 207, 2, 2, 66, 65, 10, 240, 227, 20, 2, 49, 45, 194, 167, 7, 51, 2, 52, + 2, 53, 3, 54, 26, 44, 3, 84, 89, 32, 149, 251, 3, 2, 72, 65, 24, 82, 78, + 60, 3, 80, 65, 71, 20, 3, 83, 69, 84, 233, 197, 27, 4, 68, 79, 67, 85, 8, + 36, 3, 79, 84, 69, 211, 155, 24, 69, 7, 203, 166, 13, 32, 4, 215, 255, + 25, 69, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 138, 144, 14, 82, 24, 3, + 76, 69, 70, 224, 166, 1, 2, 83, 77, 255, 159, 9, 79, 38, 86, 32, 64, 2, + 68, 32, 214, 1, 81, 20, 6, 86, 69, 76, 79, 80, 69, 223, 235, 15, 84, 6, + 42, 81, 146, 156, 26, 68, 235, 172, 1, 83, 2, 247, 192, 27, 85, 20, 32, + 3, 79, 70, 32, 131, 1, 87, 18, 88, 3, 80, 82, 79, 150, 241, 20, 71, 56, + 2, 83, 69, 166, 71, 77, 30, 84, 203, 177, 5, 76, 4, 214, 241, 20, 84, + 151, 147, 6, 79, 2, 217, 130, 20, 7, 73, 84, 72, 32, 76, 69, 70, 5, 143, + 185, 21, 85, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 42, 76, 177, 134, 24, + 4, 68, 79, 87, 78, 2, 185, 228, 22, 4, 73, 71, 72, 84, 6, 178, 132, 28, + 76, 2, 77, 3, 84, 46, 28, 2, 65, 76, 231, 6, 73, 40, 30, 32, 133, 2, 2, + 83, 32, 12, 56, 3, 84, 79, 32, 181, 155, 13, 5, 65, 78, 68, 32, 80, 10, + 68, 3, 79, 82, 32, 145, 177, 27, 8, 66, 89, 32, 68, 69, 70, 73, 78, 8, + 64, 3, 80, 82, 69, 28, 3, 83, 85, 67, 250, 235, 25, 71, 39, 76, 2, 213, + 170, 15, 2, 67, 69, 2, 181, 159, 26, 3, 67, 69, 69, 28, 72, 4, 83, 73, + 71, 78, 184, 233, 25, 4, 87, 73, 84, 72, 199, 199, 1, 67, 25, 11, 32, 22, 42, 65, 201, 1, 5, 87, 73, 84, 72, 32, 12, 112, 5, 66, 79, 86, 69, 32, 65, 19, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 32, 80, 65, 82, 65, 76, - 76, 69, 76, 8, 154, 132, 21, 80, 210, 5, 84, 182, 184, 2, 76, 139, 252, - 2, 82, 5, 243, 239, 6, 32, 8, 160, 1, 4, 66, 85, 77, 80, 20, 7, 73, 78, + 76, 69, 76, 8, 206, 151, 21, 80, 210, 5, 84, 146, 189, 2, 76, 171, 131, + 3, 82, 5, 187, 247, 6, 32, 10, 160, 1, 4, 66, 85, 77, 80, 20, 7, 73, 78, 70, 73, 78, 73, 84, 20, 18, 84, 87, 79, 32, 68, 79, 84, 83, 32, 65, 66, - 79, 86, 69, 32, 65, 78, 68, 215, 156, 15, 68, 2, 163, 223, 26, 89, 2, - 207, 174, 25, 89, 2, 17, 2, 32, 84, 2, 243, 173, 25, 87, 6, 80, 7, 86, - 65, 76, 69, 78, 84, 32, 237, 135, 21, 7, 65, 78, 71, 85, 76, 65, 82, 4, - 48, 6, 87, 73, 84, 72, 32, 70, 223, 187, 27, 84, 2, 11, 79, 2, 201, 242, + 79, 86, 69, 32, 65, 78, 68, 235, 170, 15, 68, 2, 215, 128, 27, 89, 4, + 179, 219, 25, 89, 2, 17, 2, 32, 84, 2, 211, 203, 25, 87, 6, 80, 7, 86, + 65, 76, 69, 78, 84, 32, 161, 155, 21, 7, 65, 78, 71, 85, 76, 65, 82, 4, + 48, 6, 87, 73, 84, 72, 32, 70, 147, 221, 27, 84, 2, 11, 79, 2, 237, 248, 2, 2, 85, 82, 20, 152, 1, 11, 82, 79, 82, 45, 66, 65, 82, 82, 69, 68, 32, - 152, 163, 5, 7, 73, 83, 32, 70, 79, 82, 77, 193, 195, 7, 9, 65, 83, 69, - 32, 84, 79, 32, 84, 72, 12, 160, 220, 5, 4, 87, 72, 73, 84, 13, 5, 66, - 76, 65, 67, 75, 10, 58, 67, 20, 6, 84, 73, 77, 65, 84, 69, 255, 215, 27, - 65, 5, 211, 144, 26, 65, 4, 210, 196, 26, 68, 171, 147, 1, 83, 154, 8, - 60, 7, 72, 73, 79, 80, 73, 67, 32, 150, 215, 27, 66, 3, 88, 150, 8, 204, + 184, 170, 5, 7, 73, 83, 32, 70, 79, 82, 77, 161, 202, 7, 9, 65, 83, 69, + 32, 84, 79, 32, 84, 72, 12, 216, 227, 5, 4, 87, 72, 73, 84, 13, 5, 66, + 76, 65, 67, 75, 10, 58, 67, 20, 6, 84, 73, 77, 65, 84, 69, 179, 249, 27, + 65, 5, 235, 175, 26, 65, 4, 234, 227, 26, 68, 199, 149, 1, 83, 154, 8, + 60, 7, 72, 73, 79, 80, 73, 67, 32, 202, 248, 27, 66, 3, 88, 150, 8, 204, 1, 2, 67, 79, 232, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, 80, 54, 83, - 156, 24, 11, 84, 79, 78, 65, 76, 32, 77, 65, 82, 75, 32, 226, 252, 18, - 68, 214, 235, 5, 70, 82, 81, 221, 146, 2, 4, 87, 79, 82, 68, 10, 26, 77, - 187, 133, 27, 76, 8, 52, 7, 66, 73, 78, 73, 78, 71, 32, 183, 210, 27, 77, + 156, 24, 11, 84, 79, 78, 65, 76, 32, 77, 65, 82, 75, 32, 254, 143, 19, + 68, 158, 246, 5, 70, 82, 81, 173, 150, 2, 4, 87, 79, 82, 68, 10, 26, 77, + 239, 166, 27, 76, 8, 52, 7, 66, 73, 78, 73, 78, 71, 32, 235, 243, 27, 77, 6, 60, 11, 71, 69, 77, 73, 78, 65, 84, 73, 79, 78, 32, 51, 86, 4, 44, 5, - 65, 78, 68, 32, 86, 211, 146, 27, 77, 2, 201, 246, 23, 4, 79, 87, 69, 76, - 22, 66, 84, 226, 255, 21, 72, 222, 228, 3, 69, 30, 70, 42, 78, 39, 83, 8, - 194, 199, 16, 69, 170, 158, 9, 72, 27, 87, 4, 166, 113, 65, 153, 145, 26, + 65, 78, 68, 32, 86, 135, 180, 27, 77, 2, 169, 148, 24, 4, 79, 87, 69, 76, + 22, 66, 84, 158, 152, 22, 72, 238, 234, 3, 69, 30, 70, 42, 78, 39, 83, 8, + 154, 214, 16, 69, 158, 174, 9, 72, 27, 87, 4, 202, 119, 65, 169, 172, 26, 5, 82, 69, 70, 65, 67, 198, 7, 50, 69, 37, 8, 89, 76, 76, 65, 66, 76, 69, - 32, 4, 202, 134, 24, 77, 139, 226, 2, 67, 194, 7, 210, 1, 66, 90, 67, + 32, 4, 170, 164, 24, 77, 223, 229, 2, 67, 194, 7, 210, 1, 66, 90, 67, 246, 1, 68, 186, 1, 70, 90, 71, 214, 2, 72, 162, 1, 75, 102, 77, 90, 78, 90, 80, 138, 2, 81, 174, 1, 82, 86, 83, 210, 1, 84, 122, 74, 2, 76, 138, 1, 87, 2, 89, 66, 88, 134, 1, 90, 95, 86, 38, 194, 12, 87, 230, 8, 66, - 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 78, 94, 67, - 254, 15, 72, 202, 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, - 2, 73, 3, 85, 42, 70, 72, 254, 225, 21, 65, 230, 171, 5, 69, 162, 64, 73, - 2, 79, 3, 85, 28, 166, 19, 72, 214, 206, 21, 65, 230, 171, 5, 69, 162, - 64, 73, 2, 79, 3, 85, 60, 94, 68, 214, 14, 90, 254, 209, 21, 65, 2, 79, - 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 30, 210, 14, 72, 254, - 209, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 24, - 190, 8, 87, 186, 215, 21, 65, 230, 171, 5, 69, 234, 61, 89, 186, 2, 73, + 158, 202, 23, 65, 2, 79, 210, 209, 3, 69, 162, 64, 73, 3, 85, 78, 94, 67, + 254, 15, 72, 146, 206, 23, 65, 2, 79, 210, 209, 3, 69, 234, 61, 87, 186, + 2, 73, 3, 85, 42, 70, 72, 198, 221, 23, 65, 210, 209, 3, 69, 162, 64, 73, + 2, 79, 3, 85, 28, 166, 19, 72, 158, 202, 23, 65, 210, 209, 3, 69, 162, + 64, 73, 2, 79, 3, 85, 60, 94, 68, 214, 14, 90, 198, 205, 23, 65, 2, 79, + 210, 209, 3, 69, 234, 61, 87, 186, 2, 73, 3, 85, 30, 210, 14, 72, 198, + 205, 23, 65, 2, 79, 210, 209, 3, 69, 234, 61, 87, 186, 2, 73, 3, 85, 24, + 190, 8, 87, 130, 211, 23, 65, 210, 209, 3, 69, 234, 61, 89, 186, 2, 73, 2, 79, 3, 85, 118, 142, 1, 85, 226, 7, 71, 232, 3, 7, 76, 79, 84, 84, 65, - 76, 32, 158, 2, 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, + 76, 32, 158, 2, 87, 218, 1, 89, 158, 202, 23, 65, 2, 79, 210, 209, 3, 69, 163, 64, 73, 39, 29, 5, 82, 65, 71, 69, 32, 36, 74, 66, 2, 70, 2, 77, 2, - 80, 46, 71, 2, 75, 2, 81, 191, 130, 12, 72, 4, 11, 87, 4, 198, 178, 27, - 69, 215, 22, 73, 6, 11, 87, 6, 206, 136, 27, 69, 163, 64, 73, 52, 70, 72, - 134, 220, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 36, 202, 4, - 87, 230, 8, 89, 214, 206, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, 79, 3, - 85, 64, 250, 4, 88, 134, 6, 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, - 171, 5, 69, 162, 64, 73, 3, 85, 26, 142, 3, 87, 186, 215, 21, 65, 2, 79, - 230, 171, 5, 69, 234, 61, 89, 186, 2, 73, 3, 85, 36, 166, 7, 89, 202, - 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 56, - 82, 72, 142, 1, 87, 186, 215, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, - 73, 3, 85, 32, 74, 65, 250, 215, 21, 79, 230, 171, 5, 69, 234, 61, 87, - 186, 2, 73, 3, 85, 19, 160, 9, 8, 82, 89, 78, 71, 69, 65, 76, 32, 219, - 186, 27, 65, 8, 154, 131, 27, 69, 162, 64, 65, 3, 73, 64, 94, 72, 134, 6, - 87, 218, 1, 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, - 85, 24, 130, 6, 87, 174, 208, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, - 79, 3, 85, 20, 226, 213, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 2, - 89, 186, 2, 73, 3, 85, 74, 102, 69, 226, 1, 72, 170, 3, 90, 78, 83, 214, - 206, 21, 65, 2, 79, 206, 233, 5, 87, 186, 2, 73, 3, 85, 13, 56, 8, 66, - 65, 84, 66, 69, 73, 84, 32, 243, 191, 27, 69, 8, 230, 206, 22, 66, 2, 70, - 2, 77, 3, 80, 80, 118, 72, 76, 2, 84, 72, 62, 90, 162, 2, 83, 162, 207, - 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 18, 198, - 210, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 12, - 222, 253, 26, 69, 234, 61, 65, 186, 2, 73, 2, 79, 3, 85, 16, 190, 209, - 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 40, 82, 87, 218, 1, - 89, 214, 206, 21, 65, 2, 79, 230, 171, 5, 69, 162, 64, 73, 3, 85, 10, - 170, 208, 21, 65, 230, 171, 5, 69, 163, 64, 73, 48, 90, 72, 78, 90, 214, - 206, 21, 65, 2, 79, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 3, 85, 16, - 158, 207, 21, 65, 230, 171, 5, 69, 234, 61, 87, 186, 2, 73, 2, 79, 3, 85, - 14, 210, 206, 21, 65, 230, 171, 5, 69, 162, 64, 73, 2, 79, 3, 85, 20, + 80, 46, 71, 2, 75, 2, 81, 195, 144, 12, 72, 4, 11, 87, 4, 250, 211, 27, + 69, 215, 22, 73, 6, 11, 87, 6, 130, 170, 27, 69, 163, 64, 73, 52, 70, 72, + 206, 215, 23, 65, 2, 79, 210, 209, 3, 69, 162, 64, 73, 3, 85, 36, 202, 4, + 87, 230, 8, 89, 158, 202, 23, 65, 210, 209, 3, 69, 162, 64, 73, 2, 79, 3, + 85, 64, 250, 4, 88, 134, 6, 87, 218, 1, 89, 158, 202, 23, 65, 2, 79, 210, + 209, 3, 69, 162, 64, 73, 3, 85, 26, 142, 3, 87, 130, 211, 23, 65, 2, 79, + 210, 209, 3, 69, 234, 61, 89, 186, 2, 73, 3, 85, 36, 166, 7, 89, 146, + 206, 23, 65, 2, 79, 210, 209, 3, 69, 234, 61, 87, 186, 2, 73, 3, 85, 56, + 82, 72, 142, 1, 87, 130, 211, 23, 65, 2, 79, 210, 209, 3, 69, 162, 64, + 73, 3, 85, 32, 74, 65, 194, 211, 23, 79, 210, 209, 3, 69, 234, 61, 87, + 186, 2, 73, 3, 85, 19, 160, 9, 8, 82, 89, 78, 71, 69, 65, 76, 32, 143, + 220, 27, 65, 8, 206, 164, 27, 69, 162, 64, 65, 3, 73, 64, 94, 72, 134, 6, + 87, 218, 1, 89, 158, 202, 23, 65, 2, 79, 210, 209, 3, 69, 162, 64, 73, 3, + 85, 24, 130, 6, 87, 246, 203, 23, 65, 210, 209, 3, 69, 162, 64, 73, 2, + 79, 3, 85, 20, 170, 209, 23, 65, 2, 79, 210, 209, 3, 69, 234, 61, 87, 2, + 89, 186, 2, 73, 3, 85, 74, 102, 69, 226, 1, 72, 170, 3, 90, 78, 83, 158, + 202, 23, 65, 2, 79, 186, 143, 4, 87, 186, 2, 73, 3, 85, 13, 56, 8, 66, + 65, 84, 66, 69, 73, 84, 32, 167, 225, 27, 69, 8, 198, 231, 22, 66, 2, 70, + 2, 77, 3, 80, 80, 118, 72, 76, 2, 84, 72, 62, 90, 162, 2, 83, 234, 202, + 23, 65, 2, 79, 210, 209, 3, 69, 234, 61, 87, 186, 2, 73, 3, 85, 18, 142, + 206, 23, 65, 2, 79, 210, 209, 3, 69, 234, 61, 87, 186, 2, 73, 3, 85, 12, + 146, 159, 27, 69, 234, 61, 65, 186, 2, 73, 2, 79, 3, 85, 16, 134, 205, + 23, 65, 2, 79, 210, 209, 3, 69, 162, 64, 73, 3, 85, 40, 82, 87, 218, 1, + 89, 158, 202, 23, 65, 2, 79, 210, 209, 3, 69, 162, 64, 73, 3, 85, 10, + 242, 203, 23, 65, 210, 209, 3, 69, 163, 64, 73, 48, 90, 72, 78, 90, 158, + 202, 23, 65, 2, 79, 210, 209, 3, 69, 234, 61, 87, 186, 2, 73, 3, 85, 16, + 230, 202, 23, 65, 210, 209, 3, 69, 234, 61, 87, 186, 2, 73, 2, 79, 3, 85, + 14, 154, 202, 23, 65, 210, 209, 3, 69, 162, 64, 73, 2, 79, 3, 85, 20, 130, 1, 68, 74, 72, 30, 75, 42, 82, 0, 7, 83, 72, 79, 82, 84, 32, 82, - 204, 140, 4, 3, 67, 72, 73, 165, 246, 22, 3, 89, 73, 90, 6, 48, 4, 69, - 82, 69, 84, 249, 164, 26, 2, 73, 70, 5, 17, 2, 45, 72, 2, 177, 131, 27, - 2, 73, 68, 4, 196, 164, 26, 2, 69, 78, 219, 12, 85, 2, 245, 154, 4, 3, - 73, 75, 82, 10, 68, 2, 82, 79, 197, 173, 26, 9, 76, 69, 82, 32, 67, 79, - 78, 83, 84, 8, 92, 5, 80, 69, 65, 78, 32, 204, 131, 24, 8, 45, 67, 85, - 82, 82, 69, 78, 67, 131, 172, 2, 32, 4, 214, 135, 4, 67, 19, 80, 50, 30, - 67, 102, 80, 187, 1, 84, 6, 60, 9, 76, 65, 77, 65, 84, 73, 79, 78, 32, - 247, 158, 26, 69, 4, 174, 228, 24, 81, 235, 143, 2, 77, 10, 96, 7, 76, - 79, 83, 73, 79, 78, 32, 157, 247, 26, 11, 82, 69, 83, 83, 73, 79, 78, 76, - 69, 83, 83, 8, 232, 209, 11, 4, 70, 82, 65, 77, 229, 146, 15, 8, 65, 84, - 32, 72, 79, 82, 73, 90, 34, 98, 82, 153, 179, 21, 18, 69, 78, 68, 69, 68, - 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, 14, 140, 1, 12, 69, - 77, 69, 76, 89, 32, 72, 69, 65, 86, 89, 32, 141, 222, 25, 16, 65, 84, 69, - 82, 82, 69, 83, 84, 82, 73, 65, 76, 32, 65, 76, 73, 12, 50, 83, 202, 155, - 25, 70, 234, 2, 87, 215, 10, 71, 4, 218, 156, 25, 65, 43, 73, 7, 250, - 167, 19, 71, 171, 137, 8, 83, 142, 4, 142, 1, 65, 130, 19, 69, 190, 1, - 73, 230, 7, 76, 242, 5, 79, 238, 6, 82, 234, 4, 85, 228, 183, 20, 2, 86, - 83, 202, 192, 4, 83, 223, 137, 2, 70, 92, 122, 67, 178, 15, 76, 176, 2, - 2, 88, 32, 238, 141, 3, 77, 172, 247, 5, 2, 82, 83, 188, 250, 1, 2, 84, - 72, 187, 230, 9, 73, 70, 72, 2, 69, 32, 248, 172, 16, 4, 83, 73, 77, 73, - 169, 202, 4, 2, 84, 79, 66, 226, 1, 83, 160, 1, 4, 87, 73, 84, 72, 224, - 250, 12, 2, 80, 65, 228, 136, 2, 2, 77, 65, 140, 145, 11, 13, 84, 72, 82, - 79, 87, 73, 78, 71, 32, 65, 32, 75, 73, 201, 1, 14, 72, 79, 76, 68, 73, - 78, 71, 32, 66, 65, 67, 75, 32, 84, 4, 232, 251, 16, 18, 65, 86, 79, 85, - 82, 73, 78, 71, 32, 68, 69, 76, 73, 67, 73, 79, 85, 83, 241, 128, 5, 13, - 67, 82, 69, 65, 77, 73, 78, 71, 32, 73, 78, 32, 70, 54, 38, 32, 201, 137, - 24, 3, 79, 85, 84, 52, 196, 4, 2, 67, 79, 44, 5, 72, 69, 65, 68, 45, 38, - 77, 98, 79, 92, 7, 78, 79, 32, 71, 79, 79, 68, 238, 1, 80, 164, 1, 16, - 83, 84, 85, 67, 75, 45, 79, 85, 84, 32, 84, 79, 78, 71, 85, 69, 110, 84, - 192, 177, 1, 9, 66, 65, 71, 83, 32, 85, 78, 68, 69, 224, 158, 17, 22, 70, - 73, 78, 71, 69, 82, 32, 67, 79, 86, 69, 82, 73, 78, 71, 32, 67, 76, 79, - 83, 69, 68, 244, 67, 3, 82, 79, 76, 252, 231, 1, 13, 76, 79, 79, 75, 32, - 79, 70, 32, 84, 82, 73, 85, 77, 232, 130, 3, 8, 68, 73, 65, 71, 79, 78, - 65, 76, 1, 20, 85, 78, 69, 86, 69, 78, 32, 69, 89, 69, 83, 32, 65, 78, - 68, 32, 87, 65, 86, 89, 4, 252, 4, 3, 87, 66, 79, 227, 148, 19, 76, 2, - 173, 159, 22, 4, 66, 65, 78, 68, 4, 60, 6, 69, 68, 73, 67, 65, 76, 189, - 211, 25, 3, 79, 78, 79, 2, 177, 224, 25, 3, 32, 77, 65, 12, 90, 75, 36, - 4, 80, 69, 78, 32, 205, 161, 20, 10, 78, 69, 32, 69, 89, 69, 66, 82, 79, - 87, 2, 165, 142, 22, 4, 32, 71, 69, 83, 8, 116, 5, 77, 79, 85, 84, 72, - 217, 129, 24, 18, 69, 89, 69, 83, 32, 65, 78, 68, 32, 72, 65, 78, 68, 32, - 79, 86, 69, 82, 7, 11, 32, 4, 136, 252, 9, 3, 86, 79, 77, 141, 154, 9, 5, - 65, 78, 68, 32, 67, 6, 132, 1, 18, 65, 82, 84, 89, 32, 72, 79, 82, 78, - 32, 65, 78, 68, 32, 80, 65, 82, 84, 100, 2, 69, 69, 197, 148, 19, 4, 76, - 69, 65, 68, 2, 161, 184, 22, 2, 89, 32, 7, 29, 5, 32, 65, 78, 68, 32, 4, - 36, 3, 87, 73, 78, 231, 148, 19, 84, 2, 157, 177, 1, 4, 75, 73, 78, 71, - 4, 50, 69, 225, 187, 22, 6, 72, 69, 82, 77, 79, 77, 2, 157, 142, 27, 8, - 65, 82, 83, 32, 79, 70, 32, 74, 10, 34, 76, 185, 146, 22, 2, 65, 70, 8, - 84, 13, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 189, 212, 23, - 2, 69, 78, 6, 128, 1, 9, 67, 82, 79, 83, 83, 73, 78, 71, 32, 185, 167, - 19, 16, 73, 78, 32, 87, 72, 73, 84, 69, 32, 67, 73, 82, 67, 76, 69, 32, - 4, 232, 133, 16, 3, 82, 73, 83, 143, 169, 3, 78, 4, 222, 244, 14, 73, - 255, 137, 4, 77, 14, 50, 65, 50, 77, 44, 2, 82, 82, 147, 217, 18, 78, 4, - 212, 161, 8, 3, 82, 70, 85, 147, 231, 9, 84, 4, 204, 233, 8, 2, 73, 78, - 143, 177, 7, 65, 4, 184, 238, 7, 2, 73, 83, 163, 173, 19, 89, 56, 156, 1, - 5, 71, 85, 82, 69, 32, 38, 76, 178, 1, 78, 98, 82, 182, 2, 83, 236, 13, - 4, 86, 69, 32, 68, 213, 249, 4, 10, 69, 76, 68, 32, 72, 79, 67, 75, 69, - 89, 4, 226, 177, 25, 68, 131, 170, 1, 83, 10, 32, 2, 69, 32, 65, 2, 77, - 32, 6, 166, 187, 13, 70, 136, 154, 2, 3, 67, 65, 66, 191, 150, 8, 83, 4, - 52, 4, 80, 82, 79, 74, 197, 142, 20, 3, 70, 82, 65, 2, 171, 203, 15, 69, - 4, 72, 4, 71, 69, 82, 80, 205, 144, 25, 8, 73, 84, 69, 32, 80, 65, 82, - 84, 2, 167, 255, 15, 82, 22, 34, 69, 189, 1, 3, 83, 84, 32, 13, 56, 2, - 32, 69, 68, 4, 87, 79, 82, 75, 171, 170, 15, 67, 4, 146, 179, 13, 78, - 153, 208, 4, 8, 88, 84, 73, 78, 71, 85, 73, 83, 4, 216, 172, 23, 6, 32, - 83, 80, 65, 82, 75, 223, 233, 3, 83, 10, 190, 170, 5, 81, 216, 176, 10, - 8, 83, 84, 82, 79, 78, 71, 32, 73, 239, 215, 6, 80, 10, 38, 72, 129, 200, - 23, 3, 84, 69, 68, 9, 168, 248, 3, 13, 73, 78, 71, 32, 80, 79, 76, 69, - 32, 65, 78, 68, 32, 246, 202, 5, 69, 237, 202, 16, 19, 32, 67, 65, 75, - 69, 32, 87, 73, 84, 72, 32, 83, 87, 73, 82, 76, 32, 68, 69, 42, 50, 65, - 170, 1, 69, 98, 79, 194, 1, 85, 39, 89, 12, 114, 84, 148, 157, 4, 5, 80, - 80, 73, 78, 71, 136, 158, 3, 6, 71, 32, 73, 78, 32, 72, 161, 202, 17, 3, - 77, 73, 78, 6, 198, 208, 8, 32, 250, 197, 11, 66, 139, 229, 5, 78, 4, - 160, 221, 23, 8, 88, 69, 68, 32, 66, 73, 67, 69, 153, 145, 1, 7, 85, 82, - 45, 68, 69, 45, 76, 10, 46, 82, 28, 3, 87, 69, 82, 167, 203, 25, 80, 2, - 161, 137, 26, 2, 65, 76, 7, 17, 2, 32, 80, 4, 58, 85, 169, 178, 24, 8, - 76, 65, 89, 73, 78, 71, 32, 67, 2, 189, 166, 26, 4, 78, 67, 84, 85, 4, - 162, 152, 3, 83, 219, 224, 23, 84, 13, 25, 4, 73, 78, 71, 32, 10, 64, 6, - 83, 65, 85, 67, 69, 82, 166, 130, 10, 68, 223, 130, 10, 69, 7, 29, 5, 32, - 87, 73, 84, 72, 4, 34, 32, 1, 4, 79, 85, 84, 32, 2, 21, 3, 66, 69, 65, 2, - 147, 208, 26, 77, 54, 102, 71, 20, 2, 76, 68, 68, 2, 79, 84, 22, 82, 238, - 1, 85, 148, 255, 22, 2, 78, 68, 235, 205, 3, 88, 5, 171, 251, 26, 71, 4, - 164, 192, 9, 8, 73, 78, 71, 32, 72, 65, 78, 68, 223, 147, 17, 69, 5, 139, - 148, 14, 80, 16, 102, 75, 158, 206, 20, 77, 186, 181, 4, 32, 214, 115, - 67, 201, 120, 9, 84, 85, 78, 69, 32, 67, 79, 79, 75, 8, 80, 10, 32, 65, - 78, 68, 32, 75, 78, 73, 70, 69, 162, 150, 15, 69, 183, 163, 11, 73, 5, - 249, 207, 15, 7, 32, 87, 73, 84, 72, 32, 80, 22, 26, 82, 255, 161, 23, - 78, 20, 38, 32, 218, 2, 84, 183, 221, 18, 45, 16, 110, 67, 174, 1, 68, - 134, 165, 2, 66, 248, 147, 10, 7, 76, 69, 65, 70, 32, 67, 76, 182, 111, - 84, 155, 200, 11, 80, 4, 128, 170, 13, 3, 76, 85, 66, 209, 86, 32, 79, - 82, 78, 69, 82, 32, 65, 82, 82, 79, 87, 83, 32, 67, 73, 82, 67, 76, 73, - 78, 71, 32, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 4, 21, 3, 79, 84, 32, - 4, 138, 197, 23, 80, 239, 128, 3, 77, 2, 183, 38, 72, 28, 78, 65, 226, 1, - 69, 98, 79, 209, 241, 16, 8, 73, 69, 68, 32, 83, 72, 82, 73, 10, 72, 6, - 67, 84, 73, 79, 78, 32, 69, 8, 77, 69, 32, 87, 73, 84, 72, 32, 4, 146, - 239, 19, 83, 177, 6, 9, 78, 85, 77, 69, 82, 65, 84, 79, 82, 6, 50, 80, - 194, 165, 2, 65, 217, 161, 21, 2, 84, 73, 2, 221, 238, 21, 2, 73, 67, 6, - 48, 6, 78, 67, 72, 32, 70, 82, 235, 251, 18, 69, 4, 146, 240, 25, 73, - 249, 12, 3, 65, 78, 67, 10, 52, 3, 78, 84, 45, 116, 2, 87, 78, 151, 197, - 26, 71, 4, 70, 84, 173, 138, 2, 11, 70, 65, 67, 73, 78, 71, 32, 66, 65, - 66, 89, 2, 197, 178, 12, 6, 73, 76, 84, 69, 68, 32, 5, 149, 135, 8, 6, - 73, 78, 71, 32, 70, 65, 226, 1, 80, 2, 76, 76, 134, 6, 78, 156, 233, 16, - 5, 69, 76, 32, 80, 85, 191, 251, 9, 83, 216, 1, 42, 32, 73, 6, 87, 73, - 68, 84, 72, 32, 10, 246, 255, 11, 77, 252, 169, 3, 2, 79, 85, 210, 232, - 8, 66, 151, 29, 83, 206, 1, 242, 1, 67, 42, 76, 78, 78, 30, 80, 66, 82, - 142, 1, 83, 38, 89, 158, 146, 10, 77, 198, 208, 10, 65, 158, 2, 68, 58, - 69, 98, 71, 118, 72, 202, 4, 81, 190, 143, 2, 87, 218, 28, 84, 250, 145, - 1, 70, 148, 176, 1, 6, 66, 82, 79, 75, 69, 78, 211, 7, 86, 10, 214, 230, - 20, 73, 62, 79, 159, 75, 69, 116, 42, 69, 138, 234, 20, 65, 243, 171, 4, - 79, 10, 162, 1, 70, 251, 235, 20, 83, 4, 246, 186, 21, 79, 35, 85, 6, 42, - 79, 250, 236, 20, 69, 203, 225, 2, 76, 2, 159, 148, 25, 85, 10, 36, 3, - 73, 71, 72, 131, 231, 24, 69, 8, 17, 2, 84, 32, 8, 228, 162, 20, 5, 87, - 72, 73, 84, 69, 202, 211, 2, 67, 210, 3, 80, 239, 7, 83, 4, 166, 176, 23, - 69, 147, 184, 1, 79, 2, 159, 130, 25, 69, 6, 136, 192, 11, 8, 67, 84, 73, - 79, 78, 32, 65, 80, 176, 247, 8, 5, 69, 82, 65, 76, 32, 143, 183, 1, 78, - 226, 20, 114, 65, 250, 6, 69, 198, 17, 73, 162, 1, 76, 134, 15, 79, 194, - 6, 82, 178, 92, 85, 246, 131, 22, 72, 139, 234, 3, 83, 142, 1, 42, 82, - 237, 221, 26, 4, 77, 69, 32, 68, 140, 1, 36, 3, 65, 89, 32, 235, 208, 25, - 76, 138, 1, 166, 1, 67, 210, 1, 83, 192, 2, 6, 86, 79, 87, 69, 76, 32, - 132, 166, 8, 6, 82, 69, 68, 85, 80, 76, 202, 202, 10, 72, 230, 168, 1, - 80, 170, 171, 3, 77, 191, 189, 1, 68, 52, 42, 79, 205, 1, 5, 65, 80, 73, - 84, 65, 8, 88, 10, 77, 66, 73, 78, 73, 78, 71, 32, 68, 79, 37, 8, 78, 83, - 79, 78, 65, 78, 84, 32, 4, 246, 135, 12, 85, 195, 241, 13, 84, 4, 190, - 136, 12, 78, 203, 145, 11, 71, 46, 36, 3, 77, 65, 76, 215, 250, 21, 85, - 44, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 44, 200, 1, 4, 79, 76, 68, - 32, 174, 137, 20, 78, 238, 231, 6, 66, 2, 67, 2, 68, 2, 70, 2, 71, 2, 72, - 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 88, 2, - 89, 187, 2, 65, 4, 150, 241, 26, 75, 3, 78, 12, 44, 5, 83, 73, 71, 78, - 32, 139, 177, 26, 76, 10, 226, 178, 26, 69, 162, 64, 65, 2, 73, 3, 79, - 242, 2, 112, 2, 65, 82, 110, 77, 50, 79, 204, 189, 23, 9, 82, 77, 65, 78, - 32, 80, 69, 78, 78, 218, 173, 2, 84, 239, 105, 78, 7, 29, 5, 32, 87, 73, - 84, 72, 4, 224, 147, 14, 5, 79, 85, 84, 32, 72, 149, 160, 9, 5, 32, 72, - 65, 78, 68, 4, 26, 32, 211, 244, 21, 73, 2, 251, 255, 22, 83, 226, 2, 64, - 6, 77, 69, 84, 82, 73, 67, 73, 6, 82, 71, 73, 65, 78, 32, 6, 236, 250, - 19, 4, 65, 76, 76, 89, 213, 222, 1, 5, 32, 80, 82, 79, 80, 220, 2, 228, - 1, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 172, 3, 7, 76, 69, - 84, 84, 69, 82, 32, 180, 2, 24, 77, 84, 65, 86, 82, 85, 76, 73, 32, 67, - 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 165, 6, 2, 80, - 65, 80, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 80, 142, 2, 65, 34, - 72, 166, 5, 67, 118, 71, 130, 1, 74, 34, 75, 82, 80, 34, 83, 94, 90, 220, - 180, 5, 2, 84, 65, 250, 142, 8, 76, 246, 179, 2, 82, 154, 160, 9, 66, 2, - 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, 68, 14, 69, 2, 73, - 2, 79, 2, 85, 2, 89, 143, 57, 87, 4, 162, 155, 26, 69, 227, 79, 78, 10, - 46, 65, 226, 211, 26, 73, 2, 79, 215, 22, 69, 4, 178, 234, 26, 69, 3, 82, - 94, 254, 1, 85, 178, 2, 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, - 75, 34, 76, 50, 80, 34, 83, 34, 84, 62, 90, 202, 247, 15, 82, 154, 160, - 9, 66, 2, 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, 68, 14, - 73, 2, 79, 2, 89, 142, 57, 87, 255, 2, 70, 4, 204, 248, 21, 4, 45, 66, - 82, 74, 203, 239, 4, 78, 92, 250, 1, 65, 42, 67, 74, 69, 46, 71, 34, 72, - 98, 74, 34, 75, 34, 76, 50, 80, 34, 83, 34, 84, 62, 90, 202, 247, 15, 82, - 154, 160, 9, 66, 2, 77, 2, 88, 202, 40, 78, 2, 81, 230, 33, 86, 234, 47, - 68, 14, 73, 2, 79, 2, 85, 2, 89, 142, 57, 87, 255, 2, 70, 6, 134, 150, - 26, 69, 2, 73, 227, 79, 78, 8, 38, 72, 230, 219, 25, 73, 215, 57, 65, 4, - 182, 149, 26, 73, 135, 23, 65, 4, 224, 136, 9, 2, 76, 73, 151, 220, 17, - 78, 4, 202, 154, 25, 72, 163, 122, 65, 12, 46, 65, 170, 205, 26, 73, 2, - 79, 215, 22, 69, 6, 26, 82, 227, 227, 26, 69, 5, 223, 220, 25, 68, 4, - 202, 153, 25, 72, 207, 64, 73, 4, 242, 193, 25, 72, 219, 81, 65, 4, 11, - 65, 4, 146, 210, 15, 66, 239, 144, 11, 83, 4, 162, 193, 25, 72, 223, 104, - 65, 4, 230, 226, 25, 72, 247, 47, 65, 6, 196, 183, 3, 6, 85, 82, 78, 69, - 68, 32, 211, 253, 1, 65, 4, 166, 192, 25, 72, 219, 81, 69, 2, 177, 166, - 20, 7, 82, 65, 71, 82, 65, 80, 72, 10, 48, 2, 77, 69, 20, 4, 78, 71, 69, - 82, 31, 82, 2, 179, 205, 25, 76, 2, 141, 184, 18, 2, 32, 82, 6, 38, 76, - 173, 235, 13, 3, 65, 70, 70, 5, 219, 204, 25, 83, 202, 1, 66, 65, 214, - 13, 79, 169, 144, 26, 7, 69, 73, 67, 72, 32, 83, 84, 194, 1, 84, 8, 71, - 79, 76, 73, 84, 73, 67, 32, 229, 12, 8, 83, 83, 32, 79, 70, 32, 77, 73, - 192, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 96, 45, 9, - 76, 32, 76, 69, 84, 84, 69, 82, 32, 96, 206, 1, 65, 22, 66, 42, 67, 94, - 68, 94, 70, 38, 71, 46, 73, 138, 2, 76, 58, 77, 66, 78, 34, 79, 30, 80, - 58, 82, 30, 83, 186, 1, 84, 110, 86, 22, 89, 90, 90, 226, 132, 19, 72, - 174, 128, 2, 85, 147, 174, 5, 75, 2, 163, 183, 26, 90, 4, 214, 3, 73, - 217, 198, 26, 2, 85, 75, 4, 48, 8, 65, 85, 68, 65, 84, 69, 32, 67, 15, - 72, 2, 11, 72, 2, 217, 250, 19, 2, 82, 73, 6, 42, 74, 30, 79, 221, 183, - 26, 2, 90, 69, 2, 145, 250, 19, 2, 69, 82, 2, 147, 235, 24, 66, 4, 186, - 138, 21, 82, 147, 171, 5, 73, 2, 21, 3, 76, 65, 71, 2, 207, 222, 21, 79, - 13, 38, 78, 54, 79, 141, 1, 2, 90, 72, 2, 145, 172, 26, 8, 73, 84, 73, - 65, 76, 32, 73, 90, 4, 33, 6, 84, 65, 84, 69, 68, 32, 4, 26, 66, 25, 2, - 83, 77, 2, 11, 73, 2, 35, 71, 2, 21, 3, 65, 76, 76, 2, 253, 209, 24, 2, - 32, 89, 4, 142, 213, 26, 73, 211, 2, 69, 4, 52, 9, 65, 84, 73, 78, 65, - 84, 69, 32, 77, 35, 74, 2, 173, 133, 11, 3, 89, 83, 76, 2, 249, 140, 9, - 3, 85, 68, 73, 2, 11, 65, 2, 187, 135, 21, 83, 4, 190, 177, 26, 78, 3, - 84, 4, 26, 79, 243, 213, 26, 69, 2, 133, 191, 22, 2, 75, 79, 2, 177, 235, - 21, 2, 73, 84, 14, 106, 72, 58, 76, 172, 240, 13, 6, 80, 73, 68, 69, 82, - 89, 201, 175, 10, 8, 77, 65, 76, 76, 32, 89, 85, 83, 6, 32, 2, 84, 65, - 171, 212, 26, 65, 5, 159, 172, 25, 80, 2, 239, 246, 19, 79, 6, 78, 86, - 144, 147, 22, 9, 82, 79, 75, 85, 84, 65, 83, 84, 73, 203, 172, 4, 83, 2, - 129, 162, 19, 2, 82, 73, 2, 135, 195, 24, 69, 12, 50, 69, 214, 244, 18, - 65, 250, 221, 7, 79, 3, 85, 6, 250, 130, 21, 83, 155, 143, 5, 82, 4, 156, - 145, 9, 3, 69, 77, 76, 145, 250, 15, 4, 72, 73, 86, 69, 2, 207, 206, 26, - 76, 6, 228, 238, 11, 13, 66, 69, 32, 87, 73, 84, 72, 32, 77, 69, 82, 73, - 68, 250, 198, 4, 87, 255, 135, 9, 86, 68, 162, 1, 65, 44, 12, 84, 72, 73, - 67, 32, 76, 69, 84, 84, 69, 82, 32, 230, 161, 18, 82, 206, 102, 71, 172, - 131, 5, 3, 78, 71, 71, 182, 186, 1, 79, 161, 75, 2, 76, 70, 4, 144, 183, - 15, 2, 76, 32, 183, 152, 11, 84, 54, 206, 2, 65, 50, 72, 46, 73, 46, 78, - 46, 80, 2, 81, 40, 2, 82, 65, 22, 84, 200, 137, 9, 2, 87, 73, 230, 141, - 12, 85, 188, 69, 3, 70, 65, 73, 204, 7, 4, 66, 65, 73, 82, 128, 22, 2, - 79, 84, 150, 30, 68, 158, 123, 77, 164, 131, 1, 3, 83, 65, 85, 162, 16, - 69, 184, 29, 3, 76, 65, 71, 146, 200, 1, 74, 196, 16, 2, 71, 73, 141, 12, - 2, 75, 85, 4, 204, 198, 24, 3, 73, 72, 86, 187, 131, 2, 72, 4, 214, 224, - 13, 87, 145, 226, 11, 2, 65, 71, 4, 230, 138, 9, 85, 217, 229, 14, 2, 71, - 71, 6, 154, 206, 15, 73, 253, 252, 8, 2, 65, 85, 2, 213, 134, 26, 5, 65, - 73, 82, 84, 72, 2, 235, 165, 26, 73, 4, 192, 169, 23, 2, 72, 73, 233, 69, - 2, 69, 73, 234, 9, 46, 65, 198, 5, 69, 206, 82, 73, 151, 3, 79, 152, 1, - 96, 7, 68, 85, 65, 84, 73, 79, 78, 32, 5, 78, 84, 72, 65, 32, 242, 179, - 20, 86, 139, 129, 5, 80, 2, 11, 32, 2, 151, 184, 25, 67, 146, 1, 120, 7, - 76, 69, 84, 84, 69, 82, 32, 212, 2, 5, 83, 73, 71, 78, 32, 226, 231, 22, - 65, 248, 8, 2, 86, 79, 239, 195, 3, 79, 100, 214, 1, 86, 250, 235, 22, - 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, - 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, - 8, 69, 158, 20, 72, 2, 77, 2, 82, 3, 89, 14, 60, 5, 69, 68, 73, 67, 32, - 210, 242, 22, 79, 139, 211, 3, 65, 4, 244, 143, 24, 6, 68, 79, 85, 66, - 76, 69, 235, 240, 1, 65, 16, 66, 67, 250, 176, 22, 78, 242, 60, 65, 182, - 1, 80, 179, 146, 3, 86, 4, 226, 238, 7, 79, 195, 193, 14, 65, 192, 8, 76, - 10, 65, 84, 69, 82, 45, 84, 72, 65, 78, 32, 206, 7, 69, 219, 180, 25, 89, - 56, 134, 1, 65, 150, 3, 66, 62, 79, 216, 2, 11, 69, 81, 85, 65, 76, 32, - 84, 79, 32, 79, 82, 222, 205, 6, 67, 138, 4, 87, 231, 227, 18, 83, 16, - 44, 5, 66, 79, 86, 69, 32, 187, 210, 6, 78, 12, 150, 1, 83, 180, 1, 19, - 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, 69, 81, 85, 65, 76, 32, - 65, 224, 204, 6, 4, 76, 69, 83, 83, 135, 211, 18, 82, 6, 148, 1, 7, 73, - 77, 73, 76, 65, 82, 32, 225, 207, 6, 23, 76, 65, 78, 84, 69, 68, 32, 69, - 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 76, 69, 83, 83, 4, 26, 65, - 183, 207, 6, 79, 2, 65, 3, 66, 79, 86, 6, 40, 4, 69, 83, 73, 68, 159, - 208, 6, 85, 2, 231, 2, 69, 20, 40, 2, 82, 32, 245, 1, 3, 86, 69, 82, 16, - 120, 16, 83, 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 84, 79, - 230, 209, 6, 65, 138, 247, 12, 69, 131, 227, 4, 76, 9, 49, 10, 32, 87, - 73, 84, 72, 32, 68, 79, 84, 32, 6, 44, 5, 65, 66, 79, 86, 69, 163, 153, - 18, 73, 5, 195, 138, 26, 32, 4, 52, 7, 76, 65, 80, 80, 73, 78, 71, 227, - 232, 19, 32, 2, 185, 170, 24, 2, 32, 76, 134, 8, 36, 2, 75, 32, 185, 73, - 2, 78, 32, 254, 7, 130, 3, 65, 216, 15, 8, 67, 65, 80, 73, 84, 65, 76, - 32, 182, 11, 68, 134, 1, 70, 68, 2, 73, 78, 222, 3, 75, 138, 1, 76, 174, - 3, 78, 66, 77, 84, 3, 88, 69, 83, 22, 79, 202, 1, 80, 90, 82, 182, 1, 83, - 130, 22, 84, 200, 2, 13, 85, 80, 83, 73, 76, 79, 78, 32, 87, 73, 84, 72, - 32, 150, 1, 86, 142, 2, 89, 178, 225, 7, 66, 248, 175, 3, 4, 71, 82, 65, - 77, 204, 23, 2, 90, 69, 147, 249, 11, 81, 112, 92, 10, 67, 82, 79, 80, - 72, 79, 78, 73, 67, 32, 172, 14, 6, 78, 79, 32, 84, 69, 76, 23, 82, 106, - 188, 2, 6, 65, 84, 84, 73, 67, 32, 222, 5, 67, 92, 3, 78, 65, 88, 32, 12, - 68, 69, 76, 80, 72, 73, 67, 32, 70, 73, 86, 69, 0, 14, 83, 84, 82, 65, - 84, 73, 65, 78, 32, 70, 73, 70, 84, 89, 40, 11, 69, 80, 73, 68, 65, 85, - 82, 69, 65, 78, 32, 112, 3, 72, 69, 82, 164, 1, 9, 77, 69, 83, 83, 69, - 78, 73, 65, 78, 35, 84, 48, 72, 2, 70, 73, 180, 2, 4, 79, 78, 69, 32, - 205, 1, 4, 84, 69, 78, 32, 26, 36, 3, 70, 84, 89, 105, 2, 86, 69, 11, 11, - 32, 8, 22, 84, 171, 4, 83, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 215, 3, - 65, 5, 231, 3, 32, 17, 11, 32, 14, 56, 7, 72, 85, 78, 68, 82, 69, 68, 18, - 84, 143, 3, 83, 7, 131, 2, 32, 6, 48, 7, 72, 79, 85, 83, 65, 78, 68, 187, - 2, 65, 5, 213, 1, 2, 32, 84, 14, 98, 72, 48, 7, 84, 72, 79, 85, 83, 65, - 78, 174, 174, 24, 81, 241, 225, 1, 5, 68, 82, 65, 67, 72, 6, 44, 5, 85, - 78, 68, 82, 69, 143, 174, 24, 65, 4, 17, 2, 68, 32, 4, 22, 84, 131, 1, - 83, 2, 95, 65, 8, 30, 84, 86, 83, 175, 1, 77, 4, 50, 65, 21, 8, 72, 79, - 85, 83, 65, 78, 68, 32, 2, 187, 174, 16, 76, 2, 11, 83, 2, 145, 175, 24, - 2, 84, 65, 4, 88, 5, 65, 82, 89, 83, 84, 145, 1, 12, 89, 82, 69, 78, 65, - 73, 67, 32, 84, 87, 79, 32, 2, 101, 5, 73, 65, 78, 32, 70, 2, 17, 2, 32, - 77, 2, 199, 144, 13, 78, 6, 30, 70, 29, 3, 84, 87, 79, 2, 177, 178, 15, - 2, 73, 86, 5, 11, 32, 2, 233, 144, 10, 5, 68, 82, 65, 67, 72, 8, 112, 8, - 77, 73, 79, 78, 73, 65, 78, 32, 189, 135, 25, 14, 65, 69, 85, 77, 32, 79, - 78, 69, 32, 80, 76, 69, 84, 72, 6, 238, 185, 12, 70, 134, 159, 12, 84, - 191, 58, 79, 2, 11, 32, 2, 183, 216, 24, 84, 32, 92, 8, 72, 69, 83, 80, - 73, 65, 78, 32, 129, 1, 10, 82, 79, 69, 90, 69, 78, 73, 65, 78, 32, 20, - 40, 2, 70, 73, 38, 84, 135, 151, 18, 79, 6, 162, 215, 20, 86, 231, 230, - 3, 70, 8, 202, 174, 15, 72, 178, 172, 10, 69, 239, 48, 87, 12, 36, 2, 70, - 73, 209, 24, 2, 84, 69, 8, 142, 176, 18, 86, 205, 250, 6, 3, 70, 84, 89, - 2, 159, 132, 10, 69, 4, 240, 215, 22, 2, 79, 85, 161, 181, 1, 3, 84, 65, - 66, 154, 2, 66, 76, 174, 45, 82, 66, 68, 220, 226, 7, 2, 75, 65, 135, 6, - 84, 144, 2, 44, 6, 69, 84, 84, 69, 82, 32, 239, 45, 85, 142, 2, 198, 2, - 65, 190, 1, 69, 28, 4, 73, 79, 84, 65, 128, 1, 2, 79, 77, 156, 3, 3, 82, - 72, 79, 46, 83, 48, 7, 85, 80, 83, 73, 76, 79, 78, 146, 33, 80, 170, 2, - 84, 202, 183, 5, 68, 144, 176, 2, 2, 75, 65, 166, 192, 1, 71, 190, 132, - 11, 67, 194, 149, 2, 66, 2, 72, 2, 90, 166, 1, 76, 230, 231, 2, 89, 210, - 43, 77, 2, 78, 147, 17, 88, 48, 68, 4, 76, 80, 72, 65, 213, 28, 8, 82, - 67, 72, 65, 73, 67, 32, 83, 47, 33, 6, 32, 87, 73, 84, 72, 32, 44, 242, - 2, 68, 30, 80, 226, 29, 86, 226, 5, 79, 246, 236, 3, 84, 239, 167, 5, 77, - 62, 186, 1, 84, 131, 27, 80, 31, 33, 6, 32, 87, 73, 84, 72, 32, 28, 186, - 5, 68, 136, 25, 2, 80, 83, 158, 1, 86, 226, 5, 79, 246, 236, 3, 84, 239, - 167, 5, 77, 62, 28, 2, 69, 71, 235, 34, 73, 42, 11, 65, 43, 33, 6, 32, - 87, 73, 84, 72, 32, 40, 54, 68, 30, 80, 194, 35, 79, 22, 86, 227, 236, 3, - 84, 16, 65, 4, 65, 83, 73, 65, 18, 36, 4, 83, 73, 76, 73, 211, 16, 82, - 17, 29, 5, 32, 65, 78, 68, 32, 14, 44, 2, 79, 88, 0, 3, 86, 65, 82, 23, - 80, 4, 81, 2, 73, 65, 6, 60, 10, 69, 82, 73, 83, 80, 79, 77, 69, 78, 73, - 175, 15, 82, 5, 169, 15, 7, 32, 65, 78, 68, 32, 80, 82, 5, 161, 35, 7, - 32, 87, 73, 84, 72, 32, 68, 6, 222, 140, 8, 73, 226, 194, 17, 65, 239, - 48, 72, 23, 33, 6, 32, 87, 73, 84, 72, 32, 20, 66, 68, 166, 26, 86, 226, - 5, 79, 246, 236, 3, 84, 239, 167, 5, 77, 10, 130, 24, 65, 181, 176, 22, - 5, 73, 65, 76, 89, 84, 18, 76, 9, 73, 65, 76, 89, 84, 73, 75, 65, 32, 32, - 3, 82, 65, 67, 227, 22, 65, 8, 174, 24, 65, 199, 243, 3, 84, 2, 243, 186, - 11, 72, 4, 40, 3, 73, 86, 69, 1, 3, 79, 85, 82, 2, 205, 37, 2, 32, 79, - 76, 144, 1, 27, 83, 84, 82, 85, 77, 69, 78, 84, 65, 76, 32, 78, 79, 84, - 65, 84, 73, 79, 78, 32, 83, 89, 77, 66, 79, 76, 45, 161, 154, 22, 2, 68, - 73, 74, 70, 49, 70, 50, 62, 51, 62, 52, 170, 37, 53, 206, 243, 25, 55, 3, - 56, 17, 174, 154, 26, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, 56, 3, 57, 15, - 234, 153, 26, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 174, 153, 26, - 48, 2, 50, 2, 54, 2, 55, 2, 56, 3, 57, 17, 242, 152, 26, 48, 2, 50, 2, - 51, 2, 53, 2, 55, 2, 56, 3, 57, 8, 54, 65, 38, 79, 169, 32, 6, 89, 65, - 84, 72, 79, 83, 4, 134, 255, 7, 80, 203, 133, 17, 73, 2, 11, 82, 2, 11, - 79, 2, 135, 244, 23, 78, 32, 128, 1, 6, 69, 84, 84, 69, 82, 32, 168, 2, - 6, 79, 87, 69, 82, 32, 78, 32, 6, 85, 78, 65, 84, 69, 32, 145, 194, 22, - 2, 73, 84, 24, 94, 83, 140, 13, 9, 65, 82, 67, 72, 65, 73, 67, 32, 75, 2, - 75, 182, 11, 68, 207, 172, 25, 89, 16, 88, 13, 77, 65, 76, 76, 32, 67, - 65, 80, 73, 84, 65, 76, 32, 210, 12, 65, 247, 243, 7, 84, 12, 74, 80, - 226, 192, 9, 71, 194, 156, 9, 79, 218, 206, 2, 82, 139, 176, 1, 76, 4, - 194, 128, 26, 83, 219, 19, 73, 2, 221, 134, 10, 3, 85, 77, 69, 4, 222, - 25, 83, 215, 224, 7, 69, 4, 80, 4, 69, 84, 82, 69, 241, 192, 23, 10, 85, - 83, 73, 67, 65, 76, 32, 76, 69, 73, 2, 163, 212, 2, 84, 12, 88, 3, 78, - 69, 32, 198, 236, 9, 88, 148, 155, 5, 3, 85, 78, 75, 253, 198, 5, 2, 66, - 79, 6, 64, 8, 72, 65, 76, 70, 32, 83, 73, 71, 21, 4, 81, 85, 65, 82, 4, - 139, 146, 25, 78, 2, 255, 206, 20, 84, 16, 62, 82, 206, 11, 83, 114, 69, - 154, 236, 7, 72, 211, 132, 17, 73, 2, 217, 29, 2, 79, 83, 6, 100, 3, 72, - 79, 32, 165, 246, 7, 16, 69, 86, 69, 82, 83, 69, 68, 32, 76, 85, 78, 65, - 84, 69, 32, 69, 4, 180, 243, 13, 10, 87, 73, 84, 72, 32, 83, 84, 82, 79, - 75, 135, 137, 11, 83, 226, 2, 220, 1, 5, 77, 65, 76, 76, 32, 192, 19, 22, - 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, - 84, 69, 82, 32, 72, 10, 89, 77, 66, 79, 76, 32, 84, 65, 85, 32, 189, 144, - 24, 6, 73, 78, 85, 83, 79, 73, 212, 2, 56, 7, 76, 69, 84, 84, 69, 82, 32, - 202, 17, 82, 67, 68, 206, 2, 178, 2, 65, 162, 2, 68, 38, 69, 52, 4, 73, - 79, 84, 65, 0, 7, 85, 80, 83, 73, 76, 79, 78, 254, 3, 75, 28, 2, 79, 77, - 182, 5, 80, 112, 3, 82, 72, 79, 94, 83, 94, 84, 244, 230, 7, 2, 70, 73, - 138, 193, 1, 71, 190, 132, 11, 67, 194, 149, 2, 66, 2, 72, 2, 90, 166, 1, - 76, 182, 147, 3, 77, 2, 78, 147, 17, 88, 58, 64, 4, 76, 80, 72, 65, 149, - 1, 7, 82, 67, 72, 65, 73, 67, 32, 55, 33, 6, 32, 87, 73, 84, 72, 32, 52, - 82, 86, 226, 6, 68, 30, 80, 114, 79, 142, 1, 89, 226, 238, 3, 84, 239, - 167, 5, 77, 6, 154, 5, 82, 155, 3, 65, 4, 18, 75, 23, 83, 2, 215, 244, 7, - 79, 2, 11, 65, 2, 203, 189, 13, 77, 4, 138, 223, 7, 73, 235, 239, 14, 69, - 70, 22, 80, 215, 4, 84, 20, 249, 7, 3, 83, 73, 76, 41, 33, 6, 32, 87, 73, - 84, 72, 32, 38, 78, 68, 166, 1, 80, 178, 1, 86, 226, 5, 79, 246, 236, 3, - 84, 239, 167, 5, 77, 18, 50, 65, 29, 8, 73, 65, 76, 89, 84, 73, 75, 65, - 8, 153, 1, 3, 83, 73, 65, 11, 29, 5, 32, 65, 78, 68, 32, 8, 170, 1, 80, - 154, 6, 79, 22, 86, 227, 236, 3, 84, 10, 18, 83, 115, 69, 8, 21, 3, 73, - 76, 73, 9, 17, 2, 32, 65, 6, 21, 3, 78, 68, 32, 6, 30, 80, 154, 6, 79, - 23, 86, 2, 11, 69, 2, 11, 82, 2, 181, 17, 4, 73, 83, 80, 79, 4, 22, 82, - 147, 15, 65, 2, 133, 226, 25, 2, 65, 67, 4, 206, 239, 7, 65, 3, 79, 70, - 28, 2, 69, 71, 151, 3, 73, 50, 11, 65, 51, 33, 6, 32, 87, 73, 84, 72, 32, - 48, 58, 68, 30, 80, 114, 79, 62, 86, 82, 89, 227, 238, 3, 84, 16, 61, 4, - 65, 83, 73, 65, 20, 32, 4, 83, 73, 76, 73, 91, 69, 17, 29, 5, 32, 65, 78, - 68, 32, 14, 42, 79, 12, 2, 80, 69, 50, 86, 83, 89, 4, 83, 88, 4, 89, 9, - 82, 73, 83, 80, 79, 77, 69, 78, 73, 4, 11, 65, 4, 11, 82, 4, 17, 2, 73, - 65, 5, 33, 6, 32, 65, 78, 68, 32, 89, 2, 243, 12, 80, 20, 17, 2, 67, 82, - 20, 17, 2, 79, 78, 21, 33, 6, 32, 87, 73, 84, 72, 32, 18, 88, 5, 68, 65, - 83, 73, 65, 0, 5, 80, 83, 73, 76, 73, 54, 79, 22, 86, 227, 236, 3, 84, 7, - 29, 5, 32, 65, 78, 68, 32, 4, 18, 79, 23, 86, 2, 207, 216, 9, 88, 2, 179, - 9, 65, 8, 88, 11, 65, 77, 80, 72, 89, 76, 73, 65, 78, 32, 68, 174, 233, - 25, 72, 2, 83, 219, 19, 73, 2, 139, 212, 7, 73, 7, 33, 6, 32, 87, 73, 84, - 72, 32, 4, 34, 68, 145, 129, 21, 2, 80, 83, 2, 243, 197, 8, 65, 10, 54, - 65, 186, 231, 7, 84, 230, 1, 73, 207, 243, 17, 72, 4, 142, 177, 13, 77, - 207, 202, 12, 78, 4, 246, 193, 22, 72, 219, 148, 3, 65, 4, 41, 8, 69, 86, - 69, 82, 83, 69, 68, 32, 4, 18, 68, 43, 76, 2, 37, 7, 79, 84, 84, 69, 68, - 32, 76, 2, 11, 85, 2, 33, 6, 78, 65, 84, 69, 32, 83, 2, 169, 232, 7, 3, - 73, 71, 77, 10, 158, 166, 9, 71, 190, 132, 11, 67, 2, 80, 222, 102, 82, - 231, 174, 1, 66, 2, 239, 144, 21, 82, 16, 106, 72, 104, 7, 82, 89, 66, - 76, 73, 79, 78, 44, 3, 87, 79, 32, 254, 229, 3, 79, 233, 196, 16, 2, 65, - 76, 6, 40, 4, 82, 69, 69, 32, 143, 230, 7, 69, 4, 146, 1, 79, 157, 200, - 22, 7, 81, 85, 65, 82, 84, 69, 82, 2, 21, 3, 32, 66, 65, 2, 231, 218, 23, - 83, 4, 42, 79, 221, 152, 12, 4, 84, 72, 73, 82, 2, 237, 224, 19, 2, 66, - 79, 6, 80, 5, 65, 67, 85, 84, 69, 0, 9, 68, 73, 65, 69, 82, 69, 83, 73, - 83, 39, 72, 2, 33, 6, 32, 65, 78, 68, 32, 72, 2, 197, 195, 7, 2, 79, 79, - 60, 102, 65, 21, 21, 79, 67, 65, 76, 32, 78, 79, 84, 65, 84, 73, 79, 78, - 32, 83, 89, 77, 66, 79, 76, 45, 2, 135, 207, 9, 82, 58, 90, 50, 2, 53, - 154, 179, 23, 49, 182, 192, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 13, - 202, 243, 25, 48, 2, 49, 2, 50, 2, 51, 3, 52, 4, 26, 80, 207, 177, 20, - 69, 2, 11, 79, 2, 33, 6, 71, 69, 71, 82, 65, 77, 2, 197, 246, 20, 2, 77, - 69, 8, 170, 237, 13, 65, 178, 177, 10, 66, 174, 76, 72, 253, 64, 3, 83, - 65, 76, 12, 60, 6, 78, 78, 73, 78, 71, 32, 137, 233, 24, 3, 77, 65, 67, - 10, 100, 4, 70, 65, 67, 69, 149, 228, 17, 15, 67, 65, 84, 32, 70, 65, 67, - 69, 32, 87, 73, 84, 72, 32, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 108, - 23, 79, 78, 69, 32, 76, 65, 82, 71, 69, 32, 65, 78, 68, 32, 79, 78, 69, - 32, 83, 77, 65, 76, 76, 35, 83, 2, 11, 32, 2, 191, 157, 8, 69, 4, 32, 2, - 84, 65, 203, 226, 17, 77, 2, 187, 243, 22, 82, 6, 28, 3, 85, 80, 32, 39, - 87, 4, 214, 192, 22, 83, 179, 236, 2, 77, 2, 227, 144, 18, 73, 220, 4, - 136, 1, 2, 65, 82, 70, 73, 52, 7, 74, 65, 82, 65, 84, 73, 32, 208, 6, 12, - 78, 74, 65, 76, 65, 32, 71, 79, 78, 68, 73, 32, 143, 3, 82, 4, 34, 65, - 245, 246, 20, 2, 68, 83, 2, 11, 78, 2, 187, 229, 24, 73, 4, 238, 202, 24, - 84, 217, 160, 1, 4, 68, 69, 32, 68, 182, 1, 168, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 220, 1, 5, 83, 73, 71, 78, 32, 160, 2, 6, 86, 79, 87, 69, 76, - 32, 238, 251, 19, 65, 154, 24, 82, 166, 225, 3, 68, 203, 224, 1, 79, 98, - 162, 144, 22, 65, 38, 68, 114, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, - 42, 76, 246, 14, 90, 134, 178, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, - 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, - 24, 98, 67, 28, 3, 77, 65, 68, 22, 84, 138, 202, 3, 83, 238, 137, 18, 78, - 242, 60, 65, 231, 147, 3, 86, 4, 118, 73, 219, 210, 21, 65, 2, 231, 135, - 19, 68, 4, 68, 5, 87, 79, 45, 67, 73, 29, 8, 72, 82, 69, 69, 45, 68, 79, - 84, 2, 25, 4, 82, 67, 76, 69, 2, 253, 195, 20, 5, 32, 78, 85, 75, 84, 34, - 36, 5, 83, 73, 71, 78, 32, 87, 67, 30, 82, 67, 130, 146, 22, 65, 38, 85, - 22, 86, 186, 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 245, 210, 21, 5, 65, - 78, 68, 82, 65, 126, 108, 7, 76, 69, 84, 84, 69, 82, 32, 216, 1, 5, 83, - 73, 71, 78, 32, 38, 86, 154, 241, 23, 68, 203, 224, 1, 79, 80, 130, 243, - 21, 78, 146, 23, 65, 38, 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 42, - 76, 246, 193, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, - 8, 69, 158, 20, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 89, 4, 178, 172, 23, - 86, 179, 241, 1, 65, 20, 190, 7, 79, 163, 162, 23, 73, 160, 2, 84, 6, 77, - 85, 75, 72, 73, 32, 189, 7, 10, 85, 78, 71, 32, 75, 72, 69, 77, 65, 32, - 172, 1, 194, 1, 65, 44, 7, 76, 69, 84, 84, 69, 82, 32, 238, 1, 83, 228, - 2, 2, 86, 79, 172, 144, 13, 3, 84, 73, 80, 130, 237, 5, 73, 216, 170, 2, - 5, 69, 75, 32, 79, 78, 214, 193, 2, 68, 227, 172, 1, 85, 4, 218, 200, 21, - 66, 213, 225, 1, 2, 68, 68, 96, 162, 194, 8, 71, 2, 75, 134, 195, 13, 65, - 38, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, - 78, 126, 66, 2, 67, 2, 74, 2, 80, 2, 83, 206, 40, 79, 162, 8, 69, 158, - 20, 70, 2, 72, 2, 77, 2, 86, 2, 89, 3, 90, 26, 108, 19, 69, 81, 85, 69, - 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 93, 4, 73, - 71, 78, 32, 12, 70, 71, 2, 75, 230, 225, 23, 83, 170, 216, 1, 76, 226, - 31, 70, 3, 90, 2, 227, 225, 23, 72, 14, 128, 1, 6, 65, 68, 65, 75, 32, - 66, 2, 66, 176, 140, 15, 2, 85, 68, 254, 186, 6, 78, 180, 171, 2, 3, 89, - 65, 75, 163, 165, 1, 86, 2, 255, 147, 8, 73, 18, 45, 9, 87, 69, 76, 32, - 83, 73, 71, 78, 32, 18, 250, 134, 22, 65, 38, 85, 206, 201, 1, 73, 234, - 234, 1, 79, 163, 8, 69, 116, 216, 1, 22, 67, 79, 78, 83, 79, 78, 65, 78, - 84, 32, 83, 73, 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 44, 7, 76, 69, - 84, 84, 69, 82, 32, 168, 1, 5, 83, 73, 71, 78, 32, 56, 6, 86, 79, 87, 69, - 76, 32, 251, 228, 23, 68, 8, 234, 213, 25, 72, 2, 82, 2, 86, 3, 89, 60, - 198, 230, 21, 78, 182, 23, 68, 114, 84, 206, 145, 3, 66, 2, 67, 2, 71, 2, - 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, 89, - 187, 2, 65, 4, 214, 145, 25, 65, 233, 35, 6, 84, 72, 79, 76, 72, 79, 24, - 130, 148, 20, 83, 155, 128, 5, 76, 162, 23, 110, 65, 174, 95, 69, 246, - 103, 73, 146, 9, 79, 158, 12, 84, 30, 85, 130, 1, 89, 249, 236, 12, 4, - 82, 89, 86, 78, 138, 11, 236, 1, 2, 73, 82, 60, 8, 76, 70, 87, 73, 68, - 84, 72, 32, 242, 10, 77, 210, 1, 78, 236, 76, 21, 80, 80, 89, 32, 80, 69, - 82, 83, 79, 78, 32, 82, 65, 73, 83, 73, 78, 71, 32, 79, 78, 22, 82, 38, - 84, 252, 242, 17, 2, 85, 77, 251, 241, 5, 68, 6, 26, 32, 163, 217, 24, - 67, 4, 242, 162, 24, 80, 211, 113, 83, 244, 1, 140, 2, 7, 72, 65, 78, 71, - 85, 76, 32, 216, 4, 8, 75, 65, 84, 65, 75, 65, 78, 65, 204, 3, 3, 76, 69, - 70, 0, 4, 82, 73, 71, 72, 140, 190, 6, 11, 70, 79, 82, 77, 83, 32, 76, - 73, 71, 72, 84, 142, 129, 4, 85, 174, 218, 2, 73, 250, 184, 4, 66, 242, - 227, 4, 87, 211, 35, 68, 104, 52, 7, 76, 69, 84, 84, 69, 82, 32, 167, - 217, 10, 70, 102, 206, 1, 75, 28, 5, 78, 73, 69, 85, 78, 42, 80, 24, 5, - 82, 73, 69, 85, 76, 86, 83, 98, 89, 202, 61, 67, 54, 69, 30, 73, 242, 4, - 77, 138, 1, 84, 206, 3, 87, 198, 1, 72, 230, 194, 24, 65, 2, 79, 163, 64, - 85, 6, 222, 65, 72, 155, 3, 73, 7, 11, 45, 4, 134, 72, 67, 131, 3, 72, 6, - 146, 69, 72, 35, 73, 17, 11, 45, 14, 206, 49, 84, 226, 14, 80, 130, 4, - 77, 194, 3, 75, 218, 2, 72, 99, 83, 12, 40, 4, 83, 65, 78, 71, 151, 221, - 13, 73, 10, 210, 70, 67, 42, 75, 74, 80, 34, 84, 211, 2, 83, 14, 194, - 134, 23, 69, 194, 133, 2, 65, 162, 64, 73, 2, 79, 3, 85, 118, 70, 32, - 193, 240, 2, 11, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 80, 116, 76, 7, - 76, 69, 84, 84, 69, 82, 32, 150, 240, 2, 83, 34, 86, 215, 200, 21, 77, - 110, 146, 1, 83, 138, 234, 2, 78, 150, 2, 72, 2, 75, 2, 77, 2, 82, 2, 84, - 170, 1, 89, 222, 41, 87, 154, 178, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, - 28, 76, 5, 77, 65, 76, 76, 32, 234, 200, 25, 65, 2, 69, 2, 73, 2, 79, 3, - 85, 18, 242, 236, 2, 89, 142, 183, 22, 84, 234, 36, 65, 2, 69, 2, 73, 2, - 79, 3, 85, 4, 11, 84, 4, 212, 157, 13, 2, 32, 67, 239, 138, 11, 87, 14, - 56, 3, 77, 69, 82, 106, 83, 233, 248, 21, 3, 66, 85, 82, 9, 29, 5, 32, - 65, 78, 68, 32, 6, 168, 170, 12, 3, 87, 82, 69, 144, 228, 3, 2, 83, 73, - 207, 136, 8, 80, 4, 152, 137, 25, 3, 84, 69, 82, 163, 61, 65, 194, 8, - 118, 68, 202, 2, 71, 128, 66, 13, 73, 70, 73, 32, 82, 79, 72, 73, 78, 71, - 89, 65, 32, 165, 5, 5, 85, 78, 79, 79, 32, 10, 100, 12, 32, 87, 73, 84, - 72, 32, 73, 78, 68, 69, 88, 32, 188, 1, 2, 66, 65, 193, 218, 21, 2, 83, - 72, 4, 156, 1, 18, 65, 78, 68, 32, 77, 73, 68, 68, 76, 69, 32, 70, 73, - 78, 71, 69, 82, 83, 1, 16, 70, 73, 78, 71, 69, 82, 32, 65, 78, 68, 32, - 84, 72, 85, 77, 66, 2, 233, 134, 16, 2, 32, 67, 4, 186, 185, 24, 76, 183, - 137, 1, 71, 170, 7, 84, 3, 85, 76, 32, 217, 64, 13, 90, 72, 79, 85, 32, - 78, 85, 77, 69, 82, 65, 76, 32, 146, 7, 164, 1, 9, 67, 72, 79, 83, 69, - 79, 78, 71, 32, 244, 15, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 46, 74, - 180, 31, 7, 76, 69, 84, 84, 69, 82, 32, 147, 154, 10, 70, 250, 1, 246, 1, - 67, 172, 2, 5, 73, 69, 85, 78, 71, 146, 1, 75, 132, 1, 5, 77, 73, 69, 85, - 77, 56, 5, 78, 73, 69, 85, 78, 74, 80, 172, 2, 5, 82, 73, 69, 85, 76, - 210, 1, 83, 166, 3, 84, 124, 2, 89, 69, 200, 40, 5, 72, 73, 69, 85, 72, - 199, 145, 10, 70, 30, 76, 2, 72, 73, 84, 7, 69, 79, 78, 71, 67, 72, 73, - 121, 4, 73, 69, 85, 67, 16, 40, 4, 69, 85, 67, 72, 41, 2, 84, 85, 7, 11, - 45, 4, 202, 31, 75, 243, 26, 72, 10, 21, 3, 69, 85, 77, 10, 22, 83, 155, - 46, 67, 6, 40, 4, 83, 65, 78, 71, 135, 205, 13, 73, 4, 194, 54, 67, 227, - 3, 83, 5, 215, 30, 45, 27, 11, 45, 24, 90, 80, 234, 30, 82, 242, 13, 67, - 194, 5, 77, 138, 1, 84, 186, 2, 75, 218, 2, 72, 99, 83, 6, 214, 50, 72, - 214, 3, 73, 207, 2, 65, 16, 80, 7, 65, 80, 89, 69, 79, 85, 78, 228, 20, - 5, 73, 89, 69, 79, 75, 131, 25, 72, 10, 234, 29, 82, 178, 15, 80, 30, 83, - 231, 3, 77, 11, 11, 45, 8, 158, 52, 75, 74, 80, 34, 84, 211, 2, 83, 15, - 11, 45, 12, 190, 51, 67, 42, 75, 74, 80, 34, 84, 242, 1, 72, 99, 83, 42, - 68, 6, 72, 73, 69, 85, 80, 72, 40, 4, 73, 69, 85, 80, 223, 53, 65, 7, 11, - 45, 4, 158, 51, 80, 147, 2, 72, 35, 11, 45, 32, 82, 83, 234, 20, 80, 214, - 7, 75, 162, 12, 67, 202, 6, 84, 226, 2, 78, 179, 2, 72, 14, 32, 3, 73, - 79, 83, 251, 3, 83, 13, 11, 45, 10, 242, 46, 84, 146, 2, 67, 42, 75, 75, - 80, 29, 11, 45, 26, 78, 75, 42, 83, 190, 44, 77, 154, 3, 67, 82, 78, 34, - 80, 34, 84, 243, 1, 72, 6, 170, 22, 65, 130, 19, 72, 135, 7, 73, 8, 40, - 4, 83, 65, 78, 71, 235, 197, 13, 73, 6, 206, 47, 75, 74, 80, 35, 84, 58, - 48, 3, 73, 79, 83, 217, 1, 4, 83, 65, 78, 71, 33, 11, 45, 30, 130, 1, 80, - 44, 2, 83, 83, 210, 22, 82, 210, 1, 75, 162, 12, 67, 194, 5, 77, 138, 1, - 84, 226, 2, 78, 178, 2, 72, 191, 157, 18, 73, 6, 184, 23, 4, 73, 69, 85, - 80, 179, 19, 72, 2, 233, 48, 3, 65, 78, 71, 26, 236, 17, 5, 67, 73, 69, - 85, 67, 172, 3, 4, 83, 73, 79, 83, 154, 1, 82, 186, 20, 84, 54, 89, 134, - 2, 75, 42, 78, 34, 80, 146, 2, 72, 191, 157, 18, 73, 16, 40, 5, 73, 75, - 69, 85, 84, 195, 41, 72, 15, 11, 45, 12, 226, 20, 82, 178, 19, 77, 154, - 3, 67, 42, 75, 74, 80, 243, 2, 83, 4, 150, 19, 83, 139, 22, 79, 2, 169, - 245, 8, 6, 76, 69, 32, 68, 79, 84, 216, 3, 92, 9, 79, 78, 71, 83, 69, 79, - 78, 71, 32, 165, 21, 9, 85, 78, 71, 83, 69, 79, 78, 71, 32, 154, 2, 226, - 1, 67, 80, 5, 72, 73, 69, 85, 72, 60, 5, 73, 69, 85, 78, 71, 46, 75, 220, - 1, 5, 77, 73, 69, 85, 77, 188, 1, 5, 78, 73, 69, 85, 78, 94, 80, 240, 2, - 5, 82, 73, 69, 85, 76, 190, 4, 83, 194, 3, 84, 213, 1, 2, 89, 69, 8, 36, - 4, 73, 69, 85, 67, 239, 30, 72, 7, 11, 45, 4, 162, 32, 83, 239, 7, 80, - 11, 11, 45, 8, 174, 16, 82, 178, 19, 77, 234, 3, 78, 35, 80, 9, 11, 45, - 6, 194, 17, 75, 57, 2, 83, 83, 28, 76, 7, 65, 80, 89, 69, 79, 85, 78, 40, - 5, 73, 89, 69, 79, 75, 215, 30, 72, 8, 130, 15, 82, 178, 15, 80, 131, 4, - 77, 19, 11, 45, 16, 166, 13, 75, 170, 1, 82, 42, 83, 224, 13, 2, 67, 72, - 146, 9, 78, 34, 80, 147, 2, 72, 27, 11, 45, 24, 74, 80, 30, 83, 134, 13, - 82, 242, 13, 67, 130, 9, 75, 42, 78, 179, 2, 72, 6, 174, 33, 73, 131, 6, - 65, 6, 40, 4, 83, 65, 78, 71, 227, 185, 13, 73, 4, 238, 35, 78, 147, 3, - 83, 21, 11, 45, 18, 174, 12, 82, 242, 13, 67, 202, 6, 84, 186, 2, 75, - 218, 2, 72, 62, 80, 39, 83, 36, 88, 6, 65, 78, 83, 73, 79, 83, 48, 6, 72, - 73, 69, 85, 80, 72, 53, 4, 73, 69, 85, 80, 7, 11, 45, 4, 236, 7, 2, 75, - 65, 195, 26, 80, 9, 11, 45, 6, 150, 11, 84, 234, 22, 80, 243, 2, 83, 23, - 11, 45, 20, 112, 5, 82, 73, 69, 85, 76, 24, 4, 83, 73, 79, 83, 134, 3, - 80, 246, 19, 67, 194, 5, 77, 170, 4, 84, 243, 1, 72, 5, 153, 3, 2, 45, - 80, 5, 221, 32, 2, 45, 84, 57, 11, 45, 54, 102, 75, 92, 5, 77, 73, 69, - 85, 77, 50, 80, 126, 83, 74, 84, 44, 2, 89, 69, 154, 28, 78, 179, 2, 72, - 10, 52, 5, 73, 89, 69, 79, 75, 190, 4, 65, 131, 19, 72, 7, 11, 45, 4, - 254, 32, 72, 99, 83, 9, 11, 45, 6, 130, 30, 75, 218, 2, 72, 99, 83, 14, - 48, 4, 73, 69, 85, 80, 174, 26, 72, 163, 6, 65, 11, 11, 45, 8, 42, 80, - 222, 29, 84, 242, 1, 72, 99, 83, 2, 243, 25, 72, 6, 40, 4, 83, 65, 78, - 71, 211, 178, 13, 73, 4, 182, 28, 75, 187, 3, 83, 6, 100, 5, 73, 75, 69, - 85, 84, 151, 25, 72, 6, 56, 9, 79, 82, 73, 78, 72, 73, 69, 85, 72, 191, - 3, 83, 5, 255, 29, 45, 52, 48, 3, 73, 79, 83, 161, 1, 4, 83, 65, 78, 71, - 25, 11, 45, 22, 82, 75, 162, 3, 82, 242, 13, 67, 198, 2, 80, 254, 2, 77, - 138, 1, 84, 147, 5, 72, 4, 22, 65, 135, 26, 73, 2, 237, 18, 6, 80, 89, - 69, 79, 85, 78, 28, 160, 1, 5, 82, 73, 69, 85, 76, 36, 6, 84, 73, 75, 69, - 85, 84, 16, 3, 89, 69, 83, 138, 19, 83, 178, 1, 77, 154, 3, 67, 42, 75, - 42, 78, 34, 80, 207, 159, 18, 73, 5, 17, 2, 45, 75, 2, 159, 17, 72, 5, - 255, 16, 45, 2, 139, 184, 18, 73, 20, 40, 5, 73, 75, 69, 85, 84, 155, 21, - 72, 19, 11, 45, 16, 58, 82, 42, 83, 42, 84, 162, 13, 67, 130, 9, 75, 75, - 80, 2, 17, 2, 73, 69, 2, 131, 147, 24, 85, 4, 21, 3, 73, 79, 83, 5, 223, - 1, 45, 2, 255, 19, 72, 18, 44, 6, 83, 73, 69, 85, 78, 71, 243, 19, 79, - 17, 11, 45, 14, 50, 75, 30, 83, 198, 17, 77, 154, 6, 72, 63, 80, 4, 166, - 14, 72, 135, 7, 73, 4, 26, 83, 131, 171, 13, 73, 2, 21, 3, 65, 78, 71, 2, - 207, 20, 75, 190, 1, 122, 65, 118, 69, 134, 1, 73, 92, 7, 83, 83, 65, 78, - 71, 65, 82, 106, 79, 138, 1, 85, 102, 89, 174, 15, 87, 235, 141, 10, 70, - 23, 48, 4, 82, 65, 69, 65, 98, 45, 139, 152, 25, 69, 13, 11, 45, 10, 250, - 210, 22, 69, 226, 197, 2, 65, 2, 73, 3, 85, 25, 18, 79, 55, 85, 9, 11, - 45, 6, 158, 243, 24, 69, 234, 36, 79, 3, 85, 15, 11, 45, 12, 170, 9, 69, - 170, 142, 25, 65, 2, 79, 3, 85, 31, 11, 45, 28, 66, 65, 34, 89, 166, 1, - 79, 170, 240, 24, 69, 234, 36, 73, 3, 85, 5, 11, 82, 2, 215, 144, 18, 65, - 14, 50, 65, 162, 208, 22, 69, 226, 197, 2, 79, 3, 85, 7, 138, 247, 24, - 45, 247, 30, 69, 23, 26, 45, 199, 149, 25, 69, 18, 50, 79, 22, 89, 158, - 207, 22, 69, 227, 197, 2, 85, 5, 183, 129, 25, 45, 8, 154, 207, 22, 69, - 195, 133, 2, 65, 17, 11, 45, 14, 158, 19, 89, 186, 164, 5, 73, 152, 254, - 12, 3, 69, 79, 45, 190, 158, 6, 65, 163, 64, 85, 62, 42, 65, 70, 69, 66, - 73, 22, 79, 107, 85, 11, 26, 45, 175, 147, 25, 69, 6, 182, 244, 24, 89, - 246, 30, 79, 3, 85, 11, 11, 79, 9, 11, 45, 6, 178, 144, 25, 89, 186, 2, - 79, 3, 85, 5, 219, 237, 24, 45, 19, 11, 45, 16, 58, 89, 202, 209, 24, 65, - 174, 33, 69, 246, 30, 73, 3, 79, 6, 198, 209, 24, 65, 175, 33, 69, 21, - 11, 45, 18, 142, 16, 89, 206, 187, 22, 69, 194, 133, 2, 65, 162, 64, 73, - 2, 79, 3, 85, 186, 1, 226, 1, 65, 46, 67, 54, 69, 30, 73, 22, 75, 188, 1, - 5, 77, 73, 69, 85, 77, 64, 5, 78, 73, 69, 85, 78, 70, 80, 168, 1, 5, 82, - 73, 69, 85, 76, 254, 1, 84, 90, 83, 246, 2, 87, 50, 89, 150, 1, 72, 230, - 194, 24, 79, 163, 64, 85, 9, 156, 13, 3, 82, 65, 69, 235, 129, 25, 69, 4, - 22, 72, 207, 8, 73, 2, 141, 236, 24, 2, 73, 69, 7, 166, 142, 25, 79, 3, - 85, 5, 207, 168, 18, 69, 14, 56, 7, 65, 80, 89, 69, 79, 85, 78, 106, 72, - 155, 3, 73, 8, 30, 80, 30, 83, 231, 3, 77, 4, 190, 4, 72, 215, 3, 73, 2, - 25, 4, 83, 65, 78, 71, 2, 207, 7, 80, 2, 213, 60, 2, 73, 69, 9, 11, 45, - 6, 22, 80, 247, 9, 83, 4, 142, 7, 73, 207, 2, 65, 13, 11, 45, 10, 234, 5, - 67, 146, 1, 84, 242, 1, 72, 62, 80, 39, 83, 20, 48, 4, 73, 69, 85, 80, - 170, 2, 72, 163, 6, 65, 17, 11, 45, 14, 42, 83, 186, 2, 84, 146, 2, 67, - 43, 75, 6, 21, 3, 73, 79, 83, 7, 11, 45, 4, 202, 4, 75, 107, 84, 27, 11, - 45, 24, 68, 2, 75, 73, 34, 77, 34, 80, 106, 84, 54, 89, 222, 4, 72, 99, - 83, 4, 149, 1, 4, 89, 69, 79, 75, 2, 11, 73, 2, 255, 223, 23, 69, 8, 30, - 72, 34, 73, 131, 6, 65, 2, 193, 228, 18, 3, 73, 69, 85, 4, 21, 3, 69, 85, - 80, 5, 243, 5, 45, 4, 22, 72, 151, 3, 73, 2, 225, 230, 21, 2, 73, 69, 2, - 17, 2, 69, 79, 2, 167, 4, 82, 28, 44, 3, 73, 79, 83, 57, 4, 83, 65, 78, - 71, 13, 11, 45, 10, 122, 67, 42, 75, 42, 78, 34, 80, 35, 84, 16, 78, 67, - 42, 75, 42, 78, 34, 80, 34, 84, 242, 1, 72, 98, 83, 223, 156, 18, 73, 2, - 11, 73, 2, 249, 221, 23, 2, 69, 85, 2, 11, 73, 2, 237, 219, 24, 2, 89, - 69, 2, 11, 73, 2, 247, 132, 24, 69, 2, 11, 73, 2, 147, 143, 24, 69, 2, - 11, 73, 2, 11, 75, 2, 139, 139, 24, 69, 10, 230, 190, 22, 69, 194, 133, - 2, 65, 163, 64, 73, 34, 58, 69, 206, 1, 79, 62, 85, 182, 193, 24, 65, - 163, 64, 73, 13, 42, 79, 73, 6, 83, 73, 69, 85, 78, 71, 5, 11, 82, 2, 17, - 2, 73, 78, 2, 11, 72, 2, 161, 235, 9, 2, 73, 69, 7, 11, 45, 4, 18, 80, - 39, 83, 2, 11, 65, 2, 11, 78, 2, 11, 83, 2, 223, 146, 13, 73, 9, 11, 45, - 6, 26, 89, 235, 129, 25, 73, 4, 199, 193, 24, 65, 9, 11, 45, 6, 26, 89, - 175, 129, 25, 73, 4, 203, 187, 22, 69, 24, 210, 136, 11, 84, 162, 136, - 12, 70, 30, 83, 210, 86, 78, 14, 79, 223, 110, 69, 100, 156, 1, 7, 76, - 69, 84, 84, 69, 82, 32, 196, 2, 5, 77, 65, 82, 75, 32, 72, 5, 83, 73, 71, - 78, 32, 132, 159, 2, 6, 86, 79, 87, 69, 76, 32, 255, 235, 20, 68, 58, - 202, 1, 68, 34, 75, 254, 241, 20, 84, 154, 50, 82, 130, 223, 1, 78, 238, - 178, 1, 83, 138, 69, 66, 2, 67, 2, 70, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, - 2, 80, 2, 86, 2, 87, 2, 89, 2, 90, 187, 2, 65, 4, 166, 251, 24, 68, 187, - 2, 65, 8, 56, 5, 73, 78, 78, 65, 32, 206, 250, 24, 72, 187, 2, 65, 4, - 202, 250, 24, 87, 3, 89, 4, 248, 197, 21, 6, 78, 65, 32, 75, 72, 79, 153, - 183, 2, 3, 83, 65, 75, 8, 52, 2, 84, 65, 217, 238, 22, 5, 72, 65, 82, 66, - 65, 6, 42, 72, 158, 145, 20, 83, 235, 231, 4, 78, 2, 163, 217, 24, 65, - 42, 62, 76, 144, 239, 18, 6, 83, 73, 71, 78, 32, 80, 247, 1, 86, 36, 33, - 6, 69, 84, 84, 69, 82, 32, 36, 146, 137, 21, 78, 250, 238, 3, 66, 2, 68, - 2, 71, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, - 89, 186, 2, 65, 2, 73, 3, 85, 2, 139, 235, 23, 69, 4, 130, 180, 23, 68, - 143, 197, 1, 80, 54, 52, 5, 67, 72, 73, 78, 71, 41, 4, 82, 65, 78, 32, 2, - 17, 2, 32, 67, 2, 163, 200, 23, 72, 52, 52, 7, 76, 69, 84, 84, 69, 82, - 32, 175, 179, 6, 78, 42, 218, 1, 65, 234, 173, 11, 90, 162, 47, 84, 150, - 119, 76, 50, 81, 60, 6, 68, 65, 76, 69, 84, 72, 146, 165, 4, 71, 122, 83, - 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, 78, 134, - 2, 87, 218, 103, 80, 171, 4, 77, 4, 130, 250, 16, 76, 151, 172, 7, 89, - 172, 9, 230, 1, 65, 240, 32, 5, 66, 82, 69, 87, 32, 210, 32, 76, 200, 1, - 16, 78, 84, 65, 73, 71, 65, 78, 65, 32, 76, 69, 84, 84, 69, 82, 32, 174, - 12, 82, 120, 11, 88, 65, 71, 82, 65, 77, 32, 70, 79, 82, 32, 221, 162, - 24, 4, 68, 71, 69, 72, 214, 1, 42, 68, 98, 82, 201, 1, 3, 86, 89, 32, 6, - 44, 5, 83, 84, 79, 78, 69, 187, 200, 23, 80, 5, 217, 225, 2, 7, 32, 71, - 82, 65, 86, 69, 89, 12, 32, 2, 84, 32, 139, 151, 17, 45, 10, 60, 5, 87, - 73, 84, 72, 32, 246, 163, 12, 68, 155, 143, 9, 72, 6, 76, 9, 84, 73, 80, - 32, 79, 78, 32, 84, 72, 174, 242, 12, 82, 191, 228, 10, 65, 2, 223, 190, - 24, 69, 196, 1, 134, 2, 65, 202, 1, 66, 230, 2, 67, 154, 3, 68, 162, 1, - 69, 186, 3, 70, 94, 72, 62, 76, 222, 1, 77, 110, 79, 162, 1, 82, 142, 2, - 83, 228, 1, 3, 78, 79, 82, 198, 1, 84, 128, 2, 2, 85, 80, 174, 1, 87, - 138, 140, 14, 73, 254, 238, 3, 80, 210, 131, 4, 86, 251, 77, 71, 12, 108, - 17, 82, 82, 79, 87, 32, 83, 72, 65, 70, 84, 32, 87, 73, 68, 84, 72, 32, - 182, 158, 18, 77, 155, 188, 4, 83, 8, 36, 3, 79, 78, 69, 223, 141, 23, - 84, 7, 11, 32, 4, 174, 238, 13, 84, 135, 252, 8, 72, 14, 48, 4, 65, 76, - 76, 79, 21, 4, 76, 65, 67, 75, 2, 187, 167, 5, 84, 12, 30, 32, 153, 1, 2, - 45, 70, 6, 52, 7, 67, 85, 82, 86, 69, 68, 32, 247, 228, 23, 72, 4, 40, 4, - 68, 79, 87, 78, 1, 2, 85, 80, 2, 213, 203, 23, 8, 87, 65, 82, 68, 83, 32, - 65, 78, 6, 45, 9, 69, 65, 84, 72, 69, 82, 69, 68, 32, 6, 234, 211, 13, - 83, 218, 168, 3, 78, 211, 206, 6, 82, 14, 116, 2, 72, 69, 52, 5, 73, 82, - 67, 76, 69, 149, 220, 17, 14, 79, 78, 67, 65, 86, 69, 45, 80, 79, 73, 78, - 84, 69, 68, 4, 216, 203, 20, 4, 86, 82, 79, 78, 239, 218, 2, 67, 9, 64, - 6, 32, 87, 73, 84, 72, 32, 133, 212, 22, 4, 68, 32, 83, 65, 4, 52, 7, 83, - 84, 82, 79, 75, 69, 32, 199, 254, 4, 67, 2, 29, 5, 65, 78, 68, 32, 84, 2, - 11, 87, 2, 11, 79, 2, 21, 3, 32, 68, 79, 2, 11, 84, 2, 207, 234, 23, 83, - 14, 52, 7, 65, 83, 72, 69, 68, 32, 84, 18, 73, 31, 79, 2, 175, 16, 82, 2, - 137, 144, 19, 2, 86, 73, 10, 192, 16, 3, 87, 78, 87, 198, 139, 14, 85, - 187, 180, 4, 76, 16, 120, 5, 73, 71, 72, 84, 32, 156, 2, 16, 88, 67, 76, - 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, 147, 205, 18, 81, 10, - 40, 2, 80, 79, 126, 84, 231, 207, 22, 83, 6, 33, 6, 73, 78, 84, 69, 68, - 32, 6, 202, 186, 16, 80, 196, 147, 6, 11, 82, 69, 67, 84, 73, 76, 73, 78, - 69, 65, 82, 23, 66, 2, 141, 208, 22, 24, 69, 65, 82, 68, 82, 79, 80, 45, - 83, 80, 79, 75, 69, 68, 32, 80, 82, 79, 80, 69, 76, 76, 69, 82, 4, 246, - 207, 23, 83, 199, 79, 79, 6, 44, 5, 79, 85, 82, 32, 66, 139, 254, 4, 73, - 2, 209, 131, 11, 6, 65, 76, 76, 79, 79, 78, 4, 242, 186, 17, 79, 169, - 221, 5, 6, 69, 65, 82, 84, 32, 69, 20, 74, 65, 44, 3, 69, 70, 84, 28, 2, - 79, 87, 149, 249, 4, 3, 73, 71, 65, 4, 172, 207, 21, 2, 82, 71, 147, 208, - 1, 84, 8, 254, 3, 45, 195, 6, 87, 6, 26, 32, 223, 143, 10, 69, 4, 154, - 150, 14, 68, 25, 4, 83, 73, 78, 71, 4, 56, 8, 85, 76, 84, 73, 80, 76, 73, - 67, 235, 176, 21, 73, 2, 25, 4, 65, 84, 73, 79, 2, 207, 153, 5, 78, 6, - 132, 140, 6, 9, 80, 69, 78, 32, 67, 69, 78, 84, 82, 144, 175, 10, 13, 86, - 65, 76, 32, 87, 73, 84, 72, 32, 79, 86, 65, 76, 153, 141, 6, 5, 85, 84, - 76, 73, 78, 12, 76, 4, 73, 71, 72, 84, 245, 188, 23, 9, 79, 85, 78, 68, - 45, 84, 73, 80, 80, 10, 62, 45, 109, 11, 87, 65, 82, 68, 83, 32, 65, 82, - 82, 79, 87, 4, 69, 15, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 78, 71, - 76, 69, 32, 4, 222, 229, 6, 66, 147, 173, 7, 81, 7, 139, 6, 32, 26, 106, - 65, 78, 73, 44, 2, 79, 85, 160, 154, 14, 7, 67, 82, 73, 80, 84, 32, 76, - 157, 135, 1, 3, 80, 65, 82, 4, 236, 145, 14, 10, 78, 83, 45, 83, 69, 82, - 73, 70, 32, 73, 171, 180, 8, 76, 8, 216, 143, 14, 2, 78, 71, 179, 179, 8, - 88, 10, 21, 3, 84, 72, 32, 10, 60, 5, 69, 65, 83, 84, 32, 29, 6, 87, 69, - 83, 84, 32, 80, 6, 26, 80, 203, 190, 23, 65, 4, 41, 8, 79, 73, 78, 84, - 73, 78, 71, 32, 4, 242, 237, 16, 86, 139, 191, 6, 66, 12, 88, 9, 69, 65, - 82, 68, 82, 79, 80, 45, 83, 130, 1, 82, 241, 225, 6, 4, 87, 69, 76, 86, - 6, 64, 6, 80, 79, 75, 69, 68, 32, 241, 182, 23, 4, 72, 65, 78, 75, 4, - 164, 195, 22, 8, 80, 73, 78, 87, 72, 69, 69, 76, 27, 65, 2, 193, 3, 5, - 73, 65, 78, 71, 76, 6, 26, 87, 203, 133, 10, 80, 4, 53, 11, 65, 82, 68, - 83, 32, 65, 82, 82, 79, 87, 32, 4, 29, 5, 87, 73, 84, 72, 32, 4, 192, - 251, 19, 5, 76, 65, 82, 71, 69, 203, 238, 1, 69, 12, 76, 5, 72, 73, 84, - 69, 32, 164, 1, 2, 73, 68, 237, 236, 22, 3, 69, 68, 71, 8, 64, 6, 83, 81, - 85, 65, 82, 69, 166, 214, 17, 68, 215, 171, 5, 67, 5, 173, 161, 23, 19, - 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, - 86, 2, 157, 232, 20, 2, 69, 45, 140, 2, 112, 7, 65, 67, 67, 69, 78, 84, - 32, 138, 8, 76, 164, 14, 5, 77, 65, 82, 75, 32, 114, 80, 217, 157, 21, 2, - 89, 79, 60, 128, 2, 9, 65, 84, 78, 65, 72, 32, 72, 65, 70, 22, 68, 36, 3, - 71, 69, 82, 74, 77, 124, 2, 80, 65, 28, 4, 69, 84, 78, 65, 20, 2, 81, 65, - 58, 83, 58, 84, 156, 1, 2, 89, 69, 78, 90, 196, 165, 8, 3, 82, 69, 86, - 142, 129, 15, 79, 225, 146, 1, 3, 73, 76, 85, 2, 243, 157, 18, 85, 4, - 230, 128, 19, 69, 179, 140, 5, 65, 6, 32, 3, 69, 83, 72, 179, 28, 83, 5, - 129, 221, 6, 4, 32, 77, 85, 81, 8, 84, 5, 69, 82, 75, 72, 65, 164, 238, - 17, 2, 85, 78, 249, 45, 5, 65, 72, 65, 80, 65, 5, 209, 219, 19, 4, 32, - 75, 69, 70, 4, 26, 83, 251, 143, 24, 90, 2, 151, 169, 24, 72, 4, 128, - 137, 24, 6, 82, 78, 69, 89, 32, 80, 191, 35, 68, 4, 182, 24, 69, 193, - 213, 22, 6, 72, 65, 76, 83, 72, 69, 8, 38, 69, 249, 209, 22, 3, 73, 80, - 69, 6, 48, 6, 76, 73, 83, 72, 65, 32, 207, 224, 11, 86, 4, 240, 155, 22, - 3, 81, 69, 84, 249, 141, 2, 4, 71, 69, 68, 79, 4, 128, 205, 8, 10, 82, - 65, 72, 32, 66, 69, 78, 32, 89, 79, 255, 221, 2, 84, 8, 34, 65, 237, 145, - 24, 2, 73, 78, 6, 40, 4, 81, 69, 70, 32, 183, 255, 5, 82, 4, 154, 135, - 11, 81, 157, 176, 12, 3, 71, 65, 68, 150, 1, 76, 6, 69, 84, 84, 69, 82, - 32, 201, 11, 8, 73, 71, 65, 84, 85, 82, 69, 32, 140, 1, 134, 3, 65, 204, - 1, 3, 66, 69, 84, 0, 3, 75, 65, 70, 0, 2, 80, 69, 68, 6, 70, 73, 78, 65, - 76, 32, 92, 2, 81, 79, 16, 2, 72, 69, 52, 2, 78, 85, 0, 4, 90, 65, 89, - 73, 18, 83, 48, 3, 82, 69, 83, 170, 1, 84, 52, 4, 68, 65, 76, 69, 12, 5, - 71, 73, 77, 69, 76, 0, 5, 76, 65, 77, 69, 68, 0, 3, 77, 69, 77, 48, 3, - 86, 65, 86, 80, 5, 87, 73, 68, 69, 32, 153, 1, 3, 89, 79, 68, 14, 26, 76, - 171, 198, 23, 89, 12, 64, 2, 69, 70, 73, 10, 84, 69, 82, 78, 65, 84, 73, - 86, 69, 32, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 130, 13, 77, 130, 1, 80, - 35, 81, 4, 242, 201, 16, 65, 251, 160, 1, 80, 7, 33, 6, 32, 87, 73, 84, - 72, 32, 4, 142, 15, 82, 199, 225, 13, 68, 14, 88, 2, 75, 65, 236, 2, 2, - 80, 69, 220, 153, 1, 2, 84, 83, 182, 166, 22, 78, 135, 110, 77, 4, 235, - 2, 70, 7, 244, 10, 5, 32, 87, 73, 84, 72, 167, 184, 24, 84, 4, 167, 2, - 78, 16, 44, 4, 65, 77, 69, 75, 17, 3, 72, 73, 78, 4, 231, 1, 72, 13, 33, - 6, 32, 87, 73, 84, 72, 32, 10, 40, 6, 68, 65, 71, 69, 83, 72, 39, 83, 7, - 33, 6, 32, 65, 78, 68, 32, 83, 4, 254, 254, 5, 72, 203, 192, 2, 73, 12, - 50, 69, 12, 2, 65, 86, 1, 4, 83, 65, 68, 73, 4, 11, 84, 5, 233, 236, 13, - 7, 32, 87, 73, 84, 72, 32, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 148, - 253, 1, 2, 72, 79, 131, 239, 11, 68, 16, 174, 2, 76, 202, 203, 8, 65, - 210, 196, 2, 84, 186, 217, 2, 82, 160, 244, 8, 2, 68, 65, 242, 93, 75, - 222, 106, 72, 169, 4, 7, 70, 73, 78, 65, 76, 32, 77, 7, 33, 6, 32, 87, - 73, 84, 72, 32, 4, 176, 7, 2, 72, 73, 255, 226, 13, 68, 10, 72, 6, 65, - 76, 69, 70, 32, 76, 29, 8, 89, 73, 68, 68, 73, 83, 72, 32, 2, 193, 177, - 19, 2, 65, 77, 8, 120, 7, 68, 79, 85, 66, 76, 69, 32, 232, 4, 9, 89, 79, - 68, 32, 89, 79, 68, 32, 80, 185, 128, 21, 5, 86, 65, 86, 32, 89, 4, 254, - 142, 11, 86, 163, 246, 9, 89, 6, 80, 3, 76, 79, 87, 0, 3, 85, 80, 80, - 237, 232, 22, 6, 77, 65, 83, 79, 82, 65, 2, 233, 169, 23, 2, 69, 82, 50, - 84, 5, 79, 73, 78, 84, 32, 229, 5, 11, 85, 78, 67, 84, 85, 65, 84, 73, - 79, 78, 32, 38, 228, 1, 9, 68, 65, 71, 69, 83, 72, 32, 79, 82, 46, 72, - 106, 80, 166, 1, 81, 86, 82, 22, 83, 176, 209, 9, 2, 84, 83, 132, 142, - 11, 17, 74, 85, 68, 69, 79, 45, 83, 80, 65, 78, 73, 83, 72, 32, 86, 65, - 82, 233, 143, 3, 3, 77, 69, 84, 2, 17, 2, 32, 77, 2, 201, 1, 2, 65, 80, - 12, 60, 5, 65, 84, 65, 70, 32, 106, 73, 33, 4, 79, 76, 65, 77, 6, 38, 80, - 34, 81, 145, 2, 2, 83, 69, 2, 11, 65, 2, 131, 215, 17, 84, 2, 193, 162, - 23, 3, 65, 77, 65, 2, 11, 82, 2, 143, 230, 13, 73, 5, 181, 137, 11, 12, - 32, 72, 65, 83, 69, 82, 32, 70, 79, 82, 32, 86, 6, 52, 5, 65, 77, 65, 84, - 83, 189, 191, 11, 2, 85, 66, 5, 165, 242, 10, 2, 32, 81, 2, 167, 225, 6, - 65, 8, 34, 69, 22, 72, 243, 178, 8, 73, 2, 239, 161, 23, 71, 4, 238, 178, - 8, 73, 199, 209, 7, 69, 12, 152, 1, 3, 71, 69, 82, 60, 3, 80, 65, 83, 20, - 7, 83, 79, 70, 32, 80, 65, 83, 212, 249, 21, 7, 78, 85, 78, 32, 72, 65, - 70, 185, 183, 1, 3, 77, 65, 81, 4, 26, 83, 211, 202, 22, 69, 2, 157, 176, - 23, 3, 72, 65, 89, 2, 155, 226, 13, 69, 2, 135, 226, 13, 85, 8, 114, 77, - 160, 149, 12, 15, 76, 83, 67, 72, 82, 69, 73, 66, 69, 82, 32, 80, 65, 85, - 83, 185, 223, 5, 3, 73, 67, 79, 4, 176, 222, 5, 12, 69, 84, 32, 87, 73, - 84, 72, 32, 87, 72, 73, 84, 231, 191, 17, 32, 188, 4, 164, 1, 2, 65, 45, - 50, 72, 70, 75, 230, 1, 77, 114, 78, 146, 2, 82, 66, 83, 154, 1, 84, 226, - 1, 87, 62, 89, 110, 69, 134, 241, 8, 85, 190, 170, 5, 79, 143, 191, 3, - 73, 8, 190, 144, 24, 87, 246, 30, 49, 2, 50, 3, 51, 72, 166, 6, 79, 146, - 191, 8, 65, 146, 224, 5, 85, 158, 115, 69, 3, 73, 74, 72, 2, 65, 45, 104, - 2, 79, 45, 178, 4, 73, 226, 3, 69, 223, 142, 15, 85, 24, 210, 234, 11, - 49, 206, 172, 12, 75, 214, 22, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, - 56, 3, 57, 8, 178, 153, 24, 75, 218, 19, 49, 2, 50, 3, 51, 54, 68, 2, 69, - 45, 154, 7, 79, 222, 142, 15, 65, 2, 73, 243, 203, 2, 85, 6, 218, 169, - 24, 77, 186, 2, 49, 3, 50, 68, 116, 2, 69, 45, 72, 2, 73, 45, 246, 2, 65, - 194, 243, 8, 79, 190, 170, 5, 85, 169, 136, 6, 6, 45, 77, 85, 45, 77, 79, - 14, 254, 139, 24, 75, 246, 30, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 16, - 214, 147, 24, 84, 214, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, - 54, 222, 4, 79, 2, 85, 222, 142, 15, 73, 242, 203, 2, 65, 3, 69, 68, 62, - 65, 2, 85, 226, 3, 73, 134, 241, 8, 69, 219, 157, 6, 79, 16, 11, 45, 16, - 206, 168, 24, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 64, - 74, 69, 20, 2, 79, 45, 72, 2, 85, 45, 190, 144, 15, 73, 243, 203, 2, 65, - 18, 139, 240, 18, 45, 14, 234, 164, 24, 82, 186, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 3, 54, 10, 230, 135, 24, 84, 246, 30, 49, 2, 50, 2, 51, 3, 52, - 42, 170, 242, 8, 65, 2, 73, 218, 157, 6, 79, 243, 203, 2, 69, 32, 40, 2, - 65, 45, 66, 79, 207, 218, 17, 85, 12, 198, 134, 24, 89, 246, 30, 49, 2, - 50, 2, 51, 2, 52, 3, 53, 12, 11, 45, 12, 238, 164, 24, 49, 2, 50, 2, 51, - 2, 52, 2, 53, 3, 54, 4, 140, 167, 8, 21, 77, 73, 84, 73, 65, 78, 32, 67, - 79, 78, 74, 85, 71, 65, 84, 69, 32, 77, 65, 84, 82, 167, 253, 15, 66, - 128, 1, 210, 2, 65, 110, 66, 144, 1, 2, 67, 79, 94, 68, 198, 2, 70, 66, - 71, 40, 4, 72, 79, 76, 68, 164, 1, 2, 73, 78, 116, 2, 77, 79, 58, 79, - 122, 80, 100, 2, 82, 69, 62, 83, 238, 1, 84, 210, 5, 87, 212, 155, 15, 4, - 76, 73, 77, 73, 220, 180, 5, 11, 89, 79, 85, 84, 72, 70, 85, 76, 32, 70, - 79, 181, 173, 3, 9, 69, 78, 84, 72, 85, 83, 73, 65, 83, 6, 76, 3, 80, 80, - 82, 124, 4, 70, 84, 69, 82, 189, 151, 19, 4, 66, 85, 78, 68, 2, 233, 253, - 23, 2, 79, 65, 6, 92, 5, 69, 70, 79, 82, 69, 128, 215, 3, 6, 73, 84, 73, - 78, 71, 32, 1, 4, 82, 69, 65, 75, 2, 197, 206, 23, 7, 32, 67, 79, 77, 80, - 76, 69, 6, 26, 78, 183, 142, 19, 77, 4, 228, 187, 20, 4, 84, 69, 77, 80, - 245, 211, 2, 3, 70, 76, 73, 14, 110, 69, 78, 73, 206, 146, 19, 85, 129, - 241, 2, 15, 65, 82, 75, 69, 78, 73, 78, 71, 32, 79, 70, 32, 84, 72, 69, - 6, 222, 142, 19, 67, 192, 6, 2, 76, 73, 201, 196, 4, 5, 86, 69, 76, 79, - 80, 4, 252, 132, 19, 21, 70, 70, 73, 67, 85, 76, 84, 89, 32, 65, 84, 32, - 84, 72, 69, 32, 66, 69, 71, 73, 78, 177, 248, 2, 4, 83, 80, 69, 82, 4, - 220, 137, 19, 2, 79, 76, 217, 188, 4, 5, 69, 76, 76, 79, 87, 12, 36, 5, - 65, 84, 72, 69, 82, 35, 82, 2, 225, 134, 15, 3, 73, 78, 71, 10, 40, 4, - 69, 65, 84, 32, 155, 221, 23, 65, 8, 22, 80, 211, 5, 84, 6, 22, 79, 147, - 5, 82, 4, 172, 2, 2, 83, 83, 203, 217, 23, 87, 8, 50, 78, 206, 138, 19, - 67, 217, 5, 3, 70, 76, 85, 4, 160, 144, 19, 2, 79, 67, 205, 231, 1, 5, - 69, 82, 32, 84, 82, 4, 204, 171, 22, 3, 68, 69, 83, 189, 61, 3, 85, 84, - 72, 6, 26, 66, 37, 2, 80, 80, 2, 149, 199, 23, 4, 83, 84, 82, 85, 4, 26, - 82, 239, 197, 23, 79, 2, 145, 248, 21, 2, 69, 83, 6, 196, 139, 15, 9, 85, - 83, 72, 73, 78, 71, 32, 85, 80, 204, 137, 2, 3, 82, 79, 71, 255, 196, 6, - 69, 6, 26, 84, 207, 250, 18, 86, 4, 254, 139, 19, 85, 223, 57, 82, 8, - 132, 1, 5, 77, 65, 76, 76, 32, 236, 156, 21, 6, 84, 65, 78, 68, 83, 84, - 233, 240, 1, 11, 80, 76, 73, 84, 84, 73, 78, 71, 32, 65, 80, 4, 24, 2, - 80, 82, 43, 84, 2, 225, 140, 19, 5, 69, 80, 79, 78, 68, 2, 11, 65, 2, - 155, 195, 23, 77, 30, 36, 3, 72, 69, 32, 175, 219, 17, 82, 28, 186, 2, - 65, 130, 1, 67, 132, 1, 3, 70, 65, 77, 20, 9, 82, 69, 67, 69, 80, 84, 73, - 86, 69, 30, 87, 172, 22, 12, 77, 65, 82, 82, 89, 73, 78, 71, 32, 77, 65, - 73, 168, 147, 5, 6, 71, 69, 78, 84, 76, 69, 248, 221, 10, 13, 75, 69, 69, - 80, 73, 78, 71, 32, 83, 84, 73, 76, 76, 201, 233, 3, 7, 74, 79, 89, 79, - 85, 83, 32, 6, 58, 82, 133, 181, 11, 8, 66, 89, 83, 77, 65, 76, 32, 87, - 4, 248, 168, 20, 8, 79, 85, 83, 73, 78, 71, 32, 84, 171, 214, 3, 77, 6, - 128, 131, 1, 2, 65, 85, 212, 164, 19, 9, 82, 69, 65, 84, 73, 86, 69, 32, - 72, 133, 212, 1, 9, 76, 73, 78, 71, 73, 78, 71, 32, 70, 2, 243, 235, 23, - 73, 2, 157, 166, 20, 2, 32, 69, 4, 178, 135, 22, 69, 213, 201, 1, 5, 65, - 78, 68, 69, 82, 4, 202, 231, 6, 65, 141, 166, 6, 14, 79, 82, 75, 32, 79, - 78, 32, 84, 72, 69, 32, 68, 69, 67, 222, 1, 236, 1, 2, 71, 72, 180, 2, 7, - 82, 65, 71, 65, 78, 65, 32, 212, 4, 7, 83, 84, 79, 82, 73, 67, 32, 168, - 186, 15, 7, 78, 68, 85, 32, 84, 69, 77, 240, 33, 4, 75, 73, 78, 71, 162, - 250, 1, 66, 177, 168, 4, 8, 80, 80, 79, 80, 79, 84, 65, 77, 12, 18, 32, - 119, 45, 6, 204, 135, 5, 2, 66, 82, 220, 231, 16, 6, 86, 79, 76, 84, 65, - 71, 229, 48, 9, 79, 67, 84, 69, 84, 32, 80, 82, 69, 6, 92, 11, 83, 80, - 69, 69, 68, 32, 84, 82, 65, 73, 78, 157, 200, 5, 6, 72, 69, 69, 76, 69, - 68, 5, 129, 186, 11, 14, 32, 87, 73, 84, 72, 32, 66, 85, 76, 76, 69, 84, - 32, 78, 200, 1, 112, 9, 68, 73, 71, 82, 65, 80, 72, 32, 89, 20, 7, 76, - 69, 84, 84, 69, 82, 32, 222, 172, 1, 86, 139, 185, 20, 73, 2, 243, 170, - 17, 79, 194, 1, 194, 1, 65, 74, 83, 206, 163, 1, 66, 162, 3, 78, 150, 2, - 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 80, 2, 82, 2, 84, 2, 90, 126, 87, 46, - 89, 142, 183, 22, 86, 234, 36, 69, 2, 73, 2, 79, 3, 85, 7, 37, 7, 82, 67, - 72, 65, 73, 67, 32, 4, 210, 225, 23, 87, 151, 14, 89, 42, 76, 5, 77, 65, - 76, 76, 32, 206, 133, 24, 65, 2, 69, 2, 73, 2, 79, 3, 85, 32, 170, 169, - 1, 87, 46, 89, 170, 173, 5, 75, 230, 137, 17, 84, 234, 36, 65, 2, 69, 2, - 73, 2, 79, 3, 85, 2, 139, 179, 8, 83, 108, 166, 1, 76, 52, 3, 78, 69, 89, - 46, 82, 146, 7, 84, 138, 1, 85, 154, 188, 16, 83, 246, 213, 2, 67, 156, - 192, 3, 6, 77, 79, 84, 72, 69, 84, 242, 164, 1, 79, 155, 3, 80, 6, 176, - 140, 16, 4, 76, 79, 87, 32, 247, 246, 7, 69, 4, 186, 147, 22, 66, 129, - 159, 1, 2, 32, 80, 66, 60, 8, 73, 90, 79, 78, 84, 65, 76, 32, 153, 6, 2, - 83, 69, 60, 218, 1, 66, 110, 76, 152, 1, 17, 79, 78, 69, 32, 69, 73, 71, - 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, 88, 10, 83, 67, 65, 78, 32, 76, - 73, 78, 69, 45, 46, 84, 162, 217, 21, 67, 42, 69, 238, 6, 77, 150, 1, 82, - 247, 2, 90, 6, 44, 5, 76, 65, 67, 75, 32, 163, 199, 23, 65, 4, 38, 79, - 169, 199, 22, 3, 72, 69, 88, 2, 155, 199, 22, 67, 10, 40, 4, 73, 78, 69, - 32, 135, 223, 21, 65, 8, 44, 5, 87, 73, 84, 72, 32, 171, 223, 21, 69, 6, - 26, 84, 219, 224, 21, 70, 4, 250, 224, 21, 72, 171, 90, 73, 14, 136, 225, - 16, 3, 49, 51, 53, 158, 157, 7, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 8, - 206, 253, 23, 49, 2, 51, 2, 55, 3, 57, 10, 32, 2, 65, 66, 223, 227, 21, - 82, 8, 26, 85, 223, 226, 21, 32, 6, 33, 6, 76, 65, 84, 73, 79, 78, 7, 11, - 32, 4, 188, 197, 9, 8, 87, 73, 84, 72, 32, 74, 85, 83, 223, 129, 14, 83, - 7, 11, 32, 4, 180, 191, 7, 2, 82, 65, 167, 255, 15, 70, 10, 26, 32, 235, - 241, 22, 69, 8, 86, 80, 240, 243, 18, 5, 66, 69, 86, 69, 82, 160, 80, 3, - 83, 80, 82, 215, 181, 4, 68, 2, 167, 230, 14, 69, 12, 48, 6, 82, 71, 76, - 65, 83, 83, 77, 2, 83, 69, 5, 249, 170, 20, 14, 32, 87, 73, 84, 72, 32, - 70, 76, 79, 87, 73, 78, 71, 32, 9, 11, 32, 6, 88, 8, 87, 73, 84, 72, 32, - 71, 65, 82, 217, 241, 21, 8, 66, 85, 73, 76, 68, 73, 78, 71, 2, 223, 164, - 22, 68, 7, 178, 248, 23, 74, 3, 83, 8, 106, 83, 248, 227, 22, 11, 78, 68, - 82, 69, 68, 32, 80, 79, 73, 78, 84, 148, 11, 2, 71, 71, 163, 136, 1, 84, - 2, 211, 230, 22, 72, 18, 90, 80, 224, 1, 5, 83, 84, 69, 82, 69, 236, 238, - 16, 2, 71, 73, 229, 130, 5, 2, 65, 67, 12, 60, 3, 72, 69, 78, 245, 158, - 4, 6, 79, 68, 73, 65, 83, 84, 11, 34, 32, 82, 65, 231, 238, 19, 45, 4, - 26, 87, 135, 150, 22, 66, 2, 21, 3, 73, 84, 72, 2, 185, 188, 3, 2, 32, - 68, 2, 177, 220, 12, 6, 84, 73, 79, 78, 32, 80, 2, 165, 225, 22, 2, 83, - 73, 206, 5, 160, 1, 3, 67, 69, 32, 152, 1, 2, 68, 69, 222, 24, 77, 222, - 2, 78, 200, 37, 7, 90, 65, 75, 65, 89, 65, 32, 129, 228, 19, 9, 32, 76, - 79, 86, 69, 32, 89, 79, 85, 8, 114, 67, 232, 161, 11, 18, 72, 79, 67, 75, - 69, 89, 32, 83, 84, 73, 67, 75, 32, 65, 78, 68, 32, 80, 159, 190, 1, 83, - 4, 230, 239, 15, 82, 163, 226, 2, 85, 246, 1, 68, 3, 78, 84, 73, 201, 1, - 9, 79, 71, 82, 65, 80, 72, 73, 67, 32, 8, 64, 4, 67, 65, 76, 32, 105, 8, - 70, 73, 67, 65, 84, 73, 79, 78, 6, 32, 2, 84, 79, 215, 239, 22, 87, 5, - 237, 242, 14, 12, 32, 65, 78, 68, 32, 83, 76, 65, 78, 84, 69, 68, 2, 141, - 217, 22, 2, 32, 67, 238, 1, 208, 2, 11, 65, 78, 78, 79, 84, 65, 84, 73, - 79, 78, 32, 242, 3, 67, 72, 2, 68, 69, 248, 5, 5, 78, 85, 77, 66, 69, 22, - 84, 128, 237, 5, 8, 72, 65, 76, 70, 32, 70, 73, 76, 148, 186, 1, 5, 69, - 78, 84, 69, 82, 0, 3, 82, 73, 83, 24, 5, 76, 69, 86, 69, 76, 140, 138, 9, - 5, 86, 65, 82, 73, 65, 182, 223, 4, 70, 158, 47, 73, 159, 228, 1, 83, 32, - 232, 1, 5, 66, 79, 84, 84, 79, 22, 70, 82, 77, 62, 84, 140, 1, 4, 76, 73, - 78, 75, 160, 143, 1, 4, 83, 69, 67, 79, 218, 158, 6, 79, 144, 134, 6, 6, - 82, 69, 86, 69, 82, 83, 240, 203, 9, 5, 72, 69, 65, 86, 69, 169, 39, 3, - 69, 65, 82, 2, 227, 169, 23, 77, 6, 48, 3, 79, 85, 82, 153, 160, 23, 3, - 73, 82, 83, 4, 142, 169, 23, 84, 27, 32, 4, 36, 3, 73, 68, 68, 155, 129, - 23, 65, 2, 167, 181, 13, 76, 8, 34, 72, 46, 87, 203, 130, 8, 79, 4, 194, - 142, 9, 82, 133, 143, 14, 2, 73, 82, 2, 243, 167, 23, 79, 4, 36, 3, 76, - 79, 83, 135, 151, 21, 79, 2, 181, 167, 23, 3, 73, 78, 71, 36, 104, 20, - 83, 67, 82, 73, 80, 84, 73, 79, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, - 82, 32, 159, 172, 7, 80, 34, 144, 2, 9, 65, 66, 79, 86, 69, 32, 84, 79, - 32, 84, 8, 76, 69, 70, 84, 32, 84, 79, 32, 76, 5, 79, 86, 69, 82, 76, 20, - 2, 83, 85, 182, 241, 14, 82, 252, 189, 5, 8, 70, 85, 76, 76, 32, 83, 85, - 82, 145, 228, 2, 15, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 69, - 70, 76, 4, 60, 9, 77, 73, 68, 68, 76, 69, 32, 65, 78, 143, 199, 21, 66, - 2, 215, 183, 21, 68, 4, 128, 148, 22, 10, 77, 73, 68, 68, 76, 69, 32, 65, - 78, 68, 183, 166, 1, 82, 2, 247, 220, 16, 65, 18, 72, 12, 82, 82, 79, 85, - 78, 68, 32, 70, 82, 79, 77, 32, 147, 178, 17, 66, 16, 82, 76, 220, 203, - 11, 3, 85, 80, 80, 158, 249, 9, 66, 166, 161, 1, 65, 159, 82, 82, 6, 206, - 203, 11, 79, 219, 236, 11, 69, 2, 235, 213, 8, 82, 148, 1, 92, 10, 65, - 76, 76, 89, 32, 77, 65, 82, 75, 32, 41, 9, 69, 76, 69, 71, 82, 65, 80, - 72, 32, 10, 198, 241, 21, 70, 70, 84, 183, 86, 79, 138, 1, 140, 1, 11, - 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 141, 231, 12, 17, 76, 73, 78, - 69, 32, 70, 69, 69, 68, 32, 83, 69, 80, 65, 82, 65, 84, 136, 1, 182, 1, - 65, 58, 68, 200, 2, 5, 72, 79, 85, 82, 32, 214, 1, 74, 28, 4, 70, 69, 66, - 82, 64, 2, 77, 65, 148, 192, 15, 3, 78, 79, 86, 0, 4, 83, 69, 80, 84, 25, - 4, 79, 67, 84, 79, 4, 244, 244, 19, 3, 85, 71, 85, 169, 182, 1, 2, 80, - 82, 64, 44, 3, 65, 89, 32, 221, 196, 15, 2, 69, 67, 62, 66, 84, 162, 209, - 5, 69, 66, 70, 70, 78, 26, 83, 227, 241, 16, 79, 34, 34, 72, 90, 87, 239, - 140, 23, 69, 8, 36, 3, 73, 82, 84, 199, 237, 21, 82, 6, 26, 89, 251, 136, - 22, 69, 5, 155, 178, 22, 45, 24, 26, 69, 179, 220, 23, 79, 22, 36, 3, 78, - 84, 89, 183, 223, 22, 76, 21, 247, 158, 15, 45, 50, 78, 84, 254, 206, 5, - 69, 66, 70, 70, 78, 26, 83, 234, 155, 16, 90, 251, 85, 79, 20, 42, 87, - 218, 208, 5, 72, 195, 186, 17, 69, 14, 26, 69, 223, 218, 23, 79, 12, 36, - 3, 78, 84, 89, 227, 221, 22, 76, 11, 143, 154, 17, 45, 6, 24, 2, 65, 78, - 35, 85, 2, 11, 85, 2, 243, 162, 17, 65, 4, 142, 195, 23, 78, 143, 5, 76, - 4, 154, 183, 23, 82, 171, 34, 89, 68, 40, 6, 65, 71, 69, 32, 79, 70, 83, - 80, 5, 221, 131, 9, 15, 32, 79, 82, 32, 65, 80, 80, 82, 79, 88, 73, 77, - 65, 84, 69, 65, 65, 14, 69, 82, 73, 65, 76, 32, 65, 82, 65, 77, 65, 73, - 67, 32, 62, 72, 7, 78, 85, 77, 66, 69, 82, 32, 230, 18, 76, 221, 195, 19, - 2, 83, 69, 16, 26, 84, 167, 195, 15, 79, 10, 162, 183, 11, 87, 138, 148, - 1, 69, 143, 156, 9, 72, 136, 3, 202, 1, 67, 234, 1, 68, 214, 6, 70, 132, - 2, 5, 72, 73, 66, 73, 84, 156, 1, 15, 80, 85, 84, 32, 83, 89, 77, 66, 79, - 76, 32, 70, 79, 82, 32, 206, 1, 83, 236, 5, 2, 84, 69, 206, 8, 86, 251, - 250, 9, 66, 10, 32, 2, 79, 77, 69, 2, 82, 69, 4, 240, 202, 16, 3, 73, 78, - 71, 157, 225, 2, 5, 80, 76, 69, 84, 69, 6, 36, 3, 65, 83, 69, 195, 144, - 23, 77, 4, 34, 32, 185, 181, 10, 2, 83, 32, 2, 173, 183, 11, 8, 70, 79, - 78, 84, 32, 83, 73, 90, 145, 1, 24, 2, 69, 88, 103, 73, 5, 237, 185, 22, - 20, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 84, 32, 84, 72, 69, 32, - 86, 73, 69, 138, 1, 72, 8, 67, 32, 83, 73, 89, 65, 81, 32, 249, 254, 17, - 4, 65, 78, 32, 82, 136, 1, 196, 1, 11, 65, 76, 84, 69, 82, 78, 65, 84, - 69, 32, 76, 2, 76, 28, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 100, 7, 78, - 85, 77, 66, 69, 82, 32, 190, 243, 8, 82, 149, 125, 6, 80, 76, 65, 67, 69, - 72, 2, 157, 142, 23, 2, 65, 75, 6, 68, 4, 79, 78, 69, 32, 137, 206, 21, - 7, 84, 72, 82, 69, 69, 32, 81, 4, 222, 203, 21, 72, 47, 81, 122, 212, 1, - 10, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 5, 75, 65, 82, 79, 82, 0, - 4, 76, 65, 75, 72, 198, 138, 10, 69, 66, 70, 94, 78, 26, 83, 78, 84, 240, - 130, 5, 8, 80, 82, 69, 70, 73, 88, 69, 68, 199, 41, 79, 6, 26, 84, 211, - 179, 22, 79, 4, 244, 160, 6, 2, 69, 78, 147, 141, 17, 87, 5, 239, 252, - 22, 65, 16, 72, 5, 73, 78, 73, 84, 89, 85, 9, 79, 82, 77, 65, 84, 73, 79, - 78, 32, 5, 45, 9, 32, 78, 69, 71, 65, 84, 69, 68, 32, 2, 157, 168, 8, 4, - 87, 73, 84, 72, 12, 42, 83, 137, 203, 19, 4, 68, 69, 83, 75, 10, 192, - 149, 7, 5, 69, 80, 65, 82, 65, 139, 252, 9, 79, 4, 11, 32, 4, 152, 249, - 22, 15, 65, 82, 65, 66, 73, 67, 32, 70, 79, 82, 77, 32, 83, 72, 65, 1, + 220, 147, 4, 3, 67, 72, 73, 201, 144, 23, 3, 89, 73, 90, 6, 48, 4, 69, + 82, 69, 84, 229, 208, 26, 2, 73, 70, 5, 17, 2, 45, 72, 2, 229, 164, 27, + 2, 73, 68, 4, 176, 208, 26, 2, 69, 78, 163, 2, 85, 2, 137, 162, 4, 3, 73, + 75, 82, 12, 96, 2, 82, 79, 192, 159, 11, 3, 78, 79, 77, 129, 173, 15, 9, + 76, 69, 82, 32, 67, 79, 78, 83, 84, 8, 92, 5, 80, 69, 65, 78, 32, 144, + 161, 24, 8, 45, 67, 85, 82, 82, 69, 78, 67, 215, 175, 2, 32, 4, 202, 142, + 4, 67, 19, 80, 50, 30, 67, 102, 80, 187, 1, 84, 6, 60, 9, 76, 65, 77, 65, + 84, 73, 79, 78, 32, 139, 190, 26, 69, 4, 246, 129, 25, 81, 187, 147, 2, + 77, 10, 96, 7, 76, 79, 83, 73, 79, 78, 32, 181, 152, 27, 11, 82, 69, 83, + 83, 73, 79, 78, 76, 69, 83, 83, 8, 212, 223, 11, 4, 70, 82, 65, 77, 145, + 166, 15, 8, 65, 84, 32, 72, 79, 82, 73, 90, 34, 98, 82, 233, 197, 21, 18, + 69, 78, 68, 69, 68, 32, 65, 82, 65, 66, 73, 67, 45, 73, 78, 68, 73, 67, + 14, 140, 1, 12, 69, 77, 69, 76, 89, 32, 72, 69, 65, 86, 89, 32, 137, 253, + 25, 16, 65, 84, 69, 82, 82, 69, 83, 84, 82, 73, 65, 76, 32, 65, 76, 73, + 12, 50, 83, 134, 185, 25, 70, 234, 2, 87, 203, 11, 71, 4, 150, 186, 25, + 65, 43, 73, 7, 250, 186, 19, 71, 195, 151, 8, 83, 152, 4, 142, 1, 65, + 130, 19, 69, 190, 1, 73, 134, 8, 76, 138, 6, 79, 142, 7, 82, 130, 5, 85, + 252, 201, 20, 2, 86, 83, 242, 203, 4, 83, 199, 140, 2, 70, 92, 122, 67, + 178, 15, 76, 176, 2, 2, 88, 32, 170, 148, 3, 77, 252, 253, 5, 2, 82, 83, + 140, 251, 1, 2, 84, 72, 151, 235, 9, 73, 70, 72, 2, 69, 32, 180, 187, 16, + 4, 83, 73, 77, 73, 165, 206, 4, 2, 84, 79, 66, 226, 1, 83, 160, 1, 4, 87, + 73, 84, 72, 196, 136, 13, 2, 80, 65, 252, 136, 2, 2, 77, 65, 164, 162, + 11, 13, 84, 72, 82, 79, 87, 73, 78, 71, 32, 65, 32, 75, 73, 201, 1, 14, + 72, 79, 76, 68, 73, 78, 71, 32, 66, 65, 67, 75, 32, 84, 4, 216, 138, 17, + 18, 65, 86, 79, 85, 82, 73, 78, 71, 32, 68, 69, 76, 73, 67, 73, 79, 85, + 83, 141, 138, 5, 13, 67, 82, 69, 65, 77, 73, 78, 71, 32, 73, 78, 32, 70, + 54, 38, 32, 141, 167, 24, 3, 79, 85, 84, 52, 196, 4, 2, 67, 79, 44, 5, + 72, 69, 65, 68, 45, 38, 77, 98, 79, 92, 7, 78, 79, 32, 71, 79, 79, 68, + 238, 1, 80, 164, 1, 16, 83, 84, 85, 67, 75, 45, 79, 85, 84, 32, 84, 79, + 78, 71, 85, 69, 110, 84, 204, 183, 1, 9, 66, 65, 71, 83, 32, 85, 78, 68, + 69, 212, 171, 17, 22, 70, 73, 78, 71, 69, 82, 32, 67, 79, 86, 69, 82, 73, + 78, 71, 32, 67, 76, 79, 83, 69, 68, 244, 67, 3, 82, 79, 76, 180, 231, 1, + 13, 76, 79, 79, 75, 32, 79, 70, 32, 84, 82, 73, 85, 77, 244, 141, 3, 8, + 68, 73, 65, 71, 79, 78, 65, 76, 1, 20, 85, 78, 69, 86, 69, 78, 32, 69, + 89, 69, 83, 32, 65, 78, 68, 32, 87, 65, 86, 89, 4, 252, 4, 3, 87, 66, 79, + 227, 167, 19, 76, 2, 249, 183, 22, 4, 66, 65, 78, 68, 4, 60, 6, 69, 68, + 73, 67, 65, 76, 185, 242, 25, 3, 79, 78, 79, 2, 181, 255, 25, 3, 32, 77, + 65, 12, 90, 75, 36, 4, 80, 69, 78, 32, 213, 180, 20, 10, 78, 69, 32, 69, + 89, 69, 66, 82, 79, 87, 2, 241, 166, 22, 4, 32, 71, 69, 83, 8, 116, 5, + 77, 79, 85, 84, 72, 157, 159, 24, 18, 69, 89, 69, 83, 32, 65, 78, 68, 32, + 72, 65, 78, 68, 32, 79, 86, 69, 82, 7, 11, 32, 4, 184, 137, 10, 3, 86, + 79, 77, 221, 159, 9, 5, 65, 78, 68, 32, 67, 6, 132, 1, 18, 65, 82, 84, + 89, 32, 72, 79, 82, 78, 32, 65, 78, 68, 32, 80, 65, 82, 84, 100, 2, 69, + 69, 197, 167, 19, 4, 76, 69, 65, 68, 2, 229, 208, 22, 2, 89, 32, 7, 29, + 5, 32, 65, 78, 68, 32, 4, 36, 3, 87, 73, 78, 231, 167, 19, 84, 2, 169, + 183, 1, 4, 75, 73, 78, 71, 4, 50, 69, 193, 212, 22, 6, 72, 69, 82, 77, + 79, 77, 2, 181, 175, 27, 8, 65, 82, 83, 32, 79, 70, 32, 74, 10, 34, 76, + 133, 171, 22, 2, 65, 70, 8, 84, 13, 73, 78, 71, 32, 68, 73, 65, 71, 79, + 78, 65, 76, 32, 129, 242, 23, 2, 69, 78, 6, 128, 1, 9, 67, 82, 79, 83, + 83, 73, 78, 71, 32, 201, 186, 19, 16, 73, 78, 32, 87, 72, 73, 84, 69, 32, + 67, 73, 82, 67, 76, 69, 32, 4, 164, 148, 16, 3, 82, 73, 83, 227, 173, 3, + 78, 4, 218, 130, 15, 73, 131, 143, 4, 77, 14, 50, 65, 50, 77, 44, 2, 82, + 82, 147, 236, 18, 78, 4, 232, 174, 8, 3, 82, 70, 85, 147, 237, 9, 84, 4, + 228, 246, 8, 2, 73, 78, 179, 178, 7, 65, 4, 220, 246, 7, 2, 73, 83, 151, + 198, 19, 89, 58, 138, 1, 71, 90, 76, 178, 1, 78, 98, 82, 182, 2, 83, 164, + 14, 4, 86, 69, 32, 68, 217, 255, 4, 10, 69, 76, 68, 32, 72, 79, 67, 75, + 69, 89, 6, 48, 4, 85, 82, 69, 32, 245, 140, 26, 2, 72, 84, 4, 242, 207, + 25, 68, 235, 172, 1, 83, 10, 32, 2, 69, 32, 65, 2, 77, 32, 6, 242, 200, + 13, 70, 140, 155, 2, 3, 67, 65, 66, 147, 165, 8, 83, 4, 52, 4, 80, 82, + 79, 74, 173, 161, 20, 3, 70, 82, 65, 2, 251, 217, 15, 69, 4, 72, 4, 71, + 69, 82, 80, 221, 174, 25, 8, 73, 84, 69, 32, 80, 65, 82, 84, 2, 195, 141, + 16, 82, 22, 34, 69, 189, 1, 3, 83, 84, 32, 13, 56, 2, 32, 69, 68, 4, 87, + 79, 82, 75, 251, 184, 15, 67, 4, 222, 192, 13, 78, 193, 213, 4, 8, 88, + 84, 73, 78, 71, 85, 73, 83, 4, 216, 201, 23, 6, 32, 83, 80, 65, 82, 75, + 215, 237, 3, 83, 10, 186, 177, 5, 81, 172, 184, 10, 8, 83, 84, 82, 79, + 78, 71, 32, 73, 223, 225, 6, 80, 10, 38, 72, 165, 229, 23, 3, 84, 69, 68, + 9, 128, 255, 3, 13, 73, 78, 71, 32, 80, 79, 76, 69, 32, 65, 78, 68, 32, + 174, 209, 5, 69, 213, 222, 16, 19, 32, 67, 65, 75, 69, 32, 87, 73, 84, + 72, 32, 83, 87, 73, 82, 76, 32, 68, 69, 46, 50, 65, 170, 1, 69, 98, 79, + 194, 1, 85, 39, 89, 12, 114, 84, 236, 163, 4, 5, 80, 80, 73, 78, 71, 192, + 159, 3, 6, 71, 32, 73, 78, 32, 72, 161, 224, 17, 3, 77, 73, 78, 6, 190, + 221, 8, 32, 250, 203, 11, 66, 135, 241, 5, 78, 4, 196, 250, 23, 8, 88, + 69, 68, 32, 66, 73, 67, 69, 153, 145, 1, 7, 85, 82, 45, 68, 69, 45, 76, + 12, 52, 2, 82, 65, 20, 3, 87, 69, 82, 139, 234, 25, 80, 5, 147, 170, 26, + 76, 7, 17, 2, 32, 80, 4, 58, 85, 209, 207, 24, 8, 76, 65, 89, 73, 78, 71, + 32, 67, 2, 181, 199, 26, 4, 78, 67, 84, 85, 4, 190, 158, 3, 83, 183, 251, + 23, 84, 15, 25, 4, 73, 78, 71, 32, 12, 64, 6, 83, 65, 85, 67, 69, 82, + 210, 143, 10, 68, 155, 136, 10, 69, 9, 11, 32, 6, 40, 4, 87, 73, 84, 72, + 227, 153, 26, 83, 4, 34, 32, 1, 4, 79, 85, 84, 32, 2, 21, 3, 66, 69, 65, + 2, 243, 240, 26, 77, 56, 102, 71, 20, 2, 76, 68, 68, 2, 79, 84, 22, 82, + 142, 2, 85, 128, 155, 23, 2, 78, 68, 191, 210, 3, 88, 5, 139, 156, 27, + 71, 4, 156, 205, 9, 8, 73, 78, 71, 32, 72, 65, 78, 68, 199, 167, 17, 69, + 5, 195, 161, 14, 80, 18, 78, 75, 132, 1, 3, 84, 85, 78, 134, 224, 20, 77, + 222, 192, 4, 32, 179, 116, 67, 8, 80, 10, 32, 65, 78, 68, 32, 75, 78, 73, + 70, 69, 242, 164, 15, 69, 223, 181, 11, 73, 5, 201, 222, 15, 7, 32, 87, + 73, 84, 72, 32, 80, 4, 224, 143, 27, 6, 69, 32, 67, 79, 79, 75, 179, 27, + 65, 22, 26, 82, 199, 190, 23, 78, 20, 38, 32, 218, 2, 84, 223, 239, 18, + 45, 16, 110, 67, 174, 1, 68, 182, 170, 2, 66, 216, 155, 10, 7, 76, 69, + 65, 70, 32, 67, 76, 186, 111, 84, 235, 215, 11, 80, 4, 148, 183, 13, 3, + 76, 85, 66, 181, 86, 32, 79, 82, 78, 69, 82, 32, 65, 82, 82, 79, 87, 83, + 32, 67, 73, 82, 67, 76, 73, 78, 71, 32, 65, 78, 84, 73, 67, 76, 79, 67, + 75, 87, 4, 21, 3, 79, 84, 32, 4, 246, 225, 23, 80, 195, 132, 3, 77, 2, + 231, 43, 72, 30, 78, 65, 250, 1, 69, 98, 79, 133, 131, 17, 8, 73, 69, 68, + 32, 83, 72, 82, 73, 12, 96, 6, 67, 84, 73, 79, 78, 32, 68, 8, 77, 69, 32, + 87, 73, 84, 72, 32, 193, 242, 10, 2, 71, 73, 4, 166, 129, 20, 83, 177, 6, + 9, 78, 85, 77, 69, 82, 65, 84, 79, 82, 6, 50, 80, 218, 170, 2, 65, 149, + 185, 21, 2, 84, 73, 2, 185, 134, 22, 2, 73, 67, 6, 48, 6, 78, 67, 72, 32, + 70, 82, 139, 142, 19, 69, 4, 174, 142, 26, 73, 133, 15, 3, 65, 78, 67, + 10, 52, 3, 78, 84, 45, 116, 2, 87, 78, 191, 229, 26, 71, 4, 70, 84, 217, + 143, 2, 11, 70, 65, 67, 73, 78, 71, 32, 66, 65, 66, 89, 2, 189, 191, 12, + 6, 73, 76, 84, 69, 68, 32, 5, 185, 147, 8, 6, 73, 78, 71, 32, 70, 65, + 226, 1, 80, 2, 76, 76, 134, 6, 78, 208, 250, 16, 5, 69, 76, 32, 80, 85, + 179, 138, 10, 83, 216, 1, 42, 32, 73, 6, 87, 73, 68, 84, 72, 32, 10, 234, + 140, 12, 77, 136, 171, 3, 2, 79, 85, 170, 247, 8, 66, 151, 29, 83, 206, + 1, 242, 1, 67, 42, 76, 78, 78, 30, 80, 66, 82, 142, 1, 83, 38, 89, 130, + 159, 10, 77, 174, 213, 10, 65, 158, 2, 68, 58, 69, 98, 71, 118, 72, 202, + 4, 81, 198, 153, 2, 87, 182, 29, 84, 162, 146, 1, 70, 224, 177, 1, 6, 66, + 82, 79, 75, 69, 78, 211, 7, 86, 10, 162, 248, 20, 73, 62, 79, 131, 81, + 69, 116, 42, 69, 214, 251, 20, 65, 231, 183, 4, 79, 10, 162, 1, 70, 199, + 253, 20, 83, 4, 166, 210, 21, 79, 35, 85, 6, 42, 79, 198, 254, 20, 69, + 211, 236, 2, 76, 2, 223, 177, 25, 85, 10, 36, 3, 73, 71, 72, 207, 131, + 25, 69, 8, 17, 2, 84, 32, 8, 140, 181, 20, 5, 87, 72, 73, 84, 69, 246, + 220, 2, 67, 210, 3, 80, 239, 7, 83, 4, 250, 204, 23, 69, 139, 184, 1, 79, + 2, 223, 159, 25, 69, 6, 128, 205, 11, 8, 67, 84, 73, 79, 78, 32, 65, 80, + 208, 252, 8, 5, 69, 82, 65, 76, 32, 211, 188, 1, 78, 130, 21, 114, 65, + 250, 6, 69, 222, 22, 73, 162, 1, 76, 134, 15, 79, 198, 6, 82, 178, 92, + 85, 138, 155, 22, 72, 131, 238, 3, 83, 142, 1, 42, 82, 149, 254, 26, 4, + 77, 69, 32, 68, 140, 1, 36, 3, 65, 89, 32, 255, 238, 25, 76, 138, 1, 166, + 1, 67, 210, 1, 83, 192, 2, 6, 86, 79, 87, 69, 76, 32, 172, 178, 8, 6, 82, + 69, 68, 85, 80, 76, 194, 208, 10, 72, 238, 168, 1, 80, 214, 181, 3, 77, + 171, 190, 1, 68, 52, 42, 79, 205, 1, 5, 65, 80, 73, 84, 65, 8, 88, 10, + 77, 66, 73, 78, 73, 78, 71, 32, 68, 79, 37, 8, 78, 83, 79, 78, 65, 78, + 84, 32, 4, 234, 148, 12, 85, 247, 132, 14, 84, 4, 178, 149, 12, 78, 171, + 161, 11, 71, 46, 36, 3, 77, 65, 76, 171, 146, 22, 85, 44, 45, 9, 76, 32, + 76, 69, 84, 84, 69, 82, 32, 44, 200, 1, 4, 79, 76, 68, 32, 214, 155, 20, + 78, 238, 245, 6, 66, 2, 67, 2, 68, 2, 70, 2, 71, 2, 72, 2, 74, 2, 75, 2, + 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 88, 2, 89, 187, 2, 65, + 4, 190, 145, 27, 75, 3, 78, 12, 44, 5, 83, 73, 71, 78, 32, 179, 209, 26, + 76, 10, 138, 211, 26, 69, 162, 64, 65, 2, 73, 3, 79, 146, 3, 112, 2, 65, + 82, 110, 77, 50, 79, 160, 218, 23, 9, 82, 77, 65, 78, 32, 80, 69, 78, 78, + 174, 177, 2, 84, 239, 105, 78, 7, 29, 5, 32, 87, 73, 84, 72, 4, 224, 160, + 14, 5, 79, 85, 84, 32, 72, 233, 175, 9, 5, 32, 72, 65, 78, 68, 4, 26, 32, + 167, 140, 22, 73, 2, 207, 155, 23, 83, 130, 3, 46, 77, 245, 5, 6, 82, 71, + 73, 65, 78, 32, 38, 92, 13, 65, 78, 84, 73, 67, 32, 70, 73, 71, 85, 82, + 69, 32, 205, 4, 5, 69, 84, 82, 73, 67, 32, 158, 1, 65, 82, 67, 172, 1, 9, + 70, 79, 82, 84, 85, 78, 65, 32, 77, 44, 3, 76, 65, 69, 0, 4, 84, 82, 73, + 83, 34, 80, 80, 3, 82, 85, 66, 167, 210, 10, 86, 6, 216, 1, 6, 67, 81, + 85, 73, 83, 73, 12, 4, 77, 73, 83, 83, 235, 166, 23, 76, 8, 42, 65, 97, + 6, 79, 78, 74, 85, 78, 67, 6, 56, 3, 80, 85, 84, 0, 3, 85, 68, 65, 155, + 188, 18, 82, 2, 165, 90, 5, 32, 68, 82, 65, 67, 2, 11, 84, 2, 215, 237, + 26, 73, 4, 206, 167, 2, 73, 129, 172, 24, 2, 65, 74, 2, 145, 211, 10, 3, + 84, 73, 84, 6, 44, 2, 85, 69, 177, 128, 23, 3, 79, 80, 85, 4, 162, 233, + 26, 76, 155, 34, 82, 2, 183, 130, 25, 69, 6, 252, 135, 20, 4, 65, 76, 76, + 89, 137, 228, 1, 5, 32, 80, 82, 79, 80, 220, 2, 228, 1, 6, 67, 65, 80, + 73, 84, 65, 0, 4, 83, 77, 65, 76, 172, 3, 7, 76, 69, 84, 84, 69, 82, 32, + 180, 2, 24, 77, 84, 65, 86, 82, 85, 76, 73, 32, 67, 65, 80, 73, 84, 65, + 76, 32, 76, 69, 84, 84, 69, 82, 32, 165, 6, 2, 80, 65, 80, 45, 9, 76, 32, + 76, 69, 84, 84, 69, 82, 32, 80, 142, 2, 65, 34, 72, 166, 5, 67, 118, 71, + 130, 1, 74, 34, 75, 82, 80, 34, 83, 94, 90, 252, 181, 5, 2, 84, 65, 162, + 149, 8, 76, 226, 180, 2, 82, 218, 176, 9, 66, 2, 77, 2, 88, 226, 40, 78, + 2, 81, 234, 35, 86, 234, 47, 68, 14, 69, 2, 73, 2, 79, 2, 85, 2, 89, 143, + 57, 87, 4, 178, 182, 26, 69, 227, 79, 78, 10, 46, 65, 242, 238, 26, 73, + 2, 79, 215, 22, 69, 4, 194, 133, 27, 69, 3, 82, 94, 254, 1, 85, 178, 2, + 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, 75, 34, 76, 50, 80, 34, + 83, 34, 84, 62, 90, 254, 255, 15, 82, 218, 176, 9, 66, 2, 77, 2, 88, 226, + 40, 78, 2, 81, 234, 35, 86, 234, 47, 68, 14, 73, 2, 79, 2, 89, 142, 57, + 87, 255, 2, 70, 4, 136, 139, 22, 4, 45, 66, 82, 74, 159, 248, 4, 78, 92, + 250, 1, 65, 42, 67, 74, 69, 46, 71, 34, 72, 98, 74, 34, 75, 34, 76, 50, + 80, 34, 83, 34, 84, 62, 90, 254, 255, 15, 82, 218, 176, 9, 66, 2, 77, 2, + 88, 226, 40, 78, 2, 81, 234, 35, 86, 234, 47, 68, 14, 73, 2, 79, 2, 85, + 2, 89, 142, 57, 87, 255, 2, 70, 6, 150, 177, 26, 69, 2, 73, 227, 79, 78, + 8, 38, 72, 218, 244, 25, 73, 243, 59, 65, 4, 198, 176, 26, 73, 135, 23, + 65, 4, 156, 144, 9, 2, 76, 73, 235, 239, 17, 78, 4, 190, 179, 25, 72, + 191, 124, 65, 12, 46, 65, 186, 232, 26, 73, 2, 79, 215, 22, 69, 6, 26, + 82, 243, 254, 26, 69, 5, 239, 247, 25, 68, 4, 190, 178, 25, 72, 207, 64, + 73, 4, 254, 218, 25, 72, 223, 83, 65, 4, 11, 65, 4, 198, 218, 15, 66, + 203, 163, 11, 83, 4, 174, 218, 25, 72, 227, 106, 65, 4, 246, 253, 25, 72, + 247, 47, 65, 6, 176, 184, 3, 6, 85, 82, 78, 69, 68, 32, 135, 254, 1, 65, + 4, 178, 217, 25, 72, 223, 83, 69, 2, 181, 179, 20, 7, 82, 65, 71, 82, 65, + 80, 72, 10, 48, 2, 77, 69, 20, 4, 78, 71, 69, 82, 31, 82, 2, 167, 230, + 25, 76, 2, 133, 197, 18, 2, 32, 82, 6, 38, 76, 149, 243, 13, 3, 65, 70, + 70, 5, 207, 229, 25, 83, 202, 1, 66, 65, 214, 13, 79, 185, 171, 26, 7, + 69, 73, 67, 72, 32, 83, 84, 194, 1, 84, 8, 71, 79, 76, 73, 84, 73, 67, + 32, 229, 12, 8, 83, 83, 32, 79, 70, 32, 77, 73, 192, 1, 56, 6, 67, 65, + 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 96, 45, 9, 76, 32, 76, 69, 84, 84, + 69, 82, 32, 96, 206, 1, 65, 22, 66, 42, 67, 94, 68, 94, 70, 38, 71, 46, + 73, 138, 2, 76, 58, 77, 66, 78, 34, 79, 30, 80, 58, 82, 30, 83, 186, 1, + 84, 110, 86, 22, 89, 90, 90, 234, 145, 19, 72, 190, 133, 2, 85, 139, 183, + 5, 75, 2, 179, 210, 26, 90, 4, 214, 3, 73, 233, 225, 26, 2, 85, 75, 4, + 48, 8, 65, 85, 68, 65, 84, 69, 32, 67, 15, 72, 2, 11, 72, 2, 233, 135, + 20, 2, 82, 73, 6, 42, 74, 30, 79, 237, 210, 26, 2, 90, 69, 2, 161, 135, + 20, 2, 69, 82, 2, 187, 131, 25, 66, 4, 210, 156, 21, 82, 139, 180, 5, 73, + 2, 21, 3, 76, 65, 71, 2, 139, 241, 21, 79, 13, 38, 78, 54, 79, 141, 1, 2, + 90, 72, 2, 161, 199, 26, 8, 73, 84, 73, 65, 76, 32, 73, 90, 4, 33, 6, 84, + 65, 84, 69, 68, 32, 4, 26, 66, 25, 2, 83, 77, 2, 11, 73, 2, 35, 71, 2, + 21, 3, 65, 76, 76, 2, 165, 234, 24, 2, 32, 89, 4, 158, 240, 26, 73, 211, + 2, 69, 4, 52, 9, 65, 84, 73, 78, 65, 84, 69, 32, 77, 35, 74, 2, 141, 141, + 11, 3, 89, 83, 76, 2, 161, 148, 9, 3, 85, 68, 73, 2, 11, 65, 2, 211, 153, + 21, 83, 4, 206, 204, 26, 78, 3, 84, 4, 26, 79, 131, 241, 26, 69, 2, 245, + 208, 22, 2, 75, 79, 2, 237, 253, 21, 2, 73, 84, 14, 106, 72, 58, 76, 148, + 248, 13, 6, 80, 73, 68, 69, 82, 89, 173, 191, 10, 8, 77, 65, 76, 76, 32, + 89, 85, 83, 6, 32, 2, 84, 65, 187, 239, 26, 65, 5, 155, 197, 25, 80, 2, + 255, 131, 20, 79, 6, 78, 86, 196, 165, 22, 9, 82, 79, 75, 85, 84, 65, 83, + 84, 73, 167, 181, 4, 83, 2, 253, 174, 19, 2, 82, 73, 2, 175, 219, 24, 69, + 12, 50, 69, 222, 129, 19, 65, 130, 236, 7, 79, 3, 85, 6, 146, 149, 21, + 83, 147, 152, 5, 82, 4, 196, 152, 9, 3, 69, 77, 76, 221, 139, 16, 4, 72, + 73, 86, 69, 2, 223, 233, 26, 76, 6, 196, 246, 11, 13, 66, 69, 32, 87, 73, + 84, 72, 32, 77, 69, 82, 73, 68, 230, 200, 4, 87, 183, 151, 9, 86, 68, + 162, 1, 65, 44, 12, 84, 72, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 222, + 174, 18, 82, 222, 102, 71, 228, 141, 5, 3, 78, 71, 71, 238, 187, 1, 79, + 185, 77, 2, 76, 70, 4, 196, 191, 15, 2, 76, 32, 147, 171, 11, 84, 54, + 210, 2, 65, 50, 72, 46, 73, 46, 78, 46, 80, 2, 81, 40, 2, 82, 65, 22, 84, + 236, 144, 9, 2, 87, 73, 194, 152, 12, 85, 244, 69, 3, 70, 65, 73, 204, 7, + 4, 66, 65, 73, 82, 248, 21, 2, 79, 84, 150, 30, 68, 166, 128, 1, 77, 198, + 147, 1, 69, 164, 30, 3, 76, 65, 71, 148, 41, 3, 83, 65, 85, 230, 161, 1, + 74, 196, 16, 2, 71, 73, 141, 12, 2, 75, 85, 4, 240, 222, 24, 3, 73, 72, + 86, 163, 134, 2, 72, 4, 186, 232, 13, 87, 157, 243, 11, 2, 65, 71, 4, + 138, 146, 9, 85, 241, 245, 14, 2, 71, 71, 6, 202, 214, 15, 73, 241, 140, + 9, 2, 65, 85, 2, 225, 161, 26, 5, 65, 73, 82, 84, 72, 2, 247, 192, 26, + 73, 4, 248, 192, 23, 2, 72, 73, 237, 69, 2, 69, 73, 234, 9, 46, 65, 198, + 5, 69, 206, 82, 73, 151, 3, 79, 152, 1, 96, 7, 68, 85, 65, 84, 73, 79, + 78, 32, 5, 78, 84, 72, 65, 32, 162, 192, 20, 86, 219, 141, 5, 80, 2, 11, + 32, 2, 135, 209, 25, 67, 146, 1, 120, 7, 76, 69, 84, 84, 69, 82, 32, 212, + 2, 5, 83, 73, 71, 78, 32, 154, 255, 22, 65, 248, 8, 2, 86, 79, 195, 199, + 3, 79, 100, 214, 1, 86, 178, 131, 23, 65, 38, 68, 114, 84, 230, 5, 85, + 186, 202, 1, 73, 42, 76, 226, 195, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, + 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, 8, 69, 158, 20, 72, 2, 77, 2, 82, + 3, 89, 14, 60, 5, 69, 68, 73, 67, 32, 138, 138, 23, 79, 223, 214, 3, 65, + 4, 188, 167, 24, 6, 68, 79, 85, 66, 76, 69, 175, 244, 1, 65, 16, 66, 67, + 230, 194, 22, 78, 190, 66, 65, 182, 1, 80, 135, 150, 3, 86, 4, 238, 245, + 7, 79, 163, 204, 14, 65, 192, 8, 76, 10, 65, 84, 69, 82, 45, 84, 72, 65, + 78, 32, 206, 7, 69, 231, 207, 25, 89, 56, 134, 1, 65, 150, 3, 66, 62, 79, + 216, 2, 11, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, 130, 208, 6, 67, + 138, 4, 87, 207, 252, 18, 83, 16, 44, 5, 66, 79, 86, 69, 32, 223, 212, 6, + 78, 12, 150, 1, 83, 180, 1, 19, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, + 69, 32, 69, 81, 85, 65, 76, 32, 65, 132, 207, 6, 4, 76, 69, 83, 83, 235, + 233, 18, 82, 6, 148, 1, 7, 73, 77, 73, 76, 65, 82, 32, 133, 210, 6, 23, + 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, + 32, 76, 69, 83, 83, 4, 26, 65, 219, 209, 6, 79, 2, 65, 3, 66, 79, 86, 6, + 40, 4, 69, 83, 73, 68, 195, 210, 6, 85, 2, 231, 2, 69, 20, 40, 2, 82, 32, + 245, 1, 3, 86, 69, 82, 16, 120, 16, 83, 76, 65, 78, 84, 69, 68, 32, 69, + 81, 85, 65, 76, 32, 84, 79, 138, 212, 6, 65, 242, 129, 13, 69, 167, 237, + 4, 76, 9, 49, 10, 32, 87, 73, 84, 72, 32, 68, 79, 84, 32, 6, 44, 5, 65, + 66, 79, 86, 69, 151, 166, 18, 73, 5, 207, 165, 26, 32, 4, 52, 7, 76, 65, + 80, 80, 73, 78, 71, 239, 245, 19, 32, 2, 233, 193, 24, 2, 32, 76, 134, 8, + 36, 2, 75, 32, 185, 73, 2, 78, 32, 254, 7, 130, 3, 65, 216, 15, 8, 67, + 65, 80, 73, 84, 65, 76, 32, 182, 11, 68, 134, 1, 70, 68, 2, 73, 78, 222, + 3, 75, 138, 1, 76, 174, 3, 78, 66, 77, 84, 3, 88, 69, 83, 22, 79, 202, 1, + 80, 90, 82, 182, 1, 83, 130, 22, 84, 200, 2, 13, 85, 80, 83, 73, 76, 79, + 78, 32, 87, 73, 84, 72, 32, 150, 1, 86, 142, 2, 89, 178, 232, 7, 66, 212, + 176, 3, 4, 71, 82, 65, 77, 204, 23, 2, 90, 69, 243, 136, 12, 81, 112, 92, + 10, 67, 82, 79, 80, 72, 79, 78, 73, 67, 32, 172, 14, 6, 78, 79, 32, 84, + 69, 76, 23, 82, 106, 188, 2, 6, 65, 84, 84, 73, 67, 32, 222, 5, 67, 92, + 3, 78, 65, 88, 32, 12, 68, 69, 76, 80, 72, 73, 67, 32, 70, 73, 86, 69, 0, + 14, 83, 84, 82, 65, 84, 73, 65, 78, 32, 70, 73, 70, 84, 89, 40, 11, 69, + 80, 73, 68, 65, 85, 82, 69, 65, 78, 32, 112, 3, 72, 69, 82, 164, 1, 9, + 77, 69, 83, 83, 69, 78, 73, 65, 78, 35, 84, 48, 72, 2, 70, 73, 180, 2, 4, + 79, 78, 69, 32, 205, 1, 4, 84, 69, 78, 32, 26, 36, 3, 70, 84, 89, 105, 2, + 86, 69, 11, 11, 32, 8, 22, 84, 171, 4, 83, 6, 48, 7, 72, 79, 85, 83, 65, + 78, 68, 215, 3, 65, 5, 231, 3, 32, 17, 11, 32, 14, 56, 7, 72, 85, 78, 68, + 82, 69, 68, 18, 84, 143, 3, 83, 7, 131, 2, 32, 6, 48, 7, 72, 79, 85, 83, + 65, 78, 68, 187, 2, 65, 5, 213, 1, 2, 32, 84, 14, 98, 72, 48, 7, 84, 72, + 79, 85, 83, 65, 78, 210, 198, 24, 81, 217, 228, 1, 5, 68, 82, 65, 67, 72, + 6, 44, 5, 85, 78, 68, 82, 69, 179, 198, 24, 65, 4, 17, 2, 68, 32, 4, 22, + 84, 131, 1, 83, 2, 95, 65, 8, 30, 84, 86, 83, 175, 1, 77, 4, 50, 65, 21, + 8, 72, 79, 85, 83, 65, 78, 68, 32, 2, 187, 186, 16, 76, 2, 11, 83, 2, + 181, 199, 24, 2, 84, 65, 4, 88, 5, 65, 82, 89, 83, 84, 145, 1, 12, 89, + 82, 69, 78, 65, 73, 67, 32, 84, 87, 79, 32, 2, 101, 5, 73, 65, 78, 32, + 70, 2, 17, 2, 32, 77, 2, 139, 152, 13, 78, 6, 30, 70, 29, 3, 84, 87, 79, + 2, 225, 186, 15, 2, 73, 86, 5, 11, 32, 2, 185, 152, 10, 5, 68, 82, 65, + 67, 72, 8, 112, 8, 77, 73, 79, 78, 73, 65, 78, 32, 181, 160, 25, 14, 65, + 69, 85, 77, 32, 79, 78, 69, 32, 80, 76, 69, 84, 72, 6, 206, 193, 12, 70, + 150, 176, 12, 84, 215, 58, 79, 2, 11, 32, 2, 167, 241, 24, 84, 32, 92, 8, + 72, 69, 83, 80, 73, 65, 78, 32, 129, 1, 10, 82, 79, 69, 90, 69, 78, 73, + 65, 78, 32, 20, 40, 2, 70, 73, 38, 84, 251, 163, 18, 79, 6, 182, 233, 20, + 86, 247, 236, 3, 70, 8, 250, 182, 15, 72, 142, 191, 10, 69, 239, 48, 87, + 12, 36, 2, 70, 73, 209, 24, 2, 84, 69, 8, 146, 189, 18, 86, 213, 136, 7, + 3, 70, 84, 89, 2, 231, 139, 10, 69, 4, 168, 239, 22, 2, 79, 85, 153, 181, + 1, 3, 84, 65, 66, 154, 2, 66, 76, 174, 45, 82, 66, 68, 220, 233, 7, 2, + 75, 65, 135, 6, 84, 144, 2, 44, 6, 69, 84, 84, 69, 82, 32, 239, 45, 85, + 142, 2, 198, 2, 65, 190, 1, 69, 28, 4, 73, 79, 84, 65, 128, 1, 2, 79, 77, + 156, 3, 3, 82, 72, 79, 46, 83, 48, 7, 85, 80, 83, 73, 76, 79, 78, 146, + 33, 80, 170, 2, 84, 222, 185, 5, 68, 252, 180, 2, 2, 75, 65, 238, 192, 1, + 71, 138, 143, 11, 67, 230, 154, 2, 66, 2, 72, 2, 90, 166, 1, 76, 186, + 235, 2, 89, 210, 43, 77, 2, 78, 147, 17, 88, 48, 68, 4, 76, 80, 72, 65, + 213, 28, 8, 82, 67, 72, 65, 73, 67, 32, 83, 47, 33, 6, 32, 87, 73, 84, + 72, 32, 44, 242, 2, 68, 30, 80, 226, 29, 86, 226, 5, 79, 238, 237, 3, 84, + 191, 174, 5, 77, 62, 186, 1, 84, 131, 27, 80, 31, 33, 6, 32, 87, 73, 84, + 72, 32, 28, 186, 5, 68, 136, 25, 2, 80, 83, 158, 1, 86, 226, 5, 79, 238, + 237, 3, 84, 191, 174, 5, 77, 62, 28, 2, 69, 71, 235, 34, 73, 42, 11, 65, + 43, 33, 6, 32, 87, 73, 84, 72, 32, 40, 54, 68, 30, 80, 194, 35, 79, 22, + 86, 219, 237, 3, 84, 16, 65, 4, 65, 83, 73, 65, 18, 36, 4, 83, 73, 76, + 73, 211, 16, 82, 17, 29, 5, 32, 65, 78, 68, 32, 14, 44, 2, 79, 88, 0, 3, + 86, 65, 82, 23, 80, 4, 81, 2, 73, 65, 6, 60, 10, 69, 82, 73, 83, 80, 79, + 77, 69, 78, 73, 175, 15, 82, 5, 169, 15, 7, 32, 65, 78, 68, 32, 80, 82, + 5, 161, 35, 7, 32, 87, 73, 84, 72, 32, 68, 6, 222, 147, 8, 73, 238, 214, + 17, 65, 239, 48, 72, 23, 33, 6, 32, 87, 73, 84, 72, 32, 20, 66, 68, 166, + 26, 86, 226, 5, 79, 238, 237, 3, 84, 191, 174, 5, 77, 10, 130, 24, 65, + 237, 199, 22, 5, 73, 65, 76, 89, 84, 18, 76, 9, 73, 65, 76, 89, 84, 73, + 75, 65, 32, 32, 3, 82, 65, 67, 227, 22, 65, 8, 174, 24, 65, 191, 244, 3, + 84, 2, 207, 194, 11, 72, 4, 40, 3, 73, 86, 69, 1, 3, 79, 85, 82, 2, 205, + 37, 2, 32, 79, 76, 144, 1, 27, 83, 84, 82, 85, 77, 69, 78, 84, 65, 76, + 32, 78, 79, 84, 65, 84, 73, 79, 78, 32, 83, 89, 77, 66, 79, 76, 45, 217, + 176, 22, 2, 68, 73, 74, 70, 49, 70, 50, 62, 51, 62, 52, 170, 37, 53, 218, + 142, 26, 55, 3, 56, 17, 186, 181, 26, 49, 2, 50, 2, 51, 2, 52, 2, 55, 2, + 56, 3, 57, 15, 246, 180, 26, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, + 186, 180, 26, 48, 2, 50, 2, 54, 2, 55, 2, 56, 3, 57, 17, 254, 179, 26, + 48, 2, 50, 2, 51, 2, 53, 2, 55, 2, 56, 3, 57, 8, 54, 65, 38, 79, 169, 32, + 6, 89, 65, 84, 72, 79, 83, 4, 134, 134, 8, 80, 187, 151, 17, 73, 2, 11, + 82, 2, 11, 79, 2, 191, 139, 24, 78, 32, 128, 1, 6, 69, 84, 84, 69, 82, + 32, 168, 2, 6, 79, 87, 69, 82, 32, 78, 32, 6, 85, 78, 65, 84, 69, 32, + 201, 217, 22, 2, 73, 84, 24, 94, 83, 140, 13, 9, 65, 82, 67, 72, 65, 73, + 67, 32, 75, 2, 75, 182, 11, 68, 219, 199, 25, 89, 16, 88, 13, 77, 65, 76, + 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, 210, 12, 65, 247, 250, 7, 84, 12, + 74, 80, 170, 200, 9, 71, 242, 161, 9, 79, 154, 212, 2, 82, 139, 181, 1, + 76, 4, 206, 155, 26, 83, 219, 19, 73, 2, 161, 142, 10, 3, 85, 77, 69, 4, + 222, 25, 83, 215, 231, 7, 69, 4, 80, 4, 69, 84, 82, 69, 173, 216, 23, 10, + 85, 83, 73, 67, 65, 76, 32, 76, 69, 73, 2, 235, 212, 2, 84, 12, 88, 3, + 78, 69, 32, 142, 244, 9, 88, 252, 155, 5, 3, 85, 78, 75, 225, 208, 5, 2, + 66, 79, 6, 64, 8, 72, 65, 76, 70, 32, 83, 73, 71, 21, 4, 81, 85, 65, 82, + 4, 151, 173, 25, 78, 2, 147, 225, 20, 84, 16, 62, 82, 206, 11, 83, 114, + 69, 154, 243, 7, 72, 195, 150, 17, 73, 2, 217, 29, 2, 79, 83, 6, 100, 3, + 72, 79, 32, 165, 253, 7, 16, 69, 86, 69, 82, 83, 69, 68, 32, 76, 85, 78, + 65, 84, 69, 32, 69, 4, 136, 252, 13, 10, 87, 73, 84, 72, 32, 83, 84, 82, + 79, 75, 163, 153, 11, 83, 226, 2, 220, 1, 5, 77, 65, 76, 76, 32, 192, 19, + 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, + 84, 84, 69, 82, 32, 72, 10, 89, 77, 66, 79, 76, 32, 84, 65, 85, 32, 225, + 168, 24, 6, 73, 78, 85, 83, 79, 73, 212, 2, 56, 7, 76, 69, 84, 84, 69, + 82, 32, 202, 17, 82, 67, 68, 206, 2, 178, 2, 65, 162, 2, 68, 38, 69, 52, + 4, 73, 79, 84, 65, 0, 7, 85, 80, 83, 73, 76, 79, 78, 254, 3, 75, 28, 2, + 79, 77, 182, 5, 80, 112, 3, 82, 72, 79, 94, 83, 94, 84, 244, 237, 7, 2, + 70, 73, 210, 193, 1, 71, 138, 143, 11, 67, 230, 154, 2, 66, 2, 72, 2, 90, + 166, 1, 76, 138, 151, 3, 77, 2, 78, 147, 17, 88, 58, 64, 4, 76, 80, 72, + 65, 149, 1, 7, 82, 67, 72, 65, 73, 67, 32, 55, 33, 6, 32, 87, 73, 84, 72, + 32, 52, 82, 86, 226, 6, 68, 30, 80, 114, 79, 142, 1, 89, 218, 239, 3, 84, + 191, 174, 5, 77, 6, 154, 5, 82, 155, 3, 65, 4, 18, 75, 23, 83, 2, 215, + 251, 7, 79, 2, 11, 65, 2, 171, 197, 13, 77, 4, 150, 230, 7, 73, 151, 128, + 15, 69, 70, 22, 80, 215, 4, 84, 20, 249, 7, 3, 83, 73, 76, 41, 33, 6, 32, + 87, 73, 84, 72, 32, 38, 78, 68, 166, 1, 80, 178, 1, 86, 226, 5, 79, 238, + 237, 3, 84, 191, 174, 5, 77, 18, 50, 65, 29, 8, 73, 65, 76, 89, 84, 73, + 75, 65, 8, 153, 1, 3, 83, 73, 65, 11, 29, 5, 32, 65, 78, 68, 32, 8, 170, + 1, 80, 154, 6, 79, 22, 86, 219, 237, 3, 84, 10, 18, 83, 115, 69, 8, 21, + 3, 73, 76, 73, 9, 17, 2, 32, 65, 6, 21, 3, 78, 68, 32, 6, 30, 80, 154, 6, + 79, 23, 86, 2, 11, 69, 2, 11, 82, 2, 181, 17, 4, 73, 83, 80, 79, 4, 22, + 82, 147, 15, 65, 2, 145, 253, 25, 2, 65, 67, 4, 206, 246, 7, 65, 3, 79, + 70, 28, 2, 69, 71, 151, 3, 73, 50, 11, 65, 51, 33, 6, 32, 87, 73, 84, 72, + 32, 48, 58, 68, 30, 80, 114, 79, 62, 86, 82, 89, 219, 239, 3, 84, 16, 61, + 4, 65, 83, 73, 65, 20, 32, 4, 83, 73, 76, 73, 91, 69, 17, 29, 5, 32, 65, + 78, 68, 32, 14, 42, 79, 12, 2, 80, 69, 50, 86, 83, 89, 4, 83, 88, 4, 89, + 9, 82, 73, 83, 80, 79, 77, 69, 78, 73, 4, 11, 65, 4, 11, 82, 4, 17, 2, + 73, 65, 5, 33, 6, 32, 65, 78, 68, 32, 89, 2, 243, 12, 80, 20, 17, 2, 67, + 82, 20, 17, 2, 79, 78, 21, 33, 6, 32, 87, 73, 84, 72, 32, 18, 88, 5, 68, + 65, 83, 73, 65, 0, 5, 80, 83, 73, 76, 73, 54, 79, 22, 86, 219, 237, 3, + 84, 7, 29, 5, 32, 65, 78, 68, 32, 4, 18, 79, 23, 86, 2, 151, 224, 9, 88, + 2, 179, 9, 65, 8, 88, 11, 65, 77, 80, 72, 89, 76, 73, 65, 78, 32, 68, + 186, 132, 26, 72, 2, 83, 219, 19, 73, 2, 151, 219, 7, 73, 7, 33, 6, 32, + 87, 73, 84, 72, 32, 4, 34, 68, 201, 147, 21, 2, 80, 83, 2, 151, 205, 8, + 65, 10, 54, 65, 186, 238, 7, 84, 230, 1, 73, 219, 135, 18, 72, 4, 238, + 184, 13, 77, 251, 221, 12, 78, 4, 174, 217, 22, 72, 175, 152, 3, 65, 4, + 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 18, 68, 43, 76, 2, 37, 7, 79, + 84, 84, 69, 68, 32, 76, 2, 11, 85, 2, 33, 6, 78, 65, 84, 69, 32, 83, 2, + 169, 239, 7, 3, 73, 71, 77, 10, 230, 173, 9, 71, 138, 143, 11, 67, 2, 80, + 130, 103, 82, 231, 179, 1, 66, 2, 167, 163, 21, 82, 16, 106, 72, 104, 7, + 82, 89, 66, 76, 73, 79, 78, 44, 3, 87, 79, 32, 246, 230, 3, 79, 133, 214, + 16, 2, 65, 76, 6, 40, 4, 82, 69, 69, 32, 143, 237, 7, 69, 4, 146, 1, 79, + 213, 223, 22, 7, 81, 85, 65, 82, 84, 69, 82, 2, 21, 3, 32, 66, 65, 2, + 151, 242, 23, 83, 4, 42, 79, 189, 160, 12, 4, 84, 72, 73, 82, 2, 157, + 237, 19, 2, 66, 79, 6, 80, 5, 65, 67, 85, 84, 69, 0, 9, 68, 73, 65, 69, + 82, 69, 83, 73, 83, 39, 72, 2, 33, 6, 32, 65, 78, 68, 32, 72, 2, 209, + 202, 7, 2, 79, 79, 60, 102, 65, 21, 21, 79, 67, 65, 76, 32, 78, 79, 84, + 65, 84, 73, 79, 78, 32, 83, 89, 77, 66, 79, 76, 45, 2, 207, 214, 9, 82, + 58, 90, 50, 2, 53, 214, 202, 23, 49, 134, 196, 2, 51, 2, 52, 2, 54, 2, + 55, 2, 56, 3, 57, 13, 214, 142, 26, 48, 2, 49, 2, 50, 2, 51, 3, 52, 4, + 26, 80, 227, 195, 20, 69, 2, 11, 79, 2, 33, 6, 71, 69, 71, 82, 65, 77, 2, + 253, 136, 21, 2, 77, 69, 8, 254, 245, 13, 65, 206, 193, 10, 66, 202, 78, + 72, 253, 64, 3, 83, 65, 76, 12, 60, 6, 78, 78, 73, 78, 71, 32, 149, 132, + 25, 3, 77, 65, 67, 10, 100, 4, 70, 65, 67, 69, 137, 241, 17, 15, 67, 65, + 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 83, 9, 33, 6, 32, 87, 73, + 84, 72, 32, 6, 108, 23, 79, 78, 69, 32, 76, 65, 82, 71, 69, 32, 65, 78, + 68, 32, 79, 78, 69, 32, 83, 77, 65, 76, 76, 35, 83, 2, 11, 32, 2, 227, + 164, 8, 69, 4, 32, 2, 84, 65, 191, 239, 17, 77, 2, 247, 138, 23, 82, 6, + 28, 3, 85, 80, 32, 39, 87, 4, 142, 216, 22, 83, 135, 240, 2, 77, 2, 231, + 157, 18, 73, 220, 4, 136, 1, 2, 65, 82, 70, 73, 52, 7, 74, 65, 82, 65, + 84, 73, 32, 184, 6, 12, 78, 74, 65, 76, 65, 32, 71, 79, 78, 68, 73, 32, + 143, 3, 82, 4, 34, 65, 173, 137, 21, 2, 68, 83, 2, 11, 78, 2, 199, 128, + 25, 73, 4, 246, 227, 24, 84, 221, 162, 1, 4, 68, 69, 32, 68, 182, 1, 168, + 1, 7, 76, 69, 84, 84, 69, 82, 32, 220, 1, 5, 83, 73, 71, 78, 32, 160, 2, + 6, 86, 79, 87, 69, 76, 32, 130, 142, 20, 65, 154, 24, 82, 182, 231, 3, + 68, 179, 227, 1, 79, 98, 218, 167, 22, 65, 38, 68, 114, 84, 46, 86, 186, + 5, 85, 186, 202, 1, 73, 42, 76, 246, 14, 90, 238, 180, 1, 78, 46, 83, 82, + 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, + 186, 2, 69, 3, 79, 24, 98, 67, 28, 3, 77, 65, 68, 22, 84, 130, 203, 3, + 83, 226, 154, 18, 78, 190, 66, 65, 187, 151, 3, 86, 4, 118, 73, 199, 228, + 21, 65, 2, 243, 148, 19, 68, 4, 68, 5, 87, 79, 45, 67, 73, 29, 8, 72, 82, + 69, 69, 45, 68, 79, 84, 2, 25, 4, 82, 67, 76, 69, 2, 189, 214, 20, 5, 32, + 78, 85, 75, 84, 34, 44, 5, 83, 73, 71, 78, 32, 239, 199, 15, 67, 30, 234, + 199, 15, 67, 154, 226, 6, 65, 38, 85, 22, 86, 166, 202, 1, 73, 198, 140, + 2, 69, 3, 79, 126, 108, 7, 76, 69, 84, 84, 69, 82, 32, 216, 1, 5, 83, 73, + 71, 78, 32, 38, 86, 214, 137, 24, 68, 179, 227, 1, 79, 80, 210, 137, 22, + 78, 146, 24, 65, 38, 68, 114, 84, 230, 5, 85, 186, 202, 1, 73, 42, 76, + 222, 196, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, 8, + 69, 158, 20, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 89, 4, 146, 196, 23, 86, + 247, 244, 1, 65, 20, 190, 7, 79, 131, 186, 23, 73, 160, 2, 84, 6, 77, 85, + 75, 72, 73, 32, 189, 7, 10, 85, 78, 71, 32, 75, 72, 69, 77, 65, 32, 172, + 1, 194, 1, 65, 44, 7, 76, 69, 84, 84, 69, 82, 32, 238, 1, 83, 228, 2, 2, + 86, 79, 164, 152, 13, 3, 84, 73, 80, 174, 242, 5, 73, 252, 175, 2, 5, 69, + 75, 32, 79, 78, 202, 199, 2, 68, 203, 175, 1, 85, 4, 222, 218, 21, 66, + 177, 231, 1, 2, 68, 68, 96, 250, 201, 8, 71, 2, 75, 254, 210, 13, 65, 38, + 68, 82, 82, 34, 84, 230, 5, 85, 186, 202, 1, 73, 42, 76, 226, 195, 1, 78, + 126, 66, 2, 67, 2, 74, 2, 80, 2, 83, 206, 40, 79, 162, 8, 69, 158, 20, + 70, 2, 72, 2, 77, 2, 86, 2, 89, 3, 90, 26, 108, 19, 69, 81, 85, 69, 78, + 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 93, 4, 73, 71, + 78, 32, 12, 70, 71, 2, 75, 162, 250, 23, 83, 146, 219, 1, 76, 226, 31, + 70, 3, 90, 2, 159, 250, 23, 72, 14, 128, 1, 6, 65, 68, 65, 75, 32, 66, 2, + 66, 244, 148, 15, 2, 85, 68, 190, 196, 6, 78, 236, 177, 2, 3, 89, 65, 75, + 139, 168, 1, 86, 2, 187, 155, 8, 73, 18, 45, 9, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 18, 202, 158, 22, 65, 38, 85, 186, 202, 1, 73, 210, 237, 1, + 79, 163, 8, 69, 116, 216, 1, 22, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, + 83, 73, 71, 78, 32, 77, 69, 68, 73, 65, 76, 32, 44, 7, 76, 69, 84, 84, + 69, 82, 32, 168, 1, 5, 83, 73, 71, 78, 32, 56, 6, 86, 79, 87, 69, 76, 32, + 183, 253, 23, 68, 8, 142, 241, 25, 72, 2, 82, 2, 86, 3, 89, 60, 150, 253, + 21, 78, 182, 24, 68, 114, 84, 162, 149, 3, 66, 2, 67, 2, 71, 2, 74, 2, + 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, 89, 187, 2, + 65, 4, 250, 172, 25, 65, 233, 35, 6, 84, 72, 79, 76, 72, 79, 24, 174, + 166, 20, 83, 147, 137, 5, 76, 168, 23, 110, 65, 214, 95, 69, 150, 104, + 73, 146, 9, 79, 158, 12, 84, 30, 85, 130, 1, 89, 249, 244, 12, 4, 82, 89, + 86, 78, 140, 11, 236, 1, 2, 73, 82, 100, 8, 76, 70, 87, 73, 68, 84, 72, + 32, 242, 10, 77, 210, 1, 78, 236, 76, 21, 80, 80, 89, 32, 80, 69, 82, 83, + 79, 78, 32, 82, 65, 73, 83, 73, 78, 71, 32, 79, 78, 22, 82, 38, 84, 232, + 255, 17, 2, 85, 77, 255, 253, 5, 68, 8, 66, 32, 164, 207, 20, 6, 89, 32, + 67, 82, 69, 65, 251, 164, 4, 67, 4, 218, 187, 24, 80, 231, 115, 83, 244, + 1, 140, 2, 7, 72, 65, 78, 71, 85, 76, 32, 216, 4, 8, 75, 65, 84, 65, 75, + 65, 78, 65, 204, 3, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 152, 192, 6, 11, + 70, 79, 82, 77, 83, 32, 76, 73, 71, 72, 84, 202, 134, 4, 85, 198, 218, 2, + 73, 142, 190, 4, 66, 166, 238, 4, 87, 215, 35, 68, 104, 52, 7, 76, 69, + 84, 84, 69, 82, 32, 239, 224, 10, 70, 102, 206, 1, 75, 28, 5, 78, 73, 69, + 85, 78, 42, 80, 24, 5, 82, 73, 69, 85, 76, 86, 83, 98, 89, 202, 61, 67, + 54, 69, 30, 73, 242, 4, 77, 138, 1, 84, 206, 3, 87, 198, 1, 72, 226, 221, + 24, 65, 2, 79, 163, 64, 85, 6, 222, 65, 72, 155, 3, 73, 7, 11, 45, 4, + 134, 72, 67, 131, 3, 72, 6, 146, 69, 72, 35, 73, 17, 11, 45, 14, 206, 49, + 84, 226, 14, 80, 130, 4, 77, 194, 3, 75, 218, 2, 72, 99, 83, 12, 40, 4, + 83, 65, 78, 71, 235, 229, 13, 73, 10, 210, 70, 67, 42, 75, 74, 80, 34, + 84, 211, 2, 83, 14, 238, 157, 23, 69, 146, 137, 2, 65, 162, 64, 73, 2, + 79, 3, 85, 118, 70, 32, 157, 241, 2, 11, 45, 72, 73, 82, 65, 71, 65, 78, + 65, 32, 80, 116, 76, 7, 76, 69, 84, 84, 69, 82, 32, 242, 240, 2, 83, 34, + 86, 219, 224, 21, 77, 110, 146, 1, 83, 230, 234, 2, 78, 150, 2, 72, 2, + 75, 2, 77, 2, 82, 2, 84, 170, 1, 89, 234, 41, 87, 174, 204, 22, 65, 2, + 69, 2, 73, 2, 79, 3, 85, 28, 76, 5, 77, 65, 76, 76, 32, 230, 227, 25, 65, + 2, 69, 2, 73, 2, 79, 3, 85, 18, 206, 237, 2, 89, 174, 209, 22, 84, 234, + 36, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 11, 84, 4, 180, 165, 13, 2, 32, + 67, 135, 156, 11, 87, 14, 56, 3, 77, 69, 82, 106, 83, 145, 144, 22, 3, + 66, 85, 82, 9, 29, 5, 32, 65, 78, 68, 32, 6, 220, 177, 12, 3, 87, 82, 69, + 212, 233, 3, 2, 83, 73, 191, 148, 8, 80, 4, 148, 164, 25, 3, 84, 69, 82, + 163, 61, 65, 194, 8, 118, 68, 202, 2, 71, 128, 66, 13, 73, 70, 73, 32, + 82, 79, 72, 73, 78, 71, 89, 65, 32, 165, 5, 5, 85, 78, 79, 79, 32, 10, + 100, 12, 32, 87, 73, 84, 72, 32, 73, 78, 68, 69, 88, 32, 188, 1, 2, 66, + 65, 197, 241, 21, 2, 83, 72, 4, 156, 1, 18, 65, 78, 68, 32, 77, 73, 68, + 68, 76, 69, 32, 70, 73, 78, 71, 69, 82, 83, 1, 16, 70, 73, 78, 71, 69, + 82, 32, 65, 78, 68, 32, 84, 72, 85, 77, 66, 2, 225, 147, 16, 2, 32, 67, + 4, 154, 210, 24, 76, 211, 139, 1, 71, 170, 7, 84, 3, 85, 76, 32, 217, 64, + 13, 90, 72, 79, 85, 32, 78, 85, 77, 69, 82, 65, 76, 32, 146, 7, 164, 1, + 9, 67, 72, 79, 83, 69, 79, 78, 71, 32, 244, 15, 4, 68, 79, 85, 66, 0, 4, + 83, 73, 78, 71, 46, 74, 180, 31, 7, 76, 69, 84, 84, 69, 82, 32, 219, 161, + 10, 70, 250, 1, 246, 1, 67, 172, 2, 5, 73, 69, 85, 78, 71, 146, 1, 75, + 132, 1, 5, 77, 73, 69, 85, 77, 56, 5, 78, 73, 69, 85, 78, 74, 80, 172, 2, + 5, 82, 73, 69, 85, 76, 210, 1, 83, 166, 3, 84, 124, 2, 89, 69, 200, 40, + 5, 72, 73, 69, 85, 72, 143, 153, 10, 70, 30, 76, 2, 72, 73, 84, 7, 69, + 79, 78, 71, 67, 72, 73, 121, 4, 73, 69, 85, 67, 16, 40, 4, 69, 85, 67, + 72, 41, 2, 84, 85, 7, 11, 45, 4, 202, 31, 75, 243, 26, 72, 10, 21, 3, 69, + 85, 77, 10, 22, 83, 155, 46, 67, 6, 40, 4, 83, 65, 78, 71, 219, 213, 13, + 73, 4, 194, 54, 67, 227, 3, 83, 5, 215, 30, 45, 27, 11, 45, 24, 90, 80, + 234, 30, 82, 242, 13, 67, 194, 5, 77, 138, 1, 84, 186, 2, 75, 218, 2, 72, + 99, 83, 6, 214, 50, 72, 214, 3, 73, 207, 2, 65, 16, 80, 7, 65, 80, 89, + 69, 79, 85, 78, 228, 20, 5, 73, 89, 69, 79, 75, 131, 25, 72, 10, 234, 29, + 82, 178, 15, 80, 30, 83, 231, 3, 77, 11, 11, 45, 8, 158, 52, 75, 74, 80, + 34, 84, 211, 2, 83, 15, 11, 45, 12, 190, 51, 67, 42, 75, 74, 80, 34, 84, + 242, 1, 72, 99, 83, 42, 68, 6, 72, 73, 69, 85, 80, 72, 40, 4, 73, 69, 85, + 80, 223, 53, 65, 7, 11, 45, 4, 158, 51, 80, 147, 2, 72, 35, 11, 45, 32, + 82, 83, 234, 20, 80, 214, 7, 75, 162, 12, 67, 202, 6, 84, 226, 2, 78, + 179, 2, 72, 14, 32, 3, 73, 79, 83, 251, 3, 83, 13, 11, 45, 10, 242, 46, + 84, 146, 2, 67, 42, 75, 75, 80, 29, 11, 45, 26, 78, 75, 42, 83, 190, 44, + 77, 154, 3, 67, 82, 78, 34, 80, 34, 84, 243, 1, 72, 6, 170, 22, 65, 130, + 19, 72, 135, 7, 73, 8, 40, 4, 83, 65, 78, 71, 191, 206, 13, 73, 6, 206, + 47, 75, 74, 80, 35, 84, 58, 48, 3, 73, 79, 83, 217, 1, 4, 83, 65, 78, 71, + 33, 11, 45, 30, 130, 1, 80, 44, 2, 83, 83, 210, 22, 82, 210, 1, 75, 162, + 12, 67, 194, 5, 77, 138, 1, 84, 226, 2, 78, 178, 2, 72, 187, 170, 18, 73, + 6, 184, 23, 4, 73, 69, 85, 80, 179, 19, 72, 2, 233, 48, 3, 65, 78, 71, + 26, 236, 17, 5, 67, 73, 69, 85, 67, 172, 3, 4, 83, 73, 79, 83, 154, 1, + 82, 186, 20, 84, 54, 89, 134, 2, 75, 42, 78, 34, 80, 146, 2, 72, 187, + 170, 18, 73, 16, 40, 5, 73, 75, 69, 85, 84, 195, 41, 72, 15, 11, 45, 12, + 226, 20, 82, 178, 19, 77, 154, 3, 67, 42, 75, 74, 80, 243, 2, 83, 4, 150, + 19, 83, 139, 22, 79, 2, 225, 252, 8, 6, 76, 69, 32, 68, 79, 84, 216, 3, + 92, 9, 79, 78, 71, 83, 69, 79, 78, 71, 32, 165, 21, 9, 85, 78, 71, 83, + 69, 79, 78, 71, 32, 154, 2, 226, 1, 67, 80, 5, 72, 73, 69, 85, 72, 60, 5, + 73, 69, 85, 78, 71, 46, 75, 220, 1, 5, 77, 73, 69, 85, 77, 188, 1, 5, 78, + 73, 69, 85, 78, 94, 80, 240, 2, 5, 82, 73, 69, 85, 76, 190, 4, 83, 194, + 3, 84, 213, 1, 2, 89, 69, 8, 36, 4, 73, 69, 85, 67, 239, 30, 72, 7, 11, + 45, 4, 162, 32, 83, 239, 7, 80, 11, 11, 45, 8, 174, 16, 82, 178, 19, 77, + 234, 3, 78, 35, 80, 9, 11, 45, 6, 194, 17, 75, 57, 2, 83, 83, 28, 76, 7, + 65, 80, 89, 69, 79, 85, 78, 40, 5, 73, 89, 69, 79, 75, 215, 30, 72, 8, + 130, 15, 82, 178, 15, 80, 131, 4, 77, 19, 11, 45, 16, 166, 13, 75, 170, + 1, 82, 42, 83, 224, 13, 2, 67, 72, 146, 9, 78, 34, 80, 147, 2, 72, 27, + 11, 45, 24, 74, 80, 30, 83, 134, 13, 82, 242, 13, 67, 130, 9, 75, 42, 78, + 179, 2, 72, 6, 174, 33, 73, 131, 6, 65, 6, 40, 4, 83, 65, 78, 71, 183, + 194, 13, 73, 4, 238, 35, 78, 147, 3, 83, 21, 11, 45, 18, 174, 12, 82, + 242, 13, 67, 202, 6, 84, 186, 2, 75, 218, 2, 72, 62, 80, 39, 83, 36, 88, + 6, 65, 78, 83, 73, 79, 83, 48, 6, 72, 73, 69, 85, 80, 72, 53, 4, 73, 69, + 85, 80, 7, 11, 45, 4, 236, 7, 2, 75, 65, 195, 26, 80, 9, 11, 45, 6, 150, + 11, 84, 234, 22, 80, 243, 2, 83, 23, 11, 45, 20, 112, 5, 82, 73, 69, 85, + 76, 24, 4, 83, 73, 79, 83, 134, 3, 80, 246, 19, 67, 194, 5, 77, 170, 4, + 84, 243, 1, 72, 5, 153, 3, 2, 45, 80, 5, 221, 32, 2, 45, 84, 57, 11, 45, + 54, 102, 75, 92, 5, 77, 73, 69, 85, 77, 50, 80, 126, 83, 74, 84, 44, 2, + 89, 69, 154, 28, 78, 179, 2, 72, 10, 52, 5, 73, 89, 69, 79, 75, 190, 4, + 65, 131, 19, 72, 7, 11, 45, 4, 254, 32, 72, 99, 83, 9, 11, 45, 6, 130, + 30, 75, 218, 2, 72, 99, 83, 14, 48, 4, 73, 69, 85, 80, 174, 26, 72, 163, + 6, 65, 11, 11, 45, 8, 42, 80, 222, 29, 84, 242, 1, 72, 99, 83, 2, 243, + 25, 72, 6, 40, 4, 83, 65, 78, 71, 167, 187, 13, 73, 4, 182, 28, 75, 187, + 3, 83, 6, 100, 5, 73, 75, 69, 85, 84, 151, 25, 72, 6, 56, 9, 79, 82, 73, + 78, 72, 73, 69, 85, 72, 191, 3, 83, 5, 255, 29, 45, 52, 48, 3, 73, 79, + 83, 161, 1, 4, 83, 65, 78, 71, 25, 11, 45, 22, 82, 75, 162, 3, 82, 242, + 13, 67, 198, 2, 80, 254, 2, 77, 138, 1, 84, 147, 5, 72, 4, 22, 65, 135, + 26, 73, 2, 237, 18, 6, 80, 89, 69, 79, 85, 78, 28, 160, 1, 5, 82, 73, 69, + 85, 76, 36, 6, 84, 73, 75, 69, 85, 84, 16, 3, 89, 69, 83, 138, 19, 83, + 178, 1, 77, 154, 3, 67, 42, 75, 42, 78, 34, 80, 203, 172, 18, 73, 5, 17, + 2, 45, 75, 2, 159, 17, 72, 5, 255, 16, 45, 2, 135, 197, 18, 73, 20, 40, + 5, 73, 75, 69, 85, 84, 155, 21, 72, 19, 11, 45, 16, 58, 82, 42, 83, 42, + 84, 162, 13, 67, 130, 9, 75, 75, 80, 2, 17, 2, 73, 69, 2, 227, 171, 24, + 85, 4, 21, 3, 73, 79, 83, 5, 223, 1, 45, 2, 255, 19, 72, 18, 44, 6, 83, + 73, 69, 85, 78, 71, 243, 19, 79, 17, 11, 45, 14, 50, 75, 30, 83, 198, 17, + 77, 154, 6, 72, 63, 80, 4, 166, 14, 72, 135, 7, 73, 4, 26, 83, 215, 179, + 13, 73, 2, 21, 3, 65, 78, 71, 2, 207, 20, 75, 190, 1, 122, 65, 118, 69, + 134, 1, 73, 92, 7, 83, 83, 65, 78, 71, 65, 82, 106, 79, 138, 1, 85, 102, + 89, 174, 15, 87, 179, 149, 10, 70, 23, 48, 4, 82, 65, 69, 65, 98, 45, + 135, 179, 25, 69, 13, 11, 45, 10, 166, 234, 22, 69, 178, 201, 2, 65, 2, + 73, 3, 85, 25, 18, 79, 55, 85, 9, 11, 45, 6, 154, 142, 25, 69, 234, 36, + 79, 3, 85, 15, 11, 45, 12, 170, 9, 69, 166, 169, 25, 65, 2, 79, 3, 85, + 31, 11, 45, 28, 66, 65, 34, 89, 166, 1, 79, 166, 139, 25, 69, 234, 36, + 73, 3, 85, 5, 11, 82, 2, 195, 157, 18, 65, 14, 50, 65, 206, 231, 22, 69, + 178, 201, 2, 79, 3, 85, 7, 134, 146, 25, 45, 247, 30, 69, 23, 26, 45, + 195, 176, 25, 69, 18, 50, 79, 22, 89, 202, 230, 22, 69, 179, 201, 2, 85, + 5, 179, 156, 25, 45, 8, 198, 230, 22, 69, 147, 137, 2, 65, 17, 11, 45, + 14, 158, 19, 89, 206, 166, 5, 73, 128, 137, 13, 3, 69, 79, 45, 190, 172, + 6, 65, 163, 64, 85, 62, 42, 65, 70, 69, 66, 73, 22, 79, 107, 85, 11, 26, + 45, 171, 174, 25, 69, 6, 178, 143, 25, 89, 246, 30, 79, 3, 85, 11, 11, + 79, 9, 11, 45, 6, 174, 171, 25, 89, 186, 2, 79, 3, 85, 5, 215, 136, 25, + 45, 19, 11, 45, 16, 58, 89, 198, 236, 24, 65, 174, 33, 69, 246, 30, 73, + 3, 79, 6, 194, 236, 24, 65, 175, 33, 69, 21, 11, 45, 18, 142, 16, 89, + 250, 210, 22, 69, 146, 137, 2, 65, 162, 64, 73, 2, 79, 3, 85, 186, 1, + 226, 1, 65, 46, 67, 54, 69, 30, 73, 22, 75, 188, 1, 5, 77, 73, 69, 85, + 77, 64, 5, 78, 73, 69, 85, 78, 70, 80, 168, 1, 5, 82, 73, 69, 85, 76, + 254, 1, 84, 90, 83, 246, 2, 87, 50, 89, 150, 1, 72, 226, 221, 24, 79, + 163, 64, 85, 9, 156, 13, 3, 82, 65, 69, 231, 156, 25, 69, 4, 22, 72, 207, + 8, 73, 2, 137, 135, 25, 2, 73, 69, 7, 162, 169, 25, 79, 3, 85, 5, 203, + 181, 18, 69, 14, 56, 7, 65, 80, 89, 69, 79, 85, 78, 106, 72, 155, 3, 73, + 8, 30, 80, 30, 83, 231, 3, 77, 4, 190, 4, 72, 215, 3, 73, 2, 25, 4, 83, + 65, 78, 71, 2, 207, 7, 80, 2, 241, 60, 2, 73, 69, 9, 11, 45, 6, 22, 80, + 247, 9, 83, 4, 142, 7, 73, 207, 2, 65, 13, 11, 45, 10, 234, 5, 67, 146, + 1, 84, 242, 1, 72, 62, 80, 39, 83, 20, 48, 4, 73, 69, 85, 80, 170, 2, 72, + 163, 6, 65, 17, 11, 45, 14, 42, 83, 186, 2, 84, 146, 2, 67, 43, 75, 6, + 21, 3, 73, 79, 83, 7, 11, 45, 4, 202, 4, 75, 107, 84, 27, 11, 45, 24, 68, + 2, 75, 73, 34, 77, 34, 80, 106, 84, 54, 89, 222, 4, 72, 99, 83, 4, 149, + 1, 4, 89, 69, 79, 75, 2, 11, 73, 2, 231, 248, 23, 69, 8, 30, 72, 34, 73, + 131, 6, 65, 2, 221, 240, 18, 3, 73, 69, 85, 4, 21, 3, 69, 85, 80, 5, 243, + 5, 45, 4, 22, 72, 151, 3, 73, 2, 137, 254, 21, 2, 73, 69, 2, 17, 2, 69, + 79, 2, 167, 4, 82, 28, 44, 3, 73, 79, 83, 57, 4, 83, 65, 78, 71, 13, 11, + 45, 10, 122, 67, 42, 75, 42, 78, 34, 80, 35, 84, 16, 78, 67, 42, 75, 42, + 78, 34, 80, 34, 84, 242, 1, 72, 98, 83, 219, 169, 18, 73, 2, 11, 73, 2, + 225, 246, 23, 2, 69, 85, 2, 11, 73, 2, 233, 246, 24, 2, 89, 69, 2, 11, + 73, 2, 243, 159, 24, 69, 2, 11, 73, 2, 143, 170, 24, 69, 2, 11, 73, 2, + 11, 75, 2, 135, 166, 24, 69, 10, 146, 214, 22, 69, 146, 137, 2, 65, 163, + 64, 73, 34, 58, 69, 206, 1, 79, 62, 85, 178, 220, 24, 65, 163, 64, 73, + 13, 42, 79, 73, 6, 83, 73, 69, 85, 78, 71, 5, 11, 82, 2, 17, 2, 73, 78, + 2, 11, 72, 2, 233, 242, 9, 2, 73, 69, 7, 11, 45, 4, 18, 80, 39, 83, 2, + 11, 65, 2, 11, 78, 2, 11, 83, 2, 179, 155, 13, 73, 9, 11, 45, 6, 26, 89, + 231, 156, 25, 73, 4, 195, 220, 24, 65, 9, 11, 45, 6, 26, 89, 171, 156, + 25, 73, 4, 247, 210, 22, 69, 24, 162, 144, 11, 84, 230, 152, 12, 70, 30, + 83, 182, 87, 78, 14, 79, 227, 112, 69, 100, 156, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 196, 2, 5, 77, 65, 82, 75, 32, 72, 5, 83, 73, 71, 78, 32, + 224, 159, 2, 6, 86, 79, 87, 69, 76, 32, 183, 131, 21, 68, 58, 202, 1, 68, + 34, 75, 142, 132, 21, 84, 178, 55, 82, 238, 223, 1, 78, 214, 181, 1, 83, + 138, 69, 66, 2, 67, 2, 70, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, 2, 80, 2, + 86, 2, 87, 2, 89, 2, 90, 187, 2, 65, 4, 162, 150, 25, 68, 187, 2, 65, 8, + 56, 5, 73, 78, 78, 65, 32, 202, 149, 25, 72, 187, 2, 65, 4, 198, 149, 25, + 87, 3, 89, 4, 160, 221, 21, 6, 78, 65, 32, 75, 72, 79, 237, 186, 2, 3, + 83, 65, 75, 8, 52, 2, 84, 65, 237, 134, 23, 5, 72, 65, 82, 66, 65, 6, 42, + 72, 198, 163, 20, 83, 191, 240, 4, 78, 2, 159, 244, 24, 65, 42, 62, 76, + 176, 251, 18, 6, 83, 73, 71, 78, 32, 80, 247, 1, 86, 36, 33, 6, 69, 84, + 84, 69, 82, 32, 36, 186, 159, 21, 78, 206, 243, 3, 66, 2, 68, 2, 71, 2, + 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, + 2, 65, 2, 73, 3, 85, 2, 235, 131, 24, 69, 4, 234, 204, 23, 68, 163, 199, + 1, 80, 54, 52, 5, 67, 72, 73, 78, 71, 41, 4, 82, 65, 78, 32, 2, 17, 2, + 32, 67, 2, 139, 225, 23, 72, 52, 52, 7, 76, 69, 84, 84, 69, 82, 32, 171, + 186, 6, 78, 42, 218, 1, 65, 210, 181, 11, 90, 238, 46, 84, 146, 120, 76, + 50, 81, 60, 6, 68, 65, 76, 69, 84, 72, 214, 169, 4, 71, 122, 83, 66, 89, + 198, 207, 1, 72, 234, 5, 75, 174, 81, 66, 170, 225, 4, 78, 134, 2, 87, + 218, 103, 80, 171, 4, 77, 4, 246, 134, 17, 76, 159, 186, 7, 89, 174, 9, + 210, 1, 65, 242, 32, 66, 130, 33, 76, 200, 1, 16, 78, 84, 65, 73, 71, 65, + 78, 65, 32, 76, 69, 84, 84, 69, 82, 32, 174, 12, 82, 120, 11, 88, 65, 71, + 82, 65, 77, 32, 70, 79, 82, 32, 189, 189, 24, 4, 68, 71, 69, 72, 214, 1, + 42, 68, 98, 82, 201, 1, 3, 86, 89, 32, 6, 44, 5, 83, 84, 79, 78, 69, 183, + 225, 23, 80, 5, 213, 226, 2, 7, 32, 71, 82, 65, 86, 69, 89, 12, 32, 2, + 84, 32, 147, 164, 17, 45, 10, 60, 5, 87, 73, 84, 72, 32, 222, 171, 12, + 68, 239, 158, 9, 72, 6, 76, 9, 84, 73, 80, 32, 79, 78, 32, 84, 72, 134, + 251, 12, 82, 243, 244, 10, 65, 2, 239, 217, 24, 69, 196, 1, 134, 2, 65, + 202, 1, 66, 230, 2, 67, 154, 3, 68, 162, 1, 69, 186, 3, 70, 94, 72, 62, + 76, 222, 1, 77, 110, 79, 162, 1, 82, 142, 2, 83, 228, 1, 3, 78, 79, 82, + 198, 1, 84, 128, 2, 2, 85, 80, 174, 1, 87, 186, 148, 14, 73, 222, 243, 3, + 80, 130, 142, 4, 86, 227, 78, 71, 12, 108, 17, 82, 82, 79, 87, 32, 83, + 72, 65, 70, 84, 32, 87, 73, 68, 84, 72, 32, 182, 171, 18, 77, 207, 198, + 4, 83, 8, 36, 3, 79, 78, 69, 135, 166, 23, 84, 7, 11, 32, 4, 226, 246, + 13, 84, 251, 139, 9, 72, 14, 48, 4, 65, 76, 76, 79, 21, 4, 76, 65, 67, + 75, 2, 139, 235, 5, 84, 12, 30, 32, 153, 1, 2, 45, 70, 6, 52, 7, 67, 85, + 82, 86, 69, 68, 32, 135, 128, 24, 72, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, + 80, 2, 225, 228, 23, 8, 87, 65, 82, 68, 83, 32, 65, 78, 6, 45, 9, 69, 65, + 84, 72, 69, 82, 69, 68, 32, 6, 158, 220, 13, 83, 174, 173, 3, 78, 215, + 218, 6, 82, 14, 116, 2, 72, 69, 52, 5, 73, 82, 67, 76, 69, 149, 233, 17, + 14, 79, 78, 67, 65, 86, 69, 45, 80, 79, 73, 78, 84, 69, 68, 4, 196, 221, + 20, 4, 86, 82, 79, 78, 255, 225, 2, 67, 9, 64, 6, 32, 87, 73, 84, 72, 32, + 185, 235, 22, 4, 68, 32, 83, 65, 4, 52, 7, 83, 84, 82, 79, 75, 69, 32, + 239, 128, 5, 67, 2, 29, 5, 65, 78, 68, 32, 84, 2, 11, 87, 2, 11, 79, 2, + 21, 3, 32, 68, 79, 2, 11, 84, 2, 223, 133, 24, 83, 14, 52, 7, 65, 83, 72, + 69, 68, 32, 84, 18, 73, 31, 79, 2, 175, 16, 82, 2, 161, 162, 19, 2, 86, + 73, 10, 192, 16, 3, 87, 78, 87, 246, 147, 14, 85, 191, 184, 4, 76, 16, + 120, 5, 73, 71, 72, 84, 32, 156, 2, 16, 88, 67, 76, 65, 77, 65, 84, 73, + 79, 78, 32, 77, 65, 82, 75, 32, 199, 217, 18, 81, 10, 40, 2, 80, 79, 126, + 84, 155, 231, 22, 83, 6, 33, 6, 73, 78, 84, 69, 68, 32, 6, 194, 199, 16, + 80, 128, 158, 6, 11, 82, 69, 67, 84, 73, 76, 73, 78, 69, 65, 82, 23, 66, + 2, 193, 231, 22, 24, 69, 65, 82, 68, 82, 79, 80, 45, 83, 80, 79, 75, 69, + 68, 32, 80, 82, 79, 80, 69, 76, 76, 69, 82, 4, 234, 232, 23, 83, 227, 81, + 79, 6, 44, 5, 79, 85, 82, 32, 66, 179, 128, 5, 73, 2, 181, 139, 11, 6, + 65, 76, 76, 79, 79, 78, 4, 238, 199, 17, 79, 161, 233, 5, 6, 69, 65, 82, + 84, 32, 69, 20, 74, 65, 44, 3, 69, 70, 84, 28, 2, 79, 87, 189, 251, 4, 3, + 73, 71, 65, 4, 232, 230, 21, 2, 82, 71, 211, 209, 1, 84, 8, 254, 3, 45, + 195, 6, 87, 6, 26, 32, 191, 151, 10, 69, 4, 202, 158, 14, 68, 25, 4, 83, + 73, 78, 71, 4, 56, 8, 85, 76, 84, 73, 80, 76, 73, 67, 167, 200, 21, 73, + 2, 25, 4, 65, 84, 73, 79, 2, 159, 221, 5, 78, 6, 148, 147, 6, 9, 80, 69, + 78, 32, 67, 69, 78, 84, 82, 248, 180, 10, 13, 86, 65, 76, 32, 87, 73, 84, + 72, 32, 79, 86, 65, 76, 213, 151, 6, 5, 85, 84, 76, 73, 78, 12, 76, 4, + 73, 71, 72, 84, 129, 214, 23, 9, 79, 85, 78, 68, 45, 84, 73, 80, 80, 10, + 62, 45, 109, 11, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 4, 69, 15, + 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 78, 71, 76, 69, 32, 4, 214, 236, + 6, 66, 203, 174, 7, 81, 7, 139, 6, 32, 26, 106, 65, 78, 73, 44, 2, 79, + 85, 136, 163, 14, 7, 67, 82, 73, 80, 84, 32, 76, 193, 139, 1, 3, 80, 65, + 82, 4, 156, 154, 14, 10, 78, 83, 45, 83, 69, 82, 73, 70, 32, 73, 175, + 195, 8, 76, 8, 136, 152, 14, 2, 78, 71, 183, 194, 8, 88, 10, 21, 3, 84, + 72, 32, 10, 60, 5, 69, 65, 83, 84, 32, 29, 6, 87, 69, 83, 84, 32, 80, 6, + 26, 80, 215, 215, 23, 65, 4, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 4, + 250, 250, 16, 86, 255, 202, 6, 66, 12, 88, 9, 69, 65, 82, 68, 82, 79, 80, + 45, 83, 130, 1, 82, 145, 233, 6, 4, 87, 69, 76, 86, 6, 64, 6, 80, 79, 75, + 69, 68, 32, 253, 207, 23, 4, 72, 65, 78, 75, 4, 216, 218, 22, 8, 80, 73, + 78, 87, 72, 69, 69, 76, 27, 65, 2, 193, 3, 5, 73, 65, 78, 71, 76, 6, 26, + 87, 171, 141, 10, 80, 4, 53, 11, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, + 32, 4, 29, 5, 87, 73, 84, 72, 32, 4, 152, 142, 20, 5, 76, 65, 82, 71, 69, + 179, 243, 1, 69, 12, 76, 5, 72, 73, 84, 69, 32, 164, 1, 2, 73, 68, 149, + 133, 23, 3, 69, 68, 71, 8, 64, 6, 83, 81, 85, 65, 82, 69, 182, 227, 17, + 68, 187, 183, 5, 67, 5, 169, 186, 23, 19, 32, 67, 79, 78, 84, 65, 73, 78, + 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, 86, 2, 181, 255, 20, 2, 69, 45, + 142, 2, 40, 4, 82, 69, 87, 32, 219, 237, 24, 69, 140, 2, 112, 7, 65, 67, + 67, 69, 78, 84, 32, 142, 8, 76, 164, 14, 5, 77, 65, 82, 75, 32, 114, 80, + 225, 180, 21, 2, 89, 79, 60, 128, 2, 9, 65, 84, 78, 65, 72, 32, 72, 65, + 70, 22, 68, 36, 3, 71, 69, 82, 74, 77, 124, 2, 80, 65, 28, 4, 69, 84, 78, + 65, 20, 2, 81, 65, 58, 83, 58, 84, 156, 1, 2, 89, 69, 78, 90, 224, 172, + 8, 3, 82, 69, 86, 190, 146, 15, 79, 245, 148, 1, 3, 73, 76, 85, 2, 243, + 169, 18, 85, 4, 206, 146, 19, 69, 171, 149, 5, 65, 6, 32, 3, 69, 83, 72, + 179, 28, 83, 5, 241, 227, 6, 4, 32, 77, 85, 81, 8, 84, 5, 69, 82, 75, 72, + 65, 132, 251, 17, 2, 85, 78, 153, 45, 5, 65, 72, 65, 80, 65, 5, 221, 237, + 19, 4, 32, 75, 69, 70, 4, 26, 83, 219, 170, 24, 90, 2, 247, 195, 24, 72, + 4, 224, 163, 24, 6, 82, 78, 69, 89, 32, 80, 191, 35, 68, 4, 182, 24, 69, + 185, 237, 22, 6, 72, 65, 76, 83, 72, 69, 8, 38, 69, 241, 233, 22, 3, 73, + 80, 69, 6, 48, 6, 76, 73, 83, 72, 65, 32, 135, 232, 11, 86, 4, 252, 178, + 22, 3, 81, 69, 84, 205, 145, 2, 4, 71, 69, 68, 79, 4, 176, 212, 8, 10, + 82, 65, 72, 32, 66, 69, 78, 32, 89, 79, 231, 221, 2, 84, 8, 18, 65, 95, + 73, 6, 40, 4, 81, 69, 70, 32, 167, 134, 6, 82, 4, 246, 142, 11, 81, 149, + 193, 12, 3, 71, 65, 68, 2, 255, 171, 24, 78, 150, 1, 76, 6, 69, 84, 84, + 69, 82, 32, 201, 11, 8, 73, 71, 65, 84, 85, 82, 69, 32, 140, 1, 134, 3, + 65, 204, 1, 3, 66, 69, 84, 0, 3, 75, 65, 70, 0, 2, 80, 69, 68, 6, 70, 73, + 78, 65, 76, 32, 92, 2, 81, 79, 16, 2, 72, 69, 52, 2, 78, 85, 0, 4, 90, + 65, 89, 73, 18, 83, 48, 3, 82, 69, 83, 170, 1, 84, 52, 4, 68, 65, 76, 69, + 12, 5, 71, 73, 77, 69, 76, 0, 5, 76, 65, 77, 69, 68, 0, 3, 77, 69, 77, + 48, 3, 86, 65, 86, 80, 5, 87, 73, 68, 69, 32, 153, 1, 3, 89, 79, 68, 14, + 26, 76, 135, 225, 23, 89, 12, 64, 2, 69, 70, 73, 10, 84, 69, 82, 78, 65, + 84, 73, 86, 69, 32, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 130, 13, 77, + 130, 1, 80, 35, 81, 4, 198, 214, 16, 65, 131, 161, 1, 80, 7, 33, 6, 32, + 87, 73, 84, 72, 32, 4, 138, 15, 82, 195, 233, 13, 68, 14, 88, 2, 75, 65, + 236, 2, 2, 80, 69, 148, 154, 1, 2, 84, 83, 218, 192, 22, 78, 135, 110, + 77, 4, 235, 2, 70, 7, 244, 10, 5, 32, 87, 73, 84, 72, 131, 211, 24, 84, + 4, 167, 2, 78, 16, 44, 4, 65, 77, 69, 75, 17, 3, 72, 73, 78, 4, 231, 1, + 72, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 40, 6, 68, 65, 71, 69, 83, 72, + 39, 83, 7, 33, 6, 32, 65, 78, 68, 32, 83, 4, 218, 133, 6, 72, 155, 193, + 2, 73, 12, 50, 69, 12, 2, 65, 86, 1, 4, 83, 65, 68, 73, 4, 11, 84, 5, + 225, 244, 13, 7, 32, 87, 73, 84, 72, 32, 68, 7, 33, 6, 32, 87, 73, 84, + 72, 32, 4, 208, 253, 1, 2, 72, 79, 191, 246, 11, 68, 16, 174, 2, 76, 254, + 210, 8, 65, 178, 196, 2, 84, 158, 218, 2, 82, 156, 132, 9, 2, 68, 65, + 218, 96, 75, 222, 106, 72, 169, 4, 7, 70, 73, 78, 65, 76, 32, 77, 7, 33, + 6, 32, 87, 73, 84, 72, 32, 4, 172, 7, 2, 72, 73, 251, 234, 13, 68, 10, + 72, 6, 65, 76, 69, 70, 32, 76, 29, 8, 89, 73, 68, 68, 73, 83, 72, 32, 2, + 209, 195, 19, 2, 65, 77, 8, 120, 7, 68, 79, 85, 66, 76, 69, 32, 232, 4, + 9, 89, 79, 68, 32, 89, 79, 68, 32, 80, 193, 151, 21, 5, 86, 65, 86, 32, + 89, 4, 146, 150, 11, 86, 151, 134, 10, 89, 6, 80, 3, 76, 79, 87, 0, 3, + 85, 80, 80, 173, 129, 23, 6, 77, 65, 83, 79, 82, 65, 2, 169, 194, 23, 2, + 69, 82, 50, 84, 5, 79, 73, 78, 84, 32, 225, 5, 11, 85, 78, 67, 84, 85, + 65, 84, 73, 79, 78, 32, 38, 228, 1, 9, 68, 65, 71, 69, 83, 72, 32, 79, + 82, 46, 72, 106, 80, 162, 1, 81, 86, 82, 22, 83, 224, 216, 9, 2, 84, 83, + 224, 157, 11, 17, 74, 85, 68, 69, 79, 45, 83, 80, 65, 78, 73, 83, 72, 32, + 86, 65, 82, 189, 147, 3, 3, 77, 69, 84, 2, 17, 2, 32, 77, 2, 197, 1, 2, + 65, 80, 12, 60, 5, 65, 84, 65, 70, 32, 102, 73, 33, 4, 79, 76, 65, 77, 6, + 38, 80, 34, 81, 141, 2, 2, 83, 69, 2, 11, 65, 2, 223, 227, 17, 84, 2, + 129, 201, 23, 2, 65, 77, 2, 11, 82, 2, 139, 238, 13, 73, 5, 205, 144, 11, + 12, 32, 72, 65, 83, 69, 82, 32, 70, 79, 82, 32, 86, 6, 52, 5, 65, 77, 65, + 84, 83, 245, 198, 11, 2, 85, 66, 5, 241, 249, 10, 2, 32, 81, 2, 159, 232, + 6, 65, 8, 34, 69, 22, 72, 163, 186, 8, 73, 2, 179, 186, 23, 71, 4, 158, + 186, 8, 73, 223, 214, 7, 69, 12, 152, 1, 3, 71, 69, 82, 60, 3, 80, 65, + 83, 20, 7, 83, 79, 70, 32, 80, 65, 83, 240, 144, 22, 7, 78, 85, 78, 32, + 72, 65, 70, 253, 186, 1, 3, 77, 65, 81, 4, 26, 83, 203, 226, 22, 69, 2, + 253, 202, 23, 3, 72, 65, 89, 2, 151, 234, 13, 69, 2, 131, 234, 13, 85, 8, + 114, 77, 200, 157, 12, 15, 76, 83, 67, 72, 82, 69, 73, 66, 69, 82, 32, + 80, 65, 85, 83, 229, 227, 5, 3, 73, 67, 79, 4, 144, 229, 5, 12, 69, 84, + 32, 87, 73, 84, 72, 32, 87, 72, 73, 84, 203, 209, 17, 32, 188, 4, 164, 1, + 2, 65, 45, 50, 72, 70, 75, 230, 1, 77, 114, 78, 146, 2, 82, 66, 83, 154, + 1, 84, 226, 1, 87, 62, 89, 110, 69, 182, 248, 8, 85, 226, 174, 5, 79, + 139, 192, 3, 73, 8, 158, 171, 24, 87, 246, 30, 49, 2, 50, 3, 51, 72, 166, + 6, 79, 198, 198, 8, 65, 178, 228, 5, 85, 166, 116, 69, 3, 73, 74, 72, 2, + 65, 45, 104, 2, 79, 45, 178, 4, 73, 226, 3, 69, 187, 155, 15, 85, 24, + 146, 242, 11, 49, 238, 191, 12, 75, 214, 22, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 8, 146, 180, 24, 75, 218, 19, 49, 2, 50, 3, 51, + 54, 68, 2, 69, 45, 154, 7, 79, 186, 155, 15, 65, 2, 73, 231, 203, 2, 85, + 6, 186, 196, 24, 77, 186, 2, 49, 3, 50, 68, 116, 2, 69, 45, 72, 2, 73, + 45, 246, 2, 65, 242, 250, 8, 79, 226, 174, 5, 85, 225, 146, 6, 6, 45, 77, + 85, 45, 77, 79, 14, 222, 166, 24, 75, 246, 30, 49, 2, 50, 2, 51, 2, 52, + 2, 53, 3, 54, 16, 182, 174, 24, 84, 214, 22, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 3, 55, 54, 222, 4, 79, 2, 85, 186, 155, 15, 73, 230, 203, 2, + 65, 3, 69, 68, 62, 65, 2, 85, 226, 3, 73, 182, 248, 8, 69, 135, 163, 6, + 79, 16, 11, 45, 16, 174, 195, 24, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, + 2, 55, 3, 56, 64, 74, 69, 20, 2, 79, 45, 72, 2, 85, 45, 154, 157, 15, 73, + 231, 203, 2, 65, 18, 247, 129, 19, 45, 14, 202, 191, 24, 82, 186, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 10, 198, 162, 24, 84, 246, 30, 49, 2, + 50, 2, 51, 3, 52, 42, 218, 249, 8, 65, 2, 73, 134, 163, 6, 79, 231, 203, + 2, 69, 32, 40, 2, 65, 45, 66, 79, 159, 231, 17, 85, 12, 166, 161, 24, 89, + 246, 30, 49, 2, 50, 2, 51, 2, 52, 3, 53, 12, 11, 45, 12, 206, 191, 24, + 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 4, 188, 174, 8, 21, 77, 73, 84, + 73, 65, 78, 32, 67, 79, 78, 74, 85, 71, 65, 84, 69, 32, 77, 65, 84, 82, + 215, 144, 16, 66, 128, 1, 210, 2, 65, 110, 66, 144, 1, 2, 67, 79, 94, 68, + 198, 2, 70, 66, 71, 40, 4, 72, 79, 76, 68, 164, 1, 2, 73, 78, 116, 2, 77, + 79, 58, 79, 122, 80, 100, 2, 82, 69, 66, 83, 238, 1, 84, 210, 5, 87, 172, + 168, 15, 4, 76, 73, 77, 73, 140, 191, 5, 11, 89, 79, 85, 84, 72, 70, 85, + 76, 32, 70, 79, 137, 177, 3, 9, 69, 78, 84, 72, 85, 83, 73, 65, 83, 6, + 76, 3, 80, 80, 82, 124, 4, 70, 84, 69, 82, 209, 169, 19, 4, 66, 85, 78, + 68, 2, 201, 152, 24, 2, 79, 65, 6, 92, 5, 69, 70, 79, 82, 69, 232, 216, + 3, 6, 73, 84, 73, 78, 71, 32, 1, 4, 82, 69, 65, 75, 2, 165, 233, 23, 7, + 32, 67, 79, 77, 80, 76, 69, 6, 26, 78, 203, 160, 19, 77, 4, 204, 210, 20, + 4, 84, 69, 77, 80, 209, 213, 2, 3, 70, 76, 73, 14, 110, 69, 78, 73, 226, + 164, 19, 85, 241, 245, 2, 15, 65, 82, 75, 69, 78, 73, 78, 71, 32, 79, 70, + 32, 84, 72, 69, 6, 242, 160, 19, 67, 192, 6, 2, 76, 73, 149, 205, 4, 5, + 86, 69, 76, 79, 80, 4, 144, 151, 19, 21, 70, 70, 73, 67, 85, 76, 84, 89, + 32, 65, 84, 32, 84, 72, 69, 32, 66, 69, 71, 73, 78, 169, 253, 2, 4, 83, + 80, 69, 82, 4, 240, 155, 19, 2, 79, 76, 165, 197, 4, 5, 69, 76, 76, 79, + 87, 12, 36, 5, 65, 84, 72, 69, 82, 35, 82, 2, 189, 147, 15, 3, 73, 78, + 71, 10, 40, 4, 69, 65, 84, 32, 251, 247, 23, 65, 8, 22, 80, 215, 5, 84, + 6, 22, 79, 151, 5, 82, 4, 172, 2, 2, 83, 83, 171, 244, 23, 87, 8, 50, 78, + 226, 156, 19, 67, 217, 5, 3, 70, 76, 85, 4, 180, 162, 19, 2, 79, 67, 197, + 236, 1, 5, 69, 82, 32, 84, 82, 4, 196, 195, 22, 3, 68, 69, 83, 145, 62, + 3, 85, 84, 72, 6, 26, 66, 37, 2, 80, 80, 2, 245, 225, 23, 4, 83, 84, 82, + 85, 4, 26, 82, 207, 224, 23, 79, 2, 157, 143, 22, 2, 69, 83, 6, 160, 152, + 15, 9, 85, 83, 72, 73, 78, 71, 32, 85, 80, 192, 137, 2, 3, 82, 79, 71, + 143, 211, 6, 69, 6, 26, 84, 227, 140, 19, 86, 4, 146, 158, 19, 85, 171, + 137, 4, 82, 8, 132, 1, 5, 77, 65, 76, 76, 32, 248, 179, 21, 6, 84, 65, + 78, 68, 83, 84, 185, 244, 1, 11, 80, 76, 73, 84, 84, 73, 78, 71, 32, 65, + 80, 4, 24, 2, 80, 82, 43, 84, 2, 241, 158, 19, 5, 69, 80, 79, 78, 68, 2, + 11, 65, 2, 247, 221, 23, 77, 30, 36, 3, 72, 69, 32, 251, 231, 17, 82, 28, + 186, 2, 65, 130, 1, 67, 132, 1, 3, 70, 65, 77, 20, 9, 82, 69, 67, 69, 80, + 84, 73, 86, 69, 30, 87, 172, 22, 12, 77, 65, 82, 82, 89, 73, 78, 71, 32, + 77, 65, 73, 132, 154, 5, 6, 71, 69, 78, 84, 76, 69, 240, 227, 10, 13, 75, + 69, 69, 80, 73, 78, 71, 32, 83, 84, 73, 76, 76, 173, 238, 3, 7, 74, 79, + 89, 79, 85, 83, 32, 6, 58, 82, 185, 188, 11, 8, 66, 89, 83, 77, 65, 76, + 32, 87, 4, 220, 191, 20, 8, 79, 85, 83, 73, 78, 71, 32, 84, 163, 218, 3, + 77, 6, 188, 131, 1, 2, 65, 85, 252, 186, 19, 9, 82, 69, 65, 84, 73, 86, + 69, 32, 72, 161, 212, 1, 9, 76, 73, 78, 71, 73, 78, 71, 32, 70, 2, 207, + 134, 24, 73, 2, 129, 189, 20, 2, 32, 69, 4, 166, 159, 22, 69, 189, 204, + 1, 5, 65, 78, 68, 69, 82, 4, 190, 238, 6, 65, 153, 167, 6, 14, 79, 82, + 75, 32, 79, 78, 32, 84, 72, 69, 32, 68, 69, 67, 222, 1, 236, 1, 2, 71, + 72, 180, 2, 7, 82, 65, 71, 65, 78, 65, 32, 212, 4, 7, 83, 84, 79, 82, 73, + 67, 32, 236, 198, 15, 7, 78, 68, 85, 32, 84, 69, 77, 240, 33, 4, 75, 73, + 78, 71, 218, 249, 1, 66, 169, 180, 4, 8, 80, 80, 79, 80, 79, 84, 65, 77, + 12, 18, 32, 119, 45, 6, 164, 142, 5, 2, 66, 82, 132, 248, 16, 6, 86, 79, + 76, 84, 65, 71, 217, 49, 9, 79, 67, 84, 69, 84, 32, 80, 82, 69, 6, 92, + 11, 83, 80, 69, 69, 68, 32, 84, 82, 65, 73, 78, 249, 206, 5, 6, 72, 69, + 69, 76, 69, 68, 5, 181, 193, 11, 14, 32, 87, 73, 84, 72, 32, 66, 85, 76, + 76, 69, 84, 32, 78, 200, 1, 112, 9, 68, 73, 71, 82, 65, 80, 72, 32, 89, + 20, 7, 76, 69, 84, 84, 69, 82, 32, 154, 173, 1, 86, 215, 207, 20, 73, 2, + 207, 183, 17, 79, 194, 1, 194, 1, 65, 74, 83, 138, 164, 1, 66, 162, 3, + 78, 150, 2, 68, 2, 71, 2, 72, 2, 75, 2, 77, 2, 80, 2, 82, 2, 84, 2, 90, + 126, 87, 46, 89, 174, 209, 22, 86, 234, 36, 69, 2, 73, 2, 79, 3, 85, 7, + 37, 7, 82, 67, 72, 65, 73, 67, 32, 4, 174, 252, 23, 87, 151, 14, 89, 42, + 76, 5, 77, 65, 76, 76, 32, 170, 160, 24, 65, 2, 69, 2, 73, 2, 79, 3, 85, + 32, 230, 169, 1, 87, 46, 89, 226, 179, 5, 75, 206, 157, 17, 84, 234, 36, + 65, 2, 69, 2, 73, 2, 79, 3, 85, 2, 183, 186, 8, 83, 108, 166, 1, 76, 52, + 3, 78, 69, 89, 46, 82, 146, 7, 84, 138, 1, 85, 230, 200, 16, 83, 178, + 219, 2, 67, 220, 198, 3, 6, 77, 79, 84, 72, 69, 84, 134, 167, 1, 79, 155, + 3, 80, 6, 132, 153, 16, 4, 76, 79, 87, 32, 255, 132, 8, 69, 4, 174, 171, + 22, 66, 233, 161, 1, 2, 32, 80, 66, 60, 8, 73, 90, 79, 78, 84, 65, 76, + 32, 153, 6, 2, 83, 69, 60, 218, 1, 66, 110, 76, 152, 1, 17, 79, 78, 69, + 32, 69, 73, 71, 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, 88, 10, 83, 67, + 65, 78, 32, 76, 73, 78, 69, 45, 46, 84, 170, 240, 21, 67, 42, 69, 230, 6, + 77, 150, 1, 82, 247, 2, 90, 6, 44, 5, 76, 65, 67, 75, 32, 255, 225, 23, + 65, 4, 38, 79, 241, 223, 22, 3, 72, 69, 88, 2, 227, 223, 22, 67, 10, 40, + 4, 73, 78, 69, 32, 143, 246, 21, 65, 8, 44, 5, 87, 73, 84, 72, 32, 179, + 246, 21, 69, 6, 26, 84, 219, 247, 21, 70, 4, 250, 247, 21, 72, 243, 91, + 73, 14, 208, 237, 16, 3, 49, 51, 53, 178, 171, 7, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 3, 55, 8, 170, 152, 24, 49, 2, 51, 2, 55, 3, 57, 10, 32, 2, + 65, 66, 223, 250, 21, 82, 8, 26, 85, 223, 249, 21, 32, 6, 33, 6, 76, 65, + 84, 73, 79, 78, 7, 11, 32, 4, 152, 205, 9, 8, 87, 73, 84, 72, 32, 74, 85, + 83, 223, 148, 14, 83, 7, 11, 32, 4, 204, 198, 7, 2, 82, 65, 235, 146, 16, + 70, 10, 26, 32, 171, 138, 23, 69, 8, 86, 80, 128, 134, 19, 5, 66, 69, 86, + 69, 82, 144, 80, 3, 83, 80, 82, 179, 190, 4, 68, 2, 255, 242, 14, 69, 12, + 48, 6, 82, 71, 76, 65, 83, 83, 77, 2, 83, 69, 5, 129, 194, 20, 14, 32, + 87, 73, 84, 72, 32, 70, 76, 79, 87, 73, 78, 71, 32, 9, 11, 32, 6, 88, 8, + 87, 73, 84, 72, 32, 71, 65, 82, 205, 137, 22, 8, 66, 85, 73, 76, 68, 73, + 78, 71, 2, 159, 189, 22, 68, 7, 142, 147, 24, 74, 3, 83, 8, 106, 83, 184, + 252, 22, 11, 78, 68, 82, 69, 68, 32, 80, 79, 73, 78, 84, 176, 13, 2, 71, + 71, 163, 136, 1, 84, 2, 147, 255, 22, 72, 20, 80, 2, 71, 73, 22, 80, 224, + 1, 5, 83, 84, 69, 82, 69, 185, 137, 22, 2, 65, 67, 4, 151, 252, 21, 69, + 12, 60, 3, 72, 69, 78, 221, 160, 4, 6, 79, 68, 73, 65, 83, 84, 11, 34, + 32, 82, 65, 227, 132, 20, 45, 4, 26, 87, 239, 173, 22, 66, 2, 21, 3, 73, + 84, 72, 2, 145, 190, 3, 2, 32, 68, 2, 165, 228, 12, 6, 84, 73, 79, 78, + 32, 80, 2, 217, 249, 22, 2, 83, 73, 210, 5, 172, 1, 3, 67, 69, 32, 152, + 1, 2, 68, 69, 222, 24, 77, 222, 2, 78, 230, 35, 82, 132, 2, 7, 90, 65, + 75, 65, 89, 65, 32, 209, 250, 19, 9, 32, 76, 79, 86, 69, 32, 89, 79, 85, + 8, 114, 67, 132, 169, 11, 18, 72, 79, 67, 75, 69, 89, 32, 83, 84, 73, 67, + 75, 32, 65, 78, 68, 32, 80, 235, 190, 1, 83, 4, 162, 252, 15, 82, 223, + 231, 2, 85, 246, 1, 68, 3, 78, 84, 73, 201, 1, 9, 79, 71, 82, 65, 80, 72, + 73, 67, 32, 8, 64, 4, 67, 65, 76, 32, 105, 8, 70, 73, 67, 65, 84, 73, 79, + 78, 6, 32, 2, 84, 79, 155, 138, 23, 87, 5, 173, 255, 14, 12, 32, 65, 78, + 68, 32, 83, 76, 65, 78, 84, 69, 68, 2, 205, 241, 22, 2, 32, 67, 238, 1, + 208, 2, 11, 65, 78, 78, 79, 84, 65, 84, 73, 79, 78, 32, 242, 3, 67, 72, + 2, 68, 69, 248, 5, 5, 78, 85, 77, 66, 69, 22, 84, 172, 243, 5, 8, 72, 65, + 76, 70, 32, 70, 73, 76, 232, 186, 1, 5, 69, 78, 84, 69, 82, 0, 3, 82, 73, + 83, 24, 5, 76, 69, 86, 69, 76, 188, 143, 9, 5, 86, 65, 82, 73, 65, 250, + 233, 4, 70, 154, 47, 73, 243, 231, 1, 83, 32, 232, 1, 5, 66, 79, 84, 84, + 79, 22, 70, 82, 77, 62, 84, 140, 1, 4, 76, 73, 78, 75, 196, 143, 1, 4, + 83, 69, 67, 79, 182, 165, 6, 79, 172, 135, 6, 6, 82, 69, 86, 69, 82, 83, + 152, 222, 9, 5, 72, 69, 65, 86, 69, 169, 39, 3, 69, 65, 82, 2, 167, 196, + 23, 77, 6, 48, 3, 79, 85, 82, 221, 186, 23, 3, 73, 82, 83, 4, 210, 195, + 23, 84, 27, 32, 4, 36, 3, 73, 68, 68, 223, 155, 23, 65, 2, 195, 189, 13, + 76, 8, 34, 72, 46, 87, 227, 137, 8, 79, 4, 214, 149, 9, 82, 181, 162, 14, + 2, 73, 82, 2, 183, 194, 23, 79, 4, 36, 3, 76, 79, 83, 251, 173, 21, 79, + 2, 249, 193, 23, 3, 73, 78, 71, 36, 104, 20, 83, 67, 82, 73, 80, 84, 73, + 79, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, 32, 159, 179, 7, 80, 34, + 144, 2, 9, 65, 66, 79, 86, 69, 32, 84, 79, 32, 84, 8, 76, 69, 70, 84, 32, + 84, 79, 32, 76, 5, 79, 86, 69, 82, 76, 20, 2, 83, 85, 246, 253, 14, 82, + 172, 200, 5, 8, 70, 85, 76, 76, 32, 83, 85, 82, 229, 231, 2, 15, 72, 79, + 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 69, 70, 76, 4, 60, 9, 77, 73, 68, + 68, 76, 69, 32, 65, 78, 247, 221, 21, 66, 2, 199, 206, 21, 68, 4, 168, + 172, 22, 10, 77, 73, 68, 68, 76, 69, 32, 65, 78, 68, 211, 168, 1, 82, 2, + 171, 233, 16, 65, 18, 72, 12, 82, 82, 79, 85, 78, 68, 32, 70, 82, 79, 77, + 32, 247, 189, 17, 66, 16, 82, 76, 232, 211, 11, 3, 85, 80, 80, 250, 135, + 10, 66, 130, 165, 1, 65, 159, 82, 82, 6, 218, 211, 11, 79, 147, 255, 11, + 69, 2, 251, 220, 8, 82, 148, 1, 92, 10, 65, 76, 76, 89, 32, 77, 65, 82, + 75, 32, 41, 9, 69, 76, 69, 71, 82, 65, 80, 72, 32, 10, 162, 137, 22, 70, + 70, 84, 155, 87, 79, 138, 1, 140, 1, 11, 83, 89, 77, 66, 79, 76, 32, 70, + 79, 82, 32, 245, 238, 12, 17, 76, 73, 78, 69, 32, 70, 69, 69, 68, 32, 83, + 69, 80, 65, 82, 65, 84, 136, 1, 182, 1, 65, 58, 68, 200, 2, 5, 72, 79, + 85, 82, 32, 214, 1, 74, 28, 4, 70, 69, 66, 82, 64, 2, 77, 65, 192, 204, + 15, 3, 78, 79, 86, 0, 4, 83, 69, 80, 84, 25, 4, 79, 67, 84, 79, 4, 192, + 139, 20, 3, 85, 71, 85, 165, 142, 2, 2, 80, 82, 64, 44, 3, 65, 89, 32, + 137, 209, 15, 2, 69, 67, 62, 66, 84, 218, 215, 5, 69, 66, 70, 70, 78, 26, + 83, 235, 131, 17, 79, 34, 34, 72, 90, 87, 179, 167, 23, 69, 8, 36, 3, 73, + 82, 84, 163, 133, 22, 82, 6, 26, 89, 163, 161, 22, 69, 5, 203, 202, 22, + 45, 24, 26, 69, 247, 246, 23, 79, 22, 36, 3, 78, 84, 89, 251, 249, 22, + 76, 21, 163, 171, 15, 45, 50, 78, 84, 182, 213, 5, 69, 66, 70, 70, 78, + 26, 83, 142, 173, 16, 90, 223, 86, 79, 20, 42, 87, 146, 215, 5, 72, 207, + 206, 17, 69, 14, 26, 69, 163, 245, 23, 79, 12, 36, 3, 78, 84, 89, 167, + 248, 22, 76, 11, 199, 166, 17, 45, 6, 24, 2, 65, 78, 35, 85, 2, 11, 85, + 2, 215, 174, 17, 65, 4, 210, 221, 23, 78, 143, 5, 76, 4, 222, 209, 23, + 82, 171, 34, 89, 68, 40, 6, 65, 71, 69, 32, 79, 70, 83, 80, 5, 241, 138, + 9, 15, 32, 79, 82, 32, 65, 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 65, + 65, 14, 69, 82, 73, 65, 76, 32, 65, 82, 65, 77, 65, 73, 67, 32, 62, 72, + 7, 78, 85, 77, 66, 69, 82, 32, 230, 18, 76, 205, 217, 19, 2, 83, 69, 16, + 26, 84, 211, 207, 15, 79, 10, 154, 191, 11, 87, 250, 147, 1, 69, 131, + 172, 9, 72, 136, 3, 202, 1, 67, 234, 1, 68, 214, 6, 70, 132, 2, 5, 72, + 73, 66, 73, 84, 156, 1, 15, 80, 85, 84, 32, 83, 89, 77, 66, 79, 76, 32, + 70, 79, 82, 32, 206, 1, 83, 236, 5, 2, 84, 69, 206, 8, 86, 171, 130, 10, + 66, 10, 32, 2, 79, 77, 69, 2, 82, 69, 4, 164, 215, 16, 3, 73, 78, 71, + 209, 230, 2, 5, 80, 76, 69, 84, 69, 6, 36, 3, 65, 83, 69, 135, 171, 23, + 77, 4, 34, 32, 181, 188, 10, 2, 83, 32, 2, 185, 191, 11, 8, 70, 79, 78, + 84, 32, 83, 73, 90, 145, 1, 24, 2, 69, 88, 103, 73, 5, 173, 210, 22, 20, + 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 84, 32, 84, 72, 69, 32, 86, + 73, 69, 138, 1, 72, 8, 67, 32, 83, 73, 89, 65, 81, 32, 197, 144, 18, 4, + 65, 78, 32, 82, 136, 1, 196, 1, 11, 65, 76, 84, 69, 82, 78, 65, 84, 69, + 32, 76, 2, 76, 28, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 100, 7, 78, 85, + 77, 66, 69, 82, 32, 210, 250, 8, 82, 153, 125, 6, 80, 76, 65, 67, 69, 72, + 2, 225, 168, 23, 2, 65, 75, 6, 68, 4, 79, 78, 69, 32, 229, 229, 21, 7, + 84, 72, 82, 69, 69, 32, 81, 4, 186, 227, 21, 72, 47, 81, 122, 212, 1, 10, + 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 5, 75, 65, 82, 79, 82, 0, 4, + 76, 65, 75, 72, 246, 145, 10, 69, 66, 70, 94, 78, 26, 83, 78, 84, 236, + 135, 5, 8, 80, 82, 69, 70, 73, 88, 69, 68, 199, 41, 79, 6, 26, 84, 147, + 204, 22, 79, 4, 208, 167, 6, 2, 69, 78, 251, 160, 17, 87, 5, 179, 151, + 23, 65, 16, 72, 5, 73, 78, 73, 84, 89, 85, 9, 79, 82, 77, 65, 84, 73, 79, + 78, 32, 5, 45, 9, 32, 78, 69, 71, 65, 84, 69, 68, 32, 2, 173, 175, 8, 4, + 87, 73, 84, 72, 12, 42, 83, 249, 224, 19, 4, 68, 69, 83, 75, 10, 192, + 156, 7, 5, 69, 80, 65, 82, 65, 191, 129, 10, 79, 4, 11, 32, 4, 220, 147, + 23, 15, 65, 82, 65, 66, 73, 67, 32, 70, 79, 82, 77, 32, 83, 72, 65, 1, 14, 83, 89, 77, 77, 69, 84, 82, 73, 67, 32, 83, 87, 65, 80, 10, 80, 6, - 76, 65, 84, 73, 78, 32, 198, 240, 14, 83, 165, 201, 7, 4, 78, 85, 77, 66, + 76, 65, 84, 73, 78, 32, 242, 252, 14, 83, 161, 213, 7, 4, 78, 85, 77, 66, 6, 64, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 27, 76, 2, 21, 3, - 76, 32, 76, 2, 253, 198, 21, 2, 69, 84, 116, 84, 13, 67, 82, 73, 80, 84, - 73, 79, 78, 65, 76, 32, 80, 65, 209, 196, 15, 2, 69, 82, 114, 72, 6, 72, + 76, 32, 76, 2, 217, 222, 21, 2, 69, 84, 116, 84, 13, 67, 82, 73, 80, 84, + 73, 79, 78, 65, 76, 32, 80, 65, 141, 209, 15, 2, 69, 82, 114, 72, 6, 72, 76, 65, 86, 73, 32, 225, 1, 7, 82, 84, 72, 73, 65, 78, 32, 54, 48, 7, 76, - 69, 84, 84, 69, 82, 32, 191, 3, 78, 38, 134, 173, 10, 84, 226, 118, 65, - 22, 68, 34, 76, 22, 77, 50, 87, 186, 165, 4, 71, 90, 90, 34, 83, 66, 89, - 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, 78, 223, 105, 80, + 69, 84, 84, 69, 82, 32, 191, 3, 78, 38, 130, 180, 10, 84, 222, 119, 65, + 22, 68, 34, 76, 22, 77, 50, 87, 254, 169, 4, 71, 90, 90, 34, 83, 66, 89, + 198, 207, 1, 72, 234, 5, 75, 174, 81, 66, 170, 225, 4, 78, 223, 105, 80, 60, 22, 76, 251, 1, 78, 44, 11, 69, 44, 29, 5, 84, 84, 69, 82, 32, 44, - 150, 171, 10, 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, 178, - 215, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, - 198, 75, 66, 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, 16, - 33, 6, 85, 77, 66, 69, 82, 32, 16, 146, 162, 11, 84, 174, 140, 4, 79, - 219, 128, 3, 70, 50, 36, 4, 71, 82, 65, 76, 179, 3, 82, 23, 11, 32, 20, - 58, 65, 140, 1, 5, 87, 73, 84, 72, 32, 175, 160, 21, 69, 4, 96, 6, 86, - 69, 82, 65, 71, 69, 145, 168, 16, 12, 82, 79, 85, 78, 68, 32, 65, 32, 80, - 79, 73, 78, 2, 181, 169, 16, 5, 32, 87, 73, 84, 72, 14, 160, 1, 3, 84, - 73, 77, 20, 2, 85, 78, 248, 235, 6, 16, 76, 69, 70, 84, 87, 65, 82, 68, - 83, 32, 65, 82, 82, 79, 87, 32, 210, 162, 13, 73, 198, 1, 79, 247, 64, - 68, 2, 139, 144, 20, 69, 4, 150, 144, 20, 68, 175, 222, 2, 73, 28, 94, - 76, 232, 1, 7, 83, 69, 67, 84, 73, 79, 78, 146, 7, 82, 228, 249, 11, 2, + 146, 178, 10, 84, 242, 119, 68, 34, 76, 50, 81, 214, 205, 1, 82, 142, + 220, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 198, 207, 1, 72, 234, 5, 75, + 174, 81, 66, 170, 225, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, 16, + 33, 6, 85, 77, 66, 69, 82, 32, 16, 138, 170, 11, 84, 226, 144, 4, 79, + 167, 134, 3, 70, 50, 36, 4, 71, 82, 65, 76, 179, 3, 82, 23, 11, 32, 20, + 58, 65, 140, 1, 5, 87, 73, 84, 72, 32, 159, 183, 21, 69, 4, 96, 6, 86, + 69, 82, 65, 71, 69, 193, 180, 16, 12, 82, 79, 85, 78, 68, 32, 65, 32, 80, + 79, 73, 78, 2, 229, 181, 16, 5, 32, 87, 73, 84, 72, 14, 160, 1, 3, 84, + 73, 77, 20, 2, 85, 78, 248, 242, 6, 16, 76, 69, 70, 84, 87, 65, 82, 68, + 83, 32, 65, 82, 82, 79, 87, 32, 194, 178, 13, 73, 198, 1, 79, 251, 64, + 68, 2, 251, 166, 20, 69, 4, 134, 167, 20, 68, 131, 226, 2, 73, 28, 94, + 76, 232, 1, 7, 83, 69, 67, 84, 73, 79, 78, 146, 7, 82, 128, 130, 12, 2, 67, 65, 43, 73, 8, 164, 1, 17, 73, 78, 69, 65, 82, 32, 65, 78, 78, 79, - 84, 65, 84, 73, 79, 78, 32, 229, 232, 4, 17, 79, 67, 75, 69, 68, 32, 70, - 69, 77, 65, 76, 69, 32, 65, 78, 68, 32, 6, 140, 164, 8, 3, 65, 78, 67, - 170, 153, 8, 84, 243, 208, 3, 83, 15, 11, 32, 12, 168, 1, 6, 65, 66, 79, - 86, 69, 32, 64, 5, 87, 73, 84, 72, 32, 209, 137, 20, 22, 66, 69, 83, 73, + 84, 65, 84, 73, 79, 78, 32, 169, 239, 4, 17, 79, 67, 75, 69, 68, 32, 70, + 69, 77, 65, 76, 69, 32, 65, 78, 68, 32, 6, 156, 171, 8, 3, 65, 78, 67, + 222, 158, 8, 84, 159, 219, 3, 83, 15, 11, 32, 12, 168, 1, 6, 65, 66, 79, + 86, 69, 32, 64, 5, 87, 73, 84, 72, 32, 193, 160, 20, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 87, 73, 84, 72, - 4, 192, 138, 20, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 85, 6, 202, - 171, 4, 76, 222, 223, 15, 79, 239, 221, 2, 68, 40, 56, 2, 69, 82, 189, 5, + 4, 176, 161, 20, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 85, 6, 154, + 173, 4, 76, 254, 244, 15, 79, 195, 225, 2, 68, 40, 56, 2, 69, 82, 189, 5, 7, 73, 83, 73, 66, 76, 69, 32, 34, 48, 3, 83, 69, 32, 225, 2, 4, 84, 69, 68, 32, 16, 174, 1, 66, 80, 5, 67, 72, 69, 67, 75, 68, 24, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, - 84, 73, 80, 250, 163, 20, 87, 215, 26, 77, 6, 44, 5, 76, 65, 67, 75, 32, - 159, 215, 21, 85, 4, 226, 249, 21, 68, 211, 27, 83, 4, 136, 190, 20, 8, - 69, 82, 32, 66, 79, 65, 82, 68, 231, 182, 2, 32, 2, 233, 215, 20, 2, 32, - 76, 18, 144, 1, 6, 73, 78, 84, 69, 82, 82, 22, 76, 142, 242, 11, 80, 206, - 180, 4, 69, 220, 193, 1, 2, 79, 72, 168, 153, 2, 4, 85, 78, 68, 69, 179, - 97, 81, 2, 139, 211, 4, 79, 6, 60, 9, 79, 87, 32, 75, 65, 86, 89, 75, 65, - 163, 228, 6, 65, 5, 237, 144, 18, 11, 32, 87, 73, 84, 72, 32, 75, 65, 86, - 89, 75, 6, 38, 84, 194, 172, 19, 80, 223, 88, 83, 2, 235, 168, 16, 73, - 218, 1, 94, 65, 134, 21, 69, 62, 79, 42, 85, 165, 187, 11, 10, 73, 71, - 83, 65, 87, 32, 80, 85, 90, 90, 202, 1, 120, 5, 67, 75, 45, 79, 45, 32, - 7, 80, 65, 78, 69, 83, 69, 32, 184, 2, 7, 86, 65, 78, 69, 83, 69, 32, - 171, 174, 23, 82, 2, 209, 166, 18, 3, 76, 65, 78, 16, 246, 1, 67, 18, 80, - 136, 157, 1, 10, 73, 78, 68, 85, 83, 84, 82, 73, 65, 76, 216, 223, 3, 3, - 66, 65, 78, 186, 173, 3, 68, 252, 128, 11, 15, 83, 89, 77, 66, 79, 76, - 32, 70, 79, 82, 32, 66, 69, 71, 73, 200, 246, 1, 3, 71, 79, 66, 197, 108, - 2, 79, 71, 2, 203, 32, 65, 2, 197, 136, 13, 7, 79, 83, 84, 32, 79, 70, - 70, 182, 1, 172, 2, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, - 71, 78, 32, 76, 2, 76, 69, 40, 4, 82, 73, 71, 72, 224, 6, 2, 80, 65, 176, - 3, 14, 84, 85, 82, 78, 69, 68, 32, 80, 65, 68, 65, 32, 80, 73, 88, 5, 83, - 73, 71, 78, 32, 140, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 239, 173, 21, 68, 6, 52, 2, 75, 69, 152, 189, 16, 2, 80, 69, 175, 5, 67, - 2, 163, 246, 22, 82, 96, 38, 70, 65, 5, 84, 84, 69, 82, 32, 2, 41, 8, 84, - 32, 82, 69, 82, 69, 78, 71, 2, 211, 224, 21, 71, 94, 230, 1, 68, 26, 73, - 48, 2, 75, 65, 66, 78, 130, 1, 66, 2, 67, 2, 71, 16, 2, 80, 65, 56, 2, - 82, 65, 32, 2, 83, 65, 42, 84, 74, 74, 158, 155, 21, 65, 166, 135, 2, 72, - 2, 76, 2, 77, 2, 87, 2, 89, 186, 2, 69, 2, 79, 3, 85, 8, 222, 3, 68, 15, - 65, 7, 232, 198, 5, 3, 32, 75, 65, 215, 225, 17, 73, 7, 11, 32, 4, 22, - 83, 215, 2, 77, 2, 165, 242, 20, 2, 65, 83, 14, 36, 2, 71, 65, 90, 89, - 167, 1, 65, 7, 33, 6, 32, 76, 69, 76, 69, 84, 5, 29, 5, 32, 82, 65, 83, - 87, 2, 227, 223, 5, 65, 4, 163, 1, 65, 7, 11, 32, 4, 154, 1, 77, 245, - 161, 23, 3, 67, 69, 82, 5, 201, 192, 16, 3, 32, 65, 71, 7, 17, 2, 32, 77, - 4, 70, 85, 71, 65, 8, 18, 65, 55, 84, 5, 17, 2, 32, 77, 2, 11, 85, 2, - 135, 128, 23, 82, 4, 11, 65, 5, 11, 32, 2, 11, 77, 2, 11, 65, 2, 11, 72, - 2, 141, 174, 17, 2, 65, 80, 28, 40, 3, 68, 65, 32, 165, 3, 2, 78, 71, 24, - 174, 1, 65, 90, 76, 86, 80, 140, 144, 7, 10, 84, 73, 82, 84, 65, 32, 84, - 85, 77, 69, 130, 253, 11, 87, 164, 192, 2, 7, 73, 83, 69, 78, 45, 73, 83, - 141, 209, 1, 3, 77, 65, 68, 6, 44, 3, 68, 69, 71, 169, 145, 22, 2, 78, - 68, 5, 11, 32, 2, 157, 220, 22, 2, 65, 68, 6, 38, 85, 129, 159, 23, 3, - 73, 78, 71, 4, 160, 183, 18, 2, 78, 71, 207, 241, 3, 72, 4, 38, 73, 253, - 175, 18, 3, 65, 78, 71, 2, 145, 185, 16, 3, 83, 69, 76, 4, 220, 141, 13, - 5, 82, 65, 78, 71, 75, 135, 195, 9, 75, 10, 104, 5, 67, 69, 67, 65, 75, - 144, 197, 5, 5, 80, 65, 78, 89, 65, 176, 2, 3, 87, 73, 71, 227, 240, 10, - 76, 5, 209, 158, 18, 3, 32, 84, 69, 18, 116, 4, 83, 85, 75, 85, 42, 84, - 64, 4, 87, 85, 76, 85, 238, 182, 16, 80, 213, 218, 1, 7, 68, 73, 82, 71, - 65, 32, 77, 5, 193, 164, 22, 5, 32, 77, 69, 78, 68, 6, 26, 65, 171, 184, - 16, 79, 4, 146, 184, 16, 82, 187, 148, 6, 76, 5, 25, 4, 32, 77, 69, 76, - 2, 247, 153, 23, 73, 4, 36, 3, 76, 76, 89, 255, 216, 19, 65, 2, 219, 187, - 19, 70, 4, 180, 236, 21, 2, 89, 83, 171, 96, 73, 6, 168, 188, 14, 2, 71, - 71, 182, 253, 4, 80, 199, 195, 3, 78, 186, 25, 74, 65, 178, 78, 69, 218, - 1, 72, 150, 50, 73, 198, 6, 78, 50, 79, 111, 82, 200, 10, 200, 1, 5, 73, - 84, 72, 73, 32, 234, 4, 78, 188, 45, 6, 84, 65, 75, 65, 78, 65, 208, 13, - 3, 87, 73, 32, 220, 8, 7, 89, 65, 72, 32, 76, 73, 32, 204, 195, 4, 6, 75, - 84, 79, 86, 73, 75, 251, 223, 17, 65, 136, 1, 204, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 178, 2, 83, 138, 116, 68, 208, 237, 3, 2, 86, 79, 160, 137, - 3, 11, 78, 85, 77, 66, 69, 82, 32, 83, 73, 71, 78, 230, 191, 9, 65, 229, - 211, 1, 6, 69, 78, 85, 77, 69, 82, 90, 214, 1, 68, 242, 187, 19, 65, 150, - 1, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, - 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, 82, 138, 69, 72, 2, 76, 2, 77, 2, 86, - 2, 89, 186, 2, 69, 3, 79, 10, 38, 68, 178, 147, 23, 72, 187, 2, 65, 6, - 166, 155, 21, 68, 138, 248, 1, 72, 187, 2, 65, 12, 40, 4, 73, 71, 78, 32, - 203, 148, 11, 69, 10, 202, 128, 19, 67, 98, 78, 234, 206, 3, 65, 239, 1, - 86, 230, 4, 42, 71, 241, 39, 5, 78, 65, 68, 65, 32, 174, 3, 76, 11, 88, - 73, 32, 82, 65, 68, 73, 67, 65, 76, 32, 169, 204, 20, 2, 65, 82, 172, 3, - 130, 2, 65, 94, 66, 230, 2, 67, 150, 2, 68, 158, 3, 69, 198, 1, 70, 138, - 2, 71, 146, 1, 72, 210, 2, 73, 76, 2, 74, 65, 34, 75, 30, 76, 230, 1, 77, - 254, 1, 78, 62, 79, 102, 80, 110, 82, 166, 1, 83, 230, 6, 84, 226, 2, 86, - 50, 87, 210, 2, 89, 199, 227, 21, 85, 10, 56, 2, 82, 82, 186, 228, 21, - 71, 182, 102, 78, 207, 47, 88, 4, 234, 146, 22, 79, 207, 1, 73, 34, 58, - 65, 38, 73, 46, 76, 54, 79, 102, 82, 203, 196, 21, 69, 4, 186, 196, 5, - 77, 163, 254, 13, 68, 6, 146, 216, 21, 84, 218, 113, 82, 163, 70, 71, 6, - 134, 217, 19, 79, 234, 134, 2, 65, 159, 153, 1, 85, 10, 62, 76, 194, 242, - 22, 65, 218, 5, 78, 142, 5, 68, 203, 17, 87, 2, 209, 218, 3, 4, 84, 32, - 79, 70, 6, 42, 73, 198, 241, 9, 65, 187, 180, 11, 85, 2, 191, 243, 13, - 83, 28, 58, 65, 70, 76, 74, 79, 214, 174, 10, 72, 247, 240, 10, 73, 6, - 38, 85, 250, 240, 22, 82, 219, 5, 86, 2, 129, 232, 21, 2, 76, 68, 8, 42, - 65, 206, 183, 7, 73, 131, 193, 14, 79, 4, 218, 140, 23, 78, 3, 87, 10, - 250, 233, 21, 82, 128, 2, 2, 77, 80, 218, 98, 86, 134, 5, 76, 235, 56, - 87, 34, 38, 69, 38, 73, 98, 79, 195, 1, 82, 4, 186, 137, 21, 65, 183, - 201, 1, 69, 8, 38, 83, 198, 244, 13, 86, 175, 2, 80, 4, 132, 170, 19, 5, - 84, 73, 78, 71, 85, 251, 224, 3, 72, 16, 94, 84, 76, 3, 85, 66, 76, 222, - 246, 12, 87, 136, 194, 9, 2, 32, 78, 222, 23, 79, 223, 56, 71, 7, 25, 4, - 84, 69, 68, 32, 4, 184, 180, 7, 3, 67, 76, 73, 235, 145, 15, 84, 2, 207, - 195, 3, 69, 6, 230, 208, 21, 65, 202, 167, 1, 85, 219, 16, 89, 20, 106, - 65, 38, 78, 32, 3, 86, 69, 78, 152, 252, 8, 6, 77, 66, 82, 79, 73, 68, - 206, 224, 13, 73, 243, 19, 89, 6, 234, 221, 16, 82, 131, 170, 6, 84, 4, - 178, 25, 67, 171, 176, 22, 84, 5, 175, 182, 22, 73, 28, 74, 65, 50, 73, - 62, 76, 38, 82, 170, 20, 69, 218, 160, 22, 79, 223, 23, 85, 6, 246, 242, - 13, 84, 254, 252, 8, 67, 131, 22, 78, 8, 142, 232, 12, 69, 210, 243, 9, - 71, 230, 19, 82, 199, 21, 83, 4, 226, 190, 21, 85, 251, 198, 1, 89, 4, - 168, 215, 21, 2, 65, 71, 187, 173, 1, 79, 14, 58, 79, 52, 2, 82, 65, 150, - 154, 19, 72, 147, 163, 2, 65, 7, 174, 190, 22, 76, 241, 34, 5, 32, 83, - 76, 79, 87, 4, 186, 180, 22, 73, 135, 18, 83, 22, 58, 65, 142, 1, 69, 60, - 5, 73, 68, 73, 78, 71, 19, 79, 8, 38, 76, 250, 188, 22, 78, 199, 13, 73, - 4, 34, 70, 137, 236, 21, 2, 66, 69, 2, 41, 8, 32, 84, 82, 69, 69, 32, 84, - 82, 2, 187, 161, 19, 85, 6, 26, 65, 143, 130, 23, 77, 4, 234, 229, 22, - 82, 175, 28, 68, 2, 195, 19, 32, 6, 26, 82, 183, 254, 22, 79, 4, 246, - 234, 22, 83, 215, 22, 78, 6, 26, 78, 191, 234, 22, 67, 4, 26, 83, 231, - 255, 22, 67, 2, 131, 242, 21, 69, 4, 138, 234, 22, 68, 215, 22, 82, 2, - 205, 172, 5, 2, 78, 73, 22, 46, 65, 34, 69, 78, 73, 41, 3, 79, 78, 71, 4, - 158, 233, 22, 77, 191, 19, 67, 8, 38, 65, 210, 193, 22, 71, 199, 58, 69, - 4, 218, 235, 13, 84, 211, 147, 9, 70, 6, 178, 232, 22, 70, 2, 78, 215, - 22, 68, 5, 161, 196, 11, 3, 32, 83, 84, 22, 42, 69, 34, 73, 46, 79, 235, - 173, 22, 65, 4, 166, 174, 22, 76, 195, 51, 65, 4, 188, 235, 11, 2, 78, - 73, 151, 179, 9, 76, 12, 34, 82, 34, 85, 167, 173, 22, 79, 4, 234, 219, - 21, 84, 183, 80, 78, 6, 26, 78, 219, 251, 22, 84, 4, 146, 208, 21, 84, - 215, 172, 1, 68, 6, 26, 79, 255, 223, 22, 69, 4, 210, 229, 22, 83, 215, - 22, 84, 10, 40, 2, 78, 69, 22, 80, 171, 181, 22, 76, 5, 235, 150, 6, 83, - 4, 198, 171, 10, 80, 195, 189, 2, 69, 10, 54, 82, 202, 224, 21, 76, 166, - 1, 79, 175, 152, 1, 73, 4, 156, 199, 12, 2, 73, 86, 225, 254, 6, 2, 79, - 70, 18, 62, 65, 42, 73, 242, 166, 10, 79, 130, 131, 12, 85, 195, 9, 69, - 6, 150, 170, 22, 73, 226, 79, 80, 3, 84, 6, 240, 230, 12, 3, 71, 72, 84, - 242, 212, 9, 86, 155, 39, 67, 74, 170, 1, 65, 86, 67, 54, 69, 62, 72, - 178, 1, 73, 46, 76, 62, 80, 106, 84, 186, 227, 17, 87, 166, 155, 1, 78, - 234, 63, 79, 150, 173, 1, 77, 254, 102, 81, 254, 32, 75, 247, 47, 85, 6, - 228, 193, 3, 9, 67, 82, 73, 70, 73, 67, 73, 65, 76, 214, 153, 19, 76, - 175, 28, 89, 4, 248, 238, 10, 2, 82, 73, 161, 215, 5, 2, 72, 79, 8, 158, - 186, 21, 67, 142, 51, 65, 146, 8, 76, 167, 129, 1, 69, 10, 18, 69, 39, - 79, 4, 218, 236, 21, 76, 171, 137, 1, 69, 6, 40, 4, 82, 84, 32, 84, 151, - 217, 22, 79, 4, 44, 5, 65, 73, 76, 69, 68, 175, 249, 15, 72, 2, 145, 243, - 20, 2, 32, 66, 4, 228, 229, 17, 2, 67, 75, 247, 139, 5, 76, 6, 26, 65, - 179, 183, 22, 73, 4, 214, 221, 22, 86, 199, 21, 83, 10, 50, 69, 34, 73, - 242, 254, 18, 82, 223, 164, 3, 79, 4, 250, 186, 22, 65, 183, 22, 69, 2, - 255, 214, 22, 82, 12, 34, 69, 34, 79, 235, 228, 21, 65, 4, 166, 226, 22, - 65, 219, 16, 80, 6, 26, 80, 243, 219, 22, 78, 5, 191, 161, 22, 80, 30, - 82, 65, 98, 73, 34, 79, 38, 82, 52, 2, 85, 82, 32, 2, 87, 79, 135, 160, - 22, 69, 6, 38, 78, 142, 205, 21, 66, 247, 26, 76, 2, 33, 6, 78, 69, 68, - 32, 76, 69, 2, 179, 221, 13, 65, 4, 142, 179, 22, 71, 155, 39, 76, 4, - 198, 229, 18, 78, 135, 137, 2, 79, 6, 250, 228, 16, 73, 206, 219, 4, 65, - 159, 153, 1, 69, 4, 130, 166, 21, 66, 219, 37, 84, 5, 239, 176, 19, 32, - 4, 162, 212, 12, 65, 217, 148, 5, 3, 73, 76, 76, 26, 58, 65, 110, 69, 42, - 72, 32, 2, 73, 78, 30, 79, 39, 82, 6, 32, 2, 76, 75, 215, 176, 22, 84, 5, - 11, 32, 2, 11, 69, 2, 17, 2, 78, 67, 2, 153, 226, 17, 2, 76, 79, 4, 136, - 158, 22, 2, 65, 80, 195, 51, 83, 4, 214, 166, 21, 73, 147, 51, 69, 4, - 174, 237, 22, 68, 3, 69, 4, 146, 163, 21, 77, 235, 198, 1, 82, 4, 246, - 155, 22, 79, 239, 80, 65, 2, 201, 206, 20, 2, 69, 76, 184, 1, 92, 2, 76, - 69, 148, 2, 5, 83, 73, 71, 78, 32, 150, 190, 17, 65, 250, 8, 86, 147, - 177, 3, 68, 110, 44, 5, 84, 84, 69, 82, 32, 187, 169, 22, 78, 108, 234, - 196, 17, 78, 150, 204, 1, 65, 38, 68, 46, 76, 38, 82, 34, 84, 46, 86, - 186, 5, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, 67, - 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 70, 2, 72, 2, 77, 3, 89, - 22, 94, 67, 138, 1, 83, 246, 211, 18, 78, 242, 60, 65, 174, 158, 1, 74, - 150, 3, 85, 167, 242, 1, 86, 4, 100, 19, 79, 77, 66, 73, 78, 73, 78, 71, - 32, 65, 78, 85, 83, 86, 65, 82, 65, 32, 65, 195, 211, 18, 65, 2, 225, - 230, 19, 3, 66, 79, 86, 4, 224, 190, 12, 6, 80, 65, 67, 73, 78, 71, 163, - 134, 5, 73, 162, 2, 62, 32, 173, 11, 10, 45, 72, 73, 82, 65, 71, 65, 78, - 65, 32, 154, 2, 140, 1, 7, 76, 69, 84, 84, 69, 82, 32, 242, 9, 86, 148, - 237, 18, 10, 68, 73, 71, 82, 65, 80, 72, 32, 75, 79, 246, 203, 1, 73, - 135, 145, 1, 77, 146, 2, 186, 1, 65, 178, 1, 66, 106, 77, 186, 2, 78, 54, - 83, 226, 1, 68, 2, 71, 2, 72, 2, 75, 2, 80, 2, 82, 2, 84, 2, 86, 2, 90, - 126, 87, 46, 89, 246, 219, 22, 69, 2, 73, 2, 79, 3, 85, 19, 60, 4, 73, - 78, 85, 32, 45, 7, 82, 67, 72, 65, 73, 67, 32, 8, 130, 7, 84, 234, 197, - 22, 67, 215, 22, 80, 8, 38, 89, 134, 190, 22, 87, 235, 36, 69, 4, 234, - 226, 22, 69, 3, 73, 20, 50, 73, 158, 226, 22, 65, 2, 69, 2, 79, 3, 85, - 13, 253, 4, 9, 68, 65, 75, 85, 79, 78, 32, 78, 71, 36, 50, 73, 182, 225, - 22, 65, 2, 69, 2, 79, 3, 85, 29, 29, 5, 78, 78, 65, 78, 32, 26, 96, 15, - 78, 65, 83, 65, 76, 73, 90, 69, 68, 32, 84, 79, 78, 69, 45, 69, 5, 84, - 79, 78, 69, 45, 14, 174, 224, 22, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 55, - 3, 56, 12, 234, 223, 22, 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 13, 174, - 223, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 76, 76, 5, 77, 65, 76, 76, 32, - 174, 222, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 66, 142, 1, 72, 2, 82, 54, - 75, 46, 84, 30, 87, 46, 89, 206, 136, 19, 78, 242, 146, 3, 83, 210, 27, - 77, 234, 36, 65, 2, 69, 2, 73, 2, 79, 3, 85, 10, 154, 221, 22, 65, 2, 69, - 2, 73, 2, 79, 3, 85, 8, 230, 220, 22, 65, 2, 69, 2, 79, 3, 85, 4, 186, - 220, 22, 79, 3, 85, 8, 158, 220, 22, 65, 2, 69, 2, 73, 3, 79, 6, 242, - 219, 22, 65, 2, 79, 3, 85, 2, 241, 184, 20, 5, 79, 73, 67, 69, 68, 8, 52, - 5, 68, 79, 85, 66, 76, 22, 80, 38, 83, 35, 86, 2, 139, 250, 7, 69, 2, 89, - 6, 82, 79, 76, 79, 78, 71, 2, 29, 5, 69, 77, 73, 45, 86, 2, 21, 3, 79, - 73, 67, 2, 33, 6, 69, 68, 32, 83, 79, 85, 2, 191, 141, 22, 78, 174, 1, - 216, 1, 7, 76, 69, 84, 84, 69, 82, 32, 128, 2, 12, 80, 85, 78, 67, 84, - 85, 65, 84, 73, 79, 78, 32, 148, 3, 5, 83, 73, 71, 78, 32, 88, 11, 86, - 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 170, 163, 12, 68, 195, 209, 6, - 67, 94, 226, 1, 65, 166, 160, 4, 74, 230, 219, 14, 68, 114, 84, 230, 5, - 85, 22, 86, 186, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, - 71, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 186, 2, - 69, 3, 79, 7, 130, 214, 22, 65, 3, 73, 22, 122, 67, 90, 68, 58, 70, 54, - 83, 20, 12, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 83, 69, 233, 193, 21, - 4, 84, 82, 73, 80, 4, 56, 8, 76, 79, 83, 73, 78, 71, 32, 83, 195, 130, - 21, 73, 2, 225, 202, 21, 2, 80, 73, 4, 11, 79, 4, 216, 194, 21, 2, 85, - 66, 175, 145, 1, 84, 4, 196, 129, 21, 5, 73, 76, 76, 69, 68, 139, 57, 76, - 6, 18, 69, 23, 80, 2, 195, 237, 10, 67, 4, 236, 144, 4, 2, 65, 67, 199, - 184, 17, 73, 12, 242, 219, 7, 75, 162, 226, 10, 67, 98, 78, 162, 59, 82, - 154, 162, 1, 86, 179, 241, 1, 65, 20, 66, 65, 214, 158, 4, 86, 190, 223, - 14, 69, 2, 85, 207, 201, 1, 73, 6, 148, 222, 16, 8, 76, 84, 69, 82, 78, - 65, 84, 69, 162, 243, 5, 65, 3, 73, 96, 148, 1, 7, 76, 69, 84, 84, 69, - 82, 32, 200, 1, 5, 83, 73, 71, 78, 32, 44, 5, 84, 79, 78, 69, 32, 92, 6, - 86, 79, 87, 69, 76, 32, 231, 219, 20, 68, 56, 138, 189, 18, 79, 134, 7, - 72, 154, 145, 2, 78, 238, 178, 1, 75, 2, 80, 2, 83, 2, 84, 138, 69, 66, - 2, 67, 2, 68, 2, 71, 2, 76, 2, 77, 2, 82, 2, 86, 2, 87, 2, 89, 2, 90, - 186, 2, 65, 3, 73, 4, 182, 236, 4, 67, 249, 222, 17, 2, 83, 72, 6, 36, 5, - 67, 65, 76, 89, 65, 23, 80, 5, 17, 2, 32, 80, 2, 173, 223, 17, 3, 76, 79, - 80, 10, 226, 140, 22, 69, 2, 85, 163, 64, 79, 36, 24, 2, 76, 86, 23, 89, - 2, 159, 211, 20, 73, 35, 68, 5, 66, 79, 65, 82, 68, 36, 4, 67, 65, 80, - 32, 187, 244, 2, 72, 5, 245, 134, 19, 4, 32, 65, 78, 68, 26, 254, 136, - 17, 78, 238, 174, 3, 65, 182, 34, 68, 187, 29, 84, 188, 13, 202, 1, 65, - 212, 9, 18, 73, 84, 65, 78, 32, 83, 77, 65, 76, 76, 32, 83, 67, 82, 73, - 80, 84, 32, 204, 3, 4, 77, 69, 82, 32, 204, 25, 5, 79, 74, 75, 73, 32, - 141, 6, 8, 85, 68, 65, 87, 65, 68, 73, 32, 138, 1, 56, 8, 82, 79, 83, 72, - 84, 72, 73, 32, 243, 163, 22, 78, 136, 1, 220, 1, 4, 68, 73, 71, 73, 20, - 7, 76, 69, 84, 84, 69, 82, 32, 144, 2, 7, 78, 85, 77, 66, 69, 82, 32, 36, - 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 136, 2, 5, 83, 73, - 71, 78, 32, 198, 1, 86, 183, 194, 8, 70, 8, 223, 134, 16, 84, 74, 182, 1, - 75, 42, 84, 234, 133, 7, 78, 186, 229, 11, 68, 238, 145, 3, 83, 82, 66, - 2, 67, 2, 71, 2, 80, 2, 86, 138, 69, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, - 89, 2, 90, 187, 2, 65, 6, 138, 195, 22, 72, 2, 75, 187, 2, 65, 12, 142, - 236, 18, 84, 214, 214, 3, 72, 187, 2, 65, 8, 162, 203, 14, 84, 227, 165, - 2, 79, 18, 70, 67, 74, 68, 66, 76, 36, 5, 77, 65, 78, 71, 65, 159, 240, - 20, 83, 4, 26, 82, 247, 241, 20, 73, 2, 145, 162, 21, 6, 69, 83, 67, 69, - 78, 84, 6, 26, 79, 251, 230, 18, 65, 4, 194, 230, 18, 85, 219, 220, 3, - 84, 4, 226, 220, 6, 79, 227, 190, 7, 73, 2, 215, 135, 21, 76, 12, 72, 2, - 66, 65, 22, 67, 20, 2, 68, 79, 166, 139, 20, 86, 179, 241, 1, 65, 2, 239, - 196, 21, 82, 2, 203, 147, 6, 65, 4, 56, 8, 85, 66, 76, 69, 32, 82, 73, - 78, 207, 147, 20, 84, 2, 203, 147, 20, 71, 14, 44, 5, 79, 87, 69, 76, 32, - 247, 135, 20, 73, 12, 44, 5, 83, 73, 71, 78, 32, 163, 254, 21, 76, 10, - 182, 141, 4, 86, 230, 178, 18, 69, 2, 73, 2, 79, 3, 85, 176, 7, 72, 12, - 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 179, 200, 7, 70, 174, 7, - 22, 66, 147, 1, 67, 128, 4, 170, 247, 7, 48, 2, 49, 2, 50, 2, 51, 2, 52, - 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, - 70, 174, 3, 142, 1, 68, 142, 245, 7, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 163, 198, 13, 70, - 12, 206, 188, 22, 48, 2, 49, 2, 50, 2, 51, 2, 52, 3, 53, 246, 2, 202, 1, - 67, 240, 2, 18, 73, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 86, 79, - 87, 69, 76, 32, 220, 2, 7, 76, 69, 84, 84, 69, 82, 32, 254, 2, 83, 208, - 12, 6, 86, 79, 87, 69, 76, 32, 143, 180, 20, 68, 70, 176, 1, 20, 79, 78, - 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, - 173, 172, 17, 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, - 32, 82, 73, 68, 138, 1, 78, 154, 4, 67, 2, 75, 90, 80, 74, 84, 54, 68, 2, - 76, 138, 234, 21, 83, 158, 41, 77, 2, 82, 2, 86, 2, 89, 190, 28, 66, 3, - 72, 8, 142, 153, 22, 71, 2, 89, 246, 30, 65, 3, 79, 42, 82, 81, 196, 1, - 11, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, 50, 76, 3, 82, 26, 82, - 65, 44, 6, 79, 79, 32, 84, 89, 80, 34, 85, 134, 172, 20, 73, 223, 137, 2, - 69, 8, 170, 182, 22, 65, 2, 73, 2, 81, 3, 85, 4, 11, 69, 4, 207, 142, 10, - 32, 9, 198, 185, 7, 85, 155, 252, 14, 75, 8, 18, 81, 31, 82, 4, 166, 181, - 22, 69, 3, 85, 4, 215, 149, 18, 89, 70, 138, 1, 67, 2, 75, 42, 78, 50, - 80, 30, 83, 46, 84, 54, 68, 2, 76, 166, 147, 22, 77, 2, 82, 2, 86, 2, 89, - 190, 28, 66, 2, 72, 3, 81, 8, 210, 1, 72, 154, 178, 22, 65, 3, 79, 8, - 206, 148, 22, 71, 2, 78, 2, 89, 247, 30, 79, 6, 122, 72, 155, 178, 22, - 79, 6, 130, 148, 22, 83, 190, 28, 72, 187, 2, 65, 12, 50, 72, 0, 2, 84, - 72, 154, 178, 22, 65, 3, 79, 4, 150, 178, 22, 65, 3, 79, 130, 1, 60, 4, - 73, 71, 78, 32, 221, 6, 6, 89, 77, 66, 79, 76, 32, 46, 202, 2, 65, 104, - 10, 83, 65, 77, 89, 79, 75, 32, 83, 65, 78, 22, 66, 118, 67, 86, 75, 74, - 82, 54, 84, 144, 223, 4, 3, 76, 69, 75, 168, 11, 6, 80, 72, 78, 65, 69, - 75, 248, 226, 10, 10, 89, 85, 85, 75, 65, 76, 69, 65, 80, 73, 200, 238, - 1, 3, 78, 73, 75, 180, 165, 3, 9, 77, 85, 85, 83, 73, 75, 65, 84, 79, - 133, 15, 4, 86, 73, 82, 73, 6, 100, 9, 86, 65, 75, 82, 65, 72, 65, 83, - 65, 176, 128, 18, 4, 84, 84, 72, 65, 209, 136, 4, 2, 72, 83, 2, 167, 171, - 22, 78, 8, 38, 65, 137, 164, 21, 3, 69, 89, 89, 6, 174, 6, 84, 216, 1, 2, - 78, 84, 193, 219, 20, 6, 82, 73, 89, 79, 79, 83, 4, 152, 149, 7, 12, 65, - 77, 78, 85, 67, 32, 80, 73, 73, 32, 75, 85, 211, 212, 12, 79, 6, 140, - 214, 11, 2, 65, 75, 154, 140, 9, 72, 177, 80, 4, 79, 79, 77, 85, 4, 210, - 213, 11, 79, 249, 224, 5, 4, 69, 65, 72, 77, 4, 192, 151, 21, 8, 79, 65, - 78, 68, 65, 75, 72, 73, 253, 2, 4, 82, 73, 73, 83, 84, 128, 1, 3, 68, 65, - 80, 92, 10, 76, 69, 75, 32, 65, 84, 84, 65, 75, 32, 182, 1, 80, 72, 5, - 84, 85, 84, 69, 89, 78, 66, 47, 77, 24, 22, 45, 223, 3, 32, 20, 30, 80, - 238, 2, 66, 47, 77, 8, 138, 3, 73, 37, 3, 82, 65, 77, 20, 50, 80, 98, 66, - 130, 143, 11, 77, 255, 200, 10, 83, 12, 36, 3, 82, 65, 77, 203, 148, 22, - 73, 11, 11, 45, 8, 42, 66, 130, 143, 11, 77, 239, 213, 8, 80, 4, 250, - 215, 21, 85, 151, 60, 69, 26, 44, 2, 65, 84, 44, 3, 82, 65, 77, 91, 73, - 2, 21, 3, 72, 65, 77, 2, 223, 178, 16, 65, 20, 18, 45, 119, 32, 16, 34, - 66, 32, 2, 80, 73, 15, 77, 8, 30, 69, 37, 3, 85, 79, 78, 4, 35, 73, 4, - 21, 3, 85, 79, 89, 4, 11, 32, 4, 34, 82, 169, 240, 21, 2, 75, 79, 2, 195, - 253, 20, 79, 42, 76, 10, 73, 78, 72, 69, 82, 69, 78, 84, 32, 65, 29, 5, - 83, 73, 71, 78, 32, 4, 218, 164, 22, 65, 3, 81, 38, 102, 65, 54, 73, 30, - 79, 38, 89, 184, 253, 2, 5, 67, 79, 69, 78, 71, 250, 165, 7, 85, 183, - 255, 11, 69, 10, 174, 174, 3, 65, 170, 245, 18, 69, 2, 73, 3, 85, 7, 162, - 163, 22, 69, 3, 73, 6, 134, 163, 22, 69, 2, 77, 3, 79, 7, 226, 162, 22, - 65, 3, 89, 130, 1, 146, 1, 68, 88, 7, 76, 69, 84, 84, 69, 82, 32, 170, 2, - 83, 160, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 230, 225, 15, - 87, 211, 80, 65, 6, 48, 6, 79, 85, 66, 76, 69, 32, 219, 196, 18, 65, 4, - 170, 160, 10, 83, 163, 164, 8, 68, 90, 234, 1, 83, 254, 4, 66, 42, 71, - 186, 134, 6, 68, 214, 134, 12, 74, 158, 50, 65, 150, 1, 84, 218, 207, 1, - 76, 250, 192, 1, 78, 126, 67, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 81, 2, - 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, 4, 26, 72, 211, 158, - 22, 65, 2, 189, 222, 21, 3, 79, 82, 84, 12, 40, 4, 73, 71, 78, 32, 195, - 157, 10, 69, 10, 54, 83, 238, 137, 18, 78, 206, 61, 86, 159, 145, 3, 65, - 4, 26, 72, 187, 162, 17, 85, 2, 11, 65, 2, 159, 248, 21, 68, 18, 170, - 234, 3, 86, 154, 223, 14, 65, 242, 201, 1, 73, 222, 137, 2, 69, 2, 79, 3, - 85, 138, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 176, 2, 5, 83, 73, 71, - 78, 32, 202, 177, 16, 86, 187, 246, 3, 68, 94, 222, 1, 66, 42, 71, 186, - 134, 6, 68, 174, 133, 12, 74, 198, 51, 65, 118, 82, 34, 84, 230, 5, 85, - 206, 201, 1, 73, 162, 193, 1, 78, 126, 67, 2, 75, 2, 80, 2, 83, 138, 69, - 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 182, 151, 22, 66, - 2, 72, 187, 2, 65, 6, 142, 151, 22, 71, 2, 72, 187, 2, 65, 6, 190, 133, - 18, 78, 206, 61, 86, 159, 145, 3, 65, 136, 1, 140, 1, 8, 82, 65, 84, 32, - 82, 65, 73, 32, 216, 3, 2, 83, 83, 208, 223, 20, 4, 87, 73, 70, 82, 182, - 45, 80, 240, 93, 2, 77, 79, 191, 18, 84, 116, 140, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 172, 1, 5, 83, 73, 71, 78, 32, 92, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 243, 229, 11, 68, 64, 206, 188, 18, 68, 114, 84, - 226, 222, 1, 78, 238, 178, 1, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, - 83, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 12, 208, - 202, 9, 3, 84, 79, 78, 0, 2, 89, 85, 242, 251, 1, 83, 230, 137, 10, 65, - 239, 1, 86, 16, 246, 192, 18, 65, 174, 147, 3, 85, 162, 64, 69, 2, 73, 3, - 79, 13, 40, 4, 73, 78, 71, 32, 163, 210, 21, 32, 8, 96, 4, 70, 65, 67, - 69, 181, 135, 14, 14, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, - 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 162, 134, 14, 83, 215, 144, 5, - 67, 4, 168, 146, 18, 3, 69, 69, 76, 215, 227, 3, 79, 4, 40, 4, 82, 69, - 65, 78, 187, 239, 21, 65, 2, 33, 6, 32, 83, 84, 65, 78, 68, 2, 129, 228, - 18, 2, 65, 82, 2, 11, 79, 2, 191, 161, 10, 78, 174, 43, 154, 1, 65, 150, - 232, 1, 69, 182, 62, 73, 238, 83, 79, 222, 28, 85, 94, 89, 226, 224, 7, - 82, 244, 179, 9, 5, 32, 66, 32, 66, 65, 166, 160, 1, 76, 239, 66, 70, - 252, 22, 182, 1, 66, 44, 6, 67, 82, 79, 83, 83, 69, 46, 68, 56, 2, 79, - 32, 222, 11, 82, 236, 21, 4, 83, 84, 32, 81, 76, 4, 84, 73, 78, 32, 237, - 129, 20, 8, 78, 71, 85, 65, 71, 69, 32, 84, 4, 176, 182, 14, 2, 32, 67, - 175, 206, 6, 69, 2, 153, 237, 16, 6, 32, 83, 84, 73, 67, 75, 4, 220, 242, - 12, 5, 89, 32, 66, 69, 69, 243, 220, 8, 68, 174, 1, 200, 2, 3, 72, 79, - 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 214, 5, 83, 148, 1, 9, 84, 79, 78, - 69, 32, 77, 65, 73, 32, 84, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, - 32, 188, 164, 4, 2, 75, 79, 224, 247, 12, 2, 89, 65, 230, 194, 2, 69, - 190, 50, 68, 236, 135, 1, 7, 67, 65, 78, 67, 69, 76, 76, 221, 68, 6, 78, - 73, 71, 71, 65, 72, 4, 190, 235, 21, 77, 3, 78, 94, 176, 1, 3, 70, 79, - 32, 78, 75, 96, 2, 76, 79, 50, 80, 154, 1, 83, 86, 84, 30, 72, 234, 149, - 16, 78, 166, 208, 5, 66, 2, 67, 2, 68, 2, 77, 2, 82, 2, 87, 2, 89, 247, - 30, 79, 8, 42, 70, 254, 162, 15, 83, 195, 170, 5, 84, 4, 214, 184, 21, - 79, 155, 62, 65, 10, 26, 72, 255, 135, 22, 79, 8, 32, 3, 77, 85, 32, 231, - 2, 79, 4, 146, 175, 21, 78, 211, 57, 71, 7, 17, 2, 32, 76, 4, 170, 182, - 21, 73, 67, 79, 30, 52, 4, 65, 76, 73, 32, 210, 1, 72, 131, 133, 22, 79, - 24, 154, 194, 6, 68, 34, 84, 206, 6, 78, 138, 195, 13, 66, 2, 67, 2, 71, - 2, 74, 171, 216, 1, 76, 8, 52, 9, 65, 78, 83, 75, 82, 73, 84, 32, 83, 71, - 79, 4, 254, 130, 22, 72, 3, 83, 6, 26, 72, 131, 133, 22, 79, 4, 11, 79, - 4, 11, 32, 4, 170, 159, 15, 83, 195, 170, 5, 84, 6, 112, 14, 69, 77, 73, - 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 169, 173, 18, 8, 73, 71, 78, - 32, 80, 65, 76, 73, 4, 138, 171, 21, 78, 211, 57, 76, 8, 50, 84, 220, - 145, 17, 2, 67, 65, 139, 238, 4, 69, 4, 134, 228, 21, 72, 247, 30, 73, - 32, 106, 65, 44, 5, 77, 65, 73, 32, 75, 146, 226, 17, 89, 142, 76, 85, - 206, 201, 1, 69, 2, 73, 223, 137, 2, 79, 11, 238, 129, 22, 65, 2, 73, 2, - 77, 3, 89, 4, 226, 177, 21, 65, 3, 79, 166, 1, 32, 2, 71, 69, 131, 250, - 20, 73, 164, 1, 26, 32, 199, 239, 13, 82, 160, 1, 254, 1, 66, 44, 4, 71, - 82, 69, 69, 22, 79, 250, 1, 84, 182, 231, 11, 68, 36, 2, 85, 80, 176, - 192, 3, 12, 76, 69, 70, 84, 32, 84, 82, 73, 65, 78, 71, 76, 164, 193, 4, - 5, 80, 85, 82, 80, 76, 12, 3, 82, 69, 68, 0, 6, 89, 69, 76, 76, 79, 87, - 243, 64, 67, 10, 40, 3, 82, 79, 87, 201, 1, 2, 76, 85, 4, 195, 235, 19, - 78, 10, 48, 3, 78, 69, 32, 129, 1, 4, 82, 65, 78, 71, 4, 160, 155, 8, 5, - 68, 79, 84, 32, 79, 181, 233, 2, 18, 82, 73, 78, 71, 32, 79, 86, 69, 82, - 32, 84, 87, 79, 32, 82, 73, 78, 71, 6, 11, 69, 6, 11, 32, 6, 210, 170, - 20, 67, 154, 21, 68, 139, 28, 83, 116, 156, 1, 3, 87, 79, 32, 108, 10, - 89, 80, 69, 32, 80, 73, 69, 67, 69, 32, 157, 186, 18, 17, 82, 73, 80, 76, - 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 4, 150, 225, 17, 68, - 193, 87, 19, 82, 73, 78, 71, 83, 32, 79, 86, 69, 82, 32, 79, 78, 69, 32, - 82, 73, 78, 71, 110, 182, 1, 67, 136, 2, 9, 68, 73, 65, 71, 79, 78, 65, - 76, 32, 218, 1, 76, 186, 2, 82, 158, 1, 83, 204, 3, 6, 85, 80, 80, 69, - 82, 32, 145, 2, 9, 86, 69, 82, 84, 69, 88, 32, 79, 70, 14, 80, 9, 69, 78, - 84, 82, 69, 32, 79, 70, 32, 73, 7, 82, 79, 83, 83, 66, 65, 82, 8, 252, 8, - 6, 90, 32, 87, 73, 84, 72, 142, 239, 21, 75, 2, 88, 3, 89, 7, 33, 6, 32, - 87, 73, 84, 72, 32, 4, 40, 3, 76, 79, 87, 1, 3, 85, 80, 80, 2, 181, 197, - 10, 2, 69, 82, 12, 48, 6, 85, 80, 80, 69, 82, 32, 227, 222, 9, 76, 8, 56, - 4, 76, 69, 70, 84, 157, 223, 9, 4, 82, 73, 71, 72, 5, 11, 32, 2, 21, 3, - 65, 78, 68, 2, 17, 2, 32, 76, 2, 17, 2, 79, 87, 2, 213, 194, 21, 2, 69, - 82, 26, 48, 5, 79, 87, 69, 82, 32, 129, 3, 2, 69, 70, 24, 84, 5, 76, 69, - 70, 84, 32, 56, 6, 82, 73, 71, 72, 84, 32, 158, 6, 72, 179, 1, 84, 8, 22, - 65, 207, 7, 67, 4, 194, 1, 78, 159, 202, 20, 82, 10, 22, 65, 151, 7, 67, - 6, 192, 1, 13, 78, 68, 32, 85, 80, 80, 69, 82, 32, 82, 73, 71, 72, 193, - 189, 19, 2, 82, 67, 4, 44, 4, 65, 73, 83, 69, 77, 3, 73, 71, 72, 2, 53, - 11, 68, 32, 85, 80, 80, 69, 82, 32, 76, 69, 70, 2, 187, 245, 18, 84, 2, - 141, 243, 20, 3, 84, 32, 65, 34, 48, 5, 72, 79, 82, 84, 32, 77, 3, 84, - 69, 77, 4, 40, 3, 76, 79, 87, 1, 3, 85, 80, 80, 2, 217, 4, 4, 69, 82, 32, - 84, 31, 44, 6, 32, 87, 73, 84, 72, 32, 171, 1, 45, 8, 44, 5, 76, 69, 70, - 84, 32, 30, 82, 59, 67, 4, 82, 67, 167, 214, 10, 74, 2, 21, 3, 73, 71, - 72, 2, 11, 84, 2, 17, 2, 32, 67, 2, 177, 205, 20, 4, 82, 79, 83, 83, 20, - 48, 2, 49, 50, 22, 50, 26, 52, 247, 181, 11, 51, 5, 207, 211, 14, 51, 9, - 11, 51, 7, 11, 52, 5, 243, 237, 21, 53, 18, 62, 72, 96, 3, 76, 69, 70, 0, - 4, 82, 73, 71, 72, 83, 84, 4, 65, 14, 65, 76, 70, 32, 86, 69, 82, 84, 69, - 88, 32, 79, 70, 32, 4, 218, 236, 21, 77, 3, 87, 6, 17, 2, 84, 32, 6, 26, - 67, 131, 240, 18, 65, 4, 186, 92, 82, 211, 147, 17, 79, 2, 201, 243, 8, - 3, 69, 82, 77, 2, 211, 219, 14, 32, 6, 53, 11, 85, 65, 82, 84, 69, 82, - 32, 77, 79, 79, 78, 7, 151, 234, 6, 32, 150, 20, 150, 1, 67, 148, 46, 18, - 69, 80, 73, 71, 82, 65, 80, 72, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, - 252, 1, 7, 76, 69, 84, 84, 69, 82, 32, 183, 13, 83, 200, 7, 56, 8, 65, - 80, 73, 84, 65, 76, 32, 76, 139, 168, 20, 82, 198, 7, 76, 6, 69, 84, 84, - 69, 82, 32, 221, 44, 8, 73, 71, 65, 84, 85, 82, 69, 32, 194, 7, 154, 2, - 65, 158, 3, 66, 154, 1, 67, 254, 1, 68, 250, 1, 69, 174, 2, 70, 82, 71, - 194, 1, 72, 146, 1, 73, 154, 2, 74, 118, 75, 126, 76, 234, 2, 77, 114, - 78, 134, 2, 79, 198, 2, 80, 102, 81, 62, 82, 190, 2, 83, 130, 3, 84, 142, - 3, 85, 234, 1, 86, 146, 1, 87, 118, 88, 50, 89, 167, 1, 90, 93, 172, 1, - 6, 32, 87, 73, 84, 72, 32, 166, 1, 69, 230, 65, 78, 216, 146, 14, 6, 70, - 82, 73, 67, 65, 78, 222, 182, 3, 76, 130, 182, 2, 86, 182, 162, 1, 65, 2, - 79, 2, 85, 3, 89, 66, 230, 64, 66, 34, 68, 108, 3, 82, 73, 78, 242, 33, - 77, 250, 21, 67, 162, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, - 168, 3, 73, 186, 159, 13, 84, 227, 179, 3, 83, 7, 33, 6, 32, 87, 73, 84, - 72, 32, 4, 202, 183, 1, 65, 187, 194, 3, 77, 21, 60, 6, 32, 87, 73, 84, - 72, 32, 174, 68, 82, 207, 249, 20, 69, 14, 138, 67, 84, 142, 60, 70, 218, - 56, 76, 226, 143, 16, 68, 162, 199, 2, 72, 219, 163, 1, 83, 33, 108, 6, - 32, 87, 73, 84, 72, 32, 188, 70, 7, 76, 79, 83, 69, 68, 32, 73, 54, 85, - 174, 202, 20, 79, 139, 60, 72, 20, 94, 67, 194, 180, 1, 65, 230, 227, 3, - 80, 166, 244, 14, 72, 158, 50, 66, 238, 32, 68, 211, 80, 83, 8, 182, 68, - 69, 202, 112, 73, 255, 133, 19, 65, 31, 44, 6, 32, 87, 73, 84, 72, 32, - 179, 1, 90, 24, 78, 83, 194, 15, 67, 250, 47, 84, 230, 116, 76, 226, 143, - 16, 68, 163, 199, 2, 72, 8, 92, 13, 77, 65, 76, 76, 32, 76, 69, 84, 84, - 69, 82, 32, 90, 178, 137, 1, 72, 243, 164, 20, 84, 5, 225, 72, 2, 32, 87, - 91, 104, 6, 32, 87, 73, 84, 72, 32, 152, 1, 2, 90, 72, 182, 76, 71, 166, - 239, 16, 84, 142, 159, 4, 83, 63, 78, 70, 174, 74, 67, 158, 2, 68, 194, - 16, 84, 158, 23, 77, 250, 1, 86, 250, 44, 72, 158, 1, 79, 198, 10, 71, - 186, 2, 65, 202, 168, 3, 73, 62, 66, 223, 210, 16, 83, 7, 11, 32, 4, 186, - 70, 87, 211, 8, 82, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 210, 135, 20, - 72, 138, 83, 68, 211, 80, 83, 35, 76, 6, 32, 87, 73, 84, 72, 32, 238, 81, - 76, 202, 182, 18, 65, 195, 207, 2, 72, 20, 202, 84, 67, 134, 90, 65, 134, - 169, 3, 66, 182, 25, 77, 198, 140, 4, 79, 142, 137, 11, 72, 138, 83, 68, - 211, 80, 83, 29, 76, 6, 32, 87, 73, 84, 72, 32, 182, 126, 65, 246, 238, - 7, 87, 207, 154, 12, 69, 20, 234, 82, 66, 34, 67, 46, 68, 226, 177, 19, - 72, 219, 163, 1, 83, 59, 80, 6, 32, 87, 73, 84, 72, 32, 180, 88, 2, 78, - 83, 218, 218, 20, 79, 207, 36, 83, 40, 138, 1, 68, 210, 83, 67, 242, 1, - 77, 142, 1, 84, 142, 70, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, - 168, 3, 73, 62, 66, 223, 210, 16, 83, 10, 22, 79, 239, 83, 73, 6, 150, - 121, 85, 231, 190, 18, 84, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, 67, - 234, 241, 17, 84, 227, 179, 3, 83, 4, 166, 170, 1, 73, 207, 217, 3, 82, - 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 230, 89, 67, 34, 68, 50, 83, 234, - 78, 65, 138, 1, 76, 246, 205, 7, 79, 143, 137, 11, 72, 41, 60, 6, 32, 87, - 73, 84, 72, 32, 238, 94, 65, 251, 244, 20, 74, 32, 138, 1, 66, 36, 2, 68, - 79, 52, 7, 77, 73, 68, 68, 76, 69, 32, 38, 83, 174, 2, 67, 230, 91, 72, - 242, 71, 65, 138, 1, 76, 251, 198, 16, 84, 4, 242, 186, 12, 69, 139, 223, - 8, 65, 6, 22, 85, 151, 92, 84, 2, 129, 236, 4, 2, 66, 76, 4, 162, 238, - 17, 84, 167, 147, 3, 68, 4, 214, 2, 77, 151, 159, 21, 84, 19, 44, 6, 32, - 87, 73, 84, 72, 32, 255, 94, 73, 10, 174, 165, 1, 65, 234, 144, 16, 68, - 154, 55, 84, 139, 144, 2, 72, 33, 48, 6, 32, 87, 73, 84, 72, 32, 155, - 208, 21, 74, 28, 102, 67, 44, 2, 83, 77, 226, 96, 76, 146, 64, 71, 186, - 2, 65, 102, 68, 154, 206, 7, 79, 135, 249, 8, 84, 6, 250, 131, 1, 69, 22, - 73, 131, 166, 19, 65, 2, 189, 226, 10, 10, 65, 76, 76, 32, 76, 69, 84, - 84, 69, 82, 101, 108, 6, 32, 87, 73, 84, 72, 32, 226, 103, 76, 226, 152, - 4, 80, 170, 150, 9, 77, 242, 182, 7, 73, 2, 79, 3, 85, 84, 144, 1, 2, 76, - 79, 34, 77, 146, 97, 67, 70, 68, 154, 2, 79, 38, 83, 66, 84, 106, 86, - 234, 43, 72, 242, 12, 71, 186, 2, 65, 202, 168, 3, 73, 63, 66, 4, 206, - 99, 78, 235, 232, 20, 79, 8, 202, 99, 65, 195, 153, 4, 73, 17, 33, 6, 32, - 87, 73, 84, 72, 32, 14, 178, 104, 70, 34, 83, 178, 55, 65, 138, 216, 18, - 72, 139, 83, 68, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 134, 105, 68, 39, - 83, 43, 94, 32, 156, 1, 8, 69, 86, 69, 82, 83, 69, 68, 32, 160, 111, 3, - 85, 77, 32, 243, 141, 4, 65, 28, 40, 5, 87, 73, 84, 72, 32, 159, 112, 82, - 26, 194, 78, 67, 230, 28, 68, 170, 2, 84, 174, 48, 65, 138, 1, 76, 194, - 167, 3, 73, 182, 166, 4, 79, 231, 172, 12, 83, 8, 162, 110, 72, 230, 150, - 4, 79, 202, 183, 13, 67, 155, 140, 3, 69, 49, 136, 1, 6, 32, 87, 73, 84, - 72, 32, 136, 1, 5, 77, 65, 76, 76, 32, 184, 115, 2, 65, 76, 234, 1, 72, - 152, 2, 2, 73, 71, 243, 132, 4, 67, 32, 86, 67, 210, 111, 65, 118, 68, - 138, 1, 83, 174, 1, 86, 238, 246, 7, 79, 143, 137, 11, 72, 10, 170, 112, - 65, 230, 10, 69, 82, 79, 203, 31, 73, 4, 152, 146, 19, 11, 81, 32, 87, - 73, 84, 72, 32, 72, 79, 79, 75, 233, 243, 1, 7, 67, 65, 80, 73, 84, 65, - 76, 59, 136, 1, 6, 32, 87, 73, 84, 72, 32, 168, 1, 6, 85, 82, 78, 69, 68, - 32, 132, 123, 2, 72, 79, 176, 1, 2, 79, 78, 74, 82, 251, 197, 20, 90, 22, - 82, 67, 50, 68, 198, 152, 1, 76, 182, 232, 3, 82, 206, 238, 14, 72, 219, - 163, 1, 83, 8, 146, 120, 69, 22, 73, 62, 79, 199, 165, 19, 65, 6, 134, - 142, 1, 73, 171, 154, 16, 79, 18, 178, 40, 73, 146, 133, 3, 65, 178, 149, - 18, 72, 2, 75, 2, 76, 2, 77, 2, 84, 3, 86, 79, 26, 32, 203, 129, 5, 80, - 74, 44, 5, 87, 73, 84, 72, 32, 155, 160, 20, 66, 72, 238, 131, 1, 67, 74, - 68, 150, 2, 72, 170, 1, 77, 134, 1, 79, 142, 1, 84, 186, 9, 71, 186, 2, - 65, 202, 168, 3, 73, 62, 66, 242, 209, 14, 82, 239, 128, 2, 83, 23, 88, - 6, 32, 87, 73, 84, 72, 32, 250, 138, 1, 73, 66, 79, 170, 166, 19, 69, - 251, 141, 1, 89, 8, 170, 138, 1, 68, 210, 209, 16, 84, 139, 144, 2, 72, - 19, 48, 6, 32, 87, 73, 84, 72, 32, 251, 145, 7, 89, 14, 134, 144, 1, 67, - 18, 68, 70, 71, 186, 2, 65, 139, 216, 18, 72, 7, 145, 140, 1, 7, 32, 87, - 73, 84, 72, 32, 68, 29, 48, 6, 32, 87, 73, 84, 72, 32, 135, 153, 17, 79, - 24, 226, 142, 1, 67, 18, 68, 70, 71, 22, 72, 42, 76, 254, 1, 65, 186, - 194, 3, 77, 202, 133, 13, 84, 227, 179, 3, 83, 25, 33, 6, 32, 87, 73, 84, - 72, 32, 22, 186, 57, 67, 162, 87, 65, 102, 68, 38, 76, 34, 83, 190, 226, - 3, 80, 167, 244, 14, 72, 4, 170, 207, 10, 73, 231, 213, 10, 79, 12, 128, - 1, 5, 65, 82, 67, 72, 65, 30, 73, 64, 9, 82, 69, 86, 69, 82, 83, 69, 68, - 32, 245, 204, 10, 7, 83, 73, 68, 69, 87, 65, 89, 2, 177, 235, 13, 2, 73, - 67, 4, 164, 107, 5, 78, 86, 69, 82, 84, 221, 205, 14, 3, 32, 76, 79, 4, - 222, 185, 21, 70, 3, 80, 138, 1, 198, 3, 65, 36, 2, 66, 73, 152, 1, 21, - 73, 78, 86, 69, 82, 84, 69, 68, 32, 71, 76, 79, 84, 84, 65, 76, 32, 83, - 84, 79, 80, 72, 2, 82, 69, 222, 1, 83, 128, 8, 3, 84, 87, 79, 178, 20, - 87, 216, 163, 4, 2, 68, 69, 148, 5, 2, 76, 65, 206, 11, 71, 128, 176, 1, - 21, 80, 72, 65, 82, 89, 78, 71, 69, 65, 76, 32, 86, 79, 73, 67, 69, 68, - 32, 70, 82, 73, 164, 131, 14, 20, 86, 79, 73, 67, 69, 68, 32, 76, 65, 82, - 89, 78, 71, 69, 65, 76, 32, 83, 80, 73, 179, 117, 89, 4, 174, 181, 4, 76, - 235, 176, 16, 73, 6, 76, 7, 76, 65, 66, 73, 65, 76, 32, 29, 8, 68, 69, - 78, 84, 65, 76, 32, 80, 4, 26, 80, 175, 200, 4, 67, 2, 161, 129, 16, 6, - 69, 82, 67, 85, 83, 83, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 170, 237, 4, - 67, 243, 150, 16, 83, 8, 104, 7, 86, 69, 82, 83, 69, 68, 32, 185, 237, 4, - 13, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, 75, 4, 80, 3, 69, 83, - 72, 157, 8, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 2, 217, - 132, 1, 2, 32, 76, 96, 172, 1, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, - 84, 65, 76, 32, 248, 111, 10, 84, 82, 69, 84, 67, 72, 69, 68, 32, 67, - 221, 175, 19, 10, 73, 78, 79, 76, 79, 71, 73, 67, 65, 76, 90, 230, 1, 69, - 30, 73, 22, 76, 74, 79, 42, 82, 122, 84, 210, 176, 4, 66, 214, 41, 71, - 166, 146, 16, 65, 162, 64, 67, 2, 68, 2, 70, 2, 72, 2, 74, 2, 75, 2, 77, - 2, 78, 2, 80, 2, 81, 2, 83, 2, 85, 2, 86, 2, 87, 2, 89, 3, 90, 7, 238, - 173, 21, 84, 3, 90, 5, 227, 213, 4, 78, 7, 33, 6, 32, 87, 73, 84, 72, 32, - 4, 162, 219, 8, 66, 143, 163, 12, 83, 9, 242, 94, 80, 150, 207, 20, 69, - 3, 85, 11, 88, 8, 69, 86, 69, 82, 83, 69, 68, 32, 244, 127, 5, 32, 87, - 73, 84, 72, 187, 156, 20, 85, 4, 130, 173, 21, 78, 3, 82, 13, 33, 6, 85, - 82, 78, 69, 68, 32, 10, 194, 172, 21, 69, 2, 71, 2, 75, 2, 77, 3, 82, - 184, 11, 136, 1, 5, 77, 65, 76, 76, 32, 153, 130, 1, 22, 85, 66, 83, 67, - 82, 73, 80, 84, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, - 148, 11, 76, 15, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, - 32, 43, 76, 4, 18, 73, 3, 85, 2, 211, 220, 4, 32, 144, 11, 76, 6, 69, 84, - 84, 69, 82, 32, 173, 127, 8, 73, 71, 65, 84, 85, 82, 69, 32, 254, 10, - 182, 2, 65, 190, 5, 66, 198, 3, 67, 186, 4, 68, 182, 4, 69, 174, 9, 70, - 214, 1, 71, 162, 2, 72, 230, 2, 73, 166, 7, 74, 182, 1, 75, 178, 2, 76, - 206, 6, 77, 178, 2, 78, 218, 3, 79, 242, 8, 80, 230, 1, 81, 174, 1, 82, - 142, 8, 83, 250, 10, 84, 246, 13, 85, 218, 9, 86, 130, 3, 87, 110, 88, - 226, 2, 89, 171, 3, 90, 101, 118, 32, 198, 3, 69, 82, 78, 152, 220, 4, 4, - 76, 80, 72, 65, 158, 163, 15, 86, 182, 162, 1, 65, 2, 79, 2, 85, 3, 89, - 72, 88, 5, 87, 73, 84, 72, 32, 225, 222, 5, 11, 82, 69, 86, 69, 82, 83, - 69, 68, 45, 83, 67, 70, 150, 1, 66, 34, 68, 54, 82, 170, 34, 77, 250, 21, - 67, 162, 49, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 202, 168, 3, 73, - 186, 159, 13, 84, 227, 179, 3, 83, 12, 173, 105, 4, 82, 69, 86, 69, 12, - 22, 79, 151, 57, 73, 8, 214, 57, 84, 223, 12, 85, 10, 26, 73, 135, 225, - 4, 69, 8, 26, 78, 139, 168, 4, 71, 6, 17, 2, 71, 32, 6, 236, 58, 4, 65, - 66, 79, 86, 235, 201, 18, 66, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 254, - 115, 71, 186, 2, 65, 187, 194, 3, 77, 2, 165, 191, 10, 7, 71, 76, 73, 67, - 65, 78, 65, 41, 140, 1, 6, 32, 87, 73, 84, 72, 32, 130, 1, 65, 108, 11, - 76, 65, 67, 75, 76, 69, 84, 84, 69, 82, 32, 38, 82, 134, 195, 4, 79, 203, - 182, 16, 69, 18, 110, 84, 142, 60, 70, 218, 56, 76, 178, 219, 3, 77, 174, - 7, 80, 134, 173, 12, 68, 162, 199, 2, 72, 219, 163, 1, 83, 2, 255, 7, 79, - 8, 64, 5, 82, 82, 69, 68, 32, 173, 80, 6, 83, 69, 76, 73, 78, 69, 6, 182, - 32, 65, 174, 254, 20, 69, 3, 79, 6, 202, 189, 4, 79, 243, 224, 16, 69, 2, - 165, 179, 10, 4, 79, 75, 69, 78, 47, 108, 6, 32, 87, 73, 84, 72, 32, 196, - 1, 2, 72, 73, 92, 6, 76, 79, 83, 69, 68, 32, 90, 85, 175, 202, 20, 79, - 24, 102, 67, 194, 112, 65, 230, 227, 3, 80, 218, 5, 82, 206, 238, 14, 72, - 158, 50, 66, 238, 32, 68, 211, 80, 83, 10, 54, 69, 202, 112, 73, 230, - 245, 6, 85, 155, 144, 12, 65, 4, 245, 51, 5, 68, 73, 76, 76, 65, 7, 49, - 10, 32, 87, 73, 84, 72, 32, 76, 79, 87, 32, 4, 174, 106, 82, 45, 4, 76, - 69, 70, 84, 8, 34, 73, 18, 79, 247, 193, 4, 82, 2, 175, 87, 78, 4, 218, - 214, 4, 80, 231, 140, 9, 77, 4, 37, 7, 65, 84, 82, 73, 76, 76, 79, 5, - 253, 130, 13, 5, 32, 87, 73, 84, 72, 73, 96, 6, 32, 87, 73, 84, 72, 32, - 182, 1, 69, 34, 79, 178, 1, 90, 138, 207, 4, 66, 247, 181, 16, 85, 32, - 98, 83, 34, 84, 238, 32, 67, 238, 44, 77, 170, 31, 76, 214, 211, 3, 72, - 138, 15, 80, 135, 173, 12, 68, 4, 146, 67, 72, 243, 164, 20, 84, 4, 26, - 79, 231, 227, 18, 65, 2, 243, 245, 19, 80, 8, 130, 78, 90, 215, 164, 20, - 76, 16, 60, 6, 84, 76, 69, 83, 83, 32, 49, 5, 85, 66, 76, 69, 32, 8, 26, - 74, 171, 150, 21, 73, 7, 255, 191, 4, 32, 8, 42, 87, 230, 195, 4, 82, - 199, 238, 5, 84, 2, 239, 232, 6, 89, 11, 11, 32, 8, 26, 87, 239, 191, 4, - 68, 2, 181, 89, 5, 73, 84, 72, 32, 67, 119, 112, 6, 32, 87, 73, 84, 72, - 32, 214, 4, 71, 72, 2, 78, 71, 68, 2, 83, 72, 164, 1, 2, 90, 72, 247, - 236, 16, 84, 76, 182, 1, 67, 158, 2, 68, 110, 78, 214, 15, 84, 158, 23, - 77, 250, 1, 86, 194, 3, 70, 186, 41, 72, 158, 1, 79, 198, 10, 71, 186, 2, - 65, 202, 168, 3, 73, 62, 66, 186, 64, 82, 167, 146, 16, 83, 24, 92, 6, - 69, 68, 73, 76, 76, 65, 32, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 191, - 236, 19, 65, 5, 137, 143, 4, 3, 32, 65, 78, 19, 11, 32, 16, 40, 4, 65, - 78, 68, 32, 151, 243, 18, 66, 14, 174, 85, 67, 130, 2, 72, 226, 11, 71, - 186, 2, 65, 186, 194, 3, 77, 250, 169, 4, 68, 211, 219, 8, 84, 12, 22, - 79, 239, 97, 73, 10, 28, 2, 84, 32, 239, 50, 85, 8, 204, 87, 5, 65, 66, - 79, 86, 69, 171, 154, 18, 66, 2, 167, 142, 14, 79, 4, 221, 255, 6, 13, - 89, 80, 84, 79, 76, 79, 71, 73, 67, 65, 76, 32, 65, 7, 33, 6, 32, 87, 73, - 84, 72, 32, 4, 246, 188, 4, 67, 231, 9, 80, 13, 33, 6, 32, 87, 73, 84, - 72, 32, 10, 88, 10, 68, 79, 85, 66, 76, 69, 32, 66, 65, 82, 190, 197, 4, - 80, 142, 1, 67, 207, 4, 82, 5, 177, 198, 4, 4, 32, 65, 78, 68, 15, 11, - 32, 12, 38, 82, 37, 5, 87, 73, 84, 72, 32, 2, 193, 138, 14, 4, 69, 86, - 69, 82, 10, 54, 67, 138, 196, 4, 80, 218, 5, 82, 195, 142, 14, 84, 4, - 198, 214, 7, 85, 155, 144, 12, 65, 17, 84, 6, 32, 87, 73, 84, 72, 32, 73, - 11, 69, 78, 71, 32, 68, 73, 71, 82, 65, 80, 72, 10, 222, 187, 4, 77, 174, - 7, 80, 166, 244, 14, 72, 138, 83, 68, 211, 80, 83, 5, 181, 146, 18, 8, - 32, 87, 73, 84, 72, 32, 84, 82, 37, 72, 6, 32, 87, 73, 84, 72, 32, 126, - 76, 202, 182, 18, 65, 195, 207, 2, 72, 22, 218, 3, 67, 134, 90, 65, 134, - 169, 3, 66, 182, 25, 77, 174, 33, 80, 154, 235, 3, 79, 142, 137, 11, 72, - 138, 83, 68, 211, 80, 83, 8, 33, 6, 79, 84, 84, 65, 76, 32, 8, 242, 182, - 18, 83, 170, 209, 2, 65, 2, 73, 3, 85, 39, 140, 1, 6, 32, 87, 73, 84, 72, - 32, 162, 44, 65, 232, 25, 12, 79, 79, 75, 69, 68, 32, 83, 67, 72, 87, 65, - 32, 250, 237, 3, 69, 219, 210, 16, 86, 24, 86, 66, 34, 67, 46, 68, 226, - 90, 76, 222, 226, 3, 80, 166, 244, 14, 72, 219, 163, 1, 83, 2, 233, 135, - 13, 3, 82, 69, 86, 6, 170, 58, 69, 154, 32, 73, 255, 133, 19, 65, 8, 246, - 86, 73, 174, 185, 3, 69, 171, 218, 12, 79, 75, 80, 6, 32, 87, 73, 84, 72, - 32, 222, 4, 78, 188, 1, 2, 79, 84, 155, 254, 20, 83, 48, 178, 1, 67, 34, - 68, 210, 1, 77, 64, 6, 79, 71, 79, 78, 69, 75, 78, 84, 142, 70, 72, 226, - 11, 71, 186, 2, 65, 202, 168, 3, 73, 62, 66, 136, 64, 6, 83, 84, 82, 79, - 75, 69, 51, 82, 4, 222, 87, 73, 255, 133, 19, 65, 14, 18, 73, 47, 79, 4, - 217, 26, 7, 65, 69, 82, 69, 83, 73, 83, 10, 28, 2, 84, 32, 227, 36, 85, - 8, 64, 10, 65, 66, 79, 86, 69, 32, 65, 78, 68, 32, 171, 227, 18, 66, 6, - 162, 83, 71, 186, 2, 65, 131, 200, 16, 84, 4, 29, 5, 65, 67, 82, 79, 78, - 5, 229, 35, 4, 32, 65, 78, 68, 7, 157, 72, 15, 32, 65, 78, 68, 32, 68, - 79, 84, 32, 65, 66, 79, 86, 69, 32, 4, 21, 3, 73, 76, 68, 4, 211, 135, 5, - 69, 16, 46, 83, 93, 7, 86, 69, 82, 84, 69, 68, 32, 12, 29, 5, 85, 76, 65, - 82, 32, 12, 130, 255, 20, 68, 2, 70, 2, 71, 2, 82, 2, 83, 3, 84, 4, 26, - 65, 219, 231, 20, 79, 2, 247, 165, 17, 76, 6, 166, 157, 4, 65, 129, 186, - 6, 5, 73, 70, 73, 69, 68, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 94, 67, - 236, 173, 4, 13, 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, 65, 78, 68, 247, - 158, 16, 83, 6, 190, 81, 73, 206, 217, 3, 82, 179, 172, 15, 65, 29, 48, - 6, 32, 87, 73, 84, 72, 32, 195, 249, 20, 82, 24, 98, 67, 34, 68, 50, 83, - 234, 78, 65, 138, 1, 76, 222, 226, 3, 80, 154, 235, 3, 79, 143, 137, 11, - 72, 4, 222, 47, 69, 151, 166, 19, 65, 6, 226, 69, 73, 130, 192, 3, 69, - 243, 181, 4, 79, 4, 29, 5, 84, 82, 79, 75, 69, 5, 173, 24, 6, 32, 65, 78, - 68, 32, 68, 79, 136, 1, 6, 32, 87, 73, 84, 72, 32, 250, 3, 65, 38, 69, - 48, 5, 79, 78, 71, 32, 83, 218, 173, 4, 83, 2, 90, 246, 181, 16, 85, 219, - 16, 74, 50, 178, 1, 66, 86, 67, 56, 2, 68, 79, 100, 3, 77, 73, 68, 130, - 2, 72, 242, 71, 65, 138, 1, 76, 226, 218, 3, 73, 162, 1, 82, 222, 2, 70, - 130, 4, 80, 158, 228, 12, 84, 227, 179, 3, 83, 6, 36, 3, 69, 76, 84, 187, - 190, 20, 65, 5, 153, 175, 4, 6, 32, 65, 78, 68, 32, 80, 8, 178, 43, 69, - 22, 73, 234, 149, 7, 85, 155, 144, 12, 65, 8, 38, 84, 25, 5, 85, 66, 76, - 69, 32, 4, 153, 24, 2, 32, 66, 4, 202, 166, 4, 77, 239, 173, 15, 66, 8, - 36, 4, 68, 76, 69, 32, 219, 43, 45, 6, 198, 145, 17, 84, 218, 146, 3, 82, - 79, 68, 4, 133, 148, 4, 4, 77, 66, 68, 65, 6, 246, 174, 4, 90, 205, 250, - 12, 3, 78, 73, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 18, 68, 35, 72, - 4, 218, 62, 73, 211, 180, 19, 79, 2, 157, 166, 4, 2, 73, 71, 27, 56, 6, - 32, 87, 73, 84, 72, 32, 102, 73, 187, 225, 20, 85, 16, 150, 71, 65, 130, - 218, 3, 67, 186, 2, 77, 174, 7, 80, 134, 173, 12, 68, 154, 55, 84, 139, - 144, 2, 72, 6, 25, 4, 68, 68, 76, 69, 6, 76, 7, 45, 87, 69, 76, 83, 72, - 32, 149, 252, 16, 6, 32, 83, 67, 79, 84, 83, 4, 238, 231, 19, 76, 183, - 137, 1, 86, 49, 58, 32, 216, 2, 2, 71, 32, 150, 221, 20, 85, 219, 16, 74, - 40, 88, 5, 87, 73, 84, 72, 32, 153, 136, 6, 11, 80, 82, 69, 67, 69, 68, - 69, 68, 32, 66, 89, 38, 122, 67, 74, 76, 170, 36, 77, 234, 27, 71, 186, - 2, 65, 102, 68, 130, 227, 3, 80, 218, 5, 82, 194, 229, 3, 79, 135, 249, - 8, 84, 10, 182, 35, 69, 22, 73, 210, 249, 3, 82, 154, 156, 3, 85, 155, - 144, 12, 65, 6, 144, 65, 3, 79, 78, 71, 202, 2, 73, 131, 234, 3, 69, 2, - 25, 4, 87, 73, 84, 72, 2, 153, 202, 17, 5, 32, 84, 73, 76, 68, 115, 116, - 6, 32, 87, 73, 84, 72, 32, 186, 6, 76, 52, 4, 80, 69, 78, 32, 214, 174, - 13, 77, 242, 182, 7, 73, 2, 79, 3, 85, 86, 154, 1, 67, 70, 68, 152, 1, 2, - 76, 79, 86, 77, 46, 79, 38, 83, 66, 84, 106, 86, 234, 43, 72, 242, 12, - 71, 186, 2, 65, 202, 168, 3, 73, 62, 66, 187, 64, 82, 14, 184, 48, 9, 73, - 82, 67, 85, 77, 70, 76, 69, 88, 187, 149, 19, 65, 14, 18, 73, 47, 79, 4, - 233, 12, 7, 65, 69, 82, 69, 83, 73, 83, 10, 22, 84, 183, 46, 85, 6, 11, - 32, 6, 152, 12, 5, 65, 66, 79, 86, 69, 195, 191, 18, 66, 6, 66, 78, 132, - 197, 12, 6, 87, 32, 82, 73, 78, 71, 231, 163, 8, 79, 2, 171, 20, 71, 6, - 11, 65, 6, 189, 2, 4, 67, 82, 79, 78, 4, 229, 10, 5, 71, 79, 78, 69, 75, - 4, 25, 4, 84, 82, 79, 75, 4, 11, 69, 5, 225, 48, 2, 32, 65, 8, 25, 4, 73, - 76, 68, 69, 9, 29, 5, 32, 65, 78, 68, 32, 6, 174, 46, 68, 142, 13, 65, - 187, 194, 3, 77, 6, 81, 18, 69, 82, 84, 73, 67, 65, 76, 32, 76, 73, 78, - 69, 32, 66, 69, 76, 79, 87, 7, 233, 42, 4, 32, 65, 78, 68, 2, 137, 129, - 10, 8, 68, 32, 80, 79, 76, 73, 83, 72, 16, 26, 79, 219, 159, 4, 69, 13, - 48, 6, 32, 87, 73, 84, 72, 32, 247, 228, 20, 69, 8, 222, 54, 71, 186, 2, - 65, 190, 233, 3, 82, 167, 146, 16, 83, 23, 48, 6, 32, 87, 73, 84, 72, 32, - 175, 208, 20, 72, 18, 86, 70, 34, 83, 178, 55, 65, 186, 220, 3, 77, 174, - 7, 80, 166, 244, 14, 72, 139, 83, 68, 2, 137, 185, 14, 3, 76, 79, 85, 6, - 218, 27, 84, 241, 164, 12, 6, 81, 85, 73, 82, 82, 69, 13, 48, 6, 32, 87, - 73, 84, 72, 32, 215, 155, 4, 80, 8, 42, 68, 16, 4, 72, 79, 79, 75, 23, - 83, 2, 227, 44, 73, 5, 247, 173, 18, 32, 2, 197, 26, 6, 84, 82, 79, 75, - 69, 32, 77, 90, 32, 228, 4, 8, 69, 86, 69, 82, 83, 69, 68, 32, 148, 2, 2, - 85, 77, 255, 141, 4, 65, 46, 36, 4, 87, 73, 84, 72, 235, 6, 82, 44, 26, - 32, 215, 174, 15, 79, 42, 166, 1, 67, 50, 68, 200, 1, 8, 70, 73, 83, 72, - 72, 79, 79, 75, 66, 76, 34, 84, 142, 18, 77, 162, 30, 65, 202, 168, 3, - 73, 158, 59, 80, 154, 235, 3, 79, 231, 172, 12, 83, 6, 170, 19, 69, 230, - 249, 3, 82, 179, 172, 15, 65, 8, 11, 79, 8, 24, 2, 84, 32, 111, 85, 6, - 26, 66, 131, 225, 19, 65, 4, 25, 4, 69, 76, 79, 87, 5, 29, 5, 32, 65, 78, - 68, 32, 2, 139, 244, 3, 77, 2, 21, 3, 66, 76, 69, 2, 11, 32, 2, 227, 46, - 71, 7, 29, 5, 32, 65, 78, 68, 32, 4, 162, 141, 4, 77, 175, 7, 80, 4, 222, - 49, 73, 207, 230, 3, 79, 4, 178, 168, 18, 65, 163, 26, 73, 22, 162, 1, - 72, 40, 6, 79, 80, 69, 78, 32, 69, 164, 142, 4, 8, 82, 32, 87, 73, 84, - 72, 32, 70, 168, 2, 3, 83, 67, 82, 226, 206, 14, 69, 234, 86, 67, 139, - 164, 1, 75, 2, 11, 65, 2, 225, 244, 9, 2, 76, 70, 7, 33, 6, 32, 87, 73, - 84, 72, 32, 4, 214, 151, 4, 82, 207, 238, 14, 72, 5, 11, 32, 2, 11, 82, - 2, 149, 253, 16, 3, 79, 84, 85, 85, 180, 1, 6, 32, 87, 73, 84, 72, 32, - 218, 4, 65, 66, 67, 218, 1, 72, 34, 73, 156, 2, 13, 81, 85, 65, 84, 32, - 82, 69, 86, 69, 82, 83, 69, 68, 193, 229, 9, 6, 84, 73, 82, 82, 85, 80, - 40, 110, 65, 34, 67, 86, 68, 138, 1, 83, 174, 1, 86, 210, 9, 77, 134, - 130, 4, 80, 154, 235, 3, 79, 143, 137, 11, 72, 4, 205, 1, 4, 67, 85, 84, - 69, 12, 58, 65, 230, 10, 69, 82, 79, 202, 31, 73, 231, 245, 6, 85, 4, - 113, 3, 82, 79, 78, 8, 32, 3, 79, 84, 32, 207, 32, 73, 6, 26, 66, 215, - 216, 19, 65, 4, 25, 4, 69, 76, 79, 87, 5, 11, 32, 2, 181, 212, 19, 3, 65, - 78, 68, 4, 22, 72, 203, 42, 87, 2, 21, 3, 79, 82, 84, 2, 17, 2, 32, 83, - 2, 11, 84, 2, 21, 3, 82, 79, 75, 2, 11, 69, 2, 17, 2, 32, 79, 2, 233, - 199, 19, 4, 86, 69, 82, 76, 2, 37, 7, 69, 82, 84, 73, 67, 65, 76, 2, 205, - 40, 2, 32, 76, 4, 46, 76, 245, 190, 19, 5, 75, 72, 65, 32, 89, 2, 247, - 12, 84, 18, 48, 3, 72, 87, 65, 97, 5, 82, 73, 80, 84, 32, 11, 33, 6, 32, - 87, 73, 84, 72, 32, 8, 222, 35, 71, 186, 2, 65, 190, 233, 3, 82, 207, - 238, 14, 72, 8, 26, 82, 155, 255, 3, 71, 5, 253, 175, 11, 5, 32, 87, 73, - 84, 72, 2, 225, 219, 16, 3, 65, 82, 80, 14, 48, 7, 68, 69, 87, 65, 89, - 83, 32, 199, 1, 71, 12, 110, 79, 56, 4, 84, 85, 82, 78, 176, 195, 10, 11, - 68, 73, 65, 69, 82, 69, 83, 73, 90, 69, 68, 203, 139, 10, 85, 7, 26, 80, - 211, 129, 4, 32, 2, 165, 234, 9, 2, 69, 78, 2, 237, 255, 12, 2, 69, 68, - 2, 201, 217, 16, 4, 77, 79, 73, 68, 2, 247, 250, 9, 32, 147, 1, 188, 1, - 6, 32, 87, 73, 84, 72, 32, 192, 3, 2, 69, 83, 70, 72, 130, 2, 79, 102, - 82, 54, 85, 250, 253, 3, 67, 202, 1, 83, 132, 247, 10, 9, 65, 73, 76, 76, - 69, 83, 83, 32, 80, 131, 207, 5, 90, 34, 110, 67, 182, 1, 68, 66, 77, - 170, 31, 76, 222, 226, 3, 80, 168, 5, 4, 72, 79, 79, 75, 50, 82, 167, - 146, 16, 83, 10, 58, 69, 22, 73, 62, 79, 174, 149, 7, 85, 155, 144, 12, - 65, 2, 251, 157, 12, 68, 2, 37, 7, 82, 67, 85, 77, 70, 76, 69, 2, 179, - 157, 18, 88, 2, 17, 2, 77, 77, 2, 139, 157, 18, 65, 8, 32, 2, 73, 65, - 179, 175, 16, 79, 4, 154, 21, 71, 215, 6, 69, 4, 17, 2, 73, 68, 4, 26, - 45, 203, 250, 3, 68, 2, 225, 136, 4, 6, 72, 69, 73, 71, 72, 84, 6, 45, 9, - 72, 32, 68, 73, 71, 82, 65, 80, 72, 7, 191, 128, 4, 32, 8, 64, 12, 32, - 87, 73, 84, 72, 32, 83, 84, 82, 73, 75, 69, 43, 79, 2, 205, 163, 16, 5, - 84, 72, 82, 79, 85, 6, 17, 2, 82, 78, 7, 41, 8, 32, 87, 73, 84, 72, 32, - 83, 84, 4, 25, 4, 82, 79, 75, 69, 5, 11, 32, 2, 209, 209, 3, 6, 84, 72, - 82, 79, 85, 71, 8, 26, 78, 151, 128, 4, 80, 6, 17, 2, 69, 32, 6, 134, - 201, 4, 83, 230, 197, 9, 84, 255, 131, 1, 70, 2, 17, 2, 69, 83, 2, 11, - 73, 2, 135, 163, 20, 76, 76, 44, 5, 82, 78, 69, 68, 32, 151, 197, 20, 77, - 74, 162, 1, 68, 22, 72, 66, 73, 54, 79, 142, 1, 82, 110, 84, 22, 86, 146, - 252, 3, 65, 38, 77, 198, 2, 89, 142, 174, 16, 85, 218, 19, 69, 2, 71, 2, - 75, 2, 76, 3, 87, 2, 223, 138, 17, 69, 7, 217, 236, 3, 11, 32, 87, 73, - 84, 72, 32, 70, 73, 83, 72, 72, 5, 11, 78, 2, 169, 212, 9, 5, 83, 85, 76, - 65, 82, 12, 66, 69, 180, 225, 3, 7, 32, 79, 80, 69, 78, 45, 79, 159, 29, - 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 202, 157, 18, 72, 187, 244, 1, - 83, 15, 33, 6, 32, 87, 73, 84, 72, 32, 12, 146, 242, 3, 77, 174, 7, 80, - 130, 5, 76, 154, 143, 14, 84, 143, 96, 72, 5, 211, 249, 3, 32, 7, 11, 32, - 4, 161, 5, 4, 87, 73, 84, 72, 97, 90, 32, 228, 222, 3, 6, 80, 83, 73, 76, - 79, 78, 242, 224, 16, 69, 2, 73, 2, 77, 3, 79, 82, 48, 3, 66, 65, 82, 49, - 5, 87, 73, 84, 72, 32, 5, 245, 17, 8, 32, 87, 73, 84, 72, 32, 83, 72, 78, - 142, 1, 67, 74, 68, 150, 2, 72, 170, 1, 77, 134, 1, 79, 106, 82, 38, 84, - 186, 9, 71, 82, 83, 234, 1, 65, 202, 168, 3, 73, 62, 66, 135, 66, 76, 6, - 200, 197, 4, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 247, 210, 14, 65, 18, - 52, 8, 73, 65, 69, 82, 69, 83, 73, 83, 131, 1, 79, 13, 11, 32, 10, 40, 4, - 65, 78, 68, 32, 155, 158, 18, 66, 8, 50, 67, 226, 13, 71, 186, 2, 65, - 187, 194, 3, 77, 2, 211, 150, 19, 65, 6, 26, 85, 131, 142, 18, 84, 4, 21, - 3, 66, 76, 69, 4, 11, 32, 4, 138, 13, 71, 187, 2, 65, 14, 11, 79, 14, 28, - 2, 82, 78, 191, 81, 79, 13, 29, 5, 32, 65, 78, 68, 32, 10, 66, 72, 226, - 11, 71, 186, 2, 65, 178, 236, 7, 68, 211, 219, 8, 84, 2, 213, 80, 2, 79, - 79, 10, 29, 5, 65, 67, 82, 79, 78, 11, 29, 5, 32, 65, 78, 68, 32, 8, 50, - 68, 214, 10, 71, 186, 2, 65, 131, 200, 16, 84, 2, 171, 10, 73, 6, 29, 5, - 71, 79, 78, 69, 75, 7, 11, 32, 4, 25, 4, 65, 78, 68, 32, 4, 178, 12, 65, - 131, 200, 16, 84, 4, 218, 245, 3, 69, 187, 145, 14, 73, 6, 25, 4, 73, 76, - 68, 69, 7, 11, 32, 4, 26, 65, 251, 152, 18, 66, 2, 17, 2, 78, 68, 2, 11, - 32, 2, 139, 11, 65, 29, 84, 6, 32, 87, 73, 84, 72, 32, 162, 1, 73, 66, - 79, 170, 166, 19, 69, 251, 141, 1, 89, 14, 82, 68, 182, 237, 3, 80, 142, - 1, 67, 146, 7, 82, 130, 220, 12, 84, 139, 144, 2, 72, 4, 26, 73, 243, - 245, 7, 79, 2, 17, 2, 65, 71, 2, 209, 132, 20, 2, 79, 78, 2, 41, 8, 83, - 73, 71, 79, 84, 72, 73, 67, 2, 135, 145, 16, 32, 6, 33, 6, 76, 65, 80, - 85, 75, 32, 6, 166, 157, 20, 65, 2, 79, 3, 85, 19, 33, 6, 32, 87, 73, 84, - 72, 32, 16, 202, 4, 67, 18, 68, 70, 71, 186, 2, 65, 246, 250, 17, 82, - 151, 93, 72, 17, 33, 6, 32, 87, 73, 84, 72, 32, 14, 42, 68, 32, 2, 76, - 79, 135, 234, 3, 80, 4, 222, 3, 73, 255, 173, 19, 79, 8, 60, 11, 78, 71, - 32, 76, 69, 70, 84, 32, 76, 69, 71, 79, 87, 7, 11, 32, 4, 60, 7, 65, 78, - 68, 32, 76, 79, 87, 65, 4, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 21, 3, - 73, 71, 72, 2, 167, 143, 11, 84, 2, 193, 174, 19, 4, 32, 83, 69, 82, 33, - 48, 6, 32, 87, 73, 84, 72, 32, 147, 139, 16, 79, 28, 110, 67, 18, 68, 70, - 71, 22, 72, 42, 76, 22, 83, 234, 1, 65, 186, 194, 3, 77, 202, 133, 13, - 84, 247, 178, 1, 82, 2, 203, 3, 73, 6, 26, 73, 215, 147, 16, 79, 2, 17, - 2, 65, 69, 2, 151, 172, 16, 82, 2, 223, 194, 18, 82, 4, 17, 2, 79, 79, 4, - 171, 131, 5, 75, 2, 195, 180, 19, 79, 4, 26, 72, 187, 253, 19, 84, 2, 21, - 3, 79, 82, 84, 2, 233, 246, 7, 6, 32, 82, 73, 71, 72, 84, 31, 33, 6, 32, - 87, 73, 84, 72, 32, 28, 98, 65, 22, 67, 82, 68, 38, 76, 34, 83, 146, 219, - 3, 77, 174, 7, 80, 218, 5, 82, 207, 238, 14, 72, 2, 231, 179, 13, 67, 6, - 42, 73, 230, 245, 6, 85, 155, 144, 12, 65, 2, 241, 192, 11, 4, 82, 67, - 85, 77, 6, 230, 181, 3, 69, 171, 218, 12, 79, 2, 11, 73, 2, 179, 172, 12, - 78, 4, 26, 87, 171, 250, 19, 84, 2, 179, 133, 17, 65, 18, 90, 70, 242, - 188, 9, 73, 172, 9, 6, 76, 79, 78, 71, 32, 83, 226, 198, 10, 83, 219, 5, - 79, 10, 34, 70, 134, 169, 20, 73, 3, 76, 7, 130, 169, 20, 73, 3, 76, 36, - 150, 1, 83, 210, 167, 20, 65, 2, 69, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, - 2, 77, 2, 78, 2, 79, 2, 80, 2, 82, 2, 84, 2, 85, 2, 86, 3, 88, 5, 143, - 225, 4, 67, 224, 5, 204, 1, 2, 65, 70, 144, 1, 2, 70, 84, 210, 38, 79, - 20, 5, 80, 67, 72, 65, 32, 164, 8, 8, 83, 83, 45, 84, 72, 65, 78, 32, - 254, 167, 16, 68, 228, 23, 6, 86, 69, 76, 32, 83, 76, 134, 230, 2, 77, - 239, 79, 71, 6, 120, 3, 76, 69, 83, 204, 191, 1, 14, 32, 70, 76, 85, 84, - 84, 69, 82, 73, 78, 71, 32, 73, 78, 209, 212, 11, 3, 89, 32, 71, 2, 135, - 228, 14, 83, 134, 4, 58, 32, 250, 20, 45, 217, 2, 6, 87, 65, 82, 68, 83, - 32, 246, 1, 166, 2, 65, 234, 4, 66, 202, 1, 67, 96, 2, 68, 79, 112, 2, - 72, 65, 166, 3, 76, 46, 77, 38, 79, 102, 82, 146, 3, 83, 82, 84, 222, 1, - 87, 242, 176, 8, 70, 236, 5, 14, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, - 82, 73, 71, 72, 86, 78, 202, 1, 80, 153, 13, 10, 86, 69, 82, 84, 73, 67, - 65, 76, 32, 66, 28, 22, 78, 143, 4, 82, 22, 28, 2, 68, 32, 191, 3, 71, - 16, 96, 6, 76, 79, 87, 69, 82, 32, 32, 6, 82, 73, 71, 72, 84, 32, 89, 6, - 85, 80, 80, 69, 82, 32, 4, 202, 1, 65, 235, 175, 17, 79, 6, 50, 84, 209, - 250, 17, 6, 68, 79, 85, 66, 76, 69, 4, 250, 136, 17, 82, 147, 231, 1, 65, - 6, 82, 65, 53, 16, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, 32, 66, 76, - 79, 67, 75, 2, 237, 134, 17, 8, 78, 68, 32, 82, 73, 71, 72, 84, 5, 165, - 253, 18, 17, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, - 67, 75, 6, 154, 182, 8, 69, 185, 1, 4, 76, 69, 32, 66, 6, 26, 67, 247, - 185, 8, 82, 2, 209, 185, 8, 5, 32, 76, 69, 83, 83, 12, 40, 4, 65, 82, 66, - 32, 143, 186, 8, 76, 8, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 4, 57, 12, - 32, 82, 73, 71, 72, 84, 32, 66, 65, 82, 66, 32, 4, 240, 244, 16, 4, 68, - 79, 87, 78, 1, 2, 85, 80, 14, 210, 185, 8, 85, 166, 26, 79, 206, 231, 2, - 69, 233, 168, 2, 8, 76, 79, 83, 69, 68, 32, 69, 78, 10, 44, 5, 85, 66, - 76, 69, 32, 255, 185, 8, 84, 8, 178, 187, 8, 87, 190, 30, 65, 138, 128, - 3, 81, 131, 190, 4, 80, 38, 36, 3, 76, 70, 32, 247, 191, 8, 78, 36, 132, - 2, 6, 67, 73, 82, 67, 76, 69, 246, 186, 8, 66, 90, 70, 82, 72, 50, 82, - 58, 84, 70, 87, 246, 13, 76, 22, 85, 220, 185, 8, 30, 73, 78, 86, 69, 82, - 83, 69, 32, 77, 69, 68, 73, 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, - 68, 32, 82, 73, 71, 72, 84, 251, 26, 77, 11, 33, 6, 32, 87, 73, 84, 72, - 32, 8, 42, 84, 206, 243, 17, 70, 235, 210, 1, 68, 4, 242, 197, 5, 72, - 179, 128, 13, 87, 4, 152, 158, 1, 2, 85, 71, 199, 159, 7, 79, 2, 161, - 133, 19, 4, 85, 76, 84, 73, 8, 36, 3, 78, 69, 32, 199, 190, 8, 85, 6, - 190, 162, 17, 81, 146, 4, 69, 45, 5, 84, 72, 73, 82, 68, 30, 44, 5, 73, - 71, 72, 84, 32, 155, 191, 8, 65, 28, 152, 1, 5, 65, 82, 82, 79, 87, 132, - 1, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 30, 87, 222, 236, - 8, 79, 154, 128, 8, 66, 54, 83, 207, 68, 84, 11, 11, 32, 8, 72, 5, 87, - 73, 84, 72, 32, 165, 192, 18, 7, 84, 72, 82, 79, 85, 71, 72, 6, 182, 174, - 16, 68, 242, 179, 3, 86, 79, 83, 7, 201, 228, 8, 2, 32, 87, 4, 210, 247, - 8, 65, 215, 176, 8, 72, 34, 174, 189, 8, 45, 70, 69, 78, 73, 168, 1, 3, - 80, 69, 69, 26, 81, 227, 2, 85, 26, 106, 82, 134, 194, 8, 72, 174, 212, - 7, 79, 208, 133, 1, 8, 87, 79, 32, 84, 72, 73, 82, 68, 219, 196, 1, 65, - 6, 40, 4, 73, 65, 78, 71, 255, 196, 8, 65, 4, 236, 236, 4, 8, 76, 69, 32, - 66, 69, 83, 73, 68, 207, 175, 12, 85, 16, 162, 199, 8, 72, 202, 1, 73, - 169, 184, 10, 2, 82, 73, 62, 128, 1, 9, 80, 79, 73, 78, 84, 73, 78, 71, - 32, 138, 1, 83, 206, 199, 8, 70, 226, 2, 72, 153, 7, 7, 84, 79, 45, 82, - 73, 71, 72, 30, 86, 65, 214, 204, 8, 67, 94, 68, 58, 77, 58, 82, 182, 1, - 83, 74, 84, 255, 157, 8, 69, 6, 146, 205, 8, 78, 202, 160, 8, 84, 159, 2, - 73, 4, 44, 5, 73, 68, 69, 32, 65, 131, 208, 8, 72, 2, 253, 142, 1, 2, 82, - 67, 210, 1, 172, 1, 5, 65, 82, 82, 79, 87, 174, 5, 66, 70, 72, 134, 4, - 84, 178, 2, 87, 226, 207, 8, 68, 210, 1, 70, 222, 7, 76, 26, 79, 34, 80, - 50, 82, 70, 83, 230, 191, 8, 67, 47, 81, 73, 26, 32, 143, 215, 17, 45, - 68, 102, 65, 138, 1, 84, 132, 2, 5, 87, 73, 84, 72, 32, 198, 209, 8, 70, - 181, 149, 10, 4, 79, 86, 69, 82, 12, 44, 5, 66, 79, 86, 69, 32, 179, 211, - 8, 78, 10, 64, 4, 83, 72, 79, 82, 170, 210, 8, 82, 142, 226, 4, 65, 55, - 84, 2, 143, 233, 18, 84, 12, 56, 7, 72, 82, 79, 85, 71, 72, 32, 53, 3, - 79, 32, 66, 6, 142, 178, 13, 83, 202, 196, 4, 76, 247, 145, 2, 88, 6, 32, - 2, 65, 82, 187, 212, 8, 76, 5, 169, 158, 12, 23, 32, 79, 86, 69, 82, 32, - 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 36, - 118, 76, 154, 212, 8, 68, 182, 1, 80, 30, 83, 38, 84, 178, 195, 8, 77, - 38, 78, 122, 69, 226, 151, 1, 72, 143, 163, 1, 86, 4, 158, 153, 17, 65, - 219, 243, 1, 79, 8, 190, 214, 8, 65, 224, 10, 5, 79, 84, 84, 79, 77, 207, - 185, 8, 76, 30, 26, 65, 255, 159, 17, 69, 26, 48, 6, 82, 80, 79, 79, 78, - 32, 183, 190, 19, 78, 24, 100, 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, - 32, 209, 216, 8, 9, 79, 86, 69, 82, 32, 82, 73, 71, 72, 22, 40, 4, 68, - 79, 87, 78, 117, 2, 85, 80, 10, 26, 32, 219, 165, 17, 87, 8, 222, 217, 8, - 66, 188, 2, 7, 65, 66, 79, 86, 69, 32, 82, 222, 193, 8, 70, 179, 5, 84, - 12, 26, 32, 231, 164, 17, 87, 10, 60, 6, 65, 66, 79, 86, 69, 32, 198, - 156, 17, 70, 179, 5, 84, 6, 42, 76, 201, 217, 8, 4, 82, 73, 71, 72, 4, - 238, 215, 8, 69, 199, 218, 4, 79, 54, 44, 2, 82, 73, 190, 221, 8, 79, - 159, 5, 87, 34, 44, 5, 65, 78, 71, 76, 69, 243, 225, 8, 80, 30, 56, 8, - 45, 72, 69, 65, 68, 69, 68, 32, 227, 170, 17, 32, 28, 52, 5, 65, 82, 82, - 79, 87, 178, 163, 17, 68, 39, 80, 25, 11, 32, 22, 180, 222, 8, 9, 79, 86, - 69, 82, 32, 82, 73, 71, 72, 22, 87, 251, 192, 8, 84, 6, 26, 72, 131, 228, - 8, 65, 4, 45, 9, 73, 84, 69, 32, 65, 82, 82, 79, 87, 5, 149, 168, 17, 2, - 32, 87, 5, 143, 231, 18, 80, 148, 1, 252, 1, 15, 67, 79, 78, 83, 79, 78, - 65, 78, 84, 32, 83, 73, 71, 78, 32, 124, 7, 76, 69, 84, 84, 69, 82, 32, - 236, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 198, 1, 83, - 172, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 251, 132, 18, 68, - 18, 66, 75, 22, 78, 154, 251, 19, 76, 2, 77, 2, 80, 2, 82, 3, 84, 5, 179, - 170, 19, 65, 5, 233, 201, 12, 4, 89, 73, 78, 45, 78, 194, 1, 75, 2, 80, - 190, 148, 7, 68, 194, 219, 10, 66, 2, 70, 2, 71, 2, 72, 2, 77, 138, 15, - 78, 194, 178, 1, 84, 46, 67, 2, 83, 138, 69, 74, 2, 76, 2, 82, 2, 86, 2, - 87, 2, 89, 187, 2, 65, 6, 246, 246, 19, 72, 2, 76, 187, 2, 65, 10, 82, - 84, 40, 14, 78, 89, 69, 84, 32, 84, 72, 89, 79, 79, 77, 32, 84, 65, 43, - 67, 6, 38, 65, 21, 5, 83, 72, 79, 79, 75, 2, 179, 248, 6, 45, 5, 17, 2, - 32, 67, 2, 217, 134, 15, 3, 69, 82, 45, 8, 92, 4, 73, 71, 78, 32, 37, 15, - 85, 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 4, 130, 227, - 15, 78, 227, 201, 2, 82, 4, 11, 32, 4, 250, 243, 19, 82, 3, 89, 14, 238, - 162, 16, 85, 202, 141, 1, 79, 170, 195, 2, 65, 186, 2, 69, 3, 73, 52, - 138, 1, 65, 128, 4, 11, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, 200, - 1, 2, 66, 85, 42, 67, 186, 1, 79, 210, 2, 87, 231, 227, 18, 83, 16, 40, - 5, 66, 79, 86, 69, 32, 171, 4, 78, 12, 152, 1, 7, 71, 82, 69, 65, 84, 69, - 82, 110, 83, 176, 1, 19, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, - 69, 81, 85, 65, 76, 32, 65, 227, 212, 15, 76, 2, 181, 5, 23, 45, 84, 72, - 65, 78, 32, 65, 66, 79, 86, 69, 32, 68, 79, 85, 66, 76, 69, 45, 76, 73, - 78, 69, 6, 152, 1, 7, 73, 77, 73, 76, 65, 82, 32, 93, 26, 76, 65, 78, 84, - 69, 68, 32, 69, 81, 85, 65, 76, 32, 65, 66, 79, 86, 69, 32, 71, 82, 69, - 65, 84, 69, 82, 4, 18, 65, 59, 79, 2, 25, 4, 66, 79, 86, 69, 2, 169, 222, - 17, 2, 32, 71, 2, 227, 2, 82, 2, 145, 2, 6, 45, 84, 72, 65, 78, 32, 4, - 17, 2, 68, 32, 4, 220, 3, 5, 78, 79, 84, 32, 65, 245, 151, 13, 11, 83, - 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 4, 241, 249, 12, 5, 84, 32, 78, - 79, 84, 4, 65, 14, 76, 79, 83, 69, 68, 32, 66, 89, 32, 67, 85, 82, 86, - 69, 5, 11, 32, 2, 61, 13, 65, 66, 79, 86, 69, 32, 83, 76, 65, 78, 84, 69, - 68, 2, 17, 2, 32, 69, 2, 251, 228, 14, 81, 18, 40, 2, 82, 32, 141, 220, - 11, 2, 86, 69, 16, 114, 65, 48, 16, 83, 76, 65, 78, 84, 69, 68, 32, 69, - 81, 85, 65, 76, 32, 84, 79, 218, 246, 12, 69, 223, 226, 4, 71, 2, 173, - 184, 9, 7, 80, 80, 82, 79, 88, 73, 77, 9, 49, 10, 32, 87, 73, 84, 72, 32, - 68, 79, 84, 32, 6, 26, 65, 163, 199, 11, 73, 4, 25, 4, 66, 79, 86, 69, 5, - 231, 153, 18, 32, 6, 25, 4, 73, 84, 72, 32, 6, 66, 67, 40, 8, 81, 85, 69, - 83, 84, 73, 79, 78, 143, 153, 19, 68, 2, 249, 197, 11, 5, 73, 82, 67, 76, - 69, 2, 17, 2, 32, 77, 2, 17, 2, 65, 82, 2, 151, 236, 18, 75, 136, 11, - 190, 1, 71, 186, 5, 77, 166, 7, 78, 168, 65, 2, 80, 83, 20, 3, 83, 85, - 32, 238, 199, 15, 82, 136, 35, 11, 86, 82, 69, 32, 84, 79, 85, 82, 78, - 79, 73, 210, 59, 79, 242, 219, 1, 90, 187, 82, 66, 44, 26, 65, 57, 2, 72, - 84, 2, 245, 167, 9, 9, 84, 85, 82, 69, 32, 79, 80, 69, 78, 42, 38, 32, - 137, 4, 4, 78, 73, 78, 71, 36, 146, 1, 69, 38, 70, 126, 82, 40, 3, 76, - 69, 70, 86, 83, 42, 84, 188, 198, 7, 3, 66, 76, 85, 138, 156, 9, 87, 250, - 41, 86, 250, 77, 71, 135, 69, 67, 2, 129, 209, 17, 4, 73, 71, 72, 84, 8, - 94, 73, 205, 157, 18, 17, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, - 66, 76, 65, 67, 75, 4, 149, 205, 17, 2, 86, 69, 4, 36, 3, 73, 71, 72, - 155, 176, 17, 65, 2, 165, 237, 1, 16, 84, 32, 84, 79, 82, 84, 79, 73, 83, - 69, 32, 83, 72, 69, 76, 76, 6, 170, 236, 16, 72, 138, 98, 65, 43, 73, 4, - 200, 153, 16, 2, 87, 69, 21, 3, 72, 82, 69, 7, 29, 5, 32, 77, 79, 79, 68, - 5, 155, 149, 8, 32, 138, 1, 80, 3, 66, 85, 32, 137, 157, 8, 11, 73, 84, - 69, 68, 32, 76, 73, 65, 66, 73, 76, 136, 1, 128, 1, 7, 76, 69, 84, 84, - 69, 82, 32, 222, 1, 83, 200, 2, 5, 86, 79, 87, 69, 76, 194, 206, 12, 69, - 182, 188, 4, 81, 247, 95, 68, 60, 170, 1, 71, 242, 139, 6, 84, 170, 221, - 8, 89, 186, 132, 1, 78, 162, 169, 3, 83, 82, 66, 2, 67, 2, 68, 2, 74, 2, - 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 3, 87, 6, 254, 148, 18, 89, - 202, 199, 1, 72, 187, 2, 65, 32, 108, 4, 73, 71, 78, 32, 128, 1, 12, 77, - 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 181, 250, 6, 2, 85, 66, 8, - 72, 3, 75, 69, 77, 0, 3, 77, 85, 75, 32, 2, 83, 65, 207, 149, 17, 76, 2, - 129, 154, 17, 3, 80, 72, 82, 2, 155, 201, 19, 45, 18, 174, 235, 15, 78, - 186, 172, 3, 65, 194, 66, 75, 2, 76, 2, 77, 2, 80, 2, 82, 3, 84, 20, 84, - 6, 32, 83, 73, 71, 78, 32, 177, 85, 10, 45, 67, 65, 82, 82, 73, 69, 82, - 32, 76, 18, 170, 211, 5, 65, 166, 194, 11, 79, 194, 133, 2, 69, 162, 64, - 73, 3, 85, 226, 8, 22, 69, 199, 64, 75, 222, 8, 34, 32, 177, 3, 3, 65, - 82, 32, 14, 120, 12, 73, 78, 84, 69, 71, 82, 65, 84, 73, 79, 78, 32, 186, - 156, 13, 70, 150, 143, 3, 83, 245, 147, 1, 4, 84, 65, 66, 85, 6, 108, 5, - 87, 73, 84, 72, 32, 157, 1, 17, 78, 79, 84, 32, 73, 78, 67, 76, 85, 68, - 73, 78, 71, 32, 84, 72, 69, 4, 76, 7, 82, 69, 67, 84, 65, 78, 71, 1, 8, - 83, 69, 77, 73, 67, 73, 82, 67, 2, 73, 16, 85, 76, 65, 82, 32, 80, 65, - 84, 72, 32, 65, 82, 79, 85, 78, 68, 2, 17, 2, 32, 80, 2, 223, 178, 18, - 79, 208, 8, 60, 8, 65, 32, 83, 73, 71, 78, 32, 65, 221, 24, 2, 66, 32, - 170, 5, 122, 49, 86, 51, 202, 2, 52, 214, 1, 53, 158, 4, 54, 142, 3, 55, - 148, 4, 2, 56, 48, 78, 66, 213, 136, 18, 3, 48, 50, 56, 6, 128, 188, 12, - 5, 48, 48, 45, 49, 48, 200, 221, 5, 2, 50, 48, 233, 19, 2, 51, 49, 150, - 1, 78, 48, 90, 49, 130, 1, 55, 246, 156, 14, 50, 2, 51, 2, 52, 2, 53, 3, - 54, 22, 178, 1, 57, 214, 210, 19, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, - 2, 55, 3, 56, 24, 90, 51, 214, 210, 19, 48, 2, 49, 2, 50, 2, 52, 2, 53, - 2, 54, 2, 55, 2, 56, 3, 57, 6, 210, 210, 19, 65, 2, 66, 3, 67, 4, 174, - 210, 19, 48, 3, 49, 38, 18, 48, 91, 49, 20, 162, 1, 48, 2, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 74, 48, 2, 49, 2, 50, - 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 2, 205, 178, 6, 2, 45, 86, 160, - 1, 102, 48, 78, 49, 62, 51, 86, 52, 70, 53, 86, 54, 206, 11, 50, 150, - 166, 2, 57, 198, 229, 11, 55, 3, 56, 16, 210, 207, 19, 49, 2, 50, 2, 51, - 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 12, 134, 207, 19, 48, 2, 49, 2, 50, 2, - 51, 2, 53, 3, 54, 18, 202, 206, 19, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 14, 246, 205, 19, 48, 2, 49, 2, 50, 2, 53, 2, - 55, 2, 56, 3, 57, 18, 178, 205, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, - 53, 2, 54, 2, 55, 3, 57, 12, 222, 204, 19, 51, 2, 52, 2, 53, 2, 54, 2, - 56, 3, 57, 104, 70, 48, 78, 50, 86, 51, 38, 52, 78, 54, 162, 146, 14, 53, - 243, 1, 49, 16, 218, 203, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, - 56, 3, 57, 18, 142, 203, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, - 55, 2, 56, 3, 57, 6, 186, 202, 19, 52, 2, 55, 3, 56, 16, 150, 202, 19, - 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 10, 202, 201, 19, - 48, 2, 49, 2, 50, 2, 51, 3, 52, 44, 86, 48, 134, 2, 49, 196, 252, 1, 2, - 51, 50, 177, 184, 17, 6, 50, 54, 32, 69, 89, 89, 26, 110, 57, 142, 219, - 8, 55, 182, 6, 50, 62, 54, 230, 62, 52, 226, 151, 3, 51, 110, 56, 134, - 206, 2, 49, 147, 117, 53, 10, 26, 45, 131, 190, 18, 32, 8, 96, 2, 50, 32, - 200, 169, 12, 3, 54, 32, 76, 184, 2, 3, 52, 32, 76, 201, 164, 3, 3, 51, - 32, 76, 2, 163, 173, 12, 76, 14, 110, 49, 22, 51, 32, 3, 52, 32, 65, 0, - 2, 53, 32, 238, 154, 4, 50, 238, 199, 4, 48, 241, 235, 6, 2, 55, 32, 2, - 167, 185, 18, 32, 2, 11, 32, 2, 175, 142, 12, 79, 2, 179, 137, 18, 66, - 16, 130, 197, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, - 162, 1, 22, 48, 155, 5, 49, 140, 1, 82, 49, 54, 50, 118, 51, 62, 52, 78, - 53, 86, 54, 62, 55, 70, 56, 151, 136, 14, 48, 10, 194, 195, 19, 48, 2, - 49, 2, 51, 2, 54, 3, 55, 28, 86, 49, 2, 50, 146, 77, 51, 170, 245, 18, - 48, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 7, 182, 194, 19, 70, 3, 77, 12, - 154, 194, 19, 48, 2, 49, 2, 52, 2, 55, 2, 56, 3, 57, 16, 222, 193, 19, - 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 146, 193, 19, - 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 190, 192, - 19, 48, 2, 49, 2, 53, 2, 54, 2, 55, 3, 57, 14, 130, 192, 19, 48, 2, 51, - 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 12, 190, 191, 19, 48, 2, 49, 2, 50, 2, - 53, 2, 54, 3, 55, 22, 82, 50, 36, 2, 51, 49, 30, 56, 214, 160, 12, 49, - 206, 2, 54, 146, 1, 55, 3, 57, 6, 174, 190, 19, 48, 2, 50, 3, 51, 4, 138, - 190, 19, 65, 3, 66, 4, 238, 189, 19, 48, 3, 56, 166, 3, 116, 9, 73, 68, - 69, 79, 71, 82, 65, 77, 32, 216, 17, 10, 77, 79, 78, 79, 71, 82, 65, 77, - 32, 66, 249, 1, 2, 83, 89, 234, 1, 54, 66, 249, 15, 8, 86, 69, 83, 83, - 69, 76, 32, 66, 176, 1, 22, 49, 255, 10, 50, 126, 86, 48, 210, 3, 50, - 162, 1, 51, 86, 52, 122, 53, 114, 54, 146, 1, 55, 118, 56, 71, 57, 28, - 114, 53, 98, 54, 58, 55, 78, 56, 62, 57, 192, 133, 7, 3, 50, 32, 87, 138, - 88, 48, 225, 156, 11, 4, 52, 32, 68, 69, 6, 168, 132, 7, 2, 32, 69, 196, - 149, 11, 3, 70, 32, 77, 129, 80, 7, 77, 32, 83, 84, 65, 76, 76, 4, 152, - 240, 1, 3, 70, 32, 69, 177, 142, 16, 2, 77, 32, 4, 36, 3, 70, 32, 83, 1, - 2, 77, 32, 2, 145, 225, 11, 4, 72, 69, 45, 71, 4, 188, 152, 9, 3, 77, 32, - 66, 237, 133, 9, 3, 70, 32, 83, 4, 208, 176, 17, 4, 77, 32, 66, 85, 157, - 109, 3, 70, 32, 67, 10, 196, 145, 9, 4, 51, 32, 83, 80, 240, 10, 5, 49, - 32, 66, 65, 82, 220, 231, 4, 4, 50, 32, 79, 76, 20, 6, 53, 32, 67, 89, - 80, 69, 245, 98, 4, 48, 32, 87, 72, 6, 54, 49, 172, 130, 17, 3, 48, 32, - 79, 215, 179, 2, 50, 2, 197, 156, 18, 2, 32, 87, 10, 224, 5, 3, 53, 32, - 87, 204, 143, 9, 6, 48, 32, 66, 82, 79, 78, 172, 2, 4, 49, 32, 71, 79, - 138, 158, 10, 50, 3, 54, 16, 82, 57, 214, 232, 2, 49, 198, 203, 16, 48, - 2, 50, 2, 51, 2, 52, 2, 55, 3, 56, 2, 169, 97, 3, 32, 67, 76, 20, 248, - 160, 14, 5, 51, 32, 65, 82, 77, 204, 207, 4, 5, 50, 32, 71, 65, 82, 182, - 67, 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 134, 242, - 13, 54, 200, 190, 3, 4, 51, 32, 77, 79, 158, 130, 2, 48, 2, 49, 2, 50, 2, - 52, 2, 55, 2, 56, 3, 57, 14, 246, 177, 19, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 3, 57, 4, 164, 164, 2, 3, 49, 32, 72, 143, 141, 17, 48, 50, - 42, 50, 110, 51, 130, 1, 52, 227, 1, 53, 4, 84, 8, 48, 32, 70, 79, 79, - 84, 83, 84, 169, 210, 6, 7, 53, 32, 66, 65, 84, 72, 84, 2, 143, 157, 18, - 79, 12, 104, 4, 51, 32, 83, 87, 156, 255, 13, 4, 48, 32, 83, 80, 146, - 149, 4, 49, 214, 154, 1, 50, 2, 52, 3, 54, 2, 147, 152, 18, 79, 16, 168, - 1, 9, 48, 32, 87, 72, 69, 69, 76, 69, 68, 2, 49, 34, 51, 168, 158, 17, - 12, 50, 32, 67, 72, 65, 82, 73, 79, 84, 32, 70, 82, 250, 142, 2, 53, 2, - 54, 2, 56, 3, 57, 2, 185, 163, 18, 3, 32, 67, 72, 2, 247, 190, 6, 32, 18, - 236, 165, 18, 3, 52, 32, 68, 158, 135, 1, 49, 2, 50, 2, 51, 2, 53, 2, 54, - 2, 55, 2, 56, 3, 57, 58, 50, 50, 192, 144, 12, 2, 49, 53, 1, 2, 51, 48, - 54, 50, 50, 238, 146, 12, 53, 150, 227, 1, 48, 3, 49, 12, 186, 171, 19, + 84, 73, 80, 234, 186, 20, 87, 219, 26, 77, 6, 44, 5, 76, 65, 67, 75, 32, + 251, 238, 21, 85, 4, 146, 146, 22, 68, 227, 27, 83, 4, 252, 212, 20, 8, + 69, 82, 32, 66, 79, 65, 82, 68, 183, 186, 2, 32, 2, 221, 238, 20, 2, 32, + 76, 18, 144, 1, 6, 73, 78, 84, 69, 82, 82, 22, 76, 170, 250, 11, 80, 230, + 184, 4, 69, 244, 198, 1, 2, 79, 72, 204, 158, 2, 4, 85, 78, 68, 69, 183, + 97, 81, 2, 207, 217, 4, 79, 6, 60, 9, 79, 87, 32, 75, 65, 86, 89, 75, 65, + 163, 235, 6, 65, 5, 229, 162, 18, 11, 32, 87, 73, 84, 72, 32, 75, 65, 86, + 89, 75, 6, 38, 84, 178, 194, 19, 80, 223, 89, 83, 2, 159, 181, 16, 73, 4, + 246, 177, 22, 69, 215, 93, 73, 218, 1, 94, 65, 138, 21, 69, 62, 79, 42, + 85, 157, 195, 11, 10, 73, 71, 83, 65, 87, 32, 80, 85, 90, 90, 202, 1, + 120, 5, 67, 75, 45, 79, 45, 32, 7, 80, 65, 78, 69, 83, 69, 32, 184, 2, 7, + 86, 65, 78, 69, 83, 69, 32, 207, 200, 23, 82, 2, 169, 184, 18, 3, 76, 65, + 78, 16, 246, 1, 67, 18, 80, 152, 157, 1, 10, 73, 78, 68, 85, 83, 84, 82, + 73, 65, 76, 236, 229, 3, 3, 66, 65, 78, 134, 174, 3, 68, 220, 143, 11, + 15, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 66, 69, 71, 73, 180, 248, + 1, 3, 71, 79, 66, 169, 109, 2, 79, 71, 2, 207, 32, 65, 2, 165, 145, 13, + 7, 79, 83, 84, 32, 79, 70, 70, 182, 1, 172, 2, 15, 67, 79, 78, 83, 79, + 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 76, 2, 76, 69, 40, 4, 82, 73, 71, + 72, 224, 6, 2, 80, 65, 176, 3, 14, 84, 85, 82, 78, 69, 68, 32, 80, 65, + 68, 65, 32, 80, 73, 88, 5, 83, 73, 71, 78, 32, 144, 1, 11, 86, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 167, 197, 21, 68, 6, 52, 2, 75, 69, 188, + 201, 16, 2, 80, 69, 175, 5, 67, 2, 199, 144, 23, 82, 96, 38, 70, 65, 5, + 84, 84, 69, 82, 32, 2, 41, 8, 84, 32, 82, 69, 82, 69, 78, 71, 2, 219, + 248, 21, 71, 94, 230, 1, 68, 26, 73, 48, 2, 75, 65, 66, 78, 130, 1, 66, + 2, 67, 2, 71, 16, 2, 80, 65, 56, 2, 82, 65, 32, 2, 83, 65, 42, 84, 74, + 74, 218, 178, 21, 65, 142, 138, 2, 72, 2, 76, 2, 77, 2, 87, 2, 89, 186, + 2, 69, 2, 79, 3, 85, 8, 222, 3, 68, 15, 65, 7, 184, 205, 5, 3, 32, 75, + 65, 171, 245, 17, 73, 7, 11, 32, 4, 22, 83, 215, 2, 77, 2, 133, 137, 21, + 2, 65, 83, 14, 36, 2, 71, 65, 90, 89, 167, 1, 65, 7, 33, 6, 32, 76, 69, + 76, 69, 84, 5, 29, 5, 32, 82, 65, 83, 87, 2, 159, 230, 5, 65, 4, 163, 1, + 65, 7, 11, 32, 4, 154, 1, 77, 153, 188, 23, 3, 67, 69, 82, 5, 237, 204, + 16, 3, 32, 65, 71, 7, 17, 2, 32, 77, 4, 70, 85, 71, 65, 8, 18, 65, 55, + 84, 5, 17, 2, 32, 77, 2, 11, 85, 2, 171, 154, 23, 82, 4, 11, 65, 5, 11, + 32, 2, 11, 77, 2, 11, 65, 2, 11, 72, 2, 237, 185, 17, 2, 65, 80, 28, 40, + 3, 68, 65, 32, 165, 3, 2, 78, 71, 24, 174, 1, 65, 90, 76, 86, 80, 244, + 150, 7, 10, 84, 73, 82, 84, 65, 32, 84, 85, 77, 69, 158, 135, 12, 87, + 168, 199, 2, 7, 73, 83, 69, 78, 45, 73, 83, 169, 211, 1, 3, 77, 65, 68, + 6, 44, 3, 68, 69, 71, 177, 169, 22, 2, 78, 68, 5, 11, 32, 2, 193, 246, + 22, 2, 65, 68, 6, 38, 85, 165, 185, 23, 3, 73, 78, 71, 4, 240, 200, 18, + 2, 78, 71, 163, 250, 3, 72, 4, 38, 73, 205, 193, 18, 3, 65, 78, 71, 2, + 181, 197, 16, 3, 83, 69, 76, 4, 140, 153, 13, 5, 82, 65, 78, 71, 75, 251, + 209, 9, 75, 10, 108, 5, 67, 69, 67, 65, 75, 248, 205, 5, 3, 87, 73, 71, + 202, 246, 10, 76, 249, 228, 2, 5, 80, 65, 78, 89, 65, 5, 157, 176, 18, 3, + 32, 84, 69, 18, 116, 4, 83, 85, 75, 85, 42, 84, 64, 4, 87, 85, 76, 85, + 142, 195, 16, 80, 137, 224, 1, 7, 68, 73, 82, 71, 65, 32, 77, 5, 225, + 190, 22, 5, 32, 77, 69, 78, 68, 6, 26, 65, 203, 196, 16, 79, 4, 178, 196, + 16, 82, 187, 162, 6, 76, 5, 25, 4, 32, 77, 69, 76, 2, 151, 180, 23, 73, + 4, 36, 3, 76, 76, 89, 203, 239, 19, 65, 2, 167, 210, 19, 70, 4, 192, 132, + 22, 2, 89, 83, 191, 98, 73, 6, 196, 200, 14, 2, 71, 71, 194, 135, 5, 80, + 191, 199, 3, 78, 188, 25, 74, 65, 178, 78, 69, 218, 1, 72, 162, 50, 73, + 198, 6, 78, 50, 79, 111, 82, 202, 10, 200, 1, 5, 73, 84, 72, 73, 32, 234, + 4, 78, 188, 45, 6, 84, 65, 75, 65, 78, 65, 208, 13, 3, 87, 73, 32, 220, + 8, 7, 89, 65, 72, 32, 76, 73, 32, 224, 201, 4, 6, 75, 84, 79, 86, 73, 75, + 135, 244, 17, 65, 136, 1, 204, 1, 7, 76, 69, 84, 84, 69, 82, 32, 178, 2, + 83, 150, 116, 68, 228, 243, 3, 2, 86, 79, 236, 137, 3, 11, 78, 85, 77, + 66, 69, 82, 32, 83, 73, 71, 78, 162, 202, 9, 65, 189, 211, 1, 6, 69, 78, + 85, 77, 69, 82, 90, 214, 1, 68, 190, 210, 19, 65, 150, 1, 84, 230, 5, 85, + 186, 202, 1, 73, 138, 196, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, + 75, 2, 80, 2, 82, 138, 69, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, + 79, 10, 38, 68, 210, 173, 23, 72, 187, 2, 65, 6, 222, 178, 21, 68, 242, + 250, 1, 72, 187, 2, 65, 12, 40, 4, 73, 71, 78, 32, 179, 156, 11, 69, 10, + 202, 145, 19, 67, 98, 78, 138, 216, 3, 65, 239, 1, 86, 232, 4, 42, 71, + 241, 39, 5, 78, 65, 68, 65, 32, 174, 3, 76, 11, 88, 73, 32, 82, 65, 68, + 73, 67, 65, 76, 32, 249, 226, 20, 2, 65, 82, 172, 3, 130, 2, 65, 94, 66, + 230, 2, 67, 150, 2, 68, 158, 3, 69, 198, 1, 70, 138, 2, 71, 146, 1, 72, + 210, 2, 73, 76, 2, 74, 65, 34, 75, 30, 76, 230, 1, 77, 254, 1, 78, 62, + 79, 102, 80, 110, 82, 166, 1, 83, 230, 6, 84, 226, 2, 86, 50, 87, 210, 2, + 89, 207, 251, 21, 85, 10, 56, 2, 82, 82, 198, 252, 21, 71, 202, 104, 78, + 207, 47, 88, 4, 138, 173, 22, 79, 207, 1, 73, 34, 58, 65, 38, 73, 46, 76, + 54, 79, 102, 82, 207, 220, 21, 69, 4, 242, 202, 5, 77, 183, 142, 14, 68, + 6, 158, 240, 21, 84, 238, 115, 82, 163, 70, 71, 6, 210, 239, 19, 79, 170, + 136, 2, 65, 179, 155, 1, 85, 10, 62, 76, 226, 140, 23, 65, 218, 5, 78, + 142, 5, 68, 203, 17, 87, 2, 253, 219, 3, 4, 84, 32, 79, 70, 6, 42, 73, + 158, 248, 9, 65, 155, 197, 11, 85, 2, 219, 255, 13, 83, 28, 58, 65, 70, + 76, 74, 79, 206, 181, 10, 72, 183, 129, 11, 73, 6, 38, 85, 154, 139, 23, + 82, 219, 5, 86, 2, 141, 128, 22, 2, 76, 68, 8, 42, 65, 190, 190, 7, 73, + 167, 210, 14, 79, 4, 250, 166, 23, 78, 3, 87, 10, 130, 130, 22, 82, 148, + 2, 2, 77, 80, 222, 100, 86, 134, 5, 76, 235, 56, 87, 34, 38, 69, 38, 73, + 98, 79, 195, 1, 82, 4, 242, 160, 21, 65, 159, 204, 1, 69, 8, 38, 83, 226, + 128, 14, 86, 175, 2, 80, 4, 208, 192, 19, 5, 84, 73, 78, 71, 85, 207, + 228, 3, 72, 16, 94, 84, 76, 3, 85, 66, 76, 138, 130, 13, 87, 252, 208, 9, + 2, 32, 78, 222, 23, 79, 223, 56, 71, 7, 25, 4, 84, 69, 68, 32, 4, 168, + 187, 7, 3, 67, 76, 73, 155, 165, 15, 84, 2, 175, 134, 4, 69, 6, 242, 232, + 21, 65, 222, 169, 1, 85, 219, 16, 89, 20, 106, 65, 38, 78, 32, 3, 86, 69, + 78, 140, 131, 9, 6, 77, 66, 82, 79, 73, 68, 250, 243, 13, 73, 243, 19, + 89, 6, 170, 233, 16, 82, 227, 184, 6, 84, 4, 178, 25, 67, 203, 202, 22, + 84, 5, 207, 208, 22, 73, 28, 74, 65, 50, 73, 62, 76, 38, 82, 170, 20, 69, + 250, 186, 22, 79, 223, 23, 85, 6, 146, 255, 13, 84, 130, 139, 9, 67, 131, + 22, 78, 8, 234, 240, 12, 69, 150, 133, 10, 71, 230, 19, 82, 199, 21, 83, + 4, 230, 214, 21, 85, 151, 201, 1, 89, 4, 180, 239, 21, 2, 65, 71, 207, + 175, 1, 79, 14, 58, 79, 52, 2, 82, 65, 190, 176, 19, 72, 239, 164, 2, 65, + 7, 206, 216, 22, 76, 241, 34, 5, 32, 83, 76, 79, 87, 4, 218, 206, 22, 73, + 135, 18, 83, 22, 58, 65, 142, 1, 69, 60, 5, 73, 68, 73, 78, 71, 19, 79, + 8, 38, 76, 154, 215, 22, 78, 199, 13, 73, 4, 34, 70, 165, 132, 22, 2, 66, + 69, 2, 41, 8, 32, 84, 82, 69, 69, 32, 84, 82, 2, 135, 184, 19, 85, 6, 26, + 65, 175, 156, 23, 77, 4, 138, 128, 23, 82, 175, 28, 68, 2, 195, 19, 32, + 6, 26, 82, 215, 152, 23, 79, 4, 150, 133, 23, 83, 215, 22, 78, 6, 26, 78, + 223, 132, 23, 67, 4, 26, 83, 135, 154, 23, 67, 2, 135, 138, 22, 69, 4, + 170, 132, 23, 68, 215, 22, 82, 2, 133, 179, 5, 2, 78, 73, 22, 46, 65, 34, + 69, 78, 73, 41, 3, 79, 78, 71, 4, 190, 131, 23, 77, 191, 19, 67, 8, 38, + 65, 242, 219, 22, 71, 199, 58, 69, 4, 246, 247, 13, 84, 215, 161, 9, 70, + 6, 210, 130, 23, 70, 2, 78, 215, 22, 68, 5, 153, 204, 11, 3, 32, 83, 84, + 22, 42, 69, 34, 73, 46, 79, 139, 200, 22, 65, 4, 198, 200, 22, 76, 195, + 51, 65, 4, 128, 243, 11, 2, 78, 73, 139, 195, 9, 76, 12, 34, 82, 34, 85, + 199, 199, 22, 79, 4, 134, 244, 21, 84, 187, 82, 78, 6, 26, 78, 251, 149, + 23, 84, 4, 158, 232, 21, 84, 235, 174, 1, 68, 6, 26, 79, 159, 250, 22, + 69, 4, 242, 255, 22, 83, 215, 22, 84, 10, 40, 2, 78, 69, 22, 80, 203, + 207, 22, 76, 5, 199, 157, 6, 83, 4, 190, 178, 10, 80, 247, 193, 2, 69, + 10, 54, 82, 230, 248, 21, 76, 166, 1, 79, 179, 154, 1, 73, 4, 148, 207, + 12, 2, 73, 86, 181, 141, 7, 2, 79, 70, 18, 62, 65, 42, 73, 234, 173, 10, + 79, 170, 150, 12, 85, 195, 9, 69, 6, 182, 196, 22, 73, 226, 79, 80, 3, + 84, 6, 156, 242, 12, 3, 71, 72, 84, 230, 227, 9, 86, 155, 39, 67, 74, + 170, 1, 65, 86, 67, 54, 69, 62, 72, 178, 1, 73, 46, 76, 62, 80, 106, 84, + 142, 245, 17, 87, 158, 159, 1, 78, 234, 64, 79, 130, 174, 1, 77, 226, + 103, 81, 130, 35, 75, 247, 47, 85, 6, 144, 195, 3, 9, 67, 82, 73, 70, 73, + 67, 73, 65, 76, 202, 178, 19, 76, 175, 28, 89, 4, 224, 246, 10, 2, 82, + 73, 249, 218, 5, 2, 72, 79, 8, 170, 210, 21, 67, 134, 51, 65, 174, 10, + 76, 167, 129, 1, 69, 10, 18, 69, 39, 79, 4, 222, 132, 22, 76, 199, 139, + 1, 69, 6, 40, 4, 82, 84, 32, 84, 183, 243, 22, 79, 4, 44, 5, 65, 73, 76, + 69, 68, 207, 133, 16, 72, 2, 201, 138, 21, 2, 32, 66, 4, 184, 247, 17, 2, + 67, 75, 195, 148, 5, 76, 6, 26, 65, 211, 209, 22, 73, 4, 246, 247, 22, + 86, 199, 21, 83, 10, 50, 69, 34, 73, 190, 148, 19, 82, 179, 169, 3, 79, + 4, 154, 213, 22, 65, 183, 22, 69, 2, 159, 241, 22, 82, 12, 34, 69, 34, + 79, 239, 252, 21, 65, 4, 198, 252, 22, 65, 219, 16, 80, 6, 26, 80, 147, + 246, 22, 78, 5, 223, 187, 22, 80, 30, 82, 65, 98, 73, 34, 79, 38, 82, 52, + 2, 85, 82, 32, 2, 87, 79, 167, 186, 22, 69, 6, 38, 78, 154, 229, 21, 66, + 239, 26, 76, 2, 33, 6, 78, 69, 68, 32, 76, 69, 2, 207, 233, 13, 65, 4, + 174, 205, 22, 71, 155, 39, 76, 4, 146, 251, 18, 78, 243, 138, 2, 79, 6, + 190, 240, 16, 73, 150, 232, 4, 65, 179, 155, 1, 69, 4, 134, 190, 21, 66, + 227, 37, 84, 5, 187, 199, 19, 32, 4, 254, 220, 12, 65, 209, 157, 5, 3, + 73, 76, 76, 26, 58, 65, 110, 69, 42, 72, 32, 2, 73, 78, 30, 79, 39, 82, + 6, 32, 2, 76, 75, 247, 202, 22, 84, 5, 11, 32, 2, 11, 69, 2, 17, 2, 78, + 67, 2, 237, 243, 17, 2, 76, 79, 4, 168, 184, 22, 2, 65, 80, 195, 51, 83, + 4, 218, 190, 21, 73, 231, 63, 69, 4, 206, 135, 23, 68, 3, 69, 4, 150, + 187, 21, 77, 135, 201, 1, 82, 4, 150, 182, 22, 79, 239, 80, 65, 2, 141, + 229, 20, 2, 69, 76, 186, 1, 92, 2, 76, 69, 148, 2, 5, 83, 73, 71, 78, 32, + 170, 207, 17, 65, 186, 9, 86, 247, 182, 3, 68, 110, 44, 5, 84, 84, 69, + 82, 32, 219, 195, 22, 78, 108, 190, 214, 17, 78, 142, 209, 1, 65, 38, 68, + 46, 76, 38, 82, 34, 84, 46, 86, 186, 5, 85, 206, 141, 1, 79, 238, 60, 73, + 182, 196, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, + 234, 61, 70, 2, 72, 2, 77, 3, 89, 22, 94, 67, 138, 1, 83, 246, 228, 18, + 78, 190, 66, 65, 190, 158, 1, 74, 150, 3, 85, 235, 245, 1, 86, 4, 100, + 19, 79, 77, 66, 73, 78, 73, 78, 71, 32, 65, 78, 85, 83, 86, 65, 82, 65, + 32, 65, 195, 228, 18, 65, 2, 173, 253, 19, 3, 66, 79, 86, 4, 216, 198, + 12, 6, 80, 65, 67, 73, 78, 71, 255, 143, 5, 73, 162, 2, 62, 32, 173, 11, + 10, 45, 72, 73, 82, 65, 71, 65, 78, 65, 32, 154, 2, 140, 1, 7, 76, 69, + 84, 84, 69, 82, 32, 242, 9, 86, 232, 130, 19, 10, 68, 73, 71, 82, 65, 80, + 72, 32, 75, 79, 238, 204, 1, 73, 191, 146, 1, 77, 146, 2, 186, 1, 65, + 178, 1, 66, 106, 77, 186, 2, 78, 54, 83, 226, 1, 68, 2, 71, 2, 72, 2, 75, + 2, 80, 2, 82, 2, 84, 2, 86, 2, 90, 126, 87, 46, 89, 150, 246, 22, 69, 2, + 73, 2, 79, 3, 85, 19, 60, 4, 73, 78, 85, 32, 45, 7, 82, 67, 72, 65, 73, + 67, 32, 8, 130, 7, 84, 138, 224, 22, 67, 215, 22, 80, 8, 38, 89, 166, + 216, 22, 87, 235, 36, 69, 4, 138, 253, 22, 69, 3, 73, 20, 50, 73, 190, + 252, 22, 65, 2, 69, 2, 79, 3, 85, 13, 253, 4, 9, 68, 65, 75, 85, 79, 78, + 32, 78, 71, 36, 50, 73, 214, 251, 22, 65, 2, 69, 2, 79, 3, 85, 29, 29, 5, + 78, 78, 65, 78, 32, 26, 96, 15, 78, 65, 83, 65, 76, 73, 90, 69, 68, 32, + 84, 79, 78, 69, 45, 69, 5, 84, 79, 78, 69, 45, 14, 206, 250, 22, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 55, 3, 56, 12, 138, 250, 22, 50, 2, 51, 2, + 52, 2, 53, 2, 55, 3, 56, 13, 206, 249, 22, 65, 2, 69, 2, 73, 2, 79, 3, + 85, 76, 76, 5, 77, 65, 76, 76, 32, 206, 248, 22, 65, 2, 69, 2, 73, 2, 79, + 3, 85, 66, 142, 1, 72, 2, 82, 54, 75, 46, 84, 30, 87, 46, 89, 154, 159, + 19, 78, 198, 150, 3, 83, 210, 27, 77, 234, 36, 65, 2, 69, 2, 73, 2, 79, + 3, 85, 10, 186, 247, 22, 65, 2, 69, 2, 73, 2, 79, 3, 85, 8, 134, 247, 22, + 65, 2, 69, 2, 79, 3, 85, 4, 218, 246, 22, 79, 3, 85, 8, 190, 246, 22, 65, + 2, 69, 2, 73, 3, 79, 6, 146, 246, 22, 65, 2, 79, 3, 85, 2, 189, 207, 20, + 5, 79, 73, 67, 69, 68, 8, 52, 5, 68, 79, 85, 66, 76, 22, 80, 38, 83, 35, + 86, 2, 251, 128, 8, 69, 2, 89, 6, 82, 79, 76, 79, 78, 71, 2, 29, 5, 69, + 77, 73, 45, 86, 2, 21, 3, 79, 73, 67, 2, 33, 6, 69, 68, 32, 83, 79, 85, + 2, 223, 167, 22, 78, 174, 1, 216, 1, 7, 76, 69, 84, 84, 69, 82, 32, 128, + 2, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 148, 3, 5, 83, 73, + 71, 78, 32, 88, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 162, 171, + 12, 68, 151, 224, 6, 67, 94, 226, 1, 65, 198, 166, 4, 74, 146, 236, 14, + 68, 114, 84, 230, 5, 85, 22, 86, 166, 202, 1, 73, 138, 196, 1, 78, 46, + 83, 82, 66, 2, 67, 2, 71, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, + 2, 87, 2, 89, 186, 2, 69, 3, 79, 7, 162, 240, 22, 65, 3, 73, 22, 122, 67, + 90, 68, 58, 70, 54, 83, 20, 12, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, + 83, 69, 237, 217, 21, 4, 84, 82, 73, 80, 4, 56, 8, 76, 79, 83, 73, 78, + 71, 32, 83, 199, 154, 21, 73, 2, 229, 226, 21, 2, 80, 73, 4, 11, 79, 4, + 220, 218, 21, 2, 85, 66, 203, 147, 1, 84, 4, 200, 153, 21, 5, 73, 76, 76, + 69, 68, 163, 57, 76, 6, 18, 69, 23, 80, 2, 187, 245, 10, 67, 4, 140, 151, + 4, 2, 65, 67, 171, 202, 17, 73, 12, 222, 226, 7, 75, 182, 236, 10, 67, + 98, 78, 238, 64, 82, 170, 162, 1, 86, 247, 244, 1, 65, 20, 66, 65, 246, + 164, 4, 86, 234, 239, 14, 69, 2, 85, 187, 202, 1, 73, 6, 240, 233, 16, 8, + 76, 84, 69, 82, 78, 65, 84, 69, 230, 129, 6, 65, 3, 73, 96, 148, 1, 7, + 76, 69, 84, 84, 69, 82, 32, 200, 1, 5, 83, 73, 71, 78, 32, 44, 5, 84, 79, + 78, 69, 32, 92, 6, 86, 79, 87, 69, 76, 32, 159, 243, 20, 68, 56, 138, + 206, 18, 79, 186, 7, 72, 158, 151, 2, 78, 214, 181, 1, 75, 2, 80, 2, 83, + 2, 84, 138, 69, 66, 2, 67, 2, 68, 2, 71, 2, 76, 2, 77, 2, 82, 2, 86, 2, + 87, 2, 89, 2, 90, 186, 2, 65, 3, 73, 4, 130, 243, 4, 67, 205, 242, 17, 2, + 83, 72, 6, 36, 5, 67, 65, 76, 89, 65, 23, 80, 5, 17, 2, 32, 80, 2, 249, + 240, 17, 3, 76, 79, 80, 10, 130, 167, 22, 69, 2, 85, 163, 64, 79, 36, 24, + 2, 76, 86, 23, 89, 2, 215, 234, 20, 73, 35, 68, 5, 66, 79, 65, 82, 68, + 36, 4, 67, 65, 80, 32, 243, 245, 2, 72, 5, 193, 157, 19, 4, 32, 65, 78, + 68, 26, 166, 154, 17, 78, 138, 180, 3, 65, 170, 35, 68, 135, 30, 84, 188, + 13, 202, 1, 65, 224, 9, 18, 73, 84, 65, 78, 32, 83, 77, 65, 76, 76, 32, + 83, 67, 82, 73, 80, 84, 32, 204, 3, 4, 77, 69, 82, 32, 204, 25, 5, 79, + 74, 75, 73, 32, 141, 6, 8, 85, 68, 65, 87, 65, 68, 73, 32, 138, 1, 56, 8, + 82, 79, 83, 72, 84, 72, 73, 32, 147, 190, 22, 78, 136, 1, 220, 1, 4, 68, + 73, 71, 73, 20, 7, 76, 69, 84, 84, 69, 82, 32, 144, 2, 7, 78, 85, 77, 66, + 69, 82, 32, 36, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 136, + 2, 5, 83, 73, 71, 78, 32, 210, 1, 86, 159, 201, 8, 70, 8, 243, 146, 16, + 84, 74, 182, 1, 75, 42, 84, 218, 140, 7, 78, 150, 245, 11, 68, 194, 149, + 3, 83, 82, 66, 2, 67, 2, 71, 2, 80, 2, 86, 138, 69, 72, 2, 74, 2, 76, 2, + 77, 2, 82, 2, 89, 2, 90, 187, 2, 65, 6, 170, 221, 22, 72, 2, 75, 187, 2, + 65, 12, 218, 130, 19, 84, 170, 218, 3, 72, 187, 2, 65, 8, 186, 215, 14, + 84, 243, 170, 2, 79, 18, 70, 67, 74, 68, 66, 76, 36, 5, 77, 65, 78, 71, + 65, 163, 136, 21, 83, 4, 26, 82, 251, 137, 21, 73, 2, 173, 186, 21, 6, + 69, 83, 67, 69, 78, 84, 6, 26, 79, 199, 253, 18, 65, 4, 142, 253, 18, 85, + 175, 224, 3, 84, 4, 214, 227, 6, 79, 247, 195, 7, 73, 2, 227, 159, 21, + 76, 12, 72, 2, 66, 65, 22, 67, 20, 2, 68, 79, 130, 162, 20, 86, 247, 244, + 1, 65, 2, 143, 223, 21, 82, 2, 167, 154, 6, 65, 4, 44, 5, 85, 66, 76, 69, + 32, 167, 170, 20, 84, 2, 21, 3, 82, 73, 78, 2, 139, 170, 20, 71, 14, 44, + 5, 79, 87, 69, 76, 32, 199, 158, 20, 73, 12, 44, 5, 83, 73, 71, 78, 32, + 183, 152, 22, 76, 10, 202, 147, 4, 86, 230, 198, 18, 69, 2, 73, 2, 79, 3, + 85, 176, 7, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 56, 147, + 207, 7, 70, 174, 7, 22, 66, 147, 1, 67, 128, 4, 142, 254, 7, 48, 2, 49, + 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, + 67, 2, 68, 2, 69, 3, 70, 174, 3, 142, 1, 68, 242, 251, 7, 48, 2, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, + 211, 217, 13, 70, 12, 226, 214, 22, 48, 2, 49, 2, 50, 2, 51, 2, 52, 3, + 53, 246, 2, 202, 1, 67, 240, 2, 18, 73, 78, 68, 69, 80, 69, 78, 68, 69, + 78, 84, 32, 86, 79, 87, 69, 76, 32, 220, 2, 7, 76, 69, 84, 84, 69, 82, + 32, 254, 2, 83, 208, 12, 6, 86, 79, 87, 69, 76, 32, 187, 203, 20, 68, 70, + 176, 1, 20, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 67, + 79, 69, 78, 71, 32, 245, 189, 17, 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, + 89, 77, 66, 79, 76, 32, 82, 73, 68, 138, 1, 78, 154, 4, 67, 2, 75, 90, + 80, 74, 84, 54, 68, 2, 76, 158, 132, 22, 83, 158, 41, 77, 2, 82, 2, 86, + 2, 89, 190, 28, 66, 3, 72, 8, 162, 179, 22, 71, 2, 89, 246, 30, 65, 3, + 79, 42, 82, 81, 196, 1, 11, 83, 73, 71, 78, 32, 67, 79, 69, 78, 71, 32, + 50, 76, 3, 82, 26, 82, 65, 44, 6, 79, 79, 32, 84, 89, 80, 34, 85, 178, + 195, 20, 73, 199, 140, 2, 69, 8, 190, 208, 22, 65, 2, 73, 2, 81, 3, 85, + 4, 11, 69, 4, 199, 149, 10, 32, 9, 166, 192, 7, 85, 207, 143, 15, 75, 8, + 18, 81, 31, 82, 4, 186, 207, 22, 69, 3, 85, 4, 131, 190, 18, 89, 70, 138, + 1, 67, 2, 75, 42, 78, 50, 80, 30, 83, 46, 84, 54, 68, 2, 76, 186, 173, + 22, 77, 2, 82, 2, 86, 2, 89, 190, 28, 66, 2, 72, 3, 81, 8, 210, 1, 72, + 174, 204, 22, 65, 3, 79, 8, 226, 174, 22, 71, 2, 78, 2, 89, 247, 30, 79, + 6, 122, 72, 175, 204, 22, 79, 6, 150, 174, 22, 83, 190, 28, 72, 187, 2, + 65, 12, 50, 72, 0, 2, 84, 72, 174, 204, 22, 65, 3, 79, 4, 170, 204, 22, + 65, 3, 79, 130, 1, 60, 4, 73, 71, 78, 32, 221, 6, 6, 89, 77, 66, 79, 76, + 32, 46, 202, 2, 65, 104, 10, 83, 65, 77, 89, 79, 75, 32, 83, 65, 78, 22, + 66, 118, 67, 86, 75, 74, 82, 54, 84, 188, 229, 4, 3, 76, 69, 75, 168, 11, + 6, 80, 72, 78, 65, 69, 75, 224, 232, 10, 10, 89, 85, 85, 75, 65, 76, 69, + 65, 80, 73, 244, 243, 1, 3, 78, 73, 75, 236, 171, 3, 9, 77, 85, 85, 83, + 73, 75, 65, 84, 79, 141, 15, 4, 86, 73, 82, 73, 6, 100, 9, 86, 65, 75, + 82, 65, 72, 65, 83, 65, 232, 145, 18, 4, 84, 84, 72, 65, 173, 145, 4, 2, + 72, 83, 2, 187, 197, 22, 78, 8, 38, 65, 129, 188, 21, 3, 69, 89, 89, 6, + 174, 6, 84, 216, 1, 2, 78, 84, 185, 243, 20, 6, 82, 73, 89, 79, 79, 83, + 4, 248, 155, 7, 12, 65, 77, 78, 85, 67, 32, 80, 73, 73, 32, 75, 85, 183, + 228, 12, 79, 6, 188, 221, 11, 2, 65, 75, 226, 156, 9, 72, 205, 82, 4, 79, + 79, 77, 85, 4, 130, 221, 11, 79, 137, 235, 5, 4, 69, 65, 72, 77, 4, 180, + 178, 21, 4, 82, 73, 73, 83, 217, 9, 8, 79, 65, 78, 68, 65, 75, 72, 73, + 84, 128, 1, 3, 68, 65, 80, 92, 10, 76, 69, 75, 32, 65, 84, 84, 65, 75, + 32, 182, 1, 80, 72, 5, 84, 85, 84, 69, 89, 78, 66, 47, 77, 24, 22, 45, + 223, 3, 32, 20, 30, 80, 238, 2, 66, 47, 77, 8, 138, 3, 73, 37, 3, 82, 65, + 77, 20, 50, 80, 98, 66, 186, 150, 11, 77, 219, 219, 10, 83, 12, 36, 3, + 82, 65, 77, 223, 174, 22, 73, 11, 11, 45, 8, 42, 66, 186, 150, 11, 77, + 251, 228, 8, 80, 4, 142, 242, 21, 85, 151, 60, 69, 26, 44, 2, 65, 84, 44, + 3, 82, 65, 77, 91, 73, 2, 21, 3, 72, 65, 77, 2, 175, 190, 16, 65, 20, 18, + 45, 119, 32, 16, 34, 66, 32, 2, 80, 73, 15, 77, 8, 30, 69, 37, 3, 85, 79, + 78, 4, 35, 73, 4, 21, 3, 85, 79, 89, 4, 11, 32, 4, 34, 82, 189, 138, 22, + 2, 75, 79, 2, 195, 149, 21, 79, 42, 76, 10, 73, 78, 72, 69, 82, 69, 78, + 84, 32, 65, 29, 5, 83, 73, 71, 78, 32, 4, 238, 190, 22, 65, 3, 81, 38, + 102, 65, 54, 73, 30, 79, 38, 89, 216, 254, 2, 5, 67, 79, 69, 78, 71, 182, + 172, 7, 85, 239, 145, 12, 69, 10, 194, 180, 3, 65, 170, 137, 19, 69, 2, + 73, 3, 85, 7, 182, 189, 22, 69, 3, 73, 6, 154, 189, 22, 69, 2, 77, 3, 79, + 7, 246, 188, 22, 65, 3, 89, 130, 1, 146, 1, 68, 88, 7, 76, 69, 84, 84, + 69, 82, 32, 170, 2, 83, 160, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 238, 237, 15, 87, 231, 85, 65, 6, 48, 6, 79, 85, 66, 76, 69, 32, + 155, 219, 18, 65, 4, 134, 168, 10, 83, 135, 179, 8, 68, 90, 234, 1, 83, + 254, 4, 66, 42, 71, 146, 141, 6, 68, 166, 145, 12, 74, 182, 55, 65, 150, + 1, 84, 198, 208, 1, 76, 226, 195, 1, 78, 126, 67, 2, 75, 2, 80, 138, 69, + 72, 2, 77, 2, 81, 2, 82, 2, 86, 2, 89, 186, 2, 69, 2, 73, 2, 79, 3, 85, + 4, 26, 72, 231, 184, 22, 65, 2, 209, 248, 21, 3, 79, 82, 84, 12, 40, 4, + 73, 71, 78, 32, 159, 165, 10, 69, 10, 54, 83, 226, 154, 18, 78, 154, 67, + 86, 243, 148, 3, 65, 4, 26, 72, 251, 179, 17, 85, 2, 11, 65, 2, 179, 146, + 22, 68, 18, 190, 240, 3, 86, 198, 239, 14, 65, 222, 202, 1, 73, 198, 140, + 2, 69, 2, 79, 3, 85, 138, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 176, 2, + 5, 83, 73, 71, 78, 32, 230, 194, 16, 86, 203, 252, 3, 68, 94, 222, 1, 66, + 42, 71, 146, 141, 6, 68, 254, 143, 12, 74, 222, 56, 65, 118, 82, 34, 84, + 230, 5, 85, 186, 202, 1, 73, 138, 196, 1, 78, 126, 67, 2, 75, 2, 80, 2, + 83, 138, 69, 72, 2, 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 6, 202, + 177, 22, 66, 2, 72, 187, 2, 65, 6, 162, 177, 22, 71, 2, 72, 187, 2, 65, + 6, 178, 150, 18, 78, 154, 67, 86, 243, 148, 3, 65, 136, 1, 140, 1, 8, 82, + 65, 84, 32, 82, 65, 73, 32, 216, 3, 2, 83, 83, 208, 247, 20, 4, 87, 73, + 70, 82, 202, 47, 80, 240, 93, 2, 77, 79, 191, 18, 84, 116, 140, 1, 7, 76, + 69, 84, 84, 69, 82, 32, 172, 1, 5, 83, 73, 71, 78, 32, 92, 11, 86, 79, + 87, 69, 76, 32, 83, 73, 71, 78, 32, 223, 237, 11, 68, 64, 142, 211, 18, + 68, 114, 84, 206, 223, 1, 78, 214, 181, 1, 66, 2, 67, 2, 71, 2, 74, 2, + 75, 2, 80, 2, 83, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, + 65, 12, 184, 209, 9, 3, 84, 79, 78, 0, 2, 89, 85, 190, 252, 1, 83, 198, + 156, 10, 65, 239, 1, 86, 16, 182, 215, 18, 65, 130, 151, 3, 85, 162, 64, + 69, 2, 73, 3, 79, 13, 40, 4, 73, 78, 71, 32, 183, 236, 21, 32, 8, 96, 4, + 70, 65, 67, 69, 177, 147, 14, 14, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, + 73, 84, 72, 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 158, 146, 14, 83, + 159, 155, 5, 67, 4, 232, 167, 18, 3, 69, 69, 76, 171, 232, 3, 79, 4, 40, + 4, 82, 69, 65, 78, 207, 137, 22, 65, 2, 33, 6, 32, 83, 84, 65, 78, 68, 2, + 193, 250, 18, 2, 65, 82, 2, 11, 79, 2, 171, 169, 10, 78, 200, 43, 154, 1, + 65, 178, 233, 1, 69, 198, 62, 73, 226, 83, 79, 206, 33, 85, 94, 89, 138, + 226, 7, 82, 232, 195, 9, 5, 32, 66, 32, 66, 65, 142, 163, 1, 76, 239, 66, + 70, 134, 23, 142, 1, 66, 44, 6, 67, 82, 79, 83, 83, 69, 46, 68, 58, 78, + 64, 2, 79, 32, 222, 11, 82, 236, 21, 4, 83, 84, 32, 81, 77, 4, 84, 73, + 78, 32, 4, 228, 194, 14, 2, 32, 67, 155, 218, 6, 69, 2, 137, 255, 16, 6, + 32, 83, 84, 73, 67, 75, 4, 148, 255, 12, 5, 89, 32, 66, 69, 69, 247, 234, + 8, 68, 4, 178, 202, 9, 68, 161, 241, 10, 7, 71, 85, 65, 71, 69, 32, 84, + 174, 1, 200, 2, 3, 72, 79, 32, 28, 7, 76, 69, 84, 84, 69, 82, 32, 214, 5, + 83, 148, 1, 9, 84, 79, 78, 69, 32, 77, 65, 73, 32, 84, 11, 86, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 228, 170, 4, 2, 75, 79, 224, 130, 13, 2, + 89, 65, 230, 199, 2, 69, 170, 51, 68, 212, 138, 1, 7, 67, 65, 78, 67, 69, + 76, 76, 221, 68, 6, 78, 73, 71, 71, 65, 72, 4, 186, 133, 22, 77, 3, 78, + 94, 176, 1, 3, 70, 79, 32, 78, 75, 96, 2, 76, 79, 50, 80, 154, 1, 83, 86, + 84, 30, 72, 162, 161, 16, 78, 234, 222, 5, 66, 2, 67, 2, 68, 2, 77, 2, + 82, 2, 87, 2, 89, 247, 30, 79, 8, 42, 70, 250, 174, 15, 83, 175, 182, 5, + 84, 4, 210, 210, 21, 79, 155, 62, 65, 10, 26, 72, 251, 161, 22, 79, 8, + 32, 3, 77, 85, 32, 231, 2, 79, 4, 142, 201, 21, 78, 211, 57, 71, 7, 17, + 2, 32, 76, 4, 166, 208, 21, 73, 67, 79, 30, 52, 4, 65, 76, 73, 32, 210, + 1, 72, 255, 158, 22, 79, 24, 230, 200, 6, 68, 34, 84, 206, 6, 78, 210, + 211, 13, 66, 2, 67, 2, 71, 2, 74, 147, 219, 1, 76, 8, 52, 9, 65, 78, 83, + 75, 82, 73, 84, 32, 83, 71, 79, 4, 250, 156, 22, 72, 3, 83, 6, 26, 72, + 255, 158, 22, 79, 4, 11, 79, 4, 11, 32, 4, 166, 171, 15, 83, 175, 182, 5, + 84, 6, 112, 14, 69, 77, 73, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 209, 195, 18, 8, 73, 71, 78, 32, 80, 65, 76, 73, 4, 134, 197, 21, 78, + 211, 57, 76, 8, 50, 84, 132, 163, 17, 2, 67, 65, 223, 246, 4, 69, 4, 130, + 254, 21, 72, 247, 30, 73, 32, 106, 65, 44, 5, 77, 65, 73, 32, 75, 166, + 138, 18, 89, 162, 58, 85, 186, 202, 1, 69, 2, 73, 199, 140, 2, 79, 11, + 234, 155, 22, 65, 2, 73, 2, 77, 3, 89, 4, 222, 203, 21, 65, 3, 79, 166, + 1, 32, 2, 71, 69, 255, 147, 21, 73, 164, 1, 26, 32, 171, 251, 13, 82, + 160, 1, 254, 1, 66, 44, 4, 71, 82, 69, 69, 22, 79, 250, 1, 84, 190, 242, + 11, 68, 36, 2, 85, 80, 164, 193, 3, 12, 76, 69, 70, 84, 32, 84, 82, 73, + 65, 78, 71, 76, 200, 203, 4, 5, 80, 85, 82, 80, 76, 12, 3, 82, 69, 68, 0, + 6, 89, 69, 76, 76, 79, 87, 179, 66, 67, 10, 40, 3, 82, 79, 87, 201, 1, 2, + 76, 85, 4, 227, 129, 20, 78, 10, 48, 3, 78, 69, 32, 129, 1, 4, 82, 65, + 78, 71, 4, 240, 161, 8, 5, 68, 79, 84, 32, 79, 133, 234, 2, 18, 82, 73, + 78, 71, 32, 79, 86, 69, 82, 32, 84, 87, 79, 32, 82, 73, 78, 71, 6, 11, + 69, 6, 11, 32, 6, 178, 194, 20, 67, 162, 21, 68, 155, 28, 83, 116, 156, + 1, 3, 87, 79, 32, 108, 10, 89, 80, 69, 32, 80, 73, 69, 67, 69, 32, 197, + 208, 18, 17, 82, 73, 80, 76, 69, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, + 66, 65, 4, 242, 241, 17, 68, 141, 93, 19, 82, 73, 78, 71, 83, 32, 79, 86, + 69, 82, 32, 79, 78, 69, 32, 82, 73, 78, 71, 110, 182, 1, 67, 136, 2, 9, + 68, 73, 65, 71, 79, 78, 65, 76, 32, 218, 1, 76, 186, 2, 82, 158, 1, 83, + 204, 3, 6, 85, 80, 80, 69, 82, 32, 145, 2, 9, 86, 69, 82, 84, 69, 88, 32, + 79, 70, 14, 80, 9, 69, 78, 84, 82, 69, 32, 79, 70, 32, 73, 7, 82, 79, 83, + 83, 66, 65, 82, 8, 252, 8, 6, 90, 32, 87, 73, 84, 72, 138, 137, 22, 75, + 2, 88, 3, 89, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 40, 3, 76, 79, 87, 1, + 3, 85, 80, 80, 2, 137, 205, 10, 2, 69, 82, 12, 48, 6, 85, 80, 80, 69, 82, + 32, 167, 230, 9, 76, 8, 56, 4, 76, 69, 70, 84, 225, 230, 9, 4, 82, 73, + 71, 72, 5, 11, 32, 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 76, 2, 17, 2, 79, + 87, 2, 209, 220, 21, 2, 69, 82, 26, 48, 5, 79, 87, 69, 82, 32, 129, 3, 2, + 69, 70, 24, 84, 5, 76, 69, 70, 84, 32, 56, 6, 82, 73, 71, 72, 84, 32, + 158, 6, 72, 179, 1, 84, 8, 22, 65, 207, 7, 67, 4, 194, 1, 78, 135, 226, + 20, 82, 10, 22, 65, 151, 7, 67, 6, 192, 1, 13, 78, 68, 32, 85, 80, 80, + 69, 82, 32, 82, 73, 71, 72, 249, 211, 19, 2, 82, 67, 4, 44, 4, 65, 73, + 83, 69, 77, 3, 73, 71, 72, 2, 53, 11, 68, 32, 85, 80, 80, 69, 82, 32, 76, + 69, 70, 2, 231, 139, 19, 84, 2, 137, 141, 21, 3, 84, 32, 65, 34, 48, 5, + 72, 79, 82, 84, 32, 77, 3, 84, 69, 77, 4, 40, 3, 76, 79, 87, 1, 3, 85, + 80, 80, 2, 217, 4, 4, 69, 82, 32, 84, 31, 44, 6, 32, 87, 73, 84, 72, 32, + 171, 1, 45, 8, 44, 5, 76, 69, 70, 84, 32, 30, 82, 59, 67, 4, 82, 67, 199, + 221, 10, 74, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, 67, 2, 169, + 229, 20, 4, 82, 79, 83, 83, 20, 48, 2, 49, 50, 22, 50, 26, 52, 203, 189, + 11, 51, 5, 183, 223, 14, 51, 9, 11, 51, 7, 11, 52, 5, 239, 135, 22, 53, + 18, 62, 72, 96, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 83, 84, 4, 65, 14, + 65, 76, 70, 32, 86, 69, 82, 84, 69, 88, 32, 79, 70, 32, 4, 214, 134, 22, + 77, 3, 87, 6, 17, 2, 84, 32, 6, 26, 67, 175, 134, 19, 65, 4, 202, 92, 82, + 235, 168, 17, 79, 2, 157, 250, 8, 3, 69, 82, 77, 2, 187, 231, 14, 32, 6, + 53, 11, 85, 65, 82, 84, 69, 82, 32, 77, 79, 79, 78, 7, 223, 240, 6, 32, + 158, 20, 150, 1, 67, 224, 46, 18, 69, 80, 73, 71, 82, 65, 80, 72, 73, 67, + 32, 76, 69, 84, 84, 69, 82, 32, 252, 1, 7, 76, 69, 84, 84, 69, 82, 32, + 247, 12, 83, 206, 7, 56, 8, 65, 80, 73, 84, 65, 76, 32, 76, 243, 191, 20, + 82, 204, 7, 76, 6, 69, 84, 84, 69, 82, 32, 169, 45, 8, 73, 71, 65, 84, + 85, 82, 69, 32, 200, 7, 154, 2, 65, 158, 3, 66, 154, 1, 67, 254, 1, 68, + 186, 2, 69, 174, 2, 70, 82, 71, 194, 1, 72, 146, 1, 73, 154, 2, 74, 118, + 75, 126, 76, 234, 2, 77, 114, 78, 134, 2, 79, 198, 2, 80, 114, 81, 62, + 82, 190, 2, 83, 130, 3, 84, 142, 3, 85, 234, 1, 86, 146, 1, 87, 118, 88, + 50, 89, 167, 1, 90, 93, 172, 1, 6, 32, 87, 73, 84, 72, 32, 166, 1, 69, + 246, 65, 78, 176, 158, 14, 6, 70, 82, 73, 67, 65, 78, 158, 193, 3, 76, + 210, 183, 2, 86, 186, 164, 1, 65, 2, 79, 2, 85, 3, 89, 66, 246, 64, 66, + 34, 68, 108, 3, 82, 73, 78, 242, 33, 77, 250, 21, 67, 150, 50, 72, 158, + 1, 79, 198, 10, 71, 186, 2, 65, 246, 173, 3, 73, 142, 175, 13, 84, 219, + 183, 3, 83, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 206, 184, 1, 65, 239, + 199, 3, 77, 21, 60, 6, 32, 87, 73, 84, 72, 32, 190, 68, 82, 187, 147, 21, + 69, 14, 154, 67, 84, 138, 60, 70, 210, 57, 76, 182, 159, 16, 68, 170, + 206, 2, 72, 247, 165, 1, 83, 33, 108, 6, 32, 87, 73, 84, 72, 32, 204, 70, + 7, 76, 79, 83, 69, 68, 32, 73, 54, 85, 154, 228, 20, 79, 139, 60, 72, 20, + 94, 67, 198, 181, 1, 65, 154, 233, 3, 80, 206, 133, 15, 72, 182, 50, 66, + 242, 34, 68, 211, 80, 83, 8, 198, 68, 69, 190, 113, 73, 227, 156, 19, 65, + 35, 76, 6, 32, 87, 73, 84, 72, 32, 178, 1, 90, 25, 6, 79, 85, 66, 76, 69, + 32, 24, 78, 83, 226, 15, 67, 202, 47, 84, 218, 117, 76, 182, 159, 16, 68, + 171, 206, 2, 72, 8, 92, 13, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, + 32, 90, 150, 138, 1, 72, 235, 189, 20, 84, 5, 209, 72, 2, 32, 87, 4, 254, + 71, 87, 187, 185, 10, 84, 91, 104, 6, 32, 87, 73, 84, 72, 32, 152, 1, 2, + 90, 72, 134, 76, 71, 206, 152, 17, 84, 210, 143, 4, 83, 63, 78, 70, 254, + 73, 67, 158, 2, 68, 194, 16, 84, 158, 23, 77, 250, 1, 86, 238, 45, 72, + 158, 1, 79, 198, 10, 71, 186, 2, 65, 246, 173, 3, 73, 62, 66, 171, 230, + 16, 83, 7, 11, 32, 4, 138, 70, 87, 211, 8, 82, 9, 33, 6, 32, 87, 73, 84, + 72, 32, 6, 242, 158, 20, 72, 166, 85, 68, 211, 80, 83, 35, 76, 6, 32, 87, + 73, 84, 72, 32, 190, 81, 76, 230, 204, 18, 65, 147, 211, 2, 72, 20, 154, + 84, 67, 250, 90, 65, 178, 174, 3, 66, 190, 25, 77, 226, 140, 4, 79, 154, + 154, 11, 72, 166, 85, 68, 211, 80, 83, 29, 76, 6, 32, 87, 73, 84, 72, 32, + 250, 126, 65, 198, 244, 7, 87, 247, 173, 12, 69, 20, 186, 82, 66, 34, 67, + 46, 68, 178, 201, 19, 72, 247, 165, 1, 83, 59, 80, 6, 32, 87, 73, 84, 72, + 32, 132, 88, 2, 78, 83, 198, 244, 20, 79, 207, 36, 83, 40, 138, 1, 68, + 162, 83, 67, 242, 1, 77, 142, 1, 84, 130, 71, 72, 158, 1, 79, 198, 10, + 71, 186, 2, 65, 246, 173, 3, 73, 62, 66, 171, 230, 16, 83, 10, 22, 79, + 191, 83, 73, 6, 218, 121, 85, 131, 212, 18, 84, 11, 33, 6, 32, 87, 73, + 84, 72, 32, 8, 42, 67, 174, 135, 18, 84, 219, 183, 3, 83, 4, 234, 170, 1, + 73, 131, 223, 3, 82, 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 182, 89, 67, + 34, 68, 50, 83, 222, 79, 65, 138, 1, 76, 198, 211, 7, 79, 155, 154, 11, + 72, 41, 60, 6, 32, 87, 73, 84, 72, 32, 190, 94, 65, 231, 142, 21, 74, 32, + 138, 1, 66, 36, 2, 68, 79, 52, 7, 77, 73, 68, 68, 76, 69, 32, 38, 83, + 174, 2, 67, 182, 91, 72, 230, 72, 65, 138, 1, 76, 251, 219, 16, 84, 4, + 170, 198, 12, 69, 143, 237, 8, 65, 6, 22, 85, 231, 91, 84, 2, 249, 241, + 4, 2, 66, 76, 4, 230, 131, 18, 84, 159, 151, 3, 68, 4, 214, 2, 77, 211, + 184, 21, 84, 19, 44, 6, 32, 87, 73, 84, 72, 32, 207, 94, 73, 10, 242, + 165, 1, 65, 190, 160, 16, 68, 198, 60, 84, 231, 145, 2, 72, 33, 48, 6, + 32, 87, 73, 84, 72, 32, 215, 233, 21, 74, 28, 102, 67, 44, 2, 83, 77, + 178, 96, 76, 134, 65, 71, 186, 2, 65, 102, 68, 234, 211, 7, 79, 183, 136, + 9, 84, 6, 190, 132, 1, 69, 22, 73, 231, 188, 19, 65, 2, 157, 233, 10, 10, + 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 101, 108, 6, 32, 87, 73, 84, 72, + 32, 178, 103, 76, 138, 159, 4, 80, 218, 155, 9, 77, 134, 197, 7, 73, 2, + 79, 3, 85, 84, 144, 1, 2, 76, 79, 34, 77, 226, 96, 67, 70, 68, 154, 2, + 79, 38, 83, 66, 84, 106, 86, 222, 44, 72, 242, 12, 71, 186, 2, 65, 246, + 173, 3, 73, 63, 66, 4, 158, 99, 78, 215, 130, 21, 79, 8, 154, 99, 65, + 235, 159, 4, 73, 19, 44, 6, 32, 87, 73, 84, 72, 32, 155, 23, 72, 14, 242, + 103, 70, 34, 83, 170, 56, 65, 230, 238, 18, 72, 167, 85, 68, 7, 33, 6, + 32, 87, 73, 84, 72, 32, 4, 190, 105, 68, 39, 83, 43, 94, 32, 156, 1, 8, + 69, 86, 69, 82, 83, 69, 68, 32, 216, 111, 3, 85, 77, 32, 167, 147, 4, 65, + 28, 40, 5, 87, 73, 84, 72, 32, 215, 112, 82, 26, 134, 78, 67, 218, 29, + 68, 170, 2, 84, 174, 48, 65, 138, 1, 76, 238, 172, 3, 73, 218, 166, 4, + 79, 143, 192, 12, 83, 8, 218, 110, 72, 154, 156, 4, 79, 186, 199, 13, 67, + 239, 143, 3, 69, 49, 136, 1, 6, 32, 87, 73, 84, 72, 32, 136, 1, 5, 77, + 65, 76, 76, 32, 240, 115, 2, 65, 76, 234, 1, 72, 152, 2, 2, 73, 71, 167, + 138, 4, 67, 32, 86, 67, 138, 112, 65, 118, 68, 138, 1, 83, 174, 1, 86, + 190, 252, 7, 79, 155, 154, 11, 72, 10, 226, 112, 65, 230, 10, 69, 82, 79, + 203, 31, 73, 4, 132, 168, 19, 11, 81, 32, 87, 73, 84, 72, 32, 72, 79, 79, + 75, 173, 247, 1, 7, 67, 65, 80, 73, 84, 65, 76, 59, 136, 1, 6, 32, 87, + 73, 84, 72, 32, 168, 1, 6, 85, 82, 78, 69, 68, 32, 188, 123, 2, 72, 79, + 176, 1, 2, 79, 78, 74, 82, 243, 222, 20, 90, 22, 82, 67, 50, 68, 254, + 152, 1, 76, 234, 237, 3, 82, 246, 255, 14, 72, 247, 165, 1, 83, 8, 202, + 120, 69, 22, 73, 62, 79, 171, 188, 19, 65, 6, 190, 142, 1, 73, 255, 169, + 16, 79, 18, 246, 39, 73, 242, 138, 3, 65, 190, 169, 18, 72, 2, 75, 2, 76, + 2, 77, 2, 84, 3, 86, 79, 26, 32, 183, 135, 5, 80, 74, 44, 5, 87, 73, 84, + 72, 32, 199, 183, 20, 66, 72, 166, 132, 1, 67, 74, 68, 150, 2, 72, 170, + 1, 77, 134, 1, 79, 142, 1, 84, 186, 9, 71, 186, 2, 65, 246, 173, 3, 73, + 62, 66, 234, 225, 14, 82, 195, 132, 2, 83, 23, 88, 6, 32, 87, 73, 84, 72, + 32, 178, 139, 1, 73, 66, 79, 134, 189, 19, 69, 151, 144, 1, 89, 8, 226, + 138, 1, 68, 210, 230, 16, 84, 231, 145, 2, 72, 19, 48, 6, 32, 87, 73, 84, + 72, 32, 243, 151, 7, 89, 14, 190, 144, 1, 67, 18, 68, 70, 71, 186, 2, 65, + 231, 238, 18, 72, 7, 201, 140, 1, 7, 32, 87, 73, 84, 72, 32, 68, 29, 48, + 6, 32, 87, 73, 84, 72, 32, 219, 169, 17, 79, 24, 154, 143, 1, 67, 18, 68, + 70, 71, 22, 72, 42, 76, 254, 1, 65, 238, 199, 3, 77, 150, 149, 13, 84, + 219, 183, 3, 83, 25, 33, 6, 32, 87, 73, 84, 72, 32, 22, 254, 56, 67, 150, + 88, 65, 102, 68, 38, 76, 34, 83, 242, 231, 3, 80, 207, 133, 15, 72, 4, + 254, 213, 10, 73, 195, 232, 10, 79, 12, 128, 1, 5, 65, 82, 67, 72, 65, + 30, 73, 64, 9, 82, 69, 86, 69, 82, 83, 69, 68, 32, 201, 211, 10, 7, 83, + 73, 68, 69, 87, 65, 89, 2, 217, 246, 13, 2, 73, 67, 4, 220, 107, 5, 78, + 86, 69, 82, 84, 149, 216, 14, 3, 32, 76, 79, 4, 142, 211, 21, 70, 3, 80, + 138, 1, 242, 2, 65, 36, 2, 66, 73, 152, 1, 21, 73, 78, 86, 69, 82, 84, + 69, 68, 32, 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 72, 2, 80, + 72, 16, 2, 82, 69, 222, 1, 83, 132, 8, 3, 84, 87, 79, 182, 20, 87, 128, + 170, 4, 2, 68, 69, 148, 5, 2, 76, 65, 206, 11, 71, 212, 196, 15, 20, 86, + 79, 73, 67, 69, 68, 32, 76, 65, 82, 89, 78, 71, 69, 65, 76, 32, 83, 80, + 73, 199, 119, 89, 4, 230, 187, 4, 76, 183, 196, 16, 73, 6, 76, 7, 76, 65, + 66, 73, 65, 76, 32, 29, 8, 68, 69, 78, 84, 65, 76, 32, 80, 4, 26, 80, + 239, 206, 4, 67, 2, 153, 146, 16, 6, 69, 82, 67, 85, 83, 83, 7, 33, 6, + 32, 87, 73, 84, 72, 32, 4, 234, 243, 4, 67, 183, 170, 16, 83, 2, 207, 81, + 65, 8, 104, 7, 86, 69, 82, 83, 69, 68, 32, 233, 243, 4, 13, 84, 82, 79, + 70, 76, 69, 88, 32, 67, 76, 73, 67, 75, 4, 80, 3, 69, 83, 72, 161, 8, 12, + 71, 76, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 2, 213, 133, 1, 2, 32, + 76, 96, 172, 1, 13, 77, 65, 76, 76, 32, 67, 65, 80, 73, 84, 65, 76, 32, + 244, 112, 10, 84, 82, 69, 84, 67, 72, 69, 68, 32, 67, 185, 198, 19, 10, + 73, 78, 79, 76, 79, 71, 73, 67, 65, 76, 90, 230, 1, 69, 30, 73, 22, 76, + 74, 79, 42, 82, 126, 84, 254, 182, 4, 66, 214, 41, 71, 234, 165, 16, 65, + 162, 64, 67, 2, 68, 2, 70, 2, 72, 2, 74, 2, 75, 2, 77, 2, 78, 2, 80, 2, + 81, 2, 83, 2, 85, 2, 86, 2, 87, 2, 89, 3, 90, 7, 226, 199, 21, 84, 3, 90, + 5, 147, 220, 4, 78, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 238, 225, 8, 66, + 183, 182, 12, 83, 9, 238, 95, 80, 142, 232, 20, 69, 3, 85, 11, 92, 8, 69, + 86, 69, 82, 83, 69, 68, 32, 236, 128, 1, 5, 32, 87, 73, 84, 72, 179, 181, + 20, 85, 4, 242, 198, 21, 78, 3, 82, 13, 33, 6, 85, 82, 78, 69, 68, 32, + 10, 178, 198, 21, 69, 2, 71, 2, 75, 2, 77, 3, 82, 186, 11, 136, 1, 5, 77, + 65, 76, 76, 32, 145, 131, 1, 22, 85, 66, 83, 67, 82, 73, 80, 84, 32, 83, + 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 150, 11, 76, 15, 67, 65, + 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 43, 76, 4, 18, 73, 3, + 85, 2, 255, 226, 4, 32, 146, 11, 80, 6, 69, 84, 84, 69, 82, 32, 161, 128, + 1, 8, 73, 71, 65, 84, 85, 82, 69, 32, 128, 11, 182, 2, 65, 190, 5, 66, + 198, 3, 67, 186, 4, 68, 182, 4, 69, 174, 9, 70, 214, 1, 71, 162, 2, 72, + 230, 2, 73, 166, 7, 74, 182, 1, 75, 178, 2, 76, 206, 6, 77, 178, 2, 78, + 218, 3, 79, 242, 8, 80, 218, 2, 81, 174, 1, 82, 142, 8, 83, 250, 10, 84, + 246, 13, 85, 218, 9, 86, 130, 3, 87, 110, 88, 226, 2, 89, 171, 3, 90, + 101, 118, 32, 198, 3, 69, 82, 78, 192, 226, 4, 4, 76, 80, 72, 65, 222, + 180, 15, 86, 186, 164, 1, 65, 2, 79, 2, 85, 3, 89, 72, 88, 5, 87, 73, 84, + 72, 32, 157, 229, 5, 11, 82, 69, 86, 69, 82, 83, 69, 68, 45, 83, 67, 70, + 150, 1, 66, 34, 68, 54, 82, 170, 34, 77, 250, 21, 67, 150, 50, 72, 158, + 1, 79, 198, 10, 71, 186, 2, 65, 246, 173, 3, 73, 142, 175, 13, 84, 219, + 183, 3, 83, 12, 161, 106, 4, 82, 69, 86, 69, 12, 22, 79, 151, 57, 73, 8, + 214, 57, 84, 211, 13, 85, 10, 26, 73, 175, 231, 4, 69, 8, 26, 78, 179, + 174, 4, 71, 6, 17, 2, 71, 32, 6, 236, 58, 4, 65, 66, 79, 86, 251, 223, + 18, 66, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 242, 116, 71, 186, 2, 65, + 239, 199, 3, 77, 2, 181, 198, 10, 7, 71, 76, 73, 67, 65, 78, 65, 41, 140, + 1, 6, 32, 87, 73, 84, 72, 32, 130, 1, 65, 108, 11, 76, 65, 67, 75, 76, + 69, 84, 84, 69, 82, 32, 38, 82, 174, 201, 4, 79, 143, 202, 16, 69, 18, + 110, 84, 138, 60, 70, 210, 57, 76, 230, 224, 3, 77, 174, 7, 80, 166, 183, + 12, 68, 170, 206, 2, 72, 247, 165, 1, 83, 2, 255, 7, 79, 8, 64, 5, 82, + 82, 69, 68, 32, 161, 81, 6, 83, 69, 76, 73, 78, 69, 6, 182, 32, 65, 154, + 152, 21, 69, 3, 79, 6, 242, 195, 4, 79, 183, 244, 16, 69, 2, 181, 186, + 10, 4, 79, 75, 69, 78, 47, 108, 6, 32, 87, 73, 84, 72, 32, 196, 1, 2, 72, + 73, 92, 6, 76, 79, 83, 69, 68, 32, 90, 85, 155, 228, 20, 79, 24, 102, 67, + 182, 113, 65, 154, 233, 3, 80, 218, 5, 82, 246, 255, 14, 72, 182, 50, 66, + 242, 34, 68, 211, 80, 83, 10, 54, 69, 190, 113, 73, 150, 251, 6, 85, 207, + 161, 12, 65, 4, 245, 51, 5, 68, 73, 76, 76, 65, 7, 49, 10, 32, 87, 73, + 84, 72, 32, 76, 79, 87, 32, 4, 162, 107, 82, 45, 4, 76, 69, 70, 84, 8, + 34, 73, 18, 79, 159, 200, 4, 82, 2, 163, 88, 78, 4, 130, 221, 4, 80, 151, + 146, 9, 77, 4, 37, 7, 65, 84, 82, 73, 76, 76, 79, 5, 209, 142, 13, 5, 32, + 87, 73, 84, 72, 73, 96, 6, 32, 87, 73, 84, 72, 32, 182, 1, 69, 34, 79, + 178, 1, 90, 178, 213, 4, 66, 187, 201, 16, 85, 32, 98, 83, 34, 84, 238, + 32, 67, 226, 45, 77, 170, 31, 76, 138, 217, 3, 72, 138, 15, 80, 167, 183, + 12, 68, 4, 134, 68, 72, 235, 189, 20, 84, 4, 26, 79, 215, 209, 19, 65, 2, + 219, 141, 20, 80, 8, 246, 78, 90, 207, 189, 20, 76, 16, 60, 6, 84, 76, + 69, 83, 83, 32, 49, 5, 85, 66, 76, 69, 32, 8, 26, 74, 151, 176, 21, 73, + 7, 167, 198, 4, 32, 8, 42, 87, 142, 202, 4, 82, 175, 239, 5, 84, 2, 163, + 239, 6, 89, 11, 11, 32, 8, 26, 87, 151, 198, 4, 68, 2, 169, 90, 5, 73, + 84, 72, 32, 67, 119, 112, 6, 32, 87, 73, 84, 72, 32, 214, 4, 71, 72, 2, + 78, 71, 68, 2, 83, 72, 164, 1, 2, 90, 72, 159, 150, 17, 84, 76, 182, 1, + 67, 158, 2, 68, 110, 78, 214, 15, 84, 158, 23, 77, 250, 1, 86, 190, 3, + 70, 178, 42, 72, 158, 1, 79, 198, 10, 71, 186, 2, 65, 246, 173, 3, 73, + 62, 66, 194, 64, 82, 235, 165, 16, 83, 24, 92, 6, 69, 68, 73, 76, 76, 65, + 32, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 151, 132, 20, 65, 5, 169, 149, + 4, 3, 32, 65, 78, 19, 11, 32, 16, 40, 4, 65, 78, 68, 32, 167, 137, 19, + 66, 14, 162, 86, 67, 130, 2, 72, 226, 11, 71, 186, 2, 65, 238, 199, 3, + 77, 158, 170, 4, 68, 251, 234, 8, 84, 12, 22, 79, 227, 98, 73, 10, 28, 2, + 84, 32, 227, 51, 85, 8, 192, 88, 5, 65, 66, 79, 86, 69, 199, 175, 18, 66, + 2, 131, 154, 14, 79, 4, 157, 134, 7, 13, 89, 80, 84, 79, 76, 79, 71, 73, + 67, 65, 76, 32, 65, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 158, 195, 4, 67, + 231, 9, 80, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 88, 10, 68, 79, 85, + 66, 76, 69, 32, 66, 65, 82, 230, 203, 4, 80, 142, 1, 67, 207, 4, 82, 5, + 217, 204, 4, 4, 32, 65, 78, 68, 15, 11, 32, 12, 38, 82, 37, 5, 87, 73, + 84, 72, 32, 2, 157, 150, 14, 4, 69, 86, 69, 82, 10, 54, 67, 178, 202, 4, + 80, 218, 5, 82, 195, 158, 14, 84, 4, 234, 220, 7, 85, 207, 161, 12, 65, + 17, 84, 6, 32, 87, 73, 84, 72, 32, 73, 11, 69, 78, 71, 32, 68, 73, 71, + 82, 65, 80, 72, 10, 134, 194, 4, 77, 174, 7, 80, 206, 133, 15, 72, 166, + 85, 68, 211, 80, 83, 5, 209, 168, 18, 8, 32, 87, 73, 84, 72, 32, 84, 82, + 37, 72, 6, 32, 87, 73, 84, 72, 32, 126, 76, 230, 204, 18, 65, 147, 211, + 2, 72, 22, 218, 3, 67, 250, 90, 65, 178, 174, 3, 66, 190, 25, 77, 174, + 33, 80, 182, 235, 3, 79, 154, 154, 11, 72, 166, 85, 68, 211, 80, 83, 8, + 33, 6, 79, 84, 84, 65, 76, 32, 8, 142, 205, 18, 83, 250, 212, 2, 65, 2, + 73, 3, 85, 39, 140, 1, 6, 32, 87, 73, 84, 72, 32, 150, 45, 65, 232, 25, + 12, 79, 79, 75, 69, 68, 32, 83, 67, 72, 87, 65, 32, 174, 243, 3, 69, 159, + 230, 16, 86, 24, 86, 66, 34, 67, 46, 68, 214, 91, 76, 146, 232, 3, 80, + 206, 133, 15, 72, 247, 165, 1, 83, 2, 205, 147, 13, 3, 82, 69, 86, 6, + 158, 59, 69, 154, 32, 73, 227, 156, 19, 65, 8, 234, 87, 73, 226, 190, 3, + 69, 203, 228, 12, 79, 75, 80, 6, 32, 87, 73, 84, 72, 32, 222, 4, 78, 188, + 1, 2, 79, 84, 135, 152, 21, 83, 48, 178, 1, 67, 34, 68, 210, 1, 77, 64, + 6, 79, 71, 79, 78, 69, 75, 78, 84, 130, 71, 72, 226, 11, 71, 186, 2, 65, + 246, 173, 3, 73, 62, 66, 144, 64, 6, 83, 84, 82, 79, 75, 69, 51, 82, 4, + 210, 88, 73, 227, 156, 19, 65, 14, 18, 73, 47, 79, 4, 217, 26, 7, 65, 69, + 82, 69, 83, 73, 83, 10, 28, 2, 84, 32, 215, 37, 85, 8, 64, 10, 65, 66, + 79, 86, 69, 32, 65, 78, 68, 32, 187, 249, 18, 66, 6, 150, 84, 71, 186, 2, + 65, 131, 221, 16, 84, 4, 29, 5, 65, 67, 82, 79, 78, 5, 217, 36, 4, 32, + 65, 78, 68, 7, 145, 73, 15, 32, 65, 78, 68, 32, 68, 79, 84, 32, 65, 66, + 79, 86, 69, 32, 4, 21, 3, 73, 76, 68, 4, 151, 142, 5, 69, 16, 46, 83, 93, + 7, 86, 69, 82, 84, 69, 68, 32, 12, 29, 5, 85, 76, 65, 82, 32, 12, 238, + 152, 21, 68, 2, 70, 2, 71, 2, 82, 2, 83, 3, 84, 4, 26, 65, 199, 129, 21, + 79, 2, 143, 188, 17, 76, 6, 206, 163, 4, 65, 129, 188, 6, 5, 73, 70, 73, + 69, 68, 13, 33, 6, 32, 87, 73, 84, 72, 32, 10, 94, 67, 148, 180, 4, 13, + 68, 79, 84, 32, 65, 66, 79, 86, 69, 32, 65, 78, 68, 187, 178, 16, 83, 6, + 178, 82, 73, 130, 223, 3, 82, 227, 189, 15, 65, 29, 48, 6, 32, 87, 73, + 84, 72, 32, 175, 147, 21, 82, 24, 98, 67, 34, 68, 50, 83, 222, 79, 65, + 138, 1, 76, 146, 232, 3, 80, 182, 235, 3, 79, 155, 154, 11, 72, 4, 210, + 48, 69, 251, 188, 19, 65, 6, 214, 70, 73, 182, 197, 3, 69, 151, 182, 4, + 79, 4, 29, 5, 84, 82, 79, 75, 69, 5, 161, 25, 6, 32, 65, 78, 68, 32, 68, + 79, 136, 1, 6, 32, 87, 73, 84, 72, 32, 250, 3, 65, 38, 69, 48, 5, 79, 78, + 71, 32, 83, 130, 180, 4, 83, 2, 90, 186, 201, 16, 85, 219, 16, 74, 50, + 178, 1, 66, 86, 67, 56, 2, 68, 79, 100, 3, 77, 73, 68, 130, 2, 72, 230, + 72, 65, 138, 1, 76, 150, 224, 3, 73, 162, 1, 82, 222, 2, 70, 130, 4, 80, + 234, 243, 12, 84, 219, 183, 3, 83, 6, 36, 3, 69, 76, 84, 167, 216, 20, + 65, 5, 193, 181, 4, 6, 32, 65, 78, 68, 32, 80, 8, 166, 44, 69, 22, 73, + 154, 155, 7, 85, 207, 161, 12, 65, 8, 38, 84, 25, 5, 85, 66, 76, 69, 32, + 4, 141, 25, 2, 32, 66, 4, 242, 172, 4, 77, 175, 191, 15, 66, 8, 36, 4, + 68, 76, 69, 32, 207, 44, 45, 6, 186, 167, 17, 84, 210, 150, 3, 82, 79, + 68, 4, 173, 154, 4, 4, 77, 66, 68, 65, 6, 158, 181, 4, 90, 189, 138, 13, + 3, 78, 73, 83, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 18, 68, 35, 72, 4, + 206, 63, 73, 203, 205, 19, 79, 2, 197, 172, 4, 2, 73, 71, 27, 56, 6, 32, + 87, 73, 84, 72, 32, 102, 73, 167, 251, 20, 85, 16, 138, 72, 65, 182, 223, + 3, 67, 186, 2, 77, 174, 7, 80, 166, 183, 12, 68, 198, 60, 84, 231, 145, + 2, 72, 6, 25, 4, 68, 68, 76, 69, 6, 76, 7, 45, 87, 69, 76, 83, 72, 32, + 173, 145, 17, 6, 32, 83, 67, 79, 84, 83, 4, 190, 255, 19, 76, 211, 139, + 1, 86, 49, 58, 32, 216, 2, 2, 71, 32, 130, 247, 20, 85, 219, 16, 74, 40, + 88, 5, 87, 73, 84, 72, 32, 209, 142, 6, 11, 80, 82, 69, 67, 69, 68, 69, + 68, 32, 66, 89, 38, 122, 67, 74, 76, 158, 37, 77, 234, 27, 71, 186, 2, + 65, 102, 68, 182, 232, 3, 80, 218, 5, 82, 222, 229, 3, 79, 183, 136, 9, + 84, 10, 170, 36, 69, 22, 73, 134, 255, 3, 82, 150, 156, 3, 85, 207, 161, + 12, 65, 6, 132, 66, 3, 79, 78, 71, 202, 2, 73, 183, 239, 3, 69, 2, 25, 4, + 87, 73, 84, 72, 2, 177, 224, 17, 5, 32, 84, 73, 76, 68, 115, 116, 6, 32, + 87, 73, 84, 72, 32, 186, 6, 76, 52, 4, 80, 69, 78, 32, 174, 186, 13, 77, + 134, 197, 7, 73, 2, 79, 3, 85, 86, 154, 1, 67, 70, 68, 152, 1, 2, 76, 79, + 86, 77, 46, 79, 38, 83, 66, 84, 106, 86, 222, 44, 72, 242, 12, 71, 186, + 2, 65, 246, 173, 3, 73, 62, 66, 195, 64, 82, 14, 172, 49, 9, 73, 82, 67, + 85, 77, 70, 76, 69, 88, 159, 172, 19, 65, 14, 18, 73, 47, 79, 4, 221, 13, + 7, 65, 69, 82, 69, 83, 73, 83, 10, 22, 84, 171, 47, 85, 6, 11, 32, 6, + 140, 13, 5, 65, 66, 79, 86, 69, 223, 212, 18, 66, 6, 66, 78, 216, 208, + 12, 6, 87, 32, 82, 73, 78, 71, 255, 177, 8, 79, 2, 159, 21, 71, 6, 11, + 65, 6, 189, 2, 4, 67, 82, 79, 78, 4, 217, 11, 5, 71, 79, 78, 69, 75, 4, + 25, 4, 84, 82, 79, 75, 4, 11, 69, 5, 213, 49, 2, 32, 65, 8, 25, 4, 73, + 76, 68, 69, 9, 29, 5, 32, 65, 78, 68, 32, 6, 162, 47, 68, 142, 13, 65, + 239, 199, 3, 77, 6, 81, 18, 69, 82, 84, 73, 67, 65, 76, 32, 76, 73, 78, + 69, 32, 66, 69, 76, 79, 87, 7, 221, 43, 4, 32, 65, 78, 68, 2, 153, 136, + 10, 8, 68, 32, 80, 79, 76, 73, 83, 72, 16, 26, 79, 131, 166, 4, 69, 13, + 48, 6, 32, 87, 73, 84, 72, 32, 227, 254, 20, 69, 8, 210, 55, 71, 186, 2, + 65, 242, 238, 3, 82, 235, 165, 16, 83, 25, 44, 6, 32, 87, 73, 84, 72, 32, + 179, 1, 72, 18, 86, 70, 34, 83, 170, 56, 65, 238, 225, 3, 77, 174, 7, 80, + 206, 133, 15, 72, 167, 85, 68, 2, 153, 196, 14, 3, 76, 79, 85, 6, 210, + 28, 84, 209, 175, 12, 6, 81, 85, 73, 82, 82, 69, 4, 26, 65, 171, 252, 20, + 73, 2, 193, 183, 5, 18, 82, 89, 78, 71, 69, 65, 76, 32, 86, 79, 73, 67, + 69, 68, 32, 70, 82, 73, 13, 48, 6, 32, 87, 73, 84, 72, 32, 139, 161, 4, + 80, 8, 42, 68, 16, 4, 72, 79, 79, 75, 23, 83, 2, 227, 44, 73, 5, 171, + 195, 18, 32, 2, 197, 26, 6, 84, 82, 79, 75, 69, 32, 77, 90, 32, 228, 4, + 8, 69, 86, 69, 82, 83, 69, 68, 32, 148, 2, 2, 85, 77, 179, 147, 4, 65, + 46, 36, 4, 87, 73, 84, 72, 235, 6, 82, 44, 26, 32, 195, 190, 15, 79, 42, + 166, 1, 67, 50, 68, 200, 1, 8, 70, 73, 83, 72, 72, 79, 79, 75, 66, 76, + 34, 84, 142, 18, 77, 162, 30, 65, 246, 173, 3, 73, 166, 59, 80, 182, 235, + 3, 79, 143, 192, 12, 83, 6, 170, 19, 69, 154, 255, 3, 82, 227, 189, 15, + 65, 8, 11, 79, 8, 24, 2, 84, 32, 111, 85, 6, 26, 66, 251, 249, 19, 65, 4, + 25, 4, 69, 76, 79, 87, 5, 29, 5, 32, 65, 78, 68, 32, 2, 191, 249, 3, 77, + 2, 21, 3, 66, 76, 69, 2, 11, 32, 2, 227, 46, 71, 7, 29, 5, 32, 65, 78, + 68, 32, 4, 214, 146, 4, 77, 175, 7, 80, 4, 222, 49, 73, 131, 236, 3, 79, + 4, 238, 215, 18, 73, 195, 61, 65, 22, 162, 1, 72, 40, 6, 79, 80, 69, 78, + 32, 69, 216, 147, 4, 8, 82, 32, 87, 73, 84, 72, 32, 70, 168, 2, 3, 83, + 67, 82, 190, 223, 14, 69, 190, 87, 67, 159, 166, 1, 75, 2, 11, 65, 2, + 253, 250, 9, 2, 76, 70, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 138, 157, 4, + 82, 247, 255, 14, 72, 5, 11, 32, 2, 11, 82, 2, 185, 146, 17, 3, 79, 84, + 85, 85, 180, 1, 6, 32, 87, 73, 84, 72, 32, 218, 4, 65, 66, 67, 218, 1, + 72, 34, 73, 156, 2, 13, 81, 85, 65, 84, 32, 82, 69, 86, 69, 82, 83, 69, + 68, 221, 235, 9, 6, 84, 73, 82, 82, 85, 80, 40, 110, 65, 34, 67, 86, 68, + 138, 1, 83, 174, 1, 86, 210, 9, 77, 186, 135, 4, 80, 182, 235, 3, 79, + 155, 154, 11, 72, 4, 205, 1, 4, 67, 85, 84, 69, 12, 58, 65, 230, 10, 69, + 82, 79, 202, 31, 73, 151, 251, 6, 85, 4, 113, 3, 82, 79, 78, 8, 32, 3, + 79, 84, 32, 207, 32, 73, 6, 26, 66, 207, 241, 19, 65, 4, 25, 4, 69, 76, + 79, 87, 5, 11, 32, 2, 173, 237, 19, 3, 65, 78, 68, 4, 22, 72, 203, 42, + 87, 2, 21, 3, 79, 82, 84, 2, 17, 2, 32, 83, 2, 11, 84, 2, 21, 3, 82, 79, + 75, 2, 11, 69, 2, 17, 2, 32, 79, 2, 197, 222, 19, 4, 86, 69, 82, 76, 2, + 37, 7, 69, 82, 84, 73, 67, 65, 76, 2, 205, 40, 2, 32, 76, 4, 46, 76, 165, + 226, 19, 5, 75, 72, 65, 32, 89, 2, 247, 12, 84, 18, 48, 3, 72, 87, 65, + 97, 5, 82, 73, 80, 84, 32, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 222, 35, + 71, 186, 2, 65, 242, 238, 3, 82, 247, 255, 14, 72, 8, 26, 82, 207, 132, + 4, 71, 5, 241, 186, 11, 5, 32, 87, 73, 84, 72, 2, 133, 240, 16, 3, 65, + 82, 80, 14, 48, 7, 68, 69, 87, 65, 89, 83, 32, 199, 1, 71, 12, 110, 79, + 56, 4, 84, 85, 82, 78, 156, 205, 10, 11, 68, 73, 65, 69, 82, 69, 83, 73, + 90, 69, 68, 215, 154, 10, 85, 7, 26, 80, 135, 135, 4, 32, 2, 193, 240, 9, + 2, 69, 78, 2, 221, 138, 13, 2, 69, 68, 2, 237, 237, 16, 4, 77, 79, 73, + 68, 2, 139, 129, 10, 32, 147, 1, 188, 1, 6, 32, 87, 73, 84, 72, 32, 192, + 3, 2, 69, 83, 70, 72, 130, 2, 79, 102, 82, 54, 85, 174, 131, 4, 67, 202, + 1, 83, 208, 129, 11, 9, 65, 73, 76, 76, 69, 83, 83, 32, 80, 251, 215, 5, + 90, 34, 110, 67, 182, 1, 68, 66, 77, 170, 31, 76, 146, 232, 3, 80, 168, + 5, 4, 72, 79, 79, 75, 50, 82, 235, 165, 16, 83, 10, 58, 69, 22, 73, 62, + 79, 222, 154, 7, 85, 207, 161, 12, 65, 2, 219, 168, 12, 68, 2, 37, 7, 82, + 67, 85, 77, 70, 76, 69, 2, 215, 178, 18, 88, 2, 17, 2, 77, 77, 2, 175, + 178, 18, 65, 8, 32, 2, 73, 65, 135, 191, 16, 79, 4, 154, 21, 71, 215, 6, + 69, 4, 17, 2, 73, 68, 4, 26, 45, 255, 255, 3, 68, 2, 149, 142, 4, 6, 72, + 69, 73, 71, 72, 84, 6, 45, 9, 72, 32, 68, 73, 71, 82, 65, 80, 72, 7, 243, + 133, 4, 32, 8, 64, 12, 32, 87, 73, 84, 72, 32, 83, 84, 82, 73, 75, 69, + 43, 79, 2, 233, 179, 16, 5, 84, 72, 82, 79, 85, 6, 17, 2, 82, 78, 7, 41, + 8, 32, 87, 73, 84, 72, 32, 83, 84, 4, 25, 4, 82, 79, 75, 69, 5, 11, 32, + 2, 133, 215, 3, 6, 84, 72, 82, 79, 85, 71, 8, 26, 78, 203, 133, 4, 80, 6, + 17, 2, 69, 32, 6, 206, 206, 4, 83, 134, 212, 10, 70, 155, 168, 3, 84, 2, + 17, 2, 69, 83, 2, 11, 73, 2, 255, 187, 20, 76, 76, 44, 5, 82, 78, 69, 68, + 32, 143, 222, 20, 77, 74, 162, 1, 68, 22, 72, 66, 73, 54, 79, 142, 1, 82, + 110, 84, 22, 86, 198, 129, 4, 65, 38, 77, 198, 2, 89, 210, 193, 16, 85, + 218, 19, 69, 2, 71, 2, 75, 2, 76, 3, 87, 2, 131, 160, 17, 69, 7, 141, + 242, 3, 11, 32, 87, 73, 84, 72, 32, 70, 73, 83, 72, 72, 5, 11, 78, 2, + 197, 218, 9, 5, 83, 85, 76, 65, 82, 12, 66, 69, 232, 230, 3, 7, 32, 79, + 80, 69, 78, 45, 79, 159, 29, 80, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, + 238, 178, 18, 72, 143, 248, 1, 83, 15, 33, 6, 32, 87, 73, 84, 72, 32, 12, + 198, 247, 3, 77, 174, 7, 80, 130, 5, 76, 154, 159, 14, 84, 183, 97, 72, + 5, 135, 255, 3, 32, 7, 11, 32, 4, 161, 5, 4, 87, 73, 84, 72, 97, 90, 32, + 152, 228, 3, 6, 80, 83, 73, 76, 79, 78, 182, 244, 16, 69, 2, 73, 2, 77, + 3, 79, 82, 48, 3, 66, 65, 82, 49, 5, 87, 73, 84, 72, 32, 5, 245, 17, 8, + 32, 87, 73, 84, 72, 32, 83, 72, 78, 142, 1, 67, 74, 68, 150, 2, 72, 170, + 1, 77, 134, 1, 79, 106, 82, 38, 84, 186, 9, 71, 82, 83, 234, 1, 65, 246, + 173, 3, 73, 62, 66, 143, 66, 76, 6, 152, 203, 4, 9, 73, 82, 67, 85, 77, + 70, 76, 69, 88, 139, 228, 14, 65, 18, 52, 8, 73, 65, 69, 82, 69, 83, 73, + 83, 131, 1, 79, 13, 11, 32, 10, 40, 4, 65, 78, 68, 32, 183, 179, 18, 66, + 8, 50, 67, 226, 13, 71, 186, 2, 65, 239, 199, 3, 77, 2, 183, 173, 19, 65, + 6, 26, 85, 167, 163, 18, 84, 4, 21, 3, 66, 76, 69, 4, 11, 32, 4, 138, 13, + 71, 187, 2, 65, 14, 11, 79, 14, 28, 2, 82, 78, 207, 81, 79, 13, 29, 5, + 32, 65, 78, 68, 32, 10, 66, 72, 226, 11, 71, 186, 2, 65, 138, 242, 7, 68, + 251, 234, 8, 84, 2, 229, 80, 2, 79, 79, 10, 29, 5, 65, 67, 82, 79, 78, + 11, 29, 5, 32, 65, 78, 68, 32, 8, 50, 68, 214, 10, 71, 186, 2, 65, 131, + 221, 16, 84, 2, 171, 10, 73, 6, 29, 5, 71, 79, 78, 69, 75, 7, 11, 32, 4, + 25, 4, 65, 78, 68, 32, 4, 178, 12, 65, 131, 221, 16, 84, 4, 142, 251, 3, + 69, 171, 161, 14, 73, 6, 25, 4, 73, 76, 68, 69, 7, 11, 32, 4, 26, 65, + 151, 174, 18, 66, 2, 17, 2, 78, 68, 2, 11, 32, 2, 139, 11, 65, 29, 84, 6, + 32, 87, 73, 84, 72, 32, 162, 1, 73, 66, 79, 134, 189, 19, 69, 151, 144, + 1, 89, 14, 82, 68, 234, 242, 3, 80, 142, 1, 67, 146, 7, 82, 206, 235, 12, + 84, 231, 145, 2, 72, 4, 26, 73, 203, 251, 7, 79, 2, 17, 2, 65, 71, 2, + 201, 157, 20, 2, 79, 78, 2, 41, 8, 83, 73, 71, 79, 84, 72, 73, 67, 2, + 131, 239, 18, 32, 6, 33, 6, 76, 65, 80, 85, 75, 32, 6, 158, 182, 20, 65, + 2, 79, 3, 85, 19, 33, 6, 32, 87, 73, 84, 72, 32, 16, 202, 4, 67, 18, 68, + 70, 71, 186, 2, 65, 154, 144, 18, 82, 207, 94, 72, 17, 33, 6, 32, 87, 73, + 84, 72, 32, 14, 42, 68, 32, 2, 76, 79, 187, 239, 3, 80, 4, 222, 3, 73, + 247, 198, 19, 79, 8, 60, 11, 78, 71, 32, 76, 69, 70, 84, 32, 76, 69, 71, + 79, 87, 7, 11, 32, 4, 60, 7, 65, 78, 68, 32, 76, 79, 87, 65, 4, 87, 73, + 84, 72, 2, 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 155, 154, 11, 84, 2, + 185, 199, 19, 4, 32, 83, 69, 82, 33, 48, 6, 32, 87, 73, 84, 72, 32, 175, + 155, 16, 79, 28, 110, 67, 18, 68, 70, 71, 22, 72, 42, 76, 22, 83, 234, 1, + 65, 238, 199, 3, 77, 150, 149, 13, 84, 155, 179, 1, 82, 2, 203, 3, 73, 6, + 26, 73, 171, 163, 16, 79, 2, 17, 2, 65, 69, 2, 187, 192, 16, 82, 2, 239, + 216, 18, 82, 4, 17, 2, 79, 79, 4, 239, 136, 5, 75, 2, 187, 205, 19, 79, + 4, 26, 72, 179, 150, 20, 84, 2, 21, 3, 79, 82, 84, 2, 197, 252, 7, 6, 32, + 82, 73, 71, 72, 84, 31, 33, 6, 32, 87, 73, 84, 72, 32, 28, 98, 65, 22, + 67, 82, 68, 38, 76, 34, 83, 198, 224, 3, 77, 174, 7, 80, 218, 5, 82, 247, + 255, 14, 72, 2, 223, 190, 13, 67, 6, 42, 73, 150, 251, 6, 85, 207, 161, + 12, 65, 2, 229, 203, 11, 4, 82, 67, 85, 77, 6, 154, 187, 3, 69, 203, 228, + 12, 79, 2, 11, 73, 2, 163, 183, 12, 78, 4, 26, 87, 163, 147, 20, 84, 2, + 215, 154, 17, 65, 18, 90, 70, 142, 195, 9, 73, 172, 9, 6, 76, 79, 78, 71, + 32, 83, 190, 217, 10, 83, 219, 5, 79, 10, 34, 70, 254, 193, 20, 73, 3, + 76, 7, 250, 193, 20, 73, 3, 76, 36, 150, 1, 83, 202, 192, 20, 65, 2, 69, + 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, 82, 2, + 84, 2, 85, 2, 86, 3, 88, 5, 215, 230, 4, 67, 226, 5, 240, 1, 2, 65, 70, + 144, 1, 2, 70, 84, 190, 38, 79, 20, 5, 80, 67, 72, 65, 32, 164, 8, 8, 83, + 83, 45, 84, 72, 65, 78, 32, 184, 250, 12, 5, 85, 75, 79, 84, 72, 218, + 194, 3, 68, 228, 23, 6, 86, 69, 76, 32, 83, 76, 218, 233, 2, 77, 239, 79, + 71, 6, 120, 3, 76, 69, 83, 160, 196, 1, 14, 32, 70, 76, 85, 84, 84, 69, + 82, 73, 78, 71, 32, 73, 78, 189, 218, 11, 3, 89, 32, 71, 2, 227, 243, 14, + 83, 134, 4, 58, 32, 250, 20, 45, 217, 2, 6, 87, 65, 82, 68, 83, 32, 246, + 1, 166, 2, 65, 234, 4, 66, 202, 1, 67, 96, 2, 68, 79, 112, 2, 72, 65, + 166, 3, 76, 46, 77, 38, 79, 102, 82, 146, 3, 83, 82, 84, 222, 1, 87, 158, + 183, 8, 70, 236, 5, 14, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, 82, 73, + 71, 72, 86, 78, 202, 1, 80, 153, 13, 10, 86, 69, 82, 84, 73, 67, 65, 76, + 32, 66, 28, 22, 78, 143, 4, 82, 22, 28, 2, 68, 32, 191, 3, 71, 16, 96, 6, + 76, 79, 87, 69, 82, 32, 32, 6, 82, 73, 71, 72, 84, 32, 89, 6, 85, 80, 80, + 69, 82, 32, 4, 202, 1, 65, 239, 196, 17, 79, 6, 50, 84, 209, 143, 18, 6, + 68, 79, 85, 66, 76, 69, 4, 250, 157, 17, 82, 211, 232, 1, 65, 6, 82, 65, + 53, 16, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, 32, 66, 76, 79, 67, 75, + 2, 237, 155, 17, 8, 78, 68, 32, 82, 73, 71, 72, 84, 5, 245, 147, 19, 17, + 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 6, + 198, 188, 8, 69, 185, 1, 4, 76, 69, 32, 66, 6, 26, 67, 163, 192, 8, 82, + 2, 253, 191, 8, 5, 32, 76, 69, 83, 83, 12, 40, 4, 65, 82, 66, 32, 187, + 192, 8, 76, 8, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 4, 57, 12, 32, 82, + 73, 71, 72, 84, 32, 66, 65, 82, 66, 32, 4, 240, 137, 17, 4, 68, 79, 87, + 78, 1, 2, 85, 80, 14, 254, 191, 8, 85, 166, 26, 79, 242, 235, 2, 69, 141, + 168, 2, 8, 76, 79, 83, 69, 68, 32, 69, 78, 10, 44, 5, 85, 66, 76, 69, 32, + 171, 192, 8, 84, 8, 222, 193, 8, 87, 190, 30, 65, 154, 132, 3, 81, 199, + 199, 4, 80, 38, 36, 3, 76, 70, 32, 163, 198, 8, 78, 36, 132, 2, 6, 67, + 73, 82, 67, 76, 69, 162, 193, 8, 66, 90, 70, 82, 72, 50, 82, 58, 84, 70, + 87, 246, 13, 76, 22, 85, 176, 200, 8, 30, 73, 78, 86, 69, 82, 83, 69, 32, + 77, 69, 68, 73, 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, 68, 32, 82, + 73, 71, 72, 84, 255, 26, 77, 11, 33, 6, 32, 87, 73, 84, 72, 32, 8, 42, + 84, 206, 136, 18, 70, 191, 214, 1, 68, 4, 150, 203, 5, 72, 199, 145, 13, + 87, 4, 232, 162, 1, 2, 85, 71, 163, 161, 7, 79, 2, 217, 155, 19, 4, 85, + 76, 84, 73, 8, 36, 3, 78, 69, 32, 243, 196, 8, 85, 6, 194, 183, 17, 81, + 146, 4, 69, 45, 5, 84, 72, 73, 82, 68, 30, 44, 5, 73, 71, 72, 84, 32, + 199, 197, 8, 65, 28, 152, 1, 5, 65, 82, 82, 79, 87, 132, 1, 12, 68, 79, + 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 30, 87, 214, 242, 8, 79, 162, + 143, 8, 66, 54, 83, 211, 68, 84, 11, 11, 32, 8, 72, 5, 87, 73, 84, 72, + 32, 221, 214, 18, 7, 84, 72, 82, 79, 85, 71, 72, 6, 146, 195, 16, 68, + 234, 183, 3, 86, 79, 83, 7, 245, 234, 8, 2, 32, 87, 4, 202, 253, 8, 65, + 227, 191, 8, 72, 34, 218, 195, 8, 45, 70, 69, 78, 73, 168, 1, 3, 80, 69, + 69, 26, 81, 227, 2, 85, 26, 106, 82, 178, 200, 8, 72, 130, 226, 7, 79, + 212, 134, 1, 8, 87, 79, 32, 84, 72, 73, 82, 68, 151, 198, 1, 65, 6, 40, + 4, 73, 65, 78, 71, 171, 203, 8, 65, 4, 140, 242, 4, 8, 76, 69, 32, 66, + 69, 83, 73, 68, 179, 191, 12, 85, 16, 206, 205, 8, 72, 202, 1, 73, 181, + 200, 10, 2, 82, 73, 62, 128, 1, 9, 80, 79, 73, 78, 84, 73, 78, 71, 32, + 138, 1, 83, 250, 205, 8, 70, 226, 2, 72, 153, 7, 7, 84, 79, 45, 82, 73, + 71, 72, 30, 86, 65, 130, 211, 8, 67, 94, 68, 58, 77, 58, 82, 182, 1, 83, + 74, 84, 211, 172, 8, 69, 6, 190, 211, 8, 78, 158, 175, 8, 84, 159, 2, 73, + 4, 44, 5, 73, 68, 69, 32, 65, 175, 214, 8, 72, 2, 205, 147, 1, 2, 82, 67, + 210, 1, 172, 1, 5, 65, 82, 82, 79, 87, 174, 5, 66, 70, 72, 242, 3, 84, + 178, 2, 87, 162, 214, 8, 68, 210, 1, 70, 170, 7, 76, 26, 79, 34, 80, 50, + 82, 70, 83, 242, 206, 8, 67, 47, 81, 73, 26, 32, 159, 236, 17, 45, 68, + 102, 65, 138, 1, 84, 132, 2, 5, 87, 73, 84, 72, 32, 242, 215, 8, 70, 217, + 165, 10, 4, 79, 86, 69, 82, 12, 44, 5, 66, 79, 86, 69, 32, 223, 217, 8, + 78, 10, 64, 4, 83, 72, 79, 82, 214, 216, 8, 82, 182, 230, 4, 65, 55, 84, + 2, 223, 255, 18, 84, 12, 56, 7, 72, 82, 79, 85, 71, 72, 32, 53, 3, 79, + 32, 66, 6, 226, 188, 13, 83, 238, 206, 4, 76, 211, 149, 2, 88, 6, 32, 2, + 65, 82, 231, 218, 8, 76, 5, 245, 168, 12, 23, 32, 79, 86, 69, 82, 32, 82, + 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 36, 118, + 76, 198, 218, 8, 68, 182, 1, 80, 30, 83, 38, 84, 138, 210, 8, 77, 38, 78, + 122, 69, 150, 153, 1, 72, 171, 165, 1, 86, 4, 162, 174, 17, 65, 171, 247, + 1, 79, 8, 234, 220, 8, 65, 172, 10, 5, 79, 84, 84, 79, 77, 219, 200, 8, + 76, 30, 26, 65, 131, 181, 17, 69, 26, 48, 6, 82, 80, 79, 79, 78, 32, 139, + 215, 19, 78, 24, 80, 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, 32, 249, + 245, 9, 4, 79, 86, 69, 82, 22, 40, 4, 68, 79, 87, 78, 117, 2, 85, 80, 10, + 26, 32, 243, 186, 17, 87, 8, 234, 223, 8, 66, 188, 2, 7, 65, 66, 79, 86, + 69, 32, 82, 234, 208, 8, 70, 179, 5, 84, 12, 26, 32, 255, 185, 17, 87, + 10, 60, 6, 65, 66, 79, 86, 69, 32, 222, 177, 17, 70, 179, 5, 84, 6, 42, + 76, 213, 223, 8, 4, 82, 73, 71, 72, 4, 250, 221, 8, 69, 147, 223, 4, 79, + 54, 44, 2, 82, 73, 202, 227, 8, 79, 159, 5, 87, 34, 44, 5, 65, 78, 71, + 76, 69, 255, 231, 8, 80, 30, 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 251, + 191, 17, 32, 28, 52, 5, 65, 82, 82, 79, 87, 202, 184, 17, 68, 39, 80, 25, + 11, 32, 22, 192, 228, 8, 9, 79, 86, 69, 82, 32, 82, 73, 71, 72, 22, 87, + 135, 208, 8, 84, 6, 26, 72, 143, 234, 8, 65, 4, 45, 9, 73, 84, 69, 32, + 65, 82, 82, 79, 87, 5, 173, 189, 17, 2, 32, 87, 5, 243, 253, 18, 80, 148, + 1, 252, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, + 32, 124, 7, 76, 69, 84, 84, 69, 82, 32, 236, 1, 12, 80, 85, 78, 67, 84, + 85, 65, 84, 73, 79, 78, 32, 198, 1, 83, 172, 1, 11, 86, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 251, 154, 18, 68, 18, 66, 75, 22, 78, 130, 148, + 20, 76, 2, 77, 2, 80, 2, 82, 3, 84, 5, 155, 195, 19, 65, 5, 189, 212, 12, + 4, 89, 73, 78, 45, 78, 194, 1, 75, 2, 80, 254, 153, 7, 68, 130, 236, 10, + 66, 2, 70, 2, 71, 2, 72, 2, 77, 138, 15, 78, 170, 181, 1, 84, 46, 67, 2, + 83, 138, 69, 74, 2, 76, 2, 82, 2, 86, 2, 87, 2, 89, 187, 2, 65, 6, 222, + 143, 20, 72, 2, 76, 187, 2, 65, 10, 82, 84, 40, 14, 78, 89, 69, 84, 32, + 84, 72, 89, 79, 79, 77, 32, 84, 65, 43, 67, 6, 38, 65, 21, 5, 83, 72, 79, + 79, 75, 2, 247, 253, 6, 45, 5, 17, 2, 32, 67, 2, 237, 150, 15, 3, 69, 82, + 45, 8, 92, 4, 73, 71, 78, 32, 37, 15, 85, 66, 74, 79, 73, 78, 69, 68, 32, + 76, 69, 84, 84, 69, 82, 4, 202, 242, 15, 78, 231, 208, 2, 82, 4, 11, 32, + 4, 226, 140, 20, 82, 3, 89, 14, 130, 184, 16, 85, 206, 141, 1, 79, 250, + 198, 2, 65, 186, 2, 69, 3, 73, 52, 138, 1, 65, 128, 4, 11, 69, 81, 85, + 65, 76, 32, 84, 79, 32, 79, 82, 200, 1, 2, 66, 85, 42, 67, 186, 1, 79, + 210, 2, 87, 207, 252, 18, 83, 16, 40, 5, 66, 79, 86, 69, 32, 171, 4, 78, + 12, 152, 1, 7, 71, 82, 69, 65, 84, 69, 82, 110, 83, 176, 1, 19, 68, 79, + 85, 66, 76, 69, 45, 76, 73, 78, 69, 32, 69, 81, 85, 65, 76, 32, 65, 167, + 228, 15, 76, 2, 181, 5, 23, 45, 84, 72, 65, 78, 32, 65, 66, 79, 86, 69, + 32, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 6, 152, 1, 7, 73, 77, 73, + 76, 65, 82, 32, 93, 26, 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, + 32, 65, 66, 79, 86, 69, 32, 71, 82, 69, 65, 84, 69, 82, 4, 18, 65, 59, + 79, 2, 25, 4, 66, 79, 86, 69, 2, 181, 243, 17, 2, 32, 71, 2, 227, 2, 82, + 2, 145, 2, 6, 45, 84, 72, 65, 78, 32, 4, 17, 2, 68, 32, 4, 220, 3, 5, 78, + 79, 84, 32, 65, 221, 162, 13, 11, 83, 73, 78, 71, 76, 69, 45, 76, 73, 78, + 69, 4, 217, 132, 13, 5, 84, 32, 78, 79, 84, 4, 65, 14, 76, 79, 83, 69, + 68, 32, 66, 89, 32, 67, 85, 82, 86, 69, 5, 11, 32, 2, 61, 13, 65, 66, 79, + 86, 69, 32, 83, 76, 65, 78, 84, 69, 68, 2, 17, 2, 32, 69, 2, 151, 245, + 14, 81, 18, 40, 2, 82, 32, 221, 230, 11, 2, 86, 69, 16, 114, 65, 48, 16, + 83, 76, 65, 78, 84, 69, 68, 32, 69, 81, 85, 65, 76, 32, 84, 79, 194, 129, + 13, 69, 131, 237, 4, 71, 2, 237, 190, 9, 7, 80, 80, 82, 79, 88, 73, 77, + 9, 49, 10, 32, 87, 73, 84, 72, 32, 68, 79, 84, 32, 6, 26, 65, 243, 209, + 11, 73, 4, 25, 4, 66, 79, 86, 69, 5, 179, 176, 18, 32, 6, 25, 4, 73, 84, + 72, 32, 6, 66, 67, 40, 8, 81, 85, 69, 83, 84, 73, 79, 78, 247, 177, 19, + 68, 2, 201, 208, 11, 5, 73, 82, 67, 76, 69, 2, 17, 2, 32, 77, 2, 17, 2, + 65, 82, 2, 255, 132, 19, 75, 136, 11, 190, 1, 71, 186, 5, 77, 166, 7, 78, + 156, 65, 2, 80, 83, 20, 3, 83, 85, 32, 142, 221, 15, 82, 136, 35, 11, 86, + 82, 69, 32, 84, 79, 85, 82, 78, 79, 73, 214, 59, 79, 190, 221, 1, 90, + 191, 84, 66, 44, 26, 65, 57, 2, 72, 84, 2, 181, 174, 9, 9, 84, 85, 82, + 69, 32, 79, 80, 69, 78, 42, 38, 32, 137, 4, 4, 78, 73, 78, 71, 36, 146, + 1, 69, 38, 70, 126, 82, 40, 3, 76, 69, 70, 86, 83, 42, 84, 236, 204, 7, + 3, 66, 76, 85, 238, 170, 9, 87, 254, 41, 86, 226, 78, 71, 219, 69, 67, 2, + 141, 230, 17, 4, 73, 71, 72, 84, 8, 94, 73, 153, 180, 18, 17, 79, 85, 82, + 32, 80, 79, 73, 78, 84, 69, 68, 32, 66, 76, 65, 67, 75, 4, 161, 226, 17, + 2, 86, 69, 4, 36, 3, 73, 71, 72, 135, 157, 18, 65, 2, 245, 241, 1, 16, + 84, 32, 84, 79, 82, 84, 79, 73, 83, 69, 32, 83, 72, 69, 76, 76, 6, 194, + 129, 17, 72, 254, 97, 65, 43, 73, 4, 220, 174, 16, 2, 87, 69, 21, 3, 72, + 82, 69, 7, 29, 5, 32, 77, 79, 79, 68, 5, 219, 155, 8, 32, 138, 1, 80, 3, + 66, 85, 32, 201, 163, 8, 11, 73, 84, 69, 68, 32, 76, 73, 65, 66, 73, 76, + 136, 1, 128, 1, 7, 76, 69, 84, 84, 69, 82, 32, 222, 1, 83, 200, 2, 5, 86, + 79, 87, 69, 76, 154, 217, 12, 69, 246, 198, 4, 81, 223, 96, 68, 60, 170, + 1, 71, 198, 145, 6, 84, 234, 231, 8, 89, 186, 136, 1, 78, 246, 173, 3, + 83, 82, 66, 2, 67, 2, 68, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, + 2, 82, 3, 87, 6, 202, 171, 18, 89, 230, 201, 1, 72, 187, 2, 65, 32, 108, + 4, 73, 71, 78, 32, 128, 1, 12, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, + 82, 32, 245, 255, 6, 2, 85, 66, 8, 72, 3, 75, 69, 77, 0, 3, 77, 85, 75, + 32, 2, 83, 65, 231, 170, 17, 76, 2, 153, 175, 17, 3, 80, 72, 82, 2, 131, + 226, 19, 45, 18, 194, 255, 15, 78, 142, 177, 3, 65, 194, 66, 75, 2, 76, + 2, 77, 2, 80, 2, 82, 3, 84, 20, 84, 6, 32, 83, 73, 71, 78, 32, 149, 90, + 10, 45, 67, 65, 82, 82, 73, 69, 82, 32, 76, 18, 230, 216, 5, 65, 130, + 210, 11, 79, 146, 137, 2, 69, 162, 64, 73, 3, 85, 226, 8, 22, 69, 187, + 64, 75, 222, 8, 34, 32, 177, 3, 3, 65, 82, 32, 14, 120, 12, 73, 78, 84, + 69, 71, 82, 65, 84, 73, 79, 78, 32, 150, 167, 13, 70, 206, 153, 3, 83, + 237, 147, 1, 4, 84, 65, 66, 85, 6, 108, 5, 87, 73, 84, 72, 32, 157, 1, + 17, 78, 79, 84, 32, 73, 78, 67, 76, 85, 68, 73, 78, 71, 32, 84, 72, 69, + 4, 76, 7, 82, 69, 67, 84, 65, 78, 71, 1, 8, 83, 69, 77, 73, 67, 73, 82, + 67, 2, 73, 16, 85, 76, 65, 82, 32, 80, 65, 84, 72, 32, 65, 82, 79, 85, + 78, 68, 2, 17, 2, 32, 80, 2, 179, 201, 18, 79, 208, 8, 60, 8, 65, 32, 83, + 73, 71, 78, 32, 65, 205, 24, 2, 66, 32, 170, 5, 122, 49, 86, 51, 202, 2, + 52, 214, 1, 53, 158, 4, 54, 142, 3, 55, 132, 4, 2, 56, 48, 78, 66, 185, + 159, 18, 3, 48, 50, 56, 6, 212, 198, 12, 5, 48, 48, 45, 49, 48, 200, 233, + 5, 2, 50, 48, 233, 19, 2, 51, 49, 150, 1, 78, 48, 90, 49, 130, 1, 55, + 250, 172, 14, 50, 2, 51, 2, 52, 2, 53, 3, 54, 22, 178, 1, 57, 190, 235, + 19, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 24, 90, 51, 190, + 235, 19, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 6, + 186, 235, 19, 65, 2, 66, 3, 67, 4, 150, 235, 19, 48, 3, 49, 38, 18, 48, + 91, 49, 20, 162, 1, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 2, 56, 3, 57, 18, 74, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 3, 56, 2, 237, 183, 6, 2, 45, 86, 160, 1, 102, 48, 78, 49, 62, 51, + 86, 52, 70, 53, 86, 54, 190, 11, 50, 194, 171, 2, 57, 174, 240, 11, 55, + 3, 56, 16, 186, 232, 19, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, + 57, 12, 238, 231, 19, 48, 2, 49, 2, 50, 2, 51, 2, 53, 3, 54, 18, 178, + 231, 19, 48, 2, 49, 2, 50, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 14, + 222, 230, 19, 48, 2, 49, 2, 50, 2, 53, 2, 55, 2, 56, 3, 57, 18, 154, 230, + 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 57, 12, 198, + 229, 19, 51, 2, 52, 2, 53, 2, 54, 2, 56, 3, 57, 104, 70, 48, 78, 50, 86, + 51, 38, 52, 78, 54, 150, 162, 14, 53, 131, 2, 49, 16, 194, 228, 19, 48, + 2, 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 56, 3, 57, 18, 246, 227, 19, 48, 2, + 49, 2, 50, 2, 51, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 6, 162, 227, 19, 52, + 2, 55, 3, 56, 16, 254, 226, 19, 48, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 56, 3, 57, 10, 178, 226, 19, 48, 2, 49, 2, 50, 2, 51, 3, 52, 44, 86, 48, + 134, 2, 49, 196, 129, 2, 2, 51, 50, 153, 204, 17, 6, 50, 54, 32, 69, 89, + 89, 26, 110, 57, 154, 225, 8, 55, 182, 6, 50, 62, 54, 254, 63, 52, 146, + 155, 3, 51, 114, 56, 186, 211, 2, 49, 155, 122, 53, 10, 26, 45, 207, 212, + 18, 32, 8, 96, 2, 50, 32, 156, 180, 12, 3, 54, 32, 76, 184, 2, 3, 52, 32, + 76, 137, 174, 3, 3, 51, 32, 76, 2, 247, 183, 12, 76, 14, 114, 51, 32, 3, + 52, 32, 65, 0, 2, 53, 32, 134, 65, 49, 174, 223, 3, 50, 198, 200, 4, 48, + 249, 249, 6, 2, 55, 32, 2, 11, 32, 2, 147, 153, 12, 79, 2, 151, 160, 18, + 66, 16, 250, 221, 19, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, + 55, 162, 1, 22, 48, 155, 5, 49, 140, 1, 82, 49, 54, 50, 118, 51, 62, 52, + 78, 53, 86, 54, 62, 55, 70, 56, 155, 152, 14, 48, 10, 186, 220, 19, 48, + 2, 49, 2, 51, 2, 54, 3, 55, 28, 86, 49, 2, 50, 138, 82, 51, 170, 137, 19, + 48, 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 7, 174, 219, 19, 70, 3, 77, 12, + 146, 219, 19, 48, 2, 49, 2, 52, 2, 55, 2, 56, 3, 57, 16, 214, 218, 19, + 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 138, 218, 19, + 48, 2, 49, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 12, 182, 217, + 19, 48, 2, 49, 2, 53, 2, 54, 2, 55, 3, 57, 14, 250, 216, 19, 48, 2, 51, + 2, 52, 2, 54, 2, 55, 2, 56, 3, 57, 12, 182, 216, 19, 48, 2, 49, 2, 50, 2, + 53, 2, 54, 3, 55, 22, 82, 50, 36, 2, 51, 49, 30, 56, 186, 171, 12, 49, + 206, 2, 54, 146, 1, 55, 3, 57, 6, 166, 215, 19, 48, 2, 50, 3, 51, 4, 130, + 215, 19, 65, 3, 66, 4, 230, 214, 19, 48, 3, 56, 166, 3, 116, 9, 73, 68, + 69, 79, 71, 82, 65, 77, 32, 220, 17, 10, 77, 79, 78, 79, 71, 82, 65, 77, + 32, 66, 249, 1, 2, 83, 89, 234, 1, 54, 66, 253, 15, 8, 86, 69, 83, 83, + 69, 76, 32, 66, 176, 1, 22, 49, 131, 11, 50, 126, 86, 48, 210, 3, 50, + 166, 1, 51, 86, 52, 122, 53, 114, 54, 146, 1, 55, 118, 56, 71, 57, 28, + 114, 53, 98, 54, 58, 55, 78, 56, 62, 57, 156, 139, 7, 3, 50, 32, 87, 254, + 88, 48, 137, 175, 11, 4, 52, 32, 68, 69, 6, 132, 138, 7, 2, 32, 69, 220, + 166, 11, 3, 70, 32, 77, 133, 82, 7, 77, 32, 83, 84, 65, 76, 76, 4, 168, + 245, 1, 3, 70, 32, 69, 133, 160, 16, 2, 77, 32, 4, 36, 3, 70, 32, 83, 1, + 2, 77, 32, 2, 129, 236, 11, 4, 72, 69, 45, 71, 4, 240, 159, 9, 3, 77, 32, + 66, 173, 149, 9, 3, 70, 32, 83, 4, 224, 198, 17, 4, 77, 32, 66, 85, 129, + 110, 3, 70, 32, 67, 10, 248, 152, 9, 4, 51, 32, 83, 80, 240, 10, 5, 49, + 32, 66, 65, 82, 148, 240, 4, 4, 50, 32, 79, 76, 20, 6, 53, 32, 67, 89, + 80, 69, 129, 179, 4, 4, 48, 32, 87, 72, 6, 54, 49, 164, 239, 17, 3, 48, + 32, 79, 211, 223, 1, 50, 2, 181, 179, 18, 2, 32, 87, 10, 224, 5, 3, 53, + 32, 87, 252, 150, 9, 6, 48, 32, 66, 82, 79, 78, 172, 2, 4, 49, 32, 71, + 79, 206, 175, 10, 50, 3, 54, 16, 82, 57, 134, 238, 2, 49, 138, 223, 16, + 48, 2, 50, 2, 51, 2, 52, 2, 55, 3, 56, 2, 157, 102, 3, 32, 67, 76, 20, + 160, 177, 14, 5, 51, 32, 65, 82, 77, 152, 216, 4, 5, 50, 32, 71, 65, 82, + 182, 67, 48, 2, 49, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 130, + 130, 14, 54, 216, 196, 3, 4, 51, 32, 77, 79, 134, 133, 2, 48, 2, 49, 2, + 50, 2, 52, 2, 55, 2, 56, 3, 57, 14, 234, 202, 19, 48, 2, 49, 2, 50, 2, + 51, 2, 52, 2, 53, 3, 57, 4, 204, 169, 2, 3, 49, 32, 72, 219, 160, 17, 48, + 50, 42, 50, 110, 51, 130, 1, 52, 227, 1, 53, 4, 84, 8, 48, 32, 70, 79, + 79, 84, 83, 84, 245, 215, 6, 7, 53, 32, 66, 65, 84, 72, 84, 2, 231, 179, + 18, 79, 12, 104, 4, 51, 32, 83, 87, 132, 143, 14, 4, 48, 32, 83, 80, 154, + 156, 4, 49, 218, 156, 1, 50, 2, 52, 3, 54, 2, 131, 175, 18, 79, 16, 168, + 1, 9, 48, 32, 87, 72, 69, 69, 76, 69, 68, 2, 49, 34, 51, 180, 180, 17, + 12, 50, 32, 67, 72, 65, 82, 73, 79, 84, 32, 70, 82, 226, 145, 2, 53, 2, + 54, 2, 56, 3, 57, 2, 145, 186, 18, 3, 32, 67, 72, 2, 195, 196, 6, 32, 18, + 224, 190, 18, 3, 52, 32, 68, 158, 135, 1, 49, 2, 50, 2, 51, 2, 53, 2, 54, + 2, 55, 2, 56, 3, 57, 58, 50, 50, 160, 155, 12, 2, 49, 53, 1, 2, 51, 48, + 54, 50, 50, 206, 157, 12, 53, 198, 232, 1, 48, 3, 49, 12, 174, 196, 19, 49, 2, 50, 2, 54, 2, 55, 2, 56, 3, 57, 12, 46, 49, 153, 12, 6, 50, 52, 55, 32, 68, 73, 10, 50, 50, 70, 51, 181, 11, 5, 53, 54, 32, 84, 85, 4, - 208, 174, 3, 3, 55, 32, 75, 189, 220, 15, 5, 56, 32, 75, 65, 78, 4, 56, - 3, 53, 32, 77, 201, 239, 15, 5, 51, 32, 65, 82, 69, 2, 207, 203, 12, 69, + 148, 180, 3, 3, 55, 32, 75, 237, 239, 15, 5, 56, 32, 75, 65, 78, 4, 56, + 3, 53, 32, 77, 233, 132, 16, 5, 51, 32, 65, 82, 69, 2, 195, 214, 12, 69, 176, 1, 84, 9, 76, 76, 65, 66, 76, 69, 32, 66, 48, 229, 12, 7, 77, 66, 79, 76, 32, 66, 48, 148, 1, 114, 48, 142, 1, 49, 162, 1, 50, 230, 1, 51, 174, 1, 52, 162, 1, 53, 154, 1, 54, 186, 1, 55, 198, 1, 56, 87, 57, 18, - 118, 54, 150, 198, 1, 55, 150, 8, 52, 190, 14, 57, 194, 246, 10, 53, 250, - 146, 2, 56, 198, 10, 49, 254, 2, 50, 207, 8, 51, 2, 183, 234, 17, 32, 16, - 122, 54, 18, 55, 190, 207, 1, 49, 134, 2, 50, 166, 26, 52, 248, 186, 1, - 2, 53, 32, 250, 242, 5, 48, 221, 243, 8, 2, 51, 32, 2, 183, 90, 32, 2, - 143, 133, 11, 32, 18, 166, 1, 51, 22, 54, 20, 3, 57, 32, 80, 224, 6, 2, - 53, 32, 220, 174, 3, 2, 48, 32, 196, 205, 14, 2, 55, 32, 140, 7, 2, 52, - 32, 158, 89, 56, 185, 44, 3, 49, 32, 81, 2, 215, 164, 14, 32, 2, 171, - 234, 14, 32, 2, 139, 138, 12, 85, 16, 134, 1, 48, 20, 2, 51, 32, 154, - 195, 1, 55, 254, 34, 54, 158, 241, 4, 57, 234, 163, 2, 56, 250, 168, 4, - 49, 133, 223, 5, 3, 50, 32, 81, 2, 151, 166, 14, 32, 2, 135, 1, 82, 16, - 116, 2, 51, 32, 22, 53, 142, 191, 1, 48, 222, 1, 49, 134, 6, 50, 218, 10, - 52, 178, 5, 54, 241, 216, 12, 3, 56, 32, 78, 2, 247, 170, 15, 65, 2, 223, - 144, 17, 32, 18, 130, 1, 51, 190, 191, 1, 49, 150, 1, 56, 42, 57, 102, - 55, 214, 5, 48, 194, 141, 2, 52, 212, 161, 15, 2, 50, 32, 157, 4, 2, 53, - 32, 2, 135, 194, 12, 32, 16, 132, 1, 2, 50, 32, 20, 2, 56, 32, 204, 1, 3, - 54, 32, 84, 178, 186, 1, 55, 226, 3, 57, 146, 2, 53, 206, 247, 6, 49, - 151, 154, 10, 48, 2, 191, 215, 17, 80, 2, 245, 132, 12, 2, 82, 79, 18, - 172, 1, 3, 54, 32, 82, 210, 186, 1, 55, 158, 8, 48, 170, 16, 53, 12, 3, - 49, 32, 68, 132, 169, 7, 2, 52, 32, 250, 249, 3, 50, 232, 143, 6, 3, 56, - 32, 81, 241, 2, 2, 51, 32, 2, 171, 131, 12, 65, 8, 206, 186, 1, 49, 188, - 24, 3, 55, 32, 84, 192, 249, 6, 2, 53, 32, 183, 146, 6, 48, 4, 154, 228, - 12, 49, 21, 3, 48, 32, 68, 28, 90, 52, 30, 54, 30, 56, 174, 253, 11, 53, - 158, 2, 49, 30, 51, 166, 1, 50, 235, 163, 3, 55, 4, 234, 154, 19, 55, 3, - 57, 4, 206, 154, 19, 51, 3, 52, 8, 178, 154, 19, 50, 2, 51, 2, 54, 3, 57, - 4, 136, 202, 10, 9, 69, 68, 32, 80, 65, 80, 69, 82, 67, 227, 188, 7, 32, - 5, 195, 233, 17, 84, 98, 96, 7, 76, 69, 84, 84, 69, 82, 32, 169, 234, 6, - 11, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 94, 238, 1, 84, 230, 240, - 2, 68, 178, 159, 10, 85, 150, 149, 2, 78, 138, 30, 69, 234, 139, 3, 67, + 118, 54, 182, 203, 1, 55, 130, 8, 52, 190, 14, 57, 170, 252, 10, 53, 158, + 152, 2, 56, 198, 10, 49, 254, 2, 50, 207, 8, 51, 2, 151, 129, 18, 32, 16, + 122, 54, 18, 55, 202, 212, 1, 49, 134, 2, 50, 166, 26, 52, 176, 187, 1, + 2, 53, 32, 158, 247, 5, 48, 229, 128, 9, 2, 51, 32, 2, 171, 95, 32, 2, + 235, 143, 11, 32, 18, 166, 1, 51, 22, 54, 20, 3, 57, 32, 80, 224, 6, 2, + 53, 32, 168, 180, 3, 2, 48, 32, 232, 222, 14, 2, 55, 32, 140, 7, 2, 52, + 32, 162, 91, 56, 185, 44, 3, 49, 32, 81, 2, 247, 180, 14, 32, 2, 195, + 250, 14, 32, 2, 235, 148, 12, 85, 16, 134, 1, 48, 20, 2, 51, 32, 186, + 200, 1, 55, 234, 34, 54, 218, 241, 4, 57, 210, 165, 2, 56, 246, 171, 4, + 49, 205, 237, 5, 3, 50, 32, 81, 2, 183, 182, 14, 32, 2, 135, 1, 82, 16, + 116, 2, 51, 32, 22, 53, 174, 196, 1, 48, 222, 1, 49, 242, 5, 50, 218, 10, + 52, 178, 5, 54, 133, 228, 12, 3, 56, 32, 78, 2, 151, 191, 15, 65, 2, 235, + 166, 17, 32, 18, 130, 1, 51, 222, 196, 1, 49, 150, 1, 56, 42, 57, 102, + 55, 194, 5, 48, 250, 141, 2, 52, 132, 181, 15, 2, 50, 32, 157, 4, 2, 53, + 32, 2, 251, 204, 12, 32, 16, 132, 1, 2, 50, 32, 20, 2, 56, 32, 204, 1, 3, + 54, 32, 84, 210, 191, 1, 55, 226, 3, 57, 146, 2, 53, 198, 248, 6, 49, + 243, 172, 10, 48, 2, 151, 238, 17, 80, 2, 213, 143, 12, 2, 82, 79, 18, + 172, 1, 3, 54, 32, 82, 242, 191, 1, 55, 138, 8, 48, 170, 16, 53, 12, 3, + 49, 32, 68, 168, 171, 7, 2, 52, 32, 222, 252, 3, 50, 200, 158, 6, 3, 56, + 32, 81, 241, 2, 2, 51, 32, 2, 139, 142, 12, 65, 8, 238, 191, 1, 49, 168, + 24, 3, 55, 32, 84, 196, 250, 6, 2, 53, 32, 191, 156, 6, 48, 4, 150, 160, + 17, 49, 21, 3, 48, 32, 68, 28, 90, 52, 30, 54, 30, 56, 142, 136, 12, 53, + 158, 2, 49, 30, 51, 166, 1, 50, 171, 173, 3, 55, 4, 222, 179, 19, 55, 3, + 57, 4, 194, 179, 19, 51, 3, 52, 8, 166, 179, 19, 50, 2, 51, 2, 54, 3, 57, + 4, 228, 212, 10, 9, 69, 68, 32, 80, 65, 80, 69, 82, 67, 223, 200, 7, 32, + 5, 163, 128, 18, 84, 98, 96, 7, 76, 69, 84, 84, 69, 82, 32, 129, 240, 6, + 11, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 94, 238, 1, 84, 150, 246, + 2, 68, 182, 164, 10, 85, 130, 159, 2, 78, 138, 31, 69, 190, 143, 3, 67, 2, 71, 2, 72, 2, 75, 2, 80, 2, 83, 2, 89, 2, 90, 162, 7, 65, 2, 79, 234, 61, 66, 2, 70, 2, 74, 2, 76, 2, 77, 2, 87, 2, 88, 187, 2, 73, 20, 64, 4, - 79, 78, 69, 32, 214, 206, 18, 83, 138, 69, 72, 187, 2, 65, 12, 48, 4, 77, - 89, 65, 32, 129, 192, 1, 2, 78, 65, 10, 130, 183, 12, 74, 234, 191, 6, - 66, 158, 11, 84, 254, 16, 67, 39, 78, 196, 2, 232, 1, 2, 67, 75, 132, 1, - 3, 71, 73, 67, 176, 6, 3, 78, 71, 32, 222, 4, 84, 116, 3, 86, 69, 32, 70, - 87, 172, 12, 5, 90, 69, 78, 71, 69, 134, 231, 7, 66, 148, 138, 10, 8, 85, + 79, 78, 69, 32, 202, 231, 18, 83, 138, 69, 72, 187, 2, 65, 12, 48, 4, 77, + 89, 65, 32, 141, 197, 1, 2, 78, 65, 10, 246, 193, 12, 74, 234, 205, 6, + 66, 158, 11, 84, 254, 16, 67, 39, 78, 210, 2, 232, 1, 2, 67, 75, 132, 1, + 3, 71, 73, 67, 176, 6, 3, 78, 71, 32, 206, 9, 84, 116, 3, 86, 69, 32, 70, + 87, 172, 12, 5, 90, 69, 78, 71, 69, 174, 232, 7, 66, 240, 156, 10, 8, 85, 68, 76, 89, 32, 67, 82, 89, 141, 15, 4, 76, 76, 73, 80, 9, 96, 10, 73, - 78, 71, 45, 83, 72, 73, 70, 84, 32, 153, 20, 9, 32, 87, 73, 84, 72, 32, - 73, 78, 75, 4, 162, 163, 17, 90, 251, 85, 79, 40, 56, 6, 32, 71, 65, 84, + 78, 71, 45, 83, 72, 73, 70, 84, 32, 137, 25, 9, 32, 87, 73, 84, 72, 32, + 73, 78, 75, 4, 174, 185, 17, 90, 223, 86, 79, 40, 56, 6, 32, 71, 65, 84, 69, 32, 137, 2, 3, 65, 76, 32, 12, 104, 6, 66, 85, 70, 70, 69, 82, 84, 9, - 73, 78, 86, 69, 82, 84, 69, 68, 32, 182, 130, 18, 65, 159, 85, 79, 5, + 73, 78, 86, 69, 82, 84, 69, 68, 32, 142, 153, 18, 65, 187, 87, 79, 5, 133, 1, 17, 32, 87, 73, 84, 72, 32, 73, 78, 86, 69, 82, 84, 69, 68, 32, - 73, 78, 4, 48, 3, 79, 85, 84, 129, 154, 6, 3, 73, 78, 80, 2, 167, 150, + 73, 78, 4, 48, 3, 79, 85, 84, 205, 159, 6, 3, 73, 78, 80, 2, 155, 175, 18, 80, 28, 36, 3, 65, 78, 68, 85, 2, 79, 82, 15, 33, 6, 32, 87, 73, 84, - 72, 32, 12, 226, 1, 68, 94, 72, 58, 77, 151, 161, 15, 85, 15, 11, 32, 12, + 72, 32, 12, 226, 1, 68, 94, 72, 58, 77, 147, 182, 15, 85, 15, 11, 32, 12, 88, 13, 79, 86, 69, 82, 76, 65, 80, 80, 73, 78, 71, 32, 76, 49, 5, 87, - 73, 84, 72, 32, 2, 249, 255, 17, 7, 79, 71, 73, 67, 65, 76, 32, 10, 26, - 68, 94, 72, 59, 77, 6, 11, 79, 6, 44, 5, 85, 66, 76, 69, 32, 247, 143, - 18, 84, 4, 234, 161, 15, 85, 235, 60, 79, 2, 177, 207, 15, 9, 79, 82, 73, - 90, 79, 78, 84, 65, 76, 2, 225, 218, 7, 5, 73, 68, 68, 76, 69, 34, 66, - 68, 228, 1, 4, 76, 69, 70, 84, 109, 5, 82, 73, 71, 72, 84, 6, 164, 1, 30, - 65, 83, 72, 32, 70, 82, 79, 77, 32, 76, 69, 70, 84, 32, 77, 69, 77, 66, - 69, 82, 32, 79, 70, 32, 68, 79, 85, 66, 76, 69, 166, 144, 16, 73, 219, - 208, 1, 82, 2, 17, 2, 32, 86, 2, 181, 128, 18, 5, 69, 82, 84, 73, 67, 16, - 18, 32, 119, 87, 6, 48, 6, 82, 73, 71, 72, 84, 32, 139, 238, 15, 84, 4, - 246, 227, 15, 68, 215, 138, 2, 65, 12, 26, 87, 139, 244, 8, 32, 10, 29, - 5, 65, 82, 68, 83, 32, 10, 82, 65, 0, 8, 68, 79, 85, 66, 76, 69, 32, 65, - 221, 227, 7, 4, 83, 81, 85, 73, 4, 25, 4, 82, 82, 79, 87, 5, 137, 162, - 16, 2, 32, 70, 6, 92, 5, 73, 79, 78, 32, 66, 144, 237, 17, 9, 32, 79, 70, - 32, 70, 79, 82, 84, 85, 211, 91, 85, 2, 211, 235, 9, 79, 4, 38, 76, 157, - 249, 13, 3, 72, 79, 84, 2, 133, 206, 17, 2, 69, 84, 222, 1, 34, 32, 229, - 1, 3, 69, 82, 32, 14, 138, 1, 66, 196, 195, 10, 11, 68, 79, 85, 66, 76, - 69, 32, 80, 82, 73, 77, 154, 173, 6, 65, 204, 111, 6, 75, 65, 86, 89, 75, - 65, 211, 10, 76, 4, 38, 82, 249, 248, 4, 3, 65, 84, 84, 2, 209, 240, 17, - 7, 73, 71, 72, 84, 78, 69, 83, 208, 1, 158, 1, 72, 140, 2, 5, 76, 69, 70, - 84, 32, 236, 2, 6, 82, 73, 71, 72, 84, 32, 234, 230, 15, 66, 74, 67, 74, - 70, 234, 10, 77, 150, 2, 79, 166, 18, 83, 67, 84, 20, 84, 4, 65, 76, 70, - 32, 241, 184, 7, 11, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 18, 212, - 164, 7, 9, 65, 78, 68, 32, 85, 80, 80, 69, 82, 32, 7, 73, 78, 86, 69, 82, - 83, 69, 194, 201, 8, 72, 230, 1, 86, 170, 26, 77, 130, 3, 76, 22, 82, - 202, 5, 66, 195, 156, 1, 67, 72, 186, 1, 66, 60, 8, 70, 79, 85, 78, 84, - 65, 73, 78, 22, 80, 56, 12, 83, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, - 82, 154, 1, 81, 246, 2, 84, 248, 191, 11, 3, 67, 82, 65, 139, 177, 4, 79, - 24, 56, 8, 65, 76, 76, 80, 79, 73, 78, 84, 255, 246, 15, 76, 2, 179, 240, - 9, 32, 4, 212, 243, 14, 5, 65, 73, 78, 84, 66, 163, 140, 1, 69, 2, 221, - 193, 7, 5, 32, 65, 78, 84, 73, 74, 110, 81, 166, 2, 83, 82, 84, 238, 176, - 7, 82, 202, 184, 8, 66, 238, 3, 67, 226, 3, 79, 222, 7, 68, 207, 2, 80, - 30, 17, 2, 85, 65, 30, 48, 6, 68, 82, 65, 78, 84, 32, 187, 130, 16, 82, - 28, 74, 70, 60, 2, 78, 69, 50, 83, 226, 253, 15, 67, 226, 1, 77, 143, 1, - 84, 4, 26, 65, 255, 180, 17, 82, 2, 185, 183, 7, 3, 67, 69, 32, 2, 25, 4, - 85, 84, 82, 65, 2, 191, 189, 18, 76, 4, 254, 218, 10, 77, 143, 165, 5, - 84, 6, 148, 190, 7, 11, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 82, 231, - 194, 8, 72, 6, 226, 129, 16, 82, 155, 1, 87, 5, 141, 213, 17, 25, 32, 68, - 73, 86, 73, 68, 69, 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, 78, 84, - 65, 76, 32, 82, 85, 6, 18, 71, 23, 78, 2, 215, 241, 13, 71, 4, 188, 213, - 17, 5, 65, 82, 32, 69, 67, 255, 100, 71, 114, 104, 12, 67, 73, 65, 78, - 32, 76, 69, 84, 84, 69, 82, 32, 228, 1, 5, 68, 73, 65, 78, 32, 131, 237, - 17, 73, 58, 206, 1, 77, 194, 137, 5, 75, 206, 203, 9, 66, 50, 84, 234, - 164, 2, 65, 2, 69, 2, 78, 134, 251, 1, 68, 2, 71, 2, 72, 2, 73, 2, 74, 2, - 76, 2, 80, 2, 81, 2, 82, 2, 83, 2, 85, 2, 87, 2, 88, 3, 90, 5, 167, 245, - 18, 77, 54, 88, 7, 76, 69, 84, 84, 69, 82, 32, 189, 213, 14, 9, 84, 82, - 73, 65, 78, 71, 85, 76, 65, 52, 214, 148, 13, 84, 174, 192, 1, 76, 198, - 152, 2, 83, 238, 11, 65, 2, 69, 2, 78, 134, 251, 1, 66, 2, 67, 2, 68, 2, - 70, 2, 71, 2, 73, 2, 75, 2, 77, 2, 79, 2, 81, 2, 82, 2, 85, 2, 86, 3, 89, - 248, 53, 162, 1, 65, 206, 103, 69, 246, 102, 73, 154, 25, 79, 148, 112, - 3, 82, 79, 32, 234, 3, 85, 192, 70, 7, 89, 65, 78, 77, 65, 82, 32, 218, - 145, 15, 86, 198, 61, 77, 27, 87, 204, 23, 186, 1, 71, 62, 72, 254, 10, - 75, 234, 3, 76, 252, 13, 2, 77, 77, 22, 78, 154, 17, 80, 118, 82, 174, 7, - 83, 190, 7, 84, 196, 36, 3, 89, 65, 78, 140, 129, 1, 3, 88, 73, 77, 247, - 230, 15, 67, 6, 128, 226, 17, 4, 73, 67, 32, 87, 238, 88, 78, 155, 53, - 69, 166, 1, 84, 6, 65, 74, 65, 78, 73, 32, 129, 3, 10, 74, 79, 78, 71, - 32, 84, 73, 76, 69, 32, 78, 38, 76, 162, 2, 83, 207, 129, 13, 65, 72, 88, - 6, 69, 84, 84, 69, 82, 32, 161, 144, 12, 10, 73, 71, 65, 84, 85, 82, 69, - 32, 83, 72, 70, 182, 174, 3, 78, 186, 229, 11, 68, 82, 82, 34, 84, 206, - 145, 3, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, - 2, 83, 2, 86, 186, 2, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 226, 235, 6, 69, - 229, 236, 7, 5, 73, 71, 78, 32, 78, 88, 236, 1, 2, 66, 65, 38, 69, 46, - 70, 46, 78, 42, 79, 46, 80, 22, 83, 118, 84, 194, 1, 87, 112, 5, 71, 82, - 69, 69, 78, 0, 3, 82, 69, 68, 208, 135, 6, 3, 65, 85, 84, 214, 21, 74, - 197, 158, 11, 11, 67, 72, 82, 89, 83, 65, 78, 84, 72, 69, 77, 4, 138, - 158, 1, 77, 223, 200, 17, 67, 8, 228, 2, 4, 73, 71, 72, 84, 195, 1, 65, - 12, 172, 2, 2, 73, 86, 13, 3, 79, 85, 82, 8, 192, 1, 2, 79, 82, 65, 2, - 73, 78, 8, 218, 1, 78, 189, 223, 11, 3, 82, 67, 72, 2, 159, 191, 17, 76, - 18, 88, 2, 79, 85, 76, 4, 69, 86, 69, 78, 0, 2, 73, 88, 198, 151, 12, 85, - 187, 194, 1, 80, 2, 157, 2, 2, 84, 72, 12, 36, 3, 72, 82, 69, 13, 2, 87, - 79, 6, 11, 69, 6, 25, 4, 32, 79, 70, 32, 6, 46, 67, 229, 246, 6, 5, 66, - 65, 77, 66, 79, 4, 224, 139, 6, 2, 73, 82, 217, 217, 10, 5, 72, 65, 82, - 65, 67, 6, 50, 69, 60, 4, 72, 73, 84, 69, 247, 172, 17, 73, 2, 17, 2, 83, - 84, 2, 17, 2, 32, 87, 2, 147, 215, 17, 73, 2, 17, 2, 32, 68, 2, 183, 172, - 17, 82, 52, 52, 5, 65, 83, 65, 82, 32, 233, 251, 14, 2, 69, 77, 50, 162, - 1, 69, 40, 7, 76, 69, 84, 84, 69, 82, 32, 152, 1, 5, 80, 65, 83, 83, 73, - 32, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 173, 216, 18, 3, 65, - 78, 71, 2, 161, 228, 15, 5, 78, 68, 32, 79, 70, 36, 158, 232, 16, 78, - 246, 247, 1, 66, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, - 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 187, 2, 65, 2, 11, 77, 2, 211, 136, - 18, 66, 8, 146, 225, 18, 69, 2, 73, 2, 79, 3, 85, 246, 1, 80, 7, 65, 89, - 65, 76, 65, 77, 32, 176, 11, 2, 69, 32, 225, 1, 3, 84, 69, 83, 236, 1, - 198, 1, 68, 44, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 240, 1, 7, 76, 69, - 84, 84, 69, 82, 32, 164, 5, 7, 78, 85, 77, 66, 69, 82, 32, 36, 5, 83, 73, - 71, 78, 32, 178, 180, 13, 86, 247, 196, 1, 65, 22, 204, 169, 8, 2, 65, - 84, 227, 195, 8, 73, 26, 56, 4, 79, 78, 69, 32, 121, 6, 84, 72, 82, 69, - 69, 32, 18, 82, 84, 146, 135, 5, 83, 230, 249, 7, 70, 62, 79, 130, 216, - 3, 69, 46, 72, 47, 81, 4, 254, 131, 13, 87, 239, 214, 3, 69, 8, 170, 135, - 5, 83, 170, 253, 7, 69, 110, 84, 163, 214, 3, 81, 132, 1, 226, 1, 65, 82, - 67, 158, 1, 68, 78, 84, 82, 86, 218, 130, 13, 78, 146, 251, 1, 76, 38, - 82, 134, 6, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, - 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, 2, 77, 3, 89, 11, 192, - 151, 16, 7, 82, 67, 72, 65, 73, 67, 32, 254, 194, 2, 65, 2, 73, 3, 85, - 22, 26, 72, 215, 217, 18, 65, 20, 44, 5, 73, 76, 76, 85, 32, 167, 217, - 18, 65, 18, 130, 249, 12, 76, 158, 229, 3, 78, 170, 194, 1, 82, 222, 56, - 75, 2, 77, 3, 89, 10, 220, 220, 10, 4, 79, 84, 32, 82, 182, 180, 7, 68, - 138, 69, 72, 187, 2, 65, 10, 38, 84, 170, 213, 18, 72, 187, 2, 65, 6, - 166, 213, 18, 72, 2, 84, 187, 2, 65, 12, 166, 227, 3, 69, 138, 161, 11, - 79, 139, 211, 3, 65, 6, 154, 131, 13, 79, 131, 128, 4, 84, 18, 50, 67, - 114, 86, 206, 254, 14, 65, 167, 146, 3, 80, 6, 54, 79, 120, 5, 73, 82, - 67, 85, 76, 203, 192, 14, 65, 2, 221, 178, 13, 9, 77, 66, 73, 78, 73, 78, - 71, 32, 65, 6, 60, 9, 69, 82, 84, 73, 67, 65, 76, 32, 66, 255, 145, 18, - 73, 2, 209, 254, 14, 2, 65, 82, 8, 80, 12, 87, 73, 84, 72, 32, 83, 84, - 82, 79, 75, 69, 32, 70, 65, 227, 204, 17, 83, 4, 64, 10, 65, 78, 68, 32, - 77, 65, 76, 69, 32, 65, 227, 204, 17, 83, 2, 25, 4, 78, 68, 32, 70, 2, - 11, 69, 2, 11, 77, 2, 231, 209, 7, 65, 2, 211, 145, 17, 69, 2, 219, 208, - 16, 79, 185, 1, 210, 1, 32, 220, 2, 5, 68, 65, 73, 67, 32, 224, 3, 8, 73, - 67, 72, 65, 69, 65, 78, 32, 222, 8, 83, 216, 165, 2, 3, 85, 65, 76, 190, - 217, 10, 65, 152, 168, 1, 8, 84, 69, 76, 80, 73, 69, 67, 69, 203, 251, 3, - 71, 12, 132, 1, 3, 73, 78, 32, 128, 1, 5, 87, 73, 84, 72, 32, 204, 146, - 2, 3, 68, 65, 78, 253, 252, 12, 8, 65, 78, 68, 32, 87, 79, 77, 65, 4, - 144, 135, 10, 19, 66, 85, 83, 73, 78, 69, 83, 83, 32, 83, 85, 73, 84, 32, - 76, 69, 86, 73, 84, 185, 151, 1, 4, 84, 85, 88, 69, 4, 224, 231, 13, 8, - 71, 85, 65, 32, 80, 73, 32, 77, 149, 157, 3, 4, 84, 85, 82, 66, 58, 112, - 4, 65, 70, 70, 82, 28, 7, 76, 69, 84, 84, 69, 82, 32, 244, 173, 14, 3, - 86, 79, 67, 146, 67, 71, 251, 25, 80, 2, 249, 227, 17, 2, 73, 67, 50, 82, - 65, 176, 1, 2, 68, 85, 2, 85, 28, 3, 72, 65, 76, 22, 73, 183, 132, 18, - 75, 38, 154, 1, 75, 198, 235, 12, 84, 146, 192, 1, 83, 242, 207, 3, 73, - 226, 79, 66, 2, 68, 2, 71, 2, 72, 2, 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, - 82, 3, 90, 5, 235, 200, 18, 83, 2, 153, 131, 8, 2, 83, 72, 2, 187, 200, - 18, 81, 4, 222, 202, 18, 78, 3, 84, 102, 216, 1, 7, 76, 69, 84, 84, 69, - 82, 32, 194, 4, 78, 80, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, - 32, 220, 1, 4, 83, 73, 71, 78, 129, 195, 10, 16, 65, 66, 66, 82, 69, 86, - 73, 65, 84, 73, 79, 78, 32, 77, 65, 82, 72, 218, 1, 65, 46, 66, 38, 68, - 30, 71, 30, 74, 2, 90, 30, 75, 30, 81, 38, 83, 50, 84, 54, 88, 234, 162, - 6, 76, 158, 206, 1, 82, 154, 217, 2, 89, 158, 208, 1, 72, 222, 169, 5, - 78, 134, 2, 87, 218, 103, 70, 2, 80, 171, 4, 77, 6, 150, 203, 10, 76, - 122, 65, 171, 251, 6, 89, 4, 254, 237, 12, 72, 211, 214, 3, 69, 4, 202, - 164, 6, 65, 23, 72, 4, 190, 202, 10, 72, 15, 73, 4, 250, 202, 10, 72, 15, - 65, 4, 194, 161, 12, 72, 15, 65, 4, 158, 164, 6, 72, 151, 253, 5, 79, 8, - 194, 199, 10, 83, 154, 3, 65, 251, 250, 6, 72, 6, 154, 163, 6, 72, 206, - 159, 10, 69, 243, 131, 1, 65, 4, 170, 160, 12, 65, 3, 79, 10, 33, 6, 85, - 77, 66, 69, 82, 32, 10, 130, 202, 10, 79, 58, 84, 135, 198, 2, 70, 14, - 80, 2, 68, 79, 116, 3, 76, 73, 78, 134, 167, 5, 70, 186, 202, 11, 84, - 167, 10, 83, 6, 54, 84, 13, 9, 85, 66, 76, 69, 32, 68, 79, 84, 32, 5, 11, - 32, 2, 25, 4, 87, 73, 84, 72, 2, 199, 192, 2, 73, 2, 155, 203, 3, 69, 2, - 159, 150, 17, 32, 2, 11, 32, 2, 249, 170, 18, 2, 83, 72, 4, 92, 17, 32, - 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 76, 73, 71, 72, 84, 147, 214, - 10, 76, 2, 135, 252, 14, 72, 142, 1, 142, 1, 65, 20, 5, 67, 72, 69, 78, - 32, 144, 163, 6, 4, 82, 73, 65, 71, 193, 157, 11, 13, 84, 73, 65, 76, 32, - 65, 82, 84, 83, 32, 85, 78, 73, 2, 171, 161, 5, 67, 136, 1, 152, 1, 7, - 76, 69, 84, 84, 69, 82, 32, 194, 1, 83, 236, 2, 11, 86, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 170, 237, 17, 72, 225, 5, 4, 77, 65, 82, 75, 60, - 254, 3, 84, 146, 148, 2, 68, 202, 171, 14, 78, 238, 178, 1, 67, 2, 75, 2, - 80, 2, 83, 2, 90, 138, 69, 45, 2, 66, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, - 2, 82, 2, 87, 2, 89, 187, 2, 65, 62, 96, 4, 73, 71, 78, 32, 37, 16, 85, - 66, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 32, 4, 158, 167, - 14, 67, 203, 207, 3, 65, 58, 182, 1, 84, 146, 148, 2, 68, 202, 171, 14, - 78, 238, 178, 1, 67, 2, 75, 2, 80, 2, 83, 2, 90, 138, 69, 66, 2, 71, 2, - 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, 187, 2, 65, 8, 194, 242, - 17, 83, 138, 69, 72, 187, 2, 65, 10, 158, 183, 18, 65, 186, 2, 69, 2, 73, - 2, 79, 3, 85, 156, 1, 120, 11, 65, 82, 65, 77, 32, 71, 79, 78, 68, 73, - 32, 232, 5, 3, 67, 85, 76, 64, 5, 75, 32, 87, 79, 82, 183, 240, 17, 85, - 150, 1, 100, 7, 76, 69, 84, 84, 69, 82, 32, 178, 2, 82, 56, 5, 83, 73, - 71, 78, 32, 82, 86, 223, 194, 16, 68, 94, 210, 1, 74, 42, 84, 154, 219, - 14, 65, 38, 68, 214, 6, 85, 206, 201, 1, 73, 42, 76, 214, 192, 1, 75, 38, - 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, - 86, 2, 89, 186, 2, 69, 3, 79, 6, 130, 179, 18, 78, 38, 72, 187, 2, 65, - 10, 246, 237, 17, 84, 138, 69, 72, 2, 82, 187, 2, 65, 4, 32, 2, 65, 45, - 175, 220, 14, 69, 2, 147, 240, 17, 75, 10, 178, 176, 1, 67, 198, 196, 12, - 72, 246, 43, 78, 186, 221, 1, 86, 179, 241, 1, 65, 22, 26, 79, 139, 251, - 15, 73, 20, 45, 9, 87, 69, 76, 32, 83, 73, 71, 78, 32, 20, 74, 86, 154, - 223, 14, 65, 38, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 2, 233, - 201, 7, 6, 79, 67, 65, 76, 73, 67, 2, 145, 249, 17, 11, 73, 78, 69, 32, - 79, 82, 68, 73, 78, 65, 76, 2, 207, 158, 17, 75, 226, 15, 76, 10, 72, 69, - 77, 65, 84, 73, 67, 65, 76, 32, 237, 207, 14, 3, 69, 32, 68, 224, 15, - 252, 1, 5, 66, 79, 76, 68, 32, 168, 6, 14, 68, 79, 85, 66, 76, 69, 45, - 83, 84, 82, 85, 67, 75, 32, 238, 1, 70, 164, 2, 7, 73, 84, 65, 76, 73, - 67, 32, 180, 3, 10, 77, 79, 78, 79, 83, 80, 65, 67, 69, 32, 40, 2, 82, - 73, 36, 3, 76, 69, 70, 167, 1, 83, 160, 5, 176, 1, 8, 67, 65, 80, 73, 84, - 65, 76, 32, 130, 2, 83, 210, 14, 73, 222, 3, 69, 38, 75, 38, 78, 30, 80, - 98, 82, 242, 5, 84, 56, 7, 70, 82, 65, 75, 84, 85, 82, 159, 160, 16, 68, - 104, 190, 4, 68, 174, 15, 84, 186, 4, 65, 22, 66, 2, 90, 42, 69, 98, 71, - 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 70, 85, 142, 197, - 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, 137, 2, 70, 2, 72, - 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 208, 1, 60, 5, 77, 65, 76, 76, 32, - 213, 25, 5, 67, 82, 73, 80, 84, 104, 250, 1, 68, 230, 19, 65, 22, 66, 2, - 90, 42, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, - 82, 22, 83, 34, 84, 38, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, - 206, 201, 1, 88, 222, 137, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 7, - 26, 73, 235, 239, 14, 69, 2, 187, 213, 1, 71, 110, 68, 8, 67, 65, 80, 73, - 84, 65, 76, 32, 162, 23, 83, 223, 159, 16, 68, 38, 154, 168, 18, 65, 2, - 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 79, - 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 89, 96, 52, 7, 82, 65, 75, - 84, 85, 82, 32, 219, 142, 7, 65, 94, 52, 8, 67, 65, 80, 73, 84, 65, 76, - 32, 143, 21, 83, 42, 134, 166, 18, 65, 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, - 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, - 85, 2, 86, 2, 87, 2, 88, 3, 89, 222, 1, 100, 6, 83, 77, 65, 76, 76, 32, - 222, 7, 67, 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 243, 5, 84, 104, - 242, 1, 68, 198, 12, 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, 73, - 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 142, 197, - 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, 137, 2, 74, 2, 81, - 2, 86, 2, 87, 3, 89, 9, 52, 7, 79, 84, 76, 69, 83, 83, 32, 175, 232, 14, - 69, 4, 186, 161, 18, 73, 3, 74, 124, 130, 16, 67, 34, 83, 223, 159, 16, - 68, 12, 32, 2, 71, 72, 131, 137, 7, 83, 10, 17, 2, 84, 32, 10, 112, 6, - 87, 72, 73, 84, 69, 32, 170, 244, 5, 68, 234, 106, 65, 241, 230, 4, 9, - 70, 76, 65, 84, 84, 69, 78, 69, 68, 4, 130, 165, 14, 83, 39, 84, 130, 6, - 84, 10, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 133, 14, 6, 67, 82, 73, - 80, 84, 32, 176, 5, 96, 5, 66, 79, 76, 68, 32, 176, 12, 6, 73, 84, 65, - 76, 73, 67, 34, 67, 34, 83, 223, 159, 16, 68, 204, 3, 98, 73, 122, 67, - 230, 2, 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, 83, 214, 5, 84, 215, 160, - 16, 68, 220, 1, 33, 6, 84, 65, 76, 73, 67, 32, 220, 1, 74, 67, 230, 2, - 69, 38, 75, 38, 78, 30, 80, 98, 82, 30, 83, 215, 5, 84, 102, 37, 7, 65, - 80, 73, 84, 65, 76, 32, 102, 250, 1, 84, 186, 4, 65, 22, 66, 2, 90, 22, - 68, 22, 69, 98, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, - 83, 70, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, - 222, 137, 2, 70, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 9, 40, 4, 72, - 69, 84, 65, 215, 244, 17, 65, 5, 159, 134, 17, 32, 2, 213, 161, 16, 4, - 80, 83, 73, 76, 2, 17, 2, 65, 80, 2, 159, 7, 80, 2, 197, 246, 17, 2, 65, - 66, 6, 74, 72, 172, 149, 5, 8, 65, 82, 84, 73, 65, 76, 32, 68, 167, 239, - 11, 73, 2, 207, 132, 17, 73, 2, 185, 132, 17, 2, 72, 79, 102, 29, 5, 77, - 65, 76, 76, 32, 102, 246, 1, 65, 22, 66, 2, 90, 22, 68, 22, 69, 38, 70, - 62, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, - 38, 85, 142, 197, 1, 67, 214, 249, 12, 77, 2, 78, 206, 201, 1, 88, 222, - 137, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 5, 251, 188, 14, 76, 5, - 207, 240, 17, 69, 5, 247, 219, 14, 69, 7, 186, 212, 1, 80, 131, 190, 16, - 84, 5, 11, 73, 2, 29, 5, 78, 65, 76, 32, 83, 2, 227, 1, 73, 5, 155, 194, - 15, 65, 5, 179, 239, 17, 79, 5, 11, 65, 2, 139, 218, 14, 80, 5, 187, 219, - 14, 65, 7, 11, 77, 4, 246, 170, 1, 73, 243, 165, 16, 69, 9, 174, 255, 17, - 72, 2, 83, 219, 19, 73, 5, 235, 243, 17, 72, 5, 11, 73, 2, 175, 241, 17, - 71, 7, 234, 216, 14, 72, 219, 148, 3, 65, 5, 207, 209, 1, 80, 2, 11, 72, - 2, 11, 69, 2, 11, 84, 2, 167, 254, 16, 65, 104, 11, 32, 104, 18, 67, 35, - 83, 52, 53, 5, 65, 80, 73, 84, 65, 52, 21, 3, 77, 65, 76, 52, 147, 254, - 11, 76, 82, 76, 8, 67, 65, 80, 73, 84, 65, 76, 32, 157, 1, 6, 83, 77, 65, - 76, 76, 32, 36, 254, 143, 18, 65, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, - 78, 2, 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, - 3, 90, 46, 226, 142, 18, 65, 2, 66, 2, 67, 2, 68, 2, 70, 2, 72, 2, 73, 2, - 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, - 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 40, 45, 9, 32, 78, 85, 77, 69, 82, 65, - 76, 32, 40, 70, 69, 66, 70, 70, 78, 26, 83, 66, 84, 170, 155, 16, 90, - 251, 85, 79, 6, 40, 4, 73, 71, 72, 84, 215, 247, 9, 76, 5, 147, 184, 16, - 69, 8, 30, 73, 105, 3, 79, 85, 82, 4, 206, 134, 5, 70, 167, 238, 12, 86, - 4, 65, 3, 73, 78, 69, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 5, 235, - 182, 16, 84, 10, 42, 72, 162, 247, 9, 87, 163, 195, 7, 69, 4, 154, 133, - 5, 73, 207, 149, 11, 82, 252, 8, 234, 1, 65, 192, 4, 9, 67, 72, 65, 78, - 73, 67, 65, 76, 32, 34, 68, 164, 15, 11, 69, 84, 69, 73, 32, 77, 65, 89, - 69, 75, 32, 178, 11, 76, 34, 78, 154, 50, 82, 176, 15, 8, 83, 83, 65, 71, - 69, 32, 87, 65, 20, 2, 84, 82, 235, 135, 17, 77, 26, 72, 6, 83, 85, 82, - 69, 68, 32, 253, 220, 16, 6, 84, 32, 79, 78, 32, 66, 24, 96, 5, 65, 78, - 71, 76, 69, 148, 142, 10, 9, 82, 73, 71, 72, 84, 32, 65, 78, 71, 131, - 231, 7, 66, 21, 11, 32, 18, 212, 1, 39, 87, 73, 84, 72, 32, 79, 80, 69, - 78, 32, 65, 82, 77, 32, 69, 78, 68, 73, 78, 71, 32, 73, 78, 32, 65, 82, - 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 181, 210, 17, 7, 79, - 80, 69, 78, 73, 78, 71, 16, 62, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, - 72, 43, 85, 4, 73, 3, 79, 87, 78, 4, 137, 144, 8, 5, 84, 32, 65, 78, 68, - 4, 11, 80, 4, 225, 227, 11, 3, 32, 65, 78, 4, 142, 133, 17, 65, 215, 56, - 76, 250, 1, 56, 9, 69, 70, 65, 73, 68, 82, 73, 78, 32, 159, 7, 73, 190, - 1, 174, 1, 67, 52, 6, 68, 73, 71, 73, 84, 32, 152, 1, 7, 78, 85, 77, 66, - 69, 82, 32, 114, 83, 164, 146, 7, 12, 69, 88, 67, 76, 65, 77, 65, 84, 73, - 79, 78, 32, 247, 154, 8, 70, 70, 128, 3, 5, 65, 80, 73, 84, 65, 191, 172, - 15, 79, 26, 82, 84, 48, 2, 79, 78, 222, 143, 16, 70, 30, 83, 102, 90, - 238, 85, 78, 235, 110, 69, 8, 44, 3, 72, 82, 69, 217, 128, 17, 2, 87, 79, - 4, 215, 128, 17, 69, 20, 50, 84, 230, 248, 4, 69, 46, 70, 42, 78, 31, 83, - 6, 166, 250, 4, 72, 172, 242, 4, 2, 87, 69, 135, 195, 7, 69, 70, 68, 3, - 77, 65, 76, 157, 206, 9, 8, 89, 77, 66, 79, 76, 32, 65, 73, 68, 45, 9, - 76, 32, 76, 69, 84, 84, 69, 82, 32, 68, 242, 1, 65, 38, 78, 130, 227, 3, - 72, 2, 75, 198, 197, 10, 89, 138, 147, 3, 79, 162, 64, 66, 2, 67, 2, 68, - 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, - 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 3, 90, 7, 146, 243, 3, 84, 255, - 136, 14, 73, 7, 234, 251, 17, 71, 3, 89, 60, 48, 5, 69, 86, 65, 76, 32, - 45, 3, 85, 77, 32, 6, 210, 237, 10, 69, 198, 187, 4, 67, 115, 81, 54, - 210, 1, 66, 50, 70, 164, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 248, 1, - 11, 77, 65, 84, 72, 69, 77, 65, 84, 73, 67, 65, 22, 83, 84, 4, 84, 72, - 82, 69, 182, 158, 15, 86, 130, 63, 69, 166, 4, 87, 215, 10, 71, 4, 180, - 156, 6, 3, 79, 76, 68, 255, 225, 7, 76, 10, 84, 9, 76, 65, 84, 84, 69, - 78, 69, 68, 32, 184, 3, 3, 79, 85, 82, 243, 222, 15, 73, 4, 44, 3, 76, - 69, 70, 1, 4, 82, 73, 71, 72, 2, 213, 1, 3, 84, 32, 80, 6, 11, 84, 6, 78, - 32, 41, 15, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 65, 78, 71, 76, 69, - 4, 36, 5, 67, 85, 82, 76, 89, 59, 80, 2, 17, 2, 32, 66, 2, 137, 183, 7, - 4, 82, 65, 67, 75, 2, 169, 178, 17, 10, 65, 82, 69, 78, 84, 72, 69, 83, - 73, 83, 2, 139, 183, 17, 76, 10, 212, 152, 6, 4, 77, 65, 76, 76, 250, - 229, 8, 72, 212, 95, 2, 73, 88, 183, 2, 65, 4, 11, 69, 4, 45, 9, 32, 80, - 79, 73, 78, 84, 69, 68, 32, 4, 162, 202, 9, 80, 219, 147, 6, 66, 158, 1, - 146, 1, 65, 104, 6, 67, 72, 69, 73, 75, 72, 34, 76, 144, 6, 8, 83, 89, - 76, 76, 65, 66, 76, 69, 0, 4, 87, 79, 82, 68, 50, 86, 139, 250, 15, 68, - 6, 80, 8, 72, 65, 78, 71, 32, 75, 72, 85, 164, 6, 3, 80, 85, 78, 147, - 213, 13, 78, 2, 159, 183, 16, 68, 4, 166, 162, 17, 65, 139, 60, 69, 94, + 73, 84, 72, 32, 2, 209, 150, 18, 7, 79, 71, 73, 67, 65, 76, 32, 10, 26, + 68, 94, 72, 59, 77, 6, 11, 79, 6, 44, 5, 85, 66, 76, 69, 32, 235, 168, + 18, 84, 4, 230, 182, 15, 85, 143, 61, 79, 2, 209, 228, 15, 9, 79, 82, 73, + 90, 79, 78, 84, 65, 76, 2, 173, 225, 7, 5, 73, 68, 68, 76, 69, 48, 70, + 68, 228, 1, 4, 76, 69, 70, 84, 241, 2, 5, 82, 73, 71, 72, 84, 6, 164, 1, + 30, 65, 83, 72, 32, 70, 82, 79, 77, 32, 76, 69, 70, 84, 32, 77, 69, 77, + 66, 69, 82, 32, 79, 70, 32, 68, 79, 85, 66, 76, 69, 198, 165, 16, 73, + 151, 210, 1, 82, 2, 17, 2, 32, 86, 2, 137, 151, 18, 5, 69, 82, 84, 73, + 67, 20, 46, 32, 193, 1, 6, 87, 65, 82, 68, 83, 32, 8, 48, 6, 82, 73, 71, + 72, 84, 32, 139, 131, 16, 84, 6, 44, 5, 65, 82, 82, 79, 87, 203, 248, 15, + 68, 5, 193, 248, 13, 18, 32, 87, 73, 84, 72, 32, 68, 69, 80, 69, 78, 68, + 69, 78, 84, 32, 76, 79, 12, 214, 3, 68, 42, 65, 146, 1, 83, 229, 244, 8, + 19, 72, 65, 82, 80, 79, 79, 78, 32, 65, 66, 79, 86, 69, 32, 83, 72, 79, + 82, 84, 22, 48, 6, 87, 65, 82, 68, 83, 32, 235, 251, 8, 32, 20, 88, 5, + 65, 82, 82, 79, 87, 202, 1, 68, 96, 8, 72, 65, 82, 80, 79, 79, 78, 32, + 91, 83, 11, 11, 32, 8, 164, 1, 7, 84, 72, 82, 79, 85, 71, 72, 132, 143, + 3, 10, 87, 73, 84, 72, 32, 68, 79, 85, 66, 76, 196, 232, 5, 9, 79, 86, + 69, 82, 32, 76, 79, 78, 71, 203, 188, 7, 70, 2, 203, 142, 18, 32, 4, 37, + 7, 79, 85, 66, 76, 69, 32, 65, 4, 25, 4, 82, 82, 79, 87, 5, 181, 179, 16, + 2, 32, 70, 4, 236, 246, 8, 4, 79, 86, 69, 82, 33, 11, 65, 66, 79, 86, 69, + 32, 83, 72, 79, 82, 84, 2, 233, 228, 7, 3, 81, 85, 73, 6, 92, 5, 73, 79, + 78, 32, 66, 144, 255, 17, 9, 32, 79, 70, 32, 70, 79, 82, 84, 85, 215, 93, + 85, 2, 211, 241, 9, 79, 4, 38, 76, 213, 132, 14, 3, 72, 79, 84, 2, 245, + 223, 17, 2, 69, 84, 222, 1, 34, 32, 229, 1, 3, 69, 82, 32, 14, 138, 1, + 66, 176, 201, 10, 11, 68, 79, 85, 66, 76, 69, 32, 80, 82, 73, 77, 214, + 183, 6, 65, 148, 113, 6, 75, 65, 86, 89, 75, 65, 227, 10, 76, 4, 38, 82, + 209, 249, 4, 3, 65, 84, 84, 2, 185, 130, 18, 7, 73, 71, 72, 84, 78, 69, + 83, 208, 1, 158, 1, 72, 140, 2, 5, 76, 69, 70, 84, 32, 236, 2, 6, 82, 73, + 71, 72, 84, 32, 154, 247, 15, 66, 74, 67, 74, 70, 234, 10, 77, 150, 2, + 79, 170, 18, 83, 67, 84, 20, 84, 4, 65, 76, 70, 32, 205, 186, 7, 11, 79, + 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 18, 176, 166, 7, 9, 65, 78, 68, + 32, 85, 80, 80, 69, 82, 32, 7, 73, 78, 86, 69, 82, 83, 69, 150, 216, 8, + 72, 230, 1, 86, 174, 26, 77, 130, 3, 76, 22, 82, 202, 5, 66, 247, 157, 1, + 67, 72, 186, 1, 66, 60, 8, 70, 79, 85, 78, 84, 65, 73, 78, 22, 80, 56, + 12, 83, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 82, 154, 1, 81, 246, 2, + 84, 232, 197, 11, 3, 67, 82, 65, 203, 187, 4, 79, 24, 56, 8, 65, 76, 76, + 80, 79, 73, 78, 84, 175, 135, 16, 76, 2, 179, 246, 9, 32, 4, 132, 131, + 15, 5, 65, 73, 78, 84, 66, 163, 141, 1, 69, 2, 185, 195, 7, 5, 32, 65, + 78, 84, 73, 74, 110, 81, 166, 2, 83, 82, 84, 202, 178, 7, 82, 158, 199, + 8, 66, 238, 3, 67, 226, 3, 79, 222, 7, 68, 207, 2, 80, 30, 17, 2, 85, 65, + 30, 48, 6, 68, 82, 65, 78, 84, 32, 239, 146, 16, 82, 28, 74, 70, 60, 2, + 78, 69, 50, 83, 150, 142, 16, 67, 226, 1, 77, 143, 1, 84, 4, 26, 65, 231, + 198, 17, 82, 2, 149, 185, 7, 3, 67, 69, 32, 2, 25, 4, 85, 84, 82, 65, 2, + 195, 209, 18, 76, 4, 234, 224, 10, 77, 215, 175, 5, 84, 6, 240, 191, 7, + 11, 69, 77, 73, 67, 73, 82, 67, 85, 76, 65, 82, 191, 209, 8, 72, 6, 150, + 146, 16, 82, 155, 1, 87, 5, 253, 230, 17, 25, 32, 68, 73, 86, 73, 68, 69, + 68, 32, 66, 89, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 82, 85, + 6, 18, 71, 23, 78, 2, 143, 253, 13, 71, 4, 168, 231, 17, 5, 65, 82, 32, + 69, 67, 151, 103, 71, 114, 104, 12, 67, 73, 65, 78, 32, 76, 69, 84, 84, + 69, 82, 32, 232, 1, 5, 68, 73, 65, 78, 32, 131, 129, 18, 73, 58, 210, 1, + 77, 150, 138, 5, 75, 154, 214, 9, 84, 158, 24, 66, 246, 146, 2, 65, 2, + 69, 2, 78, 238, 253, 1, 68, 2, 71, 2, 72, 2, 73, 2, 74, 2, 76, 2, 80, 2, + 81, 2, 82, 2, 83, 2, 85, 2, 87, 2, 88, 3, 90, 5, 167, 137, 19, 77, 54, + 88, 7, 76, 69, 84, 84, 69, 82, 32, 153, 224, 14, 9, 84, 82, 73, 65, 78, + 71, 85, 76, 65, 52, 222, 159, 13, 84, 190, 215, 1, 76, 198, 135, 2, 83, + 238, 11, 65, 2, 69, 2, 78, 238, 253, 1, 66, 2, 67, 2, 68, 2, 70, 2, 71, + 2, 73, 2, 75, 2, 77, 2, 79, 2, 81, 2, 82, 2, 85, 2, 86, 3, 89, 128, 54, + 162, 1, 65, 194, 103, 69, 182, 103, 73, 154, 25, 79, 164, 112, 3, 82, 79, + 32, 234, 3, 85, 204, 70, 7, 89, 65, 78, 77, 65, 82, 32, 138, 165, 15, 86, + 198, 61, 77, 27, 87, 204, 23, 186, 1, 71, 62, 72, 254, 10, 75, 234, 3, + 76, 252, 13, 2, 77, 77, 22, 78, 154, 17, 80, 118, 82, 174, 7, 83, 190, 7, + 84, 184, 36, 3, 89, 65, 78, 204, 129, 1, 3, 88, 73, 77, 175, 248, 15, 67, + 6, 228, 243, 17, 4, 73, 67, 32, 87, 138, 91, 78, 155, 53, 69, 166, 1, 84, + 6, 65, 74, 65, 78, 73, 32, 129, 3, 10, 74, 79, 78, 71, 32, 84, 73, 76, + 69, 32, 78, 38, 76, 162, 2, 83, 215, 140, 13, 65, 72, 88, 6, 69, 84, 84, + 69, 82, 32, 161, 150, 12, 10, 73, 71, 65, 84, 85, 82, 69, 32, 83, 72, 70, + 134, 175, 3, 78, 150, 245, 11, 68, 82, 82, 34, 84, 162, 149, 3, 66, 2, + 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 83, 2, 86, + 186, 2, 65, 2, 69, 2, 73, 2, 79, 3, 85, 4, 170, 237, 6, 69, 253, 245, 7, + 5, 73, 71, 78, 32, 78, 88, 236, 1, 2, 66, 65, 38, 69, 46, 70, 46, 78, 42, + 79, 46, 80, 22, 83, 118, 84, 194, 1, 87, 112, 5, 71, 82, 69, 69, 78, 0, + 3, 82, 69, 68, 168, 136, 6, 3, 65, 85, 84, 222, 21, 74, 209, 175, 11, 11, + 67, 72, 82, 89, 83, 65, 78, 84, 72, 69, 77, 4, 162, 158, 1, 77, 199, 220, + 17, 67, 8, 228, 2, 4, 73, 71, 72, 84, 195, 1, 65, 12, 172, 2, 2, 73, 86, + 13, 3, 79, 85, 82, 8, 192, 1, 2, 79, 82, 65, 2, 73, 78, 8, 218, 1, 78, + 173, 229, 11, 3, 82, 67, 72, 2, 139, 209, 17, 76, 18, 88, 2, 79, 85, 76, + 4, 69, 86, 69, 78, 0, 2, 73, 88, 182, 157, 12, 85, 255, 199, 1, 80, 2, + 157, 2, 2, 84, 72, 12, 36, 3, 72, 82, 69, 13, 2, 87, 79, 6, 11, 69, 6, + 25, 4, 32, 79, 70, 32, 6, 46, 67, 189, 248, 6, 5, 66, 65, 77, 66, 79, 4, + 184, 140, 6, 2, 73, 82, 153, 234, 10, 5, 72, 65, 82, 65, 67, 6, 50, 69, + 60, 4, 72, 73, 84, 69, 227, 190, 17, 73, 2, 17, 2, 83, 84, 2, 17, 2, 32, + 87, 2, 247, 232, 17, 73, 2, 17, 2, 32, 68, 2, 163, 190, 17, 82, 52, 52, + 5, 65, 83, 65, 82, 32, 241, 139, 15, 2, 69, 77, 50, 162, 1, 69, 40, 7, + 76, 69, 84, 84, 69, 82, 32, 152, 1, 5, 80, 65, 83, 83, 73, 32, 11, 86, + 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 173, 236, 18, 3, 65, 78, 71, 2, + 205, 244, 15, 5, 78, 68, 32, 79, 70, 36, 182, 249, 16, 78, 222, 250, 1, + 66, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, + 2, 84, 2, 86, 2, 89, 187, 2, 65, 2, 11, 77, 2, 211, 156, 18, 66, 8, 146, + 245, 18, 69, 2, 73, 2, 79, 3, 85, 246, 1, 80, 7, 65, 89, 65, 76, 65, 77, + 32, 176, 11, 2, 69, 32, 225, 1, 3, 84, 69, 83, 236, 1, 198, 1, 68, 44, 9, + 70, 82, 65, 67, 84, 73, 79, 78, 32, 240, 1, 7, 76, 69, 84, 84, 69, 82, + 32, 164, 5, 7, 78, 85, 77, 66, 69, 82, 32, 36, 5, 83, 73, 71, 78, 32, + 230, 191, 13, 86, 239, 201, 1, 65, 22, 164, 171, 8, 2, 65, 84, 163, 211, + 8, 73, 26, 56, 4, 79, 78, 69, 32, 121, 6, 84, 72, 82, 69, 69, 32, 18, 82, + 84, 254, 135, 5, 83, 130, 132, 8, 70, 62, 79, 146, 222, 3, 69, 46, 72, + 47, 81, 4, 134, 143, 13, 87, 255, 220, 3, 69, 8, 150, 136, 5, 83, 198, + 135, 8, 69, 110, 84, 179, 220, 3, 81, 132, 1, 226, 1, 65, 82, 67, 158, 1, + 68, 78, 84, 82, 86, 226, 141, 13, 78, 182, 128, 2, 76, 38, 82, 134, 6, + 85, 206, 141, 1, 79, 238, 60, 73, 182, 196, 1, 83, 82, 66, 2, 71, 2, 74, + 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, 2, 77, 3, 89, 11, 240, 167, 16, 7, + 82, 67, 72, 65, 73, 67, 32, 206, 198, 2, 65, 2, 73, 3, 85, 22, 26, 72, + 215, 237, 18, 65, 20, 44, 5, 73, 76, 76, 85, 32, 167, 237, 18, 65, 18, + 138, 132, 13, 76, 174, 235, 3, 78, 146, 197, 1, 82, 222, 56, 75, 2, 77, + 3, 89, 10, 212, 226, 10, 4, 79, 84, 32, 82, 190, 194, 7, 68, 138, 69, 72, + 187, 2, 65, 10, 38, 84, 170, 233, 18, 72, 187, 2, 65, 6, 166, 233, 18, + 72, 2, 84, 187, 2, 65, 12, 242, 227, 3, 69, 234, 176, 11, 79, 223, 214, + 3, 65, 6, 162, 142, 13, 79, 223, 134, 4, 84, 18, 50, 67, 114, 86, 250, + 142, 15, 65, 251, 149, 3, 80, 6, 54, 79, 120, 5, 73, 82, 67, 85, 76, 171, + 203, 14, 65, 2, 145, 190, 13, 9, 77, 66, 73, 78, 73, 78, 71, 32, 65, 6, + 60, 9, 69, 82, 84, 73, 67, 65, 76, 32, 66, 255, 165, 18, 73, 2, 253, 142, + 15, 2, 65, 82, 8, 80, 12, 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 69, 32, + 70, 65, 227, 224, 17, 83, 4, 64, 10, 65, 78, 68, 32, 77, 65, 76, 69, 32, + 65, 227, 224, 17, 83, 2, 25, 4, 78, 68, 32, 70, 2, 11, 69, 2, 11, 77, 2, + 139, 211, 7, 65, 2, 191, 163, 17, 69, 2, 243, 225, 16, 79, 185, 1, 210, + 1, 32, 220, 2, 5, 68, 65, 73, 67, 32, 224, 3, 8, 73, 67, 72, 65, 69, 65, + 78, 32, 222, 8, 83, 156, 166, 2, 3, 85, 65, 76, 130, 228, 10, 65, 240, + 167, 1, 8, 84, 69, 76, 80, 73, 69, 67, 69, 235, 132, 4, 71, 12, 132, 1, + 3, 73, 78, 32, 128, 1, 5, 87, 73, 84, 72, 32, 136, 147, 2, 3, 68, 65, 78, + 237, 140, 13, 8, 65, 78, 68, 32, 87, 79, 77, 65, 4, 248, 140, 10, 19, 66, + 85, 83, 73, 78, 69, 83, 83, 32, 83, 85, 73, 84, 32, 76, 69, 86, 73, 84, + 189, 151, 1, 4, 84, 85, 88, 69, 4, 140, 243, 13, 8, 71, 85, 65, 32, 80, + 73, 32, 77, 205, 163, 3, 4, 84, 85, 82, 66, 58, 112, 4, 65, 70, 70, 82, + 28, 7, 76, 69, 84, 84, 69, 82, 32, 208, 184, 14, 3, 86, 79, 67, 226, 72, + 71, 251, 25, 80, 2, 249, 247, 17, 2, 73, 67, 50, 82, 65, 176, 1, 2, 68, + 85, 2, 85, 28, 3, 72, 65, 76, 22, 73, 183, 152, 18, 75, 38, 154, 1, 75, + 206, 246, 12, 84, 250, 191, 1, 83, 130, 217, 3, 73, 226, 79, 66, 2, 68, + 2, 71, 2, 72, 2, 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 82, 3, 90, 5, 235, + 220, 18, 83, 2, 185, 132, 8, 2, 83, 72, 2, 187, 220, 18, 81, 4, 222, 222, + 18, 78, 3, 84, 102, 216, 1, 7, 76, 69, 84, 84, 69, 82, 32, 194, 4, 78, + 80, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 220, 1, 4, 83, + 73, 71, 78, 249, 200, 10, 16, 65, 66, 66, 82, 69, 86, 73, 65, 84, 73, 79, + 78, 32, 77, 65, 82, 72, 218, 1, 65, 46, 66, 38, 68, 30, 71, 30, 74, 2, + 90, 30, 75, 30, 81, 38, 83, 50, 84, 54, 88, 158, 164, 6, 76, 134, 206, 1, + 82, 246, 221, 2, 89, 198, 207, 1, 72, 190, 184, 5, 78, 134, 2, 87, 218, + 103, 70, 2, 80, 171, 4, 77, 6, 142, 209, 10, 76, 122, 65, 179, 137, 7, + 89, 4, 134, 249, 12, 72, 227, 220, 3, 69, 4, 254, 165, 6, 65, 23, 72, 4, + 182, 208, 10, 72, 15, 73, 4, 242, 208, 10, 72, 15, 65, 4, 226, 166, 12, + 72, 15, 65, 4, 210, 165, 6, 72, 131, 129, 6, 79, 8, 186, 205, 10, 83, + 154, 3, 65, 131, 137, 7, 72, 6, 206, 164, 6, 72, 178, 175, 10, 69, 219, + 134, 1, 65, 4, 202, 165, 12, 65, 3, 79, 10, 33, 6, 85, 77, 66, 69, 82, + 32, 10, 250, 207, 10, 79, 58, 84, 131, 203, 2, 70, 14, 80, 2, 68, 79, + 116, 3, 76, 73, 78, 190, 167, 5, 70, 230, 219, 11, 84, 167, 10, 83, 6, + 54, 84, 13, 9, 85, 66, 76, 69, 32, 68, 79, 84, 32, 5, 11, 32, 2, 25, 4, + 87, 73, 84, 72, 2, 151, 193, 2, 73, 2, 231, 203, 3, 69, 2, 139, 168, 17, + 32, 2, 11, 32, 2, 249, 190, 18, 2, 83, 72, 4, 92, 17, 32, 83, 89, 77, 66, + 79, 76, 32, 70, 79, 82, 32, 76, 73, 71, 72, 84, 139, 220, 10, 76, 2, 179, + 140, 15, 72, 142, 1, 142, 1, 65, 20, 5, 67, 72, 69, 78, 32, 216, 164, 6, + 4, 82, 73, 65, 71, 249, 175, 11, 13, 84, 73, 65, 76, 32, 65, 82, 84, 83, + 32, 85, 78, 73, 2, 227, 161, 5, 67, 136, 1, 152, 1, 7, 76, 69, 84, 84, + 69, 82, 32, 194, 1, 83, 236, 2, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, + 78, 32, 170, 129, 18, 72, 225, 5, 4, 77, 65, 82, 75, 60, 254, 3, 84, 206, + 148, 2, 68, 166, 188, 14, 78, 214, 181, 1, 67, 2, 75, 2, 80, 2, 83, 2, + 90, 138, 69, 45, 2, 66, 2, 71, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, + 2, 89, 187, 2, 65, 62, 96, 4, 73, 71, 78, 32, 37, 16, 85, 66, 74, 79, 73, + 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 32, 4, 254, 177, 14, 67, 235, + 216, 3, 65, 58, 182, 1, 84, 206, 148, 2, 68, 166, 188, 14, 78, 214, 181, + 1, 67, 2, 75, 2, 80, 2, 83, 2, 90, 138, 69, 66, 2, 71, 2, 72, 2, 74, 2, + 76, 2, 77, 2, 82, 2, 87, 2, 89, 187, 2, 65, 8, 194, 134, 18, 83, 138, 69, + 72, 187, 2, 65, 10, 158, 203, 18, 65, 186, 2, 69, 2, 73, 2, 79, 3, 85, + 156, 1, 120, 11, 65, 82, 65, 77, 32, 71, 79, 78, 68, 73, 32, 232, 5, 3, + 67, 85, 76, 64, 5, 75, 32, 87, 79, 82, 183, 132, 18, 85, 150, 1, 100, 7, + 76, 69, 84, 84, 69, 82, 32, 178, 2, 82, 56, 5, 83, 73, 71, 78, 32, 82, + 86, 247, 211, 16, 68, 94, 210, 1, 74, 42, 84, 198, 235, 14, 65, 38, 68, + 214, 6, 85, 186, 202, 1, 73, 42, 76, 190, 195, 1, 75, 38, 78, 46, 83, 82, + 66, 2, 67, 2, 71, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 86, 2, 89, 186, 2, + 69, 3, 79, 6, 130, 199, 18, 78, 38, 72, 187, 2, 65, 10, 246, 129, 18, 84, + 138, 69, 72, 2, 82, 187, 2, 65, 4, 32, 2, 65, 45, 219, 236, 14, 69, 2, + 147, 132, 18, 75, 10, 230, 176, 1, 67, 182, 207, 12, 72, 178, 43, 78, + 150, 227, 1, 86, 247, 244, 1, 65, 22, 26, 79, 199, 139, 16, 73, 20, 45, + 9, 87, 69, 76, 32, 83, 73, 71, 78, 32, 20, 74, 86, 198, 239, 14, 65, 38, + 85, 186, 202, 1, 73, 198, 140, 2, 69, 3, 79, 2, 141, 203, 7, 6, 79, 67, + 65, 76, 73, 67, 2, 145, 141, 18, 11, 73, 78, 69, 32, 79, 82, 68, 73, 78, + 65, 76, 2, 179, 176, 17, 75, 226, 15, 76, 10, 72, 69, 77, 65, 84, 73, 67, + 65, 76, 32, 153, 224, 14, 3, 69, 32, 68, 224, 15, 252, 1, 5, 66, 79, 76, + 68, 32, 168, 6, 14, 68, 79, 85, 66, 76, 69, 45, 83, 84, 82, 85, 67, 75, + 32, 238, 1, 70, 164, 2, 7, 73, 84, 65, 76, 73, 67, 32, 180, 3, 10, 77, + 79, 78, 79, 83, 80, 65, 67, 69, 32, 40, 2, 82, 73, 36, 3, 76, 69, 70, + 167, 1, 83, 160, 5, 176, 1, 8, 67, 65, 80, 73, 84, 65, 76, 32, 130, 2, + 83, 210, 14, 73, 210, 3, 69, 38, 75, 38, 78, 30, 80, 98, 82, 242, 5, 84, + 56, 7, 70, 82, 65, 75, 84, 85, 82, 195, 177, 16, 68, 104, 190, 4, 68, + 174, 15, 84, 174, 4, 65, 22, 66, 2, 90, 42, 69, 98, 71, 22, 73, 22, 75, + 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 70, 85, 214, 197, 1, 67, 198, + 137, 13, 77, 2, 78, 186, 202, 1, 88, 198, 140, 2, 70, 2, 72, 2, 74, 2, + 81, 2, 86, 2, 87, 3, 89, 208, 1, 60, 5, 77, 65, 76, 76, 32, 201, 25, 5, + 67, 82, 73, 80, 84, 104, 250, 1, 68, 218, 19, 65, 22, 66, 2, 90, 42, 69, + 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, + 34, 84, 38, 85, 214, 197, 1, 67, 198, 137, 13, 77, 2, 78, 186, 202, 1, + 88, 198, 140, 2, 72, 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 7, 26, 73, 151, + 128, 15, 69, 2, 247, 213, 1, 71, 110, 68, 8, 67, 65, 80, 73, 84, 65, 76, + 32, 150, 23, 83, 131, 177, 16, 68, 38, 154, 188, 18, 65, 2, 66, 2, 68, 2, + 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 79, 2, 83, 2, 84, + 2, 85, 2, 86, 2, 87, 2, 88, 3, 89, 96, 52, 7, 82, 65, 75, 84, 85, 82, 32, + 255, 143, 7, 65, 94, 52, 8, 67, 65, 80, 73, 84, 65, 76, 32, 131, 21, 83, + 42, 134, 186, 18, 65, 2, 66, 2, 68, 2, 69, 2, 70, 2, 71, 2, 74, 2, 75, 2, + 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, + 2, 88, 3, 89, 222, 1, 100, 6, 83, 77, 65, 76, 76, 32, 222, 7, 67, 218, 2, + 69, 38, 75, 38, 78, 30, 80, 98, 82, 243, 5, 84, 104, 242, 1, 68, 186, 12, + 65, 22, 66, 2, 90, 42, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, 22, + 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 214, 197, 1, 67, 198, 137, + 13, 77, 2, 78, 186, 202, 1, 88, 198, 140, 2, 74, 2, 81, 2, 86, 2, 87, 3, + 89, 9, 52, 7, 79, 84, 76, 69, 83, 83, 32, 219, 248, 14, 69, 4, 186, 181, + 18, 73, 3, 74, 124, 246, 15, 67, 34, 83, 131, 177, 16, 68, 12, 32, 2, 71, + 72, 167, 138, 7, 83, 10, 17, 2, 84, 32, 10, 112, 6, 87, 72, 73, 84, 69, + 32, 142, 245, 5, 68, 222, 107, 65, 153, 235, 4, 9, 70, 76, 65, 84, 84, + 69, 78, 69, 68, 4, 174, 180, 14, 83, 39, 84, 130, 6, 84, 10, 65, 78, 83, + 45, 83, 69, 82, 73, 70, 32, 249, 13, 6, 67, 82, 73, 80, 84, 32, 176, 5, + 96, 5, 66, 79, 76, 68, 32, 164, 12, 6, 73, 84, 65, 76, 73, 67, 34, 67, + 34, 83, 131, 177, 16, 68, 204, 3, 98, 73, 122, 67, 218, 2, 69, 38, 75, + 38, 78, 30, 80, 98, 82, 30, 83, 214, 5, 84, 251, 177, 16, 68, 220, 1, 33, + 6, 84, 65, 76, 73, 67, 32, 220, 1, 74, 67, 218, 2, 69, 38, 75, 38, 78, + 30, 80, 98, 82, 30, 83, 215, 5, 84, 102, 37, 7, 65, 80, 73, 84, 65, 76, + 32, 102, 250, 1, 84, 174, 4, 65, 22, 66, 2, 90, 22, 68, 22, 69, 98, 71, + 22, 73, 22, 75, 34, 76, 22, 79, 50, 80, 42, 82, 22, 83, 70, 85, 214, 197, + 1, 67, 198, 137, 13, 77, 2, 78, 186, 202, 1, 88, 198, 140, 2, 70, 2, 72, + 2, 74, 2, 81, 2, 86, 2, 87, 3, 89, 9, 232, 159, 11, 4, 72, 69, 84, 65, + 151, 233, 6, 65, 2, 249, 178, 16, 4, 80, 83, 73, 76, 2, 17, 2, 65, 80, 2, + 159, 7, 80, 2, 209, 138, 18, 2, 65, 66, 6, 74, 72, 148, 150, 5, 8, 65, + 82, 84, 73, 65, 76, 32, 68, 175, 128, 12, 73, 2, 191, 150, 17, 73, 2, + 169, 150, 17, 2, 72, 79, 102, 29, 5, 77, 65, 76, 76, 32, 102, 246, 1, 65, + 22, 66, 2, 90, 22, 68, 22, 69, 38, 70, 62, 71, 22, 73, 22, 75, 34, 76, + 22, 79, 50, 80, 42, 82, 22, 83, 34, 84, 38, 85, 214, 197, 1, 67, 198, + 137, 13, 77, 2, 78, 186, 202, 1, 88, 198, 140, 2, 72, 2, 74, 2, 81, 2, + 86, 2, 87, 3, 89, 5, 179, 205, 14, 76, 5, 219, 132, 18, 69, 5, 175, 236, + 14, 69, 7, 130, 213, 1, 80, 199, 209, 16, 84, 5, 11, 73, 2, 29, 5, 78, + 65, 76, 32, 83, 2, 227, 1, 73, 5, 215, 210, 15, 65, 5, 191, 131, 18, 79, + 5, 11, 65, 2, 195, 234, 14, 80, 5, 243, 235, 14, 65, 7, 11, 77, 4, 190, + 171, 1, 73, 183, 185, 16, 69, 9, 186, 147, 18, 72, 2, 83, 219, 19, 73, 5, + 247, 135, 18, 72, 5, 11, 73, 2, 187, 133, 18, 71, 7, 162, 233, 14, 72, + 175, 152, 3, 65, 5, 151, 210, 1, 80, 2, 11, 72, 2, 11, 69, 2, 11, 84, 2, + 151, 144, 17, 65, 104, 11, 32, 104, 18, 67, 35, 83, 52, 53, 5, 65, 80, + 73, 84, 65, 52, 21, 3, 77, 65, 76, 52, 195, 131, 12, 76, 82, 76, 8, 67, + 65, 80, 73, 84, 65, 76, 32, 157, 1, 6, 83, 77, 65, 76, 76, 32, 36, 138, + 164, 18, 65, 2, 67, 2, 68, 2, 71, 2, 74, 2, 75, 2, 78, 2, 79, 2, 80, 2, + 81, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 46, 238, 162, + 18, 65, 2, 66, 2, 67, 2, 68, 2, 70, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, + 77, 2, 78, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, + 2, 89, 3, 90, 40, 45, 9, 32, 78, 85, 77, 69, 82, 65, 76, 32, 40, 70, 69, + 66, 70, 70, 78, 26, 83, 66, 84, 206, 172, 16, 90, 223, 86, 79, 6, 40, 4, + 73, 71, 72, 84, 203, 253, 9, 76, 5, 131, 202, 16, 69, 8, 30, 73, 105, 3, + 79, 85, 82, 4, 146, 135, 5, 70, 239, 129, 13, 86, 4, 65, 3, 73, 78, 69, + 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 5, 219, 200, 16, 84, 10, 42, 72, + 150, 253, 9, 87, 187, 209, 7, 69, 4, 222, 133, 5, 73, 175, 166, 11, 82, + 130, 9, 226, 1, 65, 188, 4, 9, 67, 72, 65, 78, 73, 67, 65, 76, 32, 34, + 68, 204, 15, 11, 69, 84, 69, 73, 32, 77, 65, 89, 69, 75, 32, 178, 11, 76, + 62, 78, 134, 50, 82, 176, 15, 8, 83, 83, 65, 71, 69, 32, 87, 65, 22, 84, + 211, 155, 17, 77, 26, 68, 6, 83, 85, 82, 69, 68, 32, 137, 182, 14, 5, 84, + 32, 79, 78, 32, 24, 96, 5, 65, 78, 71, 76, 69, 164, 148, 10, 9, 82, 73, + 71, 72, 84, 32, 65, 78, 71, 139, 245, 7, 66, 21, 11, 32, 18, 212, 1, 39, + 87, 73, 84, 72, 32, 79, 80, 69, 78, 32, 65, 82, 77, 32, 69, 78, 68, 73, + 78, 71, 32, 73, 78, 32, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, + 78, 71, 32, 205, 230, 17, 7, 79, 80, 69, 78, 73, 78, 71, 16, 62, 68, 24, + 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 43, 85, 4, 73, 3, 79, 87, 78, 4, + 157, 150, 8, 5, 84, 32, 65, 78, 68, 4, 11, 80, 4, 153, 233, 11, 3, 32, + 65, 78, 4, 166, 153, 17, 65, 215, 56, 76, 252, 1, 56, 9, 69, 70, 65, 73, + 68, 82, 73, 78, 32, 159, 7, 73, 190, 1, 174, 1, 67, 52, 6, 68, 73, 71, + 73, 84, 32, 152, 1, 7, 78, 85, 77, 66, 69, 82, 32, 114, 83, 224, 147, 7, + 12, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 131, 170, 8, 70, 70, + 128, 3, 5, 65, 80, 73, 84, 65, 135, 189, 15, 79, 26, 82, 84, 48, 2, 79, + 78, 142, 161, 16, 70, 30, 83, 102, 90, 210, 86, 78, 239, 112, 69, 8, 44, + 3, 72, 82, 69, 241, 148, 17, 2, 87, 79, 4, 239, 148, 17, 69, 20, 50, 84, + 182, 249, 4, 69, 46, 70, 42, 78, 31, 83, 6, 246, 250, 4, 72, 220, 247, 4, + 2, 87, 69, 159, 209, 7, 69, 70, 68, 3, 77, 65, 76, 157, 212, 9, 8, 89, + 77, 66, 79, 76, 32, 65, 73, 68, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, + 32, 68, 242, 1, 65, 38, 78, 218, 227, 3, 72, 2, 75, 178, 213, 10, 89, + 222, 150, 3, 79, 162, 64, 66, 2, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, + 2, 74, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, + 87, 2, 88, 3, 90, 7, 254, 243, 3, 84, 171, 156, 14, 73, 7, 130, 144, 18, + 71, 3, 89, 62, 48, 5, 69, 86, 65, 76, 32, 45, 3, 85, 77, 32, 6, 218, 243, + 10, 69, 134, 198, 4, 67, 115, 81, 56, 210, 1, 66, 50, 70, 164, 1, 3, 76, + 69, 70, 0, 4, 82, 73, 71, 72, 248, 1, 11, 77, 65, 84, 72, 69, 77, 65, 84, + 73, 67, 65, 22, 83, 124, 4, 84, 72, 82, 69, 214, 174, 15, 86, 246, 62, + 69, 166, 4, 87, 203, 11, 71, 4, 164, 158, 6, 3, 79, 76, 68, 211, 239, 7, + 76, 10, 84, 9, 76, 65, 84, 84, 69, 78, 69, 68, 32, 224, 3, 3, 79, 85, 82, + 135, 239, 15, 73, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 213, 1, + 3, 84, 32, 80, 6, 11, 84, 6, 78, 32, 41, 15, 45, 80, 79, 73, 78, 84, 73, + 78, 71, 32, 65, 78, 71, 76, 69, 4, 36, 5, 67, 85, 82, 76, 89, 59, 80, 2, + 17, 2, 32, 66, 2, 249, 184, 7, 4, 82, 65, 67, 75, 2, 193, 198, 17, 10, + 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 2, 163, 203, 17, 76, 12, 150, + 143, 15, 72, 200, 95, 2, 73, 88, 182, 2, 65, 245, 115, 15, 77, 65, 76, + 76, 32, 87, 72, 73, 84, 69, 32, 67, 73, 82, 67, 4, 11, 69, 4, 45, 9, 32, + 80, 79, 73, 78, 84, 69, 68, 32, 4, 250, 207, 9, 80, 151, 158, 6, 66, 158, + 1, 146, 1, 65, 104, 6, 67, 72, 69, 73, 75, 72, 34, 76, 144, 6, 8, 83, 89, + 76, 76, 65, 66, 76, 69, 0, 4, 87, 79, 82, 68, 50, 86, 147, 139, 16, 68, + 6, 80, 8, 72, 65, 78, 71, 32, 75, 72, 85, 164, 6, 3, 80, 85, 78, 227, + 223, 13, 78, 2, 251, 200, 16, 68, 4, 150, 182, 17, 65, 139, 60, 69, 94, 52, 6, 69, 84, 84, 69, 82, 32, 185, 5, 2, 85, 77, 92, 250, 1, 66, 36, 2, 67, 72, 38, 68, 50, 71, 38, 74, 34, 75, 42, 78, 62, 80, 30, 83, 42, 84, - 54, 73, 0, 3, 76, 65, 73, 0, 3, 77, 73, 84, 138, 247, 12, 72, 166, 10, - 82, 2, 87, 232, 181, 2, 2, 65, 84, 182, 220, 1, 89, 246, 8, 85, 226, 79, - 69, 3, 79, 4, 182, 180, 16, 72, 255, 186, 1, 65, 4, 218, 229, 16, 73, - 183, 137, 1, 65, 8, 238, 165, 10, 72, 202, 191, 6, 73, 247, 65, 68, 4, - 190, 165, 10, 72, 231, 197, 7, 79, 4, 154, 179, 16, 72, 203, 49, 73, 6, - 216, 1, 2, 79, 75, 163, 163, 10, 72, 12, 178, 1, 65, 0, 3, 71, 79, 85, - 230, 233, 17, 78, 3, 89, 6, 118, 65, 163, 177, 16, 72, 6, 158, 220, 17, - 65, 162, 14, 72, 3, 83, 10, 48, 2, 73, 76, 162, 163, 10, 72, 191, 129, 7, - 84, 5, 181, 177, 1, 4, 32, 76, 79, 78, 2, 209, 232, 17, 3, 32, 73, 89, 2, - 173, 130, 17, 7, 32, 82, 69, 80, 69, 84, 73, 30, 64, 10, 79, 87, 69, 76, - 32, 83, 73, 71, 78, 32, 143, 178, 15, 73, 28, 130, 1, 65, 44, 4, 67, 72, - 69, 73, 2, 79, 0, 3, 83, 79, 85, 0, 2, 89, 69, 22, 73, 38, 85, 194, 131, - 11, 78, 151, 175, 4, 86, 8, 242, 216, 16, 78, 182, 80, 65, 187, 64, 85, - 2, 199, 216, 16, 78, 4, 178, 216, 16, 78, 239, 144, 1, 73, 4, 142, 216, - 16, 78, 239, 144, 1, 85, 4, 182, 224, 16, 84, 195, 56, 79, 178, 3, 160, - 1, 11, 68, 69, 32, 75, 73, 75, 65, 75, 85, 73, 32, 140, 211, 16, 20, 79, - 82, 65, 72, 32, 87, 73, 84, 72, 32, 78, 73, 78, 69, 32, 66, 82, 65, 78, - 67, 95, 83, 174, 3, 148, 1, 17, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, - 78, 85, 77, 66, 69, 82, 32, 164, 1, 10, 83, 89, 76, 76, 65, 66, 76, 69, - 32, 77, 155, 167, 9, 68, 14, 62, 84, 56, 7, 72, 85, 78, 68, 82, 69, 68, - 135, 186, 4, 77, 8, 26, 69, 207, 186, 4, 72, 6, 26, 78, 167, 161, 14, 69, - 4, 160, 186, 4, 2, 32, 84, 219, 170, 13, 83, 142, 3, 22, 48, 163, 21, 49, - 198, 1, 118, 48, 250, 1, 49, 210, 1, 50, 138, 2, 51, 254, 1, 52, 150, 2, - 53, 158, 2, 54, 242, 1, 55, 134, 2, 56, 187, 2, 57, 18, 142, 1, 49, 34, - 50, 22, 51, 22, 52, 254, 150, 2, 53, 154, 240, 3, 56, 156, 218, 10, 3, - 57, 32, 77, 92, 3, 55, 32, 77, 237, 90, 3, 54, 32, 87, 2, 11, 32, 2, 195, - 206, 17, 75, 2, 159, 217, 17, 32, 2, 215, 148, 12, 32, 2, 11, 32, 2, 251, - 205, 17, 87, 20, 130, 1, 49, 22, 54, 18, 56, 22, 57, 248, 28, 2, 48, 32, - 198, 183, 7, 53, 170, 203, 5, 52, 198, 10, 55, 222, 9, 50, 243, 235, 3, - 51, 2, 147, 179, 17, 32, 2, 167, 25, 32, 2, 163, 204, 13, 32, 2, 227, - 245, 12, 32, 20, 106, 49, 22, 50, 22, 51, 22, 52, 22, 53, 22, 54, 22, 55, - 22, 57, 130, 224, 11, 48, 145, 231, 1, 2, 56, 32, 2, 211, 169, 10, 32, 2, - 247, 128, 10, 32, 2, 143, 186, 17, 32, 2, 247, 142, 12, 32, 2, 167, 227, - 12, 32, 2, 135, 188, 17, 32, 2, 179, 221, 12, 32, 2, 239, 28, 32, 20, - 174, 1, 48, 16, 2, 49, 32, 20, 2, 52, 32, 20, 2, 56, 32, 246, 194, 4, 57, - 182, 51, 50, 22, 53, 136, 230, 7, 2, 54, 32, 232, 255, 3, 3, 55, 32, 78, - 237, 90, 3, 51, 32, 89, 2, 179, 37, 32, 2, 207, 200, 17, 89, 2, 187, 200, - 17, 70, 2, 255, 145, 16, 78, 20, 184, 1, 2, 48, 32, 20, 2, 51, 32, 30, - 53, 22, 56, 160, 8, 3, 52, 32, 75, 128, 22, 3, 54, 32, 72, 222, 2, 55, - 138, 213, 4, 49, 208, 179, 4, 3, 57, 32, 87, 241, 232, 5, 3, 50, 32, 72, - 2, 247, 234, 15, 72, 2, 213, 151, 17, 2, 78, 71, 2, 191, 162, 11, 32, 2, - 235, 186, 17, 32, 20, 178, 1, 48, 18, 53, 20, 2, 54, 32, 20, 2, 56, 32, - 22, 57, 136, 213, 9, 2, 50, 32, 236, 4, 2, 51, 32, 170, 215, 1, 49, 148, - 223, 3, 3, 52, 32, 76, 137, 143, 1, 3, 55, 32, 78, 2, 159, 4, 32, 2, 155, - 145, 16, 32, 2, 215, 170, 17, 71, 2, 247, 144, 13, 78, 2, 237, 131, 16, - 2, 32, 77, 20, 196, 1, 2, 50, 32, 22, 54, 238, 25, 51, 168, 2, 3, 55, 32, - 78, 140, 216, 1, 2, 52, 32, 234, 150, 3, 56, 248, 144, 5, 3, 48, 32, 78, - 172, 89, 3, 49, 32, 87, 158, 13, 57, 225, 227, 4, 3, 53, 32, 75, 2, 203, - 168, 17, 77, 2, 247, 151, 10, 32, 20, 196, 1, 3, 52, 32, 75, 20, 2, 53, - 32, 22, 57, 232, 7, 3, 49, 32, 71, 234, 1, 51, 236, 6, 3, 48, 32, 71, - 146, 5, 50, 236, 224, 10, 2, 55, 32, 252, 149, 4, 3, 54, 32, 75, 241, 87, - 3, 56, 32, 70, 2, 155, 175, 17, 80, 2, 155, 189, 17, 70, 2, 187, 243, 10, - 32, 20, 228, 1, 2, 48, 32, 20, 2, 49, 32, 20, 2, 52, 32, 22, 53, 172, 14, - 6, 54, 32, 76, 79, 78, 71, 224, 11, 2, 57, 32, 168, 173, 3, 3, 51, 32, - 72, 168, 154, 9, 4, 50, 32, 78, 71, 152, 239, 3, 3, 55, 32, 72, 189, 97, - 3, 56, 32, 70, 2, 139, 187, 17, 89, 2, 147, 226, 15, 80, 2, 255, 225, 15, - 76, 2, 207, 212, 16, 32, 20, 230, 1, 53, 216, 14, 4, 48, 32, 78, 71, 244, - 7, 3, 51, 32, 71, 192, 152, 12, 2, 55, 32, 138, 118, 57, 192, 112, 3, 50, - 32, 75, 156, 202, 1, 3, 49, 32, 84, 208, 27, 4, 56, 32, 78, 89, 252, 131, - 1, 3, 52, 32, 77, 225, 34, 2, 54, 32, 2, 207, 159, 17, 32, 200, 1, 118, - 48, 186, 2, 49, 210, 2, 50, 166, 2, 51, 222, 1, 52, 186, 2, 53, 214, 2, - 54, 166, 2, 55, 190, 2, 56, 223, 2, 57, 20, 222, 1, 49, 30, 52, 28, 7, - 53, 32, 76, 79, 78, 71, 32, 12, 2, 51, 32, 184, 11, 6, 54, 32, 76, 79, - 78, 71, 138, 241, 4, 50, 198, 168, 2, 48, 212, 134, 5, 3, 55, 32, 71, - 228, 176, 3, 3, 57, 32, 89, 253, 39, 3, 56, 32, 75, 2, 181, 130, 16, 2, - 32, 70, 2, 241, 132, 15, 2, 32, 84, 2, 11, 77, 2, 203, 132, 15, 66, 20, - 208, 1, 6, 48, 32, 76, 79, 78, 71, 22, 51, 34, 54, 22, 56, 32, 2, 57, 32, - 248, 17, 4, 53, 32, 78, 71, 204, 208, 2, 2, 55, 32, 184, 151, 2, 3, 50, - 32, 75, 128, 136, 10, 3, 52, 32, 87, 229, 132, 2, 2, 49, 32, 2, 227, 165, - 16, 32, 2, 11, 32, 2, 151, 179, 17, 74, 2, 203, 192, 16, 32, 2, 11, 32, - 2, 227, 178, 17, 87, 2, 151, 185, 15, 78, 20, 226, 1, 50, 32, 2, 51, 32, - 128, 4, 3, 52, 32, 71, 254, 9, 48, 148, 240, 9, 4, 56, 32, 72, 79, 156, - 220, 2, 5, 55, 32, 78, 71, 71, 224, 88, 2, 53, 32, 220, 55, 2, 57, 32, - 236, 236, 1, 3, 54, 32, 87, 245, 150, 1, 2, 49, 32, 2, 11, 32, 2, 171, - 155, 13, 77, 2, 11, 78, 2, 167, 179, 17, 68, 20, 200, 3, 2, 56, 32, 252, - 4, 3, 52, 32, 78, 216, 243, 4, 3, 50, 32, 75, 210, 210, 2, 49, 2, 53, - 236, 174, 2, 2, 55, 32, 232, 129, 5, 3, 51, 32, 70, 0, 3, 54, 32, 83, - 236, 53, 2, 48, 32, 233, 148, 1, 3, 57, 32, 87, 20, 236, 1, 8, 50, 32, + 54, 73, 0, 3, 76, 65, 73, 0, 3, 77, 73, 84, 166, 130, 13, 72, 166, 10, + 82, 2, 87, 248, 186, 2, 2, 65, 84, 250, 223, 1, 89, 246, 8, 85, 226, 79, + 69, 3, 79, 4, 146, 198, 16, 72, 147, 189, 1, 65, 4, 174, 247, 16, 73, + 211, 139, 1, 65, 8, 214, 171, 10, 72, 182, 203, 6, 73, 147, 68, 68, 4, + 166, 171, 10, 72, 239, 211, 7, 79, 4, 246, 196, 16, 72, 195, 49, 73, 6, + 216, 1, 2, 79, 75, 139, 169, 10, 72, 12, 178, 1, 65, 0, 3, 71, 79, 85, + 214, 253, 17, 78, 3, 89, 6, 118, 65, 255, 194, 16, 72, 6, 142, 240, 17, + 65, 162, 14, 72, 3, 83, 10, 48, 2, 73, 76, 138, 169, 10, 72, 199, 143, 7, + 84, 5, 225, 177, 1, 4, 32, 76, 79, 78, 2, 193, 252, 17, 3, 32, 73, 89, 2, + 157, 150, 17, 7, 32, 82, 69, 80, 69, 84, 73, 30, 64, 10, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 187, 194, 15, 73, 28, 130, 1, 65, 44, 4, 67, 72, + 69, 73, 2, 79, 0, 3, 83, 79, 85, 0, 2, 89, 69, 22, 73, 38, 85, 178, 137, + 11, 78, 211, 185, 4, 86, 8, 198, 234, 16, 78, 210, 82, 65, 187, 64, 85, + 2, 155, 234, 16, 78, 4, 134, 234, 16, 78, 139, 147, 1, 73, 4, 226, 233, + 16, 78, 139, 147, 1, 85, 6, 148, 225, 16, 4, 80, 79, 77, 69, 146, 19, 84, + 195, 56, 79, 178, 3, 160, 1, 11, 68, 69, 32, 75, 73, 75, 65, 75, 85, 73, + 32, 212, 228, 16, 20, 79, 82, 65, 72, 32, 87, 73, 84, 72, 32, 78, 73, 78, + 69, 32, 66, 82, 65, 78, 67, 79, 83, 174, 3, 148, 1, 17, 67, 79, 77, 66, + 73, 78, 73, 78, 71, 32, 78, 85, 77, 66, 69, 82, 32, 164, 1, 10, 83, 89, + 76, 76, 65, 66, 76, 69, 32, 77, 215, 172, 9, 68, 14, 62, 84, 56, 7, 72, + 85, 78, 68, 82, 69, 68, 147, 186, 4, 77, 8, 26, 69, 219, 186, 4, 72, 6, + 26, 78, 167, 177, 14, 69, 4, 172, 186, 4, 2, 32, 84, 163, 190, 13, 83, + 142, 3, 22, 48, 143, 21, 49, 198, 1, 118, 48, 250, 1, 49, 210, 1, 50, + 138, 2, 51, 254, 1, 52, 130, 2, 53, 158, 2, 54, 242, 1, 55, 134, 2, 56, + 187, 2, 57, 18, 142, 1, 49, 34, 50, 22, 51, 22, 52, 162, 151, 2, 53, 162, + 241, 3, 56, 196, 236, 10, 3, 57, 32, 77, 92, 3, 55, 32, 77, 237, 90, 3, + 54, 32, 87, 2, 11, 32, 2, 151, 226, 17, 75, 2, 243, 236, 17, 32, 2, 179, + 159, 12, 32, 2, 11, 32, 2, 207, 225, 17, 87, 20, 130, 1, 49, 22, 54, 18, + 56, 22, 57, 228, 28, 2, 48, 32, 162, 188, 7, 53, 218, 209, 5, 52, 198, + 10, 55, 222, 9, 50, 207, 244, 3, 51, 2, 231, 198, 17, 32, 2, 147, 25, 32, + 2, 215, 214, 13, 32, 2, 227, 128, 13, 32, 20, 106, 49, 22, 50, 22, 51, + 22, 52, 22, 53, 22, 54, 22, 55, 22, 57, 142, 229, 11, 48, 185, 236, 1, 2, + 56, 32, 2, 147, 175, 10, 32, 2, 195, 134, 10, 32, 2, 227, 205, 17, 32, 2, + 211, 153, 12, 32, 2, 167, 238, 12, 32, 2, 219, 207, 17, 32, 2, 179, 232, + 12, 32, 2, 219, 28, 32, 20, 174, 1, 48, 16, 2, 49, 32, 20, 2, 52, 32, 20, + 2, 56, 32, 130, 195, 4, 57, 214, 51, 50, 22, 53, 220, 240, 7, 2, 54, 32, + 188, 136, 4, 3, 55, 32, 78, 237, 90, 3, 51, 32, 89, 2, 159, 37, 32, 2, + 163, 220, 17, 89, 2, 143, 220, 17, 70, 2, 183, 163, 16, 78, 20, 192, 1, + 2, 48, 32, 22, 53, 22, 56, 160, 8, 3, 52, 32, 75, 128, 22, 3, 54, 32, 72, + 222, 2, 55, 202, 213, 4, 49, 224, 184, 4, 3, 57, 32, 87, 156, 174, 4, 2, + 51, 32, 157, 197, 1, 3, 50, 32, 72, 2, 219, 251, 15, 72, 2, 175, 217, 15, + 32, 2, 211, 206, 17, 32, 20, 178, 1, 48, 18, 53, 20, 2, 54, 32, 20, 2, + 56, 32, 22, 57, 232, 218, 9, 2, 50, 32, 236, 4, 2, 51, 32, 210, 214, 1, + 49, 164, 234, 3, 3, 52, 32, 76, 197, 144, 1, 3, 55, 32, 78, 2, 159, 4, + 32, 2, 231, 162, 16, 32, 2, 191, 190, 17, 71, 2, 131, 156, 13, 78, 2, + 185, 149, 16, 2, 32, 77, 20, 196, 1, 2, 50, 32, 22, 54, 238, 25, 51, 168, + 2, 3, 55, 32, 78, 200, 216, 1, 2, 52, 32, 234, 150, 3, 56, 144, 150, 5, + 3, 48, 32, 78, 192, 89, 3, 49, 32, 87, 158, 13, 57, 249, 238, 4, 3, 53, + 32, 75, 2, 179, 188, 17, 77, 2, 207, 157, 10, 32, 20, 196, 1, 3, 52, 32, + 75, 20, 2, 53, 32, 22, 57, 232, 7, 3, 49, 32, 71, 234, 1, 51, 236, 6, 3, + 48, 32, 71, 146, 5, 50, 212, 230, 10, 2, 55, 32, 172, 160, 4, 3, 54, 32, + 75, 217, 88, 3, 56, 32, 70, 2, 131, 195, 17, 80, 2, 131, 209, 17, 70, 2, + 163, 249, 10, 32, 20, 228, 1, 2, 48, 32, 20, 2, 49, 32, 20, 2, 52, 32, + 22, 53, 172, 14, 6, 54, 32, 76, 79, 78, 71, 224, 11, 2, 57, 32, 228, 173, + 3, 3, 51, 32, 72, 128, 165, 9, 4, 50, 32, 78, 71, 236, 247, 3, 3, 55, 32, + 72, 189, 97, 3, 56, 32, 70, 2, 243, 206, 17, 89, 2, 147, 243, 15, 80, 2, + 255, 242, 15, 76, 2, 183, 232, 16, 32, 20, 230, 1, 53, 216, 14, 4, 48, + 32, 78, 71, 244, 7, 3, 51, 32, 71, 220, 163, 12, 2, 55, 32, 250, 117, 57, + 200, 117, 3, 50, 32, 75, 136, 203, 1, 3, 49, 32, 84, 156, 28, 4, 56, 32, + 78, 89, 152, 134, 1, 3, 52, 32, 77, 225, 34, 2, 54, 32, 2, 183, 179, 17, + 32, 200, 1, 118, 48, 186, 2, 49, 210, 2, 50, 166, 2, 51, 222, 1, 52, 186, + 2, 53, 214, 2, 54, 166, 2, 55, 190, 2, 56, 223, 2, 57, 20, 222, 1, 49, + 30, 52, 28, 7, 53, 32, 76, 79, 78, 71, 32, 12, 2, 51, 32, 184, 11, 6, 54, + 32, 76, 79, 78, 71, 202, 241, 4, 50, 170, 170, 2, 48, 204, 143, 5, 3, 55, + 32, 71, 200, 182, 3, 3, 57, 32, 89, 201, 40, 3, 56, 32, 75, 2, 129, 148, + 16, 2, 32, 70, 2, 137, 149, 15, 2, 32, 84, 2, 11, 77, 2, 227, 148, 15, + 66, 20, 208, 1, 6, 48, 32, 76, 79, 78, 71, 22, 51, 34, 54, 22, 56, 32, 2, + 57, 32, 248, 17, 4, 53, 32, 78, 71, 128, 209, 2, 2, 55, 32, 196, 151, 2, + 3, 50, 32, 75, 216, 151, 10, 3, 52, 32, 87, 181, 136, 2, 2, 49, 32, 2, + 183, 183, 16, 32, 2, 11, 32, 2, 255, 198, 17, 74, 2, 155, 210, 16, 32, 2, + 11, 32, 2, 203, 198, 17, 87, 2, 151, 202, 15, 78, 20, 226, 1, 50, 32, 2, + 51, 32, 128, 4, 3, 52, 32, 71, 254, 9, 48, 244, 245, 9, 4, 56, 32, 72, + 79, 208, 225, 2, 5, 55, 32, 78, 71, 71, 148, 88, 2, 53, 32, 168, 61, 2, + 57, 32, 216, 237, 1, 3, 54, 32, 87, 221, 153, 1, 2, 49, 32, 2, 11, 32, 2, + 183, 166, 13, 77, 2, 11, 78, 2, 143, 199, 17, 68, 20, 200, 3, 2, 56, 32, + 252, 4, 3, 52, 32, 78, 148, 244, 4, 3, 50, 32, 75, 250, 215, 2, 49, 2, + 53, 232, 174, 2, 2, 55, 32, 160, 140, 5, 3, 51, 32, 70, 0, 3, 54, 32, 83, + 224, 53, 2, 48, 32, 197, 152, 1, 3, 57, 32, 87, 20, 236, 1, 8, 50, 32, 76, 79, 78, 71, 32, 77, 20, 3, 53, 32, 77, 22, 54, 224, 2, 3, 57, 32, 78, - 174, 219, 6, 55, 220, 155, 3, 3, 51, 32, 87, 236, 224, 2, 2, 48, 32, 176, - 60, 3, 56, 32, 71, 204, 228, 1, 3, 49, 32, 89, 1, 3, 52, 32, 86, 2, 171, - 164, 17, 66, 2, 179, 175, 17, 66, 2, 213, 141, 16, 3, 32, 78, 71, 20, + 186, 220, 6, 55, 176, 160, 3, 3, 51, 32, 87, 160, 230, 2, 2, 48, 32, 168, + 60, 3, 56, 32, 71, 216, 233, 1, 3, 49, 32, 89, 1, 3, 52, 32, 86, 2, 147, + 184, 17, 66, 2, 155, 195, 17, 66, 2, 169, 159, 16, 3, 32, 78, 71, 20, 192, 1, 2, 50, 32, 34, 52, 26, 53, 34, 54, 36, 2, 55, 32, 188, 7, 2, 48, - 32, 244, 250, 9, 3, 56, 32, 75, 128, 193, 2, 2, 49, 32, 156, 138, 3, 5, - 57, 32, 78, 71, 71, 253, 207, 1, 2, 51, 32, 2, 11, 78, 2, 207, 190, 17, - 74, 2, 165, 5, 2, 32, 77, 2, 11, 32, 2, 247, 172, 17, 71, 2, 145, 249, - 14, 4, 32, 78, 71, 71, 2, 223, 208, 15, 74, 20, 220, 1, 2, 48, 32, 20, 6, - 49, 32, 76, 79, 78, 71, 30, 56, 156, 2, 2, 55, 32, 220, 192, 9, 3, 52, - 32, 78, 236, 49, 4, 54, 32, 71, 85, 232, 129, 5, 2, 53, 32, 240, 87, 3, - 50, 32, 83, 0, 2, 51, 32, 165, 101, 2, 57, 32, 2, 251, 246, 14, 74, 2, - 241, 157, 12, 2, 32, 77, 2, 171, 206, 12, 32, 24, 198, 1, 50, 2, 52, 36, - 6, 53, 32, 76, 79, 78, 71, 28, 3, 55, 32, 78, 34, 56, 184, 236, 11, 2, - 54, 32, 160, 98, 3, 57, 32, 75, 168, 253, 2, 3, 51, 32, 86, 136, 111, 4, - 48, 32, 78, 89, 219, 53, 49, 4, 181, 246, 14, 4, 32, 77, 66, 79, 2, 253, - 156, 17, 2, 32, 74, 2, 11, 71, 2, 203, 241, 15, 85, 2, 207, 231, 15, 32, + 32, 204, 128, 10, 3, 56, 32, 75, 188, 198, 2, 2, 49, 32, 136, 144, 3, 5, + 57, 32, 78, 71, 71, 229, 210, 1, 2, 51, 32, 2, 11, 78, 2, 183, 210, 17, + 74, 2, 165, 5, 2, 32, 77, 2, 11, 32, 2, 223, 192, 17, 71, 2, 169, 137, + 15, 4, 32, 78, 71, 71, 2, 223, 225, 15, 74, 20, 220, 1, 2, 48, 32, 20, 6, + 49, 32, 76, 79, 78, 71, 30, 56, 156, 2, 2, 55, 32, 188, 198, 9, 3, 52, + 32, 78, 236, 49, 4, 54, 32, 71, 85, 160, 140, 5, 2, 53, 32, 216, 88, 3, + 50, 32, 83, 0, 2, 51, 32, 241, 101, 2, 57, 32, 2, 147, 135, 15, 74, 2, + 141, 169, 12, 2, 32, 77, 2, 191, 217, 12, 32, 24, 198, 1, 50, 2, 52, 36, + 6, 53, 32, 76, 79, 78, 71, 28, 3, 55, 32, 78, 34, 56, 168, 247, 11, 2, + 54, 32, 196, 98, 3, 57, 32, 75, 148, 131, 3, 3, 51, 32, 86, 240, 113, 4, + 48, 32, 78, 89, 219, 53, 49, 4, 205, 134, 15, 4, 32, 77, 66, 79, 2, 229, + 176, 17, 2, 32, 74, 2, 11, 71, 2, 151, 131, 16, 85, 2, 155, 249, 15, 32, 20, 212, 1, 2, 48, 32, 22, 49, 22, 50, 20, 6, 51, 32, 76, 79, 78, 71, 34, - 56, 216, 136, 9, 2, 53, 32, 252, 128, 1, 3, 52, 32, 78, 168, 193, 2, 2, - 54, 32, 236, 251, 2, 4, 55, 32, 77, 66, 161, 30, 4, 57, 32, 77, 85, 2, - 163, 242, 14, 68, 2, 139, 136, 10, 32, 2, 239, 216, 10, 32, 2, 165, 172, - 15, 3, 32, 78, 71, 2, 17, 2, 32, 77, 2, 163, 201, 15, 66, 16, 142, 1, 48, + 56, 168, 142, 9, 2, 53, 32, 128, 129, 1, 3, 52, 32, 78, 232, 198, 2, 2, + 54, 32, 216, 129, 3, 4, 55, 32, 77, 66, 237, 30, 4, 57, 32, 77, 85, 2, + 187, 130, 15, 68, 2, 223, 141, 10, 32, 2, 215, 222, 10, 32, 2, 165, 189, + 15, 3, 32, 78, 71, 2, 17, 2, 32, 77, 2, 163, 218, 15, 66, 16, 142, 1, 48, 32, 3, 49, 32, 78, 20, 3, 50, 32, 78, 20, 2, 51, 32, 20, 3, 52, 32, 87, - 22, 53, 20, 2, 54, 32, 213, 201, 12, 3, 55, 32, 70, 2, 11, 32, 2, 243, - 199, 15, 71, 2, 223, 199, 15, 68, 2, 155, 146, 17, 74, 2, 131, 231, 16, - 72, 2, 131, 163, 17, 85, 2, 199, 236, 15, 32, 2, 255, 144, 1, 83, 248, 1, - 72, 6, 79, 73, 84, 73, 67, 32, 204, 254, 10, 2, 67, 85, 179, 183, 2, 80, + 22, 53, 20, 2, 54, 32, 233, 212, 12, 3, 55, 32, 70, 2, 11, 32, 2, 243, + 216, 15, 71, 2, 223, 216, 15, 68, 2, 131, 166, 17, 74, 2, 235, 250, 16, + 72, 2, 235, 182, 17, 85, 2, 147, 254, 15, 32, 2, 163, 145, 1, 83, 248, 1, + 72, 6, 79, 73, 84, 73, 67, 32, 212, 131, 11, 2, 67, 85, 191, 193, 2, 80, 244, 1, 104, 8, 67, 85, 82, 83, 73, 86, 69, 32, 221, 10, 13, 72, 73, 69, 82, 79, 71, 76, 89, 80, 72, 73, 67, 32, 180, 1, 96, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 250, 2, 76, 133, 3, 7, 78, 85, 77, 66, 69, 82, 32, 24, 78, 69, 50, 70, 44, 4, 79, 78, 69, 32, 46, 83, 50, 84, 61, 3, 78, 73, 78, 4, 160, 1, 2, 76, 69, 93, 4, 73, 71, 72, 84, 4, 192, 1, 2, 73, 86, - 13, 3, 79, 85, 82, 4, 140, 175, 15, 4, 84, 87, 69, 76, 19, 72, 4, 26, 69, + 13, 3, 79, 85, 82, 4, 140, 192, 15, 4, 84, 87, 69, 76, 19, 72, 4, 26, 69, 93, 2, 73, 88, 2, 65, 2, 86, 69, 6, 46, 69, 12, 3, 72, 82, 69, 13, 2, 87, - 79, 2, 23, 78, 2, 11, 69, 2, 237, 176, 15, 5, 32, 84, 87, 69, 76, 52, 76, + 79, 2, 23, 78, 2, 11, 69, 2, 237, 193, 15, 5, 32, 84, 87, 69, 76, 52, 76, 6, 69, 84, 84, 69, 82, 32, 137, 2, 8, 79, 71, 79, 71, 82, 65, 77, 32, 48, - 182, 1, 65, 46, 84, 182, 235, 1, 78, 2, 83, 218, 200, 13, 72, 130, 179, + 182, 1, 65, 46, 84, 238, 235, 1, 78, 2, 83, 162, 217, 13, 72, 234, 181, 1, 75, 138, 69, 66, 2, 68, 2, 76, 2, 77, 2, 80, 2, 81, 2, 82, 2, 87, 2, - 89, 186, 2, 69, 2, 73, 3, 79, 5, 253, 176, 11, 6, 82, 67, 72, 65, 73, 67, - 6, 202, 174, 17, 65, 2, 69, 3, 79, 4, 138, 209, 4, 73, 241, 192, 12, 2, + 89, 186, 2, 69, 2, 73, 3, 79, 5, 157, 182, 11, 6, 82, 67, 72, 65, 73, 67, + 6, 178, 194, 17, 65, 2, 69, 3, 79, 4, 202, 209, 4, 73, 153, 212, 12, 2, 82, 77, 104, 92, 5, 69, 73, 71, 72, 84, 30, 70, 92, 4, 78, 73, 78, 69, 54, 83, 78, 84, 73, 2, 79, 78, 11, 150, 1, 89, 223, 1, 32, 24, 18, 73, - 35, 79, 12, 142, 2, 86, 207, 235, 3, 70, 12, 148, 2, 2, 85, 82, 167, 235, - 3, 82, 11, 28, 2, 84, 89, 223, 1, 32, 2, 203, 160, 6, 32, 24, 40, 4, 69, - 86, 69, 78, 1, 2, 73, 88, 13, 154, 1, 32, 167, 235, 3, 84, 28, 34, 72, - 50, 87, 131, 159, 6, 69, 12, 32, 2, 82, 69, 155, 235, 3, 73, 8, 39, 69, - 12, 26, 79, 155, 235, 3, 69, 9, 11, 32, 6, 166, 157, 6, 72, 235, 185, 5, - 84, 64, 96, 7, 76, 69, 84, 84, 69, 82, 32, 141, 167, 3, 11, 83, 89, 77, + 35, 79, 12, 142, 2, 86, 163, 236, 3, 70, 12, 148, 2, 2, 85, 82, 251, 235, + 3, 82, 11, 28, 2, 84, 89, 223, 1, 32, 2, 215, 161, 6, 32, 24, 40, 4, 69, + 86, 69, 78, 1, 2, 73, 88, 13, 154, 1, 32, 251, 235, 3, 84, 28, 34, 72, + 50, 87, 143, 160, 6, 69, 12, 32, 2, 82, 69, 239, 235, 3, 73, 8, 39, 69, + 12, 26, 79, 239, 235, 3, 69, 9, 11, 32, 6, 178, 158, 6, 72, 207, 195, 5, + 84, 64, 96, 7, 76, 69, 84, 84, 69, 82, 32, 201, 167, 3, 11, 83, 89, 77, 66, 79, 76, 32, 86, 73, 68, 74, 60, 174, 1, 66, 2, 82, 22, 78, 30, 83, - 38, 84, 222, 172, 15, 72, 130, 179, 1, 75, 138, 69, 68, 2, 76, 2, 77, 2, - 80, 2, 81, 2, 87, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 79, 4, 219, 165, 3, - 65, 8, 198, 165, 3, 65, 3, 69, 6, 170, 165, 3, 65, 151, 130, 14, 69, 10, - 134, 165, 3, 65, 2, 69, 151, 130, 14, 79, 2, 215, 160, 12, 73, 20, 44, 5, - 73, 67, 65, 76, 32, 175, 166, 17, 79, 18, 116, 10, 76, 79, 78, 71, 32, - 79, 86, 69, 82, 32, 74, 80, 26, 84, 168, 1, 7, 83, 72, 79, 82, 84, 32, - 79, 215, 32, 66, 4, 156, 246, 2, 2, 83, 72, 201, 237, 10, 7, 84, 87, 79, - 32, 83, 72, 79, 2, 109, 3, 69, 78, 84, 8, 66, 69, 34, 82, 41, 10, 87, 79, - 32, 83, 72, 79, 82, 84, 83, 32, 2, 17, 2, 84, 82, 2, 23, 65, 2, 11, 73, - 2, 245, 139, 16, 2, 83, 69, 4, 26, 79, 163, 254, 7, 74, 2, 129, 190, 10, - 4, 86, 69, 82, 32, 230, 2, 104, 3, 65, 79, 32, 136, 18, 2, 67, 82, 142, - 1, 68, 142, 1, 76, 130, 1, 78, 245, 2, 4, 82, 82, 79, 82, 170, 2, 156, 1, - 7, 76, 69, 84, 84, 69, 82, 32, 180, 10, 5, 83, 73, 71, 78, 32, 212, 1, 5, - 84, 79, 78, 69, 32, 69, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, - 178, 1, 214, 1, 65, 110, 66, 34, 68, 106, 71, 34, 76, 50, 78, 106, 82, - 170, 1, 83, 54, 84, 218, 1, 86, 32, 3, 89, 73, 32, 118, 90, 246, 165, 13, - 81, 254, 231, 1, 80, 246, 193, 1, 72, 2, 77, 138, 69, 70, 2, 75, 2, 87, - 3, 88, 10, 52, 7, 82, 67, 72, 65, 73, 67, 32, 227, 158, 17, 72, 8, 210, - 254, 8, 90, 222, 174, 4, 78, 251, 238, 3, 77, 4, 210, 138, 17, 82, 219, - 19, 65, 16, 50, 90, 154, 4, 76, 138, 151, 17, 68, 187, 2, 65, 8, 254, - 137, 17, 89, 162, 17, 72, 2, 90, 187, 2, 65, 6, 226, 213, 16, 72, 195, - 71, 65, 8, 222, 178, 10, 72, 238, 231, 6, 89, 187, 2, 65, 18, 54, 65, - 222, 212, 16, 71, 2, 78, 2, 89, 139, 69, 72, 5, 11, 83, 2, 149, 218, 13, - 4, 65, 76, 73, 90, 16, 60, 9, 69, 70, 79, 82, 77, 69, 68, 32, 84, 219, - 132, 17, 84, 14, 40, 4, 79, 78, 69, 45, 195, 160, 15, 83, 12, 254, 154, - 17, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 56, 8, 234, 134, 17, 89, 162, 17, - 72, 2, 83, 187, 2, 65, 32, 78, 76, 20, 4, 79, 78, 69, 45, 70, 83, 178, - 150, 17, 84, 186, 2, 65, 3, 69, 4, 155, 175, 10, 72, 14, 170, 153, 17, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 8, 174, 150, 17, 72, 2, 83, - 186, 2, 65, 3, 69, 4, 254, 149, 17, 70, 187, 2, 65, 16, 70, 84, 144, 157, - 15, 2, 68, 90, 190, 62, 78, 206, 185, 1, 75, 3, 80, 8, 142, 208, 16, 83, - 138, 69, 84, 187, 2, 65, 16, 50, 90, 178, 207, 16, 83, 138, 69, 72, 187, - 2, 65, 8, 202, 172, 10, 83, 238, 231, 6, 89, 187, 2, 65, 8, 144, 1, 9, - 82, 69, 70, 79, 82, 77, 69, 68, 32, 34, 65, 189, 243, 15, 18, 67, 79, 78, - 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, 70, 73, 69, 82, 4, 30, 65, - 213, 88, 2, 86, 79, 2, 133, 139, 12, 3, 83, 80, 73, 8, 218, 246, 14, 66, - 140, 77, 3, 84, 79, 80, 154, 84, 65, 159, 82, 82, 104, 146, 1, 65, 78, - 69, 62, 73, 78, 79, 74, 85, 74, 89, 160, 145, 7, 9, 82, 79, 85, 78, 68, - 69, 68, 32, 69, 178, 185, 7, 87, 202, 74, 78, 251, 124, 86, 19, 250, 151, - 15, 78, 250, 186, 1, 69, 146, 63, 72, 146, 1, 65, 2, 73, 3, 85, 15, 254, - 191, 13, 82, 178, 215, 1, 78, 154, 251, 1, 65, 3, 73, 23, 230, 150, 15, - 65, 54, 79, 158, 250, 1, 78, 86, 69, 2, 71, 2, 73, 3, 85, 13, 42, 69, - 150, 145, 17, 71, 2, 79, 3, 85, 4, 146, 145, 17, 82, 3, 89, 19, 210, 149, - 15, 65, 206, 231, 1, 69, 134, 19, 78, 2, 79, 86, 73, 3, 85, 7, 214, 252, - 16, 85, 219, 19, 73, 12, 18, 32, 63, 79, 4, 196, 252, 15, 4, 79, 78, 32, - 85, 13, 4, 68, 65, 83, 72, 8, 206, 226, 11, 83, 162, 130, 4, 80, 222, 35, - 32, 163, 112, 66, 12, 64, 4, 68, 76, 69, 32, 189, 223, 4, 6, 76, 73, 78, - 69, 32, 72, 10, 184, 192, 5, 3, 84, 72, 73, 238, 217, 8, 76, 22, 82, 207, - 163, 2, 68, 8, 76, 6, 73, 84, 65, 82, 89, 32, 232, 188, 10, 3, 75, 89, - 32, 195, 201, 5, 76, 4, 26, 72, 199, 170, 12, 77, 2, 227, 177, 4, 69, 24, - 42, 73, 76, 2, 85, 83, 131, 140, 17, 89, 6, 34, 68, 22, 77, 155, 134, 15, - 66, 2, 219, 156, 6, 73, 2, 223, 235, 6, 73, 16, 46, 32, 205, 177, 10, 5, - 45, 79, 82, 45, 80, 14, 40, 4, 83, 73, 71, 78, 203, 167, 13, 84, 13, 11, - 32, 10, 44, 5, 87, 73, 84, 72, 32, 135, 173, 6, 73, 8, 66, 67, 250, 202, - 4, 68, 234, 163, 8, 82, 21, 4, 70, 65, 76, 76, 2, 177, 231, 11, 3, 79, - 77, 77, 5, 171, 130, 15, 32, 184, 9, 184, 1, 10, 66, 73, 76, 69, 32, 80, - 72, 79, 78, 69, 166, 1, 68, 190, 76, 78, 170, 27, 79, 172, 1, 3, 83, 81, - 85, 38, 84, 250, 1, 85, 242, 176, 11, 89, 209, 219, 2, 5, 86, 73, 69, 32, - 67, 7, 11, 32, 4, 108, 21, 87, 73, 84, 72, 32, 82, 73, 71, 72, 84, 87, - 65, 82, 68, 83, 32, 65, 82, 82, 79, 87, 167, 178, 1, 79, 2, 11, 32, 2, - 225, 212, 16, 2, 65, 84, 154, 6, 58, 69, 74, 73, 149, 75, 7, 85, 76, 79, - 32, 84, 87, 79, 4, 208, 182, 16, 10, 82, 78, 32, 80, 69, 78, 84, 65, 84, - 72, 159, 18, 76, 148, 6, 42, 32, 221, 1, 5, 70, 73, 69, 82, 32, 158, 1, - 88, 5, 83, 73, 71, 78, 32, 230, 179, 3, 86, 190, 162, 3, 68, 186, 1, 76, - 195, 194, 4, 65, 10, 42, 65, 166, 212, 8, 72, 155, 237, 7, 86, 4, 44, 5, - 82, 68, 72, 65, 67, 175, 191, 16, 78, 2, 213, 191, 16, 3, 65, 78, 68, - 246, 4, 92, 12, 66, 82, 69, 86, 69, 32, 87, 73, 84, 72, 32, 73, 89, 7, - 76, 69, 84, 84, 69, 82, 32, 2, 33, 6, 78, 86, 69, 82, 84, 69, 2, 21, 3, - 68, 32, 66, 2, 129, 134, 16, 2, 82, 69, 244, 4, 198, 1, 65, 82, 66, 66, - 67, 190, 13, 68, 202, 1, 69, 182, 2, 71, 82, 72, 46, 76, 230, 4, 77, 216, - 3, 7, 79, 80, 69, 78, 32, 83, 72, 22, 80, 38, 82, 182, 4, 83, 206, 33, - 84, 114, 85, 118, 86, 63, 89, 6, 38, 76, 158, 27, 67, 251, 204, 10, 80, - 2, 245, 19, 6, 86, 69, 79, 76, 65, 82, 6, 184, 19, 5, 73, 76, 65, 66, 73, - 193, 45, 4, 69, 71, 73, 78, 160, 1, 240, 1, 7, 65, 80, 73, 84, 65, 76, - 32, 192, 2, 7, 69, 78, 84, 82, 69, 68, 32, 100, 13, 72, 73, 78, 69, 83, - 69, 32, 84, 79, 78, 69, 32, 89, 108, 8, 89, 82, 73, 76, 76, 73, 67, 32, - 242, 225, 10, 73, 244, 2, 4, 82, 79, 83, 83, 255, 196, 5, 79, 56, 206, 1, - 66, 42, 82, 130, 27, 72, 202, 141, 13, 79, 138, 147, 3, 65, 162, 64, 67, - 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, - 80, 2, 81, 2, 84, 2, 85, 2, 86, 3, 87, 5, 133, 150, 6, 5, 65, 82, 82, 69, - 68, 7, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 194, 251, 16, 69, 3, 78, - 4, 30, 76, 21, 3, 82, 73, 71, 2, 29, 2, 69, 70, 2, 11, 72, 2, 11, 84, 2, - 181, 26, 2, 32, 72, 16, 36, 3, 65, 78, 71, 1, 2, 73, 78, 8, 11, 32, 8, - 182, 134, 12, 83, 214, 162, 4, 80, 158, 44, 81, 3, 82, 78, 34, 72, 30, - 83, 187, 169, 16, 69, 2, 217, 144, 15, 2, 65, 82, 74, 40, 5, 77, 65, 76, - 76, 32, 183, 6, 79, 72, 186, 1, 66, 138, 1, 68, 38, 69, 150, 1, 80, 58, - 83, 94, 84, 34, 89, 158, 243, 15, 90, 130, 64, 73, 150, 19, 67, 2, 71, - 186, 22, 74, 2, 86, 158, 20, 72, 2, 75, 186, 2, 65, 2, 79, 3, 85, 6, 38, - 89, 198, 27, 65, 199, 219, 16, 69, 2, 221, 202, 1, 19, 69, 76, 79, 82, - 85, 83, 83, 73, 65, 78, 45, 85, 75, 82, 65, 73, 78, 73, 65, 4, 242, 213, - 6, 90, 183, 160, 10, 69, 15, 50, 83, 210, 245, 16, 70, 2, 76, 2, 77, 3, - 82, 5, 25, 4, 32, 87, 73, 84, 2, 21, 3, 72, 32, 68, 2, 11, 69, 2, 177, - 141, 13, 3, 83, 67, 69, 4, 26, 65, 215, 244, 16, 69, 2, 137, 200, 16, 2, - 76, 79, 8, 42, 84, 206, 173, 1, 67, 139, 196, 15, 72, 4, 153, 19, 8, 82, - 65, 73, 71, 72, 84, 32, 85, 4, 134, 221, 16, 83, 215, 22, 69, 6, 36, 3, - 69, 82, 85, 151, 243, 16, 85, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, - 21, 3, 65, 67, 75, 2, 229, 180, 16, 2, 32, 89, 2, 227, 175, 11, 70, 16, - 18, 69, 27, 79, 2, 169, 5, 2, 78, 84, 14, 64, 2, 84, 32, 52, 5, 85, 66, - 76, 69, 32, 129, 52, 2, 87, 78, 6, 242, 218, 9, 83, 142, 192, 4, 86, 171, - 180, 1, 72, 4, 194, 137, 2, 65, 219, 247, 2, 80, 24, 40, 5, 88, 84, 82, - 65, 45, 131, 49, 78, 20, 52, 5, 72, 73, 71, 72, 32, 81, 4, 76, 79, 87, - 32, 10, 156, 1, 9, 69, 88, 84, 82, 65, 45, 76, 79, 87, 154, 7, 68, 70, - 76, 79, 84, 10, 76, 10, 69, 88, 84, 82, 65, 45, 72, 73, 71, 72, 154, 7, - 68, 70, 76, 79, 84, 2, 145, 8, 8, 32, 67, 79, 78, 84, 79, 85, 82, 8, 162, - 9, 82, 226, 3, 76, 237, 191, 15, 9, 69, 79, 82, 71, 73, 65, 78, 32, 78, - 10, 248, 5, 4, 73, 71, 72, 32, 255, 40, 65, 44, 46, 65, 84, 2, 79, 87, - 201, 11, 2, 69, 70, 2, 21, 3, 84, 69, 82, 2, 17, 2, 65, 76, 2, 17, 2, 32, - 67, 2, 191, 188, 15, 76, 36, 86, 32, 249, 139, 12, 15, 69, 82, 32, 82, - 73, 71, 72, 84, 32, 67, 79, 82, 78, 69, 82, 34, 158, 1, 67, 20, 2, 68, - 79, 40, 4, 76, 69, 70, 84, 82, 77, 12, 2, 82, 73, 50, 84, 178, 3, 65, 42, - 71, 170, 2, 73, 184, 141, 14, 2, 85, 80, 179, 187, 1, 86, 2, 199, 210, - 10, 73, 6, 238, 2, 84, 253, 145, 14, 2, 87, 78, 6, 28, 2, 32, 65, 247, 2, - 45, 4, 25, 4, 82, 82, 79, 87, 5, 199, 148, 14, 72, 2, 111, 65, 4, 228, - 147, 14, 3, 71, 72, 84, 223, 212, 2, 78, 4, 194, 2, 79, 243, 204, 14, 73, - 18, 18, 65, 23, 73, 2, 147, 195, 15, 67, 16, 26, 68, 195, 185, 13, 78, - 14, 38, 32, 217, 1, 4, 68, 76, 69, 32, 8, 26, 68, 70, 76, 79, 84, 4, 17, - 2, 79, 84, 4, 25, 4, 84, 69, 68, 32, 4, 18, 76, 79, 84, 2, 25, 4, 69, 70, - 84, 45, 2, 25, 4, 83, 84, 69, 77, 2, 17, 2, 32, 84, 2, 11, 79, 2, 11, 78, - 2, 199, 196, 15, 69, 6, 40, 6, 68, 79, 85, 66, 76, 69, 75, 71, 4, 11, 32, - 4, 18, 65, 43, 71, 2, 11, 67, 2, 169, 208, 10, 2, 85, 84, 2, 11, 82, 2, - 247, 207, 10, 65, 2, 207, 225, 14, 69, 4, 150, 182, 13, 76, 207, 150, 2, - 82, 26, 104, 6, 65, 73, 83, 69, 68, 32, 150, 1, 69, 216, 1, 3, 73, 71, - 72, 133, 245, 8, 5, 72, 79, 84, 73, 67, 10, 70, 68, 30, 73, 162, 213, 9, - 69, 244, 242, 5, 2, 85, 80, 211, 74, 67, 2, 157, 146, 15, 2, 79, 87, 2, - 137, 213, 9, 7, 78, 86, 69, 82, 84, 69, 68, 8, 104, 7, 86, 69, 82, 83, - 69, 68, 32, 137, 28, 14, 84, 82, 79, 70, 76, 69, 88, 32, 67, 76, 73, 67, - 75, 32, 6, 26, 71, 175, 143, 14, 67, 4, 11, 76, 4, 49, 10, 79, 84, 84, - 65, 76, 32, 83, 84, 79, 80, 5, 171, 19, 32, 6, 17, 2, 84, 32, 6, 38, 72, - 166, 197, 13, 84, 235, 69, 65, 2, 133, 191, 7, 3, 65, 76, 70, 156, 2, - 138, 1, 72, 48, 5, 77, 65, 76, 76, 32, 148, 31, 8, 84, 82, 69, 83, 83, - 32, 65, 78, 53, 11, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 4, 136, - 153, 8, 3, 79, 82, 84, 203, 194, 6, 69, 144, 2, 154, 2, 65, 50, 66, 130, - 1, 67, 238, 2, 68, 238, 2, 90, 66, 69, 46, 70, 30, 71, 110, 72, 102, 77, - 34, 73, 34, 74, 98, 76, 176, 3, 7, 78, 32, 87, 73, 84, 72, 32, 30, 79, - 94, 80, 22, 82, 146, 2, 83, 190, 1, 84, 246, 7, 85, 142, 1, 86, 158, 192, - 16, 75, 2, 81, 2, 87, 2, 88, 3, 89, 9, 238, 131, 13, 76, 214, 136, 3, 73, - 227, 79, 69, 11, 46, 65, 50, 79, 222, 8, 32, 239, 173, 16, 69, 2, 17, 2, - 82, 82, 2, 193, 246, 5, 2, 69, 68, 2, 229, 20, 4, 84, 84, 79, 77, 41, - 100, 7, 65, 80, 73, 84, 65, 76, 32, 180, 1, 6, 76, 79, 83, 69, 68, 32, - 190, 17, 32, 199, 179, 16, 72, 30, 114, 73, 214, 6, 71, 226, 16, 76, 146, - 171, 16, 79, 158, 20, 65, 186, 2, 66, 2, 72, 2, 78, 2, 82, 2, 85, 3, 89, - 7, 22, 78, 191, 11, 32, 2, 249, 239, 5, 5, 86, 69, 82, 84, 69, 4, 26, 82, - 195, 161, 9, 79, 2, 217, 20, 9, 69, 86, 69, 82, 83, 69, 68, 32, 79, 23, - 104, 6, 32, 87, 73, 84, 72, 32, 86, 69, 32, 9, 79, 84, 76, 69, 83, 83, - 32, 74, 32, 105, 3, 90, 32, 68, 6, 26, 72, 163, 163, 14, 84, 4, 21, 3, - 79, 79, 75, 5, 225, 234, 13, 3, 32, 65, 78, 4, 238, 15, 90, 143, 162, 16, - 76, 4, 33, 6, 87, 73, 84, 72, 32, 83, 4, 29, 5, 84, 82, 79, 75, 69, 5, - 133, 234, 8, 4, 32, 65, 78, 68, 6, 33, 6, 73, 71, 82, 65, 80, 72, 7, 33, - 6, 32, 87, 73, 84, 72, 32, 4, 138, 14, 67, 207, 4, 82, 11, 202, 211, 16, - 83, 2, 84, 2, 90, 63, 78, 5, 225, 13, 3, 69, 78, 71, 11, 56, 5, 82, 69, - 69, 75, 32, 162, 1, 32, 195, 128, 14, 65, 4, 26, 71, 191, 132, 11, 80, 2, - 195, 129, 14, 65, 11, 40, 6, 32, 87, 73, 84, 72, 32, 39, 69, 4, 166, 255, - 14, 72, 219, 163, 1, 83, 4, 17, 2, 78, 71, 5, 11, 32, 2, 195, 230, 8, 87, - 4, 222, 4, 32, 251, 168, 16, 79, 5, 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, - 11, 82, 2, 225, 157, 14, 6, 79, 83, 83, 69, 68, 45, 25, 116, 6, 32, 87, - 73, 84, 72, 32, 226, 9, 83, 2, 90, 112, 2, 69, 90, 253, 174, 16, 8, 73, - 71, 65, 84, 85, 82, 69, 32, 12, 54, 73, 82, 77, 82, 82, 222, 6, 80, 243, - 244, 3, 66, 2, 49, 10, 78, 86, 69, 82, 84, 69, 68, 32, 76, 65, 2, 173, - 218, 12, 2, 90, 89, 2, 11, 73, 2, 17, 2, 68, 68, 2, 17, 2, 76, 69, 2, - 145, 235, 12, 2, 32, 84, 4, 61, 13, 69, 84, 82, 79, 70, 76, 69, 88, 32, - 72, 79, 79, 75, 5, 205, 12, 4, 32, 65, 78, 68, 4, 210, 11, 82, 207, 1, - 76, 9, 18, 32, 47, 80, 2, 21, 3, 87, 73, 84, 2, 151, 157, 16, 72, 4, 193, - 186, 12, 2, 69, 78, 5, 163, 185, 16, 72, 15, 80, 6, 32, 87, 73, 84, 72, - 32, 62, 65, 41, 8, 69, 86, 69, 82, 83, 69, 68, 32, 4, 26, 70, 155, 152, - 14, 84, 2, 165, 224, 8, 3, 73, 83, 72, 2, 17, 2, 77, 83, 2, 131, 232, 5, - 32, 6, 38, 71, 170, 7, 79, 227, 195, 16, 69, 2, 11, 76, 2, 173, 249, 13, - 4, 79, 84, 84, 65, 13, 72, 6, 32, 87, 73, 84, 72, 32, 34, 67, 61, 6, 73, - 68, 69, 87, 65, 89, 4, 158, 3, 67, 155, 243, 14, 72, 4, 26, 82, 203, 216, - 11, 72, 2, 233, 218, 5, 3, 73, 80, 84, 2, 231, 189, 6, 83, 51, 106, 32, - 102, 67, 116, 2, 69, 83, 44, 2, 79, 80, 42, 83, 96, 6, 85, 82, 78, 69, - 68, 32, 231, 139, 13, 72, 4, 29, 5, 87, 73, 84, 72, 32, 4, 22, 80, 219, - 5, 82, 2, 153, 220, 8, 6, 65, 76, 65, 84, 65, 76, 2, 45, 9, 32, 68, 73, - 71, 82, 65, 80, 72, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 67, 2, - 179, 145, 3, 85, 2, 11, 72, 2, 217, 150, 10, 3, 32, 68, 73, 2, 189, 225, - 5, 5, 32, 72, 65, 76, 70, 4, 37, 7, 32, 68, 73, 71, 82, 65, 80, 4, 11, - 72, 5, 11, 32, 2, 141, 3, 4, 87, 73, 84, 72, 32, 86, 65, 38, 77, 74, 79, - 42, 82, 214, 1, 89, 230, 193, 16, 72, 2, 73, 2, 86, 3, 87, 7, 150, 236, - 12, 76, 183, 216, 3, 69, 5, 41, 8, 32, 87, 73, 84, 72, 32, 76, 79, 2, - 213, 141, 4, 2, 78, 71, 2, 11, 80, 2, 225, 156, 6, 2, 69, 78, 9, 33, 6, - 32, 87, 73, 84, 72, 32, 6, 26, 76, 167, 239, 14, 72, 4, 37, 7, 79, 78, - 71, 32, 76, 69, 71, 5, 25, 4, 32, 65, 78, 68, 2, 17, 2, 32, 82, 2, 11, - 69, 2, 181, 214, 8, 7, 84, 82, 79, 70, 76, 69, 88, 5, 29, 5, 32, 87, 73, - 84, 72, 2, 185, 238, 3, 2, 32, 66, 9, 18, 32, 95, 80, 4, 40, 4, 87, 73, - 84, 72, 179, 159, 15, 66, 2, 17, 2, 32, 76, 2, 11, 69, 2, 131, 1, 70, 2, - 189, 240, 15, 2, 83, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 26, 82, - 139, 236, 14, 72, 2, 21, 3, 73, 71, 72, 2, 231, 211, 8, 84, 4, 11, 68, 4, - 11, 32, 4, 130, 210, 1, 72, 35, 76, 4, 24, 2, 72, 65, 31, 84, 2, 25, 4, - 76, 70, 32, 84, 2, 43, 82, 4, 30, 82, 53, 3, 85, 82, 78, 2, 249, 237, 15, - 8, 73, 65, 78, 71, 85, 76, 65, 82, 2, 253, 166, 8, 2, 69, 68, 8, 70, 80, - 212, 154, 10, 7, 78, 65, 83, 80, 73, 82, 65, 175, 162, 6, 83, 4, 11, 32, - 4, 242, 161, 13, 84, 235, 69, 65, 4, 26, 79, 199, 162, 15, 69, 2, 11, 73, - 2, 167, 235, 15, 67, 4, 36, 3, 65, 78, 71, 1, 2, 73, 78, 2, 25, 4, 32, - 68, 69, 80, 2, 21, 3, 65, 82, 84, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 84, - 2, 11, 79, 2, 143, 134, 6, 78, 2, 11, 32, 2, 211, 145, 15, 83, 234, 2, - 92, 2, 69, 89, 88, 7, 71, 79, 76, 73, 65, 78, 32, 206, 24, 79, 177, 130, - 12, 3, 75, 69, 89, 6, 26, 32, 135, 252, 15, 45, 4, 152, 131, 12, 6, 87, - 73, 84, 72, 32, 87, 179, 205, 2, 66, 214, 2, 194, 2, 68, 46, 70, 148, 1, - 14, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 73, 82, 71, 65, 16, 7, 76, - 69, 84, 84, 69, 82, 32, 138, 16, 83, 132, 1, 7, 82, 79, 84, 65, 84, 69, - 68, 22, 66, 98, 84, 160, 2, 4, 86, 79, 87, 69, 154, 241, 3, 67, 164, 1, - 6, 77, 65, 78, 67, 72, 85, 216, 190, 7, 4, 78, 73, 82, 85, 239, 203, 2, - 69, 22, 232, 20, 3, 79, 85, 66, 207, 176, 14, 73, 12, 112, 19, 82, 69, - 69, 32, 86, 65, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 214, - 227, 13, 85, 199, 46, 79, 8, 249, 244, 9, 3, 84, 79, 82, 5, 227, 19, 32, - 136, 2, 130, 2, 65, 244, 4, 2, 67, 72, 88, 2, 77, 65, 174, 2, 83, 246, 1, - 84, 234, 3, 90, 214, 178, 14, 72, 142, 171, 1, 75, 2, 76, 162, 7, 69, 2, - 79, 2, 85, 234, 61, 66, 2, 68, 2, 70, 2, 71, 2, 74, 2, 78, 2, 80, 2, 81, - 2, 82, 2, 87, 2, 89, 187, 2, 73, 59, 56, 8, 76, 73, 32, 71, 65, 76, 73, - 32, 231, 177, 16, 78, 54, 174, 1, 65, 52, 6, 86, 73, 83, 65, 82, 71, 22, - 68, 76, 5, 72, 65, 76, 70, 32, 34, 73, 50, 85, 34, 78, 30, 84, 66, 66, - 234, 230, 15, 80, 2, 90, 254, 68, 83, 14, 67, 3, 75, 7, 48, 6, 78, 85, - 83, 86, 65, 82, 215, 176, 16, 72, 2, 195, 160, 9, 65, 8, 26, 65, 239, - 173, 16, 68, 7, 246, 247, 8, 77, 233, 245, 6, 3, 71, 65, 76, 4, 186, 173, - 16, 89, 187, 2, 85, 5, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 85, 2, 181, - 236, 15, 3, 66, 65, 68, 4, 202, 172, 16, 71, 3, 78, 8, 60, 6, 72, 82, 69, - 69, 32, 66, 234, 230, 15, 84, 195, 71, 65, 2, 17, 2, 65, 76, 2, 143, 137, - 16, 85, 6, 26, 65, 231, 173, 16, 73, 5, 29, 5, 32, 87, 73, 84, 72, 2, - 237, 220, 14, 2, 32, 84, 41, 29, 5, 78, 67, 72, 85, 32, 38, 104, 9, 65, - 76, 73, 32, 71, 65, 76, 73, 32, 222, 177, 14, 90, 138, 248, 1, 70, 2, 75, - 2, 82, 187, 2, 73, 28, 122, 68, 254, 192, 9, 67, 226, 222, 2, 84, 134, - 145, 2, 66, 2, 71, 2, 74, 2, 76, 130, 179, 1, 90, 254, 4, 78, 131, 64, - 83, 4, 222, 176, 14, 68, 139, 248, 1, 72, 48, 52, 4, 73, 66, 69, 32, 142, - 168, 16, 72, 187, 2, 65, 44, 242, 221, 5, 71, 2, 72, 158, 173, 6, 73, - 154, 19, 84, 222, 145, 2, 67, 2, 83, 246, 7, 82, 214, 161, 1, 65, 186, 9, - 90, 162, 7, 85, 234, 61, 68, 2, 70, 2, 74, 2, 75, 2, 80, 187, 2, 69, 60, - 52, 4, 79, 68, 79, 32, 154, 166, 16, 83, 187, 2, 65, 56, 250, 1, 65, 98, - 68, 34, 74, 34, 78, 236, 164, 1, 8, 76, 79, 78, 71, 32, 86, 79, 87, 250, - 179, 4, 71, 182, 192, 6, 84, 222, 145, 2, 67, 246, 7, 72, 174, 178, 1, - 79, 2, 85, 234, 61, 66, 2, 75, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, 186, 2, - 69, 3, 73, 6, 56, 8, 76, 73, 32, 71, 65, 76, 73, 32, 199, 165, 16, 78, 4, - 214, 171, 14, 90, 139, 248, 1, 84, 4, 186, 163, 16, 90, 187, 2, 65, 4, - 154, 163, 16, 73, 187, 2, 65, 2, 251, 162, 16, 73, 6, 198, 145, 16, 72, - 162, 17, 82, 187, 2, 65, 8, 128, 1, 4, 87, 73, 82, 76, 189, 190, 4, 21, - 73, 66, 69, 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 66, 79, 85, 78, 68, - 65, 82, 89, 6, 17, 2, 32, 66, 6, 25, 4, 73, 82, 71, 65, 7, 33, 6, 32, 87, - 73, 84, 72, 32, 4, 150, 2, 68, 187, 221, 15, 79, 6, 152, 1, 3, 82, 73, - 80, 56, 18, 85, 82, 78, 69, 68, 32, 83, 87, 73, 82, 76, 32, 66, 73, 82, - 71, 65, 32, 197, 192, 1, 8, 79, 68, 79, 32, 83, 79, 70, 84, 2, 225, 221, - 15, 9, 76, 69, 32, 66, 73, 82, 71, 65, 32, 2, 33, 6, 87, 73, 84, 72, 32, - 68, 2, 161, 221, 15, 5, 79, 85, 66, 76, 69, 2, 207, 229, 9, 76, 10, 96, - 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 208, 131, 4, 5, 83, 84, 65, 66, - 76, 183, 232, 9, 82, 6, 26, 89, 219, 182, 12, 69, 4, 206, 206, 15, 65, - 155, 1, 73, 10, 48, 2, 78, 32, 230, 209, 4, 68, 191, 182, 11, 83, 6, 88, - 12, 86, 73, 69, 87, 73, 78, 71, 32, 67, 69, 82, 69, 234, 181, 12, 67, 85, - 2, 76, 65, 2, 189, 254, 15, 2, 77, 79, 4, 190, 175, 12, 73, 191, 238, 3, - 69, 10, 26, 72, 69, 2, 79, 82, 2, 45, 9, 69, 82, 32, 67, 72, 82, 73, 83, - 84, 2, 255, 254, 2, 77, 8, 50, 32, 52, 4, 73, 90, 69, 68, 143, 144, 15, - 87, 4, 226, 196, 8, 66, 233, 159, 6, 4, 83, 67, 79, 79, 2, 189, 176, 3, - 7, 32, 87, 72, 69, 69, 76, 67, 18, 52, 2, 78, 84, 152, 1, 2, 83, 69, 131, - 153, 16, 84, 10, 48, 3, 65, 73, 78, 141, 132, 12, 3, 32, 70, 85, 9, 11, - 32, 6, 142, 202, 9, 82, 24, 5, 67, 65, 66, 76, 69, 217, 235, 1, 6, 66, - 73, 67, 89, 67, 76, 7, 11, 32, 4, 148, 137, 15, 2, 84, 82, 231, 83, 70, - 86, 52, 7, 76, 69, 84, 84, 69, 82, 32, 211, 234, 5, 68, 62, 198, 1, 75, - 62, 77, 34, 78, 34, 79, 30, 80, 34, 84, 234, 105, 68, 180, 128, 8, 2, 72, - 65, 2, 82, 134, 192, 1, 69, 134, 29, 83, 154, 89, 76, 246, 7, 67, 130, - 207, 4, 89, 190, 28, 66, 2, 87, 187, 2, 65, 6, 192, 173, 5, 2, 69, 65, - 170, 255, 5, 72, 243, 234, 4, 79, 4, 242, 132, 16, 65, 215, 1, 73, 4, - 158, 199, 15, 73, 139, 60, 71, 7, 222, 150, 16, 76, 3, 79, 4, 234, 130, - 16, 72, 219, 19, 65, 6, 178, 144, 9, 72, 218, 130, 7, 69, 155, 3, 65, - 206, 4, 44, 2, 76, 84, 234, 6, 83, 151, 252, 13, 67, 102, 36, 4, 65, 78, - 73, 32, 219, 2, 73, 76, 52, 7, 76, 69, 84, 84, 69, 82, 32, 147, 148, 4, - 83, 74, 206, 1, 68, 222, 83, 78, 166, 236, 1, 82, 214, 198, 9, 74, 178, - 51, 84, 206, 145, 3, 66, 2, 67, 2, 71, 2, 75, 2, 80, 138, 69, 72, 2, 76, + 38, 84, 222, 189, 15, 72, 234, 181, 1, 75, 138, 69, 68, 2, 76, 2, 77, 2, + 80, 2, 81, 2, 87, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 79, 4, 151, 166, 3, + 65, 8, 130, 166, 3, 65, 3, 69, 6, 230, 165, 3, 65, 195, 149, 14, 69, 10, + 194, 165, 3, 65, 2, 69, 195, 149, 14, 79, 2, 243, 171, 12, 73, 22, 26, + 82, 207, 252, 16, 73, 20, 44, 5, 73, 67, 65, 76, 32, 251, 185, 17, 79, + 18, 116, 10, 76, 79, 78, 71, 32, 79, 86, 69, 82, 32, 74, 80, 26, 84, 168, + 1, 7, 83, 72, 79, 82, 84, 32, 79, 215, 32, 66, 4, 180, 246, 2, 2, 83, 72, + 169, 253, 10, 7, 84, 87, 79, 32, 83, 72, 79, 2, 109, 3, 69, 78, 84, 8, + 66, 69, 34, 82, 41, 10, 87, 79, 32, 83, 72, 79, 82, 84, 83, 32, 2, 17, 2, + 84, 82, 2, 23, 65, 2, 11, 73, 2, 189, 157, 16, 2, 83, 69, 4, 26, 79, 235, + 131, 8, 74, 2, 205, 195, 10, 4, 86, 69, 82, 32, 230, 2, 104, 3, 65, 79, + 32, 136, 18, 2, 67, 82, 142, 1, 68, 142, 1, 76, 130, 1, 78, 245, 2, 4, + 82, 82, 79, 82, 170, 2, 156, 1, 7, 76, 69, 84, 84, 69, 82, 32, 180, 10, + 5, 83, 73, 71, 78, 32, 212, 1, 5, 84, 79, 78, 69, 32, 69, 11, 86, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 178, 1, 214, 1, 65, 110, 66, 34, 68, 106, + 71, 34, 76, 50, 78, 106, 82, 170, 1, 83, 54, 84, 218, 1, 86, 32, 3, 89, + 73, 32, 118, 90, 238, 180, 13, 81, 234, 233, 1, 80, 222, 196, 1, 72, 2, + 77, 138, 69, 70, 2, 75, 2, 87, 3, 88, 10, 52, 7, 82, 67, 72, 65, 73, 67, + 32, 175, 178, 17, 72, 8, 134, 132, 9, 90, 162, 184, 4, 78, 207, 243, 3, + 77, 4, 158, 158, 17, 82, 219, 19, 65, 16, 50, 90, 154, 4, 76, 214, 170, + 17, 68, 187, 2, 65, 8, 202, 157, 17, 89, 162, 17, 72, 2, 90, 187, 2, 65, + 6, 174, 233, 16, 72, 195, 71, 65, 8, 170, 184, 10, 72, 238, 245, 6, 89, + 187, 2, 65, 18, 54, 65, 170, 232, 16, 71, 2, 78, 2, 89, 139, 69, 72, 5, + 11, 83, 2, 141, 234, 13, 4, 65, 76, 73, 90, 16, 60, 9, 69, 70, 79, 82, + 77, 69, 68, 32, 84, 167, 152, 17, 84, 14, 40, 4, 79, 78, 69, 45, 167, + 177, 15, 83, 12, 202, 174, 17, 49, 2, 50, 2, 52, 2, 53, 2, 54, 3, 56, 8, + 182, 154, 17, 89, 162, 17, 72, 2, 83, 187, 2, 65, 32, 78, 76, 20, 4, 79, + 78, 69, 45, 70, 83, 254, 169, 17, 84, 186, 2, 65, 3, 69, 4, 231, 180, 10, + 72, 14, 246, 172, 17, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 8, + 250, 169, 17, 72, 2, 83, 186, 2, 65, 3, 69, 4, 202, 169, 17, 70, 187, 2, + 65, 16, 70, 84, 244, 173, 15, 2, 68, 90, 146, 63, 78, 226, 187, 1, 75, 3, + 80, 8, 218, 227, 16, 83, 138, 69, 84, 187, 2, 65, 16, 50, 90, 254, 226, + 16, 83, 138, 69, 72, 187, 2, 65, 8, 150, 178, 10, 83, 238, 245, 6, 89, + 187, 2, 65, 8, 144, 1, 9, 82, 69, 70, 79, 82, 77, 69, 68, 32, 34, 65, + 133, 133, 16, 18, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 77, 79, 68, 73, + 70, 73, 69, 82, 4, 30, 65, 221, 88, 2, 86, 79, 2, 133, 150, 12, 3, 83, + 80, 73, 8, 202, 134, 15, 66, 204, 78, 3, 84, 79, 80, 182, 86, 65, 159, + 82, 82, 104, 146, 1, 65, 78, 69, 62, 73, 78, 79, 74, 85, 74, 89, 224, + 149, 7, 9, 82, 79, 85, 78, 68, 69, 68, 32, 69, 238, 196, 7, 87, 178, 75, + 78, 227, 127, 86, 19, 222, 168, 15, 78, 226, 189, 1, 69, 146, 63, 72, + 146, 1, 65, 2, 73, 3, 85, 15, 246, 207, 13, 82, 158, 216, 1, 78, 130, + 254, 1, 65, 3, 73, 23, 202, 167, 15, 65, 54, 79, 134, 253, 1, 78, 86, 69, + 2, 71, 2, 73, 3, 85, 13, 42, 69, 226, 164, 17, 71, 2, 79, 3, 85, 4, 222, + 164, 17, 82, 3, 89, 19, 182, 166, 15, 65, 182, 234, 1, 69, 134, 19, 78, + 2, 79, 86, 73, 3, 85, 7, 162, 144, 17, 85, 219, 19, 73, 12, 18, 32, 63, + 79, 4, 244, 141, 16, 4, 79, 78, 32, 85, 13, 4, 68, 65, 83, 72, 8, 142, + 237, 11, 83, 154, 137, 4, 80, 242, 37, 32, 163, 112, 66, 12, 64, 4, 68, + 76, 69, 32, 237, 223, 4, 6, 76, 73, 78, 69, 32, 72, 10, 220, 193, 5, 3, + 84, 72, 73, 198, 232, 8, 76, 22, 82, 159, 167, 2, 68, 8, 76, 6, 73, 84, + 65, 82, 89, 32, 164, 194, 10, 3, 75, 89, 32, 211, 215, 5, 76, 4, 26, 72, + 219, 181, 12, 77, 2, 135, 178, 4, 69, 24, 42, 73, 76, 2, 85, 83, 207, + 159, 17, 89, 6, 34, 68, 22, 77, 255, 150, 15, 66, 2, 203, 157, 6, 73, 2, + 231, 237, 6, 73, 16, 46, 32, 153, 183, 10, 5, 45, 79, 82, 45, 80, 14, 40, + 4, 83, 73, 71, 78, 159, 183, 13, 84, 13, 11, 32, 10, 44, 5, 87, 73, 84, + 72, 32, 247, 173, 6, 73, 8, 66, 67, 166, 203, 4, 68, 230, 173, 8, 82, 21, + 4, 70, 65, 76, 76, 2, 177, 242, 11, 3, 79, 77, 77, 5, 143, 147, 15, 32, + 186, 9, 184, 1, 10, 66, 73, 76, 69, 32, 80, 72, 79, 78, 69, 166, 1, 68, + 198, 76, 78, 178, 27, 79, 172, 1, 3, 83, 81, 85, 38, 84, 250, 1, 85, 218, + 187, 11, 89, 189, 225, 2, 5, 86, 73, 69, 32, 67, 7, 11, 32, 4, 108, 21, + 87, 73, 84, 72, 32, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 65, 82, + 82, 79, 87, 195, 178, 1, 79, 2, 11, 32, 2, 173, 232, 16, 2, 65, 84, 156, + 6, 58, 69, 74, 73, 157, 75, 7, 85, 76, 79, 32, 84, 87, 79, 4, 156, 202, + 16, 10, 82, 78, 32, 80, 69, 78, 84, 65, 84, 72, 159, 18, 76, 150, 6, 42, + 32, 221, 1, 5, 70, 73, 69, 82, 32, 158, 1, 88, 5, 83, 73, 71, 78, 32, + 158, 180, 3, 86, 170, 163, 3, 68, 186, 1, 76, 243, 203, 4, 65, 10, 42, + 65, 218, 217, 8, 72, 179, 251, 7, 86, 4, 44, 5, 82, 68, 72, 65, 67, 251, + 210, 16, 78, 2, 161, 211, 16, 3, 65, 78, 68, 248, 4, 92, 12, 66, 82, 69, + 86, 69, 32, 87, 73, 84, 72, 32, 73, 89, 7, 76, 69, 84, 84, 69, 82, 32, 2, + 33, 6, 78, 86, 69, 82, 84, 69, 2, 21, 3, 68, 32, 66, 2, 205, 153, 16, 2, + 82, 69, 246, 4, 198, 1, 65, 82, 66, 66, 67, 198, 13, 68, 202, 1, 69, 182, + 2, 71, 82, 72, 46, 76, 230, 4, 77, 216, 3, 7, 79, 80, 69, 78, 32, 83, 72, + 22, 80, 38, 82, 182, 4, 83, 206, 33, 84, 114, 85, 118, 86, 63, 89, 6, 38, + 76, 166, 27, 67, 227, 209, 10, 80, 2, 253, 19, 6, 86, 69, 79, 76, 65, 82, + 6, 192, 19, 5, 73, 76, 65, 66, 73, 193, 45, 4, 69, 71, 73, 78, 162, 1, + 240, 1, 7, 65, 80, 73, 84, 65, 76, 32, 200, 2, 7, 69, 78, 84, 82, 69, 68, + 32, 100, 13, 72, 73, 78, 69, 83, 69, 32, 84, 79, 78, 69, 32, 89, 108, 8, + 89, 82, 73, 76, 76, 73, 67, 32, 218, 230, 10, 73, 244, 2, 4, 82, 79, 83, + 83, 219, 211, 5, 79, 58, 214, 1, 66, 42, 82, 130, 27, 72, 186, 157, 13, + 79, 222, 150, 3, 65, 162, 64, 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 73, 2, + 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 83, 2, 84, 2, 85, 2, 86, + 3, 87, 5, 237, 150, 6, 5, 65, 82, 82, 69, 68, 7, 41, 8, 69, 86, 69, 82, + 83, 69, 68, 32, 4, 134, 143, 17, 69, 3, 78, 4, 30, 76, 21, 3, 82, 73, 71, + 2, 29, 2, 69, 70, 2, 11, 72, 2, 11, 84, 2, 181, 26, 2, 32, 72, 16, 36, 3, + 65, 78, 71, 1, 2, 73, 78, 8, 11, 32, 8, 166, 145, 12, 83, 170, 171, 4, + 80, 158, 44, 81, 3, 82, 78, 34, 72, 30, 83, 255, 188, 16, 69, 2, 181, + 161, 15, 2, 65, 82, 74, 40, 5, 77, 65, 76, 76, 32, 183, 6, 79, 72, 186, + 1, 66, 138, 1, 68, 38, 69, 150, 1, 80, 58, 83, 94, 84, 34, 89, 226, 134, + 16, 90, 130, 64, 73, 150, 19, 67, 2, 71, 186, 22, 74, 2, 86, 158, 20, 72, + 2, 75, 186, 2, 65, 2, 79, 3, 85, 6, 38, 89, 198, 27, 65, 139, 239, 16, + 69, 2, 237, 202, 1, 19, 69, 76, 79, 82, 85, 83, 83, 73, 65, 78, 45, 85, + 75, 82, 65, 73, 78, 73, 65, 4, 242, 215, 6, 90, 251, 177, 10, 69, 15, 50, + 83, 150, 137, 17, 70, 2, 76, 2, 77, 3, 82, 5, 25, 4, 32, 87, 73, 84, 2, + 21, 3, 72, 32, 68, 2, 11, 69, 2, 253, 156, 13, 3, 83, 67, 69, 4, 26, 65, + 155, 136, 17, 69, 2, 205, 219, 16, 2, 76, 79, 8, 42, 84, 226, 173, 1, 67, + 187, 215, 15, 72, 4, 153, 19, 8, 82, 65, 73, 71, 72, 84, 32, 85, 4, 202, + 240, 16, 83, 215, 22, 69, 6, 36, 3, 69, 82, 85, 219, 134, 17, 85, 5, 37, + 7, 32, 87, 73, 84, 72, 32, 66, 2, 21, 3, 65, 67, 75, 2, 169, 200, 16, 2, + 32, 89, 2, 175, 186, 11, 70, 16, 18, 69, 27, 79, 2, 169, 5, 2, 78, 84, + 14, 64, 2, 84, 32, 52, 5, 85, 66, 76, 69, 32, 129, 52, 2, 87, 78, 6, 162, + 224, 9, 83, 210, 202, 4, 86, 247, 181, 1, 72, 4, 210, 137, 2, 65, 231, + 248, 2, 80, 24, 40, 5, 88, 84, 82, 65, 45, 131, 49, 78, 20, 52, 5, 72, + 73, 71, 72, 32, 81, 4, 76, 79, 87, 32, 10, 156, 1, 9, 69, 88, 84, 82, 65, + 45, 76, 79, 87, 154, 7, 68, 70, 76, 79, 84, 10, 76, 10, 69, 88, 84, 82, + 65, 45, 72, 73, 71, 72, 154, 7, 68, 70, 76, 79, 84, 2, 145, 8, 8, 32, 67, + 79, 78, 84, 79, 85, 82, 8, 162, 9, 82, 226, 3, 76, 173, 209, 15, 9, 69, + 79, 82, 71, 73, 65, 78, 32, 78, 10, 248, 5, 4, 73, 71, 72, 32, 255, 40, + 65, 44, 46, 65, 84, 2, 79, 87, 201, 11, 2, 69, 70, 2, 21, 3, 84, 69, 82, + 2, 17, 2, 65, 76, 2, 17, 2, 32, 67, 2, 239, 205, 15, 76, 36, 86, 32, 133, + 151, 12, 15, 69, 82, 32, 82, 73, 71, 72, 84, 32, 67, 79, 82, 78, 69, 82, + 34, 158, 1, 67, 20, 2, 68, 79, 40, 4, 76, 69, 70, 84, 82, 77, 12, 2, 82, + 73, 50, 84, 178, 3, 65, 42, 71, 170, 2, 73, 172, 157, 14, 2, 85, 80, 255, + 188, 1, 86, 2, 175, 215, 10, 73, 6, 238, 2, 84, 241, 161, 14, 2, 87, 78, + 6, 28, 2, 32, 65, 247, 2, 45, 4, 25, 4, 82, 82, 79, 87, 5, 187, 164, 14, + 72, 2, 111, 65, 4, 216, 163, 14, 3, 71, 72, 84, 175, 216, 2, 78, 4, 194, + 2, 79, 219, 220, 14, 73, 18, 18, 65, 23, 73, 2, 195, 212, 15, 67, 16, 26, + 68, 179, 201, 13, 78, 14, 38, 32, 217, 1, 4, 68, 76, 69, 32, 8, 26, 68, + 70, 76, 79, 84, 4, 17, 2, 79, 84, 4, 25, 4, 84, 69, 68, 32, 4, 18, 76, + 79, 84, 2, 25, 4, 69, 70, 84, 45, 2, 25, 4, 83, 84, 69, 77, 2, 17, 2, 32, + 84, 2, 11, 79, 2, 11, 78, 2, 135, 214, 15, 69, 6, 40, 6, 68, 79, 85, 66, + 76, 69, 75, 71, 4, 11, 32, 4, 18, 65, 43, 71, 2, 11, 67, 2, 145, 213, 10, + 2, 85, 84, 2, 11, 82, 2, 223, 212, 10, 65, 2, 171, 242, 14, 69, 4, 134, + 198, 13, 76, 159, 152, 2, 82, 26, 104, 6, 65, 73, 83, 69, 68, 32, 150, 1, + 69, 216, 1, 3, 73, 71, 72, 193, 250, 8, 5, 72, 79, 84, 73, 67, 10, 70, + 68, 30, 73, 214, 218, 9, 69, 128, 255, 5, 2, 85, 80, 215, 76, 67, 2, 197, + 163, 15, 2, 79, 87, 2, 189, 218, 9, 7, 78, 86, 69, 82, 84, 69, 68, 8, + 104, 7, 86, 69, 82, 83, 69, 68, 32, 137, 28, 14, 84, 82, 79, 70, 76, 69, + 88, 32, 67, 76, 73, 67, 75, 32, 6, 26, 71, 163, 159, 14, 67, 4, 11, 76, + 4, 49, 10, 79, 84, 84, 65, 76, 32, 83, 84, 79, 80, 5, 171, 19, 32, 6, 17, + 2, 84, 32, 6, 38, 72, 150, 213, 13, 84, 239, 69, 65, 2, 197, 196, 7, 3, + 65, 76, 70, 156, 2, 138, 1, 72, 48, 5, 77, 65, 76, 76, 32, 148, 31, 8, + 84, 82, 69, 83, 83, 32, 65, 78, 53, 11, 85, 80, 69, 82, 83, 67, 82, 73, + 80, 84, 32, 4, 180, 158, 8, 3, 79, 82, 84, 251, 205, 6, 69, 144, 2, 154, + 2, 65, 50, 66, 130, 1, 67, 238, 2, 68, 238, 2, 90, 66, 69, 46, 70, 30, + 71, 110, 72, 102, 77, 34, 73, 34, 74, 98, 76, 176, 3, 7, 78, 32, 87, 73, + 84, 72, 32, 30, 79, 94, 80, 22, 82, 146, 2, 83, 190, 1, 84, 246, 7, 85, + 142, 1, 86, 226, 211, 16, 75, 2, 81, 2, 87, 2, 88, 3, 89, 9, 222, 147, + 13, 76, 170, 140, 3, 73, 227, 79, 69, 11, 46, 65, 50, 79, 222, 8, 32, + 179, 193, 16, 69, 2, 17, 2, 82, 82, 2, 169, 247, 5, 2, 69, 68, 2, 229, + 20, 4, 84, 84, 79, 77, 41, 100, 7, 65, 80, 73, 84, 65, 76, 32, 180, 1, 6, + 76, 79, 83, 69, 68, 32, 190, 17, 32, 139, 199, 16, 72, 30, 114, 73, 214, + 6, 71, 226, 16, 76, 214, 190, 16, 79, 158, 20, 65, 186, 2, 66, 2, 72, 2, + 78, 2, 82, 2, 85, 3, 89, 7, 22, 78, 191, 11, 32, 2, 225, 240, 5, 5, 86, + 69, 82, 84, 69, 4, 26, 82, 243, 166, 9, 79, 2, 217, 20, 9, 69, 86, 69, + 82, 83, 69, 68, 32, 79, 23, 104, 6, 32, 87, 73, 84, 72, 32, 86, 69, 32, + 9, 79, 84, 76, 69, 83, 83, 32, 74, 32, 105, 3, 90, 32, 68, 6, 26, 72, + 163, 179, 14, 84, 4, 21, 3, 79, 79, 75, 5, 213, 250, 13, 3, 32, 65, 78, + 4, 238, 15, 90, 211, 181, 16, 76, 4, 33, 6, 87, 73, 84, 72, 32, 83, 4, + 29, 5, 84, 82, 79, 75, 69, 5, 193, 239, 8, 4, 32, 65, 78, 68, 6, 33, 6, + 73, 71, 82, 65, 80, 72, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 138, 14, 67, + 207, 4, 82, 11, 142, 231, 16, 83, 2, 84, 2, 90, 63, 78, 5, 225, 13, 3, + 69, 78, 71, 11, 56, 5, 82, 69, 69, 75, 32, 162, 1, 32, 183, 144, 14, 65, + 4, 26, 71, 139, 143, 11, 80, 2, 183, 145, 14, 65, 11, 40, 6, 32, 87, 73, + 84, 72, 32, 39, 69, 4, 206, 144, 15, 72, 247, 165, 1, 83, 4, 17, 2, 78, + 71, 5, 11, 32, 2, 255, 235, 8, 87, 4, 222, 4, 32, 191, 188, 16, 79, 5, + 37, 7, 32, 87, 73, 84, 72, 32, 67, 2, 11, 82, 2, 225, 173, 14, 6, 79, 83, + 83, 69, 68, 45, 25, 116, 6, 32, 87, 73, 84, 72, 32, 226, 9, 83, 2, 90, + 112, 2, 69, 90, 193, 194, 16, 8, 73, 71, 65, 84, 85, 82, 69, 32, 12, 54, + 73, 82, 77, 82, 82, 222, 6, 80, 143, 245, 3, 66, 2, 49, 10, 78, 86, 69, + 82, 84, 69, 68, 32, 76, 65, 2, 157, 233, 12, 2, 90, 89, 2, 11, 73, 2, 17, + 2, 68, 68, 2, 17, 2, 76, 69, 2, 221, 250, 12, 2, 32, 84, 4, 61, 13, 69, + 84, 82, 79, 70, 76, 69, 88, 32, 72, 79, 79, 75, 5, 205, 12, 4, 32, 65, + 78, 68, 4, 210, 11, 82, 207, 1, 76, 9, 18, 32, 47, 80, 2, 21, 3, 87, 73, + 84, 2, 219, 176, 16, 72, 4, 229, 196, 12, 2, 69, 78, 5, 231, 204, 16, 72, + 15, 80, 6, 32, 87, 73, 84, 72, 32, 62, 65, 41, 8, 69, 86, 69, 82, 83, 69, + 68, 32, 4, 26, 70, 155, 168, 14, 84, 2, 225, 229, 8, 3, 73, 83, 72, 2, + 17, 2, 77, 83, 2, 235, 232, 5, 32, 6, 38, 71, 170, 7, 79, 167, 215, 16, + 69, 2, 11, 76, 2, 161, 137, 14, 4, 79, 84, 84, 65, 13, 72, 6, 32, 87, 73, + 84, 72, 32, 34, 67, 61, 6, 73, 68, 69, 87, 65, 89, 4, 158, 3, 67, 195, + 132, 15, 72, 4, 26, 82, 187, 227, 11, 72, 2, 209, 219, 5, 3, 73, 80, 84, + 2, 159, 194, 6, 83, 51, 106, 32, 102, 67, 116, 2, 69, 83, 44, 2, 79, 80, + 42, 83, 96, 6, 85, 82, 78, 69, 68, 32, 215, 155, 13, 72, 4, 29, 5, 87, + 73, 84, 72, 32, 4, 22, 80, 219, 5, 82, 2, 213, 225, 8, 6, 65, 76, 65, 84, + 65, 76, 2, 45, 9, 32, 68, 73, 71, 82, 65, 80, 72, 32, 2, 25, 4, 87, 73, + 84, 72, 2, 17, 2, 32, 67, 2, 175, 145, 3, 85, 2, 11, 72, 2, 189, 155, 10, + 3, 32, 68, 73, 2, 165, 226, 5, 5, 32, 72, 65, 76, 70, 4, 37, 7, 32, 68, + 73, 71, 82, 65, 80, 4, 11, 72, 5, 11, 32, 2, 141, 3, 4, 87, 73, 84, 72, + 32, 86, 65, 38, 77, 74, 79, 42, 82, 214, 1, 89, 170, 213, 16, 72, 2, 73, + 2, 86, 3, 87, 7, 134, 252, 12, 76, 139, 220, 3, 69, 5, 41, 8, 32, 87, 73, + 84, 72, 32, 76, 79, 2, 253, 141, 4, 2, 78, 71, 2, 11, 80, 2, 225, 158, 6, + 2, 69, 78, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 26, 76, 207, 128, 15, 72, + 4, 37, 7, 79, 78, 71, 32, 76, 69, 71, 5, 25, 4, 32, 65, 78, 68, 2, 17, 2, + 32, 82, 2, 11, 69, 2, 241, 219, 8, 7, 84, 82, 79, 70, 76, 69, 88, 5, 29, + 5, 32, 87, 73, 84, 72, 2, 213, 238, 3, 2, 32, 66, 9, 18, 32, 95, 80, 4, + 40, 4, 87, 73, 84, 72, 243, 176, 15, 66, 2, 17, 2, 32, 76, 2, 11, 69, 2, + 131, 1, 70, 2, 129, 132, 16, 2, 83, 73, 7, 33, 6, 32, 87, 73, 84, 72, 32, + 4, 26, 82, 179, 253, 14, 72, 2, 21, 3, 73, 71, 72, 2, 163, 217, 8, 84, 4, + 11, 68, 4, 11, 32, 4, 146, 210, 1, 72, 35, 76, 4, 24, 2, 72, 65, 31, 84, + 2, 25, 4, 76, 70, 32, 84, 2, 43, 82, 4, 30, 82, 53, 3, 85, 82, 78, 2, + 189, 129, 16, 8, 73, 65, 78, 71, 85, 76, 65, 82, 2, 169, 172, 8, 2, 69, + 68, 8, 70, 80, 184, 159, 10, 7, 78, 65, 83, 80, 73, 82, 65, 143, 177, 6, + 83, 4, 11, 32, 4, 226, 177, 13, 84, 239, 69, 65, 4, 26, 79, 135, 180, 15, + 69, 2, 11, 73, 2, 235, 254, 15, 67, 4, 36, 3, 65, 78, 71, 1, 2, 73, 78, + 2, 25, 4, 32, 68, 69, 80, 2, 21, 3, 65, 82, 84, 2, 21, 3, 73, 78, 71, 2, + 17, 2, 32, 84, 2, 11, 79, 2, 171, 135, 6, 78, 2, 11, 32, 2, 131, 163, 15, + 83, 234, 2, 92, 2, 69, 89, 88, 7, 71, 79, 76, 73, 65, 78, 32, 206, 24, + 79, 209, 140, 12, 3, 75, 69, 89, 6, 26, 32, 203, 143, 16, 45, 4, 128, + 142, 12, 6, 87, 73, 84, 72, 32, 87, 167, 211, 2, 66, 214, 2, 194, 2, 68, + 46, 70, 148, 1, 14, 73, 78, 86, 69, 82, 84, 69, 68, 32, 66, 73, 82, 71, + 65, 16, 7, 76, 69, 84, 84, 69, 82, 32, 138, 16, 83, 132, 1, 7, 82, 79, + 84, 65, 84, 69, 68, 22, 66, 98, 84, 160, 2, 4, 86, 79, 87, 69, 194, 241, + 3, 67, 164, 1, 6, 77, 65, 78, 67, 72, 85, 160, 201, 7, 4, 78, 73, 82, 85, + 239, 208, 2, 69, 22, 232, 20, 3, 79, 85, 66, 171, 193, 14, 73, 12, 112, + 19, 82, 69, 69, 32, 86, 65, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, + 69, 67, 202, 243, 13, 85, 195, 46, 79, 8, 177, 250, 9, 3, 84, 79, 82, 5, + 227, 19, 32, 136, 2, 130, 2, 65, 244, 4, 2, 67, 72, 88, 2, 77, 65, 174, + 2, 83, 246, 1, 84, 234, 3, 90, 178, 195, 14, 72, 246, 173, 1, 75, 2, 76, + 162, 7, 69, 2, 79, 2, 85, 234, 61, 66, 2, 68, 2, 70, 2, 71, 2, 74, 2, 78, + 2, 80, 2, 81, 2, 82, 2, 87, 2, 89, 187, 2, 73, 59, 56, 8, 76, 73, 32, 71, + 65, 76, 73, 32, 171, 197, 16, 78, 54, 174, 1, 65, 52, 6, 86, 73, 83, 65, + 82, 71, 22, 68, 76, 5, 72, 65, 76, 70, 32, 34, 73, 50, 85, 34, 78, 30, + 84, 66, 66, 174, 250, 15, 80, 2, 90, 254, 68, 83, 14, 67, 3, 75, 7, 48, + 6, 78, 85, 83, 86, 65, 82, 155, 196, 16, 72, 2, 243, 165, 9, 65, 8, 26, + 65, 179, 193, 16, 68, 7, 166, 253, 8, 77, 253, 131, 7, 3, 71, 65, 76, 4, + 254, 192, 16, 89, 187, 2, 85, 5, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, + 85, 2, 249, 255, 15, 3, 66, 65, 68, 4, 142, 192, 16, 71, 3, 78, 8, 60, 6, + 72, 82, 69, 69, 32, 66, 174, 250, 15, 84, 195, 71, 65, 2, 17, 2, 65, 76, + 2, 211, 156, 16, 85, 6, 26, 65, 171, 193, 16, 73, 5, 29, 5, 32, 87, 73, + 84, 72, 2, 149, 238, 14, 2, 32, 84, 41, 29, 5, 78, 67, 72, 85, 32, 38, + 104, 9, 65, 76, 73, 32, 71, 65, 76, 73, 32, 186, 194, 14, 90, 242, 250, + 1, 70, 2, 75, 2, 82, 187, 2, 73, 28, 122, 68, 194, 198, 9, 67, 246, 227, + 2, 84, 138, 151, 2, 66, 2, 71, 2, 74, 2, 76, 234, 181, 1, 90, 254, 4, 78, + 131, 64, 83, 4, 186, 193, 14, 68, 243, 250, 1, 72, 48, 52, 4, 73, 66, 69, + 32, 210, 187, 16, 72, 187, 2, 65, 44, 214, 222, 5, 71, 2, 72, 170, 202, + 6, 84, 238, 3, 73, 246, 147, 2, 67, 2, 83, 246, 7, 82, 190, 164, 1, 65, + 186, 9, 90, 162, 7, 85, 234, 61, 68, 2, 70, 2, 74, 2, 75, 2, 80, 187, 2, + 69, 60, 52, 4, 79, 68, 79, 32, 222, 185, 16, 83, 187, 2, 65, 56, 250, 1, + 65, 98, 68, 34, 74, 34, 78, 252, 164, 1, 8, 76, 79, 78, 71, 32, 86, 79, + 87, 206, 180, 4, 71, 170, 202, 6, 84, 226, 151, 2, 67, 246, 7, 72, 150, + 181, 1, 79, 2, 85, 234, 61, 66, 2, 75, 2, 77, 2, 80, 2, 81, 2, 87, 2, 89, + 186, 2, 69, 3, 73, 6, 56, 8, 76, 73, 32, 71, 65, 76, 73, 32, 139, 185, + 16, 78, 4, 178, 188, 14, 90, 243, 250, 1, 84, 4, 254, 182, 16, 90, 187, + 2, 65, 4, 222, 182, 16, 73, 187, 2, 65, 2, 191, 182, 16, 73, 6, 138, 165, + 16, 72, 162, 17, 82, 187, 2, 65, 8, 128, 1, 4, 87, 73, 82, 76, 217, 191, + 4, 21, 73, 66, 69, 32, 83, 89, 76, 76, 65, 66, 76, 69, 32, 66, 79, 85, + 78, 68, 65, 82, 89, 6, 17, 2, 32, 66, 6, 25, 4, 73, 82, 71, 65, 7, 33, 6, + 32, 87, 73, 84, 72, 32, 4, 150, 2, 68, 255, 240, 15, 79, 6, 152, 1, 3, + 82, 73, 80, 56, 18, 85, 82, 78, 69, 68, 32, 83, 87, 73, 82, 76, 32, 66, + 73, 82, 71, 65, 32, 217, 192, 1, 8, 79, 68, 79, 32, 83, 79, 70, 84, 2, + 165, 241, 15, 9, 76, 69, 32, 66, 73, 82, 71, 65, 32, 2, 33, 6, 87, 73, + 84, 72, 32, 68, 2, 229, 240, 15, 5, 79, 85, 66, 76, 69, 2, 135, 235, 9, + 76, 10, 84, 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 60, 4, 83, 84, 65, 66, + 215, 251, 13, 82, 6, 26, 89, 179, 198, 12, 69, 4, 158, 226, 15, 65, 155, + 1, 73, 2, 171, 132, 4, 76, 10, 48, 2, 78, 32, 250, 210, 4, 68, 231, 200, + 11, 83, 6, 88, 12, 86, 73, 69, 87, 73, 78, 71, 32, 67, 69, 82, 69, 174, + 197, 12, 67, 85, 2, 76, 65, 2, 249, 145, 16, 2, 77, 79, 4, 174, 190, 12, + 73, 139, 243, 3, 69, 10, 26, 72, 69, 2, 79, 82, 2, 45, 9, 69, 82, 32, 67, + 72, 82, 73, 83, 84, 2, 243, 254, 2, 77, 8, 50, 32, 52, 4, 73, 90, 69, 68, + 175, 161, 15, 87, 4, 150, 202, 8, 66, 221, 171, 6, 4, 83, 67, 79, 79, 2, + 209, 176, 3, 7, 32, 87, 72, 69, 69, 76, 67, 18, 52, 2, 78, 84, 152, 1, 2, + 83, 69, 191, 172, 16, 84, 10, 48, 3, 65, 73, 78, 169, 142, 12, 3, 32, 70, + 85, 9, 11, 32, 6, 186, 207, 9, 82, 24, 5, 67, 65, 66, 76, 69, 177, 241, + 1, 6, 66, 73, 67, 89, 67, 76, 7, 11, 32, 4, 180, 154, 15, 2, 84, 82, 131, + 86, 70, 86, 52, 7, 76, 69, 84, 84, 69, 82, 32, 231, 235, 5, 68, 62, 198, + 1, 75, 62, 77, 34, 78, 34, 79, 30, 80, 34, 84, 242, 105, 68, 212, 133, 8, + 2, 72, 65, 2, 82, 166, 226, 1, 83, 190, 89, 76, 246, 7, 67, 218, 100, 69, + 254, 242, 3, 89, 190, 28, 66, 2, 87, 187, 2, 65, 6, 160, 174, 5, 2, 69, + 65, 178, 137, 6, 72, 199, 243, 4, 79, 4, 174, 152, 16, 65, 215, 1, 73, 4, + 218, 218, 15, 73, 139, 60, 71, 7, 154, 170, 16, 76, 3, 79, 4, 166, 150, + 16, 72, 219, 19, 65, 6, 222, 149, 9, 72, 234, 144, 7, 69, 155, 3, 65, + 206, 4, 44, 2, 76, 84, 234, 6, 83, 247, 139, 14, 67, 102, 36, 4, 65, 78, + 73, 32, 219, 2, 73, 76, 52, 7, 76, 69, 84, 84, 69, 82, 32, 151, 149, 4, + 83, 74, 206, 1, 68, 234, 83, 78, 194, 236, 1, 82, 254, 208, 9, 74, 202, + 56, 84, 162, 149, 3, 66, 2, 67, 2, 71, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 83, 2, 86, 2, 89, 186, 2, 65, 2, 69, 2, 73, 3, 85, 10, 38, 68, - 178, 144, 16, 72, 187, 2, 65, 6, 174, 144, 16, 68, 2, 72, 187, 2, 65, 26, - 56, 2, 80, 76, 236, 2, 3, 83, 69, 84, 175, 254, 14, 77, 18, 50, 69, 89, + 238, 163, 16, 72, 187, 2, 65, 6, 234, 163, 16, 68, 2, 72, 187, 2, 65, 26, + 56, 2, 80, 76, 236, 2, 3, 83, 69, 84, 207, 143, 15, 77, 18, 50, 69, 89, 8, 73, 67, 65, 84, 73, 79, 78, 32, 2, 41, 8, 32, 77, 85, 83, 73, 67, 65, - 76, 2, 21, 3, 32, 78, 79, 2, 147, 253, 14, 84, 16, 40, 4, 83, 73, 71, 78, - 207, 144, 16, 88, 15, 11, 32, 12, 48, 3, 73, 78, 32, 81, 5, 87, 73, 84, - 72, 32, 8, 226, 206, 3, 76, 22, 82, 192, 175, 9, 5, 68, 79, 85, 66, 76, - 247, 236, 1, 84, 4, 174, 164, 12, 85, 195, 234, 2, 68, 7, 11, 32, 4, 232, - 83, 5, 77, 85, 76, 84, 73, 239, 139, 12, 85, 228, 3, 44, 5, 72, 82, 79, - 79, 77, 21, 2, 73, 67, 5, 175, 226, 14, 32, 224, 3, 30, 32, 101, 3, 65, - 76, 32, 6, 76, 4, 78, 65, 84, 85, 200, 43, 2, 70, 76, 161, 137, 11, 4, - 83, 72, 65, 82, 2, 235, 201, 10, 82, 218, 3, 64, 8, 75, 69, 89, 66, 79, - 65, 82, 68, 66, 83, 135, 247, 3, 78, 5, 41, 8, 32, 87, 73, 84, 72, 32, - 74, 65, 2, 143, 239, 13, 67, 212, 3, 48, 6, 89, 77, 66, 79, 76, 32, 243, - 166, 11, 67, 210, 3, 230, 2, 66, 238, 1, 67, 226, 9, 68, 242, 2, 69, 162, - 1, 70, 166, 2, 71, 112, 9, 65, 82, 80, 69, 71, 71, 73, 65, 84, 168, 1, 2, - 72, 65, 126, 75, 198, 3, 76, 138, 1, 77, 190, 2, 78, 162, 1, 79, 218, 2, - 80, 200, 2, 2, 81, 85, 146, 2, 82, 198, 2, 83, 230, 5, 84, 182, 10, 86, - 42, 88, 42, 87, 200, 140, 9, 9, 73, 78, 86, 69, 82, 84, 69, 68, 32, 255, - 195, 6, 90, 20, 36, 5, 69, 71, 73, 78, 32, 59, 82, 8, 130, 15, 80, 30, - 83, 218, 246, 7, 66, 143, 231, 7, 84, 12, 24, 2, 65, 67, 35, 69, 4, 198, - 210, 15, 75, 155, 53, 69, 8, 26, 86, 203, 197, 15, 65, 6, 32, 2, 73, 83, - 131, 135, 16, 69, 5, 171, 26, 32, 84, 120, 2, 65, 69, 30, 76, 94, 79, - 194, 7, 82, 234, 11, 32, 232, 32, 7, 73, 82, 67, 76, 69, 32, 88, 189, - 222, 5, 2, 85, 84, 2, 177, 193, 15, 2, 83, 85, 8, 42, 73, 245, 23, 5, 85, - 83, 84, 69, 82, 4, 26, 77, 223, 225, 13, 86, 2, 247, 214, 9, 65, 64, 26, - 77, 171, 130, 16, 68, 62, 64, 7, 66, 73, 78, 73, 78, 71, 32, 237, 145, 6, - 3, 77, 79, 78, 60, 202, 1, 65, 80, 7, 77, 65, 82, 67, 65, 84, 79, 56, 2, - 68, 79, 56, 2, 85, 80, 28, 2, 70, 76, 40, 5, 72, 65, 82, 77, 79, 22, 83, - 142, 2, 84, 226, 176, 7, 66, 224, 193, 3, 2, 76, 79, 195, 138, 5, 82, 6, - 76, 5, 67, 67, 69, 78, 84, 37, 10, 85, 71, 77, 69, 78, 84, 65, 84, 73, - 79, 5, 205, 2, 5, 45, 83, 84, 65, 67, 2, 199, 240, 14, 78, 6, 52, 2, 87, - 78, 168, 3, 2, 85, 66, 191, 225, 15, 73, 2, 233, 230, 14, 2, 32, 66, 12, - 248, 76, 2, 65, 71, 239, 179, 15, 73, 2, 195, 216, 14, 78, 12, 132, 1, 9, - 78, 65, 80, 32, 80, 73, 90, 90, 73, 22, 84, 224, 205, 4, 11, 80, 82, 69, - 67, 72, 71, 69, 83, 65, 78, 71, 219, 129, 6, 77, 2, 139, 244, 11, 67, 6, - 44, 5, 65, 67, 67, 65, 84, 151, 238, 15, 69, 4, 40, 4, 73, 83, 83, 73, - 195, 254, 15, 79, 2, 203, 223, 15, 77, 10, 34, 82, 193, 143, 12, 2, 69, - 78, 8, 28, 2, 73, 80, 191, 6, 69, 2, 209, 242, 11, 6, 76, 69, 32, 84, 79, - 78, 4, 22, 79, 191, 2, 69, 2, 255, 240, 14, 73, 24, 98, 65, 142, 1, 69, - 84, 6, 79, 85, 66, 76, 69, 32, 225, 158, 10, 7, 82, 85, 77, 32, 67, 76, - 69, 10, 96, 2, 32, 67, 20, 2, 77, 80, 204, 29, 4, 83, 72, 69, 68, 217, - 180, 15, 5, 76, 32, 83, 69, 71, 2, 223, 189, 8, 65, 5, 231, 243, 13, 32, - 4, 52, 3, 67, 82, 69, 145, 228, 8, 4, 71, 82, 69, 69, 2, 165, 25, 3, 83, - 67, 69, 6, 246, 21, 83, 130, 7, 66, 243, 134, 5, 70, 14, 32, 3, 78, 68, - 32, 155, 16, 73, 10, 74, 80, 30, 83, 208, 13, 3, 79, 70, 32, 138, 233, 7, - 66, 143, 231, 7, 84, 2, 229, 211, 12, 2, 72, 82, 2, 191, 128, 15, 76, 34, - 104, 6, 69, 82, 77, 65, 84, 65, 22, 73, 122, 79, 102, 32, 152, 36, 3, 85, - 83, 65, 205, 229, 3, 2, 76, 65, 5, 219, 202, 13, 32, 10, 22, 78, 143, 34, - 86, 8, 56, 9, 71, 69, 82, 69, 68, 32, 84, 82, 69, 243, 20, 65, 6, 237, - 237, 5, 4, 77, 79, 76, 79, 6, 216, 25, 3, 85, 82, 45, 183, 150, 14, 82, - 18, 54, 32, 56, 7, 76, 73, 83, 83, 65, 78, 68, 23, 82, 6, 25, 4, 67, 76, - 69, 70, 7, 161, 13, 3, 32, 79, 84, 4, 215, 129, 6, 79, 8, 84, 9, 65, 67, - 69, 32, 78, 79, 84, 69, 32, 37, 8, 69, 71, 79, 82, 73, 65, 78, 32, 4, - 164, 222, 8, 2, 78, 79, 27, 83, 4, 250, 2, 67, 3, 70, 8, 44, 3, 76, 70, - 32, 209, 8, 3, 85, 80, 84, 6, 52, 3, 80, 69, 68, 226, 222, 3, 78, 207, - 183, 3, 82, 2, 139, 198, 5, 65, 24, 48, 6, 73, 69, 86, 65, 78, 32, 247, - 205, 14, 79, 22, 178, 1, 67, 46, 69, 68, 7, 81, 85, 65, 82, 84, 69, 82, - 62, 70, 164, 219, 3, 4, 72, 65, 76, 70, 0, 5, 87, 72, 79, 76, 69, 157, - 224, 1, 9, 82, 69, 67, 73, 84, 65, 84, 73, 86, 2, 11, 32, 2, 11, 67, 2, - 163, 163, 5, 76, 6, 64, 5, 73, 71, 72, 84, 72, 201, 186, 14, 5, 78, 68, - 32, 79, 70, 4, 217, 252, 5, 10, 32, 78, 79, 84, 69, 32, 83, 84, 69, 77, - 4, 222, 14, 76, 197, 204, 3, 4, 73, 78, 65, 76, 8, 44, 4, 79, 78, 71, 65, - 221, 13, 2, 69, 70, 7, 11, 32, 4, 28, 3, 73, 77, 80, 3, 80, 2, 197, 2, 7, - 69, 82, 70, 69, 67, 84, 65, 18, 104, 2, 65, 88, 20, 2, 69, 90, 20, 5, 73, - 78, 73, 77, 65, 48, 3, 79, 79, 78, 25, 4, 85, 76, 84, 73, 2, 255, 204, - 15, 73, 2, 135, 207, 15, 90, 7, 11, 32, 4, 170, 144, 7, 82, 167, 194, 5, - 66, 4, 193, 17, 2, 32, 78, 4, 60, 11, 80, 76, 69, 32, 77, 69, 65, 83, 85, - 82, 69, 15, 32, 2, 11, 32, 2, 163, 143, 7, 82, 10, 120, 4, 69, 66, 69, - 78, 216, 26, 3, 85, 76, 76, 192, 220, 5, 3, 65, 84, 85, 141, 217, 6, 7, - 79, 84, 69, 72, 69, 65, 68, 2, 213, 211, 14, 4, 83, 84, 73, 77, 32, 88, - 2, 78, 69, 120, 14, 82, 78, 65, 77, 69, 78, 84, 32, 83, 84, 82, 79, 75, - 69, 107, 84, 6, 92, 18, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, 69, - 78, 84, 89, 45, 69, 73, 163, 20, 45, 4, 185, 13, 2, 71, 72, 22, 11, 45, - 22, 158, 166, 3, 49, 162, 195, 12, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, - 2, 56, 3, 57, 4, 173, 4, 3, 84, 65, 86, 18, 96, 9, 65, 82, 69, 78, 84, - 72, 69, 83, 73, 0, 2, 76, 85, 18, 69, 142, 1, 79, 223, 222, 13, 73, 2, - 247, 22, 83, 6, 68, 4, 68, 65, 76, 32, 49, 9, 83, 32, 83, 85, 66, 80, 85, - 78, 67, 4, 26, 85, 199, 165, 15, 77, 2, 171, 165, 15, 80, 2, 167, 195, - 13, 84, 6, 48, 2, 68, 65, 245, 5, 5, 82, 82, 69, 67, 84, 2, 151, 224, 13, - 84, 12, 76, 6, 65, 82, 84, 69, 82, 32, 125, 9, 73, 78, 68, 73, 67, 69, - 83, 73, 77, 8, 60, 5, 84, 79, 78, 69, 32, 246, 207, 3, 78, 207, 183, 3, - 82, 4, 26, 83, 243, 141, 5, 70, 2, 197, 228, 15, 3, 72, 65, 82, 4, 17, 2, - 65, 32, 4, 142, 171, 12, 65, 205, 182, 3, 3, 66, 65, 83, 14, 22, 69, 175, - 1, 73, 10, 72, 4, 80, 69, 65, 84, 81, 10, 86, 69, 82, 83, 69, 32, 70, 73, - 78, 65, 8, 56, 8, 69, 68, 32, 70, 73, 71, 85, 82, 163, 146, 14, 32, 6, - 147, 217, 5, 69, 2, 211, 4, 76, 4, 48, 2, 71, 72, 57, 6, 78, 70, 79, 82, - 90, 65, 2, 33, 6, 84, 32, 82, 69, 80, 69, 2, 255, 158, 10, 65, 2, 147, - 176, 8, 78, 48, 136, 1, 6, 67, 65, 78, 68, 73, 67, 62, 69, 166, 1, 72, - 54, 73, 252, 1, 6, 81, 85, 65, 82, 69, 32, 212, 153, 8, 2, 85, 66, 243, - 100, 79, 4, 17, 2, 85, 83, 5, 141, 218, 13, 5, 32, 70, 76, 69, 88, 14, - 32, 2, 77, 73, 179, 182, 15, 71, 12, 64, 6, 66, 82, 69, 86, 73, 83, 1, 6, - 77, 73, 78, 73, 77, 65, 6, 11, 32, 6, 138, 13, 87, 182, 244, 6, 82, 167, - 194, 5, 66, 6, 84, 3, 79, 82, 84, 213, 233, 5, 3, 65, 82, 80, 14, 32, 4, - 78, 71, 76, 69, 43, 88, 2, 17, 2, 32, 66, 2, 135, 246, 13, 65, 12, 18, - 45, 79, 84, 4, 242, 7, 76, 173, 190, 14, 11, 83, 84, 82, 73, 78, 71, 32, - 70, 82, 69, 84, 8, 52, 3, 69, 69, 78, 1, 6, 89, 45, 70, 79, 85, 82, 4, - 193, 12, 2, 84, 72, 6, 26, 78, 247, 219, 15, 66, 4, 229, 9, 7, 79, 84, - 69, 72, 69, 65, 68, 60, 132, 1, 6, 69, 77, 80, 85, 83, 32, 154, 4, 72, - 88, 2, 87, 79, 84, 7, 79, 82, 67, 85, 76, 85, 83, 46, 82, 141, 3, 3, 85, - 82, 78, 16, 232, 1, 27, 73, 77, 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, - 67, 85, 77, 32, 80, 82, 79, 76, 65, 84, 73, 79, 78, 69, 32, 129, 1, 25, - 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, 67, 85, 77, 32, 80, 82, 79, 76, - 65, 84, 73, 79, 78, 69, 32, 10, 60, 10, 73, 77, 80, 69, 82, 70, 69, 67, - 84, 65, 131, 1, 80, 9, 213, 206, 5, 11, 32, 68, 73, 77, 73, 78, 85, 84, - 73, 79, 78, 6, 60, 3, 73, 77, 80, 41, 8, 80, 69, 82, 70, 69, 67, 84, 65, - 2, 197, 178, 15, 5, 69, 82, 70, 69, 67, 5, 169, 189, 8, 12, 32, 68, 73, - 77, 73, 78, 85, 84, 73, 79, 78, 45, 6, 72, 2, 82, 69, 249, 5, 11, 73, 82, - 84, 89, 45, 83, 69, 67, 79, 78, 68, 2, 11, 69, 2, 11, 45, 2, 11, 76, 2, - 37, 7, 73, 78, 69, 32, 83, 84, 65, 2, 235, 211, 14, 70, 5, 225, 206, 11, - 6, 32, 82, 69, 83, 85, 80, 27, 33, 6, 73, 65, 78, 71, 76, 69, 24, 128, 1, - 10, 32, 78, 79, 84, 69, 72, 69, 65, 68, 32, 61, 17, 45, 82, 79, 85, 78, - 68, 32, 78, 79, 84, 69, 72, 69, 65, 68, 32, 68, 20, 58, 68, 24, 3, 85, - 80, 32, 38, 82, 25, 3, 76, 69, 70, 4, 93, 3, 79, 87, 78, 8, 34, 82, 78, - 87, 219, 182, 12, 66, 4, 21, 3, 73, 71, 72, 4, 11, 84, 4, 11, 32, 4, 26, - 87, 219, 182, 12, 66, 2, 11, 72, 2, 223, 138, 14, 73, 7, 11, 32, 4, 246, - 186, 8, 83, 183, 150, 7, 85, 4, 36, 3, 79, 73, 68, 159, 142, 15, 73, 2, - 237, 251, 12, 5, 32, 78, 79, 84, 69, 6, 92, 4, 72, 79, 76, 69, 221, 199, - 8, 13, 73, 84, 72, 32, 70, 73, 78, 71, 69, 82, 78, 65, 73, 4, 11, 32, 4, - 218, 186, 3, 78, 207, 183, 3, 82, 232, 3, 224, 2, 15, 67, 79, 78, 83, 79, - 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 254, 1, 76, 212, 14, 16, 77, 79, - 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, 122, 83, 80, 16, - 69, 65, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 222, 9, - 84, 204, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 160, 175, 9, - 3, 80, 65, 79, 207, 142, 4, 68, 16, 66, 77, 165, 1, 11, 83, 72, 65, 78, - 32, 77, 69, 68, 73, 65, 76, 14, 80, 6, 69, 68, 73, 65, 76, 32, 45, 10, - 79, 78, 32, 77, 69, 68, 73, 65, 76, 32, 8, 250, 200, 15, 72, 2, 82, 2, - 87, 3, 89, 6, 206, 200, 15, 76, 2, 77, 3, 78, 2, 219, 217, 10, 32, 238, - 1, 104, 6, 69, 84, 84, 69, 82, 32, 185, 13, 15, 79, 71, 79, 71, 82, 65, - 77, 32, 75, 72, 65, 77, 84, 73, 32, 232, 1, 238, 1, 65, 50, 69, 146, 1, - 71, 50, 75, 254, 1, 77, 134, 1, 78, 58, 82, 86, 83, 130, 3, 84, 198, 1, - 87, 178, 227, 11, 68, 214, 6, 85, 22, 86, 186, 201, 1, 73, 42, 76, 246, - 193, 1, 66, 2, 67, 2, 74, 2, 80, 138, 69, 72, 2, 89, 187, 2, 79, 7, 144, - 253, 14, 4, 73, 84, 79, 78, 219, 74, 85, 9, 77, 17, 65, 83, 84, 69, 82, - 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 6, 42, 71, 186, 213, 10, - 89, 135, 181, 3, 78, 2, 183, 213, 10, 72, 6, 154, 200, 9, 82, 218, 251, - 5, 72, 187, 2, 65, 44, 32, 2, 72, 65, 219, 197, 15, 65, 43, 25, 4, 77, - 84, 73, 32, 40, 134, 1, 68, 34, 84, 242, 217, 8, 78, 230, 162, 6, 67, 2, - 72, 2, 74, 170, 37, 76, 226, 31, 70, 2, 71, 2, 82, 2, 83, 2, 88, 3, 90, - 6, 242, 252, 14, 68, 139, 69, 72, 4, 211, 252, 14, 84, 12, 36, 3, 79, 78, - 32, 219, 195, 15, 65, 10, 60, 2, 66, 66, 218, 200, 13, 74, 254, 183, 1, - 78, 199, 66, 69, 4, 150, 195, 15, 65, 3, 69, 10, 214, 216, 8, 78, 238, - 231, 6, 71, 2, 89, 187, 2, 65, 4, 252, 220, 2, 12, 85, 77, 65, 73, 32, - 80, 65, 76, 65, 85, 78, 71, 199, 229, 12, 65, 50, 90, 72, 192, 221, 2, 9, - 71, 65, 87, 32, 75, 65, 82, 69, 78, 158, 225, 12, 83, 187, 2, 65, 44, 66, - 65, 197, 1, 11, 87, 69, 32, 80, 65, 76, 65, 85, 78, 71, 32, 41, 17, 2, - 78, 32, 38, 134, 1, 78, 246, 196, 13, 74, 2, 80, 2, 84, 130, 179, 1, 66, - 2, 67, 2, 71, 2, 75, 138, 69, 68, 2, 70, 2, 72, 2, 90, 187, 2, 65, 6, - 250, 188, 15, 78, 2, 89, 187, 2, 65, 4, 202, 196, 13, 67, 3, 83, 36, 38, - 65, 138, 247, 14, 84, 139, 69, 72, 31, 41, 8, 73, 32, 76, 65, 73, 78, 71, - 32, 28, 82, 78, 206, 227, 11, 68, 190, 146, 3, 66, 2, 71, 2, 74, 170, 37, - 76, 227, 31, 70, 4, 142, 187, 15, 78, 3, 89, 6, 92, 17, 69, 83, 84, 69, - 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 207, 188, 15, 65, 4, - 194, 203, 10, 80, 203, 246, 2, 84, 6, 162, 176, 14, 79, 166, 60, 81, 139, - 63, 72, 4, 56, 6, 75, 72, 65, 77, 84, 73, 1, 4, 83, 72, 65, 78, 2, 29, 5, - 32, 82, 69, 68, 85, 2, 193, 132, 1, 2, 80, 76, 90, 76, 2, 72, 65, 20, 4, - 73, 71, 78, 32, 233, 6, 6, 89, 77, 66, 79, 76, 32, 20, 199, 186, 9, 78, - 52, 202, 3, 65, 32, 12, 75, 72, 65, 77, 84, 73, 32, 84, 79, 78, 69, 45, - 30, 83, 132, 2, 15, 84, 65, 73, 32, 76, 65, 73, 78, 71, 32, 84, 79, 78, - 69, 45, 28, 22, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, - 82, 69, 78, 32, 84, 79, 78, 69, 206, 244, 2, 68, 252, 163, 5, 19, 82, 85, - 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 32, 84, 79, 78, 69, 45, 204, - 170, 3, 9, 80, 65, 79, 32, 75, 65, 82, 69, 78, 148, 114, 6, 76, 73, 84, - 84, 76, 69, 199, 187, 2, 86, 4, 234, 162, 14, 83, 255, 78, 78, 4, 178, - 182, 15, 49, 3, 51, 18, 40, 4, 72, 65, 78, 32, 147, 229, 14, 69, 16, 84, - 8, 67, 79, 85, 78, 67, 73, 76, 32, 84, 5, 84, 79, 78, 69, 45, 151, 182, - 14, 83, 6, 172, 196, 11, 8, 69, 77, 80, 72, 65, 84, 73, 67, 229, 239, 3, - 4, 84, 79, 78, 69, 8, 190, 180, 15, 50, 2, 51, 2, 53, 3, 54, 4, 146, 180, - 15, 50, 3, 53, 10, 11, 45, 10, 234, 179, 15, 49, 2, 50, 2, 51, 2, 52, 3, - 53, 18, 130, 1, 65, 132, 1, 2, 76, 79, 28, 5, 83, 72, 65, 78, 32, 192, - 173, 8, 4, 71, 69, 78, 73, 169, 97, 6, 67, 79, 77, 80, 76, 69, 8, 84, 5, - 73, 84, 79, 78, 32, 141, 174, 6, 10, 70, 79, 82, 69, 77, 69, 78, 84, 73, - 79, 6, 98, 69, 186, 249, 8, 84, 151, 158, 5, 79, 2, 217, 173, 8, 2, 67, - 65, 4, 26, 69, 207, 151, 14, 79, 2, 201, 12, 4, 88, 67, 76, 65, 24, 140, - 1, 20, 79, 78, 69, 32, 77, 65, 82, 75, 32, 83, 71, 65, 87, 32, 75, 65, - 82, 69, 78, 32, 221, 175, 9, 8, 65, 73, 32, 76, 65, 73, 78, 71, 4, 38, - 72, 129, 206, 8, 3, 75, 69, 32, 2, 139, 208, 8, 65, 56, 150, 2, 65, 76, - 9, 71, 69, 66, 65, 32, 75, 65, 82, 69, 20, 6, 75, 65, 89, 65, 72, 32, 40, - 4, 77, 79, 78, 32, 34, 83, 142, 1, 69, 40, 18, 87, 69, 83, 84, 69, 82, - 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 228, 182, 9, 2, 84, 65, - 142, 160, 2, 85, 22, 86, 187, 201, 1, 73, 8, 26, 73, 219, 172, 15, 65, 7, - 25, 4, 84, 79, 78, 32, 4, 223, 162, 13, 65, 2, 151, 236, 14, 78, 6, 190, - 149, 15, 69, 2, 79, 215, 22, 85, 4, 146, 152, 15, 73, 219, 19, 79, 10, - 80, 4, 72, 65, 78, 32, 157, 204, 8, 10, 71, 65, 87, 32, 75, 65, 82, 69, - 78, 32, 8, 54, 69, 20, 5, 70, 73, 78, 65, 76, 247, 167, 15, 65, 5, 199, - 173, 14, 32, 2, 227, 152, 15, 32, 4, 174, 133, 15, 69, 151, 14, 85, 230, - 17, 152, 2, 5, 45, 65, 82, 89, 32, 214, 4, 65, 222, 16, 66, 30, 69, 146, - 32, 73, 136, 1, 3, 75, 79, 32, 170, 10, 79, 162, 24, 85, 224, 8, 22, 89, - 73, 65, 75, 69, 78, 71, 32, 80, 85, 65, 67, 72, 85, 69, 32, 72, 77, 79, - 78, 71, 32, 216, 135, 2, 2, 80, 78, 220, 189, 12, 2, 78, 66, 27, 76, 32, - 130, 1, 67, 114, 84, 46, 83, 160, 1, 5, 85, 78, 73, 79, 78, 108, 4, 87, - 72, 73, 84, 246, 225, 11, 76, 210, 17, 73, 223, 160, 2, 80, 8, 52, 7, 73, - 82, 67, 76, 69, 68, 32, 135, 151, 14, 79, 6, 40, 2, 80, 76, 14, 84, 219, - 141, 8, 68, 2, 35, 85, 2, 21, 3, 73, 77, 69, 2, 219, 229, 11, 83, 6, 40, - 6, 81, 85, 65, 82, 69, 32, 87, 85, 4, 60, 9, 73, 78, 84, 69, 82, 83, 69, - 67, 84, 1, 2, 85, 78, 2, 215, 191, 11, 73, 2, 11, 77, 2, 179, 226, 11, - 77, 7, 69, 15, 32, 79, 80, 69, 82, 65, 84, 79, 82, 32, 87, 73, 84, 72, - 32, 4, 150, 157, 11, 80, 131, 182, 3, 68, 2, 11, 69, 2, 233, 204, 12, 2, - 32, 86, 188, 2, 170, 2, 66, 252, 4, 10, 71, 32, 77, 85, 78, 68, 65, 82, - 73, 32, 142, 4, 73, 52, 2, 78, 68, 208, 4, 7, 84, 73, 79, 78, 65, 76, 32, - 228, 213, 1, 2, 85, 83, 140, 137, 8, 5, 77, 69, 32, 66, 65, 184, 212, 3, - 7, 90, 65, 82, 32, 65, 77, 85, 188, 160, 1, 8, 82, 82, 79, 87, 32, 78, - 79, 45, 231, 62, 75, 82, 52, 7, 65, 84, 65, 69, 65, 78, 32, 231, 157, 15, - 76, 80, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, 236, 2, 7, 78, 85, 77, 66, - 69, 82, 32, 133, 243, 9, 16, 67, 82, 85, 67, 73, 70, 79, 82, 77, 32, 78, - 85, 77, 66, 69, 82, 62, 236, 1, 6, 70, 73, 78, 65, 76, 32, 154, 132, 2, - 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, 178, 215, 2, 65, 50, - 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, - 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, 18, 246, 250, 2, - 65, 54, 76, 250, 163, 4, 83, 190, 3, 89, 134, 214, 1, 75, 198, 75, 66, - 178, 216, 4, 78, 222, 105, 72, 171, 4, 77, 16, 134, 252, 2, 84, 134, 166, - 4, 79, 223, 137, 6, 70, 84, 84, 7, 76, 69, 84, 84, 69, 82, 32, 164, 2, 5, - 83, 73, 71, 78, 32, 191, 167, 13, 68, 54, 42, 65, 50, 69, 66, 73, 50, 79, - 47, 85, 11, 138, 137, 15, 78, 202, 17, 66, 2, 72, 3, 74, 15, 158, 159, - 13, 78, 210, 113, 76, 138, 109, 84, 174, 28, 71, 3, 77, 11, 194, 211, 14, - 68, 162, 70, 72, 2, 83, 3, 84, 11, 222, 152, 15, 78, 86, 76, 2, 80, 3, - 89, 11, 134, 153, 15, 67, 2, 68, 2, 75, 3, 82, 10, 100, 2, 77, 85, 20, 3, - 83, 85, 84, 182, 118, 73, 212, 234, 10, 2, 79, 74, 241, 253, 2, 3, 84, - 79, 89, 2, 139, 223, 14, 72, 2, 207, 150, 15, 85, 4, 208, 182, 11, 5, 76, - 32, 80, 79, 76, 187, 15, 82, 133, 1, 41, 8, 73, 78, 65, 71, 65, 82, 73, - 32, 130, 1, 140, 1, 7, 76, 69, 84, 84, 69, 82, 32, 248, 1, 5, 83, 73, 71, - 78, 32, 52, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 239, 229, 4, - 72, 94, 210, 1, 86, 226, 185, 11, 65, 38, 68, 82, 82, 34, 84, 230, 5, 85, - 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, - 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 89, 186, 2, 69, 3, 79, 6, - 202, 148, 5, 79, 183, 255, 9, 65, 10, 254, 240, 9, 83, 238, 203, 1, 65, - 231, 147, 3, 86, 24, 250, 235, 4, 80, 238, 39, 86, 246, 171, 6, 65, 38, - 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 158, 48, 68, 255, 160, - 14, 80, 4, 154, 146, 15, 83, 15, 72, 254, 4, 216, 1, 3, 71, 65, 84, 192, - 7, 6, 73, 84, 72, 69, 82, 32, 194, 3, 83, 136, 1, 2, 85, 84, 242, 1, 87, - 148, 16, 3, 88, 84, 32, 144, 186, 8, 4, 80, 84, 85, 78, 216, 133, 3, 2, - 67, 75, 238, 161, 2, 82, 207, 144, 1, 76, 162, 1, 152, 1, 4, 73, 86, 69, - 32, 161, 233, 12, 27, 69, 68, 32, 68, 79, 85, 66, 76, 69, 32, 86, 69, 82, - 84, 73, 67, 65, 76, 32, 66, 65, 82, 32, 68, 79, 85, 66, 160, 1, 152, 1, - 8, 67, 73, 82, 67, 76, 69, 68, 32, 224, 1, 9, 68, 73, 65, 71, 79, 78, 65, - 76, 32, 184, 1, 8, 83, 81, 85, 65, 82, 69, 68, 32, 231, 206, 8, 65, 78, - 112, 5, 68, 73, 71, 73, 84, 28, 7, 78, 85, 77, 66, 69, 82, 32, 180, 3, 2, - 76, 65, 202, 228, 13, 84, 135, 4, 83, 2, 209, 157, 13, 2, 32, 90, 20, 50, - 84, 214, 133, 2, 69, 46, 70, 42, 78, 31, 83, 6, 150, 135, 2, 72, 47, 87, - 6, 38, 77, 206, 202, 13, 67, 183, 4, 68, 2, 49, 10, 73, 68, 68, 76, 69, - 32, 82, 73, 71, 72, 2, 17, 2, 84, 32, 2, 41, 8, 84, 79, 32, 76, 79, 87, - 69, 82, 2, 153, 137, 12, 2, 32, 67, 74, 142, 1, 76, 70, 85, 230, 154, 12, - 68, 138, 29, 81, 216, 129, 1, 2, 67, 82, 218, 19, 65, 234, 19, 73, 2, 87, - 134, 8, 82, 194, 157, 1, 80, 3, 83, 54, 26, 65, 183, 237, 10, 69, 52, - 129, 246, 8, 5, 84, 73, 78, 32, 67, 2, 171, 233, 13, 80, 18, 158, 1, 65, - 216, 1, 17, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, 65, 78, 32, 78, 79, - 82, 32, 37, 14, 76, 69, 83, 83, 45, 84, 72, 65, 78, 32, 78, 79, 82, 32, - 6, 92, 3, 32, 83, 85, 85, 16, 80, 80, 82, 79, 88, 73, 77, 65, 84, 69, 76, - 89, 32, 78, 79, 82, 4, 30, 66, 1, 3, 80, 69, 82, 2, 217, 245, 6, 8, 83, - 69, 84, 32, 79, 70, 32, 78, 2, 233, 48, 5, 32, 65, 67, 84, 85, 6, 254, - 144, 8, 69, 131, 227, 4, 76, 6, 218, 144, 8, 69, 223, 226, 4, 71, 6, 26, - 84, 151, 255, 12, 83, 4, 76, 5, 73, 78, 71, 32, 68, 185, 210, 10, 8, 32, - 87, 73, 84, 72, 32, 69, 71, 2, 181, 252, 7, 2, 79, 76, 64, 40, 4, 82, 65, - 76, 32, 159, 203, 14, 69, 62, 48, 6, 67, 72, 69, 83, 83, 32, 191, 198, - 14, 70, 60, 74, 75, 234, 165, 13, 66, 38, 69, 130, 5, 80, 22, 81, 38, 82, - 131, 2, 84, 20, 44, 5, 78, 73, 71, 72, 84, 251, 166, 13, 73, 15, 167, - 167, 13, 32, 246, 2, 70, 32, 184, 8, 2, 65, 32, 252, 6, 3, 76, 73, 78, - 195, 223, 3, 83, 174, 1, 90, 77, 64, 4, 83, 72, 69, 81, 20, 8, 84, 65, - 73, 32, 76, 85, 69, 32, 255, 230, 13, 76, 4, 25, 4, 79, 79, 78, 32, 4, - 138, 143, 8, 87, 211, 222, 5, 83, 2, 163, 189, 9, 69, 166, 1, 160, 1, 7, - 76, 69, 84, 84, 69, 82, 32, 244, 2, 8, 83, 73, 71, 78, 32, 76, 65, 69, - 22, 84, 76, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 155, 138, 13, + 76, 2, 21, 3, 32, 78, 79, 2, 195, 142, 15, 84, 16, 40, 4, 83, 73, 71, 78, + 139, 164, 16, 88, 15, 11, 32, 12, 48, 3, 73, 78, 32, 81, 5, 87, 73, 84, + 72, 32, 8, 254, 206, 3, 76, 22, 82, 140, 191, 9, 5, 68, 79, 85, 66, 76, + 183, 238, 1, 84, 4, 242, 179, 12, 85, 187, 238, 2, 68, 7, 11, 32, 4, 244, + 83, 5, 77, 85, 76, 84, 73, 203, 155, 12, 85, 228, 3, 44, 5, 72, 82, 79, + 79, 77, 21, 2, 73, 67, 5, 215, 243, 14, 32, 224, 3, 30, 32, 109, 3, 65, + 76, 32, 6, 64, 4, 78, 65, 84, 85, 20, 3, 83, 72, 65, 209, 43, 2, 70, 76, + 2, 187, 212, 10, 82, 2, 227, 191, 11, 82, 218, 3, 64, 8, 75, 69, 89, 66, + 79, 65, 82, 68, 66, 83, 131, 248, 3, 78, 5, 41, 8, 32, 87, 73, 84, 72, + 32, 74, 65, 2, 231, 254, 13, 67, 212, 3, 48, 6, 89, 77, 66, 79, 76, 32, + 239, 177, 11, 67, 210, 3, 230, 2, 66, 238, 1, 67, 230, 9, 68, 250, 2, 69, + 162, 1, 70, 166, 2, 71, 112, 9, 65, 82, 80, 69, 71, 71, 73, 65, 84, 168, + 1, 2, 72, 65, 126, 75, 198, 3, 76, 138, 1, 77, 186, 2, 78, 162, 1, 79, + 218, 2, 80, 200, 2, 2, 81, 85, 146, 2, 82, 198, 2, 83, 226, 5, 84, 182, + 10, 86, 42, 88, 42, 87, 232, 145, 9, 9, 73, 78, 86, 69, 82, 84, 69, 68, + 32, 143, 210, 6, 90, 20, 36, 5, 69, 71, 73, 78, 32, 59, 82, 8, 142, 15, + 80, 30, 83, 250, 251, 7, 66, 151, 245, 7, 84, 12, 24, 2, 65, 67, 35, 69, + 4, 250, 229, 15, 75, 155, 53, 69, 8, 26, 86, 255, 216, 15, 65, 6, 32, 2, + 73, 83, 183, 154, 16, 69, 5, 179, 26, 32, 84, 120, 2, 65, 69, 34, 76, 94, + 79, 194, 7, 82, 242, 11, 32, 224, 32, 7, 73, 82, 67, 76, 69, 32, 88, 233, + 227, 5, 2, 85, 84, 2, 11, 83, 2, 219, 212, 15, 85, 8, 42, 73, 249, 23, 5, + 85, 83, 84, 69, 82, 4, 26, 77, 187, 241, 13, 86, 2, 199, 219, 9, 65, 64, + 26, 77, 219, 149, 16, 68, 62, 64, 7, 66, 73, 78, 73, 78, 71, 32, 153, + 151, 6, 3, 77, 79, 78, 60, 202, 1, 65, 80, 7, 77, 65, 82, 67, 65, 84, 79, + 56, 2, 68, 79, 56, 2, 85, 80, 28, 2, 70, 76, 40, 5, 72, 65, 82, 77, 79, + 22, 83, 142, 2, 84, 250, 181, 7, 66, 172, 199, 3, 2, 76, 79, 143, 147, 5, + 82, 6, 76, 5, 67, 67, 69, 78, 84, 37, 10, 85, 71, 77, 69, 78, 84, 65, 84, + 73, 79, 5, 205, 2, 5, 45, 83, 84, 65, 67, 2, 219, 129, 15, 78, 6, 52, 2, + 87, 78, 168, 3, 2, 85, 66, 239, 244, 15, 73, 2, 149, 248, 14, 2, 32, 66, + 12, 248, 76, 2, 65, 71, 159, 199, 15, 73, 2, 223, 233, 14, 78, 12, 132, + 1, 9, 78, 65, 80, 32, 80, 73, 90, 90, 73, 22, 84, 232, 206, 4, 11, 80, + 82, 69, 67, 72, 71, 69, 83, 65, 78, 71, 247, 138, 6, 77, 2, 231, 130, 12, + 67, 6, 44, 5, 65, 67, 67, 65, 84, 199, 129, 16, 69, 4, 40, 4, 73, 83, 83, + 73, 243, 145, 16, 79, 2, 251, 242, 15, 77, 10, 34, 82, 165, 158, 12, 2, + 69, 78, 8, 28, 2, 73, 80, 199, 6, 69, 2, 173, 129, 12, 6, 76, 69, 32, 84, + 79, 78, 4, 22, 79, 179, 2, 69, 2, 147, 130, 15, 73, 24, 98, 65, 142, 1, + 69, 92, 6, 79, 85, 66, 76, 69, 32, 145, 169, 10, 7, 82, 85, 77, 32, 67, + 76, 69, 10, 96, 2, 32, 67, 20, 2, 77, 80, 204, 29, 4, 83, 72, 69, 68, + 137, 200, 15, 5, 76, 32, 83, 69, 71, 2, 255, 194, 8, 65, 5, 175, 132, 14, + 32, 4, 40, 3, 67, 82, 69, 29, 3, 71, 82, 69, 2, 181, 25, 3, 83, 67, 69, + 2, 155, 233, 8, 69, 6, 242, 21, 83, 254, 6, 66, 191, 135, 5, 70, 14, 32, + 3, 78, 68, 32, 151, 16, 73, 10, 74, 80, 30, 83, 204, 13, 3, 79, 70, 32, + 174, 238, 7, 66, 151, 245, 7, 84, 2, 185, 227, 12, 2, 72, 82, 2, 231, + 147, 15, 76, 34, 104, 6, 69, 82, 77, 65, 84, 65, 22, 73, 122, 79, 102, + 32, 144, 36, 3, 85, 83, 65, 213, 230, 3, 2, 76, 65, 5, 175, 218, 13, 32, + 10, 22, 78, 135, 34, 86, 8, 56, 9, 71, 69, 82, 69, 68, 32, 84, 82, 69, + 239, 20, 65, 6, 137, 242, 5, 4, 77, 79, 76, 79, 6, 208, 25, 3, 85, 82, + 45, 203, 167, 14, 82, 18, 54, 32, 56, 7, 76, 73, 83, 83, 65, 78, 68, 23, + 82, 6, 25, 4, 67, 76, 69, 70, 7, 157, 13, 3, 32, 79, 84, 4, 251, 134, 6, + 79, 8, 84, 9, 65, 67, 69, 32, 78, 79, 84, 69, 32, 37, 8, 69, 71, 79, 82, + 73, 65, 78, 32, 4, 184, 227, 8, 2, 78, 79, 27, 83, 4, 250, 2, 67, 3, 70, + 8, 44, 3, 76, 70, 32, 205, 8, 3, 85, 80, 84, 6, 52, 3, 80, 69, 68, 210, + 223, 3, 78, 151, 181, 8, 82, 2, 139, 199, 5, 65, 24, 48, 6, 73, 69, 86, + 65, 78, 32, 139, 223, 14, 79, 22, 178, 1, 67, 46, 69, 68, 7, 81, 85, 65, + 82, 84, 69, 82, 62, 70, 148, 220, 3, 4, 72, 65, 76, 70, 0, 5, 87, 72, 79, + 76, 69, 173, 224, 1, 9, 82, 69, 67, 73, 84, 65, 84, 73, 86, 2, 11, 32, 2, + 11, 67, 2, 235, 163, 5, 76, 6, 64, 5, 73, 71, 72, 84, 72, 221, 203, 14, + 5, 78, 68, 32, 79, 70, 4, 253, 129, 6, 10, 32, 78, 79, 84, 69, 32, 83, + 84, 69, 77, 4, 218, 14, 76, 185, 205, 3, 4, 73, 78, 65, 76, 8, 44, 4, 79, + 78, 71, 65, 217, 13, 2, 69, 70, 7, 11, 32, 4, 28, 3, 73, 77, 80, 3, 80, + 2, 193, 2, 7, 69, 82, 70, 69, 67, 84, 65, 18, 104, 2, 65, 88, 20, 2, 69, + 90, 20, 5, 73, 78, 73, 77, 65, 44, 3, 79, 79, 78, 25, 4, 85, 76, 84, 73, + 2, 167, 224, 15, 73, 2, 175, 226, 15, 90, 7, 11, 32, 4, 226, 142, 12, 82, + 195, 83, 66, 4, 189, 17, 2, 32, 78, 4, 60, 11, 80, 76, 69, 32, 77, 69, + 65, 83, 85, 82, 69, 15, 32, 2, 11, 32, 2, 223, 141, 12, 82, 10, 120, 4, + 69, 66, 69, 78, 212, 26, 3, 85, 76, 76, 236, 225, 5, 3, 65, 84, 85, 189, + 227, 6, 7, 79, 84, 69, 72, 69, 65, 68, 2, 253, 228, 14, 4, 83, 84, 73, + 77, 32, 88, 2, 78, 69, 120, 14, 82, 78, 65, 77, 69, 78, 84, 32, 83, 84, + 82, 79, 75, 69, 107, 84, 6, 92, 18, 32, 72, 85, 78, 68, 82, 69, 68, 32, + 84, 87, 69, 78, 84, 89, 45, 69, 73, 159, 20, 45, 4, 181, 13, 2, 71, 72, + 22, 11, 45, 22, 170, 166, 3, 49, 194, 214, 12, 50, 2, 51, 2, 52, 2, 53, + 2, 54, 2, 55, 2, 56, 3, 57, 4, 173, 4, 3, 84, 65, 86, 18, 96, 9, 65, 82, + 69, 78, 84, 72, 69, 83, 73, 0, 2, 76, 85, 18, 69, 142, 1, 79, 163, 239, + 13, 73, 2, 243, 22, 83, 6, 68, 4, 68, 65, 76, 32, 49, 9, 83, 32, 83, 85, + 66, 80, 85, 78, 67, 4, 26, 85, 243, 184, 15, 77, 2, 215, 184, 15, 80, 2, + 255, 210, 13, 84, 6, 48, 2, 68, 65, 245, 5, 5, 82, 82, 69, 67, 84, 2, + 219, 240, 13, 84, 12, 76, 6, 65, 82, 84, 69, 82, 32, 125, 9, 73, 78, 68, + 73, 67, 69, 83, 73, 77, 8, 60, 5, 84, 79, 78, 69, 32, 234, 208, 3, 78, + 151, 181, 8, 82, 4, 26, 83, 187, 142, 5, 70, 2, 241, 247, 15, 3, 72, 65, + 82, 4, 17, 2, 65, 32, 4, 230, 186, 12, 65, 161, 186, 3, 3, 66, 65, 83, + 14, 22, 69, 175, 1, 73, 10, 72, 4, 80, 69, 65, 84, 81, 10, 86, 69, 82, + 83, 69, 32, 70, 73, 78, 65, 8, 56, 8, 69, 68, 32, 70, 73, 71, 85, 82, + 179, 163, 14, 32, 6, 179, 221, 5, 69, 2, 207, 4, 76, 4, 48, 2, 71, 72, + 57, 6, 78, 70, 79, 82, 90, 65, 2, 33, 6, 84, 32, 82, 69, 80, 69, 2, 179, + 169, 10, 65, 2, 171, 181, 8, 78, 48, 136, 1, 6, 67, 65, 78, 68, 73, 67, + 62, 69, 162, 1, 72, 54, 73, 252, 1, 6, 81, 85, 65, 82, 69, 32, 252, 158, + 8, 2, 85, 66, 251, 100, 79, 4, 17, 2, 85, 83, 5, 209, 234, 13, 5, 32, 70, + 76, 69, 88, 14, 32, 2, 77, 73, 223, 201, 15, 71, 12, 64, 6, 66, 82, 69, + 86, 73, 83, 1, 6, 77, 73, 78, 73, 77, 65, 6, 11, 32, 6, 134, 13, 87, 246, + 242, 11, 82, 195, 83, 66, 6, 84, 3, 79, 82, 84, 129, 239, 5, 3, 65, 82, + 80, 14, 32, 4, 78, 71, 76, 69, 43, 88, 2, 17, 2, 32, 66, 2, 207, 134, 14, + 65, 12, 18, 45, 79, 84, 4, 242, 7, 76, 217, 207, 14, 11, 83, 84, 82, 73, + 78, 71, 32, 70, 82, 69, 84, 8, 52, 3, 69, 69, 78, 1, 6, 89, 45, 70, 79, + 85, 82, 4, 193, 12, 2, 84, 72, 6, 26, 78, 167, 239, 15, 66, 4, 229, 9, 7, + 79, 84, 69, 72, 69, 65, 68, 60, 132, 1, 6, 69, 77, 80, 85, 83, 32, 154, + 4, 72, 88, 2, 87, 79, 84, 7, 79, 82, 67, 85, 76, 85, 83, 46, 82, 141, 3, + 3, 85, 82, 78, 16, 232, 1, 27, 73, 77, 80, 69, 82, 70, 69, 67, 84, 85, + 77, 32, 67, 85, 77, 32, 80, 82, 79, 76, 65, 84, 73, 79, 78, 69, 32, 129, + 1, 25, 80, 69, 82, 70, 69, 67, 84, 85, 77, 32, 67, 85, 77, 32, 80, 82, + 79, 76, 65, 84, 73, 79, 78, 69, 32, 10, 60, 10, 73, 77, 80, 69, 82, 70, + 69, 67, 84, 65, 131, 1, 80, 9, 249, 210, 5, 11, 32, 68, 73, 77, 73, 78, + 85, 84, 73, 79, 78, 6, 60, 3, 73, 77, 80, 41, 8, 80, 69, 82, 70, 69, 67, + 84, 65, 2, 245, 197, 15, 5, 69, 82, 70, 69, 67, 5, 197, 194, 8, 12, 32, + 68, 73, 77, 73, 78, 85, 84, 73, 79, 78, 45, 6, 72, 2, 82, 69, 249, 5, 11, + 73, 82, 84, 89, 45, 83, 69, 67, 79, 78, 68, 2, 11, 69, 2, 11, 45, 2, 11, + 76, 2, 37, 7, 73, 78, 69, 32, 83, 84, 65, 2, 155, 231, 14, 70, 5, 189, + 221, 11, 6, 32, 82, 69, 83, 85, 80, 27, 33, 6, 73, 65, 78, 71, 76, 69, + 24, 128, 1, 10, 32, 78, 79, 84, 69, 72, 69, 65, 68, 32, 61, 17, 45, 82, + 79, 85, 78, 68, 32, 78, 79, 84, 69, 72, 69, 65, 68, 32, 68, 20, 58, 68, + 24, 3, 85, 80, 32, 38, 82, 25, 3, 76, 69, 70, 4, 93, 3, 79, 87, 78, 8, + 34, 82, 78, 87, 183, 198, 12, 66, 4, 21, 3, 73, 71, 72, 4, 11, 84, 4, 11, + 32, 4, 26, 87, 183, 198, 12, 66, 2, 11, 72, 2, 243, 155, 14, 73, 7, 11, + 32, 4, 146, 192, 8, 83, 203, 164, 7, 85, 4, 36, 3, 79, 73, 68, 207, 161, + 15, 73, 2, 205, 139, 13, 5, 32, 78, 79, 84, 69, 6, 92, 4, 72, 79, 76, 69, + 253, 204, 8, 13, 73, 84, 72, 32, 70, 73, 78, 71, 69, 82, 78, 65, 73, 4, + 11, 32, 4, 210, 187, 3, 78, 151, 181, 8, 82, 232, 3, 224, 2, 15, 67, 79, + 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 254, 1, 76, 212, 14, + 16, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, 122, + 83, 80, 16, 69, 65, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, + 69, 218, 9, 84, 204, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 140, 180, 9, 3, 80, 65, 79, 175, 154, 4, 68, 16, 66, 77, 165, 1, 11, 83, + 72, 65, 78, 32, 77, 69, 68, 73, 65, 76, 14, 80, 6, 69, 68, 73, 65, 76, + 32, 45, 10, 79, 78, 32, 77, 69, 68, 73, 65, 76, 32, 8, 170, 220, 15, 72, + 2, 82, 2, 87, 3, 89, 6, 254, 219, 15, 76, 2, 77, 3, 78, 2, 183, 228, 10, + 32, 238, 1, 104, 6, 69, 84, 84, 69, 82, 32, 185, 13, 15, 79, 71, 79, 71, + 82, 65, 77, 32, 75, 72, 65, 77, 84, 73, 32, 232, 1, 238, 1, 65, 50, 69, + 146, 1, 71, 50, 75, 254, 1, 77, 134, 1, 78, 58, 82, 86, 83, 130, 3, 84, + 198, 1, 87, 142, 243, 11, 68, 214, 6, 85, 22, 86, 166, 202, 1, 73, 42, + 76, 222, 196, 1, 66, 2, 67, 2, 74, 2, 80, 138, 69, 72, 2, 89, 187, 2, 79, + 7, 192, 144, 15, 4, 73, 84, 79, 78, 219, 74, 85, 9, 77, 17, 65, 83, 84, + 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 6, 42, 71, 150, + 224, 10, 89, 199, 187, 3, 78, 2, 147, 224, 10, 72, 6, 130, 205, 9, 82, + 162, 138, 6, 72, 187, 2, 65, 44, 32, 2, 72, 65, 139, 217, 15, 65, 43, 25, + 4, 77, 84, 73, 32, 40, 134, 1, 68, 34, 84, 162, 223, 8, 78, 230, 176, 6, + 67, 2, 72, 2, 74, 170, 37, 76, 226, 31, 70, 2, 71, 2, 82, 2, 83, 2, 88, + 3, 90, 6, 162, 144, 15, 68, 139, 69, 72, 4, 131, 144, 15, 84, 12, 36, 3, + 79, 78, 32, 139, 215, 15, 65, 10, 60, 2, 66, 66, 162, 217, 13, 74, 230, + 186, 1, 78, 199, 66, 69, 4, 198, 214, 15, 65, 3, 69, 10, 134, 222, 8, 78, + 238, 245, 6, 71, 2, 89, 187, 2, 65, 4, 132, 221, 2, 12, 85, 77, 65, 73, + 32, 80, 65, 76, 65, 85, 78, 71, 239, 248, 12, 65, 50, 90, 72, 200, 221, + 2, 9, 71, 65, 87, 32, 75, 65, 82, 69, 78, 198, 244, 12, 83, 187, 2, 65, + 44, 66, 65, 197, 1, 11, 87, 69, 32, 80, 65, 76, 65, 85, 78, 71, 32, 41, + 17, 2, 78, 32, 38, 134, 1, 78, 190, 213, 13, 74, 2, 80, 2, 84, 234, 181, + 1, 66, 2, 67, 2, 71, 2, 75, 138, 69, 68, 2, 70, 2, 72, 2, 90, 187, 2, 65, + 6, 170, 208, 15, 78, 2, 89, 187, 2, 65, 4, 146, 213, 13, 67, 3, 83, 36, + 38, 65, 186, 138, 15, 84, 139, 69, 72, 31, 41, 8, 73, 32, 76, 65, 73, 78, + 71, 32, 28, 82, 78, 170, 243, 11, 68, 146, 150, 3, 66, 2, 71, 2, 74, 170, + 37, 76, 227, 31, 70, 4, 190, 206, 15, 78, 3, 89, 6, 92, 17, 69, 83, 84, + 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 255, 207, 15, 65, + 4, 158, 214, 10, 80, 183, 252, 2, 84, 6, 182, 193, 14, 79, 194, 62, 81, + 139, 63, 72, 4, 56, 6, 75, 72, 65, 77, 84, 73, 1, 4, 83, 72, 65, 78, 2, + 29, 5, 32, 82, 69, 68, 85, 2, 241, 132, 1, 2, 80, 76, 90, 76, 2, 72, 65, + 20, 4, 73, 71, 78, 32, 233, 6, 6, 89, 77, 66, 79, 76, 32, 20, 175, 191, + 9, 78, 52, 202, 3, 65, 32, 12, 75, 72, 65, 77, 84, 73, 32, 84, 79, 78, + 69, 45, 30, 83, 132, 2, 15, 84, 65, 73, 32, 76, 65, 73, 78, 71, 32, 84, + 79, 78, 69, 45, 28, 22, 87, 69, 83, 84, 69, 82, 78, 32, 80, 87, 79, 32, + 75, 65, 82, 69, 78, 32, 84, 79, 78, 69, 222, 244, 2, 68, 136, 169, 5, 19, + 82, 85, 77, 65, 73, 32, 80, 65, 76, 65, 85, 78, 71, 32, 84, 79, 78, 69, + 45, 140, 180, 3, 9, 80, 65, 79, 32, 75, 65, 82, 69, 78, 148, 115, 6, 76, + 73, 84, 84, 76, 69, 155, 191, 2, 86, 4, 210, 192, 14, 83, 199, 68, 78, 4, + 226, 201, 15, 49, 3, 51, 18, 40, 4, 72, 65, 78, 32, 195, 248, 14, 69, 16, + 84, 8, 67, 79, 85, 78, 67, 73, 76, 32, 84, 5, 84, 79, 78, 69, 45, 199, + 201, 14, 83, 6, 136, 211, 11, 8, 69, 77, 80, 72, 65, 84, 73, 67, 185, + 244, 3, 4, 84, 79, 78, 69, 8, 238, 199, 15, 50, 2, 51, 2, 53, 3, 54, 4, + 194, 199, 15, 50, 3, 53, 10, 11, 45, 10, 154, 199, 15, 49, 2, 50, 2, 51, + 2, 52, 3, 53, 18, 130, 1, 65, 128, 1, 2, 76, 79, 28, 5, 83, 72, 65, 78, + 32, 228, 178, 8, 4, 71, 69, 78, 73, 217, 96, 6, 67, 79, 77, 80, 76, 69, + 8, 84, 5, 73, 84, 79, 78, 32, 185, 179, 6, 10, 70, 79, 82, 69, 77, 69, + 78, 84, 73, 79, 6, 94, 69, 246, 175, 13, 84, 139, 121, 79, 2, 253, 178, + 8, 2, 67, 65, 4, 26, 69, 255, 168, 14, 79, 2, 201, 12, 4, 88, 67, 76, 65, + 24, 140, 1, 20, 79, 78, 69, 32, 77, 65, 82, 75, 32, 83, 71, 65, 87, 32, + 75, 65, 82, 69, 78, 32, 201, 180, 9, 8, 65, 73, 32, 76, 65, 73, 78, 71, + 4, 38, 72, 181, 211, 8, 3, 75, 69, 32, 2, 191, 213, 8, 65, 56, 150, 2, + 65, 76, 9, 71, 69, 66, 65, 32, 75, 65, 82, 69, 20, 6, 75, 65, 89, 65, 72, + 32, 40, 4, 77, 79, 78, 32, 34, 83, 142, 1, 69, 40, 18, 87, 69, 83, 84, + 69, 82, 78, 32, 80, 87, 79, 32, 75, 65, 82, 69, 78, 32, 212, 187, 9, 2, + 84, 65, 254, 170, 2, 85, 22, 86, 167, 202, 1, 73, 8, 26, 73, 143, 192, + 15, 65, 7, 25, 4, 84, 79, 78, 32, 4, 171, 179, 13, 65, 2, 203, 255, 14, + 78, 6, 242, 168, 15, 69, 2, 79, 215, 22, 85, 4, 198, 171, 15, 73, 219, + 19, 79, 10, 80, 4, 72, 65, 78, 32, 209, 209, 8, 10, 71, 65, 87, 32, 75, + 65, 82, 69, 78, 32, 8, 54, 69, 20, 5, 70, 73, 78, 65, 76, 171, 187, 15, + 65, 5, 251, 192, 14, 32, 2, 151, 172, 15, 32, 4, 226, 152, 15, 69, 151, + 14, 85, 232, 17, 152, 2, 5, 45, 65, 82, 89, 32, 214, 4, 65, 222, 16, 66, + 30, 69, 146, 32, 73, 136, 1, 3, 75, 79, 32, 170, 10, 79, 166, 24, 85, + 224, 8, 22, 89, 73, 65, 75, 69, 78, 71, 32, 80, 85, 65, 67, 72, 85, 69, + 32, 72, 77, 79, 78, 71, 32, 232, 135, 2, 2, 80, 78, 252, 208, 12, 2, 78, + 66, 27, 76, 32, 130, 1, 67, 114, 84, 46, 83, 160, 1, 5, 85, 78, 73, 79, + 78, 108, 4, 87, 72, 73, 84, 214, 241, 11, 76, 210, 17, 73, 151, 162, 2, + 80, 8, 52, 7, 73, 82, 67, 76, 69, 68, 32, 159, 168, 14, 79, 6, 40, 2, 80, + 76, 14, 84, 251, 146, 8, 68, 2, 35, 85, 2, 21, 3, 73, 77, 69, 2, 187, + 245, 11, 83, 6, 40, 6, 81, 85, 65, 82, 69, 32, 87, 85, 4, 60, 9, 73, 78, + 84, 69, 82, 83, 69, 67, 84, 1, 2, 85, 78, 2, 147, 207, 11, 73, 2, 11, 77, + 2, 147, 242, 11, 77, 7, 69, 15, 32, 79, 80, 69, 82, 65, 84, 79, 82, 32, + 87, 73, 84, 72, 32, 4, 246, 171, 11, 80, 215, 186, 3, 68, 2, 11, 69, 2, + 205, 220, 12, 2, 32, 86, 188, 2, 170, 2, 66, 252, 4, 10, 71, 32, 77, 85, + 78, 68, 65, 82, 73, 32, 142, 4, 73, 52, 2, 78, 68, 208, 4, 7, 84, 73, 79, + 78, 65, 76, 32, 132, 214, 1, 2, 85, 83, 148, 147, 8, 5, 77, 69, 32, 66, + 65, 220, 218, 3, 7, 90, 65, 82, 32, 65, 77, 85, 164, 163, 1, 8, 82, 82, + 79, 87, 32, 78, 79, 45, 231, 62, 75, 82, 52, 7, 65, 84, 65, 69, 65, 78, + 32, 155, 177, 15, 76, 80, 160, 1, 7, 76, 69, 84, 84, 69, 82, 32, 236, 2, + 7, 78, 85, 77, 66, 69, 82, 32, 237, 253, 9, 16, 67, 82, 85, 67, 73, 70, + 79, 82, 77, 32, 78, 85, 77, 66, 69, 82, 62, 236, 1, 6, 70, 73, 78, 65, + 76, 32, 134, 132, 2, 84, 242, 119, 68, 34, 76, 50, 81, 214, 205, 1, 82, + 142, 220, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 198, 207, 1, 72, 234, 5, + 75, 174, 81, 66, 170, 225, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, + 18, 222, 251, 2, 65, 54, 76, 190, 168, 4, 83, 190, 3, 89, 174, 213, 1, + 75, 174, 81, 66, 170, 225, 4, 78, 222, 105, 72, 171, 4, 77, 16, 238, 252, + 2, 84, 202, 170, 4, 79, 255, 148, 6, 70, 84, 84, 7, 76, 69, 84, 84, 69, + 82, 32, 164, 2, 5, 83, 73, 71, 78, 32, 139, 184, 13, 68, 54, 42, 65, 50, + 69, 66, 73, 50, 79, 47, 85, 11, 190, 156, 15, 78, 202, 17, 66, 2, 72, 3, + 74, 15, 234, 175, 13, 78, 158, 114, 76, 166, 111, 84, 174, 28, 71, 3, 77, + 11, 246, 230, 14, 68, 162, 70, 72, 2, 83, 3, 84, 11, 146, 172, 15, 78, + 86, 76, 2, 80, 3, 89, 11, 186, 172, 15, 67, 2, 68, 2, 75, 3, 82, 10, 100, + 2, 77, 85, 20, 3, 83, 85, 84, 178, 118, 73, 184, 250, 10, 2, 79, 74, 197, + 129, 3, 3, 84, 79, 89, 2, 191, 242, 14, 72, 2, 131, 170, 15, 85, 4, 176, + 198, 11, 5, 76, 32, 80, 79, 76, 187, 15, 82, 133, 1, 41, 8, 73, 78, 65, + 71, 65, 82, 73, 32, 130, 1, 140, 1, 7, 76, 69, 84, 84, 69, 82, 32, 248, + 1, 5, 83, 73, 71, 78, 32, 52, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, + 32, 251, 230, 4, 72, 94, 210, 1, 86, 194, 201, 11, 65, 38, 68, 82, 82, + 34, 84, 230, 5, 85, 186, 202, 1, 73, 42, 76, 226, 195, 1, 78, 46, 83, 82, + 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 89, 186, 2, + 69, 3, 79, 6, 242, 152, 5, 79, 195, 142, 10, 65, 10, 230, 251, 9, 83, + 230, 208, 1, 65, 187, 151, 3, 86, 24, 234, 237, 4, 80, 166, 42, 86, 174, + 183, 6, 65, 38, 85, 186, 202, 1, 73, 198, 140, 2, 69, 3, 79, 4, 162, 48, + 68, 175, 180, 14, 80, 4, 206, 165, 15, 83, 15, 72, 254, 4, 216, 1, 3, 71, + 65, 84, 192, 7, 6, 73, 84, 72, 69, 82, 32, 194, 3, 83, 136, 1, 2, 85, 84, + 242, 1, 87, 148, 16, 3, 88, 84, 32, 180, 191, 8, 4, 80, 84, 85, 78, 148, + 144, 3, 2, 67, 75, 166, 163, 2, 82, 235, 146, 1, 76, 162, 1, 152, 1, 4, + 73, 86, 69, 32, 129, 249, 12, 27, 69, 68, 32, 68, 79, 85, 66, 76, 69, 32, + 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 32, 68, 79, 85, 66, 160, + 1, 152, 1, 8, 67, 73, 82, 67, 76, 69, 68, 32, 224, 1, 9, 68, 73, 65, 71, + 79, 78, 65, 76, 32, 184, 1, 8, 83, 81, 85, 65, 82, 69, 68, 32, 143, 212, + 8, 65, 78, 112, 5, 68, 73, 71, 73, 84, 28, 7, 78, 85, 77, 66, 69, 82, 32, + 180, 3, 2, 76, 65, 234, 245, 13, 84, 151, 4, 83, 2, 157, 174, 13, 2, 32, + 90, 20, 50, 84, 194, 133, 2, 69, 46, 70, 42, 78, 31, 83, 6, 130, 135, 2, + 72, 47, 87, 6, 38, 77, 238, 219, 13, 67, 183, 4, 68, 2, 49, 10, 73, 68, + 68, 76, 69, 32, 82, 73, 71, 72, 2, 17, 2, 84, 32, 2, 41, 8, 84, 79, 32, + 76, 79, 87, 69, 82, 2, 249, 152, 12, 2, 32, 67, 74, 142, 1, 76, 70, 85, + 202, 170, 12, 68, 138, 29, 81, 140, 131, 1, 2, 67, 82, 226, 19, 65, 234, + 19, 73, 2, 87, 150, 8, 82, 198, 159, 1, 80, 3, 83, 54, 26, 65, 199, 247, + 10, 69, 52, 217, 250, 8, 5, 84, 73, 78, 32, 67, 2, 219, 250, 13, 80, 18, + 158, 1, 65, 216, 1, 17, 71, 82, 69, 65, 84, 69, 82, 45, 84, 72, 65, 78, + 32, 78, 79, 82, 32, 37, 14, 76, 69, 83, 83, 45, 84, 72, 65, 78, 32, 78, + 79, 82, 32, 6, 92, 3, 32, 83, 85, 85, 16, 80, 80, 82, 79, 88, 73, 77, 65, + 84, 69, 76, 89, 32, 78, 79, 82, 4, 30, 66, 1, 3, 80, 69, 82, 2, 245, 250, + 6, 8, 83, 69, 84, 32, 79, 70, 32, 78, 2, 237, 48, 5, 32, 65, 67, 84, 85, + 6, 178, 150, 8, 69, 167, 237, 4, 76, 6, 142, 150, 8, 69, 131, 237, 4, 71, + 6, 26, 84, 227, 143, 13, 83, 4, 76, 5, 73, 78, 71, 32, 68, 145, 221, 10, + 8, 32, 87, 73, 84, 72, 32, 69, 71, 2, 217, 129, 8, 2, 79, 76, 64, 40, 4, + 82, 65, 76, 32, 211, 222, 14, 69, 62, 48, 6, 67, 72, 69, 83, 83, 32, 243, + 217, 14, 70, 60, 74, 75, 238, 182, 13, 66, 38, 69, 150, 5, 80, 22, 81, + 38, 82, 131, 2, 84, 20, 44, 5, 78, 73, 71, 72, 84, 147, 184, 13, 73, 15, + 191, 184, 13, 32, 246, 2, 70, 32, 184, 8, 2, 65, 32, 252, 6, 3, 76, 73, + 78, 155, 224, 3, 83, 174, 1, 90, 77, 64, 4, 83, 72, 69, 81, 20, 8, 84, + 65, 73, 32, 76, 85, 69, 32, 175, 248, 13, 76, 4, 25, 4, 79, 79, 78, 32, + 4, 190, 148, 8, 87, 183, 234, 5, 83, 2, 223, 199, 9, 69, 166, 1, 160, 1, + 7, 76, 69, 84, 84, 69, 82, 32, 244, 2, 8, 83, 73, 71, 78, 32, 76, 65, 69, + 22, 84, 76, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 231, 154, 13, 68, 102, 76, 6, 70, 73, 78, 65, 76, 32, 68, 4, 72, 73, 71, 72, 1, 3, 76, - 79, 87, 14, 162, 131, 13, 78, 154, 251, 1, 66, 2, 68, 2, 75, 2, 77, 3, - 86, 44, 11, 32, 44, 146, 1, 75, 2, 88, 34, 83, 138, 139, 11, 78, 162, - 169, 3, 84, 82, 80, 138, 69, 66, 2, 68, 2, 70, 2, 72, 2, 76, 2, 77, 2, - 81, 2, 86, 3, 89, 4, 158, 250, 14, 86, 187, 2, 65, 4, 254, 249, 14, 85, - 187, 2, 65, 5, 151, 252, 14, 86, 6, 252, 154, 7, 3, 79, 78, 69, 233, 80, + 79, 87, 14, 238, 147, 13, 78, 130, 254, 1, 66, 2, 68, 2, 75, 2, 77, 3, + 86, 44, 11, 32, 44, 146, 1, 75, 2, 88, 34, 83, 234, 153, 11, 78, 246, + 173, 3, 84, 82, 80, 138, 69, 66, 2, 68, 2, 70, 2, 72, 2, 76, 2, 77, 2, + 81, 2, 86, 3, 89, 4, 210, 141, 15, 86, 187, 2, 65, 4, 178, 141, 15, 85, + 187, 2, 65, 5, 203, 143, 15, 86, 6, 168, 160, 7, 3, 79, 78, 69, 221, 80, 8, 72, 65, 77, 32, 68, 73, 71, 73, 34, 110, 65, 46, 73, 30, 79, 38, 85, - 216, 175, 12, 11, 86, 79, 87, 69, 76, 32, 83, 72, 79, 82, 84, 135, 202, - 2, 69, 8, 146, 219, 10, 65, 182, 159, 4, 69, 3, 89, 4, 154, 250, 14, 73, - 3, 89, 9, 202, 218, 10, 65, 183, 159, 4, 89, 11, 166, 218, 10, 69, 182, - 159, 4, 85, 3, 89, 194, 1, 182, 1, 68, 106, 71, 72, 7, 76, 69, 84, 84, - 69, 82, 32, 214, 2, 83, 194, 23, 80, 162, 139, 1, 86, 182, 230, 7, 65, - 144, 234, 1, 5, 73, 78, 83, 69, 82, 202, 174, 1, 67, 175, 193, 2, 79, 26, - 64, 6, 79, 85, 66, 76, 69, 32, 142, 155, 11, 65, 147, 235, 1, 73, 4, 254, - 154, 11, 68, 175, 138, 1, 67, 2, 11, 65, 2, 11, 80, 2, 17, 2, 32, 70, 2, - 129, 141, 11, 2, 73, 76, 108, 218, 1, 78, 62, 86, 142, 154, 11, 65, 38, - 68, 114, 84, 230, 5, 85, 206, 201, 1, 73, 206, 193, 1, 83, 82, 66, 2, 67, + 188, 191, 12, 11, 86, 79, 87, 69, 76, 32, 83, 72, 79, 82, 84, 215, 205, + 2, 69, 8, 222, 252, 10, 65, 158, 145, 4, 69, 3, 89, 4, 206, 141, 15, 73, + 3, 89, 9, 150, 252, 10, 65, 159, 145, 4, 89, 11, 242, 251, 10, 69, 158, + 145, 4, 85, 3, 89, 194, 1, 182, 1, 68, 106, 71, 72, 7, 76, 69, 84, 84, + 69, 82, 32, 214, 2, 83, 198, 23, 80, 190, 139, 1, 86, 210, 240, 7, 65, + 180, 238, 1, 5, 73, 78, 83, 69, 82, 206, 175, 1, 67, 255, 196, 2, 79, 26, + 64, 6, 79, 85, 66, 76, 69, 32, 238, 170, 11, 65, 255, 235, 1, 73, 4, 222, + 170, 11, 68, 179, 138, 1, 67, 2, 11, 65, 2, 11, 80, 2, 17, 2, 32, 70, 2, + 189, 156, 11, 2, 73, 76, 108, 218, 1, 78, 62, 86, 238, 169, 11, 65, 38, + 68, 114, 84, 230, 5, 85, 186, 202, 1, 73, 182, 196, 1, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, 2, 82, 138, 69, 72, 2, 87, 2, - 89, 186, 2, 69, 3, 79, 14, 166, 173, 14, 71, 2, 89, 138, 69, 72, 2, 78, - 187, 2, 65, 10, 26, 69, 139, 161, 11, 79, 2, 173, 190, 12, 3, 68, 73, 67, - 22, 26, 73, 247, 200, 4, 65, 20, 44, 3, 71, 78, 32, 165, 164, 9, 2, 68, - 68, 18, 226, 222, 10, 67, 98, 78, 242, 60, 65, 174, 158, 1, 74, 228, 2, - 5, 70, 73, 78, 65, 76, 50, 85, 167, 242, 1, 86, 4, 167, 199, 14, 69, 4, - 202, 235, 9, 80, 159, 237, 3, 76, 6, 70, 78, 185, 220, 13, 11, 71, 72, - 84, 32, 87, 73, 84, 72, 32, 83, 84, 4, 200, 241, 7, 7, 69, 32, 80, 79, - 73, 78, 84, 179, 253, 6, 74, 124, 152, 1, 3, 67, 79, 77, 130, 3, 68, 78, - 76, 156, 4, 4, 72, 73, 71, 72, 72, 7, 83, 89, 77, 66, 79, 76, 32, 226, - 217, 7, 69, 205, 188, 1, 3, 84, 65, 77, 20, 52, 7, 66, 73, 78, 73, 78, - 71, 32, 235, 236, 14, 77, 18, 88, 3, 68, 79, 85, 32, 5, 76, 79, 78, 71, - 32, 78, 78, 33, 6, 83, 72, 79, 82, 84, 32, 2, 225, 237, 13, 3, 66, 76, - 69, 8, 142, 1, 72, 34, 76, 230, 251, 10, 82, 21, 7, 68, 69, 83, 67, 69, - 78, 68, 2, 11, 65, 2, 171, 206, 10, 83, 6, 34, 72, 34, 76, 231, 251, 10, - 82, 2, 173, 252, 10, 3, 73, 71, 72, 2, 141, 252, 10, 2, 79, 87, 24, 152, - 1, 4, 65, 78, 84, 65, 144, 207, 12, 4, 79, 82, 79, 77, 155, 43, 73, 70, + 89, 186, 2, 69, 3, 79, 14, 218, 192, 14, 71, 2, 89, 138, 69, 72, 2, 78, + 187, 2, 65, 10, 26, 69, 235, 176, 11, 79, 2, 157, 206, 12, 3, 68, 73, 67, + 22, 26, 73, 131, 202, 4, 65, 20, 44, 3, 71, 78, 32, 225, 174, 9, 2, 68, + 68, 18, 246, 232, 10, 67, 98, 78, 190, 66, 65, 190, 158, 1, 74, 228, 2, + 5, 70, 73, 78, 65, 76, 50, 85, 235, 245, 1, 86, 4, 219, 218, 14, 69, 4, + 178, 246, 9, 80, 231, 243, 3, 76, 6, 70, 78, 233, 237, 13, 11, 71, 72, + 84, 32, 87, 73, 84, 72, 32, 83, 84, 4, 252, 246, 7, 7, 69, 32, 80, 79, + 73, 78, 84, 179, 139, 7, 74, 124, 152, 1, 3, 67, 79, 77, 130, 3, 68, 78, + 76, 156, 4, 4, 72, 73, 71, 72, 72, 7, 83, 89, 77, 66, 79, 76, 32, 134, + 223, 7, 69, 229, 193, 1, 3, 84, 65, 77, 20, 52, 7, 66, 73, 78, 73, 78, + 71, 32, 159, 128, 15, 77, 18, 88, 3, 68, 79, 85, 32, 5, 76, 79, 78, 71, + 32, 78, 78, 33, 6, 83, 72, 79, 82, 84, 32, 2, 149, 129, 14, 3, 66, 76, + 69, 8, 142, 1, 72, 34, 76, 198, 138, 11, 82, 21, 7, 68, 69, 83, 67, 69, + 78, 68, 2, 11, 65, 2, 187, 216, 10, 83, 6, 34, 72, 34, 76, 199, 138, 11, + 82, 2, 141, 139, 11, 3, 73, 71, 72, 2, 237, 138, 11, 2, 79, 87, 24, 152, + 1, 4, 65, 78, 84, 65, 232, 222, 12, 4, 79, 82, 79, 77, 143, 44, 73, 70, 76, 4, 65, 74, 65, 78, 32, 6, 69, 84, 84, 69, 82, 32, 173, 3, 2, 79, 87, - 2, 185, 161, 13, 3, 89, 65, 76, 66, 228, 1, 2, 68, 65, 42, 74, 90, 78, - 138, 143, 11, 82, 206, 147, 1, 79, 162, 75, 67, 162, 186, 1, 69, 250, 18, + 2, 209, 178, 13, 3, 89, 65, 76, 66, 228, 1, 2, 68, 65, 42, 74, 90, 78, + 234, 158, 11, 82, 210, 147, 1, 79, 138, 76, 67, 138, 189, 1, 69, 250, 18, 71, 242, 42, 66, 2, 70, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, 83, 2, 84, - 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 5, 197, 178, 11, 5, 71, 66, 65, - 83, 73, 8, 40, 4, 79, 78, 65, 32, 227, 232, 14, 65, 6, 158, 238, 12, 67, - 138, 248, 1, 74, 3, 82, 11, 26, 65, 1, 2, 89, 65, 5, 245, 135, 8, 5, 32, - 87, 79, 76, 79, 2, 29, 5, 32, 84, 79, 78, 69, 2, 17, 2, 32, 65, 2, 135, - 207, 8, 80, 4, 68, 7, 71, 66, 65, 75, 85, 82, 85, 1, 6, 79, 79, 32, 68, - 69, 78, 2, 247, 146, 13, 78, 184, 1, 90, 32, 156, 3, 2, 77, 73, 118, 78, - 150, 1, 82, 238, 7, 84, 250, 153, 14, 45, 147, 40, 83, 18, 202, 1, 66, - 96, 5, 69, 78, 84, 82, 89, 22, 80, 204, 186, 2, 15, 79, 78, 69, 32, 85, - 78, 68, 69, 82, 32, 69, 73, 71, 72, 84, 160, 129, 4, 9, 77, 79, 66, 73, + 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 5, 165, 194, 11, 5, 71, 66, 65, + 83, 73, 8, 40, 4, 79, 78, 65, 32, 151, 252, 14, 65, 6, 234, 254, 12, 67, + 242, 250, 1, 74, 3, 82, 11, 26, 65, 1, 2, 89, 65, 5, 169, 141, 8, 5, 32, + 87, 79, 76, 79, 2, 29, 5, 32, 84, 79, 78, 69, 2, 17, 2, 32, 65, 2, 223, + 211, 8, 80, 4, 68, 7, 71, 66, 65, 75, 85, 82, 85, 1, 6, 79, 79, 32, 68, + 69, 78, 2, 143, 164, 13, 78, 186, 1, 94, 32, 156, 3, 2, 77, 73, 118, 78, + 150, 1, 82, 238, 7, 84, 206, 213, 6, 83, 223, 215, 7, 45, 18, 202, 1, 66, + 96, 5, 69, 78, 84, 82, 89, 22, 80, 224, 186, 2, 15, 79, 78, 69, 32, 85, + 78, 68, 69, 82, 32, 69, 73, 71, 72, 84, 164, 134, 4, 9, 77, 79, 66, 73, 76, 69, 32, 80, 72, 189, 30, 3, 83, 77, 79, 4, 52, 4, 82, 69, 65, 75, - 165, 137, 2, 3, 73, 67, 89, 2, 17, 2, 32, 72, 2, 155, 195, 13, 69, 5, - 203, 220, 13, 32, 4, 60, 6, 69, 68, 69, 83, 84, 82, 221, 201, 9, 3, 73, - 82, 65, 2, 145, 159, 11, 2, 73, 65, 4, 36, 5, 78, 65, 76, 32, 68, 59, 83, - 2, 173, 206, 13, 9, 73, 71, 73, 84, 32, 83, 72, 65, 80, 2, 175, 144, 11, - 77, 6, 38, 45, 249, 217, 9, 3, 70, 79, 82, 4, 76, 8, 66, 82, 69, 65, 75, - 73, 78, 71, 161, 169, 2, 5, 80, 79, 84, 65, 66, 2, 249, 221, 6, 2, 32, - 72, 97, 56, 2, 84, 72, 146, 11, 77, 209, 229, 10, 3, 68, 73, 67, 88, 42, - 32, 201, 205, 6, 4, 69, 65, 83, 84, 86, 96, 5, 69, 65, 83, 84, 32, 148, + 173, 137, 2, 3, 73, 67, 89, 2, 17, 2, 32, 72, 2, 199, 212, 13, 69, 5, + 251, 239, 13, 32, 4, 60, 6, 69, 68, 69, 83, 84, 82, 193, 212, 9, 3, 73, + 82, 65, 2, 237, 174, 11, 2, 73, 65, 4, 36, 5, 78, 65, 76, 32, 68, 59, 83, + 2, 209, 223, 13, 9, 73, 71, 73, 84, 32, 83, 72, 65, 80, 2, 139, 160, 11, + 77, 6, 38, 45, 221, 228, 9, 3, 70, 79, 82, 4, 76, 8, 66, 82, 69, 65, 75, + 73, 78, 71, 181, 169, 2, 5, 80, 79, 84, 65, 66, 2, 161, 227, 6, 2, 32, + 72, 97, 56, 2, 84, 72, 146, 11, 77, 181, 244, 10, 3, 68, 73, 67, 88, 42, + 32, 225, 210, 6, 4, 69, 65, 83, 84, 86, 96, 5, 69, 65, 83, 84, 32, 148, 2, 6, 73, 78, 68, 73, 67, 32, 221, 1, 5, 87, 69, 83, 84, 32, 32, 82, 65, - 238, 242, 6, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, 247, 58, - 87, 14, 40, 4, 82, 82, 79, 87, 215, 238, 6, 78, 13, 11, 32, 10, 96, 9, - 67, 82, 79, 83, 83, 73, 78, 71, 32, 248, 2, 2, 65, 78, 162, 238, 6, 87, - 203, 134, 5, 70, 4, 234, 197, 3, 83, 231, 170, 3, 78, 20, 54, 80, 60, 3, - 81, 85, 65, 66, 82, 171, 132, 1, 70, 2, 37, 7, 76, 65, 67, 69, 72, 79, - 76, 2, 243, 179, 4, 68, 4, 220, 179, 4, 2, 82, 84, 209, 230, 9, 5, 78, - 84, 73, 84, 89, 2, 17, 2, 85, 80, 2, 171, 166, 4, 69, 34, 82, 65, 254, - 238, 6, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, 247, 58, 87, 16, - 34, 78, 33, 4, 82, 82, 79, 87, 2, 241, 194, 3, 3, 68, 32, 83, 15, 11, 32, - 12, 84, 3, 84, 79, 32, 142, 234, 6, 67, 40, 3, 65, 78, 68, 234, 2, 87, - 203, 134, 5, 70, 4, 150, 235, 6, 67, 169, 204, 6, 4, 76, 79, 78, 71, 56, + 150, 248, 6, 80, 130, 1, 84, 190, 207, 4, 66, 38, 68, 18, 83, 251, 58, + 87, 14, 40, 4, 82, 82, 79, 87, 255, 243, 6, 78, 13, 11, 32, 10, 96, 9, + 67, 82, 79, 83, 83, 73, 78, 71, 32, 248, 2, 2, 65, 78, 202, 243, 6, 87, + 131, 145, 5, 70, 4, 190, 198, 3, 83, 187, 175, 3, 78, 20, 54, 80, 60, 3, + 81, 85, 65, 66, 82, 199, 132, 1, 70, 2, 37, 7, 76, 65, 67, 69, 72, 79, + 76, 2, 251, 180, 4, 68, 4, 228, 180, 4, 2, 82, 84, 249, 248, 9, 5, 78, + 84, 73, 84, 89, 2, 17, 2, 85, 80, 2, 179, 167, 4, 69, 34, 82, 65, 166, + 244, 6, 80, 130, 1, 84, 190, 207, 4, 66, 38, 68, 18, 83, 251, 58, 87, 16, + 34, 78, 33, 4, 82, 82, 79, 87, 2, 197, 195, 3, 3, 68, 32, 83, 15, 11, 32, + 12, 84, 3, 84, 79, 32, 182, 239, 6, 67, 40, 3, 65, 78, 68, 234, 2, 87, + 131, 145, 5, 70, 4, 190, 240, 6, 67, 173, 216, 6, 4, 76, 79, 78, 71, 56, 54, 32, 236, 5, 5, 67, 72, 69, 68, 32, 207, 2, 69, 38, 162, 1, 65, 132, - 2, 4, 78, 79, 82, 77, 78, 80, 46, 83, 170, 1, 84, 234, 221, 7, 69, 196, - 33, 7, 73, 68, 69, 78, 84, 73, 67, 154, 193, 4, 71, 38, 76, 191, 78, 67, + 2, 4, 78, 79, 82, 77, 78, 80, 46, 83, 170, 1, 84, 154, 227, 7, 69, 196, + 33, 7, 73, 68, 69, 78, 84, 73, 67, 190, 203, 4, 71, 38, 76, 135, 80, 67, 10, 88, 3, 32, 83, 85, 52, 7, 78, 32, 69, 76, 69, 77, 69, 28, 2, 83, 89, - 251, 128, 8, 76, 4, 30, 66, 1, 3, 80, 69, 82, 2, 29, 2, 83, 69, 2, 11, - 78, 2, 183, 125, 84, 2, 37, 7, 77, 80, 84, 79, 84, 73, 67, 2, 17, 2, 65, - 76, 2, 185, 129, 8, 2, 76, 89, 4, 253, 195, 6, 14, 65, 76, 32, 83, 85, - 66, 71, 82, 79, 85, 80, 32, 79, 70, 2, 137, 129, 8, 6, 65, 82, 65, 76, - 76, 69, 6, 48, 6, 81, 85, 65, 82, 69, 32, 247, 204, 13, 73, 4, 68, 5, 73, + 171, 134, 8, 76, 4, 30, 66, 1, 3, 80, 69, 82, 2, 29, 2, 83, 69, 2, 11, + 78, 2, 211, 125, 84, 2, 37, 7, 77, 80, 84, 79, 84, 73, 67, 2, 17, 2, 65, + 76, 2, 233, 134, 8, 2, 76, 89, 4, 149, 201, 6, 14, 65, 76, 32, 83, 85, + 66, 71, 82, 79, 85, 80, 32, 79, 70, 2, 185, 134, 8, 6, 65, 82, 65, 76, + 76, 69, 6, 48, 6, 81, 85, 65, 82, 69, 32, 167, 224, 13, 73, 4, 68, 5, 73, 77, 65, 71, 69, 1, 8, 79, 82, 73, 71, 73, 78, 65, 76, 2, 21, 3, 32, 79, - 70, 2, 255, 193, 6, 32, 4, 210, 199, 10, 82, 215, 241, 1, 73, 8, 58, 76, + 70, 2, 151, 199, 6, 32, 4, 174, 214, 10, 82, 207, 242, 1, 73, 8, 58, 76, 40, 4, 82, 73, 71, 72, 133, 1, 3, 85, 80, 80, 4, 36, 2, 69, 70, 133, 1, 2, 79, 87, 2, 89, 20, 84, 32, 83, 69, 77, 73, 67, 73, 82, 67, 76, 69, 32, - 87, 73, 84, 72, 32, 84, 72, 2, 17, 2, 82, 69, 2, 167, 128, 13, 69, 2, 11, - 69, 2, 41, 8, 82, 32, 82, 73, 71, 72, 84, 45, 2, 197, 147, 3, 6, 83, 72, + 87, 73, 84, 72, 32, 84, 72, 2, 17, 2, 82, 69, 2, 187, 145, 13, 69, 2, 11, + 69, 2, 41, 8, 82, 32, 82, 73, 71, 72, 84, 45, 2, 205, 148, 3, 6, 83, 72, 65, 68, 79, 87, 11, 34, 32, 53, 4, 66, 79, 79, 75, 4, 17, 2, 80, 65, 4, - 222, 184, 14, 71, 215, 22, 68, 5, 81, 18, 32, 87, 73, 84, 72, 32, 68, 69, - 67, 79, 82, 65, 84, 73, 86, 69, 32, 67, 2, 171, 141, 4, 79, 186, 6, 102, - 77, 176, 3, 4, 83, 72, 85, 32, 172, 178, 5, 8, 84, 32, 65, 78, 68, 32, - 66, 79, 187, 155, 8, 76, 26, 36, 4, 66, 69, 82, 32, 247, 2, 69, 24, 58, + 142, 204, 14, 71, 215, 22, 68, 5, 81, 18, 32, 87, 73, 84, 72, 32, 68, 69, + 67, 79, 82, 65, 84, 73, 86, 69, 32, 67, 2, 179, 142, 4, 79, 186, 6, 102, + 77, 176, 3, 4, 83, 72, 85, 32, 216, 183, 5, 8, 84, 32, 65, 78, 68, 32, + 66, 79, 191, 169, 8, 76, 26, 36, 4, 66, 69, 82, 32, 247, 2, 69, 24, 58, 69, 50, 70, 42, 83, 66, 84, 57, 4, 78, 73, 78, 69, 4, 204, 1, 3, 73, 71, 72, 21, 3, 76, 69, 86, 4, 144, 1, 2, 79, 85, 13, 2, 73, 70, 6, 34, 73, - 85, 4, 69, 86, 69, 78, 4, 82, 88, 175, 251, 13, 71, 8, 40, 2, 72, 73, 46, - 69, 21, 2, 87, 69, 2, 11, 82, 2, 17, 2, 84, 69, 2, 11, 69, 2, 171, 199, - 7, 78, 4, 148, 199, 7, 3, 76, 86, 69, 1, 3, 78, 84, 89, 2, 215, 242, 6, - 82, 154, 6, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 251, - 166, 12, 73, 152, 6, 18, 49, 87, 50, 160, 2, 222, 1, 55, 2, 56, 2, 57, 2, + 85, 4, 69, 86, 69, 78, 4, 82, 88, 223, 142, 14, 71, 8, 40, 2, 72, 73, 46, + 69, 21, 2, 87, 69, 2, 11, 82, 2, 17, 2, 84, 69, 2, 11, 69, 2, 203, 204, + 7, 78, 4, 180, 204, 7, 3, 76, 86, 69, 1, 3, 78, 84, 89, 2, 255, 247, 6, + 82, 154, 6, 72, 12, 67, 72, 65, 82, 65, 67, 84, 69, 82, 45, 49, 66, 215, + 182, 12, 73, 152, 6, 18, 49, 87, 50, 160, 2, 222, 1, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, 70, 248, 3, 138, 1, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, - 68, 2, 69, 143, 1, 70, 32, 194, 199, 14, 48, 2, 49, 2, 50, 2, 51, 2, 52, + 68, 2, 69, 143, 1, 70, 32, 242, 218, 14, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 2, 66, 2, 67, 2, 68, 2, 69, 3, - 70, 24, 182, 198, 14, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 70, 24, 230, 217, 14, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 2, 57, 2, 65, 3, 66, 142, 1, 118, 76, 226, 3, 83, 164, 2, 5, - 84, 79, 78, 69, 45, 148, 225, 7, 8, 67, 73, 82, 67, 76, 69, 68, 32, 155, - 236, 4, 68, 92, 88, 6, 69, 84, 84, 69, 82, 32, 197, 163, 1, 10, 79, 71, - 79, 71, 82, 65, 77, 32, 78, 89, 90, 130, 2, 78, 90, 84, 246, 214, 7, 88, - 162, 126, 65, 232, 203, 1, 2, 72, 65, 242, 70, 82, 206, 147, 1, 79, 174, - 60, 68, 2, 77, 2, 80, 150, 201, 1, 69, 234, 61, 67, 2, 70, 2, 71, 2, 75, + 84, 79, 78, 69, 45, 196, 230, 7, 8, 67, 73, 82, 67, 76, 69, 68, 32, 179, + 247, 4, 68, 92, 88, 6, 69, 84, 84, 69, 82, 32, 173, 163, 1, 10, 79, 71, + 79, 71, 82, 65, 77, 32, 78, 89, 90, 130, 2, 78, 90, 84, 166, 220, 7, 88, + 182, 230, 2, 65, 144, 1, 2, 72, 65, 226, 51, 82, 210, 147, 1, 79, 150, + 61, 68, 2, 77, 2, 80, 254, 203, 1, 69, 234, 61, 67, 2, 70, 2, 71, 2, 75, 2, 76, 2, 81, 2, 83, 2, 86, 2, 89, 2, 90, 186, 2, 73, 2, 85, 3, 87, 22, - 86, 84, 230, 183, 12, 80, 254, 134, 2, 67, 2, 75, 2, 81, 2, 82, 2, 89, - 187, 2, 65, 6, 222, 190, 14, 83, 2, 88, 187, 2, 65, 14, 64, 4, 73, 71, + 86, 84, 174, 200, 12, 80, 230, 137, 2, 67, 2, 75, 2, 81, 2, 82, 2, 89, + 187, 2, 65, 6, 142, 210, 14, 83, 2, 88, 187, 2, 65, 14, 64, 4, 73, 71, 78, 32, 189, 1, 7, 89, 76, 76, 65, 66, 76, 69, 12, 56, 4, 70, 79, 82, 32, - 209, 193, 13, 4, 88, 87, 32, 88, 10, 156, 9, 2, 76, 79, 194, 164, 3, 84, - 232, 93, 8, 73, 78, 86, 69, 82, 84, 69, 66, 160, 141, 4, 3, 65, 78, 73, - 183, 167, 2, 80, 2, 209, 244, 11, 4, 32, 76, 69, 78, 14, 202, 190, 14, - 66, 2, 68, 2, 71, 2, 74, 2, 77, 2, 83, 3, 86, 250, 14, 178, 2, 66, 226, - 1, 67, 224, 5, 4, 70, 70, 73, 67, 54, 71, 248, 6, 4, 73, 76, 32, 68, 22, - 76, 186, 69, 78, 150, 5, 80, 234, 7, 82, 218, 10, 83, 188, 8, 2, 84, 84, - 154, 8, 85, 248, 1, 3, 86, 69, 82, 198, 170, 2, 89, 202, 198, 5, 72, 238, - 254, 1, 75, 154, 249, 1, 68, 194, 64, 77, 246, 9, 87, 183, 137, 1, 88, - 10, 132, 1, 6, 76, 73, 81, 85, 69, 32, 180, 158, 2, 9, 83, 69, 82, 86, - 69, 82, 32, 69, 89, 189, 29, 8, 74, 69, 67, 84, 32, 82, 69, 80, 6, 176, - 198, 4, 13, 65, 78, 71, 76, 69, 32, 79, 80, 69, 78, 73, 78, 71, 175, 241, - 1, 72, 28, 56, 2, 82, 32, 234, 4, 84, 229, 193, 5, 3, 67, 85, 76, 22, - 156, 1, 11, 65, 77, 79, 85, 78, 84, 32, 79, 70, 32, 67, 22, 66, 198, 1, - 67, 118, 68, 106, 70, 0, 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 70, 143, - 225, 12, 72, 2, 207, 179, 5, 72, 6, 136, 1, 15, 82, 65, 78, 67, 72, 32, - 66, 65, 78, 75, 32, 73, 68, 69, 78, 160, 254, 4, 5, 69, 76, 84, 32, 66, - 157, 134, 6, 3, 79, 87, 32, 2, 21, 3, 84, 73, 70, 2, 11, 73, 2, 215, 244, - 10, 67, 4, 92, 17, 85, 83, 84, 79, 77, 69, 82, 32, 65, 67, 67, 79, 85, - 78, 84, 32, 78, 155, 202, 1, 72, 2, 183, 156, 6, 85, 4, 44, 5, 79, 85, - 66, 76, 69, 251, 204, 12, 65, 2, 11, 32, 2, 11, 66, 2, 177, 158, 7, 3, - 65, 67, 75, 2, 191, 243, 13, 79, 4, 152, 241, 8, 4, 65, 71, 79, 78, 181, - 189, 3, 2, 79, 80, 2, 11, 69, 2, 221, 251, 7, 5, 32, 66, 85, 73, 76, 60, - 48, 4, 72, 65, 77, 32, 133, 176, 14, 2, 79, 78, 58, 122, 70, 0, 10, 82, + 129, 213, 13, 4, 88, 87, 32, 88, 10, 204, 9, 2, 76, 79, 230, 164, 3, 84, + 156, 94, 8, 73, 78, 86, 69, 82, 84, 69, 66, 232, 144, 4, 3, 65, 78, 73, + 195, 177, 2, 80, 2, 177, 132, 12, 4, 32, 76, 69, 78, 14, 250, 209, 14, + 66, 2, 68, 2, 71, 2, 74, 2, 77, 2, 83, 3, 86, 254, 14, 226, 2, 66, 226, + 1, 67, 226, 5, 71, 244, 6, 4, 73, 76, 32, 68, 22, 76, 198, 69, 78, 150, + 5, 80, 234, 7, 82, 242, 10, 83, 188, 8, 2, 84, 84, 154, 8, 85, 248, 1, 3, + 86, 69, 82, 254, 170, 2, 89, 148, 151, 4, 14, 70, 70, 73, 67, 69, 32, 66, + 85, 73, 76, 68, 73, 78, 71, 154, 185, 1, 72, 146, 132, 2, 75, 210, 250, + 1, 68, 194, 64, 77, 246, 9, 87, 211, 139, 1, 88, 10, 132, 1, 6, 76, 73, + 81, 85, 69, 32, 252, 158, 2, 9, 83, 69, 82, 86, 69, 82, 32, 69, 89, 189, + 29, 8, 74, 69, 67, 84, 32, 82, 69, 80, 6, 172, 203, 4, 13, 65, 78, 71, + 76, 69, 32, 79, 80, 69, 78, 73, 78, 71, 171, 241, 1, 72, 28, 56, 2, 82, + 32, 234, 4, 84, 225, 198, 5, 3, 67, 85, 76, 22, 156, 1, 11, 65, 77, 79, + 85, 78, 84, 32, 79, 70, 32, 67, 22, 66, 198, 1, 67, 118, 68, 106, 70, 0, + 10, 73, 78, 86, 69, 82, 84, 69, 68, 32, 70, 243, 241, 12, 72, 2, 203, + 184, 5, 72, 6, 136, 1, 15, 82, 65, 78, 67, 72, 32, 66, 65, 78, 75, 32, + 73, 68, 69, 78, 156, 131, 5, 5, 69, 76, 84, 32, 66, 205, 144, 6, 3, 79, + 87, 32, 2, 21, 3, 84, 73, 70, 2, 11, 73, 2, 131, 132, 11, 67, 4, 92, 17, + 85, 83, 84, 79, 77, 69, 82, 32, 65, 67, 67, 79, 85, 78, 84, 32, 78, 243, + 201, 1, 72, 2, 159, 161, 6, 85, 4, 44, 5, 79, 85, 66, 76, 69, 147, 221, + 12, 65, 2, 11, 32, 2, 11, 66, 2, 157, 163, 7, 3, 65, 67, 75, 2, 191, 134, + 14, 79, 4, 160, 251, 8, 4, 65, 71, 79, 78, 197, 195, 3, 2, 79, 80, 60, + 48, 4, 72, 65, 77, 32, 185, 195, 14, 2, 79, 78, 58, 122, 70, 0, 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 70, 36, 7, 76, 69, 84, 84, 69, 82, 32, - 137, 253, 3, 3, 83, 80, 65, 2, 149, 138, 4, 4, 69, 65, 84, 72, 52, 196, + 149, 254, 3, 3, 83, 80, 65, 2, 161, 139, 4, 4, 69, 65, 84, 72, 52, 196, 1, 2, 65, 73, 22, 66, 2, 80, 34, 67, 36, 2, 69, 65, 64, 2, 70, 69, 22, - 71, 22, 73, 58, 76, 2, 82, 22, 78, 46, 79, 34, 83, 50, 85, 206, 184, 1, - 77, 170, 9, 68, 153, 210, 11, 3, 84, 73, 78, 2, 231, 159, 14, 76, 2, 11, - 69, 2, 143, 174, 12, 73, 4, 142, 147, 8, 69, 191, 149, 4, 79, 6, 138, 1, - 66, 2, 68, 169, 141, 4, 6, 77, 72, 65, 78, 67, 72, 2, 143, 165, 9, 65, 2, - 131, 168, 13, 79, 4, 32, 2, 79, 68, 139, 175, 13, 70, 2, 239, 132, 8, 72, - 2, 147, 139, 12, 85, 4, 236, 154, 13, 3, 71, 69, 65, 219, 67, 73, 4, 166, - 222, 13, 78, 227, 79, 82, 4, 146, 250, 11, 65, 233, 177, 1, 3, 84, 82, - 65, 6, 60, 5, 73, 76, 76, 69, 65, 234, 170, 12, 65, 147, 130, 2, 82, 2, - 151, 221, 13, 78, 2, 211, 131, 13, 82, 182, 8, 38, 32, 142, 11, 68, 199, - 164, 13, 73, 184, 1, 64, 6, 67, 72, 73, 75, 73, 32, 205, 6, 5, 79, 78, - 65, 76, 32, 96, 132, 1, 7, 76, 69, 84, 84, 69, 82, 32, 148, 3, 2, 77, 85, - 62, 71, 74, 80, 204, 215, 3, 2, 82, 69, 210, 221, 8, 68, 235, 170, 1, 65, - 60, 50, 65, 98, 69, 54, 73, 50, 76, 62, 79, 51, 85, 16, 50, 65, 154, 169, - 14, 78, 86, 71, 2, 76, 3, 84, 8, 234, 169, 14, 74, 2, 75, 2, 77, 3, 87, - 8, 158, 227, 13, 68, 198, 13, 82, 222, 56, 78, 3, 80, 8, 194, 151, 14, - 78, 202, 17, 72, 2, 82, 3, 83, 12, 214, 188, 8, 65, 134, 236, 5, 69, 2, - 73, 2, 79, 3, 85, 8, 242, 139, 14, 84, 174, 28, 66, 2, 72, 3, 86, 8, 142, - 216, 13, 78, 226, 79, 67, 2, 68, 3, 89, 4, 56, 2, 45, 71, 129, 190, 12, - 6, 32, 84, 84, 85, 68, 68, 2, 253, 189, 12, 13, 65, 65, 72, 76, 65, 65, - 32, 84, 84, 85, 68, 68, 65, 6, 84, 11, 85, 78, 67, 84, 85, 65, 84, 73, - 79, 78, 32, 129, 219, 6, 4, 72, 65, 65, 82, 4, 48, 8, 68, 79, 85, 66, 76, - 69, 32, 77, 3, 77, 2, 129, 223, 13, 3, 85, 67, 65, 88, 100, 7, 76, 69, - 84, 84, 69, 82, 32, 192, 2, 5, 83, 73, 71, 78, 32, 142, 183, 8, 65, 191, - 249, 3, 68, 60, 42, 65, 54, 69, 58, 73, 54, 79, 59, 85, 13, 250, 163, 14, - 66, 2, 68, 2, 72, 2, 76, 3, 87, 13, 230, 211, 13, 78, 226, 79, 67, 2, 71, - 2, 72, 3, 83, 13, 174, 195, 8, 84, 226, 223, 5, 68, 2, 78, 3, 80, 13, - 254, 233, 13, 82, 138, 56, 78, 86, 77, 2, 79, 3, 89, 13, 130, 220, 13, - 68, 218, 52, 78, 202, 17, 74, 2, 75, 3, 82, 6, 58, 73, 236, 228, 12, 4, - 72, 79, 68, 68, 219, 151, 1, 77, 2, 243, 181, 1, 75, 252, 6, 34, 32, 153, - 57, 3, 69, 82, 32, 246, 6, 240, 2, 8, 67, 72, 73, 78, 69, 83, 69, 32, 44, - 10, 72, 85, 78, 71, 65, 82, 73, 65, 78, 32, 244, 6, 7, 73, 84, 65, 76, - 73, 67, 32, 212, 4, 14, 78, 79, 82, 84, 72, 32, 65, 82, 65, 66, 73, 65, - 78, 32, 196, 4, 3, 80, 69, 82, 216, 13, 2, 83, 79, 144, 13, 14, 84, 85, - 82, 75, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 132, 7, 7, 85, 89, 71, - 72, 85, 82, 32, 219, 209, 11, 75, 4, 174, 251, 11, 73, 177, 95, 3, 72, - 79, 79, 216, 1, 96, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 221, - 5, 7, 78, 85, 77, 66, 69, 82, 32, 102, 45, 9, 76, 32, 76, 69, 84, 84, 69, - 82, 32, 102, 226, 1, 65, 54, 69, 164, 2, 10, 78, 73, 75, 79, 76, 83, 66, - 85, 82, 71, 0, 9, 82, 85, 68, 73, 77, 69, 78, 84, 65, 42, 79, 34, 85, - 192, 240, 3, 5, 67, 76, 79, 83, 69, 150, 157, 8, 73, 241, 203, 1, 6, 83, - 72, 79, 82, 84, 32, 11, 234, 222, 12, 77, 198, 117, 78, 162, 70, 65, 3, - 75, 63, 174, 1, 77, 22, 78, 78, 83, 134, 248, 9, 67, 86, 71, 2, 76, 2, - 84, 198, 152, 2, 90, 242, 134, 2, 66, 2, 68, 2, 69, 2, 70, 2, 72, 2, 74, - 2, 75, 2, 80, 2, 82, 3, 86, 5, 235, 152, 14, 80, 11, 34, 84, 182, 152, - 14, 67, 3, 89, 5, 189, 148, 2, 5, 45, 83, 72, 65, 80, 5, 139, 152, 14, - 90, 4, 11, 32, 4, 150, 129, 14, 79, 3, 85, 7, 250, 128, 14, 69, 215, 22, - 79, 9, 150, 148, 14, 78, 154, 3, 83, 3, 85, 12, 210, 4, 70, 222, 254, 5, - 79, 243, 191, 6, 84, 78, 80, 7, 76, 69, 84, 84, 69, 82, 32, 169, 3, 8, + 71, 22, 73, 58, 76, 2, 82, 22, 78, 46, 79, 34, 83, 46, 85, 222, 184, 1, + 77, 170, 9, 68, 189, 227, 11, 3, 84, 73, 78, 2, 155, 179, 14, 76, 2, 11, + 69, 2, 219, 190, 12, 73, 4, 226, 151, 8, 69, 183, 161, 4, 79, 6, 138, 1, + 66, 2, 68, 153, 143, 4, 6, 77, 72, 65, 78, 67, 72, 2, 247, 175, 9, 65, 2, + 183, 187, 13, 79, 4, 32, 2, 79, 68, 191, 194, 13, 70, 2, 195, 137, 8, 72, + 2, 243, 154, 12, 85, 4, 132, 172, 13, 3, 71, 69, 65, 247, 69, 73, 4, 218, + 241, 13, 78, 227, 79, 82, 4, 202, 225, 12, 65, 229, 93, 3, 84, 82, 65, 6, + 60, 5, 73, 76, 76, 69, 65, 186, 187, 12, 65, 251, 132, 2, 82, 2, 207, + 240, 13, 78, 2, 247, 148, 13, 82, 182, 8, 38, 32, 142, 11, 68, 255, 183, + 13, 73, 184, 1, 64, 6, 67, 72, 73, 75, 73, 32, 205, 6, 5, 79, 78, 65, 76, + 32, 96, 132, 1, 7, 76, 69, 84, 84, 69, 82, 32, 148, 3, 2, 77, 85, 62, 71, + 74, 80, 164, 216, 3, 2, 82, 69, 202, 237, 8, 68, 211, 173, 1, 65, 60, 50, + 65, 98, 69, 54, 73, 50, 76, 62, 79, 51, 85, 16, 50, 65, 210, 188, 14, 78, + 86, 71, 2, 76, 3, 84, 8, 162, 189, 14, 74, 2, 75, 2, 77, 3, 87, 8, 214, + 246, 13, 68, 198, 13, 82, 222, 56, 78, 3, 80, 8, 250, 170, 14, 78, 202, + 17, 72, 2, 82, 3, 83, 12, 162, 170, 10, 65, 242, 145, 4, 69, 2, 73, 2, + 79, 3, 85, 8, 170, 159, 14, 84, 174, 28, 66, 2, 72, 3, 86, 8, 198, 235, + 13, 78, 226, 79, 67, 2, 68, 3, 89, 4, 56, 2, 45, 71, 209, 206, 12, 6, 32, + 84, 84, 85, 68, 68, 2, 205, 206, 12, 13, 65, 65, 72, 76, 65, 65, 32, 84, + 84, 85, 68, 68, 65, 6, 84, 11, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, + 32, 177, 224, 6, 4, 72, 65, 65, 82, 4, 48, 8, 68, 79, 85, 66, 76, 69, 32, + 77, 3, 77, 2, 185, 242, 13, 3, 85, 67, 65, 88, 100, 7, 76, 69, 84, 84, + 69, 82, 32, 192, 2, 5, 83, 73, 71, 78, 32, 206, 193, 8, 65, 207, 255, 3, + 68, 60, 42, 65, 54, 69, 58, 73, 54, 79, 59, 85, 13, 178, 183, 14, 66, 2, + 68, 2, 72, 2, 76, 3, 87, 13, 158, 231, 13, 78, 226, 79, 67, 2, 71, 2, 72, + 3, 83, 13, 238, 205, 8, 84, 218, 232, 5, 68, 2, 78, 3, 80, 13, 182, 253, + 13, 82, 138, 56, 78, 86, 77, 2, 79, 3, 89, 13, 186, 239, 13, 68, 218, 52, + 78, 202, 17, 74, 2, 75, 3, 82, 6, 58, 73, 144, 246, 12, 4, 72, 79, 68, + 68, 239, 153, 1, 77, 2, 131, 182, 1, 75, 252, 6, 34, 32, 165, 57, 3, 69, + 82, 32, 246, 6, 240, 2, 8, 67, 72, 73, 78, 69, 83, 69, 32, 44, 10, 72, + 85, 78, 71, 65, 82, 73, 65, 78, 32, 128, 7, 7, 73, 84, 65, 76, 73, 67, + 32, 212, 4, 14, 78, 79, 82, 84, 72, 32, 65, 82, 65, 66, 73, 65, 78, 32, + 196, 4, 3, 80, 69, 82, 216, 13, 2, 83, 79, 144, 13, 14, 84, 85, 82, 75, + 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 132, 7, 7, 85, 89, 71, 72, 85, + 82, 32, 171, 225, 11, 75, 4, 146, 139, 12, 73, 241, 96, 3, 72, 79, 79, + 216, 1, 96, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, 233, 5, 7, + 78, 85, 77, 66, 69, 82, 32, 102, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, + 32, 102, 214, 1, 65, 54, 69, 168, 2, 10, 78, 73, 75, 79, 76, 83, 66, 85, + 82, 71, 0, 9, 82, 85, 68, 73, 77, 69, 78, 84, 65, 42, 79, 32, 5, 83, 72, + 79, 82, 84, 22, 85, 168, 242, 3, 5, 67, 76, 79, 83, 69, 243, 171, 8, 73, + 11, 154, 240, 12, 77, 218, 119, 78, 162, 70, 65, 3, 75, 63, 178, 1, 77, + 22, 78, 78, 83, 182, 130, 10, 67, 254, 23, 71, 2, 76, 2, 84, 198, 135, 2, + 90, 218, 137, 2, 66, 2, 68, 2, 69, 2, 70, 2, 72, 2, 74, 2, 75, 2, 80, 2, + 82, 3, 86, 5, 171, 172, 14, 80, 11, 34, 84, 246, 171, 14, 67, 3, 89, 5, + 197, 149, 2, 5, 45, 83, 72, 65, 80, 5, 203, 171, 14, 90, 4, 11, 32, 4, + 214, 148, 14, 79, 3, 85, 7, 186, 148, 14, 69, 215, 22, 79, 2, 131, 237, + 13, 32, 9, 194, 167, 14, 78, 154, 3, 83, 3, 85, 12, 210, 4, 70, 242, 131, + 6, 79, 239, 203, 6, 84, 78, 80, 7, 76, 69, 84, 84, 69, 82, 32, 169, 3, 8, 78, 85, 77, 69, 82, 65, 76, 32, 70, 190, 1, 69, 90, 75, 50, 83, 36, 3, - 78, 79, 82, 242, 191, 10, 85, 206, 201, 1, 73, 190, 137, 1, 80, 2, 84, + 78, 79, 82, 202, 207, 10, 85, 186, 202, 1, 73, 166, 140, 1, 80, 2, 84, 150, 83, 67, 186, 22, 66, 2, 68, 2, 72, 2, 86, 2, 89, 2, 90, 214, 22, 65, - 3, 79, 23, 186, 244, 9, 83, 154, 153, 2, 82, 150, 201, 1, 75, 222, 61, - 70, 2, 76, 2, 77, 3, 78, 8, 150, 253, 13, 72, 214, 22, 65, 2, 69, 3, 85, - 4, 32, 2, 79, 85, 199, 252, 13, 72, 2, 29, 5, 84, 72, 69, 82, 78, 2, 233, - 137, 13, 2, 32, 84, 8, 38, 70, 206, 190, 12, 84, 191, 58, 79, 4, 11, 73, - 4, 174, 165, 12, 70, 167, 214, 1, 86, 64, 76, 7, 76, 69, 84, 84, 69, 82, + 3, 79, 23, 214, 254, 9, 83, 194, 159, 2, 82, 254, 203, 1, 75, 222, 61, + 70, 2, 76, 2, 77, 3, 78, 8, 194, 144, 14, 72, 214, 22, 65, 2, 69, 3, 85, + 4, 32, 2, 79, 85, 243, 143, 14, 72, 2, 29, 5, 84, 72, 69, 82, 78, 2, 253, + 154, 13, 2, 32, 84, 8, 38, 70, 222, 207, 12, 84, 215, 58, 79, 4, 11, 73, + 4, 242, 181, 12, 70, 143, 217, 1, 86, 64, 76, 7, 76, 69, 84, 84, 69, 82, 32, 209, 3, 7, 78, 85, 77, 66, 69, 82, 32, 58, 210, 1, 65, 38, 71, 38, - 72, 30, 75, 38, 84, 74, 90, 134, 107, 77, 208, 153, 3, 2, 69, 83, 226, - 161, 3, 66, 2, 70, 2, 82, 2, 89, 138, 193, 3, 78, 218, 235, 1, 76, 190, - 56, 68, 146, 1, 81, 134, 3, 87, 131, 56, 83, 4, 186, 193, 3, 76, 199, - 254, 9, 73, 4, 230, 226, 12, 72, 171, 154, 1, 69, 4, 134, 142, 14, 65, 3, - 69, 4, 250, 174, 7, 72, 223, 222, 5, 65, 8, 34, 72, 166, 141, 14, 65, 3, - 69, 4, 254, 132, 13, 65, 167, 136, 1, 69, 4, 11, 65, 4, 162, 190, 13, 73, - 227, 79, 72, 6, 154, 148, 6, 84, 159, 224, 6, 79, 180, 1, 64, 11, 77, 73, - 67, 32, 76, 69, 84, 84, 69, 82, 32, 183, 5, 83, 76, 228, 1, 2, 67, 72, - 22, 68, 66, 69, 22, 73, 30, 77, 2, 78, 30, 80, 22, 83, 70, 84, 50, 86, - 38, 89, 94, 90, 218, 190, 6, 76, 2, 82, 162, 222, 2, 71, 146, 165, 2, 79, - 142, 205, 1, 66, 246, 40, 65, 254, 31, 75, 174, 45, 72, 187, 2, 85, 2, - 139, 212, 7, 69, 6, 26, 90, 139, 247, 13, 79, 4, 174, 156, 9, 72, 231, - 201, 4, 73, 5, 187, 138, 14, 70, 7, 166, 138, 14, 65, 3, 69, 2, 169, 243, - 13, 2, 69, 78, 2, 223, 192, 6, 69, 6, 26, 72, 235, 245, 13, 73, 4, 140, - 155, 9, 3, 67, 72, 79, 3, 79, 4, 26, 83, 167, 245, 13, 65, 2, 147, 228, - 13, 73, 4, 182, 154, 9, 79, 215, 181, 4, 69, 14, 60, 2, 69, 82, 166, 168, - 8, 65, 154, 206, 5, 82, 203, 17, 85, 7, 130, 136, 14, 73, 3, 85, 4, 182, - 153, 9, 72, 231, 201, 4, 65, 104, 88, 4, 73, 65, 78, 32, 241, 5, 13, 79, + 72, 30, 75, 38, 84, 74, 90, 234, 106, 77, 140, 158, 3, 2, 69, 83, 238, + 162, 3, 66, 2, 70, 2, 82, 2, 89, 182, 203, 3, 78, 154, 237, 1, 76, 210, + 58, 68, 146, 1, 81, 134, 3, 87, 131, 56, 83, 4, 134, 194, 3, 76, 167, + 145, 10, 73, 4, 254, 243, 12, 72, 191, 156, 1, 69, 4, 178, 161, 14, 65, + 3, 69, 4, 166, 180, 7, 72, 223, 236, 5, 65, 8, 34, 72, 210, 160, 14, 65, + 3, 69, 4, 142, 150, 13, 65, 195, 138, 1, 69, 4, 11, 65, 4, 206, 209, 13, + 73, 227, 79, 72, 6, 190, 153, 6, 84, 163, 236, 6, 79, 180, 1, 64, 11, 77, + 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 183, 5, 83, 76, 228, 1, 2, 67, + 72, 22, 68, 66, 69, 22, 73, 30, 77, 2, 78, 30, 80, 22, 83, 70, 84, 50, + 86, 38, 89, 94, 90, 254, 195, 6, 76, 2, 82, 214, 227, 2, 71, 150, 170, 2, + 79, 222, 208, 1, 66, 246, 40, 65, 254, 31, 75, 174, 45, 72, 187, 2, 85, + 2, 215, 216, 7, 69, 6, 26, 90, 183, 138, 14, 79, 4, 134, 167, 9, 72, 187, + 210, 4, 73, 5, 231, 157, 14, 70, 7, 210, 157, 14, 65, 3, 69, 2, 213, 134, + 14, 2, 69, 78, 2, 131, 198, 6, 69, 6, 26, 72, 151, 137, 14, 73, 4, 228, + 165, 9, 3, 67, 72, 79, 3, 79, 4, 26, 83, 211, 136, 14, 65, 2, 191, 247, + 13, 73, 4, 142, 165, 9, 79, 171, 190, 4, 69, 14, 60, 2, 69, 82, 218, 178, + 8, 65, 146, 215, 5, 82, 203, 17, 85, 7, 174, 155, 14, 73, 3, 85, 4, 142, + 164, 9, 72, 187, 210, 4, 65, 104, 88, 4, 73, 65, 78, 32, 241, 5, 13, 79, 78, 65, 76, 32, 67, 79, 77, 80, 85, 84, 69, 82, 100, 80, 7, 78, 85, 77, - 66, 69, 82, 32, 80, 5, 83, 73, 71, 78, 32, 163, 207, 10, 87, 10, 42, 84, - 182, 178, 8, 72, 139, 186, 4, 79, 6, 142, 230, 1, 87, 251, 207, 11, 69, - 88, 210, 1, 65, 86, 66, 62, 68, 98, 74, 2, 86, 30, 84, 42, 88, 210, 111, - 71, 2, 75, 2, 78, 2, 82, 162, 190, 9, 77, 190, 139, 3, 83, 218, 69, 67, + 66, 69, 82, 32, 80, 5, 83, 73, 71, 78, 32, 251, 222, 10, 87, 10, 42, 84, + 234, 188, 8, 72, 255, 192, 4, 79, 6, 238, 230, 1, 87, 199, 226, 11, 69, + 88, 210, 1, 65, 86, 66, 62, 68, 98, 74, 2, 86, 30, 84, 42, 88, 182, 111, + 71, 2, 75, 2, 78, 2, 82, 150, 206, 9, 77, 146, 143, 3, 83, 218, 69, 67, 2, 70, 2, 72, 2, 76, 2, 80, 2, 89, 2, 90, 186, 2, 73, 3, 85, 9, 45, 9, - 85, 82, 65, 77, 65, 90, 68, 65, 65, 7, 146, 234, 6, 45, 247, 150, 7, 72, - 6, 38, 65, 253, 161, 10, 3, 85, 85, 77, 5, 187, 128, 14, 71, 10, 34, 65, - 190, 130, 14, 73, 3, 85, 7, 37, 7, 72, 89, 65, 65, 85, 83, 72, 5, 231, - 232, 6, 45, 4, 254, 129, 14, 65, 3, 73, 6, 170, 255, 13, 72, 186, 2, 65, - 3, 85, 4, 176, 204, 11, 8, 83, 72, 65, 65, 89, 65, 84, 72, 139, 181, 2, - 65, 5, 141, 142, 5, 31, 32, 87, 73, 84, 72, 32, 77, 79, 78, 73, 84, 79, + 85, 82, 65, 77, 65, 90, 68, 65, 65, 7, 170, 239, 6, 45, 139, 165, 7, 72, + 6, 38, 65, 213, 177, 10, 3, 85, 85, 77, 5, 231, 147, 14, 71, 10, 34, 65, + 234, 149, 14, 73, 3, 85, 7, 37, 7, 72, 89, 65, 65, 85, 83, 72, 5, 255, + 237, 6, 45, 4, 170, 149, 14, 65, 3, 73, 6, 214, 146, 14, 72, 186, 2, 65, + 3, 85, 4, 152, 220, 11, 8, 83, 72, 65, 65, 89, 65, 84, 72, 207, 184, 2, + 65, 5, 181, 147, 5, 31, 32, 87, 73, 84, 72, 32, 77, 79, 78, 73, 84, 79, 82, 32, 73, 78, 32, 80, 79, 82, 84, 82, 65, 73, 84, 32, 79, 82, 73, 69, 78, 144, 1, 92, 6, 71, 68, 73, 65, 78, 32, 141, 7, 12, 85, 84, 72, 32, 65, 82, 65, 66, 73, 65, 78, 32, 80, 58, 70, 74, 76, 145, 5, 7, 78, 85, - 77, 66, 69, 82, 32, 2, 11, 82, 2, 241, 221, 11, 10, 65, 67, 84, 73, 79, + 77, 66, 69, 82, 32, 2, 11, 82, 2, 201, 237, 11, 10, 65, 67, 84, 73, 79, 78, 32, 79, 78, 69, 60, 76, 6, 69, 84, 84, 69, 82, 32, 149, 4, 8, 73, 71, 65, 84, 85, 82, 69, 32, 58, 234, 1, 65, 96, 6, 70, 73, 78, 65, 76, 32, - 200, 1, 5, 82, 69, 83, 72, 45, 194, 215, 1, 76, 254, 165, 4, 71, 90, 90, - 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, 75, 198, 75, 66, 178, 216, 4, - 78, 134, 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 6, 26, 76, 215, 251, 12, - 89, 4, 156, 128, 6, 8, 84, 69, 82, 78, 65, 84, 69, 32, 243, 214, 1, 69, - 18, 116, 3, 78, 85, 78, 0, 5, 83, 65, 68, 72, 69, 0, 3, 84, 65, 87, 222, - 215, 1, 65, 178, 201, 6, 66, 143, 194, 5, 72, 5, 41, 8, 32, 87, 73, 84, - 72, 32, 86, 69, 2, 241, 214, 5, 4, 82, 84, 73, 67, 2, 157, 215, 1, 6, 65, - 89, 73, 78, 45, 68, 18, 42, 84, 198, 254, 5, 79, 223, 137, 6, 70, 10, 42, - 72, 194, 216, 1, 87, 251, 207, 11, 69, 4, 222, 136, 12, 82, 159, 2, 73, + 200, 1, 5, 82, 69, 83, 72, 45, 162, 216, 1, 76, 194, 170, 4, 71, 90, 90, + 34, 83, 66, 89, 198, 207, 1, 72, 234, 5, 75, 174, 81, 66, 170, 225, 4, + 78, 134, 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 6, 26, 76, 131, 143, 13, + 89, 4, 192, 133, 6, 8, 84, 69, 82, 78, 65, 84, 69, 32, 155, 214, 1, 69, + 18, 116, 3, 78, 85, 78, 0, 5, 83, 65, 68, 72, 69, 0, 3, 84, 65, 87, 190, + 216, 1, 65, 134, 211, 6, 66, 135, 203, 5, 72, 5, 41, 8, 32, 87, 73, 84, + 72, 32, 86, 69, 2, 133, 220, 5, 4, 82, 84, 73, 67, 2, 253, 215, 1, 6, 65, + 89, 73, 78, 45, 68, 18, 42, 84, 234, 131, 6, 79, 255, 148, 6, 70, 10, 42, + 72, 162, 217, 1, 87, 199, 226, 11, 69, 4, 162, 153, 12, 82, 159, 2, 73, 64, 60, 7, 76, 69, 84, 84, 69, 82, 32, 245, 3, 3, 78, 85, 77, 58, 202, 1, - 65, 38, 68, 74, 71, 34, 75, 34, 83, 78, 84, 230, 43, 90, 182, 166, 1, 76, - 50, 81, 238, 205, 1, 82, 154, 217, 2, 89, 158, 208, 1, 72, 174, 81, 66, - 178, 216, 4, 78, 134, 2, 87, 218, 103, 70, 171, 4, 77, 4, 198, 167, 3, - 76, 199, 254, 9, 89, 6, 32, 2, 72, 65, 183, 211, 1, 65, 4, 194, 156, 8, - 76, 215, 171, 5, 68, 4, 238, 44, 72, 191, 204, 5, 73, 4, 198, 208, 7, 65, - 187, 75, 72, 8, 26, 65, 211, 244, 12, 72, 6, 142, 194, 7, 77, 138, 133, - 6, 68, 143, 45, 84, 8, 130, 91, 72, 226, 150, 11, 69, 243, 131, 1, 65, 6, - 56, 4, 66, 69, 82, 32, 229, 185, 13, 4, 69, 82, 73, 67, 4, 26, 70, 195, - 217, 12, 79, 2, 199, 132, 12, 73, 146, 1, 80, 7, 79, 82, 75, 72, 79, 78, + 65, 38, 68, 74, 71, 34, 75, 34, 83, 78, 84, 254, 43, 90, 254, 166, 1, 76, + 50, 81, 214, 205, 1, 82, 246, 221, 2, 89, 198, 207, 1, 72, 150, 87, 66, + 170, 225, 4, 78, 134, 2, 87, 218, 103, 70, 171, 4, 77, 4, 146, 168, 3, + 76, 167, 145, 10, 89, 6, 32, 2, 72, 65, 151, 212, 1, 65, 4, 246, 166, 8, + 76, 207, 180, 5, 68, 4, 134, 45, 72, 203, 209, 5, 73, 4, 146, 213, 7, 65, + 163, 81, 72, 8, 26, 65, 255, 135, 13, 72, 6, 218, 198, 7, 77, 234, 147, + 6, 68, 143, 45, 84, 8, 230, 90, 72, 194, 167, 11, 69, 219, 134, 1, 65, 6, + 56, 4, 66, 69, 82, 32, 145, 205, 13, 4, 69, 82, 73, 67, 4, 26, 70, 235, + 234, 12, 79, 2, 139, 149, 12, 73, 146, 1, 80, 7, 79, 82, 75, 72, 79, 78, 32, 197, 3, 8, 89, 69, 78, 73, 83, 69, 73, 32, 84, 54, 65, 202, 1, 69, - 122, 73, 30, 79, 195, 134, 12, 66, 45, 106, 69, 194, 208, 9, 83, 158, - 160, 4, 66, 2, 68, 2, 71, 2, 76, 2, 78, 2, 81, 2, 82, 2, 84, 3, 89, 20, - 218, 240, 13, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 78, 2, 82, 2, 83, 2, 84, - 3, 89, 20, 74, 78, 138, 211, 13, 76, 158, 27, 83, 146, 1, 67, 2, 77, 2, - 80, 3, 90, 8, 178, 239, 13, 67, 2, 71, 2, 84, 3, 89, 7, 134, 239, 13, 67, - 3, 81, 13, 130, 3, 69, 234, 235, 13, 80, 2, 81, 3, 84, 62, 38, 65, 170, - 1, 69, 86, 73, 23, 79, 39, 98, 69, 162, 236, 13, 83, 62, 78, 86, 66, 2, - 68, 2, 71, 2, 76, 2, 81, 2, 82, 2, 84, 3, 89, 17, 150, 242, 11, 78, 154, - 251, 1, 66, 2, 71, 2, 75, 2, 84, 3, 89, 15, 46, 78, 174, 235, 13, 83, - 146, 1, 67, 3, 90, 6, 186, 236, 13, 67, 2, 84, 3, 89, 5, 151, 236, 13, - 81, 6, 26, 69, 235, 235, 13, 81, 5, 231, 235, 13, 75, 52, 148, 1, 10, 67, + 122, 73, 30, 79, 135, 151, 12, 66, 45, 106, 69, 170, 243, 9, 83, 226, + 144, 4, 66, 2, 68, 2, 71, 2, 76, 2, 78, 2, 81, 2, 82, 2, 84, 3, 89, 20, + 134, 132, 14, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 78, 2, 82, 2, 83, 2, 84, + 3, 89, 20, 74, 78, 182, 230, 13, 76, 158, 27, 83, 146, 1, 67, 2, 77, 2, + 80, 3, 90, 8, 222, 130, 14, 67, 2, 71, 2, 84, 3, 89, 7, 178, 130, 14, 67, + 3, 81, 13, 130, 3, 69, 150, 255, 13, 80, 2, 81, 3, 84, 62, 38, 65, 170, + 1, 69, 86, 73, 23, 79, 39, 98, 69, 206, 255, 13, 83, 62, 78, 86, 66, 2, + 68, 2, 71, 2, 76, 2, 81, 2, 82, 2, 84, 3, 89, 17, 218, 130, 12, 78, 130, + 254, 1, 66, 2, 71, 2, 75, 2, 84, 3, 89, 15, 46, 78, 218, 254, 13, 83, + 146, 1, 67, 3, 90, 6, 230, 255, 13, 67, 2, 84, 3, 89, 5, 195, 255, 13, + 81, 6, 26, 69, 151, 255, 13, 81, 5, 147, 255, 13, 75, 52, 148, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 36, 7, 76, 69, 84, 84, 69, 82, 32, - 233, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 8, 186, 236, - 5, 84, 143, 227, 3, 68, 36, 134, 200, 1, 65, 210, 206, 1, 82, 130, 216, - 2, 76, 58, 90, 34, 83, 66, 89, 128, 211, 1, 6, 70, 73, 78, 65, 76, 32, 0, - 6, 71, 73, 77, 69, 76, 45, 134, 3, 75, 198, 75, 66, 178, 216, 4, 78, 134, - 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 8, 56, 4, 84, 87, 79, 32, 214, - 196, 11, 70, 235, 129, 1, 66, 4, 246, 210, 12, 66, 71, 68, 6, 130, 181, - 1, 87, 240, 154, 3, 3, 65, 68, 85, 231, 205, 7, 77, 22, 212, 1, 34, 32, + 233, 1, 12, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 8, 222, 241, + 5, 84, 243, 231, 3, 68, 36, 230, 200, 1, 65, 186, 206, 1, 82, 222, 220, + 2, 76, 58, 90, 34, 83, 66, 89, 168, 210, 1, 6, 70, 73, 78, 65, 76, 32, 0, + 6, 71, 73, 77, 69, 76, 45, 134, 3, 75, 174, 81, 66, 170, 225, 4, 78, 134, + 2, 84, 2, 87, 218, 103, 80, 171, 4, 77, 8, 56, 4, 84, 87, 79, 32, 174, + 212, 11, 70, 187, 131, 1, 66, 4, 158, 228, 12, 66, 75, 68, 6, 146, 181, + 1, 87, 136, 160, 3, 3, 65, 68, 85, 207, 217, 7, 77, 22, 212, 1, 34, 32, 87, 73, 84, 72, 32, 69, 88, 67, 76, 65, 77, 65, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, 87, 73, 84, 72, 32, 76, 69, 70, 84, 32, 82, 44, 7, 67, - 79, 77, 73, 78, 71, 32, 194, 1, 69, 235, 147, 13, 73, 2, 21, 3, 73, 71, - 72, 2, 243, 235, 9, 84, 10, 148, 1, 6, 65, 85, 84, 79, 77, 79, 20, 7, 70, - 73, 82, 69, 32, 69, 78, 192, 224, 2, 2, 84, 65, 148, 232, 8, 6, 80, 79, - 76, 73, 67, 69, 155, 21, 66, 2, 135, 191, 11, 66, 2, 175, 202, 12, 71, 8, - 70, 32, 205, 174, 12, 11, 45, 80, 73, 69, 67, 69, 32, 83, 87, 73, 77, 6, - 40, 4, 68, 79, 84, 32, 215, 157, 10, 66, 4, 26, 79, 175, 159, 10, 76, 2, - 177, 233, 2, 11, 86, 69, 82, 32, 84, 87, 79, 32, 68, 79, 84, 46, 82, 69, - 188, 6, 2, 84, 73, 248, 212, 11, 5, 72, 73, 85, 67, 72, 171, 180, 1, 80, + 79, 77, 73, 78, 71, 32, 194, 1, 69, 151, 167, 13, 73, 2, 21, 3, 73, 71, + 72, 2, 203, 250, 9, 84, 10, 148, 1, 6, 65, 85, 84, 79, 77, 79, 20, 7, 70, + 73, 82, 69, 32, 69, 78, 144, 225, 2, 2, 84, 65, 148, 247, 8, 6, 80, 79, + 76, 73, 67, 69, 143, 22, 66, 2, 223, 206, 11, 66, 2, 215, 219, 12, 71, 8, + 70, 32, 229, 191, 12, 11, 45, 80, 73, 69, 67, 69, 32, 83, 87, 73, 77, 6, + 40, 4, 68, 79, 84, 32, 175, 173, 10, 66, 4, 26, 79, 135, 175, 10, 76, 2, + 129, 234, 2, 11, 86, 69, 82, 32, 84, 87, 79, 32, 68, 79, 84, 46, 82, 69, + 188, 6, 2, 84, 73, 188, 229, 11, 5, 72, 73, 85, 67, 72, 147, 183, 1, 80, 36, 70, 78, 201, 5, 12, 82, 65, 84, 73, 78, 71, 32, 83, 89, 83, 84, 69, 34, 22, 32, 139, 4, 45, 28, 108, 2, 66, 79, 32, 7, 67, 69, 78, 84, 82, - 69, 32, 114, 70, 70, 72, 42, 77, 226, 133, 7, 83, 215, 233, 3, 76, 4, - 198, 220, 13, 79, 155, 3, 88, 8, 50, 84, 222, 200, 11, 66, 222, 2, 65, - 191, 82, 67, 2, 37, 7, 69, 65, 82, 68, 82, 79, 80, 2, 191, 202, 11, 45, - 4, 44, 5, 73, 76, 69, 32, 70, 239, 130, 2, 79, 2, 235, 130, 2, 79, 2, 17, - 2, 65, 78, 2, 191, 175, 10, 68, 4, 57, 12, 65, 73, 76, 66, 79, 88, 32, + 69, 32, 114, 70, 70, 72, 42, 77, 142, 139, 7, 83, 135, 244, 3, 76, 4, + 242, 239, 13, 79, 155, 3, 88, 8, 50, 84, 174, 216, 11, 66, 222, 2, 65, + 135, 84, 67, 2, 37, 7, 69, 65, 82, 68, 82, 79, 80, 2, 143, 218, 11, 45, + 4, 44, 5, 73, 76, 69, 32, 70, 243, 131, 2, 79, 2, 239, 131, 2, 79, 2, 17, + 2, 65, 78, 2, 151, 191, 10, 68, 4, 57, 12, 65, 73, 76, 66, 79, 88, 32, 87, 73, 84, 72, 32, 4, 44, 3, 76, 79, 87, 21, 4, 82, 65, 73, 83, 2, 17, - 2, 69, 82, 2, 189, 243, 11, 2, 69, 68, 6, 108, 15, 67, 73, 82, 67, 85, - 73, 84, 45, 79, 85, 84, 80, 85, 84, 32, 181, 187, 12, 6, 79, 85, 84, 76, - 73, 78, 4, 18, 72, 3, 76, 2, 173, 191, 1, 4, 45, 84, 89, 80, 2, 153, 205, - 12, 6, 77, 32, 67, 79, 77, 77, 6, 64, 2, 79, 78, 237, 177, 1, 8, 67, 65, - 76, 32, 68, 73, 83, 67, 2, 167, 192, 11, 32, 206, 1, 112, 3, 65, 78, 71, - 66, 73, 212, 8, 5, 78, 65, 84, 69, 32, 32, 3, 84, 72, 79, 202, 171, 5, - 32, 175, 158, 6, 67, 6, 28, 2, 69, 32, 135, 22, 85, 4, 174, 133, 12, 66, - 175, 76, 72, 188, 1, 48, 5, 71, 73, 78, 65, 76, 21, 3, 89, 65, 32, 2, - 203, 133, 1, 32, 186, 1, 106, 65, 30, 70, 250, 1, 73, 32, 7, 76, 69, 84, - 84, 69, 82, 32, 142, 2, 83, 218, 1, 86, 243, 223, 11, 68, 4, 246, 249, 9, - 73, 3, 85, 12, 41, 8, 82, 65, 67, 84, 73, 79, 78, 32, 12, 56, 4, 79, 78, - 69, 32, 81, 6, 84, 72, 82, 69, 69, 32, 8, 42, 83, 162, 210, 11, 69, 46, - 72, 47, 81, 2, 173, 211, 11, 4, 73, 88, 84, 69, 4, 26, 83, 183, 212, 11, - 81, 2, 245, 253, 7, 4, 73, 88, 84, 69, 2, 217, 179, 12, 3, 83, 83, 72, - 104, 226, 1, 82, 238, 232, 6, 89, 134, 144, 3, 65, 38, 68, 114, 84, 46, - 86, 186, 5, 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, - 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 87, 186, 2, 69, - 3, 79, 6, 214, 208, 13, 72, 2, 82, 187, 2, 65, 18, 112, 20, 69, 81, 85, - 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, 84, 84, 69, 82, 32, 82, 29, - 4, 73, 71, 78, 32, 4, 186, 207, 13, 72, 3, 82, 14, 150, 189, 9, 67, 98, - 78, 242, 60, 65, 142, 239, 1, 79, 219, 164, 1, 86, 26, 49, 10, 79, 87, - 69, 76, 32, 83, 73, 71, 78, 32, 26, 142, 253, 9, 65, 38, 85, 22, 86, 186, - 201, 1, 73, 222, 137, 2, 69, 3, 79, 4, 174, 215, 6, 76, 243, 30, 82, 4, - 224, 240, 3, 2, 68, 79, 155, 183, 2, 71, 226, 1, 76, 4, 65, 71, 69, 32, - 140, 4, 6, 77, 65, 78, 89, 65, 32, 231, 202, 13, 67, 144, 1, 56, 6, 67, - 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 72, 45, 9, 76, 32, 76, 69, 84, - 84, 69, 82, 32, 72, 194, 1, 65, 38, 69, 90, 75, 42, 79, 22, 84, 226, 224, - 6, 72, 230, 239, 4, 67, 2, 68, 2, 71, 130, 179, 1, 83, 2, 90, 130, 3, 66, - 138, 66, 76, 2, 77, 2, 78, 2, 80, 2, 87, 186, 2, 73, 3, 85, 9, 182, 209, - 11, 73, 135, 251, 1, 72, 15, 26, 72, 159, 252, 12, 73, 10, 222, 191, 9, - 84, 222, 145, 2, 67, 138, 248, 1, 75, 3, 80, 6, 134, 201, 13, 72, 2, 89, - 187, 2, 65, 5, 183, 251, 12, 73, 6, 194, 131, 13, 83, 195, 71, 65, 80, - 52, 7, 76, 69, 84, 84, 69, 82, 32, 143, 217, 11, 68, 60, 246, 1, 65, 38, - 67, 22, 68, 38, 75, 34, 83, 30, 77, 242, 240, 2, 81, 206, 144, 8, 79, - 224, 123, 2, 76, 65, 208, 73, 2, 78, 85, 134, 2, 87, 142, 62, 69, 234, - 61, 66, 2, 70, 2, 71, 2, 72, 2, 74, 2, 82, 2, 84, 2, 88, 2, 89, 186, 2, - 73, 3, 85, 7, 142, 250, 2, 76, 167, 206, 10, 65, 2, 163, 202, 12, 65, 4, - 150, 187, 8, 69, 175, 138, 5, 72, 4, 166, 198, 12, 65, 251, 126, 72, 4, - 26, 72, 159, 199, 13, 65, 2, 199, 199, 12, 73, 124, 68, 11, 79, 77, 65, - 78, 32, 83, 73, 89, 65, 81, 32, 231, 141, 13, 69, 122, 172, 1, 17, 65, - 76, 84, 69, 82, 78, 65, 84, 69, 32, 78, 85, 77, 66, 69, 82, 32, 200, 1, - 13, 70, 82, 65, 67, 84, 73, 79, 78, 32, 79, 78, 69, 32, 48, 3, 77, 65, - 82, 47, 78, 26, 54, 70, 50, 83, 46, 84, 198, 170, 12, 78, 235, 110, 69, - 6, 236, 202, 5, 3, 79, 85, 82, 151, 253, 6, 73, 6, 188, 202, 5, 2, 73, - 88, 251, 137, 6, 69, 10, 170, 184, 2, 69, 12, 2, 87, 79, 131, 156, 9, 72, - 4, 26, 83, 131, 192, 11, 72, 2, 239, 192, 11, 73, 2, 11, 82, 2, 11, 65, - 2, 255, 248, 11, 84, 90, 33, 6, 85, 77, 66, 69, 82, 32, 90, 58, 69, 66, - 70, 94, 78, 26, 83, 78, 84, 183, 172, 5, 79, 10, 25, 4, 73, 71, 72, 84, - 11, 170, 182, 2, 89, 255, 183, 5, 32, 20, 18, 73, 35, 79, 10, 166, 2, 70, - 199, 171, 5, 86, 10, 134, 2, 82, 209, 171, 5, 2, 85, 82, 10, 65, 3, 73, - 78, 69, 20, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 11, 166, 1, 84, 191, - 235, 7, 32, 24, 34, 72, 50, 87, 235, 179, 2, 69, 10, 34, 73, 249, 171, 5, - 2, 82, 69, 4, 51, 82, 10, 26, 69, 223, 171, 5, 79, 4, 11, 78, 4, 11, 84, - 4, 191, 179, 2, 89, 84, 34, 84, 161, 162, 11, 2, 78, 67, 82, 42, 66, 37, - 6, 76, 73, 78, 69, 68, 32, 2, 141, 178, 12, 4, 79, 88, 32, 84, 80, 92, 7, - 76, 65, 84, 73, 78, 32, 67, 222, 189, 6, 87, 146, 233, 4, 66, 246, 13, - 71, 159, 23, 68, 54, 130, 170, 7, 65, 143, 210, 4, 82, 12, 18, 72, 35, - 76, 2, 137, 172, 12, 3, 69, 65, 84, 10, 32, 2, 65, 80, 247, 162, 12, 73, - 9, 29, 5, 80, 73, 78, 71, 32, 6, 40, 6, 87, 72, 73, 84, 69, 32, 51, 66, - 4, 44, 5, 65, 78, 68, 32, 66, 223, 250, 9, 83, 2, 197, 250, 9, 4, 76, 65, - 67, 75, 146, 13, 210, 1, 65, 174, 65, 68, 30, 69, 134, 13, 72, 138, 25, - 73, 206, 4, 76, 148, 15, 2, 78, 80, 54, 79, 234, 8, 82, 208, 15, 15, 83, - 65, 76, 84, 69, 82, 32, 80, 65, 72, 76, 65, 86, 73, 32, 214, 5, 85, 183, - 159, 12, 77, 154, 6, 134, 2, 68, 38, 71, 212, 1, 11, 72, 65, 87, 72, 32, - 72, 77, 79, 78, 71, 32, 134, 23, 76, 130, 6, 78, 58, 82, 160, 22, 2, 83, - 83, 132, 2, 10, 85, 32, 67, 73, 78, 32, 72, 65, 85, 32, 180, 7, 3, 87, - 32, 80, 188, 241, 7, 2, 67, 75, 157, 133, 5, 4, 80, 69, 82, 67, 5, 237, - 184, 1, 4, 68, 73, 78, 71, 14, 26, 69, 219, 145, 13, 79, 13, 34, 32, 186, - 182, 13, 82, 3, 83, 6, 72, 6, 87, 73, 84, 72, 32, 67, 241, 153, 4, 6, 70, - 65, 67, 73, 78, 71, 4, 50, 85, 193, 226, 7, 6, 73, 82, 67, 76, 69, 68, 2, - 131, 172, 12, 82, 254, 1, 190, 1, 67, 160, 6, 9, 77, 65, 82, 75, 32, 67, - 73, 77, 32, 160, 1, 7, 78, 85, 77, 66, 69, 82, 32, 244, 1, 5, 83, 73, 71, - 78, 32, 144, 10, 7, 86, 79, 87, 69, 76, 32, 75, 255, 174, 11, 68, 78, 92, - 9, 76, 65, 78, 32, 83, 73, 71, 78, 32, 137, 3, 9, 79, 78, 83, 79, 78, 65, - 78, 84, 32, 38, 132, 1, 2, 72, 65, 38, 75, 46, 76, 34, 84, 82, 86, 30, - 89, 148, 9, 2, 88, 89, 172, 5, 2, 80, 72, 174, 1, 70, 165, 2, 2, 77, 85, - 4, 190, 197, 2, 87, 187, 236, 10, 77, 6, 246, 15, 72, 234, 130, 13, 79, - 159, 14, 87, 4, 210, 12, 65, 251, 230, 12, 73, 8, 22, 83, 247, 9, 72, 6, - 180, 196, 2, 3, 72, 69, 69, 214, 176, 9, 65, 3, 87, 4, 254, 195, 2, 65, - 3, 87, 4, 226, 195, 2, 65, 211, 166, 10, 69, 40, 122, 67, 38, 72, 46, 78, - 60, 2, 80, 76, 2, 81, 250, 221, 2, 76, 2, 77, 2, 82, 2, 86, 2, 88, 2, 89, - 147, 171, 10, 65, 4, 130, 223, 2, 72, 147, 171, 10, 65, 6, 222, 222, 2, - 76, 2, 78, 147, 171, 10, 65, 12, 58, 67, 22, 84, 230, 221, 2, 75, 2, 76, - 147, 171, 10, 65, 2, 247, 221, 2, 72, 4, 226, 221, 2, 72, 3, 83, 14, 42, - 75, 50, 83, 38, 84, 223, 155, 13, 72, 4, 26, 72, 159, 239, 12, 69, 2, - 251, 156, 6, 65, 4, 230, 241, 11, 85, 255, 186, 1, 79, 4, 218, 240, 11, - 85, 215, 18, 65, 14, 52, 7, 72, 85, 78, 68, 82, 69, 68, 38, 84, 79, 77, - 4, 108, 2, 32, 77, 251, 170, 13, 83, 8, 24, 2, 69, 78, 51, 82, 6, 26, 32, - 143, 171, 13, 83, 4, 18, 66, 35, 84, 2, 205, 208, 4, 3, 73, 76, 76, 2, - 11, 72, 2, 225, 235, 9, 3, 79, 85, 83, 72, 188, 1, 4, 67, 73, 77, 32, - 246, 2, 72, 32, 3, 73, 66, 32, 22, 77, 86, 78, 50, 84, 124, 4, 86, 79, - 83, 32, 198, 1, 88, 208, 1, 6, 90, 87, 74, 32, 84, 72, 238, 176, 1, 76, - 203, 223, 4, 65, 16, 174, 1, 67, 84, 5, 78, 82, 69, 83, 32, 22, 84, 240, - 234, 11, 7, 80, 85, 66, 32, 68, 65, 87, 133, 187, 1, 16, 72, 65, 73, 83, - 32, 76, 85, 83, 32, 78, 84, 79, 71, 32, 78, 84, 4, 48, 7, 85, 65, 77, 32, - 84, 83, 72, 255, 3, 72, 2, 11, 79, 2, 195, 186, 2, 79, 2, 163, 183, 1, - 84, 6, 52, 3, 88, 87, 86, 173, 135, 10, 4, 83, 79, 86, 32, 5, 157, 150, - 6, 4, 32, 67, 72, 87, 4, 158, 72, 78, 131, 202, 12, 76, 2, 219, 234, 11, - 89, 6, 40, 4, 69, 69, 74, 32, 191, 231, 12, 85, 4, 128, 3, 2, 84, 83, 57, - 2, 83, 85, 4, 26, 84, 207, 241, 8, 81, 2, 155, 184, 2, 85, 6, 196, 1, 7, - 88, 72, 69, 69, 74, 32, 67, 236, 167, 8, 12, 72, 73, 82, 68, 45, 83, 84, - 65, 71, 69, 32, 72, 167, 214, 4, 65, 14, 54, 70, 22, 83, 30, 84, 134, 69, - 76, 239, 240, 7, 78, 2, 223, 144, 13, 69, 2, 249, 146, 6, 2, 69, 69, 6, - 42, 72, 29, 6, 83, 72, 65, 66, 32, 67, 4, 82, 73, 135, 145, 13, 79, 2, - 239, 203, 5, 69, 14, 34, 73, 22, 89, 207, 155, 11, 65, 2, 247, 229, 11, - 65, 10, 40, 4, 69, 69, 77, 32, 171, 130, 13, 79, 8, 84, 3, 78, 84, 88, - 200, 144, 6, 2, 84, 79, 218, 167, 2, 82, 181, 172, 3, 2, 70, 65, 2, 199, - 144, 6, 73, 2, 247, 179, 2, 65, 56, 50, 65, 66, 69, 38, 73, 2, 85, 38, - 79, 39, 87, 20, 170, 1, 65, 2, 73, 2, 85, 2, 87, 190, 158, 13, 66, 3, 86, - 8, 106, 69, 190, 158, 13, 66, 3, 86, 8, 70, 65, 190, 158, 13, 66, 3, 86, - 8, 34, 79, 190, 158, 13, 66, 3, 86, 4, 186, 158, 13, 66, 3, 86, 76, 18, - 76, 23, 77, 2, 175, 224, 12, 65, 74, 74, 32, 104, 6, 89, 82, 69, 78, 69, - 32, 201, 136, 4, 4, 83, 32, 85, 80, 8, 80, 3, 66, 82, 65, 150, 173, 11, - 84, 172, 97, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 175, 250, 12, 78, 64, - 80, 2, 76, 69, 40, 4, 82, 73, 71, 72, 253, 2, 7, 78, 85, 77, 66, 69, 82, - 32, 48, 38, 70, 89, 5, 84, 84, 69, 82, 32, 2, 57, 12, 84, 45, 80, 79, 73, - 78, 84, 73, 78, 71, 32, 70, 2, 177, 226, 5, 2, 76, 69, 46, 224, 1, 5, 70, - 73, 78, 65, 76, 30, 84, 246, 118, 68, 34, 76, 50, 81, 238, 205, 1, 82, - 178, 215, 2, 65, 50, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, - 75, 198, 75, 66, 178, 216, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, - 2, 217, 152, 12, 2, 32, 78, 4, 222, 150, 11, 69, 243, 131, 1, 65, 14, - 198, 120, 84, 226, 175, 10, 70, 251, 86, 79, 4, 32, 2, 67, 65, 203, 216, - 12, 68, 2, 131, 132, 12, 75, 184, 2, 94, 65, 220, 1, 11, 69, 78, 84, 72, - 69, 83, 73, 90, 69, 68, 32, 242, 16, 84, 131, 180, 12, 82, 14, 80, 5, 71, - 82, 65, 80, 72, 52, 5, 76, 76, 69, 76, 32, 137, 158, 6, 2, 67, 72, 6, - 198, 232, 9, 32, 194, 222, 1, 85, 207, 145, 1, 79, 6, 44, 5, 87, 73, 84, - 72, 32, 219, 246, 12, 84, 4, 150, 193, 6, 84, 131, 176, 4, 72, 150, 2, - 252, 1, 7, 72, 65, 78, 71, 85, 76, 32, 188, 4, 10, 73, 68, 69, 79, 71, - 82, 65, 80, 72, 32, 188, 7, 18, 75, 79, 82, 69, 65, 78, 32, 67, 72, 65, - 82, 65, 67, 84, 69, 82, 32, 79, 44, 7, 78, 85, 77, 66, 69, 82, 32, 202, - 201, 4, 68, 213, 169, 2, 2, 76, 65, 58, 102, 67, 110, 72, 30, 75, 66, 77, - 34, 78, 34, 80, 62, 82, 30, 83, 26, 84, 73, 5, 73, 69, 85, 78, 71, 10, - 34, 72, 33, 4, 73, 69, 85, 67, 4, 141, 3, 4, 73, 69, 85, 67, 7, 11, 32, - 4, 234, 145, 13, 65, 3, 85, 4, 197, 2, 3, 73, 69, 85, 8, 168, 2, 5, 72, - 73, 69, 85, 75, 13, 5, 73, 89, 69, 79, 75, 4, 245, 1, 4, 73, 69, 85, 77, - 4, 213, 1, 4, 73, 69, 85, 78, 8, 168, 1, 5, 72, 73, 69, 85, 80, 13, 4, - 73, 69, 85, 80, 4, 121, 4, 73, 69, 85, 76, 4, 93, 3, 73, 79, 83, 8, 56, - 5, 72, 73, 69, 85, 84, 13, 5, 73, 75, 69, 85, 84, 4, 11, 72, 5, 195, 140, - 13, 32, 72, 148, 1, 2, 65, 76, 30, 67, 74, 69, 82, 70, 112, 2, 76, 65, - 22, 77, 38, 78, 32, 2, 82, 69, 78, 83, 138, 2, 84, 50, 87, 158, 156, 11, - 72, 139, 82, 79, 2, 241, 133, 8, 2, 76, 73, 4, 32, 2, 79, 78, 211, 133, - 11, 65, 2, 185, 240, 7, 4, 71, 82, 65, 84, 6, 42, 78, 222, 163, 9, 65, - 163, 190, 3, 73, 2, 169, 4, 5, 84, 69, 82, 80, 82, 10, 58, 73, 160, 130, - 12, 5, 69, 83, 84, 73, 86, 239, 16, 79, 6, 208, 2, 3, 78, 65, 78, 186, - 242, 12, 82, 3, 86, 2, 195, 210, 12, 66, 4, 234, 182, 10, 69, 195, 132, - 2, 79, 4, 198, 241, 11, 73, 195, 1, 65, 8, 38, 83, 174, 250, 11, 80, 219, - 109, 65, 4, 134, 209, 6, 79, 167, 185, 6, 84, 18, 58, 69, 34, 79, 22, 80, - 34, 84, 50, 85, 167, 252, 11, 73, 4, 226, 181, 11, 86, 199, 82, 76, 2, - 243, 240, 7, 67, 2, 11, 69, 2, 179, 176, 4, 67, 4, 26, 85, 239, 216, 11, - 79, 2, 147, 247, 12, 68, 4, 26, 80, 175, 136, 13, 78, 2, 21, 3, 69, 82, - 86, 2, 135, 255, 11, 73, 6, 186, 152, 11, 72, 230, 159, 1, 69, 239, 48, - 87, 4, 246, 208, 9, 79, 227, 254, 1, 65, 4, 182, 153, 8, 32, 137, 158, 4, - 2, 74, 69, 22, 42, 69, 46, 70, 42, 78, 30, 83, 51, 84, 4, 216, 1, 3, 73, - 71, 72, 211, 240, 4, 76, 4, 160, 1, 2, 79, 85, 13, 2, 73, 70, 2, 133, 1, - 3, 73, 78, 69, 4, 34, 73, 73, 4, 69, 86, 69, 78, 2, 71, 88, 8, 34, 72, - 46, 87, 135, 181, 12, 69, 2, 11, 73, 2, 11, 82, 2, 131, 177, 11, 84, 4, - 11, 69, 4, 222, 151, 11, 78, 167, 112, 76, 18, 128, 1, 3, 73, 65, 76, - 216, 1, 3, 89, 32, 80, 168, 169, 8, 6, 78, 69, 82, 83, 72, 73, 169, 211, - 3, 6, 32, 65, 76, 84, 69, 82, 12, 62, 32, 193, 123, 10, 76, 89, 45, 82, - 69, 67, 89, 67, 76, 69, 10, 38, 68, 41, 5, 76, 73, 78, 69, 32, 2, 217, - 169, 4, 5, 73, 70, 70, 69, 82, 8, 222, 200, 3, 68, 226, 45, 70, 20, 4, - 66, 65, 67, 75, 199, 139, 9, 85, 2, 191, 237, 3, 79, 10, 98, 69, 52, 9, - 73, 86, 69, 45, 80, 85, 76, 76, 45, 89, 9, 80, 79, 82, 84, 32, 67, 79, - 78, 84, 4, 224, 226, 9, 4, 78, 71, 69, 82, 219, 138, 2, 68, 4, 40, 4, 68, - 79, 87, 78, 1, 2, 85, 80, 2, 185, 171, 5, 6, 45, 79, 85, 84, 80, 85, 2, - 231, 236, 11, 82, 114, 192, 1, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, - 84, 79, 80, 62, 76, 160, 3, 3, 82, 73, 83, 36, 14, 77, 73, 68, 45, 76, + 2, 69, 82, 2, 129, 132, 12, 2, 69, 68, 6, 108, 15, 67, 73, 82, 67, 85, + 73, 84, 45, 79, 85, 84, 80, 85, 84, 32, 221, 204, 12, 6, 79, 85, 84, 76, + 73, 78, 4, 18, 72, 3, 76, 2, 161, 192, 1, 4, 45, 84, 89, 80, 2, 169, 222, + 12, 6, 77, 32, 67, 79, 77, 77, 6, 64, 2, 79, 78, 253, 177, 1, 8, 67, 65, + 76, 32, 68, 73, 83, 67, 2, 247, 207, 11, 32, 208, 1, 104, 3, 65, 78, 71, + 66, 67, 34, 73, 212, 8, 5, 78, 65, 84, 69, 32, 32, 3, 84, 72, 79, 199, + 176, 5, 32, 6, 28, 2, 69, 32, 167, 22, 85, 4, 198, 150, 12, 66, 203, 78, + 72, 4, 186, 174, 13, 85, 223, 61, 65, 188, 1, 48, 5, 71, 73, 78, 65, 76, + 21, 3, 89, 65, 32, 2, 183, 133, 1, 32, 186, 1, 106, 65, 30, 70, 250, 1, + 73, 32, 7, 76, 69, 84, 84, 69, 82, 32, 142, 2, 83, 218, 1, 86, 159, 240, + 11, 68, 4, 182, 137, 10, 73, 3, 85, 12, 41, 8, 82, 65, 67, 84, 73, 79, + 78, 32, 12, 56, 4, 79, 78, 69, 32, 81, 6, 84, 72, 82, 69, 69, 32, 8, 42, + 83, 206, 226, 11, 69, 46, 72, 47, 81, 2, 217, 227, 11, 4, 73, 88, 84, 69, + 4, 26, 83, 227, 228, 11, 81, 2, 145, 136, 8, 4, 73, 88, 84, 69, 2, 233, + 196, 12, 3, 83, 83, 72, 104, 226, 1, 82, 130, 238, 6, 89, 178, 154, 3, + 65, 38, 68, 114, 84, 46, 86, 186, 5, 85, 186, 202, 1, 73, 42, 76, 226, + 195, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, + 72, 2, 77, 2, 87, 186, 2, 69, 3, 79, 6, 234, 227, 13, 72, 2, 82, 187, 2, + 65, 18, 112, 20, 69, 81, 85, 69, 78, 67, 69, 32, 70, 79, 82, 32, 76, 69, + 84, 84, 69, 82, 32, 82, 29, 4, 73, 71, 78, 32, 4, 206, 226, 13, 72, 3, + 82, 14, 138, 199, 9, 67, 98, 78, 190, 66, 65, 250, 239, 1, 79, 195, 167, + 1, 86, 26, 49, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 26, 206, 140, + 10, 65, 38, 85, 22, 86, 166, 202, 1, 73, 198, 140, 2, 69, 3, 79, 4, 194, + 220, 6, 76, 243, 30, 82, 4, 240, 245, 3, 2, 68, 79, 139, 183, 2, 71, 226, + 1, 76, 4, 65, 71, 69, 32, 140, 4, 6, 77, 65, 78, 89, 65, 32, 251, 221, + 13, 67, 144, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 72, + 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 72, 194, 1, 65, 38, 69, 90, + 75, 42, 79, 22, 84, 246, 229, 6, 72, 254, 250, 4, 67, 2, 68, 2, 71, 234, + 181, 1, 83, 2, 90, 130, 3, 66, 138, 66, 76, 2, 77, 2, 78, 2, 80, 2, 87, + 186, 2, 73, 3, 85, 9, 226, 225, 11, 73, 239, 253, 1, 72, 15, 26, 72, 179, + 143, 13, 73, 10, 134, 202, 9, 84, 226, 151, 2, 67, 242, 250, 1, 75, 3, + 80, 6, 154, 220, 13, 72, 2, 89, 187, 2, 65, 5, 203, 142, 13, 73, 6, 214, + 150, 13, 83, 195, 71, 65, 80, 52, 7, 76, 69, 84, 84, 69, 82, 32, 187, + 233, 11, 68, 60, 246, 1, 65, 38, 67, 22, 68, 38, 75, 34, 83, 30, 77, 162, + 241, 2, 81, 226, 159, 8, 79, 148, 125, 2, 76, 65, 236, 75, 2, 78, 85, + 134, 2, 87, 142, 62, 69, 234, 61, 66, 2, 70, 2, 71, 2, 72, 2, 74, 2, 82, + 2, 84, 2, 88, 2, 89, 186, 2, 73, 3, 85, 7, 194, 250, 2, 76, 135, 225, 10, + 65, 2, 183, 221, 12, 65, 4, 222, 197, 8, 69, 251, 146, 5, 72, 4, 186, + 217, 12, 65, 251, 126, 72, 4, 26, 72, 179, 218, 13, 65, 2, 219, 218, 12, + 73, 124, 68, 11, 79, 77, 65, 78, 32, 83, 73, 89, 65, 81, 32, 251, 160, + 13, 69, 122, 172, 1, 17, 65, 76, 84, 69, 82, 78, 65, 84, 69, 32, 78, 85, + 77, 66, 69, 82, 32, 200, 1, 13, 70, 82, 65, 67, 84, 73, 79, 78, 32, 79, + 78, 69, 32, 48, 3, 77, 65, 82, 47, 78, 26, 54, 70, 50, 83, 46, 84, 214, + 187, 12, 78, 239, 112, 69, 6, 248, 207, 5, 3, 79, 85, 82, 159, 139, 7, + 73, 6, 200, 207, 5, 2, 73, 88, 155, 149, 6, 69, 10, 226, 184, 2, 69, 12, + 2, 87, 79, 247, 171, 9, 72, 4, 26, 83, 175, 208, 11, 72, 2, 155, 209, 11, + 73, 2, 11, 82, 2, 11, 65, 2, 247, 137, 12, 84, 90, 33, 6, 85, 77, 66, 69, + 82, 32, 90, 58, 69, 66, 70, 94, 78, 26, 83, 78, 84, 179, 177, 5, 79, 10, + 25, 4, 73, 71, 72, 84, 11, 226, 182, 2, 89, 227, 193, 5, 32, 20, 18, 73, + 35, 79, 10, 166, 2, 70, 195, 176, 5, 86, 10, 134, 2, 82, 205, 176, 5, 2, + 85, 82, 10, 65, 3, 73, 78, 69, 20, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, + 11, 166, 1, 84, 219, 245, 7, 32, 24, 34, 72, 50, 87, 163, 180, 2, 69, 10, + 34, 73, 245, 176, 5, 2, 82, 69, 4, 51, 82, 10, 26, 69, 219, 176, 5, 79, + 4, 11, 78, 4, 11, 84, 4, 247, 179, 2, 89, 84, 34, 84, 217, 177, 11, 2, + 78, 67, 82, 42, 66, 37, 6, 76, 73, 78, 69, 68, 32, 2, 133, 195, 12, 4, + 79, 88, 32, 84, 80, 92, 7, 76, 65, 84, 73, 78, 32, 67, 242, 194, 6, 87, + 182, 243, 4, 66, 234, 14, 71, 159, 23, 68, 54, 186, 174, 7, 65, 215, 222, + 4, 82, 12, 18, 72, 43, 76, 2, 17, 2, 69, 65, 2, 239, 188, 12, 84, 10, 32, + 2, 65, 80, 255, 179, 12, 73, 9, 29, 5, 80, 73, 78, 71, 32, 6, 40, 6, 87, + 72, 73, 84, 69, 32, 51, 66, 4, 44, 5, 65, 78, 68, 32, 66, 151, 138, 10, + 83, 2, 253, 137, 10, 4, 76, 65, 67, 75, 152, 13, 150, 1, 65, 206, 65, 68, + 30, 69, 134, 13, 72, 138, 25, 73, 202, 4, 76, 160, 15, 2, 78, 80, 54, 79, + 238, 8, 82, 210, 15, 83, 186, 6, 85, 239, 177, 12, 77, 158, 6, 134, 2, + 68, 38, 71, 212, 1, 11, 72, 65, 87, 72, 32, 72, 77, 79, 78, 71, 32, 134, + 23, 76, 130, 6, 78, 58, 82, 196, 22, 2, 83, 83, 132, 2, 10, 85, 32, 67, + 73, 78, 32, 72, 65, 85, 32, 176, 7, 3, 87, 32, 80, 152, 252, 7, 2, 67, + 75, 233, 141, 5, 4, 80, 69, 82, 67, 5, 253, 185, 1, 4, 68, 73, 78, 71, + 14, 26, 69, 163, 165, 13, 79, 13, 34, 32, 130, 202, 13, 82, 3, 83, 6, 72, + 6, 87, 73, 84, 72, 32, 67, 181, 159, 4, 6, 70, 65, 67, 73, 78, 71, 4, 50, + 85, 145, 237, 7, 6, 73, 82, 67, 76, 69, 68, 2, 175, 189, 12, 82, 254, 1, + 190, 1, 67, 160, 6, 9, 77, 65, 82, 75, 32, 67, 73, 77, 32, 160, 1, 7, 78, + 85, 77, 66, 69, 82, 32, 244, 1, 5, 83, 73, 71, 78, 32, 144, 10, 7, 86, + 79, 87, 69, 76, 32, 75, 223, 191, 11, 68, 78, 92, 9, 76, 65, 78, 32, 83, + 73, 71, 78, 32, 137, 3, 9, 79, 78, 83, 79, 78, 65, 78, 84, 32, 38, 132, + 1, 2, 72, 65, 38, 75, 46, 76, 34, 84, 82, 86, 30, 89, 148, 9, 2, 88, 89, + 172, 5, 2, 80, 72, 174, 1, 70, 165, 2, 2, 77, 85, 4, 170, 198, 2, 87, + 151, 255, 10, 77, 6, 246, 15, 72, 178, 150, 13, 79, 159, 14, 87, 4, 210, + 12, 65, 195, 250, 12, 73, 8, 22, 83, 247, 9, 72, 6, 160, 197, 2, 3, 72, + 69, 69, 158, 193, 9, 65, 3, 87, 4, 234, 196, 2, 65, 3, 87, 4, 206, 196, + 2, 65, 175, 185, 10, 69, 40, 122, 67, 38, 72, 46, 78, 60, 2, 80, 76, 2, + 81, 222, 222, 2, 76, 2, 77, 2, 82, 2, 86, 2, 88, 2, 89, 247, 189, 10, 65, + 4, 230, 223, 2, 72, 247, 189, 10, 65, 6, 194, 223, 2, 76, 2, 78, 247, + 189, 10, 65, 12, 58, 67, 22, 84, 202, 222, 2, 75, 2, 76, 247, 189, 10, + 65, 2, 219, 222, 2, 72, 4, 198, 222, 2, 72, 3, 83, 14, 42, 75, 50, 83, + 38, 84, 167, 175, 13, 72, 4, 26, 72, 231, 130, 13, 69, 2, 175, 162, 6, + 65, 4, 154, 131, 12, 85, 147, 189, 1, 79, 4, 142, 130, 12, 85, 215, 18, + 65, 14, 52, 7, 72, 85, 78, 68, 82, 69, 68, 38, 84, 79, 77, 4, 108, 2, 32, + 77, 195, 190, 13, 83, 8, 24, 2, 69, 78, 51, 82, 6, 26, 32, 215, 190, 13, + 83, 4, 18, 66, 35, 84, 2, 253, 213, 4, 3, 73, 76, 76, 2, 11, 72, 2, 213, + 251, 9, 3, 79, 85, 83, 72, 188, 1, 4, 67, 73, 77, 32, 246, 2, 72, 32, 3, + 73, 66, 32, 22, 77, 86, 78, 50, 84, 124, 4, 86, 79, 83, 32, 198, 1, 88, + 208, 1, 6, 90, 87, 74, 32, 84, 72, 142, 178, 1, 76, 223, 227, 4, 65, 16, + 174, 1, 67, 84, 5, 78, 82, 69, 83, 32, 22, 84, 164, 252, 11, 7, 80, 85, + 66, 32, 68, 65, 87, 153, 189, 1, 16, 72, 65, 73, 83, 32, 76, 85, 83, 32, + 78, 84, 79, 71, 32, 78, 84, 4, 48, 7, 85, 65, 77, 32, 84, 83, 72, 255, 3, + 72, 2, 11, 79, 2, 175, 187, 2, 79, 2, 195, 184, 1, 84, 6, 52, 3, 88, 87, + 86, 161, 151, 10, 4, 83, 79, 86, 32, 5, 209, 155, 6, 4, 32, 67, 72, 87, + 4, 190, 72, 78, 171, 221, 12, 76, 2, 143, 252, 11, 89, 6, 40, 4, 69, 69, + 74, 32, 135, 251, 12, 85, 4, 128, 3, 2, 84, 83, 57, 2, 83, 85, 4, 26, 84, + 187, 252, 8, 81, 2, 135, 185, 2, 85, 6, 196, 1, 7, 88, 72, 69, 69, 74, + 32, 67, 224, 178, 8, 12, 72, 73, 82, 68, 45, 83, 84, 65, 71, 69, 32, 72, + 251, 222, 4, 65, 14, 54, 70, 22, 83, 30, 84, 166, 69, 76, 195, 251, 7, + 78, 2, 167, 164, 13, 69, 2, 173, 152, 6, 2, 69, 69, 6, 42, 72, 29, 6, 83, + 72, 65, 66, 32, 67, 4, 82, 73, 207, 164, 13, 79, 2, 175, 209, 5, 69, 14, + 34, 73, 22, 89, 175, 172, 11, 65, 2, 171, 247, 11, 65, 10, 40, 4, 69, 69, + 77, 32, 243, 149, 13, 79, 8, 84, 3, 78, 84, 88, 252, 149, 6, 2, 84, 79, + 154, 173, 2, 82, 245, 178, 3, 2, 70, 65, 2, 251, 149, 6, 73, 2, 227, 180, + 2, 65, 56, 50, 65, 66, 69, 38, 73, 2, 85, 38, 79, 39, 87, 20, 170, 1, 65, + 2, 73, 2, 85, 2, 87, 134, 178, 13, 66, 3, 86, 8, 106, 69, 134, 178, 13, + 66, 3, 86, 8, 70, 65, 134, 178, 13, 66, 3, 86, 8, 34, 79, 134, 178, 13, + 66, 3, 86, 4, 130, 178, 13, 66, 3, 86, 76, 18, 76, 23, 77, 2, 247, 243, + 12, 65, 74, 74, 32, 104, 6, 89, 82, 69, 78, 69, 32, 141, 142, 4, 4, 83, + 32, 85, 80, 8, 80, 3, 66, 82, 65, 246, 189, 11, 84, 248, 97, 4, 68, 79, + 87, 78, 1, 2, 85, 80, 2, 247, 141, 13, 78, 64, 80, 2, 76, 69, 40, 4, 82, + 73, 71, 72, 253, 2, 7, 78, 85, 77, 66, 69, 82, 32, 48, 38, 70, 89, 5, 84, + 84, 69, 82, 32, 2, 57, 12, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, + 70, 2, 229, 231, 5, 2, 76, 69, 46, 224, 1, 5, 70, 73, 78, 65, 76, 30, 84, + 242, 119, 68, 34, 76, 50, 81, 214, 205, 1, 82, 142, 220, 2, 65, 50, 71, + 90, 90, 34, 83, 66, 89, 198, 207, 1, 72, 234, 5, 75, 174, 81, 66, 170, + 225, 4, 78, 134, 2, 87, 218, 103, 80, 171, 4, 77, 2, 161, 172, 12, 2, 32, + 78, 4, 190, 167, 11, 69, 219, 134, 1, 65, 14, 194, 121, 84, 198, 191, 10, + 70, 223, 87, 79, 4, 32, 2, 67, 65, 147, 236, 12, 68, 2, 191, 149, 12, 75, + 188, 2, 94, 65, 220, 1, 11, 69, 78, 84, 72, 69, 83, 73, 90, 69, 68, 32, + 242, 16, 84, 203, 199, 12, 82, 14, 80, 5, 71, 82, 65, 80, 72, 52, 5, 76, + 76, 69, 76, 32, 209, 163, 6, 2, 67, 72, 6, 186, 248, 9, 32, 250, 223, 1, + 85, 235, 147, 1, 79, 6, 44, 5, 87, 73, 84, 72, 32, 163, 138, 13, 84, 4, + 222, 198, 6, 84, 175, 186, 4, 72, 150, 2, 252, 1, 7, 72, 65, 78, 71, 85, + 76, 32, 188, 4, 10, 73, 68, 69, 79, 71, 82, 65, 80, 72, 32, 188, 7, 18, + 75, 79, 82, 69, 65, 78, 32, 67, 72, 65, 82, 65, 67, 84, 69, 82, 32, 79, + 44, 7, 78, 85, 77, 66, 69, 82, 32, 250, 206, 4, 68, 145, 169, 2, 2, 76, + 65, 58, 102, 67, 110, 72, 30, 75, 66, 77, 34, 78, 34, 80, 62, 82, 30, 83, + 26, 84, 73, 5, 73, 69, 85, 78, 71, 10, 34, 72, 33, 4, 73, 69, 85, 67, 4, + 141, 3, 4, 73, 69, 85, 67, 7, 11, 32, 4, 178, 165, 13, 65, 3, 85, 4, 197, + 2, 3, 73, 69, 85, 8, 168, 2, 5, 72, 73, 69, 85, 75, 13, 5, 73, 89, 69, + 79, 75, 4, 245, 1, 4, 73, 69, 85, 77, 4, 213, 1, 4, 73, 69, 85, 78, 8, + 168, 1, 5, 72, 73, 69, 85, 80, 13, 4, 73, 69, 85, 80, 4, 121, 4, 73, 69, + 85, 76, 4, 93, 3, 73, 79, 83, 8, 56, 5, 72, 73, 69, 85, 84, 13, 5, 73, + 75, 69, 85, 84, 4, 11, 72, 5, 139, 160, 13, 32, 72, 148, 1, 2, 65, 76, + 30, 67, 74, 69, 82, 70, 112, 2, 76, 65, 22, 77, 38, 78, 32, 2, 82, 69, + 78, 83, 138, 2, 84, 50, 87, 254, 172, 11, 72, 239, 82, 79, 2, 237, 144, + 8, 2, 76, 73, 4, 32, 2, 79, 78, 179, 150, 11, 65, 2, 181, 251, 7, 4, 71, + 82, 65, 84, 6, 42, 78, 174, 179, 9, 65, 155, 194, 3, 73, 2, 169, 4, 5, + 84, 69, 82, 80, 82, 10, 58, 73, 204, 147, 12, 5, 69, 83, 84, 73, 86, 139, + 19, 79, 6, 208, 2, 3, 78, 65, 78, 130, 134, 13, 82, 3, 86, 2, 139, 230, + 12, 66, 4, 226, 198, 10, 69, 147, 136, 2, 79, 4, 138, 131, 12, 73, 195, + 1, 65, 8, 38, 83, 218, 139, 12, 80, 247, 111, 65, 4, 190, 214, 6, 79, + 183, 199, 6, 84, 18, 58, 69, 34, 79, 22, 80, 34, 84, 50, 85, 211, 141, + 12, 73, 4, 142, 199, 11, 86, 227, 84, 76, 2, 239, 251, 7, 67, 2, 11, 69, + 2, 227, 181, 4, 67, 4, 26, 85, 163, 234, 11, 79, 2, 219, 138, 13, 68, 4, + 26, 80, 247, 155, 13, 78, 2, 21, 3, 69, 82, 86, 2, 183, 144, 12, 73, 6, + 154, 169, 11, 72, 206, 162, 1, 69, 239, 48, 87, 4, 234, 224, 9, 79, 163, + 128, 2, 65, 4, 170, 164, 8, 32, 221, 166, 4, 2, 74, 69, 22, 42, 69, 46, + 70, 42, 78, 30, 83, 51, 84, 4, 216, 1, 3, 73, 71, 72, 131, 246, 4, 76, 4, + 160, 1, 2, 79, 85, 13, 2, 73, 70, 2, 133, 1, 3, 73, 78, 69, 4, 34, 73, + 73, 4, 69, 86, 69, 78, 2, 71, 88, 8, 34, 72, 46, 87, 207, 200, 12, 69, 2, + 11, 73, 2, 11, 82, 2, 175, 194, 11, 84, 4, 11, 69, 4, 190, 168, 11, 78, + 143, 115, 76, 22, 164, 1, 3, 73, 65, 76, 216, 1, 3, 89, 32, 80, 224, 206, + 6, 5, 72, 69, 78, 79, 80, 180, 229, 1, 6, 78, 69, 82, 83, 72, 73, 225, + 219, 3, 6, 32, 65, 76, 84, 69, 82, 12, 62, 32, 173, 124, 10, 76, 89, 45, + 82, 69, 67, 89, 67, 76, 69, 10, 38, 68, 41, 5, 76, 73, 78, 69, 32, 2, + 229, 174, 4, 5, 73, 70, 70, 69, 82, 8, 254, 205, 3, 68, 226, 45, 70, 20, + 4, 66, 65, 67, 75, 203, 153, 9, 85, 2, 223, 242, 3, 79, 10, 98, 69, 52, + 9, 73, 86, 69, 45, 80, 85, 76, 76, 45, 89, 9, 80, 79, 82, 84, 32, 67, 79, + 78, 84, 4, 176, 242, 9, 4, 78, 71, 69, 82, 147, 140, 2, 68, 4, 40, 4, 68, + 79, 87, 78, 1, 2, 85, 80, 2, 213, 176, 5, 6, 45, 79, 85, 84, 80, 85, 2, + 239, 253, 11, 82, 114, 192, 1, 12, 71, 76, 79, 84, 84, 65, 76, 32, 83, + 84, 79, 80, 62, 76, 156, 3, 3, 82, 73, 83, 36, 14, 77, 73, 68, 45, 76, 69, 86, 69, 76, 32, 84, 79, 78, 69, 57, 7, 83, 65, 78, 68, 72, 73, 32, 7, - 11, 32, 4, 206, 5, 70, 201, 238, 11, 4, 86, 65, 82, 73, 82, 72, 6, 69, - 84, 84, 69, 82, 32, 213, 2, 7, 79, 87, 45, 70, 65, 76, 76, 74, 206, 1, - 70, 166, 143, 7, 73, 2, 85, 214, 250, 1, 78, 242, 169, 3, 67, 2, 75, 2, - 80, 2, 84, 138, 69, 66, 2, 68, 2, 71, 2, 72, 2, 76, 2, 77, 2, 82, 2, 83, - 2, 86, 2, 90, 186, 2, 65, 2, 69, 3, 79, 20, 44, 5, 73, 78, 65, 76, 32, - 251, 250, 12, 65, 18, 222, 255, 10, 78, 154, 251, 1, 75, 2, 76, 2, 77, 2, - 80, 2, 84, 2, 87, 3, 89, 8, 157, 1, 5, 73, 78, 71, 32, 84, 7, 11, 32, 4, - 192, 1, 5, 76, 79, 78, 71, 32, 15, 70, 12, 66, 84, 73, 12, 71, 76, 79, - 84, 84, 65, 76, 32, 83, 84, 79, 80, 8, 21, 3, 79, 78, 69, 9, 11, 32, 6, - 32, 4, 76, 79, 78, 71, 27, 70, 5, 11, 32, 2, 11, 70, 2, 223, 195, 3, 73, - 2, 155, 176, 4, 82, 4, 250, 247, 12, 70, 3, 73, 80, 130, 1, 65, 122, 78, - 162, 1, 82, 162, 9, 83, 44, 3, 84, 82, 73, 130, 17, 68, 245, 136, 12, 9, - 79, 80, 76, 69, 32, 72, 85, 71, 71, 12, 50, 67, 50, 78, 190, 234, 6, 32, - 191, 139, 6, 82, 6, 182, 198, 11, 79, 202, 28, 69, 171, 147, 1, 72, 2, - 203, 225, 11, 85, 10, 118, 71, 20, 3, 83, 73, 86, 222, 178, 1, 84, 132, - 184, 4, 10, 32, 79, 86, 69, 82, 32, 83, 84, 65, 77, 167, 214, 4, 67, 2, - 151, 245, 11, 85, 2, 183, 183, 12, 69, 48, 186, 1, 32, 116, 10, 80, 69, - 78, 68, 73, 67, 85, 76, 65, 82, 42, 83, 210, 165, 7, 67, 252, 9, 10, 77, - 65, 78, 69, 78, 84, 32, 80, 65, 80, 201, 128, 2, 8, 70, 79, 82, 77, 73, - 78, 71, 32, 6, 34, 77, 30, 84, 227, 235, 11, 83, 2, 181, 241, 1, 2, 73, - 76, 2, 213, 137, 11, 8, 69, 78, 32, 84, 72, 79, 85, 83, 5, 129, 253, 8, - 5, 32, 87, 73, 84, 72, 32, 60, 2, 79, 78, 178, 75, 80, 221, 157, 11, 4, - 69, 86, 69, 82, 28, 38, 32, 149, 246, 9, 3, 65, 76, 32, 26, 216, 2, 10, - 68, 79, 73, 78, 71, 32, 67, 65, 82, 84, 32, 3, 73, 78, 32, 92, 5, 87, 73, - 84, 72, 32, 220, 213, 7, 4, 70, 82, 79, 87, 204, 13, 27, 82, 65, 73, 83, - 73, 78, 71, 32, 66, 79, 84, 72, 32, 72, 65, 78, 68, 83, 32, 73, 78, 32, - 67, 69, 76, 69, 66, 128, 185, 4, 5, 67, 76, 73, 77, 66, 213, 45, 11, 66, - 79, 87, 73, 78, 71, 32, 68, 69, 69, 80, 2, 11, 87, 2, 251, 183, 3, 72, 4, - 164, 156, 12, 6, 76, 79, 84, 85, 83, 32, 253, 64, 9, 83, 84, 69, 65, 77, - 89, 32, 82, 79, 12, 154, 1, 66, 240, 145, 2, 3, 80, 79, 85, 220, 160, 1, - 2, 67, 82, 196, 250, 5, 6, 70, 79, 76, 68, 69, 68, 221, 189, 2, 8, 72, - 69, 65, 68, 83, 67, 65, 82, 4, 36, 3, 76, 79, 78, 171, 228, 10, 65, 2, - 11, 68, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 219, 178, 12, 73, 4, 224, - 153, 9, 2, 69, 84, 195, 202, 2, 79, 2, 253, 137, 9, 2, 32, 68, 140, 2, + 11, 32, 4, 202, 5, 70, 213, 255, 11, 4, 86, 65, 82, 73, 82, 72, 6, 69, + 84, 84, 69, 82, 32, 209, 2, 7, 79, 87, 45, 70, 65, 76, 76, 74, 202, 1, + 70, 226, 252, 8, 73, 2, 85, 238, 27, 78, 198, 174, 3, 67, 2, 75, 2, 80, + 2, 84, 138, 69, 66, 2, 68, 2, 71, 2, 72, 2, 76, 2, 77, 2, 82, 2, 83, 2, + 86, 2, 90, 186, 2, 65, 2, 69, 3, 79, 20, 44, 5, 73, 78, 65, 76, 32, 163, + 142, 13, 65, 18, 158, 144, 11, 78, 130, 254, 1, 75, 2, 76, 2, 77, 2, 80, + 2, 84, 2, 87, 3, 89, 8, 157, 1, 5, 73, 78, 71, 32, 84, 7, 11, 32, 4, 192, + 1, 5, 76, 79, 78, 71, 32, 15, 70, 12, 66, 84, 73, 12, 71, 76, 79, 84, 84, + 65, 76, 32, 83, 84, 79, 80, 8, 21, 3, 79, 78, 69, 9, 11, 32, 6, 32, 4, + 76, 79, 78, 71, 27, 70, 5, 11, 32, 2, 11, 70, 2, 131, 201, 3, 73, 2, 171, + 181, 4, 82, 4, 162, 139, 13, 70, 3, 73, 80, 130, 1, 65, 122, 78, 162, 1, + 82, 162, 9, 83, 44, 3, 84, 82, 73, 130, 17, 68, 157, 156, 12, 9, 79, 80, + 76, 69, 32, 72, 85, 71, 71, 12, 50, 67, 50, 78, 138, 239, 6, 32, 155, + 154, 6, 82, 6, 202, 215, 11, 79, 194, 28, 69, 199, 149, 1, 72, 2, 227, + 128, 12, 85, 10, 118, 71, 20, 3, 83, 73, 86, 222, 179, 1, 84, 156, 188, + 4, 10, 32, 79, 86, 69, 82, 32, 83, 84, 65, 77, 187, 184, 5, 67, 2, 191, + 136, 12, 85, 2, 223, 202, 12, 69, 48, 186, 1, 32, 116, 10, 80, 69, 78, + 68, 73, 67, 85, 76, 65, 82, 42, 83, 130, 176, 7, 67, 252, 9, 10, 77, 65, + 78, 69, 78, 84, 32, 80, 65, 80, 237, 133, 2, 8, 70, 79, 82, 77, 73, 78, + 71, 32, 6, 34, 77, 30, 84, 139, 255, 11, 83, 2, 129, 242, 1, 2, 73, 76, + 2, 149, 154, 11, 8, 69, 78, 32, 84, 72, 79, 85, 83, 5, 213, 139, 9, 5, + 32, 87, 73, 84, 72, 32, 60, 2, 79, 78, 154, 75, 80, 157, 177, 11, 4, 69, + 86, 69, 82, 28, 38, 32, 237, 133, 10, 3, 65, 76, 32, 26, 216, 2, 10, 68, + 79, 73, 78, 71, 32, 67, 65, 82, 84, 32, 3, 73, 78, 32, 92, 5, 87, 73, 84, + 72, 32, 184, 224, 7, 4, 70, 82, 79, 87, 204, 13, 27, 82, 65, 73, 83, 73, + 78, 71, 32, 66, 79, 84, 72, 32, 72, 65, 78, 68, 83, 32, 73, 78, 32, 67, + 69, 76, 69, 66, 204, 193, 4, 5, 67, 76, 73, 77, 66, 213, 45, 11, 66, 79, + 87, 73, 78, 71, 32, 68, 69, 69, 80, 2, 11, 87, 2, 159, 189, 3, 72, 4, + 204, 175, 12, 6, 76, 79, 84, 85, 83, 32, 253, 64, 9, 83, 84, 69, 65, 77, + 89, 32, 82, 79, 12, 154, 1, 66, 180, 146, 2, 3, 80, 79, 85, 188, 165, 1, + 2, 67, 82, 244, 132, 6, 6, 70, 79, 76, 68, 69, 68, 177, 193, 2, 8, 72, + 69, 65, 68, 83, 67, 65, 82, 4, 36, 3, 76, 79, 78, 235, 244, 10, 65, 2, + 11, 68, 2, 11, 32, 2, 11, 72, 2, 11, 65, 2, 131, 198, 12, 73, 4, 180, + 169, 9, 2, 69, 84, 151, 206, 2, 79, 2, 209, 153, 9, 2, 32, 68, 140, 2, 66, 65, 184, 19, 9, 73, 76, 73, 80, 80, 73, 78, 69, 32, 47, 79, 204, 1, 108, 6, 71, 83, 45, 80, 65, 32, 189, 7, 16, 73, 83, 84, 79, 83, 32, 68, 73, 83, 67, 32, 83, 73, 71, 78, 32, 112, 100, 7, 76, 69, 84, 84, 69, 82, 32, 248, 4, 5, 77, 65, 82, 75, 32, 30, 83, 33, 4, 68, 79, 85, 66, 96, - 138, 2, 65, 138, 1, 67, 50, 68, 42, 83, 64, 5, 86, 79, 73, 67, 69, 222, - 242, 8, 71, 246, 168, 3, 78, 82, 84, 46, 75, 2, 80, 2, 90, 162, 7, 69, + 138, 2, 65, 138, 1, 67, 50, 68, 42, 83, 64, 5, 86, 79, 73, 67, 69, 178, + 129, 9, 71, 202, 173, 3, 78, 82, 84, 46, 75, 2, 80, 2, 90, 162, 7, 69, 234, 61, 66, 2, 70, 2, 72, 2, 74, 2, 76, 2, 77, 2, 81, 2, 82, 2, 87, 2, 88, 2, 89, 186, 2, 73, 2, 79, 3, 85, 7, 80, 8, 76, 84, 69, 82, 78, 65, - 84, 69, 21, 8, 83, 80, 73, 82, 65, 84, 69, 68, 2, 251, 226, 12, 32, 2, - 11, 32, 2, 255, 226, 12, 70, 6, 26, 65, 211, 226, 12, 72, 5, 223, 208, 8, - 78, 6, 186, 226, 12, 68, 2, 90, 187, 2, 65, 6, 168, 164, 8, 4, 77, 65, - 76, 76, 234, 189, 4, 72, 187, 2, 65, 4, 34, 68, 21, 4, 76, 69, 83, 83, 2, - 167, 233, 10, 32, 2, 215, 171, 9, 32, 4, 206, 156, 12, 68, 59, 83, 10, - 28, 3, 73, 78, 71, 31, 85, 2, 189, 150, 12, 2, 76, 69, 8, 58, 66, 173, - 151, 12, 8, 80, 69, 82, 70, 73, 88, 69, 68, 6, 133, 179, 8, 13, 74, 79, + 84, 69, 21, 8, 83, 80, 73, 82, 65, 84, 69, 68, 2, 163, 246, 12, 32, 2, + 11, 32, 2, 167, 246, 12, 70, 6, 26, 65, 251, 245, 12, 72, 5, 231, 218, 8, + 78, 6, 226, 245, 12, 68, 2, 90, 187, 2, 65, 6, 244, 174, 8, 4, 77, 65, + 76, 76, 198, 198, 4, 72, 187, 2, 65, 4, 34, 68, 21, 4, 76, 69, 83, 83, 2, + 231, 249, 10, 32, 2, 171, 187, 9, 32, 4, 246, 175, 12, 68, 59, 83, 10, + 28, 3, 73, 78, 71, 31, 85, 2, 229, 169, 12, 2, 76, 69, 8, 58, 66, 213, + 170, 12, 8, 80, 69, 82, 70, 73, 88, 69, 68, 6, 209, 189, 8, 13, 74, 79, 73, 78, 69, 68, 32, 76, 69, 84, 84, 69, 82, 92, 238, 1, 66, 146, 1, 67, 172, 2, 2, 68, 79, 38, 71, 66, 72, 64, 2, 76, 73, 32, 2, 77, 65, 70, 80, - 162, 1, 82, 38, 83, 150, 1, 84, 70, 87, 160, 223, 5, 2, 70, 76, 252, 232, - 1, 2, 79, 88, 196, 234, 3, 2, 69, 65, 250, 9, 65, 135, 1, 86, 10, 52, 2, - 69, 69, 22, 79, 133, 41, 4, 85, 76, 76, 83, 5, 247, 171, 7, 72, 4, 32, 2, - 79, 77, 135, 223, 12, 87, 2, 11, 69, 2, 163, 134, 12, 82, 16, 34, 65, 86, - 72, 22, 76, 23, 79, 6, 234, 218, 5, 80, 148, 230, 3, 8, 82, 80, 69, 78, - 84, 82, 89, 32, 195, 157, 3, 84, 2, 227, 191, 2, 73, 2, 243, 161, 11, 85, - 6, 26, 76, 33, 2, 77, 66, 2, 11, 85, 2, 187, 141, 12, 77, 5, 37, 7, 73, - 78, 73, 78, 71, 32, 79, 2, 209, 239, 9, 5, 66, 76, 73, 81, 85, 4, 154, - 179, 11, 76, 203, 146, 1, 86, 4, 42, 82, 201, 252, 10, 4, 65, 85, 78, 84, - 2, 239, 163, 11, 65, 6, 42, 69, 146, 209, 7, 79, 143, 250, 2, 73, 2, 131, - 166, 12, 76, 4, 202, 201, 12, 76, 203, 17, 68, 4, 34, 78, 161, 236, 9, 2, - 84, 84, 2, 11, 65, 2, 255, 156, 9, 67, 8, 52, 2, 69, 68, 50, 76, 153, - 166, 7, 3, 65, 80, 89, 2, 25, 4, 69, 83, 84, 82, 2, 219, 143, 11, 73, 4, - 176, 182, 4, 2, 85, 77, 177, 226, 2, 3, 65, 78, 69, 4, 166, 202, 10, 79, - 147, 254, 1, 65, 12, 108, 2, 72, 73, 222, 217, 11, 65, 158, 45, 76, 128, - 19, 3, 84, 82, 65, 177, 39, 7, 77, 65, 76, 76, 32, 65, 88, 4, 242, 185, - 2, 69, 139, 158, 10, 80, 6, 192, 180, 4, 5, 65, 84, 84, 79, 79, 202, 222, - 7, 73, 207, 36, 85, 4, 190, 225, 7, 79, 209, 231, 3, 5, 65, 86, 89, 32, - 66, 4, 174, 244, 1, 83, 25, 4, 68, 79, 85, 66, 60, 56, 8, 69, 78, 73, 67, - 73, 65, 78, 32, 251, 207, 10, 76, 58, 92, 7, 76, 69, 84, 84, 69, 82, 32, - 160, 3, 7, 78, 85, 77, 66, 69, 82, 32, 203, 150, 6, 87, 44, 234, 1, 65, - 34, 68, 22, 72, 22, 81, 22, 83, 58, 84, 158, 130, 2, 87, 138, 229, 5, 90, - 154, 180, 1, 89, 184, 206, 1, 2, 82, 79, 236, 94, 3, 71, 65, 77, 134, 8, + 162, 1, 82, 38, 83, 150, 1, 84, 70, 87, 200, 228, 5, 2, 70, 76, 176, 238, + 1, 2, 79, 88, 252, 240, 3, 2, 69, 65, 138, 10, 65, 135, 1, 86, 10, 52, 2, + 69, 69, 22, 79, 145, 41, 4, 85, 76, 76, 83, 5, 147, 182, 7, 72, 4, 32, 2, + 79, 77, 175, 242, 12, 87, 2, 11, 69, 2, 203, 153, 12, 82, 16, 34, 65, 86, + 72, 22, 76, 23, 79, 6, 130, 224, 5, 80, 208, 240, 3, 8, 82, 80, 69, 78, + 84, 82, 89, 32, 151, 161, 3, 84, 2, 199, 193, 2, 73, 2, 135, 179, 11, 85, + 6, 26, 76, 33, 2, 77, 66, 2, 11, 85, 2, 227, 160, 12, 77, 5, 37, 7, 73, + 78, 73, 78, 71, 32, 79, 2, 169, 255, 9, 5, 66, 76, 73, 81, 85, 4, 174, + 196, 11, 76, 223, 148, 1, 86, 4, 42, 82, 137, 141, 11, 4, 65, 85, 78, 84, + 2, 131, 181, 11, 65, 6, 42, 69, 238, 219, 7, 79, 243, 255, 2, 73, 2, 171, + 185, 12, 76, 4, 242, 220, 12, 76, 203, 17, 68, 4, 34, 78, 249, 251, 9, 2, + 84, 84, 2, 11, 65, 2, 211, 172, 9, 67, 8, 52, 2, 69, 68, 50, 76, 181, + 176, 7, 3, 65, 80, 89, 2, 25, 4, 69, 83, 84, 82, 2, 231, 160, 11, 73, 4, + 192, 187, 4, 2, 85, 77, 209, 231, 2, 3, 65, 78, 69, 4, 230, 218, 10, 79, + 251, 128, 2, 65, 12, 108, 2, 72, 73, 134, 237, 11, 65, 158, 45, 76, 128, + 19, 3, 84, 82, 65, 177, 39, 7, 77, 65, 76, 76, 32, 65, 88, 4, 214, 187, + 2, 69, 207, 175, 10, 80, 6, 208, 185, 4, 5, 65, 84, 84, 79, 79, 226, 236, + 7, 73, 207, 36, 85, 4, 146, 236, 7, 79, 137, 238, 3, 5, 65, 86, 89, 32, + 66, 4, 250, 244, 1, 83, 25, 4, 68, 79, 85, 66, 60, 56, 8, 69, 78, 73, 67, + 73, 65, 78, 32, 187, 224, 10, 76, 58, 92, 7, 76, 69, 84, 84, 69, 82, 32, + 160, 3, 7, 78, 85, 77, 66, 69, 82, 32, 231, 155, 6, 87, 44, 234, 1, 65, + 34, 68, 22, 72, 22, 81, 22, 83, 58, 84, 226, 130, 2, 87, 154, 239, 5, 90, + 154, 185, 1, 89, 164, 207, 1, 2, 82, 79, 184, 95, 3, 71, 65, 77, 162, 10, 75, 130, 1, 78, 144, 58, 3, 76, 65, 77, 138, 17, 66, 198, 30, 80, 171, 4, - 77, 4, 130, 210, 11, 76, 199, 49, 73, 2, 163, 187, 3, 69, 4, 147, 243, 6, - 69, 2, 187, 209, 11, 79, 6, 190, 194, 10, 65, 186, 144, 1, 72, 189, 124, - 2, 69, 77, 4, 170, 173, 12, 65, 191, 8, 69, 12, 238, 49, 84, 135, 166, 4, - 79, 40, 104, 2, 67, 75, 66, 71, 62, 76, 90, 78, 178, 1, 83, 32, 7, 84, - 67, 72, 70, 79, 82, 75, 199, 205, 12, 69, 5, 17, 2, 85, 80, 2, 21, 3, 32, - 84, 82, 2, 203, 160, 11, 85, 7, 11, 32, 4, 26, 78, 251, 146, 12, 70, 2, - 243, 198, 11, 79, 6, 52, 4, 69, 32, 79, 70, 246, 91, 67, 195, 243, 11, - 76, 2, 11, 32, 2, 255, 135, 10, 80, 14, 68, 2, 67, 72, 46, 69, 178, 164, - 4, 87, 182, 162, 7, 75, 243, 98, 65, 4, 160, 155, 3, 2, 69, 68, 255, 164, - 8, 73, 4, 28, 2, 32, 68, 255, 72, 65, 2, 221, 129, 5, 2, 69, 67, 4, 234, - 185, 11, 67, 139, 1, 84, 5, 153, 252, 9, 10, 32, 87, 73, 84, 72, 32, 84, - 69, 69, 32, 218, 1, 38, 65, 218, 9, 85, 135, 195, 12, 68, 176, 1, 78, 67, + 77, 4, 170, 229, 11, 76, 199, 49, 73, 2, 199, 192, 3, 69, 4, 195, 253, 6, + 69, 2, 227, 228, 11, 79, 6, 254, 210, 10, 65, 162, 147, 1, 72, 189, 124, + 2, 69, 77, 4, 210, 192, 12, 65, 191, 8, 69, 12, 202, 50, 84, 203, 170, 4, + 79, 40, 104, 2, 67, 75, 66, 71, 62, 76, 90, 78, 178, 1, 83, 28, 7, 84, + 67, 72, 70, 79, 82, 75, 243, 224, 12, 69, 5, 17, 2, 85, 80, 2, 21, 3, 32, + 84, 82, 2, 223, 177, 11, 85, 7, 11, 32, 4, 26, 78, 163, 166, 12, 70, 2, + 131, 216, 11, 79, 6, 52, 4, 69, 32, 79, 70, 246, 92, 67, 235, 133, 12, + 76, 2, 11, 32, 2, 215, 151, 10, 80, 14, 68, 2, 67, 72, 46, 69, 194, 169, + 4, 87, 206, 176, 7, 75, 243, 98, 65, 4, 196, 160, 3, 2, 69, 68, 231, 176, + 8, 73, 4, 28, 2, 32, 68, 239, 73, 65, 2, 253, 134, 5, 2, 69, 67, 4, 134, + 203, 11, 67, 123, 84, 5, 245, 139, 10, 10, 32, 87, 73, 84, 72, 32, 84, + 69, 69, 32, 218, 1, 38, 65, 230, 9, 85, 167, 214, 12, 68, 176, 1, 78, 67, 128, 1, 12, 78, 67, 75, 32, 67, 79, 78, 83, 84, 65, 78, 84, 83, 89, 6, - 44, 5, 69, 32, 79, 70, 32, 239, 180, 11, 65, 4, 56, 6, 73, 78, 84, 69, - 82, 69, 129, 246, 11, 2, 87, 79, 2, 199, 136, 7, 83, 5, 45, 9, 32, 79, - 86, 69, 82, 32, 84, 87, 79, 2, 11, 32, 2, 243, 182, 12, 80, 166, 1, 96, - 9, 73, 78, 71, 32, 67, 65, 82, 68, 32, 141, 166, 4, 9, 71, 82, 79, 85, - 78, 68, 32, 83, 76, 164, 1, 182, 1, 66, 44, 3, 82, 69, 68, 0, 5, 87, 72, - 73, 84, 69, 42, 70, 74, 75, 38, 69, 34, 83, 36, 3, 81, 85, 69, 14, 84, - 92, 2, 65, 67, 0, 3, 78, 73, 78, 13, 4, 74, 65, 67, 75, 4, 40, 4, 76, 65, - 67, 75, 251, 151, 11, 65, 2, 17, 2, 32, 74, 2, 159, 237, 1, 79, 18, 30, - 79, 249, 1, 2, 73, 86, 10, 128, 2, 2, 85, 82, 235, 187, 11, 79, 16, 34, - 78, 185, 1, 3, 73, 78, 71, 8, 181, 1, 4, 73, 71, 72, 84, 16, 32, 2, 69, - 86, 117, 2, 73, 88, 8, 91, 69, 66, 78, 69, 12, 3, 72, 82, 69, 12, 2, 87, - 79, 161, 1, 5, 82, 85, 77, 80, 45, 8, 23, 78, 8, 11, 69, 8, 25, 4, 32, - 79, 70, 32, 8, 88, 3, 67, 76, 85, 20, 3, 83, 80, 65, 174, 130, 9, 72, - 137, 3, 5, 68, 73, 65, 77, 79, 2, 199, 134, 12, 66, 2, 151, 176, 11, 68, - 42, 90, 50, 238, 130, 10, 49, 182, 192, 2, 51, 2, 52, 2, 53, 2, 54, 2, - 55, 2, 56, 3, 57, 7, 158, 195, 12, 48, 3, 49, 41, 46, 83, 160, 4, 2, 84, - 79, 227, 240, 8, 78, 26, 52, 5, 32, 83, 73, 71, 78, 193, 147, 9, 2, 45, - 77, 25, 11, 32, 22, 64, 3, 73, 78, 32, 124, 5, 87, 73, 84, 72, 32, 207, - 250, 3, 65, 6, 34, 76, 22, 82, 183, 156, 11, 84, 2, 41, 2, 69, 70, 2, 21, - 3, 73, 71, 72, 2, 229, 238, 10, 6, 84, 32, 72, 65, 76, 70, 14, 162, 1, - 68, 34, 83, 200, 160, 10, 4, 84, 73, 76, 68, 208, 121, 5, 66, 76, 65, 67, - 75, 181, 36, 16, 67, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, 67, 67, - 69, 78, 2, 11, 79, 2, 219, 145, 10, 84, 4, 54, 77, 161, 176, 5, 7, 85, - 66, 83, 67, 82, 73, 80, 2, 249, 154, 9, 3, 65, 76, 76, 11, 33, 6, 32, 70, - 79, 82, 77, 32, 8, 234, 205, 10, 70, 71, 84, 2, 253, 132, 12, 8, 32, 84, - 82, 65, 78, 83, 73, 83, 58, 232, 1, 5, 76, 73, 67, 69, 32, 122, 80, 136, + 44, 5, 69, 32, 79, 70, 32, 151, 198, 11, 65, 4, 56, 6, 73, 78, 84, 69, + 82, 69, 173, 137, 12, 2, 87, 79, 2, 251, 146, 7, 83, 5, 45, 9, 32, 79, + 86, 69, 82, 32, 84, 87, 79, 2, 11, 32, 2, 159, 202, 12, 80, 166, 1, 80, + 7, 71, 82, 79, 85, 78, 68, 32, 29, 9, 73, 78, 71, 32, 67, 65, 82, 68, 32, + 2, 173, 171, 4, 2, 83, 76, 164, 1, 182, 1, 66, 44, 3, 82, 69, 68, 0, 5, + 87, 72, 73, 84, 69, 42, 70, 74, 75, 38, 69, 34, 83, 36, 3, 81, 85, 69, + 14, 84, 92, 2, 65, 67, 0, 3, 78, 73, 78, 13, 4, 74, 65, 67, 75, 4, 40, 4, + 76, 65, 67, 75, 135, 169, 11, 65, 2, 17, 2, 32, 74, 2, 219, 237, 1, 79, + 18, 30, 79, 249, 1, 2, 73, 86, 10, 128, 2, 2, 85, 82, 239, 204, 11, 79, + 16, 34, 78, 185, 1, 3, 73, 78, 71, 8, 181, 1, 4, 73, 71, 72, 84, 16, 32, + 2, 69, 86, 117, 2, 73, 88, 8, 91, 69, 66, 78, 69, 12, 3, 72, 82, 69, 12, + 2, 87, 79, 161, 1, 5, 82, 85, 77, 80, 45, 8, 23, 78, 8, 11, 69, 8, 25, 4, + 32, 79, 70, 32, 8, 88, 3, 67, 76, 85, 20, 3, 83, 80, 65, 250, 145, 9, 72, + 137, 3, 5, 68, 73, 65, 77, 79, 2, 231, 153, 12, 66, 2, 171, 193, 11, 68, + 42, 90, 50, 190, 146, 10, 49, 134, 196, 2, 51, 2, 52, 2, 53, 2, 54, 2, + 55, 2, 56, 3, 57, 7, 190, 214, 12, 48, 3, 49, 41, 46, 83, 160, 4, 2, 84, + 79, 175, 128, 9, 78, 26, 52, 5, 32, 83, 73, 71, 78, 141, 163, 9, 2, 45, + 77, 25, 11, 32, 22, 64, 3, 73, 78, 32, 124, 5, 87, 73, 84, 72, 32, 215, + 255, 3, 65, 6, 34, 76, 22, 82, 195, 173, 11, 84, 2, 41, 2, 69, 70, 2, 21, + 3, 73, 71, 72, 2, 233, 255, 10, 6, 84, 32, 72, 65, 76, 70, 14, 162, 1, + 68, 34, 83, 140, 176, 10, 4, 84, 73, 76, 68, 152, 123, 5, 66, 76, 65, 67, + 75, 201, 38, 16, 67, 73, 82, 67, 85, 77, 70, 76, 69, 88, 32, 65, 67, 67, + 69, 78, 2, 11, 79, 2, 167, 161, 10, 84, 4, 54, 77, 173, 181, 5, 7, 85, + 66, 83, 67, 82, 73, 80, 2, 197, 170, 9, 3, 65, 76, 76, 11, 33, 6, 32, 70, + 79, 82, 77, 32, 8, 162, 222, 10, 70, 71, 84, 2, 157, 152, 12, 8, 32, 84, + 82, 65, 78, 83, 73, 83, 58, 232, 1, 5, 76, 73, 67, 69, 32, 122, 80, 140, 1, 11, 82, 84, 65, 66, 76, 69, 32, 83, 84, 69, 82, 22, 83, 158, 1, 84, - 146, 1, 85, 196, 1, 4, 87, 69, 82, 32, 194, 132, 7, 79, 145, 248, 4, 11, - 67, 75, 69, 84, 32, 67, 65, 76, 67, 85, 76, 6, 52, 3, 67, 65, 82, 209, - 248, 3, 4, 79, 70, 70, 73, 5, 149, 162, 10, 11, 83, 32, 82, 69, 86, 79, + 146, 1, 85, 196, 1, 4, 87, 69, 82, 32, 210, 142, 7, 79, 157, 129, 5, 11, + 67, 75, 69, 84, 32, 67, 65, 76, 67, 85, 76, 6, 52, 3, 67, 65, 82, 217, + 253, 3, 4, 79, 70, 70, 73, 5, 217, 177, 10, 11, 83, 32, 82, 69, 86, 79, 76, 86, 73, 78, 71, 6, 76, 13, 32, 68, 73, 82, 69, 67, 84, 73, 79, 78, - 65, 76, 32, 219, 214, 1, 67, 4, 166, 127, 73, 205, 180, 6, 6, 70, 79, 82, - 77, 65, 84, 2, 223, 154, 12, 69, 12, 40, 2, 69, 73, 22, 84, 235, 135, 5, - 73, 2, 167, 233, 11, 68, 8, 36, 3, 65, 76, 32, 171, 172, 11, 66, 6, 162, - 213, 1, 72, 157, 197, 6, 4, 77, 65, 82, 75, 8, 66, 65, 250, 134, 2, 32, - 141, 167, 9, 6, 84, 69, 68, 32, 80, 76, 4, 26, 66, 211, 152, 12, 84, 2, - 29, 5, 76, 69, 32, 87, 65, 2, 143, 48, 84, 12, 108, 4, 76, 84, 82, 89, - 28, 7, 82, 73, 78, 71, 32, 76, 73, 28, 2, 84, 73, 150, 205, 10, 78, 203, - 231, 1, 67, 2, 185, 240, 11, 2, 32, 76, 2, 193, 174, 5, 2, 81, 85, 4, - 225, 206, 10, 2, 78, 71, 8, 24, 2, 79, 78, 47, 83, 4, 136, 162, 11, 4, - 45, 79, 70, 70, 15, 32, 4, 212, 136, 9, 3, 76, 69, 69, 175, 153, 2, 89, - 140, 1, 74, 69, 162, 10, 73, 234, 2, 79, 157, 206, 9, 6, 65, 89, 69, 82, - 32, 66, 102, 132, 1, 6, 71, 78, 65, 78, 84, 32, 66, 83, 224, 186, 5, 4, - 67, 69, 68, 69, 204, 202, 1, 5, 86, 73, 79, 85, 83, 177, 32, 2, 84, 90, - 6, 42, 87, 130, 179, 8, 80, 215, 181, 2, 77, 2, 255, 188, 7, 79, 70, 176, + 65, 76, 32, 159, 215, 1, 67, 4, 158, 128, 1, 73, 169, 190, 6, 6, 70, 79, + 82, 77, 65, 84, 2, 251, 173, 12, 69, 12, 40, 2, 69, 73, 22, 84, 243, 140, + 5, 73, 2, 195, 252, 11, 68, 8, 36, 3, 65, 76, 32, 171, 189, 11, 66, 6, + 226, 213, 1, 72, 213, 206, 6, 4, 77, 65, 82, 75, 8, 66, 65, 238, 135, 2, + 32, 153, 183, 9, 6, 84, 69, 68, 32, 80, 76, 4, 26, 66, 239, 171, 12, 84, + 2, 29, 5, 76, 69, 32, 87, 65, 2, 243, 48, 84, 12, 108, 4, 76, 84, 82, 89, + 28, 7, 82, 73, 78, 71, 32, 76, 73, 28, 2, 84, 73, 202, 221, 10, 78, 179, + 234, 1, 67, 2, 213, 131, 12, 2, 32, 76, 2, 205, 179, 5, 2, 81, 85, 4, + 149, 223, 10, 2, 78, 71, 8, 24, 2, 79, 78, 47, 83, 4, 136, 179, 11, 4, + 45, 79, 70, 70, 15, 32, 4, 156, 152, 9, 3, 76, 69, 69, 231, 154, 2, 89, + 140, 1, 74, 69, 162, 10, 73, 230, 2, 79, 237, 221, 9, 6, 65, 89, 69, 82, + 32, 66, 102, 132, 1, 6, 71, 78, 65, 78, 84, 32, 66, 83, 252, 191, 5, 4, + 67, 69, 68, 69, 192, 207, 1, 5, 86, 73, 79, 85, 83, 241, 32, 2, 84, 90, + 6, 42, 87, 202, 193, 8, 80, 143, 184, 2, 77, 2, 199, 199, 7, 79, 70, 176, 1, 27, 69, 78, 84, 65, 84, 73, 79, 78, 32, 70, 79, 82, 77, 32, 70, 79, - 82, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 221, 200, 8, 10, 67, 82, 73, + 82, 32, 86, 69, 82, 84, 73, 67, 65, 76, 32, 129, 216, 8, 10, 67, 82, 73, 80, 84, 73, 79, 78, 32, 84, 68, 198, 1, 67, 22, 69, 46, 72, 50, 73, 94, 76, 188, 1, 6, 82, 73, 71, 72, 84, 32, 192, 2, 6, 87, 65, 86, 89, 32, 76, - 174, 151, 4, 83, 184, 207, 4, 9, 84, 87, 79, 32, 68, 79, 84, 32, 76, 135, - 114, 81, 4, 167, 185, 2, 79, 6, 222, 153, 6, 88, 174, 216, 2, 77, 3, 78, - 2, 229, 138, 9, 7, 79, 82, 73, 90, 79, 78, 84, 4, 49, 10, 68, 69, 79, 71, - 82, 65, 80, 72, 73, 67, 4, 11, 32, 4, 142, 220, 9, 67, 35, 70, 22, 40, 4, - 69, 70, 84, 32, 219, 197, 10, 79, 20, 112, 6, 87, 72, 73, 84, 69, 32, - 142, 1, 66, 42, 68, 198, 99, 67, 166, 7, 65, 138, 190, 7, 80, 238, 7, 83, - 39, 84, 4, 162, 2, 67, 139, 99, 76, 22, 110, 66, 42, 68, 36, 6, 87, 72, - 73, 84, 69, 32, 162, 99, 67, 166, 7, 65, 138, 190, 7, 80, 238, 7, 83, 39, - 84, 2, 157, 100, 6, 76, 65, 67, 75, 32, 76, 2, 209, 106, 5, 79, 85, 66, - 76, 69, 6, 74, 67, 17, 14, 76, 69, 78, 84, 73, 67, 85, 76, 65, 82, 32, - 66, 82, 65, 2, 239, 98, 79, 4, 218, 215, 11, 67, 177, 29, 2, 75, 67, 2, - 135, 194, 10, 79, 22, 46, 78, 156, 1, 2, 86, 65, 203, 145, 12, 77, 10, - 34, 84, 225, 200, 6, 2, 67, 69, 6, 26, 32, 53, 2, 69, 82, 2, 21, 3, 83, - 67, 82, 2, 153, 177, 10, 2, 69, 69, 5, 17, 2, 32, 73, 2, 195, 216, 11, - 67, 10, 60, 5, 67, 89, 32, 77, 69, 29, 6, 84, 69, 32, 85, 83, 69, 2, 133, - 161, 7, 2, 83, 83, 8, 26, 32, 159, 166, 8, 45, 4, 226, 239, 5, 84, 151, - 158, 5, 79, 14, 130, 1, 74, 30, 80, 236, 33, 5, 72, 73, 66, 73, 84, 200, - 230, 8, 6, 66, 73, 78, 71, 32, 67, 169, 225, 1, 5, 83, 69, 82, 80, 73, 2, - 165, 162, 5, 2, 69, 67, 6, 64, 6, 79, 82, 84, 73, 79, 78, 205, 139, 11, - 4, 69, 82, 84, 89, 5, 247, 209, 5, 65, 58, 172, 1, 15, 70, 79, 85, 82, - 32, 68, 79, 84, 83, 32, 87, 73, 84, 72, 32, 32, 7, 76, 69, 84, 84, 69, - 82, 32, 230, 2, 78, 134, 32, 83, 1, 8, 84, 85, 82, 78, 69, 68, 32, 83, 4, - 190, 226, 10, 67, 227, 112, 68, 36, 166, 1, 65, 22, 68, 34, 76, 22, 77, - 50, 87, 186, 165, 4, 71, 90, 90, 34, 83, 66, 89, 158, 208, 1, 72, 234, 5, - 75, 198, 75, 66, 178, 216, 4, 78, 134, 2, 84, 219, 103, 80, 2, 155, 166, - 4, 76, 2, 11, 65, 2, 143, 201, 6, 76, 2, 183, 166, 4, 65, 2, 25, 4, 69, - 77, 45, 81, 2, 147, 253, 5, 79, 2, 37, 7, 65, 87, 45, 65, 89, 73, 78, 2, - 173, 205, 1, 2, 45, 82, 14, 33, 6, 85, 77, 66, 69, 82, 32, 14, 42, 84, - 134, 166, 4, 79, 131, 231, 2, 70, 8, 42, 87, 150, 176, 10, 72, 231, 159, - 1, 69, 4, 210, 178, 10, 69, 135, 237, 1, 79, 18, 252, 1, 4, 78, 67, 84, - 85, 94, 82, 56, 19, 84, 32, 76, 73, 84, 84, 69, 82, 32, 73, 78, 32, 73, - 84, 83, 32, 80, 76, 65, 214, 151, 1, 83, 140, 42, 20, 66, 76, 73, 67, 32, - 65, 68, 68, 82, 69, 83, 83, 32, 76, 79, 85, 68, 83, 80, 69, 194, 218, 10, - 49, 3, 50, 4, 136, 206, 10, 9, 83, 32, 69, 76, 69, 86, 65, 84, 85, 229, - 144, 1, 5, 65, 84, 73, 79, 78, 4, 32, 2, 80, 76, 227, 133, 12, 83, 2, - 247, 148, 11, 69, 2, 11, 67, 2, 235, 136, 11, 69, 40, 98, 65, 180, 6, 6, - 69, 83, 84, 73, 79, 78, 162, 133, 6, 79, 165, 131, 5, 5, 73, 78, 67, 85, - 78, 30, 104, 2, 68, 82, 240, 4, 9, 84, 69, 82, 78, 73, 79, 78, 32, 73, - 48, 4, 82, 84, 69, 82, 219, 243, 10, 79, 24, 56, 4, 65, 78, 84, 32, 157, - 4, 5, 85, 80, 76, 69, 32, 20, 44, 6, 85, 80, 80, 69, 82, 32, 131, 2, 76, - 16, 56, 4, 76, 69, 70, 84, 249, 1, 5, 82, 73, 71, 72, 84, 11, 29, 5, 32, - 65, 78, 68, 32, 8, 108, 6, 76, 79, 87, 69, 82, 32, 53, 17, 85, 80, 80, - 69, 82, 32, 82, 73, 71, 72, 84, 32, 65, 78, 68, 32, 76, 4, 192, 1, 5, 76, - 69, 70, 84, 32, 231, 235, 11, 82, 4, 11, 79, 4, 11, 87, 4, 157, 236, 11, - 2, 69, 82, 7, 65, 14, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 76, 69, - 70, 4, 11, 84, 5, 11, 32, 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 76, 2, 17, - 2, 79, 87, 2, 237, 196, 10, 2, 69, 82, 4, 22, 73, 223, 37, 80, 2, 205, - 213, 8, 7, 78, 84, 69, 71, 82, 65, 76, 2, 17, 2, 32, 78, 2, 131, 206, 10, - 79, 6, 34, 32, 233, 192, 5, 2, 69, 68, 4, 250, 134, 5, 69, 159, 204, 6, - 77, 220, 9, 114, 65, 142, 8, 69, 204, 27, 6, 72, 73, 78, 79, 67, 69, 34, - 73, 174, 88, 76, 46, 79, 198, 19, 85, 151, 131, 11, 83, 62, 110, 67, 104, - 2, 68, 73, 130, 1, 73, 162, 5, 84, 152, 237, 7, 4, 66, 66, 73, 84, 178, - 229, 3, 90, 235, 56, 77, 6, 40, 4, 73, 78, 71, 32, 187, 235, 8, 67, 4, - 164, 192, 10, 7, 77, 79, 84, 79, 82, 67, 89, 175, 48, 67, 8, 66, 79, 205, - 165, 8, 10, 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 7, 232, 174, 6, 4, - 65, 67, 84, 73, 129, 175, 4, 2, 32, 66, 36, 60, 5, 76, 87, 65, 89, 32, - 46, 78, 21, 4, 83, 69, 68, 32, 4, 140, 245, 8, 2, 84, 82, 203, 249, 1, - 67, 5, 191, 245, 10, 66, 28, 152, 1, 3, 68, 79, 84, 34, 73, 48, 4, 72, - 65, 78, 68, 182, 1, 77, 38, 83, 166, 167, 7, 70, 230, 146, 2, 67, 237, - 195, 1, 7, 66, 65, 67, 75, 32, 79, 70, 5, 29, 5, 84, 69, 68, 32, 73, 2, - 193, 40, 8, 78, 84, 69, 82, 80, 79, 76, 65, 7, 33, 6, 32, 87, 73, 84, 72, - 32, 4, 198, 27, 70, 209, 190, 2, 28, 80, 65, 82, 84, 32, 66, 69, 84, 87, - 69, 69, 78, 32, 77, 73, 68, 68, 76, 69, 32, 65, 78, 68, 32, 82, 73, 78, - 71, 6, 174, 133, 11, 67, 2, 68, 3, 82, 4, 60, 9, 77, 65, 76, 76, 32, 76, - 69, 70, 84, 255, 234, 10, 81, 2, 129, 145, 8, 2, 32, 83, 5, 187, 236, 11, - 73, 250, 1, 226, 1, 67, 132, 4, 2, 68, 32, 64, 2, 71, 73, 144, 1, 5, 74, - 65, 78, 71, 32, 202, 4, 76, 32, 8, 77, 73, 78, 68, 69, 82, 32, 82, 34, - 80, 106, 83, 136, 1, 5, 84, 85, 82, 78, 32, 42, 86, 193, 199, 1, 5, 70, - 69, 82, 69, 78, 26, 114, 69, 60, 3, 89, 67, 76, 230, 204, 5, 79, 149, - 145, 4, 13, 82, 69, 65, 84, 73, 79, 78, 65, 76, 32, 86, 69, 72, 4, 38, - 73, 181, 182, 10, 3, 80, 84, 65, 2, 235, 235, 11, 80, 18, 78, 69, 53, 15, - 73, 78, 71, 32, 83, 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 2, 29, 5, 68, - 32, 80, 65, 80, 2, 131, 164, 10, 69, 16, 100, 5, 84, 89, 80, 69, 45, 133, - 254, 4, 14, 71, 69, 78, 69, 82, 73, 67, 32, 77, 65, 84, 69, 82, 73, 14, - 58, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 2, 173, 206, 5, 6, 32, - 80, 76, 65, 83, 84, 4, 42, 65, 165, 251, 4, 4, 71, 73, 70, 84, 2, 199, - 186, 3, 80, 54, 120, 4, 83, 84, 69, 82, 249, 240, 5, 20, 79, 78, 65, 76, - 32, 73, 78, 68, 73, 67, 65, 84, 79, 82, 32, 83, 89, 77, 66, 79, 2, 155, - 155, 10, 69, 74, 128, 1, 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, - 73, 71, 78, 32, 44, 7, 76, 69, 84, 84, 69, 82, 32, 226, 1, 83, 35, 86, 8, - 194, 135, 10, 78, 154, 251, 1, 72, 3, 82, 46, 162, 1, 78, 238, 243, 7, - 77, 234, 138, 4, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, - 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 187, 2, 65, 12, 206, 244, 7, 89, - 142, 27, 71, 250, 238, 3, 68, 187, 2, 65, 2, 11, 69, 2, 155, 151, 11, 67, - 18, 64, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 151, 199, 9, 73, 16, - 54, 69, 254, 190, 11, 65, 186, 64, 73, 2, 79, 3, 85, 7, 178, 255, 11, 65, - 3, 85, 2, 189, 238, 10, 3, 73, 69, 86, 2, 137, 175, 11, 3, 73, 66, 66, 2, - 41, 8, 76, 65, 67, 69, 77, 69, 78, 84, 2, 17, 2, 32, 67, 2, 157, 198, 10, - 5, 72, 65, 82, 65, 67, 8, 32, 2, 84, 82, 251, 244, 6, 80, 6, 180, 252, 7, - 16, 73, 67, 84, 69, 68, 32, 76, 69, 70, 84, 32, 69, 78, 84, 82, 89, 179, - 240, 3, 79, 6, 214, 233, 10, 83, 166, 104, 76, 31, 82, 70, 64, 4, 69, 82, - 83, 69, 165, 242, 3, 6, 79, 76, 86, 73, 78, 71, 68, 30, 32, 169, 3, 2, - 68, 32, 20, 184, 1, 6, 67, 72, 69, 67, 75, 69, 28, 2, 76, 73, 108, 7, 83, - 79, 76, 73, 68, 85, 83, 212, 220, 7, 16, 84, 73, 76, 68, 69, 32, 79, 80, - 69, 82, 65, 84, 79, 82, 32, 65, 179, 246, 2, 73, 2, 145, 227, 10, 2, 82, - 32, 4, 248, 207, 3, 18, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, - 78, 84, 69, 68, 32, 80, 251, 236, 1, 78, 9, 11, 32, 6, 184, 162, 5, 9, - 80, 82, 69, 67, 69, 68, 73, 78, 71, 250, 150, 3, 79, 251, 154, 1, 87, 48, - 232, 2, 5, 65, 78, 71, 76, 69, 20, 7, 68, 79, 85, 66, 76, 69, 32, 118, - 78, 20, 5, 70, 79, 82, 75, 69, 70, 80, 82, 82, 214, 1, 83, 106, 84, 144, - 229, 6, 30, 72, 65, 78, 68, 32, 87, 73, 84, 72, 32, 77, 73, 68, 68, 76, - 69, 32, 70, 73, 78, 71, 69, 82, 32, 69, 88, 84, 69, 78, 68, 202, 185, 2, - 67, 114, 81, 156, 65, 3, 86, 73, 67, 177, 36, 6, 69, 77, 80, 84, 89, 32, - 5, 231, 227, 3, 32, 6, 40, 3, 80, 82, 73, 41, 3, 83, 84, 82, 4, 17, 2, - 77, 69, 5, 179, 180, 3, 32, 2, 29, 5, 79, 75, 69, 32, 78, 2, 235, 177, 6, + 178, 156, 4, 83, 252, 217, 4, 9, 84, 87, 79, 32, 68, 79, 84, 32, 76, 139, + 114, 81, 4, 191, 190, 2, 79, 6, 158, 158, 6, 88, 182, 227, 2, 77, 3, 78, + 2, 173, 154, 9, 7, 79, 82, 73, 90, 79, 78, 84, 4, 49, 10, 68, 69, 79, 71, + 82, 65, 80, 72, 73, 67, 4, 11, 32, 4, 218, 235, 9, 67, 35, 70, 22, 40, 4, + 69, 70, 84, 32, 143, 214, 10, 79, 20, 112, 6, 87, 72, 73, 84, 69, 32, + 142, 1, 66, 42, 68, 186, 100, 67, 166, 7, 65, 222, 203, 7, 80, 238, 7, + 83, 39, 84, 4, 162, 2, 67, 255, 99, 76, 22, 110, 66, 42, 68, 36, 6, 87, + 72, 73, 84, 69, 32, 150, 100, 67, 166, 7, 65, 222, 203, 7, 80, 238, 7, + 83, 39, 84, 2, 145, 101, 6, 76, 65, 67, 75, 32, 76, 2, 197, 107, 5, 79, + 85, 66, 76, 69, 6, 74, 67, 17, 14, 76, 69, 78, 84, 73, 67, 85, 76, 65, + 82, 32, 66, 82, 65, 2, 227, 99, 79, 4, 246, 234, 11, 67, 177, 29, 2, 75, + 67, 2, 187, 210, 10, 79, 22, 46, 78, 156, 1, 2, 86, 65, 231, 164, 12, 77, + 10, 34, 84, 133, 211, 6, 2, 67, 69, 6, 26, 32, 53, 2, 69, 82, 2, 21, 3, + 83, 67, 82, 2, 205, 193, 10, 2, 69, 69, 5, 17, 2, 32, 73, 2, 223, 235, + 11, 67, 10, 60, 5, 67, 89, 32, 77, 69, 29, 6, 84, 69, 32, 85, 83, 69, 2, + 213, 171, 7, 2, 83, 83, 8, 26, 32, 231, 180, 8, 45, 4, 134, 166, 10, 84, + 139, 121, 79, 14, 98, 74, 30, 80, 90, 83, 156, 34, 5, 72, 73, 66, 73, 84, + 173, 245, 8, 6, 66, 73, 78, 71, 32, 67, 2, 213, 167, 5, 2, 69, 67, 6, 64, + 6, 79, 82, 84, 73, 79, 78, 137, 157, 11, 4, 69, 82, 84, 89, 5, 183, 215, + 5, 65, 2, 173, 250, 10, 4, 69, 82, 80, 73, 60, 76, 14, 65, 76, 84, 69, + 82, 32, 80, 65, 72, 76, 65, 86, 73, 32, 215, 5, 89, 58, 172, 1, 15, 70, + 79, 85, 82, 32, 68, 79, 84, 83, 32, 87, 73, 84, 72, 32, 32, 7, 76, 69, + 84, 84, 69, 82, 32, 230, 2, 78, 154, 32, 83, 1, 8, 84, 85, 82, 78, 69, + 68, 32, 83, 4, 246, 242, 10, 67, 247, 114, 68, 36, 166, 1, 65, 22, 68, + 34, 76, 22, 77, 50, 87, 254, 169, 4, 71, 90, 90, 34, 83, 66, 89, 198, + 207, 1, 72, 234, 5, 75, 174, 81, 66, 170, 225, 4, 78, 134, 2, 84, 219, + 103, 80, 2, 223, 170, 4, 76, 2, 11, 65, 2, 227, 210, 6, 76, 2, 251, 170, + 4, 65, 2, 25, 4, 69, 77, 45, 81, 2, 255, 128, 6, 79, 2, 37, 7, 65, 87, + 45, 65, 89, 73, 78, 2, 149, 205, 1, 2, 45, 82, 14, 33, 6, 85, 77, 66, 69, + 82, 32, 14, 42, 84, 202, 170, 4, 79, 191, 236, 2, 70, 8, 42, 87, 250, + 191, 10, 72, 207, 162, 1, 69, 4, 182, 194, 10, 69, 239, 239, 1, 79, 2, + 243, 132, 12, 67, 18, 252, 1, 4, 78, 67, 84, 85, 94, 82, 56, 19, 84, 32, + 76, 73, 84, 84, 69, 82, 32, 73, 78, 32, 73, 84, 83, 32, 80, 76, 65, 178, + 151, 1, 83, 132, 42, 20, 66, 76, 73, 67, 32, 65, 68, 68, 82, 69, 83, 83, + 32, 76, 79, 85, 68, 83, 80, 69, 166, 237, 10, 49, 3, 50, 4, 164, 222, 10, + 9, 83, 32, 69, 76, 69, 86, 65, 84, 85, 129, 147, 1, 5, 65, 84, 73, 79, + 78, 4, 32, 2, 80, 76, 155, 152, 12, 83, 2, 175, 167, 11, 69, 2, 11, 67, + 2, 135, 153, 11, 69, 40, 98, 65, 180, 6, 6, 69, 83, 84, 73, 79, 78, 254, + 136, 6, 79, 229, 143, 5, 5, 73, 78, 67, 85, 78, 30, 104, 2, 68, 82, 240, + 4, 9, 84, 69, 82, 78, 73, 79, 78, 32, 73, 48, 4, 82, 84, 69, 82, 143, + 132, 11, 79, 24, 56, 4, 65, 78, 84, 32, 157, 4, 5, 85, 80, 76, 69, 32, + 20, 44, 6, 85, 80, 80, 69, 82, 32, 131, 2, 76, 16, 56, 4, 76, 69, 70, 84, + 249, 1, 5, 82, 73, 71, 72, 84, 11, 29, 5, 32, 65, 78, 68, 32, 8, 108, 6, + 76, 79, 87, 69, 82, 32, 53, 17, 85, 80, 80, 69, 82, 32, 82, 73, 71, 72, + 84, 32, 65, 78, 68, 32, 76, 4, 192, 1, 5, 76, 69, 70, 84, 32, 159, 254, + 11, 82, 4, 11, 79, 4, 11, 87, 4, 213, 254, 11, 2, 69, 82, 7, 65, 14, 32, + 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 76, 69, 70, 4, 11, 84, 5, 11, 32, + 2, 21, 3, 65, 78, 68, 2, 17, 2, 32, 76, 2, 17, 2, 79, 87, 2, 137, 213, + 10, 2, 69, 82, 4, 22, 73, 239, 37, 80, 2, 177, 228, 8, 7, 78, 84, 69, 71, + 82, 65, 76, 2, 17, 2, 32, 78, 2, 159, 222, 10, 79, 6, 34, 32, 161, 197, + 5, 2, 69, 68, 4, 162, 139, 5, 69, 175, 218, 6, 77, 220, 9, 114, 65, 142, + 8, 69, 220, 27, 6, 72, 73, 78, 79, 67, 69, 34, 73, 250, 87, 76, 46, 79, + 198, 19, 85, 243, 149, 11, 83, 62, 110, 67, 104, 2, 68, 73, 130, 1, 73, + 162, 5, 84, 172, 246, 7, 4, 66, 66, 73, 84, 214, 238, 3, 90, 235, 56, 77, + 6, 40, 4, 73, 78, 71, 32, 159, 250, 8, 67, 4, 192, 208, 10, 7, 77, 79, + 84, 79, 82, 67, 89, 199, 48, 67, 8, 66, 79, 141, 180, 8, 10, 67, 65, 76, + 32, 83, 89, 77, 66, 79, 76, 7, 168, 184, 6, 4, 65, 67, 84, 73, 229, 181, + 4, 2, 32, 66, 36, 60, 5, 76, 87, 65, 89, 32, 46, 78, 21, 4, 83, 69, 68, + 32, 4, 240, 131, 9, 2, 84, 82, 155, 251, 1, 67, 5, 243, 133, 11, 66, 28, + 152, 1, 3, 68, 79, 84, 34, 73, 48, 4, 72, 65, 78, 68, 182, 1, 77, 38, 83, + 166, 177, 7, 70, 206, 151, 2, 67, 161, 197, 1, 7, 66, 65, 67, 75, 32, 79, + 70, 5, 29, 5, 84, 69, 68, 32, 73, 2, 209, 40, 8, 78, 84, 69, 82, 80, 79, + 76, 65, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 214, 27, 70, 245, 194, 2, + 28, 80, 65, 82, 84, 32, 66, 69, 84, 87, 69, 69, 78, 32, 77, 73, 68, 68, + 76, 69, 32, 65, 78, 68, 32, 82, 73, 78, 71, 6, 230, 151, 11, 67, 2, 68, + 3, 82, 4, 60, 9, 77, 65, 76, 76, 32, 76, 69, 70, 84, 179, 251, 10, 81, 2, + 229, 158, 8, 2, 32, 83, 5, 243, 254, 11, 73, 250, 1, 226, 1, 67, 132, 4, + 2, 68, 32, 64, 2, 71, 73, 144, 1, 5, 74, 65, 78, 71, 32, 202, 4, 76, 32, + 8, 77, 73, 78, 68, 69, 82, 32, 82, 34, 80, 106, 83, 136, 1, 5, 84, 85, + 82, 78, 32, 42, 86, 209, 199, 1, 5, 70, 69, 82, 69, 78, 26, 114, 69, 60, + 3, 89, 67, 76, 146, 209, 5, 79, 205, 155, 4, 13, 82, 69, 65, 84, 73, 79, + 78, 65, 76, 32, 86, 69, 72, 4, 38, 73, 209, 198, 10, 3, 80, 84, 65, 2, + 163, 254, 11, 80, 18, 78, 69, 53, 15, 73, 78, 71, 32, 83, 89, 77, 66, 79, + 76, 32, 70, 79, 82, 32, 2, 29, 5, 68, 32, 80, 65, 80, 2, 211, 179, 10, + 69, 16, 100, 5, 84, 89, 80, 69, 45, 173, 130, 5, 14, 71, 69, 78, 69, 82, + 73, 67, 32, 77, 65, 84, 69, 82, 73, 14, 58, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 3, 55, 2, 133, 210, 5, 6, 32, 80, 76, 65, 83, 84, 4, 42, 65, + 205, 255, 4, 4, 71, 73, 70, 84, 2, 231, 190, 3, 80, 54, 120, 4, 83, 84, + 69, 82, 213, 244, 5, 20, 79, 78, 65, 76, 32, 73, 78, 68, 73, 67, 65, 84, + 79, 82, 32, 83, 89, 77, 66, 79, 2, 235, 170, 10, 69, 74, 128, 1, 15, 67, + 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 44, 7, 76, 69, + 84, 84, 69, 82, 32, 226, 1, 83, 35, 86, 8, 146, 151, 10, 78, 130, 254, 1, + 72, 3, 82, 46, 162, 1, 78, 186, 253, 7, 77, 214, 147, 4, 66, 2, 67, 2, + 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, + 2, 89, 187, 2, 65, 12, 154, 254, 7, 89, 166, 31, 71, 206, 243, 3, 68, + 187, 2, 65, 2, 11, 69, 2, 211, 169, 11, 67, 18, 64, 10, 79, 87, 69, 76, + 32, 83, 73, 71, 78, 32, 139, 214, 9, 73, 16, 54, 69, 182, 209, 11, 65, + 186, 64, 73, 2, 79, 3, 85, 7, 234, 145, 12, 65, 3, 85, 2, 217, 254, 10, + 3, 73, 69, 86, 2, 193, 193, 11, 3, 73, 66, 66, 2, 41, 8, 76, 65, 67, 69, + 77, 69, 78, 84, 2, 17, 2, 32, 67, 2, 193, 214, 10, 5, 72, 65, 82, 65, 67, + 8, 32, 2, 84, 82, 231, 254, 6, 80, 6, 152, 138, 8, 16, 73, 67, 84, 69, + 68, 32, 76, 69, 70, 84, 32, 69, 78, 84, 82, 89, 135, 245, 3, 79, 6, 242, + 249, 10, 83, 194, 106, 76, 31, 82, 70, 64, 4, 69, 82, 83, 69, 197, 246, + 3, 6, 79, 76, 86, 73, 78, 71, 68, 30, 32, 169, 3, 2, 68, 32, 20, 184, 1, + 6, 67, 72, 69, 67, 75, 69, 28, 2, 76, 73, 108, 7, 83, 79, 76, 73, 68, 85, + 83, 232, 229, 7, 16, 84, 73, 76, 68, 69, 32, 79, 80, 69, 82, 65, 84, 79, + 82, 32, 65, 195, 253, 2, 73, 2, 197, 243, 10, 2, 82, 32, 4, 152, 212, 3, + 18, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, + 80, 135, 237, 1, 78, 9, 11, 32, 6, 240, 166, 5, 9, 80, 82, 69, 67, 69, + 68, 73, 78, 71, 166, 161, 3, 79, 251, 154, 1, 87, 48, 248, 2, 5, 65, 78, + 71, 76, 69, 20, 7, 68, 79, 85, 66, 76, 69, 32, 118, 78, 20, 5, 70, 79, + 82, 75, 69, 70, 80, 82, 82, 214, 1, 83, 106, 84, 236, 238, 6, 30, 72, 65, + 78, 68, 32, 87, 73, 84, 72, 32, 77, 73, 68, 68, 76, 69, 32, 70, 73, 78, + 71, 69, 82, 32, 69, 88, 84, 69, 78, 68, 198, 190, 2, 67, 114, 81, 180, + 102, 6, 69, 77, 80, 84, 89, 32, 253, 93, 7, 86, 73, 67, 84, 79, 82, 89, + 5, 247, 231, 3, 32, 6, 40, 3, 80, 82, 73, 41, 3, 83, 84, 82, 4, 17, 2, + 77, 69, 5, 195, 184, 3, 32, 2, 29, 5, 79, 75, 69, 32, 78, 2, 155, 187, 6, 79, 2, 49, 10, 68, 32, 80, 65, 82, 65, 71, 82, 65, 80, 2, 179, 4, 72, 4, - 36, 3, 73, 76, 67, 203, 219, 10, 82, 2, 11, 82, 2, 177, 236, 10, 2, 79, + 36, 3, 73, 76, 67, 239, 235, 10, 82, 2, 11, 82, 2, 217, 254, 10, 2, 79, 87, 6, 156, 1, 17, 65, 73, 83, 69, 68, 32, 72, 65, 78, 68, 32, 87, 73, - 84, 72, 32, 70, 200, 107, 8, 79, 84, 65, 84, 69, 68, 32, 70, 217, 166, 6, - 4, 73, 71, 72, 84, 2, 133, 113, 9, 73, 78, 71, 69, 82, 83, 32, 83, 80, 4, - 208, 134, 1, 17, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, - 84, 65, 76, 183, 159, 7, 69, 10, 88, 4, 73, 76, 68, 69, 28, 7, 82, 73, - 80, 76, 69, 32, 80, 241, 150, 7, 3, 72, 85, 77, 5, 213, 231, 4, 2, 32, - 69, 2, 235, 215, 10, 82, 2, 11, 82, 2, 231, 177, 11, 79, 147, 4, 228, 1, - 4, 66, 66, 79, 78, 152, 1, 3, 67, 69, 32, 60, 3, 71, 72, 84, 240, 81, 2, + 84, 72, 32, 70, 148, 107, 8, 79, 84, 65, 84, 69, 68, 32, 70, 253, 176, 6, + 4, 73, 71, 72, 84, 2, 209, 112, 9, 73, 78, 71, 69, 82, 83, 32, 83, 80, 4, + 156, 134, 1, 17, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, + 84, 65, 76, 191, 174, 7, 69, 10, 88, 4, 73, 76, 68, 69, 28, 7, 82, 73, + 80, 76, 69, 32, 80, 225, 160, 7, 3, 72, 85, 77, 5, 237, 235, 4, 2, 32, + 69, 2, 143, 232, 10, 82, 2, 11, 82, 2, 143, 196, 11, 79, 147, 4, 228, 1, + 4, 66, 66, 79, 78, 152, 1, 3, 67, 69, 32, 60, 3, 71, 72, 84, 188, 81, 2, 78, 71, 252, 1, 23, 83, 73, 78, 71, 32, 68, 73, 65, 71, 79, 78, 65, 76, - 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 226, 212, 5, 65, 255, 158, 4, 70, + 32, 67, 82, 79, 83, 83, 73, 78, 71, 32, 198, 222, 5, 65, 227, 165, 4, 70, 19, 37, 7, 32, 65, 82, 82, 79, 87, 32, 16, 88, 3, 76, 69, 70, 0, 4, 82, - 73, 71, 72, 158, 193, 4, 85, 141, 128, 7, 3, 68, 79, 87, 4, 171, 248, 1, - 84, 4, 26, 67, 199, 228, 9, 66, 2, 217, 145, 1, 3, 82, 65, 67, 224, 3, + 73, 71, 72, 178, 197, 4, 85, 161, 142, 7, 3, 68, 79, 87, 4, 207, 252, 1, + 84, 4, 26, 67, 135, 244, 9, 66, 2, 157, 145, 1, 3, 82, 65, 67, 224, 3, 110, 32, 190, 36, 45, 152, 12, 11, 72, 65, 78, 68, 32, 73, 78, 84, 69, 82, 73, 29, 6, 87, 65, 82, 68, 83, 32, 202, 1, 166, 2, 65, 132, 6, 2, 66, 76, 62, 67, 184, 1, 2, 68, 79, 242, 1, 70, 60, 2, 72, 65, 176, 5, 13, 74, @@ -6601,2251 +6675,2279 @@ static const unsigned char packed_name_dawg[] = { 69, 82, 84, 73, 67, 65, 76, 32, 147, 1, 87, 28, 22, 78, 163, 4, 82, 22, 24, 2, 68, 32, 39, 71, 4, 170, 50, 76, 21, 3, 85, 80, 80, 18, 26, 69, 17, 2, 76, 69, 2, 203, 26, 82, 17, 11, 32, 14, 154, 1, 66, 44, 8, 68, 79, 84, - 84, 69, 68, 32, 83, 2, 83, 112, 5, 87, 73, 84, 72, 32, 153, 196, 10, 12, - 86, 65, 82, 73, 65, 78, 84, 32, 87, 73, 84, 72, 4, 153, 194, 10, 6, 82, + 84, 69, 68, 32, 83, 2, 83, 112, 5, 87, 73, 84, 72, 32, 189, 212, 10, 12, + 86, 65, 82, 73, 65, 78, 84, 32, 87, 73, 84, 72, 4, 173, 210, 10, 6, 82, 65, 67, 75, 69, 84, 2, 37, 7, 85, 66, 83, 84, 73, 84, 85, 2, 25, 4, 84, - 73, 79, 78, 2, 21, 3, 32, 77, 65, 2, 231, 138, 1, 82, 4, 68, 11, 68, 79, - 87, 78, 87, 65, 82, 68, 83, 32, 90, 179, 232, 8, 65, 2, 233, 201, 10, 5, + 73, 79, 78, 2, 21, 3, 32, 77, 65, 2, 171, 138, 1, 82, 4, 68, 11, 68, 79, + 87, 78, 87, 65, 82, 68, 83, 32, 90, 139, 247, 8, 65, 2, 141, 218, 10, 5, 73, 71, 90, 65, 71, 6, 18, 67, 79, 82, 2, 41, 8, 32, 71, 82, 69, 65, 84, 69, 82, 2, 249, 24, 4, 45, 84, 72, 65, 4, 41, 8, 79, 87, 32, 87, 73, 84, - 72, 32, 4, 188, 220, 7, 7, 67, 73, 82, 67, 76, 69, 68, 151, 180, 2, 83, - 4, 25, 4, 65, 67, 75, 32, 4, 130, 27, 76, 139, 205, 7, 84, 12, 38, 85, - 166, 26, 79, 207, 231, 2, 69, 8, 53, 11, 82, 76, 89, 32, 66, 82, 65, 67, + 72, 32, 4, 144, 234, 7, 7, 67, 73, 82, 67, 76, 69, 68, 207, 182, 2, 83, + 4, 25, 4, 65, 67, 75, 32, 4, 130, 27, 76, 223, 218, 7, 84, 12, 38, 85, + 166, 26, 79, 243, 235, 2, 69, 8, 53, 11, 82, 76, 89, 32, 66, 82, 65, 67, 75, 69, 84, 9, 11, 32, 6, 44, 4, 77, 73, 68, 68, 250, 10, 76, 27, 85, 2, - 201, 170, 10, 2, 76, 69, 12, 38, 84, 41, 5, 85, 66, 76, 69, 32, 2, 137, - 17, 6, 84, 69, 68, 32, 83, 85, 10, 50, 65, 94, 87, 198, 158, 3, 81, 131, - 190, 4, 80, 4, 162, 31, 78, 157, 157, 3, 15, 82, 82, 79, 87, 32, 87, 73, - 84, 72, 32, 82, 79, 85, 78, 68, 2, 139, 24, 73, 6, 26, 73, 247, 254, 2, - 76, 4, 154, 201, 8, 86, 175, 97, 83, 28, 32, 3, 76, 70, 32, 187, 4, 78, + 221, 186, 10, 2, 76, 69, 12, 38, 84, 41, 5, 85, 66, 76, 69, 32, 2, 137, + 17, 6, 84, 69, 68, 32, 83, 85, 10, 50, 65, 94, 87, 214, 162, 3, 81, 199, + 199, 4, 80, 4, 162, 31, 78, 173, 161, 3, 15, 82, 82, 79, 87, 32, 87, 73, + 84, 72, 32, 82, 79, 85, 78, 68, 2, 139, 24, 73, 6, 26, 73, 155, 131, 3, + 76, 4, 238, 215, 8, 86, 191, 97, 83, 28, 32, 3, 76, 70, 32, 187, 4, 78, 26, 128, 1, 8, 65, 78, 68, 32, 76, 69, 70, 84, 62, 66, 90, 70, 82, 72, - 50, 82, 58, 84, 70, 87, 246, 13, 76, 22, 85, 215, 212, 8, 77, 2, 29, 5, - 32, 72, 65, 76, 70, 2, 245, 202, 8, 2, 32, 87, 6, 11, 76, 6, 40, 4, 65, - 67, 75, 32, 163, 172, 10, 79, 4, 146, 138, 10, 67, 207, 11, 83, 4, 58, - 79, 221, 152, 3, 8, 76, 89, 73, 78, 71, 32, 83, 65, 2, 175, 187, 9, 76, - 2, 253, 201, 8, 7, 79, 82, 73, 90, 79, 78, 84, 2, 33, 6, 85, 78, 78, 73, - 78, 71, 2, 247, 228, 6, 32, 2, 209, 169, 5, 12, 82, 73, 80, 76, 69, 32, - 68, 65, 83, 72, 32, 72, 2, 189, 151, 10, 4, 72, 73, 84, 69, 2, 173, 152, + 50, 82, 58, 84, 70, 87, 246, 13, 76, 22, 85, 175, 227, 8, 77, 2, 29, 5, + 32, 72, 65, 76, 70, 2, 201, 217, 8, 2, 32, 87, 6, 11, 76, 6, 40, 4, 65, + 67, 75, 32, 183, 188, 10, 79, 4, 158, 154, 10, 67, 207, 11, 83, 4, 58, + 79, 237, 156, 3, 8, 76, 89, 73, 78, 71, 32, 83, 65, 2, 131, 202, 9, 76, + 2, 209, 216, 8, 7, 79, 82, 73, 90, 79, 78, 84, 2, 33, 6, 85, 78, 78, 73, + 78, 71, 2, 203, 238, 6, 32, 2, 153, 173, 5, 12, 82, 73, 80, 76, 69, 32, + 68, 65, 83, 72, 32, 72, 2, 209, 167, 10, 4, 72, 73, 84, 69, 2, 173, 152, 1, 16, 68, 32, 84, 69, 76, 69, 80, 72, 79, 78, 69, 32, 82, 69, 67, 69, 4, - 231, 217, 7, 84, 2, 233, 133, 11, 11, 87, 32, 80, 65, 82, 65, 80, 72, 82, + 187, 231, 7, 84, 2, 145, 152, 11, 11, 87, 32, 80, 65, 82, 65, 80, 72, 82, 65, 83, 2, 177, 4, 16, 79, 82, 77, 65, 76, 32, 70, 65, 67, 84, 79, 82, - 32, 83, 69, 77, 8, 74, 85, 210, 206, 8, 78, 169, 188, 1, 8, 80, 69, 78, - 32, 83, 81, 85, 65, 2, 181, 215, 10, 6, 84, 69, 82, 32, 74, 79, 8, 49, + 32, 83, 69, 77, 8, 74, 85, 166, 221, 8, 78, 225, 189, 1, 8, 80, 69, 78, + 32, 83, 81, 85, 65, 2, 221, 233, 10, 6, 84, 69, 82, 32, 74, 79, 8, 49, 10, 65, 82, 69, 78, 84, 72, 69, 83, 73, 83, 9, 11, 32, 6, 34, 76, 26, 85, - 171, 182, 9, 69, 2, 157, 37, 2, 79, 87, 2, 133, 37, 2, 80, 80, 2, 217, + 255, 196, 9, 69, 2, 157, 37, 2, 79, 87, 2, 133, 37, 2, 80, 80, 2, 217, 10, 10, 73, 83, 69, 68, 32, 79, 77, 73, 83, 83, 40, 62, 45, 70, 69, 78, - 73, 68, 2, 80, 69, 126, 81, 227, 2, 85, 2, 253, 241, 7, 12, 83, 72, 65, - 80, 69, 68, 32, 66, 65, 71, 32, 68, 4, 26, 77, 231, 221, 8, 86, 2, 205, - 196, 10, 7, 73, 68, 73, 82, 69, 67, 84, 4, 194, 146, 3, 78, 145, 238, 7, + 73, 68, 2, 80, 69, 126, 81, 227, 2, 85, 2, 173, 128, 8, 12, 83, 72, 65, + 80, 69, 68, 32, 66, 65, 71, 32, 68, 4, 26, 77, 191, 236, 8, 86, 2, 217, + 212, 10, 7, 73, 68, 73, 82, 69, 67, 84, 4, 210, 150, 3, 78, 169, 252, 7, 8, 68, 69, 87, 65, 89, 83, 32, 85, 8, 32, 4, 65, 75, 69, 82, 67, 69, 7, - 33, 6, 32, 87, 73, 84, 72, 32, 4, 214, 248, 3, 79, 55, 84, 2, 137, 5, 2, + 33, 6, 32, 87, 73, 84, 72, 32, 4, 246, 252, 3, 79, 55, 84, 2, 137, 5, 2, 67, 72, 20, 57, 12, 85, 65, 82, 69, 32, 66, 82, 65, 67, 75, 69, 84, 21, 11, 32, 18, 84, 3, 76, 79, 87, 0, 3, 85, 80, 80, 28, 5, 87, 73, 84, 72, - 32, 143, 177, 9, 69, 2, 213, 226, 3, 2, 69, 82, 12, 96, 8, 84, 73, 67, - 75, 32, 73, 78, 32, 230, 6, 81, 246, 221, 7, 85, 222, 125, 68, 183, 189, - 2, 83, 4, 212, 225, 3, 6, 66, 79, 84, 84, 79, 77, 1, 3, 84, 79, 80, 2, - 165, 4, 6, 66, 83, 84, 73, 84, 85, 26, 54, 72, 130, 3, 82, 174, 209, 7, - 79, 171, 202, 2, 65, 14, 62, 73, 116, 5, 79, 85, 71, 72, 84, 45, 4, 82, - 69, 69, 32, 4, 21, 3, 82, 68, 32, 4, 68, 4, 73, 78, 68, 85, 185, 207, 1, - 7, 87, 72, 73, 84, 69, 32, 82, 2, 175, 148, 11, 67, 2, 11, 32, 2, 197, - 132, 3, 3, 66, 85, 66, 8, 60, 9, 81, 85, 65, 82, 84, 69, 82, 83, 32, 191, - 215, 8, 69, 6, 34, 76, 22, 85, 179, 221, 8, 66, 2, 37, 2, 79, 87, 2, 17, - 2, 80, 80, 2, 139, 216, 8, 69, 8, 34, 65, 89, 4, 73, 65, 78, 71, 2, 33, - 6, 78, 83, 80, 79, 83, 73, 2, 11, 84, 2, 17, 2, 73, 79, 2, 235, 247, 10, - 78, 6, 32, 2, 76, 69, 199, 214, 8, 85, 5, 41, 8, 32, 65, 66, 79, 86, 69, - 32, 76, 2, 225, 163, 9, 2, 69, 70, 6, 18, 66, 95, 82, 4, 68, 9, 65, 82, - 32, 87, 73, 84, 72, 32, 81, 177, 175, 10, 2, 79, 88, 2, 251, 208, 8, 85, - 2, 241, 173, 9, 3, 85, 76, 69, 14, 22, 72, 203, 1, 73, 12, 25, 4, 73, 84, - 69, 32, 12, 54, 67, 54, 76, 250, 196, 7, 80, 238, 7, 83, 39, 84, 4, 26, - 79, 207, 193, 7, 85, 2, 65, 3, 82, 78, 69, 2, 41, 8, 69, 78, 84, 73, 67, - 85, 76, 65, 2, 143, 244, 10, 82, 2, 133, 190, 6, 6, 71, 71, 76, 89, 32, + 32, 227, 191, 9, 69, 2, 245, 230, 3, 2, 69, 82, 12, 96, 8, 84, 73, 67, + 75, 32, 73, 78, 32, 230, 6, 81, 166, 236, 7, 85, 134, 126, 68, 135, 193, + 2, 83, 4, 244, 229, 3, 6, 66, 79, 84, 84, 79, 77, 1, 3, 84, 79, 80, 2, + 165, 4, 6, 66, 83, 84, 73, 84, 85, 26, 54, 72, 130, 3, 82, 130, 223, 7, + 79, 235, 204, 2, 65, 14, 62, 73, 116, 5, 79, 85, 71, 72, 84, 45, 4, 82, + 69, 69, 32, 4, 21, 3, 82, 68, 32, 4, 68, 4, 73, 78, 68, 85, 221, 211, 1, + 7, 87, 72, 73, 84, 69, 32, 82, 2, 215, 166, 11, 67, 2, 11, 32, 2, 213, + 136, 3, 3, 66, 85, 66, 8, 60, 9, 81, 85, 65, 82, 84, 69, 82, 83, 32, 151, + 230, 8, 69, 6, 34, 76, 22, 85, 139, 236, 8, 66, 2, 37, 2, 79, 87, 2, 17, + 2, 80, 80, 2, 227, 230, 8, 69, 8, 34, 65, 89, 4, 73, 65, 78, 71, 2, 33, + 6, 78, 83, 80, 79, 83, 73, 2, 11, 84, 2, 17, 2, 73, 79, 2, 147, 138, 11, + 78, 6, 32, 2, 76, 69, 159, 229, 8, 85, 5, 41, 8, 32, 65, 66, 79, 86, 69, + 32, 76, 2, 181, 178, 9, 2, 69, 70, 6, 18, 66, 95, 82, 4, 68, 9, 65, 82, + 32, 87, 73, 84, 72, 32, 81, 213, 191, 10, 2, 79, 88, 2, 211, 223, 8, 85, + 2, 189, 188, 9, 3, 85, 76, 69, 14, 22, 72, 203, 1, 73, 12, 25, 4, 73, 84, + 69, 32, 12, 54, 67, 54, 76, 206, 210, 7, 80, 238, 7, 83, 39, 84, 4, 26, + 79, 163, 207, 7, 85, 2, 65, 3, 82, 78, 69, 2, 41, 8, 69, 78, 84, 73, 67, + 85, 76, 65, 2, 183, 134, 11, 82, 2, 225, 199, 6, 6, 71, 71, 76, 89, 32, 70, 62, 118, 70, 226, 2, 72, 128, 1, 9, 80, 79, 73, 78, 84, 73, 78, 71, 32, 214, 4, 83, 197, 1, 6, 84, 79, 45, 76, 69, 70, 18, 33, 6, 65, 67, 73, 78, 71, 32, 18, 116, 14, 65, 82, 77, 69, 78, 73, 65, 78, 32, 69, 84, 69, - 82, 78, 20, 4, 66, 65, 83, 83, 16, 3, 70, 73, 83, 87, 83, 2, 171, 145, 8, - 73, 2, 227, 43, 73, 6, 26, 72, 239, 195, 11, 84, 5, 11, 32, 2, 149, 162, - 8, 6, 87, 73, 84, 72, 32, 79, 8, 236, 203, 3, 10, 86, 65, 83, 84, 73, 32, - 83, 73, 71, 78, 143, 213, 4, 78, 2, 81, 18, 65, 78, 68, 69, 68, 32, 73, - 78, 84, 69, 82, 76, 65, 67, 69, 68, 32, 80, 2, 21, 3, 69, 78, 84, 2, 231, - 134, 10, 65, 30, 90, 65, 30, 67, 94, 68, 58, 77, 58, 82, 182, 1, 83, 74, - 84, 254, 157, 8, 69, 179, 124, 71, 4, 90, 78, 203, 160, 8, 84, 2, 29, 5, - 85, 82, 86, 69, 68, 2, 17, 2, 32, 65, 2, 11, 78, 2, 177, 237, 10, 2, 71, - 76, 4, 248, 254, 2, 5, 79, 85, 66, 76, 69, 131, 177, 6, 73, 2, 129, 168, - 10, 9, 65, 71, 78, 73, 70, 89, 73, 78, 71, 10, 38, 79, 206, 163, 9, 65, - 247, 28, 73, 6, 92, 5, 67, 75, 69, 84, 32, 137, 163, 9, 12, 76, 76, 69, - 82, 32, 67, 79, 65, 83, 84, 69, 82, 4, 228, 43, 3, 66, 79, 79, 179, 189, - 10, 83, 2, 11, 84, 2, 21, 3, 73, 67, 75, 2, 221, 177, 6, 4, 32, 70, 73, - 71, 2, 155, 220, 7, 65, 4, 46, 72, 85, 7, 73, 68, 69, 32, 65, 82, 67, 2, - 17, 2, 65, 68, 2, 17, 2, 69, 68, 2, 173, 156, 10, 6, 32, 87, 72, 73, 84, - 69, 2, 11, 32, 2, 241, 224, 8, 8, 67, 76, 79, 67, 75, 87, 73, 83, 8, 17, - 2, 84, 32, 8, 86, 73, 40, 4, 79, 86, 69, 82, 152, 130, 5, 5, 69, 77, 66, - 69, 68, 251, 246, 5, 77, 2, 17, 2, 83, 79, 2, 131, 135, 1, 76, 2, 247, - 150, 3, 82, 2, 129, 171, 10, 2, 79, 82, 214, 1, 160, 1, 5, 65, 82, 82, - 79, 87, 130, 9, 66, 86, 68, 210, 1, 70, 122, 72, 230, 6, 76, 26, 79, 34, - 80, 50, 82, 70, 83, 94, 84, 206, 8, 87, 190, 182, 8, 67, 47, 81, 75, 26, - 32, 223, 132, 9, 45, 70, 94, 65, 170, 2, 70, 110, 84, 184, 1, 5, 87, 73, - 84, 72, 32, 181, 157, 1, 4, 79, 86, 69, 82, 12, 40, 5, 66, 79, 86, 69, - 32, 143, 1, 78, 10, 70, 82, 140, 161, 1, 5, 83, 72, 79, 82, 84, 130, 193, - 3, 65, 55, 84, 4, 37, 7, 69, 86, 69, 82, 83, 69, 32, 4, 226, 225, 4, 65, + 82, 78, 20, 4, 66, 65, 83, 83, 16, 3, 70, 73, 83, 87, 83, 2, 255, 159, 8, + 73, 2, 175, 43, 73, 6, 26, 72, 151, 214, 11, 84, 5, 11, 32, 2, 233, 176, + 8, 6, 87, 73, 84, 72, 32, 79, 8, 140, 208, 3, 10, 86, 65, 83, 84, 73, 32, + 83, 73, 71, 78, 195, 223, 4, 78, 2, 81, 18, 65, 78, 68, 69, 68, 32, 73, + 78, 84, 69, 82, 76, 65, 67, 69, 68, 32, 80, 2, 21, 3, 69, 78, 84, 2, 251, + 150, 10, 65, 30, 90, 65, 30, 67, 94, 68, 58, 77, 58, 82, 182, 1, 83, 74, + 84, 210, 172, 8, 69, 179, 124, 71, 4, 90, 78, 159, 175, 8, 84, 2, 29, 5, + 85, 82, 86, 69, 68, 2, 17, 2, 32, 65, 2, 11, 78, 2, 217, 255, 10, 2, 71, + 76, 4, 136, 131, 3, 5, 79, 85, 66, 76, 69, 179, 188, 6, 73, 2, 165, 184, + 10, 9, 65, 71, 78, 73, 70, 89, 73, 78, 71, 10, 38, 79, 154, 178, 9, 65, + 235, 29, 73, 6, 92, 5, 67, 75, 69, 84, 32, 213, 177, 9, 12, 76, 76, 69, + 82, 32, 67, 79, 65, 83, 84, 69, 82, 4, 176, 43, 3, 66, 79, 79, 143, 208, + 10, 83, 2, 11, 84, 2, 21, 3, 73, 67, 75, 2, 185, 187, 6, 4, 32, 70, 73, + 71, 2, 239, 234, 7, 65, 4, 46, 72, 85, 7, 73, 68, 69, 32, 65, 82, 67, 2, + 17, 2, 65, 68, 2, 17, 2, 69, 68, 2, 209, 172, 10, 6, 32, 87, 72, 73, 84, + 69, 2, 11, 32, 2, 201, 239, 8, 8, 67, 76, 79, 67, 75, 87, 73, 83, 8, 17, + 2, 84, 32, 8, 86, 73, 40, 4, 79, 86, 69, 82, 176, 134, 5, 5, 69, 77, 66, + 69, 68, 139, 133, 6, 77, 2, 17, 2, 83, 79, 2, 131, 135, 1, 76, 2, 135, + 155, 3, 82, 2, 141, 187, 10, 2, 79, 82, 214, 1, 160, 1, 5, 65, 82, 82, + 79, 87, 130, 9, 66, 86, 68, 210, 1, 70, 122, 72, 178, 6, 76, 26, 79, 34, + 80, 50, 82, 70, 83, 94, 84, 206, 8, 87, 202, 197, 8, 67, 47, 81, 75, 26, + 32, 195, 147, 9, 45, 70, 94, 65, 170, 2, 70, 110, 84, 184, 1, 5, 87, 73, + 84, 72, 32, 129, 160, 1, 4, 79, 86, 69, 82, 12, 40, 5, 66, 79, 86, 69, + 32, 143, 1, 78, 10, 70, 82, 216, 163, 1, 5, 83, 72, 79, 82, 84, 222, 194, + 3, 65, 55, 84, 4, 37, 7, 69, 86, 69, 82, 83, 69, 32, 4, 138, 230, 4, 65, 55, 84, 2, 61, 13, 68, 32, 85, 80, 80, 69, 82, 32, 65, 78, 68, 32, 76, 2, - 17, 2, 79, 87, 2, 169, 198, 8, 2, 69, 82, 6, 25, 4, 82, 79, 77, 32, 6, - 36, 3, 66, 65, 82, 227, 198, 8, 68, 5, 189, 1, 6, 32, 84, 79, 32, 66, 76, - 10, 56, 7, 72, 82, 79, 85, 71, 72, 32, 65, 3, 79, 32, 66, 6, 184, 220, 4, - 3, 83, 85, 80, 198, 197, 4, 71, 155, 146, 2, 88, 4, 26, 76, 227, 250, 10, - 65, 2, 141, 231, 9, 3, 65, 67, 75, 40, 140, 1, 6, 67, 79, 82, 78, 69, 82, - 26, 68, 98, 76, 86, 80, 30, 83, 38, 84, 178, 195, 8, 77, 38, 78, 122, 69, - 226, 151, 1, 72, 143, 163, 1, 86, 2, 133, 19, 2, 32, 68, 4, 11, 79, 4, - 40, 4, 84, 84, 69, 68, 155, 205, 7, 85, 2, 17, 2, 32, 83, 2, 251, 158, - 11, 84, 6, 26, 79, 143, 196, 8, 65, 4, 26, 87, 211, 176, 11, 79, 2, 253, - 196, 3, 2, 69, 82, 2, 237, 130, 9, 2, 76, 85, 6, 186, 196, 8, 77, 251, - 187, 2, 84, 10, 206, 16, 73, 171, 3, 65, 8, 58, 65, 128, 12, 5, 79, 84, - 84, 79, 77, 175, 184, 8, 76, 2, 145, 2, 2, 67, 75, 14, 48, 6, 79, 85, 66, - 76, 69, 32, 207, 210, 8, 65, 12, 40, 5, 65, 82, 82, 79, 87, 135, 19, 68, - 11, 26, 32, 171, 250, 8, 45, 6, 26, 87, 207, 200, 8, 70, 4, 25, 4, 73, - 84, 72, 32, 4, 146, 253, 10, 86, 79, 83, 4, 40, 4, 82, 79, 78, 84, 235, - 195, 8, 73, 2, 233, 194, 8, 14, 45, 84, 73, 76, 84, 69, 68, 32, 83, 72, - 65, 68, 79, 87, 30, 26, 65, 163, 199, 8, 69, 26, 48, 6, 82, 80, 79, 79, - 78, 32, 219, 229, 10, 78, 24, 88, 8, 79, 86, 69, 82, 32, 76, 69, 70, 45, - 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, 32, 2, 157, 132, 8, 6, 84, 87, - 65, 82, 68, 83, 22, 44, 4, 68, 79, 87, 78, 173, 1, 2, 85, 80, 10, 26, 32, - 219, 204, 8, 87, 8, 76, 8, 65, 66, 79, 86, 69, 32, 76, 69, 18, 66, 154, - 196, 8, 70, 179, 5, 84, 2, 227, 2, 70, 2, 177, 218, 4, 7, 69, 76, 79, 87, - 32, 76, 79, 12, 26, 32, 175, 203, 8, 87, 10, 60, 6, 65, 66, 79, 86, 69, - 32, 142, 195, 8, 70, 179, 5, 84, 6, 22, 76, 155, 1, 82, 4, 32, 2, 69, 70, - 239, 216, 4, 79, 2, 253, 139, 2, 24, 84, 87, 65, 82, 68, 83, 32, 72, 65, - 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 2, 21, 3, 73, - 71, 72, 2, 105, 24, 84, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, - 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 2, 11, 32, 2, 179, 236, 1, - 68, 2, 141, 1, 2, 69, 70, 2, 189, 197, 8, 3, 80, 69, 78, 4, 190, 201, 8, - 65, 181, 205, 1, 3, 85, 83, 72, 4, 36, 3, 73, 71, 72, 131, 210, 10, 79, - 2, 11, 84, 2, 171, 1, 45, 6, 32, 2, 81, 85, 203, 192, 8, 65, 4, 26, 73, - 227, 192, 8, 65, 2, 217, 200, 8, 2, 71, 71, 54, 38, 79, 60, 2, 82, 73, - 227, 4, 87, 2, 11, 80, 2, 11, 32, 2, 241, 184, 8, 4, 83, 72, 65, 68, 34, - 40, 5, 65, 78, 71, 76, 69, 255, 3, 80, 30, 56, 8, 45, 72, 69, 65, 68, 69, - 68, 32, 239, 204, 8, 32, 28, 52, 5, 65, 82, 82, 79, 87, 190, 197, 8, 68, - 39, 80, 25, 11, 32, 22, 64, 8, 79, 86, 69, 82, 32, 76, 69, 70, 22, 87, - 251, 192, 8, 84, 2, 171, 192, 8, 84, 18, 25, 4, 73, 84, 72, 32, 18, 128, - 1, 7, 68, 79, 85, 66, 76, 69, 32, 36, 7, 76, 79, 78, 71, 32, 84, 73, 218, - 192, 8, 66, 158, 1, 77, 34, 78, 34, 86, 35, 72, 4, 158, 251, 8, 72, 239, - 243, 1, 86, 4, 11, 80, 4, 11, 32, 4, 18, 68, 35, 85, 2, 169, 193, 8, 3, - 79, 87, 78, 2, 139, 193, 8, 80, 4, 21, 3, 76, 69, 32, 4, 138, 3, 68, 243, - 128, 10, 65, 18, 11, 79, 18, 56, 8, 45, 72, 69, 65, 68, 69, 68, 32, 163, - 195, 8, 32, 16, 76, 6, 65, 82, 82, 79, 87, 32, 213, 1, 8, 84, 82, 73, 80, - 76, 69, 32, 68, 14, 44, 5, 87, 73, 84, 72, 32, 167, 183, 8, 70, 12, 42, - 84, 238, 183, 7, 68, 243, 179, 3, 86, 8, 26, 65, 231, 194, 8, 82, 6, 17, - 2, 73, 76, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 150, 183, 7, 68, 243, - 179, 3, 86, 2, 137, 132, 1, 2, 65, 83, 8, 58, 65, 21, 10, 72, 73, 84, 69, - 32, 65, 82, 82, 79, 87, 2, 195, 191, 8, 86, 7, 11, 32, 4, 148, 189, 2, 4, - 70, 82, 79, 77, 147, 135, 6, 87, 19, 66, 32, 136, 1, 6, 69, 68, 32, 80, - 76, 65, 21, 3, 73, 78, 71, 12, 82, 66, 22, 80, 248, 196, 4, 2, 73, 78, - 26, 69, 238, 147, 3, 79, 239, 194, 2, 65, 2, 147, 135, 11, 85, 2, 11, 79, - 2, 171, 213, 10, 73, 2, 155, 227, 10, 78, 2, 253, 194, 3, 2, 32, 66, 4, - 24, 2, 70, 65, 75, 83, 2, 17, 2, 76, 76, 2, 21, 3, 73, 78, 71, 2, 217, - 226, 1, 2, 32, 68, 2, 233, 168, 3, 2, 79, 85, 8, 130, 151, 11, 69, 2, 73, - 2, 77, 3, 79, 126, 136, 2, 2, 67, 75, 20, 2, 76, 76, 188, 2, 4, 77, 65, - 78, 32, 194, 8, 79, 80, 2, 83, 69, 20, 6, 84, 65, 84, 69, 68, 32, 152, 3, - 3, 85, 78, 68, 214, 173, 3, 87, 200, 204, 3, 15, 65, 83, 84, 69, 68, 32, - 83, 87, 69, 69, 84, 32, 80, 79, 84, 205, 164, 2, 2, 66, 79, 5, 159, 248, - 10, 69, 10, 130, 1, 69, 64, 4, 32, 79, 70, 32, 101, 21, 73, 78, 71, 32, - 79, 78, 32, 84, 72, 69, 32, 70, 76, 79, 79, 82, 32, 76, 65, 85, 71, 6, - 60, 9, 68, 45, 85, 80, 32, 78, 69, 87, 83, 33, 2, 82, 32, 2, 11, 80, 2, - 215, 254, 1, 65, 4, 28, 3, 67, 79, 65, 23, 83, 2, 187, 218, 9, 83, 2, - 211, 94, 75, 2, 139, 193, 10, 72, 72, 140, 1, 6, 67, 69, 78, 84, 85, 82, - 22, 68, 100, 3, 81, 85, 73, 28, 8, 78, 85, 77, 69, 82, 65, 76, 32, 182, - 4, 83, 114, 85, 255, 219, 7, 65, 2, 187, 205, 5, 73, 6, 98, 69, 232, 5, - 5, 85, 80, 79, 78, 68, 61, 12, 73, 77, 73, 68, 73, 65, 32, 83, 69, 88, - 84, 85, 2, 229, 5, 3, 78, 65, 82, 48, 142, 1, 70, 136, 1, 3, 79, 78, 69, - 134, 1, 83, 66, 84, 232, 14, 10, 82, 69, 86, 69, 82, 83, 69, 68, 32, 79, - 210, 232, 2, 69, 143, 251, 6, 78, 14, 26, 73, 219, 149, 10, 79, 12, 36, - 3, 70, 84, 89, 187, 250, 2, 86, 7, 11, 32, 4, 222, 186, 5, 84, 185, 212, - 4, 5, 69, 65, 82, 76, 89, 11, 11, 32, 8, 50, 72, 41, 8, 84, 72, 79, 85, - 83, 65, 78, 68, 4, 185, 1, 6, 85, 78, 68, 82, 69, 68, 5, 197, 253, 3, 2, - 32, 67, 6, 32, 2, 73, 88, 171, 156, 9, 69, 5, 149, 141, 10, 2, 32, 76, - 10, 42, 69, 210, 248, 2, 87, 191, 163, 6, 72, 4, 11, 78, 5, 11, 32, 2, - 159, 184, 5, 84, 10, 46, 69, 181, 185, 7, 5, 73, 76, 73, 81, 85, 8, 60, - 2, 77, 85, 40, 5, 83, 84, 69, 82, 84, 21, 2, 88, 84, 2, 17, 2, 78, 67, 2, - 223, 184, 7, 73, 2, 199, 219, 7, 73, 4, 18, 65, 23, 85, 2, 171, 219, 7, - 78, 2, 143, 184, 7, 76, 4, 48, 6, 84, 32, 86, 69, 71, 69, 147, 209, 9, - 83, 2, 201, 192, 2, 2, 84, 65, 5, 243, 193, 9, 84, 10, 166, 2, 70, 32, - 11, 72, 69, 65, 86, 89, 32, 66, 76, 65, 67, 75, 52, 24, 76, 73, 71, 72, - 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, 66, 76, 65, - 67, 75, 0, 18, 87, 72, 73, 84, 69, 32, 70, 79, 85, 82, 32, 80, 79, 73, - 78, 84, 69, 68, 169, 53, 8, 67, 65, 80, 73, 84, 65, 76, 32, 2, 29, 5, 76, - 79, 82, 65, 76, 2, 213, 166, 9, 8, 32, 72, 69, 65, 82, 84, 32, 66, 2, - 149, 191, 9, 2, 32, 67, 16, 74, 32, 89, 14, 69, 68, 32, 83, 89, 77, 66, - 79, 76, 32, 70, 79, 82, 32, 4, 24, 2, 80, 85, 43, 84, 2, 11, 83, 2, 245, - 132, 10, 2, 72, 80, 2, 203, 217, 3, 65, 12, 68, 2, 83, 72, 230, 152, 6, - 67, 138, 198, 4, 70, 2, 76, 147, 17, 88, 4, 40, 4, 85, 65, 78, 71, 195, - 222, 10, 79, 2, 207, 239, 10, 88, 136, 2, 226, 1, 66, 20, 3, 71, 66, 89, - 40, 5, 76, 69, 45, 68, 69, 40, 3, 77, 73, 32, 154, 5, 78, 156, 26, 26, - 83, 83, 73, 65, 78, 32, 65, 83, 84, 82, 79, 76, 79, 71, 73, 67, 65, 76, - 32, 83, 89, 77, 66, 79, 76, 32, 231, 142, 5, 80, 2, 255, 228, 8, 76, 2, - 189, 249, 8, 5, 32, 70, 79, 79, 84, 2, 11, 76, 2, 149, 245, 5, 2, 65, 89, - 62, 68, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 102, 78, 231, 193, 2, 68, - 8, 40, 4, 79, 78, 69, 32, 199, 159, 9, 84, 6, 34, 84, 134, 252, 8, 72, - 47, 81, 2, 179, 253, 8, 72, 36, 33, 6, 85, 77, 66, 69, 82, 32, 36, 76, 5, - 69, 73, 71, 72, 84, 38, 70, 92, 2, 78, 73, 22, 79, 18, 83, 83, 84, 4, - 202, 132, 3, 32, 223, 249, 7, 89, 8, 18, 73, 35, 79, 4, 130, 2, 86, 247, - 142, 9, 70, 4, 136, 2, 2, 85, 82, 207, 142, 9, 82, 4, 77, 2, 78, 69, 2, - 167, 1, 78, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 250, 130, 3, 32, - 151, 232, 7, 84, 10, 34, 72, 50, 87, 131, 172, 10, 69, 4, 32, 2, 82, 69, - 211, 142, 9, 73, 2, 39, 69, 4, 26, 79, 195, 142, 9, 69, 2, 231, 129, 3, - 32, 182, 1, 32, 3, 73, 67, 32, 147, 25, 78, 178, 1, 220, 1, 6, 66, 69, - 76, 71, 84, 72, 20, 4, 67, 82, 79, 83, 20, 7, 76, 69, 84, 84, 69, 82, 32, - 210, 22, 83, 24, 6, 77, 85, 76, 84, 73, 80, 208, 181, 7, 5, 65, 82, 76, - 65, 85, 181, 201, 1, 7, 84, 86, 73, 77, 65, 68, 85, 2, 147, 150, 9, 79, - 2, 227, 182, 7, 83, 166, 1, 202, 4, 65, 110, 67, 98, 68, 126, 69, 82, 70, - 222, 1, 71, 104, 2, 72, 65, 50, 73, 220, 1, 5, 74, 69, 82, 65, 78, 34, - 75, 58, 76, 234, 1, 79, 128, 1, 13, 82, 65, 73, 68, 79, 32, 82, 65, 68, - 32, 82, 69, 73, 34, 83, 176, 2, 16, 66, 69, 82, 75, 65, 78, 65, 78, 32, - 66, 69, 79, 82, 67, 32, 66, 144, 1, 12, 78, 65, 85, 68, 73, 90, 32, 78, - 89, 68, 32, 78, 110, 84, 194, 1, 87, 176, 87, 7, 85, 82, 85, 90, 32, 85, - 82, 192, 188, 2, 10, 77, 65, 78, 78, 65, 90, 32, 77, 65, 78, 180, 63, 13, - 80, 69, 82, 84, 72, 79, 32, 80, 69, 79, 82, 84, 72, 142, 189, 3, 89, 202, - 210, 3, 81, 2, 86, 2, 88, 3, 90, 8, 222, 4, 69, 174, 175, 6, 67, 0, 4, - 78, 83, 85, 90, 253, 179, 3, 9, 76, 71, 73, 90, 32, 69, 79, 76, 72, 11, - 46, 69, 30, 65, 145, 138, 7, 3, 87, 69, 79, 4, 26, 65, 247, 242, 10, 78, - 2, 247, 202, 9, 76, 11, 84, 6, 79, 84, 84, 69, 68, 45, 249, 226, 3, 9, - 65, 71, 65, 90, 32, 68, 65, 69, 71, 6, 134, 242, 10, 76, 2, 78, 3, 80, - 11, 240, 74, 7, 72, 87, 65, 90, 32, 69, 72, 150, 238, 9, 65, 206, 55, 84, - 63, 78, 12, 120, 13, 82, 65, 78, 75, 83, 32, 67, 65, 83, 75, 69, 84, 32, - 137, 165, 7, 11, 69, 72, 85, 32, 70, 69, 79, 72, 32, 70, 69, 10, 46, 65, - 142, 178, 10, 73, 2, 79, 207, 60, 69, 4, 26, 69, 207, 239, 10, 67, 2, - 207, 199, 9, 83, 9, 26, 69, 195, 182, 10, 65, 4, 52, 7, 66, 79, 32, 71, - 89, 70, 85, 231, 238, 10, 82, 2, 143, 238, 10, 32, 4, 236, 8, 2, 69, 71, - 13, 4, 71, 76, 65, 90, 12, 156, 1, 2, 78, 71, 20, 9, 83, 65, 90, 32, 73, - 83, 32, 73, 83, 20, 5, 87, 65, 90, 32, 69, 252, 179, 10, 10, 67, 69, 76, - 65, 78, 68, 73, 67, 45, 89, 3, 79, 5, 199, 201, 6, 87, 2, 219, 172, 10, - 83, 2, 199, 235, 10, 79, 2, 11, 32, 2, 183, 236, 10, 74, 7, 21, 3, 65, - 85, 78, 4, 242, 232, 10, 32, 155, 3, 65, 12, 120, 15, 65, 85, 75, 65, 90, - 32, 76, 65, 71, 85, 32, 76, 79, 71, 82, 21, 11, 79, 78, 71, 45, 66, 82, - 65, 78, 67, 72, 45, 2, 187, 225, 9, 32, 10, 64, 3, 65, 82, 32, 158, 4, - 72, 62, 77, 66, 79, 167, 172, 10, 89, 2, 195, 211, 10, 65, 15, 150, 5, - 83, 0, 12, 84, 72, 65, 76, 65, 78, 32, 69, 84, 72, 69, 76, 224, 228, 10, - 4, 80, 69, 78, 45, 14, 69, 2, 78, 3, 79, 2, 11, 68, 2, 155, 176, 10, 32, - 26, 150, 1, 72, 244, 2, 18, 73, 71, 69, 76, 32, 76, 79, 78, 71, 45, 66, - 82, 65, 78, 67, 72, 45, 83, 200, 239, 6, 5, 79, 87, 73, 76, 79, 147, 171, - 2, 84, 21, 45, 9, 79, 82, 84, 45, 84, 87, 73, 71, 45, 18, 102, 66, 58, - 72, 62, 77, 30, 78, 38, 79, 42, 83, 186, 1, 84, 128, 163, 6, 2, 65, 82, - 199, 135, 4, 89, 2, 33, 6, 74, 65, 82, 75, 65, 78, 2, 171, 170, 9, 32, 2, - 25, 4, 65, 71, 65, 76, 2, 11, 76, 2, 195, 228, 10, 32, 2, 169, 150, 3, 2, - 65, 68, 2, 193, 149, 10, 4, 65, 85, 68, 32, 2, 17, 2, 83, 83, 2, 247, - 197, 10, 32, 2, 11, 79, 2, 187, 239, 6, 76, 4, 116, 15, 72, 85, 82, 73, - 83, 65, 90, 32, 84, 72, 85, 82, 83, 32, 84, 33, 10, 73, 87, 65, 90, 32, - 84, 73, 82, 32, 84, 2, 11, 72, 2, 155, 217, 5, 79, 2, 17, 2, 89, 82, 2, - 223, 198, 10, 32, 5, 41, 8, 85, 78, 74, 79, 32, 87, 89, 78, 2, 11, 78, 2, - 159, 228, 9, 32, 2, 21, 3, 73, 78, 71, 2, 229, 159, 7, 2, 76, 69, 4, 200, - 249, 8, 16, 73, 78, 71, 32, 83, 72, 73, 82, 84, 32, 87, 73, 84, 72, 32, - 83, 211, 175, 1, 69, 12, 120, 3, 66, 73, 78, 2, 78, 28, 2, 81, 85, 0, 3, - 86, 73, 71, 166, 145, 7, 83, 229, 169, 1, 6, 84, 82, 69, 68, 69, 67, 2, - 161, 187, 8, 2, 79, 86, 2, 249, 186, 8, 2, 73, 78, 250, 39, 244, 1, 2, - 32, 73, 22, 65, 238, 25, 67, 138, 5, 69, 166, 11, 72, 190, 33, 73, 198, - 232, 1, 75, 154, 2, 76, 142, 9, 77, 166, 21, 78, 174, 2, 79, 158, 39, 80, - 172, 12, 2, 81, 85, 146, 70, 83, 38, 84, 250, 16, 85, 166, 43, 87, 186, - 1, 89, 251, 166, 5, 71, 2, 239, 184, 9, 78, 196, 2, 140, 2, 5, 70, 69, - 84, 89, 32, 36, 4, 71, 73, 84, 84, 30, 76, 112, 8, 77, 65, 82, 73, 84, - 65, 78, 32, 142, 14, 78, 190, 3, 84, 100, 2, 85, 82, 244, 239, 2, 2, 73, - 76, 234, 91, 88, 144, 134, 6, 15, 75, 69, 32, 66, 79, 84, 84, 76, 69, 32, - 65, 78, 68, 32, 67, 159, 98, 82, 4, 138, 254, 1, 86, 223, 221, 7, 80, 2, - 245, 208, 3, 2, 65, 82, 6, 18, 84, 75, 85, 4, 36, 3, 32, 83, 72, 139, - 186, 9, 73, 2, 11, 65, 2, 199, 156, 10, 75, 2, 255, 209, 9, 84, 122, 184, - 1, 7, 76, 69, 84, 84, 69, 82, 32, 198, 3, 77, 248, 2, 12, 80, 85, 78, 67, - 84, 85, 65, 84, 73, 79, 78, 32, 132, 4, 11, 86, 79, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 203, 155, 4, 65, 44, 202, 1, 66, 32, 2, 68, 65, 22, 73, - 38, 75, 22, 76, 34, 83, 46, 84, 182, 2, 65, 196, 221, 5, 2, 71, 65, 222, - 148, 1, 82, 146, 141, 2, 90, 154, 81, 77, 172, 1, 2, 81, 85, 118, 78, - 226, 6, 89, 251, 101, 70, 4, 214, 186, 10, 73, 247, 25, 65, 2, 251, 194, - 9, 76, 6, 206, 214, 10, 78, 2, 84, 3, 89, 2, 251, 212, 9, 65, 2, 11, 65, - 2, 163, 194, 9, 66, 4, 152, 7, 3, 73, 78, 71, 223, 132, 9, 72, 6, 154, - 212, 9, 65, 134, 101, 73, 229, 10, 5, 83, 65, 65, 68, 73, 18, 96, 4, 65, - 82, 75, 32, 165, 1, 15, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, - 69, 82, 32, 12, 82, 68, 40, 2, 73, 78, 90, 69, 242, 2, 78, 197, 176, 8, - 5, 79, 67, 67, 76, 85, 2, 17, 2, 65, 71, 2, 159, 235, 8, 69, 5, 17, 2, - 45, 65, 2, 231, 209, 9, 76, 6, 46, 69, 180, 6, 2, 83, 72, 163, 204, 10, - 73, 2, 129, 217, 9, 11, 80, 69, 78, 84, 72, 69, 84, 73, 67, 32, 89, 28, - 130, 1, 65, 154, 1, 66, 22, 78, 30, 83, 130, 1, 90, 174, 162, 3, 84, 136, - 135, 7, 9, 77, 69, 76, 79, 68, 73, 67, 32, 81, 3, 81, 10, 76, 3, 70, 83, - 65, 34, 78, 28, 2, 84, 77, 145, 171, 10, 4, 82, 75, 65, 65, 2, 11, 65, 2, - 179, 208, 10, 81, 4, 26, 78, 187, 196, 5, 71, 2, 11, 65, 2, 143, 171, 10, - 65, 2, 201, 138, 3, 2, 69, 81, 4, 64, 4, 72, 73, 89, 89, 41, 8, 79, 70, - 32, 77, 65, 83, 72, 70, 2, 17, 2, 65, 65, 2, 163, 220, 8, 76, 2, 243, - 186, 9, 65, 4, 34, 65, 217, 219, 8, 2, 73, 81, 2, 255, 204, 9, 69, 30, - 92, 5, 76, 79, 78, 71, 32, 54, 79, 66, 83, 186, 224, 4, 65, 134, 236, 5, - 69, 2, 73, 3, 85, 10, 170, 225, 4, 65, 134, 236, 5, 69, 2, 73, 3, 85, 7, - 41, 8, 86, 69, 82, 76, 79, 78, 71, 32, 4, 203, 224, 4, 65, 4, 26, 72, - 171, 209, 5, 85, 2, 253, 139, 6, 3, 79, 82, 84, 10, 68, 8, 83, 45, 83, - 69, 82, 73, 70, 32, 145, 169, 10, 3, 68, 87, 73, 8, 44, 6, 72, 69, 65, - 86, 89, 32, 139, 2, 73, 6, 48, 3, 68, 79, 85, 81, 5, 76, 79, 87, 32, 68, - 4, 11, 66, 4, 21, 3, 76, 69, 32, 4, 84, 6, 84, 85, 82, 78, 69, 68, 23, - 67, 2, 21, 3, 79, 85, 66, 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 33, 6, - 79, 77, 77, 65, 32, 81, 2, 205, 255, 8, 3, 85, 79, 84, 2, 157, 133, 10, - 10, 78, 84, 69, 82, 82, 79, 66, 65, 78, 71, 6, 48, 6, 69, 76, 76, 73, 84, - 69, 251, 189, 5, 85, 5, 25, 4, 32, 65, 78, 84, 2, 135, 145, 7, 69, 166, - 1, 52, 7, 65, 83, 72, 84, 82, 65, 32, 235, 187, 4, 79, 164, 1, 180, 1, 7, + 17, 2, 79, 87, 2, 129, 213, 8, 2, 69, 82, 6, 25, 4, 82, 79, 77, 32, 6, + 36, 3, 66, 65, 82, 187, 213, 8, 68, 5, 189, 1, 6, 32, 84, 79, 32, 66, 76, + 10, 56, 7, 72, 82, 79, 85, 71, 72, 32, 65, 3, 79, 32, 66, 6, 224, 224, 4, + 3, 83, 85, 80, 234, 207, 4, 71, 247, 149, 2, 88, 4, 26, 76, 139, 141, 11, + 65, 2, 153, 247, 9, 3, 65, 67, 75, 40, 140, 1, 6, 67, 79, 82, 78, 69, 82, + 26, 68, 98, 76, 86, 80, 30, 83, 38, 84, 138, 210, 8, 77, 38, 78, 122, 69, + 150, 153, 1, 72, 171, 165, 1, 86, 2, 209, 18, 2, 32, 68, 4, 11, 79, 4, + 40, 4, 84, 84, 69, 68, 203, 219, 7, 85, 2, 17, 2, 32, 83, 2, 163, 177, + 11, 84, 6, 26, 79, 231, 210, 8, 65, 4, 26, 87, 251, 194, 11, 79, 2, 157, + 201, 3, 2, 69, 82, 2, 193, 145, 9, 2, 76, 85, 6, 146, 211, 8, 77, 203, + 191, 2, 84, 10, 154, 16, 73, 171, 3, 65, 8, 58, 65, 204, 11, 5, 79, 84, + 84, 79, 77, 187, 199, 8, 76, 2, 145, 2, 2, 67, 75, 14, 48, 6, 79, 85, 66, + 76, 69, 32, 167, 225, 8, 65, 12, 40, 5, 65, 82, 82, 79, 87, 211, 18, 68, + 11, 26, 32, 143, 137, 9, 45, 6, 26, 87, 167, 215, 8, 70, 4, 25, 4, 73, + 84, 72, 32, 4, 186, 143, 11, 86, 79, 83, 4, 40, 4, 82, 79, 78, 84, 195, + 210, 8, 73, 2, 193, 209, 8, 14, 45, 84, 73, 76, 84, 69, 68, 32, 83, 72, + 65, 68, 79, 87, 30, 26, 65, 251, 213, 8, 69, 26, 48, 6, 82, 80, 79, 79, + 78, 32, 131, 248, 10, 78, 24, 80, 10, 87, 73, 84, 72, 32, 66, 65, 82, 66, + 32, 197, 152, 1, 4, 79, 86, 69, 82, 22, 44, 4, 68, 79, 87, 78, 173, 1, 2, + 85, 80, 10, 26, 32, 231, 219, 8, 87, 8, 76, 8, 65, 66, 79, 86, 69, 32, + 76, 69, 18, 66, 166, 211, 8, 70, 179, 5, 84, 2, 227, 2, 70, 2, 253, 222, + 4, 7, 69, 76, 79, 87, 32, 76, 79, 12, 26, 32, 187, 218, 8, 87, 10, 60, 6, + 65, 66, 79, 86, 69, 32, 154, 210, 8, 70, 179, 5, 84, 6, 22, 76, 155, 1, + 82, 4, 32, 2, 69, 70, 187, 221, 4, 79, 2, 213, 144, 2, 24, 84, 87, 65, + 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, + 65, 82, 66, 2, 21, 3, 73, 71, 72, 2, 105, 24, 84, 87, 65, 82, 68, 83, 32, + 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 2, + 11, 32, 2, 139, 241, 1, 68, 2, 141, 1, 2, 69, 70, 2, 201, 212, 8, 3, 80, + 69, 78, 4, 202, 216, 8, 65, 233, 206, 1, 3, 85, 83, 72, 4, 36, 3, 73, 71, + 72, 223, 228, 10, 79, 2, 11, 84, 2, 171, 1, 45, 6, 32, 2, 81, 85, 215, + 207, 8, 65, 4, 26, 73, 239, 207, 8, 65, 2, 229, 215, 8, 2, 71, 71, 54, + 38, 79, 60, 2, 82, 73, 227, 4, 87, 2, 11, 80, 2, 11, 32, 2, 253, 199, 8, + 4, 83, 72, 65, 68, 34, 40, 5, 65, 78, 71, 76, 69, 255, 3, 80, 30, 56, 8, + 45, 72, 69, 65, 68, 69, 68, 32, 251, 219, 8, 32, 28, 52, 5, 65, 82, 82, + 79, 87, 202, 212, 8, 68, 39, 80, 25, 11, 32, 22, 64, 8, 79, 86, 69, 82, + 32, 76, 69, 70, 22, 87, 135, 208, 8, 84, 2, 183, 207, 8, 84, 18, 25, 4, + 73, 84, 72, 32, 18, 128, 1, 7, 68, 79, 85, 66, 76, 69, 32, 36, 7, 76, 79, + 78, 71, 32, 84, 73, 230, 207, 8, 66, 158, 1, 77, 34, 78, 34, 86, 35, 72, + 4, 166, 138, 9, 72, 195, 247, 1, 86, 4, 11, 80, 4, 11, 32, 4, 18, 68, 35, + 85, 2, 181, 208, 8, 3, 79, 87, 78, 2, 151, 208, 8, 80, 4, 21, 3, 76, 69, + 32, 4, 138, 3, 68, 203, 145, 10, 65, 18, 11, 79, 18, 56, 8, 45, 72, 69, + 65, 68, 69, 68, 32, 175, 210, 8, 32, 16, 76, 6, 65, 82, 82, 79, 87, 32, + 213, 1, 8, 84, 82, 73, 80, 76, 69, 32, 68, 14, 44, 5, 87, 73, 84, 72, 32, + 179, 198, 8, 70, 12, 42, 84, 210, 198, 7, 68, 235, 183, 3, 86, 8, 26, 65, + 243, 209, 8, 82, 6, 17, 2, 73, 76, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, + 250, 197, 7, 68, 235, 183, 3, 86, 2, 249, 132, 1, 2, 65, 83, 8, 58, 65, + 21, 10, 72, 73, 84, 69, 32, 65, 82, 82, 79, 87, 2, 207, 206, 8, 86, 7, + 11, 32, 4, 216, 193, 2, 4, 70, 82, 79, 77, 219, 145, 6, 87, 19, 66, 32, + 136, 1, 6, 69, 68, 32, 80, 76, 65, 21, 3, 73, 78, 71, 12, 82, 66, 22, 80, + 212, 201, 4, 2, 73, 78, 26, 69, 154, 158, 3, 79, 195, 198, 2, 65, 2, 239, + 153, 11, 85, 2, 11, 79, 2, 135, 232, 10, 73, 2, 247, 245, 10, 78, 2, 209, + 199, 3, 2, 32, 66, 4, 24, 2, 70, 65, 75, 83, 2, 17, 2, 76, 76, 2, 21, 3, + 73, 78, 71, 2, 177, 231, 1, 2, 32, 68, 2, 189, 173, 3, 2, 79, 85, 8, 222, + 169, 11, 69, 2, 73, 2, 77, 3, 79, 126, 136, 2, 2, 67, 75, 20, 2, 76, 76, + 188, 2, 4, 77, 65, 78, 32, 194, 8, 79, 80, 2, 83, 69, 20, 6, 84, 65, 84, + 69, 68, 32, 152, 3, 3, 85, 78, 68, 170, 178, 3, 87, 252, 213, 3, 15, 65, + 83, 84, 69, 68, 32, 83, 87, 69, 69, 84, 32, 80, 79, 84, 185, 166, 2, 2, + 66, 79, 5, 251, 138, 11, 69, 10, 130, 1, 69, 64, 4, 32, 79, 70, 32, 101, + 21, 73, 78, 71, 32, 79, 78, 32, 84, 72, 69, 32, 70, 76, 79, 79, 82, 32, + 76, 65, 85, 71, 6, 60, 9, 68, 45, 85, 80, 32, 78, 69, 87, 83, 33, 2, 82, + 32, 2, 11, 80, 2, 175, 131, 2, 65, 4, 28, 3, 67, 79, 65, 23, 83, 2, 131, + 235, 9, 83, 2, 135, 95, 75, 2, 231, 211, 10, 72, 72, 140, 1, 6, 67, 69, + 78, 84, 85, 82, 22, 68, 100, 3, 81, 85, 73, 28, 8, 78, 85, 77, 69, 82, + 65, 76, 32, 182, 4, 83, 114, 85, 135, 235, 7, 65, 2, 159, 215, 5, 73, 6, + 98, 69, 232, 5, 5, 85, 80, 79, 78, 68, 61, 12, 73, 77, 73, 68, 73, 65, + 32, 83, 69, 88, 84, 85, 2, 229, 5, 3, 78, 65, 82, 48, 142, 1, 70, 136, 1, + 3, 79, 78, 69, 134, 1, 83, 66, 84, 232, 14, 10, 82, 69, 86, 69, 82, 83, + 69, 68, 32, 79, 150, 237, 2, 69, 163, 135, 7, 78, 14, 26, 73, 183, 168, + 10, 79, 12, 36, 3, 70, 84, 89, 255, 254, 2, 86, 7, 11, 32, 4, 194, 196, + 5, 84, 177, 221, 4, 5, 69, 65, 82, 76, 89, 11, 11, 32, 8, 50, 72, 41, 8, + 84, 72, 79, 85, 83, 65, 78, 68, 4, 185, 1, 6, 85, 78, 68, 82, 69, 68, 5, + 141, 130, 4, 2, 32, 67, 6, 32, 2, 73, 88, 159, 172, 9, 69, 5, 241, 159, + 10, 2, 32, 76, 10, 42, 69, 150, 253, 2, 87, 239, 174, 6, 72, 4, 11, 78, + 5, 11, 32, 2, 131, 194, 5, 84, 10, 46, 69, 189, 200, 7, 5, 73, 76, 73, + 81, 85, 8, 60, 2, 77, 85, 40, 5, 83, 84, 69, 82, 84, 21, 2, 88, 84, 2, + 17, 2, 78, 67, 2, 231, 199, 7, 73, 2, 207, 234, 7, 73, 4, 18, 65, 23, 85, + 2, 179, 234, 7, 78, 2, 151, 199, 7, 76, 4, 48, 6, 84, 32, 86, 69, 71, 69, + 219, 225, 9, 83, 2, 141, 197, 2, 2, 84, 65, 5, 179, 210, 9, 84, 10, 166, + 2, 70, 32, 11, 72, 69, 65, 86, 89, 32, 66, 76, 65, 67, 75, 52, 24, 76, + 73, 71, 72, 84, 32, 70, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 32, + 66, 76, 65, 67, 75, 0, 18, 87, 72, 73, 84, 69, 32, 70, 79, 85, 82, 32, + 80, 79, 73, 78, 84, 69, 68, 161, 53, 8, 67, 65, 80, 73, 84, 65, 76, 32, + 2, 29, 5, 76, 79, 82, 65, 76, 2, 201, 182, 9, 8, 32, 72, 69, 65, 82, 84, + 32, 66, 2, 213, 207, 9, 2, 32, 67, 16, 74, 32, 89, 14, 69, 68, 32, 83, + 89, 77, 66, 79, 76, 32, 70, 79, 82, 32, 4, 24, 2, 80, 85, 43, 84, 2, 11, + 83, 2, 209, 151, 10, 2, 72, 80, 2, 147, 222, 3, 65, 12, 68, 2, 83, 72, + 238, 162, 6, 67, 222, 206, 4, 70, 2, 76, 147, 17, 88, 4, 40, 4, 85, 65, + 78, 71, 159, 241, 10, 79, 2, 171, 130, 11, 88, 136, 2, 226, 1, 66, 20, 3, + 71, 66, 89, 40, 5, 76, 69, 45, 68, 69, 40, 3, 77, 73, 32, 154, 5, 78, + 156, 26, 26, 83, 83, 73, 65, 78, 32, 65, 83, 84, 82, 79, 76, 79, 71, 73, + 67, 65, 76, 32, 83, 89, 77, 66, 79, 76, 32, 203, 152, 5, 80, 2, 255, 243, + 8, 76, 2, 177, 137, 9, 5, 32, 70, 79, 79, 84, 2, 11, 76, 2, 165, 255, 5, + 2, 65, 89, 62, 68, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 102, 78, 171, + 198, 2, 68, 8, 40, 4, 79, 78, 69, 32, 187, 175, 9, 84, 6, 34, 84, 250, + 139, 9, 72, 47, 81, 2, 167, 141, 9, 72, 36, 33, 6, 85, 77, 66, 69, 82, + 32, 36, 76, 5, 69, 73, 71, 72, 84, 38, 70, 92, 2, 78, 73, 22, 79, 18, 83, + 83, 84, 4, 158, 137, 3, 32, 231, 135, 8, 89, 8, 18, 73, 35, 79, 4, 130, + 2, 86, 235, 158, 9, 70, 4, 136, 2, 2, 85, 82, 195, 158, 9, 82, 4, 77, 2, + 78, 69, 2, 167, 1, 78, 8, 40, 4, 69, 86, 69, 78, 1, 2, 73, 88, 4, 206, + 135, 3, 32, 159, 246, 7, 84, 10, 34, 72, 50, 87, 223, 190, 10, 69, 4, 32, + 2, 82, 69, 199, 158, 9, 73, 2, 39, 69, 4, 26, 79, 183, 158, 9, 69, 2, + 187, 134, 3, 32, 182, 1, 32, 3, 73, 67, 32, 147, 25, 78, 178, 1, 220, 1, + 6, 66, 69, 76, 71, 84, 72, 20, 4, 67, 82, 79, 83, 20, 7, 76, 69, 84, 84, + 69, 82, 32, 210, 22, 83, 24, 6, 77, 85, 76, 84, 73, 80, 216, 196, 7, 5, + 65, 82, 76, 65, 85, 161, 202, 1, 7, 84, 86, 73, 77, 65, 68, 85, 2, 135, + 166, 9, 79, 2, 235, 197, 7, 83, 166, 1, 202, 4, 65, 110, 67, 98, 68, 126, + 69, 82, 70, 222, 1, 71, 104, 2, 72, 65, 50, 73, 220, 1, 5, 74, 69, 82, + 65, 78, 34, 75, 58, 76, 234, 1, 79, 128, 1, 13, 82, 65, 73, 68, 79, 32, + 82, 65, 68, 32, 82, 69, 73, 34, 83, 176, 2, 16, 66, 69, 82, 75, 65, 78, + 65, 78, 32, 66, 69, 79, 82, 67, 32, 66, 144, 1, 12, 78, 65, 85, 68, 73, + 90, 32, 78, 89, 68, 32, 78, 110, 84, 194, 1, 87, 128, 91, 7, 85, 82, 85, + 90, 32, 85, 82, 196, 189, 2, 10, 77, 65, 78, 78, 65, 90, 32, 77, 65, 78, + 168, 63, 13, 80, 69, 82, 84, 72, 79, 32, 80, 69, 79, 82, 84, 72, 206, + 199, 3, 89, 158, 214, 3, 81, 2, 86, 2, 88, 3, 90, 8, 222, 4, 69, 174, + 185, 6, 67, 0, 4, 78, 83, 85, 90, 189, 186, 3, 9, 76, 71, 73, 90, 32, 69, + 79, 76, 72, 11, 46, 69, 30, 65, 245, 152, 7, 3, 87, 69, 79, 4, 26, 65, + 211, 133, 11, 78, 2, 191, 219, 9, 76, 11, 84, 6, 79, 84, 84, 69, 68, 45, + 193, 231, 3, 9, 65, 71, 65, 90, 32, 68, 65, 69, 71, 6, 226, 132, 11, 76, + 2, 78, 3, 80, 11, 136, 76, 7, 72, 87, 65, 90, 32, 69, 72, 218, 255, 9, + 65, 206, 55, 84, 63, 78, 12, 120, 13, 82, 65, 78, 75, 83, 32, 67, 65, 83, + 75, 69, 84, 32, 145, 180, 7, 11, 69, 72, 85, 32, 70, 69, 79, 72, 32, 70, + 69, 10, 46, 65, 234, 196, 10, 73, 2, 79, 207, 60, 69, 4, 26, 69, 171, + 130, 11, 67, 2, 151, 216, 9, 83, 9, 26, 69, 159, 201, 10, 65, 4, 52, 7, + 66, 79, 32, 71, 89, 70, 85, 195, 129, 11, 82, 2, 235, 128, 11, 32, 4, + 236, 8, 2, 69, 71, 13, 4, 71, 76, 65, 90, 12, 156, 1, 2, 78, 71, 20, 9, + 83, 65, 90, 32, 73, 83, 32, 73, 83, 20, 5, 87, 65, 90, 32, 69, 216, 198, + 10, 10, 67, 69, 76, 65, 78, 68, 73, 67, 45, 89, 3, 79, 5, 199, 211, 6, + 87, 2, 183, 191, 10, 83, 2, 163, 254, 10, 79, 2, 11, 32, 2, 147, 255, 10, + 74, 7, 21, 3, 65, 85, 78, 4, 206, 251, 10, 32, 155, 3, 65, 12, 120, 15, + 65, 85, 75, 65, 90, 32, 76, 65, 71, 85, 32, 76, 79, 71, 82, 21, 11, 79, + 78, 71, 45, 66, 82, 65, 78, 67, 72, 45, 2, 251, 241, 9, 32, 10, 64, 3, + 65, 82, 32, 158, 4, 72, 62, 77, 66, 79, 131, 191, 10, 89, 2, 159, 230, + 10, 65, 15, 150, 5, 83, 0, 12, 84, 72, 65, 76, 65, 78, 32, 69, 84, 72, + 69, 76, 188, 247, 10, 4, 80, 69, 78, 45, 14, 69, 2, 78, 3, 79, 2, 11, 68, + 2, 247, 194, 10, 32, 26, 150, 1, 72, 244, 2, 18, 73, 71, 69, 76, 32, 76, + 79, 78, 71, 45, 66, 82, 65, 78, 67, 72, 45, 83, 208, 253, 6, 5, 79, 87, + 73, 76, 79, 203, 173, 2, 84, 21, 45, 9, 79, 82, 84, 45, 84, 87, 73, 71, + 45, 18, 102, 66, 58, 72, 62, 77, 30, 78, 38, 79, 42, 83, 186, 1, 84, 128, + 173, 6, 2, 65, 82, 163, 144, 4, 89, 2, 33, 6, 74, 65, 82, 75, 65, 78, 2, + 243, 186, 9, 32, 2, 25, 4, 65, 71, 65, 76, 2, 11, 76, 2, 159, 247, 10, + 32, 2, 253, 154, 3, 2, 65, 68, 2, 157, 168, 10, 4, 65, 85, 68, 32, 2, 17, + 2, 83, 83, 2, 211, 216, 10, 32, 2, 11, 79, 2, 195, 253, 6, 76, 4, 116, + 15, 72, 85, 82, 73, 83, 65, 90, 32, 84, 72, 85, 82, 83, 32, 84, 33, 10, + 73, 87, 65, 90, 32, 84, 73, 82, 32, 84, 2, 11, 72, 2, 171, 227, 5, 79, 2, + 17, 2, 89, 82, 2, 187, 217, 10, 32, 5, 41, 8, 85, 78, 74, 79, 32, 87, 89, + 78, 2, 11, 78, 2, 251, 246, 9, 32, 2, 21, 3, 73, 78, 71, 2, 237, 174, 7, + 2, 76, 69, 4, 188, 137, 9, 16, 73, 78, 71, 32, 83, 72, 73, 82, 84, 32, + 87, 73, 84, 72, 32, 83, 187, 178, 1, 69, 12, 120, 3, 66, 73, 78, 2, 78, + 28, 2, 81, 85, 0, 3, 86, 73, 71, 174, 160, 7, 83, 229, 169, 1, 6, 84, 82, + 69, 68, 69, 67, 2, 169, 202, 8, 2, 79, 86, 2, 129, 202, 8, 2, 73, 78, + 200, 40, 244, 1, 2, 32, 73, 22, 65, 162, 26, 67, 138, 5, 69, 166, 11, 72, + 242, 36, 73, 162, 233, 1, 75, 154, 2, 76, 142, 9, 77, 166, 21, 78, 190, + 2, 79, 158, 39, 80, 172, 12, 2, 81, 85, 138, 70, 83, 38, 84, 138, 17, 85, + 150, 43, 87, 186, 1, 89, 247, 178, 5, 71, 2, 183, 201, 9, 78, 198, 2, + 132, 2, 5, 70, 69, 84, 89, 32, 36, 4, 71, 73, 84, 84, 30, 76, 112, 8, 77, + 65, 82, 73, 84, 65, 78, 32, 146, 14, 78, 190, 3, 84, 102, 85, 204, 244, + 2, 2, 73, 76, 242, 91, 88, 144, 148, 6, 15, 75, 69, 32, 66, 79, 84, 84, + 76, 69, 32, 65, 78, 68, 32, 67, 159, 98, 82, 4, 254, 251, 6, 86, 207, + 242, 2, 80, 2, 201, 213, 3, 2, 65, 82, 6, 18, 84, 75, 85, 4, 36, 3, 32, + 83, 72, 235, 202, 9, 73, 2, 11, 65, 2, 171, 175, 10, 75, 2, 227, 228, 9, + 84, 122, 184, 1, 7, 76, 69, 84, 84, 69, 82, 32, 198, 3, 77, 248, 2, 12, + 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 136, 4, 11, 86, 79, 87, + 69, 76, 32, 83, 73, 71, 78, 32, 203, 159, 4, 65, 44, 202, 1, 66, 32, 2, + 68, 65, 22, 73, 38, 75, 22, 76, 34, 83, 46, 84, 182, 2, 65, 212, 231, 5, + 2, 71, 65, 222, 153, 1, 82, 202, 142, 2, 90, 182, 83, 77, 172, 1, 2, 81, + 85, 118, 78, 226, 6, 89, 251, 101, 70, 4, 186, 205, 10, 73, 247, 25, 65, + 2, 151, 224, 9, 76, 6, 178, 233, 10, 78, 2, 84, 3, 89, 2, 223, 231, 9, + 65, 2, 11, 65, 2, 191, 223, 9, 66, 4, 156, 7, 3, 73, 78, 71, 163, 149, 9, + 72, 6, 254, 230, 9, 65, 134, 101, 73, 229, 10, 5, 83, 65, 65, 68, 73, 18, + 96, 4, 65, 82, 75, 32, 165, 1, 15, 79, 68, 73, 70, 73, 69, 82, 32, 76, + 69, 84, 84, 69, 82, 32, 12, 82, 68, 40, 2, 73, 78, 90, 69, 242, 2, 78, + 213, 191, 8, 5, 79, 67, 67, 76, 85, 2, 17, 2, 65, 71, 2, 155, 251, 8, 69, + 5, 17, 2, 45, 65, 2, 203, 228, 9, 76, 6, 46, 69, 184, 6, 2, 83, 72, 131, + 223, 10, 73, 2, 229, 235, 9, 11, 80, 69, 78, 84, 72, 69, 84, 73, 67, 32, + 89, 28, 130, 1, 65, 154, 1, 66, 22, 78, 30, 83, 134, 1, 90, 250, 166, 3, + 84, 156, 149, 7, 9, 77, 69, 76, 79, 68, 73, 67, 32, 81, 3, 81, 10, 76, 3, + 70, 83, 65, 34, 78, 28, 2, 84, 77, 245, 189, 10, 4, 82, 75, 65, 65, 2, + 11, 65, 2, 151, 227, 10, 81, 4, 26, 78, 211, 206, 5, 71, 2, 11, 65, 2, + 243, 189, 10, 65, 2, 165, 143, 3, 2, 69, 81, 4, 64, 4, 72, 73, 89, 89, + 45, 8, 79, 70, 32, 77, 65, 83, 72, 70, 2, 11, 65, 2, 11, 65, 2, 155, 236, + 8, 76, 2, 139, 216, 9, 65, 4, 34, 65, 209, 235, 8, 2, 73, 81, 2, 223, + 223, 9, 69, 30, 92, 5, 76, 79, 78, 71, 32, 54, 79, 66, 83, 174, 205, 6, + 65, 242, 145, 4, 69, 2, 73, 3, 85, 10, 158, 206, 6, 65, 242, 145, 4, 69, + 2, 73, 3, 85, 7, 41, 8, 86, 69, 82, 76, 79, 78, 71, 32, 4, 191, 205, 6, + 65, 4, 26, 72, 183, 219, 5, 85, 2, 129, 150, 6, 3, 79, 82, 84, 10, 68, 8, + 83, 45, 83, 69, 82, 73, 70, 32, 241, 187, 10, 3, 68, 87, 73, 8, 44, 6, + 72, 69, 65, 86, 89, 32, 139, 2, 73, 6, 48, 3, 68, 79, 85, 81, 5, 76, 79, + 87, 32, 68, 4, 11, 66, 4, 21, 3, 76, 69, 32, 4, 84, 6, 84, 85, 82, 78, + 69, 68, 23, 67, 2, 21, 3, 79, 85, 66, 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, + 2, 33, 6, 79, 77, 77, 65, 32, 81, 2, 145, 144, 9, 3, 85, 79, 84, 2, 253, + 151, 10, 10, 78, 84, 69, 82, 82, 79, 66, 65, 78, 71, 6, 48, 6, 69, 76, + 76, 73, 84, 69, 143, 200, 5, 85, 5, 25, 4, 32, 65, 78, 84, 2, 147, 160, + 7, 69, 168, 1, 50, 82, 225, 141, 5, 6, 68, 73, 32, 82, 73, 89, 166, 1, + 52, 7, 65, 83, 72, 84, 82, 65, 32, 183, 191, 4, 79, 164, 1, 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 212, 1, 5, 83, 73, 71, 78, 32, 194, 21, 68, - 156, 246, 2, 16, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, - 32, 72, 139, 150, 2, 86, 100, 198, 235, 6, 65, 38, 68, 114, 84, 46, 86, - 186, 5, 85, 202, 141, 1, 79, 134, 60, 73, 42, 76, 250, 192, 1, 78, 46, + 176, 250, 2, 16, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, + 32, 72, 211, 155, 2, 86, 100, 154, 250, 6, 65, 38, 68, 114, 84, 46, 86, + 186, 5, 85, 206, 141, 1, 79, 238, 60, 73, 42, 76, 226, 195, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, - 2, 77, 2, 82, 3, 89, 8, 210, 175, 6, 67, 202, 207, 3, 65, 239, 1, 86, 52, - 66, 65, 32, 4, 72, 79, 79, 76, 46, 79, 74, 82, 143, 194, 10, 73, 4, 166, - 175, 9, 76, 215, 18, 82, 5, 153, 182, 5, 6, 32, 83, 65, 84, 67, 72, 6, - 36, 3, 82, 80, 73, 183, 138, 9, 79, 4, 202, 242, 9, 79, 135, 18, 85, 36, - 66, 69, 72, 4, 73, 80, 84, 32, 198, 246, 1, 85, 255, 194, 6, 79, 4, 36, - 3, 87, 68, 82, 195, 241, 9, 69, 2, 11, 73, 2, 167, 131, 10, 86, 28, 80, + 2, 77, 2, 82, 3, 89, 8, 218, 184, 6, 67, 234, 216, 3, 65, 239, 1, 86, 52, + 66, 65, 32, 4, 72, 79, 79, 76, 46, 79, 74, 82, 183, 212, 10, 73, 4, 194, + 191, 9, 76, 227, 20, 82, 5, 245, 191, 5, 6, 32, 83, 65, 84, 67, 72, 6, + 36, 3, 82, 80, 73, 203, 154, 9, 79, 4, 242, 132, 10, 79, 135, 18, 85, 36, + 66, 69, 72, 4, 73, 80, 84, 32, 214, 250, 1, 85, 175, 206, 6, 79, 4, 36, + 3, 87, 68, 82, 235, 131, 10, 69, 2, 11, 73, 2, 207, 149, 10, 86, 28, 80, 8, 67, 65, 80, 73, 84, 65, 76, 32, 86, 76, 81, 6, 83, 77, 65, 76, 76, 32, - 18, 170, 192, 10, 66, 2, 69, 2, 70, 2, 72, 2, 73, 2, 76, 2, 77, 2, 80, 3, - 82, 2, 37, 7, 73, 71, 65, 84, 85, 82, 69, 2, 11, 32, 2, 193, 251, 9, 2, - 69, 84, 8, 134, 191, 10, 69, 2, 71, 2, 76, 3, 79, 220, 1, 130, 2, 65, 30, + 18, 210, 210, 10, 66, 2, 69, 2, 70, 2, 72, 2, 73, 2, 76, 2, 77, 2, 80, 3, + 82, 2, 37, 7, 73, 71, 65, 84, 85, 82, 69, 2, 11, 32, 2, 233, 141, 10, 2, + 69, 84, 8, 174, 209, 10, 69, 2, 71, 2, 76, 3, 79, 220, 1, 130, 2, 65, 30, 67, 74, 69, 36, 5, 71, 77, 69, 78, 84, 28, 2, 77, 73, 172, 1, 8, 80, 65, - 82, 65, 84, 69, 68, 32, 138, 4, 82, 158, 1, 83, 68, 2, 84, 32, 140, 131, - 5, 8, 87, 73, 78, 71, 32, 78, 69, 69, 146, 140, 3, 88, 254, 104, 68, 213, - 160, 1, 2, 76, 70, 4, 210, 188, 10, 76, 3, 84, 6, 34, 84, 133, 217, 5, 2, - 79, 78, 4, 222, 187, 6, 73, 219, 199, 3, 79, 4, 130, 220, 1, 68, 195, - 132, 1, 45, 23, 221, 187, 4, 2, 69, 68, 6, 176, 80, 28, 68, 73, 82, 69, + 82, 65, 84, 69, 68, 32, 138, 4, 82, 158, 1, 83, 68, 2, 84, 32, 168, 140, + 5, 8, 87, 73, 78, 71, 32, 78, 69, 69, 202, 145, 3, 88, 190, 106, 68, 233, + 162, 1, 2, 76, 70, 4, 250, 206, 10, 76, 3, 84, 6, 34, 84, 245, 226, 5, 2, + 79, 78, 4, 178, 201, 6, 73, 175, 204, 3, 79, 4, 166, 224, 1, 68, 191, + 132, 1, 45, 23, 189, 191, 4, 2, 69, 68, 6, 212, 84, 28, 68, 73, 82, 69, 67, 84, 32, 80, 82, 79, 68, 85, 67, 84, 32, 87, 73, 84, 72, 32, 66, 79, - 84, 84, 79, 77, 32, 67, 252, 197, 7, 3, 83, 69, 88, 255, 212, 1, 67, 158, - 1, 48, 6, 66, 76, 79, 67, 75, 32, 191, 166, 9, 83, 156, 1, 88, 9, 81, 85, + 84, 84, 79, 77, 32, 67, 172, 208, 7, 3, 83, 69, 88, 211, 216, 1, 67, 158, + 1, 48, 6, 66, 76, 79, 67, 75, 32, 203, 182, 9, 83, 156, 1, 88, 9, 81, 85, 65, 68, 82, 65, 78, 84, 45, 129, 1, 8, 83, 69, 88, 84, 65, 78, 84, 45, - 30, 42, 49, 38, 50, 30, 51, 131, 184, 10, 52, 17, 34, 50, 30, 51, 131, - 184, 10, 52, 9, 26, 51, 131, 184, 10, 52, 5, 255, 183, 10, 52, 126, 58, - 49, 54, 50, 46, 51, 38, 52, 30, 53, 147, 182, 10, 54, 65, 50, 50, 46, 51, - 38, 52, 30, 53, 147, 182, 10, 54, 33, 42, 51, 38, 52, 30, 53, 147, 182, - 10, 54, 17, 34, 52, 30, 53, 147, 182, 10, 54, 9, 26, 53, 147, 182, 10, - 54, 5, 143, 182, 10, 54, 4, 120, 2, 86, 73, 241, 170, 2, 22, 73, 79, 85, + 30, 42, 49, 38, 50, 30, 51, 171, 202, 10, 52, 17, 34, 50, 30, 51, 171, + 202, 10, 52, 9, 26, 51, 171, 202, 10, 52, 5, 167, 202, 10, 52, 126, 58, + 49, 54, 50, 46, 51, 38, 52, 30, 53, 187, 200, 10, 54, 65, 50, 50, 46, 51, + 38, 52, 30, 53, 187, 200, 10, 54, 33, 42, 51, 38, 52, 30, 53, 187, 200, + 10, 54, 17, 34, 52, 30, 53, 187, 200, 10, 54, 9, 26, 53, 187, 200, 10, + 54, 5, 183, 200, 10, 54, 4, 120, 2, 86, 73, 129, 175, 2, 22, 73, 79, 85, 83, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 83, 89, 77, 66, 79, 76, - 83, 2, 11, 67, 2, 175, 243, 9, 69, 4, 52, 7, 81, 85, 73, 81, 85, 65, 68, - 219, 253, 8, 65, 2, 91, 82, 4, 64, 10, 84, 82, 65, 78, 83, 77, 73, 84, - 32, 83, 195, 173, 6, 77, 2, 11, 84, 2, 211, 236, 8, 65, 242, 2, 102, 65, - 246, 19, 73, 102, 79, 134, 11, 69, 70, 82, 136, 131, 9, 5, 85, 70, 70, - 76, 69, 167, 143, 1, 89, 176, 2, 164, 1, 12, 68, 79, 87, 69, 68, 32, 87, + 83, 2, 11, 67, 2, 215, 133, 10, 69, 4, 52, 7, 81, 85, 73, 81, 85, 65, 68, + 239, 141, 9, 65, 2, 91, 82, 4, 64, 10, 84, 82, 65, 78, 83, 77, 73, 84, + 32, 83, 151, 187, 6, 77, 2, 11, 84, 2, 223, 252, 8, 65, 134, 3, 102, 65, + 218, 20, 73, 102, 79, 214, 13, 69, 70, 82, 224, 143, 9, 5, 85, 70, 70, + 76, 69, 195, 145, 1, 89, 192, 2, 164, 1, 12, 68, 79, 87, 69, 68, 32, 87, 72, 73, 84, 69, 32, 56, 9, 76, 76, 79, 87, 32, 80, 65, 78, 32, 62, 82, - 210, 9, 86, 172, 184, 7, 2, 77, 82, 147, 230, 1, 75, 6, 162, 223, 8, 67, - 206, 11, 83, 237, 4, 3, 76, 65, 84, 2, 17, 2, 79, 70, 2, 17, 2, 32, 70, - 2, 231, 249, 6, 79, 194, 1, 40, 4, 65, 68, 65, 32, 255, 175, 10, 75, 192, + 182, 10, 86, 160, 198, 7, 2, 77, 82, 227, 233, 1, 75, 6, 174, 239, 8, 67, + 206, 11, 83, 245, 4, 3, 76, 65, 84, 2, 17, 2, 79, 70, 2, 17, 2, 32, 70, + 2, 187, 136, 7, 79, 210, 1, 40, 4, 65, 68, 65, 32, 167, 194, 10, 75, 208, 1, 162, 1, 68, 46, 69, 110, 72, 34, 76, 246, 1, 83, 212, 2, 6, 86, 79, - 87, 69, 76, 32, 250, 189, 4, 65, 228, 211, 1, 7, 67, 79, 78, 84, 73, 78, - 85, 163, 134, 4, 79, 24, 238, 209, 6, 79, 66, 65, 147, 235, 1, 73, 4, 84, - 15, 88, 84, 82, 65, 32, 83, 72, 79, 82, 84, 32, 86, 79, 87, 69, 215, 242, - 8, 75, 2, 139, 236, 9, 76, 2, 177, 253, 9, 3, 69, 65, 68, 96, 33, 6, 69, - 84, 84, 69, 82, 32, 96, 214, 210, 6, 65, 38, 68, 114, 84, 46, 86, 186, 5, - 85, 206, 201, 1, 73, 42, 76, 250, 192, 1, 78, 46, 83, 82, 66, 2, 67, 2, + 87, 69, 76, 32, 170, 199, 4, 65, 188, 211, 1, 7, 67, 79, 78, 84, 73, 78, + 85, 195, 143, 4, 79, 24, 194, 224, 6, 79, 66, 65, 255, 235, 1, 73, 4, 84, + 15, 88, 84, 82, 65, 32, 83, 72, 79, 82, 84, 32, 86, 79, 87, 69, 235, 130, + 9, 75, 2, 179, 254, 9, 76, 2, 217, 143, 10, 3, 69, 65, 68, 96, 33, 6, 69, + 84, 84, 69, 82, 32, 96, 170, 225, 6, 65, 38, 68, 114, 84, 46, 86, 186, 5, + 85, 186, 202, 1, 73, 42, 76, 226, 195, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, - 79, 30, 70, 65, 38, 69, 56, 4, 73, 71, 78, 32, 233, 164, 9, 3, 85, 84, - 82, 2, 153, 233, 9, 4, 78, 68, 72, 73, 6, 180, 201, 2, 5, 67, 84, 73, 79, - 78, 191, 179, 4, 80, 20, 106, 73, 190, 134, 5, 83, 158, 142, 1, 67, 98, - 78, 242, 60, 65, 174, 158, 1, 74, 150, 3, 85, 167, 242, 1, 86, 2, 37, 7, - 78, 86, 69, 82, 84, 69, 68, 2, 173, 148, 6, 2, 32, 67, 30, 60, 6, 77, 79, - 68, 73, 70, 73, 21, 5, 83, 73, 71, 78, 32, 2, 151, 137, 6, 69, 28, 82, - 80, 226, 211, 6, 65, 38, 85, 22, 86, 186, 201, 1, 73, 222, 137, 2, 69, 3, - 79, 2, 57, 12, 82, 73, 83, 72, 84, 72, 65, 77, 65, 84, 82, 65, 2, 155, - 144, 10, 32, 98, 72, 3, 69, 68, 32, 21, 11, 73, 65, 78, 32, 76, 69, 84, - 84, 69, 82, 32, 2, 147, 233, 9, 73, 96, 158, 2, 65, 120, 3, 67, 72, 85, - 22, 69, 70, 72, 46, 73, 46, 76, 22, 77, 38, 79, 94, 80, 18, 82, 22, 83, - 38, 84, 64, 2, 87, 79, 36, 2, 89, 69, 156, 235, 4, 2, 74, 85, 234, 222, - 2, 68, 202, 13, 90, 242, 87, 70, 182, 6, 71, 150, 45, 66, 246, 11, 75, - 218, 21, 86, 246, 25, 78, 167, 128, 1, 85, 16, 82, 82, 174, 234, 9, 73, - 234, 25, 68, 162, 8, 71, 2, 87, 198, 21, 83, 147, 1, 72, 4, 130, 151, 9, - 82, 135, 140, 1, 69, 2, 187, 128, 10, 82, 8, 38, 65, 206, 233, 9, 82, - 139, 56, 71, 4, 166, 162, 10, 82, 3, 84, 4, 200, 167, 8, 2, 65, 45, 203, - 169, 1, 85, 6, 254, 209, 9, 65, 142, 57, 67, 215, 22, 70, 2, 243, 153, 8, - 79, 4, 154, 149, 5, 69, 155, 244, 3, 73, 12, 70, 79, 130, 151, 9, 73, - 138, 109, 85, 150, 25, 65, 154, 3, 78, 3, 82, 2, 223, 137, 10, 90, 2, - 175, 13, 69, 2, 215, 254, 8, 79, 4, 194, 255, 8, 85, 187, 160, 1, 79, 6, - 26, 72, 147, 131, 10, 79, 4, 242, 250, 5, 73, 131, 147, 4, 69, 4, 226, - 149, 9, 79, 183, 137, 1, 69, 4, 242, 158, 10, 65, 3, 87, 10, 78, 69, 134, - 229, 3, 70, 252, 120, 6, 78, 84, 79, 32, 83, 72, 139, 192, 5, 80, 2, 231, - 215, 9, 76, 46, 252, 1, 2, 79, 84, 32, 6, 80, 80, 73, 78, 71, 32, 72, 2, - 82, 84, 176, 8, 7, 85, 76, 68, 69, 82, 69, 68, 220, 239, 1, 24, 67, 75, - 69, 68, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 69, 88, 80, 76, 79, - 68, 73, 78, 71, 158, 150, 3, 86, 251, 206, 4, 87, 2, 221, 133, 8, 3, 73, - 78, 71, 4, 36, 3, 84, 82, 79, 155, 233, 5, 66, 2, 11, 76, 2, 163, 129, 8, - 76, 32, 102, 32, 248, 5, 12, 72, 65, 78, 68, 32, 70, 79, 82, 77, 65, 84, - 32, 254, 171, 6, 67, 179, 232, 3, 83, 20, 154, 2, 66, 92, 11, 83, 76, 65, - 78, 84, 69, 68, 32, 78, 79, 82, 196, 1, 22, 82, 73, 71, 72, 84, 87, 65, - 82, 68, 83, 32, 65, 82, 82, 79, 87, 32, 65, 66, 79, 86, 69, 28, 7, 85, - 80, 32, 84, 65, 67, 75, 128, 1, 4, 76, 69, 70, 84, 149, 159, 2, 9, 68, - 79, 87, 78, 32, 84, 65, 67, 75, 4, 88, 14, 65, 67, 75, 83, 76, 65, 78, - 84, 69, 68, 32, 83, 79, 85, 33, 4, 69, 78, 84, 32, 2, 11, 84, 2, 203, - 252, 8, 72, 2, 245, 36, 37, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, - 73, 78, 71, 32, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, - 32, 78, 79, 82, 84, 72, 32, 69, 2, 229, 249, 5, 2, 32, 76, 7, 11, 32, 4, - 76, 13, 65, 66, 79, 86, 69, 32, 83, 72, 79, 82, 84, 32, 68, 159, 131, 2, - 87, 2, 11, 79, 2, 11, 87, 2, 11, 78, 2, 11, 32, 2, 191, 249, 6, 84, 8, + 79, 30, 70, 65, 38, 69, 56, 4, 73, 71, 78, 32, 145, 183, 9, 3, 85, 84, + 82, 2, 193, 251, 9, 4, 78, 68, 72, 73, 6, 212, 205, 2, 5, 67, 84, 73, 79, + 78, 243, 189, 4, 80, 20, 106, 73, 154, 144, 5, 83, 202, 141, 1, 67, 98, + 78, 190, 66, 65, 190, 158, 1, 74, 150, 3, 85, 235, 245, 1, 86, 2, 37, 7, + 78, 86, 69, 82, 84, 69, 68, 2, 181, 157, 6, 2, 32, 67, 46, 60, 6, 77, 79, + 68, 73, 70, 73, 21, 5, 83, 73, 71, 78, 32, 2, 155, 146, 6, 69, 44, 110, + 67, 42, 79, 34, 80, 162, 183, 4, 85, 194, 229, 1, 83, 242, 68, 65, 58, + 86, 166, 202, 1, 73, 199, 140, 2, 69, 4, 193, 157, 6, 5, 65, 78, 68, 82, + 65, 7, 186, 162, 10, 79, 215, 22, 69, 2, 57, 12, 82, 73, 83, 72, 84, 72, + 65, 77, 65, 84, 82, 65, 2, 223, 161, 10, 32, 98, 72, 3, 69, 68, 32, 21, + 11, 73, 65, 78, 32, 76, 69, 84, 84, 69, 82, 32, 2, 215, 250, 9, 73, 96, + 158, 2, 65, 120, 3, 67, 72, 85, 22, 69, 70, 72, 46, 73, 46, 76, 22, 77, + 38, 79, 94, 80, 18, 82, 22, 83, 38, 84, 64, 2, 87, 79, 36, 2, 89, 69, + 212, 243, 4, 2, 74, 85, 166, 228, 2, 68, 202, 13, 90, 218, 88, 70, 182, + 6, 71, 234, 45, 66, 246, 11, 75, 234, 21, 86, 250, 27, 78, 167, 128, 1, + 85, 16, 82, 82, 242, 251, 9, 73, 234, 25, 68, 162, 8, 71, 2, 87, 198, 21, + 83, 147, 1, 72, 4, 170, 166, 9, 82, 163, 142, 1, 69, 2, 255, 145, 10, 82, + 8, 38, 65, 146, 251, 9, 82, 139, 56, 71, 4, 234, 179, 10, 82, 3, 84, 4, + 164, 182, 8, 2, 65, 45, 179, 172, 1, 85, 6, 194, 227, 9, 65, 142, 57, 67, + 215, 22, 70, 2, 207, 168, 8, 79, 4, 146, 158, 5, 69, 227, 250, 3, 73, 12, + 70, 79, 170, 166, 9, 73, 166, 111, 85, 150, 25, 65, 154, 3, 78, 3, 82, 2, + 163, 155, 10, 90, 2, 255, 15, 69, 2, 151, 142, 9, 79, 4, 130, 143, 9, 85, + 191, 162, 1, 79, 6, 26, 72, 215, 148, 10, 79, 4, 218, 131, 6, 73, 223, + 155, 4, 69, 4, 138, 165, 9, 79, 211, 139, 1, 69, 4, 182, 176, 10, 65, 3, + 87, 10, 78, 69, 186, 232, 3, 70, 148, 126, 6, 78, 84, 79, 32, 83, 72, + 131, 201, 5, 80, 2, 171, 233, 9, 76, 50, 252, 1, 2, 79, 84, 32, 6, 80, + 80, 73, 78, 71, 32, 72, 2, 82, 84, 128, 11, 7, 85, 76, 68, 69, 82, 69, + 68, 184, 240, 1, 24, 67, 75, 69, 68, 32, 70, 65, 67, 69, 32, 87, 73, 84, + 72, 32, 69, 88, 80, 76, 79, 68, 73, 78, 71, 234, 155, 3, 86, 199, 215, 4, + 87, 2, 197, 147, 8, 3, 73, 78, 71, 4, 36, 3, 84, 82, 79, 131, 242, 5, 66, + 2, 11, 76, 2, 139, 143, 8, 76, 36, 102, 32, 200, 8, 12, 72, 65, 78, 68, + 32, 70, 79, 82, 77, 65, 84, 32, 250, 182, 6, 67, 171, 236, 3, 83, 24, + 242, 1, 66, 92, 11, 83, 76, 65, 78, 84, 69, 68, 32, 78, 79, 82, 196, 1, + 4, 76, 69, 70, 84, 156, 1, 11, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, + 32, 248, 1, 7, 85, 80, 32, 84, 65, 67, 75, 129, 161, 2, 9, 68, 79, 87, + 78, 32, 84, 65, 67, 75, 4, 88, 14, 65, 67, 75, 83, 76, 65, 78, 84, 69, + 68, 32, 83, 79, 85, 33, 4, 69, 78, 84, 32, 2, 11, 84, 2, 179, 140, 9, 72, + 2, 221, 40, 37, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, + 32, 68, 79, 87, 78, 87, 65, 82, 68, 83, 32, 84, 72, 69, 78, 32, 78, 79, + 82, 84, 72, 32, 69, 4, 116, 24, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, + 79, 79, 78, 32, 65, 66, 79, 86, 69, 32, 76, 79, 78, 71, 171, 3, 32, 2, + 237, 1, 5, 32, 82, 73, 71, 72, 4, 112, 11, 65, 82, 82, 79, 87, 32, 65, + 66, 79, 86, 69, 29, 13, 72, 65, 82, 80, 79, 79, 78, 32, 65, 66, 79, 86, + 69, 2, 157, 128, 6, 2, 32, 76, 2, 29, 5, 32, 76, 79, 78, 71, 2, 25, 4, + 32, 76, 69, 70, 2, 153, 250, 6, 6, 84, 87, 65, 82, 68, 83, 7, 11, 32, 4, + 76, 13, 65, 66, 79, 86, 69, 32, 83, 72, 79, 82, 84, 32, 68, 251, 131, 2, + 87, 2, 11, 79, 2, 11, 87, 2, 11, 78, 2, 11, 32, 2, 223, 132, 7, 84, 8, 120, 10, 67, 79, 78, 84, 73, 78, 85, 73, 78, 71, 0, 6, 76, 69, 84, 84, - 69, 82, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 205, 178, 3, 5, 32, 79, - 86, 69, 82, 2, 21, 3, 32, 83, 84, 2, 227, 146, 10, 69, 2, 25, 4, 32, 79, - 80, 69, 2, 247, 133, 9, 78, 4, 26, 73, 191, 145, 10, 85, 2, 131, 146, 10, - 77, 159, 14, 174, 1, 68, 168, 19, 2, 71, 78, 204, 181, 1, 6, 77, 73, 76, - 65, 82, 32, 158, 2, 78, 202, 24, 88, 241, 226, 6, 15, 76, 72, 79, 85, 69, - 84, 84, 69, 32, 79, 70, 32, 74, 65, 80, 200, 1, 64, 5, 68, 72, 65, 77, - 32, 133, 17, 6, 69, 87, 65, 89, 83, 32, 184, 1, 202, 1, 69, 68, 7, 76, - 69, 84, 84, 69, 82, 32, 176, 4, 15, 82, 69, 80, 69, 84, 73, 84, 73, 79, - 78, 32, 77, 65, 82, 75, 50, 83, 164, 8, 11, 86, 79, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 243, 163, 6, 68, 2, 37, 7, 78, 68, 32, 79, 70, 32, 84, 2, - 189, 195, 9, 2, 69, 88, 102, 214, 1, 65, 98, 84, 186, 177, 6, 68, 158, 1, - 86, 186, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, 83, 82, 66, 2, 67, - 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, - 2, 69, 3, 79, 11, 72, 8, 76, 84, 69, 82, 78, 65, 84, 69, 202, 139, 10, - 65, 2, 73, 3, 85, 2, 223, 230, 9, 32, 14, 134, 1, 72, 204, 225, 5, 19, - 87, 79, 45, 67, 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, - 69, 162, 225, 3, 84, 195, 71, 65, 4, 152, 202, 9, 20, 82, 69, 69, 45, 67, - 73, 82, 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, 69, 147, 64, 65, - 6, 11, 45, 6, 174, 137, 10, 49, 2, 50, 3, 51, 44, 38, 69, 181, 7, 4, 73, - 71, 78, 32, 32, 96, 11, 67, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, 177, - 6, 8, 80, 65, 82, 65, 84, 79, 82, 32, 28, 80, 11, 68, 79, 85, 66, 76, 69, - 32, 82, 73, 78, 71, 57, 5, 87, 73, 84, 72, 32, 5, 11, 32, 2, 129, 219, 8, - 6, 87, 73, 84, 72, 32, 82, 24, 212, 1, 12, 67, 73, 82, 67, 76, 69, 83, - 32, 65, 78, 68, 32, 116, 5, 81, 85, 65, 68, 82, 0, 4, 83, 69, 80, 84, 12, - 16, 82, 65, 89, 83, 32, 65, 78, 68, 32, 68, 79, 84, 84, 69, 68, 32, 46, - 68, 45, 3, 84, 82, 73, 6, 60, 4, 70, 79, 85, 82, 0, 3, 84, 87, 79, 195, - 216, 8, 82, 2, 169, 196, 6, 8, 32, 69, 78, 67, 76, 79, 83, 85, 2, 83, 85, - 6, 42, 68, 28, 3, 84, 82, 73, 215, 1, 67, 2, 197, 1, 3, 79, 85, 66, 2, - 171, 1, 80, 6, 52, 9, 68, 69, 78, 84, 32, 65, 78, 68, 32, 103, 80, 4, - 116, 6, 68, 79, 84, 84, 69, 68, 49, 14, 85, 45, 83, 72, 65, 80, 69, 68, - 32, 79, 82, 78, 65, 77, 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 25, 4, 82, - 69, 83, 67, 2, 251, 185, 1, 69, 4, 150, 224, 8, 66, 131, 81, 68, 12, 210, - 222, 4, 83, 158, 142, 1, 67, 98, 78, 234, 206, 3, 65, 239, 1, 86, 26, 74, - 65, 94, 86, 154, 172, 6, 85, 206, 201, 1, 73, 222, 137, 2, 69, 3, 79, 10, - 240, 172, 6, 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 85, 170, 211, 3, 65, - 2, 73, 3, 85, 4, 11, 79, 4, 33, 6, 67, 65, 76, 73, 67, 32, 4, 199, 172, - 6, 82, 16, 56, 5, 66, 76, 65, 67, 75, 1, 5, 87, 72, 73, 84, 69, 8, 11, - 32, 8, 70, 82, 24, 3, 76, 69, 70, 12, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, - 21, 3, 73, 71, 72, 2, 11, 84, 2, 225, 92, 6, 32, 80, 79, 73, 78, 84, 194, - 10, 96, 8, 87, 82, 73, 84, 73, 78, 71, 32, 133, 239, 1, 10, 32, 79, 70, - 32, 84, 72, 69, 32, 72, 79, 192, 10, 136, 3, 4, 65, 73, 82, 32, 192, 1, - 2, 66, 82, 102, 67, 138, 1, 68, 218, 4, 69, 242, 6, 70, 244, 3, 4, 87, - 65, 76, 76, 138, 1, 72, 234, 77, 76, 212, 6, 2, 77, 79, 222, 42, 78, 218, - 1, 82, 202, 7, 83, 162, 5, 84, 180, 10, 5, 71, 82, 65, 83, 80, 184, 5, - 30, 85, 80, 80, 69, 82, 32, 66, 79, 68, 89, 32, 84, 73, 76, 84, 73, 78, - 71, 32, 70, 82, 79, 77, 32, 72, 73, 80, 32, 74, 79, 195, 197, 4, 80, 8, - 48, 4, 66, 76, 79, 87, 29, 4, 83, 85, 67, 75, 4, 58, 32, 155, 221, 4, 73, - 4, 30, 32, 61, 3, 73, 78, 71, 2, 129, 158, 1, 10, 83, 77, 65, 76, 76, 32, - 82, 79, 84, 65, 2, 167, 248, 8, 32, 10, 52, 5, 69, 65, 84, 72, 32, 249, - 169, 1, 2, 85, 83, 4, 144, 164, 2, 2, 69, 88, 1, 2, 73, 78, 10, 40, 6, - 72, 69, 69, 75, 83, 32, 63, 79, 6, 154, 107, 83, 146, 38, 78, 225, 217, - 3, 4, 80, 85, 70, 70, 4, 174, 166, 9, 76, 223, 46, 77, 28, 108, 15, 82, - 69, 65, 77, 89, 32, 69, 89, 69, 66, 82, 79, 87, 83, 32, 165, 1, 7, 89, - 78, 65, 77, 73, 67, 32, 8, 64, 4, 68, 79, 87, 78, 0, 2, 85, 80, 29, 4, - 78, 69, 85, 84, 2, 153, 143, 1, 2, 32, 78, 4, 21, 3, 82, 65, 76, 4, 11, - 32, 4, 194, 58, 68, 187, 185, 9, 85, 20, 180, 1, 11, 69, 86, 69, 82, 89, - 32, 79, 84, 72, 69, 82, 30, 70, 22, 83, 128, 122, 9, 65, 82, 82, 79, 87, - 72, 69, 65, 68, 254, 38, 82, 188, 200, 3, 2, 84, 69, 21, 4, 71, 82, 65, - 68, 2, 181, 218, 8, 2, 32, 84, 2, 151, 136, 6, 65, 6, 68, 11, 73, 77, 85, - 76, 84, 65, 78, 69, 79, 85, 83, 151, 215, 8, 76, 5, 223, 151, 1, 32, 54, - 64, 2, 89, 69, 244, 221, 4, 4, 88, 67, 73, 84, 155, 132, 4, 65, 50, 166, - 1, 32, 56, 15, 66, 82, 79, 87, 83, 32, 83, 84, 82, 65, 73, 71, 72, 84, - 32, 44, 5, 71, 65, 90, 69, 45, 140, 2, 7, 76, 65, 83, 72, 69, 83, 32, 65, - 2, 83, 32, 6, 240, 150, 1, 5, 66, 76, 73, 78, 75, 175, 247, 4, 87, 6, - 186, 53, 68, 154, 84, 78, 163, 229, 8, 85, 18, 100, 11, 70, 76, 79, 79, - 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, - 32, 8, 82, 83, 207, 45, 67, 10, 18, 67, 43, 83, 4, 254, 45, 85, 213, 95, - 3, 73, 82, 67, 6, 37, 7, 84, 82, 65, 73, 71, 72, 84, 7, 11, 32, 4, 222, - 163, 1, 65, 55, 68, 6, 130, 51, 68, 252, 167, 4, 4, 70, 76, 85, 84, 191, - 145, 5, 85, 14, 112, 5, 72, 65, 76, 70, 32, 26, 67, 28, 4, 87, 73, 68, - 69, 230, 92, 79, 177, 130, 4, 6, 83, 81, 85, 69, 69, 90, 4, 22, 67, 131, - 93, 79, 2, 225, 232, 2, 2, 76, 79, 4, 162, 67, 32, 217, 61, 4, 78, 73, - 78, 71, 38, 204, 1, 28, 65, 67, 69, 32, 68, 73, 82, 69, 67, 84, 73, 79, - 78, 32, 80, 79, 83, 73, 84, 73, 79, 78, 32, 78, 79, 83, 69, 32, 138, 1, - 73, 82, 76, 176, 1, 8, 79, 82, 69, 72, 69, 65, 68, 32, 135, 148, 7, 85, - 6, 88, 10, 85, 80, 32, 79, 82, 32, 68, 79, 87, 78, 13, 8, 70, 79, 82, 87, - 65, 82, 68, 32, 5, 11, 32, 2, 225, 225, 4, 3, 84, 73, 76, 12, 240, 225, - 3, 11, 76, 76, 32, 77, 79, 68, 73, 70, 73, 69, 82, 151, 184, 2, 78, 12, - 44, 4, 73, 67, 75, 32, 29, 3, 79, 79, 82, 10, 146, 141, 1, 76, 35, 83, 2, - 221, 233, 8, 20, 80, 76, 65, 78, 69, 32, 83, 72, 79, 85, 76, 68, 69, 82, - 32, 72, 73, 80, 32, 77, 6, 166, 89, 87, 222, 38, 67, 47, 78, 156, 4, 34, - 65, 133, 75, 3, 69, 65, 68, 140, 4, 36, 3, 78, 68, 45, 139, 172, 9, 73, - 138, 4, 92, 5, 65, 78, 71, 76, 69, 138, 5, 67, 150, 10, 70, 186, 42, 72, - 241, 12, 4, 79, 86, 65, 76, 37, 11, 32, 34, 188, 1, 5, 73, 78, 68, 69, - 88, 176, 1, 7, 76, 73, 84, 84, 76, 69, 32, 136, 1, 5, 82, 73, 78, 71, 32, - 173, 66, 18, 77, 73, 68, 68, 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, 84, - 84, 76, 69, 17, 11, 32, 14, 128, 1, 7, 77, 73, 68, 68, 76, 69, 32, 228, - 24, 11, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 241, 42, 5, 84, 72, - 85, 77, 66, 4, 174, 70, 76, 243, 201, 8, 82, 8, 44, 5, 73, 78, 68, 69, - 88, 203, 224, 9, 85, 7, 145, 24, 18, 32, 84, 72, 85, 77, 66, 32, 73, 78, - 68, 69, 88, 32, 84, 72, 85, 77, 66, 4, 108, 22, 68, 79, 87, 78, 32, 77, - 73, 68, 68, 76, 69, 32, 84, 72, 85, 77, 66, 32, 73, 78, 68, 69, 155, 68, - 76, 2, 223, 157, 8, 88, 88, 64, 5, 73, 82, 67, 76, 69, 184, 3, 3, 76, 65, - 87, 183, 2, 85, 35, 11, 32, 32, 116, 5, 73, 78, 68, 69, 88, 200, 1, 7, - 76, 73, 84, 84, 76, 69, 32, 36, 7, 77, 73, 68, 68, 76, 69, 32, 163, 64, - 82, 21, 11, 32, 18, 72, 6, 77, 73, 68, 68, 76, 69, 198, 26, 72, 242, 38, - 82, 255, 215, 8, 66, 13, 11, 32, 10, 64, 5, 67, 82, 79, 83, 83, 198, 64, - 84, 82, 76, 243, 201, 8, 82, 4, 134, 65, 32, 227, 212, 8, 69, 4, 210, - 181, 8, 73, 139, 166, 1, 85, 6, 252, 47, 10, 82, 73, 78, 71, 32, 76, 73, - 84, 84, 76, 187, 171, 9, 85, 17, 11, 32, 14, 144, 2, 28, 77, 73, 68, 68, - 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 67, 79, 78, - 74, 79, 73, 78, 69, 68, 176, 49, 2, 70, 79, 198, 11, 78, 162, 1, 84, 217, - 118, 23, 73, 78, 68, 69, 88, 32, 84, 72, 85, 77, 66, 32, 67, 85, 82, 86, - 69, 32, 84, 72, 85, 77, 66, 5, 139, 181, 1, 32, 38, 46, 80, 149, 2, 6, - 82, 76, 73, 67, 85, 69, 31, 11, 32, 28, 188, 1, 5, 73, 78, 68, 69, 88, - 56, 19, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, - 69, 65, 68, 196, 50, 7, 77, 73, 68, 68, 76, 69, 32, 18, 79, 218, 7, 78, - 163, 1, 84, 9, 11, 32, 6, 40, 5, 84, 72, 85, 77, 66, 243, 58, 82, 5, 215, - 46, 32, 9, 11, 32, 6, 72, 5, 73, 78, 68, 69, 88, 0, 6, 77, 73, 68, 68, - 76, 69, 179, 71, 79, 2, 189, 133, 9, 13, 32, 82, 73, 78, 71, 32, 76, 73, - 84, 84, 76, 69, 32, 172, 2, 44, 3, 73, 83, 84, 177, 33, 3, 76, 65, 84, - 247, 1, 11, 32, 244, 1, 160, 2, 5, 73, 78, 68, 69, 88, 192, 16, 7, 76, - 73, 84, 84, 76, 69, 32, 196, 2, 7, 77, 73, 68, 68, 76, 69, 32, 200, 4, 5, - 82, 73, 78, 71, 32, 132, 2, 5, 84, 72, 85, 77, 66, 138, 2, 72, 205, 9, - 22, 70, 79, 85, 82, 32, 70, 73, 78, 71, 69, 82, 83, 32, 67, 79, 78, 74, - 79, 73, 78, 69, 68, 137, 1, 11, 32, 134, 1, 232, 1, 4, 66, 69, 78, 84, - 36, 6, 72, 73, 78, 71, 69, 68, 76, 6, 77, 73, 68, 68, 76, 69, 168, 7, 2, - 67, 85, 64, 6, 84, 72, 85, 77, 66, 32, 160, 5, 16, 85, 80, 32, 77, 73, - 68, 68, 76, 69, 32, 72, 73, 78, 71, 69, 68, 151, 4, 82, 5, 217, 45, 5, - 32, 79, 86, 69, 82, 9, 11, 32, 6, 252, 20, 8, 77, 73, 68, 68, 76, 69, 32, - 85, 167, 160, 8, 76, 71, 11, 32, 68, 236, 1, 4, 66, 69, 78, 84, 42, 67, - 244, 1, 10, 85, 80, 32, 83, 80, 82, 69, 65, 68, 32, 100, 6, 72, 73, 78, - 71, 69, 68, 46, 82, 80, 5, 84, 72, 85, 77, 66, 188, 28, 14, 83, 84, 82, - 65, 73, 71, 72, 84, 32, 84, 72, 85, 77, 66, 227, 17, 76, 5, 213, 92, 6, - 32, 84, 72, 85, 77, 66, 28, 68, 8, 79, 78, 74, 79, 73, 78, 69, 68, 233, - 1, 4, 82, 79, 83, 83, 23, 11, 32, 20, 144, 1, 6, 67, 85, 80, 80, 69, 68, - 28, 6, 84, 72, 85, 77, 66, 32, 68, 5, 72, 73, 78, 71, 69, 166, 22, 73, - 165, 7, 6, 77, 73, 68, 68, 76, 69, 5, 11, 32, 2, 203, 27, 84, 8, 160, 1, - 4, 83, 73, 68, 69, 219, 61, 70, 6, 22, 69, 159, 47, 32, 4, 223, 15, 68, - 5, 241, 30, 7, 32, 83, 80, 82, 69, 65, 68, 8, 32, 3, 73, 78, 71, 247, 19, - 65, 7, 11, 32, 4, 138, 36, 67, 255, 225, 8, 66, 19, 11, 32, 16, 64, 6, - 65, 78, 71, 76, 69, 68, 22, 67, 106, 72, 159, 132, 9, 66, 5, 247, 211, 5, - 32, 6, 74, 85, 230, 7, 73, 241, 25, 10, 79, 78, 74, 79, 73, 78, 69, 68, - 32, 72, 2, 145, 188, 4, 2, 80, 80, 4, 194, 33, 73, 217, 26, 2, 79, 79, - 40, 164, 1, 7, 65, 78, 71, 76, 69, 68, 32, 46, 67, 220, 1, 14, 70, 79, - 82, 87, 65, 82, 68, 32, 73, 78, 68, 69, 88, 32, 32, 4, 72, 79, 79, 75, - 53, 4, 83, 73, 68, 69, 4, 128, 1, 2, 73, 78, 1, 3, 79, 85, 84, 12, 36, 5, - 73, 82, 67, 76, 69, 15, 85, 5, 47, 68, 8, 32, 4, 80, 80, 69, 68, 39, 82, - 2, 225, 40, 5, 32, 77, 73, 68, 68, 6, 56, 9, 86, 69, 32, 84, 72, 85, 77, - 66, 32, 143, 37, 76, 4, 182, 160, 1, 73, 131, 188, 4, 85, 4, 202, 84, 83, - 255, 171, 8, 66, 7, 157, 8, 9, 69, 68, 32, 77, 73, 68, 68, 76, 69, 15, - 11, 32, 12, 92, 6, 73, 78, 68, 69, 88, 32, 156, 13, 5, 84, 72, 85, 77, - 66, 193, 8, 4, 66, 79, 84, 72, 4, 26, 72, 239, 254, 8, 66, 2, 231, 140, - 8, 73, 7, 37, 7, 32, 84, 72, 85, 77, 66, 32, 4, 178, 28, 67, 227, 129, 1, - 83, 22, 88, 4, 68, 79, 87, 78, 186, 1, 84, 158, 6, 82, 130, 21, 73, 226, - 224, 8, 66, 159, 67, 85, 9, 11, 32, 6, 80, 9, 79, 84, 72, 69, 82, 83, 32, - 67, 73, 25, 7, 82, 73, 80, 80, 76, 69, 32, 2, 225, 51, 2, 82, 67, 4, 22, - 67, 171, 80, 83, 2, 11, 85, 2, 241, 179, 4, 2, 82, 86, 4, 196, 35, 6, 79, - 85, 67, 72, 69, 83, 39, 72, 30, 134, 1, 82, 28, 6, 84, 72, 85, 77, 66, - 32, 138, 3, 85, 134, 1, 68, 210, 30, 76, 205, 244, 7, 9, 66, 69, 78, 84, - 32, 79, 86, 69, 82, 4, 238, 4, 65, 231, 29, 73, 16, 80, 7, 65, 78, 71, - 76, 69, 68, 32, 126, 67, 124, 4, 72, 79, 79, 75, 147, 32, 76, 6, 60, 10, - 79, 85, 84, 32, 73, 78, 68, 69, 88, 32, 215, 1, 73, 4, 26, 67, 151, 188, - 9, 85, 2, 133, 186, 2, 3, 82, 79, 83, 6, 76, 12, 73, 82, 67, 76, 69, 68, - 32, 73, 78, 68, 69, 88, 45, 3, 85, 80, 80, 4, 11, 32, 4, 150, 21, 72, - 131, 166, 9, 85, 2, 25, 4, 69, 68, 32, 73, 2, 233, 30, 4, 78, 68, 69, 88, - 4, 11, 80, 5, 175, 15, 32, 18, 102, 68, 20, 6, 77, 73, 68, 68, 76, 69, - 42, 82, 198, 29, 84, 82, 76, 226, 244, 7, 73, 139, 166, 1, 85, 2, 175, - 229, 7, 79, 7, 11, 32, 4, 206, 3, 82, 179, 16, 67, 2, 11, 65, 2, 37, 7, - 73, 83, 69, 68, 32, 75, 78, 2, 11, 85, 2, 11, 67, 2, 147, 148, 8, 75, 35, - 11, 32, 32, 148, 1, 8, 66, 69, 84, 87, 69, 69, 78, 32, 102, 72, 20, 5, - 79, 86, 69, 82, 32, 120, 4, 83, 73, 68, 69, 100, 6, 85, 78, 68, 69, 82, - 32, 207, 40, 70, 8, 80, 12, 73, 78, 68, 69, 88, 32, 77, 73, 68, 68, 76, - 69, 246, 20, 77, 155, 6, 82, 5, 135, 70, 32, 2, 187, 169, 4, 69, 4, 52, - 6, 70, 79, 85, 82, 32, 82, 161, 2, 2, 84, 87, 2, 11, 65, 2, 233, 80, 9, - 73, 83, 69, 68, 32, 75, 78, 85, 67, 6, 11, 32, 6, 38, 68, 190, 15, 67, - 255, 225, 8, 66, 2, 25, 4, 73, 65, 71, 79, 2, 131, 171, 8, 78, 10, 54, - 73, 34, 84, 48, 4, 70, 79, 85, 82, 131, 23, 76, 2, 161, 7, 4, 78, 68, 69, - 88, 4, 34, 87, 13, 4, 72, 82, 69, 69, 2, 11, 79, 2, 197, 164, 8, 5, 32, - 70, 73, 78, 71, 55, 11, 32, 52, 194, 1, 70, 172, 3, 4, 72, 69, 69, 76, - 192, 1, 6, 83, 80, 76, 73, 84, 32, 236, 1, 6, 84, 72, 85, 77, 66, 32, - 165, 244, 4, 16, 66, 69, 84, 87, 69, 69, 78, 32, 80, 65, 76, 77, 32, 70, - 65, 67, 24, 140, 1, 18, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, - 83, 80, 82, 69, 65, 68, 165, 1, 11, 79, 85, 82, 32, 70, 73, 78, 71, 69, - 82, 83, 15, 11, 32, 12, 68, 6, 72, 73, 78, 71, 69, 68, 42, 84, 186, 2, - 70, 199, 233, 8, 66, 7, 11, 32, 4, 190, 4, 84, 155, 15, 78, 2, 189, 35, - 6, 72, 85, 77, 66, 32, 70, 11, 11, 32, 8, 72, 9, 67, 79, 78, 74, 79, 73, - 78, 69, 68, 154, 8, 72, 231, 226, 8, 66, 5, 221, 145, 9, 3, 32, 83, 80, - 11, 11, 32, 8, 96, 19, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, - 32, 83, 80, 82, 69, 65, 68, 151, 2, 84, 7, 11, 32, 4, 26, 70, 199, 233, - 8, 66, 2, 21, 3, 79, 85, 82, 2, 167, 1, 32, 10, 72, 6, 67, 69, 78, 84, - 82, 69, 96, 5, 73, 78, 68, 69, 88, 167, 16, 76, 7, 49, 10, 32, 84, 72, - 85, 77, 66, 32, 83, 73, 68, 4, 11, 69, 5, 11, 32, 2, 131, 232, 8, 66, 2, - 11, 32, 2, 11, 84, 2, 173, 135, 1, 5, 72, 85, 77, 66, 32, 6, 242, 30, 70, - 162, 104, 83, 159, 224, 7, 66, 82, 48, 4, 73, 78, 71, 69, 181, 9, 3, 79, - 79, 75, 63, 11, 32, 60, 206, 1, 70, 172, 1, 5, 73, 78, 68, 69, 88, 196, - 2, 6, 76, 73, 84, 84, 76, 69, 80, 6, 77, 73, 68, 68, 76, 69, 30, 79, 64, - 4, 82, 73, 78, 71, 124, 6, 84, 72, 85, 77, 66, 32, 158, 6, 78, 139, 142, - 4, 83, 4, 92, 19, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, - 82, 69, 65, 68, 32, 19, 79, 2, 191, 25, 79, 2, 249, 1, 11, 85, 82, 32, - 70, 73, 78, 71, 69, 82, 83, 32, 23, 11, 32, 20, 86, 72, 40, 7, 77, 73, - 68, 68, 76, 69, 32, 104, 5, 84, 72, 85, 77, 66, 219, 9, 82, 2, 11, 73, 2, - 177, 154, 4, 2, 78, 71, 6, 36, 4, 82, 73, 78, 71, 203, 10, 76, 5, 11, 32, - 2, 11, 67, 2, 21, 3, 79, 78, 74, 2, 175, 33, 79, 11, 11, 32, 8, 34, 83, - 210, 22, 79, 203, 39, 76, 4, 190, 148, 7, 73, 195, 8, 77, 9, 11, 32, 6, - 22, 73, 199, 8, 84, 4, 25, 4, 78, 68, 69, 88, 5, 155, 8, 32, 5, 11, 32, - 2, 171, 8, 82, 8, 21, 3, 80, 69, 78, 9, 11, 32, 6, 178, 7, 78, 163, 1, - 84, 5, 97, 22, 32, 68, 79, 87, 78, 32, 73, 78, 68, 69, 88, 32, 84, 72, - 85, 77, 66, 32, 72, 79, 79, 75, 2, 177, 77, 2, 32, 77, 6, 68, 9, 66, 69, - 84, 87, 69, 69, 78, 32, 77, 53, 4, 83, 73, 68, 69, 2, 29, 5, 73, 68, 68, - 76, 69, 2, 163, 148, 4, 32, 5, 33, 6, 32, 84, 79, 85, 67, 72, 2, 185, - 250, 7, 3, 73, 78, 71, 21, 11, 32, 18, 172, 1, 4, 67, 85, 82, 76, 28, 18, - 73, 78, 68, 69, 88, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, - 48, 7, 77, 73, 68, 68, 76, 69, 32, 225, 2, 4, 82, 73, 78, 71, 2, 253, - 147, 5, 2, 73, 67, 6, 142, 183, 5, 85, 142, 238, 2, 79, 243, 41, 73, 8, - 104, 21, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 67, 79, 78, 74, - 79, 73, 78, 69, 68, 143, 2, 84, 7, 223, 228, 2, 32, 17, 11, 32, 14, 112, - 14, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, 32, 83, 22, 76, 66, - 78, 70, 82, 94, 84, 183, 244, 7, 73, 2, 215, 160, 2, 80, 2, 21, 3, 73, - 84, 84, 2, 17, 2, 76, 69, 2, 135, 166, 8, 32, 2, 11, 79, 2, 11, 32, 2, - 11, 84, 2, 11, 72, 2, 247, 251, 5, 85, 2, 11, 73, 2, 21, 3, 78, 71, 32, - 2, 11, 76, 2, 11, 73, 2, 11, 84, 2, 179, 246, 7, 84, 4, 29, 5, 72, 85, - 77, 66, 32, 4, 194, 14, 70, 163, 104, 83, 17, 11, 32, 14, 56, 8, 77, 79, - 86, 69, 77, 69, 78, 84, 247, 150, 8, 82, 12, 26, 45, 163, 199, 7, 32, 10, - 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 25, 10, 87, 65, 76, - 76, 80, 76, 65, 78, 69, 32, 4, 62, 67, 223, 40, 83, 6, 38, 67, 28, 2, 84, - 73, 195, 40, 83, 2, 169, 155, 8, 2, 85, 82, 2, 183, 251, 8, 76, 38, 50, - 73, 225, 3, 7, 79, 67, 65, 84, 73, 79, 78, 22, 32, 3, 77, 66, 32, 171, 1, - 80, 16, 56, 4, 67, 79, 77, 66, 29, 6, 76, 69, 78, 71, 84, 72, 2, 193, - 212, 5, 2, 73, 78, 14, 11, 45, 14, 146, 150, 9, 49, 2, 50, 2, 51, 2, 52, - 2, 53, 2, 54, 3, 55, 6, 58, 32, 153, 1, 9, 83, 32, 80, 82, 69, 83, 83, - 69, 68, 4, 116, 12, 76, 79, 87, 69, 82, 32, 79, 86, 69, 82, 32, 85, 133, - 251, 7, 11, 85, 80, 80, 69, 82, 32, 79, 86, 69, 82, 32, 2, 11, 80, 2, - 163, 214, 8, 80, 2, 29, 5, 32, 84, 79, 71, 69, 2, 11, 84, 2, 227, 213, 8, - 72, 16, 22, 32, 203, 1, 45, 12, 148, 1, 2, 72, 69, 236, 177, 2, 3, 84, - 79, 82, 144, 222, 4, 3, 68, 69, 80, 0, 3, 87, 73, 68, 229, 109, 10, 76, - 73, 77, 66, 83, 32, 68, 73, 71, 73, 4, 196, 13, 4, 65, 68, 32, 78, 131, - 218, 8, 73, 4, 52, 5, 70, 76, 79, 79, 82, 1, 4, 87, 65, 76, 76, 2, 221, - 210, 8, 5, 80, 76, 65, 78, 69, 156, 3, 64, 4, 85, 84, 72, 32, 161, 5, 7, - 86, 69, 77, 69, 78, 84, 45, 54, 186, 1, 67, 88, 5, 70, 82, 79, 87, 78, 0, - 5, 83, 77, 73, 76, 69, 56, 4, 75, 73, 83, 83, 36, 5, 79, 80, 69, 78, 32, - 192, 1, 5, 84, 69, 78, 83, 69, 165, 29, 5, 87, 82, 73, 78, 75, 8, 48, 6, - 76, 79, 83, 69, 68, 32, 163, 223, 7, 79, 6, 222, 2, 70, 142, 38, 67, 47, - 78, 7, 11, 32, 4, 22, 79, 203, 1, 87, 2, 147, 186, 7, 80, 7, 11, 32, 4, - 166, 1, 87, 83, 70, 18, 100, 4, 79, 86, 65, 76, 0, 9, 82, 69, 67, 84, 65, - 78, 71, 76, 69, 42, 87, 82, 70, 235, 185, 7, 67, 7, 11, 32, 4, 26, 87, - 151, 184, 7, 89, 2, 25, 4, 82, 73, 78, 75, 2, 203, 128, 4, 76, 7, 11, 32, - 4, 18, 70, 43, 83, 2, 17, 2, 79, 82, 2, 215, 244, 7, 87, 2, 17, 2, 85, - 67, 2, 219, 255, 3, 75, 230, 2, 192, 1, 9, 68, 73, 65, 71, 79, 78, 65, - 76, 32, 148, 1, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 184, 14, - 6, 72, 73, 78, 71, 69, 32, 189, 2, 10, 87, 65, 76, 76, 80, 76, 65, 78, - 69, 32, 32, 56, 8, 66, 69, 84, 87, 69, 69, 78, 32, 22, 65, 31, 84, 16, - 18, 65, 31, 84, 8, 229, 28, 3, 87, 65, 89, 8, 201, 28, 6, 79, 87, 65, 82, - 68, 83, 150, 1, 208, 2, 24, 65, 82, 77, 32, 67, 73, 82, 67, 76, 69, 32, - 72, 73, 84, 84, 73, 78, 71, 32, 87, 65, 76, 76, 32, 38, 66, 34, 67, 224, - 1, 8, 70, 73, 78, 71, 69, 82, 32, 67, 52, 5, 72, 85, 77, 80, 32, 184, 3, - 5, 76, 79, 79, 80, 32, 198, 1, 83, 108, 7, 84, 82, 73, 80, 76, 69, 32, - 110, 87, 206, 12, 68, 198, 2, 80, 154, 6, 90, 227, 232, 7, 74, 12, 178, - 7, 76, 154, 9, 77, 39, 83, 8, 150, 17, 79, 175, 230, 7, 69, 28, 86, 72, - 20, 5, 85, 82, 86, 69, 32, 196, 29, 5, 79, 82, 78, 69, 82, 227, 165, 7, - 82, 2, 211, 212, 7, 69, 18, 80, 4, 67, 79, 77, 66, 158, 8, 72, 230, 15, - 76, 202, 172, 2, 77, 183, 177, 1, 83, 2, 11, 73, 2, 131, 248, 3, 78, 6, - 128, 9, 6, 73, 82, 67, 76, 69, 83, 239, 20, 79, 18, 56, 8, 72, 73, 84, - 84, 73, 78, 71, 32, 239, 244, 3, 83, 16, 72, 8, 67, 69, 73, 76, 73, 78, - 71, 32, 101, 6, 70, 76, 79, 79, 82, 32, 8, 56, 5, 76, 65, 82, 71, 69, 1, - 5, 83, 77, 65, 76, 76, 4, 11, 32, 4, 226, 54, 84, 135, 2, 68, 8, 88, 4, - 83, 77, 65, 76, 12, 5, 76, 65, 82, 71, 69, 17, 7, 84, 82, 73, 80, 76, 69, - 32, 2, 11, 76, 2, 255, 18, 32, 4, 56, 5, 76, 65, 82, 71, 69, 1, 5, 83, - 77, 65, 76, 76, 2, 145, 53, 2, 32, 84, 18, 56, 8, 72, 73, 84, 84, 73, 78, - 71, 32, 183, 241, 3, 83, 16, 64, 7, 67, 69, 73, 76, 73, 78, 71, 1, 5, 70, - 76, 79, 79, 82, 8, 11, 32, 8, 22, 76, 191, 9, 83, 4, 249, 22, 4, 65, 82, - 71, 69, 12, 44, 6, 72, 65, 75, 73, 78, 71, 243, 16, 73, 2, 21, 3, 32, 80, - 65, 2, 221, 240, 3, 4, 82, 65, 76, 76, 8, 76, 12, 65, 76, 84, 69, 82, 78, - 65, 84, 73, 78, 71, 32, 138, 18, 87, 63, 83, 4, 134, 18, 87, 147, 21, 77, - 18, 80, 4, 65, 86, 69, 32, 169, 1, 11, 82, 73, 83, 84, 32, 67, 73, 82, - 67, 76, 69, 14, 30, 72, 102, 83, 171, 20, 76, 8, 37, 7, 73, 84, 84, 73, - 78, 71, 32, 8, 252, 2, 4, 67, 69, 73, 76, 25, 5, 70, 76, 79, 79, 82, 4, - 170, 146, 5, 78, 231, 224, 1, 77, 4, 209, 5, 10, 32, 72, 73, 84, 84, 73, - 78, 71, 32, 87, 14, 112, 3, 85, 80, 32, 184, 1, 6, 68, 79, 87, 78, 32, - 83, 149, 225, 5, 10, 83, 73, 68, 69, 32, 84, 79, 32, 83, 73, 10, 40, 5, - 68, 79, 87, 78, 32, 143, 1, 83, 8, 68, 8, 65, 76, 84, 69, 82, 78, 65, 84, - 230, 17, 76, 215, 216, 3, 83, 4, 21, 3, 73, 78, 71, 4, 11, 32, 4, 190, - 17, 76, 215, 216, 3, 83, 2, 207, 30, 69, 162, 1, 148, 2, 11, 65, 82, 77, - 32, 67, 73, 82, 67, 76, 69, 32, 98, 66, 54, 67, 174, 4, 68, 100, 8, 70, - 73, 78, 71, 69, 82, 32, 67, 64, 5, 72, 85, 77, 80, 32, 60, 5, 76, 79, 79, - 80, 32, 102, 80, 34, 83, 216, 1, 7, 84, 82, 73, 80, 76, 69, 32, 218, 1, - 87, 202, 2, 90, 227, 232, 7, 74, 8, 18, 77, 39, 83, 4, 225, 13, 5, 69, - 68, 73, 85, 77, 4, 11, 77, 4, 177, 13, 3, 65, 76, 76, 12, 34, 79, 185, - 13, 3, 69, 78, 68, 6, 183, 13, 88, 46, 100, 6, 79, 82, 78, 69, 82, 32, - 88, 4, 85, 82, 86, 69, 232, 11, 4, 72, 69, 67, 75, 227, 165, 7, 82, 8, - 54, 82, 194, 12, 76, 162, 167, 2, 77, 183, 177, 1, 83, 2, 11, 79, 2, 239, - 176, 5, 84, 30, 50, 32, 137, 2, 7, 68, 32, 67, 82, 79, 83, 83, 26, 66, - 72, 68, 2, 84, 72, 133, 5, 7, 81, 85, 65, 82, 84, 69, 82, 12, 196, 5, 10, - 65, 76, 70, 45, 67, 73, 82, 67, 76, 69, 147, 14, 73, 6, 96, 2, 69, 78, - 29, 18, 82, 69, 69, 45, 81, 85, 65, 82, 84, 69, 82, 32, 67, 73, 82, 67, - 76, 69, 2, 11, 32, 2, 131, 1, 83, 4, 11, 32, 4, 246, 176, 2, 77, 183, - 177, 1, 83, 8, 33, 6, 79, 85, 66, 76, 69, 32, 8, 30, 83, 150, 4, 65, 75, - 87, 2, 209, 196, 8, 3, 84, 82, 65, 6, 32, 3, 73, 82, 67, 151, 9, 79, 4, - 173, 7, 3, 76, 69, 83, 10, 142, 8, 76, 166, 8, 72, 254, 158, 2, 77, 183, - 177, 1, 83, 12, 68, 5, 83, 77, 65, 76, 76, 142, 7, 76, 166, 8, 72, 255, - 158, 2, 77, 5, 11, 32, 2, 227, 36, 68, 6, 181, 6, 4, 69, 65, 75, 83, 12, - 22, 73, 231, 36, 72, 10, 29, 5, 78, 71, 76, 69, 32, 10, 52, 8, 83, 84, - 82, 65, 73, 71, 72, 84, 207, 1, 87, 8, 11, 32, 8, 42, 76, 202, 172, 2, - 77, 183, 177, 1, 83, 4, 25, 4, 65, 82, 71, 69, 5, 147, 207, 8, 83, 8, 26, - 65, 74, 87, 63, 83, 4, 49, 10, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 5, - 17, 2, 32, 87, 2, 29, 5, 82, 73, 83, 84, 32, 2, 165, 196, 7, 2, 70, 76, - 2, 37, 7, 84, 82, 65, 73, 71, 72, 84, 2, 159, 20, 32, 26, 104, 4, 65, 86, - 69, 32, 185, 1, 17, 82, 73, 83, 84, 32, 67, 73, 82, 67, 76, 69, 32, 70, - 82, 79, 78, 84, 22, 108, 6, 67, 85, 82, 86, 69, 32, 140, 1, 13, 68, 73, - 65, 71, 79, 78, 65, 76, 32, 80, 65, 84, 72, 223, 8, 72, 12, 48, 4, 68, - 79, 85, 66, 1, 4, 84, 82, 73, 80, 6, 85, 2, 76, 69, 4, 11, 32, 4, 210, - 30, 68, 43, 83, 6, 29, 5, 73, 71, 90, 65, 71, 6, 11, 32, 6, 42, 76, 162, - 167, 2, 77, 183, 177, 1, 83, 2, 175, 166, 7, 65, 10, 40, 4, 79, 83, 69, - 32, 135, 182, 7, 69, 8, 26, 67, 46, 78, 35, 87, 2, 11, 79, 2, 217, 214, - 7, 3, 78, 84, 65, 2, 221, 219, 7, 3, 69, 85, 84, 4, 44, 3, 82, 73, 78, - 241, 157, 1, 2, 73, 71, 2, 151, 167, 5, 75, 72, 56, 7, 79, 84, 65, 84, - 73, 79, 78, 225, 22, 2, 85, 66, 66, 60, 10, 32, 77, 79, 68, 73, 70, 73, - 69, 82, 45, 155, 1, 45, 30, 82, 49, 250, 226, 8, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 2, 55, 2, 56, 3, 57, 14, 246, 226, 8, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 3, 54, 36, 104, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, - 32, 133, 2, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 18, 100, 4, 68, - 79, 85, 66, 0, 4, 83, 73, 78, 71, 21, 11, 65, 76, 84, 69, 82, 78, 65, 84, - 73, 78, 71, 6, 17, 2, 76, 69, 7, 45, 9, 32, 72, 73, 84, 84, 73, 78, 71, - 32, 4, 32, 2, 67, 69, 33, 2, 70, 76, 2, 11, 73, 2, 231, 142, 8, 76, 2, - 239, 166, 8, 79, 18, 88, 8, 65, 76, 84, 69, 82, 78, 65, 84, 44, 4, 68, - 79, 85, 66, 1, 4, 83, 73, 78, 71, 6, 72, 4, 73, 78, 71, 32, 159, 222, 8, - 69, 6, 17, 2, 76, 69, 7, 11, 32, 4, 11, 72, 4, 11, 73, 4, 33, 6, 84, 84, - 73, 78, 71, 32, 4, 44, 2, 67, 72, 21, 5, 70, 82, 79, 78, 84, 2, 179, 243, - 4, 69, 2, 205, 213, 6, 2, 32, 87, 30, 172, 1, 8, 72, 79, 85, 76, 68, 69, - 82, 32, 196, 1, 7, 81, 85, 69, 69, 90, 69, 32, 240, 1, 7, 85, 82, 70, 65, - 67, 69, 32, 240, 10, 5, 84, 82, 73, 75, 69, 243, 129, 5, 69, 6, 100, 4, - 72, 73, 80, 32, 129, 246, 3, 15, 84, 73, 76, 84, 73, 78, 71, 32, 70, 82, - 79, 77, 32, 87, 65, 4, 48, 4, 80, 79, 83, 73, 253, 192, 7, 2, 83, 80, 2, - 11, 84, 2, 205, 150, 5, 2, 73, 79, 12, 48, 6, 70, 76, 73, 67, 75, 32, 18, - 76, 35, 83, 2, 211, 16, 65, 4, 129, 1, 4, 65, 82, 71, 69, 6, 34, 69, 65, - 4, 77, 65, 76, 76, 2, 17, 2, 81, 85, 2, 21, 3, 69, 78, 84, 2, 151, 207, - 7, 73, 4, 11, 32, 4, 214, 11, 77, 187, 4, 83, 4, 22, 83, 135, 11, 66, 2, - 249, 207, 1, 4, 89, 77, 66, 79, 82, 58, 69, 218, 2, 79, 137, 8, 6, 82, - 65, 86, 69, 76, 45, 20, 76, 3, 69, 84, 72, 185, 1, 11, 78, 83, 69, 32, - 67, 72, 69, 69, 75, 83, 32, 15, 11, 32, 12, 56, 3, 79, 78, 32, 86, 77, - 177, 5, 4, 66, 73, 84, 69, 8, 56, 4, 76, 73, 80, 83, 1, 6, 84, 79, 78, - 71, 85, 69, 5, 11, 32, 2, 11, 77, 2, 201, 194, 3, 2, 79, 86, 6, 50, 77, - 160, 176, 4, 2, 72, 73, 167, 138, 3, 76, 2, 213, 163, 3, 2, 73, 68, 28, - 76, 5, 78, 71, 85, 69, 32, 196, 4, 4, 82, 83, 79, 45, 129, 2, 2, 85, 67, - 16, 192, 2, 7, 67, 69, 78, 84, 82, 69, 32, 52, 14, 73, 78, 83, 73, 68, - 69, 32, 77, 79, 85, 84, 72, 32, 82, 36, 4, 84, 73, 80, 32, 88, 7, 76, 73, - 67, 75, 73, 78, 71, 208, 174, 7, 14, 83, 84, 73, 67, 75, 73, 78, 71, 32, - 79, 85, 84, 32, 70, 149, 158, 1, 17, 77, 79, 86, 69, 83, 32, 65, 71, 65, - 73, 78, 83, 84, 32, 67, 72, 69, 4, 214, 1, 73, 181, 180, 3, 5, 83, 84, - 73, 67, 75, 2, 181, 197, 3, 4, 69, 76, 65, 88, 4, 84, 7, 66, 69, 84, 87, - 69, 69, 78, 41, 10, 84, 79, 85, 67, 72, 73, 78, 71, 32, 73, 2, 11, 32, 2, - 173, 155, 5, 2, 76, 73, 2, 173, 174, 5, 5, 78, 83, 73, 68, 69, 6, 120, - 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 253, 199, 3, 14, 70, 76, 79, - 79, 82, 80, 76, 65, 78, 69, 32, 84, 87, 73, 4, 108, 8, 67, 85, 82, 86, - 69, 68, 32, 66, 161, 204, 1, 13, 83, 84, 82, 65, 73, 71, 72, 84, 32, 83, - 84, 82, 69, 2, 211, 191, 7, 69, 6, 11, 72, 6, 11, 32, 6, 30, 66, 34, 77, - 187, 4, 83, 2, 137, 249, 6, 3, 69, 84, 87, 2, 149, 2, 3, 85, 76, 84, 34, - 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 29, 10, 87, 65, 76, - 76, 80, 76, 65, 78, 69, 32, 14, 178, 1, 82, 151, 2, 83, 20, 72, 11, 65, - 82, 77, 32, 83, 80, 73, 82, 65, 76, 32, 78, 82, 151, 2, 83, 6, 30, 84, - 134, 2, 68, 43, 83, 2, 11, 82, 2, 11, 73, 2, 147, 166, 7, 80, 12, 41, 8, - 79, 84, 65, 84, 73, 79, 78, 45, 12, 52, 5, 70, 76, 79, 79, 82, 1, 4, 87, - 65, 76, 76, 6, 33, 6, 80, 76, 65, 78, 69, 32, 6, 26, 65, 54, 68, 43, 83, - 2, 29, 5, 76, 84, 69, 82, 78, 2, 203, 194, 3, 65, 2, 17, 2, 79, 85, 2, - 147, 164, 7, 66, 2, 231, 163, 7, 73, 2, 11, 72, 2, 203, 192, 3, 65, 2, - 11, 73, 2, 187, 179, 7, 78, 10, 100, 6, 65, 66, 79, 86, 69, 32, 162, 1, - 79, 213, 148, 2, 10, 77, 73, 78, 85, 83, 32, 83, 73, 77, 73, 4, 60, 7, - 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 2, 37, 7, 45, 84, 72, - 65, 78, 32, 65, 2, 25, 4, 66, 79, 86, 69, 2, 161, 241, 1, 2, 32, 69, 4, - 147, 179, 6, 82, 250, 1, 68, 3, 71, 76, 69, 212, 4, 5, 72, 65, 76, 65, - 32, 143, 161, 5, 69, 20, 50, 32, 185, 195, 8, 6, 45, 83, 72, 73, 70, 84, - 16, 138, 1, 67, 0, 9, 71, 82, 65, 80, 72, 73, 67, 32, 67, 116, 2, 72, 73, - 74, 76, 36, 4, 82, 73, 71, 72, 141, 244, 4, 4, 83, 72, 73, 70, 2, 41, 8, - 72, 65, 82, 65, 67, 84, 69, 82, 2, 37, 7, 32, 73, 78, 84, 82, 79, 68, 2, - 11, 85, 2, 223, 132, 8, 67, 2, 25, 4, 71, 72, 45, 82, 2, 185, 1, 7, 69, - 86, 69, 82, 83, 69, 68, 4, 32, 2, 69, 70, 109, 2, 79, 87, 2, 49, 10, 84, - 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 21, 3, 32, 65, 78, 2, 17, 2, 71, - 76, 2, 31, 69, 2, 17, 2, 45, 57, 2, 11, 32, 2, 11, 81, 2, 233, 176, 2, 2, - 85, 79, 228, 1, 168, 2, 8, 65, 82, 67, 72, 65, 73, 67, 32, 224, 1, 15, - 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 114, 76, 188, - 8, 5, 83, 73, 71, 78, 32, 144, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, - 78, 32, 177, 227, 2, 17, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, - 75, 85, 78, 68, 68, 40, 46, 68, 109, 7, 78, 85, 77, 66, 69, 82, 32, 18, - 25, 4, 73, 71, 73, 84, 18, 11, 32, 18, 234, 204, 6, 70, 30, 83, 42, 84, - 170, 86, 78, 14, 79, 223, 110, 69, 22, 226, 232, 2, 79, 146, 229, 3, 69, - 30, 70, 42, 78, 38, 83, 39, 84, 6, 18, 82, 63, 89, 4, 56, 6, 65, 75, 65, - 65, 82, 65, 209, 184, 8, 2, 69, 80, 2, 205, 184, 8, 3, 65, 78, 83, 138, - 1, 60, 6, 69, 84, 84, 69, 82, 32, 233, 186, 2, 3, 73, 84, 72, 118, 246, - 1, 65, 106, 69, 34, 73, 62, 85, 34, 77, 220, 1, 4, 68, 65, 78, 84, 58, - 79, 28, 8, 83, 65, 78, 89, 65, 75, 65, 32, 50, 70, 2, 72, 2, 82, 2, 86, - 2, 89, 32, 8, 84, 65, 65, 76, 85, 74, 65, 32, 29, 9, 75, 65, 78, 84, 65, - 74, 65, 32, 78, 34, 102, 69, 204, 1, 2, 76, 80, 144, 2, 5, 77, 66, 65, - 32, 66, 14, 65, 2, 73, 2, 85, 151, 253, 4, 89, 4, 230, 3, 69, 151, 253, - 4, 89, 12, 46, 76, 2, 82, 154, 3, 73, 151, 253, 4, 89, 4, 11, 85, 4, 138, - 3, 85, 151, 253, 4, 89, 28, 42, 65, 177, 1, 5, 85, 85, 82, 68, 72, 22, - 32, 2, 72, 65, 179, 255, 4, 89, 20, 41, 8, 65, 80, 82, 65, 65, 78, 65, - 32, 20, 70, 84, 138, 1, 68, 22, 66, 2, 67, 2, 71, 2, 74, 2, 75, 3, 80, 4, - 154, 1, 84, 15, 65, 6, 25, 4, 65, 74, 65, 32, 6, 102, 76, 2, 78, 3, 83, - 4, 86, 79, 151, 253, 4, 89, 8, 26, 68, 22, 71, 3, 74, 4, 18, 68, 15, 65, - 2, 11, 65, 2, 147, 253, 4, 89, 6, 26, 78, 21, 2, 83, 65, 2, 89, 2, 65, - 65, 4, 68, 11, 78, 89, 79, 79, 71, 65, 32, 78, 65, 65, 75, 139, 252, 4, - 89, 2, 149, 176, 8, 4, 83, 73, 75, 89, 8, 66, 65, 218, 157, 4, 67, 217, - 145, 4, 6, 86, 73, 83, 65, 82, 71, 4, 192, 116, 5, 76, 45, 76, 65, 75, - 237, 186, 7, 6, 78, 85, 83, 86, 65, 82, 34, 56, 5, 68, 73, 71, 65, 32, - 62, 71, 82, 75, 155, 2, 65, 12, 58, 71, 48, 4, 75, 79, 77, 66, 118, 65, - 26, 73, 19, 80, 4, 11, 65, 4, 236, 2, 3, 69, 84, 84, 67, 89, 2, 11, 85, - 2, 223, 173, 8, 86, 16, 52, 5, 69, 84, 84, 73, 32, 85, 4, 79, 77, 66, 85, - 6, 26, 65, 26, 73, 19, 80, 2, 213, 1, 2, 69, 68, 2, 203, 1, 83, 2, 175, - 1, 65, 10, 40, 2, 86, 65, 137, 128, 8, 2, 32, 68, 9, 29, 5, 32, 72, 65, - 65, 32, 6, 62, 65, 0, 6, 68, 73, 71, 65, 32, 65, 85, 3, 71, 65, 89, 2, - 17, 2, 69, 76, 2, 11, 65, 2, 17, 2, 45, 80, 2, 11, 73, 2, 147, 139, 8, - 76, 2, 189, 253, 5, 5, 65, 78, 85, 75, 73, 12, 84, 2, 32, 80, 206, 2, 45, - 205, 149, 6, 10, 84, 69, 69, 78, 32, 80, 79, 73, 78, 84, 8, 140, 1, 23, - 69, 84, 65, 76, 76, 69, 68, 32, 66, 76, 65, 67, 75, 32, 65, 78, 68, 32, - 87, 72, 73, 84, 69, 49, 7, 79, 73, 78, 84, 69, 68, 32, 2, 25, 4, 32, 70, - 76, 79, 2, 223, 227, 6, 82, 6, 78, 80, 218, 147, 6, 66, 217, 132, 1, 9, - 83, 84, 65, 82, 32, 87, 73, 84, 72, 2, 21, 3, 73, 78, 87, 2, 217, 147, 6, - 4, 72, 69, 69, 76, 2, 139, 208, 3, 80, 12, 46, 73, 102, 85, 153, 145, 7, - 3, 65, 84, 69, 4, 56, 8, 32, 65, 78, 68, 32, 83, 75, 73, 243, 239, 7, 69, - 2, 17, 2, 32, 66, 2, 255, 215, 7, 79, 6, 32, 2, 76, 76, 235, 164, 8, 78, - 5, 11, 32, 2, 49, 10, 65, 78, 68, 32, 67, 82, 79, 83, 83, 66, 2, 11, 79, - 2, 187, 147, 7, 78, 42, 46, 65, 198, 4, 69, 210, 1, 73, 167, 1, 79, 14, - 64, 5, 78, 84, 69, 68, 32, 193, 146, 6, 5, 86, 79, 78, 73, 67, 12, 148, - 1, 12, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, 32, 229, 1, 19, 78, - 79, 82, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, 84, 72, 32, 72, 79, - 8, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 4, 29, 5, 45, - 84, 72, 65, 78, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 25, 4, 32, 68, - 79, 84, 2, 17, 2, 32, 73, 2, 11, 78, 2, 11, 83, 2, 183, 147, 6, 73, 4, - 24, 2, 79, 75, 43, 82, 2, 17, 2, 69, 68, 2, 131, 206, 5, 32, 2, 29, 5, - 73, 90, 79, 78, 84, 2, 11, 65, 2, 199, 238, 5, 76, 12, 80, 2, 69, 80, - 144, 144, 8, 9, 85, 84, 72, 32, 79, 82, 32, 83, 80, 203, 17, 68, 8, 40, - 4, 73, 78, 71, 32, 143, 228, 7, 89, 6, 176, 223, 4, 8, 65, 67, 67, 79, - 77, 77, 79, 68, 234, 174, 2, 83, 139, 86, 70, 6, 76, 9, 67, 69, 32, 79, - 70, 32, 80, 73, 90, 21, 6, 71, 72, 84, 76, 89, 32, 2, 211, 157, 8, 90, 4, - 40, 2, 83, 77, 173, 217, 6, 2, 70, 82, 2, 215, 234, 6, 73, 10, 18, 80, - 75, 84, 6, 152, 222, 4, 9, 73, 78, 71, 32, 76, 65, 82, 71, 69, 139, 193, - 3, 69, 4, 26, 32, 195, 158, 8, 72, 2, 11, 77, 2, 249, 132, 7, 3, 65, 67, - 72, 150, 1, 34, 65, 158, 13, 73, 207, 7, 79, 116, 32, 2, 76, 76, 155, - 142, 7, 83, 114, 30, 32, 141, 12, 2, 69, 82, 110, 194, 2, 65, 48, 3, 66, - 76, 85, 0, 5, 79, 82, 65, 78, 71, 20, 2, 67, 79, 158, 1, 68, 22, 69, 140, - 2, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, 28, 2, 76, 69, 34, 82, - 218, 3, 83, 22, 84, 36, 2, 85, 80, 56, 4, 86, 69, 69, 32, 136, 253, 1, 3, - 71, 82, 69, 34, 72, 146, 4, 80, 234, 77, 78, 222, 240, 2, 70, 83, 81, 6, - 194, 203, 4, 77, 214, 49, 73, 135, 138, 1, 83, 2, 135, 206, 6, 69, 12, - 68, 7, 78, 84, 65, 73, 78, 83, 32, 202, 130, 2, 77, 167, 199, 5, 76, 6, - 36, 4, 65, 83, 32, 77, 175, 1, 87, 2, 11, 69, 2, 11, 77, 2, 171, 219, 7, - 66, 2, 219, 130, 2, 79, 12, 84, 9, 76, 69, 77, 69, 78, 84, 32, 79, 70, - 162, 130, 2, 81, 42, 88, 175, 216, 2, 77, 7, 17, 2, 32, 87, 4, 25, 4, 73, - 84, 72, 32, 4, 26, 86, 171, 233, 4, 79, 2, 129, 243, 5, 21, 69, 82, 84, - 73, 67, 65, 76, 32, 66, 65, 82, 32, 65, 84, 32, 69, 78, 68, 32, 79, 70, - 2, 221, 196, 5, 2, 32, 67, 8, 130, 1, 70, 247, 132, 2, 83, 40, 96, 3, 73, - 71, 72, 64, 13, 79, 77, 65, 78, 32, 78, 85, 77, 69, 82, 65, 76, 32, 191, - 255, 5, 69, 6, 17, 2, 84, 32, 6, 170, 143, 4, 67, 210, 3, 80, 147, 8, 84, - 32, 62, 69, 50, 70, 62, 79, 46, 84, 166, 163, 6, 83, 211, 86, 78, 4, 26, - 76, 231, 233, 7, 73, 2, 183, 164, 6, 69, 8, 26, 73, 167, 155, 7, 79, 6, - 130, 26, 86, 255, 140, 6, 70, 6, 11, 78, 6, 11, 69, 7, 247, 191, 2, 32, - 8, 42, 87, 190, 163, 6, 72, 231, 159, 1, 69, 4, 26, 69, 231, 146, 8, 79, - 2, 143, 150, 7, 76, 2, 147, 199, 4, 69, 4, 182, 248, 3, 87, 211, 128, 2, - 73, 2, 185, 243, 4, 9, 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 11, 87, 2, - 21, 3, 73, 84, 72, 2, 11, 32, 2, 163, 166, 4, 85, 4, 29, 5, 32, 84, 72, - 65, 78, 5, 11, 32, 2, 11, 79, 2, 135, 189, 1, 82, 32, 34, 76, 141, 136, - 7, 2, 82, 75, 30, 40, 4, 73, 78, 71, 32, 131, 144, 8, 69, 28, 112, 14, - 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 41, 10, 70, 65, - 67, 69, 32, 87, 73, 84, 72, 32, 4, 160, 1, 2, 72, 69, 163, 236, 4, 79, - 24, 86, 72, 108, 10, 79, 80, 69, 78, 32, 77, 79, 85, 84, 72, 246, 1, 83, - 203, 219, 2, 84, 6, 34, 69, 54, 79, 231, 234, 7, 65, 2, 165, 146, 5, 8, - 65, 82, 84, 45, 83, 72, 65, 80, 2, 219, 201, 4, 82, 9, 29, 5, 32, 65, 78, - 68, 32, 6, 26, 67, 58, 83, 71, 84, 2, 17, 2, 79, 76, 2, 145, 188, 3, 4, - 68, 32, 83, 87, 2, 11, 77, 2, 11, 73, 2, 11, 76, 2, 217, 144, 5, 3, 73, - 78, 71, 2, 37, 7, 73, 71, 72, 84, 76, 89, 45, 2, 231, 143, 5, 67, 8, 64, - 11, 77, 73, 76, 73, 78, 71, 32, 69, 89, 69, 83, 175, 1, 85, 7, 29, 5, 32, - 65, 78, 68, 32, 4, 52, 4, 72, 65, 78, 68, 57, 5, 84, 72, 82, 69, 69, 2, - 221, 232, 4, 9, 32, 67, 79, 86, 69, 82, 73, 78, 71, 2, 241, 199, 4, 2, - 32, 72, 2, 17, 2, 78, 71, 2, 173, 245, 6, 4, 76, 65, 83, 83, 2, 171, 220, - 4, 75, 16, 42, 65, 32, 2, 69, 69, 21, 2, 79, 87, 4, 142, 255, 6, 73, 227, - 114, 75, 2, 131, 128, 7, 90, 10, 100, 7, 32, 67, 65, 80, 80, 69, 68, 28, - 3, 77, 65, 78, 136, 218, 2, 3, 66, 79, 65, 167, 143, 1, 70, 2, 137, 159, - 4, 2, 32, 77, 5, 49, 10, 32, 87, 73, 84, 72, 79, 85, 84, 32, 83, 2, 179, - 236, 6, 78, 155, 3, 162, 2, 67, 48, 2, 70, 84, 228, 1, 6, 71, 68, 73, 65, - 78, 32, 244, 10, 3, 76, 73, 68, 140, 2, 12, 79, 78, 32, 87, 73, 84, 72, - 32, 82, 73, 71, 72, 20, 11, 82, 65, 32, 83, 79, 77, 80, 69, 78, 71, 32, - 234, 2, 85, 216, 8, 6, 89, 79, 77, 66, 79, 32, 182, 226, 6, 77, 130, 135, - 1, 72, 3, 83, 4, 232, 227, 2, 3, 67, 69, 82, 215, 226, 4, 75, 10, 70, 32, - 124, 9, 87, 65, 82, 69, 45, 70, 85, 78, 67, 223, 250, 5, 66, 6, 58, 72, - 44, 6, 73, 67, 69, 32, 67, 82, 239, 188, 6, 83, 2, 11, 89, 2, 11, 80, 2, - 231, 174, 6, 72, 2, 191, 199, 6, 69, 2, 11, 84, 2, 203, 138, 6, 73, 84, - 248, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 168, 2, 13, 73, 78, - 68, 69, 80, 69, 78, 68, 69, 78, 84, 32, 83, 20, 7, 76, 69, 84, 84, 69, - 82, 32, 188, 3, 7, 78, 85, 77, 66, 69, 82, 32, 113, 12, 80, 85, 78, 67, - 84, 85, 65, 84, 73, 79, 78, 32, 22, 140, 1, 3, 72, 79, 79, 20, 4, 76, 79, - 78, 71, 50, 83, 58, 84, 142, 227, 3, 68, 176, 237, 1, 4, 82, 69, 83, 72, - 129, 15, 4, 67, 85, 82, 86, 4, 195, 224, 5, 75, 2, 25, 4, 32, 72, 79, 79, - 2, 135, 209, 5, 75, 2, 21, 3, 84, 82, 79, 2, 11, 75, 2, 207, 208, 5, 69, - 4, 193, 223, 1, 2, 87, 79, 2, 143, 254, 6, 72, 42, 190, 1, 65, 50, 71, - 34, 76, 56, 5, 82, 69, 83, 72, 45, 2, 90, 34, 83, 66, 89, 158, 208, 1, - 72, 234, 5, 75, 198, 75, 66, 2, 70, 178, 216, 4, 78, 134, 2, 84, 2, 87, - 218, 103, 80, 171, 4, 77, 4, 26, 76, 163, 252, 6, 89, 2, 219, 215, 1, 69, - 2, 11, 73, 2, 239, 238, 2, 77, 4, 26, 65, 135, 147, 6, 69, 2, 193, 209, - 1, 2, 77, 69, 2, 11, 65, 2, 167, 251, 6, 89, 6, 26, 65, 251, 250, 6, 72, - 4, 182, 200, 1, 77, 139, 133, 6, 68, 2, 199, 208, 1, 79, 8, 18, 79, 59, - 84, 4, 11, 78, 4, 11, 69, 5, 11, 32, 2, 167, 166, 2, 72, 4, 182, 140, 6, - 87, 179, 157, 1, 69, 10, 66, 67, 0, 6, 72, 65, 76, 70, 32, 67, 53, 4, 84, - 87, 79, 32, 2, 21, 3, 73, 82, 67, 2, 173, 236, 4, 2, 76, 69, 6, 100, 13, - 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 83, 13, 8, 67, 73, 82, - 67, 76, 69, 83, 32, 5, 11, 32, 2, 249, 166, 6, 4, 87, 73, 84, 72, 8, 30, - 32, 189, 1, 2, 85, 83, 4, 93, 21, 81, 85, 73, 76, 84, 32, 83, 81, 85, 65, - 82, 69, 32, 79, 82, 78, 65, 77, 69, 78, 84, 5, 11, 32, 2, 11, 73, 2, 17, - 2, 78, 32, 2, 11, 66, 2, 233, 212, 6, 4, 76, 65, 67, 75, 5, 145, 199, 4, - 7, 32, 87, 73, 84, 72, 32, 79, 2, 179, 251, 3, 84, 70, 52, 7, 76, 69, 84, - 84, 69, 82, 32, 159, 131, 6, 68, 50, 198, 1, 69, 32, 2, 77, 65, 30, 78, - 186, 146, 1, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, - 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 242, 222, 6, 65, 2, 73, 2, 79, 3, 85, - 4, 226, 241, 7, 69, 147, 1, 72, 4, 210, 242, 7, 69, 3, 72, 6, 182, 146, - 1, 71, 2, 89, 243, 222, 6, 65, 58, 104, 3, 84, 72, 32, 169, 160, 6, 17, + 69, 82, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 193, 179, 3, 5, 32, 79, + 86, 69, 82, 2, 21, 3, 32, 83, 84, 2, 215, 161, 10, 69, 2, 25, 4, 32, 79, + 80, 69, 2, 207, 146, 9, 78, 4, 26, 73, 179, 160, 10, 85, 2, 247, 160, 10, + 77, 211, 14, 174, 1, 68, 152, 20, 2, 71, 78, 184, 181, 1, 6, 77, 73, 76, + 65, 82, 32, 158, 2, 78, 202, 24, 88, 237, 238, 6, 15, 76, 72, 79, 85, 69, + 84, 84, 69, 32, 79, 70, 32, 74, 65, 80, 252, 1, 40, 5, 68, 72, 65, 77, + 32, 135, 17, 69, 184, 1, 202, 1, 69, 68, 7, 76, 69, 84, 84, 69, 82, 32, + 176, 4, 15, 82, 69, 80, 69, 84, 73, 84, 73, 79, 78, 32, 77, 65, 82, 75, + 50, 83, 164, 8, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 171, 175, + 6, 68, 2, 37, 7, 78, 68, 32, 79, 70, 32, 84, 2, 201, 210, 9, 2, 69, 88, + 102, 214, 1, 65, 98, 84, 242, 188, 6, 68, 158, 1, 86, 186, 5, 85, 186, + 202, 1, 73, 138, 196, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, + 2, 80, 138, 69, 72, 2, 76, 2, 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 11, + 72, 8, 76, 84, 69, 82, 78, 65, 84, 69, 214, 154, 10, 65, 2, 73, 3, 85, 2, + 235, 245, 9, 32, 14, 134, 1, 72, 252, 231, 5, 19, 87, 79, 45, 67, 73, 82, + 67, 76, 69, 32, 65, 76, 84, 69, 82, 78, 65, 84, 69, 254, 233, 3, 84, 195, + 71, 65, 4, 164, 217, 9, 20, 82, 69, 69, 45, 67, 73, 82, 67, 76, 69, 32, + 65, 76, 84, 69, 82, 78, 65, 84, 69, 147, 64, 65, 6, 11, 45, 6, 186, 152, + 10, 49, 2, 50, 3, 51, 44, 38, 69, 181, 7, 4, 73, 71, 78, 32, 32, 96, 11, + 67, 84, 73, 79, 78, 32, 77, 65, 82, 75, 32, 177, 6, 8, 80, 65, 82, 65, + 84, 79, 82, 32, 28, 80, 11, 68, 79, 85, 66, 76, 69, 32, 82, 73, 78, 71, + 57, 5, 87, 73, 84, 72, 32, 5, 11, 32, 2, 249, 231, 8, 6, 87, 73, 84, 72, + 32, 82, 24, 212, 1, 12, 67, 73, 82, 67, 76, 69, 83, 32, 65, 78, 68, 32, + 116, 5, 81, 85, 65, 68, 82, 0, 4, 83, 69, 80, 84, 12, 16, 82, 65, 89, 83, + 32, 65, 78, 68, 32, 68, 79, 84, 84, 69, 68, 32, 46, 68, 45, 3, 84, 82, + 73, 6, 60, 4, 70, 79, 85, 82, 0, 3, 84, 87, 79, 187, 229, 8, 82, 2, 225, + 207, 6, 8, 32, 69, 78, 67, 76, 79, 83, 85, 2, 83, 85, 6, 42, 68, 28, 3, + 84, 82, 73, 215, 1, 67, 2, 197, 1, 3, 79, 85, 66, 2, 171, 1, 80, 6, 52, + 9, 68, 69, 78, 84, 32, 65, 78, 68, 32, 103, 80, 4, 116, 6, 68, 79, 84, + 84, 69, 68, 49, 14, 85, 45, 83, 72, 65, 80, 69, 68, 32, 79, 82, 78, 65, + 77, 2, 17, 2, 76, 69, 2, 17, 2, 32, 67, 2, 25, 4, 82, 69, 83, 67, 2, 239, + 186, 1, 69, 4, 158, 237, 8, 66, 135, 83, 68, 12, 146, 229, 4, 83, 202, + 141, 1, 67, 98, 78, 138, 216, 3, 65, 239, 1, 86, 26, 74, 65, 94, 86, 210, + 183, 6, 85, 186, 202, 1, 73, 198, 140, 2, 69, 3, 79, 10, 168, 184, 6, 10, + 76, 84, 69, 82, 78, 65, 84, 69, 32, 85, 254, 214, 3, 65, 2, 73, 3, 85, 4, + 11, 79, 4, 33, 6, 67, 65, 76, 73, 67, 32, 4, 255, 183, 6, 82, 68, 84, 12, + 84, 73, 67, 32, 76, 69, 84, 84, 69, 82, 32, 78, 49, 5, 87, 65, 89, 83, + 32, 52, 178, 156, 1, 50, 222, 176, 3, 48, 131, 2, 49, 16, 56, 5, 66, 76, + 65, 67, 75, 1, 5, 87, 72, 73, 84, 69, 8, 11, 32, 8, 70, 82, 24, 3, 76, + 69, 70, 12, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 21, 3, 73, 71, 72, 2, 11, + 84, 2, 225, 92, 6, 32, 80, 79, 73, 78, 84, 194, 10, 96, 8, 87, 82, 73, + 84, 73, 78, 71, 32, 241, 238, 1, 10, 32, 79, 70, 32, 84, 72, 69, 32, 72, + 79, 192, 10, 136, 3, 4, 65, 73, 82, 32, 192, 1, 2, 66, 82, 102, 67, 138, + 1, 68, 218, 4, 69, 242, 6, 70, 244, 3, 4, 87, 65, 76, 76, 138, 1, 72, + 234, 77, 76, 212, 6, 2, 77, 79, 222, 42, 78, 218, 1, 82, 182, 7, 83, 162, + 5, 84, 180, 10, 5, 71, 82, 65, 83, 80, 184, 5, 30, 85, 80, 80, 69, 82, + 32, 66, 79, 68, 89, 32, 84, 73, 76, 84, 73, 78, 71, 32, 70, 82, 79, 77, + 32, 72, 73, 80, 32, 74, 79, 135, 207, 4, 80, 8, 48, 4, 66, 76, 79, 87, + 29, 4, 83, 85, 67, 75, 4, 58, 32, 211, 226, 4, 73, 4, 30, 32, 61, 3, 73, + 78, 71, 2, 237, 157, 1, 10, 83, 77, 65, 76, 76, 32, 82, 79, 84, 65, 2, + 171, 134, 9, 32, 10, 52, 5, 69, 65, 84, 72, 32, 229, 169, 1, 2, 85, 83, + 4, 140, 164, 2, 2, 69, 88, 1, 2, 73, 78, 10, 40, 6, 72, 69, 69, 75, 83, + 32, 63, 79, 6, 154, 107, 83, 146, 38, 78, 153, 223, 3, 4, 80, 85, 70, 70, + 4, 178, 180, 9, 76, 223, 46, 77, 28, 108, 15, 82, 69, 65, 77, 89, 32, 69, + 89, 69, 66, 82, 79, 87, 83, 32, 165, 1, 7, 89, 78, 65, 77, 73, 67, 32, 8, + 64, 4, 68, 79, 87, 78, 0, 2, 85, 80, 29, 4, 78, 69, 85, 84, 2, 153, 143, + 1, 2, 32, 78, 4, 21, 3, 82, 65, 76, 4, 11, 32, 4, 194, 58, 68, 191, 199, + 9, 85, 20, 180, 1, 11, 69, 86, 69, 82, 89, 32, 79, 84, 72, 69, 82, 30, + 70, 22, 83, 128, 122, 9, 65, 82, 82, 79, 87, 72, 69, 65, 68, 234, 38, 82, + 136, 206, 3, 2, 84, 69, 21, 4, 71, 82, 65, 68, 2, 181, 230, 8, 2, 32, 84, + 2, 163, 146, 6, 65, 6, 68, 11, 73, 77, 85, 76, 84, 65, 78, 69, 79, 85, + 83, 151, 227, 8, 76, 5, 203, 151, 1, 32, 54, 64, 2, 89, 69, 172, 227, 4, + 4, 88, 67, 73, 84, 203, 138, 4, 65, 50, 166, 1, 32, 56, 15, 66, 82, 79, + 87, 83, 32, 83, 84, 82, 65, 73, 71, 72, 84, 32, 44, 5, 71, 65, 90, 69, + 45, 140, 2, 7, 76, 65, 83, 72, 69, 83, 32, 65, 2, 83, 32, 6, 220, 150, 1, + 5, 66, 76, 73, 78, 75, 243, 129, 5, 87, 6, 186, 53, 68, 154, 84, 78, 167, + 243, 8, 85, 18, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 25, + 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 8, 82, 83, 207, 45, 67, 10, + 18, 67, 43, 83, 4, 254, 45, 85, 213, 95, 3, 73, 82, 67, 6, 37, 7, 84, 82, + 65, 73, 71, 72, 84, 7, 11, 32, 4, 202, 163, 1, 65, 55, 68, 6, 130, 51, + 68, 180, 173, 4, 4, 70, 76, 85, 84, 139, 154, 5, 85, 14, 112, 5, 72, 65, + 76, 70, 32, 26, 67, 28, 4, 87, 73, 68, 69, 230, 92, 79, 233, 135, 4, 6, + 83, 81, 85, 69, 69, 90, 4, 22, 67, 131, 93, 79, 2, 213, 232, 2, 2, 76, + 79, 4, 162, 67, 32, 217, 61, 4, 78, 73, 78, 71, 38, 204, 1, 28, 65, 67, + 69, 32, 68, 73, 82, 69, 67, 84, 73, 79, 78, 32, 80, 79, 83, 73, 84, 73, + 79, 78, 32, 78, 79, 83, 69, 32, 138, 1, 73, 82, 76, 176, 1, 8, 79, 82, + 69, 72, 69, 65, 68, 32, 187, 158, 7, 85, 6, 88, 10, 85, 80, 32, 79, 82, + 32, 68, 79, 87, 78, 13, 8, 70, 79, 82, 87, 65, 82, 68, 32, 5, 11, 32, 2, + 153, 231, 4, 3, 84, 73, 76, 12, 180, 225, 3, 11, 76, 76, 32, 77, 79, 68, + 73, 70, 73, 69, 82, 131, 195, 2, 78, 12, 44, 4, 73, 67, 75, 32, 29, 3, + 79, 79, 82, 10, 254, 140, 1, 76, 35, 83, 2, 225, 247, 8, 20, 80, 76, 65, + 78, 69, 32, 83, 72, 79, 85, 76, 68, 69, 82, 32, 72, 73, 80, 32, 77, 6, + 166, 89, 87, 222, 38, 67, 47, 78, 156, 4, 34, 65, 133, 75, 3, 69, 65, 68, + 140, 4, 36, 3, 78, 68, 45, 143, 186, 9, 73, 138, 4, 92, 5, 65, 78, 71, + 76, 69, 138, 5, 67, 150, 10, 70, 186, 42, 72, 241, 12, 4, 79, 86, 65, 76, + 37, 11, 32, 34, 188, 1, 5, 73, 78, 68, 69, 88, 176, 1, 7, 76, 73, 84, 84, + 76, 69, 32, 136, 1, 5, 82, 73, 78, 71, 32, 173, 66, 18, 77, 73, 68, 68, + 76, 69, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 17, 11, 32, 14, + 128, 1, 7, 77, 73, 68, 68, 76, 69, 32, 228, 24, 11, 82, 73, 78, 71, 32, + 76, 73, 84, 84, 76, 69, 241, 42, 5, 84, 72, 85, 77, 66, 4, 174, 70, 76, + 247, 215, 8, 82, 8, 44, 5, 73, 78, 68, 69, 88, 207, 238, 9, 85, 7, 145, + 24, 18, 32, 84, 72, 85, 77, 66, 32, 73, 78, 68, 69, 88, 32, 84, 72, 85, + 77, 66, 4, 108, 22, 68, 79, 87, 78, 32, 77, 73, 68, 68, 76, 69, 32, 84, + 72, 85, 77, 66, 32, 73, 78, 68, 69, 155, 68, 76, 2, 207, 169, 8, 88, 88, + 64, 5, 73, 82, 67, 76, 69, 184, 3, 3, 76, 65, 87, 183, 2, 85, 35, 11, 32, + 32, 116, 5, 73, 78, 68, 69, 88, 200, 1, 7, 76, 73, 84, 84, 76, 69, 32, + 36, 7, 77, 73, 68, 68, 76, 69, 32, 163, 64, 82, 21, 11, 32, 18, 72, 6, + 77, 73, 68, 68, 76, 69, 198, 26, 72, 242, 38, 82, 131, 230, 8, 66, 13, + 11, 32, 10, 64, 5, 67, 82, 79, 83, 83, 198, 64, 84, 82, 76, 247, 215, 8, + 82, 4, 134, 65, 32, 231, 226, 8, 69, 4, 194, 193, 8, 73, 159, 168, 1, 85, + 6, 252, 47, 10, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 191, 185, 9, 85, + 17, 11, 32, 14, 144, 2, 28, 77, 73, 68, 68, 76, 69, 32, 82, 73, 78, 71, + 32, 76, 73, 84, 84, 76, 69, 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 176, + 49, 2, 70, 79, 198, 11, 78, 162, 1, 84, 197, 118, 23, 73, 78, 68, 69, 88, + 32, 84, 72, 85, 77, 66, 32, 67, 85, 82, 86, 69, 32, 84, 72, 85, 77, 66, + 5, 247, 180, 1, 32, 38, 46, 80, 149, 2, 6, 82, 76, 73, 67, 85, 69, 31, + 11, 32, 28, 188, 1, 5, 73, 78, 68, 69, 88, 56, 19, 70, 73, 86, 69, 32, + 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 196, 50, 7, 77, + 73, 68, 68, 76, 69, 32, 18, 79, 218, 7, 78, 163, 1, 84, 9, 11, 32, 6, 40, + 5, 84, 72, 85, 77, 66, 243, 58, 82, 5, 215, 46, 32, 9, 11, 32, 6, 72, 5, + 73, 78, 68, 69, 88, 0, 6, 77, 73, 68, 68, 76, 69, 179, 71, 79, 2, 193, + 147, 9, 13, 32, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, 32, 172, 2, + 44, 3, 73, 83, 84, 177, 33, 3, 76, 65, 84, 247, 1, 11, 32, 244, 1, 160, + 2, 5, 73, 78, 68, 69, 88, 192, 16, 7, 76, 73, 84, 84, 76, 69, 32, 196, 2, + 7, 77, 73, 68, 68, 76, 69, 32, 200, 4, 5, 82, 73, 78, 71, 32, 132, 2, 5, + 84, 72, 85, 77, 66, 138, 2, 72, 205, 9, 22, 70, 79, 85, 82, 32, 70, 73, + 78, 71, 69, 82, 83, 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 137, 1, 11, + 32, 134, 1, 232, 1, 4, 66, 69, 78, 84, 36, 6, 72, 73, 78, 71, 69, 68, 76, + 6, 77, 73, 68, 68, 76, 69, 168, 7, 2, 67, 85, 64, 6, 84, 72, 85, 77, 66, + 32, 160, 5, 16, 85, 80, 32, 77, 73, 68, 68, 76, 69, 32, 72, 73, 78, 71, + 69, 68, 151, 4, 82, 5, 217, 45, 5, 32, 79, 86, 69, 82, 9, 11, 32, 6, 252, + 20, 8, 77, 73, 68, 68, 76, 69, 32, 85, 167, 172, 8, 76, 71, 11, 32, 68, + 236, 1, 4, 66, 69, 78, 84, 42, 67, 244, 1, 10, 85, 80, 32, 83, 80, 82, + 69, 65, 68, 32, 100, 6, 72, 73, 78, 71, 69, 68, 46, 82, 80, 5, 84, 72, + 85, 77, 66, 188, 28, 14, 83, 84, 82, 65, 73, 71, 72, 84, 32, 84, 72, 85, + 77, 66, 227, 17, 76, 5, 213, 92, 6, 32, 84, 72, 85, 77, 66, 28, 68, 8, + 79, 78, 74, 79, 73, 78, 69, 68, 233, 1, 4, 82, 79, 83, 83, 23, 11, 32, + 20, 144, 1, 6, 67, 85, 80, 80, 69, 68, 28, 6, 84, 72, 85, 77, 66, 32, 68, + 5, 72, 73, 78, 71, 69, 166, 22, 73, 165, 7, 6, 77, 73, 68, 68, 76, 69, 5, + 11, 32, 2, 203, 27, 84, 8, 160, 1, 4, 83, 73, 68, 69, 219, 61, 70, 6, 22, + 69, 159, 47, 32, 4, 223, 15, 68, 5, 241, 30, 7, 32, 83, 80, 82, 69, 65, + 68, 8, 32, 3, 73, 78, 71, 247, 19, 65, 7, 11, 32, 4, 138, 36, 67, 131, + 240, 8, 66, 19, 11, 32, 16, 64, 6, 65, 78, 71, 76, 69, 68, 22, 67, 106, + 72, 163, 146, 9, 66, 5, 167, 221, 5, 32, 6, 74, 85, 230, 7, 73, 241, 25, + 10, 79, 78, 74, 79, 73, 78, 69, 68, 32, 72, 2, 201, 193, 4, 2, 80, 80, 4, + 194, 33, 73, 217, 26, 2, 79, 79, 40, 164, 1, 7, 65, 78, 71, 76, 69, 68, + 32, 46, 67, 220, 1, 14, 70, 79, 82, 87, 65, 82, 68, 32, 73, 78, 68, 69, + 88, 32, 32, 4, 72, 79, 79, 75, 53, 4, 83, 73, 68, 69, 4, 128, 1, 2, 73, + 78, 1, 3, 79, 85, 84, 12, 36, 5, 73, 82, 67, 76, 69, 15, 85, 5, 47, 68, + 8, 32, 4, 80, 80, 69, 68, 39, 82, 2, 225, 40, 5, 32, 77, 73, 68, 68, 6, + 56, 9, 86, 69, 32, 84, 72, 85, 77, 66, 32, 143, 37, 76, 4, 162, 160, 1, + 73, 163, 198, 4, 85, 4, 202, 84, 83, 131, 186, 8, 66, 7, 157, 8, 9, 69, + 68, 32, 77, 73, 68, 68, 76, 69, 15, 11, 32, 12, 92, 6, 73, 78, 68, 69, + 88, 32, 156, 13, 5, 84, 72, 85, 77, 66, 193, 8, 4, 66, 79, 84, 72, 4, 26, + 72, 243, 140, 9, 66, 2, 215, 152, 8, 73, 7, 37, 7, 32, 84, 72, 85, 77, + 66, 32, 4, 178, 28, 67, 207, 129, 1, 83, 22, 88, 4, 68, 79, 87, 78, 186, + 1, 84, 158, 6, 82, 130, 21, 73, 230, 238, 8, 66, 159, 67, 85, 9, 11, 32, + 6, 80, 9, 79, 84, 72, 69, 82, 83, 32, 67, 73, 25, 7, 82, 73, 80, 80, 76, + 69, 32, 2, 225, 51, 2, 82, 67, 4, 22, 67, 171, 80, 83, 2, 11, 85, 2, 169, + 185, 4, 2, 82, 86, 4, 196, 35, 6, 79, 85, 67, 72, 69, 83, 39, 72, 30, + 134, 1, 82, 28, 6, 84, 72, 85, 77, 66, 32, 138, 3, 85, 134, 1, 68, 210, + 30, 76, 189, 128, 8, 9, 66, 69, 78, 84, 32, 79, 86, 69, 82, 4, 238, 4, + 65, 231, 29, 73, 16, 80, 7, 65, 78, 71, 76, 69, 68, 32, 126, 67, 124, 4, + 72, 79, 79, 75, 147, 32, 76, 6, 60, 10, 79, 85, 84, 32, 73, 78, 68, 69, + 88, 32, 215, 1, 73, 4, 26, 67, 155, 202, 9, 85, 2, 249, 185, 2, 3, 82, + 79, 83, 6, 76, 12, 73, 82, 67, 76, 69, 68, 32, 73, 78, 68, 69, 88, 45, 3, + 85, 80, 80, 4, 11, 32, 4, 150, 21, 72, 135, 180, 9, 85, 2, 25, 4, 69, 68, + 32, 73, 2, 233, 30, 4, 78, 68, 69, 88, 4, 11, 80, 5, 175, 15, 32, 18, + 102, 68, 20, 6, 77, 73, 68, 68, 76, 69, 42, 82, 198, 29, 84, 82, 76, 210, + 128, 8, 73, 159, 168, 1, 85, 2, 151, 241, 7, 79, 7, 11, 32, 4, 206, 3, + 82, 179, 16, 67, 2, 11, 65, 2, 37, 7, 73, 83, 69, 68, 32, 75, 78, 2, 11, + 85, 2, 11, 67, 2, 131, 160, 8, 75, 35, 11, 32, 32, 148, 1, 8, 66, 69, 84, + 87, 69, 69, 78, 32, 102, 72, 20, 5, 79, 86, 69, 82, 32, 120, 4, 83, 73, + 68, 69, 100, 6, 85, 78, 68, 69, 82, 32, 207, 40, 70, 8, 80, 12, 73, 78, + 68, 69, 88, 32, 77, 73, 68, 68, 76, 69, 246, 20, 77, 155, 6, 82, 5, 135, + 70, 32, 2, 243, 174, 4, 69, 4, 52, 6, 70, 79, 85, 82, 32, 82, 161, 2, 2, + 84, 87, 2, 11, 65, 2, 233, 80, 9, 73, 83, 69, 68, 32, 75, 78, 85, 67, 6, + 11, 32, 6, 38, 68, 190, 15, 67, 131, 240, 8, 66, 2, 25, 4, 73, 65, 71, + 79, 2, 235, 182, 8, 78, 10, 54, 73, 34, 84, 48, 4, 70, 79, 85, 82, 131, + 23, 76, 2, 161, 7, 4, 78, 68, 69, 88, 4, 34, 87, 13, 4, 72, 82, 69, 69, + 2, 11, 79, 2, 173, 176, 8, 5, 32, 70, 73, 78, 71, 55, 11, 32, 52, 194, 1, + 70, 172, 3, 4, 72, 69, 69, 76, 192, 1, 6, 83, 80, 76, 73, 84, 32, 236, 1, + 6, 84, 72, 85, 77, 66, 32, 205, 249, 4, 16, 66, 69, 84, 87, 69, 69, 78, + 32, 80, 65, 76, 77, 32, 70, 65, 67, 24, 140, 1, 18, 73, 86, 69, 32, 70, + 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 165, 1, 11, 79, 85, + 82, 32, 70, 73, 78, 71, 69, 82, 83, 15, 11, 32, 12, 68, 6, 72, 73, 78, + 71, 69, 68, 42, 84, 186, 2, 70, 203, 247, 8, 66, 7, 11, 32, 4, 190, 4, + 84, 155, 15, 78, 2, 189, 35, 6, 72, 85, 77, 66, 32, 70, 11, 11, 32, 8, + 72, 9, 67, 79, 78, 74, 79, 73, 78, 69, 68, 154, 8, 72, 235, 240, 8, 66, + 5, 225, 159, 9, 3, 32, 83, 80, 11, 11, 32, 8, 96, 19, 70, 73, 86, 69, 32, + 70, 73, 78, 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 151, 2, 84, 7, + 11, 32, 4, 26, 70, 203, 247, 8, 66, 2, 21, 3, 79, 85, 82, 2, 167, 1, 32, + 10, 72, 6, 67, 69, 78, 84, 82, 69, 96, 5, 73, 78, 68, 69, 88, 167, 16, + 76, 7, 49, 10, 32, 84, 72, 85, 77, 66, 32, 83, 73, 68, 4, 11, 69, 5, 11, + 32, 2, 135, 246, 8, 66, 2, 11, 32, 2, 11, 84, 2, 153, 135, 1, 5, 72, 85, + 77, 66, 32, 6, 242, 30, 70, 142, 104, 83, 183, 238, 7, 66, 82, 48, 4, 73, + 78, 71, 69, 181, 9, 3, 79, 79, 75, 63, 11, 32, 60, 206, 1, 70, 172, 1, 5, + 73, 78, 68, 69, 88, 196, 2, 6, 76, 73, 84, 84, 76, 69, 80, 6, 77, 73, 68, + 68, 76, 69, 30, 79, 64, 4, 82, 73, 78, 71, 124, 6, 84, 72, 85, 77, 66, + 32, 158, 6, 78, 195, 147, 4, 83, 4, 92, 19, 73, 86, 69, 32, 70, 73, 78, + 71, 69, 82, 83, 32, 83, 80, 82, 69, 65, 68, 32, 19, 79, 2, 191, 25, 79, + 2, 249, 1, 11, 85, 82, 32, 70, 73, 78, 71, 69, 82, 83, 32, 23, 11, 32, + 20, 86, 72, 40, 7, 77, 73, 68, 68, 76, 69, 32, 104, 5, 84, 72, 85, 77, + 66, 219, 9, 82, 2, 11, 73, 2, 233, 159, 4, 2, 78, 71, 6, 36, 4, 82, 73, + 78, 71, 203, 10, 76, 5, 11, 32, 2, 11, 67, 2, 21, 3, 79, 78, 74, 2, 175, + 33, 79, 11, 11, 32, 8, 34, 83, 210, 22, 79, 203, 39, 76, 4, 218, 159, 7, + 73, 195, 8, 77, 9, 11, 32, 6, 22, 73, 199, 8, 84, 4, 25, 4, 78, 68, 69, + 88, 5, 155, 8, 32, 5, 11, 32, 2, 171, 8, 82, 8, 21, 3, 80, 69, 78, 9, 11, + 32, 6, 178, 7, 78, 163, 1, 84, 5, 97, 22, 32, 68, 79, 87, 78, 32, 73, 78, + 68, 69, 88, 32, 84, 72, 85, 77, 66, 32, 72, 79, 79, 75, 2, 157, 77, 2, + 32, 77, 6, 68, 9, 66, 69, 84, 87, 69, 69, 78, 32, 77, 53, 4, 83, 73, 68, + 69, 2, 29, 5, 73, 68, 68, 76, 69, 2, 219, 153, 4, 32, 5, 33, 6, 32, 84, + 79, 85, 67, 72, 2, 169, 134, 8, 3, 73, 78, 71, 21, 11, 32, 18, 172, 1, 4, + 67, 85, 82, 76, 28, 18, 73, 78, 68, 69, 88, 32, 82, 73, 78, 71, 32, 76, + 73, 84, 84, 76, 69, 32, 48, 7, 77, 73, 68, 68, 76, 69, 32, 225, 2, 4, 82, + 73, 78, 71, 2, 173, 157, 5, 2, 73, 67, 6, 154, 193, 5, 85, 134, 242, 2, + 79, 243, 41, 73, 8, 104, 21, 82, 73, 78, 71, 32, 76, 73, 84, 84, 76, 69, + 32, 67, 79, 78, 74, 79, 73, 78, 69, 68, 143, 2, 84, 7, 211, 228, 2, 32, + 17, 11, 32, 14, 112, 14, 70, 73, 86, 69, 32, 70, 73, 78, 71, 69, 82, 83, + 32, 83, 22, 76, 66, 78, 70, 82, 94, 84, 167, 128, 8, 73, 2, 219, 160, 2, + 80, 2, 21, 3, 73, 84, 84, 2, 17, 2, 76, 69, 2, 139, 180, 8, 32, 2, 11, + 79, 2, 11, 32, 2, 11, 84, 2, 11, 72, 2, 167, 134, 6, 85, 2, 11, 73, 2, + 21, 3, 78, 71, 32, 2, 11, 76, 2, 11, 73, 2, 11, 84, 2, 163, 130, 8, 84, + 4, 29, 5, 72, 85, 77, 66, 32, 4, 194, 14, 70, 143, 104, 83, 17, 11, 32, + 14, 56, 8, 77, 79, 86, 69, 77, 69, 78, 84, 251, 164, 8, 82, 12, 26, 45, + 139, 211, 7, 32, 10, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, + 25, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 4, 62, 67, 223, 40, 83, + 6, 38, 67, 28, 2, 84, 73, 195, 40, 83, 2, 173, 169, 8, 2, 85, 82, 2, 187, + 137, 9, 76, 38, 50, 73, 225, 3, 7, 79, 67, 65, 84, 73, 79, 78, 22, 32, 3, + 77, 66, 32, 171, 1, 80, 16, 56, 4, 67, 79, 77, 66, 29, 6, 76, 69, 78, 71, + 84, 72, 2, 241, 222, 5, 2, 73, 78, 14, 11, 45, 14, 150, 164, 9, 49, 2, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, 6, 58, 32, 153, 1, 9, 83, 32, 80, + 82, 69, 83, 83, 69, 68, 4, 116, 12, 76, 79, 87, 69, 82, 32, 79, 86, 69, + 82, 32, 85, 133, 135, 8, 11, 85, 80, 80, 69, 82, 32, 79, 86, 69, 82, 32, + 2, 11, 80, 2, 167, 228, 8, 80, 2, 29, 5, 32, 84, 79, 71, 69, 2, 11, 84, + 2, 231, 227, 8, 72, 16, 22, 32, 203, 1, 45, 12, 148, 1, 2, 72, 69, 240, + 177, 2, 3, 84, 79, 82, 168, 233, 4, 3, 68, 69, 80, 0, 3, 87, 73, 68, 189, + 124, 10, 76, 73, 77, 66, 83, 32, 68, 73, 71, 73, 4, 196, 13, 4, 65, 68, + 32, 78, 135, 232, 8, 73, 4, 52, 5, 70, 76, 79, 79, 82, 1, 4, 87, 65, 76, + 76, 2, 225, 224, 8, 5, 80, 76, 65, 78, 69, 156, 3, 64, 4, 85, 84, 72, 32, + 161, 5, 7, 86, 69, 77, 69, 78, 84, 45, 54, 186, 1, 67, 88, 5, 70, 82, 79, + 87, 78, 0, 5, 83, 77, 73, 76, 69, 56, 4, 75, 73, 83, 83, 36, 5, 79, 80, + 69, 78, 32, 192, 1, 5, 84, 69, 78, 83, 69, 165, 29, 5, 87, 82, 73, 78, + 75, 8, 48, 6, 76, 79, 83, 69, 68, 32, 147, 235, 7, 79, 6, 222, 2, 70, + 142, 38, 67, 47, 78, 7, 11, 32, 4, 22, 79, 203, 1, 87, 2, 251, 197, 7, + 80, 7, 11, 32, 4, 166, 1, 87, 83, 70, 18, 100, 4, 79, 86, 65, 76, 0, 9, + 82, 69, 67, 84, 65, 78, 71, 76, 69, 42, 87, 82, 70, 211, 197, 7, 67, 7, + 11, 32, 4, 26, 87, 255, 195, 7, 89, 2, 25, 4, 82, 73, 78, 75, 2, 131, + 134, 4, 76, 7, 11, 32, 4, 18, 70, 43, 83, 2, 17, 2, 79, 82, 2, 215, 128, + 8, 87, 2, 17, 2, 85, 67, 2, 147, 133, 4, 75, 230, 2, 192, 1, 9, 68, 73, + 65, 71, 79, 78, 65, 76, 32, 148, 1, 11, 70, 76, 79, 79, 82, 80, 76, 65, + 78, 69, 32, 184, 14, 6, 72, 73, 78, 71, 69, 32, 189, 2, 10, 87, 65, 76, + 76, 80, 76, 65, 78, 69, 32, 32, 56, 8, 66, 69, 84, 87, 69, 69, 78, 32, + 22, 65, 31, 84, 16, 18, 65, 31, 84, 8, 229, 28, 3, 87, 65, 89, 8, 201, + 28, 6, 79, 87, 65, 82, 68, 83, 150, 1, 208, 2, 24, 65, 82, 77, 32, 67, + 73, 82, 67, 76, 69, 32, 72, 73, 84, 84, 73, 78, 71, 32, 87, 65, 76, 76, + 32, 38, 66, 34, 67, 224, 1, 8, 70, 73, 78, 71, 69, 82, 32, 67, 52, 5, 72, + 85, 77, 80, 32, 184, 3, 5, 76, 79, 79, 80, 32, 198, 1, 83, 108, 7, 84, + 82, 73, 80, 76, 69, 32, 110, 87, 206, 12, 68, 198, 2, 80, 154, 6, 90, + 231, 246, 7, 74, 12, 178, 7, 76, 154, 9, 77, 39, 83, 8, 150, 17, 79, 151, + 242, 7, 69, 28, 86, 72, 20, 5, 85, 82, 86, 69, 32, 196, 29, 5, 79, 82, + 78, 69, 82, 211, 177, 7, 82, 2, 195, 224, 7, 69, 18, 80, 4, 67, 79, 77, + 66, 158, 8, 72, 230, 15, 76, 194, 172, 2, 77, 247, 182, 1, 83, 2, 11, 73, + 2, 187, 253, 3, 78, 6, 128, 9, 6, 73, 82, 67, 76, 69, 83, 239, 20, 79, + 18, 56, 8, 72, 73, 84, 84, 73, 78, 71, 32, 167, 250, 3, 83, 16, 72, 8, + 67, 69, 73, 76, 73, 78, 71, 32, 101, 6, 70, 76, 79, 79, 82, 32, 8, 56, 5, + 76, 65, 82, 71, 69, 1, 5, 83, 77, 65, 76, 76, 4, 11, 32, 4, 206, 54, 84, + 135, 2, 68, 8, 88, 4, 83, 77, 65, 76, 12, 5, 76, 65, 82, 71, 69, 17, 7, + 84, 82, 73, 80, 76, 69, 32, 2, 11, 76, 2, 255, 18, 32, 4, 56, 5, 76, 65, + 82, 71, 69, 1, 5, 83, 77, 65, 76, 76, 2, 253, 52, 2, 32, 84, 18, 56, 8, + 72, 73, 84, 84, 73, 78, 71, 32, 239, 246, 3, 83, 16, 64, 7, 67, 69, 73, + 76, 73, 78, 71, 1, 5, 70, 76, 79, 79, 82, 8, 11, 32, 8, 22, 76, 191, 9, + 83, 4, 249, 22, 4, 65, 82, 71, 69, 12, 44, 6, 72, 65, 75, 73, 78, 71, + 243, 16, 73, 2, 21, 3, 32, 80, 65, 2, 149, 246, 3, 4, 82, 65, 76, 76, 8, + 76, 12, 65, 76, 84, 69, 82, 78, 65, 84, 73, 78, 71, 32, 138, 18, 87, 63, + 83, 4, 134, 18, 87, 255, 20, 77, 18, 80, 4, 65, 86, 69, 32, 169, 1, 11, + 82, 73, 83, 84, 32, 67, 73, 82, 67, 76, 69, 14, 30, 72, 102, 83, 171, 20, + 76, 8, 37, 7, 73, 84, 84, 73, 78, 71, 32, 8, 252, 2, 4, 67, 69, 73, 76, + 25, 5, 70, 76, 79, 79, 82, 4, 182, 156, 5, 78, 247, 225, 1, 77, 4, 209, + 5, 10, 32, 72, 73, 84, 84, 73, 78, 71, 32, 87, 14, 112, 3, 85, 80, 32, + 184, 1, 6, 68, 79, 87, 78, 32, 83, 197, 235, 5, 10, 83, 73, 68, 69, 32, + 84, 79, 32, 83, 73, 10, 40, 5, 68, 79, 87, 78, 32, 143, 1, 83, 8, 68, 8, + 65, 76, 84, 69, 82, 78, 65, 84, 230, 17, 76, 143, 222, 3, 83, 4, 21, 3, + 73, 78, 71, 4, 11, 32, 4, 190, 17, 76, 143, 222, 3, 83, 2, 187, 30, 69, + 162, 1, 148, 2, 11, 65, 82, 77, 32, 67, 73, 82, 67, 76, 69, 32, 98, 66, + 54, 67, 174, 4, 68, 100, 8, 70, 73, 78, 71, 69, 82, 32, 67, 64, 5, 72, + 85, 77, 80, 32, 60, 5, 76, 79, 79, 80, 32, 102, 80, 34, 83, 216, 1, 7, + 84, 82, 73, 80, 76, 69, 32, 218, 1, 87, 202, 2, 90, 231, 246, 7, 74, 8, + 18, 77, 39, 83, 4, 225, 13, 5, 69, 68, 73, 85, 77, 4, 11, 77, 4, 177, 13, + 3, 65, 76, 76, 12, 34, 79, 185, 13, 3, 69, 78, 68, 6, 183, 13, 88, 46, + 100, 6, 79, 82, 78, 69, 82, 32, 88, 4, 85, 82, 86, 69, 232, 11, 4, 72, + 69, 67, 75, 211, 177, 7, 82, 8, 54, 82, 194, 12, 76, 154, 167, 2, 77, + 247, 182, 1, 83, 2, 11, 79, 2, 159, 187, 5, 84, 30, 50, 32, 137, 2, 7, + 68, 32, 67, 82, 79, 83, 83, 26, 66, 72, 68, 2, 84, 72, 133, 5, 7, 81, 85, + 65, 82, 84, 69, 82, 12, 196, 5, 10, 65, 76, 70, 45, 67, 73, 82, 67, 76, + 69, 147, 14, 73, 6, 96, 2, 69, 78, 29, 18, 82, 69, 69, 45, 81, 85, 65, + 82, 84, 69, 82, 32, 67, 73, 82, 67, 76, 69, 2, 11, 32, 2, 131, 1, 83, 4, + 11, 32, 4, 238, 176, 2, 77, 247, 182, 1, 83, 8, 33, 6, 79, 85, 66, 76, + 69, 32, 8, 30, 83, 150, 4, 65, 75, 87, 2, 213, 210, 8, 3, 84, 82, 65, 6, + 32, 3, 73, 82, 67, 151, 9, 79, 4, 173, 7, 3, 76, 69, 83, 10, 142, 8, 76, + 166, 8, 72, 246, 158, 2, 77, 247, 182, 1, 83, 12, 68, 5, 83, 77, 65, 76, + 76, 142, 7, 76, 166, 8, 72, 247, 158, 2, 77, 5, 11, 32, 2, 207, 36, 68, + 6, 181, 6, 4, 69, 65, 75, 83, 12, 22, 73, 211, 36, 72, 10, 29, 5, 78, 71, + 76, 69, 32, 10, 52, 8, 83, 84, 82, 65, 73, 71, 72, 84, 207, 1, 87, 8, 11, + 32, 8, 42, 76, 194, 172, 2, 77, 247, 182, 1, 83, 4, 25, 4, 65, 82, 71, + 69, 5, 151, 221, 8, 83, 8, 26, 65, 74, 87, 63, 83, 4, 49, 10, 76, 84, 69, + 82, 78, 65, 84, 73, 78, 71, 5, 17, 2, 32, 87, 2, 29, 5, 82, 73, 83, 84, + 32, 2, 149, 208, 7, 2, 70, 76, 2, 37, 7, 84, 82, 65, 73, 71, 72, 84, 2, + 139, 20, 32, 26, 104, 4, 65, 86, 69, 32, 185, 1, 17, 82, 73, 83, 84, 32, + 67, 73, 82, 67, 76, 69, 32, 70, 82, 79, 78, 84, 22, 108, 6, 67, 85, 82, + 86, 69, 32, 140, 1, 13, 68, 73, 65, 71, 79, 78, 65, 76, 32, 80, 65, 84, + 72, 223, 8, 72, 12, 48, 4, 68, 79, 85, 66, 1, 4, 84, 82, 73, 80, 6, 85, + 2, 76, 69, 4, 11, 32, 4, 190, 30, 68, 43, 83, 6, 29, 5, 73, 71, 90, 65, + 71, 6, 11, 32, 6, 42, 76, 154, 167, 2, 77, 247, 182, 1, 83, 2, 159, 178, + 7, 65, 10, 40, 4, 79, 83, 69, 32, 247, 193, 7, 69, 8, 26, 67, 46, 78, 35, + 87, 2, 11, 79, 2, 193, 226, 7, 3, 78, 84, 65, 2, 197, 231, 7, 3, 69, 85, + 84, 4, 44, 3, 82, 73, 78, 237, 157, 1, 2, 73, 71, 2, 199, 177, 5, 75, 72, + 56, 7, 79, 84, 65, 84, 73, 79, 78, 205, 22, 2, 85, 66, 66, 60, 10, 32, + 77, 79, 68, 73, 70, 73, 69, 82, 45, 155, 1, 45, 30, 82, 49, 254, 240, 8, + 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 14, 250, 240, 8, 48, + 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 36, 104, 11, 70, 76, 79, 79, + 82, 80, 76, 65, 78, 69, 32, 133, 2, 10, 87, 65, 76, 76, 80, 76, 65, 78, + 69, 32, 18, 100, 4, 68, 79, 85, 66, 0, 4, 83, 73, 78, 71, 21, 11, 65, 76, + 84, 69, 82, 78, 65, 84, 73, 78, 71, 6, 17, 2, 76, 69, 7, 45, 9, 32, 72, + 73, 84, 84, 73, 78, 71, 32, 4, 32, 2, 67, 69, 33, 2, 70, 76, 2, 11, 73, + 2, 235, 156, 8, 76, 2, 243, 180, 8, 79, 18, 88, 8, 65, 76, 84, 69, 82, + 78, 65, 84, 44, 4, 68, 79, 85, 66, 1, 4, 83, 73, 78, 71, 6, 72, 4, 73, + 78, 71, 32, 163, 236, 8, 69, 6, 17, 2, 76, 69, 7, 11, 32, 4, 11, 72, 4, + 11, 73, 4, 33, 6, 84, 84, 73, 78, 71, 32, 4, 44, 5, 70, 82, 79, 78, 84, + 139, 249, 4, 67, 2, 253, 224, 6, 2, 32, 87, 30, 172, 1, 8, 72, 79, 85, + 76, 68, 69, 82, 32, 196, 1, 7, 81, 85, 69, 69, 90, 69, 32, 240, 1, 7, 85, + 82, 70, 65, 67, 69, 32, 240, 10, 5, 84, 82, 73, 75, 69, 183, 140, 5, 69, + 6, 100, 4, 72, 73, 80, 32, 225, 251, 3, 15, 84, 73, 76, 84, 73, 78, 71, + 32, 70, 82, 79, 77, 32, 87, 65, 4, 48, 4, 80, 79, 83, 73, 145, 205, 7, 2, + 83, 80, 2, 11, 84, 2, 145, 161, 5, 2, 73, 79, 12, 48, 6, 70, 76, 73, 67, + 75, 32, 18, 76, 35, 83, 2, 211, 16, 65, 4, 129, 1, 4, 65, 82, 71, 69, 6, + 34, 69, 65, 4, 77, 65, 76, 76, 2, 17, 2, 81, 85, 2, 21, 3, 69, 78, 84, 2, + 147, 219, 7, 73, 4, 11, 32, 4, 214, 11, 77, 187, 4, 83, 4, 22, 83, 135, + 11, 66, 2, 129, 208, 1, 4, 89, 77, 66, 79, 82, 58, 69, 218, 2, 79, 137, + 8, 6, 82, 65, 86, 69, 76, 45, 20, 76, 3, 69, 84, 72, 185, 1, 11, 78, 83, + 69, 32, 67, 72, 69, 69, 75, 83, 32, 15, 11, 32, 12, 56, 3, 79, 78, 32, + 86, 77, 177, 5, 4, 66, 73, 84, 69, 8, 56, 4, 76, 73, 80, 83, 1, 6, 84, + 79, 78, 71, 85, 69, 5, 11, 32, 2, 11, 77, 2, 149, 200, 3, 2, 79, 86, 6, + 50, 77, 220, 181, 4, 2, 72, 73, 255, 144, 3, 76, 2, 225, 168, 3, 2, 73, + 68, 28, 76, 5, 78, 71, 85, 69, 32, 196, 4, 4, 82, 83, 79, 45, 129, 2, 2, + 85, 67, 16, 192, 2, 7, 67, 69, 78, 84, 82, 69, 32, 52, 14, 73, 78, 83, + 73, 68, 69, 32, 77, 79, 85, 84, 72, 32, 82, 36, 4, 84, 73, 80, 32, 88, 7, + 76, 73, 67, 75, 73, 78, 71, 228, 186, 7, 14, 83, 84, 73, 67, 75, 73, 78, + 71, 32, 79, 85, 84, 32, 70, 153, 160, 1, 17, 77, 79, 86, 69, 83, 32, 65, + 71, 65, 73, 78, 83, 84, 32, 67, 72, 69, 4, 214, 1, 73, 129, 186, 3, 5, + 83, 84, 73, 67, 75, 2, 129, 203, 3, 4, 69, 76, 65, 88, 4, 84, 7, 66, 69, + 84, 87, 69, 69, 78, 41, 10, 84, 79, 85, 67, 72, 73, 78, 71, 32, 73, 2, + 11, 32, 2, 241, 165, 5, 2, 76, 73, 2, 241, 184, 5, 5, 78, 83, 73, 68, 69, + 6, 120, 10, 87, 65, 76, 76, 80, 76, 65, 78, 69, 32, 201, 205, 3, 14, 70, + 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 84, 87, 73, 4, 108, 8, 67, 85, + 82, 86, 69, 68, 32, 66, 169, 204, 1, 13, 83, 84, 82, 65, 73, 71, 72, 84, + 32, 83, 84, 82, 69, 2, 207, 203, 7, 69, 6, 11, 72, 6, 11, 32, 6, 30, 66, + 34, 77, 187, 4, 83, 2, 133, 133, 7, 3, 69, 84, 87, 2, 149, 2, 3, 85, 76, + 84, 34, 100, 11, 70, 76, 79, 79, 82, 80, 76, 65, 78, 69, 32, 29, 10, 87, + 65, 76, 76, 80, 76, 65, 78, 69, 32, 14, 178, 1, 82, 151, 2, 83, 20, 72, + 11, 65, 82, 77, 32, 83, 80, 73, 82, 65, 76, 32, 78, 82, 151, 2, 83, 6, + 30, 84, 134, 2, 68, 43, 83, 2, 11, 82, 2, 11, 73, 2, 151, 178, 7, 80, 12, + 41, 8, 79, 84, 65, 84, 73, 79, 78, 45, 12, 52, 5, 70, 76, 79, 79, 82, 1, + 4, 87, 65, 76, 76, 6, 33, 6, 80, 76, 65, 78, 69, 32, 6, 26, 65, 54, 68, + 43, 83, 2, 29, 5, 76, 84, 69, 82, 78, 2, 151, 200, 3, 65, 2, 17, 2, 79, + 85, 2, 151, 176, 7, 66, 2, 235, 175, 7, 73, 2, 11, 72, 2, 151, 198, 3, + 65, 2, 11, 73, 2, 195, 205, 7, 78, 10, 100, 6, 65, 66, 79, 86, 69, 32, + 162, 1, 79, 141, 148, 2, 10, 77, 73, 78, 85, 83, 32, 83, 73, 77, 73, 4, + 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, 83, 83, 2, 37, 7, 45, + 84, 72, 65, 78, 32, 65, 2, 25, 4, 66, 79, 86, 69, 2, 185, 241, 1, 2, 32, + 69, 4, 207, 189, 6, 82, 250, 1, 68, 3, 71, 76, 69, 212, 4, 5, 72, 65, 76, + 65, 32, 211, 171, 5, 69, 20, 50, 32, 209, 209, 8, 6, 45, 83, 72, 73, 70, + 84, 16, 138, 1, 67, 0, 9, 71, 82, 65, 80, 72, 73, 67, 32, 67, 116, 2, 72, + 73, 74, 76, 36, 4, 82, 73, 71, 72, 209, 254, 4, 4, 83, 72, 73, 70, 2, 41, + 8, 72, 65, 82, 65, 67, 84, 69, 82, 2, 37, 7, 32, 73, 78, 84, 82, 79, 68, + 2, 11, 85, 2, 247, 146, 8, 67, 2, 25, 4, 71, 72, 45, 82, 2, 185, 1, 7, + 69, 86, 69, 82, 83, 69, 68, 4, 32, 2, 69, 70, 109, 2, 79, 87, 2, 49, 10, + 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 2, 21, 3, 32, 65, 78, 2, 17, 2, + 71, 76, 2, 31, 69, 2, 17, 2, 45, 57, 2, 11, 32, 2, 11, 81, 2, 165, 176, + 2, 2, 85, 79, 228, 1, 168, 2, 8, 65, 82, 67, 72, 65, 73, 67, 32, 224, 1, + 15, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 114, 76, + 188, 8, 5, 83, 73, 71, 78, 32, 144, 1, 11, 86, 79, 87, 69, 76, 32, 83, + 73, 71, 78, 32, 209, 232, 2, 17, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, + 78, 32, 75, 85, 78, 68, 68, 40, 46, 68, 109, 7, 78, 85, 77, 66, 69, 82, + 32, 18, 25, 4, 73, 71, 73, 84, 18, 11, 32, 18, 154, 216, 6, 70, 30, 83, + 42, 84, 142, 87, 78, 14, 79, 227, 112, 69, 22, 130, 238, 2, 79, 162, 235, + 3, 69, 30, 70, 42, 78, 38, 83, 39, 84, 6, 18, 82, 63, 89, 4, 56, 6, 65, + 75, 65, 65, 82, 65, 233, 198, 8, 2, 69, 80, 2, 229, 198, 8, 3, 65, 78, + 83, 138, 1, 60, 6, 69, 84, 84, 69, 82, 32, 185, 186, 2, 3, 73, 84, 72, + 118, 246, 1, 65, 106, 69, 34, 73, 62, 85, 34, 77, 220, 1, 4, 68, 65, 78, + 84, 58, 79, 28, 8, 83, 65, 78, 89, 65, 75, 65, 32, 50, 70, 2, 72, 2, 82, + 2, 86, 2, 89, 32, 8, 84, 65, 65, 76, 85, 74, 65, 32, 29, 9, 75, 65, 78, + 84, 65, 74, 65, 32, 78, 34, 102, 69, 204, 1, 2, 76, 80, 144, 2, 5, 77, + 66, 65, 32, 66, 14, 65, 2, 73, 2, 85, 219, 135, 5, 89, 4, 230, 3, 69, + 219, 135, 5, 89, 12, 46, 76, 2, 82, 154, 3, 73, 219, 135, 5, 89, 4, 11, + 85, 4, 138, 3, 85, 219, 135, 5, 89, 28, 42, 65, 177, 1, 5, 85, 85, 82, + 68, 72, 22, 32, 2, 72, 65, 247, 137, 5, 89, 20, 41, 8, 65, 80, 82, 65, + 65, 78, 65, 32, 20, 70, 84, 138, 1, 68, 22, 66, 2, 67, 2, 71, 2, 74, 2, + 75, 3, 80, 4, 154, 1, 84, 15, 65, 6, 25, 4, 65, 74, 65, 32, 6, 102, 76, + 2, 78, 3, 83, 4, 86, 79, 219, 135, 5, 89, 8, 26, 68, 22, 71, 3, 74, 4, + 18, 68, 15, 65, 2, 11, 65, 2, 215, 135, 5, 89, 6, 26, 78, 21, 2, 83, 65, + 2, 89, 2, 65, 65, 4, 68, 11, 78, 89, 79, 79, 71, 65, 32, 78, 65, 65, 75, + 207, 134, 5, 89, 2, 173, 190, 8, 4, 83, 73, 75, 89, 8, 66, 65, 210, 162, + 4, 67, 249, 154, 4, 6, 86, 73, 83, 65, 82, 71, 4, 200, 116, 5, 76, 45, + 76, 65, 75, 253, 200, 7, 6, 78, 85, 83, 86, 65, 82, 34, 56, 5, 68, 73, + 71, 65, 32, 62, 71, 82, 75, 155, 2, 65, 12, 58, 71, 48, 4, 75, 79, 77, + 66, 118, 65, 26, 73, 19, 80, 4, 11, 65, 4, 236, 2, 3, 69, 84, 84, 67, 89, + 2, 11, 85, 2, 247, 187, 8, 86, 16, 52, 5, 69, 84, 84, 73, 32, 85, 4, 79, + 77, 66, 85, 6, 26, 65, 26, 73, 19, 80, 2, 213, 1, 2, 69, 68, 2, 203, 1, + 83, 2, 175, 1, 65, 10, 40, 2, 86, 65, 161, 142, 8, 2, 32, 68, 9, 29, 5, + 32, 72, 65, 65, 32, 6, 62, 65, 0, 6, 68, 73, 71, 65, 32, 65, 85, 3, 71, + 65, 89, 2, 17, 2, 69, 76, 2, 11, 65, 2, 17, 2, 45, 80, 2, 11, 73, 2, 171, + 153, 8, 76, 2, 129, 136, 6, 5, 65, 78, 85, 75, 73, 12, 84, 2, 32, 80, + 206, 2, 45, 137, 160, 6, 10, 84, 69, 69, 78, 32, 80, 79, 73, 78, 84, 8, + 140, 1, 23, 69, 84, 65, 76, 76, 69, 68, 32, 66, 76, 65, 67, 75, 32, 65, + 78, 68, 32, 87, 72, 73, 84, 69, 49, 7, 79, 73, 78, 84, 69, 68, 32, 2, 25, + 4, 32, 70, 76, 79, 2, 219, 239, 6, 82, 6, 78, 80, 150, 158, 6, 66, 153, + 134, 1, 9, 83, 84, 65, 82, 32, 87, 73, 84, 72, 2, 21, 3, 73, 78, 87, 2, + 149, 158, 6, 4, 72, 69, 69, 76, 2, 235, 213, 3, 80, 12, 46, 73, 102, 85, + 173, 157, 7, 3, 65, 84, 69, 4, 56, 8, 32, 65, 78, 68, 32, 83, 75, 73, + 139, 254, 7, 69, 2, 17, 2, 32, 66, 2, 151, 230, 7, 79, 6, 32, 2, 76, 76, + 131, 179, 8, 78, 5, 11, 32, 2, 49, 10, 65, 78, 68, 32, 67, 82, 79, 83, + 83, 66, 2, 11, 79, 2, 199, 159, 7, 78, 42, 46, 65, 198, 4, 69, 210, 1, + 73, 167, 1, 79, 14, 64, 5, 78, 84, 69, 68, 32, 253, 156, 6, 5, 86, 79, + 78, 73, 67, 12, 148, 1, 12, 69, 81, 85, 65, 76, 32, 84, 79, 32, 79, 82, + 32, 229, 1, 19, 78, 79, 82, 84, 72, 32, 65, 82, 82, 79, 87, 32, 87, 73, + 84, 72, 32, 72, 79, 8, 60, 7, 71, 82, 69, 65, 84, 69, 82, 1, 4, 76, 69, + 83, 83, 4, 29, 5, 45, 84, 72, 65, 78, 5, 11, 32, 2, 25, 4, 87, 73, 84, + 72, 2, 25, 4, 32, 68, 79, 84, 2, 17, 2, 32, 73, 2, 11, 78, 2, 11, 83, 2, + 231, 158, 6, 73, 4, 24, 2, 79, 75, 43, 82, 2, 17, 2, 69, 68, 2, 203, 216, + 5, 32, 2, 29, 5, 73, 90, 79, 78, 84, 2, 11, 65, 2, 155, 249, 5, 76, 12, + 80, 2, 69, 80, 168, 158, 8, 9, 85, 84, 72, 32, 79, 82, 32, 83, 80, 203, + 17, 68, 8, 40, 4, 73, 78, 71, 32, 167, 242, 7, 89, 6, 244, 233, 4, 8, 65, + 67, 67, 79, 77, 77, 79, 68, 162, 176, 2, 83, 167, 88, 70, 6, 76, 9, 67, + 69, 32, 79, 70, 32, 80, 73, 90, 21, 6, 71, 72, 84, 76, 89, 32, 2, 235, + 171, 8, 90, 4, 40, 2, 83, 77, 169, 229, 6, 2, 70, 82, 2, 219, 246, 6, 73, + 10, 18, 80, 75, 84, 6, 220, 232, 4, 9, 73, 78, 71, 32, 76, 65, 82, 71, + 69, 223, 196, 3, 69, 4, 26, 32, 219, 172, 8, 72, 2, 11, 77, 2, 141, 145, + 7, 3, 65, 67, 72, 150, 1, 34, 65, 158, 13, 73, 207, 7, 79, 116, 32, 2, + 76, 76, 151, 154, 7, 83, 114, 30, 32, 141, 12, 2, 69, 82, 110, 194, 2, + 65, 48, 3, 66, 76, 85, 0, 5, 79, 82, 65, 78, 71, 20, 2, 67, 79, 158, 1, + 68, 22, 69, 140, 2, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, 28, + 2, 76, 69, 34, 82, 218, 3, 83, 22, 84, 36, 2, 85, 80, 56, 4, 86, 69, 69, + 32, 196, 252, 1, 3, 71, 82, 69, 34, 72, 146, 4, 80, 206, 83, 78, 134, + 246, 2, 70, 83, 81, 6, 134, 214, 4, 77, 214, 49, 73, 255, 137, 1, 83, 2, + 131, 218, 6, 69, 12, 68, 7, 78, 84, 65, 73, 78, 83, 32, 134, 130, 2, 77, + 131, 214, 5, 76, 6, 36, 4, 65, 83, 32, 77, 175, 1, 87, 2, 11, 69, 2, 11, + 77, 2, 195, 233, 7, 66, 2, 151, 130, 2, 79, 12, 84, 9, 76, 69, 77, 69, + 78, 84, 32, 79, 70, 222, 129, 2, 81, 42, 88, 183, 227, 2, 77, 7, 17, 2, + 32, 87, 4, 25, 4, 73, 84, 72, 32, 4, 26, 86, 239, 243, 4, 79, 2, 197, + 253, 5, 21, 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 32, 65, 84, 32, + 69, 78, 68, 32, 79, 70, 2, 165, 207, 5, 2, 32, 67, 8, 130, 1, 70, 179, + 132, 2, 83, 40, 96, 3, 73, 71, 72, 64, 13, 79, 77, 65, 78, 32, 78, 85, + 77, 69, 82, 65, 76, 32, 251, 137, 6, 69, 6, 17, 2, 84, 32, 6, 238, 152, + 4, 67, 210, 3, 80, 147, 8, 84, 32, 62, 69, 50, 70, 62, 79, 46, 84, 214, + 174, 6, 83, 183, 87, 78, 4, 26, 76, 255, 247, 7, 73, 2, 231, 175, 6, 69, + 8, 26, 73, 191, 169, 7, 79, 6, 146, 26, 86, 159, 152, 6, 70, 6, 11, 78, + 6, 11, 69, 7, 151, 197, 2, 32, 8, 42, 87, 238, 174, 6, 72, 207, 162, 1, + 69, 4, 26, 69, 255, 160, 8, 79, 2, 167, 164, 7, 76, 2, 215, 209, 4, 69, + 4, 174, 253, 3, 87, 151, 134, 2, 73, 2, 253, 253, 4, 9, 45, 80, 79, 73, + 78, 84, 73, 78, 71, 2, 11, 87, 2, 21, 3, 73, 84, 72, 2, 11, 32, 2, 195, + 176, 4, 85, 4, 29, 5, 32, 84, 72, 65, 78, 5, 11, 32, 2, 11, 79, 2, 159, + 189, 1, 82, 32, 34, 76, 165, 150, 7, 2, 82, 75, 30, 40, 4, 73, 78, 71, + 32, 155, 158, 8, 69, 28, 112, 14, 67, 65, 84, 32, 70, 65, 67, 69, 32, 87, + 73, 84, 72, 32, 41, 10, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 4, 160, + 1, 2, 72, 69, 231, 246, 4, 79, 24, 86, 72, 108, 10, 79, 80, 69, 78, 32, + 77, 79, 85, 84, 72, 246, 1, 83, 215, 224, 2, 84, 6, 34, 69, 54, 79, 255, + 248, 7, 65, 2, 237, 156, 5, 8, 65, 82, 84, 45, 83, 72, 65, 80, 2, 159, + 212, 4, 82, 9, 29, 5, 32, 65, 78, 68, 32, 6, 26, 67, 58, 83, 71, 84, 2, + 17, 2, 79, 76, 2, 169, 145, 7, 4, 68, 32, 83, 87, 2, 11, 77, 2, 11, 73, + 2, 11, 76, 2, 161, 155, 5, 3, 73, 78, 71, 2, 37, 7, 73, 71, 72, 84, 76, + 89, 45, 2, 175, 154, 5, 67, 8, 64, 11, 77, 73, 76, 73, 78, 71, 32, 69, + 89, 69, 83, 175, 1, 85, 7, 29, 5, 32, 65, 78, 68, 32, 4, 52, 4, 72, 65, + 78, 68, 57, 5, 84, 72, 82, 69, 69, 2, 161, 243, 4, 9, 32, 67, 79, 86, 69, + 82, 73, 78, 71, 2, 181, 210, 4, 2, 32, 72, 2, 17, 2, 78, 71, 2, 185, 129, + 7, 4, 76, 65, 83, 83, 2, 239, 230, 4, 75, 18, 42, 65, 48, 2, 69, 69, 21, + 2, 79, 87, 6, 26, 75, 243, 138, 7, 73, 4, 195, 136, 1, 69, 2, 139, 142, + 7, 90, 10, 100, 7, 32, 67, 65, 80, 80, 69, 68, 28, 3, 77, 65, 78, 132, + 223, 2, 3, 66, 79, 65, 143, 143, 1, 70, 2, 153, 169, 4, 2, 32, 77, 5, 49, + 10, 32, 87, 73, 84, 72, 79, 85, 84, 32, 83, 2, 183, 248, 6, 78, 155, 3, + 162, 2, 67, 48, 2, 70, 84, 228, 1, 6, 71, 68, 73, 65, 78, 32, 244, 10, 3, + 76, 73, 68, 140, 2, 12, 79, 78, 32, 87, 73, 84, 72, 32, 82, 73, 71, 72, + 20, 11, 82, 65, 32, 83, 79, 77, 80, 69, 78, 71, 32, 234, 2, 85, 216, 8, + 6, 89, 79, 77, 66, 79, 32, 190, 240, 6, 77, 130, 135, 1, 72, 3, 83, 4, + 164, 233, 2, 3, 67, 69, 82, 163, 235, 4, 75, 10, 70, 32, 124, 9, 87, 65, + 82, 69, 45, 70, 85, 78, 67, 255, 133, 6, 66, 6, 58, 72, 44, 6, 73, 67, + 69, 32, 67, 82, 227, 200, 6, 83, 2, 11, 89, 2, 11, 80, 2, 211, 186, 6, + 72, 2, 179, 211, 6, 69, 2, 11, 84, 2, 235, 149, 6, 73, 84, 248, 1, 10, + 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 168, 2, 13, 73, 78, 68, 69, 80, + 69, 78, 68, 69, 78, 84, 32, 83, 20, 7, 76, 69, 84, 84, 69, 82, 32, 188, + 3, 7, 78, 85, 77, 66, 69, 82, 32, 113, 12, 80, 85, 78, 67, 84, 85, 65, + 84, 73, 79, 78, 32, 22, 140, 1, 3, 72, 79, 79, 20, 4, 76, 79, 78, 71, 50, + 83, 58, 84, 242, 231, 3, 68, 128, 243, 1, 4, 82, 69, 83, 72, 249, 14, 4, + 67, 85, 82, 86, 4, 239, 234, 5, 75, 2, 25, 4, 32, 72, 79, 79, 2, 187, + 219, 5, 75, 2, 21, 3, 84, 82, 79, 2, 11, 75, 2, 131, 219, 5, 69, 4, 233, + 222, 1, 2, 87, 79, 2, 151, 140, 7, 72, 42, 190, 1, 65, 50, 71, 34, 76, + 56, 5, 82, 69, 83, 72, 45, 2, 90, 34, 83, 66, 89, 198, 207, 1, 72, 234, + 5, 75, 174, 81, 66, 2, 70, 170, 225, 4, 78, 134, 2, 84, 2, 87, 218, 103, + 80, 171, 4, 77, 4, 26, 76, 171, 138, 7, 89, 2, 131, 215, 1, 69, 2, 11, + 73, 2, 171, 244, 2, 77, 4, 26, 65, 167, 158, 6, 69, 2, 233, 208, 1, 2, + 77, 69, 2, 11, 65, 2, 175, 137, 7, 89, 6, 26, 65, 131, 137, 7, 72, 4, + 222, 199, 1, 77, 235, 147, 6, 68, 2, 239, 207, 1, 79, 8, 18, 79, 59, 84, + 4, 11, 78, 4, 11, 69, 5, 11, 32, 2, 183, 171, 2, 72, 4, 214, 151, 6, 87, + 155, 160, 1, 69, 10, 66, 67, 0, 6, 72, 65, 76, 70, 32, 67, 53, 4, 84, 87, + 79, 32, 2, 21, 3, 73, 82, 67, 2, 225, 246, 4, 2, 76, 69, 6, 100, 13, 86, + 69, 82, 84, 73, 67, 65, 76, 32, 66, 65, 82, 83, 13, 8, 67, 73, 82, 67, + 76, 69, 83, 32, 5, 11, 32, 2, 229, 178, 6, 4, 87, 73, 84, 72, 8, 30, 32, + 189, 1, 2, 85, 83, 4, 93, 21, 81, 85, 73, 76, 84, 32, 83, 81, 85, 65, 82, + 69, 32, 79, 82, 78, 65, 77, 69, 78, 84, 5, 11, 32, 2, 11, 73, 2, 17, 2, + 78, 32, 2, 11, 66, 2, 237, 224, 6, 4, 76, 65, 67, 75, 5, 197, 209, 4, 7, + 32, 87, 73, 84, 72, 32, 79, 2, 231, 132, 4, 84, 70, 52, 7, 76, 69, 84, + 84, 69, 82, 32, 191, 142, 6, 68, 50, 198, 1, 69, 32, 2, 77, 65, 30, 78, + 194, 146, 1, 66, 2, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 80, + 2, 82, 2, 83, 2, 84, 2, 86, 2, 89, 242, 236, 6, 65, 2, 73, 2, 79, 3, 85, + 4, 234, 255, 7, 69, 147, 1, 72, 4, 218, 128, 8, 69, 3, 72, 6, 190, 146, + 1, 71, 2, 89, 243, 236, 6, 65, 58, 104, 3, 84, 72, 32, 149, 172, 6, 17, 78, 68, 32, 82, 69, 67, 79, 82, 68, 73, 78, 71, 32, 67, 79, 80, 89, 56, 60, 5, 69, 65, 83, 84, 32, 253, 2, 5, 87, 69, 83, 84, 32, 30, 96, 5, 65, - 82, 82, 79, 87, 194, 4, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, 18, 83, - 247, 58, 87, 13, 11, 32, 10, 68, 2, 65, 78, 38, 67, 120, 2, 84, 79, 154, - 2, 87, 203, 134, 5, 70, 2, 253, 2, 5, 68, 32, 83, 79, 85, 2, 37, 7, 82, - 79, 83, 83, 73, 78, 71, 2, 17, 2, 32, 78, 2, 17, 2, 79, 82, 2, 173, 133, - 5, 5, 84, 72, 32, 69, 65, 2, 17, 2, 32, 67, 2, 243, 241, 3, 79, 26, 96, - 5, 65, 82, 82, 79, 87, 198, 1, 80, 130, 1, 84, 138, 197, 4, 66, 38, 68, - 18, 83, 247, 58, 87, 9, 11, 32, 6, 52, 5, 65, 78, 68, 32, 78, 74, 87, - 203, 134, 5, 70, 2, 17, 2, 79, 82, 2, 21, 3, 84, 72, 32, 2, 137, 131, 5, - 2, 87, 69, 2, 21, 3, 73, 84, 72, 2, 11, 32, 2, 251, 151, 6, 72, 6, 41, 8, - 79, 73, 78, 84, 73, 78, 71, 32, 6, 42, 86, 242, 160, 4, 76, 155, 158, 2, - 66, 2, 17, 2, 73, 78, 2, 199, 160, 4, 69, 4, 73, 16, 82, 73, 65, 78, 71, + 82, 82, 79, 87, 194, 4, 80, 130, 1, 84, 190, 207, 4, 66, 38, 68, 18, 83, + 251, 58, 87, 13, 11, 32, 10, 68, 2, 65, 78, 38, 67, 120, 2, 84, 79, 154, + 2, 87, 131, 145, 5, 70, 2, 253, 2, 5, 68, 32, 83, 79, 85, 2, 37, 7, 82, + 79, 83, 83, 73, 78, 71, 2, 17, 2, 32, 78, 2, 17, 2, 79, 82, 2, 229, 143, + 5, 5, 84, 72, 32, 69, 65, 2, 17, 2, 32, 67, 2, 167, 251, 3, 79, 26, 96, + 5, 65, 82, 82, 79, 87, 198, 1, 80, 130, 1, 84, 190, 207, 4, 66, 38, 68, + 18, 83, 251, 58, 87, 9, 11, 32, 6, 52, 5, 65, 78, 68, 32, 78, 74, 87, + 131, 145, 5, 70, 2, 17, 2, 79, 82, 2, 21, 3, 84, 72, 32, 2, 193, 141, 5, + 2, 87, 69, 2, 21, 3, 73, 84, 72, 2, 11, 32, 2, 231, 163, 6, 72, 6, 41, 8, + 79, 73, 78, 84, 73, 78, 71, 32, 6, 42, 86, 166, 171, 4, 76, 219, 159, 2, + 66, 2, 17, 2, 73, 78, 2, 251, 170, 4, 69, 4, 73, 16, 82, 73, 65, 78, 71, 76, 69, 45, 72, 69, 65, 68, 69, 68, 32, 65, 4, 25, 4, 82, 82, 79, 87, 5, - 11, 32, 2, 203, 137, 5, 84, 166, 1, 220, 3, 23, 67, 76, 85, 83, 84, 69, + 11, 32, 2, 131, 148, 5, 84, 166, 1, 220, 3, 23, 67, 76, 85, 83, 84, 69, 82, 45, 73, 78, 73, 84, 73, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 40, 21, 70, 73, 78, 65, 76, 32, 67, 79, 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 112, 27, 72, 69, 65, 68, 32, 77, 65, 82, 75, 32, 87, 73, 84, 72, 32, 77, 79, 79, 78, 32, 65, 78, 68, 32, 83, 85, 78, 104, 7, 76, 69, 84, 84, 69, 82, 32, 180, 1, 5, 77, 65, 82, 75, 32, 54, 83, 100, 8, - 84, 69, 82, 77, 73, 78, 65, 76, 40, 6, 86, 79, 87, 69, 76, 32, 207, 132, - 4, 71, 8, 142, 158, 7, 83, 138, 69, 76, 3, 82, 24, 138, 197, 3, 83, 134, - 165, 2, 78, 226, 248, 1, 45, 186, 2, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, + 84, 69, 82, 77, 73, 78, 65, 76, 40, 6, 86, 79, 87, 69, 76, 32, 131, 143, + 4, 71, 8, 150, 172, 7, 83, 138, 69, 76, 3, 82, 24, 206, 226, 3, 83, 226, + 146, 2, 78, 202, 251, 1, 45, 186, 2, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 77, 3, 82, 7, 29, 5, 32, 65, 78, 68, 32, 4, 50, 70, 1, 8, 84, 82, 73, 80, - 76, 69, 32, 70, 2, 235, 212, 5, 76, 82, 230, 188, 2, 68, 138, 222, 4, 75, + 76, 69, 32, 70, 2, 139, 224, 5, 76, 82, 162, 194, 2, 68, 214, 230, 4, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 74, 2, 80, 2, 90, 138, - 69, 45, 2, 72, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 8, 222, - 140, 4, 80, 206, 142, 3, 68, 58, 83, 63, 84, 10, 40, 4, 73, 71, 78, 32, - 139, 159, 7, 85, 8, 246, 168, 5, 74, 158, 2, 86, 122, 85, 187, 240, 1, - 65, 4, 237, 223, 3, 5, 32, 77, 65, 82, 75, 22, 44, 5, 83, 73, 71, 78, 32, - 179, 158, 7, 76, 20, 88, 7, 86, 79, 67, 65, 76, 73, 67, 154, 159, 7, 65, - 26, 79, 2, 85, 162, 64, 69, 3, 73, 4, 11, 32, 4, 194, 223, 7, 76, 3, 82, + 69, 45, 2, 72, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 8, 146, + 151, 4, 80, 162, 146, 3, 68, 58, 83, 63, 84, 10, 40, 4, 73, 71, 78, 32, + 147, 173, 7, 85, 8, 186, 179, 5, 74, 158, 2, 86, 122, 85, 255, 243, 1, + 65, 4, 161, 233, 3, 5, 32, 77, 65, 82, 75, 22, 44, 5, 83, 73, 71, 78, 32, + 187, 172, 7, 76, 20, 88, 7, 86, 79, 67, 65, 76, 73, 67, 162, 173, 7, 65, + 26, 79, 2, 85, 162, 64, 69, 3, 73, 4, 11, 32, 4, 202, 237, 7, 76, 3, 82, 71, 122, 65, 206, 1, 69, 168, 5, 13, 72, 69, 82, 73, 67, 65, 76, 32, 65, 78, 71, 76, 69, 82, 73, 220, 1, 2, 76, 65, 91, 79, 17, 48, 4, 71, 72, 69, - 84, 22, 82, 147, 199, 7, 67, 2, 159, 202, 7, 84, 10, 24, 2, 75, 76, 59, - 83, 6, 26, 73, 191, 214, 5, 69, 2, 237, 213, 6, 2, 78, 71, 4, 17, 2, 69, - 32, 4, 214, 53, 72, 135, 3, 86, 22, 104, 2, 65, 75, 230, 3, 69, 68, 4, - 83, 77, 73, 76, 141, 214, 3, 9, 67, 75, 76, 69, 32, 70, 73, 76, 76, 12, - 70, 45, 64, 2, 69, 82, 153, 2, 8, 73, 78, 71, 32, 72, 69, 65, 68, 2, 161, - 193, 5, 11, 78, 79, 45, 69, 86, 73, 76, 32, 77, 79, 78, 9, 33, 6, 32, 87, + 84, 22, 82, 155, 213, 7, 67, 2, 167, 216, 7, 84, 10, 24, 2, 75, 76, 59, + 83, 6, 26, 73, 223, 225, 5, 69, 2, 245, 227, 6, 2, 78, 71, 4, 17, 2, 69, + 32, 4, 202, 53, 72, 135, 3, 86, 22, 104, 2, 65, 75, 230, 3, 69, 68, 4, + 83, 77, 73, 76, 193, 223, 3, 9, 67, 75, 76, 69, 32, 70, 73, 76, 76, 12, + 70, 45, 64, 2, 69, 82, 153, 2, 8, 73, 78, 71, 32, 72, 69, 65, 68, 2, 205, + 203, 5, 11, 78, 79, 45, 69, 86, 73, 76, 32, 77, 79, 78, 9, 33, 6, 32, 87, 73, 84, 72, 32, 6, 26, 67, 78, 79, 55, 84, 2, 33, 6, 65, 78, 67, 69, 76, - 76, 2, 241, 169, 7, 5, 65, 84, 73, 79, 78, 2, 209, 186, 4, 8, 78, 69, 32, - 83, 79, 85, 78, 68, 2, 25, 4, 72, 82, 69, 69, 2, 149, 197, 6, 10, 32, 83, - 79, 85, 78, 68, 32, 87, 65, 86, 2, 11, 32, 2, 185, 145, 6, 9, 73, 78, 32, - 83, 73, 76, 72, 79, 85, 4, 34, 68, 245, 245, 2, 2, 67, 72, 2, 11, 66, 2, - 247, 195, 6, 79, 2, 203, 208, 6, 79, 7, 45, 9, 32, 79, 80, 69, 78, 73, - 78, 71, 32, 4, 154, 172, 7, 76, 227, 42, 85, 10, 44, 3, 68, 69, 82, 41, - 4, 82, 65, 76, 32, 5, 17, 2, 32, 87, 2, 191, 154, 6, 69, 6, 80, 8, 67, - 65, 76, 69, 78, 68, 65, 82, 0, 4, 78, 79, 84, 69, 29, 2, 83, 72, 2, 141, - 143, 7, 2, 32, 80, 2, 223, 205, 5, 69, 4, 64, 10, 83, 72, 73, 78, 71, 32, - 83, 87, 69, 65, 211, 156, 6, 84, 2, 159, 193, 6, 84, 10, 90, 79, 60, 7, - 85, 84, 73, 78, 71, 32, 87, 192, 240, 2, 3, 82, 84, 83, 191, 173, 3, 78, - 4, 240, 87, 7, 76, 32, 79, 70, 32, 84, 72, 231, 251, 6, 78, 2, 11, 72, 2, - 231, 174, 6, 65, 222, 5, 26, 65, 191, 140, 7, 73, 220, 5, 28, 2, 82, 69, - 163, 69, 84, 218, 5, 30, 32, 245, 46, 2, 68, 32, 234, 3, 250, 1, 65, 104, + 76, 2, 249, 183, 7, 5, 65, 84, 73, 79, 78, 2, 133, 197, 4, 8, 78, 69, 32, + 83, 79, 85, 78, 68, 2, 25, 4, 72, 82, 69, 69, 2, 145, 209, 6, 10, 32, 83, + 79, 85, 78, 68, 32, 87, 65, 86, 2, 11, 32, 2, 165, 157, 6, 9, 73, 78, 32, + 83, 73, 76, 72, 79, 85, 4, 34, 68, 197, 251, 2, 2, 67, 72, 2, 11, 66, 2, + 183, 220, 6, 79, 2, 211, 222, 6, 79, 7, 45, 9, 32, 79, 80, 69, 78, 73, + 78, 71, 32, 4, 162, 186, 7, 76, 227, 42, 85, 10, 44, 3, 68, 69, 82, 41, + 4, 82, 65, 76, 32, 5, 17, 2, 32, 87, 2, 179, 166, 6, 69, 6, 80, 8, 67, + 65, 76, 69, 78, 68, 65, 82, 0, 4, 78, 79, 84, 69, 29, 2, 83, 72, 2, 149, + 157, 7, 2, 32, 80, 2, 255, 216, 5, 69, 4, 64, 10, 83, 72, 73, 78, 71, 32, + 83, 87, 69, 65, 199, 168, 6, 84, 2, 139, 205, 6, 84, 10, 90, 79, 60, 7, + 85, 84, 73, 78, 71, 32, 87, 144, 246, 2, 3, 82, 84, 83, 227, 179, 3, 78, + 4, 248, 87, 7, 76, 32, 79, 70, 32, 84, 72, 231, 137, 7, 78, 2, 11, 72, 2, + 219, 186, 6, 65, 222, 5, 26, 65, 199, 154, 7, 73, 220, 5, 28, 2, 82, 69, + 155, 69, 84, 218, 5, 30, 32, 233, 46, 2, 68, 32, 234, 3, 250, 1, 65, 104, 2, 86, 32, 94, 66, 158, 1, 67, 170, 1, 68, 102, 69, 194, 2, 70, 146, 2, - 71, 222, 1, 72, 214, 2, 73, 94, 75, 190, 4, 76, 102, 77, 210, 5, 78, 94, + 71, 222, 1, 72, 214, 2, 73, 94, 75, 182, 4, 76, 102, 77, 206, 5, 78, 94, 79, 178, 1, 80, 222, 3, 82, 154, 3, 83, 206, 2, 84, 74, 87, 230, 6, 89, - 251, 211, 6, 85, 16, 102, 32, 58, 80, 132, 73, 2, 78, 80, 144, 153, 2, 3, - 82, 85, 72, 194, 51, 65, 162, 185, 4, 77, 3, 85, 2, 21, 3, 79, 86, 69, 2, - 11, 82, 2, 175, 190, 7, 32, 4, 234, 184, 2, 69, 231, 138, 1, 65, 14, 98, - 65, 36, 4, 85, 83, 83, 89, 170, 148, 3, 73, 216, 74, 2, 79, 82, 138, 53, - 69, 195, 185, 3, 81, 4, 32, 2, 65, 82, 203, 205, 7, 82, 2, 167, 148, 3, - 69, 22, 98, 65, 30, 79, 158, 20, 77, 196, 183, 7, 7, 32, 79, 86, 69, 82, - 32, 75, 74, 85, 14, 67, 3, 68, 4, 206, 204, 7, 76, 3, 80, 5, 17, 2, 82, - 80, 2, 155, 194, 2, 79, 20, 82, 65, 162, 19, 77, 250, 205, 2, 69, 130, - 49, 79, 162, 185, 4, 66, 2, 74, 3, 76, 5, 135, 22, 65, 18, 66, 69, 22, - 82, 164, 25, 5, 83, 85, 75, 85, 85, 175, 177, 7, 86, 2, 151, 216, 5, 75, - 12, 52, 7, 65, 32, 78, 65, 77, 69, 32, 155, 202, 7, 71, 10, 132, 1, 4, - 72, 69, 73, 83, 20, 5, 84, 65, 73, 83, 89, 152, 22, 3, 77, 69, 73, 220, - 193, 2, 3, 82, 69, 73, 1, 4, 83, 89, 79, 85, 2, 183, 181, 7, 69, 2, 147, - 164, 7, 79, 12, 26, 79, 207, 200, 7, 77, 10, 60, 9, 85, 82, 32, 67, 79, - 82, 78, 69, 82, 227, 171, 7, 79, 8, 26, 32, 243, 199, 7, 83, 6, 128, 1, - 11, 66, 76, 65, 67, 75, 32, 84, 82, 73, 65, 78, 216, 62, 6, 68, 73, 65, - 71, 79, 78, 229, 199, 3, 5, 83, 65, 76, 84, 73, 2, 167, 137, 4, 71, 24, - 102, 65, 46, 73, 172, 12, 5, 85, 82, 65, 77, 85, 238, 149, 3, 72, 246, - 160, 4, 80, 186, 2, 66, 3, 89, 6, 130, 246, 6, 82, 222, 46, 78, 147, 33, - 76, 6, 42, 82, 158, 130, 5, 78, 199, 192, 2, 71, 2, 201, 210, 5, 2, 85, - 68, 30, 130, 1, 65, 22, 69, 54, 79, 62, 85, 242, 214, 1, 80, 156, 227, 5, - 10, 73, 82, 65, 71, 65, 78, 65, 32, 72, 79, 234, 8, 71, 3, 90, 5, 167, - 244, 1, 73, 4, 168, 11, 3, 75, 85, 84, 233, 232, 1, 2, 82, 85, 6, 26, 79, - 151, 195, 7, 78, 4, 170, 158, 7, 82, 235, 36, 78, 6, 54, 73, 144, 17, 4, - 65, 82, 65, 68, 175, 231, 5, 82, 2, 255, 211, 3, 73, 14, 54, 78, 212, 12, - 4, 77, 65, 71, 69, 159, 181, 7, 85, 7, 154, 210, 2, 73, 255, 219, 4, 84, - 58, 230, 1, 65, 48, 2, 89, 85, 20, 3, 73, 82, 79, 78, 77, 90, 79, 60, 2, - 85, 82, 144, 8, 2, 69, 69, 134, 146, 3, 72, 238, 153, 3, 67, 232, 120, 3, - 32, 79, 72, 162, 14, 80, 186, 2, 66, 2, 71, 2, 75, 2, 76, 2, 84, 2, 86, - 3, 87, 9, 22, 82, 131, 98, 73, 4, 22, 79, 163, 21, 65, 2, 183, 252, 4, - 82, 9, 252, 16, 3, 77, 69, 69, 140, 4, 2, 87, 65, 129, 171, 2, 3, 71, 85, - 82, 9, 11, 32, 6, 22, 67, 199, 14, 83, 4, 22, 65, 179, 6, 85, 2, 233, - 233, 4, 2, 80, 73, 4, 18, 79, 23, 82, 2, 243, 158, 7, 80, 2, 207, 129, 6, - 85, 4, 164, 206, 5, 4, 85, 90, 69, 73, 215, 68, 79, 12, 62, 79, 244, 13, - 2, 69, 70, 222, 174, 7, 77, 2, 78, 3, 88, 4, 150, 135, 6, 90, 187, 181, - 1, 71, 82, 162, 1, 32, 70, 65, 122, 66, 22, 69, 54, 73, 110, 77, 68, 2, - 85, 32, 78, 86, 2, 87, 222, 147, 3, 72, 250, 153, 3, 79, 254, 134, 1, 80, - 186, 2, 71, 2, 76, 3, 83, 10, 34, 79, 242, 2, 67, 139, 8, 83, 6, 198, 10, - 86, 199, 159, 7, 72, 13, 54, 73, 44, 2, 78, 83, 206, 84, 82, 215, 234, 4, - 72, 4, 240, 202, 5, 2, 75, 85, 179, 202, 1, 82, 2, 243, 233, 6, 89, 5, - 223, 171, 2, 32, 6, 28, 2, 71, 65, 251, 10, 69, 5, 171, 233, 6, 84, 8, - 42, 75, 20, 2, 82, 73, 199, 184, 7, 76, 2, 183, 147, 6, 85, 5, 11, 66, 2, - 11, 65, 2, 139, 255, 2, 65, 7, 11, 32, 4, 22, 67, 139, 8, 83, 2, 11, 85, - 2, 155, 172, 2, 66, 16, 210, 183, 7, 65, 2, 70, 2, 71, 2, 76, 2, 77, 2, - 83, 2, 86, 3, 87, 5, 11, 32, 2, 11, 77, 2, 171, 244, 6, 69, 16, 70, 65, - 130, 12, 79, 150, 170, 7, 70, 2, 77, 2, 83, 2, 86, 3, 87, 5, 159, 151, 7, - 78, 12, 78, 78, 20, 7, 82, 73, 71, 73, 78, 65, 76, 150, 182, 2, 79, 139, - 255, 4, 86, 2, 199, 144, 7, 83, 6, 21, 3, 32, 79, 70, 7, 25, 4, 32, 79, - 82, 32, 4, 242, 96, 78, 51, 69, 46, 118, 65, 82, 69, 94, 73, 74, 79, 142, - 161, 7, 80, 218, 16, 67, 2, 70, 2, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 87, - 9, 38, 65, 229, 254, 3, 3, 32, 65, 77, 4, 236, 1, 2, 83, 69, 219, 140, 7, - 84, 8, 34, 69, 22, 78, 223, 147, 7, 83, 2, 139, 159, 7, 90, 4, 206, 227, - 1, 73, 155, 170, 5, 83, 6, 34, 75, 233, 3, 3, 65, 83, 85, 4, 230, 248, 2, - 85, 163, 185, 4, 79, 6, 34, 73, 22, 78, 21, 2, 83, 73, 2, 131, 195, 3, - 78, 2, 183, 146, 7, 68, 2, 241, 247, 6, 4, 84, 73, 79, 78, 22, 60, 2, 65, - 68, 114, 69, 62, 73, 134, 1, 85, 207, 221, 6, 79, 7, 21, 3, 32, 79, 86, - 4, 25, 4, 69, 82, 32, 83, 5, 17, 2, 32, 83, 2, 11, 81, 2, 133, 164, 2, 2, - 85, 65, 4, 36, 3, 78, 84, 79, 183, 138, 7, 77, 2, 179, 219, 5, 71, 6, 40, - 2, 71, 72, 62, 84, 235, 171, 7, 82, 2, 233, 238, 3, 10, 84, 32, 79, 80, - 69, 78, 32, 66, 79, 88, 2, 253, 244, 2, 2, 84, 79, 4, 228, 1, 2, 85, 66, - 163, 233, 4, 80, 22, 138, 1, 65, 72, 3, 69, 78, 84, 28, 11, 80, 73, 82, - 65, 76, 32, 70, 82, 79, 77, 32, 152, 188, 2, 3, 73, 82, 73, 214, 239, 4, - 82, 3, 86, 4, 48, 2, 73, 75, 149, 173, 2, 4, 78, 84, 73, 73, 2, 251, 242, - 2, 85, 4, 134, 172, 7, 73, 3, 79, 8, 18, 66, 43, 84, 4, 201, 128, 7, 5, - 79, 84, 84, 79, 77, 4, 11, 79, 4, 151, 128, 7, 80, 6, 42, 65, 186, 135, - 3, 72, 207, 211, 3, 79, 2, 201, 245, 6, 2, 82, 71, 34, 50, 65, 20, 4, 73, - 84, 72, 32, 131, 170, 7, 66, 2, 215, 187, 3, 84, 30, 214, 1, 66, 40, 3, + 143, 226, 6, 85, 16, 102, 32, 58, 80, 252, 72, 2, 78, 80, 204, 158, 2, 3, + 82, 85, 72, 186, 51, 65, 254, 193, 4, 77, 3, 85, 2, 21, 3, 79, 86, 69, 2, + 11, 82, 2, 183, 204, 7, 32, 4, 166, 190, 2, 69, 223, 142, 1, 65, 14, 98, + 65, 36, 4, 85, 83, 83, 89, 214, 153, 3, 73, 232, 78, 2, 79, 82, 130, 54, + 69, 151, 189, 3, 81, 4, 32, 2, 65, 82, 211, 219, 7, 82, 2, 211, 153, 3, + 69, 22, 98, 65, 30, 79, 146, 20, 77, 216, 197, 7, 7, 32, 79, 86, 69, 82, + 32, 75, 74, 85, 14, 67, 3, 68, 4, 214, 218, 7, 76, 3, 80, 5, 17, 2, 82, + 80, 2, 215, 199, 2, 79, 20, 82, 65, 150, 19, 77, 186, 211, 2, 69, 250, + 48, 79, 254, 193, 4, 66, 2, 74, 3, 76, 5, 251, 21, 65, 18, 66, 69, 22, + 82, 152, 25, 5, 83, 85, 75, 85, 85, 195, 191, 7, 86, 2, 183, 227, 5, 75, + 12, 52, 7, 65, 32, 78, 65, 77, 69, 32, 163, 216, 7, 71, 10, 132, 1, 4, + 72, 69, 73, 83, 20, 5, 84, 65, 73, 83, 89, 140, 22, 3, 77, 69, 73, 156, + 199, 2, 3, 82, 69, 73, 1, 4, 83, 89, 79, 85, 2, 191, 195, 7, 69, 2, 155, + 178, 7, 79, 12, 26, 79, 215, 214, 7, 77, 10, 60, 9, 85, 82, 32, 67, 79, + 82, 78, 69, 82, 235, 185, 7, 79, 8, 26, 32, 251, 213, 7, 83, 6, 128, 1, + 11, 66, 76, 65, 67, 75, 32, 84, 82, 73, 65, 78, 208, 62, 6, 68, 73, 65, + 71, 79, 78, 161, 210, 3, 5, 83, 65, 76, 84, 73, 2, 219, 147, 4, 71, 24, + 102, 65, 46, 73, 160, 12, 5, 85, 82, 65, 77, 85, 134, 233, 5, 72, 242, + 219, 1, 80, 186, 2, 66, 3, 89, 6, 138, 132, 7, 82, 222, 46, 78, 147, 33, + 76, 6, 42, 82, 214, 140, 5, 78, 151, 196, 2, 71, 2, 233, 221, 5, 2, 85, + 68, 30, 130, 1, 65, 22, 69, 54, 79, 62, 85, 142, 191, 3, 80, 136, 137, 4, + 10, 73, 82, 65, 71, 65, 78, 65, 32, 72, 79, 234, 8, 71, 3, 90, 5, 183, + 249, 1, 73, 4, 156, 11, 3, 75, 85, 84, 133, 238, 1, 2, 82, 85, 6, 26, 79, + 159, 209, 7, 78, 4, 178, 172, 7, 82, 235, 36, 78, 6, 54, 73, 132, 17, 4, + 65, 82, 65, 68, 167, 243, 5, 82, 2, 187, 221, 3, 73, 14, 54, 78, 200, 12, + 4, 77, 65, 71, 69, 179, 195, 7, 85, 7, 206, 215, 2, 73, 211, 228, 4, 84, + 58, 234, 1, 65, 56, 3, 73, 82, 79, 78, 77, 90, 79, 60, 2, 85, 82, 140, 8, + 2, 69, 69, 208, 143, 2, 2, 89, 85, 206, 213, 3, 72, 206, 82, 67, 132, + 123, 3, 32, 79, 72, 162, 14, 80, 186, 2, 66, 2, 71, 2, 75, 2, 76, 2, 84, + 2, 86, 3, 87, 9, 22, 82, 135, 98, 73, 4, 166, 21, 65, 183, 132, 2, 79, 9, + 248, 16, 3, 77, 69, 69, 140, 4, 2, 87, 65, 193, 176, 2, 3, 71, 85, 82, 9, + 11, 32, 6, 22, 67, 195, 14, 83, 4, 22, 65, 175, 6, 85, 2, 169, 244, 4, 2, + 80, 73, 4, 18, 79, 23, 82, 2, 131, 173, 7, 80, 2, 203, 141, 6, 85, 4, + 204, 217, 5, 4, 85, 90, 69, 73, 171, 69, 79, 12, 62, 79, 240, 13, 2, 69, + 70, 242, 188, 7, 77, 2, 78, 3, 88, 4, 146, 147, 6, 90, 207, 183, 1, 71, + 82, 158, 1, 32, 70, 65, 122, 66, 22, 69, 54, 73, 110, 77, 68, 2, 85, 32, + 78, 86, 2, 87, 246, 230, 5, 72, 218, 82, 79, 154, 137, 1, 80, 186, 2, 71, + 2, 76, 3, 83, 10, 34, 79, 242, 2, 67, 139, 8, 83, 6, 198, 10, 86, 219, + 173, 7, 72, 13, 54, 73, 44, 2, 78, 83, 226, 84, 82, 239, 245, 4, 72, 4, + 156, 214, 5, 2, 75, 85, 155, 205, 1, 82, 2, 135, 248, 6, 89, 5, 167, 177, + 2, 32, 6, 28, 2, 71, 65, 251, 10, 69, 5, 191, 247, 6, 84, 8, 42, 75, 20, + 2, 82, 73, 219, 198, 7, 76, 2, 183, 159, 6, 85, 5, 11, 66, 2, 11, 65, 2, + 195, 132, 3, 65, 7, 11, 32, 4, 22, 67, 139, 8, 83, 2, 11, 85, 2, 227, + 177, 2, 66, 16, 230, 197, 7, 65, 2, 70, 2, 71, 2, 76, 2, 77, 2, 83, 2, + 86, 3, 87, 5, 11, 32, 2, 11, 77, 2, 191, 130, 7, 69, 16, 70, 65, 130, 12, + 79, 170, 184, 7, 70, 2, 77, 2, 83, 2, 86, 3, 87, 5, 179, 165, 7, 78, 12, + 78, 78, 20, 7, 82, 73, 71, 73, 78, 65, 76, 214, 187, 2, 79, 223, 135, 5, + 86, 2, 219, 158, 7, 83, 6, 21, 3, 32, 79, 70, 7, 25, 4, 32, 79, 82, 32, + 4, 134, 97, 78, 51, 69, 46, 118, 65, 82, 69, 94, 73, 74, 79, 162, 175, 7, + 80, 218, 16, 67, 2, 70, 2, 72, 2, 77, 2, 82, 2, 83, 2, 86, 3, 87, 9, 38, + 65, 165, 137, 4, 3, 32, 65, 77, 4, 236, 1, 2, 83, 69, 239, 154, 7, 84, 8, + 34, 69, 22, 78, 243, 161, 7, 83, 2, 159, 173, 7, 90, 4, 234, 232, 1, 73, + 147, 179, 5, 83, 6, 34, 75, 233, 3, 3, 65, 83, 85, 4, 158, 254, 2, 85, + 255, 193, 4, 79, 6, 34, 73, 22, 78, 21, 2, 83, 73, 2, 203, 204, 3, 78, 2, + 203, 160, 7, 68, 2, 133, 134, 7, 4, 84, 73, 79, 78, 22, 60, 2, 65, 68, + 114, 69, 62, 73, 134, 1, 85, 227, 235, 6, 79, 7, 21, 3, 32, 79, 86, 4, + 25, 4, 69, 82, 32, 83, 5, 17, 2, 32, 83, 2, 11, 81, 2, 205, 169, 2, 2, + 85, 65, 4, 36, 3, 78, 84, 79, 203, 152, 7, 77, 2, 171, 231, 5, 71, 6, 40, + 2, 71, 72, 62, 84, 255, 185, 7, 82, 2, 169, 249, 3, 10, 84, 32, 79, 80, + 69, 78, 32, 66, 79, 88, 2, 181, 250, 2, 2, 84, 79, 4, 228, 1, 2, 85, 66, + 231, 243, 4, 80, 22, 138, 1, 65, 72, 3, 69, 78, 84, 28, 11, 80, 73, 82, + 65, 76, 32, 70, 82, 79, 77, 32, 216, 193, 2, 3, 73, 82, 73, 170, 248, 4, + 82, 3, 86, 4, 48, 2, 73, 75, 213, 178, 2, 4, 78, 84, 73, 73, 2, 179, 248, + 2, 85, 4, 154, 186, 7, 73, 3, 79, 8, 18, 66, 43, 84, 4, 221, 142, 7, 5, + 79, 84, 84, 79, 77, 4, 11, 79, 4, 171, 142, 7, 80, 6, 42, 65, 210, 218, + 5, 72, 203, 142, 1, 79, 2, 221, 131, 7, 2, 82, 71, 34, 50, 65, 20, 4, 73, + 84, 72, 32, 151, 184, 7, 66, 2, 159, 197, 3, 84, 30, 214, 1, 66, 40, 3, 68, 73, 65, 0, 5, 79, 82, 84, 72, 79, 94, 72, 54, 76, 126, 84, 28, 6, 85, - 80, 80, 69, 82, 32, 186, 1, 86, 178, 136, 4, 82, 173, 130, 2, 13, 67, 79, - 78, 84, 79, 85, 82, 69, 68, 32, 79, 85, 84, 2, 217, 140, 4, 5, 79, 84, - 84, 79, 77, 2, 11, 71, 2, 213, 175, 4, 15, 79, 78, 65, 76, 32, 67, 82, + 80, 80, 69, 82, 32, 186, 1, 86, 242, 146, 4, 82, 253, 131, 2, 13, 67, 79, + 78, 84, 79, 85, 82, 69, 68, 32, 79, 85, 84, 2, 153, 151, 4, 5, 79, 84, + 84, 79, 77, 2, 11, 71, 2, 153, 186, 4, 15, 79, 78, 65, 76, 32, 67, 82, 79, 83, 83, 72, 65, 84, 67, 72, 2, 11, 79, 2, 149, 3, 6, 82, 73, 90, 79, - 78, 84, 6, 44, 5, 79, 87, 69, 82, 32, 199, 138, 4, 69, 4, 44, 3, 76, 69, - 70, 1, 4, 82, 73, 71, 72, 2, 189, 1, 3, 84, 32, 68, 2, 165, 138, 4, 2, + 78, 84, 6, 44, 5, 79, 87, 69, 82, 32, 135, 149, 4, 69, 4, 44, 3, 76, 69, + 70, 1, 4, 82, 73, 71, 72, 2, 189, 1, 3, 84, 32, 68, 2, 229, 148, 4, 2, 79, 80, 8, 60, 5, 76, 69, 70, 84, 32, 37, 6, 82, 73, 71, 72, 84, 32, 4, - 70, 68, 185, 153, 4, 2, 84, 79, 4, 34, 68, 245, 171, 4, 2, 84, 79, 2, - 129, 137, 4, 7, 73, 65, 71, 79, 78, 65, 76, 2, 29, 5, 69, 82, 84, 73, 67, - 2, 225, 171, 4, 2, 65, 76, 6, 32, 2, 65, 65, 195, 217, 5, 85, 4, 214, - 254, 6, 82, 247, 5, 68, 240, 1, 226, 1, 67, 174, 9, 68, 34, 70, 88, 3, + 70, 68, 249, 163, 4, 2, 84, 79, 4, 34, 68, 185, 182, 4, 2, 84, 79, 2, + 193, 147, 4, 7, 73, 65, 71, 79, 78, 65, 76, 2, 29, 5, 69, 82, 84, 73, 67, + 2, 165, 182, 4, 2, 65, 76, 6, 32, 2, 65, 65, 187, 229, 5, 85, 4, 234, + 140, 7, 82, 247, 5, 68, 240, 1, 226, 1, 67, 174, 9, 68, 34, 70, 88, 3, 82, 73, 83, 142, 1, 72, 66, 75, 106, 76, 166, 1, 77, 38, 78, 34, 79, 94, - 80, 34, 83, 190, 2, 84, 156, 1, 5, 69, 73, 71, 72, 84, 22, 85, 90, 86, - 230, 249, 4, 65, 218, 107, 87, 223, 97, 73, 90, 128, 1, 21, 74, 75, 32, + 80, 34, 83, 190, 2, 84, 160, 1, 5, 69, 73, 71, 72, 84, 22, 85, 90, 86, + 154, 132, 5, 65, 162, 109, 87, 243, 99, 73, 90, 128, 1, 21, 74, 75, 32, 85, 78, 73, 70, 73, 69, 68, 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, - 198, 141, 6, 79, 243, 146, 1, 76, 86, 68, 2, 52, 69, 82, 53, 206, 2, 54, - 150, 2, 55, 166, 1, 56, 95, 57, 10, 50, 48, 234, 4, 65, 214, 242, 5, 56, - 223, 97, 50, 4, 182, 159, 7, 48, 3, 57, 30, 150, 1, 50, 42, 51, 38, 52, - 48, 2, 53, 66, 0, 2, 68, 69, 22, 57, 164, 4, 2, 56, 70, 222, 162, 3, 66, - 248, 205, 2, 2, 70, 56, 221, 97, 2, 49, 56, 6, 214, 4, 55, 142, 211, 6, - 49, 3, 52, 4, 210, 167, 3, 70, 143, 206, 2, 67, 4, 26, 48, 219, 167, 3, - 51, 2, 155, 157, 7, 56, 2, 135, 157, 7, 54, 4, 174, 167, 3, 50, 143, 243, + 190, 153, 6, 79, 143, 149, 1, 76, 86, 68, 2, 52, 69, 82, 53, 206, 2, 54, + 150, 2, 55, 166, 1, 56, 95, 57, 10, 50, 48, 234, 4, 65, 214, 254, 5, 56, + 243, 99, 50, 4, 202, 173, 7, 48, 3, 57, 30, 150, 1, 50, 42, 51, 38, 52, + 48, 2, 53, 66, 0, 2, 68, 69, 22, 57, 164, 4, 2, 56, 70, 158, 172, 3, 66, + 184, 208, 2, 2, 70, 56, 241, 99, 2, 49, 56, 6, 214, 4, 55, 162, 225, 6, + 49, 3, 52, 4, 146, 177, 3, 70, 207, 208, 2, 67, 4, 26, 48, 155, 177, 3, + 51, 2, 175, 171, 7, 56, 2, 155, 171, 7, 54, 4, 238, 176, 3, 50, 227, 247, 3, 49, 24, 86, 50, 46, 51, 50, 53, 34, 54, 16, 2, 55, 48, 28, 2, 70, 49, - 129, 2, 2, 69, 56, 6, 70, 57, 170, 165, 3, 53, 167, 186, 2, 52, 4, 26, - 53, 131, 166, 3, 48, 2, 175, 155, 7, 53, 4, 202, 2, 66, 143, 163, 3, 57, - 2, 171, 2, 50, 4, 234, 154, 7, 56, 3, 57, 2, 207, 154, 7, 52, 12, 74, 53, - 38, 57, 12, 2, 49, 50, 20, 2, 68, 52, 245, 150, 7, 2, 65, 55, 4, 230, - 163, 3, 51, 231, 244, 2, 49, 2, 11, 56, 2, 191, 153, 7, 49, 2, 171, 153, - 7, 50, 6, 50, 57, 20, 2, 68, 55, 141, 163, 3, 2, 67, 65, 2, 219, 162, 3, - 69, 2, 207, 152, 7, 48, 4, 152, 210, 6, 2, 49, 52, 233, 67, 2, 48, 52, 2, - 11, 79, 2, 143, 216, 3, 84, 10, 84, 3, 65, 76, 76, 104, 4, 79, 85, 82, - 32, 240, 5, 3, 73, 86, 69, 135, 161, 5, 82, 2, 57, 12, 73, 78, 71, 32, - 68, 73, 65, 71, 79, 78, 65, 76, 2, 11, 32, 2, 11, 83, 2, 147, 174, 5, 76, - 4, 230, 211, 3, 68, 199, 194, 3, 75, 8, 212, 213, 3, 2, 73, 45, 214, 135, - 3, 68, 222, 56, 67, 3, 86, 8, 56, 8, 65, 84, 65, 75, 65, 78, 65, 32, 199, - 131, 7, 69, 6, 226, 225, 6, 75, 214, 28, 68, 159, 20, 83, 60, 36, 5, 65, + 129, 2, 2, 69, 56, 6, 70, 57, 234, 174, 3, 53, 231, 188, 2, 52, 4, 26, + 53, 195, 175, 3, 48, 2, 195, 169, 7, 53, 4, 202, 2, 66, 207, 172, 3, 57, + 2, 171, 2, 50, 4, 254, 168, 7, 56, 3, 57, 2, 227, 168, 7, 52, 12, 74, 53, + 38, 57, 12, 2, 49, 50, 20, 2, 68, 52, 137, 165, 7, 2, 65, 55, 4, 166, + 173, 3, 51, 187, 249, 2, 49, 2, 11, 56, 2, 211, 167, 7, 49, 2, 191, 167, + 7, 50, 6, 50, 57, 20, 2, 68, 55, 205, 172, 3, 2, 67, 65, 2, 155, 172, 3, + 69, 2, 227, 166, 7, 48, 4, 172, 224, 6, 2, 49, 52, 233, 67, 2, 48, 52, 2, + 11, 79, 2, 207, 226, 3, 84, 10, 84, 3, 65, 76, 76, 104, 4, 79, 85, 82, + 32, 240, 5, 3, 73, 86, 69, 179, 172, 5, 82, 2, 57, 12, 73, 78, 71, 32, + 68, 73, 65, 71, 79, 78, 65, 76, 2, 11, 32, 2, 11, 83, 2, 191, 185, 5, 76, + 4, 166, 222, 3, 68, 155, 198, 3, 75, 8, 148, 224, 3, 2, 73, 45, 170, 139, + 3, 68, 222, 56, 67, 3, 86, 8, 56, 8, 65, 84, 65, 75, 65, 78, 65, 32, 219, + 145, 7, 69, 6, 246, 239, 6, 75, 214, 28, 68, 159, 20, 83, 60, 36, 5, 65, 84, 73, 78, 32, 79, 79, 54, 164, 5, 12, 83, 77, 65, 76, 76, 32, 76, 69, - 84, 84, 69, 82, 239, 123, 67, 6, 198, 210, 3, 71, 201, 170, 2, 3, 83, 83, - 76, 4, 178, 141, 5, 73, 139, 134, 2, 86, 4, 246, 148, 6, 69, 163, 126, + 84, 84, 69, 82, 167, 123, 67, 6, 134, 221, 3, 71, 153, 172, 2, 3, 83, 83, + 76, 4, 222, 152, 5, 73, 243, 136, 2, 86, 4, 138, 163, 6, 69, 163, 126, 71, 4, 140, 3, 15, 78, 69, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 87, - 69, 78, 235, 143, 7, 75, 4, 158, 2, 80, 131, 138, 5, 76, 20, 110, 69, - 146, 1, 72, 20, 2, 73, 88, 134, 251, 4, 65, 186, 66, 77, 222, 49, 81, - 246, 98, 79, 222, 61, 68, 3, 83, 4, 56, 7, 67, 79, 78, 68, 32, 83, 67, - 21, 3, 86, 69, 78, 2, 219, 188, 5, 82, 2, 29, 5, 32, 80, 79, 73, 78, 2, - 11, 84, 2, 203, 229, 5, 32, 2, 251, 143, 7, 86, 2, 17, 2, 84, 89, 2, 199, - 143, 7, 32, 8, 44, 4, 72, 82, 69, 69, 22, 87, 243, 4, 73, 2, 239, 200, 6, - 32, 4, 68, 13, 69, 78, 84, 89, 45, 84, 87, 79, 32, 80, 79, 73, 78, 19, - 79, 2, 223, 86, 84, 2, 139, 139, 7, 32, 4, 48, 6, 80, 32, 87, 73, 84, 72, - 191, 199, 6, 72, 2, 17, 2, 32, 69, 2, 247, 119, 88, 4, 150, 199, 6, 79, - 163, 70, 83, 2, 137, 237, 5, 6, 32, 66, 76, 65, 67, 75, 6, 234, 140, 7, - 50, 2, 51, 3, 65, 93, 134, 1, 65, 178, 6, 69, 236, 1, 10, 73, 67, 75, 32, - 70, 73, 71, 85, 82, 69, 182, 1, 79, 90, 82, 174, 4, 85, 238, 252, 6, 83, - 3, 88, 38, 92, 6, 70, 70, 32, 79, 70, 32, 106, 77, 82, 82, 206, 3, 84, - 210, 133, 3, 78, 243, 214, 2, 68, 4, 60, 8, 65, 69, 83, 67, 85, 76, 65, - 80, 21, 3, 72, 69, 82, 2, 163, 132, 5, 73, 2, 147, 246, 5, 77, 2, 21, 3, - 80, 69, 68, 2, 17, 2, 32, 69, 2, 229, 220, 1, 4, 78, 86, 69, 76, 24, 42, - 32, 205, 1, 5, 84, 32, 79, 70, 32, 12, 82, 69, 54, 79, 180, 235, 1, 8, - 65, 78, 68, 32, 67, 82, 69, 83, 139, 128, 2, 87, 2, 17, 2, 81, 85, 2, 11, - 65, 2, 171, 202, 6, 76, 4, 44, 5, 70, 32, 68, 65, 86, 251, 199, 3, 80, 2, - 163, 193, 6, 73, 12, 66, 71, 30, 83, 40, 4, 80, 82, 79, 84, 242, 77, 72, - 215, 101, 84, 2, 89, 4, 85, 65, 82, 68, 4, 26, 69, 191, 249, 1, 84, 2, - 11, 76, 2, 21, 3, 69, 67, 84, 2, 29, 5, 69, 68, 32, 65, 82, 2, 183, 131, - 7, 69, 4, 188, 152, 5, 10, 85, 69, 32, 79, 70, 32, 76, 73, 66, 69, 179, - 157, 1, 73, 8, 92, 2, 65, 77, 112, 9, 78, 79, 71, 82, 65, 80, 72, 73, 67, - 209, 214, 1, 4, 84, 72, 79, 83, 4, 54, 32, 197, 250, 5, 7, 73, 78, 71, - 32, 66, 79, 87, 2, 33, 6, 76, 79, 67, 79, 77, 79, 2, 143, 208, 1, 84, 2, - 217, 177, 4, 2, 32, 70, 11, 11, 32, 8, 60, 5, 87, 73, 84, 72, 32, 233, - 163, 2, 4, 76, 69, 65, 78, 4, 32, 4, 65, 82, 77, 83, 51, 68, 2, 25, 4, - 32, 82, 65, 73, 2, 203, 246, 1, 83, 2, 167, 235, 5, 82, 4, 44, 4, 67, 75, - 32, 67, 21, 3, 80, 87, 65, 2, 163, 250, 5, 72, 2, 131, 223, 6, 84, 20, - 66, 65, 108, 10, 69, 83, 83, 32, 79, 85, 84, 76, 73, 78, 79, 73, 6, 44, - 4, 73, 71, 72, 84, 45, 3, 87, 66, 69, 4, 184, 150, 3, 2, 32, 82, 143, - 211, 2, 78, 2, 243, 72, 82, 2, 17, 2, 69, 68, 2, 17, 2, 32, 87, 2, 169, - 233, 4, 4, 72, 73, 84, 69, 12, 76, 4, 78, 71, 32, 84, 52, 4, 80, 69, 68, - 32, 241, 8, 4, 67, 84, 76, 89, 2, 17, 2, 69, 82, 2, 181, 197, 6, 3, 77, - 73, 78, 8, 70, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, 13, 2, 85, - 80, 2, 33, 3, 79, 87, 78, 2, 11, 84, 2, 11, 45, 2, 169, 216, 5, 8, 80, - 79, 73, 78, 84, 73, 78, 71, 6, 100, 8, 68, 73, 79, 32, 77, 73, 67, 82, - 20, 9, 70, 70, 69, 68, 32, 70, 76, 65, 84, 187, 249, 6, 80, 2, 167, 209, - 5, 79, 2, 11, 66, 2, 243, 166, 4, 82, 162, 3, 150, 1, 66, 212, 2, 5, 67, - 67, 69, 69, 68, 192, 3, 8, 77, 77, 65, 84, 73, 79, 78, 32, 70, 78, 204, - 22, 3, 80, 69, 82, 204, 11, 2, 82, 70, 47, 83, 63, 11, 83, 60, 76, 6, 67, - 82, 73, 80, 84, 32, 136, 1, 3, 69, 84, 32, 93, 3, 84, 73, 84, 30, 118, - 76, 186, 29, 69, 166, 1, 80, 22, 82, 218, 211, 2, 77, 246, 149, 2, 70, - 30, 83, 42, 84, 62, 90, 238, 85, 78, 15, 79, 2, 207, 30, 69, 28, 56, 6, - 65, 66, 79, 86, 69, 32, 246, 33, 79, 159, 3, 87, 6, 142, 32, 83, 143, - 184, 5, 82, 2, 251, 176, 5, 85, 22, 11, 83, 23, 11, 32, 20, 128, 1, 6, - 65, 66, 79, 86, 69, 32, 156, 1, 7, 66, 85, 84, 32, 78, 79, 84, 32, 2, 79, - 82, 141, 146, 3, 5, 85, 78, 68, 69, 82, 12, 100, 4, 78, 79, 84, 32, 28, - 12, 83, 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 32, 218, 32, 65, 39, 69, - 4, 242, 32, 65, 167, 1, 69, 4, 250, 32, 69, 83, 78, 2, 93, 5, 32, 69, 81, - 85, 73, 4, 17, 2, 32, 69, 4, 17, 2, 81, 85, 4, 22, 73, 167, 33, 65, 2, - 173, 33, 6, 86, 65, 76, 69, 78, 84, 6, 242, 136, 3, 66, 136, 228, 1, 4, - 87, 73, 84, 72, 147, 142, 1, 84, 249, 1, 194, 1, 32, 72, 7, 68, 65, 78, - 69, 83, 69, 32, 204, 12, 4, 82, 73, 83, 69, 72, 5, 85, 87, 65, 82, 32, - 132, 174, 2, 14, 83, 69, 84, 32, 79, 86, 69, 82, 32, 66, 85, 73, 76, 68, - 159, 157, 3, 70, 4, 50, 87, 225, 197, 5, 6, 66, 69, 72, 73, 78, 68, 2, - 171, 180, 6, 73, 146, 1, 198, 2, 65, 20, 17, 67, 79, 78, 83, 79, 78, 65, - 78, 84, 32, 83, 73, 71, 78, 32, 80, 65, 172, 1, 7, 76, 69, 84, 84, 69, - 82, 32, 188, 3, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 66, - 73, 78, 68, 85, 32, 228, 1, 5, 83, 73, 71, 78, 32, 204, 1, 13, 86, 79, - 87, 69, 76, 32, 83, 73, 71, 78, 32, 80, 65, 183, 245, 4, 68, 2, 171, 152, - 3, 86, 10, 68, 2, 77, 73, 40, 2, 78, 89, 33, 7, 83, 65, 78, 71, 65, 78, - 32, 2, 17, 2, 78, 71, 2, 207, 228, 5, 75, 4, 142, 5, 65, 211, 155, 1, 73, - 4, 162, 235, 6, 77, 3, 87, 76, 246, 1, 65, 58, 70, 78, 76, 2, 82, 34, 83, - 254, 150, 3, 69, 254, 216, 1, 78, 238, 178, 1, 66, 2, 75, 138, 69, 67, 2, - 68, 2, 71, 2, 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 84, 2, 86, 2, 87, 2, 88, - 2, 89, 2, 90, 186, 2, 73, 2, 79, 3, 85, 7, 180, 171, 6, 6, 82, 67, 72, - 65, 73, 67, 147, 64, 69, 6, 44, 5, 73, 78, 65, 76, 32, 227, 234, 6, 65, - 4, 222, 234, 6, 75, 3, 77, 4, 218, 197, 6, 69, 235, 36, 65, 4, 234, 231, - 6, 89, 187, 2, 65, 16, 90, 66, 2, 68, 2, 75, 12, 3, 76, 69, 85, 38, 67, - 34, 80, 253, 229, 6, 3, 83, 85, 82, 2, 11, 65, 2, 253, 103, 5, 32, 83, - 65, 84, 65, 2, 11, 65, 2, 171, 164, 6, 75, 4, 172, 3, 3, 65, 78, 71, 189, - 162, 6, 3, 85, 82, 78, 10, 32, 2, 80, 65, 235, 145, 3, 86, 8, 28, 3, 77, - 65, 65, 23, 78, 2, 203, 230, 6, 69, 6, 18, 71, 71, 89, 4, 38, 76, 225, - 160, 6, 3, 87, 73, 83, 2, 209, 197, 5, 2, 65, 89, 2, 201, 227, 6, 2, 69, - 67, 12, 18, 77, 23, 78, 2, 187, 134, 3, 69, 10, 96, 3, 69, 85, 76, 34, - 79, 22, 89, 188, 228, 1, 3, 71, 72, 85, 201, 189, 2, 4, 65, 69, 76, 65, - 2, 11, 69, 2, 195, 148, 6, 85, 2, 231, 161, 4, 76, 2, 247, 151, 1, 85, 5, - 165, 161, 3, 13, 32, 79, 86, 69, 82, 32, 77, 79, 85, 78, 84, 65, 73, 88, - 88, 7, 76, 69, 84, 84, 69, 82, 32, 184, 6, 6, 83, 73, 71, 78, 32, 80, - 135, 236, 4, 68, 66, 142, 2, 65, 66, 67, 66, 68, 44, 3, 72, 65, 77, 22, - 74, 34, 75, 42, 78, 38, 79, 2, 85, 34, 80, 38, 82, 20, 4, 83, 72, 89, 69, - 34, 84, 112, 3, 86, 65, 82, 230, 169, 4, 71, 148, 57, 4, 76, 79, 65, 67, - 136, 89, 2, 73, 77, 210, 13, 89, 150, 27, 66, 214, 89, 69, 203, 28, 77, - 6, 42, 80, 238, 215, 5, 65, 255, 134, 1, 86, 2, 251, 248, 1, 80, 4, 40, - 2, 72, 69, 253, 204, 6, 2, 65, 82, 2, 243, 207, 5, 76, 4, 22, 69, 155, - 95, 79, 2, 219, 204, 6, 86, 2, 171, 193, 6, 83, 2, 11, 89, 2, 239, 222, - 6, 65, 6, 194, 172, 6, 76, 146, 48, 73, 99, 72, 4, 138, 190, 5, 71, 171, - 160, 1, 65, 2, 11, 84, 2, 147, 144, 1, 84, 4, 198, 189, 5, 72, 175, 161, - 1, 73, 2, 243, 185, 6, 69, 4, 242, 199, 6, 76, 215, 22, 82, 8, 42, 69, - 22, 72, 209, 187, 6, 2, 65, 83, 2, 171, 142, 1, 78, 4, 26, 65, 167, 185, - 5, 69, 2, 243, 201, 6, 82, 2, 255, 218, 6, 67, 2, 175, 190, 6, 86, 72, - 54, 83, 182, 237, 4, 72, 209, 66, 4, 86, 73, 76, 76, 68, 56, 6, 67, 82, - 73, 80, 84, 32, 229, 2, 3, 69, 84, 32, 34, 114, 69, 34, 76, 134, 1, 80, - 22, 82, 218, 211, 2, 77, 246, 149, 2, 70, 30, 83, 42, 84, 62, 90, 238, - 85, 78, 15, 79, 4, 174, 69, 81, 183, 235, 5, 73, 6, 88, 18, 65, 84, 73, - 78, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 31, 69, 4, - 170, 218, 6, 73, 3, 78, 2, 55, 70, 2, 191, 171, 3, 76, 2, 21, 3, 73, 71, - 72, 2, 11, 84, 2, 139, 215, 2, 32, 34, 148, 1, 6, 65, 66, 79, 86, 69, 32, - 96, 7, 66, 69, 83, 73, 68, 69, 32, 162, 1, 79, 158, 3, 87, 221, 189, 4, - 9, 80, 82, 69, 67, 69, 68, 73, 78, 71, 6, 26, 83, 135, 188, 2, 76, 4, 11, - 85, 4, 26, 80, 151, 236, 4, 66, 2, 145, 236, 4, 2, 69, 82, 4, 108, 23, - 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 66, 89, 32, 68, 65, 83, 72, - 32, 87, 73, 84, 72, 23, 83, 2, 17, 2, 32, 83, 2, 241, 234, 4, 2, 85, 66, - 16, 11, 70, 17, 11, 32, 14, 120, 6, 65, 66, 79, 86, 69, 32, 132, 1, 4, - 87, 73, 84, 72, 249, 210, 5, 11, 79, 82, 32, 69, 81, 85, 65, 76, 32, 84, - 79, 8, 34, 65, 38, 69, 18, 84, 67, 78, 2, 11, 76, 2, 113, 3, 77, 79, 83, - 2, 203, 62, 81, 2, 21, 3, 73, 76, 68, 2, 171, 148, 3, 69, 2, 17, 2, 32, - 78, 2, 11, 79, 2, 11, 84, 2, 11, 32, 2, 11, 69, 2, 17, 2, 81, 85, 2, 11, - 65, 2, 11, 76, 2, 219, 228, 2, 32, 6, 25, 4, 73, 84, 72, 32, 6, 96, 2, - 80, 76, 24, 14, 77, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, 73, 79, 78, - 183, 129, 6, 68, 2, 11, 85, 2, 11, 83, 2, 165, 164, 4, 5, 32, 83, 73, 71, - 78, 4, 144, 202, 4, 2, 65, 67, 215, 206, 1, 69, 4, 60, 9, 80, 69, 78, 83, - 73, 79, 78, 32, 82, 131, 189, 6, 72, 2, 21, 3, 65, 73, 76, 2, 187, 196, - 5, 87, 8, 26, 65, 98, 73, 35, 85, 4, 44, 5, 83, 72, 32, 65, 77, 227, 207, - 6, 78, 2, 253, 139, 6, 7, 80, 69, 82, 83, 65, 78, 68, 2, 11, 77, 2, 183, - 145, 6, 77, 2, 233, 145, 3, 2, 78, 71, 236, 2, 92, 11, 76, 79, 84, 73, - 32, 78, 65, 71, 82, 73, 32, 226, 5, 77, 194, 17, 78, 97, 2, 82, 73, 90, - 180, 1, 7, 76, 69, 84, 84, 69, 82, 32, 168, 2, 11, 80, 79, 69, 84, 82, - 89, 32, 77, 65, 82, 75, 56, 5, 83, 73, 71, 78, 32, 145, 1, 11, 86, 79, - 87, 69, 76, 32, 83, 73, 71, 78, 32, 64, 174, 1, 68, 46, 82, 34, 84, 166, - 91, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 198, 208, 5, 72, 2, 76, 2, 77, - 2, 78, 2, 83, 246, 30, 65, 2, 69, 2, 73, 2, 79, 3, 85, 8, 238, 91, 68, - 198, 208, 5, 72, 247, 30, 79, 4, 134, 172, 6, 82, 247, 30, 79, 8, 162, - 91, 84, 198, 208, 5, 72, 247, 30, 79, 8, 11, 45, 8, 162, 202, 6, 49, 2, - 50, 2, 51, 3, 52, 8, 46, 65, 70, 72, 165, 132, 6, 3, 68, 86, 73, 4, 64, - 10, 76, 84, 69, 82, 78, 65, 84, 69, 32, 72, 155, 132, 6, 78, 2, 193, 137, - 2, 2, 65, 83, 10, 242, 169, 6, 79, 246, 30, 65, 2, 69, 2, 73, 3, 85, 88, - 60, 8, 66, 79, 76, 32, 70, 79, 82, 32, 225, 16, 2, 77, 69, 86, 238, 2, - 66, 48, 2, 67, 65, 118, 68, 234, 3, 69, 130, 2, 70, 56, 8, 72, 79, 82, - 73, 90, 79, 78, 84, 0, 6, 86, 69, 82, 84, 73, 67, 44, 3, 76, 73, 78, 52, - 9, 77, 65, 82, 75, 83, 32, 67, 72, 65, 22, 78, 90, 65, 68, 3, 82, 69, 67, - 32, 5, 71, 82, 79, 85, 80, 0, 4, 85, 78, 73, 84, 22, 83, 205, 3, 15, 84, - 89, 80, 69, 32, 65, 32, 69, 76, 69, 67, 84, 82, 79, 78, 4, 190, 189, 4, - 69, 137, 201, 1, 3, 65, 67, 75, 4, 44, 3, 82, 82, 73, 189, 183, 1, 2, 78, - 67, 2, 33, 6, 65, 71, 69, 32, 82, 69, 2, 11, 84, 2, 215, 185, 1, 85, 20, - 24, 2, 65, 84, 55, 69, 2, 173, 5, 9, 65, 32, 76, 73, 78, 75, 32, 69, 83, - 18, 84, 4, 76, 69, 84, 69, 133, 2, 12, 86, 73, 67, 69, 32, 67, 79, 78, - 84, 82, 79, 76, 11, 11, 32, 8, 160, 1, 11, 82, 69, 67, 84, 65, 78, 71, - 85, 76, 65, 82, 0, 6, 83, 81, 85, 65, 82, 69, 154, 9, 70, 129, 185, 5, - 11, 77, 69, 68, 73, 85, 77, 32, 83, 72, 65, 68, 2, 61, 13, 32, 67, 72, - 69, 67, 75, 69, 82, 32, 66, 79, 65, 82, 2, 227, 193, 5, 68, 8, 11, 32, 8, - 162, 173, 1, 70, 162, 163, 3, 84, 183, 86, 79, 12, 22, 78, 207, 1, 83, - 10, 48, 5, 68, 32, 79, 70, 32, 137, 1, 2, 81, 85, 8, 18, 77, 31, 84, 2, - 245, 149, 5, 2, 69, 68, 6, 64, 11, 82, 65, 78, 83, 77, 73, 83, 83, 73, - 79, 78, 223, 107, 69, 5, 235, 207, 3, 32, 2, 155, 7, 73, 2, 209, 246, 4, - 2, 67, 65, 4, 36, 2, 73, 76, 73, 3, 79, 82, 77, 2, 191, 2, 69, 2, 229, - 160, 1, 6, 65, 76, 32, 84, 65, 66, 2, 11, 69, 2, 17, 2, 32, 70, 2, 163, - 177, 1, 69, 2, 227, 132, 5, 80, 6, 26, 69, 239, 180, 4, 85, 4, 56, 8, 71, - 65, 84, 73, 86, 69, 32, 65, 175, 162, 5, 87, 2, 21, 3, 67, 75, 78, 2, 21, - 3, 79, 87, 76, 2, 231, 135, 1, 69, 2, 11, 79, 2, 17, 2, 82, 68, 2, 163, - 141, 3, 32, 18, 180, 1, 7, 65, 77, 65, 82, 73, 84, 65, 60, 3, 72, 73, 70, - 52, 8, 84, 65, 82, 84, 32, 79, 70, 32, 64, 8, 85, 66, 83, 84, 73, 84, 85, - 84, 200, 1, 3, 89, 78, 67, 195, 249, 5, 80, 2, 25, 4, 78, 32, 83, 79, 2, - 11, 85, 2, 155, 252, 5, 82, 4, 17, 2, 84, 32, 4, 158, 191, 5, 79, 243, - 41, 73, 4, 22, 72, 215, 101, 84, 2, 17, 2, 69, 65, 2, 159, 231, 5, 68, 4, - 11, 69, 5, 11, 32, 2, 11, 70, 2, 21, 3, 79, 82, 77, 2, 17, 2, 32, 84, 2, - 203, 152, 6, 87, 2, 205, 249, 5, 2, 73, 67, 2, 11, 84, 2, 187, 165, 6, - 82, 7, 38, 67, 181, 171, 2, 3, 65, 71, 79, 2, 181, 133, 1, 9, 72, 82, 79, - 78, 79, 85, 83, 32, 73, 180, 1, 36, 3, 65, 67, 32, 199, 128, 5, 78, 178, - 1, 150, 3, 65, 52, 4, 66, 65, 82, 82, 32, 2, 67, 79, 80, 13, 68, 79, 84, - 84, 69, 68, 32, 90, 76, 65, 77, 65, 32, 118, 69, 90, 72, 204, 2, 7, 76, - 69, 84, 84, 69, 82, 32, 236, 9, 9, 79, 66, 76, 73, 81, 85, 69, 32, 76, - 28, 4, 80, 84, 72, 65, 0, 4, 90, 81, 65, 80, 110, 82, 88, 2, 83, 85, 174, - 2, 84, 140, 247, 4, 3, 77, 85, 83, 232, 22, 7, 70, 69, 77, 73, 78, 73, - 78, 253, 136, 1, 5, 81, 85, 83, 72, 83, 2, 11, 66, 2, 133, 201, 5, 5, 66, - 82, 69, 86, 73, 2, 11, 69, 2, 247, 176, 6, 75, 6, 38, 78, 165, 17, 4, 76, - 79, 78, 32, 2, 17, 2, 84, 82, 2, 219, 224, 5, 65, 4, 28, 3, 65, 78, 71, - 35, 72, 2, 11, 85, 2, 191, 143, 5, 76, 2, 11, 79, 2, 169, 220, 3, 5, 82, - 73, 90, 79, 78, 6, 60, 10, 78, 68, 32, 79, 70, 32, 80, 65, 82, 65, 151, - 14, 83, 2, 201, 11, 2, 71, 82, 14, 104, 8, 65, 82, 75, 76, 69, 65, 78, - 32, 132, 1, 4, 66, 65, 83, 65, 65, 7, 79, 82, 73, 90, 79, 78, 84, 6, 60, - 5, 65, 83, 84, 69, 82, 40, 4, 77, 69, 84, 79, 3, 79, 2, 17, 2, 73, 83, 2, - 155, 168, 4, 67, 2, 201, 167, 2, 2, 66, 69, 6, 164, 11, 8, 45, 69, 83, - 65, 83, 65, 32, 68, 151, 132, 4, 32, 2, 137, 221, 5, 2, 65, 76, 92, 154, - 2, 68, 102, 72, 32, 3, 76, 65, 77, 34, 77, 204, 1, 2, 80, 69, 142, 1, 82, - 78, 83, 144, 1, 8, 70, 73, 78, 65, 76, 32, 83, 69, 106, 65, 14, 75, 2, - 81, 34, 84, 40, 5, 71, 65, 77, 65, 76, 40, 4, 89, 85, 68, 72, 214, 74, - 66, 142, 172, 4, 90, 166, 44, 78, 134, 2, 87, 175, 126, 69, 4, 76, 14, - 79, 84, 76, 69, 83, 83, 32, 68, 65, 76, 65, 84, 72, 32, 139, 3, 65, 2, - 167, 201, 2, 82, 4, 11, 69, 5, 239, 168, 6, 84, 2, 11, 65, 2, 207, 168, - 6, 68, 24, 60, 9, 65, 76, 65, 89, 65, 76, 65, 77, 32, 183, 152, 6, 73, - 22, 78, 76, 22, 78, 234, 173, 4, 66, 246, 213, 1, 84, 138, 34, 83, 14, - 74, 3, 82, 4, 135, 159, 4, 76, 8, 246, 83, 78, 250, 209, 5, 71, 3, 89, 9, - 33, 6, 82, 83, 73, 65, 78, 32, 6, 50, 66, 16, 3, 68, 72, 65, 17, 3, 71, - 72, 65, 2, 199, 78, 72, 2, 147, 2, 76, 2, 199, 157, 5, 77, 4, 52, 7, 69, - 86, 69, 82, 83, 69, 68, 159, 190, 4, 73, 2, 255, 222, 4, 32, 14, 142, 1, - 69, 40, 7, 79, 71, 68, 73, 65, 78, 32, 64, 12, 85, 80, 69, 82, 83, 67, - 82, 73, 80, 84, 32, 65, 222, 164, 5, 72, 201, 82, 2, 65, 68, 2, 17, 2, - 77, 75, 2, 243, 162, 4, 65, 6, 42, 90, 32, 2, 75, 72, 211, 141, 6, 70, 2, - 239, 247, 4, 72, 2, 11, 76, 2, 11, 65, 2, 135, 163, 6, 80, 6, 36, 3, 69, - 84, 72, 191, 165, 5, 65, 5, 173, 85, 6, 32, 71, 65, 82, 83, 72, 5, 167, - 246, 5, 32, 4, 217, 132, 4, 2, 73, 78, 6, 21, 3, 72, 65, 32, 6, 42, 68, - 178, 132, 4, 66, 167, 161, 1, 65, 2, 17, 2, 79, 84, 2, 223, 150, 1, 84, - 8, 58, 66, 162, 203, 2, 87, 249, 166, 1, 4, 85, 75, 75, 65, 4, 157, 240, - 3, 2, 65, 83, 14, 88, 8, 66, 76, 73, 78, 69, 65, 82, 32, 113, 10, 80, 82, - 65, 76, 73, 78, 69, 65, 82, 32, 8, 44, 5, 67, 79, 76, 79, 78, 211, 206, - 3, 70, 7, 11, 32, 4, 29, 5, 83, 75, 69, 87, 69, 4, 251, 244, 5, 68, 6, - 44, 5, 67, 79, 76, 79, 78, 227, 205, 3, 70, 5, 253, 236, 5, 7, 32, 83, - 75, 69, 87, 69, 68, 8, 62, 72, 25, 11, 87, 79, 32, 86, 69, 82, 84, 73, - 67, 65, 76, 4, 21, 3, 82, 69, 69, 4, 25, 4, 32, 68, 79, 84, 4, 231, 255, - 3, 83, 190, 43, 102, 45, 58, 65, 206, 105, 69, 154, 45, 72, 182, 46, 73, - 230, 69, 79, 170, 33, 82, 214, 16, 85, 147, 22, 87, 4, 32, 2, 83, 72, - 255, 246, 4, 82, 2, 235, 149, 5, 73, 244, 26, 182, 1, 66, 82, 71, 196, - 17, 2, 73, 32, 170, 29, 75, 144, 5, 9, 76, 76, 89, 32, 77, 65, 82, 75, - 32, 34, 77, 178, 37, 78, 132, 13, 3, 80, 69, 32, 94, 85, 190, 148, 5, 67, - 159, 11, 88, 5, 221, 122, 16, 76, 69, 32, 84, 69, 78, 78, 73, 83, 32, 80, - 65, 68, 68, 76, 69, 144, 2, 78, 32, 240, 11, 5, 65, 76, 79, 71, 32, 241, - 2, 6, 66, 65, 78, 87, 65, 32, 190, 1, 162, 1, 65, 102, 67, 186, 1, 68, - 58, 69, 98, 71, 118, 72, 46, 76, 230, 3, 80, 58, 81, 62, 82, 106, 83, - 142, 76, 78, 230, 222, 1, 84, 250, 145, 1, 70, 231, 183, 1, 86, 6, 42, - 80, 150, 201, 2, 77, 219, 187, 1, 83, 2, 17, 2, 79, 83, 2, 145, 235, 5, - 4, 84, 82, 79, 80, 8, 18, 73, 63, 79, 2, 25, 4, 82, 67, 85, 77, 2, 213, - 2, 4, 70, 76, 69, 88, 6, 26, 77, 167, 199, 5, 76, 4, 11, 77, 4, 152, 131, - 5, 7, 69, 82, 67, 73, 65, 76, 32, 235, 147, 1, 65, 22, 26, 79, 151, 165, - 4, 73, 2, 11, 76, 2, 235, 84, 76, 4, 18, 81, 43, 88, 2, 17, 2, 85, 65, 2, - 171, 231, 2, 76, 2, 153, 172, 5, 4, 67, 76, 65, 77, 4, 11, 82, 4, 18, 65, - 55, 69, 2, 11, 86, 2, 11, 69, 2, 177, 120, 3, 32, 65, 67, 2, 145, 4, 4, - 65, 84, 69, 82, 2, 145, 142, 2, 6, 89, 80, 72, 69, 78, 45, 114, 38, 65, - 254, 2, 69, 247, 168, 4, 79, 104, 25, 4, 84, 73, 78, 32, 104, 34, 67, 41, - 4, 83, 77, 65, 76, 52, 11, 65, 52, 25, 4, 80, 73, 84, 65, 52, 41, 8, 76, - 32, 76, 69, 84, 84, 69, 82, 52, 11, 32, 52, 194, 146, 6, 65, 2, 66, 2, - 67, 2, 68, 2, 69, 2, 70, 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, - 2, 78, 2, 79, 2, 80, 2, 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, - 88, 2, 89, 3, 90, 8, 22, 83, 199, 1, 70, 2, 11, 83, 2, 197, 63, 3, 45, - 84, 72, 4, 26, 69, 203, 225, 2, 76, 2, 11, 82, 2, 227, 67, 67, 4, 11, 85, - 4, 26, 79, 203, 190, 3, 69, 2, 147, 166, 5, 84, 8, 36, 3, 73, 71, 72, - 199, 249, 3, 69, 6, 17, 2, 84, 32, 6, 242, 136, 2, 67, 210, 3, 80, 239, - 7, 83, 6, 142, 195, 2, 69, 146, 184, 1, 79, 167, 214, 1, 80, 46, 80, 7, - 76, 69, 84, 84, 69, 82, 32, 208, 1, 5, 83, 73, 71, 78, 32, 147, 2, 86, - 38, 162, 1, 65, 242, 154, 2, 78, 250, 238, 3, 66, 2, 68, 2, 71, 2, 72, 2, - 75, 2, 76, 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 73, - 3, 85, 5, 197, 193, 5, 6, 82, 67, 72, 65, 73, 67, 4, 26, 80, 199, 181, 2, - 86, 2, 25, 4, 65, 77, 85, 68, 2, 227, 212, 2, 80, 36, 48, 7, 76, 69, 84, - 84, 69, 82, 32, 147, 1, 86, 32, 194, 153, 2, 78, 250, 238, 3, 66, 2, 68, - 2, 71, 2, 75, 2, 76, 2, 77, 2, 80, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, - 65, 2, 73, 3, 85, 4, 41, 8, 79, 87, 69, 76, 32, 83, 73, 71, 4, 11, 78, 4, - 243, 200, 5, 32, 212, 3, 112, 10, 76, 69, 32, 76, 69, 84, 84, 69, 82, 32, - 248, 2, 5, 84, 72, 65, 77, 32, 189, 18, 5, 86, 73, 69, 84, 32, 70, 186, - 1, 65, 34, 69, 30, 84, 250, 148, 2, 78, 210, 171, 1, 79, 162, 254, 1, 75, - 2, 80, 162, 7, 85, 234, 61, 70, 2, 72, 2, 76, 2, 77, 2, 81, 2, 83, 2, 86, - 2, 88, 2, 89, 187, 2, 73, 7, 142, 240, 5, 85, 215, 22, 73, 7, 194, 134, - 6, 69, 3, 72, 18, 60, 3, 79, 78, 69, 170, 190, 5, 83, 138, 69, 72, 187, - 2, 65, 10, 11, 45, 10, 218, 133, 6, 50, 2, 51, 2, 52, 2, 53, 3, 54, 254, - 1, 196, 1, 2, 67, 79, 240, 3, 4, 72, 79, 82, 65, 0, 4, 84, 72, 65, 77, - 28, 7, 76, 69, 84, 84, 69, 82, 32, 220, 4, 5, 83, 73, 71, 78, 32, 213, 5, - 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 20, 164, 1, 13, 78, 83, - 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 161, 241, 4, 21, 77, 66, 73, - 78, 73, 78, 71, 32, 67, 82, 89, 80, 84, 79, 71, 82, 65, 77, 77, 73, 67, - 18, 148, 1, 6, 70, 73, 78, 65, 76, 32, 22, 76, 56, 16, 72, 73, 71, 72, - 32, 82, 65, 84, 72, 65, 32, 79, 82, 32, 76, 79, 22, 77, 134, 254, 5, 66, - 3, 83, 2, 215, 190, 5, 78, 4, 54, 79, 213, 149, 1, 7, 65, 32, 84, 65, 78, - 71, 32, 2, 187, 214, 1, 87, 6, 48, 6, 69, 68, 73, 65, 76, 32, 139, 128, - 6, 65, 4, 206, 253, 5, 76, 3, 82, 20, 201, 142, 4, 2, 32, 68, 106, 188, - 1, 2, 71, 82, 44, 5, 72, 73, 71, 72, 32, 94, 76, 222, 1, 82, 134, 168, 2, - 85, 206, 201, 1, 73, 178, 15, 78, 186, 219, 1, 79, 162, 8, 69, 158, 20, - 66, 2, 68, 2, 77, 2, 87, 187, 2, 65, 2, 21, 3, 69, 65, 84, 2, 179, 251, - 5, 32, 32, 242, 1, 75, 42, 82, 186, 179, 5, 83, 82, 67, 2, 80, 2, 84, - 138, 69, 70, 2, 72, 3, 89, 36, 60, 3, 79, 87, 32, 170, 188, 5, 65, 206, - 41, 85, 159, 20, 76, 28, 86, 75, 42, 82, 138, 180, 5, 67, 2, 80, 2, 84, - 138, 69, 70, 2, 72, 2, 83, 3, 89, 6, 182, 249, 5, 72, 2, 88, 187, 2, 65, - 2, 133, 129, 4, 2, 65, 84, 8, 26, 65, 191, 228, 5, 85, 7, 214, 248, 5, - 78, 3, 84, 50, 178, 1, 72, 34, 75, 176, 1, 4, 77, 65, 73, 32, 82, 82, - 136, 1, 2, 83, 65, 92, 5, 87, 73, 65, 78, 71, 188, 126, 3, 68, 79, 75, - 224, 117, 3, 84, 79, 78, 205, 168, 3, 2, 67, 65, 4, 198, 168, 5, 65, 179, - 63, 79, 14, 52, 4, 72, 85, 69, 78, 134, 3, 65, 195, 219, 4, 69, 8, 80, 6, - 32, 84, 79, 78, 69, 45, 149, 174, 4, 8, 45, 76, 85, 69, 32, 75, 65, 82, - 6, 146, 248, 5, 51, 2, 52, 3, 53, 8, 56, 4, 75, 65, 78, 71, 134, 135, 1, - 89, 183, 181, 3, 83, 5, 183, 140, 1, 32, 4, 92, 3, 65, 32, 72, 21, 16, - 69, 86, 69, 82, 83, 69, 68, 32, 82, 79, 84, 65, 84, 69, 68, 32, 2, 195, - 187, 4, 65, 2, 155, 198, 3, 82, 8, 48, 3, 84, 75, 65, 230, 171, 4, 87, - 203, 121, 75, 4, 17, 2, 65, 78, 5, 199, 131, 4, 75, 5, 245, 191, 3, 2, - 87, 65, 38, 90, 65, 36, 4, 77, 65, 73, 32, 22, 79, 46, 84, 86, 85, 178, - 233, 3, 73, 223, 137, 2, 69, 9, 194, 244, 5, 65, 2, 69, 3, 73, 2, 183, - 224, 4, 83, 11, 210, 213, 3, 65, 186, 158, 2, 79, 3, 89, 4, 42, 65, 197, - 136, 1, 4, 72, 65, 77, 32, 2, 17, 2, 76, 76, 2, 211, 128, 4, 32, 9, 234, - 178, 5, 85, 163, 64, 69, 144, 1, 184, 1, 7, 76, 69, 84, 84, 69, 82, 32, - 188, 2, 5, 77, 65, 73, 32, 75, 32, 7, 83, 89, 77, 66, 79, 76, 32, 116, 9, + 69, 78, 255, 157, 7, 75, 4, 158, 2, 80, 175, 149, 5, 76, 20, 110, 69, + 146, 1, 72, 20, 2, 73, 88, 190, 133, 5, 65, 250, 67, 77, 246, 49, 81, + 250, 100, 79, 222, 61, 68, 3, 83, 4, 56, 7, 67, 79, 78, 68, 32, 83, 67, + 21, 3, 86, 69, 78, 2, 211, 200, 5, 82, 2, 29, 5, 32, 80, 79, 73, 78, 2, + 11, 84, 2, 203, 241, 5, 32, 2, 143, 158, 7, 86, 2, 17, 2, 84, 89, 2, 219, + 157, 7, 32, 8, 44, 4, 72, 82, 69, 69, 22, 87, 247, 4, 73, 2, 131, 215, 6, + 32, 4, 68, 13, 69, 78, 84, 89, 45, 84, 87, 79, 32, 80, 79, 73, 78, 23, + 79, 2, 251, 135, 5, 84, 2, 155, 153, 7, 32, 4, 48, 6, 80, 32, 87, 73, 84, + 72, 207, 213, 6, 72, 2, 17, 2, 32, 69, 2, 171, 119, 88, 4, 166, 213, 6, + 79, 163, 70, 83, 2, 149, 249, 5, 6, 32, 66, 76, 65, 67, 75, 6, 250, 154, + 7, 50, 2, 51, 3, 65, 95, 134, 1, 65, 178, 6, 69, 236, 1, 10, 73, 67, 75, + 32, 70, 73, 71, 85, 82, 69, 182, 1, 79, 90, 82, 190, 4, 85, 238, 138, 7, + 83, 3, 88, 38, 92, 6, 70, 70, 32, 79, 70, 32, 106, 77, 82, 82, 206, 3, + 84, 142, 143, 3, 78, 179, 217, 2, 68, 4, 60, 8, 65, 69, 83, 67, 85, 76, + 65, 80, 21, 3, 72, 69, 82, 2, 203, 143, 5, 73, 2, 151, 130, 6, 77, 2, 21, + 3, 80, 69, 68, 2, 17, 2, 32, 69, 2, 233, 225, 1, 4, 78, 86, 69, 76, 24, + 42, 32, 205, 1, 5, 84, 32, 79, 70, 32, 12, 82, 69, 54, 79, 248, 240, 1, + 8, 65, 78, 68, 32, 67, 82, 69, 83, 131, 133, 2, 87, 2, 17, 2, 81, 85, 2, + 11, 65, 2, 187, 216, 6, 76, 4, 44, 5, 70, 32, 68, 65, 86, 183, 210, 3, + 80, 2, 179, 207, 6, 73, 12, 66, 71, 30, 83, 40, 4, 80, 82, 79, 84, 242, + 77, 72, 239, 106, 84, 2, 89, 4, 85, 65, 82, 68, 4, 26, 69, 131, 255, 1, + 84, 2, 11, 76, 2, 21, 3, 69, 67, 84, 2, 29, 5, 69, 68, 32, 65, 82, 2, + 199, 145, 7, 69, 4, 228, 163, 5, 10, 85, 69, 32, 79, 70, 32, 76, 73, 66, + 69, 155, 160, 1, 73, 8, 92, 2, 65, 77, 112, 9, 78, 79, 71, 82, 65, 80, + 72, 73, 67, 213, 219, 1, 4, 84, 72, 79, 83, 4, 54, 32, 185, 134, 6, 7, + 73, 78, 71, 32, 66, 79, 87, 2, 33, 6, 76, 79, 67, 79, 77, 79, 2, 147, + 213, 1, 84, 2, 153, 188, 4, 2, 32, 70, 11, 11, 32, 8, 60, 5, 87, 73, 84, + 72, 32, 193, 169, 2, 4, 76, 69, 65, 78, 4, 32, 4, 65, 82, 77, 83, 51, 68, + 2, 25, 4, 32, 82, 65, 73, 2, 143, 252, 1, 83, 2, 179, 247, 5, 82, 4, 44, + 4, 67, 75, 32, 67, 21, 3, 80, 87, 65, 2, 179, 136, 6, 72, 2, 147, 237, 6, + 84, 22, 66, 65, 124, 10, 69, 83, 83, 32, 79, 85, 84, 76, 73, 78, 79, 73, + 8, 56, 4, 73, 71, 72, 84, 45, 6, 87, 66, 69, 82, 82, 89, 4, 196, 160, 3, + 2, 32, 82, 131, 213, 2, 78, 5, 195, 248, 5, 32, 2, 17, 2, 69, 68, 2, 17, + 2, 32, 87, 2, 205, 243, 4, 4, 72, 73, 84, 69, 12, 76, 4, 78, 71, 32, 84, + 52, 4, 80, 69, 68, 32, 241, 8, 4, 67, 84, 76, 89, 2, 17, 2, 69, 82, 2, + 181, 211, 6, 3, 77, 73, 78, 8, 70, 68, 24, 3, 76, 69, 70, 0, 4, 82, 73, + 71, 72, 13, 2, 85, 80, 2, 33, 3, 79, 87, 78, 2, 11, 84, 2, 11, 45, 2, + 149, 228, 5, 8, 80, 79, 73, 78, 84, 73, 78, 71, 6, 100, 8, 68, 73, 79, + 32, 77, 73, 67, 82, 20, 9, 70, 70, 69, 68, 32, 70, 76, 65, 84, 187, 135, + 7, 80, 2, 147, 221, 5, 79, 2, 11, 66, 2, 163, 177, 4, 82, 162, 3, 150, 1, + 66, 212, 2, 5, 67, 67, 69, 69, 68, 192, 3, 8, 77, 77, 65, 84, 73, 79, 78, + 32, 70, 78, 204, 22, 3, 80, 69, 82, 188, 11, 2, 82, 70, 47, 83, 63, 11, + 83, 60, 76, 6, 67, 82, 73, 80, 84, 32, 136, 1, 3, 69, 84, 32, 93, 3, 84, + 73, 84, 30, 118, 76, 186, 29, 69, 166, 1, 80, 22, 82, 134, 221, 2, 77, + 226, 151, 2, 70, 30, 83, 42, 84, 62, 90, 210, 86, 78, 15, 79, 2, 207, 30, + 69, 28, 56, 6, 65, 66, 79, 86, 69, 32, 246, 33, 79, 159, 3, 87, 6, 142, + 32, 83, 139, 196, 5, 82, 2, 223, 188, 5, 85, 22, 11, 83, 23, 11, 32, 20, + 128, 1, 6, 65, 66, 79, 86, 69, 32, 156, 1, 7, 66, 85, 84, 32, 78, 79, 84, + 32, 2, 79, 82, 149, 156, 3, 5, 85, 78, 68, 69, 82, 12, 100, 4, 78, 79, + 84, 32, 28, 12, 83, 73, 78, 71, 76, 69, 45, 76, 73, 78, 69, 32, 218, 32, + 65, 39, 69, 4, 242, 32, 65, 167, 1, 69, 4, 250, 32, 69, 83, 78, 2, 93, 5, + 32, 69, 81, 85, 73, 4, 17, 2, 32, 69, 4, 17, 2, 81, 85, 4, 22, 73, 167, + 33, 65, 2, 173, 33, 6, 86, 65, 76, 69, 78, 84, 6, 250, 146, 3, 66, 152, + 229, 1, 4, 87, 73, 84, 72, 251, 144, 1, 84, 249, 1, 194, 1, 32, 72, 7, + 68, 65, 78, 69, 83, 69, 32, 204, 12, 4, 82, 73, 83, 69, 72, 5, 85, 87, + 65, 82, 32, 168, 179, 2, 14, 83, 69, 84, 32, 79, 86, 69, 82, 32, 66, 85, + 73, 76, 68, 247, 163, 3, 70, 4, 50, 87, 205, 209, 5, 6, 66, 69, 72, 73, + 78, 68, 2, 171, 194, 6, 73, 146, 1, 198, 2, 65, 20, 17, 67, 79, 78, 83, + 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 80, 65, 172, 1, 7, 76, 69, + 84, 84, 69, 82, 32, 188, 3, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, + 78, 32, 66, 73, 78, 68, 85, 32, 228, 1, 5, 83, 73, 71, 78, 32, 204, 1, + 13, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 80, 65, 207, 128, 5, 68, + 2, 215, 162, 3, 86, 10, 68, 2, 77, 73, 40, 2, 78, 89, 33, 7, 83, 65, 78, + 71, 65, 78, 32, 2, 17, 2, 78, 71, 2, 179, 240, 5, 75, 4, 142, 5, 65, 219, + 160, 1, 73, 4, 162, 249, 6, 77, 3, 87, 76, 246, 1, 65, 58, 70, 78, 76, 2, + 82, 34, 83, 170, 161, 3, 69, 234, 217, 1, 78, 214, 181, 1, 66, 2, 75, + 138, 69, 67, 2, 68, 2, 71, 2, 72, 2, 74, 2, 77, 2, 80, 2, 81, 2, 84, 2, + 86, 2, 87, 2, 88, 2, 89, 2, 90, 186, 2, 73, 2, 79, 3, 85, 7, 180, 185, 6, + 6, 82, 67, 72, 65, 73, 67, 147, 64, 69, 6, 44, 5, 73, 78, 65, 76, 32, + 227, 248, 6, 65, 4, 222, 248, 6, 75, 3, 77, 4, 218, 211, 6, 69, 235, 36, + 65, 4, 234, 245, 6, 89, 187, 2, 65, 16, 90, 66, 2, 68, 2, 75, 12, 3, 76, + 69, 85, 38, 67, 34, 80, 253, 243, 6, 3, 83, 85, 82, 2, 11, 65, 2, 189, + 103, 5, 32, 83, 65, 84, 65, 2, 11, 65, 2, 171, 178, 6, 75, 4, 172, 3, 3, + 65, 78, 71, 189, 176, 6, 3, 85, 82, 78, 10, 32, 2, 80, 65, 151, 156, 3, + 86, 8, 28, 3, 77, 65, 65, 23, 78, 2, 203, 244, 6, 69, 6, 18, 71, 71, 89, + 4, 38, 76, 225, 174, 6, 3, 87, 73, 83, 2, 205, 209, 5, 2, 65, 89, 2, 201, + 241, 6, 2, 69, 67, 12, 18, 77, 23, 78, 2, 231, 144, 3, 69, 10, 96, 3, 69, + 85, 76, 34, 79, 22, 89, 232, 233, 1, 3, 71, 72, 85, 205, 194, 2, 4, 65, + 69, 76, 65, 2, 11, 69, 2, 195, 162, 6, 85, 2, 151, 172, 4, 76, 2, 255, + 156, 1, 85, 5, 209, 171, 3, 13, 32, 79, 86, 69, 82, 32, 77, 79, 85, 78, + 84, 65, 73, 88, 88, 7, 76, 69, 84, 84, 69, 82, 32, 184, 6, 6, 83, 73, 71, + 78, 32, 80, 159, 247, 4, 68, 66, 142, 2, 65, 66, 67, 66, 68, 44, 3, 72, + 65, 77, 22, 74, 34, 75, 42, 78, 38, 79, 2, 85, 34, 80, 38, 82, 20, 4, 83, + 72, 89, 69, 34, 84, 112, 3, 86, 65, 82, 144, 238, 4, 4, 76, 79, 65, 67, + 218, 29, 71, 148, 60, 2, 73, 77, 142, 26, 89, 222, 16, 66, 214, 89, 69, + 203, 28, 77, 6, 42, 80, 210, 227, 5, 65, 155, 137, 1, 86, 2, 167, 254, 1, + 80, 4, 40, 2, 72, 69, 253, 218, 6, 2, 65, 82, 2, 215, 219, 5, 76, 4, 22, + 69, 219, 94, 79, 2, 219, 218, 6, 86, 2, 171, 207, 6, 83, 2, 11, 89, 2, + 239, 236, 6, 65, 6, 194, 186, 6, 76, 146, 48, 73, 99, 72, 4, 134, 202, 5, + 71, 175, 162, 1, 65, 2, 11, 84, 2, 155, 149, 1, 84, 4, 194, 201, 5, 72, + 179, 163, 1, 73, 2, 243, 199, 6, 69, 4, 242, 213, 6, 76, 215, 22, 82, 8, + 42, 69, 22, 72, 209, 201, 6, 2, 65, 83, 2, 179, 147, 1, 78, 4, 26, 65, + 147, 197, 5, 69, 2, 243, 215, 6, 82, 2, 255, 232, 6, 67, 2, 175, 204, 6, + 86, 72, 54, 83, 206, 248, 4, 72, 165, 67, 4, 86, 73, 76, 76, 68, 56, 6, + 67, 82, 73, 80, 84, 32, 229, 2, 3, 69, 84, 32, 34, 114, 69, 34, 76, 134, + 1, 80, 22, 82, 134, 221, 2, 77, 226, 151, 2, 70, 30, 83, 42, 84, 62, 90, + 210, 86, 78, 15, 79, 4, 210, 68, 81, 147, 250, 5, 73, 6, 88, 18, 65, 84, + 73, 78, 32, 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 31, 69, + 4, 170, 232, 6, 73, 3, 78, 2, 55, 70, 2, 235, 181, 3, 76, 2, 21, 3, 73, + 71, 72, 2, 11, 84, 2, 183, 224, 2, 32, 34, 148, 1, 6, 65, 66, 79, 86, 69, + 32, 96, 7, 66, 69, 83, 73, 68, 69, 32, 162, 1, 79, 158, 3, 87, 129, 200, + 4, 9, 80, 82, 69, 67, 69, 68, 73, 78, 71, 6, 26, 83, 227, 192, 2, 76, 4, + 11, 85, 4, 26, 80, 175, 247, 4, 66, 2, 169, 247, 4, 2, 69, 82, 4, 108, + 23, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 66, 89, 32, 68, 65, 83, + 72, 32, 87, 73, 84, 72, 23, 83, 2, 17, 2, 32, 83, 2, 137, 246, 4, 2, 85, + 66, 16, 11, 70, 17, 11, 32, 14, 120, 6, 65, 66, 79, 86, 69, 32, 132, 1, + 4, 87, 73, 84, 72, 249, 224, 5, 11, 79, 82, 32, 69, 81, 85, 65, 76, 32, + 84, 79, 8, 34, 65, 38, 69, 18, 84, 67, 78, 2, 11, 76, 2, 113, 3, 77, 79, + 83, 2, 239, 61, 81, 2, 21, 3, 73, 76, 68, 2, 215, 158, 3, 69, 2, 17, 2, + 32, 78, 2, 11, 79, 2, 11, 84, 2, 11, 32, 2, 11, 69, 2, 17, 2, 81, 85, 2, + 11, 65, 2, 11, 76, 2, 143, 238, 2, 32, 6, 25, 4, 73, 84, 72, 32, 6, 104, + 14, 77, 85, 76, 84, 73, 80, 76, 73, 67, 65, 84, 73, 79, 78, 0, 4, 80, 76, + 85, 83, 199, 143, 6, 68, 2, 225, 174, 4, 5, 32, 83, 73, 71, 78, 4, 184, + 213, 4, 2, 65, 67, 191, 209, 1, 69, 4, 60, 9, 80, 69, 78, 83, 73, 79, 78, + 32, 82, 147, 203, 6, 72, 2, 21, 3, 65, 73, 76, 2, 175, 208, 5, 87, 8, 26, + 65, 98, 73, 35, 85, 4, 44, 5, 83, 72, 32, 65, 77, 243, 221, 6, 78, 2, + 141, 154, 6, 7, 80, 69, 82, 83, 65, 78, 68, 2, 11, 77, 2, 199, 159, 6, + 77, 2, 165, 156, 3, 2, 78, 71, 236, 2, 92, 11, 76, 79, 84, 73, 32, 78, + 65, 71, 82, 73, 32, 226, 5, 77, 242, 16, 78, 97, 2, 82, 73, 90, 180, 1, + 7, 76, 69, 84, 84, 69, 82, 32, 168, 2, 11, 80, 79, 69, 84, 82, 89, 32, + 77, 65, 82, 75, 56, 5, 83, 73, 71, 78, 32, 145, 1, 11, 86, 79, 87, 69, + 76, 32, 83, 73, 71, 78, 32, 64, 174, 1, 68, 46, 82, 34, 84, 242, 90, 66, + 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 223, 5, 72, 2, 76, 2, 77, 2, 78, + 2, 83, 246, 30, 65, 2, 69, 2, 73, 2, 79, 3, 85, 8, 186, 91, 68, 138, 223, + 5, 72, 247, 30, 79, 4, 150, 186, 6, 82, 247, 30, 79, 8, 238, 90, 84, 138, + 223, 5, 72, 247, 30, 79, 8, 11, 45, 8, 178, 216, 6, 49, 2, 50, 2, 51, 3, + 52, 8, 46, 65, 70, 72, 181, 146, 6, 3, 68, 86, 73, 4, 64, 10, 76, 84, 69, + 82, 78, 65, 84, 69, 32, 72, 171, 146, 6, 78, 2, 245, 142, 2, 2, 65, 83, + 10, 130, 184, 6, 79, 246, 30, 65, 2, 69, 2, 73, 3, 85, 88, 60, 8, 66, 79, + 76, 32, 70, 79, 82, 32, 145, 16, 2, 77, 69, 86, 238, 2, 66, 48, 2, 67, + 65, 118, 68, 238, 3, 69, 130, 2, 70, 56, 8, 72, 79, 82, 73, 90, 79, 78, + 84, 0, 6, 86, 69, 82, 84, 73, 67, 44, 3, 76, 73, 78, 52, 9, 77, 65, 82, + 75, 83, 32, 67, 72, 65, 22, 78, 90, 65, 68, 3, 82, 69, 67, 32, 5, 71, 82, + 79, 85, 80, 0, 4, 85, 78, 73, 84, 22, 83, 249, 2, 15, 84, 89, 80, 69, 32, + 65, 32, 69, 76, 69, 67, 84, 82, 79, 78, 4, 230, 200, 4, 69, 241, 203, 1, + 3, 65, 67, 75, 4, 44, 3, 82, 82, 73, 129, 189, 1, 2, 78, 67, 2, 33, 6, + 65, 71, 69, 32, 82, 69, 2, 11, 84, 2, 155, 191, 1, 85, 20, 24, 2, 65, 84, + 55, 69, 2, 177, 5, 9, 65, 32, 76, 73, 78, 75, 32, 69, 83, 18, 84, 4, 76, + 69, 84, 69, 137, 2, 12, 86, 73, 67, 69, 32, 67, 79, 78, 84, 82, 79, 76, + 11, 11, 32, 8, 164, 1, 11, 82, 69, 67, 84, 65, 78, 71, 85, 76, 65, 82, 0, + 6, 83, 81, 85, 65, 82, 69, 174, 186, 4, 70, 249, 149, 1, 11, 77, 69, 68, + 73, 85, 77, 32, 83, 72, 65, 68, 2, 61, 13, 32, 67, 72, 69, 67, 75, 69, + 82, 32, 66, 79, 65, 82, 2, 239, 207, 5, 68, 8, 11, 32, 8, 226, 178, 1, + 70, 134, 169, 3, 84, 155, 87, 79, 12, 22, 78, 207, 1, 83, 10, 48, 5, 68, + 32, 79, 70, 32, 137, 1, 2, 81, 85, 8, 18, 77, 31, 84, 2, 237, 161, 5, 2, + 69, 68, 6, 64, 11, 82, 65, 78, 83, 77, 73, 83, 83, 73, 79, 78, 243, 112, + 69, 5, 167, 218, 3, 32, 2, 199, 6, 73, 2, 193, 130, 5, 2, 67, 65, 4, 36, + 2, 73, 76, 73, 3, 79, 82, 77, 2, 191, 2, 69, 2, 165, 166, 1, 6, 65, 76, + 32, 84, 65, 66, 2, 11, 69, 2, 17, 2, 32, 70, 2, 227, 182, 1, 69, 2, 219, + 144, 5, 80, 6, 26, 69, 147, 192, 4, 85, 4, 56, 8, 71, 65, 84, 73, 86, 69, + 32, 65, 183, 174, 5, 87, 2, 21, 3, 67, 75, 78, 2, 21, 3, 79, 87, 76, 2, + 231, 140, 1, 69, 2, 11, 79, 2, 17, 2, 82, 68, 2, 219, 151, 3, 32, 18, + 176, 1, 7, 65, 77, 65, 82, 73, 84, 65, 60, 3, 72, 73, 70, 52, 8, 84, 65, + 82, 84, 32, 79, 70, 32, 64, 8, 85, 66, 83, 84, 73, 84, 85, 84, 120, 3, + 89, 78, 67, 163, 136, 6, 80, 2, 25, 4, 78, 32, 83, 79, 2, 11, 85, 2, 171, + 138, 6, 82, 4, 17, 2, 84, 32, 4, 174, 205, 5, 79, 243, 41, 73, 4, 22, 72, + 239, 106, 84, 2, 17, 2, 69, 65, 2, 175, 245, 5, 68, 4, 163, 177, 4, 69, + 2, 173, 136, 6, 2, 73, 67, 2, 11, 84, 2, 155, 180, 6, 82, 7, 38, 67, 193, + 181, 2, 3, 65, 71, 79, 2, 137, 139, 1, 9, 72, 82, 79, 78, 79, 85, 83, 32, + 73, 180, 1, 36, 3, 65, 67, 32, 147, 141, 5, 78, 178, 1, 150, 3, 65, 52, + 4, 66, 65, 82, 82, 32, 2, 67, 79, 80, 13, 68, 79, 84, 84, 69, 68, 32, 90, + 76, 65, 77, 65, 32, 118, 69, 90, 72, 204, 2, 7, 76, 69, 84, 84, 69, 82, + 32, 236, 9, 9, 79, 66, 76, 73, 81, 85, 69, 32, 76, 28, 4, 80, 84, 72, 65, + 0, 4, 90, 81, 65, 80, 110, 82, 88, 2, 83, 85, 174, 2, 84, 216, 131, 5, 3, + 77, 85, 83, 224, 22, 7, 70, 69, 77, 73, 78, 73, 78, 153, 139, 1, 5, 81, + 85, 83, 72, 83, 2, 11, 66, 2, 229, 215, 5, 5, 66, 82, 69, 86, 73, 2, 11, + 69, 2, 215, 191, 6, 75, 6, 38, 78, 165, 17, 4, 76, 79, 78, 32, 2, 17, 2, + 84, 82, 2, 187, 239, 5, 65, 4, 28, 3, 65, 78, 71, 35, 72, 2, 11, 85, 2, + 155, 156, 5, 76, 2, 11, 79, 2, 185, 231, 3, 5, 82, 73, 90, 79, 78, 6, 60, + 10, 78, 68, 32, 79, 70, 32, 80, 65, 82, 65, 151, 14, 83, 2, 201, 11, 2, + 71, 82, 14, 104, 8, 65, 82, 75, 76, 69, 65, 78, 32, 132, 1, 4, 66, 65, + 83, 65, 65, 7, 79, 82, 73, 90, 79, 78, 84, 6, 60, 5, 65, 83, 84, 69, 82, + 40, 4, 77, 69, 84, 79, 3, 79, 2, 17, 2, 73, 83, 2, 147, 180, 4, 67, 2, + 213, 177, 2, 2, 66, 69, 6, 164, 11, 8, 45, 69, 83, 65, 83, 65, 32, 68, + 155, 143, 4, 32, 2, 233, 235, 5, 2, 65, 76, 92, 154, 2, 68, 102, 72, 32, + 3, 76, 65, 77, 34, 77, 204, 1, 2, 80, 69, 142, 1, 82, 78, 83, 144, 1, 8, + 70, 73, 78, 65, 76, 32, 83, 69, 106, 65, 14, 75, 2, 81, 34, 84, 40, 5, + 71, 65, 77, 65, 76, 40, 4, 89, 85, 68, 72, 190, 80, 66, 242, 178, 4, 90, + 186, 46, 78, 134, 2, 87, 175, 126, 69, 4, 76, 14, 79, 84, 76, 69, 83, 83, + 32, 68, 65, 76, 65, 84, 72, 32, 139, 3, 65, 2, 179, 212, 2, 82, 4, 11, + 69, 5, 207, 183, 6, 84, 2, 11, 65, 2, 175, 183, 6, 68, 24, 60, 9, 65, 76, + 65, 89, 65, 76, 65, 77, 32, 151, 167, 6, 73, 22, 78, 76, 22, 78, 226, + 185, 4, 66, 222, 216, 1, 84, 138, 34, 83, 14, 74, 3, 82, 4, 255, 170, 4, + 76, 8, 222, 89, 78, 242, 218, 5, 71, 3, 89, 9, 33, 6, 82, 83, 73, 65, 78, + 32, 6, 50, 66, 16, 3, 68, 72, 65, 17, 3, 71, 72, 65, 2, 175, 84, 72, 2, + 147, 2, 76, 2, 139, 170, 5, 77, 4, 52, 7, 69, 86, 69, 82, 83, 69, 68, + 151, 202, 4, 73, 2, 195, 235, 4, 32, 14, 142, 1, 69, 40, 7, 79, 71, 68, + 73, 65, 78, 32, 64, 12, 85, 80, 69, 82, 83, 67, 82, 73, 80, 84, 32, 65, + 190, 179, 5, 72, 201, 82, 2, 65, 68, 2, 17, 2, 77, 75, 2, 235, 174, 4, + 65, 6, 42, 90, 32, 2, 75, 72, 179, 156, 6, 70, 2, 187, 132, 5, 72, 2, 11, + 76, 2, 11, 65, 2, 231, 177, 6, 80, 6, 36, 3, 69, 84, 72, 159, 180, 5, 65, + 5, 149, 91, 6, 32, 71, 65, 82, 83, 72, 5, 135, 133, 6, 32, 4, 221, 143, + 4, 2, 73, 78, 6, 21, 3, 72, 65, 32, 6, 42, 68, 182, 143, 4, 66, 131, 165, + 1, 65, 2, 17, 2, 79, 84, 2, 243, 156, 1, 84, 8, 58, 66, 174, 214, 2, 87, + 249, 166, 1, 4, 85, 75, 75, 65, 4, 169, 251, 3, 2, 65, 83, 14, 88, 8, 66, + 76, 73, 78, 69, 65, 82, 32, 113, 10, 80, 82, 65, 76, 73, 78, 69, 65, 82, + 32, 8, 44, 5, 67, 79, 76, 79, 78, 227, 217, 3, 70, 7, 11, 32, 4, 29, 5, + 83, 75, 69, 87, 69, 4, 219, 131, 6, 68, 6, 44, 5, 67, 79, 76, 79, 78, + 243, 216, 3, 70, 5, 221, 251, 5, 7, 32, 83, 75, 69, 87, 69, 68, 8, 62, + 72, 25, 11, 87, 79, 32, 86, 69, 82, 84, 73, 67, 65, 76, 4, 21, 3, 82, 69, + 69, 4, 25, 4, 32, 68, 79, 84, 4, 235, 138, 4, 83, 136, 47, 102, 45, 58, + 65, 162, 111, 69, 218, 45, 72, 202, 46, 73, 254, 68, 79, 246, 37, 82, + 214, 17, 85, 147, 22, 87, 4, 32, 2, 83, 72, 203, 131, 5, 82, 2, 203, 164, + 5, 73, 200, 29, 182, 1, 66, 86, 71, 196, 17, 2, 73, 32, 142, 35, 75, 144, + 5, 9, 76, 76, 89, 32, 77, 65, 82, 75, 32, 34, 77, 178, 37, 78, 240, 12, + 3, 80, 69, 32, 94, 85, 202, 157, 5, 67, 159, 11, 88, 5, 241, 128, 1, 16, + 76, 69, 32, 84, 69, 78, 78, 73, 83, 32, 80, 65, 68, 68, 76, 69, 144, 2, + 78, 32, 240, 11, 5, 65, 76, 79, 71, 32, 241, 2, 6, 66, 65, 78, 87, 65, + 32, 190, 1, 162, 1, 65, 102, 67, 186, 1, 68, 58, 69, 98, 71, 118, 72, 46, + 76, 230, 3, 80, 58, 81, 62, 82, 106, 83, 242, 81, 78, 230, 227, 1, 84, + 162, 146, 1, 70, 179, 185, 1, 86, 6, 42, 80, 158, 212, 2, 77, 211, 187, + 1, 83, 2, 17, 2, 79, 83, 2, 237, 249, 5, 4, 84, 82, 79, 80, 8, 18, 73, + 63, 79, 2, 25, 4, 82, 67, 85, 77, 2, 213, 2, 4, 70, 76, 69, 88, 6, 26, + 77, 131, 214, 5, 76, 4, 11, 77, 4, 172, 156, 5, 7, 69, 82, 67, 73, 65, + 76, 32, 179, 137, 1, 65, 22, 26, 79, 139, 177, 4, 73, 2, 11, 76, 2, 207, + 90, 76, 4, 18, 81, 43, 88, 2, 17, 2, 85, 65, 2, 179, 242, 2, 76, 2, 245, + 186, 5, 4, 67, 76, 65, 77, 4, 11, 82, 4, 18, 65, 55, 69, 2, 11, 86, 2, + 11, 69, 2, 193, 126, 3, 32, 65, 67, 2, 145, 4, 4, 65, 84, 69, 82, 2, 153, + 152, 2, 6, 89, 80, 72, 69, 78, 45, 114, 38, 65, 254, 2, 69, 235, 180, 4, + 79, 104, 25, 4, 84, 73, 78, 32, 104, 34, 67, 41, 4, 83, 77, 65, 76, 52, + 11, 65, 52, 25, 4, 80, 73, 84, 65, 52, 41, 8, 76, 32, 76, 69, 84, 84, 69, + 82, 52, 11, 32, 52, 158, 161, 6, 65, 2, 66, 2, 67, 2, 68, 2, 69, 2, 70, + 2, 71, 2, 72, 2, 73, 2, 74, 2, 75, 2, 76, 2, 77, 2, 78, 2, 79, 2, 80, 2, + 81, 2, 82, 2, 83, 2, 84, 2, 85, 2, 86, 2, 87, 2, 88, 2, 89, 3, 90, 8, 22, + 83, 199, 1, 70, 2, 11, 83, 2, 169, 69, 3, 45, 84, 72, 4, 26, 69, 211, + 236, 2, 76, 2, 11, 82, 2, 199, 73, 67, 4, 11, 85, 4, 26, 79, 215, 201, 3, + 69, 2, 239, 180, 5, 84, 8, 36, 3, 73, 71, 72, 199, 132, 4, 69, 6, 17, 2, + 84, 32, 6, 250, 146, 2, 67, 210, 3, 80, 239, 7, 83, 6, 150, 206, 2, 69, + 138, 184, 1, 79, 131, 218, 1, 80, 46, 80, 7, 76, 69, 84, 84, 69, 82, 32, + 208, 1, 5, 83, 73, 71, 78, 32, 147, 2, 86, 38, 162, 1, 65, 250, 164, 2, + 78, 206, 243, 3, 66, 2, 68, 2, 71, 2, 72, 2, 75, 2, 76, 2, 77, 2, 80, 2, + 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 73, 3, 85, 5, 161, 208, 5, 6, 82, + 67, 72, 65, 73, 67, 4, 26, 80, 207, 192, 2, 86, 2, 25, 4, 65, 77, 85, 68, + 2, 235, 223, 2, 80, 36, 48, 7, 76, 69, 84, 84, 69, 82, 32, 147, 1, 86, + 32, 202, 163, 2, 78, 206, 243, 3, 66, 2, 68, 2, 71, 2, 75, 2, 76, 2, 77, + 2, 80, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 65, 2, 73, 3, 85, 4, 41, 8, + 79, 87, 69, 76, 32, 83, 73, 71, 4, 11, 78, 4, 207, 215, 5, 32, 194, 4, + 140, 1, 10, 76, 69, 32, 76, 69, 84, 84, 69, 82, 32, 248, 2, 5, 84, 72, + 65, 77, 32, 184, 18, 5, 86, 73, 69, 84, 32, 225, 6, 3, 89, 79, 32, 70, + 186, 1, 65, 34, 69, 30, 84, 230, 158, 2, 78, 214, 172, 1, 79, 242, 129, + 2, 75, 2, 80, 162, 7, 85, 234, 61, 70, 2, 72, 2, 76, 2, 77, 2, 81, 2, 83, + 2, 86, 2, 88, 2, 89, 187, 2, 73, 7, 206, 254, 5, 85, 215, 22, 73, 7, 130, + 149, 6, 69, 3, 72, 18, 60, 3, 79, 78, 69, 234, 204, 5, 83, 138, 69, 72, + 187, 2, 65, 10, 11, 45, 10, 154, 148, 6, 50, 2, 51, 2, 52, 2, 53, 3, 54, + 254, 1, 196, 1, 2, 67, 79, 232, 3, 4, 72, 79, 82, 65, 0, 4, 84, 72, 65, + 77, 28, 7, 76, 69, 84, 84, 69, 82, 32, 220, 4, 5, 83, 73, 71, 78, 32, + 217, 5, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 20, 164, 1, 13, + 78, 83, 79, 78, 65, 78, 84, 32, 83, 73, 71, 78, 32, 197, 253, 4, 21, 77, + 66, 73, 78, 73, 78, 71, 32, 67, 82, 89, 80, 84, 79, 71, 82, 65, 77, 77, + 73, 67, 18, 148, 1, 6, 70, 73, 78, 65, 76, 32, 22, 76, 48, 16, 72, 73, + 71, 72, 32, 82, 65, 84, 72, 65, 32, 79, 82, 32, 76, 79, 22, 77, 206, 140, + 6, 66, 3, 83, 2, 151, 205, 5, 78, 4, 46, 79, 157, 26, 6, 65, 32, 84, 65, + 78, 71, 2, 167, 220, 1, 87, 6, 48, 6, 69, 68, 73, 65, 76, 32, 211, 142, + 6, 65, 4, 150, 140, 6, 76, 3, 82, 20, 169, 154, 4, 2, 32, 68, 106, 188, + 1, 2, 71, 82, 44, 5, 72, 73, 71, 72, 32, 94, 76, 222, 1, 82, 250, 178, 2, + 85, 186, 202, 1, 73, 178, 15, 78, 162, 222, 1, 79, 162, 8, 69, 158, 20, + 66, 2, 68, 2, 77, 2, 87, 187, 2, 65, 2, 21, 3, 69, 65, 84, 2, 251, 137, + 6, 32, 32, 242, 1, 75, 42, 82, 130, 194, 5, 83, 82, 67, 2, 80, 2, 84, + 138, 69, 70, 2, 72, 3, 89, 36, 60, 3, 79, 87, 32, 242, 202, 5, 65, 206, + 41, 85, 159, 20, 76, 28, 86, 75, 42, 82, 210, 194, 5, 67, 2, 80, 2, 84, + 138, 69, 70, 2, 72, 2, 83, 3, 89, 6, 254, 135, 6, 72, 2, 88, 187, 2, 65, + 2, 229, 140, 4, 2, 65, 84, 8, 26, 65, 135, 243, 5, 85, 7, 158, 135, 6, + 78, 3, 84, 50, 182, 1, 72, 34, 75, 176, 1, 4, 77, 65, 73, 32, 82, 82, + 136, 1, 2, 83, 65, 92, 5, 87, 73, 65, 78, 71, 172, 132, 1, 3, 68, 79, 75, + 224, 121, 3, 84, 79, 78, 161, 173, 3, 2, 67, 65, 4, 138, 183, 5, 65, 179, + 63, 79, 14, 52, 4, 72, 85, 69, 78, 134, 3, 65, 131, 232, 4, 69, 8, 80, 6, + 32, 84, 79, 78, 69, 45, 189, 186, 4, 8, 45, 76, 85, 69, 32, 75, 65, 82, + 6, 214, 134, 6, 51, 2, 52, 3, 53, 8, 56, 4, 75, 65, 78, 71, 246, 140, 1, + 89, 247, 187, 3, 83, 5, 167, 146, 1, 32, 4, 92, 3, 65, 32, 72, 21, 16, + 69, 86, 69, 82, 83, 69, 68, 32, 82, 79, 84, 65, 84, 69, 68, 32, 2, 243, + 199, 4, 65, 2, 139, 209, 3, 82, 8, 48, 3, 84, 75, 65, 142, 184, 4, 87, + 231, 123, 75, 4, 17, 2, 65, 78, 5, 163, 143, 4, 75, 5, 245, 202, 3, 2, + 87, 65, 38, 90, 65, 36, 4, 77, 65, 73, 32, 22, 79, 46, 84, 86, 85, 142, + 245, 3, 73, 199, 140, 2, 69, 9, 134, 131, 6, 65, 2, 69, 3, 73, 2, 179, + 249, 4, 83, 11, 186, 224, 3, 65, 150, 162, 2, 79, 3, 89, 4, 42, 65, 181, + 142, 1, 4, 72, 65, 77, 32, 2, 17, 2, 76, 76, 2, 175, 140, 4, 32, 9, 174, + 193, 5, 85, 163, 64, 69, 144, 1, 184, 1, 7, 76, 69, 84, 84, 69, 82, 32, + 188, 2, 5, 77, 65, 73, 32, 75, 32, 7, 83, 89, 77, 66, 79, 76, 32, 124, 9, 84, 79, 78, 69, 32, 77, 65, 73, 32, 81, 6, 86, 79, 87, 69, 76, 32, 96, 44, 4, 72, 73, 71, 72, 1, 3, 76, 79, 87, 48, 11, 32, 48, 154, 1, 75, 30, - 67, 2, 80, 2, 84, 34, 78, 166, 208, 5, 66, 2, 68, 2, 70, 2, 71, 2, 72, 2, - 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, 89, 247, 30, 79, 6, 26, 72, 187, 239, - 5, 79, 4, 194, 208, 5, 72, 247, 30, 79, 6, 162, 208, 5, 71, 2, 89, 247, - 30, 79, 4, 246, 157, 5, 65, 195, 52, 72, 10, 68, 2, 75, 79, 106, 78, 240, - 126, 4, 72, 79, 32, 72, 183, 179, 3, 83, 4, 212, 127, 3, 73, 32, 75, 179, - 238, 4, 78, 8, 58, 78, 254, 132, 1, 84, 230, 164, 2, 83, 171, 192, 2, 69, - 2, 143, 170, 3, 85, 26, 50, 65, 58, 85, 30, 73, 134, 236, 5, 69, 3, 79, - 10, 130, 214, 5, 85, 214, 22, 65, 2, 77, 2, 78, 3, 89, 9, 26, 69, 135, - 236, 5, 65, 5, 131, 236, 5, 65, 138, 1, 52, 3, 82, 73, 32, 237, 222, 4, - 4, 69, 79, 85, 84, 136, 1, 82, 65, 20, 7, 76, 69, 84, 84, 69, 82, 32, - 166, 2, 83, 78, 86, 187, 246, 3, 68, 2, 195, 211, 1, 66, 88, 210, 1, 65, - 250, 142, 2, 68, 82, 82, 34, 84, 230, 5, 85, 206, 201, 1, 73, 162, 193, - 1, 78, 126, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, 83, 138, 69, 72, 2, - 76, 2, 77, 2, 86, 2, 89, 186, 2, 69, 3, 79, 11, 172, 185, 3, 7, 82, 67, - 72, 65, 73, 67, 32, 198, 175, 2, 65, 2, 73, 3, 85, 8, 25, 4, 73, 71, 78, - 32, 8, 166, 212, 1, 78, 234, 206, 3, 65, 239, 1, 86, 18, 49, 10, 79, 87, - 69, 76, 32, 83, 73, 71, 78, 32, 18, 214, 147, 2, 65, 38, 85, 206, 201, 1, - 73, 222, 137, 2, 69, 3, 79, 4, 154, 51, 70, 155, 154, 4, 79, 188, 6, 36, - 3, 73, 76, 32, 239, 193, 4, 65, 186, 6, 154, 3, 65, 106, 67, 194, 2, 68, - 72, 3, 87, 69, 84, 48, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 232, 7, 9, - 73, 78, 32, 80, 79, 83, 83, 69, 83, 22, 76, 200, 2, 7, 78, 85, 77, 66, - 69, 82, 32, 152, 1, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, - 69, 78, 68, 32, 79, 70, 54, 82, 42, 83, 166, 14, 84, 228, 1, 7, 86, 79, - 87, 69, 76, 32, 83, 100, 2, 89, 69, 176, 186, 4, 5, 77, 79, 78, 84, 72, - 171, 118, 79, 6, 80, 5, 83, 32, 65, 66, 79, 242, 132, 2, 85, 217, 244, 1, - 5, 78, 68, 32, 79, 68, 2, 243, 197, 3, 86, 52, 80, 9, 79, 78, 83, 79, 78, - 65, 78, 84, 32, 156, 21, 3, 85, 82, 82, 171, 9, 82, 48, 130, 1, 75, 22, - 76, 22, 78, 46, 84, 218, 191, 1, 83, 194, 77, 82, 202, 210, 3, 67, 2, 72, - 2, 74, 2, 77, 2, 80, 2, 86, 3, 89, 5, 215, 162, 5, 83, 7, 243, 227, 4, - 76, 11, 134, 229, 3, 78, 134, 251, 1, 71, 3, 89, 5, 223, 223, 5, 84, 26, - 68, 2, 82, 89, 164, 28, 2, 69, 66, 238, 143, 2, 65, 227, 193, 1, 73, 2, - 137, 200, 1, 7, 32, 67, 85, 76, 84, 73, 86, 42, 168, 1, 4, 79, 78, 69, - 32, 220, 4, 6, 84, 72, 82, 69, 69, 32, 177, 215, 5, 22, 68, 79, 87, 78, - 83, 67, 65, 76, 73, 78, 71, 32, 70, 65, 67, 84, 79, 82, 32, 75, 73, 73, - 30, 112, 5, 69, 73, 71, 72, 84, 34, 70, 40, 3, 72, 65, 76, 22, 79, 88, 4, - 83, 73, 88, 84, 74, 84, 187, 215, 3, 81, 4, 210, 3, 73, 227, 216, 5, 72, - 4, 156, 3, 2, 79, 82, 187, 213, 3, 73, 4, 215, 218, 1, 70, 2, 225, 2, 18, - 78, 69, 45, 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 83, 73, 88, - 6, 232, 217, 1, 5, 69, 69, 78, 84, 72, 177, 24, 5, 89, 45, 70, 79, 85, 8, - 38, 72, 138, 1, 87, 239, 214, 3, 69, 4, 132, 1, 18, 82, 69, 69, 45, 72, - 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 84, 87, 201, 156, 4, 8, 73, - 82, 84, 89, 45, 83, 69, 67, 2, 17, 2, 69, 78, 2, 17, 2, 84, 73, 2, 207, - 214, 3, 69, 10, 58, 69, 28, 4, 83, 73, 88, 84, 82, 84, 163, 214, 3, 81, - 2, 129, 1, 3, 73, 71, 72, 4, 50, 69, 149, 215, 3, 6, 89, 45, 70, 79, 85, - 82, 2, 145, 215, 3, 2, 69, 78, 2, 21, 3, 87, 69, 78, 2, 221, 214, 3, 3, - 84, 73, 69, 2, 171, 214, 1, 83, 72, 48, 6, 69, 84, 84, 69, 82, 32, 195, - 237, 3, 65, 70, 194, 1, 78, 158, 201, 1, 84, 166, 49, 65, 82, 76, 38, 82, - 134, 6, 85, 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 242, 7, 69, - 234, 61, 67, 2, 72, 2, 74, 2, 75, 2, 77, 2, 80, 2, 86, 3, 89, 10, 46, 78, - 250, 209, 5, 71, 2, 89, 187, 2, 65, 4, 246, 209, 5, 78, 187, 2, 65, 8, - 38, 79, 130, 128, 4, 84, 131, 77, 83, 4, 11, 78, 4, 17, 2, 69, 32, 4, 18, - 72, 31, 84, 2, 217, 71, 3, 85, 78, 68, 2, 149, 132, 2, 3, 72, 79, 85, 2, - 17, 2, 32, 84, 2, 11, 69, 2, 171, 182, 5, 88, 2, 17, 2, 85, 80, 2, 131, - 182, 3, 69, 194, 4, 152, 1, 5, 65, 76, 84, 32, 80, 20, 4, 73, 71, 78, 32, - 206, 4, 80, 28, 9, 84, 65, 82, 84, 73, 78, 71, 32, 70, 41, 8, 89, 76, 76, - 65, 66, 76, 69, 32, 2, 219, 215, 3, 65, 40, 90, 65, 48, 3, 67, 69, 86, - 34, 75, 80, 2, 77, 85, 102, 85, 14, 80, 118, 86, 167, 64, 78, 4, 216, 2, - 4, 65, 90, 72, 65, 199, 136, 5, 78, 2, 11, 73, 2, 231, 170, 5, 84, 6, 38, - 85, 173, 170, 5, 3, 65, 65, 67, 4, 18, 90, 87, 82, 2, 167, 187, 5, 72, 6, - 60, 4, 75, 75, 85, 82, 16, 2, 84, 72, 21, 3, 85, 86, 85, 2, 167, 82, 85, - 2, 251, 149, 3, 65, 2, 75, 90, 8, 26, 65, 131, 254, 4, 79, 6, 34, 84, - 182, 146, 4, 65, 15, 78, 2, 11, 72, 2, 17, 2, 65, 75, 2, 179, 168, 5, 75, - 10, 38, 65, 214, 81, 69, 143, 184, 4, 73, 4, 180, 102, 3, 82, 65, 65, - 221, 243, 2, 6, 75, 65, 73, 89, 65, 82, 2, 11, 69, 2, 187, 9, 78, 2, 17, - 2, 82, 79, 2, 235, 196, 4, 77, 148, 4, 130, 1, 75, 166, 1, 76, 166, 1, - 78, 186, 1, 82, 86, 83, 178, 1, 84, 214, 2, 67, 2, 72, 2, 74, 2, 77, 2, - 80, 2, 86, 3, 89, 46, 84, 2, 83, 83, 174, 246, 1, 65, 38, 85, 202, 141, - 1, 79, 134, 60, 73, 191, 201, 1, 69, 24, 214, 239, 1, 65, 250, 6, 85, - 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 66, 78, 76, 146, 245, 1, - 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 44, 226, 6, - 76, 174, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, - 69, 110, 98, 78, 174, 5, 71, 2, 89, 174, 238, 1, 65, 38, 85, 202, 141, 1, - 79, 134, 60, 73, 191, 201, 1, 69, 44, 170, 5, 78, 174, 238, 1, 65, 38, - 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 44, 214, 4, 82, 174, - 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 68, - 90, 72, 170, 3, 83, 174, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, - 73, 191, 201, 1, 69, 24, 210, 241, 1, 65, 38, 85, 202, 141, 1, 79, 230, - 2, 82, 162, 57, 73, 191, 201, 1, 69, 44, 210, 2, 84, 174, 238, 1, 65, 38, - 85, 202, 141, 1, 79, 134, 60, 73, 191, 201, 1, 69, 6, 68, 2, 79, 84, 33, - 11, 82, 65, 68, 73, 84, 73, 79, 78, 65, 76, 32, 2, 11, 65, 2, 163, 188, - 4, 76, 4, 24, 2, 67, 82, 55, 78, 2, 17, 2, 69, 68, 2, 11, 73, 2, 211, - 187, 4, 84, 2, 11, 85, 2, 17, 2, 77, 66, 2, 123, 69, 22, 25, 4, 73, 71, - 78, 32, 22, 170, 238, 1, 65, 38, 85, 202, 141, 1, 79, 134, 60, 73, 191, - 201, 1, 69, 2, 11, 65, 2, 163, 186, 4, 82, 184, 13, 36, 5, 65, 66, 65, - 84, 65, 35, 71, 2, 11, 32, 2, 147, 209, 3, 84, 182, 13, 54, 69, 20, 3, - 83, 65, 32, 217, 6, 3, 85, 84, 32, 2, 207, 166, 4, 82, 178, 1, 52, 7, 76, - 69, 84, 84, 69, 82, 32, 155, 206, 3, 68, 158, 1, 210, 1, 65, 58, 70, 54, - 72, 38, 76, 58, 77, 54, 78, 50, 83, 126, 85, 114, 69, 2, 73, 2, 79, 2, - 86, 186, 241, 4, 84, 82, 67, 2, 68, 2, 71, 2, 75, 2, 80, 138, 69, 66, 2, - 82, 2, 87, 2, 88, 2, 89, 3, 90, 16, 150, 4, 87, 202, 185, 5, 67, 2, 81, - 2, 88, 3, 90, 4, 172, 236, 4, 5, 73, 78, 65, 76, 32, 251, 80, 65, 6, 174, - 177, 1, 84, 199, 139, 4, 65, 4, 204, 150, 4, 5, 79, 78, 71, 32, 85, 131, - 166, 1, 65, 10, 150, 188, 5, 65, 2, 67, 2, 81, 2, 88, 3, 90, 8, 170, 185, - 5, 71, 2, 72, 2, 89, 187, 2, 65, 8, 26, 72, 155, 187, 5, 65, 6, 40, 4, - 79, 82, 84, 32, 239, 186, 5, 65, 4, 188, 151, 1, 2, 85, 69, 137, 151, 3, - 2, 65, 87, 32, 58, 73, 54, 69, 202, 185, 5, 67, 2, 81, 2, 88, 3, 90, 16, - 50, 85, 202, 185, 5, 67, 2, 81, 2, 88, 3, 90, 8, 198, 185, 5, 67, 2, 81, - 2, 88, 3, 90, 130, 12, 64, 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, - 143, 150, 3, 73, 128, 12, 70, 48, 178, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 95, 55, 198, 1, 86, 48, 242, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 18, 170, 183, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, - 2, 54, 2, 55, 2, 56, 3, 57, 200, 1, 150, 1, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 138, 1, 58, 48, 2, 49, 2, 50, 2, - 51, 2, 52, 2, 53, 95, 54, 20, 186, 181, 5, 48, 2, 49, 2, 50, 2, 51, 2, - 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 18, 222, 180, 5, 48, 2, 49, 2, 50, - 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 3, 56, 4, 48, 6, 67, 65, 82, 84, 82, - 73, 21, 2, 68, 82, 2, 183, 254, 3, 68, 2, 239, 182, 4, 73, 2, 179, 173, - 3, 82, 142, 3, 154, 1, 65, 152, 2, 5, 68, 68, 89, 32, 66, 22, 76, 166, - 14, 78, 144, 1, 5, 83, 84, 32, 84, 85, 21, 12, 84, 82, 65, 71, 82, 65, - 77, 32, 70, 79, 82, 32, 10, 68, 9, 67, 85, 80, 32, 87, 73, 84, 72, 79, - 58, 82, 199, 224, 4, 80, 2, 33, 6, 85, 84, 32, 72, 65, 78, 2, 231, 140, - 4, 68, 6, 72, 9, 45, 79, 70, 70, 32, 67, 65, 76, 69, 29, 5, 68, 82, 79, - 80, 45, 2, 245, 142, 4, 2, 78, 68, 4, 250, 155, 3, 83, 245, 115, 4, 66, - 65, 82, 66, 2, 171, 142, 4, 69, 216, 1, 38, 69, 213, 2, 4, 85, 71, 85, - 32, 16, 60, 6, 80, 72, 79, 78, 69, 32, 246, 1, 83, 151, 179, 2, 86, 12, - 132, 1, 3, 82, 69, 67, 228, 150, 1, 3, 76, 79, 67, 146, 144, 3, 83, 189, - 116, 13, 79, 78, 32, 84, 79, 80, 32, 79, 70, 32, 77, 79, 68, 6, 36, 5, - 69, 73, 86, 69, 82, 51, 79, 5, 29, 5, 32, 87, 73, 84, 72, 2, 179, 38, 32, - 2, 255, 140, 3, 82, 2, 11, 67, 2, 167, 229, 3, 79, 200, 1, 162, 1, 65, - 20, 15, 70, 82, 65, 67, 84, 73, 79, 78, 32, 68, 73, 71, 73, 84, 32, 164, - 2, 2, 76, 69, 252, 3, 5, 83, 73, 71, 78, 32, 198, 2, 86, 147, 177, 3, 68, - 2, 247, 205, 1, 73, 14, 74, 84, 40, 2, 79, 78, 81, 10, 90, 69, 82, 79, - 32, 70, 79, 82, 32, 79, 8, 36, 3, 72, 82, 69, 13, 2, 87, 79, 4, 11, 69, - 4, 29, 5, 32, 70, 79, 82, 32, 4, 34, 79, 21, 4, 69, 86, 69, 78, 2, 17, 2, - 68, 68, 2, 49, 10, 32, 80, 79, 87, 69, 82, 83, 32, 79, 70, 2, 133, 22, 2, - 32, 70, 114, 44, 5, 84, 84, 69, 82, 32, 219, 230, 4, 78, 112, 214, 1, 68, - 54, 78, 106, 82, 38, 84, 138, 203, 1, 65, 82, 76, 114, 86, 186, 5, 85, - 202, 141, 1, 79, 134, 60, 73, 206, 193, 1, 83, 82, 66, 2, 67, 2, 71, 2, - 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, 2, 77, 3, 89, 10, 166, 223, 4, - 68, 138, 69, 72, 2, 90, 187, 2, 65, 10, 42, 65, 210, 163, 5, 71, 2, 78, - 3, 89, 5, 41, 8, 75, 65, 65, 82, 65, 32, 80, 79, 2, 131, 37, 76, 6, 158, - 204, 1, 82, 175, 217, 3, 65, 10, 230, 221, 4, 84, 138, 69, 72, 2, 83, - 187, 2, 65, 20, 86, 67, 194, 1, 83, 212, 35, 3, 84, 85, 85, 170, 107, 78, - 242, 60, 65, 231, 147, 3, 86, 6, 60, 9, 79, 77, 66, 73, 78, 73, 78, 71, - 32, 171, 143, 1, 65, 4, 70, 65, 145, 166, 4, 11, 67, 65, 78, 68, 82, 65, - 66, 73, 78, 68, 85, 2, 33, 6, 78, 85, 83, 86, 65, 82, 2, 235, 165, 4, 65, - 2, 11, 73, 2, 209, 231, 3, 3, 68, 68, 72, 30, 49, 10, 79, 87, 69, 76, 32, - 83, 73, 71, 78, 32, 30, 174, 206, 1, 65, 38, 85, 22, 86, 182, 141, 1, 79, - 134, 60, 73, 191, 201, 1, 69, 6, 80, 10, 78, 73, 83, 32, 82, 65, 67, 81, - 85, 69, 174, 132, 3, 71, 175, 156, 2, 84, 2, 11, 84, 2, 25, 4, 32, 65, - 78, 68, 2, 215, 152, 3, 32, 2, 199, 137, 5, 66, 162, 1, 210, 2, 65, 134, - 1, 66, 174, 1, 67, 222, 2, 68, 202, 3, 69, 154, 2, 70, 182, 1, 71, 198, - 1, 72, 64, 8, 89, 79, 85, 84, 72, 70, 85, 76, 52, 2, 73, 78, 46, 75, 98, - 76, 146, 1, 77, 134, 1, 79, 62, 80, 142, 1, 82, 210, 1, 83, 224, 1, 14, - 86, 65, 83, 84, 78, 69, 83, 83, 32, 79, 82, 32, 87, 65, 12, 2, 87, 65, - 154, 94, 85, 203, 150, 4, 74, 8, 88, 4, 67, 67, 85, 77, 22, 83, 228, 20, - 2, 68, 86, 229, 197, 1, 5, 71, 71, 82, 65, 86, 2, 239, 185, 1, 85, 2, - 151, 217, 4, 67, 6, 92, 7, 79, 76, 68, 32, 82, 69, 83, 32, 5, 82, 65, 78, - 67, 72, 177, 146, 4, 3, 65, 82, 82, 2, 249, 202, 4, 3, 79, 76, 85, 2, 11, - 73, 2, 213, 166, 1, 3, 78, 71, 32, 22, 54, 72, 20, 3, 76, 79, 83, 66, 79, - 175, 152, 2, 69, 2, 155, 229, 3, 65, 6, 26, 69, 227, 249, 3, 85, 4, 230, - 248, 1, 68, 215, 138, 2, 78, 12, 28, 3, 77, 80, 76, 35, 78, 4, 246, 17, - 73, 143, 183, 4, 69, 8, 32, 4, 83, 84, 65, 78, 23, 84, 2, 199, 135, 5, - 67, 6, 46, 69, 20, 3, 82, 65, 82, 203, 137, 4, 65, 2, 251, 199, 4, 78, 2, - 183, 170, 3, 73, 20, 80, 4, 65, 82, 75, 69, 22, 69, 210, 1, 73, 74, 85, - 249, 248, 4, 3, 79, 85, 66, 2, 203, 198, 4, 78, 6, 132, 1, 19, 70, 69, - 67, 84, 73, 86, 69, 78, 69, 83, 83, 32, 79, 82, 32, 68, 73, 83, 84, 34, - 80, 137, 7, 6, 67, 73, 83, 73, 86, 69, 2, 11, 79, 2, 215, 197, 4, 82, 2, - 11, 65, 2, 11, 82, 2, 143, 10, 84, 8, 68, 6, 70, 70, 73, 67, 85, 76, 34, - 77, 161, 12, 4, 86, 69, 82, 71, 2, 11, 84, 2, 159, 129, 4, 73, 4, 140, 1, - 2, 73, 78, 243, 194, 4, 77, 14, 100, 5, 77, 66, 69, 76, 76, 34, 78, 124, - 4, 88, 72, 65, 85, 188, 106, 3, 84, 69, 82, 147, 159, 3, 65, 2, 189, 208, - 4, 3, 73, 83, 72, 6, 80, 4, 68, 69, 65, 86, 20, 4, 76, 65, 82, 71, 129, - 146, 3, 4, 67, 79, 85, 78, 2, 171, 154, 4, 79, 2, 183, 207, 4, 69, 2, - 135, 194, 4, 83, 12, 70, 79, 76, 3, 85, 76, 76, 196, 5, 3, 65, 73, 76, - 159, 225, 4, 76, 4, 18, 76, 35, 83, 2, 225, 192, 4, 3, 76, 79, 87, 2, - 205, 4, 2, 84, 69, 4, 166, 191, 3, 32, 175, 59, 78, 10, 96, 8, 65, 84, - 72, 69, 82, 73, 78, 71, 22, 79, 64, 3, 82, 69, 65, 65, 5, 85, 65, 82, 68, - 69, 5, 215, 144, 4, 32, 2, 41, 8, 73, 78, 71, 32, 84, 79, 32, 77, 2, 211, - 218, 4, 69, 2, 75, 84, 4, 48, 2, 65, 82, 33, 6, 79, 76, 68, 73, 78, 71, - 2, 11, 68, 2, 175, 248, 3, 78, 2, 11, 32, 2, 235, 243, 1, 66, 4, 26, 67, - 211, 208, 4, 78, 2, 143, 5, 82, 4, 60, 7, 69, 69, 80, 73, 78, 71, 32, - 221, 184, 4, 2, 73, 78, 2, 11, 83, 2, 147, 134, 3, 77, 6, 18, 65, 107, - 69, 4, 60, 3, 66, 79, 85, 21, 8, 87, 32, 79, 82, 32, 77, 79, 68, 2, 243, - 187, 4, 82, 2, 175, 131, 4, 69, 2, 139, 188, 4, 71, 6, 26, 65, 30, 69, - 47, 73, 2, 153, 187, 4, 2, 83, 83, 2, 11, 65, 2, 11, 83, 2, 179, 235, 3, - 85, 2, 11, 82, 2, 171, 197, 4, 69, 4, 164, 203, 3, 7, 78, 32, 84, 72, 69, - 32, 86, 179, 110, 80, 8, 54, 65, 64, 4, 69, 78, 69, 84, 245, 97, 2, 85, - 82, 4, 22, 84, 243, 2, 67, 2, 17, 2, 84, 69, 2, 183, 186, 4, 82, 2, 139, - 200, 1, 82, 12, 30, 69, 157, 1, 2, 73, 84, 10, 34, 76, 22, 83, 243, 230, - 4, 65, 2, 227, 227, 1, 69, 6, 18, 73, 51, 80, 4, 30, 68, 137, 1, 2, 83, - 84, 2, 147, 1, 69, 2, 11, 79, 2, 187, 255, 3, 78, 2, 243, 254, 3, 85, 10, - 34, 69, 64, 2, 73, 78, 23, 84, 2, 11, 86, 2, 17, 2, 69, 82, 2, 11, 65, 2, - 211, 202, 4, 78, 2, 183, 182, 4, 75, 6, 42, 79, 237, 132, 3, 4, 82, 69, - 78, 71, 4, 26, 80, 143, 240, 4, 86, 2, 11, 80, 2, 179, 209, 3, 65, 2, 39, - 83, 4, 26, 73, 243, 227, 4, 84, 2, 147, 181, 4, 84, 224, 2, 86, 65, 180, - 31, 2, 69, 82, 246, 1, 73, 206, 1, 79, 100, 3, 82, 69, 69, 187, 8, 85, - 146, 2, 44, 4, 65, 78, 65, 32, 145, 10, 2, 73, 32, 100, 122, 65, 38, 69, - 76, 7, 76, 69, 84, 84, 69, 82, 32, 154, 7, 79, 76, 3, 73, 66, 73, 0, 3, - 85, 66, 85, 45, 2, 83, 85, 4, 184, 8, 3, 65, 66, 65, 3, 66, 6, 58, 66, 0, - 3, 69, 66, 69, 245, 7, 4, 89, 66, 69, 89, 2, 243, 7, 69, 78, 202, 1, 65, - 42, 68, 82, 70, 2, 81, 14, 71, 50, 72, 38, 75, 62, 76, 32, 3, 77, 69, 69, - 20, 2, 67, 72, 2, 74, 2, 80, 18, 78, 42, 83, 94, 84, 102, 86, 2, 87, 42, - 90, 138, 138, 3, 66, 2, 82, 3, 89, 4, 252, 1, 2, 76, 73, 203, 218, 4, 73, - 6, 30, 65, 29, 3, 72, 65, 65, 4, 218, 2, 65, 255, 1, 86, 2, 239, 219, 4, - 76, 2, 123, 65, 6, 110, 65, 86, 78, 129, 218, 4, 3, 72, 65, 73, 4, 186, - 141, 3, 72, 151, 240, 1, 65, 4, 26, 65, 255, 140, 3, 72, 2, 11, 65, 2, - 211, 218, 4, 70, 4, 18, 65, 35, 72, 2, 11, 65, 2, 159, 218, 4, 77, 2, - 211, 1, 65, 4, 236, 217, 4, 2, 79, 79, 191, 34, 65, 8, 32, 2, 65, 65, 18, - 72, 23, 69, 2, 163, 16, 68, 4, 18, 69, 87, 65, 2, 255, 216, 4, 69, 10, - 62, 65, 16, 3, 72, 65, 65, 198, 138, 3, 84, 207, 242, 1, 79, 2, 131, 1, - 86, 5, 167, 216, 4, 76, 2, 17, 2, 65, 65, 2, 255, 215, 4, 86, 6, 26, 65, - 187, 252, 4, 79, 4, 26, 86, 159, 252, 4, 65, 2, 21, 3, 73, 89, 65, 2, - 171, 232, 4, 78, 6, 48, 3, 65, 66, 79, 14, 66, 1, 3, 79, 66, 79, 2, 23, - 65, 2, 11, 79, 2, 11, 70, 2, 11, 73, 2, 179, 231, 4, 76, 2, 199, 250, 3, - 75, 174, 1, 26, 67, 167, 137, 3, 68, 154, 1, 132, 1, 9, 72, 65, 82, 65, - 67, 84, 69, 82, 32, 129, 207, 4, 17, 85, 82, 82, 69, 78, 67, 89, 32, 83, - 89, 77, 66, 79, 76, 32, 66, 65, 152, 1, 176, 2, 6, 66, 79, 32, 66, 65, - 73, 16, 6, 67, 72, 79, 32, 67, 72, 56, 3, 68, 79, 32, 44, 2, 70, 79, 76, - 3, 72, 79, 32, 66, 75, 190, 1, 76, 138, 1, 77, 252, 1, 7, 65, 78, 71, 75, - 72, 65, 78, 46, 78, 158, 1, 80, 234, 1, 82, 70, 83, 238, 2, 84, 224, 2, - 3, 87, 79, 32, 34, 89, 141, 142, 4, 2, 79, 32, 2, 143, 12, 77, 8, 212, - 251, 2, 2, 65, 78, 162, 170, 1, 73, 167, 58, 79, 4, 176, 209, 4, 3, 67, - 72, 65, 227, 33, 68, 6, 32, 2, 32, 70, 21, 2, 78, 71, 4, 227, 250, 2, 65, - 2, 211, 171, 3, 77, 4, 40, 4, 78, 79, 75, 72, 235, 243, 4, 72, 2, 251, - 241, 4, 85, 14, 40, 2, 72, 79, 229, 9, 3, 79, 32, 75, 12, 26, 32, 235, - 250, 3, 77, 10, 36, 2, 75, 72, 57, 3, 82, 65, 75, 8, 158, 9, 87, 138, - 215, 3, 85, 138, 68, 79, 139, 60, 65, 2, 131, 155, 4, 72, 8, 76, 2, 79, - 32, 224, 11, 8, 65, 75, 75, 72, 65, 78, 71, 89, 151, 231, 4, 85, 4, 32, - 2, 67, 72, 207, 161, 4, 76, 2, 183, 208, 4, 85, 16, 28, 2, 65, 73, 219, - 52, 79, 14, 42, 32, 176, 1, 3, 84, 65, 73, 19, 89, 10, 76, 4, 67, 72, 65, - 84, 44, 5, 72, 65, 78, 45, 65, 22, 84, 203, 237, 4, 69, 2, 11, 84, 2, 11, - 65, 2, 207, 238, 4, 87, 2, 139, 221, 3, 75, 4, 234, 209, 4, 72, 159, 11, - 82, 2, 203, 2, 75, 2, 205, 198, 4, 2, 65, 77, 8, 60, 3, 71, 79, 32, 32, - 3, 73, 75, 72, 29, 3, 79, 32, 78, 2, 11, 78, 2, 223, 202, 4, 71, 2, 249, - 210, 4, 2, 65, 72, 4, 182, 159, 4, 69, 227, 79, 85, 12, 68, 6, 65, 73, - 89, 65, 78, 78, 22, 72, 133, 204, 4, 3, 79, 32, 80, 2, 215, 218, 4, 79, - 8, 36, 3, 73, 78, 84, 21, 2, 79, 32, 2, 139, 201, 4, 72, 6, 44, 2, 80, - 72, 161, 5, 4, 83, 65, 77, 80, 4, 182, 156, 4, 85, 155, 1, 65, 4, 32, 2, - 79, 32, 239, 236, 4, 85, 2, 11, 82, 2, 167, 234, 4, 85, 36, 44, 4, 65, - 82, 65, 32, 225, 1, 2, 79, 32, 28, 62, 65, 130, 1, 85, 130, 225, 2, 73, - 222, 137, 2, 69, 3, 79, 13, 64, 6, 73, 32, 77, 65, 73, 77, 154, 235, 4, - 65, 2, 69, 3, 77, 4, 26, 65, 255, 160, 3, 85, 2, 11, 76, 2, 151, 215, 4, - 65, 9, 186, 170, 4, 69, 163, 64, 85, 8, 24, 2, 82, 85, 23, 83, 2, 199, - 214, 4, 83, 6, 242, 199, 4, 65, 226, 31, 85, 187, 2, 79, 18, 30, 72, 133, - 2, 2, 79, 32, 14, 48, 6, 65, 78, 84, 72, 65, 75, 21, 2, 79, 32, 2, 163, - 213, 3, 72, 12, 80, 8, 78, 65, 78, 71, 77, 79, 78, 84, 20, 4, 80, 72, 85, - 84, 13, 2, 84, 72, 2, 175, 201, 4, 72, 2, 123, 72, 8, 34, 65, 234, 150, - 4, 79, 3, 85, 4, 222, 157, 3, 72, 131, 202, 1, 78, 4, 38, 84, 209, 177, - 2, 3, 80, 65, 84, 2, 159, 200, 4, 65, 2, 11, 87, 2, 139, 147, 3, 65, 6, - 30, 65, 45, 3, 79, 32, 89, 2, 21, 3, 77, 65, 75, 2, 167, 156, 3, 75, 4, - 154, 149, 4, 73, 227, 77, 65, 10, 30, 69, 145, 1, 2, 77, 79, 6, 18, 32, - 107, 70, 4, 84, 11, 68, 79, 69, 83, 32, 78, 79, 84, 32, 69, 88, 189, 208, - 3, 4, 69, 88, 73, 83, 2, 235, 122, 73, 2, 159, 196, 3, 79, 4, 46, 77, - 141, 188, 3, 5, 68, 89, 78, 65, 77, 2, 155, 172, 3, 69, 10, 18, 78, 95, - 82, 8, 26, 32, 179, 219, 3, 75, 6, 26, 83, 251, 218, 2, 71, 4, 206, 206, - 2, 65, 223, 215, 1, 80, 2, 21, 3, 68, 32, 80, 2, 25, 4, 76, 65, 67, 69, - 2, 17, 2, 32, 77, 2, 131, 207, 3, 69, 4, 56, 4, 85, 71, 72, 84, 161, 206, - 3, 4, 78, 71, 32, 83, 2, 161, 187, 1, 5, 32, 66, 65, 76, 76, 46, 22, 32, - 207, 3, 45, 30, 154, 1, 68, 90, 76, 118, 82, 198, 153, 1, 66, 86, 67, - 194, 6, 83, 134, 167, 1, 80, 137, 22, 15, 78, 69, 84, 87, 79, 82, 75, 69, - 68, 32, 67, 79, 77, 80, 85, 4, 64, 10, 73, 77, 69, 78, 83, 73, 79, 78, - 65, 76, 163, 157, 1, 79, 2, 139, 187, 3, 32, 6, 80, 12, 73, 78, 69, 83, - 32, 67, 79, 78, 86, 69, 82, 71, 149, 150, 1, 2, 69, 70, 4, 225, 179, 4, - 3, 73, 78, 71, 10, 40, 4, 65, 89, 83, 32, 183, 149, 1, 73, 8, 146, 192, - 2, 66, 166, 161, 1, 65, 130, 82, 76, 31, 82, 16, 44, 2, 68, 32, 254, 3, - 80, 147, 156, 1, 69, 12, 196, 2, 12, 84, 79, 80, 45, 76, 73, 71, 72, 84, - 69, 68, 32, 80, 17, 76, 69, 70, 84, 45, 76, 73, 71, 72, 84, 69, 68, 32, - 68, 79, 87, 78, 0, 16, 82, 73, 71, 72, 84, 45, 76, 73, 71, 72, 84, 69, - 68, 32, 85, 80, 245, 132, 2, 25, 66, 79, 84, 84, 79, 77, 45, 76, 73, 71, - 72, 84, 69, 68, 32, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 6, 76, 4, 76, - 69, 70, 84, 69, 11, 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 2, 11, - 87, 2, 25, 4, 65, 82, 68, 83, 2, 197, 238, 1, 2, 32, 69, 4, 170, 238, 1, - 69, 159, 22, 65, 2, 249, 154, 4, 5, 69, 82, 45, 69, 77, 8, 34, 77, 85, 4, - 78, 68, 69, 82, 4, 21, 3, 66, 83, 32, 4, 38, 85, 149, 223, 2, 3, 68, 79, - 87, 2, 179, 209, 3, 80, 4, 180, 171, 3, 10, 32, 67, 76, 79, 85, 68, 32, - 65, 78, 68, 161, 46, 2, 83, 84, 230, 5, 204, 1, 6, 66, 69, 84, 65, 78, - 32, 184, 45, 6, 69, 32, 79, 86, 69, 82, 72, 7, 70, 73, 78, 65, 71, 72, - 32, 150, 9, 71, 152, 1, 3, 76, 68, 69, 252, 2, 2, 77, 69, 56, 2, 78, 89, - 78, 82, 139, 199, 3, 67, 160, 3, 228, 2, 18, 65, 83, 84, 82, 79, 76, 79, - 71, 73, 67, 65, 76, 32, 83, 73, 71, 78, 32, 172, 1, 18, 67, 65, 78, 84, - 73, 76, 76, 65, 84, 73, 79, 78, 32, 83, 73, 71, 78, 32, 208, 1, 6, 68, - 73, 71, 73, 84, 32, 100, 9, 75, 85, 32, 82, 85, 32, 75, 72, 65, 26, 76, - 176, 3, 5, 77, 65, 82, 75, 32, 222, 18, 83, 181, 15, 11, 86, 79, 87, 69, - 76, 32, 83, 73, 71, 78, 32, 6, 46, 83, 205, 40, 6, 45, 75, 72, 89, 85, - 68, 4, 104, 8, 68, 79, 78, 71, 32, 84, 83, 72, 205, 21, 13, 71, 82, 65, - 32, 71, 67, 65, 78, 32, 45, 67, 72, 65, 2, 235, 31, 85, 8, 144, 1, 5, 72, - 69, 65, 86, 89, 0, 5, 76, 73, 71, 72, 84, 40, 7, 83, 66, 85, 66, 32, 45, - 67, 193, 171, 4, 8, 67, 65, 78, 71, 32, 84, 69, 45, 2, 17, 2, 32, 66, 2, - 211, 188, 3, 69, 2, 231, 198, 3, 72, 40, 156, 223, 2, 4, 72, 65, 76, 70, - 82, 70, 30, 83, 42, 84, 62, 90, 238, 85, 78, 14, 79, 223, 110, 69, 5, - 221, 10, 2, 32, 66, 92, 96, 6, 69, 84, 84, 69, 82, 32, 153, 2, 13, 79, - 71, 79, 84, 89, 80, 69, 32, 83, 73, 71, 78, 32, 88, 230, 1, 75, 162, 115, - 82, 212, 142, 3, 10, 70, 73, 88, 69, 68, 45, 70, 79, 82, 77, 202, 1, 68, - 86, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 80, 2, 90, 138, 69, 45, - 2, 72, 2, 74, 2, 76, 2, 77, 2, 87, 2, 89, 187, 2, 65, 8, 134, 202, 4, 83, - 14, 72, 2, 75, 187, 2, 65, 4, 224, 25, 3, 76, 72, 65, 13, 4, 67, 72, 65, - 68, 80, 254, 2, 66, 176, 1, 8, 77, 78, 89, 65, 77, 32, 89, 73, 114, 67, - 172, 2, 13, 89, 73, 71, 32, 77, 71, 79, 32, 84, 83, 72, 69, 71, 190, 1, - 71, 212, 2, 9, 65, 78, 71, 32, 75, 72, 65, 78, 71, 58, 72, 48, 2, 73, 78, - 246, 1, 78, 222, 1, 82, 118, 83, 46, 84, 36, 4, 76, 69, 65, 68, 212, 100, - 2, 80, 65, 181, 191, 1, 17, 68, 69, 76, 73, 77, 73, 84, 69, 82, 32, 84, - 83, 72, 69, 71, 32, 66, 10, 52, 9, 75, 65, 45, 32, 83, 72, 79, 71, 32, - 27, 83, 4, 142, 1, 71, 67, 89, 6, 34, 75, 201, 21, 3, 68, 85, 83, 4, 56, - 6, 65, 45, 32, 83, 72, 79, 89, 4, 85, 82, 32, 89, 2, 21, 3, 71, 32, 71, - 2, 41, 8, 73, 32, 77, 71, 79, 32, 82, 71, 2, 243, 252, 2, 89, 2, 221, 2, - 2, 73, 71, 12, 84, 5, 65, 82, 69, 84, 32, 240, 1, 2, 72, 69, 29, 7, 76, - 79, 83, 73, 78, 71, 32, 6, 112, 12, 45, 68, 90, 85, 68, 32, 82, 84, 65, - 71, 83, 32, 97, 12, 89, 73, 71, 32, 77, 71, 79, 32, 80, 72, 85, 82, 4, - 42, 66, 37, 6, 77, 69, 32, 76, 79, 78, 2, 33, 6, 90, 72, 73, 32, 77, 73, - 2, 203, 22, 71, 2, 225, 3, 3, 32, 83, 72, 2, 149, 183, 2, 2, 32, 77, 4, - 68, 13, 66, 82, 68, 65, 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, 2, - 213, 5, 11, 73, 71, 32, 77, 71, 79, 32, 83, 71, 65, 66, 12, 68, 4, 84, - 69, 82, 32, 141, 2, 8, 85, 71, 32, 82, 84, 65, 71, 83, 8, 56, 8, 89, 73, - 71, 32, 77, 71, 79, 32, 227, 251, 3, 84, 6, 68, 4, 45, 85, 77, 32, 117, - 9, 84, 82, 85, 78, 67, 65, 84, 69, 68, 4, 88, 7, 82, 78, 65, 77, 32, 66, - 67, 245, 2, 10, 71, 84, 69, 82, 32, 84, 83, 72, 69, 71, 2, 241, 2, 2, 65, - 68, 2, 231, 189, 4, 32, 4, 21, 3, 32, 71, 89, 4, 146, 240, 3, 79, 135, - 18, 65, 2, 17, 2, 65, 76, 2, 241, 154, 4, 2, 65, 78, 6, 92, 6, 73, 84, - 73, 65, 76, 32, 181, 248, 3, 11, 84, 69, 82, 83, 89, 76, 76, 65, 66, 73, - 67, 4, 68, 13, 66, 82, 68, 65, 32, 82, 78, 89, 73, 78, 71, 32, 89, 3, 89, - 2, 53, 11, 73, 71, 32, 77, 71, 79, 32, 77, 68, 85, 78, 2, 179, 156, 4, - 32, 10, 72, 10, 71, 65, 83, 32, 66, 90, 85, 78, 71, 32, 77, 4, 89, 73, - 83, 32, 4, 56, 3, 83, 71, 79, 145, 154, 4, 5, 78, 89, 73, 32, 90, 2, 251, - 9, 82, 6, 44, 5, 84, 83, 72, 69, 71, 179, 245, 3, 83, 5, 163, 245, 3, 32, - 4, 128, 245, 3, 8, 71, 89, 65, 32, 71, 82, 65, 77, 1, 14, 73, 78, 32, 67, - 72, 69, 78, 32, 83, 80, 85, 78, 71, 83, 4, 140, 244, 3, 4, 66, 82, 85, - 76, 39, 72, 6, 32, 4, 82, 65, 73, 76, 55, 83, 2, 225, 7, 9, 73, 78, 71, - 32, 77, 67, 72, 65, 78, 4, 56, 5, 65, 32, 45, 80, 72, 209, 242, 3, 3, 72, - 69, 71, 2, 183, 148, 4, 82, 156, 1, 84, 4, 73, 71, 78, 32, 228, 6, 9, 85, - 66, 74, 79, 73, 78, 69, 68, 32, 143, 4, 89, 42, 176, 1, 4, 71, 82, 85, - 32, 96, 2, 76, 67, 30, 77, 36, 11, 78, 89, 73, 32, 90, 76, 65, 32, 78, - 65, 65, 22, 82, 252, 2, 2, 89, 65, 130, 4, 73, 165, 4, 5, 83, 78, 65, 32, - 76, 4, 40, 3, 67, 65, 78, 1, 3, 77, 69, 68, 2, 25, 4, 32, 82, 71, 89, 2, - 169, 4, 2, 73, 78, 4, 238, 3, 73, 179, 4, 69, 4, 136, 4, 2, 65, 82, 231, - 3, 67, 2, 215, 144, 4, 32, 20, 116, 4, 68, 69, 76, 32, 240, 1, 10, 74, - 69, 83, 32, 83, 85, 32, 78, 71, 65, 181, 236, 3, 6, 78, 65, 77, 32, 66, - 67, 16, 52, 5, 68, 75, 65, 82, 32, 53, 4, 78, 65, 71, 32, 8, 94, 71, 181, - 202, 2, 6, 82, 68, 69, 76, 32, 78, 8, 42, 71, 69, 6, 82, 68, 69, 76, 32, - 68, 6, 46, 67, 212, 143, 2, 2, 78, 89, 167, 122, 83, 2, 179, 178, 4, 73, - 2, 187, 145, 3, 75, 2, 199, 195, 2, 32, 4, 18, 78, 71, 82, 2, 11, 71, 2, - 21, 3, 32, 82, 84, 2, 11, 65, 2, 171, 244, 3, 71, 2, 17, 2, 32, 84, 2, - 219, 157, 3, 83, 94, 68, 7, 76, 69, 84, 84, 69, 82, 32, 145, 2, 5, 83, - 73, 71, 78, 32, 88, 220, 1, 10, 70, 73, 88, 69, 68, 45, 70, 79, 82, 77, - 150, 230, 3, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, - 80, 2, 90, 138, 69, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 87, 2, 89, - 187, 2, 65, 6, 11, 32, 6, 222, 172, 4, 82, 2, 87, 3, 89, 6, 38, 73, 50, - 77, 33, 3, 76, 67, 69, 2, 45, 9, 78, 86, 69, 82, 84, 69, 68, 32, 77, 2, - 11, 67, 2, 45, 2, 72, 85, 2, 25, 4, 32, 84, 83, 65, 2, 11, 32, 2, 211, - 227, 2, 67, 20, 60, 6, 76, 76, 65, 66, 76, 69, 21, 5, 77, 66, 79, 76, 32, - 2, 155, 156, 4, 32, 18, 104, 4, 68, 82, 73, 76, 32, 6, 78, 79, 82, 32, - 66, 85, 130, 1, 80, 93, 7, 82, 68, 79, 32, 82, 74, 69, 2, 11, 32, 2, 139, - 135, 4, 66, 9, 11, 32, 6, 72, 4, 66, 90, 72, 73, 0, 4, 71, 83, 85, 77, 1, - 4, 78, 89, 73, 83, 2, 177, 247, 1, 5, 32, 45, 75, 72, 89, 4, 52, 6, 65, - 68, 77, 65, 32, 71, 21, 3, 72, 85, 82, 2, 167, 224, 2, 68, 2, 191, 112, - 32, 5, 237, 238, 2, 6, 32, 82, 71, 89, 65, 32, 30, 116, 8, 82, 69, 86, - 69, 82, 83, 69, 68, 186, 85, 85, 22, 86, 182, 141, 1, 79, 134, 60, 73, - 190, 201, 1, 69, 235, 61, 65, 4, 129, 159, 2, 2, 32, 73, 2, 25, 4, 32, - 73, 78, 70, 2, 11, 73, 2, 11, 78, 2, 151, 187, 2, 73, 118, 216, 1, 9, 67, - 79, 78, 83, 79, 78, 65, 78, 84, 20, 7, 76, 69, 84, 84, 69, 82, 32, 236, - 6, 20, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, 32, - 76, 65, 66, 73, 37, 8, 83, 69, 80, 65, 82, 65, 84, 79, 2, 131, 232, 3, - 32, 112, 106, 65, 108, 17, 66, 69, 82, 66, 69, 82, 32, 65, 67, 65, 68, - 69, 77, 89, 32, 89, 65, 30, 84, 223, 1, 89, 4, 84, 6, 89, 69, 82, 32, 89, - 65, 173, 163, 4, 9, 72, 65, 71, 71, 65, 82, 32, 89, 65, 2, 183, 163, 4, - 71, 4, 178, 164, 4, 72, 3, 74, 18, 92, 11, 65, 87, 69, 76, 76, 69, 77, - 69, 84, 32, 89, 33, 8, 85, 65, 82, 69, 71, 32, 89, 65, 2, 11, 65, 2, 171, - 163, 4, 90, 16, 62, 71, 190, 2, 75, 142, 159, 4, 90, 62, 78, 86, 72, 3, - 81, 4, 214, 162, 4, 72, 3, 78, 86, 54, 65, 210, 2, 69, 182, 159, 4, 73, - 2, 79, 3, 85, 77, 190, 1, 68, 30, 71, 2, 75, 14, 66, 2, 72, 22, 83, 30, - 84, 30, 90, 138, 77, 82, 186, 209, 3, 67, 146, 1, 65, 2, 70, 2, 74, 2, - 76, 2, 77, 2, 78, 2, 80, 2, 81, 2, 86, 2, 87, 3, 89, 9, 38, 68, 159, 160, - 4, 72, 7, 11, 72, 5, 155, 160, 4, 72, 7, 134, 160, 4, 72, 3, 83, 7, 234, - 159, 4, 72, 3, 84, 7, 206, 159, 4, 72, 3, 90, 5, 179, 159, 4, 89, 2, 233, - 181, 3, 4, 65, 76, 73, 90, 2, 183, 221, 3, 82, 6, 76, 2, 69, 82, 21, 13, - 72, 84, 32, 84, 82, 73, 70, 79, 76, 73, 65, 84, 69, 5, 131, 225, 3, 32, - 2, 33, 6, 32, 83, 78, 79, 87, 70, 2, 175, 53, 76, 19, 11, 32, 16, 72, 8, - 79, 80, 69, 82, 65, 84, 79, 82, 233, 1, 5, 87, 73, 84, 72, 32, 11, 11, - 32, 8, 38, 65, 101, 5, 87, 73, 84, 72, 32, 4, 25, 4, 66, 79, 86, 69, 4, - 11, 32, 4, 26, 76, 139, 252, 2, 82, 2, 11, 69, 2, 147, 252, 2, 70, 4, 26, - 82, 251, 154, 3, 68, 2, 17, 2, 73, 83, 2, 241, 202, 2, 3, 73, 78, 71, 6, - 26, 68, 143, 234, 1, 82, 4, 11, 79, 4, 175, 252, 1, 84, 6, 22, 82, 239, - 125, 83, 2, 11, 32, 2, 243, 171, 1, 67, 5, 21, 3, 32, 84, 87, 2, 21, 3, - 79, 32, 68, 2, 237, 86, 3, 79, 84, 83, 170, 1, 100, 5, 72, 85, 84, 65, - 32, 184, 6, 11, 79, 78, 73, 65, 78, 32, 83, 73, 71, 78, 32, 227, 129, 3, - 69, 164, 1, 178, 1, 65, 88, 7, 76, 69, 84, 84, 69, 82, 32, 204, 1, 2, 83, - 73, 200, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 162, 162, 2, - 68, 204, 152, 1, 2, 71, 86, 255, 71, 79, 4, 18, 66, 51, 78, 2, 29, 5, 66, - 82, 69, 86, 73, 2, 171, 22, 65, 2, 143, 131, 4, 74, 94, 178, 60, 65, 38, - 68, 114, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, 162, 193, 1, 78, 46, - 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, 77, - 2, 82, 2, 89, 186, 2, 69, 3, 79, 12, 21, 3, 71, 78, 32, 12, 46, 67, 98, - 78, 242, 60, 65, 231, 147, 3, 86, 2, 11, 65, 2, 11, 78, 2, 25, 4, 68, 82, - 65, 66, 2, 11, 73, 2, 11, 78, 2, 139, 239, 3, 68, 2, 11, 85, 2, 135, 239, - 3, 75, 30, 78, 83, 166, 63, 65, 38, 85, 22, 86, 186, 201, 1, 73, 222, - 137, 2, 69, 3, 79, 4, 25, 4, 72, 79, 82, 84, 4, 11, 32, 4, 198, 146, 4, - 69, 3, 79, 4, 144, 221, 3, 8, 67, 65, 80, 73, 84, 65, 76, 32, 239, 24, - 69, 146, 2, 140, 2, 12, 68, 72, 82, 73, 32, 76, 69, 84, 84, 69, 82, 32, - 182, 4, 77, 18, 78, 34, 79, 80, 2, 80, 32, 168, 17, 23, 82, 84, 79, 73, - 83, 69, 32, 83, 72, 69, 76, 76, 32, 66, 82, 65, 67, 75, 69, 84, 69, 68, - 32, 130, 4, 84, 222, 149, 2, 73, 149, 70, 5, 75, 89, 79, 32, 84, 104, - 242, 1, 71, 42, 74, 30, 77, 34, 78, 70, 72, 34, 80, 34, 83, 210, 50, 82, - 206, 147, 1, 79, 134, 60, 69, 42, 76, 198, 2, 65, 178, 191, 1, 67, 2, 68, - 2, 75, 2, 84, 2, 88, 2, 90, 138, 69, 66, 2, 70, 2, 81, 2, 86, 186, 2, 73, - 2, 85, 3, 89, 6, 170, 139, 4, 72, 2, 74, 187, 2, 65, 4, 186, 141, 4, 65, - 3, 89, 4, 230, 138, 4, 66, 187, 2, 65, 14, 66, 71, 190, 194, 2, 74, 194, - 130, 1, 88, 138, 69, 68, 187, 2, 65, 4, 130, 138, 4, 74, 187, 2, 65, 4, - 226, 137, 4, 83, 187, 2, 65, 10, 54, 72, 198, 193, 2, 75, 202, 199, 1, - 84, 187, 2, 65, 4, 138, 137, 4, 84, 187, 2, 65, 2, 231, 28, 65, 2, 11, - 71, 2, 179, 244, 3, 85, 6, 32, 2, 84, 72, 151, 254, 2, 76, 5, 11, 66, 2, - 11, 82, 2, 151, 162, 2, 85, 72, 252, 1, 4, 65, 82, 67, 32, 162, 2, 67, - 44, 2, 72, 65, 166, 3, 80, 140, 3, 13, 74, 85, 83, 84, 73, 70, 73, 69, - 68, 32, 76, 79, 87, 84, 5, 76, 69, 70, 84, 32, 228, 1, 6, 82, 73, 71, 72, - 84, 32, 170, 2, 83, 38, 84, 81, 7, 87, 73, 84, 72, 32, 85, 80, 6, 180, 1, - 19, 65, 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, - 79, 87, 73, 21, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, - 87, 32, 87, 73, 84, 72, 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, - 80, 2, 187, 128, 2, 76, 2, 11, 77, 2, 143, 128, 2, 73, 2, 11, 85, 2, 133, - 179, 3, 3, 82, 76, 89, 34, 36, 3, 76, 70, 32, 179, 133, 4, 84, 32, 70, - 70, 186, 1, 76, 22, 82, 178, 2, 83, 250, 5, 66, 231, 243, 1, 73, 8, 136, - 1, 15, 79, 82, 87, 65, 82, 68, 45, 70, 65, 67, 73, 78, 71, 32, 82, 157, - 2, 13, 76, 65, 73, 76, 73, 78, 71, 32, 82, 79, 66, 79, 84, 4, 22, 85, - 243, 1, 79, 2, 203, 197, 3, 78, 8, 41, 2, 69, 70, 8, 21, 3, 73, 71, 72, - 8, 11, 84, 8, 54, 32, 69, 9, 45, 70, 65, 67, 73, 78, 71, 32, 82, 2, 11, - 80, 2, 33, 6, 65, 82, 69, 78, 84, 72, 2, 207, 222, 1, 69, 6, 38, 79, 21, - 5, 85, 78, 78, 69, 82, 2, 155, 177, 3, 66, 4, 29, 5, 32, 70, 82, 65, 77, - 4, 11, 69, 4, 11, 45, 4, 134, 129, 4, 49, 3, 50, 4, 18, 69, 59, 84, 2, - 11, 67, 2, 11, 84, 2, 11, 73, 2, 147, 135, 2, 79, 2, 17, 2, 65, 78, 2, - 11, 68, 2, 21, 3, 73, 78, 71, 2, 17, 2, 32, 80, 2, 11, 69, 2, 11, 82, 2, - 207, 175, 3, 83, 4, 17, 2, 69, 82, 4, 33, 6, 32, 72, 65, 76, 70, 32, 4, - 250, 3, 66, 139, 105, 87, 10, 180, 1, 13, 74, 85, 83, 84, 73, 70, 73, 69, - 68, 32, 76, 79, 87, 126, 67, 50, 72, 225, 214, 2, 21, 66, 76, 65, 67, 75, - 32, 76, 69, 70, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 83, 2, 237, - 1, 7, 69, 82, 32, 82, 73, 71, 72, 8, 78, 67, 50, 72, 33, 13, 74, 85, 83, - 84, 73, 70, 73, 69, 68, 32, 76, 79, 87, 4, 26, 79, 243, 130, 3, 82, 2, - 139, 190, 3, 82, 2, 253, 168, 3, 3, 65, 76, 70, 2, 33, 6, 69, 82, 32, 76, - 69, 70, 2, 53, 11, 84, 32, 81, 85, 65, 82, 84, 69, 82, 32, 66, 2, 11, 76, - 2, 209, 168, 2, 3, 65, 67, 75, 2, 201, 167, 3, 4, 81, 85, 65, 82, 2, 11, - 79, 2, 165, 167, 3, 12, 82, 84, 79, 73, 83, 69, 32, 83, 72, 69, 76, 76, - 2, 29, 5, 87, 65, 82, 68, 83, 2, 17, 2, 32, 65, 2, 153, 252, 2, 4, 82, - 82, 79, 87, 20, 188, 1, 22, 67, 74, 75, 32, 85, 78, 73, 70, 73, 69, 68, - 32, 73, 68, 69, 79, 71, 82, 65, 80, 72, 45, 161, 2, 19, 76, 65, 84, 73, - 78, 32, 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 18, 40, 2, - 52, 69, 34, 53, 54, 54, 87, 55, 4, 202, 1, 48, 203, 205, 2, 56, 4, 30, - 50, 141, 1, 2, 66, 56, 2, 175, 176, 3, 68, 6, 48, 2, 50, 53, 22, 53, 249, - 205, 2, 2, 55, 50, 2, 135, 246, 3, 51, 2, 67, 53, 4, 32, 2, 48, 66, 21, - 2, 54, 68, 2, 195, 245, 3, 57, 2, 175, 245, 3, 55, 2, 11, 82, 2, 179, - 183, 3, 32, 64, 48, 6, 65, 76, 32, 82, 85, 78, 21, 2, 79, 32, 2, 247, - 250, 2, 79, 62, 72, 7, 76, 69, 84, 84, 69, 82, 32, 217, 2, 6, 83, 73, 71, - 78, 32, 82, 60, 206, 1, 66, 106, 78, 138, 30, 73, 202, 141, 1, 69, 162, - 75, 67, 162, 186, 1, 65, 234, 61, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, - 2, 77, 2, 80, 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 79, 3, 85, 12, - 52, 7, 82, 69, 65, 84, 72, 89, 32, 227, 241, 3, 65, 10, 182, 30, 73, 202, - 141, 1, 69, 143, 175, 2, 65, 4, 246, 238, 3, 71, 187, 2, 65, 2, 17, 2, - 73, 83, 2, 21, 3, 73, 78, 71, 2, 11, 32, 2, 147, 198, 2, 84, 98, 46, 65, - 254, 1, 73, 246, 11, 79, 195, 1, 85, 16, 66, 67, 36, 2, 68, 69, 34, 77, - 20, 2, 78, 83, 155, 159, 3, 73, 4, 246, 231, 1, 75, 243, 206, 1, 84, 2, - 141, 16, 4, 32, 77, 65, 82, 5, 131, 212, 1, 32, 4, 44, 2, 80, 76, 33, 5, - 86, 69, 82, 83, 65, 2, 11, 85, 2, 199, 207, 3, 84, 2, 171, 62, 76, 66, - 156, 1, 3, 65, 78, 71, 176, 3, 8, 68, 69, 78, 84, 32, 69, 77, 66, 20, 9, - 71, 82, 65, 77, 32, 70, 79, 82, 32, 140, 2, 4, 80, 76, 69, 32, 167, 151, - 3, 67, 16, 48, 2, 76, 69, 245, 1, 5, 85, 76, 65, 82, 32, 10, 44, 6, 32, - 87, 73, 84, 72, 32, 163, 1, 45, 8, 80, 9, 83, 69, 82, 73, 70, 83, 32, 65, - 84, 54, 85, 138, 168, 2, 82, 187, 66, 68, 2, 17, 2, 32, 66, 2, 197, 218, - 3, 3, 79, 84, 84, 2, 241, 60, 2, 78, 68, 2, 205, 202, 2, 4, 72, 69, 65, - 68, 6, 68, 9, 70, 76, 65, 71, 32, 79, 78, 32, 80, 34, 82, 179, 138, 2, - 66, 2, 11, 79, 2, 207, 205, 3, 83, 2, 11, 85, 2, 239, 171, 3, 76, 2, 155, - 215, 3, 76, 16, 66, 69, 34, 72, 34, 76, 22, 77, 46, 84, 42, 87, 255, 210, - 1, 70, 2, 11, 65, 2, 211, 230, 1, 82, 2, 11, 69, 2, 211, 248, 1, 65, 2, - 215, 184, 3, 65, 2, 21, 3, 79, 85, 78, 2, 175, 187, 2, 84, 2, 17, 2, 72, - 85, 2, 215, 199, 1, 78, 4, 206, 175, 2, 65, 131, 42, 73, 30, 176, 2, 3, - 67, 79, 76, 24, 20, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, 65, - 82, 32, 87, 73, 84, 72, 32, 98, 80, 34, 84, 20, 13, 86, 69, 82, 84, 73, - 67, 65, 76, 32, 66, 65, 82, 32, 44, 9, 83, 79, 76, 73, 68, 85, 83, 32, - 66, 188, 21, 2, 68, 65, 168, 187, 1, 6, 78, 69, 83, 84, 69, 68, 171, 11, - 73, 2, 129, 37, 2, 79, 78, 4, 34, 68, 33, 4, 84, 82, 73, 80, 2, 17, 2, - 79, 85, 2, 11, 66, 2, 185, 179, 3, 2, 76, 69, 4, 254, 221, 1, 76, 135, - 110, 82, 2, 167, 202, 1, 73, 8, 42, 66, 74, 68, 134, 189, 1, 82, 115, 87, - 2, 29, 5, 73, 78, 65, 82, 89, 2, 21, 3, 32, 82, 69, 2, 235, 32, 76, 2, - 25, 4, 69, 76, 73, 77, 2, 191, 170, 2, 73, 10, 24, 2, 76, 76, 35, 80, 5, - 137, 220, 1, 3, 69, 89, 66, 6, 44, 5, 73, 67, 65, 76, 32, 243, 207, 3, - 72, 4, 18, 68, 47, 70, 2, 11, 82, 2, 11, 73, 2, 243, 221, 3, 78, 2, 215, - 248, 1, 73, 6, 18, 69, 79, 77, 5, 129, 188, 3, 14, 32, 76, 73, 71, 72, - 84, 32, 77, 79, 79, 78, 32, 65, 82, 2, 235, 170, 3, 80, 216, 1, 76, 3, - 71, 82, 73, 22, 76, 214, 12, 82, 221, 186, 2, 5, 77, 66, 76, 69, 82, 2, - 155, 216, 2, 75, 162, 1, 68, 11, 85, 45, 84, 73, 71, 65, 76, 65, 82, 73, - 32, 179, 222, 3, 73, 160, 1, 122, 65, 42, 67, 30, 68, 106, 71, 32, 7, 76, - 69, 84, 84, 69, 82, 32, 178, 3, 82, 32, 5, 83, 73, 71, 78, 32, 131, 3, - 86, 2, 11, 85, 2, 177, 155, 3, 2, 32, 76, 2, 245, 158, 3, 2, 79, 78, 4, - 18, 79, 67, 65, 2, 11, 85, 2, 11, 66, 2, 25, 4, 76, 69, 32, 68, 2, 11, - 65, 2, 159, 183, 3, 78, 2, 129, 215, 2, 3, 69, 77, 73, 100, 206, 1, 65, - 38, 68, 46, 76, 38, 82, 34, 84, 46, 86, 186, 5, 85, 206, 201, 1, 73, 162, - 193, 1, 78, 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 206, 40, - 79, 162, 8, 69, 158, 20, 72, 2, 77, 3, 89, 9, 158, 218, 3, 65, 2, 73, 3, - 85, 8, 186, 146, 3, 68, 138, 69, 72, 187, 2, 65, 6, 154, 208, 1, 76, 183, - 137, 2, 65, 4, 242, 214, 3, 82, 187, 2, 65, 8, 202, 145, 3, 84, 138, 69, - 72, 187, 2, 65, 10, 214, 5, 79, 139, 211, 3, 65, 2, 11, 69, 2, 243, 221, - 1, 80, 18, 174, 1, 65, 72, 6, 76, 79, 79, 80, 69, 68, 40, 2, 79, 77, 0, - 5, 83, 72, 82, 73, 73, 48, 13, 67, 65, 78, 68, 82, 65, 32, 65, 78, 85, - 78, 65, 83, 22, 80, 179, 146, 3, 86, 4, 26, 86, 235, 145, 3, 78, 2, 21, - 3, 65, 71, 82, 2, 251, 219, 1, 65, 2, 17, 2, 32, 86, 2, 187, 157, 1, 73, - 2, 17, 2, 32, 80, 2, 25, 4, 85, 83, 72, 80, 2, 231, 204, 3, 73, 2, 237, - 176, 3, 2, 76, 85, 30, 56, 10, 69, 68, 73, 67, 32, 84, 79, 78, 69, 32, - 35, 79, 4, 226, 164, 1, 65, 235, 4, 83, 26, 45, 9, 87, 69, 76, 32, 83, - 73, 71, 78, 32, 26, 70, 65, 38, 85, 22, 86, 186, 201, 1, 73, 234, 234, 1, - 79, 163, 8, 69, 6, 202, 211, 3, 65, 2, 73, 3, 85, 5, 167, 211, 3, 85, 8, - 11, 79, 8, 33, 6, 67, 65, 76, 73, 67, 32, 8, 26, 82, 159, 214, 2, 76, 5, - 199, 210, 3, 82, 50, 50, 75, 84, 4, 78, 69, 68, 32, 135, 173, 2, 84, 4, - 48, 6, 73, 83, 72, 32, 76, 73, 135, 192, 3, 69, 2, 11, 82, 2, 191, 202, - 2, 65, 44, 238, 1, 65, 80, 6, 66, 76, 65, 67, 75, 32, 40, 7, 87, 72, 73, - 84, 69, 32, 83, 22, 67, 50, 68, 96, 2, 78, 79, 32, 2, 79, 75, 30, 83, - 153, 168, 3, 21, 71, 82, 69, 69, 75, 32, 83, 77, 65, 76, 76, 32, 76, 69, - 84, 84, 69, 82, 32, 73, 79, 4, 26, 77, 239, 170, 2, 78, 2, 17, 2, 80, 69, - 2, 11, 82, 2, 251, 192, 2, 83, 4, 18, 80, 23, 83, 2, 147, 150, 2, 69, 2, - 131, 152, 2, 72, 4, 144, 3, 5, 65, 80, 73, 84, 65, 167, 121, 79, 6, 30, - 65, 33, 3, 73, 71, 73, 2, 11, 71, 2, 231, 143, 3, 71, 4, 221, 221, 1, 3, - 84, 32, 84, 6, 162, 2, 82, 255, 195, 2, 84, 2, 141, 228, 1, 2, 32, 72, - 14, 128, 1, 18, 65, 78, 83, 45, 83, 69, 82, 73, 70, 32, 67, 65, 80, 73, - 84, 65, 76, 32, 38, 69, 32, 3, 77, 65, 76, 33, 2, 79, 85, 6, 222, 203, 3, - 71, 2, 76, 3, 89, 2, 11, 77, 2, 159, 251, 2, 73, 2, 11, 76, 2, 235, 201, - 2, 32, 4, 21, 3, 84, 72, 32, 4, 32, 2, 69, 65, 1, 2, 87, 69, 2, 53, 11, - 83, 84, 32, 80, 79, 73, 78, 84, 73, 78, 71, 2, 17, 2, 32, 76, 2, 195, - 200, 2, 69, 38, 58, 69, 52, 8, 73, 83, 84, 69, 68, 32, 82, 73, 63, 79, 2, - 17, 2, 76, 86, 2, 153, 178, 1, 3, 69, 32, 80, 2, 17, 2, 71, 72, 2, 137, - 109, 6, 84, 87, 65, 82, 68, 83, 34, 30, 32, 229, 10, 2, 45, 69, 32, 134, - 3, 66, 86, 67, 116, 3, 68, 79, 84, 226, 1, 72, 44, 14, 73, 78, 84, 69, - 82, 83, 69, 67, 84, 73, 78, 71, 32, 76, 92, 6, 74, 79, 73, 78, 69, 68, - 64, 8, 76, 79, 71, 73, 67, 65, 76, 32, 98, 77, 0, 3, 87, 79, 77, 116, 13, - 82, 73, 78, 71, 83, 32, 65, 76, 73, 71, 78, 69, 68, 48, 24, 65, 83, 84, - 69, 82, 73, 83, 75, 83, 32, 65, 76, 73, 71, 78, 69, 68, 32, 86, 69, 82, - 84, 73, 67, 35, 83, 2, 29, 5, 85, 84, 84, 79, 78, 2, 17, 2, 32, 77, 2, - 11, 79, 2, 191, 187, 2, 85, 2, 93, 21, 79, 78, 83, 69, 67, 85, 84, 73, - 86, 69, 32, 69, 81, 85, 65, 76, 83, 32, 83, 73, 71, 2, 251, 133, 3, 78, - 6, 18, 32, 55, 83, 4, 22, 76, 131, 1, 80, 2, 145, 163, 1, 2, 69, 65, 2, - 53, 11, 32, 79, 86, 69, 82, 32, 79, 78, 69, 32, 68, 2, 11, 79, 2, 11, 84, - 2, 17, 2, 32, 80, 2, 29, 5, 85, 78, 67, 84, 85, 2, 167, 241, 2, 65, 2, - 11, 69, 2, 11, 65, 2, 159, 173, 2, 82, 4, 17, 2, 79, 71, 4, 25, 4, 73, - 67, 65, 76, 4, 11, 32, 4, 130, 179, 2, 65, 159, 85, 79, 2, 17, 2, 32, 83, - 2, 21, 3, 81, 85, 65, 2, 183, 172, 2, 82, 4, 30, 79, 13, 3, 65, 78, 68, - 2, 11, 82, 2, 11, 32, 2, 11, 79, 2, 11, 80, 2, 143, 18, 69, 2, 11, 69, 2, - 45, 9, 78, 32, 72, 79, 76, 68, 73, 78, 71, 2, 11, 32, 2, 11, 72, 2, 11, - 65, 2, 143, 223, 1, 78, 2, 45, 9, 32, 72, 79, 82, 73, 90, 79, 78, 84, 2, - 11, 65, 2, 219, 154, 3, 76, 2, 49, 10, 80, 69, 69, 67, 72, 32, 66, 85, - 66, 66, 2, 203, 169, 2, 76, 2, 11, 77, 2, 245, 212, 1, 2, 32, 68, 248, 3, - 140, 1, 8, 71, 65, 82, 73, 84, 73, 67, 32, 152, 6, 7, 77, 66, 82, 69, 76, - 76, 65, 166, 1, 78, 158, 8, 80, 138, 166, 1, 82, 151, 134, 2, 83, 62, 48, - 7, 76, 69, 84, 84, 69, 82, 32, 159, 5, 87, 60, 206, 1, 65, 28, 2, 81, 79, - 22, 66, 22, 68, 50, 71, 44, 2, 72, 79, 22, 75, 34, 76, 32, 2, 82, 65, 22, - 83, 74, 84, 74, 89, 22, 90, 150, 182, 2, 78, 202, 91, 80, 246, 5, 87, - 202, 12, 77, 174, 18, 73, 3, 85, 4, 26, 76, 247, 233, 2, 73, 2, 155, 183, - 3, 80, 2, 243, 148, 3, 69, 4, 26, 69, 211, 175, 2, 72, 2, 195, 148, 3, - 76, 4, 166, 140, 2, 72, 189, 138, 1, 2, 65, 77, 5, 151, 182, 3, 84, 4, - 150, 183, 2, 65, 239, 126, 72, 2, 11, 65, 2, 139, 147, 3, 77, 2, 187, - 189, 1, 83, 8, 38, 65, 238, 183, 2, 72, 239, 90, 83, 4, 234, 160, 3, 68, - 239, 13, 77, 6, 38, 72, 206, 154, 3, 69, 175, 28, 79, 2, 11, 65, 2, 231, - 250, 1, 78, 2, 183, 240, 2, 79, 4, 246, 145, 3, 69, 207, 36, 85, 2, 21, - 3, 79, 82, 68, 2, 25, 4, 32, 68, 73, 86, 2, 235, 149, 1, 73, 7, 11, 32, - 4, 84, 4, 79, 78, 32, 71, 45, 13, 87, 73, 84, 72, 32, 82, 65, 73, 78, 32, - 68, 82, 79, 2, 11, 82, 2, 11, 79, 2, 231, 166, 2, 85, 2, 239, 246, 2, 80, - 32, 160, 1, 3, 65, 77, 85, 20, 7, 67, 69, 82, 84, 65, 73, 78, 34, 68, 62, - 73, 245, 5, 18, 77, 65, 82, 82, 73, 69, 68, 32, 80, 65, 82, 84, 78, 69, - 82, 83, 72, 73, 2, 187, 162, 2, 83, 2, 11, 84, 2, 243, 171, 2, 89, 4, 26, - 69, 159, 159, 2, 79, 2, 11, 82, 2, 135, 151, 3, 84, 22, 60, 2, 79, 78, - 234, 3, 84, 106, 86, 173, 58, 3, 67, 79, 82, 15, 11, 32, 12, 160, 1, 6, - 65, 66, 79, 86, 69, 32, 108, 22, 66, 69, 83, 73, 68, 69, 32, 65, 78, 68, - 32, 74, 79, 73, 78, 69, 68, 32, 87, 73, 84, 72, 41, 5, 87, 73, 84, 72, - 32, 4, 52, 9, 66, 65, 82, 32, 65, 66, 79, 86, 69, 23, 73, 2, 17, 2, 32, - 73, 2, 161, 49, 4, 78, 84, 69, 82, 2, 17, 2, 32, 85, 2, 239, 222, 2, 78, - 6, 66, 77, 58, 79, 189, 245, 2, 8, 76, 79, 71, 73, 67, 65, 76, 32, 2, 11, - 73, 2, 11, 78, 2, 11, 85, 2, 183, 167, 2, 83, 2, 11, 86, 2, 209, 140, 2, - 2, 69, 82, 4, 18, 32, 67, 69, 2, 11, 83, 2, 17, 2, 69, 80, 2, 11, 65, 2, - 187, 244, 2, 82, 2, 255, 153, 2, 68, 2, 57, 12, 69, 82, 83, 65, 76, 32, - 82, 69, 67, 89, 67, 76, 2, 17, 2, 73, 78, 2, 155, 153, 2, 71, 2, 135, - 153, 2, 80, 144, 3, 130, 1, 32, 246, 7, 45, 180, 8, 4, 80, 69, 82, 32, - 132, 40, 8, 83, 73, 68, 69, 45, 68, 79, 87, 21, 6, 87, 65, 82, 68, 83, - 32, 38, 140, 1, 5, 65, 82, 82, 79, 87, 224, 1, 5, 66, 65, 82, 66, 32, - 228, 1, 5, 68, 79, 87, 78, 32, 210, 1, 70, 30, 82, 93, 4, 84, 65, 67, 75, - 8, 64, 4, 72, 69, 65, 68, 137, 51, 7, 32, 84, 72, 82, 79, 85, 71, 7, 11, - 32, 4, 112, 22, 66, 69, 84, 87, 69, 69, 78, 32, 84, 87, 79, 32, 72, 79, - 82, 73, 90, 79, 78, 84, 65, 76, 171, 155, 2, 73, 2, 193, 147, 2, 2, 32, - 66, 8, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 4, 57, 12, 84, 32, 68, - 79, 87, 78, 32, 66, 65, 82, 66, 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, - 71, 72, 2, 11, 84, 2, 25, 4, 32, 72, 65, 82, 2, 11, 80, 2, 231, 214, 2, - 79, 14, 76, 5, 65, 82, 82, 79, 87, 62, 66, 38, 68, 18, 83, 246, 58, 87, - 219, 9, 84, 5, 37, 7, 32, 87, 73, 84, 72, 32, 66, 2, 187, 156, 2, 65, 2, - 221, 138, 2, 4, 76, 65, 67, 75, 2, 191, 59, 79, 2, 179, 65, 65, 2, 11, - 73, 2, 223, 112, 83, 2, 53, 11, 73, 71, 72, 84, 32, 68, 73, 65, 71, 79, - 78, 2, 189, 128, 1, 4, 65, 76, 32, 69, 5, 29, 5, 32, 87, 73, 84, 72, 2, - 33, 6, 32, 67, 73, 82, 67, 76, 2, 175, 166, 2, 69, 32, 58, 70, 225, 1, 9, - 80, 79, 73, 78, 84, 73, 78, 71, 32, 4, 41, 8, 65, 67, 73, 78, 71, 32, 83, - 78, 4, 65, 14, 65, 75, 69, 32, 72, 69, 65, 68, 32, 87, 73, 84, 72, 32, 4, - 42, 79, 25, 6, 67, 76, 79, 83, 69, 68, 2, 21, 3, 80, 69, 78, 2, 21, 3, - 32, 77, 79, 2, 135, 159, 1, 85, 28, 130, 1, 65, 86, 69, 62, 70, 20, 8, - 77, 73, 76, 73, 84, 65, 82, 89, 26, 82, 88, 6, 83, 77, 65, 76, 76, 32, - 118, 84, 255, 121, 71, 4, 22, 84, 159, 2, 73, 2, 37, 7, 79, 77, 73, 67, - 32, 66, 79, 2, 219, 227, 1, 77, 2, 29, 5, 78, 69, 82, 71, 89, 2, 213, - 179, 1, 2, 32, 87, 2, 143, 158, 3, 82, 2, 129, 1, 2, 32, 65, 8, 64, 5, - 79, 67, 75, 69, 84, 118, 69, 222, 129, 1, 65, 247, 28, 73, 2, 239, 200, - 2, 32, 4, 18, 65, 67, 82, 2, 11, 73, 2, 11, 82, 2, 17, 2, 80, 76, 2, 255, - 131, 2, 65, 2, 11, 69, 2, 151, 248, 1, 68, 4, 37, 7, 82, 73, 65, 78, 71, - 76, 69, 4, 11, 32, 4, 11, 87, 4, 25, 4, 73, 84, 72, 32, 4, 18, 76, 27, - 82, 2, 11, 69, 2, 35, 70, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, - 72, 2, 21, 3, 65, 76, 70, 2, 17, 2, 32, 66, 2, 11, 76, 2, 143, 235, 1, - 65, 200, 1, 192, 1, 4, 65, 78, 68, 32, 210, 2, 66, 74, 67, 74, 70, 36, 5, - 72, 65, 76, 70, 32, 200, 4, 5, 76, 69, 70, 84, 32, 254, 5, 77, 150, 2, - 79, 60, 6, 82, 73, 71, 72, 84, 32, 234, 17, 83, 67, 84, 8, 34, 76, 53, 4, - 82, 73, 71, 72, 6, 48, 2, 69, 70, 173, 1, 5, 79, 87, 69, 82, 32, 2, 53, - 11, 84, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 2, 201, 35, 25, 32, 84, - 82, 73, 65, 78, 71, 85, 76, 65, 82, 32, 84, 72, 82, 69, 69, 32, 81, 85, - 65, 82, 84, 69, 82, 4, 28, 2, 84, 82, 231, 39, 79, 2, 225, 4, 7, 73, 65, - 78, 71, 85, 76, 65, 2, 17, 2, 76, 65, 2, 17, 2, 68, 69, 2, 185, 223, 1, - 3, 32, 83, 67, 10, 33, 6, 69, 78, 84, 82, 69, 32, 10, 174, 12, 76, 22, - 82, 251, 21, 79, 2, 17, 2, 73, 86, 2, 179, 31, 69, 20, 140, 1, 5, 66, 76, - 79, 67, 75, 110, 72, 32, 8, 73, 78, 86, 69, 82, 83, 69, 32, 198, 1, 86, - 170, 26, 77, 130, 3, 76, 22, 82, 139, 162, 1, 67, 5, 225, 28, 23, 32, 65, - 78, 68, 32, 76, 79, 87, 69, 82, 32, 72, 65, 76, 70, 32, 73, 78, 86, 69, - 82, 83, 69, 2, 225, 18, 4, 69, 65, 86, 89, 4, 100, 21, 77, 69, 68, 73, - 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, 68, 32, 76, 79, 87, 69, 51, - 87, 2, 11, 82, 2, 185, 35, 5, 32, 72, 65, 76, 70, 2, 21, 3, 72, 73, 84, - 2, 191, 191, 1, 69, 2, 29, 5, 69, 82, 84, 73, 67, 2, 197, 115, 14, 65, - 76, 32, 76, 73, 78, 69, 32, 87, 73, 84, 72, 32, 84, 62, 50, 66, 238, 3, - 67, 54, 79, 74, 84, 179, 13, 81, 22, 65, 14, 76, 79, 67, 75, 32, 68, 73, - 65, 71, 79, 78, 65, 76, 32, 22, 144, 1, 6, 76, 79, 87, 69, 82, 32, 233, - 8, 24, 85, 80, 80, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, - 84, 32, 84, 79, 32, 85, 80, 80, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, - 32, 84, 79, 32, 36, 8, 76, 69, 70, 84, 32, 84, 79, 32, 229, 10, 18, 77, - 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 85, 80, 80, 6, - 70, 76, 253, 8, 3, 85, 80, 80, 6, 34, 76, 213, 9, 3, 85, 80, 80, 2, 149, - 10, 2, 79, 87, 2, 29, 5, 79, 82, 78, 69, 82, 2, 155, 232, 1, 32, 4, 182, - 12, 78, 53, 12, 82, 32, 76, 79, 87, 69, 82, 32, 82, 73, 71, 72, 8, 34, - 79, 166, 19, 82, 155, 1, 87, 2, 237, 18, 11, 32, 76, 79, 87, 69, 82, 32, - 82, 73, 71, 72, 12, 33, 6, 73, 68, 68, 76, 69, 32, 12, 52, 7, 67, 69, 78, - 84, 82, 69, 32, 74, 76, 23, 82, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, - 72, 2, 117, 3, 84, 32, 79, 4, 41, 2, 69, 70, 4, 21, 3, 73, 71, 72, 4, 17, - 2, 84, 32, 4, 30, 79, 253, 17, 2, 84, 87, 2, 139, 9, 78, 4, 11, 78, 4, - 17, 2, 69, 32, 4, 150, 21, 81, 147, 4, 69, 68, 84, 2, 66, 76, 186, 6, 68, - 114, 79, 222, 1, 80, 38, 81, 224, 4, 2, 83, 72, 51, 84, 22, 61, 13, 79, - 67, 75, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 22, 140, 1, 24, 76, 79, - 87, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, - 32, 76, 79, 87, 57, 6, 85, 80, 80, 69, 82, 32, 4, 21, 3, 69, 82, 32, 4, - 246, 3, 67, 135, 215, 2, 82, 18, 176, 1, 10, 67, 69, 78, 84, 82, 69, 32, - 84, 79, 32, 92, 8, 76, 69, 70, 84, 32, 84, 79, 32, 141, 1, 18, 77, 73, - 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, 76, 79, 87, 6, 32, 3, - 76, 79, 87, 139, 1, 85, 4, 21, 3, 69, 82, 32, 4, 142, 2, 77, 223, 214, 2, - 82, 6, 28, 3, 76, 79, 87, 51, 85, 4, 21, 3, 69, 82, 32, 4, 142, 1, 67, - 43, 77, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 117, 2, 32, 77, 6, 21, 3, - 69, 82, 32, 6, 34, 67, 42, 77, 223, 214, 2, 82, 2, 11, 69, 2, 141, 225, - 1, 2, 78, 84, 2, 25, 4, 73, 68, 68, 76, 2, 139, 176, 1, 69, 2, 57, 12, - 82, 79, 80, 45, 83, 72, 65, 68, 79, 87, 69, 68, 2, 17, 2, 32, 87, 2, 213, - 190, 1, 3, 72, 73, 84, 4, 62, 78, 53, 11, 82, 32, 76, 79, 87, 69, 82, 32, - 76, 69, 70, 2, 225, 16, 9, 69, 32, 83, 73, 88, 84, 69, 69, 78, 2, 69, 15, - 84, 32, 67, 85, 82, 76, 89, 32, 66, 82, 65, 67, 75, 69, 84, 2, 11, 32, 2, - 219, 173, 2, 83, 2, 11, 69, 2, 205, 74, 2, 78, 67, 26, 17, 2, 85, 65, 26, - 44, 6, 68, 82, 65, 78, 84, 32, 255, 3, 82, 24, 90, 67, 112, 10, 70, 65, - 67, 69, 32, 87, 73, 84, 72, 32, 114, 77, 76, 2, 83, 84, 67, 84, 14, 68, - 7, 73, 82, 67, 85, 76, 65, 82, 145, 167, 1, 4, 72, 69, 83, 83, 2, 17, 2, - 32, 65, 2, 147, 212, 1, 82, 4, 34, 67, 45, 4, 79, 80, 69, 78, 2, 21, 3, - 76, 79, 83, 2, 17, 2, 69, 68, 2, 177, 231, 1, 3, 32, 69, 89, 2, 25, 4, - 73, 67, 82, 79, 2, 11, 67, 2, 233, 194, 1, 4, 79, 77, 80, 85, 2, 41, 8, - 65, 78, 68, 73, 78, 71, 32, 75, 2, 195, 207, 2, 78, 2, 21, 3, 69, 76, 69, - 2, 11, 86, 2, 187, 90, 73, 2, 181, 167, 1, 3, 84, 69, 82, 4, 153, 102, 8, - 65, 68, 79, 87, 69, 68, 32, 87, 8, 30, 79, 106, 82, 155, 1, 87, 2, 49, - 10, 32, 76, 79, 87, 69, 82, 32, 76, 69, 70, 2, 11, 84, 2, 11, 32, 2, 11, - 70, 2, 195, 112, 73, 4, 25, 4, 73, 65, 78, 71, 4, 40, 4, 85, 76, 65, 82, - 219, 224, 2, 76, 2, 17, 2, 32, 77, 2, 41, 8, 69, 68, 73, 85, 77, 32, 83, - 72, 2, 223, 102, 65, 2, 21, 3, 69, 76, 70, 2, 11, 84, 2, 163, 164, 1, 72, - 2, 17, 2, 69, 86, 2, 17, 2, 69, 78, 2, 145, 1, 2, 32, 69, 10, 64, 5, 72, - 82, 69, 69, 32, 193, 1, 6, 82, 73, 65, 78, 71, 85, 8, 54, 69, 49, 9, 81, - 85, 65, 82, 84, 69, 82, 83, 32, 2, 29, 5, 73, 71, 72, 84, 72, 2, 243, 5, - 83, 6, 30, 76, 22, 82, 203, 5, 66, 2, 41, 2, 69, 70, 2, 21, 3, 73, 71, - 72, 2, 43, 84, 2, 17, 2, 76, 65, 2, 11, 82, 2, 17, 2, 32, 79, 2, 25, 4, - 78, 69, 32, 81, 2, 185, 4, 6, 85, 65, 82, 84, 69, 82, 2, 199, 181, 2, 78, - 128, 1, 154, 1, 65, 184, 6, 2, 66, 76, 150, 1, 68, 50, 70, 82, 72, 150, - 4, 67, 46, 81, 42, 82, 22, 83, 102, 84, 146, 7, 80, 173, 3, 6, 87, 72, - 73, 84, 69, 32, 32, 34, 78, 33, 4, 82, 82, 79, 87, 2, 11, 67, 2, 195, - 172, 2, 79, 31, 11, 32, 28, 134, 1, 65, 192, 1, 14, 76, 69, 70, 84, 87, - 65, 82, 68, 83, 32, 79, 70, 32, 68, 32, 5, 87, 73, 84, 72, 32, 210, 8, - 70, 179, 5, 84, 2, 41, 8, 78, 68, 32, 82, 73, 71, 72, 84, 2, 17, 2, 32, - 79, 2, 25, 4, 78, 69, 32, 69, 2, 21, 3, 73, 71, 72, 2, 17, 2, 84, 72, 2, - 11, 32, 2, 11, 66, 2, 11, 76, 2, 191, 190, 1, 79, 2, 197, 206, 1, 3, 79, - 87, 78, 20, 74, 68, 58, 76, 42, 77, 38, 78, 58, 83, 66, 69, 250, 12, 84, - 139, 58, 72, 2, 21, 3, 79, 85, 66, 2, 11, 76, 2, 247, 188, 2, 69, 2, 11, - 65, 2, 253, 22, 3, 82, 71, 69, 2, 225, 22, 5, 69, 68, 73, 85, 77, 2, 25, - 4, 79, 84, 67, 72, 2, 11, 69, 2, 139, 56, 68, 4, 11, 77, 4, 25, 4, 65, - 76, 76, 32, 4, 22, 69, 223, 21, 84, 2, 129, 22, 10, 81, 85, 73, 76, 65, - 84, 69, 82, 65, 76, 4, 25, 4, 65, 67, 75, 32, 4, 26, 67, 139, 208, 1, 65, - 2, 25, 4, 73, 82, 67, 76, 2, 25, 4, 69, 68, 32, 87, 2, 11, 72, 2, 237, - 14, 2, 73, 84, 4, 22, 79, 195, 13, 65, 2, 177, 14, 2, 85, 66, 2, 11, 73, - 2, 37, 7, 78, 71, 69, 82, 45, 80, 79, 2, 173, 206, 1, 2, 83, 84, 20, 88, - 17, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, 66, 65, 82, 66, 32, - 131, 3, 69, 16, 56, 4, 76, 69, 70, 84, 245, 1, 5, 82, 73, 71, 72, 84, 10, - 22, 32, 239, 9, 87, 8, 60, 7, 66, 69, 83, 73, 68, 69, 32, 206, 1, 70, - 179, 5, 84, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, 197, 149, 1, 23, - 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, - 32, 66, 65, 82, 66, 6, 22, 32, 251, 7, 87, 4, 22, 70, 179, 5, 84, 2, 217, - 195, 1, 3, 82, 79, 77, 4, 25, 4, 65, 86, 89, 32, 4, 26, 67, 155, 202, 1, - 65, 2, 189, 10, 7, 79, 77, 80, 82, 69, 83, 83, 2, 137, 9, 6, 85, 65, 68, - 82, 85, 80, 2, 171, 145, 2, 79, 4, 30, 65, 53, 3, 81, 85, 65, 2, 245, - 200, 1, 8, 78, 83, 45, 83, 69, 82, 73, 70, 2, 131, 9, 82, 36, 36, 2, 82, - 73, 229, 7, 2, 87, 79, 30, 40, 5, 65, 78, 71, 76, 69, 155, 7, 80, 28, 52, - 8, 45, 72, 69, 65, 68, 69, 68, 32, 219, 12, 32, 26, 48, 5, 65, 82, 82, - 79, 87, 174, 5, 68, 39, 80, 23, 11, 32, 20, 92, 17, 76, 69, 70, 84, 87, - 65, 82, 68, 83, 32, 79, 70, 32, 68, 79, 87, 78, 98, 84, 23, 87, 2, 37, 7, - 87, 65, 82, 68, 83, 32, 84, 2, 37, 7, 82, 73, 65, 78, 71, 76, 69, 2, 219, - 5, 45, 2, 171, 190, 1, 79, 16, 25, 4, 73, 84, 72, 32, 16, 114, 66, 28, 6, - 76, 79, 78, 71, 32, 84, 130, 1, 77, 34, 78, 34, 86, 34, 72, 149, 56, 6, - 68, 79, 85, 66, 76, 69, 2, 149, 2, 3, 79, 76, 68, 4, 17, 2, 73, 80, 4, - 11, 32, 4, 34, 76, 21, 4, 82, 73, 71, 72, 2, 17, 2, 69, 70, 2, 11, 84, 2, - 11, 87, 2, 243, 125, 65, 2, 121, 5, 69, 68, 73, 85, 77, 2, 89, 5, 65, 82, - 82, 79, 87, 2, 29, 5, 69, 82, 89, 32, 72, 2, 25, 4, 69, 65, 86, 89, 2, - 221, 177, 2, 4, 32, 83, 72, 65, 2, 11, 65, 2, 249, 1, 2, 83, 72, 2, 11, - 65, 2, 25, 4, 73, 82, 69, 68, 2, 29, 5, 32, 65, 82, 82, 79, 2, 211, 157, - 2, 87, 2, 11, 76, 2, 187, 192, 1, 69, 6, 74, 32, 61, 14, 45, 72, 69, 65, - 68, 69, 68, 32, 65, 82, 82, 79, 87, 32, 2, 25, 4, 72, 69, 65, 68, 2, 11, - 69, 2, 179, 191, 1, 68, 4, 42, 87, 253, 134, 1, 4, 70, 82, 79, 77, 2, 33, - 6, 73, 84, 72, 32, 84, 82, 2, 61, 13, 73, 65, 78, 71, 76, 69, 32, 65, 82, - 82, 79, 87, 72, 2, 149, 121, 2, 69, 65, 18, 88, 5, 65, 82, 82, 79, 87, - 149, 3, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, 87, 15, 11, 32, - 12, 108, 11, 79, 78, 32, 80, 69, 68, 69, 83, 84, 65, 76, 106, 87, 217, - 179, 1, 8, 70, 82, 79, 77, 32, 66, 65, 82, 7, 33, 6, 32, 87, 73, 84, 72, - 32, 4, 26, 86, 171, 180, 1, 72, 2, 205, 180, 1, 5, 69, 82, 84, 73, 67, 2, - 29, 5, 73, 84, 72, 73, 78, 2, 17, 2, 32, 84, 2, 37, 7, 82, 73, 65, 78, - 71, 76, 69, 2, 11, 32, 2, 11, 65, 2, 25, 4, 82, 82, 79, 87, 2, 11, 72, 2, - 195, 142, 2, 69, 5, 45, 9, 32, 79, 78, 32, 80, 69, 68, 69, 83, 2, 235, - 202, 1, 84, 240, 15, 86, 65, 254, 16, 83, 174, 3, 69, 234, 45, 73, 186, - 8, 79, 134, 2, 84, 21, 2, 85, 76, 218, 8, 116, 2, 73, 32, 128, 16, 17, - 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 84, 79, 82, 45, 245, - 45, 2, 77, 80, 216, 4, 54, 67, 34, 70, 82, 81, 40, 2, 83, 89, 207, 95, - 68, 2, 11, 79, 2, 231, 176, 2, 77, 2, 11, 85, 2, 11, 76, 2, 11, 76, 2, - 11, 32, 2, 11, 83, 2, 147, 216, 1, 84, 2, 17, 2, 85, 69, 2, 215, 231, 1, - 83, 190, 4, 68, 7, 76, 76, 65, 66, 76, 69, 32, 197, 11, 5, 77, 66, 79, - 76, 32, 164, 4, 214, 1, 68, 70, 66, 2, 83, 2, 84, 2, 90, 70, 71, 122, 72, - 82, 75, 134, 1, 76, 130, 1, 77, 86, 78, 134, 3, 67, 2, 70, 2, 74, 2, 80, - 2, 82, 2, 86, 2, 89, 78, 87, 50, 69, 34, 79, 250, 73, 65, 2, 73, 3, 85, - 42, 66, 72, 162, 8, 79, 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 28, - 230, 7, 72, 58, 79, 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 34, 62, - 66, 202, 6, 69, 86, 79, 226, 197, 2, 65, 2, 73, 3, 85, 18, 106, 79, 222, - 5, 69, 182, 198, 2, 65, 2, 73, 3, 85, 24, 50, 79, 222, 5, 69, 178, 75, - 65, 2, 73, 3, 85, 7, 142, 204, 2, 78, 3, 79, 34, 70, 80, 206, 5, 79, 222, - 74, 65, 230, 186, 1, 69, 162, 64, 73, 3, 85, 18, 246, 4, 69, 86, 79, 222, - 74, 65, 134, 251, 1, 73, 3, 85, 16, 54, 69, 218, 4, 79, 226, 197, 2, 65, - 2, 73, 3, 85, 7, 26, 78, 159, 202, 2, 69, 2, 21, 3, 71, 84, 72, 2, 139, - 140, 2, 69, 42, 214, 3, 66, 0, 2, 71, 66, 58, 79, 194, 133, 2, 69, 162, - 64, 65, 2, 73, 3, 85, 90, 90, 68, 174, 1, 71, 126, 74, 2, 89, 58, 79, - 194, 133, 2, 69, 162, 64, 65, 2, 73, 3, 85, 24, 54, 79, 234, 135, 2, 69, - 162, 64, 65, 2, 73, 3, 85, 15, 36, 3, 76, 69, 32, 227, 199, 2, 79, 10, - 54, 83, 182, 168, 2, 68, 190, 28, 70, 2, 75, 3, 77, 2, 179, 168, 2, 79, - 25, 42, 71, 138, 247, 1, 65, 2, 69, 3, 79, 16, 50, 69, 86, 79, 226, 197, - 2, 65, 2, 73, 3, 85, 7, 178, 198, 2, 69, 3, 78, 14, 54, 79, 194, 133, 2, - 69, 162, 64, 65, 2, 73, 3, 85, 5, 223, 197, 2, 79, 28, 46, 69, 34, 79, - 250, 73, 65, 2, 73, 3, 85, 9, 150, 74, 69, 135, 251, 1, 78, 9, 246, 73, - 79, 135, 251, 1, 78, 26, 66, 68, 62, 70, 30, 74, 22, 75, 50, 78, 22, 84, - 151, 234, 1, 66, 6, 26, 79, 135, 243, 1, 65, 4, 130, 243, 1, 79, 135, 50, - 45, 4, 74, 69, 219, 192, 2, 65, 2, 199, 242, 1, 79, 4, 26, 69, 155, 242, - 1, 85, 2, 151, 242, 1, 69, 2, 163, 175, 2, 73, 6, 238, 241, 1, 73, 2, 79, - 195, 78, 65, 128, 4, 74, 49, 94, 50, 98, 51, 2, 52, 2, 53, 2, 54, 2, 55, - 2, 56, 3, 57, 223, 1, 182, 1, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, - 54, 2, 55, 2, 56, 3, 57, 137, 1, 90, 48, 2, 49, 2, 50, 2, 51, 2, 52, 94, - 53, 218, 191, 2, 54, 2, 55, 2, 56, 3, 57, 23, 178, 192, 2, 48, 2, 49, 2, - 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 17, 214, 191, 2, 48, - 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 194, 1, 116, 4, 68, 73, 67, 32, - 206, 19, 82, 168, 155, 1, 13, 67, 84, 79, 82, 32, 79, 82, 32, 67, 82, 79, - 83, 83, 219, 106, 83, 86, 60, 5, 83, 73, 71, 78, 32, 153, 10, 5, 84, 79, - 78, 69, 32, 48, 218, 2, 65, 216, 1, 17, 68, 79, 85, 66, 76, 69, 32, 65, - 78, 85, 83, 86, 65, 82, 65, 32, 65, 98, 74, 52, 6, 78, 73, 72, 83, 72, - 86, 22, 82, 240, 1, 8, 72, 69, 88, 73, 70, 79, 82, 77, 22, 76, 52, 4, 84, - 73, 82, 89, 22, 85, 60, 8, 86, 73, 83, 65, 82, 71, 65, 32, 237, 6, 17, - 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, 77, 73, 68, 76, 73, 78, 14, - 76, 8, 78, 85, 83, 86, 65, 82, 65, 32, 212, 1, 3, 84, 73, 75, 151, 2, 82, - 10, 134, 1, 65, 24, 4, 66, 65, 72, 73, 24, 9, 85, 66, 72, 65, 89, 65, 84, - 79, 32, 201, 4, 10, 86, 65, 77, 65, 71, 79, 77, 85, 75, 72, 2, 21, 3, 78, - 84, 65, 2, 21, 3, 82, 71, 79, 2, 11, 77, 2, 163, 9, 85, 2, 235, 245, 1, - 82, 2, 33, 6, 73, 72, 86, 65, 77, 85, 2, 151, 3, 76, 2, 203, 181, 2, 65, - 8, 144, 1, 15, 69, 86, 69, 82, 83, 69, 68, 32, 86, 73, 83, 65, 82, 71, - 65, 36, 9, 79, 84, 65, 84, 69, 68, 32, 65, 82, 57, 5, 84, 72, 65, 78, 71, - 4, 11, 32, 4, 218, 6, 65, 23, 85, 2, 25, 4, 68, 72, 65, 86, 2, 201, 243, - 1, 2, 73, 83, 2, 17, 2, 32, 76, 2, 21, 3, 79, 78, 71, 2, 229, 240, 1, 2, - 32, 65, 2, 171, 178, 2, 65, 2, 37, 7, 80, 65, 68, 72, 77, 65, 78, 2, 171, - 178, 2, 73, 10, 40, 3, 65, 78, 85, 2, 85, 179, 9, 83, 4, 25, 4, 68, 65, - 84, 84, 4, 11, 65, 5, 25, 4, 32, 87, 73, 84, 2, 11, 72, 2, 11, 32, 2, 11, - 84, 2, 11, 65, 2, 159, 170, 1, 73, 38, 128, 2, 5, 67, 65, 78, 68, 82, 16, - 2, 68, 79, 96, 2, 75, 65, 136, 1, 4, 80, 82, 69, 78, 16, 2, 82, 73, 106, - 84, 164, 1, 11, 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, 180, 1, 12, - 65, 84, 72, 65, 82, 86, 65, 86, 69, 68, 73, 67, 169, 231, 1, 2, 83, 72, - 4, 131, 19, 65, 6, 40, 5, 85, 66, 76, 69, 32, 191, 3, 84, 4, 22, 82, 211, - 5, 83, 2, 11, 73, 2, 251, 1, 78, 4, 56, 3, 82, 83, 72, 17, 7, 84, 72, 65, - 75, 65, 32, 65, 2, 139, 116, 65, 2, 17, 2, 78, 85, 2, 17, 2, 68, 65, 2, - 139, 139, 2, 84, 2, 131, 53, 75, 4, 82, 78, 237, 2, 15, 71, 86, 69, 68, - 73, 67, 32, 75, 65, 83, 72, 77, 73, 82, 73, 2, 219, 177, 1, 71, 6, 42, - 72, 24, 4, 82, 73, 80, 76, 19, 87, 2, 49, 3, 82, 69, 69, 2, 219, 2, 69, - 2, 11, 79, 2, 25, 4, 32, 68, 79, 84, 2, 11, 83, 2, 11, 32, 2, 167, 15, - 66, 8, 176, 1, 10, 65, 71, 71, 82, 65, 86, 65, 84, 69, 68, 22, 73, 105, - 27, 75, 65, 84, 72, 65, 75, 65, 32, 73, 78, 68, 69, 80, 69, 78, 68, 69, - 78, 84, 32, 83, 86, 65, 82, 73, 84, 65, 2, 17, 2, 32, 73, 2, 49, 10, 78, - 68, 69, 80, 69, 78, 68, 69, 78, 84, 2, 17, 2, 32, 83, 2, 189, 134, 2, 3, - 86, 65, 82, 5, 241, 10, 7, 32, 83, 67, 72, 82, 79, 69, 104, 62, 83, 16, - 6, 84, 73, 67, 65, 76, 32, 181, 17, 2, 89, 32, 2, 167, 88, 73, 76, 200, - 2, 4, 66, 65, 82, 32, 174, 3, 67, 42, 69, 62, 70, 38, 71, 32, 11, 73, 68, - 69, 79, 71, 82, 65, 80, 72, 73, 67, 48, 12, 75, 65, 78, 65, 32, 82, 69, - 80, 69, 65, 84, 32, 254, 1, 76, 194, 3, 77, 88, 17, 79, 78, 69, 32, 69, - 73, 71, 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, 62, 82, 158, 1, 84, 218, - 1, 90, 173, 72, 3, 83, 73, 88, 8, 108, 6, 66, 69, 83, 73, 68, 69, 64, 8, - 68, 79, 85, 66, 76, 69, 32, 76, 20, 4, 84, 82, 73, 80, 143, 1, 87, 2, 17, - 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 159, 129, 1, 84, 2, 69, 2, 69, 70, 2, - 25, 4, 76, 69, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 29, 5, 32, 84, - 85, 82, 78, 2, 11, 83, 2, 11, 84, 2, 203, 128, 1, 73, 2, 21, 3, 73, 84, - 72, 2, 17, 2, 32, 72, 2, 137, 244, 1, 7, 79, 82, 73, 90, 79, 78, 84, 2, - 141, 235, 1, 5, 65, 80, 65, 67, 73, 2, 25, 4, 76, 76, 73, 80, 2, 11, 83, - 2, 215, 229, 1, 73, 2, 11, 79, 2, 213, 82, 2, 85, 82, 2, 221, 155, 1, 3, - 79, 45, 75, 2, 17, 2, 32, 73, 2, 133, 185, 1, 2, 84, 69, 10, 120, 4, 77, - 65, 82, 75, 45, 22, 87, 73, 84, 72, 32, 86, 79, 73, 67, 69, 68, 32, 83, - 79, 85, 78, 68, 32, 77, 65, 82, 75, 7, 11, 32, 4, 50, 85, 21, 3, 76, 79, - 87, 5, 17, 2, 32, 85, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, 153, 29, 2, - 32, 72, 16, 30, 65, 33, 3, 73, 78, 69, 2, 11, 68, 2, 151, 226, 1, 68, 15, - 11, 32, 12, 38, 69, 49, 5, 87, 73, 84, 72, 32, 2, 25, 4, 88, 84, 69, 78, - 2, 219, 206, 1, 83, 10, 56, 4, 67, 73, 82, 67, 98, 70, 26, 84, 151, 140, - 1, 77, 4, 11, 76, 4, 11, 69, 4, 11, 32, 4, 26, 66, 167, 161, 1, 65, 2, - 11, 69, 2, 223, 131, 1, 76, 2, 57, 3, 79, 85, 82, 2, 11, 72, 2, 21, 3, - 82, 69, 69, 2, 45, 9, 32, 84, 73, 67, 75, 32, 77, 65, 82, 2, 167, 223, 1, - 75, 2, 65, 14, 65, 76, 69, 32, 87, 73, 84, 72, 32, 83, 84, 82, 79, 75, 2, - 171, 149, 1, 69, 12, 150, 156, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 3, 55, - 4, 42, 65, 57, 6, 69, 83, 73, 83, 84, 79, 2, 25, 4, 67, 73, 78, 71, 2, - 11, 32, 2, 207, 121, 67, 2, 21, 3, 82, 32, 83, 2, 11, 69, 2, 159, 215, 1, - 71, 10, 32, 2, 65, 66, 106, 73, 19, 82, 6, 18, 32, 35, 85, 2, 11, 75, 2, - 179, 136, 2, 69, 4, 33, 6, 76, 65, 84, 73, 79, 78, 5, 251, 45, 32, 2, - 167, 9, 76, 2, 29, 5, 65, 70, 70, 73, 67, 2, 177, 238, 1, 2, 32, 76, 2, - 233, 106, 4, 73, 71, 90, 65, 26, 80, 6, 72, 69, 65, 86, 89, 32, 176, 5, - 3, 77, 85, 67, 241, 9, 3, 66, 79, 76, 20, 62, 69, 190, 1, 70, 38, 82, 82, - 83, 246, 1, 87, 215, 10, 71, 4, 25, 4, 73, 71, 72, 84, 4, 11, 32, 4, 22, - 80, 223, 2, 83, 2, 25, 4, 79, 73, 78, 84, 2, 17, 2, 69, 68, 2, 17, 2, 32, - 66, 2, 25, 4, 76, 65, 67, 75, 2, 11, 32, 2, 215, 79, 83, 2, 11, 73, 2, - 185, 1, 2, 86, 69, 2, 11, 69, 2, 29, 5, 86, 69, 82, 83, 69, 2, 17, 2, 32, - 83, 2, 231, 1, 79, 6, 30, 65, 42, 73, 147, 1, 79, 2, 11, 76, 2, 11, 84, - 2, 147, 116, 73, 2, 11, 88, 2, 11, 32, 2, 11, 83, 2, 21, 3, 80, 79, 75, - 2, 17, 2, 69, 68, 2, 11, 32, 2, 11, 65, 2, 11, 83, 2, 205, 78, 3, 84, 69, - 82, 2, 173, 13, 3, 76, 73, 68, 4, 21, 3, 72, 73, 84, 4, 11, 69, 4, 11, - 32, 4, 226, 64, 67, 163, 49, 83, 4, 11, 72, 4, 11, 32, 4, 18, 71, 39, 76, - 2, 69, 6, 82, 69, 65, 84, 69, 82, 2, 11, 69, 2, 11, 83, 2, 11, 83, 2, 17, - 2, 45, 84, 2, 187, 71, 72, 160, 1, 140, 1, 9, 66, 82, 65, 84, 73, 79, 78, - 32, 77, 34, 67, 36, 3, 68, 69, 79, 126, 69, 222, 1, 79, 22, 82, 21, 7, - 84, 72, 75, 85, 81, 73, 32, 2, 11, 79, 2, 183, 249, 1, 68, 2, 221, 129, - 1, 4, 84, 79, 82, 89, 6, 30, 32, 77, 3, 67, 65, 83, 4, 18, 67, 43, 71, 2, - 17, 2, 65, 77, 2, 203, 202, 1, 69, 2, 139, 119, 65, 2, 219, 71, 83, 6, + 67, 2, 80, 2, 84, 34, 78, 234, 222, 5, 66, 2, 68, 2, 70, 2, 71, 2, 72, 2, + 76, 2, 77, 2, 82, 2, 83, 2, 86, 2, 89, 247, 30, 79, 6, 26, 72, 255, 253, + 5, 79, 4, 134, 223, 5, 72, 247, 30, 79, 6, 230, 222, 5, 71, 2, 89, 247, + 30, 79, 4, 186, 172, 5, 65, 195, 52, 72, 10, 72, 2, 75, 79, 110, 78, 216, + 132, 1, 4, 72, 79, 32, 72, 247, 185, 3, 83, 4, 192, 133, 1, 3, 73, 32, + 75, 135, 247, 4, 78, 8, 58, 78, 230, 138, 1, 84, 234, 169, 2, 83, 251, + 195, 2, 69, 2, 251, 180, 3, 85, 26, 62, 65, 206, 4, 85, 206, 228, 1, 73, + 242, 145, 4, 69, 3, 79, 10, 178, 228, 5, 85, 214, 22, 65, 2, 77, 2, 78, + 3, 89, 110, 72, 7, 76, 69, 84, 84, 69, 82, 32, 238, 3, 83, 157, 1, 3, 88, + 65, 77, 96, 206, 1, 65, 80, 2, 76, 79, 12, 4, 72, 73, 71, 72, 126, 78, + 34, 85, 198, 133, 1, 80, 2, 84, 138, 95, 73, 194, 200, 1, 79, 190, 170, + 2, 66, 2, 67, 2, 68, 2, 71, 2, 77, 2, 81, 2, 86, 2, 89, 247, 30, 69, 16, + 222, 225, 5, 85, 214, 22, 65, 2, 69, 2, 75, 2, 77, 2, 78, 2, 80, 3, 84, + 21, 11, 87, 18, 11, 32, 18, 70, 75, 194, 158, 5, 78, 210, 57, 70, 2, 72, + 2, 80, 2, 84, 3, 88, 6, 142, 216, 5, 72, 2, 86, 247, 30, 79, 4, 230, 215, + 5, 71, 247, 30, 79, 9, 202, 228, 1, 69, 243, 145, 4, 65, 12, 76, 4, 73, + 71, 78, 32, 245, 156, 5, 9, 89, 77, 66, 79, 76, 32, 77, 85, 69, 10, 38, + 65, 206, 222, 5, 85, 255, 5, 79, 6, 202, 244, 5, 78, 86, 85, 3, 89, 2, + 171, 129, 1, 32, 138, 1, 52, 3, 82, 73, 32, 201, 229, 4, 4, 69, 79, 85, + 84, 136, 1, 82, 65, 20, 7, 76, 69, 84, 84, 69, 82, 32, 166, 2, 83, 78, + 86, 203, 252, 3, 68, 2, 155, 211, 1, 66, 88, 210, 1, 65, 158, 148, 2, 68, + 82, 82, 34, 84, 230, 5, 85, 186, 202, 1, 73, 138, 196, 1, 78, 126, 66, 2, + 67, 2, 71, 2, 74, 2, 75, 2, 80, 2, 83, 138, 69, 72, 2, 76, 2, 77, 2, 86, + 2, 89, 186, 2, 69, 3, 79, 11, 208, 190, 3, 7, 82, 67, 72, 65, 73, 67, 32, + 154, 179, 2, 65, 2, 73, 3, 85, 8, 25, 4, 73, 71, 78, 32, 8, 254, 211, 1, + 78, 138, 216, 3, 65, 239, 1, 86, 18, 49, 10, 79, 87, 69, 76, 32, 83, 73, + 71, 78, 32, 18, 250, 152, 2, 65, 38, 85, 186, 202, 1, 73, 198, 140, 2, + 69, 3, 79, 4, 134, 51, 70, 163, 161, 4, 79, 188, 6, 36, 3, 73, 76, 32, + 211, 200, 4, 65, 186, 6, 154, 3, 65, 106, 67, 194, 2, 68, 72, 3, 87, 69, + 84, 48, 9, 70, 82, 65, 67, 84, 73, 79, 78, 32, 232, 7, 9, 73, 78, 32, 80, + 79, 83, 83, 69, 83, 22, 76, 200, 2, 7, 78, 85, 77, 66, 69, 82, 32, 152, + 1, 18, 80, 85, 78, 67, 84, 85, 65, 84, 73, 79, 78, 32, 69, 78, 68, 32, + 79, 70, 54, 82, 42, 83, 166, 14, 84, 228, 1, 7, 86, 79, 87, 69, 76, 32, + 83, 100, 2, 89, 69, 168, 195, 4, 5, 77, 79, 78, 84, 72, 171, 118, 79, 6, + 80, 5, 83, 32, 65, 66, 79, 150, 138, 2, 85, 197, 245, 1, 5, 78, 68, 32, + 79, 68, 2, 143, 203, 3, 86, 52, 80, 9, 79, 78, 83, 79, 78, 65, 78, 84, + 32, 156, 21, 3, 85, 82, 82, 171, 9, 82, 48, 130, 1, 75, 22, 76, 22, 78, + 46, 84, 194, 191, 1, 83, 254, 82, 82, 158, 214, 3, 67, 2, 72, 2, 74, 2, + 77, 2, 80, 2, 86, 3, 89, 5, 207, 171, 5, 83, 7, 235, 236, 4, 76, 11, 150, + 235, 3, 78, 238, 253, 1, 71, 3, 89, 5, 215, 232, 5, 84, 26, 68, 2, 82, + 89, 164, 28, 2, 69, 66, 146, 149, 2, 65, 207, 194, 1, 73, 2, 225, 199, 1, + 7, 32, 67, 85, 76, 84, 73, 86, 42, 168, 1, 4, 79, 78, 69, 32, 220, 4, 6, + 84, 72, 82, 69, 69, 32, 169, 224, 5, 22, 68, 79, 87, 78, 83, 67, 65, 76, + 73, 78, 71, 32, 70, 65, 67, 84, 79, 82, 32, 75, 73, 73, 30, 112, 5, 69, + 73, 71, 72, 84, 34, 70, 40, 3, 72, 65, 76, 22, 79, 88, 4, 83, 73, 88, 84, + 74, 84, 203, 221, 3, 81, 4, 210, 3, 73, 219, 225, 5, 72, 4, 156, 3, 2, + 79, 82, 203, 219, 3, 73, 4, 251, 222, 1, 70, 2, 225, 2, 18, 78, 69, 45, + 72, 85, 78, 68, 82, 69, 68, 45, 65, 78, 68, 45, 83, 73, 88, 6, 140, 222, + 1, 5, 69, 69, 78, 84, 72, 141, 25, 5, 89, 45, 70, 79, 85, 8, 38, 72, 138, + 1, 87, 255, 220, 3, 69, 4, 132, 1, 18, 82, 69, 69, 45, 72, 85, 78, 68, + 82, 69, 68, 45, 65, 78, 68, 45, 84, 87, 173, 163, 4, 8, 73, 82, 84, 89, + 45, 83, 69, 67, 2, 17, 2, 69, 78, 2, 17, 2, 84, 73, 2, 223, 220, 3, 69, + 10, 58, 69, 28, 4, 83, 73, 88, 84, 82, 84, 179, 220, 3, 81, 2, 129, 1, 3, + 73, 71, 72, 4, 50, 69, 165, 221, 3, 6, 89, 45, 70, 79, 85, 82, 2, 161, + 221, 3, 2, 69, 78, 2, 21, 3, 87, 69, 78, 2, 237, 220, 3, 3, 84, 73, 69, + 2, 207, 218, 1, 83, 72, 48, 6, 69, 84, 84, 69, 82, 32, 211, 243, 3, 65, + 70, 194, 1, 78, 170, 201, 1, 84, 190, 54, 65, 82, 76, 38, 82, 134, 6, 85, + 206, 141, 1, 79, 238, 60, 73, 182, 196, 1, 83, 242, 7, 69, 234, 61, 67, + 2, 72, 2, 74, 2, 75, 2, 77, 2, 80, 2, 86, 3, 89, 10, 46, 78, 242, 218, 5, + 71, 2, 89, 187, 2, 65, 4, 238, 218, 5, 78, 187, 2, 65, 8, 38, 79, 222, + 134, 4, 84, 159, 79, 83, 4, 11, 78, 4, 17, 2, 69, 32, 4, 18, 72, 31, 84, + 2, 133, 72, 3, 85, 78, 68, 2, 185, 137, 2, 3, 72, 79, 85, 2, 17, 2, 32, + 84, 2, 11, 69, 2, 163, 191, 5, 88, 2, 17, 2, 85, 80, 2, 159, 187, 3, 69, + 194, 4, 152, 1, 5, 65, 76, 84, 32, 80, 20, 4, 73, 71, 78, 32, 206, 4, 80, + 28, 9, 84, 65, 82, 84, 73, 78, 71, 32, 70, 41, 8, 89, 76, 76, 65, 66, 76, + 69, 32, 2, 235, 221, 3, 65, 40, 90, 65, 48, 3, 67, 69, 86, 34, 75, 80, 2, + 77, 85, 102, 85, 14, 80, 118, 86, 211, 64, 78, 4, 216, 2, 4, 65, 90, 72, + 65, 191, 145, 5, 78, 2, 11, 73, 2, 223, 179, 5, 84, 6, 38, 85, 165, 179, + 5, 3, 65, 65, 67, 4, 18, 90, 87, 82, 2, 159, 196, 5, 72, 6, 60, 4, 75, + 75, 85, 82, 16, 2, 84, 72, 21, 3, 85, 86, 85, 2, 203, 82, 85, 2, 175, + 155, 3, 65, 2, 75, 90, 8, 26, 65, 251, 134, 5, 79, 6, 34, 84, 154, 153, + 4, 65, 15, 78, 2, 11, 72, 2, 17, 2, 65, 75, 2, 171, 177, 5, 75, 10, 38, + 65, 250, 81, 69, 227, 192, 4, 73, 4, 216, 102, 3, 82, 65, 65, 201, 249, + 2, 6, 75, 65, 73, 89, 65, 82, 2, 11, 69, 2, 187, 9, 78, 2, 17, 2, 82, 79, + 2, 227, 205, 4, 77, 148, 4, 130, 1, 75, 166, 1, 76, 166, 1, 78, 186, 1, + 82, 86, 83, 178, 1, 84, 214, 2, 67, 2, 72, 2, 74, 2, 77, 2, 80, 2, 86, 3, + 89, 46, 84, 2, 83, 83, 210, 251, 1, 65, 38, 85, 206, 141, 1, 79, 238, 60, + 73, 167, 204, 1, 69, 24, 250, 244, 1, 65, 250, 6, 85, 206, 141, 1, 79, + 238, 60, 73, 167, 204, 1, 69, 66, 78, 76, 182, 250, 1, 65, 38, 85, 206, + 141, 1, 79, 238, 60, 73, 167, 204, 1, 69, 44, 226, 6, 76, 210, 243, 1, + 65, 38, 85, 206, 141, 1, 79, 238, 60, 73, 167, 204, 1, 69, 110, 98, 78, + 174, 5, 71, 2, 89, 210, 243, 1, 65, 38, 85, 206, 141, 1, 79, 238, 60, 73, + 167, 204, 1, 69, 44, 170, 5, 78, 210, 243, 1, 65, 38, 85, 206, 141, 1, + 79, 238, 60, 73, 167, 204, 1, 69, 44, 214, 4, 82, 210, 243, 1, 65, 38, + 85, 206, 141, 1, 79, 238, 60, 73, 167, 204, 1, 69, 68, 90, 72, 170, 3, + 83, 210, 243, 1, 65, 38, 85, 206, 141, 1, 79, 238, 60, 73, 167, 204, 1, + 69, 24, 246, 246, 1, 65, 38, 85, 206, 141, 1, 79, 230, 2, 82, 138, 58, + 73, 167, 204, 1, 69, 44, 210, 2, 84, 210, 243, 1, 65, 38, 85, 206, 141, + 1, 79, 238, 60, 73, 167, 204, 1, 69, 6, 68, 2, 79, 84, 33, 11, 82, 65, + 68, 73, 84, 73, 79, 78, 65, 76, 32, 2, 11, 65, 2, 155, 197, 4, 76, 4, 24, + 2, 67, 82, 55, 78, 2, 17, 2, 69, 68, 2, 11, 73, 2, 203, 196, 4, 84, 2, + 11, 85, 2, 17, 2, 77, 66, 2, 123, 69, 22, 25, 4, 73, 71, 78, 32, 22, 206, + 243, 1, 65, 38, 85, 206, 141, 1, 79, 238, 60, 73, 167, 204, 1, 69, 2, 11, + 65, 2, 155, 195, 4, 82, 158, 15, 36, 5, 65, 66, 65, 84, 65, 35, 71, 2, + 11, 32, 2, 163, 215, 3, 84, 156, 15, 54, 69, 20, 3, 83, 65, 32, 213, 6, + 3, 85, 84, 32, 2, 195, 173, 4, 82, 178, 1, 52, 7, 76, 69, 84, 84, 69, 82, + 32, 171, 212, 3, 68, 158, 1, 210, 1, 65, 58, 70, 54, 72, 38, 76, 58, 77, + 54, 78, 50, 83, 122, 85, 114, 69, 2, 73, 2, 79, 2, 86, 182, 250, 4, 84, + 82, 67, 2, 68, 2, 71, 2, 75, 2, 80, 138, 69, 66, 2, 82, 2, 87, 2, 88, 2, + 89, 3, 90, 16, 146, 4, 87, 198, 194, 5, 67, 2, 81, 2, 88, 3, 90, 4, 164, + 245, 4, 5, 73, 78, 65, 76, 32, 251, 80, 65, 6, 186, 177, 1, 84, 179, 148, + 4, 65, 4, 176, 157, 4, 5, 79, 78, 71, 32, 85, 151, 168, 1, 65, 10, 142, + 197, 5, 65, 2, 67, 2, 81, 2, 88, 3, 90, 8, 162, 194, 5, 71, 2, 72, 2, 89, + 187, 2, 65, 8, 26, 72, 147, 196, 5, 65, 6, 40, 4, 79, 82, 84, 32, 231, + 195, 5, 65, 4, 184, 229, 3, 2, 85, 69, 233, 79, 2, 65, 87, 32, 58, 73, + 54, 69, 198, 194, 5, 67, 2, 81, 2, 88, 3, 90, 16, 50, 85, 198, 194, 5, + 67, 2, 81, 2, 88, 3, 90, 8, 194, 194, 5, 67, 2, 81, 2, 88, 3, 90, 232, + 13, 64, 10, 67, 79, 77, 80, 79, 78, 69, 78, 84, 45, 183, 155, 3, 73, 230, + 13, 78, 48, 178, 1, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 95, 56, + 198, 1, 86, 48, 130, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, + 56, 3, 57, 18, 158, 192, 5, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, + 2, 56, 3, 57, 200, 1, 166, 1, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, + 54, 2, 55, 2, 56, 3, 57, 168, 1, 74, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, + 53, 2, 54, 2, 55, 95, 56, 20, 158, 190, 5, 48, 2, 49, 2, 50, 2, 51, 2, + 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 8, 194, 189, 5, 48, 2, 49, 2, 50, + 3, 51, 4, 48, 6, 67, 65, 82, 84, 82, 73, 21, 2, 68, 82, 2, 175, 133, 4, + 68, 2, 251, 191, 4, 73, 2, 215, 179, 3, 82, 144, 3, 154, 1, 65, 152, 2, + 5, 68, 68, 89, 32, 66, 22, 76, 230, 14, 78, 144, 1, 5, 83, 84, 32, 84, + 85, 21, 12, 84, 82, 65, 71, 82, 65, 77, 32, 70, 79, 82, 32, 10, 68, 9, + 67, 85, 80, 32, 87, 73, 84, 72, 79, 58, 82, 211, 233, 4, 80, 2, 33, 6, + 85, 84, 32, 72, 65, 78, 2, 223, 147, 4, 68, 6, 72, 9, 45, 79, 70, 70, 32, + 67, 65, 76, 69, 29, 5, 68, 82, 79, 80, 45, 2, 253, 149, 4, 2, 78, 68, 4, + 170, 161, 3, 83, 205, 117, 4, 66, 65, 82, 66, 2, 179, 149, 4, 69, 218, 1, + 38, 69, 213, 2, 4, 85, 71, 85, 32, 16, 60, 6, 80, 72, 79, 78, 69, 32, + 246, 1, 83, 211, 184, 2, 86, 12, 132, 1, 3, 82, 69, 67, 208, 150, 1, 3, + 76, 79, 67, 178, 153, 3, 83, 189, 116, 13, 79, 78, 32, 84, 79, 80, 32, + 79, 70, 32, 77, 79, 68, 6, 36, 5, 69, 73, 86, 69, 82, 51, 79, 5, 29, 5, + 32, 87, 73, 84, 72, 2, 243, 38, 32, 2, 183, 146, 3, 82, 2, 11, 67, 2, + 151, 236, 3, 79, 202, 1, 162, 1, 65, 84, 15, 70, 82, 65, 67, 84, 73, 79, + 78, 32, 68, 73, 71, 73, 84, 32, 164, 2, 2, 76, 69, 252, 3, 5, 83, 73, 71, + 78, 32, 198, 2, 86, 247, 182, 3, 68, 4, 60, 9, 82, 67, 72, 65, 73, 67, + 32, 83, 72, 243, 210, 1, 73, 2, 167, 237, 2, 82, 14, 74, 84, 40, 2, 79, + 78, 81, 10, 90, 69, 82, 79, 32, 70, 79, 82, 32, 79, 8, 36, 3, 72, 82, 69, + 13, 2, 87, 79, 4, 11, 69, 4, 29, 5, 32, 70, 79, 82, 32, 4, 34, 79, 21, 4, + 69, 86, 69, 78, 2, 17, 2, 68, 68, 2, 49, 10, 32, 80, 79, 87, 69, 82, 83, + 32, 79, 70, 2, 133, 22, 2, 32, 70, 114, 44, 5, 84, 84, 69, 82, 32, 167, + 239, 4, 78, 112, 214, 1, 68, 54, 78, 106, 82, 38, 84, 130, 208, 1, 65, + 82, 76, 114, 86, 186, 5, 85, 206, 141, 1, 79, 238, 60, 73, 182, 196, 1, + 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 162, 7, 69, 234, 61, 72, + 2, 77, 3, 89, 10, 242, 231, 4, 68, 138, 69, 72, 2, 90, 187, 2, 65, 10, + 42, 65, 158, 172, 5, 71, 2, 78, 3, 89, 5, 41, 8, 75, 65, 65, 82, 65, 32, + 80, 79, 2, 251, 36, 76, 6, 150, 209, 1, 82, 131, 221, 3, 65, 10, 178, + 230, 4, 84, 138, 69, 72, 2, 83, 187, 2, 65, 20, 86, 67, 194, 1, 83, 204, + 35, 3, 84, 85, 85, 222, 106, 78, 190, 66, 65, 187, 151, 3, 86, 6, 60, 9, + 79, 77, 66, 73, 78, 73, 78, 71, 32, 215, 142, 1, 65, 4, 70, 65, 221, 174, + 4, 11, 67, 65, 78, 68, 82, 65, 66, 73, 78, 68, 85, 2, 33, 6, 78, 85, 83, + 86, 65, 82, 2, 183, 174, 4, 65, 2, 11, 73, 2, 137, 238, 3, 3, 68, 68, 72, + 30, 49, 10, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 30, 166, 211, 1, 65, + 38, 85, 22, 86, 186, 141, 1, 79, 238, 60, 73, 167, 204, 1, 69, 6, 80, 10, + 78, 73, 83, 32, 82, 65, 67, 81, 85, 69, 158, 137, 3, 71, 139, 160, 2, 84, + 2, 11, 84, 2, 25, 4, 32, 65, 78, 68, 2, 187, 158, 3, 32, 2, 147, 146, 5, + 66, 162, 1, 210, 2, 65, 134, 1, 66, 174, 1, 67, 222, 2, 68, 202, 3, 69, + 154, 2, 70, 182, 1, 71, 198, 1, 72, 64, 8, 89, 79, 85, 84, 72, 70, 85, + 76, 52, 2, 73, 78, 46, 75, 98, 76, 146, 1, 77, 134, 1, 79, 62, 80, 142, + 1, 82, 210, 1, 83, 224, 1, 14, 86, 65, 83, 84, 78, 69, 83, 83, 32, 79, + 82, 32, 87, 65, 12, 2, 87, 65, 138, 94, 85, 167, 159, 4, 74, 8, 88, 4, + 67, 67, 85, 77, 22, 83, 228, 20, 2, 68, 86, 221, 202, 1, 5, 71, 71, 82, + 65, 86, 2, 195, 190, 1, 85, 2, 227, 225, 4, 67, 6, 92, 7, 79, 76, 68, 32, + 82, 69, 83, 32, 5, 82, 65, 78, 67, 72, 229, 152, 4, 3, 65, 82, 82, 2, + 197, 211, 4, 3, 79, 76, 85, 2, 11, 73, 2, 205, 170, 1, 3, 78, 71, 32, 22, + 54, 72, 20, 3, 76, 79, 83, 66, 79, 167, 157, 2, 69, 2, 211, 235, 3, 65, + 6, 26, 69, 171, 128, 4, 85, 4, 222, 253, 1, 68, 167, 140, 2, 78, 12, 28, + 3, 77, 80, 76, 35, 78, 4, 246, 17, 73, 219, 191, 4, 69, 8, 32, 4, 83, 84, + 65, 78, 23, 84, 2, 147, 144, 5, 67, 6, 46, 69, 20, 3, 82, 65, 82, 251, + 143, 4, 65, 2, 199, 208, 4, 78, 2, 155, 176, 3, 73, 20, 80, 4, 65, 82, + 75, 69, 22, 69, 210, 1, 73, 74, 85, 197, 129, 5, 3, 79, 85, 66, 2, 151, + 207, 4, 78, 6, 132, 1, 19, 70, 69, 67, 84, 73, 86, 69, 78, 69, 83, 83, + 32, 79, 82, 32, 68, 73, 83, 84, 34, 80, 137, 7, 6, 67, 73, 83, 73, 86, + 69, 2, 11, 79, 2, 163, 206, 4, 82, 2, 11, 65, 2, 11, 82, 2, 143, 10, 84, + 8, 68, 6, 70, 70, 73, 67, 85, 76, 34, 77, 161, 12, 4, 86, 69, 82, 71, 2, + 11, 84, 2, 223, 135, 4, 73, 4, 140, 1, 2, 73, 78, 191, 203, 4, 77, 14, + 100, 5, 77, 66, 69, 76, 76, 34, 78, 124, 4, 88, 72, 65, 85, 172, 106, 3, + 84, 69, 82, 215, 165, 3, 65, 2, 137, 217, 4, 3, 73, 83, 72, 6, 80, 4, 68, + 69, 65, 86, 20, 4, 76, 65, 82, 71, 229, 151, 3, 4, 67, 79, 85, 78, 2, + 247, 162, 4, 79, 2, 131, 216, 4, 69, 2, 211, 202, 4, 83, 12, 70, 79, 76, + 3, 85, 76, 76, 196, 5, 3, 65, 73, 76, 235, 233, 4, 76, 4, 18, 76, 35, 83, + 2, 173, 201, 4, 3, 76, 79, 87, 2, 205, 4, 2, 84, 69, 4, 214, 197, 3, 32, + 199, 59, 78, 10, 96, 8, 65, 84, 72, 69, 82, 73, 78, 71, 22, 79, 64, 3, + 82, 69, 65, 65, 5, 85, 65, 82, 68, 69, 5, 163, 153, 4, 32, 2, 41, 8, 73, + 78, 71, 32, 84, 79, 32, 77, 2, 159, 227, 4, 69, 2, 75, 84, 4, 48, 2, 65, + 82, 33, 6, 79, 76, 68, 73, 78, 71, 2, 11, 68, 2, 247, 254, 3, 78, 2, 11, + 32, 2, 227, 248, 1, 66, 4, 26, 67, 159, 217, 4, 78, 2, 143, 5, 82, 4, 60, + 7, 69, 69, 80, 73, 78, 71, 32, 169, 193, 4, 2, 73, 78, 2, 11, 83, 2, 247, + 139, 3, 77, 6, 18, 65, 107, 69, 4, 60, 3, 66, 79, 85, 21, 8, 87, 32, 79, + 82, 32, 77, 79, 68, 2, 191, 196, 4, 82, 2, 223, 137, 4, 69, 2, 215, 196, + 4, 71, 6, 26, 65, 30, 69, 47, 73, 2, 229, 195, 4, 2, 83, 83, 2, 11, 65, + 2, 11, 83, 2, 251, 241, 3, 85, 2, 11, 82, 2, 247, 205, 4, 69, 4, 220, + 209, 3, 7, 78, 32, 84, 72, 69, 32, 86, 199, 112, 80, 8, 54, 65, 64, 4, + 69, 78, 69, 84, 229, 97, 2, 85, 82, 4, 22, 84, 243, 2, 67, 2, 17, 2, 84, + 69, 2, 131, 195, 4, 82, 2, 131, 205, 1, 82, 12, 30, 69, 157, 1, 2, 73, + 84, 10, 34, 76, 22, 83, 191, 239, 4, 65, 2, 219, 232, 1, 69, 6, 18, 73, + 51, 80, 4, 30, 68, 137, 1, 2, 83, 84, 2, 147, 1, 69, 2, 11, 79, 2, 239, + 133, 4, 78, 2, 163, 133, 4, 85, 10, 34, 69, 64, 2, 73, 78, 23, 84, 2, 11, + 86, 2, 17, 2, 69, 82, 2, 11, 65, 2, 159, 211, 4, 78, 2, 131, 191, 4, 75, + 6, 42, 79, 209, 138, 3, 4, 82, 69, 78, 71, 4, 26, 80, 219, 248, 4, 86, 2, + 11, 80, 2, 235, 215, 3, 65, 2, 39, 83, 4, 26, 73, 191, 236, 4, 84, 2, + 223, 189, 4, 84, 226, 2, 78, 65, 182, 31, 69, 146, 2, 73, 206, 1, 79, + 100, 3, 82, 69, 69, 187, 8, 85, 146, 2, 44, 4, 65, 78, 65, 32, 145, 10, + 2, 73, 32, 100, 122, 65, 38, 69, 76, 7, 76, 69, 84, 84, 69, 82, 32, 154, + 7, 79, 76, 3, 73, 66, 73, 0, 3, 85, 66, 85, 45, 2, 83, 85, 4, 184, 8, 3, + 65, 66, 65, 3, 66, 6, 58, 66, 0, 3, 69, 66, 69, 245, 7, 4, 89, 66, 69, + 89, 2, 243, 7, 69, 78, 202, 1, 65, 42, 68, 82, 70, 2, 81, 14, 71, 50, 72, + 38, 75, 62, 76, 32, 3, 77, 69, 69, 20, 2, 67, 72, 2, 74, 2, 80, 18, 78, + 42, 83, 94, 84, 102, 86, 2, 87, 42, 90, 246, 143, 3, 66, 2, 82, 3, 89, 4, + 252, 1, 2, 76, 73, 159, 227, 4, 73, 6, 30, 65, 29, 3, 72, 65, 65, 4, 218, + 2, 65, 255, 1, 86, 2, 195, 228, 4, 76, 2, 123, 65, 6, 110, 65, 86, 78, + 213, 226, 4, 3, 72, 65, 73, 4, 166, 147, 3, 72, 255, 242, 1, 65, 4, 26, + 65, 235, 146, 3, 72, 2, 11, 65, 2, 167, 227, 4, 70, 4, 18, 65, 35, 72, 2, + 11, 65, 2, 243, 226, 4, 77, 2, 211, 1, 65, 4, 192, 226, 4, 2, 79, 79, + 191, 34, 65, 8, 32, 2, 65, 65, 18, 72, 23, 69, 2, 163, 16, 68, 4, 18, 69, + 87, 65, 2, 211, 225, 4, 69, 10, 62, 65, 16, 3, 72, 65, 65, 178, 144, 3, + 84, 183, 245, 1, 79, 2, 131, 1, 86, 5, 251, 224, 4, 76, 2, 17, 2, 65, 65, + 2, 211, 224, 4, 86, 6, 26, 65, 143, 133, 5, 79, 4, 26, 86, 243, 132, 5, + 65, 2, 21, 3, 73, 89, 65, 2, 255, 240, 4, 78, 6, 48, 3, 65, 66, 79, 14, + 66, 1, 3, 79, 66, 79, 2, 23, 65, 2, 11, 79, 2, 11, 70, 2, 11, 73, 2, 135, + 240, 4, 76, 2, 155, 131, 4, 75, 174, 1, 26, 67, 147, 143, 3, 68, 154, 1, + 132, 1, 9, 72, 65, 82, 65, 67, 84, 69, 82, 32, 213, 215, 4, 17, 85, 82, + 82, 69, 78, 67, 89, 32, 83, 89, 77, 66, 79, 76, 32, 66, 65, 152, 1, 176, + 2, 6, 66, 79, 32, 66, 65, 73, 16, 6, 67, 72, 79, 32, 67, 72, 56, 3, 68, + 79, 32, 44, 2, 70, 79, 76, 3, 72, 79, 32, 66, 75, 190, 1, 76, 138, 1, 77, + 252, 1, 7, 65, 78, 71, 75, 72, 65, 78, 46, 78, 158, 1, 80, 234, 1, 82, + 70, 83, 238, 2, 84, 224, 2, 3, 87, 79, 32, 34, 89, 225, 150, 4, 2, 79, + 32, 2, 143, 12, 77, 8, 192, 129, 3, 2, 65, 78, 138, 173, 1, 73, 167, 58, + 79, 4, 132, 218, 4, 3, 67, 72, 65, 227, 33, 68, 6, 32, 2, 32, 70, 21, 2, + 78, 71, 4, 207, 128, 3, 65, 2, 139, 178, 3, 77, 4, 40, 4, 78, 79, 75, 72, + 191, 252, 4, 72, 2, 207, 250, 4, 85, 14, 40, 2, 72, 79, 229, 9, 3, 79, + 32, 75, 12, 26, 32, 191, 131, 4, 77, 10, 36, 2, 75, 72, 57, 3, 82, 65, + 75, 8, 158, 9, 87, 150, 234, 3, 85, 210, 57, 79, 139, 60, 65, 2, 215, + 163, 4, 72, 8, 76, 2, 79, 32, 224, 11, 8, 65, 75, 75, 72, 65, 78, 71, 89, + 235, 239, 4, 85, 4, 32, 2, 67, 72, 163, 170, 4, 76, 2, 139, 217, 4, 85, + 16, 28, 2, 65, 73, 211, 52, 79, 14, 42, 32, 176, 1, 3, 84, 65, 73, 19, + 89, 10, 76, 4, 67, 72, 65, 84, 44, 5, 72, 65, 78, 45, 65, 22, 84, 159, + 246, 4, 69, 2, 11, 84, 2, 11, 65, 2, 163, 247, 4, 87, 2, 151, 240, 3, 75, + 4, 190, 218, 4, 72, 159, 11, 82, 2, 203, 2, 75, 2, 161, 207, 4, 2, 65, + 77, 8, 60, 3, 71, 79, 32, 32, 3, 73, 75, 72, 29, 3, 79, 32, 78, 2, 11, + 78, 2, 179, 211, 4, 71, 2, 205, 219, 4, 2, 65, 72, 4, 138, 168, 4, 69, + 227, 79, 85, 12, 68, 6, 65, 73, 89, 65, 78, 78, 22, 72, 217, 212, 4, 3, + 79, 32, 80, 2, 171, 227, 4, 79, 8, 36, 3, 73, 78, 84, 21, 2, 79, 32, 2, + 223, 209, 4, 72, 6, 44, 2, 80, 72, 161, 5, 4, 83, 65, 77, 80, 4, 138, + 165, 4, 85, 155, 1, 65, 4, 32, 2, 79, 32, 195, 245, 4, 85, 2, 11, 82, 2, + 251, 242, 4, 85, 36, 44, 4, 65, 82, 65, 32, 225, 1, 2, 79, 32, 28, 62, + 65, 130, 1, 85, 238, 230, 2, 73, 198, 140, 2, 69, 3, 79, 13, 64, 6, 73, + 32, 77, 65, 73, 77, 238, 243, 4, 65, 2, 69, 3, 77, 4, 26, 65, 183, 167, + 3, 85, 2, 11, 76, 2, 235, 223, 4, 65, 9, 142, 179, 4, 69, 163, 64, 85, 8, + 24, 2, 82, 85, 23, 83, 2, 155, 223, 4, 83, 6, 198, 208, 4, 65, 226, 31, + 85, 187, 2, 79, 18, 30, 72, 133, 2, 2, 79, 32, 14, 48, 6, 65, 78, 84, 72, + 65, 75, 21, 2, 79, 32, 2, 175, 232, 3, 72, 12, 80, 8, 78, 65, 78, 71, 77, + 79, 78, 84, 20, 4, 80, 72, 85, 84, 13, 2, 84, 72, 2, 131, 210, 4, 72, 2, + 123, 72, 8, 34, 65, 190, 159, 4, 79, 3, 85, 4, 150, 164, 3, 72, 159, 204, + 1, 78, 4, 38, 84, 225, 182, 2, 3, 80, 65, 84, 2, 243, 208, 4, 65, 2, 11, + 87, 2, 195, 153, 3, 65, 6, 30, 65, 45, 3, 79, 32, 89, 2, 21, 3, 77, 65, + 75, 2, 223, 162, 3, 75, 4, 238, 157, 4, 73, 227, 77, 65, 12, 26, 82, 167, + 199, 2, 84, 10, 30, 69, 145, 1, 2, 77, 79, 6, 18, 32, 107, 70, 4, 84, 11, + 68, 79, 69, 83, 32, 78, 79, 84, 32, 69, 88, 229, 228, 3, 4, 69, 88, 73, + 83, 2, 171, 127, 73, 2, 211, 202, 3, 79, 4, 46, 77, 177, 194, 3, 5, 68, + 89, 78, 65, 77, 2, 191, 178, 3, 69, 10, 18, 78, 95, 82, 8, 26, 32, 235, + 227, 3, 75, 6, 26, 83, 203, 224, 2, 71, 4, 170, 211, 2, 65, 187, 219, 1, + 80, 2, 21, 3, 68, 32, 80, 2, 25, 4, 76, 65, 67, 69, 2, 17, 2, 32, 77, 2, + 159, 213, 3, 69, 4, 56, 4, 85, 71, 72, 84, 189, 212, 3, 4, 78, 71, 32, + 83, 2, 133, 192, 1, 5, 32, 66, 65, 76, 76, 46, 22, 32, 207, 3, 45, 30, + 154, 1, 68, 90, 76, 118, 82, 170, 158, 1, 66, 86, 67, 194, 6, 83, 254, + 166, 1, 80, 253, 22, 15, 78, 69, 84, 87, 79, 82, 75, 69, 68, 32, 67, 79, + 77, 80, 85, 4, 64, 10, 73, 77, 69, 78, 83, 73, 79, 78, 65, 76, 135, 162, + 1, 79, 2, 175, 193, 3, 32, 6, 80, 12, 73, 78, 69, 83, 32, 67, 79, 78, 86, + 69, 82, 71, 249, 154, 1, 2, 69, 70, 4, 153, 188, 4, 3, 73, 78, 71, 10, + 40, 4, 65, 89, 83, 32, 155, 154, 1, 73, 8, 238, 196, 2, 66, 130, 165, 1, + 65, 130, 82, 76, 31, 82, 16, 44, 2, 68, 32, 254, 3, 80, 247, 160, 1, 69, + 12, 196, 2, 12, 84, 79, 80, 45, 76, 73, 71, 72, 84, 69, 68, 32, 80, 17, + 76, 69, 70, 84, 45, 76, 73, 71, 72, 84, 69, 68, 32, 68, 79, 87, 78, 0, + 16, 82, 73, 71, 72, 84, 45, 76, 73, 71, 72, 84, 69, 68, 32, 85, 80, 221, + 137, 2, 25, 66, 79, 84, 84, 79, 77, 45, 76, 73, 71, 72, 84, 69, 68, 32, + 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 6, 76, 4, 76, 69, 70, 84, 69, 11, + 82, 73, 71, 72, 84, 87, 65, 82, 68, 83, 32, 2, 11, 87, 2, 25, 4, 65, 82, + 68, 83, 2, 173, 243, 1, 2, 32, 69, 4, 146, 243, 1, 69, 159, 22, 65, 2, + 177, 163, 4, 5, 69, 82, 45, 69, 77, 8, 34, 77, 85, 4, 78, 68, 69, 82, 4, + 21, 3, 66, 83, 32, 4, 38, 85, 229, 228, 2, 3, 68, 79, 87, 2, 235, 217, 3, + 80, 4, 216, 177, 3, 10, 32, 67, 76, 79, 85, 68, 32, 65, 78, 68, 181, 48, + 2, 83, 84, 230, 5, 204, 1, 6, 66, 69, 84, 65, 78, 32, 148, 45, 6, 69, 32, + 79, 86, 69, 82, 72, 7, 70, 73, 78, 65, 71, 72, 32, 206, 8, 71, 152, 1, 3, + 76, 68, 69, 252, 2, 2, 77, 69, 60, 2, 78, 89, 78, 82, 171, 208, 3, 67, + 160, 3, 228, 2, 18, 65, 83, 84, 82, 79, 76, 79, 71, 73, 67, 65, 76, 32, + 83, 73, 71, 78, 32, 172, 1, 18, 67, 65, 78, 84, 73, 76, 76, 65, 84, 73, + 79, 78, 32, 83, 73, 71, 78, 32, 172, 1, 6, 68, 73, 71, 73, 84, 32, 100, + 9, 75, 85, 32, 82, 85, 32, 75, 72, 65, 26, 76, 176, 3, 5, 77, 65, 82, 75, + 32, 222, 18, 83, 181, 15, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, + 6, 46, 83, 169, 40, 6, 45, 75, 72, 89, 85, 68, 4, 104, 8, 68, 79, 78, 71, + 32, 84, 83, 72, 169, 21, 13, 71, 82, 65, 32, 71, 67, 65, 78, 32, 45, 67, + 72, 65, 2, 199, 31, 85, 8, 148, 1, 7, 83, 66, 85, 66, 32, 45, 67, 180, + 207, 3, 5, 72, 69, 65, 86, 89, 0, 5, 76, 73, 71, 72, 84, 233, 100, 8, 67, + 65, 78, 71, 32, 84, 69, 45, 2, 167, 205, 3, 72, 40, 144, 229, 2, 4, 72, + 65, 76, 70, 82, 70, 30, 83, 42, 84, 62, 90, 210, 86, 78, 14, 79, 227, + 112, 69, 5, 221, 10, 2, 32, 66, 92, 96, 6, 69, 84, 84, 69, 82, 32, 153, + 2, 13, 79, 71, 79, 84, 89, 80, 69, 32, 83, 73, 71, 78, 32, 88, 230, 1, + 75, 170, 120, 82, 168, 146, 3, 10, 70, 73, 88, 69, 68, 45, 70, 79, 82, + 77, 202, 1, 68, 86, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 80, 2, + 90, 138, 69, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 87, 2, 89, 187, 2, 65, 8, + 226, 210, 4, 83, 14, 72, 2, 75, 187, 2, 65, 4, 224, 25, 3, 76, 72, 65, + 13, 4, 67, 72, 65, 68, 80, 254, 2, 66, 176, 1, 8, 77, 78, 89, 65, 77, 32, + 89, 73, 114, 67, 172, 2, 13, 89, 73, 71, 32, 77, 71, 79, 32, 84, 83, 72, + 69, 71, 190, 1, 71, 212, 2, 9, 65, 78, 71, 32, 75, 72, 65, 78, 71, 58, + 72, 48, 2, 73, 78, 246, 1, 78, 222, 1, 82, 118, 83, 46, 84, 36, 4, 76, + 69, 65, 68, 220, 105, 2, 80, 65, 173, 191, 1, 17, 68, 69, 76, 73, 77, 73, + 84, 69, 82, 32, 84, 83, 72, 69, 71, 32, 66, 10, 52, 9, 75, 65, 45, 32, + 83, 72, 79, 71, 32, 27, 83, 4, 142, 1, 71, 67, 89, 6, 34, 75, 201, 21, 3, + 68, 85, 83, 4, 56, 6, 65, 45, 32, 83, 72, 79, 89, 4, 85, 82, 32, 89, 2, + 21, 3, 71, 32, 71, 2, 41, 8, 73, 32, 77, 71, 79, 32, 82, 71, 2, 179, 131, + 3, 89, 2, 221, 2, 2, 73, 71, 12, 84, 5, 65, 82, 69, 84, 32, 240, 1, 2, + 72, 69, 29, 7, 76, 79, 83, 73, 78, 71, 32, 6, 112, 12, 45, 68, 90, 85, + 68, 32, 82, 84, 65, 71, 83, 32, 97, 12, 89, 73, 71, 32, 77, 71, 79, 32, + 80, 72, 85, 82, 4, 42, 66, 37, 6, 77, 69, 32, 76, 79, 78, 2, 33, 6, 90, + 72, 73, 32, 77, 73, 2, 203, 22, 71, 2, 225, 3, 3, 32, 83, 72, 2, 137, + 189, 2, 2, 32, 77, 4, 68, 13, 66, 82, 68, 65, 32, 82, 78, 89, 73, 78, 71, + 32, 89, 3, 89, 2, 213, 5, 11, 73, 71, 32, 77, 71, 79, 32, 83, 71, 65, 66, + 12, 68, 4, 84, 69, 82, 32, 141, 2, 8, 85, 71, 32, 82, 84, 65, 71, 83, 8, + 56, 8, 89, 73, 71, 32, 77, 71, 79, 32, 191, 132, 4, 84, 6, 68, 4, 45, 85, + 77, 32, 117, 9, 84, 82, 85, 78, 67, 65, 84, 69, 68, 4, 88, 7, 82, 78, 65, + 77, 32, 66, 67, 245, 2, 10, 71, 84, 69, 82, 32, 84, 83, 72, 69, 71, 2, + 241, 2, 2, 65, 68, 2, 195, 198, 4, 32, 4, 21, 3, 32, 71, 89, 4, 238, 248, + 3, 79, 135, 18, 65, 2, 17, 2, 65, 76, 2, 205, 163, 4, 2, 65, 78, 6, 92, + 6, 73, 84, 73, 65, 76, 32, 145, 129, 4, 11, 84, 69, 82, 83, 89, 76, 76, + 65, 66, 73, 67, 4, 68, 13, 66, 82, 68, 65, 32, 82, 78, 89, 73, 78, 71, + 32, 89, 3, 89, 2, 53, 11, 73, 71, 32, 77, 71, 79, 32, 77, 68, 85, 78, 2, + 143, 165, 4, 32, 10, 72, 10, 71, 65, 83, 32, 66, 90, 85, 78, 71, 32, 77, + 4, 89, 73, 83, 32, 4, 56, 3, 83, 71, 79, 237, 162, 4, 5, 78, 89, 73, 32, + 90, 2, 251, 9, 82, 6, 44, 5, 84, 83, 72, 69, 71, 143, 254, 3, 83, 5, 255, + 253, 3, 32, 4, 220, 253, 3, 8, 71, 89, 65, 32, 71, 82, 65, 77, 1, 14, 73, + 78, 32, 67, 72, 69, 78, 32, 83, 80, 85, 78, 71, 83, 4, 232, 252, 3, 4, + 66, 82, 85, 76, 39, 72, 6, 32, 4, 82, 65, 73, 76, 55, 83, 2, 225, 7, 9, + 73, 78, 71, 32, 77, 67, 72, 65, 78, 4, 56, 5, 65, 32, 45, 80, 72, 173, + 251, 3, 3, 72, 69, 71, 2, 147, 157, 4, 82, 156, 1, 84, 4, 73, 71, 78, 32, + 228, 6, 9, 85, 66, 74, 79, 73, 78, 69, 68, 32, 143, 4, 89, 42, 176, 1, 4, + 71, 82, 85, 32, 96, 2, 76, 67, 30, 77, 36, 11, 78, 89, 73, 32, 90, 76, + 65, 32, 78, 65, 65, 22, 82, 252, 2, 2, 89, 65, 130, 4, 73, 165, 4, 5, 83, + 78, 65, 32, 76, 4, 40, 3, 67, 65, 78, 1, 3, 77, 69, 68, 2, 25, 4, 32, 82, + 71, 89, 2, 169, 4, 2, 73, 78, 4, 238, 3, 73, 179, 4, 69, 4, 136, 4, 2, + 65, 82, 231, 3, 67, 2, 179, 153, 4, 32, 20, 116, 4, 68, 69, 76, 32, 240, + 1, 10, 74, 69, 83, 32, 83, 85, 32, 78, 71, 65, 145, 245, 3, 6, 78, 65, + 77, 32, 66, 67, 16, 52, 5, 68, 75, 65, 82, 32, 53, 4, 78, 65, 71, 32, 8, + 94, 71, 169, 208, 2, 6, 82, 68, 69, 76, 32, 78, 8, 42, 71, 69, 6, 82, 68, + 69, 76, 32, 68, 6, 46, 67, 220, 148, 2, 2, 78, 89, 231, 123, 83, 2, 143, + 187, 4, 73, 2, 147, 152, 3, 75, 2, 187, 201, 2, 32, 4, 18, 78, 71, 82, 2, + 11, 71, 2, 21, 3, 32, 82, 84, 2, 11, 65, 2, 135, 253, 3, 71, 2, 17, 2, + 32, 84, 2, 171, 164, 3, 83, 94, 68, 7, 76, 69, 84, 84, 69, 82, 32, 145, + 2, 5, 83, 73, 71, 78, 32, 88, 220, 1, 10, 70, 73, 88, 69, 68, 45, 70, 79, + 82, 77, 242, 238, 3, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, + 2, 71, 2, 80, 2, 90, 138, 69, 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, + 87, 2, 89, 187, 2, 65, 6, 11, 32, 6, 186, 181, 4, 82, 2, 87, 3, 89, 6, + 38, 73, 50, 77, 33, 3, 76, 67, 69, 2, 45, 9, 78, 86, 69, 82, 84, 69, 68, + 32, 77, 2, 11, 67, 2, 45, 2, 72, 85, 2, 25, 4, 32, 84, 83, 65, 2, 11, 32, + 2, 147, 234, 2, 67, 20, 60, 6, 76, 76, 65, 66, 76, 69, 21, 5, 77, 66, 79, + 76, 32, 2, 247, 164, 4, 32, 18, 104, 4, 68, 82, 73, 76, 32, 6, 78, 79, + 82, 32, 66, 85, 130, 1, 80, 93, 7, 82, 68, 79, 32, 82, 74, 69, 2, 11, 32, + 2, 231, 143, 4, 66, 9, 11, 32, 6, 72, 4, 66, 90, 72, 73, 0, 4, 71, 83, + 85, 77, 1, 4, 78, 89, 73, 83, 2, 145, 212, 2, 5, 32, 45, 75, 72, 89, 4, + 52, 6, 65, 68, 77, 65, 32, 71, 21, 3, 72, 85, 82, 2, 231, 230, 2, 68, 2, + 199, 117, 32, 5, 181, 245, 2, 6, 32, 82, 71, 89, 65, 32, 30, 116, 8, 82, + 69, 86, 69, 82, 83, 69, 68, 194, 90, 85, 22, 86, 186, 141, 1, 79, 238, + 60, 73, 166, 204, 1, 69, 235, 61, 65, 4, 245, 164, 2, 2, 32, 73, 2, 25, + 4, 32, 73, 78, 70, 2, 11, 73, 2, 11, 78, 2, 139, 193, 2, 73, 118, 216, 1, + 9, 67, 79, 78, 83, 79, 78, 65, 78, 84, 20, 7, 76, 69, 84, 84, 69, 82, 32, + 164, 6, 20, 77, 79, 68, 73, 70, 73, 69, 82, 32, 76, 69, 84, 84, 69, 82, + 32, 76, 65, 66, 73, 37, 8, 83, 69, 80, 65, 82, 65, 84, 79, 2, 223, 240, + 3, 32, 112, 106, 65, 108, 17, 66, 69, 82, 66, 69, 82, 32, 65, 67, 65, 68, + 69, 77, 89, 32, 89, 65, 30, 84, 211, 1, 89, 4, 84, 6, 89, 69, 82, 32, 89, + 65, 137, 172, 4, 9, 72, 65, 71, 71, 65, 82, 32, 89, 65, 2, 147, 172, 4, + 71, 4, 142, 173, 4, 72, 3, 74, 18, 92, 11, 65, 87, 69, 76, 76, 69, 77, + 69, 84, 32, 89, 21, 8, 85, 65, 82, 69, 71, 32, 89, 65, 2, 235, 205, 2, + 65, 16, 62, 71, 226, 26, 75, 210, 143, 4, 90, 62, 78, 86, 72, 3, 81, 4, + 190, 171, 4, 72, 3, 78, 86, 54, 65, 210, 25, 69, 158, 145, 4, 73, 2, 79, + 3, 85, 77, 194, 1, 71, 2, 75, 18, 83, 30, 84, 30, 90, 134, 23, 68, 126, + 66, 2, 72, 198, 58, 82, 142, 213, 3, 67, 146, 1, 65, 2, 70, 2, 74, 2, 76, + 2, 77, 2, 78, 2, 80, 2, 81, 2, 86, 2, 87, 3, 89, 7, 199, 24, 72, 7, 150, + 169, 4, 72, 3, 83, 7, 250, 168, 4, 72, 3, 84, 7, 222, 168, 4, 72, 3, 90, + 2, 141, 191, 3, 4, 65, 76, 73, 90, 2, 219, 230, 3, 82, 6, 76, 2, 69, 82, + 21, 13, 72, 84, 32, 84, 82, 73, 70, 79, 76, 73, 65, 84, 69, 5, 167, 234, + 3, 32, 2, 33, 6, 32, 83, 78, 79, 87, 70, 2, 219, 58, 76, 19, 11, 32, 16, + 72, 8, 79, 80, 69, 82, 65, 84, 79, 82, 233, 1, 5, 87, 73, 84, 72, 32, 11, + 11, 32, 8, 38, 65, 101, 5, 87, 73, 84, 72, 32, 4, 25, 4, 66, 79, 86, 69, + 4, 11, 32, 4, 26, 76, 171, 131, 3, 82, 2, 11, 69, 2, 179, 131, 3, 70, 4, + 26, 82, 159, 164, 3, 68, 2, 17, 2, 73, 83, 2, 249, 209, 2, 3, 73, 78, 71, + 6, 26, 68, 223, 239, 1, 82, 4, 11, 79, 4, 247, 129, 2, 84, 6, 26, 82, + 187, 131, 1, 83, 2, 11, 32, 2, 195, 177, 1, 67, 5, 21, 3, 32, 84, 87, 2, + 21, 3, 79, 32, 68, 2, 185, 92, 3, 79, 84, 83, 170, 1, 100, 5, 72, 85, 84, + 65, 32, 184, 6, 11, 79, 78, 73, 65, 78, 32, 83, 73, 71, 78, 32, 231, 136, + 3, 69, 164, 1, 178, 1, 65, 88, 7, 76, 69, 84, 84, 69, 82, 32, 204, 1, 2, + 83, 73, 200, 1, 11, 86, 79, 87, 69, 76, 32, 83, 73, 71, 78, 32, 218, 168, + 2, 68, 180, 155, 1, 2, 71, 86, 255, 71, 79, 4, 18, 66, 51, 78, 2, 29, 5, + 66, 82, 69, 86, 73, 2, 247, 26, 65, 2, 175, 140, 4, 74, 94, 254, 65, 65, + 38, 68, 114, 84, 46, 86, 186, 5, 85, 186, 202, 1, 73, 138, 196, 1, 78, + 46, 83, 82, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, 80, 138, 69, 72, 2, 76, 2, + 77, 2, 82, 2, 89, 186, 2, 69, 3, 79, 12, 21, 3, 71, 78, 32, 12, 46, 67, + 98, 78, 190, 66, 65, 187, 151, 3, 86, 2, 11, 65, 2, 11, 78, 2, 25, 4, 68, + 82, 65, 66, 2, 11, 73, 2, 11, 78, 2, 171, 248, 3, 68, 2, 11, 85, 2, 167, + 248, 3, 75, 30, 78, 83, 242, 68, 65, 38, 85, 22, 86, 166, 202, 1, 73, + 198, 140, 2, 69, 3, 79, 4, 25, 4, 72, 79, 82, 84, 4, 11, 32, 4, 230, 155, + 4, 69, 3, 79, 4, 176, 230, 3, 8, 67, 65, 80, 73, 84, 65, 76, 32, 239, 24, + 69, 254, 2, 192, 2, 12, 68, 72, 82, 73, 32, 76, 69, 84, 84, 69, 82, 32, + 180, 4, 10, 76, 79, 78, 71, 32, 83, 73, 75, 73, 32, 154, 4, 77, 18, 78, + 34, 79, 80, 2, 80, 32, 168, 17, 23, 82, 84, 79, 73, 83, 69, 32, 83, 72, + 69, 76, 76, 32, 66, 82, 65, 67, 75, 69, 84, 69, 68, 32, 130, 4, 84, 202, + 151, 2, 73, 249, 70, 5, 75, 89, 79, 32, 84, 104, 242, 1, 71, 42, 74, 30, + 77, 34, 78, 70, 72, 34, 80, 34, 83, 234, 55, 82, 210, 147, 1, 79, 238, + 60, 69, 42, 76, 198, 2, 65, 154, 194, 1, 67, 2, 68, 2, 75, 2, 84, 2, 88, + 2, 90, 138, 69, 66, 2, 70, 2, 81, 2, 86, 186, 2, 73, 2, 85, 3, 89, 6, + 150, 148, 4, 72, 2, 74, 187, 2, 65, 4, 166, 150, 4, 65, 3, 89, 4, 210, + 147, 4, 66, 187, 2, 65, 14, 66, 71, 142, 201, 2, 74, 222, 132, 1, 88, + 138, 69, 68, 187, 2, 65, 4, 238, 146, 4, 74, 187, 2, 65, 4, 206, 146, 4, + 83, 187, 2, 65, 10, 54, 72, 150, 200, 2, 75, 230, 201, 1, 84, 187, 2, 65, + 4, 246, 145, 4, 84, 187, 2, 65, 108, 92, 7, 76, 69, 84, 84, 69, 82, 32, + 228, 2, 5, 83, 73, 71, 78, 32, 58, 85, 143, 156, 2, 68, 82, 190, 1, 65, + 22, 68, 30, 78, 58, 82, 14, 84, 30, 66, 2, 67, 2, 71, 2, 74, 2, 75, 2, + 80, 226, 144, 4, 69, 2, 72, 2, 73, 2, 76, 2, 77, 2, 79, 2, 83, 2, 85, 2, + 86, 2, 88, 3, 89, 5, 239, 145, 4, 65, 9, 122, 68, 227, 144, 4, 72, 11, + 34, 78, 158, 145, 4, 71, 3, 89, 5, 155, 145, 4, 89, 7, 39, 82, 9, 26, 84, + 227, 144, 4, 72, 5, 223, 144, 4, 72, 4, 176, 238, 3, 2, 83, 69, 177, 25, + 4, 72, 69, 67, 65, 2, 205, 205, 3, 2, 78, 71, 2, 239, 28, 65, 2, 11, 71, + 2, 135, 249, 3, 85, 6, 32, 2, 84, 72, 207, 128, 3, 76, 5, 11, 66, 2, 11, + 82, 2, 131, 164, 2, 85, 72, 252, 1, 4, 65, 82, 67, 32, 162, 2, 67, 44, 2, + 72, 65, 166, 3, 80, 140, 3, 13, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, + 76, 79, 87, 84, 5, 76, 69, 70, 84, 32, 228, 1, 6, 82, 73, 71, 72, 84, 32, + 170, 2, 83, 38, 84, 81, 7, 87, 73, 84, 72, 32, 85, 80, 6, 180, 1, 19, 65, + 78, 84, 73, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, 87, + 73, 21, 67, 76, 79, 67, 75, 87, 73, 83, 69, 32, 65, 82, 82, 79, 87, 32, + 87, 73, 84, 72, 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 80, 2, + 167, 130, 2, 76, 2, 11, 77, 2, 251, 129, 2, 73, 2, 11, 85, 2, 217, 183, + 3, 3, 82, 76, 89, 34, 36, 3, 76, 70, 32, 135, 138, 4, 84, 32, 70, 70, + 186, 1, 76, 22, 82, 178, 2, 83, 250, 5, 66, 211, 245, 1, 73, 8, 136, 1, + 15, 79, 82, 87, 65, 82, 68, 45, 70, 65, 67, 73, 78, 71, 32, 82, 157, 2, + 13, 76, 65, 73, 76, 73, 78, 71, 32, 82, 79, 66, 79, 84, 4, 22, 85, 243, + 1, 79, 2, 159, 202, 3, 78, 8, 41, 2, 69, 70, 8, 21, 3, 73, 71, 72, 8, 11, + 84, 8, 54, 32, 69, 9, 45, 70, 65, 67, 73, 78, 71, 32, 82, 2, 11, 80, 2, + 33, 6, 65, 82, 69, 78, 84, 72, 2, 207, 223, 1, 69, 6, 38, 79, 21, 5, 85, + 78, 78, 69, 82, 2, 239, 181, 3, 66, 4, 29, 5, 32, 70, 82, 65, 77, 4, 11, + 69, 4, 11, 45, 4, 218, 133, 4, 49, 3, 50, 4, 18, 69, 59, 84, 2, 11, 67, + 2, 11, 84, 2, 11, 73, 2, 255, 136, 2, 79, 2, 17, 2, 65, 78, 2, 11, 68, 2, + 21, 3, 73, 78, 71, 2, 17, 2, 32, 80, 2, 11, 69, 2, 11, 82, 2, 163, 180, + 3, 83, 4, 17, 2, 69, 82, 4, 33, 6, 32, 72, 65, 76, 70, 32, 4, 250, 3, 66, + 139, 106, 87, 10, 180, 1, 13, 74, 85, 83, 84, 73, 70, 73, 69, 68, 32, 76, + 79, 87, 126, 67, 50, 72, 161, 217, 2, 21, 66, 76, 65, 67, 75, 32, 76, 69, + 70, 84, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 83, 2, 237, 1, 7, 69, 82, + 32, 82, 73, 71, 72, 8, 78, 67, 50, 72, 33, 13, 74, 85, 83, 84, 73, 70, + 73, 69, 68, 32, 76, 79, 87, 4, 26, 79, 199, 135, 3, 82, 2, 223, 194, 3, + 82, 2, 209, 173, 3, 3, 65, 76, 70, 2, 33, 6, 69, 82, 32, 76, 69, 70, 2, + 53, 11, 84, 32, 81, 85, 65, 82, 84, 69, 82, 32, 66, 2, 11, 76, 2, 137, + 171, 2, 3, 65, 67, 75, 2, 157, 172, 3, 4, 81, 85, 65, 82, 2, 11, 79, 2, + 249, 171, 3, 12, 82, 84, 79, 73, 83, 69, 32, 83, 72, 69, 76, 76, 2, 29, + 5, 87, 65, 82, 68, 83, 2, 17, 2, 32, 65, 2, 237, 128, 3, 4, 82, 82, 79, + 87, 20, 188, 1, 22, 67, 74, 75, 32, 85, 78, 73, 70, 73, 69, 68, 32, 73, + 68, 69, 79, 71, 82, 65, 80, 72, 45, 161, 2, 19, 76, 65, 84, 73, 78, 32, + 67, 65, 80, 73, 84, 65, 76, 32, 76, 69, 84, 84, 69, 18, 40, 2, 52, 69, + 34, 53, 54, 54, 87, 55, 4, 202, 1, 48, 139, 208, 2, 56, 4, 30, 50, 141, + 1, 2, 66, 56, 2, 131, 181, 3, 68, 6, 48, 2, 50, 53, 22, 53, 185, 208, 2, + 2, 55, 50, 2, 219, 250, 3, 51, 2, 67, 53, 4, 32, 2, 48, 66, 21, 2, 54, + 68, 2, 151, 250, 3, 57, 2, 131, 250, 3, 55, 2, 11, 82, 2, 135, 188, 3, + 32, 64, 48, 6, 65, 76, 32, 82, 85, 78, 21, 2, 79, 32, 2, 203, 255, 2, 79, + 62, 72, 7, 76, 69, 84, 84, 69, 82, 32, 217, 2, 6, 83, 73, 71, 78, 32, 82, + 60, 206, 1, 66, 106, 78, 138, 31, 73, 206, 141, 1, 69, 138, 76, 67, 138, + 189, 1, 65, 234, 61, 68, 2, 71, 2, 72, 2, 74, 2, 75, 2, 76, 2, 77, 2, 80, + 2, 82, 2, 83, 2, 84, 2, 87, 2, 89, 186, 2, 79, 3, 85, 12, 52, 7, 82, 69, + 65, 84, 72, 89, 32, 183, 246, 3, 65, 10, 182, 31, 73, 206, 141, 1, 69, + 223, 178, 2, 65, 4, 202, 243, 3, 71, 187, 2, 65, 2, 17, 2, 73, 83, 2, 21, + 3, 73, 78, 71, 2, 11, 32, 2, 211, 200, 2, 84, 104, 54, 65, 254, 1, 69, + 86, 73, 246, 11, 79, 231, 1, 85, 16, 66, 67, 36, 2, 68, 69, 34, 77, 20, + 2, 78, 83, 231, 163, 3, 73, 4, 218, 233, 1, 75, 219, 209, 1, 84, 2, 133, + 17, 4, 32, 77, 65, 82, 5, 243, 212, 1, 32, 4, 44, 2, 80, 76, 33, 5, 86, + 69, 82, 83, 65, 2, 11, 85, 2, 147, 212, 3, 84, 2, 163, 63, 76, 4, 52, 7, + 65, 83, 85, 82, 69, 32, 67, 235, 220, 2, 69, 2, 11, 72, 2, 171, 4, 69, + 66, 156, 1, 3, 65, 78, 71, 176, 3, 8, 68, 69, 78, 84, 32, 69, 77, 66, 20, + 9, 71, 82, 65, 77, 32, 70, 79, 82, 32, 140, 2, 4, 80, 76, 69, 32, 159, + 155, 3, 67, 16, 48, 2, 76, 69, 245, 1, 5, 85, 76, 65, 82, 32, 10, 44, 6, + 32, 87, 73, 84, 72, 32, 163, 1, 45, 8, 80, 9, 83, 69, 82, 73, 70, 83, 32, + 65, 84, 54, 85, 238, 169, 2, 82, 207, 68, 68, 2, 17, 2, 32, 66, 2, 189, + 222, 3, 3, 79, 84, 84, 2, 149, 61, 2, 78, 68, 2, 193, 204, 2, 4, 72, 69, + 65, 68, 6, 68, 9, 70, 76, 65, 71, 32, 79, 78, 32, 80, 34, 82, 195, 139, + 2, 66, 2, 11, 79, 2, 199, 209, 3, 83, 2, 11, 85, 2, 231, 175, 3, 76, 2, + 147, 219, 3, 76, 16, 66, 69, 34, 72, 34, 76, 22, 77, 46, 84, 42, 87, 155, + 211, 1, 70, 2, 11, 65, 2, 227, 231, 1, 82, 2, 11, 69, 2, 227, 249, 1, 65, + 2, 207, 188, 3, 65, 2, 21, 3, 79, 85, 78, 2, 147, 189, 2, 84, 2, 17, 2, + 72, 85, 2, 251, 199, 1, 78, 4, 178, 177, 2, 65, 251, 41, 73, 30, 176, 2, + 3, 67, 79, 76, 24, 20, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 66, + 65, 82, 32, 87, 73, 84, 72, 32, 98, 80, 34, 84, 20, 13, 86, 69, 82, 84, + 73, 67, 65, 76, 32, 66, 65, 82, 32, 44, 9, 83, 79, 76, 73, 68, 85, 83, + 32, 66, 224, 21, 2, 68, 65, 160, 187, 1, 6, 78, 69, 83, 84, 69, 68, 159, + 12, 73, 2, 165, 37, 2, 79, 78, 4, 34, 68, 33, 4, 84, 82, 73, 80, 2, 17, + 2, 79, 85, 2, 11, 66, 2, 177, 183, 3, 2, 76, 69, 4, 142, 223, 1, 76, 235, + 110, 82, 2, 195, 202, 1, 73, 8, 42, 66, 74, 68, 170, 189, 1, 82, 115, 87, + 2, 29, 5, 73, 78, 65, 82, 89, 2, 21, 3, 32, 82, 69, 2, 143, 33, 76, 2, + 25, 4, 69, 76, 73, 77, 2, 163, 172, 2, 73, 12, 32, 2, 76, 76, 42, 77, 23, + 80, 5, 17, 2, 69, 89, 2, 255, 220, 1, 66, 2, 247, 184, 2, 66, 6, 44, 5, + 73, 67, 65, 76, 32, 199, 211, 3, 72, 4, 18, 68, 47, 70, 2, 11, 82, 2, 11, + 73, 2, 199, 225, 3, 78, 2, 195, 249, 1, 73, 6, 18, 69, 79, 77, 5, 213, + 191, 3, 14, 32, 76, 73, 71, 72, 84, 32, 77, 79, 79, 78, 32, 65, 82, 2, + 191, 174, 3, 80, 216, 1, 76, 3, 71, 82, 73, 22, 76, 214, 12, 82, 173, + 188, 2, 5, 77, 66, 76, 69, 82, 2, 239, 219, 2, 75, 162, 1, 68, 11, 85, + 45, 84, 73, 71, 65, 76, 65, 82, 73, 32, 135, 226, 3, 73, 160, 1, 122, 65, + 42, 67, 30, 68, 106, 71, 32, 7, 76, 69, 84, 84, 69, 82, 32, 178, 3, 82, + 32, 5, 83, 73, 71, 78, 32, 131, 3, 86, 2, 11, 85, 2, 133, 159, 3, 2, 32, + 76, 2, 201, 162, 3, 2, 79, 78, 4, 18, 79, 67, 65, 2, 11, 85, 2, 11, 66, + 2, 25, 4, 76, 69, 32, 68, 2, 11, 65, 2, 243, 186, 3, 78, 2, 213, 218, 2, + 3, 69, 77, 73, 100, 206, 1, 65, 38, 68, 46, 76, 38, 82, 34, 84, 46, 86, + 186, 5, 85, 186, 202, 1, 73, 138, 196, 1, 78, 46, 83, 82, 66, 2, 67, 2, + 71, 2, 74, 2, 75, 2, 80, 206, 40, 79, 162, 8, 69, 158, 20, 72, 2, 77, 3, + 89, 9, 242, 221, 3, 65, 2, 73, 3, 85, 8, 142, 150, 3, 68, 138, 69, 72, + 187, 2, 65, 6, 134, 209, 1, 76, 159, 140, 2, 65, 4, 198, 218, 3, 82, 187, + 2, 65, 8, 158, 149, 3, 84, 138, 69, 72, 187, 2, 65, 10, 214, 5, 79, 223, + 214, 3, 65, 2, 11, 69, 2, 223, 222, 1, 80, 18, 174, 1, 65, 72, 6, 76, 79, + 79, 80, 69, 68, 40, 2, 79, 77, 0, 5, 83, 72, 82, 73, 73, 48, 13, 67, 65, + 78, 68, 82, 65, 32, 65, 78, 85, 78, 65, 83, 22, 80, 135, 150, 3, 86, 4, + 26, 86, 191, 149, 3, 78, 2, 21, 3, 65, 71, 82, 2, 231, 220, 1, 65, 2, 17, + 2, 32, 86, 2, 203, 157, 1, 73, 2, 17, 2, 32, 80, 2, 25, 4, 85, 83, 72, + 80, 2, 187, 208, 3, 73, 2, 193, 180, 3, 2, 76, 85, 30, 56, 10, 69, 68, + 73, 67, 32, 84, 79, 78, 69, 32, 35, 79, 4, 226, 164, 1, 65, 235, 4, 83, + 26, 45, 9, 87, 69, 76, 32, 83, 73, 71, 78, 32, 26, 70, 65, 38, 85, 22, + 86, 166, 202, 1, 73, 210, 237, 1, 79, 163, 8, 69, 6, 158, 215, 3, 65, 2, + 73, 3, 85, 5, 251, 214, 3, 85, 8, 11, 79, 8, 33, 6, 67, 65, 76, 73, 67, + 32, 8, 26, 82, 243, 217, 2, 76, 5, 155, 214, 3, 82, 50, 50, 75, 84, 4, + 78, 69, 68, 32, 199, 174, 2, 84, 4, 48, 6, 73, 83, 72, 32, 76, 73, 219, + 195, 3, 69, 2, 11, 82, 2, 147, 206, 2, 65, 44, 238, 1, 65, 80, 6, 66, 76, + 65, 67, 75, 32, 40, 7, 87, 72, 73, 84, 69, 32, 83, 22, 67, 50, 68, 96, 2, + 78, 79, 32, 2, 79, 75, 30, 83, 237, 171, 3, 21, 71, 82, 69, 69, 75, 32, + 83, 77, 65, 76, 76, 32, 76, 69, 84, 84, 69, 82, 32, 73, 79, 4, 26, 77, + 175, 172, 2, 78, 2, 17, 2, 80, 69, 2, 11, 82, 2, 179, 194, 2, 83, 4, 18, + 80, 23, 83, 2, 211, 151, 2, 69, 2, 195, 153, 2, 72, 4, 144, 3, 5, 65, 80, + 73, 84, 65, 171, 121, 79, 6, 30, 65, 33, 3, 73, 71, 73, 2, 11, 71, 2, + 187, 147, 3, 71, 4, 201, 222, 1, 3, 84, 32, 84, 6, 162, 2, 82, 211, 199, + 2, 84, 2, 249, 228, 1, 2, 32, 72, 14, 128, 1, 18, 65, 78, 83, 45, 83, 69, + 82, 73, 70, 32, 67, 65, 80, 73, 84, 65, 76, 32, 38, 69, 32, 3, 77, 65, + 76, 33, 2, 79, 85, 6, 178, 207, 3, 71, 2, 76, 3, 89, 2, 11, 77, 2, 243, + 254, 2, 73, 2, 11, 76, 2, 191, 205, 2, 32, 4, 21, 3, 84, 72, 32, 4, 32, + 2, 69, 65, 1, 2, 87, 69, 2, 53, 11, 83, 84, 32, 80, 79, 73, 78, 84, 73, + 78, 71, 2, 17, 2, 32, 76, 2, 151, 204, 2, 69, 38, 58, 69, 52, 8, 73, 83, + 84, 69, 68, 32, 82, 73, 63, 79, 2, 17, 2, 76, 86, 2, 145, 178, 1, 3, 69, + 32, 80, 2, 17, 2, 71, 72, 2, 141, 109, 6, 84, 87, 65, 82, 68, 83, 34, 30, + 32, 229, 10, 2, 45, 69, 32, 134, 3, 66, 86, 67, 116, 3, 68, 79, 84, 226, + 1, 72, 44, 14, 73, 78, 84, 69, 82, 83, 69, 67, 84, 73, 78, 71, 32, 76, + 92, 6, 74, 79, 73, 78, 69, 68, 64, 8, 76, 79, 71, 73, 67, 65, 76, 32, 98, + 77, 0, 3, 87, 79, 77, 116, 13, 82, 73, 78, 71, 83, 32, 65, 76, 73, 71, + 78, 69, 68, 48, 24, 65, 83, 84, 69, 82, 73, 83, 75, 83, 32, 65, 76, 73, + 71, 78, 69, 68, 32, 86, 69, 82, 84, 73, 67, 35, 83, 2, 29, 5, 85, 84, 84, + 79, 78, 2, 17, 2, 32, 77, 2, 11, 79, 2, 251, 188, 2, 85, 2, 93, 21, 79, + 78, 83, 69, 67, 85, 84, 73, 86, 69, 32, 69, 81, 85, 65, 76, 83, 32, 83, + 73, 71, 2, 207, 137, 3, 78, 6, 18, 32, 55, 83, 4, 22, 76, 131, 1, 80, 2, + 145, 163, 1, 2, 69, 65, 2, 53, 11, 32, 79, 86, 69, 82, 32, 79, 78, 69, + 32, 68, 2, 11, 79, 2, 11, 84, 2, 17, 2, 32, 80, 2, 29, 5, 85, 78, 67, 84, + 85, 2, 251, 244, 2, 65, 2, 11, 69, 2, 11, 65, 2, 227, 188, 2, 82, 4, 17, + 2, 79, 71, 4, 25, 4, 73, 67, 65, 76, 4, 11, 32, 4, 186, 180, 2, 65, 187, + 87, 79, 2, 17, 2, 32, 83, 2, 21, 3, 81, 85, 65, 2, 255, 173, 2, 82, 4, + 30, 79, 13, 3, 65, 78, 68, 2, 11, 82, 2, 11, 32, 2, 11, 79, 2, 11, 80, 2, + 143, 18, 69, 2, 11, 69, 2, 45, 9, 78, 32, 72, 79, 76, 68, 73, 78, 71, 2, + 11, 32, 2, 11, 72, 2, 11, 65, 2, 251, 223, 1, 78, 2, 45, 9, 32, 72, 79, + 82, 73, 90, 79, 78, 84, 2, 11, 65, 2, 175, 158, 3, 76, 2, 49, 10, 80, 69, + 69, 67, 72, 32, 66, 85, 66, 66, 2, 147, 171, 2, 76, 2, 11, 77, 2, 225, + 213, 1, 2, 32, 68, 248, 3, 140, 1, 8, 71, 65, 82, 73, 84, 73, 67, 32, + 152, 6, 7, 77, 66, 82, 69, 76, 76, 65, 166, 1, 78, 158, 8, 80, 246, 166, + 1, 82, 255, 136, 2, 83, 62, 48, 7, 76, 69, 84, 84, 69, 82, 32, 159, 5, + 87, 60, 206, 1, 65, 28, 2, 81, 79, 22, 66, 22, 68, 50, 71, 44, 2, 72, 79, + 22, 75, 34, 76, 32, 2, 82, 65, 22, 83, 74, 84, 74, 89, 22, 90, 234, 185, + 2, 78, 202, 91, 80, 246, 5, 87, 202, 12, 77, 174, 18, 73, 3, 85, 4, 26, + 76, 203, 237, 2, 73, 2, 239, 186, 3, 80, 2, 199, 152, 3, 69, 4, 26, 69, + 139, 177, 2, 72, 2, 151, 152, 3, 76, 4, 230, 141, 2, 72, 209, 140, 1, 2, + 65, 77, 5, 235, 185, 3, 84, 4, 234, 186, 2, 65, 239, 126, 72, 2, 11, 65, + 2, 223, 150, 3, 77, 2, 167, 190, 1, 83, 8, 38, 65, 194, 187, 2, 72, 239, + 90, 83, 4, 190, 164, 3, 68, 239, 13, 77, 6, 38, 72, 162, 158, 3, 69, 175, + 28, 79, 2, 11, 65, 2, 167, 252, 1, 78, 2, 139, 244, 2, 79, 4, 202, 149, + 3, 69, 207, 36, 85, 2, 21, 3, 79, 82, 68, 2, 25, 4, 32, 68, 73, 86, 2, + 235, 149, 1, 73, 7, 11, 32, 4, 84, 4, 79, 78, 32, 71, 45, 13, 87, 73, 84, + 72, 32, 82, 65, 73, 78, 32, 68, 82, 79, 2, 11, 82, 2, 11, 79, 2, 159, + 168, 2, 85, 2, 195, 250, 2, 80, 32, 160, 1, 3, 65, 77, 85, 20, 7, 67, 69, + 82, 84, 65, 73, 78, 34, 68, 62, 73, 245, 5, 18, 77, 65, 82, 82, 73, 69, + 68, 32, 80, 65, 82, 84, 78, 69, 82, 83, 72, 73, 2, 243, 163, 2, 83, 2, + 11, 84, 2, 199, 175, 2, 89, 4, 26, 69, 215, 160, 2, 79, 2, 11, 82, 2, + 219, 154, 3, 84, 22, 60, 2, 79, 78, 234, 3, 84, 106, 86, 177, 58, 3, 67, + 79, 82, 15, 11, 32, 12, 160, 1, 6, 65, 66, 79, 86, 69, 32, 108, 22, 66, + 69, 83, 73, 68, 69, 32, 65, 78, 68, 32, 74, 79, 73, 78, 69, 68, 32, 87, + 73, 84, 72, 41, 5, 87, 73, 84, 72, 32, 4, 52, 9, 66, 65, 82, 32, 65, 66, + 79, 86, 69, 23, 73, 2, 17, 2, 32, 73, 2, 161, 49, 4, 78, 84, 69, 82, 2, + 17, 2, 32, 85, 2, 195, 226, 2, 78, 6, 66, 77, 58, 79, 145, 249, 2, 8, 76, + 79, 71, 73, 67, 65, 76, 32, 2, 11, 73, 2, 11, 78, 2, 11, 85, 2, 139, 171, + 2, 83, 2, 11, 86, 2, 161, 142, 2, 2, 69, 82, 4, 18, 32, 67, 69, 2, 11, + 83, 2, 17, 2, 69, 80, 2, 11, 65, 2, 143, 248, 2, 82, 2, 183, 155, 2, 68, + 2, 57, 12, 69, 82, 83, 65, 76, 32, 82, 69, 67, 89, 67, 76, 2, 17, 2, 73, + 78, 2, 211, 154, 2, 71, 2, 191, 154, 2, 80, 144, 3, 130, 1, 32, 246, 7, + 45, 180, 8, 4, 80, 69, 82, 32, 136, 40, 8, 83, 73, 68, 69, 45, 68, 79, + 87, 21, 6, 87, 65, 82, 68, 83, 32, 38, 140, 1, 5, 65, 82, 82, 79, 87, + 224, 1, 5, 66, 65, 82, 66, 32, 228, 1, 5, 68, 79, 87, 78, 32, 210, 1, 70, + 30, 82, 93, 4, 84, 65, 67, 75, 8, 64, 4, 72, 69, 65, 68, 141, 51, 7, 32, + 84, 72, 82, 79, 85, 71, 7, 11, 32, 4, 112, 22, 66, 69, 84, 87, 69, 69, + 78, 32, 84, 87, 79, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 227, 156, + 2, 73, 2, 145, 149, 2, 2, 32, 66, 8, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, + 72, 4, 57, 12, 84, 32, 68, 79, 87, 78, 32, 66, 65, 82, 66, 32, 4, 44, 3, + 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 11, 84, 2, 25, 4, 32, 72, 65, 82, 2, + 11, 80, 2, 187, 218, 2, 79, 14, 76, 5, 65, 82, 82, 79, 87, 62, 66, 38, + 68, 18, 83, 250, 58, 87, 219, 9, 84, 5, 37, 7, 32, 87, 73, 84, 72, 32, + 66, 2, 247, 157, 2, 65, 2, 173, 140, 2, 4, 76, 65, 67, 75, 2, 195, 59, + 79, 2, 183, 65, 65, 2, 11, 73, 2, 239, 112, 83, 2, 53, 11, 73, 71, 72, + 84, 32, 68, 73, 65, 71, 79, 78, 2, 189, 128, 1, 4, 65, 76, 32, 69, 5, 29, + 5, 32, 87, 73, 84, 72, 2, 33, 6, 32, 67, 73, 82, 67, 76, 2, 131, 170, 2, + 69, 32, 58, 70, 225, 1, 9, 80, 79, 73, 78, 84, 73, 78, 71, 32, 4, 41, 8, + 65, 67, 73, 78, 71, 32, 83, 78, 4, 65, 14, 65, 75, 69, 32, 72, 69, 65, + 68, 32, 87, 73, 84, 72, 32, 4, 42, 79, 25, 6, 67, 76, 79, 83, 69, 68, 2, + 21, 3, 80, 69, 78, 2, 21, 3, 32, 77, 79, 2, 243, 159, 1, 85, 28, 130, 1, + 65, 86, 69, 62, 70, 20, 8, 77, 73, 76, 73, 84, 65, 82, 89, 26, 82, 88, 6, + 83, 77, 65, 76, 76, 32, 118, 84, 255, 121, 71, 4, 22, 84, 159, 2, 73, 2, + 37, 7, 79, 77, 73, 67, 32, 66, 79, 2, 155, 229, 1, 77, 2, 29, 5, 78, 69, + 82, 71, 89, 2, 193, 180, 1, 2, 32, 87, 2, 227, 161, 3, 82, 2, 129, 1, 2, + 32, 65, 8, 64, 5, 79, 67, 75, 69, 84, 118, 69, 214, 129, 1, 65, 235, 29, + 73, 2, 195, 204, 2, 32, 4, 18, 65, 67, 82, 2, 11, 73, 2, 11, 82, 2, 17, + 2, 80, 76, 2, 207, 133, 2, 65, 2, 11, 69, 2, 215, 249, 1, 68, 4, 37, 7, + 82, 73, 65, 78, 71, 76, 69, 4, 11, 32, 4, 11, 87, 4, 25, 4, 73, 84, 72, + 32, 4, 18, 76, 27, 82, 2, 11, 69, 2, 35, 70, 2, 21, 3, 73, 71, 72, 2, 11, + 84, 2, 17, 2, 32, 72, 2, 21, 3, 65, 76, 70, 2, 17, 2, 32, 66, 2, 11, 76, + 2, 207, 236, 1, 65, 200, 1, 192, 1, 4, 65, 78, 68, 32, 210, 2, 66, 74, + 67, 74, 70, 36, 5, 72, 65, 76, 70, 32, 200, 4, 5, 76, 69, 70, 84, 32, + 254, 5, 77, 150, 2, 79, 60, 6, 82, 73, 71, 72, 84, 32, 238, 17, 83, 67, + 84, 8, 34, 76, 53, 4, 82, 73, 71, 72, 6, 48, 2, 69, 70, 173, 1, 5, 79, + 87, 69, 82, 32, 2, 53, 11, 84, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 2, + 205, 35, 25, 32, 84, 82, 73, 65, 78, 71, 85, 76, 65, 82, 32, 84, 72, 82, + 69, 69, 32, 81, 85, 65, 82, 84, 69, 82, 4, 28, 2, 84, 82, 235, 39, 79, 2, + 225, 4, 7, 73, 65, 78, 71, 85, 76, 65, 2, 17, 2, 76, 65, 2, 17, 2, 68, + 69, 2, 249, 224, 1, 3, 32, 83, 67, 10, 33, 6, 69, 78, 84, 82, 69, 32, 10, + 174, 12, 76, 22, 82, 255, 21, 79, 2, 17, 2, 73, 86, 2, 183, 31, 69, 20, + 140, 1, 5, 66, 76, 79, 67, 75, 110, 72, 32, 8, 73, 78, 86, 69, 82, 83, + 69, 32, 198, 1, 86, 174, 26, 77, 130, 3, 76, 22, 82, 191, 163, 1, 67, 5, + 229, 28, 23, 32, 65, 78, 68, 32, 76, 79, 87, 69, 82, 32, 72, 65, 76, 70, + 32, 73, 78, 86, 69, 82, 83, 69, 2, 225, 18, 4, 69, 65, 86, 89, 4, 100, + 21, 77, 69, 68, 73, 85, 77, 32, 83, 72, 65, 68, 69, 32, 65, 78, 68, 32, + 76, 79, 87, 69, 51, 87, 2, 11, 82, 2, 189, 35, 5, 32, 72, 65, 76, 70, 2, + 21, 3, 72, 73, 84, 2, 247, 192, 1, 69, 2, 29, 5, 69, 82, 84, 73, 67, 2, + 189, 115, 14, 65, 76, 32, 76, 73, 78, 69, 32, 87, 73, 84, 72, 32, 84, 62, + 50, 66, 238, 3, 67, 54, 79, 74, 84, 183, 13, 81, 22, 65, 14, 76, 79, 67, + 75, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 22, 144, 1, 6, 76, 79, 87, + 69, 82, 32, 233, 8, 24, 85, 80, 80, 69, 82, 32, 77, 73, 68, 68, 76, 69, + 32, 76, 69, 70, 84, 32, 84, 79, 32, 85, 80, 80, 18, 176, 1, 10, 67, 69, + 78, 84, 82, 69, 32, 84, 79, 32, 36, 8, 76, 69, 70, 84, 32, 84, 79, 32, + 229, 10, 18, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, + 85, 80, 80, 6, 70, 76, 253, 8, 3, 85, 80, 80, 6, 34, 76, 213, 9, 3, 85, + 80, 80, 2, 149, 10, 2, 79, 87, 2, 29, 5, 79, 82, 78, 69, 82, 2, 219, 233, + 1, 32, 4, 182, 12, 78, 53, 12, 82, 32, 76, 79, 87, 69, 82, 32, 82, 73, + 71, 72, 8, 34, 79, 170, 19, 82, 155, 1, 87, 2, 241, 18, 11, 32, 76, 79, + 87, 69, 82, 32, 82, 73, 71, 72, 12, 33, 6, 73, 68, 68, 76, 69, 32, 12, + 52, 7, 67, 69, 78, 84, 82, 69, 32, 74, 76, 23, 82, 4, 44, 3, 76, 69, 70, + 1, 4, 82, 73, 71, 72, 2, 117, 3, 84, 32, 79, 4, 41, 2, 69, 70, 4, 21, 3, + 73, 71, 72, 4, 17, 2, 84, 32, 4, 30, 79, 129, 18, 2, 84, 87, 2, 139, 9, + 78, 4, 11, 78, 4, 17, 2, 69, 32, 4, 154, 21, 81, 147, 4, 69, 68, 84, 2, + 66, 76, 186, 6, 68, 114, 79, 222, 1, 80, 42, 81, 224, 4, 2, 83, 72, 51, + 84, 22, 61, 13, 79, 67, 75, 32, 68, 73, 65, 71, 79, 78, 65, 76, 32, 22, + 140, 1, 24, 76, 79, 87, 69, 82, 32, 77, 73, 68, 68, 76, 69, 32, 76, 69, + 70, 84, 32, 84, 79, 32, 76, 79, 87, 57, 6, 85, 80, 80, 69, 82, 32, 4, 21, + 3, 69, 82, 32, 4, 246, 3, 67, 219, 218, 2, 82, 18, 176, 1, 10, 67, 69, + 78, 84, 82, 69, 32, 84, 79, 32, 92, 8, 76, 69, 70, 84, 32, 84, 79, 32, + 141, 1, 18, 77, 73, 68, 68, 76, 69, 32, 76, 69, 70, 84, 32, 84, 79, 32, + 76, 79, 87, 6, 32, 3, 76, 79, 87, 139, 1, 85, 4, 21, 3, 69, 82, 32, 4, + 142, 2, 77, 179, 218, 2, 82, 6, 28, 3, 76, 79, 87, 51, 85, 4, 21, 3, 69, + 82, 32, 4, 142, 1, 67, 43, 77, 2, 17, 2, 80, 80, 2, 17, 2, 69, 82, 2, + 117, 2, 32, 77, 6, 21, 3, 69, 82, 32, 6, 34, 67, 42, 77, 179, 218, 2, 82, + 2, 11, 69, 2, 221, 226, 1, 2, 78, 84, 2, 25, 4, 73, 68, 68, 76, 2, 195, + 177, 1, 69, 2, 57, 12, 82, 79, 80, 45, 83, 72, 65, 68, 79, 87, 69, 68, 2, + 17, 2, 32, 87, 2, 149, 192, 1, 3, 72, 73, 84, 4, 62, 78, 53, 11, 82, 32, + 76, 79, 87, 69, 82, 32, 76, 69, 70, 2, 229, 16, 9, 69, 32, 83, 73, 88, + 84, 69, 69, 78, 2, 69, 15, 84, 32, 67, 85, 82, 76, 89, 32, 66, 82, 65, + 67, 75, 69, 84, 2, 11, 32, 2, 175, 177, 2, 83, 2, 11, 69, 2, 165, 162, 1, + 2, 78, 67, 26, 17, 2, 85, 65, 26, 44, 6, 68, 82, 65, 78, 84, 32, 255, 3, + 82, 24, 90, 67, 112, 10, 70, 65, 67, 69, 32, 87, 73, 84, 72, 32, 114, 77, + 76, 2, 83, 84, 67, 84, 14, 68, 7, 73, 82, 67, 85, 76, 65, 82, 197, 168, + 1, 4, 72, 69, 83, 83, 2, 17, 2, 32, 65, 2, 207, 213, 1, 82, 4, 34, 67, + 45, 4, 79, 80, 69, 78, 2, 21, 3, 76, 79, 83, 2, 17, 2, 69, 68, 2, 245, + 232, 1, 3, 32, 69, 89, 2, 25, 4, 73, 67, 82, 79, 2, 11, 67, 2, 165, 196, + 1, 4, 79, 77, 80, 85, 2, 41, 8, 65, 78, 68, 73, 78, 71, 32, 75, 2, 147, + 211, 2, 78, 2, 21, 3, 69, 76, 69, 2, 11, 86, 2, 183, 90, 73, 2, 233, 168, + 1, 3, 84, 69, 82, 4, 141, 102, 8, 65, 68, 79, 87, 69, 68, 32, 87, 8, 30, + 79, 106, 82, 155, 1, 87, 2, 49, 10, 32, 76, 79, 87, 69, 82, 32, 76, 69, + 70, 2, 11, 84, 2, 11, 32, 2, 11, 70, 2, 171, 113, 73, 4, 25, 4, 73, 65, + 78, 71, 4, 40, 4, 85, 76, 65, 82, 171, 228, 2, 76, 2, 17, 2, 32, 77, 2, + 41, 8, 69, 68, 73, 85, 77, 32, 83, 72, 2, 199, 103, 65, 2, 21, 3, 69, 76, + 70, 2, 11, 84, 2, 215, 165, 1, 72, 2, 17, 2, 69, 86, 2, 17, 2, 69, 78, 2, + 145, 1, 2, 32, 69, 10, 64, 5, 72, 82, 69, 69, 32, 193, 1, 6, 82, 73, 65, + 78, 71, 85, 8, 54, 69, 49, 9, 81, 85, 65, 82, 84, 69, 82, 83, 32, 2, 29, + 5, 73, 71, 72, 84, 72, 2, 243, 5, 83, 6, 30, 76, 22, 82, 203, 5, 66, 2, + 41, 2, 69, 70, 2, 21, 3, 73, 71, 72, 2, 43, 84, 2, 17, 2, 76, 65, 2, 11, + 82, 2, 17, 2, 32, 79, 2, 25, 4, 78, 69, 32, 81, 2, 185, 4, 6, 85, 65, 82, + 84, 69, 82, 2, 151, 185, 2, 78, 128, 1, 154, 1, 65, 184, 6, 2, 66, 76, + 150, 1, 68, 50, 70, 82, 72, 150, 4, 67, 46, 81, 42, 82, 22, 83, 102, 84, + 146, 7, 80, 173, 3, 6, 87, 72, 73, 84, 69, 32, 32, 34, 78, 33, 4, 82, 82, + 79, 87, 2, 11, 67, 2, 147, 176, 2, 79, 31, 11, 32, 28, 134, 1, 65, 192, + 1, 14, 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 68, 32, 5, 87, + 73, 84, 72, 32, 210, 8, 70, 179, 5, 84, 2, 41, 8, 78, 68, 32, 82, 73, 71, + 72, 84, 2, 17, 2, 32, 79, 2, 25, 4, 78, 69, 32, 69, 2, 21, 3, 73, 71, 72, + 2, 17, 2, 84, 72, 2, 11, 32, 2, 11, 66, 2, 11, 76, 2, 251, 191, 1, 79, 2, + 145, 208, 1, 3, 79, 87, 78, 20, 74, 68, 58, 76, 42, 77, 38, 78, 58, 83, + 66, 69, 250, 12, 84, 135, 58, 72, 2, 21, 3, 79, 85, 66, 2, 11, 76, 2, + 199, 192, 2, 69, 2, 11, 65, 2, 253, 22, 3, 82, 71, 69, 2, 225, 22, 5, 69, + 68, 73, 85, 77, 2, 25, 4, 79, 84, 67, 72, 2, 11, 69, 2, 151, 56, 68, 4, + 11, 77, 4, 25, 4, 65, 76, 76, 32, 4, 22, 69, 223, 21, 84, 2, 129, 22, 10, + 81, 85, 73, 76, 65, 84, 69, 82, 65, 76, 4, 25, 4, 65, 67, 75, 32, 4, 26, + 67, 215, 209, 1, 65, 2, 25, 4, 73, 82, 67, 76, 2, 25, 4, 69, 68, 32, 87, + 2, 11, 72, 2, 237, 14, 2, 73, 84, 4, 22, 79, 195, 13, 65, 2, 177, 14, 2, + 85, 66, 2, 11, 73, 2, 37, 7, 78, 71, 69, 82, 45, 80, 79, 2, 249, 207, 1, + 2, 83, 84, 20, 88, 17, 65, 82, 80, 79, 79, 78, 32, 87, 73, 84, 72, 32, + 66, 65, 82, 66, 32, 131, 3, 69, 16, 56, 4, 76, 69, 70, 84, 245, 1, 5, 82, + 73, 71, 72, 84, 10, 22, 32, 239, 9, 87, 8, 60, 7, 66, 69, 83, 73, 68, 69, + 32, 206, 1, 70, 179, 5, 84, 4, 40, 4, 68, 79, 87, 78, 1, 2, 85, 80, 2, + 249, 150, 1, 23, 87, 65, 82, 68, 83, 32, 72, 65, 82, 80, 79, 79, 78, 32, + 87, 73, 84, 72, 32, 66, 65, 82, 66, 6, 22, 32, 251, 7, 87, 4, 22, 70, + 179, 5, 84, 2, 165, 197, 1, 3, 82, 79, 77, 4, 25, 4, 65, 86, 89, 32, 4, + 26, 67, 231, 203, 1, 65, 2, 189, 10, 7, 79, 77, 80, 82, 69, 83, 83, 2, + 137, 9, 6, 85, 65, 68, 82, 85, 80, 2, 251, 148, 2, 79, 4, 30, 65, 53, 3, + 81, 85, 65, 2, 193, 202, 1, 8, 78, 83, 45, 83, 69, 82, 73, 70, 2, 131, 9, + 82, 36, 36, 2, 82, 73, 229, 7, 2, 87, 79, 30, 40, 5, 65, 78, 71, 76, 69, + 155, 7, 80, 28, 52, 8, 45, 72, 69, 65, 68, 69, 68, 32, 219, 12, 32, 26, + 48, 5, 65, 82, 82, 79, 87, 174, 5, 68, 39, 80, 23, 11, 32, 20, 92, 17, + 76, 69, 70, 84, 87, 65, 82, 68, 83, 32, 79, 70, 32, 68, 79, 87, 78, 98, + 84, 23, 87, 2, 37, 7, 87, 65, 82, 68, 83, 32, 84, 2, 37, 7, 82, 73, 65, + 78, 71, 76, 69, 2, 219, 5, 45, 2, 247, 191, 1, 79, 16, 25, 4, 73, 84, 72, + 32, 16, 114, 66, 28, 6, 76, 79, 78, 71, 32, 84, 130, 1, 77, 34, 78, 34, + 86, 34, 72, 145, 56, 6, 68, 79, 85, 66, 76, 69, 2, 149, 2, 3, 79, 76, 68, + 4, 17, 2, 73, 80, 4, 11, 32, 4, 34, 76, 21, 4, 82, 73, 71, 72, 2, 17, 2, + 69, 70, 2, 11, 84, 2, 11, 87, 2, 219, 126, 65, 2, 121, 5, 69, 68, 73, 85, + 77, 2, 89, 5, 65, 82, 82, 79, 87, 2, 29, 5, 69, 82, 89, 32, 72, 2, 25, 4, + 69, 65, 86, 89, 2, 173, 181, 2, 4, 32, 83, 72, 65, 2, 11, 65, 2, 249, 1, + 2, 83, 72, 2, 11, 65, 2, 25, 4, 73, 82, 69, 68, 2, 29, 5, 32, 65, 82, 82, + 79, 2, 163, 161, 2, 87, 2, 11, 76, 2, 135, 194, 1, 69, 6, 74, 32, 61, 14, + 45, 72, 69, 65, 68, 69, 68, 32, 65, 82, 82, 79, 87, 32, 2, 25, 4, 72, 69, + 65, 68, 2, 11, 69, 2, 255, 192, 1, 68, 4, 42, 87, 177, 136, 1, 4, 70, 82, + 79, 77, 2, 33, 6, 73, 84, 72, 32, 84, 82, 2, 61, 13, 73, 65, 78, 71, 76, + 69, 32, 65, 82, 82, 79, 87, 72, 2, 253, 121, 2, 69, 65, 18, 88, 5, 65, + 82, 82, 79, 87, 149, 3, 12, 68, 79, 85, 66, 76, 69, 32, 65, 82, 82, 79, + 87, 15, 11, 32, 12, 108, 11, 79, 78, 32, 80, 69, 68, 69, 83, 84, 65, 76, + 106, 87, 165, 181, 1, 8, 70, 82, 79, 77, 32, 66, 65, 82, 7, 33, 6, 32, + 87, 73, 84, 72, 32, 4, 26, 86, 247, 181, 1, 72, 2, 153, 182, 1, 5, 69, + 82, 84, 73, 67, 2, 29, 5, 73, 84, 72, 73, 78, 2, 17, 2, 32, 84, 2, 37, 7, + 82, 73, 65, 78, 71, 76, 69, 2, 11, 32, 2, 11, 65, 2, 25, 4, 82, 82, 79, + 87, 2, 11, 72, 2, 147, 146, 2, 69, 5, 45, 9, 32, 79, 78, 32, 80, 69, 68, + 69, 83, 2, 159, 204, 1, 84, 244, 15, 86, 65, 254, 16, 83, 174, 3, 69, + 194, 46, 73, 202, 8, 79, 134, 2, 84, 21, 2, 85, 76, 218, 8, 116, 2, 73, + 32, 128, 16, 17, 82, 73, 65, 84, 73, 79, 78, 32, 83, 69, 76, 69, 67, 84, + 79, 82, 45, 233, 45, 2, 77, 80, 216, 4, 54, 67, 34, 70, 82, 81, 40, 2, + 83, 89, 183, 96, 68, 2, 11, 79, 2, 183, 180, 2, 77, 2, 11, 85, 2, 11, 76, + 2, 11, 76, 2, 11, 32, 2, 11, 83, 2, 227, 219, 1, 84, 2, 17, 2, 85, 69, 2, + 167, 235, 1, 83, 190, 4, 68, 7, 76, 76, 65, 66, 76, 69, 32, 197, 11, 5, + 77, 66, 79, 76, 32, 164, 4, 214, 1, 68, 70, 66, 2, 83, 2, 84, 2, 90, 70, + 71, 122, 72, 82, 75, 134, 1, 76, 130, 1, 77, 86, 78, 134, 3, 67, 2, 70, + 2, 74, 2, 80, 2, 82, 2, 86, 2, 89, 78, 87, 50, 69, 34, 79, 226, 74, 65, + 2, 73, 3, 85, 42, 66, 72, 162, 8, 79, 146, 137, 2, 69, 162, 64, 65, 2, + 73, 3, 85, 28, 230, 7, 72, 58, 79, 146, 137, 2, 69, 162, 64, 65, 2, 73, + 3, 85, 34, 62, 66, 202, 6, 69, 86, 79, 178, 201, 2, 65, 2, 73, 3, 85, 18, + 106, 79, 222, 5, 69, 134, 202, 2, 65, 2, 73, 3, 85, 24, 50, 79, 222, 5, + 69, 154, 76, 65, 2, 73, 3, 85, 7, 222, 207, 2, 78, 3, 79, 34, 70, 80, + 206, 5, 79, 198, 75, 65, 206, 189, 1, 69, 162, 64, 73, 3, 85, 18, 246, 4, + 69, 86, 79, 198, 75, 65, 238, 253, 1, 73, 3, 85, 16, 54, 69, 218, 4, 79, + 178, 201, 2, 65, 2, 73, 3, 85, 7, 26, 78, 239, 205, 2, 69, 2, 21, 3, 71, + 84, 72, 2, 219, 143, 2, 69, 42, 214, 3, 66, 0, 2, 71, 66, 58, 79, 146, + 137, 2, 69, 162, 64, 65, 2, 73, 3, 85, 90, 90, 68, 174, 1, 71, 126, 74, + 2, 89, 58, 79, 146, 137, 2, 69, 162, 64, 65, 2, 73, 3, 85, 24, 54, 79, + 186, 139, 2, 69, 162, 64, 65, 2, 73, 3, 85, 15, 36, 3, 76, 69, 32, 179, + 203, 2, 79, 10, 54, 83, 134, 172, 2, 68, 190, 28, 70, 2, 75, 3, 77, 2, + 131, 172, 2, 79, 25, 42, 71, 218, 250, 1, 65, 2, 69, 3, 79, 16, 50, 69, + 86, 79, 178, 201, 2, 65, 2, 73, 3, 85, 7, 130, 202, 2, 69, 3, 78, 14, 54, + 79, 146, 137, 2, 69, 162, 64, 65, 2, 73, 3, 85, 5, 175, 201, 2, 79, 28, + 46, 69, 34, 79, 226, 74, 65, 2, 73, 3, 85, 9, 254, 74, 69, 239, 253, 1, + 78, 9, 222, 74, 79, 239, 253, 1, 78, 26, 66, 68, 62, 70, 30, 74, 22, 75, + 50, 78, 22, 84, 231, 237, 1, 66, 6, 26, 79, 215, 246, 1, 65, 4, 210, 246, + 1, 79, 135, 50, 45, 4, 74, 69, 171, 196, 2, 65, 2, 151, 246, 1, 79, 4, + 26, 69, 235, 245, 1, 85, 2, 231, 245, 1, 69, 2, 243, 178, 2, 73, 6, 190, + 245, 1, 73, 2, 79, 195, 78, 65, 128, 4, 74, 49, 94, 50, 98, 51, 2, 52, 2, + 53, 2, 54, 2, 55, 2, 56, 3, 57, 223, 1, 182, 1, 48, 2, 49, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 137, 1, 90, 48, 2, 49, 2, 50, + 2, 51, 2, 52, 94, 53, 170, 195, 2, 54, 2, 55, 2, 56, 3, 57, 23, 130, 196, + 2, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 2, 54, 2, 55, 2, 56, 3, 57, 17, + 166, 195, 2, 48, 2, 49, 2, 50, 2, 51, 2, 52, 2, 53, 3, 54, 196, 1, 128, + 1, 4, 68, 73, 67, 32, 190, 19, 82, 152, 25, 2, 83, 84, 201, 131, 1, 13, + 67, 84, 79, 82, 32, 79, 82, 32, 67, 82, 79, 83, 83, 86, 60, 5, 83, 73, + 71, 78, 32, 137, 10, 5, 84, 79, 78, 69, 32, 48, 218, 2, 65, 216, 1, 17, + 68, 79, 85, 66, 76, 69, 32, 65, 78, 85, 83, 86, 65, 82, 65, 32, 65, 98, + 74, 52, 6, 78, 73, 72, 83, 72, 86, 22, 82, 240, 1, 8, 72, 69, 88, 73, 70, + 79, 82, 77, 22, 76, 52, 4, 84, 73, 82, 89, 22, 85, 60, 8, 86, 73, 83, 65, + 82, 71, 65, 32, 221, 6, 17, 89, 65, 74, 85, 82, 86, 69, 68, 73, 67, 32, + 77, 73, 68, 76, 73, 78, 14, 76, 8, 78, 85, 83, 86, 65, 82, 65, 32, 212, + 1, 3, 84, 73, 75, 151, 2, 82, 10, 134, 1, 65, 24, 4, 66, 65, 72, 73, 24, + 9, 85, 66, 72, 65, 89, 65, 84, 79, 32, 201, 4, 10, 86, 65, 77, 65, 71, + 79, 77, 85, 75, 72, 2, 21, 3, 78, 84, 65, 2, 21, 3, 82, 71, 79, 2, 11, + 77, 2, 147, 9, 85, 2, 175, 249, 1, 82, 2, 33, 6, 73, 72, 86, 65, 77, 85, + 2, 151, 3, 76, 2, 143, 185, 2, 65, 8, 144, 1, 15, 69, 86, 69, 82, 83, 69, + 68, 32, 86, 73, 83, 65, 82, 71, 65, 36, 9, 79, 84, 65, 84, 69, 68, 32, + 65, 82, 57, 5, 84, 72, 65, 78, 71, 4, 11, 32, 4, 202, 6, 65, 23, 85, 2, + 25, 4, 68, 72, 65, 86, 2, 141, 247, 1, 2, 73, 83, 2, 17, 2, 32, 76, 2, + 21, 3, 79, 78, 71, 2, 169, 244, 1, 2, 32, 65, 2, 239, 181, 2, 65, 2, 37, + 7, 80, 65, 68, 72, 77, 65, 78, 2, 239, 181, 2, 73, 10, 40, 3, 65, 78, 85, + 2, 85, 163, 9, 83, 4, 25, 4, 68, 65, 84, 84, 4, 11, 65, 5, 25, 4, 32, 87, + 73, 84, 2, 11, 72, 2, 11, 32, 2, 11, 84, 2, 211, 87, 65, 38, 128, 2, 5, + 67, 65, 78, 68, 82, 16, 2, 68, 79, 96, 2, 75, 65, 136, 1, 4, 80, 82, 69, + 78, 16, 2, 82, 73, 106, 84, 164, 1, 11, 89, 65, 74, 85, 82, 86, 69, 68, + 73, 67, 32, 180, 1, 12, 65, 84, 72, 65, 82, 86, 65, 86, 69, 68, 73, 67, + 253, 234, 1, 2, 83, 72, 4, 251, 18, 65, 6, 40, 5, 85, 66, 76, 69, 32, + 191, 3, 84, 4, 22, 82, 211, 5, 83, 2, 11, 73, 2, 251, 1, 78, 4, 56, 3, + 82, 83, 72, 17, 7, 84, 72, 65, 75, 65, 32, 65, 2, 203, 117, 65, 2, 17, 2, + 78, 85, 2, 17, 2, 68, 65, 2, 223, 142, 2, 84, 2, 239, 53, 75, 4, 82, 78, + 237, 2, 15, 71, 86, 69, 68, 73, 67, 32, 75, 65, 83, 72, 77, 73, 82, 73, + 2, 175, 181, 1, 71, 6, 42, 72, 24, 4, 82, 73, 80, 76, 19, 87, 2, 49, 3, + 82, 69, 69, 2, 219, 2, 69, 2, 11, 79, 2, 25, 4, 32, 68, 79, 84, 2, 11, + 83, 2, 11, 32, 2, 159, 15, 66, 8, 176, 1, 10, 65, 71, 71, 82, 65, 86, 65, + 84, 69, 68, 22, 73, 105, 27, 75, 65, 84, 72, 65, 75, 65, 32, 73, 78, 68, + 69, 80, 69, 78, 68, 69, 78, 84, 32, 83, 86, 65, 82, 73, 84, 65, 2, 17, 2, + 32, 73, 2, 49, 10, 78, 68, 69, 80, 69, 78, 68, 69, 78, 84, 2, 17, 2, 32, + 83, 2, 145, 138, 2, 3, 86, 65, 82, 5, 241, 10, 7, 32, 83, 67, 72, 82, 79, + 69, 104, 62, 83, 16, 6, 84, 73, 67, 65, 76, 32, 173, 17, 2, 89, 32, 2, + 223, 89, 73, 76, 200, 2, 4, 66, 65, 82, 32, 174, 3, 67, 42, 69, 62, 70, + 38, 71, 32, 11, 73, 68, 69, 79, 71, 82, 65, 80, 72, 73, 67, 48, 12, 75, + 65, 78, 65, 32, 82, 69, 80, 69, 65, 84, 32, 254, 1, 76, 186, 3, 77, 88, + 17, 79, 78, 69, 32, 69, 73, 71, 72, 84, 72, 32, 66, 76, 79, 67, 75, 45, + 62, 82, 158, 1, 84, 218, 1, 90, 237, 73, 3, 83, 73, 88, 8, 108, 6, 66, + 69, 83, 73, 68, 69, 64, 8, 68, 79, 85, 66, 76, 69, 32, 76, 20, 4, 84, 82, + 73, 80, 143, 1, 87, 2, 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 223, 130, + 1, 84, 2, 69, 2, 69, 70, 2, 25, 4, 76, 69, 32, 82, 2, 21, 3, 73, 71, 72, + 2, 11, 84, 2, 29, 5, 32, 84, 85, 82, 78, 2, 11, 83, 2, 11, 84, 2, 139, + 130, 1, 73, 2, 21, 3, 73, 84, 72, 2, 17, 2, 32, 72, 2, 221, 247, 1, 7, + 79, 82, 73, 90, 79, 78, 84, 2, 225, 238, 1, 5, 65, 80, 65, 67, 73, 2, 25, + 4, 76, 76, 73, 80, 2, 11, 83, 2, 171, 233, 1, 73, 2, 11, 79, 2, 141, 84, + 2, 85, 82, 2, 177, 159, 1, 3, 79, 45, 75, 2, 17, 2, 32, 73, 2, 217, 188, + 1, 2, 84, 69, 10, 120, 4, 77, 65, 82, 75, 45, 22, 87, 73, 84, 72, 32, 86, + 79, 73, 67, 69, 68, 32, 83, 79, 85, 78, 68, 32, 77, 65, 82, 75, 7, 11, + 32, 4, 50, 85, 21, 3, 76, 79, 87, 5, 17, 2, 32, 85, 2, 17, 2, 80, 80, 2, + 17, 2, 69, 82, 2, 133, 30, 2, 32, 72, 16, 30, 65, 33, 3, 73, 78, 69, 2, + 11, 68, 2, 235, 229, 1, 68, 15, 11, 32, 12, 38, 69, 49, 5, 87, 73, 84, + 72, 32, 2, 25, 4, 88, 84, 69, 78, 2, 175, 210, 1, 83, 10, 60, 5, 67, 73, + 82, 67, 76, 86, 70, 26, 84, 215, 141, 1, 77, 4, 11, 69, 4, 11, 32, 4, 26, + 66, 131, 165, 1, 65, 2, 11, 69, 2, 183, 133, 1, 76, 2, 57, 3, 79, 85, 82, + 2, 11, 72, 2, 21, 3, 82, 69, 69, 2, 45, 9, 32, 84, 73, 67, 75, 32, 77, + 65, 82, 2, 131, 227, 1, 75, 2, 65, 14, 65, 76, 69, 32, 87, 73, 84, 72, + 32, 83, 84, 82, 79, 75, 2, 135, 153, 1, 69, 12, 242, 159, 2, 50, 2, 51, + 2, 52, 2, 53, 2, 54, 3, 55, 4, 42, 65, 57, 6, 69, 83, 73, 83, 84, 79, 2, + 25, 4, 67, 73, 78, 71, 2, 11, 32, 2, 167, 123, 67, 2, 21, 3, 82, 32, 83, + 2, 11, 69, 2, 251, 218, 1, 71, 10, 32, 2, 65, 66, 106, 73, 19, 82, 6, 18, + 32, 35, 85, 2, 11, 75, 2, 143, 140, 2, 69, 4, 33, 6, 76, 65, 84, 73, 79, + 78, 5, 239, 46, 32, 2, 155, 10, 76, 2, 29, 5, 65, 70, 70, 73, 67, 2, 141, + 242, 1, 2, 32, 76, 2, 177, 108, 4, 73, 71, 90, 65, 26, 80, 6, 72, 69, 65, + 86, 89, 32, 176, 5, 3, 77, 85, 67, 229, 10, 3, 66, 79, 76, 20, 62, 69, + 190, 1, 70, 38, 82, 82, 83, 246, 1, 87, 203, 11, 71, 4, 25, 4, 73, 71, + 72, 84, 4, 11, 32, 4, 22, 80, 223, 2, 83, 2, 25, 4, 79, 73, 78, 84, 2, + 17, 2, 69, 68, 2, 17, 2, 32, 66, 2, 25, 4, 76, 65, 67, 75, 2, 11, 32, 2, + 151, 81, 83, 2, 11, 73, 2, 185, 1, 2, 86, 69, 2, 11, 69, 2, 29, 5, 86, + 69, 82, 83, 69, 2, 17, 2, 32, 83, 2, 231, 1, 79, 6, 30, 65, 42, 73, 147, + 1, 79, 2, 11, 76, 2, 11, 84, 2, 235, 117, 73, 2, 11, 88, 2, 11, 32, 2, + 11, 83, 2, 21, 3, 80, 79, 75, 2, 17, 2, 69, 68, 2, 11, 32, 2, 11, 65, 2, + 11, 83, 2, 149, 80, 3, 84, 69, 82, 2, 161, 14, 3, 76, 73, 68, 4, 21, 3, + 72, 73, 84, 4, 11, 69, 4, 11, 32, 4, 162, 66, 67, 187, 49, 83, 4, 11, 72, + 4, 11, 32, 4, 18, 71, 39, 76, 2, 69, 6, 82, 69, 65, 84, 69, 82, 2, 11, + 69, 2, 11, 83, 2, 11, 83, 2, 17, 2, 45, 84, 2, 251, 72, 72, 4, 11, 65, 5, + 11, 32, 2, 11, 70, 2, 21, 3, 79, 82, 77, 2, 17, 2, 32, 84, 2, 195, 245, + 1, 87, 162, 1, 156, 1, 9, 66, 82, 65, 84, 73, 79, 78, 32, 77, 32, 4, 67, + 84, 79, 82, 36, 3, 68, 69, 79, 126, 69, 222, 1, 79, 22, 82, 21, 7, 84, + 72, 75, 85, 81, 73, 32, 2, 11, 79, 2, 159, 252, 1, 68, 4, 170, 130, 1, + 89, 255, 141, 1, 73, 6, 30, 32, 77, 3, 67, 65, 83, 4, 18, 67, 43, 71, 2, + 17, 2, 65, 77, 2, 179, 205, 1, 69, 2, 239, 119, 65, 2, 167, 72, 83, 6, 168, 1, 31, 84, 78, 65, 77, 69, 83, 69, 32, 65, 76, 84, 69, 82, 78, 65, - 84, 69, 32, 82, 69, 65, 68, 73, 78, 71, 32, 77, 65, 82, 75, 32, 189, 108, - 5, 87, 68, 65, 84, 65, 4, 26, 78, 219, 138, 2, 67, 2, 139, 129, 1, 72, 2, - 167, 141, 1, 76, 2, 243, 237, 1, 71, 140, 1, 56, 6, 67, 65, 80, 73, 84, + 84, 69, 32, 82, 69, 65, 68, 73, 78, 71, 32, 77, 65, 82, 75, 32, 161, 109, + 5, 87, 68, 65, 84, 65, 4, 26, 78, 195, 141, 2, 67, 2, 215, 129, 1, 72, 2, + 143, 144, 1, 76, 2, 219, 240, 1, 71, 140, 1, 56, 6, 67, 65, 80, 73, 84, 65, 1, 4, 83, 77, 65, 76, 70, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, - 70, 230, 1, 66, 34, 69, 22, 73, 22, 76, 34, 78, 246, 136, 1, 67, 2, 68, + 70, 230, 1, 66, 34, 69, 22, 73, 22, 76, 34, 78, 222, 139, 1, 67, 2, 68, 2, 83, 2, 84, 226, 56, 72, 238, 48, 70, 2, 74, 2, 77, 2, 80, 2, 82, 2, 86, 2, 88, 2, 90, 158, 20, 71, 2, 75, 2, 81, 186, 2, 65, 2, 79, 2, 85, 3, - 89, 4, 166, 243, 1, 66, 215, 22, 69, 5, 219, 137, 2, 73, 5, 243, 242, 1, - 74, 4, 250, 134, 2, 76, 187, 2, 65, 4, 190, 242, 1, 74, 215, 22, 69, 8, - 28, 3, 73, 68, 69, 59, 76, 2, 21, 3, 68, 32, 71, 2, 141, 71, 4, 82, 69, - 69, 75, 6, 46, 67, 20, 3, 76, 69, 89, 41, 2, 85, 77, 2, 219, 222, 1, 65, - 2, 11, 66, 2, 11, 65, 2, 139, 126, 76, 2, 11, 69, 2, 17, 2, 32, 73, 2, - 193, 125, 4, 78, 84, 69, 71, 5, 239, 134, 2, 83, 40, 70, 67, 45, 13, 71, + 89, 4, 142, 246, 1, 66, 215, 22, 69, 5, 195, 140, 2, 73, 5, 219, 245, 1, + 74, 4, 226, 137, 2, 76, 187, 2, 65, 4, 166, 245, 1, 74, 215, 22, 69, 8, + 28, 3, 73, 68, 69, 59, 76, 2, 21, 3, 68, 32, 71, 2, 225, 71, 4, 82, 69, + 69, 75, 6, 46, 67, 20, 3, 76, 69, 89, 41, 2, 85, 77, 2, 195, 225, 1, 65, + 2, 11, 66, 2, 11, 65, 2, 215, 126, 76, 2, 11, 69, 2, 17, 2, 32, 73, 2, + 141, 126, 4, 78, 84, 69, 71, 5, 215, 137, 2, 83, 40, 70, 67, 45, 13, 71, 65, 82, 32, 70, 82, 65, 67, 84, 73, 79, 78, 32, 2, 11, 65, 2, 11, 78, 2, - 159, 200, 1, 85, 38, 106, 70, 96, 4, 79, 78, 69, 32, 178, 2, 84, 80, 7, + 135, 203, 1, 85, 38, 106, 70, 96, 4, 79, 78, 69, 32, 178, 2, 84, 80, 7, 83, 69, 86, 69, 78, 32, 69, 153, 33, 3, 90, 69, 82, 6, 56, 4, 73, 86, 69, 32, 253, 3, 5, 79, 85, 82, 32, 70, 4, 162, 3, 69, 113, 3, 83, 73, 88, 18, 66, 69, 28, 2, 70, 73, 18, 72, 34, 78, 14, 81, 30, 83, 55, 84, 2, 197, 1, - 3, 73, 71, 72, 2, 171, 1, 70, 2, 11, 65, 2, 251, 129, 1, 76, 2, 111, 73, - 2, 133, 75, 3, 85, 65, 82, 4, 24, 2, 69, 86, 15, 73, 2, 43, 69, 2, 43, - 88, 4, 18, 69, 35, 72, 2, 11, 78, 2, 255, 128, 2, 84, 2, 143, 107, 73, + 3, 73, 71, 72, 2, 171, 1, 70, 2, 11, 65, 2, 227, 132, 1, 76, 2, 111, 73, + 2, 217, 75, 3, 85, 65, 82, 4, 24, 2, 69, 86, 15, 73, 2, 43, 69, 2, 43, + 88, 4, 18, 69, 35, 72, 2, 11, 78, 2, 231, 131, 2, 84, 2, 243, 107, 73, 10, 48, 5, 72, 82, 69, 69, 32, 93, 3, 87, 79, 32, 6, 26, 69, 26, 81, 67, - 70, 2, 109, 3, 73, 71, 72, 2, 21, 3, 85, 65, 82, 2, 139, 114, 84, 4, 22, - 70, 219, 32, 84, 2, 11, 73, 2, 11, 70, 2, 205, 194, 1, 2, 84, 72, 164, 6, - 86, 65, 250, 23, 69, 158, 3, 72, 250, 68, 73, 234, 8, 79, 134, 6, 82, - 191, 144, 1, 74, 204, 2, 114, 70, 18, 78, 220, 5, 2, 88, 73, 158, 1, 82, - 210, 10, 84, 194, 1, 86, 221, 152, 1, 6, 83, 84, 69, 66, 65, 83, 2, 147, + 70, 2, 109, 3, 73, 71, 72, 2, 21, 3, 85, 65, 82, 2, 215, 114, 84, 4, 22, + 70, 219, 32, 84, 2, 11, 73, 2, 11, 70, 2, 181, 197, 1, 2, 84, 72, 168, 6, + 86, 65, 250, 23, 69, 158, 3, 72, 222, 69, 73, 234, 8, 79, 238, 5, 82, + 219, 146, 1, 74, 204, 2, 114, 70, 18, 78, 220, 5, 2, 88, 73, 158, 1, 82, + 210, 10, 84, 194, 1, 86, 197, 155, 1, 6, 83, 84, 69, 66, 65, 83, 2, 231, 90, 70, 122, 36, 4, 67, 72, 79, 32, 183, 5, 73, 118, 100, 7, 76, 69, 84, 84, 69, 82, 32, 252, 3, 3, 78, 71, 85, 16, 5, 84, 79, 78, 69, 32, 243, 7, 68, 88, 210, 1, 65, 38, 79, 34, 69, 22, 73, 22, 76, 50, 78, 42, 84, 50, - 85, 22, 89, 130, 178, 1, 75, 2, 80, 2, 83, 138, 69, 66, 2, 67, 2, 68, 2, + 85, 22, 89, 234, 180, 1, 75, 2, 80, 2, 83, 138, 69, 66, 2, 67, 2, 68, 2, 70, 2, 71, 2, 72, 2, 74, 2, 77, 2, 82, 2, 86, 2, 87, 3, 90, 13, 34, 65, - 210, 250, 1, 78, 87, 85, 7, 11, 78, 5, 151, 251, 1, 71, 5, 131, 251, 1, - 78, 5, 155, 250, 1, 78, 4, 26, 76, 195, 250, 1, 65, 2, 135, 248, 1, 72, - 6, 242, 247, 1, 71, 2, 89, 187, 2, 65, 8, 202, 247, 1, 72, 2, 82, 2, 83, - 187, 2, 65, 5, 243, 169, 1, 69, 4, 174, 248, 1, 73, 147, 1, 65, 2, 159, - 114, 78, 8, 40, 3, 75, 79, 73, 1, 3, 84, 85, 80, 5, 139, 229, 1, 78, 4, + 186, 253, 1, 78, 87, 85, 7, 11, 78, 5, 255, 253, 1, 71, 5, 235, 253, 1, + 78, 5, 131, 253, 1, 78, 4, 26, 76, 171, 253, 1, 65, 2, 239, 250, 1, 72, + 6, 218, 250, 1, 71, 2, 89, 187, 2, 65, 8, 178, 250, 1, 72, 2, 82, 2, 83, + 187, 2, 65, 5, 219, 172, 1, 69, 4, 150, 251, 1, 73, 147, 1, 65, 2, 135, + 117, 78, 8, 40, 3, 75, 79, 73, 1, 3, 84, 85, 80, 5, 243, 231, 1, 78, 4, 21, 3, 78, 71, 32, 4, 76, 8, 67, 82, 69, 83, 67, 69, 78, 84, 1, 7, 71, - 73, 66, 66, 79, 85, 83, 2, 21, 3, 32, 77, 79, 2, 11, 79, 2, 155, 100, 78, - 170, 1, 72, 9, 65, 78, 71, 32, 67, 73, 84, 73, 32, 229, 111, 4, 78, 73, + 73, 66, 66, 79, 85, 83, 2, 21, 3, 32, 77, 79, 2, 11, 79, 2, 231, 100, 78, + 170, 1, 72, 9, 65, 78, 71, 32, 67, 73, 84, 73, 32, 205, 114, 4, 78, 73, 78, 71, 168, 1, 128, 1, 6, 67, 65, 80, 73, 84, 65, 0, 4, 83, 77, 65, 76, - 190, 4, 68, 156, 2, 7, 78, 85, 77, 66, 69, 82, 32, 175, 222, 1, 79, 64, + 190, 4, 68, 156, 2, 7, 78, 85, 77, 66, 69, 82, 32, 151, 225, 1, 79, 64, 45, 9, 76, 32, 76, 69, 84, 84, 69, 82, 32, 64, 174, 1, 65, 38, 69, 42, - 72, 74, 78, 50, 79, 22, 83, 50, 85, 30, 89, 226, 42, 84, 180, 110, 2, 86, + 72, 74, 78, 50, 79, 22, 83, 50, 85, 30, 89, 174, 43, 84, 208, 112, 2, 86, 73, 222, 51, 66, 2, 80, 246, 5, 75, 158, 11, 73, 2, 87, 162, 17, 68, 3, - 71, 9, 166, 243, 1, 78, 86, 77, 3, 84, 7, 11, 78, 4, 202, 243, 1, 78, 3, - 89, 8, 38, 79, 198, 154, 1, 73, 235, 31, 65, 4, 170, 186, 1, 82, 235, 25, - 76, 4, 26, 71, 215, 161, 1, 85, 2, 147, 240, 1, 65, 5, 151, 172, 1, 68, - 4, 26, 83, 179, 222, 1, 73, 2, 159, 205, 1, 85, 4, 242, 241, 1, 67, 3, - 85, 8, 34, 85, 182, 241, 1, 65, 3, 79, 5, 179, 241, 1, 74, 20, 11, 73, + 71, 9, 142, 246, 1, 78, 86, 77, 3, 84, 7, 11, 78, 4, 178, 246, 1, 78, 3, + 89, 8, 38, 79, 174, 157, 1, 73, 235, 31, 65, 4, 146, 189, 1, 82, 235, 25, + 76, 4, 26, 71, 191, 164, 1, 85, 2, 251, 242, 1, 65, 5, 255, 174, 1, 68, + 4, 26, 83, 155, 225, 1, 73, 2, 135, 208, 1, 85, 4, 218, 244, 1, 67, 3, + 85, 8, 34, 85, 158, 244, 1, 65, 3, 79, 5, 155, 244, 1, 74, 20, 11, 73, 20, 11, 71, 20, 17, 2, 73, 84, 20, 11, 32, 20, 66, 70, 30, 83, 42, 84, - 62, 90, 238, 85, 78, 14, 79, 223, 110, 69, 4, 206, 115, 73, 131, 4, 79, - 4, 22, 69, 207, 99, 73, 2, 135, 28, 86, 4, 26, 72, 211, 208, 1, 87, 2, - 11, 82, 2, 227, 216, 1, 69, 2, 11, 69, 2, 163, 208, 1, 82, 18, 42, 69, + 62, 90, 210, 86, 78, 14, 79, 227, 112, 69, 4, 182, 118, 73, 131, 4, 79, + 4, 22, 69, 155, 100, 73, 2, 211, 28, 86, 4, 26, 72, 187, 211, 1, 87, 2, + 11, 82, 2, 203, 219, 1, 69, 2, 11, 69, 2, 139, 211, 1, 82, 18, 42, 69, 30, 70, 42, 78, 38, 83, 39, 84, 2, 221, 1, 3, 73, 71, 72, 4, 22, 73, 139, 1, 79, 2, 171, 1, 70, 2, 17, 2, 73, 78, 2, 135, 1, 69, 4, 92, 2, 69, 86, - 25, 2, 73, 88, 6, 34, 72, 26, 87, 179, 157, 1, 69, 2, 11, 73, 2, 35, 82, - 2, 11, 69, 2, 11, 78, 2, 175, 219, 1, 84, 12, 32, 2, 69, 82, 179, 235, 1, - 67, 10, 34, 32, 165, 156, 1, 2, 77, 69, 8, 80, 3, 67, 76, 79, 22, 87, - 228, 200, 1, 5, 66, 85, 70, 70, 65, 1, 2, 80, 79, 2, 175, 182, 1, 83, 2, - 223, 110, 65, 20, 60, 2, 69, 32, 124, 4, 73, 78, 71, 32, 161, 1, 2, 89, - 32, 6, 182, 2, 68, 161, 189, 1, 23, 65, 82, 82, 79, 87, 32, 80, 79, 73, + 25, 2, 73, 88, 6, 34, 72, 26, 87, 155, 160, 1, 69, 2, 11, 73, 2, 35, 82, + 2, 11, 69, 2, 11, 78, 2, 151, 222, 1, 84, 12, 32, 2, 69, 82, 155, 238, 1, + 67, 10, 34, 32, 141, 159, 1, 2, 77, 69, 8, 80, 3, 67, 76, 79, 22, 87, + 204, 203, 1, 5, 66, 85, 70, 70, 65, 1, 2, 80, 79, 2, 151, 185, 1, 83, 2, + 199, 113, 65, 20, 60, 2, 69, 32, 124, 4, 73, 78, 71, 32, 161, 1, 2, 89, + 32, 6, 182, 2, 68, 137, 192, 1, 23, 65, 82, 82, 79, 87, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, 68, 73, 82, 69, 67, 84, 76, 89, 6, 64, 5, 66, 76, 65, 67, 75, 0, 5, 87, 72, 73, 84, 69, 55, 72, 2, 17, 2, 32, 70, 2, 11, - 76, 2, 175, 232, 1, 65, 2, 11, 65, 2, 11, 78, 2, 215, 97, 68, 8, 26, 68, - 34, 76, 43, 79, 2, 11, 65, 2, 143, 231, 1, 83, 4, 22, 79, 203, 78, 73, 2, - 163, 78, 87, 2, 11, 86, 2, 11, 69, 2, 135, 78, 82, 14, 48, 3, 65, 82, 89, - 70, 68, 70, 73, 171, 1, 83, 4, 11, 32, 4, 32, 2, 67, 65, 211, 169, 1, 70, - 2, 183, 169, 1, 84, 4, 26, 71, 167, 149, 1, 68, 2, 129, 70, 6, 69, 45, + 76, 2, 151, 235, 1, 65, 2, 11, 65, 2, 11, 78, 2, 191, 100, 68, 8, 26, 68, + 34, 76, 43, 79, 2, 11, 65, 2, 247, 233, 1, 83, 4, 22, 79, 175, 79, 73, 2, + 135, 79, 87, 2, 11, 86, 2, 11, 69, 2, 235, 78, 82, 14, 48, 3, 65, 82, 89, + 70, 68, 70, 73, 171, 1, 83, 4, 11, 32, 4, 32, 2, 67, 65, 187, 172, 1, 70, + 2, 159, 172, 1, 84, 4, 26, 71, 143, 152, 1, 68, 2, 229, 70, 6, 69, 45, 84, 65, 73, 76, 4, 116, 17, 69, 82, 83, 84, 82, 65, 83, 83, 32, 69, 76, - 76, 73, 80, 84, 73, 67, 145, 45, 7, 71, 72, 84, 32, 76, 73, 70, 2, 17, 2, - 32, 70, 2, 149, 148, 1, 2, 85, 78, 2, 37, 7, 84, 32, 83, 89, 82, 73, 65, - 2, 131, 35, 67, 188, 2, 52, 3, 69, 69, 76, 100, 3, 73, 84, 69, 219, 62, - 65, 7, 60, 7, 32, 79, 70, 32, 68, 72, 65, 21, 4, 67, 72, 65, 73, 2, 147, - 194, 1, 82, 2, 231, 79, 82, 180, 2, 54, 32, 161, 66, 8, 45, 70, 69, 65, - 84, 72, 69, 82, 178, 2, 148, 2, 18, 65, 82, 82, 79, 87, 32, 83, 72, 65, - 70, 84, 32, 87, 73, 68, 84, 72, 32, 114, 66, 46, 67, 250, 15, 68, 166, 5, - 69, 50, 70, 178, 3, 72, 246, 3, 76, 210, 4, 77, 222, 1, 78, 34, 80, 146, + 76, 73, 80, 84, 73, 67, 229, 45, 7, 71, 72, 84, 32, 76, 73, 70, 2, 17, 2, + 32, 70, 2, 253, 150, 1, 2, 85, 78, 2, 37, 7, 84, 32, 83, 89, 82, 73, 65, + 2, 215, 35, 67, 192, 2, 52, 3, 69, 69, 76, 100, 3, 73, 84, 69, 175, 63, + 65, 7, 60, 7, 32, 79, 70, 32, 68, 72, 65, 21, 4, 67, 72, 65, 73, 2, 251, + 196, 1, 82, 2, 179, 80, 82, 184, 2, 54, 32, 133, 67, 8, 45, 70, 69, 65, + 84, 72, 69, 82, 182, 2, 148, 2, 18, 65, 82, 82, 79, 87, 32, 83, 72, 65, + 70, 84, 32, 87, 73, 68, 84, 72, 32, 114, 66, 46, 67, 198, 16, 68, 166, 5, + 69, 50, 70, 186, 3, 72, 246, 3, 76, 210, 4, 77, 222, 1, 78, 34, 80, 146, 1, 81, 78, 82, 142, 2, 83, 158, 12, 84, 228, 3, 2, 85, 80, 149, 4, 3, 86, - 69, 82, 4, 22, 84, 231, 70, 79, 2, 11, 87, 2, 21, 3, 79, 32, 84, 2, 17, - 2, 72, 73, 2, 11, 82, 2, 239, 161, 1, 68, 2, 11, 85, 2, 11, 76, 2, 135, - 170, 1, 76, 94, 154, 1, 72, 176, 10, 5, 73, 82, 67, 76, 69, 162, 3, 76, + 69, 82, 4, 22, 84, 203, 71, 79, 2, 11, 87, 2, 21, 3, 79, 32, 84, 2, 17, + 2, 72, 73, 2, 11, 82, 2, 215, 164, 1, 68, 2, 11, 85, 2, 11, 76, 2, 239, + 172, 1, 76, 98, 154, 1, 72, 252, 10, 5, 73, 82, 67, 76, 69, 162, 3, 76, 24, 20, 79, 78, 67, 65, 86, 69, 45, 83, 73, 68, 69, 68, 32, 68, 73, 65, - 77, 79, 78, 68, 79, 82, 66, 25, 4, 69, 83, 83, 32, 66, 66, 66, 38, 69, - 118, 75, 142, 4, 80, 22, 81, 38, 82, 131, 2, 84, 6, 241, 5, 5, 73, 83, - 72, 79, 80, 4, 45, 9, 81, 85, 73, 72, 79, 80, 80, 69, 82, 5, 17, 2, 32, - 82, 2, 129, 6, 8, 79, 84, 65, 84, 69, 68, 32, 78, 26, 38, 73, 25, 5, 78, - 73, 71, 72, 84, 6, 177, 4, 2, 78, 71, 21, 22, 32, 151, 3, 45, 12, 41, 8, - 82, 79, 84, 65, 84, 69, 68, 32, 12, 104, 2, 70, 79, 0, 15, 79, 78, 69, + 77, 79, 78, 68, 79, 82, 70, 25, 4, 69, 83, 83, 32, 70, 104, 3, 65, 76, + 70, 18, 66, 38, 69, 116, 3, 70, 69, 82, 22, 75, 142, 4, 80, 22, 81, 38, + 82, 131, 2, 84, 2, 255, 83, 73, 6, 133, 6, 5, 73, 83, 72, 79, 80, 4, 45, + 9, 81, 85, 73, 72, 79, 80, 80, 69, 82, 5, 17, 2, 32, 82, 2, 149, 6, 8, + 79, 84, 65, 84, 69, 68, 32, 78, 2, 167, 222, 1, 90, 26, 38, 73, 25, 5, + 78, 73, 71, 72, 84, 6, 177, 4, 2, 78, 71, 21, 22, 32, 151, 3, 45, 12, 41, + 8, 82, 79, 84, 65, 84, 69, 68, 32, 12, 104, 2, 70, 79, 0, 15, 79, 78, 69, 32, 72, 85, 78, 68, 82, 69, 68, 32, 84, 72, 73, 18, 84, 215, 3, 78, 2, 207, 1, 82, 6, 148, 1, 11, 87, 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, 133, 3, 20, 72, 82, 69, 69, 32, 72, 85, 78, 68, 82, 69, 68, 32, 70, 73, @@ -8854,374 +8956,377 @@ static const unsigned char packed_name_dawg[] = { 87, 6, 21, 3, 85, 69, 69, 6, 35, 78, 6, 21, 3, 79, 79, 75, 7, 45, 9, 32, 82, 79, 84, 65, 84, 69, 68, 32, 4, 70, 78, 25, 13, 84, 87, 79, 32, 72, 85, 78, 68, 82, 69, 68, 32, 83, 2, 49, 3, 73, 78, 69, 2, 25, 4, 69, 86, - 69, 78, 2, 17, 2, 84, 89, 2, 225, 65, 6, 32, 68, 69, 71, 82, 69, 12, 29, + 69, 78, 2, 17, 2, 84, 89, 2, 241, 65, 6, 32, 68, 69, 71, 82, 69, 12, 29, 5, 85, 82, 78, 69, 68, 12, 11, 32, 12, 42, 66, 30, 75, 34, 80, 34, 81, - 47, 82, 2, 197, 91, 3, 73, 83, 72, 4, 198, 131, 1, 73, 171, 38, 78, 2, - 11, 65, 2, 179, 132, 1, 87, 2, 11, 85, 2, 11, 69, 2, 135, 132, 1, 69, 2, - 243, 169, 1, 79, 19, 11, 32, 16, 100, 16, 67, 79, 78, 84, 65, 73, 78, 73, + 47, 82, 2, 225, 93, 3, 73, 83, 72, 4, 226, 133, 1, 73, 171, 38, 78, 2, + 11, 65, 2, 207, 134, 1, 87, 2, 11, 85, 2, 11, 69, 2, 163, 134, 1, 69, 2, + 143, 172, 1, 79, 19, 11, 32, 16, 100, 16, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 121, 5, 87, 73, 84, 72, 32, 2, 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 11, 32, 2, 11, 67, 2, 11, 73, 2, - 11, 82, 2, 187, 45, 67, 14, 56, 2, 68, 79, 70, 84, 228, 31, 2, 76, 79, - 231, 1, 85, 4, 18, 84, 35, 87, 2, 11, 32, 2, 167, 166, 1, 82, 2, 143, 54, - 78, 2, 11, 87, 2, 11, 79, 2, 11, 32, 2, 247, 59, 68, 2, 209, 27, 2, 85, - 66, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 228, 30, 2, 76, 69, 49, 2, 82, - 73, 2, 17, 2, 79, 83, 2, 227, 141, 1, 83, 28, 76, 6, 73, 65, 77, 79, 78, + 11, 82, 2, 195, 45, 67, 14, 56, 2, 68, 79, 70, 84, 236, 31, 2, 76, 79, + 231, 1, 85, 4, 18, 84, 35, 87, 2, 11, 32, 2, 195, 168, 1, 82, 2, 167, 54, + 78, 2, 11, 87, 2, 11, 79, 2, 11, 32, 2, 147, 60, 68, 2, 217, 27, 2, 85, + 66, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 236, 30, 2, 76, 69, 49, 2, 82, + 73, 2, 17, 2, 79, 83, 2, 255, 143, 1, 83, 28, 76, 6, 73, 65, 77, 79, 78, 68, 216, 2, 3, 79, 87, 78, 177, 1, 2, 82, 65, 15, 11, 32, 12, 160, 1, 17, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, 65, 67, 75, 32, 128, - 1, 9, 87, 73, 84, 72, 32, 67, 69, 78, 84, 226, 23, 83, 197, 19, 2, 73, + 1, 9, 87, 73, 84, 72, 32, 67, 69, 78, 84, 234, 23, 83, 213, 19, 2, 73, 78, 6, 74, 83, 0, 6, 86, 69, 82, 89, 32, 83, 29, 6, 77, 69, 68, 73, 85, - 77, 2, 25, 4, 77, 65, 76, 76, 2, 197, 15, 2, 32, 68, 2, 11, 82, 2, 11, + 77, 2, 25, 4, 77, 65, 76, 76, 2, 205, 15, 2, 32, 68, 2, 11, 82, 2, 11, 69, 2, 231, 58, 68, 10, 96, 10, 32, 80, 79, 73, 78, 84, 73, 78, 71, 32, - 53, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 170, 36, 66, 24, 5, - 76, 69, 70, 84, 32, 51, 73, 4, 194, 37, 83, 51, 84, 4, 33, 6, 85, 71, 72, - 84, 83, 32, 4, 22, 77, 255, 120, 75, 2, 159, 122, 65, 2, 29, 5, 88, 67, - 76, 65, 77, 2, 167, 15, 65, 14, 74, 76, 144, 2, 11, 79, 85, 82, 32, 80, + 53, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 178, 36, 66, 24, 5, + 76, 69, 70, 84, 32, 51, 73, 4, 202, 37, 83, 51, 84, 4, 33, 6, 85, 71, 72, + 84, 83, 32, 4, 22, 77, 155, 123, 75, 2, 187, 124, 65, 2, 29, 5, 88, 67, + 76, 65, 77, 2, 175, 15, 65, 14, 74, 76, 144, 2, 11, 79, 85, 82, 32, 80, 79, 73, 78, 84, 69, 68, 71, 82, 8, 28, 2, 65, 71, 175, 1, 79, 5, 149, 1, 34, 32, 87, 73, 84, 72, 32, 72, 79, 82, 73, 90, 79, 78, 84, 65, 76, 32, 77, 73, 68, 68, 76, 69, 32, 66, 76, 65, 67, 75, 32, 83, 84, 82, 73, 2, - 231, 176, 1, 80, 4, 26, 82, 163, 137, 1, 87, 2, 17, 2, 69, 84, 2, 163, - 176, 1, 84, 4, 11, 32, 4, 18, 67, 23, 83, 2, 171, 198, 1, 85, 2, 247, 36, - 84, 2, 235, 61, 79, 16, 34, 65, 150, 1, 69, 231, 1, 79, 2, 25, 4, 82, 68, - 32, 83, 2, 45, 9, 72, 69, 76, 76, 32, 70, 76, 79, 80, 2, 17, 2, 80, 89, - 2, 17, 2, 32, 68, 2, 11, 73, 2, 211, 193, 1, 83, 10, 22, 65, 151, 12, 88, - 8, 30, 82, 29, 3, 86, 89, 32, 4, 11, 84, 5, 211, 15, 32, 4, 74, 67, 53, - 14, 83, 65, 76, 84, 73, 82, 69, 32, 87, 73, 84, 72, 32, 82, 2, 17, 2, 72, - 69, 2, 11, 67, 2, 211, 129, 1, 75, 2, 167, 19, 79, 4, 60, 8, 82, 73, 90, - 79, 78, 84, 65, 76, 233, 42, 2, 85, 82, 2, 205, 31, 2, 32, 69, 22, 42, - 65, 116, 3, 69, 70, 84, 175, 1, 79, 4, 24, 2, 82, 71, 19, 84, 2, 227, 32, - 69, 2, 11, 73, 2, 11, 78, 2, 17, 2, 32, 67, 2, 11, 82, 2, 187, 42, 79, - 12, 58, 32, 77, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 6, 44, 6, 76, - 65, 78, 69, 32, 77, 247, 24, 80, 2, 11, 69, 2, 227, 10, 82, 6, 150, 2, - 80, 206, 24, 83, 51, 84, 6, 160, 1, 4, 87, 69, 82, 32, 221, 8, 30, 90, - 69, 78, 71, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, - 65, 67, 75, 32, 83, 77, 65, 76, 76, 32, 76, 4, 44, 3, 76, 69, 70, 1, 4, - 82, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, 80, 2, 203, 5, 79, 12, 68, 6, - 69, 68, 73, 85, 77, 32, 117, 7, 79, 79, 78, 32, 83, 69, 76, 10, 30, 68, - 54, 83, 227, 6, 76, 2, 11, 73, 2, 11, 65, 2, 11, 77, 2, 199, 46, 79, 6, - 246, 26, 84, 50, 77, 59, 81, 2, 11, 69, 2, 203, 185, 1, 78, 2, 11, 73, 2, - 227, 187, 1, 66, 6, 18, 65, 87, 69, 2, 37, 7, 82, 65, 76, 76, 69, 76, 79, - 2, 11, 71, 2, 11, 82, 2, 163, 170, 1, 65, 4, 11, 78, 4, 174, 2, 84, 211, - 46, 78, 2, 21, 3, 85, 69, 83, 2, 197, 118, 9, 84, 73, 79, 78, 32, 77, 65, - 82, 75, 14, 34, 69, 25, 4, 73, 71, 72, 84, 2, 129, 21, 2, 67, 84, 12, 60, - 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 211, 17, 32, 8, 30, 80, 202, - 19, 83, 51, 84, 4, 18, 69, 55, 79, 2, 11, 78, 2, 11, 84, 2, 11, 65, 2, - 179, 104, 71, 2, 11, 73, 2, 11, 78, 2, 139, 122, 84, 56, 118, 67, 32, 3, - 69, 83, 65, 18, 72, 58, 77, 166, 1, 80, 68, 5, 81, 85, 65, 82, 69, 180, - 6, 2, 85, 78, 183, 12, 84, 2, 153, 40, 4, 73, 83, 83, 79, 2, 175, 37, 77, - 2, 21, 3, 79, 71, 73, 2, 169, 121, 4, 32, 80, 73, 69, 8, 32, 4, 65, 76, - 76, 32, 115, 73, 6, 18, 76, 71, 83, 2, 11, 79, 2, 11, 90, 2, 11, 69, 2, - 11, 78, 2, 203, 158, 1, 71, 4, 210, 19, 84, 107, 81, 2, 211, 44, 76, 2, - 21, 3, 65, 68, 69, 2, 11, 32, 2, 11, 83, 2, 251, 151, 1, 85, 29, 11, 32, - 26, 114, 66, 36, 17, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, 66, 76, - 65, 67, 75, 32, 85, 5, 87, 73, 84, 72, 32, 2, 17, 2, 85, 84, 2, 159, 99, - 84, 6, 54, 86, 178, 17, 83, 37, 6, 77, 69, 68, 73, 85, 77, 2, 153, 17, 3, - 69, 82, 89, 18, 150, 1, 76, 50, 82, 214, 1, 85, 144, 1, 17, 86, 69, 82, - 84, 73, 67, 65, 76, 32, 66, 73, 83, 69, 67, 84, 73, 78, 233, 19, 6, 67, - 69, 78, 84, 82, 69, 6, 18, 69, 15, 79, 2, 67, 70, 4, 247, 1, 87, 4, 18, - 73, 115, 79, 2, 17, 2, 71, 72, 2, 33, 6, 84, 87, 65, 82, 68, 83, 2, 11, - 32, 2, 11, 84, 2, 11, 73, 2, 215, 172, 1, 67, 2, 29, 5, 85, 78, 68, 69, - 68, 2, 21, 3, 32, 67, 79, 2, 193, 32, 2, 82, 78, 4, 17, 2, 80, 80, 4, 21, - 3, 69, 82, 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, 2, 33, 6, 84, - 32, 81, 85, 65, 68, 2, 175, 36, 82, 2, 171, 20, 71, 11, 11, 32, 8, 84, - 12, 66, 69, 72, 73, 78, 68, 32, 67, 76, 79, 85, 68, 69, 5, 87, 73, 84, - 72, 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 255, 44, 65, - 4, 38, 82, 29, 5, 83, 77, 65, 76, 76, 2, 11, 65, 2, 179, 110, 89, 2, 11, - 32, 2, 21, 3, 67, 76, 79, 2, 187, 101, 85, 10, 38, 79, 54, 69, 62, 82, - 203, 1, 87, 2, 49, 10, 85, 67, 72, 84, 79, 78, 69, 32, 84, 69, 2, 17, 2, - 76, 69, 2, 11, 80, 2, 11, 72, 2, 151, 17, 79, 4, 148, 1, 4, 65, 80, 69, - 90, 33, 28, 73, 65, 78, 71, 76, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, - 78, 71, 32, 83, 77, 65, 76, 76, 32, 87, 72, 73, 84, 2, 11, 73, 2, 183, - 152, 1, 85, 2, 135, 4, 69, 2, 11, 79, 2, 85, 19, 45, 87, 65, 89, 32, 76, - 69, 70, 84, 32, 87, 65, 89, 32, 84, 82, 65, 70, 70, 2, 11, 73, 2, 251, - 167, 1, 67, 12, 62, 32, 185, 1, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, - 32, 4, 11, 80, 4, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 4, 18, 66, 75, - 73, 2, 21, 3, 65, 67, 75, 2, 25, 4, 72, 65, 78, 68, 2, 17, 2, 32, 73, 2, - 17, 2, 78, 68, 2, 219, 25, 69, 8, 54, 67, 42, 83, 125, 7, 84, 82, 73, 65, - 78, 71, 76, 2, 21, 3, 72, 69, 86, 2, 179, 85, 82, 2, 25, 4, 77, 65, 76, - 76, 2, 17, 2, 32, 84, 2, 17, 2, 82, 73, 2, 11, 65, 2, 11, 78, 2, 11, 71, - 2, 211, 141, 1, 76, 4, 11, 69, 5, 11, 32, 2, 11, 87, 2, 217, 18, 3, 73, - 84, 72, 10, 44, 6, 84, 73, 67, 65, 76, 32, 255, 1, 89, 8, 62, 69, 48, 9, - 82, 69, 67, 84, 65, 78, 71, 76, 69, 127, 66, 2, 11, 76, 2, 17, 2, 76, 73, - 2, 183, 25, 80, 5, 37, 7, 32, 87, 73, 84, 72, 32, 72, 2, 37, 7, 79, 82, - 73, 90, 79, 78, 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 66, 2, 219, 104, - 65, 2, 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, 2, 17, 2, 32, 83, - 2, 11, 81, 2, 11, 85, 2, 11, 65, 2, 227, 137, 1, 82, 2, 11, 69, 2, 11, - 68, 2, 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, 11, 87, 2, 241, - 4, 4, 65, 82, 68, 83, 100, 140, 1, 10, 68, 69, 45, 72, 69, 65, 68, 69, - 68, 32, 132, 4, 4, 71, 71, 76, 89, 124, 6, 76, 84, 69, 68, 32, 70, 42, - 78, 189, 1, 2, 82, 69, 80, 128, 1, 3, 76, 69, 70, 0, 4, 82, 73, 71, 72, - 12, 4, 68, 79, 87, 78, 0, 2, 85, 80, 32, 3, 78, 79, 82, 1, 3, 83, 79, 85, - 10, 11, 84, 10, 109, 5, 87, 65, 82, 68, 83, 20, 21, 3, 84, 72, 32, 20, - 32, 2, 69, 65, 1, 2, 87, 69, 10, 17, 2, 83, 84, 10, 11, 32, 10, 110, 72, - 0, 6, 86, 69, 82, 89, 32, 72, 28, 5, 76, 73, 71, 72, 84, 0, 6, 77, 69, - 68, 73, 85, 77, 23, 66, 2, 25, 4, 69, 65, 86, 89, 2, 17, 2, 32, 66, 2, - 21, 3, 65, 82, 66, 2, 11, 32, 2, 11, 65, 2, 11, 82, 2, 11, 82, 2, 131, - 28, 79, 2, 17, 2, 32, 86, 2, 11, 69, 2, 33, 6, 82, 84, 73, 67, 65, 76, 2, - 11, 32, 2, 11, 76, 2, 11, 73, 2, 215, 130, 1, 78, 2, 11, 76, 2, 11, 79, - 2, 147, 91, 87, 12, 46, 68, 106, 69, 186, 15, 75, 163, 136, 1, 71, 6, 22, - 32, 139, 26, 79, 4, 44, 2, 67, 72, 217, 15, 4, 66, 76, 79, 87, 2, 11, 73, - 2, 151, 129, 1, 77, 2, 11, 32, 2, 121, 3, 71, 76, 65, 4, 36, 5, 68, 32, - 75, 69, 89, 51, 76, 2, 17, 2, 66, 79, 2, 11, 65, 2, 203, 80, 82, 2, 11, - 69, 2, 243, 88, 83, 30, 66, 77, 158, 3, 82, 226, 11, 78, 226, 64, 79, - 129, 9, 2, 76, 70, 14, 36, 2, 65, 78, 161, 2, 2, 69, 78, 13, 72, 12, 32, - 87, 73, 84, 72, 32, 66, 85, 78, 78, 89, 32, 29, 2, 83, 32, 2, 11, 69, 2, - 167, 6, 65, 8, 48, 2, 66, 79, 28, 2, 67, 76, 54, 72, 19, 83, 2, 11, 79, - 2, 207, 86, 84, 2, 11, 79, 2, 11, 84, 2, 11, 72, 2, 155, 86, 69, 2, 187, - 119, 65, 2, 17, 2, 65, 78, 2, 131, 10, 68, 2, 11, 83, 2, 11, 32, 2, 11, - 83, 2, 11, 89, 2, 17, 2, 77, 66, 2, 187, 9, 79, 10, 72, 2, 68, 32, 156, - 1, 3, 76, 68, 32, 32, 2, 82, 73, 219, 144, 1, 77, 4, 56, 9, 83, 69, 80, - 65, 82, 65, 84, 79, 82, 203, 83, 74, 2, 17, 2, 32, 77, 2, 21, 3, 73, 68, - 68, 2, 11, 76, 2, 11, 69, 2, 11, 32, 2, 207, 64, 68, 2, 11, 77, 2, 223, - 144, 1, 65, 2, 11, 69, 2, 171, 83, 68, 10, 56, 7, 65, 80, 80, 69, 68, 32, - 80, 30, 69, 163, 1, 73, 2, 213, 76, 3, 82, 69, 83, 6, 48, 2, 65, 84, 80, - 3, 83, 84, 76, 187, 108, 78, 2, 11, 72, 2, 11, 32, 2, 11, 80, 2, 25, 4, - 82, 79, 68, 85, 2, 195, 114, 67, 2, 11, 69, 2, 247, 80, 82, 2, 11, 84, 2, - 11, 73, 2, 17, 2, 78, 71, 2, 17, 2, 32, 72, 2, 11, 65, 2, 215, 71, 78, - 34, 76, 2, 32, 73, 138, 1, 45, 28, 7, 73, 65, 78, 71, 81, 73, 32, 155, - 83, 79, 2, 53, 11, 78, 32, 65, 32, 82, 69, 67, 84, 65, 78, 71, 2, 11, 76, - 2, 11, 69, 2, 11, 32, 2, 11, 66, 2, 11, 79, 2, 163, 140, 1, 88, 2, 11, - 82, 2, 187, 122, 65, 28, 48, 5, 66, 76, 65, 67, 75, 1, 3, 82, 69, 68, 14, - 11, 32, 14, 130, 1, 67, 72, 4, 69, 76, 69, 80, 28, 4, 71, 69, 78, 69, 46, - 72, 36, 4, 83, 79, 76, 68, 169, 9, 6, 77, 65, 78, 68, 65, 82, 4, 24, 2, - 65, 78, 19, 72, 2, 167, 58, 78, 2, 201, 57, 3, 65, 82, 73, 2, 11, 72, 2, - 191, 70, 65, 2, 11, 82, 2, 11, 65, 2, 179, 137, 1, 76, 2, 17, 2, 79, 82, - 2, 183, 114, 83, 2, 143, 75, 73, 242, 19, 50, 65, 58, 69, 226, 11, 73, - 217, 35, 2, 79, 45, 2, 17, 2, 87, 78, 2, 11, 73, 2, 241, 74, 2, 78, 71, - 98, 60, 4, 76, 76, 79, 87, 62, 78, 53, 5, 90, 73, 68, 73, 32, 2, 17, 2, - 32, 72, 2, 11, 69, 2, 11, 65, 2, 227, 106, 82, 2, 11, 32, 2, 11, 83, 2, - 11, 73, 2, 251, 54, 71, 94, 112, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, - 32, 76, 5, 72, 89, 80, 72, 69, 17, 7, 76, 69, 84, 84, 69, 82, 32, 4, 44, - 3, 77, 65, 68, 13, 4, 72, 65, 77, 90, 2, 11, 68, 2, 215, 67, 65, 2, 215, - 27, 78, 88, 250, 1, 67, 50, 77, 18, 68, 42, 69, 58, 72, 30, 75, 22, 71, - 2, 81, 32, 3, 76, 65, 77, 98, 78, 18, 80, 30, 83, 66, 84, 28, 2, 86, 65, - 126, 87, 14, 79, 18, 88, 52, 3, 89, 79, 84, 154, 1, 90, 134, 53, 82, 238, - 48, 66, 254, 5, 85, 162, 14, 70, 3, 74, 6, 22, 72, 147, 114, 73, 4, 22, - 72, 251, 113, 73, 2, 247, 113, 73, 4, 11, 65, 4, 178, 130, 1, 68, 3, 76, - 8, 42, 76, 142, 50, 89, 226, 79, 84, 3, 87, 2, 71, 73, 4, 150, 112, 65, - 147, 15, 72, 4, 18, 72, 15, 65, 2, 11, 65, 2, 163, 129, 1, 70, 5, 11, 32, - 2, 11, 87, 2, 21, 3, 73, 84, 72, 2, 17, 2, 32, 68, 2, 11, 79, 2, 187, 3, - 84, 2, 207, 48, 85, 4, 202, 105, 72, 215, 22, 69, 8, 46, 72, 246, 47, 73, - 194, 9, 65, 163, 70, 69, 2, 243, 47, 73, 4, 238, 104, 72, 215, 22, 65, 5, - 37, 7, 32, 65, 76, 84, 69, 82, 78, 2, 17, 2, 65, 84, 2, 11, 69, 2, 11, - 32, 2, 11, 70, 2, 11, 79, 2, 227, 109, 82, 2, 11, 65, 2, 159, 126, 87, 4, - 22, 72, 251, 125, 65, 2, 11, 69, 2, 139, 46, 89, 5, 37, 7, 32, 87, 73, - 84, 72, 32, 67, 2, 45, 9, 73, 82, 67, 85, 77, 70, 76, 69, 88, 2, 11, 32, - 2, 11, 65, 2, 11, 66, 2, 11, 79, 2, 255, 101, 86, 6, 22, 65, 175, 124, - 69, 5, 171, 124, 76, 140, 19, 34, 32, 161, 35, 3, 78, 32, 89, 138, 19, - 88, 8, 82, 65, 68, 73, 67, 65, 76, 32, 181, 7, 9, 83, 89, 76, 76, 65, 66, - 76, 69, 32, 110, 170, 1, 66, 42, 67, 86, 68, 42, 71, 74, 72, 66, 74, 66, - 75, 30, 76, 30, 77, 26, 78, 74, 80, 26, 83, 74, 84, 30, 86, 30, 89, 30, - 90, 250, 35, 81, 198, 49, 87, 235, 30, 79, 4, 22, 66, 247, 64, 85, 2, - 163, 93, 85, 12, 42, 85, 18, 89, 178, 98, 72, 203, 22, 73, 2, 135, 121, - 79, 7, 130, 121, 80, 3, 84, 4, 22, 68, 215, 120, 85, 2, 247, 63, 85, 10, - 42, 71, 238, 91, 79, 162, 28, 69, 15, 65, 4, 162, 89, 85, 235, 30, 79, 8, - 22, 88, 243, 88, 77, 6, 238, 88, 85, 202, 2, 73, 163, 28, 79, 8, 22, 74, - 167, 119, 79, 6, 246, 90, 85, 218, 5, 73, 215, 22, 89, 4, 206, 90, 73, - 175, 28, 69, 6, 190, 54, 73, 199, 7, 89, 4, 182, 118, 79, 15, 73, 8, 30, - 89, 26, 90, 199, 90, 66, 4, 254, 117, 73, 3, 79, 2, 231, 117, 85, 4, 182, - 89, 85, 3, 89, 10, 22, 72, 223, 97, 83, 8, 214, 60, 85, 178, 28, 65, 162, - 28, 79, 15, 89, 4, 214, 88, 65, 175, 28, 85, 4, 138, 60, 85, 211, 56, 69, - 4, 158, 88, 73, 175, 28, 79, 10, 54, 85, 224, 62, 2, 90, 73, 238, 24, 79, - 175, 28, 65, 4, 246, 115, 80, 3, 82, 156, 18, 134, 2, 66, 134, 1, 67, - 162, 1, 68, 110, 70, 50, 71, 150, 1, 72, 138, 3, 73, 134, 1, 74, 98, 75, - 54, 76, 62, 77, 134, 1, 78, 234, 4, 80, 54, 81, 2, 89, 46, 82, 162, 1, - 83, 134, 1, 84, 102, 86, 82, 87, 58, 88, 50, 90, 140, 2, 2, 85, 79, 66, - 65, 2, 79, 107, 69, 132, 1, 66, 66, 238, 21, 85, 150, 1, 69, 26, 73, 42, - 65, 2, 79, 67, 89, 64, 234, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 3, - 89, 122, 66, 72, 238, 20, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, 89, - 54, 46, 85, 146, 22, 65, 2, 69, 2, 79, 67, 89, 19, 142, 22, 79, 106, 82, - 230, 88, 80, 3, 88, 106, 58, 68, 138, 15, 85, 134, 5, 73, 94, 69, 66, 65, - 3, 79, 54, 210, 19, 85, 58, 73, 94, 69, 66, 65, 3, 79, 42, 182, 20, 79, - 66, 65, 2, 73, 2, 89, 67, 85, 116, 58, 71, 214, 15, 85, 146, 4, 73, 42, - 65, 2, 69, 3, 79, 56, 50, 73, 162, 15, 85, 186, 4, 65, 2, 69, 3, 79, 13, - 150, 19, 69, 142, 90, 84, 3, 88, 246, 1, 78, 73, 30, 76, 58, 77, 54, 78, - 110, 88, 50, 85, 254, 15, 69, 66, 65, 3, 79, 6, 198, 19, 69, 231, 88, 84, - 64, 238, 16, 85, 58, 73, 94, 69, 2, 79, 66, 65, 67, 89, 58, 182, 16, 85, - 58, 73, 158, 1, 65, 2, 79, 35, 89, 42, 46, 79, 34, 85, 202, 16, 69, 26, - 73, 43, 65, 6, 242, 106, 80, 2, 84, 3, 88, 6, 238, 17, 79, 231, 88, 84, - 46, 46, 85, 254, 15, 69, 26, 73, 42, 65, 3, 79, 8, 187, 16, 79, 19, 42, - 84, 130, 16, 69, 206, 89, 80, 3, 88, 5, 11, 69, 2, 11, 82, 2, 11, 65, 2, - 11, 84, 2, 11, 73, 2, 11, 79, 2, 191, 39, 78, 106, 50, 74, 190, 10, 85, - 146, 4, 73, 42, 79, 67, 89, 50, 158, 13, 85, 174, 1, 73, 42, 79, 3, 89, - 56, 242, 12, 85, 58, 73, 158, 1, 65, 2, 69, 3, 79, 70, 218, 9, 85, 250, - 3, 69, 26, 73, 42, 65, 2, 79, 67, 89, 106, 70, 71, 218, 8, 85, 158, 3, - 73, 158, 1, 65, 2, 79, 2, 89, 107, 69, 44, 186, 11, 85, 150, 1, 69, 66, - 65, 2, 79, 105, 2, 73, 69, 248, 2, 102, 66, 54, 68, 94, 71, 90, 74, 46, - 82, 50, 89, 78, 90, 134, 7, 85, 58, 73, 94, 65, 2, 69, 67, 79, 54, 202, - 10, 73, 158, 1, 65, 2, 79, 66, 85, 3, 89, 46, 46, 73, 198, 10, 69, 66, - 65, 2, 79, 67, 85, 13, 234, 11, 69, 230, 88, 80, 2, 84, 3, 88, 34, 60, 2, - 85, 79, 218, 9, 69, 0, 2, 73, 69, 66, 65, 3, 79, 7, 226, 99, 84, 3, 88, - 50, 230, 1, 85, 242, 7, 73, 42, 79, 67, 89, 46, 146, 9, 79, 66, 65, 2, - 69, 66, 85, 3, 89, 38, 30, 85, 222, 8, 73, 43, 79, 15, 194, 8, 79, 142, - 90, 80, 2, 84, 3, 88, 56, 62, 85, 206, 4, 79, 178, 2, 73, 158, 1, 65, 66, - 89, 43, 69, 15, 254, 8, 79, 2, 82, 230, 88, 80, 3, 88, 60, 150, 6, 85, - 58, 73, 158, 1, 65, 2, 79, 67, 89, 56, 254, 2, 85, 146, 4, 73, 42, 79, - 67, 89, 100, 58, 82, 254, 4, 85, 150, 1, 69, 66, 65, 2, 79, 67, 89, 48, - 46, 85, 162, 6, 69, 2, 79, 66, 89, 43, 65, 17, 134, 7, 79, 2, 82, 230, - 88, 80, 2, 84, 3, 88, 176, 1, 70, 83, 158, 3, 72, 50, 85, 58, 73, 94, 69, - 66, 65, 2, 79, 67, 89, 56, 130, 4, 73, 94, 69, 66, 65, 2, 79, 2, 85, 67, - 89, 56, 46, 85, 158, 3, 73, 94, 69, 66, 65, 3, 79, 21, 182, 4, 79, 106, - 82, 230, 88, 80, 2, 84, 3, 88, 60, 54, 69, 166, 3, 73, 42, 65, 2, 79, 66, - 85, 3, 89, 4, 150, 93, 80, 3, 88, 28, 38, 85, 206, 2, 69, 2, 79, 67, 65, - 9, 203, 2, 79, 40, 210, 2, 73, 42, 79, 66, 89, 41, 2, 85, 79, 178, 1, 66, - 72, 50, 85, 58, 73, 42, 90, 54, 69, 66, 65, 2, 79, 67, 89, 54, 46, 85, - 214, 1, 65, 2, 69, 2, 79, 67, 89, 19, 146, 1, 79, 170, 1, 82, 230, 88, - 80, 2, 84, 3, 88, 15, 90, 69, 142, 90, 80, 2, 84, 3, 88, 58, 50, 69, 2, - 79, 26, 73, 42, 65, 34, 85, 35, 89, 7, 138, 90, 80, 3, 88, 17, 38, 69, - 206, 89, 80, 2, 84, 3, 88, 9, 202, 89, 80, 2, 84, 3, 88, 11, 70, 82, 230, - 88, 80, 3, 88, 13, 38, 82, 230, 88, 80, 2, 84, 3, 88, 5, 227, 88, 88, 2, - 219, 7, 65, 2, 207, 57, 89, 180, 4, 252, 1, 10, 32, 78, 79, 84, 65, 84, - 73, 79, 78, 32, 224, 6, 16, 65, 78, 65, 66, 65, 90, 65, 82, 32, 83, 81, - 85, 65, 82, 69, 32, 210, 15, 69, 180, 2, 6, 73, 80, 80, 69, 82, 45, 96, - 8, 78, 65, 77, 69, 78, 78, 89, 32, 188, 33, 3, 79, 77, 66, 235, 26, 87, - 26, 144, 1, 9, 66, 65, 71, 32, 77, 69, 77, 66, 69, 42, 82, 104, 6, 68, - 79, 77, 65, 73, 78, 60, 3, 76, 69, 70, 154, 1, 83, 145, 2, 3, 84, 89, 80, - 2, 11, 82, 2, 11, 83, 2, 215, 83, 72, 8, 100, 4, 65, 78, 71, 69, 60, 3, - 73, 71, 72, 221, 1, 11, 69, 76, 65, 84, 73, 79, 78, 65, 76, 32, 67, 2, - 173, 3, 11, 32, 65, 78, 84, 73, 82, 69, 83, 84, 82, 73, 4, 17, 2, 84, 32, - 4, 60, 4, 73, 77, 65, 71, 13, 7, 66, 73, 78, 68, 73, 78, 71, 2, 11, 69, - 2, 25, 4, 32, 66, 82, 65, 2, 11, 67, 2, 175, 29, 75, 8, 44, 6, 67, 72, - 69, 77, 65, 32, 211, 1, 80, 6, 18, 67, 71, 80, 2, 17, 2, 79, 77, 2, 11, - 80, 2, 11, 79, 2, 11, 83, 2, 107, 73, 4, 30, 73, 41, 3, 82, 79, 74, 2, - 11, 80, 2, 11, 73, 2, 163, 80, 78, 2, 11, 69, 2, 11, 67, 2, 11, 84, 2, - 87, 73, 2, 139, 52, 79, 2, 11, 69, 2, 11, 32, 2, 11, 67, 2, 11, 79, 2, - 11, 76, 2, 11, 79, 2, 223, 79, 78, 144, 1, 240, 1, 2, 67, 76, 68, 7, 73, - 78, 73, 84, 73, 65, 76, 204, 2, 14, 70, 73, 78, 65, 76, 32, 67, 79, 78, - 83, 79, 78, 65, 78, 16, 7, 76, 69, 84, 84, 69, 82, 32, 148, 3, 5, 77, 65, - 82, 75, 32, 206, 1, 83, 181, 3, 6, 86, 79, 87, 69, 76, 32, 14, 64, 5, 79, - 83, 73, 78, 71, 137, 1, 6, 85, 83, 84, 69, 82, 45, 4, 11, 32, 4, 64, 12, - 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, 68, 23, 72, 2, 17, 2, 32, 72, - 2, 17, 2, 69, 65, 2, 215, 10, 68, 10, 96, 13, 70, 73, 78, 65, 76, 32, 76, - 69, 84, 84, 69, 82, 32, 41, 7, 73, 78, 73, 84, 73, 65, 76, 8, 238, 72, - 76, 2, 82, 2, 86, 3, 89, 2, 37, 7, 32, 76, 69, 84, 84, 69, 82, 2, 151, 6, - 32, 2, 131, 9, 84, 82, 166, 1, 68, 50, 75, 38, 78, 46, 83, 38, 84, 46, - 66, 2, 67, 2, 71, 2, 80, 2, 90, 138, 69, 45, 2, 72, 2, 74, 2, 76, 2, 77, - 2, 82, 2, 86, 2, 89, 187, 2, 65, 12, 206, 1, 68, 2, 90, 138, 69, 72, 187, - 2, 65, 6, 154, 70, 83, 14, 72, 187, 2, 65, 8, 130, 70, 71, 2, 78, 2, 89, - 187, 2, 65, 6, 214, 69, 72, 2, 83, 187, 2, 65, 12, 42, 83, 2, 84, 138, - 69, 72, 187, 2, 65, 4, 134, 69, 72, 187, 2, 65, 8, 50, 68, 58, 83, 40, 4, - 76, 79, 78, 71, 23, 84, 2, 29, 5, 79, 85, 66, 76, 69, 2, 11, 32, 2, 11, - 83, 2, 11, 72, 2, 11, 65, 2, 159, 70, 68, 2, 17, 2, 32, 84, 2, 17, 2, 83, - 72, 2, 147, 69, 69, 14, 36, 4, 73, 71, 78, 32, 255, 2, 85, 12, 54, 65, - 72, 6, 67, 65, 78, 68, 82, 65, 167, 1, 86, 2, 11, 78, 2, 11, 85, 2, 17, - 2, 83, 86, 2, 11, 65, 2, 135, 66, 82, 6, 36, 5, 66, 73, 78, 68, 85, 15, - 32, 5, 11, 32, 2, 25, 4, 87, 73, 84, 72, 2, 17, 2, 32, 79, 2, 21, 3, 82, - 78, 65, 2, 11, 77, 2, 11, 69, 2, 239, 38, 78, 4, 11, 73, 4, 18, 82, 19, - 83, 2, 219, 33, 65, 2, 11, 65, 2, 11, 82, 2, 139, 64, 71, 2, 151, 4, 66, - 20, 38, 76, 109, 5, 83, 73, 71, 78, 32, 2, 17, 2, 69, 78, 2, 11, 71, 2, - 11, 84, 2, 11, 72, 2, 11, 32, 2, 11, 77, 2, 11, 65, 2, 135, 62, 82, 18, - 86, 65, 26, 79, 2, 85, 16, 8, 82, 69, 86, 69, 82, 83, 69, 68, 146, 64, - 69, 3, 73, 4, 182, 64, 73, 3, 85, 5, 159, 64, 69, 2, 183, 44, 32, 12, 72, - 2, 66, 82, 16, 9, 82, 79, 32, 87, 73, 68, 84, 72, 32, 203, 1, 85, 2, 147, - 2, 65, 8, 32, 2, 78, 79, 86, 83, 31, 74, 4, 26, 45, 73, 2, 78, 45, 2, 29, - 5, 66, 82, 69, 65, 75, 2, 11, 32, 2, 11, 83, 2, 163, 1, 80, 2, 11, 74, 2, - 11, 79, 2, 11, 73, 2, 11, 78, 2, 143, 5, 69, 2, 219, 61, 83, 2, 21, 3, - 77, 79, 85, 2, 17, 2, 84, 72, 2, 11, 32, 2, 11, 70, 2, 11, 65, 2, 167, - 38, 67, 242, 2, 168, 1, 10, 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 224, - 18, 6, 78, 69, 85, 77, 69, 32, 181, 38, 17, 80, 82, 73, 90, 78, 65, 75, - 32, 77, 79, 68, 73, 70, 73, 69, 82, 32, 128, 1, 144, 2, 17, 76, 79, 87, - 69, 82, 32, 84, 79, 78, 65, 76, 32, 82, 65, 78, 71, 69, 88, 5, 77, 65, - 82, 75, 32, 148, 3, 18, 65, 84, 84, 65, 67, 72, 73, 78, 71, 32, 86, 69, - 82, 84, 73, 67, 65, 76, 233, 11, 17, 84, 79, 78, 65, 76, 32, 82, 65, 78, - 71, 69, 32, 77, 65, 82, 75, 32, 2, 33, 6, 32, 73, 78, 68, 73, 67, 2, 11, - 65, 2, 11, 84, 2, 11, 79, 2, 219, 56, 82, 118, 182, 2, 67, 142, 1, 68, - 104, 8, 71, 79, 82, 65, 90, 68, 79, 32, 34, 78, 106, 75, 114, 79, 72, 2, - 80, 79, 152, 2, 8, 77, 65, 76, 79, 32, 80, 79, 86, 100, 2, 82, 65, 50, - 83, 186, 1, 84, 108, 7, 86, 89, 83, 79, 75, 79, 32, 234, 1, 90, 224, 10, - 2, 76, 79, 224, 22, 4, 85, 68, 65, 82, 129, 6, 4, 66, 79, 82, 90, 6, 60, - 6, 72, 65, 83, 72, 75, 65, 29, 5, 85, 82, 86, 69, 68, 5, 197, 47, 3, 32, - 80, 79, 2, 17, 2, 32, 79, 2, 11, 77, 2, 235, 24, 69, 4, 240, 10, 13, 69, - 77, 69, 83, 84, 86, 69, 78, 78, 89, 32, 90, 65, 217, 14, 6, 86, 79, 69, - 84, 79, 67, 10, 30, 78, 89, 3, 86, 89, 83, 8, 29, 5, 73, 90, 75, 79, 32, - 8, 164, 8, 8, 83, 32, 75, 82, 89, 90, 72, 69, 35, 79, 2, 167, 20, 79, 8, - 56, 4, 82, 89, 90, 72, 150, 6, 65, 233, 39, 2, 85, 80, 5, 21, 3, 32, 79, - 78, 2, 11, 32, 2, 203, 7, 76, 6, 220, 5, 3, 84, 83, 69, 200, 13, 5, 66, - 76, 65, 67, 72, 131, 31, 78, 18, 22, 68, 131, 2, 86, 6, 60, 7, 67, 72, - 65, 83, 72, 73, 69, 201, 22, 3, 86, 69, 82, 5, 17, 2, 32, 87, 2, 21, 3, - 73, 84, 72, 2, 17, 2, 32, 86, 2, 29, 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, - 76, 2, 11, 32, 2, 11, 83, 2, 11, 84, 2, 11, 82, 2, 11, 79, 2, 131, 25, - 75, 12, 29, 5, 89, 83, 72, 69, 32, 12, 22, 83, 251, 3, 79, 8, 154, 3, 32, - 229, 2, 4, 84, 82, 65, 78, 4, 28, 2, 90, 83, 183, 5, 86, 2, 219, 37, 69, - 10, 136, 1, 2, 75, 79, 16, 16, 84, 82, 65, 78, 78, 79, 32, 77, 65, 76, - 79, 32, 80, 79, 86, 89, 236, 1, 5, 82, 69, 68, 78, 69, 231, 26, 79, 2, - 239, 42, 66, 2, 11, 83, 2, 183, 22, 72, 10, 50, 79, 16, 5, 83, 65, 84, - 65, 32, 139, 36, 73, 2, 187, 28, 67, 6, 158, 1, 79, 201, 24, 3, 83, 32, - 75, 10, 24, 2, 83, 32, 95, 79, 6, 11, 75, 6, 44, 6, 72, 79, 75, 72, 76, - 79, 247, 24, 82, 4, 11, 77, 4, 17, 2, 32, 79, 4, 11, 78, 4, 11, 32, 4, - 18, 76, 31, 82, 2, 11, 69, 2, 179, 14, 70, 2, 11, 73, 2, 11, 71, 2, 139, - 14, 72, 6, 18, 65, 31, 69, 2, 249, 25, 3, 68, 69, 82, 4, 22, 86, 239, 10, - 76, 2, 199, 38, 79, 6, 60, 5, 77, 82, 65, 67, 72, 18, 83, 1, 4, 84, 82, - 69, 83, 2, 155, 10, 78, 2, 17, 2, 86, 69, 2, 155, 6, 84, 232, 1, 128, 2, - 2, 67, 72, 38, 68, 218, 1, 70, 28, 10, 71, 79, 76, 85, 66, 67, 72, 73, - 75, 32, 158, 1, 75, 160, 2, 6, 77, 69, 67, 72, 73, 75, 184, 1, 2, 78, 69, - 18, 79, 138, 2, 80, 146, 2, 82, 114, 83, 180, 20, 9, 86, 82, 65, 75, 72, - 73, 89, 65, 32, 151, 2, 90, 4, 246, 11, 69, 209, 10, 2, 65, 83, 10, 74, - 69, 98, 85, 16, 9, 86, 65, 32, 86, 32, 67, 72, 69, 76, 187, 27, 79, 4, - 76, 2, 82, 66, 213, 2, 12, 77, 69, 83, 84, 86, 69, 78, 78, 89, 32, 75, - 76, 2, 195, 34, 73, 2, 203, 34, 68, 2, 11, 78, 2, 231, 36, 85, 2, 11, 73, - 2, 147, 34, 84, 10, 78, 84, 38, 83, 240, 3, 5, 77, 82, 65, 67, 72, 145, - 14, 4, 66, 79, 82, 90, 4, 32, 3, 82, 69, 83, 251, 1, 73, 2, 21, 3, 86, - 69, 84, 2, 231, 17, 76, 14, 76, 4, 72, 65, 77, 73, 18, 76, 40, 3, 79, 66, - 89, 16, 2, 82, 89, 87, 85, 2, 219, 3, 76, 2, 11, 89, 2, 11, 85, 2, 151, - 33, 67, 2, 223, 31, 76, 6, 28, 2, 85, 75, 219, 32, 90, 5, 21, 3, 32, 84, - 73, 2, 11, 75, 2, 251, 15, 72, 2, 11, 70, 2, 11, 73, 2, 11, 83, 2, 215, - 30, 77, 11, 11, 32, 8, 44, 7, 75, 76, 89, 85, 67, 72, 69, 83, 80, 6, 64, - 9, 78, 69, 80, 79, 83, 84, 79, 89, 65, 14, 80, 163, 14, 86, 2, 39, 78, 2, - 25, 4, 79, 86, 79, 68, 2, 143, 14, 78, 2, 223, 22, 77, 14, 40, 2, 66, 76, - 41, 4, 83, 79, 75, 65, 2, 11, 65, 2, 11, 75, 2, 243, 30, 79, 13, 11, 32, - 10, 30, 75, 142, 26, 84, 39, 83, 6, 104, 11, 76, 89, 85, 67, 72, 69, 86, - 65, 89, 65, 32, 185, 25, 10, 82, 89, 85, 75, 79, 86, 65, 89, 65, 32, 4, - 202, 16, 78, 251, 8, 83, 14, 58, 65, 88, 8, 69, 82, 69, 86, 79, 68, 75, - 65, 27, 79, 6, 44, 3, 82, 65, 75, 222, 19, 76, 211, 5, 85, 2, 11, 76, 2, - 11, 73, 2, 171, 28, 84, 5, 153, 15, 2, 32, 78, 4, 68, 5, 68, 67, 72, 65, - 83, 245, 9, 7, 76, 75, 85, 76, 73, 90, 77, 2, 11, 72, 2, 219, 4, 73, 4, - 64, 11, 69, 86, 69, 82, 83, 69, 68, 32, 67, 72, 69, 139, 26, 79, 2, 25, - 4, 76, 89, 85, 83, 2, 215, 17, 84, 126, 96, 9, 75, 65, 77, 69, 89, 84, - 83, 65, 32, 148, 2, 8, 76, 79, 90, 72, 73, 84, 73, 69, 119, 84, 22, 128, - 1, 13, 68, 86, 79, 69, 67, 72, 69, 76, 78, 65, 89, 65, 32, 44, 7, 75, 76, - 89, 85, 67, 72, 69, 74, 84, 218, 18, 77, 119, 83, 8, 218, 10, 75, 110, - 78, 178, 8, 80, 75, 83, 6, 40, 5, 86, 65, 89, 65, 32, 243, 10, 78, 4, - 178, 15, 84, 183, 4, 83, 4, 162, 15, 73, 147, 4, 82, 9, 11, 32, 6, 48, 2, - 83, 32, 25, 6, 90, 65, 75, 82, 89, 84, 4, 166, 4, 75, 95, 90, 2, 11, 79, - 2, 211, 22, 69, 96, 92, 4, 65, 84, 89, 65, 172, 4, 6, 79, 80, 73, 84, 83, - 65, 189, 1, 5, 82, 69, 76, 65, 32, 23, 11, 32, 20, 76, 2, 83, 32, 236, 2, - 9, 90, 65, 75, 82, 89, 84, 65, 89, 65, 159, 5, 78, 14, 160, 1, 14, 68, - 86, 85, 77, 89, 65, 32, 90, 65, 80, 89, 65, 84, 89, 28, 7, 75, 82, 89, - 90, 72, 69, 77, 24, 2, 82, 79, 17, 8, 90, 65, 80, 89, 65, 84, 79, 89, 2, - 11, 77, 2, 215, 19, 73, 5, 189, 1, 2, 32, 73, 2, 203, 2, 71, 7, 21, 3, - 32, 73, 32, 4, 54, 75, 37, 9, 80, 79, 68, 67, 72, 65, 83, 72, 73, 2, 11, - 82, 2, 21, 3, 89, 90, 72, 2, 211, 1, 69, 5, 17, 2, 32, 83, 2, 17, 2, 32, - 90, 2, 29, 5, 65, 80, 89, 65, 84, 2, 11, 79, 2, 199, 17, 89, 7, 11, 32, - 4, 68, 6, 83, 32, 79, 67, 72, 75, 29, 7, 87, 73, 84, 72, 32, 83, 79, 2, - 11, 79, 2, 215, 16, 77, 2, 45, 9, 82, 79, 67, 72, 89, 65, 32, 78, 79, 2, - 11, 90, 2, 163, 7, 72, 68, 140, 1, 9, 68, 86, 79, 69, 67, 72, 69, 76, 78, - 162, 1, 75, 78, 78, 174, 1, 71, 152, 3, 8, 77, 82, 65, 67, 72, 78, 79, - 84, 42, 80, 75, 84, 10, 18, 79, 71, 65, 6, 64, 7, 80, 79, 86, 79, 68, 78, - 65, 189, 5, 4, 75, 82, 89, 90, 4, 17, 2, 89, 65, 5, 17, 2, 32, 75, 2, - 145, 5, 4, 76, 89, 85, 67, 26, 48, 6, 76, 89, 85, 67, 72, 69, 77, 2, 82, - 89, 4, 22, 78, 203, 6, 80, 2, 157, 8, 9, 69, 80, 79, 83, 84, 79, 89, 65, - 78, 22, 48, 7, 85, 75, 79, 86, 65, 89, 65, 195, 3, 90, 21, 11, 32, 18, - 54, 71, 236, 2, 5, 84, 82, 89, 65, 83, 179, 2, 80, 14, 21, 3, 82, 79, 77, - 14, 32, 4, 78, 65, 89, 65, 47, 79, 5, 237, 1, 7, 32, 87, 73, 84, 72, 32, - 83, 10, 92, 10, 75, 82, 89, 90, 72, 69, 86, 65, 89, 65, 17, 9, 80, 79, - 86, 79, 68, 78, 65, 89, 65, 5, 175, 2, 32, 7, 33, 6, 32, 87, 73, 84, 72, - 32, 4, 24, 2, 68, 79, 23, 83, 2, 41, 2, 85, 66, 2, 21, 3, 73, 78, 71, 2, - 133, 6, 6, 76, 69, 32, 90, 65, 80, 2, 175, 6, 75, 2, 237, 5, 3, 72, 69, - 86, 2, 11, 73, 2, 11, 75, 2, 187, 5, 72, 6, 22, 79, 187, 3, 82, 4, 28, 2, - 76, 85, 187, 1, 86, 2, 163, 1, 80, 8, 80, 5, 82, 89, 65, 83, 79, 153, 2, - 10, 73, 75, 72, 65, 89, 65, 32, 80, 85, 84, 6, 62, 80, 44, 4, 83, 84, 82, - 69, 173, 1, 4, 71, 76, 65, 83, 2, 17, 2, 79, 86, 2, 193, 1, 2, 79, 68, 2, - 171, 1, 76, 16, 88, 12, 75, 76, 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, - 38, 77, 46, 80, 38, 84, 39, 83, 8, 34, 77, 46, 80, 38, 84, 39, 83, 2, 25, - 4, 82, 65, 67, 72, 2, 247, 1, 78, 2, 11, 82, 2, 205, 1, 2, 79, 83, 2, 11, - 82, 2, 11, 69, 2, 11, 83, 2, 157, 1, 4, 86, 69, 84, 76, 6, 30, 65, 121, - 3, 77, 69, 89, 4, 32, 4, 78, 79, 90, 72, 31, 80, 2, 11, 69, 2, 151, 3, - 75, 2, 17, 2, 89, 65, 2, 11, 84, 2, 11, 65, 2, 35, 89, 2, 11, 84, 2, 11, - 83, 2, 183, 2, 65, 10, 108, 11, 68, 73, 82, 69, 67, 84, 73, 79, 78, 32, - 70, 28, 3, 75, 82, 89, 28, 5, 76, 69, 86, 69, 76, 35, 82, 2, 11, 76, 2, - 159, 1, 73, 2, 11, 90, 2, 143, 1, 72, 4, 11, 45, 4, 114, 50, 3, 51, 2, - 11, 79, 2, 83, 71, 8, 26, 78, 34, 83, 15, 74, 4, 18, 66, 27, 74, 2, 11, - 83, 2, 11, 80, 3, 0, + 131, 179, 1, 80, 4, 26, 82, 191, 139, 1, 87, 2, 17, 2, 69, 84, 2, 191, + 178, 1, 84, 4, 11, 32, 4, 18, 67, 23, 83, 2, 199, 200, 1, 85, 2, 143, 37, + 84, 2, 141, 64, 2, 79, 87, 16, 34, 65, 150, 1, 69, 231, 1, 79, 2, 25, 4, + 82, 68, 32, 83, 2, 45, 9, 72, 69, 76, 76, 32, 70, 76, 79, 80, 2, 17, 2, + 80, 89, 2, 17, 2, 32, 68, 2, 11, 73, 2, 231, 195, 1, 83, 10, 22, 65, 151, + 12, 88, 8, 30, 82, 29, 3, 86, 89, 32, 4, 11, 84, 5, 211, 15, 32, 4, 74, + 67, 53, 14, 83, 65, 76, 84, 73, 82, 69, 32, 87, 73, 84, 72, 32, 82, 2, + 17, 2, 72, 69, 2, 11, 67, 2, 231, 131, 1, 75, 2, 167, 19, 79, 4, 60, 8, + 82, 73, 90, 79, 78, 84, 65, 76, 249, 42, 2, 85, 82, 2, 201, 31, 2, 32, + 69, 22, 42, 65, 116, 3, 69, 70, 84, 175, 1, 79, 4, 24, 2, 82, 71, 19, 84, + 2, 243, 32, 69, 2, 11, 73, 2, 11, 78, 2, 17, 2, 32, 67, 2, 11, 82, 2, + 203, 42, 79, 12, 58, 32, 77, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, + 6, 44, 6, 76, 65, 78, 69, 32, 77, 247, 24, 80, 2, 11, 69, 2, 227, 10, 82, + 6, 150, 2, 80, 206, 24, 83, 51, 84, 6, 160, 1, 4, 87, 69, 82, 32, 221, 8, + 30, 90, 69, 78, 71, 69, 32, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, + 66, 76, 65, 67, 75, 32, 83, 77, 65, 76, 76, 32, 76, 4, 44, 3, 76, 69, 70, + 1, 4, 82, 73, 71, 72, 2, 11, 84, 2, 17, 2, 32, 80, 2, 203, 5, 79, 12, 68, + 6, 69, 68, 73, 85, 77, 32, 117, 7, 79, 79, 78, 32, 83, 69, 76, 10, 30, + 68, 54, 83, 227, 6, 76, 2, 11, 73, 2, 11, 65, 2, 11, 77, 2, 191, 46, 79, + 6, 134, 27, 84, 50, 77, 59, 81, 2, 11, 69, 2, 223, 187, 1, 78, 2, 11, 73, + 2, 247, 189, 1, 66, 6, 18, 65, 87, 69, 2, 37, 7, 82, 65, 76, 76, 69, 76, + 79, 2, 11, 71, 2, 11, 82, 2, 183, 172, 1, 65, 4, 11, 78, 4, 174, 2, 84, + 203, 46, 78, 2, 21, 3, 85, 69, 83, 2, 217, 120, 9, 84, 73, 79, 78, 32, + 77, 65, 82, 75, 14, 34, 69, 25, 4, 73, 71, 72, 84, 2, 129, 21, 2, 67, 84, + 12, 60, 10, 45, 80, 79, 73, 78, 84, 73, 78, 71, 32, 211, 17, 32, 8, 30, + 80, 202, 19, 83, 51, 84, 4, 18, 69, 55, 79, 2, 11, 78, 2, 11, 84, 2, 11, + 65, 2, 199, 106, 71, 2, 11, 73, 2, 11, 78, 2, 159, 124, 84, 56, 118, 67, + 32, 3, 69, 83, 65, 18, 72, 58, 77, 166, 1, 80, 68, 5, 81, 85, 65, 82, 69, + 180, 6, 2, 85, 78, 199, 12, 84, 2, 145, 40, 4, 73, 83, 83, 79, 2, 167, + 37, 77, 2, 21, 3, 79, 71, 73, 2, 189, 123, 4, 32, 80, 73, 69, 8, 32, 4, + 65, 76, 76, 32, 115, 73, 6, 18, 76, 71, 83, 2, 11, 79, 2, 11, 90, 2, 11, + 69, 2, 11, 78, 2, 223, 160, 1, 71, 4, 226, 19, 84, 107, 81, 2, 231, 46, + 76, 2, 21, 3, 65, 68, 69, 2, 11, 32, 2, 11, 83, 2, 143, 154, 1, 85, 29, + 11, 32, 26, 114, 66, 36, 17, 67, 79, 78, 84, 65, 73, 78, 73, 78, 71, 32, + 66, 76, 65, 67, 75, 32, 85, 5, 87, 73, 84, 72, 32, 2, 17, 2, 85, 84, 2, + 179, 101, 84, 6, 54, 86, 194, 17, 83, 37, 6, 77, 69, 68, 73, 85, 77, 2, + 169, 17, 3, 69, 82, 89, 18, 150, 1, 76, 50, 82, 214, 1, 85, 144, 1, 17, + 86, 69, 82, 84, 73, 67, 65, 76, 32, 66, 73, 83, 69, 67, 84, 73, 78, 249, + 19, 6, 67, 69, 78, 84, 82, 69, 6, 18, 69, 15, 79, 2, 67, 70, 4, 247, 1, + 87, 4, 18, 73, 115, 79, 2, 17, 2, 71, 72, 2, 33, 6, 84, 87, 65, 82, 68, + 83, 2, 11, 32, 2, 11, 84, 2, 11, 73, 2, 235, 174, 1, 67, 2, 29, 5, 85, + 78, 68, 69, 68, 2, 21, 3, 32, 67, 79, 2, 185, 32, 2, 82, 78, 4, 17, 2, + 80, 80, 4, 21, 3, 69, 82, 32, 4, 44, 3, 76, 69, 70, 1, 4, 82, 73, 71, 72, + 2, 33, 6, 84, 32, 81, 85, 65, 68, 2, 167, 36, 82, 2, 187, 20, 71, 11, 11, + 32, 8, 84, 12, 66, 69, 72, 73, 78, 68, 32, 67, 76, 79, 85, 68, 69, 5, 87, + 73, 84, 72, 32, 5, 29, 5, 32, 87, 73, 84, 72, 2, 17, 2, 32, 82, 2, 147, + 47, 65, 4, 38, 82, 29, 5, 83, 77, 65, 76, 76, 2, 11, 65, 2, 199, 112, 89, + 2, 11, 32, 2, 21, 3, 67, 76, 79, 2, 207, 103, 85, 10, 38, 79, 54, 69, 62, + 82, 203, 1, 87, 2, 49, 10, 85, 67, 72, 84, 79, 78, 69, 32, 84, 69, 2, 17, + 2, 76, 69, 2, 11, 80, 2, 11, 72, 2, 167, 17, 79, 4, 148, 1, 4, 65, 80, + 69, 90, 33, 28, 73, 65, 78, 71, 76, 69, 32, 67, 79, 78, 84, 65, 73, 78, + 73, 78, 71, 32, 83, 77, 65, 76, 76, 32, 87, 72, 73, 84, 2, 11, 73, 2, + 203, 154, 1, 85, 2, 135, 4, 69, 2, 11, 79, 2, 85, 19, 45, 87, 65, 89, 32, + 76, 69, 70, 84, 32, 87, 65, 89, 32, 84, 82, 65, 70, 70, 2, 11, 73, 2, + 143, 170, 1, 67, 12, 62, 32, 185, 1, 10, 45, 80, 79, 73, 78, 84, 73, 78, + 71, 32, 4, 11, 80, 4, 41, 8, 79, 73, 78, 84, 73, 78, 71, 32, 4, 18, 66, + 75, 73, 2, 21, 3, 65, 67, 75, 2, 25, 4, 72, 65, 78, 68, 2, 17, 2, 32, 73, + 2, 17, 2, 78, 68, 2, 211, 25, 69, 8, 54, 67, 42, 83, 125, 7, 84, 82, 73, + 65, 78, 71, 76, 2, 21, 3, 72, 69, 86, 2, 199, 87, 82, 2, 25, 4, 77, 65, + 76, 76, 2, 17, 2, 32, 84, 2, 17, 2, 82, 73, 2, 11, 65, 2, 11, 78, 2, 11, + 71, 2, 231, 143, 1, 76, 4, 11, 69, 5, 11, 32, 2, 11, 87, 2, 209, 18, 3, + 73, 84, 72, 10, 44, 6, 84, 73, 67, 65, 76, 32, 143, 2, 89, 8, 58, 69, 48, + 7, 82, 69, 67, 84, 65, 78, 71, 147, 1, 66, 2, 11, 76, 2, 17, 2, 76, 73, + 2, 183, 25, 80, 4, 17, 2, 76, 69, 5, 37, 7, 32, 87, 73, 84, 72, 32, 72, + 2, 37, 7, 79, 82, 73, 90, 79, 78, 84, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, + 66, 2, 223, 106, 65, 2, 17, 2, 32, 83, 2, 11, 77, 2, 21, 3, 65, 76, 76, + 2, 17, 2, 32, 83, 2, 11, 81, 2, 11, 85, 2, 11, 65, 2, 231, 139, 1, 82, 2, + 11, 69, 2, 11, 68, 2, 17, 2, 32, 82, 2, 21, 3, 73, 71, 72, 2, 11, 84, 2, + 11, 87, 2, 241, 4, 4, 65, 82, 68, 83, 100, 140, 1, 10, 68, 69, 45, 72, + 69, 65, 68, 69, 68, 32, 132, 4, 4, 71, 71, 76, 89, 124, 6, 76, 84, 69, + 68, 32, 70, 42, 78, 189, 1, 2, 82, 69, 80, 128, 1, 3, 76, 69, 70, 0, 4, + 82, 73, 71, 72, 12, 4, 68, 79, 87, 78, 0, 2, 85, 80, 32, 3, 78, 79, 82, + 1, 3, 83, 79, 85, 10, 11, 84, 10, 109, 5, 87, 65, 82, 68, 83, 20, 21, 3, + 84, 72, 32, 20, 32, 2, 69, 65, 1, 2, 87, 69, 10, 17, 2, 83, 84, 10, 11, + 32, 10, 110, 72, 0, 6, 86, 69, 82, 89, 32, 72, 28, 5, 76, 73, 71, 72, 84, + 0, 6, 77, 69, 68, 73, 85, 77, 23, 66, 2, 25, 4, 69, 65, 86, 89, 2, 17, 2, + 32, 66, 2, 21, 3, 65, 82, 66, 2, 11, 32, 2, 11, 65, 2, 11, 82, 2, 11, 82, + 2, 135, 30, 79, 2, 17, 2, 32, 86, 2, 11, 69, 2, 33, 6, 82, 84, 73, 67, + 65, 76, 2, 11, 32, 2, 11, 76, 2, 11, 73, 2, 219, 132, 1, 78, 2, 11, 76, + 2, 11, 79, 2, 151, 93, 87, 12, 46, 68, 106, 69, 190, 17, 75, 163, 136, 1, + 71, 6, 22, 32, 143, 28, 79, 4, 44, 2, 67, 72, 221, 17, 4, 66, 76, 79, 87, + 2, 11, 73, 2, 155, 131, 1, 77, 2, 11, 32, 2, 121, 3, 71, 76, 65, 4, 36, + 5, 68, 32, 75, 69, 89, 51, 76, 2, 17, 2, 66, 79, 2, 11, 65, 2, 207, 82, + 82, 2, 11, 69, 2, 247, 90, 83, 30, 66, 77, 134, 3, 82, 254, 13, 78, 226, + 64, 79, 129, 9, 2, 76, 70, 14, 36, 2, 65, 78, 137, 2, 2, 69, 78, 13, 72, + 12, 32, 87, 73, 84, 72, 32, 66, 85, 78, 78, 89, 32, 29, 2, 83, 32, 2, 11, + 69, 2, 143, 6, 65, 8, 52, 2, 66, 79, 16, 2, 67, 76, 54, 83, 199, 12, 72, + 2, 251, 13, 79, 2, 11, 79, 2, 11, 84, 2, 11, 72, 2, 167, 88, 69, 2, 17, + 2, 65, 78, 2, 131, 10, 68, 2, 11, 83, 2, 11, 32, 2, 11, 83, 2, 11, 89, 2, + 17, 2, 77, 66, 2, 187, 9, 79, 10, 72, 2, 68, 32, 156, 1, 3, 76, 68, 32, + 32, 2, 82, 73, 247, 146, 1, 77, 4, 56, 9, 83, 69, 80, 65, 82, 65, 84, 79, + 82, 231, 85, 74, 2, 17, 2, 32, 77, 2, 21, 3, 73, 68, 68, 2, 11, 76, 2, + 11, 69, 2, 11, 32, 2, 235, 66, 68, 2, 11, 77, 2, 251, 146, 1, 65, 2, 11, + 69, 2, 199, 85, 68, 10, 56, 7, 65, 80, 80, 69, 68, 32, 80, 30, 69, 163, + 1, 73, 2, 241, 78, 3, 82, 69, 83, 6, 48, 2, 65, 84, 80, 3, 83, 84, 76, + 215, 110, 78, 2, 11, 72, 2, 11, 32, 2, 11, 80, 2, 25, 4, 82, 79, 68, 85, + 2, 223, 116, 67, 2, 11, 69, 2, 147, 83, 82, 2, 11, 84, 2, 11, 73, 2, 17, + 2, 78, 71, 2, 17, 2, 32, 72, 2, 11, 65, 2, 243, 73, 78, 34, 76, 2, 32, + 73, 138, 1, 45, 28, 7, 73, 65, 78, 71, 81, 73, 32, 183, 85, 79, 2, 53, + 11, 78, 32, 65, 32, 82, 69, 67, 84, 65, 78, 71, 2, 11, 76, 2, 11, 69, 2, + 11, 32, 2, 11, 66, 2, 11, 79, 2, 191, 142, 1, 88, 2, 11, 82, 2, 215, 124, + 65, 28, 48, 5, 66, 76, 65, 67, 75, 1, 3, 82, 69, 68, 14, 11, 32, 14, 130, + 1, 67, 72, 4, 69, 76, 69, 80, 28, 4, 71, 69, 78, 69, 46, 72, 40, 4, 83, + 79, 76, 68, 193, 11, 6, 77, 65, 78, 68, 65, 82, 4, 24, 2, 65, 78, 19, 72, + 2, 195, 60, 78, 2, 229, 59, 3, 65, 82, 73, 2, 11, 72, 2, 219, 72, 65, 2, + 11, 82, 2, 11, 65, 2, 207, 139, 1, 76, 2, 11, 79, 2, 11, 82, 2, 207, 116, + 83, 2, 167, 77, 73, 248, 19, 54, 65, 206, 2, 69, 226, 11, 73, 217, 35, 2, + 79, 45, 8, 84, 16, 78, 71, 81, 73, 78, 32, 83, 73, 71, 78, 32, 83, 76, + 79, 87, 32, 199, 1, 87, 6, 28, 3, 79, 78, 69, 51, 84, 2, 17, 2, 32, 66, + 2, 11, 69, 2, 131, 109, 65, 4, 60, 9, 72, 82, 69, 69, 32, 72, 65, 76, 70, + 1, 2, 87, 79, 2, 21, 3, 32, 66, 69, 2, 11, 65, 2, 223, 74, 84, 2, 11, 78, + 2, 11, 73, 2, 241, 74, 2, 78, 71, 98, 60, 4, 76, 76, 79, 87, 62, 78, 53, + 5, 90, 73, 68, 73, 32, 2, 17, 2, 32, 72, 2, 11, 69, 2, 11, 65, 2, 227, + 106, 82, 2, 11, 32, 2, 11, 83, 2, 11, 73, 2, 251, 54, 71, 94, 112, 10, + 67, 79, 77, 66, 73, 78, 73, 78, 71, 32, 76, 5, 72, 89, 80, 72, 69, 17, 7, + 76, 69, 84, 84, 69, 82, 32, 4, 44, 3, 77, 65, 68, 13, 4, 72, 65, 77, 90, + 2, 11, 68, 2, 215, 67, 65, 2, 215, 27, 78, 88, 250, 1, 67, 50, 77, 18, + 68, 42, 69, 58, 72, 30, 75, 22, 71, 2, 81, 32, 3, 76, 65, 77, 98, 78, 18, + 80, 30, 83, 66, 84, 28, 2, 86, 65, 126, 87, 14, 79, 18, 88, 52, 3, 89, + 79, 84, 154, 1, 90, 134, 53, 82, 238, 48, 66, 254, 5, 85, 162, 14, 70, 3, + 74, 6, 22, 72, 147, 114, 73, 4, 22, 72, 251, 113, 73, 2, 247, 113, 73, 4, + 11, 65, 4, 178, 130, 1, 68, 3, 76, 8, 42, 76, 142, 50, 89, 226, 79, 84, + 3, 87, 2, 71, 73, 4, 150, 112, 65, 147, 15, 72, 4, 18, 72, 15, 65, 2, 11, + 65, 2, 163, 129, 1, 70, 5, 11, 32, 2, 11, 87, 2, 21, 3, 73, 84, 72, 2, + 17, 2, 32, 68, 2, 11, 79, 2, 187, 3, 84, 2, 207, 48, 85, 4, 202, 105, 72, + 215, 22, 69, 8, 46, 72, 246, 47, 73, 194, 9, 65, 163, 70, 69, 2, 243, 47, + 73, 4, 238, 104, 72, 215, 22, 65, 5, 37, 7, 32, 65, 76, 84, 69, 82, 78, + 2, 17, 2, 65, 84, 2, 11, 69, 2, 11, 32, 2, 11, 70, 2, 11, 79, 2, 227, + 109, 82, 2, 11, 65, 2, 159, 126, 87, 4, 22, 72, 251, 125, 65, 2, 11, 69, + 2, 139, 46, 89, 5, 41, 8, 32, 87, 73, 84, 72, 32, 67, 73, 2, 41, 8, 82, + 67, 85, 77, 70, 76, 69, 88, 2, 11, 32, 2, 11, 65, 2, 11, 66, 2, 11, 79, + 2, 255, 101, 86, 6, 22, 65, 175, 124, 69, 5, 171, 124, 76, 140, 19, 34, + 32, 161, 35, 3, 78, 32, 89, 138, 19, 88, 8, 82, 65, 68, 73, 67, 65, 76, + 32, 181, 7, 9, 83, 89, 76, 76, 65, 66, 76, 69, 32, 110, 170, 1, 66, 42, + 67, 86, 68, 42, 71, 74, 72, 66, 74, 66, 75, 30, 76, 30, 77, 26, 78, 74, + 80, 26, 83, 74, 84, 30, 86, 30, 89, 30, 90, 250, 35, 81, 198, 49, 87, + 235, 30, 79, 4, 22, 66, 247, 64, 85, 2, 163, 93, 85, 12, 42, 85, 18, 89, + 178, 98, 72, 203, 22, 73, 2, 135, 121, 79, 7, 130, 121, 80, 3, 84, 4, 22, + 68, 215, 120, 85, 2, 247, 63, 85, 10, 42, 71, 238, 91, 79, 162, 28, 69, + 15, 65, 4, 162, 89, 85, 235, 30, 79, 8, 22, 88, 243, 88, 77, 6, 238, 88, + 85, 202, 2, 73, 163, 28, 79, 8, 22, 74, 167, 119, 79, 6, 246, 90, 85, + 218, 5, 73, 215, 22, 89, 4, 206, 90, 73, 175, 28, 69, 6, 190, 54, 73, + 199, 7, 89, 4, 182, 118, 79, 15, 73, 8, 30, 89, 26, 90, 199, 90, 66, 4, + 254, 117, 73, 3, 79, 2, 231, 117, 85, 4, 182, 89, 85, 3, 89, 10, 22, 72, + 223, 97, 83, 8, 214, 60, 85, 178, 28, 65, 162, 28, 79, 15, 89, 4, 214, + 88, 65, 175, 28, 85, 4, 138, 60, 85, 211, 56, 69, 4, 158, 88, 73, 175, + 28, 79, 10, 54, 85, 224, 62, 2, 90, 73, 238, 24, 79, 175, 28, 65, 4, 246, + 115, 80, 3, 82, 156, 18, 134, 2, 66, 134, 1, 67, 162, 1, 68, 110, 70, 50, + 71, 150, 1, 72, 138, 3, 73, 134, 1, 74, 98, 75, 54, 76, 62, 77, 134, 1, + 78, 234, 4, 80, 54, 81, 2, 89, 46, 82, 162, 1, 83, 134, 1, 84, 102, 86, + 82, 87, 58, 88, 50, 90, 140, 2, 2, 85, 79, 66, 65, 2, 79, 107, 69, 132, + 1, 66, 66, 238, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, 89, 64, + 234, 21, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 3, 89, 122, 66, 72, 238, + 20, 85, 150, 1, 69, 26, 73, 42, 65, 2, 79, 67, 89, 54, 46, 85, 146, 22, + 65, 2, 69, 2, 79, 67, 89, 19, 142, 22, 79, 106, 82, 230, 88, 80, 3, 88, + 106, 58, 68, 138, 15, 85, 134, 5, 73, 94, 69, 66, 65, 3, 79, 54, 210, 19, + 85, 58, 73, 94, 69, 66, 65, 3, 79, 42, 182, 20, 79, 66, 65, 2, 73, 2, 89, + 67, 85, 116, 58, 71, 214, 15, 85, 146, 4, 73, 42, 65, 2, 69, 3, 79, 56, + 50, 73, 162, 15, 85, 186, 4, 65, 2, 69, 3, 79, 13, 150, 19, 69, 142, 90, + 84, 3, 88, 246, 1, 78, 73, 30, 76, 58, 77, 54, 78, 110, 88, 50, 85, 254, + 15, 69, 66, 65, 3, 79, 6, 198, 19, 69, 231, 88, 84, 64, 238, 16, 85, 58, + 73, 94, 69, 2, 79, 66, 65, 67, 89, 58, 182, 16, 85, 58, 73, 158, 1, 65, + 2, 79, 35, 89, 42, 46, 79, 34, 85, 202, 16, 69, 26, 73, 43, 65, 6, 242, + 106, 80, 2, 84, 3, 88, 6, 238, 17, 79, 231, 88, 84, 46, 46, 85, 254, 15, + 69, 26, 73, 42, 65, 3, 79, 8, 187, 16, 79, 19, 42, 84, 130, 16, 69, 206, + 89, 80, 3, 88, 5, 11, 69, 2, 11, 82, 2, 11, 65, 2, 11, 84, 2, 11, 73, 2, + 11, 79, 2, 191, 39, 78, 106, 50, 74, 190, 10, 85, 146, 4, 73, 42, 79, 67, + 89, 50, 158, 13, 85, 174, 1, 73, 42, 79, 3, 89, 56, 242, 12, 85, 58, 73, + 158, 1, 65, 2, 69, 3, 79, 70, 218, 9, 85, 250, 3, 69, 26, 73, 42, 65, 2, + 79, 67, 89, 106, 70, 71, 218, 8, 85, 158, 3, 73, 158, 1, 65, 2, 79, 2, + 89, 107, 69, 44, 186, 11, 85, 150, 1, 69, 66, 65, 2, 79, 105, 2, 73, 69, + 248, 2, 102, 66, 54, 68, 94, 71, 90, 74, 46, 82, 50, 89, 78, 90, 134, 7, + 85, 58, 73, 94, 65, 2, 69, 67, 79, 54, 202, 10, 73, 158, 1, 65, 2, 79, + 66, 85, 3, 89, 46, 46, 73, 198, 10, 69, 66, 65, 2, 79, 67, 85, 13, 234, + 11, 69, 230, 88, 80, 2, 84, 3, 88, 34, 60, 2, 85, 79, 218, 9, 69, 0, 2, + 73, 69, 66, 65, 3, 79, 7, 226, 99, 84, 3, 88, 50, 230, 1, 85, 242, 7, 73, + 42, 79, 67, 89, 46, 146, 9, 79, 66, 65, 2, 69, 66, 85, 3, 89, 38, 30, 85, + 222, 8, 73, 43, 79, 15, 194, 8, 79, 142, 90, 80, 2, 84, 3, 88, 56, 62, + 85, 206, 4, 79, 178, 2, 73, 158, 1, 65, 66, 89, 43, 69, 15, 254, 8, 79, + 2, 82, 230, 88, 80, 3, 88, 60, 150, 6, 85, 58, 73, 158, 1, 65, 2, 79, 67, + 89, 56, 254, 2, 85, 146, 4, 73, 42, 79, 67, 89, 100, 58, 82, 254, 4, 85, + 150, 1, 69, 66, 65, 2, 79, 67, 89, 48, 46, 85, 162, 6, 69, 2, 79, 66, 89, + 43, 65, 17, 134, 7, 79, 2, 82, 230, 88, 80, 2, 84, 3, 88, 176, 1, 70, 83, + 158, 3, 72, 50, 85, 58, 73, 94, 69, 66, 65, 2, 79, 67, 89, 56, 130, 4, + 73, 94, 69, 66, 65, 2, 79, 2, 85, 67, 89, 56, 46, 85, 158, 3, 73, 94, 69, + 66, 65, 3, 79, 21, 182, 4, 79, 106, 82, 230, 88, 80, 2, 84, 3, 88, 60, + 54, 69, 166, 3, 73, 42, 65, 2, 79, 66, 85, 3, 89, 4, 150, 93, 80, 3, 88, + 28, 38, 85, 206, 2, 69, 2, 79, 67, 65, 9, 203, 2, 79, 40, 210, 2, 73, 42, + 79, 66, 89, 41, 2, 85, 79, 178, 1, 66, 72, 50, 85, 58, 73, 42, 90, 54, + 69, 66, 65, 2, 79, 67, 89, 54, 46, 85, 214, 1, 65, 2, 69, 2, 79, 67, 89, + 19, 146, 1, 79, 170, 1, 82, 230, 88, 80, 2, 84, 3, 88, 15, 90, 69, 142, + 90, 80, 2, 84, 3, 88, 58, 50, 69, 2, 79, 26, 73, 42, 65, 34, 85, 35, 89, + 7, 138, 90, 80, 3, 88, 17, 38, 69, 206, 89, 80, 2, 84, 3, 88, 9, 202, 89, + 80, 2, 84, 3, 88, 11, 70, 82, 230, 88, 80, 3, 88, 13, 38, 82, 230, 88, + 80, 2, 84, 3, 88, 5, 227, 88, 88, 2, 219, 7, 65, 2, 207, 57, 89, 180, 4, + 252, 1, 10, 32, 78, 79, 84, 65, 84, 73, 79, 78, 32, 224, 6, 16, 65, 78, + 65, 66, 65, 90, 65, 82, 32, 83, 81, 85, 65, 82, 69, 32, 210, 15, 69, 180, + 2, 6, 73, 80, 80, 69, 82, 45, 96, 8, 78, 65, 77, 69, 78, 78, 89, 32, 188, + 33, 3, 79, 77, 66, 235, 26, 87, 26, 144, 1, 9, 66, 65, 71, 32, 77, 69, + 77, 66, 69, 42, 82, 104, 6, 68, 79, 77, 65, 73, 78, 60, 3, 76, 69, 70, + 154, 1, 83, 145, 2, 3, 84, 89, 80, 2, 11, 82, 2, 11, 83, 2, 215, 83, 72, + 8, 100, 4, 65, 78, 71, 69, 60, 3, 73, 71, 72, 221, 1, 11, 69, 76, 65, 84, + 73, 79, 78, 65, 76, 32, 67, 2, 173, 3, 11, 32, 65, 78, 84, 73, 82, 69, + 83, 84, 82, 73, 4, 17, 2, 84, 32, 4, 60, 4, 73, 77, 65, 71, 13, 7, 66, + 73, 78, 68, 73, 78, 71, 2, 11, 69, 2, 25, 4, 32, 66, 82, 65, 2, 11, 67, + 2, 175, 29, 75, 8, 44, 6, 67, 72, 69, 77, 65, 32, 211, 1, 80, 6, 18, 67, + 71, 80, 2, 17, 2, 79, 77, 2, 11, 80, 2, 11, 79, 2, 11, 83, 2, 107, 73, 4, + 30, 73, 41, 3, 82, 79, 74, 2, 11, 80, 2, 11, 73, 2, 163, 80, 78, 2, 11, + 69, 2, 11, 67, 2, 11, 84, 2, 87, 73, 2, 139, 52, 79, 2, 11, 69, 2, 11, + 32, 2, 11, 67, 2, 11, 79, 2, 11, 76, 2, 11, 79, 2, 223, 79, 78, 144, 1, + 240, 1, 2, 67, 76, 68, 7, 73, 78, 73, 84, 73, 65, 76, 204, 2, 14, 70, 73, + 78, 65, 76, 32, 67, 79, 78, 83, 79, 78, 65, 78, 16, 7, 76, 69, 84, 84, + 69, 82, 32, 148, 3, 5, 77, 65, 82, 75, 32, 206, 1, 83, 181, 3, 6, 86, 79, + 87, 69, 76, 32, 14, 64, 5, 79, 83, 73, 78, 71, 137, 1, 6, 85, 83, 84, 69, + 82, 45, 4, 11, 32, 4, 64, 12, 68, 79, 85, 66, 76, 69, 45, 76, 73, 78, 69, + 68, 23, 72, 2, 17, 2, 32, 72, 2, 17, 2, 69, 65, 2, 215, 10, 68, 10, 96, + 13, 70, 73, 78, 65, 76, 32, 76, 69, 84, 84, 69, 82, 32, 41, 7, 73, 78, + 73, 84, 73, 65, 76, 8, 238, 72, 76, 2, 82, 2, 86, 3, 89, 2, 37, 7, 32, + 76, 69, 84, 84, 69, 82, 2, 151, 6, 32, 2, 131, 9, 84, 82, 166, 1, 68, 50, + 75, 38, 78, 46, 83, 38, 84, 46, 66, 2, 67, 2, 71, 2, 80, 2, 90, 138, 69, + 45, 2, 72, 2, 74, 2, 76, 2, 77, 2, 82, 2, 86, 2, 89, 187, 2, 65, 12, 206, + 1, 68, 2, 90, 138, 69, 72, 187, 2, 65, 6, 154, 70, 83, 14, 72, 187, 2, + 65, 8, 130, 70, 71, 2, 78, 2, 89, 187, 2, 65, 6, 214, 69, 72, 2, 83, 187, + 2, 65, 12, 42, 83, 2, 84, 138, 69, 72, 187, 2, 65, 4, 134, 69, 72, 187, + 2, 65, 8, 50, 68, 58, 83, 40, 4, 76, 79, 78, 71, 23, 84, 2, 29, 5, 79, + 85, 66, 76, 69, 2, 11, 32, 2, 11, 83, 2, 11, 72, 2, 11, 65, 2, 159, 70, + 68, 2, 17, 2, 32, 84, 2, 17, 2, 83, 72, 2, 147, 69, 69, 14, 36, 4, 73, + 71, 78, 32, 255, 2, 85, 12, 54, 65, 72, 6, 67, 65, 78, 68, 82, 65, 167, + 1, 86, 2, 11, 78, 2, 11, 85, 2, 17, 2, 83, 86, 2, 11, 65, 2, 135, 66, 82, + 6, 36, 5, 66, 73, 78, 68, 85, 15, 32, 5, 11, 32, 2, 25, 4, 87, 73, 84, + 72, 2, 17, 2, 32, 79, 2, 21, 3, 82, 78, 65, 2, 11, 77, 2, 11, 69, 2, 239, + 38, 78, 4, 11, 73, 4, 18, 82, 19, 83, 2, 219, 33, 65, 2, 11, 65, 2, 11, + 82, 2, 139, 64, 71, 2, 151, 4, 66, 20, 38, 76, 109, 5, 83, 73, 71, 78, + 32, 2, 17, 2, 69, 78, 2, 11, 71, 2, 11, 84, 2, 11, 72, 2, 11, 32, 2, 11, + 77, 2, 11, 65, 2, 135, 62, 82, 18, 86, 65, 26, 79, 2, 85, 16, 8, 82, 69, + 86, 69, 82, 83, 69, 68, 146, 64, 69, 3, 73, 4, 182, 64, 73, 3, 85, 5, + 159, 64, 69, 2, 183, 44, 32, 12, 72, 2, 66, 82, 16, 9, 82, 79, 32, 87, + 73, 68, 84, 72, 32, 203, 1, 85, 2, 147, 2, 65, 8, 32, 2, 78, 79, 86, 83, + 31, 74, 4, 26, 45, 73, 2, 78, 45, 2, 29, 5, 66, 82, 69, 65, 75, 2, 11, + 32, 2, 11, 83, 2, 163, 1, 80, 2, 11, 74, 2, 11, 79, 2, 11, 73, 2, 11, 78, + 2, 143, 5, 69, 2, 219, 61, 83, 2, 21, 3, 77, 79, 85, 2, 17, 2, 84, 72, 2, + 11, 32, 2, 11, 70, 2, 11, 65, 2, 167, 38, 67, 242, 2, 168, 1, 10, 67, 79, + 77, 66, 73, 78, 73, 78, 71, 32, 224, 18, 6, 78, 69, 85, 77, 69, 32, 181, + 38, 17, 80, 82, 73, 90, 78, 65, 75, 32, 77, 79, 68, 73, 70, 73, 69, 82, + 32, 128, 1, 144, 2, 17, 76, 79, 87, 69, 82, 32, 84, 79, 78, 65, 76, 32, + 82, 65, 78, 71, 69, 88, 5, 77, 65, 82, 75, 32, 148, 3, 18, 65, 84, 84, + 65, 67, 72, 73, 78, 71, 32, 86, 69, 82, 84, 73, 67, 65, 76, 233, 11, 17, + 84, 79, 78, 65, 76, 32, 82, 65, 78, 71, 69, 32, 77, 65, 82, 75, 32, 2, + 33, 6, 32, 73, 78, 68, 73, 67, 2, 11, 65, 2, 11, 84, 2, 11, 79, 2, 219, + 56, 82, 118, 182, 2, 67, 142, 1, 68, 104, 8, 71, 79, 82, 65, 90, 68, 79, + 32, 34, 78, 106, 75, 114, 79, 72, 2, 80, 79, 152, 2, 8, 77, 65, 76, 79, + 32, 80, 79, 86, 100, 2, 82, 65, 50, 83, 186, 1, 84, 108, 7, 86, 89, 83, + 79, 75, 79, 32, 234, 1, 90, 224, 10, 2, 76, 79, 224, 22, 4, 85, 68, 65, + 82, 129, 6, 4, 66, 79, 82, 90, 6, 60, 6, 72, 65, 83, 72, 75, 65, 29, 5, + 85, 82, 86, 69, 68, 5, 197, 47, 3, 32, 80, 79, 2, 17, 2, 32, 79, 2, 11, + 77, 2, 235, 24, 69, 4, 240, 10, 13, 69, 77, 69, 83, 84, 86, 69, 78, 78, + 89, 32, 90, 65, 217, 14, 6, 86, 79, 69, 84, 79, 67, 10, 30, 78, 89, 3, + 86, 89, 83, 8, 29, 5, 73, 90, 75, 79, 32, 8, 164, 8, 8, 83, 32, 75, 82, + 89, 90, 72, 69, 35, 79, 2, 167, 20, 79, 8, 56, 4, 82, 89, 90, 72, 150, 6, + 65, 233, 39, 2, 85, 80, 5, 21, 3, 32, 79, 78, 2, 11, 32, 2, 203, 7, 76, + 6, 220, 5, 3, 84, 83, 69, 200, 13, 5, 66, 76, 65, 67, 72, 131, 31, 78, + 18, 22, 68, 131, 2, 86, 6, 60, 7, 67, 72, 65, 83, 72, 73, 69, 201, 22, 3, + 86, 69, 82, 5, 17, 2, 32, 87, 2, 21, 3, 73, 84, 72, 2, 17, 2, 32, 86, 2, + 29, 5, 69, 82, 84, 73, 67, 2, 17, 2, 65, 76, 2, 11, 32, 2, 11, 83, 2, 11, + 84, 2, 11, 82, 2, 11, 79, 2, 131, 25, 75, 12, 29, 5, 89, 83, 72, 69, 32, + 12, 22, 83, 251, 3, 79, 8, 154, 3, 32, 229, 2, 4, 84, 82, 65, 78, 4, 28, + 2, 90, 83, 183, 5, 86, 2, 219, 37, 69, 10, 136, 1, 2, 75, 79, 16, 16, 84, + 82, 65, 78, 78, 79, 32, 77, 65, 76, 79, 32, 80, 79, 86, 89, 236, 1, 5, + 82, 69, 68, 78, 69, 231, 26, 79, 2, 239, 42, 66, 2, 11, 83, 2, 183, 22, + 72, 10, 50, 79, 16, 5, 83, 65, 84, 65, 32, 139, 36, 73, 2, 187, 28, 67, + 6, 158, 1, 79, 201, 24, 3, 83, 32, 75, 10, 24, 2, 83, 32, 95, 79, 6, 11, + 75, 6, 44, 6, 72, 79, 75, 72, 76, 79, 247, 24, 82, 4, 11, 77, 4, 17, 2, + 32, 79, 4, 11, 78, 4, 11, 32, 4, 18, 76, 31, 82, 2, 11, 69, 2, 179, 14, + 70, 2, 11, 73, 2, 11, 71, 2, 139, 14, 72, 6, 18, 65, 31, 69, 2, 249, 25, + 3, 68, 69, 82, 4, 22, 86, 239, 10, 76, 2, 199, 38, 79, 6, 60, 5, 77, 82, + 65, 67, 72, 18, 83, 1, 4, 84, 82, 69, 83, 2, 155, 10, 78, 2, 17, 2, 86, + 69, 2, 155, 6, 84, 232, 1, 128, 2, 2, 67, 72, 38, 68, 218, 1, 70, 28, 10, + 71, 79, 76, 85, 66, 67, 72, 73, 75, 32, 158, 1, 75, 160, 2, 6, 77, 69, + 67, 72, 73, 75, 184, 1, 2, 78, 69, 18, 79, 138, 2, 80, 146, 2, 82, 114, + 83, 180, 20, 9, 86, 82, 65, 75, 72, 73, 89, 65, 32, 151, 2, 90, 4, 246, + 11, 69, 209, 10, 2, 65, 83, 10, 74, 69, 98, 85, 16, 9, 86, 65, 32, 86, + 32, 67, 72, 69, 76, 187, 27, 79, 4, 76, 2, 82, 66, 213, 2, 12, 77, 69, + 83, 84, 86, 69, 78, 78, 89, 32, 75, 76, 2, 195, 34, 73, 2, 203, 34, 68, + 2, 11, 78, 2, 231, 36, 85, 2, 11, 73, 2, 147, 34, 84, 10, 78, 84, 38, 83, + 240, 3, 5, 77, 82, 65, 67, 72, 145, 14, 4, 66, 79, 82, 90, 4, 32, 3, 82, + 69, 83, 251, 1, 73, 2, 21, 3, 86, 69, 84, 2, 231, 17, 76, 14, 76, 4, 72, + 65, 77, 73, 18, 76, 40, 3, 79, 66, 89, 16, 2, 82, 89, 87, 85, 2, 219, 3, + 76, 2, 11, 89, 2, 11, 85, 2, 151, 33, 67, 2, 223, 31, 76, 6, 28, 2, 85, + 75, 219, 32, 90, 5, 21, 3, 32, 84, 73, 2, 11, 75, 2, 251, 15, 72, 2, 11, + 70, 2, 11, 73, 2, 11, 83, 2, 215, 30, 77, 11, 11, 32, 8, 44, 7, 75, 76, + 89, 85, 67, 72, 69, 83, 80, 6, 64, 9, 78, 69, 80, 79, 83, 84, 79, 89, 65, + 14, 80, 163, 14, 86, 2, 39, 78, 2, 25, 4, 79, 86, 79, 68, 2, 143, 14, 78, + 2, 223, 22, 77, 14, 40, 2, 66, 76, 41, 4, 83, 79, 75, 65, 2, 11, 65, 2, + 11, 75, 2, 243, 30, 79, 13, 11, 32, 10, 30, 75, 142, 26, 84, 39, 83, 6, + 104, 11, 76, 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, 185, 25, 10, 82, 89, + 85, 75, 79, 86, 65, 89, 65, 32, 4, 202, 16, 78, 251, 8, 83, 14, 58, 65, + 88, 8, 69, 82, 69, 86, 79, 68, 75, 65, 27, 79, 6, 44, 3, 82, 65, 75, 222, + 19, 76, 211, 5, 85, 2, 11, 76, 2, 11, 73, 2, 171, 28, 84, 5, 153, 15, 2, + 32, 78, 4, 68, 5, 68, 67, 72, 65, 83, 245, 9, 7, 76, 75, 85, 76, 73, 90, + 77, 2, 11, 72, 2, 219, 4, 73, 4, 64, 11, 69, 86, 69, 82, 83, 69, 68, 32, + 67, 72, 69, 139, 26, 79, 2, 25, 4, 76, 89, 85, 83, 2, 215, 17, 84, 126, + 96, 9, 75, 65, 77, 69, 89, 84, 83, 65, 32, 148, 2, 8, 76, 79, 90, 72, 73, + 84, 73, 69, 119, 84, 22, 128, 1, 13, 68, 86, 79, 69, 67, 72, 69, 76, 78, + 65, 89, 65, 32, 44, 7, 75, 76, 89, 85, 67, 72, 69, 74, 84, 218, 18, 77, + 119, 83, 8, 218, 10, 75, 110, 78, 178, 8, 80, 75, 83, 6, 40, 5, 86, 65, + 89, 65, 32, 243, 10, 78, 4, 178, 15, 84, 183, 4, 83, 4, 162, 15, 73, 147, + 4, 82, 9, 11, 32, 6, 48, 2, 83, 32, 25, 6, 90, 65, 75, 82, 89, 84, 4, + 166, 4, 75, 95, 90, 2, 11, 79, 2, 211, 22, 69, 96, 92, 4, 65, 84, 89, 65, + 172, 4, 6, 79, 80, 73, 84, 83, 65, 189, 1, 5, 82, 69, 76, 65, 32, 23, 11, + 32, 20, 76, 2, 83, 32, 236, 2, 9, 90, 65, 75, 82, 89, 84, 65, 89, 65, + 159, 5, 78, 14, 160, 1, 14, 68, 86, 85, 77, 89, 65, 32, 90, 65, 80, 89, + 65, 84, 89, 28, 7, 75, 82, 89, 90, 72, 69, 77, 24, 2, 82, 79, 17, 8, 90, + 65, 80, 89, 65, 84, 79, 89, 2, 11, 77, 2, 215, 19, 73, 5, 189, 1, 2, 32, + 73, 2, 203, 2, 71, 7, 21, 3, 32, 73, 32, 4, 54, 75, 37, 9, 80, 79, 68, + 67, 72, 65, 83, 72, 73, 2, 11, 82, 2, 21, 3, 89, 90, 72, 2, 211, 1, 69, + 5, 17, 2, 32, 83, 2, 17, 2, 32, 90, 2, 29, 5, 65, 80, 89, 65, 84, 2, 11, + 79, 2, 199, 17, 89, 7, 11, 32, 4, 68, 6, 83, 32, 79, 67, 72, 75, 29, 7, + 87, 73, 84, 72, 32, 83, 79, 2, 11, 79, 2, 215, 16, 77, 2, 45, 9, 82, 79, + 67, 72, 89, 65, 32, 78, 79, 2, 11, 90, 2, 163, 7, 72, 68, 140, 1, 9, 68, + 86, 79, 69, 67, 72, 69, 76, 78, 162, 1, 75, 78, 78, 174, 1, 71, 152, 3, + 8, 77, 82, 65, 67, 72, 78, 79, 84, 42, 80, 75, 84, 10, 18, 79, 71, 65, 6, + 64, 7, 80, 79, 86, 79, 68, 78, 65, 189, 5, 4, 75, 82, 89, 90, 4, 17, 2, + 89, 65, 5, 17, 2, 32, 75, 2, 145, 5, 4, 76, 89, 85, 67, 26, 48, 6, 76, + 89, 85, 67, 72, 69, 77, 2, 82, 89, 4, 22, 78, 203, 6, 80, 2, 157, 8, 9, + 69, 80, 79, 83, 84, 79, 89, 65, 78, 22, 48, 7, 85, 75, 79, 86, 65, 89, + 65, 195, 3, 90, 21, 11, 32, 18, 54, 71, 236, 2, 5, 84, 82, 89, 65, 83, + 179, 2, 80, 14, 21, 3, 82, 79, 77, 14, 32, 4, 78, 65, 89, 65, 47, 79, 5, + 237, 1, 7, 32, 87, 73, 84, 72, 32, 83, 10, 92, 10, 75, 82, 89, 90, 72, + 69, 86, 65, 89, 65, 17, 9, 80, 79, 86, 79, 68, 78, 65, 89, 65, 5, 175, 2, + 32, 7, 33, 6, 32, 87, 73, 84, 72, 32, 4, 24, 2, 68, 79, 23, 83, 2, 41, 2, + 85, 66, 2, 21, 3, 73, 78, 71, 2, 133, 6, 6, 76, 69, 32, 90, 65, 80, 2, + 175, 6, 75, 2, 237, 5, 3, 72, 69, 86, 2, 11, 73, 2, 11, 75, 2, 187, 5, + 72, 6, 22, 79, 187, 3, 82, 4, 28, 2, 76, 85, 187, 1, 86, 2, 163, 1, 80, + 8, 80, 5, 82, 89, 65, 83, 79, 153, 2, 10, 73, 75, 72, 65, 89, 65, 32, 80, + 85, 84, 6, 62, 80, 44, 4, 83, 84, 82, 69, 173, 1, 4, 71, 76, 65, 83, 2, + 17, 2, 79, 86, 2, 193, 1, 2, 79, 68, 2, 171, 1, 76, 16, 88, 12, 75, 76, + 89, 85, 67, 72, 69, 86, 65, 89, 65, 32, 38, 77, 46, 80, 38, 84, 39, 83, + 8, 34, 77, 46, 80, 38, 84, 39, 83, 2, 25, 4, 82, 65, 67, 72, 2, 247, 1, + 78, 2, 11, 82, 2, 205, 1, 2, 79, 83, 2, 11, 82, 2, 11, 69, 2, 11, 83, 2, + 157, 1, 4, 86, 69, 84, 76, 6, 30, 65, 121, 3, 77, 69, 89, 4, 32, 4, 78, + 79, 90, 72, 31, 80, 2, 11, 69, 2, 151, 3, 75, 2, 17, 2, 89, 65, 2, 11, + 84, 2, 11, 65, 2, 35, 89, 2, 11, 84, 2, 11, 83, 2, 183, 2, 65, 10, 108, + 11, 68, 73, 82, 69, 67, 84, 73, 79, 78, 32, 70, 28, 3, 75, 82, 89, 28, 5, + 76, 69, 86, 69, 76, 35, 82, 2, 11, 76, 2, 159, 1, 73, 2, 11, 90, 2, 143, + 1, 72, 4, 11, 45, 4, 114, 50, 3, 51, 2, 11, 79, 2, 83, 71, 8, 26, 78, 34, + 83, 15, 74, 4, 18, 66, 27, 74, 2, 11, 83, 2, 11, 80, 3, 0, }; static const unsigned int dawg_pos_to_codepoint[] = { @@ -9248,809 +9353,821 @@ static const unsigned int dawg_pos_to_codepoint[] = { 71425, 71424, 71431, 71430, 71441, 71433, 71437, 71439, 71484, 71485, 71467, 71486, 71487, 71456, 71457, 71465, 71466, 71463, 71460, 71461, 71458, 71459, 71462, 71464, 71483, 71482, 71477, 71476, 71479, 71478, - 71475, 71474, 71472, 71481, 71473, 71480, 9992, 128747, 128748, 128874, - 128822, 128823, 128837, 128776, 128777, 128774, 128775, 128773, 128811, - 128859, 128826, 128829, 128855, 128769, 128875, 128876, 128830, 128834, - 128835, 128836, 128783, 128857, 128848, 128846, 128844, 128805, 128800, - 128803, 128804, 128798, 128869, 128870, 128871, 128872, 128873, 128787, - 128865, 128866, 128864, 128880, 128794, 128841, 128883, 128882, 128854, - 128878, 128796, 128797, 128801, 128810, 128851, 128784, 128785, 128786, - 128824, 128881, 128789, 128879, 128782, 128843, 128858, 128856, 128868, - 128867, 128863, 128768, 128833, 128818, 128816, 128817, 128799, 128819, - 128820, 128821, 128827, 128828, 128877, 128792, 128793, 128788, 128806, - 128813, 128825, 128850, 128860, 128861, 128814, 128807, 128812, 128802, - 128862, 128781, 128839, 128795, 128852, 128847, 128831, 128832, 128840, - 128809, 128849, 128845, 128778, 128815, 128779, 128780, 128790, 128791, - 128808, 128772, 128842, 128853, 128771, 128838, 128770, 9879, 8501, - 983054, 128126, 117837, 117836, 117844, 117845, 117842, 117843, 117839, - 117838, 117841, 117840, 8780, 9006, 983203, 8776, 10863, 8778, 9095, - 9941, 9200, 38, 127994, 127944, 128657, 10815, 82970, 82971, 82964, - 82965, 82966, 82967, 82968, 82969, 82972, 82973, 82974, 82994, 82995, - 82996, 82987, 82988, 82992, 82993, 82986, 82989, 82990, 82991, 82997, - 82998, 82999, 83016, 83017, 83018, 83019, 83010, 83011, 83012, 83013, - 83014, 83015, 83020, 83021, 83022, 83050, 83051, 83052, 83053, 83043, - 83044, 83045, 83046, 83047, 83048, 83049, 83054, 82984, 82985, 82975, - 82976, 82977, 82978, 82979, 82980, 82981, 82982, 82983, 82953, 82954, - 82955, 82956, 82957, 82958, 82959, 82960, 82961, 82962, 82963, 82944, - 82945, 82946, 82947, 82948, 82949, 82950, 82951, 82952, 83000, 83001, - 83002, 83003, 83004, 83005, 83006, 83007, 83008, 83009, 83023, 83024, - 83025, 83026, 83027, 83028, 83029, 83030, 83031, 83032, 83033, 83034, - 83035, 83036, 83037, 83038, 83039, 83040, 83041, 83042, 83062, 83063, - 83064, 83065, 83070, 83071, 83072, 83073, 83066, 83067, 83068, 83055, - 83056, 83057, 83058, 83059, 83060, 83061, 83069, 83074, 83075, 83076, - 83077, 83078, 83083, 83084, 83079, 83080, 83081, 83082, 83085, 83086, - 83087, 83088, 83094, 83095, 83089, 83090, 83091, 83092, 83093, 83096, - 83097, 83098, 83099, 83105, 83106, 83100, 83101, 83102, 83103, 83104, - 83107, 83108, 83109, 83110, 83111, 83112, 83113, 83114, 83115, 83116, - 83117, 83118, 83119, 83120, 83121, 83122, 83123, 83124, 83125, 83126, - 83127, 83128, 83129, 83130, 83131, 83132, 83133, 83134, 83135, 83136, - 83137, 83138, 83139, 83140, 83141, 83142, 83143, 83144, 83145, 83146, - 83147, 83148, 83149, 83150, 83151, 83152, 83153, 83154, 83155, 83156, - 83157, 83158, 83159, 83160, 83161, 83162, 83163, 83164, 83165, 83166, - 83167, 83168, 83169, 83170, 83173, 83174, 83175, 83180, 83181, 83183, - 83184, 83171, 83172, 83176, 83177, 83178, 83179, 83182, 83190, 83191, - 83192, 83193, 83185, 83186, 83187, 83188, 83189, 83194, 83195, 83196, - 83274, 83275, 83280, 83281, 83270, 83271, 83272, 83273, 83276, 83277, - 83278, 83279, 83268, 83269, 83259, 83260, 83261, 83262, 83263, 83264, - 83265, 83266, 83267, 83204, 83205, 83197, 83198, 83199, 83200, 83201, - 83202, 83203, 83206, 83207, 83245, 83246, 83238, 83239, 83240, 83241, - 83242, 83243, 83244, 83247, 83248, 83208, 83209, 83210, 83211, 83212, - 83213, 83214, 83215, 83216, 83217, 83218, 83219, 83220, 83221, 83222, - 83223, 83224, 83225, 83226, 83227, 83228, 83229, 83230, 83231, 83232, - 83233, 83234, 83235, 83236, 83237, 83249, 83250, 83251, 83252, 83253, - 83254, 83255, 83256, 83257, 83258, 83291, 83292, 83282, 83283, 83284, - 83285, 83286, 83287, 83288, 83289, 83290, 83312, 83313, 83303, 83304, - 83305, 83306, 83307, 83308, 83309, 83310, 83311, 83348, 83349, 83339, - 83340, 83341, 83342, 83343, 83344, 83345, 83346, 83347, 83322, 83323, - 83324, 83325, 83316, 83317, 83318, 83314, 83315, 83319, 83320, 83321, - 83326, 83327, 83328, 83354, 83355, 83359, 83360, 83350, 83351, 83352, - 83353, 83356, 83357, 83358, 83361, 83377, 83378, 83374, 83375, 83381, - 83382, 83373, 83376, 83379, 83380, 83383, 83384, 83385, 83389, 83386, - 83387, 83388, 83390, 83391, 83392, 83393, 83394, 83395, 83363, 83364, - 83362, 83365, 83366, 83367, 83368, 83369, 83370, 83371, 83372, 83293, - 83294, 83295, 83296, 83297, 83298, 83299, 83300, 83301, 83302, 83329, - 83330, 83331, 83332, 83333, 83334, 83335, 83336, 83337, 83338, 83406, - 83407, 83408, 83409, 83410, 83411, 83412, 83413, 83414, 83415, 83416, - 83447, 83448, 83455, 83456, 83449, 83450, 83451, 83452, 83453, 83454, - 83457, 83458, 83489, 83490, 83491, 83492, 83493, 83494, 83495, 83496, - 83396, 83397, 83398, 83399, 83400, 83401, 83402, 83403, 83404, 83405, - 83417, 83418, 83419, 83420, 83421, 83422, 83423, 83424, 83425, 83426, - 83427, 83428, 83429, 83430, 83431, 83432, 83433, 83434, 83435, 83436, - 83437, 83438, 83439, 83440, 83441, 83442, 83443, 83444, 83445, 83446, - 83459, 83460, 83461, 83462, 83463, 83464, 83465, 83466, 83467, 83468, - 83469, 83470, 83471, 83472, 83473, 83474, 83475, 83476, 83477, 83478, - 83479, 83480, 83481, 83482, 83483, 83484, 83485, 83486, 83487, 83488, - 83526, 83497, 83498, 83499, 83500, 83501, 83502, 83503, 83504, 83505, - 83506, 83507, 83508, 83509, 83510, 83511, 83512, 83513, 83514, 83515, - 83516, 83517, 83518, 83519, 83520, 83521, 83522, 83523, 83524, 83525, - 129728, 8736, 10654, 10660, 128551, 8491, 128162, 128544, 128028, 117768, - 128246, 11150, 11148, 11149, 11151, 11119, 8630, 10560, 8755, 128260, - 10226, 8634, 10769, 9875, 10193, 9765, 9021, 9055, 9061, 9033, 9052, - 9022, 9066, 9058, 9067, 9042, 9049, 9035, 9034, 9038, 9062, 9073, 9046, - 9050, 9075, 9080, 9014, 9082, 9078, 9077, 9081, 9060, 9051, 9063, 9029, - 9109, 9020, 9056, 9017, 9018, 9036, 9047, 9044, 9037, 9043, 9040, 9027, - 9031, 9072, 9071, 9016, 9026, 9025, 9028, 9032, 9019, 9054, 9048, 9030, - 9076, 9070, 9023, 9059, 9069, 9015, 9079, 9024, 9074, 9057, 9041, 9045, - 9053, 9039, 9065, 9064, 9068, 39, 11236, 8773, 8786, 10864, 8774, 8784, - 983195, 983196, 69372, 1548, 2277, 2280, 2276, 2279, 2278, 2281, 1615, - 65144, 65145, 2302, 1612, 65138, 1549, 2206, 2299, 2300, 1643, 2274, - 1771, 1770, 1565, 1757, 1614, 1630, 2293, 2292, 65142, 65143, 1611, - 65136, 1538, 1645, 1748, 2207, 1621, 1620, 1616, 2294, 65146, 65147, - 1613, 65140, 2258, 2255, 2254, 2257, 2236, 2244, 2235, 2237, 1593, 1886, - 2227, 1696, 1887, 1885, 65226, 65227, 65225, 65228, 1575, 2165, 2176, - 2173, 2161, 2174, 2171, 2167, 2177, 2169, 2166, 2168, 2178, 2164, 2160, - 2162, 2175, 2172, 1571, 65156, 65155, 1573, 65160, 65159, 1651, 1650, - 1908, 1907, 1570, 65154, 65153, 2163, 2170, 1649, 64337, 64336, 1609, - 65264, 65263, 65166, 65165, 1749, 1576, 2230, 1878, 2208, 1874, 1875, - 1872, 1876, 1877, 1873, 2209, 65168, 65169, 65167, 65170, 1664, 64347, - 64348, 64346, 64349, 1659, 64339, 64340, 64338, 64341, 1583, 1882, 1774, - 1679, 2222, 69314, 1881, 1674, 1675, 1680, 1673, 65194, 65193, 1676, - 64389, 64388, 1590, 1787, 65214, 65215, 65213, 65216, 1677, 64387, 64386, - 1672, 64393, 64392, 1646, 1697, 1647, 1668, 64371, 64372, 64370, 64373, - 1678, 64391, 64390, 1740, 1911, 1910, 1909, 1599, 1598, 1597, 64509, - 64510, 64508, 64511, 1601, 1699, 2212, 1698, 1889, 1701, 1888, 65234, - 65235, 65233, 65236, 1711, 1716, 1714, 1712, 2224, 64403, 64404, 64402, - 64405, 1594, 2243, 1788, 65230, 65231, 65229, 65232, 1715, 64407, 64408, - 64406, 64409, 2248, 1581, 1916, 2186, 1903, 1906, 1902, 1880, 1669, 1666, - 1879, 1665, 65186, 65187, 65185, 65188, 1569, 65152, 1607, 1729, 1730, - 64423, 64424, 64422, 64425, 1791, 1728, 64421, 64420, 1726, 64427, 64428, - 64426, 64429, 65258, 65259, 65257, 65260, 1652, 1653, 1656, 1654, 1580, - 2210, 2246, 2245, 65182, 65183, 65181, 65184, 1688, 64395, 64394, 1603, - 69316, 1919, 1710, 2228, 1708, 1707, 65242, 65243, 65241, 65244, 1568, - 1705, 1892, 1596, 1891, 2189, 1595, 2242, 1890, 64399, 64400, 64398, - 64401, 1733, 64481, 64480, 1737, 64483, 64482, 1582, 65190, 65191, 65189, - 65192, 1604, 2214, 1718, 2247, 1717, 1720, 1719, 1898, 65246, 65247, - 65245, 65248, 2221, 1605, 2215, 1894, 1893, 65250, 65251, 65249, 65252, - 1564, 1709, 1713, 64411, 64412, 64410, 64413, 64468, 64469, 64467, 64470, - 1667, 64375, 64376, 64374, 64377, 1606, 1722, 64415, 64414, 1896, 1897, - 1725, 1895, 2185, 1721, 1724, 65254, 65255, 65253, 65256, 1662, 2231, - 2238, 64343, 64344, 64342, 64345, 1702, 64367, 64368, 64366, 64369, 1602, - 2213, 2229, 1703, 1704, 65238, 65239, 65237, 65240, 1585, 1905, 2233, - 1682, 1685, 1883, 1899, 1687, 1775, 1900, 1689, 1684, 1686, 2218, 1683, - 65198, 65197, 1681, 64397, 64396, 1723, 64417, 64418, 64416, 64419, 2220, - 1589, 2223, 1694, 1693, 65210, 65211, 65209, 65212, 1587, 1690, 1918, - 1904, 1691, 1692, 1901, 1884, 1917, 65202, 65203, 65201, 65204, 1588, - 1786, 65206, 65207, 65205, 65208, 1648, 2225, 1706, 1591, 69315, 2211, - 2188, 1695, 2187, 65218, 65219, 65217, 65220, 1670, 2241, 1727, 64379, - 64380, 64378, 64381, 1671, 64383, 64384, 64382, 64385, 1578, 1577, 65172, - 65171, 1731, 65176, 2232, 2239, 1661, 1660, 65174, 65175, 65173, 1663, - 64355, 64356, 64354, 64357, 1584, 65196, 65195, 1579, 65178, 65179, - 65177, 65180, 2182, 1657, 2240, 64359, 64360, 64358, 64361, 1658, 64351, - 64352, 64350, 64353, 1735, 1655, 64477, 64472, 64471, 64488, 64489, 1739, - 1700, 64363, 64364, 64362, 64365, 64479, 64478, 1608, 2219, 1743, 1913, - 1912, 1572, 65158, 65157, 1738, 1732, 65262, 65261, 1610, 1746, 1915, - 1914, 1747, 64433, 64432, 64431, 64430, 1574, 65162, 65163, 65161, 65164, - 1742, 2216, 2234, 2217, 1741, 1745, 65266, 65267, 65265, 65268, 1736, - 64476, 64475, 1734, 64474, 64473, 1592, 65222, 65223, 65221, 65224, 1586, - 65200, 65199, 2226, 1744, 64485, 64486, 64484, 64487, 2297, 2295, 64888, - 64887, 64886, 64950, 64699, 64554, 64964, 64885, 64698, 64553, 64787, - 64759, 64788, 64760, 64842, 64839, 64841, 64840, 64845, 65015, 64656, - 64605, 64828, 64829, 65010, 65011, 65023, 64962, 64669, 64518, 64672, - 64738, 64926, 64670, 64519, 64622, 64521, 64668, 64517, 64620, 64671, - 64520, 64737, 64621, 64619, 64618, 64623, 64522, 65021, 64878, 64939, - 64693, 64547, 64880, 64879, 64694, 64548, 64803, 64775, 64692, 64546, - 64695, 64549, 64812, 64784, 64804, 64776, 64893, 64892, 64704, 64559, - 64636, 64561, 64702, 64557, 64961, 64705, 64560, 64703, 64558, 64637, - 64562, 64889, 64891, 64890, 64701, 64556, 64789, 64761, 64700, 64555, - 64790, 64762, 64859, 64858, 64682, 64536, 64795, 64767, 64959, 64681, + 71475, 71474, 71472, 71481, 71473, 71480, 9992, 128747, 128748, 130042, + 9200, 128874, 128822, 128823, 128837, 128776, 128777, 128774, 128775, + 128773, 128811, 128859, 128826, 128829, 128855, 128769, 128875, 128876, + 128830, 128834, 128835, 128836, 128783, 128857, 128848, 128846, 128844, + 128805, 128800, 128803, 128804, 128798, 128869, 128870, 128871, 128872, + 128873, 128787, 128865, 128866, 128864, 128880, 128794, 128841, 128883, + 128882, 128854, 128878, 128796, 128797, 128801, 128810, 128851, 128784, + 128785, 128786, 128824, 128881, 128789, 128879, 128782, 128843, 128858, + 128856, 128868, 128867, 128863, 128768, 128833, 128818, 128816, 128817, + 128799, 128819, 128820, 128821, 128827, 128828, 128877, 128792, 128793, + 128788, 128806, 128813, 128825, 128850, 128860, 128861, 128814, 128807, + 128812, 128802, 128862, 128781, 128839, 128795, 128852, 128847, 128831, + 128832, 128840, 128809, 128849, 128845, 128778, 128815, 128779, 128780, + 128790, 128791, 128808, 128772, 128842, 128853, 128771, 128770, 128838, + 9879, 8501, 983054, 128126, 117837, 117836, 117844, 117845, 117842, + 117843, 117839, 117838, 117841, 117840, 8780, 9006, 983203, 8776, 10863, + 8778, 9095, 9941, 118479, 127994, 38, 127944, 128657, 10815, 82970, + 82971, 82964, 82965, 82966, 82967, 82968, 82969, 82972, 82973, 82974, + 82994, 82995, 82996, 82987, 82988, 82992, 82993, 82986, 82989, 82990, + 82991, 82997, 82998, 82999, 83016, 83017, 83018, 83019, 83010, 83011, + 83012, 83013, 83014, 83015, 83020, 83021, 83022, 83050, 83051, 83052, + 83053, 83043, 83044, 83045, 83046, 83047, 83048, 83049, 83054, 82984, + 82985, 82975, 82976, 82977, 82978, 82979, 82980, 82981, 82982, 82983, + 82953, 82954, 82955, 82956, 82957, 82958, 82959, 82960, 82961, 82962, + 82963, 82944, 82945, 82946, 82947, 82948, 82949, 82950, 82951, 82952, + 83000, 83001, 83002, 83003, 83004, 83005, 83006, 83007, 83008, 83009, + 83023, 83024, 83025, 83026, 83027, 83028, 83029, 83030, 83031, 83032, + 83033, 83034, 83035, 83036, 83037, 83038, 83039, 83040, 83041, 83042, + 83062, 83063, 83064, 83065, 83070, 83071, 83072, 83073, 83066, 83067, + 83068, 83055, 83056, 83057, 83058, 83059, 83060, 83061, 83069, 83074, + 83075, 83076, 83077, 83078, 83083, 83084, 83079, 83080, 83081, 83082, + 83085, 83086, 83087, 83088, 83094, 83095, 83089, 83090, 83091, 83092, + 83093, 83096, 83097, 83098, 83099, 83105, 83106, 83100, 83101, 83102, + 83103, 83104, 83107, 83108, 83109, 83110, 83111, 83112, 83113, 83114, + 83115, 83116, 83117, 83118, 83119, 83120, 83121, 83122, 83123, 83124, + 83125, 83126, 83127, 83128, 83129, 83130, 83131, 83132, 83133, 83134, + 83135, 83136, 83137, 83138, 83139, 83140, 83141, 83142, 83143, 83144, + 83145, 83146, 83147, 83148, 83149, 83150, 83151, 83152, 83153, 83154, + 83155, 83156, 83157, 83158, 83159, 83160, 83161, 83162, 83163, 83164, + 83165, 83166, 83167, 83168, 83169, 83170, 83173, 83174, 83175, 83180, + 83181, 83183, 83184, 83171, 83172, 83176, 83177, 83178, 83179, 83182, + 83190, 83191, 83192, 83193, 83185, 83186, 83187, 83188, 83189, 83194, + 83195, 83196, 83274, 83275, 83280, 83281, 83270, 83271, 83272, 83273, + 83276, 83277, 83278, 83279, 83268, 83269, 83259, 83260, 83261, 83262, + 83263, 83264, 83265, 83266, 83267, 83204, 83205, 83197, 83198, 83199, + 83200, 83201, 83202, 83203, 83206, 83207, 83245, 83246, 83238, 83239, + 83240, 83241, 83242, 83243, 83244, 83247, 83248, 83208, 83209, 83210, + 83211, 83212, 83213, 83214, 83215, 83216, 83217, 83218, 83219, 83220, + 83221, 83222, 83223, 83224, 83225, 83226, 83227, 83228, 83229, 83230, + 83231, 83232, 83233, 83234, 83235, 83236, 83237, 83249, 83250, 83251, + 83252, 83253, 83254, 83255, 83256, 83257, 83258, 83291, 83292, 83282, + 83283, 83284, 83285, 83286, 83287, 83288, 83289, 83290, 83312, 83313, + 83303, 83304, 83305, 83306, 83307, 83308, 83309, 83310, 83311, 83348, + 83349, 83339, 83340, 83341, 83342, 83343, 83344, 83345, 83346, 83347, + 83322, 83323, 83324, 83325, 83316, 83317, 83318, 83314, 83315, 83319, + 83320, 83321, 83326, 83327, 83328, 83354, 83355, 83359, 83360, 83350, + 83351, 83352, 83353, 83356, 83357, 83358, 83361, 83377, 83378, 83374, + 83375, 83381, 83382, 83373, 83376, 83379, 83380, 83383, 83384, 83385, + 83389, 83386, 83387, 83388, 83390, 83391, 83392, 83393, 83394, 83395, + 83363, 83364, 83362, 83365, 83366, 83367, 83368, 83369, 83370, 83371, + 83372, 83293, 83294, 83295, 83296, 83297, 83298, 83299, 83300, 83301, + 83302, 83329, 83330, 83331, 83332, 83333, 83334, 83335, 83336, 83337, + 83338, 83406, 83407, 83408, 83409, 83410, 83411, 83412, 83413, 83414, + 83415, 83416, 83447, 83448, 83455, 83456, 83449, 83450, 83451, 83452, + 83453, 83454, 83457, 83458, 83489, 83490, 83491, 83492, 83493, 83494, + 83495, 83496, 83396, 83397, 83398, 83399, 83400, 83401, 83402, 83403, + 83404, 83405, 83417, 83418, 83419, 83420, 83421, 83422, 83423, 83424, + 83425, 83426, 83427, 83428, 83429, 83430, 83431, 83432, 83433, 83434, + 83435, 83436, 83437, 83438, 83439, 83440, 83441, 83442, 83443, 83444, + 83445, 83446, 83459, 83460, 83461, 83462, 83463, 83464, 83465, 83466, + 83467, 83468, 83469, 83470, 83471, 83472, 83473, 83474, 83475, 83476, + 83477, 83478, 83479, 83480, 83481, 83482, 83483, 83484, 83485, 83486, + 83487, 83488, 83526, 83497, 83498, 83499, 83500, 83501, 83502, 83503, + 83504, 83505, 83506, 83507, 83508, 83509, 83510, 83511, 83512, 83513, + 83514, 83515, 83516, 83517, 83518, 83519, 83520, 83521, 83522, 83523, + 83524, 83525, 129728, 8736, 10654, 10660, 128551, 8491, 128162, 128544, + 128028, 117768, 128246, 11150, 11148, 11149, 11151, 11119, 8630, 10560, + 8755, 128260, 10226, 8634, 10769, 9875, 10193, 9765, 9021, 9055, 9061, + 9033, 9052, 9022, 9066, 9058, 9067, 9042, 9049, 9035, 9034, 9038, 9062, + 9073, 9046, 9050, 9075, 9080, 9014, 9082, 9078, 9077, 9081, 9060, 9051, + 9063, 9029, 9109, 9020, 9056, 9017, 9018, 9036, 9047, 9044, 9037, 9043, + 9040, 9027, 9031, 9072, 9071, 9016, 9026, 9025, 9028, 9032, 9019, 9054, + 9048, 9030, 9076, 9070, 9023, 9059, 9069, 9015, 9079, 9024, 9074, 9057, + 9041, 9045, 9053, 9039, 9065, 9064, 9068, 39, 11236, 983195, 118461, + 8773, 8786, 10864, 8774, 8784, 983196, 2183, 69328, 69372, 1548, 2277, + 2280, 2276, 2279, 2278, 2281, 1615, 65144, 65145, 2302, 1612, 65138, + 1549, 2299, 2300, 69370, 2206, 1643, 2274, 1771, 1770, 1565, 1757, 1614, + 1630, 2293, 2292, 65142, 65143, 1611, 65136, 1538, 1645, 1748, 2207, + 1621, 1620, 1616, 2294, 65146, 65147, 1613, 65140, 2258, 2255, 2254, + 2257, 2236, 2244, 2235, 2237, 1593, 1886, 2227, 1696, 1887, 1885, 65226, + 65227, 65225, 65228, 1575, 2165, 2176, 2173, 2161, 2174, 2171, 2167, + 2177, 2169, 2166, 2168, 2178, 2164, 2160, 2162, 2175, 2172, 1571, 65156, + 65155, 1573, 65160, 65159, 1651, 1650, 1908, 1907, 1570, 65154, 65153, + 2163, 2170, 1649, 64337, 64336, 1609, 65264, 65263, 65166, 65165, 1749, + 1576, 2230, 1878, 2208, 1874, 1875, 1872, 1876, 1877, 1873, 2209, 65168, + 65169, 65167, 65170, 1664, 64347, 64348, 64346, 64349, 1659, 64339, + 64340, 64338, 64341, 1583, 1882, 1774, 1679, 2222, 69314, 1881, 1674, + 1675, 1680, 1673, 65194, 65193, 1676, 64389, 64388, 1590, 1787, 65214, + 65215, 65213, 65216, 1677, 64387, 64386, 1672, 64393, 64392, 1646, 1697, + 1647, 1668, 64371, 64372, 64370, 64373, 1678, 64391, 64390, 1740, 1911, + 1910, 1909, 1599, 1598, 1597, 64509, 64510, 64508, 64511, 1601, 1699, + 2212, 1698, 1889, 1701, 1888, 65234, 65235, 65233, 65236, 1711, 1716, + 1714, 1712, 2224, 64403, 64404, 64402, 64405, 1594, 2243, 1788, 65230, + 65231, 65229, 65232, 1715, 64407, 64408, 64406, 64409, 2248, 1581, 1916, + 2186, 1903, 1906, 1902, 1880, 1669, 1666, 1879, 1665, 65186, 65187, + 65185, 65188, 1569, 65152, 1607, 1729, 1730, 64423, 64424, 64422, 64425, + 1791, 1728, 64421, 64420, 1726, 64427, 64428, 64426, 64429, 65258, 65259, + 65257, 65260, 1652, 1653, 1656, 1654, 1580, 2210, 2246, 2245, 65182, + 65183, 65181, 65184, 1688, 64395, 64394, 1603, 69316, 1919, 1710, 2228, + 1708, 1707, 65242, 65243, 65241, 65244, 1568, 1705, 1892, 1596, 1891, + 2189, 1595, 2242, 1890, 64399, 64400, 64398, 64401, 1733, 64481, 64480, + 1737, 64483, 64482, 1582, 65190, 65191, 65189, 65192, 1604, 2214, 1718, + 2247, 1717, 1720, 1719, 1898, 65246, 65247, 65245, 65248, 2221, 1605, + 2215, 1894, 1893, 65250, 65251, 65249, 65252, 1564, 1709, 1713, 64411, + 64412, 64410, 64413, 64468, 64469, 64467, 64470, 1667, 64375, 64376, + 64374, 64377, 1606, 1722, 64415, 64414, 1896, 1897, 1725, 1895, 2185, + 1724, 2191, 1721, 65254, 65255, 65253, 65256, 1662, 2231, 2238, 64343, + 64344, 64342, 64345, 1702, 64367, 64368, 64366, 64369, 1602, 2213, 2229, + 1703, 1704, 65238, 65239, 65237, 65240, 1585, 1905, 2233, 1682, 1685, + 1883, 1899, 1687, 1775, 1900, 1689, 1684, 1686, 2218, 1683, 65198, 65197, + 1681, 64397, 64396, 1723, 64417, 64418, 64416, 64419, 2220, 1589, 2223, + 1694, 1693, 65210, 65211, 65209, 65212, 1587, 1690, 1918, 1904, 1691, + 1692, 1901, 1884, 1917, 65202, 65203, 65201, 65204, 1588, 1786, 65206, + 65207, 65205, 65208, 1648, 2225, 1706, 1591, 69315, 2211, 2188, 1695, + 2187, 65218, 65219, 65217, 65220, 1670, 2241, 1727, 64379, 64380, 64378, + 64381, 1671, 64383, 64384, 64382, 64385, 1578, 1577, 65172, 65171, 1731, + 65176, 2232, 2239, 1661, 1660, 65174, 65175, 65173, 1663, 64355, 64356, + 64354, 64357, 1584, 65196, 65195, 2182, 69318, 1579, 65178, 65179, 65177, + 65180, 1657, 2240, 64359, 64360, 64358, 64361, 1658, 64351, 64352, 64350, + 64353, 1735, 1655, 64477, 64472, 64471, 64488, 64489, 1739, 1700, 64363, + 64364, 64362, 64365, 64479, 64478, 1608, 2219, 1743, 1913, 1912, 1572, + 65158, 65157, 1738, 1732, 65262, 65261, 1610, 1746, 1915, 1914, 1747, + 64433, 64432, 64431, 64430, 1574, 65162, 65163, 65161, 65164, 1742, 2216, + 2234, 2217, 1745, 1741, 69319, 65266, 65267, 65265, 65268, 1736, 64476, + 64475, 1734, 64474, 64473, 1592, 65222, 65223, 65221, 65224, 1586, 65200, + 65199, 2226, 1744, 64485, 64486, 64484, 64487, 2297, 2295, 64888, 64887, + 64886, 64950, 64699, 64554, 64964, 64885, 64698, 64553, 64787, 64759, + 64788, 64760, 64465, 64973, 64842, 64839, 64466, 69331, 64841, 69330, + 64840, 69329, 64845, 65015, 64656, 64605, 64828, 64829, 65010, 65011, + 65023, 64962, 64669, 64518, 64672, 64738, 64926, 64670, 64519, 64622, + 64521, 64668, 64517, 64620, 64671, 64520, 64737, 64621, 64619, 64618, + 64623, 64522, 65021, 64878, 64939, 64693, 64547, 64880, 64879, 64694, + 64548, 64803, 64775, 64692, 64546, 64695, 64549, 64812, 64784, 64804, + 64776, 64452, 64893, 64892, 64704, 64559, 64636, 64561, 64702, 64557, + 64961, 64705, 64560, 64703, 64558, 64637, 64562, 64889, 64891, 64890, + 64701, 64556, 64789, 64761, 64700, 64555, 64790, 64762, 64462, 64463, + 64460, 64461, 64859, 64858, 64682, 64536, 64795, 64767, 64959, 64681, 64535, 64796, 64768, 64915, 64916, 64728, 64594, 64595, 64596, 64727, - 64593, 64729, 64934, 64958, 64679, 64533, 64857, 64856, 64935, 64933, - 64680, 64534, 64797, 64769, 64798, 64770, 65019, 64643, 64573, 64640, - 64567, 64708, 64568, 64641, 64711, 64571, 64747, 64710, 64570, 64709, - 64569, 64963, 64955, 64951, 64642, 64712, 64572, 64748, 64644, 64574, - 64538, 64799, 64771, 64683, 64537, 64684, 64539, 64800, 64772, 65272, - 65271, 65274, 65273, 65270, 65269, 64646, 64579, 65276, 65275, 64898, - 64949, 64896, 64897, 64714, 64576, 64717, 64899, 64900, 64954, 64956, - 64940, 64713, 64575, 64902, 64901, 64715, 64577, 64904, 64903, 64941, - 64645, 64716, 64578, 64749, 64647, 64580, 64585, 64648, 64905, 64906, - 64907, 64719, 64582, 64910, 64911, 64953, 64720, 64583, 64909, 64914, - 64908, 64960, 64718, 64581, 64945, 64649, 64721, 64584, 64586, 65012, - 64918, 64917, 64947, 64723, 64588, 64726, 64751, 64952, 64957, 64921, - 64920, 64919, 64967, 64722, 64587, 64923, 64922, 64652, 64725, 64590, - 64750, 64654, 64591, 64653, 64651, 64724, 64589, 64650, 64655, 64592, - 64895, 64948, 64894, 64946, 64707, 64564, 64638, 64565, 64706, 64563, - 64639, 64566, 65009, 64843, 64833, 64835, 64836, 64837, 64834, 64832, - 64847, 65014, 64604, 64869, 64868, 64937, 64689, 64544, 64965, 64870, + 64593, 64729, 65019, 64451, 64934, 64958, 64679, 64533, 64857, 64856, + 64935, 64933, 64680, 64534, 64797, 64769, 64798, 64770, 64643, 64573, + 64640, 64567, 64708, 64568, 64641, 64711, 64571, 64747, 64710, 64570, + 64709, 64569, 64963, 64955, 64951, 64642, 64712, 64572, 64748, 64644, + 64574, 64974, 64538, 64799, 64771, 64683, 64537, 64684, 64539, 64800, + 64772, 65272, 65271, 65274, 65273, 65270, 65269, 64646, 64579, 65276, + 65275, 64898, 64949, 64896, 64897, 64714, 64576, 64717, 64899, 64900, + 64954, 64956, 64940, 64713, 64575, 64902, 64901, 64715, 64577, 64904, + 64903, 64941, 64645, 64716, 64578, 64749, 64647, 64580, 64585, 64648, + 64905, 64906, 64907, 64719, 64582, 64910, 64911, 64953, 64720, 64583, + 64909, 64914, 64908, 64960, 64718, 64581, 64945, 64649, 64721, 64584, + 64586, 65012, 64918, 64917, 64947, 64723, 64588, 64726, 64751, 64952, + 64957, 64921, 64920, 64919, 64967, 64722, 64587, 64923, 64922, 64652, + 64725, 64590, 64750, 64654, 64591, 64653, 64651, 64724, 64589, 64650, + 64655, 64592, 69336, 64895, 64948, 64894, 64946, 64707, 64564, 64638, + 64565, 64706, 64563, 64639, 64566, 65009, 69332, 69333, 69334, 64843, + 69335, 64833, 64835, 64836, 64837, 64834, 64969, 64459, 64971, 64970, + 64832, 64968, 64847, 64456, 64457, 64458, 64912, 64454, 64455, 64913, + 64453, 65014, 64604, 64869, 64868, 64937, 64689, 64544, 64965, 64870, 64691, 64545, 64801, 64773, 64690, 64811, 64783, 64802, 64774, 65013, - 64975, 65008, 65017, 65018, 64844, 64838, 64860, 64686, 64541, 64821, - 64817, 64744, 64862, 64861, 64685, 64540, 64820, 64936, 64966, 64687, - 64542, 64822, 64864, 64863, 64865, 64867, 64866, 64688, 64543, 64743, - 64791, 64763, 64810, 64782, 64792, 64764, 64606, 64609, 64755, 64607, - 64610, 64756, 64611, 64608, 64754, 64818, 64746, 64872, 64871, 64938, - 64806, 64814, 64778, 64824, 64873, 64805, 64813, 64777, 64823, 64875, - 64874, 64877, 64876, 64808, 64816, 64780, 64745, 64793, 64765, 64807, - 64815, 64779, 64825, 64809, 64781, 64794, 64766, 65022, 64846, 64882, - 64881, 64883, 64884, 64819, 64551, 64826, 64785, 64757, 64696, 64550, - 64786, 64758, 64851, 64850, 64849, 64674, 64524, 64677, 64740, 64930, - 64852, 64929, 64675, 64525, 64928, 64848, 64927, 64673, 64523, 64932, - 64853, 64855, 64854, 64931, 64626, 64676, 64526, 64739, 64628, 64527, - 64627, 64625, 64624, 64629, 64528, 64603, 64529, 64634, 64531, 64632, - 64678, 64530, 64741, 64633, 64631, 64630, 64635, 64532, 64742, 65016, - 64661, 64601, 64616, 64515, 64491, 64490, 64493, 64492, 64503, 64504, - 64502, 64667, 64736, 64664, 64513, 64663, 64512, 64665, 64614, 64666, - 64514, 64735, 64615, 64499, 64498, 64495, 64494, 64617, 64516, 64501, - 64500, 64613, 64612, 64497, 64496, 64942, 64731, 64598, 64734, 64753, - 64660, 64658, 64943, 64730, 64597, 64732, 64599, 64925, 64924, 64944, - 64659, 64733, 64600, 64752, 64657, 64662, 64602, 64506, 64507, 64505, - 64697, 64552, 64827, 2204, 1619, 1624, 2303, 126492, 126494, 126493, - 126495, 126644, 126638, 126641, 126647, 126648, 126646, 126632, 126645, - 126630, 126650, 126626, 126636, 126625, 126640, 126643, 126633, 126631, - 126651, 126637, 126635, 126649, 126627, 126642, 126639, 126629, 126489, - 126467, 126518, 126517, 126516, 126510, 126513, 126503, 126500, 126519, - 126506, 126498, 126508, 126497, 126512, 126505, 126523, 126509, 126507, - 126514, 126511, 126521, 126612, 126606, 126609, 126615, 126592, 126607, - 126599, 126596, 126616, 126614, 126600, 126613, 126598, 126618, 126594, - 126604, 126593, 126608, 126611, 126601, 126619, 126605, 126603, 126617, - 126595, 126610, 126597, 126475, 126704, 126705, 126588, 126590, 126585, - 126582, 126568, 126581, 126580, 126574, 126577, 126567, 126564, 126583, - 126570, 126562, 126572, 126561, 126576, 126569, 126586, 126587, 126573, - 126578, 126575, 126484, 126478, 126481, 126557, 126559, 126553, 126548, - 126542, 126545, 126551, 126530, 126537, 126535, 126555, 126541, 126539, - 126546, 126543, 126472, 126488, 126486, 126485, 126464, 126479, 126487, - 126474, 126470, 126490, 126466, 126476, 126465, 126480, 126483, 126473, - 126471, 126491, 126477, 126482, 126469, 1541, 1536, 2290, 2289, 2288, - 1642, 2199, 1550, 2192, 1769, 2193, 2184, 1544, 1629, 2296, 2301, 2298, - 1772, 2183, 1623, 983633, 983635, 983640, 983634, 983638, 983636, 983639, - 983637, 983641, 1563, 1617, 65148, 65149, 1554, 1555, 1552, 1537, 1539, - 1540, 1790, 1789, 1553, 1551, 1556, 2249, 1560, 1761, 2250, 2272, 1558, - 983202, 1751, 1750, 1753, 1752, 1762, 1764, 1768, 2264, 2273, 1756, 2261, - 1755, 1557, 2200, 2260, 2266, 2268, 2267, 2269, 2252, 2271, 2270, 2291, - 1767, 2251, 1760, 1759, 1559, 2253, 1754, 2263, 2262, 2265, 2202, 2201, - 69375, 2203, 69373, 69374, 2259, 1773, 1763, 1562, 1561, 1766, 1765, - 1622, 1618, 65150, 65151, 2256, 2205, 64444, 64435, 64434, 64441, 64440, - 64439, 64438, 64446, 64445, 64437, 64436, 64449, 64448, 64443, 64442, - 64450, 64447, 1758, 1600, 2179, 2180, 65137, 2181, 65139, 2285, 2282, - 2286, 2283, 2287, 2284, 1566, 2275, 1644, 1627, 1626, 1628, 2190, 1631, - 1567, 1625, 1545, 1546, 1542, 1543, 1637, 1636, 1639, 1638, 1635, 1634, - 1632, 1641, 1633, 1640, 1375, 1370, 1359, 1337, 1349, 1362, 1347, 1353, - 1342, 1361, 1360, 1356, 1333, 1335, 1336, 1346, 1331, 1355, 1345, 1364, - 1343, 1363, 1354, 1351, 1357, 1329, 1358, 1352, 1340, 1366, 1341, 1339, - 1330, 1348, 1350, 1338, 1334, 1344, 1332, 1365, 1373, 1372, 1371, 1395, - 1401, 1390, 1409, 1408, 1404, 1381, 1383, 1384, 1394, 1379, 1403, 1393, - 1412, 1391, 1411, 1402, 1399, 1405, 1376, 1407, 1385, 1377, 1406, 1400, - 1397, 1416, 1410, 1388, 1414, 1389, 1387, 1378, 1396, 1398, 1386, 1382, - 1392, 1380, 1413, 1415, 64279, 64277, 64275, 64276, 64278, 1369, 1418, - 1423, 1417, 1374, 129201, 10549, 10548, 129200, 10550, 10551, 129968, - 128667, 127912, 9800, 8978, 9738, 65948, 42, 8727, 8258, 128562, 11225, - 9954, 8771, 8870, 128095, 9883, 128663, 127975, 128762, 8371, 127814, - 68352, 68353, 68357, 68355, 68358, 68359, 68356, 68354, 68373, 68374, - 68372, 68393, 68405, 68388, 68387, 68386, 68391, 68390, 68389, 68371, - 68370, 68369, 68403, 68401, 68404, 68399, 68394, 68395, 68378, 68381, - 68377, 68366, 68367, 68362, 68363, 68364, 68365, 68385, 68384, 68380, - 68379, 68402, 68400, 68360, 68361, 68375, 68383, 68376, 68368, 68398, - 68392, 68382, 68397, 68396, 68409, 129361, 1547, 129518, 8525, 9810, - 129683, 128118, 128036, 127868, 128124, 128700, 128281, 128386, 11101, - 11099, 983056, 10155, 128043, 129363, 127992, 129441, 129366, 129391, - 128708, 7007, 7005, 7006, 6991, 6990, 6917, 6918, 6987, 6988, 6928, 6953, - 6954, 6936, 6937, 6948, 6944, 6943, 6949, 6984, 6927, 6933, 6934, 6919, - 6920, 6929, 6930, 6921, 6922, 6938, 6939, 6931, 6981, 6932, 6982, 6958, - 6925, 6926, 6950, 6945, 6935, 6940, 6951, 6952, 6957, 6923, 6924, 6962, - 6960, 6961, 6946, 6942, 6941, 6947, 6983, 6985, 6986, 6963, 6955, 6959, - 6956, 7022, 7025, 7021, 7024, 7023, 7026, 7020, 7019, 7027, 7012, 7013, - 7018, 7015, 7017, 7016, 7010, 7014, 7009, 7011, 7032, 7036, 7033, 7034, - 7035, 7031, 7030, 7029, 7028, 7003, 7038, 7008, 7002, 7037, 7039, 6915, - 6913, 6912, 6914, 6916, 6964, 6972, 6973, 6970, 6971, 6968, 6969, 6974, - 6975, 6977, 6976, 6965, 6978, 6979, 6966, 6967, 6980, 7004, 6997, 6996, - 6999, 6998, 6995, 6994, 6992, 7001, 6993, 7000, 127880, 10057, 9744, - 128503, 128505, 128499, 128501, 11197, 9745, 9746, 128502, 128500, 10007, - 129526, 129648, 42737, 42736, 42741, 42740, 42733, 42699, 42713, 42712, - 42692, 42706, 42683, 42719, 42734, 42735, 42682, 42729, 42657, 42725, - 42659, 42670, 42718, 42666, 42716, 42701, 42675, 42671, 42722, 42720, - 42727, 42723, 42702, 42726, 42677, 42707, 42708, 42709, 42686, 42694, - 42674, 42731, 42695, 42685, 42684, 42691, 42673, 42664, 42715, 42703, - 92193, 92188, 92161, 92199, 92240, 92179, 92211, 92219, 92213, 92243, - 92207, 92216, 92183, 92184, 92238, 92171, 983268, 92174, 92203, 92186, - 92210, 92230, 92175, 92221, 92232, 92246, 92198, 92214, 92190, 92176, - 92197, 92196, 92164, 92245, 92212, 92201, 92160, 92182, 92173, 92229, - 92227, 92180, 92237, 92241, 92223, 92185, 92194, 92178, 92239, 92225, - 92233, 92167, 92191, 92231, 92244, 92204, 92226, 92236, 92200, 92189, - 92187, 92162, 92163, 92169, 92170, 92202, 92205, 92222, 92168, 92165, - 92215, 92224, 92208, 92220, 92235, 92177, 92181, 92195, 92234, 92209, - 92166, 92172, 92206, 92192, 92228, 92217, 92242, 92218, 92262, 92271, - 92272, 92270, 92294, 92253, 92301, 92256, 92273, 92259, 92251, 92297, - 92300, 92296, 92295, 92255, 92252, 92292, 92286, 92268, 92290, 92281, - 92267, 92284, 92287, 92282, 92283, 92277, 92298, 92276, 92302, 92247, - 92299, 92288, 92269, 92260, 92261, 92257, 92274, 92289, 92263, 92279, - 92265, 92266, 92250, 92249, 92285, 92248, 92264, 92280, 92258, 92254, - 92278, 92291, 92293, 92275, 92310, 92320, 92312, 92394, 92393, 92319, - 92384, 92321, 92386, 92361, 92373, 92366, 92328, 92329, 92357, 92342, - 92397, 92365, 92376, 92339, 92382, 92363, 92318, 92387, 92383, 92315, - 92385, 92354, 92311, 92381, 92343, 92326, 92364, 92324, 92391, 92344, - 92390, 92338, 92396, 92308, 92349, 92375, 92378, 92317, 92331, 92362, - 92336, 92341, 92370, 92347, 92307, 92303, 92309, 92395, 92368, 92356, - 92367, 92360, 92389, 92333, 92359, 92374, 92351, 92325, 92371, 92350, - 92345, 92372, 92314, 92340, 92323, 92304, 92313, 92316, 92398, 92399, - 92380, 92330, 92379, 92392, 92335, 92332, 92346, 92400, 92358, 92334, - 92353, 92337, 92306, 92369, 92388, 92322, 92305, 92327, 92377, 92352, - 92348, 92355, 92436, 92517, 92488, 92434, 92415, 92431, 92454, 92460, - 92420, 92489, 92422, 92474, 92479, 92449, 92502, 92439, 92484, 92495, - 92464, 92511, 92406, 92446, 92497, 92426, 92482, 92448, 92515, 92401, - 92478, 92427, 92496, 92458, 92424, 92445, 92404, 92466, 92444, 92438, - 92442, 92441, 92510, 92499, 92425, 92437, 92440, 92471, 92465, 92409, - 92483, 92475, 92467, 92485, 92457, 92432, 92476, 92435, 92412, 92414, - 92430, 92407, 92403, 92405, 92487, 92418, 92486, 92408, 92447, 92459, - 92480, 92472, 92505, 92514, 92450, 92463, 92410, 92493, 92507, 92503, - 92462, 92506, 92443, 92509, 92469, 92461, 92490, 92512, 92494, 92455, - 92417, 92501, 92413, 92508, 92500, 92504, 92473, 92419, 92498, 92423, - 92516, 92428, 92452, 92451, 92481, 92416, 92456, 92433, 92477, 92492, - 92491, 92513, 92411, 92402, 92421, 92453, 92468, 92470, 92429, 92661, - 92626, 92616, 92596, 92612, 92603, 92673, 92651, 92627, 92597, 92585, - 92578, 92615, 92565, 92552, 92602, 92674, 92570, 92646, 92599, 92532, - 92587, 92538, 92620, 92670, 92665, 92575, 92521, 92633, 92595, 92523, - 92556, 92539, 92664, 92653, 92667, 92600, 92542, 92555, 92668, 92520, - 92582, 92591, 92581, 92551, 92654, 92611, 92614, 92666, 92671, 92617, - 92637, 92562, 92518, 92592, 92558, 92528, 92607, 92535, 92557, 92563, - 92550, 92624, 92640, 92657, 92531, 92601, 92553, 92543, 92619, 92598, - 92541, 92544, 92579, 92658, 92554, 92628, 92648, 92547, 92527, 92545, - 92540, 92604, 92622, 92589, 92608, 92583, 92609, 92662, 92613, 92584, - 92625, 92634, 92524, 92536, 92605, 92647, 92548, 92663, 92593, 92621, - 92568, 92610, 92576, 92529, 92618, 92649, 92561, 92656, 92526, 92655, - 92546, 92534, 92560, 92580, 92659, 92660, 92638, 92571, 92525, 92549, - 92559, 92635, 92577, 92530, 92636, 92669, 92572, 92672, 92537, 92519, - 92630, 92586, 92569, 92566, 92573, 92652, 92522, 92574, 92650, 92533, - 92567, 92623, 92606, 92639, 92564, 92590, 92642, 92643, 92594, 92641, - 92645, 92644, 92588, 92629, 92632, 92631, 92710, 92695, 92694, 92726, - 92675, 92719, 92677, 92718, 92682, 92717, 92689, 92720, 92724, 92685, - 92722, 92723, 92711, 92712, 92698, 92688, 92697, 92696, 92702, 92687, - 92704, 92681, 92708, 92703, 92706, 92714, 92709, 92679, 92721, 92684, - 92683, 92707, 92691, 92713, 92700, 92693, 92727, 92690, 92692, 92686, - 92680, 92725, 92699, 92701, 92705, 92716, 92728, 92678, 92715, 92676, - 42693, 42698, 42711, 42696, 42667, 42717, 42704, 42661, 42721, 42669, - 42668, 42705, 42700, 42680, 42678, 42710, 42688, 42681, 42732, 42676, - 42679, 42730, 42728, 42672, 42662, 42724, 42687, 42689, 42690, 42697, - 42714, 42660, 42656, 42665, 42663, 42658, 42738, 42742, 42739, 42743, - 127974, 128183, 128180, 128182, 128181, 127820, 129685, 128202, 129532, - 128136, 129530, 127936, 92916, 92912, 92915, 92913, 92914, 92887, 92894, - 92908, 92880, 92907, 92893, 92886, 92888, 92881, 92906, 92896, 92891, - 92902, 92900, 92885, 92890, 92884, 92904, 92905, 92899, 92889, 92897, - 92892, 92895, 92882, 92898, 92883, 92901, 92903, 92909, 92917, 9918, - 129415, 7152, 7153, 7124, 7108, 7114, 7130, 7139, 7127, 7138, 7133, 7136, - 7113, 7111, 7117, 7119, 7107, 7135, 7125, 7112, 7123, 7129, 7116, 7132, - 7105, 7126, 7128, 7110, 7109, 7137, 7121, 7118, 7106, 7120, 7134, 7122, - 7115, 7131, 7104, 7140, 7141, 7154, 7155, 7167, 7165, 7166, 7164, 7142, - 7150, 7151, 7144, 7147, 7149, 7143, 7145, 7146, 7148, 128704, 128705, - 128267, 127900, 127901, 9835, 9836, 129492, 128059, 127958, 128147, - 129451, 129752, 127866, 129714, 983055, 128276, 129745, 128277, 9086, - 128718, 2557, 2432, 2519, 2548, 2552, 2551, 2550, 2549, 2553, 2454, 2510, - 983661, 2453, 2480, 2545, 2544, 2525, 2524, 2556, 2443, 2528, 2444, 2529, - 2527, 2479, 2437, 2438, 2448, 2452, 2466, 2465, 2471, 2470, 2464, 2463, - 2469, 2468, 2441, 2442, 2439, 2440, 2457, 2467, 2462, 2472, 2486, 2487, - 2488, 2477, 2476, 2459, 2458, 2456, 2455, 2461, 2460, 2475, 2474, 2489, - 2482, 2478, 2447, 2451, 2547, 2546, 983651, 983650, 983652, 2558, 2433, - 2492, 2493, 2434, 2509, 2435, 2554, 2494, 2504, 2508, 2497, 2498, 2499, - 2500, 2530, 2531, 2495, 2496, 2503, 2507, 2539, 2538, 2541, 2540, 2537, - 2536, 2534, 2543, 2535, 2542, 2555, 11102, 127857, 9004, 9187, 8812, - 8502, 8757, 129475, 128719, 72710, 72711, 72712, 72746, 72704, 72705, - 72715, 72717, 72731, 72730, 72736, 72735, 72729, 72728, 72734, 72733, - 72708, 72709, 72706, 72707, 72722, 72732, 72727, 72737, 72747, 72748, - 72749, 72741, 72740, 72724, 72723, 72721, 72720, 72726, 72725, 72719, - 72718, 72739, 72738, 72750, 72745, 72742, 72744, 72743, 72714, 72716, - 72801, 72810, 72806, 72797, 72807, 72798, 72802, 72811, 72800, 72809, - 72799, 72808, 72796, 72805, 72804, 72795, 72803, 72794, 72764, 72768, - 72765, 72767, 72766, 72756, 72757, 72758, 72751, 72761, 72763, 72754, - 72755, 72752, 72753, 72760, 72762, 72770, 72769, 72789, 72788, 72791, - 72790, 72787, 72786, 72784, 72793, 72785, 72792, 72771, 72772, 72773, - 72812, 128692, 128690, 10745, 10744, 129506, 127921, 127874, 128038, - 8383, 129766, 9763, 128089, 129452, 9679, 9864, 10733, 9865, 9210, 11176, - 11177, 11178, 11179, 11180, 11182, 11181, 11183, 9960, 10028, 9821, - 129554, 129596, 129609, 129612, 9818, 129551, 129593, 9822, 129543, - 129564, 129585, 129597, 129606, 129555, 129619, 129617, 129618, 9823, - 129556, 129598, 9819, 129552, 129594, 9820, 129553, 129595, 129575, - 129572, 129576, 129577, 129573, 129574, 9827, 9670, 11201, 10070, 10730, - 11230, 9830, 128419, 128899, 9196, 9662, 9660, 11167, 127778, 9922, 9923, - 10047, 9873, 10022, 128447, 128420, 9829, 11042, 128426, 11052, 10711, - 11044, 117867, 117870, 117869, 117868, 11035, 9194, 9198, 128896, 9668, - 9666, 9664, 11164, 8268, 9944, 128412, 9754, 9699, 9698, 10731, 9207, - 11206, 11045, 9204, 11207, 11047, 9205, 11208, 9206, 11205, 128921, - 128927, 9726, 9724, 9912, 117871, 10002, 128392, 9648, 11039, 127986, - 118451, 128413, 9755, 9193, 9197, 9654, 9199, 128898, 11091, 9658, 9656, - 10145, 10148, 11166, 8269, 127990, 9644, 11049, 11050, 11089, 9642, - 129997, 9787, 9632, 11200, 9209, 128306, 9728, 128369, 9927, 9984, - 128900, 128909, 9986, 9751, 9824, 9733, 128919, 128925, 128908, 9951, - 128383, 9742, 9942, 128418, 128897, 9195, 9652, 9650, 9700, 9701, 11165, - 9851, 9646, 11054, 128920, 128926, 11037, 11204, 10707, 10067, 8493, - 8460, 8465, 8476, 8488, 10164, 10166, 10165, 9250, 118018, 118039, - 118128, 118187, 118218, 118159, 118054, 118112, 118234, 118084, 118202, - 118143, 118031, 118062, 118120, 118240, 118179, 118090, 118210, 118151, - 118047, 118104, 118226, 118165, 118076, 118194, 118135, 118021, 118035, - 118066, 118124, 118243, 118183, 118093, 118214, 118155, 118050, 118108, - 118230, 118168, 118080, 118198, 118139, 118028, 118058, 118116, 118237, - 118175, 118087, 118206, 118147, 118043, 118100, 118222, 118162, 118072, - 118190, 118131, 118029, 118060, 118118, 118177, 118208, 118149, 118045, - 118102, 118224, 118023, 118037, 118068, 118126, 118244, 118185, 118095, - 118216, 118157, 118052, 118110, 118232, 118170, 118082, 118200, 118141, - 118074, 118192, 118133, 118020, 118033, 118064, 118122, 118242, 118181, - 118092, 118212, 118153, 118048, 118106, 118228, 118167, 118078, 118196, - 118137, 118026, 118056, 118114, 118235, 118173, 118085, 118204, 118145, - 118041, 118098, 118220, 118160, 118070, 118188, 118129, 118034, 118065, - 118123, 118182, 118213, 118154, 118049, 118107, 118229, 118079, 118197, - 118138, 118017, 118024, 118038, 118069, 118127, 118245, 118186, 118096, - 118217, 118158, 118053, 118111, 118233, 118171, 118083, 118201, 118142, - 118030, 118061, 118119, 118239, 118178, 118089, 118209, 118150, 118046, - 118103, 118225, 118164, 118075, 118193, 118134, 118027, 118057, 118115, - 118236, 118174, 118086, 118205, 118146, 118042, 118099, 118221, 118161, - 118071, 118189, 118130, 118025, 118055, 118113, 118172, 118203, 118144, - 118040, 118097, 118219, 118016, 118022, 118036, 118067, 118125, 118184, - 118094, 118215, 118156, 118051, 118109, 118231, 118169, 118081, 118199, - 118140, 118059, 118117, 118238, 118176, 118088, 118207, 118148, 118044, - 118101, 118223, 118163, 118073, 118191, 118132, 118019, 118105, 118227, - 118166, 118032, 118063, 118121, 118241, 118180, 118091, 118211, 118152, - 118077, 118195, 118136, 129792, 129794, 129798, 129806, 129821, 129836, - 129813, 129844, 129829, 129802, 129817, 129848, 129832, 129810, 129840, - 129825, 129796, 129804, 129819, 129850, 129834, 129842, 129827, 129800, - 129815, 129846, 129831, 129808, 129838, 129823, 129793, 129801, 129816, - 129847, 129797, 129805, 129820, 129851, 129835, 129812, 129843, 129828, - 129809, 129839, 129824, 129795, 129803, 129818, 129849, 129833, 129811, - 129841, 129826, 129799, 129814, 129845, 129830, 129807, 129837, 129822, - 127804, 128033, 128216, 128153, 129744, 128939, 128951, 128957, 128945, - 128902, 128912, 128932, 128366, 128278, 128209, 128218, 129667, 12731, - 12727, 12726, 12724, 12725, 12570, 12574, 12718, 12576, 12719, 12578, - 12580, 12713, 12735, 12720, 12584, 12715, 12572, 12579, 12581, 12709, - 12708, 12573, 12575, 12582, 12557, 12728, 12588, 12707, 12732, 12583, - 12714, 12723, 12589, 12716, 12712, 12585, 12555, 12587, 12717, 12591, - 12571, 12722, 12711, 12590, 12734, 12721, 12710, 12577, 12567, 12563, - 12705, 12730, 12558, 12733, 12568, 12564, 12556, 12729, 12569, 12565, - 12549, 12704, 12560, 12706, 12553, 12552, 12559, 12551, 12550, 12561, - 12566, 12554, 12586, 12562, 118257, 118258, 118259, 118260, 117854, - 11867, 117853, 118253, 118255, 11868, 117855, 118249, 118251, 118247, - 11211, 8993, 130029, 8990, 8973, 11812, 8991, 8972, 11813, 130030, 9141, - 9142, 10555, 9183, 9181, 130026, 130018, 9185, 127870, 128144, 127893, - 128335, 129379, 127923, 8904, 10705, 10706, 127993, 118282, 117792, - 117791, 118281, 9574, 9559, 9556, 9577, 9565, 9562, 9553, 9580, 9571, - 9568, 9552, 9511, 9490, 9503, 9486, 9537, 9520, 9513, 9489, 9505, 9485, - 9543, 9519, 9558, 9555, 9573, 9557, 9554, 9572, 9592, 9598, 9593, 9599, - 9499, 9531, 9495, 9475, 9547, 9515, 9507, 9595, 9523, 9491, 9487, 9551, - 9549, 9483, 9481, 9479, 9477, 9473, 9594, 9541, 9525, 9517, 9533, 9530, - 9522, 9546, 9539, 9582, 9581, 9583, 9584, 130010, 130014, 129954, 129958, - 130003, 129959, 129964, 129965, 129955, 130000, 129952, 129956, 129963, - 129960, 129953, 129961, 129957, 129962, 130007, 130005, 130004, 130012, - 9586, 130008, 130011, 130002, 130015, 130006, 9585, 130009, 130001, - 130013, 9587, 129966, 9591, 9516, 9488, 9484, 9550, 9548, 9472, 117787, - 117788, 129967, 9588, 9596, 9482, 9480, 9478, 9476, 117789, 9589, 9597, - 9524, 9496, 9492, 9474, 118297, 118295, 118296, 118294, 9532, 9508, 9500, - 117790, 9590, 9542, 9526, 9518, 9534, 9529, 9521, 9545, 9540, 9536, 9510, - 9498, 9502, 9494, 9528, 9544, 9514, 9497, 9506, 9493, 9527, 9564, 9561, - 9576, 9563, 9560, 9575, 9570, 9567, 9579, 9538, 9512, 9504, 9535, 9509, - 9501, 9569, 9566, 9578, 129354, 983263, 128163, 128102, 128713, 128023, - 129460, 69649, 69685, 69749, 69745, 69746, 69687, 69686, 69637, 69638, - 69648, 69650, 69664, 69663, 69669, 69668, 69662, 69661, 69667, 69666, - 69643, 69644, 69645, 69646, 69679, 69641, 69642, 69639, 69640, 69684, - 69678, 69655, 69665, 69660, 69670, 69680, 69681, 69682, 69674, 69673, - 69657, 69656, 69654, 69653, 69659, 69658, 69652, 69651, 69672, 69671, - 69683, 69675, 69677, 69676, 69647, 69721, 69730, 69726, 69717, 69727, - 69718, 69722, 69731, 69720, 69729, 69719, 69728, 69716, 69725, 69724, - 69715, 69723, 69714, 69732, 69733, 69759, 69709, 69707, 69706, 69705, - 69708, 69632, 69744, 69635, 69634, 69636, 69633, 69700, 69747, 69748, - 69689, 69688, 69699, 69701, 69692, 69693, 69694, 69695, 69696, 69697, - 69690, 69691, 69698, 69702, 69704, 69703, 69739, 69738, 69741, 69740, - 69737, 69736, 69734, 69743, 69735, 69742, 10241, 10243, 10247, 10255, - 10271, 10303, 10367, 10495, 10431, 10335, 10463, 10399, 10287, 10351, - 10479, 10415, 10319, 10447, 10383, 10263, 10295, 10359, 10487, 10423, - 10327, 10455, 10391, 10279, 10343, 10471, 10407, 10311, 10439, 10375, - 10251, 10267, 10299, 10363, 10491, 10427, 10331, 10459, 10395, 10283, - 10347, 10475, 10411, 10315, 10443, 10379, 10259, 10291, 10355, 10483, - 10419, 10323, 10451, 10387, 10275, 10339, 10467, 10403, 10307, 10435, - 10371, 10245, 10253, 10269, 10301, 10365, 10493, 10429, 10333, 10461, - 10397, 10285, 10349, 10477, 10413, 10317, 10445, 10381, 10261, 10293, - 10357, 10485, 10421, 10325, 10453, 10389, 10277, 10341, 10469, 10405, - 10309, 10437, 10373, 10249, 10265, 10297, 10361, 10489, 10425, 10329, - 10457, 10393, 10281, 10345, 10473, 10409, 10313, 10441, 10377, 10257, - 10289, 10353, 10481, 10417, 10321, 10449, 10385, 10273, 10337, 10465, - 10401, 10305, 10433, 10369, 10242, 10246, 10254, 10270, 10302, 10366, - 10494, 10430, 10334, 10462, 10398, 10286, 10350, 10478, 10414, 10318, - 10446, 10382, 10262, 10294, 10358, 10486, 10422, 10326, 10454, 10390, - 10278, 10342, 10470, 10406, 10310, 10438, 10374, 10250, 10266, 10298, - 10362, 10490, 10426, 10330, 10458, 10394, 10282, 10346, 10474, 10410, - 10314, 10442, 10378, 10258, 10290, 10354, 10482, 10418, 10322, 10450, - 10386, 10274, 10338, 10466, 10402, 10306, 10434, 10370, 10244, 10252, - 10268, 10300, 10364, 10492, 10428, 10332, 10460, 10396, 10284, 10348, - 10476, 10412, 10316, 10444, 10380, 10260, 10292, 10356, 10484, 10420, - 10324, 10452, 10388, 10276, 10340, 10468, 10404, 10308, 10436, 10372, - 10248, 10264, 10296, 10360, 10488, 10424, 10328, 10456, 10392, 10280, - 10344, 10472, 10408, 10312, 10440, 10376, 10256, 10288, 10352, 10480, - 10416, 10320, 10448, 10384, 10272, 10336, 10464, 10400, 10304, 10432, - 10368, 10240, 129504, 983125, 129329, 127838, 728, 127753, 128112, - 128188, 129650, 129521, 9099, 166, 128148, 129382, 129294, 129529, - 129483, 129767, 128027, 6663, 6662, 6659, 6658, 6671, 6670, 6667, 6666, - 6661, 6668, 6665, 6657, 6678, 6669, 6656, 6674, 6660, 6673, 6676, 6664, - 6675, 6672, 6677, 6683, 6681, 6679, 6682, 6680, 6687, 6686, 5957, 5960, - 5962, 5959, 5956, 5969, 5955, 5966, 5963, 5961, 5965, 5968, 5958, 5967, - 5964, 5952, 5953, 5954, 5970, 5971, 8226, 8729, 128363, 128364, 9678, - 128652, 128101, 128100, 128655, 129480, 129419, 127959, 127791, 129699, - 118939, 118940, 118944, 118943, 118941, 118942, 118938, 118945, 118882, - 118876, 118824, 118835, 118797, 118866, 118957, 118801, 118802, 118865, - 118818, 118916, 118819, 118917, 118899, 118935, 119018, 119022, 119021, - 119020, 119023, 119019, 119017, 118986, 118984, 118985, 118843, 118887, - 118808, 118870, 119003, 119002, 119004, 119005, 118937, 118991, 118995, - 118990, 118992, 118994, 118993, 118930, 118933, 118931, 118932, 119014, - 118918, 118912, 119015, 118785, 118831, 118907, 118966, 118900, 118880, - 118798, 118888, 118884, 118869, 118960, 118959, 118958, 118836, 118969, - 118977, 118978, 118971, 118976, 118975, 118973, 118970, 118987, 118979, - 118980, 118972, 118983, 983274, 118982, 118974, 118988, 118981, 119000, - 119001, 118929, 118928, 118806, 119029, 118927, 118898, 118964, 118965, - 118853, 118968, 118837, 118967, 118936, 118956, 118810, 118854, 118847, - 118839, 118906, 118791, 118800, 119024, 119026, 118862, 118812, 119025, - 119027, 118863, 118811, 118820, 119028, 118911, 118842, 118850, 118921, - 118858, 118913, 118914, 118915, 118834, 118860, 118868, 118796, 118881, - 118922, 118926, 118925, 118924, 118923, 118830, 118877, 118949, 118947, - 118948, 118963, 118955, 118962, 118946, 118961, 118953, 118952, 118951, - 118950, 118954, 118871, 118805, 118855, 118828, 118901, 118787, 118788, - 118856, 118816, 118875, 118845, 118879, 118793, 118846, 118878, 118814, - 118822, 118873, 118859, 118857, 118849, 118840, 118861, 118786, 118895, - 118841, 118874, 118892, 118897, 118807, 118784, 118821, 118844, 118825, - 118889, 119010, 119012, 119013, 119011, 119006, 119008, 119009, 119007, - 118910, 118815, 118852, 119016, 118826, 118885, 118827, 118803, 118886, - 118792, 118813, 118920, 118799, 118833, 118829, 118904, 118902, 118903, - 118905, 118804, 118934, 118919, 118832, 118893, 118838, 118851, 118883, - 118894, 118891, 118896, 118823, 118789, 118790, 118872, 118817, 118809, - 118908, 118909, 118996, 118998, 118997, 118999, 118989, 118794, 118795, - 118867, 118864, 118848, 118890, 983262, 983126, 983057, 9764, 8454, - 129305, 128197, 128247, 128248, 127957, 983098, 5130, 5131, 5122, 6322, - 5148, 5551, 5310, 5382, 5166, 6321, 5759, 5559, 5556, 5557, 5558, 5567, - 5564, 5565, 5566, 5563, 5560, 5561, 5562, 5555, 5552, 5553, 5554, 5384, - 5439, 6387, 6388, 5281, 5264, 6382, 6389, 5203, 5674, 5675, 5677, 5676, - 5673, 5672, 5706, 5707, 5709, 5708, 5705, 5704, 5204, 5574, 5575, 5577, - 5576, 5573, 5572, 6384, 6381, 5620, 6383, 5617, 5618, 5619, 5616, 5615, - 5195, 5592, 5593, 5595, 5594, 5591, 5590, 5174, 5175, 5129, 5703, 5662, - 5663, 5665, 5664, 5661, 5660, 5655, 5656, 6386, 5659, 5657, 5654, 5652, - 5633, 5629, 5630, 5632, 5631, 5628, 5627, 5623, 5624, 5626, 5625, 5622, - 5621, 5636, 5637, 5639, 5329, 5638, 5635, 5634, 5722, 5718, 5719, 5721, - 5720, 5717, 5716, 5712, 5713, 5715, 5714, 5711, 5710, 5614, 5610, 5611, - 5613, 5612, 5609, 5608, 5702, 5698, 5699, 5701, 5700, 5697, 5696, 5686, - 5687, 5689, 5688, 5685, 5684, 5692, 5693, 5695, 5694, 5691, 5690, 5737, - 5738, 5740, 5739, 5736, 5735, 5604, 5605, 5607, 5606, 5603, 5602, 5598, - 5599, 5601, 5600, 5597, 5596, 5725, 5726, 5728, 5727, 5724, 5723, 5680, - 5681, 5683, 5682, 5679, 5678, 5668, 5669, 5671, 5670, 5667, 5666, 5731, - 5732, 5734, 5733, 5730, 5729, 5642, 5643, 5645, 5644, 5641, 5640, 5580, - 5581, 5583, 5582, 5579, 5578, 5586, 5587, 5589, 5588, 5585, 5584, 5648, - 5649, 5651, 5650, 5647, 5646, 5128, 5265, 5258, 5276, 5278, 5272, 5274, - 5268, 5270, 5266, 5741, 5261, 5262, 5259, 5260, 5257, 5121, 6364, 5163, - 5469, 5465, 5466, 5460, 5461, 5158, 5157, 5162, 5155, 5156, 6367, 6366, - 5160, 5152, 5151, 5153, 5154, 5161, 5159, 5462, 5742, 5463, 5464, 5467, - 5459, 5120, 5501, 5123, 5124, 5164, 5251, 5252, 5246, 5248, 6329, 5242, - 5244, 5238, 5240, 5236, 5234, 5235, 5228, 6328, 5231, 5232, 5229, 5230, - 5227, 5354, 5542, 5540, 5541, 5538, 5539, 5536, 5537, 5338, 5339, 5332, - 6333, 5350, 5352, 5346, 5348, 5342, 5344, 5340, 5335, 5336, 5333, 5334, - 5331, 5307, 5283, 5356, 5458, 5287, 5288, 5385, 5290, 5291, 5284, 6330, - 5302, 5304, 5298, 5300, 5294, 5296, 5292, 5285, 5286, 5309, 5328, 5473, - 5475, 5471, 5319, 5386, 5390, 5391, 5388, 5389, 5380, 5387, 5142, 5147, - 5280, 5250, 5306, 5327, 5221, 5437, 72372, 72373, 72370, 72371, 72368, - 72369, 72378, 72379, 72376, 72377, 72374, 72375, 5320, 5313, 6332, 5525, - 5523, 5524, 5518, 5744, 5521, 5522, 5519, 5520, 5526, 5749, 5750, 5747, - 5748, 5745, 5746, 5499, 5497, 5498, 5495, 5496, 5493, 5494, 5492, 5500, - 5316, 5317, 6331, 5323, 5325, 6346, 6348, 6342, 6344, 5321, 5314, 5315, - 5312, 5330, 5509, 5507, 5508, 5502, 5743, 5505, 5506, 5503, 5504, 5125, - 6361, 6347, 6349, 6343, 6345, 6362, 6363, 6359, 6358, 6360, 6356, 6357, - 5165, 5126, 6320, 5193, 5184, 5186, 6326, 5188, 5190, 5180, 5182, 5178, - 5176, 5177, 5168, 6325, 5171, 5172, 6324, 5169, 5170, 5167, 5456, 6368, - 5443, 6355, 5454, 6353, 6354, 6351, 6352, 6350, 5451, 5452, 5445, 6341, - 5448, 5449, 5446, 5447, 5442, 5381, 5364, 6335, 5570, 6380, 5571, 5568, - 5569, 5653, 6385, 5658, 5529, 6379, 6378, 5530, 5527, 5528, 5441, 5282, - 5311, 5365, 5358, 5413, 5405, 5407, 6338, 5409, 5411, 5401, 5403, 5399, - 5395, 5396, 6336, 5397, 5398, 6337, 5393, 5394, 5392, 5361, 5256, 5253, - 5254, 5255, 5362, 6334, 5383, 5376, 5378, 5372, 5374, 5368, 5370, 5366, - 72383, 72380, 72381, 72382, 5359, 5360, 5357, 5222, 5482, 5550, 5548, - 5549, 5546, 5547, 5544, 5545, 5543, 6372, 5480, 6371, 5478, 5479, 5476, - 5477, 5472, 5474, 5470, 5512, 6377, 6376, 5513, 5510, 5511, 5487, 5486, - 6375, 5485, 6374, 6373, 5483, 5484, 5226, 5223, 5224, 5225, 5205, 5206, - 5197, 6327, 5217, 5219, 5213, 5215, 5209, 5211, 5207, 5491, 5488, 5489, - 5490, 5200, 5201, 5198, 5199, 5196, 5132, 5355, 5351, 5353, 5347, 5349, - 5343, 5345, 5341, 5453, 6370, 5450, 6369, 5444, 5308, 5303, 5305, 5299, - 5301, 5295, 5297, 5293, 5194, 5189, 5191, 5185, 5187, 5181, 5183, 5179, - 5440, 5434, 5436, 5430, 5432, 5426, 5428, 5424, 5324, 5326, 5322, 5457, - 5455, 5517, 5514, 5515, 5516, 5410, 5412, 5406, 5408, 5402, 5404, 5400, - 5377, 5379, 5373, 5375, 5369, 5371, 5367, 5277, 5279, 5273, 5275, 5269, - 5271, 5267, 5247, 5249, 5243, 5245, 5239, 5241, 5237, 5481, 5218, 5220, - 5214, 5216, 5210, 5212, 5208, 5468, 5144, 5146, 5139, 5141, 5135, 5137, - 5133, 6365, 5138, 5140, 5535, 5756, 5757, 5754, 5755, 5752, 5753, 5751, - 5534, 5531, 5532, 5533, 5758, 5143, 5145, 6323, 5134, 5136, 5438, 5192, - 5173, 5263, 5233, 5337, 5289, 5318, 5363, 5202, 5420, 5127, 5149, 5421, - 5422, 5415, 6340, 5418, 5419, 6339, 5433, 5435, 5429, 5431, 5425, 5427, - 5423, 5416, 5417, 5414, 5150, 983097, 983171, 917631, 128473, 9803, - 128367, 127852, 129387, 128758, 11839, 9809, 128199, 128450, 128451, - 8248, 8257, 8453, 66225, 66246, 66211, 66214, 66254, 66218, 66250, 66251, - 66252, 66253, 66229, 66238, 66244, 66227, 66224, 66222, 66223, 66242, - 66243, 66232, 66221, 66247, 66230, 66226, 66239, 66212, 66248, 66256, - 66235, 66208, 66215, 66210, 66220, 66234, 66255, 66240, 66241, 66236, - 66237, 66231, 66209, 66213, 66249, 66233, 66245, 66217, 66219, 66216, - 66228, 127904, 711, 127887, 129690, 983073, 129365, 9936, 128008, 128049, - 128569, 128572, 66888, 66864, 66912, 66882, 66873, 66902, 66889, 66890, - 66911, 66891, 66895, 66901, 66881, 66867, 66870, 66868, 66904, 66866, - 66876, 66910, 66879, 66883, 66897, 66915, 66884, 66878, 66885, 66914, - 66903, 66877, 66896, 66909, 66906, 66908, 66899, 66872, 66913, 66874, - 66871, 66875, 66869, 66893, 66887, 66892, 66907, 66900, 66894, 66905, - 66880, 66898, 66865, 66886, 66927, 9761, 127797, 9963, 8373, 184, 65102, - 65098, 8452, 162, 128328, 9907, 9939, 129681, 69908, 69907, 69913, 69912, - 69899, 69909, 69904, 69914, 69906, 69905, 69911, 69910, 69920, 69921, - 69918, 69917, 69901, 69900, 69898, 69897, 69903, 69902, 69896, 69895, - 69956, 69923, 69916, 69915, 69926, 69919, 69922, 69925, 69959, 69924, - 69891, 69894, 69892, 69893, 69888, 69890, 69889, 69952, 69927, 69957, - 69933, 69935, 69930, 69931, 69932, 69958, 69928, 69929, 69934, 69936, - 69939, 69954, 69953, 69947, 69946, 69949, 69948, 69945, 69944, 69942, - 69951, 69943, 69950, 69955, 69940, 69938, 69937, 43587, 43597, 43596, - 43573, 43572, 43574, 43571, 43590, 43586, 43595, 43588, 43585, 43584, - 43594, 43591, 43593, 43589, 43592, 43530, 43531, 43536, 43538, 43537, - 43543, 43544, 43551, 43552, 43548, 43547, 43546, 43553, 43550, 43549, - 43545, 43542, 43541, 43558, 43559, 43520, 43524, 43533, 43532, 43529, - 43528, 43535, 43534, 43527, 43526, 43540, 43539, 43560, 43556, 43555, - 43557, 43554, 43523, 43521, 43525, 43522, 43612, 43614, 43613, 43615, - 43561, 43568, 43569, 43562, 43563, 43567, 43566, 43565, 43570, 43564, - 43605, 43604, 43607, 43606, 43603, 43602, 43600, 43609, 43601, 43608, - 983058, 983140, 983137, 8256, 128200, 128185, 128201, 128638, 129941, - 10003, 129472, 128227, 5084, 5075, 5077, 5079, 5081, 5082, 5083, 5055, - 5037, 5038, 5039, 5040, 5041, 5042, 5054, 5056, 5057, 5058, 5059, 5060, - 5061, 5069, 5068, 5070, 5071, 5072, 5073, 5074, 5085, 5086, 5087, 5088, - 5089, 5090, 5091, 5092, 5093, 5094, 5095, 5096, 5076, 5078, 5080, 5030, - 5032, 5033, 5034, 5035, 5036, 5043, 5044, 5045, 5046, 5047, 5048, 5049, - 5050, 5051, 5052, 5053, 5109, 5062, 5063, 5064, 5065, 5066, 5067, 5097, - 5098, 5099, 5100, 5101, 5102, 5103, 5104, 5105, 5106, 5107, 5108, 5031, - 5024, 5025, 5026, 5027, 5028, 5029, 43948, 43939, 43941, 43943, 43945, - 43946, 43947, 43919, 43901, 43902, 43903, 43904, 43905, 43906, 43918, - 43920, 43921, 43922, 43923, 43924, 43925, 43933, 43932, 43934, 43935, - 43936, 43937, 43938, 43949, 43950, 43951, 43952, 43953, 43954, 43955, - 43956, 43957, 43958, 43959, 43960, 43940, 43942, 43944, 43894, 43896, - 43897, 43898, 43899, 43900, 43907, 43908, 43909, 43910, 43911, 43912, - 43913, 43914, 43915, 43916, 43917, 5117, 43926, 43927, 43928, 43929, - 43930, 43931, 43961, 43962, 43963, 43964, 43965, 43966, 43967, 5112, - 5113, 5114, 5115, 5116, 43895, 43888, 43889, 43890, 43891, 43892, 43893, - 127800, 127826, 127792, 127937, 129490, 128696, 9767, 128063, 128020, - 9911, 69559, 69553, 69567, 69571, 69556, 69564, 69570, 69552, 69568, - 69555, 69560, 69562, 69557, 69561, 69563, 69554, 69566, 69572, 69558, - 69569, 69565, 69578, 69574, 69575, 69577, 69573, 69579, 69576, 129378, - 127851, 127876, 9962, 127910, 9680, 9682, 10690, 10683, 10691, 9684, - 9683, 9685, 9677, 9681, 10677, 10682, 10684, 127246, 8859, 11199, 10687, - 129282, 129280, 129281, 128320, 9938, 127342, 127341, 127277, 8856, - 10808, 9316, 9315, 9318, 9317, 9314, 9313, 9450, 9320, 9312, 9319, - 127247, 8857, 8861, 12905, 12919, 12904, 12918, 12903, 12917, 12926, - 12909, 12923, 12906, 12920, 12896, 12910, 12900, 12914, 12897, 12911, - 12908, 12922, 12901, 12915, 12899, 12913, 12902, 12916, 12907, 12921, - 12898, 12912, 9097, 127343, 10162, 12975, 127569, 12959, 127568, 12963, - 12951, 12962, 12965, 12973, 12943, 12957, 12935, 12950, 12939, 12932, - 12955, 12931, 12964, 12946, 12869, 12871, 12952, 12966, 12967, 12969, - 12942, 12954, 12938, 12976, 12936, 12948, 12868, 12974, 12961, 12970, - 12968, 12953, 12934, 12972, 12956, 12944, 12870, 12947, 12949, 12971, - 12945, 12933, 12958, 12930, 12937, 12929, 12941, 12940, 12960, 12928, - 127275, 127276, 128712, 13033, 13036, 13034, 13037, 13035, 13013, 13016, - 13014, 13017, 13015, 13038, 13041, 13039, 13042, 13040, 13028, 13031, - 13029, 13032, 13030, 13046, 13049, 13047, 13050, 13048, 13018, 13021, - 13019, 13022, 13020, 13023, 13026, 13024, 13027, 13025, 13051, 13053, - 13052, 13054, 13043, 13045, 13044, 13008, 13011, 13009, 13012, 13010, - 12925, 12924, 9398, 9399, 9400, 9401, 9402, 9403, 9404, 9405, 9406, 9407, - 9408, 9409, 9410, 9411, 9412, 9413, 9414, 9415, 9416, 9417, 9418, 9419, - 9420, 9421, 9422, 9423, 9424, 9425, 9426, 9427, 9428, 9429, 9430, 9431, - 9432, 9433, 9434, 9435, 9436, 9437, 9438, 9439, 9440, 9441, 9442, 9443, - 9444, 9445, 9446, 9447, 9448, 9449, 10688, 10806, 8854, 12879, 9329, - 9322, 12991, 12876, 9326, 12981, 12875, 12982, 12986, 12985, 12988, - 12987, 12984, 12983, 12990, 12989, 9325, 12878, 9328, 12877, 9327, 9321, - 12872, 12890, 12874, 12891, 12895, 12894, 12978, 12977, 12893, 12892, - 12980, 12979, 9324, 9331, 12873, 12881, 12885, 12884, 12887, 12886, - 12883, 12882, 12889, 12888, 9323, 9330, 10050, 12342, 10679, 10681, 8853, - 8858, 10680, 128981, 9098, 8855, 10686, 10026, 127278, 127245, 8860, - 10678, 10689, 128983, 11198, 94, 10768, 127914, 127961, 127750, 195088, - 195089, 195090, 195091, 195092, 195093, 195094, 195095, 195096, 195097, - 195098, 195099, 195100, 195101, 195072, 195073, 195074, 195075, 195076, - 195077, 195078, 195079, 195080, 195081, 195082, 195083, 195084, 195085, - 195086, 195087, 194560, 194561, 194562, 194563, 194564, 194565, 194566, - 194567, 194568, 194569, 194570, 194571, 194572, 194573, 194574, 194575, - 194576, 194577, 194578, 194579, 194580, 194581, 194582, 194583, 194584, - 194585, 194586, 194587, 194588, 194589, 194590, 194591, 194592, 194593, - 194594, 194595, 194596, 194597, 194598, 194599, 194600, 194601, 194602, - 194603, 194604, 194605, 194606, 194607, 194608, 194609, 194610, 194611, - 194612, 194613, 194614, 194615, 194616, 194617, 194618, 194619, 194620, - 194621, 194622, 194623, 194624, 194625, 194626, 194627, 194628, 194629, - 194630, 194631, 194632, 194633, 194634, 194635, 194636, 194637, 194638, - 194639, 194640, 194641, 194642, 194643, 194644, 194645, 194646, 194647, - 194648, 194649, 194650, 194651, 194652, 194653, 194654, 194655, 194656, - 194657, 194658, 194659, 194660, 194661, 194662, 194663, 194664, 194665, - 194666, 194667, 194668, 194669, 194670, 194671, 194672, 194673, 194674, - 194675, 194676, 194677, 194678, 194679, 194680, 194681, 194682, 194683, - 194684, 194685, 194686, 194687, 194688, 194689, 194690, 194691, 194692, - 194693, 194694, 194695, 194696, 194697, 194698, 194699, 194700, 194701, - 194702, 194703, 194704, 194705, 194706, 194707, 194708, 194709, 194710, - 194711, 194712, 194713, 194714, 194715, 194716, 194717, 194718, 194719, - 194720, 194721, 194722, 194723, 194724, 194725, 194726, 194727, 194728, - 194729, 194730, 194731, 194732, 194733, 194734, 194735, 194736, 194737, - 194738, 194739, 194740, 194741, 194742, 194743, 194744, 194745, 194746, - 194747, 194748, 194749, 194750, 194751, 194752, 194753, 194754, 194755, - 194756, 194757, 194758, 194759, 194760, 194761, 194762, 194763, 194764, - 194765, 194766, 194767, 194768, 194769, 194770, 194771, 194772, 194773, - 194774, 194775, 194776, 194777, 194778, 194779, 194780, 194781, 194782, - 194783, 194784, 194785, 194786, 194787, 194788, 194789, 194790, 194791, - 194792, 194793, 194794, 194795, 194796, 194797, 194798, 194799, 194800, - 194801, 194802, 194803, 194804, 194805, 194806, 194807, 194808, 194809, - 194810, 194811, 194812, 194813, 194814, 194815, 194816, 194817, 194818, - 194819, 194820, 194821, 194822, 194823, 194824, 194825, 194826, 194827, - 194828, 194829, 194830, 194831, 194832, 194833, 194834, 194835, 194836, - 194837, 194838, 194839, 194840, 194841, 194842, 194843, 194844, 194845, - 194846, 194847, 194848, 194849, 194850, 194851, 194852, 194853, 194854, - 194855, 194856, 194857, 194858, 194859, 194860, 194861, 194862, 194863, - 194864, 194865, 194866, 194867, 194868, 194869, 194870, 194871, 194872, - 194873, 194874, 194875, 194876, 194877, 194878, 194879, 194880, 194881, - 194882, 194883, 194884, 194885, 194886, 194887, 194888, 194889, 194890, - 194891, 194892, 194893, 194894, 194895, 194896, 194897, 194898, 194899, - 194900, 194901, 194902, 194903, 194904, 194905, 194906, 194907, 194908, - 194909, 194910, 194911, 194912, 194913, 194914, 194915, 194916, 194917, - 194918, 194919, 194920, 194921, 194922, 194923, 194924, 194925, 194926, - 194927, 194928, 194929, 194930, 194931, 194932, 194933, 194934, 194935, - 194936, 194937, 194938, 194939, 194940, 194941, 194942, 194943, 194944, - 194945, 194946, 194947, 194948, 194949, 194950, 194951, 194952, 194953, - 194954, 194955, 194956, 194957, 194958, 194959, 194960, 194961, 194962, - 194963, 194964, 194965, 194966, 194967, 194968, 194969, 194970, 194971, - 194972, 194973, 194974, 194975, 194976, 194977, 194978, 194979, 194980, - 194981, 194982, 194983, 194984, 194985, 194986, 194987, 194988, 194989, - 194990, 194991, 194992, 194993, 194994, 194995, 194996, 194997, 194998, - 194999, 195000, 195001, 195002, 195003, 195004, 195005, 195006, 195007, - 195008, 195009, 195010, 195011, 195012, 195013, 195014, 195015, 195016, - 195017, 195018, 195019, 195020, 195021, 195022, 195023, 195024, 195025, - 195026, 195027, 195028, 195029, 195030, 195031, 195032, 195033, 195034, - 195035, 195036, 195037, 195038, 195039, 195040, 195041, 195042, 195043, - 195044, 195045, 195046, 195047, 195048, 195049, 195050, 195051, 195052, - 195053, 195054, 195055, 195056, 195057, 195058, 195059, 195060, 195061, - 195062, 195063, 195064, 195065, 195066, 195067, 195068, 195069, 195070, - 195071, 64096, 64097, 64098, 64099, 64100, 64101, 64102, 64103, 64104, - 64105, 64106, 64107, 64108, 64109, 64000, 64001, 64002, 64003, 64004, - 64005, 64006, 64007, 64008, 64009, 64010, 64011, 64012, 64013, 64014, - 64015, 64016, 64017, 64018, 64019, 64020, 64021, 64022, 64023, 64024, - 64025, 64026, 64027, 64028, 64029, 64030, 64031, 64032, 64033, 64034, - 64035, 64036, 64037, 64038, 64039, 64040, 64041, 64042, 64043, 64044, - 64045, 64046, 64047, 64048, 64049, 64050, 64051, 64052, 64053, 64054, - 64055, 64056, 64057, 64058, 64059, 64060, 64061, 64062, 64063, 64064, - 64065, 64066, 64067, 64068, 64069, 64070, 64071, 64072, 64073, 64074, - 64075, 64076, 64077, 64078, 64079, 64080, 64081, 64082, 64083, 64084, - 64085, 64086, 64087, 64088, 64089, 64090, 64091, 64092, 64093, 64094, - 64095, 64112, 64113, 64114, 64115, 64116, 64117, 64118, 64119, 64120, - 64121, 64122, 64123, 64124, 64125, 64126, 64127, 64128, 64129, 64130, - 64131, 64132, 64133, 64134, 64135, 64136, 64137, 64138, 64139, 64140, - 64141, 64142, 64143, 64144, 64145, 64146, 64147, 64148, 64149, 64150, - 64151, 64152, 64153, 64154, 64155, 64156, 64157, 64158, 64159, 64160, - 64161, 64162, 64163, 64164, 64165, 64166, 64167, 64168, 64169, 64170, - 64171, 64172, 64173, 64174, 64175, 64176, 64177, 64178, 64179, 64180, - 64181, 64182, 64183, 64184, 64185, 64186, 64187, 64188, 64189, 64190, - 64191, 64192, 64193, 64194, 64195, 64196, 64197, 64198, 64199, 64200, - 64201, 64202, 64203, 64204, 64205, 64206, 64207, 64208, 64209, 64210, - 64211, 64212, 64213, 64214, 64215, 64216, 64217, 63744, 63745, 63746, - 63747, 63748, 63749, 63750, 63751, 63752, 63753, 63754, 63755, 63756, - 63757, 63758, 63759, 63760, 63761, 63762, 63763, 63764, 63765, 63766, - 63767, 63768, 63769, 63770, 63771, 63772, 63773, 63774, 63775, 63776, - 63777, 63778, 63779, 63780, 63781, 63782, 63783, 63784, 63785, 63786, - 63787, 63788, 63789, 63790, 63791, 63792, 63793, 63794, 63795, 63796, - 63797, 63798, 63799, 63800, 63801, 63802, 63803, 63804, 63805, 63806, - 63807, 63808, 63809, 63810, 63811, 63812, 63813, 63814, 63815, 63816, - 63817, 63818, 63819, 63820, 63821, 63822, 63823, 63824, 63825, 63826, - 63827, 63828, 63829, 63830, 63831, 63832, 63833, 63834, 63835, 63836, - 63837, 63838, 63839, 63840, 63841, 63842, 63843, 63844, 63845, 63846, - 63847, 63848, 63849, 63850, 63851, 63852, 63853, 63854, 63855, 63856, - 63857, 63858, 63859, 63860, 63861, 63862, 63863, 63864, 63865, 63866, - 63867, 63868, 63869, 63870, 63871, 63872, 63873, 63874, 63875, 63876, - 63877, 63878, 63879, 63880, 63881, 63882, 63883, 63884, 63885, 63886, - 63887, 63888, 63889, 63890, 63891, 63892, 63893, 63894, 63895, 63896, - 63897, 63898, 63899, 63900, 63901, 63902, 63903, 63904, 63905, 63906, - 63907, 63908, 63909, 63910, 63911, 63912, 63913, 63914, 63915, 63916, - 63917, 63918, 63919, 63920, 63921, 63922, 63923, 63924, 63925, 63926, - 63927, 63928, 63929, 63930, 63931, 63932, 63933, 63934, 63935, 63936, - 63937, 63938, 63939, 63940, 63941, 63942, 63943, 63944, 63945, 63946, - 63947, 63948, 63949, 63950, 63951, 63952, 63953, 63954, 63955, 63956, - 63957, 63958, 63959, 63960, 63961, 63962, 63963, 63964, 63965, 63966, - 63967, 63968, 63969, 63970, 63971, 63972, 63973, 63974, 63975, 63976, - 63977, 63978, 63979, 63980, 63981, 63982, 63983, 63984, 63985, 63986, - 63987, 63988, 63989, 63990, 63991, 63992, 63993, 63994, 63995, 63996, - 63997, 63998, 63999, 11946, 12003, 11910, 11963, 11962, 11950, 11992, - 12012, 12000, 12005, 11996, 12010, 11984, 11988, 11994, 11987, 11952, - 12007, 11977, 11976, 11973, 12019, 11993, 12014, 11995, 12016, 12006, - 12002, 11979, 11936, 11983, 11905, 11970, 11943, 11931, 11914, 11934, - 11944, 11999, 11998, 11997, 11960, 11947, 11939, 11978, 11968, 11967, - 11966, 12004, 11927, 11926, 12001, 11975, 11928, 12018, 12013, 12015, - 12011, 11945, 11986, 11985, 11921, 11920, 11919, 11918, 11964, 11957, - 11990, 11989, 11935, 11965, 11933, 11941, 11940, 11909, 11991, 11959, - 11929, 11904, 11908, 11907, 11906, 11915, 11942, 11974, 11980, 12008, - 12009, 11951, 11925, 11924, 11922, 11949, 11948, 11917, 11916, 11958, - 11932, 12017, 11911, 11923, 11969, 11982, 11981, 11938, 11937, 11972, - 11971, 11956, 11955, 11954, 11953, 11913, 11912, 11961, 12752, 12743, - 12748, 12768, 12772, 12757, 12741, 12750, 12769, 12747, 12749, 12744, - 12742, 12746, 12758, 12754, 12763, 12770, 12764, 12753, 12740, 12767, - 12760, 12759, 12745, 12773, 12766, 12762, 12755, 12761, 12736, 12765, - 12739, 12737, 12738, 12756, 12751, 12771, 128079, 127916, 127963, 128385, - 129346, 127867, 128203, 128346, 128358, 128343, 128355, 128340, 128352, - 128339, 128351, 128344, 128356, 128336, 128348, 128342, 128354, 128341, - 128353, 128345, 128357, 128347, 128359, 128337, 128349, 128338, 128350, - 10561, 8754, 128259, 10227, 128472, 128257, 128258, 8631, 11118, 8635, - 8753, 10959, 10961, 10960, 10962, 127746, 10828, 10832, 128234, 128235, - 128272, 128213, 10829, 8272, 9729, 127786, 127785, 127784, 127783, - 129313, 9114, 129715, 127864, 129381, 58, 8788, 8353, 128165, 7625, 7623, - 769, 791, 833, 8410, 8404, 8423, 857, 8432, 7677, 844, 774, 7627, 814, - 810, 838, 70459, 780, 812, 784, 8409, 8405, 787, 789, 806, 65062, 65069, - 42609, 1160, 11774, 11744, 11768, 11747, 11757, 11765, 42654, 11751, - 11752, 11753, 11756, 42613, 11775, 11772, 42655, 11767, 11754, 42619, - 11763, 11762, 42618, 42615, 42612, 42617, 11770, 42614, 11771, 11773, - 11769, 11759, 42616, 11760, 11758, 11748, 11749, 11761, 11746, 11764, - 11755, 11745, 11750, 11766, 42621, 1156, 1158, 1159, 1157, 42608, 42610, - 1155, 65070, 65071, 1161, 42620, 123023, 42607, 770, 813, 807, 43248, - 43244, 43245, 43246, 43247, 43242, 43243, 43249, 43237, 43236, 43239, - 43238, 43235, 43234, 43232, 43241, 43233, 43240, 7675, 776, 6833, 804, - 775, 7672, 856, 803, 7674, 7617, 7616, 861, 860, 865, 7676, 862, 863, - 6840, 831, 6858, 6857, 6844, 866, 858, 864, 65058, 65059, 8422, 840, 782, - 779, 783, 819, 7629, 6832, 798, 6835, 8413, 8416, 8418, 8414, 8419, 8420, - 8415, 839, 850, 8412, 122892, 122884, 122891, 122890, 122921, 122919, - 122889, 122916, 122900, 122907, 122910, 122901, 122908, 122880, 122920, - 122881, 122909, 122903, 122922, 122883, 122895, 122894, 122896, 122898, - 122899, 122882, 122885, 122912, 122911, 122913, 122918, 122915, 122888, - 122886, 122904, 122902, 122897, 122893, 70508, 70507, 70506, 70505, - 70504, 70502, 70503, 70515, 70513, 70514, 70516, 70512, 768, 790, 832, - 7624, 7621, 847, 119363, 119362, 119364, 835, 834, 837, 836, 777, 843, - 795, 785, 815, 826, 811, 6855, 6834, 7632, 12442, 12441, 7671, 7670, - 7643, 7646, 7647, 7649, 7650, 867, 7666, 7655, 7636, 7637, 7638, 872, - 7639, 868, 7663, 7641, 7659, 7635, 869, 7640, 6860, 6861, 6862, 7645, - 7660, 7653, 870, 7667, 7661, 871, 7668, 7664, 876, 7651, 7626, 877, 6848, - 7652, 7658, 7656, 7657, 7665, 6847, 873, 7642, 874, 7644, 875, 7648, - 7662, 878, 879, 7654, 852, 7678, 8430, 8406, 841, 794, 6849, 6851, 796, - 849, 8400, 792, 845, 8417, 8429, 8426, 65056, 65063, 65057, 65064, 6841, - 8427, 824, 822, 8402, 818, 772, 65060, 65067, 65061, 65068, 817, 7622, - 7620, 7628, 800, 6854, 842, 66424, 66425, 66423, 66426, 66422, 6839, 808, - 7630, 773, 6846, 6845, 6843, 801, 799, 6856, 8421, 788, 802, 7679, 854, - 848, 853, 8431, 8407, 825, 855, 8401, 6850, 6852, 793, 8428, 8408, 805, - 778, 7631, 859, 823, 821, 8403, 6853, 827, 6842, 7619, 7618, 828, 8411, - 771, 65065, 65066, 820, 816, 6859, 8424, 6836, 786, 846, 797, 7669, 7633, - 7634, 809, 781, 830, 7673, 8425, 6838, 6837, 851, 829, 8274, 64, 44, + 64975, 65008, 65017, 64464, 64838, 64972, 64844, 65018, 64860, 64686, + 64541, 64821, 64817, 64744, 64862, 64861, 64685, 64540, 64820, 64936, + 64966, 64687, 64542, 64822, 64864, 64863, 64865, 64867, 64866, 64688, + 64543, 64743, 64791, 64763, 64810, 64782, 64792, 64764, 64606, 64609, + 64755, 64607, 64610, 64756, 64611, 64608, 64754, 64818, 64746, 64872, + 64871, 64938, 64806, 64814, 64778, 64824, 64873, 64805, 64813, 64777, + 64823, 64875, 64874, 64877, 64876, 64808, 64816, 64780, 64745, 64793, + 64765, 64807, 64815, 64779, 64825, 64809, 64781, 64794, 64766, 65022, + 64846, 64882, 64881, 64883, 64884, 64819, 64551, 64826, 64785, 64757, + 64696, 64550, 64786, 64758, 64851, 64850, 64849, 64674, 64524, 64677, + 64740, 64930, 64852, 64929, 64675, 64525, 64928, 64848, 64927, 64673, + 64523, 64932, 64853, 64855, 64854, 64931, 64626, 64676, 64526, 64739, + 64628, 64527, 64627, 64625, 64624, 64629, 64528, 64603, 64529, 64634, + 64531, 64632, 64678, 64530, 64741, 64633, 64631, 64630, 64635, 64532, + 64742, 65016, 64661, 64601, 64616, 64515, 64491, 64490, 64493, 64492, + 64503, 64504, 64502, 64667, 64736, 64664, 64513, 64663, 64512, 64665, + 64614, 64666, 64514, 64735, 64615, 64499, 64498, 64495, 64494, 64617, + 64516, 64501, 64500, 64613, 64612, 64497, 64496, 64942, 64731, 64598, + 64734, 64753, 64660, 64658, 64943, 64730, 64597, 64732, 64599, 64925, + 64924, 64944, 64659, 64733, 64600, 64752, 64657, 64662, 64602, 64506, + 64507, 64505, 64697, 64552, 64827, 2204, 1619, 1624, 2303, 126492, + 126494, 126493, 126495, 126644, 126638, 126641, 126647, 126648, 126646, + 126632, 126645, 126630, 126650, 126626, 126636, 126625, 126640, 126643, + 126633, 126631, 126651, 126637, 126635, 126649, 126627, 126642, 126639, + 126629, 126489, 126467, 126518, 126517, 126516, 126510, 126513, 126503, + 126500, 126519, 126506, 126498, 126508, 126497, 126512, 126505, 126523, + 126509, 126507, 126514, 126511, 126521, 126612, 126606, 126609, 126615, + 126592, 126607, 126599, 126596, 126616, 126614, 126600, 126613, 126598, + 126618, 126594, 126604, 126593, 126608, 126611, 126601, 126619, 126605, + 126603, 126617, 126595, 126610, 126597, 126475, 126704, 126705, 126588, + 126590, 126585, 126582, 126568, 126581, 126580, 126574, 126577, 126567, + 126564, 126583, 126570, 126562, 126572, 126561, 126576, 126569, 126586, + 126587, 126573, 126578, 126575, 126484, 126478, 126481, 126557, 126559, + 126553, 126548, 126542, 126545, 126551, 126530, 126537, 126535, 126555, + 126541, 126539, 126546, 126543, 126472, 126488, 126486, 126485, 126464, + 126479, 126487, 126474, 126470, 126490, 126466, 126476, 126465, 126480, + 126483, 126473, 126471, 126491, 126477, 126482, 126469, 1541, 1536, 2290, + 2289, 2288, 1642, 2199, 1550, 2192, 1769, 2193, 2184, 1544, 1629, 2296, + 2301, 2298, 1772, 1623, 983633, 983635, 983640, 983634, 983638, 983636, + 983639, 983637, 983641, 1563, 1617, 65148, 65149, 1554, 1555, 1552, 1537, + 1539, 1540, 1790, 1789, 1553, 1551, 1556, 2249, 1560, 1761, 2250, 2272, + 1558, 983202, 1751, 1750, 1753, 1752, 1762, 1764, 2273, 1756, 2261, 1755, + 1557, 2200, 2260, 2266, 2268, 2267, 2269, 2252, 2271, 2270, 2291, 1767, + 2251, 1768, 2264, 1760, 1759, 1559, 2253, 1754, 2263, 2262, 1766, 69317, + 69371, 2265, 2202, 2201, 69375, 2203, 69373, 69374, 2259, 1773, 1763, + 1562, 1561, 1765, 1622, 1618, 65150, 65151, 2256, 2205, 64444, 64435, + 64434, 64441, 64440, 64439, 64438, 64446, 64445, 64437, 64436, 64449, + 64448, 64443, 64442, 64450, 64447, 1758, 1600, 2179, 2180, 65137, 2181, + 65139, 2285, 2282, 2286, 2283, 2287, 2284, 1566, 2275, 1644, 1627, 1626, + 1628, 2190, 1631, 1567, 1625, 1545, 1546, 1542, 1543, 1637, 1636, 1639, + 1638, 1635, 1634, 1632, 1641, 1633, 1640, 1375, 1370, 1359, 1337, 1349, + 1362, 1347, 1353, 1342, 1361, 1360, 1356, 1333, 1335, 1336, 1346, 1331, + 1355, 1345, 1364, 1343, 1363, 1354, 1351, 1357, 1329, 1358, 1352, 1340, + 1366, 1341, 1339, 1330, 1348, 1350, 1338, 1334, 1344, 1332, 1365, 1373, + 1372, 1371, 1395, 1401, 1390, 1409, 1408, 1404, 1381, 1383, 1384, 1394, + 1379, 1403, 1393, 1412, 1391, 1411, 1402, 1399, 1405, 1376, 1407, 1385, + 1377, 1406, 1400, 1397, 1416, 1410, 1388, 1414, 1389, 1387, 1378, 1396, + 1398, 1386, 1382, 1392, 1380, 1413, 1415, 64279, 64277, 64275, 64276, + 64278, 1369, 1418, 1423, 1417, 1374, 129201, 10549, 10548, 129200, 10550, + 10551, 129968, 128667, 127912, 9800, 8978, 9738, 65948, 42, 8727, 8258, + 128562, 118477, 9954, 11225, 128888, 8771, 8870, 128095, 9883, 128663, + 127975, 128762, 8371, 127814, 68352, 68353, 68357, 68355, 68358, 68359, + 68356, 68354, 68373, 68374, 68372, 68393, 68405, 68388, 68387, 68386, + 68391, 68390, 68389, 68371, 68370, 68369, 68403, 68401, 68404, 68399, + 68394, 68395, 68378, 68381, 68377, 68366, 68367, 68362, 68363, 68364, + 68365, 68385, 68384, 68380, 68379, 68402, 68400, 68360, 68361, 68375, + 68383, 68376, 68368, 68398, 68392, 68382, 68397, 68396, 68409, 129361, + 8525, 1547, 129518, 9810, 129683, 128118, 128036, 127868, 128124, 128700, + 128281, 128386, 11101, 11099, 983056, 10155, 128043, 129363, 127992, + 129441, 129366, 129391, 128708, 7007, 7005, 7006, 6991, 6990, 6917, 6918, + 6987, 6988, 6928, 6953, 6954, 6936, 6937, 6948, 6944, 6943, 6949, 6984, + 6927, 6933, 6934, 6919, 6920, 6929, 6930, 6921, 6922, 6938, 6939, 6931, + 6981, 6932, 6982, 6958, 6925, 6926, 6950, 6945, 6935, 6940, 6951, 6952, + 6957, 6923, 6924, 6962, 6960, 6961, 6946, 6942, 6941, 6947, 6983, 6985, + 6986, 6963, 6955, 6959, 6956, 7022, 7025, 7021, 7024, 7023, 7026, 7020, + 7019, 7027, 7012, 7013, 7018, 7015, 7017, 7016, 7010, 7014, 7009, 7011, + 7032, 7036, 7033, 7034, 7035, 7031, 7030, 7029, 7028, 7003, 7038, 7008, + 7002, 7037, 7039, 6915, 6913, 6912, 6914, 6916, 6964, 6972, 6973, 6970, + 6971, 6968, 6969, 6974, 6975, 6977, 6976, 6965, 6978, 6979, 6966, 6967, + 6980, 7004, 6997, 6996, 6999, 6998, 6995, 6994, 6992, 7001, 6993, 7000, + 127880, 10057, 9744, 128503, 128505, 128499, 128501, 11197, 9745, 9746, + 128502, 128500, 10007, 129526, 129648, 42737, 42736, 42741, 42740, 42733, + 42699, 42713, 42712, 42692, 42706, 42683, 42719, 42734, 42735, 42682, + 42729, 42657, 42725, 42659, 42670, 42718, 42666, 42716, 42701, 42675, + 42671, 42722, 42720, 42727, 42723, 42702, 42726, 42677, 42707, 42708, + 42709, 42686, 42694, 42674, 42731, 42695, 42685, 42684, 42691, 42673, + 42664, 42715, 42703, 92193, 92188, 92161, 92199, 92240, 92179, 92211, + 92219, 92213, 92243, 92207, 92216, 92183, 92184, 92238, 92171, 983268, + 92174, 92203, 92186, 92210, 92230, 92175, 92221, 92232, 92246, 92198, + 92214, 92190, 92176, 92197, 92196, 92164, 92245, 92212, 92201, 92160, + 92182, 92173, 92229, 92227, 92180, 92237, 92241, 92223, 92185, 92194, + 92178, 92239, 92225, 92233, 92167, 92191, 92231, 92244, 92204, 92226, + 92236, 92200, 92189, 92187, 92162, 92163, 92169, 92170, 92202, 92205, + 92222, 92168, 92165, 92215, 92224, 92208, 92220, 92235, 92177, 92181, + 92195, 92234, 92209, 92166, 92172, 92206, 92192, 92228, 92217, 92242, + 92218, 92262, 92271, 92272, 92270, 92294, 92253, 92301, 92256, 92273, + 92259, 92251, 92297, 92300, 92296, 92295, 92255, 92252, 92292, 92286, + 92268, 92290, 92281, 92267, 92284, 92287, 92282, 92283, 92277, 92298, + 983270, 92276, 92302, 92247, 92299, 92288, 92269, 92260, 92261, 983269, + 92289, 92257, 92274, 92263, 92279, 92265, 92266, 92250, 92249, 92285, + 92248, 92264, 92280, 92258, 92254, 92278, 92291, 92293, 92275, 92310, + 92320, 92312, 92394, 92393, 92319, 92384, 92321, 92386, 92361, 92373, + 92366, 92328, 92329, 92357, 92342, 92397, 92365, 92376, 92339, 92382, + 92363, 92318, 92387, 92383, 92315, 92385, 92354, 92311, 92381, 92343, + 92326, 92364, 92324, 92391, 92344, 92390, 92338, 92396, 92308, 92349, + 92375, 92378, 92317, 92331, 92362, 92336, 92341, 92370, 92347, 92307, + 92303, 92309, 92395, 92368, 92356, 92367, 92360, 92389, 92333, 92359, + 92374, 92351, 92325, 92371, 92350, 92345, 92372, 92314, 92340, 92323, + 92304, 92313, 92316, 92398, 92399, 92380, 92330, 983271, 92379, 92392, + 92335, 92332, 92346, 92400, 92358, 92334, 92353, 92337, 92306, 92369, + 92388, 92322, 92305, 92327, 92377, 92352, 92348, 92355, 92436, 92517, + 92488, 92434, 92415, 92431, 92454, 92460, 92420, 92489, 92422, 92474, + 92479, 92449, 92502, 92439, 92484, 92495, 92464, 92511, 92406, 92446, + 92497, 92426, 92482, 92448, 92515, 92401, 92478, 92427, 92496, 92458, + 92424, 92445, 92404, 92466, 92444, 92438, 92442, 92441, 92510, 92499, + 92425, 92437, 92440, 92471, 92465, 92409, 92483, 92475, 92467, 92485, + 92457, 92432, 92476, 92435, 92412, 92414, 92430, 92407, 92403, 92405, + 92487, 92418, 92486, 92408, 92447, 92459, 92480, 92472, 92505, 92514, + 92450, 92463, 92410, 92493, 92507, 92503, 92462, 92506, 92443, 92509, + 92469, 92461, 92490, 92512, 92494, 92455, 92417, 92501, 92413, 92508, + 92500, 92504, 92473, 92419, 92498, 92423, 92516, 92428, 92452, 92451, + 92481, 92416, 92456, 92433, 92477, 92492, 92491, 92513, 92411, 92402, + 92421, 92453, 92468, 92470, 92429, 92661, 92626, 92616, 92596, 92612, + 92603, 92673, 92651, 92627, 92597, 92585, 92578, 92615, 92565, 92552, + 92602, 92674, 92570, 92646, 92599, 92532, 92587, 92538, 92620, 92670, + 92665, 92575, 92521, 92633, 92595, 92523, 92556, 92539, 92664, 92653, + 92667, 92600, 92542, 92555, 92668, 92520, 92582, 92591, 92581, 92551, + 92654, 92611, 92614, 92666, 92671, 92617, 92637, 92562, 92518, 92592, + 92558, 92528, 92607, 92535, 92557, 92563, 92550, 92624, 983272, 92640, + 92657, 92531, 92601, 92553, 92543, 92619, 92598, 92541, 92544, 92579, + 92658, 92554, 92628, 92648, 92547, 92527, 92545, 92540, 92604, 92622, + 92589, 92608, 92583, 92609, 92662, 92613, 92584, 92625, 92634, 92524, + 92536, 92605, 92647, 92548, 92663, 92593, 92621, 92568, 92610, 92576, + 92529, 92618, 92649, 92561, 92656, 92526, 92655, 92546, 92534, 92560, + 92580, 92659, 92660, 92638, 92571, 92525, 92549, 92559, 92635, 92577, + 92530, 92636, 92669, 92572, 92672, 92537, 92519, 92630, 92586, 92569, + 92566, 92573, 92652, 92522, 92574, 92650, 92533, 92567, 92623, 92606, + 92639, 92564, 92590, 92642, 92643, 92594, 92641, 92645, 92644, 92588, + 92629, 92632, 92631, 92710, 92695, 92694, 92726, 92675, 92719, 92677, + 92718, 92682, 92717, 92689, 92720, 92724, 92685, 92722, 92723, 92711, + 92712, 92698, 92688, 92697, 92696, 92702, 92687, 92704, 92681, 92708, + 92703, 92706, 92714, 92709, 92679, 92721, 92684, 92683, 92707, 92691, + 92713, 92700, 92693, 92727, 92690, 92692, 92686, 92680, 92725, 92699, + 92701, 92705, 92716, 92728, 92678, 92715, 92676, 42693, 42698, 42711, + 42696, 42667, 42717, 42704, 42661, 42721, 42669, 42668, 42705, 42700, + 42680, 42678, 42710, 42688, 42681, 42732, 42676, 42679, 42730, 42728, + 42672, 42662, 42724, 42687, 42689, 42690, 42697, 42714, 42660, 42656, + 42665, 42663, 42658, 42738, 42742, 42739, 42743, 127974, 128183, 128180, + 128182, 128181, 127820, 129685, 128202, 129532, 128136, 129530, 127936, + 92916, 92912, 92915, 92913, 92914, 92887, 92894, 92908, 92880, 92907, + 92893, 92886, 92888, 92881, 92906, 92896, 92891, 92902, 92900, 92885, + 92890, 92884, 92904, 92905, 92899, 92889, 92897, 92892, 92895, 92882, + 92898, 92883, 92901, 92903, 92909, 92917, 9918, 129415, 7152, 7153, 7124, + 7108, 7114, 7130, 7139, 7127, 7138, 7133, 7136, 7113, 7111, 7117, 7119, + 7107, 7135, 7125, 7112, 7123, 7129, 7116, 7132, 7105, 7126, 7128, 7110, + 7109, 7137, 7121, 7118, 7106, 7120, 7134, 7122, 7115, 7131, 7104, 7140, + 7141, 7154, 7155, 7167, 7165, 7166, 7164, 7142, 7150, 7151, 7144, 7147, + 7149, 7143, 7145, 7146, 7148, 128704, 128705, 128267, 127900, 127901, + 9835, 9836, 129492, 128059, 127958, 128147, 129451, 129752, 127866, + 129714, 983055, 128276, 129745, 128277, 9086, 128718, 118478, 2557, 2432, + 2519, 2548, 2552, 2551, 2550, 2549, 2553, 2454, 2510, 983661, 2453, 2480, + 2545, 2544, 2525, 2524, 2556, 2443, 2528, 2444, 2529, 2527, 2479, 2437, + 2438, 2448, 2452, 2466, 2465, 2471, 2470, 2464, 2463, 2469, 2468, 2441, + 2442, 2439, 2440, 2457, 2467, 2462, 2472, 2486, 2487, 2488, 2477, 2476, + 2459, 2458, 2456, 2455, 2461, 2460, 2475, 2474, 2489, 2482, 2478, 2447, + 2451, 2547, 2546, 983651, 983650, 983652, 2558, 2433, 2492, 2493, 2434, + 2509, 2435, 2554, 2494, 2504, 2508, 2497, 2498, 2499, 2500, 2530, 2531, + 2495, 2496, 2503, 2507, 2539, 2538, 2541, 2540, 2537, 2536, 2534, 2543, + 2535, 2542, 2555, 11102, 127857, 9004, 9187, 93856, 93880, 93858, 93864, + 93873, 93874, 93859, 93861, 93869, 93868, 93870, 93876, 93875, 93867, + 93862, 93865, 93877, 93857, 93879, 93860, 93866, 93871, 93872, 93878, + 93863, 93883, 93907, 93885, 93891, 93900, 93901, 93886, 93888, 93896, + 93895, 93897, 93903, 93902, 93894, 93889, 93892, 93904, 93884, 93906, + 93887, 93893, 93898, 93899, 93905, 93890, 8812, 8502, 8757, 129475, + 128719, 72710, 72711, 72712, 72746, 72704, 72705, 72715, 72717, 72731, + 72730, 72736, 72735, 72729, 72728, 72734, 72733, 72708, 72709, 72706, + 72707, 72722, 72732, 72727, 72737, 72747, 72748, 72749, 72741, 72740, + 72724, 72723, 72721, 72720, 72726, 72725, 72719, 72718, 72739, 72738, + 72750, 72745, 72742, 72744, 72743, 72714, 72716, 72801, 72810, 72806, + 72797, 72807, 72798, 72802, 72811, 72800, 72809, 72799, 72808, 72796, + 72805, 72804, 72795, 72803, 72794, 72764, 72768, 72765, 72767, 72766, + 72756, 72757, 72758, 72751, 72761, 72763, 72754, 72755, 72752, 72753, + 72760, 72762, 72770, 72769, 72789, 72788, 72791, 72790, 72787, 72786, + 72784, 72793, 72785, 72792, 72771, 72772, 72773, 72812, 128692, 128690, + 10745, 10744, 129506, 127921, 127874, 128038, 8383, 129766, 9763, 128089, + 129452, 9679, 9864, 10733, 9865, 9210, 11176, 11177, 11178, 11179, 11180, + 11182, 11181, 11183, 9960, 10028, 129623, 9821, 129554, 129596, 129609, + 129612, 129622, 9818, 129551, 129593, 9822, 129543, 129564, 129585, + 129597, 129606, 129555, 129619, 129617, 129618, 9823, 129556, 129598, + 9819, 129552, 129594, 9820, 129553, 129595, 129575, 129572, 129576, + 129577, 129573, 129574, 9827, 9670, 11201, 10070, 10730, 11230, 9830, + 128419, 128899, 9196, 9662, 9660, 11167, 127778, 9922, 9923, 10047, 9873, + 10022, 128447, 128420, 9829, 11042, 128426, 11052, 10711, 11044, 117867, + 117870, 117869, 117868, 11035, 9194, 9198, 128896, 9668, 9666, 9664, + 11164, 8268, 9944, 128412, 9754, 9699, 9698, 10731, 9207, 11206, 11045, + 9204, 11207, 11047, 9205, 11208, 9206, 11205, 128921, 128927, 9726, 9724, + 9912, 117871, 10002, 128392, 9648, 11039, 127986, 118451, 128413, 9755, + 9193, 9197, 9654, 9199, 128898, 11091, 9658, 9656, 10145, 10148, 11166, + 8269, 127990, 9644, 11049, 11050, 11089, 9642, 129997, 9787, 9632, 11200, + 9209, 128306, 9728, 128369, 9927, 9984, 128900, 128909, 9986, 9751, 9824, + 9733, 128919, 128925, 128908, 9951, 128383, 9742, 9942, 128418, 128897, + 9195, 9652, 9650, 9700, 9701, 11165, 9851, 9646, 11054, 128920, 128926, + 11037, 11204, 10707, 10067, 8493, 8460, 8465, 8476, 8488, 10164, 10166, + 10165, 9250, 118018, 118039, 118128, 118187, 118218, 118159, 118054, + 118112, 118234, 118084, 118202, 118143, 118031, 118062, 118120, 118240, + 118179, 118090, 118210, 118151, 118047, 118104, 118226, 118165, 118076, + 118194, 118135, 118021, 118035, 118066, 118124, 118243, 118183, 118093, + 118214, 118155, 118050, 118108, 118230, 118168, 118080, 118198, 118139, + 118028, 118058, 118116, 118237, 118175, 118087, 118206, 118147, 118043, + 118100, 118222, 118162, 118072, 118190, 118131, 118029, 118060, 118118, + 118177, 118208, 118149, 118045, 118102, 118224, 118023, 118037, 118068, + 118126, 118244, 118185, 118095, 118216, 118157, 118052, 118110, 118232, + 118170, 118082, 118200, 118141, 118074, 118192, 118133, 118020, 118033, + 118064, 118122, 118242, 118181, 118092, 118212, 118153, 118048, 118106, + 118228, 118167, 118078, 118196, 118137, 118026, 118056, 118114, 118235, + 118173, 118085, 118204, 118145, 118041, 118098, 118220, 118160, 118070, + 118188, 118129, 118034, 118065, 118123, 118182, 118213, 118154, 118049, + 118107, 118229, 118079, 118197, 118138, 118017, 118024, 118038, 118069, + 118127, 118245, 118186, 118096, 118217, 118158, 118053, 118111, 118233, + 118171, 118083, 118201, 118142, 118030, 118061, 118119, 118239, 118178, + 118089, 118209, 118150, 118046, 118103, 118225, 118164, 118075, 118193, + 118134, 118027, 118057, 118115, 118236, 118174, 118086, 118205, 118146, + 118042, 118099, 118221, 118161, 118071, 118189, 118130, 118025, 118055, + 118113, 118172, 118203, 118144, 118040, 118097, 118219, 118016, 118022, + 118036, 118067, 118125, 118184, 118094, 118215, 118156, 118051, 118109, + 118231, 118169, 118081, 118199, 118140, 118059, 118117, 118238, 118176, + 118088, 118207, 118148, 118044, 118101, 118223, 118163, 118073, 118191, + 118132, 118019, 118105, 118227, 118166, 118032, 118063, 118121, 118241, + 118180, 118091, 118211, 118152, 118077, 118195, 118136, 129792, 129794, + 129798, 129806, 129821, 129836, 129813, 129844, 129829, 129802, 129817, + 129848, 129832, 129810, 129840, 129825, 129796, 129804, 129819, 129850, + 129834, 129842, 129827, 129800, 129815, 129846, 129831, 129808, 129838, + 129823, 129793, 129801, 129816, 129847, 129797, 129805, 129820, 129851, + 129835, 129812, 129843, 129828, 129809, 129839, 129824, 129795, 129803, + 129818, 129849, 129833, 129811, 129841, 129826, 129799, 129814, 129845, + 129830, 129807, 129837, 129822, 127804, 128033, 128216, 128153, 129744, + 128939, 128951, 128957, 128945, 128902, 128912, 128932, 128366, 128278, + 128209, 128218, 129667, 12731, 12727, 12726, 12724, 12725, 12570, 12574, + 12718, 12576, 12719, 12578, 12580, 12713, 12735, 12720, 12584, 12715, + 12572, 12579, 12581, 12709, 12708, 12573, 12575, 12582, 12557, 12728, + 12588, 12707, 12732, 12583, 12714, 12723, 12589, 12716, 12712, 12585, + 12555, 12587, 12717, 12591, 12571, 12722, 12711, 12590, 12734, 12721, + 12710, 12577, 12567, 12563, 12705, 12730, 12558, 12733, 12568, 12564, + 12556, 12729, 12569, 12565, 12549, 12704, 12560, 12706, 12553, 12552, + 12559, 12551, 12550, 12561, 12566, 12554, 12586, 12562, 118257, 118258, + 118259, 118260, 117854, 11867, 117853, 118253, 118255, 11868, 117855, + 118249, 118251, 118247, 11211, 8993, 130029, 8990, 8973, 11812, 8991, + 8972, 11813, 130030, 9141, 9142, 10555, 9183, 9181, 130026, 130018, 9185, + 127870, 128144, 127893, 128335, 129379, 127923, 8904, 10705, 10706, + 127993, 118282, 117792, 117791, 118281, 9574, 9559, 9556, 9577, 9565, + 9562, 9553, 9580, 9571, 9568, 9552, 9511, 9490, 9503, 9486, 9537, 9520, + 9513, 9489, 9505, 9485, 9543, 9519, 9558, 9555, 9573, 9557, 9554, 9572, + 9592, 9598, 9593, 9599, 9499, 9531, 9495, 9475, 9547, 9515, 9507, 9595, + 9523, 9491, 9487, 9551, 9549, 9483, 9481, 9479, 9477, 9473, 9594, 9541, + 9525, 9517, 9533, 9530, 9522, 9546, 9539, 9582, 9581, 9583, 9584, 130010, + 130014, 129954, 129958, 130003, 129959, 129964, 129965, 129955, 130000, + 129952, 129956, 129963, 129960, 129953, 129961, 129957, 129962, 130007, + 130005, 130004, 130012, 9586, 130008, 130011, 130002, 130015, 130006, + 9585, 130009, 130001, 130013, 9587, 129966, 9591, 9516, 9488, 9484, 9550, + 9548, 9472, 117787, 117788, 129967, 9588, 9596, 9482, 9480, 9478, 9476, + 117789, 9589, 9597, 9524, 9496, 9492, 9474, 118297, 118295, 118296, + 118294, 9532, 9508, 9500, 117790, 9590, 9542, 9526, 9518, 9534, 9529, + 9521, 9545, 9540, 9536, 9510, 9498, 9502, 9494, 9528, 9544, 9514, 9497, + 9506, 9493, 9527, 9564, 9561, 9576, 9563, 9560, 9575, 9570, 9567, 9579, + 9538, 9512, 9504, 9535, 9509, 9501, 9569, 9566, 9578, 129354, 983263, + 128163, 128102, 128713, 128023, 129460, 69649, 69685, 69749, 69745, + 69746, 69687, 69686, 69637, 69638, 69648, 69650, 69664, 69663, 69669, + 69668, 69662, 69661, 69667, 69666, 69643, 69644, 69645, 69646, 69679, + 69641, 69642, 69639, 69640, 69684, 69678, 69655, 69665, 69660, 69670, + 69680, 69681, 69682, 69674, 69673, 69657, 69656, 69654, 69653, 69659, + 69658, 69652, 69651, 69672, 69671, 69683, 69675, 69677, 69676, 69647, + 69721, 69730, 69726, 69717, 69727, 69718, 69722, 69731, 69720, 69729, + 69719, 69728, 69716, 69725, 69724, 69715, 69723, 69714, 69732, 69733, + 69759, 69709, 69707, 69706, 69705, 69708, 69632, 69744, 69635, 69634, + 69636, 69633, 69700, 69747, 69748, 69689, 69688, 69699, 69701, 69692, + 69693, 69694, 69695, 69696, 69697, 69690, 69691, 69698, 69702, 69704, + 69703, 69739, 69738, 69741, 69740, 69737, 69736, 69734, 69743, 69735, + 69742, 10241, 10243, 10247, 10255, 10271, 10303, 10367, 10495, 10431, + 10335, 10463, 10399, 10287, 10351, 10479, 10415, 10319, 10447, 10383, + 10263, 10295, 10359, 10487, 10423, 10327, 10455, 10391, 10279, 10343, + 10471, 10407, 10311, 10439, 10375, 10251, 10267, 10299, 10363, 10491, + 10427, 10331, 10459, 10395, 10283, 10347, 10475, 10411, 10315, 10443, + 10379, 10259, 10291, 10355, 10483, 10419, 10323, 10451, 10387, 10275, + 10339, 10467, 10403, 10307, 10435, 10371, 10245, 10253, 10269, 10301, + 10365, 10493, 10429, 10333, 10461, 10397, 10285, 10349, 10477, 10413, + 10317, 10445, 10381, 10261, 10293, 10357, 10485, 10421, 10325, 10453, + 10389, 10277, 10341, 10469, 10405, 10309, 10437, 10373, 10249, 10265, + 10297, 10361, 10489, 10425, 10329, 10457, 10393, 10281, 10345, 10473, + 10409, 10313, 10441, 10377, 10257, 10289, 10353, 10481, 10417, 10321, + 10449, 10385, 10273, 10337, 10465, 10401, 10305, 10433, 10369, 10242, + 10246, 10254, 10270, 10302, 10366, 10494, 10430, 10334, 10462, 10398, + 10286, 10350, 10478, 10414, 10318, 10446, 10382, 10262, 10294, 10358, + 10486, 10422, 10326, 10454, 10390, 10278, 10342, 10470, 10406, 10310, + 10438, 10374, 10250, 10266, 10298, 10362, 10490, 10426, 10330, 10458, + 10394, 10282, 10346, 10474, 10410, 10314, 10442, 10378, 10258, 10290, + 10354, 10482, 10418, 10322, 10450, 10386, 10274, 10338, 10466, 10402, + 10306, 10434, 10370, 10244, 10252, 10268, 10300, 10364, 10492, 10428, + 10332, 10460, 10396, 10284, 10348, 10476, 10412, 10316, 10444, 10380, + 10260, 10292, 10356, 10484, 10420, 10324, 10452, 10388, 10276, 10340, + 10468, 10404, 10308, 10436, 10372, 10248, 10264, 10296, 10360, 10488, + 10424, 10328, 10456, 10392, 10280, 10344, 10472, 10408, 10312, 10440, + 10376, 10256, 10288, 10352, 10480, 10416, 10320, 10448, 10384, 10272, + 10336, 10464, 10400, 10304, 10432, 10368, 10240, 129504, 983125, 129329, + 127838, 728, 127753, 128112, 128188, 129650, 129521, 9099, 166, 128148, + 129382, 129294, 129529, 129483, 129767, 128027, 6663, 6662, 6659, 6658, + 6671, 6670, 6667, 6666, 6661, 6668, 6665, 6657, 6678, 6669, 6656, 6674, + 6660, 6673, 6676, 6664, 6675, 6672, 6677, 6683, 6681, 6679, 6682, 6680, + 6687, 6686, 5957, 5960, 5962, 5959, 5956, 5969, 5955, 5966, 5963, 5961, + 5965, 5968, 5958, 5967, 5964, 5952, 5953, 5954, 5970, 5971, 8226, 8729, + 128363, 128364, 9678, 128652, 128101, 128100, 128655, 129480, 129419, + 127959, 127791, 129699, 118939, 118940, 118944, 118943, 118941, 118942, + 118938, 118945, 118882, 118876, 118824, 118835, 118797, 118866, 118957, + 118801, 118802, 118865, 118818, 118916, 118819, 118917, 118899, 118935, + 119018, 119022, 119021, 119020, 119023, 119019, 119017, 118986, 118984, + 118985, 118843, 118887, 118808, 118870, 119003, 119002, 119004, 119005, + 118937, 118991, 118995, 118990, 118992, 118994, 118993, 118930, 118933, + 118931, 118932, 119014, 118918, 118912, 119015, 118785, 118831, 118907, + 118966, 118900, 118880, 118798, 118888, 118884, 118869, 118960, 118959, + 118958, 118836, 118969, 118977, 118978, 118971, 118976, 118975, 118973, + 118970, 118987, 118979, 118980, 118972, 118983, 983278, 118982, 118974, + 118988, 118981, 119000, 119001, 118929, 118928, 118806, 119029, 118927, + 118898, 118964, 118965, 118853, 118968, 118837, 118967, 118936, 118956, + 118810, 118854, 118847, 118839, 118906, 118791, 118800, 119024, 119026, + 118862, 118812, 119025, 119027, 118863, 118811, 118820, 119028, 118911, + 118842, 118850, 118921, 118858, 118913, 118914, 118915, 118834, 118860, + 118868, 118796, 118881, 118922, 118926, 118925, 118924, 118923, 118830, + 118877, 118949, 118947, 118948, 118963, 118955, 118962, 118946, 118961, + 118953, 118952, 118951, 118950, 118954, 118871, 118805, 118855, 118828, + 118901, 118787, 118788, 118856, 118816, 118875, 118845, 118879, 118793, + 118846, 118878, 118814, 118822, 118873, 118859, 118857, 118849, 118840, + 118861, 118786, 118895, 118841, 118874, 118892, 118897, 118807, 118784, + 118821, 118844, 118825, 118889, 119010, 119012, 119013, 119011, 119006, + 119008, 119009, 119007, 118910, 118815, 118852, 119016, 118826, 118885, + 118827, 118803, 118886, 118792, 118813, 118920, 118799, 118833, 118829, + 118904, 118902, 118903, 118905, 118804, 118934, 118919, 118832, 118893, + 118838, 118851, 118883, 118894, 118891, 118896, 118823, 118789, 118790, + 118872, 118817, 118809, 118908, 118909, 118996, 118998, 118997, 118999, + 118989, 118794, 118795, 118867, 118864, 118848, 118890, 983262, 983126, + 983057, 9764, 8454, 129305, 128197, 128247, 128248, 127957, 983098, 5130, + 5131, 5122, 6322, 5148, 5551, 5310, 5382, 5166, 6321, 5759, 5559, 5556, + 5557, 5558, 5567, 5564, 5565, 5566, 5563, 5560, 5561, 5562, 5555, 5552, + 5553, 5554, 5384, 5439, 6387, 6388, 5281, 5264, 6382, 6389, 5203, 5674, + 5675, 5677, 5676, 5673, 5672, 5706, 5707, 5709, 5708, 5705, 5704, 5204, + 5574, 5575, 5577, 5576, 5573, 5572, 6384, 6381, 5620, 6383, 5617, 5618, + 5619, 5616, 5615, 5195, 5592, 5593, 5595, 5594, 5591, 5590, 5174, 5175, + 5129, 5703, 5662, 5663, 5665, 5664, 5661, 5660, 5655, 5656, 6386, 5659, + 5657, 5654, 5652, 5633, 5629, 5630, 5632, 5631, 5628, 5627, 5623, 5624, + 5626, 5625, 5622, 5621, 5636, 5637, 5639, 5329, 5638, 5635, 5634, 5722, + 5718, 5719, 5721, 5720, 5717, 5716, 5712, 5713, 5715, 5714, 5711, 5710, + 5614, 5610, 5611, 5613, 5612, 5609, 5608, 5702, 5698, 5699, 5701, 5700, + 5697, 5696, 5686, 5687, 5689, 5688, 5685, 5684, 5692, 5693, 5695, 5694, + 5691, 5690, 5737, 5738, 5740, 5739, 5736, 5735, 5604, 5605, 5607, 5606, + 5603, 5602, 5598, 5599, 5601, 5600, 5597, 5596, 5725, 5726, 5728, 5727, + 5724, 5723, 5680, 5681, 5683, 5682, 5679, 5678, 5668, 5669, 5671, 5670, + 5667, 5666, 5731, 5732, 5734, 5733, 5730, 5729, 5642, 5643, 5645, 5644, + 5641, 5640, 5580, 5581, 5583, 5582, 5579, 5578, 5586, 5587, 5589, 5588, + 5585, 5584, 5648, 5649, 5651, 5650, 5647, 5646, 5128, 5265, 5258, 5276, + 5278, 5272, 5274, 5268, 5270, 5266, 5741, 5261, 5262, 5259, 5260, 5257, + 5121, 6364, 5163, 5469, 5465, 5466, 5460, 5461, 5158, 5157, 5162, 5155, + 5156, 6367, 6366, 5160, 5152, 5151, 5153, 5154, 5161, 5159, 5462, 5742, + 5463, 5464, 5467, 5459, 5120, 5501, 5123, 5124, 5164, 5251, 5252, 5246, + 5248, 6329, 5242, 5244, 5238, 5240, 5236, 5234, 5235, 5228, 6328, 5231, + 5232, 5229, 5230, 5227, 5354, 5542, 5540, 5541, 5538, 5539, 5536, 5537, + 5338, 5339, 5332, 6333, 5350, 5352, 5346, 5348, 5342, 5344, 5340, 5335, + 5336, 5333, 5334, 5331, 5307, 5283, 5356, 5458, 5287, 5288, 5385, 5290, + 5291, 5284, 6330, 5302, 5304, 5298, 5300, 5294, 5296, 5292, 5285, 5286, + 5309, 5328, 5473, 5475, 5471, 5319, 5391, 5388, 5389, 5386, 5390, 5380, + 5387, 5142, 5147, 5280, 5250, 5306, 5327, 5221, 5437, 72372, 72373, + 72370, 72371, 72368, 72369, 72378, 72379, 72376, 72377, 72374, 72375, + 5320, 5313, 6332, 5525, 5523, 5524, 5518, 5744, 5521, 5522, 5519, 5520, + 5526, 5749, 5750, 5747, 5748, 5745, 5746, 5499, 5497, 5498, 5495, 5496, + 5493, 5494, 5492, 5500, 5316, 5317, 6331, 5323, 5325, 6346, 6348, 6342, + 6344, 5321, 5314, 5315, 5312, 5330, 5509, 5507, 5508, 5502, 5743, 5505, + 5506, 5503, 5504, 5125, 6361, 6347, 6349, 6343, 6345, 6362, 6363, 6359, + 6358, 6360, 6356, 6357, 5165, 5126, 6320, 5193, 5184, 5186, 6326, 5188, + 5190, 5180, 5182, 5178, 5176, 5177, 5168, 6325, 5171, 5172, 6324, 5169, + 5170, 5167, 5456, 6368, 5443, 6355, 5454, 6353, 6354, 6351, 6352, 6350, + 5451, 5452, 5445, 6341, 5448, 5449, 5446, 5447, 5442, 5381, 5364, 6335, + 5570, 6380, 5571, 5568, 5569, 5653, 6385, 5658, 5529, 6379, 6378, 5530, + 5527, 5528, 5441, 5282, 5311, 5365, 5358, 5413, 5405, 5407, 6338, 5409, + 5411, 5401, 5403, 5399, 5395, 5396, 6336, 5397, 5398, 6337, 5393, 5394, + 5392, 5361, 5256, 5253, 5254, 5255, 5362, 6334, 5383, 5376, 5378, 5372, + 5374, 5368, 5370, 5366, 72383, 72380, 72381, 72382, 5359, 5360, 5357, + 5222, 5482, 5550, 5548, 5549, 5546, 5547, 5544, 5545, 5543, 6372, 5480, + 6371, 5478, 5479, 5476, 5477, 5472, 5474, 5470, 5512, 6377, 6376, 5513, + 5510, 5511, 5487, 5486, 6375, 5485, 6374, 6373, 5483, 5484, 5226, 5223, + 5224, 5225, 5205, 5206, 5197, 6327, 5217, 5219, 5213, 5215, 5209, 5211, + 5207, 5491, 5488, 5489, 5490, 5200, 5201, 5198, 5199, 5196, 5132, 5355, + 5351, 5353, 5347, 5349, 5343, 5345, 5341, 5453, 6370, 5450, 6369, 5444, + 5308, 5303, 5305, 5299, 5301, 5295, 5297, 5293, 5194, 5189, 5191, 5185, + 5187, 5181, 5183, 5179, 5440, 5434, 5436, 5430, 5432, 5426, 5428, 5424, + 5324, 5326, 5322, 5457, 5455, 5517, 5514, 5515, 5516, 5410, 5412, 5406, + 5408, 5402, 5404, 5400, 5377, 5379, 5373, 5375, 5369, 5371, 5367, 5277, + 5279, 5273, 5275, 5269, 5271, 5267, 5247, 5249, 5243, 5245, 5239, 5241, + 5237, 5481, 5218, 5220, 5214, 5216, 5210, 5212, 5208, 5468, 5144, 5146, + 5139, 5141, 5135, 5137, 5133, 6365, 5138, 5140, 5535, 5756, 5757, 5754, + 5755, 5752, 5753, 5751, 5534, 5531, 5532, 5533, 5758, 5143, 5145, 6323, + 5134, 5136, 5438, 5192, 5173, 5263, 5233, 5337, 5289, 5318, 5363, 5202, + 5420, 5127, 5149, 5421, 5422, 5415, 6340, 5418, 5419, 6339, 5433, 5435, + 5429, 5431, 5425, 5427, 5423, 5416, 5417, 5414, 5150, 983097, 983171, + 917631, 128473, 9803, 128367, 127852, 129387, 128758, 11839, 9809, + 128199, 128450, 128451, 8248, 8257, 8453, 66225, 66246, 66211, 66214, + 66254, 66218, 66250, 66251, 66252, 66253, 66229, 66238, 66244, 66227, + 66224, 66222, 66223, 66242, 66243, 66232, 66221, 66247, 66230, 66226, + 66239, 66212, 66248, 66256, 66235, 66208, 66215, 66210, 66220, 66234, + 66255, 66240, 66241, 66236, 66237, 66231, 66209, 66213, 66249, 66233, + 66245, 66217, 66219, 66216, 66228, 127904, 711, 127887, 129690, 983073, + 129365, 9936, 128008, 128049, 128569, 128572, 66888, 66864, 66912, 66882, + 66873, 66902, 66889, 66890, 66911, 66891, 66895, 66901, 66881, 66867, + 66870, 66868, 66904, 66866, 66876, 66910, 66879, 66883, 66897, 66915, + 66884, 66878, 66885, 66914, 66903, 66877, 66896, 66909, 66906, 66908, + 66899, 66872, 66913, 66874, 66871, 66875, 66869, 66893, 66887, 66892, + 66907, 66900, 66894, 66905, 66880, 66898, 66865, 66886, 66927, 9761, + 127797, 9963, 8373, 184, 65102, 65098, 8452, 162, 128328, 9907, 9939, + 129681, 69908, 69907, 69913, 69912, 69899, 69909, 69904, 69914, 69906, + 69905, 69911, 69910, 69920, 69921, 69918, 69917, 69901, 69900, 69898, + 69897, 69903, 69902, 69896, 69895, 69956, 69923, 69916, 69915, 69926, + 69919, 69922, 69925, 69959, 69924, 69891, 69894, 69892, 69893, 69888, + 69890, 69889, 69952, 69927, 69957, 69933, 69935, 69930, 69931, 69932, + 69958, 69928, 69929, 69934, 69936, 69939, 69954, 69953, 69947, 69946, + 69949, 69948, 69945, 69944, 69942, 69951, 69943, 69950, 69955, 69940, + 69938, 69937, 43587, 43597, 43596, 43573, 43572, 43574, 43571, 43590, + 43586, 43595, 43588, 43585, 43584, 43594, 43591, 43593, 43589, 43592, + 43530, 43531, 43536, 43538, 43537, 43543, 43544, 43551, 43552, 43548, + 43547, 43546, 43553, 43550, 43549, 43545, 43542, 43541, 43558, 43559, + 43520, 43524, 43533, 43532, 43529, 43528, 43535, 43534, 43527, 43526, + 43540, 43539, 43560, 43556, 43555, 43557, 43554, 43523, 43521, 43525, + 43522, 43612, 43614, 43613, 43615, 43561, 43568, 43569, 43562, 43563, + 43567, 43566, 43565, 43570, 43564, 43605, 43604, 43607, 43606, 43603, + 43602, 43600, 43609, 43601, 43608, 983058, 983140, 983137, 8256, 128200, + 128185, 128201, 128638, 129941, 10003, 129472, 128227, 5084, 5075, 5077, + 5079, 5081, 5082, 5083, 5055, 5037, 5038, 5039, 5040, 5041, 5042, 5054, + 5056, 5057, 5058, 5059, 5060, 5061, 5069, 5068, 5070, 5071, 5072, 5073, + 5074, 5085, 5086, 5087, 5088, 5089, 5090, 5091, 5092, 5093, 5094, 5095, + 5096, 5076, 5078, 5080, 5030, 5032, 5033, 5034, 5035, 5036, 5043, 5044, + 5045, 5046, 5047, 5048, 5049, 5050, 5051, 5052, 5053, 5109, 5062, 5063, + 5064, 5065, 5066, 5067, 5097, 5098, 5099, 5100, 5101, 5102, 5103, 5104, + 5105, 5106, 5107, 5108, 5031, 5024, 5025, 5026, 5027, 5028, 5029, 43948, + 43939, 43941, 43943, 43945, 43946, 43947, 43919, 43901, 43902, 43903, + 43904, 43905, 43906, 43918, 43920, 43921, 43922, 43923, 43924, 43925, + 43933, 43932, 43934, 43935, 43936, 43937, 43938, 43949, 43950, 43951, + 43952, 43953, 43954, 43955, 43956, 43957, 43958, 43959, 43960, 43940, + 43942, 43944, 43894, 43896, 43897, 43898, 43899, 43900, 43907, 43908, + 43909, 43910, 43911, 43912, 43913, 43914, 43915, 43916, 43917, 5117, + 43926, 43927, 43928, 43929, 43930, 43931, 43961, 43962, 43963, 43964, + 43965, 43966, 43967, 5112, 5113, 5114, 5115, 5116, 43895, 43888, 43889, + 43890, 43891, 43892, 43893, 127800, 118462, 127826, 127792, 127937, + 129490, 128696, 94194, 94195, 9767, 128063, 128020, 9911, 69559, 69553, + 69567, 69571, 69556, 69564, 69570, 69552, 69568, 69555, 69560, 69562, + 69557, 69561, 69563, 69554, 69566, 69572, 69558, 69569, 69565, 69578, + 69574, 69575, 69577, 69573, 69579, 69576, 129378, 127851, 127876, 9962, + 127910, 9680, 9682, 10690, 10683, 10691, 9684, 9683, 9685, 9677, 9681, + 10677, 10682, 10684, 127246, 8859, 11199, 10687, 129282, 129280, 129281, + 128320, 9938, 127342, 127341, 127277, 8856, 10808, 9316, 9315, 9318, + 9317, 9314, 9313, 9450, 9320, 9312, 9319, 127247, 8857, 8861, 12905, + 12919, 12904, 12918, 12903, 12917, 12926, 12909, 12923, 12906, 12920, + 12896, 12910, 12900, 12914, 12897, 12911, 12908, 12922, 12901, 12915, + 12899, 12913, 12902, 12916, 12907, 12921, 12898, 12912, 9097, 127343, + 10162, 12975, 127569, 12959, 127568, 12963, 12951, 12962, 12965, 12973, + 12943, 12957, 12935, 12950, 12939, 12932, 12955, 12931, 12964, 12946, + 12869, 12871, 12952, 12966, 12967, 12969, 12942, 12954, 12938, 12976, + 12936, 12948, 12868, 12974, 12961, 12970, 12968, 12953, 12934, 12972, + 12956, 12944, 12870, 12947, 12949, 12971, 12945, 12933, 12958, 12930, + 12937, 12929, 12941, 12940, 12960, 12928, 127275, 127276, 128712, 13033, + 13036, 13034, 13037, 13035, 13013, 13016, 13014, 13017, 13015, 13038, + 13041, 13039, 13042, 13040, 13028, 13031, 13029, 13032, 13030, 13046, + 13049, 13047, 13050, 13048, 13018, 13021, 13019, 13022, 13020, 13023, + 13026, 13024, 13027, 13025, 13051, 13053, 13052, 13054, 13043, 13045, + 13044, 13008, 13011, 13009, 13012, 13010, 12925, 12924, 9398, 9399, 9400, + 9401, 9402, 9403, 9404, 9405, 9406, 9407, 9408, 9409, 9410, 9411, 9412, + 9413, 9414, 9415, 9416, 9417, 9418, 9419, 9420, 9421, 9422, 9423, 9424, + 9425, 9426, 9427, 9428, 9429, 9430, 9431, 9432, 9433, 9434, 9435, 9436, + 9437, 9438, 9439, 9440, 9441, 9442, 9443, 9444, 9445, 9446, 9447, 9448, + 9449, 10688, 10806, 8854, 12879, 9329, 9322, 12991, 12876, 9326, 12981, + 12875, 12982, 12986, 12985, 12988, 12987, 12984, 12983, 12990, 12989, + 9325, 12878, 9328, 12877, 9327, 9321, 12872, 12890, 12874, 12891, 12895, + 12894, 12978, 12977, 12893, 12892, 12980, 12979, 9324, 9331, 12873, + 12881, 12885, 12884, 12887, 12886, 12883, 12882, 12889, 12888, 9323, + 9330, 10050, 12342, 10679, 10681, 8853, 8858, 10680, 128981, 9098, 8855, + 10686, 10026, 127278, 127245, 8860, 10678, 10689, 128983, 11198, 94, + 10768, 127914, 127961, 127750, 195088, 195089, 195090, 195091, 195092, + 195093, 195094, 195095, 195096, 195097, 195098, 195099, 195100, 195101, + 195072, 195073, 195074, 195075, 195076, 195077, 195078, 195079, 195080, + 195081, 195082, 195083, 195084, 195085, 195086, 195087, 194560, 194561, + 194562, 194563, 194564, 194565, 194566, 194567, 194568, 194569, 194570, + 194571, 194572, 194573, 194574, 194575, 194576, 194577, 194578, 194579, + 194580, 194581, 194582, 194583, 194584, 194585, 194586, 194587, 194588, + 194589, 194590, 194591, 194592, 194593, 194594, 194595, 194596, 194597, + 194598, 194599, 194600, 194601, 194602, 194603, 194604, 194605, 194606, + 194607, 194608, 194609, 194610, 194611, 194612, 194613, 194614, 194615, + 194616, 194617, 194618, 194619, 194620, 194621, 194622, 194623, 194624, + 194625, 194626, 194627, 194628, 194629, 194630, 194631, 194632, 194633, + 194634, 194635, 194636, 194637, 194638, 194639, 194640, 194641, 194642, + 194643, 194644, 194645, 194646, 194647, 194648, 194649, 194650, 194651, + 194652, 194653, 194654, 194655, 194656, 194657, 194658, 194659, 194660, + 194661, 194662, 194663, 194664, 194665, 194666, 194667, 194668, 194669, + 194670, 194671, 194672, 194673, 194674, 194675, 194676, 194677, 194678, + 194679, 194680, 194681, 194682, 194683, 194684, 194685, 194686, 194687, + 194688, 194689, 194690, 194691, 194692, 194693, 194694, 194695, 194696, + 194697, 194698, 194699, 194700, 194701, 194702, 194703, 194704, 194705, + 194706, 194707, 194708, 194709, 194710, 194711, 194712, 194713, 194714, + 194715, 194716, 194717, 194718, 194719, 194720, 194721, 194722, 194723, + 194724, 194725, 194726, 194727, 194728, 194729, 194730, 194731, 194732, + 194733, 194734, 194735, 194736, 194737, 194738, 194739, 194740, 194741, + 194742, 194743, 194744, 194745, 194746, 194747, 194748, 194749, 194750, + 194751, 194752, 194753, 194754, 194755, 194756, 194757, 194758, 194759, + 194760, 194761, 194762, 194763, 194764, 194765, 194766, 194767, 194768, + 194769, 194770, 194771, 194772, 194773, 194774, 194775, 194776, 194777, + 194778, 194779, 194780, 194781, 194782, 194783, 194784, 194785, 194786, + 194787, 194788, 194789, 194790, 194791, 194792, 194793, 194794, 194795, + 194796, 194797, 194798, 194799, 194800, 194801, 194802, 194803, 194804, + 194805, 194806, 194807, 194808, 194809, 194810, 194811, 194812, 194813, + 194814, 194815, 194816, 194817, 194818, 194819, 194820, 194821, 194822, + 194823, 194824, 194825, 194826, 194827, 194828, 194829, 194830, 194831, + 194832, 194833, 194834, 194835, 194836, 194837, 194838, 194839, 194840, + 194841, 194842, 194843, 194844, 194845, 194846, 194847, 194848, 194849, + 194850, 194851, 194852, 194853, 194854, 194855, 194856, 194857, 194858, + 194859, 194860, 194861, 194862, 194863, 194864, 194865, 194866, 194867, + 194868, 194869, 194870, 194871, 194872, 194873, 194874, 194875, 194876, + 194877, 194878, 194879, 194880, 194881, 194882, 194883, 194884, 194885, + 194886, 194887, 194888, 194889, 194890, 194891, 194892, 194893, 194894, + 194895, 194896, 194897, 194898, 194899, 194900, 194901, 194902, 194903, + 194904, 194905, 194906, 194907, 194908, 194909, 194910, 194911, 194912, + 194913, 194914, 194915, 194916, 194917, 194918, 194919, 194920, 194921, + 194922, 194923, 194924, 194925, 194926, 194927, 194928, 194929, 194930, + 194931, 194932, 194933, 194934, 194935, 194936, 194937, 194938, 194939, + 194940, 194941, 194942, 194943, 194944, 194945, 194946, 194947, 194948, + 194949, 194950, 194951, 194952, 194953, 194954, 194955, 194956, 194957, + 194958, 194959, 194960, 194961, 194962, 194963, 194964, 194965, 194966, + 194967, 194968, 194969, 194970, 194971, 194972, 194973, 194974, 194975, + 194976, 194977, 194978, 194979, 194980, 194981, 194982, 194983, 194984, + 194985, 194986, 194987, 194988, 194989, 194990, 194991, 194992, 194993, + 194994, 194995, 194996, 194997, 194998, 194999, 195000, 195001, 195002, + 195003, 195004, 195005, 195006, 195007, 195008, 195009, 195010, 195011, + 195012, 195013, 195014, 195015, 195016, 195017, 195018, 195019, 195020, + 195021, 195022, 195023, 195024, 195025, 195026, 195027, 195028, 195029, + 195030, 195031, 195032, 195033, 195034, 195035, 195036, 195037, 195038, + 195039, 195040, 195041, 195042, 195043, 195044, 195045, 195046, 195047, + 195048, 195049, 195050, 195051, 195052, 195053, 195054, 195055, 195056, + 195057, 195058, 195059, 195060, 195061, 195062, 195063, 195064, 195065, + 195066, 195067, 195068, 195069, 195070, 195071, 64096, 64097, 64098, + 64099, 64100, 64101, 64102, 64103, 64104, 64105, 64106, 64107, 64108, + 64109, 64000, 64001, 64002, 64003, 64004, 64005, 64006, 64007, 64008, + 64009, 64010, 64011, 64012, 64013, 64014, 64015, 64016, 64017, 64018, + 64019, 64020, 64021, 64022, 64023, 64024, 64025, 64026, 64027, 64028, + 64029, 64030, 64031, 64032, 64033, 64034, 64035, 64036, 64037, 64038, + 64039, 64040, 64041, 64042, 64043, 64044, 64045, 64046, 64047, 64048, + 64049, 64050, 64051, 64052, 64053, 64054, 64055, 64056, 64057, 64058, + 64059, 64060, 64061, 64062, 64063, 64064, 64065, 64066, 64067, 64068, + 64069, 64070, 64071, 64072, 64073, 64074, 64075, 64076, 64077, 64078, + 64079, 64080, 64081, 64082, 64083, 64084, 64085, 64086, 64087, 64088, + 64089, 64090, 64091, 64092, 64093, 64094, 64095, 64112, 64113, 64114, + 64115, 64116, 64117, 64118, 64119, 64120, 64121, 64122, 64123, 64124, + 64125, 64126, 64127, 64128, 64129, 64130, 64131, 64132, 64133, 64134, + 64135, 64136, 64137, 64138, 64139, 64140, 64141, 64142, 64143, 64144, + 64145, 64146, 64147, 64148, 64149, 64150, 64151, 64152, 64153, 64154, + 64155, 64156, 64157, 64158, 64159, 64160, 64161, 64162, 64163, 64164, + 64165, 64166, 64167, 64168, 64169, 64170, 64171, 64172, 64173, 64174, + 64175, 64176, 64177, 64178, 64179, 64180, 64181, 64182, 64183, 64184, + 64185, 64186, 64187, 64188, 64189, 64190, 64191, 64192, 64193, 64194, + 64195, 64196, 64197, 64198, 64199, 64200, 64201, 64202, 64203, 64204, + 64205, 64206, 64207, 64208, 64209, 64210, 64211, 64212, 64213, 64214, + 64215, 64216, 64217, 63744, 63745, 63746, 63747, 63748, 63749, 63750, + 63751, 63752, 63753, 63754, 63755, 63756, 63757, 63758, 63759, 63760, + 63761, 63762, 63763, 63764, 63765, 63766, 63767, 63768, 63769, 63770, + 63771, 63772, 63773, 63774, 63775, 63776, 63777, 63778, 63779, 63780, + 63781, 63782, 63783, 63784, 63785, 63786, 63787, 63788, 63789, 63790, + 63791, 63792, 63793, 63794, 63795, 63796, 63797, 63798, 63799, 63800, + 63801, 63802, 63803, 63804, 63805, 63806, 63807, 63808, 63809, 63810, + 63811, 63812, 63813, 63814, 63815, 63816, 63817, 63818, 63819, 63820, + 63821, 63822, 63823, 63824, 63825, 63826, 63827, 63828, 63829, 63830, + 63831, 63832, 63833, 63834, 63835, 63836, 63837, 63838, 63839, 63840, + 63841, 63842, 63843, 63844, 63845, 63846, 63847, 63848, 63849, 63850, + 63851, 63852, 63853, 63854, 63855, 63856, 63857, 63858, 63859, 63860, + 63861, 63862, 63863, 63864, 63865, 63866, 63867, 63868, 63869, 63870, + 63871, 63872, 63873, 63874, 63875, 63876, 63877, 63878, 63879, 63880, + 63881, 63882, 63883, 63884, 63885, 63886, 63887, 63888, 63889, 63890, + 63891, 63892, 63893, 63894, 63895, 63896, 63897, 63898, 63899, 63900, + 63901, 63902, 63903, 63904, 63905, 63906, 63907, 63908, 63909, 63910, + 63911, 63912, 63913, 63914, 63915, 63916, 63917, 63918, 63919, 63920, + 63921, 63922, 63923, 63924, 63925, 63926, 63927, 63928, 63929, 63930, + 63931, 63932, 63933, 63934, 63935, 63936, 63937, 63938, 63939, 63940, + 63941, 63942, 63943, 63944, 63945, 63946, 63947, 63948, 63949, 63950, + 63951, 63952, 63953, 63954, 63955, 63956, 63957, 63958, 63959, 63960, + 63961, 63962, 63963, 63964, 63965, 63966, 63967, 63968, 63969, 63970, + 63971, 63972, 63973, 63974, 63975, 63976, 63977, 63978, 63979, 63980, + 63981, 63982, 63983, 63984, 63985, 63986, 63987, 63988, 63989, 63990, + 63991, 63992, 63993, 63994, 63995, 63996, 63997, 63998, 63999, 11946, + 12003, 11910, 11963, 11962, 11950, 11992, 12012, 12000, 12005, 11996, + 12010, 11984, 11988, 11994, 11987, 11952, 12007, 11977, 11976, 11973, + 12019, 11993, 12014, 11995, 12016, 12006, 12002, 11979, 11936, 11983, + 11905, 11970, 11943, 11931, 11914, 11934, 11944, 11999, 11998, 11997, + 11960, 11947, 11939, 11978, 11968, 11967, 11966, 12004, 11927, 11926, + 12001, 11975, 11928, 12018, 12013, 12015, 12011, 11945, 11986, 11985, + 11921, 11920, 11919, 11918, 11964, 11957, 11990, 11989, 11935, 11965, + 11933, 11941, 11940, 11909, 11991, 11959, 11929, 11904, 11908, 11907, + 11906, 11915, 11942, 11974, 11980, 12008, 12009, 11951, 11925, 11924, + 11922, 11949, 11948, 11917, 11916, 11958, 11932, 12017, 11911, 11923, + 11969, 11982, 11981, 11938, 11937, 11972, 11971, 11956, 11955, 11954, + 11953, 11913, 11912, 11961, 12752, 12743, 12748, 12768, 12772, 12757, + 12741, 12750, 12769, 12747, 12749, 12744, 12742, 12746, 12758, 12754, + 12763, 12770, 12764, 12753, 12740, 12767, 12760, 12759, 12745, 12773, + 12766, 12762, 12755, 12761, 12736, 12765, 12739, 12737, 12738, 12756, + 12751, 12771, 128079, 127916, 127963, 128385, 129346, 127867, 128203, + 128346, 128358, 128343, 128355, 128340, 128352, 128339, 128351, 128344, + 128356, 128336, 128348, 128342, 128354, 128341, 128353, 128345, 128357, + 128347, 128359, 128337, 128349, 128338, 128350, 10561, 8754, 128259, + 10227, 128472, 128257, 128258, 8631, 11118, 8635, 8753, 10959, 10961, + 10960, 10962, 127746, 10828, 10832, 128234, 128235, 128272, 128213, + 10829, 8272, 9729, 127786, 127785, 127784, 127783, 129313, 9114, 129715, + 127864, 129381, 58, 8788, 8353, 128165, 6867, 7625, 7623, 769, 791, 833, + 8410, 8404, 8423, 857, 8432, 7677, 844, 774, 7627, 814, 810, 838, 70459, + 780, 812, 784, 8409, 8405, 787, 789, 806, 65062, 65069, 42609, 1160, + 11774, 11744, 11768, 11747, 11757, 11765, 42654, 11751, 11752, 11753, + 11756, 42613, 11775, 11772, 42655, 11767, 11754, 42619, 11763, 11762, + 42618, 42615, 42612, 42617, 11770, 42614, 11771, 11773, 11769, 11759, + 42616, 11760, 11758, 11748, 11749, 11761, 11746, 11764, 11755, 11745, + 11750, 11766, 42621, 1156, 1158, 1159, 1157, 42608, 42610, 1155, 65070, + 65071, 1161, 42620, 123023, 42607, 770, 813, 807, 43248, 43244, 43245, + 43246, 43247, 43242, 43243, 43249, 43237, 43236, 43239, 43238, 43235, + 43234, 43232, 43241, 43233, 43240, 7675, 776, 804, 6876, 6833, 775, 7672, + 856, 803, 7674, 6877, 7617, 7616, 6886, 6887, 779, 861, 860, 865, 7676, + 6863, 7629, 862, 863, 6840, 831, 6858, 6857, 6844, 866, 6891, 858, 864, + 65058, 65059, 8422, 840, 782, 783, 819, 6832, 798, 6875, 6835, 8413, + 8416, 8418, 8414, 8419, 8420, 8415, 839, 6888, 850, 8412, 6874, 122892, + 122884, 122891, 122890, 122921, 122919, 122889, 122916, 122900, 122907, + 122910, 122901, 122908, 122880, 122920, 122881, 122909, 122903, 122922, + 122883, 122895, 122894, 122896, 122898, 122899, 122882, 122885, 122912, + 122911, 122913, 122918, 122915, 122888, 122886, 122904, 122902, 122897, + 122893, 70508, 70507, 70506, 70505, 70504, 70502, 70503, 70515, 70513, + 70514, 70516, 70512, 768, 790, 832, 6865, 7624, 7621, 847, 119363, + 119362, 119364, 835, 834, 837, 836, 777, 843, 795, 785, 815, 826, 6883, + 811, 6855, 6834, 7632, 12442, 12441, 7671, 7670, 7643, 7646, 7647, 7649, + 7650, 867, 7666, 7655, 7636, 7637, 7638, 872, 7639, 868, 7663, 7641, + 7659, 7635, 869, 7640, 6860, 6861, 6862, 7645, 7660, 7653, 870, 7667, + 7661, 871, 7668, 7664, 876, 7651, 7626, 877, 6848, 7652, 7658, 7656, + 7657, 7665, 6847, 873, 7642, 874, 7644, 875, 7648, 7662, 878, 879, 7654, + 6889, 841, 794, 852, 7678, 8430, 8406, 6849, 6851, 796, 849, 8400, 792, + 6880, 845, 8417, 8429, 8426, 65056, 65063, 65057, 65064, 6841, 8427, 824, + 822, 8402, 818, 772, 65060, 65067, 65061, 65068, 817, 6869, 7620, 6872, + 7622, 7628, 800, 6882, 6854, 842, 66424, 66425, 66423, 66426, 66422, + 6839, 808, 7630, 773, 6846, 6845, 6843, 801, 799, 6856, 8421, 788, 802, + 7679, 854, 848, 853, 8431, 8407, 825, 855, 8401, 6850, 6852, 793, 6881, + 8428, 8408, 805, 778, 7631, 859, 823, 821, 8403, 6873, 6853, 827, 6884, + 6842, 7619, 7618, 828, 6885, 8411, 771, 65065, 65066, 820, 816, 6859, + 8424, 6836, 786, 797, 7669, 846, 6890, 7633, 7634, 809, 781, 830, 6864, + 6870, 6866, 6871, 6868, 7673, 8425, 6838, 6837, 851, 829, 8274, 64, 44, 128476, 9092, 8705, 129517, 9732, 127882, 128533, 128534, 128119, 128679, 8715, 8883, 8885, 8954, 8955, 8957, 8750, 127899, 983187, 9089, 127978, 9010, 9740, 10861, 127859, 127850, 127834, 11505, 11504, 11503, 11464, @@ -10258,30 +10375,30 @@ static const unsigned int dawg_pos_to_codepoint[] = { 983084, 983086, 983088, 983090, 983162, 9110, 9192, 127795, 128475, 8451, 176, 8457, 983120, 128666, 8796, 983119, 9161, 9159, 9153, 9156, 9162, 9160, 9154, 9157, 9164, 9151, 9163, 9150, 9158, 9152, 9155, 117829, - 117828, 127980, 9739, 66589, 66591, 66596, 66597, 66587, 66585, 66594, - 66595, 66593, 66599, 66562, 66563, 66564, 66565, 66561, 66560, 66568, - 66569, 66570, 66571, 66567, 66566, 66598, 66573, 66588, 66579, 66592, - 66590, 66581, 66578, 66580, 66582, 66577, 66586, 66575, 66584, 66583, - 66574, 66572, 66576, 66629, 66631, 66636, 66637, 66627, 66625, 66634, - 66635, 66633, 66639, 66602, 66603, 66604, 66605, 66601, 66600, 66608, - 66609, 66610, 66611, 66607, 66606, 66638, 66613, 66628, 66619, 66632, - 66630, 66621, 66618, 66620, 66622, 66617, 66626, 66615, 66624, 66623, - 66614, 66612, 66616, 127964, 127965, 128421, 128468, 2388, 2416, 43258, - 2387, 43257, 72448, 72449, 43259, 2309, 2310, 2320, 2324, 2421, 43262, - 2330, 2418, 2317, 2321, 2331, 2396, 2430, 2338, 2337, 2343, 2342, 2429, - 2394, 2328, 2427, 2327, 2426, 2361, 2350, 2424, 2308, 2318, 2322, 2358, - 2359, 2360, 2323, 2420, 2419, 2431, 2349, 2348, 2393, 2326, 2325, 2397, - 2353, 2352, 2399, 2351, 2313, 2314, 2423, 2422, 2345, 2339, 2329, 2334, - 2344, 2333, 2428, 2332, 2356, 2355, 2354, 2336, 2335, 2341, 2340, 2315, - 2400, 2316, 2401, 2357, 2311, 2312, 2347, 2346, 2425, 2395, 2398, 2392, - 2319, 983644, 983643, 983646, 983647, 983649, 983648, 983642, 983645, - 72450, 72451, 72452, 72453, 2305, 43255, 43251, 43254, 43253, 72455, - 72454, 72456, 43250, 43260, 2304, 72457, 2364, 2365, 2306, 43252, 43256, - 2417, 2381, 2307, 2386, 2385, 2366, 2376, 2380, 2383, 43263, 2389, 2373, - 2377, 2379, 2363, 2362, 2382, 2369, 2370, 2391, 2390, 2374, 2378, 2371, - 2372, 2402, 2403, 2367, 2368, 2375, 2405, 2404, 2411, 2410, 2413, 2412, - 2409, 2408, 2406, 2415, 2407, 2414, 43261, 2384, 983089, 983161, 983087, - 983085, 983083, 127962, 129487, 129420, 11033, 11032, 11030, 11031, + 117828, 127980, 127962, 9739, 66589, 66591, 66596, 66597, 66587, 66585, + 66594, 66595, 66593, 66599, 66562, 66563, 66564, 66565, 66561, 66560, + 66568, 66569, 66570, 66571, 66567, 66566, 66598, 66573, 66588, 66579, + 66592, 66590, 66581, 66578, 66580, 66582, 66577, 66586, 66575, 66584, + 66583, 66574, 66572, 66576, 66629, 66631, 66636, 66637, 66627, 66625, + 66634, 66635, 66633, 66639, 66602, 66603, 66604, 66605, 66601, 66600, + 66608, 66609, 66610, 66611, 66607, 66606, 66638, 66613, 66628, 66619, + 66632, 66630, 66621, 66618, 66620, 66622, 66617, 66626, 66615, 66624, + 66623, 66614, 66612, 66616, 127964, 127965, 128421, 128468, 2388, 2416, + 43258, 2387, 43257, 72448, 72449, 43259, 2309, 2310, 2320, 2324, 2421, + 43262, 2330, 2418, 2317, 2321, 2331, 2396, 2430, 2338, 2337, 2343, 2342, + 2429, 2394, 2328, 2427, 2327, 2426, 2361, 2350, 2424, 2308, 2318, 2322, + 2358, 2359, 2360, 2431, 2349, 2348, 2393, 2326, 2325, 2397, 2353, 2352, + 2323, 2420, 2419, 2399, 2351, 2313, 2314, 2423, 2422, 2345, 2339, 2329, + 2334, 2344, 2333, 2428, 2332, 2356, 2355, 2354, 2336, 2335, 2341, 2340, + 2315, 2400, 2316, 2401, 2357, 2311, 2312, 2347, 2346, 2425, 2395, 2398, + 2392, 2319, 983644, 983643, 983646, 983647, 983649, 983648, 983642, + 983645, 72450, 72451, 72452, 72453, 2305, 43255, 43251, 43254, 43253, + 72455, 72454, 72456, 43250, 43260, 2304, 72457, 2364, 2365, 2306, 43252, + 43256, 2417, 2381, 2307, 2386, 2385, 2366, 2376, 2380, 2383, 43263, 2389, + 2373, 2377, 2379, 2363, 2362, 2382, 2369, 2370, 2391, 2390, 2374, 2378, + 2371, 2372, 2402, 2403, 2367, 2368, 2375, 2405, 2404, 2411, 2410, 2413, + 2412, 2409, 2408, 2406, 2415, 2407, 2414, 43261, 2384, 983089, 983161, + 983087, 983085, 983083, 129487, 129420, 11033, 11032, 11030, 11031, 128924, 128160, 8900, 8960, 168, 117827, 9856, 9857, 9858, 9859, 9860, 9861, 128754, 53, 127238, 9356, 52, 127237, 9355, 57, 127242, 9360, 49, 127234, 9352, 55, 127240, 9358, 54, 127239, 9357, 51, 127236, 9354, 50, @@ -10290,645 +10407,646 @@ static const unsigned int dawg_pos_to_codepoint[] = { 10128, 10127, 10124, 10123, 127244, 10130, 10122, 10129, 10111, 10106, 10105, 10108, 10107, 10104, 10103, 10110, 10102, 10109, 10121, 10116, 10115, 10118, 10117, 10114, 10113, 127243, 10120, 10112, 10119, 9107, - 127919, 128549, 128542, 9933, 9090, 129400, 72004, 72021, 72020, 72023, - 72022, 72019, 72018, 72016, 72025, 72017, 72024, 71964, 71958, 71963, - 71974, 71973, 71936, 71937, 71961, 71960, 71966, 71965, 71940, 71941, - 71938, 71939, 71982, 71976, 71952, 71962, 71957, 71967, 71978, 71979, - 71980, 71971, 71970, 71954, 71953, 71951, 71950, 71949, 71948, 71969, - 71968, 71981, 71955, 71972, 71975, 71977, 71983, 71942, 71945, 71997, - 71996, 72003, 71995, 71984, 71991, 71987, 71988, 71985, 71986, 71989, - 71992, 71998, 72002, 72000, 72005, 72006, 71999, 72001, 8725, 247, 8903, - 129343, 8739, 9902, 129684, 128171, 128565, 12291, 8783, 9009, 128462, - 128441, 128442, 128443, 8939, 8941, 8716, 8740, 10990, 8832, 8928, 8876, - 8833, 8929, 8878, 128021, 71680, 71681, 71687, 71689, 71703, 71702, - 71708, 71707, 71723, 71716, 71701, 71700, 71706, 71705, 71684, 71685, - 71682, 71683, 71694, 71704, 71699, 71709, 71719, 71720, 71721, 71713, - 71712, 71696, 71695, 71693, 71692, 71698, 71697, 71691, 71690, 71711, - 71710, 71722, 71717, 71714, 71718, 71715, 71686, 71688, 71729, 71730, - 71724, 71732, 71734, 71727, 71728, 71725, 71726, 71731, 71733, 71739, - 71738, 71735, 71737, 71736, 128054, 128044, 36, 127025, 127026, 127027, - 127028, 127029, 127030, 127031, 127032, 127033, 127034, 127035, 127036, - 127037, 127038, 127039, 127040, 127041, 127042, 127043, 127044, 127045, - 127046, 127047, 127048, 127049, 127050, 127051, 127052, 127053, 127054, - 127055, 127056, 127057, 127058, 127059, 127060, 127061, 127062, 127063, - 127064, 127065, 127066, 127067, 127068, 127069, 127070, 127071, 127072, - 127073, 127024, 127075, 127076, 127077, 127078, 127079, 127080, 127081, - 127082, 127083, 127084, 127085, 127086, 127087, 127088, 127089, 127090, - 127091, 127092, 127093, 127094, 127095, 127096, 127097, 127098, 127099, - 127100, 127101, 127102, 127103, 127104, 127105, 127106, 127107, 127108, - 127109, 127110, 127111, 127112, 127113, 127114, 127115, 127116, 127117, - 127118, 127119, 127120, 127121, 127122, 127123, 127074, 129743, 8363, - 8724, 8760, 8901, 729, 9676, 8284, 11850, 11034, 129765, 11795, 11784, - 10649, 11798, 9470, 9465, 9464, 9467, 9466, 9463, 9462, 9469, 9461, 9468, - 10175, 10868, 10986, 8225, 8223, 11840, 8914, 8748, 11842, 8222, 8215, - 10835, 10836, 10645, 10913, 10915, 10914, 10939, 8243, 12318, 10746, - 10830, 10831, 11849, 10988, 11844, 10940, 8913, 8912, 11005, 10987, 8915, - 9208, 10981, 8875, 10979, 8214, 11799, 10646, 733, 8252, 8263, 65100, - 10908, 10907, 11002, 11001, 10906, 10905, 8510, 8473, 8511, 8450, 8461, - 8469, 8474, 8477, 8484, 8518, 8519, 8520, 8521, 8517, 8509, 8508, 8512, - 10719, 9890, 9891, 11260, 127849, 8868, 10993, 10623, 8945, 8964, 117764, - 128317, 117859, 118264, 117883, 118268, 117849, 128315, 117914, 117864, - 10728, 10729, 117875, 117879, 129288, 129290, 129289, 129291, 8595, 8629, - 8671, 129035, 129031, 129179, 129043, 129027, 129047, 8626, 8627, 10504, - 8693, 129975, 8615, 10515, 11796, 11015, 129203, 11147, 11107, 11139, - 11123, 129067, 11168, 11169, 129063, 129059, 129075, 129071, 11133, - 11085, 11117, 11143, 129171, 10507, 8609, 11247, 8681, 129175, 8623, - 8659, 8675, 129079, 10597, 10607, 10593, 10585, 8643, 10589, 10581, 8642, - 129091, 129095, 129087, 10225, 128623, 129107, 129083, 8650, 128687, - 128330, 128682, 129444, 8367, 10139, 128009, 128050, 128042, 129656, - 128167, 129316, 129345, 9946, 128087, 113784, 113788, 113785, 113783, - 113782, 113786, 113787, 113795, 113798, 113793, 113800, 113781, 113794, - 113792, 113799, 113797, 113796, 113776, 113808, 113817, 113811, 113814, - 113809, 113816, 113779, 113810, 113815, 113813, 113812, 113778, 113777, - 113780, 113729, 113733, 113672, 113677, 113683, 113735, 113739, 113746, - 113668, 113678, 113674, 113726, 113691, 113699, 113700, 113695, 113709, - 113712, 113713, 113705, 113711, 113669, 113725, 113679, 113684, 113670, - 113743, 113749, 113687, 113689, 113693, 113707, 113697, 113703, 113690, - 113694, 113708, 113698, 113704, 113764, 113763, 113762, 113761, 113732, - 113753, 113731, 113755, 113754, 113666, 113766, 113765, 113676, 113675, - 113741, 113750, 113680, 113688, 113692, 113696, 113710, 113727, 113728, - 113716, 113717, 113714, 113715, 113701, 113702, 113724, 113723, 113706, - 113742, 113740, 113767, 113769, 113730, 113768, 113682, 113685, 113752, - 113737, 113667, 113719, 113718, 113681, 113745, 113748, 113751, 113738, - 113673, 113770, 113720, 113757, 113760, 113722, 113759, 113756, 113721, - 113758, 113665, 113747, 113664, 113686, 113734, 113736, 113744, 113671, - 113821, 113822, 113823, 113820, 129375, 129414, 129516, 128192, 983082, - 128066, 127805, 127806, 129467, 9793, 127758, 127759, 127757, 9178, 9841, - 129413, 11790, 77830, 77831, 77832, 77828, 77829, 77824, 77825, 77826, - 77827, 77833, 77834, 77835, 77840, 77841, 77844, 77845, 77836, 77837, - 77838, 77839, 77842, 77843, 77846, 77847, 77869, 77870, 77872, 77873, - 77874, 77875, 77877, 77878, 77871, 77876, 77879, 77880, 77881, 77882, - 77860, 77861, 77858, 77859, 77862, 77863, 77864, 77865, 77866, 77867, - 77868, 77903, 77848, 77849, 77850, 77851, 77852, 77853, 77854, 77855, - 77856, 77857, 77883, 77884, 77885, 77886, 77887, 77888, 77889, 77890, - 77891, 77892, 77893, 77894, 77895, 77896, 77897, 77898, 77899, 77900, - 77901, 77902, 78867, 78868, 78869, 78861, 78862, 78863, 78864, 78865, - 78866, 78870, 78871, 78892, 78893, 78894, 78872, 78873, 78874, 78875, - 78876, 78877, 78878, 78879, 78880, 78881, 78882, 78883, 78884, 78885, - 78886, 78887, 78888, 78889, 78890, 78891, 78910, 78908, 78903, 77908, - 77909, 77904, 77905, 77906, 77907, 77910, 77911, 77912, 77913, 77915, - 77916, 77917, 77918, 77914, 77919, 77920, 77921, 77922, 77923, 77924, - 77925, 77926, 77927, 77928, 77929, 77930, 77931, 77932, 77933, 77934, - 77935, 77936, 77937, 77938, 77939, 77940, 77941, 77949, 77950, 77942, - 77943, 77944, 77945, 77946, 77947, 77948, 77951, 77974, 77975, 77978, - 77979, 77973, 77976, 77977, 77980, 77981, 77982, 77983, 77984, 77991, - 77992, 77994, 77995, 77985, 77986, 77987, 77988, 77989, 77990, 77993, - 77996, 77997, 77998, 77999, 78000, 78001, 78002, 78003, 78004, 78005, - 78006, 78008, 78009, 78011, 78012, 78007, 78010, 78013, 78014, 78015, - 78016, 78017, 78025, 78026, 78027, 78028, 78029, 78030, 78031, 78032, - 78033, 78018, 78019, 78020, 78021, 78022, 78023, 78024, 77969, 77970, - 77962, 77963, 77964, 77965, 77966, 77967, 77968, 77971, 77972, 77952, - 77953, 77954, 77955, 77956, 77957, 77958, 77959, 77960, 77961, 78041, - 78042, 78043, 78044, 78034, 78035, 78036, 78037, 78038, 78039, 78040, - 78057, 78058, 78066, 78067, 78059, 78060, 78061, 78062, 78063, 78064, - 78065, 78068, 78073, 78074, 78069, 78070, 78071, 78072, 78075, 78076, - 78077, 78051, 78052, 78053, 78054, 78045, 78046, 78047, 78048, 78049, - 78050, 78055, 78056, 78911, 78909, 78904, 78078, 78079, 78080, 78081, - 78082, 78083, 78084, 78085, 78086, 78087, 78091, 78092, 78088, 78089, - 78090, 78093, 78094, 78095, 78096, 78097, 78098, 78111, 78112, 78118, - 78119, 78120, 78121, 78110, 78113, 78114, 78115, 78116, 78117, 78122, - 78128, 78129, 78130, 78131, 78132, 78133, 78123, 78124, 78125, 78126, - 78127, 78134, 78135, 78137, 78138, 78139, 78140, 78136, 78141, 78142, - 78100, 78101, 78099, 78102, 78103, 78104, 78105, 78106, 78107, 78108, - 78109, 78913, 78150, 78151, 78152, 78148, 78149, 78143, 78144, 78145, - 78146, 78147, 78153, 78154, 78156, 78157, 78155, 78158, 78159, 78160, - 78161, 78162, 78163, 78164, 78165, 78184, 78185, 78186, 78187, 78178, - 78179, 78180, 78181, 78182, 78183, 78188, 78189, 78193, 78194, 78196, - 78197, 78190, 78191, 78192, 78195, 78198, 78199, 78200, 78201, 78166, - 78167, 78173, 78174, 78168, 78169, 78170, 78171, 78172, 78175, 78176, - 78177, 78202, 78203, 78204, 78205, 78206, 78212, 78213, 78207, 78208, - 78209, 78210, 78211, 78214, 78215, 78914, 78916, 78897, 78220, 78221, - 78225, 78226, 78216, 78217, 78218, 78219, 78222, 78223, 78224, 78227, - 78228, 78229, 78230, 78231, 78232, 78233, 78234, 78907, 78901, 78899, - 78906, 78900, 78898, 78905, 78244, 78245, 78249, 78250, 78243, 78246, - 78247, 78248, 78251, 78252, 78915, 78253, 78254, 78255, 78257, 78258, - 78256, 78259, 78260, 78261, 78262, 78263, 78264, 78268, 78269, 78270, - 78271, 78272, 78273, 78274, 78275, 78276, 78265, 78266, 78279, 78280, - 78281, 78282, 78283, 78284, 78267, 78277, 78278, 78285, 78286, 78289, - 78290, 78292, 78293, 78297, 78298, 78287, 78288, 78291, 78294, 78295, - 78296, 78299, 78304, 78305, 78306, 78301, 78302, 78300, 78303, 78307, - 78308, 78309, 78310, 78311, 78312, 78313, 78314, 78315, 78316, 78317, - 78318, 78933, 78928, 78920, 78924, 78932, 78926, 78921, 78929, 78925, - 78923, 78931, 78919, 78927, 78922, 78930, 78912, 78336, 78337, 78338, - 78328, 78329, 78330, 78331, 78332, 78333, 78334, 78335, 78339, 78354, - 78355, 78356, 78357, 78358, 78359, 78361, 78362, 78351, 78352, 78353, - 78360, 78363, 78364, 78345, 78346, 78340, 78341, 78342, 78343, 78344, - 78347, 78348, 78349, 78350, 78365, 78366, 78367, 78319, 78320, 78321, - 78322, 78323, 78324, 78325, 78326, 78327, 78372, 78373, 78368, 78369, - 78370, 78371, 78374, 78375, 78376, 78377, 78385, 78386, 78378, 78379, - 78380, 78381, 78382, 78383, 78384, 78387, 78388, 78389, 78399, 78400, - 78401, 78402, 78409, 78410, 78403, 78404, 78405, 78406, 78407, 78408, - 78411, 78414, 78415, 78412, 78413, 78390, 78391, 78392, 78393, 78394, - 78395, 78396, 78397, 78398, 78423, 78424, 78425, 78426, 78427, 78428, - 78429, 78416, 78417, 78421, 78422, 78418, 78419, 78420, 78430, 78431, - 78432, 78433, 78434, 78435, 78436, 78445, 78446, 78437, 78438, 78439, - 78440, 78441, 78442, 78443, 78444, 78447, 78448, 78452, 78453, 78454, - 78455, 78459, 78460, 78449, 78450, 78451, 78456, 78457, 78458, 78469, - 78470, 78471, 78472, 78473, 78461, 78462, 78465, 78466, 78463, 78464, - 78467, 78468, 78474, 78475, 78476, 78487, 78488, 78489, 78490, 78477, - 78478, 78479, 78480, 78481, 78482, 78483, 78484, 78485, 78486, 78902, - 78491, 78492, 78494, 78495, 78493, 78496, 78497, 78498, 78499, 78500, - 78501, 78502, 78503, 78514, 78515, 78516, 78512, 78513, 78511, 78517, - 78518, 78519, 78520, 78521, 78522, 78523, 78524, 78530, 78531, 78525, - 78526, 78527, 78528, 78529, 78532, 78533, 78534, 78535, 78536, 78537, - 78538, 78539, 78540, 78541, 78542, 78543, 78544, 78546, 78547, 78551, - 78552, 78545, 78548, 78549, 78550, 78553, 78554, 78555, 78560, 78561, - 78562, 78565, 78566, 78556, 78557, 78558, 78559, 78563, 78564, 78567, - 78568, 78575, 78576, 78577, 78569, 78570, 78571, 78572, 78573, 78574, - 78578, 78579, 78580, 78586, 78587, 78581, 78582, 78583, 78584, 78585, - 78588, 78589, 78590, 78591, 78592, 78593, 78594, 78595, 78596, 78597, - 78598, 78601, 78602, 78606, 78607, 78608, 78609, 78610, 78611, 78599, - 78600, 78603, 78604, 78605, 78613, 78614, 78619, 78620, 78612, 78615, - 78616, 78617, 78618, 78621, 78622, 78623, 78636, 78637, 78638, 78639, - 78634, 78635, 78640, 78641, 78642, 78624, 78625, 78626, 78627, 78628, - 78629, 78630, 78631, 78632, 78633, 78917, 78648, 78649, 78650, 78643, - 78644, 78645, 78646, 78647, 78651, 78652, 78653, 78667, 78668, 78674, - 78675, 78664, 78665, 78666, 78669, 78670, 78671, 78672, 78673, 78678, - 78679, 78676, 78677, 78680, 78681, 78682, 78683, 78684, 78685, 78686, - 78687, 78688, 78689, 78654, 78655, 78656, 78657, 78658, 78659, 78660, - 78661, 78662, 78663, 78706, 78707, 78708, 78690, 78691, 78692, 78693, - 78694, 78695, 78696, 78697, 78698, 78699, 78700, 78701, 78702, 78703, - 78704, 78705, 78709, 78710, 78712, 78713, 78714, 78715, 78895, 78716, - 78717, 78718, 78711, 78719, 78720, 78721, 78722, 78723, 78724, 78725, - 78726, 78727, 78728, 78729, 78730, 78731, 78732, 78733, 78734, 78735, - 78736, 78737, 78738, 78741, 78742, 78747, 78748, 78749, 78750, 78739, - 78740, 78743, 78744, 78745, 78746, 78751, 78752, 78753, 78754, 78756, - 78757, 78761, 78762, 78755, 78758, 78759, 78760, 78763, 78764, 78765, - 78766, 78896, 78769, 78770, 78776, 78777, 78767, 78768, 78771, 78772, - 78773, 78774, 78775, 78778, 78779, 78783, 78784, 78787, 78788, 78789, - 78790, 78780, 78781, 78782, 78785, 78786, 78791, 78796, 78797, 78792, - 78793, 78794, 78795, 78798, 78918, 78802, 78803, 78804, 78806, 78807, - 78809, 78810, 78799, 78800, 78801, 78805, 78808, 78811, 78812, 78813, - 78814, 78815, 78816, 78817, 78818, 78819, 78821, 78822, 78823, 78824, - 78825, 78826, 78827, 78828, 78829, 78830, 78831, 78832, 78820, 78833, - 78834, 78835, 78836, 78842, 78843, 78844, 78845, 78846, 78847, 78848, - 78849, 78850, 78851, 78852, 78853, 78854, 78855, 78856, 78857, 78858, - 78859, 78860, 78837, 78838, 78839, 78840, 78841, 78235, 78236, 78237, - 78238, 78239, 78240, 78241, 78242, 78504, 78505, 78506, 78507, 78508, - 78509, 78510, 78944, 78945, 78946, 78947, 78948, 78949, 78950, 78951, - 78952, 78953, 78954, 78955, 78956, 78957, 78958, 78959, 78960, 78961, - 78962, 78963, 78964, 78965, 78966, 78967, 78968, 78969, 78970, 78971, - 78972, 78973, 78974, 78975, 78976, 78977, 78978, 78979, 78980, 78981, - 78982, 78983, 78984, 78985, 78986, 78987, 78988, 78989, 78990, 78991, - 78992, 78993, 78994, 78995, 78996, 78997, 78998, 78999, 79000, 79001, - 79002, 79003, 79004, 79005, 79006, 79007, 79008, 79009, 79010, 79011, - 79012, 79013, 79014, 79015, 79016, 79017, 79018, 79019, 79020, 79021, - 79022, 79023, 79024, 79025, 79026, 79027, 79028, 79029, 79030, 79031, - 79032, 79033, 79034, 79035, 79036, 79037, 79038, 79039, 79040, 79041, - 79042, 79043, 79044, 79045, 79046, 79047, 79048, 79049, 79050, 79051, - 79052, 79053, 79054, 79055, 79056, 79057, 79058, 79059, 79060, 79061, - 79062, 79063, 79064, 79065, 79066, 79067, 79068, 79069, 79070, 79071, - 79072, 79073, 79074, 79075, 79076, 79077, 79078, 79079, 79080, 79081, - 79082, 79083, 79084, 79085, 79086, 79087, 79088, 79089, 79090, 79091, - 79092, 79093, 79094, 79095, 79096, 79097, 79098, 79099, 79100, 79101, - 79102, 79103, 79104, 79105, 79106, 79107, 79108, 79109, 79110, 79111, - 79112, 79113, 79114, 79115, 79116, 79117, 79118, 79119, 79120, 79121, - 79122, 79123, 79124, 79125, 79126, 79127, 79128, 79129, 79130, 79131, - 79132, 79133, 79134, 79135, 79136, 79137, 79138, 79139, 79140, 79141, - 79142, 79143, 79144, 79145, 79146, 79147, 79148, 79149, 79150, 79151, - 79152, 79153, 79154, 79155, 79156, 79157, 79158, 79159, 79160, 79161, - 79162, 79163, 79164, 79165, 79166, 79167, 79168, 79169, 79170, 79171, - 79172, 79173, 79174, 79175, 79176, 79177, 79178, 79179, 79180, 79181, - 79182, 79183, 79184, 79185, 79186, 79187, 79188, 79189, 79190, 79191, - 79192, 79193, 79194, 79195, 79196, 79197, 79198, 79199, 79200, 79201, - 79202, 79203, 79204, 79205, 79206, 79207, 79208, 79209, 79210, 79211, - 79212, 79213, 79214, 79215, 79216, 79217, 79218, 79219, 79220, 79221, - 79222, 79223, 79224, 79225, 79226, 79227, 79228, 79229, 79230, 79231, - 79232, 79233, 79234, 79235, 79236, 79237, 79238, 79239, 79240, 79241, - 79242, 79243, 79244, 79245, 79246, 79247, 79248, 79249, 79250, 79251, - 79252, 79253, 79254, 79255, 79256, 79257, 79258, 79259, 79260, 79261, - 79262, 79263, 79264, 79265, 79266, 79267, 79268, 79269, 79270, 79271, - 79272, 79273, 79274, 79275, 79276, 79277, 79278, 79279, 79280, 79281, - 79282, 79283, 79284, 79285, 79286, 79287, 79288, 79289, 79290, 79291, - 79292, 79293, 79294, 79295, 79296, 79297, 79298, 79299, 79300, 79301, - 79302, 79303, 79304, 79305, 79306, 79307, 79308, 79309, 79310, 79311, - 79312, 79313, 79314, 79315, 79316, 79317, 79318, 79319, 79320, 79321, - 79322, 79323, 79324, 79325, 79326, 79327, 79328, 79329, 79330, 79331, - 79332, 79333, 79334, 79335, 79336, 79337, 79338, 79339, 79340, 79341, - 79342, 79343, 79344, 79345, 79346, 79347, 79348, 79349, 79350, 79351, - 79352, 79353, 79354, 79355, 79356, 79357, 79358, 79359, 79360, 79361, - 79362, 79363, 79364, 79365, 79366, 79367, 79368, 79369, 79370, 79371, - 79372, 79373, 79374, 79375, 79376, 79377, 79378, 79379, 79380, 79381, - 79382, 79383, 79384, 79385, 79386, 79387, 79388, 79389, 79390, 79391, - 79392, 79393, 79394, 79395, 79396, 79397, 79398, 79399, 79400, 79401, - 79402, 79403, 79404, 79405, 79406, 79407, 79408, 79409, 79410, 79411, - 79412, 79413, 79414, 79415, 79416, 79417, 79418, 79419, 79420, 79421, - 79422, 79423, 79424, 79425, 79426, 79427, 79428, 79429, 79430, 79431, - 79432, 79433, 79434, 79435, 79436, 79437, 79438, 79439, 79440, 79441, - 79442, 79443, 79444, 79445, 79446, 79447, 79448, 79449, 79450, 79451, - 79452, 79453, 79454, 79455, 79456, 79457, 79458, 79459, 79460, 79461, - 79462, 79463, 79464, 79465, 79466, 79467, 79468, 79469, 79470, 79471, - 79472, 79473, 79474, 79475, 79476, 79477, 79478, 79479, 79480, 79481, - 79482, 79483, 79484, 79485, 79486, 79487, 79488, 79489, 79490, 79491, - 79492, 79493, 79494, 79495, 79496, 79497, 79498, 79499, 79500, 79501, - 79502, 79503, 79504, 79505, 79506, 79507, 79508, 79509, 79510, 79511, - 79512, 79513, 79514, 79515, 79516, 79517, 79518, 79519, 79520, 79521, - 79522, 79523, 79524, 79525, 79526, 79527, 79528, 79529, 79530, 79531, - 79532, 79533, 79534, 79535, 79536, 79537, 79538, 79539, 79540, 79541, - 79542, 79543, 79544, 79545, 79546, 79547, 79548, 79549, 79550, 79551, - 79552, 79553, 79554, 79555, 79556, 79557, 79558, 79559, 79560, 79561, - 79562, 79563, 79564, 79565, 79566, 79567, 79568, 79569, 79570, 79571, - 79572, 79573, 79574, 79575, 79576, 79577, 79578, 79579, 79580, 79581, - 79582, 79583, 79584, 79585, 79586, 79587, 79588, 79589, 79590, 79591, - 79592, 79593, 79594, 79595, 79596, 79597, 79598, 79599, 79600, 79601, - 79602, 79603, 79604, 79605, 79606, 79607, 79608, 79609, 79610, 79611, - 79612, 79613, 79614, 79615, 79616, 79617, 79618, 79619, 79620, 79621, - 79622, 79623, 79624, 79625, 79626, 79627, 79628, 79629, 79630, 79631, - 79632, 79633, 79634, 79635, 79636, 79637, 79638, 79639, 79640, 79641, - 79642, 79643, 79644, 79645, 79646, 79647, 79648, 79649, 79650, 79651, - 79652, 79653, 79654, 79655, 79656, 79657, 79658, 79659, 79660, 79661, - 79662, 79663, 79664, 79665, 79666, 79667, 79668, 79669, 79670, 79671, - 79672, 79673, 79674, 79675, 79676, 79677, 79678, 79679, 79680, 79681, - 79682, 79683, 79684, 79685, 79686, 79687, 79688, 79689, 79690, 79691, - 79692, 79693, 79694, 79695, 79696, 79697, 79698, 79699, 79700, 79701, - 79702, 79703, 79704, 79705, 79706, 79707, 79708, 79709, 79710, 79711, - 79712, 79713, 79714, 79715, 79716, 79717, 79718, 79719, 79720, 79721, - 79722, 79723, 79724, 79725, 79726, 79727, 79728, 79729, 79730, 79731, - 79732, 79733, 79734, 79735, 79736, 79737, 79738, 79739, 79740, 79741, - 79742, 79743, 79744, 79745, 79746, 79747, 79748, 79749, 79750, 79751, - 79752, 79753, 79754, 79755, 79756, 79757, 79758, 79759, 79760, 79761, - 79762, 79763, 79764, 79765, 79766, 79767, 79768, 79769, 79770, 79771, - 79772, 79773, 79774, 79775, 79776, 79777, 79778, 79779, 79780, 79781, - 79782, 79783, 79784, 79785, 79786, 79787, 79788, 79789, 79790, 79791, - 79792, 79793, 79794, 79795, 79796, 79797, 79798, 79799, 79800, 79801, - 79802, 79803, 79804, 79805, 79806, 79807, 79808, 79809, 79810, 79811, - 79812, 79813, 79814, 79815, 79816, 79817, 79818, 79819, 79820, 79821, - 79822, 79823, 79824, 79825, 79826, 79827, 79828, 79829, 79830, 79831, - 79832, 79833, 79834, 79835, 79836, 79837, 79838, 79839, 79840, 79841, - 79842, 79843, 79844, 79845, 79846, 79847, 79848, 79849, 79850, 79851, - 79852, 79853, 79854, 79855, 79856, 79857, 79858, 79859, 79860, 79861, - 79862, 79863, 79864, 79865, 79866, 79867, 79868, 79869, 79870, 79871, - 79872, 79873, 79874, 79875, 79876, 79877, 79878, 79879, 79880, 79881, - 79882, 79883, 79884, 79885, 79886, 79887, 79888, 79889, 79890, 79891, - 79892, 79893, 79894, 79895, 79896, 79897, 79898, 79899, 79900, 79901, - 79902, 79903, 79904, 79905, 79906, 79907, 79908, 79909, 79910, 79911, - 79912, 79913, 79914, 79915, 79916, 79917, 79918, 79919, 79920, 79921, - 79922, 79923, 79924, 79925, 79926, 79927, 79928, 79929, 79930, 79931, - 79932, 79933, 79934, 79935, 79936, 79937, 79938, 79939, 79940, 79941, - 79942, 79943, 79944, 79945, 79946, 79947, 79948, 79949, 79950, 79951, - 79952, 79953, 79954, 79955, 79956, 79957, 79958, 79959, 79960, 79961, - 79962, 79963, 79964, 79965, 79966, 79967, 79968, 79969, 79970, 79971, - 79972, 79973, 79974, 79975, 79976, 79977, 79978, 79979, 79980, 79981, - 79982, 79983, 79984, 79985, 79986, 79987, 79988, 79989, 79990, 79991, - 79992, 79993, 79994, 79995, 79996, 79997, 79998, 79999, 80000, 80001, - 80002, 80003, 80004, 80005, 80006, 80007, 80008, 80009, 80010, 80011, - 80012, 80013, 80014, 80015, 80016, 80017, 80018, 80019, 80020, 80021, - 80022, 80023, 80024, 80025, 80026, 80027, 80028, 80029, 80030, 80031, - 80032, 80033, 80034, 80035, 80036, 80037, 80038, 80039, 80040, 80041, - 80042, 80043, 80044, 80045, 80046, 80047, 80048, 80049, 80050, 80051, - 80052, 80053, 80054, 80055, 80056, 80057, 80058, 80059, 80060, 80061, - 80062, 80063, 80064, 80065, 80066, 80067, 80068, 80069, 80070, 80071, - 80072, 80073, 80074, 80075, 80076, 80077, 80078, 80079, 80080, 80081, - 80082, 80083, 80084, 80085, 80086, 80087, 80088, 80089, 80090, 80091, - 80092, 80093, 80094, 80095, 80096, 80097, 80098, 80099, 80100, 80101, - 80102, 80103, 80104, 80105, 80106, 80107, 80108, 80109, 80110, 80111, - 80112, 80113, 80114, 80115, 80116, 80117, 80118, 80119, 80120, 80121, - 80122, 80123, 80124, 80125, 80126, 80127, 80128, 80129, 80130, 80131, - 80132, 80133, 80134, 80135, 80136, 80137, 80138, 80139, 80140, 80141, - 80142, 80143, 80144, 80145, 80146, 80147, 80148, 80149, 80150, 80151, - 80152, 80153, 80154, 80155, 80156, 80157, 80158, 80159, 80160, 80161, - 80162, 80163, 80164, 80165, 80166, 80167, 80168, 80169, 80170, 80171, - 80172, 80173, 80174, 80175, 80176, 80177, 80178, 80179, 80180, 80181, - 80182, 80183, 80184, 80185, 80186, 80187, 80188, 80189, 80190, 80191, - 80192, 80193, 80194, 80195, 80196, 80197, 80198, 80199, 80200, 80201, - 80202, 80203, 80204, 80205, 80206, 80207, 80208, 80209, 80210, 80211, - 80212, 80213, 80214, 80215, 80216, 80217, 80218, 80219, 80220, 80221, - 80222, 80223, 80224, 80225, 80226, 80227, 80228, 80229, 80230, 80231, - 80232, 80233, 80234, 80235, 80236, 80237, 80238, 80239, 80240, 80241, - 80242, 80243, 80244, 80245, 80246, 80247, 80248, 80249, 80250, 80251, - 80252, 80253, 80254, 80255, 80256, 80257, 80258, 80259, 80260, 80261, - 80262, 80263, 80264, 80265, 80266, 80267, 80268, 80269, 80270, 80271, - 80272, 80273, 80274, 80275, 80276, 80277, 80278, 80279, 80280, 80281, - 80282, 80283, 80284, 80285, 80286, 80287, 80288, 80289, 80290, 80291, - 80292, 80293, 80294, 80295, 80296, 80297, 80298, 80299, 80300, 80301, - 80302, 80303, 80304, 80305, 80306, 80307, 80308, 80309, 80310, 80311, - 80312, 80313, 80314, 80315, 80316, 80317, 80318, 80319, 80320, 80321, - 80322, 80323, 80324, 80325, 80326, 80327, 80328, 80329, 80330, 80331, - 80332, 80333, 80334, 80335, 80336, 80337, 80338, 80339, 80340, 80341, - 80342, 80343, 80344, 80345, 80346, 80347, 80348, 80349, 80350, 80351, - 80352, 80353, 80354, 80355, 80356, 80357, 80358, 80359, 80360, 80361, - 80362, 80363, 80364, 80365, 80366, 80367, 80368, 80369, 80370, 80371, - 80372, 80373, 80374, 80375, 80376, 80377, 80378, 80379, 80380, 80381, - 80382, 80383, 80384, 80385, 80386, 80387, 80388, 80389, 80390, 80391, - 80392, 80393, 80394, 80395, 80396, 80397, 80398, 80399, 80400, 80401, - 80402, 80403, 80404, 80405, 80406, 80407, 80408, 80409, 80410, 80411, - 80412, 80413, 80414, 80415, 80416, 80417, 80418, 80419, 80420, 80421, - 80422, 80423, 80424, 80425, 80426, 80427, 80428, 80429, 80430, 80431, - 80432, 80433, 80434, 80435, 80436, 80437, 80438, 80439, 80440, 80441, - 80442, 80443, 80444, 80445, 80446, 80447, 80448, 80449, 80450, 80451, - 80452, 80453, 80454, 80455, 80456, 80457, 80458, 80459, 80460, 80461, - 80462, 80463, 80464, 80465, 80466, 80467, 80468, 80469, 80470, 80471, - 80472, 80473, 80474, 80475, 80476, 80477, 80478, 80479, 80480, 80481, - 80482, 80483, 80484, 80485, 80486, 80487, 80488, 80489, 80490, 80491, - 80492, 80493, 80494, 80495, 80496, 80497, 80498, 80499, 80500, 80501, - 80502, 80503, 80504, 80505, 80506, 80507, 80508, 80509, 80510, 80511, - 80512, 80513, 80514, 80515, 80516, 80517, 80518, 80519, 80520, 80521, - 80522, 80523, 80524, 80525, 80526, 80527, 80528, 80529, 80530, 80531, - 80532, 80533, 80534, 80535, 80536, 80537, 80538, 80539, 80540, 80541, - 80542, 80543, 80544, 80545, 80546, 80547, 80548, 80549, 80550, 80551, - 80552, 80553, 80554, 80555, 80556, 80557, 80558, 80559, 80560, 80561, - 80562, 80563, 80564, 80565, 80566, 80567, 80568, 80569, 80570, 80571, - 80572, 80573, 80574, 80575, 80576, 80577, 80578, 80579, 80580, 80581, - 80582, 80583, 80584, 80585, 80586, 80587, 80588, 80589, 80590, 80591, - 80592, 80593, 80594, 80595, 80596, 80597, 80598, 80599, 80600, 80601, - 80602, 80603, 80604, 80605, 80606, 80607, 80608, 80609, 80610, 80611, - 80612, 80613, 80614, 80615, 80616, 80617, 80618, 80619, 80620, 80621, - 80622, 80623, 80624, 80625, 80626, 80627, 80628, 80629, 80630, 80631, - 80632, 80633, 80634, 80635, 80636, 80637, 80638, 80639, 80640, 80641, - 80642, 80643, 80644, 80645, 80646, 80647, 80648, 80649, 80650, 80651, - 80652, 80653, 80654, 80655, 80656, 80657, 80658, 80659, 80660, 80661, - 80662, 80663, 80664, 80665, 80666, 80667, 80668, 80669, 80670, 80671, - 80672, 80673, 80674, 80675, 80676, 80677, 80678, 80679, 80680, 80681, - 80682, 80683, 80684, 80685, 80686, 80687, 80688, 80689, 80690, 80691, - 80692, 80693, 80694, 80695, 80696, 80697, 80698, 80699, 80700, 80701, - 80702, 80703, 80704, 80705, 80706, 80707, 80708, 80709, 80710, 80711, - 80712, 80713, 80714, 80715, 80716, 80717, 80718, 80719, 80720, 80721, - 80722, 80723, 80724, 80725, 80726, 80727, 80728, 80729, 80730, 80731, - 80732, 80733, 80734, 80735, 80736, 80737, 80738, 80739, 80740, 80741, - 80742, 80743, 80744, 80745, 80746, 80747, 80748, 80749, 80750, 80751, - 80752, 80753, 80754, 80755, 80756, 80757, 80758, 80759, 80760, 80761, - 80762, 80763, 80764, 80765, 80766, 80767, 80768, 80769, 80770, 80771, - 80772, 80773, 80774, 80775, 80776, 80777, 80778, 80779, 80780, 80781, - 80782, 80783, 80784, 80785, 80786, 80787, 80788, 80789, 80790, 80791, - 80792, 80793, 80794, 80795, 80796, 80797, 80798, 80799, 80800, 80801, - 80802, 80803, 80804, 80805, 80806, 80807, 80808, 80809, 80810, 80811, - 80812, 80813, 80814, 80815, 80816, 80817, 80818, 80819, 80820, 80821, - 80822, 80823, 80824, 80825, 80826, 80827, 80828, 80829, 80830, 80831, - 80832, 80833, 80834, 80835, 80836, 80837, 80838, 80839, 80840, 80841, - 80842, 80843, 80844, 80845, 80846, 80847, 80848, 80849, 80850, 80851, - 80852, 80853, 80854, 80855, 80856, 80857, 80858, 80859, 80860, 80861, - 80862, 80863, 80864, 80865, 80866, 80867, 80868, 80869, 80870, 80871, - 80872, 80873, 80874, 80875, 80876, 80877, 80878, 80879, 80880, 80881, - 80882, 80883, 80884, 80885, 80886, 80887, 80888, 80889, 80890, 80891, - 80892, 80893, 80894, 80895, 80896, 80897, 80898, 80899, 80900, 80901, - 80902, 80903, 80904, 80905, 80906, 80907, 80908, 80909, 80910, 80911, - 80912, 80913, 80914, 80915, 80916, 80917, 80918, 80919, 80920, 80921, - 80922, 80923, 80924, 80925, 80926, 80927, 80928, 80929, 80930, 80931, - 80932, 80933, 80934, 80935, 80936, 80937, 80938, 80939, 80940, 80941, - 80942, 80943, 80944, 80945, 80946, 80947, 80948, 80949, 80950, 80951, - 80952, 80953, 80954, 80955, 80956, 80957, 80958, 80959, 80960, 80961, - 80962, 80963, 80964, 80965, 80966, 80967, 80968, 80969, 80970, 80971, - 80972, 80973, 80974, 80975, 80976, 80977, 80978, 80979, 80980, 80981, - 80982, 80983, 80984, 80985, 80986, 80987, 80988, 80989, 80990, 80991, - 80992, 80993, 80994, 80995, 80996, 80997, 80998, 80999, 81000, 81001, - 81002, 81003, 81004, 81005, 81006, 81007, 81008, 81009, 81010, 81011, - 81012, 81013, 81014, 81015, 81016, 81017, 81018, 81019, 81020, 81021, - 81022, 81023, 81024, 81025, 81026, 81027, 81028, 81029, 81030, 81031, - 81032, 81033, 81034, 81035, 81036, 81037, 81038, 81039, 81040, 81041, - 81042, 81043, 81044, 81045, 81046, 81047, 81048, 81049, 81050, 81051, - 81052, 81053, 81054, 81055, 81056, 81057, 81058, 81059, 81060, 81061, - 81062, 81063, 81064, 81065, 81066, 81067, 81068, 81069, 81070, 81071, - 81072, 81073, 81074, 81075, 81076, 81077, 81078, 81079, 81080, 81081, - 81082, 81083, 81084, 81085, 81086, 81087, 81088, 81089, 81090, 81091, - 81092, 81093, 81094, 81095, 81096, 81097, 81098, 81099, 81100, 81101, - 81102, 81103, 81104, 81105, 81106, 81107, 81108, 81109, 81110, 81111, - 81112, 81113, 81114, 81115, 81116, 81117, 81118, 81119, 81120, 81121, - 81122, 81123, 81124, 81125, 81126, 81127, 81128, 81129, 81130, 81131, - 81132, 81133, 81134, 81135, 81136, 81137, 81138, 81139, 81140, 81141, - 81142, 81143, 81144, 81145, 81146, 81147, 81148, 81149, 81150, 81151, - 81152, 81153, 81154, 81155, 81156, 81157, 81158, 81159, 81160, 81161, - 81162, 81163, 81164, 81165, 81166, 81167, 81168, 81169, 81170, 81171, - 81172, 81173, 81174, 81175, 81176, 81177, 81178, 81179, 81180, 81181, - 81182, 81183, 81184, 81185, 81186, 81187, 81188, 81189, 81190, 81191, - 81192, 81193, 81194, 81195, 81196, 81197, 81198, 81199, 81200, 81201, - 81202, 81203, 81204, 81205, 81206, 81207, 81208, 81209, 81210, 81211, - 81212, 81213, 81214, 81215, 81216, 81217, 81218, 81219, 81220, 81221, - 81222, 81223, 81224, 81225, 81226, 81227, 81228, 81229, 81230, 81231, - 81232, 81233, 81234, 81235, 81236, 81237, 81238, 81239, 81240, 81241, - 81242, 81243, 81244, 81245, 81246, 81247, 81248, 81249, 81250, 81251, - 81252, 81253, 81254, 81255, 81256, 81257, 81258, 81259, 81260, 81261, - 81262, 81263, 81264, 81265, 81266, 81267, 81268, 81269, 81270, 81271, - 81272, 81273, 81274, 81275, 81276, 81277, 81278, 81279, 81280, 81281, - 81282, 81283, 81284, 81285, 81286, 81287, 81288, 81289, 81290, 81291, - 81292, 81293, 81294, 81295, 81296, 81297, 81298, 81299, 81300, 81301, - 81302, 81303, 81304, 81305, 81306, 81307, 81308, 81309, 81310, 81311, - 81312, 81313, 81314, 81315, 81316, 81317, 81318, 81319, 81320, 81321, - 81322, 81323, 81324, 81325, 81326, 81327, 81328, 81329, 81330, 81331, - 81332, 81333, 81334, 81335, 81336, 81337, 81338, 81339, 81340, 81341, - 81342, 81343, 81344, 81345, 81346, 81347, 81348, 81349, 81350, 81351, - 81352, 81353, 81354, 81355, 81356, 81357, 81358, 81359, 81360, 81361, - 81362, 81363, 81364, 81365, 81366, 81367, 81368, 81369, 81370, 81371, - 81372, 81373, 81374, 81375, 81376, 81377, 81378, 81379, 81380, 81381, - 81382, 81383, 81384, 81385, 81386, 81387, 81388, 81389, 81390, 81391, - 81392, 81393, 81394, 81395, 81396, 81397, 81398, 81399, 81400, 81401, - 81402, 81403, 81404, 81405, 81406, 81407, 81408, 81409, 81410, 81411, - 81412, 81413, 81414, 81415, 81416, 81417, 81418, 81419, 81420, 81421, - 81422, 81423, 81424, 81425, 81426, 81427, 81428, 81429, 81430, 81431, - 81432, 81433, 81434, 81435, 81436, 81437, 81438, 81439, 81440, 81441, - 81442, 81443, 81444, 81445, 81446, 81447, 81448, 81449, 81450, 81451, - 81452, 81453, 81454, 81455, 81456, 81457, 81458, 81459, 81460, 81461, - 81462, 81463, 81464, 81465, 81466, 81467, 81468, 81469, 81470, 81471, - 81472, 81473, 81474, 81475, 81476, 81477, 81478, 81479, 81480, 81481, - 81482, 81483, 81484, 81485, 81486, 81487, 81488, 81489, 81490, 81491, - 81492, 81493, 81494, 81495, 81496, 81497, 81498, 81499, 81500, 81501, - 81502, 81503, 81504, 81505, 81506, 81507, 81508, 81509, 81510, 81511, - 81512, 81513, 81514, 81515, 81516, 81517, 81518, 81519, 81520, 81521, - 81522, 81523, 81524, 81525, 81526, 81527, 81528, 81529, 81530, 81531, - 81532, 81533, 81534, 81535, 81536, 81537, 81538, 81539, 81540, 81541, - 81542, 81543, 81544, 81545, 81546, 81547, 81548, 81549, 81550, 81551, - 81552, 81553, 81554, 81555, 81556, 81557, 81558, 81559, 81560, 81561, - 81562, 81563, 81564, 81565, 81566, 81567, 81568, 81569, 81570, 81571, - 81572, 81573, 81574, 81575, 81576, 81577, 81578, 81579, 81580, 81581, - 81582, 81583, 81584, 81585, 81586, 81587, 81588, 81589, 81590, 81591, - 81592, 81593, 81594, 81595, 81596, 81597, 81598, 81599, 81600, 81601, - 81602, 81603, 81604, 81605, 81606, 81607, 81608, 81609, 81610, 81611, - 81612, 81613, 81614, 81615, 81616, 81617, 81618, 81619, 81620, 81621, - 81622, 81623, 81624, 81625, 81626, 81627, 81628, 81629, 81630, 81631, - 81632, 81633, 81634, 81635, 81636, 81637, 81638, 81639, 81640, 81641, - 81642, 81643, 81644, 81645, 81646, 81647, 81648, 81649, 81650, 81651, - 81652, 81653, 81654, 81655, 81656, 81657, 81658, 81659, 81660, 81661, - 81662, 81663, 81664, 81665, 81666, 81667, 81668, 81669, 81670, 81671, - 81672, 81673, 81674, 81675, 81676, 81677, 81678, 81679, 81680, 81681, - 81682, 81683, 81684, 81685, 81686, 81687, 81688, 81689, 81690, 81691, - 81692, 81693, 81694, 81695, 81696, 81697, 81698, 81699, 81700, 81701, - 81702, 81703, 81704, 81705, 81706, 81707, 81708, 81709, 81710, 81711, - 81712, 81713, 81714, 81715, 81716, 81717, 81718, 81719, 81720, 81721, - 81722, 81723, 81724, 81725, 81726, 81727, 81728, 81729, 81730, 81731, - 81732, 81733, 81734, 81735, 81736, 81737, 81738, 81739, 81740, 81741, - 81742, 81743, 81744, 81745, 81746, 81747, 81748, 81749, 81750, 81751, - 81752, 81753, 81754, 81755, 81756, 81757, 81758, 81759, 81760, 81761, - 81762, 81763, 81764, 81765, 81766, 81767, 81768, 81769, 81770, 81771, - 81772, 81773, 81774, 81775, 81776, 81777, 81778, 81779, 81780, 81781, - 81782, 81783, 81784, 81785, 81786, 81787, 81788, 81789, 81790, 81791, - 81792, 81793, 81794, 81795, 81796, 81797, 81798, 81799, 81800, 81801, - 81802, 81803, 81804, 81805, 81806, 81807, 81808, 81809, 81810, 81811, - 81812, 81813, 81814, 81815, 81816, 81817, 81818, 81819, 81820, 81821, - 81822, 81823, 81824, 81825, 81826, 81827, 81828, 81829, 81830, 81831, - 81832, 81833, 81834, 81835, 81836, 81837, 81838, 81839, 81840, 81841, - 81842, 81843, 81844, 81845, 81846, 81847, 81848, 81849, 81850, 81851, - 81852, 81853, 81854, 81855, 81856, 81857, 81858, 81859, 81860, 81861, - 81862, 81863, 81864, 81865, 81866, 81867, 81868, 81869, 81870, 81871, - 81872, 81873, 81874, 81875, 81876, 81877, 81878, 81879, 81880, 81881, - 81882, 81883, 81884, 81885, 81886, 81887, 81888, 81889, 81890, 81891, - 81892, 81893, 81894, 81895, 81896, 81897, 81898, 81899, 81900, 81901, - 81902, 81903, 81904, 81905, 81906, 81907, 81908, 81909, 81910, 81911, - 81912, 81913, 81914, 81915, 81916, 81917, 81918, 81919, 82928, 82929, - 82930, 82931, 82932, 82933, 82934, 82935, 82936, 82937, 82938, 82688, - 82689, 82690, 82691, 82692, 82693, 82694, 82695, 82696, 82697, 82698, - 82699, 82700, 82701, 82702, 82703, 82704, 82705, 82706, 82707, 82708, - 82709, 82710, 82711, 82712, 82713, 82714, 82715, 82716, 82717, 82718, - 82719, 82720, 82721, 82722, 82723, 82724, 82725, 82726, 82727, 82728, - 82729, 82730, 82731, 82732, 82733, 82734, 82735, 82736, 82737, 82738, - 82739, 82740, 82741, 82742, 82743, 82744, 82745, 82746, 82747, 82748, - 82749, 82750, 82751, 82752, 82753, 82754, 82755, 82756, 82757, 82758, - 82759, 82760, 82761, 82762, 82763, 82764, 82765, 82766, 82767, 82768, - 82769, 82770, 82771, 82772, 82773, 82774, 82775, 82776, 82777, 82778, - 82779, 82780, 82781, 82782, 82783, 82784, 82785, 82786, 82787, 82788, - 82789, 82790, 82791, 82792, 82793, 82794, 82795, 82796, 82797, 82798, - 82799, 82800, 82801, 82802, 82803, 82804, 82805, 82806, 82807, 82808, - 82809, 82810, 82811, 82812, 82813, 82814, 82815, 82816, 82817, 82818, - 82819, 82820, 82821, 82822, 82823, 82824, 82825, 82826, 82827, 82828, - 82829, 82830, 82831, 82832, 82833, 82834, 82835, 82836, 82837, 82838, - 82839, 82840, 82841, 82842, 82843, 82844, 82845, 82846, 82847, 82848, - 82849, 82850, 82851, 82852, 82853, 82854, 82855, 82856, 82857, 82858, - 82859, 82860, 82861, 82862, 82863, 82864, 82865, 82866, 82867, 82868, - 82869, 82870, 82871, 82872, 82873, 82874, 82875, 82876, 82877, 82878, - 82879, 82880, 82881, 82882, 82883, 82884, 82885, 82886, 82887, 82888, - 82889, 82890, 82891, 82892, 82893, 82894, 82895, 82896, 82897, 82898, - 82899, 82900, 82901, 82902, 82903, 82904, 82905, 82906, 82907, 82908, - 82909, 82910, 82911, 82912, 82913, 82914, 82915, 82916, 82917, 82918, - 82919, 82920, 82921, 82922, 82923, 82924, 82925, 82926, 82927, 81920, - 81921, 81922, 81923, 81924, 81925, 81926, 81927, 81928, 81929, 81930, - 81931, 81932, 81933, 81934, 81935, 81936, 81937, 81938, 81939, 81940, - 81941, 81942, 81943, 81944, 81945, 81946, 81947, 81948, 81949, 81950, - 81951, 81952, 81953, 81954, 81955, 81956, 81957, 81958, 81959, 81960, - 81961, 81962, 81963, 81964, 81965, 81966, 81967, 81968, 81969, 81970, - 81971, 81972, 81973, 81974, 81975, 81976, 81977, 81978, 81979, 81980, - 81981, 81982, 81983, 81984, 81985, 81986, 81987, 81988, 81989, 81990, - 81991, 81992, 81993, 81994, 81995, 81996, 81997, 81998, 81999, 82000, - 82001, 82002, 82003, 82004, 82005, 82006, 82007, 82008, 82009, 82010, - 82011, 82012, 82013, 82014, 82015, 82016, 82017, 82018, 82019, 82020, - 82021, 82022, 82023, 82024, 82025, 82026, 82027, 82028, 82029, 82030, - 82031, 82032, 82033, 82034, 82035, 82036, 82037, 82038, 82039, 82040, - 82041, 82042, 82043, 82044, 82045, 82046, 82047, 82048, 82049, 82050, - 82051, 82052, 82053, 82054, 82055, 82056, 82057, 82058, 82059, 82060, - 82061, 82062, 82063, 82064, 82065, 82066, 82067, 82068, 82069, 82070, - 82071, 82072, 82073, 82074, 82075, 82076, 82077, 82078, 82079, 82080, - 82081, 82082, 82083, 82084, 82085, 82086, 82087, 82088, 82089, 82090, - 82091, 82092, 82093, 82094, 82095, 82096, 82097, 82098, 82099, 82100, - 82101, 82102, 82103, 82104, 82105, 82106, 82107, 82108, 82109, 82110, - 82111, 82112, 82113, 82114, 82115, 82116, 82117, 82118, 82119, 82120, - 82121, 82122, 82123, 82124, 82125, 82126, 82127, 82128, 82129, 82130, - 82131, 82132, 82133, 82134, 82135, 82136, 82137, 82138, 82139, 82140, - 82141, 82142, 82143, 82144, 82145, 82146, 82147, 82148, 82149, 82150, - 82151, 82152, 82153, 82154, 82155, 82156, 82157, 82158, 82159, 82160, - 82161, 82162, 82163, 82164, 82165, 82166, 82167, 82168, 82169, 82170, - 82171, 82172, 82173, 82174, 82175, 82176, 82177, 82178, 82179, 82180, - 82181, 82182, 82183, 82184, 82185, 82186, 82187, 82188, 82189, 82190, - 82191, 82192, 82193, 82194, 82195, 82196, 82197, 82198, 82199, 82200, - 82201, 82202, 82203, 82204, 82205, 82206, 82207, 82208, 82209, 82210, - 82211, 82212, 82213, 82214, 82215, 82216, 82217, 82218, 82219, 82220, - 82221, 82222, 82223, 82224, 82225, 82226, 82227, 82228, 82229, 82230, - 82231, 82232, 82233, 82234, 82235, 82236, 82237, 82238, 82239, 82240, - 82241, 82242, 82243, 82244, 82245, 82246, 82247, 82248, 82249, 82250, - 82251, 82252, 82253, 82254, 82255, 82256, 82257, 82258, 82259, 82260, - 82261, 82262, 82263, 82264, 82265, 82266, 82267, 82268, 82269, 82270, - 82271, 82272, 82273, 82274, 82275, 82276, 82277, 82278, 82279, 82280, - 82281, 82282, 82283, 82284, 82285, 82286, 82287, 82288, 82289, 82290, - 82291, 82292, 82293, 82294, 82295, 82296, 82297, 82298, 82299, 82300, - 82301, 82302, 82303, 82304, 82305, 82306, 82307, 82308, 82309, 82310, - 82311, 82312, 82313, 82314, 82315, 82316, 82317, 82318, 82319, 82320, - 82321, 82322, 82323, 82324, 82325, 82326, 82327, 82328, 82329, 82330, - 82331, 82332, 82333, 82334, 82335, 82336, 82337, 82338, 82339, 82340, - 82341, 82342, 82343, 82344, 82345, 82346, 82347, 82348, 82349, 82350, - 82351, 82352, 82353, 82354, 82355, 82356, 82357, 82358, 82359, 82360, - 82361, 82362, 82363, 82364, 82365, 82366, 82367, 82368, 82369, 82370, - 82371, 82372, 82373, 82374, 82375, 82376, 82377, 82378, 82379, 82380, - 82381, 82382, 82383, 82384, 82385, 82386, 82387, 82388, 82389, 82390, - 82391, 82392, 82393, 82394, 82395, 82396, 82397, 82398, 82399, 82400, - 82401, 82402, 82403, 82404, 82405, 82406, 82407, 82408, 82409, 82410, - 82411, 82412, 82413, 82414, 82415, 82416, 82417, 82418, 82419, 82420, - 82421, 82422, 82423, 82424, 82425, 82426, 82427, 82428, 82429, 82430, - 82431, 82432, 82433, 82434, 82435, 82436, 82437, 82438, 82439, 82440, - 82441, 82442, 82443, 82444, 82445, 82446, 82447, 82448, 82449, 82450, - 82451, 82452, 82453, 82454, 82455, 82456, 82457, 82458, 82459, 82460, - 82461, 82462, 82463, 82464, 82465, 82466, 82467, 82468, 82469, 82470, - 82471, 82472, 82473, 82474, 82475, 82476, 82477, 82478, 82479, 82480, - 82481, 82482, 82483, 82484, 82485, 82486, 82487, 82488, 82489, 82490, - 82491, 82492, 82493, 82494, 82495, 82496, 82497, 82498, 82499, 82500, - 82501, 82502, 82503, 82504, 82505, 82506, 82507, 82508, 82509, 82510, - 82511, 82512, 82513, 82514, 82515, 82516, 82517, 82518, 82519, 82520, - 82521, 82522, 82523, 82524, 82525, 82526, 82527, 82528, 82529, 82530, - 82531, 82532, 82533, 82534, 82535, 82536, 82537, 82538, 82539, 82540, - 82541, 82542, 82543, 82544, 82545, 82546, 82547, 82548, 82549, 82550, - 82551, 82552, 82553, 82554, 82555, 82556, 82557, 82558, 82559, 82560, - 82561, 82562, 82563, 82564, 82565, 82566, 82567, 82568, 82569, 82570, - 82571, 82572, 82573, 82574, 82575, 82576, 82577, 82578, 82579, 82580, - 82581, 82582, 82583, 82584, 82585, 82586, 82587, 82588, 82589, 82590, - 82591, 82592, 82593, 82594, 82595, 82596, 82597, 82598, 82599, 82600, - 82601, 82602, 82603, 82604, 82605, 82606, 82607, 82608, 82609, 82610, - 82611, 82612, 82613, 82614, 82615, 82616, 82617, 82618, 82619, 82620, - 82621, 82622, 82623, 82624, 82625, 82626, 82627, 82628, 82629, 82630, - 82631, 82632, 82633, 82634, 82635, 82636, 82637, 82638, 82639, 82640, - 82641, 82642, 82643, 82644, 82645, 82646, 82647, 82648, 82649, 82650, - 82651, 82652, 82653, 82654, 82655, 82656, 82657, 82658, 82659, 82660, - 82661, 82662, 82663, 82664, 82665, 82666, 82667, 82668, 82669, 82670, - 82671, 82672, 82673, 82674, 82675, 82676, 82677, 82678, 82679, 82680, - 82681, 82682, 82683, 82684, 82685, 82686, 82687, 129370, 10037, 10039, - 10036, 10049, 117865, 117866, 10058, 10035, 9834, 66854, 66853, 66827, - 66826, 66833, 66832, 66821, 66837, 66836, 66835, 66842, 66841, 66824, - 66823, 66819, 66818, 66822, 66820, 66855, 66831, 66844, 66843, 66846, - 66845, 66852, 66851, 66817, 66825, 66828, 66830, 66834, 66839, 66840, - 66848, 66849, 66816, 66829, 66838, 66847, 66850, 128268, 128294, 128161, - 8961, 9191, 8712, 8946, 8953, 8947, 8952, 8950, 8949, 10969, 10194, - 128024, 128727, 69608, 69621, 69603, 69611, 69618, 69619, 69600, 69615, - 69602, 69606, 69614, 69617, 69620, 69609, 69604, 69607, 69610, 69601, - 69613, 69605, 69616, 69612, 69622, 129501, 983101, 129456, 129457, - 129459, 129458, 127995, 127996, 127997, 127998, 127999, 128453, 128454, - 128455, 129721, 128460, 128461, 8709, 10675, 10676, 10674, 10673, 128459, - 9091, 8193, 8212, 8195, 8192, 8211, 8194, 983179, 8718, 983178, 983135, - 983099, 983048, 983095, 983046, 983064, 128282, 983051, 983050, 9993, - 128388, 128233, 9094, 983067, 983100, 983049, 8926, 8927, 8925, 8924, - 8797, 8917, 61, 10865, 10867, 11072, 10609, 10723, 10724, 10926, 11257, - 10871, 10854, 10862, 8789, 10872, 8781, 8794, 10738, 10736, 10734, 10739, - 10737, 10735, 11249, 11248, 9003, 8998, 983105, 983104, 8494, 8793, - 983136, 4957, 4959, 4958, 4963, 4965, 4978, 4988, 4980, 4979, 4987, 4985, - 4982, 4981, 4986, 4984, 4983, 4968, 4966, 4964, 4960, 4999, 4998, 4711, - 4997, 43816, 43819, 43821, 43820, 43818, 43822, 43817, 4704, 4707, 4710, - 11653, 4709, 4708, 4706, 4705, 43808, 43811, 43813, 43812, 43810, 43814, - 43809, 11704, 11707, 11709, 11708, 11706, 11710, 11705, 11688, 11691, - 11693, 11692, 11690, 11694, 11689, 4904, 4907, 4910, 11664, 4909, 4908, - 4911, 4906, 4905, 4728, 4731, 4734, 11655, 4733, 4732, 4735, 4730, 4729, - 43789, 43788, 43787, 43786, 43790, 43785, 4856, 4859, 4862, 11661, 4861, - 4860, 4863, 4858, 4857, 43797, 43796, 43795, 43794, 43798, 43793, 4848, - 4851, 4854, 11660, 4853, 4852, 4855, 4850, 4849, 5003, 5002, 4943, 5001, - 4936, 4939, 4941, 4940, 4954, 4938, 4942, 4937, 4873, 124916, 124915, - 124924, 124923, 124910, 124909, 124926, 124925, 124922, 124921, 124920, - 124919, 124918, 124917, 124914, 124913, 124912, 124904, 11667, 4895, - 11670, 11669, 11668, 4888, 4891, 4893, 4892, 4890, 4894, 4889, 4768, - 4771, 4774, 11658, 4773, 4772, 4775, 4770, 4769, 4880, 4883, 4885, 4884, - 4882, 11736, 11739, 11741, 11740, 11738, 11742, 11737, 4872, 4875, 4878, - 4879, 4877, 4876, 4874, 124907, 124906, 4631, 124905, 124896, 124899, - 124901, 124900, 124898, 124902, 124897, 4624, 4627, 4629, 4628, 4626, - 4630, 4625, 4608, 4611, 4614, 4615, 4613, 4612, 4610, 4609, 4800, 4803, - 4805, 4804, 4802, 4792, 4795, 4797, 4796, 4794, 4798, 4793, 4784, 4787, - 4789, 4788, 4786, 11720, 11723, 11725, 11724, 11722, 11726, 11721, 4776, - 4779, 4782, 4783, 4781, 4780, 4778, 4777, 4995, 4994, 4639, 4993, 4632, - 4635, 4638, 11649, 4637, 4636, 4953, 4634, 4633, 4760, 4763, 4766, 11657, - 4765, 4764, 4767, 4762, 4761, 4752, 4755, 4758, 11656, 4757, 4756, 4759, - 4754, 4753, 4912, 4816, 4819, 4821, 4820, 4818, 4822, 4817, 4915, 4918, - 11665, 4917, 4916, 4919, 4914, 4913, 5007, 5006, 4951, 5005, 4944, 4947, - 4950, 11666, 4949, 4948, 4946, 4945, 4696, 4699, 4701, 4700, 4698, 4688, - 4691, 4693, 4692, 4690, 4694, 4689, 4680, 4683, 4685, 4684, 4682, 11712, - 11715, 11717, 11716, 11714, 11718, 11713, 4672, 4675, 4678, 4679, 4677, - 4676, 4674, 4673, 4648, 4651, 4654, 11650, 4653, 4652, 4655, 4952, 4650, - 4649, 4661, 4996, 5000, 4992, 5004, 4660, 4664, 4667, 4670, 11652, 4669, - 4668, 4671, 4666, 4665, 4640, 4643, 4645, 4644, 4647, 4642, 4646, 4641, - 11680, 11683, 11685, 11684, 11682, 11686, 11681, 4656, 4659, 4662, 11651, - 4663, 4658, 4657, 4896, 4899, 4902, 11663, 4901, 4900, 4903, 4898, 4897, - 43781, 43780, 43779, 43778, 43782, 43777, 4928, 4931, 4934, 4935, 4933, - 4932, 4930, 4929, 4920, 4923, 4925, 4924, 4927, 4922, 4926, 4921, 4720, - 4723, 4726, 11654, 4725, 4724, 4727, 4722, 4721, 4864, 4867, 4870, 11662, - 4869, 4868, 4871, 4866, 4865, 4616, 4619, 4622, 11648, 4621, 4620, 4623, - 4618, 4617, 4808, 4811, 4814, 4815, 4813, 4812, 4810, 4809, 4840, 4843, - 4846, 4847, 4845, 4844, 4842, 4841, 4744, 4747, 4749, 4748, 4746, 11728, - 11731, 11733, 11732, 11730, 11734, 11729, 4736, 4739, 4742, 4743, 4741, - 4740, 4738, 4737, 4832, 4835, 4837, 4836, 4839, 4834, 4838, 4833, 11696, - 11699, 11701, 11700, 11698, 11702, 11697, 4824, 4827, 4830, 11659, 4829, - 4828, 4831, 4826, 4825, 4712, 4715, 4717, 4716, 4719, 4714, 4718, 4713, - 5009, 5016, 5012, 5015, 5013, 5017, 5010, 5011, 5014, 5008, 4973, 4972, - 4975, 4974, 4971, 4970, 4977, 4969, 4976, 4962, 4967, 4961, 983096, - 983047, 127984, 127972, 8352, 8364, 8455, 8265, 33, 8761, 118269, 118270, + 127919, 128549, 128542, 9933, 9090, 129770, 129400, 72004, 72021, 72020, + 72023, 72022, 72019, 72018, 72016, 72025, 72017, 72024, 71964, 71958, + 71963, 71974, 71973, 71936, 71937, 71961, 71960, 71966, 71965, 71940, + 71941, 71938, 71939, 71982, 71976, 71952, 71962, 71957, 71967, 71978, + 71979, 71980, 71971, 71970, 71954, 71953, 71951, 71950, 71949, 71948, + 71969, 71968, 71981, 71955, 71972, 71975, 71977, 71983, 71942, 71945, + 71997, 71996, 72003, 71995, 71984, 71991, 71987, 71988, 71985, 71986, + 71989, 71992, 71998, 72002, 72000, 72005, 72006, 71999, 72001, 8725, 247, + 8903, 129343, 8739, 9902, 129684, 128171, 128565, 12291, 8783, 9009, + 128462, 128441, 128442, 128443, 8939, 8941, 8716, 8740, 10990, 8832, + 8928, 8876, 8833, 8929, 8878, 128021, 71680, 71681, 71687, 71689, 71703, + 71702, 71708, 71707, 71723, 71716, 71701, 71700, 71706, 71705, 71684, + 71685, 71682, 71683, 71694, 71704, 71699, 71709, 71719, 71720, 71721, + 71713, 71712, 71696, 71695, 71693, 71692, 71698, 71697, 71691, 71690, + 71711, 71710, 71722, 71717, 71714, 71718, 71715, 71686, 71688, 71729, + 71730, 71724, 71732, 71734, 71727, 71728, 71725, 71726, 71731, 71733, + 71739, 71738, 71735, 71737, 71736, 128054, 128044, 36, 127025, 127026, + 127027, 127028, 127029, 127030, 127031, 127032, 127033, 127034, 127035, + 127036, 127037, 127038, 127039, 127040, 127041, 127042, 127043, 127044, + 127045, 127046, 127047, 127048, 127049, 127050, 127051, 127052, 127053, + 127054, 127055, 127056, 127057, 127058, 127059, 127060, 127061, 127062, + 127063, 127064, 127065, 127066, 127067, 127068, 127069, 127070, 127071, + 127072, 127073, 127024, 127075, 127076, 127077, 127078, 127079, 127080, + 127081, 127082, 127083, 127084, 127085, 127086, 127087, 127088, 127089, + 127090, 127091, 127092, 127093, 127094, 127095, 127096, 127097, 127098, + 127099, 127100, 127101, 127102, 127103, 127104, 127105, 127106, 127107, + 127108, 127109, 127110, 127111, 127112, 127113, 127114, 127115, 127116, + 127117, 127118, 127119, 127120, 127121, 127122, 127123, 127074, 129743, + 8363, 8724, 8760, 8901, 729, 9676, 8284, 11850, 11034, 129765, 11795, + 11784, 10649, 11798, 9470, 9465, 9464, 9467, 9466, 9463, 9462, 9469, + 9461, 9468, 10175, 10868, 10986, 8225, 8223, 11840, 8914, 8748, 11842, + 8222, 8215, 10835, 10836, 10645, 10913, 10915, 10914, 10939, 8243, 12318, + 10746, 10830, 10831, 11849, 10988, 11844, 10940, 8913, 8912, 11005, + 10987, 8915, 9208, 10981, 8875, 10979, 8214, 11799, 10646, 733, 8252, + 8263, 65100, 10908, 10907, 11002, 11001, 10906, 10905, 8510, 8473, 8511, + 8450, 8461, 8469, 8474, 8477, 8484, 8518, 8519, 8520, 8521, 8517, 8509, + 8508, 8512, 10719, 9890, 9891, 11260, 127849, 8868, 10993, 10623, 8945, + 8964, 117764, 128317, 117859, 118264, 117883, 118268, 117849, 128315, + 117914, 117864, 10728, 10729, 117875, 117879, 129288, 129290, 129289, + 129291, 8595, 8629, 8671, 129035, 129031, 129179, 129043, 129027, 129047, + 8626, 8627, 10504, 8693, 129975, 8615, 10515, 11796, 11015, 129203, + 11147, 11107, 11139, 11123, 129067, 11168, 11169, 129063, 129059, 129075, + 129071, 11133, 11085, 11117, 11143, 129171, 10507, 8609, 11247, 8681, + 129175, 8623, 8659, 8675, 129079, 10597, 10607, 10593, 10585, 8643, + 10589, 10581, 8642, 129091, 129095, 129087, 10225, 128623, 129107, + 129083, 8650, 128687, 128330, 128682, 129444, 8367, 10139, 128009, + 128050, 128042, 129656, 128167, 129316, 129345, 9946, 128087, 113784, + 113788, 113785, 113783, 113782, 113786, 113787, 113795, 113798, 113793, + 113800, 113781, 113794, 113792, 113799, 113797, 113796, 113776, 113808, + 113817, 113811, 113814, 113809, 113816, 113779, 113810, 113815, 113813, + 113812, 113778, 113777, 113780, 113729, 113733, 113672, 113677, 113683, + 113735, 113739, 113746, 113668, 113678, 113674, 113726, 113691, 113699, + 113700, 113695, 113709, 113712, 113713, 113705, 113711, 113669, 113725, + 113679, 113684, 113670, 113743, 113749, 113687, 113689, 113693, 113707, + 113697, 113703, 113690, 113694, 113708, 113698, 113704, 113764, 113763, + 113762, 113761, 113732, 113753, 113731, 113755, 113754, 113666, 113766, + 113765, 113676, 113675, 113741, 113750, 113680, 113688, 113692, 113696, + 113710, 113727, 113728, 113716, 113717, 113714, 113715, 113701, 113702, + 113724, 113723, 113706, 113742, 113740, 113767, 113769, 113730, 113768, + 113682, 113685, 113752, 113737, 113667, 113719, 113718, 113681, 113745, + 113748, 113751, 113738, 113673, 113770, 113720, 113757, 113760, 113722, + 113759, 113756, 113721, 113758, 113665, 113747, 113664, 113686, 113734, + 113736, 113744, 113671, 113821, 113822, 113823, 113820, 129375, 129414, + 129516, 128192, 983082, 128066, 127805, 127806, 129467, 9793, 127758, + 127759, 127757, 9178, 9841, 129413, 11790, 77830, 77831, 77832, 77828, + 77829, 77824, 77825, 77826, 77827, 77833, 77834, 77835, 77840, 77841, + 77844, 77845, 77836, 77837, 77838, 77839, 77842, 77843, 77846, 77847, + 77869, 77870, 77872, 77873, 77874, 77875, 77877, 77878, 77871, 77876, + 77879, 77880, 77881, 77882, 77860, 77861, 77858, 77859, 77862, 77863, + 77864, 77865, 77866, 77867, 77868, 77903, 77848, 77849, 77850, 77851, + 77852, 77853, 77854, 77855, 77856, 77857, 77883, 77884, 77885, 77886, + 77887, 77888, 77889, 77890, 77891, 77892, 77893, 77894, 77895, 77896, + 77897, 77898, 77899, 77900, 77901, 77902, 78867, 78868, 78869, 78861, + 78862, 78863, 78864, 78865, 78866, 78870, 78871, 78892, 78893, 78894, + 78872, 78873, 78874, 78875, 78876, 78877, 78878, 78879, 78880, 78881, + 78882, 78883, 78884, 78885, 78886, 78887, 78888, 78889, 78890, 78891, + 78910, 78908, 78903, 77908, 77909, 77904, 77905, 77906, 77907, 77910, + 77911, 77912, 77913, 77915, 77916, 77917, 77918, 77914, 77919, 77920, + 77921, 77922, 77923, 77924, 77925, 77926, 77927, 77928, 77929, 77930, + 77931, 77932, 77933, 77934, 77935, 77936, 77937, 77938, 77939, 77940, + 77941, 77949, 77950, 77942, 77943, 77944, 77945, 77946, 77947, 77948, + 77951, 77974, 77975, 77978, 77979, 77973, 77976, 77977, 77980, 77981, + 77982, 77983, 77984, 77991, 77992, 77994, 77995, 77985, 77986, 77987, + 77988, 77989, 77990, 77993, 77996, 77997, 77998, 77999, 78000, 78001, + 78002, 78003, 78004, 78005, 78006, 78008, 78009, 78011, 78012, 78007, + 78010, 78013, 78014, 78015, 78016, 78017, 78025, 78026, 78027, 78028, + 78029, 78030, 78031, 78032, 78033, 78018, 78019, 78020, 78021, 78022, + 78023, 78024, 77969, 77970, 77962, 77963, 77964, 77965, 77966, 77967, + 77968, 77971, 77972, 77952, 77953, 77954, 77955, 77956, 77957, 77958, + 77959, 77960, 77961, 78041, 78042, 78043, 78044, 78034, 78035, 78036, + 78037, 78038, 78039, 78040, 78057, 78058, 78066, 78067, 78059, 78060, + 78061, 78062, 78063, 78064, 78065, 78068, 78073, 78074, 78069, 78070, + 78071, 78072, 78075, 78076, 78077, 78051, 78052, 78053, 78054, 78045, + 78046, 78047, 78048, 78049, 78050, 78055, 78056, 78911, 78909, 78904, + 78078, 78079, 78080, 78081, 78082, 78083, 78084, 78085, 78086, 78087, + 78091, 78092, 78088, 78089, 78090, 78093, 78094, 78095, 78096, 78097, + 78098, 78111, 78112, 78118, 78119, 78120, 78121, 78110, 78113, 78114, + 78115, 78116, 78117, 78122, 78128, 78129, 78130, 78131, 78132, 78133, + 78123, 78124, 78125, 78126, 78127, 78134, 78135, 78137, 78138, 78139, + 78140, 78136, 78141, 78142, 78100, 78101, 78099, 78102, 78103, 78104, + 78105, 78106, 78107, 78108, 78109, 78913, 78150, 78151, 78152, 78148, + 78149, 78143, 78144, 78145, 78146, 78147, 78153, 78154, 78156, 78157, + 78155, 78158, 78159, 78160, 78161, 78162, 78163, 78164, 78165, 78184, + 78185, 78186, 78187, 78178, 78179, 78180, 78181, 78182, 78183, 78188, + 78189, 78193, 78194, 78196, 78197, 78190, 78191, 78192, 78195, 78198, + 78199, 78200, 78201, 78166, 78167, 78173, 78174, 78168, 78169, 78170, + 78171, 78172, 78175, 78176, 78177, 78202, 78203, 78204, 78205, 78206, + 78212, 78213, 78207, 78208, 78209, 78210, 78211, 78214, 78215, 78914, + 78916, 78897, 78220, 78221, 78225, 78226, 78216, 78217, 78218, 78219, + 78222, 78223, 78224, 78227, 78228, 78229, 78230, 78231, 78232, 78233, + 78234, 78907, 78901, 78899, 78906, 78900, 78898, 78905, 78244, 78245, + 78249, 78250, 78243, 78246, 78247, 78248, 78251, 78252, 78915, 78253, + 78254, 78255, 78257, 78258, 78256, 78259, 78260, 78261, 78262, 78263, + 78264, 78268, 78269, 78270, 78271, 78272, 78273, 78274, 78275, 78276, + 78265, 78266, 78279, 78280, 78281, 78282, 78283, 78284, 78267, 78277, + 78278, 78285, 78286, 78289, 78290, 78292, 78293, 78297, 78298, 78287, + 78288, 78291, 78294, 78295, 78296, 78299, 78304, 78305, 78306, 78301, + 78302, 78300, 78303, 78307, 78308, 78309, 78310, 78311, 78312, 78313, + 78314, 78315, 78316, 78317, 78318, 78933, 78928, 78920, 78924, 78932, + 78926, 78921, 78929, 78925, 78923, 78931, 78919, 78927, 78922, 78930, + 78912, 78336, 78337, 78338, 78328, 78329, 78330, 78331, 78332, 78333, + 78334, 78335, 78339, 78354, 78355, 78356, 78357, 78358, 78359, 78361, + 78362, 78351, 78352, 78353, 78360, 78363, 78364, 78345, 78346, 78340, + 78341, 78342, 78343, 78344, 78347, 78348, 78349, 78350, 78365, 78366, + 78367, 78319, 78320, 78321, 78322, 78323, 78324, 78325, 78326, 78327, + 78372, 78373, 78368, 78369, 78370, 78371, 78374, 78375, 78376, 78377, + 78385, 78386, 78378, 78379, 78380, 78381, 78382, 78383, 78384, 78387, + 78388, 78389, 78399, 78400, 78401, 78402, 78409, 78410, 78403, 78404, + 78405, 78406, 78407, 78408, 78411, 78414, 78415, 78412, 78413, 78390, + 78391, 78392, 78393, 78394, 78395, 78396, 78397, 78398, 78423, 78424, + 78425, 78426, 78427, 78428, 78429, 78416, 78417, 78421, 78422, 78418, + 78419, 78420, 78430, 78431, 78432, 78433, 78434, 78435, 78436, 78445, + 78446, 78437, 78438, 78439, 78440, 78441, 78442, 78443, 78444, 78447, + 78448, 78452, 78453, 78454, 78455, 78459, 78460, 78449, 78450, 78451, + 78456, 78457, 78458, 78469, 78470, 78471, 78472, 78473, 78461, 78462, + 78465, 78466, 78463, 78464, 78467, 78468, 78474, 78475, 78476, 78487, + 78488, 78489, 78490, 78477, 78478, 78479, 78480, 78481, 78482, 78483, + 78484, 78485, 78486, 78902, 78491, 78492, 78494, 78495, 78493, 78496, + 78497, 78498, 78499, 78500, 78501, 78502, 78503, 78514, 78515, 78516, + 78512, 78513, 78511, 78517, 78518, 78519, 78520, 78521, 78522, 78523, + 78524, 78530, 78531, 78525, 78526, 78527, 78528, 78529, 78532, 78533, + 78534, 78535, 78536, 78537, 78538, 78539, 78540, 78541, 78542, 78543, + 78544, 78546, 78547, 78551, 78552, 78545, 78548, 78549, 78550, 78553, + 78554, 78555, 78560, 78561, 78562, 78565, 78566, 78556, 78557, 78558, + 78559, 78563, 78564, 78567, 78568, 78575, 78576, 78577, 78569, 78570, + 78571, 78572, 78573, 78574, 78578, 78579, 78580, 78586, 78587, 78581, + 78582, 78583, 78584, 78585, 78588, 78589, 78590, 78591, 78592, 78593, + 78594, 78595, 78596, 78597, 78598, 78601, 78602, 78606, 78607, 78608, + 78609, 78610, 78611, 78599, 78600, 78603, 78604, 78605, 78613, 78614, + 78619, 78620, 78612, 78615, 78616, 78617, 78618, 78621, 78622, 78623, + 78636, 78637, 78638, 78639, 78634, 78635, 78640, 78641, 78642, 78624, + 78625, 78626, 78627, 78628, 78629, 78630, 78631, 78632, 78633, 78917, + 78648, 78649, 78650, 78643, 78644, 78645, 78646, 78647, 78651, 78652, + 78653, 78667, 78668, 78674, 78675, 78664, 78665, 78666, 78669, 78670, + 78671, 78672, 78673, 78678, 78679, 78676, 78677, 78680, 78681, 78682, + 78683, 78684, 78685, 78686, 78687, 78688, 78689, 78654, 78655, 78656, + 78657, 78658, 78659, 78660, 78661, 78662, 78663, 78706, 78707, 78708, + 78690, 78691, 78692, 78693, 78694, 78695, 78696, 78697, 78698, 78699, + 78700, 78701, 78702, 78703, 78704, 78705, 78709, 78710, 78712, 78713, + 78714, 78715, 78895, 78716, 78717, 78718, 78711, 78719, 78720, 78721, + 78722, 78723, 78724, 78725, 78726, 78727, 78728, 78729, 78730, 78731, + 78732, 78733, 78734, 78735, 78736, 78737, 78738, 78741, 78742, 78747, + 78748, 78749, 78750, 78739, 78740, 78743, 78744, 78745, 78746, 78751, + 78752, 78753, 78754, 78756, 78757, 78761, 78762, 78755, 78758, 78759, + 78760, 78763, 78764, 78765, 78766, 78896, 78769, 78770, 78776, 78777, + 78767, 78768, 78771, 78772, 78773, 78774, 78775, 78778, 78779, 78783, + 78784, 78787, 78788, 78789, 78790, 78780, 78781, 78782, 78785, 78786, + 78791, 78796, 78797, 78792, 78793, 78794, 78795, 78798, 78918, 78802, + 78803, 78804, 78806, 78807, 78809, 78810, 78799, 78800, 78801, 78805, + 78808, 78811, 78812, 78813, 78814, 78815, 78816, 78817, 78818, 78819, + 78821, 78822, 78823, 78824, 78825, 78826, 78827, 78828, 78829, 78830, + 78831, 78832, 78820, 78833, 78834, 78835, 78836, 78842, 78843, 78844, + 78845, 78846, 78847, 78848, 78849, 78850, 78851, 78852, 78853, 78854, + 78855, 78856, 78857, 78858, 78859, 78860, 78837, 78838, 78839, 78840, + 78841, 78235, 78236, 78237, 78238, 78239, 78240, 78241, 78242, 78504, + 78505, 78506, 78507, 78508, 78509, 78510, 78944, 78945, 78946, 78947, + 78948, 78949, 78950, 78951, 78952, 78953, 78954, 78955, 78956, 78957, + 78958, 78959, 78960, 78961, 78962, 78963, 78964, 78965, 78966, 78967, + 78968, 78969, 78970, 78971, 78972, 78973, 78974, 78975, 78976, 78977, + 78978, 78979, 78980, 78981, 78982, 78983, 78984, 78985, 78986, 78987, + 78988, 78989, 78990, 78991, 78992, 78993, 78994, 78995, 78996, 78997, + 78998, 78999, 79000, 79001, 79002, 79003, 79004, 79005, 79006, 79007, + 79008, 79009, 79010, 79011, 79012, 79013, 79014, 79015, 79016, 79017, + 79018, 79019, 79020, 79021, 79022, 79023, 79024, 79025, 79026, 79027, + 79028, 79029, 79030, 79031, 79032, 79033, 79034, 79035, 79036, 79037, + 79038, 79039, 79040, 79041, 79042, 79043, 79044, 79045, 79046, 79047, + 79048, 79049, 79050, 79051, 79052, 79053, 79054, 79055, 79056, 79057, + 79058, 79059, 79060, 79061, 79062, 79063, 79064, 79065, 79066, 79067, + 79068, 79069, 79070, 79071, 79072, 79073, 79074, 79075, 79076, 79077, + 79078, 79079, 79080, 79081, 79082, 79083, 79084, 79085, 79086, 79087, + 79088, 79089, 79090, 79091, 79092, 79093, 79094, 79095, 79096, 79097, + 79098, 79099, 79100, 79101, 79102, 79103, 79104, 79105, 79106, 79107, + 79108, 79109, 79110, 79111, 79112, 79113, 79114, 79115, 79116, 79117, + 79118, 79119, 79120, 79121, 79122, 79123, 79124, 79125, 79126, 79127, + 79128, 79129, 79130, 79131, 79132, 79133, 79134, 79135, 79136, 79137, + 79138, 79139, 79140, 79141, 79142, 79143, 79144, 79145, 79146, 79147, + 79148, 79149, 79150, 79151, 79152, 79153, 79154, 79155, 79156, 79157, + 79158, 79159, 79160, 79161, 79162, 79163, 79164, 79165, 79166, 79167, + 79168, 79169, 79170, 79171, 79172, 79173, 79174, 79175, 79176, 79177, + 79178, 79179, 79180, 79181, 79182, 79183, 79184, 79185, 79186, 79187, + 79188, 79189, 79190, 79191, 79192, 79193, 79194, 79195, 79196, 79197, + 79198, 79199, 79200, 79201, 79202, 79203, 79204, 79205, 79206, 79207, + 79208, 79209, 79210, 79211, 79212, 79213, 79214, 79215, 79216, 79217, + 79218, 79219, 79220, 79221, 79222, 79223, 79224, 79225, 79226, 79227, + 79228, 79229, 79230, 79231, 79232, 79233, 79234, 79235, 79236, 79237, + 79238, 79239, 79240, 79241, 79242, 79243, 79244, 79245, 79246, 79247, + 79248, 79249, 79250, 79251, 79252, 79253, 79254, 79255, 79256, 79257, + 79258, 79259, 79260, 79261, 79262, 79263, 79264, 79265, 79266, 79267, + 79268, 79269, 79270, 79271, 79272, 79273, 79274, 79275, 79276, 79277, + 79278, 79279, 79280, 79281, 79282, 79283, 79284, 79285, 79286, 79287, + 79288, 79289, 79290, 79291, 79292, 79293, 79294, 79295, 79296, 79297, + 79298, 79299, 79300, 79301, 79302, 79303, 79304, 79305, 79306, 79307, + 79308, 79309, 79310, 79311, 79312, 79313, 79314, 79315, 79316, 79317, + 79318, 79319, 79320, 79321, 79322, 79323, 79324, 79325, 79326, 79327, + 79328, 79329, 79330, 79331, 79332, 79333, 79334, 79335, 79336, 79337, + 79338, 79339, 79340, 79341, 79342, 79343, 79344, 79345, 79346, 79347, + 79348, 79349, 79350, 79351, 79352, 79353, 79354, 79355, 79356, 79357, + 79358, 79359, 79360, 79361, 79362, 79363, 79364, 79365, 79366, 79367, + 79368, 79369, 79370, 79371, 79372, 79373, 79374, 79375, 79376, 79377, + 79378, 79379, 79380, 79381, 79382, 79383, 79384, 79385, 79386, 79387, + 79388, 79389, 79390, 79391, 79392, 79393, 79394, 79395, 79396, 79397, + 79398, 79399, 79400, 79401, 79402, 79403, 79404, 79405, 79406, 79407, + 79408, 79409, 79410, 79411, 79412, 79413, 79414, 79415, 79416, 79417, + 79418, 79419, 79420, 79421, 79422, 79423, 79424, 79425, 79426, 79427, + 79428, 79429, 79430, 79431, 79432, 79433, 79434, 79435, 79436, 79437, + 79438, 79439, 79440, 79441, 79442, 79443, 79444, 79445, 79446, 79447, + 79448, 79449, 79450, 79451, 79452, 79453, 79454, 79455, 79456, 79457, + 79458, 79459, 79460, 79461, 79462, 79463, 79464, 79465, 79466, 79467, + 79468, 79469, 79470, 79471, 79472, 79473, 79474, 79475, 79476, 79477, + 79478, 79479, 79480, 79481, 79482, 79483, 79484, 79485, 79486, 79487, + 79488, 79489, 79490, 79491, 79492, 79493, 79494, 79495, 79496, 79497, + 79498, 79499, 79500, 79501, 79502, 79503, 79504, 79505, 79506, 79507, + 79508, 79509, 79510, 79511, 79512, 79513, 79514, 79515, 79516, 79517, + 79518, 79519, 79520, 79521, 79522, 79523, 79524, 79525, 79526, 79527, + 79528, 79529, 79530, 79531, 79532, 79533, 79534, 79535, 79536, 79537, + 79538, 79539, 79540, 79541, 79542, 79543, 79544, 79545, 79546, 79547, + 79548, 79549, 79550, 79551, 79552, 79553, 79554, 79555, 79556, 79557, + 79558, 79559, 79560, 79561, 79562, 79563, 79564, 79565, 79566, 79567, + 79568, 79569, 79570, 79571, 79572, 79573, 79574, 79575, 79576, 79577, + 79578, 79579, 79580, 79581, 79582, 79583, 79584, 79585, 79586, 79587, + 79588, 79589, 79590, 79591, 79592, 79593, 79594, 79595, 79596, 79597, + 79598, 79599, 79600, 79601, 79602, 79603, 79604, 79605, 79606, 79607, + 79608, 79609, 79610, 79611, 79612, 79613, 79614, 79615, 79616, 79617, + 79618, 79619, 79620, 79621, 79622, 79623, 79624, 79625, 79626, 79627, + 79628, 79629, 79630, 79631, 79632, 79633, 79634, 79635, 79636, 79637, + 79638, 79639, 79640, 79641, 79642, 79643, 79644, 79645, 79646, 79647, + 79648, 79649, 79650, 79651, 79652, 79653, 79654, 79655, 79656, 79657, + 79658, 79659, 79660, 79661, 79662, 79663, 79664, 79665, 79666, 79667, + 79668, 79669, 79670, 79671, 79672, 79673, 79674, 79675, 79676, 79677, + 79678, 79679, 79680, 79681, 79682, 79683, 79684, 79685, 79686, 79687, + 79688, 79689, 79690, 79691, 79692, 79693, 79694, 79695, 79696, 79697, + 79698, 79699, 79700, 79701, 79702, 79703, 79704, 79705, 79706, 79707, + 79708, 79709, 79710, 79711, 79712, 79713, 79714, 79715, 79716, 79717, + 79718, 79719, 79720, 79721, 79722, 79723, 79724, 79725, 79726, 79727, + 79728, 79729, 79730, 79731, 79732, 79733, 79734, 79735, 79736, 79737, + 79738, 79739, 79740, 79741, 79742, 79743, 79744, 79745, 79746, 79747, + 79748, 79749, 79750, 79751, 79752, 79753, 79754, 79755, 79756, 79757, + 79758, 79759, 79760, 79761, 79762, 79763, 79764, 79765, 79766, 79767, + 79768, 79769, 79770, 79771, 79772, 79773, 79774, 79775, 79776, 79777, + 79778, 79779, 79780, 79781, 79782, 79783, 79784, 79785, 79786, 79787, + 79788, 79789, 79790, 79791, 79792, 79793, 79794, 79795, 79796, 79797, + 79798, 79799, 79800, 79801, 79802, 79803, 79804, 79805, 79806, 79807, + 79808, 79809, 79810, 79811, 79812, 79813, 79814, 79815, 79816, 79817, + 79818, 79819, 79820, 79821, 79822, 79823, 79824, 79825, 79826, 79827, + 79828, 79829, 79830, 79831, 79832, 79833, 79834, 79835, 79836, 79837, + 79838, 79839, 79840, 79841, 79842, 79843, 79844, 79845, 79846, 79847, + 79848, 79849, 79850, 79851, 79852, 79853, 79854, 79855, 79856, 79857, + 79858, 79859, 79860, 79861, 79862, 79863, 79864, 79865, 79866, 79867, + 79868, 79869, 79870, 79871, 79872, 79873, 79874, 79875, 79876, 79877, + 79878, 79879, 79880, 79881, 79882, 79883, 79884, 79885, 79886, 79887, + 79888, 79889, 79890, 79891, 79892, 79893, 79894, 79895, 79896, 79897, + 79898, 79899, 79900, 79901, 79902, 79903, 79904, 79905, 79906, 79907, + 79908, 79909, 79910, 79911, 79912, 79913, 79914, 79915, 79916, 79917, + 79918, 79919, 79920, 79921, 79922, 79923, 79924, 79925, 79926, 79927, + 79928, 79929, 79930, 79931, 79932, 79933, 79934, 79935, 79936, 79937, + 79938, 79939, 79940, 79941, 79942, 79943, 79944, 79945, 79946, 79947, + 79948, 79949, 79950, 79951, 79952, 79953, 79954, 79955, 79956, 79957, + 79958, 79959, 79960, 79961, 79962, 79963, 79964, 79965, 79966, 79967, + 79968, 79969, 79970, 79971, 79972, 79973, 79974, 79975, 79976, 79977, + 79978, 79979, 79980, 79981, 79982, 79983, 79984, 79985, 79986, 79987, + 79988, 79989, 79990, 79991, 79992, 79993, 79994, 79995, 79996, 79997, + 79998, 79999, 80000, 80001, 80002, 80003, 80004, 80005, 80006, 80007, + 80008, 80009, 80010, 80011, 80012, 80013, 80014, 80015, 80016, 80017, + 80018, 80019, 80020, 80021, 80022, 80023, 80024, 80025, 80026, 80027, + 80028, 80029, 80030, 80031, 80032, 80033, 80034, 80035, 80036, 80037, + 80038, 80039, 80040, 80041, 80042, 80043, 80044, 80045, 80046, 80047, + 80048, 80049, 80050, 80051, 80052, 80053, 80054, 80055, 80056, 80057, + 80058, 80059, 80060, 80061, 80062, 80063, 80064, 80065, 80066, 80067, + 80068, 80069, 80070, 80071, 80072, 80073, 80074, 80075, 80076, 80077, + 80078, 80079, 80080, 80081, 80082, 80083, 80084, 80085, 80086, 80087, + 80088, 80089, 80090, 80091, 80092, 80093, 80094, 80095, 80096, 80097, + 80098, 80099, 80100, 80101, 80102, 80103, 80104, 80105, 80106, 80107, + 80108, 80109, 80110, 80111, 80112, 80113, 80114, 80115, 80116, 80117, + 80118, 80119, 80120, 80121, 80122, 80123, 80124, 80125, 80126, 80127, + 80128, 80129, 80130, 80131, 80132, 80133, 80134, 80135, 80136, 80137, + 80138, 80139, 80140, 80141, 80142, 80143, 80144, 80145, 80146, 80147, + 80148, 80149, 80150, 80151, 80152, 80153, 80154, 80155, 80156, 80157, + 80158, 80159, 80160, 80161, 80162, 80163, 80164, 80165, 80166, 80167, + 80168, 80169, 80170, 80171, 80172, 80173, 80174, 80175, 80176, 80177, + 80178, 80179, 80180, 80181, 80182, 80183, 80184, 80185, 80186, 80187, + 80188, 80189, 80190, 80191, 80192, 80193, 80194, 80195, 80196, 80197, + 80198, 80199, 80200, 80201, 80202, 80203, 80204, 80205, 80206, 80207, + 80208, 80209, 80210, 80211, 80212, 80213, 80214, 80215, 80216, 80217, + 80218, 80219, 80220, 80221, 80222, 80223, 80224, 80225, 80226, 80227, + 80228, 80229, 80230, 80231, 80232, 80233, 80234, 80235, 80236, 80237, + 80238, 80239, 80240, 80241, 80242, 80243, 80244, 80245, 80246, 80247, + 80248, 80249, 80250, 80251, 80252, 80253, 80254, 80255, 80256, 80257, + 80258, 80259, 80260, 80261, 80262, 80263, 80264, 80265, 80266, 80267, + 80268, 80269, 80270, 80271, 80272, 80273, 80274, 80275, 80276, 80277, + 80278, 80279, 80280, 80281, 80282, 80283, 80284, 80285, 80286, 80287, + 80288, 80289, 80290, 80291, 80292, 80293, 80294, 80295, 80296, 80297, + 80298, 80299, 80300, 80301, 80302, 80303, 80304, 80305, 80306, 80307, + 80308, 80309, 80310, 80311, 80312, 80313, 80314, 80315, 80316, 80317, + 80318, 80319, 80320, 80321, 80322, 80323, 80324, 80325, 80326, 80327, + 80328, 80329, 80330, 80331, 80332, 80333, 80334, 80335, 80336, 80337, + 80338, 80339, 80340, 80341, 80342, 80343, 80344, 80345, 80346, 80347, + 80348, 80349, 80350, 80351, 80352, 80353, 80354, 80355, 80356, 80357, + 80358, 80359, 80360, 80361, 80362, 80363, 80364, 80365, 80366, 80367, + 80368, 80369, 80370, 80371, 80372, 80373, 80374, 80375, 80376, 80377, + 80378, 80379, 80380, 80381, 80382, 80383, 80384, 80385, 80386, 80387, + 80388, 80389, 80390, 80391, 80392, 80393, 80394, 80395, 80396, 80397, + 80398, 80399, 80400, 80401, 80402, 80403, 80404, 80405, 80406, 80407, + 80408, 80409, 80410, 80411, 80412, 80413, 80414, 80415, 80416, 80417, + 80418, 80419, 80420, 80421, 80422, 80423, 80424, 80425, 80426, 80427, + 80428, 80429, 80430, 80431, 80432, 80433, 80434, 80435, 80436, 80437, + 80438, 80439, 80440, 80441, 80442, 80443, 80444, 80445, 80446, 80447, + 80448, 80449, 80450, 80451, 80452, 80453, 80454, 80455, 80456, 80457, + 80458, 80459, 80460, 80461, 80462, 80463, 80464, 80465, 80466, 80467, + 80468, 80469, 80470, 80471, 80472, 80473, 80474, 80475, 80476, 80477, + 80478, 80479, 80480, 80481, 80482, 80483, 80484, 80485, 80486, 80487, + 80488, 80489, 80490, 80491, 80492, 80493, 80494, 80495, 80496, 80497, + 80498, 80499, 80500, 80501, 80502, 80503, 80504, 80505, 80506, 80507, + 80508, 80509, 80510, 80511, 80512, 80513, 80514, 80515, 80516, 80517, + 80518, 80519, 80520, 80521, 80522, 80523, 80524, 80525, 80526, 80527, + 80528, 80529, 80530, 80531, 80532, 80533, 80534, 80535, 80536, 80537, + 80538, 80539, 80540, 80541, 80542, 80543, 80544, 80545, 80546, 80547, + 80548, 80549, 80550, 80551, 80552, 80553, 80554, 80555, 80556, 80557, + 80558, 80559, 80560, 80561, 80562, 80563, 80564, 80565, 80566, 80567, + 80568, 80569, 80570, 80571, 80572, 80573, 80574, 80575, 80576, 80577, + 80578, 80579, 80580, 80581, 80582, 80583, 80584, 80585, 80586, 80587, + 80588, 80589, 80590, 80591, 80592, 80593, 80594, 80595, 80596, 80597, + 80598, 80599, 80600, 80601, 80602, 80603, 80604, 80605, 80606, 80607, + 80608, 80609, 80610, 80611, 80612, 80613, 80614, 80615, 80616, 80617, + 80618, 80619, 80620, 80621, 80622, 80623, 80624, 80625, 80626, 80627, + 80628, 80629, 80630, 80631, 80632, 80633, 80634, 80635, 80636, 80637, + 80638, 80639, 80640, 80641, 80642, 80643, 80644, 80645, 80646, 80647, + 80648, 80649, 80650, 80651, 80652, 80653, 80654, 80655, 80656, 80657, + 80658, 80659, 80660, 80661, 80662, 80663, 80664, 80665, 80666, 80667, + 80668, 80669, 80670, 80671, 80672, 80673, 80674, 80675, 80676, 80677, + 80678, 80679, 80680, 80681, 80682, 80683, 80684, 80685, 80686, 80687, + 80688, 80689, 80690, 80691, 80692, 80693, 80694, 80695, 80696, 80697, + 80698, 80699, 80700, 80701, 80702, 80703, 80704, 80705, 80706, 80707, + 80708, 80709, 80710, 80711, 80712, 80713, 80714, 80715, 80716, 80717, + 80718, 80719, 80720, 80721, 80722, 80723, 80724, 80725, 80726, 80727, + 80728, 80729, 80730, 80731, 80732, 80733, 80734, 80735, 80736, 80737, + 80738, 80739, 80740, 80741, 80742, 80743, 80744, 80745, 80746, 80747, + 80748, 80749, 80750, 80751, 80752, 80753, 80754, 80755, 80756, 80757, + 80758, 80759, 80760, 80761, 80762, 80763, 80764, 80765, 80766, 80767, + 80768, 80769, 80770, 80771, 80772, 80773, 80774, 80775, 80776, 80777, + 80778, 80779, 80780, 80781, 80782, 80783, 80784, 80785, 80786, 80787, + 80788, 80789, 80790, 80791, 80792, 80793, 80794, 80795, 80796, 80797, + 80798, 80799, 80800, 80801, 80802, 80803, 80804, 80805, 80806, 80807, + 80808, 80809, 80810, 80811, 80812, 80813, 80814, 80815, 80816, 80817, + 80818, 80819, 80820, 80821, 80822, 80823, 80824, 80825, 80826, 80827, + 80828, 80829, 80830, 80831, 80832, 80833, 80834, 80835, 80836, 80837, + 80838, 80839, 80840, 80841, 80842, 80843, 80844, 80845, 80846, 80847, + 80848, 80849, 80850, 80851, 80852, 80853, 80854, 80855, 80856, 80857, + 80858, 80859, 80860, 80861, 80862, 80863, 80864, 80865, 80866, 80867, + 80868, 80869, 80870, 80871, 80872, 80873, 80874, 80875, 80876, 80877, + 80878, 80879, 80880, 80881, 80882, 80883, 80884, 80885, 80886, 80887, + 80888, 80889, 80890, 80891, 80892, 80893, 80894, 80895, 80896, 80897, + 80898, 80899, 80900, 80901, 80902, 80903, 80904, 80905, 80906, 80907, + 80908, 80909, 80910, 80911, 80912, 80913, 80914, 80915, 80916, 80917, + 80918, 80919, 80920, 80921, 80922, 80923, 80924, 80925, 80926, 80927, + 80928, 80929, 80930, 80931, 80932, 80933, 80934, 80935, 80936, 80937, + 80938, 80939, 80940, 80941, 80942, 80943, 80944, 80945, 80946, 80947, + 80948, 80949, 80950, 80951, 80952, 80953, 80954, 80955, 80956, 80957, + 80958, 80959, 80960, 80961, 80962, 80963, 80964, 80965, 80966, 80967, + 80968, 80969, 80970, 80971, 80972, 80973, 80974, 80975, 80976, 80977, + 80978, 80979, 80980, 80981, 80982, 80983, 80984, 80985, 80986, 80987, + 80988, 80989, 80990, 80991, 80992, 80993, 80994, 80995, 80996, 80997, + 80998, 80999, 81000, 81001, 81002, 81003, 81004, 81005, 81006, 81007, + 81008, 81009, 81010, 81011, 81012, 81013, 81014, 81015, 81016, 81017, + 81018, 81019, 81020, 81021, 81022, 81023, 81024, 81025, 81026, 81027, + 81028, 81029, 81030, 81031, 81032, 81033, 81034, 81035, 81036, 81037, + 81038, 81039, 81040, 81041, 81042, 81043, 81044, 81045, 81046, 81047, + 81048, 81049, 81050, 81051, 81052, 81053, 81054, 81055, 81056, 81057, + 81058, 81059, 81060, 81061, 81062, 81063, 81064, 81065, 81066, 81067, + 81068, 81069, 81070, 81071, 81072, 81073, 81074, 81075, 81076, 81077, + 81078, 81079, 81080, 81081, 81082, 81083, 81084, 81085, 81086, 81087, + 81088, 81089, 81090, 81091, 81092, 81093, 81094, 81095, 81096, 81097, + 81098, 81099, 81100, 81101, 81102, 81103, 81104, 81105, 81106, 81107, + 81108, 81109, 81110, 81111, 81112, 81113, 81114, 81115, 81116, 81117, + 81118, 81119, 81120, 81121, 81122, 81123, 81124, 81125, 81126, 81127, + 81128, 81129, 81130, 81131, 81132, 81133, 81134, 81135, 81136, 81137, + 81138, 81139, 81140, 81141, 81142, 81143, 81144, 81145, 81146, 81147, + 81148, 81149, 81150, 81151, 81152, 81153, 81154, 81155, 81156, 81157, + 81158, 81159, 81160, 81161, 81162, 81163, 81164, 81165, 81166, 81167, + 81168, 81169, 81170, 81171, 81172, 81173, 81174, 81175, 81176, 81177, + 81178, 81179, 81180, 81181, 81182, 81183, 81184, 81185, 81186, 81187, + 81188, 81189, 81190, 81191, 81192, 81193, 81194, 81195, 81196, 81197, + 81198, 81199, 81200, 81201, 81202, 81203, 81204, 81205, 81206, 81207, + 81208, 81209, 81210, 81211, 81212, 81213, 81214, 81215, 81216, 81217, + 81218, 81219, 81220, 81221, 81222, 81223, 81224, 81225, 81226, 81227, + 81228, 81229, 81230, 81231, 81232, 81233, 81234, 81235, 81236, 81237, + 81238, 81239, 81240, 81241, 81242, 81243, 81244, 81245, 81246, 81247, + 81248, 81249, 81250, 81251, 81252, 81253, 81254, 81255, 81256, 81257, + 81258, 81259, 81260, 81261, 81262, 81263, 81264, 81265, 81266, 81267, + 81268, 81269, 81270, 81271, 81272, 81273, 81274, 81275, 81276, 81277, + 81278, 81279, 81280, 81281, 81282, 81283, 81284, 81285, 81286, 81287, + 81288, 81289, 81290, 81291, 81292, 81293, 81294, 81295, 81296, 81297, + 81298, 81299, 81300, 81301, 81302, 81303, 81304, 81305, 81306, 81307, + 81308, 81309, 81310, 81311, 81312, 81313, 81314, 81315, 81316, 81317, + 81318, 81319, 81320, 81321, 81322, 81323, 81324, 81325, 81326, 81327, + 81328, 81329, 81330, 81331, 81332, 81333, 81334, 81335, 81336, 81337, + 81338, 81339, 81340, 81341, 81342, 81343, 81344, 81345, 81346, 81347, + 81348, 81349, 81350, 81351, 81352, 81353, 81354, 81355, 81356, 81357, + 81358, 81359, 81360, 81361, 81362, 81363, 81364, 81365, 81366, 81367, + 81368, 81369, 81370, 81371, 81372, 81373, 81374, 81375, 81376, 81377, + 81378, 81379, 81380, 81381, 81382, 81383, 81384, 81385, 81386, 81387, + 81388, 81389, 81390, 81391, 81392, 81393, 81394, 81395, 81396, 81397, + 81398, 81399, 81400, 81401, 81402, 81403, 81404, 81405, 81406, 81407, + 81408, 81409, 81410, 81411, 81412, 81413, 81414, 81415, 81416, 81417, + 81418, 81419, 81420, 81421, 81422, 81423, 81424, 81425, 81426, 81427, + 81428, 81429, 81430, 81431, 81432, 81433, 81434, 81435, 81436, 81437, + 81438, 81439, 81440, 81441, 81442, 81443, 81444, 81445, 81446, 81447, + 81448, 81449, 81450, 81451, 81452, 81453, 81454, 81455, 81456, 81457, + 81458, 81459, 81460, 81461, 81462, 81463, 81464, 81465, 81466, 81467, + 81468, 81469, 81470, 81471, 81472, 81473, 81474, 81475, 81476, 81477, + 81478, 81479, 81480, 81481, 81482, 81483, 81484, 81485, 81486, 81487, + 81488, 81489, 81490, 81491, 81492, 81493, 81494, 81495, 81496, 81497, + 81498, 81499, 81500, 81501, 81502, 81503, 81504, 81505, 81506, 81507, + 81508, 81509, 81510, 81511, 81512, 81513, 81514, 81515, 81516, 81517, + 81518, 81519, 81520, 81521, 81522, 81523, 81524, 81525, 81526, 81527, + 81528, 81529, 81530, 81531, 81532, 81533, 81534, 81535, 81536, 81537, + 81538, 81539, 81540, 81541, 81542, 81543, 81544, 81545, 81546, 81547, + 81548, 81549, 81550, 81551, 81552, 81553, 81554, 81555, 81556, 81557, + 81558, 81559, 81560, 81561, 81562, 81563, 81564, 81565, 81566, 81567, + 81568, 81569, 81570, 81571, 81572, 81573, 81574, 81575, 81576, 81577, + 81578, 81579, 81580, 81581, 81582, 81583, 81584, 81585, 81586, 81587, + 81588, 81589, 81590, 81591, 81592, 81593, 81594, 81595, 81596, 81597, + 81598, 81599, 81600, 81601, 81602, 81603, 81604, 81605, 81606, 81607, + 81608, 81609, 81610, 81611, 81612, 81613, 81614, 81615, 81616, 81617, + 81618, 81619, 81620, 81621, 81622, 81623, 81624, 81625, 81626, 81627, + 81628, 81629, 81630, 81631, 81632, 81633, 81634, 81635, 81636, 81637, + 81638, 81639, 81640, 81641, 81642, 81643, 81644, 81645, 81646, 81647, + 81648, 81649, 81650, 81651, 81652, 81653, 81654, 81655, 81656, 81657, + 81658, 81659, 81660, 81661, 81662, 81663, 81664, 81665, 81666, 81667, + 81668, 81669, 81670, 81671, 81672, 81673, 81674, 81675, 81676, 81677, + 81678, 81679, 81680, 81681, 81682, 81683, 81684, 81685, 81686, 81687, + 81688, 81689, 81690, 81691, 81692, 81693, 81694, 81695, 81696, 81697, + 81698, 81699, 81700, 81701, 81702, 81703, 81704, 81705, 81706, 81707, + 81708, 81709, 81710, 81711, 81712, 81713, 81714, 81715, 81716, 81717, + 81718, 81719, 81720, 81721, 81722, 81723, 81724, 81725, 81726, 81727, + 81728, 81729, 81730, 81731, 81732, 81733, 81734, 81735, 81736, 81737, + 81738, 81739, 81740, 81741, 81742, 81743, 81744, 81745, 81746, 81747, + 81748, 81749, 81750, 81751, 81752, 81753, 81754, 81755, 81756, 81757, + 81758, 81759, 81760, 81761, 81762, 81763, 81764, 81765, 81766, 81767, + 81768, 81769, 81770, 81771, 81772, 81773, 81774, 81775, 81776, 81777, + 81778, 81779, 81780, 81781, 81782, 81783, 81784, 81785, 81786, 81787, + 81788, 81789, 81790, 81791, 81792, 81793, 81794, 81795, 81796, 81797, + 81798, 81799, 81800, 81801, 81802, 81803, 81804, 81805, 81806, 81807, + 81808, 81809, 81810, 81811, 81812, 81813, 81814, 81815, 81816, 81817, + 81818, 81819, 81820, 81821, 81822, 81823, 81824, 81825, 81826, 81827, + 81828, 81829, 81830, 81831, 81832, 81833, 81834, 81835, 81836, 81837, + 81838, 81839, 81840, 81841, 81842, 81843, 81844, 81845, 81846, 81847, + 81848, 81849, 81850, 81851, 81852, 81853, 81854, 81855, 81856, 81857, + 81858, 81859, 81860, 81861, 81862, 81863, 81864, 81865, 81866, 81867, + 81868, 81869, 81870, 81871, 81872, 81873, 81874, 81875, 81876, 81877, + 81878, 81879, 81880, 81881, 81882, 81883, 81884, 81885, 81886, 81887, + 81888, 81889, 81890, 81891, 81892, 81893, 81894, 81895, 81896, 81897, + 81898, 81899, 81900, 81901, 81902, 81903, 81904, 81905, 81906, 81907, + 81908, 81909, 81910, 81911, 81912, 81913, 81914, 81915, 81916, 81917, + 81918, 81919, 82928, 82929, 82930, 82931, 82932, 82933, 82934, 82935, + 82936, 82937, 82938, 82688, 82689, 82690, 82691, 82692, 82693, 82694, + 82695, 82696, 82697, 82698, 82699, 82700, 82701, 82702, 82703, 82704, + 82705, 82706, 82707, 82708, 82709, 82710, 82711, 82712, 82713, 82714, + 82715, 82716, 82717, 82718, 82719, 82720, 82721, 82722, 82723, 82724, + 82725, 82726, 82727, 82728, 82729, 82730, 82731, 82732, 82733, 82734, + 82735, 82736, 82737, 82738, 82739, 82740, 82741, 82742, 82743, 82744, + 82745, 82746, 82747, 82748, 82749, 82750, 82751, 82752, 82753, 82754, + 82755, 82756, 82757, 82758, 82759, 82760, 82761, 82762, 82763, 82764, + 82765, 82766, 82767, 82768, 82769, 82770, 82771, 82772, 82773, 82774, + 82775, 82776, 82777, 82778, 82779, 82780, 82781, 82782, 82783, 82784, + 82785, 82786, 82787, 82788, 82789, 82790, 82791, 82792, 82793, 82794, + 82795, 82796, 82797, 82798, 82799, 82800, 82801, 82802, 82803, 82804, + 82805, 82806, 82807, 82808, 82809, 82810, 82811, 82812, 82813, 82814, + 82815, 82816, 82817, 82818, 82819, 82820, 82821, 82822, 82823, 82824, + 82825, 82826, 82827, 82828, 82829, 82830, 82831, 82832, 82833, 82834, + 82835, 82836, 82837, 82838, 82839, 82840, 82841, 82842, 82843, 82844, + 82845, 82846, 82847, 82848, 82849, 82850, 82851, 82852, 82853, 82854, + 82855, 82856, 82857, 82858, 82859, 82860, 82861, 82862, 82863, 82864, + 82865, 82866, 82867, 82868, 82869, 82870, 82871, 82872, 82873, 82874, + 82875, 82876, 82877, 82878, 82879, 82880, 82881, 82882, 82883, 82884, + 82885, 82886, 82887, 82888, 82889, 82890, 82891, 82892, 82893, 82894, + 82895, 82896, 82897, 82898, 82899, 82900, 82901, 82902, 82903, 82904, + 82905, 82906, 82907, 82908, 82909, 82910, 82911, 82912, 82913, 82914, + 82915, 82916, 82917, 82918, 82919, 82920, 82921, 82922, 82923, 82924, + 82925, 82926, 82927, 81920, 81921, 81922, 81923, 81924, 81925, 81926, + 81927, 81928, 81929, 81930, 81931, 81932, 81933, 81934, 81935, 81936, + 81937, 81938, 81939, 81940, 81941, 81942, 81943, 81944, 81945, 81946, + 81947, 81948, 81949, 81950, 81951, 81952, 81953, 81954, 81955, 81956, + 81957, 81958, 81959, 81960, 81961, 81962, 81963, 81964, 81965, 81966, + 81967, 81968, 81969, 81970, 81971, 81972, 81973, 81974, 81975, 81976, + 81977, 81978, 81979, 81980, 81981, 81982, 81983, 81984, 81985, 81986, + 81987, 81988, 81989, 81990, 81991, 81992, 81993, 81994, 81995, 81996, + 81997, 81998, 81999, 82000, 82001, 82002, 82003, 82004, 82005, 82006, + 82007, 82008, 82009, 82010, 82011, 82012, 82013, 82014, 82015, 82016, + 82017, 82018, 82019, 82020, 82021, 82022, 82023, 82024, 82025, 82026, + 82027, 82028, 82029, 82030, 82031, 82032, 82033, 82034, 82035, 82036, + 82037, 82038, 82039, 82040, 82041, 82042, 82043, 82044, 82045, 82046, + 82047, 82048, 82049, 82050, 82051, 82052, 82053, 82054, 82055, 82056, + 82057, 82058, 82059, 82060, 82061, 82062, 82063, 82064, 82065, 82066, + 82067, 82068, 82069, 82070, 82071, 82072, 82073, 82074, 82075, 82076, + 82077, 82078, 82079, 82080, 82081, 82082, 82083, 82084, 82085, 82086, + 82087, 82088, 82089, 82090, 82091, 82092, 82093, 82094, 82095, 82096, + 82097, 82098, 82099, 82100, 82101, 82102, 82103, 82104, 82105, 82106, + 82107, 82108, 82109, 82110, 82111, 82112, 82113, 82114, 82115, 82116, + 82117, 82118, 82119, 82120, 82121, 82122, 82123, 82124, 82125, 82126, + 82127, 82128, 82129, 82130, 82131, 82132, 82133, 82134, 82135, 82136, + 82137, 82138, 82139, 82140, 82141, 82142, 82143, 82144, 82145, 82146, + 82147, 82148, 82149, 82150, 82151, 82152, 82153, 82154, 82155, 82156, + 82157, 82158, 82159, 82160, 82161, 82162, 82163, 82164, 82165, 82166, + 82167, 82168, 82169, 82170, 82171, 82172, 82173, 82174, 82175, 82176, + 82177, 82178, 82179, 82180, 82181, 82182, 82183, 82184, 82185, 82186, + 82187, 82188, 82189, 82190, 82191, 82192, 82193, 82194, 82195, 82196, + 82197, 82198, 82199, 82200, 82201, 82202, 82203, 82204, 82205, 82206, + 82207, 82208, 82209, 82210, 82211, 82212, 82213, 82214, 82215, 82216, + 82217, 82218, 82219, 82220, 82221, 82222, 82223, 82224, 82225, 82226, + 82227, 82228, 82229, 82230, 82231, 82232, 82233, 82234, 82235, 82236, + 82237, 82238, 82239, 82240, 82241, 82242, 82243, 82244, 82245, 82246, + 82247, 82248, 82249, 82250, 82251, 82252, 82253, 82254, 82255, 82256, + 82257, 82258, 82259, 82260, 82261, 82262, 82263, 82264, 82265, 82266, + 82267, 82268, 82269, 82270, 82271, 82272, 82273, 82274, 82275, 82276, + 82277, 82278, 82279, 82280, 82281, 82282, 82283, 82284, 82285, 82286, + 82287, 82288, 82289, 82290, 82291, 82292, 82293, 82294, 82295, 82296, + 82297, 82298, 82299, 82300, 82301, 82302, 82303, 82304, 82305, 82306, + 82307, 82308, 82309, 82310, 82311, 82312, 82313, 82314, 82315, 82316, + 82317, 82318, 82319, 82320, 82321, 82322, 82323, 82324, 82325, 82326, + 82327, 82328, 82329, 82330, 82331, 82332, 82333, 82334, 82335, 82336, + 82337, 82338, 82339, 82340, 82341, 82342, 82343, 82344, 82345, 82346, + 82347, 82348, 82349, 82350, 82351, 82352, 82353, 82354, 82355, 82356, + 82357, 82358, 82359, 82360, 82361, 82362, 82363, 82364, 82365, 82366, + 82367, 82368, 82369, 82370, 82371, 82372, 82373, 82374, 82375, 82376, + 82377, 82378, 82379, 82380, 82381, 82382, 82383, 82384, 82385, 82386, + 82387, 82388, 82389, 82390, 82391, 82392, 82393, 82394, 82395, 82396, + 82397, 82398, 82399, 82400, 82401, 82402, 82403, 82404, 82405, 82406, + 82407, 82408, 82409, 82410, 82411, 82412, 82413, 82414, 82415, 82416, + 82417, 82418, 82419, 82420, 82421, 82422, 82423, 82424, 82425, 82426, + 82427, 82428, 82429, 82430, 82431, 82432, 82433, 82434, 82435, 82436, + 82437, 82438, 82439, 82440, 82441, 82442, 82443, 82444, 82445, 82446, + 82447, 82448, 82449, 82450, 82451, 82452, 82453, 82454, 82455, 82456, + 82457, 82458, 82459, 82460, 82461, 82462, 82463, 82464, 82465, 82466, + 82467, 82468, 82469, 82470, 82471, 82472, 82473, 82474, 82475, 82476, + 82477, 82478, 82479, 82480, 82481, 82482, 82483, 82484, 82485, 82486, + 82487, 82488, 82489, 82490, 82491, 82492, 82493, 82494, 82495, 82496, + 82497, 82498, 82499, 82500, 82501, 82502, 82503, 82504, 82505, 82506, + 82507, 82508, 82509, 82510, 82511, 82512, 82513, 82514, 82515, 82516, + 82517, 82518, 82519, 82520, 82521, 82522, 82523, 82524, 82525, 82526, + 82527, 82528, 82529, 82530, 82531, 82532, 82533, 82534, 82535, 82536, + 82537, 82538, 82539, 82540, 82541, 82542, 82543, 82544, 82545, 82546, + 82547, 82548, 82549, 82550, 82551, 82552, 82553, 82554, 82555, 82556, + 82557, 82558, 82559, 82560, 82561, 82562, 82563, 82564, 82565, 82566, + 82567, 82568, 82569, 82570, 82571, 82572, 82573, 82574, 82575, 82576, + 82577, 82578, 82579, 82580, 82581, 82582, 82583, 82584, 82585, 82586, + 82587, 82588, 82589, 82590, 82591, 82592, 82593, 82594, 82595, 82596, + 82597, 82598, 82599, 82600, 82601, 82602, 82603, 82604, 82605, 82606, + 82607, 82608, 82609, 82610, 82611, 82612, 82613, 82614, 82615, 82616, + 82617, 82618, 82619, 82620, 82621, 82622, 82623, 82624, 82625, 82626, + 82627, 82628, 82629, 82630, 82631, 82632, 82633, 82634, 82635, 82636, + 82637, 82638, 82639, 82640, 82641, 82642, 82643, 82644, 82645, 82646, + 82647, 82648, 82649, 82650, 82651, 82652, 82653, 82654, 82655, 82656, + 82657, 82658, 82659, 82660, 82661, 82662, 82663, 82664, 82665, 82666, + 82667, 82668, 82669, 82670, 82671, 82672, 82673, 82674, 82675, 82676, + 82677, 82678, 82679, 82680, 82681, 82682, 82683, 82684, 82685, 82686, + 82687, 118470, 129370, 10037, 10039, 10036, 10049, 117865, 117866, 10058, + 10035, 9834, 66854, 66853, 66827, 66826, 66833, 66832, 66821, 66837, + 66836, 66835, 66842, 66841, 66824, 66823, 66819, 66818, 66822, 66820, + 66855, 66831, 66844, 66843, 66846, 66845, 66852, 66851, 66817, 66825, + 66828, 66830, 66834, 66839, 66840, 66848, 66849, 66816, 66829, 66838, + 66847, 66850, 128268, 128294, 128161, 8961, 9191, 8712, 8946, 8953, 8947, + 8952, 8950, 8949, 10969, 10194, 128024, 128727, 69608, 69621, 69603, + 69611, 69618, 69619, 69600, 69615, 69602, 69606, 69614, 69617, 69620, + 69609, 69604, 69607, 69610, 69601, 69613, 69605, 69616, 69612, 69622, + 129501, 983101, 129456, 129457, 129459, 129458, 127995, 127996, 127997, + 127998, 127999, 128453, 128454, 128455, 129721, 128460, 128461, 8709, + 10675, 10676, 10674, 10673, 128459, 9091, 8193, 8212, 8195, 8192, 8211, + 8194, 983179, 8718, 983178, 983135, 983099, 983048, 983095, 983046, + 983064, 128282, 983051, 983050, 9993, 128388, 128233, 9094, 983067, + 983100, 983049, 8926, 8927, 8925, 8924, 8797, 8917, 61, 10865, 10867, + 11072, 10609, 10723, 10724, 10926, 11257, 11158, 10871, 10854, 10862, + 8789, 10872, 8781, 8794, 10738, 10736, 10734, 10739, 10737, 10735, 11249, + 11248, 9003, 8998, 983105, 983104, 8494, 8793, 983136, 4957, 4959, 4958, + 4963, 4965, 4978, 4988, 4980, 4979, 4987, 4985, 4982, 4981, 4986, 4984, + 4983, 4968, 4966, 4964, 4960, 4999, 4998, 4711, 4997, 43816, 43819, + 43821, 43820, 43818, 43822, 43817, 4704, 4707, 4710, 11653, 4709, 4708, + 4706, 4705, 43808, 43811, 43813, 43812, 43810, 43814, 43809, 11704, + 11707, 11709, 11708, 11706, 11710, 11705, 11688, 11691, 11693, 11692, + 11690, 11694, 11689, 4904, 4907, 4910, 11664, 4909, 4908, 4911, 4906, + 4905, 4728, 4731, 4734, 11655, 4733, 4732, 4735, 4730, 4729, 43789, + 43788, 43787, 43786, 43790, 43785, 4856, 4859, 4862, 11661, 4861, 4860, + 4863, 4858, 4857, 43797, 43796, 43795, 43794, 43798, 43793, 4848, 4851, + 4854, 11660, 4853, 4852, 4855, 4850, 4849, 5003, 5002, 4943, 5001, 4936, + 4939, 4941, 4940, 4954, 4938, 4942, 4937, 4873, 124916, 124915, 124924, + 124923, 124910, 124909, 124926, 124925, 124922, 124921, 124920, 124919, + 124918, 124917, 124914, 124913, 124912, 124904, 11667, 4895, 11670, + 11669, 11668, 4888, 4891, 4893, 4892, 4890, 4894, 4889, 4768, 4771, 4774, + 11658, 4773, 4772, 4775, 4770, 4769, 4880, 4883, 4885, 4884, 4882, 11736, + 11739, 11741, 11740, 11738, 11742, 11737, 4872, 4875, 4878, 4879, 4877, + 4876, 4874, 124907, 124906, 4631, 124905, 124896, 124899, 124901, 124900, + 124898, 124902, 124897, 4624, 4627, 4629, 4628, 4626, 4630, 4625, 4608, + 4611, 4614, 4615, 4613, 4612, 4610, 4609, 4800, 4803, 4805, 4804, 4802, + 4792, 4795, 4797, 4796, 4794, 4798, 4793, 4784, 4787, 4789, 4788, 4786, + 11720, 11723, 11725, 11724, 11722, 11726, 11721, 4776, 4779, 4782, 4783, + 4781, 4780, 4778, 4777, 4995, 4994, 4639, 4993, 4632, 4635, 4638, 11649, + 4637, 4636, 4953, 4634, 4633, 4760, 4763, 4766, 11657, 4765, 4764, 4767, + 4762, 4761, 4752, 4755, 4758, 11656, 4757, 4756, 4759, 4754, 4753, 4912, + 4816, 4819, 4821, 4820, 4818, 4822, 4817, 4915, 4918, 11665, 4917, 4916, + 4919, 4914, 4913, 5007, 5006, 4951, 5005, 4944, 4947, 4950, 11666, 4949, + 4948, 4946, 4945, 4696, 4699, 4701, 4700, 4698, 4688, 4691, 4693, 4692, + 4690, 4694, 4689, 4680, 4683, 4685, 4684, 4682, 11712, 11715, 11717, + 11716, 11714, 11718, 11713, 4672, 4675, 4678, 4679, 4677, 4676, 4674, + 4673, 4648, 4651, 4654, 11650, 4653, 4652, 4655, 4952, 4650, 4649, 4661, + 4996, 5000, 4992, 5004, 4660, 4664, 4667, 4670, 11652, 4669, 4668, 4671, + 4666, 4665, 4640, 4643, 4645, 4644, 4647, 4642, 4646, 4641, 11680, 11683, + 11685, 11684, 11682, 11686, 11681, 4656, 4659, 4662, 11651, 4663, 4658, + 4657, 4896, 4899, 4902, 11663, 4901, 4900, 4903, 4898, 4897, 43781, + 43780, 43779, 43778, 43782, 43777, 4928, 4931, 4934, 4935, 4933, 4932, + 4930, 4929, 4920, 4923, 4925, 4924, 4927, 4922, 4926, 4921, 4720, 4723, + 4726, 11654, 4725, 4724, 4727, 4722, 4721, 4864, 4867, 4870, 11662, 4869, + 4868, 4871, 4866, 4865, 4616, 4619, 4622, 11648, 4621, 4620, 4623, 4618, + 4617, 4808, 4811, 4814, 4815, 4813, 4812, 4810, 4809, 4840, 4843, 4846, + 4847, 4845, 4844, 4842, 4841, 4744, 4747, 4749, 4748, 4746, 11728, 11731, + 11733, 11732, 11730, 11734, 11729, 4736, 4739, 4742, 4743, 4741, 4740, + 4738, 4737, 4832, 4835, 4837, 4836, 4839, 4834, 4838, 4833, 11696, 11699, + 11701, 11700, 11698, 11702, 11697, 4824, 4827, 4830, 11659, 4829, 4828, + 4831, 4826, 4825, 4712, 4715, 4717, 4716, 4719, 4714, 4718, 4713, 5009, + 5016, 5012, 5015, 5013, 5017, 5010, 5011, 5014, 5008, 4973, 4972, 4975, + 4974, 4971, 4970, 4977, 4969, 4976, 4962, 4967, 4961, 983096, 983047, + 127984, 127972, 8352, 8364, 118472, 8455, 8265, 33, 8761, 118269, 118270, 118271, 118274, 128529, 128942, 128954, 128948, 128905, 128915, 128935, 128125, 1781, 1780, 1783, 1782, 1779, 1778, 1776, 1785, 1777, 1784, 128065, 128083, 128064, 128231, 9167, 127794, 983180, 128523, 128561, @@ -10937,150 +11055,153 @@ static const unsigned int dawg_pos_to_codepoint[] = { 128514, 129298, 129769, 129323, 128580, 128548, 129764, 129396, 128566, 129318, 128134, 128536, 129401, 8507, 127981, 10540, 10543, 9950, 127810, 129478, 128439, 128224, 128106, 9771, 127877, 129498, 128552, 129718, - 170, 9792, 127905, 9972, 129338, 8210, 8199, 128193, 128452, 983107, - 128253, 127902, 129734, 10765, 128293, 128658, 129519, 127879, 127878, - 129512, 9789, 127771, 127763, 8296, 129351, 128031, 127907, 9673, 127845, - 128074, 8281, 11821, 127953, 129407, 129747, 9189, 117910, 9971, 129449, - 128170, 9884, 10086, 9880, 8277, 127924, 128190, 128563, 129672, 129712, - 128760, 117834, 117835, 129359, 128389, 127787, 127745, 129709, 128448, - 129462, 128099, 127860, 127869, 11792, 10972, 983071, 8704, 8873, 129376, - 10021, 11156, 8280, 8283, 10019, 127808, 10018, 128966, 8732, 8197, 9970, - 129749, 129418, 8260, 8543, 128444, 128446, 128445, 127839, 8355, 129398, - 10156, 128037, 8994, 128550, 128056, 127844, 127773, 127765, 10199, 9608, - 46, 65342, 65312, 65292, 65306, 65504, 65375, 65371, 65288, 65339, 65308, - 65313, 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, - 65323, 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, - 65333, 65334, 65335, 65336, 65337, 65338, 65345, 65346, 65347, 65348, - 65349, 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, - 65359, 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, - 65369, 65370, 65343, 65506, 65283, 65505, 65285, 65291, 65376, 65373, - 65289, 65341, 65340, 65307, 65295, 65509, 65507, 65287, 65286, 65290, - 65284, 65301, 65300, 65303, 65302, 65299, 65298, 65296, 65305, 65297, - 65304, 65309, 65281, 65344, 65310, 65293, 65282, 65311, 65510, 65374, - 65294, 65508, 65372, 8289, 9905, 118280, 9981, 9179, 983215, 983216, - 983217, 983219, 983108, 983236, 983072, 68972, 68971, 68973, 68970, - 68964, 68965, 68959, 68961, 68948, 68945, 68954, 68960, 68953, 68963, - 68949, 68947, 68952, 68946, 68962, 68958, 68950, 68957, 68951, 68955, - 68956, 68944, 68996, 68997, 68991, 68993, 68980, 68977, 68986, 68992, - 68985, 68995, 68981, 68979, 68984, 68978, 68994, 68990, 68982, 68989, - 68983, 68987, 68988, 68976, 68943, 68969, 68941, 68938, 68939, 68940, - 68942, 68975, 68974, 69006, 69007, 68933, 68932, 68935, 68934, 68931, - 68930, 68928, 68937, 68929, 68936, 129476, 127922, 9881, 9965, 9966, - 128142, 9802, 8782, 8785, 8762, 4301, 4256, 4288, 4292, 4290, 4293, 4289, - 4281, 4285, 4284, 4282, 4278, 4258, 4287, 4283, 4277, 4265, 4276, 4270, - 4280, 4273, 4271, 4262, 4263, 4274, 4266, 4272, 4257, 4267, 4286, 4268, - 4279, 4261, 4259, 4260, 4264, 4269, 4275, 4295, 4291, 11565, 11520, - 11552, 11556, 11554, 11557, 11553, 11545, 11549, 11548, 11546, 11542, - 11522, 11551, 11547, 11541, 11529, 11540, 11534, 11544, 11537, 11535, - 11526, 11527, 11538, 11530, 11536, 11521, 11531, 11550, 11532, 11543, - 11525, 11523, 11524, 11528, 11533, 11539, 11559, 11555, 983955, 4323, - 4349, 4346, 4304, 4329, 4333, 4332, 4330, 4344, 4308, 4326, 4306, 4340, - 4350, 4336, 4338, 4341, 4337, 4335, 4331, 4325, 4313, 4351, 4314, 4324, - 4318, 4328, 4321, 4345, 4311, 4322, 4319, 4310, 4320, 4305, 4315, 4334, - 4316, 4327, 4309, 4307, 4312, 4317, 4343, 4339, 4342, 7357, 7354, 7312, - 7337, 7341, 7340, 7338, 7352, 7316, 7334, 7314, 7348, 7358, 7344, 7346, - 7349, 7345, 7343, 7339, 7333, 7321, 7359, 7322, 7332, 7326, 7336, 7329, - 7353, 7319, 7330, 7327, 7318, 7328, 7313, 7323, 7342, 7324, 7335, 7317, - 7315, 7320, 7325, 7331, 7351, 7347, 7350, 4347, 8368, 12307, 129502, - 8503, 129754, 128103, 128714, 129426, 11264, 11304, 11265, 11311, 11293, - 11276, 11268, 11271, 11287, 11306, 11267, 11275, 11274, 11305, 11303, - 11307, 11273, 11310, 11278, 11279, 11280, 11281, 11289, 11282, 11290, - 11283, 11291, 11308, 11294, 11284, 11298, 11300, 11301, 11285, 11309, - 11292, 11266, 11269, 11296, 11295, 11297, 11302, 11299, 11272, 11270, - 11288, 11286, 11277, 11312, 11352, 11313, 11359, 11341, 11324, 11316, - 11319, 11335, 11354, 11315, 11323, 11322, 11353, 11351, 11355, 11321, - 11358, 11326, 11327, 11328, 11329, 11337, 11330, 11338, 11331, 11339, - 11356, 11342, 11332, 11346, 11348, 11349, 11333, 11357, 11340, 11314, - 11317, 11344, 11343, 11345, 11350, 11347, 11320, 11318, 11336, 11334, - 11325, 129371, 127760, 127775, 129508, 10726, 129349, 128016, 66356, - 66352, 66376, 66359, 66358, 66375, 66378, 66369, 66365, 66368, 66357, - 66370, 66360, 66372, 66373, 66367, 66374, 66353, 66377, 66355, 66364, - 66371, 66361, 66363, 66366, 66354, 66362, 129421, 129405, 128893, 129727, - 127948, 127891, 70495, 70494, 70411, 70496, 70412, 70497, 70453, 70405, - 70406, 70416, 70420, 70434, 70433, 70439, 70438, 70432, 70431, 70437, - 70436, 70409, 70410, 70407, 70408, 70451, 70450, 70425, 70435, 70430, - 70440, 70454, 70455, 70456, 70445, 70444, 70427, 70426, 70424, 70423, - 70429, 70428, 70422, 70421, 70443, 70442, 70419, 70415, 70457, 70446, - 70448, 70447, 70400, 70401, 70460, 70461, 70402, 70493, 70477, 70403, - 70487, 70462, 70472, 70476, 70465, 70466, 70467, 70468, 70498, 70499, - 70463, 70464, 70475, 70471, 70480, 96, 127815, 10896, 10894, 10900, - 10892, 10898, 10616, 10890, 10888, 10917, 8935, 8809, 10878, 10882, - 10884, 10880, 10886, 8819, 8805, 8823, 10916, 8807, 8923, 10919, 10921, - 10874, 10876, 8919, 62, 65860, 65863, 65878, 65866, 65873, 65859, 65861, - 65868, 65875, 65862, 65870, 65864, 65871, 65867, 65874, 65857, 65869, - 65876, 65856, 65858, 65865, 65877, 65872, 65879, 65903, 65885, 65904, - 65907, 65908, 65900, 65883, 65886, 65896, 65890, 65882, 65880, 65891, - 65902, 65906, 65897, 65899, 65893, 65892, 65884, 65881, 65898, 65905, - 65887, 65901, 65894, 65895, 65888, 65889, 903, 65927, 65926, 913, 7945, - 7949, 8077, 7947, 8075, 7951, 8079, 8073, 7944, 7948, 8076, 7946, 8074, - 7950, 8078, 8072, 8124, 8120, 8122, 8123, 902, 8121, 882, 919, 7977, - 7981, 8093, 7979, 8091, 7983, 8095, 8089, 7976, 7980, 8092, 7978, 8090, - 7982, 8094, 8088, 8140, 8139, 8138, 905, 917, 7961, 7965, 7963, 7960, - 7964, 7962, 8137, 8136, 904, 921, 7993, 7999, 7997, 7995, 938, 7992, - 7998, 7996, 7994, 8152, 8154, 8155, 906, 8153, 937, 8041, 8045, 8109, - 8043, 8107, 8047, 8111, 8105, 8040, 8044, 8108, 8042, 8106, 8046, 8110, - 8104, 8188, 8187, 8186, 911, 927, 8009, 8013, 8011, 8008, 8012, 8010, - 8185, 8184, 908, 929, 8172, 931, 1018, 1015, 933, 8025, 8031, 8029, 8027, - 939, 8168, 8170, 8171, 910, 8169, 886, 934, 936, 928, 920, 932, 916, 922, - 915, 935, 914, 880, 918, 923, 895, 924, 925, 926, 1017, 1023, 1021, 1022, - 975, 1012, 8129, 8174, 8173, 901, 65915, 8190, 8159, 8158, 8157, 65920, - 65919, 119325, 119331, 119332, 119333, 119334, 119335, 119336, 119337, - 119326, 119338, 119339, 119340, 119341, 119342, 119343, 119344, 119345, - 119346, 119347, 119348, 119349, 119327, 119350, 119351, 119352, 119353, - 119354, 119355, 119356, 119328, 119357, 119358, 119359, 119360, 119361, - 119329, 119330, 65933, 1008, 983, 8125, 65922, 7466, 7464, 7462, 43877, - 7465, 7463, 992, 986, 984, 990, 988, 1011, 885, 1010, 1013, 65923, 884, - 65921, 119365, 65925, 65909, 65910, 65931, 8189, 65924, 65916, 8126, - 8127, 8143, 8142, 8141, 8128, 981, 982, 1020, 1009, 1014, 945, 8112, - 8048, 8114, 7937, 7941, 8069, 7943, 8071, 7939, 8067, 8065, 7936, 7940, - 8068, 7942, 8070, 7938, 8066, 8064, 8118, 8119, 8049, 8116, 8115, 940, - 8113, 985, 883, 989, 948, 949, 7953, 7957, 7955, 7952, 7956, 7954, 8051, - 8050, 941, 951, 7969, 7973, 8085, 7975, 8087, 7971, 8083, 8081, 7968, - 7972, 8084, 7974, 8086, 7970, 8082, 8080, 8134, 8135, 8053, 8132, 8052, - 8130, 8131, 942, 953, 7985, 7991, 7989, 7987, 970, 8151, 8147, 8146, 912, - 7984, 7990, 7988, 7986, 8150, 8144, 8054, 8055, 943, 8145, 965, 8017, - 8023, 8021, 8019, 971, 8167, 8163, 8162, 944, 8016, 8022, 8020, 8018, - 8166, 8160, 8058, 8059, 973, 8161, 954, 991, 969, 8033, 8037, 8101, 8039, - 8103, 8035, 8099, 8097, 8032, 8036, 8100, 8038, 8102, 8034, 8098, 8096, - 8182, 8183, 8061, 8180, 8060, 8178, 8179, 974, 959, 8001, 8005, 8003, - 8000, 8004, 8002, 8057, 8056, 972, 887, 966, 968, 960, 961, 8165, 8164, - 993, 1019, 987, 963, 1016, 952, 964, 962, 947, 967, 946, 881, 950, 955, - 956, 957, 958, 893, 891, 892, 7527, 7530, 7529, 7528, 7526, 65952, 65932, - 65918, 65912, 977, 65929, 65917, 65911, 900, 65914, 979, 980, 978, 8175, - 119297, 119315, 119316, 119317, 119318, 119319, 119300, 119320, 119321, - 119322, 119323, 119324, 119296, 119305, 119306, 119307, 119308, 119309, - 119310, 119311, 119312, 119313, 119314, 119298, 119299, 119301, 119302, - 119303, 119304, 890, 65913, 976, 65928, 65930, 894, 127823, 128215, - 128154, 129367, 129654, 128512, 129322, 129321, 128513, 128568, 128556, - 983110, 11218, 128151, 8370, 128130, 127928, 129454, 2693, 2694, 2704, - 2708, 2722, 2721, 2727, 2726, 2720, 2719, 2725, 2724, 2699, 2784, 2700, - 2785, 2741, 2697, 2698, 2695, 2696, 2739, 2738, 2809, 2713, 2723, 2718, - 2728, 2742, 2743, 2744, 2733, 2732, 2715, 2714, 2712, 2711, 2717, 2716, - 2710, 2709, 2731, 2730, 2745, 2734, 2736, 2735, 2703, 2707, 2814, 2689, - 2812, 2815, 2813, 2811, 2810, 2748, 2749, 2690, 2765, 2691, 2757, 2761, - 2750, 2760, 2764, 2753, 2754, 2755, 2756, 2786, 2787, 2751, 2752, 2759, - 2763, 2701, 2705, 2800, 2801, 2795, 2794, 2797, 2796, 2793, 2792, 2790, - 2799, 2791, 2798, 2768, 73092, 73082, 73056, 73057, 73064, 73067, 73091, - 73090, 73081, 73080, 73086, 73085, 73076, 73075, 73060, 73061, 73058, - 73059, 73087, 73077, 73071, 73070, 73084, 73083, 73079, 73078, 73089, - 73088, 73074, 73073, 73094, 73093, 73066, 73063, 73095, 73072, 73096, - 73097, 73069, 73068, 73110, 73109, 73098, 73105, 73108, 73101, 73102, - 73099, 73100, 73107, 73104, 73111, 73125, 73124, 73127, 73126, 73123, - 73122, 73120, 73129, 73121, 73128, 73112, 2678, 2673, 2650, 2584, 2583, - 2649, 2582, 2581, 2565, 2566, 2576, 2580, 2594, 2593, 2599, 2598, 2652, - 2608, 2592, 2591, 2597, 2596, 2569, 2570, 2567, 2568, 2611, 2610, 2585, - 2595, 2590, 2600, 2605, 2604, 2587, 2586, 2589, 2588, 2603, 2602, 2614, - 2616, 2579, 2575, 2654, 2617, 2606, 2613, 2607, 2651, 983656, 983655, - 983654, 983653, 983658, 983657, 2561, 2562, 2641, 2620, 2677, 2637, 2563, - 2622, 2632, 2636, 2625, 2626, 2623, 2624, 2635, 2631, 2672, 2674, 2676, - 2667, 2666, 2669, 2668, 2665, 2664, 2662, 2671, 2663, 2670, 2675, 90412, - 90414, 90411, 90410, 90373, 90388, 90382, 90381, 90387, 90386, 90380, - 90379, 90385, 90384, 90392, 90391, 90375, 90374, 90372, 90371, 90377, - 90376, 90370, 90369, 90390, 90389, 90378, 90396, 90393, 90395, 90397, - 90383, 90394, 90368, 90413, 90415, 90398, 90405, 90408, 90401, 90402, - 90406, 90407, 90399, 90400, 90403, 90404, 90409, 90421, 90420, 90423, - 90422, 90419, 90418, 90416, 90425, 90417, 90424, 128123, 983111, 129710, - 8202, 128135, 65467, 65441, 65443, 65444, 65445, 65446, 65469, 65458, + 170, 9792, 127905, 9972, 129338, 8210, 8199, 129775, 128193, 128452, + 983107, 128253, 127902, 129734, 10765, 128293, 128658, 129519, 127879, + 127878, 129512, 9789, 127771, 127763, 8296, 129351, 128031, 127907, 9673, + 127845, 128074, 8281, 11821, 127953, 129407, 129747, 9189, 117910, 9971, + 129449, 128170, 9884, 118466, 10086, 9880, 8277, 127924, 128190, 128563, + 129672, 129712, 128760, 117834, 117835, 118011, 129359, 128389, 127787, + 127745, 129709, 128448, 129462, 128099, 127860, 127869, 11792, 10972, + 129376, 118476, 983071, 8704, 8873, 10021, 11156, 8280, 8283, 10019, + 127808, 10018, 128966, 8732, 8197, 9970, 129749, 129418, 8260, 8543, + 128444, 128446, 128445, 118458, 127839, 8355, 129398, 10156, 128037, + 8994, 128550, 128056, 127844, 127773, 127765, 10199, 9608, 46, 65342, + 65312, 65292, 65306, 65504, 65375, 65371, 65288, 65339, 65308, 65313, + 65314, 65315, 65316, 65317, 65318, 65319, 65320, 65321, 65322, 65323, + 65324, 65325, 65326, 65327, 65328, 65329, 65330, 65331, 65332, 65333, + 65334, 65335, 65336, 65337, 65338, 65345, 65346, 65347, 65348, 65349, + 65350, 65351, 65352, 65353, 65354, 65355, 65356, 65357, 65358, 65359, + 65360, 65361, 65362, 65363, 65364, 65365, 65366, 65367, 65368, 65369, + 65370, 65343, 65506, 65283, 65505, 65285, 65291, 65376, 65373, 65289, + 65341, 65340, 65307, 65295, 65509, 65507, 65287, 65286, 65290, 65284, + 65301, 65300, 65303, 65302, 65299, 65298, 65296, 65305, 65297, 65304, + 65309, 65281, 65344, 65310, 65293, 65282, 65311, 65510, 65374, 65294, + 65508, 65372, 8289, 9905, 118280, 9981, 9179, 983215, 983216, 983217, + 983219, 983108, 983236, 983072, 68972, 68971, 68973, 68970, 68964, 68965, + 68959, 68961, 68948, 68945, 68954, 68960, 68953, 68963, 68949, 68947, + 68952, 68946, 68962, 68958, 68950, 68957, 68951, 68955, 68956, 68944, + 68996, 68997, 68991, 68993, 68980, 68977, 68986, 68992, 68985, 68995, + 68981, 68979, 68984, 68978, 68994, 68990, 68982, 68989, 68983, 68987, + 68988, 68976, 68943, 68969, 68941, 68938, 68939, 68940, 68942, 68975, + 68974, 69006, 69007, 68933, 68932, 68935, 68934, 68931, 68930, 68928, + 68937, 68929, 68936, 129476, 127922, 9881, 9965, 9966, 128142, 9802, + 118501, 118506, 118498, 118503, 118510, 118505, 118502, 118508, 118499, + 118504, 118497, 118507, 118509, 118496, 118500, 118511, 8782, 8785, 8762, + 4301, 4256, 4288, 4292, 4290, 4293, 4289, 4281, 4285, 4284, 4282, 4278, + 4258, 4287, 4283, 4277, 4265, 4276, 4270, 4280, 4273, 4271, 4262, 4263, + 4274, 4266, 4272, 4257, 4267, 4286, 4268, 4279, 4261, 4259, 4260, 4264, + 4269, 4275, 4295, 4291, 11565, 11520, 11552, 11556, 11554, 11557, 11553, + 11545, 11549, 11548, 11546, 11542, 11522, 11551, 11547, 11541, 11529, + 11540, 11534, 11544, 11537, 11535, 11526, 11527, 11538, 11530, 11536, + 11521, 11531, 11550, 11532, 11543, 11525, 11523, 11524, 11528, 11533, + 11539, 11559, 11555, 983955, 4323, 4349, 4346, 4304, 4329, 4333, 4332, + 4330, 4344, 4308, 4326, 4306, 4340, 4350, 4336, 4338, 4341, 4337, 4335, + 4331, 4325, 4313, 4351, 4314, 4324, 4318, 4328, 4321, 4345, 4311, 4322, + 4319, 4310, 4320, 4305, 4315, 4334, 4316, 4327, 4309, 4307, 4312, 4317, + 4343, 4339, 4342, 7357, 7354, 7312, 7337, 7341, 7340, 7338, 7352, 7316, + 7334, 7314, 7348, 7358, 7344, 7346, 7349, 7345, 7343, 7339, 7333, 7321, + 7359, 7322, 7332, 7326, 7336, 7329, 7353, 7319, 7330, 7327, 7318, 7328, + 7313, 7323, 7342, 7324, 7335, 7317, 7315, 7320, 7325, 7331, 7351, 7347, + 7350, 4347, 8368, 12307, 129502, 8503, 129754, 128103, 128714, 129426, + 11264, 11304, 11265, 11311, 11293, 11276, 11268, 11271, 11287, 11306, + 11267, 11275, 11274, 11305, 11303, 11307, 11273, 11310, 11278, 11279, + 11280, 11281, 11289, 11282, 11290, 11283, 11291, 11308, 11294, 11284, + 11298, 11300, 11301, 11285, 11309, 11292, 11266, 11269, 11296, 11295, + 11297, 11302, 11299, 11272, 11270, 11288, 11286, 11277, 11312, 11352, + 11313, 11359, 11341, 11324, 11316, 11319, 11335, 11354, 11315, 11323, + 11322, 11353, 11351, 11355, 11321, 11358, 11326, 11327, 11328, 11329, + 11337, 11330, 11338, 11331, 11339, 11356, 11342, 11332, 11346, 11348, + 11349, 11333, 11357, 11340, 11314, 11317, 11344, 11343, 11345, 11350, + 11347, 11320, 11318, 11336, 11334, 11325, 129371, 127760, 127775, 129508, + 10726, 129349, 128016, 66356, 66352, 66376, 66359, 66358, 66375, 66378, + 66369, 66365, 66368, 66357, 66370, 66360, 66372, 66373, 66367, 66374, + 66353, 66377, 66355, 66364, 66361, 66363, 66371, 66366, 66354, 66362, + 129421, 129405, 128893, 129727, 127948, 127891, 70495, 70494, 70411, + 70496, 70412, 70497, 70453, 70405, 70406, 70416, 70420, 70434, 70433, + 70439, 70438, 70432, 70431, 70437, 70436, 70409, 70410, 70407, 70408, + 70451, 70450, 70425, 70435, 70430, 70440, 70454, 70455, 70456, 70445, + 70444, 70427, 70426, 70424, 70423, 70429, 70428, 70422, 70421, 70443, + 70442, 70419, 70415, 70457, 70446, 70448, 70447, 70400, 70401, 70460, + 70461, 70402, 70493, 70477, 70403, 70487, 70462, 70472, 70476, 70465, + 70466, 70467, 70468, 70498, 70499, 70463, 70464, 70475, 70471, 70480, 96, + 127815, 10896, 10894, 10900, 10892, 10898, 10616, 10890, 10888, 10917, + 8935, 8809, 10878, 10882, 10884, 10880, 10886, 8819, 8805, 8823, 10916, + 8807, 8923, 10919, 10921, 10874, 10876, 8919, 62, 65860, 65863, 65878, + 65866, 65873, 65859, 65861, 65868, 65875, 65862, 65870, 65864, 65871, + 65867, 65874, 65857, 65869, 65876, 65856, 65858, 65865, 65877, 65872, + 65879, 65903, 65885, 65904, 65907, 65908, 65900, 65883, 65886, 65896, + 65890, 65882, 65880, 65891, 65902, 65906, 65897, 65899, 65893, 65892, + 65884, 65881, 65898, 65905, 65887, 65901, 65894, 65895, 65888, 65889, + 903, 65927, 65926, 913, 7945, 7949, 8077, 7947, 8075, 7951, 8079, 8073, + 7944, 7948, 8076, 7946, 8074, 7950, 8078, 8072, 8124, 8120, 8122, 8123, + 902, 8121, 882, 919, 7977, 7981, 8093, 7979, 8091, 7983, 8095, 8089, + 7976, 7980, 8092, 7978, 8090, 7982, 8094, 8088, 8140, 8139, 8138, 905, + 917, 7961, 7965, 7963, 7960, 7964, 7962, 8137, 8136, 904, 921, 7993, + 7999, 7997, 7995, 938, 7992, 7998, 7996, 7994, 8152, 8154, 8155, 906, + 8153, 937, 8041, 8045, 8109, 8043, 8107, 8047, 8111, 8105, 8040, 8044, + 8108, 8042, 8106, 8046, 8110, 8104, 8188, 8187, 8186, 911, 927, 8009, + 8013, 8011, 8008, 8012, 8010, 8185, 8184, 908, 929, 8172, 931, 1018, + 1015, 933, 8025, 8031, 8029, 8027, 939, 8168, 8170, 8171, 910, 8169, 886, + 934, 936, 928, 920, 932, 916, 922, 915, 935, 914, 880, 918, 923, 895, + 924, 925, 926, 1017, 1023, 1021, 1022, 975, 1012, 8129, 8174, 8173, 901, + 65915, 8190, 8159, 8158, 8157, 65920, 65919, 119325, 119331, 119332, + 119333, 119334, 119335, 119336, 119337, 119326, 119338, 119339, 119340, + 119341, 119342, 119343, 119344, 119345, 119346, 119347, 119348, 119349, + 119327, 119350, 119351, 119352, 119353, 119354, 119355, 119356, 119328, + 119357, 119358, 119359, 119360, 119361, 119329, 119330, 65933, 1008, 983, + 8125, 65922, 7466, 7464, 7462, 43877, 7465, 7463, 992, 986, 984, 990, + 988, 1011, 885, 1010, 1013, 65923, 884, 65921, 119365, 65925, 65909, + 65910, 65931, 8189, 65924, 65916, 8126, 8127, 8143, 8142, 8141, 8128, + 981, 982, 1020, 1009, 1014, 945, 8112, 8048, 8114, 7937, 7941, 8069, + 7943, 8071, 7939, 8067, 8065, 7936, 7940, 8068, 7942, 8070, 7938, 8066, + 8064, 8118, 8119, 8049, 8116, 8115, 940, 8113, 985, 883, 989, 948, 949, + 7953, 7957, 7955, 7952, 7956, 7954, 8051, 8050, 941, 951, 7969, 7973, + 8085, 7975, 8087, 7971, 8083, 8081, 7968, 7972, 8084, 7974, 8086, 7970, + 8082, 8080, 8134, 8135, 8053, 8132, 8052, 8130, 8131, 942, 953, 7985, + 7991, 7989, 7987, 970, 8151, 8147, 8146, 912, 7984, 7990, 7988, 7986, + 8150, 8144, 8054, 8055, 943, 8145, 965, 8017, 8023, 8021, 8019, 971, + 8167, 8163, 8162, 944, 8016, 8022, 8020, 8018, 8166, 8160, 8058, 8059, + 973, 8161, 954, 991, 969, 8033, 8037, 8101, 8039, 8103, 8035, 8099, 8097, + 8032, 8036, 8100, 8038, 8102, 8034, 8098, 8096, 8182, 8183, 8061, 8180, + 8060, 8178, 8179, 974, 959, 8001, 8005, 8003, 8000, 8004, 8002, 8057, + 8056, 972, 887, 966, 968, 960, 961, 8165, 8164, 993, 1019, 987, 963, + 1016, 952, 964, 962, 947, 967, 946, 881, 950, 955, 956, 957, 958, 893, + 891, 892, 7527, 7530, 7529, 7528, 7526, 65952, 65932, 65918, 65912, 977, + 65929, 65917, 65911, 900, 65914, 979, 980, 978, 8175, 119297, 119315, + 119316, 119317, 119318, 119319, 119300, 119320, 119321, 119322, 119323, + 119324, 119296, 119305, 119306, 119307, 119308, 119309, 119310, 119311, + 119312, 119313, 119314, 119298, 119299, 119301, 119302, 119303, 119304, + 890, 65913, 976, 65928, 65930, 894, 127823, 128215, 128154, 129367, + 129654, 128512, 129322, 129321, 128513, 128568, 128556, 983110, 11218, + 128151, 8370, 128130, 127928, 129454, 2693, 2694, 2704, 2708, 2722, 2721, + 2727, 2726, 2720, 2719, 2725, 2724, 2699, 2784, 2700, 2785, 2741, 2697, + 2698, 2695, 2696, 2739, 2738, 2809, 2713, 2723, 2718, 2728, 2742, 2743, + 2744, 2733, 2732, 2715, 2714, 2712, 2711, 2717, 2716, 2710, 2709, 2731, + 2730, 2745, 2734, 2736, 2735, 2703, 2707, 2814, 2689, 2812, 2815, 2813, + 2811, 2810, 2748, 2749, 2690, 2765, 2691, 2757, 2761, 2750, 2760, 2764, + 2753, 2754, 2755, 2756, 2786, 2787, 2751, 2752, 2759, 2763, 2701, 2705, + 2800, 2801, 2795, 2794, 2797, 2796, 2793, 2792, 2790, 2799, 2791, 2798, + 2768, 73092, 73082, 73056, 73057, 73064, 73067, 73091, 73090, 73081, + 73080, 73086, 73085, 73076, 73075, 73060, 73061, 73058, 73059, 73087, + 73077, 73071, 73070, 73084, 73083, 73079, 73078, 73089, 73088, 73074, + 73073, 73094, 73093, 73066, 73063, 73095, 73072, 73096, 73097, 73069, + 73068, 73110, 73109, 73098, 73105, 73108, 73101, 73102, 73099, 73100, + 73107, 73104, 73111, 73125, 73124, 73127, 73126, 73123, 73122, 73120, + 73129, 73121, 73128, 73112, 2678, 2673, 2650, 2584, 2583, 2649, 2582, + 2581, 2565, 2566, 2576, 2580, 2594, 2593, 2599, 2598, 2652, 2608, 2592, + 2591, 2597, 2596, 2569, 2570, 2567, 2568, 2611, 2610, 2585, 2595, 2590, + 2600, 2605, 2604, 2587, 2586, 2589, 2588, 2603, 2602, 2614, 2616, 2579, + 2575, 2654, 2617, 2606, 2613, 2607, 2651, 983656, 983655, 983654, 983653, + 983658, 983657, 2561, 2562, 2641, 2620, 2677, 2637, 2563, 2622, 2632, + 2636, 2625, 2626, 2623, 2624, 2635, 2631, 2672, 2674, 2676, 2667, 2666, + 2669, 2668, 2665, 2664, 2662, 2671, 2663, 2670, 2675, 90412, 90414, + 90411, 90410, 90373, 90388, 90382, 90381, 90387, 90386, 90380, 90379, + 90385, 90384, 90392, 90391, 90375, 90374, 90372, 90371, 90377, 90376, + 90370, 90369, 90390, 90389, 90378, 90396, 90393, 90395, 90397, 90383, + 90394, 90368, 90413, 90415, 90398, 90405, 90408, 90401, 90402, 90406, + 90407, 90399, 90400, 90403, 90404, 90409, 90421, 90420, 90423, 90422, + 90419, 90418, 90416, 90425, 90417, 90424, 128123, 983111, 129710, 8202, + 129736, 128135, 65467, 65441, 65443, 65444, 65445, 65446, 65469, 65458, 65460, 65449, 65454, 65455, 65452, 65451, 65450, 65456, 65453, 65465, 65442, 65459, 65448, 65462, 65461, 65483, 65482, 65476, 65477, 65499, 65490, 65495, 65466, 65464, 65479, 65478, 65498, 65500, 65463, 65457, @@ -11169,81 +11290,81 @@ static const unsigned int dawg_pos_to_codepoint[] = { 64292, 64291, 64294, 1497, 64285, 64313, 64335, 1520, 1522, 64287, 1521, 1477, 1476, 1455, 1468, 1458, 1459, 1457, 1460, 1465, 1466, 1463, 1464, 1479, 1467, 1471, 1462, 1473, 1456, 1474, 1461, 64286, 1469, 1524, 1523, - 1472, 1475, 1478, 1470, 1519, 9937, 9096, 11263, 128641, 110597, 110594, - 110595, 110596, 110778, 110779, 110780, 110781, 110782, 110783, 110784, - 110785, 110750, 110759, 110760, 110751, 110752, 110753, 110754, 110755, - 110756, 110757, 110758, 110768, 110769, 110770, 110771, 110772, 110773, - 110774, 110775, 110776, 110777, 110761, 110762, 110763, 110764, 110765, - 110766, 110767, 110615, 110624, 110625, 110626, 110616, 110617, 110618, - 110619, 110620, 110621, 110622, 110623, 110651, 110648, 110649, 110650, - 110627, 110628, 110629, 110630, 110631, 110632, 110633, 110634, 110642, - 110643, 110644, 110645, 110646, 110647, 110635, 110636, 110637, 110638, - 110639, 110640, 110641, 110806, 110804, 110805, 110807, 110808, 110809, - 110810, 110811, 110812, 110786, 110787, 110788, 110789, 110790, 110791, - 110792, 110793, 110794, 110795, 110796, 110797, 110798, 110799, 110800, - 110801, 110802, 110803, 110744, 110738, 110739, 110740, 110741, 110742, - 110743, 110734, 110727, 110728, 110729, 110730, 110731, 110732, 110733, - 110718, 110719, 110720, 110721, 110722, 110723, 110724, 110725, 110726, - 110745, 110746, 110747, 110748, 110749, 110735, 110736, 110737, 110877, - 110878, 110850, 110851, 110852, 110853, 110854, 110855, 110840, 110841, - 110842, 110843, 110844, 110845, 110833, 110834, 110835, 110836, 110837, - 110838, 110839, 110829, 110830, 110831, 110832, 110846, 110847, 110848, - 110849, 110652, 110653, 110654, 110655, 110656, 110657, 110658, 110659, - 110666, 110667, 110668, 110669, 110670, 110671, 110672, 110673, 110660, - 110661, 110662, 110663, 110664, 110665, 110674, 110675, 110676, 110677, - 110678, 110679, 110680, 110681, 110682, 110683, 110684, 110685, 110702, - 110703, 110704, 110705, 110706, 110707, 110708, 110709, 110710, 110717, - 110711, 110712, 110713, 110714, 110715, 110716, 110701, 110697, 110698, - 110699, 110700, 110690, 110691, 110692, 110693, 110694, 110695, 110696, - 110686, 110687, 110688, 110689, 110856, 110857, 110858, 110859, 110860, - 110861, 110862, 110863, 110864, 110865, 110870, 110871, 110872, 110873, - 110874, 110875, 110876, 110866, 110867, 110868, 110869, 110818, 110813, - 110814, 110815, 110816, 110817, 110823, 110824, 110825, 110826, 110827, - 110828, 110819, 110820, 110821, 110822, 983273, 110607, 110608, 110609, - 110610, 110611, 110602, 110603, 110604, 110605, 110606, 110612, 110613, - 110614, 110598, 110599, 110600, 110601, 8889, 127807, 19922, 19966, - 19958, 19967, 19924, 19946, 19923, 19909, 19947, 19944, 19943, 19956, - 19906, 19962, 19935, 19939, 19920, 19916, 19948, 19917, 19937, 19931, - 19929, 19925, 19911, 19928, 19964, 19945, 19934, 19918, 19930, 19942, - 19950, 19941, 19949, 19938, 19914, 19927, 19936, 19952, 19965, 19912, - 19915, 19926, 19954, 19910, 19932, 19953, 19904, 19933, 19940, 19905, - 19951, 19959, 19957, 19960, 19955, 19961, 19913, 19908, 19921, 19963, - 19907, 19919, 129428, 128262, 9889, 983123, 128644, 128645, 128096, - 12447, 12354, 110879, 110593, 12430, 110929, 110928, 110930, 12419, - 12423, 12421, 12437, 12438, 110898, 12387, 12353, 12359, 12355, 12361, - 12357, 12373, 12379, 12375, 12381, 12377, 12403, 983997, 984000, 983998, - 984001, 983999, 12400, 12409, 12412, 12406, 12435, 12394, 12397, 12395, - 12398, 12396, 12384, 12391, 12386, 12393, 12389, 12364, 12370, 12366, - 12372, 12368, 12399, 12408, 12402, 12411, 12405, 12363, 12369, 12365, - 12371, 12367, 12414, 12417, 12415, 12418, 12416, 12401, 12410, 12404, - 12413, 12407, 12425, 12428, 12426, 12429, 12427, 12383, 12390, 12385, - 12392, 12388, 12374, 12380, 12376, 12382, 12378, 12431, 12433, 12432, - 12434, 12420, 12424, 12422, 12436, 12360, 12356, 12362, 12358, 12446, - 12445, 9964, 128725, 129406, 127802, 129435, 128616, 128617, 128371, - 128029, 127855, 11203, 11043, 8213, 118290, 118287, 117905, 9135, 117893, - 129921, 129910, 129911, 129912, 129913, 129914, 129915, 9146, 9147, 9148, - 9149, 983059, 983141, 983138, 11134, 128677, 117779, 8230, 9897, 117915, - 117769, 118448, 128014, 127943, 128052, 127798, 9749, 9832, 127789, - 127976, 8987, 9203, 8962, 127969, 127968, 127960, 127973, 128298, 8763, - 129693, 983124, 983060, 983142, 983139, 128559, 128175, 129303, 128726, - 8208, 11802, 8259, 8231, 45, 11794, 9102, 11226, 129723, 8372, 127848, - 129482, 127954, 9976, 8801, 10725, 10855, 129706, 12696, 12700, 12693, - 12697, 12695, 12703, 12692, 12699, 12691, 12694, 12688, 12698, 12690, - 12689, 12701, 12702, 12294, 12289, 12275, 12273, 12274, 12272, 12283, - 12282, 12285, 12279, 12280, 12281, 12278, 12277, 12284, 12783, 12287, - 12276, 12286, 12332, 12295, 119670, 119669, 119668, 119667, 119666, - 12999, 12995, 13309, 13310, 13292, 13282, 13299, 13304, 13303, 13306, - 13305, 13302, 13301, 13308, 13300, 13307, 13291, 13281, 13289, 13287, - 13297, 13290, 13294, 13284, 13283, 13293, 13288, 13298, 13286, 13296, - 13285, 13295, 13280, 13003, 13164, 13168, 13167, 13166, 13165, 13156, - 13146, 13157, 13147, 13154, 13152, 13162, 13155, 13159, 13149, 13148, - 13158, 13153, 13163, 13151, 13161, 13150, 13160, 13144, 13145, 12992, - 12997, 12998, 12993, 12994, 12996, 13002, 13000, 13001, 12343, 12351, - 12333, 12331, 12330, 12350, 12290, 12293, 12288, 8887, 8787, 128127, - 67676, 67673, 67675, 67679, 67674, 67672, 67677, 67678, 67656, 67669, - 67651, 67659, 67666, 67667, 67648, 67663, 67650, 67654, 67662, 67665, - 67668, 67657, 67652, 67655, 67658, 67649, 67661, 67653, 67664, 67660, - 67671, 128232, 10716, 128474, 10721, 8710, 983130, 983129, 129781, + 1472, 1475, 1478, 1470, 1519, 118464, 9937, 9096, 11263, 128641, 110597, + 110594, 110595, 110596, 110778, 110779, 110780, 110781, 110782, 110783, + 110784, 110785, 110750, 110759, 110760, 110751, 110752, 110753, 110754, + 110755, 110756, 110757, 110758, 110768, 110769, 110770, 110771, 110772, + 110773, 110774, 110775, 110776, 110777, 110761, 110762, 110763, 110764, + 110765, 110766, 110767, 110615, 110624, 110625, 110626, 110616, 110617, + 110618, 110619, 110620, 110621, 110622, 110623, 110651, 110648, 110649, + 110650, 110627, 110628, 110629, 110630, 110631, 110632, 110633, 110634, + 110642, 110643, 110644, 110645, 110646, 110647, 110635, 110636, 110637, + 110638, 110639, 110640, 110641, 110806, 110804, 110805, 110807, 110808, + 110809, 110810, 110811, 110812, 110786, 110787, 110788, 110789, 110790, + 110791, 110792, 110793, 110794, 110795, 110796, 110797, 110798, 110799, + 110800, 110801, 110802, 110803, 110744, 110738, 110739, 110740, 110741, + 110742, 110743, 110734, 110727, 110728, 110729, 110730, 110731, 110732, + 110733, 110718, 110719, 110720, 110721, 110722, 110723, 110724, 110725, + 110726, 110745, 110746, 110747, 110748, 110749, 110735, 110736, 110737, + 110877, 110878, 110850, 110851, 110852, 110853, 110854, 110855, 110840, + 110841, 110842, 110843, 110844, 110845, 110833, 110834, 110835, 110836, + 110837, 110838, 110839, 110829, 110830, 110831, 110832, 110846, 110847, + 110848, 110849, 110652, 110653, 110654, 110655, 110656, 110657, 110658, + 110659, 110666, 110667, 110668, 110669, 110670, 110671, 110672, 110673, + 110660, 110661, 110662, 110663, 110664, 110665, 110674, 110675, 110676, + 110677, 110678, 110679, 110680, 110681, 110682, 110683, 110684, 110685, + 110702, 110703, 110704, 110705, 110706, 110707, 110708, 110709, 110710, + 110717, 110711, 110712, 110713, 110714, 110715, 110716, 110701, 110697, + 110698, 110699, 110700, 110690, 110691, 110692, 110693, 110694, 110695, + 110696, 110686, 110687, 110688, 110689, 110856, 110857, 110858, 110859, + 110860, 110861, 110862, 110863, 110864, 110865, 110870, 110871, 110872, + 110873, 110874, 110875, 110876, 110866, 110867, 110868, 110869, 110818, + 110813, 110814, 110815, 110816, 110817, 110823, 110824, 110825, 110826, + 110827, 110828, 110819, 110820, 110821, 110822, 983277, 110607, 110608, + 110609, 110610, 110611, 110602, 110603, 110604, 110605, 110606, 110612, + 110613, 110614, 110598, 110599, 110600, 110601, 8889, 127807, 19922, + 19966, 19958, 19967, 19924, 19946, 19923, 19909, 19947, 19944, 19943, + 19956, 19906, 19962, 19935, 19939, 19920, 19916, 19948, 19917, 19937, + 19931, 19929, 19925, 19911, 19928, 19964, 19945, 19934, 19918, 19930, + 19942, 19950, 19941, 19949, 19938, 19914, 19927, 19936, 19952, 19965, + 19912, 19915, 19926, 19954, 19910, 19932, 19953, 19904, 19933, 19940, + 19905, 19951, 19959, 19957, 19960, 19955, 19961, 19913, 19908, 19921, + 19963, 19907, 19919, 129428, 128262, 9889, 983123, 128644, 128645, + 128096, 12447, 12354, 110879, 110593, 12430, 110929, 110928, 110930, + 12419, 12423, 12421, 12437, 12438, 110898, 12387, 12353, 12359, 12355, + 12361, 12357, 12373, 12379, 12375, 12381, 12377, 12403, 983997, 984000, + 983998, 984001, 983999, 12400, 12409, 12412, 12406, 12435, 12394, 12397, + 12395, 12398, 12396, 12384, 12391, 12386, 12393, 12389, 12364, 12370, + 12366, 12372, 12368, 12399, 12408, 12402, 12411, 12405, 12363, 12369, + 12365, 12371, 12367, 12414, 12417, 12415, 12418, 12416, 12401, 12410, + 12404, 12413, 12407, 12425, 12428, 12426, 12429, 12427, 12383, 12390, + 12385, 12392, 12388, 12374, 12380, 12376, 12382, 12378, 12431, 12433, + 12432, 12434, 12420, 12424, 12422, 12436, 12360, 12356, 12362, 12358, + 12446, 12445, 9964, 128725, 129406, 127802, 129435, 128616, 128617, + 128371, 128029, 127855, 11203, 11043, 8213, 118290, 118287, 117905, 9135, + 117893, 129921, 129910, 129911, 129912, 129913, 129914, 129915, 9146, + 9147, 9148, 9149, 983059, 983141, 983138, 11134, 128677, 117779, 8230, + 9897, 117915, 117769, 118448, 128014, 127943, 128052, 127798, 9749, 9832, + 127789, 127976, 8987, 9203, 8962, 127969, 127968, 127960, 127973, 128298, + 8763, 129693, 983124, 983060, 983142, 983139, 128559, 128175, 129303, + 128726, 11226, 128889, 8208, 11802, 8259, 8231, 45, 11794, 9102, 129723, + 8372, 127848, 129482, 127954, 9976, 8801, 10725, 10855, 129706, 12696, + 12700, 12693, 12697, 12695, 12703, 12692, 12699, 12691, 12694, 12688, + 12698, 12690, 12689, 12701, 12702, 12294, 12289, 12275, 12273, 12274, + 12272, 12283, 12282, 12285, 12279, 12280, 12281, 12278, 12277, 12284, + 12783, 12287, 12276, 12286, 12332, 12295, 119670, 119669, 119668, 119667, + 119666, 12999, 12995, 13309, 13310, 13292, 13282, 13299, 13304, 13303, + 13306, 13305, 13302, 13301, 13308, 13300, 13307, 13291, 13281, 13289, + 13287, 13297, 13290, 13294, 13284, 13283, 13293, 13288, 13298, 13286, + 13296, 13285, 13295, 13280, 13003, 13164, 13168, 13167, 13166, 13165, + 13156, 13146, 13157, 13147, 13154, 13152, 13162, 13155, 13159, 13149, + 13148, 13158, 13153, 13163, 13151, 13161, 13150, 13160, 13144, 13145, + 12992, 12997, 12998, 12993, 12994, 12996, 13002, 13000, 13001, 12343, + 12351, 12333, 12331, 12330, 12350, 12290, 12293, 12288, 8887, 8787, + 128127, 67676, 67673, 67675, 67679, 67674, 67672, 67677, 67678, 67656, + 67669, 67651, 67659, 67666, 67667, 67648, 67663, 67650, 67654, 67662, + 67665, 67668, 67657, 67652, 67655, 67658, 67649, 67661, 67653, 67664, + 67660, 67671, 128232, 10716, 128474, 10721, 8710, 983130, 983129, 129781, 126132, 126112, 126126, 126125, 126127, 126131, 126130, 126129, 126113, 126114, 126110, 126111, 126072, 126081, 126108, 126090, 126099, 126078, 126105, 126069, 126087, 126096, 126077, 126104, 126068, 126086, 126095, @@ -11263,254 +11384,255 @@ static const unsigned int dawg_pos_to_codepoint[] = { 9134, 65529, 65531, 65530, 9892, 8745, 10825, 10823, 10820, 10819, 10816, 10827, 8253, 8890, 10812, 117901, 117903, 9688, 129942, 129969, 129972, 9689, 129936, 11800, 11845, 11846, 8766, 9959, 161, 8487, 8276, 191, - 8290, 8292, 8291, 128229, 127982, 129311, 127875, 127983, 127971, 12292, - 9979, 127886, 128304, 128122, 128121, 43453, 43454, 43455, 43457, 43421, - 43422, 43426, 43427, 43398, 43397, 43399, 43407, 43408, 43409, 43412, - 43402, 43403, 43418, 43416, 43428, 43423, 43431, 43432, 43413, 43414, - 43410, 43411, 43429, 43430, 43401, 43435, 43436, 43441, 43439, 43440, - 43424, 43425, 43419, 43420, 43415, 43417, 43396, 43405, 43442, 43437, - 43433, 43438, 43434, 43404, 43406, 43400, 43458, 43466, 43467, 43459, - 43465, 43461, 43464, 43468, 43463, 43486, 43462, 43487, 43460, 43471, - 43456, 43469, 43393, 43443, 43392, 43395, 43394, 43448, 43449, 43444, - 43450, 43445, 43446, 43447, 43452, 43451, 43477, 43476, 43479, 43478, - 43475, 43474, 43472, 43481, 43473, 43480, 129753, 129724, 128086, 128377, - 10781, 129337, 9795, 9909, 129513, 69786, 69787, 69785, 69793, 69792, - 69763, 69764, 69770, 69772, 69784, 69783, 69791, 69790, 69767, 69768, - 69765, 69766, 69777, 69789, 69782, 69794, 69804, 69805, 69806, 69798, - 69797, 69779, 69778, 69776, 69775, 69781, 69780, 69774, 69773, 69796, - 69795, 69788, 69801, 69807, 69802, 69799, 69803, 69800, 69769, 69771, - 69760, 69818, 69761, 69817, 69762, 69822, 69823, 69825, 69824, 69826, - 69808, 69814, 69816, 69811, 69812, 69809, 69810, 69813, 69815, 69821, - 69837, 69819, 69820, 12142, 12164, 12060, 12157, 12100, 12149, 12184, - 12191, 12227, 12068, 12174, 12234, 12205, 12134, 12168, 12219, 12189, - 12088, 12090, 12096, 12160, 12182, 12224, 12190, 12147, 12114, 12118, - 12058, 12176, 12075, 12112, 12045, 12170, 12124, 12070, 12194, 12109, - 12229, 12196, 12139, 12056, 12099, 12034, 12084, 12136, 12120, 12044, - 12111, 12094, 12125, 12243, 12238, 12082, 12159, 12063, 12215, 12062, - 12042, 12241, 12067, 12235, 12043, 12140, 12119, 12207, 12123, 12133, - 12222, 12117, 12226, 12245, 12214, 12217, 12236, 12155, 12188, 12113, - 12065, 12198, 12066, 12146, 12171, 12225, 12200, 12121, 12093, 12095, - 12221, 12092, 12216, 12231, 12054, 12218, 12179, 12037, 12173, 12072, - 12046, 12127, 12152, 12049, 12074, 12107, 12208, 12212, 12041, 12210, - 12131, 12033, 12039, 12199, 12085, 12128, 12161, 12162, 12233, 12165, - 12192, 12077, 12201, 12061, 12105, 12040, 12240, 12102, 12153, 12032, - 12080, 12167, 12048, 12156, 12059, 12126, 12158, 12050, 12183, 12204, - 12097, 12239, 12053, 12078, 12150, 12071, 12187, 12186, 12223, 12228, - 12104, 12098, 12064, 12036, 12057, 12163, 12178, 12185, 12154, 12203, - 12083, 12087, 12135, 12151, 12202, 12035, 12122, 12141, 12180, 12144, - 12076, 12052, 12115, 12091, 12108, 12169, 12143, 12148, 12130, 12089, - 12211, 12073, 12101, 12138, 12103, 12209, 12047, 12220, 12172, 12129, - 12166, 12242, 12237, 12145, 12106, 12081, 12244, 12038, 12086, 12055, - 12181, 12197, 12193, 12175, 12116, 12110, 12177, 12137, 12230, 12213, - 12195, 12069, 12079, 12206, 12051, 12232, 12132, 129432, 3240, 3293, - 3225, 3235, 3230, 3205, 3206, 3216, 3220, 3234, 3233, 3239, 3238, 983205, - 3251, 3250, 3249, 3248, 3232, 3231, 3237, 3236, 3211, 3296, 3212, 3297, - 3253, 3209, 3210, 3218, 3219, 3207, 3208, 3254, 3255, 3256, 3245, 3244, - 3227, 3226, 3224, 3223, 3229, 3228, 3222, 3221, 3243, 3242, 3214, 3215, - 3294, 3257, 3246, 3247, 3285, 3315, 3201, 3200, 3204, 3260, 3261, 3202, - 3313, 3314, 3277, 3203, 3286, 3262, 3272, 3276, 3265, 3266, 3267, 3268, - 3298, 3299, 3274, 3275, 3263, 3264, 3270, 3271, 3307, 3306, 3309, 3308, - 3305, 3304, 3302, 3311, 3303, 3310, 12450, 984009, 984008, 984007, - 984010, 110881, 110880, 110882, 110592, 12499, 984002, 984005, 984003, - 984006, 984004, 12496, 12505, 12508, 12502, 12511, 110583, 110584, - 110585, 110586, 110587, 110589, 110590, 110576, 110577, 110578, 110579, - 110581, 110582, 12510, 12513, 12514, 12512, 12531, 12490, 12493, 12491, - 12494, 12492, 12789, 12792, 12790, 12793, 12791, 12795, 12798, 12796, - 12799, 12797, 12533, 12534, 110933, 12784, 12787, 12483, 12526, 110949, - 110948, 110950, 12515, 12519, 12517, 110951, 12788, 12785, 12786, 12794, - 12449, 12455, 12451, 12457, 12453, 12469, 12475, 12471, 12477, 12473, - 12480, 12487, 12482, 12489, 12485, 12460, 12466, 12462, 12468, 12464, - 12495, 12504, 12498, 12507, 12501, 12459, 12465, 12461, 12467, 12463, - 12497, 12506, 12500, 12509, 12503, 12521, 12524, 12522, 12525, 12523, - 12479, 12486, 12481, 12488, 12484, 12535, 12537, 12536, 12538, 12532, - 12470, 12476, 12472, 12478, 12474, 12527, 12529, 12528, 12530, 12516, - 12520, 12518, 12456, 12452, 12458, 12454, 12542, 12543, 12541, 12539, - 12448, 12540, 12444, 12443, 73476, 73477, 73487, 73523, 73498, 73497, - 73503, 73502, 73508, 73507, 73501, 73500, 73506, 73505, 73480, 73481, - 73482, 73483, 73484, 73485, 73478, 73479, 73494, 73504, 73499, 73509, - 73519, 73520, 73521, 73513, 73512, 73496, 73495, 73493, 73492, 73491, - 73490, 73511, 73510, 73522, 73517, 73514, 73516, 73518, 73515, 73486, - 73488, 73551, 73548, 73546, 73545, 73549, 73543, 73541, 73544, 73550, - 73542, 73547, 73537, 73472, 73562, 73474, 73475, 73473, 73525, 73524, - 73535, 73530, 73534, 73536, 73528, 73529, 73526, 73527, 73540, 73539, - 73557, 73556, 73559, 73558, 73555, 73554, 73552, 73561, 73553, 73560, - 73538, 43299, 43301, 43283, 43295, 43277, 43281, 43284, 43275, 43274, - 43286, 43285, 43279, 43278, 43294, 43282, 43289, 43297, 43288, 43276, - 43292, 43287, 43290, 43296, 43293, 43291, 43280, 43298, 43300, 43310, - 43311, 43308, 43309, 43307, 43303, 43305, 43304, 43302, 43306, 43269, - 43268, 43271, 43270, 43267, 43266, 43264, 43273, 43265, 43272, 119496, - 119506, 119499, 119503, 119493, 119492, 119502, 119497, 119507, 119495, - 119505, 119494, 119504, 119501, 119491, 119500, 119490, 119498, 119488, - 119489, 128331, 8490, 128273, 9000, 128422, 983552, 983553, 983559, - 983558, 983561, 983560, 983557, 983556, 983554, 983563, 983555, 983562, - 128287, 118449, 68163, 68162, 68161, 68160, 68113, 68146, 68112, 68147, - 68148, 68123, 68122, 68128, 68127, 68126, 68121, 68131, 68125, 68124, - 68130, 68129, 68141, 68142, 68143, 68135, 68134, 68118, 68117, 68115, - 68114, 68133, 68132, 68149, 68140, 68145, 68119, 68139, 68136, 68138, - 68137, 68144, 68096, 68165, 68164, 68166, 68167, 68179, 68178, 68183, - 68176, 68182, 68181, 68184, 68180, 68177, 68152, 68153, 68109, 68154, - 68111, 68110, 68099, 68101, 68097, 68102, 68098, 68108, 68159, 68168, - 129711, 101120, 101121, 101122, 101123, 101124, 101125, 101126, 101127, - 101128, 101129, 101130, 101131, 101132, 101133, 101134, 101135, 101136, - 101137, 101138, 101139, 101140, 101141, 101142, 101143, 101144, 101145, - 101146, 101147, 101148, 101149, 101150, 101151, 101152, 101153, 101154, - 101155, 101156, 101157, 101158, 101159, 101160, 101161, 101162, 101163, - 101164, 101165, 101166, 101167, 101168, 101169, 101170, 101171, 101172, - 101173, 101174, 101175, 101176, 101177, 101178, 101179, 101180, 101181, - 101182, 101183, 101184, 101185, 101186, 101187, 101188, 101189, 101190, - 101191, 101192, 101193, 101194, 101195, 101196, 101197, 101198, 101199, - 101200, 101201, 101202, 101203, 101204, 101205, 101206, 101207, 101208, - 101209, 101210, 101211, 101212, 101213, 101214, 101215, 101216, 101217, - 101218, 101219, 101220, 101221, 101222, 101223, 101224, 101225, 101226, - 101227, 101228, 101229, 101230, 101231, 101232, 101233, 101234, 101235, - 101236, 101237, 101238, 101239, 101240, 101241, 101242, 101243, 101244, - 101245, 101246, 101247, 101248, 101249, 101250, 101251, 101252, 101253, - 101254, 101255, 101256, 101257, 101258, 101259, 101260, 101261, 101262, - 101263, 101264, 101265, 101266, 101267, 101268, 101269, 101270, 101271, - 101272, 101273, 101274, 101275, 101276, 101277, 101278, 101279, 101280, - 101281, 101282, 101283, 101284, 101285, 101286, 101287, 101288, 101289, - 101290, 101291, 101292, 101293, 101294, 101295, 101296, 101297, 101298, - 101299, 101300, 101301, 101302, 101303, 101304, 101305, 101306, 101307, - 101308, 101309, 101310, 101311, 101312, 101313, 101314, 101315, 101316, - 101317, 101318, 101319, 101320, 101321, 101322, 101323, 101324, 101325, - 101326, 101327, 101328, 101329, 101330, 101331, 101332, 101333, 101334, - 101335, 101336, 101337, 101338, 101339, 101340, 101341, 101342, 101343, - 101344, 101345, 101346, 101347, 101348, 101349, 101350, 101351, 101352, - 101353, 101354, 101355, 101356, 101357, 101358, 101359, 101360, 101361, - 101362, 101363, 101364, 101365, 101366, 101367, 101368, 101369, 101370, - 101371, 101372, 101373, 101374, 101375, 101584, 101585, 101586, 101587, - 101588, 101589, 101376, 101377, 101378, 101379, 101380, 101381, 101382, - 101383, 101384, 101385, 101386, 101387, 101388, 101389, 101390, 101391, - 101392, 101393, 101394, 101395, 101396, 101397, 101398, 101399, 101400, - 101401, 101402, 101403, 101404, 101405, 101406, 101407, 101408, 101409, - 101410, 101411, 101412, 101413, 101414, 101415, 101416, 101417, 101418, - 101419, 101420, 101421, 101422, 101423, 101424, 101425, 101426, 101427, - 101428, 101429, 101430, 101431, 101432, 101433, 101434, 101435, 101436, - 101437, 101438, 101439, 101440, 101441, 101442, 101443, 101444, 101445, - 101446, 101447, 101448, 101449, 101450, 101451, 101452, 101453, 101454, - 101455, 101456, 101457, 101458, 101459, 101460, 101461, 101462, 101463, - 101464, 101465, 101466, 101467, 101468, 101469, 101470, 101471, 101472, - 101473, 101474, 101475, 101476, 101477, 101478, 101479, 101480, 101481, - 101482, 101483, 101484, 101485, 101486, 101487, 101488, 101489, 101490, - 101491, 101492, 101493, 101494, 101495, 101496, 101497, 101498, 101499, - 101500, 101501, 101502, 101503, 101504, 101505, 101506, 101507, 101508, - 101509, 101510, 101511, 101512, 101513, 101514, 101515, 101516, 101517, - 101518, 101519, 101520, 101521, 101522, 101523, 101524, 101525, 101526, - 101527, 101528, 101529, 101530, 101531, 101532, 101533, 101534, 101535, - 101536, 101537, 101538, 101539, 101540, 101541, 101542, 101543, 101544, - 101545, 101546, 101547, 101548, 101549, 101550, 101551, 101552, 101553, - 101554, 101555, 101556, 101557, 101558, 101559, 101560, 101561, 101562, - 101563, 101564, 101565, 101566, 101567, 101568, 101569, 101570, 101571, - 101572, 101573, 101574, 101575, 101576, 101577, 101578, 101579, 101580, - 101581, 101582, 101583, 101631, 94180, 983960, 983965, 983970, 983975, - 983962, 983964, 983961, 983963, 983957, 983959, 983956, 983958, 983977, - 983979, 983978, 983972, 983974, 983967, 983969, 983971, 983973, 983966, - 983968, 983989, 983983, 983985, 983986, 983987, 983980, 983982, 983984, - 983981, 983976, 983988, 6107, 6052, 6064, 6051, 6067, 6066, 6065, 6055, - 6057, 6058, 6056, 6053, 6054, 6063, 983994, 983991, 983992, 983993, 6061, - 6062, 6059, 6060, 6022, 6024, 6021, 6023, 6017, 6019, 6016, 6018, 6020, - 6030, 6025, 6035, 6037, 6039, 6038, 6046, 6045, 6047, 6032, 6034, 6027, - 6029, 6031, 6033, 6026, 6028, 6049, 6043, 6040, 6042, 6044, 6041, 6036, - 6048, 6050, 6108, 6109, 6095, 6096, 6099, 6091, 6101, 6104, 6102, 6098, - 6094, 6100, 6106, 6092, 6087, 6093, 6090, 6103, 6105, 6088, 6086, 6089, - 6097, 6652, 6636, 6655, 6639, 6653, 6637, 6654, 6638, 6651, 6635, 6650, - 6634, 6133, 6137, 6136, 6134, 6135, 6130, 6132, 6131, 6129, 6128, 6624, - 6648, 6632, 6649, 6633, 6647, 6631, 6646, 6630, 6645, 6629, 6642, 6626, - 6640, 6643, 6627, 6644, 6628, 6641, 6625, 6069, 6068, 6070, 983996, 6082, - 6083, 6085, 6071, 6080, 6072, 6078, 983995, 6084, 6073, 6079, 6074, - 983990, 6075, 6077, 6076, 6081, 6117, 6116, 6119, 6118, 6115, 6114, 6112, - 6121, 6113, 6120, 70204, 70201, 70200, 70208, 70185, 70178, 70179, 70177, - 70155, 70156, 70154, 70172, 70167, 70166, 70173, 70171, 70161, 70160, - 70144, 70145, 70149, 70151, 70165, 70164, 70170, 70169, 70187, 70183, - 70157, 70168, 70163, 70174, 70159, 70158, 70153, 70152, 70176, 70175, - 70186, 70180, 70207, 70182, 70184, 70181, 70148, 70146, 70150, 70147, - 70199, 70206, 70198, 70197, 70196, 70203, 70209, 70188, 70193, 70195, - 70189, 70190, 70192, 70194, 70191, 70202, 70205, 70357, 70358, 70356, - 70333, 70334, 70332, 70345, 70347, 70344, 70352, 70351, 70340, 70339, - 70338, 70320, 70321, 70327, 70329, 70346, 70361, 70343, 70342, 70350, - 70349, 70324, 70325, 70322, 70323, 70335, 70348, 70341, 70353, 70337, - 70336, 70331, 70330, 70355, 70354, 70364, 70365, 70366, 70362, 70359, - 70363, 70360, 70326, 70328, 70377, 70378, 70367, 70368, 70374, 70376, - 70371, 70372, 70369, 70370, 70373, 70375, 70389, 70388, 70391, 70390, - 70387, 70386, 70384, 70393, 70385, 70392, 93521, 93520, 93525, 93524, - 93519, 93518, 93523, 93522, 93512, 93517, 93526, 93530, 93529, 93514, - 93513, 93511, 93510, 93516, 93515, 93509, 93508, 93528, 93527, 93537, - 93536, 93538, 93534, 93531, 93533, 93535, 93532, 93507, 93505, 93549, - 93548, 93504, 93547, 93506, 93539, 93544, 93546, 93541, 93542, 93543, - 93540, 93545, 93551, 93550, 93557, 93556, 93559, 93558, 93555, 93554, - 93552, 93561, 93553, 93560, 128143, 128535, 128537, 128538, 128573, - 128139, 129373, 8365, 128088, 129665, 129486, 129698, 12927, 128040, - 11235, 129404, 127991, 129357, 128030, 129692, 3805, 3804, 983206, - 983207, 3743, 3741, 3807, 3806, 3714, 3716, 3713, 983209, 3747, 3749, - 3730, 3729, 3736, 3728, 3727, 3731, 3726, 3744, 3721, 3718, 3724, 3756, - 3740, 3742, 3739, 3752, 3753, 3754, 3722, 3734, 3735, 3733, 3755, 3758, - 3719, 3725, 3737, 3738, 3720, 3732, 3745, 983208, 3751, 3746, 3757, 3773, - 3772, 3770, 3785, 3786, 3787, 3784, 3760, 3762, 3780, 3763, 3779, 3761, - 3771, 3766, 3767, 3768, 3769, 3776, 3777, 3764, 3765, 3778, 3782, 3790, - 3759, 3797, 3796, 3799, 3798, 3795, 3794, 3792, 3801, 3793, 3800, 3788, - 3789, 128996, 129003, 128309, 128311, 128998, 128994, 129001, 68413, - 68415, 128992, 128310, 128999, 68412, 68414, 118324, 118319, 118322, - 118323, 118303, 118304, 118336, 118331, 118328, 118315, 118314, 118306, - 118316, 118318, 118334, 118335, 118333, 118327, 118339, 118341, 118342, - 118340, 118320, 118338, 118332, 118302, 118325, 118309, 118317, 118307, - 118313, 118326, 118329, 118312, 118330, 118352, 118348, 118351, 118350, - 118347, 118344, 118345, 118343, 118349, 118346, 118305, 118321, 118301, - 118299, 118298, 118310, 118311, 118308, 118300, 118337, 11004, 10201, - 10200, 10782, 128995, 129002, 128308, 128997, 128993, 129000, 9711, - 10923, 10925, 8382, 9790, 127772, 127767, 65, 258, 7858, 7856, 7854, - 7862, 7860, 550, 480, 7840, 512, 196, 478, 197, 506, 7680, 256, 983564, - 194, 7848, 7846, 7844, 7852, 7850, 461, 7842, 260, 983590, 983592, 192, - 193, 514, 195, 570, 198, 508, 482, 42946, 393, 11373, 42808, 42810, - 42802, 42804, 42806, 42812, 66, 386, 42902, 7686, 7684, 7682, 385, 579, - 42822, 42932, 67, 199, 7688, 264, 268, 262, 42948, 391, 42898, 266, 571, - 42960, 42796, 42798, 42862, 42931, 68, 498, 453, 42951, 272, 7696, 7698, - 270, 395, 7694, 7692, 7690, 394, 497, 452, 69, 552, 7708, 202, 983586, - 7874, 7872, 7870, 983584, 7878, 7876, 7704, 282, 278, 983598, 983600, - 7864, 516, 203, 7868, 7706, 274, 7700, 7702, 983570, 983572, 983574, - 7866, 280, 983594, 983596, 200, 201, 518, 276, 582, 439, 494, 440, 42786, - 42788, 42858, 208, 425, 330, 70, 401, 7710, 42904, 71, 290, 284, 486, - 500, 286, 7712, 42912, 403, 288, 484, 577, 42938, 42940, 42942, 404, - 983199, 72, 7722, 7720, 292, 542, 7718, 11367, 7716, 7714, 42922, 294, - 11381, 502, 42790, 73, 520, 7882, 304, 207, 7726, 206, 463, 298, 983566, - 296, 7724, 7880, 302, 983604, 983606, 204, 205, 522, 300, 407, 42873, - 42875, 42877, 42882, 42884, 42886, 406, 42860, 74, 308, 42930, 983608, - 584, 75, 310, 488, 42818, 11369, 7730, 42816, 42820, 7728, 7732, 42914, - 408, 76, 42925, 573, 11360, 7734, 7736, 11362, 319, 456, 321, 315, 7740, - 317, 42824, 313, 7738, 983610, 42970, 42972, 455, 77, 7742, 7746, 7744, - 983612, 11374, 7930, 7932, 42966, 78, 325, 7754, 327, 459, 544, 7752, - 413, 504, 323, 42896, 7750, 7748, 42916, 209, 458, 79, 42826, 42828, 332, - 7760, 7762, 415, 212, 7892, 7890, 7888, 7896, 7894, 465, 214, 554, 558, - 560, 7884, 524, 336, 490, 492, 216, 510, 213, 7758, 7756, 556, 983576, - 983578, 983580, 416, 7902, 7900, 7898, 7906, 7904, 7886, 210, 211, 526, - 334, 42944, 400, 390, 42934, 418, 42830, 546, 80, 42834, 11363, 42832, - 42836, 7764, 420, 7766, 81, 42840, 42838, 82, 342, 344, 7770, 7772, 7768, - 528, 11364, 983614, 340, 7774, 530, 42918, 588, 42842, 42997, 42923, - 42814, 398, 42844, 42955, 83, 352, 7782, 350, 536, 348, 346, 7780, 7778, - 7784, 7776, 42956, 42953, 11390, 983582, 42920, 42949, 586, 42926, 42891, - 7838, 42968, 42924, 399, 84, 354, 7792, 538, 356, 574, 7788, 7786, 7790, - 430, 428, 358, 42878, 11375, 11376, 42893, 42928, 42880, 412, 42929, 581, - 222, 42852, 42854, 388, 423, 444, 42794, 42792, 85, 219, 7798, 467, 220, - 473, 475, 471, 469, 7794, 532, 368, 7908, 431, 7916, 7914, 7912, 7920, - 7918, 7910, 362, 7802, 983568, 983620, 983622, 370, 983616, 983618, 360, - 7800, 7796, 217, 218, 534, 364, 366, 42936, 580, 433, 86, 42846, 7806, - 7804, 434, 42850, 42906, 42908, 42910, 42856, 42848, 87, 372, 7812, 7816, - 7814, 7808, 7810, 11378, 503, 88, 7820, 7818, 89, 374, 376, 7924, 7822, - 7922, 435, 7926, 7934, 221, 562, 7928, 590, 540, 90, 7824, 381, 377, - 11371, 7826, 379, 7828, 11391, 437, 42950, 548, 306, 338, 10013, 43007, - 43005, 43006, 43003, 43004, 42999, 450, 7461, 684, 664, 685, 662, 122638, - 446, 426, 674, 451, 122634, 7431, 7430, 7459, 618, 641, 671, 122628, + 8290, 8292, 8291, 128229, 118471, 118465, 127982, 129311, 127875, 127983, + 127971, 12292, 9979, 127886, 128304, 128122, 128121, 43453, 43454, 43455, + 43457, 43421, 43422, 43426, 43427, 43398, 43397, 43399, 43407, 43408, + 43409, 43412, 43402, 43403, 43418, 43416, 43428, 43423, 43431, 43432, + 43413, 43414, 43410, 43411, 43429, 43430, 43401, 43435, 43436, 43441, + 43439, 43440, 43424, 43425, 43419, 43420, 43415, 43417, 43396, 43405, + 43442, 43437, 43433, 43438, 43434, 43404, 43406, 43400, 43458, 43466, + 43467, 43459, 43465, 43461, 43464, 43468, 43463, 43486, 43462, 43487, + 43460, 43471, 43456, 43469, 43393, 43443, 43395, 43394, 43392, 43448, + 43449, 43444, 43450, 43445, 43446, 43447, 43452, 43451, 43477, 43476, + 43479, 43478, 43475, 43474, 43472, 43481, 43473, 43480, 129753, 129724, + 128086, 128377, 10781, 129337, 9795, 9909, 129513, 69786, 69787, 69785, + 69793, 69792, 69763, 69764, 69770, 69772, 69784, 69783, 69791, 69790, + 69767, 69768, 69765, 69766, 69777, 69789, 69782, 69794, 69804, 69805, + 69806, 69798, 69797, 69779, 69778, 69776, 69775, 69781, 69780, 69774, + 69773, 69796, 69795, 69788, 69801, 69807, 69802, 69799, 69803, 69800, + 69769, 69771, 69760, 69818, 69761, 69817, 69762, 69822, 69823, 69825, + 69824, 69826, 69808, 69814, 69816, 69811, 69812, 69809, 69810, 69813, + 69815, 69821, 69837, 69819, 69820, 12142, 12164, 12060, 12157, 12100, + 12149, 12184, 12191, 12227, 12068, 12174, 12234, 12205, 12134, 12168, + 12219, 12189, 12088, 12090, 12096, 12160, 12182, 12224, 12190, 12147, + 12114, 12118, 12058, 12176, 12075, 12112, 12045, 12170, 12124, 12070, + 12194, 12109, 12229, 12196, 12139, 12056, 12099, 12034, 12084, 12136, + 12120, 12044, 12111, 12094, 12125, 12243, 12238, 12082, 12159, 12063, + 12215, 12062, 12042, 12241, 12067, 12235, 12043, 12140, 12119, 12207, + 12123, 12133, 12222, 12117, 12226, 12245, 12214, 12217, 12236, 12155, + 12188, 12113, 12065, 12198, 12066, 12146, 12171, 12225, 12200, 12121, + 12093, 12095, 12221, 12092, 12216, 12231, 12054, 12218, 12179, 12037, + 12173, 12072, 12046, 12127, 12152, 12049, 12074, 12107, 12208, 12212, + 12041, 12210, 12131, 12033, 12039, 12199, 12085, 12128, 12161, 12162, + 12233, 12165, 12192, 12077, 12201, 12061, 12105, 12040, 12240, 12102, + 12153, 12032, 12080, 12167, 12048, 12156, 12059, 12126, 12158, 12050, + 12183, 12204, 12097, 12239, 12053, 12078, 12150, 12071, 12187, 12186, + 12223, 12228, 12104, 12098, 12064, 12036, 12057, 12163, 12178, 12185, + 12154, 12203, 12083, 12087, 12135, 12151, 12202, 12035, 12122, 12141, + 12180, 12144, 12076, 12052, 12115, 12091, 12108, 12169, 12143, 12148, + 12130, 12089, 12211, 12073, 12101, 12138, 12103, 12209, 12047, 12220, + 12172, 12129, 12166, 12242, 12237, 12145, 12106, 12081, 12244, 12038, + 12086, 12055, 12181, 12197, 12193, 12175, 12116, 12110, 12177, 12137, + 12230, 12213, 12195, 12069, 12079, 12206, 12051, 12232, 12132, 129432, + 3240, 3293, 3225, 3235, 3230, 3205, 3206, 3216, 3220, 3234, 3233, 3239, + 3238, 983205, 3251, 3250, 3249, 3248, 3232, 3231, 3237, 3236, 3211, 3296, + 3212, 3297, 3253, 3209, 3210, 3218, 3219, 3207, 3208, 3254, 3255, 3256, + 3245, 3244, 3227, 3226, 3224, 3223, 3229, 3228, 3222, 3221, 3243, 3242, + 3214, 3215, 3294, 3257, 3246, 3247, 3285, 3315, 3201, 3200, 3204, 3260, + 3261, 3202, 3313, 3314, 3277, 3203, 3292, 3286, 3262, 3272, 3276, 3265, + 3266, 3267, 3268, 3298, 3299, 3274, 3275, 3263, 3264, 3270, 3271, 3307, + 3306, 3309, 3308, 3305, 3304, 3302, 3311, 3303, 3310, 12450, 984009, + 984008, 984007, 984010, 110881, 110880, 110882, 110592, 12499, 984002, + 984005, 984003, 984006, 984004, 12496, 12505, 12508, 12502, 12511, + 110583, 110584, 110585, 110586, 110587, 110589, 110590, 110576, 110577, + 110578, 110579, 110581, 110582, 12510, 12513, 12514, 12512, 12531, 12490, + 12493, 12491, 12494, 12492, 12789, 12792, 12790, 12793, 12791, 12795, + 12798, 12796, 12799, 12797, 12533, 12534, 110933, 12784, 12787, 12483, + 12526, 110949, 110948, 110950, 12515, 12519, 12517, 110951, 12788, 12785, + 12786, 12794, 12449, 12455, 12451, 12457, 12453, 12469, 12475, 12471, + 12477, 12473, 12480, 12487, 12482, 12489, 12485, 12460, 12466, 12462, + 12468, 12464, 12495, 12504, 12498, 12507, 12501, 12459, 12465, 12461, + 12467, 12463, 12497, 12506, 12500, 12509, 12503, 12521, 12524, 12522, + 12525, 12523, 12479, 12486, 12481, 12488, 12484, 12535, 12537, 12536, + 12538, 12532, 12470, 12476, 12472, 12478, 12474, 12527, 12529, 12528, + 12530, 12516, 12520, 12518, 12456, 12452, 12458, 12454, 12542, 12543, + 12541, 12539, 12448, 12540, 12444, 12443, 73476, 73477, 73487, 73523, + 73498, 73497, 73503, 73502, 73508, 73507, 73501, 73500, 73506, 73505, + 73480, 73481, 73482, 73483, 73484, 73485, 73478, 73479, 73494, 73504, + 73499, 73509, 73519, 73520, 73521, 73513, 73512, 73496, 73495, 73493, + 73492, 73491, 73490, 73511, 73510, 73522, 73517, 73514, 73516, 73518, + 73515, 73486, 73488, 73551, 73548, 73546, 73545, 73549, 73543, 73541, + 73544, 73550, 73542, 73547, 73537, 73472, 73562, 73474, 73475, 73473, + 73525, 73524, 73535, 73530, 73534, 73536, 73528, 73529, 73526, 73527, + 73540, 73539, 73557, 73556, 73559, 73558, 73555, 73554, 73552, 73561, + 73553, 73560, 73538, 43299, 43301, 43283, 43295, 43277, 43281, 43284, + 43275, 43274, 43286, 43285, 43279, 43278, 43294, 43282, 43289, 43297, + 43288, 43276, 43292, 43287, 43290, 43296, 43293, 43291, 43280, 43298, + 43300, 43310, 43311, 43308, 43309, 43307, 43303, 43305, 43304, 43302, + 43306, 43269, 43268, 43271, 43270, 43267, 43266, 43264, 43273, 43265, + 43272, 119496, 119506, 119499, 119503, 119493, 119492, 119502, 119497, + 119507, 119495, 119505, 119494, 119504, 119501, 119491, 119500, 119490, + 119498, 119488, 119489, 128331, 8490, 128273, 9000, 128422, 983552, + 983553, 983559, 983558, 983561, 983560, 983557, 983556, 983554, 983563, + 983555, 983562, 128287, 118449, 68163, 68162, 68161, 68160, 68113, 68146, + 68112, 68147, 68148, 68123, 68122, 68128, 68127, 68126, 68121, 68131, + 68125, 68124, 68130, 68129, 68141, 68142, 68143, 68135, 68134, 68118, + 68117, 68115, 68114, 68133, 68132, 68149, 68140, 68145, 68119, 68139, + 68136, 68138, 68137, 68144, 68096, 68165, 68164, 68166, 68167, 68179, + 68178, 68183, 68176, 68182, 68181, 68184, 68180, 68177, 68152, 68153, + 68109, 68154, 68111, 68110, 68099, 68101, 68097, 68102, 68098, 68108, + 68159, 68168, 129711, 101120, 101121, 101122, 101123, 101124, 101125, + 101126, 101127, 101128, 101129, 101130, 101131, 101132, 101133, 101134, + 101135, 101136, 101137, 101138, 101139, 101140, 101141, 101142, 101143, + 101144, 101145, 101146, 101147, 101148, 101149, 101150, 101151, 101152, + 101153, 101154, 101155, 101156, 101157, 101158, 101159, 101160, 101161, + 101162, 101163, 101164, 101165, 101166, 101167, 101168, 101169, 101170, + 101171, 101172, 101173, 101174, 101175, 101176, 101177, 101178, 101179, + 101180, 101181, 101182, 101183, 101184, 101185, 101186, 101187, 101188, + 101189, 101190, 101191, 101192, 101193, 101194, 101195, 101196, 101197, + 101198, 101199, 101200, 101201, 101202, 101203, 101204, 101205, 101206, + 101207, 101208, 101209, 101210, 101211, 101212, 101213, 101214, 101215, + 101216, 101217, 101218, 101219, 101220, 101221, 101222, 101223, 101224, + 101225, 101226, 101227, 101228, 101229, 101230, 101231, 101232, 101233, + 101234, 101235, 101236, 101237, 101238, 101239, 101240, 101241, 101242, + 101243, 101244, 101245, 101246, 101247, 101248, 101249, 101250, 101251, + 101252, 101253, 101254, 101255, 101256, 101257, 101258, 101259, 101260, + 101261, 101262, 101263, 101264, 101265, 101266, 101267, 101268, 101269, + 101270, 101271, 101272, 101273, 101274, 101275, 101276, 101277, 101278, + 101279, 101280, 101281, 101282, 101283, 101284, 101285, 101286, 101287, + 101288, 101289, 101290, 101291, 101292, 101293, 101294, 101295, 101296, + 101297, 101298, 101299, 101300, 101301, 101302, 101303, 101304, 101305, + 101306, 101307, 101308, 101309, 101310, 101311, 101312, 101313, 101314, + 101315, 101316, 101317, 101318, 101319, 101320, 101321, 101322, 101323, + 101324, 101325, 101326, 101327, 101328, 101329, 101330, 101331, 101332, + 101333, 101334, 101335, 101336, 101337, 101338, 101339, 101340, 101341, + 101342, 101343, 101344, 101345, 101346, 101347, 101348, 101349, 101350, + 101351, 101352, 101353, 101354, 101355, 101356, 101357, 101358, 101359, + 101360, 101361, 101362, 101363, 101364, 101365, 101366, 101367, 101368, + 101369, 101370, 101371, 101372, 101373, 101374, 101375, 101584, 101585, + 101586, 101587, 101588, 101589, 101376, 101377, 101378, 101379, 101380, + 101381, 101382, 101383, 101384, 101385, 101386, 101387, 101388, 101389, + 101390, 101391, 101392, 101393, 101394, 101395, 101396, 101397, 101398, + 101399, 101400, 101401, 101402, 101403, 101404, 101405, 101406, 101407, + 101408, 101409, 101410, 101411, 101412, 101413, 101414, 101415, 101416, + 101417, 101418, 101419, 101420, 101421, 101422, 101423, 101424, 101425, + 101426, 101427, 101428, 101429, 101430, 101431, 101432, 101433, 101434, + 101435, 101436, 101437, 101438, 101439, 101440, 101441, 101442, 101443, + 101444, 101445, 101446, 101447, 101448, 101449, 101450, 101451, 101452, + 101453, 101454, 101455, 101456, 101457, 101458, 101459, 101460, 101461, + 101462, 101463, 101464, 101465, 101466, 101467, 101468, 101469, 101470, + 101471, 101472, 101473, 101474, 101475, 101476, 101477, 101478, 101479, + 101480, 101481, 101482, 101483, 101484, 101485, 101486, 101487, 101488, + 101489, 101490, 101491, 101492, 101493, 101494, 101495, 101496, 101497, + 101498, 101499, 101500, 101501, 101502, 101503, 101504, 101505, 101506, + 101507, 101508, 101509, 101510, 101511, 101512, 101513, 101514, 101515, + 101516, 101517, 101518, 101519, 101520, 101521, 101522, 101523, 101524, + 101525, 101526, 101527, 101528, 101529, 101530, 101531, 101532, 101533, + 101534, 101535, 101536, 101537, 101538, 101539, 101540, 101541, 101542, + 101543, 101544, 101545, 101546, 101547, 101548, 101549, 101550, 101551, + 101552, 101553, 101554, 101555, 101556, 101557, 101558, 101559, 101560, + 101561, 101562, 101563, 101564, 101565, 101566, 101567, 101568, 101569, + 101570, 101571, 101572, 101573, 101574, 101575, 101576, 101577, 101578, + 101579, 101580, 101581, 101582, 101583, 101631, 94180, 983960, 983965, + 983970, 983975, 983962, 983964, 983961, 983963, 983957, 983959, 983956, + 983958, 983977, 983979, 983978, 983972, 983974, 983967, 983969, 983971, + 983973, 983966, 983968, 983989, 983983, 983985, 983986, 983987, 983980, + 983982, 983984, 983981, 983976, 983988, 6107, 6052, 6064, 6051, 6067, + 6066, 6065, 6055, 6057, 6058, 6056, 6053, 6054, 6063, 983994, 983991, + 983992, 983993, 6061, 6062, 6059, 6060, 6022, 6024, 6021, 6023, 6017, + 6019, 6016, 6018, 6020, 6030, 6025, 6035, 6037, 6039, 6038, 6046, 6045, + 6047, 6032, 6034, 6027, 6029, 6031, 6033, 6026, 6028, 6049, 6043, 6040, + 6042, 6044, 6041, 6036, 6048, 6050, 6108, 6109, 6095, 6096, 6099, 6091, + 6101, 6104, 6102, 6098, 6094, 6100, 6106, 6092, 6087, 6090, 6093, 6103, + 6105, 6088, 6086, 6089, 6097, 6652, 6636, 6655, 6639, 6653, 6637, 6654, + 6638, 6651, 6635, 6650, 6634, 6133, 6137, 6136, 6134, 6135, 6130, 6132, + 6131, 6129, 6128, 6624, 6648, 6632, 6649, 6633, 6647, 6631, 6646, 6630, + 6645, 6629, 6642, 6626, 6640, 6643, 6627, 6644, 6628, 6641, 6625, 6069, + 6068, 6070, 983996, 6082, 6083, 6085, 6071, 6080, 6072, 6078, 983995, + 6084, 6073, 6079, 6074, 983990, 6075, 6077, 6076, 6081, 6117, 6116, 6119, + 6118, 6115, 6114, 6112, 6121, 6113, 6120, 70204, 70201, 70200, 70208, + 70185, 70178, 70179, 70177, 70155, 70156, 70154, 70172, 70167, 70166, + 70173, 70171, 70161, 70160, 70144, 70145, 70149, 70151, 70165, 70164, + 70170, 70169, 70187, 70183, 70157, 70168, 70163, 70174, 70159, 70158, + 70153, 70152, 70176, 70175, 70186, 70180, 70207, 70182, 70184, 70181, + 70148, 70146, 70150, 70147, 70199, 70206, 70198, 70197, 70196, 70203, + 70209, 70188, 70193, 70195, 70189, 70190, 70192, 70194, 70191, 70202, + 70205, 70357, 70358, 70356, 70333, 70334, 70332, 70345, 70347, 70344, + 70352, 70351, 70340, 70339, 70338, 70320, 70321, 70327, 70329, 70346, + 70361, 70343, 70342, 70350, 70349, 70324, 70325, 70322, 70323, 70335, + 70348, 70341, 70353, 70337, 70336, 70331, 70330, 70355, 70354, 70364, + 70365, 70366, 70362, 70359, 70363, 70360, 70326, 70328, 70377, 70378, + 70367, 70368, 70374, 70376, 70371, 70372, 70369, 70370, 70373, 70375, + 70389, 70388, 70391, 70390, 70387, 70386, 70384, 70393, 70385, 70392, + 93521, 93520, 93525, 93524, 93519, 93518, 93523, 93522, 93512, 93517, + 93526, 93530, 93529, 93514, 93513, 93511, 93510, 93516, 93515, 93509, + 93508, 93528, 93527, 93537, 93536, 93538, 93534, 93531, 93533, 93535, + 93532, 93507, 93505, 93549, 93548, 93504, 93547, 93506, 93539, 93544, + 93546, 93541, 93542, 93543, 93540, 93545, 93551, 93550, 93557, 93556, + 93559, 93558, 93555, 93554, 93552, 93561, 93553, 93560, 128143, 128535, + 128537, 128538, 128573, 128139, 129373, 8365, 128088, 129665, 129486, + 129698, 12927, 128040, 11235, 129404, 127991, 129357, 128030, 129692, + 128728, 917505, 3805, 3804, 983206, 983207, 3743, 3741, 3807, 3806, 3714, + 3716, 3713, 983209, 3747, 3749, 3730, 3729, 3736, 3728, 3727, 3731, 3726, + 3744, 3721, 3718, 3724, 3756, 3740, 3742, 3739, 3752, 3753, 3754, 3722, + 3734, 3735, 3733, 3755, 3758, 3719, 3725, 3737, 3738, 3720, 3732, 3745, + 983208, 3751, 3746, 3757, 3773, 3772, 3770, 3785, 3786, 3787, 3784, 3760, + 3762, 3780, 3763, 3779, 3761, 3771, 3766, 3767, 3768, 3769, 3776, 3777, + 3764, 3765, 3778, 3782, 3790, 3759, 3797, 3796, 3799, 3798, 3795, 3794, + 3792, 3801, 3793, 3800, 3788, 3789, 128996, 129003, 128309, 128311, + 128998, 128994, 129001, 68413, 68415, 128992, 128310, 128999, 68412, + 68414, 118324, 118319, 118322, 118323, 118303, 118304, 118336, 118331, + 118328, 118315, 118314, 118306, 118316, 118318, 118334, 118335, 118333, + 118327, 118339, 118341, 118342, 118340, 118320, 118338, 118332, 118302, + 118325, 118309, 118317, 118307, 118313, 118326, 118329, 118312, 118330, + 118352, 118348, 118351, 118350, 118347, 118344, 118345, 118343, 118349, + 118346, 118305, 118321, 118301, 118299, 118298, 118310, 118311, 118308, + 118300, 118337, 11004, 10201, 10200, 10782, 128995, 129002, 128308, + 128997, 128993, 129000, 9711, 10923, 10925, 8382, 9790, 127772, 127767, + 65, 258, 7858, 7856, 7854, 7862, 7860, 550, 480, 7840, 512, 196, 478, + 197, 506, 7680, 256, 983564, 194, 7848, 7846, 7844, 7852, 7850, 461, + 7842, 260, 983590, 983592, 192, 193, 514, 195, 570, 198, 508, 482, 42946, + 393, 11373, 42808, 42810, 42802, 42804, 42806, 42812, 66, 386, 42902, + 7686, 7684, 7682, 385, 579, 42822, 42932, 67, 199, 7688, 264, 268, 262, + 42948, 391, 42898, 266, 571, 42960, 42796, 42798, 42862, 42931, 68, 498, + 453, 42951, 272, 7696, 7698, 270, 395, 7694, 7692, 7690, 394, 497, 452, + 42964, 42962, 69, 552, 7708, 202, 983586, 7874, 7872, 7870, 983584, 7878, + 7876, 7704, 282, 278, 983598, 983600, 7864, 516, 203, 7868, 7706, 274, + 7700, 7702, 983570, 983572, 983574, 7866, 280, 983594, 983596, 200, 201, + 518, 276, 582, 439, 494, 440, 42786, 42788, 42858, 208, 425, 330, 70, + 401, 7710, 42904, 71, 290, 284, 486, 500, 286, 7712, 42912, 403, 288, + 484, 577, 42938, 42940, 42942, 404, 983199, 72, 7722, 7720, 292, 542, + 7718, 11367, 7716, 7714, 42922, 294, 11381, 502, 42790, 73, 520, 7882, + 304, 207, 7726, 206, 463, 298, 983566, 296, 7724, 7880, 302, 983604, + 983606, 204, 205, 522, 300, 407, 42873, 42875, 42877, 42882, 42884, + 42886, 406, 42860, 74, 308, 42930, 983608, 584, 75, 310, 488, 42818, + 11369, 7730, 42816, 42820, 7728, 7732, 42914, 408, 76, 42925, 573, 11360, + 7734, 7736, 11362, 319, 456, 321, 315, 7740, 317, 42824, 313, 7738, + 983610, 42970, 42972, 455, 77, 7742, 7746, 7744, 983612, 11374, 7930, + 7932, 42966, 78, 325, 7754, 327, 459, 544, 7752, 413, 504, 323, 42896, + 7750, 7748, 42916, 209, 458, 79, 42826, 42828, 332, 7760, 7762, 415, 212, + 7892, 7890, 7888, 7896, 7894, 465, 214, 554, 558, 560, 7884, 524, 336, + 490, 492, 216, 510, 213, 7758, 7756, 556, 983576, 983578, 983580, 416, + 7902, 7900, 7898, 7906, 7904, 7886, 210, 211, 526, 334, 42944, 400, 390, + 42934, 418, 42830, 546, 80, 42834, 11363, 42832, 42836, 7764, 420, 7766, + 42958, 81, 42840, 42838, 82, 342, 344, 7770, 7772, 7768, 528, 983614, + 11364, 340, 7774, 530, 42918, 588, 42842, 42997, 42923, 42814, 398, + 42844, 42955, 83, 352, 7782, 350, 536, 348, 346, 7780, 7778, 7784, 7776, + 42956, 42953, 11390, 983582, 42920, 42949, 586, 42926, 42891, 7838, + 42968, 42924, 399, 84, 354, 7792, 538, 356, 574, 7788, 7786, 7790, 430, + 428, 358, 42878, 11375, 11376, 42893, 42928, 42880, 412, 42929, 581, 222, + 42852, 42854, 388, 444, 423, 42794, 42792, 85, 219, 7798, 467, 220, 473, + 475, 471, 469, 7794, 532, 368, 7908, 431, 7916, 7914, 7912, 7920, 7918, + 7910, 362, 7802, 983568, 983620, 983622, 370, 983616, 983618, 360, 7800, + 7796, 217, 218, 534, 364, 366, 42936, 580, 433, 86, 42846, 7806, 7804, + 434, 42850, 42906, 42908, 42910, 42856, 42848, 87, 372, 7812, 7816, 7814, + 7808, 7810, 11378, 503, 88, 7820, 7818, 89, 374, 376, 7924, 7822, 7922, + 435, 7926, 7934, 221, 562, 7928, 590, 540, 90, 7824, 381, 377, 11371, + 7826, 379, 7828, 11391, 437, 42950, 548, 306, 338, 10013, 43007, 43005, + 43006, 43003, 43004, 42999, 450, 7461, 684, 664, 685, 662, 122638, 446, + 661, 426, 674, 451, 122634, 7431, 7430, 7459, 618, 641, 671, 122628, 7436, 7439, 7440, 630, 7445, 640, 7438, 7449, 43846, 42870, 7451, 11387, 122626, 122640, 43002, 7450, 665, 7427, 610, 667, 7424, 7425, 7428, 7429, 42800, 668, 7434, 7435, 7437, 628, 7448, 42927, 42801, 7452, 7456, 7457, - 655, 7458, 663, 122639, 42895, 443, 447, 448, 449, 660, 673, 661, 7460, - 422, 7547, 7550, 97, 259, 7859, 7857, 7855, 7863, 7861, 551, 481, 7841, - 513, 228, 479, 229, 507, 7681, 7834, 7567, 257, 983565, 226, 7849, 7847, - 7845, 7853, 7851, 462, 7843, 261, 983591, 983593, 224, 225, 515, 227, - 11365, 43825, 230, 983624, 509, 483, 42947, 593, 7568, 42809, 42811, - 42803, 42805, 42807, 42813, 98, 387, 42903, 7687, 7532, 7552, 7685, 7683, - 595, 384, 43824, 43827, 629, 43853, 43837, 43838, 43826, 42823, 7447, - 42933, 99, 231, 7689, 265, 597, 269, 263, 42900, 122653, 392, 42899, 267, - 572, 43859, 43860, 43861, 42961, 666, 631, 606, 42797, 42799, 42863, 100, + 655, 7458, 663, 122639, 42895, 443, 447, 448, 449, 660, 673, 7460, 422, + 7547, 7550, 97, 259, 7859, 7857, 7855, 7863, 7861, 551, 481, 7841, 513, + 228, 479, 229, 507, 7681, 7834, 7567, 257, 983565, 226, 7849, 7847, 7845, + 7853, 7851, 462, 7843, 261, 983591, 983593, 224, 225, 515, 227, 11365, + 43825, 230, 983624, 509, 483, 42947, 593, 7568, 42809, 42811, 42803, + 42805, 42807, 42813, 98, 387, 42903, 7687, 7532, 7552, 7685, 7683, 595, + 384, 43824, 43827, 629, 43853, 43837, 43838, 43826, 42823, 7447, 42933, + 99, 231, 7689, 265, 597, 269, 263, 42900, 122653, 392, 42899, 267, 572, + 43859, 43860, 43861, 42961, 666, 631, 606, 42797, 42799, 42863, 100, 42952, 273, 396, 598, 7697, 7699, 545, 271, 122661, 7533, 7695, 599, 7569, 7553, 7693, 7691, 676, 122642, 122649, 7839, 567, 607, 644, 305, 42965, 43848, 43850, 42963, 499, 454, 675, 677, 43878, 568, 42865, 101, @@ -11539,41 +11661,41 @@ static const unsigned int dawg_pos_to_codepoint[] = { 7757, 557, 983577, 983579, 983581, 417, 7903, 7901, 7899, 7907, 7905, 7887, 242, 243, 527, 335, 122651, 42945, 596, 983625, 983626, 7575, 43839, 43874, 603, 7571, 42935, 419, 42831, 547, 112, 42835, 7549, 42833, - 42837, 7765, 7537, 7560, 421, 7767, 632, 113, 42841, 672, 587, 42839, - 569, 114, 343, 43849, 345, 7771, 7773, 7769, 529, 638, 7539, 122646, - 7775, 636, 637, 983615, 122664, 7538, 341, 531, 7561, 42919, 589, 43847, - 42843, 42998, 604, 7572, 605, 639, 122625, 600, 122631, 8580, 42815, - 122627, 42869, 42845, 612, 115, 347, 7781, 353, 7783, 351, 537, 349, - 122654, 7779, 7785, 7777, 42957, 42954, 575, 983583, 122665, 7540, 7562, - 42921, 642, 42892, 43872, 601, 983629, 983630, 7573, 602, 43851, 43852, - 609, 43830, 223, 7441, 7442, 7443, 7455, 7454, 7453, 42969, 645, 43845, - 116, 355, 7793, 539, 566, 357, 11366, 7831, 7789, 7787, 122666, 7541, - 7791, 427, 429, 122633, 648, 359, 679, 122647, 122652, 7546, 254, 42853, - 42855, 389, 424, 445, 7446, 42795, 397, 613, 686, 687, 7433, 42879, 7444, - 43842, 43841, 43843, 43844, 7432, 633, 43880, 122645, 634, 122632, 11385, - 635, 647, 122637, 652, 983627, 983628, 592, 594, 7426, 623, 624, 654, - 122630, 43857, 477, 7543, 670, 42881, 653, 42871, 680, 678, 43879, 11383, - 42793, 117, 649, 43855, 251, 7799, 468, 252, 474, 476, 472, 470, 7795, - 533, 369, 7909, 432, 7917, 7915, 7913, 7921, 7919, 7911, 363, 7803, - 983569, 983621, 983623, 371, 983617, 983619, 7577, 367, 361, 7801, 7797, - 249, 43854, 42937, 250, 535, 365, 43858, 650, 7551, 7531, 43856, 42872, - 43875, 118, 42847, 7807, 7564, 11380, 11377, 7805, 651, 42851, 42907, - 42909, 42911, 42857, 42849, 119, 373, 7813, 7817, 7815, 7809, 7811, 7832, - 11379, 120, 7821, 7819, 43863, 43864, 43865, 43862, 7565, 121, 375, 255, - 7925, 7823, 7923, 436, 7927, 7935, 43866, 591, 253, 563, 7929, 7833, 541, - 122, 378, 7825, 657, 382, 11372, 7827, 380, 7829, 576, 438, 7542, 7566, - 656, 549, 64256, 64259, 64260, 64257, 64258, 307, 64261, 64262, 339, - 8347, 8340, 8336, 8337, 8341, 7522, 11388, 8342, 8343, 8344, 8345, 8338, - 8346, 7523, 8348, 7524, 7525, 8339, 917505, 129726, 127811, 129388, - 129897, 129916, 129947, 10203, 10202, 129899, 129917, 117902, 128494, - 12296, 10641, 10643, 11058, 11056, 10576, 10571, 10570, 10574, 12304, - 10647, 123, 9128, 9129, 9127, 12300, 8968, 9948, 10714, 12298, 8220, - 11816, 11780, 129287, 129284, 129285, 129283, 129286, 9686, 11240, 9612, - 129977, 117924, 118288, 129970, 118285, 118283, 118435, 118440, 129940, - 129932, 128379, 128709, 11804, 10204, 9614, 9615, 129999, 10197, 8596, - 8700, 8697, 8622, 10568, 8660, 10500, 8654, 8621, 11012, 8703, 11020, - 129112, 11108, 11788, 10181, 8907, 9609, 8216, 11814, 128488, 91, 9123, - 9121, 10639, 10637, 8261, 10635, 11863, 11861, 9122, 11778, 10703, + 42837, 7765, 7537, 7560, 421, 7767, 42959, 632, 113, 42841, 672, 587, + 42839, 569, 114, 343, 43849, 345, 7771, 7773, 7769, 529, 638, 7539, + 122646, 7775, 636, 983615, 637, 122664, 7538, 341, 531, 7561, 42919, 589, + 43847, 42843, 42998, 604, 7572, 605, 639, 122625, 600, 122631, 8580, + 42815, 122627, 42869, 42845, 612, 115, 347, 7781, 353, 7783, 351, 537, + 349, 122654, 7779, 7785, 7777, 42957, 42954, 575, 983583, 122665, 7540, + 7562, 42921, 642, 42892, 43872, 601, 983629, 983630, 7573, 602, 43851, + 43852, 609, 43830, 223, 7441, 7442, 7443, 7455, 7454, 7453, 42969, 645, + 43845, 116, 355, 7793, 539, 566, 357, 11366, 7831, 7789, 7787, 122666, + 7541, 7791, 427, 429, 122633, 648, 359, 679, 122647, 122652, 7546, 254, + 42853, 42855, 389, 445, 424, 7446, 42795, 397, 613, 686, 687, 7433, + 42879, 7444, 43842, 43841, 43843, 43844, 7432, 633, 43880, 122645, 634, + 122632, 11385, 635, 647, 122637, 652, 983627, 983628, 592, 594, 7426, + 623, 624, 654, 122630, 43857, 477, 7543, 670, 42881, 653, 42871, 680, + 678, 43879, 11383, 42793, 117, 649, 43855, 251, 7799, 468, 252, 474, 476, + 472, 470, 7795, 533, 369, 7909, 432, 7917, 7915, 7913, 7921, 7919, 7911, + 363, 7803, 983569, 983621, 983623, 371, 983617, 983619, 7577, 367, 361, + 7801, 7797, 249, 43854, 42937, 250, 535, 365, 43858, 650, 7551, 7531, + 43856, 42872, 43875, 118, 42847, 7807, 7564, 11380, 11377, 7805, 651, + 42851, 42907, 42909, 42911, 42857, 42849, 119, 373, 7813, 7817, 7815, + 7809, 7811, 7832, 11379, 120, 7821, 7819, 43863, 43864, 43865, 43862, + 7565, 121, 375, 255, 7925, 7823, 7923, 436, 7927, 7935, 43866, 591, 253, + 563, 7929, 7833, 541, 122, 378, 7825, 657, 382, 11372, 7827, 380, 7829, + 576, 438, 7542, 7566, 656, 549, 64256, 64259, 64260, 64257, 64258, 307, + 64261, 64262, 339, 8347, 8340, 8336, 8337, 8341, 7522, 11388, 8342, 8343, + 8344, 8345, 8338, 8346, 7523, 8348, 7524, 7525, 8339, 129726, 127811, + 129388, 129897, 129916, 129947, 10203, 10202, 129899, 129917, 117902, + 128494, 12296, 10641, 10643, 11058, 11056, 10576, 10571, 10570, 10574, + 12304, 10647, 123, 9128, 9129, 9127, 12300, 8968, 9948, 10714, 12298, + 8220, 11816, 11780, 129287, 129284, 129285, 129283, 129286, 9686, 11240, + 9612, 129977, 117924, 118288, 129970, 118285, 118283, 118435, 118440, + 129940, 129932, 128379, 128709, 11804, 10204, 9614, 9615, 129999, 10197, + 8596, 8700, 8697, 8622, 10568, 8660, 10500, 8654, 8621, 11012, 8703, + 11020, 129112, 11108, 11788, 10181, 8907, 9609, 8216, 11814, 128488, 91, + 9123, 9121, 10639, 10637, 8261, 10635, 11863, 11861, 9122, 11778, 10703, 129900, 11785, 117771, 129985, 128492, 118434, 118441, 9610, 9613, 12308, 129998, 8867, 12302, 10627, 12310, 10629, 12314, 12312, 10712, 128398, 9611, 10620, 8970, 130027, 130019, 8905, 40, 9117, 9115, 9116, 11808, @@ -11599,96 +11721,97 @@ static const unsigned int dawg_pos_to_codepoint[] = { 7206, 7212, 7207, 7237, 7236, 7239, 7238, 7235, 7234, 7232, 7241, 7233, 7240, 10897, 10895, 10893, 10899, 10891, 10614, 10889, 10887, 8922, 8934, 8808, 10918, 10920, 10885, 10877, 10881, 10883, 10879, 8818, 8804, 8822, - 8806, 10873, 10875, 8918, 60, 128210, 127898, 127819, 129461, 128626, - 128955, 128969, 128943, 11212, 128964, 10099, 128648, 10098, 9617, - 128937, 128949, 128978, 128960, 129653, 128910, 10072, 128930, 128504, - 9735, 128498, 128497, 6429, 6404, 6403, 6412, 6430, 6411, 6421, 6410, - 6405, 6415, 6425, 6426, 6427, 6419, 6418, 6407, 6406, 6414, 6413, 6409, - 6408, 6402, 6401, 6417, 6416, 6428, 6423, 6420, 6422, 6424, 6458, 6457, - 6459, 6464, 6449, 6452, 6450, 6448, 6456, 6454, 6453, 6455, 6451, 6442, - 6443, 6441, 6432, 6436, 6438, 6440, 6437, 6439, 6435, 6433, 6434, 6400, - 6468, 6469, 6475, 6474, 6477, 6476, 6473, 6472, 6470, 6479, 6471, 6478, - 13007, 10770, 10771, 10772, 983062, 8232, 983068, 983143, 67143, 67146, - 67151, 67165, 67166, 67167, 67157, 67158, 67159, 67160, 67161, 67162, - 67163, 67164, 67171, 67172, 67173, 67168, 67169, 67170, 67174, 67175, - 67176, 67177, 67178, 67179, 67230, 67231, 67180, 67181, 67182, 67183, - 67184, 67185, 67186, 67187, 67188, 67189, 67190, 67191, 67192, 67193, - 67194, 67195, 67196, 67197, 67198, 67199, 67200, 67201, 67202, 67203, - 67204, 67205, 67206, 67207, 67208, 67209, 67210, 67211, 67212, 67213, - 67214, 67215, 67216, 67217, 67218, 67219, 67220, 67221, 67222, 67223, - 67224, 67225, 67226, 67227, 67228, 67229, 67232, 67233, 67234, 67235, - 67236, 67237, 67238, 67239, 67240, 67241, 67242, 67243, 67244, 67245, - 67246, 67247, 67248, 67249, 67250, 67251, 67252, 67253, 67254, 67255, - 67256, 67257, 67258, 67259, 67260, 67261, 67262, 67263, 67264, 67274, - 67275, 67276, 67277, 67278, 67279, 67280, 67281, 67282, 67283, 67284, - 67285, 67286, 67287, 67288, 67289, 67290, 67291, 67292, 67293, 67294, - 67295, 67296, 67297, 67298, 67299, 67300, 67301, 67302, 67303, 67304, - 67265, 67266, 67267, 67268, 67269, 67270, 67271, 67272, 67273, 67325, - 67326, 67327, 67328, 67329, 67330, 67305, 67306, 67307, 67308, 67309, - 67310, 67311, 67312, 67313, 67314, 67315, 67316, 67317, 67318, 67319, - 67320, 67321, 67322, 67323, 67324, 67331, 67332, 67333, 67334, 67335, - 67336, 67337, 67338, 67349, 67350, 67351, 67352, 67353, 67354, 67355, - 67356, 67357, 67358, 67359, 67360, 67361, 67362, 67363, 67364, 67365, - 67366, 67367, 67368, 67378, 67379, 67380, 67381, 67382, 67369, 67370, - 67371, 67372, 67373, 67374, 67375, 67376, 67377, 67339, 67340, 67341, - 67342, 67343, 67344, 67345, 67346, 67347, 67348, 67401, 67404, 67403, - 67402, 67400, 67398, 67393, 67397, 67395, 67394, 67399, 67392, 67396, - 67406, 67408, 67409, 67410, 67407, 67405, 67411, 67413, 67412, 67424, - 67425, 67426, 67427, 67428, 67429, 67430, 67431, 67081, 67082, 67083, - 67084, 67085, 67087, 67088, 67089, 67090, 67091, 67092, 67093, 67094, - 67086, 67095, 67096, 67097, 67098, 67100, 67101, 67102, 67103, 67104, - 67105, 67106, 67107, 67108, 67109, 67110, 67111, 67112, 67113, 67114, - 67115, 67116, 67117, 67118, 67119, 67120, 67121, 67122, 67123, 67124, - 67125, 67126, 67127, 67128, 67129, 67130, 67131, 67132, 67133, 67134, - 67135, 67136, 67137, 67138, 67139, 67140, 67141, 67142, 67072, 67073, - 67074, 67075, 67076, 67077, 67078, 67079, 67080, 67145, 67147, 67148, - 67149, 67150, 67154, 67155, 67144, 67152, 67153, 67156, 67099, 65667, - 65668, 65669, 65670, 65671, 65672, 65673, 65675, 65674, 65677, 65676, - 65665, 65664, 65666, 65681, 65679, 65680, 65682, 65678, 65686, 65685, - 65687, 65693, 65690, 65691, 65692, 65694, 65703, 65696, 65695, 65697, - 65698, 65699, 65701, 65702, 65707, 65706, 65704, 65705, 65708, 65709, - 65710, 65711, 65712, 65713, 65719, 65717, 65714, 65715, 65716, 65718, - 65720, 65721, 65722, 65723, 65724, 65725, 65726, 65727, 65728, 65729, - 65731, 65730, 65732, 65733, 65737, 65734, 65735, 65736, 65738, 65739, - 65740, 65741, 65743, 65742, 65744, 65745, 65747, 65748, 65752, 65749, - 65750, 65751, 65753, 65754, 65755, 65756, 65757, 65779, 65780, 65781, - 65782, 65783, 65784, 65785, 65759, 65760, 65761, 65762, 65763, 65764, - 65765, 65766, 65767, 65768, 65769, 65770, 65771, 65772, 65773, 65774, - 65775, 65776, 65777, 65778, 65758, 65786, 65683, 65684, 65689, 65688, - 65700, 65746, 65561, 65543, 65587, 65582, 65589, 65536, 65541, 65579, - 65566, 65571, 65596, 65569, 65584, 65544, 65559, 65540, 65557, 65560, - 65580, 65606, 65600, 65599, 65577, 65562, 65538, 65573, 65563, 65609, - 65588, 65549, 65568, 65537, 65581, 65574, 65601, 65542, 65593, 65583, - 65594, 65552, 65547, 65605, 65578, 65545, 65585, 65586, 65546, 65570, - 65591, 65564, 65565, 65607, 65610, 65611, 65553, 65590, 65550, 65539, - 65576, 65608, 65551, 65554, 65592, 65603, 65597, 65567, 65572, 65558, - 65555, 65612, 65602, 65556, 65613, 65604, 65620, 65621, 65623, 65624, - 65626, 65627, 65628, 65629, 65622, 65616, 65617, 65619, 65618, 65625, - 128391, 128279, 128482, 128132, 42237, 42235, 42232, 42234, 42236, 42233, - 42206, 42205, 42197, 42196, 42204, 42195, 42228, 42229, 42230, 42213, - 42208, 42224, 42225, 42203, 42202, 42221, 42198, 42216, 42214, 42200, - 42199, 42194, 42193, 42219, 42210, 73648, 42220, 42211, 42212, 42222, - 42223, 42227, 42231, 42192, 42217, 42201, 42209, 42207, 42218, 42215, - 42226, 42238, 42239, 8356, 8374, 129409, 129422, 9806, 128274, 983079, - 983076, 128271, 117785, 117786, 117784, 117783, 117782, 117781, 8743, - 10848, 10846, 10833, 10844, 10842, 10847, 8744, 10841, 10851, 10850, - 10834, 10845, 10843, 10982, 10188, 129688, 10234, 10231, 10206, 10229, - 10235, 10232, 10237, 11059, 10230, 10236, 10233, 10238, 10239, 10205, - 129524, 128884, 129719, 128140, 127977, 128261, 129707, 12319, 8270, - 11847, 11848, 95, 118273, 9691, 118276, 118291, 129935, 118436, 118447, - 9604, 9697, 117765, 128394, 129852, 129853, 129870, 129872, 129868, - 129856, 129871, 129869, 129854, 129873, 129855, 128395, 128396, 128393, - 10559, 117934, 117936, 117932, 117930, 117972, 9695, 117960, 117948, - 117964, 117968, 117952, 117956, 117944, 117940, 117817, 129951, 9722, - 117820, 128397, 118428, 117935, 117937, 117933, 117931, 117973, 9694, - 117961, 117949, 117965, 117969, 117953, 117957, 117945, 117941, 117818, - 10558, 128318, 10065, 129950, 9727, 117823, 117767, 129863, 129865, - 129867, 129864, 129861, 129866, 129859, 129862, 129860, 129857, 129858, - 10195, 118431, 10063, 9998, 9987, 118429, 117821, 118430, 117822, 130021, - 9605, 118425, 118426, 118424, 117816, 118427, 117819, 9602, 9601, 9607, - 9603, 118437, 118446, 9606, 129903, 9674, 10208, 129438, 128557, 127853, - 129523, 128886, 129729, 66190, 66192, 66187, 66196, 66178, 66179, 66199, - 66185, 66200, 66176, 66201, 66177, 66202, 66191, 66193, 66181, 66180, + 8806, 10873, 10875, 8918, 60, 118480, 128210, 127898, 127819, 129461, + 128626, 128955, 128969, 128943, 11212, 128964, 10099, 128648, 10098, + 9617, 128937, 128949, 128978, 128960, 129653, 128910, 10072, 128930, + 128504, 9735, 128498, 128497, 6429, 6404, 6403, 6412, 6430, 6411, 6421, + 6410, 6405, 6415, 6425, 6426, 6427, 6419, 6418, 6407, 6406, 6414, 6413, + 6409, 6408, 6402, 6401, 6417, 6416, 6428, 6423, 6420, 6422, 6424, 6458, + 6457, 6459, 6464, 6449, 6452, 6450, 6448, 6456, 6454, 6453, 6455, 6451, + 6442, 6443, 6441, 6432, 6436, 6438, 6440, 6437, 6439, 6435, 6433, 6434, + 6400, 6468, 6469, 6475, 6474, 6477, 6476, 6473, 6472, 6470, 6479, 6471, + 6478, 13007, 10770, 10771, 10772, 983062, 8232, 983068, 983143, 67143, + 67146, 67151, 67165, 67166, 67167, 67157, 67158, 67159, 67160, 67161, + 67162, 67163, 67164, 67171, 67172, 67173, 67168, 67169, 67170, 67174, + 67175, 67176, 67177, 67178, 67179, 67230, 67231, 67180, 67181, 67182, + 67183, 67184, 67185, 67186, 67187, 67188, 67189, 67190, 67191, 67192, + 67193, 67194, 67195, 67196, 67197, 67198, 67199, 67200, 67201, 67202, + 67203, 67204, 67205, 67206, 67207, 67208, 67209, 67210, 67211, 67212, + 67213, 67214, 67215, 67216, 67217, 67218, 67219, 67220, 67221, 67222, + 67223, 67224, 67225, 67226, 67227, 67228, 67229, 67232, 67233, 67234, + 67235, 67236, 67237, 67238, 67239, 67240, 67241, 67242, 67243, 67244, + 67245, 67246, 67247, 67248, 67249, 67250, 67251, 67252, 67253, 67254, + 67255, 67256, 67257, 67258, 67259, 67260, 67261, 67262, 67263, 67264, + 67274, 67275, 67276, 67277, 67278, 67279, 67280, 67281, 67282, 67283, + 67284, 67285, 67286, 67287, 67288, 67289, 67290, 67291, 67292, 67293, + 67294, 67295, 67296, 67297, 67298, 67299, 67300, 67301, 67302, 67303, + 67304, 67265, 67266, 67267, 67268, 67269, 67270, 67271, 67272, 67273, + 67325, 67326, 67327, 67328, 67329, 67330, 67305, 67306, 67307, 67308, + 67309, 67310, 67311, 67312, 67313, 67314, 67315, 67316, 67317, 67318, + 67319, 67320, 67321, 67322, 67323, 67324, 67331, 67332, 67333, 67334, + 67335, 67336, 67337, 67338, 67349, 67350, 67351, 67352, 67353, 67354, + 67355, 67356, 67357, 67358, 67359, 67360, 67361, 67362, 67363, 67364, + 67365, 67366, 67367, 67368, 67378, 67379, 67380, 67381, 67382, 67369, + 67370, 67371, 67372, 67373, 67374, 67375, 67376, 67377, 67339, 67340, + 67341, 67342, 67343, 67344, 67345, 67346, 67347, 67348, 67401, 67404, + 67403, 67402, 67400, 67398, 67393, 67397, 67395, 67394, 67399, 67392, + 67396, 67408, 67409, 67410, 67406, 67407, 67405, 67411, 67413, 67412, + 67424, 67425, 67426, 67427, 67428, 67429, 67430, 67431, 67081, 67082, + 67083, 67084, 67085, 67087, 67088, 67089, 67090, 67091, 67092, 67093, + 67094, 67086, 67095, 67096, 67097, 67098, 67100, 67101, 67102, 67103, + 67104, 67105, 67106, 67107, 67108, 67109, 67110, 67111, 67112, 67113, + 67114, 67115, 67116, 67117, 67118, 67119, 67120, 67121, 67122, 67123, + 67124, 67125, 67126, 67127, 67128, 67129, 67130, 67131, 67132, 67133, + 67134, 67135, 67136, 67137, 67138, 67139, 67140, 67141, 67142, 67072, + 67073, 67074, 67075, 67076, 67077, 67078, 67079, 67080, 67145, 67147, + 67148, 67149, 67150, 67154, 67155, 67144, 67152, 67153, 67156, 67099, + 65667, 65668, 65669, 65670, 65671, 65672, 65673, 65675, 65674, 65677, + 65676, 65665, 65664, 65666, 65681, 65679, 65680, 65682, 65678, 65686, + 65685, 65687, 65693, 65690, 65691, 65692, 65694, 65703, 65696, 65695, + 65697, 65698, 65699, 65701, 65702, 65707, 65706, 65704, 65705, 65708, + 65709, 65710, 65711, 65712, 65713, 65719, 65717, 65714, 65715, 65716, + 65718, 65720, 65721, 65722, 65723, 65724, 65725, 65726, 65727, 65728, + 65729, 65731, 65730, 65732, 65733, 65737, 65734, 65735, 65736, 65738, + 65739, 65740, 65741, 65743, 65742, 65744, 65745, 65747, 65748, 65752, + 65749, 65750, 65751, 65753, 65754, 65755, 65756, 65757, 65779, 65780, + 65781, 65782, 65783, 65784, 65785, 65759, 65760, 65761, 65762, 65763, + 65764, 65765, 65766, 65767, 65768, 65769, 65770, 65771, 65772, 65773, + 65774, 65775, 65776, 65777, 65778, 65758, 65786, 65683, 65684, 65689, + 65688, 65700, 65746, 65561, 65543, 65587, 65582, 65589, 65536, 65541, + 65579, 65566, 65571, 65596, 65569, 65584, 65544, 65559, 65540, 65557, + 65560, 65580, 65606, 65600, 65599, 65577, 65562, 65538, 65573, 65563, + 65609, 65588, 65549, 65568, 65537, 65581, 65574, 65601, 65542, 65593, + 65583, 65594, 65552, 65547, 65605, 65578, 65545, 65585, 65586, 65546, + 65570, 65591, 65564, 65565, 65607, 65610, 65611, 65553, 65590, 65550, + 65539, 65576, 65608, 65551, 65554, 65592, 65603, 65597, 65567, 65572, + 65558, 65555, 65612, 65602, 65556, 65613, 65604, 65620, 65621, 65623, + 65624, 65626, 65627, 65628, 65629, 65622, 65616, 65617, 65619, 65618, + 65625, 128391, 128279, 128482, 128132, 42237, 42235, 42232, 42234, 42236, + 42233, 42206, 42205, 42197, 42196, 42204, 42195, 42228, 42229, 42230, + 42213, 42208, 42224, 42225, 42203, 42202, 42221, 42198, 42216, 42214, + 42200, 42199, 42194, 42193, 42219, 42210, 73648, 42220, 42211, 42212, + 42222, 42223, 42227, 42231, 42192, 42217, 42201, 42209, 42207, 42218, + 42215, 42226, 42238, 42239, 8356, 8374, 129409, 129422, 9806, 128274, + 983079, 983076, 128271, 117785, 117786, 117784, 117783, 117782, 117781, + 8743, 10848, 10846, 10833, 10844, 10842, 10847, 8744, 10841, 10851, + 10850, 10834, 10845, 10843, 10982, 10188, 129688, 10231, 129240, 10234, + 10206, 10232, 10237, 10229, 10235, 11059, 129236, 10230, 129238, 129239, + 129232, 10236, 10233, 10238, 129233, 129234, 10239, 10205, 129524, + 128884, 129719, 128140, 127977, 128261, 129707, 12319, 8270, 11847, + 11848, 95, 118273, 9691, 118276, 118291, 129935, 118436, 118447, 9604, + 9697, 117765, 128394, 129852, 129853, 129870, 129872, 129868, 129856, + 129871, 129869, 129854, 129873, 129855, 128395, 128396, 128393, 10559, + 117934, 117936, 117932, 117930, 117972, 9695, 117960, 117948, 117964, + 117968, 117952, 117956, 117944, 117940, 117817, 129951, 9722, 117820, + 128397, 118428, 117935, 117937, 117933, 117931, 117973, 9694, 117961, + 117949, 117965, 117969, 117953, 117957, 117945, 117941, 117818, 10558, + 128318, 10065, 129950, 9727, 117823, 117767, 129863, 129865, 129867, + 129864, 129861, 129866, 129859, 129862, 129860, 129857, 129858, 10195, + 118431, 10063, 9998, 9987, 118429, 117821, 118430, 117822, 130021, 9605, + 118425, 118426, 118424, 117816, 118427, 117819, 9602, 9601, 9607, 9603, + 118437, 118446, 9606, 129903, 9674, 10208, 129438, 128557, 127853, + 129523, 128886, 129729, 66190, 66192, 66187, 66196, 66199, 66185, 66200, + 66178, 66179, 66176, 66201, 66177, 66202, 66191, 66193, 66181, 66180, 66203, 66182, 66186, 66189, 66195, 66188, 66197, 66198, 66194, 66183, 66204, 66184, 67887, 67892, 67881, 67895, 67891, 67886, 67872, 67893, 67876, 67894, 67883, 67896, 67873, 67897, 67875, 67889, 67874, 67878, @@ -11856,132 +11979,132 @@ static const unsigned int dawg_pos_to_codepoint[] = { 119526, 119536, 119533, 119523, 119532, 119522, 119530, 119520, 119521, 128470, 175, 8737, 10667, 10666, 10671, 10669, 10670, 10668, 10665, 10664, 10651, 10653, 8798, 127830, 129470, 129471, 93773, 93764, 93790, - 93787, 983270, 93783, 983269, 93782, 93772, 93766, 93791, 93779, 93789, + 93787, 983274, 93783, 983273, 93782, 93772, 93766, 93791, 93779, 93789, 93786, 93776, 93777, 93785, 93775, 93770, 93769, 93771, 93774, 93780, 93760, 93767, 93781, 93788, 93761, 93768, 93778, 93762, 93763, 93784, 93765, 93847, 93827, 93846, 93826, 93845, 93825, 93844, 93829, 93828, 93831, 93830, 93824, 93833, 93832, 93837, 93836, 93834, 93842, 93835, - 93838, 93839, 93843, 93840, 93841, 93805, 93796, 93822, 93819, 983272, - 93815, 983271, 93814, 93804, 93798, 93823, 93811, 93821, 93818, 93808, + 93838, 93839, 93843, 93840, 93841, 93805, 93796, 93822, 93819, 983276, + 93815, 983275, 93814, 93804, 93798, 93823, 93811, 93821, 93818, 93808, 93809, 93817, 93807, 93802, 93801, 93803, 93806, 93812, 93792, 93799, 93813, 93820, 93793, 93800, 93810, 93794, 93795, 93816, 93797, 93849, 93850, 93848, 11859, 11852, 11860, 128901, 9899, 10090, 10091, 128967, - 128965, 128944, 10100, 10088, 10092, 10101, 10089, 10093, 8287, 9900, - 9618, 128971, 128950, 128938, 128963, 128961, 10073, 128974, 128956, - 9898, 128911, 128931, 43761, 44013, 43762, 43760, 44011, 43994, 43989, - 43974, 43746, 43993, 43991, 43751, 43750, 43992, 43986, 43987, 43990, - 43968, 43995, 43976, 43973, 43999, 43977, 44001, 43752, 43747, 43972, - 43998, 43984, 43969, 43753, 43754, 43975, 44000, 43978, 43749, 43748, - 43983, 44002, 43970, 43996, 43971, 43997, 43981, 43988, 43979, 43985, - 43980, 43982, 43744, 43745, 44012, 43763, 43764, 44005, 43757, 43759, - 43758, 44009, 44003, 44007, 44006, 44004, 43755, 44008, 43756, 44010, - 43765, 43766, 44021, 44020, 44023, 44022, 44019, 44018, 44016, 44025, - 44017, 44024, 129760, 127816, 125140, 125137, 125136, 125139, 125141, - 125138, 125142, 124928, 124929, 124930, 124936, 124937, 124949, 124950, - 124948, 124938, 124956, 124990, 124992, 124974, 124955, 124964, 124963, - 124991, 124957, 124962, 124976, 124996, 124997, 124998, 124982, 124983, - 124984, 125004, 124975, 125003, 125005, 125011, 125018, 125028, 125029, - 125012, 125019, 125020, 125027, 125013, 125035, 125048, 124942, 124934, - 125090, 125046, 125078, 125033, 124946, 125037, 125070, 125000, 125095, - 125121, 124951, 125044, 125041, 125072, 124987, 125066, 125076, 125074, - 125009, 125107, 125108, 125068, 125124, 124945, 125002, 124931, 125089, - 125022, 124980, 125099, 124986, 125100, 125080, 125119, 124933, 125021, - 125015, 125071, 124985, 125117, 125056, 124993, 125039, 125049, 125043, - 125024, 124932, 125047, 125097, 124959, 125069, 125088, 124999, 125123, - 124952, 125036, 125026, 125001, 125085, 124960, 125057, 125073, 124966, - 125098, 125014, 125091, 124989, 125007, 124978, 124940, 125106, 125050, - 125030, 125092, 124941, 125060, 125077, 125102, 125094, 125053, 125040, - 125055, 125104, 125103, 124939, 125017, 124961, 125112, 125087, 124970, - 124971, 124969, 125023, 124979, 125042, 124947, 125086, 125075, 125051, - 125111, 124968, 124944, 125038, 125096, 125016, 125118, 125109, 124953, - 125059, 125052, 125006, 124958, 125093, 125115, 125054, 124988, 125008, - 125084, 125061, 125064, 125120, 125063, 124967, 124977, 124965, 125031, - 983275, 125081, 125082, 983276, 125010, 125067, 124973, 125032, 124935, - 125116, 125122, 125101, 124994, 124995, 125113, 125058, 125079, 125114, - 125065, 125034, 125083, 124954, 125062, 125105, 125110, 125045, 124943, - 124972, 124981, 125025, 125131, 125130, 125133, 125132, 125129, 125128, - 125135, 125127, 125134, 128334, 128697, 68028, 68093, 68090, 68089, - 68086, 68029, 68092, 68091, 68095, 68088, 68087, 68094, 68000, 68016, - 68020, 68021, 68022, 68009, 68010, 68015, 68017, 68014, 68013, 68018, - 68006, 68023, 68012, 68008, 68007, 68019, 68011, 68005, 68004, 68001, - 68002, 68003, 68031, 68030, 68039, 68075, 68057, 68084, 68066, 68036, - 68054, 68081, 68063, 68045, 68072, 68035, 68053, 68080, 68062, 68044, - 68071, 68040, 68076, 68058, 68085, 68067, 68038, 68056, 68083, 68065, - 68047, 68074, 68037, 68055, 68082, 68064, 68046, 68073, 68034, 68052, - 68079, 68061, 68043, 68070, 68033, 68051, 68078, 68060, 68042, 68069, - 68041, 68068, 68032, 68050, 68077, 68059, 67974, 67975, 67982, 67983, - 67978, 67979, 67980, 67981, 67987, 67988, 67989, 67992, 67993, 67994, - 67995, 67996, 67986, 67985, 67990, 67997, 67984, 67977, 67976, 67991, - 67973, 67972, 67968, 67969, 67970, 67971, 67998, 67999, 9791, 129500, - 983173, 9170, 9172, 9177, 9176, 9175, 9173, 9174, 9171, 9169, 128647, - 128221, 94015, 93989, 93971, 93958, 94019, 94021, 93953, 94023, 93999, - 93995, 94008, 93981, 93979, 93967, 93963, 93993, 93992, 93983, 93977, - 93976, 93975, 93974, 93968, 94032, 93988, 93987, 93973, 93972, 93997, - 93996, 93969, 94106, 94107, 94108, 94109, 94110, 94111, 94002, 94026, - 94022, 94003, 94004, 94010, 93980, 93978, 94099, 94100, 94101, 94102, - 94103, 94104, 94105, 93998, 93994, 94007, 94025, 93966, 93962, 94024, - 93961, 93960, 94000, 94009, 93964, 93965, 94001, 93970, 93984, 93954, - 94017, 94014, 94016, 94013, 94006, 94012, 94005, 94011, 93986, 93985, - 93955, 93952, 94020, 93990, 93957, 93956, 93959, 93982, 94018, 93991, - 94035, 94034, 94033, 94031, 94098, 94096, 94097, 94095, 94036, 94039, - 94040, 94067, 94068, 94038, 94037, 94073, 94075, 94045, 94071, 94069, - 94046, 94047, 94085, 94074, 94049, 94050, 94051, 94052, 94053, 94086, - 94057, 94054, 94084, 94055, 94056, 94041, 94082, 94048, 94081, 94042, - 94076, 94058, 94059, 94060, 94061, 94063, 94064, 94079, 94087, 94062, - 94065, 94080, 94066, 94072, 94070, 94044, 94043, 94077, 94078, 94083, - 983239, 983240, 128300, 127908, 181, 129440, 117772, 129986, 130022, - 130023, 183, 8943, 129686, 127894, 127756, 8357, 128189, 128469, 128656, - 8722, 10793, 10794, 10796, 10795, 10810, 8770, 8723, 10751, 129694, - 129705, 128241, 128242, 128244, 129339, 8871, 71232, 71229, 71236, 71231, - 71230, 71216, 71226, 71228, 71219, 71220, 71221, 71222, 71223, 71224, - 71217, 71218, 71225, 71227, 71234, 71233, 71253, 71252, 71255, 71254, - 71251, 71250, 71248, 71257, 71249, 71256, 71168, 71169, 71179, 71181, - 71195, 71194, 71200, 71199, 71193, 71192, 71198, 71197, 71174, 71175, - 71176, 71177, 71210, 71172, 71173, 71170, 71171, 71215, 71209, 71186, - 71196, 71191, 71201, 71211, 71212, 71213, 71205, 71204, 71188, 71187, - 71185, 71184, 71190, 71189, 71183, 71182, 71203, 71202, 71214, 71206, - 71208, 71207, 71178, 71180, 71235, 43867, 67512, 714, 700, 67509, 761, - 763, 7470, 7471, 7487, 7474, 7483, 7476, 43000, 7484, 7485, 7468, 7469, - 42994, 7472, 7473, 42995, 7475, 7477, 7478, 7479, 7480, 7481, 7482, 7486, - 42996, 7488, 7489, 11389, 7490, 723, 722, 42755, 42753, 42757, 42759, - 42754, 42752, 42756, 42758, 42652, 122956, 122958, 122929, 122954, - 122932, 122952, 122943, 122987, 122946, 122938, 122939, 122942, 122960, - 122941, 122959, 122989, 122955, 122950, 122948, 122944, 122951, 122988, - 122953, 122934, 122935, 122936, 122933, 122949, 122931, 122957, 122930, - 122947, 122937, 122928, 122940, 122945, 42653, 7544, 710, 735, 42889, - 67510, 42776, 42775, 42777, 750, 698, 725, 709, 984011, 42765, 42760, - 42770, 741, 984012, 42769, 42764, 42774, 745, 762, 764, 715, 704, 67507, - 4348, 42766, 42761, 42771, 742, 721, 67511, 42888, 42768, 42763, 751, - 767, 753, 42773, 717, 754, 755, 744, 759, 719, 718, 42783, 752, 716, - 42778, 703, 43882, 706, 713, 42767, 42762, 42772, 743, 758, 757, 756, - 727, 766, 726, 697, 42780, 42782, 42781, 42779, 760, 705, 67508, 701, - 67513, 702, 43883, 707, 734, 42890, 765, 7491, 7493, 7516, 67459, 7495, - 7601, 7509, 67461, 7517, 7580, 7590, 694, 7591, 67474, 67476, 7595, - 67484, 67491, 67456, 67460, 67478, 7600, 67498, 7608, 67506, 67471, - 67492, 7581, 7521, 7496, 67468, 67469, 67467, 67466, 7519, 7585, 67480, - 67463, 67465, 67464, 7611, 7613, 7612, 7497, 7604, 7582, 7614, 7505, - 7584, 67472, 7501, 7518, 7520, 67475, 736, 688, 689, 67477, 43868, 67479, - 7504, 7596, 7588, 7589, 690, 7592, 737, 43869, 43870, 7593, 67485, 7594, - 67483, 67481, 67482, 67486, 67487, 43001, 7599, 7598, 7506, 67490, 7499, - 7507, 7510, 7602, 691, 67497, 67496, 67473, 740, 7583, 67470, 738, 67514, - 7603, 7586, 7498, 7513, 7511, 7605, 67503, 67499, 67502, 7508, 67500, - 67501, 7492, 7579, 7494, 7514, 7597, 7500, 692, 67494, 67495, 693, 67488, - 67489, 7587, 7502, 7610, 43881, 7615, 7512, 43871, 7606, 7607, 7515, - 67504, 7609, 7503, 67493, 695, 739, 696, 42784, 42785, 67458, 67457, 720, - 699, 724, 708, 749, 42864, 748, 712, 747, 746, 10762, 128184, 128176, - 129297, 71266, 6165, 6164, 6167, 6166, 6163, 6162, 6160, 6169, 6161, - 6168, 6159, 6157, 6156, 6155, 6147, 6149, 71271, 71272, 6176, 6279, 6272, - 6295, 6273, 6289, 6274, 6313, 6286, 6311, 6310, 6280, 6276, 6275, 6282, - 6287, 6278, 6285, 6284, 6288, 6277, 6291, 6290, 6293, 6294, 6292, 6283, - 6281, 6185, 6196, 6264, 6210, 6190, 6303, 6305, 6307, 6300, 6302, 6304, - 6312, 6298, 6301, 6314, 6308, 6309, 6299, 6306, 6263, 6262, 6260, 6261, - 6259, 6244, 6252, 6245, 6253, 6238, 6239, 6254, 6248, 6257, 6247, 6256, - 6242, 6258, 6255, 6241, 6240, 6249, 6251, 6250, 6243, 6246, 6237, 6193, - 6192, 6297, 6296, 6218, 6236, 6225, 6234, 6227, 6235, 6211, 6222, 6232, - 6228, 6224, 6226, 6233, 6214, 6216, 6215, 6217, 6219, 6231, 6223, 6220, - 6221, 6230, 6229, 6212, 6213, 6204, 6194, 6209, 6207, 6205, 6206, 6203, - 6202, 6208, 6191, 6177, 6183, 6179, 6181, 6180, 6182, 6186, 6195, 6201, - 6189, 6197, 6184, 6187, 6188, 6199, 6200, 6198, 6178, 71273, 71275, + 128965, 128944, 10100, 10088, 10092, 10101, 10089, 10093, 8287, 9618, + 128971, 128950, 128938, 9900, 118512, 128963, 128961, 10073, 128974, + 128956, 9898, 128911, 128931, 43761, 44013, 43762, 43760, 44011, 43994, + 43989, 43974, 43746, 43993, 43991, 43751, 43750, 43992, 43986, 43987, + 43990, 43968, 43995, 43976, 43973, 43999, 43977, 44001, 43752, 43747, + 43972, 43998, 43984, 43969, 43753, 43754, 43975, 44000, 43978, 43749, + 43748, 43983, 44002, 43970, 43996, 43971, 43997, 43981, 43988, 43979, + 43985, 43980, 43982, 43744, 43745, 44012, 43763, 43764, 44005, 43757, + 43759, 43758, 44009, 44003, 44007, 44006, 44004, 43755, 44008, 43756, + 44010, 43765, 43766, 44021, 44020, 44023, 44022, 44019, 44018, 44016, + 44025, 44017, 44024, 118475, 129760, 127816, 125140, 125137, 125136, + 125139, 125141, 125138, 125142, 124928, 124929, 124930, 124936, 124937, + 124949, 124950, 124948, 124938, 124956, 124990, 124992, 124974, 124955, + 124964, 124963, 124991, 124957, 124962, 124976, 124996, 124997, 124998, + 124982, 124983, 124984, 125004, 124975, 125003, 125005, 125011, 125018, + 125028, 125029, 125012, 125019, 125020, 125027, 125013, 125035, 124942, + 124934, 125090, 125046, 125078, 125033, 124946, 125048, 125037, 125070, + 125000, 125095, 125121, 124951, 125044, 125041, 125072, 124987, 125066, + 125076, 125074, 125009, 125107, 125108, 125068, 125124, 124945, 125002, + 124931, 125089, 125022, 124980, 125099, 124986, 125100, 125080, 125119, + 124933, 125021, 125015, 125071, 124985, 125117, 125056, 124993, 125039, + 125049, 125043, 125024, 124932, 125047, 125097, 124959, 125069, 125088, + 124999, 125123, 124952, 125036, 125026, 125001, 125085, 124960, 125057, + 125073, 124966, 125098, 125014, 125091, 124989, 125007, 124978, 124940, + 125106, 125050, 125030, 125092, 124941, 125060, 125077, 125102, 125094, + 125053, 125040, 125055, 125104, 125103, 124939, 125017, 124961, 125112, + 125087, 124970, 124971, 124969, 125023, 124979, 125042, 124947, 125086, + 125075, 125051, 125111, 124968, 124944, 125038, 125096, 125016, 125118, + 125109, 124953, 125059, 125052, 125006, 124958, 125093, 125115, 125054, + 124988, 125008, 125084, 125061, 125064, 125120, 125063, 124967, 124977, + 124965, 125031, 983279, 125081, 125082, 983280, 125010, 125067, 124973, + 125032, 124935, 125116, 125122, 125101, 124994, 124995, 125113, 125058, + 125079, 125114, 125065, 125034, 125083, 124954, 125062, 125105, 125110, + 125045, 124943, 124972, 124981, 125025, 125131, 125130, 125133, 125132, + 125129, 125128, 125135, 125127, 125134, 128334, 128697, 68028, 68093, + 68090, 68089, 68086, 68029, 68092, 68091, 68095, 68088, 68087, 68094, + 68000, 68016, 68020, 68021, 68022, 68009, 68010, 68015, 68017, 68014, + 68013, 68018, 68006, 68023, 68012, 68008, 68007, 68019, 68011, 68005, + 68004, 68001, 68002, 68003, 68031, 68030, 68039, 68075, 68057, 68084, + 68066, 68036, 68054, 68081, 68063, 68045, 68072, 68035, 68053, 68080, + 68062, 68044, 68071, 68040, 68076, 68058, 68085, 68067, 68038, 68056, + 68083, 68065, 68047, 68074, 68037, 68055, 68082, 68064, 68046, 68073, + 68034, 68052, 68079, 68061, 68043, 68070, 68033, 68051, 68078, 68060, + 68042, 68069, 68041, 68068, 68032, 68050, 68077, 68059, 67974, 67975, + 67982, 67983, 67978, 67979, 67980, 67981, 67987, 67988, 67989, 67992, + 67993, 67994, 67995, 67996, 67986, 67985, 67990, 67997, 67984, 67977, + 67976, 67991, 67973, 67972, 67968, 67969, 67970, 67971, 67998, 67999, + 9791, 129500, 983173, 9170, 9172, 9177, 9176, 9175, 9173, 9174, 9171, + 9169, 128647, 118467, 128221, 94015, 93989, 93971, 93958, 94019, 94021, + 93953, 94023, 93999, 93995, 94008, 93981, 93979, 93967, 93963, 93993, + 93992, 93983, 93977, 93976, 93975, 93974, 93968, 94032, 93988, 93987, + 93973, 93972, 93997, 93996, 93969, 94106, 94107, 94108, 94109, 94110, + 94111, 94002, 94026, 94022, 94003, 94004, 94010, 93980, 93978, 94099, + 94100, 94101, 94102, 94103, 94104, 94105, 93998, 93994, 94007, 94025, + 93966, 93962, 94024, 93961, 93960, 94000, 94009, 93964, 93965, 94001, + 93970, 93984, 93954, 94017, 94014, 94016, 94013, 94006, 94012, 94005, + 94011, 93986, 93985, 93955, 93952, 94020, 93990, 93957, 93956, 93959, + 93982, 94018, 93991, 94035, 94034, 94033, 94031, 94098, 94096, 94097, + 94095, 94036, 94039, 94040, 94067, 94068, 94038, 94037, 94073, 94075, + 94045, 94071, 94069, 94046, 94047, 94085, 94074, 94049, 94050, 94051, + 94052, 94053, 94086, 94057, 94054, 94084, 94055, 94056, 94041, 94082, + 94048, 94081, 94042, 94076, 94058, 94059, 94060, 94061, 94063, 94064, + 94079, 94087, 94062, 94065, 94080, 94066, 94072, 94070, 94044, 94043, + 94077, 94078, 94083, 983239, 983240, 128300, 127908, 181, 129440, 117772, + 129986, 130022, 130023, 183, 8943, 129686, 127894, 127756, 8357, 128189, + 128469, 128656, 8722, 10793, 10794, 10796, 10795, 10810, 8770, 8723, + 10751, 129694, 129705, 128241, 128242, 128244, 129339, 8871, 71232, + 71229, 71236, 71231, 71230, 71216, 71226, 71228, 71219, 71220, 71221, + 71222, 71223, 71224, 71217, 71218, 71225, 71227, 71234, 71233, 71253, + 71252, 71255, 71254, 71251, 71250, 71248, 71257, 71249, 71256, 71168, + 71169, 71179, 71181, 71195, 71194, 71200, 71199, 71193, 71192, 71198, + 71197, 71174, 71175, 71176, 71177, 71210, 71172, 71173, 71170, 71171, + 71215, 71209, 71186, 71196, 71191, 71201, 71211, 71212, 71213, 71205, + 71204, 71188, 71187, 71185, 71184, 71190, 71189, 71183, 71182, 71203, + 71202, 71214, 71206, 71208, 71207, 71178, 71180, 71235, 43867, 67512, + 714, 700, 67509, 761, 763, 7470, 7471, 7487, 7474, 7483, 7476, 43000, + 7484, 7485, 7468, 7469, 42994, 7472, 7473, 42995, 7475, 7477, 7478, 7479, + 7480, 7481, 7482, 7486, 42996, 42993, 7488, 7489, 11389, 7490, 723, 722, + 42755, 42753, 42757, 42759, 42754, 42752, 42756, 42758, 42652, 122956, + 122958, 122929, 122954, 122932, 122952, 122943, 122987, 122946, 122938, + 122939, 122942, 122960, 122941, 122959, 122989, 122955, 122950, 122948, + 122944, 122951, 122988, 122953, 122934, 122935, 122936, 122933, 122949, + 122931, 122957, 122930, 122947, 122937, 122928, 122940, 122945, 42653, + 7544, 710, 735, 42889, 67510, 42776, 42775, 42777, 750, 698, 725, 709, + 984011, 42765, 42760, 42770, 741, 984012, 42769, 42764, 42774, 745, 762, + 764, 715, 704, 67507, 4348, 42766, 42761, 42771, 742, 721, 67511, 42888, + 42768, 42763, 751, 767, 753, 42773, 717, 754, 755, 744, 759, 719, 718, + 42783, 752, 716, 42778, 703, 43882, 706, 713, 42767, 42762, 42772, 743, + 758, 757, 756, 727, 766, 726, 697, 42780, 42782, 42781, 42779, 760, 705, + 67508, 701, 67513, 702, 43883, 707, 734, 42890, 765, 7491, 7493, 7516, + 67459, 7495, 7601, 7509, 67461, 7517, 7580, 7590, 694, 7591, 67474, + 67476, 7595, 67484, 67491, 67456, 67460, 67478, 7600, 67498, 7608, 67506, + 67471, 67492, 7581, 7521, 7496, 67468, 67469, 67467, 67466, 7519, 7585, + 67480, 67463, 67465, 67464, 7611, 7613, 7612, 7497, 7604, 7582, 7614, + 7505, 7584, 67472, 7501, 7518, 7520, 67475, 736, 688, 689, 67477, 43868, + 67479, 7504, 7596, 7588, 7589, 690, 7592, 737, 43869, 43870, 7593, 67485, + 7594, 67483, 67481, 67482, 67486, 67487, 43001, 7599, 7598, 7506, 67490, + 7499, 7507, 7510, 7602, 691, 67497, 67496, 67473, 740, 7583, 67470, 738, + 67514, 7603, 7586, 7498, 7513, 7511, 7605, 67503, 67499, 67502, 7508, + 67500, 67501, 7492, 7579, 7494, 7514, 7597, 7500, 692, 67494, 67495, 693, + 67488, 67489, 7587, 7502, 7610, 43881, 7615, 7512, 43871, 7606, 7607, + 7515, 67504, 7609, 7503, 67493, 695, 739, 696, 42784, 42785, 67458, + 67457, 720, 699, 724, 708, 749, 42864, 748, 712, 747, 746, 10762, 128184, + 128176, 129297, 71266, 6165, 6164, 6167, 6166, 6163, 6162, 6160, 6169, + 6161, 6168, 6159, 6157, 6156, 6155, 6147, 6149, 71271, 71272, 6176, 6279, + 6272, 6295, 6273, 6289, 6274, 6313, 6286, 6311, 6310, 6280, 6276, 6275, + 6282, 6287, 6278, 6285, 6284, 6288, 6277, 6291, 6290, 6293, 6294, 6292, + 6283, 6281, 6185, 6196, 6264, 6210, 6190, 6303, 6305, 6307, 6300, 6302, + 6304, 6312, 6298, 6301, 6314, 6308, 6309, 6299, 6306, 6263, 6262, 6260, + 6261, 6259, 6244, 6252, 6245, 6253, 6254, 6248, 6238, 6239, 6257, 6247, + 6256, 6242, 6258, 6255, 6241, 6240, 6249, 6251, 6250, 6243, 6246, 6237, + 6193, 6192, 6297, 6296, 6218, 6236, 6225, 6234, 6227, 6235, 6211, 6222, + 6232, 6228, 6224, 6226, 6233, 6214, 6216, 6215, 6217, 6219, 6231, 6223, + 6220, 6221, 6230, 6229, 6212, 6213, 6204, 6194, 6209, 6207, 6205, 6206, + 6203, 6202, 6208, 6191, 6177, 6183, 6179, 6181, 6180, 6182, 6186, 6195, + 6201, 6189, 6197, 6184, 6187, 6188, 6199, 6200, 6198, 6178, 71273, 71275, 71274, 6151, 71265, 71270, 71269, 6144, 71268, 71264, 71267, 71276, 6150, 6158, 6148, 6146, 6152, 6153, 6154, 6145, 9866, 9867, 119552, 9101, 128669, 128018, 128053, 127889, 129390, 118261, 128496, 129742, 129439, @@ -11989,14 +12112,14 @@ static const unsigned int dawg_pos_to_codepoint[] = { 128693, 128507, 128001, 129700, 128045, 128068, 128511, 127909, 92748, 92744, 92761, 92750, 92739, 92751, 92737, 92754, 92749, 92753, 92743, 92752, 92757, 92766, 92736, 92741, 92746, 92764, 92745, 92765, 92755, - 92760, 92758, 92756, 92763, 92762, 92747, 92738, 92740, 92759, 92742, + 92756, 92763, 92762, 92747, 92760, 92758, 92738, 92740, 92759, 92742, 92783, 92782, 92773, 92772, 92775, 92774, 92771, 92770, 92768, 92777, 92769, 92776, 70291, 70292, 70290, 70297, 70296, 70293, 70287, 70298, 70312, 70311, 70306, 70285, 70284, 70289, 70288, 70295, 70294, 70303, 70301, 70283, 70282, 70280, 70278, 70277, 70276, 70300, 70299, 70310, 70307, 70304, 70309, 70308, 70305, 70272, 70275, 70273, 70274, 70313, 127926, 215, 10804, 10805, 10807, 10811, 10801, 10800, 10005, 8844, 8845, - 8846, 8888, 127812, 117860, 9838, 9837, 9839, 127929, 127896, 119161, + 8846, 8888, 127812, 117860, 9838, 9839, 9837, 127929, 127896, 119161, 119159, 119155, 119157, 119061, 119060, 119224, 119235, 119132, 119058, 119059, 119255, 119253, 119130, 119131, 119163, 119169, 119149, 119167, 119168, 119210, 119178, 119173, 119211, 119150, 119151, 119152, 119153, @@ -12108,9 +12231,9 @@ static const unsigned int dawg_pos_to_codepoint[] = { 129208, 128600, 128592, 128608, 11110, 11126, 11017, 8662, 129108, 11009, 128746, 8882, 8884, 8379, 8836, 8837, 8713, 8772, 8777, 8938, 8940, 8742, 8930, 8931, 172, 8877, 8769, 8813, 8800, 8802, 8815, 8814, 9083, 128323, - 10159, 128324, 10161, 128456, 128457, 128458, 128211, 128212, 160, - 128067, 9369, 9362, 9365, 9366, 9367, 35, 9368, 9364, 9361, 9363, 9371, - 9370, 8470, 110960, 110961, 110962, 110963, 110964, 110965, 110966, + 10159, 128324, 10161, 128456, 128457, 128458, 128211, 128212, 128067, + 118012, 160, 9369, 9362, 9365, 9366, 9367, 35, 9368, 9364, 9361, 9363, + 9371, 9370, 8470, 110960, 110961, 110962, 110963, 110964, 110965, 110966, 110967, 110968, 110969, 110970, 110971, 110972, 110973, 110974, 110975, 110976, 110977, 110978, 110979, 110980, 110981, 110982, 110983, 110984, 110985, 110986, 110987, 110988, 110989, 110990, 110991, 110992, 110993, @@ -12164,76 +12287,76 @@ static const unsigned int dawg_pos_to_codepoint[] = { 123185, 123188, 123187, 123215, 123205, 123204, 123207, 123206, 123203, 123202, 123200, 123209, 123201, 123208, 117776, 983231, 983066, 10663, 10662, 11869, 9215, 65532, 9287, 9286, 9284, 9285, 9289, 9281, 9290, - 9288, 9282, 9283, 9280, 128721, 128025, 128885, 127970, 5787, 5788, 5776, - 5761, 5786, 5770, 5769, 5781, 5779, 5785, 5763, 5772, 5780, 5784, 5762, - 5775, 5773, 5765, 5777, 5782, 5764, 5774, 5783, 5766, 5778, 5771, 5767, - 5768, 5760, 731, 128738, 7265, 7264, 7266, 7267, 7261, 7260, 7262, 7259, - 7280, 7282, 7281, 7279, 7271, 7270, 7272, 7269, 7258, 7263, 7278, 7268, - 7283, 7273, 7284, 7285, 7287, 7286, 7276, 7274, 7275, 7277, 7290, 7288, - 7289, 7295, 7294, 7292, 7291, 7253, 7252, 7255, 7254, 7251, 7250, 7248, - 7257, 7249, 7256, 7293, 124374, 124376, 124375, 124377, 124378, 124379, - 124392, 124396, 124395, 124397, 124394, 124393, 124380, 124381, 124383, - 124384, 124385, 124382, 124368, 124371, 124370, 124369, 124372, 124373, - 124386, 124388, 124390, 124389, 124387, 124391, 124399, 124400, 124398, - 124415, 124406, 124405, 124408, 124407, 124404, 124403, 124401, 124410, - 124402, 124409, 94179, 94178, 68736, 68739, 68744, 68737, 68756, 68745, - 68760, 68769, 68761, 68775, 68785, 68741, 68762, 68772, 68773, 68740, - 68777, 68742, 68749, 68750, 68758, 68759, 68774, 68776, 68783, 68784, - 68738, 68743, 68747, 68748, 68751, 68754, 68755, 68768, 68770, 68782, - 68765, 68780, 68766, 68781, 68763, 68767, 68764, 68778, 68757, 68786, - 68779, 68746, 68752, 68753, 68771, 68800, 68803, 68808, 68801, 68820, - 68809, 68824, 68833, 68825, 68839, 68849, 68805, 68826, 68836, 68837, - 68804, 68841, 68806, 68813, 68814, 68822, 68823, 68838, 68840, 68847, - 68848, 68802, 68807, 68811, 68812, 68815, 68818, 68819, 68832, 68834, - 68846, 68829, 68844, 68830, 68845, 68827, 68831, 68828, 68842, 68821, - 68850, 68843, 68810, 68816, 68817, 68835, 68861, 68859, 68858, 68862, - 68863, 68860, 66308, 66324, 66318, 66335, 66323, 66331, 66327, 66330, - 66315, 66316, 66317, 66329, 66314, 66306, 66322, 66351, 66321, 66350, - 66326, 66334, 66313, 66333, 66328, 66320, 66312, 66325, 66332, 66305, - 66307, 66311, 66309, 66349, 66310, 66304, 66319, 66339, 66337, 66338, - 66336, 68241, 68242, 68246, 68244, 68226, 68224, 68237, 68235, 68249, - 68251, 68247, 68233, 68248, 68252, 68227, 68234, 68230, 68239, 68232, - 68240, 68231, 68250, 68236, 68225, 68243, 68245, 68228, 68229, 68238, - 68255, 68254, 68253, 66404, 66390, 66392, 66387, 66388, 66411, 66393, - 66421, 66418, 66396, 66397, 66399, 66406, 66405, 66401, 66413, 66402, - 66398, 66414, 66415, 66416, 66408, 66420, 66417, 66407, 66419, 66389, - 66391, 66395, 66400, 66386, 66409, 66410, 66385, 66384, 66394, 66412, - 66403, 66516, 66514, 66515, 66517, 66513, 66464, 66504, 66505, 66506, - 66482, 66510, 66511, 66477, 66508, 66509, 66478, 66479, 66473, 66474, - 66490, 66491, 66480, 66475, 66476, 66507, 66471, 66469, 66470, 66467, - 66468, 66484, 66485, 66492, 66493, 66486, 66487, 66488, 66497, 66498, - 66495, 66472, 66483, 66499, 66494, 66481, 66489, 66496, 66465, 66466, - 66512, 128435, 118450, 69414, 69395, 69376, 69394, 69391, 69392, 69398, - 69399, 69403, 69404, 69377, 69379, 69382, 69400, 69388, 69380, 69384, - 69393, 69397, 69401, 69386, 69381, 69385, 69387, 69378, 69390, 69402, - 69383, 69396, 69389, 69415, 69407, 69412, 69411, 69406, 69410, 69405, - 69413, 69409, 69408, 68209, 68210, 68217, 68211, 68213, 68214, 68212, - 68203, 68205, 68207, 68206, 68202, 68198, 68220, 68219, 68215, 68201, - 68216, 68193, 68196, 68199, 68218, 68192, 68194, 68200, 68204, 68197, - 68208, 68195, 68222, 68221, 68223, 68608, 68619, 68627, 68623, 68634, - 68640, 68644, 68668, 68670, 68677, 68632, 68669, 68671, 68617, 68625, - 68621, 68638, 68643, 68660, 68666, 68675, 68630, 68648, 68653, 68646, - 68650, 68641, 68673, 68658, 68642, 68655, 68628, 68611, 68657, 68662, - 68614, 68615, 68636, 68656, 68664, 68679, 68680, 68609, 68610, 68645, - 68654, 68620, 68624, 68635, 68678, 68633, 68672, 68652, 68618, 68626, - 68622, 68639, 68661, 68667, 68676, 68631, 68613, 68649, 68647, 68651, - 68674, 68659, 68629, 68612, 68663, 68616, 68637, 68665, 69509, 69508, - 69507, 69506, 69488, 69502, 69496, 69505, 69492, 69499, 69501, 69503, - 69494, 69493, 69490, 69495, 69489, 69498, 69504, 69491, 69500, 69497, - 69511, 69512, 69513, 69510, 128477, 128117, 129491, 128116, 129746, - 128283, 128664, 128753, 128662, 128660, 128653, 11819, 8228, 128431, - 129649, 129477, 128214, 9251, 10044, 10027, 10034, 10011, 128194, 128449, - 128080, 128237, 128236, 10180, 10179, 128275, 9103, 9104, 10174, 983191, - 8997, 128191, 128440, 9934, 9741, 128217, 129505, 129447, 8886, 2902, - 2903, 2933, 2934, 2931, 2930, 2935, 2932, 2928, 2909, 2908, 2864, 2911, - 2863, 2821, 2822, 2832, 2836, 2850, 2849, 2855, 2854, 2848, 2847, 2853, - 2852, 2827, 2912, 2828, 2913, 2869, 2825, 2826, 2823, 2824, 2867, 2866, - 2841, 2851, 2846, 2856, 2870, 2871, 2872, 2861, 2860, 2843, 2842, 2840, - 2839, 2845, 2844, 2838, 2837, 2859, 2858, 2873, 2862, 2929, 2831, 2835, - 983660, 983659, 2817, 2876, 2877, 2818, 2901, 2893, 2819, 2878, 2888, - 2892, 2881, 2882, 2883, 2884, 2914, 2915, 2879, 2880, 2887, 2891, 2923, - 2922, 2925, 2924, 2921, 2920, 2918, 2927, 2919, 2926, 64830, 64831, 9766, - 117826, 10183, 128895, 66736, 66737, 66738, 66739, 66743, 66763, 66761, + 9288, 9282, 9283, 9280, 128721, 128025, 128885, 5787, 5788, 5776, 5761, + 5786, 5770, 5769, 5781, 5779, 5785, 5763, 5772, 5780, 5784, 5762, 5775, + 5773, 5765, 5777, 5782, 5764, 5774, 5783, 5766, 5778, 5771, 5767, 5768, + 5760, 731, 128738, 7265, 7264, 7266, 7267, 7261, 7260, 7262, 7259, 7280, + 7282, 7281, 7279, 7271, 7270, 7272, 7269, 7258, 7263, 7278, 7268, 7283, + 7273, 7284, 7285, 7287, 7286, 7276, 7274, 7275, 7277, 7290, 7288, 7289, + 7295, 7294, 7292, 7291, 7253, 7252, 7255, 7254, 7251, 7250, 7248, 7257, + 7249, 7256, 7293, 124374, 124376, 124375, 124377, 124378, 124379, 124392, + 124396, 124395, 124397, 124394, 124393, 124380, 124381, 124383, 124384, + 124385, 124382, 124368, 124371, 124370, 124369, 124372, 124373, 124386, + 124388, 124390, 124389, 124387, 124391, 124399, 124400, 124398, 124415, + 124406, 124405, 124408, 124407, 124404, 124403, 124401, 124410, 124402, + 124409, 94179, 94178, 68736, 68739, 68744, 68737, 68756, 68745, 68760, + 68769, 68761, 68775, 68785, 68741, 68762, 68772, 68773, 68740, 68777, + 68742, 68749, 68750, 68758, 68759, 68774, 68776, 68783, 68784, 68738, + 68743, 68747, 68748, 68751, 68754, 68755, 68768, 68770, 68782, 68765, + 68780, 68766, 68781, 68763, 68767, 68764, 68771, 68778, 68757, 68786, + 68779, 68746, 68752, 68753, 68800, 68803, 68808, 68801, 68820, 68809, + 68824, 68833, 68825, 68839, 68849, 68805, 68826, 68836, 68837, 68804, + 68841, 68806, 68813, 68814, 68822, 68823, 68838, 68840, 68847, 68848, + 68802, 68807, 68811, 68812, 68815, 68818, 68819, 68832, 68834, 68846, + 68829, 68844, 68830, 68845, 68827, 68831, 68828, 68835, 68842, 68821, + 68850, 68843, 68810, 68816, 68817, 68861, 68859, 68858, 68862, 68863, + 68860, 66308, 66324, 66318, 66335, 66323, 66331, 66327, 66330, 66315, + 66316, 66317, 66329, 66314, 66306, 66322, 66351, 66321, 66350, 66326, + 66334, 66313, 66333, 66328, 66320, 66312, 66325, 66332, 66305, 66307, + 66311, 66309, 66349, 66310, 66304, 66319, 66339, 66337, 66338, 66336, + 68241, 68242, 68246, 68244, 68226, 68224, 68237, 68235, 68249, 68251, + 68247, 68233, 68248, 68252, 68227, 68234, 68230, 68239, 68232, 68240, + 68231, 68250, 68236, 68225, 68243, 68245, 68228, 68229, 68238, 68255, + 68254, 68253, 66404, 66390, 66392, 66387, 66388, 66411, 66393, 66421, + 66418, 66396, 66397, 66399, 66406, 66405, 66401, 66413, 66402, 66398, + 66414, 66415, 66416, 66408, 66420, 66417, 66407, 66419, 66389, 66391, + 66395, 66400, 66386, 66409, 66410, 66385, 66384, 66394, 66412, 66403, + 66516, 66514, 66515, 66517, 66513, 66464, 66504, 66505, 66506, 66482, + 66510, 66511, 66477, 66508, 66509, 66478, 66479, 66473, 66474, 66490, + 66491, 66480, 66475, 66476, 66507, 66471, 66469, 66470, 66467, 66468, + 66484, 66485, 66492, 66493, 66486, 66487, 66488, 66497, 66498, 66495, + 66472, 66483, 66499, 66494, 66481, 66489, 66496, 66465, 66466, 66512, + 128435, 118450, 69414, 69395, 69376, 69394, 69391, 69392, 69398, 69399, + 69403, 69404, 69377, 69379, 69382, 69400, 69388, 69380, 69384, 69393, + 69397, 69401, 69386, 69381, 69385, 69387, 69378, 69390, 69402, 69383, + 69396, 69389, 69415, 69407, 69412, 69411, 69406, 69410, 69405, 69413, + 69409, 69408, 68209, 68210, 68217, 68211, 68213, 68214, 68212, 68203, + 68205, 68207, 68206, 68202, 68198, 68220, 68219, 68215, 68201, 68216, + 68193, 68196, 68199, 68218, 68192, 68194, 68200, 68204, 68197, 68208, + 68195, 68222, 68221, 68223, 68608, 68619, 68627, 68623, 68634, 68640, + 68644, 68668, 68670, 68677, 68632, 68669, 68671, 68617, 68625, 68621, + 68638, 68643, 68660, 68666, 68675, 68630, 68648, 68653, 68646, 68650, + 68641, 68673, 68658, 68642, 68655, 68628, 68611, 68657, 68662, 68614, + 68615, 68636, 68656, 68664, 68679, 68680, 68609, 68610, 68645, 68654, + 68620, 68624, 68635, 68678, 68633, 68672, 68652, 68618, 68626, 68622, + 68639, 68661, 68667, 68676, 68631, 68613, 68649, 68647, 68651, 68674, + 68659, 68629, 68612, 68663, 68616, 68637, 68665, 69509, 69508, 69507, + 69506, 69488, 69502, 69496, 69505, 69492, 69499, 69501, 69503, 69494, + 69493, 69490, 69495, 69489, 69498, 69504, 69491, 69500, 69497, 69511, + 69512, 69513, 69510, 128477, 128117, 129491, 128116, 129746, 128283, + 128664, 128753, 128662, 128660, 128653, 11819, 8228, 128431, 129649, + 129477, 128214, 9251, 10044, 10027, 10034, 10011, 128194, 128449, 128080, + 128237, 128236, 10180, 10179, 128275, 9103, 9104, 10174, 983191, 8997, + 128191, 128440, 9934, 9741, 128217, 129505, 129447, 128895, 129741, 8886, + 2902, 2903, 2933, 2934, 2931, 2930, 2935, 2932, 2928, 2909, 2908, 2864, + 2911, 2863, 2821, 2822, 2832, 2836, 2850, 2849, 2855, 2854, 2848, 2847, + 2853, 2852, 2827, 2912, 2828, 2913, 2869, 2825, 2826, 2823, 2824, 2867, + 2866, 2841, 2851, 2846, 2856, 2870, 2871, 2872, 2861, 2860, 2843, 2842, + 2840, 2839, 2845, 2844, 2838, 2837, 2859, 2858, 2873, 2862, 2929, 2831, + 2835, 983660, 983659, 2817, 2876, 2877, 2818, 2901, 2893, 2819, 2878, + 2888, 2892, 2881, 2882, 2883, 2884, 2914, 2915, 2879, 2880, 2887, 2891, + 2923, 2922, 2925, 2924, 2921, 2920, 2918, 2927, 2919, 2926, 64830, 64831, + 9766, 117826, 10183, 66736, 66737, 66738, 66739, 66743, 66763, 66761, 66742, 66749, 66757, 66744, 66768, 66750, 66748, 66754, 66755, 66764, 66762, 66760, 66746, 66745, 66741, 66765, 66769, 66759, 66758, 66771, 66770, 66740, 66751, 66752, 66753, 66756, 66767, 66747, 66766, 66776, @@ -12256,119 +12379,120 @@ static const unsigned int dawg_pos_to_codepoint[] = { 117986, 117987, 117988, 117989, 117990, 117991, 117992, 117993, 117994, 117995, 117996, 117997, 117998, 117999, 10015, 9885, 10029, 10009, 118005, 118004, 118007, 118006, 118003, 118002, 118000, 118009, 118001, - 118008, 8485, 129397, 128471, 11195, 11194, 11196, 8254, 129450, 8486, - 128076, 127842, 128329, 129417, 128002, 983122, 983121, 128463, 128195, - 128479, 128196, 128223, 128464, 128724, 93059, 93065, 93064, 93058, - 93070, 93056, 93055, 93053, 93062, 93069, 93061, 93066, 93071, 93068, - 93054, 93057, 93063, 93067, 93060, 92967, 92975, 92965, 92969, 92959, - 92968, 92971, 92957, 92962, 92960, 92972, 92970, 92963, 92958, 92966, - 92961, 92956, 92974, 92964, 92973, 92979, 92978, 92980, 92977, 92976, - 92982, 92981, 93023, 93020, 93024, 93021, 93019, 93025, 93022, 93043, - 92985, 93047, 93044, 93045, 92997, 93046, 93042, 93032, 93029, 92995, - 93038, 92993, 93041, 93035, 93033, 93037, 93030, 93039, 92987, 92992, - 92986, 92983, 92984, 93027, 92994, 93034, 92988, 92990, 92989, 92991, - 93028, 92996, 93031, 93040, 93036, 92954, 92955, 92938, 92939, 92932, - 92933, 92942, 92943, 92950, 92951, 92928, 92929, 92936, 92937, 92948, - 92949, 92930, 92931, 92944, 92945, 92934, 92935, 92940, 92941, 92946, - 92947, 92952, 92953, 93013, 93012, 93015, 93014, 93011, 93010, 93008, - 93017, 93009, 93016, 9908, 11801, 127796, 129779, 129780, 67703, 67693, - 67688, 67702, 67683, 67691, 67699, 67700, 67680, 67696, 67682, 67686, - 67695, 67698, 67701, 67689, 67684, 67687, 67690, 67681, 67694, 67685, - 67697, 67692, 67704, 67711, 67706, 67707, 67710, 67709, 67708, 67705, - 129330, 129374, 128060, 8233, 11853, 11791, 10995, 10994, 8741, 129666, - 12809, 12823, 12808, 12822, 12828, 12813, 12827, 12810, 12824, 12800, - 12814, 12804, 12818, 12801, 12815, 12812, 12826, 12805, 12819, 12803, - 12817, 12806, 12820, 12811, 12825, 12802, 12816, 12807, 12821, 12863, - 12855, 12858, 12861, 12847, 12839, 12854, 12843, 12836, 12864, 12835, - 12856, 12846, 12842, 12840, 12852, 12862, 12865, 12857, 12867, 12838, - 12866, 12851, 12853, 12859, 12849, 12860, 12848, 12837, 12834, 12841, - 12833, 12845, 12844, 12850, 12832, 12830, 12829, 9349, 9342, 9345, 9346, - 9350, 9347, 9348, 9344, 9351, 9343, 9341, 9336, 9335, 9338, 9337, 9334, - 9333, 9340, 9332, 9339, 127248, 127249, 127250, 127251, 127252, 127253, - 127254, 127255, 127256, 127257, 127258, 127259, 127260, 127261, 127262, - 127263, 127264, 127265, 127266, 127267, 127268, 127269, 127270, 127271, - 127272, 127273, 9372, 9373, 9374, 9375, 9376, 9377, 9378, 9379, 9380, - 9381, 9382, 9383, 9384, 9385, 9386, 9387, 9388, 9389, 9390, 9391, 9392, - 9393, 9394, 9395, 9396, 9397, 8706, 983147, 983146, 983149, 983150, 9853, - 127881, 12880, 12349, 129436, 128755, 11261, 9105, 9106, 128706, 72437, - 72440, 72432, 72416, 72419, 72413, 72417, 72415, 72412, 72414, 72418, - 72420, 72403, 72407, 72411, 72409, 72410, 72391, 72400, 72404, 72397, - 72394, 72385, 72401, 72384, 72399, 72398, 72396, 72388, 72393, 72392, - 72386, 72387, 72402, 72395, 72390, 72389, 72405, 72406, 72408, 72436, - 72435, 72438, 72439, 72422, 72421, 72424, 72425, 72431, 72433, 72434, - 72428, 72427, 72429, 72430, 72423, 72426, 128062, 128230, 128206, 983228, - 983237, 129434, 9774, 127825, 129372, 129755, 127824, 128039, 128532, - 9956, 128390, 9999, 8240, 8241, 8524, 10178, 10977, 129336, 129496, - 129494, 128113, 9977, 128590, 129733, 128591, 129493, 128589, 128588, - 129495, 128583, 128187, 8966, 128547, 37, 9854, 127917, 8359, 8369, - 129515, 128694, 129730, 43101, 43117, 43120, 43076, 43123, 43077, 43115, - 43090, 43082, 43094, 43098, 43099, 43119, 43118, 43109, 43074, 43075, - 43116, 43079, 43083, 43089, 43088, 43114, 43113, 43081, 43080, 43073, - 43072, 43085, 43084, 43092, 43093, 43104, 43110, 43086, 43108, 43100, - 43078, 43097, 43087, 43106, 43096, 43091, 43107, 43095, 43102, 43105, - 43103, 43127, 43126, 43124, 43121, 43111, 43112, 43122, 43125, 66033, - 66023, 66017, 66010, 66027, 66003, 66018, 66028, 66004, 66012, 66022, - 66020, 66045, 66019, 66031, 66041, 66007, 66006, 66025, 66026, 66038, - 66016, 66013, 66014, 66000, 66001, 66034, 66036, 66037, 66029, 66011, - 66024, 66015, 66021, 66042, 66043, 66002, 66008, 66032, 66005, 66044, - 66040, 66039, 66030, 66009, 66035, 5941, 5942, 67840, 67855, 67843, - 67844, 67847, 67858, 67857, 67860, 67854, 67861, 67848, 67845, 67846, - 67849, 67859, 67842, 67850, 67853, 67851, 67841, 67856, 67852, 67864, - 67866, 67867, 67863, 67862, 67865, 67871, 11227, 9935, 128763, 128022, - 128061, 128055, 128169, 182, 128138, 129292, 129295, 127885, 127821, - 10031, 129655, 129669, 9811, 128299, 8916, 10970, 129383, 8984, 128720, - 129703, 8462, 8463, 127183, 127136, 127167, 127199, 127188, 127140, - 127156, 127172, 127200, 127189, 127141, 127157, 127173, 127196, 127148, - 127164, 127180, 127198, 127150, 127166, 127182, 127192, 127144, 127160, - 127176, 127191, 127143, 127159, 127175, 127190, 127142, 127158, 127174, - 127197, 127149, 127165, 127181, 127194, 127146, 127162, 127178, 127187, - 127139, 127155, 127171, 127186, 127138, 127154, 127170, 127202, 127220, - 127221, 127201, 127210, 127211, 127212, 127213, 127214, 127215, 127216, - 127217, 127218, 127219, 127203, 127204, 127205, 127206, 127207, 127208, - 127209, 127185, 127137, 127153, 127169, 127193, 127145, 127161, 127177, - 127195, 127147, 127163, 127179, 128733, 983151, 43, 10797, 10798, 10809, - 10789, 10786, 10791, 10790, 10788, 10792, 10787, 10866, 177, 9799, 11222, - 11221, 11220, 11219, 129696, 983148, 117777, 128659, 128680, 128110, - 8297, 8236, 127871, 128254, 11239, 128239, 12306, 12320, 128238, 8982, - 128688, 129364, 127858, 129716, 127831, 129751, 128574, 128545, 163, - 128093, 9212, 9213, 9214, 9211, 128041, 128425, 129328, 129732, 129731, - 65043, 65040, 65045, 65073, 65074, 65049, 65041, 65042, 65091, 65047, - 65083, 65085, 65089, 65079, 65087, 65077, 65095, 65081, 65075, 65084, - 65086, 65092, 983261, 65048, 65090, 65080, 65088, 65078, 65096, 65082, - 65076, 65044, 65072, 65046, 8478, 8826, 10937, 10933, 10927, 10929, - 10935, 10931, 8936, 8830, 8828, 8880, 9111, 129384, 9113, 128424, 128438, - 129332, 128120, 983193, 983166, 983163, 983164, 983167, 8242, 8965, 8759, - 8733, 8522, 128711, 129455, 11224, 128255, 68507, 68508, 68480, 68483, - 68490, 68491, 68485, 68482, 68486, 68493, 68495, 68496, 68488, 68484, - 68487, 68489, 68481, 68492, 68497, 68494, 68526, 68522, 68523, 68525, - 68521, 68527, 68524, 68505, 68506, 11854, 8200, 128156, 128091, 128686, - 128204, 128226, 983165, 983168, 983194, 9624, 9625, 9626, 9627, 9628, - 9629, 9630, 9631, 9622, 9623, 10764, 8279, 10774, 9833, 128894, 8264, 63, - 8799, 34, 9915, 127949, 127950, 129437, 128251, 9762, 128280, 9143, - 128740, 128643, 9926, 127752, 11827, 11783, 11782, 9995, 128400, 128406, - 127338, 127339, 127340, 129996, 11787, 9994, 11828, 129306, 128000, 8758, - 128007, 128048, 129682, 128015, 129534, 117778, 9852, 9843, 9844, 9845, - 9846, 9847, 9848, 9849, 9850, 983113, 128665, 127822, 129511, 174, - 127462, 127463, 127464, 127465, 127466, 127467, 127468, 127469, 127470, - 127471, 127472, 127473, 127474, 127475, 127476, 127477, 127478, 127479, - 127480, 127481, 127482, 127483, 127484, 127485, 127486, 127487, 43344, - 43343, 43346, 43345, 43333, 43323, 43331, 43314, 43332, 43317, 43330, - 43320, 43319, 43321, 43316, 43313, 43329, 43322, 43312, 43326, 43318, - 43325, 43324, 43315, 43328, 43327, 43334, 43359, 43337, 43342, 43341, - 43338, 43340, 43335, 43339, 43336, 43347, 128524, 127895, 65533, 9952, - 9953, 128699, 8479, 9166, 11152, 11153, 128639, 128968, 983152, 92, - 10184, 10741, 10743, 11073, 11079, 983153, 10659, 10661, 8246, 12317, - 10989, 8976, 11793, 8267, 8245, 128401, 9753, 11262, 8515, 8271, 8765, - 8909, 8247, 128402, 128403, 128405, 11841, 11822, 128404, 10672, 128158, - 8251, 129423, 983154, 127872, 11190, 11188, 11191, 11189, 11186, 11187, - 11184, 11185, 127832, 127833, 129919, 129918, 128495, 8735, 12297, 10642, - 11777, 11776, 9084, 8894, 10652, 10644, 10228, 8692, 12305, 10648, 125, - 9132, 9133, 9131, 12301, 8969, 11781, 12299, 10608, 10715, 8221, 11817, - 129929, 10621, 8971, 118272, 9687, 11241, 9616, 129978, 117925, 118289, - 129971, 118286, 118284, 118432, 118443, 129933, 128381, 130025, 130017, - 11805, 8906, 10198, 129927, 9621, 129980, 41, 9120, 9118, 9119, 11789, - 10182, 8908, 129931, 8217, 11815, 128360, 128361, 128362, 128489, 93, - 9126, 9124, 10638, 10640, 8262, 10636, 11864, 11862, 9125, 11779, 117773, + 118008, 8485, 129397, 128471, 11195, 11194, 11196, 8254, 129450, 127970, + 118459, 8486, 128076, 127842, 128329, 129417, 128002, 983122, 983121, + 128463, 128195, 128479, 128196, 128223, 128464, 128724, 93059, 93065, + 93064, 93058, 93070, 93056, 93055, 93053, 93062, 93069, 93061, 93066, + 93071, 93068, 93054, 93057, 93063, 93067, 93060, 92967, 92975, 92965, + 92969, 92959, 92968, 92971, 92957, 92962, 92960, 92972, 92970, 92963, + 92958, 92966, 92961, 92956, 92974, 92964, 92973, 92979, 92978, 92980, + 92977, 92976, 92982, 92981, 93023, 93020, 93024, 93021, 93019, 93025, + 93022, 93043, 92985, 93047, 93044, 93045, 92997, 93046, 93042, 93032, + 93029, 92995, 93038, 92993, 93041, 93035, 93033, 93037, 93030, 93039, + 92987, 92992, 92986, 92983, 92984, 93027, 92994, 93034, 92988, 92990, + 92989, 92991, 93028, 92996, 93031, 93040, 93036, 92954, 92955, 92938, + 92939, 92932, 92933, 92942, 92943, 92950, 92951, 92928, 92929, 92936, + 92937, 92948, 92949, 92930, 92931, 92944, 92945, 92934, 92935, 92940, + 92941, 92946, 92947, 92952, 92953, 93013, 93012, 93015, 93014, 93011, + 93010, 93008, 93017, 93009, 93016, 9908, 11801, 127796, 129779, 129780, + 67703, 67693, 67688, 67702, 67683, 67691, 67699, 67700, 67680, 67696, + 67682, 67686, 67695, 67698, 67701, 67689, 67684, 67687, 67690, 67681, + 67694, 67685, 67697, 67692, 67704, 67711, 67706, 67707, 67710, 67709, + 67708, 67705, 129330, 129374, 128060, 8233, 11853, 11791, 10995, 10994, + 8741, 129666, 12809, 12823, 12808, 12822, 12828, 12813, 12827, 12810, + 12824, 12800, 12814, 12804, 12818, 12801, 12815, 12812, 12826, 12805, + 12819, 12803, 12817, 12806, 12820, 12811, 12825, 12802, 12816, 12807, + 12821, 12863, 12855, 12858, 12861, 12847, 12839, 12854, 12843, 12836, + 12864, 12835, 12856, 12846, 12842, 12840, 12852, 12862, 12865, 12857, + 12867, 12838, 12866, 12851, 12853, 12859, 12849, 12860, 12848, 12837, + 12834, 12841, 12833, 12845, 12844, 12850, 12832, 12830, 12829, 9349, + 9342, 9345, 9346, 9350, 9347, 9348, 9344, 9351, 9343, 9341, 9336, 9335, + 9338, 9337, 9334, 9333, 9340, 9332, 9339, 127248, 127249, 127250, 127251, + 127252, 127253, 127254, 127255, 127256, 127257, 127258, 127259, 127260, + 127261, 127262, 127263, 127264, 127265, 127266, 127267, 127268, 127269, + 127270, 127271, 127272, 127273, 9372, 9373, 9374, 9375, 9376, 9377, 9378, + 9379, 9380, 9381, 9382, 9383, 9384, 9385, 9386, 9387, 9388, 9389, 9390, + 9391, 9392, 9393, 9394, 9395, 9396, 9397, 8706, 983147, 983146, 983149, + 983150, 9853, 127881, 118468, 128890, 12880, 12349, 129436, 128755, + 11261, 9105, 9106, 128706, 72437, 72440, 72432, 72416, 72419, 72413, + 72417, 72415, 72412, 72414, 72418, 72420, 72403, 72407, 72411, 72409, + 72410, 72391, 72400, 72404, 72397, 72394, 72385, 72401, 72384, 72399, + 72398, 72396, 72388, 72393, 72392, 72386, 72387, 72402, 72395, 72390, + 72389, 72405, 72406, 72408, 72436, 72435, 72438, 72439, 72422, 72421, + 72424, 72425, 72431, 72433, 72434, 72428, 72427, 72429, 72430, 72423, + 72426, 128062, 128230, 128206, 983228, 983237, 129434, 9774, 127825, + 129372, 129755, 127824, 128039, 128532, 9956, 128390, 9999, 8240, 8241, + 8524, 10178, 10977, 129336, 129496, 129494, 128113, 9977, 128590, 129733, + 128591, 129493, 128589, 128588, 129495, 128583, 128187, 8966, 128547, 37, + 9854, 127917, 8359, 8369, 129515, 128694, 129730, 43101, 43117, 43120, + 43076, 43123, 43077, 43115, 43090, 43082, 43094, 43098, 43099, 43119, + 43118, 43109, 43074, 43075, 43116, 43079, 43083, 43089, 43088, 43114, + 43113, 43081, 43080, 43073, 43072, 43085, 43084, 43092, 43093, 43104, + 43110, 43086, 43108, 43100, 43078, 43097, 43087, 43106, 43096, 43091, + 43107, 43095, 43102, 43105, 43103, 43127, 43126, 43124, 43121, 43111, + 43112, 43122, 43125, 66033, 66023, 66017, 66010, 66027, 66003, 66018, + 66028, 66004, 66012, 66022, 66020, 66045, 66019, 66031, 66041, 66007, + 66006, 66025, 66026, 66038, 66016, 66013, 66014, 66000, 66001, 66034, + 66036, 66037, 66029, 66011, 66024, 66015, 66021, 66042, 66043, 66002, + 66008, 66032, 66005, 66044, 66040, 66039, 66030, 66009, 66035, 5941, + 5942, 67840, 67855, 67843, 67844, 67847, 67858, 67857, 67860, 67854, + 67861, 67848, 67845, 67846, 67849, 67859, 67842, 67850, 67853, 67851, + 67841, 67856, 67852, 67864, 67866, 67867, 67863, 67862, 67865, 67871, + 11227, 9935, 128763, 128022, 128061, 128055, 128169, 182, 128138, 129292, + 129295, 127885, 127821, 10031, 129655, 129669, 9811, 128299, 8916, 10970, + 129383, 8984, 128720, 129703, 8462, 8463, 128733, 127183, 127136, 127167, + 127199, 127188, 127140, 127156, 127172, 127200, 127189, 127141, 127157, + 127173, 127196, 127148, 127164, 127180, 127198, 127150, 127166, 127182, + 127192, 127144, 127160, 127176, 127191, 127143, 127159, 127175, 127190, + 127142, 127158, 127174, 127197, 127149, 127165, 127181, 127194, 127146, + 127162, 127178, 127187, 127139, 127155, 127171, 127186, 127138, 127154, + 127170, 127202, 127220, 127221, 127201, 127210, 127211, 127212, 127213, + 127214, 127215, 127216, 127217, 127218, 127219, 127203, 127204, 127205, + 127206, 127207, 127208, 127209, 127185, 127137, 127153, 127169, 127193, + 127145, 127161, 127177, 127195, 127147, 127163, 127179, 983151, 43, + 10797, 10798, 10809, 10789, 10786, 10791, 10790, 10788, 10792, 10787, + 10866, 177, 9799, 11222, 11221, 11220, 11219, 129696, 983148, 117777, + 128659, 128680, 128110, 8297, 8236, 127871, 128254, 11239, 128239, 12306, + 12320, 128238, 8982, 128688, 129364, 127858, 129716, 127831, 129751, + 128574, 128545, 163, 128093, 9212, 9213, 9214, 9211, 128041, 128425, + 129328, 129732, 129731, 65043, 65040, 65045, 65073, 65074, 65049, 65041, + 65042, 65091, 65047, 65083, 65085, 65089, 65079, 65087, 65077, 65095, + 65081, 65075, 65084, 65086, 65092, 983261, 65048, 65090, 65080, 65088, + 65078, 65096, 65082, 65076, 65044, 65072, 65046, 8478, 8826, 10937, + 10933, 10927, 10929, 10935, 10931, 8936, 8830, 8828, 8880, 9111, 129384, + 9113, 128424, 128438, 129332, 128120, 983193, 983166, 983163, 983164, + 983167, 8242, 8965, 8759, 8733, 8522, 11224, 128711, 129455, 128255, + 68507, 68508, 68480, 68483, 68490, 68491, 68485, 68482, 68486, 68493, + 68495, 68496, 68488, 68484, 68487, 68489, 68481, 68492, 68497, 68494, + 68526, 68522, 68523, 68525, 68521, 68527, 68524, 68505, 68506, 118473, + 11854, 8200, 128156, 128091, 128686, 128204, 128226, 983165, 983168, + 983194, 9624, 9625, 9626, 9627, 9628, 9629, 9630, 9631, 9622, 9623, + 10764, 8279, 10774, 9833, 128894, 8264, 63, 8799, 34, 9915, 127949, + 127950, 129437, 128251, 9762, 128280, 9143, 128740, 128643, 9926, 127752, + 11827, 11783, 11782, 9995, 128400, 128406, 127338, 127339, 127340, + 129996, 11787, 9994, 11828, 129306, 128000, 8758, 128007, 128048, 129682, + 128015, 129534, 117778, 9852, 9843, 9844, 9845, 9846, 9847, 9848, 9849, + 9850, 983113, 128665, 127822, 129511, 174, 127462, 127463, 127464, + 127465, 127466, 127467, 127468, 127469, 127470, 127471, 127472, 127473, + 127474, 127475, 127476, 127477, 127478, 127479, 127480, 127481, 127482, + 127483, 127484, 127485, 127486, 127487, 43344, 43343, 43346, 43345, + 43333, 43323, 43331, 43314, 43332, 43317, 43330, 43320, 43319, 43321, + 43316, 43313, 43329, 43322, 43312, 43326, 43318, 43325, 43324, 43315, + 43328, 43327, 43334, 43359, 43337, 43342, 43341, 43338, 43340, 43335, + 43339, 43336, 43347, 128524, 127895, 65533, 9952, 9953, 128699, 8479, + 9166, 11152, 11153, 128639, 128968, 983152, 92, 10184, 10741, 10743, + 11073, 11079, 983153, 10659, 10661, 8246, 12317, 10989, 8976, 11793, + 8267, 8245, 128401, 9753, 11262, 8515, 8271, 8765, 8909, 8247, 128402, + 128403, 128405, 11841, 11822, 10672, 128404, 128158, 8251, 129423, + 983154, 127872, 11190, 11188, 11191, 11189, 11186, 11187, 11184, 11185, + 127832, 127833, 129919, 129918, 128495, 8735, 12297, 10642, 11777, 11776, + 9084, 8894, 10652, 10644, 10228, 8692, 12305, 10648, 125, 9132, 9133, + 9131, 12301, 8969, 11781, 12299, 10608, 10715, 8221, 11817, 129929, + 10621, 8971, 118272, 9687, 11241, 9616, 129978, 117925, 118289, 129971, + 118286, 118284, 118432, 118443, 129933, 128381, 130025, 130017, 11805, + 8906, 10198, 129927, 9621, 129980, 41, 9120, 9118, 9119, 11789, 10182, + 8908, 129931, 8217, 11815, 128360, 128361, 128362, 128489, 93, 9126, + 9124, 10638, 10640, 8262, 10636, 11864, 11862, 9125, 11779, 117773, 129987, 128493, 118433, 118442, 129930, 129928, 11786, 8895, 10702, 129902, 12309, 8866, 11809, 9145, 117766, 12303, 10628, 12311, 10630, 12315, 12313, 10713, 1421, 117833, 117907, 117909, 129308, 4053, 4055, @@ -12379,8 +12503,8 @@ static const unsigned int dawg_pos_to_codepoint[] = { 8628, 10513, 8699, 129202, 8620, 129034, 10565, 129042, 129026, 8603, 11022, 11023, 8611, 10517, 10516, 129030, 129178, 129046, 8618, 8696, 8644, 10522, 129193, 129185, 11146, 11157, 8658, 10499, 8655, 10503, - 10524, 10509, 8674, 129195, 129078, 8652, 10601, 10605, 10591, 10583, - 8641, 10600, 10604, 10596, 10587, 10579, 8640, 129777, 129090, 129094, + 10524, 10509, 8674, 129195, 129078, 10601, 10605, 10591, 10583, 8641, + 10600, 10604, 10596, 10587, 10579, 8640, 8652, 129777, 129090, 129094, 129191, 8702, 8649, 129784, 129189, 128622, 8669, 129082, 129106, 129187, 11106, 11138, 11132, 983242, 11175, 11173, 129066, 129062, 129058, 129074, 129070, 11122, 11116, 11142, 129170, 10511, 8667, 10518, 10520, @@ -12419,50 +12543,54 @@ static const unsigned int dawg_pos_to_codepoint[] = { 43149, 43186, 43178, 43180, 43179, 43205, 43136, 43204, 43137, 43215, 43214, 43221, 43220, 43223, 43222, 43219, 43218, 43216, 43225, 43217, 43224, 43188, 43189, 43200, 43203, 43192, 43193, 43194, 43195, 43196, - 43197, 43201, 43202, 43190, 43191, 43198, 43199, 129429, 9973, 127927, - 127862, 129403, 9878, 129507, 127979, 127890, 129410, 9807, 128756, - 129691, 128437, 8492, 8496, 8497, 8459, 8464, 8466, 8499, 8472, 8475, - 128624, 8495, 8458, 8467, 8500, 8456, 128220, 983186, 129453, 128186, - 167, 8980, 129352, 127793, 128584, 8979, 130037, 130036, 130039, 130038, - 130035, 130034, 130032, 130041, 130033, 130040, 10802, 9914, 59, 117793, - 117795, 117799, 117807, 117803, 117797, 117805, 117801, 117794, 117798, - 117806, 117802, 117796, 117804, 117800, 118353, 118355, 118359, 118367, - 118383, 118415, 118399, 118375, 118407, 118391, 118363, 118379, 118411, - 118395, 118371, 118403, 118387, 118357, 118365, 118381, 118413, 118397, - 118373, 118405, 118389, 118361, 118377, 118409, 118393, 118369, 118401, - 118385, 118354, 118358, 118366, 118382, 118414, 118398, 118374, 118406, - 118390, 118362, 118378, 118410, 118394, 118370, 118402, 118386, 118356, - 118364, 118380, 118412, 118396, 118372, 118404, 118388, 118360, 118376, - 118408, 118392, 118368, 118400, 118384, 11259, 8480, 129324, 9916, 65093, - 983169, 8726, 129697, 9913, 11250, 129331, 10061, 10032, 10014, 129368, - 70086, 70085, 70101, 70100, 70103, 70102, 70099, 70098, 70096, 70105, - 70097, 70104, 70092, 70106, 70108, 70019, 70020, 70030, 70032, 70046, - 70045, 70051, 70050, 70044, 70043, 70049, 70048, 70025, 70026, 70027, - 70028, 70062, 70023, 70024, 70021, 70022, 70061, 70060, 70037, 70047, - 70042, 70052, 70063, 70064, 70065, 70056, 70055, 70039, 70038, 70036, - 70035, 70041, 70040, 70034, 70033, 70054, 70053, 70066, 70057, 70059, - 70058, 70029, 70031, 70089, 70110, 70111, 70088, 70095, 70107, 70016, - 70090, 70081, 70017, 70082, 70083, 70080, 70018, 70093, 70091, 70094, - 70067, 70077, 70079, 70070, 70071, 70072, 70073, 70074, 70075, 70068, - 70069, 70076, 70078, 70087, 70109, 70084, 129416, 127847, 66684, 66680, + 43197, 43201, 43202, 43190, 43191, 43198, 43199, 129429, 8385, 9973, + 127927, 127862, 129403, 9878, 129507, 127979, 127890, 129410, 9807, + 128756, 129691, 128437, 8492, 8496, 8497, 8459, 8464, 8466, 8499, 8472, + 8475, 128624, 8495, 8458, 8467, 8500, 8456, 128220, 983186, 129453, + 128186, 167, 8980, 129352, 127793, 128584, 8979, 130037, 130036, 130039, + 130038, 130035, 130034, 130032, 130041, 130033, 130040, 10802, 9914, 59, + 117793, 117795, 117799, 117807, 117803, 117797, 117805, 117801, 117794, + 117798, 117806, 117802, 117796, 117804, 117800, 118353, 118355, 118359, + 118367, 118383, 118415, 118399, 118375, 118407, 118391, 118363, 118379, + 118411, 118395, 118371, 118403, 118387, 118357, 118365, 118381, 118413, + 118397, 118373, 118405, 118389, 118361, 118377, 118409, 118393, 118369, + 118401, 118385, 118354, 118358, 118366, 118382, 118414, 118398, 118374, + 118406, 118390, 118362, 118378, 118410, 118394, 118370, 118402, 118386, + 118356, 118364, 118380, 118412, 118396, 118372, 118404, 118388, 118360, + 118376, 118408, 118392, 118368, 118400, 118384, 11259, 8480, 129324, + 9916, 65093, 983169, 8726, 129697, 9913, 11250, 129331, 10061, 10032, + 10014, 129368, 70086, 70085, 70101, 70100, 70103, 70102, 70099, 70098, + 70096, 70105, 70097, 70104, 70092, 70106, 70108, 70019, 70020, 70030, + 70032, 70046, 70045, 70051, 70050, 70044, 70043, 70049, 70048, 70025, + 70026, 70027, 70028, 70062, 70023, 70024, 70021, 70022, 70061, 70060, + 70037, 70047, 70042, 70052, 70063, 70064, 70065, 70056, 70055, 70039, + 70038, 70036, 70035, 70041, 70040, 70034, 70033, 70054, 70053, 70066, + 70057, 70059, 70058, 70029, 70031, 70089, 70110, 70111, 70088, 70095, + 70107, 70016, 70090, 70081, 70017, 70082, 70083, 70080, 70018, 70093, + 70091, 72550, 72551, 70078, 72545, 72544, 70094, 70070, 70071, 72547, + 72546, 72548, 72549, 70067, 70077, 70079, 70072, 70073, 70074, 70075, + 70068, 70069, 70076, 70087, 70109, 70084, 129416, 127847, 66684, 66680, 66682, 66665, 66673, 66679, 66664, 66669, 66647, 66685, 66672, 66683, 66663, 66659, 66649, 66686, 66674, 66662, 66660, 66656, 66661, 66677, 66678, 66668, 66676, 66666, 66681, 66640, 66670, 66646, 66645, 66644, 66654, 66641, 66667, 66658, 66648, 66687, 66657, 66651, 66655, 66643, 66652, 66650, 66642, 66653, 66671, 66675, 9752, 129768, 128737, 983075, - 983078, 9961, 128674, 127776, 128722, 128717, 11087, 11103, 11086, 10564, - 10976, 10985, 10984, 10974, 10975, 10983, 113825, 113824, 113826, 113827, - 127856, 129651, 9085, 129327, 129679, 128703, 128017, 129424, 129335, - 10722, 983198, 983080, 71113, 71040, 71131, 71041, 71051, 71053, 71128, - 71070, 71129, 71130, 71065, 71064, 71069, 71067, 71066, 71072, 71071, - 71046, 71047, 71048, 71049, 71082, 71044, 71045, 71042, 71043, 71058, - 71068, 71063, 71073, 71083, 71084, 71085, 71077, 71076, 71060, 71059, - 71057, 71056, 71062, 71061, 71055, 71054, 71075, 71074, 71086, 71081, - 71078, 71080, 71079, 71050, 71052, 71110, 71111, 71112, 71119, 71120, - 71127, 71126, 71125, 71123, 71124, 71117, 71118, 71116, 71121, 71115, - 71114, 71122, 71109, 71108, 71105, 71100, 71104, 71101, 71103, 71102, - 71132, 71133, 71087, 71097, 71099, 71092, 71093, 71090, 71091, 71088, - 71089, 71096, 71098, 71107, 71106, 128411, 128410, 128417, 128416, + 983078, 9961, 128674, 127776, 128722, 128717, 11087, 11103, 11086, + 129237, 10974, 10564, 129235, 10976, 10985, 10984, 10975, 10983, 113825, + 113824, 113826, 113827, 127856, 129651, 9085, 129327, 129679, 128703, + 128017, 129424, 129335, 10722, 983198, 983080, 71113, 71040, 71131, + 71041, 71051, 71053, 71128, 71070, 71129, 71130, 71065, 71064, 71069, + 71067, 71066, 71072, 71071, 71046, 71047, 71048, 71049, 71082, 71044, + 71045, 71042, 71043, 71058, 71068, 71063, 71073, 71083, 71084, 71085, + 71077, 71076, 71060, 71059, 71057, 71056, 71062, 71061, 71055, 71054, + 71075, 71074, 71086, 71081, 71078, 71080, 71079, 71050, 71052, 71110, + 71111, 71112, 71119, 71120, 71127, 71126, 71125, 71123, 71124, 71117, + 71118, 71116, 71121, 71115, 71114, 71122, 71109, 71108, 71105, 71100, + 71104, 71101, 71103, 71102, 71132, 71133, 71087, 71097, 71099, 71092, + 71093, 71090, 71091, 71088, 71089, 71096, 71098, 71107, 71106, 67923, + 67924, 67925, 67926, 67927, 67928, 67929, 67904, 67905, 67906, 67907, + 67908, 67909, 67910, 67911, 67912, 67913, 67914, 67915, 67916, 67917, + 67918, 67919, 67920, 67921, 67922, 128411, 128410, 128417, 128416, 128409, 128408, 128415, 128414, 121399, 121397, 121400, 121398, 121402, 121401, 121104, 121103, 121102, 121388, 121387, 121386, 121482, 121479, 121358, 121359, 121357, 121360, 121341, 121335, 121339, 121340, 121336, @@ -12519,18 +12647,18 @@ static const unsigned int dawg_pos_to_codepoint[] = { 121197, 121195, 121216, 121215, 121214, 121213, 121212, 121211, 121449, 121125, 121126, 121121, 121122, 121123, 121124, 121127, 121318, 121316, 121317, 121315, 121156, 121155, 121154, 121146, 121145, 121144, 121150, - 121149, 121148, 121147, 121230, 121231, 121229, 121228, 121261, 121254, + 121149, 121148, 121147, 121230, 121231, 121229, 121228, 121254, 121261, 121247, 121233, 121232, 121226, 121227, 121225, 121224, 121249, 121248, 121153, 121152, 121151, 121139, 121135, 121137, 121138, 121136, 121330, - 121329, 121128, 121236, 121262, 121255, 121235, 121234, 121237, 121240, - 121239, 121263, 121256, 121238, 121162, 121161, 121160, 121132, 121133, + 121329, 121128, 121236, 121255, 121262, 121235, 121234, 121237, 121240, + 121239, 121256, 121263, 121238, 121162, 121161, 121160, 121132, 121133, 121131, 121130, 121134, 121253, 121142, 121143, 121141, 121140, 121243, - 121242, 121241, 121246, 121245, 121244, 121270, 121269, 121268, 121264, - 121257, 121326, 121325, 121159, 121158, 121157, 121448, 121394, 121393, + 121242, 121241, 121246, 121245, 121244, 121270, 121269, 121268, 121257, + 121264, 121326, 121325, 121159, 121158, 121157, 121448, 121394, 121393, 121395, 121396, 121450, 121513, 121514, 121515, 121516, 121517, 121518, 121519, 121505, 121506, 121507, 121508, 121509, 121510, 121511, 121512, 121312, 121284, 121299, 121311, 121283, 121298, 121313, 121285, 121300, - 121267, 121260, 121252, 121251, 121266, 121259, 121250, 121265, 121258, + 121260, 121267, 121252, 121251, 121259, 121266, 121250, 121258, 121265, 121107, 121106, 121105, 121454, 121453, 121457, 121120, 121112, 121110, 121114, 121113, 121111, 121108, 121109, 121101, 121100, 121099, 121481, 121441, 121445, 121446, 121443, 121444, 121442, 121447, 121390, 121389, @@ -12560,326 +12688,346 @@ static const unsigned int dawg_pos_to_codepoint[] = { 65128, 65108, 68411, 732, 118266, 10849, 65125, 65123, 65130, 65122, 65119, 65106, 65110, 10922, 10924, 10803, 128571, 128570, 128525, 128520, 128519, 128515, 128517, 128516, 128518, 128522, 129325, 129392, 128526, - 129394, 8995, 128527, 128684, 128012, 128013, 129319, 127956, 9731, 9924, - 127938, 10052, 983077, 9917, 129510, 173, 127846, 128428, 9108, 129358, - 69453, 69452, 69454, 69456, 69447, 69449, 69446, 69448, 69455, 69451, - 69450, 69445, 69424, 69437, 69426, 69433, 69444, 69440, 69429, 69436, - 69439, 69441, 69431, 69427, 69430, 69432, 69425, 69443, 69435, 69442, - 69428, 69438, 69434, 69457, 69460, 69459, 69458, 69463, 69465, 69461, - 69462, 69464, 128618, 128619, 47, 10742, 128284, 69859, 69863, 69864, - 69846, 69847, 69857, 69849, 69842, 69843, 69844, 69845, 69854, 69856, - 69855, 69848, 69851, 69853, 69840, 69841, 69850, 69852, 69858, 69860, - 69862, 69861, 69877, 69876, 69879, 69878, 69875, 69874, 69872, 69881, - 69873, 69880, 8600, 10537, 10541, 8690, 10533, 129210, 128603, 128595, - 128611, 11112, 11128, 11018, 8664, 129110, 11010, 8601, 10538, 10534, - 129211, 128601, 128593, 128609, 11113, 11129, 11019, 8665, 129111, 11011, - 8471, 72328, 72329, 72327, 72326, 72340, 72339, 72334, 72332, 72341, - 72335, 72333, 72330, 72331, 72338, 72336, 72337, 72352, 72351, 72350, - 72297, 72296, 72302, 72311, 72301, 72323, 72285, 72284, 72288, 72298, - 72293, 72303, 72319, 72320, 72321, 72310, 72309, 72295, 72294, 72300, - 72299, 72307, 72306, 72290, 72289, 72287, 72286, 72292, 72291, 72305, - 72304, 72312, 72313, 72314, 72322, 72317, 72308, 72316, 72318, 72315, - 72272, 72349, 72348, 72347, 72346, 72324, 72343, 72325, 72342, 72345, - 72353, 72354, 72282, 72281, 72279, 72280, 72277, 72278, 72275, 72274, - 72276, 72273, 72283, 72344, 8384, 983043, 983182, 983118, 983177, 127837, - 128150, 10055, 10024, 117824, 117825, 32, 128586, 128264, 128263, 128265, - 128266, 128483, 128676, 128172, 8375, 117830, 117831, 8738, 10656, 10657, - 128375, 128376, 128467, 128466, 128026, 128166, 129759, 129525, 129348, - 128051, 127941, 129533, 13279, 117900, 13056, 13058, 13057, 13059, 13250, - 13171, 13278, 13101, 13172, 13108, 13105, 13118, 13116, 13251, 13192, - 8851, 13255, 13183, 13213, 13220, 13216, 13254, 8852, 13252, 13253, - 13170, 13092, 13175, 13177, 13176, 13093, 13094, 13256, 127376, 13207, - 13064, 13179, 13181, 13182, 13055, 13180, 13005, 13063, 13006, 117899, - 117898, 117897, 9974, 9165, 13209, 13070, 13071, 13311, 13075, 13073, - 13072, 13080, 13081, 13203, 13228, 13191, 13257, 13258, 13098, 13110, - 13113, 13121, 13122, 13119, 13107, 13106, 13109, 13259, 13169, 127488, - 13004, 13200, 13260, 13060, 13061, 8847, 8932, 8849, 13178, 13188, 13069, - 13068, 13067, 13074, 13076, 13078, 13079, 13077, 13214, 13262, 13222, - 13218, 13086, 13085, 13082, 13083, 13084, 13201, 13193, 13248, 13226, - 13189, 13199, 13261, 13208, 13263, 13240, 13246, 8977, 13266, 10957, - 13264, 13265, 13267, 13223, 13224, 13249, 13221, 13217, 13187, 13123, - 13124, 13127, 13126, 13125, 13190, 13268, 13131, 13132, 13133, 13128, - 13129, 13130, 13269, 13212, 13219, 13215, 13186, 13196, 13197, 13205, - 13211, 13234, 13238, 13244, 13239, 13241, 13245, 13247, 13202, 13270, - 13227, 13198, 13206, 13235, 13185, 13096, 13097, 13195, 13210, 13233, - 13237, 13243, 13065, 8848, 8933, 8850, 13066, 13173, 13225, 13099, 13100, - 13184, 13115, 13112, 13114, 13111, 13103, 13104, 13102, 13117, 13120, - 11216, 13273, 13174, 13194, 13271, 13272, 13274, 13232, 13236, 13242, - 13229, 13230, 13231, 13142, 13141, 10958, 13137, 13138, 13140, 13139, - 8730, 13087, 13088, 13090, 13091, 117887, 117886, 117884, 117885, 13089, - 13275, 13276, 128918, 13204, 13095, 13143, 11027, 9641, 9638, 9636, - 11029, 9706, 9703, 11026, 9705, 9639, 11028, 9640, 9637, 9704, 10720, - 13277, 13135, 13134, 13136, 13062, 127529, 127530, 127512, 127508, - 127533, 127545, 127520, 127516, 127534, 127506, 127540, 127525, 127546, - 127532, 127511, 127509, 127524, 127505, 127517, 127518, 127527, 127537, - 127504, 127528, 127535, 127519, 127515, 127513, 127543, 127542, 127526, - 127541, 127544, 127522, 127538, 127514, 127521, 127539, 127510, 127536, - 127523, 127547, 127531, 127378, 127377, 8865, 10693, 11820, 127390, - 127392, 127379, 10692, 127400, 127399, 127398, 127306, 127489, 127507, - 127490, 9919, 127397, 127280, 127281, 127282, 127283, 127284, 127285, - 127286, 127287, 127288, 127289, 127290, 127291, 127292, 127293, 127294, - 127295, 127296, 127297, 127298, 127299, 127300, 127301, 127302, 127303, - 127304, 127305, 10190, 10191, 127401, 8863, 127307, 127381, 127382, - 127396, 127383, 127310, 8862, 127388, 127393, 127402, 127395, 9949, - 10695, 10696, 127384, 127308, 127309, 127387, 127394, 127389, 8864, - 127391, 127385, 127403, 127404, 127386, 10694, 127311, 127380, 10151, - 129425, 983157, 983160, 983134, 983190, 9877, 9882, 128387, 8795, 10017, - 8902, 9770, 11242, 11243, 983175, 983133, 983181, 983176, 983042, 983044, - 128509, 128649, 129485, 127967, 128642, 127836, 11836, 129658, 129989, - 129990, 129993, 129991, 129992, 128480, 9201, 128207, 9188, 127827, - 10025, 983189, 117891, 117888, 117890, 117889, 8803, 127897, 129369, - 128723, 983170, 983045, 983103, 8333, 8332, 8328, 8330, 8334, 8331, 8325, - 8324, 8327, 8326, 8323, 8322, 8320, 8329, 8321, 10963, 10965, 10617, - 8834, 10953, 10949, 10951, 10955, 8842, 8838, 10947, 10943, 10945, 10941, - 983102, 8827, 10938, 10934, 10928, 10930, 10936, 10932, 8937, 8831, 8829, - 8881, 9139, 10763, 9138, 9737, 127774, 9925, 7098, 7073, 7074, 7075, - 7084, 7085, 7043, 983220, 7046, 7102, 7103, 7062, 7100, 7068, 7099, 7067, - 7087, 7070, 7048, 7049, 7053, 7057, 7060, 7101, 7064, 7086, 7050, 7054, - 7059, 7052, 7072, 7055, 7065, 7061, 7051, 7058, 7063, 7069, 7071, 7066, - 7056, 7044, 7047, 7045, 7367, 7366, 7365, 7364, 7363, 7361, 7362, 7360, - 7082, 7041, 7042, 7040, 7083, 7080, 7081, 7079, 7077, 7076, 7078, 7093, - 7092, 7095, 7094, 7091, 7090, 7088, 7097, 7089, 7096, 127749, 127748, - 72648, 72662, 72661, 72669, 72652, 72640, 72663, 72651, 72655, 72672, - 72646, 72667, 72666, 72653, 72657, 72645, 72665, 72649, 72644, 72658, - 72668, 72670, 72664, 72671, 72641, 72659, 72650, 72656, 72643, 72660, - 72654, 72642, 72647, 72673, 72693, 72692, 72695, 72694, 72691, 72690, - 72688, 72697, 72689, 72696, 127751, 127803, 8316, 8312, 8305, 8319, 8317, - 8314, 8318, 8315, 8309, 8308, 8311, 8310, 179, 178, 8304, 8313, 185, - 10966, 10964, 10619, 10968, 10967, 8835, 10954, 10950, 10952, 10956, - 8843, 8839, 10948, 10944, 10946, 10942, 10185, 129464, 129465, 8751, - 127940, 128671, 127843, 128629, 129442, 127946, 8275, 43027, 43026, - 43031, 43030, 43040, 43038, 43025, 43024, 43029, 43028, 43036, 43035, - 43021, 43020, 43018, 43017, 43023, 43022, 43016, 43015, 43034, 43033, - 43042, 43039, 43037, 43032, 43041, 43008, 43012, 43009, 43013, 43011, - 43048, 43049, 43050, 43051, 43052, 43019, 43014, 43010, 43047, 43043, - 43046, 43044, 43045, 9223, 9224, 9229, 9240, 9232, 9249, 9256, 9255, - 9253, 9257, 9236, 9235, 9234, 9233, 9241, 9220, 9239, 9219, 9221, 9243, - 9244, 9228, 9225, 9227, 9226, 128325, 9237, 9252, 9216, 9222, 9246, 9245, - 9247, 8527, 9230, 9231, 9217, 9218, 9242, 9254, 9238, 9248, 11159, 9007, - 983094, 983093, 128333, 1807, 1866, 1802, 1798, 1799, 1849, 1848, 1792, - 1854, 1853, 1805, 1804, 1803, 1852, 1851, 1850, 1797, 1814, 1813, 1815, - 1818, 1824, 2153, 2152, 2149, 2148, 2144, 2146, 2150, 2147, 2154, 2145, - 2151, 1825, 1830, 1837, 1839, 1838, 1831, 1834, 1827, 1869, 1870, 1871, - 1809, 1835, 1832, 1828, 1808, 1823, 1833, 1819, 1820, 1836, 1811, 1812, - 1821, 1822, 1810, 1817, 1826, 1816, 1829, 1864, 1863, 1842, 1841, 1840, - 1845, 1844, 1843, 1847, 1846, 1855, 1858, 1796, 983204, 1801, 1794, 1795, - 1800, 1793, 1862, 1861, 1860, 1859, 1865, 1856, 1857, 128137, 983184, - 128085, 129430, 983061, 127955, 917543, 917542, 917546, 917598, 917568, - 917548, 917562, 917540, 917557, 917556, 917559, 917558, 917555, 917554, - 917552, 917561, 917553, 917560, 917565, 917537, 917600, 917566, 917549, - 917569, 917570, 917571, 917572, 917573, 917574, 917575, 917576, 917577, - 917578, 917579, 917580, 917581, 917582, 917583, 917584, 917585, 917586, - 917587, 917588, 917589, 917590, 917591, 917592, 917593, 917594, 917601, - 917602, 917603, 917604, 917605, 917606, 917607, 917608, 917609, 917610, - 917611, 917612, 917613, 917614, 917615, 917616, 917617, 917618, 917619, - 917620, 917621, 917622, 917623, 917624, 917625, 917626, 917564, 917627, - 917544, 917595, 917599, 917541, 917547, 917538, 917567, 917629, 917545, - 917597, 917596, 917563, 917551, 917536, 917539, 917630, 917550, 917628, - 5888, 5919, 5893, 5896, 5898, 5895, 5892, 5905, 5891, 5902, 5899, 5897, - 5901, 5904, 5894, 5903, 5900, 5889, 5890, 5909, 5908, 5906, 5907, 5989, - 5992, 5994, 5991, 5988, 5987, 5998, 5995, 5993, 6000, 5990, 5999, 5996, - 5984, 5985, 5986, 6002, 6003, 6499, 6508, 6509, 6507, 6501, 6502, 6512, - 6513, 6514, 6515, 6516, 6497, 6483, 6487, 6486, 6482, 6498, 6505, 6504, - 6496, 6480, 6490, 6489, 6503, 6506, 6492, 6494, 6488, 6491, 6495, 6484, - 6493, 6481, 6485, 6500, 6745, 6746, 6743, 6747, 6742, 6741, 6748, 6749, - 6750, 6783, 6789, 6788, 6791, 6790, 6787, 6786, 6784, 6793, 6785, 6792, - 6805, 6804, 6807, 6806, 6803, 6802, 6800, 6809, 6801, 6808, 6740, 6689, - 6690, 6688, 6702, 6726, 6727, 6728, 6696, 6695, 6713, 6712, 6707, 6706, - 6714, 6729, 6720, 6693, 6692, 6691, 6704, 6699, 6697, 6717, 6715, 6709, - 6708, 6716, 6732, 6698, 6719, 6723, 6739, 6724, 6730, 6721, 6705, 6701, - 6722, 6735, 6736, 6733, 6734, 6694, 6700, 6710, 6738, 6737, 6711, 6703, - 6718, 6725, 6731, 6828, 6820, 6775, 6776, 6777, 6780, 6824, 6825, 6819, - 6772, 6744, 6823, 6779, 6778, 6822, 6826, 6827, 6818, 6752, 6816, 6817, - 6821, 6773, 6774, 6829, 6753, 6755, 6767, 6769, 6754, 6763, 6764, 6771, - 6768, 6765, 6756, 6770, 6761, 6762, 6760, 6759, 6757, 6758, 6766, 43653, - 43651, 43649, 43661, 43659, 43679, 43677, 43671, 43669, 43657, 43665, - 43673, 43675, 43667, 43681, 43655, 43693, 43689, 43683, 43687, 43663, - 43691, 43685, 43695, 43652, 43650, 43648, 43660, 43658, 43678, 43676, - 43670, 43668, 43656, 43664, 43672, 43674, 43666, 43680, 43654, 43692, - 43688, 43682, 43686, 43662, 43690, 43684, 43694, 43696, 43703, 43743, - 43739, 43740, 43742, 43741, 43712, 43713, 43714, 43711, 43707, 43697, - 43710, 43709, 43708, 43700, 43699, 43705, 43706, 43698, 43704, 43701, - 43702, 71353, 71296, 71352, 71297, 71303, 71305, 71319, 71318, 71324, - 71323, 71338, 71332, 71317, 71316, 71322, 71321, 71300, 71301, 71298, - 71299, 71310, 71320, 71315, 71325, 71329, 71328, 71312, 71311, 71309, - 71308, 71314, 71313, 71307, 71306, 71327, 71326, 71335, 71336, 71337, - 71333, 71330, 71334, 71331, 71302, 71304, 71351, 71339, 71350, 71340, - 71341, 71347, 71349, 71344, 71345, 71342, 71343, 71346, 71348, 71365, - 71364, 71367, 71366, 71363, 71362, 71360, 71369, 71361, 71368, 129377, - 119672, 119671, 3064, 3031, 73707, 983662, 983685, 983674, 983677, - 983676, 983669, 983667, 983679, 983663, 983665, 983668, 983666, 983683, - 983681, 983682, 983673, 983678, 983664, 983684, 983680, 983671, 983670, - 983675, 983672, 73706, 3063, 73701, 3062, 3059, 3051, 3050, 3053, 3052, - 3049, 3048, 3046, 3055, 3047, 3054, 73700, 73666, 73676, 73668, 73679, - 73681, 73682, 73665, 73673, 73674, 73667, 73664, 73669, 73672, 73675, - 73680, 73670, 73678, 73671, 73677, 73683, 73684, 73710, 2985, 2979, 2969, - 2974, 2984, 2975, 2980, 2949, 2950, 2960, 2964, 2996, 2995, 2994, 2993, - 2992, 2953, 2954, 2962, 2963, 2951, 2952, 2998, 2999, 3000, 2958, 2959, - 2970, 3001, 2972, 2965, 2990, 2986, 2997, 2991, 73702, 3057, 3058, 3056, - 3066, 73727, 3065, 73703, 73687, 2946, 73686, 73698, 73690, 73693, 73692, - 73712, 73689, 73688, 73691, 73697, 73694, 73695, 73696, 73713, 73699, - 3021, 2947, 73685, 73708, 73711, 983939, 983940, 983947, 983950, 983943, - 983944, 983948, 983949, 983941, 983942, 983945, 983946, 983686, 983693, - 983696, 983689, 983690, 983694, 983695, 983687, 983688, 983691, 983692, - 983840, 983847, 983850, 983843, 983844, 983848, 983849, 983841, 983842, - 983845, 983846, 983851, 983858, 983861, 983854, 983855, 983859, 983860, - 983852, 983853, 983856, 983857, 983818, 983825, 983828, 983821, 983822, - 983826, 983827, 983819, 983820, 983823, 983824, 983873, 983880, 983883, - 983876, 983877, 983881, 983882, 983874, 983875, 983878, 983879, 983741, - 983748, 983751, 983744, 983745, 983749, 983750, 983742, 983743, 983746, - 983747, 983697, 983704, 983707, 983700, 983701, 983705, 983706, 983698, - 983699, 983702, 983703, 983719, 983726, 983729, 983722, 983723, 983727, - 983728, 983720, 983721, 983724, 983725, 983763, 983770, 983773, 983766, - 983767, 983771, 983772, 983764, 983765, 983768, 983769, 983862, 983869, - 983872, 983865, 983866, 983870, 983871, 983863, 983864, 983867, 983868, - 983807, 983814, 983817, 983810, 983811, 983815, 983816, 983808, 983809, - 983812, 983813, 983895, 983902, 983905, 983898, 983899, 983903, 983904, - 983951, 983896, 983897, 983900, 983901, 983906, 983913, 983916, 983909, - 983910, 983914, 983915, 983907, 983908, 983911, 983912, 983917, 983924, - 983927, 983920, 983921, 983925, 983926, 983918, 983919, 983922, 983923, - 983730, 983737, 983740, 983733, 983734, 983738, 983739, 983731, 983732, - 983735, 983736, 983752, 983759, 983762, 983755, 983756, 983760, 983761, - 983753, 983754, 983757, 983758, 983708, 983715, 983718, 983711, 983712, - 983716, 983717, 983709, 983710, 983713, 983714, 983928, 983935, 983938, - 983931, 983932, 983936, 983937, 983929, 983930, 983933, 983934, 983884, - 983891, 983894, 983887, 983888, 983892, 983893, 983885, 983886, 983889, - 983890, 983785, 983792, 983795, 983788, 983789, 983793, 983794, 983786, - 983787, 983790, 983791, 983774, 983781, 983784, 983777, 983778, 983782, - 983783, 983775, 983776, 983779, 983780, 983829, 983836, 983839, 983832, - 983833, 983837, 983838, 983830, 983831, 983834, 983835, 983796, 983803, - 983806, 983799, 983800, 983804, 983805, 983797, 983798, 983801, 983802, - 73709, 73704, 73705, 3006, 3016, 3020, 3009, 3010, 3018, 3019, 3007, - 3008, 3014, 3015, 3061, 3060, 3024, 129748, 127883, 127818, 92809, 92810, - 92811, 92808, 92789, 92790, 92791, 92788, 92816, 92859, 92856, 92847, - 92845, 92817, 92846, 92843, 92829, 92830, 92831, 92828, 92835, 92851, - 92840, 92844, 92818, 92819, 92852, 92836, 92825, 92826, 92827, 92824, - 92813, 92814, 92815, 92812, 92820, 92822, 92823, 92821, 92805, 92806, - 92807, 92804, 92797, 92798, 92799, 92796, 92801, 92802, 92803, 92800, - 92785, 92786, 92787, 92784, 92793, 92794, 92795, 92792, 92857, 92854, - 92848, 92861, 92853, 92860, 92849, 92855, 92834, 92833, 92832, 92841, - 92839, 92842, 92850, 92838, 92858, 92837, 92862, 92869, 92868, 92871, - 92870, 92867, 92866, 92864, 92873, 92865, 92872, 100352, 100353, 100354, - 100355, 100356, 100357, 100358, 100359, 100360, 100361, 100362, 100363, - 100364, 100365, 100366, 100367, 100368, 100369, 100370, 100371, 100372, - 100373, 100374, 100375, 100376, 100377, 100378, 100379, 100380, 100381, - 100382, 100383, 100384, 100385, 100386, 100387, 100388, 100389, 100390, - 100391, 100392, 100393, 100394, 100395, 100396, 100397, 100398, 100399, - 100400, 100401, 100402, 100403, 100404, 100405, 100406, 100407, 100408, - 100409, 100410, 100411, 100412, 100413, 100414, 100415, 100416, 100417, - 100418, 100419, 100420, 100421, 100422, 100423, 100424, 100425, 100426, - 100427, 100428, 100429, 100430, 100431, 100432, 100433, 100434, 100435, - 100436, 100437, 100438, 100439, 100440, 100441, 100442, 100443, 100444, - 100445, 100446, 100447, 100448, 100449, 100450, 100451, 100452, 100453, - 100454, 100455, 100456, 100457, 100458, 100459, 100460, 100461, 100462, - 100463, 100464, 100465, 100466, 100467, 100468, 100469, 100470, 100471, - 100472, 100473, 100474, 100475, 100476, 100477, 100478, 100479, 100480, - 100481, 100482, 100483, 100484, 100485, 100486, 100487, 100488, 100489, - 100490, 100491, 100492, 100493, 100494, 100495, 100496, 100497, 100498, - 100499, 100500, 100501, 100502, 100503, 100504, 100505, 100506, 100507, - 100508, 100509, 100510, 100511, 100512, 100513, 100514, 100515, 100516, - 100517, 100518, 100519, 100520, 100521, 100522, 100523, 100524, 100525, - 100526, 100527, 100528, 100529, 100530, 100531, 100532, 100533, 100534, - 100535, 100536, 100537, 100538, 100539, 100540, 100541, 100542, 100543, - 100544, 100545, 100546, 100547, 100548, 100549, 100550, 100551, 100552, - 100553, 100554, 100555, 100556, 100557, 100558, 100559, 100560, 100561, - 100562, 100563, 100564, 100565, 100566, 100567, 100568, 100569, 100570, - 100571, 100572, 100573, 100574, 100575, 100576, 100577, 100578, 100579, - 100580, 100581, 100582, 100583, 100584, 100585, 100586, 100587, 100588, - 100589, 100590, 100591, 100592, 100593, 100594, 100595, 100596, 100597, - 100598, 100599, 100600, 100601, 100602, 100603, 100604, 100605, 100606, - 100607, 100608, 100609, 100610, 100611, 100612, 100613, 100614, 100615, - 100616, 100617, 100618, 100619, 100620, 100621, 100622, 100623, 100624, - 100625, 100626, 100627, 100628, 100629, 100630, 100631, 100632, 100633, - 100634, 100635, 100636, 100637, 100638, 100639, 100640, 100641, 100642, - 100643, 100644, 100645, 100646, 100647, 100648, 100649, 100650, 100651, - 100652, 100653, 100654, 100655, 100656, 100657, 100658, 100659, 100660, - 100661, 100662, 100663, 100664, 100665, 100666, 100667, 100668, 100669, - 100670, 100671, 100672, 100673, 100674, 100675, 100676, 100677, 100678, - 100679, 100680, 100681, 100682, 100683, 100684, 100685, 100686, 100687, - 100688, 100689, 100690, 100691, 100692, 100693, 100694, 100695, 100696, - 100697, 100698, 100699, 100700, 100701, 100702, 100703, 100704, 100705, - 100706, 100707, 100708, 100709, 100710, 100711, 100712, 100713, 100714, - 100715, 100716, 100717, 100718, 100719, 100720, 100721, 100722, 100723, - 100724, 100725, 100726, 100727, 100728, 100729, 100730, 100731, 100732, - 100733, 100734, 100735, 100736, 100737, 100738, 100739, 100740, 100741, - 100742, 100743, 100744, 100745, 100746, 100747, 100748, 100749, 100750, - 100751, 100752, 100753, 100754, 100755, 100756, 100757, 100758, 100759, - 100760, 100761, 100762, 100763, 100764, 100765, 100766, 100767, 100768, - 100769, 100770, 100771, 100772, 100773, 100774, 100775, 100776, 100777, - 100778, 100779, 100780, 100781, 100782, 100783, 100784, 100785, 100786, - 100787, 100788, 100789, 100790, 100791, 100792, 100793, 100794, 100795, - 100796, 100797, 100798, 100799, 100800, 100801, 100802, 100803, 100804, - 100805, 100806, 100807, 100808, 100809, 100810, 100811, 100812, 100813, - 100814, 100815, 100816, 100817, 100818, 100819, 100820, 100821, 100822, - 100823, 100824, 100825, 100826, 100827, 100828, 100829, 100830, 100831, - 100832, 100833, 100834, 100835, 100836, 100837, 100838, 100839, 100840, - 100841, 100842, 100843, 100844, 100845, 100846, 100847, 100848, 100849, - 100850, 100851, 100852, 100853, 100854, 100855, 100856, 100857, 100858, - 100859, 100860, 100861, 100862, 100863, 100864, 100865, 100866, 100867, - 100868, 100869, 100870, 100871, 100872, 100873, 100874, 100875, 100876, - 100877, 100878, 100879, 100880, 100881, 100882, 100883, 100884, 100885, - 100886, 100887, 100888, 100889, 100890, 100891, 100892, 100893, 100894, - 100895, 100896, 100897, 100898, 100899, 100900, 100901, 100902, 100903, - 100904, 100905, 100906, 100907, 100908, 100909, 100910, 100911, 100912, - 100913, 100914, 100915, 100916, 100917, 100918, 100919, 100920, 100921, - 100922, 100923, 100924, 100925, 100926, 100927, 100928, 100929, 100930, - 100931, 100932, 100933, 100934, 100935, 100936, 100937, 100938, 100939, - 100940, 100941, 100942, 100943, 100944, 100945, 100946, 100947, 100948, - 100949, 100950, 100951, 100952, 100953, 100954, 100955, 100956, 100957, - 100958, 100959, 100960, 100961, 100962, 100963, 100964, 100965, 100966, - 100967, 100968, 100969, 100970, 100971, 100972, 100973, 100974, 100975, - 100976, 100977, 100978, 100979, 100980, 100981, 100982, 100983, 100984, - 100985, 100986, 100987, 100988, 100989, 100990, 100991, 100992, 100993, - 100994, 100995, 100996, 100997, 100998, 100999, 101000, 101001, 101002, - 101003, 101004, 101005, 101006, 101007, 101008, 101009, 101010, 101011, - 101012, 101013, 101014, 101015, 101016, 101017, 101018, 101019, 101020, - 101021, 101022, 101023, 101024, 101025, 101026, 101027, 101028, 101029, - 101030, 101031, 101032, 101033, 101034, 101035, 101036, 101037, 101038, - 101039, 101040, 101041, 101042, 101043, 101044, 101045, 101046, 101047, - 101048, 101049, 101050, 101051, 101052, 101053, 101054, 101055, 101056, - 101057, 101058, 101059, 101060, 101061, 101062, 101063, 101064, 101065, - 101066, 101067, 101068, 101069, 101070, 101071, 101072, 101073, 101074, - 101075, 101076, 101077, 101078, 101079, 101080, 101081, 101082, 101083, - 101084, 101085, 101086, 101087, 101088, 101089, 101090, 101091, 101092, - 101093, 101094, 101095, 101096, 101097, 101098, 101099, 101100, 101101, - 101102, 101103, 101104, 101105, 101106, 101107, 101108, 101109, 101110, - 101111, 101112, 101113, 101114, 101115, 101116, 101117, 101118, 101119, - 94176, 128429, 9991, 9801, 127790, 128661, 127861, 128198, 10043, 10170, - 129750, 129528, 128222, 128380, 8981, 9990, 8481, 128384, 128301, 128250, - 3158, 3195, 3198, 3194, 3197, 3193, 3196, 3192, 3106, 3105, 3111, 3161, - 3110, 3112, 3165, 3097, 3107, 3102, 3162, 3121, 3120, 3104, 3103, 3109, - 3160, 3108, 3077, 3078, 3088, 3092, 3124, 3123, 3122, 3083, 3168, 3084, - 3169, 3125, 3081, 3082, 3090, 3091, 3079, 3080, 3126, 3127, 3128, 3117, - 3116, 3099, 3098, 3096, 3095, 3101, 3100, 3094, 3093, 3115, 3114, 3086, - 3087, 3129, 3118, 3119, 3157, 3076, 3072, 3073, 3191, 3199, 3132, 3133, - 3074, 3149, 3075, 3134, 3144, 3148, 3137, 3138, 3139, 3140, 3170, 3171, - 3146, 3147, 3135, 3136, 3142, 3143, 3179, 3178, 3181, 3180, 3177, 3176, - 3174, 3183, 3175, 3182, 127934, 8376, 9978, 129514, 119617, 119564, - 119577, 119633, 119587, 119566, 119561, 119585, 119613, 119590, 119631, - 119634, 119630, 119608, 119582, 119563, 119573, 119558, 119624, 119567, - 119623, 119586, 119636, 119612, 119625, 119568, 119584, 119619, 119618, - 119583, 119603, 119600, 119626, 119610, 119580, 119576, 119638, 119559, - 119595, 119632, 119606, 119592, 119615, 119599, 119602, 119614, 119629, - 119574, 119569, 119570, 119622, 119562, 119591, 119637, 119597, 119589, - 119616, 119609, 119560, 119635, 119565, 119604, 119588, 119571, 119594, - 119578, 119596, 119579, 119598, 119572, 119605, 119627, 119621, 119628, - 119601, 119593, 119607, 119575, 119620, 119611, 119581, 1959, 1958, 1964, - 1961, 1965, 1927, 1954, 1951, 1937, 1931, 1930, 1956, 1934, 1935, 1955, - 1945, 1920, 1926, 1946, 1933, 1925, 1929, 1943, 1942, 1941, 1922, 1969, - 1950, 1949, 1921, 1936, 1939, 1932, 1947, 1944, 1952, 1928, 1957, 1938, - 1948, 1953, 1924, 1923, 1940, 1967, 1966, 1963, 1960, 1962, 1968, 3610, - 3592, 3594, 3593, 3596, 3598, 3604, 3613, 3615, 3663, 3630, 3627, 3588, - 3587, 3589, 3586, 3590, 3675, 3585, 3628, 3621, 3653, 3622, 3659, 3633, - 3657, 3658, 3656, 3655, 3654, 3617, 3674, 3591, 3661, 3603, 3609, 3631, - 3642, 3612, 3614, 3616, 3611, 3619, 3620, 3632, 3652, 3651, 3634, 3649, - 3635, 3640, 3638, 3639, 3641, 3636, 3637, 3648, 3650, 3625, 3624, 3626, - 3595, 3660, 3601, 3602, 3607, 3600, 3608, 3606, 3605, 3599, 3623, 3662, - 3597, 3618, 3629, 3647, 3669, 3668, 3671, 3670, 3667, 3666, 3664, 3673, - 3665, 3672, 8708, 8707, 8756, 127777, 10727, 128936, 8201, 128929, + 129394, 8995, 128527, 128684, 128013, 118010, 128012, 129319, 127956, + 9731, 9924, 127938, 10052, 983077, 9917, 129510, 173, 127846, 128428, + 9108, 129358, 69453, 69452, 69454, 69456, 69447, 69449, 69446, 69448, + 69455, 69451, 69450, 69445, 69424, 69437, 69426, 69433, 69444, 69440, + 69429, 69436, 69439, 69441, 69431, 69427, 69430, 69432, 69425, 69443, + 69435, 69442, 69428, 69438, 69434, 69457, 69460, 69459, 69458, 69463, + 69465, 69461, 69462, 69464, 128618, 128619, 47, 10742, 128284, 69859, + 69863, 69864, 69846, 69847, 69857, 69849, 69842, 69843, 69844, 69845, + 69854, 69856, 69855, 69848, 69851, 69853, 69840, 69841, 69850, 69852, + 69858, 69860, 69862, 69861, 69877, 69876, 69879, 69878, 69875, 69874, + 69872, 69881, 69873, 69880, 8600, 10537, 10541, 8690, 10533, 129210, + 128603, 128595, 128611, 11112, 11128, 11018, 8664, 129110, 11010, 8601, + 10538, 10534, 129211, 128601, 128593, 128609, 11113, 11129, 11019, 8665, + 129111, 11011, 8471, 72328, 72329, 72327, 72326, 72340, 72339, 72334, + 72332, 72341, 72335, 72333, 72330, 72331, 72338, 72336, 72337, 72352, + 72351, 72350, 72297, 72296, 72302, 72311, 72301, 72323, 72285, 72284, + 72288, 72298, 72293, 72303, 72319, 72320, 72321, 72310, 72309, 72295, + 72294, 72300, 72299, 72307, 72306, 72290, 72289, 72287, 72286, 72292, + 72291, 72305, 72304, 72312, 72313, 72314, 72322, 72317, 72308, 72316, + 72318, 72315, 72272, 72349, 72348, 72347, 72346, 72324, 72343, 72325, + 72342, 72345, 72353, 72354, 72282, 72281, 72279, 72280, 72277, 72278, + 72275, 72274, 72276, 72273, 72283, 72344, 8384, 983043, 983182, 983118, + 983177, 127837, 128150, 10055, 10024, 117824, 117825, 32, 128586, 128264, + 128263, 128265, 128266, 128483, 128676, 128172, 8375, 117830, 117831, + 8738, 10656, 10657, 128375, 128376, 128467, 128466, 128026, 128166, + 129759, 129525, 129348, 128051, 127941, 129533, 13279, 117900, 13056, + 13058, 13057, 13059, 13250, 13171, 13278, 13101, 13172, 13108, 13105, + 13118, 13116, 13251, 13192, 8851, 13255, 13183, 13213, 13220, 13216, + 13254, 8852, 13252, 13253, 13170, 13092, 13175, 13177, 13176, 13093, + 13094, 13256, 127376, 13207, 13064, 13179, 13181, 13182, 13055, 13180, + 13005, 13063, 13006, 117899, 117898, 117897, 9974, 9165, 13209, 13070, + 13071, 13311, 13075, 13073, 13072, 13080, 13081, 13203, 13228, 13191, + 13257, 13258, 13098, 13110, 13113, 13121, 13122, 13119, 13107, 13106, + 13109, 13259, 13169, 127488, 13004, 13200, 13260, 13060, 13061, 8847, + 8932, 8849, 13178, 13188, 13068, 13069, 13067, 13076, 13078, 13079, + 13077, 13214, 13262, 13222, 13218, 13086, 13085, 13082, 13083, 13084, + 13074, 13201, 13193, 13248, 13226, 13189, 13199, 13261, 13208, 13263, + 13240, 13246, 8977, 13266, 10957, 13264, 13265, 13267, 13223, 13224, + 13249, 13221, 13217, 13187, 13123, 13124, 13127, 13126, 13125, 13190, + 13268, 13131, 13132, 13133, 13128, 13129, 13130, 13269, 13212, 13219, + 13215, 13186, 13196, 13197, 13205, 13211, 13234, 13238, 13244, 13239, + 13241, 13245, 13247, 13202, 13270, 13227, 13198, 13206, 13235, 13185, + 13096, 13097, 13195, 13210, 13233, 13237, 13243, 13065, 8848, 8933, 8850, + 13066, 13173, 13225, 13099, 13100, 13184, 13115, 13112, 13114, 13111, + 13103, 13104, 13102, 13117, 13120, 11216, 13273, 13174, 13194, 13271, + 13272, 13274, 13232, 13236, 13242, 13229, 13230, 13231, 13142, 13141, + 10958, 13137, 13138, 13140, 13139, 8730, 13087, 13088, 13090, 13091, + 117887, 117886, 117884, 117885, 13089, 13275, 13276, 128918, 13204, + 13095, 13143, 11027, 9641, 9638, 9636, 11029, 9706, 9703, 11026, 9705, + 9639, 11028, 9640, 9637, 9704, 10720, 13277, 13135, 13134, 13136, 13062, + 127529, 127530, 127512, 127508, 127533, 127545, 127520, 127516, 127534, + 127506, 127540, 127525, 127546, 127532, 127511, 127509, 127524, 127505, + 127517, 127518, 127527, 127537, 127504, 127528, 127535, 127519, 127515, + 127513, 127543, 127542, 127526, 127541, 127544, 127522, 127538, 127514, + 127521, 127539, 127510, 127536, 127523, 127547, 127531, 127378, 127377, + 8865, 10693, 11820, 127390, 127392, 127379, 10692, 127400, 127399, + 127398, 127306, 127489, 127507, 127490, 9919, 127397, 127280, 127281, + 127282, 127283, 127284, 127285, 127286, 127287, 127288, 127289, 127290, + 127291, 127292, 127293, 127294, 127295, 127296, 127297, 127298, 127299, + 127300, 127301, 127302, 127303, 127304, 127305, 10190, 10191, 127401, + 8863, 127307, 127381, 127382, 127396, 127383, 127310, 8862, 127388, + 127393, 127402, 127395, 9949, 10695, 10696, 127384, 127308, 127309, + 127387, 127394, 127389, 8864, 127391, 127385, 127403, 127404, 127386, + 10694, 127311, 127380, 10151, 129425, 983157, 983160, 983134, 983190, + 9877, 9882, 128387, 8795, 10017, 8902, 9770, 11242, 11243, 983175, + 983133, 983181, 983176, 983042, 983044, 128509, 128649, 129485, 127967, + 128642, 127836, 11836, 129658, 129989, 129990, 129993, 129991, 129992, + 128480, 9201, 128207, 9188, 127827, 118463, 10025, 983189, 117891, + 117888, 117890, 117889, 8803, 127897, 129369, 128723, 983170, 983045, + 983103, 8333, 8332, 8328, 8330, 8334, 8331, 8325, 8324, 8327, 8326, 8323, + 8322, 8320, 8329, 8321, 10963, 10965, 10617, 8834, 10953, 10949, 10951, + 10955, 8842, 8838, 10947, 10945, 10943, 10941, 983102, 8827, 10938, + 10934, 10928, 10930, 10936, 10932, 8937, 8831, 8829, 8881, 9139, 10763, + 9138, 9737, 127774, 9925, 7098, 7073, 7074, 7075, 7084, 7085, 7043, + 983220, 7046, 7102, 7103, 7062, 7100, 7068, 7099, 7067, 7087, 7070, 7048, + 7049, 7053, 7057, 7060, 7101, 7064, 7086, 7050, 7054, 7059, 7052, 7072, + 7055, 7065, 7061, 7051, 7058, 7063, 7069, 7071, 7066, 7056, 7044, 7047, + 7045, 7367, 7366, 7365, 7364, 7363, 7361, 7362, 7360, 7082, 7041, 7042, + 7040, 7083, 7080, 7081, 7079, 7077, 7076, 7078, 7093, 7092, 7095, 7094, + 7091, 7090, 7088, 7097, 7089, 7096, 127749, 127748, 72648, 72662, 72661, + 72669, 72652, 72640, 72663, 72651, 72655, 72672, 72646, 72667, 72666, + 72653, 72657, 72645, 72665, 72649, 72644, 72658, 72668, 72670, 72664, + 72671, 72641, 72659, 72656, 72650, 72643, 72660, 72654, 72642, 72647, + 72673, 72693, 72692, 72695, 72694, 72691, 72690, 72688, 72697, 72689, + 72696, 127751, 127803, 8316, 8312, 8305, 8319, 8317, 8314, 8318, 8315, + 8309, 8308, 8311, 8310, 179, 178, 8304, 8313, 185, 10966, 10964, 10619, + 10968, 10967, 8835, 10954, 10950, 10952, 10956, 8843, 8839, 10948, 10946, + 10944, 10942, 10185, 129464, 129465, 8751, 127940, 128671, 127843, + 128629, 129442, 127946, 8275, 43027, 43026, 43031, 43030, 43040, 43038, + 43025, 43024, 43029, 43028, 43036, 43035, 43021, 43020, 43018, 43017, + 43023, 43022, 43016, 43015, 43034, 43033, 43042, 43039, 43037, 43032, + 43041, 43008, 43012, 43009, 43013, 43011, 43048, 43049, 43050, 43051, + 43052, 43019, 43014, 43010, 43047, 43043, 43046, 43044, 43045, 9223, + 9224, 9229, 9240, 9232, 9249, 9256, 9255, 9253, 9257, 9236, 9235, 9234, + 9233, 9241, 9220, 9239, 9219, 9221, 9243, 9244, 9228, 9225, 9227, 9226, + 128325, 9237, 9252, 9216, 9222, 9246, 9245, 9247, 8527, 9230, 9231, 9217, + 9218, 9242, 9254, 9238, 9248, 11159, 9007, 983094, 983093, 128333, 1807, + 1866, 1802, 1798, 1799, 1849, 1848, 1792, 1854, 1853, 1805, 1804, 1803, + 1852, 1851, 1850, 1797, 1814, 1813, 1815, 1818, 1824, 2153, 2152, 2149, + 2148, 2144, 2146, 2150, 2147, 2154, 2145, 2151, 1825, 1830, 1837, 1839, + 1838, 1831, 1834, 1827, 1869, 1870, 1871, 1809, 1835, 1832, 1828, 1808, + 1823, 1833, 1819, 1820, 1836, 1811, 1812, 1821, 1822, 1810, 1817, 1826, + 1816, 1829, 1864, 1863, 1842, 1841, 1840, 1845, 1844, 1843, 1847, 1846, + 1855, 1858, 1796, 983204, 1801, 1794, 1795, 1800, 1793, 1862, 1861, 1860, + 1859, 1865, 1856, 1857, 128137, 983184, 128085, 129430, 983061, 127955, + 917543, 917542, 917546, 917598, 917568, 917548, 917562, 917540, 917557, + 917556, 917559, 917558, 917555, 917554, 917552, 917561, 917553, 917560, + 917565, 917537, 917600, 917566, 917549, 917569, 917570, 917571, 917572, + 917573, 917574, 917575, 917576, 917577, 917578, 917579, 917580, 917581, + 917582, 917583, 917584, 917585, 917586, 917587, 917588, 917589, 917590, + 917591, 917592, 917593, 917594, 917601, 917602, 917603, 917604, 917605, + 917606, 917607, 917608, 917609, 917610, 917611, 917612, 917613, 917614, + 917615, 917616, 917617, 917618, 917619, 917620, 917621, 917622, 917623, + 917624, 917625, 917626, 917564, 917627, 917544, 917595, 917599, 917541, + 917547, 917538, 917567, 917629, 917545, 917597, 917596, 917563, 917551, + 917536, 917539, 917630, 917550, 917628, 5888, 5919, 5893, 5896, 5898, + 5895, 5892, 5905, 5891, 5902, 5899, 5897, 5901, 5904, 5894, 5903, 5900, + 5889, 5890, 5909, 5908, 5906, 5907, 5989, 5992, 5994, 5991, 5988, 5987, + 5998, 5995, 5993, 6000, 5990, 5999, 5996, 5984, 5985, 5986, 6002, 6003, + 6499, 6508, 6509, 6507, 6501, 6502, 6512, 6513, 6514, 6515, 6516, 6497, + 6483, 6487, 6486, 6482, 6498, 6505, 6504, 6496, 6480, 6490, 6489, 6503, + 6506, 6492, 6494, 6488, 6491, 6495, 6484, 6493, 6481, 6485, 6500, 6745, + 6746, 6743, 6747, 6742, 6741, 6748, 6749, 6750, 6783, 6789, 6788, 6791, + 6790, 6787, 6786, 6784, 6793, 6785, 6792, 6805, 6804, 6807, 6806, 6803, + 6802, 6800, 6809, 6801, 6808, 6740, 6689, 6690, 6688, 6702, 6726, 6727, + 6728, 6696, 6695, 6713, 6712, 6707, 6706, 6714, 6729, 6720, 6693, 6692, + 6691, 6704, 6699, 6697, 6717, 6715, 6709, 6708, 6716, 6732, 6698, 6719, + 6723, 6739, 6724, 6730, 6721, 6705, 6701, 6722, 6735, 6736, 6733, 6734, + 6694, 6700, 6710, 6738, 6737, 6711, 6703, 6718, 6725, 6731, 6828, 6820, + 6775, 6776, 6777, 6780, 6824, 6825, 6819, 6772, 6744, 6823, 6779, 6778, + 6822, 6826, 6827, 6818, 6752, 6816, 6817, 6821, 6773, 6774, 6829, 6753, + 6755, 6767, 6769, 6754, 6763, 6764, 6771, 6768, 6765, 6756, 6770, 6761, + 6762, 6760, 6759, 6757, 6758, 6766, 43653, 43651, 43649, 43661, 43659, + 43679, 43677, 43671, 43669, 43657, 43665, 43673, 43675, 43667, 43681, + 43655, 43693, 43689, 43683, 43687, 43663, 43691, 43685, 43695, 43652, + 43650, 43648, 43660, 43658, 43678, 43676, 43670, 43668, 43656, 43664, + 43672, 43674, 43666, 43680, 43654, 43692, 43688, 43682, 43686, 43662, + 43690, 43684, 43694, 43696, 43703, 43743, 43739, 43740, 43742, 43741, + 43712, 43713, 43714, 43711, 43707, 43697, 43710, 43709, 43708, 43700, + 43699, 43705, 43706, 43698, 43704, 43701, 43702, 124653, 124640, 124645, + 124658, 124657, 124656, 124660, 124659, 124632, 124610, 124637, 124608, + 124617, 124628, 124634, 124625, 124620, 124615, 124611, 124638, 124609, + 124618, 124629, 124635, 124626, 124621, 124616, 124613, 124623, 124644, + 124642, 124650, 124651, 124627, 124622, 124641, 124649, 124647, 124652, + 124624, 124614, 124619, 124612, 124630, 124636, 124633, 124631, 124648, + 124655, 124646, 124654, 124643, 124661, 124670, 124671, 71353, 71296, + 71352, 71297, 71303, 71305, 71319, 71318, 71324, 71323, 71338, 71332, + 71317, 71316, 71322, 71321, 71300, 71301, 71298, 71299, 71310, 71320, + 71315, 71325, 71329, 71328, 71312, 71311, 71309, 71308, 71314, 71313, + 71307, 71306, 71327, 71326, 71335, 71336, 71337, 71333, 71330, 71334, + 71331, 71302, 71304, 71351, 71339, 71350, 71340, 71341, 71347, 71349, + 71344, 71345, 71342, 71343, 71346, 71348, 71365, 71364, 71367, 71366, + 71363, 71362, 71360, 71369, 71361, 71368, 129377, 119672, 119671, 3064, + 3031, 73707, 983662, 983685, 983674, 983677, 983676, 983669, 983667, + 983679, 983663, 983665, 983668, 983666, 983683, 983681, 983682, 983673, + 983678, 983664, 983684, 983680, 983671, 983670, 983675, 983672, 73706, + 3063, 73701, 3062, 3059, 3051, 3050, 3053, 3052, 3049, 3048, 3046, 3055, + 3047, 3054, 73700, 73666, 73676, 73668, 73679, 73681, 73682, 73665, + 73673, 73674, 73667, 73664, 73669, 73672, 73675, 73680, 73670, 73678, + 73671, 73677, 73683, 73684, 73710, 2985, 2979, 2969, 2974, 2984, 2975, + 2980, 2949, 2950, 2960, 2964, 2996, 2995, 2994, 2993, 2992, 2953, 2954, + 2962, 2963, 2951, 2952, 2998, 2999, 3000, 2958, 2959, 2970, 3001, 2972, + 2965, 2990, 2986, 2997, 2991, 73702, 3057, 3058, 3056, 3066, 73727, 3065, + 73703, 73687, 2946, 73686, 73698, 73690, 73693, 73692, 73712, 73689, + 73688, 73691, 73697, 73694, 73695, 73696, 73713, 73699, 3021, 2947, + 73685, 73708, 73711, 983939, 983940, 983947, 983950, 983943, 983944, + 983948, 983949, 983941, 983942, 983945, 983946, 983686, 983693, 983696, + 983689, 983690, 983694, 983695, 983687, 983688, 983691, 983692, 983840, + 983847, 983850, 983843, 983844, 983848, 983849, 983841, 983842, 983845, + 983846, 983851, 983858, 983861, 983854, 983855, 983859, 983860, 983852, + 983853, 983856, 983857, 983818, 983825, 983828, 983821, 983822, 983826, + 983827, 983819, 983820, 983823, 983824, 983873, 983880, 983883, 983876, + 983877, 983881, 983882, 983874, 983875, 983878, 983879, 983741, 983748, + 983751, 983744, 983745, 983749, 983750, 983742, 983743, 983746, 983747, + 983697, 983704, 983707, 983700, 983701, 983705, 983706, 983698, 983699, + 983702, 983703, 983719, 983726, 983729, 983722, 983723, 983727, 983728, + 983720, 983721, 983724, 983725, 983763, 983770, 983773, 983766, 983767, + 983771, 983772, 983764, 983765, 983768, 983769, 983862, 983869, 983872, + 983865, 983866, 983870, 983871, 983863, 983864, 983867, 983868, 983807, + 983814, 983817, 983810, 983811, 983815, 983816, 983808, 983809, 983812, + 983813, 983895, 983902, 983905, 983898, 983899, 983903, 983904, 983951, + 983896, 983897, 983900, 983901, 983906, 983913, 983916, 983909, 983910, + 983914, 983915, 983907, 983908, 983911, 983912, 983917, 983924, 983927, + 983920, 983921, 983925, 983926, 983918, 983919, 983922, 983923, 983730, + 983737, 983740, 983733, 983734, 983738, 983739, 983731, 983732, 983735, + 983736, 983752, 983759, 983762, 983755, 983756, 983760, 983761, 983753, + 983754, 983757, 983758, 983708, 983715, 983718, 983711, 983712, 983716, + 983717, 983709, 983710, 983713, 983714, 983928, 983935, 983938, 983931, + 983932, 983936, 983937, 983929, 983930, 983933, 983934, 983884, 983891, + 983894, 983887, 983888, 983892, 983893, 983885, 983886, 983889, 983890, + 983785, 983792, 983795, 983788, 983789, 983793, 983794, 983786, 983787, + 983790, 983791, 983774, 983781, 983784, 983777, 983778, 983782, 983783, + 983775, 983776, 983779, 983780, 983829, 983836, 983839, 983832, 983833, + 983837, 983838, 983830, 983831, 983834, 983835, 983796, 983803, 983806, + 983799, 983800, 983804, 983805, 983797, 983798, 983801, 983802, 73709, + 73704, 73705, 3006, 3016, 3020, 3009, 3010, 3018, 3019, 3007, 3008, 3014, + 3015, 3061, 3060, 3024, 129748, 127883, 127818, 92809, 92810, 92811, + 92808, 92789, 92790, 92791, 92788, 92816, 92859, 92856, 92847, 92845, + 92817, 92846, 92843, 92829, 92830, 92831, 92828, 92835, 92851, 92840, + 92844, 92818, 92819, 92852, 92836, 92825, 92826, 92827, 92824, 92813, + 92814, 92815, 92812, 92820, 92822, 92823, 92821, 92805, 92806, 92807, + 92804, 92797, 92798, 92799, 92796, 92801, 92802, 92803, 92800, 92785, + 92786, 92787, 92784, 92793, 92794, 92795, 92792, 92857, 92854, 92848, + 92861, 92853, 92860, 92849, 92855, 92834, 92833, 92832, 92841, 92839, + 92842, 92850, 92838, 92858, 92837, 92862, 92869, 92868, 92871, 92870, + 92867, 92866, 92864, 92873, 92865, 92872, 100352, 100353, 100354, 100355, + 100356, 100357, 100358, 100359, 100360, 100361, 100362, 100363, 100364, + 100365, 100366, 100367, 100368, 100369, 100370, 100371, 100372, 100373, + 100374, 100375, 100376, 100377, 100378, 100379, 100380, 100381, 100382, + 100383, 100384, 100385, 100386, 100387, 100388, 100389, 100390, 100391, + 100392, 100393, 100394, 100395, 100396, 100397, 100398, 100399, 100400, + 100401, 100402, 100403, 100404, 100405, 100406, 100407, 100408, 100409, + 100410, 100411, 100412, 100413, 100414, 100415, 100416, 100417, 100418, + 100419, 100420, 100421, 100422, 100423, 100424, 100425, 100426, 100427, + 100428, 100429, 100430, 100431, 100432, 100433, 100434, 100435, 100436, + 100437, 100438, 100439, 100440, 100441, 100442, 100443, 100444, 100445, + 100446, 100447, 100448, 100449, 100450, 100451, 100452, 100453, 100454, + 100455, 100456, 100457, 100458, 100459, 100460, 100461, 100462, 100463, + 100464, 100465, 100466, 100467, 100468, 100469, 100470, 100471, 100472, + 100473, 100474, 100475, 100476, 100477, 100478, 100479, 100480, 100481, + 100482, 100483, 100484, 100485, 100486, 100487, 100488, 100489, 100490, + 100491, 100492, 100493, 100494, 100495, 100496, 100497, 100498, 100499, + 100500, 100501, 100502, 100503, 100504, 100505, 100506, 100507, 100508, + 100509, 100510, 100511, 100512, 100513, 100514, 100515, 100516, 100517, + 100518, 100519, 100520, 100521, 100522, 100523, 100524, 100525, 100526, + 100527, 100528, 100529, 100530, 100531, 100532, 100533, 100534, 100535, + 100536, 100537, 100538, 100539, 100540, 100541, 100542, 100543, 100544, + 100545, 100546, 100547, 100548, 100549, 100550, 100551, 100552, 100553, + 100554, 100555, 100556, 100557, 100558, 100559, 100560, 100561, 100562, + 100563, 100564, 100565, 100566, 100567, 100568, 100569, 100570, 100571, + 100572, 100573, 100574, 100575, 100576, 100577, 100578, 100579, 100580, + 100581, 100582, 100583, 100584, 100585, 100586, 100587, 100588, 100589, + 100590, 100591, 100592, 100593, 100594, 100595, 100596, 100597, 100598, + 100599, 100600, 100601, 100602, 100603, 100604, 100605, 100606, 100607, + 100608, 100609, 100610, 100611, 100612, 100613, 100614, 100615, 100616, + 100617, 100618, 100619, 100620, 100621, 100622, 100623, 100624, 100625, + 100626, 100627, 100628, 100629, 100630, 100631, 100632, 100633, 100634, + 100635, 100636, 100637, 100638, 100639, 100640, 100641, 100642, 100643, + 100644, 100645, 100646, 100647, 100648, 100649, 100650, 100651, 100652, + 100653, 100654, 100655, 100656, 100657, 100658, 100659, 100660, 100661, + 100662, 100663, 100664, 100665, 100666, 100667, 100668, 100669, 100670, + 100671, 100672, 100673, 100674, 100675, 100676, 100677, 100678, 100679, + 100680, 100681, 100682, 100683, 100684, 100685, 100686, 100687, 100688, + 100689, 100690, 100691, 100692, 100693, 100694, 100695, 100696, 100697, + 100698, 100699, 100700, 100701, 100702, 100703, 100704, 100705, 100706, + 100707, 100708, 100709, 100710, 100711, 100712, 100713, 100714, 100715, + 100716, 100717, 100718, 100719, 100720, 100721, 100722, 100723, 100724, + 100725, 100726, 100727, 100728, 100729, 100730, 100731, 100732, 100733, + 100734, 100735, 100736, 100737, 100738, 100739, 100740, 100741, 100742, + 100743, 100744, 100745, 100746, 100747, 100748, 100749, 100750, 100751, + 100752, 100753, 100754, 100755, 100756, 100757, 100758, 100759, 100760, + 100761, 100762, 100763, 100764, 100765, 100766, 100767, 100768, 100769, + 100770, 100771, 100772, 100773, 100774, 100775, 100776, 100777, 100778, + 100779, 100780, 100781, 100782, 100783, 100784, 100785, 100786, 100787, + 100788, 100789, 100790, 100791, 100792, 100793, 100794, 100795, 100796, + 100797, 100798, 100799, 100800, 100801, 100802, 100803, 100804, 100805, + 100806, 100807, 100808, 100809, 100810, 100811, 100812, 100813, 100814, + 100815, 100816, 100817, 100818, 100819, 100820, 100821, 100822, 100823, + 100824, 100825, 100826, 100827, 100828, 100829, 100830, 100831, 100832, + 100833, 100834, 100835, 100836, 100837, 100838, 100839, 100840, 100841, + 100842, 100843, 100844, 100845, 100846, 100847, 100848, 100849, 100850, + 100851, 100852, 100853, 100854, 100855, 100856, 100857, 100858, 100859, + 100860, 100861, 100862, 100863, 100864, 100865, 100866, 100867, 100868, + 100869, 100870, 100871, 100872, 100873, 100874, 100875, 100876, 100877, + 100878, 100879, 100880, 100881, 100882, 100883, 100884, 100885, 100886, + 100887, 100888, 100889, 100890, 100891, 100892, 100893, 100894, 100895, + 100896, 100897, 100898, 100899, 100900, 100901, 100902, 100903, 100904, + 100905, 100906, 100907, 100908, 100909, 100910, 100911, 100912, 100913, + 100914, 100915, 100916, 100917, 100918, 100919, 100920, 100921, 100922, + 100923, 100924, 100925, 100926, 100927, 100928, 100929, 100930, 100931, + 100932, 100933, 100934, 100935, 100936, 100937, 100938, 100939, 100940, + 100941, 100942, 100943, 100944, 100945, 100946, 100947, 100948, 100949, + 100950, 100951, 100952, 100953, 100954, 100955, 100956, 100957, 100958, + 100959, 100960, 100961, 100962, 100963, 100964, 100965, 100966, 100967, + 100968, 100969, 100970, 100971, 100972, 100973, 100974, 100975, 100976, + 100977, 100978, 100979, 100980, 100981, 100982, 100983, 100984, 100985, + 100986, 100987, 100988, 100989, 100990, 100991, 100992, 100993, 100994, + 100995, 100996, 100997, 100998, 100999, 101000, 101001, 101002, 101003, + 101004, 101005, 101006, 101007, 101008, 101009, 101010, 101011, 101012, + 101013, 101014, 101015, 101016, 101017, 101018, 101019, 101020, 101021, + 101022, 101023, 101024, 101025, 101026, 101027, 101028, 101029, 101030, + 101031, 101032, 101033, 101034, 101035, 101036, 101037, 101038, 101039, + 101040, 101041, 101042, 101043, 101044, 101045, 101046, 101047, 101048, + 101049, 101050, 101051, 101052, 101053, 101054, 101055, 101056, 101057, + 101058, 101059, 101060, 101061, 101062, 101063, 101064, 101065, 101066, + 101067, 101068, 101069, 101070, 101071, 101072, 101073, 101074, 101075, + 101076, 101077, 101078, 101079, 101080, 101081, 101082, 101083, 101084, + 101085, 101086, 101087, 101088, 101089, 101090, 101091, 101092, 101093, + 101094, 101095, 101096, 101097, 101098, 101099, 101100, 101101, 101102, + 101103, 101104, 101105, 101106, 101107, 101108, 101109, 101110, 101111, + 101112, 101113, 101114, 101115, 101116, 101117, 101118, 101119, 101760, + 101761, 101762, 101763, 101764, 101765, 101766, 101767, 101768, 101769, + 101770, 101771, 101772, 101773, 101774, 101775, 101776, 101777, 101778, + 101779, 101780, 101781, 101782, 101783, 101784, 101785, 101786, 101787, + 101788, 101789, 101790, 101791, 101792, 101793, 101794, 101795, 101796, + 101797, 101798, 101799, 101800, 101801, 101802, 101803, 101804, 101805, + 101806, 101807, 101808, 101809, 101810, 101811, 101812, 101813, 101814, + 101815, 101816, 101817, 101818, 101819, 101820, 101821, 101822, 101823, + 101824, 101825, 101826, 101827, 101828, 101829, 101830, 101831, 101832, + 101833, 101834, 101835, 101836, 101837, 101838, 101839, 101840, 101841, + 101842, 101843, 101844, 101845, 101846, 101847, 101848, 101849, 101850, + 101851, 101852, 101853, 101854, 101855, 101856, 101857, 101858, 101859, + 101860, 101861, 101862, 101863, 101864, 101865, 101866, 101867, 101868, + 101869, 101870, 101871, 101872, 101873, 101874, 94176, 128429, 9991, + 9801, 127790, 128661, 127861, 128198, 10043, 10170, 129750, 129528, + 128222, 128380, 8981, 9990, 8481, 128384, 128301, 128250, 3164, 3158, + 3195, 3198, 3194, 3197, 3193, 3196, 3192, 3106, 3105, 3111, 3161, 3110, + 3112, 3165, 3097, 3107, 3102, 3162, 3121, 3120, 3104, 3103, 3109, 3160, + 3108, 3077, 3078, 3088, 3092, 3124, 3123, 3122, 3083, 3168, 3084, 3169, + 3125, 3081, 3082, 3090, 3091, 3079, 3080, 3126, 3127, 3128, 3117, 3116, + 3099, 3098, 3096, 3095, 3101, 3100, 3094, 3093, 3115, 3114, 3086, 3087, + 3129, 3118, 3119, 3157, 3076, 3072, 3073, 3191, 3199, 3132, 3133, 3074, + 3149, 3075, 3134, 3144, 3148, 3137, 3138, 3139, 3140, 3170, 3171, 3146, + 3147, 3135, 3136, 3142, 3143, 3179, 3178, 3181, 3180, 3177, 3176, 3174, + 3183, 3175, 3182, 127934, 8376, 9978, 129514, 119617, 119564, 119577, + 119633, 119587, 119566, 119561, 119585, 119613, 119590, 119631, 119634, + 119630, 119608, 119582, 119563, 119573, 119558, 119624, 119567, 119623, + 119586, 119636, 119612, 119625, 119568, 119584, 119619, 119618, 119583, + 119603, 119600, 119626, 119610, 119580, 119576, 119638, 119559, 119595, + 119632, 119606, 119592, 119615, 119599, 119602, 119614, 119629, 119574, + 119569, 119570, 119622, 119562, 119591, 119637, 119597, 119589, 119616, + 119609, 119560, 119635, 119565, 119604, 119588, 119571, 119594, 119578, + 119596, 119579, 119598, 119572, 119605, 119627, 119621, 119628, 119601, + 119593, 119607, 119575, 119620, 119611, 119581, 1959, 1958, 1964, 1961, + 1965, 1927, 1954, 1951, 1937, 1931, 1930, 1956, 1934, 1935, 1955, 1945, + 1920, 1926, 1946, 1933, 1925, 1929, 1943, 1942, 1941, 1922, 1969, 1950, + 1949, 1921, 1936, 1939, 1932, 1947, 1944, 1952, 1928, 1957, 1938, 1948, + 1953, 1924, 1923, 1940, 1967, 1966, 1963, 1960, 1962, 1968, 3610, 3592, + 3594, 3593, 3596, 3598, 3604, 3613, 3615, 3663, 3630, 3627, 3588, 3587, + 3589, 3586, 3590, 3675, 3585, 3628, 3621, 3653, 3622, 3659, 3633, 3657, + 3658, 3656, 3655, 3654, 3617, 3674, 3591, 3661, 3603, 3609, 3631, 3642, + 3612, 3614, 3616, 3611, 3619, 3620, 3632, 3652, 3651, 3634, 3649, 3635, + 3640, 3638, 3639, 3641, 3636, 3637, 3648, 3650, 3625, 3624, 3626, 3595, + 3660, 3601, 3602, 3607, 3600, 3608, 3606, 3605, 3599, 3623, 3662, 3597, + 3618, 3629, 3647, 3669, 3668, 3671, 3670, 3667, 3666, 3664, 3673, 3665, + 3672, 8708, 8707, 8756, 127777, 10727, 118474, 128936, 8201, 128929, 129300, 129353, 128173, 129652, 10176, 8278, 9887, 9886, 11057, 128485, 128484, 128486, 128487, 8694, 128433, 10870, 128491, 128962, 128423, 11160, 11162, 10146, 11163, 11161, 10147, 8196, 11835, 128077, 128078, - 9928, 9736, 3865, 3863, 3864, 4032, 4033, 4035, 4034, 3886, 3885, 3888, + 9928, 9736, 3865, 3863, 3864, 4035, 4032, 4033, 4034, 3886, 3885, 3888, 3887, 3884, 3883, 3891, 3890, 3882, 3889, 3877, 3876, 3879, 3878, 3875, 3874, 3872, 3881, 3873, 3880, 4030, 4031, 3945, 3905, 3947, 3904, 3948, 3938, 3946, 3917, 3916, 3932, 3931, 3922, 3921, 3908, 3918, 3913, 3923, @@ -12898,9 +13046,9 @@ static const unsigned int dawg_pos_to_codepoint[] = { 4040, 4037, 4039, 3968, 3969, 3956, 3957, 3958, 3959, 3960, 3961, 3964, 3965, 3954, 3955, 3962, 3963, 3953, 10717, 11647, 11608, 11595, 11585, 11573, 11620, 11607, 11600, 11582, 11590, 11596, 11601, 11586, 11592, - 11568, 11575, 11577, 11578, 11576, 11571, 11606, 11572, 11581, 11589, - 11583, 11569, 11570, 11584, 11587, 11609, 11611, 11610, 11612, 11613, - 11615, 11619, 11594, 11621, 11604, 11605, 11614, 11588, 11580, 11574, + 11568, 11571, 11606, 11572, 11581, 11589, 11583, 11609, 11611, 11610, + 11612, 11613, 11615, 11619, 11594, 11621, 11575, 11577, 11578, 11576, + 11569, 11570, 11584, 11587, 11604, 11605, 11614, 11588, 11580, 11574, 11597, 11598, 11599, 11602, 11591, 11616, 11617, 11618, 11622, 11579, 11593, 11623, 11603, 11631, 11632, 128005, 128047, 10053, 126, 8764, 11081, 10610, 10859, 10858, 11807, 11806, 11803, 9202, 10708, 10709, @@ -12918,155 +13066,161 @@ static const unsigned int dawg_pos_to_codepoint[] = { 67039, 67038, 67037, 67034, 67059, 67018, 67017, 67030, 67029, 67008, 67009, 67013, 67012, 67016, 67014, 67057, 67028, 67043, 67042, 67048, 67046, 67053, 67052, 67010, 67019, 67036, 67045, 67026, 67044, 67050, - 127813, 128069, 129463, 129701, 129520, 10554, 10557, 10556, 9182, - 118256, 117851, 118262, 118263, 11865, 117850, 118252, 118254, 11866, - 117852, 118248, 118250, 11833, 118246, 11210, 8992, 127913, 9180, 130024, - 130016, 130031, 8988, 8975, 11810, 118279, 8989, 8974, 11811, 130028, - 9140, 9184, 128285, 127553, 127554, 127559, 127555, 127557, 127560, - 127552, 127556, 127558, 127274, 9008, 123554, 123556, 123559, 123561, - 123564, 123537, 123544, 123543, 123553, 123555, 123558, 123560, 123546, - 123565, 123563, 123539, 123541, 123550, 123549, 123540, 123552, 123542, - 123536, 123551, 123545, 123538, 123548, 123547, 123562, 123557, 123566, - 128701, 128508, 128434, 128668, 8482, 128650, 128651, 11223, 10971, - 128646, 10701, 10699, 128710, 10698, 10141, 128681, 128208, 8227, 128305, - 9783, 9776, 9777, 9782, 9779, 9781, 9780, 9778, 10998, 10856, 10857, - 10747, 8244, 8779, 10996, 10624, 8874, 10997, 11003, 11851, 11000, 10999, - 8749, 8285, 129484, 128654, 127865, 128032, 127942, 8872, 11231, 127930, - 8366, 70601, 70608, 70613, 70612, 70610, 70528, 70529, 70542, 70545, - 70559, 70558, 70564, 70563, 70581, 70579, 70573, 70580, 70572, 70557, - 70556, 70562, 70561, 70534, 70535, 70536, 70537, 70574, 70532, 70533, - 70530, 70531, 70550, 70560, 70555, 70565, 70575, 70576, 70577, 70569, - 70568, 70552, 70551, 70549, 70548, 70554, 70553, 70547, 70546, 70567, - 70566, 70544, 70539, 70578, 70570, 70571, 70609, 70583, 70604, 70607, - 70615, 70616, 70602, 70611, 70606, 70605, 70626, 70625, 70584, 70597, - 70600, 70587, 70588, 70589, 70590, 70591, 70592, 70585, 70586, 70599, - 70594, 127799, 8378, 129411, 8523, 10658, 11202, 9930, 9929, 8498, 11826, - 11832, 8587, 8586, 128598, 128596, 8985, 128399, 8513, 8514, 8516, 11829, - 8526, 128599, 128597, 8489, 128034, 129347, 10041, 128256, 128432, 10869, - 8229, 8282, 11818, 128149, 10837, 10838, 10697, 10760, 10759, 128108, - 128109, 117896, 8273, 128490, 11834, 66432, 66451, 66454, 66433, 66436, - 66447, 66457, 66434, 66437, 66440, 66443, 66435, 66445, 66455, 66453, - 66450, 66444, 66461, 66456, 66441, 66458, 66442, 66439, 66449, 66448, - 66452, 66438, 66446, 66459, 66460, 66463, 9730, 9969, 9748, 128530, - 11217, 8255, 9100, 8746, 10824, 10822, 10826, 10817, 10818, 10821, - 983116, 11258, 9842, 129412, 9903, 8963, 8996, 11193, 10685, 10577, - 10573, 10572, 10575, 8597, 8616, 11021, 8661, 129113, 8691, 11109, 10622, - 8944, 8869, 10207, 117873, 117877, 117857, 128743, 117881, 118267, - 128742, 117847, 128314, 117912, 117862, 128744, 128316, 9709, 9710, - 117760, 129898, 129946, 129920, 129896, 9985, 118417, 117809, 118418, - 117810, 130020, 129924, 9600, 129937, 118275, 129938, 9690, 118292, - 129934, 118439, 118444, 9696, 129885, 129887, 129889, 129886, 129883, - 129888, 129881, 129884, 129882, 129879, 129880, 10196, 118416, 9136, - 129944, 129948, 9720, 117808, 9692, 117958, 117946, 117962, 117966, - 117950, 117954, 117928, 117926, 117942, 117970, 117938, 117813, 118421, - 118422, 118420, 117812, 118423, 117815, 129922, 9620, 129874, 129875, - 129892, 129894, 129890, 129878, 129893, 129891, 129876, 129895, 129877, - 10064, 118419, 9137, 10000, 9693, 117959, 117947, 117963, 117967, 117951, - 117955, 117929, 117927, 117943, 117971, 117939, 117814, 128319, 10066, - 129945, 129949, 9721, 117811, 129926, 129923, 118438, 118445, 129925, - 129901, 128579, 11797, 8593, 129976, 8645, 8670, 129033, 129029, 129177, - 129041, 129025, 129045, 8624, 8625, 10505, 8613, 10514, 11145, 11014, - 8657, 8673, 129077, 10606, 10595, 10592, 10584, 8639, 10588, 10580, 8638, - 129089, 129093, 129085, 10224, 128621, 129105, 129081, 11105, 11137, - 11121, 129065, 11170, 11171, 129061, 129057, 129073, 129069, 11131, - 11115, 11141, 129169, 10506, 8607, 11245, 10569, 8648, 8679, 8683, 8685, - 8684, 129173, 8682, 11192, 8686, 8687, 9797, 983117, 42509, 42510, 42511, - 42446, 42370, 42486, 42257, 42333, 42294, 42407, 42445, 42369, 42485, - 42256, 42332, 42293, 42406, 42449, 42373, 42489, 42260, 42336, 42297, - 42410, 42434, 42359, 42473, 42246, 42321, 42283, 42396, 42435, 42360, - 42474, 42247, 42322, 42284, 42397, 42452, 42376, 42492, 42263, 42339, - 42300, 42413, 42451, 42375, 42491, 42262, 42338, 42299, 42412, 42444, - 42368, 42484, 42255, 42331, 42292, 42405, 42443, 42367, 42483, 42254, - 42330, 42291, 42404, 42454, 42378, 42494, 42265, 42341, 42302, 42415, - 42453, 42377, 42493, 42264, 42340, 42301, 42414, 42439, 42440, 42364, - 42479, 42251, 42480, 42327, 42288, 42401, 42502, 42272, 42503, 42461, - 42385, 42349, 42309, 42422, 42429, 42430, 42355, 42468, 42242, 42469, - 42316, 42317, 42278, 42279, 42391, 42392, 42476, 42249, 42477, 42437, - 42362, 42324, 42325, 42286, 42399, 42459, 42383, 42346, 42347, 42499, - 42270, 42307, 42420, 42487, 42508, 42258, 42447, 42371, 42334, 42295, - 42408, 42436, 42361, 42475, 42248, 42323, 42285, 42398, 42438, 42363, - 42478, 42250, 42326, 42287, 42400, 42462, 42386, 42504, 42273, 42350, - 42310, 42423, 42450, 42514, 42539, 42512, 42513, 42538, 42374, 42490, - 42261, 42337, 42298, 42411, 42507, 42500, 42271, 42501, 42460, 42384, - 42348, 42308, 42421, 42315, 42467, 42428, 42457, 42381, 42497, 42268, - 42344, 42305, 42418, 42464, 42388, 42506, 42275, 42352, 42312, 42425, - 42463, 42387, 42505, 42274, 42351, 42311, 42424, 42455, 42379, 42495, - 42266, 42342, 42303, 42416, 42441, 42365, 42481, 42252, 42328, 42289, - 42402, 42456, 42380, 42496, 42267, 42343, 42304, 42417, 42433, 42358, - 42472, 42245, 42320, 42282, 42395, 42448, 42372, 42488, 42259, 42335, - 42296, 42409, 42442, 42366, 42482, 42253, 42329, 42290, 42403, 42458, - 42382, 42498, 42269, 42345, 42306, 42419, 42470, 42243, 42244, 42471, - 42431, 42356, 42357, 42432, 42318, 42319, 42280, 42281, 42393, 42394, - 42465, 42240, 42241, 42466, 42426, 42353, 42354, 42427, 42313, 42314, - 42276, 42277, 42389, 42390, 42523, 42526, 42522, 42515, 42520, 42527, - 42516, 42524, 42518, 42517, 42525, 42521, 42519, 42533, 42532, 42535, - 42534, 42531, 42530, 42528, 42537, 42529, 42536, 65024, 65033, 917843, - 917844, 917845, 917846, 917847, 917848, 917849, 917850, 917851, 917852, - 65034, 917853, 917854, 917855, 917856, 917857, 917858, 917859, 917860, - 917861, 917862, 65035, 917863, 917864, 917865, 917866, 917867, 917868, - 917869, 917870, 917871, 917872, 65036, 917873, 917874, 917875, 917876, - 917877, 917878, 917879, 917880, 917881, 917882, 65037, 917883, 917884, - 917885, 917886, 917887, 917888, 917889, 917890, 917891, 917892, 65038, - 917893, 917894, 917895, 917896, 917897, 917898, 917899, 917900, 917901, - 917902, 65039, 917903, 917904, 917905, 917906, 917907, 917908, 917909, - 917910, 917911, 917912, 917760, 917913, 917914, 917915, 917916, 917917, - 917918, 917919, 917920, 917921, 917922, 917761, 917923, 917924, 917925, - 917926, 917927, 917928, 917929, 917930, 917931, 917932, 917762, 917933, - 917934, 917935, 917936, 917937, 917938, 917939, 917940, 917941, 917942, - 65025, 917763, 917943, 917944, 917945, 917946, 917947, 917948, 917949, - 917950, 917951, 917952, 917764, 917953, 917954, 917955, 917956, 917957, - 917958, 917959, 917960, 917961, 917962, 917765, 917963, 917964, 917965, - 917966, 917967, 917968, 917969, 917970, 917971, 917972, 917766, 917973, - 917974, 917975, 917976, 917977, 917978, 917979, 917980, 917981, 917982, - 917767, 917983, 917984, 917985, 917986, 917987, 917988, 917989, 917990, - 917991, 917992, 917768, 917993, 917994, 917995, 917996, 917997, 917998, - 917999, 917769, 917770, 917771, 917772, 65026, 917773, 917774, 917775, - 917776, 917777, 917778, 917779, 917780, 917781, 917782, 65027, 917783, - 917784, 917785, 917786, 917787, 917788, 917789, 917790, 917791, 917792, - 65028, 917793, 917794, 917795, 917796, 917797, 917798, 917799, 917800, - 917801, 917802, 65029, 917803, 917804, 917805, 917806, 917807, 917808, - 917809, 917810, 917811, 917812, 65030, 917813, 917814, 917815, 917816, - 917817, 917818, 917819, 917820, 917821, 917822, 65031, 917823, 917824, - 917825, 917826, 917827, 917828, 917829, 917830, 917831, 917832, 65032, - 917833, 917834, 917835, 917836, 917837, 917838, 917839, 917840, 917841, - 917842, 129499, 983245, 983254, 983360, 983361, 983362, 983363, 983364, - 983365, 983366, 983367, 983368, 983369, 983255, 983370, 983371, 983372, - 983373, 983374, 983375, 983376, 983377, 983378, 983379, 983256, 983380, - 983381, 983382, 983383, 983384, 983385, 983386, 983387, 983388, 983389, - 983257, 983390, 983391, 983392, 983393, 983394, 983395, 983396, 983397, - 983398, 983399, 983258, 983400, 983401, 983402, 983403, 983404, 983405, - 983406, 983407, 983408, 983409, 983259, 983410, 983411, 983412, 983413, - 983414, 983415, 983416, 983417, 983418, 983419, 983260, 983420, 983421, - 983422, 983423, 983424, 983425, 983426, 983427, 983428, 983429, 983277, - 983430, 983431, 983432, 983433, 983434, 983435, 983436, 983437, 983438, - 983439, 983278, 983440, 983441, 983442, 983443, 983444, 983445, 983446, - 983447, 983448, 983449, 983279, 983450, 983451, 983452, 983453, 983454, - 983455, 983456, 983457, 983458, 983459, 983246, 983280, 983460, 983461, - 983462, 983463, 983464, 983465, 983466, 983467, 983468, 983469, 983281, - 983470, 983471, 983472, 983473, 983474, 983475, 983476, 983477, 983478, - 983479, 983282, 983480, 983481, 983482, 983483, 983484, 983485, 983486, - 983487, 983488, 983489, 983283, 983490, 983491, 983492, 983493, 983494, - 983495, 983496, 983497, 983498, 983499, 983284, 983500, 983501, 983502, - 983503, 983504, 983505, 983506, 983507, 983508, 983509, 983285, 983510, - 983511, 983512, 983513, 983514, 983515, 983516, 983286, 983287, 983288, - 983289, 983247, 983290, 983291, 983292, 983293, 983294, 983295, 983296, - 983297, 983298, 983299, 983248, 983300, 983301, 983302, 983303, 983304, - 983305, 983306, 983307, 983308, 983309, 983249, 983310, 983311, 983312, - 983313, 983314, 983315, 983316, 983317, 983318, 983319, 983250, 983320, - 983321, 983322, 983323, 983324, 983325, 983326, 983327, 983328, 983329, - 983251, 983330, 983331, 983332, 983333, 983334, 983335, 983336, 983337, - 983338, 983339, 983252, 983340, 983341, 983342, 983343, 983344, 983345, - 983346, 983347, 983348, 983349, 983253, 983350, 983351, 983352, 983353, - 983354, 983355, 983356, 983357, 983358, 983359, 7401, 7402, 7409, 7403, - 7404, 7415, 7410, 7418, 7413, 7379, 7398, 7396, 7411, 7408, 7406, 7407, - 7405, 7414, 7397, 7400, 7395, 7399, 7394, 7380, 7384, 7412, 7417, 7386, - 7389, 7376, 7388, 7378, 7416, 7392, 7391, 7387, 7390, 7381, 7382, 7383, - 7385, 7393, 7377, 8483, 10704, 10980, 10978, 10186, 117780, 8942, 8286, - 117917, 12347, 12337, 12339, 12341, 12338, 12340, 117892, 124, 9168, - 10992, 10991, 117904, 118293, 9087, 9896, 129904, 129905, 129906, 129907, - 129908, 129909, 117916, 117770, 11135, 983069, 983144, 11823, 128678, - 10650, 11837, 128976, 128959, 128947, 128637, 128941, 128953, 128636, - 128904, 128914, 128934, 8921, 8920, 128933, 10799, 9910, 128243, 9996, + 73140, 73141, 73149, 73154, 73155, 73150, 73151, 73156, 73171, 73166, + 73161, 73168, 73175, 73176, 73147, 73152, 73153, 73148, 73144, 73145, + 73157, 73158, 73164, 73165, 73159, 73160, 73162, 73163, 73142, 73143, + 73137, 73173, 73136, 73169, 73146, 73139, 73172, 73138, 73170, 73174, + 73167, 73177, 73178, 73179, 73189, 73188, 73191, 73190, 73187, 73186, + 73184, 73193, 73185, 73192, 127813, 128069, 129463, 129701, 129520, + 10554, 10557, 10556, 9182, 118256, 117851, 118262, 118263, 11865, 117850, + 118252, 118254, 11866, 117852, 118248, 118250, 11833, 118246, 11210, + 8992, 127913, 9180, 130024, 130016, 130031, 8988, 8975, 11810, 118279, + 8989, 8974, 11811, 130028, 9140, 9184, 128285, 127553, 127554, 127559, + 127555, 127557, 127560, 127552, 127556, 127558, 127274, 9008, 123554, + 123556, 123559, 123561, 123564, 123537, 123544, 123543, 123553, 123555, + 123558, 123560, 123546, 123565, 123563, 123539, 123541, 123550, 123549, + 123540, 123552, 123542, 123536, 123551, 123545, 123538, 123548, 123547, + 123562, 123557, 123566, 128701, 128508, 128434, 128668, 8482, 128650, + 128651, 11223, 10971, 128646, 129678, 118460, 10701, 10699, 128710, + 10698, 10141, 128681, 128208, 8227, 128305, 9783, 9776, 9777, 9782, 9779, + 9781, 9780, 9778, 10998, 10856, 10857, 10747, 8244, 8779, 10996, 10624, + 8874, 10997, 11003, 11851, 11000, 10999, 8749, 8285, 129484, 128654, + 129674, 127865, 128032, 127942, 8872, 11231, 127930, 8366, 70601, 70608, + 70613, 70612, 70610, 70528, 70529, 70542, 70545, 70559, 70558, 70564, + 70563, 70581, 70579, 70573, 70580, 70572, 70557, 70556, 70562, 70561, + 70534, 70535, 70536, 70537, 70574, 70532, 70533, 70530, 70531, 70550, + 70560, 70555, 70565, 70575, 70576, 70577, 70569, 70568, 70552, 70551, + 70549, 70548, 70554, 70553, 70547, 70546, 70567, 70566, 70544, 70539, + 70578, 70570, 70571, 70609, 70583, 70604, 70607, 70615, 70616, 70602, + 70611, 70606, 70605, 70626, 70625, 70584, 70597, 70600, 70587, 70588, + 70589, 70590, 70591, 70592, 70585, 70586, 70599, 70594, 127799, 8378, + 129411, 8523, 10658, 11202, 9930, 9929, 8498, 11826, 11832, 8587, 8586, + 128598, 128596, 8985, 128399, 8513, 8514, 8516, 11829, 8526, 128599, + 128597, 8489, 128034, 129347, 10041, 128256, 128432, 10869, 8229, 8282, + 11818, 128149, 10837, 10838, 10697, 10760, 10759, 128108, 128109, 117896, + 8273, 128490, 11834, 66432, 66451, 66454, 66433, 66436, 66447, 66457, + 66434, 66437, 66440, 66443, 66435, 66445, 66455, 66453, 66450, 66444, + 66461, 66456, 66441, 66458, 66442, 66439, 66449, 66448, 66452, 66438, + 66446, 66459, 66460, 66463, 9730, 9969, 9748, 128530, 11217, 8255, 9100, + 8746, 10824, 10822, 10826, 10817, 10818, 10821, 983116, 11258, 9842, + 129412, 9903, 8963, 8996, 11193, 10685, 10577, 10573, 10572, 10575, 8597, + 8616, 11021, 8661, 129113, 8691, 11109, 10622, 8944, 8869, 10207, 117873, + 117877, 117857, 128743, 117881, 118267, 128742, 117847, 128314, 117912, + 117862, 128744, 128316, 9709, 9710, 117760, 129898, 129946, 129920, + 129896, 9985, 118417, 117809, 118418, 117810, 130020, 129924, 9600, + 129937, 118275, 129938, 9690, 118292, 129934, 118439, 118444, 9696, + 129885, 129887, 129889, 129886, 129883, 129888, 129881, 129884, 129882, + 129879, 129880, 10196, 118416, 9136, 129944, 129948, 9720, 117808, 9692, + 117958, 117946, 117962, 117966, 117950, 117954, 117928, 117926, 117942, + 117970, 117938, 117813, 118421, 118422, 118420, 117812, 118423, 117815, + 129922, 9620, 129874, 129875, 129892, 129894, 129890, 129878, 129893, + 129891, 129876, 129895, 129877, 10064, 118419, 9137, 10000, 9693, 117959, + 117947, 117963, 117967, 117951, 117955, 117929, 117927, 117943, 117971, + 117939, 117814, 128319, 10066, 129945, 129949, 9721, 117811, 129926, + 129923, 118438, 118445, 129925, 129901, 128579, 11797, 8593, 129976, + 8645, 8670, 129033, 129029, 129177, 129041, 129025, 129045, 8624, 8625, + 10505, 8613, 10514, 11145, 11014, 8657, 8673, 129077, 10606, 10595, + 10592, 10584, 8639, 10588, 10580, 8638, 129089, 129093, 129085, 10224, + 128621, 129105, 129081, 11105, 11137, 11121, 129065, 11170, 11171, + 129061, 129057, 129073, 129069, 11131, 11115, 11141, 129169, 10506, 8607, + 11245, 10569, 8648, 8679, 8683, 8685, 8684, 129173, 8682, 11192, 8686, + 8687, 9797, 983117, 42509, 42510, 42511, 42446, 42370, 42486, 42257, + 42333, 42294, 42407, 42445, 42369, 42485, 42256, 42332, 42293, 42406, + 42449, 42373, 42489, 42260, 42336, 42297, 42410, 42434, 42359, 42473, + 42246, 42321, 42283, 42396, 42435, 42360, 42474, 42247, 42322, 42284, + 42397, 42452, 42376, 42492, 42263, 42339, 42300, 42413, 42451, 42375, + 42491, 42262, 42338, 42299, 42412, 42444, 42368, 42484, 42255, 42331, + 42292, 42405, 42443, 42367, 42483, 42254, 42330, 42291, 42404, 42454, + 42378, 42494, 42265, 42341, 42302, 42415, 42453, 42377, 42493, 42264, + 42340, 42301, 42414, 42439, 42440, 42364, 42479, 42251, 42480, 42327, + 42288, 42401, 42502, 42272, 42503, 42461, 42385, 42349, 42309, 42422, + 42429, 42430, 42355, 42468, 42242, 42469, 42316, 42317, 42278, 42279, + 42391, 42392, 42476, 42249, 42477, 42437, 42362, 42324, 42325, 42286, + 42399, 42459, 42383, 42346, 42347, 42499, 42270, 42307, 42420, 42487, + 42508, 42258, 42447, 42371, 42334, 42295, 42408, 42436, 42361, 42475, + 42248, 42323, 42285, 42398, 42438, 42363, 42478, 42250, 42326, 42287, + 42400, 42462, 42386, 42504, 42273, 42350, 42310, 42423, 42450, 42514, + 42539, 42512, 42513, 42538, 42374, 42490, 42261, 42337, 42298, 42411, + 42507, 42500, 42271, 42501, 42460, 42384, 42348, 42308, 42421, 42315, + 42467, 42428, 42457, 42381, 42497, 42268, 42344, 42305, 42418, 42464, + 42388, 42506, 42275, 42352, 42312, 42425, 42463, 42387, 42505, 42274, + 42351, 42311, 42424, 42455, 42379, 42495, 42266, 42342, 42303, 42416, + 42441, 42365, 42481, 42252, 42328, 42289, 42402, 42456, 42380, 42496, + 42267, 42343, 42304, 42417, 42433, 42358, 42472, 42245, 42320, 42282, + 42395, 42448, 42372, 42488, 42259, 42335, 42296, 42409, 42442, 42366, + 42482, 42253, 42329, 42290, 42403, 42458, 42382, 42498, 42269, 42345, + 42306, 42419, 42470, 42243, 42244, 42471, 42431, 42356, 42357, 42432, + 42318, 42319, 42280, 42281, 42393, 42394, 42465, 42240, 42241, 42466, + 42426, 42353, 42354, 42427, 42313, 42314, 42276, 42277, 42389, 42390, + 42523, 42526, 42522, 42515, 42520, 42527, 42516, 42524, 42518, 42517, + 42525, 42521, 42519, 42533, 42532, 42535, 42534, 42531, 42530, 42528, + 42537, 42529, 42536, 65024, 65033, 917843, 917844, 917845, 917846, + 917847, 917848, 917849, 917850, 917851, 917852, 65034, 917853, 917854, + 917855, 917856, 917857, 917858, 917859, 917860, 917861, 917862, 65035, + 917863, 917864, 917865, 917866, 917867, 917868, 917869, 917870, 917871, + 917872, 65036, 917873, 917874, 917875, 917876, 917877, 917878, 917879, + 917880, 917881, 917882, 65037, 917883, 917884, 917885, 917886, 917887, + 917888, 917889, 917890, 917891, 917892, 65038, 917893, 917894, 917895, + 917896, 917897, 917898, 917899, 917900, 917901, 917902, 65039, 917903, + 917904, 917905, 917906, 917907, 917908, 917909, 917910, 917911, 917912, + 917760, 917913, 917914, 917915, 917916, 917917, 917918, 917919, 917920, + 917921, 917922, 917761, 917923, 917924, 917925, 917926, 917927, 917928, + 917929, 917930, 917931, 917932, 917762, 917933, 917934, 917935, 917936, + 917937, 917938, 917939, 917940, 917941, 917942, 65025, 917763, 917943, + 917944, 917945, 917946, 917947, 917948, 917949, 917950, 917951, 917952, + 917764, 917953, 917954, 917955, 917956, 917957, 917958, 917959, 917960, + 917961, 917962, 917765, 917963, 917964, 917965, 917966, 917967, 917968, + 917969, 917970, 917971, 917972, 917766, 917973, 917974, 917975, 917976, + 917977, 917978, 917979, 917980, 917981, 917982, 917767, 917983, 917984, + 917985, 917986, 917987, 917988, 917989, 917990, 917991, 917992, 917768, + 917993, 917994, 917995, 917996, 917997, 917998, 917999, 917769, 917770, + 917771, 917772, 65026, 917773, 917774, 917775, 917776, 917777, 917778, + 917779, 917780, 917781, 917782, 65027, 917783, 917784, 917785, 917786, + 917787, 917788, 917789, 917790, 917791, 917792, 65028, 917793, 917794, + 917795, 917796, 917797, 917798, 917799, 917800, 917801, 917802, 65029, + 917803, 917804, 917805, 917806, 917807, 917808, 917809, 917810, 917811, + 917812, 65030, 917813, 917814, 917815, 917816, 917817, 917818, 917819, + 917820, 917821, 917822, 65031, 917823, 917824, 917825, 917826, 917827, + 917828, 917829, 917830, 917831, 917832, 65032, 917833, 917834, 917835, + 917836, 917837, 917838, 917839, 917840, 917841, 917842, 129499, 983245, + 983254, 983364, 983365, 983366, 983367, 983368, 983369, 983370, 983371, + 983372, 983373, 983255, 983374, 983375, 983376, 983377, 983378, 983379, + 983380, 983381, 983382, 983383, 983256, 983384, 983385, 983386, 983387, + 983388, 983389, 983390, 983391, 983392, 983393, 983257, 983394, 983395, + 983396, 983397, 983398, 983399, 983400, 983401, 983402, 983403, 983258, + 983404, 983405, 983406, 983407, 983408, 983409, 983410, 983411, 983412, + 983413, 983259, 983414, 983415, 983416, 983417, 983418, 983419, 983420, + 983421, 983422, 983423, 983260, 983424, 983425, 983426, 983427, 983428, + 983429, 983430, 983431, 983432, 983433, 983281, 983434, 983435, 983436, + 983437, 983438, 983439, 983440, 983441, 983442, 983443, 983282, 983444, + 983445, 983446, 983447, 983448, 983449, 983450, 983451, 983452, 983453, + 983283, 983454, 983455, 983456, 983457, 983458, 983459, 983460, 983461, + 983462, 983463, 983246, 983284, 983464, 983465, 983466, 983467, 983468, + 983469, 983470, 983471, 983472, 983473, 983285, 983474, 983475, 983476, + 983477, 983478, 983479, 983480, 983481, 983482, 983483, 983286, 983484, + 983485, 983486, 983487, 983488, 983489, 983490, 983491, 983492, 983493, + 983287, 983494, 983495, 983496, 983497, 983498, 983499, 983500, 983501, + 983502, 983503, 983288, 983504, 983505, 983506, 983507, 983508, 983509, + 983510, 983511, 983512, 983513, 983289, 983514, 983515, 983516, 983517, + 983518, 983519, 983520, 983290, 983291, 983292, 983293, 983247, 983294, + 983295, 983296, 983297, 983298, 983299, 983300, 983301, 983302, 983303, + 983248, 983304, 983305, 983306, 983307, 983308, 983309, 983310, 983311, + 983312, 983313, 983249, 983314, 983315, 983316, 983317, 983318, 983319, + 983320, 983321, 983322, 983323, 983250, 983324, 983325, 983326, 983327, + 983328, 983329, 983330, 983331, 983332, 983333, 983251, 983334, 983335, + 983336, 983337, 983338, 983339, 983340, 983341, 983342, 983343, 983252, + 983344, 983345, 983346, 983347, 983348, 983349, 983350, 983351, 983352, + 983353, 983253, 983354, 983355, 983356, 983357, 983358, 983359, 983360, + 983361, 983362, 983363, 7401, 7402, 7409, 7403, 7404, 7415, 7410, 7418, + 7413, 7379, 7398, 7396, 7411, 7408, 7406, 7407, 7405, 7414, 7397, 7400, + 7395, 7399, 7394, 7380, 7384, 7412, 7417, 7386, 7389, 7376, 7388, 7378, + 7416, 7392, 7391, 7387, 7390, 7381, 7382, 7383, 7385, 7393, 7377, 8483, + 10704, 10980, 10978, 10186, 117780, 8942, 8286, 117917, 12347, 12337, + 12339, 12341, 12338, 12340, 117892, 124, 9168, 10992, 10991, 117904, + 118293, 9087, 9896, 129904, 129905, 129906, 129907, 129908, 129909, + 117916, 117770, 11135, 983069, 983144, 11823, 128678, 10650, 11837, + 128976, 128959, 128947, 128637, 128941, 128953, 128636, 128904, 128914, + 128934, 8921, 8920, 128933, 9910, 128887, 10799, 128243, 9996, 118469, 128249, 127918, 128252, 94193, 94192, 8983, 127931, 9805, 66929, 66930, 66936, 66935, 66942, 66943, 66947, 66946, 66950, 66949, 66932, 66931, 66934, 66933, 66957, 66956, 66959, 66958, 66941, 66940, 66937, 66944, @@ -13095,193 +13249,194 @@ static const unsigned int dawg_pos_to_codepoint[] = { 71920, 71919, 71916, 71915, 71914, 71935, 9888, 128702, 127754, 128003, 129341, 127817, 8986, 12316, 11071, 10547, 127988, 127987, 128075, 12336, 65103, 8967, 65099, 128465, 128576, 128553, 10172, 128146, 983238, - 127947, 9840, 128734, 9784, 9855, 129197, 129196, 9702, 9815, 129548, - 129590, 129608, 129611, 9812, 129545, 129587, 9816, 129542, 129563, - 129584, 129591, 129605, 129549, 129616, 129614, 129615, 9817, 129550, - 129592, 9813, 129546, 129588, 9814, 129547, 129589, 129569, 129566, - 129570, 129571, 129567, 129568, 9675, 128906, 9862, 10732, 9863, 9717, - 9718, 9716, 9719, 9831, 10209, 10210, 10211, 129995, 9671, 9672, 128922, - 128923, 10192, 9826, 9931, 128071, 128407, 9759, 9663, 9661, 9920, 9921, - 10069, 9872, 9983, 10048, 128174, 11214, 10023, 9785, 128427, 129293, - 9825, 9989, 129984, 11041, 11053, 10710, 11036, 128326, 9945, 128072, - 9756, 9669, 9667, 9665, 117894, 117895, 128928, 11046, 11088, 9725, 9723, - 11048, 11229, 10001, 9649, 11040, 127985, 10068, 9645, 11092, 9659, 9657, - 9655, 128073, 9758, 9988, 65094, 9750, 11051, 11090, 9643, 9786, 9828, - 9633, 128307, 128916, 9635, 128917, 10212, 9713, 9714, 10213, 9634, 9712, - 9715, 9707, 9093, 127779, 127781, 127782, 9788, 127780, 9734, 128382, - 9743, 9186, 10177, 9943, 128070, 9757, 129994, 9653, 9651, 9708, 11055, - 9647, 118278, 11006, 11038, 10163, 128011, 129144, 129152, 129120, + 127947, 9840, 128734, 9784, 9855, 129197, 129196, 9702, 129621, 9815, + 129548, 129590, 129608, 129611, 129620, 9812, 129545, 129587, 9816, + 129542, 129563, 129584, 129591, 129605, 129549, 129616, 129614, 129615, + 9817, 129550, 129592, 9813, 129546, 129588, 9814, 129547, 129589, 129569, + 129566, 129570, 129571, 129567, 129568, 9675, 128906, 9862, 10732, 9863, + 9717, 9718, 9716, 9719, 9831, 10209, 10210, 10211, 129995, 9671, 9672, + 128922, 128923, 10192, 9826, 9931, 128071, 128407, 9759, 9663, 9661, + 9920, 9921, 10069, 9872, 9983, 10048, 128174, 11214, 10023, 9785, 128427, + 129293, 9825, 9989, 129984, 11041, 11053, 10710, 11036, 128326, 9945, + 128072, 9756, 9669, 9667, 9665, 117894, 117895, 128928, 11046, 11088, + 9725, 9723, 11048, 11229, 10001, 9649, 11040, 127985, 10068, 9645, 11092, + 9659, 9657, 9655, 128073, 9758, 9988, 65094, 9750, 11051, 11090, 9643, + 9786, 9828, 9633, 128307, 128916, 9635, 128917, 10212, 9713, 9714, 10213, + 9634, 9712, 9715, 9707, 9093, 127779, 127781, 127782, 9788, 127780, 9734, + 128382, 9743, 9186, 10177, 9943, 128070, 9757, 129994, 9653, 9651, 9708, + 11055, 9647, 118278, 11006, 11038, 10163, 128011, 129144, 129152, 129120, 129136, 129128, 129146, 129154, 129122, 129138, 129130, 129147, 129155, 129123, 129139, 129131, 129145, 129153, 129121, 129137, 129129, 129149, 129157, 129125, 129141, 129133, 129148, 129156, 129124, 129140, 129132, 129150, 129158, 129126, 129142, 129134, 129151, 129159, 129127, 129143, 129135, 11838, 129344, 127888, 127788, 129695, 127863, 128521, 129725, - 128430, 128732, 128105, 128111, 128098, 128090, 128082, 128097, 128698, + 128430, 128732, 128105, 128111, 128098, 128090, 128097, 128082, 128698, 11825, 8288, 128506, 128543, 129713, 8361, 129717, 128058, 127873, 8768, 129340, 128295, 9997, 983233, 8999, 129659, 129644, 129643, 129641, 129639, 129642, 129645, 129640, 129637, 129636, 129634, 129632, 129635, - 129638, 129633, 8891, 129393, 128155, 165, 69292, 69291, 69293, 69256, - 69255, 69254, 69281, 69268, 69259, 69248, 69271, 69289, 69286, 69287, - 69257, 69278, 69277, 69279, 69276, 69280, 69296, 69282, 69251, 69250, - 69266, 69265, 69267, 69253, 69252, 69269, 69274, 69275, 69284, 69285, - 69272, 69258, 69288, 69297, 69263, 69260, 69270, 69262, 69261, 69249, - 69283, 69273, 69264, 42139, 42149, 42173, 42172, 42132, 42147, 42179, - 42174, 42148, 42169, 42150, 42134, 42166, 42135, 42145, 42143, 42137, - 42175, 42157, 42154, 42167, 42165, 42163, 42130, 42182, 42129, 42171, - 42138, 42140, 42136, 42131, 42151, 42164, 42181, 42142, 42156, 42170, - 42176, 42178, 42160, 42133, 42144, 42152, 42159, 42161, 42158, 42141, - 42146, 42177, 42180, 42155, 42162, 42128, 42168, 42153, 41070, 41059, - 41060, 41058, 41073, 41072, 41071, 41068, 41069, 41066, 41067, 41065, - 41048, 41052, 41053, 41050, 41051, 41049, 41046, 41047, 41056, 41057, - 41054, 41055, 41063, 41064, 41061, 41062, 41076, 41077, 41074, 41075, - 41006, 40995, 40996, 40994, 41009, 41008, 41007, 41004, 41005, 41002, - 41003, 41001, 40984, 40988, 40989, 40986, 40987, 40985, 40982, 40983, - 40992, 40993, 40990, 40991, 40999, 41000, 40997, 40998, 41012, 41015, - 41014, 41013, 41010, 41011, 41842, 41831, 41832, 41829, 41830, 41845, - 41844, 41843, 41841, 41827, 41828, 41825, 41826, 41839, 41840, 41837, - 41838, 41835, 41836, 41833, 41834, 41848, 41851, 41850, 41849, 41846, - 41847, 41670, 41659, 41660, 41658, 41673, 41672, 41671, 41668, 41669, - 41666, 41667, 41665, 41648, 41652, 41653, 41650, 41651, 41649, 41646, - 41647, 41656, 41657, 41654, 41655, 41663, 41664, 41661, 41662, 41676, - 41679, 41678, 41677, 41674, 41675, 41293, 41282, 41283, 41281, 41296, - 41295, 41294, 41291, 41292, 41272, 41275, 41276, 41274, 41273, 41270, - 41271, 41289, 41290, 41288, 41279, 41280, 41277, 41278, 41286, 41287, - 41284, 41285, 41238, 41228, 41227, 41241, 41240, 41239, 41236, 41237, - 41218, 41221, 41222, 41220, 41219, 41216, 41217, 41234, 41235, 41233, - 41225, 41226, 41223, 41224, 41231, 41232, 41229, 41230, 41174, 41175, - 41173, 41171, 41172, 41169, 41170, 41167, 41168, 41165, 41166, 41184, - 41185, 41182, 41183, 41178, 41181, 41180, 41179, 41176, 41177, 41494, - 41496, 41497, 41495, 41492, 41493, 41516, 41504, 41505, 41502, 41503, - 41519, 41518, 41517, 41514, 41515, 41500, 41501, 41498, 41499, 41512, - 41513, 41510, 41511, 41508, 41509, 41506, 41507, 41460, 41448, 41449, - 41446, 41447, 41463, 41462, 41461, 41458, 41459, 41436, 41440, 41441, - 41438, 41439, 41437, 41434, 41435, 41444, 41445, 41442, 41443, 41456, - 41457, 41454, 41455, 41452, 41453, 41450, 41451, 41584, 41583, 41582, - 41389, 41379, 41380, 41378, 41392, 41391, 41390, 41387, 41388, 41369, - 41372, 41373, 41371, 41370, 41367, 41368, 41385, 41386, 41384, 41382, - 41383, 41381, 41376, 41377, 41374, 41375, 41395, 41398, 41397, 41396, - 41393, 41394, 41125, 41117, 41118, 41116, 41128, 41127, 41126, 41123, - 41124, 41107, 41110, 41111, 41109, 41108, 41105, 41106, 41114, 41115, - 41112, 41113, 41121, 41122, 41119, 41120, 41130, 41133, 41132, 41131, - 41129, 41336, 41334, 41335, 41333, 41332, 41340, 41338, 41339, 41337, - 41322, 41326, 41327, 41324, 41325, 41323, 41320, 41321, 41330, 41331, - 41328, 41329, 41556, 41557, 41554, 41555, 41563, 41564, 41562, 41544, - 41548, 41549, 41546, 41547, 41545, 41542, 41543, 41552, 41553, 41550, - 41551, 41560, 41561, 41558, 41559, 41591, 41592, 41589, 41590, 41598, - 41599, 41597, 41587, 41588, 41585, 41586, 41595, 41596, 41593, 41594, - 40962, 40960, 983243, 40966, 40967, 40964, 40965, 40963, 40961, 42025, - 42017, 42018, 42016, 42028, 42027, 42026, 42023, 42024, 42010, 42014, - 42015, 42012, 42013, 42011, 42008, 42009, 42021, 42022, 42019, 42020, - 42031, 42032, 42029, 42030, 41970, 41962, 41963, 41960, 41961, 41973, - 41972, 41971, 41968, 41969, 41954, 41958, 41959, 41956, 41957, 41955, - 41952, 41953, 41966, 41967, 41964, 41965, 41976, 41979, 41978, 41977, - 41974, 41975, 41488, 41476, 41477, 41475, 41491, 41490, 41489, 41486, - 41487, 41466, 41469, 41470, 41468, 41467, 41464, 41465, 41473, 41474, - 41471, 41472, 41484, 41485, 41482, 41483, 41480, 41481, 41478, 41479, - 41424, 41413, 41414, 41411, 41412, 41427, 41426, 41425, 41422, 41423, - 41420, 41421, 41419, 41401, 41405, 41406, 41403, 41404, 41402, 41399, - 41400, 41409, 41410, 41407, 41408, 41417, 41418, 41415, 41416, 41430, - 41433, 41432, 41431, 41428, 41429, 41538, 41527, 41528, 41526, 41541, - 41540, 41539, 41536, 41537, 41534, 41535, 41533, 41524, 41525, 41522, - 41523, 41531, 41532, 41529, 41530, 41521, 41520, 41157, 41147, 41148, - 41145, 41146, 41160, 41159, 41158, 41155, 41156, 41136, 41139, 41140, - 41138, 41137, 41134, 41135, 41143, 41144, 41141, 41142, 41151, 41152, - 41149, 41150, 41163, 41164, 41161, 41162, 41154, 41153, 41080, 41083, - 41084, 41082, 41081, 41078, 41079, 41087, 41088, 41085, 41086, 41091, - 41092, 41089, 41090, 41095, 41098, 41097, 41096, 41093, 41094, 41101, - 41104, 41103, 41102, 41099, 41100, 41299, 41302, 41301, 41300, 41297, - 41298, 41312, 41313, 41311, 41305, 41306, 41303, 41304, 41309, 41310, - 41307, 41308, 41316, 41319, 41318, 41317, 41314, 41315, 41574, 41572, - 41573, 41580, 41581, 41579, 41566, 41567, 41565, 41570, 41571, 41568, - 41569, 41577, 41578, 41575, 41576, 42048, 42042, 42041, 42051, 42050, - 42049, 42047, 42035, 42039, 42040, 42037, 42038, 42036, 42033, 42034, - 42045, 42046, 42043, 42044, 42054, 42057, 42056, 42055, 42052, 42053, - 41881, 41882, 41880, 41878, 41879, 41876, 41877, 41885, 41886, 41883, - 41884, 41889, 41892, 41891, 41890, 41887, 41888, 41895, 41898, 41897, - 41896, 41893, 41894, 42075, 42067, 42068, 42066, 42076, 42073, 42074, - 42060, 42064, 42065, 42062, 42063, 42061, 42058, 42059, 42071, 42072, - 42069, 42070, 41727, 41721, 41720, 41730, 41729, 41728, 41726, 41723, - 41722, 41711, 41714, 41715, 41713, 41712, 41709, 41710, 41718, 41719, - 41716, 41717, 41733, 41736, 41735, 41734, 41731, 41732, 41725, 41724, - 41363, 41352, 41353, 41351, 41366, 41365, 41364, 41361, 41362, 41343, - 41346, 41347, 41345, 41344, 41341, 41342, 41349, 41350, 41348, 41359, - 41360, 41358, 41356, 41357, 41354, 41355, 41036, 41028, 41029, 41027, - 41039, 41038, 41037, 41034, 41035, 41018, 41021, 41022, 41020, 41019, - 41016, 41017, 41025, 41026, 41023, 41024, 41032, 41033, 41030, 41031, - 41042, 41045, 41044, 41043, 41040, 41041, 41998, 41990, 41991, 41988, - 41989, 42001, 42000, 41999, 41996, 41997, 41982, 41986, 41987, 41984, - 41985, 41983, 41980, 41981, 41994, 41995, 41992, 41993, 42004, 42007, - 42006, 42005, 42002, 42003, 42115, 42107, 42108, 42105, 42106, 42118, - 42117, 42116, 42113, 42114, 42099, 42103, 42104, 42101, 42102, 42100, - 42097, 42098, 42111, 42112, 42109, 42110, 42121, 42124, 42123, 42122, - 42119, 42120, 41866, 41855, 41854, 41869, 41868, 41867, 41864, 41865, - 41862, 41863, 41860, 41861, 41858, 41859, 41856, 41857, 41872, 41875, - 41874, 41873, 41870, 41871, 41853, 41852, 41942, 41931, 41932, 41930, - 41945, 41944, 41943, 41940, 41941, 41938, 41939, 41937, 41928, 41929, - 41926, 41927, 41935, 41936, 41933, 41934, 41948, 41951, 41950, 41949, - 41946, 41947, 41772, 41775, 41776, 41774, 41773, 41770, 41771, 41786, - 41787, 41785, 41779, 41780, 41777, 41778, 41783, 41784, 41781, 41782, - 41790, 41791, 41788, 41789, 41794, 41797, 41796, 41795, 41792, 41793, - 41916, 41904, 41905, 41903, 41919, 41918, 41917, 41914, 41915, 41901, - 41902, 41899, 41900, 41912, 41913, 41910, 41911, 41908, 41909, 41906, - 41907, 41922, 41925, 41924, 41923, 41920, 41921, 41760, 41749, 41750, - 41748, 41763, 41762, 41761, 41758, 41759, 41739, 41742, 41743, 41741, - 41740, 41737, 41738, 41756, 41757, 41755, 41746, 41747, 41744, 41745, - 41753, 41754, 41751, 41752, 41766, 41769, 41768, 41767, 41764, 41765, - 41266, 41255, 41256, 41253, 41254, 41269, 41268, 41267, 41264, 41265, - 41244, 41247, 41248, 41246, 41245, 41242, 41243, 41262, 41263, 41261, - 41251, 41252, 41249, 41250, 41259, 41260, 41257, 41258, 41203, 41202, - 41188, 41192, 41193, 41190, 41191, 41189, 41186, 41187, 41196, 41197, - 41194, 41195, 41200, 41201, 41198, 41199, 41206, 41209, 41208, 41207, - 41204, 41205, 41212, 41215, 41214, 41213, 41210, 41211, 40981, 41605, - 41606, 41604, 41611, 41612, 41610, 41608, 41609, 41607, 41602, 41603, - 41600, 41601, 42079, 42083, 42084, 42081, 42082, 42080, 42077, 42078, - 42089, 42090, 42087, 42088, 42093, 42096, 42095, 42094, 42091, 42092, - 42086, 42085, 41815, 41803, 41804, 41802, 41818, 41817, 41816, 41813, - 41814, 41800, 41801, 41798, 41799, 41811, 41812, 41809, 41810, 41807, - 41808, 41805, 41806, 41821, 41824, 41823, 41822, 41819, 41820, 41636, - 41625, 41626, 41624, 41639, 41638, 41637, 41634, 41635, 41615, 41618, - 41619, 41617, 41616, 41613, 41614, 41696, 41697, 41695, 41693, 41694, - 41692, 41682, 41686, 41687, 41684, 41685, 41683, 41680, 41681, 41690, - 41691, 41688, 41689, 41699, 41702, 41701, 41700, 41698, 41705, 41708, - 41707, 41706, 41703, 41704, 41632, 41633, 41631, 41622, 41623, 41620, - 41621, 41629, 41630, 41627, 41628, 41642, 41645, 41644, 41643, 41640, - 41641, 40973, 40974, 40972, 40970, 40971, 40968, 40969, 40977, 40978, - 40975, 40976, 40980, 40979, 9775, 129664, 8959, 10853, 10632, 10634, - 10814, 10852, 10631, 10633, 10783, 10784, 10785, 10625, 10626, 72262, - 72256, 72253, 72252, 72254, 72251, 72250, 72261, 72255, 72243, 72215, - 72214, 72230, 72229, 72220, 72219, 72242, 72204, 72203, 72207, 72216, - 72211, 72221, 72238, 72239, 72240, 72228, 72227, 72213, 72212, 72218, - 72217, 72225, 72224, 72209, 72208, 72206, 72205, 72223, 72222, 72231, - 72232, 72233, 72241, 72210, 72236, 72226, 72235, 72237, 72234, 72192, - 72259, 72258, 72260, 72257, 72248, 72245, 72246, 72247, 72244, 72249, - 72263, 72202, 72199, 72200, 72198, 72197, 72195, 72194, 72201, 72196, - 72193, 129427, 65279, 8204, 8203, 8205, 11234, 129296, 118593, 118584, - 118585, 118591, 118580, 118589, 118528, 118540, 118531, 118543, 118559, - 118529, 118541, 118532, 118544, 118592, 118573, 118569, 118568, 118581, - 118586, 118561, 118582, 118583, 118566, 118538, 118550, 118555, 118556, - 118535, 118547, 118537, 118549, 118553, 118558, 118534, 118546, 118572, - 118562, 118571, 118554, 118533, 118545, 118587, 118588, 118530, 118542, - 118552, 118563, 118539, 118551, 118557, 118536, 118548, 118579, 118570, - 118560, 118567, 118565, 118564, 118590, 118576, 118577, 118578, 118619, - 118639, 118637, 118645, 118721, 118611, 118635, 118659, 118626, 118623, - 118625, 118624, 118622, 118638, 118612, 118660, 118608, 118609, 118657, - 118719, 118695, 118699, 118698, 118697, 118696, 118722, 118720, 118703, - 118708, 118707, 118706, 118705, 118704, 118610, 118620, 118723, 118616, - 118617, 118640, 118672, 118636, 118658, 118652, 118651, 118649, 118650, - 118648, 118646, 118647, 118644, 118643, 118641, 118642, 118653, 118656, - 118654, 118655, 118662, 118670, 118664, 118666, 118669, 118663, 118665, - 118671, 118667, 118668, 118673, 118614, 118615, 118618, 118685, 118687, - 118684, 118683, 118686, 118681, 118680, 118709, 118713, 118711, 118716, - 118717, 118714, 118715, 118712, 118718, 118710, 118676, 118679, 118690, - 118688, 118693, 118694, 118691, 118692, 118689, 118675, 118677, 118678, - 118674, 118701, 118702, 118700, 118682, 118632, 118631, 118634, 118633, - 118628, 118627, 118630, 118629, 118613, 118621, 118661, 118596, 118597, - 118594, 118595, 118598, 129503, 983264, 983222, 983221, 983223, + 129638, 129633, 8891, 94196, 94197, 94198, 129393, 128155, 165, 69292, + 69291, 69293, 69256, 69255, 69254, 69281, 69268, 69259, 69248, 69271, + 69289, 69286, 69287, 69257, 69278, 69277, 69279, 69276, 69280, 69296, + 69282, 69251, 69250, 69266, 69265, 69267, 69253, 69252, 69269, 69274, + 69275, 69284, 69285, 69272, 69258, 69288, 69297, 69263, 69260, 69270, + 69262, 69261, 69249, 69283, 69273, 69264, 42139, 42149, 42173, 42172, + 42132, 42147, 42179, 42174, 42148, 42169, 42150, 42134, 42166, 42135, + 42145, 42143, 42137, 42175, 42157, 42154, 42167, 42165, 42163, 42130, + 42182, 42129, 42171, 42138, 42140, 42136, 42131, 42151, 42164, 42181, + 42142, 42156, 42170, 42176, 42178, 42160, 42133, 42144, 42152, 42159, + 42161, 42158, 42141, 42146, 42177, 42180, 42155, 42162, 42128, 42168, + 42153, 41070, 41059, 41060, 41058, 41073, 41072, 41071, 41068, 41069, + 41066, 41067, 41065, 41048, 41052, 41053, 41050, 41051, 41049, 41046, + 41047, 41056, 41057, 41054, 41055, 41063, 41064, 41061, 41062, 41076, + 41077, 41074, 41075, 41006, 40995, 40996, 40994, 41009, 41008, 41007, + 41004, 41005, 41002, 41003, 41001, 40984, 40988, 40989, 40986, 40987, + 40985, 40982, 40983, 40992, 40993, 40990, 40991, 40999, 41000, 40997, + 40998, 41012, 41015, 41014, 41013, 41010, 41011, 41842, 41831, 41832, + 41829, 41830, 41845, 41844, 41843, 41841, 41827, 41828, 41825, 41826, + 41839, 41840, 41837, 41838, 41835, 41836, 41833, 41834, 41848, 41851, + 41850, 41849, 41846, 41847, 41670, 41659, 41660, 41658, 41673, 41672, + 41671, 41668, 41669, 41666, 41667, 41665, 41648, 41652, 41653, 41650, + 41651, 41649, 41646, 41647, 41656, 41657, 41654, 41655, 41663, 41664, + 41661, 41662, 41676, 41679, 41678, 41677, 41674, 41675, 41293, 41282, + 41283, 41281, 41296, 41295, 41294, 41291, 41292, 41272, 41275, 41276, + 41274, 41273, 41270, 41271, 41289, 41290, 41288, 41279, 41280, 41277, + 41278, 41286, 41287, 41284, 41285, 41238, 41228, 41227, 41241, 41240, + 41239, 41236, 41237, 41218, 41221, 41222, 41220, 41219, 41216, 41217, + 41234, 41235, 41233, 41225, 41226, 41223, 41224, 41231, 41232, 41229, + 41230, 41174, 41175, 41173, 41171, 41172, 41169, 41170, 41167, 41168, + 41165, 41166, 41184, 41185, 41182, 41183, 41178, 41181, 41180, 41179, + 41176, 41177, 41494, 41496, 41497, 41495, 41492, 41493, 41516, 41504, + 41505, 41502, 41503, 41519, 41518, 41517, 41514, 41515, 41500, 41501, + 41498, 41499, 41512, 41513, 41510, 41511, 41508, 41509, 41506, 41507, + 41460, 41448, 41449, 41446, 41447, 41463, 41462, 41461, 41458, 41459, + 41436, 41440, 41441, 41438, 41439, 41437, 41434, 41435, 41444, 41445, + 41442, 41443, 41456, 41457, 41454, 41455, 41452, 41453, 41450, 41451, + 41584, 41583, 41582, 41389, 41379, 41380, 41378, 41392, 41391, 41390, + 41387, 41388, 41369, 41372, 41373, 41371, 41370, 41367, 41368, 41385, + 41386, 41384, 41382, 41383, 41381, 41376, 41377, 41374, 41375, 41395, + 41398, 41397, 41396, 41393, 41394, 41125, 41117, 41118, 41116, 41128, + 41127, 41126, 41123, 41124, 41107, 41110, 41111, 41109, 41108, 41105, + 41106, 41114, 41115, 41112, 41113, 41121, 41122, 41119, 41120, 41130, + 41133, 41132, 41131, 41129, 41336, 41334, 41335, 41333, 41332, 41340, + 41338, 41339, 41337, 41322, 41326, 41327, 41324, 41325, 41323, 41320, + 41321, 41330, 41331, 41328, 41329, 41556, 41557, 41554, 41555, 41563, + 41564, 41562, 41544, 41548, 41549, 41546, 41547, 41545, 41542, 41543, + 41552, 41553, 41550, 41551, 41560, 41561, 41558, 41559, 41591, 41592, + 41589, 41590, 41598, 41599, 41597, 41587, 41588, 41585, 41586, 41595, + 41596, 41593, 41594, 40962, 40960, 983243, 40966, 40967, 40964, 40965, + 40963, 40961, 42025, 42017, 42018, 42016, 42028, 42027, 42026, 42023, + 42024, 42010, 42014, 42015, 42012, 42013, 42011, 42008, 42009, 42021, + 42022, 42019, 42020, 42031, 42032, 42029, 42030, 41970, 41962, 41963, + 41960, 41961, 41973, 41972, 41971, 41968, 41969, 41954, 41958, 41959, + 41956, 41957, 41955, 41952, 41953, 41966, 41967, 41964, 41965, 41976, + 41979, 41978, 41977, 41974, 41975, 41488, 41476, 41477, 41475, 41491, + 41490, 41489, 41486, 41487, 41466, 41469, 41470, 41468, 41467, 41464, + 41465, 41473, 41474, 41471, 41472, 41484, 41485, 41482, 41483, 41480, + 41481, 41478, 41479, 41424, 41413, 41414, 41411, 41412, 41427, 41426, + 41425, 41422, 41423, 41420, 41421, 41419, 41401, 41405, 41406, 41403, + 41404, 41402, 41399, 41400, 41409, 41410, 41407, 41408, 41417, 41418, + 41415, 41416, 41430, 41433, 41432, 41431, 41428, 41429, 41538, 41527, + 41528, 41526, 41541, 41540, 41539, 41536, 41537, 41534, 41535, 41533, + 41524, 41525, 41522, 41523, 41531, 41532, 41529, 41530, 41521, 41520, + 41157, 41147, 41148, 41145, 41146, 41160, 41159, 41158, 41155, 41156, + 41136, 41139, 41140, 41138, 41137, 41134, 41135, 41143, 41144, 41141, + 41142, 41151, 41152, 41149, 41150, 41163, 41164, 41161, 41162, 41154, + 41153, 41080, 41083, 41084, 41082, 41081, 41078, 41079, 41087, 41088, + 41085, 41086, 41091, 41092, 41089, 41090, 41095, 41098, 41097, 41096, + 41093, 41094, 41101, 41104, 41103, 41102, 41099, 41100, 41299, 41302, + 41301, 41300, 41297, 41298, 41312, 41313, 41311, 41305, 41306, 41303, + 41304, 41309, 41310, 41307, 41308, 41316, 41319, 41318, 41317, 41314, + 41315, 41574, 41572, 41573, 41580, 41581, 41579, 41566, 41567, 41565, + 41570, 41571, 41568, 41569, 41577, 41578, 41575, 41576, 42048, 42042, + 42041, 42051, 42050, 42049, 42047, 42035, 42039, 42040, 42037, 42038, + 42036, 42033, 42034, 42045, 42046, 42043, 42044, 42054, 42057, 42056, + 42055, 42052, 42053, 41881, 41882, 41880, 41878, 41879, 41876, 41877, + 41885, 41886, 41883, 41884, 41889, 41892, 41891, 41890, 41887, 41888, + 41895, 41898, 41897, 41896, 41893, 41894, 42075, 42067, 42068, 42066, + 42076, 42073, 42074, 42060, 42064, 42065, 42062, 42063, 42061, 42058, + 42059, 42071, 42072, 42069, 42070, 41727, 41721, 41720, 41730, 41729, + 41728, 41726, 41723, 41722, 41711, 41714, 41715, 41713, 41712, 41709, + 41710, 41718, 41719, 41716, 41717, 41733, 41736, 41735, 41734, 41731, + 41732, 41725, 41724, 41363, 41352, 41353, 41351, 41366, 41365, 41364, + 41361, 41362, 41343, 41346, 41347, 41345, 41344, 41341, 41342, 41349, + 41350, 41348, 41359, 41360, 41358, 41356, 41357, 41354, 41355, 41036, + 41028, 41029, 41027, 41039, 41038, 41037, 41034, 41035, 41018, 41021, + 41022, 41020, 41019, 41016, 41017, 41025, 41026, 41023, 41024, 41032, + 41033, 41030, 41031, 41042, 41045, 41044, 41043, 41040, 41041, 41998, + 41990, 41991, 41988, 41989, 42001, 42000, 41999, 41996, 41997, 41982, + 41986, 41987, 41984, 41985, 41983, 41980, 41981, 41994, 41995, 41992, + 41993, 42004, 42007, 42006, 42005, 42002, 42003, 42115, 42107, 42108, + 42105, 42106, 42118, 42117, 42116, 42113, 42114, 42099, 42103, 42104, + 42101, 42102, 42100, 42097, 42098, 42111, 42112, 42109, 42110, 42121, + 42124, 42123, 42122, 42119, 42120, 41866, 41855, 41854, 41869, 41868, + 41867, 41864, 41865, 41862, 41863, 41860, 41861, 41858, 41859, 41856, + 41857, 41872, 41875, 41874, 41873, 41870, 41871, 41853, 41852, 41942, + 41931, 41932, 41930, 41945, 41944, 41943, 41940, 41941, 41938, 41939, + 41937, 41928, 41929, 41926, 41927, 41935, 41936, 41933, 41934, 41948, + 41951, 41950, 41949, 41946, 41947, 41772, 41775, 41776, 41774, 41773, + 41770, 41771, 41786, 41787, 41785, 41779, 41780, 41777, 41778, 41783, + 41784, 41781, 41782, 41790, 41791, 41788, 41789, 41794, 41797, 41796, + 41795, 41792, 41793, 41916, 41904, 41905, 41903, 41919, 41918, 41917, + 41914, 41915, 41901, 41902, 41899, 41900, 41912, 41913, 41910, 41911, + 41908, 41909, 41906, 41907, 41922, 41925, 41924, 41923, 41920, 41921, + 41760, 41749, 41750, 41748, 41763, 41762, 41761, 41758, 41759, 41739, + 41742, 41743, 41741, 41740, 41737, 41738, 41756, 41757, 41755, 41746, + 41747, 41744, 41745, 41753, 41754, 41751, 41752, 41766, 41769, 41768, + 41767, 41764, 41765, 41266, 41255, 41256, 41253, 41254, 41269, 41268, + 41267, 41264, 41265, 41244, 41247, 41248, 41246, 41245, 41242, 41243, + 41262, 41263, 41261, 41251, 41252, 41249, 41250, 41259, 41260, 41257, + 41258, 41203, 41202, 41188, 41192, 41193, 41190, 41191, 41189, 41186, + 41187, 41196, 41197, 41194, 41195, 41200, 41201, 41198, 41199, 41206, + 41209, 41208, 41207, 41204, 41205, 41212, 41215, 41214, 41213, 41210, + 41211, 40981, 41605, 41606, 41604, 41611, 41612, 41610, 41608, 41609, + 41607, 41602, 41603, 41600, 41601, 42079, 42083, 42084, 42081, 42082, + 42080, 42077, 42078, 42089, 42090, 42087, 42088, 42093, 42096, 42095, + 42094, 42091, 42092, 42086, 42085, 41815, 41803, 41804, 41802, 41818, + 41817, 41816, 41813, 41814, 41800, 41801, 41798, 41799, 41811, 41812, + 41809, 41810, 41807, 41808, 41805, 41806, 41821, 41824, 41823, 41822, + 41819, 41820, 41636, 41625, 41626, 41624, 41639, 41638, 41637, 41634, + 41635, 41615, 41618, 41619, 41617, 41616, 41613, 41614, 41696, 41697, + 41695, 41693, 41694, 41692, 41682, 41686, 41687, 41684, 41685, 41683, + 41680, 41681, 41690, 41691, 41688, 41689, 41699, 41702, 41701, 41700, + 41698, 41705, 41708, 41707, 41706, 41703, 41704, 41632, 41633, 41631, + 41622, 41623, 41620, 41621, 41629, 41630, 41627, 41628, 41642, 41645, + 41644, 41643, 41640, 41641, 40973, 40974, 40972, 40970, 40971, 40968, + 40969, 40977, 40978, 40975, 40976, 40980, 40979, 9775, 129664, 8959, + 10853, 10632, 10634, 10814, 10852, 10631, 10633, 10783, 10784, 10785, + 10625, 10626, 72262, 72256, 72253, 72252, 72254, 72251, 72250, 72261, + 72255, 72243, 72215, 72214, 72230, 72229, 72220, 72219, 72242, 72204, + 72203, 72207, 72216, 72211, 72221, 72238, 72239, 72240, 72228, 72227, + 72213, 72212, 72218, 72217, 72225, 72224, 72209, 72208, 72206, 72205, + 72223, 72222, 72231, 72232, 72233, 72241, 72210, 72236, 72226, 72235, + 72237, 72234, 72192, 72259, 72258, 72260, 72257, 72248, 72245, 72246, + 72247, 72244, 72249, 72263, 72202, 72199, 72200, 72198, 72197, 72195, + 72194, 72201, 72196, 72193, 129427, 65279, 8204, 8203, 8205, 11234, + 129296, 118593, 118584, 118585, 118591, 118580, 118589, 118528, 118540, + 118531, 118543, 118559, 118529, 118541, 118532, 118544, 118592, 118573, + 118569, 118568, 118581, 118586, 118561, 118582, 118583, 118566, 118538, + 118550, 118555, 118556, 118535, 118547, 118537, 118549, 118553, 118558, + 118534, 118546, 118572, 118562, 118571, 118554, 118533, 118545, 118587, + 118588, 118530, 118542, 118552, 118563, 118539, 118551, 118557, 118536, + 118548, 118579, 118570, 118560, 118567, 118565, 118564, 118590, 118576, + 118577, 118578, 118619, 118639, 118637, 118645, 118721, 118611, 118635, + 118659, 118626, 118623, 118625, 118624, 118622, 118638, 118612, 118660, + 118608, 118609, 118657, 118719, 118695, 118699, 118698, 118697, 118696, + 118722, 118720, 118703, 118708, 118707, 118706, 118705, 118704, 118610, + 118620, 118723, 118616, 118617, 118640, 118672, 118636, 118658, 118652, + 118651, 118649, 118650, 118648, 118646, 118647, 118644, 118643, 118641, + 118642, 118653, 118656, 118654, 118655, 118662, 118670, 118664, 118666, + 118669, 118663, 118665, 118671, 118667, 118668, 118673, 118614, 118615, + 118618, 118685, 118687, 118684, 118683, 118686, 118681, 118680, 118709, + 118713, 118711, 118716, 118717, 118714, 118715, 118712, 118718, 118710, + 118676, 118679, 118690, 118688, 118693, 118694, 118691, 118692, 118689, + 118675, 118677, 118678, 118674, 118701, 118702, 118700, 118682, 118632, + 118631, 118634, 118633, 118628, 118627, 118630, 118629, 118613, 118621, + 118661, 118596, 118597, 118594, 118595, 118598, 129503, 983264, 983222, + 983221, 983223, }; #define DAWG_CODEPOINT_TO_POS_SHIFT 8 -#define DAWG_CODEPOINT_TO_POS_NOTFOUND 40951 +#define DAWG_CODEPOINT_TO_POS_NOTFOUND 41412 static const unsigned char dawg_codepoint_to_pos_index1[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, @@ -13305,14 +13460,14 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 136, 52, 52, 52, 52, 52, 52, 137, 138, 139, 140, 52, 141, 142, 143, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 144, 145, 146, 147, 148, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 144, 145, 146, 147, 148, 149, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 52, 52, 52, 52, 149, 150, 151, 152, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 153, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, - 166, 167, 168, 52, 52, 52, 52, 169, 170, 171, 172, 52, 173, 174, 52, 175, - 176, 177, 52, 52, 178, 179, 180, 52, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 150, 151, 152, 153, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 154, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, + 167, 168, 169, 52, 52, 52, 52, 170, 171, 172, 173, 52, 174, 175, 176, + 177, 178, 179, 52, 52, 180, 181, 182, 52, 183, 184, 185, 186, 187, 188, + 189, 190, 191, 192, 193, 194, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -13326,7 +13481,7 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 52, 193, 194, 195, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 195, 196, 197, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -13483,7 +13638,7 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 196, 197, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 198, 199, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -13497,7 +13652,7 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, - 52, 52, 52, 52, 52, 52, 52, 52, 52, 198, 199, 200, 201, 52, 52, 52, 52, + 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 200, 201, 202, 203, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, @@ -13526,1694 +13681,1684 @@ static const unsigned char dawg_codepoint_to_pos_index1[] = { 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, + 52, }; static const unsigned short dawg_codepoint_to_pos_index2[] = { - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 34087, 17354, 31745, 29447, 11088, 31317, 368, 1052, 24153, - 31969, 2527, 31568, 8453, 20460, 17532, 33926, 10872, 10857, 10869, - 10866, 10851, 10848, 10863, 10860, 10875, 10854, 8056, 32578, 24396, - 16792, 18147, 31743, 8452, 22741, 22787, 22797, 22813, 22828, 22873, - 22877, 22894, 22908, 22937, 22942, 22954, 22974, 22983, 22999, 23049, - 23057, 23060, 23081, 23105, 23134, 23173, 23184, 23193, 23196, 23210, - 24115, 31870, 31983, 6819, 25161, 18118, 23302, 23352, 23372, 23395, - 23431, 23490, 23498, 23516, 23535, 23572, 23578, 23592, 23631, 23644, - 23668, 23725, 23736, 23742, 23780, 23822, 23895, 23943, 23957, 23966, - 23974, 23990, 24056, 38833, 31932, 37296, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 29440, - 20815, 6131, 31610, 9951, 39398, 4937, 32560, 10839, 8647, 17433, 24163, - 29422, 33877, 31793, 26847, 10554, 31580, 34712, 34711, 4, 27575, 31465, - 27581, 6127, 34715, 25814, 32030, 38965, 38963, 38971, 20818, 22770, - 22771, 22759, 22773, 22752, 22754, 22775, 22798, 22859, 22860, 22831, - 22846, 22924, 22925, 22914, 22912, 22870, 22997, 23038, 23039, 23006, - 23024, 23013, 28287, 23022, 23165, 23166, 23135, 23138, 23205, 23126, - 23812, 23333, 23334, 23322, 23336, 23313, 23315, 23339, 23373, 23464, - 23465, 23434, 23449, 23553, 23554, 23536, 23538, 23489, 23663, 23707, - 23708, 23669, 23693, 23676, 10999, 23691, 23930, 23933, 23898, 23901, - 23985, 23844, 23976, 22757, 23320, 22742, 23303, 22767, 23330, 22802, - 23378, 22800, 23375, 22806, 23383, 22801, 23377, 22820, 23403, 22817, - 23397, 22849, 23453, 22862, 23467, 22841, 23444, 22856, 23461, 22840, - 23443, 22879, 23500, 22882, 23503, 22886, 23508, 22878, 23499, 22897, - 23519, 22904, 23528, 22918, 23550, 22916, 23545, 22927, 23556, 22921, - 23547, 22911, 23419, 23222, 24010, 22938, 23573, 22943, 23579, 23591, - 22968, 23609, 22964, 23596, 22966, 23599, 22961, 23606, 22963, 23617, - 22992, 23656, 22984, 23645, 22986, 23649, 23664, 22872, 23472, 23002, - 23686, 23041, 23710, 23019, 23682, 23223, 24013, 23069, 23759, 23061, - 23743, 23062, 23745, 23087, 23781, 23086, 23787, 23084, 23785, 23082, - 23783, 23106, 23823, 23109, 23827, 23116, 23839, 23162, 23927, 23154, - 23917, 23168, 23935, 23169, 23926, 23145, 23908, 23159, 23922, 23185, - 23958, 23197, 23975, 23198, 23213, 23991, 23216, 23997, 23212, 23994, - 23623, 23361, 22793, 22788, 23353, 23129, 23847, 23044, 22804, 23381, - 22779, 22825, 22821, 23398, 23852, 23078, 23104, 23043, 22874, 23493, - 22885, 22892, 23534, 22935, 22928, 22953, 23590, 23595, 23619, 23123, - 22990, 23650, 23005, 23031, 23700, 23046, 23722, 23055, 23733, 23299, - 23130, 23848, 22871, 23239, 23835, 23115, 23836, 23114, 23147, 23910, - 23172, 23177, 23202, 23980, 23219, 24000, 22864, 22866, 23482, 23487, - 23291, 23131, 23849, 23238, 23292, 23293, 23294, 23231, 23241, 22827, - 22815, 23425, 22973, 22962, 23630, 22998, 22987, 23667, 22765, 23328, - 22915, 23537, 23012, 23675, 23137, 23900, 23142, 23905, 23141, 23904, - 23139, 23902, 23140, 23903, 23884, 22753, 23314, 22749, 23310, 22777, - 23342, 22887, 23509, 22880, 23501, 22944, 23580, 23020, 23689, 23021, - 23690, 22865, 23484, 23575, 22826, 22814, 23424, 22881, 23502, 22906, - 23192, 22991, 23655, 22755, 23316, 22776, 23341, 23023, 23692, 22751, - 23312, 22772, 23335, 22845, 23448, 22861, 23466, 22909, 23544, 22926, - 23555, 23018, 23681, 23040, 23709, 23066, 23749, 23071, 23760, 23144, - 23907, 23167, 23934, 23085, 23786, 23108, 23825, 23209, 23989, 22898, - 23520, 22988, 23402, 23048, 23724, 23221, 24004, 22748, 23309, 22829, - 23432, 23014, 23677, 23027, 23696, 23015, 23678, 23016, 23679, 23206, - 23986, 23598, 23648, 23826, 23416, 23429, 23741, 22774, 22807, 23384, - 22956, 23110, 23794, 23999, 22888, 23510, 22794, 23171, 23125, 22863, - 23469, 22941, 23577, 23098, 23739, 23073, 23763, 23208, 23984, 23876, - 23344, 23877, 23360, 23713, 23376, 23399, 23407, 23772, 23803, 23807, - 23719, 23767, 23769, 23391, 23417, 23507, 23810, 23268, 23514, 23779, - 23853, 23527, 23533, 23557, 23568, 23246, 23604, 23593, 23612, 23620, - 23879, 23880, 23639, 23652, 23661, 23279, 23364, 23253, 23390, 23735, - 23864, 23867, 23870, 23754, 23755, 23750, 23770, 23255, 23247, 23800, - 23475, 23418, 23820, 23479, 23871, 23838, 23896, 23937, 23950, 23873, - 23888, 23881, 23286, 24003, 23993, 23481, 23483, 23295, 23297, 23236, - 23288, 23234, 23266, 23389, 23269, 23275, 23574, 23886, 23248, 23738, - 23296, 23240, 23426, 23412, 23427, 23891, 23840, 23890, 23496, 23627, - 23628, 23233, 23235, 23854, 23855, 27905, 27906, 27914, 27936, 27963, - 27966, 27861, 27983, 27985, 27834, 27777, 27991, 27688, 27842, 27844, - 27820, 27793, 27840, 27822, 27846, 27993, 27779, 27769, 6060, 27997, - 27823, 27687, 27792, 27818, 27809, 27815, 27814, 27990, 27800, 27721, - 27720, 27992, 27778, 27833, 27831, 4930, 11194, 32162, 29978, 33836, - 11253, 27847, 27770, 27904, 27916, 27943, 27984, 27940, 27784, 27799, - 27827, 27812, 27789, 27999, 27998, 27996, 27994, 27776, 27805, 27817, - 27807, 27810, 27811, 27830, 27829, 27828, 27813, 27839, 27690, 27790, - 27691, 27791, 27849, 27832, 27806, 8264, 8062, 8146, 8428, 8370, 8390, - 8072, 8171, 8168, 8277, 8415, 8197, 8078, 8443, 8196, 8198, 8080, 8280, - 8436, 8083, 8398, 8084, 8265, 8063, 8355, 8411, 8349, 8279, 8352, 8438, - 8202, 8395, 8379, 8394, 8399, 8174, 8170, 8414, 8085, 8148, 8388, 8442, - 8075, 8283, 8079, 8147, 8074, 8281, 8432, 8375, 8369, 8199, 8431, 8419, - 8367, 8418, 8366, 8406, 8282, 8422, 8426, 8450, 8444, 8185, 8266, 8064, - 8274, 8273, 8276, 8275, 8076, 8211, 8195, 8348, 8381, 8278, 8071, 8356, - 8437, 8269, 8402, 8353, 8212, 8449, 8344, 8403, 8401, 8407, 8173, 8068, - 8190, 8417, 8179, 8178, 8182, 8183, 8191, 8180, 8189, 8296, 8304, 8309, - 8317, 8320, 8302, 8334, 8336, 8338, 8323, 8326, 8341, 8342, 18332, 18596, - 18227, 18463, 18414, 18410, 18321, 18578, 40951, 40951, 18653, 18603, - 18604, 18602, 18658, 18335, 40951, 40951, 40951, 40951, 18618, 18348, - 18225, 18201, 18258, 18248, 18272, 40951, 18304, 40951, 18319, 18294, - 18510, 18204, 18331, 18329, 18327, 18249, 18333, 18228, 18325, 18259, - 18328, 18334, 18336, 18337, 18338, 18295, 18324, 18305, 40951, 18307, - 18326, 18310, 18322, 18330, 18323, 18274, 18264, 18315, 18460, 18475, - 18500, 18519, 18530, 18435, 18595, 18593, 18465, 18466, 18597, 18476, - 18590, 18501, 18541, 18598, 18599, 18600, 18601, 18568, 18581, 18582, - 18592, 18588, 18591, 18521, 18579, 18594, 18580, 18543, 18506, 18526, - 18577, 18539, 18567, 18343, 18655, 18614, 18622, 18620, 18621, 18430, - 18431, 18395, 18406, 18462, 18405, 18587, 18408, 18464, 18407, 18542, - 18404, 18585, 8528, 8622, 8506, 8600, 8502, 8596, 8500, 8594, 8498, 8592, - 8527, 8621, 8497, 8591, 18394, 18433, 18411, 18409, 18344, 18412, 18434, - 18309, 18589, 18339, 18308, 18586, 18432, 18341, 18342, 18340, 10218, - 10220, 10167, 10206, 10142, 10171, 10158, 10277, 10290, 10113, 10116, - 10130, 10245, 10215, 10258, 10175, 10143, 10159, 10291, 10200, 10179, - 10217, 10284, 10280, 10213, 10256, 10230, 10181, 10197, 10186, 10249, - 10117, 10192, 10195, 10127, 10137, 10199, 10207, 10133, 10160, 10263, - 10261, 10211, 10274, 10266, 10180, 10279, 10271, 10302, 10318, 10492, - 10359, 10338, 10376, 10485, 10481, 10372, 10434, 10389, 10340, 10356, - 10345, 10415, 10420, 10351, 10354, 10452, 10463, 10358, 10366, 10458, - 10319, 10441, 10439, 10370, 10475, 10444, 10339, 10480, 10472, 10377, - 10379, 10326, 10365, 10468, 10330, 10317, 10478, 10491, 10408, 10414, - 10455, 10404, 10374, 10436, 10334, 10250, 10416, 10273, 10474, 10226, - 10385, 10112, 10406, 10222, 10381, 10155, 10314, 10223, 10382, 10246, - 10405, 10120, 10424, 10289, 10490, 10228, 10387, 10229, 10388, 10141, - 10467, 10121, 10429, 10251, 10417, 10253, 10419, 10243, 10402, 10523, - 8139, 8133, 8136, 8134, 8135, 8089, 8142, 10257, 10435, 10270, 10448, - 10193, 10352, 10202, 10361, 10204, 10363, 10201, 10360, 10286, 10487, - 10282, 10483, 10231, 10390, 10233, 10392, 10234, 10393, 10153, 10312, - 10188, 10347, 10296, 10496, 10118, 10421, 10149, 10308, 10196, 10355, - 10129, 10454, 10268, 10446, 10269, 10447, 10208, 10367, 10295, 10495, - 10162, 10321, 10163, 10322, 10259, 10437, 10146, 10305, 10147, 10306, - 10299, 10287, 10488, 10232, 10391, 10184, 10343, 10191, 10350, 10190, - 10349, 10244, 10403, 10198, 10357, 10423, 10145, 10304, 10144, 10303, - 10294, 10494, 10219, 10378, 10254, 10432, 10255, 10433, 10285, 10486, - 10281, 10482, 10148, 10307, 10216, 10375, 10214, 10373, 10252, 10418, - 10151, 10310, 10152, 10311, 10194, 10353, 10140, 10466, 10139, 10465, - 10138, 10464, 10161, 10320, 10203, 10362, 10275, 10476, 10205, 10364, - 10209, 10368, 10210, 10369, 10237, 10396, 10236, 10395, 10242, 10401, - 10235, 10394, 10238, 10397, 10239, 10398, 10240, 10399, 10241, 10400, - 10125, 10428, 10185, 10344, 10114, 10409, 10126, 10431, 10272, 10473, - 10293, 10493, 10292, 10471, 10150, 10309, 10182, 10341, 10187, 10346, - 10119, 10422, 10260, 10438, 10189, 10348, 10173, 10332, 10177, 10336, - 10183, 10342, 40951, 2445, 2452, 2436, 2458, 2432, 2456, 2433, 2434, - 2423, 2455, 2451, 2448, 2450, 2428, 2440, 2457, 2438, 2435, 2426, 2453, - 2424, 2454, 2443, 2447, 2427, 2442, 2437, 2431, 2444, 2446, 2422, 2430, - 2429, 2425, 2441, 2439, 2459, 2449, 40951, 40951, 2509, 2421, 2462, 2461, - 2460, 2513, 2420, 2482, 2485, 2495, 2473, 2501, 2469, 2499, 2470, 2471, - 2484, 2498, 2494, 2491, 2493, 2465, 2477, 2500, 2475, 2472, 2463, 2496, - 2488, 2497, 2480, 2487, 2464, 2479, 2474, 2468, 2481, 2486, 2483, 2467, - 2466, 2490, 2478, 2476, 2502, 2492, 2503, 2489, 2512, 2510, 40951, 40951, - 32017, 24176, 2511, 40951, 19805, 19808, 19809, 19816, 19817, 19813, - 19820, 19818, 19803, 19815, 19812, 19796, 19797, 19798, 19806, 19811, - 19804, 19793, 19801, 19802, 19799, 19800, 19795, 19807, 19810, 19814, - 19821, 19822, 19794, 19819, 19900, 19915, 19904, 19902, 19903, 19905, - 19917, 19913, 19908, 19909, 19906, 19907, 19911, 19901, 19919, 19925, - 19912, 19922, 19914, 19916, 19923, 19899, 19898, 19924, 19910, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 19823, 19830, 19873, - 19871, 19848, 19879, 19853, 19850, 19865, 19890, 19839, 19833, 19875, - 19845, 19877, 19844, 19851, 19855, 19829, 19841, 19836, 19843, 19869, - 19846, 19863, 19857, 19867, 40951, 40951, 40951, 40951, 19926, 19894, - 19897, 19895, 19921, 19920, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 2264, 2299, 1093, 2300, 2301, 2263, - 2408, 2409, 2275, 2406, 2407, 2598, 1062, 1075, 2270, 2305, 2298, 2304, - 2296, 2297, 2306, 2326, 2312, 2341, 2308, 2357, 2356, 2292, 1381, 1083, - 2396, 2404, 1336, 1288, 1151, 1141, 1569, 1144, 1585, 1123, 1165, 1512, - 1511, 1532, 1314, 1273, 1355, 1190, 1529, 1433, 1611, 1467, 1480, 1459, - 1206, 1489, 1606, 1113, 1260, 1342, 1339, 1235, 1234, 1233, 2384, 1240, - 1424, 1325, 1360, 1373, 1397, 1290, 1564, 1159, 1576, 1091, 1073, 1103, - 1085, 1069, 1099, 2293, 2361, 2117, 1098, 1097, 2360, 2282, 2118, 2405, - 2400, 2399, 2401, 2276, 1086, 2403, 2416, 2418, 2415, 2414, 2411, 2410, - 2413, 2412, 2419, 2417, 2268, 1079, 2398, 1094, 1218, 1220, 1486, 1156, - 1148, 1147, 1310, 1311, 1313, 1550, 1312, 1538, 1544, 1185, 1520, 1519, - 1412, 1524, 1180, 1283, 1281, 1392, 1221, 1280, 1499, 1506, 1215, 1200, - 1197, 1198, 1203, 1212, 1226, 1193, 1199, 1450, 1436, 1447, 1444, 1437, - 1445, 1440, 1322, 1443, 1468, 1471, 1472, 1462, 1461, 1493, 1116, 1219, - 1243, 1241, 1557, 1245, 1419, 1427, 1428, 1337, 1488, 1331, 1330, 1382, - 1328, 1251, 1254, 1383, 1253, 1267, 1252, 1364, 1362, 1366, 1365, 1406, - 1398, 1453, 1407, 1403, 1301, 1501, 1298, 1291, 1292, 1515, 1573, 1349, - 1603, 1549, 1600, 1352, 1572, 1556, 1229, 1594, 1590, 1566, 1615, 1595, - 1577, 1580, 1095, 1164, 2315, 2314, 2317, 2316, 2343, 2325, 2323, 1084, - 2383, 2340, 2339, 2309, 2318, 2355, 2319, 2359, 2358, 2337, 2320, 2272, - 1082, 1081, 2280, 2354, 1192, 1441, 17374, 17376, 17373, 17372, 17369, - 17368, 17371, 17370, 17377, 17375, 1481, 1207, 1262, 2303, 2302, 1297, - 34842, 34916, 34913, 34914, 34910, 34851, 34838, 34839, 34915, 34912, - 34837, 34847, 34846, 34845, 40951, 34835, 34883, 34879, 34893, 34889, - 34890, 34853, 34852, 34854, 34896, 34894, 34855, 34886, 34887, 34891, - 34892, 34884, 34856, 34868, 34895, 34875, 34882, 34897, 34869, 34873, - 34881, 34885, 34874, 34880, 34888, 34870, 34872, 34871, 34902, 34901, - 34900, 34905, 34904, 34903, 34907, 34906, 34841, 34840, 34850, 34849, - 34848, 34844, 34843, 34908, 34922, 34923, 34909, 34920, 34919, 34918, - 34917, 34899, 34898, 34921, 34836, 40951, 40951, 34876, 34877, 34878, - 1171, 1174, 1169, 1170, 1172, 1173, 1167, 1282, 1279, 1196, 1191, 1438, - 1474, 1118, 1114, 1117, 1246, 1244, 1344, 1340, 1338, 1376, 1375, 1404, - 1401, 1402, 1367, 1439, 1442, 1473, 1278, 1276, 1470, 1434, 1277, 1150, - 1149, 1232, 1231, 1230, 1568, 1567, 1579, 1578, 1274, 1475, 1469, 1327, - 36865, 36878, 36874, 36891, 36890, 36869, 36866, 36854, 36885, 36870, - 36859, 36858, 36881, 36868, 36861, 36862, 36879, 36857, 36887, 36880, - 36892, 36873, 36872, 36871, 36883, 36864, 36867, 36882, 36888, 36877, - 36876, 36856, 36884, 36889, 36855, 36863, 36860, 36886, 36850, 36849, - 36896, 36852, 36897, 36895, 36851, 36853, 36894, 36893, 36898, 36875, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 29306, 29308, 29305, 29304, 29301, 29300, - 29303, 29302, 29309, 29307, 29341, 29328, 29342, 29327, 29343, 29325, - 29324, 29312, 29317, 29330, 29336, 29338, 29316, 29326, 29311, 29323, - 29322, 29337, 29329, 29331, 29333, 29334, 29319, 29335, 29320, 29318, - 29332, 29339, 29340, 29321, 29314, 29313, 29315, 29294, 29295, 29296, - 29292, 29289, 29290, 29291, 29293, 29288, 29345, 29344, 29347, 29346, - 29297, 29348, 29310, 40951, 40951, 29298, 29299, 29349, 32389, 32376, - 32390, 32378, 32381, 32377, 32392, 32380, 32387, 32396, 32382, 32383, - 32393, 32395, 32384, 32379, 32397, 32388, 32394, 32391, 32385, 32386, - 32399, 32400, 32403, 32398, 32404, 32401, 32423, 32433, 32428, 32422, - 32432, 32427, 32421, 32431, 32405, 32429, 32425, 32435, 32406, 32424, - 32434, 32426, 32430, 32402, 40951, 40951, 32413, 32407, 32409, 32412, - 32410, 32414, 32436, 32419, 32417, 32420, 32416, 32418, 32411, 32415, - 32408, 40951, 25603, 25590, 25592, 25591, 25593, 25602, 25600, 25605, - 25585, 25583, 25582, 25594, 25595, 25596, 25586, 25604, 25597, 25588, - 25598, 25599, 25587, 25584, 25601, 25606, 25589, 25581, 25607, 25608, - 40951, 40951, 25609, 40951, 34861, 34866, 34862, 34864, 34860, 34859, - 34863, 34867, 34858, 34857, 34865, 40951, 40951, 40951, 40951, 40951, - 1137, 1127, 1138, 1154, 1136, 1124, 1133, 1130, 1134, 1132, 1155, 1129, - 1140, 1126, 1128, 1139, 1125, 1131, 1135, 2385, 2386, 2388, 1537, 2281, - 2274, 1405, 1275, 1494, 1492, 1341, 2402, 40951, 2271, 2273, 40951, - 40951, 40951, 40951, 40951, 2269, 2327, 2348, 2347, 2350, 2116, 2365, - 1076, 1096, 1168, 1175, 1315, 1491, 1242, 1425, 1361, 1374, 1591, 1593, - 1446, 1565, 1458, 1372, 1194, 1460, 1255, 1487, 1614, 1115, 1329, 1426, - 1166, 1413, 1517, 1435, 1592, 1111, 1109, 1112, 1414, 1518, 1539, 1500, - 1343, 1261, 1110, 1317, 1316, 1363, 1272, 2307, 2310, 2338, 2333, 2342, - 1107, 1106, 2364, 1108, 1105, 2353, 2328, 2324, 2345, 2344, 2321, 2346, - 2329, 2331, 2330, 2332, 2335, 2334, 2311, 2322, 1080, 2397, 1065, 1063, - 1067, 1066, 1064, 1068, 2391, 2393, 2395, 2390, 2392, 2394, 2267, 2266, - 2265, 2336, 1088, 1087, 1100, 1621, 2277, 1620, 2279, 1077, 1078, 2278, - 1072, 2119, 10772, 10762, 10776, 10781, 10697, 10671, 10672, 10741, - 10742, 10717, 10718, 10736, 10738, 10679, 10698, 10749, 10673, 10680, - 10699, 10703, 10674, 10711, 10710, 10692, 10690, 10723, 10677, 10681, - 10728, 10726, 10724, 10733, 10732, 10685, 10684, 10722, 10735, 10734, - 10687, 10686, 10725, 10721, 10744, 10743, 10708, 10707, 10695, 10716, - 10714, 10713, 10731, 10730, 10729, 10740, 10700, 10701, 10702, 10694, - 10794, 10793, 10774, 10775, 10784, 10806, 10807, 10796, 10797, 10802, - 10803, 10790, 10800, 10808, 10785, 10791, 10801, 10792, 10786, 10780, - 10795, 10787, 10822, 10783, 10782, 10666, 10663, 10789, 10799, 10798, - 10748, 10709, 10689, 10746, 10682, 10712, 10747, 10715, 10737, 10739, - 10804, 10805, 10810, 10809, 10817, 10819, 10816, 10815, 10812, 10811, - 10814, 10813, 10820, 10818, 10664, 10779, 10678, 10705, 10704, 10675, - 10720, 10719, 10696, 10745, 10693, 10691, 10727, 10688, 10683, 10706, - 3549, 3617, 3620, 3622, 40951, 3573, 3574, 3587, 3588, 3585, 3586, 3567, - 3569, 40951, 40951, 3609, 3575, 40951, 40951, 3610, 3576, 3560, 3557, - 3601, 3600, 3589, 3599, 3598, 3603, 3602, 3591, 3582, 3581, 3578, 3577, - 3590, 3584, 3583, 3580, 3579, 3592, 40951, 3605, 3604, 3597, 3596, 3608, - 3572, 3561, 40951, 3607, 40951, 40951, 40951, 3593, 3594, 3595, 3606, - 40951, 40951, 3618, 3619, 3624, 3633, 3634, 3627, 3628, 3629, 3630, - 40951, 40951, 3635, 3625, 40951, 40951, 3636, 3626, 3621, 3558, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3550, 40951, 40951, - 40951, 40951, 3565, 3564, 40951, 3571, 3568, 3570, 3631, 3632, 40951, - 40951, 3643, 3645, 3642, 3641, 3638, 3637, 3640, 3639, 3646, 3644, 3563, - 3562, 3612, 3611, 3551, 3555, 3554, 3553, 3552, 3556, 3623, 3647, 3566, - 3548, 3616, 40951, 40951, 18887, 18888, 18893, 40951, 18839, 18840, - 18855, 18856, 18853, 18854, 40951, 40951, 40951, 40951, 18874, 18841, - 40951, 40951, 18873, 18842, 18838, 18837, 18835, 18834, 18859, 18866, - 18865, 18868, 18867, 18861, 18850, 18849, 18844, 18843, 18860, 18852, - 18851, 18846, 18845, 18862, 40951, 18870, 18869, 18864, 18863, 18877, - 18879, 18848, 40951, 18858, 18857, 40951, 18878, 18871, 40951, 18872, - 18876, 40951, 40951, 18890, 40951, 18894, 18899, 18900, 18897, 18898, - 40951, 40951, 40951, 40951, 18902, 18895, 40951, 40951, 18901, 18896, - 18892, 40951, 40951, 40951, 18889, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 18836, 18833, 18880, 18847, 40951, 18875, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 18912, 18914, 18911, 18910, 18907, - 18906, 18909, 18908, 18915, 18913, 18903, 18832, 18904, 18916, 18905, - 18891, 18831, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 18727, 18735, 18737, 40951, 18677, 18678, 18696, 18697, - 18694, 18695, 18689, 18691, 18753, 40951, 18724, 18679, 18754, 40951, - 18725, 18680, 18717, 18716, 18713, 18712, 18701, 18711, 18710, 18715, - 18714, 18703, 18686, 18685, 18682, 18681, 18702, 18688, 18687, 18684, - 18683, 18704, 40951, 18719, 18718, 18709, 18708, 18721, 18723, 18722, - 40951, 18699, 18698, 40951, 18693, 18705, 18706, 18707, 18720, 40951, - 40951, 18733, 18734, 18740, 18749, 18750, 18743, 18744, 18745, 18746, - 18738, 40951, 18751, 18741, 18739, 40951, 18752, 18742, 18736, 40951, - 40951, 18767, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18690, 18692, 18747, - 18748, 40951, 40951, 18763, 18765, 18762, 18761, 18758, 18757, 18760, - 18759, 18766, 18764, 18755, 18756, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 18700, 18732, 18731, 18728, 18730, 18726, 18729, 40951, - 30620, 30623, 30626, 40951, 30571, 30572, 30590, 30591, 30588, 30589, - 30583, 30585, 40951, 40951, 30616, 30573, 40951, 40951, 30617, 30574, - 30610, 30609, 30606, 30605, 30594, 30604, 30603, 30608, 30607, 30596, - 30580, 30579, 30576, 30575, 30595, 30582, 30581, 30578, 30577, 30597, - 40951, 30612, 30611, 30602, 30601, 30614, 30570, 30568, 40951, 30593, - 30592, 40951, 30587, 30598, 30599, 30600, 30613, 40951, 40951, 30621, - 30622, 30627, 30636, 30637, 30630, 30631, 30632, 30633, 40951, 40951, - 30638, 30628, 40951, 40951, 30639, 30629, 30625, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 30624, 30557, 30558, 40951, 40951, 40951, - 40951, 30567, 30566, 40951, 30569, 30584, 30586, 30634, 30635, 40951, - 40951, 30646, 30648, 30645, 30644, 30641, 30640, 30643, 30642, 30649, - 30647, 30565, 30615, 30562, 30561, 30564, 30559, 30560, 30563, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35480, - 35497, 40951, 35443, 35444, 35456, 35457, 35452, 35453, 40951, 40951, - 40951, 35461, 35462, 35445, 40951, 35454, 35455, 35446, 35466, 40951, - 40951, 40951, 35438, 35463, 40951, 35465, 40951, 35439, 35441, 40951, - 40951, 40951, 35437, 35442, 40951, 40951, 40951, 35440, 35436, 35468, - 40951, 40951, 40951, 35467, 35470, 35451, 35450, 35449, 35448, 35447, - 35469, 35458, 35459, 35460, 35464, 40951, 40951, 40951, 40951, 35770, - 35777, 35778, 35773, 35774, 40951, 40951, 40951, 35779, 35780, 35771, - 40951, 35775, 35776, 35772, 35496, 40951, 40951, 35783, 40951, 40951, - 40951, 40951, 40951, 40951, 35372, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35409, - 35411, 35408, 35407, 35404, 35403, 35406, 35405, 35412, 35410, 35474, - 35472, 35473, 35402, 35782, 35781, 35401, 35399, 35371, 35477, 35475, - 40951, 40951, 40951, 40951, 40951, 36730, 36731, 36736, 36738, 36729, - 36690, 36691, 36706, 36707, 36702, 36703, 36697, 36699, 40951, 36723, - 36724, 36692, 40951, 36704, 36705, 36693, 36720, 36719, 36716, 36715, - 36679, 36714, 36713, 36718, 36717, 36681, 36686, 36685, 36673, 36672, - 36680, 36689, 36687, 36676, 36674, 36677, 40951, 36722, 36721, 36712, - 36711, 36726, 36727, 36684, 36683, 36696, 36695, 36694, 36701, 36708, - 36709, 36710, 36725, 40951, 40951, 36734, 36735, 36739, 36750, 36751, - 36742, 36743, 36744, 36745, 40951, 36752, 36753, 36740, 40951, 36748, - 36749, 36741, 36737, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 36728, 36664, 40951, 36688, 36675, 36682, 40951, 40951, 36678, 40951, - 40951, 36698, 36700, 36746, 36747, 40951, 40951, 36760, 36762, 36759, - 36758, 36755, 36754, 36757, 36756, 36763, 36761, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 36732, 36671, 36669, 36667, 36665, 36670, - 36668, 36666, 36733, 21274, 21273, 21278, 21282, 21275, 21222, 21223, - 21248, 21249, 21244, 21245, 21239, 21241, 40951, 21265, 21266, 21224, - 40951, 21246, 21247, 21225, 21262, 21261, 21258, 21257, 21219, 21256, - 21255, 21260, 21259, 21221, 21236, 21235, 21227, 21226, 21220, 21238, - 21237, 21229, 21228, 21217, 40951, 21264, 21263, 21254, 21253, 21269, - 21270, 21234, 21233, 21232, 21231, 40951, 21243, 21250, 21251, 21252, - 21268, 40951, 40951, 21276, 21277, 21284, 21295, 21296, 21287, 21288, - 21289, 21290, 40951, 21297, 21298, 21285, 40951, 21293, 21294, 21286, - 21281, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21271, 21283, - 40951, 40951, 40951, 40951, 40951, 40951, 21218, 21267, 40951, 21240, - 21242, 21291, 21292, 40951, 40951, 21305, 21307, 21304, 21303, 21300, - 21299, 21302, 21301, 21308, 21306, 40951, 21279, 21280, 21272, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 25543, 25545, 25550, 25548, 25500, 25474, 25476, 25520, 25521, - 25516, 25517, 25501, 25503, 40951, 25535, 25536, 25477, 40951, 25518, - 25519, 25478, 25532, 25531, 25528, 25527, 25508, 25489, 25488, 25530, - 25529, 25509, 25497, 25495, 25492, 25491, 25507, 25499, 25498, 25494, - 25493, 25510, 25506, 25534, 25533, 25526, 25525, 25538, 25539, 25515, - 25514, 25513, 25512, 25511, 25505, 25522, 25523, 25524, 25537, 25496, - 25546, 25544, 25549, 25552, 25563, 25564, 25555, 25556, 25557, 25558, - 40951, 25565, 25566, 25553, 40951, 25561, 25562, 25554, 25547, 25490, - 25551, 40951, 40951, 40951, 40951, 25486, 25487, 25481, 25567, 25466, - 25464, 25471, 25461, 25462, 25472, 25465, 25475, 25502, 25504, 25559, - 25560, 40951, 40951, 25457, 25459, 25456, 25455, 25452, 25451, 25454, - 25453, 25460, 25458, 25542, 25540, 25541, 25469, 25468, 25473, 25463, - 25467, 25470, 25450, 25483, 25482, 25484, 25479, 25480, 25485, 40951, - 33736, 33735, 33737, 40951, 33681, 33678, 33666, 33665, 33689, 33688, - 33691, 33690, 33687, 33686, 33685, 33684, 33683, 33682, 33679, 33710, - 33709, 33680, 40951, 40951, 40951, 33675, 33700, 33673, 33698, 33723, - 33713, 33672, 33697, 33674, 33699, 33720, 33721, 33714, 33667, 33692, - 33669, 33694, 33704, 33711, 33668, 33693, 33670, 33695, 33707, 40951, - 33712, 33676, 33701, 33671, 33696, 33702, 33677, 33719, 33717, 40951, - 33706, 40951, 40951, 33718, 33722, 33705, 33708, 33716, 33703, 33715, - 40951, 40951, 40951, 33734, 40951, 40951, 40951, 40951, 33754, 33746, - 33741, 33747, 33742, 33748, 40951, 33743, 40951, 33744, 33749, 33740, - 33753, 33750, 33751, 33752, 33745, 40951, 40951, 40951, 40951, 40951, - 40951, 33730, 33732, 33729, 33728, 33725, 33724, 33727, 33726, 33733, - 33731, 40951, 40951, 33738, 33739, 33755, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 36917, 36914, - 36912, 36911, 36913, 36915, 36931, 36900, 36902, 36901, 36960, 36903, - 36972, 36904, 36969, 36965, 36962, 36963, 36933, 36905, 36968, 36967, - 36964, 36966, 36934, 36899, 36940, 36937, 36906, 36938, 36907, 36939, - 36929, 36973, 36941, 36942, 36919, 36921, 36970, 36958, 36957, 36959, - 36910, 36918, 36974, 36909, 36935, 36943, 36923, 36946, 36948, 36953, - 36954, 36950, 36951, 36949, 36952, 36936, 40951, 40951, 40951, 40951, - 36975, 36955, 36947, 36956, 36945, 36944, 36920, 36928, 36927, 36926, - 36924, 36925, 36922, 36961, 36932, 36971, 36908, 36982, 36984, 36981, - 36980, 36977, 36976, 36979, 36978, 36985, 36983, 36930, 36916, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 22578, 22576, 40951, 22577, - 40951, 22591, 22606, 22610, 22590, 22600, 40951, 22592, 22607, 22588, - 22586, 22585, 22583, 22582, 22587, 22611, 22603, 22601, 22602, 22584, - 22608, 22609, 22596, 22594, 22573, 22595, 22572, 22589, 22612, 22615, - 22580, 40951, 22581, 40951, 22614, 22597, 22598, 22599, 22604, 22593, - 22616, 22605, 22642, 22624, 22629, 22625, 22627, 22637, 22638, 22631, - 22632, 22633, 22634, 22619, 22630, 22618, 22617, 40951, 40951, 22635, - 22636, 22639, 22628, 22626, 40951, 22640, 40951, 22623, 22620, 22621, - 22622, 22653, 22654, 22641, 40951, 22649, 22651, 22648, 22647, 22644, - 22643, 22646, 22645, 22652, 22650, 40951, 40951, 22569, 22568, 22575, - 22574, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 37208, 37115, 37113, 37114, 37123, 37111, 37108, - 37112, 37132, 37103, 37101, 37124, 37139, 37133, 37129, 37136, 37128, - 37131, 37130, 37107, 37116, 37099, 37098, 37026, 37027, 37025, 37147, - 37148, 37149, 37151, 37152, 37150, 37048, 37050, 37047, 37046, 37043, - 37042, 37045, 37044, 37051, 37049, 37040, 37037, 37036, 37033, 37032, - 37035, 37034, 37041, 37039, 37038, 37104, 37126, 37106, 37125, 37109, - 37135, 37117, 37118, 37119, 37120, 37158, 37144, 37057, 37055, 37085, - 37084, 37067, 37083, 37082, 37092, 40951, 37069, 37077, 37076, 37062, - 37061, 37068, 37079, 37078, 37066, 37065, 37070, 37087, 37086, 37081, - 37080, 37094, 37075, 37074, 37064, 37063, 37095, 37088, 37089, 37090, - 37096, 37059, 37093, 37071, 37072, 37073, 37091, 37097, 37054, 37060, - 37056, 37058, 40951, 40951, 40951, 40951, 37232, 37228, 37229, 37220, - 37221, 37222, 37223, 37224, 37225, 37230, 37231, 37226, 37227, 37155, - 37156, 37218, 37219, 37146, 37160, 37121, 37138, 37142, 37157, 37143, - 37145, 37140, 37141, 37159, 37207, 37206, 37205, 37172, 37171, 37191, - 37190, 37173, 37189, 37188, 37198, 40951, 37175, 37183, 37182, 37165, - 37164, 37174, 37185, 37184, 37169, 37168, 37176, 37193, 37192, 37187, - 37186, 37200, 37181, 37180, 37167, 37166, 37202, 37194, 37195, 37196, - 37203, 37201, 37199, 37177, 37178, 37179, 37197, 37204, 37170, 37162, - 37163, 37161, 40951, 37052, 37053, 37028, 37029, 37031, 37030, 37209, - 37216, 37214, 37217, 37215, 37210, 37213, 37212, 37211, 40951, 37154, - 37153, 37102, 37105, 37127, 37122, 37110, 32022, 24181, 32023, 24182, - 37137, 37134, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28582, - 28561, 28560, 28559, 28591, 28659, 28658, 28661, 28660, 28592, 28589, - 28637, 28636, 28643, 28642, 28590, 28621, 28638, 28645, 28644, 28593, - 28663, 28662, 28657, 28656, 28588, 28665, 28595, 28655, 28641, 28620, - 28664, 28654, 28551, 28615, 28652, 28653, 28646, 28647, 28554, 28587, - 28666, 28553, 28758, 28742, 28765, 28766, 28759, 28760, 28754, 28739, - 28747, 28748, 28755, 28683, 28702, 28707, 28706, 28682, 28546, 28544, - 28545, 28543, 28558, 28783, 28785, 28782, 28781, 28778, 28777, 28780, - 28779, 28786, 28784, 28705, 28694, 28712, 28716, 28711, 28715, 28596, - 28619, 28648, 28649, 28650, 28651, 28761, 28762, 28763, 28764, 28586, - 28585, 28583, 28584, 28549, 28548, 28547, 28618, 28753, 28727, 28728, - 28640, 28639, 28756, 28757, 28697, 28698, 28699, 28700, 28701, 28557, - 28556, 28555, 28743, 28745, 28746, 28744, 28610, 28609, 28608, 28606, - 28614, 28598, 28611, 28599, 28601, 28612, 28604, 28602, 28613, 28550, - 28752, 28749, 28750, 28751, 28689, 28690, 28691, 28692, 28687, 28688, - 28686, 28594, 28703, 28678, 28680, 28677, 28676, 28673, 28672, 28675, - 28674, 28681, 28679, 28684, 28685, 28740, 28741, 28714, 28713, 17728, - 17754, 17739, 17760, 17761, 17759, 17749, 17750, 17762, 17743, 17752, - 17755, 17757, 17763, 17745, 17748, 17753, 17747, 17751, 17764, 17744, - 17742, 17738, 17758, 17746, 17734, 17737, 17741, 17736, 17735, 17756, - 17740, 17729, 17733, 17731, 17766, 17730, 17732, 40951, 17765, 40951, - 40951, 40951, 40951, 40951, 17727, 40951, 40951, 17811, 17842, 17819, - 17848, 17817, 17847, 17840, 17837, 17849, 17829, 17831, 17843, 17845, - 17850, 17833, 17839, 17841, 17835, 17838, 17808, 17832, 17828, 17818, - 17846, 17834, 17812, 17815, 17827, 17814, 17813, 17844, 17826, 17822, - 17825, 17823, 17852, 17820, 17824, 17853, 17851, 17816, 17836, 17810, - 17900, 27795, 17809, 17821, 17830, 19147, 19221, 19155, 19226, 19219, - 19183, 19150, 19165, 19223, 19197, 19215, 19129, 19127, 19213, 19114, - 19149, 19233, 19162, 19236, 19157, 19222, 19159, 19158, 19230, 19193, - 19217, 19196, 19142, 19152, 19146, 19175, 19180, 19179, 19166, 19170, - 19168, 19171, 19172, 19169, 19177, 19176, 19178, 19173, 19144, 19145, - 19204, 19210, 19209, 19202, 19207, 19198, 19199, 19201, 19212, 19206, - 19205, 19203, 19208, 19200, 19211, 19119, 19118, 19124, 19123, 19182, - 19139, 19138, 19136, 19131, 19141, 19132, 19225, 19135, 19134, 19137, - 19130, 19234, 19128, 19121, 19117, 19126, 19122, 19115, 19116, 19120, - 19125, 19163, 19143, 19224, 19235, 19148, 19161, 19156, 19160, 19227, - 19238, 19476, 19382, 19392, 19440, 19444, 19394, 19393, 19446, 19445, - 19421, 19473, 19474, 19431, 19452, 19432, 19472, 19471, 19475, 19461, - 19398, 19450, 19405, 19390, 19391, 19442, 19441, 19396, 19397, 19395, - 19448, 19449, 19429, 19428, 19424, 19422, 19430, 19453, 19454, 19455, - 19460, 19459, 19437, 19438, 19436, 19433, 19439, 19466, 19465, 19464, - 19463, 19462, 19470, 19468, 19404, 19401, 19451, 19406, 19408, 19415, - 19419, 19417, 19407, 19383, 19385, 19388, 19387, 19420, 19389, 19443, - 19447, 19426, 19427, 19258, 19359, 19261, 19281, 19284, 19288, 19363, - 19309, 19310, 19315, 19319, 19328, 19331, 19324, 19336, 19268, 19298, - 19301, 19337, 19354, 19250, 19241, 19244, 19267, 19372, 19294, 19245, - 19260, 19262, 19287, 19286, 19290, 19289, 19285, 19370, 19364, 19312, - 19335, 19329, 19330, 19349, 19316, 19318, 19323, 19322, 19313, 19327, - 19325, 19314, 19332, 19278, 19275, 19269, 19274, 19273, 19271, 19276, - 19280, 19257, 19299, 19303, 19308, 19256, 19339, 19347, 19340, 19343, - 19291, 19252, 19253, 19362, 19251, 19373, 19377, 19380, 19296, 19255, - 19248, 19246, 19247, 19249, 19381, 19264, 19265, 19263, 19259, 19266, - 19360, 17020, 17027, 17026, 17021, 17025, 17024, 17022, 17023, 17247, - 17255, 17254, 17248, 17252, 17251, 17249, 17253, 17013, 17019, 17017, - 17014, 17016, 17015, 17018, 17004, 17064, 17072, 17071, 17065, 17069, - 17068, 17066, 17062, 17176, 17183, 17181, 17177, 17179, 17178, 17182, - 17180, 17151, 17160, 17159, 17152, 17156, 17155, 17153, 17157, 17191, - 17197, 17196, 17192, 17166, 17161, 17193, 17195, 17167, 17175, 17174, - 17168, 17172, 17171, 17169, 17173, 17143, 17150, 17149, 17144, 17148, - 17147, 17145, 17146, 17131, 40951, 17135, 17132, 17134, 17133, 40951, - 40951, 17124, 17130, 17128, 17125, 17127, 17126, 17129, 40951, 17119, - 40951, 17123, 17120, 17122, 17121, 40951, 40951, 16854, 16861, 16860, - 16855, 16859, 16858, 16856, 16845, 17316, 17323, 17321, 17317, 17319, - 17318, 17322, 17320, 17229, 17237, 17236, 17230, 17234, 17233, 17231, - 17235, 16892, 16900, 16899, 16893, 16897, 16896, 16894, 16898, 17284, - 17291, 17290, 17285, 17289, 17288, 17286, 17287, 17272, 40951, 17276, - 17273, 17275, 17274, 40951, 40951, 17082, 17090, 17089, 17083, 17087, - 17086, 17084, 17088, 17073, 17081, 17080, 17074, 17078, 17077, 17075, - 17079, 16974, 16982, 16981, 16975, 16979, 16978, 16976, 16980, 17052, - 17059, 17058, 17053, 17057, 17056, 17054, 17055, 17040, 40951, 17044, - 17041, 17043, 17042, 40951, 40951, 17033, 17039, 17037, 17034, 17036, - 17035, 17038, 40951, 17028, 40951, 17032, 17029, 17031, 17030, 40951, - 40951, 17256, 17263, 17262, 17257, 17261, 17260, 17258, 17259, 17092, - 17098, 17096, 17093, 17095, 17094, 17097, 40951, 17307, 17315, 17314, - 17308, 17312, 17311, 17309, 17313, 17292, 17299, 17297, 17293, 17295, - 17294, 17298, 17296, 17264, 17271, 17270, 17265, 17269, 17268, 17266, - 17267, 16922, 16930, 16929, 16923, 16927, 16926, 16924, 16928, 16907, - 16915, 16914, 16908, 16912, 16911, 16909, 16913, 17238, 17246, 17245, - 17239, 17243, 17242, 17240, 17244, 16995, 16943, 17001, 16996, 17000, - 16999, 16997, 16998, 16983, 40951, 16987, 16984, 16986, 16985, 40951, - 40951, 16967, 16973, 16971, 16968, 16970, 16969, 16972, 16963, 17198, - 17206, 17205, 17199, 17203, 17202, 17200, 17204, 16883, 16891, 16890, - 16884, 16888, 16887, 16885, 16889, 17091, 17106, 17105, 17099, 17103, - 17102, 17100, 17104, 17221, 17228, 17226, 17222, 17224, 17223, 17227, - 17225, 17213, 17220, 17219, 17214, 17218, 17217, 17215, 17216, 16935, - 16942, 16940, 16936, 16938, 16937, 16941, 16933, 17111, 17118, 17117, - 17112, 17116, 17115, 17113, 17109, 17158, 17070, 16939, 40951, 40951, - 16823, 16825, 16824, 16842, 17345, 17343, 16826, 16841, 16827, 16840, - 17344, 16839, 17341, 17339, 17338, 17335, 17334, 17337, 17336, 17342, - 17340, 16828, 16831, 16830, 16835, 16834, 16838, 16837, 16833, 16836, - 16832, 16829, 40951, 40951, 40951, 17164, 17063, 17061, 17060, 17162, - 16846, 16844, 16843, 17163, 16934, 16932, 16931, 17165, 17110, 17108, - 17107, 17333, 17324, 17330, 17331, 17326, 17328, 17332, 17327, 17325, - 17329, 40951, 40951, 40951, 40951, 40951, 40951, 6382, 6383, 6384, 6385, - 6386, 6387, 6345, 6381, 6346, 6347, 6348, 6349, 6350, 6310, 6311, 6312, - 6313, 6314, 6315, 6351, 6352, 6353, 6354, 6355, 6356, 6357, 6358, 6359, - 6360, 6361, 6316, 6309, 6317, 6318, 6319, 6320, 6321, 6322, 6363, 6364, - 6365, 6366, 6367, 6368, 6324, 6323, 6325, 6326, 6327, 6328, 6329, 6303, - 6342, 6304, 6343, 6305, 6344, 6306, 6307, 6308, 6302, 6330, 6331, 6332, - 6333, 6334, 6335, 6336, 6337, 6338, 6339, 6340, 6341, 6369, 6370, 6371, - 6372, 6373, 6374, 6375, 6376, 6377, 6378, 6379, 6380, 6362, 40951, 40951, - 6462, 6463, 6464, 6465, 6466, 6448, 40951, 40951, 5524, 5496, 5269, 5526, - 5527, 5676, 5690, 5973, 5480, 5340, 5267, 5268, 5850, 5940, 5960, 5938, - 5961, 5939, 5942, 5936, 5943, 5937, 5605, 5957, 5934, 5958, 5935, 5606, - 5271, 5974, 5992, 5513, 5512, 5514, 5515, 5507, 5508, 5505, 5504, 5517, - 5511, 5516, 5506, 5498, 5528, 5689, 5275, 5710, 5703, 5708, 5709, 5705, - 5706, 5964, 5338, 5339, 5701, 5702, 5700, 5879, 5698, 5877, 5699, 5878, - 5693, 5875, 5694, 5876, 5696, 5873, 5697, 5874, 5963, 5692, 5872, 5331, - 5849, 5832, 5847, 5848, 5845, 5846, 5971, 5302, 5315, 5830, 5831, 5840, - 5932, 5838, 5930, 5839, 5931, 5836, 5928, 5837, 5929, 5834, 5926, 5835, - 5927, 5611, 5792, 5827, 5828, 5829, 5826, 5547, 5541, 5545, 5546, 5543, - 5544, 5966, 5539, 5540, 5538, 5924, 5536, 5922, 5537, 5923, 5534, 5920, - 5535, 5921, 5531, 5918, 5532, 5919, 5608, 5529, 5530, 5772, 5773, 5774, - 5771, 5495, 5482, 5493, 5494, 5491, 5492, 5965, 5299, 5481, 5489, 5917, - 5487, 5915, 5488, 5916, 5485, 5913, 5486, 5914, 5483, 5911, 5484, 5912, - 5607, 5298, 5748, 5573, 5581, 5590, 5591, 5576, 5577, 5968, 5579, 5580, - 5589, 5871, 5587, 5869, 5588, 5870, 5585, 5867, 5586, 5868, 5583, 5865, - 5584, 5866, 5609, 5572, 5864, 5592, 5273, 5749, 5665, 5626, 5663, 5664, - 5653, 5654, 5969, 5597, 5625, 5662, 5890, 5656, 5888, 5657, 5889, 5610, - 5593, 5371, 5666, 5571, 5558, 5569, 5570, 5567, 5568, 5967, 5556, 5557, - 5566, 5858, 5564, 5856, 5565, 5857, 5562, 5854, 5563, 5855, 5560, 5852, - 5561, 5853, 5548, 5851, 5574, 5791, 5751, 5789, 5790, 5770, 5775, 5970, - 5731, 5750, 5784, 5910, 5782, 5908, 5783, 5909, 5780, 5906, 5781, 5907, - 5778, 5904, 5779, 5905, 5603, 5730, 5274, 5777, 5294, 5578, 5598, 5604, - 5601, 5602, 5599, 5600, 5769, 5767, 5768, 5761, 5762, 5764, 5765, 5760, - 5903, 5758, 5901, 5759, 5902, 5753, 5899, 5754, 5900, 5756, 5897, 5757, - 5898, 5752, 5991, 5977, 5989, 5990, 5979, 5980, 5972, 5975, 5976, 5988, - 5887, 5986, 5885, 5987, 5886, 5984, 5883, 5985, 5884, 5982, 5881, 5983, - 5882, 5612, 5962, 5295, 5880, 5747, 5729, 5713, 5863, 5723, 5727, 5728, - 5725, 5726, 5861, 5721, 5722, 5859, 5715, 5892, 5711, 5891, 5575, 5523, - 5502, 5503, 5518, 5520, 5521, 5500, 5501, 5522, 5933, 5499, 5811, 5596, - 5809, 5594, 5810, 5595, 5807, 5808, 5805, 5806, 5803, 5925, 5793, 5824, - 5825, 5821, 5819, 5818, 5842, 5843, 5844, 5841, 5651, 5649, 5650, 5647, - 5648, 5645, 5646, 5644, 5652, 5525, 5670, 5674, 5675, 5672, 5673, 5668, - 5669, 5667, 5816, 5817, 5812, 5815, 5894, 5895, 5896, 5893, 5631, 5635, - 5636, 5633, 5634, 5629, 5630, 5628, 5637, 5745, 5746, 5741, 5744, 5953, - 5954, 5955, 5952, 5944, 5554, 5555, 5552, 5553, 5550, 5551, 5549, 5801, - 5799, 5800, 5797, 5798, 5795, 5796, 5794, 5272, 5291, 5292, 5293, 5290, - 5279, 5280, 5281, 5278, 5287, 5288, 5289, 5286, 5283, 5284, 5285, 5282, - 5736, 5737, 5733, 5735, 5321, 5320, 5316, 5317, 5319, 5318, 5467, 5466, - 5462, 5463, 5465, 5464, 5473, 5472, 5468, 5469, 5471, 5470, 5337, 5336, - 5332, 5333, 5335, 5334, 5431, 5430, 5426, 5427, 5429, 5428, 5425, 5424, - 5420, 5421, 5423, 5422, 5394, 5393, 5389, 5390, 5392, 5391, 5388, 5330, - 5329, 5326, 5327, 5328, 5324, 5367, 5366, 5362, 5363, 5365, 5364, 5361, - 5360, 5356, 5357, 5359, 5358, 5355, 5374, 5373, 5368, 5369, 5372, 5370, - 5461, 5460, 5456, 5457, 5459, 5458, 5479, 5478, 5474, 5475, 5477, 5476, - 5354, 5738, 5353, 5348, 5349, 5352, 5740, 5351, 5347, 5346, 5342, 5343, - 5345, 5344, 5449, 5448, 5444, 5445, 5447, 5446, 5308, 5307, 5303, 5304, - 5306, 5305, 5443, 5442, 5438, 5439, 5441, 5440, 5407, 5406, 5402, 5403, - 5405, 5404, 5413, 5412, 5408, 5409, 5411, 5410, 5401, 5400, 5396, 5397, - 5399, 5398, 5395, 5341, 5314, 5313, 5309, 5310, 5312, 5311, 5387, 5386, - 5382, 5383, 5385, 5384, 5381, 5380, 5376, 5377, 5379, 5378, 5375, 5437, - 5436, 5432, 5433, 5435, 5434, 5455, 5454, 5450, 5451, 5453, 5452, 5419, - 5418, 5414, 5415, 5417, 5416, 5490, 5519, 5671, 5632, 5642, 5643, 5640, - 5641, 5638, 5639, 5951, 5949, 5950, 5947, 5948, 5945, 5946, 5956, 5277, - 29977, 29952, 29963, 29959, 29969, 29966, 29972, 29975, 29976, 29955, - 29954, 29974, 29960, 29965, 29970, 29964, 29951, 29967, 29973, 29957, - 29961, 29956, 29968, 29971, 29962, 29958, 29953, 29949, 29950, 40951, - 40951, 40951, 32296, 32352, 32346, 32350, 32349, 32344, 32342, 32289, - 32274, 32320, 32273, 32272, 32317, 32332, 32319, 32323, 32324, 32326, - 32312, 32278, 32311, 32297, 32290, 32298, 32300, 32345, 32302, 32301, - 32315, 32329, 32341, 32331, 32283, 32305, 32286, 32309, 32299, 32314, - 32335, 32306, 32348, 32275, 32338, 32337, 32333, 32276, 32354, 32343, - 32334, 32281, 32340, 32328, 32284, 32322, 32287, 32347, 32316, 32330, - 32313, 32282, 32304, 32303, 32285, 32321, 32288, 32308, 32280, 32279, - 32277, 32339, 32318, 32336, 32307, 32351, 32353, 32355, 32356, 32271, - 32357, 32358, 32270, 32310, 32327, 32325, 32294, 32293, 32295, 32292, - 32291, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35025, 35042, - 35043, 35033, 35031, 35027, 35039, 35030, 35028, 35036, 35029, 35035, - 35041, 35037, 35034, 35040, 35038, 35032, 35046, 35047, 35045, 35044, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 35026, - 19648, 19649, 19650, 19639, 19637, 19633, 19645, 19636, 19634, 19642, - 19635, 19641, 19647, 19643, 19640, 19646, 19644, 19638, 19652, 19653, - 19651, 31427, 31428, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 4990, 4991, 4992, 4981, 4979, 4975, 4987, 4978, 4976, 4984, - 4977, 4983, 4989, 4985, 4982, 4988, 4986, 4980, 4993, 4994, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 35061, 35062, 35063, 35053, 35052, 35048, 35058, 35051, 35049, 35056, - 35050, 35055, 35060, 40951, 35054, 35059, 35057, 40951, 35064, 35065, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 22231, 22229, 22232, 22230, 22233, 22227, 22225, 22228, - 22226, 22235, 22249, 22245, 22250, 22246, 22234, 22247, 22243, 22248, - 22244, 22236, 22257, 22237, 22239, 22238, 22253, 22256, 22254, 22252, - 22255, 22241, 22240, 22242, 22258, 22251, 22259, 22206, 22204, 22214, - 22215, 22210, 22213, 22211, 22212, 22223, 22224, 22221, 22222, 22216, - 22205, 22209, 22208, 22207, 22326, 22325, 22327, 22332, 22334, 22338, - 22340, 22342, 22344, 22343, 22335, 22339, 22333, 22345, 22329, 22330, - 22337, 22331, 22280, 22274, 22279, 22281, 22276, 22265, 22273, 22275, - 22270, 22262, 22263, 22282, 22269, 22264, 22271, 22266, 22268, 22277, - 22267, 22278, 22272, 22203, 22260, 22261, 40951, 40951, 22352, 22354, - 22351, 22350, 22347, 22346, 22349, 22348, 22355, 22353, 40951, 40951, - 40951, 40951, 40951, 40951, 22304, 22303, 22300, 22302, 22301, 22295, - 22298, 22299, 22297, 22296, 40951, 40951, 40951, 40951, 40951, 40951, - 28162, 28174, 28170, 28019, 28169, 28020, 28167, 28158, 28171, 28172, - 28173, 28018, 28017, 28016, 28168, 28015, 28011, 28013, 28010, 28009, - 28006, 28005, 28008, 28007, 28014, 28012, 40951, 40951, 40951, 40951, - 40951, 40951, 28023, 28137, 28154, 28139, 28141, 28140, 28142, 28138, - 28148, 28051, 28143, 28149, 28150, 28146, 28055, 28136, 28098, 28097, - 28128, 28144, 28052, 28147, 28153, 28151, 28152, 28145, 28134, 28133, - 28127, 28131, 28132, 28130, 28135, 28129, 28054, 28107, 28125, 28126, - 28114, 28116, 28115, 28117, 28101, 28118, 28121, 28122, 28108, 28120, - 28111, 28103, 28112, 28105, 28110, 28124, 28123, 28119, 28109, 28113, - 28104, 28106, 28102, 28096, 28079, 28080, 28090, 28089, 28086, 28094, - 28075, 28077, 28095, 28084, 28082, 28091, 28093, 28092, 28076, 28078, - 28081, 28088, 28085, 28083, 28087, 28074, 28072, 28073, 28071, 28070, - 28053, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28025, 28027, - 28029, 28036, 28035, 28043, 28039, 28024, 28034, 28050, 28037, 28049, - 28041, 28040, 28031, 28038, 28042, 28028, 28045, 28044, 28048, 28046, - 28047, 28026, 28100, 28099, 28063, 28068, 28059, 28064, 28060, 28056, - 28061, 28057, 28069, 28058, 28066, 28067, 28033, 28032, 28062, 28030, - 28065, 40951, 40951, 40951, 40951, 40951, 5691, 5276, 5270, 5959, 5707, - 5704, 5695, 5833, 5542, 5533, 5582, 5655, 5627, 5559, 5776, 5732, 5763, - 5766, 5755, 5981, 5978, 5724, 5660, 5680, 5661, 5681, 5658, 5678, 5659, - 5679, 5720, 5718, 5719, 5716, 5717, 5714, 5687, 5688, 5685, 5684, 5686, - 5677, 5682, 5683, 5497, 5941, 5510, 5509, 5712, 5862, 5860, 5804, 5802, - 5823, 5822, 5820, 5814, 5813, 5743, 5742, 5734, 5323, 5300, 5325, 5322, - 5739, 5350, 5296, 5297, 5301, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 24478, 24445, 24444, 24425, 24424, 24431, - 24439, 24438, 24443, 24442, 24430, 24428, 24426, 24441, 24440, 24432, - 24447, 24446, 24437, 24436, 24450, 24429, 24451, 24449, 24452, 24433, - 24434, 24435, 24448, 24423, 24427, 40951, 24469, 24476, 24477, 24475, - 24470, 24473, 24471, 24474, 24472, 24468, 24466, 24467, 40951, 40951, - 40951, 40951, 24460, 24457, 24459, 24465, 24458, 24463, 24462, 24464, - 24461, 24454, 24453, 24455, 40951, 40951, 40951, 40951, 24456, 40951, - 40951, 40951, 24479, 24480, 24487, 24489, 24486, 24485, 24482, 24481, - 24484, 24483, 24490, 24488, 35086, 35098, 35081, 35078, 35096, 35099, - 35080, 35079, 35093, 35088, 35087, 35094, 35091, 35097, 35092, 35095, - 35085, 35077, 35082, 35066, 35100, 35070, 35071, 35089, 35084, 35083, - 35090, 35069, 35067, 35068, 40951, 40951, 35072, 35073, 35074, 35075, - 35076, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 29120, 29142, 29102, 29104, 29107, 29124, 29126, 29129, - 29110, 29106, 29122, 29132, 29128, 29144, 29111, 29109, 29108, 29133, - 29131, 29130, 29113, 29112, 29119, 29135, 29134, 29141, 29116, 29121, - 29118, 29138, 29143, 29140, 29117, 29115, 29114, 29139, 29137, 29136, - 29101, 29103, 29123, 29125, 29105, 29127, 40951, 40951, 40951, 40951, - 29165, 29150, 29154, 29160, 29163, 29166, 29152, 29156, 29157, 29161, - 29153, 29151, 29164, 29159, 29158, 29162, 29155, 29100, 29095, 29094, - 29099, 29098, 29097, 29096, 29147, 29148, 40951, 40951, 40951, 40951, - 40951, 40951, 29173, 29175, 29172, 29171, 29168, 29167, 29170, 29169, - 29176, 29174, 29149, 40951, 40951, 40951, 29145, 29146, 22305, 22324, - 22317, 22320, 22322, 22315, 22313, 22311, 22307, 22309, 22294, 22292, - 22284, 22288, 22290, 22286, 22318, 22323, 22316, 22319, 22321, 22314, - 22312, 22310, 22306, 22308, 22293, 22291, 22283, 22287, 22289, 22285, - 4959, 4956, 4948, 4947, 4961, 4953, 4946, 4945, 4964, 4955, 4952, 4951, - 4954, 4958, 4950, 4949, 4966, 4962, 4960, 4965, 4963, 4967, 4957, 4970, - 4972, 4969, 4971, 4968, 40951, 40951, 4974, 4973, 35134, 35132, 35133, - 35150, 35149, 35148, 35174, 35140, 35139, 35153, 35160, 35152, 35175, - 35168, 35135, 35180, 35151, 35167, 35144, 35143, 35157, 35156, 35176, - 35179, 35142, 35141, 35145, 35155, 35158, 35154, 35181, 35161, 35147, - 35166, 35169, 35162, 35164, 35182, 35136, 35137, 35138, 35146, 35165, - 35183, 35159, 35172, 35173, 35170, 35171, 35178, 35177, 35163, 35131, - 35106, 35105, 35103, 35194, 35101, 35102, 35104, 35107, 35108, 35109, - 40951, 35202, 35209, 35213, 35210, 35219, 35225, 35226, 35224, 35223, - 35221, 35222, 35214, 35215, 35218, 35227, 35211, 35217, 35212, 35220, - 35216, 35193, 35206, 35207, 35186, 35187, 35188, 35197, 35196, 35189, - 40951, 40951, 35110, 35117, 35119, 35116, 35115, 35112, 35111, 35114, - 35113, 35120, 35118, 40951, 40951, 40951, 40951, 40951, 40951, 35127, - 35129, 35126, 35125, 35122, 35121, 35124, 35123, 35130, 35128, 40951, - 40951, 40951, 40951, 40951, 40951, 35203, 35204, 35201, 35192, 35185, - 35205, 35198, 35195, 35190, 35191, 35199, 35200, 35184, 35208, 40951, - 40951, 8201, 8169, 8285, 8203, 8435, 8448, 8447, 8387, 8184, 8364, 8423, - 8393, 8188, 8392, 8391, 8333, 8327, 8350, 8409, 8351, 8410, 8421, 8380, - 8284, 8396, 8187, 8186, 8433, 8311, 8312, 8313, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 2717, 2716, 2718, 2715, 2719, - 2626, 2627, 2643, 2644, 2647, 2648, 2665, 2666, 2656, 2657, 2640, 2630, - 2645, 2646, 2651, 2653, 2641, 2642, 2660, 2633, 2634, 2649, 2650, 2661, - 2672, 2671, 2637, 2636, 2659, 2670, 2673, 2635, 2638, 2658, 2662, 2663, - 2631, 2632, 2678, 2680, 2664, 2655, 2679, 2668, 2669, 2667, 2677, 2720, - 2731, 2734, 2735, 2725, 2726, 2723, 2724, 2721, 2722, 2727, 2728, 2730, - 2729, 2732, 2733, 2736, 2652, 2654, 2674, 2639, 2675, 2676, 2628, 2629, - 40951, 2625, 2624, 2744, 2746, 2743, 2742, 2739, 2738, 2741, 2740, 2747, - 2745, 2712, 2709, 2737, 2622, 2623, 2621, 2711, 2698, 2696, 2699, 2690, - 2691, 2697, 2693, 2695, 2694, 2692, 2688, 2687, 2683, 2681, 2685, 2684, - 2682, 2686, 2689, 2708, 2707, 2706, 2705, 2700, 2702, 2703, 2704, 2701, - 2713, 2710, 2714, 34633, 34631, 34632, 34584, 34619, 34621, 34586, 34620, - 34596, 34597, 34604, 34612, 34607, 34598, 34605, 34609, 34618, 34599, - 34613, 34606, 34600, 34611, 34589, 34614, 34602, 34610, 34617, 34593, - 34591, 34615, 34595, 34616, 34608, 34579, 34580, 34581, 34639, 34638, - 34640, 34637, 34635, 34636, 34630, 34634, 34582, 34583, 34603, 34594, - 34647, 34649, 34646, 34645, 34642, 34641, 34644, 34643, 34650, 34648, - 34578, 34592, 34590, 34601, 34587, 34588, 3508, 3494, 3502, 3486, 3474, - 3498, 3497, 3483, 3489, 3482, 3475, 3506, 3492, 3484, 3501, 3485, 3503, - 3500, 3505, 3490, 3473, 3488, 3495, 3478, 3496, 3491, 3476, 3507, 3493, - 3480, 3504, 3487, 3481, 3499, 3479, 3477, 3509, 3510, 3517, 3523, 3520, - 3524, 3525, 3521, 3526, 3522, 3518, 3519, 3471, 3472, 3511, 3512, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3516, 3514, 3515, 3513, - 24308, 24307, 24306, 24320, 24319, 24325, 24335, 24334, 24338, 24326, - 24333, 24332, 24314, 24327, 24311, 24310, 24309, 24318, 24317, 24316, - 24315, 24324, 24323, 24329, 24328, 24313, 24343, 24340, 24339, 24322, - 24321, 24341, 24337, 24336, 24342, 24344, 24353, 24352, 24358, 24360, - 24356, 24357, 24354, 24355, 24359, 24297, 24302, 24301, 24299, 24303, - 24304, 24305, 24300, 24298, 24351, 24350, 40951, 40951, 40951, 24345, - 24348, 24349, 24347, 24346, 24367, 24369, 24366, 24365, 24362, 24361, - 24364, 24363, 24370, 24368, 40951, 40951, 40951, 24331, 24330, 24312, - 30023, 30025, 30022, 30021, 30018, 30017, 30020, 30019, 30026, 30024, - 29996, 29987, 29985, 29984, 29986, 29997, 29981, 29980, 29982, 29983, - 29999, 29995, 29993, 29992, 29994, 30001, 30007, 30008, 30006, 30009, - 29998, 29991, 29988, 29990, 29989, 30000, 30002, 30003, 30005, 30004, - 30011, 30012, 30010, 30016, 30015, 30027, 30014, 30013, 10430, 10407, - 10413, 10470, 10451, 10459, 10449, 10450, 10469, 10135, 10461, 40951, - 40951, 40951, 40951, 40951, 17856, 17887, 17864, 17893, 17862, 17892, - 17885, 17882, 17894, 17874, 17876, 17888, 17890, 17895, 17878, 17884, - 17886, 17880, 17883, 17896, 17877, 17873, 17863, 17891, 17879, 17857, - 17860, 17872, 17859, 17858, 17889, 17871, 17867, 17870, 17868, 17898, - 17865, 17869, 17899, 17897, 17861, 17881, 17855, 40951, 40951, 17854, - 17866, 17875, 34629, 34627, 34628, 34626, 34625, 34624, 34623, 34622, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 38803, 38816, - 38805, 38783, 38797, 38811, 38812, 38813, 38798, 38814, 38801, 38809, - 38804, 38802, 38810, 38808, 38807, 38815, 38796, 38794, 38785, 38792, - 38784, 38795, 38793, 38774, 38775, 38777, 38778, 38790, 38788, 38789, - 38787, 38776, 38780, 38786, 38799, 38782, 38791, 38779, 38806, 38800, - 38781, 40951, 40951, 40951, 40951, 40951, 23270, 23271, 23878, 23267, - 23272, 23273, 23244, 23243, 23863, 23856, 23276, 23277, 23250, 23278, - 23256, 23251, 23252, 23813, 23814, 23815, 23858, 23254, 23850, 23370, - 23280, 23257, 23265, 23260, 23283, 23818, 23817, 23816, 23284, 23285, - 23287, 23245, 23298, 23232, 18400, 18403, 18399, 18402, 18398, 10300, - 27701, 27702, 27692, 27693, 27704, 27705, 27695, 27707, 27697, 27708, - 27709, 27710, 27711, 27712, 27713, 27696, 27699, 27700, 27714, 27694, - 27716, 27717, 27719, 27850, 27957, 27851, 27959, 27854, 27879, 27893, - 27947, 27932, 27962, 27900, 27970, 27981, 27910, 27897, 27930, 27933, - 27954, 27856, 27934, 27949, 27974, 27948, 27960, 27978, 27852, 27858, - 27901, 27884, 27902, 27878, 24019, 24027, 24029, 24030, 18609, 18605, - 18608, 18607, 18606, 23939, 23356, 23405, 23491, 23634, 23654, 23731, - 23758, 23751, 23797, 23833, 24001, 23885, 27768, 23562, 23843, 23300, - 23569, 23727, 23301, 23938, 23357, 23409, 23492, 23505, 23588, 23615, - 23635, 23660, 23732, 23761, 23798, 23478, 23946, 23973, 24002, 23319, - 23345, 23408, 23468, 23720, 23768, 23806, 23559, 23716, 23480, 23925, - 23486, 27958, 27859, 27877, 27895, 27941, 27898, 27885, 27946, 27969, - 27912, 27913, 27860, 27862, 27915, 27919, 27921, 27865, 27911, 27961, - 27929, 27928, 27871, 27855, 27935, 27945, 27894, 27950, 27976, 27977, - 27873, 27980, 27971, 27890, 27892, 27891, 27896, 27973, 8177, 8176, 8425, - 8424, 8377, 8268, 8376, 8061, 8267, 8060, 8325, 8073, 8378, 8200, 8389, - 8416, 8286, 8440, 8441, 8308, 8299, 8300, 8301, 8303, 8310, 8306, 8335, - 8291, 8337, 8314, 8292, 8293, 8339, 8294, 8295, 8324, 8328, 8316, 8343, - 8298, 8330, 8331, 8329, 8307, 8315, 8319, 8340, 8305, 8322, 8332, 8297, - 8318, 8321, 8439, 8290, 8289, 8172, 8445, 8175, 8167, 8181, 8070, 8345, - 8400, 22756, 23317, 22792, 23359, 22791, 23358, 22790, 23355, 22799, - 23374, 22824, 23411, 22823, 23410, 22822, 23406, 22818, 23400, 22819, - 23401, 22850, 23454, 22851, 23455, 22839, 23442, 22848, 23452, 22830, - 23433, 22875, 23494, 22883, 23504, 22902, 23524, 22901, 23523, 22899, - 23521, 22896, 23518, 22895, 23517, 22919, 23551, 22913, 23539, 22950, - 23586, 22947, 23583, 22951, 23587, 22958, 23600, 22959, 23601, 22969, - 23610, 22965, 23597, 22975, 23632, 22977, 23637, 22976, 23636, 22995, - 23659, 22994, 23658, 22989, 23651, 22985, 23646, 23026, 23695, 23025, - 23694, 23003, 23687, 23004, 23688, 23054, 23730, 23056, 23734, 23065, - 23748, 23063, 23746, 23064, 23747, 23070, 23753, 23091, 23791, 23089, - 23789, 23088, 23782, 23083, 23784, 23090, 23790, 23112, 23831, 23111, - 23830, 23113, 23834, 23107, 23824, 23143, 23906, 23164, 23929, 23136, - 23899, 23163, 23928, 23155, 23918, 23176, 23949, 23175, 23945, 23189, - 23962, 23190, 23963, 23186, 23959, 23188, 23961, 23187, 23960, 23195, - 23968, 23194, 23967, 23200, 23978, 23211, 23992, 23215, 23996, 23217, - 23998, 23525, 23829, 23964, 23988, 23318, 23625, 23624, 23626, 23101, - 23415, 22750, 23311, 22766, 23329, 22762, 23325, 22761, 23324, 22760, - 23323, 22764, 23327, 22763, 23326, 22745, 23306, 22744, 23305, 22743, - 23304, 22747, 23308, 22746, 23307, 22844, 23447, 22855, 23460, 22847, - 23451, 22835, 23438, 22834, 23437, 22833, 23436, 22838, 23441, 22837, - 23440, 22920, 23552, 22910, 23543, 23017, 23680, 23037, 23706, 23009, - 23672, 23008, 23671, 23007, 23670, 23011, 23674, 23010, 23673, 23034, - 23703, 23033, 23702, 23032, 23701, 23036, 23705, 23035, 23704, 23146, - 23909, 23153, 23916, 23150, 23913, 23149, 23912, 23148, 23911, 23152, - 23915, 23151, 23914, 23201, 23979, 23199, 23977, 23203, 23981, 23207, - 23987, 22980, 23640, 22981, 23641, 23204, 23982, 18447, 18439, 18452, - 18444, 18448, 18440, 18450, 18442, 18213, 18205, 18216, 18208, 18214, - 18206, 18218, 18210, 18470, 18467, 18472, 18469, 18471, 18468, 40951, - 40951, 18253, 18250, 18255, 18252, 18254, 18251, 40951, 40951, 18485, - 18477, 18490, 18482, 18486, 18478, 18488, 18480, 18237, 18229, 18240, - 18232, 18238, 18230, 18242, 18234, 18511, 18502, 18514, 18505, 18513, - 18504, 18512, 18503, 18265, 18260, 18268, 18263, 18267, 18262, 18266, - 18261, 18572, 18569, 18574, 18571, 18573, 18570, 40951, 40951, 18299, - 18296, 18301, 18298, 18300, 18297, 40951, 40951, 18531, 18522, 18534, - 18525, 18533, 18524, 18532, 18523, 40951, 18311, 40951, 18314, 40951, - 18313, 40951, 18312, 18552, 18544, 18557, 18549, 18553, 18545, 18555, - 18547, 18283, 18275, 18286, 18278, 18284, 18276, 18288, 18280, 18437, - 18457, 18474, 18473, 18497, 18495, 18517, 18518, 18576, 18575, 18537, - 18538, 18564, 18562, 40951, 40951, 18454, 18446, 18453, 18445, 18449, - 18441, 18451, 18443, 18220, 18212, 18217, 18209, 18215, 18207, 18219, - 18211, 18492, 18484, 18491, 18483, 18487, 18479, 18489, 18481, 18244, - 18236, 18241, 18233, 18239, 18231, 18243, 18235, 18559, 18551, 18558, - 18550, 18554, 18546, 18556, 18548, 18290, 18282, 18287, 18279, 18285, - 18277, 18289, 18281, 18436, 18461, 18438, 18459, 18458, 40951, 18455, - 18456, 18222, 18226, 18223, 18224, 18221, 18396, 18424, 18425, 18429, - 18345, 18498, 18499, 18496, 40951, 18493, 18494, 18257, 18256, 18247, - 18246, 18245, 18428, 18427, 18426, 18516, 18520, 18509, 18508, 40951, - 40951, 18515, 18507, 18269, 18273, 18270, 18271, 40951, 18353, 18352, - 18351, 18536, 18540, 18529, 18528, 18584, 18583, 18535, 18527, 18316, - 18320, 18317, 18318, 18306, 18347, 18346, 18623, 40951, 40951, 18565, - 18566, 18563, 40951, 18560, 18561, 18303, 18302, 18293, 18292, 18291, - 18421, 18350, 40951, 16764, 16761, 16766, 16763, 37019, 17510, 33761, - 17439, 31718, 36992, 18978, 40757, 40756, 40758, 24189, 32047, 20456, - 29361, 17438, 16765, 16762, 20402, 11250, 11224, 24112, 31977, 33636, - 33634, 24065, 31942, 11223, 11218, 10529, 11217, 4995, 37548, 30526, - 37694, 20425, 20459, 24496, 31062, 24188, 32046, 31593, 24187, 32045, - 28962, 31296, 31297, 31679, 11232, 37562, 31885, 31879, 31893, 6007, - 33635, 33637, 31902, 11254, 20799, 30878, 37745, 6293, 6008, 2529, 20458, - 17514, 24120, 31988, 11255, 31742, 17353, 37393, 31884, 3854, 3896, - 25158, 31890, 8045, 37706, 8451, 34742, 20817, 17476, 36999, 31738, - 17503, 17463, 37695, 17504, 11196, 37573, 38824, 26974, 39366, 17636, - 20819, 20821, 20820, 40951, 24186, 32044, 17456, 31592, 20713, 7, 20712, - 6, 28957, 29359, 34713, 34701, 40951, 40951, 34708, 34707, 34710, 34709, - 34700, 34714, 34704, 34706, 34699, 34703, 34705, 34702, 34543, 34545, - 34542, 34541, 34538, 34537, 34540, 34539, 34533, 34544, 34534, 34536, - 34532, 34531, 34535, 40951, 24016, 24017, 24025, 24031, 24015, 24018, - 24021, 24022, 24023, 24024, 24026, 24014, 24028, 40951, 40951, 40951, - 17350, 8058, 8697, 17520, 25104, 27586, 28890, 31320, 32367, 39370, - 29093, 11190, 17351, 22555, 37582, 11372, 17901, 31321, 18673, 2540, - 20465, 6126, 25105, 34096, 36765, 20703, 37664, 29411, 25663, 32236, - 22737, 3762, 34076, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8354, 8408, 8368, - 8420, 8066, 8082, 8347, 8405, 8413, 8081, 8065, 8427, 8213, 8204, 8207, - 8210, 8205, 8357, 8206, 8208, 8209, 8397, 8194, 8067, 8434, 8446, 8359, - 8365, 8412, 8358, 8346, 8404, 8069, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 0, - 100, 11266, 10553, 6130, 6009, 5260, 17352, 32555, 10555, 32552, 32544, - 3944, 11267, 31482, 31483, 32545, 3945, 32546, 32553, 25335, 11268, - 29454, 33992, 32548, 11264, 11269, 32549, 3946, 11270, 31655, 31863, - 32658, 36660, 37535, 38817, 11271, 30872, 30880, 20816, 3947, 37687, - 21610, 961, 32541, 3943, 16820, 32551, 32542, 32543, 37671, 32547, 32554, - 346, 3653, 17904, 10542, 20710, 32224, 17418, 11278, 11277, 11263, 11265, - 11279, 37680, 37681, 31889, 37682, 11276, 11272, 11273, 11274, 11275, - 31683, 37666, 31298, 2600, 37684, 34821, 38966, 38964, 38968, 38969, - 38974, 38962, 38973, 38972, 38960, 38967, 38959, 38961, 38970, 38958, - 38975, 17515, 32192, 32203, 32204, 32191, 32188, 32197, 32199, 32207, - 32208, 32200, 32206, 32202, 32185, 32193, 32189, 32195, 33823, 33827, - 33828, 33822, 33819, 33831, 33830, 33818, 33832, 33829, 33817, 33826, - 33821, 33824, 33820, 33825, 32196, 32190, 32201, 32205, 23774, 32198, - 32187, 32186, 32194, 38976, 37675, 37674, 40951, 40951, 40951, 40951, - 24190, 37896, 32049, 11308, 24094, 37767, 29392, 29365, 33964, 33979, - 24210, 32073, 24274, 32150, 24271, 37946, 32149, 11344, 24213, 32076, - 24221, 37909, 32056, 11322, 37768, 24219, 32082, 24204, 32068, 24102, - 24097, 11348, 37906, 37907, 11317, 11318, 32064, 11309, 972, 8029, 29394, - 24201, 977, 8031, 24240, 24234, 37923, 37920, 32110, 32104, 11359, 11356, - 32084, 37898, 24224, 24286, 37949, 32116, 11367, 24241, 32099, 24277, - 24101, 32092, 24275, 37913, 32090, 11349, 24099, 37770, 29405, 29378, - 33976, 33989, 24261, 32140, 24290, 32120, 37899, 11310, 24281, 37914, - 32096, 11350, 24200, 32063, 24272, 37950, 32151, 11346, 37955, 37951, - 37953, 37952, 37957, 37958, 32152, 29393, 33967, 37772, 31929, 11320, - 37007, 24220, 32083, 24096, 24206, 32066, 24095, 24285, 32115, 24104, - 17498, 8456, 31208, 36987, 36986, 16754, 20631, 28846, 16703, 29414, - 33803, 8464, 11016, 33796, 16768, 28805, 28793, 28797, 27590, 27597, - 11191, 10998, 32663, 2528, 32161, 4996, 34323, 8703, 17509, 31682, 20704, - 31919, 957, 26848, 34099, 11002, 11017, 31067, 29419, 25119, 25126, - 20792, 37747, 20777, 11221, 37572, 8470, 34735, 38954, 8032, 8023, 974, - 36988, 3654, 31773, 31681, 11192, 17355, 17726, 20446, 37297, 31891, - 20813, 33756, 39374, 29424, 27596, 2533, 29415, 1054, 1057, 29049, 362, - 29416, 364, 37563, 359, 16806, 17724, 11008, 1058, 17725, 1055, 20594, - 8057, 16804, 32159, 32160, 8649, 16821, 16807, 34488, 10558, 16790, - 26859, 31744, 29426, 20470, 29427, 34524, 24390, 18137, 24392, 18140, - 24381, 18130, 28542, 28541, 3652, 29425, 29429, 29428, 29054, 29051, - 24389, 18136, 29053, 29050, 24391, 18138, 29055, 29052, 31656, 34561, - 31665, 34570, 31664, 34569, 11019, 11022, 34549, 34721, 29412, 29413, - 34555, 34727, 29047, 29048, 34554, 34726, 28295, 28296, 28297, 34196, - 34285, 34198, 34287, 34131, 34138, 6804, 6750, 6809, 6542, 6555, 6805, - 6531, 6814, 6556, 34456, 34449, 34470, 34404, 32006, 24138, 11285, 37776, - 2534, 27605, 37579, 17499, 37566, 11248, 11021, 29423, 11024, 29046, - 31666, 34571, 29409, 8465, 29410, 8466, 30556, 20593, 28298, 20217, - 20800, 39395, 28891, 29364, 31925, 32002, 28802, 28803, 28804, 28798, - 10837, 11193, 34490, 11000, 4372, 24152, 31964, 24110, 31975, 31892, - 9946, 9945, 11242, 11241, 11220, 11245, 31476, 16791, 24395, 18146, - 38867, 38866, 24379, 18141, 16789, 16788, 16786, 16787, 11020, 11023, - 29420, 29421, 34197, 34286, 24380, 18129, 31663, 34568, 29417, 11014, - 29418, 11015, 38823, 27582, 37775, 11288, 16704, 16706, 33804, 16709, - 16708, 33805, 16707, 16705, 8467, 8468, 33797, 8469, 33798, 40669, 10838, - 16701, 20440, 37759, 11289, 31680, 31315, 39140, 24061, 31937, 24149, - 31946, 4355, 4352, 37483, 37479, 31882, 34229, 2524, 32565, 32561, 36658, - 31601, 38878, 31479, 37678, 39131, 20438, 37478, 37482, 4351, 4354, - 37472, 4349, 17524, 33863, 37760, 30548, 16817, 39379, 21612, 24159, - 32027, 16816, 3650, 10524, 360, 34831, 37499, 11009, 8475, 33788, 8651, - 8652, 1002, 1040, 1026, 1014, 1015, 1031, 1012, 982, 987, 1037, 1042, - 1028, 1027, 1022, 1029, 1010, 1034, 1023, 1030, 985, 994, 993, 1016, - 1019, 995, 1048, 1021, 1045, 991, 1020, 1018, 1046, 998, 1017, 1033, 992, - 999, 1008, 986, 1047, 1032, 983, 1013, 1044, 989, 1038, 1007, 984, 996, - 1009, 1050, 1049, 988, 990, 1051, 1039, 1036, 1025, 1024, 997, 1043, - 1000, 1035, 1005, 1004, 1041, 1001, 1006, 1003, 29430, 31924, 32844, - 3546, 38839, 20776, 8473, 10924, 16760, 8455, 39283, 16782, 365, 19928, - 6586, 6808, 4936, 37746, 28178, 20462, 30544, 30545, 31220, 31221, 10919, - 33880, 1011, 10549, 31667, 29278, 31669, 8052, 24155, 24156, 24154, - 31971, 31972, 31970, 24117, 24124, 24116, 31985, 31992, 31984, 24059, - 24057, 24058, 9948, 31935, 31933, 31934, 20787, 20406, 37828, 37867, - 34574, 34572, 37486, 4358, 4359, 31753, 24158, 32008, 20415, 20416, - 20417, 20418, 10571, 10569, 10573, 10562, 10566, 10574, 10563, 10567, - 10572, 10561, 10565, 10560, 10564, 10570, 10568, 34164, 31864, 17382, - 38834, 27419, 27411, 27418, 27412, 27416, 27417, 27415, 27414, 27413, - 11539, 17640, 37474, 4362, 37456, 4361, 37487, 4365, 39292, 3651, 34516, - 17468, 8, 16702, 10550, 3885, 3847, 3928, 3824, 3886, 3848, 3888, 367, - 34514, 37305, 20439, 3864, 3867, 3869, 3861, 11246, 3907, 3771, 31615, - 31612, 31613, 31614, 29932, 34816, 34824, 34825, 34805, 34803, 34806, - 34817, 34788, 34789, 34810, 34812, 34811, 34809, 34790, 34822, 34823, - 34792, 34801, 34800, 34799, 34798, 34814, 34828, 34804, 34791, 34802, - 34826, 34807, 34808, 34819, 34818, 34820, 34829, 34793, 3951, 30531, - 34815, 34796, 34827, 34795, 34794, 34797, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 29944, 29939, - 29942, 29943, 29936, 29937, 29935, 29934, 29941, 29938, 29940, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 6552, 6549, 6548, 6545, 6544, 6547, 6546, 6553, 6551, 6773, 6753, 6798, - 6786, 6768, 6756, 6772, 6770, 6752, 6799, 6787, 31154, 31152, 31151, - 31148, 31147, 31150, 31149, 31155, 31153, 31146, 31137, 31145, 31143, - 31138, 31139, 31141, 31142, 31136, 31140, 31144, 10859, 10871, 10868, - 10853, 10850, 10865, 10862, 10877, 10856, 29450, 29443, 29451, 29449, - 29444, 29445, 29446, 29448, 29442, 29453, 29452, 31182, 31183, 31184, - 31185, 31186, 31187, 31188, 31189, 31190, 31191, 31192, 31193, 31194, - 31195, 31196, 31197, 31198, 31199, 31200, 31201, 31202, 31203, 31204, - 31205, 31206, 31207, 6696, 6697, 6698, 6699, 6700, 6701, 6702, 6703, - 6704, 6705, 6706, 6707, 6708, 6709, 6710, 6711, 6712, 6713, 6714, 6715, - 6716, 6717, 6718, 6719, 6720, 6721, 6722, 6723, 6724, 6725, 6726, 6727, - 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, 6737, 6738, 6739, - 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6550, 28971, 28969, - 28967, 28972, 28973, 28975, 28976, 28970, 28974, 28968, 11212, 11210, - 11209, 11206, 11205, 11208, 11207, 11213, 11211, 11204, 28966, 4484, - 4430, 4500, 4416, 4493, 4429, 4492, 4428, 4491, 4427, 4490, 4426, 4481, - 4400, 4394, 4423, 4480, 4398, 4392, 4422, 4499, 4528, 4522, 4415, 4498, - 4526, 4520, 4413, 4507, 4544, 4521, 4393, 4541, 4399, 4527, 4419, 4506, - 4543, 4519, 4391, 4540, 4397, 4525, 4418, 4479, 4434, 4512, 4402, 4396, - 4515, 4437, 4421, 4497, 4433, 4511, 4529, 4523, 4514, 4436, 4414, 4505, - 4435, 4513, 4542, 4518, 4395, 4539, 4439, 4517, 4432, 4510, 4401, 4524, - 4516, 4438, 4417, 4483, 4425, 4482, 4424, 4390, 4386, 4407, 4404, 4382, - 4406, 4403, 4381, 4534, 4531, 4385, 4533, 4530, 4384, 4546, 4537, 4389, - 4545, 4536, 4388, 4408, 4405, 4380, 4535, 4532, 4383, 4547, 4538, 4387, - 4441, 4440, 4442, 4443, 4472, 4466, 4476, 4488, 4495, 4509, 4478, 4409, - 4411, 4431, 4420, 4489, 4496, 4410, 4412, 37805, 25259, 25258, 25261, - 25169, 25251, 25264, 25260, 17531, 24111, 24134, 24147, 24075, 24135, - 24090, 24091, 31950, 24410, 26976, 10536, 37853, 31967, 31735, 31736, - 31727, 31728, 31729, 31730, 31731, 31732, 31733, 31734, 3905, 39270, - 39279, 39273, 34342, 34351, 34341, 34348, 34350, 34340, 3902, 39267, - 3898, 39255, 3935, 39302, 3879, 39251, 3930, 39299, 3929, 39298, 3887, - 39259, 3892, 39258, 3891, 39257, 3826, 39214, 3825, 39213, 3852, 39240, - 3851, 39239, 3850, 39238, 3816, 39203, 39204, 17460, 25266, 39189, 11195, - 6525, 4999, 3767, 6517, 6526, 6518, 6523, 6522, 6524, 24073, 31948, - 20804, 20808, 37809, 25163, 37833, 37869, 25213, 25193, 37814, 25170, - 3859, 3858, 3931, 3932, 39155, 34345, 34352, 34347, 34344, 39282, 39300, - 37791, 37792, 22734, 39280, 39276, 39277, 39281, 39196, 39194, 39195, - 39197, 37831, 37886, 25204, 39247, 3874, 39246, 3873, 25227, 3909, 8046, - 37740, 33870, 8458, 3918, 39289, 24420, 37024, 34575, 2525, 10578, 8476, - 30552, 3924, 39291, 2750, 2756, 2757, 32374, 37742, 20434, 39264, 3916, - 32818, 31887, 3857, 3884, 39237, 39296, 39261, 39212, 33767, 6123, 31751, - 3764, 5259, 981, 30652, 6480, 8684, 8683, 34491, 17428, 102, 19104, - 31286, 40667, 37551, 37552, 37557, 37554, 37556, 37555, 37553, 37550, - 39151, 39224, 39268, 3904, 39287, 17453, 22738, 27408, 17434, 11535, - 25571, 20931, 32444, 37959, 29280, 31581, 2523, 36647, 17723, 5997, - 24295, 38880, 25108, 32537, 32372, 6003, 2601, 31474, 39161, 39177, - 39180, 39156, 39164, 39174, 3787, 3803, 3806, 3782, 3790, 3800, 3917, - 39227, 39208, 3815, 39269, 3836, 3821, 39198, 20435, 31740, 16657, 3532, - 3533, 28302, 28301, 28303, 39149, 11540, 37756, 31781, 31782, 31783, - 31784, 31785, 31786, 31787, 31788, 3934, 31780, 31213, 31318, 39152, - 10841, 10842, 10843, 10844, 10845, 10846, 39191, 39193, 3768, 3770, - 28175, 28176, 10881, 10884, 10883, 10882, 39218, 3832, 19105, 979, 8692, - 34485, 32532, 345, 17475, 17719, 34486, 2536, 17473, 30859, 37001, 37000, - 39125, 20285, 11281, 11282, 20791, 25570, 25569, 25568, 38840, 20426, - 26985, 26962, 26975, 25737, 11003, 37758, 8675, 17637, 29090, 6133, - 31022, 20932, 38870, 6483, 3875, 32665, 32577, 31746, 32660, 33875, 3469, - 34418, 39215, 39216, 3829, 3830, 33871, 34577, 31756, 3911, 37023, 37670, - 37669, 39209, 8693, 10923, 30551, 31459, 6065, 19927, 6538, 6134, 29352, - 366, 3925, 39294, 3855, 39235, 11381, 19785, 24062, 34461, 17422, 3922, - 31860, 31861, 2532, 19711, 31293, 32026, 24185, 20814, 3780, 32823, 6515, - 6125, 20390, 17720, 17721, 25666, 28194, 37741, 17511, 17470, 17436, - 32528, 34163, 33765, 20469, 31305, 36766, 20829, 19687, 17639, 9941, - 39219, 3912, 37798, 3915, 25245, 39262, 39228, 36659, 36646, 226, 16779, - 31769, 31761, 38872, 39377, 25244, 31295, 37868, 39250, 3877, 6299, - 19709, 28294, 19745, 2760, 19701, 30861, 19792, 30535, 19747, 23224, - 32670, 30858, 25572, 34489, 17507, 17505, 19730, 17501, 3833, 39223, - 34084, 34518, 6811, 30533, 3781, 30860, 19749, 31471, 32669, 19700, - 30534, 16656, 16651, 16649, 33759, 16650, 19723, 37690, 33762, 36652, - 30532, 19775, 33757, 3831, 39220, 16652, 6800, 19774, 33873, 37295, - 19708, 34083, 19768, 2749, 16655, 19725, 8689, 32668, 29038, 25243, - 37865, 25225, 37883, 3942, 39254, 39217, 3818, 19727, 24417, 26982, - 19791, 19758, 19759, 19719, 19720, 19742, 19741, 9953, 19728, 19734, - 19704, 32221, 17474, 32220, 26969, 26972, 26963, 26964, 26970, 26973, - 19738, 19751, 19737, 19750, 24409, 24407, 26968, 26971, 10906, 10904, - 10903, 10900, 10899, 10902, 10901, 10907, 10905, 10898, 10917, 10914, - 10913, 10910, 10909, 10912, 10911, 10918, 10916, 10908, 10896, 10893, - 10892, 10889, 10888, 10891, 10890, 10897, 10895, 10887, 19787, 19790, - 19746, 19716, 19764, 19752, 19771, 11373, 19755, 37545, 19777, 10539, - 19715, 3893, 37015, 37018, 3894, 19702, 19703, 34479, 19714, 32042, - 24175, 2613, 17522, 19743, 19782, 29432, 9947, 29434, 6588, 39306, 3948, - 3950, 3949, 19705, 19707, 19706, 36653, 19776, 39145, 19788, 30546, - 11214, 36998, 39293, 31299, 30542, 30541, 24109, 31974, 30654, 31871, - 34732, 38821, 26434, 25134, 26255, 34446, 34447, 39207, 980, 16711, - 25241, 37826, 24093, 31965, 17530, 22726, 22725, 24040, 24039, 24089, - 25149, 25138, 37777, 25267, 39199, 39200, 39201, 39275, 39278, 26435, - 26429, 26438, 26432, 26437, 26431, 26436, 26430, 26439, 26433, 37927, - 11363, 976, 8025, 31928, 25139, 25144, 25137, 25141, 25146, 25136, 25140, - 25145, 25142, 25147, 25148, 4925, 4670, 4798, 4671, 4862, 4735, 4799, - 4672, 4894, 4767, 4831, 4704, 4863, 4736, 4800, 4673, 4910, 4783, 4847, - 4720, 4879, 4752, 4816, 4689, 4895, 4768, 4832, 4705, 4864, 4737, 4801, - 4674, 4918, 4791, 4855, 4728, 4887, 4760, 4824, 4697, 4903, 4776, 4840, - 4713, 4872, 4745, 4809, 4682, 4911, 4784, 4848, 4721, 4880, 4753, 4817, - 4690, 4896, 4769, 4833, 4706, 4865, 4738, 4802, 4675, 4922, 4795, 4859, - 4732, 4891, 4764, 4828, 4701, 4907, 4780, 4844, 4717, 4876, 4749, 4813, - 4686, 4915, 4788, 4852, 4725, 4884, 4757, 4821, 4694, 4900, 4773, 4837, - 4710, 4869, 4742, 4806, 4679, 4919, 4792, 4856, 4729, 4888, 4761, 4825, - 4698, 4904, 4777, 4841, 4714, 4873, 4746, 4810, 4683, 4912, 4785, 4849, - 4722, 4881, 4754, 4818, 4691, 4897, 4770, 4834, 4707, 4866, 4739, 4803, - 4676, 4924, 4797, 4861, 4734, 4893, 4766, 4830, 4703, 4909, 4782, 4846, - 4719, 4878, 4751, 4815, 4688, 4917, 4790, 4854, 4727, 4886, 4759, 4823, - 4696, 4902, 4775, 4839, 4712, 4871, 4744, 4808, 4681, 4921, 4794, 4858, - 4731, 4890, 4763, 4827, 4700, 4906, 4779, 4843, 4716, 4875, 4748, 4812, - 4685, 4914, 4787, 4851, 4724, 4883, 4756, 4820, 4693, 4899, 4772, 4836, - 4709, 4868, 4741, 4805, 4678, 4923, 4796, 4860, 4733, 4892, 4765, 4829, - 4702, 4908, 4781, 4845, 4718, 4877, 4750, 4814, 4687, 4916, 4789, 4853, - 4726, 4885, 4758, 4822, 4695, 4901, 4774, 4838, 4711, 4870, 4743, 4807, - 4680, 4920, 4793, 4857, 4730, 4889, 4762, 4826, 4699, 4905, 4778, 4842, - 4715, 4874, 4747, 4811, 4684, 4913, 4786, 4850, 4723, 4882, 4755, 4819, - 4692, 4898, 4771, 4835, 4708, 4867, 4740, 4804, 4677, 32146, 32145, - 24276, 32091, 24100, 32147, 24278, 32093, 11319, 37908, 37945, 11343, - 24280, 32095, 24260, 32139, 32148, 32065, 37910, 11323, 32078, 32077, - 32141, 32143, 32142, 24225, 32085, 24279, 32094, 24202, 32062, 24222, - 32057, 29391, 29371, 29397, 29369, 33968, 33981, 29396, 29368, 33965, - 33980, 32165, 17420, 33966, 29366, 17421, 32166, 29367, 29395, 39134, - 2516, 2515, 2518, 2519, 32043, 24174, 37453, 4360, 37455, 37454, 25223, - 25187, 973, 8022, 32052, 24191, 32831, 32070, 24207, 32061, 24098, 37948, - 24052, 24051, 37765, 37764, 24053, 37766, 24050, 37763, 24239, 32109, - 37922, 11358, 24233, 32103, 37919, 11355, 24238, 32108, 37921, 11357, - 24232, 32102, 37918, 11354, 24235, 37917, 32107, 11352, 24237, 24231, - 32105, 32100, 24236, 24230, 32106, 32101, 37916, 11353, 31940, 16796, - 37299, 24195, 32054, 32053, 24376, 24198, 18125, 34548, 24197, 34718, - 24148, 31945, 37774, 11287, 37565, 40680, 40681, 24140, 32011, 24142, - 32013, 40675, 40671, 40676, 40672, 24121, 31989, 24119, 31986, 24118, - 31987, 24046, 31921, 24047, 31927, 11227, 11252, 24055, 31931, 11202, - 38854, 26857, 31926, 26858, 958, 5, 34100, 34101, 37667, 31877, 959, - 31878, 29930, 29929, 26856, 26855, 26850, 26849, 26854, 26852, 26853, - 26851, 31900, 16758, 16757, 16755, 16756, 6527, 6815, 6802, 6806, 6803, - 6528, 6520, 6529, 37762, 6810, 6533, 6748, 6816, 6519, 6521, 34410, - 34405, 34476, 34462, 34463, 37700, 37544, 37542, 32369, 37541, 32003, - 24126, 38818, 4373, 4374, 3941, 37306, 37307, 39232, 3840, 24145, 32016, - 24063, 31941, 20628, 37233, 20705, 11280, 34353, 20630, 32851, 16797, - 16798, 20471, 18009, 36990, 11300, 11301, 3819, 3860, 39192, 3769, 16810, - 16813, 16809, 16812, 16808, 16811, 32238, 31872, 33927, 31873, 3757, - 3756, 11234, 37561, 24162, 32029, 37308, 27598, 28792, 28790, 28791, - 28800, 28799, 28795, 28796, 37702, 37701, 28794, 28000, 34573, 31737, - 17446, 20786, 20778, 6820, 978, 24492, 24493, 24494, 20779, 31739, 20783, - 20780, 20784, 20782, 20785, 20781, 20929, 22727, 40677, 40678, 40679, - 31573, 31578, 31576, 31572, 31575, 31574, 31577, 27591, 27592, 27594, - 27593, 31569, 31570, 38869, 28293, 28292, 32576, 33848, 28288, 28289, - 6749, 28290, 6543, 31571, 27595, 28291, 20801, 32048, 40673, 372, 20797, - 37751, 37752, 20796, 20795, 37753, 37749, 20794, 37748, 20793, 37750, - 20798, 8038, 8044, 11235, 11236, 8039, 25122, 25130, 11225, 11226, 37698, - 37699, 33787, 33786, 25127, 25124, 25132, 25123, 25131, 25121, 25125, - 25120, 33838, 25129, 25128, 40674, 40670, 16802, 20472, 37559, 37560, - 37301, 37300, 33631, 8477, 16803, 363, 1056, 16793, 31579, 16794, 11215, - 37693, 37009, 16801, 16805, 24393, 18144, 24394, 18145, 24385, 18131, - 24388, 18134, 24386, 18132, 24387, 18133, 24384, 18135, 24378, 18127, - 24377, 18126, 24375, 18123, 24373, 18121, 24372, 18120, 24371, 18124, - 24374, 18122, 33772, 33770, 33773, 33771, 11262, 11261, 11258, 11257, - 33630, 33629, 33628, 33627, 11228, 11230, 11229, 18139, 18128, 24382, - 18142, 24383, 18143, 33846, 22735, 33847, 22736, 16799, 31659, 34564, - 31660, 34565, 31662, 34567, 31658, 34563, 31661, 34566, 31657, 34562, - 11231, 11240, 34559, 34731, 34557, 34729, 34558, 34730, 34556, 34728, - 34551, 34723, 34552, 34724, 34550, 34722, 34553, 34725, 34231, 34318, - 8033, 8035, 8034, 8036, 34546, 34717, 34547, 34716, 34720, 34719, 16710, - 31477, 37539, 17496, 29363, 32835, 32836, 32832, 31300, 38820, 11249, - 38819, 11247, 25133, 32837, 32834, 32833, 11216, 11244, 11238, 31881, - 11018, 38836, 38835, 11286, 31066, 31065, 37564, 37567, 37558, 37571, - 37570, 11260, 11259, 37568, 22724, 11243, 39304, 28801, 29380, 29407, - 33978, 33991, 24103, 24229, 37912, 11325, 29377, 29404, 33975, 33988, - 24105, 37769, 32074, 32075, 24211, 24212, 34346, 34339, 34349, 34343, - 10833, 10834, 10832, 10831, 11198, 3846, 39233, 3939, 39305, 3880, 39252, - 39230, 3837, 20401, 3841, 3863, 39244, 3866, 39248, 3899, 3900, 39265, - 3839, 39231, 3936, 39301, 24049, 37002, 24048, 25143, 24268, 24267, - 24269, 24270, 24205, 24215, 24214, 24263, 24265, 24264, 24199, 39133, - 16795, 31874, 24192, 32060, 32059, 24294, 32155, 31875, 32050, 37298, - 24194, 24193, 32051, 11339, 32830, 32828, 39245, 3901, 39266, 3890, - 39256, 19735, 19748, 19712, 19710, 19713, 33774, 2611, 33775, 2610, 3648, - 32829, 24245, 37931, 32124, 11328, 24107, 37773, 29402, 29375, 33973, - 33986, 24257, 37942, 32136, 11340, 8030, 971, 24256, 37933, 32135, 11330, - 40951, 40951, 29403, 29376, 33974, 33987, 24247, 37941, 32126, 11338, - 20422, 38849, 24246, 37932, 32125, 11329, 24258, 37943, 32137, 11341, - 24228, 37911, 32088, 11327, 968, 969, 967, 970, 31865, 31866, 29275, - 29276, 17502, 32089, 40951, 34830, 37013, 37017, 37014, 37016, 3853, - 3933, 3895, 3827, 11332, 11333, 37935, 37936, 24250, 32129, 24249, 32128, - 3772, 3773, 3774, 3775, 3776, 3778, 3777, 3779, 31912, 31913, 31910, - 31911, 31907, 31909, 31906, 31908, 37956, 37761, 30876, 30875, 30877, - 2755, 6818, 6532, 3906, 3817, 37668, 20400, 3940, 3870, 3862, 3865, 3868, - 29281, 37471, 4348, 24405, 32222, 39222, 32223, 34303, 37744, 18671, - 31585, 31584, 31583, 31582, 37538, 31686, 2531, 20463, 31458, 29058, - 39249, 3820, 37580, 9943, 19685, 40759, 22562, 1053, 97, 38957, 31596, - 24074, 31949, 34492, 34493, 24266, 37947, 32144, 11345, 16815, 16814, - 32666, 32364, 32362, 32363, 32361, 32365, 32366, 16800, 37755, 32657, - 11283, 31219, 31888, 19929, 17909, 17911, 17945, 17919, 17915, 17946, - 17953, 17916, 17952, 17925, 17921, 17920, 17914, 17956, 17927, 17928, - 17929, 17930, 17932, 17934, 17938, 17942, 17955, 17917, 17954, 17931, - 17933, 17935, 17944, 17913, 17937, 17948, 17947, 17949, 17939, 17951, - 17940, 17941, 17950, 17923, 17910, 17922, 17918, 17924, 17936, 17943, - 17926, 17912, 17957, 17959, 17993, 17967, 17963, 17994, 18001, 17964, - 18000, 17973, 17969, 17968, 17962, 18004, 17975, 17976, 17977, 17978, - 17980, 17982, 17986, 17990, 18003, 17965, 18002, 17979, 17981, 17983, - 17992, 17961, 17985, 17996, 17995, 17997, 17987, 17999, 17988, 17989, - 17998, 17971, 17958, 17970, 17966, 17972, 17984, 17991, 17974, 17960, - 22957, 23603, 22960, 23051, 23067, 23337, 23828, 22900, 23522, 22946, - 23582, 23214, 23995, 22780, 22979, 23118, 23119, 23948, 23191, 23965, - 23947, 22905, 23529, 23893, 23450, 23869, 23684, 23261, 24020, 27718, - 23094, 23218, 8485, 8579, 8535, 8629, 8499, 8593, 8496, 8590, 8540, 8634, - 8530, 8624, 8534, 8628, 8501, 8595, 8532, 8626, 8538, 8632, 8504, 8598, - 8509, 8603, 8541, 8635, 8542, 8636, 8505, 8599, 8510, 8604, 8537, 8631, - 8539, 8633, 8531, 8625, 8533, 8627, 8543, 8637, 8507, 8601, 8503, 8597, - 8536, 8630, 8526, 8620, 8493, 8587, 8521, 8615, 8489, 8583, 8494, 8588, - 8495, 8589, 8490, 8584, 8519, 8613, 8529, 8623, 8491, 8585, 8517, 8611, - 8520, 8614, 8484, 8578, 8492, 8586, 8512, 8606, 8513, 8607, 8508, 8602, - 8515, 8609, 8514, 8608, 8511, 8605, 8518, 8612, 8516, 8610, 8524, 8618, - 8522, 8616, 8523, 8617, 8525, 8619, 8639, 8640, 8641, 8643, 8644, 8638, - 8642, 8487, 8581, 8488, 8582, 8483, 8482, 8481, 8486, 8580, 40951, 40951, - 40951, 40951, 40951, 8577, 8574, 8575, 8576, 8572, 8573, 8645, 17768, - 17794, 17779, 17800, 17801, 17799, 17789, 17790, 17802, 17783, 17792, - 17795, 17797, 17803, 17785, 17788, 17793, 17787, 17791, 17804, 17784, - 17782, 17778, 17798, 17786, 17774, 17777, 17781, 17776, 17775, 17796, - 17780, 17769, 17773, 17771, 17806, 17770, 17772, 40951, 17805, 40951, - 40951, 40951, 40951, 40951, 17767, 40951, 40951, 37248, 37259, 37260, - 37253, 37255, 37238, 37277, 37249, 37252, 37250, 37251, 37287, 37276, - 37256, 37242, 37258, 37261, 37237, 37246, 37262, 37275, 37257, 37243, - 37282, 37247, 37288, 37270, 37236, 37244, 37278, 37279, 37280, 37241, - 37245, 37281, 37290, 37272, 37273, 37254, 37240, 37235, 37263, 37265, - 37264, 37266, 37267, 37274, 37268, 37283, 37284, 37285, 37269, 37239, - 37271, 37286, 37289, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 37291, 37292, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 37234, 17250, 17067, 17154, - 17194, 17170, 16857, 17232, 16895, 17085, 17076, 16977, 17310, 16925, - 16910, 17241, 17201, 16886, 17101, 17114, 16962, 16966, 16965, 16964, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 17184, - 17190, 17188, 17185, 17187, 17186, 17189, 40951, 16876, 16882, 16880, - 16877, 16879, 16878, 16881, 40951, 17300, 17306, 17304, 17301, 17303, - 17302, 17305, 40951, 16869, 16875, 16873, 16870, 16872, 16871, 16874, - 40951, 17136, 17142, 17140, 17137, 17139, 17138, 17141, 40951, 17045, - 17051, 17049, 17046, 17048, 17047, 17050, 40951, 17277, 17283, 17281, - 17278, 17280, 17279, 17282, 40951, 16988, 16994, 16992, 16989, 16991, - 16990, 16993, 40951, 8091, 8129, 8126, 8093, 8123, 8124, 8130, 8097, - 8098, 8099, 8106, 8128, 8100, 8094, 8122, 8119, 8121, 8125, 8109, 8108, - 8127, 8095, 8131, 8105, 8092, 8118, 8114, 8116, 8103, 8117, 8090, 8102, - 31923, 31922, 24125, 31993, 24067, 31938, 31760, 31759, 11201, 24128, - 32001, 31768, 24108, 31973, 11542, 31064, 17495, 31883, 20461, 11200, - 11324, 37895, 11203, 11251, 20810, 31023, 20457, 37304, 24088, 31963, - 37303, 37302, 24157, 32007, 37480, 37484, 4353, 4356, 24113, 31978, - 24066, 31943, 37696, 30525, 34406, 17464, 31898, 38852, 32158, 39365, - 37672, 31758, 31770, 37683, 10530, 10531, 37673, 37469, 37708, 37020, - 34506, 38855, 39348, 6002, 11219, 31897, 11222, 10537, 11239, 20811, - 20812, 25159, 25160, 11237, 11197, 37569, 26959, 31063, 31717, 8648, - 8685, 8686, 37392, 26958, 26960, 24123, 31991, 24122, 31990, 37461, - 37465, 4339, 4343, 29931, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 7916, 7869, 7919, - 7918, 7917, 7912, 7840, 7937, 7951, 7950, 7873, 7920, 7933, 7932, 7902, - 7901, 7900, 7899, 7929, 7938, 7928, 7927, 7888, 7887, 7891, 7915, 40951, - 7872, 7935, 7909, 7874, 7907, 7867, 7943, 7942, 7881, 7911, 7910, 7921, - 7871, 7875, 7896, 7838, 7880, 7931, 7930, 7843, 7926, 7854, 7949, 7948, - 7947, 7946, 7904, 7934, 7914, 7879, 7952, 7842, 7841, 7903, 7908, 7885, - 7884, 7883, 7939, 7870, 7945, 7944, 7858, 7922, 7890, 7857, 7856, 7882, - 7866, 7923, 7941, 7940, 7868, 7850, 7898, 7897, 7853, 7851, 7906, 7905, - 7913, 7844, 7860, 7852, 7862, 7848, 7878, 7877, 7876, 7846, 7889, 7865, - 7839, 7886, 7847, 7864, 7855, 7924, 7925, 7849, 7895, 7845, 7893, 7861, - 7894, 7863, 7936, 7892, 7859, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 21128, 21110, 21044, 21164, - 21152, 21096, 21196, 21111, 21124, 21107, 21059, 21063, 21048, 21033, - 21099, 21185, 21131, 21102, 21136, 21213, 21170, 21141, 21093, 21198, - 21042, 21153, 21029, 21133, 21004, 21122, 21058, 21056, 21151, 21079, - 21081, 21061, 21011, 21210, 21036, 21144, 21098, 21180, 21103, 21031, - 21169, 21120, 21142, 21211, 21129, 21194, 21054, 21159, 21045, 21113, - 21197, 21160, 21019, 21178, 21020, 21172, 21090, 21087, 21050, 21088, - 21021, 21139, 21150, 21043, 21006, 21181, 21126, 21183, 21149, 21123, - 21193, 21104, 21173, 21038, 21204, 21049, 21032, 21078, 21027, 21171, - 21203, 21070, 21028, 21065, 21047, 21086, 21165, 21067, 21035, 21051, - 21134, 21100, 21114, 21188, 21177, 21109, 21215, 21068, 21015, 21161, - 21046, 21206, 21182, 21041, 21064, 21166, 21002, 21175, 21168, 21192, - 21082, 21026, 21176, 21007, 21143, 21162, 21101, 21127, 21157, 21076, - 21132, 21005, 21135, 21055, 21022, 21115, 21116, 21154, 21003, 21118, - 21189, 21130, 21016, 21174, 21034, 21083, 21187, 21097, 21012, 21202, - 21030, 21205, 21155, 21095, 21167, 21199, 21023, 21137, 21008, 21156, - 21146, 21145, 21077, 21018, 21025, 21009, 21119, 21201, 21037, 21209, - 21040, 21200, 21080, 21112, 21085, 21121, 21163, 21158, 21138, 21014, - 21212, 21066, 21105, 21184, 21108, 21179, 21106, 21208, 21073, 21057, - 21091, 21074, 21094, 21017, 21186, 21089, 21069, 21147, 21024, 21084, - 21071, 21010, 21148, 21039, 21207, 21092, 21214, 21117, 21013, 21062, - 21075, 21191, 21053, 21140, 21125, 21060, 21190, 21052, 21195, 21072, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 20495, 20493, 20494, 20492, - 20507, 20503, 20502, 20499, 20500, 20501, 20497, 20496, 20504, 20498, - 20508, 20506, 20592, 20491, 20590, 11007, 20828, 20591, 20490, 20510, - 24045, 31920, 24064, 31939, 24060, 31936, 24139, 32010, 24054, 31930, - 31598, 17902, 24136, 32005, 24141, 32012, 24144, 32015, 24143, 32014, - 39132, 31880, 11233, 25157, 31599, 19581, 19574, 19571, 19577, 19576, - 19579, 19578, 19582, 19580, 20588, 20587, 20509, 20586, 19240, 19239, - 39138, 38827, 38830, 38828, 38831, 38829, 6801, 20584, 19575, 19573, - 19572, 38826, 25816, 31216, 20589, 20585, 40951, 20305, 20291, 20307, - 20385, 20309, 20387, 20306, 20384, 20308, 20386, 20346, 20336, 20348, - 20338, 20350, 20340, 20347, 20337, 20349, 20339, 20310, 20371, 20312, - 20373, 20314, 20375, 20311, 20372, 20313, 20374, 20366, 20331, 20368, - 20333, 20304, 20370, 20335, 20367, 20332, 20369, 20334, 20326, 20328, - 20330, 20327, 20329, 20341, 20321, 20356, 20343, 20315, 20358, 20345, - 20324, 20360, 20342, 20322, 20357, 20344, 20323, 20359, 20351, 20353, - 20355, 20352, 20354, 20298, 20380, 20300, 20382, 20299, 20381, 20361, - 20363, 20365, 20362, 20364, 20294, 20376, 20378, 20377, 20379, 20325, - 20383, 20301, 20302, 40951, 40951, 8288, 8287, 21453, 21452, 20389, - 20388, 20290, 21450, 21380, 21309, 21382, 21443, 21384, 21445, 21381, - 21442, 21383, 21444, 21405, 21395, 21407, 21397, 21409, 21399, 21406, - 21396, 21408, 21398, 21385, 21430, 21387, 21432, 21389, 21434, 21386, - 21431, 21388, 21433, 21420, 21390, 21422, 21392, 21367, 21424, 21394, - 21421, 21391, 21423, 21393, 21347, 21349, 21351, 21348, 21350, 21400, - 21324, 21410, 21402, 21318, 21412, 21404, 21327, 21414, 21401, 21325, - 21411, 21403, 21326, 21413, 21342, 21328, 21345, 21343, 21344, 21372, - 21439, 21374, 21441, 21373, 21440, 21415, 21417, 21419, 21416, 21418, - 21368, 21435, 21437, 21436, 21438, 21346, 21429, 21362, 21363, 21425, - 21427, 21426, 21428, 21449, 21451, 21448, 21446, 21447, 40951, 40951, - 40951, 40951, 40951, 4320, 4328, 4327, 4325, 4324, 4331, 4296, 4316, - 4284, 4312, 4326, 4322, 4329, 4333, 4309, 4315, 4319, 4330, 4308, 4314, - 4318, 4264, 4300, 4276, 4281, 4265, 4282, 4267, 4307, 4269, 4277, 4270, - 4278, 4283, 4289, 4274, 4295, 4332, 4297, 4286, 4292, 4303, 4299, 40951, - 19493, 19537, 19494, 19499, 19500, 19502, 19529, 19540, 19515, 19516, - 19518, 19520, 19527, 19523, 19519, 19526, 19495, 19505, 19539, 19506, - 19530, 19542, 19487, 19482, 19536, 19481, 19492, 19528, 19513, 19566, - 19477, 19480, 19563, 19564, 19484, 19483, 19550, 19549, 19567, 19546, - 19547, 19568, 19555, 19569, 19545, 19544, 19548, 19559, 19485, 19565, - 19486, 19570, 19538, 19501, 19504, 19503, 19517, 19524, 19521, 19522, - 19525, 19496, 19498, 19497, 19491, 19512, 19510, 19507, 19508, 19511, - 19509, 19489, 19490, 19532, 19533, 19535, 19534, 19531, 19514, 19543, - 19552, 19554, 19553, 19488, 19541, 19551, 19556, 19557, 19558, 19561, - 19560, 19562, 19478, 19479, 40951, 20484, 20487, 20486, 20482, 20480, - 20476, 20483, 20478, 20474, 20477, 20485, 20481, 20475, 20488, 20489, - 20479, 4321, 4310, 4323, 4287, 4280, 4279, 4306, 4302, 4294, 4271, 4290, - 4275, 4293, 4298, 4266, 4268, 4273, 4305, 4301, 4291, 4262, 4263, 4261, - 4260, 4285, 4317, 4311, 4259, 4288, 4313, 4304, 4272, 7983, 7986, 7987, - 7985, 7973, 7959, 7965, 7954, 7964, 7977, 7966, 7962, 7955, 7963, 7960, - 7989, 7953, 7972, 7968, 7981, 7988, 7958, 7967, 7976, 7975, 7982, 7980, - 7969, 7971, 7984, 7979, 7974, 7956, 7961, 7970, 7990, 7957, 7978, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20505, 21365, - 21377, 21378, 21366, 21376, 21352, 21354, 21356, 21353, 21355, 21379, - 21357, 21359, 21361, 21358, 21360, 31078, 31082, 31094, 31088, 31080, - 31086, 31090, 31096, 31071, 31069, 31076, 31092, 31084, 31074, 31079, - 31083, 31095, 31089, 31081, 31087, 31091, 31097, 31072, 31070, 31077, - 31093, 31085, 31075, 31073, 31135, 31134, 40951, 31133, 31129, 31127, - 31108, 31106, 31126, 31118, 31103, 31112, 31128, 31111, 31105, 31131, - 31130, 31110, 31102, 31125, 31123, 31132, 31120, 31113, 31121, 31104, - 31099, 31109, 31116, 31100, 31122, 31124, 31101, 31114, 31098, 31107, - 31115, 31119, 31117, 6620, 6608, 6630, 6609, 6774, 6788, 6776, 6758, - 6755, 6771, 6769, 6751, 31215, 6789, 6795, 6794, 6791, 6790, 6793, 6792, - 6797, 6796, 6775, 6777, 6783, 6782, 6779, 6778, 6568, 6572, 6584, 6578, - 6570, 6576, 6580, 6561, 6559, 6557, 6566, 6582, 6574, 6564, 6569, 6573, - 6585, 6579, 6571, 6577, 6581, 6562, 6560, 6558, 6567, 6583, 6575, 6565, - 6695, 6694, 6563, 22560, 6643, 6639, 6637, 6605, 6603, 6635, 6626, 6600, - 6618, 6638, 6616, 6602, 6641, 6640, 6614, 6598, 6629, 6634, 6607, 6631, - 6619, 6632, 6601, 6594, 6610, 6625, 6615, 6604, 6628, 6599, 6636, 6591, - 6642, 6622, 6595, 6593, 6606, 6596, 6611, 6612, 6624, 6613, 6623, 6633, - 6627, 6597, 6621, 6589, 6617, 6781, 6780, 6785, 6784, 6757, 6759, 6765, - 6764, 6761, 6760, 6763, 6762, 6767, 6766, 6754, 20575, 20578, 20579, - 20517, 20580, 20576, 20577, 20516, 20582, 20583, 20581, 20549, 34191, - 34157, 34159, 24491, 6689, 6691, 6693, 6690, 6692, 6652, 6654, 6656, - 6653, 6655, 6672, 6674, 6676, 6673, 6675, 6677, 6679, 6681, 6678, 6680, - 6662, 6664, 6666, 6663, 6665, 6647, 6649, 6651, 6648, 6650, 6657, 6659, - 6661, 6658, 6660, 6686, 6688, 6687, 6667, 6669, 6671, 6668, 6670, 6682, - 6684, 6683, 6685, 34155, 34116, 34118, 34117, 34119, 34194, 34195, 34358, - 34158, 34151, 34284, 34288, 34203, 34202, 34201, 34166, 34167, 34171, - 34170, 34204, 34169, 34205, 34208, 34206, 34207, 34172, 34173, 34215, - 34216, 34217, 34214, 34213, 34324, 34325, 34332, 34326, 34327, 34142, - 34146, 34147, 34337, 34277, 34278, 34179, 34291, 34292, 34123, 34300, - 34298, 34299, 34126, 34186, 34185, 34125, 34187, 34180, 34297, 34295, - 34181, 34296, 34294, 34128, 34301, 34127, 34184, 34302, 34182, 34183, - 34241, 34242, 34245, 34244, 34243, 34251, 34252, 34253, 34248, 34249, - 34250, 34356, 34355, 34357, 34319, 34320, 34322, 34321, 34317, 34316, - 34338, 20573, 20574, 20556, 20558, 20565, 20564, 20571, 20569, 20560, - 20567, 20559, 20562, 20555, 20557, 20566, 20563, 20572, 20570, 20561, - 20568, 20550, 20554, 20553, 20552, 20551, 34189, 34141, 34121, 34124, - 34289, 34305, 34143, 34145, 34144, 34199, 34152, 34156, 34153, 34154, - 34133, 34293, 34276, 34258, 34240, 34200, 34222, 34246, 34176, 34130, - 34219, 34306, 34279, 34259, 34260, 34273, 34223, 34192, 34218, 34270, - 34174, 34336, 34261, 34274, 34150, 34225, 34165, 34280, 34262, 34255, - 34134, 34209, 34257, 34136, 34239, 34212, 34256, 34135, 34238, 34211, - 34235, 34236, 34290, 34221, 34272, 34175, 34313, 34314, 34315, 34310, - 34281, 34263, 34275, 34311, 34282, 34264, 34266, 34227, 34267, 34312, - 34283, 34265, 34268, 34228, 34269, 34220, 34237, 34120, 34129, 34139, - 34140, 34137, 34132, 34148, 34177, 34178, 34188, 34193, 34224, 34210, - 34226, 34232, 34233, 34230, 34234, 34247, 34254, 34271, 34307, 34308, - 34304, 34309, 34333, 34334, 34354, 34122, 34114, 20548, 20533, 20521, - 20540, 20539, 20546, 20544, 20535, 20542, 20534, 20537, 20532, 20520, - 20541, 20538, 20547, 20545, 20536, 20543, 20522, 20530, 20528, 20527, - 20524, 20523, 20526, 20525, 20531, 20529, 20518, 20519, 34168, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20267, 20270, 20231, - 20281, 20278, 20226, 20264, 20243, 20260, 20277, 20255, 20261, 20236, - 20238, 20248, 20282, 20235, 20279, 20219, 20225, 20223, 20242, 20262, - 20256, 20244, 20241, 20249, 20240, 20265, 20268, 20247, 20233, 20257, - 20239, 20254, 20234, 20269, 20252, 20250, 20229, 20228, 20246, 20224, - 20227, 20237, 20253, 20251, 20271, 20258, 20266, 20263, 20275, 20230, - 20273, 20221, 20272, 20274, 20276, 20232, 20280, 20245, 20259, 20220, - 20222, 39884, 39891, 39883, 39890, 39888, 39889, 39886, 39887, 40659, - 40660, 40657, 40658, 40656, 40654, 40655, 40663, 40664, 40661, 40662, - 40666, 40665, 40531, 39551, 39552, 39545, 39550, 39548, 39549, 39546, - 39547, 39555, 39556, 39553, 39554, 39536, 39534, 39535, 39559, 39560, - 39557, 39558, 39544, 39542, 39543, 39540, 39541, 39533, 39539, 39538, - 39537, 39565, 39566, 39561, 39564, 39563, 39562, 40263, 40264, 40258, - 40262, 40261, 40259, 40260, 40267, 40268, 40265, 40266, 40252, 40250, - 40251, 40271, 40272, 40269, 40270, 40256, 40257, 40249, 40255, 40254, - 40253, 40277, 40278, 40273, 40276, 40275, 40274, 39519, 39520, 39513, - 39518, 39516, 39517, 39514, 39515, 39523, 39524, 39521, 39522, 39504, - 39502, 39503, 39527, 39528, 39525, 39526, 39512, 39510, 39511, 39508, - 39509, 39501, 39507, 39506, 39505, 39531, 39532, 39529, 39530, 40066, - 40067, 40061, 40065, 40064, 40062, 40063, 40070, 40071, 40068, 40069, - 40074, 40075, 40072, 40073, 40080, 40081, 40076, 40079, 40078, 40077, - 40086, 40087, 40082, 40085, 40084, 40083, 39809, 39810, 39804, 39808, - 39807, 39805, 39806, 39813, 39814, 39811, 39812, 39798, 39796, 39797, - 39817, 39818, 39815, 39816, 39802, 39803, 39795, 39801, 39800, 39799, - 39823, 39819, 39822, 39821, 39820, 40045, 40046, 40040, 40044, 40043, - 40041, 40042, 40049, 40050, 40047, 40048, 40033, 40034, 40031, 40032, - 40053, 40054, 40051, 40052, 40060, 40059, 40038, 40039, 40030, 40037, - 40036, 40035, 40057, 40058, 40055, 40056, 39690, 39691, 39688, 39689, - 39686, 39687, 39684, 39685, 39683, 39681, 39682, 39700, 39701, 39696, - 39699, 39698, 39697, 39694, 39695, 39692, 39693, 40509, 40510, 40503, - 40508, 40506, 40507, 40504, 40505, 40513, 40514, 40511, 40512, 40517, - 40518, 40515, 40516, 40502, 40501, 40523, 40524, 40519, 40522, 40521, - 40520, 40529, 40530, 40525, 40528, 40527, 40526, 39668, 39669, 39663, - 39667, 39666, 39664, 39665, 39675, 39676, 39673, 39674, 39657, 39656, - 39679, 39680, 39677, 39678, 39672, 39670, 39671, 39661, 39662, 39655, - 39660, 39659, 39658, 40488, 40489, 40483, 40487, 40486, 40484, 40485, - 40495, 40496, 40493, 40494, 40476, 40477, 40474, 40475, 40499, 40500, - 40497, 40498, 40492, 40490, 40491, 40481, 40482, 40473, 40480, 40479, - 40478, 39642, 39643, 39637, 39641, 39640, 39638, 39639, 39649, 39650, - 39647, 39648, 39631, 39629, 39630, 39653, 39654, 39651, 39652, 39646, - 39644, 39645, 39635, 39636, 39628, 39634, 39633, 39632, 40092, 40093, - 40088, 40091, 40090, 40089, 40099, 40100, 40097, 40098, 40103, 40104, - 40101, 40102, 40096, 40094, 40095, 40109, 40110, 40105, 40108, 40107, - 40106, 39839, 39840, 39833, 39838, 39836, 39837, 39834, 39835, 39843, - 39844, 39841, 39842, 39828, 39827, 39825, 39826, 39824, 39832, 39830, - 39831, 39829, 40237, 40238, 40232, 40236, 40235, 40233, 40234, 40241, - 40239, 40240, 40226, 40224, 40225, 40247, 40248, 40245, 40246, 40244, - 40242, 40243, 40230, 40231, 40223, 40229, 40228, 40227, 39777, 39778, - 39772, 39776, 39775, 39773, 39774, 39787, 39788, 39785, 39786, 39766, - 39764, 39765, 39784, 39782, 39783, 39781, 39779, 39780, 39770, 39771, - 39763, 39769, 39768, 39767, 39793, 39794, 39789, 39792, 39791, 39790, - 39992, 39993, 39986, 39991, 39989, 39990, 39987, 39988, 39996, 39997, - 39994, 39995, 39976, 39977, 39974, 39975, 40000, 40001, 39998, 39999, - 39985, 39983, 39984, 39981, 39982, 39973, 39980, 39979, 39978, 40006, - 40007, 40002, 40005, 40004, 40003, 39746, 39747, 39740, 39745, 39743, - 39744, 39741, 39742, 39750, 39751, 39748, 39749, 39733, 39734, 39731, - 39732, 39758, 39759, 39756, 39757, 39754, 39755, 39752, 39753, 39738, - 39739, 39730, 39737, 39736, 39735, 39959, 39960, 39954, 39958, 39957, - 39955, 39956, 39963, 39964, 39961, 39962, 39948, 39946, 39947, 39971, - 39972, 39969, 39970, 39967, 39968, 39965, 39966, 39952, 39953, 39945, - 39951, 39950, 39949, 39706, 39707, 39702, 39705, 39703, 39704, 39720, - 39721, 39718, 39719, 39711, 39712, 39709, 39710, 39728, 39729, 39726, - 39727, 39724, 39725, 39722, 39723, 39716, 39717, 39708, 39715, 39714, - 39713, 40029, 40028, 40022, 40023, 40020, 40021, 40011, 40009, 40010, - 40026, 40027, 40024, 40025, 40019, 40017, 40018, 40015, 40016, 40008, - 40014, 40013, 40012, 39858, 39859, 39852, 39857, 39855, 39856, 39853, - 39854, 39862, 39863, 39860, 39861, 39847, 39848, 39845, 39846, 39866, - 39867, 39864, 39865, 39851, 39849, 39850, 40119, 40117, 40118, 40122, - 40123, 40120, 40121, 40112, 40113, 40111, 40126, 40127, 40124, 40125, - 40116, 40114, 40115, 39762, 39761, 39760, 39877, 39878, 39875, 39876, - 39870, 39871, 39868, 39869, 39881, 39882, 39879, 39880, 39874, 39872, - 39873, 40543, 40544, 40541, 40542, 40534, 40532, 40533, 40540, 40538, - 40539, 40537, 40535, 40536, 40606, 40607, 40601, 40605, 40604, 40602, - 40603, 40642, 40643, 40640, 40641, 40595, 40593, 40594, 40646, 40647, - 40644, 40645, 40639, 40637, 40638, 40599, 40600, 40592, 40598, 40597, - 40596, 40652, 40653, 40648, 40651, 40650, 40649, 39612, 39613, 39606, - 39611, 39609, 39610, 39607, 39608, 39616, 39617, 39614, 39615, 39597, - 39595, 39596, 39620, 39621, 39618, 39619, 39605, 39603, 39604, 39601, - 39602, 39594, 39600, 39599, 39598, 39626, 39627, 39622, 39625, 39624, - 39623, 40620, 40621, 40614, 40619, 40617, 40618, 40615, 40616, 40624, - 40625, 40622, 40623, 40613, 40611, 40612, 40610, 40608, 40609, 40630, - 40626, 40629, 40628, 40627, 40635, 40636, 40631, 40634, 40633, 40632, - 40209, 40210, 40204, 40208, 40207, 40205, 40206, 40213, 40214, 40211, - 40212, 40197, 40196, 40203, 40202, 40222, 40221, 40201, 40195, 40200, - 40199, 40198, 40219, 40220, 40215, 40218, 40217, 40216, 40454, 40455, - 40449, 40453, 40452, 40450, 40451, 40461, 40462, 40459, 40460, 40443, - 40441, 40442, 40465, 40466, 40463, 40464, 40458, 40456, 40457, 40447, - 40448, 40440, 40446, 40445, 40444, 40471, 40472, 40467, 40470, 40469, - 40468, 40390, 40391, 40385, 40389, 40388, 40386, 40387, 40397, 40398, - 40395, 40396, 40401, 40402, 40399, 40400, 40394, 40392, 40393, 40405, - 40406, 40403, 40404, 40411, 40412, 40407, 40410, 40409, 40408, 40576, - 40577, 40574, 40575, 40568, 40566, 40567, 40584, 40585, 40582, 40583, - 40580, 40581, 40578, 40579, 40572, 40573, 40565, 40571, 40570, 40569, - 40590, 40591, 40586, 40589, 40588, 40587, 39578, 39579, 39576, 39577, - 39570, 39571, 39568, 39569, 39586, 39587, 39584, 39585, 39582, 39583, - 39580, 39581, 39575, 39567, 39574, 39573, 39572, 39592, 39593, 39588, - 39591, 39590, 39589, 40358, 40357, 40337, 40336, 40349, 40350, 40347, - 40348, 40345, 40346, 40343, 40344, 40341, 40342, 40335, 40340, 40339, - 40338, 40355, 40356, 40351, 40354, 40353, 40352, 40158, 40159, 40156, - 40157, 40155, 40153, 40154, 40162, 40163, 40160, 40161, 40168, 40169, - 40164, 40167, 40166, 40165, 40174, 40175, 40170, 40173, 40172, 40171, - 40424, 40425, 40422, 40423, 40416, 40414, 40415, 40432, 40433, 40430, - 40431, 40428, 40429, 40426, 40427, 40420, 40421, 40413, 40419, 40418, - 40417, 40438, 40439, 40434, 40437, 40436, 40435, 40373, 40374, 40371, - 40372, 40362, 40360, 40361, 40377, 40378, 40375, 40376, 40370, 40368, - 40369, 40366, 40367, 40359, 40365, 40364, 40363, 40383, 40384, 40379, - 40382, 40381, 40380, 39933, 39934, 39927, 39932, 39930, 39931, 39928, - 39929, 39920, 39921, 39918, 39919, 39937, 39938, 39935, 39936, 39925, - 39926, 39917, 39924, 39923, 39922, 39943, 39944, 39939, 39942, 39941, - 39940, 40295, 40296, 40289, 40294, 40292, 40293, 40290, 40291, 40282, - 40283, 40280, 40281, 40299, 40300, 40297, 40298, 40287, 40288, 40279, - 40286, 40285, 40284, 40305, 40306, 40301, 40304, 40303, 40302, 39907, - 39908, 39901, 39906, 39904, 39905, 39902, 39903, 39895, 39893, 39894, - 39911, 39912, 39909, 39910, 39899, 39900, 39892, 39898, 39897, 39896, - 39915, 39916, 39913, 39914, 40141, 40142, 40135, 40140, 40138, 40139, - 40136, 40137, 40130, 40129, 40145, 40146, 40143, 40144, 40134, 40128, - 40133, 40132, 40131, 40151, 40152, 40147, 40150, 40149, 40148, 40189, - 40190, 40183, 40188, 40186, 40187, 40184, 40185, 40179, 40177, 40178, - 40193, 40194, 40191, 40192, 40181, 40182, 40176, 40180, 40551, 40552, - 40545, 40550, 40548, 40549, 40546, 40547, 40564, 40563, 40555, 40556, - 40553, 40554, 40561, 40562, 40557, 40560, 40559, 40558, 40323, 40324, - 40317, 40322, 40320, 40321, 40318, 40319, 40310, 40311, 40308, 40309, - 40327, 40328, 40325, 40326, 40315, 40316, 40307, 40314, 40313, 40312, - 40333, 40334, 40329, 40332, 40331, 40330, 40951, 40951, 40951, 39498, - 39471, 39469, 39476, 39450, 39486, 39457, 39459, 39475, 39462, 39473, - 39446, 39474, 39492, 39480, 39461, 39487, 39460, 39493, 39451, 39454, - 39447, 39456, 39477, 39488, 39500, 39465, 39496, 39481, 39464, 39491, - 39489, 39485, 39490, 39497, 39468, 39478, 39467, 39458, 39466, 39499, - 39455, 39482, 39472, 39449, 39448, 39453, 39463, 39483, 39494, 39484, - 39452, 39495, 39479, 39470, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 25094, 25083, 25082, 25066, 25064, 25063, 25077, - 25081, 25080, 25096, 25075, 25074, 25065, 25062, 25061, 25098, 25071, - 25097, 25085, 25088, 25089, 25070, 25079, 25100, 25078, 25095, 25099, - 25084, 25087, 25076, 25090, 25091, 25072, 25073, 25101, 25092, 25067, - 25068, 25069, 25093, 25057, 25060, 25058, 25056, 25059, 25055, 25102, - 25103, 38225, 38226, 38062, 38211, 38212, 38185, 37988, 37995, 38098, - 38071, 38105, 38045, 38171, 38199, 38023, 38016, 37974, 37967, 38089, - 38192, 37981, 38124, 38009, 38002, 38037, 38030, 38164, 38178, 38143, - 38206, 38084, 38130, 38051, 38112, 38157, 38150, 38234, 38235, 38066, - 38067, 38220, 38221, 38187, 37990, 37997, 38100, 38077, 38107, 38048, - 38173, 38201, 38025, 38018, 37976, 37969, 38093, 38194, 37983, 38126, - 38011, 38004, 38039, 38032, 38166, 38180, 38145, 38208, 38085, 38135, - 38056, 38114, 38159, 38152, 38232, 38233, 38137, 38064, 38065, 38218, - 38219, 38186, 37989, 37996, 38099, 38075, 38076, 38106, 38047, 38172, - 38200, 38024, 38017, 37975, 37968, 38092, 38193, 37982, 38125, 38010, - 38003, 38038, 38031, 38165, 38179, 38144, 38207, 38081, 38082, 38134, - 38055, 38113, 38158, 38151, 38229, 38230, 38060, 38215, 38216, 38183, - 37986, 37993, 38096, 38074, 38103, 38043, 38169, 38197, 38021, 38014, - 37972, 37965, 38091, 38190, 37979, 38122, 38007, 38000, 38035, 38028, - 38162, 38176, 38141, 38204, 38080, 38133, 38054, 38110, 38155, 38148, - 38236, 38237, 38068, 38069, 38222, 38223, 38188, 37991, 37998, 38101, - 38078, 38108, 38049, 38174, 38202, 38026, 38019, 37977, 37970, 38094, - 38195, 37984, 38127, 38012, 38005, 38040, 38033, 38167, 38181, 38146, - 38209, 38086, 38136, 38057, 38115, 38160, 38153, 38228, 38231, 38139, - 38058, 38059, 38214, 38217, 38182, 37985, 37992, 38095, 38073, 38102, - 38041, 38042, 38168, 38196, 38020, 38013, 37971, 37964, 38090, 38189, - 37978, 38116, 38006, 37999, 38034, 38027, 38161, 38175, 38140, 38203, - 38079, 38132, 38053, 38109, 38154, 38147, 38224, 38227, 38138, 38061, - 38063, 38210, 38213, 38184, 37987, 37994, 38097, 38070, 38072, 38104, - 38044, 38046, 38170, 38198, 38022, 38015, 37973, 37966, 38087, 38191, - 37980, 38123, 38008, 38001, 38036, 38029, 38163, 38177, 38142, 38205, - 38083, 38129, 38131, 38050, 38052, 38111, 38156, 38149, 38128, 38088, - 37961, 37962, 37963, 38119, 38120, 38117, 38241, 38244, 38247, 38246, - 38250, 38242, 38249, 38240, 38238, 38245, 38248, 38239, 38243, 38257, - 38259, 38256, 38255, 38252, 38251, 38254, 38253, 38260, 38258, 38121, - 38118, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 10283, 10484, 10172, 10331, 10122, 10425, 10227, 10386, 10168, - 10327, 10248, 10411, 10156, 10315, 10115, 10412, 10276, 10477, 10224, - 10383, 10124, 10427, 10225, 10384, 10164, 10323, 10157, 10316, 10221, - 10380, 10278, 10479, 10123, 10426, 10267, 10445, 10264, 10442, 10265, - 10443, 10247, 10410, 10154, 10313, 10169, 10328, 10298, 8145, 8137, 8088, - 8138, 33776, 8112, 8101, 8115, 8111, 8120, 8113, 8110, 8107, 8143, 8132, - 10297, 10301, 10178, 10337, 10176, 10335, 10288, 10489, 10166, 10325, - 10174, 10333, 10128, 10453, 10136, 10462, 10132, 10457, 10131, 10456, - 10134, 10460, 10212, 10371, 10262, 10440, 10170, 10329, 10165, 10324, - 27730, 27767, 8096, 8104, 3413, 2779, 3416, 2781, 3412, 3388, 3405, 3415, - 2808, 3414, 2784, 3385, 3391, 3390, 2782, 2788, 3404, 2807, 2801, 2787, - 3400, 2795, 3395, 3401, 3394, 3398, 2777, 2773, 2805, 2804, 2799, 3407, - 3397, 3408, 3409, 2806, 2771, 3381, 2800, 2803, 3384, 3410, 3382, 2768, - 3393, 2786, 2793, 2810, 3387, 3392, 2772, 2796, 2797, 2798, 3396, 3383, - 2770, 2769, 3411, 2809, 2785, 3386, 2783, 2774, 2790, 3389, 2789, 2792, - 3406, 2780, 2794, 2791, 3403, 2778, 3402, 2802, 3399, 2767, 2775, 2776, - 2764, 2763, 3417, 3419, 2766, 2765, 3418, 3420, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 27727, 27723, 27726, 27722, 27728, - 27724, 27729, 27725, 27782, 27797, 27825, 27804, 27787, 27781, 27796, - 27824, 27803, 27786, 27783, 27798, 27826, 27808, 27788, 27774, 27773, - 27775, 27819, 27838, 27835, 27837, 27836, 27816, 27986, 27987, 22867, - 23470, 22868, 23471, 22907, 23532, 23133, 23894, 23132, 23851, 22809, - 23392, 22810, 23393, 23274, 23282, 22783, 23348, 22784, 23349, 22785, - 23350, 22781, 23346, 22782, 23347, 22786, 23351, 23077, 23775, 22948, - 23584, 22945, 23581, 22949, 23585, 22795, 23369, 22967, 23608, 23000, - 23683, 23001, 23685, 23047, 23723, 23052, 23728, 23050, 23726, 23053, - 23729, 23059, 23740, 23058, 23737, 23074, 23765, 23079, 23778, 23174, - 23944, 23183, 23956, 23178, 23951, 23127, 23845, 23128, 23846, 23182, - 23955, 22869, 23488, 22936, 23571, 22811, 23394, 27995, 23430, 23629, - 23643, 23666, 23777, 23259, 23889, 23941, 22929, 23560, 22930, 23561, - 22931, 23117, 23857, 23122, 23887, 22932, 23563, 22933, 23564, 22934, - 23565, 27802, 27771, 27848, 23100, 23801, 23120, 23613, 23290, 22993, - 23657, 22805, 23382, 23379, 23526, 22789, 23354, 22876, 23495, 23179, - 23952, 23180, 23953, 23181, 23954, 22884, 23506, 22952, 23589, 22996, - 23662, 23072, 23762, 23096, 23799, 22903, 23076, 23103, 22955, 23099, - 23281, 23121, 23124, 22939, 22812, 22796, 23371, 23045, 23721, 23170, - 23932, 22889, 23511, 22890, 23512, 22891, 23513, 23042, 23712, 22778, - 23343, 22803, 23097, 23220, 22816, 23396, 23093, 23793, 23080, 23092, - 23792, 40951, 40951, 22808, 23388, 40951, 23423, 40951, 23420, 22982, - 23642, 23102, 23819, 22971, 23618, 22972, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27703, 27706, 27715, - 23075, 23766, 23230, 27698, 27927, 23264, 23228, 23229, 23226, 23227, - 23225, 34770, 34772, 34782, 34774, 34771, 34773, 34781, 34762, 34761, - 34758, 34757, 34780, 34756, 34755, 34760, 34759, 34750, 34749, 34744, - 34743, 34752, 34751, 34746, 34745, 34768, 34764, 34763, 34754, 34753, - 34767, 34748, 34766, 34747, 34769, 34765, 34784, 34786, 34787, 34785, - 34783, 34775, 34776, 34777, 34778, 34779, 40951, 40951, 40951, 29388, - 29387, 29390, 29385, 29386, 29389, 29382, 29381, 29384, 29383, 40951, - 40951, 40951, 40951, 40951, 40951, 31352, 31351, 31340, 31341, 31328, - 31330, 31362, 31343, 31350, 31349, 31333, 31344, 31354, 31353, 31359, - 31364, 31346, 31345, 31332, 31367, 31355, 31356, 31334, 31369, 31366, - 31363, 31335, 31336, 31361, 31325, 31370, 31372, 31357, 31371, 31365, - 31368, 31360, 31339, 31358, 31377, 31378, 31348, 31347, 31331, 31342, - 31326, 31338, 31337, 31327, 31376, 31379, 31329, 31375, 31380, 31374, - 31373, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32496, - 32498, 32445, 32446, 32466, 32467, 32462, 32463, 32457, 32458, 32459, - 32460, 32489, 32490, 32447, 32464, 32465, 32448, 32486, 32485, 32482, - 32481, 32470, 32480, 32479, 32484, 32483, 32472, 32454, 32453, 32450, - 32449, 32471, 32456, 32455, 32452, 32451, 32473, 32488, 32487, 32478, - 32477, 32492, 32494, 32493, 32469, 32461, 32474, 32475, 32476, 32491, - 32468, 32511, 32512, 32523, 32524, 32515, 32516, 32517, 32518, 32519, - 32520, 32525, 32526, 32513, 32521, 32522, 32514, 32497, 32495, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32500, 32499, 32507, - 32509, 32506, 32505, 32502, 32501, 32504, 32503, 32510, 32508, 40951, - 40951, 40951, 40951, 40951, 40951, 8163, 8165, 8162, 8161, 8158, 8157, - 8160, 8159, 8166, 8164, 8154, 8155, 8150, 8151, 8152, 8153, 8149, 8156, - 10770, 10764, 10777, 10766, 10765, 10763, 10778, 10667, 10665, 10670, - 10771, 10821, 10676, 10788, 21585, 21587, 21584, 21583, 21580, 21579, - 21582, 21581, 21588, 21586, 21549, 21548, 21559, 21545, 21553, 21552, - 21566, 21546, 21555, 21543, 21547, 21551, 21550, 21561, 21558, 21556, - 21562, 21565, 21560, 21564, 21554, 21544, 21563, 21557, 21567, 21541, - 21568, 21542, 21577, 21574, 21576, 21575, 21578, 21573, 21571, 21572, - 21569, 21570, 31838, 31835, 31827, 31843, 31834, 31829, 31840, 31832, - 31831, 31833, 31837, 31825, 31842, 31841, 31839, 31845, 31844, 31836, - 31830, 31826, 31828, 31824, 31846, 31853, 31855, 31848, 31851, 31854, - 31852, 31850, 31849, 31821, 31820, 31823, 31822, 31856, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31847, - 19228, 19231, 19232, 19229, 19186, 19187, 19195, 19189, 19191, 19194, - 19188, 19184, 19190, 19192, 19185, 19151, 19153, 19154, 19167, 19174, - 19181, 19216, 19133, 19140, 19214, 19218, 19164, 19237, 19220, 40951, - 40951, 40951, 20903, 20901, 20905, 20904, 20875, 20843, 20842, 20844, - 20884, 20863, 20849, 20850, 20882, 20876, 20883, 20845, 20846, 20847, - 20859, 20860, 20848, 20857, 20858, 20873, 20852, 20874, 20851, 20871, - 20872, 20838, 20839, 20854, 20869, 20870, 20840, 20841, 20853, 20861, - 20862, 20855, 20856, 20879, 20881, 20864, 20865, 20878, 20880, 20867, - 20868, 20866, 20877, 20902, 20908, 20910, 20911, 20912, 20906, 20907, - 20909, 20914, 20913, 20834, 20835, 20836, 20899, 20837, 20885, 20888, - 20897, 20890, 20895, 20893, 20891, 20889, 20886, 20887, 20892, 20900, - 40951, 20898, 20921, 20923, 20920, 20919, 20916, 20915, 20918, 20917, - 20924, 20922, 40951, 40951, 40951, 40951, 20894, 20896, 28607, 28605, - 28600, 28597, 28603, 28693, 28671, 28623, 28635, 28631, 28630, 28633, - 28632, 28625, 28624, 28622, 28735, 28737, 28734, 28733, 28730, 28729, - 28732, 28731, 28738, 28736, 28634, 28627, 28626, 28629, 28628, 40951, - 6245, 6263, 6265, 6262, 6246, 6264, 6254, 6253, 6250, 6249, 6225, 6226, - 6248, 6247, 6252, 6251, 6227, 6229, 6228, 6256, 6255, 6242, 6241, 6230, - 6231, 6240, 6236, 6235, 6234, 6239, 6238, 6232, 6233, 6237, 6261, 6259, - 6258, 6260, 6243, 6244, 6257, 6270, 6273, 6274, 6279, 6277, 6276, 6275, - 6271, 6272, 6278, 6213, 6211, 6210, 6212, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 6219, 6218, 6215, 6207, 6217, 6223, - 6214, 6221, 6224, 6222, 6220, 6216, 6209, 6208, 40951, 40951, 6286, 6288, - 6285, 6284, 6281, 6280, 6283, 6282, 6289, 6287, 40951, 40951, 6266, 6268, - 6267, 6269, 28577, 28570, 28569, 28574, 28573, 28567, 28566, 28565, - 28563, 28562, 28564, 28568, 28579, 28572, 28571, 28576, 28670, 28580, - 28581, 28578, 28667, 28668, 28669, 28708, 28710, 28709, 28552, 28704, - 28695, 28696, 28616, 28617, 35254, 35230, 35253, 35229, 35252, 35228, - 35267, 35243, 35261, 35237, 35256, 35232, 35255, 35231, 35272, 35248, - 35262, 35238, 35265, 35241, 35260, 35236, 35259, 35235, 35263, 35239, - 35264, 35240, 35258, 35234, 35257, 35233, 35266, 35242, 35270, 35246, - 35274, 35250, 35271, 35247, 35269, 35245, 35273, 35249, 35268, 35244, - 35275, 35251, 35276, 35288, 35296, 35293, 35292, 35298, 35299, 35277, - 35297, 35294, 35295, 35287, 35291, 35290, 35289, 35286, 35283, 35284, - 35285, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 35279, 35280, 35282, 35281, 35278, - 27037, 27038, 26996, 27013, 27024, 27023, 27000, 26999, 27012, 27018, - 27019, 27051, 27053, 27043, 27045, 27044, 26991, 26988, 26990, 27040, - 27041, 27055, 27056, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 17212, 17210, 17209, 17208, 17207, 17211, 40951, - 40951, 16906, 16904, 16903, 16902, 16901, 16905, 40951, 40951, 16921, - 16919, 16918, 16917, 16916, 16920, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 16862, 16868, 16866, 16863, 16865, 16864, - 16867, 40951, 16847, 16853, 16851, 16848, 16850, 16849, 16852, 40951, - 23362, 23338, 23368, 23363, 23459, 23622, 23811, 23611, 23602, 23605, - 23633, 23647, 23473, 23366, 23367, 23717, 23567, 23860, 23859, 23861, - 23862, 23821, 23258, 23764, 23421, 23744, 23422, 23808, 23809, 23365, - 23931, 23897, 23940, 23883, 23936, 23385, 23386, 23387, 23972, 23969, - 23970, 23971, 23983, 27685, 27908, 27917, 27918, 27975, 23802, 23570, - 23718, 23942, 23566, 18401, 23428, 23892, 23865, 27972, 27821, 27845, - 40951, 40951, 40951, 40951, 6468, 6469, 6470, 6471, 6472, 6473, 6431, - 6467, 6432, 6433, 6434, 6435, 6436, 6396, 6397, 6398, 6399, 6400, 6401, - 6437, 6438, 6439, 6440, 6441, 6442, 6443, 6444, 6445, 6446, 6447, 6402, - 6395, 6403, 6404, 6405, 6406, 6407, 6408, 6449, 6450, 6451, 6452, 6453, - 6454, 6410, 6409, 6411, 6412, 6413, 6414, 6415, 6389, 6428, 6390, 6429, - 6391, 6430, 6392, 6393, 6394, 6388, 6416, 6417, 6418, 6419, 6420, 6421, - 6422, 6423, 6424, 6425, 6426, 6427, 6455, 6456, 6457, 6458, 6459, 6460, - 6461, 27005, 27017, 27027, 27029, 27014, 27008, 26995, 27020, 27007, - 27010, 27022, 27033, 27035, 27031, 27036, 27025, 27016, 27034, 27002, - 27003, 27032, 26994, 27004, 26998, 27001, 26997, 26993, 27006, 27028, - 27030, 27015, 27009, 27021, 27011, 27026, 27047, 27050, 27042, 27049, - 27048, 27052, 27046, 27054, 26992, 27039, 26989, 40951, 40951, 27063, - 27065, 27062, 27061, 27058, 27057, 27060, 27059, 27066, 27064, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 19425, 19423, 19456, 19457, 19458, 19434, 19435, 19467, 19469, - 19402, 19400, 19399, 19403, 19409, 19410, 19412, 19411, 19416, 19413, - 19414, 19418, 19386, 19384, 40951, 40951, 40951, 40951, 19282, 19283, - 19351, 19352, 19371, 19365, 19366, 19369, 19368, 19367, 19326, 19311, - 19350, 19317, 19321, 19320, 19334, 19333, 19254, 19279, 19272, 19357, - 19270, 19277, 19307, 19300, 19306, 19361, 19302, 19305, 19304, 19345, - 19338, 19355, 19356, 19344, 19342, 19341, 19346, 19348, 19293, 19292, - 19378, 19379, 19243, 19242, 19358, 19297, 19295, 40951, 40951, 40951, - 40951, 7582, 7583, 7584, 7585, 7586, 7587, 7588, 7589, 7590, 7591, 7592, - 7593, 7594, 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, - 7605, 7606, 7607, 7608, 7609, 7610, 7611, 7612, 7613, 7614, 7615, 7616, - 7617, 7618, 7619, 7620, 7621, 7622, 7623, 7624, 7625, 7626, 7627, 7628, - 7629, 7630, 7631, 7632, 7633, 7634, 7635, 7636, 7637, 7638, 7639, 7640, - 7641, 7642, 7643, 7644, 7645, 7646, 7647, 7648, 7649, 7650, 7651, 7652, - 7653, 7654, 7655, 7656, 7657, 7658, 7659, 7660, 7661, 7662, 7663, 7664, - 7665, 7666, 7667, 7668, 7669, 7670, 7671, 7672, 7673, 7674, 7675, 7676, - 7677, 7678, 7679, 7680, 7681, 7682, 7683, 7684, 7685, 7686, 7687, 7688, - 7689, 7690, 7691, 7692, 7693, 7694, 7695, 7696, 7697, 7698, 7699, 7700, - 7701, 7702, 7703, 7704, 7705, 7706, 7707, 7708, 7709, 7710, 7711, 7712, - 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, 7721, 7722, 7723, 7724, - 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, 7733, 7734, 7735, 7736, - 7737, 7738, 7739, 7740, 7741, 7742, 7743, 7744, 7745, 7746, 7747, 7748, - 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756, 7757, 7758, 7759, 7760, - 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, 7769, 7770, 7771, 7772, - 7773, 7774, 7775, 7776, 7777, 7778, 7779, 7780, 7781, 7782, 7783, 7784, - 7785, 7786, 7787, 7788, 7789, 7790, 7791, 7792, 7793, 7794, 7795, 7796, - 7797, 7798, 7799, 7800, 7801, 7802, 7803, 7804, 7805, 7806, 7807, 7808, - 7809, 7810, 7811, 7812, 7813, 7814, 7815, 7816, 7817, 7818, 7819, 7820, - 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, - 7833, 7834, 7835, 7836, 7837, 7380, 7381, 7382, 7383, 7384, 7385, 7386, - 7387, 7388, 7389, 7390, 7391, 7392, 7393, 7394, 7395, 7396, 7397, 7398, - 7399, 7400, 7401, 7402, 7403, 7404, 7405, 7406, 7407, 7408, 7409, 7410, - 7411, 7412, 7413, 7414, 7415, 7416, 7417, 7418, 7419, 7420, 7421, 7422, - 7423, 7424, 7425, 7426, 7427, 7428, 7429, 7430, 7431, 7432, 7433, 7434, - 7435, 7436, 7437, 7438, 7439, 7440, 7441, 7442, 7443, 7444, 7445, 7446, - 7447, 7448, 7449, 7450, 7451, 7452, 7453, 7454, 7455, 7456, 7457, 7458, - 7459, 7460, 7461, 7462, 7463, 7464, 7465, 7466, 7467, 7468, 7469, 7470, - 7471, 7472, 7473, 7474, 7475, 7366, 7367, 7368, 7369, 7370, 7371, 7372, - 7373, 7374, 7375, 7376, 7377, 7378, 7379, 40951, 40951, 7476, 7477, 7478, - 7479, 7480, 7481, 7482, 7483, 7484, 7485, 7486, 7487, 7488, 7489, 7490, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 34311, 17490, 31931, 29628, 11221, 31502, 371, 1054, 24321, + 32155, 2570, 31753, 8585, 20621, 17673, 34150, 11004, 10989, 11001, + 10998, 10983, 10980, 10995, 10992, 11007, 10986, 8161, 32765, 24564, + 16926, 18304, 31929, 8584, 22906, 22952, 22962, 22978, 22995, 23040, + 23044, 23061, 23075, 23104, 23109, 23121, 23141, 23150, 23166, 23216, + 23225, 23228, 23249, 23273, 23302, 23341, 23352, 23361, 23364, 23378, + 24283, 32056, 32169, 6924, 25337, 18275, 23470, 23520, 23540, 23563, + 23599, 23658, 23666, 23684, 23703, 23740, 23746, 23760, 23799, 23812, + 23836, 23893, 23905, 23911, 23949, 23991, 24064, 24112, 24126, 24135, + 24143, 24159, 24224, 39287, 32118, 37693, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 29622, + 20975, 6233, 31795, 10083, 39859, 5039, 32747, 10971, 8779, 17569, 24331, + 29602, 34101, 31979, 27023, 10686, 31765, 34937, 34936, 4, 27754, 31650, + 27760, 6229, 34940, 25990, 32216, 39421, 39419, 39427, 20978, 22935, + 22936, 22924, 22938, 22917, 22919, 22940, 22963, 23026, 23027, 22998, + 23013, 23091, 23092, 23081, 23079, 23037, 23164, 23205, 23206, 23173, + 23191, 23180, 28467, 23189, 23333, 23334, 23303, 23306, 23373, 23294, + 23981, 23501, 23502, 23490, 23504, 23481, 23483, 23507, 23541, 23632, + 23633, 23602, 23617, 23721, 23722, 23704, 23706, 23657, 23831, 23875, + 23876, 23837, 23861, 23844, 11132, 23859, 24099, 24102, 24067, 24070, + 24154, 24013, 24145, 22922, 23488, 22907, 23471, 22932, 23498, 22967, + 23546, 22965, 23543, 22971, 23551, 22966, 23545, 22985, 23571, 22982, + 23565, 23016, 23621, 23029, 23635, 23008, 23612, 23023, 23629, 23007, + 23611, 23046, 23668, 23049, 23671, 23053, 23676, 23045, 23667, 23064, + 23687, 23071, 23696, 23085, 23718, 23083, 23713, 23094, 23724, 23088, + 23715, 23078, 23587, 23390, 24179, 23105, 23741, 23110, 23747, 23759, + 23135, 23777, 23131, 23764, 23133, 23767, 23128, 23774, 23130, 23785, + 23159, 23824, 23151, 23813, 23153, 23817, 23832, 23039, 23640, 23169, + 23854, 23208, 23878, 23186, 23850, 23391, 24182, 23237, 23928, 23229, + 23912, 23230, 23914, 23255, 23950, 23254, 23956, 23252, 23954, 23250, + 23952, 23274, 23992, 23277, 23996, 23284, 24008, 23330, 24096, 23322, + 24086, 23336, 24104, 23337, 24095, 23313, 24077, 23327, 24091, 23353, + 24127, 23365, 24144, 23366, 23381, 24160, 23384, 24166, 23380, 24163, + 23791, 23529, 22958, 22953, 23521, 23297, 24016, 23211, 22969, 23549, + 22944, 22990, 22986, 23566, 24021, 23246, 23272, 23210, 23041, 23661, + 23052, 23059, 23702, 23102, 23095, 23120, 23758, 23763, 23787, 23291, + 23157, 23818, 23172, 23198, 23868, 23213, 23890, 23222, 23901, 23467, + 23299, 24018, 23038, 23408, 24004, 23283, 24005, 23282, 23315, 24079, + 23340, 23345, 23370, 24149, 23387, 24169, 23031, 23033, 23650, 23655, + 23460, 23298, 24017, 23406, 23461, 23462, 23463, 23399, 23410, 22992, + 22980, 23593, 23140, 23129, 23798, 23165, 23154, 23835, 22930, 23496, + 23082, 23705, 23179, 23843, 23305, 24069, 23310, 24074, 23309, 24073, + 23307, 24071, 23308, 24072, 24053, 22918, 23482, 22914, 23478, 22942, + 23510, 23054, 23677, 23047, 23669, 23111, 23748, 23187, 23857, 23188, + 23858, 23032, 23652, 23743, 22991, 22979, 23592, 23048, 23670, 23073, + 23360, 23158, 23823, 22920, 23484, 22941, 23509, 23190, 23860, 22916, + 23480, 22937, 23503, 23012, 23616, 23028, 23634, 23076, 23712, 23093, + 23723, 23185, 23849, 23207, 23877, 23234, 23918, 23239, 23929, 23312, + 24076, 23335, 24103, 23253, 23955, 23276, 23994, 23377, 24158, 23065, + 23688, 23155, 23570, 23215, 23892, 23389, 24173, 22913, 23477, 22996, + 23600, 23181, 23845, 23194, 23864, 23182, 23846, 23183, 23847, 23374, + 24155, 23766, 23816, 23995, 23584, 23597, 23910, 22939, 22972, 23552, + 23123, 23278, 23963, 24168, 23055, 23678, 22959, 23339, 23293, 23030, + 23637, 23108, 23745, 23266, 23908, 23241, 23932, 23376, 24153, 24045, + 23512, 24046, 23528, 23881, 23544, 23567, 23575, 23941, 23972, 23976, + 23887, 23936, 23938, 23559, 23585, 23675, 23979, 23437, 23682, 23948, + 24022, 23695, 23701, 23725, 23736, 23415, 23772, 23761, 23780, 23788, + 24048, 24049, 23807, 23820, 23829, 23448, 23532, 23422, 23558, 23904, + 24033, 24036, 24039, 23923, 23925, 23919, 23939, 23424, 23416, 23969, + 23643, 23586, 23989, 23647, 24040, 24007, 24065, 24106, 24119, 24042, + 24057, 24050, 23455, 24172, 24162, 23649, 23651, 23464, 23407, 23404, + 23457, 23402, 23435, 23557, 23438, 23444, 23742, 24055, 23417, 23907, + 23465, 23409, 23594, 23580, 23595, 24060, 24009, 24059, 23664, 23795, + 23796, 23401, 23403, 24023, 24024, 28085, 28086, 28094, 28116, 28143, + 28146, 28041, 28163, 28165, 28014, 27957, 28171, 27867, 28022, 28024, + 28000, 27973, 28020, 28002, 28026, 28173, 27959, 27949, 6162, 28177, + 28003, 27866, 27972, 27998, 27989, 27995, 27994, 28170, 27980, 27901, + 27900, 28172, 27958, 28013, 28011, 5032, 11327, 32348, 30158, 34059, + 11386, 28027, 27950, 28084, 28096, 28123, 28164, 28120, 27964, 27979, + 28007, 27992, 27969, 28179, 28178, 28176, 28174, 27956, 27985, 27997, + 27987, 27990, 27991, 28010, 28009, 28008, 27993, 28019, 27869, 27970, + 27870, 27971, 28029, 28012, 27986, 8379, 8168, 8252, 8554, 8489, 8512, + 8178, 8278, 8274, 8393, 8538, 8288, 8184, 8570, 8310, 8311, 8186, 8396, + 8562, 8189, 8520, 8190, 8380, 8169, 8473, 8533, 8463, 8395, 8470, 8563, + 8314, 8517, 8500, 8516, 8521, 8281, 8275, 8537, 8191, 8254, 8510, 8569, + 8181, 8400, 8185, 8253, 8180, 8397, 8558, 8494, 8488, 8312, 8557, 8542, + 8486, 8541, 8485, 8528, 8398, 8546, 8551, 8582, 8571, 8298, 8381, 8170, + 8390, 8389, 8392, 8391, 8182, 8324, 8309, 8462, 8503, 8394, 8177, 8475, + 8565, 8385, 8524, 8471, 8326, 8581, 8464, 8525, 8523, 8529, 8280, 8174, + 8304, 8540, 8290, 8289, 8295, 8296, 8305, 8291, 8302, 8413, 8421, 8426, + 8434, 8437, 8419, 8451, 8453, 8455, 8440, 8443, 8458, 8459, 18489, 18753, + 18384, 18620, 18571, 18567, 18478, 18735, 41412, 41412, 18810, 18760, + 18761, 18759, 18815, 18492, 41412, 41412, 41412, 41412, 18775, 18505, + 18382, 18358, 18415, 18405, 18429, 41412, 18461, 41412, 18476, 18451, + 18667, 18361, 18488, 18486, 18484, 18406, 18490, 18385, 18482, 18416, + 18485, 18491, 18493, 18494, 18495, 18452, 18481, 18462, 41412, 18464, + 18483, 18467, 18479, 18487, 18480, 18431, 18421, 18472, 18617, 18632, + 18657, 18676, 18687, 18592, 18752, 18750, 18622, 18623, 18754, 18633, + 18747, 18658, 18698, 18755, 18756, 18757, 18758, 18725, 18738, 18739, + 18749, 18745, 18748, 18678, 18736, 18751, 18737, 18700, 18663, 18683, + 18734, 18696, 18724, 18500, 18812, 18771, 18779, 18777, 18778, 18587, + 18588, 18552, 18563, 18619, 18562, 18744, 18565, 18621, 18564, 18699, + 18561, 18742, 8660, 8754, 8638, 8732, 8634, 8728, 8632, 8726, 8630, 8724, + 8659, 8753, 8629, 8723, 18551, 18590, 18568, 18566, 18501, 18569, 18591, + 18466, 18746, 18496, 18465, 18743, 18589, 18498, 18499, 18497, 10350, + 10352, 10299, 10338, 10274, 10303, 10290, 10409, 10422, 10245, 10248, + 10262, 10377, 10347, 10390, 10307, 10275, 10291, 10423, 10332, 10311, + 10349, 10416, 10412, 10345, 10388, 10362, 10313, 10329, 10318, 10381, + 10249, 10324, 10327, 10259, 10269, 10331, 10339, 10265, 10292, 10395, + 10393, 10343, 10406, 10398, 10312, 10411, 10403, 10434, 10450, 10624, + 10491, 10470, 10508, 10617, 10613, 10504, 10566, 10521, 10472, 10488, + 10477, 10547, 10552, 10483, 10486, 10584, 10595, 10490, 10498, 10590, + 10451, 10573, 10571, 10502, 10607, 10576, 10471, 10612, 10604, 10509, + 10511, 10458, 10497, 10600, 10462, 10449, 10610, 10623, 10540, 10546, + 10587, 10536, 10506, 10568, 10466, 10382, 10548, 10405, 10606, 10358, + 10517, 10244, 10538, 10354, 10513, 10287, 10446, 10355, 10514, 10378, + 10537, 10252, 10556, 10421, 10622, 10360, 10519, 10361, 10520, 10273, + 10599, 10253, 10561, 10383, 10549, 10385, 10551, 10375, 10534, 10655, + 8245, 8239, 8242, 8240, 8241, 8195, 8248, 10389, 10567, 10402, 10580, + 10325, 10484, 10334, 10493, 10336, 10495, 10333, 10492, 10418, 10619, + 10414, 10615, 10363, 10522, 10365, 10524, 10366, 10525, 10285, 10444, + 10320, 10479, 10428, 10628, 10250, 10553, 10281, 10440, 10328, 10487, + 10261, 10586, 10400, 10578, 10401, 10579, 10340, 10499, 10427, 10627, + 10294, 10453, 10295, 10454, 10391, 10569, 10278, 10437, 10279, 10438, + 10431, 10419, 10620, 10364, 10523, 10316, 10475, 10323, 10482, 10322, + 10481, 10376, 10535, 10330, 10489, 10555, 10277, 10436, 10276, 10435, + 10426, 10626, 10351, 10510, 10386, 10564, 10387, 10565, 10417, 10618, + 10413, 10614, 10280, 10439, 10348, 10507, 10346, 10505, 10384, 10550, + 10283, 10442, 10284, 10443, 10326, 10485, 10272, 10598, 10271, 10597, + 10270, 10596, 10293, 10452, 10335, 10494, 10407, 10608, 10337, 10496, + 10341, 10500, 10342, 10501, 10369, 10528, 10368, 10527, 10374, 10533, + 10367, 10526, 10370, 10529, 10371, 10530, 10372, 10531, 10373, 10532, + 10257, 10560, 10317, 10476, 10246, 10541, 10258, 10563, 10404, 10605, + 10425, 10625, 10424, 10603, 10282, 10441, 10314, 10473, 10319, 10478, + 10251, 10554, 10392, 10570, 10321, 10480, 10305, 10464, 10309, 10468, + 10315, 10474, 41412, 2488, 2495, 2479, 2501, 2475, 2499, 2476, 2477, + 2466, 2498, 2494, 2491, 2493, 2471, 2483, 2500, 2481, 2478, 2469, 2496, + 2467, 2497, 2486, 2490, 2470, 2485, 2480, 2474, 2487, 2489, 2465, 2473, + 2472, 2468, 2484, 2482, 2502, 2492, 41412, 41412, 2552, 2464, 2505, 2504, + 2503, 2556, 2463, 2525, 2528, 2538, 2516, 2544, 2512, 2542, 2513, 2514, + 2527, 2541, 2537, 2534, 2536, 2508, 2520, 2543, 2518, 2515, 2506, 2539, + 2531, 2540, 2523, 2530, 2507, 2522, 2517, 2511, 2524, 2529, 2526, 2510, + 2509, 2533, 2521, 2519, 2545, 2535, 2546, 2532, 2555, 2553, 41412, 41412, + 32203, 24344, 2554, 41412, 19963, 19966, 19967, 19974, 19975, 19971, + 19978, 19976, 19961, 19973, 19970, 19954, 19955, 19956, 19964, 19969, + 19962, 19951, 19959, 19960, 19957, 19958, 19953, 19965, 19968, 19972, + 19979, 19980, 19952, 19977, 20058, 20073, 20062, 20060, 20061, 20063, + 20075, 20071, 20066, 20067, 20064, 20065, 20069, 20059, 20077, 20083, + 20070, 20080, 20072, 20074, 20081, 20057, 20056, 20082, 20068, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 19981, 19988, 20031, + 20029, 20006, 20037, 20011, 20008, 20023, 20048, 19997, 19991, 20033, + 20003, 20035, 20002, 20009, 20013, 19987, 19999, 19994, 20001, 20027, + 20004, 20021, 20015, 20025, 41412, 41412, 41412, 41412, 20084, 20052, + 20055, 20053, 20079, 20078, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 2306, 2340, 1099, 2341, 2342, 2305, + 2451, 2452, 2317, 2449, 2450, 2644, 1067, 1080, 2312, 2346, 2339, 2345, + 2337, 2338, 2347, 2365, 2353, 2382, 2349, 2401, 2400, 2333, 1387, 1089, + 2439, 2447, 1342, 1294, 1157, 1147, 1577, 1150, 1593, 1129, 1171, 1519, + 1518, 1541, 1320, 1279, 1361, 1196, 1536, 1440, 1620, 1474, 1487, 1466, + 1212, 1496, 1615, 1119, 1266, 1348, 1345, 1241, 1240, 1239, 2427, 1246, + 1431, 1331, 1366, 1379, 1403, 1296, 1572, 1165, 1584, 1097, 1078, 1109, + 1091, 1074, 1105, 2334, 2404, 2159, 1104, 1103, 2403, 2323, 2160, 2448, + 2443, 2442, 2444, 2318, 1092, 2446, 2459, 2461, 2458, 2457, 2454, 2453, + 2456, 2455, 2462, 2460, 2310, 1085, 2441, 1100, 1224, 1226, 1493, 1162, + 1154, 1153, 1316, 1317, 1319, 1558, 1318, 1546, 1552, 1191, 1527, 1526, + 1419, 1531, 1186, 1289, 1287, 1398, 1227, 1286, 1506, 1513, 1221, 1206, + 1203, 1204, 1209, 1218, 1232, 1199, 1205, 1457, 1443, 1454, 1451, 1444, + 1452, 1447, 1328, 1450, 1475, 1478, 1479, 1469, 1468, 1500, 1122, 1225, + 1249, 1247, 1565, 1251, 1426, 1434, 1435, 1343, 1495, 1337, 1336, 1388, + 1334, 1257, 1260, 1389, 1259, 1273, 1258, 1370, 1368, 1372, 1371, 1414, + 1404, 1460, 1412, 1409, 1307, 1508, 1304, 1297, 1298, 1522, 1581, 1355, + 1612, 1557, 1609, 1358, 1580, 1564, 1235, 1603, 1598, 1574, 1624, 1602, + 1585, 1588, 1101, 1170, 2356, 2355, 2358, 2357, 2384, 2364, 2362, 1090, + 2426, 2381, 2380, 2350, 2359, 2399, 2360, 2402, 2387, 2376, 2378, 2314, + 1088, 1087, 2322, 2398, 1198, 1448, 17510, 17512, 17509, 17508, 17505, + 17504, 17507, 17506, 17513, 17511, 1488, 1213, 1268, 2344, 2343, 1303, + 35067, 35141, 35138, 35139, 35135, 35076, 35063, 35064, 35140, 35137, + 35062, 35072, 35071, 35070, 41412, 35060, 35108, 35104, 35118, 35114, + 35115, 35078, 35077, 35079, 35121, 35119, 35080, 35111, 35112, 35116, + 35117, 35109, 35081, 35093, 35120, 35100, 35107, 35122, 35094, 35098, + 35106, 35110, 35099, 35105, 35113, 35095, 35097, 35096, 35127, 35126, + 35125, 35130, 35129, 35128, 35132, 35131, 35066, 35065, 35075, 35074, + 35073, 35069, 35068, 35133, 35147, 35148, 35134, 35145, 35144, 35143, + 35142, 35124, 35123, 35146, 35061, 41412, 41412, 35101, 35102, 35103, + 1177, 1180, 1175, 1176, 1178, 1179, 1173, 1288, 1285, 1202, 1197, 1445, + 1481, 1124, 1120, 1123, 1252, 1250, 1350, 1346, 1344, 1382, 1381, 1410, + 1407, 1408, 1373, 1446, 1449, 1480, 1284, 1282, 1477, 1441, 1283, 1156, + 1155, 1238, 1237, 1236, 1576, 1575, 1587, 1586, 1280, 1482, 1476, 1333, + 37261, 37274, 37270, 37287, 37286, 37265, 37262, 37250, 37281, 37266, + 37255, 37254, 37277, 37264, 37257, 37258, 37275, 37253, 37283, 37276, + 37288, 37269, 37268, 37267, 37279, 37260, 37263, 37278, 37284, 37273, + 37272, 37252, 37280, 37285, 37251, 37259, 37256, 37282, 37246, 37245, + 37292, 37248, 37293, 37291, 37247, 37249, 37290, 37289, 37294, 37271, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 29486, 29488, 29485, 29484, 29481, 29480, + 29483, 29482, 29489, 29487, 29521, 29508, 29522, 29507, 29523, 29505, + 29504, 29492, 29497, 29510, 29516, 29518, 29496, 29506, 29491, 29503, + 29502, 29517, 29509, 29511, 29513, 29514, 29499, 29515, 29500, 29498, + 29512, 29519, 29520, 29501, 29494, 29493, 29495, 29474, 29475, 29476, + 29472, 29469, 29470, 29471, 29473, 29468, 29525, 29524, 29527, 29526, + 29477, 29528, 29490, 41412, 41412, 29478, 29479, 29529, 32575, 32562, + 32576, 32564, 32567, 32563, 32578, 32566, 32573, 32582, 32568, 32569, + 32579, 32581, 32570, 32565, 32583, 32574, 32580, 32577, 32571, 32572, + 32585, 32586, 32589, 32584, 32590, 32587, 32609, 32619, 32614, 32608, + 32618, 32613, 32607, 32617, 32591, 32615, 32611, 32621, 32592, 32610, + 32620, 32612, 32616, 32588, 41412, 41412, 32599, 32593, 32595, 32598, + 32596, 32600, 32622, 32605, 32603, 32606, 32602, 32604, 32597, 32601, + 32594, 41412, 25779, 25766, 25768, 25767, 25769, 25778, 25776, 25781, + 25761, 25759, 25758, 25770, 25771, 25772, 25762, 25780, 25773, 25764, + 25774, 25775, 25763, 25760, 25777, 25782, 25765, 25757, 25783, 25784, + 41412, 41412, 25785, 41412, 35086, 35091, 35087, 35089, 35085, 35084, + 35088, 35092, 35083, 35082, 35090, 41412, 41412, 41412, 41412, 41412, + 1143, 1133, 1144, 1160, 1142, 1130, 1139, 1136, 1140, 1138, 1161, 1135, + 1146, 1132, 1134, 1145, 1131, 1137, 1141, 2428, 2429, 2431, 1539, 1064, + 2316, 1411, 1281, 1501, 1499, 1347, 2445, 1413, 2313, 2315, 41412, 41412, + 41412, 41412, 41412, 2311, 2366, 2392, 2391, 2394, 2158, 2408, 1084, + 1102, 1174, 1181, 1321, 1498, 1248, 1432, 1367, 1380, 1599, 1601, 1453, + 1573, 1465, 1378, 1200, 1467, 1261, 1494, 1623, 1121, 1335, 1433, 1172, + 1420, 1524, 1442, 1600, 1117, 1115, 1118, 1421, 1525, 1547, 1507, 1349, + 1267, 1116, 1323, 1322, 1369, 1278, 2348, 2351, 2377, 2372, 2383, 1113, + 1112, 2407, 1114, 1111, 2397, 2367, 2363, 2386, 2385, 2379, 2390, 2368, + 2370, 2369, 2371, 2374, 2373, 2352, 2361, 1086, 2440, 1070, 1068, 1072, + 1071, 1069, 1073, 2434, 2436, 2438, 2433, 2435, 2437, 2309, 2308, 2307, + 2375, 1094, 1093, 1106, 1630, 2319, 1629, 2321, 1081, 1082, 2320, 1077, + 2161, 10905, 10895, 10909, 10914, 10830, 10804, 10805, 10874, 10875, + 10850, 10851, 10869, 10871, 10812, 10831, 10882, 10806, 10813, 10832, + 10845, 10807, 10841, 10840, 10825, 10823, 10856, 10810, 10814, 10861, + 10859, 10857, 10866, 10865, 10818, 10817, 10855, 10868, 10867, 10820, + 10819, 10858, 10854, 10877, 10876, 10838, 10837, 10828, 10849, 10844, + 10843, 10864, 10863, 10862, 10873, 10833, 10834, 10835, 10827, 10927, + 10926, 10907, 10908, 10917, 10939, 10940, 10929, 10930, 10935, 10936, + 10923, 10933, 10941, 10918, 10924, 10934, 10925, 10919, 10913, 10928, + 10920, 10955, 10916, 10915, 10799, 10796, 10922, 10932, 10931, 10881, + 10839, 10822, 10879, 10815, 10842, 10880, 10848, 10870, 10872, 10937, + 10938, 10943, 10942, 10950, 10952, 10949, 10948, 10945, 10944, 10947, + 10946, 10953, 10951, 10797, 10912, 10811, 10847, 10846, 10808, 10853, + 10852, 10829, 10878, 10826, 10824, 10860, 10821, 10816, 10836, 3599, + 3667, 3670, 3672, 41412, 3623, 3624, 3637, 3638, 3635, 3636, 3617, 3619, + 41412, 41412, 3659, 3625, 41412, 41412, 3660, 3626, 3610, 3607, 3651, + 3650, 3639, 3649, 3648, 3653, 3652, 3641, 3632, 3631, 3628, 3627, 3640, + 3634, 3633, 3630, 3629, 3642, 41412, 3655, 3654, 3647, 3646, 3658, 3622, + 3611, 41412, 3657, 41412, 41412, 41412, 3643, 3644, 3645, 3656, 41412, + 41412, 3668, 3669, 3674, 3683, 3684, 3677, 3678, 3679, 3680, 41412, + 41412, 3685, 3675, 41412, 41412, 3686, 3676, 3671, 3608, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 3600, 41412, 41412, 41412, + 41412, 3615, 3614, 41412, 3621, 3618, 3620, 3681, 3682, 41412, 41412, + 3693, 3695, 3692, 3691, 3688, 3687, 3690, 3689, 3696, 3694, 3613, 3612, + 3662, 3661, 3601, 3605, 3604, 3603, 3602, 3606, 3673, 3697, 3616, 3598, + 3666, 41412, 41412, 19044, 19045, 19050, 41412, 18996, 18997, 19012, + 19013, 19010, 19011, 41412, 41412, 41412, 41412, 19031, 18998, 41412, + 41412, 19030, 18999, 18995, 18994, 18992, 18991, 19016, 19023, 19022, + 19025, 19024, 19018, 19007, 19006, 19001, 19000, 19017, 19009, 19008, + 19003, 19002, 19019, 41412, 19027, 19026, 19021, 19020, 19034, 19036, + 19005, 41412, 19015, 19014, 41412, 19035, 19028, 41412, 19029, 19033, + 41412, 41412, 19047, 41412, 19051, 19056, 19057, 19054, 19055, 41412, + 41412, 41412, 41412, 19059, 19052, 41412, 41412, 19058, 19053, 19049, + 41412, 41412, 41412, 19046, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 18993, 18990, 19037, 19004, 41412, 19032, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 19069, 19071, 19068, 19067, 19064, 19063, + 19066, 19065, 19072, 19070, 19060, 18989, 19061, 19073, 19062, 19048, + 18988, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 18884, 18892, 18894, 41412, 18834, 18835, 18853, 18854, 18851, + 18852, 18846, 18848, 18910, 41412, 18881, 18836, 18911, 41412, 18882, + 18837, 18874, 18873, 18870, 18869, 18858, 18868, 18867, 18872, 18871, + 18860, 18843, 18842, 18839, 18838, 18859, 18845, 18844, 18841, 18840, + 18861, 41412, 18876, 18875, 18866, 18865, 18878, 18880, 18879, 41412, + 18856, 18855, 41412, 18850, 18862, 18863, 18864, 18877, 41412, 41412, + 18890, 18891, 18897, 18906, 18907, 18900, 18901, 18902, 18903, 18895, + 41412, 18908, 18898, 18896, 41412, 18909, 18899, 18893, 41412, 41412, + 18924, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 18847, 18849, 18904, 18905, + 41412, 41412, 18920, 18922, 18919, 18918, 18915, 18914, 18917, 18916, + 18923, 18921, 18912, 18913, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 18857, 18889, 18888, 18885, 18887, 18883, 18886, 41412, 30802, + 30805, 30808, 41412, 30753, 30754, 30772, 30773, 30770, 30771, 30765, + 30767, 41412, 41412, 30798, 30755, 41412, 41412, 30799, 30756, 30792, + 30791, 30788, 30787, 30776, 30786, 30785, 30790, 30789, 30778, 30762, + 30761, 30758, 30757, 30777, 30764, 30763, 30760, 30759, 30779, 41412, + 30794, 30793, 30784, 30783, 30796, 30752, 30750, 41412, 30775, 30774, + 41412, 30769, 30780, 30781, 30782, 30795, 41412, 41412, 30803, 30804, + 30809, 30818, 30819, 30812, 30813, 30814, 30815, 41412, 41412, 30820, + 30810, 41412, 41412, 30821, 30811, 30807, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 30806, 30739, 30740, 41412, 41412, 41412, 41412, + 30749, 30748, 41412, 30751, 30766, 30768, 30816, 30817, 41412, 41412, + 30828, 30830, 30827, 30826, 30823, 30822, 30825, 30824, 30831, 30829, + 30747, 30797, 30744, 30743, 30746, 30741, 30742, 30745, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 35760, 35777, + 41412, 35723, 35724, 35736, 35737, 35732, 35733, 41412, 41412, 41412, + 35741, 35742, 35725, 41412, 35734, 35735, 35726, 35746, 41412, 41412, + 41412, 35718, 35743, 41412, 35745, 41412, 35719, 35721, 41412, 41412, + 41412, 35717, 35722, 41412, 41412, 41412, 35720, 35716, 35748, 41412, + 41412, 41412, 35747, 35750, 35731, 35730, 35729, 35728, 35727, 35749, + 35738, 35739, 35740, 35744, 41412, 41412, 41412, 41412, 36050, 36057, + 36058, 36053, 36054, 41412, 41412, 41412, 36059, 36060, 36051, 41412, + 36055, 36056, 36052, 35776, 41412, 41412, 36063, 41412, 41412, 41412, + 41412, 41412, 41412, 35652, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 35689, 35691, + 35688, 35687, 35684, 35683, 35686, 35685, 35692, 35690, 35754, 35752, + 35753, 35682, 36062, 36061, 35681, 35679, 35651, 35757, 35755, 41412, + 41412, 41412, 41412, 41412, 37126, 37127, 37132, 37134, 37125, 37086, + 37087, 37102, 37103, 37098, 37099, 37093, 37095, 41412, 37119, 37120, + 37088, 41412, 37100, 37101, 37089, 37116, 37115, 37112, 37111, 37075, + 37110, 37109, 37114, 37113, 37077, 37082, 37081, 37069, 37068, 37076, + 37085, 37083, 37072, 37070, 37073, 41412, 37118, 37117, 37108, 37107, + 37122, 37123, 37080, 37079, 37092, 37091, 37090, 37097, 37104, 37105, + 37106, 37121, 41412, 41412, 37130, 37131, 37135, 37146, 37147, 37138, + 37139, 37140, 37141, 41412, 37148, 37149, 37136, 41412, 37144, 37145, + 37137, 37133, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 37124, + 37060, 41412, 37084, 37071, 37078, 41412, 37059, 37074, 41412, 41412, + 37094, 37096, 37142, 37143, 41412, 41412, 37156, 37158, 37155, 37154, + 37151, 37150, 37153, 37152, 37159, 37157, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 37128, 37067, 37065, 37063, 37061, 37066, 37064, + 37062, 37129, 21436, 21435, 21440, 21444, 21437, 21384, 21385, 21410, + 21411, 21406, 21407, 21401, 21403, 41412, 21427, 21428, 21386, 41412, + 21408, 21409, 21387, 21424, 21423, 21420, 21419, 21381, 21418, 21417, + 21422, 21421, 21383, 21398, 21397, 21389, 21388, 21382, 21400, 21399, + 21391, 21390, 21379, 41412, 21426, 21425, 21416, 21415, 21431, 21432, + 21396, 21395, 21394, 21393, 41412, 21405, 21412, 21413, 21414, 21430, + 41412, 41412, 21438, 21439, 21447, 21458, 21459, 21450, 21451, 21452, + 21453, 41412, 21460, 21461, 21448, 41412, 21456, 21457, 21449, 21443, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 21433, 21446, 41412, + 41412, 41412, 41412, 41412, 21445, 21380, 21429, 41412, 21402, 21404, + 21454, 21455, 41412, 41412, 21468, 21470, 21467, 21466, 21463, 21462, + 21465, 21464, 21471, 21469, 41412, 21441, 21442, 21434, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 25719, 25721, 25726, 25724, 25676, 25650, 25652, 25696, 25697, 25692, + 25693, 25677, 25679, 41412, 25711, 25712, 25653, 41412, 25694, 25695, + 25654, 25708, 25707, 25704, 25703, 25684, 25665, 25664, 25706, 25705, + 25685, 25673, 25671, 25668, 25667, 25683, 25675, 25674, 25670, 25669, + 25686, 25682, 25710, 25709, 25702, 25701, 25714, 25715, 25691, 25690, + 25689, 25688, 25687, 25681, 25698, 25699, 25700, 25713, 25672, 25722, + 25720, 25725, 25728, 25739, 25740, 25731, 25732, 25733, 25734, 41412, + 25741, 25742, 25729, 41412, 25737, 25738, 25730, 25723, 25666, 25727, + 41412, 41412, 41412, 41412, 25662, 25663, 25657, 25743, 25642, 25640, + 25647, 25637, 25638, 25648, 25641, 25651, 25678, 25680, 25735, 25736, + 41412, 41412, 25633, 25635, 25632, 25631, 25628, 25627, 25630, 25629, + 25636, 25634, 25718, 25716, 25717, 25645, 25644, 25649, 25639, 25643, + 25646, 25626, 25659, 25658, 25660, 25655, 25656, 25661, 41412, 33959, + 33958, 33960, 41412, 33904, 33901, 33889, 33888, 33912, 33911, 33914, + 33913, 33910, 33909, 33908, 33907, 33906, 33905, 33902, 33933, 33932, + 33903, 41412, 41412, 41412, 33898, 33923, 33896, 33921, 33946, 33936, + 33895, 33920, 33897, 33922, 33943, 33944, 33937, 33890, 33915, 33892, + 33917, 33927, 33934, 33891, 33916, 33893, 33918, 33930, 41412, 33935, + 33899, 33924, 33894, 33919, 33925, 33900, 33942, 33940, 41412, 33929, + 41412, 41412, 33941, 33945, 33928, 33931, 33939, 33926, 33938, 41412, + 41412, 41412, 33957, 41412, 41412, 41412, 41412, 33977, 33969, 33964, + 33970, 33965, 33971, 41412, 33966, 41412, 33967, 33972, 33963, 33976, + 33973, 33974, 33975, 33968, 41412, 41412, 41412, 41412, 41412, 41412, + 33953, 33955, 33952, 33951, 33948, 33947, 33950, 33949, 33956, 33954, + 41412, 41412, 33961, 33962, 33978, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 37313, 37310, 37308, + 37307, 37309, 37311, 37327, 37296, 37298, 37297, 37356, 37299, 37368, + 37300, 37365, 37361, 37358, 37359, 37329, 37301, 37364, 37363, 37360, + 37362, 37330, 37295, 37336, 37333, 37302, 37334, 37303, 37335, 37325, + 37369, 37337, 37338, 37315, 37317, 37366, 37354, 37353, 37355, 37306, + 37314, 37370, 37305, 37331, 37339, 37319, 37342, 37344, 37349, 37350, + 37346, 37347, 37345, 37348, 37332, 41412, 41412, 41412, 41412, 37371, + 37351, 37343, 37352, 37341, 37340, 37316, 37324, 37323, 37322, 37320, + 37321, 37318, 37357, 37328, 37367, 37304, 37378, 37380, 37377, 37376, + 37373, 37372, 37375, 37374, 37381, 37379, 37326, 37312, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 22743, 22741, 41412, 22742, 41412, + 22756, 22771, 22775, 22755, 22765, 41412, 22757, 22772, 22753, 22751, + 22750, 22748, 22747, 22752, 22776, 22768, 22766, 22767, 22749, 22773, + 22774, 22761, 22759, 22738, 22760, 22737, 22754, 22777, 22780, 22745, + 41412, 22746, 41412, 22779, 22762, 22763, 22764, 22769, 22758, 22781, + 22770, 22807, 22789, 22794, 22790, 22792, 22802, 22803, 22796, 22797, + 22798, 22799, 22784, 22795, 22783, 22782, 41412, 41412, 22800, 22801, + 22804, 22793, 22791, 41412, 22805, 41412, 22788, 22785, 22786, 22787, + 22818, 22819, 22806, 41412, 22814, 22816, 22813, 22812, 22809, 22808, + 22811, 22810, 22817, 22815, 41412, 41412, 22734, 22733, 22740, 22739, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 37605, 37512, 37510, 37511, 37520, 37508, 37505, 37509, + 37529, 37500, 37498, 37521, 37536, 37530, 37526, 37533, 37525, 37528, + 37527, 37504, 37513, 37496, 37495, 37423, 37424, 37422, 37544, 37545, + 37546, 37548, 37549, 37547, 37445, 37447, 37444, 37443, 37440, 37439, + 37442, 37441, 37448, 37446, 37437, 37434, 37433, 37430, 37429, 37432, + 37431, 37438, 37436, 37435, 37501, 37523, 37503, 37522, 37506, 37532, + 37514, 37515, 37516, 37517, 37555, 37541, 37454, 37452, 37482, 37481, + 37464, 37480, 37479, 37489, 41412, 37466, 37474, 37473, 37459, 37458, + 37465, 37476, 37475, 37463, 37462, 37467, 37484, 37483, 37478, 37477, + 37491, 37472, 37471, 37461, 37460, 37492, 37485, 37486, 37487, 37493, + 37456, 37490, 37468, 37469, 37470, 37488, 37494, 37451, 37457, 37453, + 37455, 41412, 41412, 41412, 41412, 37629, 37625, 37626, 37617, 37618, + 37619, 37620, 37621, 37622, 37627, 37628, 37623, 37624, 37552, 37553, + 37615, 37616, 37543, 37557, 37518, 37535, 37539, 37554, 37540, 37542, + 37537, 37538, 37556, 37604, 37603, 37602, 37569, 37568, 37588, 37587, + 37570, 37586, 37585, 37595, 41412, 37572, 37580, 37579, 37562, 37561, + 37571, 37582, 37581, 37566, 37565, 37573, 37590, 37589, 37584, 37583, + 37597, 37578, 37577, 37564, 37563, 37599, 37591, 37592, 37593, 37600, + 37598, 37596, 37574, 37575, 37576, 37594, 37601, 37567, 37559, 37560, + 37558, 41412, 37449, 37450, 37426, 37427, 37428, 37425, 37606, 37613, + 37611, 37614, 37612, 37607, 37610, 37609, 37608, 41412, 37551, 37550, + 37499, 37502, 37524, 37519, 37507, 32208, 24349, 32209, 24350, 37534, + 37531, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 28762, 28741, + 28740, 28739, 28771, 28839, 28838, 28841, 28840, 28772, 28769, 28817, + 28816, 28823, 28822, 28770, 28801, 28818, 28825, 28824, 28773, 28843, + 28842, 28837, 28836, 28768, 28845, 28775, 28835, 28821, 28800, 28844, + 28834, 28731, 28795, 28832, 28833, 28826, 28827, 28734, 28767, 28846, + 28733, 28938, 28922, 28945, 28946, 28939, 28940, 28934, 28919, 28927, + 28928, 28935, 28863, 28882, 28887, 28886, 28862, 28726, 28724, 28725, + 28723, 28738, 28963, 28965, 28962, 28961, 28958, 28957, 28960, 28959, + 28966, 28964, 28885, 28874, 28892, 28896, 28891, 28895, 28776, 28799, + 28828, 28829, 28830, 28831, 28941, 28942, 28943, 28944, 28766, 28765, + 28763, 28764, 28729, 28728, 28727, 28798, 28933, 28907, 28908, 28820, + 28819, 28936, 28937, 28877, 28878, 28879, 28880, 28881, 28737, 28736, + 28735, 28923, 28925, 28926, 28924, 28790, 28789, 28788, 28786, 28794, + 28778, 28791, 28779, 28781, 28792, 28784, 28782, 28793, 28730, 28932, + 28929, 28930, 28931, 28869, 28870, 28871, 28872, 28867, 28868, 28866, + 28774, 28883, 28858, 28860, 28857, 28856, 28853, 28852, 28855, 28854, + 28861, 28859, 28864, 28865, 28920, 28921, 28894, 28893, 17885, 17911, + 17896, 17917, 17918, 17916, 17906, 17907, 17919, 17900, 17909, 17912, + 17914, 17920, 17902, 17905, 17910, 17904, 17908, 17921, 17901, 17899, + 17895, 17915, 17903, 17891, 17894, 17898, 17893, 17892, 17913, 17897, + 17886, 17890, 17888, 17923, 17887, 17889, 41412, 17922, 41412, 41412, + 41412, 41412, 41412, 17884, 41412, 41412, 17968, 17999, 17976, 18005, + 17974, 18004, 17997, 17994, 18006, 17986, 17988, 18000, 18002, 18007, + 17990, 17996, 17998, 17992, 17995, 17965, 17989, 17985, 17975, 18003, + 17991, 17969, 17972, 17984, 17971, 17970, 18001, 17983, 17979, 17982, + 17980, 18009, 17977, 17981, 18010, 18008, 17973, 17993, 17967, 18057, + 27975, 17966, 17978, 17987, 19305, 19379, 19313, 19384, 19377, 19341, + 19308, 19323, 19381, 19355, 19373, 19287, 19285, 19371, 19272, 19307, + 19391, 19320, 19394, 19315, 19380, 19317, 19316, 19388, 19351, 19375, + 19354, 19300, 19310, 19304, 19333, 19338, 19337, 19324, 19328, 19326, + 19329, 19330, 19327, 19335, 19334, 19336, 19331, 19302, 19303, 19362, + 19368, 19367, 19360, 19365, 19356, 19357, 19359, 19370, 19364, 19363, + 19361, 19366, 19358, 19369, 19277, 19276, 19282, 19281, 19340, 19297, + 19296, 19294, 19289, 19299, 19290, 19383, 19293, 19292, 19295, 19288, + 19392, 19286, 19279, 19275, 19284, 19280, 19273, 19274, 19278, 19283, + 19321, 19301, 19382, 19393, 19306, 19319, 19314, 19318, 19385, 19396, + 19634, 19540, 19550, 19598, 19602, 19552, 19551, 19604, 19603, 19579, + 19631, 19632, 19589, 19610, 19590, 19630, 19629, 19633, 19619, 19556, + 19608, 19563, 19548, 19549, 19600, 19599, 19554, 19555, 19553, 19606, + 19607, 19587, 19586, 19582, 19580, 19588, 19611, 19612, 19613, 19618, + 19617, 19595, 19596, 19594, 19591, 19597, 19624, 19623, 19622, 19621, + 19620, 19628, 19626, 19562, 19559, 19609, 19564, 19566, 19573, 19577, + 19575, 19565, 19541, 19543, 19546, 19545, 19578, 19547, 19601, 19605, + 19584, 19585, 19416, 19517, 19419, 19439, 19442, 19446, 19521, 19467, + 19468, 19473, 19477, 19486, 19489, 19482, 19494, 19426, 19456, 19459, + 19495, 19512, 19408, 19399, 19402, 19425, 19530, 19452, 19403, 19418, + 19420, 19445, 19444, 19448, 19447, 19443, 19528, 19522, 19470, 19493, + 19487, 19488, 19507, 19474, 19476, 19481, 19480, 19471, 19485, 19483, + 19472, 19490, 19436, 19433, 19427, 19432, 19431, 19429, 19434, 19438, + 19415, 19457, 19461, 19466, 19414, 19497, 19505, 19498, 19501, 19449, + 19410, 19411, 19520, 19409, 19531, 19535, 19538, 19454, 19413, 19406, + 19404, 19405, 19407, 19539, 19422, 19423, 19421, 19417, 19424, 19518, + 17155, 17162, 17161, 17156, 17160, 17159, 17157, 17158, 17382, 17390, + 17389, 17383, 17387, 17386, 17384, 17388, 17148, 17154, 17152, 17149, + 17151, 17150, 17153, 17139, 17199, 17207, 17206, 17200, 17204, 17203, + 17201, 17197, 17311, 17318, 17316, 17312, 17314, 17313, 17317, 17315, + 17286, 17295, 17294, 17287, 17291, 17290, 17288, 17292, 17326, 17332, + 17331, 17327, 17301, 17296, 17328, 17330, 17302, 17310, 17309, 17303, + 17307, 17306, 17304, 17308, 17278, 17285, 17284, 17279, 17283, 17282, + 17280, 17281, 17266, 41412, 17270, 17267, 17269, 17268, 41412, 41412, + 17259, 17265, 17263, 17260, 17262, 17261, 17264, 41412, 17254, 41412, + 17258, 17255, 17257, 17256, 41412, 41412, 16989, 16996, 16995, 16990, + 16994, 16993, 16991, 16980, 17451, 17458, 17456, 17452, 17454, 17453, + 17457, 17455, 17364, 17372, 17371, 17365, 17369, 17368, 17366, 17370, + 17027, 17035, 17034, 17028, 17032, 17031, 17029, 17033, 17419, 17426, + 17425, 17420, 17424, 17423, 17421, 17422, 17407, 41412, 17411, 17408, + 17410, 17409, 41412, 41412, 17217, 17225, 17224, 17218, 17222, 17221, + 17219, 17223, 17208, 17216, 17215, 17209, 17213, 17212, 17210, 17214, + 17109, 17117, 17116, 17110, 17114, 17113, 17111, 17115, 17187, 17194, + 17193, 17188, 17192, 17191, 17189, 17190, 17175, 41412, 17179, 17176, + 17178, 17177, 41412, 41412, 17168, 17174, 17172, 17169, 17171, 17170, + 17173, 41412, 17163, 41412, 17167, 17164, 17166, 17165, 41412, 41412, + 17391, 17398, 17397, 17392, 17396, 17395, 17393, 17394, 17227, 17233, + 17231, 17228, 17230, 17229, 17232, 41412, 17442, 17450, 17449, 17443, + 17447, 17446, 17444, 17448, 17427, 17434, 17432, 17428, 17430, 17429, + 17433, 17431, 17399, 17406, 17405, 17400, 17404, 17403, 17401, 17402, + 17057, 17065, 17064, 17058, 17062, 17061, 17059, 17063, 17042, 17050, + 17049, 17043, 17047, 17046, 17044, 17048, 17373, 17381, 17380, 17374, + 17378, 17377, 17375, 17379, 17130, 17078, 17136, 17131, 17135, 17134, + 17132, 17133, 17118, 41412, 17122, 17119, 17121, 17120, 41412, 41412, + 17102, 17108, 17106, 17103, 17105, 17104, 17107, 17098, 17333, 17341, + 17340, 17334, 17338, 17337, 17335, 17339, 17018, 17026, 17025, 17019, + 17023, 17022, 17020, 17024, 17226, 17241, 17240, 17234, 17238, 17237, + 17235, 17239, 17356, 17363, 17361, 17357, 17359, 17358, 17362, 17360, + 17348, 17355, 17354, 17349, 17353, 17352, 17350, 17351, 17070, 17077, + 17075, 17071, 17073, 17072, 17076, 17068, 17246, 17253, 17252, 17247, + 17251, 17250, 17248, 17244, 17293, 17205, 17074, 41412, 41412, 16958, + 16960, 16959, 16977, 17480, 17478, 16961, 16976, 16962, 16975, 17479, + 16974, 17476, 17474, 17473, 17470, 17469, 17472, 17471, 17477, 17475, + 16963, 16966, 16965, 16970, 16969, 16973, 16972, 16968, 16971, 16967, + 16964, 41412, 41412, 41412, 17299, 17198, 17196, 17195, 17297, 16981, + 16979, 16978, 17298, 17069, 17067, 17066, 17300, 17245, 17243, 17242, + 17468, 17459, 17465, 17466, 17461, 17463, 17467, 17462, 17460, 17464, + 41412, 41412, 41412, 41412, 41412, 41412, 6484, 6485, 6486, 6487, 6488, + 6489, 6447, 6483, 6448, 6449, 6450, 6451, 6452, 6412, 6413, 6414, 6415, + 6416, 6417, 6453, 6454, 6455, 6456, 6457, 6458, 6459, 6460, 6461, 6462, + 6463, 6418, 6411, 6419, 6420, 6421, 6422, 6423, 6424, 6465, 6466, 6467, + 6468, 6469, 6470, 6426, 6425, 6427, 6428, 6429, 6430, 6431, 6405, 6444, + 6406, 6445, 6407, 6446, 6408, 6409, 6410, 6404, 6432, 6433, 6434, 6435, + 6436, 6437, 6438, 6439, 6440, 6441, 6442, 6443, 6471, 6472, 6473, 6474, + 6475, 6476, 6477, 6478, 6479, 6480, 6481, 6482, 6464, 41412, 41412, 6564, + 6565, 6566, 6567, 6568, 6550, 41412, 41412, 5626, 5598, 5371, 5628, 5629, + 5778, 5792, 6075, 5582, 5442, 5369, 5370, 5952, 6042, 6062, 6040, 6063, + 6041, 6044, 6038, 6045, 6039, 5707, 6059, 6036, 6060, 6037, 5708, 5373, + 6076, 6094, 5615, 5614, 5616, 5617, 5609, 5610, 5607, 5606, 5619, 5613, + 5618, 5608, 5600, 5630, 5791, 5377, 5812, 5805, 5810, 5811, 5807, 5808, + 6066, 5440, 5441, 5803, 5804, 5802, 5981, 5800, 5979, 5801, 5980, 5795, + 5977, 5796, 5978, 5798, 5975, 5799, 5976, 6065, 5794, 5974, 5433, 5951, + 5934, 5949, 5950, 5947, 5948, 6073, 5404, 5417, 5932, 5933, 5942, 6034, + 5940, 6032, 5941, 6033, 5938, 6030, 5939, 6031, 5936, 6028, 5937, 6029, + 5713, 5894, 5929, 5930, 5931, 5928, 5649, 5643, 5647, 5648, 5645, 5646, + 6068, 5641, 5642, 5640, 6026, 5638, 6024, 5639, 6025, 5636, 6022, 5637, + 6023, 5633, 6020, 5634, 6021, 5710, 5631, 5632, 5874, 5875, 5876, 5873, + 5597, 5584, 5595, 5596, 5593, 5594, 6067, 5401, 5583, 5591, 6019, 5589, + 6017, 5590, 6018, 5587, 6015, 5588, 6016, 5585, 6013, 5586, 6014, 5709, + 5400, 5850, 5675, 5683, 5692, 5693, 5678, 5679, 6070, 5681, 5682, 5691, + 5973, 5689, 5971, 5690, 5972, 5687, 5969, 5688, 5970, 5685, 5967, 5686, + 5968, 5711, 5674, 5966, 5694, 5375, 5851, 5767, 5728, 5765, 5766, 5755, + 5756, 6071, 5699, 5727, 5764, 5992, 5758, 5990, 5759, 5991, 5712, 5695, + 5473, 5768, 5673, 5660, 5671, 5672, 5669, 5670, 6069, 5658, 5659, 5668, + 5960, 5666, 5958, 5667, 5959, 5664, 5956, 5665, 5957, 5662, 5954, 5663, + 5955, 5650, 5953, 5676, 5893, 5853, 5891, 5892, 5872, 5877, 6072, 5833, + 5852, 5886, 6012, 5884, 6010, 5885, 6011, 5882, 6008, 5883, 6009, 5880, + 6006, 5881, 6007, 5705, 5832, 5376, 5879, 5396, 5680, 5703, 5706, 5701, + 5702, 5704, 5700, 5871, 5869, 5870, 5863, 5864, 5866, 5867, 5862, 6005, + 5860, 6003, 5861, 6004, 5855, 6001, 5856, 6002, 5858, 5999, 5859, 6000, + 5854, 6093, 6079, 6091, 6092, 6081, 6082, 6074, 6077, 6078, 6090, 5989, + 6088, 5987, 6089, 5988, 6086, 5985, 6087, 5986, 6084, 5983, 6085, 5984, + 5714, 6064, 5397, 5982, 5849, 5831, 5815, 5965, 5825, 5829, 5830, 5827, + 5828, 5963, 5823, 5824, 5961, 5817, 5994, 5813, 5993, 5677, 5625, 5604, + 5605, 5620, 5622, 5623, 5602, 5603, 5624, 6035, 5601, 5913, 5698, 5911, + 5696, 5912, 5697, 5909, 5910, 5907, 5908, 5905, 6027, 5895, 5926, 5927, + 5923, 5921, 5920, 5944, 5945, 5946, 5943, 5753, 5751, 5752, 5749, 5750, + 5747, 5748, 5746, 5754, 5627, 5772, 5776, 5777, 5774, 5775, 5770, 5771, + 5769, 5918, 5919, 5914, 5917, 5996, 5997, 5998, 5995, 5733, 5737, 5738, + 5735, 5736, 5731, 5732, 5730, 5739, 5847, 5848, 5843, 5846, 6055, 6056, + 6057, 6054, 6046, 5656, 5657, 5654, 5655, 5652, 5653, 5651, 5903, 5901, + 5902, 5899, 5900, 5897, 5898, 5896, 5374, 5393, 5394, 5395, 5392, 5381, + 5382, 5383, 5380, 5389, 5390, 5391, 5388, 5385, 5386, 5387, 5384, 5838, + 5839, 5835, 5837, 5423, 5422, 5418, 5419, 5421, 5420, 5569, 5568, 5564, + 5565, 5567, 5566, 5575, 5574, 5570, 5571, 5573, 5572, 5439, 5438, 5434, + 5435, 5437, 5436, 5533, 5532, 5528, 5529, 5531, 5530, 5527, 5526, 5522, + 5523, 5525, 5524, 5496, 5495, 5491, 5492, 5494, 5493, 5490, 5432, 5431, + 5428, 5429, 5430, 5426, 5469, 5468, 5464, 5465, 5467, 5466, 5463, 5462, + 5458, 5459, 5461, 5460, 5457, 5476, 5475, 5470, 5471, 5474, 5472, 5563, + 5562, 5558, 5559, 5561, 5560, 5581, 5580, 5576, 5577, 5579, 5578, 5456, + 5840, 5455, 5450, 5451, 5454, 5842, 5453, 5449, 5448, 5444, 5445, 5447, + 5446, 5551, 5550, 5546, 5547, 5549, 5548, 5410, 5409, 5405, 5406, 5408, + 5407, 5545, 5544, 5540, 5541, 5543, 5542, 5509, 5508, 5504, 5505, 5507, + 5506, 5515, 5514, 5510, 5511, 5513, 5512, 5503, 5502, 5498, 5499, 5501, + 5500, 5497, 5443, 5416, 5415, 5411, 5412, 5414, 5413, 5489, 5488, 5484, + 5485, 5487, 5486, 5483, 5482, 5478, 5479, 5481, 5480, 5477, 5539, 5538, + 5534, 5535, 5537, 5536, 5557, 5556, 5552, 5553, 5555, 5554, 5521, 5520, + 5516, 5517, 5519, 5518, 5592, 5621, 5773, 5734, 5744, 5745, 5742, 5743, + 5740, 5741, 6053, 6051, 6052, 6049, 6050, 6047, 6048, 6058, 5379, 30157, + 30132, 30143, 30139, 30149, 30146, 30152, 30155, 30156, 30135, 30134, + 30154, 30140, 30145, 30150, 30144, 30131, 30147, 30153, 30137, 30141, + 30136, 30148, 30151, 30142, 30138, 30133, 30129, 30130, 41412, 41412, + 41412, 32482, 32538, 32532, 32536, 32535, 32530, 32528, 32475, 32460, + 32506, 32459, 32458, 32503, 32518, 32505, 32509, 32510, 32512, 32498, + 32464, 32497, 32483, 32476, 32484, 32486, 32531, 32488, 32487, 32501, + 32515, 32527, 32517, 32469, 32491, 32472, 32495, 32485, 32500, 32521, + 32492, 32534, 32461, 32524, 32523, 32519, 32462, 32540, 32529, 32520, + 32467, 32526, 32514, 32470, 32508, 32473, 32533, 32502, 32516, 32499, + 32468, 32490, 32489, 32471, 32507, 32474, 32494, 32466, 32465, 32463, + 32525, 32504, 32522, 32493, 32537, 32539, 32541, 32542, 32457, 32543, + 32544, 32456, 32496, 32513, 32511, 32480, 32479, 32481, 32478, 32477, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 35250, 35267, 35268, + 35258, 35256, 35252, 35264, 35255, 35253, 35261, 35254, 35260, 35266, + 35262, 35259, 35265, 35263, 35257, 35271, 35272, 35270, 35269, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 35251, 19806, + 19807, 19808, 19797, 19795, 19791, 19803, 19794, 19792, 19800, 19793, + 19799, 19805, 19801, 19798, 19804, 19802, 19796, 19810, 19811, 19809, + 31612, 31613, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 5092, 5093, 5094, 5083, 5081, 5077, 5089, 5080, 5078, 5086, 5079, + 5085, 5091, 5087, 5084, 5090, 5088, 5082, 5095, 5096, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 35286, 35287, 35288, 35278, 35277, 35273, 35283, 35276, 35274, 35281, + 35275, 35280, 35285, 41412, 35279, 35284, 35282, 41412, 35289, 35290, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 22394, 22392, 22395, 22393, 22396, 22390, 22388, 22391, + 22389, 22398, 22412, 22408, 22413, 22409, 22397, 22410, 22406, 22411, + 22407, 22399, 22420, 22400, 22402, 22401, 22416, 22419, 22417, 22415, + 22418, 22404, 22403, 22405, 22421, 22414, 22422, 22369, 22367, 22377, + 22378, 22373, 22376, 22374, 22375, 22386, 22387, 22384, 22385, 22379, + 22368, 22372, 22371, 22370, 22489, 22488, 22490, 22495, 22497, 22501, + 22503, 22505, 22507, 22506, 22498, 22502, 22496, 22508, 22492, 22493, + 22500, 22494, 22443, 22437, 22442, 22444, 22438, 22428, 22436, 22439, + 22433, 22425, 22426, 22445, 22432, 22427, 22434, 22429, 22431, 22440, + 22430, 22441, 22435, 22366, 22423, 22424, 41412, 41412, 22515, 22517, + 22514, 22513, 22510, 22509, 22512, 22511, 22518, 22516, 41412, 41412, + 41412, 41412, 41412, 41412, 22467, 22466, 22463, 22465, 22464, 22458, + 22461, 22462, 22460, 22459, 41412, 41412, 41412, 41412, 41412, 41412, + 28342, 28354, 28350, 28199, 28349, 28200, 28347, 28338, 28351, 28352, + 28353, 28198, 28197, 28196, 28348, 28195, 28191, 28193, 28190, 28189, + 28186, 28185, 28188, 28187, 28194, 28192, 41412, 41412, 41412, 41412, + 41412, 41412, 28203, 28317, 28334, 28319, 28321, 28320, 28322, 28318, + 28328, 28231, 28323, 28329, 28330, 28326, 28235, 28316, 28278, 28277, + 28308, 28324, 28232, 28327, 28333, 28331, 28332, 28325, 28314, 28313, + 28307, 28311, 28312, 28310, 28315, 28309, 28234, 28287, 28305, 28306, + 28294, 28296, 28295, 28297, 28281, 28298, 28301, 28302, 28288, 28300, + 28291, 28283, 28292, 28285, 28290, 28304, 28303, 28299, 28289, 28293, + 28284, 28286, 28282, 28276, 28261, 28262, 28270, 28269, 28266, 28274, + 28255, 28257, 28275, 28264, 28260, 28271, 28273, 28272, 28256, 28258, + 28259, 28268, 28265, 28263, 28267, 28254, 28252, 28253, 28251, 28250, + 28233, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 28205, 28207, + 28209, 28216, 28215, 28223, 28219, 28204, 28214, 28230, 28217, 28229, + 28221, 28220, 28211, 28218, 28222, 28208, 28225, 28224, 28228, 28226, + 28227, 28206, 28280, 28279, 28243, 28248, 28239, 28244, 28240, 28236, + 28241, 28237, 28249, 28238, 28246, 28247, 28213, 28212, 28242, 28210, + 28245, 41412, 41412, 41412, 41412, 41412, 5793, 5378, 5372, 6061, 5809, + 5806, 5797, 5935, 5644, 5635, 5684, 5757, 5729, 5661, 5878, 5834, 5865, + 5868, 5857, 6083, 6080, 5826, 5762, 5782, 5763, 5783, 5760, 5780, 5761, + 5781, 5822, 5820, 5821, 5818, 5819, 5816, 5789, 5790, 5787, 5786, 5788, + 5779, 5784, 5785, 5599, 6043, 5612, 5611, 5814, 5964, 5962, 5906, 5904, + 5925, 5924, 5922, 5916, 5915, 5845, 5844, 5836, 5425, 5402, 5427, 5424, + 5841, 5452, 5398, 5399, 5403, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 24647, 24614, 24613, 24594, 24593, 24600, + 24608, 24607, 24612, 24611, 24599, 24597, 24595, 24610, 24609, 24601, + 24616, 24615, 24606, 24605, 24619, 24598, 24620, 24618, 24621, 24602, + 24603, 24604, 24617, 24592, 24596, 41412, 24638, 24645, 24646, 24644, + 24639, 24642, 24640, 24643, 24641, 24637, 24635, 24636, 41412, 41412, + 41412, 41412, 24629, 24626, 24628, 24634, 24627, 24632, 24631, 24633, + 24630, 24623, 24622, 24624, 41412, 41412, 41412, 41412, 24625, 41412, + 41412, 41412, 24648, 24649, 24656, 24658, 24655, 24654, 24651, 24650, + 24653, 24652, 24659, 24657, 35311, 35323, 35306, 35303, 35321, 35324, + 35305, 35304, 35318, 35313, 35312, 35319, 35316, 35322, 35317, 35320, + 35310, 35302, 35307, 35291, 35325, 35295, 35296, 35314, 35309, 35308, + 35315, 35294, 35292, 35293, 41412, 41412, 35297, 35298, 35299, 35300, + 35301, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 29300, 29322, 29282, 29284, 29287, 29304, 29306, 29309, + 29290, 29286, 29302, 29312, 29308, 29324, 29291, 29289, 29288, 29313, + 29311, 29310, 29293, 29292, 29299, 29315, 29314, 29321, 29296, 29301, + 29298, 29318, 29323, 29320, 29297, 29295, 29294, 29319, 29317, 29316, + 29281, 29283, 29303, 29305, 29285, 29307, 41412, 41412, 41412, 41412, + 29345, 29330, 29334, 29340, 29343, 29346, 29332, 29336, 29337, 29341, + 29333, 29331, 29344, 29339, 29338, 29342, 29335, 29280, 29275, 29274, + 29279, 29278, 29277, 29276, 29327, 29328, 41412, 41412, 41412, 41412, + 41412, 41412, 29353, 29355, 29352, 29351, 29348, 29347, 29350, 29349, + 29356, 29354, 29329, 41412, 41412, 41412, 29325, 29326, 22468, 22487, + 22480, 22483, 22485, 22478, 22476, 22474, 22470, 22472, 22457, 22455, + 22447, 22451, 22453, 22449, 22481, 22486, 22479, 22482, 22484, 22477, + 22475, 22473, 22469, 22471, 22456, 22454, 22446, 22450, 22452, 22448, + 5061, 5058, 5050, 5049, 5063, 5055, 5048, 5047, 5066, 5057, 5054, 5053, + 5056, 5060, 5052, 5051, 5068, 5064, 5062, 5067, 5065, 5069, 5059, 5072, + 5074, 5071, 5073, 5070, 41412, 41412, 5076, 5075, 35359, 35357, 35358, + 35375, 35374, 35373, 35399, 35365, 35364, 35378, 35385, 35377, 35400, + 35393, 35360, 35405, 35376, 35392, 35369, 35368, 35382, 35381, 35401, + 35404, 35367, 35366, 35370, 35380, 35383, 35379, 35406, 35386, 35372, + 35391, 35394, 35387, 35389, 35407, 35361, 35362, 35363, 35371, 35390, + 35408, 35384, 35397, 35398, 35395, 35396, 35403, 35402, 35388, 35356, + 35331, 35330, 35328, 35419, 35326, 35327, 35329, 35332, 35333, 35334, + 41412, 35427, 35434, 35438, 35435, 35444, 35450, 35451, 35449, 35448, + 35446, 35447, 35439, 35440, 35443, 35452, 35436, 35442, 35437, 35445, + 35441, 35418, 35431, 35432, 35411, 35412, 35413, 35422, 35421, 35414, + 41412, 41412, 35335, 35342, 35344, 35341, 35340, 35337, 35336, 35339, + 35338, 35345, 35343, 41412, 41412, 41412, 41412, 41412, 41412, 35352, + 35354, 35351, 35350, 35347, 35346, 35349, 35348, 35355, 35353, 41412, + 41412, 41412, 41412, 41412, 41412, 35428, 35429, 35426, 35417, 35410, + 35430, 35423, 35420, 35415, 35416, 35424, 35425, 35409, 35433, 41412, + 41412, 8313, 8277, 8402, 8316, 8561, 8580, 8579, 8509, 8297, 8483, 8548, + 8515, 8301, 8514, 8513, 8450, 8444, 8468, 8531, 8469, 8532, 8545, 8502, + 8401, 8518, 8300, 8299, 8559, 8428, 8429, 8430, 8293, 8572, 8382, 8574, + 8165, 8576, 8495, 8573, 8575, 8497, 8544, 8328, 8315, 8276, 8283, 41412, + 41412, 8474, 8534, 8501, 8399, 8547, 8552, 8286, 8287, 8325, 8461, 8566, + 8303, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 2762, 2761, 2763, 2760, 2764, 2671, 2672, 2688, 2689, 2692, 2693, + 2710, 2711, 2701, 2702, 2685, 2675, 2690, 2691, 2696, 2698, 2686, 2687, + 2705, 2678, 2679, 2694, 2695, 2706, 2717, 2716, 2682, 2681, 2704, 2715, + 2718, 2680, 2683, 2703, 2707, 2708, 2676, 2677, 2723, 2725, 2709, 2700, + 2724, 2713, 2714, 2712, 2722, 2765, 2776, 2779, 2780, 2770, 2771, 2768, + 2769, 2766, 2767, 2772, 2773, 2775, 2774, 2777, 2778, 2781, 2697, 2699, + 2719, 2684, 2720, 2721, 2673, 2674, 41412, 2670, 2669, 2789, 2791, 2788, + 2787, 2784, 2783, 2786, 2785, 2792, 2790, 2757, 2754, 2782, 2667, 2668, + 2666, 2756, 2743, 2741, 2744, 2735, 2736, 2742, 2738, 2740, 2739, 2737, + 2733, 2732, 2728, 2726, 2730, 2729, 2727, 2731, 2734, 2753, 2752, 2751, + 2750, 2745, 2747, 2748, 2749, 2746, 2758, 2755, 2759, 34858, 34856, + 34857, 34809, 34844, 34846, 34811, 34845, 34821, 34822, 34829, 34837, + 34832, 34823, 34830, 34834, 34843, 34824, 34838, 34831, 34825, 34836, + 34814, 34839, 34827, 34835, 34842, 34818, 34816, 34840, 34820, 34841, + 34833, 34804, 34805, 34806, 34864, 34863, 34865, 34862, 34860, 34861, + 34855, 34859, 34807, 34808, 34828, 34819, 34872, 34874, 34871, 34870, + 34867, 34866, 34869, 34868, 34875, 34873, 34803, 34817, 34815, 34826, + 34812, 34813, 3557, 3543, 3551, 3535, 3523, 3547, 3546, 3532, 3538, 3531, + 3524, 3555, 3541, 3533, 3550, 3534, 3552, 3549, 3554, 3539, 3522, 3537, + 3544, 3527, 3545, 3540, 3525, 3556, 3542, 3529, 3553, 3536, 3530, 3548, + 3528, 3526, 3558, 3559, 3566, 3572, 3569, 3573, 3574, 3570, 3575, 3571, + 3567, 3568, 3520, 3521, 3560, 3561, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 3565, 3563, 3564, 3562, 24476, 24475, 24474, 24488, + 24487, 24493, 24503, 24502, 24506, 24494, 24501, 24500, 24482, 24495, + 24479, 24478, 24477, 24486, 24485, 24484, 24483, 24492, 24491, 24497, + 24496, 24481, 24511, 24508, 24507, 24490, 24489, 24509, 24505, 24504, + 24510, 24512, 24521, 24520, 24526, 24528, 24524, 24525, 24522, 24523, + 24527, 24465, 24470, 24469, 24467, 24471, 24472, 24473, 24468, 24466, + 24519, 24518, 41412, 41412, 41412, 24513, 24516, 24517, 24515, 24514, + 24535, 24537, 24534, 24533, 24530, 24529, 24532, 24531, 24538, 24536, + 41412, 41412, 41412, 24499, 24498, 24480, 30203, 30205, 30202, 30201, + 30198, 30197, 30200, 30199, 30206, 30204, 30176, 30167, 30165, 30164, + 30166, 30177, 30161, 30160, 30162, 30163, 30179, 30175, 30173, 30172, + 30174, 30181, 30187, 30188, 30186, 30189, 30178, 30171, 30168, 30170, + 30169, 30180, 30182, 30183, 30185, 30184, 30191, 30192, 30190, 30196, + 30195, 30207, 30194, 30193, 10562, 10539, 10545, 10602, 10583, 10591, + 10581, 10582, 10601, 10267, 10593, 41412, 41412, 41412, 41412, 41412, + 18013, 18044, 18021, 18050, 18019, 18049, 18042, 18039, 18051, 18031, + 18033, 18045, 18047, 18052, 18035, 18041, 18043, 18037, 18040, 18053, + 18034, 18030, 18020, 18048, 18036, 18014, 18017, 18029, 18016, 18015, + 18046, 18028, 18024, 18027, 18025, 18055, 18022, 18026, 18056, 18054, + 18018, 18038, 18012, 41412, 41412, 18011, 18023, 18032, 34854, 34852, + 34853, 34851, 34850, 34849, 34848, 34847, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 39257, 39270, 39259, 39237, 39251, 39265, + 39266, 39267, 39252, 39268, 39255, 39263, 39258, 39256, 39264, 39262, + 39261, 39269, 39250, 39248, 39239, 39246, 39238, 39249, 39247, 39228, + 39229, 39231, 39232, 39244, 39242, 39243, 39241, 39230, 39234, 39240, + 39253, 39236, 39245, 39233, 39260, 39254, 39235, 41412, 41412, 41412, + 41412, 41412, 23439, 23440, 24047, 23436, 23441, 23442, 23413, 23412, + 24032, 24025, 23445, 23446, 23419, 23447, 23425, 23420, 23421, 23982, + 23983, 23984, 24027, 23423, 24019, 23538, 23449, 23426, 23434, 23429, + 23452, 23987, 23986, 23985, 23453, 23454, 23456, 23414, 23466, 23400, + 18557, 18560, 18556, 18559, 18555, 10432, 27880, 27881, 27871, 27872, + 27883, 27884, 27874, 27886, 27876, 27887, 27888, 27889, 27890, 27891, + 27892, 27875, 27878, 27879, 27893, 27873, 27896, 27897, 27899, 28030, + 28137, 28031, 28139, 28034, 28059, 28073, 28127, 28112, 28142, 28080, + 28150, 28161, 28090, 28077, 28110, 28113, 28134, 28036, 28114, 28129, + 28154, 28128, 28140, 28158, 28032, 28038, 28081, 28064, 28082, 28058, + 24188, 24196, 24198, 24199, 18766, 18762, 18765, 18764, 18763, 24108, + 23524, 23573, 23659, 23802, 23822, 23899, 23927, 23920, 23966, 24002, + 24170, 24054, 27948, 23730, 24012, 23468, 23737, 23895, 23469, 24107, + 23525, 23577, 23660, 23673, 23756, 23783, 23803, 23828, 23900, 23930, + 23967, 23646, 24115, 24142, 24171, 23487, 23513, 23576, 23636, 23888, + 23937, 23975, 23727, 23884, 23648, 24094, 23654, 28138, 28039, 28057, + 28075, 28121, 28078, 28065, 28126, 28149, 28092, 28093, 28040, 28042, + 28095, 28099, 28101, 28045, 28091, 28141, 28109, 28108, 28051, 28035, + 28115, 28125, 28074, 28130, 28156, 28157, 28053, 28160, 28151, 28070, + 28072, 28071, 28076, 28153, 8285, 8284, 8550, 8549, 8496, 8384, 8498, + 8167, 8383, 8166, 8442, 8179, 8499, 8294, 8511, 8539, 8403, 8567, 8568, + 8425, 8416, 8417, 8418, 8420, 8427, 8423, 8452, 8408, 8454, 8431, 8409, + 8410, 8456, 8411, 8412, 8441, 8445, 8433, 8460, 8415, 8447, 8448, 8446, + 8424, 8432, 8436, 8457, 8422, 8439, 8449, 8414, 8435, 8438, 8564, 8407, + 8406, 8279, 8577, 8282, 8273, 8292, 8176, 8465, 8522, 22921, 23485, + 22957, 23527, 22956, 23526, 22955, 23523, 22964, 23542, 22989, 23579, + 22988, 23578, 22987, 23574, 22983, 23568, 22984, 23569, 23017, 23622, + 23018, 23623, 23006, 23610, 23015, 23620, 22997, 23601, 23042, 23662, + 23050, 23672, 23069, 23692, 23068, 23691, 23066, 23689, 23063, 23686, + 23062, 23685, 23086, 23719, 23080, 23707, 23117, 23754, 23114, 23751, + 23118, 23755, 23125, 23768, 23126, 23769, 23136, 23778, 23132, 23765, + 23142, 23800, 23144, 23805, 23143, 23804, 23162, 23827, 23161, 23826, + 23156, 23819, 23152, 23814, 23193, 23863, 23192, 23862, 23170, 23855, + 23171, 23856, 23221, 23898, 23223, 23902, 23233, 23917, 23231, 23915, + 23232, 23916, 23238, 23922, 23259, 23960, 23257, 23958, 23256, 23951, + 23251, 23953, 23258, 23959, 23280, 24000, 23279, 23999, 23281, 24003, + 23275, 23993, 23311, 24075, 23332, 24098, 23304, 24068, 23331, 24097, + 23323, 24087, 23344, 24118, 23343, 24114, 23357, 24131, 23358, 24132, + 23354, 24128, 23356, 24130, 23355, 24129, 23363, 24137, 23362, 24136, + 23368, 24147, 23379, 24161, 23383, 24165, 23385, 24167, 23693, 23998, + 24133, 24157, 23486, 23793, 23792, 23794, 23269, 23583, 22915, 23479, + 22931, 23497, 22927, 23493, 22926, 23492, 22925, 23491, 22929, 23495, + 22928, 23494, 22910, 23474, 22909, 23473, 22908, 23472, 22912, 23476, + 22911, 23475, 23011, 23615, 23022, 23628, 23014, 23619, 23002, 23606, + 23001, 23605, 23000, 23604, 23005, 23609, 23004, 23608, 23087, 23720, + 23077, 23711, 23184, 23848, 23204, 23874, 23176, 23840, 23175, 23839, + 23174, 23838, 23178, 23842, 23177, 23841, 23201, 23871, 23200, 23870, + 23199, 23869, 23203, 23873, 23202, 23872, 23314, 24078, 23321, 24085, + 23318, 24082, 23317, 24081, 23316, 24080, 23320, 24084, 23319, 24083, + 23369, 24148, 23367, 24146, 23371, 24150, 23375, 24156, 23147, 23808, + 23148, 23809, 23372, 24151, 18604, 18596, 18609, 18601, 18605, 18597, + 18607, 18599, 18370, 18362, 18373, 18365, 18371, 18363, 18375, 18367, + 18627, 18624, 18629, 18626, 18628, 18625, 41412, 41412, 18410, 18407, + 18412, 18409, 18411, 18408, 41412, 41412, 18642, 18634, 18647, 18639, + 18643, 18635, 18645, 18637, 18394, 18386, 18397, 18389, 18395, 18387, + 18399, 18391, 18668, 18659, 18671, 18662, 18670, 18661, 18669, 18660, + 18422, 18417, 18425, 18420, 18424, 18419, 18423, 18418, 18729, 18726, + 18731, 18728, 18730, 18727, 41412, 41412, 18456, 18453, 18458, 18455, + 18457, 18454, 41412, 41412, 18688, 18679, 18691, 18682, 18690, 18681, + 18689, 18680, 41412, 18468, 41412, 18471, 41412, 18470, 41412, 18469, + 18709, 18701, 18714, 18706, 18710, 18702, 18712, 18704, 18440, 18432, + 18443, 18435, 18441, 18433, 18445, 18437, 18594, 18614, 18631, 18630, + 18654, 18652, 18674, 18675, 18733, 18732, 18694, 18695, 18721, 18719, + 41412, 41412, 18611, 18603, 18610, 18602, 18606, 18598, 18608, 18600, + 18377, 18369, 18374, 18366, 18372, 18364, 18376, 18368, 18649, 18641, + 18648, 18640, 18644, 18636, 18646, 18638, 18401, 18393, 18398, 18390, + 18396, 18388, 18400, 18392, 18716, 18708, 18715, 18707, 18711, 18703, + 18713, 18705, 18447, 18439, 18444, 18436, 18442, 18434, 18446, 18438, + 18593, 18618, 18595, 18616, 18615, 41412, 18612, 18613, 18379, 18383, + 18380, 18381, 18378, 18553, 18581, 18582, 18586, 18502, 18655, 18656, + 18653, 41412, 18650, 18651, 18414, 18413, 18404, 18403, 18402, 18585, + 18584, 18583, 18673, 18677, 18666, 18665, 41412, 41412, 18672, 18664, + 18426, 18430, 18427, 18428, 41412, 18510, 18509, 18508, 18693, 18697, + 18686, 18685, 18741, 18740, 18692, 18684, 18473, 18477, 18474, 18475, + 18463, 18504, 18503, 18780, 41412, 41412, 18722, 18723, 18720, 41412, + 18717, 18718, 18460, 18459, 18450, 18449, 18448, 18578, 18507, 41412, + 16898, 16895, 16900, 16897, 37416, 17650, 33984, 17575, 31904, 37389, + 19135, 41218, 41217, 41219, 24357, 32233, 20617, 29541, 17574, 16899, + 16896, 20561, 11383, 11357, 24280, 32163, 33859, 33857, 24233, 32128, + 11356, 11351, 10661, 11350, 5097, 38001, 30706, 38148, 20584, 20620, + 24665, 31245, 24356, 32232, 31778, 24355, 32231, 29142, 31481, 31482, + 31864, 11365, 38015, 32071, 32065, 32079, 6109, 33858, 33860, 32088, + 11387, 20959, 31059, 38199, 6395, 6110, 2572, 20619, 17654, 24288, 32174, + 11388, 31928, 17489, 37790, 32070, 3956, 3998, 25334, 32076, 8150, 38160, + 8583, 34967, 20977, 17614, 37396, 31924, 17643, 17600, 38149, 17644, + 11329, 38026, 39278, 27150, 39824, 17777, 20979, 20981, 20980, 41412, + 24354, 32230, 17593, 31777, 20873, 7, 20872, 6, 29137, 29539, 34938, + 34926, 41412, 41412, 34933, 34932, 34935, 34934, 34925, 34939, 34929, + 34931, 34924, 34928, 34930, 34927, 34768, 34770, 34767, 34766, 34763, + 34762, 34765, 34764, 34758, 34769, 34759, 34761, 34757, 34756, 34760, + 41412, 24185, 24186, 24194, 24200, 24184, 24187, 24190, 24191, 24192, + 24193, 24195, 24183, 24197, 41412, 41412, 41412, 17485, 8163, 8829, + 17661, 25273, 27765, 29070, 31505, 32553, 39828, 29273, 11323, 17486, + 22718, 38036, 11505, 18058, 31506, 18830, 2585, 20625, 6228, 25274, + 34320, 37161, 20863, 38118, 29591, 25839, 32422, 22902, 3862, 34300, + 32714, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 8472, 8530, 8487, 8543, 8172, 8188, + 8467, 8527, 8536, 8187, 8171, 8553, 8327, 8317, 8320, 8323, 8318, 8476, + 8319, 8321, 8322, 8519, 8308, 8173, 8560, 8578, 8478, 8484, 8535, 8477, + 8466, 8526, 8175, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 0, 100, 11399, 10685, + 6232, 6111, 5362, 17488, 32742, 10687, 32739, 32731, 4046, 11400, 31667, + 31668, 32732, 4047, 32733, 32740, 25511, 11401, 29635, 34216, 32735, + 11397, 11402, 32736, 4048, 11403, 31840, 32049, 32845, 37055, 37986, + 39271, 11404, 31053, 31063, 20976, 4049, 38141, 21773, 963, 32728, 4045, + 16955, 32738, 32729, 32730, 38125, 32734, 32741, 348, 3753, 18061, 10674, + 20870, 32410, 17554, 11411, 11410, 11396, 11398, 11412, 38134, 38135, + 32075, 38136, 11409, 11405, 11406, 11407, 11408, 31868, 38120, 31483, + 2643, 38138, 35046, 39422, 39420, 39424, 39425, 39430, 39418, 39429, + 39428, 39416, 39423, 39415, 39417, 39426, 39414, 39431, 17655, 32378, + 32389, 32390, 32377, 32374, 32383, 32385, 32393, 32394, 32386, 32392, + 32388, 32371, 32379, 32375, 32381, 34046, 34050, 34051, 34045, 34042, + 34054, 34053, 34041, 34055, 34052, 34040, 34049, 34044, 34047, 34043, + 34048, 32382, 32376, 32387, 32391, 23943, 32384, 32373, 32372, 32380, + 39432, 38129, 38128, 41412, 41412, 41412, 41412, 24358, 38350, 32235, + 11441, 24262, 38221, 29572, 29545, 34188, 34203, 24378, 32259, 24442, + 32336, 24439, 38400, 32335, 11477, 24381, 32262, 24389, 38363, 32242, + 11455, 38222, 24387, 32268, 24372, 32254, 24270, 24265, 11481, 38360, + 38361, 11450, 11451, 32250, 11442, 974, 8134, 29574, 24369, 979, 8136, + 24408, 24402, 38377, 38374, 32295, 32289, 11492, 11489, 32270, 38352, + 24392, 24454, 38403, 32302, 11500, 24409, 32296, 24445, 24269, 32278, + 24443, 38367, 32276, 11482, 24267, 38224, 29585, 29558, 34200, 34213, + 24429, 32326, 24458, 32306, 38353, 11443, 24449, 38368, 32282, 11483, + 24368, 32249, 24440, 38404, 32337, 11479, 38409, 38405, 38407, 38406, + 38411, 38412, 32338, 29573, 34191, 38226, 32115, 11453, 37404, 24388, + 32269, 24264, 24374, 32252, 24263, 24453, 32301, 24272, 17639, 8588, + 31391, 37383, 37382, 16888, 20791, 29026, 16837, 29594, 34026, 8596, + 11149, 34019, 16902, 28985, 28973, 28977, 27769, 27776, 11324, 11131, + 32850, 2571, 32347, 5098, 34547, 8835, 17649, 31867, 20864, 32105, 959, + 27024, 34323, 11135, 11150, 31250, 29599, 25288, 25295, 20952, 38201, + 20937, 11354, 38025, 8602, 34960, 39410, 8137, 8128, 976, 37384, 3754, + 31959, 31866, 11325, 17491, 17883, 20605, 37694, 32077, 20973, 33979, + 39832, 29604, 27775, 2578, 29595, 1058, 1061, 29229, 364, 29596, 366, + 38016, 361, 16941, 17881, 11141, 1062, 17882, 1059, 20754, 8162, 16939, + 32345, 32346, 8781, 16956, 16942, 34712, 10690, 16924, 27035, 31930, + 29606, 20630, 29607, 34749, 24558, 18294, 24560, 18297, 24549, 18287, + 28722, 28721, 3752, 29605, 29609, 29608, 29234, 29231, 24557, 18293, + 29233, 29230, 24559, 18295, 29235, 29232, 31841, 34786, 31850, 34795, + 31849, 34794, 11152, 11155, 34774, 34946, 29592, 29593, 34780, 34952, + 29227, 29228, 34779, 34951, 28475, 28476, 28477, 34420, 34509, 34422, + 34511, 34355, 34362, 6909, 6855, 6914, 6647, 6660, 6910, 6636, 6919, + 6661, 34680, 34673, 34694, 34628, 32192, 24306, 11418, 38230, 2579, + 27784, 38033, 17640, 38019, 11381, 11154, 29603, 11157, 29226, 31851, + 34796, 29589, 8597, 29590, 8598, 30738, 20753, 28478, 20376, 20960, + 39853, 29071, 29544, 32111, 32188, 28982, 28983, 28984, 28978, 10969, + 11326, 34714, 11133, 4474, 24320, 32150, 24278, 32161, 32078, 10078, + 10077, 11375, 11374, 11353, 11378, 31661, 16925, 24563, 18303, 39321, + 39320, 24547, 18298, 16923, 16922, 16920, 16921, 11153, 11156, 29600, + 29601, 34421, 34510, 24548, 18286, 31848, 34793, 29597, 11147, 29598, + 11148, 39277, 27761, 38229, 11421, 16838, 16840, 34027, 16843, 16842, + 34028, 16841, 16839, 8599, 8600, 34020, 8601, 34021, 41130, 10970, 16835, + 20599, 38213, 11422, 31865, 31500, 39596, 24229, 32123, 24317, 32132, + 4457, 4454, 37934, 37930, 32068, 34453, 2567, 32752, 32748, 37053, 31786, + 39334, 31664, 38132, 39587, 20597, 37929, 37933, 4453, 4456, 37923, 4451, + 17665, 34086, 38214, 30728, 16952, 39837, 21775, 24327, 32213, 16951, + 3700, 10656, 362, 35056, 37950, 11142, 8607, 34011, 8783, 8784, 1004, + 1042, 1028, 1016, 1017, 1033, 1014, 984, 989, 1039, 1044, 1030, 1029, + 1024, 1031, 1012, 1036, 1025, 1032, 987, 996, 995, 1018, 1021, 997, 1050, + 1023, 1047, 993, 1022, 1020, 1048, 1000, 1019, 1035, 994, 1001, 1010, + 988, 1049, 1034, 985, 1015, 1046, 991, 1040, 1009, 986, 998, 1011, 1052, + 1051, 990, 992, 1053, 1041, 1038, 1027, 1026, 999, 1045, 1002, 1037, + 1007, 1006, 1043, 1003, 1008, 1005, 29610, 32110, 33041, 3595, 39293, + 20936, 8605, 11056, 16894, 8587, 39741, 16916, 367, 20087, 6691, 6913, + 5038, 38200, 28358, 20623, 30724, 30725, 31405, 31406, 11051, 34104, + 1013, 10681, 31852, 29458, 31854, 8157, 24323, 24324, 24322, 32157, + 32158, 32156, 24285, 24292, 24284, 32171, 32178, 32170, 24227, 24225, + 24226, 10080, 32121, 32119, 32120, 20947, 20565, 38282, 38321, 34799, + 34797, 37937, 4460, 4461, 31939, 24326, 32194, 20574, 20575, 20576, + 20577, 10703, 10701, 10705, 10694, 10698, 10706, 10695, 10699, 10704, + 10693, 10697, 10692, 10696, 10702, 10700, 34388, 32050, 17518, 39288, + 27597, 27589, 27596, 27590, 27594, 27595, 27593, 27592, 27591, 11672, + 17781, 37925, 4464, 37907, 4463, 37938, 4467, 39750, 3701, 34740, 17605, + 8, 16836, 10682, 3987, 3949, 4030, 3926, 3988, 3950, 3990, 230, 34738, + 37702, 20598, 3966, 3969, 3971, 3963, 11379, 4009, 3871, 31800, 31797, + 31798, 31799, 30113, 35041, 35049, 35050, 35030, 35028, 35031, 35042, + 35013, 35014, 35035, 35037, 35036, 35034, 35015, 35047, 35048, 35017, + 35026, 35025, 35024, 35023, 35039, 35053, 35029, 35016, 35027, 35051, + 35032, 35033, 35044, 35043, 35045, 35054, 35018, 4053, 30711, 35040, + 35021, 35052, 35020, 35019, 35022, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 30125, 30120, 30123, + 30124, 30117, 30118, 30116, 30115, 30122, 30119, 30121, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 6657, + 6654, 6653, 6650, 6649, 6652, 6651, 6658, 6656, 6878, 6858, 6903, 6891, + 6873, 6861, 6877, 6875, 6857, 6904, 6892, 31337, 31335, 31334, 31331, + 31330, 31333, 31332, 31338, 31336, 31329, 31320, 31328, 31326, 31321, + 31322, 31324, 31325, 31319, 31323, 31327, 10991, 11003, 11000, 10985, + 10982, 10997, 10994, 11009, 10988, 29631, 29624, 29632, 29630, 29625, + 29626, 29627, 29629, 29623, 29634, 29633, 31365, 31366, 31367, 31368, + 31369, 31370, 31371, 31372, 31373, 31374, 31375, 31376, 31377, 31378, + 31379, 31380, 31381, 31382, 31383, 31384, 31385, 31386, 31387, 31388, + 31389, 31390, 6801, 6802, 6803, 6804, 6805, 6806, 6807, 6808, 6809, 6810, + 6811, 6812, 6813, 6814, 6815, 6816, 6817, 6818, 6819, 6820, 6821, 6822, + 6823, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, 6833, 6834, + 6835, 6836, 6837, 6838, 6839, 6840, 6841, 6842, 6843, 6844, 6845, 6846, + 6847, 6848, 6849, 6850, 6851, 6852, 6655, 29151, 29149, 29147, 29152, + 29153, 29155, 29156, 29150, 29154, 29148, 11345, 11343, 11342, 11339, + 11338, 11341, 11340, 11346, 11344, 11337, 29146, 4586, 4532, 4602, 4518, + 4595, 4531, 4594, 4530, 4593, 4529, 4592, 4528, 4583, 4502, 4496, 4525, + 4582, 4500, 4494, 4524, 4601, 4630, 4624, 4517, 4600, 4628, 4622, 4515, + 4609, 4646, 4623, 4495, 4643, 4501, 4629, 4521, 4608, 4645, 4621, 4493, + 4642, 4499, 4627, 4520, 4581, 4536, 4614, 4504, 4498, 4617, 4539, 4523, + 4599, 4535, 4613, 4631, 4625, 4616, 4538, 4516, 4607, 4537, 4615, 4644, + 4620, 4497, 4641, 4541, 4619, 4534, 4612, 4503, 4626, 4618, 4540, 4519, + 4585, 4527, 4584, 4526, 4492, 4488, 4509, 4506, 4484, 4508, 4505, 4483, + 4636, 4633, 4487, 4635, 4632, 4486, 4648, 4639, 4491, 4647, 4638, 4490, + 4510, 4507, 4482, 4637, 4634, 4485, 4649, 4640, 4489, 4543, 4542, 4544, + 4545, 4574, 4568, 4578, 4590, 4597, 4611, 4580, 4511, 4513, 4533, 4522, + 4591, 4598, 4512, 4514, 38259, 25435, 25434, 25437, 25345, 25427, 25440, + 25436, 17672, 24279, 24302, 24315, 24243, 24303, 24258, 24259, 32136, + 24579, 27151, 10668, 38307, 32153, 31921, 31922, 31913, 31914, 31915, + 31916, 31917, 31918, 31919, 31920, 4007, 39728, 39737, 39731, 34566, + 34575, 34565, 34572, 34574, 34564, 4004, 39725, 4000, 39713, 4037, 39760, + 3981, 39709, 4032, 39757, 4031, 39756, 3989, 39717, 3994, 39716, 3993, + 39715, 3928, 39672, 3927, 39671, 3954, 39698, 3953, 39697, 3952, 39696, + 3918, 39661, 39662, 17597, 25442, 39647, 11328, 6630, 5101, 3867, 6622, + 6631, 6623, 6628, 6627, 6629, 24241, 32134, 20964, 20968, 38263, 25339, + 38287, 38323, 25389, 25369, 38268, 25346, 3961, 3960, 4033, 4034, 39611, + 34569, 34576, 34571, 34568, 39740, 39758, 38245, 38246, 22899, 39738, + 39734, 39735, 39739, 39654, 39652, 39653, 39655, 38285, 38340, 25380, + 39705, 3976, 39704, 3975, 25403, 4011, 8151, 38194, 34094, 8590, 4020, + 39747, 24589, 37421, 34800, 2568, 10711, 8608, 30732, 4026, 39749, 2795, + 2801, 2802, 32560, 38196, 20593, 39722, 4018, 33013, 32073, 3959, 3986, + 39695, 39754, 39719, 39670, 33990, 6225, 31937, 3864, 5361, 983, 30834, + 6585, 8816, 8815, 34715, 17564, 102, 19262, 31471, 41128, 38004, 38005, + 38010, 38007, 38009, 38008, 38006, 38003, 39607, 39682, 39726, 4006, + 39745, 17590, 22903, 27586, 17570, 11668, 25747, 21093, 32630, 38413, + 29460, 31766, 2566, 37042, 17864, 6099, 24463, 39336, 25277, 32724, + 32558, 6105, 2646, 31659, 39619, 39635, 39638, 39613, 39622, 39632, 3889, + 3905, 3908, 3883, 3892, 3902, 4019, 39685, 39666, 3917, 39727, 3938, + 3923, 39656, 20594, 31926, 16791, 3581, 3582, 28483, 28481, 28482, 39605, + 11673, 38210, 31967, 31968, 31969, 31970, 31971, 31972, 31973, 31974, + 4036, 31966, 31396, 31503, 39608, 10973, 10974, 10975, 10976, 10977, + 10978, 39649, 39651, 3868, 3870, 28355, 28356, 11013, 11016, 11015, + 11014, 39676, 3934, 19263, 981, 8824, 34709, 32719, 347, 17613, 17860, + 34710, 2581, 17610, 31040, 37398, 37397, 39581, 20444, 11414, 11415, + 20951, 25746, 25745, 25744, 39294, 20585, 27162, 27138, 27155, 25913, + 11136, 38212, 8807, 17778, 29270, 6235, 31205, 21094, 39323, 6588, 3977, + 32852, 32764, 31932, 32847, 34099, 3518, 34642, 39673, 39674, 3931, 3932, + 34095, 34802, 31942, 4013, 37420, 38124, 38123, 39667, 8825, 11055, + 30731, 31644, 6167, 20086, 6643, 6236, 29532, 368, 4027, 39752, 3957, + 39693, 11514, 19943, 24230, 34685, 17558, 4024, 32046, 32047, 2575, + 19869, 31478, 32212, 24353, 20974, 3880, 33018, 6620, 6227, 20549, 17861, + 17862, 25842, 28374, 38195, 17651, 17607, 17572, 32715, 34387, 33988, + 20629, 31490, 37162, 20991, 19845, 17780, 10073, 39677, 4014, 38252, + 4017, 25421, 39720, 39686, 37054, 37041, 226, 16913, 31955, 31947, 39327, + 39835, 25420, 31480, 38322, 39708, 3979, 6401, 19867, 28474, 19903, 2805, + 19859, 31042, 19950, 30715, 19905, 23392, 32857, 31039, 25748, 34713, + 17647, 17645, 19888, 17641, 3935, 39681, 34308, 34743, 6916, 30713, 3881, + 31041, 19907, 31656, 32856, 19858, 30714, 16790, 16785, 16783, 33982, + 16784, 19881, 38144, 33985, 37047, 30712, 19933, 33980, 3933, 39678, + 16786, 6905, 19932, 34097, 37692, 19866, 34307, 19926, 2794, 16789, + 19883, 8821, 32855, 29218, 25419, 38319, 25401, 38337, 4044, 39712, + 39675, 3920, 19885, 24586, 27159, 19949, 19916, 19917, 19877, 19878, + 19900, 19899, 10085, 19886, 19892, 19862, 32407, 17612, 32406, 27145, + 27148, 27139, 27140, 27146, 27149, 19896, 19909, 19895, 19908, 24578, + 24576, 27144, 27147, 11038, 11036, 11035, 11032, 11031, 11034, 11033, + 11039, 11037, 11030, 11049, 11046, 11045, 11042, 11041, 11044, 11043, + 11050, 11048, 11040, 11028, 11025, 11024, 11021, 11020, 11023, 11022, + 11029, 11027, 11019, 19945, 19948, 19904, 19874, 19922, 19910, 19929, + 11506, 19913, 37998, 19935, 10671, 19873, 3995, 37412, 37415, 3996, + 19860, 19861, 34703, 19872, 32228, 24343, 2658, 17663, 19901, 19940, + 29612, 10079, 29614, 6693, 39764, 4050, 4052, 4051, 19863, 19865, 19864, + 37048, 19934, 39601, 19946, 30726, 11347, 37395, 39751, 31484, 30722, + 30721, 24277, 32160, 30836, 32057, 34957, 39275, 26610, 25303, 26431, + 34670, 34671, 39665, 982, 16845, 25417, 38280, 24261, 32151, 17671, + 22891, 22890, 24208, 24207, 24257, 25325, 25308, 38231, 25443, 39657, + 39658, 39659, 39733, 39736, 26611, 26605, 26614, 26608, 26613, 26607, + 26612, 26606, 26615, 26609, 38381, 11496, 978, 8130, 32114, 25311, 25315, + 25305, 25309, 25320, 25307, 25312, 25319, 25310, 25321, 25324, 5027, + 4772, 4900, 4773, 4964, 4837, 4901, 4774, 4996, 4869, 4933, 4806, 4965, + 4838, 4902, 4775, 5012, 4885, 4949, 4822, 4981, 4854, 4918, 4791, 4997, + 4870, 4934, 4807, 4966, 4839, 4903, 4776, 5020, 4893, 4957, 4830, 4989, + 4862, 4926, 4799, 5005, 4878, 4942, 4815, 4974, 4847, 4911, 4784, 5013, + 4886, 4950, 4823, 4982, 4855, 4919, 4792, 4998, 4871, 4935, 4808, 4967, + 4840, 4904, 4777, 5024, 4897, 4961, 4834, 4993, 4866, 4930, 4803, 5009, + 4882, 4946, 4819, 4978, 4851, 4915, 4788, 5017, 4890, 4954, 4827, 4986, + 4859, 4923, 4796, 5002, 4875, 4939, 4812, 4971, 4844, 4908, 4781, 5021, + 4894, 4958, 4831, 4990, 4863, 4927, 4800, 5006, 4879, 4943, 4816, 4975, + 4848, 4912, 4785, 5014, 4887, 4951, 4824, 4983, 4856, 4920, 4793, 4999, + 4872, 4936, 4809, 4968, 4841, 4905, 4778, 5026, 4899, 4963, 4836, 4995, + 4868, 4932, 4805, 5011, 4884, 4948, 4821, 4980, 4853, 4917, 4790, 5019, + 4892, 4956, 4829, 4988, 4861, 4925, 4798, 5004, 4877, 4941, 4814, 4973, + 4846, 4910, 4783, 5023, 4896, 4960, 4833, 4992, 4865, 4929, 4802, 5008, + 4881, 4945, 4818, 4977, 4850, 4914, 4787, 5016, 4889, 4953, 4826, 4985, + 4858, 4922, 4795, 5001, 4874, 4938, 4811, 4970, 4843, 4907, 4780, 5025, + 4898, 4962, 4835, 4994, 4867, 4931, 4804, 5010, 4883, 4947, 4820, 4979, + 4852, 4916, 4789, 5018, 4891, 4955, 4828, 4987, 4860, 4924, 4797, 5003, + 4876, 4940, 4813, 4972, 4845, 4909, 4782, 5022, 4895, 4959, 4832, 4991, + 4864, 4928, 4801, 5007, 4880, 4944, 4817, 4976, 4849, 4913, 4786, 5015, + 4888, 4952, 4825, 4984, 4857, 4921, 4794, 5000, 4873, 4937, 4810, 4969, + 4842, 4906, 4779, 32332, 32331, 24444, 32277, 24268, 32333, 24446, 32279, + 11452, 38362, 38399, 11476, 24448, 32281, 24428, 32325, 32334, 32251, + 38364, 11456, 32264, 32263, 32327, 32329, 32328, 24393, 32271, 24447, + 32280, 24370, 32248, 24390, 32243, 29571, 29551, 29577, 29549, 34192, + 34205, 29576, 29548, 34189, 34204, 32351, 17556, 34190, 29546, 17557, + 32352, 29547, 29575, 39590, 2559, 2558, 2561, 2562, 32229, 24342, 37904, + 4462, 37906, 37905, 25399, 25363, 975, 8127, 32238, 24359, 33028, 32256, + 24375, 32247, 24266, 38402, 24220, 24219, 38219, 38218, 24221, 38220, + 24218, 38217, 24407, 32294, 38376, 11491, 24401, 32288, 38373, 11488, + 24406, 32293, 38375, 11490, 24400, 32287, 38372, 11487, 24403, 38371, + 32292, 11485, 24405, 24399, 32290, 32285, 24404, 24398, 32291, 32286, + 38370, 11486, 32126, 16930, 37696, 24363, 32240, 32239, 24544, 24366, + 18282, 34773, 24365, 34943, 24316, 32131, 38228, 11420, 38018, 41141, + 41142, 24308, 32197, 24310, 32199, 41136, 41132, 41137, 41133, 24289, + 32175, 24287, 32172, 24286, 32173, 24214, 32107, 24215, 32113, 11360, + 11385, 24223, 32117, 11335, 39308, 27033, 32112, 27034, 960, 5, 34324, + 34325, 38121, 32063, 961, 32064, 30111, 30110, 27032, 27031, 27026, + 27025, 27030, 27028, 27029, 27027, 32085, 16892, 16891, 16889, 16890, + 6632, 6920, 6907, 6911, 6908, 6633, 6625, 6634, 38216, 6915, 6638, 6853, + 6921, 6624, 6626, 34634, 34629, 34700, 34686, 34687, 38154, 37997, 37995, + 32555, 37994, 32189, 24294, 39272, 4475, 4476, 4043, 37703, 37704, 39690, + 3942, 24313, 32202, 24231, 32127, 20788, 37630, 20865, 11413, 34577, + 20790, 33048, 16931, 16932, 20631, 18166, 37386, 11433, 11434, 3921, + 3962, 39650, 3869, 16945, 16948, 16944, 16947, 16943, 16946, 32424, + 32058, 34151, 32059, 3857, 3856, 11367, 38014, 24330, 32215, 37705, + 27777, 28972, 28970, 28971, 28980, 28979, 28975, 28976, 38156, 38155, + 28974, 28180, 34798, 31923, 17583, 20946, 20938, 6925, 980, 24661, 24662, + 24663, 20939, 31925, 20943, 20940, 20944, 20942, 20945, 20941, 21091, + 22892, 41138, 41139, 41140, 31758, 31763, 31761, 31757, 31760, 31759, + 31762, 27770, 27771, 27773, 27772, 31754, 31755, 39325, 28473, 28472, + 32763, 34071, 28468, 28469, 6854, 28470, 6648, 31756, 27774, 28471, + 20961, 32234, 41134, 374, 20957, 38205, 38206, 20956, 20955, 38207, + 38203, 20954, 38202, 20953, 38204, 20958, 8143, 8149, 11368, 11369, 8144, + 25291, 25299, 11358, 11359, 38152, 38153, 34010, 34009, 25296, 25293, + 25301, 25292, 25300, 25290, 25294, 25289, 34061, 25298, 25297, 41135, + 41131, 16937, 20632, 38012, 38013, 37698, 37697, 33854, 8609, 16938, 365, + 1060, 16927, 31764, 16928, 11348, 38147, 37406, 16936, 16940, 24561, + 18301, 24562, 18302, 24553, 18288, 24556, 18291, 24554, 18289, 24555, + 18290, 24552, 18292, 24546, 18284, 24545, 18283, 24543, 18280, 24541, + 18278, 24540, 18277, 24539, 18281, 24542, 18279, 33995, 33993, 33996, + 33994, 11395, 11394, 11391, 11390, 33853, 33852, 33851, 33850, 11361, + 11363, 11362, 18296, 18285, 24550, 18299, 24551, 18300, 34069, 22900, + 34070, 22901, 16933, 31844, 34789, 31845, 34790, 31847, 34792, 31843, + 34788, 31846, 34791, 31842, 34787, 11364, 11373, 34784, 34956, 34783, + 34955, 34782, 34954, 34781, 34953, 34776, 34948, 34777, 34949, 34775, + 34947, 34778, 34950, 34455, 34542, 8138, 8140, 8139, 8141, 34771, 34942, + 34772, 34941, 34945, 34944, 16844, 31662, 37990, 17635, 29543, 33027, + 33033, 33030, 31485, 39274, 11382, 39273, 11380, 25302, 33034, 33032, + 33031, 11349, 11377, 11371, 32067, 11151, 39290, 39289, 11419, 31249, + 31248, 38017, 38020, 38011, 38024, 38023, 11393, 11392, 38021, 22889, + 11376, 39762, 28981, 29560, 29587, 34202, 34215, 24271, 24397, 38366, + 11458, 29557, 29584, 34199, 34212, 24273, 38223, 32260, 32261, 24379, + 24380, 34570, 34563, 34573, 34567, 10965, 10966, 10964, 10963, 11331, + 3948, 39691, 4041, 39763, 3982, 39710, 39688, 3939, 20560, 3943, 3965, + 39702, 3968, 39706, 4001, 4002, 39723, 3941, 39689, 4038, 39759, 24217, + 37399, 24216, 25313, 24436, 24435, 24437, 24438, 24373, 24383, 24382, + 24431, 24433, 24432, 24367, 39589, 16929, 32060, 24360, 32246, 32245, + 24462, 32341, 32061, 32236, 37695, 24362, 24361, 32237, 11472, 33025, + 33023, 39703, 4003, 39724, 3992, 39714, 19893, 19906, 19870, 19868, + 19871, 33997, 2656, 33998, 2655, 3698, 33024, 24413, 38385, 32310, 11461, + 24275, 38227, 29582, 29555, 34197, 34210, 24425, 38396, 32322, 11473, + 8135, 973, 24424, 38387, 32321, 11463, 41412, 41412, 29583, 29556, 34198, + 34211, 24415, 38395, 32312, 11471, 20581, 39303, 24414, 38386, 32311, + 11462, 24426, 38397, 32323, 11474, 24396, 38365, 32274, 11460, 970, 971, + 969, 972, 32051, 32052, 29455, 29456, 17642, 32275, 16935, 35055, 37410, + 37414, 37411, 37413, 3955, 4035, 3997, 3929, 11465, 11466, 38389, 38390, + 24418, 32315, 24417, 32314, 3872, 3873, 3874, 3875, 3876, 3878, 3877, + 3879, 32098, 32099, 32096, 32097, 32093, 32095, 32092, 32094, 38410, + 38215, 31057, 31056, 31058, 2800, 6923, 6637, 4008, 3919, 38122, 20559, + 4042, 3972, 3964, 3967, 3970, 29461, 37922, 4450, 24574, 32408, 39680, + 32409, 34527, 38198, 18828, 31770, 31769, 31768, 31767, 37989, 31869, + 2576, 20615, 31643, 29238, 39707, 3922, 38034, 10075, 19843, 41220, + 22725, 1055, 97, 39413, 31781, 24242, 32135, 34716, 34717, 24434, 38401, + 32330, 11478, 16950, 16949, 32853, 32550, 32548, 32549, 32547, 32551, + 32552, 16934, 38209, 32844, 11416, 31404, 32074, 20088, 18066, 18068, + 18102, 18076, 18072, 18103, 18110, 18073, 18109, 18082, 18078, 18077, + 18071, 18113, 18084, 18085, 18086, 18087, 18089, 18091, 18095, 18099, + 18112, 18074, 18111, 18088, 18090, 18092, 18101, 18070, 18094, 18105, + 18104, 18106, 18096, 18108, 18097, 18098, 18107, 18080, 18067, 18079, + 18075, 18081, 18093, 18100, 18083, 18069, 18114, 18116, 18150, 18124, + 18120, 18151, 18158, 18121, 18157, 18130, 18126, 18125, 18119, 18161, + 18132, 18133, 18134, 18135, 18137, 18139, 18143, 18147, 18160, 18122, + 18159, 18136, 18138, 18140, 18149, 18118, 18142, 18153, 18152, 18154, + 18144, 18156, 18145, 18146, 18155, 18128, 18115, 18127, 18123, 18129, + 18141, 18148, 18131, 18117, 23124, 23771, 23127, 23218, 23236, 23505, + 23997, 23067, 23690, 23113, 23750, 23382, 24164, 22945, 23146, 23286, + 23287, 24117, 23359, 24134, 24116, 23072, 23697, 24062, 23618, 24038, + 23852, 23430, 24189, 27898, 23262, 23386, 8617, 8711, 8667, 8761, 8631, + 8725, 8628, 8722, 8672, 8766, 8662, 8756, 8666, 8760, 8633, 8727, 8664, + 8758, 8670, 8764, 8636, 8730, 8641, 8735, 8673, 8767, 8674, 8768, 8637, + 8731, 8642, 8736, 8669, 8763, 8671, 8765, 8663, 8757, 8665, 8759, 8675, + 8769, 8639, 8733, 8635, 8729, 8668, 8762, 8658, 8752, 8625, 8719, 8653, + 8747, 8621, 8715, 8626, 8720, 8627, 8721, 8622, 8716, 8651, 8745, 8661, + 8755, 8623, 8717, 8649, 8743, 8652, 8746, 8616, 8710, 8624, 8718, 8644, + 8738, 8645, 8739, 8640, 8734, 8647, 8741, 8646, 8740, 8643, 8737, 8650, + 8744, 8648, 8742, 8656, 8750, 8654, 8748, 8655, 8749, 8657, 8751, 8771, + 8772, 8773, 8775, 8776, 8770, 8774, 8619, 8713, 8620, 8714, 8615, 8614, + 8613, 8618, 8712, 41412, 41412, 41412, 41412, 41412, 8709, 8706, 8707, + 8708, 8704, 8705, 8777, 17925, 17951, 17936, 17957, 17958, 17956, 17946, + 17947, 17959, 17940, 17949, 17952, 17954, 17960, 17942, 17945, 17950, + 17944, 17948, 17961, 17941, 17939, 17935, 17955, 17943, 17931, 17934, + 17938, 17933, 17932, 17953, 17937, 17926, 17930, 17928, 17963, 17927, + 17929, 41412, 17962, 41412, 41412, 41412, 41412, 41412, 17924, 41412, + 41412, 37645, 37665, 37666, 37646, 37648, 37635, 37674, 37661, 37664, + 37662, 37663, 37684, 37673, 37649, 37639, 37651, 37667, 37634, 37643, + 37668, 37672, 37650, 37640, 37679, 37644, 37685, 37659, 37633, 37641, + 37675, 37676, 37677, 37638, 37642, 37678, 37687, 37669, 37670, 37647, + 37637, 37632, 37652, 37654, 37653, 37655, 37656, 37671, 37657, 37680, + 37681, 37682, 37658, 37636, 37660, 37683, 37686, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 37688, 37689, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 37631, 17385, 17202, 17289, 17329, 17305, 16992, 17367, 17030, 17220, + 17211, 17112, 17445, 17060, 17045, 17376, 17336, 17021, 17236, 17249, + 17097, 17101, 17100, 17099, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 17319, 17325, 17323, 17320, 17322, 17321, 17324, + 41412, 17011, 17017, 17015, 17012, 17014, 17013, 17016, 41412, 17435, + 17441, 17439, 17436, 17438, 17437, 17440, 41412, 17004, 17010, 17008, + 17005, 17007, 17006, 17009, 41412, 17271, 17277, 17275, 17272, 17274, + 17273, 17276, 41412, 17180, 17186, 17184, 17181, 17183, 17182, 17185, + 41412, 17412, 17418, 17416, 17413, 17415, 17414, 17417, 41412, 17123, + 17129, 17127, 17124, 17126, 17125, 17128, 41412, 8197, 8235, 8232, 8199, + 8229, 8230, 8236, 8203, 8204, 8205, 8212, 8234, 8206, 8200, 8228, 8225, + 8227, 8231, 8215, 8214, 8233, 8201, 8237, 8211, 8198, 8224, 8220, 8222, + 8209, 8223, 8196, 8208, 32109, 32108, 24293, 32179, 24235, 32124, 31946, + 31945, 11334, 24296, 32187, 31954, 24276, 32159, 11675, 31247, 17634, + 32069, 20622, 11333, 11457, 38349, 11336, 11384, 20970, 31206, 20618, + 37701, 24256, 32149, 37700, 37699, 24325, 32193, 37931, 37935, 4455, + 4458, 24281, 32164, 24234, 32129, 38150, 30705, 34630, 17601, 32084, + 39306, 32344, 39823, 38126, 31944, 31956, 38137, 10662, 10663, 38127, + 37920, 38162, 37417, 34730, 39309, 39806, 6104, 11352, 32083, 11355, + 10669, 11372, 20971, 20972, 25335, 25336, 11370, 11330, 38022, 27135, + 31246, 31903, 8780, 8817, 8818, 37789, 27134, 27136, 24291, 32177, 24290, + 32176, 37912, 37916, 4441, 4445, 30112, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 8021, 7974, 8024, 8023, 8022, 8017, 7945, 8042, 8056, 8055, 7978, 8025, + 8038, 8037, 8007, 8006, 8005, 8004, 8034, 8043, 8033, 8032, 7993, 7992, + 7996, 8020, 41412, 7977, 8040, 8014, 7979, 8012, 7972, 8048, 8047, 7986, + 8016, 8015, 8026, 7976, 7980, 8001, 7943, 7985, 8036, 8035, 7948, 8031, + 7959, 8054, 8053, 8052, 8051, 8009, 8039, 8019, 7984, 8057, 7947, 7946, + 8008, 8013, 7990, 7989, 7988, 8044, 7975, 8050, 8049, 7963, 8027, 7995, + 7962, 7961, 7987, 7971, 8028, 8046, 8045, 7973, 7955, 8003, 8002, 7958, + 7956, 8011, 8010, 8018, 7949, 7965, 7957, 7967, 7953, 7983, 7982, 7981, + 7951, 7994, 7970, 7944, 7991, 7952, 7969, 7960, 8029, 8030, 7954, 8000, + 7950, 7998, 7966, 7999, 7968, 8041, 7997, 7964, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 21290, + 21272, 21206, 21326, 21314, 21258, 21358, 21273, 21286, 21269, 21221, + 21225, 21210, 21195, 21261, 21347, 21293, 21264, 21298, 21375, 21332, + 21303, 21255, 21360, 21204, 21315, 21191, 21295, 21166, 21284, 21220, + 21218, 21313, 21241, 21243, 21223, 21173, 21372, 21198, 21306, 21260, + 21342, 21265, 21193, 21331, 21282, 21304, 21373, 21291, 21356, 21216, + 21321, 21207, 21275, 21359, 21322, 21181, 21340, 21182, 21334, 21252, + 21249, 21212, 21250, 21183, 21301, 21312, 21205, 21168, 21343, 21288, + 21345, 21311, 21285, 21355, 21266, 21335, 21200, 21366, 21211, 21194, + 21240, 21189, 21333, 21365, 21232, 21190, 21227, 21209, 21248, 21327, + 21229, 21197, 21213, 21296, 21262, 21276, 21350, 21339, 21271, 21377, + 21230, 21177, 21323, 21208, 21368, 21344, 21203, 21226, 21328, 21164, + 21337, 21330, 21354, 21244, 21188, 21338, 21169, 21305, 21324, 21263, + 21289, 21319, 21238, 21294, 21167, 21297, 21217, 21184, 21277, 21278, + 21316, 21165, 21280, 21351, 21292, 21178, 21336, 21196, 21245, 21349, + 21259, 21174, 21364, 21192, 21367, 21317, 21257, 21329, 21361, 21185, + 21299, 21170, 21318, 21308, 21307, 21239, 21180, 21187, 21171, 21281, + 21363, 21199, 21371, 21202, 21362, 21242, 21274, 21247, 21283, 21325, + 21320, 21300, 21176, 21374, 21228, 21267, 21346, 21270, 21341, 21268, + 21370, 21235, 21219, 21253, 21236, 21256, 21179, 21348, 21251, 21231, + 21309, 21186, 21246, 21233, 21172, 21310, 21201, 21369, 21254, 21376, + 21279, 21175, 21224, 21237, 21353, 21215, 21302, 21287, 21222, 21352, + 21214, 21357, 21234, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 20655, + 20653, 20654, 20652, 20667, 20663, 20662, 20659, 20660, 20661, 20657, + 20656, 20664, 20658, 20668, 20666, 20752, 20651, 20750, 11140, 20990, + 20751, 20650, 20670, 24213, 32106, 24232, 32125, 24228, 32122, 24307, + 32196, 24222, 32116, 31783, 18059, 24304, 32191, 24309, 32198, 24312, + 32201, 24311, 32200, 39588, 32066, 11366, 25333, 31784, 19739, 19732, + 19729, 19735, 19734, 19737, 19736, 19740, 19738, 20748, 20747, 20669, + 20746, 19398, 19397, 39594, 39281, 39284, 39282, 39285, 39283, 6906, + 20744, 19733, 19731, 19730, 39280, 25992, 31401, 20749, 20745, 41412, + 20464, 20450, 20466, 20544, 20468, 20546, 20465, 20543, 20467, 20545, + 20505, 20495, 20507, 20497, 20509, 20499, 20506, 20496, 20508, 20498, + 20469, 20530, 20471, 20532, 20473, 20534, 20470, 20531, 20472, 20533, + 20525, 20490, 20527, 20492, 20463, 20529, 20494, 20526, 20491, 20528, + 20493, 20485, 20487, 20489, 20486, 20488, 20500, 20480, 20515, 20502, + 20474, 20517, 20504, 20483, 20519, 20501, 20481, 20516, 20503, 20482, + 20518, 20510, 20512, 20514, 20511, 20513, 20457, 20539, 20459, 20541, + 20458, 20540, 20520, 20522, 20524, 20521, 20523, 20453, 20535, 20537, + 20536, 20538, 20484, 20542, 20460, 20461, 41412, 41412, 8405, 8404, + 21616, 21615, 20548, 20547, 20449, 21613, 21543, 21472, 21545, 21606, + 21547, 21608, 21544, 21605, 21546, 21607, 21568, 21558, 21570, 21560, + 21572, 21562, 21569, 21559, 21571, 21561, 21548, 21593, 21550, 21595, + 21552, 21597, 21549, 21594, 21551, 21596, 21583, 21553, 21585, 21555, + 21530, 21587, 21557, 21584, 21554, 21586, 21556, 21510, 21512, 21514, + 21511, 21513, 21563, 21487, 21573, 21565, 21481, 21575, 21567, 21490, + 21577, 21564, 21488, 21574, 21566, 21489, 21576, 21505, 21491, 21508, + 21506, 21507, 21535, 21602, 21537, 21604, 21536, 21603, 21578, 21580, + 21582, 21579, 21581, 21531, 21598, 21600, 21599, 21601, 21509, 21592, + 21525, 21526, 21588, 21590, 21589, 21591, 21612, 21614, 21611, 21609, + 21610, 41412, 41412, 41412, 41412, 41412, 4422, 4430, 4429, 4427, 4426, + 4433, 4398, 4418, 4386, 4414, 4428, 4424, 4431, 4435, 4411, 4417, 4421, + 4432, 4410, 4416, 4420, 4366, 4402, 4378, 4383, 4367, 4384, 4369, 4409, + 4371, 4379, 4372, 4380, 4385, 4391, 4376, 4397, 4434, 4399, 4388, 4394, + 4405, 4401, 41412, 19651, 19695, 19652, 19657, 19658, 19660, 19687, + 19698, 19673, 19674, 19676, 19678, 19685, 19681, 19677, 19684, 19653, + 19663, 19697, 19664, 19688, 19700, 19645, 19640, 19694, 19639, 19650, + 19686, 19671, 19724, 19635, 19638, 19721, 19722, 19642, 19641, 19708, + 19707, 19725, 19704, 19705, 19726, 19713, 19727, 19703, 19702, 19706, + 19717, 19643, 19723, 19644, 19728, 19696, 19659, 19662, 19661, 19675, + 19682, 19679, 19680, 19683, 19654, 19656, 19655, 19649, 19670, 19668, + 19665, 19666, 19669, 19667, 19647, 19648, 19690, 19691, 19693, 19692, + 19689, 19672, 19701, 19710, 19712, 19711, 19646, 19699, 19709, 19714, + 19715, 19716, 19719, 19718, 19720, 19636, 19637, 41412, 20644, 20647, + 20646, 20642, 20640, 20636, 20643, 20638, 20634, 20637, 20645, 20641, + 20635, 20648, 20649, 20639, 4423, 4412, 4425, 4389, 4382, 4381, 4408, + 4404, 4396, 4373, 4392, 4377, 4395, 4400, 4368, 4370, 4375, 4407, 4403, + 4393, 4364, 4365, 4363, 4362, 4387, 4419, 4413, 4361, 4390, 4415, 4406, + 4374, 8088, 8091, 8092, 8090, 8078, 8064, 8070, 8059, 8069, 8082, 8071, + 8067, 8060, 8068, 8065, 8094, 8058, 8077, 8073, 8086, 8093, 8063, 8072, + 8081, 8080, 8087, 8085, 8074, 8076, 8089, 8084, 8079, 8061, 8066, 8075, + 8095, 8062, 8083, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 20665, 21528, 21540, 21541, 21529, 21539, 21515, 21517, 21519, + 21516, 21518, 21542, 21520, 21522, 21524, 21521, 21523, 31261, 31265, + 31277, 31271, 31263, 31269, 31273, 31279, 31254, 31252, 31259, 31275, + 31267, 31257, 31262, 31266, 31278, 31272, 31264, 31270, 31274, 31280, + 31255, 31253, 31260, 31276, 31268, 31258, 31256, 31318, 31317, 41412, + 31316, 31312, 31310, 31291, 31289, 31309, 31301, 31286, 31295, 31311, + 31294, 31288, 31314, 31313, 31293, 31285, 31308, 31306, 31315, 31303, + 31296, 31304, 31287, 31282, 31292, 31299, 31283, 31305, 31307, 31284, + 31297, 31281, 31290, 31298, 31302, 31300, 6725, 6713, 6735, 6714, 6879, + 6893, 6881, 6863, 6860, 6876, 6874, 6856, 31400, 6894, 6900, 6899, 6896, + 6895, 6898, 6897, 6902, 6901, 6880, 6882, 6888, 6887, 6884, 6883, 6673, + 6677, 6689, 6683, 6675, 6681, 6685, 6666, 6664, 6662, 6671, 6687, 6679, + 6669, 6674, 6678, 6690, 6684, 6676, 6682, 6686, 6667, 6665, 6663, 6672, + 6688, 6680, 6670, 6800, 6799, 6668, 22723, 6748, 6744, 6742, 6710, 6708, + 6740, 6731, 6705, 6723, 6743, 6721, 6707, 6746, 6745, 6719, 6703, 6734, + 6739, 6712, 6736, 6724, 6737, 6706, 6699, 6715, 6730, 6720, 6709, 6733, + 6704, 6741, 6696, 6747, 6727, 6700, 6698, 6711, 6701, 6716, 6717, 6729, + 6718, 6728, 6738, 6732, 6702, 6726, 6694, 6722, 6886, 6885, 6890, 6889, + 6862, 6864, 6870, 6869, 6866, 6865, 6868, 6867, 6872, 6871, 6859, 20735, + 20738, 20739, 20677, 20740, 20736, 20737, 20676, 20742, 20743, 20741, + 20709, 34415, 34381, 34383, 24660, 6794, 6796, 6798, 6795, 6797, 6757, + 6759, 6761, 6758, 6760, 6777, 6779, 6781, 6778, 6780, 6782, 6784, 6786, + 6783, 6785, 6767, 6769, 6771, 6768, 6770, 6752, 6754, 6756, 6753, 6755, + 6762, 6764, 6766, 6763, 6765, 6791, 6793, 6792, 6772, 6774, 6776, 6773, + 6775, 6787, 6789, 6788, 6790, 34379, 34340, 34342, 34341, 34343, 34418, + 34419, 34582, 34382, 34375, 34508, 34512, 34427, 34425, 34426, 34390, + 34391, 34395, 34394, 34441, 34393, 34428, 34431, 34429, 34430, 34396, + 34397, 34438, 34439, 34440, 34437, 34436, 34548, 34549, 34556, 34550, + 34551, 34366, 34370, 34371, 34561, 34501, 34502, 34403, 34515, 34516, + 34347, 34524, 34522, 34523, 34350, 34410, 34409, 34349, 34411, 34404, + 34521, 34519, 34405, 34520, 34518, 34352, 34525, 34351, 34408, 34526, + 34406, 34407, 34465, 34466, 34469, 34468, 34467, 34475, 34476, 34477, + 34472, 34473, 34474, 34580, 34579, 34581, 34543, 34544, 34546, 34545, + 34541, 34540, 34562, 20733, 20734, 20716, 20718, 20725, 20724, 20731, + 20729, 20720, 20727, 20719, 20722, 20715, 20717, 20726, 20723, 20732, + 20730, 20721, 20728, 20710, 20714, 20713, 20712, 20711, 34413, 34365, + 34345, 34348, 34513, 34529, 34367, 34369, 34368, 34423, 34376, 34380, + 34377, 34378, 34357, 34517, 34500, 34482, 34464, 34424, 34446, 34470, + 34400, 34354, 34443, 34530, 34503, 34483, 34484, 34497, 34447, 34416, + 34442, 34494, 34398, 34560, 34485, 34498, 34374, 34449, 34389, 34504, + 34486, 34479, 34358, 34432, 34481, 34360, 34463, 34435, 34480, 34359, + 34462, 34434, 34459, 34460, 34514, 34445, 34496, 34399, 34537, 34538, + 34539, 34534, 34505, 34487, 34499, 34535, 34506, 34488, 34490, 34451, + 34491, 34536, 34507, 34489, 34492, 34452, 34493, 34444, 34461, 34344, + 34353, 34363, 34364, 34361, 34356, 34372, 34401, 34402, 34412, 34417, + 34448, 34433, 34450, 34456, 34457, 34454, 34458, 34471, 34478, 34495, + 34531, 34532, 34528, 34533, 34557, 34558, 34578, 34346, 34338, 20708, + 20693, 20681, 20700, 20699, 20706, 20704, 20695, 20702, 20694, 20697, + 20692, 20680, 20701, 20698, 20707, 20705, 20696, 20703, 20682, 20690, + 20688, 20687, 20684, 20683, 20686, 20685, 20691, 20689, 20678, 20679, + 34392, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 20426, + 20429, 20390, 20440, 20437, 20385, 20423, 20402, 20419, 20436, 20414, + 20420, 20395, 20397, 20407, 20441, 20394, 20438, 20378, 20384, 20382, + 20401, 20421, 20415, 20403, 20400, 20408, 20399, 20424, 20427, 20406, + 20392, 20416, 20398, 20413, 20393, 20428, 20411, 20409, 20388, 20387, + 20405, 20383, 20386, 20396, 20412, 20410, 20430, 20417, 20425, 20422, + 20434, 20389, 20432, 20380, 20431, 20433, 20435, 20391, 20439, 20404, + 20418, 20379, 20381, 40345, 40352, 40344, 40351, 40349, 40350, 40347, + 40348, 41120, 41121, 41118, 41119, 41117, 41115, 41116, 41124, 41125, + 41122, 41123, 41127, 41126, 40992, 40012, 40013, 40006, 40011, 40009, + 40010, 40007, 40008, 40016, 40017, 40014, 40015, 39997, 39995, 39996, + 40020, 40021, 40018, 40019, 40005, 40003, 40004, 40001, 40002, 39994, + 40000, 39999, 39998, 40026, 40027, 40022, 40025, 40024, 40023, 40724, + 40725, 40719, 40723, 40722, 40720, 40721, 40728, 40729, 40726, 40727, + 40713, 40711, 40712, 40732, 40733, 40730, 40731, 40717, 40718, 40710, + 40716, 40715, 40714, 40738, 40739, 40734, 40737, 40736, 40735, 39980, + 39981, 39974, 39979, 39977, 39978, 39975, 39976, 39984, 39985, 39982, + 39983, 39965, 39963, 39964, 39988, 39989, 39986, 39987, 39973, 39971, + 39972, 39969, 39970, 39962, 39968, 39967, 39966, 39992, 39993, 39990, + 39991, 40527, 40528, 40522, 40526, 40525, 40523, 40524, 40531, 40532, + 40529, 40530, 40535, 40536, 40533, 40534, 40541, 40542, 40537, 40540, + 40539, 40538, 40547, 40548, 40543, 40546, 40545, 40544, 40270, 40271, + 40265, 40269, 40268, 40266, 40267, 40274, 40275, 40272, 40273, 40259, + 40257, 40258, 40278, 40279, 40276, 40277, 40263, 40264, 40256, 40262, + 40261, 40260, 40284, 40280, 40283, 40282, 40281, 40506, 40507, 40501, + 40505, 40504, 40502, 40503, 40510, 40511, 40508, 40509, 40494, 40495, + 40492, 40493, 40514, 40515, 40512, 40513, 40521, 40520, 40499, 40500, + 40491, 40498, 40497, 40496, 40518, 40519, 40516, 40517, 40151, 40152, + 40149, 40150, 40147, 40148, 40145, 40146, 40144, 40142, 40143, 40161, + 40162, 40157, 40160, 40159, 40158, 40155, 40156, 40153, 40154, 40970, + 40971, 40964, 40969, 40967, 40968, 40965, 40966, 40974, 40975, 40972, + 40973, 40978, 40979, 40976, 40977, 40963, 40962, 40984, 40985, 40980, + 40983, 40982, 40981, 40990, 40991, 40986, 40989, 40988, 40987, 40129, + 40130, 40124, 40128, 40127, 40125, 40126, 40136, 40137, 40134, 40135, + 40118, 40117, 40140, 40141, 40138, 40139, 40133, 40131, 40132, 40122, + 40123, 40116, 40121, 40120, 40119, 40949, 40950, 40944, 40948, 40947, + 40945, 40946, 40956, 40957, 40954, 40955, 40937, 40938, 40935, 40936, + 40960, 40961, 40958, 40959, 40953, 40951, 40952, 40942, 40943, 40934, + 40941, 40940, 40939, 40103, 40104, 40098, 40102, 40101, 40099, 40100, + 40110, 40111, 40108, 40109, 40092, 40090, 40091, 40114, 40115, 40112, + 40113, 40107, 40105, 40106, 40096, 40097, 40089, 40095, 40094, 40093, + 40553, 40554, 40549, 40552, 40551, 40550, 40560, 40561, 40558, 40559, + 40564, 40565, 40562, 40563, 40557, 40555, 40556, 40570, 40571, 40566, + 40569, 40568, 40567, 40300, 40301, 40294, 40299, 40297, 40298, 40295, + 40296, 40304, 40305, 40302, 40303, 40289, 40288, 40286, 40287, 40285, + 40293, 40291, 40292, 40290, 40698, 40699, 40693, 40697, 40696, 40694, + 40695, 40702, 40700, 40701, 40687, 40685, 40686, 40708, 40709, 40706, + 40707, 40705, 40703, 40704, 40691, 40692, 40684, 40690, 40689, 40688, + 40238, 40239, 40233, 40237, 40236, 40234, 40235, 40248, 40249, 40246, + 40247, 40227, 40225, 40226, 40245, 40243, 40244, 40242, 40240, 40241, + 40231, 40232, 40224, 40230, 40229, 40228, 40254, 40255, 40250, 40253, + 40252, 40251, 40453, 40454, 40447, 40452, 40450, 40451, 40448, 40449, + 40457, 40458, 40455, 40456, 40437, 40438, 40435, 40436, 40461, 40462, + 40459, 40460, 40446, 40444, 40445, 40442, 40443, 40434, 40441, 40440, + 40439, 40467, 40468, 40463, 40466, 40465, 40464, 40207, 40208, 40201, + 40206, 40204, 40205, 40202, 40203, 40211, 40212, 40209, 40210, 40194, + 40195, 40192, 40193, 40219, 40220, 40217, 40218, 40215, 40216, 40213, + 40214, 40199, 40200, 40191, 40198, 40197, 40196, 40420, 40421, 40415, + 40419, 40418, 40416, 40417, 40424, 40425, 40422, 40423, 40409, 40407, + 40408, 40432, 40433, 40430, 40431, 40428, 40429, 40426, 40427, 40413, + 40414, 40406, 40412, 40411, 40410, 40167, 40168, 40163, 40166, 40164, + 40165, 40181, 40182, 40179, 40180, 40172, 40173, 40170, 40171, 40189, + 40190, 40187, 40188, 40185, 40186, 40183, 40184, 40177, 40178, 40169, + 40176, 40175, 40174, 40490, 40489, 40483, 40484, 40481, 40482, 40472, + 40470, 40471, 40487, 40488, 40485, 40486, 40480, 40478, 40479, 40476, + 40477, 40469, 40475, 40474, 40473, 40319, 40320, 40313, 40318, 40316, + 40317, 40314, 40315, 40323, 40324, 40321, 40322, 40308, 40309, 40306, + 40307, 40327, 40328, 40325, 40326, 40312, 40310, 40311, 40580, 40578, + 40579, 40583, 40584, 40581, 40582, 40573, 40574, 40572, 40587, 40588, + 40585, 40586, 40577, 40575, 40576, 40223, 40222, 40221, 40338, 40339, + 40336, 40337, 40331, 40332, 40329, 40330, 40342, 40343, 40340, 40341, + 40335, 40333, 40334, 41004, 41005, 41002, 41003, 40995, 40993, 40994, + 41001, 40999, 41000, 40998, 40996, 40997, 41067, 41068, 41062, 41066, + 41065, 41063, 41064, 41103, 41104, 41101, 41102, 41056, 41054, 41055, + 41107, 41108, 41105, 41106, 41100, 41098, 41099, 41060, 41061, 41053, + 41059, 41058, 41057, 41113, 41114, 41109, 41112, 41111, 41110, 40073, + 40074, 40067, 40072, 40070, 40071, 40068, 40069, 40077, 40078, 40075, + 40076, 40058, 40056, 40057, 40081, 40082, 40079, 40080, 40066, 40064, + 40065, 40062, 40063, 40055, 40061, 40060, 40059, 40087, 40088, 40083, + 40086, 40085, 40084, 41081, 41082, 41075, 41080, 41078, 41079, 41076, + 41077, 41085, 41086, 41083, 41084, 41074, 41072, 41073, 41071, 41069, + 41070, 41091, 41087, 41090, 41089, 41088, 41096, 41097, 41092, 41095, + 41094, 41093, 40670, 40671, 40665, 40669, 40668, 40666, 40667, 40674, + 40675, 40672, 40673, 40658, 40657, 40664, 40663, 40683, 40682, 40662, + 40656, 40661, 40660, 40659, 40680, 40681, 40676, 40679, 40678, 40677, + 40915, 40916, 40910, 40914, 40913, 40911, 40912, 40922, 40923, 40920, + 40921, 40904, 40902, 40903, 40926, 40927, 40924, 40925, 40919, 40917, + 40918, 40908, 40909, 40901, 40907, 40906, 40905, 40932, 40933, 40928, + 40931, 40930, 40929, 40851, 40852, 40846, 40850, 40849, 40847, 40848, + 40858, 40859, 40856, 40857, 40862, 40863, 40860, 40861, 40855, 40853, + 40854, 40866, 40867, 40864, 40865, 40872, 40873, 40868, 40871, 40870, + 40869, 41037, 41038, 41035, 41036, 41029, 41027, 41028, 41045, 41046, + 41043, 41044, 41041, 41042, 41039, 41040, 41033, 41034, 41026, 41032, + 41031, 41030, 41051, 41052, 41047, 41050, 41049, 41048, 40039, 40040, + 40037, 40038, 40031, 40032, 40029, 40030, 40047, 40048, 40045, 40046, + 40043, 40044, 40041, 40042, 40036, 40028, 40035, 40034, 40033, 40053, + 40054, 40049, 40052, 40051, 40050, 40819, 40818, 40798, 40797, 40810, + 40811, 40808, 40809, 40806, 40807, 40804, 40805, 40802, 40803, 40796, + 40801, 40800, 40799, 40816, 40817, 40812, 40815, 40814, 40813, 40619, + 40620, 40617, 40618, 40616, 40614, 40615, 40623, 40624, 40621, 40622, + 40629, 40630, 40625, 40628, 40627, 40626, 40635, 40636, 40631, 40634, + 40633, 40632, 40885, 40886, 40883, 40884, 40877, 40875, 40876, 40893, + 40894, 40891, 40892, 40889, 40890, 40887, 40888, 40881, 40882, 40874, + 40880, 40879, 40878, 40899, 40900, 40895, 40898, 40897, 40896, 40834, + 40835, 40832, 40833, 40823, 40821, 40822, 40838, 40839, 40836, 40837, + 40831, 40829, 40830, 40827, 40828, 40820, 40826, 40825, 40824, 40844, + 40845, 40840, 40843, 40842, 40841, 40394, 40395, 40388, 40393, 40391, + 40392, 40389, 40390, 40381, 40382, 40379, 40380, 40398, 40399, 40396, + 40397, 40386, 40387, 40378, 40385, 40384, 40383, 40404, 40405, 40400, + 40403, 40402, 40401, 40756, 40757, 40750, 40755, 40753, 40754, 40751, + 40752, 40743, 40744, 40741, 40742, 40760, 40761, 40758, 40759, 40748, + 40749, 40740, 40747, 40746, 40745, 40766, 40767, 40762, 40765, 40764, + 40763, 40368, 40369, 40362, 40367, 40365, 40366, 40363, 40364, 40356, + 40354, 40355, 40372, 40373, 40370, 40371, 40360, 40361, 40353, 40359, + 40358, 40357, 40376, 40377, 40374, 40375, 40602, 40603, 40596, 40601, + 40599, 40600, 40597, 40598, 40591, 40590, 40606, 40607, 40604, 40605, + 40595, 40589, 40594, 40593, 40592, 40612, 40613, 40608, 40611, 40610, + 40609, 40650, 40651, 40644, 40649, 40647, 40648, 40645, 40646, 40640, + 40638, 40639, 40654, 40655, 40652, 40653, 40642, 40643, 40637, 40641, + 41012, 41013, 41006, 41011, 41009, 41010, 41007, 41008, 41025, 41024, + 41016, 41017, 41014, 41015, 41022, 41023, 41018, 41021, 41020, 41019, + 40784, 40785, 40778, 40783, 40781, 40782, 40779, 40780, 40771, 40772, + 40769, 40770, 40788, 40789, 40786, 40787, 40776, 40777, 40768, 40775, + 40774, 40773, 40794, 40795, 40790, 40793, 40792, 40791, 41412, 41412, + 41412, 39959, 39932, 39930, 39937, 39911, 39947, 39918, 39920, 39936, + 39923, 39934, 39907, 39935, 39953, 39941, 39922, 39948, 39921, 39954, + 39912, 39915, 39908, 39917, 39938, 39949, 39961, 39926, 39957, 39942, + 39925, 39952, 39950, 39946, 39951, 39958, 39929, 39939, 39928, 39919, + 39927, 39960, 39916, 39943, 39933, 39910, 39909, 39914, 39924, 39944, + 39955, 39945, 39913, 39956, 39940, 39931, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 25263, 25252, 25251, 25235, 25233, + 25232, 25246, 25250, 25249, 25265, 25244, 25243, 25234, 25231, 25230, + 25267, 25240, 25266, 25254, 25257, 25258, 25239, 25248, 25269, 25247, + 25264, 25268, 25253, 25256, 25245, 25259, 25260, 25241, 25242, 25270, + 25261, 25236, 25237, 25238, 25262, 25226, 25229, 25227, 25225, 25228, + 25224, 25271, 25272, 38679, 38680, 38516, 38665, 38666, 38639, 38442, + 38449, 38552, 38525, 38559, 38499, 38625, 38653, 38477, 38470, 38428, + 38421, 38543, 38646, 38435, 38578, 38463, 38456, 38491, 38484, 38618, + 38632, 38597, 38660, 38538, 38584, 38505, 38566, 38611, 38604, 38688, + 38689, 38520, 38521, 38674, 38675, 38641, 38444, 38451, 38554, 38531, + 38561, 38502, 38627, 38655, 38479, 38472, 38430, 38423, 38547, 38648, + 38437, 38580, 38465, 38458, 38493, 38486, 38620, 38634, 38599, 38662, + 38539, 38589, 38510, 38568, 38613, 38606, 38686, 38687, 38591, 38518, + 38519, 38672, 38673, 38640, 38443, 38450, 38553, 38529, 38530, 38560, + 38501, 38626, 38654, 38478, 38471, 38429, 38422, 38546, 38647, 38436, + 38579, 38464, 38457, 38492, 38485, 38619, 38633, 38598, 38661, 38535, + 38536, 38588, 38509, 38567, 38612, 38605, 38683, 38684, 38514, 38669, + 38670, 38637, 38440, 38447, 38550, 38528, 38557, 38497, 38623, 38651, + 38475, 38468, 38426, 38419, 38545, 38644, 38433, 38576, 38461, 38454, + 38489, 38482, 38616, 38630, 38595, 38658, 38534, 38587, 38508, 38564, + 38609, 38602, 38690, 38691, 38522, 38523, 38676, 38677, 38642, 38445, + 38452, 38555, 38532, 38562, 38503, 38628, 38656, 38480, 38473, 38431, + 38424, 38548, 38649, 38438, 38581, 38466, 38459, 38494, 38487, 38621, + 38635, 38600, 38663, 38540, 38590, 38511, 38569, 38614, 38607, 38682, + 38685, 38593, 38512, 38513, 38668, 38671, 38636, 38439, 38446, 38549, + 38527, 38556, 38495, 38496, 38622, 38650, 38474, 38467, 38425, 38418, + 38544, 38643, 38432, 38570, 38460, 38453, 38488, 38481, 38615, 38629, + 38594, 38657, 38533, 38586, 38507, 38563, 38608, 38601, 38678, 38681, + 38592, 38515, 38517, 38664, 38667, 38638, 38441, 38448, 38551, 38524, + 38526, 38558, 38498, 38500, 38624, 38652, 38476, 38469, 38427, 38420, + 38541, 38645, 38434, 38577, 38462, 38455, 38490, 38483, 38617, 38631, + 38596, 38659, 38537, 38583, 38585, 38504, 38506, 38565, 38610, 38603, + 38582, 38542, 38415, 38416, 38417, 38573, 38574, 38571, 38695, 38698, + 38701, 38700, 38704, 38696, 38703, 38694, 38692, 38699, 38702, 38693, + 38697, 38711, 38713, 38710, 38709, 38706, 38705, 38708, 38707, 38714, + 38712, 38575, 38572, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 10415, 10616, 10304, 10463, 10254, 10557, 10359, + 10518, 10300, 10459, 10380, 10543, 10288, 10447, 10247, 10544, 10408, + 10609, 10356, 10515, 10256, 10559, 10357, 10516, 10296, 10455, 10289, + 10448, 10353, 10512, 10410, 10611, 10255, 10558, 10399, 10577, 10396, + 10574, 10397, 10575, 10379, 10542, 10286, 10445, 10301, 10460, 10430, + 8251, 8243, 8194, 8244, 33999, 8218, 8207, 8221, 8217, 8226, 8219, 8216, + 8213, 8249, 8238, 10429, 10433, 10310, 10469, 10308, 10467, 10420, 10621, + 10298, 10457, 10306, 10465, 10260, 10585, 10268, 10594, 10264, 10589, + 10263, 10588, 10266, 10592, 10344, 10503, 10394, 10572, 10302, 10461, + 10297, 10456, 27910, 27947, 8202, 8210, 3462, 2824, 3465, 2826, 3461, + 3437, 3454, 3464, 2853, 3463, 2829, 3434, 3440, 3439, 2827, 2833, 3453, + 2852, 2846, 2832, 3449, 2840, 3444, 3450, 3443, 3447, 2822, 2818, 2850, + 2849, 2844, 3456, 3446, 3457, 3458, 2851, 2816, 3430, 2845, 2848, 3433, + 3459, 3431, 2813, 3442, 2831, 2838, 2855, 3436, 3441, 2817, 2841, 2842, + 2843, 3445, 3432, 2815, 2814, 3460, 2854, 2830, 3435, 2828, 2819, 2835, + 3438, 2834, 2837, 3455, 2825, 2839, 2836, 3452, 2823, 3451, 2847, 3448, + 2812, 2820, 2821, 2809, 2808, 3466, 3468, 2811, 2810, 3467, 3469, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 27907, 27903, 27906, + 27902, 27908, 27904, 27909, 27905, 27962, 27977, 28005, 27984, 27967, + 27961, 27976, 28004, 27983, 27966, 27963, 27978, 28006, 27988, 27968, + 27954, 27953, 27955, 27999, 28018, 28015, 28017, 28016, 27996, 28166, + 28167, 23034, 23638, 23035, 23639, 23074, 23700, 23301, 24063, 23300, + 24020, 22974, 23560, 22975, 23561, 23443, 23451, 22948, 23516, 22949, + 23517, 22950, 23518, 22946, 23514, 22947, 23515, 22951, 23519, 23245, + 23944, 23115, 23752, 23112, 23749, 23116, 23753, 22960, 23537, 23134, + 23776, 23167, 23851, 23168, 23853, 23214, 23891, 23219, 23896, 23217, + 23894, 23220, 23897, 23227, 23909, 23226, 23906, 23242, 23934, 23247, + 23947, 23342, 24113, 23351, 24125, 23346, 24120, 23295, 24014, 23296, + 24015, 23350, 24124, 23036, 23656, 23103, 23739, 22976, 23562, 28175, + 23598, 23797, 23811, 23834, 23946, 23428, 24058, 24110, 23096, 23728, + 23097, 23729, 23098, 23285, 24026, 23290, 24056, 23099, 23731, 23100, + 23732, 23101, 23733, 27982, 27951, 28028, 23268, 23970, 23288, 23781, + 23459, 23160, 23825, 22970, 23550, 23547, 23694, 22954, 23522, 23043, + 23663, 23347, 24121, 23348, 24122, 23349, 24123, 23051, 23674, 23119, + 23757, 23163, 23830, 23240, 23931, 23264, 23968, 23070, 23244, 23271, + 23122, 23267, 23450, 23289, 23292, 23106, 22977, 22961, 23539, 23212, + 23889, 23338, 24101, 23056, 23679, 23057, 23680, 23058, 23681, 23209, + 23880, 22943, 23511, 22968, 23265, 23388, 22981, 23564, 23261, 23962, + 23248, 23260, 23961, 23224, 23903, 22973, 23556, 22994, 23591, 22993, + 23588, 23149, 23810, 23270, 23988, 23138, 23786, 23139, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 27895, 27882, + 27885, 27894, 23243, 23935, 23398, 27877, 28107, 23433, 23396, 23397, + 23394, 23395, 23393, 34995, 34997, 35007, 34999, 34996, 34998, 35006, + 34987, 34986, 34983, 34982, 35005, 34981, 34980, 34985, 34984, 34975, + 34974, 34969, 34968, 34977, 34976, 34971, 34970, 34993, 34989, 34988, + 34979, 34978, 34992, 34973, 34991, 34972, 34994, 34990, 35009, 35011, + 35012, 35010, 35008, 35000, 35001, 35002, 35003, 35004, 41412, 41412, + 41412, 29568, 29567, 29570, 29565, 29566, 29569, 29562, 29561, 29564, + 29563, 41412, 41412, 41412, 41412, 41412, 41412, 31537, 31536, 31525, + 31526, 31513, 31515, 31547, 31528, 31535, 31534, 31518, 31529, 31539, + 31538, 31544, 31549, 31531, 31530, 31517, 31552, 31540, 31541, 31519, + 31554, 31551, 31548, 31520, 31521, 31546, 31510, 31555, 31557, 31542, + 31556, 31550, 31553, 31545, 31524, 31543, 31562, 31563, 31533, 31532, + 31516, 31527, 31511, 31523, 31522, 31512, 31561, 31564, 31514, 31560, + 31565, 31559, 31558, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 32682, 32684, 32631, 32632, 32652, 32653, 32648, 32649, 32643, + 32644, 32645, 32646, 32675, 32676, 32633, 32650, 32651, 32634, 32672, + 32671, 32668, 32667, 32656, 32666, 32665, 32670, 32669, 32658, 32640, + 32639, 32636, 32635, 32657, 32642, 32641, 32638, 32637, 32659, 32674, + 32673, 32664, 32663, 32678, 32680, 32679, 32655, 32647, 32660, 32661, + 32662, 32677, 32654, 32697, 32698, 32709, 32710, 32701, 32702, 32703, + 32704, 32705, 32706, 32711, 32712, 32699, 32707, 32708, 32700, 32683, + 32681, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 32686, + 32685, 32693, 32695, 32692, 32691, 32688, 32687, 32690, 32689, 32696, + 32694, 41412, 41412, 41412, 41412, 41412, 41412, 8269, 8271, 8268, 8267, + 8264, 8263, 8266, 8265, 8272, 8270, 8260, 8261, 8256, 8257, 8258, 8259, + 8255, 8262, 10903, 10897, 10910, 10899, 10898, 10896, 10911, 10800, + 10798, 10803, 10904, 10954, 10809, 10921, 21748, 21750, 21747, 21746, + 21743, 21742, 21745, 21744, 21751, 21749, 21712, 21711, 21722, 21708, + 21716, 21715, 21729, 21709, 21718, 21706, 21710, 21714, 21713, 21724, + 21721, 21719, 21725, 21728, 21723, 21727, 21717, 21707, 21726, 21720, + 21730, 21704, 21731, 21705, 21740, 21737, 21739, 21738, 21741, 21736, + 21734, 21735, 21732, 21733, 32024, 32021, 32013, 32029, 32020, 32015, + 32026, 32018, 32017, 32019, 32023, 32011, 32028, 32027, 32025, 32031, + 32030, 32022, 32016, 32012, 32014, 32010, 32032, 32039, 32041, 32034, + 32037, 32040, 32038, 32036, 32035, 32007, 32006, 32009, 32008, 32042, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 32033, 19386, 19389, 19390, 19387, 19344, 19345, 19353, 19347, + 19349, 19352, 19346, 19342, 19348, 19350, 19343, 19309, 19311, 19312, + 19325, 19332, 19339, 19374, 19291, 19298, 19372, 19376, 19322, 19395, + 19378, 41412, 41412, 41412, 21067, 21063, 21066, 21065, 21037, 21005, + 21004, 21006, 21046, 21025, 21011, 21012, 21044, 21038, 21045, 21007, + 21008, 21009, 21021, 21022, 21010, 21019, 21020, 21035, 21014, 21036, + 21013, 21033, 21034, 21000, 21001, 21016, 21031, 21032, 21002, 21003, + 21015, 21023, 21024, 21017, 21018, 21041, 21043, 21026, 21027, 21040, + 21042, 21029, 21030, 21028, 21039, 21064, 21070, 21072, 21073, 21074, + 21068, 21069, 21071, 21076, 21075, 20996, 20997, 20998, 21061, 20999, + 21047, 21050, 21059, 21052, 21057, 21055, 21053, 21051, 21048, 21049, + 21054, 21062, 41412, 21060, 21083, 21085, 21082, 21081, 21078, 21077, + 21080, 21079, 21086, 21084, 41412, 41412, 41412, 41412, 21056, 21058, + 28787, 28785, 28780, 28777, 28783, 28873, 28851, 28803, 28815, 28811, + 28810, 28813, 28812, 28805, 28804, 28802, 28915, 28917, 28914, 28913, + 28910, 28909, 28912, 28911, 28918, 28916, 28814, 28807, 28806, 28809, + 28808, 41412, 6347, 6365, 6367, 6364, 6348, 6366, 6356, 6355, 6352, 6351, + 6327, 6328, 6350, 6349, 6354, 6353, 6329, 6331, 6330, 6358, 6357, 6344, + 6343, 6332, 6333, 6342, 6338, 6337, 6336, 6341, 6340, 6334, 6335, 6339, + 6363, 6361, 6360, 6362, 6345, 6346, 6359, 6372, 6375, 6376, 6381, 6379, + 6378, 6377, 6373, 6374, 6380, 6315, 6313, 6312, 6314, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 6321, 6320, 6317, 6309, + 6319, 6325, 6316, 6323, 6326, 6324, 6322, 6318, 6311, 6310, 41412, 41412, + 6388, 6390, 6387, 6386, 6383, 6382, 6385, 6384, 6391, 6389, 41412, 41412, + 6368, 6370, 6369, 6371, 28757, 28750, 28749, 28754, 28753, 28747, 28746, + 28745, 28743, 28742, 28744, 28748, 28759, 28752, 28751, 28756, 28850, + 28760, 28761, 28758, 28847, 28848, 28849, 28888, 28890, 28889, 28732, + 28884, 28875, 28876, 28796, 28797, 35479, 35455, 35478, 35454, 35477, + 35453, 35492, 35468, 35486, 35462, 35481, 35457, 35480, 35456, 35497, + 35473, 35487, 35463, 35490, 35466, 35485, 35461, 35484, 35460, 35488, + 35464, 35489, 35465, 35483, 35459, 35482, 35458, 35491, 35467, 35495, + 35471, 35499, 35475, 35496, 35472, 35494, 35470, 35498, 35474, 35493, + 35469, 35500, 35476, 35501, 35513, 35521, 35518, 35517, 35523, 35524, + 35502, 35522, 35519, 35520, 35512, 35516, 35515, 35514, 35511, 35508, + 35509, 35510, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 35504, 35505, 35507, 35506, + 35503, 27214, 27215, 27173, 27190, 27201, 27200, 27177, 27176, 27189, + 27195, 27196, 27228, 27230, 27220, 27222, 27221, 27168, 27165, 27167, + 27217, 27218, 27232, 27233, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 17347, 17345, 17344, 17343, 17342, 17346, + 41412, 41412, 17041, 17039, 17038, 17037, 17036, 17040, 41412, 41412, + 17056, 17054, 17053, 17052, 17051, 17055, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 16997, 17003, 17001, 16998, 17000, + 16999, 17002, 41412, 16982, 16988, 16986, 16983, 16985, 16984, 16987, + 41412, 23530, 23506, 23536, 23531, 23627, 23790, 23980, 23779, 23770, + 23773, 23801, 23815, 23641, 23534, 23535, 23885, 23735, 24029, 24028, + 24030, 24031, 23990, 23427, 23933, 23589, 23913, 23590, 23977, 23978, + 23533, 24100, 24066, 24109, 24052, 24105, 23553, 23554, 23555, 24141, + 24138, 24139, 24140, 24152, 27864, 28088, 28097, 28098, 28155, 23971, + 23738, 23886, 24111, 23734, 18558, 23596, 24061, 24034, 28152, 28001, + 28025, 41412, 41412, 41412, 41412, 6570, 6571, 6572, 6573, 6574, 6575, + 6533, 6569, 6534, 6535, 6536, 6537, 6538, 6498, 6499, 6500, 6501, 6502, + 6503, 6539, 6540, 6541, 6542, 6543, 6544, 6545, 6546, 6547, 6548, 6549, + 6504, 6497, 6505, 6506, 6507, 6508, 6509, 6510, 6551, 6552, 6553, 6554, + 6555, 6556, 6512, 6511, 6513, 6514, 6515, 6516, 6517, 6491, 6530, 6492, + 6531, 6493, 6532, 6494, 6495, 6496, 6490, 6518, 6519, 6520, 6521, 6522, + 6523, 6524, 6525, 6526, 6527, 6528, 6529, 6557, 6558, 6559, 6560, 6561, + 6562, 6563, 27182, 27194, 27204, 27206, 27191, 27185, 27172, 27197, + 27184, 27187, 27199, 27210, 27212, 27208, 27213, 27202, 27193, 27211, + 27179, 27180, 27209, 27171, 27181, 27175, 27178, 27174, 27170, 27183, + 27205, 27207, 27192, 27186, 27198, 27188, 27203, 27224, 27227, 27219, + 27226, 27225, 27229, 27223, 27231, 27169, 27216, 27166, 41412, 41412, + 27240, 27242, 27239, 27238, 27235, 27234, 27237, 27236, 27243, 27241, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 19583, 19581, 19614, 19615, 19616, 19592, 19593, 19625, + 19627, 19560, 19558, 19557, 19561, 19567, 19568, 19570, 19569, 19574, + 19571, 19572, 19576, 19544, 19542, 41412, 41412, 41412, 41412, 19440, + 19441, 19509, 19510, 19529, 19523, 19524, 19527, 19526, 19525, 19484, + 19469, 19508, 19475, 19479, 19478, 19492, 19491, 19412, 19437, 19430, + 19515, 19428, 19435, 19465, 19458, 19464, 19519, 19460, 19463, 19462, + 19503, 19496, 19513, 19514, 19502, 19500, 19499, 19504, 19506, 19451, + 19450, 19536, 19537, 19401, 19400, 19516, 19455, 19453, 41412, 41412, + 41412, 41412, 7687, 7688, 7689, 7690, 7691, 7692, 7693, 7694, 7695, 7696, + 7697, 7698, 7699, 7700, 7701, 7702, 7703, 7704, 7705, 7706, 7707, 7708, + 7709, 7710, 7711, 7712, 7713, 7714, 7715, 7716, 7717, 7718, 7719, 7720, + 7721, 7722, 7723, 7724, 7725, 7726, 7727, 7728, 7729, 7730, 7731, 7732, + 7733, 7734, 7735, 7736, 7737, 7738, 7739, 7740, 7741, 7742, 7743, 7744, + 7745, 7746, 7747, 7748, 7749, 7750, 7751, 7752, 7753, 7754, 7755, 7756, + 7757, 7758, 7759, 7760, 7761, 7762, 7763, 7764, 7765, 7766, 7767, 7768, + 7769, 7770, 7771, 7772, 7773, 7774, 7775, 7776, 7777, 7778, 7779, 7780, + 7781, 7782, 7783, 7784, 7785, 7786, 7787, 7788, 7789, 7790, 7791, 7792, + 7793, 7794, 7795, 7796, 7797, 7798, 7799, 7800, 7801, 7802, 7803, 7804, + 7805, 7806, 7807, 7808, 7809, 7810, 7811, 7812, 7813, 7814, 7815, 7816, + 7817, 7818, 7819, 7820, 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, + 7829, 7830, 7831, 7832, 7833, 7834, 7835, 7836, 7837, 7838, 7839, 7840, + 7841, 7842, 7843, 7844, 7845, 7846, 7847, 7848, 7849, 7850, 7851, 7852, + 7853, 7854, 7855, 7856, 7857, 7858, 7859, 7860, 7861, 7862, 7863, 7864, + 7865, 7866, 7867, 7868, 7869, 7870, 7871, 7872, 7873, 7874, 7875, 7876, + 7877, 7878, 7879, 7880, 7881, 7882, 7883, 7884, 7885, 7886, 7887, 7888, + 7889, 7890, 7891, 7892, 7893, 7894, 7895, 7896, 7897, 7898, 7899, 7900, + 7901, 7902, 7903, 7904, 7905, 7906, 7907, 7908, 7909, 7910, 7911, 7912, + 7913, 7914, 7915, 7916, 7917, 7918, 7919, 7920, 7921, 7922, 7923, 7924, + 7925, 7926, 7927, 7928, 7929, 7930, 7931, 7932, 7933, 7934, 7935, 7936, + 7937, 7938, 7939, 7940, 7941, 7942, 7485, 7486, 7487, 7488, 7489, 7490, 7491, 7492, 7493, 7494, 7495, 7496, 7497, 7498, 7499, 7500, 7501, 7502, 7503, 7504, 7505, 7506, 7507, 7508, 7509, 7510, 7511, 7512, 7513, 7514, 7515, 7516, 7517, 7518, 7519, 7520, 7521, 7522, 7523, 7524, 7525, 7526, @@ -15221,3308 +15366,3370 @@ static const unsigned short dawg_codepoint_to_pos_index2[] = { 7539, 7540, 7541, 7542, 7543, 7544, 7545, 7546, 7547, 7548, 7549, 7550, 7551, 7552, 7553, 7554, 7555, 7556, 7557, 7558, 7559, 7560, 7561, 7562, 7563, 7564, 7565, 7566, 7567, 7568, 7569, 7570, 7571, 7572, 7573, 7574, - 7575, 7576, 7577, 7578, 7579, 7580, 7581, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 24005, 24008, 24009, 24006, 24007, 24011, - 24012, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 2506, 2507, 2505, 2508, 2504, 40951, 40951, 40951, - 40951, 40951, 19891, 19918, 19896, 19827, 19883, 19886, 19888, 19887, - 19882, 19889, 19885, 19884, 19828, 19861, 19862, 19859, 19860, 19825, - 19826, 19824, 19832, 19874, 19872, 19849, 19881, 19854, 40951, 19866, - 19892, 19840, 19835, 19876, 40951, 19878, 40951, 19852, 19856, 40951, - 19842, 19838, 40951, 19870, 19847, 19864, 19858, 19868, 19880, 19831, - 19834, 19837, 19893, 1158, 1157, 1188, 1186, 1187, 1189, 1417, 1415, - 1416, 1418, 1183, 1181, 1182, 1184, 1547, 1545, 1546, 1548, 1527, 1525, - 1526, 1528, 1542, 1540, 1541, 1543, 1560, 1558, 1559, 1561, 1422, 1420, - 1421, 1423, 1224, 1222, 1223, 1225, 1395, 1393, 1394, 1396, 1504, 1502, - 1503, 1505, 1509, 1507, 1508, 1510, 1214, 1213, 1205, 1204, 1228, 1227, - 1217, 1216, 1324, 1323, 1452, 1451, 1347, 1345, 1346, 1348, 1258, 1256, - 1257, 1259, 1270, 1268, 1269, 1271, 1386, 1384, 1385, 1387, 1400, 1399, - 1456, 1454, 1455, 1457, 1300, 1299, 1295, 1293, 1294, 1296, 1304, 1302, - 1303, 1305, 1584, 1583, 1582, 1581, 2368, 2367, 2376, 2375, 2372, 2371, - 2370, 2369, 2380, 2379, 2366, 2374, 2373, 2382, 2378, 2377, 2381, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 1390, 1388, 1389, 1391, 1553, 1552, - 1605, 1604, 1602, 1601, 1551, 1563, 1562, 1351, 1350, 1354, 1353, 1618, - 1616, 1617, 1619, 1554, 1555, 2058, 2057, 2060, 2059, 2079, 2078, 2087, - 2086, 2077, 2076, 2083, 2082, 2063, 2061, 2062, 2112, 2110, 2111, 1238, - 1236, 1237, 1239, 2069, 2067, 2073, 2056, 2081, 1660, 1651, 1656, 1663, - 1658, 1669, 2021, 2009, 2016, 2029, 2032, 2037, 2039, 2044, 2041, 2050, - 1738, 1744, 1723, 1718, 1777, 1773, 1779, 1928, 1921, 1933, 1941, 1900, - 1904, 1682, 1674, 1678, 1684, 2002, 1997, 2114, 1631, 1627, 1712, 1708, - 1696, 1701, 1692, 1699, 1694, 1703, 1882, 1878, 1880, 1884, 1753, 1755, - 1763, 1761, 1758, 1769, 1751, 1772, 1805, 1797, 1809, 1815, 1789, 1818, - 1836, 1825, 1830, 1840, 1819, 1841, 1857, 1847, 1869, 1862, 1865, 1872, - 1733, 1729, 1730, 1731, 2097, 2090, 2099, 2105, 2054, 2109, 2038, 1895, - 1643, 1949, 1952, 1956, 1950, 1953, 1955, 2085, 2084, 2071, 2075, 2055, - 2080, 1667, 1666, 1661, 1665, 1657, 1668, 2035, 2034, 2027, 2033, 2031, - 2036, 2048, 2047, 2042, 2046, 2040, 2049, 1693, 1702, 1879, 1883, 1752, - 1756, 1767, 1750, 1771, 1813, 1788, 1817, 1820, 1838, 1870, 1867, 1860, - 1866, 1864, 1871, 1642, 2107, 2094, 2103, 2093, 2053, 2108, 2068, 2066, - 2070, 2072, 2064, 1659, 1650, 1655, 1662, 1652, 2020, 2008, 2015, 2028, - 2010, 2043, 1737, 1743, 1722, 1717, 1776, 1778, 1927, 1920, 1932, 1940, - 1899, 1907, 1903, 1681, 1673, 1677, 1683, 2001, 2113, 1630, 1626, 1711, - 1707, 1695, 1700, 1691, 1698, 1881, 1877, 1754, 1762, 1760, 1757, 1768, - 1804, 1796, 1808, 1814, 1798, 1835, 1824, 1829, 1839, 1856, 1846, 1868, - 1861, 1848, 1732, 1728, 1734, 2096, 2089, 2098, 2104, 2091, 2074, 2065, - 1664, 1653, 2030, 2011, 2045, 2051, 1942, 1924, 1979, 1959, 1759, 1770, - 1816, 1863, 1849, 2106, 2092, 1957, 1951, 1954, 2000, 2004, 1633, 1635, - 1710, 1714, 1944, 1948, 1981, 1989, 1720, 1725, 1746, 1748, 1775, 1781, - 1906, 1911, 1680, 1688, 1970, 1965, 1984, 1978, 1987, 1946, 1909, 1686, - 1999, 2003, 1632, 1634, 1709, 1713, 1943, 1947, 1980, 1988, 1719, 1724, - 1745, 1747, 1774, 1780, 1905, 1910, 1679, 1687, 1968, 1963, 1982, 1976, - 1986, 1945, 1908, 1685, 1969, 1964, 1983, 1977, 1923, 1958, 1996, 1929, - 1922, 1934, 1971, 1966, 1985, 1998, 2115, 1644, 1645, 30650, 30651, 1892, - 1887, 1891, 1888, 1889, 1890, 1918, 1637, 1639, 1638, 1636, 1886, 1917, - 1640, 1991, 1893, 2018, 2007, 2006, 2005, 2013, 2023, 2025, 2024, 1740, - 1739, 1716, 1715, 1919, 1926, 1925, 1936, 1935, 1937, 1939, 1938, 1897, - 1896, 1902, 1961, 1960, 1967, 1973, 1972, 1975, 1974, 1671, 1676, 1675, - 1993, 1992, 1994, 1995, 1629, 1624, 1623, 1622, 1704, 1706, 1705, 1690, - 1689, 1875, 1873, 1794, 1795, 1792, 1799, 1800, 1807, 1806, 1811, 1810, - 1821, 1822, 1823, 1833, 1831, 1826, 1827, 40951, 40951, 1832, 1726, 1727, - 1844, 1843, 1854, 1853, 1852, 1859, 1858, 2101, 2100, 1654, 2019, 2017, - 2014, 2012, 2026, 2022, 1742, 1735, 1741, 1930, 1898, 1962, 1672, 1803, - 1812, 2088, 2095, 2102, 1837, 1876, 1845, 1874, 1793, 1625, 1766, 1850, - 1828, 1801, 1765, 1802, 1851, 1736, 1721, 1834, 1697, 1649, 1764, 1628, - 1901, 1931, 1855, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 1913, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 1914, 1885, 1646, 1647, 1842, 1912, 1894, 1641, 2052, 1915, - 1916, 1749, 32167, 1670, 1990, 1648, 38261, 38372, 38440, 38451, 38462, - 38473, 38484, 38495, 38506, 38262, 38273, 38284, 38295, 38306, 38317, - 38328, 31622, 31627, 31628, 31621, 31652, 31623, 31654, 31630, 31644, - 31626, 40951, 40951, 40951, 40951, 40951, 40951, 8360, 8362, 8192, 8193, - 8371, 8373, 8086, 8361, 8363, 8429, 8430, 8372, 8374, 8087, 8140, 8141, - 31653, 31624, 31625, 31639, 31651, 31636, 31648, 31634, 31646, 31638, - 31650, 31631, 31640, 31632, 31641, 31635, 31647, 31633, 31645, 31629, - 31642, 32661, 39263, 31637, 31649, 10541, 6129, 39141, 11256, 10540, - 6128, 39139, 33800, 33809, 33844, 40951, 33834, 33801, 33845, 33807, - 33808, 33811, 33815, 33810, 33814, 33812, 33816, 33843, 33791, 33793, - 33842, 33840, 33813, 33839, 33806, 40951, 33833, 33802, 33841, 33799, - 40951, 40951, 40951, 40951, 1092, 2387, 1074, 2389, 1104, 40951, 1089, - 1090, 1070, 1071, 1101, 1102, 2294, 2295, 2362, 2363, 1289, 1153, 1152, - 1143, 1142, 1571, 1570, 1146, 1145, 1588, 1586, 1587, 1589, 1163, 1162, - 1178, 1176, 1177, 1179, 1514, 1513, 1523, 1521, 1522, 1516, 1535, 1533, - 1534, 1536, 1320, 1318, 1319, 1321, 1286, 1284, 1285, 1287, 1358, 1356, - 1357, 1359, 1202, 1201, 1531, 1530, 1449, 1448, 1613, 1612, 1478, 1476, - 1477, 1479, 1484, 1482, 1483, 1485, 1465, 1463, 1464, 1466, 1210, 1208, - 1209, 1211, 1497, 1495, 1496, 1498, 1609, 1607, 1608, 1610, 1121, 1119, - 1120, 1122, 1265, 1263, 1264, 1266, 1249, 1247, 1248, 1250, 1431, 1429, - 1430, 1432, 1334, 1332, 1333, 1335, 1370, 1368, 1369, 1371, 1379, 1377, - 1378, 1380, 1410, 1408, 1409, 1411, 1308, 1306, 1307, 1309, 1575, 1574, - 1161, 1160, 1598, 1596, 1597, 1599, 1787, 1786, 1783, 1782, 1785, 1784, - 1791, 1790, 40951, 40951, 40755, 40951, 17625, 17629, 17597, 17613, - 17599, 17611, 17610, 17540, 17603, 17612, 17600, 17535, 17628, 17633, - 17607, 17620, 17622, 17619, 17618, 17615, 17614, 17617, 17616, 17623, - 17621, 17536, 17606, 17542, 17624, 17627, 17630, 17534, 17543, 17544, - 17545, 17546, 17547, 17548, 17549, 17550, 17551, 17552, 17553, 17554, - 17555, 17556, 17557, 17558, 17559, 17560, 17561, 17562, 17563, 17564, - 17565, 17566, 17567, 17568, 17541, 17605, 17604, 17533, 17595, 17626, - 17569, 17570, 17571, 17572, 17573, 17574, 17575, 17576, 17577, 17578, - 17579, 17580, 17581, 17582, 17583, 17584, 17585, 17586, 17587, 17588, - 17589, 17590, 17591, 17592, 17593, 17594, 17539, 17635, 17602, 17632, - 17538, 17601, 19098, 19091, 19093, 19097, 19089, 19081, 19036, 19038, - 19040, 19037, 19039, 19032, 19034, 19033, 19035, 19090, 19082, 19084, - 19086, 19083, 19085, 19057, 19059, 19061, 19058, 19060, 19041, 19043, - 19045, 19042, 19044, 19072, 19074, 19076, 19073, 19075, 19047, 19049, - 19051, 19048, 19050, 19052, 19054, 19056, 19053, 19055, 19062, 19064, - 19066, 19063, 19065, 19077, 19079, 19078, 19067, 19069, 19071, 19068, - 19070, 19080, 19046, 19088, 19087, 19031, 18981, 18998, 18982, 18983, - 18984, 18985, 19019, 19000, 18989, 18994, 18993, 18992, 18996, 18990, - 18991, 18995, 19017, 18987, 18999, 18988, 19002, 19001, 19016, 19011, - 18997, 19010, 18980, 19018, 18986, 19025, 40951, 40951, 40951, 19026, - 19027, 19005, 19006, 19013, 19012, 40951, 40951, 19004, 19003, 19028, - 19022, 19023, 19029, 40951, 40951, 19008, 19030, 19021, 19020, 19024, - 19009, 40951, 40951, 19014, 19007, 19015, 40951, 40951, 40951, 17537, - 17598, 17596, 17609, 17634, 17608, 17631, 40951, 19095, 19092, 19096, - 19094, 19101, 19099, 19100, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 20788, 20790, 20789, 29933, 31859, 40951, - 40951, 24968, 24994, 24987, 25020, 24978, 24969, 24998, 24964, 24976, - 25006, 25009, 25003, 40951, 24992, 25019, 25023, 25002, 25017, 25024, - 25031, 25034, 24979, 25030, 24977, 24980, 24963, 24986, 24989, 25012, - 25013, 24971, 25028, 24993, 24974, 25010, 24972, 25029, 24988, 24996, - 40951, 25021, 24985, 25005, 24970, 24981, 24995, 24966, 25000, 24975, - 25007, 25008, 24965, 24991, 24967, 25018, 25011, 25025, 24999, 25001, - 40951, 24973, 25027, 40951, 24984, 24983, 24997, 25033, 25026, 25036, - 25004, 24982, 25014, 25022, 24990, 25015, 25016, 25032, 25035, 40951, - 40951, 25046, 25047, 25049, 25048, 25037, 25038, 25045, 25039, 25040, - 25050, 25041, 25042, 25043, 25044, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24852, - 24851, 24853, 24840, 24841, 24842, 24843, 24844, 24845, 24846, 24848, - 24847, 24850, 24849, 24858, 24855, 24856, 24854, 24857, 24957, 24958, - 24860, 24859, 24861, 24960, 24959, 24863, 24864, 24865, 24862, 24866, - 24869, 24868, 24870, 24871, 24872, 24961, 24873, 24874, 24867, 24877, - 24878, 24876, 24875, 24879, 24880, 24881, 24882, 24883, 24884, 24887, - 24888, 24889, 24886, 24890, 24885, 24891, 24892, 24893, 24894, 24895, - 24896, 24897, 24898, 24899, 24900, 24902, 24901, 24903, 24904, 24906, - 24907, 24908, 24905, 24909, 24910, 24911, 24912, 24914, 24913, 24915, - 24916, 24962, 24917, 24918, 24920, 24921, 24922, 24919, 24923, 24924, - 24925, 24926, 24927, 24955, 24935, 24936, 24937, 24938, 24939, 24940, - 24941, 24942, 24943, 24944, 24945, 24946, 24947, 24948, 24949, 24950, - 24951, 24952, 24953, 24954, 24928, 24929, 24930, 24931, 24932, 24933, - 24934, 24956, 40951, 40951, 40951, 40951, 40951, 112, 113, 159, 40951, - 40951, 40951, 40951, 156, 151, 146, 126, 121, 139, 134, 114, 129, 154, - 149, 144, 124, 119, 140, 135, 115, 130, 157, 152, 147, 127, 122, 142, - 137, 117, 132, 158, 153, 148, 128, 123, 143, 138, 118, 133, 155, 150, - 145, 125, 120, 141, 136, 116, 131, 40951, 40951, 40951, 111, 107, 109, - 110, 108, 103, 104, 105, 106, 18166, 18163, 18167, 18153, 18148, 18154, - 18157, 18149, 18159, 18168, 18151, 18161, 18155, 18164, 18158, 18160, - 18170, 18152, 18162, 18156, 18165, 18169, 18150, 18171, 18183, 18192, - 18182, 18178, 18191, 18173, 18179, 18195, 18199, 18200, 18181, 18184, - 18190, 18189, 18197, 18198, 18180, 18187, 18193, 18188, 18177, 18196, - 18185, 18172, 18174, 18194, 18186, 18175, 18176, 18418, 18419, 18617, - 18613, 18654, 18619, 18349, 18423, 18616, 18612, 18355, 18354, 18415, - 18397, 18413, 18422, 18417, 18203, 18202, 18656, 18615, 18657, 18420, - 18611, 18393, 29360, 40951, 32211, 32214, 32209, 32212, 32183, 32213, - 32181, 32184, 32210, 32182, 32215, 32180, 2526, 40951, 40951, 40951, - 18610, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31405, 31406, - 31417, 31386, 31389, 31420, 31398, 31397, 31418, 31425, 31384, 31411, - 31390, 31403, 31404, 31413, 31402, 31383, 31387, 31394, 31392, 31414, - 31391, 31382, 31412, 31399, 31400, 31385, 31388, 31410, 31424, 31395, - 31419, 31381, 31407, 31426, 31408, 31409, 31401, 31423, 31422, 31396, - 31415, 31416, 31421, 31393, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 25283, 25285, 25278, 25279, 25290, 25289, - 25292, 25300, 25302, 25281, 25293, 25276, 25296, 25294, 25274, 25287, - 25275, 25288, 25299, 25295, 25277, 25297, 25298, 25280, 25282, 25284, - 25286, 25291, 25301, 40951, 40951, 40951, 6039, 6050, 6041, 6012, 6035, - 6051, 6013, 6040, 6057, 6055, 6015, 6056, 6042, 6030, 6025, 6026, 6024, - 6010, 6033, 6023, 6058, 6020, 6032, 6049, 6029, 6053, 6043, 6038, 6047, - 6048, 6021, 6034, 6045, 6046, 6027, 6028, 6022, 6054, 6011, 6031, 6036, - 6052, 6016, 6017, 6018, 6019, 6014, 6044, 6037, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 8571, 8569, 8567, 8566, 8563, 8562, 8565, 8564, 8570, 8568, - 8561, 8560, 8558, 8549, 8547, 8556, 8554, 8545, 8551, 8552, 8559, 8557, - 8548, 8546, 8555, 8553, 8544, 8550, 40951, 40951, 40951, 40951, 30215, - 30209, 30195, 30210, 30182, 30212, 30214, 30211, 30206, 30202, 30194, - 30190, 30191, 30192, 30184, 30216, 30205, 30198, 30196, 30186, 30183, - 30207, 30200, 30188, 30204, 30193, 30189, 30187, 30208, 30203, 30201, - 30185, 30220, 30218, 30219, 30217, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 30213, 30199, 30197, 18013, 18029, 18037, - 18031, 18012, 18022, 18016, 18015, 18024, 18034, 18038, 18035, 18032, - 18020, 18036, 18027, 18021, 18019, 18023, 18033, 18025, 18026, 18028, - 18017, 18014, 18030, 18018, 40951, 40951, 40951, 40951, 40951, 30287, - 30286, 30283, 30256, 30257, 30279, 30254, 30280, 30255, 30259, 30288, - 30281, 30262, 30263, 30270, 30264, 30282, 30267, 30269, 30290, 30253, - 30266, 30265, 30277, 30274, 30284, 30285, 30258, 30289, 30268, 30271, - 30272, 30273, 30276, 30261, 30278, 30275, 30260, 8386, 8384, 8382, 8383, - 8385, 40951, 40951, 40951, 40951, 40951, 37709, 37712, 37716, 37720, - 37713, 37717, 37735, 37731, 37718, 37728, 37730, 37719, 37725, 37721, - 37736, 37714, 37733, 37732, 37724, 37710, 37734, 37723, 37711, 37722, - 37727, 37715, 37729, 37737, 37738, 37726, 40951, 37739, 30296, 30338, - 30339, 30319, 30320, 30317, 30318, 30316, 30331, 30308, 30309, 30313, - 30314, 30303, 30306, 30307, 30312, 30335, 30300, 30332, 30321, 30322, - 30325, 30326, 30327, 30336, 30310, 30311, 30323, 30324, 30334, 30330, - 30337, 30328, 30329, 30333, 40951, 40951, 40951, 40951, 30297, 30298, - 30299, 30315, 30304, 30305, 30301, 30302, 30340, 30295, 30292, 30293, - 30291, 30294, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 10594, 10593, 10589, 10590, 10591, 10592, - 10600, 10599, 10595, 10596, 10597, 10598, 10617, 10602, 10616, 10613, - 10618, 10611, 10608, 10604, 10609, 10607, 10610, 10615, 10614, 10584, - 10612, 10583, 10603, 10579, 10606, 10580, 10605, 10587, 10585, 10586, - 10581, 10582, 10601, 10588, 10634, 10633, 10629, 10630, 10631, 10632, - 10640, 10639, 10635, 10636, 10637, 10638, 10657, 10642, 10656, 10653, - 10658, 10651, 10648, 10644, 10649, 10647, 10650, 10655, 10654, 10624, - 10652, 10623, 10643, 10619, 10646, 10620, 10645, 10627, 10625, 10626, - 10621, 10622, 10641, 10628, 32797, 32803, 32814, 32811, 32801, 32800, - 32799, 32778, 32806, 32784, 32813, 32809, 32812, 32815, 32802, 32810, - 32789, 32808, 32805, 32783, 32788, 32790, 32787, 32782, 32776, 32773, - 32795, 32804, 32793, 32777, 32798, 32816, 32780, 32774, 32786, 32817, - 32794, 32791, 32792, 32775, 32771, 32796, 32772, 32781, 32770, 32779, - 32785, 32807, 30729, 30747, 30753, 30751, 30754, 30735, 30732, 30752, - 30737, 30736, 30733, 30731, 30749, 30748, 30739, 30734, 30742, 30738, - 30743, 30744, 30750, 30755, 30728, 30745, 30756, 30740, 30757, 30730, - 30746, 30741, 40951, 40951, 30764, 30766, 30763, 30762, 30759, 30758, - 30761, 30760, 30767, 30765, 40951, 40951, 40951, 40951, 40951, 40951, - 30656, 30657, 30658, 30659, 30684, 30677, 30663, 30660, 30666, 30676, - 30675, 30690, 30669, 30664, 30668, 30685, 30686, 30687, 30670, 30671, - 30688, 30665, 30681, 30680, 30674, 30662, 30673, 30661, 30672, 30678, - 30691, 30689, 30667, 30679, 30683, 30682, 40951, 40951, 40951, 40951, - 30692, 30693, 30694, 30695, 30720, 30713, 30699, 30696, 30702, 30712, - 30711, 30726, 30705, 30700, 30704, 30721, 30722, 30723, 30706, 30707, - 30724, 30701, 30717, 30716, 30710, 30698, 30709, 30697, 30708, 30714, - 30727, 30725, 30703, 30715, 30719, 30718, 40951, 40951, 40951, 40951, - 16693, 16684, 16673, 16672, 16675, 16664, 16674, 16671, 16670, 16685, - 16661, 16660, 16686, 16694, 16687, 16677, 16663, 16662, 16688, 16667, - 16666, 16665, 16695, 16689, 16690, 16669, 16668, 16679, 16678, 16681, - 16680, 16696, 16691, 16692, 16697, 16683, 16682, 16659, 16658, 16676, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6071, 6120, 6087, - 6083, 6085, 6110, 6084, 6108, 6105, 6074, 6107, 6109, 6088, 6099, 6095, - 6090, 6118, 6082, 6073, 6091, 6094, 6096, 6121, 6112, 6070, 6076, 6077, - 6079, 6113, 6111, 6116, 6080, 6100, 6092, 6119, 6104, 6115, 6081, 6075, - 6098, 6086, 6117, 6102, 6114, 6103, 6101, 6089, 6078, 6072, 6106, 6097, - 6093, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 6122, 38912, 38881, 38882, 38892, 38891, 38894, 38893, - 38884, 38883, 38901, 38909, 40951, 38900, 38899, 38885, 38886, 38902, - 38910, 38888, 38887, 38903, 38890, 38889, 38913, 38904, 38911, 38905, - 40951, 38896, 38895, 38898, 38897, 38914, 38906, 38907, 40951, 38915, - 38908, 40951, 38947, 38916, 38917, 38927, 38926, 38929, 38928, 38919, - 38918, 38936, 38944, 40951, 38935, 38934, 38920, 38921, 38937, 38945, - 38923, 38922, 38938, 38925, 38924, 38948, 38939, 38946, 38940, 40951, - 38931, 38930, 38933, 38932, 38949, 38941, 38942, 40951, 38950, 38943, - 40951, 40951, 40951, 37427, 37428, 37441, 37401, 37430, 37429, 37432, - 37408, 37431, 37424, 37423, 37442, 37398, 37404, 37397, 37403, 37411, - 37410, 37445, 37399, 37434, 37426, 37425, 37402, 37409, 37405, 37421, - 37413, 37443, 37420, 37419, 37418, 37415, 37414, 37436, 37435, 37446, - 37444, 37438, 37407, 37437, 37406, 37447, 37400, 37440, 37439, 37396, - 37417, 37416, 37433, 37412, 37422, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24819, 24820, 24821, - 24822, 24823, 24824, 24825, 24826, 24827, 24758, 24759, 24760, 24761, - 24762, 24771, 24763, 24764, 24765, 24766, 24767, 24768, 24769, 24770, - 24772, 24773, 24774, 24775, 24839, 24776, 24777, 24778, 24779, 24780, - 24781, 24782, 24783, 24784, 24785, 24786, 24787, 24788, 24789, 24790, - 24791, 24792, 24793, 24794, 24795, 24796, 24797, 24798, 24799, 24800, - 24801, 24802, 24803, 24804, 24805, 24806, 24807, 24808, 24809, 24810, - 24811, 24812, 24813, 24814, 24815, 24816, 24817, 24818, 24499, 24835, - 24828, 24500, 24829, 24830, 24831, 24832, 24501, 24836, 24837, 24833, - 24834, 24838, 24505, 24506, 24507, 24508, 24509, 24510, 24511, 24512, - 24502, 24503, 24504, 24516, 24517, 24518, 24513, 24514, 24515, 24519, - 24520, 24521, 24522, 24523, 24524, 24527, 24528, 24529, 24530, 24531, - 24532, 24533, 24534, 24535, 24536, 24537, 24538, 24539, 24540, 24541, - 24542, 24543, 24544, 24545, 24546, 24547, 24548, 24549, 24550, 24551, - 24552, 24553, 24554, 24555, 24556, 24557, 24558, 24559, 24560, 24561, - 24562, 24563, 24564, 24565, 24566, 24567, 24568, 24569, 24570, 24571, - 24572, 24573, 24574, 24575, 24576, 24525, 24526, 24577, 24578, 24579, - 24580, 24581, 24582, 24583, 24584, 24585, 24586, 24587, 24588, 24589, - 24590, 24591, 24592, 24593, 24594, 24595, 24596, 24597, 24598, 24599, - 24600, 24601, 24602, 24603, 24604, 24605, 24606, 24607, 24608, 24609, - 24641, 24642, 24643, 24644, 24645, 24646, 24647, 24648, 24649, 24610, - 24611, 24612, 24613, 24614, 24615, 24616, 24617, 24618, 24619, 24620, - 24621, 24622, 24623, 24624, 24625, 24626, 24627, 24628, 24629, 24630, - 24631, 24632, 24633, 24634, 24635, 24636, 24637, 24638, 24639, 24640, - 24656, 24657, 24658, 24659, 24660, 24661, 24662, 24663, 24664, 24665, - 24666, 24667, 24668, 24669, 24670, 24671, 24672, 24673, 24674, 24675, - 24650, 24651, 24652, 24653, 24654, 24655, 24676, 24677, 24678, 24679, - 24680, 24681, 24682, 24683, 24718, 24719, 24720, 24721, 24722, 24723, - 24724, 24725, 24726, 24727, 24684, 24685, 24686, 24687, 24688, 24689, - 24690, 24691, 24692, 24693, 24694, 24695, 24696, 24697, 24698, 24699, - 24700, 24701, 24702, 24703, 24709, 24710, 24711, 24712, 24713, 24714, - 24715, 24716, 24717, 24704, 24705, 24706, 24707, 24708, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24739, 24734, 24737, - 24736, 24740, 24735, 24733, 24738, 24732, 24728, 24731, 24730, 24729, - 24746, 24741, 24745, 24742, 24743, 24744, 24747, 24749, 24748, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 24750, - 24751, 24752, 24753, 24754, 24755, 24756, 24757, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 27868, 27989, 27988, 27853, 27869, 27857, 40951, 27887, 27889, - 27888, 27883, 27882, 27880, 27881, 27942, 27875, 27899, 27939, 27863, - 27903, 27864, 27907, 27870, 27909, 27886, 27923, 27924, 27922, 27866, - 27920, 27925, 27926, 27967, 27968, 27931, 27867, 27876, 27982, 27964, - 27965, 27938, 27937, 27872, 27952, 27955, 27956, 27953, 27951, 27979, - 40951, 27874, 27794, 27841, 27689, 27772, 27801, 27686, 27843, 27944, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 10008, - 10009, 10010, 10011, 10012, 10002, 40951, 40951, 10003, 40951, 9958, - 9959, 9960, 9961, 9962, 9963, 9964, 9965, 9966, 9967, 9968, 9969, 9970, - 9971, 9972, 9973, 9974, 9975, 9976, 9977, 9978, 9979, 9980, 9981, 9982, - 9983, 9984, 9985, 9986, 9987, 9988, 9989, 9990, 9991, 9992, 9993, 9994, - 9995, 9996, 9997, 9998, 9999, 10000, 10001, 40951, 10006, 10007, 40951, - 40951, 40951, 10004, 40951, 40951, 10005, 20610, 20621, 20612, 20606, - 20618, 20623, 20613, 20619, 20604, 20617, 20620, 20607, 20625, 20622, - 20614, 20611, 20624, 20615, 20608, 20609, 20616, 20605, 40951, 20626, - 20601, 20597, 20600, 20598, 20596, 20602, 20603, 20599, 31035, 31046, - 31037, 31031, 31043, 31048, 31038, 31044, 31029, 31042, 31045, 31032, - 31050, 31028, 31047, 31039, 31036, 31049, 31040, 31033, 31034, 31041, - 31030, 31027, 31051, 31058, 31053, 31054, 31057, 31056, 31055, 31052, - 28806, 28821, 28811, 28832, 28823, 28817, 28813, 28829, 28834, 28824, - 28830, 28815, 28809, 28828, 28810, 28831, 28807, 28818, 28814, 28836, - 28812, 28833, 28825, 28822, 28835, 28826, 28819, 28820, 28808, 28827, - 28816, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 28841, - 28838, 28839, 28844, 28845, 28843, 28840, 28837, 28842, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 19658, 19674, 19666, 19665, - 19671, 19676, 19660, 19672, 19661, 19670, 19673, 19663, 19678, 19675, - 19667, 19659, 19677, 19668, 19664, 40951, 19669, 19662, 40951, 40951, - 40951, 40951, 40951, 19679, 19683, 19682, 19681, 19680, 31429, 31448, - 31444, 31431, 31432, 31440, 31441, 31433, 31439, 31442, 31445, 31447, - 31450, 31446, 31437, 31430, 31449, 31435, 31434, 31443, 31436, 31438, - 31455, 31454, 31451, 31456, 31452, 31453, 40951, 40951, 40951, 31457, - 25309, 25315, 25319, 25317, 25311, 25327, 25320, 25328, 25321, 25305, - 25322, 25313, 25323, 25325, 25308, 25303, 25326, 25318, 25324, 25307, - 25304, 25310, 25312, 25306, 25314, 25316, 40951, 40951, 40951, 40951, - 40951, 25329, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 27402, 27403, 27404, 27405, - 27401, 27400, 27376, 27377, 27398, 27397, 27380, 27381, 27382, 27383, - 27378, 27379, 27396, 27393, 27392, 27384, 27385, 27386, 27394, 27399, - 27387, 27388, 27389, 27390, 27391, 27395, 27406, 27407, 27298, 27319, - 27320, 27321, 27318, 27317, 27310, 27314, 27313, 27303, 27304, 27316, - 27312, 27308, 27307, 27305, 27299, 27306, 27309, 27315, 27300, 27301, - 27302, 27311, 40951, 40951, 40951, 40951, 27286, 27291, 27323, 27322, - 27372, 27364, 27358, 27335, 27329, 27352, 27346, 27324, 27341, 27370, - 27368, 27362, 27339, 27333, 27356, 27350, 40951, 40951, 27373, 27365, - 27359, 27336, 27330, 27353, 27347, 27326, 27343, 27375, 27367, 27361, - 27338, 27332, 27355, 27349, 27328, 27345, 27371, 27369, 27363, 27340, - 27334, 27357, 27351, 27325, 27342, 27374, 27366, 27360, 27337, 27331, - 27354, 27348, 27327, 27344, 27290, 27296, 27295, 27289, 27288, 27293, - 27292, 27287, 27297, 27294, 21668, 21690, 21692, 21688, 40951, 21689, - 21691, 40951, 40951, 40951, 40951, 40951, 21693, 21684, 21687, 21686, - 21634, 21632, 21656, 21655, 40951, 21654, 21653, 21662, 40951, 21642, - 21638, 21637, 21645, 21644, 21641, 21640, 21639, 21647, 21646, 21643, - 21658, 21657, 21652, 21651, 21664, 21666, 21665, 21663, 21660, 21648, - 21649, 21650, 21667, 21661, 21633, 21635, 21636, 21659, 40951, 40951, - 21682, 21683, 21685, 40951, 40951, 40951, 40951, 21694, 21631, 21630, - 21629, 21628, 21670, 21669, 21671, 21672, 21695, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 21676, 21681, 21674, 21673, 21680, 21678, - 21677, 21675, 21679, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 30405, 30401, 30406, 30411, 30402, 30409, 30395, 30403, 30407, 30399, - 30394, 30390, 30408, 30391, 30393, 30392, 30410, 30383, 30384, 30386, - 30389, 30387, 30388, 30398, 30400, 30385, 30404, 30397, 30396, 30413, - 30412, 30414, 30226, 30244, 30225, 30235, 30247, 30248, 30237, 30241, - 30239, 30232, 30236, 30228, 30243, 30227, 30249, 30238, 30240, 30221, - 30222, 30245, 30224, 30246, 30223, 30231, 30233, 30229, 30242, 30230, - 30234, 30252, 30251, 30250, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 25610, 25614, 25613, 25618, - 25617, 25615, 25639, 25642, 25658, 25622, 25621, 25620, 25619, 25640, - 25632, 25638, 25624, 25634, 25623, 25636, 25616, 25631, 25645, 25641, - 25628, 25612, 25611, 25644, 25643, 25629, 25626, 25635, 25625, 25637, - 25630, 25627, 25633, 25660, 25659, 40951, 40951, 40951, 40951, 25646, - 25650, 25649, 25648, 25647, 25657, 25655, 25653, 25652, 25651, 25656, - 25654, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 2542, 2543, 2549, 2545, 2548, 2544, 2546, 2547, 2585, 2586, 2575, 2576, - 2577, 2578, 2573, 2574, 2590, 2563, 2562, 2561, 2552, 2550, 2551, 2587, - 2589, 2572, 2570, 2582, 2581, 2571, 2593, 2588, 2580, 2579, 2557, 2556, - 2555, 2560, 2559, 2558, 2592, 2553, 2568, 2569, 2595, 2594, 2591, 2567, - 2584, 2565, 2583, 2564, 2566, 2554, 40951, 40951, 40951, 2596, 37309, - 33835, 22667, 22662, 22668, 22663, 20752, 20763, 20754, 20748, 20760, - 20765, 20755, 20761, 20746, 20759, 20762, 20749, 20767, 20764, 20756, - 20753, 20766, 20757, 20750, 20751, 20758, 20747, 40951, 40951, 20772, - 20769, 20770, 20775, 20771, 20768, 20773, 20774, 20721, 20735, 20726, - 20722, 20732, 20725, 20727, 20733, 20719, 20731, 20734, 20723, 20724, - 20736, 20728, 20737, 20729, 20730, 20720, 40951, 40951, 40951, 40951, - 40951, 20742, 20739, 20740, 20745, 20741, 20738, 20743, 20744, 31690, - 31704, 31695, 31691, 31701, 31694, 31696, 31702, 31700, 31703, 31692, - 31693, 31705, 31697, 31707, 31698, 31699, 31706, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 31715, 31716, 31688, 31689, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 31712, 31709, 31710, 31714, 31711, 31708, 31713, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30415, 30457, 30458, - 30447, 30483, 30476, 30450, 30451, 30485, 30428, 30468, 30416, 30461, - 30430, 30470, 30418, 30462, 30429, 30469, 30417, 30446, 30482, 30436, - 30475, 30425, 30465, 30419, 30463, 30452, 30486, 30431, 30471, 30420, - 30441, 30444, 30432, 30421, 30459, 30439, 30478, 30437, 30477, 30440, - 30479, 30467, 30438, 30460, 30445, 30453, 30448, 30443, 30481, 30433, - 30472, 30449, 30484, 30454, 30487, 30434, 30473, 30422, 30426, 30423, - 30427, 30466, 30442, 30480, 30435, 30474, 30424, 30464, 30455, 30456, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 30074, 30077, 30100, 30075, 30089, - 30085, 30091, 30101, 30076, 30079, 30121, 30102, 30103, 30092, 30093, - 30104, 30122, 30123, 30105, 30106, 30078, 30118, 30094, 30095, 30080, - 30082, 30086, 30114, 30116, 30110, 30112, 30115, 30107, 30081, 30108, - 30124, 30087, 30088, 30096, 30083, 30097, 30090, 30117, 30120, 30111, - 30113, 30109, 30098, 30099, 30084, 30119, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30125, - 30128, 30151, 30126, 30140, 30136, 30142, 30152, 30127, 30130, 30172, - 30153, 30154, 30143, 30144, 30155, 30173, 30174, 30156, 30157, 30129, - 30169, 30145, 30146, 30131, 30133, 30137, 30165, 30167, 30161, 30163, - 30166, 30158, 30132, 30159, 30175, 30138, 30139, 30147, 30134, 30148, - 30141, 30168, 30171, 30162, 30164, 30160, 30149, 30150, 30135, 30170, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30178, 30177, 30181, - 30176, 30179, 30180, 19611, 19598, 19606, 19590, 19589, 19603, 19599, - 19602, 19587, 19600, 19584, 19583, 19592, 19591, 19610, 19597, 19596, - 19588, 19601, 19604, 19605, 19595, 19608, 19585, 19609, 19586, 19593, - 19594, 19607, 19618, 19620, 19622, 19619, 19621, 19613, 19612, 19617, - 19614, 19616, 19615, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 19629, 19631, 19628, 19627, 19624, 19623, 19626, 19625, 19632, - 19630, 40951, 40951, 40951, 40951, 40951, 40951, 17713, 17715, 17712, - 17711, 17708, 17707, 17710, 17709, 17716, 17714, 17699, 17700, 17701, - 17698, 17702, 17696, 17673, 17657, 17665, 17663, 17656, 17662, 17668, - 17670, 17664, 17660, 17658, 17671, 17672, 17669, 17667, 17654, 17659, - 17655, 17666, 17661, 17652, 17653, 40951, 40951, 40951, 17697, 17651, - 17649, 17648, 17650, 17704, 17703, 17695, 17679, 17687, 17685, 17678, - 17684, 17690, 17692, 17686, 17682, 17680, 17693, 17694, 17691, 17689, - 17676, 17681, 17677, 17688, 17683, 17674, 17675, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 17705, 17706, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 32268, 32266, 32265, 32262, 32261, - 32264, 32263, 32269, 32267, 32260, 32259, 32257, 32248, 32246, 32255, - 32253, 32244, 32250, 32251, 32258, 32256, 32247, 32245, 32254, 32252, - 32243, 32249, 32240, 32241, 32239, 32242, 40951, 39408, 39442, 39422, - 39421, 39427, 39426, 39404, 39403, 39402, 39413, 39434, 39407, 39438, - 39441, 39440, 39437, 39445, 39424, 39423, 39425, 39406, 39428, 39439, - 39409, 39433, 39444, 39429, 39430, 39417, 39415, 39414, 39416, 39418, - 39405, 39420, 39443, 39431, 39432, 39411, 39412, 39435, 39410, 40951, - 39400, 39399, 39401, 40951, 40951, 39419, 39436, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 1195, 1490, 1326, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 1061, 2351, 2352, 2349, 30345, 30353, 30367, 30354, 30358, 30364, 30355, - 30370, 30359, 30365, 30363, 30366, 30357, 30372, 30368, 30347, 30348, - 30360, 30346, 30344, 30371, 30361, 30349, 30350, 30356, 30362, 30369, - 30351, 30352, 30379, 30377, 30374, 30382, 30381, 30378, 30376, 30375, - 30380, 30343, 30373, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 33894, 33908, 33896, 33905, 33912, 33900, 33906, 33904, 33907, - 33897, 33914, 33910, 33901, 33895, 33913, 33902, 33899, 33903, 33911, - 33909, 33898, 33893, 33888, 33886, 33889, 33887, 33892, 33891, 33883, - 33882, 33884, 33890, 33885, 33915, 33918, 33917, 33916, 33921, 33922, - 33919, 33923, 33920, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 30492, 30504, 30502, 30507, 30496, - 30501, 30500, 30503, 30494, 30509, 30505, 30497, 30508, 30498, 30493, - 30499, 30506, 30495, 30491, 30490, 30489, 30488, 30513, 30510, 30511, - 30512, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6491, - 6485, 6499, 6493, 6488, 6496, 6502, 6484, 6494, 6497, 6495, 6498, 6489, - 6504, 6500, 6486, 6492, 6503, 6490, 6487, 6501, 6509, 6506, 6507, 6511, - 6508, 6505, 6510, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 16720, 16731, 16722, 16716, 16728, 16733, 16723, 16729, - 16714, 16727, 16730, 16717, 16735, 16732, 16724, 16721, 16734, 16725, - 16718, 16719, 16726, 16715, 16736, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 4635, 4640, 4638, 4637, 4639, 4562, 4563, - 4581, 4582, 4579, 4580, 4574, 4575, 4576, 4577, 4608, 4564, 4555, 4565, - 4601, 4600, 4597, 4596, 4585, 4595, 4594, 4599, 4598, 4587, 4571, 4570, - 4567, 4566, 4586, 4573, 4572, 4569, 4568, 4588, 4603, 4602, 4593, 4592, - 4605, 4607, 4606, 4584, 4578, 4589, 4590, 4591, 4604, 4583, 4556, 4561, - 4560, 4645, 4644, 4654, 4655, 4648, 4649, 4650, 4651, 4652, 4653, 4656, - 4646, 4641, 4647, 4657, 4659, 4658, 4633, 4632, 4631, 4634, 4630, 40951, - 40951, 40951, 40951, 4626, 4624, 4621, 4612, 4614, 4619, 4617, 4609, - 4615, 4625, 4623, 4622, 4611, 4613, 4620, 4618, 4610, 4616, 4627, 4628, - 4666, 4668, 4665, 4664, 4661, 4660, 4663, 4662, 4669, 4667, 4636, 4558, - 4559, 4642, 4643, 4557, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 4629, 20979, 20981, 20983, 20939, 20940, 20949, 20950, - 20947, 20948, 20977, 20941, 20978, 20942, 20967, 20966, 20963, 20962, - 20951, 20961, 20960, 20965, 20964, 20953, 20944, 20943, 20936, 20934, - 20935, 20970, 20952, 20946, 20945, 20938, 20937, 20954, 20969, 20968, - 20959, 20958, 20974, 20976, 20971, 20973, 20975, 20955, 20956, 20957, - 20972, 20989, 20994, 20995, 20992, 20993, 20996, 20990, 20997, 20991, - 20982, 20980, 21000, 21001, 20998, 20984, 20985, 20987, 20986, 20988, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 20999, 40951, 40951, 33946, 33947, 33936, 33937, 33938, 33939, 33932, - 33933, 33943, 33935, 33948, 33944, 33949, 33945, 33940, 33942, 33941, - 33934, 33950, 33929, 33951, 33953, 33952, 33930, 33931, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 33960, 33962, 33959, 33958, 33955, - 33954, 33957, 33956, 33963, 33961, 40951, 40951, 40951, 40951, 40951, - 40951, 6174, 6176, 6175, 6170, 6172, 6173, 6171, 6159, 6158, 6155, 6154, - 6140, 6153, 6152, 6157, 6156, 6142, 6145, 6144, 6137, 6136, 6141, 6147, - 6146, 6139, 6138, 6143, 6163, 6162, 6151, 6150, 6165, 6148, 6149, 6166, - 6161, 6169, 6167, 6164, 6178, 6186, 6187, 6182, 6183, 6184, 6180, 6188, - 6181, 6189, 6206, 6205, 6190, 6204, 40951, 6199, 6201, 6198, 6197, 6194, - 6193, 6196, 6195, 6202, 6200, 6177, 6192, 6191, 6203, 6160, 6179, 6185, - 6168, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25371, - 25373, 25375, 25372, 25374, 25363, 25362, 25359, 25358, 25357, 25356, - 25361, 25360, 25342, 25351, 25350, 25345, 25344, 25341, 25353, 25352, - 25347, 25346, 25343, 25365, 25364, 25355, 25354, 25368, 25349, 25367, - 25370, 25369, 25366, 25348, 25378, 25379, 25377, 25376, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32741, 32744, 32748, - 32687, 32688, 32706, 32707, 32704, 32705, 32699, 32700, 32701, 32702, - 32733, 32689, 32734, 32690, 32726, 32725, 32722, 32721, 32710, 32720, - 32719, 32724, 32723, 32712, 32696, 32695, 32692, 32691, 32711, 32698, - 32697, 32694, 32693, 32713, 32728, 32727, 32718, 32717, 32730, 32732, - 32731, 32709, 32708, 32703, 32714, 32715, 32716, 32729, 32752, 32761, - 32762, 32755, 32756, 32757, 32758, 32759, 32760, 32763, 32753, 32764, - 32754, 32747, 32743, 32745, 32746, 32767, 32673, 32672, 32765, 32738, - 32735, 32742, 32750, 32684, 32749, 32751, 32739, 32680, 32682, 32679, - 32678, 32675, 32674, 32677, 32676, 32683, 32681, 32685, 32740, 32686, - 32766, 32736, 32737, 40951, 33649, 33647, 33646, 33643, 33642, 33645, - 33644, 33650, 33648, 33661, 33660, 33659, 33655, 33654, 33658, 33657, - 33653, 33656, 33651, 33652, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 22374, 22375, 22401, 22403, 22400, - 22376, 22402, 22377, 22391, 22390, 22366, 22364, 22365, 22384, 22389, - 22388, 22373, 22372, 40951, 22386, 22379, 22378, 22369, 22368, 22385, - 22381, 22380, 22371, 22367, 22370, 22387, 22393, 22392, 22363, 22361, - 22362, 22395, 22399, 22397, 22383, 22398, 22360, 22394, 22382, 22411, - 22414, 22415, 22418, 22416, 22412, 22417, 22413, 22408, 22407, 22406, - 22404, 22358, 22357, 22419, 22409, 22356, 22420, 22405, 22396, 22359, - 22410, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 28281, 28283, 28284, 28282, 28272, 28271, 28270, - 40951, 28269, 40951, 28268, 28267, 28260, 28259, 40951, 28254, 28262, - 28261, 28250, 28248, 28249, 28253, 28264, 28263, 28252, 28251, 28255, - 28274, 28273, 28266, 40951, 28265, 28277, 28280, 28258, 28276, 28279, - 28278, 28275, 28257, 28256, 28285, 40951, 40951, 40951, 40951, 40951, - 40951, 22435, 22436, 22447, 22448, 22445, 22446, 22466, 22437, 22467, - 22438, 22456, 22455, 22426, 22424, 22425, 22449, 22454, 22453, 22434, - 22433, 22432, 22451, 22442, 22441, 22429, 22427, 22439, 22428, 22450, - 22444, 22443, 22431, 22430, 22452, 22458, 22457, 22423, 22421, 22422, - 22463, 22465, 22440, 22462, 22464, 22459, 22460, 22461, 22470, 22471, - 22476, 22477, 22474, 22475, 22478, 22472, 22479, 22473, 22468, 22469, - 40951, 40951, 40951, 40951, 40951, 22486, 22488, 22485, 22484, 22481, - 22480, 22483, 22482, 22489, 22487, 40951, 40951, 40951, 40951, 40951, - 40951, 18095, 18096, 18099, 18102, 40951, 18052, 18053, 18066, 18067, - 18064, 18065, 18047, 18049, 40951, 40951, 18090, 18054, 40951, 40951, - 18089, 18055, 18086, 18085, 18082, 18081, 18070, 18080, 18079, 18084, - 18083, 18072, 18061, 18060, 18057, 18056, 18071, 18063, 18062, 18059, - 18058, 18073, 40951, 18088, 18087, 18078, 18077, 18092, 18094, 18093, - 40951, 18069, 18068, 40951, 18051, 18074, 18075, 18076, 18091, 40951, - 8077, 18097, 18098, 18104, 18113, 18114, 18107, 18108, 18109, 18110, - 40951, 40951, 18116, 18105, 40951, 40951, 18115, 18106, 18101, 40951, - 40951, 18117, 40951, 40951, 40951, 40951, 40951, 40951, 18103, 40951, - 40951, 40951, 40951, 40951, 18100, 18046, 18045, 18048, 18050, 18111, - 18112, 40951, 40951, 8257, 8258, 8256, 8255, 8254, 8253, 8252, 40951, - 40951, 40951, 8263, 8260, 8261, 8259, 8262, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 37588, 37589, 37612, - 37613, 37610, 37611, 37605, 37606, 37607, 37608, 40951, 37634, 40951, - 40951, 37590, 40951, 37633, 37591, 37630, 37629, 37626, 37625, 37614, - 37624, 37623, 37628, 37627, 37616, 37602, 37601, 37593, 37592, 37615, - 37604, 37603, 37595, 37594, 37617, 37632, 37631, 37622, 37621, 37636, - 37637, 37600, 37598, 37609, 37618, 37619, 37620, 37635, 37597, 37599, - 37596, 40951, 37639, 37650, 37659, 37660, 37653, 37654, 37655, 37656, - 37657, 37658, 40951, 37662, 40951, 40951, 37651, 40951, 37661, 37652, - 37583, 37644, 40951, 37640, 37647, 37646, 37641, 37584, 37638, 37587, - 37645, 37586, 37585, 40951, 37642, 37643, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 37649, 37648, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 29204, 29205, 29218, 29219, 29216, - 29217, 29200, 29201, 29202, 29203, 29244, 29206, 29245, 29207, 29232, - 29231, 29228, 29227, 29193, 29192, 29226, 29225, 29230, 29229, 29195, - 29194, 29213, 29212, 29209, 29208, 29197, 29215, 29214, 29211, 29210, - 29198, 29196, 29238, 29237, 29224, 29223, 29236, 29235, 29243, 29240, - 29239, 29234, 29233, 29242, 29220, 29221, 29222, 29241, 29258, 29267, - 29268, 29261, 29262, 29263, 29264, 29265, 29266, 29269, 29259, 29270, - 29260, 29253, 29246, 29249, 29254, 29247, 29248, 29251, 29274, 29255, - 29180, 29178, 29273, 29191, 29271, 29187, 29189, 29186, 29185, 29182, - 29181, 29184, 29183, 29190, 29188, 29179, 29257, 40951, 29272, 29256, - 29199, 29250, 29252, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 37311, 37312, 37313, 37331, 37332, 37329, 37330, - 37324, 37325, 37326, 37327, 37357, 37314, 37358, 37315, 37349, 37348, - 37345, 37344, 37333, 37343, 37342, 37347, 37346, 37335, 37321, 37320, - 37317, 37316, 37334, 37323, 37322, 37319, 37318, 37336, 37351, 37350, - 37341, 37340, 37354, 37356, 37355, 37353, 37328, 37337, 37338, 37339, - 37352, 37367, 37376, 37377, 37370, 37371, 37372, 37373, 37374, 37375, - 37378, 37365, 37368, 37379, 37366, 37369, 37359, 37362, 37364, 37363, - 37360, 37361, 37390, 37310, 37391, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 37386, 37388, 37385, 37384, 37381, 37380, 37383, - 37382, 37389, 37387, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 32855, - 32857, 32878, 32879, 32876, 32877, 32871, 32872, 32873, 32874, 32904, - 32858, 32905, 32859, 32896, 32895, 32892, 32891, 32880, 32890, 32889, - 32894, 32893, 32882, 32865, 32864, 32868, 32867, 32881, 32866, 32861, - 32870, 32869, 32883, 32898, 32897, 32888, 32887, 32901, 32903, 32902, - 32900, 32875, 32884, 32885, 32886, 32899, 32933, 32940, 32941, 32938, - 32939, 32936, 32937, 40951, 40951, 32942, 32934, 32943, 32935, 32926, - 32928, 32930, 32929, 32927, 32925, 32945, 32944, 32924, 32923, 32906, - 32907, 32908, 32854, 32921, 32920, 32918, 32916, 32917, 32909, 32910, - 32919, 32922, 32914, 32915, 32913, 32912, 32911, 32860, 32862, 32863, - 32856, 32931, 32932, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27636, 27637, 27655, - 27656, 27653, 27654, 27648, 27649, 27650, 27651, 27682, 27638, 27683, - 27639, 27675, 27674, 27671, 27670, 27659, 27669, 27668, 27673, 27672, - 27661, 27645, 27644, 27641, 27640, 27660, 27647, 27646, 27643, 27642, - 27662, 27677, 27676, 27667, 27666, 27679, 27681, 27680, 27658, 27652, - 27663, 27664, 27665, 27678, 27657, 27611, 27620, 27621, 27614, 27615, - 27616, 27617, 27618, 27619, 27622, 27612, 27623, 27613, 27607, 27610, - 27609, 27606, 27625, 27624, 27684, 27608, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27632, 27634, 27631, - 27630, 27627, 27626, 27629, 27628, 27635, 27633, 40951, 40951, 40951, - 40951, 40951, 40951, 28164, 28159, 28004, 28165, 28163, 28161, 28160, - 28021, 28022, 28155, 28157, 28156, 28166, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 35301, 35303, 35318, 35319, 35316, - 35317, 35343, 35304, 35344, 35305, 35333, 35332, 35329, 35328, 35320, - 35327, 35326, 35331, 35330, 35322, 35313, 35312, 35307, 35306, 35321, - 35315, 35314, 35309, 35308, 35323, 35335, 35334, 35325, 35324, 35340, - 35342, 35311, 35339, 35341, 35336, 35337, 35338, 35310, 35346, 35348, - 35349, 35354, 35355, 35352, 35353, 35356, 35350, 35357, 35351, 35347, - 35345, 35302, 35300, 40951, 40951, 40951, 40951, 40951, 40951, 35364, - 35366, 35363, 35362, 35359, 35358, 35361, 35360, 35367, 35365, 40951, - 40951, 40951, 40951, 40951, 40951, 28773, 28775, 28772, 28771, 28768, - 28767, 28770, 28769, 28776, 28774, 28723, 28725, 28722, 28721, 28718, - 28717, 28720, 28719, 28726, 28724, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 191, 190, 178, 181, 175, 167, 193, 192, 183, 195, - 189, 184, 174, 196, 177, 197, 180, 194, 164, 171, 170, 187, 166, 186, - 182, 188, 165, 40951, 40951, 162, 163, 161, 203, 204, 210, 211, 208, 209, - 212, 207, 213, 205, 206, 200, 40951, 40951, 40951, 40951, 222, 224, 221, - 220, 217, 216, 219, 218, 225, 223, 215, 214, 198, 199, 201, 202, 185, - 173, 172, 169, 168, 179, 176, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 11026, - 11027, 11042, 11043, 11040, 11041, 11068, 11028, 11069, 11029, 11060, - 11059, 11056, 11055, 11044, 11054, 11053, 11058, 11057, 11046, 11037, - 11036, 11031, 11030, 11045, 11039, 11038, 11033, 11032, 11047, 11062, - 11061, 11052, 11051, 11065, 11067, 11035, 11064, 11066, 11048, 11049, - 11050, 11063, 11034, 11072, 11077, 11078, 11075, 11076, 11070, 11071, - 11079, 11073, 11080, 11074, 11083, 11085, 11084, 11082, 11081, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 39052, - 39041, 39070, 39060, 39062, 39063, 39069, 39059, 39045, 39054, 39042, - 39072, 39068, 39047, 39061, 39058, 39046, 39055, 39064, 39053, 39071, - 39044, 39043, 39066, 39067, 39050, 39049, 39048, 39051, 39056, 39057, - 39065, 39084, 39073, 39102, 39092, 39094, 39095, 39101, 39091, 39077, - 39086, 39074, 39104, 39100, 39079, 39093, 39090, 39078, 39087, 39096, - 39085, 39103, 39076, 39075, 39098, 39099, 39082, 39081, 39080, 39083, - 39088, 39089, 39097, 39111, 39113, 39110, 39109, 39106, 39105, 39108, - 39107, 39114, 39112, 39123, 39122, 39121, 39117, 39116, 39120, 39119, - 39115, 39118, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 39124, 10942, 10943, 10950, 10951, 10948, - 10949, 10977, 40951, 40951, 10978, 40951, 40951, 10968, 10967, 10966, - 10965, 10954, 10964, 10963, 10972, 40951, 10956, 10938, 40951, 10945, - 10944, 10955, 10939, 10937, 10947, 10946, 10957, 10970, 10969, 10962, - 10961, 10973, 10941, 10940, 10974, 10953, 10975, 10958, 10959, 10960, - 10971, 10952, 10976, 10983, 10987, 10988, 10985, 10986, 10989, 40951, - 10984, 10990, 40951, 40951, 10982, 10980, 10979, 10991, 10996, 10993, - 10997, 10992, 10981, 10926, 10994, 10995, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 10933, 10935, 10932, 10931, 10928, - 10927, 10930, 10929, 10936, 10934, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 28895, 28896, 28911, 28912, 28909, - 28910, 28892, 28893, 40951, 40951, 28937, 28897, 28938, 28898, 28931, - 28930, 28927, 28926, 28915, 28925, 28924, 28929, 28928, 28917, 28906, - 28905, 28900, 28899, 28916, 28908, 28907, 28902, 28901, 28918, 28933, - 28932, 28923, 28922, 28935, 28936, 28904, 28914, 28894, 28919, 28920, - 28921, 28934, 28913, 28903, 28947, 28952, 28953, 28950, 28951, 28945, - 28946, 40951, 40951, 28954, 28948, 28955, 28949, 28941, 28943, 28942, - 28940, 28939, 28956, 28944, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40732, 40753, 40750, 40749, 40752, 40748, 40747, 40745, 40746, - 40751, 40744, 40700, 40699, 40719, 40718, 40701, 40717, 40716, 40726, - 40703, 40711, 40710, 40693, 40692, 40702, 40713, 40712, 40697, 40696, - 40704, 40721, 40720, 40715, 40714, 40728, 40709, 40708, 40695, 40694, - 40722, 40723, 40724, 40731, 40729, 40727, 40730, 40705, 40706, 40707, - 40725, 40698, 40691, 40741, 40738, 40739, 40740, 40737, 40742, 40688, - 40687, 40685, 40684, 40686, 40690, 40683, 40736, 40734, 40733, 40735, - 40689, 40682, 40743, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 34052, 34073, 34071, 34070, 34072, 34068, 34069, 34066, 34067, - 34065, 34064, 34074, 34019, 34018, 34038, 34037, 34020, 34036, 34035, - 34040, 34039, 34022, 34030, 34029, 34013, 34012, 34021, 34032, 34031, - 34016, 34014, 34023, 34042, 34041, 34034, 34033, 34048, 34028, 34027, - 34015, 34043, 34044, 34045, 34051, 34049, 34047, 34050, 34024, 34025, - 34026, 34046, 34017, 34057, 34059, 33996, 33995, 33993, 33994, 34004, - 34005, 34000, 34003, 33999, 34002, 34007, 34008, 34006, 33998, 33997, - 34001, 34060, 34058, 34075, 34061, 34056, 34055, 34054, 34053, 34011, - 34010, 34009, 34062, 34063, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 5617, 5618, 5615, 5616, - 5613, 5614, 5623, 5624, 5621, 5622, 5619, 5620, 5786, 5787, 5788, 5785, - 31247, 31245, 31254, 31255, 31251, 31259, 31258, 31240, 31253, 31252, - 31244, 31257, 31250, 31243, 31249, 31248, 31241, 31246, 31256, 31235, - 31242, 31260, 31261, 31236, 31262, 31238, 31239, 31237, 31231, 31228, - 31232, 31230, 31226, 31229, 31233, 31227, 31234, 31268, 31267, 31278, - 31269, 31270, 31279, 31275, 31274, 31276, 31277, 31271, 31225, 31272, - 31273, 31264, 31263, 31223, 31265, 31266, 31224, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 10668, 10669, 10758, 10759, 10760, 10761, - 10768, 10767, 10769, 10773, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 34658, 34677, 34684, 34681, - 34671, 34668, 34663, 34685, 34653, 34670, 34679, 34660, 34657, 34666, - 34683, 34661, 34680, 34667, 34672, 34678, 34682, 34655, 34654, 34659, - 34675, 34669, 34665, 34664, 34673, 34656, 34674, 34676, 34662, 34686, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 34693, 34695, 34692, 34691, 34688, 34687, - 34690, 34689, 34696, 34694, 40951, 40951, 40951, 40951, 40951, 40951, - 3661, 3662, 3675, 3676, 3673, 3674, 3657, 3658, 3659, 40951, 3701, 3663, - 3702, 3664, 3693, 3692, 3689, 3688, 3677, 3687, 3686, 3691, 3690, 3679, - 3670, 3669, 3666, 3665, 3678, 3672, 3671, 3668, 3667, 3680, 3695, 3694, - 3685, 3684, 3698, 3700, 3699, 3697, 3660, 3681, 3682, 3683, 3696, 3729, - 3734, 3735, 3732, 3733, 3726, 3727, 3728, 40951, 3736, 3730, 3737, 3731, - 3721, 3723, 3725, 3724, 3722, 3739, 3738, 3750, 3751, 3752, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 3746, 3748, 3745, - 3744, 3741, 3740, 3743, 3742, 3749, 3747, 3720, 3718, 3715, 3706, 3708, - 3713, 3711, 3703, 3709, 3719, 3717, 3716, 3705, 3707, 3714, 3712, 3704, - 3710, 3753, 40951, 40951, 40951, 25735, 25736, 25681, 25680, 25690, - 25675, 25679, 25678, 25692, 25676, 25672, 25671, 25674, 25677, 25683, - 25682, 25689, 25694, 25670, 25669, 25673, 25696, 25686, 25687, 25688, - 25697, 25695, 25693, 25684, 25685, 25691, 25698, 40951, 40951, 25713, - 25712, 25721, 25707, 25711, 25710, 25723, 25708, 25704, 25703, 25706, - 25709, 25715, 25714, 25720, 25725, 25702, 25701, 25705, 25727, 25718, - 25719, 40951, 25728, 25726, 25724, 25716, 25717, 25722, 25729, 25730, - 25732, 25734, 25731, 25733, 25700, 25699, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25747, - 25748, 25757, 25758, 25755, 25756, 25784, 40951, 25749, 25785, 40951, - 25750, 25763, 25762, 25776, 25775, 25764, 25774, 25773, 25741, 25740, - 25766, 25743, 25742, 25752, 25751, 25765, 25746, 25744, 25754, 25753, - 25767, 25778, 25777, 25772, 25771, 25780, 25783, 25781, 25760, 25782, - 25768, 25769, 25770, 25779, 25759, 25761, 25739, 25745, 25794, 25799, - 25800, 25797, 25798, 25793, 40951, 40951, 40951, 25801, 40951, 25795, - 25802, 40951, 25796, 25792, 25791, 25790, 25788, 25789, 25803, 25787, - 25786, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25810, - 25812, 25809, 25808, 25805, 25804, 25807, 25806, 25813, 25811, 40951, - 40951, 40951, 40951, 40951, 40951, 18770, 18771, 18784, 18785, 18782, - 18783, 40951, 18801, 18772, 40951, 18800, 18773, 18807, 18806, 18789, - 18788, 18803, 18797, 18796, 18781, 18780, 18787, 18793, 18792, 18777, - 18776, 18769, 18791, 18790, 18779, 18778, 18786, 18795, 18794, 18775, - 18774, 18768, 18799, 18798, 18802, 18804, 18805, 18810, 18815, 18816, - 18813, 18814, 40951, 18818, 18811, 40951, 18817, 18812, 18809, 18808, - 18819, 18830, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18826, - 18828, 18825, 18824, 18821, 18820, 18823, 18822, 18829, 18827, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25433, - 25431, 25425, 25436, 25428, 25435, 25439, 25430, 25427, 25429, 25432, - 25426, 25441, 25437, 25434, 25440, 25438, 25442, 25448, 25445, 25447, - 25444, 25446, 25443, 25424, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 21513, 21517, 21515, 21516, 21454, 21455, 21474, 21475, 21468, - 21469, 21470, 21471, 21472, 21473, 21499, 21456, 21500, 40951, 21490, - 21489, 21488, 21487, 21476, 21486, 21485, 21459, 21458, 21478, 21465, - 21464, 21461, 21460, 21477, 21467, 21466, 21463, 21462, 21479, 21492, - 21491, 21484, 21483, 21495, 21498, 21496, 21494, 21497, 21480, 21481, - 21482, 21493, 21457, 21519, 21518, 21526, 21527, 21524, 21525, 21521, - 40951, 40951, 40951, 21522, 21520, 21523, 21512, 21540, 21529, 21528, - 21507, 21510, 21506, 21508, 21504, 21503, 21511, 21502, 21505, 21509, - 21501, 21536, 21538, 21535, 21534, 21531, 21530, 21533, 21532, 21539, - 21537, 21514, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25086, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 35424, 35420, 35414, 35423, 35416, 35425, 35429, - 35431, 35426, 35421, 35422, 35427, 35415, 35432, 35430, 35417, 35428, - 35418, 35419, 35433, 35434, 35498, 35481, 35479, 35488, 35487, 35483, - 35489, 35485, 35484, 35491, 35492, 35493, 35490, 35482, 35495, 35413, - 35400, 35471, 35478, 35768, 35769, 35398, 35373, 35499, 35767, 35435, - 35500, 35486, 35494, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 35476, 8820, 8828, 8826, 8822, - 8827, 8823, 8821, 8824, 8825, 8889, 8829, 8838, 8837, 8831, 8830, 8842, - 8832, 8833, 8841, 8835, 8836, 8843, 8844, 8849, 8846, 8845, 8847, 8848, - 8851, 8853, 8855, 8854, 8856, 8862, 8859, 8860, 8864, 8857, 8858, 8863, - 8861, 8866, 8865, 8867, 8869, 8870, 8874, 8873, 8871, 8872, 8875, 8888, - 8876, 8877, 8878, 8887, 8879, 8883, 8884, 8882, 8880, 8881, 8886, 8885, - 8890, 8891, 8902, 8893, 8897, 8898, 8899, 8900, 8901, 8903, 8906, 8905, - 8904, 8907, 8909, 8910, 8911, 8912, 8913, 8914, 8915, 8916, 8917, 8918, - 8919, 8920, 8921, 8922, 8923, 8924, 8925, 8926, 8941, 8927, 8928, 8938, - 8932, 8929, 8930, 8931, 8939, 8936, 8940, 8937, 8933, 8935, 8948, 8944, - 8945, 8946, 8949, 8960, 8950, 8953, 8954, 8956, 8957, 8958, 8961, 8964, - 8962, 8963, 8965, 8966, 8968, 8969, 8998, 9005, 8999, 9000, 9001, 9002, - 9003, 9004, 9006, 9008, 9007, 9009, 9012, 9013, 9016, 9010, 9011, 9017, - 9062, 9061, 9063, 9018, 9021, 9022, 9023, 9019, 9020, 9024, 9027, 9025, - 9028, 9030, 9039, 9040, 9041, 9042, 9060, 9043, 9044, 9057, 9058, 9056, - 9045, 9046, 9047, 9048, 9049, 9050, 9051, 9054, 9055, 9064, 9154, 9065, - 9066, 9068, 9067, 9074, 9069, 9071, 9073, 9077, 9076, 9078, 9079, 9086, - 9080, 9081, 9085, 9089, 9090, 9087, 9088, 9096, 9093, 9097, 9098, 9099, - 9100, 9101, 9103, 9104, 9106, 9105, 9108, 9107, 9110, 9109, 9111, 9112, - 9114, 9115, 9119, 9121, 9125, 9126, 9139, 9131, 9132, 9127, 9128, 9129, - 9133, 9137, 9134, 9135, 9136, 9140, 9141, 9143, 9144, 9145, 9146, 9147, - 9148, 9158, 9149, 9150, 9153, 9152, 9151, 9155, 9156, 9157, 9159, 9160, - 9163, 9164, 9165, 9166, 9167, 9169, 9168, 9185, 9176, 9177, 9170, 9171, - 9173, 9174, 9172, 9175, 9184, 9178, 9183, 9181, 9180, 9182, 9187, 9206, - 9188, 9189, 9190, 9193, 9192, 9194, 9195, 9197, 9198, 9199, 9207, 9200, - 9201, 9202, 9205, 9204, 9203, 9208, 9209, 9211, 9212, 9213, 9214, 9216, - 9220, 9217, 9221, 9219, 9218, 9222, 9223, 9224, 9225, 9229, 9228, 9226, - 9227, 9230, 9231, 9233, 9252, 9254, 9234, 9236, 9235, 9237, 9238, 9241, - 9242, 9240, 9239, 9243, 9244, 9245, 9246, 9249, 9247, 9248, 9250, 9251, - 9255, 9256, 9253, 9257, 9258, 9259, 9260, 9262, 9265, 9264, 9266, 9267, - 9269, 9270, 9271, 9275, 9274, 9272, 9273, 9276, 9280, 9279, 9278, 9281, - 9282, 9284, 9285, 9289, 9286, 9287, 9292, 9290, 9293, 9294, 9296, 9295, - 9297, 9298, 9320, 9319, 9322, 9323, 9304, 9305, 9302, 9303, 9301, 9299, - 9307, 9306, 9308, 9310, 9316, 9317, 9314, 9315, 9324, 9325, 9326, 9344, - 9329, 9330, 9331, 9327, 9328, 9332, 9333, 9334, 9335, 9336, 9338, 9339, - 9340, 9341, 9342, 9367, 9345, 9348, 9346, 9347, 9353, 9354, 9351, 9352, - 9349, 9350, 9355, 9356, 9364, 9357, 9358, 9365, 9362, 9363, 9366, 9359, - 9360, 9361, 9368, 9369, 9370, 9371, 9372, 9373, 9374, 9376, 9377, 9375, - 9378, 9379, 9420, 9421, 9382, 9383, 9380, 9381, 9386, 9387, 9385, 9391, - 9388, 9390, 9389, 9395, 9396, 9394, 9392, 9393, 9397, 9398, 9399, 9400, - 9401, 9402, 9403, 9422, 9408, 9404, 9405, 9406, 9407, 9409, 9411, 9410, - 9412, 9413, 9415, 9414, 9416, 9418, 9417, 9423, 9424, 9427, 9428, 9425, - 9426, 9502, 9497, 9498, 9499, 9500, 9501, 9503, 9506, 9504, 9505, 9507, - 9549, 9508, 9538, 9539, 9515, 9517, 9534, 9518, 9540, 9522, 9520, 9521, - 9523, 9524, 9525, 9526, 9535, 9536, 9529, 9530, 9532, 9541, 9509, 9510, - 9514, 9512, 9550, 9542, 9544, 9543, 9545, 9551, 9552, 9546, 9547, 9548, - 9553, 9554, 9555, 9558, 9559, 9560, 9556, 9557, 9561, 9562, 9564, 9566, - 9567, 9585, 9583, 9584, 9587, 9586, 9568, 9575, 9573, 9574, 9569, 9570, - 9576, 9577, 9578, 9579, 9580, 9582, 9588, 9597, 9589, 9591, 9592, 9590, - 9593, 9595, 9594, 9596, 9599, 9601, 9600, 9602, 9603, 9636, 9638, 9604, - 9606, 9605, 9608, 9611, 9609, 9610, 9614, 9618, 9621, 9620, 9623, 9626, - 9624, 9625, 9629, 9631, 9637, 9639, 9640, 9644, 9651, 9649, 9647, 9648, - 9650, 9653, 9652, 9645, 9646, 9654, 9657, 9663, 9658, 9661, 9656, 9655, - 9664, 9662, 9659, 9660, 9665, 9666, 9667, 9668, 9669, 9670, 9671, 9673, - 9676, 9677, 9680, 9681, 9682, 9678, 9679, 9674, 9675, 9683, 9684, 9685, - 9686, 9687, 9688, 9690, 9691, 9692, 9693, 9694, 9698, 9695, 9719, 9712, - 9713, 9718, 9701, 9700, 9714, 9717, 9715, 9704, 9703, 9706, 9708, 9709, - 9710, 9711, 9707, 9720, 9696, 9721, 9722, 9723, 9724, 9725, 9726, 9734, - 9732, 9730, 9733, 9729, 9731, 9727, 9728, 9735, 9737, 9738, 9739, 9746, - 9741, 9742, 9750, 9751, 9747, 9749, 9748, 9752, 9754, 9753, 9755, 9766, - 9757, 9756, 9765, 9763, 9758, 9759, 9760, 9762, 9761, 9764, 9770, 9767, - 9769, 9768, 9771, 9772, 9773, 9774, 9777, 9778, 9780, 9781, 9782, 9783, - 9785, 9784, 9786, 9793, 9787, 9788, 9794, 9789, 9790, 9791, 9792, 9795, - 9799, 9796, 9797, 9798, 9800, 9801, 9802, 9803, 9809, 9807, 9805, 9806, - 9804, 9808, 9810, 9812, 9829, 9830, 9813, 9818, 9820, 9814, 9817, 9815, - 9816, 9821, 9827, 9828, 9822, 9825, 9826, 9831, 9837, 9836, 9832, 9834, - 9833, 9918, 9919, 9838, 9839, 9844, 9845, 9842, 9843, 9846, 9840, 9841, - 9847, 9850, 9851, 9853, 9854, 9855, 9859, 9856, 9857, 9858, 9848, 9849, - 9860, 9862, 9861, 9863, 9865, 9866, 9867, 9873, 9872, 9868, 9869, 9870, - 9905, 9874, 9875, 9876, 9877, 9878, 9897, 9880, 9881, 9883, 9882, 9884, - 9885, 9901, 9886, 9888, 9887, 9900, 9890, 9898, 9902, 9893, 9892, 9899, - 9894, 9896, 9895, 9903, 9904, 9906, 9910, 9908, 9909, 9907, 9913, 9912, - 9911, 9917, 9914, 9915, 9916, 9920, 9922, 9921, 9925, 9923, 9940, 9926, - 9929, 9931, 9927, 9928, 9932, 9930, 9933, 9935, 9937, 9938, 9939, 9343, - 8839, 8850, 8868, 8934, 8943, 8959, 8967, 9059, 9052, 9070, 9072, 9162, - 9186, 9232, 9261, 9263, 9277, 9283, 9318, 9291, 9321, 9300, 9309, 9313, - 9384, 9513, 9516, 9531, 9563, 9581, 9598, 9607, 9635, 9633, 9612, 9643, - 9672, 9689, 9699, 9819, 9852, 9835, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8804, 8799, 8736, 8722, - 8783, 8779, 8711, 8756, 8803, 8744, 8728, 8789, 8778, 8710, 8755, 8742, - 8726, 8785, 8775, 8705, 8752, 8765, 8809, 8801, 8738, 8724, 8787, 8777, - 8707, 8754, 8766, 8810, 8802, 8739, 8725, 8811, 8793, 8794, 8740, 8716, - 8782, 8774, 8704, 8751, 8769, 8812, 8795, 8796, 8741, 8717, 8780, 8781, - 8764, 8807, 8790, 8791, 8731, 8721, 8797, 8798, 8732, 8735, 8733, 8734, - 8788, 8773, 8771, 8772, 8708, 8709, 8747, 8749, 8750, 8748, 8805, 8800, - 8737, 8723, 8784, 8763, 8806, 8792, 8729, 8730, 8719, 8720, 8746, 8745, - 8759, 8808, 8768, 8814, 8718, 8767, 8813, 8760, 8761, 8757, 8758, 8762, - 8770, 8712, 8715, 8714, 8713, 8743, 8727, 8786, 8776, 8706, 8753, 40951, - 8819, 8818, 8817, 8815, 8816, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 8840, 8834, 8852, 8892, 8894, 8895, - 8896, 8908, 8947, 8942, 8952, 8951, 8955, 8972, 8970, 8971, 8973, 8974, - 8992, 8978, 8975, 8976, 8977, 8994, 8995, 8993, 8982, 8981, 8979, 8980, - 8985, 8983, 8984, 8986, 8987, 8988, 8989, 8996, 8997, 8991, 8990, 9015, - 9014, 9026, 9029, 9034, 9038, 9031, 9035, 9036, 9032, 9033, 9037, 9053, - 9075, 9082, 9083, 9084, 9091, 9092, 9095, 9094, 9102, 9113, 9116, 9117, - 9118, 9120, 9122, 9123, 9124, 9130, 9138, 9142, 9161, 9179, 9191, 9196, - 9210, 9215, 9268, 9288, 9311, 9312, 9419, 9439, 9429, 9430, 9438, 9434, - 9435, 9436, 9433, 9432, 9431, 9437, 9441, 9440, 9447, 9448, 9442, 9443, - 9444, 9449, 9445, 9446, 9450, 9451, 9452, 9453, 9454, 9455, 9462, 9456, - 9461, 9460, 9458, 9457, 9459, 9463, 9464, 9469, 9470, 9465, 9466, 9467, - 9468, 9496, 9492, 9471, 9479, 9480, 9478, 9477, 9481, 9472, 9473, 9475, - 9476, 9474, 9493, 9482, 9489, 9491, 9486, 9487, 9490, 9485, 9488, 9484, - 9483, 9494, 9495, 9511, 9537, 9519, 9527, 9528, 9533, 9565, 9572, 9571, - 9632, 9613, 9615, 9634, 9616, 9617, 9619, 9622, 9627, 9628, 9630, 9697, - 9716, 9702, 9705, 9736, 9740, 9743, 9744, 9745, 9776, 9775, 9779, 9811, - 9823, 9824, 9864, 9871, 9879, 9891, 9889, 9924, 9934, 9936, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 10013, 10014, 10015, 10016, 10017, 10018, 10019, 10020, 10023, 10024, - 10021, 10022, 10025, 10026, 10027, 10028, 10029, 10030, 10031, 10032, - 10033, 10034, 10035, 10036, 10037, 10038, 10039, 10040, 10041, 10042, - 10043, 10044, 10045, 10046, 10047, 10048, 10049, 10050, 10051, 10052, - 10053, 10054, 10055, 10056, 10057, 10058, 10059, 10079, 10080, 10081, - 10082, 10083, 10084, 10085, 10086, 10087, 10062, 10063, 10064, 10065, - 10066, 10060, 10061, 10067, 10068, 10069, 10088, 10089, 10090, 10091, - 10092, 10093, 10094, 10095, 10096, 10097, 10070, 10071, 10072, 10073, - 10074, 10075, 10076, 10077, 10078, 10098, 10099, 10100, 10101, 10102, - 10103, 10104, 10105, 10106, 10107, 10108, 10109, 10110, 10111, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 11548, 11549, 11550, 11551, 11546, 11547, 11543, 11544, - 11545, 11552, 11553, 11554, 11559, 11560, 11561, 11562, 11555, 11556, - 11563, 11564, 11557, 11558, 11565, 11566, 11593, 11594, 11595, 11596, - 11597, 11598, 11599, 11600, 11601, 11602, 11583, 11584, 11581, 11582, - 11585, 11586, 11587, 11588, 11589, 11590, 11591, 11567, 11568, 11575, - 11569, 11570, 11571, 11572, 11576, 11573, 11574, 11577, 11578, 11579, - 11580, 11603, 11604, 11605, 11606, 11607, 11608, 11609, 11610, 11611, - 11612, 11613, 11614, 11615, 11616, 11617, 11618, 11619, 11620, 11621, - 11622, 11592, 11662, 11663, 11664, 11665, 11660, 11661, 11666, 11667, - 11668, 11669, 11674, 11670, 11671, 11672, 11673, 11675, 11676, 11677, - 11678, 11679, 11680, 11681, 11682, 11683, 11684, 11685, 11686, 11687, - 11688, 11689, 11690, 11691, 11692, 11693, 11694, 11695, 11696, 11697, - 11700, 11701, 11702, 11703, 11704, 11705, 11706, 11698, 11699, 11707, - 11780, 11781, 11782, 11783, 11784, 11785, 11786, 11787, 11788, 11789, - 11771, 11772, 11773, 11774, 11775, 11776, 11777, 11769, 11770, 11778, - 11779, 11712, 11708, 11709, 11713, 11714, 11710, 11711, 11715, 11716, - 11717, 11718, 11719, 11724, 11725, 11726, 11727, 11728, 11729, 11720, - 11721, 11730, 11722, 11723, 11731, 11732, 11733, 11734, 11735, 11736, - 11737, 11738, 11739, 11740, 11741, 11746, 11742, 11743, 11747, 11744, - 11745, 11748, 11749, 11750, 11751, 11752, 11762, 11763, 11764, 11765, - 11766, 11767, 11768, 11753, 11754, 11755, 11756, 11757, 11758, 11759, - 11760, 11761, 11794, 11795, 11796, 11797, 11798, 11799, 11800, 11790, - 11791, 11792, 11793, 11826, 11827, 11828, 11829, 11830, 11831, 11822, - 11823, 11824, 11825, 11832, 11833, 11801, 11802, 11805, 11806, 11807, - 11808, 11809, 11810, 11811, 11803, 11804, 11812, 11815, 11816, 11817, - 11818, 11813, 11814, 11819, 11820, 11821, 11837, 11838, 11839, 11840, - 11841, 11842, 11843, 11844, 11845, 11846, 11849, 11850, 11851, 11847, - 11848, 11852, 11853, 11854, 11855, 11856, 11857, 11893, 11891, 11892, - 11894, 11895, 11896, 11897, 11898, 11899, 11900, 11901, 11864, 11858, - 11859, 11865, 11866, 11867, 11868, 11869, 11860, 11861, 11862, 11863, - 11870, 11877, 11878, 11879, 11880, 11881, 11871, 11872, 11873, 11874, - 11875, 11876, 11882, 11883, 11888, 11884, 11885, 11886, 11887, 11889, - 11890, 11908, 11909, 11910, 11911, 11912, 11906, 11907, 11903, 11904, - 11905, 11913, 11914, 11917, 11915, 11916, 11918, 11919, 11920, 11921, - 11922, 11923, 11924, 11925, 11950, 11951, 11954, 11955, 11956, 11957, - 11958, 11952, 11953, 11959, 11960, 11961, 11930, 11931, 11932, 11933, - 11934, 11935, 11926, 11927, 11928, 11929, 11936, 11937, 11942, 11943, - 11944, 11938, 11939, 11945, 11940, 11941, 11946, 11947, 11948, 11949, - 11962, 11963, 11964, 11965, 11966, 11969, 11970, 11971, 11972, 11973, - 11967, 11968, 11974, 11975, 11983, 11984, 11985, 11986, 11979, 11980, - 11987, 11988, 11989, 11981, 11982, 11990, 11991, 11992, 11993, 11994, - 11995, 11996, 11997, 12638, 12639, 12640, 12641, 12642, 12643, 12644, - 12645, 12009, 12005, 12006, 12010, 12011, 12012, 12007, 12008, 12013, - 12014, 12016, 12017, 12018, 12021, 12019, 12020, 12022, 12023, 12024, - 12025, 12026, 12027, 12037, 12038, 12045, 12028, 12029, 12030, 12031, - 12032, 12033, 12034, 12035, 12036, 12046, 12047, 12039, 12040, 12041, - 12042, 12043, 12044, 12048, 12049, 12056, 12057, 12050, 12051, 12058, - 12052, 12053, 12059, 12060, 12061, 12054, 12055, 12062, 12068, 12066, - 12067, 12069, 12063, 12064, 12065, 12070, 12071, 12072, 12073, 12074, - 12075, 12076, 12077, 12078, 12079, 12080, 12081, 12138, 12139, 12140, - 12141, 12142, 12143, 12144, 12145, 12146, 12101, 12102, 12103, 12104, - 12105, 12106, 12107, 12108, 12098, 12099, 12100, 12109, 12126, 12127, - 12128, 12129, 12130, 12124, 12125, 12131, 12132, 12133, 12134, 12118, - 12119, 12120, 12110, 12111, 12112, 12113, 12114, 12115, 12121, 12116, - 12117, 12122, 12123, 12135, 12136, 12137, 12149, 12150, 12151, 12152, - 12147, 12148, 12153, 12154, 12155, 12156, 12159, 12160, 12161, 12162, - 12163, 12164, 12165, 12157, 12158, 12166, 12167, 12168, 12186, 12187, - 12188, 12189, 12190, 12191, 12192, 12193, 12194, 12169, 12170, 12171, - 12172, 12175, 12176, 12177, 12178, 12179, 12180, 12173, 12174, 12181, - 12184, 12185, 12182, 12183, 12202, 12203, 12206, 12207, 12208, 12204, - 12205, 12195, 12196, 12197, 12198, 12199, 12200, 12201, 12209, 12210, - 12211, 12212, 12213, 12214, 12215, 12218, 12219, 12220, 12221, 12222, - 12223, 12224, 12225, 12216, 12217, 12226, 12227, 12234, 12235, 12236, - 12228, 12229, 12230, 12231, 12237, 12238, 12239, 12232, 12233, 12245, - 12246, 12249, 12250, 12247, 12248, 12251, 12252, 12240, 12241, 12242, - 12243, 12244, 12253, 12254, 12255, 12260, 12261, 12262, 12263, 12264, - 12265, 12266, 12267, 12268, 12269, 12256, 12257, 12258, 12259, 12271, - 12272, 12275, 12273, 12274, 12276, 12277, 12278, 12279, 12280, 12281, - 12282, 12283, 12646, 12647, 12648, 12649, 12650, 12651, 12652, 12289, - 12287, 12288, 12284, 12285, 12286, 12290, 12291, 12292, 12293, 12294, - 12295, 12296, 12297, 12300, 12301, 12302, 12303, 12304, 12298, 12299, - 12305, 12306, 12307, 12308, 12309, 12310, 12311, 12312, 12313, 12314, - 12315, 12316, 12317, 12322, 12318, 12319, 12323, 12324, 12325, 12320, - 12321, 12326, 12327, 12328, 12334, 12335, 12336, 12337, 12329, 12330, - 12331, 12338, 12339, 12332, 12333, 12340, 12341, 12345, 12346, 12347, - 12348, 12349, 12350, 12342, 12343, 12344, 12351, 12352, 12353, 12356, - 12357, 12358, 12359, 12360, 12354, 12355, 12361, 12362, 12363, 12364, - 12365, 12366, 12367, 12368, 12369, 12370, 12371, 12380, 12381, 12372, - 12373, 12382, 12383, 12384, 12374, 12375, 12376, 12377, 12378, 12379, - 12389, 12385, 12386, 12390, 12391, 12392, 12393, 12387, 12388, 12394, - 12395, 12396, 12406, 12407, 12408, 12409, 12410, 12411, 12412, 12413, - 12414, 12415, 12401, 12402, 12397, 12398, 12399, 12400, 12403, 12404, - 12405, 12420, 12421, 12422, 12423, 12424, 12417, 12418, 12419, 12425, - 12426, 12427, 12454, 12455, 12456, 12457, 12458, 12459, 12460, 12461, - 12462, 12463, 12432, 12433, 12434, 12428, 12429, 12435, 12436, 12437, - 12438, 12439, 12430, 12431, 12442, 12443, 12440, 12441, 12444, 12445, - 12446, 12447, 12448, 12449, 12450, 12451, 12452, 12453, 12467, 12468, - 12469, 12470, 12471, 12472, 12473, 12474, 12475, 12476, 12477, 12478, - 12479, 12480, 12481, 12482, 12464, 12465, 12466, 12483, 12484, 12493, - 12485, 12486, 12487, 12488, 12490, 12491, 12492, 12494, 12495, 12496, - 12497, 12498, 12499, 12500, 12501, 12502, 12503, 12504, 12505, 12506, - 12507, 12508, 12509, 12510, 12511, 12512, 12513, 12520, 12521, 12514, - 12515, 12522, 12523, 12524, 12525, 12516, 12517, 12518, 12519, 12526, - 12527, 12528, 12529, 12534, 12530, 12531, 12535, 12536, 12537, 12532, - 12533, 12538, 12539, 12540, 12541, 12547, 12548, 12543, 12544, 12549, - 12550, 12551, 12552, 12553, 12545, 12546, 12554, 12555, 12562, 12563, - 12564, 12556, 12557, 12565, 12566, 12558, 12559, 12560, 12561, 12567, - 12570, 12571, 12572, 12573, 12568, 12569, 12574, 12583, 12584, 12585, - 12576, 12577, 12578, 12586, 12579, 12580, 12587, 12581, 12582, 12588, - 12589, 12590, 12591, 12592, 12593, 12594, 12595, 12596, 12609, 12597, - 12598, 12599, 12600, 12601, 12602, 12603, 12604, 12605, 12606, 12607, - 12608, 12610, 12611, 12612, 12613, 12633, 12634, 12635, 12636, 12637, - 12614, 12615, 12616, 12617, 12618, 12619, 12620, 12621, 12622, 12623, - 12624, 12625, 12626, 12627, 12628, 12629, 12630, 12631, 12632, 11626, - 11627, 11628, 11629, 11630, 11631, 11623, 11624, 11625, 11632, 11633, - 11637, 11638, 11639, 11640, 11641, 11642, 11643, 11644, 11645, 11646, - 11647, 11648, 11649, 11650, 11651, 11652, 11653, 11654, 11655, 11656, - 11634, 11635, 11636, 12489, 12542, 11978, 12003, 12000, 12002, 11999, - 12270, 11659, 11836, 12004, 12001, 11998, 11658, 11835, 11657, 11834, - 12097, 11902, 11976, 12015, 11977, 12416, 12575, 12093, 12084, 12088, - 12095, 12091, 12085, 12090, 12087, 12094, 12083, 12089, 12096, 12092, - 12086, 12082, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 12653, 12654, 12655, 12656, 12657, 12658, 12659, 12660, - 12661, 12662, 12663, 12664, 12665, 12666, 12667, 12668, 12669, 12670, - 12671, 12672, 12673, 12674, 12675, 12676, 12677, 12678, 12679, 12680, - 12681, 12682, 12683, 12684, 12685, 12686, 12687, 12688, 12689, 12690, - 12691, 12692, 12693, 12694, 12695, 12696, 12697, 12698, 12699, 12700, - 12701, 12702, 12703, 12704, 12705, 12706, 12707, 12708, 12709, 12710, - 12711, 12712, 12713, 12714, 12715, 12716, 12717, 12718, 12719, 12720, - 12721, 12722, 12723, 12724, 12725, 12726, 12727, 12728, 12729, 12730, - 12731, 12732, 12733, 12734, 12735, 12736, 12737, 12738, 12739, 12740, - 12741, 12742, 12743, 12744, 12745, 12746, 12747, 12748, 12749, 12750, - 12751, 12752, 12753, 12754, 12755, 12756, 12757, 12758, 12759, 12760, - 12761, 12762, 12763, 12764, 12765, 12766, 12767, 12768, 12769, 12770, - 12771, 12772, 12773, 12774, 12775, 12776, 12777, 12778, 12779, 12780, - 12781, 12782, 12783, 12784, 12785, 12786, 12787, 12788, 12789, 12790, - 12791, 12792, 12793, 12794, 12795, 12796, 12797, 12798, 12799, 12800, - 12801, 12802, 12803, 12804, 12805, 12806, 12807, 12808, 12809, 12810, - 12811, 12812, 12813, 12814, 12815, 12816, 12817, 12818, 12819, 12820, - 12821, 12822, 12823, 12824, 12825, 12826, 12827, 12828, 12829, 12830, - 12831, 12832, 12833, 12834, 12835, 12836, 12837, 12838, 12839, 12840, - 12841, 12842, 12843, 12844, 12845, 12846, 12847, 12848, 12849, 12850, - 12851, 12852, 12853, 12854, 12855, 12856, 12857, 12858, 12859, 12860, - 12861, 12862, 12863, 12864, 12865, 12866, 12867, 12868, 12869, 12870, - 12871, 12872, 12873, 12874, 12875, 12876, 12877, 12878, 12879, 12880, - 12881, 12882, 12883, 12884, 12885, 12886, 12887, 12888, 12889, 12890, - 12891, 12892, 12893, 12894, 12895, 12896, 12897, 12898, 12899, 12900, - 12901, 12902, 12903, 12904, 12905, 12906, 12907, 12908, 12909, 12910, - 12911, 12912, 12913, 12914, 12915, 12916, 12917, 12918, 12919, 12920, - 12921, 12922, 12923, 12924, 12925, 12926, 12927, 12928, 12929, 12930, - 12931, 12932, 12933, 12934, 12935, 12936, 12937, 12938, 12939, 12940, - 12941, 12942, 12943, 12944, 12945, 12946, 12947, 12948, 12949, 12950, - 12951, 12952, 12953, 12954, 12955, 12956, 12957, 12958, 12959, 12960, - 12961, 12962, 12963, 12964, 12965, 12966, 12967, 12968, 12969, 12970, - 12971, 12972, 12973, 12974, 12975, 12976, 12977, 12978, 12979, 12980, - 12981, 12982, 12983, 12984, 12985, 12986, 12987, 12988, 12989, 12990, - 12991, 12992, 12993, 12994, 12995, 12996, 12997, 12998, 12999, 13000, - 13001, 13002, 13003, 13004, 13005, 13006, 13007, 13008, 13009, 13010, - 13011, 13012, 13013, 13014, 13015, 13016, 13017, 13018, 13019, 13020, - 13021, 13022, 13023, 13024, 13025, 13026, 13027, 13028, 13029, 13030, - 13031, 13032, 13033, 13034, 13035, 13036, 13037, 13038, 13039, 13040, - 13041, 13042, 13043, 13044, 13045, 13046, 13047, 13048, 13049, 13050, - 13051, 13052, 13053, 13054, 13055, 13056, 13057, 13058, 13059, 13060, - 13061, 13062, 13063, 13064, 13065, 13066, 13067, 13068, 13069, 13070, - 13071, 13072, 13073, 13074, 13075, 13076, 13077, 13078, 13079, 13080, - 13081, 13082, 13083, 13084, 13085, 13086, 13087, 13088, 13089, 13090, - 13091, 13092, 13093, 13094, 13095, 13096, 13097, 13098, 13099, 13100, - 13101, 13102, 13103, 13104, 13105, 13106, 13107, 13108, 13109, 13110, - 13111, 13112, 13113, 13114, 13115, 13116, 13117, 13118, 13119, 13120, - 13121, 13122, 13123, 13124, 13125, 13126, 13127, 13128, 13129, 13130, - 13131, 13132, 13133, 13134, 13135, 13136, 13137, 13138, 13139, 13140, - 13141, 13142, 13143, 13144, 13145, 13146, 13147, 13148, 13149, 13150, - 13151, 13152, 13153, 13154, 13155, 13156, 13157, 13158, 13159, 13160, - 13161, 13162, 13163, 13164, 13165, 13166, 13167, 13168, 13169, 13170, - 13171, 13172, 13173, 13174, 13175, 13176, 13177, 13178, 13179, 13180, - 13181, 13182, 13183, 13184, 13185, 13186, 13187, 13188, 13189, 13190, - 13191, 13192, 13193, 13194, 13195, 13196, 13197, 13198, 13199, 13200, - 13201, 13202, 13203, 13204, 13205, 13206, 13207, 13208, 13209, 13210, - 13211, 13212, 13213, 13214, 13215, 13216, 13217, 13218, 13219, 13220, - 13221, 13222, 13223, 13224, 13225, 13226, 13227, 13228, 13229, 13230, - 13231, 13232, 13233, 13234, 13235, 13236, 13237, 13238, 13239, 13240, - 13241, 13242, 13243, 13244, 13245, 13246, 13247, 13248, 13249, 13250, - 13251, 13252, 13253, 13254, 13255, 13256, 13257, 13258, 13259, 13260, - 13261, 13262, 13263, 13264, 13265, 13266, 13267, 13268, 13269, 13270, - 13271, 13272, 13273, 13274, 13275, 13276, 13277, 13278, 13279, 13280, - 13281, 13282, 13283, 13284, 13285, 13286, 13287, 13288, 13289, 13290, - 13291, 13292, 13293, 13294, 13295, 13296, 13297, 13298, 13299, 13300, - 13301, 13302, 13303, 13304, 13305, 13306, 13307, 13308, 13309, 13310, - 13311, 13312, 13313, 13314, 13315, 13316, 13317, 13318, 13319, 13320, - 13321, 13322, 13323, 13324, 13325, 13326, 13327, 13328, 13329, 13330, - 13331, 13332, 13333, 13334, 13335, 13336, 13337, 13338, 13339, 13340, - 13341, 13342, 13343, 13344, 13345, 13346, 13347, 13348, 13349, 13350, - 13351, 13352, 13353, 13354, 13355, 13356, 13357, 13358, 13359, 13360, - 13361, 13362, 13363, 13364, 13365, 13366, 13367, 13368, 13369, 13370, - 13371, 13372, 13373, 13374, 13375, 13376, 13377, 13378, 13379, 13380, - 13381, 13382, 13383, 13384, 13385, 13386, 13387, 13388, 13389, 13390, - 13391, 13392, 13393, 13394, 13395, 13396, 13397, 13398, 13399, 13400, - 13401, 13402, 13403, 13404, 13405, 13406, 13407, 13408, 13409, 13410, - 13411, 13412, 13413, 13414, 13415, 13416, 13417, 13418, 13419, 13420, - 13421, 13422, 13423, 13424, 13425, 13426, 13427, 13428, 13429, 13430, - 13431, 13432, 13433, 13434, 13435, 13436, 13437, 13438, 13439, 13440, - 13441, 13442, 13443, 13444, 13445, 13446, 13447, 13448, 13449, 13450, - 13451, 13452, 13453, 13454, 13455, 13456, 13457, 13458, 13459, 13460, - 13461, 13462, 13463, 13464, 13465, 13466, 13467, 13468, 13469, 13470, - 13471, 13472, 13473, 13474, 13475, 13476, 13477, 13478, 13479, 13480, - 13481, 13482, 13483, 13484, 13485, 13486, 13487, 13488, 13489, 13490, - 13491, 13492, 13493, 13494, 13495, 13496, 13497, 13498, 13499, 13500, - 13501, 13502, 13503, 13504, 13505, 13506, 13507, 13508, 13509, 13510, - 13511, 13512, 13513, 13514, 13515, 13516, 13517, 13518, 13519, 13520, - 13521, 13522, 13523, 13524, 13525, 13526, 13527, 13528, 13529, 13530, - 13531, 13532, 13533, 13534, 13535, 13536, 13537, 13538, 13539, 13540, - 13541, 13542, 13543, 13544, 13545, 13546, 13547, 13548, 13549, 13550, - 13551, 13552, 13553, 13554, 13555, 13556, 13557, 13558, 13559, 13560, - 13561, 13562, 13563, 13564, 13565, 13566, 13567, 13568, 13569, 13570, - 13571, 13572, 13573, 13574, 13575, 13576, 13577, 13578, 13579, 13580, - 13581, 13582, 13583, 13584, 13585, 13586, 13587, 13588, 13589, 13590, - 13591, 13592, 13593, 13594, 13595, 13596, 13597, 13598, 13599, 13600, - 13601, 13602, 13603, 13604, 13605, 13606, 13607, 13608, 13609, 13610, - 13611, 13612, 13613, 13614, 13615, 13616, 13617, 13618, 13619, 13620, - 13621, 13622, 13623, 13624, 13625, 13626, 13627, 13628, 13629, 13630, - 13631, 13632, 13633, 13634, 13635, 13636, 13637, 13638, 13639, 13640, - 13641, 13642, 13643, 13644, 13645, 13646, 13647, 13648, 13649, 13650, - 13651, 13652, 13653, 13654, 13655, 13656, 13657, 13658, 13659, 13660, - 13661, 13662, 13663, 13664, 13665, 13666, 13667, 13668, 13669, 13670, - 13671, 13672, 13673, 13674, 13675, 13676, 13677, 13678, 13679, 13680, - 13681, 13682, 13683, 13684, 13685, 13686, 13687, 13688, 13689, 13690, - 13691, 13692, 13693, 13694, 13695, 13696, 13697, 13698, 13699, 13700, - 13701, 13702, 13703, 13704, 13705, 13706, 13707, 13708, 13709, 13710, - 13711, 13712, 13713, 13714, 13715, 13716, 13717, 13718, 13719, 13720, - 13721, 13722, 13723, 13724, 13725, 13726, 13727, 13728, 13729, 13730, - 13731, 13732, 13733, 13734, 13735, 13736, 13737, 13738, 13739, 13740, - 13741, 13742, 13743, 13744, 13745, 13746, 13747, 13748, 13749, 13750, - 13751, 13752, 13753, 13754, 13755, 13756, 13757, 13758, 13759, 13760, - 13761, 13762, 13763, 13764, 13765, 13766, 13767, 13768, 13769, 13770, - 13771, 13772, 13773, 13774, 13775, 13776, 13777, 13778, 13779, 13780, - 13781, 13782, 13783, 13784, 13785, 13786, 13787, 13788, 13789, 13790, - 13791, 13792, 13793, 13794, 13795, 13796, 13797, 13798, 13799, 13800, - 13801, 13802, 13803, 13804, 13805, 13806, 13807, 13808, 13809, 13810, - 13811, 13812, 13813, 13814, 13815, 13816, 13817, 13818, 13819, 13820, - 13821, 13822, 13823, 13824, 13825, 13826, 13827, 13828, 13829, 13830, - 13831, 13832, 13833, 13834, 13835, 13836, 13837, 13838, 13839, 13840, - 13841, 13842, 13843, 13844, 13845, 13846, 13847, 13848, 13849, 13850, - 13851, 13852, 13853, 13854, 13855, 13856, 13857, 13858, 13859, 13860, - 13861, 13862, 13863, 13864, 13865, 13866, 13867, 13868, 13869, 13870, - 13871, 13872, 13873, 13874, 13875, 13876, 13877, 13878, 13879, 13880, - 13881, 13882, 13883, 13884, 13885, 13886, 13887, 13888, 13889, 13890, - 13891, 13892, 13893, 13894, 13895, 13896, 13897, 13898, 13899, 13900, - 13901, 13902, 13903, 13904, 13905, 13906, 13907, 13908, 13909, 13910, - 13911, 13912, 13913, 13914, 13915, 13916, 13917, 13918, 13919, 13920, - 13921, 13922, 13923, 13924, 13925, 13926, 13927, 13928, 13929, 13930, - 13931, 13932, 13933, 13934, 13935, 13936, 13937, 13938, 13939, 13940, - 13941, 13942, 13943, 13944, 13945, 13946, 13947, 13948, 13949, 13950, - 13951, 13952, 13953, 13954, 13955, 13956, 13957, 13958, 13959, 13960, - 13961, 13962, 13963, 13964, 13965, 13966, 13967, 13968, 13969, 13970, - 13971, 13972, 13973, 13974, 13975, 13976, 13977, 13978, 13979, 13980, - 13981, 13982, 13983, 13984, 13985, 13986, 13987, 13988, 13989, 13990, - 13991, 13992, 13993, 13994, 13995, 13996, 13997, 13998, 13999, 14000, - 14001, 14002, 14003, 14004, 14005, 14006, 14007, 14008, 14009, 14010, - 14011, 14012, 14013, 14014, 14015, 14016, 14017, 14018, 14019, 14020, - 14021, 14022, 14023, 14024, 14025, 14026, 14027, 14028, 14029, 14030, - 14031, 14032, 14033, 14034, 14035, 14036, 14037, 14038, 14039, 14040, - 14041, 14042, 14043, 14044, 14045, 14046, 14047, 14048, 14049, 14050, - 14051, 14052, 14053, 14054, 14055, 14056, 14057, 14058, 14059, 14060, - 14061, 14062, 14063, 14064, 14065, 14066, 14067, 14068, 14069, 14070, - 14071, 14072, 14073, 14074, 14075, 14076, 14077, 14078, 14079, 14080, - 14081, 14082, 14083, 14084, 14085, 14086, 14087, 14088, 14089, 14090, - 14091, 14092, 14093, 14094, 14095, 14096, 14097, 14098, 14099, 14100, - 14101, 14102, 14103, 14104, 14105, 14106, 14107, 14108, 14109, 14110, - 14111, 14112, 14113, 14114, 14115, 14116, 14117, 14118, 14119, 14120, - 14121, 14122, 14123, 14124, 14125, 14126, 14127, 14128, 14129, 14130, - 14131, 14132, 14133, 14134, 14135, 14136, 14137, 14138, 14139, 14140, - 14141, 14142, 14143, 14144, 14145, 14146, 14147, 14148, 14149, 14150, - 14151, 14152, 14153, 14154, 14155, 14156, 14157, 14158, 14159, 14160, - 14161, 14162, 14163, 14164, 14165, 14166, 14167, 14168, 14169, 14170, - 14171, 14172, 14173, 14174, 14175, 14176, 14177, 14178, 14179, 14180, - 14181, 14182, 14183, 14184, 14185, 14186, 14187, 14188, 14189, 14190, - 14191, 14192, 14193, 14194, 14195, 14196, 14197, 14198, 14199, 14200, - 14201, 14202, 14203, 14204, 14205, 14206, 14207, 14208, 14209, 14210, - 14211, 14212, 14213, 14214, 14215, 14216, 14217, 14218, 14219, 14220, - 14221, 14222, 14223, 14224, 14225, 14226, 14227, 14228, 14229, 14230, - 14231, 14232, 14233, 14234, 14235, 14236, 14237, 14238, 14239, 14240, - 14241, 14242, 14243, 14244, 14245, 14246, 14247, 14248, 14249, 14250, - 14251, 14252, 14253, 14254, 14255, 14256, 14257, 14258, 14259, 14260, - 14261, 14262, 14263, 14264, 14265, 14266, 14267, 14268, 14269, 14270, - 14271, 14272, 14273, 14274, 14275, 14276, 14277, 14278, 14279, 14280, - 14281, 14282, 14283, 14284, 14285, 14286, 14287, 14288, 14289, 14290, - 14291, 14292, 14293, 14294, 14295, 14296, 14297, 14298, 14299, 14300, - 14301, 14302, 14303, 14304, 14305, 14306, 14307, 14308, 14309, 14310, - 14311, 14312, 14313, 14314, 14315, 14316, 14317, 14318, 14319, 14320, - 14321, 14322, 14323, 14324, 14325, 14326, 14327, 14328, 14329, 14330, - 14331, 14332, 14333, 14334, 14335, 14336, 14337, 14338, 14339, 14340, - 14341, 14342, 14343, 14344, 14345, 14346, 14347, 14348, 14349, 14350, - 14351, 14352, 14353, 14354, 14355, 14356, 14357, 14358, 14359, 14360, - 14361, 14362, 14363, 14364, 14365, 14366, 14367, 14368, 14369, 14370, - 14371, 14372, 14373, 14374, 14375, 14376, 14377, 14378, 14379, 14380, - 14381, 14382, 14383, 14384, 14385, 14386, 14387, 14388, 14389, 14390, - 14391, 14392, 14393, 14394, 14395, 14396, 14397, 14398, 14399, 14400, - 14401, 14402, 14403, 14404, 14405, 14406, 14407, 14408, 14409, 14410, - 14411, 14412, 14413, 14414, 14415, 14416, 14417, 14418, 14419, 14420, - 14421, 14422, 14423, 14424, 14425, 14426, 14427, 14428, 14429, 14430, - 14431, 14432, 14433, 14434, 14435, 14436, 14437, 14438, 14439, 14440, - 14441, 14442, 14443, 14444, 14445, 14446, 14447, 14448, 14449, 14450, - 14451, 14452, 14453, 14454, 14455, 14456, 14457, 14458, 14459, 14460, - 14461, 14462, 14463, 14464, 14465, 14466, 14467, 14468, 14469, 14470, - 14471, 14472, 14473, 14474, 14475, 14476, 14477, 14478, 14479, 14480, - 14481, 14482, 14483, 14484, 14485, 14486, 14487, 14488, 14489, 14490, - 14491, 14492, 14493, 14494, 14495, 14496, 14497, 14498, 14499, 14500, - 14501, 14502, 14503, 14504, 14505, 14506, 14507, 14508, 14509, 14510, - 14511, 14512, 14513, 14514, 14515, 14516, 14517, 14518, 14519, 14520, - 14521, 14522, 14523, 14524, 14525, 14526, 14527, 14528, 14529, 14530, - 14531, 14532, 14533, 14534, 14535, 14536, 14537, 14538, 14539, 14540, - 14541, 14542, 14543, 14544, 14545, 14546, 14547, 14548, 14549, 14550, - 14551, 14552, 14553, 14554, 14555, 14556, 14557, 14558, 14559, 14560, - 14561, 14562, 14563, 14564, 14565, 14566, 14567, 14568, 14569, 14570, - 14571, 14572, 14573, 14574, 14575, 14576, 14577, 14578, 14579, 14580, - 14581, 14582, 14583, 14584, 14585, 14586, 14587, 14588, 14589, 14590, - 14591, 14592, 14593, 14594, 14595, 14596, 14597, 14598, 14599, 14600, - 14601, 14602, 14603, 14604, 14605, 14606, 14607, 14608, 14609, 14610, - 14611, 14612, 14613, 14614, 14615, 14616, 14617, 14618, 14619, 14620, - 14621, 14622, 14623, 14624, 14625, 14626, 14627, 14628, 14629, 14630, - 14631, 14632, 14633, 14634, 14635, 14636, 14637, 14638, 14639, 14640, - 14641, 14642, 14643, 14644, 14645, 14646, 14647, 14648, 14649, 14650, - 14651, 14652, 14653, 14654, 14655, 14656, 14657, 14658, 14659, 14660, - 14661, 14662, 14663, 14664, 14665, 14666, 14667, 14668, 14669, 14670, - 14671, 14672, 14673, 14674, 14675, 14676, 14677, 14678, 14679, 14680, - 14681, 14682, 14683, 14684, 14685, 14686, 14687, 14688, 14689, 14690, - 14691, 14692, 14693, 14694, 14695, 14696, 14697, 14698, 14699, 14700, - 14701, 14702, 14703, 14704, 14705, 14706, 14707, 14708, 14709, 14710, - 14711, 14712, 14713, 14714, 14715, 14716, 14717, 14718, 14719, 14720, - 14721, 14722, 14723, 14724, 14725, 14726, 14727, 14728, 14729, 14730, - 14731, 14732, 14733, 14734, 14735, 14736, 14737, 14738, 14739, 14740, - 14741, 14742, 14743, 14744, 14745, 14746, 14747, 14748, 14749, 14750, - 14751, 14752, 14753, 14754, 14755, 14756, 14757, 14758, 14759, 14760, - 14761, 14762, 14763, 14764, 14765, 14766, 14767, 14768, 14769, 14770, - 14771, 14772, 14773, 14774, 14775, 14776, 14777, 14778, 14779, 14780, - 14781, 14782, 14783, 14784, 14785, 14786, 14787, 14788, 14789, 14790, - 14791, 14792, 14793, 14794, 14795, 14796, 14797, 14798, 14799, 14800, - 14801, 14802, 14803, 14804, 14805, 14806, 14807, 14808, 14809, 14810, - 14811, 14812, 14813, 14814, 14815, 14816, 14817, 14818, 14819, 14820, - 14821, 14822, 14823, 14824, 14825, 14826, 14827, 14828, 14829, 14830, - 14831, 14832, 14833, 14834, 14835, 14836, 14837, 14838, 14839, 14840, - 14841, 14842, 14843, 14844, 14845, 14846, 14847, 14848, 14849, 14850, - 14851, 14852, 14853, 14854, 14855, 14856, 14857, 14858, 14859, 14860, - 14861, 14862, 14863, 14864, 14865, 14866, 14867, 14868, 14869, 14870, - 14871, 14872, 14873, 14874, 14875, 14876, 14877, 14878, 14879, 14880, - 14881, 14882, 14883, 14884, 14885, 14886, 14887, 14888, 14889, 14890, - 14891, 14892, 14893, 14894, 14895, 14896, 14897, 14898, 14899, 14900, - 14901, 14902, 14903, 14904, 14905, 14906, 14907, 14908, 14909, 14910, - 14911, 14912, 14913, 14914, 14915, 14916, 14917, 14918, 14919, 14920, - 14921, 14922, 14923, 14924, 14925, 14926, 14927, 14928, 14929, 14930, - 14931, 14932, 14933, 14934, 14935, 14936, 14937, 14938, 14939, 14940, - 14941, 14942, 14943, 14944, 14945, 14946, 14947, 14948, 14949, 14950, - 14951, 14952, 14953, 14954, 14955, 14956, 14957, 14958, 14959, 14960, - 14961, 14962, 14963, 14964, 14965, 14966, 14967, 14968, 14969, 14970, - 14971, 14972, 14973, 14974, 14975, 14976, 14977, 14978, 14979, 14980, - 14981, 14982, 14983, 14984, 14985, 14986, 14987, 14988, 14989, 14990, - 14991, 14992, 14993, 14994, 14995, 14996, 14997, 14998, 14999, 15000, - 15001, 15002, 15003, 15004, 15005, 15006, 15007, 15008, 15009, 15010, - 15011, 15012, 15013, 15014, 15015, 15016, 15017, 15018, 15019, 15020, - 15021, 15022, 15023, 15024, 15025, 15026, 15027, 15028, 15029, 15030, - 15031, 15032, 15033, 15034, 15035, 15036, 15037, 15038, 15039, 15040, - 15041, 15042, 15043, 15044, 15045, 15046, 15047, 15048, 15049, 15050, - 15051, 15052, 15053, 15054, 15055, 15056, 15057, 15058, 15059, 15060, - 15061, 15062, 15063, 15064, 15065, 15066, 15067, 15068, 15069, 15070, - 15071, 15072, 15073, 15074, 15075, 15076, 15077, 15078, 15079, 15080, - 15081, 15082, 15083, 15084, 15085, 15086, 15087, 15088, 15089, 15090, - 15091, 15092, 15093, 15094, 15095, 15096, 15097, 15098, 15099, 15100, - 15101, 15102, 15103, 15104, 15105, 15106, 15107, 15108, 15109, 15110, - 15111, 15112, 15113, 15114, 15115, 15116, 15117, 15118, 15119, 15120, - 15121, 15122, 15123, 15124, 15125, 15126, 15127, 15128, 15129, 15130, - 15131, 15132, 15133, 15134, 15135, 15136, 15137, 15138, 15139, 15140, - 15141, 15142, 15143, 15144, 15145, 15146, 15147, 15148, 15149, 15150, - 15151, 15152, 15153, 15154, 15155, 15156, 15157, 15158, 15159, 15160, - 15161, 15162, 15163, 15164, 15165, 15166, 15167, 15168, 15169, 15170, - 15171, 15172, 15173, 15174, 15175, 15176, 15177, 15178, 15179, 15180, - 15181, 15182, 15183, 15184, 15185, 15186, 15187, 15188, 15189, 15190, - 15191, 15192, 15193, 15194, 15195, 15196, 15197, 15198, 15199, 15200, - 15201, 15202, 15203, 15204, 15205, 15206, 15207, 15208, 15209, 15210, - 15211, 15212, 15213, 15214, 15215, 15216, 15217, 15218, 15219, 15220, - 15221, 15222, 15223, 15224, 15225, 15226, 15227, 15228, 15229, 15230, - 15231, 15232, 15233, 15234, 15235, 15236, 15237, 15238, 15239, 15240, - 15241, 15242, 15243, 15244, 15245, 15246, 15247, 15248, 15249, 15250, - 15251, 15252, 15253, 15254, 15255, 15256, 15257, 15258, 15259, 15260, - 15261, 15262, 15263, 15264, 15265, 15266, 15267, 15268, 15269, 15270, - 15271, 15272, 15273, 15274, 15275, 15276, 15277, 15278, 15279, 15280, - 15281, 15282, 15283, 15284, 15285, 15286, 15287, 15288, 15289, 15290, - 15291, 15292, 15293, 15294, 15295, 15296, 15297, 15298, 15299, 15300, - 15301, 15302, 15303, 15304, 15305, 15306, 15307, 15308, 15309, 15310, - 15311, 15312, 15313, 15314, 15315, 15316, 15317, 15318, 15319, 15320, - 15321, 15322, 15323, 15324, 15325, 15326, 15327, 15328, 15329, 15330, - 15331, 15332, 15333, 15334, 15335, 15336, 15337, 15338, 15339, 15340, - 15341, 15342, 15343, 15344, 15345, 15346, 15347, 15348, 15349, 15350, - 15351, 15352, 15353, 15354, 15355, 15356, 15357, 15358, 15359, 15360, - 15361, 15362, 15363, 15364, 15365, 15366, 15367, 15368, 15369, 15370, - 15371, 15372, 15373, 15374, 15375, 15376, 15377, 15378, 15379, 15380, - 15381, 15382, 15383, 15384, 15385, 15386, 15387, 15388, 15389, 15390, - 15391, 15392, 15393, 15394, 15395, 15396, 15397, 15398, 15399, 15400, - 15401, 15402, 15403, 15404, 15405, 15406, 15407, 15408, 15409, 15410, - 15411, 15412, 15413, 15414, 15415, 15416, 15417, 15418, 15419, 15420, - 15421, 15422, 15423, 15424, 15425, 15426, 15427, 15428, 15429, 15430, - 15431, 15432, 15433, 15434, 15435, 15436, 15437, 15438, 15439, 15440, - 15441, 15442, 15443, 15444, 15445, 15446, 15447, 15448, 15449, 15450, - 15451, 15452, 15453, 15454, 15455, 15456, 15457, 15458, 15459, 15460, - 15461, 15462, 15463, 15464, 15465, 15466, 15467, 15468, 15469, 15470, - 15471, 15472, 15473, 15474, 15475, 15476, 15477, 15478, 15479, 15480, - 15481, 15482, 15483, 15484, 15485, 15486, 15487, 15488, 15489, 15490, - 15491, 15492, 15493, 15494, 15495, 15496, 15497, 15498, 15499, 15500, - 15501, 15502, 15503, 15504, 15505, 15506, 15507, 15508, 15509, 15510, - 15511, 15512, 15513, 15514, 15515, 15516, 15517, 15518, 15519, 15520, - 15521, 15522, 15523, 15524, 15525, 15526, 15527, 15528, 15529, 15530, - 15531, 15532, 15533, 15534, 15535, 15536, 15537, 15538, 15539, 15540, - 15541, 15542, 15543, 15544, 15545, 15546, 15547, 15548, 15549, 15550, - 15551, 15552, 15553, 15554, 15555, 15556, 15557, 15558, 15559, 15560, - 15561, 15562, 15563, 15564, 15565, 15566, 15567, 15568, 15569, 15570, - 15571, 15572, 15573, 15574, 15575, 15576, 15577, 15578, 15579, 15580, - 15581, 15582, 15583, 15584, 15585, 15586, 15587, 15588, 15589, 15590, - 15591, 15592, 15593, 15594, 15595, 15596, 15597, 15598, 15599, 15600, - 15601, 15602, 15603, 15604, 15605, 15606, 15607, 15608, 15609, 15610, - 15611, 15612, 15613, 15614, 15615, 15616, 15617, 15618, 15619, 15620, - 15621, 15622, 15623, 15624, 15625, 15626, 15627, 15628, 15880, 15881, - 15882, 15883, 15884, 15885, 15886, 15887, 15888, 15889, 15890, 15891, - 15892, 15893, 15894, 15895, 15896, 15897, 15898, 15899, 15900, 15901, - 15902, 15903, 15904, 15905, 15906, 15907, 15908, 15909, 15910, 15911, - 15912, 15913, 15914, 15915, 15916, 15917, 15918, 15919, 15920, 15921, - 15922, 15923, 15924, 15925, 15926, 15927, 15928, 15929, 15930, 15931, - 15932, 15933, 15934, 15935, 15936, 15937, 15938, 15939, 15940, 15941, - 15942, 15943, 15944, 15945, 15946, 15947, 15948, 15949, 15950, 15951, - 15952, 15953, 15954, 15955, 15956, 15957, 15958, 15959, 15960, 15961, - 15962, 15963, 15964, 15965, 15966, 15967, 15968, 15969, 15970, 15971, - 15972, 15973, 15974, 15975, 15976, 15977, 15978, 15979, 15980, 15981, - 15982, 15983, 15984, 15985, 15986, 15987, 15988, 15989, 15990, 15991, - 15992, 15993, 15994, 15995, 15996, 15997, 15998, 15999, 16000, 16001, - 16002, 16003, 16004, 16005, 16006, 16007, 16008, 16009, 16010, 16011, - 16012, 16013, 16014, 16015, 16016, 16017, 16018, 16019, 16020, 16021, - 16022, 16023, 16024, 16025, 16026, 16027, 16028, 16029, 16030, 16031, - 16032, 16033, 16034, 16035, 16036, 16037, 16038, 16039, 16040, 16041, - 16042, 16043, 16044, 16045, 16046, 16047, 16048, 16049, 16050, 16051, - 16052, 16053, 16054, 16055, 16056, 16057, 16058, 16059, 16060, 16061, - 16062, 16063, 16064, 16065, 16066, 16067, 16068, 16069, 16070, 16071, - 16072, 16073, 16074, 16075, 16076, 16077, 16078, 16079, 16080, 16081, - 16082, 16083, 16084, 16085, 16086, 16087, 16088, 16089, 16090, 16091, - 16092, 16093, 16094, 16095, 16096, 16097, 16098, 16099, 16100, 16101, - 16102, 16103, 16104, 16105, 16106, 16107, 16108, 16109, 16110, 16111, - 16112, 16113, 16114, 16115, 16116, 16117, 16118, 16119, 16120, 16121, - 16122, 16123, 16124, 16125, 16126, 16127, 16128, 16129, 16130, 16131, - 16132, 16133, 16134, 16135, 16136, 16137, 16138, 16139, 16140, 16141, - 16142, 16143, 16144, 16145, 16146, 16147, 16148, 16149, 16150, 16151, - 16152, 16153, 16154, 16155, 16156, 16157, 16158, 16159, 16160, 16161, - 16162, 16163, 16164, 16165, 16166, 16167, 16168, 16169, 16170, 16171, - 16172, 16173, 16174, 16175, 16176, 16177, 16178, 16179, 16180, 16181, - 16182, 16183, 16184, 16185, 16186, 16187, 16188, 16189, 16190, 16191, - 16192, 16193, 16194, 16195, 16196, 16197, 16198, 16199, 16200, 16201, - 16202, 16203, 16204, 16205, 16206, 16207, 16208, 16209, 16210, 16211, - 16212, 16213, 16214, 16215, 16216, 16217, 16218, 16219, 16220, 16221, - 16222, 16223, 16224, 16225, 16226, 16227, 16228, 16229, 16230, 16231, - 16232, 16233, 16234, 16235, 16236, 16237, 16238, 16239, 16240, 16241, - 16242, 16243, 16244, 16245, 16246, 16247, 16248, 16249, 16250, 16251, - 16252, 16253, 16254, 16255, 16256, 16257, 16258, 16259, 16260, 16261, - 16262, 16263, 16264, 16265, 16266, 16267, 16268, 16269, 16270, 16271, - 16272, 16273, 16274, 16275, 16276, 16277, 16278, 16279, 16280, 16281, - 16282, 16283, 16284, 16285, 16286, 16287, 16288, 16289, 16290, 16291, - 16292, 16293, 16294, 16295, 16296, 16297, 16298, 16299, 16300, 16301, - 16302, 16303, 16304, 16305, 16306, 16307, 16308, 16309, 16310, 16311, - 16312, 16313, 16314, 16315, 16316, 16317, 16318, 16319, 16320, 16321, - 16322, 16323, 16324, 16325, 16326, 16327, 16328, 16329, 16330, 16331, - 16332, 16333, 16334, 16335, 16336, 16337, 16338, 16339, 16340, 16341, - 16342, 16343, 16344, 16345, 16346, 16347, 16348, 16349, 16350, 16351, - 16352, 16353, 16354, 16355, 16356, 16357, 16358, 16359, 16360, 16361, - 16362, 16363, 16364, 16365, 16366, 16367, 16368, 16369, 16370, 16371, - 16372, 16373, 16374, 16375, 16376, 16377, 16378, 16379, 16380, 16381, - 16382, 16383, 16384, 16385, 16386, 16387, 16388, 16389, 16390, 16391, - 16392, 16393, 16394, 16395, 16396, 16397, 16398, 16399, 16400, 16401, - 16402, 16403, 16404, 16405, 16406, 16407, 16408, 16409, 16410, 16411, - 16412, 16413, 16414, 16415, 16416, 16417, 16418, 16419, 16420, 16421, - 16422, 16423, 16424, 16425, 16426, 16427, 16428, 16429, 16430, 16431, - 16432, 16433, 16434, 16435, 16436, 16437, 16438, 16439, 16440, 16441, - 16442, 16443, 16444, 16445, 16446, 16447, 16448, 16449, 16450, 16451, - 16452, 16453, 16454, 16455, 16456, 16457, 16458, 16459, 16460, 16461, - 16462, 16463, 16464, 16465, 16466, 16467, 16468, 16469, 16470, 16471, - 16472, 16473, 16474, 16475, 16476, 16477, 16478, 16479, 16480, 16481, - 16482, 16483, 16484, 16485, 16486, 16487, 16488, 16489, 16490, 16491, - 16492, 16493, 16494, 16495, 16496, 16497, 16498, 16499, 16500, 16501, - 16502, 16503, 16504, 16505, 16506, 16507, 16508, 16509, 16510, 16511, - 16512, 16513, 16514, 16515, 16516, 16517, 16518, 16519, 16520, 16521, - 16522, 16523, 16524, 16525, 16526, 16527, 16528, 16529, 16530, 16531, - 16532, 16533, 16534, 16535, 16536, 16537, 16538, 16539, 16540, 16541, - 16542, 16543, 16544, 16545, 16546, 16547, 16548, 16549, 16550, 16551, - 16552, 16553, 16554, 16555, 16556, 16557, 16558, 16559, 16560, 16561, - 16562, 16563, 16564, 16565, 16566, 16567, 16568, 16569, 16570, 16571, - 16572, 16573, 16574, 16575, 16576, 16577, 16578, 16579, 16580, 16581, - 16582, 16583, 16584, 16585, 16586, 16587, 16588, 16589, 16590, 16591, - 16592, 16593, 16594, 16595, 16596, 16597, 16598, 16599, 16600, 16601, - 16602, 16603, 16604, 16605, 16606, 16607, 16608, 16609, 16610, 16611, - 16612, 16613, 16614, 16615, 16616, 16617, 16618, 16619, 16620, 16621, - 16622, 16623, 16624, 16625, 16626, 16627, 16628, 16629, 16630, 16631, - 16632, 16633, 16634, 16635, 16636, 16637, 16638, 16639, 16640, 16641, - 16642, 16643, 16644, 16645, 16646, 16647, 15640, 15641, 15642, 15643, - 15644, 15645, 15646, 15647, 15648, 15649, 15650, 15651, 15652, 15653, - 15654, 15655, 15656, 15657, 15658, 15659, 15660, 15661, 15662, 15663, - 15664, 15665, 15666, 15667, 15668, 15669, 15670, 15671, 15672, 15673, - 15674, 15675, 15676, 15677, 15678, 15679, 15680, 15681, 15682, 15683, - 15684, 15685, 15686, 15687, 15688, 15689, 15690, 15691, 15692, 15693, - 15694, 15695, 15696, 15697, 15698, 15699, 15700, 15701, 15702, 15703, - 15704, 15705, 15706, 15707, 15708, 15709, 15710, 15711, 15712, 15713, - 15714, 15715, 15716, 15717, 15718, 15719, 15720, 15721, 15722, 15723, - 15724, 15725, 15726, 15727, 15728, 15729, 15730, 15731, 15732, 15733, - 15734, 15735, 15736, 15737, 15738, 15739, 15740, 15741, 15742, 15743, - 15744, 15745, 15746, 15747, 15748, 15749, 15750, 15751, 15752, 15753, - 15754, 15755, 15756, 15757, 15758, 15759, 15760, 15761, 15762, 15763, - 15764, 15765, 15766, 15767, 15768, 15769, 15770, 15771, 15772, 15773, - 15774, 15775, 15776, 15777, 15778, 15779, 15780, 15781, 15782, 15783, - 15784, 15785, 15786, 15787, 15788, 15789, 15790, 15791, 15792, 15793, - 15794, 15795, 15796, 15797, 15798, 15799, 15800, 15801, 15802, 15803, - 15804, 15805, 15806, 15807, 15808, 15809, 15810, 15811, 15812, 15813, - 15814, 15815, 15816, 15817, 15818, 15819, 15820, 15821, 15822, 15823, - 15824, 15825, 15826, 15827, 15828, 15829, 15830, 15831, 15832, 15833, - 15834, 15835, 15836, 15837, 15838, 15839, 15840, 15841, 15842, 15843, - 15844, 15845, 15846, 15847, 15848, 15849, 15850, 15851, 15852, 15853, - 15854, 15855, 15856, 15857, 15858, 15859, 15860, 15861, 15862, 15863, - 15864, 15865, 15866, 15867, 15868, 15869, 15870, 15871, 15872, 15873, - 15874, 15875, 15876, 15877, 15878, 15879, 15629, 15630, 15631, 15632, - 15633, 15634, 15635, 15636, 15637, 15638, 15639, 40951, 40951, 40951, - 40951, 40951, 445, 446, 447, 448, 449, 450, 451, 452, 453, 434, 435, 436, - 437, 438, 439, 440, 441, 442, 443, 444, 375, 376, 377, 378, 379, 380, - 373, 374, 381, 382, 383, 425, 426, 427, 428, 429, 430, 431, 432, 433, - 423, 424, 391, 387, 388, 392, 393, 394, 389, 390, 384, 385, 386, 395, - 396, 397, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 402, 403, - 404, 405, 406, 407, 398, 399, 400, 401, 408, 409, 410, 464, 465, 466, - 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, - 481, 482, 483, 415, 416, 417, 418, 419, 420, 421, 411, 412, 413, 414, - 422, 495, 496, 497, 498, 499, 500, 501, 484, 485, 486, 487, 492, 493, - 494, 502, 488, 489, 490, 491, 503, 504, 505, 506, 507, 510, 511, 512, - 513, 508, 509, 514, 515, 516, 517, 520, 521, 522, 523, 524, 518, 519, - 525, 526, 527, 528, 531, 532, 533, 534, 535, 529, 530, 536, 537, 538, - 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, - 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, - 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, - 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, - 595, 596, 597, 598, 599, 607, 608, 600, 601, 602, 609, 610, 611, 612, - 603, 604, 613, 605, 606, 618, 619, 620, 621, 622, 614, 615, 616, 617, - 623, 624, 625, 651, 652, 653, 654, 655, 656, 657, 649, 650, 658, 659, - 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, - 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, - 699, 700, 662, 663, 664, 665, 666, 667, 668, 660, 661, 669, 670, 701, - 702, 703, 704, 705, 706, 707, 708, 709, 710, 640, 641, 642, 643, 644, - 645, 646, 647, 648, 638, 639, 630, 631, 632, 633, 626, 627, 634, 635, - 636, 637, 628, 629, 713, 714, 715, 716, 717, 718, 719, 720, 721, 711, - 712, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 724, 725, 726, - 727, 728, 729, 730, 731, 732, 722, 723, 751, 752, 748, 749, 750, 753, - 754, 755, 744, 745, 746, 747, 756, 757, 758, 815, 816, 817, 818, 819, - 820, 821, 822, 823, 824, 735, 736, 737, 738, 739, 740, 741, 742, 743, - 733, 734, 763, 764, 765, 766, 759, 760, 767, 768, 769, 761, 762, 770, - 796, 794, 795, 797, 798, 799, 800, 801, 802, 803, 804, 777, 773, 774, - 778, 771, 772, 779, 780, 775, 776, 781, 782, 783, 785, 786, 787, 784, - 788, 789, 790, 791, 792, 793, 856, 857, 858, 859, 860, 861, 862, 863, - 864, 865, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 866, - 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, - 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, - 895, 836, 837, 840, 841, 842, 843, 844, 845, 838, 839, 846, 847, 896, - 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, - 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, - 925, 848, 849, 850, 851, 852, 853, 854, 855, 927, 928, 929, 930, 931, - 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, - 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 926, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 18950, 18940, 18939, 18936, 18935, 18921, 18934, - 18933, 18938, 18937, 18943, 18928, 18927, 18924, 18923, 18948, 18930, - 18929, 18926, 18925, 18922, 18942, 18941, 18932, 18931, 18945, 18949, - 18946, 18944, 18947, 18953, 18960, 18961, 18956, 18957, 18962, 18963, - 18954, 18958, 18959, 18955, 18964, 18920, 18919, 18917, 18951, 18918, - 18952, 18971, 18973, 18970, 18969, 18966, 18965, 18968, 18967, 18974, - 18972, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 2847, - 2813, 2872, 2873, 2843, 2880, 2891, 2862, 2879, 2874, 2875, 2826, 2892, - 2849, 2828, 2833, 2840, 2886, 2858, 2816, 2852, 2887, 2848, 2823, 2824, - 2856, 2830, 2871, 2812, 2870, 2839, 2863, 2894, 2811, 2857, 2888, 2842, - 2841, 2837, 2814, 2869, 2846, 2876, 2829, 2866, 2877, 2893, 2821, 2883, - 2890, 2831, 2817, 2845, 2819, 2838, 2881, 2822, 2896, 2898, 2818, 2884, - 2834, 2878, 2855, 2882, 2860, 2867, 2851, 2895, 2850, 2832, 2864, 2835, - 2861, 2889, 2885, 2868, 2853, 2825, 2859, 2815, 2854, 2897, 2820, 2865, - 2844, 2836, 2930, 2946, 2944, 2943, 2909, 2915, 2904, 2950, 2914, 2906, - 2936, 2949, 2908, 2934, 2935, 2899, 2939, 2947, 2941, 2942, 2921, 2918, - 2933, 2902, 2900, 2901, 2907, 2937, 2954, 2928, 2926, 2951, 2940, 2948, - 2920, 2924, 2925, 2922, 2945, 2917, 2923, 2932, 2938, 2919, 2952, 2916, - 2953, 2903, 2913, 2912, 2910, 2927, 2931, 2911, 2905, 2929, 3006, 3026, - 3047, 3043, 3005, 2994, 3007, 2955, 2983, 2957, 3027, 3023, 2980, 3028, - 2998, 2977, 2960, 2956, 2962, 3046, 3025, 2988, 3018, 2986, 3048, 2967, - 2968, 3032, 2999, 3036, 3014, 3040, 3035, 3001, 3042, 2992, 2974, 3024, - 3002, 2970, 2985, 2990, 3021, 3037, 3004, 3051, 2995, 3020, 3017, 3050, - 3041, 2982, 3052, 3010, 2969, 3039, 3015, 3012, 2964, 3000, 2976, 2987, - 2972, 2966, 3011, 3009, 3044, 3003, 3019, 3022, 2965, 3016, 2996, 2973, - 3049, 2997, 3033, 3031, 2984, 2975, 2979, 2961, 2981, 2963, 2978, 3045, - 3013, 2991, 2989, 3034, 2959, 2958, 3008, 2993, 2971, 3029, 3030, 3038, - 3080, 3164, 3113, 3087, 3114, 3073, 3112, 3118, 3100, 3127, 3163, 3109, - 3143, 3110, 3057, 3156, 3141, 3116, 3148, 3061, 3165, 3063, 3150, 3085, - 3095, 3076, 3082, 3152, 3169, 3111, 3058, 3106, 3158, 3056, 3108, 3053, - 3096, 3090, 3068, 3097, 3092, 3091, 3133, 3089, 3086, 3074, 3119, 3078, - 3066, 3125, 3154, 3153, 3166, 3059, 3140, 3157, 3105, 3084, 3120, 3060, - 3136, 3131, 3126, 3071, 3099, 3088, 3103, 3167, 3135, 3168, 3098, 3122, - 3147, 3064, 3102, 3107, 3159, 3081, 3065, 3121, 3155, 3077, 3101, 3069, - 3104, 3117, 3115, 3055, 3062, 3137, 3161, 3160, 3128, 3139, 3070, 3083, - 3075, 3149, 3094, 3145, 3142, 3067, 3130, 3146, 3123, 3132, 3129, 3144, - 3134, 3093, 3072, 3138, 3162, 3124, 3079, 3151, 3054, 3223, 3300, 3210, - 3197, 3307, 3200, 3263, 3289, 3279, 3249, 3226, 3274, 3294, 3235, 3190, - 3310, 3282, 3228, 3264, 3299, 3192, 3202, 3251, 3241, 3207, 3238, 3242, - 3250, 3281, 3248, 3267, 3290, 3231, 3214, 3184, 3237, 3245, 3208, 3201, - 3229, 3225, 3291, 3283, 3277, 3222, 3230, 3315, 3183, 3304, 3311, 3271, - 3303, 3187, 3288, 3297, 3305, 3308, 3196, 3273, 3293, 3181, 3243, 3284, - 3213, 3211, 3256, 3260, 3180, 3302, 3191, 3323, 3254, 3316, 3212, 3224, - 3269, 3319, 3199, 3173, 3179, 3240, 3189, 3206, 3236, 3185, 3175, 3252, - 3265, 3313, 3227, 3255, 3257, 3272, 3216, 3174, 3259, 3217, 3182, 3172, - 3220, 3275, 3239, 3193, 3270, 3253, 3312, 3232, 3261, 3171, 3178, 3246, - 3324, 3301, 3326, 3325, 3198, 3262, 3292, 3295, 3221, 3287, 3314, 3233, - 3320, 3317, 3318, 3322, 3321, 3188, 3266, 3247, 3276, 3309, 3177, 3306, - 3204, 3215, 3280, 3278, 3234, 3244, 3285, 3286, 3170, 3258, 3268, 3203, - 3195, 3218, 3205, 3209, 3296, 3194, 3219, 3298, 3176, 3186, 3331, 3380, - 3333, 3378, 3358, 3371, 3352, 3335, 3361, 3360, 3340, 3370, 3350, 3346, - 3337, 3368, 3363, 3369, 3366, 3329, 3328, 3348, 3347, 3345, 3373, 3365, - 3374, 3349, 3354, 3351, 3375, 3355, 3362, 3353, 3357, 3327, 3343, 3344, - 3364, 3356, 3379, 3376, 3336, 3334, 3332, 3338, 3359, 3341, 3342, 3339, - 3372, 3330, 3367, 3377, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 28219, 28211, 28232, 28209, 28233, 28220, 28235, 28215, 28206, 28223, - 28221, 28231, 28205, 28213, 28208, 28210, 28216, 28214, 28212, 28225, - 28228, 28217, 28227, 28234, 28226, 28207, 28230, 28229, 28222, 28224, - 28218, 40951, 28244, 28246, 28243, 28242, 28239, 28238, 28241, 28240, - 28247, 28245, 40951, 40951, 40951, 40951, 28237, 28236, 35842, 35839, - 35840, 35841, 35794, 35791, 35792, 35793, 35846, 35843, 35844, 35845, - 35834, 35831, 35832, 35833, 35838, 35835, 35836, 35837, 35830, 35827, - 35828, 35829, 35790, 35787, 35788, 35789, 35822, 35819, 35820, 35821, - 35795, 35800, 35811, 35812, 35823, 35826, 35824, 35825, 35818, 35815, - 35816, 35817, 35806, 35803, 35804, 35805, 35857, 35856, 35855, 35807, - 35814, 35864, 35862, 35859, 35809, 35858, 35860, 35802, 35810, 35799, - 35801, 35798, 35849, 35853, 35861, 35808, 35813, 35851, 35848, 35854, - 35797, 35847, 35863, 35796, 35852, 35850, 35865, 40951, 35872, 35874, - 35871, 35870, 35867, 35866, 35869, 35868, 35875, 35873, 40951, 40951, - 40951, 40951, 40951, 40951, 3441, 3446, 3462, 3464, 3454, 3452, 3444, - 3438, 3445, 3458, 3453, 3449, 3460, 3443, 3439, 3461, 3448, 3459, 3463, - 3457, 3451, 3465, 3450, 3466, 3455, 3456, 3447, 3442, 3440, 3467, 40951, - 40951, 3434, 3436, 3437, 3435, 3433, 3468, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 30994, 30995, 31000, 31001, - 30988, 30989, 31004, 31005, 30996, 30997, 30986, 30987, 31006, 31007, - 30990, 30991, 31002, 31003, 31008, 31009, 30998, 30999, 30992, 30993, - 31010, 31011, 30984, 30985, 30930, 30921, 30927, 30918, 30923, 30929, - 30922, 30926, 30932, 30916, 30928, 30914, 30919, 30917, 30925, 30920, - 30924, 30933, 30931, 30915, 30938, 30937, 30935, 30934, 30936, 30940, - 30939, 30970, 30971, 30949, 30969, 30967, 30975, 30977, 30976, 30978, - 30968, 30960, 30973, 30958, 30980, 30953, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 31018, 31020, 31017, 31016, - 31013, 31012, 31015, 31014, 31021, 31019, 40951, 30945, 30942, 30944, - 30947, 30941, 30943, 30946, 40951, 30972, 30979, 30957, 30965, 30981, - 30956, 30963, 30974, 30962, 30983, 30964, 30959, 30966, 30982, 30961, - 30955, 30948, 30951, 30952, 30954, 30950, 40951, 40951, 40951, 40951, - 40951, 30902, 30909, 30901, 30900, 30910, 30898, 30895, 30913, 30905, - 30903, 30911, 30897, 30896, 30906, 30912, 30908, 30904, 30899, 30907, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 22525, 22522, 22527, 22521, - 22510, 22509, 22506, 22505, 22498, 22504, 22503, 22508, 22507, 22499, - 22495, 22494, 22491, 22490, 22497, 22496, 22493, 22492, 22500, 22512, - 22511, 22502, 22501, 22517, 22520, 22518, 22516, 22519, 22514, 22513, - 22515, 22528, 22534, 22531, 22532, 22533, 22529, 22535, 22530, 22526, - 22524, 22523, 22537, 22536, 22544, 22546, 22543, 22542, 22539, 22538, - 22541, 22540, 22547, 22545, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 26886, 26890, 26893, 26894, 26864, 26896, 26872, 26887, - 26891, 26882, 26881, 26883, 26871, 26863, 26884, 26880, 26877, 26878, - 26892, 26874, 26885, 26888, 26870, 26868, 26895, 26879, 26876, 26866, - 26889, 26875, 26865, 26873, 26944, 26948, 26951, 26952, 26922, 26954, - 26930, 26945, 26949, 26940, 26939, 26941, 26929, 26921, 26942, 26938, - 26935, 26936, 26950, 26932, 26943, 26946, 26928, 26926, 26953, 26937, - 26934, 26924, 26947, 26933, 26923, 26931, 26908, 26902, 26900, 26898, - 26905, 26904, 26907, 26906, 26910, 26909, 26913, 26915, 26912, 26911, - 26916, 26917, 26919, 26920, 26914, 26918, 26903, 26901, 26899, 26897, - 26957, 26955, 26956, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 27502, 27428, 27490, 27501, 27506, 27505, - 27425, 27507, 27482, 27481, 27479, 27436, 27485, 27486, 27478, 27435, - 27444, 27452, 27488, 27424, 27449, 27448, 27443, 27442, 27441, 27440, - 27466, 27434, 27465, 27433, 27508, 27439, 27489, 27500, 27499, 27447, - 27446, 27423, 27504, 27510, 27438, 27437, 27475, 27431, 27451, 27450, - 27474, 27430, 27483, 27487, 27459, 27462, 27463, 27497, 27495, 27476, - 27432, 27484, 27464, 27498, 27496, 27494, 27492, 27422, 27493, 27491, - 27509, 27426, 27503, 27427, 27461, 27429, 27480, 27477, 27460, 40951, - 40951, 40951, 40951, 27514, 27445, 27513, 27512, 27511, 27519, 27525, - 27524, 27520, 27521, 27546, 27550, 27567, 27566, 27528, 27531, 27532, - 27548, 27535, 27536, 27537, 27538, 27539, 27542, 27544, 27545, 27541, - 27552, 27553, 27554, 27555, 27560, 27556, 27557, 27561, 27563, 27522, - 27523, 27530, 27565, 27529, 27564, 27526, 27534, 27527, 27551, 27568, - 27569, 27558, 27562, 27549, 27547, 27570, 27543, 27533, 27540, 27559, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27518, 27516, 27517, - 27515, 27467, 27468, 27469, 27470, 27471, 27472, 27473, 27453, 27454, - 27455, 27456, 27457, 27458, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 36644, 29851, - 30073, 30072, 22168, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 38877, 38876, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 35876, 35877, 35878, 35879, 35880, 35881, 35882, 35883, 35884, 35885, - 35886, 35887, 35888, 35889, 35890, 35891, 35892, 35893, 35894, 35895, - 35896, 35897, 35898, 35899, 35900, 35901, 35902, 35903, 35904, 35905, - 35906, 35907, 35908, 35909, 35910, 35911, 35912, 35913, 35914, 35915, - 35916, 35917, 35918, 35919, 35920, 35921, 35922, 35923, 35924, 35925, - 35926, 35927, 35928, 35929, 35930, 35931, 35932, 35933, 35934, 35935, - 35936, 35937, 35938, 35939, 35940, 35941, 35942, 35943, 35944, 35945, - 35946, 35947, 35948, 35949, 35950, 35951, 35952, 35953, 35954, 35955, - 35956, 35957, 35958, 35959, 35960, 35961, 35962, 35963, 35964, 35965, - 35966, 35967, 35968, 35969, 35970, 35971, 35972, 35973, 35974, 35975, - 35976, 35977, 35978, 35979, 35980, 35981, 35982, 35983, 35984, 35985, - 35986, 35987, 35988, 35989, 35990, 35991, 35992, 35993, 35994, 35995, - 35996, 35997, 35998, 35999, 36000, 36001, 36002, 36003, 36004, 36005, - 36006, 36007, 36008, 36009, 36010, 36011, 36012, 36013, 36014, 36015, - 36016, 36017, 36018, 36019, 36020, 36021, 36022, 36023, 36024, 36025, - 36026, 36027, 36028, 36029, 36030, 36031, 36032, 36033, 36034, 36035, - 36036, 36037, 36038, 36039, 36040, 36041, 36042, 36043, 36044, 36045, - 36046, 36047, 36048, 36049, 36050, 36051, 36052, 36053, 36054, 36055, - 36056, 36057, 36058, 36059, 36060, 36061, 36062, 36063, 36064, 36065, - 36066, 36067, 36068, 36069, 36070, 36071, 36072, 36073, 36074, 36075, - 36076, 36077, 36078, 36079, 36080, 36081, 36082, 36083, 36084, 36085, - 36086, 36087, 36088, 36089, 36090, 36091, 36092, 36093, 36094, 36095, - 36096, 36097, 36098, 36099, 36100, 36101, 36102, 36103, 36104, 36105, - 36106, 36107, 36108, 36109, 36110, 36111, 36112, 36113, 36114, 36115, - 36116, 36117, 36118, 36119, 36120, 36121, 36122, 36123, 36124, 36125, - 36126, 36127, 36128, 36129, 36130, 36131, 36132, 36133, 36134, 36135, - 36136, 36137, 36138, 36139, 36140, 36141, 36142, 36143, 36144, 36145, - 36146, 36147, 36148, 36149, 36150, 36151, 36152, 36153, 36154, 36155, - 36156, 36157, 36158, 36159, 36160, 36161, 36162, 36163, 36164, 36165, - 36166, 36167, 36168, 36169, 36170, 36171, 36172, 36173, 36174, 36175, - 36176, 36177, 36178, 36179, 36180, 36181, 36182, 36183, 36184, 36185, - 36186, 36187, 36188, 36189, 36190, 36191, 36192, 36193, 36194, 36195, - 36196, 36197, 36198, 36199, 36200, 36201, 36202, 36203, 36204, 36205, - 36206, 36207, 36208, 36209, 36210, 36211, 36212, 36213, 36214, 36215, - 36216, 36217, 36218, 36219, 36220, 36221, 36222, 36223, 36224, 36225, - 36226, 36227, 36228, 36229, 36230, 36231, 36232, 36233, 36234, 36235, - 36236, 36237, 36238, 36239, 36240, 36241, 36242, 36243, 36244, 36245, - 36246, 36247, 36248, 36249, 36250, 36251, 36252, 36253, 36254, 36255, - 36256, 36257, 36258, 36259, 36260, 36261, 36262, 36263, 36264, 36265, - 36266, 36267, 36268, 36269, 36270, 36271, 36272, 36273, 36274, 36275, - 36276, 36277, 36278, 36279, 36280, 36281, 36282, 36283, 36284, 36285, - 36286, 36287, 36288, 36289, 36290, 36291, 36292, 36293, 36294, 36295, - 36296, 36297, 36298, 36299, 36300, 36301, 36302, 36303, 36304, 36305, - 36306, 36307, 36308, 36309, 36310, 36311, 36312, 36313, 36314, 36315, - 36316, 36317, 36318, 36319, 36320, 36321, 36322, 36323, 36324, 36325, - 36326, 36327, 36328, 36329, 36330, 36331, 36332, 36333, 36334, 36335, - 36336, 36337, 36338, 36339, 36340, 36341, 36342, 36343, 36344, 36345, - 36346, 36347, 36348, 36349, 36350, 36351, 36352, 36353, 36354, 36355, - 36356, 36357, 36358, 36359, 36360, 36361, 36362, 36363, 36364, 36365, - 36366, 36367, 36368, 36369, 36370, 36371, 36372, 36373, 36374, 36375, - 36376, 36377, 36378, 36379, 36380, 36381, 36382, 36383, 36384, 36385, - 36386, 36387, 36388, 36389, 36390, 36391, 36392, 36393, 36394, 36395, - 36396, 36397, 36398, 36399, 36400, 36401, 36402, 36403, 36404, 36405, - 36406, 36407, 36408, 36409, 36410, 36411, 36412, 36413, 36414, 36415, - 36416, 36417, 36418, 36419, 36420, 36421, 36422, 36423, 36424, 36425, - 36426, 36427, 36428, 36429, 36430, 36431, 36432, 36433, 36434, 36435, - 36436, 36437, 36438, 36439, 36440, 36441, 36442, 36443, 36444, 36445, - 36446, 36447, 36448, 36449, 36450, 36451, 36452, 36453, 36454, 36455, - 36456, 36457, 36458, 36459, 36460, 36461, 36462, 36463, 36464, 36465, - 36466, 36467, 36468, 36469, 36470, 36471, 36472, 36473, 36474, 36475, - 36476, 36477, 36478, 36479, 36480, 36481, 36482, 36483, 36484, 36485, - 36486, 36487, 36488, 36489, 36490, 36491, 36492, 36493, 36494, 36495, - 36496, 36497, 36498, 36499, 36500, 36501, 36502, 36503, 36504, 36505, - 36506, 36507, 36508, 36509, 36510, 36511, 36512, 36513, 36514, 36515, - 36516, 36517, 36518, 36519, 36520, 36521, 36522, 36523, 36524, 36525, - 36526, 36527, 36528, 36529, 36530, 36531, 36532, 36533, 36534, 36535, - 36536, 36537, 36538, 36539, 36540, 36541, 36542, 36543, 36544, 36545, - 36546, 36547, 36548, 36549, 36550, 36551, 36552, 36553, 36554, 36555, - 36556, 36557, 36558, 36559, 36560, 36561, 36562, 36563, 36564, 36565, - 36566, 36567, 36568, 36569, 36570, 36571, 36572, 36573, 36574, 36575, - 36576, 36577, 36578, 36579, 36580, 36581, 36582, 36583, 36584, 36585, - 36586, 36587, 36588, 36589, 36590, 36591, 36592, 36593, 36594, 36595, - 36596, 36597, 36598, 36599, 36600, 36601, 36602, 36603, 36604, 36605, - 36606, 36607, 36608, 36609, 36610, 36611, 36612, 36613, 36614, 36615, - 36616, 36617, 36618, 36619, 36620, 36621, 36622, 36623, 36624, 36625, - 36626, 36627, 36628, 36629, 36630, 36631, 36632, 36633, 36634, 36635, - 36636, 36637, 36638, 36639, 36640, 36641, 36642, 36643, 21697, 21698, - 21699, 21700, 21701, 21702, 21703, 21704, 21705, 21706, 21707, 21708, - 21709, 21710, 21711, 21712, 21713, 21714, 21715, 21716, 21717, 21718, - 21719, 21720, 21721, 21722, 21723, 21724, 21725, 21726, 21727, 21728, - 21729, 21730, 21731, 21732, 21733, 21734, 21735, 21736, 21737, 21738, - 21739, 21740, 21741, 21742, 21743, 21744, 21745, 21746, 21747, 21748, - 21749, 21750, 21751, 21752, 21753, 21754, 21755, 21756, 21757, 21758, - 21759, 21760, 21761, 21762, 21763, 21764, 21765, 21766, 21767, 21768, - 21769, 21770, 21771, 21772, 21773, 21774, 21775, 21776, 21777, 21778, - 21779, 21780, 21781, 21782, 21783, 21784, 21785, 21786, 21787, 21788, - 21789, 21790, 21791, 21792, 21793, 21794, 21795, 21796, 21797, 21798, - 21799, 21800, 21801, 21802, 21803, 21804, 21805, 21806, 21807, 21808, - 21809, 21810, 21811, 21812, 21813, 21814, 21815, 21816, 21817, 21818, - 21819, 21820, 21821, 21822, 21823, 21824, 21825, 21826, 21827, 21828, - 21829, 21830, 21831, 21832, 21833, 21834, 21835, 21836, 21837, 21838, - 21839, 21840, 21841, 21842, 21843, 21844, 21845, 21846, 21847, 21848, - 21849, 21850, 21851, 21852, 21853, 21854, 21855, 21856, 21857, 21858, - 21859, 21860, 21861, 21862, 21863, 21864, 21865, 21866, 21867, 21868, - 21869, 21870, 21871, 21872, 21873, 21874, 21875, 21876, 21877, 21878, - 21879, 21880, 21881, 21882, 21883, 21884, 21885, 21886, 21887, 21888, - 21889, 21890, 21891, 21892, 21893, 21894, 21895, 21896, 21897, 21898, - 21899, 21900, 21901, 21902, 21903, 21904, 21905, 21906, 21907, 21908, - 21909, 21910, 21911, 21912, 21913, 21914, 21915, 21916, 21917, 21918, - 21919, 21920, 21921, 21922, 21923, 21924, 21925, 21926, 21927, 21928, - 21929, 21930, 21931, 21932, 21933, 21934, 21935, 21936, 21937, 21938, - 21939, 21940, 21941, 21942, 21943, 21944, 21945, 21946, 21947, 21948, - 21949, 21950, 21951, 21952, 21959, 21960, 21961, 21962, 21963, 21964, - 21965, 21966, 21967, 21968, 21969, 21970, 21971, 21972, 21973, 21974, - 21975, 21976, 21977, 21978, 21979, 21980, 21981, 21982, 21983, 21984, - 21985, 21986, 21987, 21988, 21989, 21990, 21991, 21992, 21993, 21994, - 21995, 21996, 21997, 21998, 21999, 22000, 22001, 22002, 22003, 22004, - 22005, 22006, 22007, 22008, 22009, 22010, 22011, 22012, 22013, 22014, - 22015, 22016, 22017, 22018, 22019, 22020, 22021, 22022, 22023, 22024, - 22025, 22026, 22027, 22028, 22029, 22030, 22031, 22032, 22033, 22034, - 22035, 22036, 22037, 22038, 22039, 22040, 22041, 22042, 22043, 22044, - 22045, 22046, 22047, 22048, 22049, 22050, 22051, 22052, 22053, 22054, - 22055, 22056, 22057, 22058, 22059, 22060, 22061, 22062, 22063, 22064, - 22065, 22066, 22067, 22068, 22069, 22070, 22071, 22072, 22073, 22074, - 22075, 22076, 22077, 22078, 22079, 22080, 22081, 22082, 22083, 22084, - 22085, 22086, 22087, 22088, 22089, 22090, 22091, 22092, 22093, 22094, - 22095, 22096, 22097, 22098, 22099, 22100, 22101, 22102, 22103, 22104, - 22105, 22106, 22107, 22108, 22109, 22110, 22111, 22112, 22113, 22114, - 22115, 22116, 22117, 22118, 22119, 22120, 22121, 22122, 22123, 22124, - 22125, 22126, 22127, 22128, 22129, 22130, 22131, 22132, 22133, 22134, - 22135, 22136, 22137, 22138, 22139, 22140, 22141, 22142, 22143, 22144, - 22145, 22146, 22147, 22148, 22149, 22150, 22151, 22152, 22153, 22154, - 22155, 22156, 22157, 22158, 22159, 22160, 22161, 22162, 22163, 22164, - 22165, 22166, 21953, 21954, 21955, 21956, 21957, 21958, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 22167, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 21336, 21337, 21338, 21339, 40951, 21340, 21341, 21329, 21330, 21331, - 21332, 21333, 40951, 21334, 21335, 40951, 21317, 20293, 19932, 19933, - 19934, 19931, 20213, 20214, 20215, 20216, 20205, 20206, 20207, 20208, - 20209, 20200, 20201, 20202, 20203, 20204, 20210, 20211, 20212, 19971, - 19975, 19976, 19977, 19978, 19979, 19980, 19981, 19982, 19972, 19973, - 19974, 19987, 19988, 19989, 19990, 19991, 19992, 19993, 19994, 20001, - 20002, 20003, 20004, 20005, 20006, 20007, 19995, 19996, 19997, 19998, - 19999, 20000, 19984, 19985, 19986, 19983, 20096, 20097, 20098, 20099, - 20100, 20101, 20102, 20103, 20112, 20113, 20114, 20115, 20116, 20117, - 20104, 20105, 20106, 20107, 20108, 20109, 20110, 20111, 20118, 20119, - 20120, 20121, 20122, 20123, 20124, 20125, 20126, 20127, 20128, 20129, - 20158, 20159, 20160, 20161, 20151, 20152, 20153, 20154, 20155, 20156, - 20157, 20147, 20148, 20149, 20150, 20146, 20130, 20131, 20132, 20133, - 20134, 20135, 20136, 20137, 20138, 20140, 20141, 20142, 20143, 20144, - 20145, 20139, 20050, 20051, 20052, 20053, 20054, 20055, 20056, 20057, - 20058, 20043, 20044, 20045, 20046, 20047, 20048, 20049, 20042, 20064, - 20065, 20066, 20036, 20037, 20038, 20039, 20040, 20041, 20035, 20059, - 20060, 20061, 20062, 20063, 19943, 19946, 19947, 19948, 19949, 19950, - 19951, 19952, 19953, 19944, 19945, 19964, 19965, 19966, 19967, 19968, - 19969, 19970, 19954, 19955, 19956, 19957, 19958, 19959, 19960, 19961, - 19962, 19963, 19935, 19936, 19937, 19938, 19939, 19940, 19941, 19942, - 20017, 20018, 20019, 20020, 20021, 20022, 20023, 20024, 20025, 20026, - 20027, 20028, 20029, 20030, 20031, 20032, 20033, 20034, 20009, 20010, - 20008, 20011, 20012, 20013, 20014, 20015, 20016, 20184, 20185, 20186, - 20187, 20188, 20183, 20195, 20196, 20197, 20198, 20189, 20190, 20191, - 20192, 20193, 20194, 20088, 20089, 20090, 20091, 20081, 20082, 20083, - 20084, 20085, 20086, 20087, 20075, 20076, 20077, 20078, 20079, 20080, - 20092, 20093, 20094, 20095, 20069, 20070, 20071, 20072, 20073, 20074, - 20162, 20163, 20164, 20165, 20166, 20167, 20168, 20169, 20170, 20171, - 20179, 20180, 20181, 20182, 20172, 20173, 20174, 20175, 20176, 20177, - 20178, 20067, 20068, 20292, 21315, 21314, 21316, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 20303, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 20296, 20295, 20297, 40951, 40951, 21364, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 21370, 21369, 21371, 21375, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 29455, 29456, 29457, 29458, 29459, 29460, - 29461, 29462, 29463, 29464, 29465, 29466, 29467, 29468, 29469, 29470, - 29471, 29472, 29473, 29474, 29475, 29476, 29477, 29478, 29479, 29480, - 29481, 29482, 29483, 29484, 29485, 29486, 29487, 29488, 29489, 29490, - 29491, 29492, 29493, 29494, 29495, 29496, 29497, 29498, 29499, 29500, - 29501, 29502, 29503, 29504, 29505, 29506, 29507, 29508, 29509, 29510, - 29511, 29512, 29513, 29514, 29515, 29516, 29517, 29518, 29519, 29520, - 29521, 29522, 29523, 29524, 29525, 29526, 29527, 29528, 29529, 29530, - 29531, 29532, 29533, 29534, 29535, 29536, 29537, 29538, 29539, 29540, - 29541, 29542, 29543, 29544, 29545, 29546, 29547, 29548, 29549, 29550, - 29551, 29552, 29553, 29554, 29555, 29556, 29557, 29558, 29559, 29560, - 29561, 29562, 29563, 29564, 29565, 29566, 29567, 29568, 29569, 29570, - 29571, 29572, 29573, 29574, 29575, 29576, 29577, 29578, 29579, 29580, - 29581, 29582, 29583, 29584, 29585, 29586, 29587, 29588, 29589, 29590, - 29591, 29592, 29593, 29594, 29595, 29596, 29597, 29598, 29599, 29600, - 29601, 29602, 29603, 29604, 29605, 29606, 29607, 29608, 29609, 29610, - 29611, 29612, 29613, 29614, 29615, 29616, 29617, 29618, 29619, 29620, - 29621, 29622, 29623, 29624, 29625, 29626, 29627, 29628, 29629, 29630, - 29631, 29632, 29633, 29634, 29635, 29636, 29637, 29638, 29639, 29640, - 29641, 29642, 29643, 29644, 29645, 29646, 29647, 29648, 29649, 29650, - 29651, 29652, 29653, 29654, 29655, 29656, 29657, 29658, 29659, 29660, - 29661, 29662, 29663, 29664, 29665, 29666, 29667, 29668, 29669, 29670, - 29671, 29672, 29673, 29674, 29675, 29676, 29677, 29678, 29679, 29680, - 29681, 29682, 29683, 29684, 29685, 29686, 29687, 29688, 29689, 29690, - 29691, 29692, 29693, 29694, 29695, 29696, 29697, 29698, 29699, 29700, - 29701, 29702, 29703, 29704, 29705, 29706, 29707, 29708, 29709, 29710, - 29711, 29712, 29713, 29714, 29715, 29716, 29717, 29718, 29719, 29720, - 29721, 29722, 29723, 29724, 29725, 29726, 29727, 29728, 29729, 29730, - 29731, 29732, 29733, 29734, 29735, 29736, 29737, 29738, 29739, 29740, - 29741, 29742, 29743, 29744, 29745, 29746, 29747, 29748, 29749, 29750, - 29751, 29752, 29753, 29754, 29755, 29756, 29757, 29758, 29759, 29760, - 29761, 29762, 29763, 29764, 29765, 29766, 29767, 29768, 29769, 29770, - 29771, 29772, 29773, 29774, 29775, 29776, 29777, 29778, 29779, 29780, - 29781, 29782, 29783, 29784, 29785, 29786, 29787, 29788, 29789, 29790, - 29791, 29792, 29793, 29794, 29795, 29796, 29797, 29798, 29799, 29800, - 29801, 29802, 29803, 29804, 29805, 29806, 29807, 29808, 29809, 29810, - 29811, 29812, 29813, 29814, 29815, 29816, 29817, 29818, 29819, 29820, - 29821, 29822, 29823, 29824, 29825, 29826, 29827, 29828, 29829, 29830, - 29831, 29832, 29833, 29834, 29835, 29836, 29837, 29838, 29839, 29840, - 29841, 29842, 29843, 29844, 29845, 29846, 29847, 29848, 29849, 29850, - 40951, 40951, 40951, 40951, 11516, 11514, 11463, 11496, 11423, 11436, - 11440, 11521, 11417, 11504, 11425, 11467, 11466, 11418, 11424, 11438, - 11470, 11499, 11492, 11419, 11439, 11493, 11517, 11443, 11471, 11444, - 11449, 11427, 11472, 11445, 11450, 11430, 11473, 11447, 11452, 11428, - 11429, 11481, 11482, 11448, 11453, 11434, 11485, 11446, 11451, 11431, - 11474, 11435, 11432, 11433, 11479, 11480, 11477, 11478, 11498, 11497, - 11506, 11512, 11509, 11484, 11483, 11437, 11426, 11475, 11476, 11415, - 11490, 11460, 11458, 11416, 11518, 11420, 11519, 11495, 11503, 11421, - 11487, 11468, 11486, 11441, 11520, 11500, 11422, 11515, 11501, 11442, - 11469, 11502, 11494, 11459, 11462, 11461, 11511, 11507, 11513, 11510, - 11508, 11457, 11456, 11455, 11454, 11465, 11464, 11488, 11491, 11489, - 11505, 40951, 40951, 40951, 40951, 40951, 11400, 11413, 11412, 11407, - 11414, 11394, 11387, 11386, 11383, 11385, 11388, 11389, 11384, 40951, - 40951, 40951, 11396, 11392, 11395, 11390, 11399, 11398, 11391, 11397, - 11393, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 11401, 11405, - 11408, 11403, 11411, 11410, 11404, 11409, 11406, 11402, 40951, 40951, - 11525, 11522, 11523, 11524, 32839, 32838, 32840, 32841, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 37793, 32041, 24171, 32038, 11290, 25171, 32009, 25229, 965, 20428, - 38848, 24129, 27577, 31994, 24164, 32031, 29926, 31588, 31779, 20424, - 38822, 25118, 25117, 25116, 25115, 25113, 25114, 4485, 4486, 4494, 4508, - 4378, 4377, 32579, 32587, 32580, 32591, 32584, 32588, 32581, 32593, - 32586, 32590, 32583, 32592, 32585, 32589, 32582, 37832, 37800, 37802, - 37887, 37849, 37845, 37881, 37851, 25255, 25202, 25222, 25257, 25205, - 25247, 25249, 25228, 34085, 34086, 30653, 10840, 10576, 10575, 34097, - 34098, 24177, 32018, 17483, 17484, 350, 349, 356, 355, 358, 357, 353, - 354, 351, 352, 24167, 37785, 32034, 11296, 37462, 37458, 37466, 4340, - 4338, 4344, 24160, 37780, 32028, 11292, 28300, 24170, 37788, 32037, - 11299, 16653, 16654, 3842, 3845, 3844, 3843, 3876, 24183, 37778, 32024, - 11302, 24184, 37779, 32025, 11303, 24173, 37782, 32040, 11294, 34330, - 34331, 34329, 34328, 34521, 34523, 34522, 34520, 38832, 20407, 39241, - 39242, 37705, 34162, 34161, 34160, 34115, 20802, 24043, 20803, 38837, - 20405, 24178, 32019, 24179, 32020, 17469, 24169, 37787, 32036, 11298, - 20427, 38847, 38825, 24172, 32039, 24166, 32033, 24168, 32035, 24077, - 31952, 37841, 37877, 37840, 37876, 25191, 25211, 25190, 25210, 25188, - 25208, 25189, 25209, 37844, 37880, 25201, 25221, 37842, 37878, 25200, - 25220, 37835, 37871, 25195, 25215, 37838, 37874, 25198, 25218, 37839, - 37875, 25199, 25219, 37834, 37870, 25194, 25214, 37836, 37872, 25196, - 25216, 37837, 37873, 25197, 25217, 37843, 37879, 25192, 25212, 30832, - 30833, 30834, 30835, 30836, 30837, 30838, 30839, 30840, 30841, 30842, - 30843, 30844, 30845, 30846, 30847, 30848, 30849, 30850, 30851, 30852, - 30853, 30854, 30855, 30856, 30857, 30868, 30870, 30867, 30866, 30863, - 30862, 30865, 30864, 30871, 30869, 40951, 40951, 40951, 40951, 40951, - 40951, 4137, 4081, 3952, 4167, 4038, 3979, 4138, 4019, 4082, 4128, 4054, - 4113, 3995, 4010, 4098, 3964, 4171, 4039, 4069, 3980, 4139, 4020, 4083, - 3953, 4134, 4062, 4121, 4003, 4160, 4016, 4106, 3972, 4047, 4075, 3988, - 4146, 4028, 4091, 3958, 4129, 4055, 4114, 3996, 4153, 4011, 4099, 3965, - 4172, 4040, 4070, 3981, 4140, 4021, 4084, 4066, 4125, 4007, 4164, 4035, - 4110, 3976, 4179, 4051, 4078, 3992, 4150, 4032, 4095, 3961, 4059, 4118, - 4000, 4157, 4103, 3969, 4176, 4044, 3985, 4143, 4025, 4088, 4135, 4063, - 4122, 4004, 4161, 4017, 4107, 3973, 4168, 4048, 4076, 3989, 4147, 4029, - 4092, 3959, 4130, 4056, 4115, 3997, 4154, 4012, 4100, 3966, 4173, 4041, - 4071, 3982, 4141, 4022, 4085, 3954, 4068, 4127, 4009, 4166, 4037, 4112, - 3978, 4181, 4053, 4080, 3994, 4152, 4034, 4097, 3963, 4133, 4061, 4120, - 4002, 4159, 4015, 4105, 3971, 4178, 4046, 4074, 3987, 4145, 4027, 4090, - 3957, 4065, 4124, 4006, 4163, 4109, 3975, 4170, 4050, 3991, 4149, 4031, - 4094, 4131, 4058, 4117, 3999, 4156, 4013, 4102, 3968, 4175, 4043, 4072, - 3984, 4142, 4024, 4087, 3955, 4067, 4126, 4008, 4165, 4036, 4111, 3977, - 4180, 4052, 4079, 3993, 4151, 4033, 4096, 3962, 4132, 4060, 4119, 4001, - 4158, 4014, 4104, 3970, 4177, 4045, 4073, 3986, 4144, 4026, 4089, 3956, - 4136, 4064, 4123, 4005, 4162, 4018, 4108, 3974, 4169, 4049, 4077, 3990, - 4148, 4030, 4093, 3960, 4057, 4116, 3998, 4155, 4101, 3967, 4174, 4042, - 3983, 4023, 4086, 37470, 4347, 37467, 4345, 37468, 4346, 37463, 4341, - 37464, 4342, 37457, 4334, 4335, 4336, 4337, 28184, 37459, 37460, 11293, - 24161, 33837, 37783, 11295, 17356, 17357, 17358, 31947, 25162, 17359, - 37807, 25164, 19784, 39303, 37481, 17638, 4379, 4376, 24081, 31956, - 24080, 31955, 20404, 24078, 31953, 20403, 25165, 37810, 38838, 4504, - 4502, 4503, 4501, 22718, 22717, 22722, 22716, 22694, 22673, 22674, 22714, - 22680, 22698, 22721, 22696, 22719, 22720, 22702, 22699, 22679, 22678, - 22681, 22697, 22682, 22670, 22691, 22715, 22671, 22672, 22669, 22695, - 22700, 22686, 22677, 22701, 22703, 22676, 22693, 22685, 22683, 22684, - 22675, 22723, 22692, 22687, 22690, 22688, 22689, 22711, 22709, 22710, - 22713, 22708, 22705, 22712, 22707, 22706, 22704, 32594, 32626, 32595, - 32642, 32611, 32627, 32596, 32650, 32619, 32635, 32604, 32643, 32612, - 32628, 32597, 32654, 32623, 32639, 32608, 32647, 32616, 32632, 32601, - 32651, 32620, 32636, 32605, 32644, 32613, 32629, 32598, 32656, 32625, - 32641, 32610, 32649, 32618, 32634, 32603, 32653, 32622, 32638, 32607, - 32646, 32615, 32631, 32600, 32655, 32624, 32640, 32609, 32648, 32617, - 32633, 32602, 32652, 32621, 32637, 32606, 32645, 32614, 32630, 32599, - 37827, 37799, 37801, 37866, 37848, 37846, 37847, 37850, 25254, 25252, - 25253, 25256, 25207, 25246, 25248, 25242, 31957, 31997, 24132, 24082, - 25167, 25262, 37890, 37812, 24083, 24133, 31998, 31958, 37813, 37891, - 25263, 25168, 20429, 21627, 30342, 3882, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40767, 40772, 40806, 40769, 40774, 40802, 40796, 40790, - 40813, 40792, 40786, 40810, 40768, 40773, 40807, 40770, 40775, 40803, - 40797, 40791, 40814, 40793, 40787, 40811, 40808, 40794, 40801, 40788, - 40789, 40812, 40795, 40771, 40817, 40782, 40799, 40809, 40820, 40819, - 40785, 40818, 40779, 40778, 40816, 40800, 40798, 40777, 40951, 40951, - 40822, 40823, 40824, 40815, 40765, 40780, 40783, 40784, 40762, 40763, - 40781, 40804, 40805, 40766, 40821, 40764, 40776, 40761, 40943, 40944, - 40941, 40942, 40945, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40841, 40842, 40858, 40830, 40839, 40938, 40893, 40894, - 40861, 40862, 40895, 40825, 40859, 40939, 40837, 40834, 40836, 40835, - 40833, 40935, 40934, 40937, 40936, 40931, 40930, 40933, 40932, 40831, - 40865, 40827, 40838, 40826, 40863, 40876, 40877, 40875, 40874, 40828, - 40872, 40873, 40871, 40869, 40870, 40868, 40867, 40878, 40880, 40881, - 40879, 40843, 40866, 40832, 40840, 40940, 40882, 40887, 40884, 40888, - 40885, 40890, 40891, 40886, 40883, 40889, 40864, 40892, 40925, 40922, - 40913, 40923, 40924, 40914, 40902, 40901, 40929, 40899, 40898, 40896, - 40900, 40897, 40916, 40921, 40915, 40919, 40920, 40917, 40918, 40845, - 40849, 40848, 40847, 40846, 40928, 40926, 40927, 40852, 40857, 40856, - 40855, 40854, 40853, 40903, 40912, 40905, 40910, 40904, 40908, 40909, - 40906, 40907, 40911, 40844, 40851, 40829, 40850, 40860, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 5194, 5066, 5187, - 5169, 5170, 5238, 5239, 5119, 5216, 5176, 5250, 5251, 5142, 5021, 5072, - 5219, 5120, 5024, 5025, 5214, 5226, 5165, 5102, 5193, 5045, 5242, 5114, - 5128, 5124, 5217, 5179, 5208, 5172, 5241, 5027, 5029, 5129, 5195, 5180, - 5237, 5019, 5197, 5211, 5213, 5167, 5221, 5149, 5067, 5229, 5220, 5139, - 5020, 5079, 5110, 5231, 5117, 5185, 5189, 5132, 5043, 5196, 5174, 5177, - 5116, 5254, 5184, 5133, 5232, 5209, 5108, 5115, 5166, 5171, 5183, 5135, - 5182, 5140, 5186, 5123, 5127, 5253, 5026, 5022, 5252, 5141, 5075, 5046, - 5164, 5240, 5181, 5190, 5173, 5018, 5150, 5178, 5175, 5071, 5143, 5017, - 5233, 5074, 5212, 5215, 5044, 5073, 5198, 5255, 5235, 5191, 5230, 5234, - 5188, 5236, 5192, 5105, 5031, 5070, 5168, 5223, 5224, 5222, 5225, 5118, - 5068, 5243, 5244, 5207, 5131, 5064, 5136, 5137, 5138, 5028, 5030, 5063, - 5228, 5218, 5134, 5144, 5148, 5147, 5146, 5145, 5104, 5101, 5100, 5058, - 5060, 5061, 5059, 5227, 5032, 5112, 5051, 5015, 5009, 5010, 5013, 5014, - 5012, 5011, 5016, 5157, 5152, 5153, 5151, 5162, 5161, 5160, 5159, 5163, - 5155, 5113, 5023, 5078, 5077, 5076, 5158, 5156, 5154, 5106, 5107, 5069, - 5111, 5109, 5080, 5087, 5083, 5091, 5086, 5095, 5085, 5084, 5081, 5082, - 5089, 5090, 5097, 5094, 5092, 5041, 5042, 5040, 5088, 5096, 5249, 5054, - 5052, 5055, 5057, 5056, 5053, 5245, 5247, 5246, 5248, 5098, 5099, 5048, - 5047, 5049, 5050, 5203, 5206, 5204, 5205, 5199, 5202, 5200, 5201, 5062, - 5065, 5210, 5039, 5033, 5038, 5036, 5035, 5034, 5037, 5121, 5125, 5122, - 5126, 5130, 5103, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 28489, 28366, 28382, 28474, 28361, 28486, 28424, 28475, - 28473, 28362, 28358, 28485, 28352, 28470, 28471, 28472, 28377, 28378, - 28315, 28316, 28311, 28310, 28441, 28512, 28509, 28384, 28383, 28490, - 28491, 28385, 28394, 28395, 28396, 28355, 28387, 28388, 28389, 28368, - 28369, 40951, 40951, 28432, 28365, 28367, 28393, 28392, 28437, 28436, - 28488, 28487, 28464, 28465, 28351, 28357, 28453, 28454, 28468, 28469, - 28433, 28535, 28407, 28467, 28376, 28493, 28511, 28495, 28440, 28533, - 28456, 28356, 28496, 28497, 28520, 28521, 28524, 28525, 28522, 28523, - 28516, 28517, 28518, 28519, 28430, 28431, 28526, 28527, 28455, 28531, - 28438, 28435, 28319, 28320, 28314, 28534, 28406, 28466, 28375, 28492, - 28510, 28494, 28439, 28340, 28341, 28344, 28345, 28346, 28379, 28380, - 28381, 28323, 28330, 28331, 28332, 28333, 28334, 28308, 28373, 28309, - 28374, 28307, 28371, 28306, 28370, 28321, 28339, 28347, 28338, 28324, - 28325, 28322, 28349, 28404, 28403, 28328, 28350, 28335, 28342, 28348, - 28327, 28343, 28476, 28499, 28538, 28463, 28426, 28386, 28354, 28363, - 28400, 28399, 28515, 28528, 28537, 28529, 28530, 28442, 28445, 28446, - 28447, 28448, 28449, 28450, 28451, 28452, 28443, 28444, 28408, 28434, - 28372, 28364, 28326, 28329, 28336, 28337, 28458, 28457, 28405, 28398, - 28397, 28536, 28359, 28360, 28425, 28421, 28312, 28479, 28481, 28427, - 28429, 28482, 28484, 28390, 28391, 28423, 28422, 28313, 28480, 28428, - 28483, 28507, 28506, 28508, 28505, 28501, 28502, 28503, 28504, 28353, - 28401, 28402, 28498, 28532, 28460, 28318, 28477, 28317, 28513, 28461, - 28462, 28478, 28514, 28459, 28409, 28412, 28416, 28419, 28418, 28417, - 28413, 28414, 28410, 28411, 28415, 28500, 28420, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 18636, 18624, - 18647, 18648, 18630, 18649, 18650, 18651, 18652, 18637, 18638, 18639, - 18640, 18641, 18642, 18643, 18644, 18645, 18646, 18625, 18626, 18627, - 18628, 18629, 18631, 18632, 18633, 18634, 18635, 18356, 18364, 18377, - 18385, 18391, 18392, 18357, 18358, 18359, 18360, 18361, 18362, 18363, - 18365, 18366, 18367, 18368, 18369, 18370, 18371, 18372, 18373, 18374, - 18375, 18376, 18378, 18379, 18380, 18381, 18382, 18383, 18384, 18386, - 18387, 18388, 18389, 18390, 8271, 8270, 8272, 18416, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21607, - 21608, 21605, 21603, 21594, 21593, 21600, 21598, 21589, 21596, 21606, - 21591, 21604, 21602, 21595, 21592, 21601, 21599, 21590, 21597, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 26844, 26845, 26842, 26840, 26831, 26830, 26837, 26835, 26826, - 26833, 26843, 26828, 26841, 26839, 26832, 26829, 26838, 26836, 26827, - 26834, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 28177, 10885, 10886, 10880, 10879, 10878, 36785, - 36805, 36826, 36774, 36819, 36783, 36769, 36828, 36773, 36787, 36793, - 36816, 36817, 36831, 36837, 36784, 36815, 36845, 36803, 36770, 36833, - 36835, 36802, 36848, 36782, 36797, 36794, 36775, 36789, 36772, 36830, - 36823, 36777, 36820, 36809, 36843, 36832, 36806, 36834, 36822, 36836, - 36811, 36799, 36842, 36812, 36798, 36829, 36838, 36808, 36844, 36781, - 36825, 36801, 36847, 36791, 36776, 36813, 36810, 36824, 36768, 36796, - 36795, 36846, 36840, 36818, 36788, 36786, 36792, 36800, 36839, 36841, - 36814, 36780, 36778, 36807, 36771, 36779, 36827, 36790, 36821, 36804, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 8669, - 8667, 8666, 8663, 8662, 8665, 8664, 8670, 8668, 8660, 8658, 8657, 8654, - 8653, 8656, 8655, 8661, 8659, 20515, 20514, 20513, 20512, 20511, 35370, - 35369, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 25824, 25826, 25854, 25817, - 25830, 25862, 25833, 25863, 25835, 25864, 25837, 25839, 25856, 25858, - 25841, 25844, 25865, 25848, 25850, 25820, 25852, 25866, 25867, 25860, - 25868, 25828, 25872, 25874, 25907, 25869, 25878, 25881, 25883, 25915, - 25885, 25916, 25887, 25889, 25909, 25911, 25891, 25894, 25917, 25898, - 25900, 25902, 25905, 25918, 25919, 25913, 25920, 25876, 26312, 26314, - 26344, 26318, 26320, 26352, 26323, 26353, 26325, 26354, 26327, 26329, - 26346, 26348, 26331, 26334, 26355, 26338, 26340, 26308, 26342, 26356, - 26357, 26350, 26358, 26316, 26260, 26262, 26295, 26256, 26266, 26269, - 26271, 40951, 26273, 26303, 26275, 26277, 26297, 26299, 26279, 26282, - 26304, 26286, 26288, 26290, 26293, 26305, 26306, 26301, 26307, 26264, - 25977, 25979, 26009, 25983, 25985, 26017, 25988, 26018, 25990, 26019, - 25992, 25994, 26011, 26013, 25996, 25999, 26020, 26003, 26005, 25973, - 26007, 26021, 26022, 26015, 26023, 25981, 26031, 26033, 26068, 26037, - 26039, 26042, 26044, 26076, 26046, 26077, 26048, 26050, 26070, 26072, - 26052, 26055, 26078, 26059, 26061, 26063, 26066, 26079, 26080, 26074, - 26081, 26035, 26784, 40951, 26785, 26786, 40951, 40951, 26787, 40951, - 40951, 26788, 26789, 40951, 40951, 26790, 26791, 26792, 26793, 40951, - 26794, 26795, 26796, 26797, 26798, 26799, 26800, 26801, 26802, 26803, - 26804, 26805, 40951, 26806, 40951, 26807, 26808, 26809, 26810, 26811, - 26812, 26813, 40951, 26814, 26815, 26816, 26817, 26818, 26819, 26820, - 26821, 26822, 26823, 26824, 25921, 25922, 25923, 25924, 25925, 25926, - 25927, 25928, 25929, 25930, 25931, 25932, 25933, 25934, 25935, 25936, - 25937, 25938, 25939, 25940, 25941, 25942, 25943, 25944, 25945, 25946, - 25947, 25948, 25949, 25950, 25951, 25952, 25953, 25954, 25955, 25956, - 25957, 25958, 25959, 25960, 25961, 25962, 25963, 25964, 25965, 25966, - 25967, 25968, 25969, 25970, 25971, 25972, 26208, 26209, 40951, 26210, - 26211, 26212, 26213, 40951, 40951, 26214, 26215, 26216, 26217, 26218, - 26219, 26220, 26221, 40951, 26222, 26223, 26224, 26225, 26226, 26227, - 26228, 40951, 26229, 26230, 26231, 26232, 26233, 26234, 26235, 26236, - 26237, 26238, 26239, 26240, 26241, 26242, 26243, 26244, 26245, 26246, - 26247, 26248, 26249, 26250, 26251, 26252, 26253, 26254, 26153, 26154, - 40951, 26155, 26156, 26157, 26158, 40951, 26159, 26160, 26161, 26162, - 26163, 40951, 26164, 40951, 40951, 40951, 26165, 26166, 26167, 26168, - 26169, 26170, 26171, 40951, 26172, 26173, 26174, 26175, 26176, 26177, - 26178, 26179, 26180, 26181, 26182, 26183, 26184, 26185, 26186, 26187, - 26188, 26189, 26190, 26191, 26192, 26193, 26194, 26195, 26196, 26197, - 26091, 26092, 26093, 26094, 26095, 26096, 26097, 26098, 26099, 26100, - 26101, 26102, 26103, 26104, 26105, 26106, 26107, 26108, 26109, 26110, - 26111, 26112, 26113, 26114, 26115, 26116, 26117, 26118, 26119, 26120, - 26121, 26122, 26123, 26124, 26125, 26126, 26127, 26128, 26129, 26130, - 26131, 26132, 26133, 26134, 26135, 26136, 26137, 26138, 26139, 26140, - 26141, 26142, 26722, 26723, 26724, 26725, 26726, 26727, 26728, 26729, - 26730, 26731, 26732, 26733, 26734, 26735, 26736, 26737, 26738, 26739, - 26740, 26741, 26742, 26743, 26744, 26745, 26746, 26747, 26748, 26749, - 26750, 26751, 26752, 26753, 26754, 26755, 26756, 26757, 26758, 26759, - 26760, 26761, 26762, 26763, 26764, 26765, 26766, 26767, 26768, 26769, - 26770, 26771, 26772, 26773, 26554, 26556, 26586, 26560, 26562, 26594, - 26565, 26595, 26567, 26596, 26569, 26571, 26588, 26590, 26573, 26576, - 26597, 26580, 26582, 26550, 26584, 26598, 26599, 26592, 26600, 26558, - 26608, 26610, 26645, 26614, 26616, 26619, 26621, 26653, 26623, 26654, - 26625, 26627, 26647, 26649, 26629, 26632, 26655, 26636, 26638, 26640, - 26643, 26656, 26657, 26651, 26658, 26612, 26670, 26671, 26672, 26673, - 26674, 26675, 26676, 26677, 26678, 26679, 26680, 26681, 26682, 26683, - 26684, 26685, 26686, 26687, 26688, 26689, 26690, 26691, 26692, 26693, - 26694, 26695, 26696, 26697, 26698, 26699, 26700, 26701, 26702, 26703, - 26704, 26705, 26706, 26707, 26708, 26709, 26710, 26711, 26712, 26713, - 26714, 26715, 26716, 26717, 26718, 26719, 26720, 26721, 26444, 26446, - 26476, 26450, 26452, 26484, 26455, 26485, 26457, 26486, 26459, 26461, - 26478, 26480, 26463, 26466, 26487, 26470, 26472, 26440, 26474, 26488, - 26489, 26482, 26490, 26448, 26498, 26500, 26535, 26504, 26506, 26509, - 26511, 26543, 26513, 26544, 26515, 26517, 26537, 26539, 26519, 26522, - 26545, 26526, 26528, 26530, 26533, 26546, 26547, 26541, 26548, 26502, - 26367, 26368, 26369, 26370, 26371, 26372, 26373, 26374, 26375, 26376, - 26377, 26378, 26379, 26380, 26381, 26382, 26383, 26384, 26385, 26386, - 26387, 26388, 26389, 26390, 26391, 26392, 26393, 26394, 26395, 26396, - 26397, 26398, 26399, 26400, 26401, 26402, 26403, 26404, 26405, 26406, - 26407, 26408, 26409, 26410, 26411, 26412, 26413, 26414, 26415, 26416, - 26417, 26418, 26257, 26258, 40951, 40951, 25825, 25827, 25834, 25819, - 25831, 25829, 25832, 25821, 25836, 25838, 25840, 25857, 25859, 25861, - 25842, 25847, 25849, 25822, 25851, 25823, 25853, 25845, 25855, 25846, - 25843, 26085, 25873, 25875, 25884, 25871, 25879, 25877, 25880, 25903, - 25886, 25888, 25890, 25910, 25912, 25914, 25892, 25897, 25899, 25882, - 25901, 25904, 25906, 25895, 25908, 25896, 25893, 26087, 26083, 26090, - 26084, 26086, 26089, 26088, 26313, 26315, 26324, 26319, 26321, 26317, - 26322, 26309, 26326, 26328, 26330, 26347, 26349, 26351, 26332, 26337, - 26339, 26310, 26341, 26311, 26343, 26335, 26345, 26336, 26333, 26361, - 26261, 26263, 26272, 26259, 26267, 26265, 26268, 26291, 26274, 26276, - 26278, 26298, 26300, 26302, 26280, 26285, 26287, 26270, 26289, 26292, - 26294, 26283, 26296, 26284, 26281, 26363, 26359, 26366, 26360, 26362, - 26365, 26364, 25978, 25980, 25989, 25984, 25986, 25982, 25987, 25974, - 25991, 25993, 25995, 26012, 26014, 26016, 25997, 26002, 26004, 25975, - 26006, 25976, 26008, 26000, 26010, 26001, 25998, 26026, 26032, 26034, - 26045, 26038, 26040, 26036, 26041, 26064, 26047, 26049, 26051, 26071, - 26073, 26075, 26053, 26058, 26060, 26043, 26062, 26065, 26067, 26056, - 26069, 26057, 26054, 26028, 26024, 26082, 26025, 26027, 26030, 26029, - 26555, 26557, 26566, 26561, 26563, 26559, 26564, 26551, 26568, 26570, - 26572, 26589, 26591, 26593, 26574, 26579, 26581, 26552, 26583, 26553, - 26585, 26577, 26587, 26578, 26575, 26603, 26609, 26611, 26622, 26615, - 26617, 26613, 26618, 26641, 26624, 26626, 26628, 26648, 26650, 26652, - 26630, 26635, 26637, 26620, 26639, 26642, 26644, 26633, 26646, 26634, - 26631, 26605, 26601, 26659, 26602, 26604, 26607, 26606, 26445, 26447, - 26456, 26451, 26453, 26449, 26454, 26441, 26458, 26460, 26462, 26479, - 26481, 26483, 26464, 26469, 26471, 26442, 26473, 26443, 26475, 26467, - 26477, 26468, 26465, 26493, 26499, 26501, 26512, 26505, 26507, 26503, - 26508, 26531, 26514, 26516, 26518, 26538, 26540, 26542, 26520, 26525, - 26527, 26510, 26529, 26532, 26534, 26523, 26536, 26524, 26521, 26495, - 26491, 26549, 26492, 26494, 26497, 26496, 25818, 25870, 40951, 40951, - 26149, 26151, 26148, 26147, 26144, 26143, 26146, 26145, 26152, 26150, - 26204, 26206, 26203, 26202, 26199, 26198, 26201, 26200, 26207, 26205, - 26780, 26782, 26779, 26778, 26775, 26774, 26777, 26776, 26783, 26781, - 26666, 26668, 26665, 26664, 26661, 26660, 26663, 26662, 26669, 26667, - 26425, 26427, 26424, 26423, 26420, 26419, 26422, 26421, 26428, 26426, - 33092, 33048, 33073, 33289, 33244, 33030, 33093, 33057, 33206, 33158, - 33134, 33095, 33098, 33055, 33099, 33049, 33100, 33122, 33117, 33155, - 33096, 33102, 33111, 33112, 33103, 33109, 33113, 33051, 33185, 33094, - 33123, 33052, 33132, 33101, 33131, 33118, 33157, 33156, 33097, 33116, - 33126, 33127, 33130, 33129, 33197, 33105, 33106, 33107, 33179, 33147, - 33110, 33114, 33108, 33104, 33178, 33139, 33177, 33176, 33136, 33135, - 33138, 33128, 33125, 33124, 33174, 33173, 33175, 33146, 33222, 33226, - 33225, 33223, 33224, 33067, 33213, 33243, 33215, 33228, 33220, 33229, - 33221, 33230, 33219, 33077, 33078, 33242, 33283, 33216, 33217, 33218, - 33214, 33240, 33227, 33238, 33231, 33239, 33237, 33235, 33232, 33233, - 33234, 33236, 33064, 33070, 33068, 33069, 33273, 33272, 33080, 33072, - 33083, 33086, 33081, 33084, 33082, 33085, 33090, 33087, 33047, 33282, - 33288, 33285, 33287, 33261, 33263, 33241, 33271, 33264, 33268, 33262, - 33270, 33269, 33267, 33029, 33119, 33054, 33246, 33032, 33255, 33121, - 33120, 33247, 33160, 33163, 33162, 33161, 33170, 33210, 33059, 33284, - 33041, 33166, 33169, 33164, 33165, 33258, 33168, 33257, 33040, 33039, - 33167, 33058, 33256, 33038, 33133, 33053, 33248, 33265, 33031, 33115, - 33050, 33186, 33266, 33042, 33194, 33190, 33192, 33063, 33286, 33043, - 33187, 33189, 33188, 33193, 33191, 33281, 33159, 33056, 33088, 33275, - 33276, 33274, 33076, 33254, 33034, 33033, 33183, 33259, 33181, 33062, - 33171, 33182, 33280, 33180, 33184, 33172, 33060, 33089, 33079, 33260, - 33045, 33046, 33044, 33061, 33065, 33066, 33278, 33279, 33277, 33245, - 33148, 33250, 33151, 33152, 33153, 33150, 33154, 33149, 33143, 33144, - 33145, 33142, 33140, 33071, 33141, 33137, 33074, 33075, 33252, 33253, - 33249, 33251, 33036, 33037, 33035, 33195, 33200, 33203, 33204, 33205, - 33211, 33196, 33198, 33199, 33207, 33202, 33208, 33209, 33201, 33091, - 33212, 33603, 33602, 33601, 33623, 33622, 33621, 33578, 33577, 33576, - 32962, 32961, 32960, 33564, 33563, 33562, 33574, 33575, 33570, 33573, - 33569, 33572, 33571, 33019, 33022, 33018, 33021, 33020, 33568, 33438, - 33439, 33440, 33441, 33436, 33437, 33442, 33482, 33387, 33500, 33499, - 33497, 33498, 33501, 33476, 33479, 33477, 33478, 33475, 33506, 33505, - 33503, 33504, 33452, 33451, 33450, 33456, 33455, 33454, 33453, 33474, - 33473, 33472, 33449, 33448, 33447, 33522, 33521, 33520, 33496, 33495, - 33494, 33619, 33618, 33617, 33616, 33615, 33614, 33620, 33613, 33612, - 33611, 33356, 33355, 33353, 33354, 33360, 33359, 33357, 33358, 33348, - 33347, 33345, 33346, 33352, 33351, 33349, 33350, 33410, 33409, 33407, - 33408, 33411, 33425, 33428, 33426, 33427, 33384, 33415, 33414, 33413, - 33412, 33370, 33383, 33382, 33381, 33371, 33369, 33368, 33367, 33434, - 33433, 33432, 33431, 33430, 33429, 33606, 33605, 33604, 33609, 33608, - 33607, 33610, 33469, 33468, 33466, 33467, 33460, 33459, 33457, 33458, - 33465, 33464, 33487, 33486, 33483, 33488, 33493, 33490, 33489, 33509, - 33508, 33507, 33512, 33511, 33510, 33463, 33471, 33470, 33559, 33556, - 33555, 33502, 33462, 33485, 33492, 33517, 33561, 33558, 33554, 33461, - 33484, 33491, 33516, 33560, 33557, 33553, 33515, 33514, 33513, 33374, - 33373, 33391, 33389, 33390, 33388, 33400, 33398, 33399, 33397, 33417, - 33416, 33548, 33545, 33551, 33376, 33375, 33392, 33393, 33395, 33394, - 33404, 33402, 33403, 33401, 33419, 33418, 33549, 33546, 33552, 33380, - 33379, 33377, 33378, 33372, 33396, 33405, 33420, 33421, 33422, 33547, - 33544, 33550, 33406, 33446, 33444, 33445, 33443, 33366, 33364, 33362, - 33365, 33363, 33361, 33519, 33518, 33424, 33423, 33481, 33480, 33386, - 33385, 32978, 32977, 32973, 32976, 32980, 32979, 32974, 32975, 32972, - 32981, 33291, 33298, 33296, 33295, 33293, 33294, 33292, 33297, 33011, - 33009, 33010, 32987, 32986, 32985, 32970, 32968, 32969, 32971, 33026, - 33025, 33024, 33005, 33006, 33002, 32983, 32982, 33001, 33003, 33000, - 33004, 32984, 32999, 32997, 32998, 32994, 32996, 32995, 32988, 32990, - 32989, 32992, 32991, 32993, 32965, 32964, 32963, 33588, 33587, 33589, - 33008, 33525, 33524, 33526, 33527, 32955, 32957, 32954, 32956, 32959, - 32958, 33320, 33318, 33319, 33325, 33327, 33326, 33322, 33324, 33323, - 33339, 33338, 33337, 33331, 33332, 33333, 33334, 33335, 33336, 33328, - 33330, 33329, 33340, 33341, 33342, 33309, 33307, 33308, 33321, 33344, - 33343, 33596, 33595, 33593, 33594, 33592, 33597, 33591, 33590, 33580, - 33585, 33583, 33584, 33581, 33582, 33586, 33523, 33435, 33528, 33290, - 33007, 33566, 33565, 33028, 33023, 33567, 33599, 33598, 33600, 33624, - 33299, 33300, 33301, 33302, 33303, 33304, 33305, 33306, 33017, 33317, - 33316, 33311, 33314, 33313, 33310, 33312, 33315, 32967, 33027, 33579, - 32966, 33625, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 33012, 33013, 33014, - 33015, 33016, 40951, 33536, 33537, 33538, 33539, 33540, 33541, 33542, - 33543, 33529, 33530, 33531, 33532, 33533, 33534, 33535, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 23497, 23771, - 23262, 23776, 23249, 23621, 23882, 23773, 23868, 23837, 23242, 23476, - 23477, 23872, 23237, 23289, 23263, 23614, 23413, 23594, 23474, 23866, - 23752, 23841, 23485, 23414, 23558, 23711, 23842, 23380, 23788, 40951, - 40951, 40951, 40951, 40951, 40951, 23404, 23607, 23653, 23757, 23796, - 23832, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 8227, 8229, 8239, 8233, 8215, 8240, 8247, - 40951, 8246, 8220, 8217, 8216, 8214, 8251, 8235, 8234, 8236, 8250, 8237, - 8238, 8222, 8225, 8249, 8231, 8248, 40951, 40951, 8223, 8226, 8230, 8224, - 8242, 8241, 8243, 40951, 8245, 8221, 40951, 8244, 8219, 8228, 8218, 8232, - 40951, 40951, 40951, 40951, 40951, 27764, 27733, 27761, 27759, 27735, - 27757, 27754, 27755, 27756, 27763, 27740, 27741, 27765, 27744, 27742, - 27737, 27750, 27766, 27739, 27762, 27749, 27758, 27748, 27751, 27736, - 27753, 27734, 27747, 27731, 27760, 27732, 27745, 27743, 10520, 10498, - 10518, 10505, 10501, 10515, 10512, 10513, 10514, 10519, 10503, 10521, - 10517, 10504, 10522, 10502, 10507, 10511, 10516, 10510, 10508, 10509, - 10506, 10497, 10500, 10499, 27738, 27752, 27746, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 8144, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 29882, 29866, 29857, 29868, 29873, 29865, 29870, - 29861, 29887, 29891, 29893, 29896, 29860, 29855, 29890, 29880, 29864, - 29863, 29894, 29856, 29867, 29888, 29876, 29892, 29895, 29862, 29884, - 29869, 29859, 29879, 29858, 29874, 29881, 29883, 29889, 29875, 29871, - 29872, 29897, 29898, 29877, 29878, 29885, 29886, 29899, 40951, 40951, - 40951, 29908, 29912, 29911, 29914, 29913, 29910, 29909, 29905, 29902, - 29901, 29904, 29903, 29906, 29907, 40951, 40951, 29922, 29924, 29921, - 29920, 29917, 29916, 29919, 29918, 29925, 29923, 40951, 40951, 40951, - 40951, 29900, 29915, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 37522, 37505, 37525, 37515, 37519, 37516, 37521, - 37507, 37506, 37524, 37512, 37527, 37526, 37518, 37517, 37523, 37520, - 37508, 37500, 37509, 37501, 37529, 37510, 37502, 37511, 37503, 37528, - 37514, 37504, 37513, 37530, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 38979, 38978, 39010, 39011, 39012, 39014, 39003, 39006, 38992, - 38995, 39007, 38999, 38996, 39013, 39009, 39008, 39016, 39021, 39020, - 39019, 39005, 38984, 38983, 39018, 39017, 39004, 39015, 38987, 38989, - 38993, 39000, 38991, 38998, 38997, 38986, 38981, 38982, 38990, 38985, - 38988, 38980, 38994, 39001, 39002, 39025, 39026, 39023, 39024, 39033, - 39035, 39032, 39031, 39028, 39027, 39030, 39029, 39036, 39034, 40951, - 40951, 40951, 40951, 40951, 39022, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 28864, 28867, 28866, 28868, 28865, 28847, 28851, - 28849, 28848, 28850, 28859, 28862, 28860, 28863, 28861, 28869, 28870, - 28871, 28872, 28873, 28852, 28854, 28857, 28858, 28853, 28856, 28855, - 28877, 28874, 28878, 28876, 28875, 28885, 28887, 28884, 28883, 28880, - 28879, 28882, 28881, 28888, 28886, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 30046, - 30049, 30048, 30047, 30050, 30051, 30028, 30030, 30029, 30031, 30032, - 30033, 30040, 30041, 30045, 30042, 30043, 30044, 30052, 30056, 30053, - 30055, 30054, 30057, 30034, 30039, 30038, 30036, 30035, 30037, 30060, - 30058, 30059, 30068, 30070, 30067, 30066, 30063, 30062, 30065, 30064, - 30071, 30069, 40951, 40951, 40951, 40951, 30061, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 17006, 17012, 17010, 17007, 17009, 17008, 17011, 40951, 16961, - 17005, 17003, 17002, 40951, 16949, 16948, 40951, 16960, 16959, 16958, - 16945, 16944, 16957, 16956, 16955, 16954, 16953, 16952, 16947, 16946, - 16951, 16950, 40951, 27076, 27077, 27078, 27144, 27165, 27153, 27118, - 27253, 27079, 27080, 27084, 27203, 27188, 27193, 27117, 27271, 27220, - 27142, 27123, 27214, 27083, 27081, 27082, 27129, 27173, 27226, 27266, - 27089, 27085, 27093, 27230, 27168, 27178, 27205, 27094, 27091, 27090, - 27243, 27181, 27241, 27219, 27210, 27208, 27209, 27272, 27251, 27088, - 27103, 27095, 27242, 27187, 27212, 27147, 27273, 27099, 27100, 27101, - 27157, 27149, 27133, 27234, 27185, 27086, 27092, 27087, 27160, 27257, - 27258, 27096, 27097, 27098, 27171, 27126, 27176, 27143, 27104, 27102, - 27105, 27229, 27186, 27235, 27137, 27249, 27106, 27110, 27114, 27183, - 27155, 27223, 27204, 27107, 27111, 27112, 27154, 27146, 27211, 27164, - 27274, 27175, 27113, 27108, 27109, 27191, 27244, 27252, 27122, 27264, - 27115, 27174, 27124, 27221, 27161, 27199, 27131, 27213, 27163, 27130, - 27270, 27120, 27166, 27116, 27162, 27190, 27217, 27228, 27198, 27233, - 27200, 27159, 27179, 27260, 27227, 27194, 27237, 27267, 27240, 27238, - 27263, 27134, 27250, 27140, 27169, 27125, 27156, 27132, 27180, 27136, - 27216, 27135, 27195, 27121, 27261, 27151, 27246, 27247, 27265, 27236, - 27177, 27215, 27207, 27170, 27145, 27119, 27184, 27192, 27231, 27197, - 27127, 27222, 27167, 27182, 27148, 27150, 27256, 27196, 27202, 27201, - 27268, 27189, 27138, 27139, 27225, 27269, 27218, 27206, 27259, 27262, - 27232, 27254, 27158, 27224, 27152, 27239, 27128, 27255, 27172, 27141, - 40951, 40951, 27282, 27280, 27279, 27276, 27275, 27278, 27277, 27283, - 27281, 27071, 27070, 27074, 27072, 27069, 27073, 27075, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 27, 9, 24, - 14, 29, 21, 34, 28, 37, 39, 35, 40, 41, 10, 30, 32, 18, 15, 31, 42, 13, - 25, 36, 23, 12, 20, 33, 19, 38, 17, 11, 26, 16, 22, 62, 44, 59, 49, 64, - 56, 69, 63, 72, 74, 70, 75, 76, 45, 65, 67, 53, 50, 66, 77, 48, 60, 71, - 58, 47, 55, 68, 54, 73, 52, 46, 61, 51, 57, 85, 86, 79, 84, 43, 78, 83, - 82, 40951, 40951, 40951, 40951, 93, 95, 92, 91, 88, 87, 90, 89, 96, 94, - 40951, 40951, 40951, 40951, 80, 81, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 20698, 20684, - 20679, 20659, 20654, 20672, 20667, 20647, 20662, 20687, 20682, 20677, - 20657, 20652, 20673, 20668, 20648, 20663, 20699, 20685, 20680, 20660, - 20655, 20675, 20670, 20650, 20665, 20700, 20686, 20681, 20661, 20656, - 20676, 20671, 20651, 20666, 20688, 20683, 20678, 20658, 20653, 20674, - 20669, 20649, 20664, 20645, 20646, 20636, 20643, 20644, 20696, 20694, - 20693, 20690, 20689, 20692, 20691, 20697, 20695, 20702, 20638, 20637, - 20639, 20701, 20642, 20641, 20640, 20635, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 30827, 30822, 30817, 30797, 30792, 30810, 30805, 30785, - 30800, 30825, 30820, 30815, 30795, 30790, 30811, 30806, 30786, 30801, - 30828, 30823, 30818, 30798, 30793, 30813, 30808, 30788, 30803, 30829, - 30824, 30819, 30799, 30794, 30814, 30809, 30789, 30804, 30826, 30821, - 30816, 30796, 30791, 30812, 30807, 30787, 30802, 30784, 30777, 30779, - 30769, 30771, 30772, 30774, 30781, 30780, 30775, 30770, 30773, 30778, - 30776, 30783, 30782, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 2246, 2254, 2252, 2150, - 40951, 2262, 2250, 2258, 2242, 2257, 2249, 2198, 2253, 2260, 2225, 2247, - 2255, 2226, 2261, 2256, 2224, 2245, 2244, 2248, 2243, 2149, 2251, 2259, - 2120, 2122, 2121, 2123, 40951, 2162, 2160, 40951, 2157, 40951, 40951, - 2156, 40951, 2164, 2159, 2167, 2161, 2166, 2154, 2169, 2163, 2155, 2168, - 40951, 2153, 2152, 2151, 2158, 40951, 2170, 40951, 2165, 40951, 40951, - 40951, 40951, 40951, 40951, 2234, 40951, 40951, 40951, 40951, 2236, - 40951, 2235, 40951, 2239, 40951, 2238, 2231, 2241, 40951, 2232, 2240, - 40951, 2230, 40951, 40951, 2233, 40951, 2229, 40951, 2237, 40951, 2227, - 40951, 2228, 40951, 2216, 2214, 40951, 2211, 40951, 40951, 2210, 2205, - 2218, 2213, 40951, 2215, 2221, 2208, 2223, 2217, 2209, 2222, 40951, 2207, - 2206, 2204, 2212, 40951, 2203, 2219, 2220, 2201, 40951, 2202, 40951, - 2175, 2187, 2185, 2195, 2178, 2197, 2183, 2177, 2181, 2190, 40951, 2193, - 2186, 2192, 2172, 2176, 2188, 2173, 2196, 2189, 2171, 2182, 2180, 2174, - 2179, 2194, 2184, 2191, 40951, 40951, 40951, 40951, 40951, 2136, 2134, - 2145, 40951, 2148, 2132, 2140, 2130, 2139, 40951, 2143, 2135, 2142, 2125, - 2147, 2137, 2126, 2146, 2138, 2124, 2131, 2129, 2127, 2128, 2144, 2133, - 2141, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 2199, 2200, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 25385, - 25401, 25416, 25392, 25420, 25419, 25417, 25397, 25414, 25411, 25390, - 25387, 25406, 25403, 25383, 25394, 25398, 25415, 25412, 25391, 25388, - 25407, 25404, 25384, 25395, 25396, 25413, 25410, 25389, 25386, 25405, - 25402, 25382, 25393, 25400, 25399, 25380, 25423, 25409, 25408, 25421, - 25418, 25422, 25381, 40951, 40951, 40951, 40951, 11138, 11089, 11090, - 11091, 11092, 11093, 11094, 11095, 11096, 11097, 11098, 11099, 11100, - 11101, 11102, 11103, 11104, 11105, 11106, 11107, 11108, 11109, 11110, - 11111, 11112, 11113, 11114, 11115, 11116, 11117, 11118, 11119, 11120, - 11121, 11122, 11123, 11124, 11125, 11126, 11127, 11128, 11129, 11130, - 11131, 11132, 11133, 11134, 11135, 11136, 11137, 11188, 11139, 11140, - 11141, 11142, 11143, 11144, 11145, 11146, 11147, 11148, 11149, 11150, - 11151, 11152, 11153, 11154, 11155, 11156, 11157, 11158, 11159, 11160, - 11161, 11162, 11163, 11164, 11165, 11166, 11167, 11168, 11169, 11170, - 11171, 11172, 11173, 11174, 11175, 11176, 11177, 11178, 11179, 11180, - 11181, 11182, 11183, 11184, 11185, 11186, 11187, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 31485, - 31555, 31530, 31526, 31489, 31494, 31514, 31510, 31506, 31559, 31522, - 31563, 31498, 31518, 31502, 40951, 40951, 31556, 31531, 31527, 31490, - 31495, 31515, 31511, 31507, 31560, 31523, 31564, 31499, 31519, 31503, - 31486, 40951, 31557, 31532, 31528, 31491, 31496, 31516, 31512, 31508, - 31561, 31524, 31565, 31500, 31520, 31504, 31484, 40951, 31554, 31529, - 31525, 31488, 31493, 31513, 31509, 31505, 31558, 31521, 31562, 31497, - 31517, 31501, 31487, 31492, 31536, 31533, 31547, 31548, 31549, 31550, - 31551, 31552, 31553, 31537, 31538, 31539, 31540, 31541, 31542, 31543, - 31544, 31545, 31546, 31534, 31535, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 10874, 10873, 10858, 10870, 10867, - 10852, 10849, 10864, 10861, 10876, 10855, 10915, 10894, 6813, 6530, 6554, - 31156, 31157, 31158, 31159, 31160, 31161, 31162, 31163, 31164, 31165, - 31166, 31167, 31168, 31169, 31170, 31171, 31172, 31173, 31174, 31175, - 31176, 31177, 31178, 31179, 31180, 31181, 37498, 6644, 6645, 6541, 6812, - 8646, 34420, 34421, 34422, 34423, 34424, 34425, 34426, 34427, 34428, - 34429, 34430, 34431, 34432, 34433, 34434, 34435, 34436, 34437, 34438, - 34439, 34440, 34441, 34442, 34443, 34444, 34445, 34414, 34450, 34465, - 34466, 34455, 34477, 28977, 28978, 28979, 28980, 28981, 28982, 28983, - 28984, 28985, 28986, 28987, 28988, 28989, 28990, 28991, 28992, 28993, - 28994, 28995, 28996, 28997, 28998, 28999, 29000, 29001, 29002, 31764, - 31765, 31766, 6540, 6539, 6587, 29008, 29009, 29010, 29011, 29012, 29013, - 29014, 29015, 29016, 29017, 29018, 29019, 29020, 29021, 29022, 29023, - 29024, 29025, 29026, 29027, 29028, 29029, 29030, 29031, 29032, 29033, - 8690, 29040, 29043, 29044, 29039, 29041, 34149, 34403, 34402, 34409, - 34478, 34451, 34452, 34454, 34464, 34472, 34475, 34467, 34457, 34469, - 34407, 34471, 34408, 34458, 34468, 34460, 34453, 34419, 34413, 34412, - 34411, 34448, 34459, 34473, 34474, 25815, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 31794, 31795, 31796, 31797, 31798, 31799, 31800, 31801, - 31802, 31803, 31804, 31805, 31806, 31807, 31808, 31809, 31810, 31811, - 31812, 31813, 31814, 31815, 31816, 31817, 31818, 31819, 34190, 34415, - 34417, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 34381, 34376, 34368, 34416, 34362, 34374, - 34397, 34373, 34361, 34386, 34394, 34385, 34366, 34377, 34378, 34384, - 34365, 34395, 34392, 34399, 34375, 34370, 34389, 34379, 34382, 34359, - 34360, 34401, 34372, 34363, 34367, 34383, 34398, 34380, 34393, 34396, - 34369, 34390, 34388, 34387, 34391, 34364, 34371, 34400, 40951, 40951, - 40951, 40951, 37495, 37489, 37490, 37492, 37496, 37493, 37497, 37491, - 37494, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 6592, 6590, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 32230, 32231, 32228, 32232, 32227, 32229, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 10525, 17488, 8037, 29287, 34652, 34651, - 6823, 34697, 31757, 4931, 39127, 38952, 27585, 11538, 11536, 11537, - 18006, 29092, 39039, 17455, 39040, 17529, 39038, 22740, 39037, 8680, - 29091, 17454, 22739, 17528, 34576, 18007, 32825, 36989, 3828, 39284, - 39288, 39285, 39286, 8050, 8049, 8048, 8047, 17487, 39351, 20436, 36648, - 5007, 6476, 32563, 17383, 10551, 31024, 6124, 20433, 37663, 6474, 32218, - 20393, 34698, 4242, 11532, 11533, 20218, 17506, 25667, 17423, 24034, - 28299, 37448, 2541, 18119, 27068, 39130, 35786, 24399, 3426, 31470, - 31791, 18659, 31290, 31287, 6475, 34517, 19108, 33783, 26860, 31606, - 31914, 31915, 8480, 9952, 34505, 34081, 4929, 17519, 32234, 10534, 30882, - 34738, 17527, 17461, 33878, 32769, 20466, 11284, 8479, 6513, 5999, 25270, - 9955, 20399, 32842, 3649, 31604, 8478, 17493, 36650, 32530, 39353, 8054, - 37576, 3540, 7996, 2605, 17494, 4366, 31594, 31905, 39373, 3760, 20825, - 6514, 17429, 17451, 17450, 2748, 31214, 8459, 35785, 8691, 31469, 20830, - 6061, 39350, 28182, 32535, 18044, 19689, 4368, 27584, 31858, 28305, - 34525, 24398, 8471, 3530, 3531, 17444, 98, 6059, 17435, 32176, 17459, - 27574, 28204, 6516, 19688, 2522, 37473, 6821, 37395, 7992, 31319, 38874, - 10920, 33789, 3759, 17718, 4371, 17477, 28540, 28286, 32529, 18675, - 28304, 37581, 38879, 28539, 32359, 36764, 33764, 3432, 6477, 33872, - 32360, 34736, 34112, 37578, 20431, 370, 32237, 34741, 39148, 18043, - 31747, 31748, 8682, 38953, 17465, 20468, 34929, 33869, 5265, 3536, 5006, - 20443, 6822, 10828, 7993, 10659, 10660, 28958, 34503, 20442, 20441, - 29948, 20827, 17349, 20444, 3421, 2538, 20437, 25154, 8474, 32534, 10577, - 17419, 20823, 20826, 17348, 39253, 3881, 39136, 39135, 32219, 3897, - 22564, 2616, 4375, 369, 16743, 16744, 16745, 16746, 16747, 31772, 28199, - 30885, 39128, 8673, 37293, 24296, 31774, 6066, 11374, 8694, 39307, 33866, - 33867, 20430, 31777, 18011, 32848, 28180, 32217, 6482, 11025, 31461, - 4553, 16712, 29946, 34106, 4944, 964, 20398, 22566, 17458, 37577, 4243, - 37688, 19657, 2604, 17523, 3761, 31291, 22561, 31616, 11376, 2614, 11087, - 28201, 8674, 37294, 31775, 6067, 11375, 34111, 20432, 28181, 11086, - 31463, 17526, 19106, 39372, 3535, 31061, 31462, 31280, 6481, 17380, - 17378, 11531, 29441, 28202, 37449, 39295, 39210, 39236, 39260, 17462, - 39137, 30881, 37021, 37022, 7991, 30538, 8696, 39362, 17379, 29282, - 34926, 20927, 11382, 22556, 3765, 39361, 31720, 19112, 31611, 25661, - 2535, 20289, 39363, 39360, 17492, 5002, 5001, 4551, 17906, 25574, 39358, - 17427, 25580, 37703, 37704, 31591, 39359, 4932, 31304, 25577, 25578, - 30517, 30515, 2603, 8462, 31673, 20833, 20832, 18975, 2606, 17367, 348, - 20595, 33766, 20711, 18674, 10533, 25054, 28889, 17415, 18979, 3430, - 34924, 31466, 22553, 25153, 32156, 17722, 22548, 4367, 8672, 39146, 3537, - 4938, 37697, 34082, 18672, 19691, 4245, 18661, 39397, 31719, 19690, - 31901, 19692, 10836, 16700, 962, 4550, 33778, 8059, 34107, 11378, 10538, - 31464, 17472, 11005, 34095, 36996, 39221, 20453, 28002, 9950, 19721, - 8679, 3423, 3425, 3424, 3422, 28001, 6295, 32559, 31314, 4933, 27587, - 17478, 30549, 11529, 17440, 30536, 30889, 30891, 5262, 36651, 6004, 6294, - 6296, 3428, 7997, 31722, 32225, 31282, 34515, 37547, 4256, 24397, 29438, - 29439, 8043, 30530, 18660, 4244, 30553, 4257, 28960, 32556, 27421, 36656, - 30892, 17426, 32443, 31723, 6301, 30831, 20822, 31281, 17381, 20627, - 16781, 8040, 8041, 30540, 30539, 31600, 31597, 29277, 27601, 27602, - 38871, 27603, 29357, 966, 5263, 5264, 38873, 36663, 31750, 38875, 17443, - 31595, 31687, 37691, 8027, 8028, 8024, 975, 25155, 20284, 34090, 34089, - 34091, 34092, 3529, 16698, 24165, 32032, 25112, 8042, 21611, 25109, - 30543, 3543, 3545, 4255, 25052, 31752, 2608, 16776, 30519, 33928, 37488, - 29356, 21626, 20714, 20715, 20718, 20717, 20716, 17447, 16699, 39376, - 19102, 29852, 20445, 31475, 27573, 36662, 8701, 33760, 20831, 37549, - 3908, 39271, 22730, 22657, 22665, 22658, 33795, 33794, 37786, 11297, - 37790, 11291, 25224, 37882, 6537, 8688, 8687, 29431, 29433, 34813, 39234, - 19736, 6132, 30883, 11369, 21609, 28188, 34834, 27284, 4369, 8008, 8018, - 8020, 8004, 8002, 8012, 8010, 8000, 8006, 8014, 7998, 8016, 8009, 8019, - 8021, 8005, 8003, 8013, 8011, 8001, 8007, 8015, 7999, 8017, 31979, 31980, - 31981, 4997, 4998, 32164, 4254, 5998, 25664, 3910, 29355, 20397, 25575, - 33781, 10535, 34102, 34103, 20928, 25579, 24086, 36657, 31960, 39290, - 3923, 36661, 7994, 2609, 34487, 16780, 17486, 31294, 25051, 3878, 25186, - 25172, 25184, 25185, 25206, 24146, 37679, 31762, 31886, 31894, 31895, - 31899, 31896, 31763, 39211, 32951, 32950, 32947, 32946, 3856, 3883, - 32953, 32952, 32949, 32948, 3926, 3822, 3835, 10661, 21613, 37012, 31670, - 31617, 3838, 39225, 33879, 36645, 39356, 30527, 37692, 37008, 37533, - 30341, 19655, 32540, 31671, 17425, 30550, 11011, 11012, 11013, 17516, - 17518, 17517, 3834, 17490, 30537, 6005, 6006, 17441, 16748, 16749, 16750, - 29435, 29436, 29437, 16759, 16752, 16753, 11010, 30888, 30893, 39142, - 34105, 34104, 10662, 27588, 26846, 30874, 8026, 5996, 20629, 10552, 8454, - 30514, 32175, 30890, 34513, 10532, 25053, 34093, 37004, 37003, 37005, - 37006, 24114, 31982, 37707, 37010, 24131, 31996, 24044, 31918, 28185, - 24422, 24421, 2753, 2759, 2754, 2758, 2751, 24419, 2752, 39367, 28198, - 37532, 34500, 33763, 28203, 18664, 18667, 17405, 33854, 33856, 33855, - 33857, 33853, 33852, 39354, 33858, 17385, 31857, 33851, 33861, 33864, - 29089, 17360, 37743, 17388, 31292, 8460, 8461, 22549, 17416, 22550, - 22551, 17402, 17403, 17404, 10922, 39368, 963, 31609, 8700, 31316, 17410, - 10921, 17525, 960, 17431, 39144, 33780, 37394, 18669, 25269, 17393, - 20452, 17395, 17386, 2530, 17479, 33779, 11006, 17413, 17390, 18668, - 6068, 33850, 33849, 6069, 22552, 31608, 8699, 39143, 33785, 33784, 37894, - 17409, 17398, 17392, 31313, 32564, 19694, 34088, 19654, 31311, 31310, - 31306, 31308, 29400, 33984, 29373, 33971, 37677, 37686, 37676, 37685, - 29399, 33983, 29372, 33970, 19772, 19765, 19769, 19762, 29401, 33985, - 29374, 33972, 19773, 19766, 19770, 19763, 20395, 20396, 33924, 33925, - 24289, 37928, 32119, 11364, 32550, 19767, 24401, 19744, 19699, 34739, - 32437, 32438, 32439, 19789, 32440, 19756, 38862, 38859, 6297, 31867, - 32174, 19930, 34504, 31755, 20287, 20288, 37540, 27420, 24408, 34501, - 37536, 37537, 5000, 30524, 37575, 5003, 27589, 371, 17448, 31589, 30523, - 36649, 30522, 2537, 30520, 31790, 10557, 2521, 37534, 28179, 28195, - 34737, 28196, 160, 32824, 32233, 34094, 20423, 38853, 8463, 31590, 37546, - 11370, 29353, 33865, 29358, 31721, 11368, 31602, 29362, 3755, 29351, - 3754, 28197, 31323, 29354, 6479, 27285, 39364, 31862, 2607, 37531, 39126, - 32847, 3527, 3528, 31222, 9954, 2620, 24087, 37543, 31684, 6646, 4552, - 17907, 8671, 33777, 32827, 3547, 3656, 31480, 29945, 32826, 34527, 30894, - 20391, 20455, 16713, 40951, 40951, 40951, 40951, 39357, 31566, 39150, - 32157, 19103, 32820, 29979, 28193, 31754, 28190, 37784, 37781, 37789, - 33792, 29408, 227, 228, 40951, 40951, 40951, 32442, 30521, 10847, 31218, - 32538, 28191, 6001, 33782, 17482, 33769, 2539, 31460, 32177, 40951, - 40951, 40951, 295, 243, 344, 342, 339, 237, 235, 236, 233, 234, 332, 334, - 335, 321, 288, 250, 281, 282, 283, 265, 309, 286, 336, 337, 307, 308, - 270, 323, 276, 277, 259, 300, 256, 278, 319, 257, 258, 255, 310, 317, - 338, 329, 279, 238, 318, 311, 316, 333, 298, 299, 297, 301, 302, 303, - 230, 231, 284, 312, 240, 304, 305, 241, 246, 326, 327, 296, 247, 248, - 249, 232, 343, 322, 328, 271, 340, 289, 254, 331, 253, 325, 252, 330, - 313, 280, 324, 341, 274, 242, 291, 251, 290, 239, 314, 315, 320, 294, - 268, 266, 267, 293, 292, 260, 261, 262, 263, 264, 229, 244, 245, 306, - 275, 287, 269, 285, 273, 272, 25151, 29947, 25272, 40951, 40951, 40951, - 40951, 19684, 25449, 18041, 31741, 30655, 3849, 3927, 3889, 3823, 3913, - 26961, 4251, 19786, 38863, 17364, 39190, 32226, 3921, 3914, 24416, 26986, - 4252, 19783, 38864, 17365, 39272, 39274, 34335, 3919, 3937, 3871, 39205, - 39206, 10835, 3920, 3938, 3872, 39243, 36993, 24418, 26987, 4253, 38868, - 38865, 17366, 36991, 24411, 26979, 4247, 19757, 38860, 17361, 24404, - 26967, 4250, 19732, 38858, 17363, 24412, 26978, 4248, 19761, 38861, - 17362, 24402, 26984, 4249, 19726, 38857, 24414, 26981, 37011, 26980, - 24406, 26966, 17508, 26965, 31868, 24403, 19731, 26977, 19760, 33758, - 26983, 19724, 38856, 19722, 24413, 19779, 19778, 6807, 29003, 6817, - 29004, 29285, 40951, 40951, 40951, 40951, 40951, 40951, 22664, 22732, - 22660, 22728, 22655, 22731, 22659, 22666, 22733, 22661, 22729, 22656, - 40951, 40951, 40951, 40951, 19729, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 24209, 37904, 32072, 11315, 24216, 37901, 32079, 11312, 24203, 37900, - 32069, 11311, 40951, 40951, 40951, 40951, 24208, 37903, 32071, 11314, - 24218, 37905, 32081, 11316, 19740, 19781, 19754, 19718, 19739, 19780, - 19753, 19717, 24253, 37938, 32132, 11335, 24252, 37937, 32131, 11334, - 24251, 37934, 32130, 11331, 24255, 37940, 32134, 11337, 24254, 37939, - 32133, 11336, 24283, 37915, 32098, 11351, 24291, 37930, 32121, 11366, - 24293, 37926, 32154, 11362, 24243, 37924, 32112, 11360, 24244, 37925, - 32113, 11361, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 24292, 37929, 32122, 11365, 29406, 29379, 33977, 33990, 24106, 37771, - 40951, 40951, 40951, 40951, 40951, 40951, 39310, 39325, 39315, 39320, - 39335, 39330, 39340, 39345, 39312, 39327, 39317, 39322, 39337, 39332, - 39342, 39347, 39311, 39326, 39316, 39321, 39336, 39331, 39341, 39346, - 39308, 39323, 39313, 39318, 39333, 39328, 39338, 39343, 39309, 39324, - 39314, 39319, 39334, 39329, 39339, 39344, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 24259, 37944, 32138, 11342, 24273, 37954, - 32153, 11347, 24217, 37902, 32080, 11313, 19695, 19698, 19697, 19696, - 24227, 32087, 24262, 32123, 24284, 32118, 24288, 32114, 24226, 32086, - 24282, 32097, 39154, 39153, 40951, 40951, 2517, 2514, 32067, 11326, - 29034, 29035, 29042, 29036, 29398, 29370, 33969, 33982, 40951, 40951, - 40951, 40951, 24223, 32058, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 6535, 6536, 6534, 24071, 24069, - 24070, 24072, 24068, 11304, 11306, 11305, 11307, 31467, 39226, 4940, - 31468, 40760, 28003, 17406, 29283, 36994, 17389, 32235, 20454, 33626, - 5261, 31771, 24180, 32021, 19113, 19109, 20824, 17387, 8051, 28959, - 32179, 11379, 25330, 17414, 33868, 17397, 18666, 18665, 17408, 32659, - 33859, 17394, 32845, 31618, 4928, 31059, 32667, 31672, 25576, 28189, - 32850, 31301, 20930, 17437, 27604, 39375, 39129, 19111, 11001, 39349, - 11380, 7995, 37689, 34110, 18010, 32168, 17457, 32562, 36995, 4548, - 25738, 9949, 22565, 33881, 17485, 8695, 2597, 9956, 2615, 31603, 6064, - 2618, 18662, 32671, 34526, 16648, 18005, 31288, 22554, 31060, 11526, - 17500, 35368, 6512, 4370, 9942, 8055, 4939, 31478, 31668, 9957, 32441, - 6000, 24035, 25665, 28183, 2619, 33860, 39396, 33862, 17399, 17412, - 30873, 17521, 29286, 10925, 17417, 17401, 32531, 22563, 18040, 20392, - 17466, 8702, 25106, 32536, 37665, 37757, 11541, 11527, 3470, 32768, - 30884, 17513, 5005, 10830, 18039, 25107, 31903, 32849, 34480, 17908, - 40754, 20283, 32527, 34927, 8681, 21216, 25336, 31285, 20394, 31217, - 31749, 25268, 28187, 27576, 2617, 34740, 25573, 11371, 33790, 30830, - 30555, 33768, 17471, 30879, 3538, 3766, 32558, 18676, 31685, 16739, - 16740, 16742, 16741, 4554, 24400, 17491, 37450, 34733, 34734, 32370, - 11534, 28192, 25662, 26861, 26862, 6300, 9944, 32373, 3655, 17717, 30529, - 17424, 38977, 5004, 26825, 20467, 4942, 37574, 34502, 22558, 10829, - 17391, 101, 6478, 30516, 3534, 31309, 31303, 31312, 31302, 25340, 17430, - 38517, 27409, 16737, 17903, 40946, 4926, 30554, 3758, 32533, 18008, 8677, - 33876, 31792, 17452, 20933, 36767, 31322, 11528, 8457, 2599, 17449, - 37452, 4935, 25339, 25271, 25150, 34109, 2761, 32371, 36655, 4941, 3431, - 32178, 3429, 34113, 31778, 28961, 29066, 29077, 29080, 29069, 29059, - 29074, 39165, 3791, 29060, 39162, 39178, 39181, 39157, 39170, 39175, - 3788, 3804, 3807, 3783, 3796, 3801, 29067, 29078, 29081, 29070, 29065, - 29075, 39166, 3792, 29061, 39184, 39187, 39188, 39183, 39185, 39186, - 3810, 3813, 3814, 3809, 3811, 3812, 29084, 29087, 29088, 29083, 29085, - 29086, 39167, 3793, 29062, 39163, 39179, 39182, 39158, 39168, 39176, - 3789, 3805, 3808, 3784, 3794, 3802, 29068, 29079, 29082, 29071, 29063, - 29076, 39169, 3795, 29064, 39159, 3785, 29072, 39160, 3786, 29073, 39172, - 39173, 39171, 3798, 3799, 3797, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 39391, 39394, 39390, 39392, - 39389, 39388, 39393, 39384, 39387, 39383, 39385, 39382, 39381, 39386, - 40951, 40951, 2762, 30528, 4934, 32843, 36997, 24415, 18663, 31472, - 11377, 99, 34507, 39380, 8698, 40951, 40951, 40951, 40668, 22557, 31068, - 4258, 25338, 31473, 29056, 25668, 17480, 19656, 40951, 40951, 40951, - 40951, 40951, 32846, 32163, 6135, 31776, 2602, 11004, 3427, 27583, 1, - 25135, 8676, 6062, 32539, 22567, 20447, 27599, 39352, 31586, 32664, - 22559, 5008, 28200, 37451, 19686, 31481, 32173, 27600, 20473, 25156, - 19107, 17489, 18977, 21696, 17481, 39369, 3541, 8053, 31605, 39371, - 17432, 25152, 8650, 16751, 29057, 20464, 20926, 39355, 24033, 18042, 956, - 25273, 31324, 31620, 31619, 31307, 17445, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 28186, 11189, 4246, 3544, 30518, 17467, 35784, - 17512, 36654, 31607, 3539, 20925, 17905, 31289, 32216, 40951, 40951, - 34108, 27067, 32375, 17396, 17400, 17411, 11199, 3763, 4943, 32819, - 17407, 40951, 40951, 40951, 40951, 40951, 40951, 19110, 32111, 24242, - 31025, 31026, 20634, 19693, 24287, 32117, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 4182, 4212, 4183, 4227, 4198, 4216, 4184, 4235, - 4205, 4213, 4191, 4228, 4199, 4217, 4185, 4239, 4209, 4224, 4195, 4232, - 4221, 4188, 4236, 4206, 4214, 4192, 4229, 4200, 4218, 4186, 4241, 4211, - 4226, 4197, 4234, 4204, 4223, 4190, 4238, 4208, 4194, 4231, 4202, 4220, - 4187, 4240, 4210, 4225, 4196, 4233, 4203, 4222, 4189, 4237, 4207, 4215, - 4193, 4230, 4201, 4219, 25173, 25174, 25181, 25183, 25178, 25239, 25240, - 25236, 25238, 25234, 25237, 25230, 25233, 25231, 25235, 25232, 25177, - 25180, 25175, 25179, 25176, 25182, 37854, 37855, 37862, 37864, 37859, - 37824, 37825, 37821, 37823, 37819, 37822, 37815, 37818, 37816, 37820, - 37817, 37858, 37861, 37856, 37860, 37857, 37863, 37797, 24036, 37794, - 24041, 24127, 37893, 32004, 25265, 38841, 38842, 38843, 38844, 38845, - 38846, 20409, 20410, 20411, 20412, 20413, 20414, 24037, 24042, 31917, - 31916, 37796, 20408, 37852, 37889, 37804, 37892, 37888, 31966, 32000, - 31944, 31999, 31976, 24085, 31959, 37811, 25166, 20809, 37806, 37808, - 40951, 24084, 6298, 20805, 19733, 37829, 37884, 37795, 24038, 37830, - 37885, 25226, 25203, 4454, 4458, 4446, 4452, 4455, 4460, 4447, 4449, - 4457, 4459, 4461, 4456, 4450, 4451, 4477, 4487, 2520, 20806, 24079, - 31954, 20807, 24196, 32055, 11321, 37897, 24076, 31951, 38951, 31968, - 29006, 29005, 29007, 39229, 24130, 27578, 31995, 29037, 34508, 34509, - 34511, 34512, 34510, 39297, 39202, 31767, 3903, 24137, 24092, 4453, 4474, - 4469, 4448, 4464, 4463, 4471, 4462, 4467, 4473, 4444, 4468, 4465, 4475, - 4445, 4470, 37476, 31962, 4364, 24151, 37803, 25250, 27579, 27580, 37475, - 31961, 4363, 24150, 37485, 4350, 4357, 37477, 32572, 32574, 32571, 32570, - 32567, 32566, 32569, 32568, 32575, 32573, 40951, 40951, 40951, 40951, - 40951, 40951, 6854, 6855, 6856, 6857, 6858, 6859, 6860, 6861, 6862, 6863, - 6864, 6865, 6866, 6867, 6868, 6869, 6870, 6871, 6872, 6873, 6874, 6875, - 6876, 6877, 6878, 6879, 6880, 6881, 6882, 6883, 6884, 6885, 6886, 6887, - 6888, 6889, 6890, 6891, 6892, 6893, 6894, 6895, 6896, 6897, 6898, 6899, - 6900, 6901, 6902, 6903, 6904, 6905, 6906, 6907, 6908, 6909, 6910, 6911, - 6912, 6913, 6914, 6915, 6916, 6917, 6918, 6919, 6920, 6921, 6922, 6923, - 6924, 6925, 6926, 6927, 6928, 6929, 6930, 6931, 6932, 6933, 6934, 6935, - 6936, 6937, 6938, 6939, 6940, 6941, 6942, 6943, 6944, 6945, 6946, 6947, - 6948, 6949, 6950, 6951, 6952, 6953, 6954, 6955, 6956, 6957, 6958, 6959, - 6960, 6961, 6962, 6963, 6964, 6965, 6966, 6967, 6968, 6969, 6970, 6971, - 6972, 6973, 6974, 6975, 6976, 6977, 6978, 6979, 6980, 6981, 6982, 6983, - 6984, 6985, 6986, 6987, 6988, 6989, 6990, 6991, 6992, 6993, 6994, 6995, - 6996, 6997, 6998, 6999, 7000, 7001, 7002, 7003, 7004, 7005, 7006, 7007, - 7008, 7009, 7010, 7011, 7012, 7013, 7014, 7015, 7016, 7017, 7018, 7019, - 7020, 7021, 7022, 7023, 7024, 7025, 7026, 7027, 7028, 7029, 7030, 7031, - 7032, 7033, 7034, 7035, 7036, 7037, 7038, 7039, 7040, 7041, 7042, 7043, - 7044, 7045, 7046, 7047, 7048, 7049, 7050, 7051, 7052, 7053, 7054, 7055, - 7056, 7057, 7058, 7059, 7060, 7061, 7062, 7063, 7064, 7065, 7066, 7067, - 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, 7077, 7078, 7079, - 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, 7089, 7090, 7091, - 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, 7101, 7102, 7103, - 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, 7113, 7114, 7115, - 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, 7125, 7126, 7127, - 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, 7137, 7138, 7139, - 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7147, 7148, 7149, 7150, 7151, - 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160, 7161, 7162, 7163, - 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, 7173, 7174, 7175, - 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, 7184, 7185, 7186, 7187, - 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, 7196, 7197, 7198, 7199, - 7200, 7201, 7202, 7203, 7204, 7205, 7206, 7207, 7208, 7209, 7210, 7211, - 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, 7220, 7221, 7222, 7223, - 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, 7232, 7233, 7234, 7235, - 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, 7244, 7245, 7246, 7247, - 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, 7256, 7257, 7258, 7259, - 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, 7268, 7269, 7270, 7271, - 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, 7280, 7281, 7282, 7283, - 7284, 7285, 7286, 7287, 7288, 7289, 7290, 7291, 7292, 7293, 7294, 7295, - 7296, 7297, 7298, 7299, 7300, 7301, 7302, 7303, 7304, 7305, 7306, 7307, - 7308, 7309, 7310, 7311, 7312, 7313, 7314, 7315, 7316, 7317, 7318, 7319, - 7320, 7321, 7322, 7323, 7324, 7325, 7326, 7327, 7328, 7329, 7330, 7331, - 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339, 7340, 7341, 7342, 7343, - 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, 7353, 7354, 7355, - 7356, 7357, 7358, 7359, 7360, 7361, 7362, 7363, 7364, 7365, 6838, 6839, - 6840, 6841, 6842, 6843, 6844, 6845, 6846, 6847, 6848, 6849, 6850, 6851, - 6852, 6853, 6824, 6825, 6826, 6827, 6828, 6829, 6830, 6831, 6832, 6833, - 6834, 6835, 6836, 6837, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 24032, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 35020, 34949, 35012, 35021, 34937, 35010, 34931, 34930, 35007, - 35015, 34932, 35011, 34935, 34952, 35023, 35019, 34944, 34946, 34943, - 34942, 34939, 34938, 34941, 34940, 34947, 34945, 34936, 35018, 35005, - 34948, 34951, 35013, 34934, 34953, 34954, 34955, 34956, 34957, 34958, - 34959, 34960, 34961, 34962, 34963, 34964, 34965, 34966, 34967, 34968, - 34969, 34970, 34971, 34972, 34973, 34974, 34975, 34976, 34977, 34978, - 35008, 35017, 35016, 34933, 35009, 34950, 34979, 34980, 34981, 34982, - 34983, 34984, 34985, 34986, 34987, 34988, 34989, 34990, 34991, 34992, - 34993, 34994, 34995, 34996, 34997, 34998, 34999, 35000, 35001, 35002, - 35003, 35004, 35006, 35024, 35014, 35022, 5995, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 38339, 38350, 38361, 38373, 38384, - 38395, 38406, 38417, 38428, 38436, 38437, 38438, 38439, 38441, 38442, - 38443, 38444, 38445, 38446, 38447, 38448, 38449, 38450, 38452, 38453, - 38454, 38455, 38456, 38457, 38458, 38459, 38460, 38461, 38463, 38464, - 38465, 38466, 38467, 38468, 38469, 38470, 38471, 38472, 38474, 38475, - 38476, 38477, 38478, 38479, 38480, 38481, 38482, 38483, 38485, 38486, - 38487, 38488, 38489, 38490, 38491, 38492, 38493, 38494, 38496, 38497, - 38498, 38499, 38500, 38501, 38502, 38503, 38504, 38505, 38507, 38508, - 38509, 38510, 38511, 38512, 38513, 38514, 38515, 38516, 38263, 38264, - 38265, 38266, 38267, 38268, 38269, 38270, 38271, 38272, 38274, 38275, - 38276, 38277, 38278, 38279, 38280, 38281, 38282, 38283, 38285, 38286, - 38287, 38288, 38289, 38290, 38291, 38292, 38293, 38294, 38296, 38297, - 38298, 38299, 38300, 38301, 38302, 38303, 38304, 38305, 38307, 38308, - 38309, 38310, 38311, 38312, 38313, 38314, 38315, 38316, 38318, 38319, - 38320, 38321, 38322, 38323, 38324, 38325, 38326, 38327, 38329, 38330, - 38331, 38332, 38333, 38334, 38335, 38336, 38337, 38338, 38340, 38341, - 38342, 38343, 38344, 38345, 38346, 38347, 38348, 38349, 38351, 38352, - 38353, 38354, 38355, 38356, 38357, 38358, 38359, 38360, 38362, 38363, - 38364, 38365, 38366, 38367, 38368, 38369, 38370, 38371, 38374, 38375, - 38376, 38377, 38378, 38379, 38380, 38381, 38382, 38383, 38385, 38386, - 38387, 38388, 38389, 38390, 38391, 38392, 38393, 38394, 38396, 38397, - 38398, 38399, 38400, 38401, 38402, 38403, 38404, 38405, 38407, 38408, - 38409, 38410, 38411, 38412, 38413, 38414, 38415, 38416, 38418, 38419, - 38420, 38421, 38422, 38423, 38424, 38425, 38426, 38427, 38429, 38430, - 38431, 38432, 38433, 38434, 38435, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 29854, 29853, 34498, 34077, 34499, 34529, 16774, 17347, 16772, - 16785, 16778, 16777, 3, 2, 347, 3542, 2612, 5258, 6290, 20419, 20449, - 34928, 24495, 29177, 16775, 25337, 29928, 16783, 24497, 38850, 38955, - 17497, 17647, 6063, 8678, 32821, 25111, 33874, 32822, 25110, 32853, - 10543, 11530, 10827, 10544, 10826, 10545, 10825, 10546, 10823, 10547, - 29045, 28963, 34833, 34832, 16773, 17346, 5993, 5266, 16771, 16784, - 16738, 34560, 34530, 16819, 16818, 20706, 17442, 17645, 20707, 18670, - 18976, 20708, 31789, 32368, 20709, 37754, 37960, 34079, 10559, 10556, - 30887, 30886, 20286, 20448, 4927, 5257, 29350, 28965, 20633, 20632, - 29279, 29284, 34495, 34483, 16770, 16822, 6292, 20421, 20451, 6291, - 20420, 20450, 24498, 38851, 38956, 31210, 31209, 31587, 31211, 31212, - 31567, 31869, 31876, 31904, 33639, 33640, 34481, 33638, 33641, 34482, - 10824, 10548, 31676, 31677, 31724, 31675, 31678, 31725, 32662, 34528, - 5994, 10528, 27410, 28789, 34494, 34497, 34080, 16769, 16767, 17384, - 34496, 34078, 33633, 34925, 33632, 32557, 8472, 10527, 34519, 34484, - 30547, 30768, 31674, 31726, 1059, 1060, 28964, 32852, 22893, 23515, - 10526, 2313, 361, 34911, 21230, 22570, 22571, 22613, 22579, 37100, 19375, - 19376, 19353, 19374, 17641, 17642, 17643, 28787, 17644, 34585, 40949, - 40948, 40950, 25333, 32171, 25331, 32169, 31283, 25334, 32172, 29927, - 28788, 39378, 25332, 32170, 17646, 31284, 39147, 27571, 27572, 24248, - 32127, 39885, 28575, 38518, 38629, 38697, 38708, 38719, 38730, 38741, - 38752, 38763, 38519, 38530, 38541, 38552, 38563, 38574, 38585, 31643, - 5256, 4549, 40947, 9642, 9641, 9337, 2827, 26869, 26867, 26927, 26925, - 20199, 5093, 27245, 27248, 38596, 38607, 38618, 38630, 38641, 38652, - 38663, 38674, 38685, 38693, 38694, 38695, 38696, 38698, 38699, 38700, - 38701, 38702, 38703, 38704, 38705, 38706, 38707, 38709, 38710, 38711, - 38712, 38713, 38714, 38715, 38716, 38717, 38718, 38720, 38721, 38722, - 38723, 38724, 38725, 38726, 38727, 38728, 38729, 38731, 38732, 38733, - 38734, 38735, 38736, 38737, 38738, 38739, 38740, 38742, 38743, 38744, - 38745, 38746, 38747, 38748, 38749, 38750, 38751, 38753, 38754, 38755, - 38756, 38757, 38758, 38759, 38760, 38761, 38762, 38764, 38765, 38766, - 38767, 38768, 38769, 38770, 38771, 38772, 38773, 38520, 38521, 38522, - 38523, 38524, 38525, 38526, 38527, 38528, 38529, 38531, 38532, 38533, - 38534, 38535, 38536, 38537, 38538, 38539, 38540, 38542, 38543, 38544, - 38545, 38546, 38547, 38548, 38549, 38550, 38551, 38553, 38554, 38555, - 38556, 38557, 38558, 38559, 38560, 38561, 38562, 38564, 38565, 38566, - 38567, 38568, 38569, 38570, 38571, 38572, 38573, 38575, 38576, 38577, - 38578, 38579, 38580, 38581, 38582, 38583, 38584, 38586, 38587, 38588, - 38589, 38590, 38591, 38592, 38593, 38594, 38595, 38597, 38598, 38599, - 38600, 38601, 38602, 38603, 38604, 38605, 38606, 38608, 38609, 38610, - 38611, 38612, 38613, 38614, 38615, 38616, 38617, 38619, 38620, 38621, - 38622, 38623, 38624, 38625, 38626, 38627, 38628, 38631, 38632, 38633, - 38634, 38635, 38636, 38637, 38638, 38639, 38640, 38642, 38643, 38644, - 38645, 38646, 38647, 38648, 38649, 38650, 38651, 38653, 38654, 38655, - 38656, 38657, 38658, 38659, 38660, 38661, 38662, 38664, 38665, 38666, - 38667, 38668, 38669, 38670, 38671, 38672, 38673, 38675, 38676, 38677, - 38678, 38679, 38680, 38681, 38682, 38683, 38684, 38686, 38687, 38688, - 38689, 38690, 38691, 38692, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 21614, - 21615, 21622, 21624, 21621, 21620, 21617, 21616, 21619, 21618, 21625, - 21623, 22758, 23321, 22917, 23546, 23156, 23919, 22852, 23456, 22853, - 23457, 22854, 23458, 23028, 23697, 23029, 23698, 23030, 23699, 23095, - 23795, 22836, 23439, 22832, 23435, 23541, 23665, 22768, 23331, 22769, - 23332, 22857, 23462, 22858, 23463, 22842, 23445, 22843, 23446, 23540, - 23542, 22922, 23548, 22923, 23549, 22940, 23576, 22970, 23616, 22978, - 23638, 23068, 23756, 23160, 23923, 23161, 23924, 23157, 23920, 23158, - 23921, 23340, 23714, 23715, 23874, 23875, 23804, 23805, 23530, 23531, - 2283, 2286, 2284, 2288, 2290, 2287, 2289, 2285, 2291, 10756, 10751, - 10750, 10757, 10752, 10753, 10755, 10754, 3614, 3613, 3615, 18884, 18883, - 18882, 18881, 18886, 18885, 30619, 30618, 3559, 35374, 35382, 35391, - 35383, 35385, 35380, 35384, 35379, 35395, 35394, 35397, 35389, 35376, - 35396, 35378, 35377, 35390, 35381, 35393, 35387, 35388, 35386, 35392, - 35375, 35513, 35520, 35521, 35516, 35517, 35522, 35523, 35514, 35518, - 35519, 35515, 35579, 35586, 35587, 35582, 35583, 35588, 35589, 35580, - 35584, 35585, 35581, 35690, 35697, 35698, 35693, 35694, 35699, 35700, - 35691, 35695, 35696, 35692, 35590, 35597, 35598, 35593, 35594, 35599, - 35600, 35591, 35595, 35596, 35592, 35668, 35675, 35676, 35671, 35672, - 35677, 35678, 35669, 35673, 35674, 35670, 35568, 35575, 35576, 35571, - 35572, 35577, 35578, 35569, 35573, 35574, 35570, 35679, 35686, 35687, - 35682, 35683, 35688, 35689, 35680, 35684, 35685, 35681, 35601, 35608, - 35609, 35604, 35605, 35610, 35611, 35602, 35606, 35607, 35603, 35734, - 35741, 35742, 35737, 35738, 35743, 35744, 35735, 35739, 35740, 35736, - 35723, 35730, 35731, 35726, 35727, 35732, 35733, 35724, 35728, 35729, - 35725, 35756, 35763, 35764, 35759, 35760, 35765, 35766, 35757, 35761, - 35762, 35758, 35623, 35630, 35631, 35626, 35627, 35632, 35633, 35624, - 35628, 35629, 35625, 35546, 35553, 35554, 35549, 35550, 35555, 35556, - 35547, 35551, 35552, 35548, 35745, 35752, 35753, 35748, 35749, 35754, - 35755, 35746, 35750, 35751, 35747, 35524, 35531, 35532, 35527, 35528, - 35533, 35534, 35525, 35529, 35530, 35526, 35535, 35542, 35543, 35538, - 35539, 35544, 35545, 35536, 35540, 35541, 35537, 35612, 35619, 35620, - 35615, 35616, 35621, 35622, 35613, 35617, 35618, 35614, 35557, 35564, - 35565, 35560, 35561, 35566, 35567, 35558, 35562, 35563, 35559, 35712, - 35719, 35720, 35715, 35716, 35721, 35722, 35713, 35717, 35718, 35714, - 35634, 35642, 35643, 35637, 35638, 35644, 35645, 35635, 35639, 35640, - 35636, 35646, 35653, 35654, 35649, 35650, 35655, 35656, 35647, 35651, - 35652, 35648, 35657, 35664, 35665, 35660, 35661, 35666, 35667, 35658, - 35662, 35663, 35659, 35701, 35708, 35709, 35704, 35705, 35710, 35711, - 35702, 35706, 35707, 35703, 35501, 35502, 35509, 35510, 35505, 35506, - 35511, 35512, 35503, 35507, 35508, 35504, 35641, 33664, 33662, 33663, - 17807, 22179, 22177, 22180, 22178, 22169, 22175, 22173, 22176, 22174, - 22170, 22190, 22186, 22191, 22187, 22171, 22188, 22184, 22189, 22185, - 22172, 22201, 22181, 22183, 22182, 22197, 22200, 22198, 22193, 22199, - 22194, 22195, 22196, 22202, 22192, 22341, 22218, 22219, 22220, 22217, - 22336, 22328, 20316, 20318, 20320, 20317, 20319, 21319, 21321, 21323, - 21320, 21322, 21312, 21311, 21310, 21313, 27780, 27785, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, - 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, 40951, + 7575, 7576, 7577, 7578, 7579, 7580, 7471, 7472, 7473, 7474, 7475, 7476, + 7477, 7478, 7479, 7480, 7481, 7482, 7483, 7484, 41412, 41412, 7581, 7582, + 7583, 7584, 7585, 7586, 7587, 7588, 7589, 7590, 7591, 7592, 7593, 7594, + 7595, 7596, 7597, 7598, 7599, 7600, 7601, 7602, 7603, 7604, 7605, 7606, + 7607, 7608, 7609, 7610, 7611, 7612, 7613, 7614, 7615, 7616, 7617, 7618, + 7619, 7620, 7621, 7622, 7623, 7624, 7625, 7626, 7627, 7628, 7629, 7630, + 7631, 7632, 7633, 7634, 7635, 7636, 7637, 7638, 7639, 7640, 7641, 7642, + 7643, 7644, 7645, 7646, 7647, 7648, 7649, 7650, 7651, 7652, 7653, 7654, + 7655, 7656, 7657, 7658, 7659, 7660, 7661, 7662, 7663, 7664, 7665, 7666, + 7667, 7668, 7669, 7670, 7671, 7672, 7673, 7674, 7675, 7676, 7677, 7678, + 7679, 7680, 7681, 7682, 7683, 7684, 7685, 7686, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 24174, 24177, 24178, 24175, 24176, + 24180, 24181, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 2549, 2550, 2548, 2551, 2547, 41412, 41412, + 41412, 41412, 41412, 20049, 20076, 20054, 19985, 20041, 20044, 20046, + 20045, 20040, 20047, 20043, 20042, 19986, 20019, 20020, 20017, 20018, + 19983, 19984, 19982, 19990, 20032, 20030, 20007, 20039, 20012, 41412, + 20024, 20050, 19998, 19993, 20034, 41412, 20036, 41412, 20010, 20014, + 41412, 20000, 19996, 41412, 20028, 20005, 20022, 20016, 20026, 20038, + 19989, 19992, 19995, 20051, 1164, 1163, 1194, 1192, 1193, 1195, 1424, + 1422, 1423, 1425, 1189, 1187, 1188, 1190, 1555, 1553, 1554, 1556, 1534, + 1532, 1533, 1535, 1550, 1548, 1549, 1551, 1568, 1566, 1567, 1569, 1429, + 1427, 1428, 1430, 1230, 1228, 1229, 1231, 1401, 1399, 1400, 1402, 1511, + 1509, 1510, 1512, 1516, 1514, 1515, 1517, 1220, 1219, 1211, 1210, 1234, + 1233, 1223, 1222, 1330, 1329, 1459, 1458, 1353, 1351, 1352, 1354, 1264, + 1262, 1263, 1265, 1276, 1274, 1275, 1277, 1392, 1390, 1391, 1393, 1406, + 1405, 1463, 1461, 1462, 1464, 1306, 1305, 1301, 1299, 1300, 1302, 1310, + 1308, 1309, 1311, 1592, 1591, 1590, 1589, 2411, 2410, 2419, 2418, 2415, + 2414, 2413, 2412, 2423, 2422, 2409, 2417, 2416, 2425, 2421, 2420, 2424, + 1756, 1704, 1933, 1930, 1931, 1926, 1927, 1928, 1920, 1733, 1734, 1731, + 1732, 1956, 1645, 1649, 1396, 1394, 1395, 1397, 1561, 1560, 1614, 1613, + 1611, 1610, 1559, 1571, 1570, 1357, 1356, 1360, 1359, 1627, 1625, 1626, + 1628, 1562, 1563, 2100, 2099, 2102, 2101, 2121, 2120, 2129, 2128, 2119, + 2118, 2125, 2124, 2105, 2103, 2104, 2154, 2152, 2153, 1244, 1242, 1243, + 1245, 2111, 2109, 2115, 2098, 2123, 1675, 1666, 1671, 1678, 1673, 1684, + 2063, 2051, 2058, 2071, 2074, 2079, 2081, 2086, 2083, 2092, 1760, 1766, + 1743, 1738, 1799, 1795, 1801, 1970, 1963, 1975, 1983, 1940, 1944, 1697, + 1689, 1693, 1699, 2044, 2039, 2156, 1640, 1636, 1728, 1724, 1712, 1717, + 1708, 1715, 1710, 1719, 1905, 1901, 1903, 1907, 1774, 1776, 1784, 1782, + 1779, 1790, 1772, 1793, 1827, 1819, 1831, 1837, 1811, 1840, 1858, 1847, + 1852, 1862, 1841, 1863, 1879, 1869, 1891, 1884, 1887, 1894, 1753, 1749, + 1750, 1751, 2139, 2132, 2141, 2147, 2096, 2151, 2080, 1935, 1658, 1991, + 1994, 1998, 1992, 1995, 1997, 2127, 2126, 2113, 2117, 2097, 2122, 1682, + 1681, 1676, 1680, 1672, 1683, 2077, 2076, 2069, 2075, 2073, 2078, 2090, + 2089, 2084, 2088, 2082, 2091, 1709, 1718, 1902, 1906, 1773, 1777, 1788, + 1771, 1792, 1835, 1810, 1839, 1842, 1860, 1892, 1889, 1882, 1888, 1886, + 1893, 1657, 2149, 2136, 2145, 2135, 2095, 2150, 2110, 2108, 2112, 2114, + 2106, 1674, 1665, 1670, 1677, 1667, 2062, 2050, 2057, 2070, 2052, 2085, + 1759, 1765, 1742, 1737, 1798, 1800, 1969, 1962, 1974, 1982, 1939, 1947, + 1943, 1696, 1688, 1692, 1698, 2043, 2155, 1639, 1635, 1727, 1723, 1711, + 1716, 1707, 1714, 1904, 1900, 1775, 1783, 1781, 1778, 1789, 1826, 1818, + 1830, 1836, 1820, 1857, 1846, 1851, 1861, 1878, 1868, 1890, 1883, 1870, + 1752, 1748, 1754, 2138, 2131, 2140, 2146, 2133, 2116, 2107, 1679, 1668, + 2072, 2053, 2087, 2093, 1984, 1966, 2021, 2001, 1780, 1791, 1838, 1885, + 1871, 2148, 2134, 1999, 1993, 1996, 2042, 2046, 1642, 1644, 1726, 1730, + 1986, 1990, 2023, 2031, 1740, 1745, 1768, 1770, 1797, 1803, 1946, 1951, + 1695, 1703, 2012, 2007, 2026, 2020, 2029, 1988, 1949, 1701, 2041, 2045, + 1641, 1643, 1725, 1729, 1985, 1989, 2022, 2030, 1739, 1744, 1767, 1769, + 1796, 1802, 1945, 1950, 1694, 1702, 2010, 2005, 2024, 2018, 2028, 1987, + 1948, 1700, 2011, 2006, 2025, 2019, 1965, 2000, 2038, 1971, 1964, 1976, + 2013, 2008, 2027, 2040, 2157, 1659, 1660, 30832, 30833, 1923, 1914, 1918, + 1915, 1916, 1917, 1957, 1648, 1653, 1651, 1647, 1912, 1959, 1655, 2033, + 1925, 2060, 2049, 2048, 2047, 2055, 2065, 2067, 2066, 1762, 1761, 1736, + 1735, 1961, 1968, 1967, 1978, 1977, 1979, 1981, 1980, 1937, 1936, 1942, + 2003, 2002, 2009, 2015, 2014, 2017, 2016, 1686, 1691, 1690, 2035, 2034, + 2036, 2037, 1638, 1633, 1632, 1631, 1720, 1722, 1721, 1706, 1705, 1898, + 1896, 1816, 1817, 1814, 1821, 1822, 1829, 1828, 1833, 1832, 1843, 1844, + 1845, 1855, 1853, 1848, 1849, 1929, 1932, 1854, 1746, 1747, 1866, 1865, + 1876, 1875, 1874, 1881, 1880, 2143, 2142, 1669, 2061, 2059, 2056, 2054, + 2068, 2064, 1764, 1757, 1763, 1972, 1938, 2004, 1687, 1825, 1834, 2130, + 2137, 2144, 1859, 1899, 1867, 1897, 1815, 1634, 1787, 1872, 1850, 1823, + 1786, 1824, 1873, 1758, 1741, 1856, 1713, 1664, 1785, 1637, 1941, 1973, + 1877, 1924, 1919, 1922, 1921, 1958, 1646, 1794, 1953, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 1954, 1908, 1661, 1662, 1864, 1952, 1934, 1656, 2094, 1955, 1960, 1755, + 32353, 1685, 2032, 1663, 38715, 38826, 38894, 38905, 38916, 38927, 38938, + 38949, 38960, 38716, 38727, 38738, 38749, 38760, 38771, 38782, 31807, + 31812, 31813, 31806, 31837, 31808, 31839, 31815, 31829, 31811, 41412, + 41412, 41412, 41412, 41412, 41412, 8479, 8481, 8306, 8307, 8490, 8492, + 8192, 8480, 8482, 8555, 8556, 8491, 8493, 8193, 8246, 8247, 31838, 31809, + 31810, 31824, 31836, 31821, 31833, 31819, 31831, 31823, 31835, 31816, + 31825, 31817, 31826, 31820, 31832, 31818, 31830, 31814, 31827, 32848, + 39721, 31822, 31834, 10673, 6231, 39597, 11389, 10672, 6230, 39595, + 34023, 34032, 34067, 41412, 34057, 34024, 34068, 34030, 34031, 34034, + 34038, 34033, 34037, 34035, 34039, 34066, 34014, 34016, 34065, 34063, + 34036, 34062, 34029, 41412, 34056, 34025, 34064, 34022, 41412, 41412, + 41412, 41412, 1098, 2430, 1079, 2432, 1110, 41412, 1095, 1096, 1075, + 1076, 1107, 1108, 2335, 2336, 2405, 2406, 1295, 1159, 1158, 1149, 1148, + 1579, 1578, 1152, 1151, 1596, 1594, 1595, 1597, 1169, 1168, 1184, 1182, + 1183, 1185, 1521, 1520, 1530, 1528, 1529, 1523, 1544, 1542, 1543, 1545, + 1326, 1324, 1325, 1327, 1292, 1290, 1291, 1293, 1364, 1362, 1363, 1365, + 1208, 1207, 1538, 1537, 1456, 1455, 1622, 1621, 1485, 1483, 1484, 1486, + 1491, 1489, 1490, 1492, 1472, 1470, 1471, 1473, 1216, 1214, 1215, 1217, + 1504, 1502, 1503, 1505, 1618, 1616, 1617, 1619, 1127, 1125, 1126, 1128, + 1271, 1269, 1270, 1272, 1255, 1253, 1254, 1256, 1438, 1436, 1437, 1439, + 1340, 1338, 1339, 1341, 1376, 1374, 1375, 1377, 1385, 1383, 1384, 1386, + 1417, 1415, 1416, 1418, 1314, 1312, 1313, 1315, 1583, 1582, 1167, 1166, + 1607, 1605, 1606, 1608, 1809, 1808, 1805, 1804, 1807, 1806, 1813, 1812, + 41412, 41412, 41216, 41412, 17766, 17770, 17738, 17754, 17740, 17752, + 17751, 17681, 17744, 17753, 17741, 17676, 17769, 17774, 17748, 17761, + 17763, 17760, 17759, 17756, 17755, 17758, 17757, 17764, 17762, 17677, + 17747, 17683, 17765, 17768, 17771, 17675, 17684, 17685, 17686, 17687, + 17688, 17689, 17690, 17691, 17692, 17693, 17694, 17695, 17696, 17697, + 17698, 17699, 17700, 17701, 17702, 17703, 17704, 17705, 17706, 17707, + 17708, 17709, 17682, 17746, 17745, 17674, 17736, 17767, 17710, 17711, + 17712, 17713, 17714, 17715, 17716, 17717, 17718, 17719, 17720, 17721, + 17722, 17723, 17724, 17725, 17726, 17727, 17728, 17729, 17730, 17731, + 17732, 17733, 17734, 17735, 17680, 17776, 17743, 17773, 17679, 17742, + 19256, 19249, 19251, 19255, 19247, 19239, 19194, 19196, 19198, 19195, + 19197, 19190, 19192, 19191, 19193, 19248, 19240, 19242, 19244, 19241, + 19243, 19215, 19217, 19219, 19216, 19218, 19199, 19201, 19203, 19200, + 19202, 19230, 19232, 19234, 19231, 19233, 19205, 19207, 19209, 19206, + 19208, 19210, 19212, 19214, 19211, 19213, 19220, 19222, 19224, 19221, + 19223, 19235, 19237, 19236, 19225, 19227, 19229, 19226, 19228, 19238, + 19204, 19246, 19245, 19189, 19139, 19156, 19140, 19141, 19142, 19143, + 19177, 19158, 19147, 19152, 19151, 19150, 19154, 19148, 19149, 19153, + 19175, 19145, 19157, 19146, 19160, 19159, 19174, 19169, 19155, 19168, + 19138, 19176, 19144, 19183, 41412, 41412, 41412, 19184, 19185, 19163, + 19164, 19171, 19170, 41412, 41412, 19162, 19161, 19186, 19180, 19181, + 19187, 41412, 41412, 19166, 19188, 19179, 19178, 19182, 19167, 41412, + 41412, 19172, 19165, 19173, 41412, 41412, 41412, 17678, 17739, 17737, + 17750, 17775, 17749, 17772, 41412, 19253, 19250, 19254, 19252, 19259, + 19257, 19258, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 20948, 20950, 20949, 30114, 32045, 41412, 41412, 25137, + 25163, 25156, 25189, 25147, 25138, 25167, 25133, 25145, 25175, 25178, + 25172, 41412, 25161, 25188, 25192, 25171, 25186, 25193, 25200, 25203, + 25148, 25199, 25146, 25149, 25132, 25155, 25158, 25181, 25182, 25140, + 25197, 25162, 25143, 25179, 25141, 25198, 25157, 25165, 41412, 25190, + 25154, 25174, 25139, 25150, 25164, 25135, 25169, 25144, 25176, 25177, + 25134, 25160, 25136, 25187, 25180, 25194, 25168, 25170, 41412, 25142, + 25196, 41412, 25153, 25152, 25166, 25202, 25195, 25205, 25173, 25151, + 25183, 25191, 25159, 25184, 25185, 25201, 25204, 41412, 41412, 25215, + 25216, 25218, 25217, 25206, 25207, 25214, 25208, 25209, 25219, 25210, + 25211, 25212, 25213, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 25021, 25020, 25022, + 25009, 25010, 25011, 25012, 25013, 25014, 25015, 25017, 25016, 25019, + 25018, 25027, 25024, 25025, 25023, 25026, 25126, 25127, 25029, 25028, + 25030, 25129, 25128, 25032, 25033, 25034, 25031, 25035, 25038, 25037, + 25039, 25040, 25041, 25130, 25042, 25043, 25036, 25046, 25047, 25045, + 25044, 25048, 25049, 25050, 25051, 25052, 25053, 25056, 25057, 25058, + 25055, 25059, 25054, 25060, 25061, 25062, 25063, 25064, 25065, 25066, + 25067, 25068, 25069, 25071, 25070, 25072, 25073, 25075, 25076, 25077, + 25074, 25078, 25079, 25080, 25081, 25083, 25082, 25084, 25085, 25131, + 25086, 25087, 25089, 25090, 25091, 25088, 25092, 25093, 25094, 25095, + 25096, 25124, 25104, 25105, 25106, 25107, 25108, 25109, 25110, 25111, + 25112, 25113, 25114, 25115, 25116, 25117, 25118, 25119, 25120, 25121, + 25122, 25123, 25097, 25098, 25099, 25100, 25101, 25102, 25103, 25125, + 41412, 41412, 41412, 41412, 41412, 112, 113, 159, 41412, 41412, 41412, + 41412, 156, 151, 146, 126, 121, 139, 134, 114, 129, 154, 149, 144, 124, + 119, 140, 135, 115, 130, 157, 152, 147, 127, 122, 142, 137, 117, 132, + 158, 153, 148, 128, 123, 143, 138, 118, 133, 155, 150, 145, 125, 120, + 141, 136, 116, 131, 41412, 41412, 41412, 111, 107, 109, 110, 108, 103, + 104, 105, 106, 18323, 18320, 18324, 18310, 18305, 18311, 18314, 18306, + 18316, 18325, 18308, 18318, 18312, 18321, 18315, 18317, 18327, 18309, + 18319, 18313, 18322, 18326, 18307, 18328, 18340, 18349, 18339, 18335, + 18348, 18330, 18336, 18352, 18356, 18357, 18338, 18341, 18347, 18346, + 18354, 18355, 18337, 18344, 18350, 18345, 18334, 18353, 18342, 18329, + 18331, 18351, 18343, 18332, 18333, 18575, 18576, 18774, 18770, 18811, + 18776, 18506, 18580, 18773, 18769, 18512, 18511, 18572, 18554, 18570, + 18579, 18574, 18360, 18359, 18813, 18772, 18814, 18577, 18768, 18550, + 29540, 41412, 32397, 32400, 32395, 32398, 32369, 32399, 32367, 32370, + 32396, 32368, 32401, 32366, 2569, 41412, 41412, 41412, 18767, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 31590, 31591, 31602, 31571, + 31574, 31605, 31583, 31582, 31603, 31610, 31569, 31596, 31575, 31588, + 31589, 31598, 31587, 31568, 31572, 31579, 31577, 31599, 31576, 31567, + 31597, 31584, 31585, 31570, 31573, 31595, 31609, 31580, 31604, 31566, + 31592, 31611, 31593, 31594, 31586, 31608, 31607, 31581, 31600, 31601, + 31606, 31578, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 25459, 25461, 25457, 25458, 25466, 25465, 25468, 25476, + 25478, 25455, 25469, 25452, 25472, 25470, 25450, 25463, 25451, 25464, + 25475, 25471, 25453, 25473, 25474, 25454, 25456, 25460, 25462, 25467, + 25477, 41412, 41412, 41412, 6141, 6152, 6143, 6114, 6137, 6153, 6115, + 6142, 6159, 6157, 6117, 6158, 6144, 6132, 6127, 6128, 6126, 6112, 6135, + 6125, 6160, 6122, 6134, 6151, 6131, 6155, 6145, 6140, 6149, 6150, 6123, + 6136, 6147, 6148, 6129, 6130, 6124, 6156, 6113, 6133, 6138, 6154, 6118, + 6119, 6120, 6121, 6116, 6146, 6139, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 8703, 8701, 8699, 8698, 8695, 8694, 8697, 8696, 8702, 8700, 8693, 8692, + 8690, 8681, 8679, 8688, 8686, 8677, 8683, 8684, 8691, 8689, 8680, 8678, + 8687, 8685, 8676, 8682, 41412, 41412, 41412, 41412, 30395, 30389, 30375, + 30390, 30362, 30392, 30394, 30391, 30386, 30382, 30374, 30370, 30371, + 30372, 30364, 30396, 30385, 30378, 30376, 30366, 30363, 30387, 30380, + 30368, 30384, 30373, 30369, 30367, 30388, 30383, 30381, 30365, 30400, + 30398, 30399, 30397, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 30393, 30379, 30377, 18170, 18186, 18194, 18188, 18169, + 18179, 18173, 18172, 18181, 18190, 18195, 18191, 18189, 18177, 18193, + 18184, 18178, 18176, 18180, 18192, 18182, 18183, 18185, 18174, 18171, + 18187, 18175, 41412, 41412, 41412, 41412, 41412, 30467, 30466, 30463, + 30436, 30437, 30459, 30434, 30460, 30435, 30439, 30468, 30461, 30442, + 30443, 30450, 30444, 30462, 30447, 30449, 30470, 30433, 30446, 30445, + 30457, 30454, 30464, 30465, 30438, 30469, 30448, 30451, 30452, 30453, + 30456, 30441, 30458, 30455, 30440, 8508, 8506, 8504, 8505, 8507, 41412, + 41412, 41412, 41412, 41412, 38163, 38166, 38170, 38174, 38167, 38171, + 38189, 38185, 38172, 38182, 38184, 38173, 38179, 38175, 38190, 38168, + 38187, 38186, 38178, 38164, 38188, 38177, 38165, 38176, 38181, 38169, + 38183, 38191, 38192, 38180, 41412, 38193, 30476, 30518, 30519, 30499, + 30500, 30497, 30498, 30496, 30511, 30488, 30489, 30493, 30494, 30483, + 30486, 30487, 30492, 30515, 30480, 30512, 30501, 30502, 30505, 30506, + 30507, 30516, 30490, 30491, 30503, 30504, 30514, 30510, 30517, 30508, + 30509, 30513, 41412, 41412, 41412, 41412, 30477, 30478, 30479, 30495, + 30484, 30485, 30481, 30482, 30520, 30475, 30472, 30473, 30471, 30474, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 10727, 10726, 10722, 10723, 10724, 10725, 10733, 10732, + 10728, 10729, 10730, 10731, 10750, 10735, 10749, 10746, 10751, 10744, + 10741, 10737, 10742, 10740, 10743, 10748, 10747, 10717, 10745, 10716, + 10736, 10712, 10739, 10713, 10738, 10720, 10718, 10719, 10714, 10715, + 10734, 10721, 10767, 10766, 10762, 10763, 10764, 10765, 10773, 10772, + 10768, 10769, 10770, 10771, 10790, 10775, 10789, 10786, 10791, 10784, + 10781, 10777, 10782, 10780, 10783, 10788, 10787, 10757, 10785, 10756, + 10776, 10752, 10779, 10753, 10778, 10760, 10758, 10759, 10754, 10755, + 10774, 10761, 32992, 32998, 33009, 33006, 32996, 32995, 32994, 32973, + 33001, 32979, 33008, 33004, 33007, 33010, 32997, 33005, 32984, 33003, + 33000, 32978, 32983, 32985, 32982, 32977, 32971, 32968, 32990, 32999, + 32988, 32972, 32993, 33011, 32975, 32969, 32981, 33012, 32989, 32986, + 32987, 32970, 32966, 32991, 32967, 32976, 32965, 32974, 32980, 33002, + 30910, 30928, 30934, 30932, 30935, 30916, 30913, 30933, 30918, 30917, + 30914, 30912, 30930, 30929, 30920, 30915, 30923, 30919, 30924, 30925, + 30931, 30936, 30909, 30926, 30937, 30921, 30938, 30911, 30927, 30922, + 41412, 41412, 30945, 30947, 30944, 30943, 30940, 30939, 30942, 30941, + 30948, 30946, 41412, 41412, 41412, 41412, 41412, 41412, 30837, 30838, + 30839, 30840, 30865, 30858, 30844, 30841, 30847, 30857, 30856, 30871, + 30850, 30845, 30849, 30866, 30867, 30868, 30851, 30852, 30869, 30846, + 30862, 30861, 30855, 30843, 30854, 30842, 30853, 30859, 30872, 30870, + 30848, 30860, 30864, 30863, 41412, 41412, 41412, 41412, 30873, 30874, + 30875, 30876, 30901, 30894, 30880, 30877, 30883, 30893, 30892, 30907, + 30886, 30881, 30885, 30902, 30903, 30904, 30887, 30888, 30905, 30882, + 30898, 30897, 30891, 30879, 30890, 30878, 30889, 30895, 30908, 30906, + 30884, 30896, 30900, 30899, 41412, 41412, 41412, 41412, 16827, 16818, + 16807, 16806, 16809, 16798, 16808, 16805, 16804, 16819, 16795, 16794, + 16820, 16828, 16821, 16811, 16797, 16796, 16822, 16801, 16800, 16799, + 16829, 16823, 16824, 16803, 16802, 16813, 16812, 16815, 16814, 16830, + 16825, 16826, 16831, 16817, 16816, 16793, 16792, 16810, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 6173, 6222, 6189, 6185, 6187, + 6212, 6186, 6210, 6207, 6176, 6209, 6211, 6190, 6201, 6197, 6192, 6220, + 6184, 6175, 6193, 6196, 6198, 6223, 6214, 6172, 6178, 6179, 6181, 6215, + 6213, 6218, 6182, 6202, 6194, 6221, 6206, 6217, 6183, 6177, 6200, 6188, + 6219, 6204, 6216, 6205, 6203, 6191, 6180, 6174, 6208, 6199, 6195, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 6224, 39368, 39337, 39338, 39348, 39347, 39350, 39349, 39340, 39339, + 39357, 39365, 41412, 39356, 39355, 39341, 39342, 39358, 39366, 39344, + 39343, 39359, 39346, 39345, 39369, 39360, 39367, 39361, 41412, 39352, + 39351, 39354, 39353, 39370, 39362, 39363, 41412, 39371, 39364, 41412, + 39403, 39372, 39373, 39383, 39382, 39385, 39384, 39375, 39374, 39392, + 39400, 41412, 39391, 39390, 39376, 39377, 39393, 39401, 39379, 39378, + 39394, 39381, 39380, 39404, 39395, 39402, 39396, 41412, 39387, 39386, + 39389, 39388, 39405, 39397, 39398, 41412, 39406, 39399, 41412, 41412, + 41412, 37824, 37825, 37838, 37798, 37827, 37826, 37829, 37805, 37828, + 37821, 37820, 37839, 37795, 37801, 37794, 37800, 37808, 37807, 37842, + 37796, 37831, 37823, 37822, 37799, 37806, 37802, 37818, 37810, 37840, + 37817, 37816, 37815, 37812, 37811, 37833, 37832, 37843, 37841, 37835, + 37804, 37834, 37803, 37844, 37797, 37837, 37836, 37793, 37814, 37813, + 37830, 37809, 37819, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 24988, 24989, 24990, 24991, 24992, + 24993, 24994, 24995, 24996, 24927, 24928, 24929, 24930, 24931, 24940, + 24932, 24933, 24934, 24935, 24936, 24937, 24938, 24939, 24941, 24942, + 24943, 24944, 25008, 24945, 24946, 24947, 24948, 24949, 24950, 24951, + 24952, 24953, 24954, 24955, 24956, 24957, 24958, 24959, 24960, 24961, + 24962, 24963, 24964, 24965, 24966, 24967, 24968, 24969, 24970, 24971, + 24972, 24973, 24974, 24975, 24976, 24977, 24978, 24979, 24980, 24981, + 24982, 24983, 24984, 24985, 24986, 24987, 24668, 25004, 24997, 24669, + 24998, 24999, 25000, 25001, 24670, 25005, 25006, 25002, 25003, 25007, + 24674, 24675, 24676, 24677, 24678, 24679, 24680, 24681, 24671, 24672, + 24673, 24685, 24686, 24687, 24682, 24683, 24684, 24688, 24689, 24690, + 24691, 24692, 24693, 24696, 24697, 24698, 24699, 24700, 24701, 24702, + 24703, 24704, 24705, 24706, 24707, 24708, 24709, 24710, 24711, 24712, + 24713, 24714, 24715, 24716, 24717, 24718, 24719, 24720, 24721, 24722, + 24723, 24724, 24725, 24726, 24727, 24728, 24729, 24730, 24731, 24732, + 24733, 24734, 24735, 24736, 24737, 24738, 24739, 24740, 24741, 24742, + 24743, 24744, 24745, 24694, 24695, 24746, 24747, 24748, 24749, 24750, + 24751, 24752, 24753, 24754, 24755, 24756, 24757, 24758, 24759, 24760, + 24761, 24762, 24763, 24764, 24765, 24766, 24767, 24768, 24769, 24770, + 24771, 24772, 24773, 24774, 24775, 24776, 24777, 24778, 24810, 24811, + 24812, 24813, 24814, 24815, 24816, 24817, 24818, 24779, 24780, 24781, + 24782, 24783, 24784, 24785, 24786, 24787, 24788, 24789, 24790, 24791, + 24792, 24793, 24794, 24795, 24796, 24797, 24798, 24799, 24800, 24801, + 24802, 24803, 24804, 24805, 24806, 24807, 24808, 24809, 24825, 24826, + 24827, 24828, 24829, 24830, 24831, 24832, 24833, 24834, 24835, 24836, + 24837, 24838, 24839, 24840, 24841, 24842, 24843, 24844, 24819, 24820, + 24821, 24822, 24823, 24824, 24845, 24846, 24847, 24848, 24849, 24850, + 24851, 24852, 24887, 24888, 24889, 24890, 24891, 24892, 24893, 24894, + 24895, 24896, 24853, 24854, 24855, 24856, 24857, 24858, 24859, 24860, + 24861, 24862, 24863, 24864, 24865, 24866, 24867, 24868, 24869, 24870, + 24871, 24872, 24878, 24879, 24880, 24881, 24882, 24883, 24884, 24885, + 24886, 24873, 24874, 24875, 24876, 24877, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 24908, 24903, 24906, 24905, 24909, + 24904, 24902, 24907, 24901, 24897, 24900, 24899, 24898, 24915, 24913, + 24914, 24910, 24911, 24912, 24916, 24918, 24917, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 24919, 24920, 24921, + 24922, 24923, 24924, 24925, 24926, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 28048, + 28169, 28168, 28033, 28049, 28037, 41412, 28067, 28069, 28068, 28063, + 28062, 28060, 28061, 28122, 28055, 28079, 28119, 28043, 28083, 28044, + 28087, 28050, 28089, 28066, 28103, 28104, 28102, 28046, 28100, 28105, + 28106, 28147, 28148, 28111, 28047, 28056, 28162, 28144, 28145, 28118, + 28117, 28052, 28132, 28135, 28136, 28133, 28131, 28159, 41412, 28054, + 27974, 28021, 27868, 27952, 27981, 27865, 28023, 28124, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 10140, 10141, 10142, + 10143, 10144, 10134, 41412, 41412, 10135, 41412, 10090, 10091, 10092, + 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, 10102, + 10103, 10104, 10105, 10106, 10107, 10108, 10109, 10110, 10111, 10112, + 10113, 10114, 10115, 10116, 10117, 10118, 10119, 10120, 10121, 10122, + 10123, 10124, 10125, 10126, 10127, 10128, 10129, 10130, 10131, 10132, + 10133, 41412, 10138, 10139, 41412, 41412, 41412, 10136, 41412, 41412, + 10137, 20770, 20781, 20772, 20766, 20778, 20783, 20773, 20779, 20764, + 20777, 20780, 20767, 20785, 20782, 20774, 20771, 20784, 20775, 20768, + 20769, 20776, 20765, 41412, 20786, 20761, 20757, 20760, 20758, 20756, + 20762, 20763, 20759, 31218, 31229, 31220, 31214, 31226, 31231, 31221, + 31227, 31212, 31225, 31228, 31215, 31233, 31211, 31230, 31222, 31219, + 31232, 31223, 31216, 31217, 31224, 31213, 31210, 31234, 31241, 31236, + 31237, 31240, 31239, 31238, 31235, 28986, 29001, 28991, 29012, 29003, + 28997, 28993, 29009, 29014, 29004, 29010, 28995, 28989, 29008, 28990, + 29011, 28987, 28998, 28994, 29016, 28992, 29013, 29005, 29002, 29015, + 29006, 28999, 29000, 28988, 29007, 28996, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 29021, 29018, 29019, 29024, 29025, 29023, + 29020, 29017, 29022, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 19816, 19832, 19824, 19823, 19829, 19834, 19818, 19830, 19819, + 19828, 19831, 19821, 19836, 19833, 19825, 19817, 19835, 19826, 19822, + 41412, 19827, 19820, 41412, 41412, 41412, 41412, 41412, 19837, 19841, + 19840, 19839, 19838, 31614, 31633, 31629, 31616, 31617, 31625, 31626, + 31618, 31624, 31627, 31630, 31632, 31635, 31631, 31622, 31615, 31634, + 31620, 31619, 31628, 31621, 31623, 31640, 31639, 31636, 31641, 31637, + 31638, 41412, 41412, 41412, 31642, 25485, 25491, 25495, 25493, 25487, + 25503, 25496, 25504, 25497, 25481, 25498, 25489, 25499, 25501, 25484, + 25479, 25502, 25494, 25500, 25483, 25480, 25486, 25488, 25482, 25490, + 25492, 41412, 41412, 41412, 41412, 41412, 25505, 33150, 33151, 33152, + 33153, 33154, 33155, 33156, 33157, 33158, 33159, 33160, 33161, 33162, + 33163, 33164, 33165, 33166, 33167, 33168, 33143, 33144, 33145, 33146, + 33147, 33148, 33149, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 27580, 27581, 27582, 27583, 27579, 27578, 27554, 27555, 27576, + 27575, 27558, 27559, 27560, 27561, 27556, 27557, 27574, 27571, 27570, + 27562, 27563, 27564, 27572, 27577, 27565, 27566, 27567, 27568, 27569, + 27573, 27584, 27585, 27476, 27497, 27498, 27499, 27496, 27495, 27488, + 27492, 27491, 27481, 27482, 27494, 27490, 27486, 27485, 27483, 27477, + 27484, 27487, 27493, 27478, 27479, 27480, 27489, 41412, 41412, 41412, + 41412, 27464, 27469, 27501, 27500, 27550, 27542, 27536, 27513, 27507, + 27530, 27524, 27502, 27519, 27548, 27546, 27540, 27517, 27511, 27534, + 27528, 41412, 41412, 27551, 27543, 27537, 27514, 27508, 27531, 27525, + 27504, 27521, 27553, 27545, 27539, 27516, 27510, 27533, 27527, 27506, + 27523, 27549, 27547, 27541, 27518, 27512, 27535, 27529, 27503, 27520, + 27552, 27544, 27538, 27515, 27509, 27532, 27526, 27505, 27522, 27468, + 27474, 27473, 27467, 27466, 27471, 27470, 27465, 27475, 27472, 21831, + 21853, 21855, 21851, 41412, 21852, 21854, 41412, 41412, 41412, 41412, + 41412, 21856, 21847, 21850, 21849, 21797, 21795, 21819, 21818, 41412, + 21817, 21816, 21825, 41412, 21805, 21801, 21800, 21808, 21807, 21804, + 21803, 21802, 21810, 21809, 21806, 21821, 21820, 21815, 21814, 21827, + 21829, 21828, 21826, 21823, 21811, 21812, 21813, 21830, 21824, 21796, + 21798, 21799, 21822, 41412, 41412, 21845, 21846, 21848, 41412, 41412, + 41412, 41412, 21857, 21794, 21793, 21792, 21791, 21833, 21832, 21834, + 21835, 21858, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 21839, + 21844, 21837, 21836, 21843, 21841, 21840, 21838, 21842, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 30585, 30581, 30586, 30591, 30582, + 30589, 30575, 30583, 30587, 30579, 30574, 30570, 30588, 30571, 30573, + 30572, 30590, 30563, 30564, 30566, 30569, 30567, 30568, 30578, 30580, + 30565, 30584, 30577, 30576, 30593, 30592, 30594, 30406, 30424, 30405, + 30415, 30427, 30428, 30417, 30421, 30419, 30412, 30416, 30408, 30423, + 30407, 30429, 30418, 30420, 30401, 30402, 30425, 30404, 30426, 30403, + 30411, 30413, 30409, 30422, 30410, 30414, 30432, 30431, 30430, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 25786, 25790, 25789, 25794, 25793, 25791, 25815, 25818, 25834, + 25798, 25797, 25796, 25795, 25816, 25808, 25814, 25800, 25810, 25799, + 25812, 25792, 25807, 25821, 25817, 25804, 25788, 25787, 25820, 25819, + 25805, 25802, 25811, 25801, 25813, 25806, 25803, 25809, 25836, 25835, + 41412, 41412, 41412, 41412, 25822, 25826, 25825, 25824, 25823, 25833, + 25831, 25829, 25828, 25827, 25832, 25830, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 2587, 2588, 2594, 2590, 2593, 2589, + 2591, 2592, 2630, 2631, 2620, 2621, 2622, 2623, 2618, 2619, 2635, 2608, + 2607, 2606, 2597, 2595, 2596, 2632, 2634, 2617, 2615, 2627, 2626, 2616, + 2638, 2633, 2625, 2624, 2602, 2601, 2600, 2605, 2604, 2603, 2637, 2598, + 2613, 2614, 2640, 2639, 2636, 2612, 2629, 2610, 2628, 2609, 2611, 2599, + 41412, 41412, 41412, 2641, 37706, 34058, 22832, 22827, 22833, 22828, + 20912, 20923, 20914, 20908, 20920, 20925, 20915, 20921, 20906, 20919, + 20922, 20909, 20927, 20924, 20916, 20913, 20926, 20917, 20910, 20911, + 20918, 20907, 41412, 41412, 20932, 20929, 20930, 20935, 20931, 20928, + 20933, 20934, 20881, 20895, 20886, 20882, 20892, 20885, 20887, 20893, + 20879, 20891, 20894, 20883, 20884, 20896, 20888, 20897, 20889, 20890, + 20880, 41412, 41412, 41412, 41412, 41412, 20902, 20899, 20900, 20905, + 20901, 20898, 20903, 20904, 31875, 31889, 31880, 31876, 31886, 31879, + 31881, 31887, 31885, 31888, 31877, 31878, 31890, 31882, 31892, 31883, + 31884, 31891, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 31900, + 31901, 31873, 31874, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 31897, 31894, 31895, 31899, 31896, + 31893, 31898, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 30595, 30637, 30638, 30627, 30663, 30656, 30630, 30631, + 30665, 30608, 30648, 30596, 30641, 30610, 30650, 30598, 30642, 30609, + 30649, 30597, 30626, 30662, 30616, 30655, 30605, 30645, 30599, 30643, + 30632, 30666, 30611, 30651, 30600, 30621, 30624, 30612, 30601, 30639, + 30619, 30658, 30617, 30657, 30620, 30659, 30647, 30618, 30640, 30625, + 30633, 30628, 30623, 30661, 30613, 30652, 30629, 30664, 30634, 30667, + 30614, 30653, 30602, 30606, 30603, 30607, 30646, 30622, 30660, 30615, + 30654, 30604, 30644, 30635, 30636, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 30254, 30257, 30280, 30255, 30269, 30265, 30271, 30281, 30256, 30259, + 30302, 30282, 30283, 30272, 30273, 30284, 30303, 30304, 30285, 30286, + 30258, 30299, 30274, 30275, 30260, 30262, 30266, 30294, 30296, 30290, + 30292, 30295, 30287, 30261, 30288, 30297, 30267, 30268, 30276, 30263, + 30277, 30270, 30298, 30301, 30291, 30293, 30289, 30278, 30279, 30264, + 30300, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 30305, 30308, 30331, 30306, 30320, 30316, + 30322, 30332, 30307, 30310, 30353, 30333, 30334, 30323, 30324, 30335, + 30354, 30355, 30336, 30337, 30309, 30350, 30325, 30326, 30311, 30313, + 30317, 30345, 30347, 30341, 30343, 30346, 30338, 30312, 30339, 30348, + 30318, 30319, 30327, 30314, 30328, 30321, 30349, 30352, 30342, 30344, + 30340, 30329, 30330, 30315, 30351, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 30358, 30357, 30361, 30356, 30359, 30360, 19769, 19756, + 19764, 19748, 19747, 19761, 19757, 19760, 19745, 19758, 19742, 19741, + 19750, 19749, 19768, 19755, 19754, 19746, 19759, 19762, 19763, 19753, + 19766, 19743, 19767, 19744, 19751, 19752, 19765, 19776, 19778, 19780, + 19777, 19779, 19771, 19770, 19775, 19772, 19774, 19773, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 19787, 19789, 19786, 19785, + 19782, 19781, 19784, 19783, 19790, 19788, 41412, 41412, 41412, 41412, + 41412, 41412, 17854, 17856, 17853, 17852, 17849, 17848, 17851, 17850, + 17857, 17855, 17840, 17841, 17842, 17839, 17843, 17837, 17814, 17798, + 17806, 17804, 17797, 17803, 17809, 17811, 17805, 17801, 17799, 17812, + 17813, 17810, 17808, 17795, 17800, 17796, 17807, 17802, 17793, 17794, + 41412, 41412, 41412, 17838, 17792, 17790, 17789, 17791, 17845, 17844, + 17836, 17820, 17828, 17826, 17819, 17825, 17831, 17833, 17827, 17823, + 17821, 17834, 17835, 17832, 17830, 17817, 17822, 17818, 17829, 17824, + 17815, 17816, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 17846, 17847, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 32454, 32452, 32451, 32448, 32447, 32450, 32449, 32455, 32453, 32446, + 32445, 32443, 32434, 32432, 32441, 32439, 32430, 32436, 32437, 32444, + 32442, 32433, 32431, 32440, 32438, 32429, 32435, 32426, 32427, 32425, + 32428, 41412, 39869, 39903, 39883, 39882, 39888, 39887, 39865, 39864, + 39863, 39874, 39895, 39868, 39899, 39902, 39901, 39898, 39906, 39885, + 39884, 39886, 39867, 39889, 39900, 39870, 39894, 39905, 39890, 39891, + 39878, 39876, 39875, 39877, 39879, 39866, 39881, 39904, 39892, 39893, + 39872, 39873, 39896, 39871, 41412, 39861, 39860, 39862, 41412, 41412, + 39880, 39897, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 1201, 1497, 1332, + 2388, 1540, 1604, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 1065, 1654, 1652, 1650, 1909, 1910, 1911, 1913, 1895, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 1083, 2389, 1066, 2395, 2396, 2393, 30525, 30533, 30547, 30534, + 30538, 30544, 30535, 30550, 30539, 30545, 30543, 30546, 30537, 30552, + 30548, 30527, 30528, 30540, 30526, 30524, 30551, 30541, 30529, 30530, + 30536, 30542, 30549, 30531, 30532, 30559, 30557, 30554, 30562, 30561, + 30558, 30556, 30555, 30560, 30523, 30553, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 34118, 34132, 34120, 34129, 34136, 34124, + 34130, 34128, 34131, 34121, 34138, 34134, 34125, 34119, 34137, 34126, + 34123, 34127, 34135, 34133, 34122, 34117, 34112, 34110, 34113, 34111, + 34116, 34115, 34107, 34106, 34108, 34114, 34109, 34139, 34142, 34141, + 34140, 34145, 34146, 34143, 34147, 34144, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 30672, 30684, + 30682, 30687, 30676, 30681, 30680, 30683, 30674, 30689, 30685, 30677, + 30688, 30678, 30673, 30679, 30686, 30675, 30671, 30670, 30669, 30668, + 30693, 30690, 30691, 30692, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 6596, 6590, 6604, 6598, 6593, 6601, 6607, 6589, 6599, 6602, + 6600, 6603, 6594, 6609, 6605, 6591, 6597, 6608, 6595, 6592, 6606, 6614, + 6611, 6612, 6616, 6613, 6610, 6615, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 16854, 16865, 16856, 16850, 16862, + 16867, 16857, 16863, 16848, 16861, 16864, 16851, 16869, 16866, 16858, + 16855, 16868, 16859, 16852, 16853, 16860, 16849, 16870, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 4737, 4742, 4740, 4739, + 4741, 4664, 4665, 4683, 4684, 4681, 4682, 4676, 4677, 4678, 4679, 4710, + 4666, 4657, 4667, 4703, 4702, 4699, 4698, 4687, 4697, 4696, 4701, 4700, + 4689, 4673, 4672, 4669, 4668, 4688, 4675, 4674, 4671, 4670, 4690, 4705, + 4704, 4695, 4694, 4707, 4709, 4708, 4686, 4680, 4691, 4692, 4693, 4706, + 4685, 4658, 4663, 4662, 4747, 4746, 4756, 4757, 4750, 4751, 4752, 4753, + 4754, 4755, 4758, 4748, 4743, 4749, 4759, 4761, 4760, 4735, 4734, 4733, + 4736, 4732, 41412, 41412, 41412, 41412, 4728, 4726, 4723, 4714, 4716, + 4721, 4719, 4711, 4717, 4727, 4725, 4724, 4713, 4715, 4722, 4720, 4712, + 4718, 4729, 4730, 4768, 4770, 4767, 4766, 4763, 4762, 4765, 4764, 4771, + 4769, 4738, 4660, 4661, 4744, 4745, 4659, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 4731, 21141, 21143, 21145, 21101, + 21102, 21111, 21112, 21109, 21110, 21139, 21103, 21140, 21104, 21129, + 21128, 21125, 21124, 21113, 21123, 21122, 21127, 21126, 21115, 21106, + 21105, 21098, 21096, 21097, 21132, 21114, 21108, 21107, 21100, 21099, + 21116, 21131, 21130, 21121, 21120, 21136, 21138, 21133, 21135, 21137, + 21117, 21118, 21119, 21134, 21151, 21156, 21157, 21154, 21155, 21158, + 21152, 21159, 21153, 21144, 21142, 21162, 21163, 21160, 21146, 21147, + 21149, 21148, 21150, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 21161, 41412, 41412, 34170, 34171, 34160, 34161, + 34162, 34163, 34156, 34157, 34167, 34159, 34172, 34168, 34173, 34169, + 34164, 34166, 34165, 34158, 34174, 34153, 34175, 34177, 34176, 34154, + 34155, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 34184, 34186, + 34183, 34182, 34179, 34178, 34181, 34180, 34187, 34185, 41412, 41412, + 41412, 41412, 41412, 41412, 6276, 6278, 6277, 6272, 6274, 6275, 6273, + 6261, 6260, 6257, 6256, 6242, 6255, 6254, 6259, 6258, 6244, 6247, 6246, + 6239, 6238, 6243, 6249, 6248, 6241, 6240, 6245, 6265, 6264, 6253, 6252, + 6267, 6250, 6251, 6268, 6263, 6271, 6269, 6266, 6280, 6288, 6289, 6284, + 6285, 6286, 6282, 6290, 6283, 6291, 6308, 6307, 6292, 6306, 41412, 6301, + 6303, 6300, 6299, 6296, 6295, 6298, 6297, 6304, 6302, 6279, 6294, 6293, + 6305, 6262, 6281, 6287, 6270, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 25547, 25549, 25551, 25548, 25550, 25539, 25538, 25535, + 25534, 25533, 25532, 25537, 25536, 25518, 25527, 25526, 25521, 25520, + 25517, 25529, 25528, 25523, 25522, 25519, 25541, 25540, 25531, 25530, + 25544, 25525, 25543, 25546, 25545, 25542, 25524, 25554, 25555, 25553, + 25552, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 32928, 32931, 32935, 32874, 32875, 32893, 32894, 32891, 32892, 32886, + 32887, 32888, 32889, 32920, 32876, 32921, 32877, 32913, 32912, 32909, + 32908, 32897, 32907, 32906, 32911, 32910, 32899, 32883, 32882, 32879, + 32878, 32898, 32885, 32884, 32881, 32880, 32900, 32915, 32914, 32905, + 32904, 32917, 32919, 32918, 32896, 32895, 32890, 32901, 32902, 32903, + 32916, 32950, 32957, 32958, 32944, 32945, 32953, 32954, 32955, 32956, + 32959, 32951, 32940, 32952, 32934, 32930, 32932, 32933, 32962, 32860, + 32859, 32960, 32925, 32922, 32929, 32937, 32871, 32936, 32943, 32926, + 32867, 32869, 32866, 32865, 32862, 32861, 32864, 32863, 32870, 32868, + 32872, 32927, 32873, 32961, 32923, 32924, 41412, 33872, 33870, 33869, + 33866, 33865, 33868, 33867, 33873, 33871, 33884, 33883, 33882, 33878, + 33877, 33881, 33880, 33876, 33879, 33874, 33875, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 22537, 22538, + 22564, 22566, 22563, 22539, 22565, 22540, 22554, 22553, 22529, 22527, + 22528, 22547, 22552, 22551, 22536, 22535, 41412, 22549, 22542, 22541, + 22532, 22531, 22548, 22544, 22543, 22534, 22530, 22533, 22550, 22556, + 22555, 22526, 22524, 22525, 22558, 22562, 22560, 22546, 22561, 22523, + 22557, 22545, 22574, 22577, 22578, 22581, 22579, 22575, 22580, 22576, + 22571, 22570, 22569, 22567, 22521, 22520, 22582, 22572, 22519, 22583, + 22568, 22559, 22522, 22573, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 28461, 28463, 28464, 28462, + 28452, 28451, 28450, 41412, 28449, 41412, 28448, 28447, 28440, 28439, + 41412, 28434, 28442, 28441, 28430, 28428, 28429, 28433, 28444, 28443, + 28432, 28431, 28435, 28454, 28453, 28446, 41412, 28445, 28457, 28460, + 28438, 28456, 28459, 28458, 28455, 28437, 28436, 28465, 41412, 41412, + 41412, 41412, 41412, 41412, 22598, 22599, 22610, 22611, 22608, 22609, + 22629, 22600, 22630, 22601, 22619, 22618, 22589, 22587, 22588, 22612, + 22617, 22616, 22597, 22596, 22595, 22614, 22605, 22604, 22592, 22590, + 22602, 22591, 22613, 22607, 22606, 22594, 22593, 22615, 22621, 22620, + 22586, 22584, 22585, 22626, 22628, 22603, 22625, 22627, 22622, 22623, + 22624, 22633, 22634, 22639, 22640, 22637, 22638, 22641, 22635, 22642, + 22636, 22631, 22632, 41412, 41412, 41412, 41412, 41412, 22649, 22651, + 22648, 22647, 22644, 22643, 22646, 22645, 22652, 22650, 41412, 41412, + 41412, 41412, 41412, 41412, 18252, 18253, 18256, 18259, 41412, 18209, + 18210, 18223, 18224, 18221, 18222, 18204, 18206, 41412, 41412, 18247, + 18211, 41412, 41412, 18246, 18212, 18243, 18242, 18239, 18238, 18227, + 18237, 18236, 18241, 18240, 18229, 18218, 18217, 18214, 18213, 18228, + 18220, 18219, 18216, 18215, 18230, 41412, 18245, 18244, 18235, 18234, + 18249, 18251, 18250, 41412, 18226, 18225, 41412, 18208, 18231, 18232, + 18233, 18248, 41412, 8183, 18254, 18255, 18261, 18270, 18271, 18264, + 18265, 18266, 18267, 41412, 41412, 18273, 18262, 41412, 41412, 18272, + 18263, 18258, 41412, 41412, 18274, 41412, 41412, 41412, 41412, 41412, + 41412, 18260, 41412, 41412, 41412, 41412, 41412, 18257, 18203, 18202, + 18205, 18207, 18268, 18269, 41412, 41412, 8372, 8373, 8371, 8370, 8369, + 8368, 8367, 41412, 41412, 41412, 8378, 8375, 8376, 8374, 8377, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 38042, 38043, 38066, 38067, 38064, 38065, 38059, 38060, 38061, 38062, + 41412, 38088, 41412, 41412, 38044, 41412, 38087, 38045, 38084, 38083, + 38080, 38079, 38068, 38078, 38077, 38082, 38081, 38070, 38056, 38055, + 38047, 38046, 38069, 38058, 38057, 38049, 38048, 38071, 38086, 38085, + 38076, 38075, 38090, 38091, 38054, 38052, 38063, 38072, 38073, 38074, + 38089, 38051, 38053, 38050, 41412, 38093, 38104, 38113, 38114, 38107, + 38108, 38109, 38110, 38111, 38112, 41412, 38116, 41412, 41412, 38105, + 41412, 38115, 38106, 38037, 38098, 41412, 38094, 38101, 38100, 38095, + 38038, 38092, 38041, 38099, 38040, 38039, 41412, 38096, 38097, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 38103, 38102, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 29384, 29385, + 29398, 29399, 29396, 29397, 29380, 29381, 29382, 29383, 29424, 29386, + 29425, 29387, 29412, 29411, 29408, 29407, 29373, 29372, 29406, 29405, + 29410, 29409, 29375, 29374, 29393, 29392, 29389, 29388, 29377, 29395, + 29394, 29391, 29390, 29378, 29376, 29418, 29417, 29404, 29403, 29416, + 29415, 29423, 29420, 29419, 29414, 29413, 29422, 29400, 29401, 29402, + 29421, 29438, 29447, 29448, 29441, 29442, 29443, 29444, 29445, 29446, + 29449, 29439, 29450, 29440, 29433, 29426, 29429, 29434, 29427, 29428, + 29431, 29454, 29435, 29360, 29358, 29453, 29371, 29451, 29367, 29369, + 29366, 29365, 29362, 29361, 29364, 29363, 29370, 29368, 29359, 29437, + 41412, 29452, 29436, 29379, 29430, 29432, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 37708, 37709, 37710, 37728, + 37729, 37726, 37727, 37721, 37722, 37723, 37724, 37754, 37711, 37755, + 37712, 37746, 37745, 37742, 37741, 37730, 37740, 37739, 37744, 37743, + 37732, 37718, 37717, 37714, 37713, 37731, 37720, 37719, 37716, 37715, + 37733, 37748, 37747, 37738, 37737, 37751, 37753, 37752, 37750, 37725, + 37734, 37735, 37736, 37749, 37764, 37773, 37774, 37767, 37768, 37769, + 37770, 37771, 37772, 37775, 37762, 37765, 37776, 37763, 37766, 37756, + 37759, 37761, 37760, 37757, 37758, 37787, 37707, 37788, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 37783, 37785, 37782, 37781, + 37778, 37777, 37780, 37779, 37786, 37784, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 33052, 33054, 33075, 33076, 33073, 33074, 33068, 33069, + 33070, 33071, 33101, 33055, 33102, 33056, 33093, 33092, 33089, 33088, + 33077, 33087, 33086, 33091, 33090, 33079, 33062, 33061, 33065, 33064, + 33078, 33063, 33058, 33067, 33066, 33080, 33095, 33094, 33085, 33084, + 33098, 33100, 33099, 33097, 33072, 33081, 33082, 33083, 33096, 33130, + 33137, 33138, 33135, 33136, 33133, 33134, 41412, 41412, 33139, 33131, + 33140, 33132, 33123, 33125, 33127, 33126, 33124, 33122, 33142, 33141, + 33121, 33120, 33103, 33104, 33105, 33051, 33118, 33117, 33115, 33113, + 33114, 33106, 33107, 33116, 33119, 33111, 33112, 33110, 33109, 33108, + 33057, 33059, 33060, 33053, 33128, 33129, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 27815, 27816, 27834, 27835, 27832, 27833, 27827, 27828, 27829, 27830, + 27861, 27817, 27862, 27818, 27854, 27853, 27850, 27849, 27838, 27848, + 27847, 27852, 27851, 27840, 27824, 27823, 27820, 27819, 27839, 27826, + 27825, 27822, 27821, 27841, 27856, 27855, 27846, 27845, 27858, 27860, + 27859, 27837, 27831, 27842, 27843, 27844, 27857, 27836, 27790, 27799, + 27800, 27793, 27794, 27795, 27796, 27797, 27798, 27801, 27791, 27802, + 27792, 27786, 27789, 27788, 27785, 27804, 27803, 27863, 27787, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 27811, 27813, 27810, 27809, 27806, 27805, 27808, 27807, 27814, 27812, + 41412, 41412, 41412, 41412, 41412, 41412, 28344, 28339, 28184, 28345, + 28343, 28341, 28340, 28201, 28202, 28335, 28337, 28336, 28346, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 35581, 35583, + 35598, 35599, 35596, 35597, 35623, 35584, 35624, 35585, 35613, 35612, + 35609, 35608, 35600, 35607, 35606, 35611, 35610, 35602, 35593, 35592, + 35587, 35586, 35601, 35595, 35594, 35589, 35588, 35603, 35615, 35614, + 35605, 35604, 35620, 35622, 35591, 35619, 35621, 35616, 35617, 35618, + 35590, 35626, 35628, 35629, 35634, 35635, 35632, 35633, 35636, 35630, + 35637, 35631, 35627, 35625, 35582, 35580, 41412, 41412, 41412, 41412, + 41412, 41412, 35644, 35646, 35643, 35642, 35639, 35638, 35641, 35640, + 35647, 35645, 41412, 41412, 41412, 41412, 41412, 41412, 28953, 28955, + 28952, 28951, 28948, 28947, 28950, 28949, 28956, 28954, 28903, 28905, + 28902, 28901, 28898, 28897, 28900, 28899, 28906, 28904, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 191, 190, 178, 181, 175, 167, + 193, 192, 183, 195, 189, 184, 174, 196, 177, 197, 180, 194, 164, 171, + 170, 187, 166, 186, 182, 188, 165, 41412, 41412, 162, 163, 161, 203, 204, + 210, 211, 208, 209, 212, 207, 213, 205, 206, 200, 41412, 41412, 41412, + 41412, 222, 224, 221, 220, 217, 216, 219, 218, 225, 223, 215, 214, 198, + 199, 201, 202, 185, 173, 172, 169, 168, 179, 176, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 11159, 11160, 11175, 11176, 11173, 11174, 11201, 11161, + 11202, 11162, 11193, 11192, 11189, 11188, 11177, 11187, 11186, 11191, + 11190, 11179, 11170, 11169, 11164, 11163, 11178, 11172, 11171, 11166, + 11165, 11180, 11195, 11194, 11185, 11184, 11198, 11200, 11168, 11197, + 11199, 11181, 11182, 11183, 11196, 11167, 11205, 11210, 11211, 11208, + 11209, 11203, 11204, 11212, 11206, 11213, 11207, 11216, 11218, 11217, + 11215, 11214, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 39508, 39497, 39526, 39516, 39518, 39519, 39525, 39515, + 39501, 39510, 39498, 39528, 39524, 39503, 39517, 39514, 39502, 39511, + 39520, 39509, 39527, 39500, 39499, 39522, 39523, 39506, 39505, 39504, + 39507, 39512, 39513, 39521, 39540, 39529, 39558, 39548, 39550, 39551, + 39557, 39547, 39533, 39542, 39530, 39560, 39556, 39535, 39549, 39546, + 39534, 39543, 39552, 39541, 39559, 39532, 39531, 39554, 39555, 39538, + 39537, 39536, 39539, 39544, 39545, 39553, 39567, 39569, 39566, 39565, + 39562, 39561, 39564, 39563, 39570, 39568, 39579, 39578, 39577, 39573, + 39572, 39576, 39575, 39571, 39574, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 39580, 11075, 11076, + 11083, 11084, 11081, 11082, 11110, 41412, 41412, 11111, 41412, 41412, + 11101, 11100, 11099, 11098, 11087, 11097, 11096, 11105, 41412, 11089, + 11071, 41412, 11078, 11077, 11088, 11072, 11070, 11080, 11079, 11090, + 11103, 11102, 11095, 11094, 11106, 11074, 11073, 11107, 11086, 11108, + 11091, 11092, 11093, 11104, 11085, 11109, 11116, 11120, 11121, 11118, + 11119, 11122, 41412, 11117, 11123, 41412, 41412, 11115, 11113, 11112, + 11124, 11129, 11126, 11130, 11125, 11114, 11059, 11127, 11128, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 11066, 11068, + 11065, 11064, 11061, 11060, 11063, 11062, 11069, 11067, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 29075, 29076, + 29091, 29092, 29089, 29090, 29072, 29073, 41412, 41412, 29117, 29077, + 29118, 29078, 29111, 29110, 29107, 29106, 29095, 29105, 29104, 29109, + 29108, 29097, 29086, 29085, 29080, 29079, 29096, 29088, 29087, 29082, + 29081, 29098, 29113, 29112, 29103, 29102, 29115, 29116, 29084, 29094, + 29074, 29099, 29100, 29101, 29114, 29093, 29083, 29127, 29132, 29133, + 29130, 29131, 29125, 29126, 41412, 41412, 29134, 29128, 29135, 29129, + 29121, 29123, 29122, 29120, 29119, 29136, 29124, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41193, 41214, 41211, 41210, 41213, 41209, + 41208, 41206, 41207, 41212, 41205, 41161, 41160, 41180, 41179, 41162, + 41178, 41177, 41187, 41164, 41172, 41171, 41154, 41153, 41163, 41174, + 41173, 41158, 41157, 41165, 41182, 41181, 41176, 41175, 41189, 41170, + 41169, 41156, 41155, 41183, 41184, 41185, 41192, 41190, 41188, 41191, + 41166, 41167, 41168, 41186, 41159, 41152, 41202, 41199, 41200, 41201, + 41198, 41203, 41149, 41148, 41146, 41145, 41147, 41151, 41144, 41197, + 41195, 41194, 41196, 41150, 41143, 41204, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 34276, 34297, 34295, 34294, 34296, 34292, + 34293, 34290, 34291, 34289, 34288, 34298, 34243, 34242, 34262, 34261, + 34244, 34260, 34259, 34264, 34263, 34246, 34254, 34253, 34237, 34236, + 34245, 34256, 34255, 34240, 34238, 34247, 34266, 34265, 34258, 34257, + 34272, 34252, 34251, 34239, 34267, 34268, 34269, 34275, 34273, 34271, + 34274, 34248, 34249, 34250, 34270, 34241, 34281, 34283, 34220, 34219, + 34217, 34218, 34228, 34229, 34224, 34227, 34223, 34226, 34231, 34232, + 34230, 34222, 34221, 34225, 34284, 34282, 34299, 34285, 34280, 34279, + 34278, 34277, 34235, 34234, 34233, 34286, 34287, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 5719, 5720, 5717, 5718, 5715, 5716, 5725, 5726, 5723, 5724, 5721, 5722, + 5888, 5889, 5890, 5887, 31432, 31430, 31439, 31440, 31436, 31444, 31443, + 31425, 31438, 31437, 31429, 31442, 31435, 31428, 31434, 31433, 31426, + 31431, 31441, 31420, 31427, 31445, 31446, 31421, 31447, 31423, 31424, + 31422, 31416, 31413, 31417, 31415, 31411, 31414, 31418, 31412, 31419, + 31453, 31452, 31463, 31454, 31455, 31464, 31460, 31459, 31461, 31462, + 31456, 31410, 31457, 31458, 31449, 31448, 31408, 31450, 31451, 31409, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 10801, 10802, 10891, + 10892, 10893, 10894, 10901, 10900, 10902, 10906, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 32942, 32941, 32947, 32946, 32948, 32949, 32938, + 32939, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 34883, + 34902, 34909, 34906, 34896, 34893, 34888, 34910, 34878, 34895, 34905, + 34885, 34882, 34891, 34908, 34886, 34904, 34892, 34897, 34903, 34907, + 34880, 34879, 34884, 34900, 34894, 34890, 34889, 34898, 34881, 34899, + 34901, 34887, 34911, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 34918, 34920, 34917, + 34916, 34913, 34912, 34915, 34914, 34921, 34919, 41412, 41412, 41412, + 41412, 41412, 41412, 3761, 3762, 3775, 3776, 3773, 3774, 3757, 3758, + 3759, 41412, 3801, 3763, 3802, 3764, 3793, 3792, 3789, 3788, 3777, 3787, + 3786, 3791, 3790, 3779, 3770, 3769, 3766, 3765, 3778, 3772, 3771, 3768, + 3767, 3780, 3795, 3794, 3785, 3784, 3798, 3800, 3799, 3797, 3760, 3781, + 3782, 3783, 3796, 3829, 3834, 3835, 3832, 3833, 3826, 3827, 3828, 41412, + 3836, 3830, 3837, 3831, 3821, 3823, 3825, 3824, 3822, 3839, 3838, 3850, + 3851, 3852, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 3846, 3848, 3845, 3844, 3841, 3840, 3843, 3842, 3849, 3847, + 3820, 3818, 3815, 3806, 3808, 3813, 3811, 3803, 3809, 3819, 3817, 3816, + 3805, 3807, 3814, 3812, 3804, 3810, 3853, 41412, 41412, 41412, 25911, + 25912, 25857, 25856, 25866, 25851, 25855, 25854, 25868, 25852, 25848, + 25847, 25850, 25853, 25859, 25858, 25865, 25870, 25846, 25845, 25849, + 25872, 25862, 25863, 25864, 25873, 25871, 25869, 25860, 25861, 25867, + 25874, 41412, 41412, 25889, 25888, 25897, 25883, 25887, 25886, 25899, + 25884, 25880, 25879, 25882, 25885, 25891, 25890, 25896, 25901, 25878, + 25877, 25881, 25903, 25894, 25895, 41412, 25904, 25902, 25900, 25892, + 25893, 25898, 25905, 25906, 25908, 25910, 25907, 25909, 25876, 25875, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 25923, 25924, 25933, 25934, 25931, 25932, 25960, + 41412, 25925, 25961, 41412, 25926, 25939, 25938, 25952, 25951, 25940, + 25950, 25949, 25917, 25916, 25942, 25919, 25918, 25928, 25927, 25941, + 25922, 25920, 25930, 25929, 25943, 25954, 25953, 25948, 25947, 25956, + 25959, 25957, 25936, 25958, 25944, 25945, 25946, 25955, 25935, 25937, + 25915, 25921, 25970, 25975, 25976, 25973, 25974, 25969, 41412, 41412, + 41412, 25977, 41412, 25971, 25978, 41412, 25972, 25968, 25967, 25966, + 25964, 25965, 25979, 25963, 25962, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 25986, 25988, 25985, 25984, 25981, 25980, 25983, + 25982, 25989, 25987, 41412, 41412, 41412, 41412, 41412, 41412, 18927, + 18928, 18941, 18942, 18939, 18940, 41412, 18958, 18929, 41412, 18957, + 18930, 18964, 18963, 18946, 18945, 18960, 18954, 18953, 18938, 18937, + 18944, 18950, 18949, 18934, 18933, 18926, 18948, 18947, 18936, 18935, + 18943, 18952, 18951, 18932, 18931, 18925, 18956, 18955, 18959, 18961, + 18962, 18967, 18972, 18973, 18970, 18971, 41412, 18975, 18968, 41412, + 18974, 18969, 18966, 18965, 18976, 18987, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 18983, 18985, 18982, 18981, 18978, 18977, 18980, + 18979, 18986, 18984, 41412, 41412, 41412, 41412, 41412, 41412, 37877, + 37875, 37882, 37880, 37845, 37846, 37873, 37874, 37863, 37864, 37879, + 37859, 37862, 37847, 37850, 37851, 37860, 37861, 37848, 37849, 37852, + 37865, 37866, 37869, 37870, 37855, 37871, 37872, 37867, 37868, 37854, + 37885, 37856, 37878, 37883, 37853, 37881, 37876, 37884, 37857, 37858, + 37886, 37887, 37888, 41412, 41412, 41412, 41412, 37895, 37897, 37894, + 37893, 37890, 37889, 37892, 37891, 37898, 37896, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 25609, 25607, 25601, 25612, 25604, 25611, 25615, + 25606, 25603, 25605, 25608, 25602, 25617, 25613, 25610, 25616, 25614, + 25618, 25624, 25621, 25623, 25620, 25622, 25619, 25600, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 21676, 21680, 21678, 21679, 21617, + 21618, 21637, 21638, 21631, 21632, 21633, 21634, 21635, 21636, 21662, + 21619, 21663, 41412, 21653, 21652, 21651, 21650, 21639, 21649, 21648, + 21622, 21621, 21641, 21628, 21627, 21624, 21623, 21640, 21630, 21629, + 21626, 21625, 21642, 21655, 21654, 21647, 21646, 21658, 21661, 21659, + 21657, 21660, 21643, 21644, 21645, 21656, 21620, 21682, 21681, 21689, + 21690, 21687, 21688, 21684, 41412, 41412, 41412, 21685, 21683, 21686, + 21675, 21703, 21692, 21691, 21670, 21673, 21669, 21671, 21667, 21666, + 21674, 21665, 21668, 21672, 21664, 21699, 21701, 21698, 21697, 21694, + 21693, 21696, 21695, 21702, 21700, 21677, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 25255, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 35704, 35700, 35694, + 35703, 35696, 35705, 35709, 35711, 35706, 35701, 35702, 35707, 35695, + 35712, 35710, 35697, 35708, 35698, 35699, 35713, 35714, 35778, 35761, + 35759, 35768, 35767, 35763, 35769, 35765, 35764, 35771, 35772, 35773, + 35770, 35762, 35775, 35693, 35680, 35751, 35758, 36048, 36049, 35678, + 35653, 35779, 36047, 35715, 35780, 35766, 35774, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 35756, 8952, 8960, 8958, 8954, 8959, 8955, 8953, 8956, 8957, 9021, 8961, + 8970, 8969, 8963, 8962, 8974, 8964, 8965, 8973, 8967, 8968, 8975, 8976, + 8981, 8978, 8977, 8979, 8980, 8983, 8985, 8987, 8986, 8988, 8994, 8991, + 8992, 8996, 8989, 8990, 8995, 8993, 8998, 8997, 8999, 9001, 9002, 9006, + 9005, 9003, 9004, 9007, 9020, 9008, 9009, 9010, 9019, 9011, 9015, 9016, + 9014, 9012, 9013, 9018, 9017, 9022, 9023, 9034, 9025, 9029, 9030, 9031, + 9032, 9033, 9035, 9038, 9037, 9036, 9039, 9041, 9042, 9043, 9044, 9045, + 9046, 9047, 9048, 9049, 9050, 9051, 9052, 9053, 9054, 9055, 9056, 9057, + 9058, 9073, 9059, 9060, 9070, 9064, 9061, 9062, 9063, 9071, 9068, 9072, + 9069, 9065, 9067, 9080, 9076, 9077, 9078, 9081, 9092, 9082, 9085, 9086, + 9088, 9089, 9090, 9093, 9096, 9094, 9095, 9097, 9098, 9100, 9101, 9130, + 9137, 9131, 9132, 9133, 9134, 9135, 9136, 9138, 9140, 9139, 9141, 9144, + 9145, 9148, 9142, 9143, 9149, 9194, 9193, 9195, 9150, 9153, 9154, 9155, + 9151, 9152, 9156, 9159, 9157, 9160, 9162, 9171, 9172, 9173, 9174, 9192, + 9175, 9176, 9189, 9190, 9188, 9177, 9178, 9179, 9180, 9181, 9182, 9183, + 9186, 9187, 9196, 9286, 9197, 9198, 9200, 9199, 9206, 9201, 9203, 9205, + 9209, 9208, 9210, 9211, 9218, 9212, 9213, 9217, 9221, 9222, 9219, 9220, + 9228, 9225, 9229, 9230, 9231, 9232, 9233, 9235, 9236, 9238, 9237, 9240, + 9239, 9242, 9241, 9243, 9244, 9246, 9247, 9251, 9253, 9257, 9258, 9271, + 9263, 9264, 9259, 9260, 9261, 9265, 9269, 9266, 9267, 9268, 9272, 9273, + 9275, 9276, 9277, 9278, 9279, 9280, 9290, 9281, 9282, 9285, 9284, 9283, + 9287, 9288, 9289, 9291, 9292, 9295, 9296, 9297, 9298, 9299, 9301, 9300, + 9317, 9308, 9309, 9302, 9303, 9305, 9306, 9304, 9307, 9316, 9310, 9315, + 9313, 9312, 9314, 9319, 9338, 9320, 9321, 9322, 9325, 9324, 9326, 9327, + 9329, 9330, 9331, 9339, 9332, 9333, 9334, 9337, 9336, 9335, 9340, 9341, + 9343, 9344, 9345, 9346, 9348, 9352, 9349, 9353, 9351, 9350, 9354, 9355, + 9356, 9357, 9361, 9360, 9358, 9359, 9362, 9363, 9365, 9384, 9386, 9366, + 9368, 9367, 9369, 9370, 9373, 9374, 9372, 9371, 9375, 9376, 9377, 9378, + 9381, 9379, 9380, 9382, 9383, 9387, 9388, 9385, 9389, 9390, 9391, 9392, + 9394, 9397, 9396, 9398, 9399, 9401, 9402, 9403, 9407, 9406, 9404, 9405, + 9408, 9412, 9411, 9410, 9413, 9414, 9416, 9417, 9421, 9418, 9419, 9424, + 9422, 9425, 9426, 9428, 9427, 9429, 9430, 9452, 9451, 9454, 9455, 9436, + 9437, 9434, 9435, 9433, 9431, 9439, 9438, 9440, 9442, 9448, 9449, 9446, + 9447, 9456, 9457, 9458, 9476, 9461, 9462, 9463, 9459, 9460, 9464, 9465, + 9466, 9467, 9468, 9470, 9471, 9472, 9473, 9474, 9499, 9477, 9480, 9478, + 9479, 9485, 9486, 9483, 9484, 9481, 9482, 9487, 9488, 9496, 9489, 9490, + 9497, 9494, 9495, 9498, 9491, 9492, 9493, 9500, 9501, 9502, 9503, 9504, + 9505, 9506, 9508, 9509, 9507, 9510, 9511, 9552, 9553, 9514, 9515, 9512, + 9513, 9518, 9519, 9517, 9523, 9520, 9522, 9521, 9527, 9528, 9526, 9524, + 9525, 9529, 9530, 9531, 9532, 9533, 9534, 9535, 9554, 9540, 9536, 9537, + 9538, 9539, 9541, 9543, 9542, 9544, 9545, 9547, 9546, 9548, 9550, 9549, + 9555, 9556, 9559, 9560, 9557, 9558, 9634, 9629, 9630, 9631, 9632, 9633, + 9635, 9638, 9636, 9637, 9639, 9681, 9640, 9670, 9671, 9647, 9649, 9666, + 9650, 9672, 9654, 9652, 9653, 9655, 9656, 9657, 9658, 9667, 9668, 9661, + 9662, 9664, 9673, 9641, 9642, 9646, 9644, 9682, 9674, 9676, 9675, 9677, + 9683, 9684, 9678, 9679, 9680, 9685, 9686, 9687, 9690, 9691, 9692, 9688, + 9689, 9693, 9694, 9696, 9698, 9699, 9717, 9715, 9716, 9719, 9718, 9700, + 9707, 9705, 9706, 9701, 9702, 9708, 9709, 9710, 9711, 9712, 9714, 9720, + 9729, 9721, 9723, 9724, 9722, 9725, 9727, 9726, 9728, 9731, 9733, 9732, + 9734, 9735, 9768, 9770, 9736, 9738, 9737, 9740, 9743, 9741, 9742, 9746, + 9750, 9753, 9752, 9755, 9758, 9756, 9757, 9761, 9763, 9769, 9771, 9772, + 9776, 9783, 9781, 9779, 9780, 9782, 9785, 9784, 9777, 9778, 9786, 9789, + 9795, 9790, 9793, 9788, 9787, 9796, 9794, 9791, 9792, 9797, 9798, 9799, + 9800, 9801, 9802, 9803, 9805, 9808, 9809, 9812, 9813, 9814, 9810, 9811, + 9806, 9807, 9815, 9816, 9817, 9818, 9819, 9820, 9822, 9823, 9824, 9825, + 9826, 9830, 9827, 9851, 9844, 9845, 9850, 9833, 9832, 9846, 9849, 9847, + 9836, 9835, 9838, 9840, 9841, 9842, 9843, 9839, 9852, 9828, 9853, 9854, + 9855, 9856, 9857, 9858, 9866, 9864, 9862, 9865, 9861, 9863, 9859, 9860, + 9867, 9869, 9870, 9871, 9878, 9873, 9874, 9882, 9883, 9879, 9881, 9880, + 9884, 9886, 9885, 9887, 9898, 9889, 9888, 9897, 9895, 9890, 9891, 9892, + 9894, 9893, 9896, 9902, 9899, 9901, 9900, 9903, 9904, 9905, 9906, 9909, + 9910, 9912, 9913, 9914, 9915, 9917, 9916, 9918, 9925, 9919, 9920, 9926, + 9921, 9922, 9923, 9924, 9927, 9931, 9928, 9929, 9930, 9932, 9933, 9934, + 9935, 9941, 9939, 9937, 9938, 9936, 9940, 9942, 9944, 9961, 9962, 9945, + 9950, 9952, 9946, 9949, 9947, 9948, 9953, 9959, 9960, 9954, 9957, 9958, + 9963, 9969, 9968, 9964, 9966, 9965, 10050, 10051, 9970, 9971, 9976, 9977, + 9974, 9975, 9978, 9972, 9973, 9979, 9982, 9983, 9985, 9986, 9987, 9991, + 9988, 9989, 9990, 9980, 9981, 9992, 9994, 9993, 9995, 9997, 9998, 9999, + 10005, 10004, 10000, 10001, 10002, 10037, 10006, 10007, 10008, 10009, + 10010, 10029, 10012, 10013, 10015, 10014, 10016, 10017, 10033, 10018, + 10020, 10019, 10032, 10022, 10030, 10034, 10025, 10024, 10031, 10026, + 10028, 10027, 10035, 10036, 10038, 10042, 10040, 10041, 10039, 10045, + 10044, 10043, 10049, 10046, 10047, 10048, 10052, 10054, 10053, 10057, + 10055, 10072, 10058, 10061, 10063, 10059, 10060, 10064, 10062, 10065, + 10067, 10069, 10070, 10071, 9475, 8971, 8982, 9000, 9066, 9075, 9091, + 9099, 9191, 9184, 9202, 9204, 9294, 9318, 9364, 9393, 9395, 9409, 9415, + 9450, 9423, 9453, 9432, 9441, 9445, 9516, 9645, 9648, 9663, 9695, 9713, + 9730, 9739, 9767, 9765, 9744, 9775, 9804, 9821, 9831, 9951, 9984, 9967, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 8936, 8931, 8868, 8854, 8915, 8911, 8843, 8888, 8935, 8876, + 8860, 8921, 8910, 8842, 8887, 8874, 8858, 8917, 8907, 8837, 8884, 8897, + 8941, 8933, 8870, 8856, 8919, 8909, 8839, 8886, 8898, 8942, 8934, 8871, + 8857, 8943, 8925, 8926, 8872, 8848, 8914, 8906, 8836, 8883, 8901, 8944, + 8927, 8928, 8873, 8849, 8912, 8913, 8896, 8939, 8922, 8923, 8863, 8853, + 8929, 8930, 8864, 8867, 8865, 8866, 8920, 8905, 8903, 8904, 8840, 8841, + 8879, 8881, 8882, 8880, 8937, 8932, 8869, 8855, 8916, 8895, 8938, 8924, + 8861, 8862, 8851, 8852, 8878, 8877, 8891, 8940, 8900, 8946, 8850, 8899, + 8945, 8892, 8893, 8889, 8890, 8894, 8902, 8844, 8847, 8846, 8845, 8875, + 8859, 8918, 8908, 8838, 8885, 41412, 8951, 8950, 8949, 8947, 8948, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 8972, 8966, 8984, 9024, 9026, 9027, 9028, 9040, 9079, 9074, 9084, 9083, + 9087, 9104, 9102, 9103, 9105, 9106, 9124, 9110, 9107, 9108, 9109, 9126, + 9127, 9125, 9114, 9113, 9111, 9112, 9117, 9115, 9116, 9118, 9119, 9120, + 9121, 9128, 9129, 9123, 9122, 9147, 9146, 9158, 9161, 9166, 9170, 9163, + 9167, 9168, 9164, 9165, 9169, 9185, 9207, 9214, 9215, 9216, 9223, 9224, + 9227, 9226, 9234, 9245, 9248, 9249, 9250, 9252, 9254, 9255, 9256, 9262, + 9270, 9274, 9293, 9311, 9323, 9328, 9342, 9347, 9400, 9420, 9443, 9444, + 9551, 9571, 9561, 9562, 9570, 9566, 9567, 9568, 9565, 9564, 9563, 9569, + 9573, 9572, 9579, 9580, 9574, 9575, 9576, 9581, 9577, 9578, 9582, 9583, + 9584, 9585, 9586, 9587, 9594, 9588, 9593, 9592, 9590, 9589, 9591, 9595, + 9596, 9601, 9602, 9597, 9598, 9599, 9600, 9628, 9624, 9603, 9611, 9612, + 9610, 9609, 9613, 9604, 9605, 9607, 9608, 9606, 9625, 9614, 9621, 9623, + 9618, 9619, 9622, 9617, 9620, 9616, 9615, 9626, 9627, 9643, 9669, 9651, + 9659, 9660, 9665, 9697, 9704, 9703, 9764, 9745, 9747, 9766, 9748, 9749, + 9751, 9754, 9759, 9760, 9762, 9829, 9848, 9834, 9837, 9868, 9872, 9875, + 9876, 9877, 9908, 9907, 9911, 9943, 9955, 9956, 9996, 10003, 10011, + 10023, 10021, 10056, 10066, 10068, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 10145, 10146, 10147, + 10148, 10149, 10150, 10151, 10152, 10155, 10156, 10153, 10154, 10157, + 10158, 10159, 10160, 10161, 10162, 10163, 10164, 10165, 10166, 10167, + 10168, 10169, 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, + 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185, 10186, 10187, + 10188, 10189, 10190, 10191, 10211, 10212, 10213, 10214, 10215, 10216, + 10217, 10218, 10219, 10194, 10195, 10196, 10197, 10198, 10192, 10193, + 10199, 10200, 10201, 10220, 10221, 10222, 10223, 10224, 10225, 10226, + 10227, 10228, 10229, 10202, 10203, 10204, 10205, 10206, 10207, 10208, + 10209, 10210, 10230, 10231, 10232, 10233, 10234, 10235, 10236, 10237, + 10238, 10239, 10240, 10241, 10242, 10243, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 11681, + 11682, 11683, 11684, 11679, 11680, 11676, 11677, 11678, 11685, 11686, + 11687, 11692, 11693, 11694, 11695, 11688, 11689, 11696, 11697, 11690, + 11691, 11698, 11699, 11726, 11727, 11728, 11729, 11730, 11731, 11732, + 11733, 11734, 11735, 11716, 11717, 11714, 11715, 11718, 11719, 11720, + 11721, 11722, 11723, 11724, 11700, 11701, 11708, 11702, 11703, 11704, + 11705, 11709, 11706, 11707, 11710, 11711, 11712, 11713, 11736, 11737, + 11738, 11739, 11740, 11741, 11742, 11743, 11744, 11745, 11746, 11747, + 11748, 11749, 11750, 11751, 11752, 11753, 11754, 11755, 11725, 11795, + 11796, 11797, 11798, 11793, 11794, 11799, 11800, 11801, 11802, 11807, + 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, 11813, + 11814, 11815, 11816, 11817, 11818, 11819, 11820, 11821, 11822, 11823, + 11824, 11825, 11826, 11827, 11828, 11829, 11830, 11833, 11834, 11835, + 11836, 11837, 11838, 11839, 11831, 11832, 11840, 11913, 11914, 11915, + 11916, 11917, 11918, 11919, 11920, 11921, 11922, 11904, 11905, 11906, + 11907, 11908, 11909, 11910, 11902, 11903, 11911, 11912, 11845, 11841, + 11842, 11846, 11847, 11843, 11844, 11848, 11849, 11850, 11851, 11852, + 11857, 11858, 11859, 11860, 11861, 11862, 11853, 11854, 11863, 11855, + 11856, 11864, 11865, 11866, 11867, 11868, 11869, 11870, 11871, 11872, + 11873, 11874, 11879, 11875, 11876, 11880, 11877, 11878, 11881, 11882, + 11883, 11884, 11885, 11895, 11896, 11897, 11898, 11899, 11900, 11901, + 11886, 11887, 11888, 11889, 11890, 11891, 11892, 11893, 11894, 11927, + 11928, 11929, 11930, 11931, 11932, 11933, 11923, 11924, 11925, 11926, + 11959, 11960, 11961, 11962, 11963, 11964, 11955, 11956, 11957, 11958, + 11965, 11966, 11934, 11935, 11938, 11939, 11940, 11941, 11942, 11943, + 11944, 11936, 11937, 11945, 11948, 11949, 11950, 11951, 11946, 11947, + 11952, 11953, 11954, 11970, 11971, 11972, 11973, 11974, 11975, 11976, + 11977, 11978, 11979, 11982, 11983, 11984, 11980, 11981, 11985, 11986, + 11987, 11988, 11989, 11990, 12026, 12024, 12025, 12027, 12028, 12029, + 12030, 12031, 12032, 12033, 12034, 11997, 11991, 11992, 11998, 11999, + 12000, 12001, 12002, 11993, 11994, 11995, 11996, 12003, 12010, 12011, + 12012, 12013, 12014, 12004, 12005, 12006, 12007, 12008, 12009, 12015, + 12016, 12021, 12017, 12018, 12019, 12020, 12022, 12023, 12041, 12042, + 12043, 12044, 12045, 12039, 12040, 12036, 12037, 12038, 12046, 12047, + 12050, 12048, 12049, 12051, 12052, 12053, 12054, 12055, 12056, 12057, + 12058, 12083, 12084, 12087, 12088, 12089, 12090, 12091, 12085, 12086, + 12092, 12093, 12094, 12063, 12064, 12065, 12066, 12067, 12068, 12059, + 12060, 12061, 12062, 12069, 12070, 12075, 12076, 12077, 12071, 12072, + 12078, 12073, 12074, 12079, 12080, 12081, 12082, 12095, 12096, 12097, + 12098, 12099, 12102, 12103, 12104, 12105, 12106, 12100, 12101, 12107, + 12108, 12116, 12117, 12118, 12119, 12112, 12113, 12120, 12121, 12122, + 12114, 12115, 12123, 12124, 12125, 12126, 12127, 12128, 12129, 12130, + 12771, 12772, 12773, 12774, 12775, 12776, 12777, 12778, 12142, 12138, + 12139, 12143, 12144, 12145, 12140, 12141, 12146, 12147, 12149, 12150, + 12151, 12154, 12152, 12153, 12155, 12156, 12157, 12158, 12159, 12160, + 12170, 12171, 12178, 12161, 12162, 12163, 12164, 12165, 12166, 12167, + 12168, 12169, 12179, 12180, 12172, 12173, 12174, 12175, 12176, 12177, + 12181, 12182, 12189, 12190, 12183, 12184, 12191, 12185, 12186, 12192, + 12193, 12194, 12187, 12188, 12195, 12201, 12199, 12200, 12202, 12196, + 12197, 12198, 12203, 12204, 12205, 12206, 12207, 12208, 12209, 12210, + 12211, 12212, 12213, 12214, 12271, 12272, 12273, 12274, 12275, 12276, + 12277, 12278, 12279, 12234, 12235, 12236, 12237, 12238, 12239, 12240, + 12241, 12231, 12232, 12233, 12242, 12259, 12260, 12261, 12262, 12263, + 12257, 12258, 12264, 12265, 12266, 12267, 12251, 12252, 12253, 12243, + 12244, 12245, 12246, 12247, 12248, 12254, 12249, 12250, 12255, 12256, + 12268, 12269, 12270, 12282, 12283, 12284, 12285, 12280, 12281, 12286, + 12287, 12288, 12289, 12292, 12293, 12294, 12295, 12296, 12297, 12298, + 12290, 12291, 12299, 12300, 12301, 12319, 12320, 12321, 12322, 12323, + 12324, 12325, 12326, 12327, 12302, 12303, 12304, 12305, 12308, 12309, + 12310, 12311, 12312, 12313, 12306, 12307, 12314, 12317, 12318, 12315, + 12316, 12335, 12336, 12339, 12340, 12341, 12337, 12338, 12328, 12329, + 12330, 12331, 12332, 12333, 12334, 12342, 12343, 12344, 12345, 12346, + 12347, 12348, 12351, 12352, 12353, 12354, 12355, 12356, 12357, 12358, + 12349, 12350, 12359, 12360, 12367, 12368, 12369, 12361, 12362, 12363, + 12364, 12370, 12371, 12372, 12365, 12366, 12378, 12379, 12382, 12383, + 12380, 12381, 12384, 12385, 12373, 12374, 12375, 12376, 12377, 12386, + 12387, 12388, 12393, 12394, 12395, 12396, 12397, 12398, 12399, 12400, + 12401, 12402, 12389, 12390, 12391, 12392, 12404, 12405, 12408, 12406, + 12407, 12409, 12410, 12411, 12412, 12413, 12414, 12415, 12416, 12779, + 12780, 12781, 12782, 12783, 12784, 12785, 12422, 12420, 12421, 12417, + 12418, 12419, 12423, 12424, 12425, 12426, 12427, 12428, 12429, 12430, + 12433, 12434, 12435, 12436, 12437, 12431, 12432, 12438, 12439, 12440, + 12441, 12442, 12443, 12444, 12445, 12446, 12447, 12448, 12449, 12450, + 12455, 12451, 12452, 12456, 12457, 12458, 12453, 12454, 12459, 12460, + 12461, 12467, 12468, 12469, 12470, 12462, 12463, 12464, 12471, 12472, + 12465, 12466, 12473, 12474, 12478, 12479, 12480, 12481, 12482, 12483, + 12475, 12476, 12477, 12484, 12485, 12486, 12489, 12490, 12491, 12492, + 12493, 12487, 12488, 12494, 12495, 12496, 12497, 12498, 12499, 12500, + 12501, 12502, 12503, 12504, 12513, 12514, 12505, 12506, 12515, 12516, + 12517, 12507, 12508, 12509, 12510, 12511, 12512, 12522, 12518, 12519, + 12523, 12524, 12525, 12526, 12520, 12521, 12527, 12528, 12529, 12539, + 12540, 12541, 12542, 12543, 12544, 12545, 12546, 12547, 12548, 12534, + 12535, 12530, 12531, 12532, 12533, 12536, 12537, 12538, 12553, 12554, + 12555, 12556, 12557, 12550, 12551, 12552, 12558, 12559, 12560, 12587, + 12588, 12589, 12590, 12591, 12592, 12593, 12594, 12595, 12596, 12565, + 12566, 12567, 12561, 12562, 12568, 12569, 12570, 12571, 12572, 12563, + 12564, 12575, 12576, 12573, 12574, 12577, 12578, 12579, 12580, 12581, + 12582, 12583, 12584, 12585, 12586, 12600, 12601, 12602, 12603, 12604, + 12605, 12606, 12607, 12608, 12609, 12610, 12611, 12612, 12613, 12614, + 12615, 12597, 12598, 12599, 12616, 12617, 12626, 12618, 12619, 12620, + 12621, 12623, 12624, 12625, 12627, 12628, 12629, 12630, 12631, 12632, + 12633, 12634, 12635, 12636, 12637, 12638, 12639, 12640, 12641, 12642, + 12643, 12644, 12645, 12646, 12653, 12654, 12647, 12648, 12655, 12656, + 12657, 12658, 12649, 12650, 12651, 12652, 12659, 12660, 12661, 12662, + 12667, 12663, 12664, 12668, 12669, 12670, 12665, 12666, 12671, 12672, + 12673, 12674, 12680, 12681, 12676, 12677, 12682, 12683, 12684, 12685, + 12686, 12678, 12679, 12687, 12688, 12695, 12696, 12697, 12689, 12690, + 12698, 12699, 12691, 12692, 12693, 12694, 12700, 12703, 12704, 12705, + 12706, 12701, 12702, 12707, 12716, 12717, 12718, 12709, 12710, 12711, + 12719, 12712, 12713, 12720, 12714, 12715, 12721, 12722, 12723, 12724, + 12725, 12726, 12727, 12728, 12729, 12742, 12730, 12731, 12732, 12733, + 12734, 12735, 12736, 12737, 12738, 12739, 12740, 12741, 12743, 12744, + 12745, 12746, 12766, 12767, 12768, 12769, 12770, 12747, 12748, 12749, + 12750, 12751, 12752, 12753, 12754, 12755, 12756, 12757, 12758, 12759, + 12760, 12761, 12762, 12763, 12764, 12765, 11759, 11760, 11761, 11762, + 11763, 11764, 11756, 11757, 11758, 11765, 11766, 11770, 11771, 11772, + 11773, 11774, 11775, 11776, 11777, 11778, 11779, 11780, 11781, 11782, + 11783, 11784, 11785, 11786, 11787, 11788, 11789, 11767, 11768, 11769, + 12622, 12675, 12111, 12136, 12133, 12135, 12132, 12403, 11792, 11969, + 12137, 12134, 12131, 11791, 11968, 11790, 11967, 12230, 12035, 12109, + 12148, 12110, 12549, 12708, 12226, 12217, 12221, 12228, 12224, 12218, + 12223, 12220, 12227, 12216, 12222, 12229, 12225, 12219, 12215, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 12786, + 12787, 12788, 12789, 12790, 12791, 12792, 12793, 12794, 12795, 12796, + 12797, 12798, 12799, 12800, 12801, 12802, 12803, 12804, 12805, 12806, + 12807, 12808, 12809, 12810, 12811, 12812, 12813, 12814, 12815, 12816, + 12817, 12818, 12819, 12820, 12821, 12822, 12823, 12824, 12825, 12826, + 12827, 12828, 12829, 12830, 12831, 12832, 12833, 12834, 12835, 12836, + 12837, 12838, 12839, 12840, 12841, 12842, 12843, 12844, 12845, 12846, + 12847, 12848, 12849, 12850, 12851, 12852, 12853, 12854, 12855, 12856, + 12857, 12858, 12859, 12860, 12861, 12862, 12863, 12864, 12865, 12866, + 12867, 12868, 12869, 12870, 12871, 12872, 12873, 12874, 12875, 12876, + 12877, 12878, 12879, 12880, 12881, 12882, 12883, 12884, 12885, 12886, + 12887, 12888, 12889, 12890, 12891, 12892, 12893, 12894, 12895, 12896, + 12897, 12898, 12899, 12900, 12901, 12902, 12903, 12904, 12905, 12906, + 12907, 12908, 12909, 12910, 12911, 12912, 12913, 12914, 12915, 12916, + 12917, 12918, 12919, 12920, 12921, 12922, 12923, 12924, 12925, 12926, + 12927, 12928, 12929, 12930, 12931, 12932, 12933, 12934, 12935, 12936, + 12937, 12938, 12939, 12940, 12941, 12942, 12943, 12944, 12945, 12946, + 12947, 12948, 12949, 12950, 12951, 12952, 12953, 12954, 12955, 12956, + 12957, 12958, 12959, 12960, 12961, 12962, 12963, 12964, 12965, 12966, + 12967, 12968, 12969, 12970, 12971, 12972, 12973, 12974, 12975, 12976, + 12977, 12978, 12979, 12980, 12981, 12982, 12983, 12984, 12985, 12986, + 12987, 12988, 12989, 12990, 12991, 12992, 12993, 12994, 12995, 12996, + 12997, 12998, 12999, 13000, 13001, 13002, 13003, 13004, 13005, 13006, + 13007, 13008, 13009, 13010, 13011, 13012, 13013, 13014, 13015, 13016, + 13017, 13018, 13019, 13020, 13021, 13022, 13023, 13024, 13025, 13026, + 13027, 13028, 13029, 13030, 13031, 13032, 13033, 13034, 13035, 13036, + 13037, 13038, 13039, 13040, 13041, 13042, 13043, 13044, 13045, 13046, + 13047, 13048, 13049, 13050, 13051, 13052, 13053, 13054, 13055, 13056, + 13057, 13058, 13059, 13060, 13061, 13062, 13063, 13064, 13065, 13066, + 13067, 13068, 13069, 13070, 13071, 13072, 13073, 13074, 13075, 13076, + 13077, 13078, 13079, 13080, 13081, 13082, 13083, 13084, 13085, 13086, + 13087, 13088, 13089, 13090, 13091, 13092, 13093, 13094, 13095, 13096, + 13097, 13098, 13099, 13100, 13101, 13102, 13103, 13104, 13105, 13106, + 13107, 13108, 13109, 13110, 13111, 13112, 13113, 13114, 13115, 13116, + 13117, 13118, 13119, 13120, 13121, 13122, 13123, 13124, 13125, 13126, + 13127, 13128, 13129, 13130, 13131, 13132, 13133, 13134, 13135, 13136, + 13137, 13138, 13139, 13140, 13141, 13142, 13143, 13144, 13145, 13146, + 13147, 13148, 13149, 13150, 13151, 13152, 13153, 13154, 13155, 13156, + 13157, 13158, 13159, 13160, 13161, 13162, 13163, 13164, 13165, 13166, + 13167, 13168, 13169, 13170, 13171, 13172, 13173, 13174, 13175, 13176, + 13177, 13178, 13179, 13180, 13181, 13182, 13183, 13184, 13185, 13186, + 13187, 13188, 13189, 13190, 13191, 13192, 13193, 13194, 13195, 13196, + 13197, 13198, 13199, 13200, 13201, 13202, 13203, 13204, 13205, 13206, + 13207, 13208, 13209, 13210, 13211, 13212, 13213, 13214, 13215, 13216, + 13217, 13218, 13219, 13220, 13221, 13222, 13223, 13224, 13225, 13226, + 13227, 13228, 13229, 13230, 13231, 13232, 13233, 13234, 13235, 13236, + 13237, 13238, 13239, 13240, 13241, 13242, 13243, 13244, 13245, 13246, + 13247, 13248, 13249, 13250, 13251, 13252, 13253, 13254, 13255, 13256, + 13257, 13258, 13259, 13260, 13261, 13262, 13263, 13264, 13265, 13266, + 13267, 13268, 13269, 13270, 13271, 13272, 13273, 13274, 13275, 13276, + 13277, 13278, 13279, 13280, 13281, 13282, 13283, 13284, 13285, 13286, + 13287, 13288, 13289, 13290, 13291, 13292, 13293, 13294, 13295, 13296, + 13297, 13298, 13299, 13300, 13301, 13302, 13303, 13304, 13305, 13306, + 13307, 13308, 13309, 13310, 13311, 13312, 13313, 13314, 13315, 13316, + 13317, 13318, 13319, 13320, 13321, 13322, 13323, 13324, 13325, 13326, + 13327, 13328, 13329, 13330, 13331, 13332, 13333, 13334, 13335, 13336, + 13337, 13338, 13339, 13340, 13341, 13342, 13343, 13344, 13345, 13346, + 13347, 13348, 13349, 13350, 13351, 13352, 13353, 13354, 13355, 13356, + 13357, 13358, 13359, 13360, 13361, 13362, 13363, 13364, 13365, 13366, + 13367, 13368, 13369, 13370, 13371, 13372, 13373, 13374, 13375, 13376, + 13377, 13378, 13379, 13380, 13381, 13382, 13383, 13384, 13385, 13386, + 13387, 13388, 13389, 13390, 13391, 13392, 13393, 13394, 13395, 13396, + 13397, 13398, 13399, 13400, 13401, 13402, 13403, 13404, 13405, 13406, + 13407, 13408, 13409, 13410, 13411, 13412, 13413, 13414, 13415, 13416, + 13417, 13418, 13419, 13420, 13421, 13422, 13423, 13424, 13425, 13426, + 13427, 13428, 13429, 13430, 13431, 13432, 13433, 13434, 13435, 13436, + 13437, 13438, 13439, 13440, 13441, 13442, 13443, 13444, 13445, 13446, + 13447, 13448, 13449, 13450, 13451, 13452, 13453, 13454, 13455, 13456, + 13457, 13458, 13459, 13460, 13461, 13462, 13463, 13464, 13465, 13466, + 13467, 13468, 13469, 13470, 13471, 13472, 13473, 13474, 13475, 13476, + 13477, 13478, 13479, 13480, 13481, 13482, 13483, 13484, 13485, 13486, + 13487, 13488, 13489, 13490, 13491, 13492, 13493, 13494, 13495, 13496, + 13497, 13498, 13499, 13500, 13501, 13502, 13503, 13504, 13505, 13506, + 13507, 13508, 13509, 13510, 13511, 13512, 13513, 13514, 13515, 13516, + 13517, 13518, 13519, 13520, 13521, 13522, 13523, 13524, 13525, 13526, + 13527, 13528, 13529, 13530, 13531, 13532, 13533, 13534, 13535, 13536, + 13537, 13538, 13539, 13540, 13541, 13542, 13543, 13544, 13545, 13546, + 13547, 13548, 13549, 13550, 13551, 13552, 13553, 13554, 13555, 13556, + 13557, 13558, 13559, 13560, 13561, 13562, 13563, 13564, 13565, 13566, + 13567, 13568, 13569, 13570, 13571, 13572, 13573, 13574, 13575, 13576, + 13577, 13578, 13579, 13580, 13581, 13582, 13583, 13584, 13585, 13586, + 13587, 13588, 13589, 13590, 13591, 13592, 13593, 13594, 13595, 13596, + 13597, 13598, 13599, 13600, 13601, 13602, 13603, 13604, 13605, 13606, + 13607, 13608, 13609, 13610, 13611, 13612, 13613, 13614, 13615, 13616, + 13617, 13618, 13619, 13620, 13621, 13622, 13623, 13624, 13625, 13626, + 13627, 13628, 13629, 13630, 13631, 13632, 13633, 13634, 13635, 13636, + 13637, 13638, 13639, 13640, 13641, 13642, 13643, 13644, 13645, 13646, + 13647, 13648, 13649, 13650, 13651, 13652, 13653, 13654, 13655, 13656, + 13657, 13658, 13659, 13660, 13661, 13662, 13663, 13664, 13665, 13666, + 13667, 13668, 13669, 13670, 13671, 13672, 13673, 13674, 13675, 13676, + 13677, 13678, 13679, 13680, 13681, 13682, 13683, 13684, 13685, 13686, + 13687, 13688, 13689, 13690, 13691, 13692, 13693, 13694, 13695, 13696, + 13697, 13698, 13699, 13700, 13701, 13702, 13703, 13704, 13705, 13706, + 13707, 13708, 13709, 13710, 13711, 13712, 13713, 13714, 13715, 13716, + 13717, 13718, 13719, 13720, 13721, 13722, 13723, 13724, 13725, 13726, + 13727, 13728, 13729, 13730, 13731, 13732, 13733, 13734, 13735, 13736, + 13737, 13738, 13739, 13740, 13741, 13742, 13743, 13744, 13745, 13746, + 13747, 13748, 13749, 13750, 13751, 13752, 13753, 13754, 13755, 13756, + 13757, 13758, 13759, 13760, 13761, 13762, 13763, 13764, 13765, 13766, + 13767, 13768, 13769, 13770, 13771, 13772, 13773, 13774, 13775, 13776, + 13777, 13778, 13779, 13780, 13781, 13782, 13783, 13784, 13785, 13786, + 13787, 13788, 13789, 13790, 13791, 13792, 13793, 13794, 13795, 13796, + 13797, 13798, 13799, 13800, 13801, 13802, 13803, 13804, 13805, 13806, + 13807, 13808, 13809, 13810, 13811, 13812, 13813, 13814, 13815, 13816, + 13817, 13818, 13819, 13820, 13821, 13822, 13823, 13824, 13825, 13826, + 13827, 13828, 13829, 13830, 13831, 13832, 13833, 13834, 13835, 13836, + 13837, 13838, 13839, 13840, 13841, 13842, 13843, 13844, 13845, 13846, + 13847, 13848, 13849, 13850, 13851, 13852, 13853, 13854, 13855, 13856, + 13857, 13858, 13859, 13860, 13861, 13862, 13863, 13864, 13865, 13866, + 13867, 13868, 13869, 13870, 13871, 13872, 13873, 13874, 13875, 13876, + 13877, 13878, 13879, 13880, 13881, 13882, 13883, 13884, 13885, 13886, + 13887, 13888, 13889, 13890, 13891, 13892, 13893, 13894, 13895, 13896, + 13897, 13898, 13899, 13900, 13901, 13902, 13903, 13904, 13905, 13906, + 13907, 13908, 13909, 13910, 13911, 13912, 13913, 13914, 13915, 13916, + 13917, 13918, 13919, 13920, 13921, 13922, 13923, 13924, 13925, 13926, + 13927, 13928, 13929, 13930, 13931, 13932, 13933, 13934, 13935, 13936, + 13937, 13938, 13939, 13940, 13941, 13942, 13943, 13944, 13945, 13946, + 13947, 13948, 13949, 13950, 13951, 13952, 13953, 13954, 13955, 13956, + 13957, 13958, 13959, 13960, 13961, 13962, 13963, 13964, 13965, 13966, + 13967, 13968, 13969, 13970, 13971, 13972, 13973, 13974, 13975, 13976, + 13977, 13978, 13979, 13980, 13981, 13982, 13983, 13984, 13985, 13986, + 13987, 13988, 13989, 13990, 13991, 13992, 13993, 13994, 13995, 13996, + 13997, 13998, 13999, 14000, 14001, 14002, 14003, 14004, 14005, 14006, + 14007, 14008, 14009, 14010, 14011, 14012, 14013, 14014, 14015, 14016, + 14017, 14018, 14019, 14020, 14021, 14022, 14023, 14024, 14025, 14026, + 14027, 14028, 14029, 14030, 14031, 14032, 14033, 14034, 14035, 14036, + 14037, 14038, 14039, 14040, 14041, 14042, 14043, 14044, 14045, 14046, + 14047, 14048, 14049, 14050, 14051, 14052, 14053, 14054, 14055, 14056, + 14057, 14058, 14059, 14060, 14061, 14062, 14063, 14064, 14065, 14066, + 14067, 14068, 14069, 14070, 14071, 14072, 14073, 14074, 14075, 14076, + 14077, 14078, 14079, 14080, 14081, 14082, 14083, 14084, 14085, 14086, + 14087, 14088, 14089, 14090, 14091, 14092, 14093, 14094, 14095, 14096, + 14097, 14098, 14099, 14100, 14101, 14102, 14103, 14104, 14105, 14106, + 14107, 14108, 14109, 14110, 14111, 14112, 14113, 14114, 14115, 14116, + 14117, 14118, 14119, 14120, 14121, 14122, 14123, 14124, 14125, 14126, + 14127, 14128, 14129, 14130, 14131, 14132, 14133, 14134, 14135, 14136, + 14137, 14138, 14139, 14140, 14141, 14142, 14143, 14144, 14145, 14146, + 14147, 14148, 14149, 14150, 14151, 14152, 14153, 14154, 14155, 14156, + 14157, 14158, 14159, 14160, 14161, 14162, 14163, 14164, 14165, 14166, + 14167, 14168, 14169, 14170, 14171, 14172, 14173, 14174, 14175, 14176, + 14177, 14178, 14179, 14180, 14181, 14182, 14183, 14184, 14185, 14186, + 14187, 14188, 14189, 14190, 14191, 14192, 14193, 14194, 14195, 14196, + 14197, 14198, 14199, 14200, 14201, 14202, 14203, 14204, 14205, 14206, + 14207, 14208, 14209, 14210, 14211, 14212, 14213, 14214, 14215, 14216, + 14217, 14218, 14219, 14220, 14221, 14222, 14223, 14224, 14225, 14226, + 14227, 14228, 14229, 14230, 14231, 14232, 14233, 14234, 14235, 14236, + 14237, 14238, 14239, 14240, 14241, 14242, 14243, 14244, 14245, 14246, + 14247, 14248, 14249, 14250, 14251, 14252, 14253, 14254, 14255, 14256, + 14257, 14258, 14259, 14260, 14261, 14262, 14263, 14264, 14265, 14266, + 14267, 14268, 14269, 14270, 14271, 14272, 14273, 14274, 14275, 14276, + 14277, 14278, 14279, 14280, 14281, 14282, 14283, 14284, 14285, 14286, + 14287, 14288, 14289, 14290, 14291, 14292, 14293, 14294, 14295, 14296, + 14297, 14298, 14299, 14300, 14301, 14302, 14303, 14304, 14305, 14306, + 14307, 14308, 14309, 14310, 14311, 14312, 14313, 14314, 14315, 14316, + 14317, 14318, 14319, 14320, 14321, 14322, 14323, 14324, 14325, 14326, + 14327, 14328, 14329, 14330, 14331, 14332, 14333, 14334, 14335, 14336, + 14337, 14338, 14339, 14340, 14341, 14342, 14343, 14344, 14345, 14346, + 14347, 14348, 14349, 14350, 14351, 14352, 14353, 14354, 14355, 14356, + 14357, 14358, 14359, 14360, 14361, 14362, 14363, 14364, 14365, 14366, + 14367, 14368, 14369, 14370, 14371, 14372, 14373, 14374, 14375, 14376, + 14377, 14378, 14379, 14380, 14381, 14382, 14383, 14384, 14385, 14386, + 14387, 14388, 14389, 14390, 14391, 14392, 14393, 14394, 14395, 14396, + 14397, 14398, 14399, 14400, 14401, 14402, 14403, 14404, 14405, 14406, + 14407, 14408, 14409, 14410, 14411, 14412, 14413, 14414, 14415, 14416, + 14417, 14418, 14419, 14420, 14421, 14422, 14423, 14424, 14425, 14426, + 14427, 14428, 14429, 14430, 14431, 14432, 14433, 14434, 14435, 14436, + 14437, 14438, 14439, 14440, 14441, 14442, 14443, 14444, 14445, 14446, + 14447, 14448, 14449, 14450, 14451, 14452, 14453, 14454, 14455, 14456, + 14457, 14458, 14459, 14460, 14461, 14462, 14463, 14464, 14465, 14466, + 14467, 14468, 14469, 14470, 14471, 14472, 14473, 14474, 14475, 14476, + 14477, 14478, 14479, 14480, 14481, 14482, 14483, 14484, 14485, 14486, + 14487, 14488, 14489, 14490, 14491, 14492, 14493, 14494, 14495, 14496, + 14497, 14498, 14499, 14500, 14501, 14502, 14503, 14504, 14505, 14506, + 14507, 14508, 14509, 14510, 14511, 14512, 14513, 14514, 14515, 14516, + 14517, 14518, 14519, 14520, 14521, 14522, 14523, 14524, 14525, 14526, + 14527, 14528, 14529, 14530, 14531, 14532, 14533, 14534, 14535, 14536, + 14537, 14538, 14539, 14540, 14541, 14542, 14543, 14544, 14545, 14546, + 14547, 14548, 14549, 14550, 14551, 14552, 14553, 14554, 14555, 14556, + 14557, 14558, 14559, 14560, 14561, 14562, 14563, 14564, 14565, 14566, + 14567, 14568, 14569, 14570, 14571, 14572, 14573, 14574, 14575, 14576, + 14577, 14578, 14579, 14580, 14581, 14582, 14583, 14584, 14585, 14586, + 14587, 14588, 14589, 14590, 14591, 14592, 14593, 14594, 14595, 14596, + 14597, 14598, 14599, 14600, 14601, 14602, 14603, 14604, 14605, 14606, + 14607, 14608, 14609, 14610, 14611, 14612, 14613, 14614, 14615, 14616, + 14617, 14618, 14619, 14620, 14621, 14622, 14623, 14624, 14625, 14626, + 14627, 14628, 14629, 14630, 14631, 14632, 14633, 14634, 14635, 14636, + 14637, 14638, 14639, 14640, 14641, 14642, 14643, 14644, 14645, 14646, + 14647, 14648, 14649, 14650, 14651, 14652, 14653, 14654, 14655, 14656, + 14657, 14658, 14659, 14660, 14661, 14662, 14663, 14664, 14665, 14666, + 14667, 14668, 14669, 14670, 14671, 14672, 14673, 14674, 14675, 14676, + 14677, 14678, 14679, 14680, 14681, 14682, 14683, 14684, 14685, 14686, + 14687, 14688, 14689, 14690, 14691, 14692, 14693, 14694, 14695, 14696, + 14697, 14698, 14699, 14700, 14701, 14702, 14703, 14704, 14705, 14706, + 14707, 14708, 14709, 14710, 14711, 14712, 14713, 14714, 14715, 14716, + 14717, 14718, 14719, 14720, 14721, 14722, 14723, 14724, 14725, 14726, + 14727, 14728, 14729, 14730, 14731, 14732, 14733, 14734, 14735, 14736, + 14737, 14738, 14739, 14740, 14741, 14742, 14743, 14744, 14745, 14746, + 14747, 14748, 14749, 14750, 14751, 14752, 14753, 14754, 14755, 14756, + 14757, 14758, 14759, 14760, 14761, 14762, 14763, 14764, 14765, 14766, + 14767, 14768, 14769, 14770, 14771, 14772, 14773, 14774, 14775, 14776, + 14777, 14778, 14779, 14780, 14781, 14782, 14783, 14784, 14785, 14786, + 14787, 14788, 14789, 14790, 14791, 14792, 14793, 14794, 14795, 14796, + 14797, 14798, 14799, 14800, 14801, 14802, 14803, 14804, 14805, 14806, + 14807, 14808, 14809, 14810, 14811, 14812, 14813, 14814, 14815, 14816, + 14817, 14818, 14819, 14820, 14821, 14822, 14823, 14824, 14825, 14826, + 14827, 14828, 14829, 14830, 14831, 14832, 14833, 14834, 14835, 14836, + 14837, 14838, 14839, 14840, 14841, 14842, 14843, 14844, 14845, 14846, + 14847, 14848, 14849, 14850, 14851, 14852, 14853, 14854, 14855, 14856, + 14857, 14858, 14859, 14860, 14861, 14862, 14863, 14864, 14865, 14866, + 14867, 14868, 14869, 14870, 14871, 14872, 14873, 14874, 14875, 14876, + 14877, 14878, 14879, 14880, 14881, 14882, 14883, 14884, 14885, 14886, + 14887, 14888, 14889, 14890, 14891, 14892, 14893, 14894, 14895, 14896, + 14897, 14898, 14899, 14900, 14901, 14902, 14903, 14904, 14905, 14906, + 14907, 14908, 14909, 14910, 14911, 14912, 14913, 14914, 14915, 14916, + 14917, 14918, 14919, 14920, 14921, 14922, 14923, 14924, 14925, 14926, + 14927, 14928, 14929, 14930, 14931, 14932, 14933, 14934, 14935, 14936, + 14937, 14938, 14939, 14940, 14941, 14942, 14943, 14944, 14945, 14946, + 14947, 14948, 14949, 14950, 14951, 14952, 14953, 14954, 14955, 14956, + 14957, 14958, 14959, 14960, 14961, 14962, 14963, 14964, 14965, 14966, + 14967, 14968, 14969, 14970, 14971, 14972, 14973, 14974, 14975, 14976, + 14977, 14978, 14979, 14980, 14981, 14982, 14983, 14984, 14985, 14986, + 14987, 14988, 14989, 14990, 14991, 14992, 14993, 14994, 14995, 14996, + 14997, 14998, 14999, 15000, 15001, 15002, 15003, 15004, 15005, 15006, + 15007, 15008, 15009, 15010, 15011, 15012, 15013, 15014, 15015, 15016, + 15017, 15018, 15019, 15020, 15021, 15022, 15023, 15024, 15025, 15026, + 15027, 15028, 15029, 15030, 15031, 15032, 15033, 15034, 15035, 15036, + 15037, 15038, 15039, 15040, 15041, 15042, 15043, 15044, 15045, 15046, + 15047, 15048, 15049, 15050, 15051, 15052, 15053, 15054, 15055, 15056, + 15057, 15058, 15059, 15060, 15061, 15062, 15063, 15064, 15065, 15066, + 15067, 15068, 15069, 15070, 15071, 15072, 15073, 15074, 15075, 15076, + 15077, 15078, 15079, 15080, 15081, 15082, 15083, 15084, 15085, 15086, + 15087, 15088, 15089, 15090, 15091, 15092, 15093, 15094, 15095, 15096, + 15097, 15098, 15099, 15100, 15101, 15102, 15103, 15104, 15105, 15106, + 15107, 15108, 15109, 15110, 15111, 15112, 15113, 15114, 15115, 15116, + 15117, 15118, 15119, 15120, 15121, 15122, 15123, 15124, 15125, 15126, + 15127, 15128, 15129, 15130, 15131, 15132, 15133, 15134, 15135, 15136, + 15137, 15138, 15139, 15140, 15141, 15142, 15143, 15144, 15145, 15146, + 15147, 15148, 15149, 15150, 15151, 15152, 15153, 15154, 15155, 15156, + 15157, 15158, 15159, 15160, 15161, 15162, 15163, 15164, 15165, 15166, + 15167, 15168, 15169, 15170, 15171, 15172, 15173, 15174, 15175, 15176, + 15177, 15178, 15179, 15180, 15181, 15182, 15183, 15184, 15185, 15186, + 15187, 15188, 15189, 15190, 15191, 15192, 15193, 15194, 15195, 15196, + 15197, 15198, 15199, 15200, 15201, 15202, 15203, 15204, 15205, 15206, + 15207, 15208, 15209, 15210, 15211, 15212, 15213, 15214, 15215, 15216, + 15217, 15218, 15219, 15220, 15221, 15222, 15223, 15224, 15225, 15226, + 15227, 15228, 15229, 15230, 15231, 15232, 15233, 15234, 15235, 15236, + 15237, 15238, 15239, 15240, 15241, 15242, 15243, 15244, 15245, 15246, + 15247, 15248, 15249, 15250, 15251, 15252, 15253, 15254, 15255, 15256, + 15257, 15258, 15259, 15260, 15261, 15262, 15263, 15264, 15265, 15266, + 15267, 15268, 15269, 15270, 15271, 15272, 15273, 15274, 15275, 15276, + 15277, 15278, 15279, 15280, 15281, 15282, 15283, 15284, 15285, 15286, + 15287, 15288, 15289, 15290, 15291, 15292, 15293, 15294, 15295, 15296, + 15297, 15298, 15299, 15300, 15301, 15302, 15303, 15304, 15305, 15306, + 15307, 15308, 15309, 15310, 15311, 15312, 15313, 15314, 15315, 15316, + 15317, 15318, 15319, 15320, 15321, 15322, 15323, 15324, 15325, 15326, + 15327, 15328, 15329, 15330, 15331, 15332, 15333, 15334, 15335, 15336, + 15337, 15338, 15339, 15340, 15341, 15342, 15343, 15344, 15345, 15346, + 15347, 15348, 15349, 15350, 15351, 15352, 15353, 15354, 15355, 15356, + 15357, 15358, 15359, 15360, 15361, 15362, 15363, 15364, 15365, 15366, + 15367, 15368, 15369, 15370, 15371, 15372, 15373, 15374, 15375, 15376, + 15377, 15378, 15379, 15380, 15381, 15382, 15383, 15384, 15385, 15386, + 15387, 15388, 15389, 15390, 15391, 15392, 15393, 15394, 15395, 15396, + 15397, 15398, 15399, 15400, 15401, 15402, 15403, 15404, 15405, 15406, + 15407, 15408, 15409, 15410, 15411, 15412, 15413, 15414, 15415, 15416, + 15417, 15418, 15419, 15420, 15421, 15422, 15423, 15424, 15425, 15426, + 15427, 15428, 15429, 15430, 15431, 15432, 15433, 15434, 15435, 15436, + 15437, 15438, 15439, 15440, 15441, 15442, 15443, 15444, 15445, 15446, + 15447, 15448, 15449, 15450, 15451, 15452, 15453, 15454, 15455, 15456, + 15457, 15458, 15459, 15460, 15461, 15462, 15463, 15464, 15465, 15466, + 15467, 15468, 15469, 15470, 15471, 15472, 15473, 15474, 15475, 15476, + 15477, 15478, 15479, 15480, 15481, 15482, 15483, 15484, 15485, 15486, + 15487, 15488, 15489, 15490, 15491, 15492, 15493, 15494, 15495, 15496, + 15497, 15498, 15499, 15500, 15501, 15502, 15503, 15504, 15505, 15506, + 15507, 15508, 15509, 15510, 15511, 15512, 15513, 15514, 15515, 15516, + 15517, 15518, 15519, 15520, 15521, 15522, 15523, 15524, 15525, 15526, + 15527, 15528, 15529, 15530, 15531, 15532, 15533, 15534, 15535, 15536, + 15537, 15538, 15539, 15540, 15541, 15542, 15543, 15544, 15545, 15546, + 15547, 15548, 15549, 15550, 15551, 15552, 15553, 15554, 15555, 15556, + 15557, 15558, 15559, 15560, 15561, 15562, 15563, 15564, 15565, 15566, + 15567, 15568, 15569, 15570, 15571, 15572, 15573, 15574, 15575, 15576, + 15577, 15578, 15579, 15580, 15581, 15582, 15583, 15584, 15585, 15586, + 15587, 15588, 15589, 15590, 15591, 15592, 15593, 15594, 15595, 15596, + 15597, 15598, 15599, 15600, 15601, 15602, 15603, 15604, 15605, 15606, + 15607, 15608, 15609, 15610, 15611, 15612, 15613, 15614, 15615, 15616, + 15617, 15618, 15619, 15620, 15621, 15622, 15623, 15624, 15625, 15626, + 15627, 15628, 15629, 15630, 15631, 15632, 15633, 15634, 15635, 15636, + 15637, 15638, 15639, 15640, 15641, 15642, 15643, 15644, 15645, 15646, + 15647, 15648, 15649, 15650, 15651, 15652, 15653, 15654, 15655, 15656, + 15657, 15658, 15659, 15660, 15661, 15662, 15663, 15664, 15665, 15666, + 15667, 15668, 15669, 15670, 15671, 15672, 15673, 15674, 15675, 15676, + 15677, 15678, 15679, 15680, 15681, 15682, 15683, 15684, 15685, 15686, + 15687, 15688, 15689, 15690, 15691, 15692, 15693, 15694, 15695, 15696, + 15697, 15698, 15699, 15700, 15701, 15702, 15703, 15704, 15705, 15706, + 15707, 15708, 15709, 15710, 15711, 15712, 15713, 15714, 15715, 15716, + 15717, 15718, 15719, 15720, 15721, 15722, 15723, 15724, 15725, 15726, + 15727, 15728, 15729, 15730, 15731, 15732, 15733, 15734, 15735, 15736, + 15737, 15738, 15739, 15740, 15741, 15742, 15743, 15744, 15745, 15746, + 15747, 15748, 15749, 15750, 15751, 15752, 15753, 15754, 15755, 15756, + 15757, 15758, 15759, 15760, 15761, 16013, 16014, 16015, 16016, 16017, + 16018, 16019, 16020, 16021, 16022, 16023, 16024, 16025, 16026, 16027, + 16028, 16029, 16030, 16031, 16032, 16033, 16034, 16035, 16036, 16037, + 16038, 16039, 16040, 16041, 16042, 16043, 16044, 16045, 16046, 16047, + 16048, 16049, 16050, 16051, 16052, 16053, 16054, 16055, 16056, 16057, + 16058, 16059, 16060, 16061, 16062, 16063, 16064, 16065, 16066, 16067, + 16068, 16069, 16070, 16071, 16072, 16073, 16074, 16075, 16076, 16077, + 16078, 16079, 16080, 16081, 16082, 16083, 16084, 16085, 16086, 16087, + 16088, 16089, 16090, 16091, 16092, 16093, 16094, 16095, 16096, 16097, + 16098, 16099, 16100, 16101, 16102, 16103, 16104, 16105, 16106, 16107, + 16108, 16109, 16110, 16111, 16112, 16113, 16114, 16115, 16116, 16117, + 16118, 16119, 16120, 16121, 16122, 16123, 16124, 16125, 16126, 16127, + 16128, 16129, 16130, 16131, 16132, 16133, 16134, 16135, 16136, 16137, + 16138, 16139, 16140, 16141, 16142, 16143, 16144, 16145, 16146, 16147, + 16148, 16149, 16150, 16151, 16152, 16153, 16154, 16155, 16156, 16157, + 16158, 16159, 16160, 16161, 16162, 16163, 16164, 16165, 16166, 16167, + 16168, 16169, 16170, 16171, 16172, 16173, 16174, 16175, 16176, 16177, + 16178, 16179, 16180, 16181, 16182, 16183, 16184, 16185, 16186, 16187, + 16188, 16189, 16190, 16191, 16192, 16193, 16194, 16195, 16196, 16197, + 16198, 16199, 16200, 16201, 16202, 16203, 16204, 16205, 16206, 16207, + 16208, 16209, 16210, 16211, 16212, 16213, 16214, 16215, 16216, 16217, + 16218, 16219, 16220, 16221, 16222, 16223, 16224, 16225, 16226, 16227, + 16228, 16229, 16230, 16231, 16232, 16233, 16234, 16235, 16236, 16237, + 16238, 16239, 16240, 16241, 16242, 16243, 16244, 16245, 16246, 16247, + 16248, 16249, 16250, 16251, 16252, 16253, 16254, 16255, 16256, 16257, + 16258, 16259, 16260, 16261, 16262, 16263, 16264, 16265, 16266, 16267, + 16268, 16269, 16270, 16271, 16272, 16273, 16274, 16275, 16276, 16277, + 16278, 16279, 16280, 16281, 16282, 16283, 16284, 16285, 16286, 16287, + 16288, 16289, 16290, 16291, 16292, 16293, 16294, 16295, 16296, 16297, + 16298, 16299, 16300, 16301, 16302, 16303, 16304, 16305, 16306, 16307, + 16308, 16309, 16310, 16311, 16312, 16313, 16314, 16315, 16316, 16317, + 16318, 16319, 16320, 16321, 16322, 16323, 16324, 16325, 16326, 16327, + 16328, 16329, 16330, 16331, 16332, 16333, 16334, 16335, 16336, 16337, + 16338, 16339, 16340, 16341, 16342, 16343, 16344, 16345, 16346, 16347, + 16348, 16349, 16350, 16351, 16352, 16353, 16354, 16355, 16356, 16357, + 16358, 16359, 16360, 16361, 16362, 16363, 16364, 16365, 16366, 16367, + 16368, 16369, 16370, 16371, 16372, 16373, 16374, 16375, 16376, 16377, + 16378, 16379, 16380, 16381, 16382, 16383, 16384, 16385, 16386, 16387, + 16388, 16389, 16390, 16391, 16392, 16393, 16394, 16395, 16396, 16397, + 16398, 16399, 16400, 16401, 16402, 16403, 16404, 16405, 16406, 16407, + 16408, 16409, 16410, 16411, 16412, 16413, 16414, 16415, 16416, 16417, + 16418, 16419, 16420, 16421, 16422, 16423, 16424, 16425, 16426, 16427, + 16428, 16429, 16430, 16431, 16432, 16433, 16434, 16435, 16436, 16437, + 16438, 16439, 16440, 16441, 16442, 16443, 16444, 16445, 16446, 16447, + 16448, 16449, 16450, 16451, 16452, 16453, 16454, 16455, 16456, 16457, + 16458, 16459, 16460, 16461, 16462, 16463, 16464, 16465, 16466, 16467, + 16468, 16469, 16470, 16471, 16472, 16473, 16474, 16475, 16476, 16477, + 16478, 16479, 16480, 16481, 16482, 16483, 16484, 16485, 16486, 16487, + 16488, 16489, 16490, 16491, 16492, 16493, 16494, 16495, 16496, 16497, + 16498, 16499, 16500, 16501, 16502, 16503, 16504, 16505, 16506, 16507, + 16508, 16509, 16510, 16511, 16512, 16513, 16514, 16515, 16516, 16517, + 16518, 16519, 16520, 16521, 16522, 16523, 16524, 16525, 16526, 16527, + 16528, 16529, 16530, 16531, 16532, 16533, 16534, 16535, 16536, 16537, + 16538, 16539, 16540, 16541, 16542, 16543, 16544, 16545, 16546, 16547, + 16548, 16549, 16550, 16551, 16552, 16553, 16554, 16555, 16556, 16557, + 16558, 16559, 16560, 16561, 16562, 16563, 16564, 16565, 16566, 16567, + 16568, 16569, 16570, 16571, 16572, 16573, 16574, 16575, 16576, 16577, + 16578, 16579, 16580, 16581, 16582, 16583, 16584, 16585, 16586, 16587, + 16588, 16589, 16590, 16591, 16592, 16593, 16594, 16595, 16596, 16597, + 16598, 16599, 16600, 16601, 16602, 16603, 16604, 16605, 16606, 16607, + 16608, 16609, 16610, 16611, 16612, 16613, 16614, 16615, 16616, 16617, + 16618, 16619, 16620, 16621, 16622, 16623, 16624, 16625, 16626, 16627, + 16628, 16629, 16630, 16631, 16632, 16633, 16634, 16635, 16636, 16637, + 16638, 16639, 16640, 16641, 16642, 16643, 16644, 16645, 16646, 16647, + 16648, 16649, 16650, 16651, 16652, 16653, 16654, 16655, 16656, 16657, + 16658, 16659, 16660, 16661, 16662, 16663, 16664, 16665, 16666, 16667, + 16668, 16669, 16670, 16671, 16672, 16673, 16674, 16675, 16676, 16677, + 16678, 16679, 16680, 16681, 16682, 16683, 16684, 16685, 16686, 16687, + 16688, 16689, 16690, 16691, 16692, 16693, 16694, 16695, 16696, 16697, + 16698, 16699, 16700, 16701, 16702, 16703, 16704, 16705, 16706, 16707, + 16708, 16709, 16710, 16711, 16712, 16713, 16714, 16715, 16716, 16717, + 16718, 16719, 16720, 16721, 16722, 16723, 16724, 16725, 16726, 16727, + 16728, 16729, 16730, 16731, 16732, 16733, 16734, 16735, 16736, 16737, + 16738, 16739, 16740, 16741, 16742, 16743, 16744, 16745, 16746, 16747, + 16748, 16749, 16750, 16751, 16752, 16753, 16754, 16755, 16756, 16757, + 16758, 16759, 16760, 16761, 16762, 16763, 16764, 16765, 16766, 16767, + 16768, 16769, 16770, 16771, 16772, 16773, 16774, 16775, 16776, 16777, + 16778, 16779, 16780, 15773, 15774, 15775, 15776, 15777, 15778, 15779, + 15780, 15781, 15782, 15783, 15784, 15785, 15786, 15787, 15788, 15789, + 15790, 15791, 15792, 15793, 15794, 15795, 15796, 15797, 15798, 15799, + 15800, 15801, 15802, 15803, 15804, 15805, 15806, 15807, 15808, 15809, + 15810, 15811, 15812, 15813, 15814, 15815, 15816, 15817, 15818, 15819, + 15820, 15821, 15822, 15823, 15824, 15825, 15826, 15827, 15828, 15829, + 15830, 15831, 15832, 15833, 15834, 15835, 15836, 15837, 15838, 15839, + 15840, 15841, 15842, 15843, 15844, 15845, 15846, 15847, 15848, 15849, + 15850, 15851, 15852, 15853, 15854, 15855, 15856, 15857, 15858, 15859, + 15860, 15861, 15862, 15863, 15864, 15865, 15866, 15867, 15868, 15869, + 15870, 15871, 15872, 15873, 15874, 15875, 15876, 15877, 15878, 15879, + 15880, 15881, 15882, 15883, 15884, 15885, 15886, 15887, 15888, 15889, + 15890, 15891, 15892, 15893, 15894, 15895, 15896, 15897, 15898, 15899, + 15900, 15901, 15902, 15903, 15904, 15905, 15906, 15907, 15908, 15909, + 15910, 15911, 15912, 15913, 15914, 15915, 15916, 15917, 15918, 15919, + 15920, 15921, 15922, 15923, 15924, 15925, 15926, 15927, 15928, 15929, + 15930, 15931, 15932, 15933, 15934, 15935, 15936, 15937, 15938, 15939, + 15940, 15941, 15942, 15943, 15944, 15945, 15946, 15947, 15948, 15949, + 15950, 15951, 15952, 15953, 15954, 15955, 15956, 15957, 15958, 15959, + 15960, 15961, 15962, 15963, 15964, 15965, 15966, 15967, 15968, 15969, + 15970, 15971, 15972, 15973, 15974, 15975, 15976, 15977, 15978, 15979, + 15980, 15981, 15982, 15983, 15984, 15985, 15986, 15987, 15988, 15989, + 15990, 15991, 15992, 15993, 15994, 15995, 15996, 15997, 15998, 15999, + 16000, 16001, 16002, 16003, 16004, 16005, 16006, 16007, 16008, 16009, + 16010, 16011, 16012, 15762, 15763, 15764, 15765, 15766, 15767, 15768, + 15769, 15770, 15771, 15772, 41412, 41412, 41412, 41412, 41412, 447, 448, + 449, 450, 451, 452, 453, 454, 455, 436, 437, 438, 439, 440, 441, 442, + 443, 444, 445, 446, 377, 378, 379, 380, 381, 382, 375, 376, 383, 384, + 385, 427, 428, 429, 430, 431, 432, 433, 434, 435, 425, 426, 393, 389, + 390, 394, 395, 396, 391, 392, 386, 387, 388, 397, 398, 399, 456, 457, + 458, 459, 460, 461, 462, 463, 464, 465, 404, 405, 406, 407, 408, 409, + 400, 401, 402, 403, 410, 411, 412, 466, 467, 468, 469, 470, 471, 472, + 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 417, + 418, 419, 420, 421, 422, 423, 413, 414, 415, 416, 424, 497, 498, 499, + 500, 501, 502, 503, 486, 487, 488, 489, 494, 495, 496, 504, 490, 491, + 492, 493, 505, 506, 507, 508, 509, 512, 513, 514, 515, 510, 511, 516, + 517, 518, 519, 522, 523, 524, 525, 526, 520, 521, 527, 528, 529, 530, + 533, 534, 535, 536, 537, 531, 532, 538, 539, 540, 541, 542, 543, 544, + 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, + 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, + 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, + 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, + 601, 609, 610, 602, 603, 604, 611, 612, 613, 614, 605, 606, 615, 607, + 608, 620, 621, 622, 623, 624, 616, 617, 618, 619, 625, 626, 627, 653, + 654, 655, 656, 657, 658, 659, 651, 652, 660, 661, 673, 674, 675, 676, + 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, + 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 664, 665, + 666, 667, 668, 669, 670, 662, 663, 671, 672, 703, 704, 705, 706, 707, + 708, 709, 710, 711, 712, 642, 643, 644, 645, 646, 647, 648, 649, 650, + 640, 641, 632, 633, 634, 635, 628, 629, 636, 637, 638, 639, 630, 631, + 715, 716, 717, 718, 719, 720, 721, 722, 723, 713, 714, 807, 808, 809, + 810, 811, 812, 813, 814, 815, 816, 726, 727, 728, 729, 730, 731, 732, + 733, 734, 724, 725, 753, 754, 750, 751, 752, 755, 756, 757, 746, 747, + 748, 749, 758, 759, 760, 817, 818, 819, 820, 821, 822, 823, 824, 825, + 826, 737, 738, 739, 740, 741, 742, 743, 744, 745, 735, 736, 765, 766, + 767, 768, 761, 762, 769, 770, 771, 763, 764, 772, 798, 796, 797, 799, + 800, 801, 802, 803, 804, 805, 806, 779, 775, 776, 780, 773, 774, 781, + 782, 777, 778, 783, 784, 785, 787, 788, 789, 786, 790, 791, 792, 793, + 794, 795, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 827, 828, + 829, 830, 831, 832, 833, 834, 835, 836, 837, 868, 869, 870, 871, 872, + 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, + 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 838, 839, 842, + 843, 844, 845, 846, 847, 840, 841, 848, 849, 898, 899, 900, 901, 902, + 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, + 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 850, 851, 852, + 853, 854, 855, 856, 857, 929, 930, 931, 932, 933, 934, 935, 936, 937, + 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, + 952, 953, 954, 955, 956, 957, 928, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 19107, 19097, 19096, 19093, 19092, 19078, 19091, 19090, 19095, 19094, + 19100, 19085, 19084, 19081, 19080, 19105, 19087, 19086, 19083, 19082, + 19079, 19099, 19098, 19089, 19088, 19102, 19106, 19103, 19101, 19104, + 19110, 19117, 19118, 19113, 19114, 19119, 19120, 19111, 19115, 19116, + 19112, 19121, 19077, 19076, 19074, 19108, 19075, 19109, 19128, 19130, + 19127, 19126, 19123, 19122, 19125, 19124, 19131, 19129, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 2892, 2858, 2917, 2918, 2888, + 2925, 2936, 2907, 2924, 2919, 2920, 2871, 2937, 2894, 2873, 2878, 2885, + 2931, 2903, 2861, 2897, 2932, 2893, 2868, 2869, 2901, 2875, 2916, 2857, + 2915, 2884, 2908, 2939, 2856, 2902, 2933, 2887, 2886, 2882, 2859, 2914, + 2891, 2921, 2874, 2911, 2922, 2938, 2866, 2928, 2935, 2876, 2862, 2890, + 2864, 2883, 2926, 2867, 2941, 2943, 2863, 2929, 2879, 2923, 2900, 2927, + 2905, 2912, 2896, 2940, 2895, 2877, 2909, 2880, 2906, 2934, 2930, 2913, + 2898, 2870, 2904, 2860, 2899, 2942, 2865, 2910, 2889, 2881, 2976, 2993, + 2991, 2990, 2954, 2960, 2949, 2997, 2959, 2951, 2984, 2996, 2953, 2980, + 2981, 2944, 2986, 2994, 2988, 2989, 2966, 2963, 2979, 2947, 2945, 2946, + 2952, 2985, 3001, 2974, 2971, 2998, 2987, 2995, 2965, 2969, 2970, 2967, + 2992, 2962, 2968, 2978, 2983, 2964, 2999, 2961, 3000, 2948, 2958, 2957, + 2955, 2972, 2977, 2956, 2950, 2975, 3053, 3073, 3095, 3091, 3052, 3041, + 3054, 3002, 3030, 3004, 3074, 3070, 3027, 3075, 3045, 3024, 3007, 3003, + 3009, 3094, 3072, 3035, 3065, 3033, 3096, 3014, 3015, 3079, 3046, 3084, + 3061, 3088, 3083, 3048, 3090, 3039, 3021, 3071, 3049, 3017, 3032, 3037, + 3068, 3085, 3051, 3099, 3042, 3067, 3064, 3098, 3089, 3029, 3100, 3057, + 3016, 3087, 3062, 3059, 3011, 3047, 3023, 3034, 3019, 3013, 3058, 3056, + 3092, 3050, 3066, 3069, 3012, 3063, 3043, 3020, 3097, 3044, 3081, 3078, + 3031, 3022, 3026, 3008, 3028, 3010, 3025, 3093, 3060, 3038, 3036, 3082, + 3006, 3005, 3055, 3040, 3018, 3076, 3077, 3086, 3128, 3212, 3161, 3135, + 3162, 3121, 3160, 3166, 3148, 3175, 3211, 3157, 3191, 3158, 3105, 3204, + 3189, 3164, 3196, 3109, 3213, 3111, 3198, 3133, 3143, 3124, 3130, 3200, + 3217, 3159, 3106, 3154, 3206, 3104, 3156, 3101, 3144, 3138, 3116, 3145, + 3140, 3139, 3181, 3137, 3134, 3122, 3167, 3126, 3114, 3173, 3202, 3201, + 3214, 3107, 3188, 3205, 3153, 3132, 3168, 3108, 3184, 3179, 3174, 3119, + 3147, 3136, 3151, 3215, 3183, 3216, 3146, 3170, 3195, 3112, 3150, 3155, + 3207, 3129, 3113, 3169, 3203, 3125, 3149, 3117, 3152, 3165, 3163, 3103, + 3110, 3185, 3209, 3208, 3176, 3187, 3118, 3131, 3123, 3197, 3142, 3193, + 3190, 3115, 3178, 3194, 3171, 3180, 3177, 3192, 3182, 3141, 3120, 3186, + 3210, 3172, 3127, 3199, 3102, 3271, 3349, 3258, 3245, 3356, 3248, 3312, + 3338, 3328, 3298, 3274, 3323, 3343, 3284, 3238, 3359, 3331, 3276, 3313, + 3348, 3240, 3250, 3300, 3290, 3255, 3287, 3291, 3299, 3330, 3297, 3316, + 3339, 3279, 3262, 3232, 3286, 3294, 3256, 3249, 3277, 3273, 3340, 3332, + 3326, 3270, 3278, 3364, 3231, 3353, 3360, 3320, 3352, 3235, 3337, 3346, + 3354, 3357, 3244, 3322, 3342, 3229, 3292, 3333, 3261, 3259, 3305, 3309, + 3228, 3351, 3239, 3372, 3303, 3365, 3260, 3272, 3318, 3368, 3247, 3221, + 3227, 3289, 3237, 3254, 3285, 3233, 3223, 3301, 3314, 3362, 3275, 3304, + 3306, 3321, 3264, 3222, 3308, 3265, 3230, 3220, 3268, 3324, 3288, 3241, + 3319, 3302, 3361, 3280, 3310, 3219, 3226, 3295, 3373, 3350, 3375, 3374, + 3246, 3311, 3341, 3344, 3269, 3336, 3363, 3282, 3369, 3366, 3367, 3371, + 3370, 3236, 3315, 3296, 3325, 3358, 3225, 3355, 3252, 3263, 3329, 3327, + 3283, 3293, 3334, 3335, 3218, 3307, 3317, 3251, 3243, 3266, 3253, 3257, + 3345, 3242, 3267, 3347, 3224, 3234, 3380, 3429, 3382, 3427, 3407, 3420, + 3401, 3384, 3410, 3409, 3389, 3419, 3399, 3395, 3386, 3417, 3412, 3418, + 3415, 3378, 3377, 3397, 3396, 3394, 3422, 3414, 3423, 3398, 3403, 3400, + 3424, 3404, 3411, 3402, 3406, 3376, 3392, 3393, 3413, 3405, 3428, 3425, + 3385, 3383, 3381, 3387, 3408, 3390, 3391, 3388, 3421, 3379, 3416, 3426, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 28399, 28391, 28412, + 28389, 28413, 28400, 28415, 28395, 28386, 28403, 28401, 28409, 28385, + 28393, 28388, 28390, 28396, 28394, 28392, 28405, 28406, 28397, 28411, + 28414, 28410, 28387, 28408, 28407, 28402, 28404, 28398, 41412, 28424, + 28426, 28423, 28422, 28419, 28418, 28421, 28420, 28427, 28425, 41412, + 41412, 41412, 41412, 28417, 28416, 36122, 36119, 36120, 36121, 36074, + 36071, 36072, 36073, 36126, 36123, 36124, 36125, 36114, 36111, 36112, + 36113, 36118, 36115, 36116, 36117, 36110, 36107, 36108, 36109, 36070, + 36067, 36068, 36069, 36102, 36099, 36100, 36101, 36075, 36080, 36091, + 36092, 36103, 36106, 36104, 36105, 36098, 36095, 36096, 36097, 36086, + 36083, 36084, 36085, 36137, 36136, 36135, 36087, 36094, 36144, 36142, + 36139, 36089, 36138, 36140, 36082, 36090, 36079, 36081, 36078, 36129, + 36133, 36141, 36088, 36093, 36131, 36128, 36134, 36077, 36127, 36143, + 36076, 36132, 36130, 36145, 41412, 36152, 36154, 36151, 36150, 36147, + 36146, 36149, 36148, 36155, 36153, 41412, 41412, 41412, 41412, 41412, + 41412, 3490, 3495, 3511, 3513, 3503, 3501, 3493, 3487, 3494, 3507, 3502, + 3498, 3509, 3492, 3488, 3510, 3497, 3508, 3512, 3506, 3500, 3514, 3499, + 3515, 3504, 3505, 3496, 3491, 3489, 3516, 41412, 41412, 3483, 3485, 3486, + 3484, 3482, 3517, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 31177, 31178, 31183, 31184, 31171, 31172, 31187, 31188, + 31179, 31180, 31169, 31170, 31189, 31190, 31173, 31174, 31185, 31186, + 31191, 31192, 31181, 31182, 31175, 31176, 31193, 31194, 31167, 31168, + 31113, 31104, 31110, 31101, 31106, 31112, 31105, 31109, 31115, 31099, + 31111, 31097, 31102, 31100, 31108, 31103, 31107, 31116, 31114, 31098, + 31121, 31120, 31118, 31117, 31119, 31123, 31122, 31153, 31154, 31132, + 31152, 31150, 31158, 31160, 31159, 31161, 31151, 31143, 31156, 31141, + 31163, 31136, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 31201, 31203, 31200, 31199, 31196, 31195, 31198, 31197, + 31204, 31202, 41412, 31128, 31125, 31127, 31130, 31124, 31126, 31129, + 41412, 31155, 31162, 31140, 31148, 31164, 31139, 31146, 31157, 31145, + 31166, 31147, 31142, 31149, 31165, 31144, 31138, 31131, 31134, 31135, + 31137, 31133, 41412, 41412, 41412, 41412, 41412, 31085, 31092, 31084, + 31083, 31093, 31081, 31078, 31096, 31088, 31086, 31094, 31080, 31079, + 31089, 31095, 31091, 31087, 31082, 31090, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 22688, 22685, 22690, 22684, 22673, 22672, 22669, 22668, + 22661, 22667, 22666, 22671, 22670, 22662, 22658, 22657, 22654, 22653, + 22660, 22659, 22656, 22655, 22663, 22675, 22674, 22665, 22664, 22680, + 22683, 22681, 22679, 22682, 22677, 22676, 22678, 22691, 22697, 22694, + 22695, 22696, 22692, 22698, 22693, 22689, 22687, 22686, 22700, 22699, + 22707, 22709, 22706, 22705, 22702, 22701, 22704, 22703, 22710, 22708, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 27062, 27066, + 27069, 27070, 27040, 27072, 27048, 27063, 27067, 27058, 27057, 27059, + 27047, 27039, 27060, 27056, 27053, 27054, 27068, 27050, 27061, 27064, + 27046, 27044, 27071, 27055, 27052, 27042, 27065, 27051, 27041, 27049, + 27120, 27124, 27127, 27128, 27098, 27130, 27106, 27121, 27125, 27116, + 27115, 27117, 27105, 27097, 27118, 27114, 27111, 27112, 27126, 27108, + 27119, 27122, 27104, 27102, 27129, 27113, 27110, 27100, 27123, 27109, + 27099, 27107, 27084, 27078, 27076, 27074, 27081, 27080, 27083, 27082, + 27086, 27085, 27089, 27091, 27088, 27087, 27092, 27093, 27095, 27096, + 27090, 27094, 27079, 27077, 27075, 27073, 27133, 27131, 27132, 41412, + 41412, 41412, 41412, 41412, 3702, 3719, 3704, 3708, 3721, 3709, 3716, + 3726, 3705, 3717, 3722, 3715, 3711, 3710, 3712, 3723, 3724, 3706, 3707, + 3714, 3713, 3718, 3725, 3720, 3703, 41412, 41412, 3727, 3744, 3729, 3733, + 3746, 3734, 3741, 3751, 3730, 3742, 3747, 3740, 3736, 3735, 3737, 3748, + 3749, 3731, 3732, 3739, 3738, 3743, 3750, 3745, 3728, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 27681, 27607, 27669, 27680, 27685, 27684, 27604, 27686, + 27661, 27660, 27658, 27615, 27664, 27665, 27657, 27614, 27623, 27631, + 27667, 27603, 27628, 27627, 27622, 27621, 27620, 27619, 27645, 27613, + 27644, 27612, 27687, 27618, 27668, 27679, 27678, 27626, 27625, 27602, + 27683, 27689, 27617, 27616, 27654, 27610, 27630, 27629, 27653, 27609, + 27662, 27666, 27638, 27641, 27642, 27676, 27674, 27655, 27611, 27663, + 27643, 27677, 27675, 27673, 27671, 27601, 27672, 27670, 27688, 27605, + 27682, 27606, 27640, 27608, 27659, 27656, 27639, 41412, 41412, 41412, + 41412, 27693, 27624, 27692, 27691, 27690, 27698, 27704, 27703, 27699, + 27700, 27725, 27729, 27746, 27745, 27707, 27710, 27711, 27727, 27714, + 27715, 27716, 27717, 27718, 27721, 27723, 27724, 27720, 27731, 27732, + 27733, 27734, 27739, 27735, 27736, 27740, 27742, 27701, 27702, 27709, + 27744, 27708, 27743, 27705, 27713, 27706, 27730, 27747, 27748, 27737, + 27741, 27728, 27726, 27749, 27722, 27712, 27719, 27738, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 27697, 27695, 27696, 27694, 27646, + 27647, 27648, 27649, 27650, 27651, 27652, 27632, 27633, 27634, 27635, + 27636, 27637, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 37039, 30032, 30253, 30252, + 22331, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 39333, 39332, 6583, 6584, 39854, 39855, 39856, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 36156, 36157, + 36158, 36159, 36160, 36161, 36162, 36163, 36164, 36165, 36166, 36167, + 36168, 36169, 36170, 36171, 36172, 36173, 36174, 36175, 36176, 36177, + 36178, 36179, 36180, 36181, 36182, 36183, 36184, 36185, 36186, 36187, + 36188, 36189, 36190, 36191, 36192, 36193, 36194, 36195, 36196, 36197, + 36198, 36199, 36200, 36201, 36202, 36203, 36204, 36205, 36206, 36207, + 36208, 36209, 36210, 36211, 36212, 36213, 36214, 36215, 36216, 36217, + 36218, 36219, 36220, 36221, 36222, 36223, 36224, 36225, 36226, 36227, + 36228, 36229, 36230, 36231, 36232, 36233, 36234, 36235, 36236, 36237, + 36238, 36239, 36240, 36241, 36242, 36243, 36244, 36245, 36246, 36247, + 36248, 36249, 36250, 36251, 36252, 36253, 36254, 36255, 36256, 36257, + 36258, 36259, 36260, 36261, 36262, 36263, 36264, 36265, 36266, 36267, + 36268, 36269, 36270, 36271, 36272, 36273, 36274, 36275, 36276, 36277, + 36278, 36279, 36280, 36281, 36282, 36283, 36284, 36285, 36286, 36287, + 36288, 36289, 36290, 36291, 36292, 36293, 36294, 36295, 36296, 36297, + 36298, 36299, 36300, 36301, 36302, 36303, 36304, 36305, 36306, 36307, + 36308, 36309, 36310, 36311, 36312, 36313, 36314, 36315, 36316, 36317, + 36318, 36319, 36320, 36321, 36322, 36323, 36324, 36325, 36326, 36327, + 36328, 36329, 36330, 36331, 36332, 36333, 36334, 36335, 36336, 36337, + 36338, 36339, 36340, 36341, 36342, 36343, 36344, 36345, 36346, 36347, + 36348, 36349, 36350, 36351, 36352, 36353, 36354, 36355, 36356, 36357, + 36358, 36359, 36360, 36361, 36362, 36363, 36364, 36365, 36366, 36367, + 36368, 36369, 36370, 36371, 36372, 36373, 36374, 36375, 36376, 36377, + 36378, 36379, 36380, 36381, 36382, 36383, 36384, 36385, 36386, 36387, + 36388, 36389, 36390, 36391, 36392, 36393, 36394, 36395, 36396, 36397, + 36398, 36399, 36400, 36401, 36402, 36403, 36404, 36405, 36406, 36407, + 36408, 36409, 36410, 36411, 36412, 36413, 36414, 36415, 36416, 36417, + 36418, 36419, 36420, 36421, 36422, 36423, 36424, 36425, 36426, 36427, + 36428, 36429, 36430, 36431, 36432, 36433, 36434, 36435, 36436, 36437, + 36438, 36439, 36440, 36441, 36442, 36443, 36444, 36445, 36446, 36447, + 36448, 36449, 36450, 36451, 36452, 36453, 36454, 36455, 36456, 36457, + 36458, 36459, 36460, 36461, 36462, 36463, 36464, 36465, 36466, 36467, + 36468, 36469, 36470, 36471, 36472, 36473, 36474, 36475, 36476, 36477, + 36478, 36479, 36480, 36481, 36482, 36483, 36484, 36485, 36486, 36487, + 36488, 36489, 36490, 36491, 36492, 36493, 36494, 36495, 36496, 36497, + 36498, 36499, 36500, 36501, 36502, 36503, 36504, 36505, 36506, 36507, + 36508, 36509, 36510, 36511, 36512, 36513, 36514, 36515, 36516, 36517, + 36518, 36519, 36520, 36521, 36522, 36523, 36524, 36525, 36526, 36527, + 36528, 36529, 36530, 36531, 36532, 36533, 36534, 36535, 36536, 36537, + 36538, 36539, 36540, 36541, 36542, 36543, 36544, 36545, 36546, 36547, + 36548, 36549, 36550, 36551, 36552, 36553, 36554, 36555, 36556, 36557, + 36558, 36559, 36560, 36561, 36562, 36563, 36564, 36565, 36566, 36567, + 36568, 36569, 36570, 36571, 36572, 36573, 36574, 36575, 36576, 36577, + 36578, 36579, 36580, 36581, 36582, 36583, 36584, 36585, 36586, 36587, + 36588, 36589, 36590, 36591, 36592, 36593, 36594, 36595, 36596, 36597, + 36598, 36599, 36600, 36601, 36602, 36603, 36604, 36605, 36606, 36607, + 36608, 36609, 36610, 36611, 36612, 36613, 36614, 36615, 36616, 36617, + 36618, 36619, 36620, 36621, 36622, 36623, 36624, 36625, 36626, 36627, + 36628, 36629, 36630, 36631, 36632, 36633, 36634, 36635, 36636, 36637, + 36638, 36639, 36640, 36641, 36642, 36643, 36644, 36645, 36646, 36647, + 36648, 36649, 36650, 36651, 36652, 36653, 36654, 36655, 36656, 36657, + 36658, 36659, 36660, 36661, 36662, 36663, 36664, 36665, 36666, 36667, + 36668, 36669, 36670, 36671, 36672, 36673, 36674, 36675, 36676, 36677, + 36678, 36679, 36680, 36681, 36682, 36683, 36684, 36685, 36686, 36687, + 36688, 36689, 36690, 36691, 36692, 36693, 36694, 36695, 36696, 36697, + 36698, 36699, 36700, 36701, 36702, 36703, 36704, 36705, 36706, 36707, + 36708, 36709, 36710, 36711, 36712, 36713, 36714, 36715, 36716, 36717, + 36718, 36719, 36720, 36721, 36722, 36723, 36724, 36725, 36726, 36727, + 36728, 36729, 36730, 36731, 36732, 36733, 36734, 36735, 36736, 36737, + 36738, 36739, 36740, 36741, 36742, 36743, 36744, 36745, 36746, 36747, + 36748, 36749, 36750, 36751, 36752, 36753, 36754, 36755, 36756, 36757, + 36758, 36759, 36760, 36761, 36762, 36763, 36764, 36765, 36766, 36767, + 36768, 36769, 36770, 36771, 36772, 36773, 36774, 36775, 36776, 36777, + 36778, 36779, 36780, 36781, 36782, 36783, 36784, 36785, 36786, 36787, + 36788, 36789, 36790, 36791, 36792, 36793, 36794, 36795, 36796, 36797, + 36798, 36799, 36800, 36801, 36802, 36803, 36804, 36805, 36806, 36807, + 36808, 36809, 36810, 36811, 36812, 36813, 36814, 36815, 36816, 36817, + 36818, 36819, 36820, 36821, 36822, 36823, 36824, 36825, 36826, 36827, + 36828, 36829, 36830, 36831, 36832, 36833, 36834, 36835, 36836, 36837, + 36838, 36839, 36840, 36841, 36842, 36843, 36844, 36845, 36846, 36847, + 36848, 36849, 36850, 36851, 36852, 36853, 36854, 36855, 36856, 36857, + 36858, 36859, 36860, 36861, 36862, 36863, 36864, 36865, 36866, 36867, + 36868, 36869, 36870, 36871, 36872, 36873, 36874, 36875, 36876, 36877, + 36878, 36879, 36880, 36881, 36882, 36883, 36884, 36885, 36886, 36887, + 36888, 36889, 36890, 36891, 36892, 36893, 36894, 36895, 36896, 36897, + 36898, 36899, 36900, 36901, 36902, 36903, 36904, 36905, 36906, 36907, + 36908, 36909, 36910, 36911, 36912, 36913, 36914, 36915, 36916, 36917, + 36918, 36919, 36920, 36921, 36922, 36923, 21860, 21861, 21862, 21863, + 21864, 21865, 21866, 21867, 21868, 21869, 21870, 21871, 21872, 21873, + 21874, 21875, 21876, 21877, 21878, 21879, 21880, 21881, 21882, 21883, + 21884, 21885, 21886, 21887, 21888, 21889, 21890, 21891, 21892, 21893, + 21894, 21895, 21896, 21897, 21898, 21899, 21900, 21901, 21902, 21903, + 21904, 21905, 21906, 21907, 21908, 21909, 21910, 21911, 21912, 21913, + 21914, 21915, 21916, 21917, 21918, 21919, 21920, 21921, 21922, 21923, + 21924, 21925, 21926, 21927, 21928, 21929, 21930, 21931, 21932, 21933, + 21934, 21935, 21936, 21937, 21938, 21939, 21940, 21941, 21942, 21943, + 21944, 21945, 21946, 21947, 21948, 21949, 21950, 21951, 21952, 21953, + 21954, 21955, 21956, 21957, 21958, 21959, 21960, 21961, 21962, 21963, + 21964, 21965, 21966, 21967, 21968, 21969, 21970, 21971, 21972, 21973, + 21974, 21975, 21976, 21977, 21978, 21979, 21980, 21981, 21982, 21983, + 21984, 21985, 21986, 21987, 21988, 21989, 21990, 21991, 21992, 21993, + 21994, 21995, 21996, 21997, 21998, 21999, 22000, 22001, 22002, 22003, + 22004, 22005, 22006, 22007, 22008, 22009, 22010, 22011, 22012, 22013, + 22014, 22015, 22016, 22017, 22018, 22019, 22020, 22021, 22022, 22023, + 22024, 22025, 22026, 22027, 22028, 22029, 22030, 22031, 22032, 22033, + 22034, 22035, 22036, 22037, 22038, 22039, 22040, 22041, 22042, 22043, + 22044, 22045, 22046, 22047, 22048, 22049, 22050, 22051, 22052, 22053, + 22054, 22055, 22056, 22057, 22058, 22059, 22060, 22061, 22062, 22063, + 22064, 22065, 22066, 22067, 22068, 22069, 22070, 22071, 22072, 22073, + 22074, 22075, 22076, 22077, 22078, 22079, 22080, 22081, 22082, 22083, + 22084, 22085, 22086, 22087, 22088, 22089, 22090, 22091, 22092, 22093, + 22094, 22095, 22096, 22097, 22098, 22099, 22100, 22101, 22102, 22103, + 22104, 22105, 22106, 22107, 22108, 22109, 22110, 22111, 22112, 22113, + 22114, 22115, 22122, 22123, 22124, 22125, 22126, 22127, 22128, 22129, + 22130, 22131, 22132, 22133, 22134, 22135, 22136, 22137, 22138, 22139, + 22140, 22141, 22142, 22143, 22144, 22145, 22146, 22147, 22148, 22149, + 22150, 22151, 22152, 22153, 22154, 22155, 22156, 22157, 22158, 22159, + 22160, 22161, 22162, 22163, 22164, 22165, 22166, 22167, 22168, 22169, + 22170, 22171, 22172, 22173, 22174, 22175, 22176, 22177, 22178, 22179, + 22180, 22181, 22182, 22183, 22184, 22185, 22186, 22187, 22188, 22189, + 22190, 22191, 22192, 22193, 22194, 22195, 22196, 22197, 22198, 22199, + 22200, 22201, 22202, 22203, 22204, 22205, 22206, 22207, 22208, 22209, + 22210, 22211, 22212, 22213, 22214, 22215, 22216, 22217, 22218, 22219, + 22220, 22221, 22222, 22223, 22224, 22225, 22226, 22227, 22228, 22229, + 22230, 22231, 22232, 22233, 22234, 22235, 22236, 22237, 22238, 22239, + 22240, 22241, 22242, 22243, 22244, 22245, 22246, 22247, 22248, 22249, + 22250, 22251, 22252, 22253, 22254, 22255, 22256, 22257, 22258, 22259, + 22260, 22261, 22262, 22263, 22264, 22265, 22266, 22267, 22268, 22269, + 22270, 22271, 22272, 22273, 22274, 22275, 22276, 22277, 22278, 22279, + 22280, 22281, 22282, 22283, 22284, 22285, 22286, 22287, 22288, 22289, + 22290, 22291, 22292, 22293, 22294, 22295, 22296, 22297, 22298, 22299, + 22300, 22301, 22302, 22303, 22304, 22305, 22306, 22307, 22308, 22309, + 22310, 22311, 22312, 22313, 22314, 22315, 22316, 22317, 22318, 22319, + 22320, 22321, 22322, 22323, 22324, 22325, 22326, 22327, 22328, 22329, + 22116, 22117, 22118, 22119, 22120, 22121, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 22330, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 36924, 36925, 36926, 36927, + 36928, 36929, 36930, 36931, 36932, 36933, 36934, 36935, 36936, 36937, + 36938, 36939, 36940, 36941, 36942, 36943, 36944, 36945, 36946, 36947, + 36948, 36949, 36950, 36951, 36952, 36953, 36954, 36955, 36956, 36957, + 36958, 36959, 36960, 36961, 36962, 36963, 36964, 36965, 36966, 36967, + 36968, 36969, 36970, 36971, 36972, 36973, 36974, 36975, 36976, 36977, + 36978, 36979, 36980, 36981, 36982, 36983, 36984, 36985, 36986, 36987, + 36988, 36989, 36990, 36991, 36992, 36993, 36994, 36995, 36996, 36997, + 36998, 36999, 37000, 37001, 37002, 37003, 37004, 37005, 37006, 37007, + 37008, 37009, 37010, 37011, 37012, 37013, 37014, 37015, 37016, 37017, + 37018, 37019, 37020, 37021, 37022, 37023, 37024, 37025, 37026, 37027, + 37028, 37029, 37030, 37031, 37032, 37033, 37034, 37035, 37036, 37037, + 37038, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 21499, 21500, 21501, 21502, 41412, 21503, + 21504, 21492, 21493, 21494, 21495, 21496, 41412, 21497, 21498, 41412, + 21480, 20452, 20091, 20092, 20093, 20090, 20372, 20373, 20374, 20375, + 20364, 20365, 20366, 20367, 20368, 20359, 20360, 20361, 20362, 20363, + 20369, 20370, 20371, 20130, 20134, 20135, 20136, 20137, 20138, 20139, + 20140, 20141, 20131, 20132, 20133, 20146, 20147, 20148, 20149, 20150, + 20151, 20152, 20153, 20160, 20161, 20162, 20163, 20164, 20165, 20166, + 20154, 20155, 20156, 20157, 20158, 20159, 20143, 20144, 20145, 20142, + 20255, 20256, 20257, 20258, 20259, 20260, 20261, 20262, 20271, 20272, + 20273, 20274, 20275, 20276, 20263, 20264, 20265, 20266, 20267, 20268, + 20269, 20270, 20277, 20278, 20279, 20280, 20281, 20282, 20283, 20284, + 20285, 20286, 20287, 20288, 20317, 20318, 20319, 20320, 20310, 20311, + 20312, 20313, 20314, 20315, 20316, 20306, 20307, 20308, 20309, 20305, + 20289, 20290, 20291, 20292, 20293, 20294, 20295, 20296, 20297, 20299, + 20300, 20301, 20302, 20303, 20304, 20298, 20209, 20210, 20211, 20212, + 20213, 20214, 20215, 20216, 20217, 20202, 20203, 20204, 20205, 20206, + 20207, 20208, 20201, 20223, 20224, 20225, 20195, 20196, 20197, 20198, + 20199, 20200, 20194, 20218, 20219, 20220, 20221, 20222, 20102, 20105, + 20106, 20107, 20108, 20109, 20110, 20111, 20112, 20103, 20104, 20123, + 20124, 20125, 20126, 20127, 20128, 20129, 20113, 20114, 20115, 20116, + 20117, 20118, 20119, 20120, 20121, 20122, 20094, 20095, 20096, 20097, + 20098, 20099, 20100, 20101, 20176, 20177, 20178, 20179, 20180, 20181, + 20182, 20183, 20184, 20185, 20186, 20187, 20188, 20189, 20190, 20191, + 20192, 20193, 20168, 20169, 20167, 20170, 20171, 20172, 20173, 20174, + 20175, 20343, 20344, 20345, 20346, 20347, 20342, 20354, 20355, 20356, + 20357, 20348, 20349, 20350, 20351, 20352, 20353, 20247, 20248, 20249, + 20250, 20240, 20241, 20242, 20243, 20244, 20245, 20246, 20234, 20235, + 20236, 20237, 20238, 20239, 20251, 20252, 20253, 20254, 20228, 20229, + 20230, 20231, 20232, 20233, 20321, 20322, 20323, 20324, 20325, 20326, + 20327, 20328, 20329, 20330, 20338, 20339, 20340, 20341, 20331, 20332, + 20333, 20334, 20335, 20336, 20337, 20226, 20227, 20451, 21478, 21477, + 21479, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 20462, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 20455, 20454, 20456, 41412, + 41412, 21527, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 21533, 21532, 21534, 21538, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 29636, 29637, + 29638, 29639, 29640, 29641, 29642, 29643, 29644, 29645, 29646, 29647, + 29648, 29649, 29650, 29651, 29652, 29653, 29654, 29655, 29656, 29657, + 29658, 29659, 29660, 29661, 29662, 29663, 29664, 29665, 29666, 29667, + 29668, 29669, 29670, 29671, 29672, 29673, 29674, 29675, 29676, 29677, + 29678, 29679, 29680, 29681, 29682, 29683, 29684, 29685, 29686, 29687, + 29688, 29689, 29690, 29691, 29692, 29693, 29694, 29695, 29696, 29697, + 29698, 29699, 29700, 29701, 29702, 29703, 29704, 29705, 29706, 29707, + 29708, 29709, 29710, 29711, 29712, 29713, 29714, 29715, 29716, 29717, + 29718, 29719, 29720, 29721, 29722, 29723, 29724, 29725, 29726, 29727, + 29728, 29729, 29730, 29731, 29732, 29733, 29734, 29735, 29736, 29737, + 29738, 29739, 29740, 29741, 29742, 29743, 29744, 29745, 29746, 29747, + 29748, 29749, 29750, 29751, 29752, 29753, 29754, 29755, 29756, 29757, + 29758, 29759, 29760, 29761, 29762, 29763, 29764, 29765, 29766, 29767, + 29768, 29769, 29770, 29771, 29772, 29773, 29774, 29775, 29776, 29777, + 29778, 29779, 29780, 29781, 29782, 29783, 29784, 29785, 29786, 29787, + 29788, 29789, 29790, 29791, 29792, 29793, 29794, 29795, 29796, 29797, + 29798, 29799, 29800, 29801, 29802, 29803, 29804, 29805, 29806, 29807, + 29808, 29809, 29810, 29811, 29812, 29813, 29814, 29815, 29816, 29817, + 29818, 29819, 29820, 29821, 29822, 29823, 29824, 29825, 29826, 29827, + 29828, 29829, 29830, 29831, 29832, 29833, 29834, 29835, 29836, 29837, + 29838, 29839, 29840, 29841, 29842, 29843, 29844, 29845, 29846, 29847, + 29848, 29849, 29850, 29851, 29852, 29853, 29854, 29855, 29856, 29857, + 29858, 29859, 29860, 29861, 29862, 29863, 29864, 29865, 29866, 29867, + 29868, 29869, 29870, 29871, 29872, 29873, 29874, 29875, 29876, 29877, + 29878, 29879, 29880, 29881, 29882, 29883, 29884, 29885, 29886, 29887, + 29888, 29889, 29890, 29891, 29892, 29893, 29894, 29895, 29896, 29897, + 29898, 29899, 29900, 29901, 29902, 29903, 29904, 29905, 29906, 29907, + 29908, 29909, 29910, 29911, 29912, 29913, 29914, 29915, 29916, 29917, + 29918, 29919, 29920, 29921, 29922, 29923, 29924, 29925, 29926, 29927, + 29928, 29929, 29930, 29931, 29932, 29933, 29934, 29935, 29936, 29937, + 29938, 29939, 29940, 29941, 29942, 29943, 29944, 29945, 29946, 29947, + 29948, 29949, 29950, 29951, 29952, 29953, 29954, 29955, 29956, 29957, + 29958, 29959, 29960, 29961, 29962, 29963, 29964, 29965, 29966, 29967, + 29968, 29969, 29970, 29971, 29972, 29973, 29974, 29975, 29976, 29977, + 29978, 29979, 29980, 29981, 29982, 29983, 29984, 29985, 29986, 29987, + 29988, 29989, 29990, 29991, 29992, 29993, 29994, 29995, 29996, 29997, + 29998, 29999, 30000, 30001, 30002, 30003, 30004, 30005, 30006, 30007, + 30008, 30009, 30010, 30011, 30012, 30013, 30014, 30015, 30016, 30017, + 30018, 30019, 30020, 30021, 30022, 30023, 30024, 30025, 30026, 30027, + 30028, 30029, 30030, 30031, 41412, 41412, 41412, 41412, 11649, 11647, + 11596, 11629, 11556, 11569, 11573, 11654, 11550, 11637, 11558, 11600, + 11599, 11551, 11557, 11571, 11603, 11632, 11625, 11552, 11572, 11626, + 11650, 11576, 11604, 11577, 11582, 11560, 11605, 11578, 11583, 11563, + 11606, 11580, 11585, 11561, 11562, 11614, 11615, 11581, 11586, 11567, + 11618, 11579, 11584, 11564, 11607, 11568, 11565, 11566, 11612, 11613, + 11610, 11611, 11631, 11630, 11639, 11645, 11642, 11617, 11616, 11570, + 11559, 11608, 11609, 11548, 11623, 11593, 11591, 11549, 11651, 11553, + 11652, 11628, 11636, 11554, 11620, 11601, 11619, 11574, 11653, 11633, + 11555, 11648, 11634, 11575, 11602, 11635, 11627, 11592, 11595, 11594, + 11644, 11640, 11646, 11643, 11641, 11590, 11589, 11588, 11587, 11598, + 11597, 11621, 11624, 11622, 11638, 41412, 41412, 41412, 41412, 41412, + 11533, 11546, 11545, 11540, 11547, 11527, 11520, 11519, 11516, 11518, + 11521, 11522, 11517, 41412, 41412, 41412, 11529, 11525, 11528, 11523, + 11532, 11531, 11524, 11530, 11526, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 11534, 11538, 11541, 11536, 11544, 11543, 11537, 11542, + 11539, 11535, 41412, 41412, 11658, 11655, 11656, 11657, 33036, 33035, + 33037, 33038, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 38247, 32227, 24339, 32224, 11423, 25347, + 32195, 25405, 967, 20587, 39302, 24297, 27756, 32180, 24332, 32217, + 30107, 31773, 31965, 20583, 39276, 25287, 25286, 25285, 25284, 25282, + 25283, 4587, 4588, 4596, 4610, 4480, 4479, 32766, 32774, 32767, 32778, + 32771, 32775, 32768, 32780, 32773, 32777, 32770, 32779, 32772, 32776, + 32769, 38286, 38254, 38256, 38341, 38303, 38299, 38335, 38305, 25431, + 25378, 25398, 25433, 25381, 25423, 25425, 25404, 34309, 34310, 30835, + 10972, 10708, 10707, 34321, 34322, 24345, 32204, 17621, 17622, 352, 351, + 358, 357, 360, 359, 355, 356, 353, 354, 24335, 38239, 32220, 11429, + 37913, 37909, 37917, 4442, 4440, 4446, 24328, 38234, 32214, 11425, 28480, + 24338, 38242, 32223, 11432, 16787, 16788, 3944, 3947, 3946, 3945, 3978, + 24351, 38232, 32210, 11435, 24352, 38233, 32211, 11436, 24341, 38236, + 32226, 11427, 34554, 34555, 34553, 34552, 34746, 34748, 34747, 34745, + 39286, 20566, 39699, 39700, 38159, 34386, 34385, 34384, 34339, 20962, + 24211, 20963, 39291, 20564, 24346, 32205, 24347, 32206, 17606, 24337, + 38241, 32222, 11431, 20586, 39301, 39279, 24340, 32225, 24334, 32219, + 24336, 32221, 24245, 32138, 38295, 38331, 38294, 38330, 25367, 25387, + 25366, 25386, 25364, 25384, 25365, 25385, 38298, 38334, 25377, 25397, + 38296, 38332, 25376, 25396, 38289, 38325, 25371, 25391, 38292, 38328, + 25374, 25394, 38293, 38329, 25375, 25395, 38288, 38324, 25370, 25390, + 38290, 38326, 25372, 25392, 38291, 38327, 25373, 25393, 38297, 38333, + 25368, 25388, 31013, 31014, 31015, 31016, 31017, 31018, 31019, 31020, + 31021, 31022, 31023, 31024, 31025, 31026, 31027, 31028, 31029, 31030, + 31031, 31032, 31033, 31034, 31035, 31036, 31037, 31038, 31049, 31051, + 31048, 31047, 31044, 31043, 31046, 31045, 31052, 31050, 34090, 17623, + 29621, 41412, 41412, 41412, 4239, 4183, 4054, 4269, 4140, 4081, 4240, + 4121, 4184, 4230, 4156, 4215, 4097, 4112, 4200, 4066, 4273, 4141, 4171, + 4082, 4241, 4122, 4185, 4055, 4236, 4164, 4223, 4105, 4262, 4118, 4208, + 4074, 4149, 4177, 4090, 4248, 4130, 4193, 4060, 4231, 4157, 4216, 4098, + 4255, 4113, 4201, 4067, 4274, 4142, 4172, 4083, 4242, 4123, 4186, 4168, + 4227, 4109, 4266, 4137, 4212, 4078, 4281, 4153, 4180, 4094, 4252, 4134, + 4197, 4063, 4161, 4220, 4102, 4259, 4205, 4071, 4278, 4146, 4087, 4245, + 4127, 4190, 4237, 4165, 4224, 4106, 4263, 4119, 4209, 4075, 4270, 4150, + 4178, 4091, 4249, 4131, 4194, 4061, 4232, 4158, 4217, 4099, 4256, 4114, + 4202, 4068, 4275, 4143, 4173, 4084, 4243, 4124, 4187, 4056, 4170, 4229, + 4111, 4268, 4139, 4214, 4080, 4283, 4155, 4182, 4096, 4254, 4136, 4199, + 4065, 4235, 4163, 4222, 4104, 4261, 4117, 4207, 4073, 4280, 4148, 4176, + 4089, 4247, 4129, 4192, 4059, 4167, 4226, 4108, 4265, 4211, 4077, 4272, + 4152, 4093, 4251, 4133, 4196, 4233, 4160, 4219, 4101, 4258, 4115, 4204, + 4070, 4277, 4145, 4174, 4086, 4244, 4126, 4189, 4057, 4169, 4228, 4110, + 4267, 4138, 4213, 4079, 4282, 4154, 4181, 4095, 4253, 4135, 4198, 4064, + 4234, 4162, 4221, 4103, 4260, 4116, 4206, 4072, 4279, 4147, 4175, 4088, + 4246, 4128, 4191, 4058, 4238, 4166, 4225, 4107, 4264, 4120, 4210, 4076, + 4271, 4151, 4179, 4092, 4250, 4132, 4195, 4062, 4159, 4218, 4100, 4257, + 4203, 4069, 4276, 4144, 4085, 4125, 4188, 37921, 4449, 37918, 4447, + 37919, 4448, 37914, 4443, 37915, 4444, 37908, 4436, 4437, 4438, 4439, + 28364, 37910, 37911, 11426, 24329, 34060, 38237, 11428, 17492, 17493, + 17494, 32133, 25338, 17495, 38261, 25340, 19942, 39761, 37932, 17779, + 4481, 4478, 24249, 32142, 24248, 32141, 20563, 24246, 32139, 20562, + 25341, 38264, 39292, 4606, 4604, 4605, 4603, 22883, 22882, 22887, 22881, + 22859, 22838, 22839, 22879, 22845, 22863, 22886, 22861, 22884, 22885, + 22867, 22864, 22844, 22843, 22846, 22862, 22847, 22835, 22856, 22880, + 22836, 22837, 22834, 22860, 22865, 22851, 22842, 22866, 22868, 22841, + 22858, 22850, 22848, 22849, 22840, 22888, 22857, 22852, 22855, 22853, + 22854, 22876, 22874, 22875, 22878, 22873, 22870, 22877, 22872, 22871, + 22869, 32781, 32813, 32782, 32829, 32798, 32814, 32783, 32837, 32806, + 32822, 32791, 32830, 32799, 32815, 32784, 32841, 32810, 32826, 32795, + 32834, 32803, 32819, 32788, 32838, 32807, 32823, 32792, 32831, 32800, + 32816, 32785, 32843, 32812, 32828, 32797, 32836, 32805, 32821, 32790, + 32840, 32809, 32825, 32794, 32833, 32802, 32818, 32787, 32842, 32811, + 32827, 32796, 32835, 32804, 32820, 32789, 32839, 32808, 32824, 32793, + 32832, 32801, 32817, 32786, 38281, 38253, 38255, 38320, 38302, 38300, + 38301, 38304, 25430, 25428, 25429, 25432, 25383, 25422, 25424, 25418, + 32143, 32183, 24300, 24250, 25343, 25438, 38344, 38266, 24251, 24301, + 32184, 32144, 38267, 38345, 25439, 25344, 20588, 21790, 30522, 3984, + 41412, 41412, 41412, 41412, 41412, 41412, 17659, 31062, 37993, 1057, + 6577, 34742, 20085, 20984, 17611, 27599, 31398, 39328, 16781, 20983, + 17487, 31902, 37387, 27244, 17637, 2574, 3597, 369, 24565, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 17878, 17875, 17867, 17873, 17879, 17865, 17871, + 17868, 17874, 17870, 17866, 17876, 17872, 17877, 17869, 17880, 27156, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41228, 41233, 41267, 41230, 41235, + 41263, 41257, 41251, 41274, 41253, 41247, 41271, 41229, 41234, 41268, + 41231, 41236, 41264, 41258, 41252, 41275, 41254, 41248, 41272, 41269, + 41255, 41262, 41249, 41250, 41273, 41256, 41232, 41278, 41243, 41260, + 41270, 41281, 41280, 41246, 41279, 41240, 41239, 41277, 41261, 41259, + 41238, 41412, 41412, 41283, 41284, 41285, 41276, 41226, 41241, 41244, + 41245, 41223, 41224, 41242, 41265, 41266, 41227, 41282, 41225, 41237, + 41222, 41404, 41405, 41402, 41403, 41406, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41302, 41303, 41319, 41291, 41300, + 41399, 41354, 41355, 41322, 41323, 41356, 41286, 41320, 41400, 41298, + 41295, 41297, 41296, 41294, 41396, 41395, 41398, 41397, 41392, 41391, + 41394, 41393, 41292, 41326, 41288, 41299, 41287, 41324, 41337, 41338, + 41336, 41335, 41289, 41333, 41334, 41332, 41330, 41331, 41329, 41328, + 41339, 41341, 41342, 41340, 41304, 41327, 41293, 41301, 41401, 41343, + 41348, 41345, 41349, 41346, 41351, 41352, 41347, 41344, 41350, 41325, + 41353, 41386, 41383, 41374, 41384, 41385, 41375, 41363, 41362, 41390, + 41360, 41359, 41357, 41361, 41358, 41377, 41382, 41376, 41380, 41381, + 41378, 41379, 41306, 41310, 41309, 41308, 41307, 41389, 41387, 41388, + 41313, 41318, 41317, 41316, 41315, 41314, 41364, 41373, 41366, 41371, + 41365, 41369, 41370, 41367, 41368, 41372, 41305, 41312, 41290, 41311, + 41321, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 5296, 5168, 5289, 5271, 5272, 5340, 5341, 5221, 5318, 5278, 5352, + 5353, 5244, 5123, 5174, 5321, 5222, 5126, 5127, 5316, 5328, 5267, 5204, + 5295, 5147, 5344, 5216, 5230, 5226, 5319, 5281, 5310, 5274, 5343, 5129, + 5131, 5231, 5297, 5282, 5339, 5121, 5299, 5313, 5315, 5269, 5323, 5251, + 5169, 5331, 5322, 5241, 5122, 5181, 5212, 5333, 5219, 5287, 5291, 5234, + 5145, 5298, 5276, 5279, 5218, 5356, 5286, 5235, 5334, 5311, 5210, 5217, + 5268, 5273, 5285, 5237, 5284, 5242, 5288, 5225, 5229, 5355, 5128, 5124, + 5354, 5243, 5177, 5148, 5266, 5342, 5283, 5292, 5275, 5120, 5252, 5280, + 5277, 5173, 5245, 5119, 5335, 5176, 5314, 5317, 5146, 5175, 5300, 5357, + 5337, 5293, 5332, 5336, 5290, 5338, 5294, 5207, 5133, 5172, 5270, 5325, + 5326, 5324, 5327, 5220, 5170, 5345, 5346, 5309, 5233, 5166, 5238, 5239, + 5240, 5130, 5132, 5165, 5330, 5320, 5236, 5246, 5250, 5249, 5248, 5247, + 5206, 5203, 5202, 5160, 5162, 5163, 5161, 5329, 5134, 5214, 5153, 5117, + 5111, 5112, 5115, 5116, 5114, 5113, 5118, 5259, 5254, 5255, 5253, 5264, + 5263, 5262, 5261, 5265, 5257, 5215, 5125, 5180, 5179, 5178, 5260, 5258, + 5256, 5208, 5209, 5171, 5213, 5211, 5182, 5189, 5185, 5193, 5188, 5197, + 5187, 5186, 5183, 5184, 5191, 5192, 5199, 5196, 5194, 5143, 5144, 5142, + 5190, 5198, 5351, 5156, 5154, 5157, 5159, 5158, 5155, 5347, 5349, 5348, + 5350, 5200, 5201, 5150, 5149, 5151, 5152, 5305, 5308, 5306, 5307, 5301, + 5304, 5302, 5303, 5164, 5167, 5312, 5141, 5135, 5140, 5138, 5137, 5136, + 5139, 5223, 5227, 5224, 5228, 5232, 5205, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 28669, 28546, 28562, 28654, + 28541, 28666, 28604, 28655, 28653, 28542, 28538, 28665, 28532, 28650, + 28651, 28652, 28557, 28558, 28495, 28496, 28491, 28490, 28621, 28692, + 28689, 28564, 28563, 28670, 28671, 28565, 28574, 28575, 28576, 28535, + 28567, 28568, 28569, 28548, 28549, 41412, 41412, 28612, 28545, 28547, + 28573, 28572, 28617, 28616, 28668, 28667, 28644, 28645, 28531, 28537, + 28633, 28634, 28648, 28649, 28613, 28715, 28587, 28647, 28556, 28673, + 28691, 28675, 28620, 28713, 28636, 28536, 28676, 28677, 28700, 28701, + 28704, 28705, 28702, 28703, 28696, 28697, 28698, 28699, 28610, 28611, + 28706, 28707, 28635, 28711, 28618, 28615, 28499, 28500, 28494, 28714, + 28586, 28646, 28555, 28672, 28690, 28674, 28619, 28520, 28521, 28524, + 28525, 28526, 28559, 28560, 28561, 28503, 28510, 28511, 28512, 28513, + 28514, 28488, 28553, 28489, 28554, 28487, 28551, 28486, 28550, 28501, + 28519, 28527, 28518, 28504, 28505, 28502, 28529, 28584, 28583, 28508, + 28530, 28515, 28522, 28528, 28507, 28523, 28656, 28679, 28718, 28643, + 28606, 28566, 28534, 28543, 28580, 28579, 28695, 28708, 28717, 28709, + 28710, 28622, 28625, 28626, 28627, 28628, 28629, 28630, 28631, 28632, + 28623, 28624, 28588, 28614, 28552, 28544, 28506, 28509, 28516, 28517, + 28638, 28637, 28585, 28578, 28577, 28716, 28539, 28540, 28605, 28601, + 28492, 28659, 28661, 28607, 28609, 28662, 28664, 28570, 28571, 28603, + 28602, 28493, 28660, 28608, 28663, 28687, 28686, 28688, 28685, 28681, + 28682, 28683, 28684, 28533, 28581, 28582, 28678, 28712, 28640, 28498, + 28657, 28497, 28693, 28641, 28642, 28658, 28694, 28639, 28589, 28592, + 28596, 28599, 28598, 28597, 28593, 28594, 28590, 28591, 28595, 28680, + 28600, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 18793, 18781, 18804, 18805, 18787, 18806, 18807, 18808, + 18809, 18794, 18795, 18796, 18797, 18798, 18799, 18800, 18801, 18802, + 18803, 18782, 18783, 18784, 18785, 18786, 18788, 18789, 18790, 18791, + 18792, 18513, 18521, 18534, 18542, 18548, 18549, 18514, 18515, 18516, + 18517, 18518, 18519, 18520, 18522, 18523, 18524, 18525, 18526, 18527, + 18528, 18529, 18530, 18531, 18532, 18533, 18535, 18536, 18537, 18538, + 18539, 18540, 18541, 18543, 18544, 18545, 18546, 18547, 8387, 8386, 8388, + 18573, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 21770, 21771, 21768, 21766, 21757, 21756, 21763, + 21761, 21752, 21759, 21769, 21754, 21767, 21765, 21758, 21755, 21764, + 21762, 21753, 21760, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 27020, 27021, 27018, 27016, 27007, + 27006, 27013, 27011, 27002, 27009, 27019, 27004, 27017, 27015, 27008, + 27005, 27014, 27012, 27003, 27010, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 28357, 11017, 11018, + 11012, 11011, 11010, 37181, 37201, 37222, 37170, 37215, 37179, 37165, + 37224, 37169, 37183, 37189, 37212, 37213, 37227, 37233, 37180, 37211, + 37241, 37199, 37166, 37229, 37231, 37198, 37244, 37178, 37193, 37190, + 37171, 37185, 37168, 37226, 37219, 37173, 37216, 37205, 37239, 37228, + 37202, 37230, 37218, 37232, 37207, 37195, 37238, 37208, 37194, 37225, + 37234, 37204, 37240, 37177, 37221, 37197, 37243, 37187, 37172, 37209, + 37206, 37220, 37164, 37192, 37191, 37242, 37236, 37214, 37184, 37182, + 37188, 37196, 37235, 37237, 37210, 37176, 37174, 37203, 37167, 37175, + 37223, 37186, 37217, 37200, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 8801, 8799, 8798, 8795, 8794, 8797, 8796, 8802, + 8800, 8792, 8790, 8789, 8786, 8785, 8788, 8787, 8793, 8791, 20675, 20674, + 20673, 20672, 20671, 35650, 35649, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 26000, 26002, 26030, 25993, 26006, 26038, 26009, 26039, 26011, 26040, + 26013, 26015, 26032, 26034, 26017, 26020, 26041, 26024, 26026, 25996, + 26028, 26042, 26043, 26036, 26044, 26004, 26048, 26050, 26083, 26045, + 26054, 26057, 26059, 26091, 26061, 26092, 26063, 26065, 26085, 26087, + 26067, 26070, 26093, 26074, 26076, 26078, 26081, 26094, 26095, 26089, + 26096, 26052, 26488, 26490, 26520, 26494, 26496, 26528, 26499, 26529, + 26501, 26530, 26503, 26505, 26522, 26524, 26507, 26510, 26531, 26514, + 26516, 26484, 26518, 26532, 26533, 26526, 26534, 26492, 26436, 26438, + 26471, 26432, 26442, 26445, 26447, 41412, 26449, 26479, 26451, 26453, + 26473, 26475, 26455, 26458, 26480, 26462, 26464, 26466, 26469, 26481, + 26482, 26477, 26483, 26440, 26153, 26155, 26185, 26159, 26161, 26193, + 26164, 26194, 26166, 26195, 26168, 26170, 26187, 26189, 26172, 26175, + 26196, 26179, 26181, 26149, 26183, 26197, 26198, 26191, 26199, 26157, + 26207, 26209, 26244, 26213, 26215, 26218, 26220, 26252, 26222, 26253, + 26224, 26226, 26246, 26248, 26228, 26231, 26254, 26235, 26237, 26239, + 26242, 26255, 26256, 26250, 26257, 26211, 26960, 41412, 26961, 26962, + 41412, 41412, 26963, 41412, 41412, 26964, 26965, 41412, 41412, 26966, + 26967, 26968, 26969, 41412, 26970, 26971, 26972, 26973, 26974, 26975, + 26976, 26977, 26978, 26979, 26980, 26981, 41412, 26982, 41412, 26983, + 26984, 26985, 26986, 26987, 26988, 26989, 41412, 26990, 26991, 26992, + 26993, 26994, 26995, 26996, 26997, 26998, 26999, 27000, 26097, 26098, + 26099, 26100, 26101, 26102, 26103, 26104, 26105, 26106, 26107, 26108, + 26109, 26110, 26111, 26112, 26113, 26114, 26115, 26116, 26117, 26118, + 26119, 26120, 26121, 26122, 26123, 26124, 26125, 26126, 26127, 26128, + 26129, 26130, 26131, 26132, 26133, 26134, 26135, 26136, 26137, 26138, + 26139, 26140, 26141, 26142, 26143, 26144, 26145, 26146, 26147, 26148, + 26384, 26385, 41412, 26386, 26387, 26388, 26389, 41412, 41412, 26390, + 26391, 26392, 26393, 26394, 26395, 26396, 26397, 41412, 26398, 26399, + 26400, 26401, 26402, 26403, 26404, 41412, 26405, 26406, 26407, 26408, + 26409, 26410, 26411, 26412, 26413, 26414, 26415, 26416, 26417, 26418, + 26419, 26420, 26421, 26422, 26423, 26424, 26425, 26426, 26427, 26428, + 26429, 26430, 26329, 26330, 41412, 26331, 26332, 26333, 26334, 41412, + 26335, 26336, 26337, 26338, 26339, 41412, 26340, 41412, 41412, 41412, + 26341, 26342, 26343, 26344, 26345, 26346, 26347, 41412, 26348, 26349, + 26350, 26351, 26352, 26353, 26354, 26355, 26356, 26357, 26358, 26359, + 26360, 26361, 26362, 26363, 26364, 26365, 26366, 26367, 26368, 26369, + 26370, 26371, 26372, 26373, 26267, 26268, 26269, 26270, 26271, 26272, + 26273, 26274, 26275, 26276, 26277, 26278, 26279, 26280, 26281, 26282, + 26283, 26284, 26285, 26286, 26287, 26288, 26289, 26290, 26291, 26292, + 26293, 26294, 26295, 26296, 26297, 26298, 26299, 26300, 26301, 26302, + 26303, 26304, 26305, 26306, 26307, 26308, 26309, 26310, 26311, 26312, + 26313, 26314, 26315, 26316, 26317, 26318, 26898, 26899, 26900, 26901, + 26902, 26903, 26904, 26905, 26906, 26907, 26908, 26909, 26910, 26911, + 26912, 26913, 26914, 26915, 26916, 26917, 26918, 26919, 26920, 26921, + 26922, 26923, 26924, 26925, 26926, 26927, 26928, 26929, 26930, 26931, + 26932, 26933, 26934, 26935, 26936, 26937, 26938, 26939, 26940, 26941, + 26942, 26943, 26944, 26945, 26946, 26947, 26948, 26949, 26730, 26732, + 26762, 26736, 26738, 26770, 26741, 26771, 26743, 26772, 26745, 26747, + 26764, 26766, 26749, 26752, 26773, 26756, 26758, 26726, 26760, 26774, + 26775, 26768, 26776, 26734, 26784, 26786, 26821, 26790, 26792, 26795, + 26797, 26829, 26799, 26830, 26801, 26803, 26823, 26825, 26805, 26808, + 26831, 26812, 26814, 26816, 26819, 26832, 26833, 26827, 26834, 26788, + 26846, 26847, 26848, 26849, 26850, 26851, 26852, 26853, 26854, 26855, + 26856, 26857, 26858, 26859, 26860, 26861, 26862, 26863, 26864, 26865, + 26866, 26867, 26868, 26869, 26870, 26871, 26872, 26873, 26874, 26875, + 26876, 26877, 26878, 26879, 26880, 26881, 26882, 26883, 26884, 26885, + 26886, 26887, 26888, 26889, 26890, 26891, 26892, 26893, 26894, 26895, + 26896, 26897, 26620, 26622, 26652, 26626, 26628, 26660, 26631, 26661, + 26633, 26662, 26635, 26637, 26654, 26656, 26639, 26642, 26663, 26646, + 26648, 26616, 26650, 26664, 26665, 26658, 26666, 26624, 26674, 26676, + 26711, 26680, 26682, 26685, 26687, 26719, 26689, 26720, 26691, 26693, + 26713, 26715, 26695, 26698, 26721, 26702, 26704, 26706, 26709, 26722, + 26723, 26717, 26724, 26678, 26543, 26544, 26545, 26546, 26547, 26548, + 26549, 26550, 26551, 26552, 26553, 26554, 26555, 26556, 26557, 26558, + 26559, 26560, 26561, 26562, 26563, 26564, 26565, 26566, 26567, 26568, + 26569, 26570, 26571, 26572, 26573, 26574, 26575, 26576, 26577, 26578, + 26579, 26580, 26581, 26582, 26583, 26584, 26585, 26586, 26587, 26588, + 26589, 26590, 26591, 26592, 26593, 26594, 26433, 26434, 41412, 41412, + 26001, 26003, 26010, 25995, 26007, 26005, 26008, 25997, 26012, 26014, + 26016, 26033, 26035, 26037, 26018, 26023, 26025, 25998, 26027, 25999, + 26029, 26021, 26031, 26022, 26019, 26261, 26049, 26051, 26060, 26047, + 26055, 26053, 26056, 26079, 26062, 26064, 26066, 26086, 26088, 26090, + 26068, 26073, 26075, 26058, 26077, 26080, 26082, 26071, 26084, 26072, + 26069, 26263, 26259, 26266, 26260, 26262, 26265, 26264, 26489, 26491, + 26500, 26495, 26497, 26493, 26498, 26485, 26502, 26504, 26506, 26523, + 26525, 26527, 26508, 26513, 26515, 26486, 26517, 26487, 26519, 26511, + 26521, 26512, 26509, 26537, 26437, 26439, 26448, 26435, 26443, 26441, + 26444, 26467, 26450, 26452, 26454, 26474, 26476, 26478, 26456, 26461, + 26463, 26446, 26465, 26468, 26470, 26459, 26472, 26460, 26457, 26539, + 26535, 26542, 26536, 26538, 26541, 26540, 26154, 26156, 26165, 26160, + 26162, 26158, 26163, 26150, 26167, 26169, 26171, 26188, 26190, 26192, + 26173, 26178, 26180, 26151, 26182, 26152, 26184, 26176, 26186, 26177, + 26174, 26202, 26208, 26210, 26221, 26214, 26216, 26212, 26217, 26240, + 26223, 26225, 26227, 26247, 26249, 26251, 26229, 26234, 26236, 26219, + 26238, 26241, 26243, 26232, 26245, 26233, 26230, 26204, 26200, 26258, + 26201, 26203, 26206, 26205, 26731, 26733, 26742, 26737, 26739, 26735, + 26740, 26727, 26744, 26746, 26748, 26765, 26767, 26769, 26750, 26755, + 26757, 26728, 26759, 26729, 26761, 26753, 26763, 26754, 26751, 26779, + 26785, 26787, 26798, 26791, 26793, 26789, 26794, 26817, 26800, 26802, + 26804, 26824, 26826, 26828, 26806, 26811, 26813, 26796, 26815, 26818, + 26820, 26809, 26822, 26810, 26807, 26781, 26777, 26835, 26778, 26780, + 26783, 26782, 26621, 26623, 26632, 26627, 26629, 26625, 26630, 26617, + 26634, 26636, 26638, 26655, 26657, 26659, 26640, 26645, 26647, 26618, + 26649, 26619, 26651, 26643, 26653, 26644, 26641, 26669, 26675, 26677, + 26688, 26681, 26683, 26679, 26684, 26707, 26690, 26692, 26694, 26714, + 26716, 26718, 26696, 26701, 26703, 26686, 26705, 26708, 26710, 26699, + 26712, 26700, 26697, 26671, 26667, 26725, 26668, 26670, 26673, 26672, + 25994, 26046, 41412, 41412, 26325, 26327, 26324, 26323, 26320, 26319, + 26322, 26321, 26328, 26326, 26380, 26382, 26379, 26378, 26375, 26374, + 26377, 26376, 26383, 26381, 26956, 26958, 26955, 26954, 26951, 26950, + 26953, 26952, 26959, 26957, 26842, 26844, 26841, 26840, 26837, 26836, + 26839, 26838, 26845, 26843, 26601, 26603, 26600, 26599, 26596, 26595, + 26598, 26597, 26604, 26602, 33315, 33271, 33296, 33512, 33467, 33253, + 33316, 33280, 33429, 33381, 33357, 33318, 33321, 33278, 33322, 33272, + 33323, 33345, 33340, 33378, 33319, 33325, 33334, 33335, 33326, 33332, + 33336, 33274, 33408, 33317, 33346, 33275, 33355, 33324, 33354, 33341, + 33380, 33379, 33320, 33339, 33349, 33350, 33353, 33352, 33420, 33328, + 33329, 33330, 33402, 33370, 33333, 33337, 33331, 33327, 33401, 33362, + 33400, 33399, 33359, 33358, 33361, 33351, 33348, 33347, 33397, 33396, + 33398, 33369, 33445, 33449, 33448, 33446, 33447, 33290, 33436, 33466, + 33438, 33451, 33443, 33452, 33444, 33453, 33442, 33300, 33301, 33465, + 33506, 33439, 33440, 33441, 33437, 33463, 33450, 33461, 33454, 33462, + 33460, 33458, 33455, 33456, 33457, 33459, 33287, 33293, 33291, 33292, + 33496, 33495, 33303, 33295, 33306, 33309, 33304, 33307, 33305, 33308, + 33313, 33310, 33270, 33505, 33511, 33508, 33510, 33484, 33486, 33464, + 33494, 33487, 33491, 33485, 33493, 33492, 33490, 33252, 33342, 33277, + 33469, 33255, 33478, 33344, 33343, 33470, 33383, 33386, 33385, 33384, + 33393, 33433, 33282, 33507, 33264, 33389, 33392, 33387, 33388, 33481, + 33391, 33480, 33263, 33262, 33390, 33281, 33479, 33261, 33356, 33276, + 33471, 33488, 33254, 33338, 33273, 33409, 33489, 33265, 33417, 33413, + 33415, 33286, 33509, 33266, 33410, 33412, 33411, 33416, 33414, 33504, + 33382, 33279, 33311, 33498, 33499, 33497, 33299, 33477, 33257, 33256, + 33406, 33482, 33404, 33285, 33394, 33405, 33503, 33403, 33407, 33395, + 33283, 33312, 33302, 33483, 33268, 33269, 33267, 33284, 33288, 33289, + 33501, 33502, 33500, 33468, 33371, 33473, 33374, 33375, 33376, 33373, + 33377, 33372, 33366, 33367, 33368, 33365, 33363, 33294, 33364, 33360, + 33297, 33298, 33475, 33476, 33472, 33474, 33259, 33260, 33258, 33418, + 33423, 33426, 33427, 33428, 33434, 33419, 33421, 33422, 33430, 33425, + 33431, 33432, 33424, 33314, 33435, 33826, 33825, 33824, 33846, 33845, + 33844, 33801, 33800, 33799, 33185, 33184, 33183, 33787, 33786, 33785, + 33797, 33798, 33793, 33796, 33792, 33795, 33794, 33242, 33245, 33241, + 33244, 33243, 33791, 33661, 33662, 33663, 33664, 33659, 33660, 33665, + 33705, 33610, 33723, 33722, 33720, 33721, 33724, 33699, 33702, 33700, + 33701, 33698, 33729, 33728, 33726, 33727, 33675, 33674, 33673, 33679, + 33678, 33677, 33676, 33697, 33696, 33695, 33672, 33671, 33670, 33745, + 33744, 33743, 33719, 33718, 33717, 33842, 33841, 33840, 33839, 33838, + 33837, 33843, 33836, 33835, 33834, 33579, 33578, 33576, 33577, 33583, + 33582, 33580, 33581, 33571, 33570, 33568, 33569, 33575, 33574, 33572, + 33573, 33633, 33632, 33630, 33631, 33634, 33648, 33651, 33649, 33650, + 33607, 33638, 33637, 33636, 33635, 33593, 33606, 33605, 33604, 33594, + 33592, 33591, 33590, 33657, 33656, 33655, 33654, 33653, 33652, 33829, + 33828, 33827, 33832, 33831, 33830, 33833, 33692, 33691, 33689, 33690, + 33683, 33682, 33680, 33681, 33688, 33687, 33710, 33709, 33706, 33711, + 33716, 33713, 33712, 33732, 33731, 33730, 33735, 33734, 33733, 33686, + 33694, 33693, 33782, 33779, 33778, 33725, 33684, 33707, 33714, 33739, + 33783, 33780, 33776, 33685, 33708, 33715, 33740, 33784, 33781, 33777, + 33738, 33737, 33736, 33597, 33596, 33614, 33612, 33613, 33611, 33623, + 33621, 33622, 33620, 33640, 33639, 33771, 33768, 33774, 33599, 33598, + 33615, 33616, 33618, 33617, 33627, 33625, 33626, 33624, 33642, 33641, + 33772, 33769, 33775, 33603, 33602, 33600, 33601, 33595, 33619, 33628, + 33643, 33644, 33645, 33770, 33767, 33773, 33629, 33669, 33667, 33668, + 33666, 33589, 33587, 33585, 33588, 33586, 33584, 33742, 33741, 33647, + 33646, 33704, 33703, 33609, 33608, 33201, 33200, 33196, 33199, 33203, + 33202, 33197, 33198, 33195, 33204, 33514, 33521, 33519, 33518, 33516, + 33517, 33515, 33520, 33234, 33232, 33233, 33210, 33209, 33208, 33193, + 33191, 33192, 33194, 33249, 33248, 33247, 33228, 33229, 33225, 33206, + 33205, 33224, 33226, 33223, 33227, 33207, 33222, 33220, 33221, 33217, + 33219, 33218, 33211, 33213, 33212, 33215, 33214, 33216, 33188, 33187, + 33186, 33811, 33810, 33812, 33231, 33748, 33747, 33749, 33750, 33178, + 33180, 33177, 33179, 33182, 33181, 33543, 33541, 33542, 33548, 33550, + 33549, 33545, 33547, 33546, 33562, 33561, 33560, 33554, 33555, 33556, + 33557, 33558, 33559, 33551, 33553, 33552, 33563, 33564, 33565, 33532, + 33530, 33531, 33544, 33567, 33566, 33819, 33818, 33816, 33817, 33815, + 33820, 33814, 33813, 33803, 33808, 33806, 33807, 33804, 33805, 33809, + 33746, 33658, 33751, 33513, 33230, 33789, 33788, 33251, 33246, 33790, + 33822, 33821, 33823, 33847, 33522, 33523, 33524, 33525, 33526, 33527, + 33528, 33529, 33240, 33540, 33539, 33534, 33537, 33536, 33533, 33535, + 33538, 33190, 33250, 33802, 33189, 33848, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 33235, 33236, 33237, 33238, 33239, 41412, 33759, 33760, 33761, + 33762, 33763, 33764, 33765, 33766, 33752, 33753, 33754, 33755, 33756, + 33757, 33758, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 23665, 23940, 23431, 23945, 23418, 23789, 24051, 23942, + 24037, 24006, 23411, 23644, 23645, 24041, 23405, 23458, 23432, 23782, + 23581, 23762, 23642, 24035, 23921, 24010, 23653, 23582, 23726, 23879, + 24011, 23548, 23957, 41412, 41412, 41412, 41412, 41412, 41412, 23572, + 23775, 23821, 23926, 23965, 24001, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 8342, 8344, 8354, + 8348, 8330, 8355, 8362, 41412, 8361, 8335, 8332, 8331, 8329, 8366, 8350, + 8349, 8351, 8365, 8352, 8353, 8337, 8340, 8364, 8346, 8363, 41412, 41412, + 8338, 8341, 8345, 8339, 8357, 8356, 8358, 41412, 8360, 8336, 41412, 8359, + 8334, 8343, 8333, 8347, 41412, 41412, 41412, 41412, 41412, 27944, 27913, + 27941, 27939, 27915, 27937, 27934, 27935, 27936, 27943, 27920, 27921, + 27945, 27924, 27922, 27917, 27930, 27946, 27919, 27942, 27929, 27938, + 27928, 27931, 27916, 27933, 27914, 27927, 27911, 27940, 27912, 27925, + 27923, 10652, 10630, 10650, 10637, 10633, 10647, 10644, 10645, 10646, + 10651, 10635, 10653, 10649, 10636, 10654, 10634, 10639, 10643, 10648, + 10642, 10640, 10641, 10638, 10629, 10632, 10631, 27918, 27932, 27926, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 8250, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 30063, 30047, 30038, 30049, + 30054, 30046, 30051, 30042, 30068, 30072, 30074, 30077, 30041, 30036, + 30071, 30061, 30045, 30044, 30075, 30037, 30048, 30069, 30057, 30073, + 30076, 30043, 30065, 30050, 30040, 30060, 30039, 30055, 30062, 30064, + 30070, 30056, 30052, 30053, 30078, 30079, 30058, 30059, 30066, 30067, + 30080, 41412, 41412, 41412, 30089, 30093, 30092, 30095, 30094, 30091, + 30090, 30086, 30083, 30082, 30085, 30084, 30087, 30088, 41412, 41412, + 30103, 30105, 30102, 30101, 30098, 30097, 30100, 30099, 30106, 30104, + 41412, 41412, 41412, 41412, 30081, 30096, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 37973, 37956, 37976, 37966, + 37970, 37967, 37972, 37958, 37957, 37975, 37963, 37978, 37977, 37969, + 37968, 37974, 37971, 37959, 37951, 37960, 37952, 37980, 37961, 37953, + 37962, 37954, 37979, 37965, 37955, 37964, 37981, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 39435, 39434, 39466, 39467, 39468, 39470, + 39459, 39462, 39448, 39451, 39463, 39455, 39452, 39469, 39465, 39464, + 39472, 39477, 39476, 39475, 39461, 39440, 39439, 39474, 39473, 39460, + 39471, 39443, 39445, 39449, 39456, 39447, 39454, 39453, 39442, 39437, + 39438, 39446, 39441, 39444, 39436, 39450, 39457, 39458, 39481, 39482, + 39479, 39480, 39489, 39491, 39488, 39487, 39484, 39483, 39486, 39485, + 39492, 39490, 41412, 41412, 41412, 41412, 41412, 39478, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 29044, 29047, 29046, 29048, + 29045, 29027, 29031, 29029, 29028, 29030, 29039, 29042, 29040, 29043, + 29041, 29049, 29050, 29051, 29052, 29053, 29032, 29034, 29037, 29038, + 29033, 29036, 29035, 29057, 29054, 29058, 29056, 29055, 29065, 29067, + 29064, 29063, 29060, 29059, 29062, 29061, 29068, 29066, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 30226, 30229, 30228, 30227, 30230, 30231, 30208, 30210, + 30209, 30211, 30212, 30213, 30220, 30221, 30225, 30222, 30223, 30224, + 30232, 30236, 30233, 30235, 30234, 30237, 30214, 30219, 30218, 30216, + 30215, 30217, 30240, 30238, 30239, 30248, 30250, 30247, 30246, 30243, + 30242, 30245, 30244, 30251, 30249, 41412, 41412, 41412, 41412, 30241, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 35536, 35545, 35534, 35543, 35567, 35552, 35565, 35542, + 35551, 35537, 35546, 35566, 35541, 35550, 35559, 35553, 35564, 35540, + 35549, 35558, 35538, 35547, 35568, 35571, 35533, 35570, 35539, 35548, + 35569, 35535, 35544, 41412, 35526, 35560, 35555, 35576, 35554, 35527, + 35574, 35562, 35572, 35561, 35556, 35557, 35563, 35525, 35575, 35573, + 35530, 35529, 35528, 35532, 35531, 35577, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 35578, 35579, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 17141, 17147, 17145, 17142, 17144, 17143, 17146, 41412, 17096, 17140, + 17138, 17137, 41412, 17084, 17083, 41412, 17095, 17094, 17093, 17080, + 17079, 17092, 17091, 17090, 17089, 17088, 17087, 17082, 17081, 17086, + 17085, 41412, 27254, 27255, 27256, 27322, 27343, 27331, 27295, 27431, + 27257, 27258, 27262, 27381, 27366, 27371, 27294, 27449, 27398, 27320, + 27300, 27392, 27261, 27259, 27260, 27307, 27351, 27404, 27444, 27267, + 27263, 27271, 27408, 27346, 27356, 27383, 27272, 27269, 27268, 27421, + 27359, 27419, 27397, 27388, 27386, 27387, 27450, 27429, 27266, 27281, + 27273, 27420, 27365, 27390, 27325, 27451, 27277, 27278, 27279, 27335, + 27327, 27311, 27412, 27363, 27264, 27270, 27265, 27338, 27435, 27436, + 27274, 27275, 27276, 27349, 27304, 27354, 27321, 27282, 27280, 27283, + 27407, 27364, 27413, 27315, 27427, 27284, 27288, 27292, 27361, 27333, + 27401, 27382, 27285, 27289, 27290, 27332, 27324, 27389, 27342, 27452, + 27353, 27291, 27286, 27287, 27369, 27422, 27430, 27299, 27442, 27293, + 27352, 27302, 27399, 27339, 27377, 27309, 27391, 27341, 27308, 27448, + 27297, 27344, 27301, 27340, 27368, 27395, 27406, 27376, 27411, 27378, + 27337, 27357, 27438, 27405, 27372, 27415, 27445, 27418, 27416, 27441, + 27312, 27428, 27318, 27347, 27303, 27334, 27310, 27358, 27314, 27394, + 27313, 27373, 27298, 27439, 27329, 27424, 27425, 27443, 27414, 27355, + 27393, 27385, 27348, 27323, 27296, 27362, 27370, 27409, 27375, 27305, + 27400, 27345, 27360, 27326, 27328, 27434, 27374, 27380, 27379, 27446, + 27367, 27316, 27317, 27403, 27447, 27396, 27384, 27437, 27440, 27410, + 27432, 27336, 27402, 27330, 27417, 27306, 27433, 27350, 27319, 41412, + 41412, 27460, 27458, 27457, 27454, 27453, 27456, 27455, 27461, 27459, + 27249, 27248, 27252, 27250, 27247, 27251, 27253, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 27, 9, 24, 14, + 29, 21, 34, 28, 37, 39, 35, 40, 41, 10, 30, 32, 18, 15, 31, 42, 13, 25, + 36, 23, 12, 20, 33, 19, 38, 17, 11, 26, 16, 22, 62, 44, 59, 49, 64, 56, + 69, 63, 72, 74, 70, 75, 76, 45, 65, 67, 53, 50, 66, 77, 48, 60, 71, 58, + 47, 55, 68, 54, 73, 52, 46, 61, 51, 57, 85, 86, 79, 84, 43, 78, 83, 82, + 41412, 41412, 41412, 41412, 93, 95, 92, 91, 88, 87, 90, 89, 96, 94, + 41412, 41412, 41412, 41412, 80, 81, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 20858, 20844, + 20839, 20819, 20814, 20832, 20827, 20807, 20822, 20847, 20842, 20837, + 20817, 20812, 20833, 20828, 20808, 20823, 20859, 20845, 20840, 20820, + 20815, 20835, 20830, 20810, 20825, 20860, 20846, 20841, 20821, 20816, + 20836, 20831, 20811, 20826, 20848, 20843, 20838, 20818, 20813, 20834, + 20829, 20809, 20824, 20805, 20806, 20796, 20803, 20804, 20856, 20854, + 20853, 20850, 20849, 20852, 20851, 20857, 20855, 20862, 20798, 20797, + 20799, 20861, 20802, 20801, 20800, 20795, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 31008, 31003, 30998, 30978, 30973, 30991, 30986, 30966, + 30981, 31006, 31001, 30996, 30976, 30971, 30992, 30987, 30967, 30982, + 31009, 31004, 30999, 30979, 30974, 30994, 30989, 30969, 30984, 31010, + 31005, 31000, 30980, 30975, 30995, 30990, 30970, 30985, 31007, 31002, + 30997, 30977, 30972, 30993, 30988, 30968, 30983, 30965, 30958, 30960, + 30950, 30952, 30953, 30955, 30962, 30961, 30956, 30951, 30954, 30959, + 30957, 30964, 30963, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 2288, 2296, 2294, 2192, + 41412, 2304, 2292, 2300, 2284, 2299, 2291, 2240, 2295, 2302, 2267, 2289, + 2297, 2268, 2303, 2298, 2266, 2287, 2286, 2290, 2285, 2191, 2293, 2301, + 2162, 2164, 2163, 2165, 41412, 2204, 2202, 41412, 2199, 41412, 41412, + 2198, 41412, 2206, 2201, 2209, 2203, 2208, 2196, 2211, 2205, 2197, 2210, + 41412, 2195, 2194, 2193, 2200, 41412, 2212, 41412, 2207, 41412, 41412, + 41412, 41412, 41412, 41412, 2276, 41412, 41412, 41412, 41412, 2278, + 41412, 2277, 41412, 2281, 41412, 2280, 2273, 2283, 41412, 2274, 2282, + 41412, 2272, 41412, 41412, 2275, 41412, 2271, 41412, 2279, 41412, 2269, + 41412, 2270, 41412, 2258, 2256, 41412, 2253, 41412, 41412, 2252, 2247, + 2260, 2255, 41412, 2257, 2263, 2250, 2265, 2259, 2251, 2264, 41412, 2249, + 2248, 2246, 2254, 41412, 2245, 2261, 2262, 2243, 41412, 2244, 41412, + 2217, 2229, 2227, 2237, 2220, 2239, 2225, 2219, 2223, 2232, 41412, 2235, + 2228, 2234, 2214, 2218, 2230, 2215, 2238, 2231, 2213, 2224, 2222, 2216, + 2221, 2236, 2226, 2233, 41412, 41412, 41412, 41412, 41412, 2178, 2176, + 2187, 41412, 2190, 2174, 2182, 2172, 2181, 41412, 2185, 2177, 2184, 2167, + 2189, 2179, 2168, 2188, 2180, 2166, 2173, 2171, 2169, 2170, 2186, 2175, + 2183, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 2241, 2242, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 25561, + 25577, 25592, 25568, 25596, 25595, 25593, 25573, 25590, 25587, 25566, + 25563, 25582, 25579, 25559, 25570, 25574, 25591, 25588, 25567, 25564, + 25583, 25580, 25560, 25571, 25572, 25589, 25586, 25565, 25562, 25581, + 25578, 25558, 25569, 25576, 25575, 25556, 25599, 25585, 25584, 25597, + 25594, 25598, 25557, 41412, 41412, 41412, 41412, 11271, 11222, 11223, + 11224, 11225, 11226, 11227, 11228, 11229, 11230, 11231, 11232, 11233, + 11234, 11235, 11236, 11237, 11238, 11239, 11240, 11241, 11242, 11243, + 11244, 11245, 11246, 11247, 11248, 11249, 11250, 11251, 11252, 11253, + 11254, 11255, 11256, 11257, 11258, 11259, 11260, 11261, 11262, 11263, + 11264, 11265, 11266, 11267, 11268, 11269, 11270, 11321, 11272, 11273, + 11274, 11275, 11276, 11277, 11278, 11279, 11280, 11281, 11282, 11283, + 11284, 11285, 11286, 11287, 11288, 11289, 11290, 11291, 11292, 11293, + 11294, 11295, 11296, 11297, 11298, 11299, 11300, 11301, 11302, 11303, + 11304, 11305, 11306, 11307, 11308, 11309, 11310, 11311, 11312, 11313, + 11314, 11315, 11316, 11317, 11318, 11319, 11320, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 31671, + 31741, 31716, 31712, 31675, 31680, 31700, 31696, 31692, 31745, 31708, + 31749, 31684, 31704, 31688, 41412, 41412, 31742, 31717, 31713, 31676, + 31681, 31701, 31697, 31693, 31746, 31709, 31750, 31685, 31705, 31689, + 31672, 41412, 31743, 31718, 31714, 31677, 31682, 31702, 31698, 31694, + 31747, 31710, 31751, 31686, 31706, 31690, 31670, 41412, 31740, 31715, + 31711, 31674, 31679, 31699, 31695, 31691, 31744, 31707, 31748, 31683, + 31703, 31687, 31673, 31678, 31722, 31719, 31733, 31734, 31735, 31736, + 31737, 31738, 31739, 31723, 31724, 31725, 31726, 31727, 31728, 31729, + 31730, 31731, 31732, 31720, 31721, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 11006, 11005, 10990, 11002, 10999, + 10984, 10981, 10996, 10993, 11008, 10987, 11047, 11026, 6918, 6635, 6659, + 31339, 31340, 31341, 31342, 31343, 31344, 31345, 31346, 31347, 31348, + 31349, 31350, 31351, 31352, 31353, 31354, 31355, 31356, 31357, 31358, + 31359, 31360, 31361, 31362, 31363, 31364, 37949, 6749, 6750, 6646, 6917, + 8778, 34644, 34645, 34646, 34647, 34648, 34649, 34650, 34651, 34652, + 34653, 34654, 34655, 34656, 34657, 34658, 34659, 34660, 34661, 34662, + 34663, 34664, 34665, 34666, 34667, 34668, 34669, 34638, 34674, 34689, + 34690, 34679, 34701, 29157, 29158, 29159, 29160, 29161, 29162, 29163, + 29164, 29165, 29166, 29167, 29168, 29169, 29170, 29171, 29172, 29173, + 29174, 29175, 29176, 29177, 29178, 29179, 29180, 29181, 29182, 31950, + 31951, 31952, 6645, 6644, 6692, 29188, 29189, 29190, 29191, 29192, 29193, + 29194, 29195, 29196, 29197, 29198, 29199, 29200, 29201, 29202, 29203, + 29204, 29205, 29206, 29207, 29208, 29209, 29210, 29211, 29212, 29213, + 8822, 29220, 29223, 29224, 29219, 29221, 34373, 34627, 34626, 34633, + 34702, 34675, 34676, 34678, 34688, 34696, 34699, 34691, 34681, 34693, + 34631, 34695, 34632, 34682, 34692, 34684, 34677, 34643, 34637, 34636, + 34635, 34672, 34683, 34697, 34698, 25991, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 31980, 31981, 31982, 31983, 31984, 31985, 31986, 31987, + 31988, 31989, 31990, 31991, 31992, 31993, 31994, 31995, 31996, 31997, + 31998, 31999, 32000, 32001, 32002, 32003, 32004, 32005, 34414, 34639, + 34641, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 34605, 34600, 34592, 34640, 34586, 34598, + 34621, 34597, 34585, 34610, 34618, 34609, 34590, 34601, 34602, 34608, + 34589, 34619, 34616, 34623, 34599, 34594, 34613, 34603, 34606, 34583, + 34584, 34625, 34596, 34587, 34591, 34607, 34622, 34604, 34617, 34620, + 34593, 34614, 34612, 34611, 34615, 34588, 34595, 34624, 41412, 41412, + 41412, 41412, 37946, 37940, 37941, 37943, 37947, 37944, 37948, 37942, + 37945, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 6697, 6695, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 32416, 32417, 32414, 32418, 32413, 32415, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 10657, 17627, 8142, 29467, 34877, 34876, + 6928, 34922, 31943, 5033, 39583, 39408, 27764, 11671, 11669, 11670, + 18163, 29272, 39495, 17592, 39496, 17670, 39494, 22905, 39493, 8812, + 29271, 17591, 22904, 17669, 34801, 18164, 33020, 37385, 3930, 39742, + 39746, 39743, 39744, 8155, 8154, 8153, 8152, 17626, 39809, 20595, 37043, + 5109, 6579, 32750, 17519, 10683, 31207, 6226, 20592, 38117, 6576, 32404, + 20552, 34923, 4344, 11665, 11666, 20377, 17646, 25843, 17559, 24202, + 28479, 37899, 2586, 18276, 27246, 39586, 36066, 24568, 3475, 31655, + 31977, 18816, 31475, 31472, 6578, 34741, 19266, 34006, 27036, 31791, + 32100, 32101, 8612, 10084, 34729, 34305, 5031, 17660, 32420, 10666, + 31065, 34963, 17668, 17598, 34102, 32964, 20626, 11417, 8611, 6618, 6101, + 25446, 10087, 20558, 33039, 3699, 31789, 8610, 17632, 37045, 32717, + 39811, 8159, 38030, 3589, 8101, 2650, 17633, 4468, 31779, 32091, 39831, + 3860, 20987, 6619, 17565, 17588, 17587, 2793, 31397, 8591, 36065, 8823, + 31654, 20992, 6163, 39808, 28362, 32722, 18201, 19847, 4470, 27763, + 32044, 28485, 34750, 24567, 8603, 3579, 3580, 17581, 98, 6161, 17571, + 32362, 17596, 27753, 28384, 6621, 19846, 2565, 37924, 6926, 37792, 8097, + 31504, 39330, 11052, 34012, 3859, 17859, 4473, 17615, 28720, 28466, + 32716, 18832, 28484, 38035, 39335, 28719, 32545, 37160, 33987, 3481, + 6580, 34096, 32546, 34961, 34336, 38032, 20590, 372, 32423, 34966, 39604, + 18200, 31933, 31934, 8814, 39409, 17602, 20628, 35154, 34093, 5367, 3585, + 5108, 20602, 6927, 10710, 8098, 10792, 10793, 29138, 34727, 20601, 20600, + 31061, 20989, 17484, 20603, 3470, 2583, 20596, 25330, 8606, 32721, 10709, + 17555, 20985, 20988, 17483, 39711, 3983, 39592, 39591, 32405, 3999, + 22727, 2661, 4477, 370, 16877, 16878, 16879, 16880, 16881, 31958, 28379, + 31068, 39584, 8805, 37690, 24464, 31960, 6168, 11507, 8826, 39765, 34091, + 34089, 20589, 31963, 18168, 33045, 28360, 32403, 6587, 11158, 31646, + 4655, 16846, 30127, 34330, 5046, 966, 20557, 22729, 17595, 38031, 4345, + 38142, 19815, 2649, 17664, 3861, 31476, 22724, 31801, 11509, 2659, 11220, + 28381, 8806, 37691, 31961, 6169, 11508, 34335, 20591, 28361, 11219, + 31648, 17667, 19264, 39830, 3584, 31244, 31647, 31465, 6586, 17516, + 17514, 11664, 29620, 28382, 37900, 39753, 39668, 39694, 39718, 17599, + 39593, 31064, 37418, 37419, 8096, 30718, 8828, 39821, 17515, 29462, + 35151, 21089, 11515, 22719, 3865, 39819, 31906, 19270, 31796, 25837, + 2580, 20448, 39820, 39818, 17631, 5104, 5103, 4653, 18063, 25750, 39816, + 17563, 25756, 38157, 38158, 31776, 39817, 5034, 31489, 25753, 25754, + 30697, 30695, 2648, 8594, 31858, 20995, 20994, 19132, 2651, 17503, 350, + 20755, 33989, 20871, 18831, 10665, 25223, 29069, 17551, 19137, 3479, + 35149, 31651, 22716, 25329, 32342, 17863, 22711, 4469, 8804, 39602, 3586, + 5040, 38151, 34306, 18829, 19849, 4347, 18818, 39858, 31905, 19848, + 32087, 19850, 10968, 16834, 964, 4652, 34001, 8164, 34331, 11511, 10670, + 31649, 17609, 11138, 34319, 37393, 39679, 20612, 28182, 10082, 19879, + 8811, 3472, 3474, 3473, 3471, 28181, 6397, 32746, 31499, 5035, 27766, + 17616, 30729, 11662, 17577, 30716, 31072, 31074, 5364, 37046, 6106, 6396, + 6398, 3477, 8102, 31908, 32411, 31467, 34739, 38000, 4358, 24566, 29618, + 29619, 8148, 30710, 18817, 4346, 30733, 4359, 29140, 32743, 27600, 37051, + 31075, 17562, 32629, 31909, 6403, 31012, 20982, 31466, 17517, 20787, + 16915, 8145, 8146, 30720, 30719, 31785, 31782, 29457, 27780, 27781, + 39326, 27782, 29537, 968, 5365, 5366, 39329, 37058, 31936, 39331, 17580, + 31780, 31872, 38145, 8132, 8133, 8129, 977, 25331, 20443, 34314, 34313, + 34315, 34316, 3578, 16832, 24333, 32218, 25281, 8147, 21774, 25278, + 30723, 3592, 3594, 4357, 25221, 31938, 2653, 16910, 30699, 34152, 37939, + 29536, 21789, 20874, 20875, 20878, 20877, 20876, 17584, 16833, 39834, + 19260, 30033, 20604, 31660, 27752, 37057, 8833, 33983, 20993, 38002, + 4010, 39729, 22895, 22822, 22830, 22823, 34018, 34017, 38240, 11430, + 38244, 11424, 25400, 38336, 6642, 8820, 8819, 29611, 29613, 35038, 39692, + 19894, 6234, 31066, 11502, 21772, 28368, 35059, 27462, 4471, 8113, 8123, + 8125, 8109, 8107, 8117, 8115, 8105, 8111, 8119, 8103, 8121, 8114, 8124, + 8126, 8110, 8108, 8118, 8116, 8106, 8112, 8120, 8104, 8122, 32165, 32166, + 32167, 5099, 5100, 32350, 4356, 6100, 25840, 4012, 29535, 20556, 25751, + 34004, 10667, 34326, 34327, 21090, 25755, 24254, 37052, 32146, 39748, + 4025, 37056, 8099, 2654, 34711, 16914, 17625, 31479, 25220, 3980, 25362, + 25348, 25360, 25361, 25382, 24314, 38133, 31948, 32072, 32080, 32081, + 32086, 32082, 31949, 39669, 33174, 33173, 33170, 33169, 3958, 3985, + 33176, 33175, 33172, 33171, 4028, 3924, 3937, 10794, 21776, 37409, 31855, + 31802, 3940, 39683, 34103, 37040, 39814, 30707, 38146, 37405, 37984, + 30521, 19813, 32727, 31856, 17561, 30730, 11144, 11145, 11146, 17656, + 17658, 17657, 3936, 17629, 30717, 6107, 6108, 17578, 16882, 16883, 16884, + 29615, 29616, 29617, 16893, 16886, 16887, 11143, 31071, 31076, 39598, + 34329, 34328, 10795, 27767, 27022, 31055, 8131, 6098, 20789, 10684, 8586, + 30694, 32361, 31073, 34737, 10664, 25222, 34317, 37401, 37400, 37402, + 37403, 24282, 32168, 38161, 37407, 24299, 32182, 24212, 32104, 28365, + 24591, 24590, 2798, 2804, 2799, 2803, 2796, 24588, 2797, 39825, 28378, + 37983, 34724, 33986, 28383, 18821, 18824, 17541, 34077, 34079, 34078, + 34080, 34076, 34075, 39812, 34081, 17521, 32043, 34074, 34084, 34087, + 29269, 17496, 38197, 17524, 31477, 8592, 8593, 22712, 17552, 22713, + 22714, 17538, 17539, 17540, 11054, 39826, 965, 31794, 8832, 31501, 17546, + 11053, 17666, 962, 17567, 39600, 34003, 37791, 18826, 25445, 17529, + 20611, 17531, 17522, 2573, 17617, 34002, 11139, 17549, 17526, 18825, + 6170, 34073, 34072, 6171, 22715, 31793, 8831, 39599, 34008, 34007, 38348, + 17545, 17534, 17528, 31498, 32751, 19852, 34312, 19812, 31496, 31495, + 31491, 31493, 29580, 34208, 29553, 34195, 38131, 38140, 38130, 38139, + 29579, 34207, 29552, 34194, 19930, 19923, 19927, 19920, 29581, 34209, + 29554, 34196, 19931, 19924, 19928, 19921, 20554, 20555, 34148, 34149, + 24457, 38382, 32305, 11497, 32737, 19925, 24570, 19902, 19857, 34964, + 32623, 32624, 32625, 19947, 32626, 19914, 39316, 39313, 6399, 32053, + 32360, 20089, 34728, 31941, 20446, 20447, 37991, 27598, 24577, 34725, + 37987, 37988, 5102, 30704, 38028, 5105, 27768, 373, 17585, 31774, 30703, + 37044, 30702, 2582, 30700, 31976, 10689, 2564, 37985, 28359, 28375, + 34962, 28376, 160, 33019, 32419, 34318, 20582, 39307, 8595, 31775, 37999, + 11503, 29533, 34088, 29538, 31907, 11501, 31787, 29542, 3855, 29531, + 3854, 28377, 31508, 29534, 6582, 27463, 39822, 32048, 2652, 37982, 39582, + 33044, 3576, 3577, 31407, 10086, 2665, 24255, 37996, 31870, 6751, 4654, + 18064, 8803, 34000, 33022, 3596, 3756, 31665, 30126, 33021, 34752, 31077, + 20550, 20614, 16847, 22731, 41412, 41412, 41412, 39815, 31669, 39606, + 32343, 19261, 33015, 30159, 28373, 31940, 28370, 38238, 38235, 38243, + 34015, 29588, 227, 228, 41412, 41412, 41412, 32628, 30701, 10979, 31403, + 32725, 28371, 6103, 34005, 17620, 33992, 2584, 31645, 32363, 41412, + 41412, 41412, 297, 245, 345, 344, 341, 239, 237, 238, 235, 236, 334, 336, + 337, 323, 290, 252, 283, 284, 285, 267, 311, 288, 338, 339, 309, 310, + 272, 325, 278, 279, 261, 302, 258, 280, 321, 259, 260, 257, 312, 319, + 340, 331, 281, 240, 320, 313, 318, 335, 300, 301, 299, 303, 304, 305, + 232, 233, 286, 314, 242, 306, 307, 243, 248, 328, 329, 298, 249, 250, + 251, 234, 346, 324, 330, 273, 342, 291, 256, 333, 255, 327, 254, 332, + 315, 282, 326, 343, 276, 244, 293, 253, 292, 241, 316, 317, 322, 296, + 270, 268, 269, 295, 294, 262, 263, 264, 265, 266, 231, 246, 247, 308, + 277, 289, 271, 287, 275, 274, 25327, 30128, 25448, 39324, 2577, 20616, + 31399, 19842, 25625, 18198, 31927, 30736, 3951, 4029, 3991, 3925, 4015, + 27137, 4353, 19944, 39317, 17500, 39648, 32412, 4023, 4016, 24585, 27163, + 4354, 19941, 39318, 17501, 39730, 39732, 34559, 4021, 4039, 3973, 39663, + 39664, 10967, 4022, 4040, 3974, 39701, 37390, 24587, 27164, 4355, 39322, + 39319, 17502, 37388, 24580, 27154, 4349, 19915, 39314, 17497, 24573, + 27143, 4352, 19890, 39312, 17499, 24581, 27153, 4350, 19919, 39315, + 17498, 24571, 27161, 4351, 19884, 39311, 24583, 27158, 37408, 27157, + 24575, 27142, 17648, 27141, 32054, 24572, 19889, 27152, 19918, 33981, + 27160, 19882, 39310, 19880, 24582, 19937, 19936, 6912, 29183, 6922, + 29184, 29465, 41412, 41412, 41412, 41412, 41412, 41412, 22829, 22897, + 22825, 22893, 22820, 22896, 22824, 22831, 22898, 22826, 22894, 22821, + 41412, 41412, 41412, 41412, 19887, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 24377, 38358, 32258, 11448, 24384, 38355, 32265, 11445, 24371, 38354, + 32255, 11444, 41412, 41412, 41412, 41412, 24376, 38357, 32257, 11447, + 24386, 38359, 32267, 11449, 19898, 19939, 19912, 19876, 19897, 19938, + 19911, 19875, 24421, 38392, 32318, 11468, 24420, 38391, 32317, 11467, + 24419, 38388, 32316, 11464, 24423, 38394, 32320, 11470, 24422, 38393, + 32319, 11469, 24451, 38369, 32284, 11484, 24459, 38384, 32307, 11499, + 24461, 38380, 32340, 11495, 24411, 38378, 32298, 11493, 24412, 38379, + 32299, 11494, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 24460, 38383, 32308, 11498, 29586, 29559, 34201, 34214, 24274, 38225, + 41412, 41412, 41412, 41412, 41412, 41412, 39768, 39783, 39773, 39778, + 39793, 39788, 39798, 39803, 39770, 39785, 39775, 39780, 39795, 39790, + 39800, 39805, 39769, 39784, 39774, 39779, 39794, 39789, 39799, 39804, + 39766, 39781, 39771, 39776, 39791, 39786, 39796, 39801, 39767, 39782, + 39772, 39777, 39792, 39787, 39797, 39802, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 24427, 38398, 32324, 11475, 24441, 38408, + 32339, 11480, 24385, 38356, 32266, 11446, 19853, 19856, 19855, 19854, + 24395, 32273, 24430, 32309, 24452, 32304, 24456, 32300, 24394, 32272, + 24450, 32283, 39610, 39609, 41412, 41412, 2560, 2557, 32253, 11459, + 29214, 29215, 29222, 29216, 29578, 29550, 34193, 34206, 41412, 41412, + 41412, 41412, 24391, 32244, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 25318, 25322, + 25323, 33029, 25314, 33026, 25316, 25317, 25306, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 6640, 6641, 6639, 24239, 24237, + 24238, 24240, 24236, 11437, 11439, 11438, 11440, 31652, 39684, 5042, + 31653, 41221, 28183, 17542, 29463, 37391, 17525, 32421, 20613, 33849, + 5363, 31957, 24348, 32207, 19271, 19267, 20986, 17523, 8156, 29139, + 32365, 11512, 25506, 17550, 34092, 17533, 18823, 18822, 17544, 32846, + 34082, 17530, 33042, 31803, 5030, 31242, 32854, 31857, 25752, 28369, + 33047, 31486, 21092, 17573, 27783, 39833, 39585, 19269, 11134, 39807, + 11513, 8100, 38143, 34334, 18167, 32354, 17594, 32749, 37392, 4650, + 25914, 10081, 22728, 34105, 17624, 8827, 2642, 10088, 2660, 31788, 6166, + 2663, 18819, 32858, 34751, 16782, 18162, 31473, 22717, 31243, 11659, + 17636, 35648, 6617, 4472, 10074, 8160, 5041, 31663, 31853, 10089, 32627, + 6102, 24203, 25841, 28363, 2664, 34083, 39857, 34085, 17535, 17548, + 31054, 17662, 29466, 11058, 17553, 17537, 32718, 22726, 18197, 20551, + 17603, 8834, 25275, 32723, 38119, 38211, 11674, 11660, 3519, 32963, + 31067, 17653, 5107, 10962, 18196, 25276, 32089, 33046, 34704, 18065, + 41215, 20442, 32713, 35152, 8813, 21378, 25512, 31470, 20553, 31402, + 31935, 25444, 28367, 27755, 2662, 34965, 25749, 11504, 34013, 31011, + 30735, 33991, 17608, 31060, 3587, 3866, 32745, 18833, 31871, 16873, + 16874, 16876, 16875, 4656, 24569, 17630, 37901, 34958, 34959, 32556, + 11667, 28372, 25838, 27037, 27038, 6402, 10076, 32559, 3755, 17858, + 30709, 17560, 39433, 5106, 27001, 20627, 5044, 38027, 34726, 22721, + 10961, 17527, 101, 6581, 30696, 3583, 31494, 31488, 31497, 31487, 25516, + 17566, 38971, 27587, 16871, 18060, 41407, 5028, 30734, 3858, 32720, + 18165, 8809, 34100, 31978, 17589, 21095, 37163, 31507, 11661, 8589, 2645, + 17586, 37903, 5037, 25515, 25447, 25326, 34333, 2806, 32557, 37050, 5043, + 3480, 32364, 3478, 34337, 31964, 29141, 29246, 29257, 29260, 29249, + 29239, 29254, 39623, 3893, 29240, 39620, 39636, 39639, 39614, 39628, + 39633, 3890, 3906, 3909, 3884, 3898, 3903, 29247, 29258, 29261, 29250, + 29245, 29255, 39624, 3894, 29241, 39642, 39645, 39646, 39641, 39643, + 39644, 3912, 3915, 3916, 3911, 3913, 3914, 29264, 29267, 29268, 29263, + 29265, 29266, 39625, 3895, 29242, 39621, 39637, 39640, 39615, 39626, + 39634, 3891, 3907, 3910, 3885, 3896, 3904, 29248, 29259, 29262, 29251, + 29243, 29256, 39627, 3897, 29244, 39616, 3886, 29252, 39617, 3887, 29253, + 39630, 39631, 39629, 3900, 3901, 3899, 39618, 39612, 3888, 3882, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 39849, 39852, 39848, + 39850, 39847, 39846, 39851, 39842, 39845, 39841, 39843, 39840, 39839, + 39844, 41412, 41412, 2807, 30708, 5036, 33040, 37394, 24584, 18820, + 31657, 11510, 99, 34731, 39838, 8830, 41412, 41412, 41412, 41129, 22720, + 31251, 4360, 25514, 31658, 29236, 25844, 17618, 19814, 38029, 41412, + 41412, 41412, 37992, 33043, 32349, 6237, 31962, 2647, 11137, 3476, 27762, + 1, 25304, 8808, 6164, 32726, 22730, 20606, 27778, 39810, 31771, 32851, + 22722, 5110, 28380, 37902, 19844, 31666, 32359, 27779, 20633, 25332, + 19265, 17628, 19134, 21859, 17619, 39827, 3590, 8158, 31790, 39829, + 17568, 25328, 8782, 16885, 29237, 20624, 21088, 39813, 24201, 18199, 958, + 25449, 31509, 31805, 31804, 31492, 17582, 41412, 19136, 41412, 41412, + 41412, 41412, 30737, 28366, 11322, 4348, 3593, 30698, 17604, 36064, + 17652, 37049, 31792, 3588, 21087, 18062, 31474, 32402, 41412, 41412, + 34332, 27245, 32561, 17532, 17536, 17547, 11332, 3863, 5045, 33014, + 17543, 11057, 41412, 41412, 41412, 41412, 17576, 19268, 32297, 24410, + 31208, 31209, 20794, 19851, 24455, 32303, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 4284, 4314, 4285, 4329, 4300, 4318, 4286, 4337, + 4307, 4315, 4293, 4330, 4301, 4319, 4287, 4341, 4311, 4326, 4297, 4334, + 4323, 4290, 4338, 4308, 4316, 4294, 4331, 4302, 4320, 4288, 4343, 4313, + 4328, 4299, 4336, 4306, 4325, 4292, 4340, 4310, 4296, 4333, 4304, 4322, + 4289, 4342, 4312, 4327, 4298, 4335, 4305, 4324, 4291, 4339, 4309, 4317, + 4295, 4332, 4303, 4321, 25349, 25350, 25357, 25359, 25354, 25415, 25416, + 25412, 25414, 25410, 25413, 25406, 25409, 25407, 25411, 25408, 25353, + 25356, 25351, 25355, 25352, 25358, 38308, 38309, 38316, 38318, 38313, + 38278, 38279, 38275, 38277, 38273, 38276, 38269, 38272, 38270, 38274, + 38271, 38312, 38315, 38310, 38314, 38311, 38317, 38251, 24204, 38248, + 24209, 24295, 38347, 32190, 25441, 39295, 39296, 39297, 39298, 39299, + 39300, 20568, 20569, 20570, 20571, 20572, 20573, 24205, 24210, 32103, + 32102, 38250, 20567, 38306, 38343, 38258, 38346, 38342, 32152, 32186, + 32130, 32185, 32162, 24253, 32145, 38265, 25342, 20969, 38260, 38262, + 41412, 24252, 6400, 20965, 19891, 38283, 38338, 38249, 24206, 38284, + 38339, 25402, 25379, 4556, 4560, 4548, 4554, 4557, 4562, 4549, 4551, + 4559, 4561, 4563, 4558, 4552, 4553, 4579, 4589, 2563, 20966, 24247, + 32140, 20967, 24364, 32241, 11454, 38351, 24244, 32137, 39407, 32154, + 29186, 29185, 29187, 39687, 24298, 27757, 32181, 29217, 34732, 34733, + 34735, 34736, 34734, 39755, 39660, 31953, 4005, 24305, 24260, 4555, 4576, + 4571, 4550, 4566, 4565, 4573, 4564, 4569, 4575, 4546, 4570, 4567, 4577, + 4547, 4572, 37927, 32148, 4466, 24319, 38257, 25426, 27758, 27759, 37926, + 32147, 4465, 24318, 37936, 4452, 4459, 37928, 32759, 32761, 32758, 32757, + 32754, 32753, 32756, 32755, 32762, 32760, 229, 41412, 41412, 41412, + 41412, 41412, 6959, 6960, 6961, 6962, 6963, 6964, 6965, 6966, 6967, 6968, + 6969, 6970, 6971, 6972, 6973, 6974, 6975, 6976, 6977, 6978, 6979, 6980, + 6981, 6982, 6983, 6984, 6985, 6986, 6987, 6988, 6989, 6990, 6991, 6992, + 6993, 6994, 6995, 6996, 6997, 6998, 6999, 7000, 7001, 7002, 7003, 7004, + 7005, 7006, 7007, 7008, 7009, 7010, 7011, 7012, 7013, 7014, 7015, 7016, + 7017, 7018, 7019, 7020, 7021, 7022, 7023, 7024, 7025, 7026, 7027, 7028, + 7029, 7030, 7031, 7032, 7033, 7034, 7035, 7036, 7037, 7038, 7039, 7040, + 7041, 7042, 7043, 7044, 7045, 7046, 7047, 7048, 7049, 7050, 7051, 7052, + 7053, 7054, 7055, 7056, 7057, 7058, 7059, 7060, 7061, 7062, 7063, 7064, + 7065, 7066, 7067, 7068, 7069, 7070, 7071, 7072, 7073, 7074, 7075, 7076, + 7077, 7078, 7079, 7080, 7081, 7082, 7083, 7084, 7085, 7086, 7087, 7088, + 7089, 7090, 7091, 7092, 7093, 7094, 7095, 7096, 7097, 7098, 7099, 7100, + 7101, 7102, 7103, 7104, 7105, 7106, 7107, 7108, 7109, 7110, 7111, 7112, + 7113, 7114, 7115, 7116, 7117, 7118, 7119, 7120, 7121, 7122, 7123, 7124, + 7125, 7126, 7127, 7128, 7129, 7130, 7131, 7132, 7133, 7134, 7135, 7136, + 7137, 7138, 7139, 7140, 7141, 7142, 7143, 7144, 7145, 7146, 7147, 7148, + 7149, 7150, 7151, 7152, 7153, 7154, 7155, 7156, 7157, 7158, 7159, 7160, + 7161, 7162, 7163, 7164, 7165, 7166, 7167, 7168, 7169, 7170, 7171, 7172, + 7173, 7174, 7175, 7176, 7177, 7178, 7179, 7180, 7181, 7182, 7183, 7184, + 7185, 7186, 7187, 7188, 7189, 7190, 7191, 7192, 7193, 7194, 7195, 7196, + 7197, 7198, 7199, 7200, 7201, 7202, 7203, 7204, 7205, 7206, 7207, 7208, + 7209, 7210, 7211, 7212, 7213, 7214, 7215, 7216, 7217, 7218, 7219, 7220, + 7221, 7222, 7223, 7224, 7225, 7226, 7227, 7228, 7229, 7230, 7231, 7232, + 7233, 7234, 7235, 7236, 7237, 7238, 7239, 7240, 7241, 7242, 7243, 7244, + 7245, 7246, 7247, 7248, 7249, 7250, 7251, 7252, 7253, 7254, 7255, 7256, + 7257, 7258, 7259, 7260, 7261, 7262, 7263, 7264, 7265, 7266, 7267, 7268, + 7269, 7270, 7271, 7272, 7273, 7274, 7275, 7276, 7277, 7278, 7279, 7280, + 7281, 7282, 7283, 7284, 7285, 7286, 7287, 7288, 7289, 7290, 7291, 7292, + 7293, 7294, 7295, 7296, 7297, 7298, 7299, 7300, 7301, 7302, 7303, 7304, + 7305, 7306, 7307, 7308, 7309, 7310, 7311, 7312, 7313, 7314, 7315, 7316, + 7317, 7318, 7319, 7320, 7321, 7322, 7323, 7324, 7325, 7326, 7327, 7328, + 7329, 7330, 7331, 7332, 7333, 7334, 7335, 7336, 7337, 7338, 7339, 7340, + 7341, 7342, 7343, 7344, 7345, 7346, 7347, 7348, 7349, 7350, 7351, 7352, + 7353, 7354, 7355, 7356, 7357, 7358, 7359, 7360, 7361, 7362, 7363, 7364, + 7365, 7366, 7367, 7368, 7369, 7370, 7371, 7372, 7373, 7374, 7375, 7376, + 7377, 7378, 7379, 7380, 7381, 7382, 7383, 7384, 7385, 7386, 7387, 7388, + 7389, 7390, 7391, 7392, 7393, 7394, 7395, 7396, 7397, 7398, 7399, 7400, + 7401, 7402, 7403, 7404, 7405, 7406, 7407, 7408, 7409, 7410, 7411, 7412, + 7413, 7414, 7415, 7416, 7417, 7418, 7419, 7420, 7421, 7422, 7423, 7424, + 7425, 7426, 7427, 7428, 7429, 7430, 7431, 7432, 7433, 7434, 7435, 7436, + 7437, 7438, 7439, 7440, 7441, 7442, 7443, 7444, 7445, 7446, 7447, 7448, + 7449, 7450, 7451, 7452, 7453, 7454, 7455, 7456, 7457, 7458, 7459, 7460, + 7461, 7462, 7463, 7464, 7465, 7466, 7467, 7468, 7469, 7470, 6943, 6944, + 6945, 6946, 6947, 6948, 6949, 6950, 6951, 6952, 6953, 6954, 6955, 6956, + 6957, 6958, 6929, 6930, 6931, 6932, 6933, 6934, 6935, 6936, 6937, 6938, + 6939, 6940, 6941, 6942, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 22732, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 35245, 35174, 35237, 35246, 35162, 35235, 35156, 35155, 35232, + 35240, 35157, 35236, 35160, 35177, 35248, 35244, 35169, 35171, 35168, + 35167, 35164, 35163, 35166, 35165, 35172, 35170, 35161, 35243, 35230, + 35173, 35176, 35238, 35159, 35178, 35179, 35180, 35181, 35182, 35183, + 35184, 35185, 35186, 35187, 35188, 35189, 35190, 35191, 35192, 35193, + 35194, 35195, 35196, 35197, 35198, 35199, 35200, 35201, 35202, 35203, + 35233, 35242, 35241, 35158, 35234, 35175, 35204, 35205, 35206, 35207, + 35208, 35209, 35210, 35211, 35212, 35213, 35214, 35215, 35216, 35217, + 35218, 35219, 35220, 35221, 35222, 35223, 35224, 35225, 35226, 35227, + 35228, 35229, 35231, 35249, 35239, 35247, 6097, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 38793, 38804, 38815, 38827, 38838, + 38849, 38860, 38871, 38882, 38890, 38891, 38892, 38893, 38895, 38896, + 38897, 38898, 38899, 38900, 38901, 38902, 38903, 38904, 38906, 38907, + 38908, 38909, 38910, 38911, 38912, 38913, 38914, 38915, 38917, 38918, + 38919, 38920, 38921, 38922, 38923, 38924, 38925, 38926, 38928, 38929, + 38930, 38931, 38932, 38933, 38934, 38935, 38936, 38937, 38939, 38940, + 38941, 38942, 38943, 38944, 38945, 38946, 38947, 38948, 38950, 38951, + 38952, 38953, 38954, 38955, 38956, 38957, 38958, 38959, 38961, 38962, + 38963, 38964, 38965, 38966, 38967, 38968, 38969, 38970, 38717, 38718, + 38719, 38720, 38721, 38722, 38723, 38724, 38725, 38726, 38728, 38729, + 38730, 38731, 38732, 38733, 38734, 38735, 38736, 38737, 38739, 38740, + 38741, 38742, 38743, 38744, 38745, 38746, 38747, 38748, 38750, 38751, + 38752, 38753, 38754, 38755, 38756, 38757, 38758, 38759, 38761, 38762, + 38763, 38764, 38765, 38766, 38767, 38768, 38769, 38770, 38772, 38773, + 38774, 38775, 38776, 38777, 38778, 38779, 38780, 38781, 38783, 38784, + 38785, 38786, 38787, 38788, 38789, 38790, 38791, 38792, 38794, 38795, + 38796, 38797, 38798, 38799, 38800, 38801, 38802, 38803, 38805, 38806, + 38807, 38808, 38809, 38810, 38811, 38812, 38813, 38814, 38816, 38817, + 38818, 38819, 38820, 38821, 38822, 38823, 38824, 38825, 38828, 38829, + 38830, 38831, 38832, 38833, 38834, 38835, 38836, 38837, 38839, 38840, + 38841, 38842, 38843, 38844, 38845, 38846, 38847, 38848, 38850, 38851, + 38852, 38853, 38854, 38855, 38856, 38857, 38858, 38859, 38861, 38862, + 38863, 38864, 38865, 38866, 38867, 38868, 38869, 38870, 38872, 38873, + 38874, 38875, 38876, 38877, 38878, 38879, 38880, 38881, 38883, 38884, + 38885, 38886, 38887, 38888, 38889, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 30035, 30034, 34722, 34301, 34723, 34754, 16908, 17482, 16906, + 16919, 16912, 16911, 3, 2, 349, 3591, 2657, 5360, 6392, 20578, 20608, + 35153, 24664, 29357, 16909, 25513, 30109, 16917, 24666, 39304, 39411, + 17638, 17788, 6165, 8810, 33016, 25280, 34098, 33017, 25279, 33050, + 10675, 11663, 10960, 10676, 10959, 10677, 10958, 10678, 10956, 10679, + 29225, 29143, 35058, 35057, 16907, 17481, 6095, 5368, 16905, 16918, + 16872, 34785, 34755, 16954, 16953, 20866, 17579, 17786, 20867, 18827, + 19133, 20868, 31975, 32554, 20869, 38208, 38414, 34303, 10691, 10688, + 31070, 31069, 20445, 20607, 5029, 5359, 29530, 29145, 20793, 20792, + 29459, 29464, 34719, 34707, 16904, 16957, 6394, 20580, 20610, 6393, + 20579, 20609, 24667, 39305, 39412, 31393, 31392, 31772, 31394, 31395, + 31752, 32055, 32062, 32090, 33862, 33863, 34705, 33861, 33864, 34706, + 10957, 10680, 31861, 31862, 31910, 31860, 31863, 31911, 32849, 34753, + 6096, 10660, 27588, 28969, 34718, 34721, 34304, 16903, 16901, 17520, + 34720, 34302, 33856, 35150, 33855, 32744, 8604, 10659, 34744, 34708, + 30727, 30949, 31859, 31912, 1056, 1063, 29144, 33049, 23060, 23683, + 10658, 2354, 363, 35136, 21392, 22735, 22736, 22778, 22744, 37497, 19533, + 19534, 19511, 19532, 17782, 17783, 17784, 28967, 17785, 34810, 41410, + 41409, 41411, 25509, 32357, 25507, 32355, 31468, 25510, 32358, 30108, + 28968, 39836, 25508, 32356, 17787, 31469, 39603, 27750, 27751, 24416, + 32313, 40346, 28755, 38972, 39083, 39151, 39162, 39173, 39184, 39195, + 39206, 39217, 38973, 38984, 38995, 39006, 39017, 39028, 39039, 31828, + 5358, 4651, 41408, 9774, 9773, 9469, 2872, 2982, 2973, 3080, 3281, 27045, + 27043, 27103, 27101, 20358, 5195, 27423, 27426, 39050, 39061, 39072, + 39084, 39095, 39106, 39117, 39128, 39139, 39147, 39148, 39149, 39150, + 39152, 39153, 39154, 39155, 39156, 39157, 39158, 39159, 39160, 39161, + 39163, 39164, 39165, 39166, 39167, 39168, 39169, 39170, 39171, 39172, + 39174, 39175, 39176, 39177, 39178, 39179, 39180, 39181, 39182, 39183, + 39185, 39186, 39187, 39188, 39189, 39190, 39191, 39192, 39193, 39194, + 39196, 39197, 39198, 39199, 39200, 39201, 39202, 39203, 39204, 39205, + 39207, 39208, 39209, 39210, 39211, 39212, 39213, 39214, 39215, 39216, + 39218, 39219, 39220, 39221, 39222, 39223, 39224, 39225, 39226, 39227, + 38974, 38975, 38976, 38977, 38978, 38979, 38980, 38981, 38982, 38983, + 38985, 38986, 38987, 38988, 38989, 38990, 38991, 38992, 38993, 38994, + 38996, 38997, 38998, 38999, 39000, 39001, 39002, 39003, 39004, 39005, + 39007, 39008, 39009, 39010, 39011, 39012, 39013, 39014, 39015, 39016, + 39018, 39019, 39020, 39021, 39022, 39023, 39024, 39025, 39026, 39027, + 39029, 39030, 39031, 39032, 39033, 39034, 39035, 39036, 39037, 39038, + 39040, 39041, 39042, 39043, 39044, 39045, 39046, 39047, 39048, 39049, + 39051, 39052, 39053, 39054, 39055, 39056, 39057, 39058, 39059, 39060, + 39062, 39063, 39064, 39065, 39066, 39067, 39068, 39069, 39070, 39071, + 39073, 39074, 39075, 39076, 39077, 39078, 39079, 39080, 39081, 39082, + 39085, 39086, 39087, 39088, 39089, 39090, 39091, 39092, 39093, 39094, + 39096, 39097, 39098, 39099, 39100, 39101, 39102, 39103, 39104, 39105, + 39107, 39108, 39109, 39110, 39111, 39112, 39113, 39114, 39115, 39116, + 39118, 39119, 39120, 39121, 39122, 39123, 39124, 39125, 39126, 39127, + 39129, 39130, 39131, 39132, 39133, 39134, 39135, 39136, 39137, 39138, + 39140, 39141, 39142, 39143, 39144, 39145, 39146, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 21777, 21778, + 21785, 21787, 21784, 21783, 21780, 21779, 21782, 21781, 21788, 21786, + 22923, 23489, 23084, 23714, 23324, 24088, 23019, 23624, 23020, 23625, + 23021, 23626, 23195, 23865, 23196, 23866, 23197, 23867, 23263, 23964, + 23003, 23607, 22999, 23603, 23709, 23833, 22933, 23499, 22934, 23500, + 23024, 23630, 23025, 23631, 23009, 23613, 23010, 23614, 23708, 23710, + 23089, 23716, 23090, 23717, 23107, 23744, 23137, 23784, 23145, 23806, + 23235, 23924, 23328, 24092, 23329, 24093, 23325, 24089, 23326, 24090, + 23508, 23882, 23883, 24043, 24044, 23973, 23974, 23698, 23699, 2324, + 2327, 2325, 2329, 2331, 2328, 2330, 2326, 2332, 10889, 10884, 10883, + 10890, 10885, 10886, 10888, 10887, 3664, 3663, 3665, 19041, 19040, 19039, + 19038, 19043, 19042, 30801, 30800, 3609, 35654, 35662, 35671, 35663, + 35665, 35660, 35664, 35659, 35675, 35674, 35677, 35669, 35656, 35676, + 35658, 35657, 35670, 35661, 35673, 35667, 35668, 35666, 35672, 35655, + 35793, 35800, 35801, 35796, 35797, 35802, 35803, 35794, 35798, 35799, + 35795, 35859, 35866, 35867, 35862, 35863, 35868, 35869, 35860, 35864, + 35865, 35861, 35970, 35977, 35978, 35973, 35974, 35979, 35980, 35971, + 35975, 35976, 35972, 35870, 35877, 35878, 35873, 35874, 35879, 35880, + 35871, 35875, 35876, 35872, 35948, 35955, 35956, 35951, 35952, 35957, + 35958, 35949, 35953, 35954, 35950, 35848, 35855, 35856, 35851, 35852, + 35857, 35858, 35849, 35853, 35854, 35850, 35959, 35966, 35967, 35962, + 35963, 35968, 35969, 35960, 35964, 35965, 35961, 35881, 35888, 35889, + 35884, 35885, 35890, 35891, 35882, 35886, 35887, 35883, 36014, 36021, + 36022, 36017, 36018, 36023, 36024, 36015, 36019, 36020, 36016, 36003, + 36010, 36011, 36006, 36007, 36012, 36013, 36004, 36008, 36009, 36005, + 36036, 36043, 36044, 36039, 36040, 36045, 36046, 36037, 36041, 36042, + 36038, 35903, 35910, 35911, 35906, 35907, 35912, 35913, 35904, 35908, + 35909, 35905, 35826, 35833, 35834, 35829, 35830, 35835, 35836, 35827, + 35831, 35832, 35828, 36025, 36032, 36033, 36028, 36029, 36034, 36035, + 36026, 36030, 36031, 36027, 35804, 35811, 35812, 35807, 35808, 35813, + 35814, 35805, 35809, 35810, 35806, 35815, 35822, 35823, 35818, 35819, + 35824, 35825, 35816, 35820, 35821, 35817, 35892, 35899, 35900, 35895, + 35896, 35901, 35902, 35893, 35897, 35898, 35894, 35837, 35844, 35845, + 35840, 35841, 35846, 35847, 35838, 35842, 35843, 35839, 35992, 35999, + 36000, 35995, 35996, 36001, 36002, 35993, 35997, 35998, 35994, 35914, + 35922, 35923, 35917, 35918, 35924, 35925, 35915, 35919, 35920, 35916, + 35926, 35933, 35934, 35929, 35930, 35935, 35936, 35927, 35931, 35932, + 35928, 35937, 35944, 35945, 35940, 35941, 35946, 35947, 35938, 35942, + 35943, 35939, 35981, 35988, 35989, 35984, 35985, 35990, 35991, 35982, + 35986, 35987, 35983, 35781, 35782, 35789, 35790, 35785, 35786, 35791, + 35792, 35783, 35787, 35788, 35784, 35921, 33887, 33885, 33886, 17964, + 22342, 22340, 22343, 22341, 22332, 22338, 22336, 22339, 22337, 22333, + 22353, 22349, 22354, 22350, 22334, 22351, 22347, 22352, 22348, 22335, + 22364, 22344, 22346, 22345, 22360, 22363, 22361, 22356, 22362, 22357, + 22358, 22359, 22365, 22355, 22504, 22381, 22382, 22383, 22380, 22499, + 22491, 20475, 20477, 20479, 20476, 20478, 21482, 21484, 21486, 21483, + 21485, 21475, 21474, 21473, 21476, 27960, 27965, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, + 41412, 41412, 41412, 41412, 41412, 41412, 41412, 41412, }; static const unsigned int aliases_start = 0xf0000; -static const unsigned int aliases_end = 0xf01dd; +static const unsigned int aliases_end = 0xf01e1; static const unsigned int name_aliases[] = { 0x0000, 0x0000, @@ -18753,6 +18960,10 @@ static const unsigned int name_aliases[] = { 0x122D5, 0x12327, 0x1680B, + 0x16881, + 0x1688E, + 0x168DC, + 0x1697D, 0x16E56, 0x16E57, 0x16E76, diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 26ac35734fb..09c8d9487f5 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -14,7 +14,9 @@ This module roughly corresponds to:: class Xxo: - """A class that explicitly stores attributes in an internal dict""" + """A class that explicitly stores attributes in an internal dict + (to simulate custom attribute handling). + """ def __init__(self): # In the C class, "_x_attr" is not accessible from Python code @@ -85,13 +87,16 @@ typedef struct { // Instance state typedef struct { PyObject_HEAD - PyObject *x_attr; /* Attributes dictionary */ + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ Py_ssize_t x_exports; /* how many buffer are exported */ } XxoObject; #define XxoObject_CAST(op) ((XxoObject *)(op)) -// XXX: no good way to do this yet +// TODO: full support for type-checking was added in 3.14 (Py_tp_token) // #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) static XxoObject * @@ -112,8 +117,13 @@ newXxoObject(PyObject *module) return self; } -/* Xxo finalization */ +/* Xxo finalization. + * + * Types that store references to other PyObjects generally need to implement + * the GC slots: traverse, clear, dealloc, and (optionally) finalize. + */ +// traverse: Visit all references from an object, including its type static int Xxo_traverse(PyObject *op, visitproc visit, void *arg) { @@ -126,6 +136,7 @@ Xxo_traverse(PyObject *op, visitproc visit, void *arg) return 0; } +// clear: drop references in order to break all reference cycles static int Xxo_clear(PyObject *op) { @@ -134,6 +145,8 @@ Xxo_clear(PyObject *op) return 0; } +// finalize: like clear, but should leave the object in a consistent state. +// Equivalent to `__del__` in Python. static void Xxo_finalize(PyObject *op) { @@ -141,6 +154,7 @@ Xxo_finalize(PyObject *op) Py_CLEAR(self->x_attr); } +// dealloc: drop all remaining references and free memory static void Xxo_dealloc(PyObject *self) { @@ -155,6 +169,7 @@ Xxo_dealloc(PyObject *self) /* Xxo attribute handling */ +// Get an attribute. static PyObject * Xxo_getattro(PyObject *op, PyObject *name) { @@ -168,9 +183,12 @@ Xxo_getattro(PyObject *op, PyObject *name) return NULL; } } + // Fall back to generic implementation (this handles special attributes, + // raising AttributeError, etc.) return PyObject_GenericGetAttr(op, name); } +// Set or delete an attribute. static int Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) { @@ -198,7 +216,9 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) } } -/* Xxo methods */ +/* Xxo methods: C functions plus a PyMethodDef array that lists them and + * specifies metadata. + */ static PyObject * Xxo_demo(PyObject *op, PyTypeObject *defining_class, @@ -234,7 +254,10 @@ static PyMethodDef Xxo_methods[] = { {NULL, NULL} /* sentinel */ }; -/* Xxo buffer interface */ +/* Xxo buffer interface: C functions later referenced from PyType_Slot array. + * Other interfaces (e.g. for sequence-like or number-like types) are defined + * similarly. + */ static int Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) @@ -300,6 +323,7 @@ static PyType_Spec Xxo_Type_spec = { /* Str type definition*/ static PyType_Slot Str_Type_slots[] = { + // slots array intentionally kept empty {0, 0}, /* sentinel */ }; @@ -400,12 +424,32 @@ xx_modexec(PyObject *m) } static PyModuleDef_Slot xx_slots[] = { + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ {Py_mod_exec, xx_modexec}, + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} }; +// Module finalization: modules that hold references in their module state +// need to implement the fullowing GC hooks. They're similar to the ones for +// types (see "Xxo finalization"). + static int xx_traverse(PyObject *module, visitproc visit, void *arg) { @@ -424,6 +468,13 @@ xx_clear(PyObject *module) return 0; } +static void +xx_free(void *module) +{ + // allow xx_modexec to omit calling xx_clear on error + (void)xx_clear((PyObject *)module); +} + static struct PyModuleDef xxmodule = { PyModuleDef_HEAD_INIT, .m_name = "xxlimited", @@ -433,13 +484,13 @@ static struct PyModuleDef xxmodule = { .m_slots = xx_slots, .m_traverse = xx_traverse, .m_clear = xx_clear, - /* m_free is not necessary here: xx_clear clears all references, - * and the module state is deallocated along with the module. - */ + .m_free = xx_free, }; -/* Export function for the module (*must* be called PyInit_xx) */ +/* Export function for the module. *Must* be called PyInit_xx; usually it is + * the only non-`static` object in a module definition. + */ PyMODINIT_FUNC PyInit_xxlimited(void) diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index d4b4b91697c..6bac09aa6c2 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -17,6 +17,16 @@ #error "At least zlib version 1.2.2.1 is required" #endif +#if (SIZEOF_OFF_T == SIZEOF_SIZE_T) +# define convert_to_z_off_t PyLong_AsSsize_t +#elif (SIZEOF_OFF_T == SIZEOF_LONG_LONG) +# define convert_to_z_off_t PyLong_AsLongLong +#elif (SIZEOF_OFF_T == SIZEOF_LONG) +# define convert_to_z_off_t PyLong_AsLong +#else +# error off_t does not match either size_t, long, or long long! +#endif + // Blocks output buffer wrappers #include "pycore_blocks_output_buffer.h" @@ -263,7 +273,9 @@ static compobject * newcompobject(PyTypeObject *type) { compobject *self; - self = PyObject_New(compobject, type); + assert(type != NULL); + assert(type->tp_alloc != NULL); + self = _compobject_CAST(type->tp_alloc(type, 0)); if (self == NULL) return NULL; self->eof = 0; @@ -332,7 +344,7 @@ zlib_compress_impl(PyObject *module, Py_buffer *data, int level, int wbits) PyObject *return_value; int flush; z_stream zst; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; zlibstate *state = get_zlib_state(module); @@ -417,7 +429,7 @@ zlib.decompress / wbits: int(c_default="MAX_WBITS") = MAX_WBITS The window buffer size and container format. - bufsize: Py_ssize_t(c_default="DEF_BUF_SIZE") = DEF_BUF_SIZE + bufsize: Py_ssize_t(c_default="DEF_BUF_SIZE", allow_negative=False) = DEF_BUF_SIZE The initial output buffer size. Returns a bytes object containing the uncompressed data. @@ -426,22 +438,19 @@ Returns a bytes object containing the uncompressed data. static PyObject * zlib_decompress_impl(PyObject *module, Py_buffer *data, int wbits, Py_ssize_t bufsize) -/*[clinic end generated code: output=77c7e35111dc8c42 input=a9ac17beff1f893f]*/ +/*[clinic end generated code: output=77c7e35111dc8c42 input=530077065b3a2233]*/ { PyObject *return_value; Byte *ibuf; Py_ssize_t ibuflen; int err, flush; z_stream zst; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; _Uint32Window window; // output buffer's UINT32_MAX sliding window zlibstate *state = get_zlib_state(module); - if (bufsize < 0) { - PyErr_SetString(PyExc_ValueError, "bufsize must be non-negative"); - return NULL; - } else if (bufsize == 0) { + if (bufsize == 0) { bufsize = 1; } @@ -706,33 +715,41 @@ zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) } static void -Dealloc(compobject *self) +compobject_dealloc_impl(PyObject *op, int (*dealloc)(z_streamp)) { - PyTypeObject *type = Py_TYPE(self); + PyTypeObject *type = Py_TYPE(op); + PyObject_GC_UnTrack(op); + compobject *self = _compobject_CAST(op); + if (self->is_initialised) { + (void)dealloc(&self->zst); + } PyThread_free_lock(self->lock); Py_XDECREF(self->unused_data); Py_XDECREF(self->unconsumed_tail); Py_XDECREF(self->zdict); - PyObject_Free(self); + type->tp_free(self); Py_DECREF(type); } +static int +compobject_traverse(PyObject *op, visitproc visit, void *arg) +{ + compobject *self = _compobject_CAST(op); + Py_VISIT(Py_TYPE(op)); + Py_VISIT(self->zdict); + return 0; +} + static void Comp_dealloc(PyObject *op) { - compobject *self = _compobject_CAST(op); - if (self->is_initialised) - (void)deflateEnd(&self->zst); - Dealloc(self); + compobject_dealloc_impl(op, &deflateEnd); } static void Decomp_dealloc(PyObject *op) { - compobject *self = _compobject_CAST(op); - if (self->is_initialised) - (void)inflateEnd(&self->zst); - Dealloc(self); + compobject_dealloc_impl(op, &inflateEnd); } /*[clinic input] @@ -757,7 +774,7 @@ zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, { PyObject *return_value; int err; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; zlibstate *state = PyType_GetModuleState(cls); ENTER_ZLIB(self); @@ -816,22 +833,24 @@ save_unconsumed_input(compobject *self, Py_buffer *data, int err) input data in self->unused_data. */ if (self->zst.avail_in > 0) { Py_ssize_t old_size = PyBytes_GET_SIZE(self->unused_data); - Py_ssize_t new_size, left_size; - PyObject *new_data; + Py_ssize_t left_size; left_size = (Byte *)data->buf + data->len - self->zst.next_in; if (left_size > (PY_SSIZE_T_MAX - old_size)) { PyErr_NoMemory(); return -1; } - new_size = old_size + left_size; - new_data = PyBytes_FromStringAndSize(NULL, new_size); - if (new_data == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(old_size + left_size); + if (writer == NULL) { return -1; - memcpy(PyBytes_AS_STRING(new_data), - PyBytes_AS_STRING(self->unused_data), old_size); - memcpy(PyBytes_AS_STRING(new_data) + old_size, - self->zst.next_in, left_size); - Py_SETREF(self->unused_data, new_data); + } + char *new_data = PyBytesWriter_GetData(writer); + memcpy(new_data, PyBytes_AS_STRING(self->unused_data), old_size); + memcpy(new_data + old_size, self->zst.next_in, left_size); + PyObject *new_unused_data = PyBytesWriter_Finish(writer); + if (new_unused_data == NULL) { + return -1; + } + Py_SETREF(self->unused_data, new_unused_data); self->zst.avail_in = 0; } } @@ -852,13 +871,14 @@ save_unconsumed_input(compobject *self, Py_buffer *data, int err) } /*[clinic input] +@permit_long_docstring_body zlib.Decompress.decompress cls: defining_class data: Py_buffer The binary data to decompress. / - max_length: Py_ssize_t = 0 + max_length: Py_ssize_t(allow_negative=False) = 0 The maximum allowable length of the decompressed data. Unconsumed input data will be stored in the unconsumed_tail attribute. @@ -873,22 +893,19 @@ Call the flush() method to clear these buffers. static PyObject * zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=b024a93c2c922d57 input=bfb37b3864cfb606]*/ +/*[clinic end generated code: output=b024a93c2c922d57 input=77de124bd2a2ecc0]*/ { int err = Z_OK; Py_ssize_t ibuflen; PyObject *return_value; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; PyObject *module = PyType_GetModule(cls); if (module == NULL) return NULL; zlibstate *state = get_zlib_state(module); - if (max_length < 0) { - PyErr_SetString(PyExc_ValueError, "max_length must be non-negative"); - return NULL; - } else if (max_length == 0) { + if (max_length == 0) { max_length = -1; } @@ -988,13 +1005,13 @@ zlib_Compress_flush_impl(compobject *self, PyTypeObject *cls, int mode) { int err; PyObject *return_value; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; zlibstate *state = PyType_GetModuleState(cls); /* Flushing with Z_NO_FLUSH is a no-op, so there's no point in doing any work at all; just return an empty string. */ if (mode == Z_NO_FLUSH) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } ENTER_ZLIB(self); @@ -1250,7 +1267,7 @@ zlib_Decompress_flush_impl(compobject *self, PyTypeObject *cls, Py_buffer data; PyObject *return_value; Py_ssize_t ibuflen; - _BlocksOutputBuffer buffer = {.list = NULL}; + _BlocksOutputBuffer buffer = {.writer = NULL}; _Uint32Window window; // output buffer's UINT32_MAX sliding window PyObject *module = PyType_GetModule(cls); @@ -1357,16 +1374,19 @@ typedef struct { char needs_input; } ZlibDecompressor; +#define ZlibDecompressor_CAST(op) ((ZlibDecompressor *)(op)) + /*[clinic input] -class zlib.ZlibDecompressor "ZlibDecompressor *" "&ZlibDecompressorType" +class zlib._ZlibDecompressor "ZlibDecompressor *" "&ZlibDecompressorType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=0658178ab94645df]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=49151d1d703e6bcc]*/ static void ZlibDecompressor_dealloc(PyObject *op) { - ZlibDecompressor *self = (ZlibDecompressor*)op; - PyObject *type = (PyObject *)Py_TYPE(self); + PyTypeObject *type = Py_TYPE(op); + PyObject_GC_UnTrack(op); + ZlibDecompressor *self = ZlibDecompressor_CAST(op); PyThread_free_lock(self->lock); if (self->is_initialised) { inflateEnd(&self->zst); @@ -1374,10 +1394,19 @@ ZlibDecompressor_dealloc(PyObject *op) PyMem_Free(self->input_buffer); Py_CLEAR(self->unused_data); Py_CLEAR(self->zdict); - PyObject_Free(self); + type->tp_free(self); Py_DECREF(type); } +static int +ZlibDecompressor_traverse(PyObject *op, visitproc visit, void *arg) +{ + ZlibDecompressor *self = ZlibDecompressor_CAST(op); + Py_VISIT(Py_TYPE(op)); + Py_VISIT(self->zdict); + return 0; +} + static int set_inflate_zdict_ZlibDecompressor(zlibstate *state, ZlibDecompressor *self) { @@ -1658,7 +1687,8 @@ decompress(ZlibDecompressor *self, uint8_t *data, } /*[clinic input] -zlib.ZlibDecompressor.decompress +@permit_long_docstring_body +zlib._ZlibDecompressor.decompress data: Py_buffer max_length: Py_ssize_t=-1 @@ -1680,9 +1710,10 @@ the unused_data attribute. [clinic start generated code]*/ static PyObject * -zlib_ZlibDecompressor_decompress_impl(ZlibDecompressor *self, - Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=990d32787b775f85 input=0b29d99715250b96]*/ +zlib__ZlibDecompressor_decompress_impl(ZlibDecompressor *self, + Py_buffer *data, + Py_ssize_t max_length) +/*[clinic end generated code: output=ac00dcf73e843e99 input=c9278e791be1152b]*/ { PyObject *result = NULL; @@ -1698,38 +1729,29 @@ zlib_ZlibDecompressor_decompress_impl(ZlibDecompressor *self, return result; } -PyDoc_STRVAR(ZlibDecompressor__new____doc__, -"_ZlibDecompressor(wbits=15, zdict=b\'\')\n" -"--\n" -"\n" -"Create a decompressor object for decompressing data incrementally.\n" -"\n" -" wbits = 15\n" -" zdict\n" -" The predefined compression dictionary. This is a sequence of bytes\n" -" (such as a bytes object) containing subsequences that are expected\n" -" to occur frequently in the data that is to be compressed. Those\n" -" subsequences that are expected to be most common should come at the\n" -" end of the dictionary. This must be the same dictionary as used by the\n" -" compressor that produced the input data.\n" -"\n"); +/*[clinic input] +@classmethod +zlib._ZlibDecompressor.__new__ + + wbits: int(c_default='MAX_WBITS') = MAX_WBITS + zdict: object(c_default='NULL') = b'' + The predefined compression dictionary. This is a sequence of bytes + (such as a bytes object) containing subsequences that are expected + to occur frequently in the data that is to be compressed. Those + subsequences that are expected to be most common should come at the + end of the dictionary. This must be the same dictionary as used by the + compressor that produced the input data. + +Create a decompressor object for decompressing data incrementally. +[clinic start generated code]*/ static PyObject * -ZlibDecompressor__new__(PyTypeObject *cls, - PyObject *args, - PyObject *kwargs) +zlib__ZlibDecompressor_impl(PyTypeObject *type, int wbits, PyObject *zdict) +/*[clinic end generated code: output=1065607df0d33baa input=9ebad0be6de226e2]*/ { - static char *keywords[] = {"wbits", "zdict", NULL}; - static const char * const format = "|iO:_ZlibDecompressor"; - int wbits = MAX_WBITS; - PyObject *zdict = NULL; - zlibstate *state = PyType_GetModuleState(cls); - - if (!PyArg_ParseTupleAndKeywords( - args, kwargs, format, keywords, &wbits, &zdict)) { - return NULL; - } - ZlibDecompressor *self = PyObject_New(ZlibDecompressor, cls); + assert(type != NULL && type->tp_alloc != NULL); + zlibstate *state = PyType_GetModuleState(type); + ZlibDecompressor *self = ZlibDecompressor_CAST(type->tp_alloc(type, 0)); if (self == NULL) { return NULL; } @@ -1744,11 +1766,7 @@ ZlibDecompressor__new__(PyTypeObject *cls, self->zst.zfree = PyZlib_Free; self->zst.next_in = NULL; self->zst.avail_in = 0; - self->unused_data = PyBytes_FromStringAndSize(NULL, 0); - if (self->unused_data == NULL) { - Py_CLEAR(self); - return NULL; - } + self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); self->lock = PyThread_allocate_lock(); if (self->lock == NULL) { Py_DECREF(self); @@ -1805,7 +1823,7 @@ static PyMethodDef Decomp_methods[] = }; static PyMethodDef ZlibDecompressor_methods[] = { - ZLIB_ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF + ZLIB__ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF {NULL} }; @@ -1876,6 +1894,44 @@ zlib_adler32_impl(PyObject *module, Py_buffer *data, unsigned int value) return PyLong_FromUnsignedLong(value & 0xffffffffU); } +/*[clinic input] +zlib.adler32_combine -> unsigned_int + + adler1: unsigned_int(bitwise=True) + Adler-32 checksum for sequence A + + adler2: unsigned_int(bitwise=True) + Adler-32 checksum for sequence B + + len2: object(subclass_of='&PyLong_Type') + Length of sequence B + / + +Combine two Adler-32 checksums into one. + +Given the Adler-32 checksum 'adler1' of a sequence A and the +Adler-32 checksum 'adler2' of a sequence B of length 'len2', +return the Adler-32 checksum of A and B concatenated. +[clinic start generated code]*/ + +static unsigned int +zlib_adler32_combine_impl(PyObject *module, unsigned int adler1, + unsigned int adler2, PyObject *len2) +/*[clinic end generated code: output=61842cefb16afb1b input=51bb045c95130c6f]*/ +{ +#if defined(Z_WANT64) + z_off64_t len = convert_to_z_off_t(len2); +#else + z_off_t len = convert_to_z_off_t(len2); +#endif + if (PyErr_Occurred()) { + return (unsigned int)-1; + } + return adler32_combine(adler1, adler2, len); +} + + + /*[clinic input] zlib.crc32 -> unsigned_int @@ -1923,20 +1979,80 @@ zlib_crc32_impl(PyObject *module, Py_buffer *data, unsigned int value) return value; } +/*[clinic input] +zlib.crc32_combine -> unsigned_int + + crc1: unsigned_int(bitwise=True) + CRC-32 checksum for sequence A + + crc2: unsigned_int(bitwise=True) + CRC-32 checksum for sequence B + + len2: object(subclass_of='&PyLong_Type') + Length of sequence B + / + +Combine two CRC-32 checksums into one. + +Given the CRC-32 checksum 'crc1' of a sequence A and the +CRC-32 checksum 'crc2' of a sequence B of length 'len2', +return the CRC-32 checksum of A and B concatenated. +[clinic start generated code]*/ + +static unsigned int +zlib_crc32_combine_impl(PyObject *module, unsigned int crc1, + unsigned int crc2, PyObject *len2) +/*[clinic end generated code: output=c4def907c602e6eb input=9c8a065d9040dc66]*/ +{ +#if defined(Z_WANT64) + z_off64_t len = convert_to_z_off_t(len2); +#else + z_off_t len = convert_to_z_off_t(len2); +#endif + if (PyErr_Occurred()) { + return (unsigned int)-1; + } + return crc32_combine(crc1, crc2, len); +} + +static PyObject * +zlib_getattr(PyObject *self, PyObject *args) +{ + PyObject *name; + if (!PyArg_UnpackTuple(args, "__getattr__", 1, 1, &name)) { + return NULL; + } + + if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "__version__")) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + 1) < 0) { + return NULL; + } + return PyUnicode_FromString("1.0"); + } + + PyErr_Format(PyExc_AttributeError, "module 'zlib' has no attribute %R", name); + return NULL; +} static PyMethodDef zlib_methods[] = { ZLIB_ADLER32_METHODDEF + ZLIB_ADLER32_COMBINE_METHODDEF ZLIB_COMPRESS_METHODDEF ZLIB_COMPRESSOBJ_METHODDEF ZLIB_CRC32_METHODDEF + ZLIB_CRC32_COMBINE_METHODDEF ZLIB_DECOMPRESS_METHODDEF ZLIB_DECOMPRESSOBJ_METHODDEF + {"__getattr__", zlib_getattr, METH_VARARGS, "Module __getattr__"}, {NULL, NULL} }; static PyType_Slot Comptype_slots[] = { {Py_tp_dealloc, Comp_dealloc}, + {Py_tp_traverse, compobject_traverse}, {Py_tp_methods, comp_methods}, {0, 0}, }; @@ -1944,12 +2060,18 @@ static PyType_Slot Comptype_slots[] = { static PyType_Spec Comptype_spec = { .name = "zlib.Compress", .basicsize = sizeof(compobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC + ), .slots= Comptype_slots, }; static PyType_Slot Decomptype_slots[] = { {Py_tp_dealloc, Decomp_dealloc}, + {Py_tp_traverse, compobject_traverse}, {Py_tp_methods, Decomp_methods}, {Py_tp_members, Decomp_members}, {0, 0}, @@ -1958,15 +2080,21 @@ static PyType_Slot Decomptype_slots[] = { static PyType_Spec Decomptype_spec = { .name = "zlib.Decompress", .basicsize = sizeof(compobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC + ), .slots = Decomptype_slots, }; static PyType_Slot ZlibDecompressor_type_slots[] = { {Py_tp_dealloc, ZlibDecompressor_dealloc}, + {Py_tp_traverse, ZlibDecompressor_traverse}, {Py_tp_members, ZlibDecompressor_members}, - {Py_tp_new, ZlibDecompressor__new__}, - {Py_tp_doc, (char *)ZlibDecompressor__new____doc__}, + {Py_tp_new, zlib__ZlibDecompressor}, + {Py_tp_doc, (char *)zlib__ZlibDecompressor__doc__}, {Py_tp_methods, ZlibDecompressor_methods}, {0, 0}, }; @@ -1978,17 +2106,24 @@ static PyType_Spec ZlibDecompressor_type_spec = { // ZlibDecompressor_type_spec does not have Py_TPFLAGS_BASETYPE flag // which prevents to create a subclass. // So calling PyType_GetModuleState() in this file is always safe. - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC + ), .slots = ZlibDecompressor_type_slots, }; + PyDoc_STRVAR(zlib_module_documentation, "The functions in this module allow compression and decompression using the\n" "zlib library, which is based on GNU zip.\n" "\n" "adler32(string[, start]) -- Compute an Adler-32 checksum.\n" +"adler32_combine(adler1, adler2, len2, /) -- Combine two Adler-32 checksums.\n" "compress(data[, level]) -- Compress data, with compression level 0-9 or -1.\n" "compressobj([level[, ...]]) -- Return a compressor object.\n" "crc32(string[, start]) -- Compute a CRC-32 checksum.\n" +"crc32_combine(crc1, crc2, len2, /) -- Combine two CRC-32 checksums.\n" "decompress(string,[wbits],[bufsize]) -- Decompresses a compressed string.\n" "decompressobj([wbits[, zdict]]) -- Return a decompressor object.\n" "\n" @@ -2108,9 +2243,6 @@ zlib_exec(PyObject *mod) return -1; } #endif - if (PyModule_AddStringConstant(mod, "__version__", "1.0") < 0) { - return -1; - } return 0; } diff --git a/Objects/abstract.c b/Objects/abstract.c index df96b935ecc..8adad8407d0 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -132,8 +132,9 @@ PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) return defaultvalue; } if (!PyLong_Check(result)) { - PyErr_Format(PyExc_TypeError, "__length_hint__ must be an integer, not %.100s", - Py_TYPE(result)->tp_name); + PyErr_Format(PyExc_TypeError, + "%T.__length_hint__() must return an int, not %T", + o, result); Py_DECREF(result); return -1; } @@ -143,7 +144,8 @@ PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) return -1; } if (res < 0) { - PyErr_Format(PyExc_ValueError, "__length_hint__() should return >= 0"); + PyErr_Format(PyExc_ValueError, + "%T.__length_hint__() must return a non-negative int", o); return -1; } return res; @@ -887,8 +889,8 @@ PyObject_Format(PyObject *obj, PyObject *format_spec) if (result && !PyUnicode_Check(result)) { PyErr_Format(PyExc_TypeError, - "__format__ must return a str, not %.200s", - Py_TYPE(result)->tp_name); + "%T.__format__() must return a str, not %T", + obj, result); Py_SETREF(result, NULL); goto done; } @@ -1421,17 +1423,17 @@ _PyNumber_Index(PyObject *item) if (!PyLong_Check(result)) { PyErr_Format(PyExc_TypeError, - "__index__ returned non-int (type %.200s)", - Py_TYPE(result)->tp_name); + "%T.__index__() must return an int, not %T", + item, result); Py_DECREF(result); return NULL; } /* Issue #17576: warn if 'result' not of exact type int. */ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "__index__ returned non-int (type %.200s). " + "%T.__index__() must return an int, not %T. " "The ability to return an instance of a strict subclass of int " "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(result)->tp_name)) { + item, result)) { Py_DECREF(result); return NULL; } @@ -1531,17 +1533,17 @@ PyNumber_Long(PyObject *o) if (!PyLong_Check(result)) { PyErr_Format(PyExc_TypeError, - "__int__ returned non-int (type %.200s)", - Py_TYPE(result)->tp_name); + "%T.__int__() must return an int, not %T", + o, result); Py_DECREF(result); return NULL; } /* Issue #17576: warn if 'result' not of exact type int. */ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "__int__ returned non-int (type %.200s). " + "%T.__int__() must return an int, not %T. " "The ability to return an instance of a strict subclass of int " "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(result)->tp_name)) { + o, result)) { Py_DECREF(result); return NULL; } @@ -1609,17 +1611,16 @@ PyNumber_Float(PyObject *o) if (!PyFloat_Check(res)) { PyErr_Format(PyExc_TypeError, - "%.50s.__float__ returned non-float (type %.50s)", - Py_TYPE(o)->tp_name, Py_TYPE(res)->tp_name); + "%T.__float__() must return a float, not %T", o, res); Py_DECREF(res); return NULL; } /* Issue #26983: warn if 'res' not of exact type float. */ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "%.50s.__float__ returned non-float (type %.50s). " + "%T.__float__() must return a float, not %T. " "The ability to return an instance of a strict subclass of float " "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(o)->tp_name, Py_TYPE(res)->tp_name)) { + o, res)) { Py_DECREF(res); return NULL; } @@ -2435,10 +2436,8 @@ method_output_as_list(PyObject *o, PyObject *meth) PyThreadState *tstate = _PyThreadState_GET(); if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "%.200s.%U() returned a non-iterable (type %.200s)", - Py_TYPE(o)->tp_name, - meth, - Py_TYPE(meth_output)->tp_name); + "%T.%U() must return an iterable, not %T", + o, meth, meth_output); } Py_DECREF(meth_output); return NULL; @@ -2818,9 +2817,8 @@ PyObject_GetIter(PyObject *o) PyObject *res = (*f)(o); if (res != NULL && !PyIter_Check(res)) { PyErr_Format(PyExc_TypeError, - "iter() returned non-iterator " - "of type '%.100s'", - Py_TYPE(res)->tp_name); + "%T.__iter__() must return an iterator, not %T", + o, res); Py_SETREF(res, NULL); } return res; @@ -2839,8 +2837,8 @@ PyObject_GetAIter(PyObject *o) { PyObject *it = (*f)(o); if (it != NULL && !PyAIter_Check(it)) { PyErr_Format(PyExc_TypeError, - "aiter() returned not an async iterator of type '%.100s'", - Py_TYPE(it)->tp_name); + "%T.__aiter__() must return an async iterator, not %T", + o, it); Py_SETREF(it, NULL); } return it; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b5d5ca9178e..99e1c9b13f7 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -17,8 +17,8 @@ class bytearray "PyByteArrayObject *" "&PyByteArray_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=5535b77c37a119e0]*/ -/* For PyByteArray_AS_STRING(). */ -char _PyByteArray_empty_string[] = ""; +/* Max number of bytes a bytearray can contain */ +#define PyByteArray_SIZE_MAX ((Py_ssize_t)(PY_SSIZE_T_MAX - _PyBytesObject_SIZE)) /* Helpers */ @@ -43,6 +43,14 @@ _getbytevalue(PyObject* arg, int *value) return 1; } +static void +bytearray_reinit_from_bytes(PyByteArrayObject *self, Py_ssize_t size, + Py_ssize_t alloc) { + self->ob_bytes = self->ob_start = PyBytes_AS_STRING(self->ob_bytes_object); + Py_SET_SIZE(self, size); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->ob_alloc, alloc); +} + static int bytearray_getbuffer_lock_held(PyObject *self, Py_buffer *view, int flags) { @@ -127,7 +135,6 @@ PyObject * PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) { PyByteArrayObject *new; - Py_ssize_t alloc; if (size < 0) { PyErr_SetString(PyExc_SystemError, @@ -135,35 +142,32 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) return NULL; } - /* Prevent buffer overflow when setting alloc to size+1. */ - if (size == PY_SSIZE_T_MAX) { - return PyErr_NoMemory(); - } - new = PyObject_New(PyByteArrayObject, &PyByteArray_Type); - if (new == NULL) + if (new == NULL) { return NULL; + } - if (size == 0) { - new->ob_bytes = NULL; - alloc = 0; - } - else { - alloc = size + 1; - new->ob_bytes = PyMem_Malloc(alloc); - if (new->ob_bytes == NULL) { - Py_DECREF(new); - return PyErr_NoMemory(); - } - if (bytes != NULL && size > 0) - memcpy(new->ob_bytes, bytes, size); - new->ob_bytes[size] = '\0'; /* Trailing null byte */ - } - Py_SET_SIZE(new, size); - new->ob_alloc = alloc; - new->ob_start = new->ob_bytes; + /* Fill values used in bytearray_dealloc. + + In an optimized build the memory isn't zeroed and ob_exports would be + uninitialized when when PyBytes_FromStringAndSize errored leading to + intermittent test failures. */ new->ob_exports = 0; + /* Optimization: size=0 bytearray should not allocate space + + PyBytes_FromStringAndSize returns the empty bytes global when size=0 so + no allocation occurs. */ + new->ob_bytes_object = PyBytes_FromStringAndSize(NULL, size); + if (new->ob_bytes_object == NULL) { + Py_DECREF(new); + return NULL; + } + bytearray_reinit_from_bytes(new, size, size); + if (bytes != NULL && size > 0) { + memcpy(new->ob_bytes, bytes, size); + } + return (PyObject *)new; } @@ -189,7 +193,6 @@ static int bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - void *sval; PyByteArrayObject *obj = ((PyByteArrayObject *)self); /* All computations are done unsigned to avoid integer overflows (see issue #22335). */ @@ -214,16 +217,17 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) return -1; } - if (size + logical_offset + 1 <= alloc) { + if (size + logical_offset <= alloc) { /* Current buffer is large enough to host the requested size, decide on a strategy. */ if (size < alloc / 2) { /* Major downsize; resize down to exact size */ - alloc = size + 1; + alloc = size; } else { /* Minor downsize; quick exit */ Py_SET_SIZE(self, size); + /* Add mid-buffer null; end provided by bytes. */ PyByteArray_AS_STRING(self)[size] = '\0'; /* Trailing null */ return 0; } @@ -236,38 +240,36 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) } else { /* Major upsize; resize up to exact size */ - alloc = size + 1; + alloc = size; } } - if (alloc > PY_SSIZE_T_MAX) { + if (alloc > PyByteArray_SIZE_MAX) { PyErr_NoMemory(); return -1; } + /* Re-align data to the start of the allocation. */ if (logical_offset > 0) { - sval = PyMem_Malloc(alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } - memcpy(sval, PyByteArray_AS_STRING(self), - Py_MIN((size_t)requested_size, (size_t)Py_SIZE(self))); - PyMem_Free(obj->ob_bytes); - } - else { - sval = PyMem_Realloc(obj->ob_bytes, alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } + /* optimization tradeoff: This is faster than a new allocation when + the number of bytes being removed in a resize is small; for large + size changes it may be better to just make a new bytes object as + _PyBytes_Resize will do a malloc + memcpy internally. */ + memmove(obj->ob_bytes, obj->ob_start, + Py_MIN(requested_size, Py_SIZE(self))); } - obj->ob_bytes = obj->ob_start = sval; - Py_SET_SIZE(self, size); - FT_ATOMIC_STORE_SSIZE_RELAXED(obj->ob_alloc, alloc); - obj->ob_bytes[size] = '\0'; /* Trailing null byte */ + int ret = _PyBytes_Resize(&obj->ob_bytes_object, alloc); + if (ret == -1) { + obj->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + size = alloc = 0; + } + bytearray_reinit_from_bytes(obj, size, alloc); + if (alloc != size) { + /* Add mid-buffer null; end provided by bytes. */ + obj->ob_bytes[size] = '\0'; + } - return 0; + return ret; } int @@ -295,7 +297,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) goto done; } - if (va.len > PY_SSIZE_T_MAX - vb.len) { + if (va.len > PyByteArray_SIZE_MAX - vb.len) { PyErr_NoMemory(); goto done; } @@ -339,7 +341,7 @@ bytearray_iconcat_lock_held(PyObject *op, PyObject *other) } Py_ssize_t size = Py_SIZE(self); - if (size > PY_SSIZE_T_MAX - vo.len) { + if (size > PyByteArray_SIZE_MAX - vo.len) { PyBuffer_Release(&vo); return PyErr_NoMemory(); } @@ -373,7 +375,7 @@ bytearray_repeat_lock_held(PyObject *op, Py_ssize_t count) count = 0; } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { + if (count > 0 && mysize > PyByteArray_SIZE_MAX / count) { return PyErr_NoMemory(); } Py_ssize_t size = mysize * count; @@ -409,7 +411,7 @@ bytearray_irepeat_lock_held(PyObject *op, Py_ssize_t count) } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { + if (count > 0 && mysize > PyByteArray_SIZE_MAX / count) { return PyErr_NoMemory(); } const Py_ssize_t size = mysize * count; @@ -585,7 +587,7 @@ bytearray_setslice_linear(PyByteArrayObject *self, buf = PyByteArray_AS_STRING(self); } else if (growth > 0) { - if (Py_SIZE(self) > (Py_ssize_t)PY_SSIZE_T_MAX - growth) { + if (Py_SIZE(self) > PyByteArray_SIZE_MAX - growth) { PyErr_NoMemory(); return -1; } @@ -709,7 +711,9 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - char *buf = PyByteArray_AS_STRING(self); + // Do not store a reference to the internal buffer since + // index.__index__() or _getbytevalue() may alter 'self'. + // See https://github.com/python/cpython/issues/91153. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -744,7 +748,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value } else { assert(0 <= ival && ival < 256); - buf[i] = (char)ival; + PyByteArray_AS_STRING(self)[i] = (char)ival; return 0; } } @@ -805,6 +809,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Delete slice */ size_t cur; Py_ssize_t i; + char *buf = PyByteArray_AS_STRING(self); if (!_canresize(self)) return -1; @@ -845,6 +850,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Assign slice */ Py_ssize_t i; size_t cur; + char *buf = PyByteArray_AS_STRING(self); if (needed != slicelen) { PyErr_Format(PyExc_ValueError, @@ -895,6 +901,13 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, PyObject *it; PyObject *(*iternext)(PyObject *); + /* First __init__; set ob_bytes_object so ob_bytes is always non-null. */ + if (self->ob_bytes_object == NULL) { + self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + bytearray_reinit_from_bytes(self, 0, 0); + self->ob_exports = 0; + } + if (Py_SIZE(self) != 0) { /* Empty previous contents (yes, do this first of all!) */ if (PyByteArray_Resize((PyObject *)self, 0) < 0) @@ -1063,95 +1076,20 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, return -1; } -/* Mostly copied from string_repr, but without the - "smart quote" functionality. */ static PyObject * bytearray_repr_lock_held(PyObject *op) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - PyByteArrayObject *self = _PyByteArray_CAST(op); - const char *className = _PyType_Name(Py_TYPE(self)); - const char *quote_prefix = "(b"; - const char *quote_postfix = ")"; - Py_ssize_t length = Py_SIZE(self); - /* 6 == strlen(quote_prefix) + 2 + strlen(quote_postfix) + 1 */ - Py_ssize_t newsize; - PyObject *v; - Py_ssize_t i; - char *bytes; - char c; - char *p; - int quote; - char *test, *start; - char *buffer; - - newsize = strlen(className); - if (length > (PY_SSIZE_T_MAX - 6 - newsize) / 4) { - PyErr_SetString(PyExc_OverflowError, - "bytearray object is too large to make repr"); + const char *className = _PyType_Name(Py_TYPE(op)); + PyObject *bytes_repr = _Py_bytes_repr(PyByteArray_AS_STRING(op), + PyByteArray_GET_SIZE(op), 1, + "bytearray"); + if (bytes_repr == NULL) { return NULL; } - - newsize += 6 + length * 4; - buffer = PyMem_Malloc(newsize); - if (buffer == NULL) { - PyErr_NoMemory(); - return NULL; - } - - /* Figure out which quote to use; single is preferred */ - quote = '\''; - start = PyByteArray_AS_STRING(self); - for (test = start; test < start+length; ++test) { - if (*test == '"') { - quote = '\''; /* back to single */ - break; - } - else if (*test == '\'') - quote = '"'; - } - - p = buffer; - while (*className) - *p++ = *className++; - while (*quote_prefix) - *p++ = *quote_prefix++; - *p++ = quote; - - bytes = PyByteArray_AS_STRING(self); - for (i = 0; i < length; i++) { - /* There's at least enough room for a hex escape - and a closing quote. */ - assert(newsize - (p - buffer) >= 5); - c = bytes[i]; - if (c == '\'' || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c == 0) - *p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = Py_hexdigits[(c & 0xf0) >> 4]; - *p++ = Py_hexdigits[c & 0xf]; - } - else - *p++ = c; - } - assert(newsize - (p - buffer) >= 1); - *p++ = quote; - while (*quote_postfix) { - *p++ = *quote_postfix++; - } - - v = PyUnicode_FromStringAndSize(buffer, p - buffer); - PyMem_Free(buffer); - return v; + PyObject *res = PyUnicode_FromFormat("%s(%U)", className, bytes_repr); + Py_DECREF(bytes_repr); + return res; } static PyObject * @@ -1240,9 +1178,7 @@ bytearray_dealloc(PyObject *op) "deallocated bytearray object has exported buffers"); PyErr_Print(); } - if (self->ob_bytes != 0) { - PyMem_Free(self->ob_bytes); - } + Py_XDECREF(self->ob_bytes_object); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1275,6 +1211,7 @@ bytearray_dealloc(PyObject *op) /*[clinic input] +@permit_long_summary @critical_section @text_signature "($self, sub[, start[, end]], /)" bytearray.find @@ -1294,13 +1231,14 @@ Return -1 on failure. static PyObject * bytearray_find_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=413e1cab2ae87da0 input=1de9f4558df68336]*/ +/*[clinic end generated code: output=413e1cab2ae87da0 input=df3aa94840d893a7]*/ { return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), sub, start, end); } /*[clinic input] +@permit_long_summary @critical_section bytearray.count = bytearray.find @@ -1310,7 +1248,7 @@ Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[ static PyObject * bytearray_count_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=a21ee2692e4f1233 input=2608c30644614724]*/ +/*[clinic end generated code: output=a21ee2692e4f1233 input=e8fcdca8272857e0]*/ { return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), sub, start, end); @@ -1347,6 +1285,7 @@ bytearray_copy_impl(PyByteArrayObject *self) } /*[clinic input] +@permit_long_summary @critical_section bytearray.index = bytearray.find @@ -1358,13 +1297,14 @@ Raise ValueError if the subsection is not found. static PyObject * bytearray_index_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=067a1e78efc672a7 input=0086ba0ab9bf44a5]*/ +/*[clinic end generated code: output=067a1e78efc672a7 input=c37f177cfee19fe4]*/ { return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), sub, start, end); } /*[clinic input] +@permit_long_summary @critical_section bytearray.rfind = bytearray.find @@ -1376,13 +1316,14 @@ Return -1 on failure. static PyObject * bytearray_rfind_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=51bf886f932b283c input=ac73593305d5c1d1]*/ +/*[clinic end generated code: output=51bf886f932b283c input=1265b11c437d2750]*/ { return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), sub, start, end); } /*[clinic input] +@permit_long_summary @critical_section bytearray.rindex = bytearray.find @@ -1394,7 +1335,7 @@ Raise ValueError if the subsection is not found. static PyObject * bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=38e1cf66bafb08b9 input=0cf331bf5ebe0e91]*/ +/*[clinic end generated code: output=38e1cf66bafb08b9 input=7d198b3d6b0a62ce]*/ { return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), sub, start, end); @@ -1413,6 +1354,7 @@ bytearray_contains(PyObject *self, PyObject *arg) } /*[clinic input] +@permit_long_summary @critical_section @text_signature "($self, prefix[, start[, end]], /)" bytearray.startswith @@ -1431,13 +1373,14 @@ Return True if the bytearray starts with the specified prefix, False otherwise. static PyObject * bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=a3d9b6d44d3662a6 input=ea8d036d09df34b2]*/ +/*[clinic end generated code: output=a3d9b6d44d3662a6 input=93f9ffee684f109a]*/ { return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), subobj, start, end); } /*[clinic input] +@permit_long_summary @critical_section @text_signature "($self, suffix[, start[, end]], /)" bytearray.endswith @@ -1456,7 +1399,7 @@ Return True if the bytearray ends with the specified suffix, False otherwise. static PyObject * bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=e75ea8c227954caa input=c61b90bb23a689ce]*/ +/*[clinic end generated code: output=e75ea8c227954caa input=d158b030a11d0b06]*/ { return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), subobj, start, end); @@ -1533,14 +1476,14 @@ bytearray_removesuffix_impl(PyByteArrayObject *self, Py_buffer *suffix) /*[clinic input] bytearray.resize size: Py_ssize_t - New size to resize to.. + New size to resize to. / Resize the internal buffer of bytearray to len. [clinic start generated code]*/ static PyObject * bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) -/*[clinic end generated code: output=f73524922990b2d9 input=75fd4d17c4aa47d3]*/ +/*[clinic end generated code: output=f73524922990b2d9 input=6c9a260ca7f72071]*/ { Py_ssize_t start_size = PyByteArray_GET_SIZE(self); int result = PyByteArray_Resize((PyObject *)self, size); @@ -1555,6 +1498,94 @@ bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) } +/*[clinic input] +@critical_section +bytearray.take_bytes + n: object = None + Bytes to take, negative indexes from end. None indicates all bytes. + / +Take *n* bytes from the bytearray and return them as a bytes object. +[clinic start generated code]*/ + +static PyObject * +bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n) +/*[clinic end generated code: output=3147fbc0bbbe8d94 input=b15b5172cdc6deda]*/ +{ + Py_ssize_t to_take; + Py_ssize_t size = Py_SIZE(self); + if (Py_IsNone(n)) { + to_take = size; + } + // Integer index, from start (zero, positive) or end (negative). + else if (_PyIndex_Check(n)) { + to_take = PyNumber_AsSsize_t(n, PyExc_IndexError); + if (to_take == -1 && PyErr_Occurred()) { + return NULL; + } + if (to_take < 0) { + to_take += size; + } + } + else { + PyErr_SetString(PyExc_TypeError, "n must be an integer or None"); + return NULL; + } + + if (to_take < 0 || to_take > size) { + PyErr_Format(PyExc_IndexError, + "can't take %zd bytes outside size %zd", + to_take, size); + return NULL; + } + + // Exports may change the contents. No mutable bytes allowed. + if (!_canresize(self)) { + return NULL; + } + + if (to_take == 0 || size == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + + Py_ssize_t remaining_length = size - to_take; + // optimization: If taking less than leaving, just copy the small to_take + // portion out and move ob_start. + if (to_take < remaining_length) { + PyObject *ret = PyBytes_FromStringAndSize(self->ob_start, to_take); + if (ret == NULL) { + return NULL; + } + self->ob_start += to_take; + Py_SET_SIZE(self, remaining_length); + return ret; + } + + // Copy remaining bytes to a new bytes. + PyObject *remaining = PyBytes_FromStringAndSize(self->ob_start + to_take, + remaining_length); + if (remaining == NULL) { + return NULL; + } + + // If the bytes are offset inside the buffer must first align. + if (self->ob_start != self->ob_bytes) { + memmove(self->ob_bytes, self->ob_start, to_take); + self->ob_start = self->ob_bytes; + } + + if (_PyBytes_Resize(&self->ob_bytes_object, to_take) == -1) { + Py_DECREF(remaining); + return NULL; + } + + // Point the bytearray towards the buffer with the remaining data. + PyObject *result = self->ob_bytes_object; + self->ob_bytes_object = remaining; + bytearray_reinit_from_bytes(self, remaining_length, remaining_length); + return result; +} + + /*[clinic input] @critical_section bytearray.translate @@ -1662,6 +1693,8 @@ bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @staticmethod bytearray.maketrans @@ -1679,13 +1712,14 @@ The bytes objects frm and to must be of the same length. static PyObject * bytearray_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=1df267d99f56b15e input=b10de38c85950a63]*/ +/*[clinic end generated code: output=1df267d99f56b15e input=1146b43a592eca13]*/ { return _Py_bytes_maketrans(frm, to); } /*[clinic input] +@permit_long_docstring_body @critical_section bytearray.replace @@ -1705,7 +1739,7 @@ replaced. static PyObject * bytearray_replace_impl(PyByteArrayObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count) -/*[clinic end generated code: output=d39884c4dc59412a input=6992755672c8a807]*/ +/*[clinic end generated code: output=d39884c4dc59412a input=66afec32f4e095e0]*/ { return stringlib_replace((PyObject *)self, (const char *)old->buf, old->len, @@ -1713,6 +1747,7 @@ bytearray_replace_impl(PyByteArrayObject *self, Py_buffer *old, } /*[clinic input] +@permit_long_summary @critical_section bytearray.split @@ -1730,7 +1765,7 @@ Return a list of the sections in the bytearray, using sep as the delimiter. static PyObject * bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=833e2cf385d9a04d input=1c367486b9938909]*/ +/*[clinic end generated code: output=833e2cf385d9a04d input=dd9f6e2910cc3a34]*/ { Py_ssize_t len = PyByteArray_GET_SIZE(self), n; const char *s = PyByteArray_AS_STRING(self), *sub; @@ -1756,6 +1791,7 @@ bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, } /*[clinic input] +@permit_long_docstring_body @critical_section bytearray.partition @@ -1774,7 +1810,7 @@ original bytearray object and two empty bytearray objects. static PyObject * bytearray_partition_impl(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=b5fa1e03f10cfccb input=632855f986733f34]*/ +/*[clinic end generated code: output=b5fa1e03f10cfccb input=b87276af883f39d9]*/ { PyObject *bytesep, *result; @@ -1794,6 +1830,7 @@ bytearray_partition_impl(PyByteArrayObject *self, PyObject *sep) } /*[clinic input] +@permit_long_docstring_body @critical_section bytearray.rpartition @@ -1813,7 +1850,7 @@ objects and the copy of the original bytearray object. static PyObject * bytearray_rpartition_impl(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=0186ce7b1ef61289 input=4318e3d125497450]*/ +/*[clinic end generated code: output=0186ce7b1ef61289 input=5bdcfc4c333bcfab]*/ { PyObject *bytesep, *result; @@ -1833,6 +1870,8 @@ bytearray_rpartition_impl(PyByteArrayObject *self, PyObject *sep) } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @critical_section bytearray.rsplit = bytearray.split @@ -1844,7 +1883,7 @@ Splitting is done starting at the end of the bytearray and working to the front. static PyObject * bytearray_rsplit_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=a55e0b5a03cb6190 input=3cd513c2b94a53c1]*/ +/*[clinic end generated code: output=a55e0b5a03cb6190 input=60e9abf305128ff4]*/ { Py_ssize_t len = PyByteArray_GET_SIZE(self), n; const char *s = PyByteArray_AS_STRING(self), *sub; @@ -1924,11 +1963,6 @@ bytearray_insert_impl(PyByteArrayObject *self, Py_ssize_t index, int item) Py_ssize_t n = Py_SIZE(self); char *buf; - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytearray"); - return NULL; - } if (bytearray_resize_lock_held((PyObject *)self, n + 1) < 0) return NULL; buf = PyByteArray_AS_STRING(self); @@ -2043,11 +2077,6 @@ bytearray_append_impl(PyByteArrayObject *self, int item) { Py_ssize_t n = Py_SIZE(self); - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytearray"); - return NULL; - } if (bytearray_resize_lock_held((PyObject *)self, n + 1) < 0) return NULL; @@ -2087,6 +2116,7 @@ bytearray_expandtabs(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py } /*[clinic input] +@permit_long_summary @critical_section bytearray.extend @@ -2099,7 +2129,7 @@ Append all the items from the iterator or sequence to the end of the bytearray. static PyObject * bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) -/*[clinic end generated code: output=2f25e0ce72b98748 input=86e65beaba444650]*/ +/*[clinic end generated code: output=2f25e0ce72b98748 input=aeed44b025146632]*/ { PyObject *it, *item, *bytearray_obj; Py_ssize_t buf_size = 0, len = 0; @@ -2154,16 +2184,16 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) if (len >= buf_size) { Py_ssize_t addition; - if (len == PY_SSIZE_T_MAX) { + if (len == PyByteArray_SIZE_MAX) { Py_DECREF(it); Py_DECREF(bytearray_obj); return PyErr_NoMemory(); } addition = len >> 1; - if (addition > PY_SSIZE_T_MAX - len - 1) - buf_size = PY_SSIZE_T_MAX; + if (addition > PyByteArray_SIZE_MAX - len) + buf_size = PyByteArray_SIZE_MAX; else - buf_size = len + addition + 1; + buf_size = len + addition; if (bytearray_resize_lock_held((PyObject *)bytearray_obj, buf_size) < 0) { Py_DECREF(it); Py_DECREF(bytearray_obj); @@ -2318,6 +2348,7 @@ bytearray_strip_impl_helper(PyByteArrayObject* self, PyObject* bytes, int stript } /*[clinic input] +@permit_long_docstring_body @critical_section bytearray.strip @@ -2331,7 +2362,7 @@ If the argument is omitted or None, strip leading and trailing ASCII whitespace. static PyObject * bytearray_strip_impl(PyByteArrayObject *self, PyObject *bytes) -/*[clinic end generated code: output=760412661a34ad5a input=1f9026e5ad35388a]*/ +/*[clinic end generated code: output=760412661a34ad5a input=6acaf88b2ec9daa7]*/ { return bytearray_strip_impl_helper(self, bytes, BOTHSTRIP); } @@ -2459,7 +2490,11 @@ static PyObject * bytearray_alloc(PyObject *op, PyObject *Py_UNUSED(ignored)) { PyByteArrayObject *self = _PyByteArray_CAST(op); - return PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc)); + Py_ssize_t alloc = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc); + if (alloc > 0) { + alloc += _PyBytesObject_SIZE; + } + return PyLong_FromSsize_t(alloc); } /*[clinic input] @@ -2508,6 +2543,8 @@ bytearray_rjust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @critical_section bytearray.splitlines @@ -2521,7 +2558,7 @@ true. static PyObject * bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) -/*[clinic end generated code: output=4223c94b895f6ad9 input=874cd662866a66a1]*/ +/*[clinic end generated code: output=4223c94b895f6ad9 input=21bc3f02bf1be832]*/ { return stringlib_splitlines( (PyObject*) self, PyByteArray_AS_STRING(self), @@ -2653,9 +2690,13 @@ static PyObject * bytearray_sizeof_impl(PyByteArrayObject *self) /*[clinic end generated code: output=738abdd17951c427 input=e27320fd98a4bc5a]*/ { - size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc) * sizeof(char); - return PyLong_FromSize_t(res); + Py_ssize_t res = _PyObject_SIZE(Py_TYPE(self)); + Py_ssize_t alloc = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc); + if (alloc > 0) { + res += _PyBytesObject_SIZE + alloc; + } + + return PyLong_FromSsize_t(res); } static PySequenceMethods bytearray_as_sequence = { @@ -2738,6 +2779,7 @@ static PyMethodDef bytearray_methods[] = { BYTEARRAY_STARTSWITH_METHODDEF BYTEARRAY_STRIP_METHODDEF {"swapcase", bytearray_swapcase, METH_NOARGS, _Py_swapcase__doc__}, + BYTEARRAY_TAKE_BYTES_METHODDEF {"title", bytearray_title, METH_NOARGS, _Py_title__doc__}, BYTEARRAY_TRANSLATE_METHODDEF {"upper", bytearray_upper, METH_NOARGS, _Py_upper__doc__}, diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index c239ae18a59..56a461d0dd0 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -356,26 +356,24 @@ The bytes objects frm and to must be of the same length."); PyObject * _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to) { - PyObject *res = NULL; - Py_ssize_t i; - char *p; - if (frm->len != to->len) { PyErr_Format(PyExc_ValueError, "maketrans arguments must have same length"); return NULL; } - res = PyBytes_FromStringAndSize(NULL, 256); - if (!res) + PyBytesWriter *writer = PyBytesWriter_Create(256); + if (!writer) { return NULL; - p = PyBytes_AS_STRING(res); + } + char *p = PyBytesWriter_GetData(writer); + Py_ssize_t i; for (i = 0; i < 256; i++) p[i] = (char) i; for (i = 0; i < frm->len; i++) { p[((unsigned char *)frm->buf)[i]] = ((char *)to->buf)[i]; } - return res; + return PyBytesWriter_Finish(writer); } #define FASTSEARCH fastsearch diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index fc407ec6bf9..2b0925017f2 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -7,6 +7,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_format.h" // F_LJUST +#include "pycore_freelist.h" // _Py_FREELIST_FREE() #include "pycore_global_objects.h"// _Py_GET_GLOBAL_OBJECT() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _PyLong_DigitValue @@ -24,17 +25,12 @@ class bytes "PyBytesObject *" "&PyBytes_Type" #include "clinic/bytesobject.c.h" -/* PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation - for a bytes object of length n should request PyBytesObject_SIZE + n bytes. - - Using PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves - 3 or 7 bytes per bytes object allocation on a typical system. -*/ -#define PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) +#define PyBytesObject_SIZE _PyBytesObject_SIZE /* Forward declaration */ -Py_LOCAL_INLINE(Py_ssize_t) _PyBytesWriter_GetSize(_PyBytesWriter *writer, - char *str); +static void* _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, + Py_ssize_t size, void *data); +static Py_ssize_t _PyBytesWriter_GetAllocated(PyBytesWriter *writer); #define CHARACTERS _Py_SINGLETON(bytes_characters) @@ -195,10 +191,11 @@ PyBytes_FromString(const char *str) return (PyObject *) op; } -PyObject * -PyBytes_FromFormatV(const char *format, va_list vargs) + +static char* +bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, + const char *format, va_list vargs) { - char *s; const char *f; const char *p; Py_ssize_t prec; @@ -212,21 +209,20 @@ PyBytes_FromFormatV(const char *format, va_list vargs) Longest 64-bit pointer representation: "0xffffffffffffffff\0" (19 bytes). */ char buffer[21]; - _PyBytesWriter writer; - _PyBytesWriter_Init(&writer); + char *s = (char*)PyBytesWriter_GetData(writer) + writer_pos; - s = _PyBytesWriter_Alloc(&writer, strlen(format)); - if (s == NULL) - return NULL; - writer.overallocate = 1; - -#define WRITE_BYTES(str) \ +#define WRITE_BYTES_LEN(str, len_expr) \ do { \ - s = _PyBytesWriter_WriteBytes(&writer, s, (str), strlen(str)); \ - if (s == NULL) \ + size_t len = (len_expr); \ + s = PyBytesWriter_GrowAndUpdatePointer(writer, len, s); \ + if (s == NULL) { \ goto error; \ + } \ + memcpy(s, (str), len); \ + s += len; \ } while (0) +#define WRITE_BYTES(str) WRITE_BYTES_LEN(str, strlen(str)) for (f = format; *f; f++) { if (*f != '%') { @@ -267,10 +263,6 @@ PyBytes_FromFormatV(const char *format, va_list vargs) ++f; } - /* subtract bytes preallocated for the format string - (ex: 2 for "%s") */ - writer.min_size -= (f - p + 1); - switch (*f) { case 'c': { @@ -281,7 +273,6 @@ PyBytes_FromFormatV(const char *format, va_list vargs) "expects an integer in range [0; 255]"); goto error; } - writer.min_size++; *s++ = (unsigned char)c; break; } @@ -340,9 +331,7 @@ PyBytes_FromFormatV(const char *format, va_list vargs) i++; } } - s = _PyBytesWriter_WriteBytes(&writer, s, p, i); - if (s == NULL) - goto error; + WRITE_BYTES_LEN(p, i); break; } @@ -361,31 +350,45 @@ PyBytes_FromFormatV(const char *format, va_list vargs) break; case '%': - writer.min_size++; *s++ = '%'; break; default: - if (*f == 0) { - /* fix min_size if we reached the end of the format string */ - writer.min_size++; - } - /* invalid format string: copy unformatted string and exit */ WRITE_BYTES(p); - return _PyBytesWriter_Finish(&writer, s); + return s; } } #undef WRITE_BYTES +#undef WRITE_BYTES_LEN - return _PyBytesWriter_Finish(&writer, s); + return s; error: - _PyBytesWriter_Dealloc(&writer); return NULL; } + +PyObject * +PyBytes_FromFormatV(const char *format, va_list vargs) +{ + Py_ssize_t alloc = strlen(format); + PyBytesWriter *writer = PyBytesWriter_Create(alloc); + if (writer == NULL) { + return NULL; + } + + char *s = bytes_fromformat(writer, 0, format, vargs); + if (s == NULL) { + PyBytesWriter_Discard(writer); + return NULL; + } + + return PyBytesWriter_FinishWithPointer(writer, s); +} + + PyObject * PyBytes_FromFormat(const char *format, ...) { @@ -398,6 +401,7 @@ PyBytes_FromFormat(const char *format, ...) return ret; } + /* Helpers for formatstring */ Py_LOCAL_INLINE(PyObject *) @@ -420,7 +424,7 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) static char* formatfloat(PyObject *v, int flags, int prec, int type, - PyObject **p_result, _PyBytesWriter *writer, char *str) + PyObject **p_result, PyBytesWriter *writer, char *str) { char *p; PyObject *result; @@ -448,7 +452,7 @@ formatfloat(PyObject *v, int flags, int prec, int type, len = strlen(p); if (writer != NULL) { - str = _PyBytesWriter_Prepare(writer, str, len); + str = PyBytesWriter_GrowAndUpdatePointer(writer, len, str); if (str == NULL) { PyMem_Free(p); return NULL; @@ -566,8 +570,8 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) return NULL; if (!PyBytes_Check(result)) { PyErr_Format(PyExc_TypeError, - "__bytes__ returned non-bytes (type %.200s)", - Py_TYPE(result)->tp_name); + "%T.__bytes__() must return a bytes, not %T", + v, result); Py_DECREF(result); return NULL; } @@ -599,12 +603,10 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, PyObject *args, int use_bytearray) { const char *fmt; - char *res; Py_ssize_t arglen, argidx; Py_ssize_t fmtcnt; int args_owned = 0; PyObject *dict = NULL; - _PyBytesWriter writer; if (args == NULL) { PyErr_BadInternalCall(); @@ -613,14 +615,17 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, fmt = format; fmtcnt = format_len; - _PyBytesWriter_Init(&writer); - writer.use_bytearray = use_bytearray; - - res = _PyBytesWriter_Alloc(&writer, fmtcnt); - if (res == NULL) + PyBytesWriter *writer; + if (use_bytearray) { + writer = _PyBytesWriter_CreateByteArray(fmtcnt); + } + else { + writer = PyBytesWriter_Create(fmtcnt); + } + if (writer == NULL) { return NULL; - if (!use_bytearray) - writer.overallocate = 1; + } + char *res = PyBytesWriter_GetData(writer); if (PyTuple_Check(args)) { arglen = PyTuple_GET_SIZE(args); @@ -825,7 +830,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, if (fmtcnt == 0) { /* last write: disable writer overallocation */ - writer.overallocate = 0; + writer->overallocate = 0; } sign = 0; @@ -888,8 +893,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, } /* Fast path */ - writer.min_size -= 2; /* size preallocated for "%d" */ - res = _PyLong_FormatBytesWriter(&writer, res, + res = _PyLong_FormatBytesWriter(writer, res, v, base, alternate); if (res == NULL) goto error; @@ -917,8 +921,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, && !(flags & (F_SIGN | F_BLANK))) { /* Fast path */ - writer.min_size -= 2; /* size preallocated for "%f" */ - res = formatfloat(v, flags, prec, c, NULL, &writer, res); + res = formatfloat(v, flags, prec, c, NULL, writer, res); if (res == NULL) goto error; continue; @@ -974,9 +977,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, alloc++; /* 2: size preallocated for %s */ if (alloc > 2) { - res = _PyBytesWriter_Prepare(&writer, res, alloc - 2); - if (res == NULL) + res = PyBytesWriter_GrowAndUpdatePointer(writer, alloc - 2, res); + if (res == NULL) { + Py_XDECREF(temp); goto error; + } } #ifndef NDEBUG char *before = res; @@ -1052,7 +1057,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, /* If overallocation was disabled, ensure that it was the last write. Otherwise, we missed an optimization */ - assert(writer.overallocate || fmtcnt == 0 || use_bytearray); + assert(writer->overallocate || fmtcnt == 0 || use_bytearray); } /* until end */ if (argidx < arglen && !dict) { @@ -1064,10 +1069,10 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, if (args_owned) { Py_DECREF(args); } - return _PyBytesWriter_Finish(&writer, res); + return PyBytesWriter_FinishWithPointer(writer, res); error: - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); if (args_owned) { Py_DECREF(args); } @@ -1075,26 +1080,22 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, } /* Unescape a backslash-escaped string. */ -PyObject *_PyBytes_DecodeEscape(const char *s, +PyObject *_PyBytes_DecodeEscape2(const char *s, Py_ssize_t len, const char *errors, - const char **first_invalid_escape) + int *first_invalid_escape_char, + const char **first_invalid_escape_ptr) { - int c; - char *p; - const char *end; - _PyBytesWriter writer; - - _PyBytesWriter_Init(&writer); - - p = _PyBytesWriter_Alloc(&writer, len); - if (p == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { return NULL; - writer.overallocate = 1; + } + char *p = PyBytesWriter_GetData(writer); - *first_invalid_escape = NULL; + *first_invalid_escape_char = -1; + *first_invalid_escape_ptr = NULL; - end = s + len; + const char *end = s + len; while (s < end) { if (*s != '\\') { *p++ = *s++; @@ -1123,20 +1124,23 @@ PyObject *_PyBytes_DecodeEscape(const char *s, case 'a': *p++ = '\007'; break; /* BEL, not classic C */ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': - c = s[-1] - '0'; + { + int c = s[-1] - '0'; if (s < end && '0' <= *s && *s <= '7') { c = (c<<3) + *s++ - '0'; if (s < end && '0' <= *s && *s <= '7') c = (c<<3) + *s++ - '0'; } if (c > 0377) { - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-3; /* Back up 3 chars, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = c; + /* Back up 3 chars, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 3; } } *p++ = c; break; + } case 'x': if (s+1 < end) { int digit1, digit2; @@ -1173,19 +1177,20 @@ PyObject *_PyBytes_DecodeEscape(const char *s, break; default: - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-1; /* Back up one char, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = (unsigned char)s[-1]; + /* Back up one char, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 1; } *p++ = '\\'; s--; } } - return _PyBytesWriter_Finish(&writer, p); + return PyBytesWriter_FinishWithPointer(writer, p); failed: - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return NULL; } @@ -1195,18 +1200,19 @@ PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t Py_UNUSED(unicode), const char *Py_UNUSED(recode_encoding)) { - const char* first_invalid_escape; - PyObject *result = _PyBytes_DecodeEscape(s, len, errors, - &first_invalid_escape); + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + PyObject *result = _PyBytes_DecodeEscape2(s, len, errors, + &first_invalid_escape_char, + &first_invalid_escape_ptr); if (result == NULL) return NULL; - if (first_invalid_escape != NULL) { - unsigned char c = *first_invalid_escape; - if ('4' <= c && c <= '7') { + if (first_invalid_escape_char != -1) { + if (first_invalid_escape_char > 0xff) { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "b\"\\%.3s\" is an invalid octal escape sequence. " + "b\"\\%o\" is an invalid octal escape sequence. " "Such sequences will not work in the future. ", - first_invalid_escape) < 0) + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; @@ -1216,7 +1222,7 @@ PyObject *PyBytes_DecodeEscape(const char *s, if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "b\"\\%c\" is an invalid escape sequence. " "Such sequences will not work in the future. ", - c) < 0) + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; @@ -1335,27 +1341,33 @@ _PyBytes_ReverseFind(const char *haystack, Py_ssize_t len_haystack, PyObject * PyBytes_Repr(PyObject *obj, int smartquotes) { - PyBytesObject* op = (PyBytesObject*) obj; - Py_ssize_t i, length = Py_SIZE(op); + return _Py_bytes_repr(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj), + smartquotes, "bytes"); +} + +PyObject * +_Py_bytes_repr(const char *data, Py_ssize_t length, int smartquotes, + const char *classname) +{ + Py_ssize_t i; Py_ssize_t newsize, squotes, dquotes; PyObject *v; unsigned char quote; - const unsigned char *s; Py_UCS1 *p; /* Compute size of output string */ squotes = dquotes = 0; newsize = 3; /* b'' */ - s = (const unsigned char*)op->ob_sval; for (i = 0; i < length; i++) { + unsigned char c = data[i]; Py_ssize_t incr = 1; - switch(s[i]) { + switch(c) { case '\'': squotes++; break; case '"': dquotes++; break; case '\\': case '\t': case '\n': case '\r': incr = 2; break; /* \C */ default: - if (s[i] < ' ' || s[i] >= 0x7f) + if (c < ' ' || c >= 0x7f) incr = 4; /* \xHH */ } if (newsize > PY_SSIZE_T_MAX - incr) @@ -1379,7 +1391,7 @@ PyBytes_Repr(PyObject *obj, int smartquotes) *p++ = 'b', *p++ = quote; for (i = 0; i < length; i++) { - unsigned char c = op->ob_sval[i]; + unsigned char c = data[i]; if (c == quote || c == '\\') *p++ = '\\', *p++ = c; else if (c == '\t') @@ -1402,8 +1414,8 @@ PyBytes_Repr(PyObject *obj, int smartquotes) return v; overflow: - PyErr_SetString(PyExc_OverflowError, - "bytes object is too large to make repr"); + PyErr_Format(PyExc_OverflowError, + "%s object is too large to make repr", classname); return NULL; } @@ -1781,6 +1793,7 @@ bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) } /*[clinic input] +@permit_long_docstring_body bytes.partition sep: Py_buffer @@ -1798,7 +1811,7 @@ object and two empty bytes objects. static PyObject * bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=f532b392a17ff695 input=61cca95519406099]*/ +/*[clinic end generated code: output=f532b392a17ff695 input=31c55a0cebaf7722]*/ { return stringlib_partition( (PyObject*) self, @@ -1808,6 +1821,7 @@ bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) } /*[clinic input] +@permit_long_docstring_body bytes.rpartition sep: Py_buffer @@ -1825,7 +1839,7 @@ objects and the original bytes object. static PyObject * bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=191b114cbb028e50 input=d78db010c8cfdbe1]*/ +/*[clinic end generated code: output=191b114cbb028e50 input=9ea5a3ab0b02bf52]*/ { return stringlib_rpartition( (PyObject*) self, @@ -1835,6 +1849,7 @@ bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) } /*[clinic input] +@permit_long_docstring_body bytes.rsplit = bytes.split Return a list of the sections in the bytes, using sep as the delimiter. @@ -1844,7 +1859,7 @@ Splitting is done starting at the end of the bytes and working to the front. static PyObject * bytes_rsplit_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=ba698d9ea01e1c8f input=0f86c9f28f7d7b7b]*/ +/*[clinic end generated code: output=ba698d9ea01e1c8f input=55b6eaea1f3d7046]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -1905,6 +1920,7 @@ PyBytes_Join(PyObject *sep, PyObject *iterable) } /*[clinic input] +@permit_long_summary @text_signature "($self, sub[, start[, end]], /)" bytes.find @@ -1923,13 +1939,14 @@ Return -1 on failure. static PyObject * bytes_find_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=d5961a1c77b472a1 input=3171e62a8ae7f240]*/ +/*[clinic end generated code: output=d5961a1c77b472a1 input=47d0929adafc6b0b]*/ { return _Py_bytes_find(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), sub, start, end); } /*[clinic input] +@permit_long_summary bytes.index = bytes.find Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. @@ -1940,13 +1957,14 @@ Raise ValueError if the subsection is not found. static PyObject * bytes_index_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=0da25cc74683ba42 input=aa34ad71ba0bafe3]*/ +/*[clinic end generated code: output=0da25cc74683ba42 input=1cb45ce71456a269]*/ { return _Py_bytes_index(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), sub, start, end); } /*[clinic input] +@permit_long_summary bytes.rfind = bytes.find Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. @@ -1957,13 +1975,14 @@ Return -1 on failure. static PyObject * bytes_rfind_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=51b60fa4ad011c09 input=864c3e7f3010b33c]*/ +/*[clinic end generated code: output=51b60fa4ad011c09 input=c9473d714251f1ab]*/ { return _Py_bytes_rfind(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), sub, start, end); } /*[clinic input] +@permit_long_summary bytes.rindex = bytes.find Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end]. @@ -1974,7 +1993,7 @@ Raise ValueError if the subsection is not found. static PyObject * bytes_rindex_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=42bf674e0a0aabf6 input=21051fc5cfeacf2c]*/ +/*[clinic end generated code: output=42bf674e0a0aabf6 input=bb5f473c64610c43]*/ { return _Py_bytes_rindex(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), sub, start, end); @@ -2060,6 +2079,7 @@ do_argstrip(PyBytesObject *self, int striptype, PyObject *bytes) } /*[clinic input] +@permit_long_docstring_body bytes.strip bytes: object = None @@ -2072,7 +2092,7 @@ If the argument is omitted or None, strip leading and trailing ASCII whitespace. static PyObject * bytes_strip_impl(PyBytesObject *self, PyObject *bytes) -/*[clinic end generated code: output=c7c228d3bd104a1b input=8a354640e4e0b3ef]*/ +/*[clinic end generated code: output=c7c228d3bd104a1b input=71904cd278c0ee03]*/ { return do_argstrip(self, BOTHSTRIP, bytes); } @@ -2115,6 +2135,7 @@ bytes_rstrip_impl(PyBytesObject *self, PyObject *bytes) /*[clinic input] +@permit_long_summary bytes.count = bytes.find Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end]. @@ -2123,7 +2144,7 @@ Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[ static PyObject * bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=9848140b9be17d0f input=b6e4a5ed515e1e59]*/ +/*[clinic end generated code: output=9848140b9be17d0f input=bb2f136f83f0d30e]*/ { return _Py_bytes_count(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), sub, start, end); @@ -2260,6 +2281,8 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @staticmethod bytes.maketrans @@ -2277,13 +2300,14 @@ The bytes objects frm and to must be of the same length. static PyObject * bytes_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=a36f6399d4b77f6f input=a3bd00d430a0979f]*/ +/*[clinic end generated code: output=a36f6399d4b77f6f input=a06b75f44d933fb3]*/ { return _Py_bytes_maketrans(frm, to); } /*[clinic input] +@permit_long_docstring_body bytes.replace old: Py_buffer @@ -2302,7 +2326,7 @@ replaced. static PyObject * bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, Py_ssize_t count) -/*[clinic end generated code: output=994fa588b6b9c104 input=b2fbbf0bf04de8e5]*/ +/*[clinic end generated code: output=994fa588b6b9c104 input=8b99a9ab32bc06a2]*/ { return stringlib_replace((PyObject *)self, (const char *)old->buf, old->len, @@ -2386,6 +2410,7 @@ bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) } /*[clinic input] +@permit_long_summary @text_signature "($self, prefix[, start[, end]], /)" bytes.startswith @@ -2403,13 +2428,14 @@ Return True if the bytes starts with the specified prefix, False otherwise. static PyObject * bytes_startswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=b1e8da1cbd528e8c input=8a4165df8adfa6c9]*/ +/*[clinic end generated code: output=b1e8da1cbd528e8c input=a14efd070f15be80]*/ { return _Py_bytes_startswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), subobj, start, end); } /*[clinic input] +@permit_long_summary @text_signature "($self, suffix[, start[, end]], /)" bytes.endswith @@ -2427,7 +2453,7 @@ Return True if the bytes ends with the specified suffix, False otherwise. static PyObject * bytes_endswith_impl(PyBytesObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=038b633111f3629d input=b5c3407a2a5c9aac]*/ +/*[clinic end generated code: output=038b633111f3629d input=49e383eaaf292713]*/ { return _Py_bytes_endswith(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), subobj, start, end); @@ -2459,6 +2485,7 @@ bytes_decode_impl(PyBytesObject *self, const char *encoding, /*[clinic input] +@permit_long_docstring_body bytes.splitlines keepends: bool = False @@ -2471,7 +2498,7 @@ true. static PyObject * bytes_splitlines_impl(PyBytesObject *self, int keepends) -/*[clinic end generated code: output=3484149a5d880ffb input=5d7b898af2fe55c0]*/ +/*[clinic end generated code: output=3484149a5d880ffb input=d17968d2a355fe55]*/ { return stringlib_splitlines( (PyObject*) self, PyBytes_AS_STRING(self), @@ -2506,17 +2533,13 @@ bytes_fromhex_impl(PyTypeObject *type, PyObject *string) PyObject* _PyBytes_FromHex(PyObject *string, int use_bytearray) { - char *buf; Py_ssize_t hexlen, invalid_char; unsigned int top, bot; const Py_UCS1 *str, *start, *end; - _PyBytesWriter writer; + PyBytesWriter *writer = NULL; Py_buffer view; view.obj = NULL; - _PyBytesWriter_Init(&writer); - writer.use_bytearray = use_bytearray; - if (PyUnicode_Check(string)) { hexlen = PyUnicode_GET_LENGTH(string); @@ -2552,10 +2575,16 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) } /* This overestimates if there are spaces */ - buf = _PyBytesWriter_Alloc(&writer, hexlen / 2); - if (buf == NULL) { + if (use_bytearray) { + writer = _PyBytesWriter_CreateByteArray(hexlen / 2); + } + else { + writer = PyBytesWriter_Create(hexlen / 2); + } + if (writer == NULL) { goto release_buffer; } + char *buf = PyBytesWriter_GetData(writer); start = str; end = str + hexlen; @@ -2594,7 +2623,7 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) if (view.obj != NULL) { PyBuffer_Release(&view); } - return _PyBytesWriter_Finish(&writer, buf); + return PyBytesWriter_FinishWithPointer(writer, buf); error: if (invalid_char == -1) { @@ -2605,7 +2634,7 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) "non-hexadecimal number found in " "fromhex() arg at position %zd", invalid_char); } - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); release_buffer: if (view.obj != NULL) { @@ -2760,7 +2789,7 @@ bytes_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, "errors without a string argument"); return NULL; } - bytes = PyBytes_FromStringAndSize(NULL, 0); + bytes = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else if (encoding != NULL) { /* Encode via the codec registry */ @@ -2788,8 +2817,8 @@ bytes_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, return NULL; if (!PyBytes_Check(bytes)) { PyErr_Format(PyExc_TypeError, - "__bytes__ returned non-bytes (type %.200s)", - Py_TYPE(bytes)->tp_name); + "%T.__bytes__() must return a bytes, not %T", + x, bytes); Py_DECREF(bytes); return NULL; } @@ -2832,23 +2861,25 @@ bytes_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, static PyObject* _PyBytes_FromBuffer(PyObject *x) { - PyObject *new; Py_buffer view; - if (PyObject_GetBuffer(x, &view, PyBUF_FULL_RO) < 0) return NULL; - new = PyBytes_FromStringAndSize(NULL, view.len); - if (!new) + PyBytesWriter *writer = PyBytesWriter_Create(view.len); + if (writer == NULL) { goto fail; - if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, - &view, view.len, 'C') < 0) + } + + if (PyBuffer_ToContiguous(PyBytesWriter_GetData(writer), + &view, view.len, 'C') < 0) { goto fail; + } + PyBuffer_Release(&view); - return new; + return PyBytesWriter_Finish(writer); fail: - Py_XDECREF(new); + PyBytesWriter_Discard(writer); PyBuffer_Release(&view); return NULL; } @@ -2856,23 +2887,18 @@ _PyBytes_FromBuffer(PyObject *x) static PyObject* _PyBytes_FromList(PyObject *x) { - Py_ssize_t i, size = PyList_GET_SIZE(x); - Py_ssize_t value; - char *str; - PyObject *item; - _PyBytesWriter writer; - - _PyBytesWriter_Init(&writer); - str = _PyBytesWriter_Alloc(&writer, size); - if (str == NULL) + Py_ssize_t size = PyList_GET_SIZE(x); + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; - writer.overallocate = 1; - size = writer.allocated; + } + char *str = PyBytesWriter_GetData(writer); + size = _PyBytesWriter_GetAllocated(writer); - for (i = 0; i < PyList_GET_SIZE(x); i++) { - item = PyList_GET_ITEM(x, i); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(x); i++) { + PyObject *item = PyList_GET_ITEM(x, i); Py_INCREF(item); - value = PyNumber_AsSsize_t(item, NULL); + Py_ssize_t value = PyNumber_AsSsize_t(item, NULL); Py_DECREF(item); if (value == -1 && PyErr_Occurred()) goto error; @@ -2884,33 +2910,33 @@ _PyBytes_FromList(PyObject *x) } if (i >= size) { - str = _PyBytesWriter_Resize(&writer, str, size+1); - if (str == NULL) - return NULL; - size = writer.allocated; + str = _PyBytesWriter_ResizeAndUpdatePointer(writer, size + 1, str); + if (str == NULL) { + goto error; + } + size = _PyBytesWriter_GetAllocated(writer); } *str++ = (char) value; } - return _PyBytesWriter_Finish(&writer, str); + return PyBytesWriter_FinishWithPointer(writer, str); - error: - _PyBytesWriter_Dealloc(&writer); +error: + PyBytesWriter_Discard(writer); return NULL; } static PyObject* _PyBytes_FromTuple(PyObject *x) { - PyObject *bytes; Py_ssize_t i, size = PyTuple_GET_SIZE(x); Py_ssize_t value; - char *str; PyObject *item; - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; - str = ((PyBytesObject *)bytes)->ob_sval; + } + char *str = PyBytesWriter_GetData(writer); for (i = 0; i < size; i++) { item = PyTuple_GET_ITEM(x, i); @@ -2925,31 +2951,29 @@ _PyBytes_FromTuple(PyObject *x) } *str++ = (char) value; } - return bytes; + return PyBytesWriter_Finish(writer); error: - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); return NULL; } static PyObject * _PyBytes_FromIterator(PyObject *it, PyObject *x) { - char *str; Py_ssize_t i, size; - _PyBytesWriter writer; /* For iterator version, create a bytes object and resize as needed */ size = PyObject_LengthHint(x, 64); if (size == -1 && PyErr_Occurred()) return NULL; - _PyBytesWriter_Init(&writer); - str = _PyBytesWriter_Alloc(&writer, size); - if (str == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; - writer.overallocate = 1; - size = writer.allocated; + } + char *str = PyBytesWriter_GetData(writer); + size = _PyBytesWriter_GetAllocated(writer); /* Run the iterator to exhaustion */ for (i = 0; ; i++) { @@ -2979,18 +3003,18 @@ _PyBytes_FromIterator(PyObject *it, PyObject *x) /* Append the byte */ if (i >= size) { - str = _PyBytesWriter_Resize(&writer, str, size+1); - if (str == NULL) - return NULL; - size = writer.allocated; + str = _PyBytesWriter_ResizeAndUpdatePointer(writer, size + 1, str); + if (str == NULL) { + goto error; + } + size = _PyBytesWriter_GetAllocated(writer); } *str++ = (char) value; } - - return _PyBytesWriter_Finish(&writer, str); + return PyBytesWriter_FinishWithPointer(writer, str); error: - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return NULL; } @@ -3142,7 +3166,7 @@ PyBytes_Concat(PyObject **pv, PyObject *w) return; } - if (Py_REFCNT(*pv) == 1 && PyBytes_CheckExact(*pv)) { + if (_PyObject_IsUniquelyReferenced(*pv) && PyBytes_CheckExact(*pv)) { /* Only one reference, so we can resize in place */ Py_ssize_t oldsize; Py_buffer wb; @@ -3227,7 +3251,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) Py_DECREF(v); return 0; } - if (Py_REFCNT(v) != 1) { + if (!_PyObject_IsUniquelyReferenced(v)) { if (oldsize < newsize) { *pv = _PyBytes_FromSize(newsize, 0); if (*pv) { @@ -3422,288 +3446,6 @@ bytes_iter(PyObject *seq) } -/* _PyBytesWriter API */ - -#ifdef MS_WINDOWS - /* On Windows, overallocate by 50% is the best factor */ -# define OVERALLOCATE_FACTOR 2 -#else - /* On Linux, overallocate by 25% is the best factor */ -# define OVERALLOCATE_FACTOR 4 -#endif - -void -_PyBytesWriter_Init(_PyBytesWriter *writer) -{ - /* Set all attributes before small_buffer to 0 */ - memset(writer, 0, offsetof(_PyBytesWriter, small_buffer)); -#ifndef NDEBUG - memset(writer->small_buffer, PYMEM_CLEANBYTE, - sizeof(writer->small_buffer)); -#endif -} - -void -_PyBytesWriter_Dealloc(_PyBytesWriter *writer) -{ - Py_CLEAR(writer->buffer); -} - -Py_LOCAL_INLINE(char*) -_PyBytesWriter_AsString(_PyBytesWriter *writer) -{ - if (writer->use_small_buffer) { - assert(writer->buffer == NULL); - return writer->small_buffer; - } - else if (writer->use_bytearray) { - assert(writer->buffer != NULL); - return PyByteArray_AS_STRING(writer->buffer); - } - else { - assert(writer->buffer != NULL); - return PyBytes_AS_STRING(writer->buffer); - } -} - -Py_LOCAL_INLINE(Py_ssize_t) -_PyBytesWriter_GetSize(_PyBytesWriter *writer, char *str) -{ - const char *start = _PyBytesWriter_AsString(writer); - assert(str != NULL); - assert(str >= start); - assert(str - start <= writer->allocated); - return str - start; -} - -#ifndef NDEBUG -Py_LOCAL_INLINE(int) -_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) -{ - const char *start, *end; - - if (writer->use_small_buffer) { - assert(writer->buffer == NULL); - } - else { - assert(writer->buffer != NULL); - if (writer->use_bytearray) - assert(PyByteArray_CheckExact(writer->buffer)); - else - assert(PyBytes_CheckExact(writer->buffer)); - assert(Py_REFCNT(writer->buffer) == 1); - } - - if (writer->use_bytearray) { - /* bytearray has its own overallocation algorithm, - writer overallocation must be disabled */ - assert(!writer->overallocate); - } - - assert(0 <= writer->allocated); - assert(0 <= writer->min_size && writer->min_size <= writer->allocated); - /* the last byte must always be null */ - start = _PyBytesWriter_AsString(writer); - assert(start[writer->allocated] == 0); - - end = start + writer->allocated; - assert(str != NULL); - assert(start <= str && str <= end); - return 1; -} -#endif - -void* -_PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size) -{ - Py_ssize_t allocated, pos; - - assert(_PyBytesWriter_CheckConsistency(writer, str)); - assert(writer->allocated < size); - - allocated = size; - if (writer->overallocate - && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - allocated += allocated / OVERALLOCATE_FACTOR; - } - - pos = _PyBytesWriter_GetSize(writer, str); - if (!writer->use_small_buffer) { - if (writer->use_bytearray) { - if (PyByteArray_Resize(writer->buffer, allocated)) - goto error; - /* writer->allocated can be smaller than writer->buffer->ob_alloc, - but we cannot use ob_alloc because bytes may need to be moved - to use the whole buffer. bytearray uses an internal optimization - to avoid moving or copying bytes when bytes are removed at the - beginning (ex: del bytearray[:1]). */ - } - else { - if (_PyBytes_Resize(&writer->buffer, allocated)) - goto error; - } - } - else { - /* convert from stack buffer to bytes object buffer */ - assert(writer->buffer == NULL); - - if (writer->use_bytearray) - writer->buffer = PyByteArray_FromStringAndSize(NULL, allocated); - else - writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); - if (writer->buffer == NULL) - goto error; - - if (pos != 0) { - char *dest; - if (writer->use_bytearray) - dest = PyByteArray_AS_STRING(writer->buffer); - else - dest = PyBytes_AS_STRING(writer->buffer); - memcpy(dest, - writer->small_buffer, - pos); - } - - writer->use_small_buffer = 0; -#ifndef NDEBUG - memset(writer->small_buffer, PYMEM_CLEANBYTE, - sizeof(writer->small_buffer)); -#endif - } - writer->allocated = allocated; - - str = _PyBytesWriter_AsString(writer) + pos; - assert(_PyBytesWriter_CheckConsistency(writer, str)); - return str; - -error: - _PyBytesWriter_Dealloc(writer); - return NULL; -} - -void* -_PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) -{ - Py_ssize_t new_min_size; - - assert(_PyBytesWriter_CheckConsistency(writer, str)); - assert(size >= 0); - - if (size == 0) { - /* nothing to do */ - return str; - } - - if (writer->min_size > PY_SSIZE_T_MAX - size) { - PyErr_NoMemory(); - _PyBytesWriter_Dealloc(writer); - return NULL; - } - new_min_size = writer->min_size + size; - - if (new_min_size > writer->allocated) - str = _PyBytesWriter_Resize(writer, str, new_min_size); - - writer->min_size = new_min_size; - return str; -} - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -void* -_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) -{ - /* ensure that _PyBytesWriter_Alloc() is only called once */ - assert(writer->min_size == 0 && writer->buffer == NULL); - assert(size >= 0); - - writer->use_small_buffer = 1; -#ifndef NDEBUG - writer->allocated = sizeof(writer->small_buffer) - 1; - /* In debug mode, don't use the full small buffer because it is less - efficient than bytes and bytearray objects to detect buffer underflow - and buffer overflow. Use 10 bytes of the small buffer to test also - code using the smaller buffer in debug mode. - - Don't modify the _PyBytesWriter structure (use a shorter small buffer) - in debug mode to also be able to detect stack overflow when running - tests in debug mode. The _PyBytesWriter is large (more than 512 bytes), - if _Py_EnterRecursiveCall() is not used in deep C callback, we may hit a - stack overflow. */ - writer->allocated = Py_MIN(writer->allocated, 10); - /* _PyBytesWriter_CheckConsistency() requires the last byte to be 0, - to detect buffer overflow */ - writer->small_buffer[writer->allocated] = 0; -#else - writer->allocated = sizeof(writer->small_buffer); -#endif - return _PyBytesWriter_Prepare(writer, writer->small_buffer, size); -} - -PyObject * -_PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) -{ - Py_ssize_t size; - PyObject *result; - - assert(_PyBytesWriter_CheckConsistency(writer, str)); - - size = _PyBytesWriter_GetSize(writer, str); - if (size == 0 && !writer->use_bytearray) { - Py_CLEAR(writer->buffer); - /* Get the empty byte string singleton */ - result = PyBytes_FromStringAndSize(NULL, 0); - } - else if (writer->use_small_buffer) { - if (writer->use_bytearray) { - result = PyByteArray_FromStringAndSize(writer->small_buffer, size); - } - else { - result = PyBytes_FromStringAndSize(writer->small_buffer, size); - } - } - else { - result = writer->buffer; - writer->buffer = NULL; - - if (size != writer->allocated) { - if (writer->use_bytearray) { - if (PyByteArray_Resize(result, size)) { - Py_DECREF(result); - return NULL; - } - } - else { - if (_PyBytes_Resize(&result, size)) { - assert(result == NULL); - return NULL; - } - } - } - } - return result; -} - -void* -_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *ptr, - const void *bytes, Py_ssize_t size) -{ - char *str = (char *)ptr; - - str = _PyBytesWriter_Prepare(writer, str, size); - if (str == NULL) - return NULL; - - memcpy(str, bytes, size); - str += size; - - return str; -} - - void _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src) @@ -3727,3 +3469,341 @@ _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, } } + +// --- PyBytesWriter API ----------------------------------------------------- + +static inline char* +byteswriter_data(PyBytesWriter *writer) +{ + return _PyBytesWriter_GetData(writer); +} + + +static inline Py_ssize_t +byteswriter_allocated(PyBytesWriter *writer) +{ + if (writer->obj == NULL) { + return sizeof(writer->small_buffer); + } + else if (writer->use_bytearray) { + return PyByteArray_GET_SIZE(writer->obj); + } + else { + return PyBytes_GET_SIZE(writer->obj); + } +} + + +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ +# define OVERALLOCATE_FACTOR 2 +#else + /* On Linux, overallocate by 25% is the best factor */ +# define OVERALLOCATE_FACTOR 4 +#endif + +static inline int +byteswriter_resize(PyBytesWriter *writer, Py_ssize_t size, int resize) +{ + assert(size >= 0); + + Py_ssize_t old_allocated = byteswriter_allocated(writer); + if (size <= old_allocated) { + return 0; + } + + if (resize & writer->overallocate) { + if (size <= (PY_SSIZE_T_MAX - size / OVERALLOCATE_FACTOR)) { + size += size / OVERALLOCATE_FACTOR; + } + } + + if (writer->obj != NULL) { + if (writer->use_bytearray) { + if (PyByteArray_Resize(writer->obj, size)) { + return -1; + } + } + else { + if (_PyBytes_Resize(&writer->obj, size)) { + return -1; + } + } + assert(writer->obj != NULL); + } + else if (writer->use_bytearray) { + writer->obj = PyByteArray_FromStringAndSize(NULL, size); + if (writer->obj == NULL) { + return -1; + } + if (resize) { + assert((size_t)size > sizeof(writer->small_buffer)); + memcpy(PyByteArray_AS_STRING(writer->obj), + writer->small_buffer, + sizeof(writer->small_buffer)); + } + } + else { + writer->obj = PyBytes_FromStringAndSize(NULL, size); + if (writer->obj == NULL) { + return -1; + } + if (resize) { + assert((size_t)size > sizeof(writer->small_buffer)); + memcpy(PyBytes_AS_STRING(writer->obj), + writer->small_buffer, + sizeof(writer->small_buffer)); + } + } + +#ifdef Py_DEBUG + Py_ssize_t allocated = byteswriter_allocated(writer); + if (resize && allocated > old_allocated) { + memset(byteswriter_data(writer) + old_allocated, 0xff, + allocated - old_allocated); + } +#endif + + return 0; +} + + +static PyBytesWriter* +byteswriter_create(Py_ssize_t size, int use_bytearray) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be >= 0"); + return NULL; + } + + PyBytesWriter *writer = _Py_FREELIST_POP_MEM(bytes_writers); + if (writer == NULL) { + writer = (PyBytesWriter *)PyMem_Malloc(sizeof(PyBytesWriter)); + if (writer == NULL) { + PyErr_NoMemory(); + return NULL; + } + } + writer->obj = NULL; + writer->size = 0; + writer->use_bytearray = use_bytearray; + writer->overallocate = !use_bytearray; + + if (size >= 1) { + if (byteswriter_resize(writer, size, 0) < 0) { + PyBytesWriter_Discard(writer); + return NULL; + } + writer->size = size; + } +#ifdef Py_DEBUG + memset(byteswriter_data(writer), 0xff, byteswriter_allocated(writer)); +#endif + return writer; +} + +PyBytesWriter* +PyBytesWriter_Create(Py_ssize_t size) +{ + return byteswriter_create(size, 0); +} + +PyBytesWriter* +_PyBytesWriter_CreateByteArray(Py_ssize_t size) +{ + return byteswriter_create(size, 1); +} + + +void +PyBytesWriter_Discard(PyBytesWriter *writer) +{ + if (writer == NULL) { + return; + } + + Py_XDECREF(writer->obj); + _Py_FREELIST_FREE(bytes_writers, writer, PyMem_Free); +} + + +PyObject* +PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) +{ + PyObject *result; + if (size == 0) { + result = bytes_get_empty(); + } + else if (writer->obj != NULL) { + if (writer->use_bytearray) { + if (size != PyByteArray_GET_SIZE(writer->obj)) { + if (PyByteArray_Resize(writer->obj, size)) { + goto error; + } + } + } + else { + if (size != PyBytes_GET_SIZE(writer->obj)) { + if (_PyBytes_Resize(&writer->obj, size)) { + goto error; + } + } + } + result = writer->obj; + writer->obj = NULL; + } + else if (writer->use_bytearray) { + result = PyByteArray_FromStringAndSize(writer->small_buffer, size); + } + else { + result = PyBytes_FromStringAndSize(writer->small_buffer, size); + } + PyBytesWriter_Discard(writer); + return result; + +error: + PyBytesWriter_Discard(writer); + return NULL; +} + +PyObject* +PyBytesWriter_Finish(PyBytesWriter *writer) +{ + return PyBytesWriter_FinishWithSize(writer, writer->size); +} + + +PyObject* +PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) +{ + Py_ssize_t size = (char*)buf - byteswriter_data(writer); + if (size < 0 || size > byteswriter_allocated(writer)) { + PyBytesWriter_Discard(writer); + PyErr_SetString(PyExc_ValueError, "invalid end pointer"); + return NULL; + } + + return PyBytesWriter_FinishWithSize(writer, size); +} + + +void* +PyBytesWriter_GetData(PyBytesWriter *writer) +{ + return byteswriter_data(writer); +} + + +Py_ssize_t +PyBytesWriter_GetSize(PyBytesWriter *writer) +{ + return _PyBytesWriter_GetSize(writer); +} + + +static Py_ssize_t +_PyBytesWriter_GetAllocated(PyBytesWriter *writer) +{ + return byteswriter_allocated(writer); +} + + +int +PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be >= 0"); + return -1; + } + if (byteswriter_resize(writer, size, 1) < 0) { + return -1; + } + writer->size = size; + return 0; +} + + +static void* +_PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, + void *data) +{ + Py_ssize_t pos = (char*)data - byteswriter_data(writer); + if (PyBytesWriter_Resize(writer, size) < 0) { + return NULL; + } + return byteswriter_data(writer) + pos; +} + + +int +PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t size) +{ + if (size < 0 && writer->size + size < 0) { + PyErr_SetString(PyExc_ValueError, "invalid size"); + return -1; + } + if (size > PY_SSIZE_T_MAX - writer->size) { + PyErr_NoMemory(); + return -1; + } + size = writer->size + size; + + if (byteswriter_resize(writer, size, 1) < 0) { + return -1; + } + writer->size = size; + return 0; +} + + +void* +PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, + void *buf) +{ + Py_ssize_t pos = (char*)buf - byteswriter_data(writer); + if (PyBytesWriter_Grow(writer, size) < 0) { + return NULL; + } + return byteswriter_data(writer) + pos; +} + + +int +PyBytesWriter_WriteBytes(PyBytesWriter *writer, + const void *bytes, Py_ssize_t size) +{ + if (size < 0) { + size_t len = strlen(bytes); + if (len > (size_t)PY_SSIZE_T_MAX) { + PyErr_NoMemory(); + return -1; + } + size = (Py_ssize_t)len; + } + + Py_ssize_t pos = writer->size; + if (PyBytesWriter_Grow(writer, size) < 0) { + return -1; + } + char *buf = byteswriter_data(writer); + memcpy(buf + pos, bytes, size); + return 0; +} + + +int +PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) +{ + Py_ssize_t pos = writer->size; + if (PyBytesWriter_Grow(writer, strlen(format)) < 0) { + return -1; + } + + va_list vargs; + va_start(vargs, format); + char *buf = bytes_fromformat(writer, pos, format, vargs); + va_end(vargs); + + Py_ssize_t size = buf - byteswriter_data(writer); + return PyBytesWriter_Resize(writer, size); +} diff --git a/Objects/call.c b/Objects/call.c index b1610dababd..c69015abfb3 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -213,7 +213,7 @@ _PyObject_MakeTpCall(PyThreadState *tstate, PyObject *callable, return NULL; } - PyObject *argstuple = _PyTuple_FromArray(args, nargs); + PyObject *argstuple = PyTuple_FromArray(args, nargs); if (argstuple == NULL) { return NULL; } @@ -834,12 +834,15 @@ PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, assert(PyVectorcall_NARGS(nargsf) >= 1); PyThreadState *tstate = _PyThreadState_GET(); - PyObject *callable = NULL; + _PyCStackRef method; + _PyThreadState_PushCStackRef(tstate, &method); /* Use args[0] as "self" argument */ - int unbound = _PyObject_GetMethod(args[0], name, &callable); - if (callable == NULL) { + int unbound = _PyObject_GetMethodStackRef(tstate, args[0], name, &method.ref); + if (PyStackRef_IsNull(method.ref)) { + _PyThreadState_PopCStackRef(tstate, &method); return NULL; } + PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); if (unbound) { /* We must remove PY_VECTORCALL_ARGUMENTS_OFFSET since @@ -855,7 +858,7 @@ PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_METHOD, callable); PyObject *result = _PyObject_VectorcallTstate(tstate, callable, args, nargsf, kwnames); - Py_DECREF(callable); + _PyThreadState_PopCStackRef(tstate, &method); return result; } @@ -868,11 +871,14 @@ PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) return null_error(tstate); } - PyObject *callable = NULL; - int is_method = _PyObject_GetMethod(obj, name, &callable); - if (callable == NULL) { + _PyCStackRef method; + _PyThreadState_PushCStackRef(tstate, &method); + int is_method = _PyObject_GetMethodStackRef(tstate, obj, name, &method.ref); + if (PyStackRef_IsNull(method.ref)) { + _PyThreadState_PopCStackRef(tstate, &method); return NULL; } + PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); obj = is_method ? obj : NULL; va_list vargs; @@ -880,37 +886,7 @@ PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) PyObject *result = object_vacall(tstate, obj, callable, vargs); va_end(vargs); - Py_DECREF(callable); - return result; -} - - -PyObject * -_PyObject_CallMethodIdObjArgs(PyObject *obj, _Py_Identifier *name, ...) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (obj == NULL || name == NULL) { - return null_error(tstate); - } - - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return NULL; - } - - PyObject *callable = NULL; - int is_method = _PyObject_GetMethod(obj, oname, &callable); - if (callable == NULL) { - return NULL; - } - obj = is_method ? obj : NULL; - - va_list vargs; - va_start(vargs, name); - PyObject *result = object_vacall(tstate, obj, callable, vargs); - va_end(vargs); - - Py_DECREF(callable); + _PyThreadState_PopCStackRef(tstate, &method); return result; } diff --git a/Objects/classobject.c b/Objects/classobject.c index 58e1d179773..e71f301f2ef 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -7,6 +7,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/classobject.c.h" @@ -245,8 +246,7 @@ method_dealloc(PyObject *self) { PyMethodObject *im = _PyMethodObject_CAST(self); _PyObject_GC_UNTRACK(im); - if (im->im_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)im); + FT_CLEAR_WEAKREFS(self, im->im_weakreflist); Py_DECREF(im->im_func); Py_XDECREF(im->im_self); assert(Py_IS_TYPE(self, &PyMethod_Type)); diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index ffb45ade11f..be704ccf68f 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -599,7 +599,7 @@ PyDoc_STRVAR(bytearray_resize__doc__, "Resize the internal buffer of bytearray to len.\n" "\n" " size\n" -" New size to resize to.."); +" New size to resize to."); #define BYTEARRAY_RESIZE_METHODDEF \ {"resize", (PyCFunction)bytearray_resize, METH_O, bytearray_resize__doc__}, @@ -631,6 +631,43 @@ exit: return return_value; } +PyDoc_STRVAR(bytearray_take_bytes__doc__, +"take_bytes($self, n=None, /)\n" +"--\n" +"\n" +"Take *n* bytes from the bytearray and return them as a bytes object.\n" +"\n" +" n\n" +" Bytes to take, negative indexes from end. None indicates all bytes."); + +#define BYTEARRAY_TAKE_BYTES_METHODDEF \ + {"take_bytes", _PyCFunction_CAST(bytearray_take_bytes), METH_FASTCALL, bytearray_take_bytes__doc__}, + +static PyObject * +bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n); + +static PyObject * +bytearray_take_bytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *n = Py_None; + + if (!_PyArg_CheckPositional("take_bytes", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + n = args[0]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = bytearray_take_bytes_impl((PyByteArrayObject *)self, n); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_translate__doc__, "translate($self, table, /, delete=b\'\')\n" "--\n" @@ -1796,4 +1833,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=be6d28193bc96a2c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5eddefde2a001ceb input=a9049054013a1b77]*/ diff --git a/Objects/clinic/enumobject.c.h b/Objects/clinic/enumobject.c.h index 29ff11e89b9..1bda482f495 100644 --- a/Objects/clinic/enumobject.c.h +++ b/Objects/clinic/enumobject.c.h @@ -82,7 +82,7 @@ exit: } PyDoc_STRVAR(reversed_new__doc__, -"reversed(sequence, /)\n" +"reversed(object, /)\n" "--\n" "\n" "Return a reverse iterator over the values of the given sequence."); @@ -110,4 +110,4 @@ reversed_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=3300305b351674d3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=155cc9483d5f9eab input=a9049054013a1b77]*/ diff --git a/Objects/clinic/interpolationobject.c.h b/Objects/clinic/interpolationobject.c.h index 7a94dabafc9..2030e17e49e 100644 --- a/Objects/clinic/interpolationobject.c.h +++ b/Objects/clinic/interpolationobject.c.h @@ -47,26 +47,31 @@ interpolation_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *argsbuf[4]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *value; - PyObject *expression; + PyObject *expression = &_Py_STR(empty); PyObject *conversion = Py_None; PyObject *format_spec = &_Py_STR(empty); fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } value = fastargs[0]; - if (!PyUnicode_Check(fastargs[1])) { - _PyArg_BadArgument("Interpolation", "argument 'expression'", "str", fastargs[1]); - goto exit; - } - expression = fastargs[1]; if (!noptargs) { goto skip_optional_pos; } + if (fastargs[1]) { + if (!PyUnicode_Check(fastargs[1])) { + _PyArg_BadArgument("Interpolation", "argument 'expression'", "str", fastargs[1]); + goto exit; + } + expression = fastargs[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } if (fastargs[2]) { if (!_conversion_converter(fastargs[2], &conversion)) { goto exit; @@ -86,4 +91,4 @@ skip_optional_pos: exit: return return_value; } -/*[clinic end generated code: output=599742a5ccd6f060 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2391391e2d7708c0 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index a236a32c091..c88772030ec 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -340,6 +340,11 @@ int_to_bytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject * goto exit; } length = ival; + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length cannot be negative"); + goto exit; + } } if (!--noptargs) { goto skip_optional_pos; @@ -485,4 +490,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=d23f8ce5bdf08a30 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e68f4e23ead3f649 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index e71c29a1b26..894e9be91bb 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(OrderedDict_fromkeys__doc__, @@ -73,6 +74,53 @@ exit: return return_value; } +PyDoc_STRVAR(OrderedDict___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n"); + +#define ORDEREDDICT___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)OrderedDict___sizeof__, METH_NOARGS, OrderedDict___sizeof____doc__}, + +static Py_ssize_t +OrderedDict___sizeof___impl(PyODictObject *self); + +static PyObject * +OrderedDict___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + Py_ssize_t _return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + _return_value = OrderedDict___sizeof___impl((PyODictObject *)self); + Py_END_CRITICAL_SECTION(); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(OrderedDict___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling"); + +#define ORDEREDDICT___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)OrderedDict___reduce__, METH_NOARGS, OrderedDict___reduce____doc__}, + +static PyObject * +OrderedDict___reduce___impl(PyODictObject *od); + +static PyObject * +OrderedDict___reduce__(PyObject *od, PyObject *Py_UNUSED(ignored)) +{ + return OrderedDict___reduce___impl((PyODictObject *)od); +} + PyDoc_STRVAR(OrderedDict_setdefault__doc__, "setdefault($self, /, key, default=None)\n" "--\n" @@ -135,7 +183,9 @@ OrderedDict_setdefault(PyObject *self, PyObject *const *args, Py_ssize_t nargs, } default_value = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = OrderedDict_setdefault_impl((PyODictObject *)self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -204,7 +254,9 @@ OrderedDict_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObjec } default_value = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = OrderedDict_pop_impl((PyODictObject *)self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -272,12 +324,62 @@ OrderedDict_popitem(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyO goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = OrderedDict_popitem_impl((PyODictObject *)self, last); + Py_END_CRITICAL_SECTION(); exit: return return_value; } +PyDoc_STRVAR(OrderedDict_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from ordered dict."); + +#define ORDEREDDICT_CLEAR_METHODDEF \ + {"clear", (PyCFunction)OrderedDict_clear, METH_NOARGS, OrderedDict_clear__doc__}, + +static PyObject * +OrderedDict_clear_impl(PyODictObject *self); + +static PyObject * +OrderedDict_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = OrderedDict_clear_impl((PyODictObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(OrderedDict_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"A shallow copy of ordered dict."); + +#define ORDEREDDICT_COPY_METHODDEF \ + {"copy", (PyCFunction)OrderedDict_copy, METH_NOARGS, OrderedDict_copy__doc__}, + +static PyObject * +OrderedDict_copy_impl(PyObject *od); + +static PyObject * +OrderedDict_copy(PyObject *od, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(od); + return_value = OrderedDict_copy_impl(od); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(OrderedDict_move_to_end__doc__, "move_to_end($self, /, key, last=True)\n" "--\n" @@ -342,9 +444,11 @@ OrderedDict_move_to_end(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = OrderedDict_move_to_end_impl((PyODictObject *)self, key, last); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=7d8206823bb1f419 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7bc997ca7900f06f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 9ae2f51f758..bd4c7a0e64f 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -688,14 +688,14 @@ typealias_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(typealias_new__doc__, -"typealias(name, value, *, type_params=<unrepresentable>)\n" +"typealias(name, value, *, type_params=<unrepresentable>, qualname=None)\n" "--\n" "\n" "Create a TypeAliasType."); static PyObject * typealias_new_impl(PyTypeObject *type, PyObject *name, PyObject *value, - PyObject *type_params); + PyObject *type_params, PyObject *qualname); static PyObject * typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -703,7 +703,7 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -712,7 +712,7 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(value), &_Py_ID(type_params), }, + .ob_item = { &_Py_ID(name), &_Py_ID(value), &_Py_ID(type_params), &_Py_ID(qualname), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -721,20 +721,21 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "value", "type_params", NULL}; + static const char * const _keywords[] = {"name", "value", "type_params", "qualname", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typealias", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; PyObject *name; PyObject *value; PyObject *type_params = NULL; + PyObject *qualname = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -750,11 +751,17 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!noptargs) { goto skip_optional_kwonly; } - type_params = fastargs[2]; + if (fastargs[2]) { + type_params = fastargs[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + qualname = fastargs[3]; skip_optional_kwonly: - return_value = typealias_new_impl(type, name, value, type_params); + return_value = typealias_new_impl(type, name, value, type_params, qualname); exit: return return_value; } -/*[clinic end generated code: output=9dad71445e079303 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=67ab9a5d1869f2c9 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 92eaf5e00bc..3aea2038fd1 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -17,6 +17,7 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() #include "pycore_uniqueid.h" // _PyObject_AssignUniqueId() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/codeobject.c.h" #include <stdbool.h> @@ -549,16 +550,12 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->func_state.mutex); -#endif + FT_MUTEX_LOCK(&interp->func_state.mutex); co->co_version = interp->func_state.next_version; if (interp->func_state.next_version != 0) { interp->func_state.next_version++; } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->func_state.mutex); -#endif + FT_MUTEX_UNLOCK(&interp->func_state.mutex); co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ @@ -688,7 +685,7 @@ intern_code_constants(struct _PyCodeConstructor *con) #ifdef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); struct _py_code_state *state = &interp->code_state; - PyMutex_Lock(&state->mutex); + FT_MUTEX_LOCK(&state->mutex); #endif if (intern_strings(con->names) < 0) { goto error; @@ -699,15 +696,11 @@ intern_code_constants(struct _PyCodeConstructor *con) if (intern_strings(con->localsplusnames) < 0) { goto error; } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&state->mutex); -#endif + FT_MUTEX_UNLOCK(&state->mutex); return 0; error: -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&state->mutex); -#endif + FT_MUTEX_UNLOCK(&state->mutex); return -1; } @@ -1012,8 +1005,8 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) * source location tracking (co_lines/co_positions) ******************/ -int -PyCode_Addr2Line(PyCodeObject *co, int addrq) +static int +_PyCode_Addr2Line(PyCodeObject *co, int addrq) { if (addrq < 0) { return co->co_firstlineno; @@ -1027,6 +1020,33 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) return _PyCode_CheckLineNumber(addrq, &bounds); } +int +_PyCode_SafeAddr2Line(PyCodeObject *co, int addrq) +{ + if (addrq < 0) { + return co->co_firstlineno; + } + if (co->_co_monitoring && co->_co_monitoring->lines) { + return _Py_Instrumentation_GetLine(co, addrq/sizeof(_Py_CODEUNIT)); + } + if (!(addrq >= 0 && addrq < _PyCode_NBYTES(co))) { + return -1; + } + PyCodeAddressRange bounds; + _PyCode_InitAddressRange(co, &bounds); + return _PyCode_CheckLineNumber(addrq, &bounds); +} + +int +PyCode_Addr2Line(PyCodeObject *co, int addrq) +{ + int lineno; + Py_BEGIN_CRITICAL_SECTION(co); + lineno = _PyCode_Addr2Line(co, addrq); + Py_END_CRITICAL_SECTION(); + return lineno; +} + void _PyLineTable_InitAddressRange(const char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range) { @@ -1714,7 +1734,7 @@ static int identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, PyObject *globalnames, PyObject *attrnames, PyObject *globalsns, PyObject *builtinsns, - struct co_unbound_counts *counts) + struct co_unbound_counts *counts, int *p_numdupes) { // This function is inspired by inspect.getclosurevars(). // It would be nicer if we had something similar to co_localspluskinds, @@ -1729,6 +1749,7 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, assert(builtinsns == NULL || PyDict_Check(builtinsns)); assert(counts == NULL || counts->total == 0); struct co_unbound_counts unbound = {0}; + int numdupes = 0; Py_ssize_t len = Py_SIZE(co); for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); @@ -1747,6 +1768,12 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, if (PySet_Add(attrnames, name) < 0) { return -1; } + if (PySet_Contains(globalnames, name)) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + numdupes += 1; + } } else if (inst.op.code == LOAD_GLOBAL) { int oparg = GET_OPARG(co, i, inst.op.arg); @@ -1778,11 +1805,20 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, if (PySet_Add(globalnames, name) < 0) { return -1; } + if (PySet_Contains(attrnames, name)) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + numdupes += 1; + } } } if (counts != NULL) { *counts = unbound; } + if (p_numdupes != NULL) { + *p_numdupes = numdupes; + } return 0; } @@ -1932,20 +1968,24 @@ _PyCode_SetUnboundVarCounts(PyThreadState *tstate, // Fill in unbound.globals and unbound.numattrs. struct co_unbound_counts unbound = {0}; + int numdupes = 0; Py_BEGIN_CRITICAL_SECTION(co); res = identify_unbound_names( tstate, co, globalnames, attrnames, globalsns, builtinsns, - &unbound); + &unbound, &numdupes); Py_END_CRITICAL_SECTION(); if (res < 0) { goto finally; } assert(unbound.numunknown == 0); - assert(unbound.total <= counts->unbound.total); + assert(unbound.total - numdupes <= counts->unbound.total); assert(counts->unbound.numunknown == counts->unbound.total); - unbound.numunknown = counts->unbound.total - unbound.total; - unbound.total = counts->unbound.total; + // There may be a name that is both a global and an attr. + int totalunbound = counts->unbound.total + numdupes; + unbound.numunknown = totalunbound - unbound.total; + unbound.total = totalunbound; counts->unbound = unbound; + counts->total += numdupes; res = 0; finally: @@ -1955,12 +1995,129 @@ _PyCode_SetUnboundVarCounts(PyThreadState *tstate, } +int +_PyCode_CheckNoInternalState(PyCodeObject *co, const char **p_errmsg) +{ + const char *errmsg = NULL; + // We don't worry about co_executors, co_instrumentation, + // or co_monitoring. They are essentially ephemeral. + if (co->co_extra != NULL) { + errmsg = "only basic code objects are supported"; + } + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + +int +_PyCode_CheckNoExternalState(PyCodeObject *co, _PyCode_var_counts_t *counts, + const char **p_errmsg) +{ + const char *errmsg = NULL; + if (counts->numfree > 0) { // It's a closure. + errmsg = "closures not supported"; + } + else if (counts->unbound.globals.numglobal > 0) { + errmsg = "globals not supported"; + } + else if (counts->unbound.globals.numbuiltin > 0 + && counts->unbound.globals.numunknown > 0) + { + errmsg = "globals not supported"; + } + // Otherwise we don't check counts.unbound.globals.numunknown since we can't + // distinguish beween globals and builtins here. + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + +int +_PyCode_VerifyStateless(PyThreadState *tstate, + PyCodeObject *co, PyObject *globalnames, + PyObject *globalsns, PyObject *builtinsns) +{ + const char *errmsg; + _PyCode_var_counts_t counts = {0}; + _PyCode_GetVarCounts(co, &counts); + if (_PyCode_SetUnboundVarCounts( + tstate, co, &counts, globalnames, NULL, + globalsns, builtinsns) < 0) + { + return -1; + } + // We may consider relaxing the internal state constraints + // if it becomes a problem. + if (!_PyCode_CheckNoInternalState(co, &errmsg)) { + _PyErr_SetString(tstate, PyExc_ValueError, errmsg); + return -1; + } + if (builtinsns != NULL) { + // Make sure the next check will fail for globals, + // even if there aren't any builtins. + counts.unbound.globals.numbuiltin += 1; + } + if (!_PyCode_CheckNoExternalState(co, &counts, &errmsg)) { + _PyErr_SetString(tstate, PyExc_ValueError, errmsg); + return -1; + } + // Note that we don't check co->co_flags & CO_NESTED for anything here. + return 0; +} + + +int +_PyCode_CheckPureFunction(PyCodeObject *co, const char **p_errmsg) +{ + const char *errmsg = NULL; + if (co->co_flags & CO_GENERATOR) { + errmsg = "generators not supported"; + } + else if (co->co_flags & CO_COROUTINE) { + errmsg = "coroutines not supported"; + } + else if (co->co_flags & CO_ITERABLE_COROUTINE) { + errmsg = "coroutines not supported"; + } + else if (co->co_flags & CO_ASYNC_GENERATOR) { + errmsg = "generators not supported"; + } + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + /* Here "value" means a non-None value, since a bare return is identical * to returning None explicitly. Likewise a missing return statement * at the end of the function is turned into "return None". */ static int code_returns_only_none(PyCodeObject *co) { + if (!_PyCode_CheckPureFunction(co, NULL)) { + return 0; + } + int len = (int)Py_SIZE(co); + assert(len > 0); + + // The last instruction either returns or raises. We can take advantage + // of that for a quick exit. + _Py_CODEUNIT final = _Py_GetBaseCodeUnit(co, len-1); + // Look up None in co_consts. Py_ssize_t nconsts = PyTuple_Size(co->co_consts); int none_index = 0; @@ -1971,27 +2128,43 @@ code_returns_only_none(PyCodeObject *co) } if (none_index == nconsts) { // None wasn't there, which means there was no implicit return, - // "return", or "return None". That means there must be - // an explicit return (non-None). - return 0; - } + // "return", or "return None". - // Walk the bytecode, looking for RETURN_VALUE. - Py_ssize_t len = Py_SIZE(co); - for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { - _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); - if (IS_RETURN_OPCODE(inst.op.code)) { - assert(i != 0); - // Ignore it if it returns None. - _Py_CODEUNIT prev = _Py_GetBaseCodeUnit(co, i-1); - if (prev.op.code == LOAD_CONST) { - // We don't worry about EXTENDED_ARG for now. - if (prev.op.arg == none_index) { - continue; - } - } + // That means there must be + // an explicit return (non-None), or it only raises. + if (IS_RETURN_OPCODE(final.op.code)) { + // It was an explicit return (non-None). return 0; } + // It must end with a raise then. We still have to walk the + // bytecode to see if there's any explicit return (non-None). + assert(IS_RAISE_OPCODE(final.op.code)); + for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); + if (IS_RETURN_OPCODE(inst.op.code)) { + // We alraedy know it isn't returning None. + return 0; + } + } + // It must only raise. + } + else { + // Walk the bytecode, looking for RETURN_VALUE. + for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); + if (IS_RETURN_OPCODE(inst.op.code)) { + assert(i != 0); + // Ignore it if it returns None. + _Py_CODEUNIT prev = _Py_GetBaseCodeUnit(co, i-1); + if (prev.op.code == LOAD_CONST) { + // We don't worry about EXTENDED_ARG for now. + if (prev.op.arg == none_index) { + continue; + } + } + return 0; + } + } } return 1; } @@ -2230,6 +2403,8 @@ free_monitoring_data(_PyCoMonitoringData *data) static void code_dealloc(PyObject *self) { + PyThreadState *tstate = PyThreadState_GET(); + _Py_atomic_add_uint64(&tstate->interp->_code_object_generation, 1); PyCodeObject *co = _PyCodeObject_CAST(self); _PyObject_ResurrectStart(self); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); @@ -2257,6 +2432,7 @@ code_dealloc(PyObject *self) PyMem_Free(co_extra); } #ifdef _Py_TIER2 + _PyJit_Tracer_InvalidateDependency(tstate, self); if (co->co_executors != NULL) { clear_executors(co); } @@ -2281,9 +2457,7 @@ code_dealloc(PyObject *self) Py_XDECREF(co->_co_cached->_co_varnames); PyMem_Free(co->_co_cached); } - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, co->co_weakreflist); free_monitoring_data(co->_co_monitoring); #ifdef Py_GIL_DISABLED // The first element always points to the mutable bytecode at the end of @@ -2569,6 +2743,7 @@ code_branchesiterator(PyObject *self, PyObject *Py_UNUSED(args)) } /*[clinic input] +@permit_long_summary @text_signature "($self, /, **changes)" code.replace @@ -2605,7 +2780,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, PyObject *co_filename, PyObject *co_name, PyObject *co_qualname, PyObject *co_linetable, PyObject *co_exceptiontable) -/*[clinic end generated code: output=e75c48a15def18b9 input=a455a89c57ac9d42]*/ +/*[clinic end generated code: output=e75c48a15def18b9 input=e944fdac8b456114]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -3176,12 +3351,29 @@ _PyCodeArray_New(Py_ssize_t size) return arr; } +// Get the underlying code unit, leaving instrumentation +static _Py_CODEUNIT +deopt_code_unit(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i; + _Py_CODEUNIT inst = { + .cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)}; + int opcode = inst.op.code; + if (opcode < MIN_INSTRUMENTED_OPCODE) { + inst.op.code = _PyOpcode_Deopt[opcode]; + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + } + // JIT should not be enabled with free-threading + assert(inst.op.code != ENTER_EXECUTOR); + return inst; +} + static void copy_code(_Py_CODEUNIT *dst, PyCodeObject *co) { int code_len = (int) Py_SIZE(co); for (int i = 0; i < code_len; i += _PyInstruction_GetLength(co, i)) { - dst[i] = _Py_GetBaseCodeUnit(co, i); + dst[i] = deopt_code_unit(co, i); } _PyCode_Quicken(dst, code_len, 1); } @@ -3214,7 +3406,7 @@ create_tlbc_lock_held(PyCodeObject *co, Py_ssize_t idx) } memcpy(new_tlbc->entries, tlbc->entries, tlbc->size * sizeof(void *)); _Py_atomic_store_ptr_release(&co->co_tlbc, new_tlbc); - _PyMem_FreeDelayed(tlbc); + _PyMem_FreeDelayed(tlbc, tlbc->size * sizeof(void *)); tlbc = new_tlbc; } char *bc = PyMem_Calloc(1, _PyCode_NBYTES(co)); diff --git a/Objects/complexobject.c b/Objects/complexobject.c index c2dd320ae73..3612c2699a5 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -1,4 +1,3 @@ - /* Complex object implementation */ /* Borrows heavily from floatobject.c */ @@ -9,6 +8,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_complexobject.h" // _PyComplex_FormatAdvancedWriter() #include "pycore_floatobject.h" // _Py_convert_int_to_double() +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _Py_ADJUST_ERANGE2() @@ -139,8 +139,8 @@ _Py_c_prod(Py_complex z, Py_complex w) recalc = 1; } if (recalc) { - r.real = Py_INFINITY*(a*c - b*d); - r.imag = Py_INFINITY*(a*d + b*c); + r.real = INFINITY*(a*c - b*d); + r.imag = INFINITY*(a*d + b*c); } } @@ -229,8 +229,8 @@ _Py_c_quot(Py_complex a, Py_complex b) { const double x = copysign(isinf(a.real) ? 1.0 : 0.0, a.real); const double y = copysign(isinf(a.imag) ? 1.0 : 0.0, a.imag); - r.real = Py_INFINITY * (x*b.real + y*b.imag); - r.imag = Py_INFINITY * (y*b.real - x*b.imag); + r.real = INFINITY * (x*b.real + y*b.imag); + r.imag = INFINITY * (y*b.real - x*b.imag); } else if ((isinf(abs_breal) || isinf(abs_bimag)) && isfinite(a.real) && isfinite(a.imag)) @@ -410,16 +410,32 @@ complex_subtype_from_c_complex(PyTypeObject *type, Py_complex cval) PyObject * PyComplex_FromCComplex(Py_complex cval) { - /* Inline PyObject_New */ - PyComplexObject *op = PyObject_Malloc(sizeof(PyComplexObject)); + PyComplexObject *op = _Py_FREELIST_POP(PyComplexObject, complexes); + if (op == NULL) { - return PyErr_NoMemory(); + /* Inline PyObject_New */ + op = PyObject_Malloc(sizeof(PyComplexObject)); + if (op == NULL) { + return PyErr_NoMemory(); + } + _PyObject_Init((PyObject*)op, &PyComplex_Type); } - _PyObject_Init((PyObject*)op, &PyComplex_Type); op->cval = cval; return (PyObject *) op; } +static void +complex_dealloc(PyObject *op) +{ + assert(PyComplex_Check(op)); + if (PyComplex_CheckExact(op)) { + _Py_FREELIST_FREE(complexes, op, PyObject_Free); + } + else { + Py_TYPE(op)->tp_free(op); + } +} + static PyObject * complex_subtype_from_doubles(PyTypeObject *type, double real, double imag) { @@ -499,17 +515,17 @@ try_complex_special_method(PyObject *op) } if (!PyComplex_Check(res)) { PyErr_Format(PyExc_TypeError, - "__complex__ returned non-complex (type %.200s)", - Py_TYPE(res)->tp_name); + "%T.__complex__() must return a complex, not %T", + op, res); Py_DECREF(res); return NULL; } /* Issue #29894: warn if 'res' not of exact type complex. */ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "__complex__ returned non-complex (type %.200s). " + "%T.__complex__() must return a complex, not %T. " "The ability to return an instance of a strict subclass of complex " "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(res)->tp_name)) { + op, res)) { Py_DECREF(res); return NULL; } @@ -628,7 +644,7 @@ complex_hash(PyObject *op) * compare equal must have the same hash value, so that * hash(x + 0*j) must equal hash(x). */ - combined = hashreal + _PyHASH_IMAG * hashimag; + combined = hashreal + PyHASH_IMAG * hashimag; if (combined == (Py_uhash_t)-1) combined = (Py_uhash_t)-2; return (Py_hash_t)combined; @@ -853,6 +869,7 @@ complex_richcompare(PyObject *v, PyObject *w, int op) } /*[clinic input] +@permit_long_summary complex.conjugate Return the complex conjugate of its argument. (3-4j).conjugate() == 3+4j. @@ -860,7 +877,7 @@ Return the complex conjugate of its argument. (3-4j).conjugate() == 3+4j. static PyObject * complex_conjugate_impl(PyComplexObject *self) -/*[clinic end generated code: output=5059ef162edfc68e input=5fea33e9747ec2c4]*/ +/*[clinic end generated code: output=5059ef162edfc68e input=71b8ab003e1cec95]*/ { Py_complex c = self->cval; c.imag = -c.imag; @@ -1383,7 +1400,7 @@ PyTypeObject PyComplex_Type = { "complex", sizeof(PyComplexObject), 0, - 0, /* tp_dealloc */ + complex_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 10c465b95ac..5ac4fbd8129 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -39,41 +39,41 @@ descr_name(PyDescrObject *descr) } static PyObject * -descr_repr(PyDescrObject *descr, const char *format) +descr_repr(PyDescrObject *descr, const char *kind) { PyObject *name = NULL; if (descr->d_name != NULL && PyUnicode_Check(descr->d_name)) name = descr->d_name; - return PyUnicode_FromFormat(format, name, "?", descr->d_type->tp_name); + if (descr->d_type == &PyBaseObject_Type) { + return PyUnicode_FromFormat("<%s '%V'>", kind, name, "?"); + } + return PyUnicode_FromFormat("<%s '%V' of '%s' objects>", + kind, name, "?", descr->d_type->tp_name); } static PyObject * method_repr(PyObject *descr) { - return descr_repr((PyDescrObject *)descr, - "<method '%V' of '%s' objects>"); + return descr_repr((PyDescrObject *)descr, "method"); } static PyObject * member_repr(PyObject *descr) { - return descr_repr((PyDescrObject *)descr, - "<member '%V' of '%s' objects>"); + return descr_repr((PyDescrObject *)descr, "member"); } static PyObject * getset_repr(PyObject *descr) { - return descr_repr((PyDescrObject *)descr, - "<attribute '%V' of '%s' objects>"); + return descr_repr((PyDescrObject *)descr, "attribute"); } static PyObject * wrapperdescr_repr(PyObject *descr) { - return descr_repr((PyDescrObject *)descr, - "<slot wrapper '%V' of '%s' objects>"); + return descr_repr((PyDescrObject *)descr, "slot wrapper"); } static int @@ -313,7 +313,7 @@ method_vectorcall_VARARGS( if (method_check_args(func, args, nargs, kwnames)) { return NULL; } - PyObject *argstuple = _PyTuple_FromArray(args+1, nargs-1); + PyObject *argstuple = PyTuple_FromArray(args+1, nargs-1); if (argstuple == NULL) { return NULL; } @@ -338,7 +338,7 @@ method_vectorcall_VARARGS_KEYWORDS( if (method_check_args(func, args, nargs, NULL)) { return NULL; } - PyObject *argstuple = _PyTuple_FromArray(args+1, nargs-1); + PyObject *argstuple = PyTuple_FromArray(args+1, nargs-1); if (argstuple == NULL) { return NULL; } @@ -1233,7 +1233,10 @@ static PyObject * mappingproxy_richcompare(PyObject *self, PyObject *w, int op) { mappingproxyobject *v = (mappingproxyobject *)self; - return PyObject_RichCompare(v->mapping, w, op); + if (op == Py_EQ || op == Py_NE) { + return PyObject_RichCompare(v->mapping, w, op); + } + Py_RETURN_NOTIMPLEMENTED; } static int diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 59b0cf1ce7d..e0eef7b46df 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -217,7 +217,7 @@ set_values(PyDictObject *mp, PyDictValues *values) #define LOAD_KEYS_NENTRIES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) #define INCREF_KEYS_FT(dk) dictkeys_incref(dk) -#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(_PyInterpreterState_GET(), dk, shared) +#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(dk, shared) static inline void split_keys_entry_added(PyDictKeysObject *keys) { @@ -380,8 +380,7 @@ equally good collision statistics, needed less code & used less memory. */ -static int dictresize(PyInterpreterState *interp, PyDictObject *mp, - uint8_t log_newsize, int unicode); +static int dictresize(PyDictObject *mp, uint8_t log_newsize, int unicode); static PyObject* dict_iter(PyObject *dict); @@ -401,8 +400,7 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); static inline Py_hash_t unicode_get_hash(PyObject *o) { - assert(PyUnicode_CheckExact(o)); - return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(o)->hash); + return PyUnstable_Unicode_GET_CACHED_HASH(o); } /* Print summary info about the state of the optimized allocator */ @@ -444,7 +442,7 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) +dictkeys_decref(PyDictKeysObject *dk, bool use_qsbr) { if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); @@ -547,13 +545,13 @@ static inline uint8_t calculate_log2_keysize(Py_ssize_t minsize) { #if SIZEOF_LONG == SIZEOF_SIZE_T - minsize = (minsize | PyDict_MINSIZE) - 1; - return _Py_bit_length(minsize | (PyDict_MINSIZE-1)); + minsize = Py_MAX(minsize, PyDict_MINSIZE); + return _Py_bit_length(minsize - 1); #elif defined(_MSC_VER) - // On 64bit Windows, sizeof(long) == 4. - minsize = (minsize | PyDict_MINSIZE) - 1; + // On 64bit Windows, sizeof(long) == 4. We cannot use _Py_bit_length. + minsize = Py_MAX(minsize, PyDict_MINSIZE); unsigned long msb; - _BitScanReverse64(&msb, (uint64_t)minsize); + _BitScanReverse64(&msb, (uint64_t)minsize - 1); return (uint8_t)(msb + 1); #else uint8_t log2_size; @@ -753,7 +751,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* -new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) +new_keys_object(uint8_t log2_size, bool unicode) { Py_ssize_t usable; int log2_bytes; @@ -813,7 +811,7 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(keys); + _PyMem_FreeDelayed(keys, _PyDict_KeysSize(keys)); return; } #endif @@ -858,7 +856,7 @@ free_values(PyDictValues *values, bool use_qsbr) assert(values->embedded == 0); #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(values); + _PyMem_FreeDelayed(values, values_size_from_count(values->capacity)); return; } #endif @@ -867,8 +865,7 @@ free_values(PyDictValues *values, bool use_qsbr) /* Consumes a reference to the keys object */ static PyObject * -new_dict(PyInterpreterState *interp, - PyDictKeysObject *keys, PyDictValues *values, +new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { assert(keys != NULL); @@ -876,7 +873,7 @@ new_dict(PyInterpreterState *interp, if (mp == NULL) { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); if (free_values_on_failure) { free_values(values, false); } @@ -894,7 +891,7 @@ new_dict(PyInterpreterState *interp, } static PyObject * -new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) +new_dict_with_shared_keys(PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); @@ -905,7 +902,7 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - return new_dict(interp, keys, values, 0, 1); + return new_dict(keys, values, 0, 1); } @@ -971,9 +968,8 @@ clone_combined_dict_keys(PyDictObject *orig) PyObject * PyDict_New(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); /* We don't incref Py_EMPTY_KEYS here because it is immortal. */ - return new_dict(interp, Py_EMPTY_KEYS, NULL, 0, 0); + return new_dict(Py_EMPTY_KEYS, NULL, 0, 0); } /* Search index of hash table from offset of entry table */ @@ -1584,32 +1580,47 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb return ix; } +static Py_ssize_t +lookup_threadsafe_unicode(PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr) +{ + assert(dk->dk_kind == DICT_KEYS_UNICODE); + assert(PyUnicode_CheckExact(key)); + + Py_ssize_t ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_EMPTY) { + *value_addr = PyStackRef_NULL; + return ix; + } + else if (ix >= 0) { + PyObject **addr_of_value = &DK_UNICODE_ENTRIES(dk)[ix].me_value; + PyObject *value = _Py_atomic_load_ptr(addr_of_value); + if (value == NULL) { + *value_addr = PyStackRef_NULL; + return DKIX_EMPTY; + } + if (_PyObject_HasDeferredRefcount(value)) { + *value_addr = (_PyStackRef){ .bits = (uintptr_t)value | Py_TAG_DEFERRED }; + return ix; + } + if (_Py_TryIncrefCompare(addr_of_value, value)) { + *value_addr = PyStackRef_FromPyObjectSteal(value); + return ix; + } + return DKIX_KEY_CHANGED; + } + assert(ix == DKIX_KEY_CHANGED); + return ix; +} + Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr) { PyDictKeysObject *dk = _Py_atomic_load_ptr(&mp->ma_keys); if (dk->dk_kind == DICT_KEYS_UNICODE && PyUnicode_CheckExact(key)) { - Py_ssize_t ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); - if (ix == DKIX_EMPTY) { - *value_addr = PyStackRef_NULL; + Py_ssize_t ix = lookup_threadsafe_unicode(dk, key, hash, value_addr); + if (ix != DKIX_KEY_CHANGED) { return ix; } - else if (ix >= 0) { - PyObject **addr_of_value = &DK_UNICODE_ENTRIES(dk)[ix].me_value; - PyObject *value = _Py_atomic_load_ptr(addr_of_value); - if (value == NULL) { - *value_addr = PyStackRef_NULL; - return DKIX_EMPTY; - } - if (_PyObject_HasDeferredRefcount(value)) { - *value_addr = (_PyStackRef){ .bits = (uintptr_t)value | Py_TAG_DEFERRED }; - return ix; - } - if (_Py_TryIncrefCompare(addr_of_value, value)) { - *value_addr = PyStackRef_FromPyObjectSteal(value); - return ix; - } - } } PyObject *obj; @@ -1649,6 +1660,46 @@ _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t h #endif +// Looks up the unicode key `key` in the dictionary. Note that `*method` may +// already contain a valid value! See _PyObject_GetMethodStackRef(). +int +_PyDict_GetMethodStackRef(PyDictObject *mp, PyObject *key, _PyStackRef *method) +{ + assert(PyUnicode_CheckExact(key)); + Py_hash_t hash = hash_unicode_key(key); + +#ifdef Py_GIL_DISABLED + PyDictKeysObject *dk = _Py_atomic_load_ptr_acquire(&mp->ma_keys); + if (dk->dk_kind == DICT_KEYS_UNICODE) { + _PyStackRef ref; + Py_ssize_t ix = lookup_threadsafe_unicode(dk, key, hash, &ref); + if (ix >= 0) { + assert(!PyStackRef_IsNull(ref)); + PyStackRef_XSETREF(*method, ref); + return 1; + } + else if (ix == DKIX_EMPTY) { + return 0; + } + assert(ix == DKIX_KEY_CHANGED); + } +#endif + + PyObject *obj; + Py_INCREF(mp); + Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &obj); + Py_DECREF(mp); + if (ix == DKIX_ERROR) { + PyStackRef_CLEAR(*method); + return -1; + } + else if (ix >= 0 && obj != NULL) { + PyStackRef_XSETREF(*method, PyStackRef_FromPyObjectSteal(obj)); + return 1; + } + return 0; // not found +} + int _PyDict_HasOnlyStringKeys(PyObject *dict) { @@ -1714,18 +1765,26 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) } static int -insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) +insertion_resize(PyDictObject *mp, int unicode) { - return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); + return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } static inline int insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, Py_hash_t hash, PyObject *key, PyObject *value) { + // gh-140551: If dict was cleared in _Py_dict_lookup, + // we have to resize one more time to force general key kind. + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { + if (insertion_resize(mp, 0) < 0) + return -1; + assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + } + if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(interp, mp, 1) < 0) { + if (insertion_resize(mp, 1) < 0) { return -1; } } @@ -1819,38 +1878,31 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { PyObject *old_value; + Py_ssize_t ix; ASSERT_DICT_LOCKED(mp); - if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(interp, mp, 0) < 0) - goto Fail; - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); - } - - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { + ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { insert_split_value(interp, mp, key, value, ix); Py_DECREF(key); Py_DECREF(value); return 0; } - - /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { + // No space in shared keys. Go to insert_combined_dict() below. + } + else { + ix = _Py_dict_lookup(mp, key, hash, &old_value); + if (ix == DKIX_ERROR) goto Fail; - } } - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) - goto Fail; - if (ix == DKIX_EMPTY) { - assert(!_PyDict_HasSplitTable(mp)); - /* Insert into new slot. */ - assert(old_value == NULL); + // insert_combined_dict() will convert from non DICT_KEYS_GENERAL table + // into DICT_KEYS_GENERAL table if key is not Unicode. + // We don't convert it before _Py_dict_lookup because non-Unicode key + // may change generic table into Unicode table. if (insert_combined_dict(interp, mp, hash, key, value) < 0) { goto Fail; } @@ -1862,10 +1914,14 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (old_value != value) { _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); assert(old_value != NULL); - assert(!_PyDict_HasSplitTable(mp)); if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); + if (_PyDict_HasSplitTable(mp)) { + STORE_SPLIT_VALUE(mp, ix, value); + } + else { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; @@ -1893,8 +1949,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, ASSERT_DICT_LOCKED(mp); int unicode = PyUnicode_CheckExact(key); - PyDictKeysObject *newkeys = new_keys_object( - interp, PyDict_LOG_MINSIZE, unicode); + PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); if (newkeys == NULL) { Py_DECREF(key); Py_DECREF(value); @@ -1989,7 +2044,7 @@ This function supports: - Generic -> Generic */ static int -dictresize(PyInterpreterState *interp, PyDictObject *mp, +dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) { PyDictKeysObject *oldkeys, *newkeys; @@ -2017,7 +2072,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, */ /* Allocate a new table. */ - newkeys = new_keys_object(interp, log2_newsize, unicode); + newkeys = new_keys_object(log2_newsize, unicode); if (newkeys == NULL) { return -1; } @@ -2060,7 +2115,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } UNLOCK_KEYS(oldkeys); set_keys(mp, newkeys); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); set_values(mp, NULL); if (oldvalues->embedded) { assert(oldvalues->embedded == 1); @@ -2141,7 +2196,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } static PyObject * -dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) +dict_new_presized(Py_ssize_t minused, bool unicode) { const uint8_t log2_max_presize = 17; const Py_ssize_t max_presize = ((Py_ssize_t)1) << log2_max_presize; @@ -2162,17 +2217,16 @@ dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) log2_newsize = estimate_log2_keysize(minused); } - new_keys = new_keys_object(interp, log2_newsize, unicode); + new_keys = new_keys_object(log2_newsize, unicode); if (new_keys == NULL) return NULL; - return new_dict(interp, new_keys, NULL, 0, 0); + return new_dict(new_keys, NULL, 0, 0); } PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return dict_new_presized(interp, minused, false); + return dict_new_presized(minused, false); } PyObject * @@ -2182,7 +2236,6 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, { bool unicode = true; PyObject *const *ks = keys; - PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = 0; i < length; i++) { if (!PyUnicode_CheckExact(*ks)) { @@ -2192,7 +2245,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, ks += keys_offset; } - PyObject *dict = dict_new_presized(interp, length, unicode); + PyObject *dict = dict_new_presized(length, unicode); if (dict == NULL) { return NULL; } @@ -2478,18 +2531,6 @@ _PyDict_GetItemWithError(PyObject *dp, PyObject *kv) return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference } -PyObject * -_PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key) -{ - PyObject *kv; - kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return NULL; - Py_hash_t hash = unicode_get_hash(kv); - assert (hash != -1); /* interned strings have their hash value initialised */ - return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference -} - PyObject * _PyDict_GetItemStringWithError(PyObject *v, const char *key) { @@ -2770,8 +2811,8 @@ PyDict_DelItem(PyObject *op, PyObject *key) return _PyDict_DelItem_KnownHash(op, key, hash); } -static int -delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) +int +_PyDict_DelItem_KnownHash_LockHeld(PyObject *op, PyObject *key, Py_hash_t hash) { Py_ssize_t ix; PyDictObject *mp; @@ -2806,7 +2847,7 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) { int res; Py_BEGIN_CRITICAL_SECTION(op); - res = delitem_knownhash_lock_held(op, key, hash); + res = _PyDict_DelItem_KnownHash_LockHeld(op, key, hash); Py_END_CRITICAL_SECTION(); return res; } @@ -2895,7 +2936,7 @@ clear_lock_held(PyObject *op) if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); } else { n = oldkeys->dk_nentries; @@ -2909,12 +2950,17 @@ clear_lock_held(PyObject *op) set_values(mp, NULL); set_keys(mp, Py_EMPTY_KEYS); free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(interp, oldkeys, false); + dictkeys_decref(oldkeys, false); } } ASSERT_CONSISTENT(mp); } +void +_PyDict_Clear_LockHeld(PyObject *op) { + clear_lock_held(op); +} + void PyDict_Clear(PyObject *op) { @@ -3156,7 +3202,7 @@ dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, uint8_t new_size = Py_MAX( estimate_log2_keysize(PyDict_GET_SIZE(iterable)), DK_LOG_SIZE(mp->ma_keys)); - if (dictresize(interp, mp, new_size, unicode)) { + if (dictresize(mp, new_size, unicode)) { Py_DECREF(mp); return NULL; } @@ -3178,9 +3224,10 @@ dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, Py_ssize_t pos = 0; PyObject *key; Py_hash_t hash; - - if (dictresize(interp, mp, - estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + uint8_t new_size = Py_MAX( + estimate_log2_keysize(PySet_GET_SIZE(iterable)), + DK_LOG_SIZE(mp->ma_keys)); + if (dictresize(mp, new_size, 0)) { Py_DECREF(mp); return NULL; } @@ -3292,11 +3339,11 @@ dict_dealloc(PyObject *self) } free_values(values, false); } - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } if (Py_IS_TYPE(mp, &PyDict_Type)) { _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free); @@ -3826,7 +3873,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe return -1; ensure_shared_on_resize(mp); - dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); + dictkeys_decref(mp->ma_keys, IS_DICT_SHARED(mp)); set_keys(mp, keys); STORE_USED(mp, other->ma_used); ASSERT_CONSISTENT(mp); @@ -3845,14 +3892,13 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe */ if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(mp->ma_used + other->ma_used), + if (dictresize(mp, estimate_log2_keysize(mp->ma_used + other->ma_used), unicode)) { return -1; } } - Py_ssize_t orig_size = other->ma_keys->dk_nentries; + Py_ssize_t orig_size = other->ma_used; Py_ssize_t pos = 0; Py_hash_t hash; PyObject *key, *value; @@ -3886,7 +3932,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe if (err != 0) return -1; - if (orig_size != other->ma_keys->dk_nentries) { + if (orig_size != other->ma_used) { PyErr_SetString(PyExc_RuntimeError, "dict mutated during update"); return -1; @@ -4111,7 +4157,7 @@ copy_lock_held(PyObject *o) if (keys == NULL) { return NULL; } - PyDictObject *new = (PyDictObject *)new_dict(interp, keys, NULL, 0, 0); + PyDictObject *new = (PyDictObject *)new_dict(keys, NULL, 0, 0); if (new == NULL) { /* In case of an error, `new_dict()` takes care of cleaning up `keys`. */ @@ -4320,6 +4366,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; + Py_ssize_t ix; PyInterpreterState *interp = _PyInterpreterState_GET(); ASSERT_DICT_LOCKED(d); @@ -4355,17 +4402,8 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu return 0; } - if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(interp, mp, 0) < 0) { - if (result) { - *result = NULL; - } - return -1; - } - } - - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { + ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { PyObject *value = mp->ma_values->values[ix]; int already_present = value != NULL; @@ -4378,33 +4416,29 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } return already_present; } - - /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { - goto error; - } + // No space in shared keys. Go to insert_combined_dict() below. } - - assert(!_PyDict_HasSplitTable(mp)); - - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) { - if (result) { - *result = NULL; + else { + ix = _Py_dict_lookup(mp, key, hash, &value); + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; } - return -1; } if (ix == DKIX_EMPTY) { - assert(!_PyDict_HasSplitTable(mp)); value = default_value; + // See comment to this function in insertdict. if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { Py_DECREF(key); Py_DECREF(value); if (result) { *result = NULL; } + return -1; } STORE_USED(mp, mp->ma_used + 1); @@ -4422,12 +4456,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu *result = incref_result ? Py_NewRef(value) : value; } return 1; - -error: - if (result) { - *result = NULL; - } - return -1; } int @@ -4490,6 +4518,7 @@ dict_clear_impl(PyDictObject *self) } /*[clinic input] +@permit_long_summary dict.pop key: object @@ -4504,7 +4533,7 @@ raise a KeyError. static PyObject * dict_pop_impl(PyDictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=3abb47b89f24c21c input=e221baa01044c44c]*/ +/*[clinic end generated code: output=3abb47b89f24c21c input=d409c7eb2de67e38]*/ { return dict_pop_default((PyObject*)self, key, default_value); } @@ -4548,7 +4577,7 @@ dict_popitem_impl(PyDictObject *self) } /* Convert split table to combined table */ if (_PyDict_HasSplitTable(self)) { - if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { + if (dictresize(self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { Py_DECREF(res); return NULL; } @@ -4647,9 +4676,11 @@ dict_tp_clear(PyObject *op) static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -static Py_ssize_t -sizeof_lock_held(PyDictObject *mp) +Py_ssize_t +_PyDict_SizeOf_LockHeld(PyDictObject *mp) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + size_t res = _PyObject_SIZE(Py_TYPE(mp)); if (_PyDict_HasSplitTable(mp)) { res += shared_keys_usable_size(mp->ma_keys) * sizeof(PyObject*); @@ -4668,7 +4699,7 @@ _PyDict_SizeOf(PyDictObject *mp) { Py_ssize_t res; Py_BEGIN_CRITICAL_SECTION(mp); - res = sizeof_lock_held(mp); + res = _PyDict_SizeOf_LockHeld(mp); Py_END_CRITICAL_SECTION(); return res; @@ -4806,16 +4837,6 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return 0; } -int -_PyDict_ContainsId(PyObject *op, _Py_Identifier *key) -{ - PyObject *kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) { - return -1; - } - return PyDict_Contains(op, kv); -} - /* Hack to implement "key in dict" */ static PySequenceMethods dict_as_sequence = { 0, /* sq_length */ @@ -4996,16 +5017,6 @@ PyDict_GetItemStringRef(PyObject *v, const char *key, PyObject **result) return res; } -int -_PyDict_SetItemId(PyObject *v, _Py_Identifier *key, PyObject *item) -{ - PyObject *kv; - kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return -1; - return PyDict_SetItem(v, kv, item); -} - int PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) { @@ -5021,15 +5032,6 @@ PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) return err; } -int -_PyDict_DelItemId(PyObject *v, _Py_Identifier *key) -{ - PyObject *kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return -1; - return PyDict_DelItem(v, kv); -} - int PyDict_DelItemString(PyObject *v, const char *key) { @@ -5609,22 +5611,10 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, #endif -static bool -has_unique_reference(PyObject *op) -{ -#ifdef Py_GIL_DISABLED - return (_Py_IsOwnedByCurrentThread(op) && - op->ob_ref_local == 1 && - _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); -#else - return Py_REFCNT(op) == 1; -#endif -} - static bool acquire_iter_result(PyObject *result) { - if (has_unique_reference(result)) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); return true; } @@ -5661,8 +5651,11 @@ dictiter_iternextitem(PyObject *self) } else { result = PyTuple_New(2); - if (result == NULL) + if (result == NULL) { + Py_DECREF(key); + Py_DECREF(value); return NULL; + } PyTuple_SET_ITEM(result, 0, key); PyTuple_SET_ITEM(result, 1, value); } @@ -5770,7 +5763,7 @@ dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) } else if (Py_IS_TYPE(di, &PyDictRevIterItem_Type)) { result = di->di_result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { PyObject *oldkey = PyTuple_GET_ITEM(result, 0); PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); @@ -6718,10 +6711,7 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(PyHeapTypeObject *cls) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - - PyDictKeysObject *keys = new_keys_object( - interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); + PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { PyErr_Clear(); } @@ -6785,8 +6775,7 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) } static PyDictObject * -make_dict_from_instance_attributes(PyInterpreterState *interp, - PyDictKeysObject *keys, PyDictValues *values) +make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) { dictkeys_incref(keys); Py_ssize_t used = 0; @@ -6797,7 +6786,7 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, used += 1; } } - PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); + PyDictObject *res = (PyDictObject *)new_dict(keys, values, used, 0); return res; } @@ -6811,9 +6800,8 @@ _PyObject_MaterializeManagedDict_LockHeld(PyObject *obj) PyDictValues *values = _PyObject_InlineValues(obj); PyDictObject *dict; if (values->valid) { - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); - dict = make_dict_from_instance_attributes(interp, keys, values); + dict = make_dict_from_instance_attributes(keys, values); } else { dict = (PyDictObject *)PyDict_New(); @@ -6858,7 +6846,7 @@ _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) dict_unhashable_type(name); return -1; } - return delitem_knownhash_lock_held((PyObject *)dict, name, hash); + return _PyDict_DelItem_KnownHash_LockHeld((PyObject *)dict, name, hash); } else { return setitem_lock_held(dict, name, value); } @@ -6909,7 +6897,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, if (dict == NULL) { // Make the dict but don't publish it in the object // so that no one else will see it. - dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); + dict = make_dict_from_instance_attributes(keys, values); if (dict == NULL || _PyDict_SetItem_LockHeld(dict, name, value) < 0) { Py_XDECREF(dict); @@ -6932,6 +6920,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, PyErr_Format(PyExc_AttributeError, "'%.100s' object has no attribute '%U'", Py_TYPE(obj)->tp_name, name); + (void)_PyObject_SetAttributeErrorContext(obj, name); return -1; } @@ -7442,11 +7431,10 @@ PyObject_ClearManagedDict(PyObject *obj) "clearing an object managed dict"); /* Clear the dict */ Py_BEGIN_CRITICAL_SECTION(dict); - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *oldkeys = dict->ma_keys; set_keys(dict, Py_EMPTY_KEYS); dict->ma_values = NULL; - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(dict)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(dict)); STORE_USED(dict, 0); clear_inline_values(_PyObject_InlineValues(obj)); Py_END_CRITICAL_SECTION(); @@ -7483,8 +7471,7 @@ ensure_managed_dict(PyObject *obj) goto done; } #endif - dict = (PyDictObject *)new_dict_with_shared_keys(_PyInterpreterState_GET(), - CACHED_KEYS(tp)); + dict = (PyDictObject *)new_dict_with_shared_keys(CACHED_KEYS(tp)); FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, (PyDictObject *)dict); @@ -7513,9 +7500,8 @@ ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) #endif PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { - PyInterpreterState *interp = _PyInterpreterState_GET(); assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); - dict = new_dict_with_shared_keys(interp, cached); + dict = new_dict_with_shared_keys(cached); } else { dict = PyDict_New(); @@ -7571,8 +7557,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, void _PyDictKeys_DecRef(PyDictKeysObject *keys) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } static inline uint32_t diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 1123b140c7f..814ce4f9195 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -367,7 +367,7 @@ typedef struct { @classmethod reversed.__new__ as reversed_new - sequence as seq: object + object as seq: object / Return a reverse iterator over the values of the given sequence. @@ -375,7 +375,7 @@ Return a reverse iterator over the values of the given sequence. static PyObject * reversed_new_impl(PyTypeObject *type, PyObject *seq) -/*[clinic end generated code: output=f7854cc1df26f570 input=aeb720361e5e3f1d]*/ +/*[clinic end generated code: output=f7854cc1df26f570 input=4781869729e3ba50]*/ { Py_ssize_t n; PyObject *reversed_meth; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index b17cac83551..9a43057b383 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -13,7 +13,6 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" #include "pycore_pyerrors.h" // struct _PyErr_SetRaisedException -#include "pycore_tuple.h" // _PyTuple_FromArray() #include "osdefs.h" // SEP #include "clinic/exceptions.c.h" @@ -119,7 +118,7 @@ BaseException_vectorcall(PyObject *type_obj, PyObject * const*args, self->context = NULL; self->suppress_context = 0; - self->args = _PyTuple_FromArray(args, PyVectorcall_NARGS(nargsf)); + self->args = PyTuple_FromArray(args, PyVectorcall_NARGS(nargsf)); if (!self->args) { Py_DECREF(self); return NULL; @@ -695,12 +694,12 @@ PyTypeObject _PyExc_ ## EXCNAME = { \ #define ComplexExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCNEW, \ EXCMETHODS, EXCMEMBERS, EXCGETSET, \ - EXCSTR, EXCDOC) \ + EXCSTR, EXCREPR, EXCDOC) \ static PyTypeObject _PyExc_ ## EXCNAME = { \ PyVarObject_HEAD_INIT(NULL, 0) \ # EXCNAME, \ sizeof(Py ## EXCSTORE ## Object), 0, \ - EXCSTORE ## _dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + EXCSTORE ## _dealloc, 0, 0, 0, 0, EXCREPR, 0, 0, 0, 0, 0, \ EXCSTR, 0, 0, 0, \ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, \ PyDoc_STR(EXCDOC), EXCSTORE ## _traverse, \ @@ -793,7 +792,7 @@ StopIteration_traverse(PyObject *op, visitproc visit, void *arg) } ComplexExtendsException(PyExc_Exception, StopIteration, StopIteration, - 0, 0, StopIteration_members, 0, 0, + 0, 0, StopIteration_members, 0, 0, 0, "Signal the end from iterator.__next__()."); @@ -866,7 +865,7 @@ static PyMemberDef SystemExit_members[] = { }; ComplexExtendsException(PyExc_BaseException, SystemExit, SystemExit, - 0, 0, SystemExit_members, 0, 0, + 0, 0, SystemExit_members, 0, 0, 0, "Request to exit from the interpreter."); /* @@ -891,6 +890,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *message = NULL; PyObject *exceptions = NULL; + PyObject *exceptions_str = NULL; if (!PyArg_ParseTuple(args, "UO:BaseExceptionGroup.__new__", @@ -906,6 +906,18 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } + /* Save initial exceptions sequence as a string in case sequence is mutated */ + if (!PyList_Check(exceptions) && !PyTuple_Check(exceptions)) { + exceptions_str = PyObject_Repr(exceptions); + if (exceptions_str == NULL) { + /* We don't hold a reference to exceptions, so clear it before + * attempting a decref in the cleanup. + */ + exceptions = NULL; + goto error; + } + } + exceptions = PySequence_Tuple(exceptions); if (!exceptions) { return NULL; @@ -989,9 +1001,11 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->msg = Py_NewRef(message); self->excs = exceptions; + self->excs_str = exceptions_str; return (PyObject*)self; error: - Py_DECREF(exceptions); + Py_XDECREF(exceptions); + Py_XDECREF(exceptions_str); return NULL; } @@ -1030,6 +1044,7 @@ BaseExceptionGroup_clear(PyObject *op) PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); Py_CLEAR(self->msg); Py_CLEAR(self->excs); + Py_CLEAR(self->excs_str); return BaseException_clear(op); } @@ -1047,6 +1062,7 @@ BaseExceptionGroup_traverse(PyObject *op, visitproc visit, void *arg) PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); Py_VISIT(self->msg); Py_VISIT(self->excs); + Py_VISIT(self->excs_str); return BaseException_traverse(op, visit, arg); } @@ -1064,6 +1080,54 @@ BaseExceptionGroup_str(PyObject *op) self->msg, num_excs, num_excs > 1 ? "s" : ""); } +static PyObject * +BaseExceptionGroup_repr(PyObject *op) +{ + PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); + assert(self->msg); + + PyObject *exceptions_str = NULL; + + /* Use the saved exceptions string for custom sequences. */ + if (self->excs_str) { + exceptions_str = Py_NewRef(self->excs_str); + } + else { + assert(self->excs); + + /* Older versions delegated to BaseException, inserting the current + * value of self.args[1]; but this can be mutable and go out-of-sync + * with self.exceptions. Instead, use self.exceptions for accuracy, + * making it look like self.args[1] for backwards compatibility. */ + if (PyList_Check(PyTuple_GET_ITEM(self->args, 1))) { + PyObject *exceptions_list = PySequence_List(self->excs); + if (!exceptions_list) { + return NULL; + } + + exceptions_str = PyObject_Repr(exceptions_list); + Py_DECREF(exceptions_list); + } + else { + exceptions_str = PyObject_Repr(self->excs); + } + + if (!exceptions_str) { + return NULL; + } + } + + assert(exceptions_str != NULL); + + const char *name = _PyType_Name(Py_TYPE(self)); + PyObject *repr = PyUnicode_FromFormat( + "%s(%R, %U)", name, + self->msg, exceptions_str); + + Py_DECREF(exceptions_str); + return repr; +} + /*[clinic input] @critical_section BaseExceptionGroup.derive @@ -1698,7 +1762,7 @@ static PyMethodDef BaseExceptionGroup_methods[] = { ComplexExtendsException(PyExc_BaseException, BaseExceptionGroup, BaseExceptionGroup, BaseExceptionGroup_new /* new */, BaseExceptionGroup_methods, BaseExceptionGroup_members, - 0 /* getset */, BaseExceptionGroup_str, + 0 /* getset */, BaseExceptionGroup_str, BaseExceptionGroup_repr, "A combination of multiple unrelated exceptions."); /* @@ -1864,6 +1928,62 @@ ImportError_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) return res; } +static PyObject * +ImportError_repr(PyObject *self) +{ + int hasargs = PyTuple_GET_SIZE(((PyBaseExceptionObject *)self)->args) != 0; + PyImportErrorObject *exc = PyImportErrorObject_CAST(self); + if (exc->name == NULL && exc->path == NULL) { + return BaseException_repr(self); + } + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + goto error; + } + PyObject *r = BaseException_repr(self); + if (r == NULL) { + goto error; + } + if (PyUnicodeWriter_WriteSubstring( + writer, r, 0, PyUnicode_GET_LENGTH(r) - 1) < 0) + { + Py_DECREF(r); + goto error; + } + Py_DECREF(r); + if (exc->name) { + if (hasargs) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { + goto error; + } + } + if (PyUnicodeWriter_Format(writer, "name=%R", exc->name) < 0) { + goto error; + } + hasargs = 1; + } + if (exc->path) { + if (hasargs) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { + goto error; + } + } + if (PyUnicodeWriter_Format(writer, "path=%R", exc->path) < 0) { + goto error; + } + } + + if (PyUnicodeWriter_WriteChar(writer, ')') < 0) { + goto error; + } + + return PyUnicodeWriter_Finish(writer); + +error: + PyUnicodeWriter_Discard(writer); + return NULL; +} + static PyMemberDef ImportError_members[] = { {"msg", _Py_T_OBJECT, offsetof(PyImportErrorObject, msg), 0, PyDoc_STR("exception message")}, @@ -1881,12 +2001,26 @@ static PyMethodDef ImportError_methods[] = { {NULL} }; -ComplexExtendsException(PyExc_Exception, ImportError, - ImportError, 0 /* new */, - ImportError_methods, ImportError_members, - 0 /* getset */, ImportError_str, - "Import can't find module, or can't find name in " - "module."); +static PyTypeObject _PyExc_ImportError = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "ImportError", + .tp_basicsize = sizeof(PyImportErrorObject), + .tp_dealloc = ImportError_dealloc, + .tp_repr = ImportError_repr, + .tp_str = ImportError_str, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .tp_doc = PyDoc_STR( + "Import can't find module, " + "or can't find name in module."), + .tp_traverse = ImportError_traverse, + .tp_clear = ImportError_clear, + .tp_methods = ImportError_methods, + .tp_members = ImportError_members, + .tp_base = &_PyExc_Exception, + .tp_dictoffset = offsetof(PyImportErrorObject, dict), + .tp_init = ImportError_init, +}; +PyObject *PyExc_ImportError = (PyObject *)&_PyExc_ImportError; /* * ModuleNotFoundError extends ImportError @@ -2356,7 +2490,7 @@ static PyGetSetDef OSError_getset[] = { ComplexExtendsException(PyExc_Exception, OSError, OSError, OSError_new, OSError_methods, OSError_members, OSError_getset, - OSError_str, + OSError_str, 0, "Base class for I/O related errors."); @@ -2497,7 +2631,7 @@ static PyMethodDef NameError_methods[] = { ComplexExtendsException(PyExc_Exception, NameError, NameError, 0, NameError_methods, NameError_members, - 0, BaseException_str, "Name not found globally."); + 0, BaseException_str, 0, "Name not found globally."); /* * UnboundLocalError extends NameError @@ -2631,7 +2765,7 @@ static PyMethodDef AttributeError_methods[] = { ComplexExtendsException(PyExc_Exception, AttributeError, AttributeError, 0, AttributeError_methods, AttributeError_members, - 0, BaseException_str, "Attribute not found."); + 0, BaseException_str, 0, "Attribute not found."); /* * SyntaxError extends Exception @@ -2830,7 +2964,7 @@ static PyMemberDef SyntaxError_members[] = { ComplexExtendsException(PyExc_Exception, SyntaxError, SyntaxError, 0, 0, SyntaxError_members, 0, - SyntaxError_str, "Invalid syntax."); + SyntaxError_str, 0, "Invalid syntax."); /* @@ -2890,7 +3024,7 @@ KeyError_str(PyObject *op) } ComplexExtendsException(PyExc_LookupError, KeyError, BaseException, - 0, 0, 0, 0, KeyError_str, "Mapping key not found."); + 0, 0, 0, 0, KeyError_str, 0, "Mapping key not found."); /* diff --git a/Objects/fileobject.c b/Objects/fileobject.c index e624405bd5f..05c3e75b464 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -68,9 +68,9 @@ PyFile_GetLine(PyObject *f, int n) } if (result != NULL && !PyBytes_Check(result) && !PyUnicode_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%T.readline() must return a str, not %T", f, result); Py_SETREF(result, NULL); - PyErr_SetString(PyExc_TypeError, - "object.readline() returned non-string"); } if (n < 0 && result != NULL && PyBytes_Check(result)) { @@ -193,8 +193,8 @@ PyObject_AsFileDescriptor(PyObject *o) Py_DECREF(fno); } else { - PyErr_SetString(PyExc_TypeError, - "fileno() returned a non-integer"); + PyErr_Format(PyExc_TypeError, + "%T.fileno() must return an int, not %T", o, fno); Py_DECREF(fno); return -1; } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 93e1973d6b3..78006783c6e 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -288,16 +288,16 @@ PyFloat_AsDouble(PyObject *op) if (!PyFloat_CheckExact(res)) { if (!PyFloat_Check(res)) { PyErr_Format(PyExc_TypeError, - "%.50s.__float__ returned non-float (type %.50s)", - Py_TYPE(op)->tp_name, Py_TYPE(res)->tp_name); + "%T.__float__() must return a float, not %T", + op, res); Py_DECREF(res); return -1; } if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "%.50s.__float__ returned non-float (type %.50s). " + "%T.__float__() must return a float, not %T. " "The ability to return an instance of a strict subclass of float " "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(op)->tp_name, Py_TYPE(res)->tp_name)) { + op, res)) { Py_DECREF(res); return -1; } @@ -1484,6 +1484,7 @@ float_fromhex_impl(PyTypeObject *type, PyObject *string) } /*[clinic input] +@permit_long_summary float.as_integer_ratio Return a pair of integers, whose ratio is exactly equal to the original float. @@ -1501,7 +1502,7 @@ OverflowError on infinities and a ValueError on NaNs. static PyObject * float_as_integer_ratio_impl(PyObject *self) -/*[clinic end generated code: output=65f25f0d8d30a712 input=d5ba7765655d75bd]*/ +/*[clinic end generated code: output=65f25f0d8d30a712 input=75ae9be7cecd82a3]*/ { double self_double; double float_part; @@ -1698,6 +1699,7 @@ typedef enum _py_float_format_type float_format_type; /*[clinic input] +@permit_long_docstring_body @classmethod float.__getformat__ @@ -1716,7 +1718,7 @@ C type named by typestr. static PyObject * float___getformat___impl(PyTypeObject *type, const char *typestr) -/*[clinic end generated code: output=2bfb987228cc9628 input=90d5e246409a246e]*/ +/*[clinic end generated code: output=2bfb987228cc9628 input=d2735823bfe8e81e]*/ { float_format_type r; @@ -2028,6 +2030,10 @@ PyFloat_Pack2(double x, char *data, int le) memcpy(&v, &x, sizeof(v)); v &= 0xffc0000000000ULL; bits = (unsigned short)(v >> 42); /* NaN's type & payload */ + /* set qNaN if no payload */ + if (!bits) { + bits |= (1<<9); + } } else { sign = (x < 0.0); @@ -2200,16 +2206,16 @@ PyFloat_Pack4(double x, char *data, int le) if ((v & (1ULL << 51)) == 0) { uint32_t u32; memcpy(&u32, &y, 4); - u32 &= ~(1 << 22); /* make sNaN */ + /* if have payload, make sNaN */ + if (u32 & 0x3fffff) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); } #else uint32_t u32; memcpy(&u32, &y, 4); - if ((v & (1ULL << 51)) == 0) { - u32 &= ~(1 << 22); - } /* Workaround RISC-V: "If a NaN value is converted to a * different floating-point type, the result is the * canonical NaN of the new type". The canonical NaN here @@ -2220,6 +2226,10 @@ PyFloat_Pack4(double x, char *data, int le) /* add payload */ u32 -= (u32 & 0x3fffff); u32 += (uint32_t)((v & 0x7ffffffffffffULL) >> 29); + /* if have payload, make sNaN */ + if ((v & (1ULL << 51)) == 0 && (u32 & 0x3fffff)) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); #endif @@ -2405,7 +2415,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_INFINITY : Py_INFINITY; + return sign ? -INFINITY : INFINITY; } else { /* NaN */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 76b52efccf8..b652973600c 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,6 +17,7 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG +#include "pycore_optimizer.h" #include "clinic/frameobject.c.h" @@ -260,7 +261,10 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } - _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); +#if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), co, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), co); +#endif _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); _PyStackRef oldvalue = fast[i]; @@ -913,6 +917,15 @@ static PyMethodDef framelocalsproxy_methods[] = { {NULL, NULL} /* sentinel */ }; +PyDoc_STRVAR(framelocalsproxy_doc, +"FrameLocalsProxy($frame)\n" +"--\n" +"\n" +"Create a write-through view of the locals dictionary for a frame.\n" +"\n" +" frame\n" +" the frame object to wrap."); + PyTypeObject PyFrameLocalsProxy_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "FrameLocalsProxy", @@ -933,6 +946,7 @@ PyTypeObject PyFrameLocalsProxy_Type = { .tp_alloc = PyType_GenericAlloc, .tp_new = framelocalsproxy_new, .tp_free = PyObject_GC_Del, + .tp_doc = framelocalsproxy_doc, }; PyObject * @@ -1386,6 +1400,10 @@ mark_stacks(PyCodeObject *code_obj, int len) stacks[j] = next_stack; break; case GET_ITER: + next_stack = push_value(pop_value(next_stack), Iterator); + next_stack = push_value(next_stack, Iterator); + stacks[next_i] = next_stack; + break; case GET_AITER: next_stack = push_value(pop_value(next_stack), Iterator); stacks[next_i] = next_stack; @@ -1671,6 +1689,8 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) case PY_MONITORING_EVENT_PY_RESUME: case PY_MONITORING_EVENT_JUMP: case PY_MONITORING_EVENT_BRANCH: + case PY_MONITORING_EVENT_BRANCH_LEFT: + case PY_MONITORING_EVENT_BRANCH_RIGHT: case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: /* Setting f_lineno is allowed for the above events */ @@ -1833,6 +1853,7 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) } /*[clinic input] +@permit_long_summary @critical_section @getter frame.f_trace as frame_trace @@ -1842,7 +1863,7 @@ Return the trace function for this frame, or None if no trace function is set. static PyObject * frame_trace_get_impl(PyFrameObject *self) -/*[clinic end generated code: output=5475cbfce07826cd input=f382612525829773]*/ +/*[clinic end generated code: output=5475cbfce07826cd input=e4eacf2c68cac577]*/ { PyObject* trace = self->f_trace; if (trace == NULL) { @@ -1852,6 +1873,7 @@ frame_trace_get_impl(PyFrameObject *self) } /*[clinic input] +@permit_long_summary @critical_section @setter frame.f_trace as frame_trace @@ -1859,7 +1881,7 @@ frame.f_trace as frame_trace static int frame_trace_set_impl(PyFrameObject *self, PyObject *value) -/*[clinic end generated code: output=d6fe08335cf76ae4 input=d96a18bda085707f]*/ +/*[clinic end generated code: output=d6fe08335cf76ae4 input=e57380734815dac5]*/ { if (value == Py_None) { value = NULL; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 56df5730db0..b659ac80233 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1,14 +1,17 @@ /* Function object implementation */ #include "Python.h" +#include "pycore_code.h" // _PyCode_VerifyStateless() #include "pycore_dict.h" // _Py_INCREF_DICT() #include "pycore_function.h" // _PyFunction_Vectorcall #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" - +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() +#include "pycore_optimizer.h" // _PyJit_Tracer_InvalidateDependency static const char * func_event_name(PyFunction_WatchEvent event) { @@ -59,6 +62,7 @@ handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, case PyFunction_EVENT_MODIFY_CODE: case PyFunction_EVENT_MODIFY_DEFAULTS: case PyFunction_EVENT_MODIFY_KWDEFAULTS: + case PyFunction_EVENT_MODIFY_QUALNAME: RARE_EVENT_INTERP_INC(interp, func_modification); break; default: @@ -557,8 +561,9 @@ func_get_annotation_dict(PyFunctionObject *op) return NULL; } if (!PyDict_Check(ann_dict)) { - PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", - Py_TYPE(ann_dict)->tp_name); + PyErr_Format(PyExc_TypeError, + "__annotate__() must return a dict, not %T", + ann_dict); Py_DECREF(ann_dict); return NULL; } @@ -743,6 +748,7 @@ func_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) "__qualname__ must be set to a string object"); return -1; } + handle_func_event(PyFunction_EVENT_MODIFY_QUALNAME, (PyFunctionObject *) op, value); Py_XSETREF(op->func_qualname, Py_NewRef(value)); return 0; } @@ -1145,10 +1151,12 @@ func_dealloc(PyObject *self) if (_PyObject_ResurrectEnd(self)) { return; } +#if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), self, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), self); +#endif _PyObject_GC_UNTRACK(op); - if (op->func_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) op); - } + FT_CLEAR_WEAKREFS(self, op->func_weakreflist); (void)func_clear((PyObject*)op); // These aren't cleared by func_clear(). _Py_DECREF_CODE((PyCodeObject *)op->func_code); @@ -1240,6 +1248,64 @@ PyTypeObject PyFunction_Type = { }; +int +_PyFunction_VerifyStateless(PyThreadState *tstate, PyObject *func) +{ + assert(!PyErr_Occurred()); + assert(PyFunction_Check(func)); + + // Check the globals. + PyObject *globalsns = PyFunction_GET_GLOBALS(func); + if (globalsns != NULL && !PyDict_Check(globalsns)) { + _PyErr_Format(tstate, PyExc_TypeError, + "unsupported globals %R", globalsns); + return -1; + } + // Check the builtins. + PyObject *builtinsns = _PyFunction_GET_BUILTINS(func); + if (builtinsns != NULL && !PyDict_Check(builtinsns)) { + _PyErr_Format(tstate, PyExc_TypeError, + "unsupported builtins %R", builtinsns); + return -1; + } + // Disallow __defaults__. + PyObject *defaults = PyFunction_GET_DEFAULTS(func); + if (defaults != NULL) { + assert(PyTuple_Check(defaults)); // per PyFunction_New() + if (PyTuple_GET_SIZE(defaults) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, + "defaults not supported"); + return -1; + } + } + // Disallow __kwdefaults__. + PyObject *kwdefaults = PyFunction_GET_KW_DEFAULTS(func); + if (kwdefaults != NULL) { + assert(PyDict_Check(kwdefaults)); // per PyFunction_New() + if (PyDict_Size(kwdefaults) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, + "keyword defaults not supported"); + return -1; + } + } + // Disallow __closure__. + PyObject *closure = PyFunction_GET_CLOSURE(func); + if (closure != NULL) { + assert(PyTuple_Check(closure)); // per PyFunction_New() + if (PyTuple_GET_SIZE(closure) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, "closures not supported"); + return -1; + } + } + // Check the code. + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + if (_PyCode_VerifyStateless(tstate, co, NULL, globalsns, builtinsns) < 0) { + return -1; + } + return 0; +} + + static int functools_copy_attr(PyObject *wrapper, PyObject *wrapped, PyObject *name) { diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index ec3d01f00a3..8b526f43f1e 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -7,6 +7,7 @@ #include "pycore_typevarobject.h" // _Py_typing_type_repr #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> @@ -33,9 +34,7 @@ ga_dealloc(PyObject *self) gaobject *alias = (gaobject *)self; _PyObject_GC_UNTRACK(self); - if (alias->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)alias); - } + FT_CLEAR_WEAKREFS(self, alias->weakreflist); Py_XDECREF(alias->origin); Py_XDECREF(alias->args); Py_XDECREF(alias->parameters); @@ -65,7 +64,7 @@ ga_repr_items_list(PyUnicodeWriter *writer, PyObject *p) for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { return -1; } } @@ -109,7 +108,7 @@ ga_repr(PyObject *self) } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -126,7 +125,7 @@ ga_repr(PyObject *self) } if (len == 0) { // for something like tuple[()] we should print a "()" - if (PyUnicodeWriter_WriteUTF8(writer, "()", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "()", 2) < 0) { goto error; } } @@ -526,12 +525,24 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje return NULL; } if (unpack) { + if (!PyTuple_Check(arg)) { + Py_DECREF(newargs); + Py_DECREF(item); + Py_XDECREF(tuple_args); + PyObject *original = PyTuple_GET_ITEM(args, iarg); + PyErr_Format(PyExc_TypeError, + "expected __typing_subst__ of %T objects to return a tuple, not %T", + original, arg); + Py_DECREF(arg); + return NULL; + } jarg = tuple_extend(&newargs, jarg, &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); Py_DECREF(arg); if (jarg < 0) { Py_DECREF(item); Py_XDECREF(tuple_args); + assert(newargs == NULL); return NULL; } } @@ -637,7 +648,6 @@ ga_vectorcall(PyObject *self, PyObject *const *args, static const char* const attr_exceptions[] = { "__class__", - "__bases__", "__origin__", "__args__", "__unpacked__", @@ -646,6 +656,11 @@ static const char* const attr_exceptions[] = { "__mro_entries__", "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ "__reduce__", + NULL, +}; + +static const char* const attr_blocked[] = { + "__bases__", "__copy__", "__deepcopy__", NULL, @@ -656,15 +671,29 @@ ga_getattro(PyObject *self, PyObject *name) { gaobject *alias = (gaobject *)self; if (PyUnicode_Check(name)) { + // When we check blocked attrs, we don't allow to proxy them to `__origin__`. + // Otherwise, we can break existing code. + for (const char * const *p = attr_blocked; ; p++) { + if (*p == NULL) { + break; + } + if (_PyUnicode_EqualToASCIIString(name, *p)) { + goto generic_getattr; + } + } + + // When we see own attrs, it has a priority over `__origin__`'s attr. for (const char * const *p = attr_exceptions; ; p++) { if (*p == NULL) { return PyObject_GetAttr(alias->origin, name); } if (_PyUnicode_EqualToASCIIString(name, *p)) { - break; + goto generic_getattr; } } } + +generic_getattr: return PyObject_GenericGetAttr(self, name); } diff --git a/Objects/genobject.c b/Objects/genobject.c index 98b2c5004df..e47180a23d2 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -17,6 +17,7 @@ #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_warnings.h" // _PyErr_WarnUnawaitedCoroutine() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "opcode_ids.h" // RESUME, etc @@ -161,8 +162,7 @@ gen_dealloc(PyObject *self) _PyObject_GC_UNTRACK(gen); - if (gen->gi_weakreflist != NULL) - PyObject_ClearWeakRefs(self); + FT_CLEAR_WEAKREFS(self, gen->gi_weakreflist); _PyObject_GC_TRACK(self); @@ -315,7 +315,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) } PyDoc_STRVAR(send_doc, -"send(arg) -> send 'arg' into generator,\n\ +"send(value) -> send 'value' into generator,\n\ return next yielded value or raise StopIteration."); static PyObject * @@ -407,11 +407,12 @@ gen_close(PyObject *self, PyObject *args) } _PyInterpreterFrame *frame = &gen->gi_iframe; if (is_resume(frame->instr_ptr)) { + bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET()); /* We can safely ignore the outermost try block * as it is automatically generated to handle * StopIteration. */ int oparg = frame->instr_ptr->op.arg; - if (oparg & RESUME_OPARG_DEPTH1_MASK) { + if (oparg & RESUME_OPARG_DEPTH1_MASK && no_unwind_tools) { // RESUME after YIELD_VALUE and exception depth is 1 assert((oparg & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START); gen->gi_frame_state = FRAME_COMPLETED; @@ -704,7 +705,8 @@ static PyObject * gen_get_name(PyObject *self, void *Py_UNUSED(ignored)) { PyGenObject *op = _PyGen_CAST(self); - return Py_NewRef(op->gi_name); + PyObject *name = FT_ATOMIC_LOAD_PTR_ACQUIRE(op->gi_name); + return Py_NewRef(name); } static int @@ -718,7 +720,11 @@ gen_set_name(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) "__name__ must be set to a string object"); return -1; } - Py_XSETREF(op->gi_name, Py_NewRef(value)); + Py_BEGIN_CRITICAL_SECTION(self); + // gh-133931: To prevent use-after-free from other threads that reference + // the gi_name. + _PyObject_XSetRefDelayed(&op->gi_name, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -726,7 +732,8 @@ static PyObject * gen_get_qualname(PyObject *self, void *Py_UNUSED(ignored)) { PyGenObject *op = _PyGen_CAST(self); - return Py_NewRef(op->gi_qualname); + PyObject *qualname = FT_ATOMIC_LOAD_PTR_ACQUIRE(op->gi_qualname); + return Py_NewRef(qualname); } static int @@ -740,7 +747,11 @@ gen_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) "__qualname__ must be set to a string object"); return -1; } - Py_XSETREF(op->gi_qualname, Py_NewRef(value)); + Py_BEGIN_CRITICAL_SECTION(self); + // gh-133931: To prevent use-after-free from other threads that reference + // the gi_qualname. + _PyObject_XSetRefDelayed(&op->gi_qualname, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -922,6 +933,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; + gen->gi_iframe.f_executable = PyStackRef_None; assert(func->func_name != NULL); gen->gi_name = Py_NewRef(func->func_name); assert(func->func_qualname != NULL); @@ -1082,14 +1094,14 @@ _PyCoro_GetAwaitableIter(PyObject *o) if (PyCoro_CheckExact(res) || gen_is_coroutine(res)) { /* __await__ must return an *iterator*, not a coroutine or another awaitable (see PEP 492) */ - PyErr_SetString(PyExc_TypeError, - "__await__() returned a coroutine"); + PyErr_Format(PyExc_TypeError, + "%T.__await__() must return an iterator, " + "not coroutine", o); Py_CLEAR(res); } else if (!PyIter_Check(res)) { PyErr_Format(PyExc_TypeError, - "__await__() returned non-iterator " - "of type '%.100s'", - Py_TYPE(res)->tp_name); + "%T.__await__() must return an iterator, " + "not %T", o, res); Py_CLEAR(res); } } @@ -1451,7 +1463,9 @@ typedef struct PyAsyncGenAThrow { /* Can be NULL, when in the "aclose()" mode (equivalent of "athrow(GeneratorExit)") */ - PyObject *agt_args; + PyObject *agt_typ; + PyObject *agt_tb; + PyObject *agt_val; AwaitableState agt_state; } PyAsyncGenAThrow; @@ -2078,7 +2092,9 @@ async_gen_athrow_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); Py_CLEAR(agt->agt_gen); - Py_CLEAR(agt->agt_args); + Py_XDECREF(agt->agt_typ); + Py_XDECREF(agt->agt_tb); + Py_XDECREF(agt->agt_val); PyObject_GC_Del(self); } @@ -2088,7 +2104,9 @@ async_gen_athrow_traverse(PyObject *self, visitproc visit, void *arg) { PyAsyncGenAThrow *agt = _PyAsyncGenAThrow_CAST(self); Py_VISIT(agt->agt_gen); - Py_VISIT(agt->agt_args); + Py_VISIT(agt->agt_typ); + Py_VISIT(agt->agt_tb); + Py_VISIT(agt->agt_val); return 0; } @@ -2116,7 +2134,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) if (o->agt_state == AWAITABLE_STATE_INIT) { if (o->agt_gen->ag_running_async) { o->agt_state = AWAITABLE_STATE_CLOSED; - if (o->agt_args == NULL) { + if (o->agt_typ == NULL) { PyErr_SetString( PyExc_RuntimeError, "aclose(): asynchronous generator is already running"); @@ -2143,7 +2161,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) o->agt_state = AWAITABLE_STATE_ITER; o->agt_gen->ag_running_async = 1; - if (o->agt_args == NULL) { + if (o->agt_typ == NULL) { /* aclose() mode */ o->agt_gen->ag_closed = 1; @@ -2157,19 +2175,10 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) goto yield_close; } } else { - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; - - if (!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, - &typ, &val, &tb)) { - return NULL; - } - retval = _gen_throw((PyGenObject *)gen, 0, /* Do not close generator when PyExc_GeneratorExit is passed */ - typ, val, tb); + o->agt_typ, o->agt_val, o->agt_tb); retval = async_gen_unwrap_value(o->agt_gen, retval); } if (retval == NULL) { @@ -2181,7 +2190,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) assert(o->agt_state == AWAITABLE_STATE_ITER); retval = gen_send((PyObject *)gen, arg); - if (o->agt_args) { + if (o->agt_typ) { return async_gen_unwrap_value(o->agt_gen, retval); } else { /* aclose() mode */ @@ -2212,7 +2221,7 @@ async_gen_athrow_send(PyObject *self, PyObject *arg) if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - if (o->agt_args == NULL) { + if (o->agt_typ == NULL) { /* when aclose() is called we don't want to propagate StopAsyncIteration or GeneratorExit; just raise StopIteration, signalling that this 'aclose()' await @@ -2241,7 +2250,7 @@ async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (o->agt_state == AWAITABLE_STATE_INIT) { if (o->agt_gen->ag_running_async) { o->agt_state = AWAITABLE_STATE_CLOSED; - if (o->agt_args == NULL) { + if (o->agt_typ == NULL) { PyErr_SetString( PyExc_RuntimeError, "aclose(): asynchronous generator is already running"); @@ -2259,7 +2268,7 @@ async_gen_athrow_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } PyObject *retval = gen_throw((PyObject*)o->agt_gen, args, nargs); - if (o->agt_args) { + if (o->agt_typ) { retval = async_gen_unwrap_value(o->agt_gen, retval); if (retval == NULL) { o->agt_gen->ag_running_async = 0; @@ -2334,7 +2343,7 @@ async_gen_athrow_finalize(PyObject *op) { PyAsyncGenAThrow *o = (PyAsyncGenAThrow*)op; if (o->agt_state == AWAITABLE_STATE_INIT) { - PyObject *method = o->agt_args ? &_Py_ID(athrow) : &_Py_ID(aclose); + PyObject *method = o->agt_typ ? &_Py_ID(athrow) : &_Py_ID(aclose); _PyErr_WarnUnawaitedAgenMethod(o->agt_gen, method); } } @@ -2403,13 +2412,23 @@ PyTypeObject _PyAsyncGenAThrow_Type = { static PyObject * async_gen_athrow_new(PyAsyncGenObject *gen, PyObject *args) { + PyObject *typ = NULL; + PyObject *tb = NULL; + PyObject *val = NULL; + if (args && !PyArg_UnpackTuple(args, "athrow", 1, 3, &typ, &val, &tb)) { + return NULL; + } + PyAsyncGenAThrow *o; o = PyObject_GC_New(PyAsyncGenAThrow, &_PyAsyncGenAThrow_Type); if (o == NULL) { return NULL; } o->agt_gen = (PyAsyncGenObject*)Py_NewRef(gen); - o->agt_args = Py_XNewRef(args); + o->agt_typ = Py_XNewRef(typ); + o->agt_tb = Py_XNewRef(tb); + o->agt_val = Py_XNewRef(val); + o->agt_state = AWAITABLE_STATE_INIT; _PyObject_GC_TRACK((PyObject*)o); return (PyObject*)o; diff --git a/Objects/interpolationobject.c b/Objects/interpolationobject.c index aaea3b8c067..b58adb693f0 100644 --- a/Objects/interpolationobject.c +++ b/Objects/interpolationobject.c @@ -54,7 +54,7 @@ typedef struct { Interpolation.__new__ as interpolation_new value: object - expression: object(subclass_of='&PyUnicode_Type') + expression: object(subclass_of='&PyUnicode_Type', c_default='&_Py_STR(empty)') = "" conversion: object(converter='_conversion_converter') = None format_spec: object(subclass_of='&PyUnicode_Type', c_default='&_Py_STR(empty)') = "" [clinic start generated code]*/ @@ -63,7 +63,7 @@ static PyObject * interpolation_new_impl(PyTypeObject *type, PyObject *value, PyObject *expression, PyObject *conversion, PyObject *format_spec) -/*[clinic end generated code: output=6488e288765bc1a9 input=d91711024068528c]*/ +/*[clinic end generated code: output=6488e288765bc1a9 input=fc5c285c1dd23d36]*/ { interpolationobject *self = PyObject_GC_New(interpolationobject, type); if (!self) { @@ -137,6 +137,8 @@ interpolation_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef interpolation_methods[] = { {"__reduce__", interpolation_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, + {"__class_getitem__", Py_GenericAlias, + METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL}, }; diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 5712e02ae82..e323987601d 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -357,8 +357,9 @@ anextawaitable_getiter(anextawaitableobject *obj) } Py_SETREF(awaitable, new_awaitable); if (!PyIter_Check(awaitable)) { - PyErr_SetString(PyExc_TypeError, - "__await__ returned a non-iterable"); + PyErr_Format(PyExc_TypeError, + "%T.__await__() must return an iterable, not %T", + obj, awaitable); Py_DECREF(awaitable); return NULL; } diff --git a/Objects/listobject.c b/Objects/listobject.c index c5895645a2d..1722ea60cdc 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -6,16 +6,16 @@ #include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_dict.h" // _PyDictViewObject #include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() -#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_interp.h" // PyInterpreterState.list #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() -#include "pycore_stackref.h" // _Py_TryIncrefCompareStackRef() -#include "pycore_tuple.h" // _PyTuple_FromArray() -#include "pycore_typeobject.h" // _Py_TYPE_VERSION_LIST +#include "pycore_pyatomic_ft_wrappers.h" #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_stackref.h" // _Py_TryIncrefCompareStackRef() +#include "pycore_tuple.h" // _PyTuple_FromArraySteal() +#include "pycore_typeobject.h" // _Py_TYPE_VERSION_LIST #include <stddef.h> /*[clinic input] @@ -61,7 +61,8 @@ free_list_items(PyObject** items, bool use_qsbr) #ifdef Py_GIL_DISABLED _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); if (use_qsbr) { - _PyMem_FreeDelayed(array); + size_t size = sizeof(_PyListArray) + array->allocated * sizeof(PyObject *); + _PyMem_FreeDelayed(array, size); } else { PyMem_Free(array); @@ -1381,9 +1382,9 @@ list_extend_dictitems(PyListObject *self, PyDictObject *dict) PyObject **dest = self->ob_item + m; Py_ssize_t pos = 0; Py_ssize_t i = 0; - PyObject *key, *value; - while (_PyDict_Next((PyObject *)dict, &pos, &key, &value, NULL)) { - PyObject *item = PyTuple_Pack(2, key, value); + PyObject *key_value[2]; + while (_PyDict_Next((PyObject *)dict, &pos, &key_value[0], &key_value[1], NULL)) { + PyObject *item = PyTuple_FromArray(key_value, 2); if (item == NULL) { Py_SET_SIZE(self, m + i); return -1; @@ -1684,10 +1685,7 @@ sortslice_advance(sortslice *slice, Py_ssize_t n) /* Avoid malloc for small temp arrays. */ #define MERGESTATE_TEMP_SIZE 256 -/* The largest value of minrun. This must be a power of 2, and >= 1, so that - * the compute_minrun() algorithm guarantees to return a result no larger than - * this, - */ +/* The largest value of minrun. This must be a power of 2, and >= 1 */ #define MAX_MINRUN 64 #if ((MAX_MINRUN) < 1) || ((MAX_MINRUN) & ((MAX_MINRUN) - 1)) #error "MAX_MINRUN must be a power of 2, and >= 1" @@ -1748,6 +1746,11 @@ struct s_MergeState { * of tuples. It may be set to safe_object_compare, but the idea is that hopefully * we can assume more, and use one of the special-case compares. */ int (*tuple_elem_compare)(PyObject *, PyObject *, MergeState *); + + /* Varisbles used for minrun computation. The "ideal" minrun length is + * the infinite precision listlen / 2**e. See listsort.txt. + */ + Py_ssize_t mr_current, mr_e, mr_mask; }; /* binarysort is the best method for sorting small arrays: it does few @@ -2209,6 +2212,14 @@ merge_init(MergeState *ms, Py_ssize_t list_size, int has_keyfunc, ms->min_gallop = MIN_GALLOP; ms->listlen = list_size; ms->basekeys = lo->keys; + + /* State for generating minrun values. See listsort.txt. */ + ms->mr_e = 0; + while (list_size >> ms->mr_e >= MAX_MINRUN) { + ++ms->mr_e; + } + ms->mr_mask = (1 << ms->mr_e) - 1; + ms->mr_current = 0; } /* Free all the temp memory owned by the MergeState. This must be called @@ -2686,27 +2697,15 @@ merge_force_collapse(MergeState *ms) return 0; } -/* Compute a good value for the minimum run length; natural runs shorter - * than this are boosted artificially via binary insertion. - * - * If n < MAX_MINRUN return n (it's too small to bother with fancy stuff). - * Else if n is an exact power of 2, return MAX_MINRUN / 2. - * Else return an int k, MAX_MINRUN / 2 <= k <= MAX_MINRUN, such that n/k is - * close to, but strictly less than, an exact power of 2. - * - * See listsort.txt for more info. - */ -static Py_ssize_t -merge_compute_minrun(Py_ssize_t n) +/* Return the next minrun value to use. See listsort.txt. */ +Py_LOCAL_INLINE(Py_ssize_t) +minrun_next(MergeState *ms) { - Py_ssize_t r = 0; /* becomes 1 if any 1 bits are shifted off */ - - assert(n >= 0); - while (n >= MAX_MINRUN) { - r |= n & 1; - n >>= 1; - } - return n + r; + ms->mr_current += ms->listlen; + assert(ms->mr_current >= 0); /* no overflow */ + Py_ssize_t result = ms->mr_current >> ms->mr_e; + ms->mr_current &= ms->mr_mask; + return result; } /* Here we define custom comparison functions to optimize for the cases one commonly @@ -2877,6 +2876,7 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms) * duplicated). */ /*[clinic input] +@permit_long_docstring_body @critical_section list.sort @@ -2897,7 +2897,7 @@ The reverse flag can be set to sort in descending order. static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=667bf25d0e3a3676]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=e4f6b6069181ad7d]*/ { MergeState ms; Py_ssize_t nremaining; @@ -3074,7 +3074,6 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) /* March over the array once, left to right, finding natural runs, * and extending short natural runs to minrun elements. */ - minrun = merge_compute_minrun(nremaining); do { Py_ssize_t n; @@ -3083,6 +3082,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) if (n < 0) goto fail; /* If short, extend to min(minrun, nremaining). */ + minrun = minrun_next(&ms); if (n < minrun) { const Py_ssize_t force = nremaining <= minrun ? nremaining : minrun; @@ -3221,7 +3221,7 @@ PyList_AsTuple(PyObject *v) PyObject *ret; PyListObject *self = (PyListObject *)v; Py_BEGIN_CRITICAL_SECTION(self); - ret = _PyTuple_FromArray(self->ob_item, Py_SIZE(v)); + ret = PyTuple_FromArray(self->ob_item, Py_SIZE(v)); Py_END_CRITICAL_SECTION(); return ret; } diff --git a/Objects/listsort.txt b/Objects/listsort.txt index f387d9c116e..5b2fc7d50a2 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -270,8 +270,8 @@ result. This has two primary good effects: Computing minrun ---------------- -If N < MAX_MINRUN, minrun is N. IOW, binary insertion sort is used for the -whole array then; it's hard to beat that given the overheads of trying +If N < MAX_MINRUN, minrun is N. IOW, binary insertion sort is used for the +whole array then; it's hard to beat that given the overheads of trying something fancier (see note BINSORT). When N is a power of 2, testing on random data showed that minrun values of @@ -288,7 +288,6 @@ that 32 isn't a good choice for the general case! Consider N=2112: >>> divmod(2112, 32) (66, 0) ->>> If the data is randomly ordered, we're very likely to end up with 66 runs each of length 32. The first 64 of these trigger a sequence of perfectly @@ -301,22 +300,94 @@ to get 64 elements into place). If we take minrun=33 in this case, then we're very likely to end up with 64 runs each of length 33, and then all merges are perfectly balanced. Better! -What we want to avoid is picking minrun such that in +The original code used a cheap heuristic to pick a minrun that avoided the +very worst cases of imbalance for the final merge, but "pretty bad" cases +still existed. - q, r = divmod(N, minrun) +In 2025, Stefan Pochmann found a much better approach, based on letting minrun +vary a bit from one run to the next. Under his scheme, at _all_ levels of the +merge tree: -q is a power of 2 and r>0 (then the last merge only gets r elements into -place, and r < minrun is small compared to N), or q a little larger than a -power of 2 regardless of r (then we've got a case similar to "2112", again -leaving too little work for the last merge to do). +- The number of runs is a power of 2. +- At most two different run lengths appear. +- When two do appear, the smaller is one less than the larger. +- The lengths of run pairs merged never differ by more than one. -Instead we pick a minrun in range(MAX_MINRUN / 2, MAX_MINRUN + 1) such that -N/minrun is exactly a power of 2, or if that isn't possible, is close to, but -strictly less than, a power of 2. This is easier to do than it may sound: -take the first log2(MAX_MINRUN) bits of N, and add 1 if any of the remaining -bits are set. In fact, that rule covers every case in this section, including -small N and exact powers of 2; merge_compute_minrun() is a deceptively simple -function. +So, in all respects, as perfectly balanced as possible. + +For the 2112 case, that also keeps minrun at 33, but we were lucky there +that 2112 is 33 times a power of 2. The new approach doesn't rely on luck. + +For example, with 315 random elements, the old scheme uses fixed minrun=40 and +produces runs of length 40, except for the last. The new scheme produces a +mix of lengths 39 and 40: + +old: 40 40 40 40 40 40 40 35 +new: 39 39 40 39 39 40 39 40 + +Both schemes produce eight runs, a power of 2. That's good for a balanced +merge tree. But the new scheme allows merges where left and right length +never differ by more than 1: + +39 39 40 39 39 40 39 40 + 78 79 79 79 + 157 158 + 315 + +(This shows merges downward, e.g., two runs of length 39 are merged and +become a run of length 78.) + +With larger lists, the old scheme can get even more unbalanced. For example, +with 32769 elements (that's 2**15 + 1), it uses minrun=33 and produces 993 +runs (of length 33). That's not even a power of 2. The new scheme instead +produces 1024 runs, all with length 32 except for the last one with length 33. + +How does it work? Ideally, all runs would be exactly equally long. For the +above example, each run would have 315/8 = 39.375 elements. Which of course +doesn't work. But we can get close: + +For the first run, we'd like 39.375 elements. Since that's impossible, we +instead use 39 (the floor) and remember the current leftover fraction 0.375. +For the second run, we add 0.375 + 39.375 = 39.75. Again impossible, so we +instead use 39 and remember 0.75. For the third run, we add 0.75 + 39.375 = +40.125. This time we get 40 and remember 0.125. And so on. Here's a Python +generator doing that: + +def gen_minruns_with_floats(n): + mr = n + while mr >= MAX_MINRUN: + mr /= 2 + + mr_current = 0 + while True: + mr_current += mr + yield int(mr_current) + mr_current %= 1 + +But while all arithmetic here can be done exactly using binery floating point, +floats have less precision that a Py_ssize_t, and mixing floats with ints is +needlessly expensive anyway. + +So here's an integer version, where the internal numbers are scaled up by +2**e, or rather not divided by 2**e. Instead, only each yielded minrun gets +divided (by right-shifting). For example instead of adding 39.375 and +reducing modulo 1, it just adds 315 and reduces modulo 8. And always divides +by 8 to get each actual minrun value: + +def gen_minruns_simpler(n): + e = 0 + while (n >> e) >= MAX_MINRUN: + e += 1 + mask = (1 << e) - 1 + + mr_current = 0 + while True: + mr_current += n + yield mr_current >> e + mr_current &= mask + +See note MINRUN CODE for a full implementation and a driver that exhaustively +verifies the claims above for all list lengths through 2 million. The Merge Pattern @@ -820,3 +891,75 @@ partially mitigated by pre-scanning the data to determine whether the data is homogeneous with respect to type. If so, it is sometimes possible to substitute faster type-specific comparisons for the slower, generic PyObject_RichCompareBool. + +MINRUN CODE +from itertools import accumulate +try: + from itertools import batched +except ImportError: + from itertools import islice + def batched(xs, k): + it = iter(xs) + while chunk := tuple(islice(it, k)): + yield chunk + +MAX_MINRUN = 64 + +def gen_minruns(n): + # In listobject.c, initialization is done in merge_init(), and + # the body of the loop in minrun_next(). + mr_e = 0 + while (n >> mr_e) >= MAX_MINRUN: + mr_e += 1 + mr_mask = (1 << mr_e) - 1 + + mr_current = 0 + while True: + mr_current += n + yield mr_current >> mr_e + mr_current &= mr_mask + +def chew(n, show=False): + if n < 1: + return + + sizes = [] + tot = 0 + for size in gen_minruns(n): + sizes.append(size) + tot += size + if tot >= n: + break + assert tot == n + print(n, len(sizes)) + + small, large = MAX_MINRUN // 2, MAX_MINRUN + while len(sizes) > 1: + assert not len(sizes) & 1 + assert len(sizes).bit_count() == 1 # i.e., power of 2 + assert sum(sizes) == n + assert min(sizes) >= min(n, small) + assert max(sizes) <= large + + d = set(sizes) + assert len(d) <= 2 + if len(d) == 2: + lo, hi = sorted(d) + assert lo + 1 == hi + + mr = n / len(sizes) + for i, s in enumerate(accumulate(sizes, initial=0)): + assert int(mr * i) == s + + newsizes = [] + for a, b in batched(sizes, 2): + assert abs(a - b) <= 1 + newsizes.append(a + b) + sizes = newsizes + smsll = large + large *= 2 + + assert sizes[0] == n + +for n in range(2_000_001): + chew(n) \ No newline at end of file diff --git a/Objects/longobject.c b/Objects/longobject.c index 40d90ecf4fa..43c0db753a0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -10,6 +10,7 @@ #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS +#include "pycore_stackref.h" #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include "pycore_unicodeobject.h" // _PyUnicode_Equal() @@ -316,6 +317,33 @@ _PyLong_FromSTwoDigits(stwodigits x) return (PyLongObject*)_PyLong_FromLarge(x); } +/* Create a new medium int object from a medium int. + * Do not raise. Return NULL if not medium or can't allocate. */ +static inline _PyStackRef +medium_from_stwodigits(stwodigits x) +{ + if (IS_SMALL_INT(x)) { + return PyStackRef_FromPyObjectBorrow(get_small_int((sdigit)x)); + } + assert(x != 0); + if(!is_medium_int(x)) { + return PyStackRef_NULL; + } + PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); + if (v == NULL) { + v = PyObject_Malloc(sizeof(PyLongObject)); + if (v == NULL) { + return PyStackRef_NULL; + } + _PyObject_Init((PyObject*)v, &PyLong_Type); + } + digit abs_x = x < 0 ? (digit)(-x) : (digit)x; + _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); + v->long_value.ob_digit[0] = abs_x; + return PyStackRef_FromPyObjectStealMortal((PyObject *)v); +} + + /* If a freshly-allocated int is already shared, it must be a small integer, so negating it must go to PyLong_FromLong */ Py_LOCAL_INLINE(void) @@ -324,7 +352,7 @@ _PyLong_Negate(PyLongObject **x_p) PyLongObject *x; x = (PyLongObject *)*x_p; - if (Py_REFCNT(x) == 1) { + if (_PyObject_IsUniquelyReferenced((PyObject *)x)) { _PyLong_FlipSign(x); return; } @@ -498,6 +526,54 @@ PyLong_FromDouble(double dval) #define PY_ABS_LONG_MIN (0-(unsigned long)LONG_MIN) #define PY_ABS_SSIZE_T_MIN (0-(size_t)PY_SSIZE_T_MIN) +static inline unsigned long +unroll_digits_ulong(PyLongObject *v, Py_ssize_t *iptr) +{ + assert(ULONG_MAX >= ((1UL << PyLong_SHIFT) - 1)); + + Py_ssize_t i = *iptr; + assert(i >= 2); + + /* unroll 1 digit */ + --i; + digit *digits = v->long_value.ob_digit; + unsigned long x = digits[i]; + +#if (ULONG_MAX >> PyLong_SHIFT) >= ((1UL << PyLong_SHIFT) - 1) + /* unroll another digit */ + x <<= PyLong_SHIFT; + --i; + x |= digits[i]; +#endif + + *iptr = i; + return x; +} + +static inline size_t +unroll_digits_size_t(PyLongObject *v, Py_ssize_t *iptr) +{ + assert(SIZE_MAX >= ((1UL << PyLong_SHIFT) - 1)); + + Py_ssize_t i = *iptr; + assert(i >= 2); + + /* unroll 1 digit */ + --i; + digit *digits = v->long_value.ob_digit; + size_t x = digits[i]; + +#if (SIZE_MAX >> PyLong_SHIFT) >= ((1 << PyLong_SHIFT) - 1) + /* unroll another digit */ + x <<= PyLong_SHIFT; + --i; + x |= digits[i]; +#endif + + *iptr = i; + return x; +} + /* Get a C long int from an int object or any object that has an __index__ method. @@ -507,13 +583,11 @@ PyLong_FromDouble(double dval) For other errors (e.g., TypeError), return -1 and set an error condition. In this case *overflow will be 0. */ - long PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) { - /* This version by Tim Peters */ + /* This version originally by Tim Peters */ PyLongObject *v; - unsigned long x, prev; long res; Py_ssize_t i; int sign; @@ -556,14 +630,14 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) res = -1; i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; + + unsigned long x = unroll_digits_ulong(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) { + if (x > (ULONG_MAX >> PyLong_SHIFT)) { *overflow = sign; goto exit; } + x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } /* Haven't lost any bits, but casting to long requires extra * care (see comment above). @@ -627,7 +701,6 @@ PyLong_AsInt(PyObject *obj) Py_ssize_t PyLong_AsSsize_t(PyObject *vv) { PyLongObject *v; - size_t x, prev; Py_ssize_t i; int sign; @@ -646,12 +719,13 @@ PyLong_AsSsize_t(PyObject *vv) { } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; + + size_t x = unroll_digits_size_t(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) + if (x > (SIZE_MAX >> PyLong_SHIFT)) { goto overflow; + } + x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } /* Haven't lost any bits, but casting to a signed type requires * extra care (see comment above). @@ -677,7 +751,6 @@ unsigned long PyLong_AsUnsignedLong(PyObject *vv) { PyLongObject *v; - unsigned long x, prev; Py_ssize_t i; if (vv == NULL) { @@ -708,13 +781,13 @@ PyLong_AsUnsignedLong(PyObject *vv) return (unsigned long) -1; } i = _PyLong_DigitCount(v); - x = 0; + + unsigned long x = unroll_digits_ulong(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) { + if (x > (ULONG_MAX >> PyLong_SHIFT)) { goto overflow; } + x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } return x; overflow: @@ -731,7 +804,6 @@ size_t PyLong_AsSize_t(PyObject *vv) { PyLongObject *v; - size_t x, prev; Py_ssize_t i; if (vv == NULL) { @@ -753,16 +825,16 @@ PyLong_AsSize_t(PyObject *vv) return (size_t) -1; } i = _PyLong_DigitCount(v); - x = 0; + + size_t x = unroll_digits_size_t(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) { - PyErr_SetString(PyExc_OverflowError, - "Python int too large to convert to C size_t"); - return (size_t) -1; + if (x > (SIZE_MAX >> PyLong_SHIFT)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C size_t"); + return (size_t) -1; + } + x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } - } return x; } @@ -773,7 +845,6 @@ static unsigned long _PyLong_AsUnsignedLongMask(PyObject *vv) { PyLongObject *v; - unsigned long x; Py_ssize_t i; if (vv == NULL || !PyLong_Check(vv)) { @@ -790,7 +861,7 @@ _PyLong_AsUnsignedLongMask(PyObject *vv) } i = _PyLong_DigitCount(v); int sign = _PyLong_NonCompactSign(v); - x = 0; + unsigned long x = unroll_digits_ulong(v, &i); while (--i >= 0) { x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } @@ -971,16 +1042,9 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, ++numsignificantbytes; } - /* How many Python int digits do we need? We have - 8*numsignificantbytes bits, and each Python int digit has - PyLong_SHIFT bits, so it's the ceiling of the quotient. */ - /* catch overflow before it happens */ - if (numsignificantbytes > (PY_SSIZE_T_MAX - PyLong_SHIFT) / 8) { - PyErr_SetString(PyExc_OverflowError, - "byte array too long to convert to int"); - return NULL; - } - ndigits = (numsignificantbytes * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; + /* avoid integer overflow */ + ndigits = numsignificantbytes / PyLong_SHIFT * 8 + + (numsignificantbytes % PyLong_SHIFT * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; v = long_alloc(ndigits); if (v == NULL) return NULL; @@ -1145,13 +1209,19 @@ _PyLong_AsByteArray(PyLongObject* v, *p = (unsigned char)(accum & 0xff); p += pincr; } - else if (j == n && n > 0 && is_signed) { + else if (j == n && is_signed) { /* The main loop filled the byte array exactly, so the code just above didn't get to ensure there's a sign bit, and the loop below wouldn't add one either. Make sure a sign bit exists. */ - unsigned char msb = *(p - pincr); - int sign_bit_set = msb >= 0x80; + int sign_bit_set; + if (n > 0) { + unsigned char msb = *(p - pincr); + sign_bit_set = msb >= 0x80; + } + else { + sign_bit_set = 0; + } assert(accumbits == 0); if (sign_bit_set == do_twos_comp) return 0; @@ -1598,7 +1668,6 @@ static unsigned long long _PyLong_AsUnsignedLongLongMask(PyObject *vv) { PyLongObject *v; - unsigned long long x; Py_ssize_t i; int sign; @@ -1616,7 +1685,7 @@ _PyLong_AsUnsignedLongLongMask(PyObject *vv) } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; + unsigned long long x = unroll_digits_ulong(v, &i); while (--i >= 0) { x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } @@ -1662,7 +1731,6 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) { /* This version by Tim Peters */ PyLongObject *v; - unsigned long long x, prev; long long res; Py_ssize_t i; int sign; @@ -1704,15 +1772,14 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) else { i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; + unsigned long long x = unroll_digits_ulong(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) + v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) { + if (x > ULLONG_MAX >> PyLong_SHIFT) { *overflow = sign; res = -1; goto exit; } + x = (x << PyLong_SHIFT) + v->long_value.ob_digit[i]; } /* Haven't lost any bits, but casting to long requires extra * care (see comment above). @@ -1760,6 +1827,10 @@ UNSIGNED_INT_CONVERTER(UnsignedInt, unsigned int) UNSIGNED_INT_CONVERTER(UnsignedLong, unsigned long) UNSIGNED_INT_CONVERTER(UnsignedLongLong, unsigned long long) UNSIGNED_INT_CONVERTER(Size_t, size_t) +UNSIGNED_INT_CONVERTER(UInt8, uint8_t) +UNSIGNED_INT_CONVERTER(UInt16, uint16_t) +UNSIGNED_INT_CONVERTER(UInt32, uint32_t) +UNSIGNED_INT_CONVERTER(UInt64, uint64_t) #define CHECK_BINOP(v,w) \ @@ -1949,7 +2020,7 @@ static int pylong_int_to_decimal_string(PyObject *aa, PyObject **p_output, _PyUnicodeWriter *writer, - _PyBytesWriter *bytes_writer, + PyBytesWriter *bytes_writer, char **bytes_str) { PyObject *s = NULL; @@ -1980,7 +2051,8 @@ pylong_int_to_decimal_string(PyObject *aa, Py_ssize_t size = PyUnicode_GET_LENGTH(s); const void *data = PyUnicode_DATA(s); int kind = PyUnicode_KIND(s); - *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, size); + *bytes_str = PyBytesWriter_GrowAndUpdatePointer(bytes_writer, size, + *bytes_str); if (*bytes_str == NULL) { goto error; } @@ -2017,7 +2089,7 @@ static int long_to_decimal_string_internal(PyObject *aa, PyObject **p_output, _PyUnicodeWriter *writer, - _PyBytesWriter *bytes_writer, + PyBytesWriter *bytes_writer, char **bytes_str) { PyLongObject *scratch, *a; @@ -2143,7 +2215,8 @@ long_to_decimal_string_internal(PyObject *aa, } } else if (bytes_writer) { - *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); + *bytes_str = PyBytesWriter_GrowAndUpdatePointer(bytes_writer, strlen, + *bytes_str); if (*bytes_str == NULL) { Py_DECREF(scratch); return -1; @@ -2253,7 +2326,7 @@ long_to_decimal_string(PyObject *aa) static int long_format_binary(PyObject *aa, int base, int alternate, PyObject **p_output, _PyUnicodeWriter *writer, - _PyBytesWriter *bytes_writer, char **bytes_str) + PyBytesWriter *bytes_writer, char **bytes_str) { PyLongObject *a = (PyLongObject *)aa; PyObject *v = NULL; @@ -2314,7 +2387,8 @@ long_format_binary(PyObject *aa, int base, int alternate, return -1; } else if (bytes_writer) { - *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); + *bytes_str = PyBytesWriter_GrowAndUpdatePointer(bytes_writer, sz, + *bytes_str); if (*bytes_str == NULL) return -1; } @@ -2443,7 +2517,7 @@ _PyLong_FormatWriter(_PyUnicodeWriter *writer, } char* -_PyLong_FormatBytesWriter(_PyBytesWriter *writer, char *str, +_PyLong_FormatBytesWriter(PyBytesWriter *writer, char *str, PyObject *obj, int base, int alternate) { @@ -3611,38 +3685,54 @@ long_hash(PyObject *obj) } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; - while (--i >= 0) { - /* Here x is a quantity in the range [0, _PyHASH_MODULUS); we - want to compute x * 2**PyLong_SHIFT + v->long_value.ob_digit[i] modulo - _PyHASH_MODULUS. - The computation of x * 2**PyLong_SHIFT % _PyHASH_MODULUS + // unroll first digit + Py_BUILD_ASSERT(PyHASH_BITS > PyLong_SHIFT); + assert(i >= 1); + --i; + x = v->long_value.ob_digit[i]; + assert(x < PyHASH_MODULUS); + +#if PyHASH_BITS >= 2 * PyLong_SHIFT + // unroll second digit + assert(i >= 1); + --i; + x <<= PyLong_SHIFT; + x += v->long_value.ob_digit[i]; + assert(x < PyHASH_MODULUS); +#endif + + while (--i >= 0) { + /* Here x is a quantity in the range [0, PyHASH_MODULUS); we + want to compute x * 2**PyLong_SHIFT + v->long_value.ob_digit[i] modulo + PyHASH_MODULUS. + + The computation of x * 2**PyLong_SHIFT % PyHASH_MODULUS amounts to a rotation of the bits of x. To see this, write - x * 2**PyLong_SHIFT = y * 2**_PyHASH_BITS + z + x * 2**PyLong_SHIFT = y * 2**PyHASH_BITS + z - where y = x >> (_PyHASH_BITS - PyLong_SHIFT) gives the top + where y = x >> (PyHASH_BITS - PyLong_SHIFT) gives the top PyLong_SHIFT bits of x (those that are shifted out of the - original _PyHASH_BITS bits, and z = (x << PyLong_SHIFT) & - _PyHASH_MODULUS gives the bottom _PyHASH_BITS - PyLong_SHIFT - bits of x, shifted up. Then since 2**_PyHASH_BITS is - congruent to 1 modulo _PyHASH_MODULUS, y*2**_PyHASH_BITS is - congruent to y modulo _PyHASH_MODULUS. So + original PyHASH_BITS bits, and z = (x << PyLong_SHIFT) & + PyHASH_MODULUS gives the bottom PyHASH_BITS - PyLong_SHIFT + bits of x, shifted up. Then since 2**PyHASH_BITS is + congruent to 1 modulo PyHASH_MODULUS, y*2**PyHASH_BITS is + congruent to y modulo PyHASH_MODULUS. So - x * 2**PyLong_SHIFT = y + z (mod _PyHASH_MODULUS). + x * 2**PyLong_SHIFT = y + z (mod PyHASH_MODULUS). The right-hand side is just the result of rotating the - _PyHASH_BITS bits of x left by PyLong_SHIFT places; since - not all _PyHASH_BITS bits of x are 1s, the same is true - after rotation, so 0 <= y+z < _PyHASH_MODULUS and y + z is + PyHASH_BITS bits of x left by PyLong_SHIFT places; since + not all PyHASH_BITS bits of x are 1s, the same is true + after rotation, so 0 <= y+z < PyHASH_MODULUS and y + z is the reduction of x*2**PyLong_SHIFT modulo - _PyHASH_MODULUS. */ - x = ((x << PyLong_SHIFT) & _PyHASH_MODULUS) | - (x >> (_PyHASH_BITS - PyLong_SHIFT)); + PyHASH_MODULUS. */ + x = ((x << PyLong_SHIFT) & PyHASH_MODULUS) | + (x >> (PyHASH_BITS - PyLong_SHIFT)); x += v->long_value.ob_digit[i]; - if (x >= _PyHASH_MODULUS) - x -= _PyHASH_MODULUS; + if (x >= PyHASH_MODULUS) + x -= PyHASH_MODULUS; } x = x * sign; if (x == (Py_uhash_t)-1) @@ -3774,10 +3864,12 @@ long_add(PyLongObject *a, PyLongObject *b) return z; } -PyObject * -_PyLong_Add(PyLongObject *a, PyLongObject *b) +_PyStackRef +_PyCompactLong_Add(PyLongObject *a, PyLongObject *b) { - return (PyObject*)long_add(a, b); + assert(_PyLong_BothAreCompact(a, b)); + stwodigits v = medium_value(a) + medium_value(b); + return medium_from_stwodigits(v); } static PyObject * @@ -3817,10 +3909,12 @@ long_sub(PyLongObject *a, PyLongObject *b) return z; } -PyObject * -_PyLong_Subtract(PyLongObject *a, PyLongObject *b) +_PyStackRef +_PyCompactLong_Subtract(PyLongObject *a, PyLongObject *b) { - return (PyObject*)long_sub(a, b); + assert(_PyLong_BothAreCompact(a, b)); + stwodigits v = medium_value(a) - medium_value(b); + return medium_from_stwodigits(v); } static PyObject * @@ -4264,10 +4358,14 @@ long_mul(PyLongObject *a, PyLongObject *b) return z; } -PyObject * -_PyLong_Multiply(PyLongObject *a, PyLongObject *b) +/* This function returns NULL if the result is not compact, + * or if it fails to allocate, but never raises */ +_PyStackRef +_PyCompactLong_Multiply(PyLongObject *a, PyLongObject *b) { - return (PyObject*)long_mul(a, b); + assert(_PyLong_BothAreCompact(a, b)); + stwodigits v = medium_value(a) * medium_value(b); + return medium_from_stwodigits(v); } static PyObject * @@ -5751,7 +5849,7 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) assert(size_a >= 0); _PyLong_SetSignAndDigitCount(c, 1, size_a); } - else if (Py_REFCNT(a) == 1) { + else if (_PyObject_IsUniquelyReferenced((PyObject *)a)) { c = (PyLongObject*)Py_NewRef(a); } else { @@ -5765,7 +5863,8 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) assert(size_a >= 0); _PyLong_SetSignAndDigitCount(d, 1, size_a); } - else if (Py_REFCNT(b) == 1 && size_a <= alloc_b) { + else if (_PyObject_IsUniquelyReferenced((PyObject *)b) + && size_a <= alloc_b) { d = (PyLongObject*)Py_NewRef(b); assert(size_a >= 0); _PyLong_SetSignAndDigitCount(d, 1, size_a); @@ -6220,6 +6319,7 @@ popcount_digit(digit d) } /*[clinic input] +@permit_long_summary int.bit_count Number of ones in the binary representation of the absolute value of self. @@ -6234,7 +6334,7 @@ Also known as the population count. static PyObject * int_bit_count_impl(PyObject *self) -/*[clinic end generated code: output=2e571970daf1e5c3 input=7e0adef8e8ccdf2e]*/ +/*[clinic end generated code: output=2e571970daf1e5c3 input=f2510a306761db15]*/ { assert(self != NULL); assert(PyLong_Check(self)); @@ -6282,7 +6382,7 @@ int_as_integer_ratio_impl(PyObject *self) /*[clinic input] int.to_bytes - length: Py_ssize_t = 1 + length: Py_ssize_t(allow_negative=False) = 1 Length of bytes object to use. An OverflowError is raised if the integer is not representable with the given number of bytes. Default is length 1. @@ -6304,11 +6404,9 @@ Return an array of bytes representing an integer. static PyObject * int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=89c801df114050a3 input=a0103d0e9ad85c2b]*/ +/*[clinic end generated code: output=89c801df114050a3 input=66f9d0c20529b44f]*/ { int little_endian; - PyObject *bytes; - if (byteorder == NULL) little_endian = 0; else if (_PyUnicode_Equal(byteorder, &_Py_ID(little))) @@ -6321,24 +6419,19 @@ int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, return NULL; } - if (length < 0) { - PyErr_SetString(PyExc_ValueError, - "length argument must be non-negative"); + PyBytesWriter *writer = PyBytesWriter_Create(length); + if (writer == NULL) { return NULL; } - bytes = PyBytes_FromStringAndSize(NULL, length); - if (bytes == NULL) - return NULL; - if (_PyLong_AsByteArray((PyLongObject *)self, - (unsigned char *)PyBytes_AS_STRING(bytes), + PyBytesWriter_GetData(writer), length, little_endian, is_signed, 1) < 0) { - Py_DECREF(bytes); + PyBytesWriter_Discard(writer); return NULL; } - return bytes; + return PyBytesWriter_Finish(writer); } /*[clinic input] diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index cf673fb379e..f1232f38921 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2265,6 +2265,7 @@ memoryview_tolist_impl(PyMemoryViewObject *self) } /*[clinic input] +@permit_long_docstring_body memoryview.tobytes order: str(accept={str, NoneType}, c_default="NULL") = 'C' @@ -2280,11 +2281,10 @@ to C first. order=None is the same as order='C'. static PyObject * memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) -/*[clinic end generated code: output=1288b62560a32a23 input=0efa3ddaeda573a8]*/ +/*[clinic end generated code: output=1288b62560a32a23 input=23c9faf372cfdbcc]*/ { Py_buffer *src = VIEW_ADDR(self); char ord = 'C'; - PyObject *bytes; CHECK_RELEASED(self); @@ -2302,16 +2302,18 @@ memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) } } - bytes = PyBytes_FromStringAndSize(NULL, src->len); - if (bytes == NULL) - return NULL; - - if (PyBuffer_ToContiguous(PyBytes_AS_STRING(bytes), src, src->len, ord) < 0) { - Py_DECREF(bytes); + PyBytesWriter *writer = PyBytesWriter_Create(src->len); + if (writer == NULL) { return NULL; } - return bytes; + if (PyBuffer_ToContiguous(PyBytesWriter_GetData(writer), + src, src->len, ord) < 0) { + PyBytesWriter_Discard(writer); + return NULL; + } + + return PyBytesWriter_Finish(writer); } /*[clinic input] @@ -2343,8 +2345,6 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, /*[clinic end generated code: output=430ca760f94f3ca7 input=539f6a3a5fb56946]*/ { Py_buffer *src = VIEW_ADDR(self); - PyObject *bytes; - PyObject *ret; CHECK_RELEASED(self); @@ -2352,19 +2352,22 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, return _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); } - bytes = PyBytes_FromStringAndSize(NULL, src->len); - if (bytes == NULL) - return NULL; - - if (PyBuffer_ToContiguous(PyBytes_AS_STRING(bytes), src, src->len, 'C') < 0) { - Py_DECREF(bytes); + PyBytesWriter *writer = PyBytesWriter_Create(src->len); + if (writer == NULL) { return NULL; } - ret = _Py_strhex_with_sep( - PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes), - sep, bytes_per_sep); - Py_DECREF(bytes); + if (PyBuffer_ToContiguous(PyBytesWriter_GetData(writer), + src, src->len, 'C') < 0) { + PyBytesWriter_Discard(writer); + return NULL; + } + + PyObject *ret = _Py_strhex_with_sep( + PyBytesWriter_GetData(writer), + PyBytesWriter_GetSize(writer), + sep, bytes_per_sep); + PyBytesWriter_Discard(writer); return ret; } diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c3dcd09ad1c..e6e469ca270 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -8,6 +8,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() /* undefine macro trampoline to PyCFunction_NewEx */ @@ -167,9 +168,7 @@ meth_dealloc(PyObject *self) { PyCFunctionObject *m = _PyCFunctionObject_CAST(self); PyObject_GC_UnTrack(m); - if (m->m_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*) m); - } + FT_CLEAR_WEAKREFS(self, m->m_weakreflist); // We need to access ml_flags here rather than later. // `m->m_ml` might have the same lifetime // as `m_self` when it's dynamically allocated. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f363ef173cb..5a0b16ba572 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -2,18 +2,19 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_EnableGILTransient() #include "pycore_dict.h" // _PyDict_EnablePerThreadRefcounting() #include "pycore_fileutils.h" // _Py_wgetcwd #include "pycore_import.h" // _PyImport_GetNextModuleIndex() #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyModule_CreateInitialized() -#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "osdefs.h" // MAXPATHLEN @@ -27,6 +28,27 @@ static PyMemberDef module_members[] = { {0} }; +static void +assert_def_missing_or_redundant(PyModuleObject *m) +{ + /* We copy all relevant info into the module object. + * Modules created using a def keep a reference to that (statically + * allocated) def; the info there should match what we have in the module. + */ +#ifndef NDEBUG + if (m->md_token_is_def) { + PyModuleDef *def = (PyModuleDef *)m->md_token; + assert(def); +#define DO_ASSERT(F) assert (def->m_ ## F == m->md_state_ ## F); + DO_ASSERT(size); + DO_ASSERT(traverse); + DO_ASSERT(clear); + DO_ASSERT(free); +#undef DO_ASSERT + } +#endif // NDEBUG +} + PyTypeObject PyModuleDef_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -44,14 +66,73 @@ _PyModule_IsExtension(PyObject *obj) } PyModuleObject *module = (PyModuleObject*)obj; - PyModuleDef *def = module->md_def; - return (def != NULL && def->m_methods != NULL); + if (module->md_exec) { + return 1; + } + if (module->md_token_is_def) { + PyModuleDef *def = (PyModuleDef *)module->md_token; + return (module->md_token_is_def && def->m_methods != NULL); + } + return 0; } PyObject* PyModuleDef_Init(PyModuleDef* def) { +#ifdef Py_GIL_DISABLED + // Check that this def does not come from a non-free-threading ABI. + // + // This is meant as a "sanity check"; users should never rely on it. + // In particular, if we run out of ob_flags bits, or otherwise need to + // change some of the internals, this check can go away. Still, it + // would be nice to keep it for the free-threading transition. + // + // A PyModuleDef must be initialized with PyModuleDef_HEAD_INIT, + // which (via PyObject_HEAD_INIT) sets _Py_STATICALLY_ALLOCATED_FLAG + // and not _Py_LEGACY_ABI_CHECK_FLAG. For PyModuleDef, these flags never + // change. + // This means that the lower nibble of a valid PyModuleDef's ob_flags is + // always `_10_` (in binary; `_` is don't care). + // + // So, a check for these bits won't reject valid PyModuleDef. + // Rejecting incompatible extensions is slightly less important; here's + // how that works: + // + // In the pre-free-threading stable ABI, PyModuleDef_HEAD_INIT is big + // enough to overlap with free-threading ABI's ob_flags, is all zeros + // except for the refcount field. + // The refcount field can be: + // - 1 (3.11 and below) + // - UINT_MAX >> 2 (32-bit 3.12 & 3.13) + // - UINT_MAX (64-bit 3.12 & 3.13) + // - 7L << 28 (3.14) + // + // This means that the lower nibble of *any byte* in PyModuleDef_HEAD_INIT + // is not `_10_` -- it can be: + // - 0b0000 + // - 0b0001 + // - 0b0011 (from UINT_MAX >> 2) + // - 0b0111 (from 7L << 28) + // - 0b1111 (e.g. from UINT_MAX) + // (The values may change at runtime as the PyModuleDef is used, but + // PyModuleDef_Init is required before using the def as a Python object, + // so we check at least once with the initial values. + uint16_t flags = ((PyObject*)def)->ob_flags; + uint16_t bits = _Py_STATICALLY_ALLOCATED_FLAG | _Py_LEGACY_ABI_CHECK_FLAG; + if ((flags & bits) != _Py_STATICALLY_ALLOCATED_FLAG) { + const char *message = "invalid PyModuleDef, extension possibly " + "compiled for non-free-threaded Python"; + // Write the error as unraisable: if the extension tries calling + // any API, it's likely to segfault and lose the exception. + PyErr_SetString(PyExc_SystemError, message); + PyErr_WriteUnraisable(NULL); + // But also raise the exception normally -- this is technically + // a recoverable state. + PyErr_SetString(PyExc_SystemError, message); + return NULL; + } +#endif assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY); if (def->m_base.m_index == 0) { Py_SET_REFCNT(def, 1); @@ -93,10 +174,19 @@ new_module_notrack(PyTypeObject *mt) m = (PyModuleObject *)_PyType_AllocNoTrack(mt, 0); if (m == NULL) return NULL; - m->md_def = NULL; m->md_state = NULL; m->md_weaklist = NULL; m->md_name = NULL; + m->md_token_is_def = false; +#ifdef Py_GIL_DISABLED + m->md_requires_gil = true; +#endif + m->md_state_size = 0; + m->md_state_traverse = NULL; + m->md_state_clear = NULL; + m->md_state_free = NULL; + m->md_exec = NULL; + m->md_token = NULL; m->md_dict = PyDict_New(); if (m->md_dict == NULL) { Py_DECREF(m); @@ -211,6 +301,17 @@ PyModule_Create2(PyModuleDef* module, int module_api_version) return _PyModule_CreateInitialized(module, module_api_version); } +static void +module_copy_members_from_deflike( + PyModuleObject *md, + PyModuleDef *def_like /* not necessarily a valid Python object */) +{ + md->md_state_size = def_like->m_size; + md->md_state_traverse = def_like->m_traverse; + md->md_state_clear = def_like->m_clear; + md->md_state_free = def_like->m_free; +} + PyObject * _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) { @@ -257,15 +358,21 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return NULL; } } - m->md_def = module; + m->md_token = module; + m->md_token_is_def = true; + module_copy_members_from_deflike(m, module); #ifdef Py_GIL_DISABLED - m->md_gil = Py_MOD_GIL_USED; + m->md_requires_gil = true; #endif return (PyObject*)m; } -PyObject * -PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) +static PyObject * +module_from_def_and_spec( + PyModuleDef* def_like, /* not necessarily a valid Python object */ + PyObject *spec, + int module_api_version, + PyModuleDef* original_def /* NULL if not defined by a def */) { PyModuleDef_Slot* cur_slot; PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; @@ -274,14 +381,14 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; int has_gil_slot = 0; - void *gil_slot = Py_MOD_GIL_USED; + bool requires_gil = true; int has_execution_slots = 0; const char *name; int ret; + void *token = NULL; + _Py_modexecfunc m_exec = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); - PyModuleDef_Init(def); - nameobj = PyObject_GetAttrString(spec, "name"); if (nameobj == NULL) { return NULL; @@ -295,7 +402,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } - if (def->m_size < 0) { + if (def_like->m_size < 0) { PyErr_Format( PyExc_SystemError, "module %s: m_size may not be negative for multi-phase initialization", @@ -303,7 +410,35 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio goto error; } - for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { + for (cur_slot = def_like->m_slots; cur_slot && cur_slot->slot; cur_slot++) { + // Macro to copy a non-NULL, non-repeatable slot that's unusable with + // PyModuleDef. The destination must be initially NULL. +#define COPY_COMMON_SLOT(SLOT, TYPE, DEST) \ + do { \ + if (!(TYPE)(cur_slot->value)) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: " #SLOT " must not be NULL", \ + name); \ + goto error; \ + } \ + if (original_def) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: " #SLOT " used with PyModuleDef", \ + name); \ + goto error; \ + } \ + if (DEST) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s has more than one " #SLOT " slot", \ + name); \ + goto error; \ + } \ + DEST = (TYPE)(cur_slot->value); \ + } while (0); \ + ///////////////////////////////////////////////////////////////// switch (cur_slot->slot) { case Py_mod_create: if (create) { @@ -317,6 +452,9 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio break; case Py_mod_exec: has_execution_slots = 1; + if (!original_def) { + COPY_COMMON_SLOT(Py_mod_exec, _Py_modexecfunc, m_exec); + } break; case Py_mod_multiple_interpreters: if (has_multiple_interpreters_slot) { @@ -337,9 +475,43 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name); goto error; } - gil_slot = cur_slot->value; + requires_gil = (cur_slot->value != Py_MOD_GIL_NOT_USED); has_gil_slot = 1; break; + case Py_mod_abi: + if (PyABIInfo_Check((PyABIInfo *)cur_slot->value, name) < 0) { + goto error; + } + break; + case Py_mod_name: + COPY_COMMON_SLOT(Py_mod_name, char*, def_like->m_name); + break; + case Py_mod_doc: + COPY_COMMON_SLOT(Py_mod_doc, char*, def_like->m_doc); + break; + case Py_mod_state_size: + COPY_COMMON_SLOT(Py_mod_state_size, Py_ssize_t, + def_like->m_size); + break; + case Py_mod_methods: + COPY_COMMON_SLOT(Py_mod_methods, PyMethodDef*, + def_like->m_methods); + break; + case Py_mod_state_traverse: + COPY_COMMON_SLOT(Py_mod_state_traverse, traverseproc, + def_like->m_traverse); + break; + case Py_mod_state_clear: + COPY_COMMON_SLOT(Py_mod_state_clear, inquiry, + def_like->m_clear); + break; + case Py_mod_state_free: + COPY_COMMON_SLOT(Py_mod_state_free, freefunc, + def_like->m_free); + break; + case Py_mod_token: + COPY_COMMON_SLOT(Py_mod_token, void*, token); + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -348,8 +520,25 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name, cur_slot->slot); goto error; } +#undef COPY_COMMON_SLOT } +#ifdef Py_GIL_DISABLED + // For modules created directly from slots (not from a def), we enable + // the GIL here (pairing `_PyEval_EnableGILTransient` with + // an immediate `_PyImport_EnableGILAndWarn`). + // For modules created from a def, the caller is responsible for this. + if (!original_def && requires_gil) { + PyThreadState *tstate = _PyThreadState_GET(); + if (_PyEval_EnableGILTransient(tstate) < 0) { + goto error; + } + if (_PyImport_EnableGILAndWarn(tstate, nameobj) < 0) { + goto error; + } + } +#endif + /* By default, multi-phase init modules are expected to work under multiple interpreters. */ if (!has_multiple_interpreters_slot) { @@ -371,7 +560,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } if (create) { - m = create(spec, def); + m = create(spec, original_def); if (m == NULL) { if (!PyErr_Occurred()) { PyErr_Format( @@ -397,15 +586,27 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio } if (PyModule_Check(m)) { - ((PyModuleObject*)m)->md_state = NULL; - ((PyModuleObject*)m)->md_def = def; + PyModuleObject *mod = (PyModuleObject*)m; + mod->md_state = NULL; + module_copy_members_from_deflike(mod, def_like); + if (original_def) { + assert (!token); + mod->md_token = original_def; + mod->md_token_is_def = 1; + } + else { + mod->md_token = token; + } #ifdef Py_GIL_DISABLED - ((PyModuleObject*)m)->md_gil = gil_slot; + mod->md_requires_gil = requires_gil; #else - (void)gil_slot; + (void)requires_gil; #endif + mod->md_exec = m_exec; } else { - if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { + if (def_like->m_size > 0 || def_like->m_traverse || def_like->m_clear + || def_like->m_free) + { PyErr_Format( PyExc_SystemError, "module %s is not a module object, but requests module state", @@ -420,17 +621,25 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio name); goto error; } + if (token) { + PyErr_Format( + PyExc_SystemError, + "module %s specifies a token, but did not create " + "a ModuleType instance", + name); + goto error; + } } - if (def->m_methods != NULL) { - ret = _add_methods_to_object(m, nameobj, def->m_methods); + if (def_like->m_methods != NULL) { + ret = _add_methods_to_object(m, nameobj, def_like->m_methods); if (ret != 0) { goto error; } } - if (def->m_doc != NULL) { - ret = PyModule_SetDocString(m, def->m_doc); + if (def_like->m_doc != NULL) { + ret = PyModule_SetDocString(m, def_like->m_doc); if (ret != 0) { goto error; } @@ -445,83 +654,131 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +PyObject * +PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) +{ + PyModuleDef_Init(def); + return module_from_def_and_spec(def, spec, module_api_version, def); +} + +PyObject * +PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) +{ + if (!slots) { + PyErr_SetString( + PyExc_SystemError, + "PyModule_FromSlotsAndSpec called with NULL slots"); + return NULL; + } + // Fill in enough of a PyModuleDef to pass to common machinery + PyModuleDef def_like = {.m_slots = (PyModuleDef_Slot *)slots}; + + return module_from_def_and_spec(&def_like, spec, PYTHON_API_VERSION, + NULL); +} + #ifdef Py_GIL_DISABLED int PyUnstable_Module_SetGIL(PyObject *module, void *gil) { + bool requires_gil = (gil != Py_MOD_GIL_NOT_USED); if (!PyModule_Check(module)) { PyErr_BadInternalCall(); return -1; } - ((PyModuleObject *)module)->md_gil = gil; + ((PyModuleObject *)module)->md_requires_gil = requires_gil; return 0; } #endif +static int +run_exec_func(PyObject *module, int (*exec)(PyObject *)) +{ + int ret = exec(module); + if (ret != 0) { + if (!PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "execution of %R failed without setting an exception", + module); + } + return -1; + } + if (PyErr_Occurred()) { + _PyErr_FormatFromCause( + PyExc_SystemError, + "execution of module %R raised unreported exception", + module); + return -1; + } + return 0; +} + +static int +alloc_state(PyObject *module) +{ + if (!PyModule_Check(module)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", module); + return -1; + } + PyModuleObject *md = (PyModuleObject*)module; + + if (md->md_state_size >= 0) { + if (md->md_state == NULL) { + /* Always set a state pointer; this serves as a marker to skip + * multiple initialization (importlib.reload() is no-op) */ + md->md_state = PyMem_Malloc(md->md_state_size); + if (!md->md_state) { + PyErr_NoMemory(); + return -1; + } + memset(md->md_state, 0, md->md_state_size); + } + } + return 0; +} + +int +PyModule_Exec(PyObject *module) +{ + if (alloc_state(module) < 0) { + return -1; + } + PyModuleObject *md = (PyModuleObject*)module; + if (md->md_exec) { + assert(!md->md_token_is_def); + return run_exec_func(module, md->md_exec); + } + + PyModuleDef *def = _PyModule_GetDefOrNull(module); + if (def) { + return PyModule_ExecDef(module, def); + } + return 0; +} + int PyModule_ExecDef(PyObject *module, PyModuleDef *def) { PyModuleDef_Slot *cur_slot; - const char *name; - int ret; - name = PyModule_GetName(module); - if (name == NULL) { + if (alloc_state(module) < 0) { return -1; } - if (def->m_size >= 0) { - PyModuleObject *md = (PyModuleObject*)module; - if (md->md_state == NULL) { - /* Always set a state pointer; this serves as a marker to skip - * multiple initialization (importlib.reload() is no-op) */ - md->md_state = PyMem_Malloc(def->m_size); - if (!md->md_state) { - PyErr_NoMemory(); - return -1; - } - memset(md->md_state, 0, def->m_size); - } - } + assert(PyModule_Check(module)); if (def->m_slots == NULL) { return 0; } for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { - switch (cur_slot->slot) { - case Py_mod_create: - /* handled in PyModule_FromDefAndSpec2 */ - break; - case Py_mod_exec: - ret = ((int (*)(PyObject *))cur_slot->value)(module); - if (ret != 0) { - if (!PyErr_Occurred()) { - PyErr_Format( - PyExc_SystemError, - "execution of module %s failed without setting an exception", - name); - } - return -1; - } - if (PyErr_Occurred()) { - _PyErr_FormatFromCause( - PyExc_SystemError, - "execution of module %s raised unreported exception", - name); - return -1; - } - break; - case Py_mod_multiple_interpreters: - case Py_mod_gil: - /* handled in PyModule_FromDefAndSpec2 */ - break; - default: - PyErr_Format( - PyExc_SystemError, - "module %s initialized with unknown slot %i", - name, cur_slot->slot); + if (cur_slot->slot == Py_mod_exec) { + int (*func)(PyObject *) = cur_slot->value; + if (run_exec_func(module, func) < 0) { return -1; + } + continue; } } return 0; @@ -565,6 +822,31 @@ PyModule_GetDict(PyObject *m) return _PyModule_GetDict(m); // borrowed reference } +int +PyModule_GetStateSize(PyObject *m, Py_ssize_t *size_p) +{ + *size_p = -1; + if (!PyModule_Check(m)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", m); + return -1; + } + PyModuleObject *mod = (PyModuleObject *)m; + *size_p = mod->md_state_size; + return 0; +} + +int +PyModule_GetToken(PyObject *m, void **token_p) +{ + *token_p = NULL; + if (!PyModule_Check(m)) { + PyErr_Format(PyExc_TypeError, "expected module, got %T", m); + return -1; + } + *token_p = _PyModule_GetToken(m); + return 0; +} + PyObject* PyModule_GetNameObject(PyObject *mod) { @@ -705,7 +987,7 @@ PyModule_GetDef(PyObject* m) PyErr_BadArgument(); return NULL; } - return _PyModule_GetDef(m); + return _PyModule_GetDefOrNull(m); } void* @@ -827,20 +1109,20 @@ module_dealloc(PyObject *self) if (verbose && m->md_name) { PySys_FormatStderr("# destroy %U\n", m->md_name); } - if (m->md_weaklist != NULL) - PyObject_ClearWeakRefs((PyObject *) m); + FT_CLEAR_WEAKREFS(self, m->md_weaklist); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_free - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_free && (m->md_state_size <= 0 || m->md_state != NULL)) { - m->md_def->m_free(m); + m->md_state_free(m); } Py_XDECREF(m->md_dict); Py_XDECREF(m->md_name); - if (m->md_state != NULL) + if (m->md_state != NULL) { PyMem_Free(m->md_state); + } Py_TYPE(m)->tp_free((PyObject *)m); } @@ -1058,7 +1340,7 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) int is_possibly_shadowing_stdlib = 0; if (is_possibly_shadowing) { PyObject *stdlib_modules; - if (_PySys_GetOptionalAttrString("stdlib_module_names", &stdlib_modules) < 0) { + if (PySys_GetOptionalAttrString("stdlib_module_names", &stdlib_modules) < 0) { goto done; } if (stdlib_modules && PyAnySet_Check(stdlib_modules)) { @@ -1148,11 +1430,11 @@ module_traverse(PyObject *self, visitproc visit, void *arg) { PyModuleObject *m = _PyModule_CAST(self); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_traverse() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_traverse - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_traverse && (m->md_state_size <= 0 || m->md_state != NULL)) { - int res = m->md_def->m_traverse((PyObject*)m, visit, arg); + int res = m->md_state_traverse((PyObject*)m, visit, arg); if (res) return res; } @@ -1166,18 +1448,19 @@ module_clear(PyObject *self) { PyModuleObject *m = _PyModule_CAST(self); + assert_def_missing_or_redundant(m); /* bpo-39824: Don't call m_clear() if m_size > 0 and md_state=NULL */ - if (m->md_def && m->md_def->m_clear - && (m->md_def->m_size <= 0 || m->md_state != NULL)) + if (m->md_state_clear && (m->md_state_size <= 0 || m->md_state != NULL)) { - int res = m->md_def->m_clear((PyObject*)m); + int res = m->md_state_clear((PyObject*)m); if (PyErr_Occurred()) { PyErr_FormatUnraisable("Exception ignored in m_clear of module%s%V", m->md_name ? " " : "", m->md_name, ""); } - if (res) + if (res) { return res; + } } Py_CLEAR(m->md_dict); return 0; @@ -1330,8 +1613,9 @@ module_get_annotations(PyObject *self, void *Py_UNUSED(ignored)) return NULL; } if (!PyDict_Check(annotations)) { - PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", - Py_TYPE(annotations)->tp_name); + PyErr_Format(PyExc_TypeError, + "__annotate__() must return a dict, not %T", + annotations); Py_DECREF(annotate); Py_DECREF(annotations); Py_DECREF(dict); diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index caebe6bf543..201cb8a7df8 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -124,9 +124,10 @@ namespace_repr(PyObject *ns) if (PyUnicode_Check(key) && PyUnicode_GET_LENGTH(key) > 0) { PyObject *value, *item; - value = PyDict_GetItemWithError(d, key); - if (value != NULL) { + int has_key = PyDict_GetItemRef(d, key, &value); + if (has_key == 1) { item = PyUnicode_FromFormat("%U=%R", key, value); + Py_DECREF(value); if (item == NULL) { loop_error = 1; } @@ -135,7 +136,7 @@ namespace_repr(PyObject *ns) Py_DECREF(item); } } - else if (PyErr_Occurred()) { + else if (has_key < 0) { loop_error = 1; } } @@ -193,10 +194,14 @@ namespace_clear(PyObject *op) static PyObject * namespace_richcompare(PyObject *self, PyObject *other, int op) { - if (PyObject_TypeCheck(self, &_PyNamespace_Type) && - PyObject_TypeCheck(other, &_PyNamespace_Type)) + if ( + (op == Py_EQ || op == Py_NE) && + PyObject_TypeCheck(self, &_PyNamespace_Type) && + PyObject_TypeCheck(other, &_PyNamespace_Type) + ) { return PyObject_RichCompare(((_PyNamespaceObject *)self)->ns_dict, ((_PyNamespaceObject *)other)->ns_dict, op); + } Py_RETURN_NOTIMPLEMENTED; } diff --git a/Objects/object.c b/Objects/object.c index 723b0427e69..fcea3503de8 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -713,7 +713,7 @@ _PyObject_IsFreed(PyObject *op) /* For debugging convenience. See Misc/gdbinit for some useful gdb hooks */ void -_PyObject_Dump(PyObject* op) +PyUnstable_Object_Dump(PyObject* op) { if (_PyObject_IsFreed(op)) { /* It seems like the object memory has been freed: @@ -784,8 +784,7 @@ PyObject_Repr(PyObject *v) } if (!PyUnicode_Check(res)) { _PyErr_Format(tstate, PyExc_TypeError, - "__repr__ returned non-string (type %.200s)", - Py_TYPE(res)->tp_name); + "%T.__repr__() must return a str, not %T", v, res); Py_DECREF(res); return NULL; } @@ -827,8 +826,7 @@ PyObject_Str(PyObject *v) } if (!PyUnicode_Check(res)) { _PyErr_Format(tstate, PyExc_TypeError, - "__str__ returned non-string (type %.200s)", - Py_TYPE(res)->tp_name); + "%T.__str__() must return a str, not %T", v, res); Py_DECREF(res); return NULL; } @@ -883,8 +881,8 @@ PyObject_Bytes(PyObject *v) return NULL; if (!PyBytes_Check(result)) { PyErr_Format(PyExc_TypeError, - "__bytes__ returned non-bytes (type %.200s)", - Py_TYPE(result)->tp_name); + "%T.__bytes__() must return a bytes, not %T", + v, result); Py_DECREF(result); return NULL; } @@ -925,6 +923,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() clear_freelist(&freelists->floats, is_finalization, free_object); + clear_freelist(&freelists->complexes, is_finalization, free_object); for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { clear_freelist(&freelists->tuples[i], is_finalization, free_object); } @@ -946,6 +945,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); } clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); + clear_freelist(&freelists->bytes_writers, is_finalization, PyMem_Free); clear_freelist(&freelists->ints, is_finalization, free_object); clear_freelist(&freelists->pycfunctionobject, is_finalization, PyObject_GC_Del); clear_freelist(&freelists->pycmethodobject, is_finalization, PyObject_GC_Del); @@ -1130,11 +1130,14 @@ PyObject_RichCompareBool(PyObject *v, PyObject *w, int op) res = PyObject_RichCompare(v, w, op); if (res == NULL) return -1; - if (PyBool_Check(res)) + if (PyBool_Check(res)) { ok = (res == Py_True); - else + assert(_Py_IsImmortal(res)); + } + else { ok = PyObject_IsTrue(res); - Py_DECREF(res); + Py_DECREF(res); + } return ok; } @@ -1209,16 +1212,27 @@ PyObject_HasAttrString(PyObject *obj, const char *name) int PyObject_SetAttrString(PyObject *v, const char *name, PyObject *w) { - PyObject *s; - int res; - - if (Py_TYPE(v)->tp_setattr != NULL) - return (*Py_TYPE(v)->tp_setattr)(v, (char*)name, w); - s = PyUnicode_InternFromString(name); - if (s == NULL) + PyThreadState *tstate = _PyThreadState_GET(); + if (w == NULL && _PyErr_Occurred(tstate)) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyErr_SetString(tstate, PyExc_SystemError, + "PyObject_SetAttrString() must not be called with NULL value " + "and an exception set"); + _PyErr_ChainExceptions1Tstate(tstate, exc); return -1; - res = PyObject_SetAttr(v, s, w); - Py_XDECREF(s); + } + + if (Py_TYPE(v)->tp_setattr != NULL) { + return (*Py_TYPE(v)->tp_setattr)(v, (char*)name, w); + } + + PyObject *s = PyUnicode_InternFromString(name); + if (s == NULL) { + return -1; + } + + int res = PyObject_SetAttr(v, s, w); + Py_DECREF(s); return res; } @@ -1436,6 +1450,16 @@ PyObject_HasAttr(PyObject *obj, PyObject *name) int PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) { + PyThreadState *tstate = _PyThreadState_GET(); + if (value == NULL && _PyErr_Occurred(tstate)) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyErr_SetString(tstate, PyExc_SystemError, + "PyObject_SetAttr() must not be called with NULL value " + "and an exception set"); + _PyErr_ChainExceptions1Tstate(tstate, exc); + return -1; + } + PyTypeObject *tp = Py_TYPE(v); int err; @@ -1447,8 +1471,7 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) } Py_INCREF(name); - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUnicode_InternMortal(interp, &name); + _PyUnicode_InternMortal(tstate->interp, &name); if (tp->tp_setattro != NULL) { err = (*tp->tp_setattro)(v, name, value); Py_DECREF(name); @@ -1664,6 +1687,111 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } +int +_PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, + PyObject *name, _PyStackRef *method) +{ + int meth_found = 0; + + assert(PyStackRef_IsNull(*method)); + + PyTypeObject *tp = Py_TYPE(obj); + if (!_PyType_IsReady(tp)) { + if (PyType_Ready(tp) < 0) { + return 0; + } + } + + if (tp->tp_getattro != PyObject_GenericGetAttr || !PyUnicode_CheckExact(name)) { + PyObject *res = PyObject_GetAttr(obj, name); + if (res != NULL) { + *method = PyStackRef_FromPyObjectSteal(res); + } + return 0; + } + + _PyType_LookupStackRefAndVersion(tp, name, method); + PyObject *descr = PyStackRef_AsPyObjectBorrow(*method); + descrgetfunc f = NULL; + if (descr != NULL) { + if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { + meth_found = 1; + } + else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + PyObject *value = f(descr, obj, (PyObject *)Py_TYPE(obj)); + PyStackRef_CLEAR(*method); + if (value != NULL) { + *method = PyStackRef_FromPyObjectSteal(value); + } + return 0; + } + } + } + PyObject *dict, *attr; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(obj, name, &attr)) { + if (attr != NULL) { + PyStackRef_CLEAR(*method); + *method = PyStackRef_FromPyObjectSteal(attr); + return 0; + } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); + } + else { + PyObject **dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr != NULL) { + dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); + } + else { + dict = NULL; + } + } + if (dict != NULL) { + assert(PyUnicode_CheckExact(name)); + int found = _PyDict_GetMethodStackRef((PyDictObject *)dict, name, method); + if (found < 0) { + assert(PyStackRef_IsNull(*method)); + return -1; + } + else if (found) { + return 0; + } + } + + if (meth_found) { + assert(!PyStackRef_IsNull(*method)); + return 1; + } + + if (f != NULL) { + PyObject *value = f(descr, obj, (PyObject *)Py_TYPE(obj)); + PyStackRef_CLEAR(*method); + if (value) { + *method = PyStackRef_FromPyObjectSteal(value); + } + return 0; + } + + if (descr != NULL) { + assert(!PyStackRef_IsNull(*method)); + return 0; + } + + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + + _PyObject_SetAttributeErrorContext(obj, name); + assert(PyStackRef_IsNull(*method)); + return 0; +} + + /* Generic GetAttr functions - put these in your tp_[gs]etattro slot. */ PyObject * @@ -1906,34 +2034,11 @@ PyObject_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) int PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { - PyObject **dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && - _PyObject_GetManagedDict(obj) == NULL - ) { - /* Was unable to convert to dict */ - PyErr_NoMemory(); - } - else { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - } - return -1; - } if (value == NULL) { PyErr_SetString(PyExc_TypeError, "cannot delete __dict__"); return -1; } - if (!PyDict_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__dict__ must be set to a dictionary, " - "not a '%.200s'", Py_TYPE(value)->tp_name); - return -1; - } - Py_BEGIN_CRITICAL_SECTION(obj); - Py_XSETREF(*dictptr, Py_NewRef(value)); - Py_END_CRITICAL_SECTION(); - return 0; + return _PyObject_SetDict(obj, value); } @@ -1996,9 +2101,25 @@ _dir_locals(void) PyObject *names; PyObject *locals; - locals = _PyEval_GetFrameLocals(); - if (locals == NULL) + if (_PyEval_GetFrame() != NULL) { + locals = _PyEval_GetFrameLocals(); + } + else { + PyThreadState *tstate = _PyThreadState_GET(); + locals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (locals == NULL) { + if (!_PyErr_Occurred(tstate)) { + locals = _PyEval_GetFrameLocals(); + assert(_PyErr_Occurred(tstate)); + } + } + else { + Py_INCREF(locals); + } + } + if (locals == NULL) { return NULL; + } names = PyMapping_Keys(locals); Py_DECREF(locals); @@ -2930,62 +3051,33 @@ Py_ReprLeave(PyObject *obj) /* Trashcan support. */ -#ifndef Py_GIL_DISABLED -/* We need to store a pointer in the refcount field of - * an object. It is important that we never store 0 (NULL). -* It is also important to not make the object appear immortal, -* or it might be untracked by the cycle GC. */ -static uintptr_t -pointer_to_safe_refcount(void *ptr) -{ - uintptr_t full = (uintptr_t)ptr; - assert((full & 3) == 0); -#if SIZEOF_VOID_P > 4 - uint32_t refcnt = (uint32_t)full; - if (refcnt >= (uint32_t)_Py_IMMORTAL_MINIMUM_REFCNT) { - full = full - ((uintptr_t)_Py_IMMORTAL_MINIMUM_REFCNT) + 1; - } - return full + 2; -#else - // Make the top two bits 0, so it appears mortal. - return (full >> 2) + 1; -#endif -} - -static void * -safe_refcount_to_pointer(uintptr_t refcnt) -{ -#if SIZEOF_VOID_P > 4 - if (refcnt & 1) { - refcnt += _Py_IMMORTAL_MINIMUM_REFCNT - 1; - } - return (void *)(refcnt - 2); -#else - return (void *)((refcnt -1) << 2); -#endif -} -#endif - -/* Add op to the gcstate->trash_delete_later list. Called when the current - * call-stack depth gets large. op must be a currently untracked gc'ed - * object, with refcount 0. Py_DECREF must already have been called on it. +/* Add op to the tstate->delete_later list. Called when the current + * call-stack depth gets large. op must be a gc'ed object, with refcount 0. + * Py_DECREF must already have been called on it. */ void _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op) { _PyObject_ASSERT(op, Py_REFCNT(op) == 0); + PyTypeObject *tp = Py_TYPE(op); + assert(tp->tp_flags & Py_TPFLAGS_HAVE_GC); + int tracked = 0; + if (tp->tp_is_gc == NULL || tp->tp_is_gc(op)) { + tracked = _PyObject_GC_IS_TRACKED(op); + if (tracked) { + _PyObject_GC_UNTRACK(op); + } + } + uintptr_t tagged_ptr = ((uintptr_t)tstate->delete_later) | tracked; #ifdef Py_GIL_DISABLED - op->ob_tid = (uintptr_t)tstate->delete_later; + op->ob_tid = tagged_ptr; #else - /* Store the delete_later pointer in the refcnt field. */ - uintptr_t refcnt = pointer_to_safe_refcount(tstate->delete_later); - *((uintptr_t*)op) = refcnt; - assert(!_Py_IsImmortal(op)); + _Py_AS_GC(op)->_gc_next = tagged_ptr; #endif tstate->delete_later = op; } -/* Deallocate all the objects in the gcstate->trash_delete_later list. +/* Deallocate all the objects in the tstate->delete_later list. * Called when the call-stack unwinds again. */ void _PyTrash_thread_destroy_chain(PyThreadState *tstate) @@ -2995,17 +3087,17 @@ _PyTrash_thread_destroy_chain(PyThreadState *tstate) destructor dealloc = Py_TYPE(op)->tp_dealloc; #ifdef Py_GIL_DISABLED - tstate->delete_later = (PyObject*) op->ob_tid; + uintptr_t tagged_ptr = op->ob_tid; op->ob_tid = 0; _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED); #else - /* Get the delete_later pointer from the refcnt field. - * See _PyTrash_thread_deposit_object(). */ - uintptr_t refcnt = *((uintptr_t*)op); - tstate->delete_later = safe_refcount_to_pointer(refcnt); - op->ob_refcnt = 0; + uintptr_t tagged_ptr = _Py_AS_GC(op)->_gc_next; + _Py_AS_GC(op)->_gc_next = 0; #endif - + tstate->delete_later = (PyObject *)(tagged_ptr & ~1); + if (tagged_ptr & 1) { + _PyObject_GC_TRACK(op); + } /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but * Py_DECREF was already called on this object, and in @@ -3058,7 +3150,7 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, /* This might succeed or fail, but we're about to abort, so at least try to provide any extra info we can: */ - _PyObject_Dump(obj); + PyUnstable_Object_Dump(obj); fprintf(stderr, "\n"); fflush(stderr); @@ -3079,10 +3171,11 @@ void _Py_Dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); + unsigned long gc_flag = type->tp_flags & Py_TPFLAGS_HAVE_GC; destructor dealloc = type->tp_dealloc; PyThreadState *tstate = _PyThreadState_GET(); intptr_t margin = _Py_RecursionLimit_GetMargin(tstate); - if (margin < 2) { + if (margin < 2 && gc_flag) { _PyTrash_thread_deposit_object(tstate, (PyObject *)op); return; } @@ -3128,7 +3221,7 @@ _Py_Dealloc(PyObject *op) Py_XDECREF(old_exc); Py_DECREF(type); #endif - if (tstate->delete_later && margin >= 4) { + if (tstate->delete_later && margin >= 4 && gc_flag) { _PyTrash_thread_destroy_chain(tstate); } } @@ -3194,8 +3287,18 @@ _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { _Py_AssertHoldsTstate(); + + _PyEval_StopTheWorldAll(&_PyRuntime); + if (_PyRuntime.ref_tracer.tracer_func != NULL) { + _PyReftracerTrack(NULL, PyRefTracer_TRACKER_REMOVED); + if (PyErr_Occurred()) { + _PyEval_StartTheWorldAll(&_PyRuntime); + return -1; + } + } _PyRuntime.ref_tracer.tracer_func = tracer; _PyRuntime.ref_tracer.tracer_data = data; + _PyEval_StartTheWorldAll(&_PyRuntime); return 0; } @@ -3258,24 +3361,6 @@ Py_GetConstantBorrowed(unsigned int constant_id) return Py_GetConstant(constant_id); } - -// Py_TYPE() implementation for the stable ABI -#undef Py_TYPE -PyTypeObject* -Py_TYPE(PyObject *ob) -{ - return _Py_TYPE(ob); -} - - -// Py_REFCNT() implementation for the stable ABI -#undef Py_REFCNT -Py_ssize_t -Py_REFCNT(PyObject *ob) -{ - return _Py_REFCNT(ob); -} - int PyUnstable_IsImmortal(PyObject *op) { @@ -3292,3 +3377,26 @@ PyUnstable_Object_IsUniquelyReferenced(PyObject *op) assert(op != NULL); return _PyObject_IsUniquelyReferenced(op); } + +int +_PyObject_VisitType(PyObject *op, visitproc visit, void *arg) +{ + assert(op != NULL); + PyTypeObject *tp = Py_TYPE(op); + _PyObject_ASSERT((PyObject *)tp, PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); + Py_VISIT(tp); + return 0; +} + +// Implementations for the stable ABI +// Keep these at the end. +#undef Py_TYPE +#undef Py_REFCNT +#undef Py_SIZE +#undef Py_IS_TYPE +#undef Py_SET_SIZE +PyTypeObject* Py_TYPE(PyObject *ob) { return _Py_TYPE_impl(ob); } +Py_ssize_t Py_REFCNT(PyObject *ob) { return _Py_REFCNT(ob); } +Py_ssize_t Py_SIZE(PyObject *o) { return _Py_SIZE_impl(o); } +int Py_IS_TYPE(PyObject *o, PyTypeObject *t) { return _Py_IS_TYPE_impl(o, t); } +void Py_SET_SIZE(PyVarObject *o, Py_ssize_t s) { _Py_SET_SIZE_impl(o, s); } diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png index a63d095ea0b..9ccb99f9db1 100644 Binary files a/Objects/object_layout_312.png and b/Objects/object_layout_312.png differ diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png index f7059c286c8..f3df59253da 100644 Binary files a/Objects/object_layout_313.png and b/Objects/object_layout_313.png differ diff --git a/Objects/object_layout_full_312.png b/Objects/object_layout_full_312.png index 4f46ca86091..8c0aca4e06a 100644 Binary files a/Objects/object_layout_full_312.png and b/Objects/object_layout_full_312.png differ diff --git a/Objects/object_layout_full_313.png b/Objects/object_layout_full_313.png index 7352ec69e5d..2d5c8569f62 100644 Binary files a/Objects/object_layout_full_313.png and b/Objects/object_layout_full_313.png differ diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index b209808da90..2b95ebbf8e5 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -124,6 +124,33 @@ _PyMem_mi_page_is_safe_to_free(mi_page_t *page) } +#ifdef Py_GIL_DISABLED + +// If we are deferring collection of more than this amount of memory for +// mimalloc pages, advance the write sequence. Advancing allows these +// pages to be re-used in a different thread or for a different size class. +#define QSBR_PAGE_MEM_LIMIT 4096*20 + +// Return true if the global write sequence should be advanced for a mimalloc +// page that is deferred from collection. +static bool +should_advance_qsbr_for_page(struct _qsbr_thread_state *qsbr, mi_page_t *page) +{ + size_t bsize = mi_page_block_size(page); + size_t page_size = page->capacity*bsize; + if (page_size > QSBR_PAGE_MEM_LIMIT) { + qsbr->deferred_page_memory = 0; + return true; + } + qsbr->deferred_page_memory += page_size; + if (qsbr->deferred_page_memory > QSBR_PAGE_MEM_LIMIT) { + qsbr->deferred_page_memory = 0; + return true; + } + return false; +} +#endif + static bool _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) { @@ -139,7 +166,14 @@ _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) _PyMem_mi_page_clear_qsbr(page); page->retire_expire = 0; - page->qsbr_goal = _Py_qsbr_deferred_advance(tstate->qsbr); + + if (should_advance_qsbr_for_page(tstate->qsbr, page)) { + page->qsbr_goal = _Py_qsbr_advance(tstate->qsbr->shared); + } + else { + page->qsbr_goal = _Py_qsbr_shared_next(tstate->qsbr->shared); + } + llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); return false; } @@ -1141,8 +1175,44 @@ free_work_item(uintptr_t ptr, delayed_dealloc_cb cb, void *state) } } + +#ifdef Py_GIL_DISABLED + +// For deferred advance on free: the number of deferred items before advancing +// the write sequence. This is based on WORK_ITEMS_PER_CHUNK. We ideally +// want to process a chunk before it overflows. +#define QSBR_DEFERRED_LIMIT 127 + +// If the deferred memory exceeds 1 MiB, advance the write sequence. This +// helps limit memory usage due to QSBR delaying frees too long. +#define QSBR_FREE_MEM_LIMIT 1024*1024 + +// Return true if the global write sequence should be advanced for a deferred +// memory free. +static bool +should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size) +{ + if (size > QSBR_FREE_MEM_LIMIT) { + qsbr->deferred_count = 0; + qsbr->deferred_memory = 0; + qsbr->should_process = true; + return true; + } + qsbr->deferred_count++; + qsbr->deferred_memory += size; + if (qsbr->deferred_count > QSBR_DEFERRED_LIMIT || + qsbr->deferred_memory > QSBR_FREE_MEM_LIMIT) { + qsbr->deferred_count = 0; + qsbr->deferred_memory = 0; + qsbr->should_process = true; + return true; + } + return false; +} +#endif + static void -free_delayed(uintptr_t ptr) +free_delayed(uintptr_t ptr, size_t size) { #ifndef Py_GIL_DISABLED free_work_item(ptr, NULL, NULL); @@ -1200,23 +1270,32 @@ free_delayed(uintptr_t ptr) } assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK); - uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr); + uint64_t seq; + if (should_advance_qsbr_for_free(tstate->qsbr, size)) { + seq = _Py_qsbr_advance(tstate->qsbr->shared); + } + else { + seq = _Py_qsbr_shared_next(tstate->qsbr->shared); + } buf->array[buf->wr_idx].ptr = ptr; buf->array[buf->wr_idx].qsbr_goal = seq; buf->wr_idx++; if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + // Normally the processing of delayed items is done from the eval + // breaker. Processing here is a safety measure to ensure too much + // work does not accumulate. _PyMem_ProcessDelayed((PyThreadState *)tstate); } #endif } void -_PyMem_FreeDelayed(void *ptr) +_PyMem_FreeDelayed(void *ptr, size_t size) { assert(!((uintptr_t)ptr & 0x01)); if (ptr != NULL) { - free_delayed((uintptr_t)ptr); + free_delayed((uintptr_t)ptr, size); } } @@ -1226,7 +1305,25 @@ _PyObject_XDecRefDelayed(PyObject *ptr) { assert(!((uintptr_t)ptr & 0x01)); if (ptr != NULL) { - free_delayed(((uintptr_t)ptr)|0x01); + // We use 0 as the size since we don't have an easy way to know the + // actual size. If we are freeing many objects, the write sequence + // will be advanced due to QSBR_DEFERRED_LIMIT. + free_delayed(((uintptr_t)ptr)|0x01, 0); + } +} +#endif + +#ifdef Py_GIL_DISABLED +void +_PyObject_XSetRefDelayed(PyObject **ptr, PyObject *value) +{ + PyObject *old = *ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*ptr, value); + if (old == NULL) { + return; + } + if (!_Py_IsImmortal(old)) { + _PyObject_XDecRefDelayed(old); } } #endif @@ -1238,7 +1335,7 @@ work_queue_first(struct llist_node *head) } static void -process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, +process_queue(struct llist_node *head, _PyThreadStateImpl *tstate, bool keep_empty, delayed_dealloc_cb cb, void *state) { while (!llist_empty(head)) { @@ -1246,7 +1343,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, if (buf->rd_idx < buf->wr_idx) { struct _mem_work_item *item = &buf->array[buf->rd_idx]; - if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) { + if (!_Py_qsbr_poll(tstate->qsbr, item->qsbr_goal)) { return; } @@ -1270,11 +1367,11 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, static void process_interp_queue(struct _Py_mem_interp_free_queue *queue, - struct _qsbr_thread_state *qsbr, delayed_dealloc_cb cb, + _PyThreadStateImpl *tstate, delayed_dealloc_cb cb, void *state) { assert(PyMutex_IsLocked(&queue->mutex)); - process_queue(&queue->head, qsbr, false, cb, state); + process_queue(&queue->head, tstate, false, cb, state); int more_work = !llist_empty(&queue->head); _Py_atomic_store_int_relaxed(&queue->has_work, more_work); @@ -1282,7 +1379,7 @@ process_interp_queue(struct _Py_mem_interp_free_queue *queue, static void maybe_process_interp_queue(struct _Py_mem_interp_free_queue *queue, - struct _qsbr_thread_state *qsbr, delayed_dealloc_cb cb, + _PyThreadStateImpl *tstate, delayed_dealloc_cb cb, void *state) { if (!_Py_atomic_load_int_relaxed(&queue->has_work)) { @@ -1291,7 +1388,7 @@ maybe_process_interp_queue(struct _Py_mem_interp_free_queue *queue, // Try to acquire the lock, but don't block if it's already held. if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) { - process_interp_queue(queue, qsbr, cb, state); + process_interp_queue(queue, tstate, cb, state); PyMutex_Unlock(&queue->mutex); } } @@ -1302,11 +1399,13 @@ _PyMem_ProcessDelayed(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + tstate_impl->qsbr->should_process = false; + // Process thread-local work - process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true, NULL, NULL); + process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, NULL, NULL); // Process shared interpreter work - maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr, NULL, NULL); + maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl, NULL, NULL); } void @@ -1316,10 +1415,10 @@ _PyMem_ProcessDelayedNoDealloc(PyThreadState *tstate, delayed_dealloc_cb cb, voi _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; // Process thread-local work - process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true, cb, state); + process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, cb, state); // Process shared interpreter work - maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr, cb, state); + maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl, cb, state); } void @@ -1348,7 +1447,7 @@ _PyMem_AbandonDelayed(PyThreadState *tstate) // Process the merged queue now (see gh-130794). _PyThreadStateImpl *this_tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); - process_interp_queue(&interp->mem_free_queue, this_tstate->qsbr, NULL, NULL); + process_interp_queue(&interp->mem_free_queue, this_tstate, NULL, NULL); PyMutex_Unlock(&interp->mem_free_queue.mutex); @@ -1424,9 +1523,9 @@ PyObject_Free(void *ptr) } -/* If we're using GCC, use __builtin_expect() to reduce overhead of +/* Use __builtin_expect() where available to reduce overhead of the valgrind checks */ -#if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__) +#if (defined(__clang__) || (defined(__GNUC__) && (__GNUC__ > 2))) && defined(__OPTIMIZE__) # define UNLIKELY(value) __builtin_expect((value), 0) # define LIKELY(value) __builtin_expect((value), 1) #else diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 891f6197401..25928028919 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -223,7 +223,6 @@ PyDict_DelItem PyMapping_DelItem PyDict_DelItemString PyMapping_DelItemString PyDict_GetItem - PyDict_GetItemWithError PyObject_GetItem -_PyDict_GetItemIdWithError - PyDict_GetItemString PyMapping_GetItemString PyDict_Items PyMapping_Items PyDict_Keys PyMapping_Keys @@ -473,6 +472,7 @@ Potential Optimizations #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_tuple.h" // _PyTuple_Recycle() #include <stddef.h> // offsetof() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/odictobject.c.h" @@ -535,6 +535,7 @@ struct _odictnode { static Py_ssize_t _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); PyObject *value = NULL; PyDictKeysObject *keys = ((PyDictObject *)od)->ma_keys; Py_ssize_t ix; @@ -559,6 +560,7 @@ _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) static int _odict_resize(PyODictObject *od) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t size, i; _ODictNode **fast_nodes, *node; @@ -595,6 +597,7 @@ _odict_resize(PyODictObject *od) static Py_ssize_t _odict_get_index(PyODictObject *od, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); PyDictKeysObject *keys; assert(key != NULL); @@ -615,6 +618,7 @@ _odict_get_index(PyODictObject *od, PyObject *key, Py_hash_t hash) static _ODictNode * _odict_find_node_hash(PyODictObject *od, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t index; if (_odict_EMPTY(od)) @@ -629,6 +633,7 @@ _odict_find_node_hash(PyODictObject *od, PyObject *key, Py_hash_t hash) static _ODictNode * _odict_find_node(PyODictObject *od, PyObject *key) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t index; Py_hash_t hash; @@ -647,6 +652,7 @@ _odict_find_node(PyODictObject *od, PyObject *key) static void _odict_add_head(PyODictObject *od, _ODictNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); _odictnode_PREV(node) = NULL; _odictnode_NEXT(node) = _odict_FIRST(od); if (_odict_FIRST(od) == NULL) @@ -660,6 +666,7 @@ _odict_add_head(PyODictObject *od, _ODictNode *node) static void _odict_add_tail(PyODictObject *od, _ODictNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); _odictnode_PREV(node) = _odict_LAST(od); _odictnode_NEXT(node) = NULL; if (_odict_LAST(od) == NULL) @@ -674,6 +681,7 @@ _odict_add_tail(PyODictObject *od, _ODictNode *node) static int _odict_add_new_node(PyODictObject *od, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t i; _ODictNode *node; @@ -718,6 +726,7 @@ _odict_add_new_node(PyODictObject *od, PyObject *key, Py_hash_t hash) static void _odict_remove_node(PyODictObject *od, _ODictNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); if (_odict_FIRST(od) == node) _odict_FIRST(od) = _odictnode_NEXT(node); else if (_odictnode_PREV(node) != NULL) @@ -753,6 +762,7 @@ static int _odict_clear_node(PyODictObject *od, _ODictNode *node, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t i; assert(key != NULL); @@ -934,6 +944,7 @@ static PyNumberMethods odict_as_number = { /* fromkeys() */ /*[clinic input] +@permit_long_summary @classmethod OrderedDict.fromkeys @@ -945,36 +956,39 @@ Create a new ordered dictionary with keys from iterable and values set to value. static PyObject * OrderedDict_fromkeys_impl(PyTypeObject *type, PyObject *seq, PyObject *value) -/*[clinic end generated code: output=c10390d452d78d6d input=1a0476c229c597b3]*/ +/*[clinic end generated code: output=c10390d452d78d6d input=1277ae0769083848]*/ { return _PyDict_FromKeys((PyObject *)type, seq, value); } -/* __sizeof__() */ +/*[clinic input] +@critical_section +OrderedDict.__sizeof__ -> Py_ssize_t +[clinic start generated code]*/ -/* OrderedDict.__sizeof__() does not have a docstring. */ -PyDoc_STRVAR(odict_sizeof__doc__, ""); - -static PyObject * -odict_sizeof(PyObject *op, PyObject *Py_UNUSED(ignored)) +static Py_ssize_t +OrderedDict___sizeof___impl(PyODictObject *self) +/*[clinic end generated code: output=1a8560db8cf83ac5 input=655e989ae24daa6a]*/ { - PyODictObject *od = _PyODictObject_CAST(op); - Py_ssize_t res = _PyDict_SizeOf((PyDictObject *)od); - res += sizeof(_ODictNode *) * od->od_fast_nodes_size; /* od_fast_nodes */ - if (!_odict_EMPTY(od)) { - res += sizeof(_ODictNode) * PyODict_SIZE(od); /* linked-list */ + Py_ssize_t res = _PyDict_SizeOf_LockHeld((PyDictObject *)self); + res += sizeof(_ODictNode *) * self->od_fast_nodes_size; /* od_fast_nodes */ + if (!_odict_EMPTY(self)) { + res += sizeof(_ODictNode) * PyODict_SIZE(self); /* linked-list */ } - return PyLong_FromSsize_t(res); + return res; } -/* __reduce__() */ +/*[clinic input] +OrderedDict.__reduce__ + self as od: self(type="PyODictObject *") -PyDoc_STRVAR(odict_reduce__doc__, "Return state information for pickling"); +Return state information for pickling +[clinic start generated code]*/ static PyObject * -odict_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +OrderedDict___reduce___impl(PyODictObject *od) +/*[clinic end generated code: output=71eeb81f760a6a8e input=b0467c7ec400fe5e]*/ { - register PyODictObject *od = _PyODictObject_CAST(op); PyObject *state, *result = NULL; PyObject *items_iter, *items, *args = NULL; @@ -1009,8 +1023,10 @@ odict_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) /* setdefault(): Skips __missing__() calls. */ +static int PyODict_SetItem_LockHeld(PyObject *self, PyObject *key, PyObject *value); /*[clinic input] +@critical_section OrderedDict.setdefault key: object @@ -1024,7 +1040,7 @@ Return the value for key if key is in the dictionary, else default. static PyObject * OrderedDict_setdefault_impl(PyODictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=97537cb7c28464b6 input=38e098381c1efbc6]*/ +/*[clinic end generated code: output=97537cb7c28464b6 input=d7b93e92734f99b5]*/ { PyObject *result = NULL; @@ -1034,7 +1050,7 @@ OrderedDict_setdefault_impl(PyODictObject *self, PyObject *key, if (PyErr_Occurred()) return NULL; assert(_odict_find_node(self, key) == NULL); - if (PyODict_SetItem((PyObject *)self, key, default_value) >= 0) { + if (PyODict_SetItem_LockHeld((PyObject *)self, key, default_value) >= 0) { result = Py_NewRef(default_value); } } @@ -1064,10 +1080,9 @@ static PyObject * _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); + PyObject *value = NULL; - - Py_BEGIN_CRITICAL_SECTION(od); - _ODictNode *node = _odict_find_node_hash(_PyODictObject_CAST(od), key, hash); if (node != NULL) { /* Pop the node first to avoid a possible dict resize (due to @@ -1092,7 +1107,6 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, PyErr_SetObject(PyExc_KeyError, key); } } - Py_END_CRITICAL_SECTION(); done: return value; @@ -1100,6 +1114,8 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, /* Skips __missing__() calls. */ /*[clinic input] +@critical_section +@permit_long_summary OrderedDict.pop key: object @@ -1114,7 +1130,7 @@ raise a KeyError. static PyObject * OrderedDict_pop_impl(PyODictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=7a6447d104e7494b input=7efe36601007dff7]*/ +/*[clinic end generated code: output=7a6447d104e7494b input=0742e3c9bf076a72]*/ { Py_hash_t hash = PyObject_Hash(key); if (hash == -1) @@ -1126,6 +1142,7 @@ OrderedDict_pop_impl(PyODictObject *self, PyObject *key, /* popitem() */ /*[clinic input] +@critical_section OrderedDict.popitem last: bool = True @@ -1137,7 +1154,7 @@ Pairs are returned in LIFO order if last is true or FIFO order if false. static PyObject * OrderedDict_popitem_impl(PyODictObject *self, int last) -/*[clinic end generated code: output=98e7d986690d49eb input=d992ac5ee8305e1a]*/ +/*[clinic end generated code: output=98e7d986690d49eb input=8aafc7433e0a40e7]*/ { PyObject *key, *value, *item = NULL; _ODictNode *node; @@ -1166,6 +1183,9 @@ OrderedDict_popitem_impl(PyODictObject *self, int last) PyDoc_STRVAR(odict_keys__doc__, ""); static PyObject * odictkeys_new(PyObject *od, PyObject *Py_UNUSED(ignored)); /* forward */ +static int +_PyODict_SetItem_KnownHash_LockHeld(PyObject *od, PyObject *key, PyObject *value, + Py_hash_t hash); /* forward */ /* values() */ @@ -1191,32 +1211,36 @@ static PyObject * mutablemapping_update(PyObject *, PyObject *, PyObject *); #define odict_update mutablemapping_update -/* clear() */ +/*[clinic input] +@critical_section +OrderedDict.clear -PyDoc_STRVAR(odict_clear__doc__, - "od.clear() -> None. Remove all items from od."); +Remove all items from ordered dict. +[clinic start generated code]*/ static PyObject * -odict_clear(PyObject *op, PyObject *Py_UNUSED(ignored)) +OrderedDict_clear_impl(PyODictObject *self) +/*[clinic end generated code: output=a1a76d1322f556c5 input=08b12322e74c535c]*/ { - register PyODictObject *od = _PyODictObject_CAST(op); - PyDict_Clear(op); - _odict_clear_nodes(od); + _PyDict_Clear_LockHeld((PyObject *)self); + _odict_clear_nodes(self); Py_RETURN_NONE; } /* copy() */ -/* forward */ -static int _PyODict_SetItem_KnownHash(PyObject *, PyObject *, PyObject *, - Py_hash_t); +/*[clinic input] +@critical_section +OrderedDict.copy + self as od: self -PyDoc_STRVAR(odict_copy__doc__, "od.copy() -> a shallow copy of od"); +A shallow copy of ordered dict. +[clinic start generated code]*/ static PyObject * -odict_copy(PyObject *op, PyObject *Py_UNUSED(ignored)) +OrderedDict_copy_impl(PyObject *od) +/*[clinic end generated code: output=9cdbe7394aecc576 input=e329951ae617ed48]*/ { - register PyODictObject *od = _PyODictObject_CAST(op); _ODictNode *node; PyObject *od_copy; @@ -1236,8 +1260,8 @@ odict_copy(PyObject *op, PyObject *Py_UNUSED(ignored)) PyErr_SetObject(PyExc_KeyError, key); goto fail; } - if (_PyODict_SetItem_KnownHash((PyObject *)od_copy, key, value, - _odictnode_HASH(node)) != 0) + if (_PyODict_SetItem_KnownHash_LockHeld((PyObject *)od_copy, key, value, + _odictnode_HASH(node)) != 0) goto fail; } } @@ -1285,6 +1309,7 @@ odict_reversed(PyObject *op, PyObject *Py_UNUSED(ignored)) /* move_to_end() */ /*[clinic input] +@critical_section OrderedDict.move_to_end key: object @@ -1297,7 +1322,7 @@ Raise KeyError if the element does not exist. static PyObject * OrderedDict_move_to_end_impl(PyODictObject *self, PyObject *key, int last) -/*[clinic end generated code: output=fafa4c5cc9b92f20 input=d6ceff7132a2fcd7]*/ +/*[clinic end generated code: output=fafa4c5cc9b92f20 input=09f8bc7053c0f6d4]*/ { _ODictNode *node; @@ -1338,10 +1363,8 @@ static PyMethodDef odict_methods[] = { /* overridden dict methods */ ORDEREDDICT_FROMKEYS_METHODDEF - {"__sizeof__", odict_sizeof, METH_NOARGS, - odict_sizeof__doc__}, - {"__reduce__", odict_reduce, METH_NOARGS, - odict_reduce__doc__}, + ORDEREDDICT___SIZEOF___METHODDEF + ORDEREDDICT___REDUCE___METHODDEF ORDEREDDICT_SETDEFAULT_METHODDEF ORDEREDDICT_POP_METHODDEF ORDEREDDICT_POPITEM_METHODDEF @@ -1353,11 +1376,8 @@ static PyMethodDef odict_methods[] = { odict_items__doc__}, {"update", _PyCFunction_CAST(odict_update), METH_VARARGS | METH_KEYWORDS, odict_update__doc__}, - {"clear", odict_clear, METH_NOARGS, - odict_clear__doc__}, - {"copy", odict_copy, METH_NOARGS, - odict_copy__doc__}, - + ORDEREDDICT_CLEAR_METHODDEF + ORDEREDDICT_COPY_METHODDEF /* new methods */ {"__reversed__", odict_reversed, METH_NOARGS, odict_reversed__doc__}, @@ -1391,8 +1411,7 @@ odict_dealloc(PyObject *op) PyObject_GC_UnTrack(self); Py_XDECREF(self->od_inst_dict); - if (self->od_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); + FT_CLEAR_WEAKREFS(op, self->od_weakreflist); _odict_clear_nodes(self); PyDict_Type.tp_dealloc((PyObject *)self); @@ -1457,7 +1476,8 @@ odict_tp_clear(PyObject *op) { PyODictObject *od = _PyODictObject_CAST(op); Py_CLEAR(od->od_inst_dict); - PyDict_Clear((PyObject *)od); + // cannot use lock held variant as critical section is not held here + PyDict_Clear(op); _odict_clear_nodes(od); return 0; } @@ -1465,7 +1485,7 @@ odict_tp_clear(PyObject *op) /* tp_richcompare */ static PyObject * -odict_richcompare(PyObject *v, PyObject *w, int op) +odict_richcompare_lock_held(PyObject *v, PyObject *w, int op) { if (!PyODict_Check(v) || !PyDict_Check(w)) { Py_RETURN_NOTIMPLEMENTED; @@ -1498,6 +1518,16 @@ odict_richcompare(PyObject *v, PyObject *w, int op) } } +static PyObject * +odict_richcompare(PyObject *v, PyObject *w, int op) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION2(v, w); + res = odict_richcompare_lock_held(v, w, op); + Py_END_CRITICAL_SECTION2(); + return res; +} + /* tp_iter */ static PyObject * @@ -1517,7 +1547,7 @@ odict_init(PyObject *self, PyObject *args, PyObject *kwds) if (len == -1) return -1; if (len > 1) { - const char *msg = "expected at most 1 arguments, got %zd"; + const char *msg = "expected at most 1 argument, got %zd"; PyErr_Format(PyExc_TypeError, msg, len); return -1; } @@ -1588,10 +1618,11 @@ PyODict_New(void) } static int -_PyODict_SetItem_KnownHash(PyObject *od, PyObject *key, PyObject *value, - Py_hash_t hash) +_PyODict_SetItem_KnownHash_LockHeld(PyObject *od, PyObject *key, PyObject *value, + Py_hash_t hash) { - int res = _PyDict_SetItem_KnownHash(od, key, value, hash); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); + int res = _PyDict_SetItem_KnownHash_LockHeld((PyDictObject *)od, key, value, hash); if (res == 0) { res = _odict_add_new_node(_PyODictObject_CAST(od), key, hash); if (res < 0) { @@ -1604,18 +1635,32 @@ _PyODict_SetItem_KnownHash(PyObject *od, PyObject *key, PyObject *value, return res; } -int -PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + +static int +PyODict_SetItem_LockHeld(PyObject *od, PyObject *key, PyObject *value) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_hash_t hash = PyObject_Hash(key); - if (hash == -1) + if (hash == -1) { return -1; - return _PyODict_SetItem_KnownHash(od, key, value, hash); + } + return _PyODict_SetItem_KnownHash_LockHeld(od, key, value, hash); } int -PyODict_DelItem(PyObject *od, PyObject *key) +PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) { + int res; + Py_BEGIN_CRITICAL_SECTION(od); + res = PyODict_SetItem_LockHeld(od, key, value); + Py_END_CRITICAL_SECTION(); + return res; +} + +int +PyODict_DelItem_LockHeld(PyObject *od, PyObject *key) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); int res; Py_hash_t hash = PyObject_Hash(key); if (hash == -1) @@ -1623,9 +1668,18 @@ PyODict_DelItem(PyObject *od, PyObject *key) res = _odict_clear_node(_PyODictObject_CAST(od), NULL, key, hash); if (res < 0) return -1; - return _PyDict_DelItem_KnownHash(od, key, hash); + return _PyDict_DelItem_KnownHash_LockHeld(od, key, hash); } +int +PyODict_DelItem(PyObject *od, PyObject *key) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(od); + res = PyODict_DelItem_LockHeld(od, key); + Py_END_CRITICAL_SECTION(); + return res; +} /* ------------------------------------------- * The OrderedDict views (keys/values/items) @@ -1667,14 +1721,14 @@ odictiter_traverse(PyObject *op, visitproc visit, void *arg) /* In order to protect against modifications during iteration, we track * the current key instead of the current node. */ static PyObject * -odictiter_nextkey(odictiterobject *di) +odictiter_nextkey_lock_held(odictiterobject *di) { + assert(di->di_odict != NULL); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(di->di_odict); PyObject *key = NULL; _ODictNode *node; int reversed = di->kind & _odict_ITER_REVERSED; - if (di->di_odict == NULL) - return NULL; if (di->di_current == NULL) goto done; /* We're already done. */ @@ -1719,8 +1773,23 @@ odictiter_nextkey(odictiterobject *di) return key; } + static PyObject * -odictiter_iternext(PyObject *op) +odictiter_nextkey(odictiterobject *di) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(di); + if (di->di_odict == NULL) { + return NULL; + } + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(di->di_odict); + res = odictiter_nextkey_lock_held(di); + Py_END_CRITICAL_SECTION(); + return res; +} + +static PyObject * +odictiter_iternext_lock_held(PyObject *op) { odictiterobject *di = (odictiterobject*)op; PyObject *result, *value; @@ -1734,14 +1803,12 @@ odictiter_iternext(PyObject *op) return key; } - value = PyODict_GetItem((PyObject *)di->di_odict, key); /* borrowed */ - if (value == NULL) { + if (PyDict_GetItemRef((PyObject *)di->di_odict, key, &value) != 1) { if (!PyErr_Occurred()) PyErr_SetObject(PyExc_KeyError, key); Py_DECREF(key); goto done; } - Py_INCREF(value); /* Handle the values case. */ if (!(di->kind & _odict_ITER_KEYS)) { @@ -1752,7 +1819,7 @@ odictiter_iternext(PyObject *op) /* Handle the items case. */ result = di->di_result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { /* not in use so we can reuse it * (the common case during iteration) */ Py_INCREF(result); @@ -1781,6 +1848,17 @@ odictiter_iternext(PyObject *op) return NULL; } +static PyObject * +odictiter_iternext(PyObject *op) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(op); + res = odictiter_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return res; +} + + /* No need for tp_clear because odictiterobject is not mutable. */ PyDoc_STRVAR(reduce_doc, "Return state information for pickling"); diff --git a/Objects/picklebufobject.c b/Objects/picklebufobject.c index 3ce800de04c..50f17687bc4 100644 --- a/Objects/picklebufobject.c +++ b/Objects/picklebufobject.c @@ -1,6 +1,7 @@ /* PickleBuffer object implementation */ #include "Python.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> typedef struct { @@ -111,8 +112,7 @@ picklebuf_dealloc(PyObject *op) { PyPickleBufferObject *self = (PyPickleBufferObject*)op; PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); + FT_CLEAR_WEAKREFS(op, self->weakreflist); PyBuffer_Release(&self->view); Py_TYPE(self)->tp_free((PyObject *) self); } diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index f8cdfe68a64..e93346fb277 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -1042,6 +1042,11 @@ longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) static PyObject * longrangeiter_setstate(PyObject *op, PyObject *state) { + if (!PyLong_CheckExact(state)) { + PyErr_Format(PyExc_TypeError, "state must be an int, not %T", state); + return NULL; + } + longrangeiterobject *r = (longrangeiterobject*)op; PyObject *zero = _PyLong_GetZero(); // borrowed reference int cmp; diff --git a/Objects/setobject.c b/Objects/setobject.c index 8aa6b0d1809..85f4d7d4031 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -40,6 +40,7 @@ #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() #include "pycore_pyerrors.h" // _PyErr_SetKeyError() #include "pycore_setobject.h" // _PySet_NextEntry() definition +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "stringlib/eq.h" // unicode_eq() #include <stddef.h> // offsetof() @@ -85,6 +86,8 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) int probes; int cmp; + int frozenset = PyFrozenSet_CheckExact(so); + while (1) { entry = &so->table[i]; probes = (i + LINEAR_PROBES <= mask) ? LINEAR_PROBES: 0; @@ -101,13 +104,20 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) && unicode_eq(startkey, key)) return entry; table = so->table; - Py_INCREF(startkey); - cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) - return NULL; - if (table != so->table || entry->key != startkey) - return set_lookkey(so, key, hash); + if (frozenset) { + cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + if (cmp < 0) + return NULL; + } else { + // incref startkey because it can be removed from the set by the compare + Py_INCREF(startkey); + cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) + return NULL; + if (table != so->table || entry->key != startkey) + return set_lookkey(so, key, hash); + } if (cmp > 0) return entry; mask = so->mask; @@ -536,8 +546,7 @@ set_dealloc(PyObject *self) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(so); - if (so->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) so); + FT_CLEAR_WEAKREFS(self, so->weakreflist); for (entry = so->table; used > 0; entry++) { if (entry->key && entry->key != dummy) { @@ -1962,6 +1971,7 @@ set_symmetric_difference_update_set(PySetObject *so, PySetObject *other) } /*[clinic input] +@permit_long_summary set.symmetric_difference_update so: setobject other: object @@ -1972,7 +1982,7 @@ Update the set, keeping only elements found in either set, but not in both. static PyObject * set_symmetric_difference_update_impl(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=79f80b4ee5da66c1 input=a50acf0365e1f0a5]*/ +/*[clinic end generated code: output=79f80b4ee5da66c1 input=86a3dddac9bfb15e]*/ { if (Py_Is((PyObject *)so, other)) { return set_clear((PyObject *)so, NULL); @@ -2234,10 +2244,16 @@ set_contains_lock_held(PySetObject *so, PyObject *key) int _PySet_Contains(PySetObject *so, PyObject *key) { + assert(so); + int rv; - Py_BEGIN_CRITICAL_SECTION(so); - rv = set_contains_lock_held(so, key); - Py_END_CRITICAL_SECTION(); + if (PyFrozenSet_CheckExact(so)) { + rv = set_contains_lock_held(so, key); + } else { + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_contains_lock_held(so, key); + Py_END_CRITICAL_SECTION(); + } return rv; } @@ -2428,7 +2444,7 @@ set_init(PyObject *so, PyObject *args, PyObject *kwds) if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; - if (Py_REFCNT(self) == 1 && self->fill == 0) { + if (_PyObject_IsUniquelyReferenced((PyObject *)self) && self->fill == 0) { self->hash = -1; if (iterable == NULL) { return 0; @@ -2731,7 +2747,9 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - + if (PyFrozenSet_CheckExact(anyset)) { + return set_contains_key((PySetObject *)anyset, key); + } int rv; Py_BEGIN_CRITICAL_SECTION(anyset); rv = set_contains_key((PySetObject *)anyset, key); @@ -2757,17 +2775,24 @@ PySet_Discard(PyObject *set, PyObject *key) int PySet_Add(PyObject *anyset, PyObject *key) { - if (!PySet_Check(anyset) && - (!PyFrozenSet_Check(anyset) || Py_REFCNT(anyset) != 1)) { - PyErr_BadInternalCall(); - return -1; + if (PySet_Check(anyset)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(anyset); + rv = set_add_key((PySetObject *)anyset, key); + Py_END_CRITICAL_SECTION(); + return rv; } - int rv; - Py_BEGIN_CRITICAL_SECTION(anyset); - rv = set_add_key((PySetObject *)anyset, key); - Py_END_CRITICAL_SECTION(); - return rv; + if (PyFrozenSet_Check(anyset) && _PyObject_IsUniquelyReferenced(anyset)) { + // We can only change frozensets if they are uniquely referenced. The + // API limits the usage of `PySet_Add` to "fill in the values of brand + // new frozensets before they are exposed to other code". In this case, + // this can be done without a lock. + return set_add_key((PySetObject *)anyset, key); + } + + PyErr_BadInternalCall(); + return -1; } int diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 440410d0aef..9e53fab8429 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -257,16 +257,14 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end, /* UTF-8 encoder specialized for a Unicode kind to avoid the slow PyUnicode_READ() macro. Delete some parts of the code depending on the kind: UCS-1 strings don't need to handle surrogates for example. */ -Py_LOCAL_INLINE(char *) -STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, - PyObject *unicode, +Py_LOCAL_INLINE(PyBytesWriter*) +STRINGLIB(utf8_encoder)(PyObject *unicode, const STRINGLIB_CHAR *data, Py_ssize_t size, _Py_error_handler error_handler, - const char *errors) + const char *errors, + char **end) { - Py_ssize_t i; /* index into data of next input character */ - char *p; /* next free byte in output buffer */ #if STRINGLIB_SIZEOF_CHAR > 1 PyObject *error_handler_obj = NULL; PyObject *exc = NULL; @@ -284,14 +282,19 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, if (size > PY_SSIZE_T_MAX / max_char_size) { /* integer overflow */ PyErr_NoMemory(); + *end = NULL; return NULL; } - _PyBytesWriter_Init(writer); - p = _PyBytesWriter_Alloc(writer, size * max_char_size); - if (p == NULL) + PyBytesWriter *writer = PyBytesWriter_Create(size * max_char_size); + if (writer == NULL) { + *end = NULL; return NULL; + } + /* next free byte in output buffer */ + char *p = PyBytesWriter_GetData(writer); + Py_ssize_t i; /* index into data of next input character */ for (i = 0; i < size;) { Py_UCS4 ch = data[i++]; @@ -348,7 +351,7 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, case _Py_ERROR_BACKSLASHREPLACE: /* subtract preallocated bytes */ - writer->min_size -= max_char_size * (endpos - startpos); + writer->size -= max_char_size * (endpos - startpos); p = backslashreplace(writer, p, unicode, startpos, endpos); if (p == NULL) @@ -358,7 +361,7 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, case _Py_ERROR_XMLCHARREFREPLACE: /* subtract preallocated bytes */ - writer->min_size -= max_char_size * (endpos - startpos); + writer->size -= max_char_size * (endpos - startpos); p = xmlcharrefreplace(writer, p, unicode, startpos, endpos); if (p == NULL) @@ -389,22 +392,25 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, if (newpos < startpos) { writer->overallocate = 1; - p = _PyBytesWriter_Prepare(writer, p, - max_char_size * (startpos - newpos)); - if (p == NULL) + p = PyBytesWriter_GrowAndUpdatePointer(writer, + max_char_size * (startpos - newpos), + p); + if (p == NULL) { goto error; + } } else { /* subtract preallocated bytes */ - writer->min_size -= max_char_size * (newpos - startpos); + writer->size -= max_char_size * (newpos - startpos); /* Only overallocate the buffer if it's not the last write */ writer->overallocate = (newpos < size); } + char *rep_str; + Py_ssize_t rep_len; if (PyBytes_Check(rep)) { - p = _PyBytesWriter_WriteBytes(writer, p, - PyBytes_AS_STRING(rep), - PyBytes_GET_SIZE(rep)); + rep_str = PyBytes_AS_STRING(rep); + rep_len = PyBytes_GET_SIZE(rep); } else { /* rep is unicode */ @@ -415,13 +421,16 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, goto error; } - p = _PyBytesWriter_WriteBytes(writer, p, - PyUnicode_DATA(rep), - PyUnicode_GET_LENGTH(rep)); + rep_str = PyUnicode_DATA(rep); + rep_len = PyUnicode_GET_LENGTH(rep); } - if (p == NULL) + p = PyBytesWriter_GrowAndUpdatePointer(writer, rep_len, p); + if (p == NULL) { goto error; + } + memcpy(p, rep_str, rep_len); + p += rep_len; Py_CLEAR(rep); i = newpos; @@ -458,13 +467,16 @@ STRINGLIB(utf8_encoder)(_PyBytesWriter *writer, Py_XDECREF(error_handler_obj); Py_XDECREF(exc); #endif - return p; + *end = p; + return writer; #if STRINGLIB_SIZEOF_CHAR > 1 error: + PyBytesWriter_Discard(writer); Py_XDECREF(rep); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); + *end = NULL; return NULL; #endif } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 05e700b0625..b447865c986 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -595,7 +595,7 @@ STRINGLIB(default_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, continue; } /* miss: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } else { @@ -604,7 +604,7 @@ STRINGLIB(default_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } else { /* skip: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } } @@ -668,7 +668,7 @@ STRINGLIB(adaptive_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } } /* miss: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } else { @@ -677,7 +677,7 @@ STRINGLIB(adaptive_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } else { /* skip: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } } diff --git a/Objects/stringlib/localeutil.h b/Objects/stringlib/localeutil.h deleted file mode 100644 index a4ab701de00..00000000000 --- a/Objects/stringlib/localeutil.h +++ /dev/null @@ -1,97 +0,0 @@ -/* _PyUnicode_InsertThousandsGrouping() helper functions */ - -typedef struct { - const char *grouping; - char previous; - Py_ssize_t i; /* Where we're currently pointing in grouping. */ -} GroupGenerator; - - -static void -GroupGenerator_init(GroupGenerator *self, const char *grouping) -{ - self->grouping = grouping; - self->i = 0; - self->previous = 0; -} - - -/* Returns the next grouping, or 0 to signify end. */ -static Py_ssize_t -GroupGenerator_next(GroupGenerator *self) -{ - /* Note that we don't really do much error checking here. If a - grouping string contains just CHAR_MAX, for example, then just - terminate the generator. That shouldn't happen, but at least we - fail gracefully. */ - switch (self->grouping[self->i]) { - case 0: - return self->previous; - case CHAR_MAX: - /* Stop the generator. */ - return 0; - default: { - char ch = self->grouping[self->i]; - self->previous = ch; - self->i++; - return (Py_ssize_t)ch; - } - } -} - - -/* Fill in some digits, leading zeros, and thousands separator. All - are optional, depending on when we're called. */ -static void -InsertThousandsGrouping_fill(_PyUnicodeWriter *writer, Py_ssize_t *buffer_pos, - PyObject *digits, Py_ssize_t *digits_pos, - Py_ssize_t n_chars, Py_ssize_t n_zeros, - PyObject *thousands_sep, Py_ssize_t thousands_sep_len, - Py_UCS4 *maxchar, int forward) -{ - if (!writer) { - /* if maxchar > 127, maxchar is already set */ - if (*maxchar == 127 && thousands_sep) { - Py_UCS4 maxchar2 = PyUnicode_MAX_CHAR_VALUE(thousands_sep); - *maxchar = Py_MAX(*maxchar, maxchar2); - } - return; - } - - if (thousands_sep) { - if (!forward) { - *buffer_pos -= thousands_sep_len; - } - /* Copy the thousands_sep chars into the buffer. */ - _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, - thousands_sep, 0, - thousands_sep_len); - if (forward) { - *buffer_pos += thousands_sep_len; - } - } - - if (!forward) { - *buffer_pos -= n_chars; - *digits_pos -= n_chars; - } - _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, - digits, *digits_pos, - n_chars); - if (forward) { - *buffer_pos += n_chars; - *digits_pos += n_chars; - } - - if (n_zeros) { - if (!forward) { - *buffer_pos -= n_zeros; - } - int kind = PyUnicode_KIND(writer->buffer); - void *data = PyUnicode_DATA(writer->buffer); - unicode_fill(kind, data, '0', *buffer_pos, n_zeros); - if (forward) { - *buffer_pos += n_zeros; - } - } -} diff --git a/Objects/stringlib/transmogrify.h b/Objects/stringlib/transmogrify.h index 71099bb586e..591bf55d1a9 100644 --- a/Objects/stringlib/transmogrify.h +++ b/Objects/stringlib/transmogrify.h @@ -207,6 +207,7 @@ stringlib_center_impl(PyObject *self, Py_ssize_t width, char fillchar) } /*[clinic input] +@permit_long_summary B.zfill as stringlib_zfill width: Py_ssize_t @@ -219,7 +220,7 @@ The original string is never truncated. static PyObject * stringlib_zfill_impl(PyObject *self, Py_ssize_t width) -/*[clinic end generated code: output=0b3c684a7f1b2319 input=2da6d7b8e9bcb19a]*/ +/*[clinic end generated code: output=0b3c684a7f1b2319 input=dfb9cbb16f521756]*/ { Py_ssize_t fill; PyObject *s; diff --git a/Objects/structseq.c b/Objects/structseq.c index c05bcde24c4..7a159338b9b 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -12,7 +12,7 @@ #include "pycore_modsupport.h" // _PyArg_NoPositional() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_structseq.h" // PyStructSequence_InitType() -#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_tuple.h" // _PyTuple_RESET_HASH_CACHE() #include "pycore_typeobject.h" // _PyStaticType_FiniBuiltin() static const char visible_length_key[] = "n_sequence_fields"; @@ -353,7 +353,7 @@ structseq_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) if (n_unnamed_fields < 0) { return NULL; } - tup = _PyTuple_FromArray(self->ob_item, n_visible_fields); + tup = PyTuple_FromArray(self->ob_item, n_visible_fields); if (!tup) goto error; diff --git a/Objects/templateobject.c b/Objects/templateobject.c index 7d356980b56..ac38e4de435 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -23,11 +23,15 @@ templateiter_next(PyObject *op) if (self->from_strings) { item = PyIter_Next(self->stringsiter); self->from_strings = 0; + if (item == NULL) { + return NULL; + } if (PyUnicode_GET_LENGTH(item) == 0) { Py_SETREF(item, PyIter_Next(self->interpolationsiter)); self->from_strings = 1; } - } else { + } + else { item = PyIter_Next(self->interpolationsiter); self->from_strings = 1; } @@ -242,54 +246,6 @@ template_iter(PyObject *op) return (PyObject *)iter; } -static PyObject * -template_strings_append_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1); - PyObject *concat = PyUnicode_Concat(string, str); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - for (Py_ssize_t i = 0; i < stringslen - 1; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - PyTuple_SET_ITEM(newstrings, stringslen - 1, concat); - - return newstrings; -} - -static PyObject * -template_strings_prepend_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, 0); - PyObject *concat = PyUnicode_Concat(str, string); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - PyTuple_SET_ITEM(newstrings, 0, concat); - for (Py_ssize_t i = 1; i < stringslen; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - - return newstrings; -} - static PyObject * template_strings_concat(PyObject *left, PyObject *right) { @@ -341,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other) return newtemplate; } -static PyObject * -template_concat_template_str(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_append_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - -static PyObject * -template_concat_str_template(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_prepend_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - PyObject * _PyTemplate_Concat(PyObject *self, PyObject *other) { if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) { return template_concat_templates((templateobject *) self, (templateobject *) other); } - else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) { - return template_concat_template_str((templateobject *) self, other); - } - else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) { - return template_concat_str_template((templateobject *) other, self); - } - else { - Py_RETURN_NOTIMPLEMENTED; - } + + PyErr_Format(PyExc_TypeError, + "can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template", + other); + return NULL; } static PyObject * @@ -444,6 +370,8 @@ template_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef template_methods[] = { {"__reduce__", template_reduce, METH_NOARGS, NULL}, + {"__class_getitem__", Py_GenericAlias, + METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL}, }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 9b31758485c..169ac69701d 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -118,7 +118,7 @@ int PyTuple_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) { PyObject **p; - if (!PyTuple_Check(op) || Py_REFCNT(op) != 1) { + if (!PyTuple_Check(op) || !_PyObject_IsUniquelyReferenced(op)) { Py_XDECREF(newitem); PyErr_BadInternalCall(); return -1; @@ -156,6 +156,18 @@ _PyTuple_MaybeUntrack(PyObject *op) _PyObject_GC_UNTRACK(op); } +/* Fast, but conservative check if an object maybe tracked + May return true for an object that is not tracked, + Will always return true for an object that is tracked. + This is a temporary workaround until _PyObject_GC_IS_TRACKED + becomes fast and safe to call on non-GC objects. +*/ +static bool +maybe_tracked(PyObject *ob) +{ + return _PyType_IS_GC(Py_TYPE(ob)); +} + PyObject * PyTuple_Pack(Py_ssize_t n, ...) { @@ -163,6 +175,7 @@ PyTuple_Pack(Py_ssize_t n, ...) PyObject *o; PyObject **items; va_list vargs; + bool track = false; if (n == 0) { return tuple_get_empty(); @@ -177,10 +190,15 @@ PyTuple_Pack(Py_ssize_t n, ...) items = result->ob_item; for (i = 0; i < n; i++) { o = va_arg(vargs, PyObject *); + if (!track && maybe_tracked(o)) { + track = true; + } items[i] = Py_NewRef(o); } va_end(vargs); - _PyObject_GC_TRACK(result); + if (track) { + _PyObject_GC_TRACK(result); + } return (PyObject *)result; } @@ -366,7 +384,7 @@ tuple_item(PyObject *op, Py_ssize_t i) } PyObject * -_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) { if (n == 0) { return tuple_get_empty(); @@ -377,11 +395,17 @@ _PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) return NULL; } PyObject **dst = tuple->ob_item; + bool track = false; for (Py_ssize_t i = 0; i < n; i++) { PyObject *item = src[i]; + if (!track && maybe_tracked(item)) { + track = true; + } dst[i] = Py_NewRef(item); } - _PyObject_GC_TRACK(tuple); + if (track) { + _PyObject_GC_TRACK(tuple); + } return (PyObject *)tuple; } @@ -396,10 +420,17 @@ _PyTuple_FromStackRefStealOnSuccess(const _PyStackRef *src, Py_ssize_t n) return NULL; } PyObject **dst = tuple->ob_item; + bool track = false; for (Py_ssize_t i = 0; i < n; i++) { - dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + PyObject *item = PyStackRef_AsPyObjectSteal(src[i]); + if (!track && maybe_tracked(item)) { + track = true; + } + dst[i] = item; + } + if (track) { + _PyObject_GC_TRACK(tuple); } - _PyObject_GC_TRACK(tuple); return (PyObject *)tuple; } @@ -438,7 +469,7 @@ tuple_slice(PyTupleObject *a, Py_ssize_t ilow, if (ilow == 0 && ihigh == Py_SIZE(a) && PyTuple_CheckExact(a)) { return Py_NewRef(a); } - return _PyTuple_FromArray(a->ob_item + ilow, ihigh - ilow); + return PyTuple_FromArray(a->ob_item + ilow, ihigh - ilow); } PyObject * @@ -923,7 +954,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) v = (PyTupleObject *) *pv; if (v == NULL || !Py_IS_TYPE(v, &PyTuple_Type) || - (Py_SIZE(v) != 0 && Py_REFCNT(v) != 1)) { + (Py_SIZE(v) != 0 && !_PyObject_IsUniquelyReferenced(*pv))) { *pv = 0; Py_XDECREF(v); PyErr_BadInternalCall(); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a7ab69fef4c..cbe0215359e 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -48,34 +48,133 @@ class object "PyObject *" "&PyBaseObject_Type" & ((1 << MCACHE_SIZE_EXP) - 1)) #define MCACHE_HASH_METHOD(type, name) \ - MCACHE_HASH(FT_ATOMIC_LOAD_UINT32_RELAXED((type)->tp_version_tag), \ + MCACHE_HASH(FT_ATOMIC_LOAD_UINT_RELAXED((type)->tp_version_tag), \ ((Py_ssize_t)(name)) >> 3) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ (PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE) -#define NEXT_GLOBAL_VERSION_TAG _PyRuntime.types.next_version_tag #define NEXT_VERSION_TAG(interp) \ (interp)->types.next_version_tag #ifdef Py_GIL_DISABLED -// There's a global lock for mutation of types. This avoids having to take -// additional locks while doing various subclass processing which may result -// in odd behaviors w.r.t. running with the GIL as the outer type lock could -// be released and reacquired during a subclass update if there's contention -// on the subclass lock. -#define TYPE_LOCK &PyInterpreterState_Get()->types.mutex -#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUT(TYPE_LOCK) +// There's a global lock for types that ensures that tp_version_tag and +// _spec_cache are correctly updated if the type is modified. It also protects +// tp_mro, tp_bases, and tp_base. This avoids having to take additional locks +// while doing various subclass processing which may result in odd behaviors +// w.r.t. running with the GIL as the outer type lock could be released and +// reacquired during a subclass update if there's contention on the subclass +// lock. +// +// Note that this lock does not protect updates of other type slots or the +// tp_flags member. Instead, we either ensure those updates are done before +// the type has been revealed to other threads or we only do those updates +// while the stop-the-world mechanism is active. The slots and flags are read +// in many places without holding a lock and without atomics. +#define TYPE_LOCK &_PyInterpreterState_GET()->types.mutex +#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUTEX(TYPE_LOCK) #define END_TYPE_LOCK() Py_END_CRITICAL_SECTION() #define BEGIN_TYPE_DICT_LOCK(d) \ - Py_BEGIN_CRITICAL_SECTION2_MUT(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) + Py_BEGIN_CRITICAL_SECTION2_MUTEX(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) #define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() +#if !defined(NDEBUG) || defined(Py_DEBUG) +// Return true if the world is currently stopped. +static bool +types_world_is_stopped(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return interp->stoptheworld.world_stopped; +} +#endif + +// Checks that the type has not yet been revealed (exposed) to other +// threads. The _Py_TYPE_REVEALED_FLAG flag is set by type_new() and +// PyType_FromMetaclass() to indicate that a newly initialized type might be +// revealed. We only have ob_flags on 64-bit platforms. +#if SIZEOF_VOID_P > 4 +#define TYPE_IS_REVEALED(tp) ((((PyObject *)(tp))->ob_flags & _Py_TYPE_REVEALED_FLAG) != 0) +#else +#define TYPE_IS_REVEALED(tp) 0 +#endif + +#ifdef Py_DEBUG #define ASSERT_TYPE_LOCK_HELD() \ - _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) + if (!types_world_is_stopped()) { _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK); } + +// Checks if we can safely update type slots or tp_flags. +#define ASSERT_WORLD_STOPPED_OR_NEW_TYPE(tp) \ + assert(!TYPE_IS_REVEALED(tp) || types_world_is_stopped()) + +#define ASSERT_NEW_TYPE_OR_LOCKED(tp) \ + if (TYPE_IS_REVEALED(tp)) { ASSERT_TYPE_LOCK_HELD(); } +#else +#define ASSERT_TYPE_LOCK_HELD() +#define ASSERT_WORLD_STOPPED_OR_NEW_TYPE(tp) +#define ASSERT_NEW_TYPE_OR_LOCKED(tp) +#endif + +static void +types_stop_world(void) +{ + assert(!types_world_is_stopped()); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + assert(types_world_is_stopped()); +} + +static void +types_start_world(void) +{ + assert(types_world_is_stopped()); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StartTheWorld(interp); + assert(!types_world_is_stopped()); +} + +// This is used to temporarily prevent the TYPE_LOCK from being suspended +// when held by the topmost critical section. +static void +type_lock_prevent_release(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t *tagptr = &tstate->critical_section; + PyCriticalSection *c = (PyCriticalSection *)(*tagptr & ~_Py_CRITICAL_SECTION_MASK); + if (!(*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { + assert(c->_cs_mutex == TYPE_LOCK); + c->_cs_mutex = NULL; + } + else { + PyCriticalSection2 *c2 = (PyCriticalSection2 *)c; + if (c->_cs_mutex == TYPE_LOCK) { + c->_cs_mutex = c2->_cs_mutex2; + c2->_cs_mutex2 = NULL; + } else { + assert(c2->_cs_mutex2 == TYPE_LOCK); + c2->_cs_mutex2 = NULL; + } + } +} + +static void +type_lock_allow_release(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t *tagptr = &tstate->critical_section; + PyCriticalSection *c = (PyCriticalSection *)(*tagptr & ~_Py_CRITICAL_SECTION_MASK); + if (!(*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { + assert(c->_cs_mutex == NULL); + c->_cs_mutex = TYPE_LOCK; + } + else { + PyCriticalSection2 *c2 = (PyCriticalSection2 *)c; + assert(c2->_cs_mutex2 == NULL); + c2->_cs_mutex2 = TYPE_LOCK; + } +} #else @@ -84,6 +183,14 @@ class object "PyObject *" "&PyBaseObject_Type" #define BEGIN_TYPE_DICT_LOCK(d) #define END_TYPE_DICT_LOCK() #define ASSERT_TYPE_LOCK_HELD() +#define TYPE_IS_REVEALED(tp) 0 +#define ASSERT_WORLD_STOPPED_OR_NEW_TYPE(tp) +#define ASSERT_NEW_TYPE_OR_LOCKED(tp) +#define types_world_is_stopped() 1 +#define types_stop_world() +#define types_start_world() +#define type_lock_prevent_release() +#define type_lock_allow_release() #endif @@ -106,6 +213,9 @@ slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds); static int slot_tp_setattro(PyObject *self, PyObject *name, PyObject *value); +static PyObject * +slot_tp_call(PyObject *self, PyObject *args, PyObject *kwds); + static inline PyTypeObject * type_from_ref(PyObject *ref) { @@ -117,7 +227,7 @@ type_from_ref(PyObject *ref) } -/* helpers for for static builtin types */ +/* helpers for static builtin types */ #ifndef NDEBUG static inline int @@ -157,8 +267,8 @@ static_ext_type_lookup(PyInterpreterState *interp, size_t index, assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; - int64_t interp_count = - _PyRuntime.types.managed_static.types[full_index].interp_count; + int64_t interp_count = _Py_atomic_load_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count); assert((interp_count == 0) == (_PyRuntime.types.managed_static.types[full_index].type == NULL)); *p_interp_count = interp_count; @@ -235,7 +345,7 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; assert((initial == 1) == - (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + (_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count) == 0)); (void)_Py_atomic_add_int64( &_PyRuntime.types.managed_static.types[full_index].interp_count, 1); @@ -284,7 +394,7 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, : &(interp->types.for_extensions.initialized[index]); assert(state != NULL); - assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count) > 0); assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); @@ -294,7 +404,7 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, (void)_Py_atomic_add_int64( &_PyRuntime.types.managed_static.types[full_index].interp_count, -1); if (final) { - assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + assert(!_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count)); _PyRuntime.types.managed_static.types[full_index].type = NULL; managed_static_type_index_clear(self); @@ -346,21 +456,14 @@ _PyStaticType_GetBuiltins(void) static void type_set_flags(PyTypeObject *tp, unsigned long flags) { - if (tp->tp_flags & Py_TPFLAGS_READY) { - // It's possible the type object has been exposed to other threads - // if it's been marked ready. In that case, the type lock should be - // held when flags are modified. - ASSERT_TYPE_LOCK_HELD(); - } - // Since PyType_HasFeature() reads the flags without holding the type - // lock, we need an atomic store here. - FT_ATOMIC_STORE_ULONG_RELAXED(tp->tp_flags, flags); + ASSERT_WORLD_STOPPED_OR_NEW_TYPE(tp); + tp->tp_flags = flags; } static void type_set_flags_with_mask(PyTypeObject *tp, unsigned long mask, unsigned long flags) { - ASSERT_TYPE_LOCK_HELD(); + ASSERT_WORLD_STOPPED_OR_NEW_TYPE(tp); unsigned long new_flags = (tp->tp_flags & ~mask) | flags; type_set_flags(tp, new_flags); } @@ -498,6 +601,7 @@ static inline void set_tp_bases(PyTypeObject *self, PyObject *bases, int initial) { assert(PyTuple_Check(bases)); + ASSERT_NEW_TYPE_OR_LOCKED(self); if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { // XXX tp_bases can probably be statically allocated for each // static builtin type. @@ -542,7 +646,7 @@ clear_tp_bases(PyTypeObject *self, int final) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { - ASSERT_TYPE_LOCK_HELD(); + ASSERT_NEW_TYPE_OR_LOCKED(self); return self->tp_mro; } @@ -650,6 +754,22 @@ lookup_tp_subclasses(PyTypeObject *self) return (PyObject *)self->tp_subclasses; } +PyObject * +_PyType_LookupSubclasses(PyTypeObject *self) +{ + return lookup_tp_subclasses(self); +} + +PyObject * +_PyType_InitSubclasses(PyTypeObject *self) +{ + PyObject *existing = lookup_tp_subclasses(self); + if (existing != NULL) { + return existing; + } + return init_tp_subclasses(self); +} + int _PyType_HasSubclasses(PyTypeObject *self) { @@ -1027,7 +1147,6 @@ PyType_Unwatch(int watcher_id, PyObject* obj) static void set_version_unlocked(PyTypeObject *tp, unsigned int version) { - ASSERT_TYPE_LOCK_HELD(); assert(version == 0 || (tp->tp_versions_used != _Py_ATTR_CACHE_UNUSED)); #ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -1075,7 +1194,12 @@ type_modified_unlocked(PyTypeObject *type) We don't assign new version tags eagerly, but only as needed. */ - ASSERT_TYPE_LOCK_HELD(); + ASSERT_NEW_TYPE_OR_LOCKED(type); +#ifdef Py_GIL_DISABLED + // This function is re-entrant and it's not safe to call it + // with the world stopped. + assert(!types_world_is_stopped()); +#endif if (type->tp_version_tag == 0) { return; } @@ -1106,6 +1230,8 @@ type_modified_unlocked(PyTypeObject *type) while (bits) { assert(i < TYPE_MAX_WATCHERS); if (bits & 1) { + // Note that PyErr_FormatUnraisable is potentially re-entrant + // and the watcher callback might be too. PyType_WatchCallback cb = interp->type_watchers[i]; if (cb && (cb(type) < 0)) { PyErr_FormatUnraisable( @@ -1245,19 +1371,24 @@ _PyType_LookupByVersion(unsigned int version) #endif } -unsigned int -_PyType_GetVersionForCurrentState(PyTypeObject *tp) -{ - return tp->tp_version_tag; -} - - - #define MAX_VERSIONS_PER_CLASS 1000 #if _Py_ATTR_CACHE_UNUSED < MAX_VERSIONS_PER_CLASS #error "_Py_ATTR_CACHE_UNUSED must be bigger than max" #endif +static inline unsigned int +next_global_version_tag(void) +{ + unsigned int old; + do { + old = _Py_atomic_load_uint_relaxed(&_PyRuntime.types.next_version_tag); + if (old >= _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { + return 0; + } + } while (!_Py_atomic_compare_exchange_uint(&_PyRuntime.types.next_version_tag, &old, old + 1)); + return old + 1; +} + static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { @@ -1288,11 +1419,12 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) } if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ - if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { + unsigned int next_version_tag = next_global_version_tag(); + if (next_version_tag == 0) { /* We have run out of version numbers */ return 0; } - set_version_unlocked(type, NEXT_GLOBAL_VERSION_TAG++); + set_version_unlocked(type, next_version_tag); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -1335,15 +1467,15 @@ static PyMemberDef type_members[] = { static int check_set_special_type_attr(PyTypeObject *type, PyObject *value, const char *name) { - if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { + if (!value) { PyErr_Format(PyExc_TypeError, - "cannot set '%s' attribute of immutable type '%s'", + "cannot delete '%s' attribute of type '%s'", name, type->tp_name); return 0; } - if (!value) { + if (_PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)) { PyErr_Format(PyExc_TypeError, - "cannot delete '%s' attribute of immutable type '%s'", + "cannot set '%s' attribute of immutable type '%s'", name, type->tp_name); return 0; } @@ -1421,9 +1553,13 @@ type_set_name(PyObject *tp, PyObject *value, void *Py_UNUSED(closure)) return -1; } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); type->tp_name = tp_name; - Py_SETREF(((PyHeapTypeObject*)type)->ht_name, Py_NewRef(value)); - + PyObject *old_name = ((PyHeapTypeObject*)type)->ht_name; + ((PyHeapTypeObject*)type)->ht_name = Py_NewRef(value); + _PyEval_StartTheWorld(interp); + Py_DECREF(old_name); return 0; } @@ -1586,10 +1722,13 @@ type_set_abstractmethods(PyObject *tp, PyObject *value, void *Py_UNUSED(closure) BEGIN_TYPE_LOCK(); type_modified_unlocked(type); + types_stop_world(); if (abstract) type_add_flags(type, Py_TPFLAGS_IS_ABSTRACT); else type_clear_flags(type, Py_TPFLAGS_IS_ABSTRACT); + types_start_world(); + ASSERT_TYPE_LOCK_HELD(); END_TYPE_LOCK(); return 0; @@ -1624,15 +1763,15 @@ type_get_mro(PyObject *tp, void *Py_UNUSED(closure)) return mro; } -static PyTypeObject *best_base(PyObject *); -static int mro_internal(PyTypeObject *, PyObject **); +static PyTypeObject *find_best_base(PyObject *); +static int mro_internal(PyTypeObject *, int, PyObject **); static int type_is_subtype_base_chain(PyTypeObject *, PyTypeObject *); -static int compatible_for_assignment(PyTypeObject *, PyTypeObject *, const char *); +static int compatible_for_assignment(PyTypeObject *, PyTypeObject *, const char *, int); static int add_subclass(PyTypeObject*, PyTypeObject*); static int add_all_subclasses(PyTypeObject *type, PyObject *bases); static void remove_subclass(PyTypeObject *, PyTypeObject *); static void remove_all_subclasses(PyTypeObject *type, PyObject *bases); -static void update_all_slots(PyTypeObject *); +static int update_all_slots(PyTypeObject *); typedef int (*update_callback)(PyTypeObject *, void *); static int update_subclasses(PyTypeObject *type, PyObject *attr_name, @@ -1640,18 +1779,21 @@ static int update_subclasses(PyTypeObject *type, PyObject *attr_name, static int recurse_down_subclasses(PyTypeObject *type, PyObject *name, update_callback callback, void *data); +// Compute tp_mro for this type and all of its subclasses. This +// is called after __bases__ is assigned to an existing type. static int -mro_hierarchy(PyTypeObject *type, PyObject *temp) +mro_hierarchy_for_complete_type(PyTypeObject *type, PyObject *temp) { ASSERT_TYPE_LOCK_HELD(); PyObject *old_mro; - int res = mro_internal(type, &old_mro); + int res = mro_internal(type, 0, &old_mro); if (res <= 0) { /* error / reentrance */ return res; } PyObject *new_mro = lookup_tp_mro(type); + assert(new_mro != NULL); PyObject *tuple; if (old_mro != NULL) { @@ -1696,7 +1838,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_ssize_t n = PyList_GET_SIZE(subclasses); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *subclass = _PyType_CAST(PyList_GET_ITEM(subclasses, i)); - res = mro_hierarchy(subclass, temp); + res = mro_hierarchy_for_complete_type(subclass, temp); if (res < 0) { break; } @@ -1708,9 +1850,9 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) } static int -type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases) +type_check_new_bases(PyTypeObject *type, PyObject *new_bases, PyTypeObject **best_base) { - // Check arguments + // Check arguments, this is re-entrant due to the PySys_Audit() call if (!check_set_special_type_attr(type, new_bases, "__bases__")) { return -1; } @@ -1759,26 +1901,39 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases) } // Compute the new MRO and the new base class - PyTypeObject *new_base = best_base(new_bases); - if (new_base == NULL) + *best_base = find_best_base(new_bases); + if (*best_base == NULL) return -1; - if (!compatible_for_assignment(type->tp_base, new_base, "__bases__")) { + if (!compatible_for_assignment(type, *best_base, "__bases__", 0)) { return -1; } + return 0; +} + +static int +type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases, PyTypeObject *best_base) +{ + ASSERT_TYPE_LOCK_HELD(); + + Py_ssize_t n; PyObject *old_bases = lookup_tp_bases(type); assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; + type_lock_prevent_release(); + types_stop_world(); set_tp_bases(type, Py_NewRef(new_bases), 0); - type->tp_base = (PyTypeObject *)Py_NewRef(new_base); + type->tp_base = (PyTypeObject *)Py_NewRef(best_base); + types_start_world(); + type_lock_allow_release(); PyObject *temp = PyList_New(0); if (temp == NULL) { goto bail; } - if (mro_hierarchy(type, temp) < 0) { + if (mro_hierarchy_for_complete_type(type, temp) < 0) { goto undo; } Py_DECREF(temp); @@ -1796,7 +1951,11 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases) add to all new_bases */ remove_all_subclasses(type, old_bases); res = add_all_subclasses(type, new_bases); - update_all_slots(type); + if (update_all_slots(type) < 0) { + goto bail; + } + /* Clear the VALID_VERSION flag of 'type' and all its subclasses. */ + type_modified_unlocked(type); } else { res = 0; @@ -1827,13 +1986,17 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases) bail: if (lookup_tp_bases(type) == new_bases) { - assert(type->tp_base == new_base); + assert(type->tp_base == best_base); + type_lock_prevent_release(); + types_stop_world(); set_tp_bases(type, old_bases, 0); type->tp_base = old_base; + types_start_world(); + type_lock_allow_release(); Py_DECREF(new_bases); - Py_DECREF(new_base); + Py_DECREF(best_base); } else { Py_DECREF(old_bases); @@ -1848,9 +2011,13 @@ static int type_set_bases(PyObject *tp, PyObject *new_bases, void *Py_UNUSED(closure)) { PyTypeObject *type = PyTypeObject_CAST(tp); + PyTypeObject *best_base; int res; BEGIN_TYPE_LOCK(); - res = type_set_bases_unlocked(type, new_bases); + res = type_check_new_bases(type, new_bases, &best_base); + if (res == 0) { + res = type_set_bases_unlocked(type, new_bases, best_base); + } END_TYPE_LOCK(); return res; } @@ -2028,8 +2195,9 @@ type_get_annotations(PyObject *tp, void *Py_UNUSED(closure)) return NULL; } if (!PyDict_Check(annotations)) { - PyErr_Format(PyExc_TypeError, "__annotate__ returned non-dict of type '%.100s'", - Py_TYPE(annotations)->tp_name); + PyErr_Format(PyExc_TypeError, + "__annotate__() must return a dict, not %T", + annotations); Py_DECREF(annotations); Py_DECREF(annotate); Py_DECREF(dict); @@ -2065,19 +2233,46 @@ type_set_annotations(PyObject *tp, PyObject *value, void *Py_UNUSED(closure)) return -1; } - int result; PyObject *dict = PyType_GetDict(type); - if (value != NULL) { - /* set */ - result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value); - } else { - /* delete */ - result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); - if (result == 0) { - PyErr_SetString(PyExc_AttributeError, "__annotations__"); + int result = PyDict_ContainsString(dict, "__annotations__"); + if (result < 0) { + Py_DECREF(dict); + return -1; + } + if (result) { + // If __annotations__ is currently in the dict, we update it, + if (value != NULL) { + result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); + } else { + result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (result == 0) { + // Somebody else just deleted it? + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; + } + } + if (result < 0) { Py_DECREF(dict); return -1; } + // Also clear __annotations_cache__ just in case. + result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); + } + else { + // Else we update only __annotations_cache__. + if (value != NULL) { + /* set */ + result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value); + } else { + /* delete */ + result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); + if (result == 0) { + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; + } + } } if (result < 0) { Py_DECREF(dict); @@ -3051,6 +3246,7 @@ static PyObject * class_name(PyObject *cls) { PyObject *name; + // Note that this is potentially re-entrant. if (PyObject_GetOptionalAttr(cls, &_Py_ID(__name__), &name) == 0) { name = PyObject_Repr(cls); } @@ -3247,6 +3443,7 @@ mro_implementation_unlocked(PyTypeObject *type) */ PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, 0)); PyObject *base_mro = lookup_tp_mro(base); + assert(base_mro != NULL); Py_ssize_t k = PyTuple_GET_SIZE(base_mro); PyObject *result = PyTuple_New(k + 1); if (result == NULL) { @@ -3281,9 +3478,12 @@ mro_implementation_unlocked(PyTypeObject *type) return NULL; } + PyObject *mro_to_merge; for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); - to_merge[i] = lookup_tp_mro(base); + mro_to_merge = lookup_tp_mro(base); + assert(mro_to_merge != NULL); + to_merge[i] = mro_to_merge; } to_merge[n] = bases; @@ -3342,10 +3542,8 @@ mro_check(PyTypeObject *type, PyObject *mro) for (i = 0; i < n; i++) { PyObject *obj = PyTuple_GET_ITEM(mro, i); if (!PyType_Check(obj)) { - PyErr_Format( - PyExc_TypeError, - "mro() returned a non-class ('%.500s')", - Py_TYPE(obj)->tp_name); + PyErr_Format(PyExc_TypeError, + "%N.mro() returned a non-class ('%T')", type, obj); return -1; } PyTypeObject *base = (PyTypeObject*)obj; @@ -3353,8 +3551,8 @@ mro_check(PyTypeObject *type, PyObject *mro) if (!is_subtype_with_mro(lookup_tp_mro(solid), solid, solid_base(base))) { PyErr_Format( PyExc_TypeError, - "mro() returned base with unsuitable layout ('%.500s')", - base->tp_name); + "%N.mro() returned base with unsuitable layout ('%N')", + type, base); return -1; } } @@ -3387,9 +3585,13 @@ mro_invoke(PyTypeObject *type) const int custom = !Py_IS_TYPE(type, &PyType_Type); if (custom) { + // Custom mro() method on metaclass. This is potentially re-entrant. + // We are called either from type_ready() or from type_set_bases(). mro_result = call_method_noarg((PyObject *)type, &_Py_ID(mro)); } else { + // In this case, the mro() method on the type object is being used and + // we know that these calls are not re-entrant. mro_result = mro_implementation_unlocked(type); } if (mro_result == NULL) @@ -3437,7 +3639,7 @@ mro_invoke(PyTypeObject *type) - Returns -1 in case of an error. */ static int -mro_internal_unlocked(PyTypeObject *type, int initial, PyObject **p_old_mro) +mro_internal(PyTypeObject *type, int initial, PyObject **p_old_mro) { ASSERT_TYPE_LOCK_HELD(); @@ -3485,21 +3687,11 @@ mro_internal_unlocked(PyTypeObject *type, int initial, PyObject **p_old_mro) return 1; } -static int -mro_internal(PyTypeObject *type, PyObject **p_old_mro) -{ - int res; - BEGIN_TYPE_LOCK(); - res = mro_internal_unlocked(type, 0, p_old_mro); - END_TYPE_LOCK(); - return res; -} - /* Calculate the best base amongst multiple base classes. This is the first one that's on the path to the "solid base". */ static PyTypeObject * -best_base(PyObject *bases) +find_best_base(PyObject *bases) { Py_ssize_t i, n; PyTypeObject *base, *winner, *candidate; @@ -3581,13 +3773,167 @@ solid_base(PyTypeObject *type) } } +#ifdef Py_GIL_DISABLED + +// The structures and functions below are used in the free-threaded build +// to safely make updates to type slots, on type_setattro() for a slot +// or when __bases__ is re-assigned. Since the slots are read without atomic +// operations and without locking, we can only safely update them while the +// world is stopped. However, with the world stopped, we are very limited on +// which APIs can be safely used. For example, calling _PyObject_HashFast() +// or _PyDict_GetItemRef_KnownHash() are not safe and can potentially cause +// deadlocks. Hashing can be re-entrant and _PyDict_GetItemRef_KnownHash can +// acquire a lock if the dictionary is not owned by the current thread, to +// mark it shared on reading. +// +// We do the slot updates in two steps. First, with TYPE_LOCK held, we lookup +// the descriptor for each slot, for each subclass. We build a queue of +// updates to perform but don't actually update the type structures. After we +// are finished the lookups, we stop-the-world and apply all of the updates. +// The apply_slot_updates() code is simple and easy to confirm that it is +// safe. + +typedef struct { + PyTypeObject *type; + void **slot_ptr; + void *slot_value; +} slot_update_item_t; + +// The number of slot updates performed is based on the number of changed +// slots and the number of subclasses. It's possible there are many updates +// required if there are many subclasses (potentially an unbounded amount). +// Usually the number of slot updates is small, most often zero or one. When +// running the unit tests, we don't exceed 20. The chunk size is set to +// handle the common case with a single chunk and to not require too many +// chunk allocations if there are many subclasses. +#define SLOT_UPDATE_CHUNK_SIZE 30 + +typedef struct _slot_update { + struct _slot_update *prev; + Py_ssize_t n; + slot_update_item_t updates[SLOT_UPDATE_CHUNK_SIZE]; +} slot_update_chunk_t; + +// a queue of updates to be performed +typedef struct { + slot_update_chunk_t *head; +} slot_update_t; + +static slot_update_chunk_t * +slot_update_new_chunk(void) +{ + slot_update_chunk_t *chunk = PyMem_Malloc(sizeof(slot_update_chunk_t)); + if (chunk == NULL) { + PyErr_NoMemory(); + return NULL; + } + chunk->prev = NULL; + chunk->n = 0; + return chunk; +} + +static void +slot_update_free_chunks(slot_update_t *updates) +{ + slot_update_chunk_t *chunk = updates->head; + while (chunk != NULL) { + slot_update_chunk_t *prev = chunk->prev; + PyMem_Free(chunk); + chunk = prev; + } +} + +static int +queue_slot_update(slot_update_t *updates, PyTypeObject *type, + void **slot_ptr, void *slot_value) +{ + if (*slot_ptr == slot_value) { + return 0; // slot pointer not actually changed, don't queue update + } + if (updates->head == NULL || updates->head->n == SLOT_UPDATE_CHUNK_SIZE) { + slot_update_chunk_t *chunk = slot_update_new_chunk(); + if (chunk == NULL) { + return -1; // out-of-memory + } + chunk->prev = updates->head; + updates->head = chunk; + } + slot_update_item_t *item = &updates->head->updates[updates->head->n]; + item->type = type; + item->slot_ptr = slot_ptr; + item->slot_value = slot_value; + updates->head->n++; + assert(updates->head->n <= SLOT_UPDATE_CHUNK_SIZE); + return 0; +} + +static void +apply_slot_updates(slot_update_t *updates) +{ + assert(types_world_is_stopped()); + slot_update_chunk_t *chunk = updates->head; + while (chunk != NULL) { + for (Py_ssize_t i = 0; i < chunk->n; i++) { + slot_update_item_t *item = &chunk->updates[i]; + *(item->slot_ptr) = item->slot_value; + if (item->slot_value == slot_tp_call) { + /* A generic __call__ is incompatible with vectorcall */ + type_clear_flags(item->type, Py_TPFLAGS_HAVE_VECTORCALL); + } + } + chunk = chunk->prev; + } +} + +static void +apply_type_slot_updates(slot_update_t *updates) +{ + // This must be done carefully to avoid data races and deadlocks. We + // have just updated the type __dict__, while holding TYPE_LOCK. We have + // collected all of the required type slot updates into the 'updates' + // queue. Note that those updates can apply to multiple types since + // subclasses might also be affected by the dict change. + // + // We need to prevent other threads from writing to the dict before we can + // finish updating the slots. The actual stores to the slots are done + // with the world stopped. If we block on the stop-the-world mutex then + // we could release TYPE_LOCK mutex and potentially allow other threads + // to update the dict. That's because TYPE_LOCK was acquired using a + // critical section. + // + // The type_lock_prevent_release() call prevents the TYPE_LOCK mutex from + // being released even if we block on the STM mutex. We need to take care + // that we do not deadlock because of that. It is safe because we always + // acquire locks in the same order: first the TYPE_LOCK mutex and then the + // STM mutex. + type_lock_prevent_release(); + types_stop_world(); + apply_slot_updates(updates); + types_start_world(); + type_lock_allow_release(); +} + +#else + +// dummy definition, this parameter is only NULL in the default build +typedef void slot_update_t; + +#endif + +/// data passed to update_slots_callback() +typedef struct { + slot_update_t *queued_updates; + pytype_slotdef **defs; +} update_callback_data_t; + static void object_dealloc(PyObject *); static PyObject *object_new(PyTypeObject *, PyObject *, PyObject *); static int object_init(PyObject *, PyObject *, PyObject *); -static int update_slot(PyTypeObject *, PyObject *); +static int update_slot(PyTypeObject *, PyObject *, slot_update_t *update); static void fixup_slot_dispatchers(PyTypeObject *); static int type_new_set_names(PyTypeObject *); static int type_new_init_subclass(PyTypeObject *, PyObject *); +static bool has_slotdef(PyObject *); /* * Helpers for __dict__ descriptor. We don't want to expose the dicts @@ -3649,10 +3995,35 @@ subtype_dict(PyObject *obj, void *context) return PyObject_GenericGetDict(obj, context); } +int +_PyObject_SetDict(PyObject *obj, PyObject *value) +{ + if (value != NULL && !PyDict_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__dict__ must be set to a dictionary, " + "not a '%.200s'", Py_TYPE(value)->tp_name); + return -1; + } + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + return _PyObject_SetManagedDict(obj, value); + } + PyObject **dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + Py_BEGIN_CRITICAL_SECTION(obj); + // gh-133980: To prevent use-after-free from other threads that reference + // the __dict__ + _PyObject_XSetRefDelayed(dictptr, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); + return 0; +} + static int subtype_setdict(PyObject *obj, PyObject *value, void *context) { - PyObject **dictptr; PyTypeObject *base; base = get_builtin_base_with_dict(Py_TYPE(obj)); @@ -3670,28 +4041,7 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) } return func(descr, obj, value); } - /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ - if (value != NULL && !PyDict_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__dict__ must be set to a dictionary, " - "not a '%.200s'", Py_TYPE(value)->tp_name); - return -1; - } - - if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - return _PyObject_SetManagedDict(obj, value); - } - else { - dictptr = _PyObject_ComputedDictPointer(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return -1; - } - Py_CLEAR(*dictptr); - *dictptr = Py_XNewRef(value); - } - return 0; + return _PyObject_SetDict(obj, value); } static PyObject * @@ -3720,26 +4070,15 @@ subtype_getweakref(PyObject *obj, void *context) return Py_NewRef(result); } -/* Three variants on the subtype_getsets list. */ - -static PyGetSetDef subtype_getsets_full[] = { - {"__dict__", subtype_dict, subtype_setdict, - PyDoc_STR("dictionary for instance variables")}, - {"__weakref__", subtype_getweakref, NULL, - PyDoc_STR("list of weak references to the object")}, - {0} +/* getset definitions for common descriptors */ +static PyGetSetDef subtype_getset_dict = { + "__dict__", subtype_dict, subtype_setdict, + PyDoc_STR("dictionary for instance variables"), }; -static PyGetSetDef subtype_getsets_dict_only[] = { - {"__dict__", subtype_dict, subtype_setdict, - PyDoc_STR("dictionary for instance variables")}, - {0} -}; - -static PyGetSetDef subtype_getsets_weakref_only[] = { - {"__weakref__", subtype_getweakref, NULL, - PyDoc_STR("list of weak references to the object")}, - {0} +static PyGetSetDef subtype_getset_weakref = { + "__weakref__", subtype_getweakref, NULL, + PyDoc_STR("list of weak references to the object"), }; static int @@ -3785,7 +4124,7 @@ type_init(PyObject *cls, PyObject *args, PyObject *kwds) unsigned long PyType_GetFlags(PyTypeObject *type) { - return FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags); + return type->tp_flags; } @@ -4004,14 +4343,6 @@ type_new_slots_bases(type_new_ctx *ctx) static int type_new_slots_impl(type_new_ctx *ctx, PyObject *dict) { - /* Are slots allowed? */ - if (ctx->nslot > 0 && ctx->base->tp_itemsize != 0) { - PyErr_Format(PyExc_TypeError, - "nonempty __slots__ not supported for subtype of '%s'", - ctx->base->tp_name); - return -1; - } - if (type_new_visit_slots(ctx) < 0) { return -1; } @@ -4038,14 +4369,13 @@ type_new_slots(type_new_ctx *ctx, PyObject *dict) ctx->add_dict = 0; ctx->add_weak = 0; ctx->may_add_dict = (ctx->base->tp_dictoffset == 0); - ctx->may_add_weak = (ctx->base->tp_weaklistoffset == 0 - && ctx->base->tp_itemsize == 0); + ctx->may_add_weak = (ctx->base->tp_weaklistoffset == 0); if (ctx->slots == NULL) { if (ctx->may_add_dict) { ctx->add_dict++; } - if (ctx->may_add_weak) { + if (ctx->may_add_weak && ctx->base->tp_itemsize == 0) { ctx->add_weak++; } } @@ -4275,16 +4605,48 @@ type_new_classmethod(PyObject *dict, PyObject *attr) return 0; } +/* Add __dict__ or __weakref__ descriptor */ +static int +type_add_common_descriptor(PyInterpreterState *interp, + PyObject **cache, + PyGetSetDef *getset_def, + PyObject *dict) +{ +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->cached_objects.descriptor_mutex); +#endif + PyObject *descr = *cache; + if (!descr) { + descr = PyDescr_NewGetSet(&PyBaseObject_Type, getset_def); + *cache = descr; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->cached_objects.descriptor_mutex); +#endif + if (!descr) { + return -1; + } + if (PyDict_SetDefaultRef(dict, PyDescr_NAME(descr), descr, NULL) < 0) { + return -1; + } + return 0; +} /* Add descriptors for custom slots from __slots__, or for __dict__ */ static int -type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) +type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type, PyObject *dict) { PyHeapTypeObject *et = (PyHeapTypeObject *)type; Py_ssize_t slotoffset = ctx->base->tp_basicsize; if (et->ht_slots != NULL) { PyMemberDef *mp = _PyHeapType_GET_MEMBERS(et); Py_ssize_t nslot = PyTuple_GET_SIZE(et->ht_slots); + if (ctx->base->tp_itemsize != 0) { + PyErr_Format(PyExc_TypeError, + "arbitrary __slots__ not supported for subtype of '%s'", + ctx->base->tp_name); + return -1; + } for (Py_ssize_t i = 0; i < nslot; i++, mp++) { mp->name = PyUnicode_AsUTF8( PyTuple_GET_ITEM(et->ht_slots, i)); @@ -4316,6 +4678,30 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) type->tp_basicsize = slotoffset; type->tp_itemsize = ctx->base->tp_itemsize; type->tp_members = _PyHeapType_GET_MEMBERS(et); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + + if (type->tp_dictoffset) { + if (type_add_common_descriptor( + interp, + &interp->cached_objects.dict_descriptor, + &subtype_getset_dict, + dict) < 0) + { + return -1; + } + } + if (type->tp_weaklistoffset) { + if (type_add_common_descriptor( + interp, + &interp->cached_objects.weakref_descriptor, + &subtype_getset_weakref, + dict) < 0) + { + return -1; + } + } + return 0; } @@ -4323,18 +4709,7 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type) static void type_new_set_slots(const type_new_ctx *ctx, PyTypeObject *type) { - if (type->tp_weaklistoffset && type->tp_dictoffset) { - type->tp_getset = subtype_getsets_full; - } - else if (type->tp_weaklistoffset && !type->tp_dictoffset) { - type->tp_getset = subtype_getsets_weakref_only; - } - else if (!type->tp_weaklistoffset && type->tp_dictoffset) { - type->tp_getset = subtype_getsets_dict_only; - } - else { - type->tp_getset = NULL; - } + type->tp_getset = NULL; /* Special case some slots */ if (type->tp_dictoffset != 0 || ctx->nslot > 0) { @@ -4439,7 +4814,7 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type) return -1; } - if (type_new_descriptors(ctx, type) < 0) { + if (type_new_descriptors(ctx, type, dict) < 0) { return -1; } @@ -4511,8 +4886,14 @@ type_new_init(type_new_ctx *ctx) set_tp_dict(type, dict); PyHeapTypeObject *et = (PyHeapTypeObject*)type; - et->ht_slots = ctx->slots; - ctx->slots = NULL; + if (ctx->slots && PyTuple_GET_SIZE(ctx->slots)) { + et->ht_slots = ctx->slots; + ctx->slots = NULL; + } + else { + et->ht_slots = NULL; + Py_CLEAR(ctx->slots); + } return type; @@ -4563,6 +4944,10 @@ type_new_impl(type_new_ctx *ctx) } assert(_PyType_CheckConsistency(type)); +#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) && SIZEOF_VOID_P > 4 + // After this point, other threads can potentally use this type. + ((PyObject*)type)->ob_flags |= _Py_TYPE_REVEALED_FLAG; +#endif return (PyObject *)type; @@ -4625,7 +5010,7 @@ type_new_get_bases(type_new_ctx *ctx, PyObject **type) } /* Calculate best base, and check that all bases are type objects */ - PyTypeObject *base = best_base(ctx->bases); + PyTypeObject *base = find_best_base(ctx->bases); if (base == NULL) { return -1; } @@ -5040,12 +5425,12 @@ PyType_FromMetaclass( } /* Calculate best base, and check that all bases are type objects */ - PyTypeObject *base = best_base(bases); // borrowed ref + PyTypeObject *base = find_best_base(bases); // borrowed ref if (base == NULL) { goto finally; } - // best_base should check Py_TPFLAGS_BASETYPE & raise a proper exception, - // here we just check its work + // find_best_base() should check Py_TPFLAGS_BASETYPE & raise a proper + // exception, here we just check its work assert(_PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)); /* Calculate sizes */ @@ -5276,6 +5661,10 @@ PyType_FromMetaclass( } assert(_PyType_CheckConsistency(type)); +#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) && SIZEOF_VOID_P > 4 + // After this point, other threads can potentally use this type. + ((PyObject*)type)->ob_flags |= _Py_TYPE_REVEALED_FLAG; +#endif finally: if (PyErr_Occurred()) { @@ -5388,11 +5777,11 @@ PyType_GetModuleState(PyTypeObject *type) } -/* Get the module of the first superclass where the module has the - * given PyModuleDef. +/* Return borrowed ref to the module of the first superclass where the module + * has the given token. */ -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +static PyObject * +borrow_module_by_token(PyTypeObject *type, const void *token) { assert(PyType_Check(type)); @@ -5404,7 +5793,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) else { PyHeapTypeObject *ht = (PyHeapTypeObject*)type; PyObject *module = ht->ht_module; - if (module && _PyModule_GetDef(module) == def) { + if (module && _PyModule_GetToken(module) == token) { return module; } } @@ -5432,7 +5821,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) PyHeapTypeObject *ht = (PyHeapTypeObject*)super; PyObject *module = ht->ht_module; - if (module && _PyModule_GetDef(module) == def) { + if (module && _PyModule_GetToken(module) == token) { res = module; break; } @@ -5450,6 +5839,18 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) return NULL; } +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +{ + return borrow_module_by_token(type, def); +} + +PyObject * +PyType_GetModuleByToken(PyTypeObject *type, const void *token) +{ + return Py_XNewRef(borrow_module_by_token(type, token)); +} + static PyTypeObject * get_base_by_token_recursive(PyObject *bases, void *token) @@ -5571,8 +5972,6 @@ PyObject_GetItemData(PyObject *obj) static PyObject * find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) { - ASSERT_TYPE_LOCK_HELD(); - Py_hash_t hash = _PyObject_HashFast(name); if (hash == -1) { *error = -1; @@ -5640,7 +6039,7 @@ static PyObject * update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) { _Py_atomic_store_ptr_relaxed(&entry->value, value); /* borrowed */ - assert(_PyASCIIObject_CAST(name)->hash != -1); + assert(PyUnstable_Unicode_GET_CACHED_HASH(name) != -1); OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); // We're releasing this under the lock for simplicity sake because it's always a // exact unicode object or Py_None so it's safe to do so. @@ -5881,9 +6280,13 @@ _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor void _PyType_SetFlags(PyTypeObject *self, unsigned long mask, unsigned long flags) { - BEGIN_TYPE_LOCK(); - type_set_flags_with_mask(self, mask, flags); - END_TYPE_LOCK(); + unsigned long new_flags = (self->tp_flags & ~mask) | flags; + if (new_flags != self->tp_flags) { + types_stop_world(); + // can't use new_flags here since they could be out-of-date + self->tp_flags = (self->tp_flags & ~mask) | flags; + types_start_world(); + } } int @@ -5930,9 +6333,9 @@ set_flags_recursive(PyTypeObject *self, unsigned long mask, unsigned long flags) void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags) { - BEGIN_TYPE_LOCK(); + types_stop_world(); set_flags_recursive(self, mask, flags); - END_TYPE_LOCK(); + types_start_world(); } /* This is similar to PyObject_GenericGetAttr(), @@ -6046,6 +6449,8 @@ _Py_type_getattro(PyObject *tp, PyObject *name) return _Py_type_getattro_impl(type, name, NULL); } +// Called by type_setattro(). Updates both the type dict and +// the type versions. static int type_update_dict(PyTypeObject *type, PyDictObject *dict, PyObject *name, PyObject *value, PyObject **old_value) @@ -6075,10 +6480,30 @@ type_update_dict(PyTypeObject *type, PyDictObject *dict, PyObject *name, return -1; } - if (is_dunder_name(name)) { - return update_slot(type, name); - } + return 0; +} +static int +update_slot_after_setattr(PyTypeObject *type, PyObject *name) +{ +#ifdef Py_GIL_DISABLED + // stack allocate one chunk since that's all we need + assert(SLOT_UPDATE_CHUNK_SIZE >= MAX_EQUIV); + slot_update_chunk_t chunk = {0}; + slot_update_t queued_updates = {&chunk}; + + if (update_slot(type, name, &queued_updates) < 0) { + return -1; + } + if (queued_updates.head->n > 0) { + apply_type_slot_updates(&queued_updates); + ASSERT_TYPE_LOCK_HELD(); + // should never allocate another chunk + assert(chunk.prev == NULL); + } +#else + update_slot(type, name, NULL); +#endif return 0; } @@ -6124,6 +6549,18 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); +#ifdef Py_GIL_DISABLED + // gh-139103: Enable deferred refcounting for functions assigned + // to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically. + if (value != NULL && + PyFunction_Check(value) && + !_PyObject_HasDeferredRefcount(value)) + { + PyUnstable_Object_EnableDeferredRefcount(value); + } +#endif + PyObject *old_value = NULL; PyObject *descr = _PyType_LookupRef(metatype, name); if (descr != NULL) { @@ -6136,7 +6573,9 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) PyObject *dict = type->tp_dict; if (dict == NULL) { - // We don't just do PyType_Ready because we could already be readying + // This is an unlikely case. PyType_Ready has not yet been done and + // we need to initialize tp_dict. We don't just do PyType_Ready + // because we could already be readying. BEGIN_TYPE_LOCK(); dict = type->tp_dict; if (dict == NULL) { @@ -6152,6 +6591,12 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) BEGIN_TYPE_DICT_LOCK(dict); res = type_update_dict(type, (PyDictObject *)dict, name, value, &old_value); assert(_PyType_CheckConsistency(type)); + if (res == 0) { + if (is_dunder_name(name) && has_slotdef(name)) { + // The name corresponds to a type slot. + res = update_slot_after_setattr(type, name); + } + } END_TYPE_DICT_LOCK(); done: @@ -6283,6 +6728,14 @@ _PyStaticType_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) } +void +_PyTypes_FiniCachedDescriptors(PyInterpreterState *interp) +{ + Py_CLEAR(interp->cached_objects.dict_descriptor); + Py_CLEAR(interp->cached_objects.weakref_descriptor); +} + + static void type_dealloc(PyObject *self) { @@ -6868,10 +7321,6 @@ compatible_with_tp_base(PyTypeObject *child) return (parent != NULL && child->tp_basicsize == parent->tp_basicsize && child->tp_itemsize == parent->tp_itemsize && - child->tp_dictoffset == parent->tp_dictoffset && - child->tp_weaklistoffset == parent->tp_weaklistoffset && - ((child->tp_flags & Py_TPFLAGS_HAVE_GC) == - (parent->tp_flags & Py_TPFLAGS_HAVE_GC)) && (child->tp_dealloc == subtype_dealloc || child->tp_dealloc == parent->tp_dealloc)); } @@ -6906,11 +7355,24 @@ same_slots_added(PyTypeObject *a, PyTypeObject *b) } static int -compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* attr) +compatible_flags(int setclass, PyTypeObject *origto, PyTypeObject *newto, unsigned long flags) +{ + /* For __class__ assignment, the flags should be the same. + For __bases__ assignment, the new base flags can only be set + if the original class flags are set. + */ + return setclass ? (origto->tp_flags & flags) == (newto->tp_flags & flags) + : !(~(origto->tp_flags & flags) & (newto->tp_flags & flags)); +} + +static int +compatible_for_assignment(PyTypeObject *origto, PyTypeObject *newto, + const char *attr, int setclass) { PyTypeObject *newbase, *oldbase; + PyTypeObject *oldto = setclass ? origto : origto->tp_base; - if (newto->tp_free != oldto->tp_free) { + if (setclass && newto->tp_free != oldto->tp_free) { PyErr_Format(PyExc_TypeError, "%s assignment: " "'%s' deallocator differs from '%s'", @@ -6919,6 +7381,28 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* oldto->tp_name); return 0; } + if (!compatible_flags(setclass, origto, newto, + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_INLINE_VALUES | + Py_TPFLAGS_PREHEADER)) + { + goto differs; + } + /* For __class__ assignment, tp_dictoffset and tp_weaklistoffset should + be the same for old and new types. + For __bases__ assignment, they can only be set in the new base + if they are set in the original class with the same value. + */ + if ((setclass || newto->tp_dictoffset) + && origto->tp_dictoffset != newto->tp_dictoffset) + { + goto differs; + } + if ((setclass || newto->tp_weaklistoffset) + && origto->tp_weaklistoffset != newto->tp_weaklistoffset) + { + goto differs; + } /* It's tricky to tell if two arbitrary types are sufficiently compatible as to be interchangeable; e.g., even if they have the same tp_basicsize, they @@ -6940,17 +7424,7 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* !same_slots_added(newbase, oldbase))) { goto differs; } - if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) != - ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES))) - { - goto differs; - } - /* The above does not check for the preheader */ - if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == - ((newto->tp_flags & Py_TPFLAGS_PREHEADER))) - { - return 1; - } + return 1; differs: PyErr_Format(PyExc_TypeError, "%s assignment: " @@ -7027,7 +7501,7 @@ object_set_class_world_stopped(PyObject *self, PyTypeObject *newto) return -1; } - if (compatible_for_assignment(oldto, newto, "__class__")) { + if (compatible_for_assignment(oldto, newto, "__class__", 1)) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { @@ -7081,15 +7555,10 @@ object_set_class(PyObject *self, PyObject *value, void *closure) return -1; } -#ifdef Py_GIL_DISABLED - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); -#endif + types_stop_world(); PyTypeObject *oldto = Py_TYPE(self); int res = object_set_class_world_stopped(self, newto); -#ifdef Py_GIL_DISABLED - _PyEval_StartTheWorld(interp); -#endif + types_start_world(); if (res == 0) { if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) { Py_DECREF(oldto); @@ -8466,6 +8935,13 @@ type_ready_preheader(PyTypeObject *type) type->tp_name); return -1; } + if (!(type->tp_flags & Py_TPFLAGS_HAVE_GC)) { + PyErr_Format(PyExc_SystemError, + "type %s has the Py_TPFLAGS_MANAGED_DICT flag " + "but not Py_TPFLAGS_HAVE_GC flag", + type->tp_name); + return -1; + } type->tp_dictoffset = -1; } if (type->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF) { @@ -8478,6 +8954,13 @@ type_ready_preheader(PyTypeObject *type) type->tp_name); return -1; } + if (!(type->tp_flags & Py_TPFLAGS_HAVE_GC)) { + PyErr_Format(PyExc_SystemError, + "type %s has the Py_TPFLAGS_MANAGED_WEAKREF flag " + "but not Py_TPFLAGS_HAVE_GC flag", + type->tp_name); + return -1; + } type->tp_weaklistoffset = MANAGED_WEAKREF_OFFSET; } return 0; @@ -8497,7 +8980,7 @@ type_ready_mro(PyTypeObject *type, int initial) } /* Calculate method resolution order */ - if (mro_internal_unlocked(type, initial, NULL) < 0) { + if (mro_internal(type, initial, NULL) < 0) { return -1; } PyObject *mro = lookup_tp_mro(type); @@ -8571,6 +9054,7 @@ type_ready_inherit(PyTypeObject *type) // Inherit slots PyObject *mro = lookup_tp_mro(type); + assert(mro != NULL); Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(mro, i); @@ -8669,7 +9153,11 @@ type_ready_set_new(PyTypeObject *type, int initial) && base == &PyBaseObject_Type && !(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - type_add_flags(type, Py_TPFLAGS_DISALLOW_INSTANTIATION); + if (initial) { + type_add_flags(type, Py_TPFLAGS_DISALLOW_INSTANTIATION); + } else { + assert(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION); + } } if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { @@ -8683,13 +9171,17 @@ type_ready_set_new(PyTypeObject *type, int initial) } } else { - // tp_new is NULL: inherit tp_new from base - type->tp_new = base->tp_new; + if (initial) { + // tp_new is NULL: inherit tp_new from base + type->tp_new = base->tp_new; + } } } else { // Py_TPFLAGS_DISALLOW_INSTANTIATION sets tp_new to NULL - type->tp_new = NULL; + if (initial) { + type->tp_new = NULL; + } } return 0; } @@ -8822,7 +9314,12 @@ type_ready(PyTypeObject *type, int initial) } /* All done -- set the ready flag */ - type_add_flags(type, Py_TPFLAGS_READY); + if (initial) { + type_add_flags(type, Py_TPFLAGS_READY); + } else { + assert(type->tp_flags & Py_TPFLAGS_READY); + } + stop_readying(type); assert(_PyType_CheckConsistency(type)); @@ -8871,15 +9368,16 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self, assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); - if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { - assert(initial); + if (initial) { + assert((self->tp_flags & Py_TPFLAGS_READY) == 0); type_add_flags(self, _Py_TPFLAGS_STATIC_BUILTIN); type_add_flags(self, Py_TPFLAGS_IMMUTABLETYPE); - assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); if (self->tp_version_tag == 0) { - _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); + unsigned int next_version_tag = next_global_version_tag(); + assert(next_version_tag != 0); + _PyType_SetVersion(self, next_version_tag); } } else { @@ -9682,6 +10180,11 @@ tp_new_wrapper(PyObject *self, PyObject *args, PyObject *kwds) /* If staticbase is NULL now, it is a really weird type. In the spirit of backwards compatibility (?), just shut up. */ if (staticbase && staticbase->tp_new != type->tp_new) { + if (staticbase->tp_new == NULL) { + PyErr_Format(PyExc_TypeError, + "cannot create '%s' instances", subtype->tp_name); + return NULL; + } PyErr_Format(PyExc_TypeError, "%s.__new__(%s) is not safe, use %s.__new__()", type->tp_name, @@ -10045,9 +10548,8 @@ slot_nb_bool(PyObject *self) } else { PyErr_Format(PyExc_TypeError, - "__bool__ should return " - "bool, returned %s", - Py_TYPE(value)->tp_name); + "%T.__bool__() must return a bool, not %T", + self, value); result = -1; } Py_DECREF(value); @@ -10124,6 +10626,7 @@ slot_tp_hash(PyObject *self) return PyObject_HashNotImplemented(self); } if (!PyLong_Check(res)) { + Py_DECREF(res); PyErr_SetString(PyExc_TypeError, "__hash__ method should return an integer"); return -1; @@ -10211,7 +10714,10 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ +#ifndef Py_GIL_DISABLED + // Replacing the slot is only thread-safe if there is a GIL. tp->tp_getattro = _Py_slot_tp_getattro; +#endif return _Py_slot_tp_getattro(self, name); } /* speed hack: we could use lookup_maybe, but that would resolve the @@ -10336,9 +10842,11 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { +#ifndef Py_GIL_DISABLED /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) tp->tp_descr_get = NULL; +#endif return Py_NewRef(self); } if (obj == NULL) @@ -10522,7 +11030,8 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) } if (!PyMemoryView_Check(ret)) { PyErr_Format(PyExc_TypeError, - "__buffer__ returned non-memoryview object"); + "%T.__buffer__() must return a memoryview, not %T", + self, ret); goto fail; } @@ -10971,6 +11480,11 @@ static pytype_slotdef slotdefs[] = { {NULL} }; +/* Stores the number of times where slotdefs has elements with same name. + This counter precalculated by _PyType_InitSlotDefs() when the main + interpreter starts. */ +static uint8_t slotdefs_name_counts[Py_ARRAY_LENGTH(slotdefs)]; + /* Given a type pointer and an offset gotten from a slotdef entry, return a pointer to the actual slot. This is not quite the same as simply adding the offset to the type pointer, since it takes care to indirect through the @@ -11013,49 +11527,19 @@ slotptr(PyTypeObject *type, int ioffset) return (void **)ptr; } -/* Return a slot pointer for a given name, but ONLY if the attribute has - exactly one slot function. The name must be an interned string. */ -static void ** -resolve_slotdups(PyTypeObject *type, PyObject *name) +// Return true if "name" corresponds to at least one slot definition. This is +// a more accurate but more expensive test compared to is_dunder_name(). +static bool +has_slotdef(PyObject *name) { - /* XXX Maybe this could be optimized more -- but is it worth it? */ - - /* pname and ptrs act as a little cache */ - PyInterpreterState *interp = _PyInterpreterState_GET(); -#define pname _Py_INTERP_CACHED_OBJECT(interp, type_slots_pname) -#define ptrs _Py_INTERP_CACHED_OBJECT(interp, type_slots_ptrs) - pytype_slotdef *p, **pp; - void **res, **ptr; - - if (pname != name) { - /* Collect all slotdefs that match name into ptrs. */ - pname = name; - pp = ptrs; - for (p = slotdefs; p->name_strobj; p++) { - if (p->name_strobj == name) - *pp++ = p; + for (pytype_slotdef *p = slotdefs; p->name_strobj; p++) { + if (p->name_strobj == name) { + return true; } - *pp = NULL; } - - /* Look in all slots of the type matching the name. If exactly one of these - has a filled-in slot, return a pointer to that slot. - Otherwise, return NULL. */ - res = NULL; - for (pp = ptrs; *pp; pp++) { - ptr = slotptr(type, (*pp)->offset); - if (ptr == NULL || *ptr == NULL) - continue; - if (res != NULL) - return NULL; - res = ptr; - } - return res; -#undef pname -#undef ptrs + return false; } - /* Common code for update_slots_callback() and fixup_slot_dispatchers(). * * This is meant to set a "slot" like type->tp_repr or @@ -11107,13 +11591,22 @@ resolve_slotdups(PyTypeObject *type, PyObject *name) * There are some further special cases for specific slots, like supporting * __hash__ = None for tp_hash and special code for tp_new. * - * When done, return a pointer to the next slotdef with a different offset, - * because that's convenient for fixup_slot_dispatchers(). This function never - * sets an exception: if an internal error happens (unlikely), it's ignored. */ -static pytype_slotdef * -update_one_slot(PyTypeObject *type, pytype_slotdef *p) + * When done, next_p is set to the next slotdef with a different offset, + * because that's convenient for fixup_slot_dispatchers(). + * + * If the queued_updates pointer is provided, the actual updates to the slot + * pointers are queued, rather than being immediately performed. That argument + * is only used for the free-threaded build since those updates need to be + * done while the world is stopped. + * + * This function will only return an error if the queued_updates argument is + * provided and allocating memory for the queue fails. Other exceptions that + * occur internally are ignored, such as when looking up descriptors. */ +static int +update_one_slot(PyTypeObject *type, pytype_slotdef *p, pytype_slotdef **next_p, + slot_update_t *queued_updates) { - ASSERT_TYPE_LOCK_HELD(); + ASSERT_NEW_TYPE_OR_LOCKED(type); PyObject *descr; PyWrapperDescrObject *d; @@ -11136,7 +11629,10 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) do { ++p; } while (p->offset == offset); - return p; + if (next_p != NULL) { + *next_p = p; + } + return 0; } /* We may end up clearing live exceptions below, so make sure it's ours. */ assert(!PyErr_Occurred()); @@ -11157,7 +11653,15 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) } if (Py_IS_TYPE(descr, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)descr)->d_base->name_strobj == p->name_strobj) { - void **tptr = resolve_slotdups(type, p->name_strobj); + void **tptr; + size_t index = (p - slotdefs) / sizeof(slotdefs[0]); + if (slotdefs_name_counts[index] == 1) { + tptr = slotptr(type, p->offset); + } + else { + tptr = NULL; + } + if (tptr == NULL || tptr == ptr) generic = p->function; d = (PyWrapperDescrObject *)descr; @@ -11219,16 +11723,41 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) } if (p->function == slot_tp_call) { /* A generic __call__ is incompatible with vectorcall */ - type_clear_flags(type, Py_TPFLAGS_HAVE_VECTORCALL); + if (queued_updates == NULL) { + type_clear_flags(type, Py_TPFLAGS_HAVE_VECTORCALL); + } } } Py_DECREF(descr); } while ((++p)->offset == offset); - if (specific && !use_generic) - *ptr = specific; - else - *ptr = generic; - return p; + + void *slot_value; + if (specific && !use_generic) { + slot_value = specific; + } else { + slot_value = generic; + } + +#ifdef Py_GIL_DISABLED + if (queued_updates != NULL) { + // queue the update to perform later, while world is stopped + if (queue_slot_update(queued_updates, type, ptr, slot_value) < 0) { + return -1; + } + } else { + // do the update to the type structure now + *ptr = slot_value; + } +#else + // always do the update immediately + assert(queued_updates == NULL); + *ptr = slot_value; +#endif + + if (next_p != NULL) { + *next_p = p; + } + return 0; } /* In the type, update the slots whose slotdefs are gathered in the pp array. @@ -11236,18 +11765,21 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) static int update_slots_callback(PyTypeObject *type, void *data) { - ASSERT_TYPE_LOCK_HELD(); + ASSERT_NEW_TYPE_OR_LOCKED(type); - pytype_slotdef **pp = (pytype_slotdef **)data; + update_callback_data_t *update_data = (update_callback_data_t *)data; + pytype_slotdef **pp = update_data->defs; for (; *pp; pp++) { - update_one_slot(type, *pp); + if (update_one_slot(type, *pp, NULL, update_data->queued_updates) < 0) { + return -1; + } } return 0; } /* Update the slots after assignment to a class (type) attribute. */ static int -update_slot(PyTypeObject *type, PyObject *name) +update_slot(PyTypeObject *type, PyObject *name, slot_update_t *queued_updates) { pytype_slotdef *ptrs[MAX_EQUIV]; pytype_slotdef *p; @@ -11278,8 +11810,12 @@ update_slot(PyTypeObject *type, PyObject *name) } if (ptrs[0] == NULL) return 0; /* Not an attribute that affects any slots */ + + update_callback_data_t callback_data; + callback_data.defs = ptrs; + callback_data.queued_updates = queued_updates; return update_subclasses(type, name, - update_slots_callback, (void *)ptrs); + update_slots_callback, (void *)&callback_data); } /* Store the proper functions in the slot dispatches at class (type) @@ -11288,33 +11824,124 @@ update_slot(PyTypeObject *type, PyObject *name) static void fixup_slot_dispatchers(PyTypeObject *type) { - // This lock isn't strictly necessary because the type has not been - // exposed to anyone else yet, but update_ont_slot calls find_name_in_mro - // where we'd like to assert that the type is locked. - BEGIN_TYPE_LOCK(); - assert(!PyErr_Occurred()); for (pytype_slotdef *p = slotdefs; p->name; ) { - p = update_one_slot(type, p); + update_one_slot(type, p, &p, NULL); } - - END_TYPE_LOCK(); } -static void +#ifdef Py_GIL_DISABLED + +// Called when __bases__ is re-assigned. +static int +update_all_slots(PyTypeObject* type) +{ + // Note that update_slot() can fail due to out-of-memory when allocating + // the queue chunks to hold the updates. That's unlikely since the number + // of updates is normally small but we handle that case. update_slot() + // can fail internally for other reasons (a lookup fails) but those + // errors are suppressed. + slot_update_t queued_updates = {0}; + for (pytype_slotdef *p = slotdefs; p->name; p++) { + if (update_slot(type, p->name_strobj, &queued_updates) < 0) { + if (queued_updates.head) { + slot_update_free_chunks(&queued_updates); + } + return -1; + } + } + if (queued_updates.head != NULL) { + apply_type_slot_updates(&queued_updates); + ASSERT_TYPE_LOCK_HELD(); + slot_update_free_chunks(&queued_updates); + } + return 0; +} + +#else + +// Called when __bases__ is re-assigned. +static int update_all_slots(PyTypeObject* type) { pytype_slotdef *p; - - ASSERT_TYPE_LOCK_HELD(); - - /* Clear the VALID_VERSION flag of 'type' and all its subclasses. */ - type_modified_unlocked(type); - for (p = slotdefs; p->name; p++) { - /* update_slot returns int but can't actually fail */ - update_slot(type, p->name_strobj); + /* update_slot returns int but can't actually fail in this case*/ + update_slot(type, p->name_strobj, NULL); } + return 0; +} + +#endif + +int +_PyType_InitSlotDefs(PyInterpreterState *interp) +{ + if (!_Py_IsMainInterpreter(interp)) { + return 0; + } + PyObject *bytearray = NULL; + PyObject *cache = PyDict_New(); + if (!cache) { + return -1; + } + + pytype_slotdef *p; + Py_ssize_t idx = 0; + for (p = slotdefs; p->name_strobj; p++, idx++) { + assert(idx < 255); + + if (PyDict_GetItemRef(cache, p->name_strobj, &bytearray) < 0) { + goto error; + } + + if (!bytearray) { + Py_ssize_t size = sizeof(uint8_t) * (1 + MAX_EQUIV); + bytearray = PyByteArray_FromStringAndSize(NULL, size); + if (!bytearray) { + goto error; + } + + uint8_t *data = (uint8_t *)PyByteArray_AS_STRING(bytearray); + data[0] = 0; + + if (PyDict_SetItem(cache, p->name_strobj, bytearray) < 0) { + goto error; + } + } + + assert(PyByteArray_CheckExact(bytearray)); + uint8_t *data = (uint8_t *)PyByteArray_AS_STRING(bytearray); + + data[0] += 1; + assert(data[0] < MAX_EQUIV); + + data[data[0]] = (uint8_t)idx; + + Py_CLEAR(bytearray); + } + + memset(slotdefs_name_counts, 0, sizeof(slotdefs_name_counts)); + + Py_ssize_t pos = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(cache, &pos, &key, &value)) { + uint8_t *data = (uint8_t *)PyByteArray_AS_STRING(value); + uint8_t n = data[0]; + for (uint8_t i = 0; i < n; i++) { + uint8_t idx = data[i + 1]; + slotdefs_name_counts[idx] = n; + } + } + + Py_DECREF(cache); + return 0; + +error: + Py_XDECREF(bytearray); + Py_DECREF(cache); + return -1; } @@ -11586,7 +12213,10 @@ PyType_Freeze(PyTypeObject *type) } BEGIN_TYPE_LOCK(); + types_stop_world(); type_add_flags(type, Py_TPFLAGS_IMMUTABLETYPE); + types_start_world(); + ASSERT_TYPE_LOCK_HELD(); type_modified_unlocked(type); END_TYPE_LOCK(); diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 6c199a52aa0..8e43962c7e3 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -53,6 +53,7 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *qualname; PyObject *type_params; PyObject *compute_value; PyObject *value; @@ -192,7 +193,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(value); i++) { PyObject *item = PyTuple_GET_ITEM(value, i); if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { PyUnicodeWriter_Discard(writer); return NULL; } @@ -250,7 +251,7 @@ static PyType_Slot constevaluator_slots[] = { {0, NULL}, }; -PyType_Spec constevaluator_spec = { +static PyType_Spec constevaluator_spec = { .name = "_typing._ConstEvaluator", .basicsize = sizeof(constevaluatorobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -273,7 +274,7 @@ _Py_typing_type_repr(PyUnicodeWriter *writer, PyObject *p) } if (p == (PyObject *)&_PyNone_Type) { - return PyUnicodeWriter_WriteUTF8(writer, "None", 4); + return PyUnicodeWriter_WriteASCII(writer, "None", 4); } if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && @@ -472,7 +473,7 @@ typevar_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); - Py_DECREF(tv->name); + Py_XDECREF(tv->name); Py_XDECREF(tv->bound); Py_XDECREF(tv->evaluate_bound); Py_XDECREF(tv->constraints); @@ -491,6 +492,7 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); typevarobject *tv = typevarobject_CAST(self); + Py_VISIT(tv->name); Py_VISIT(tv->bound); Py_VISIT(tv->evaluate_bound); Py_VISIT(tv->constraints); @@ -505,6 +507,7 @@ static int typevar_clear(PyObject *op) { typevarobject *self = typevarobject_CAST(op); + Py_CLEAR(self->name); Py_CLEAR(self->bound); Py_CLEAR(self->evaluate_bound); Py_CLEAR(self->constraints); @@ -927,7 +930,7 @@ static PyType_Slot typevar_slots[] = { {0, NULL}, }; -PyType_Spec typevar_spec = { +static PyType_Spec typevar_spec = { .name = "typing.TypeVar", .basicsize = sizeof(typevarobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1075,7 +1078,7 @@ static PyType_Slot paramspecargs_slots[] = { {0, NULL}, }; -PyType_Spec paramspecargs_spec = { +static PyType_Spec paramspecargs_spec = { .name = "typing.ParamSpecArgs", .basicsize = sizeof(paramspecattrobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1155,7 +1158,7 @@ static PyType_Slot paramspeckwargs_slots[] = { {0, NULL}, }; -PyType_Spec paramspeckwargs_spec = { +static PyType_Spec paramspeckwargs_spec = { .name = "typing.ParamSpecKwargs", .basicsize = sizeof(paramspecattrobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1171,7 +1174,7 @@ paramspec_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); - Py_DECREF(ps->name); + Py_XDECREF(ps->name); Py_XDECREF(ps->bound); Py_XDECREF(ps->default_value); Py_XDECREF(ps->evaluate_default); @@ -1187,6 +1190,7 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); paramspecobject *ps = paramspecobject_CAST(self); + Py_VISIT(ps->name); Py_VISIT(ps->bound); Py_VISIT(ps->default_value); Py_VISIT(ps->evaluate_default); @@ -1198,6 +1202,7 @@ static int paramspec_clear(PyObject *op) { paramspecobject *self = paramspecobject_CAST(op); + Py_CLEAR(self->name); Py_CLEAR(self->bound); Py_CLEAR(self->default_value); Py_CLEAR(self->evaluate_default); @@ -1504,7 +1509,7 @@ static PyType_Slot paramspec_slots[] = { {0, 0}, }; -PyType_Spec paramspec_spec = { +static PyType_Spec paramspec_spec = { .name = "typing.ParamSpec", .basicsize = sizeof(paramspecobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE @@ -1519,7 +1524,7 @@ typevartuple_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); typevartupleobject *tvt = typevartupleobject_CAST(self); - Py_DECREF(tvt->name); + Py_XDECREF(tvt->name); Py_XDECREF(tvt->default_value); Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1683,6 +1688,7 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); typevartupleobject *tvt = typevartupleobject_CAST(self); + Py_VISIT(tvt->name); Py_VISIT(tvt->default_value); Py_VISIT(tvt->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); @@ -1693,6 +1699,7 @@ static int typevartuple_clear(PyObject *self) { typevartupleobject *tvt = typevartupleobject_CAST(self); + Py_CLEAR(tvt->name); Py_CLEAR(tvt->default_value); Py_CLEAR(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1782,7 +1789,7 @@ Note that only TypeVarTuples defined in the global scope can be\n\ pickled.\n\ "); -PyType_Slot typevartuple_slots[] = { +static PyType_Slot typevartuple_slots[] = { {Py_tp_doc, (void *)typevartuple_doc}, {Py_tp_members, typevartuple_members}, {Py_tp_methods, typevartuple_methods}, @@ -1798,7 +1805,7 @@ PyType_Slot typevartuple_slots[] = { {0, 0}, }; -PyType_Spec typevartuple_spec = { +static PyType_Spec typevartuple_spec = { .name = "typing.TypeVarTuple", .basicsize = sizeof(typevartupleobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_DICT @@ -1851,7 +1858,8 @@ typealias_dealloc(PyObject *self) PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); typealiasobject *ta = typealiasobject_CAST(self); - Py_DECREF(ta->name); + Py_XDECREF(ta->name); + Py_XDECREF(ta->qualname); Py_XDECREF(ta->type_params); Py_XDECREF(ta->compute_value); Py_XDECREF(ta->value); @@ -1878,11 +1886,12 @@ static PyObject * typealias_repr(PyObject *self) { typealiasobject *ta = (typealiasobject *)self; - return Py_NewRef(ta->name); + return Py_NewRef(ta->qualname); } static PyMemberDef typealias_members[] = { {"__name__", _Py_T_OBJECT, offsetof(typealiasobject, name), Py_READONLY}, + {"__qualname__", _Py_T_OBJECT, offsetof(typealiasobject, qualname), Py_READONLY}, {0} }; @@ -1997,7 +2006,7 @@ typealias_check_type_params(PyObject *type_params, int *err) { } static PyObject * -typelias_convert_type_params(PyObject *type_params) +typealias_convert_type_params(PyObject *type_params) { if ( type_params == NULL @@ -2012,14 +2021,15 @@ typelias_convert_type_params(PyObject *type_params) } static typealiasobject * -typealias_alloc(PyObject *name, PyObject *type_params, PyObject *compute_value, - PyObject *value, PyObject *module) +typealias_alloc(PyObject *name, PyObject *qualname, PyObject *type_params, + PyObject *compute_value, PyObject *value, PyObject *module) { typealiasobject *ta = PyObject_GC_New(typealiasobject, &_PyTypeAlias_Type); if (ta == NULL) { return NULL; } ta->name = Py_NewRef(name); + ta->qualname = Py_NewRef(qualname); ta->type_params = Py_XNewRef(type_params); ta->compute_value = Py_XNewRef(compute_value); ta->value = Py_XNewRef(value); @@ -2032,6 +2042,8 @@ static int typealias_traverse(PyObject *op, visitproc visit, void *arg) { typealiasobject *self = typealiasobject_CAST(op); + Py_VISIT(self->name); + Py_VISIT(self->qualname); Py_VISIT(self->type_params); Py_VISIT(self->compute_value); Py_VISIT(self->value); @@ -2043,6 +2055,8 @@ static int typealias_clear(PyObject *op) { typealiasobject *self = typealiasobject_CAST(op); + Py_CLEAR(self->name); + Py_CLEAR(self->qualname); Py_CLEAR(self->type_params); Py_CLEAR(self->compute_value); Py_CLEAR(self->value); @@ -2088,14 +2102,15 @@ typealias.__new__ as typealias_new value: object * type_params: object = NULL + qualname: object(c_default="NULL") = None Create a TypeAliasType. [clinic start generated code]*/ static PyObject * typealias_new_impl(PyTypeObject *type, PyObject *name, PyObject *value, - PyObject *type_params) -/*[clinic end generated code: output=8920ce6bdff86f00 input=df163c34e17e1a35]*/ + PyObject *type_params, PyObject *qualname) +/*[clinic end generated code: output=b7f6d9f1c577cd9c input=cbec290f8c4886ef]*/ { if (type_params != NULL && !PyTuple_Check(type_params)) { PyErr_SetString(PyExc_TypeError, "type_params must be a tuple"); @@ -2108,12 +2123,23 @@ typealias_new_impl(PyTypeObject *type, PyObject *name, PyObject *value, return NULL; } + if (qualname == NULL || qualname == Py_None) { + // If qualname was not set directly, we use name instead. + qualname = name; + } else { + if (!PyUnicode_Check(qualname)) { + PyErr_SetString(PyExc_TypeError, "qualname must be a string"); + return NULL; + } + } + PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *ta = (PyObject *)typealias_alloc(name, checked_params, NULL, value, - module); + + PyObject *ta = (PyObject *)typealias_alloc( + name, qualname, checked_params, NULL, value, module); Py_DECREF(module); return ta; } @@ -2179,10 +2205,17 @@ _Py_make_typealias(PyThreadState* unused, PyObject *args) assert(PyTuple_GET_SIZE(args) == 3); PyObject *name = PyTuple_GET_ITEM(args, 0); assert(PyUnicode_Check(name)); - PyObject *type_params = typelias_convert_type_params(PyTuple_GET_ITEM(args, 1)); + PyObject *type_params = typealias_convert_type_params(PyTuple_GET_ITEM(args, 1)); PyObject *compute_value = PyTuple_GET_ITEM(args, 2); assert(PyFunction_Check(compute_value)); - return (PyObject *)typealias_alloc(name, type_params, compute_value, NULL, NULL); + + PyFunctionObject *compute_func = (PyFunctionObject *)compute_value; + PyCodeObject *code_obj = (PyCodeObject *)compute_func->func_code; + PyObject *qualname = code_obj->co_qualname; + assert(qualname != NULL); + + return (PyObject *)typealias_alloc( + name, qualname, type_params, compute_value, NULL, NULL); } PyDoc_STRVAR(generic_doc, @@ -2304,24 +2337,17 @@ generic_dealloc(PyObject *self) Py_DECREF(tp); } -static int -generic_traverse(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static PyType_Slot generic_slots[] = { {Py_tp_doc, (void *)generic_doc}, {Py_tp_methods, generic_methods}, {Py_tp_dealloc, generic_dealloc}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_free, PyObject_GC_Del}, - {Py_tp_traverse, generic_traverse}, + {Py_tp_traverse, _PyObject_VisitType}, {0, NULL}, }; -PyType_Spec generic_spec = { +static PyType_Spec generic_spec = { .name = "typing.Generic", .basicsize = sizeof(PyObject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, diff --git a/Objects/unicode_format.c b/Objects/unicode_format.c new file mode 100644 index 00000000000..26bdae55d8b --- /dev/null +++ b/Objects/unicode_format.c @@ -0,0 +1,1002 @@ +/* + +Unicode implementation based on original code by Fredrik Lundh, +modified by Marc-Andre Lemburg <mal@lemburg.com>. + +Major speed upgrades to the method implementations at the Reykjavik +NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke. + +Copyright (c) Corporation for National Research Initiatives. + +-------------------------------------------------------------------- +The original string type implementation is: + + Copyright (c) 1999 by Secret Labs AB + Copyright (c) 1999 by Fredrik Lundh + +By obtaining, using, and/or copying this software and/or its +associated documentation, you agree that you have read, understood, +and will comply with the following terms and conditions: + +Permission to use, copy, modify, and distribute this software and its +associated documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appears in all +copies, and that both that copyright notice and this permission notice +appear in supporting documentation, and that the name of Secret Labs +AB or the author not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR +ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +-------------------------------------------------------------------- + +*/ + +// PyUnicode_Format() implementation + +#include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_format.h" // F_ALT +#include "pycore_long.h" // _PyLong_FormatWriter() +#include "pycore_object.h" // _PyObject_IsUniquelyReferenced() +#include "pycore_unicodeobject.h" // _Py_MAX_UNICODE + + +#define MAX_UNICODE _Py_MAX_UNICODE +#define ensure_unicode _PyUnicode_EnsureUnicode + +struct unicode_formatter_t { + PyObject *args; + int args_owned; + Py_ssize_t arglen, argidx; + PyObject *dict; + + int fmtkind; + Py_ssize_t fmtcnt, fmtpos; + const void *fmtdata; + PyObject *fmtstr; + + _PyUnicodeWriter writer; +}; + + +struct unicode_format_arg_t { + Py_UCS4 ch; + int flags; + Py_ssize_t width; + int prec; + int sign; +}; + + +static PyObject * +unicode_format_getnextarg(struct unicode_formatter_t *ctx) +{ + Py_ssize_t argidx = ctx->argidx; + + if (argidx < ctx->arglen) { + ctx->argidx++; + if (ctx->arglen < 0) + return ctx->args; + else + return PyTuple_GetItem(ctx->args, argidx); + } + PyErr_SetString(PyExc_TypeError, + "not enough arguments for format string"); + return NULL; +} + + +/* Returns a new reference to a PyUnicode object, or NULL on failure. */ + +/* Format a float into the writer if the writer is not NULL, or into *p_output + otherwise. + + Return 0 on success, raise an exception and return -1 on error. */ +static int +formatfloat(PyObject *v, struct unicode_format_arg_t *arg, + PyObject **p_output, + _PyUnicodeWriter *writer) +{ + char *p; + double x; + Py_ssize_t len; + int prec; + int dtoa_flags = 0; + + x = PyFloat_AsDouble(v); + if (x == -1.0 && PyErr_Occurred()) + return -1; + + prec = arg->prec; + if (prec < 0) + prec = 6; + + if (arg->flags & F_ALT) + dtoa_flags |= Py_DTSF_ALT; + p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL); + if (p == NULL) + return -1; + len = strlen(p); + if (writer) { + if (_PyUnicodeWriter_WriteASCIIString(writer, p, len) < 0) { + PyMem_Free(p); + return -1; + } + } + else + *p_output = _PyUnicode_FromASCII(p, len); + PyMem_Free(p); + return 0; +} + + +/* formatlong() emulates the format codes d, u, o, x and X, and + * the F_ALT flag, for Python's long (unbounded) ints. It's not used for + * Python's regular ints. + * Return value: a new PyUnicodeObject*, or NULL if error. + * The output string is of the form + * "-"? ("0x" | "0X")? digit+ + * "0x"/"0X" are present only for x and X conversions, with F_ALT + * set in flags. The case of hex digits will be correct, + * There will be at least prec digits, zero-filled on the left if + * necessary to get that many. + * val object to be converted + * flags bitmask of format flags; only F_ALT is looked at + * prec minimum number of digits; 0-fill on left if needed + * type a character in [duoxX]; u acts the same as d + * + * CAUTION: o, x and X conversions on regular ints can never + * produce a '-' sign, but can for Python's unbounded ints. + */ +PyObject * +_PyUnicode_FormatLong(PyObject *val, int alt, int prec, int type) +{ + PyObject *result = NULL; + char *buf; + Py_ssize_t i; + int sign; /* 1 if '-', else 0 */ + int len; /* number of characters */ + Py_ssize_t llen; + int numdigits; /* len == numnondigits + numdigits */ + int numnondigits = 0; + + /* Avoid exceeding SSIZE_T_MAX */ + if (prec > INT_MAX-3) { + PyErr_SetString(PyExc_OverflowError, + "precision too large"); + return NULL; + } + + assert(PyLong_Check(val)); + + switch (type) { + default: + Py_UNREACHABLE(); + case 'd': + case 'i': + case 'u': + /* int and int subclasses should print numerically when a numeric */ + /* format code is used (see issue18780) */ + result = PyNumber_ToBase(val, 10); + break; + case 'o': + numnondigits = 2; + result = PyNumber_ToBase(val, 8); + break; + case 'x': + case 'X': + numnondigits = 2; + result = PyNumber_ToBase(val, 16); + break; + } + if (!result) + return NULL; + + assert(_PyUnicode_IsModifiable(result)); + assert(PyUnicode_IS_ASCII(result)); + + /* To modify the string in-place, there can only be one reference. */ + if (!_PyObject_IsUniquelyReferenced(result)) { + Py_DECREF(result); + PyErr_BadInternalCall(); + return NULL; + } + buf = PyUnicode_DATA(result); + llen = PyUnicode_GET_LENGTH(result); + if (llen > INT_MAX) { + Py_DECREF(result); + PyErr_SetString(PyExc_ValueError, + "string too large in _PyUnicode_FormatLong"); + return NULL; + } + len = (int)llen; + sign = buf[0] == '-'; + numnondigits += sign; + numdigits = len - numnondigits; + assert(numdigits > 0); + + /* Get rid of base marker unless F_ALT */ + if (((alt) == 0 && + (type == 'o' || type == 'x' || type == 'X'))) { + assert(buf[sign] == '0'); + assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || + buf[sign+1] == 'o'); + numnondigits -= 2; + buf += 2; + len -= 2; + if (sign) + buf[0] = '-'; + assert(len == numnondigits + numdigits); + assert(numdigits > 0); + } + + /* Fill with leading zeroes to meet minimum width. */ + if (prec > numdigits) { + PyObject *r1 = PyBytes_FromStringAndSize(NULL, + numnondigits + prec); + char *b1; + if (!r1) { + Py_DECREF(result); + return NULL; + } + b1 = PyBytes_AS_STRING(r1); + for (i = 0; i < numnondigits; ++i) + *b1++ = *buf++; + for (i = 0; i < prec - numdigits; i++) + *b1++ = '0'; + for (i = 0; i < numdigits; i++) + *b1++ = *buf++; + *b1 = '\0'; + Py_SETREF(result, r1); + buf = PyBytes_AS_STRING(result); + len = numnondigits + prec; + } + + /* Fix up case for hex conversions. */ + if (type == 'X') { + /* Need to convert all lower case letters to upper case. + and need to convert 0x to 0X (and -0x to -0X). */ + for (i = 0; i < len; i++) + if (buf[i] >= 'a' && buf[i] <= 'x') + buf[i] -= 'a'-'A'; + } + if (!PyUnicode_Check(result) + || buf != PyUnicode_DATA(result)) { + PyObject *unicode; + unicode = _PyUnicode_FromASCII(buf, len); + Py_SETREF(result, unicode); + } + else if (len != PyUnicode_GET_LENGTH(result)) { + if (PyUnicode_Resize(&result, len) < 0) + Py_CLEAR(result); + } + return result; +} + + +/* Format an integer or a float as an integer. + * Return 1 if the number has been formatted into the writer, + * 0 if the number has been formatted into *p_output + * -1 and raise an exception on error */ +static int +mainformatlong(PyObject *v, + struct unicode_format_arg_t *arg, + PyObject **p_output, + _PyUnicodeWriter *writer) +{ + PyObject *iobj, *res; + char type = (char)arg->ch; + + if (!PyNumber_Check(v)) + goto wrongtype; + + /* make sure number is a type of integer for o, x, and X */ + if (!PyLong_Check(v)) { + if (type == 'o' || type == 'x' || type == 'X') { + iobj = _PyNumber_Index(v); + } + else { + iobj = PyNumber_Long(v); + } + if (iobj == NULL ) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) + goto wrongtype; + return -1; + } + assert(PyLong_Check(iobj)); + } + else { + iobj = Py_NewRef(v); + } + + if (PyLong_CheckExact(v) + && arg->width == -1 && arg->prec == -1 + && !(arg->flags & (F_SIGN | F_BLANK)) + && type != 'X') + { + /* Fast path */ + int alternate = arg->flags & F_ALT; + int base; + + switch(type) + { + default: + Py_UNREACHABLE(); + case 'd': + case 'i': + case 'u': + base = 10; + break; + case 'o': + base = 8; + break; + case 'x': + case 'X': + base = 16; + break; + } + + if (_PyLong_FormatWriter(writer, v, base, alternate) == -1) { + Py_DECREF(iobj); + return -1; + } + Py_DECREF(iobj); + return 1; + } + + res = _PyUnicode_FormatLong(iobj, arg->flags & F_ALT, arg->prec, type); + Py_DECREF(iobj); + if (res == NULL) + return -1; + *p_output = res; + return 0; + +wrongtype: + switch(type) + { + case 'o': + case 'x': + case 'X': + PyErr_Format(PyExc_TypeError, + "%%%c format: an integer is required, " + "not %.200s", + type, Py_TYPE(v)->tp_name); + break; + default: + PyErr_Format(PyExc_TypeError, + "%%%c format: a real number is required, " + "not %.200s", + type, Py_TYPE(v)->tp_name); + break; + } + return -1; +} + + +static Py_UCS4 +formatchar(PyObject *v) +{ + /* presume that the buffer is at least 3 characters long */ + if (PyUnicode_Check(v)) { + if (PyUnicode_GET_LENGTH(v) == 1) { + return PyUnicode_READ_CHAR(v, 0); + } + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(v)); + return (Py_UCS4) -1; + } + else { + int overflow; + long x = PyLong_AsLongAndOverflow(v, &overflow); + if (x == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Format(PyExc_TypeError, + "%%c requires an int or a unicode character, not %T", + v); + return (Py_UCS4) -1; + } + return (Py_UCS4) -1; + } + + if (x < 0 || x > MAX_UNICODE) { + /* this includes an overflow in converting to C long */ + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x110000)"); + return (Py_UCS4) -1; + } + + return (Py_UCS4) x; + } +} + + +/* Parse options of an argument: flags, width, precision. + Handle also "%(name)" syntax. + + Return 0 if the argument has been formatted into arg->str. + Return 1 if the argument has been written into ctx->writer, + Raise an exception and return -1 on error. */ +static int +unicode_format_arg_parse(struct unicode_formatter_t *ctx, + struct unicode_format_arg_t *arg) +{ +#define FORMAT_READ(ctx) \ + PyUnicode_READ((ctx)->fmtkind, (ctx)->fmtdata, (ctx)->fmtpos) + + PyObject *v; + + if (arg->ch == '(') { + /* Get argument value from a dictionary. Example: "%(name)s". */ + Py_ssize_t keystart; + Py_ssize_t keylen; + PyObject *key; + int pcount = 1; + + if (ctx->dict == NULL) { + PyErr_SetString(PyExc_TypeError, + "format requires a mapping"); + return -1; + } + ++ctx->fmtpos; + --ctx->fmtcnt; + keystart = ctx->fmtpos; + /* Skip over balanced parentheses */ + while (pcount > 0 && --ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + if (arg->ch == ')') + --pcount; + else if (arg->ch == '(') + ++pcount; + ctx->fmtpos++; + } + keylen = ctx->fmtpos - keystart - 1; + if (ctx->fmtcnt < 0 || pcount > 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format key"); + return -1; + } + key = PyUnicode_Substring(ctx->fmtstr, + keystart, keystart + keylen); + if (key == NULL) + return -1; + if (ctx->args_owned) { + ctx->args_owned = 0; + Py_DECREF(ctx->args); + } + ctx->args = PyObject_GetItem(ctx->dict, key); + Py_DECREF(key); + if (ctx->args == NULL) + return -1; + ctx->args_owned = 1; + ctx->arglen = -1; + ctx->argidx = -2; + } + + /* Parse flags. Example: "%+i" => flags=F_SIGN. */ + while (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + switch (arg->ch) { + case '-': arg->flags |= F_LJUST; continue; + case '+': arg->flags |= F_SIGN; continue; + case ' ': arg->flags |= F_BLANK; continue; + case '#': arg->flags |= F_ALT; continue; + case '0': arg->flags |= F_ZERO; continue; + } + break; + } + + /* Parse width. Example: "%10s" => width=10 */ + if (arg->ch == '*') { + v = unicode_format_getnextarg(ctx); + if (v == NULL) + return -1; + if (!PyLong_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "* wants int"); + return -1; + } + arg->width = PyLong_AsSsize_t(v); + if (arg->width == -1 && PyErr_Occurred()) + return -1; + if (arg->width < 0) { + arg->flags |= F_LJUST; + arg->width = -arg->width; + } + if (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + } + } + else if (arg->ch >= '0' && arg->ch <= '9') { + arg->width = arg->ch - '0'; + while (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + if (arg->ch < '0' || arg->ch > '9') + break; + /* Since arg->ch is unsigned, the RHS would end up as unsigned, + mixing signed and unsigned comparison. Since arg->ch is between + '0' and '9', casting to int is safe. */ + if (arg->width > (PY_SSIZE_T_MAX - ((int)arg->ch - '0')) / 10) { + PyErr_SetString(PyExc_ValueError, + "width too big"); + return -1; + } + arg->width = arg->width*10 + (arg->ch - '0'); + } + } + + /* Parse precision. Example: "%.3f" => prec=3 */ + if (arg->ch == '.') { + arg->prec = 0; + if (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + } + if (arg->ch == '*') { + v = unicode_format_getnextarg(ctx); + if (v == NULL) + return -1; + if (!PyLong_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "* wants int"); + return -1; + } + arg->prec = PyLong_AsInt(v); + if (arg->prec == -1 && PyErr_Occurred()) + return -1; + if (arg->prec < 0) + arg->prec = 0; + if (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + } + } + else if (arg->ch >= '0' && arg->ch <= '9') { + arg->prec = arg->ch - '0'; + while (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + if (arg->ch < '0' || arg->ch > '9') + break; + if (arg->prec > (INT_MAX - ((int)arg->ch - '0')) / 10) { + PyErr_SetString(PyExc_ValueError, + "precision too big"); + return -1; + } + arg->prec = arg->prec*10 + (arg->ch - '0'); + } + } + } + + /* Ignore "h", "l" and "L" format prefix (ex: "%hi" or "%ls") */ + if (ctx->fmtcnt >= 0) { + if (arg->ch == 'h' || arg->ch == 'l' || arg->ch == 'L') { + if (--ctx->fmtcnt >= 0) { + arg->ch = FORMAT_READ(ctx); + ctx->fmtpos++; + } + } + } + if (ctx->fmtcnt < 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format"); + return -1; + } + return 0; + +#undef FORMAT_READ +} + + +/* Format one argument. Supported conversion specifiers: + + - "s", "r", "a": any type + - "i", "d", "u": int or float + - "o", "x", "X": int + - "e", "E", "f", "F", "g", "G": float + - "c": int or str (1 character) + + When possible, the output is written directly into the Unicode writer + (ctx->writer). A string is created when padding is required. + + Return 0 if the argument has been formatted into *p_str, + 1 if the argument has been written into ctx->writer, + -1 on error. */ +static int +unicode_format_arg_format(struct unicode_formatter_t *ctx, + struct unicode_format_arg_t *arg, + PyObject **p_str) +{ + PyObject *v; + _PyUnicodeWriter *writer = &ctx->writer; + + if (ctx->fmtcnt == 0) + ctx->writer.overallocate = 0; + + v = unicode_format_getnextarg(ctx); + if (v == NULL) + return -1; + + + switch (arg->ch) { + case 's': + case 'r': + case 'a': + if (PyLong_CheckExact(v) && arg->width == -1 && arg->prec == -1) { + /* Fast path */ + if (_PyLong_FormatWriter(writer, v, 10, arg->flags & F_ALT) == -1) + return -1; + return 1; + } + + if (PyUnicode_CheckExact(v) && arg->ch == 's') { + *p_str = Py_NewRef(v); + } + else { + if (arg->ch == 's') + *p_str = PyObject_Str(v); + else if (arg->ch == 'r') + *p_str = PyObject_Repr(v); + else + *p_str = PyObject_ASCII(v); + } + break; + + case 'i': + case 'd': + case 'u': + case 'o': + case 'x': + case 'X': + { + int ret = mainformatlong(v, arg, p_str, writer); + if (ret != 0) + return ret; + arg->sign = 1; + break; + } + + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + if (arg->width == -1 && arg->prec == -1 + && !(arg->flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (formatfloat(v, arg, NULL, writer) == -1) + return -1; + return 1; + } + + arg->sign = 1; + if (formatfloat(v, arg, p_str, NULL) == -1) + return -1; + break; + + case 'c': + { + Py_UCS4 ch = formatchar(v); + if (ch == (Py_UCS4) -1) + return -1; + if (arg->width == -1 && arg->prec == -1) { + /* Fast path */ + if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) + return -1; + return 1; + } + *p_str = PyUnicode_FromOrdinal(ch); + break; + } + + default: + PyErr_Format(PyExc_ValueError, + "unsupported format character '%c' (0x%x) " + "at index %zd", + (31<=arg->ch && arg->ch<=126) ? (char)arg->ch : '?', + (int)arg->ch, + ctx->fmtpos - 1); + return -1; + } + if (*p_str == NULL) + return -1; + assert (PyUnicode_Check(*p_str)); + return 0; +} + + +static int +unicode_format_arg_output(struct unicode_formatter_t *ctx, + struct unicode_format_arg_t *arg, + PyObject *str) +{ + Py_ssize_t len; + int kind; + const void *pbuf; + Py_ssize_t pindex; + Py_UCS4 signchar; + Py_ssize_t buflen; + Py_UCS4 maxchar; + Py_ssize_t sublen; + _PyUnicodeWriter *writer = &ctx->writer; + Py_UCS4 fill; + + fill = ' '; + if (arg->sign && arg->flags & F_ZERO) + fill = '0'; + + len = PyUnicode_GET_LENGTH(str); + if ((arg->width == -1 || arg->width <= len) + && (arg->prec == -1 || arg->prec >= len) + && !(arg->flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (_PyUnicodeWriter_WriteStr(writer, str) == -1) + return -1; + return 0; + } + + /* Truncate the string for "s", "r" and "a" formats + if the precision is set */ + if (arg->ch == 's' || arg->ch == 'r' || arg->ch == 'a') { + if (arg->prec >= 0 && len > arg->prec) + len = arg->prec; + } + + /* Adjust sign and width */ + kind = PyUnicode_KIND(str); + pbuf = PyUnicode_DATA(str); + pindex = 0; + signchar = '\0'; + if (arg->sign) { + Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex); + if (ch == '-' || ch == '+') { + signchar = ch; + len--; + pindex++; + } + else if (arg->flags & F_SIGN) + signchar = '+'; + else if (arg->flags & F_BLANK) + signchar = ' '; + else + arg->sign = 0; + } + if (arg->width < len) + arg->width = len; + + /* Prepare the writer */ + maxchar = writer->maxchar; + if (!(arg->flags & F_LJUST)) { + if (arg->sign) { + if ((arg->width-1) > len) + maxchar = Py_MAX(maxchar, fill); + } + else { + if (arg->width > len) + maxchar = Py_MAX(maxchar, fill); + } + } + if (PyUnicode_MAX_CHAR_VALUE(str) > maxchar) { + Py_UCS4 strmaxchar = _PyUnicode_FindMaxChar(str, 0, pindex+len); + maxchar = Py_MAX(maxchar, strmaxchar); + } + + buflen = arg->width; + if (arg->sign && len == arg->width) + buflen++; + if (_PyUnicodeWriter_Prepare(writer, buflen, maxchar) == -1) + return -1; + + /* Write the sign if needed */ + if (arg->sign) { + if (fill != ' ') { + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); + writer->pos += 1; + } + if (arg->width > len) + arg->width--; + } + + /* Write the numeric prefix for "x", "X" and "o" formats + if the alternate form is used. + For example, write "0x" for the "%#x" format. */ + if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { + assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); + assert(PyUnicode_READ(kind, pbuf, pindex + 1) == arg->ch); + if (fill != ' ') { + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); + PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); + writer->pos += 2; + pindex += 2; + } + arg->width -= 2; + if (arg->width < 0) + arg->width = 0; + len -= 2; + } + + /* Pad left with the fill character if needed */ + if (arg->width > len && !(arg->flags & F_LJUST)) { + sublen = arg->width - len; + _PyUnicode_Fill(writer->kind, writer->data, fill, writer->pos, sublen); + writer->pos += sublen; + arg->width = len; + } + + /* If padding with spaces: write sign if needed and/or numeric prefix if + the alternate form is used */ + if (fill == ' ') { + if (arg->sign) { + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); + writer->pos += 1; + } + if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { + assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); + assert(PyUnicode_READ(kind, pbuf, pindex+1) == arg->ch); + PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); + PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); + writer->pos += 2; + pindex += 2; + } + } + + /* Write characters */ + if (len) { + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + str, pindex, len); + writer->pos += len; + } + + /* Pad right with the fill character if needed */ + if (arg->width > len) { + sublen = arg->width - len; + _PyUnicode_Fill(writer->kind, writer->data, ' ', writer->pos, sublen); + writer->pos += sublen; + } + return 0; +} + + +/* Helper of PyUnicode_Format(): format one arg. + Return 0 on success, raise an exception and return -1 on error. */ +static int +unicode_format_arg(struct unicode_formatter_t *ctx) +{ + struct unicode_format_arg_t arg; + PyObject *str; + int ret; + + arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos); + if (arg.ch == '%') { + ctx->fmtpos++; + ctx->fmtcnt--; + if (_PyUnicodeWriter_WriteCharInline(&ctx->writer, '%') < 0) + return -1; + return 0; + } + arg.flags = 0; + arg.width = -1; + arg.prec = -1; + arg.sign = 0; + str = NULL; + + ret = unicode_format_arg_parse(ctx, &arg); + if (ret == -1) + return -1; + + ret = unicode_format_arg_format(ctx, &arg, &str); + if (ret == -1) + return -1; + + if (ret != 1) { + ret = unicode_format_arg_output(ctx, &arg, str); + Py_DECREF(str); + if (ret == -1) + return -1; + } + + if (ctx->dict && (ctx->argidx < ctx->arglen)) { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + return -1; + } + return 0; +} + + +PyObject * +PyUnicode_Format(PyObject *format, PyObject *args) +{ + struct unicode_formatter_t ctx; + + if (format == NULL || args == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + if (ensure_unicode(format) < 0) + return NULL; + + ctx.fmtstr = format; + ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr); + ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr); + ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr); + ctx.fmtpos = 0; + + _PyUnicodeWriter_Init(&ctx.writer); + ctx.writer.min_length = ctx.fmtcnt + 100; + ctx.writer.overallocate = 1; + + if (PyTuple_Check(args)) { + ctx.arglen = PyTuple_Size(args); + ctx.argidx = 0; + } + else { + ctx.arglen = -1; + ctx.argidx = -2; + } + ctx.args_owned = 0; + if (PyMapping_Check(args) && !PyTuple_Check(args) && !PyUnicode_Check(args)) + ctx.dict = args; + else + ctx.dict = NULL; + ctx.args = args; + + while (--ctx.fmtcnt >= 0) { + if (PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { + Py_ssize_t nonfmtpos; + + nonfmtpos = ctx.fmtpos++; + while (ctx.fmtcnt >= 0 && + PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { + ctx.fmtpos++; + ctx.fmtcnt--; + } + if (ctx.fmtcnt < 0) { + ctx.fmtpos--; + ctx.writer.overallocate = 0; + } + + if (_PyUnicodeWriter_WriteSubstring(&ctx.writer, ctx.fmtstr, + nonfmtpos, ctx.fmtpos) < 0) + goto onError; + } + else { + ctx.fmtpos++; + if (unicode_format_arg(&ctx) == -1) + goto onError; + } + } + + if (ctx.argidx < ctx.arglen && !ctx.dict) { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + goto onError; + } + + if (ctx.args_owned) { + Py_DECREF(ctx.args); + } + return _PyUnicodeWriter_Finish(&ctx.writer); + + onError: + _PyUnicodeWriter_Dealloc(&ctx.writer); + if (ctx.args_owned) { + Py_DECREF(ctx.args); + } + return NULL; +} diff --git a/Python/formatter_unicode.c b/Objects/unicode_formatter.c similarity index 88% rename from Python/formatter_unicode.c rename to Objects/unicode_formatter.c index 30807f428c7..b8604d13559 100644 --- a/Python/formatter_unicode.c +++ b/Objects/unicode_formatter.c @@ -8,6 +8,241 @@ #include "pycore_unicodeobject.h" // PyUnicode_MAX_CHAR_VALUE() #include <locale.h> + +/* _PyUnicode_InsertThousandsGrouping() helper functions */ + +typedef struct { + const char *grouping; + char previous; + Py_ssize_t i; /* Where we're currently pointing in grouping. */ +} GroupGenerator; + + +static void +GroupGenerator_init(GroupGenerator *self, const char *grouping) +{ + self->grouping = grouping; + self->i = 0; + self->previous = 0; +} + + +/* Returns the next grouping, or 0 to signify end. */ +static Py_ssize_t +GroupGenerator_next(GroupGenerator *self) +{ + /* Note that we don't really do much error checking here. If a + grouping string contains just CHAR_MAX, for example, then just + terminate the generator. That shouldn't happen, but at least we + fail gracefully. */ + switch (self->grouping[self->i]) { + case 0: + return self->previous; + case CHAR_MAX: + /* Stop the generator. */ + return 0; + default: { + char ch = self->grouping[self->i]; + self->previous = ch; + self->i++; + return (Py_ssize_t)ch; + } + } +} + + +/* Fill in some digits, leading zeros, and thousands separator. All + are optional, depending on when we're called. */ +static void +InsertThousandsGrouping_fill(_PyUnicodeWriter *writer, Py_ssize_t *buffer_pos, + PyObject *digits, Py_ssize_t *digits_pos, + Py_ssize_t n_chars, Py_ssize_t n_zeros, + PyObject *thousands_sep, Py_ssize_t thousands_sep_len, + Py_UCS4 *maxchar, int forward) +{ + if (!writer) { + /* if maxchar > 127, maxchar is already set */ + if (*maxchar == 127 && thousands_sep) { + Py_UCS4 maxchar2 = PyUnicode_MAX_CHAR_VALUE(thousands_sep); + *maxchar = Py_MAX(*maxchar, maxchar2); + } + return; + } + + if (thousands_sep) { + if (!forward) { + *buffer_pos -= thousands_sep_len; + } + /* Copy the thousands_sep chars into the buffer. */ + _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, + thousands_sep, 0, + thousands_sep_len); + if (forward) { + *buffer_pos += thousands_sep_len; + } + } + + if (!forward) { + *buffer_pos -= n_chars; + *digits_pos -= n_chars; + } + _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, + digits, *digits_pos, + n_chars); + if (forward) { + *buffer_pos += n_chars; + *digits_pos += n_chars; + } + + if (n_zeros) { + if (!forward) { + *buffer_pos -= n_zeros; + } + int kind = PyUnicode_KIND(writer->buffer); + void *data = PyUnicode_DATA(writer->buffer); + _PyUnicode_Fill(kind, data, '0', *buffer_pos, n_zeros); + if (forward) { + *buffer_pos += n_zeros; + } + } +} + + +/** + * InsertThousandsGrouping: + * @writer: Unicode writer. + * @n_buffer: Number of characters in @buffer. + * @digits: Digits we're reading from. If count is non-NULL, this is unused. + * @d_pos: Start of digits string. + * @n_digits: The number of digits in the string, in which we want + * to put the grouping chars. + * @min_width: The minimum width of the digits in the output string. + * Output will be zero-padded on the left to fill. + * @grouping: see definition in localeconv(). + * @thousands_sep: see definition in localeconv(). + * + * There are 2 modes: counting and filling. If @writer is NULL, + * we are in counting mode, else filling mode. + * If counting, the required buffer size is returned. + * If filling, we know the buffer will be large enough, so we don't + * need to pass in the buffer size. + * Inserts thousand grouping characters (as defined by grouping and + * thousands_sep) into @writer. + * + * Return value: -1 on error, number of characters otherwise. + **/ +static Py_ssize_t +_PyUnicode_InsertThousandsGrouping( + _PyUnicodeWriter *writer, + Py_ssize_t n_buffer, + PyObject *digits, + Py_ssize_t d_pos, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + PyObject *thousands_sep, + Py_UCS4 *maxchar, + int forward) +{ + min_width = Py_MAX(0, min_width); + if (writer) { + assert(digits != NULL); + assert(maxchar == NULL); + } + else { + assert(digits == NULL); + assert(maxchar != NULL); + } + assert(0 <= d_pos); + assert(0 <= n_digits); + assert(grouping != NULL); + + Py_ssize_t count = 0; + Py_ssize_t n_zeros; + int loop_broken = 0; + int use_separator = 0; /* First time through, don't append the + separator. They only go between + groups. */ + Py_ssize_t buffer_pos; + Py_ssize_t digits_pos; + Py_ssize_t len; + Py_ssize_t n_chars; + Py_ssize_t remaining = n_digits; /* Number of chars remaining to + be looked at */ + /* A generator that returns all of the grouping widths, until it + returns 0. */ + GroupGenerator groupgen; + GroupGenerator_init(&groupgen, grouping); + const Py_ssize_t thousands_sep_len = PyUnicode_GET_LENGTH(thousands_sep); + + /* if digits are not grouped, thousands separator + should be an empty string */ + assert(!(grouping[0] == CHAR_MAX && thousands_sep_len != 0)); + + digits_pos = d_pos + (forward ? 0 : n_digits); + if (writer) { + buffer_pos = writer->pos + (forward ? 0 : n_buffer); + assert(buffer_pos <= PyUnicode_GET_LENGTH(writer->buffer)); + assert(digits_pos <= PyUnicode_GET_LENGTH(digits)); + } + else { + buffer_pos = forward ? 0 : n_buffer; + } + + if (!writer) { + *maxchar = 127; + } + + while ((len = GroupGenerator_next(&groupgen)) > 0) { + len = Py_MIN(len, Py_MAX(Py_MAX(remaining, min_width), 1)); + n_zeros = Py_MAX(0, len - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, len)); + + /* Use n_zero zero's and n_chars chars */ + + /* Count only, don't do anything. */ + count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; + + /* Copy into the writer. */ + InsertThousandsGrouping_fill(writer, &buffer_pos, + digits, &digits_pos, + n_chars, n_zeros, + use_separator ? thousands_sep : NULL, + thousands_sep_len, maxchar, forward); + + /* Use a separator next time. */ + use_separator = 1; + + remaining -= n_chars; + min_width -= len; + + if (remaining <= 0 && min_width <= 0) { + loop_broken = 1; + break; + } + min_width -= thousands_sep_len; + } + if (!loop_broken) { + /* We left the loop without using a break statement. */ + + len = Py_MAX(Py_MAX(remaining, min_width), 1); + n_zeros = Py_MAX(0, len - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, len)); + + /* Use n_zero zero's and n_chars chars */ + count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; + + /* Copy into the writer. */ + InsertThousandsGrouping_fill(writer, &buffer_pos, + digits, &digits_pos, + n_chars, n_zeros, + use_separator ? thousands_sep : NULL, + thousands_sep_len, maxchar, forward); + } + return count; +} + + /* Raises an exception about an unknown presentation type for this * type. */ diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c new file mode 100644 index 00000000000..2b944bf1ea8 --- /dev/null +++ b/Objects/unicode_writer.c @@ -0,0 +1,639 @@ +/* + +Unicode implementation based on original code by Fredrik Lundh, +modified by Marc-Andre Lemburg <mal@lemburg.com>. + +Major speed upgrades to the method implementations at the Reykjavik +NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke. + +Copyright (c) Corporation for National Research Initiatives. + +-------------------------------------------------------------------- +The original string type implementation is: + + Copyright (c) 1999 by Secret Labs AB + Copyright (c) 1999 by Fredrik Lundh + +By obtaining, using, and/or copying this software and/or its +associated documentation, you agree that you have read, understood, +and will comply with the following terms and conditions: + +Permission to use, copy, modify, and distribute this software and its +associated documentation for any purpose and without fee is hereby +granted, provided that the above copyright notice appears in all +copies, and that both that copyright notice and this permission notice +appear in supporting documentation, and that the name of Secret Labs +AB or the author not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR +ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +-------------------------------------------------------------------- + +*/ + +#include "Python.h" +#include "pycore_freelist.h" // _Py_FREELIST_FREE() +#include "pycore_long.h" // _PyLong_FormatWriter() +#include "pycore_unicodeobject.h" // _PyUnicode_Result() + + +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ +# define OVERALLOCATE_FACTOR 2 +#else + /* On Linux, overallocate by 25% is the best factor */ +# define OVERALLOCATE_FACTOR 4 +#endif + + +/* Compilation of templated routines */ + +#define STRINGLIB_GET_EMPTY() _PyUnicode_GetEmpty() + +#include "stringlib/ucs1lib.h" +#include "stringlib/find_max_char.h" +#include "stringlib/undef.h" + + +/* Copy an ASCII or latin1 char* string into a Python Unicode string. + + WARNING: The function doesn't copy the terminating null character and + doesn't check the maximum character (may write a latin1 character in an + ASCII string). */ +static void +unicode_write_cstr(PyObject *unicode, Py_ssize_t index, + const char *str, Py_ssize_t len) +{ + int kind = PyUnicode_KIND(unicode); + const void *data = PyUnicode_DATA(unicode); + const char *end = str + len; + + assert(index + len <= PyUnicode_GET_LENGTH(unicode)); + switch (kind) { + case PyUnicode_1BYTE_KIND: { +#ifdef Py_DEBUG + if (PyUnicode_IS_ASCII(unicode)) { + Py_UCS4 maxchar = ucs1lib_find_max_char( + (const Py_UCS1*)str, + (const Py_UCS1*)str + len); + assert(maxchar < 128); + } +#endif + memcpy((char *) data + index, str, len); + break; + } + case PyUnicode_2BYTE_KIND: { + Py_UCS2 *start = (Py_UCS2 *)data + index; + Py_UCS2 *ucs2 = start; + + for (; str < end; ++ucs2, ++str) + *ucs2 = (Py_UCS2)*str; + + assert((ucs2 - start) <= PyUnicode_GET_LENGTH(unicode)); + break; + } + case PyUnicode_4BYTE_KIND: { + Py_UCS4 *start = (Py_UCS4 *)data + index; + Py_UCS4 *ucs4 = start; + + for (; str < end; ++ucs4, ++str) + *ucs4 = (Py_UCS4)*str; + + assert((ucs4 - start) <= PyUnicode_GET_LENGTH(unicode)); + break; + } + default: + Py_UNREACHABLE(); + } +} + + +static inline void +_PyUnicodeWriter_Update(_PyUnicodeWriter *writer) +{ + writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); + writer->data = PyUnicode_DATA(writer->buffer); + + if (!writer->readonly) { + writer->kind = PyUnicode_KIND(writer->buffer); + writer->size = PyUnicode_GET_LENGTH(writer->buffer); + } + else { + /* use a value smaller than PyUnicode_1BYTE_KIND() so + _PyUnicodeWriter_PrepareKind() will copy the buffer. */ + writer->kind = 0; + assert(writer->kind <= PyUnicode_1BYTE_KIND); + + /* Copy-on-write mode: set buffer size to 0 so + * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on + * next write. */ + writer->size = 0; + } +} + + +void +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer) +{ + memset(writer, 0, sizeof(*writer)); + + /* ASCII is the bare minimum */ + writer->min_char = 127; + + /* use a kind value smaller than PyUnicode_1BYTE_KIND so + _PyUnicodeWriter_PrepareKind() will copy the buffer. */ + assert(writer->kind == 0); + assert(writer->kind < PyUnicode_1BYTE_KIND); +} + + +PyUnicodeWriter* +PyUnicodeWriter_Create(Py_ssize_t length) +{ + if (length < 0) { + PyErr_SetString(PyExc_ValueError, + "length must be positive"); + return NULL; + } + + const size_t size = sizeof(_PyUnicodeWriter); + PyUnicodeWriter *pub_writer; + pub_writer = _Py_FREELIST_POP_MEM(unicode_writers); + if (pub_writer == NULL) { + pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); + if (pub_writer == NULL) { + return (PyUnicodeWriter *)PyErr_NoMemory(); + } + } + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; + + _PyUnicodeWriter_Init(writer); + if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { + PyUnicodeWriter_Discard(pub_writer); + return NULL; + } + writer->overallocate = 1; + + return pub_writer; +} + + +void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) +{ + if (writer == NULL) { + return; + } + _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); + _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); +} + + +// Initialize _PyUnicodeWriter with initial buffer +void +_PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) +{ + memset(writer, 0, sizeof(*writer)); + writer->buffer = buffer; + _PyUnicodeWriter_Update(writer); + writer->min_length = writer->size; +} + + +int +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar) +{ + Py_ssize_t newlen; + PyObject *newbuffer; + + assert(length >= 0); + assert(maxchar <= _Py_MAX_UNICODE); + + /* ensure that the _PyUnicodeWriter_Prepare macro was used */ + assert((maxchar > writer->maxchar && length >= 0) + || length > 0); + + if (length > PY_SSIZE_T_MAX - writer->pos) { + PyErr_NoMemory(); + return -1; + } + newlen = writer->pos + length; + + maxchar = Py_MAX(maxchar, writer->min_char); + + if (writer->buffer == NULL) { + assert(!writer->readonly); + if (writer->overallocate + && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { + /* overallocate to limit the number of realloc() */ + newlen += newlen / OVERALLOCATE_FACTOR; + } + if (newlen < writer->min_length) + newlen = writer->min_length; + + writer->buffer = PyUnicode_New(newlen, maxchar); + if (writer->buffer == NULL) + return -1; + } + else if (newlen > writer->size) { + if (writer->overallocate + && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { + /* overallocate to limit the number of realloc() */ + newlen += newlen / OVERALLOCATE_FACTOR; + } + if (newlen < writer->min_length) + newlen = writer->min_length; + + if (maxchar > writer->maxchar || writer->readonly) { + /* resize + widen */ + maxchar = Py_MAX(maxchar, writer->maxchar); + newbuffer = PyUnicode_New(newlen, maxchar); + if (newbuffer == NULL) + return -1; + _PyUnicode_FastCopyCharacters(newbuffer, 0, + writer->buffer, 0, writer->pos); + Py_DECREF(writer->buffer); + writer->readonly = 0; + } + else { + newbuffer = _PyUnicode_ResizeCompact(writer->buffer, newlen); + if (newbuffer == NULL) + return -1; + } + writer->buffer = newbuffer; + } + else if (maxchar > writer->maxchar) { + assert(!writer->readonly); + newbuffer = PyUnicode_New(writer->size, maxchar); + if (newbuffer == NULL) + return -1; + _PyUnicode_FastCopyCharacters(newbuffer, 0, + writer->buffer, 0, writer->pos); + Py_SETREF(writer->buffer, newbuffer); + } + _PyUnicodeWriter_Update(writer); + return 0; + +#undef OVERALLOCATE_FACTOR +} + +int +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + int kind) +{ + Py_UCS4 maxchar; + + /* ensure that the _PyUnicodeWriter_PrepareKind macro was used */ + assert(writer->kind < kind); + + switch (kind) + { + case PyUnicode_1BYTE_KIND: maxchar = 0xff; break; + case PyUnicode_2BYTE_KIND: maxchar = 0xffff; break; + case PyUnicode_4BYTE_KIND: maxchar = _Py_MAX_UNICODE; break; + default: + Py_UNREACHABLE(); + } + + return _PyUnicodeWriter_PrepareInternal(writer, 0, maxchar); +} + + +int +_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, Py_UCS4 ch) +{ + return _PyUnicodeWriter_WriteCharInline(writer, ch); +} + + +int +PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) +{ + if (ch > _Py_MAX_UNICODE) { + PyErr_SetString(PyExc_ValueError, + "character must be in range(0x110000)"); + return -1; + } + + return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); +} + + +int +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) +{ + assert(PyUnicode_Check(str)); + + Py_UCS4 maxchar; + Py_ssize_t len; + + len = PyUnicode_GET_LENGTH(str); + if (len == 0) + return 0; + maxchar = PyUnicode_MAX_CHAR_VALUE(str); + if (maxchar > writer->maxchar || len > writer->size - writer->pos) { + if (writer->buffer == NULL && !writer->overallocate) { + assert(_PyUnicode_CheckConsistency(str, 1)); + writer->readonly = 1; + writer->buffer = Py_NewRef(str); + _PyUnicodeWriter_Update(writer); + writer->pos += len; + return 0; + } + if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1) + return -1; + } + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + str, 0, len); + writer->pos += len; + return 0; +} + + +int +PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + if (type == &PyUnicode_Type) { + return _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, obj); + } + + if (type == &PyLong_Type) { + return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + } + + PyObject *str = PyObject_Str(obj); + if (str == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); + Py_DECREF(str); + return res; +} + + +int +PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) +{ + if (Py_TYPE(obj) == &PyLong_Type) { + return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); + } + + PyObject *repr = PyObject_Repr(obj); + if (repr == NULL) { + return -1; + } + + int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, repr); + Py_DECREF(repr); + return res; +} + + +int +_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + assert(0 <= start); + assert(end <= PyUnicode_GET_LENGTH(str)); + assert(start <= end); + + if (start == 0 && end == PyUnicode_GET_LENGTH(str)) + return _PyUnicodeWriter_WriteStr(writer, str); + + Py_ssize_t len = end - start; + if (len == 0) { + return 0; + } + + Py_UCS4 maxchar; + if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) { + maxchar = _PyUnicode_FindMaxChar(str, start, end); + } + else { + maxchar = writer->maxchar; + } + if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) { + return -1; + } + + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + str, start, len); + writer->pos += len; + return 0; +} + + +int +PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, + Py_ssize_t start, Py_ssize_t end) +{ + if (!PyUnicode_Check(str)) { + PyErr_Format(PyExc_TypeError, "expect str, not %T", str); + return -1; + } + if (start < 0 || start > end) { + PyErr_Format(PyExc_ValueError, "invalid start argument"); + return -1; + } + if (end > PyUnicode_GET_LENGTH(str)) { + PyErr_Format(PyExc_ValueError, "invalid end argument"); + return -1; + } + + return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, + start, end); +} + + +int +_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, + const char *ascii, Py_ssize_t len) +{ + if (len == -1) + len = strlen(ascii); + + assert(ucs1lib_find_max_char((const Py_UCS1*)ascii, (const Py_UCS1*)ascii + len) < 128); + + if (writer->buffer == NULL && !writer->overallocate) { + PyObject *str; + + str = _PyUnicode_FromASCII(ascii, len); + if (str == NULL) + return -1; + + writer->readonly = 1; + writer->buffer = str; + _PyUnicodeWriter_Update(writer); + writer->pos += len; + return 0; + } + + if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) + return -1; + + switch (writer->kind) + { + case PyUnicode_1BYTE_KIND: + { + const Py_UCS1 *str = (const Py_UCS1 *)ascii; + Py_UCS1 *data = writer->data; + + memcpy(data + writer->pos, str, len); + break; + } + case PyUnicode_2BYTE_KIND: + { + _PyUnicode_CONVERT_BYTES( + Py_UCS1, Py_UCS2, + ascii, ascii + len, + (Py_UCS2 *)writer->data + writer->pos); + break; + } + case PyUnicode_4BYTE_KIND: + { + _PyUnicode_CONVERT_BYTES( + Py_UCS1, Py_UCS4, + ascii, ascii + len, + (Py_UCS4 *)writer->data + writer->pos); + break; + } + default: + Py_UNREACHABLE(); + } + + writer->pos += len; + return 0; +} + + +int +PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size) +{ + assert(writer != NULL); + _Py_AssertHoldsTstate(); + + _PyUnicodeWriter *priv_writer = (_PyUnicodeWriter*)writer; + return _PyUnicodeWriter_WriteASCIIString(priv_writer, str, size); +} + + +int +PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size) +{ + if (size < 0) { + size = strlen(str); + } + + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + int res = _PyUnicode_DecodeUTF8Writer(_writer, str, size, + _Py_ERROR_STRICT, NULL, NULL); + if (res < 0) { + _writer->pos = old_pos; + } + return res; +} + + +int +PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, + const char *string, + Py_ssize_t length, + const char *errors, + Py_ssize_t *consumed) +{ + if (length < 0) { + length = strlen(string); + } + + _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; + Py_ssize_t old_pos = _writer->pos; + int res = _PyUnicode_DecodeUTF8Writer(_writer, string, length, + _Py_ERROR_UNKNOWN, errors, + consumed); + if (res < 0) { + _writer->pos = old_pos; + if (consumed) { + *consumed = 0; + } + } + return res; +} + + +int +_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, + const char *str, Py_ssize_t len) +{ + Py_UCS4 maxchar; + + maxchar = ucs1lib_find_max_char((const Py_UCS1*)str, (const Py_UCS1*)str + len); + if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1) + return -1; + unicode_write_cstr(writer->buffer, writer->pos, str, len); + writer->pos += len; + return 0; +} + + +PyObject * +_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) +{ + PyObject *str; + + if (writer->pos == 0) { + Py_CLEAR(writer->buffer); + return _PyUnicode_GetEmpty(); + } + + str = writer->buffer; + writer->buffer = NULL; + + if (writer->readonly) { + assert(PyUnicode_GET_LENGTH(str) == writer->pos); + return str; + } + + if (PyUnicode_GET_LENGTH(str) != writer->pos) { + PyObject *str2; + str2 = _PyUnicode_ResizeCompact(str, writer->pos); + if (str2 == NULL) { + Py_DECREF(str); + return NULL; + } + str = str2; + } + + assert(_PyUnicode_CheckConsistency(str, 1)); + return _PyUnicode_Result(str); +} + + +PyObject* +PyUnicodeWriter_Finish(PyUnicodeWriter *writer) +{ + PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); + assert(((_PyUnicodeWriter*)writer)->buffer == NULL); + _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); + return str; +} + + +void +_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) +{ + Py_CLEAR(writer->buffer); +} diff --git a/Objects/unicodectype.c b/Objects/unicodectype.c index 7cd0dca3d13..fdd380190ac 100644 --- a/Objects/unicodectype.c +++ b/Objects/unicodectype.c @@ -9,6 +9,7 @@ */ #include "Python.h" +#include "pycore_unicodectype.h" // export _PyUnicode_IsXidStart(), _PyUnicode_IsXidContinue() #define ALPHA_MASK 0x01 #define DECIMAL_MASK 0x02 diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index eb3e1c48fd4..f737a885f19 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -46,7 +46,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_codecs.h" // _PyCodec_Lookup() #include "pycore_critical_section.h" // Py_*_CRITICAL_SECTION_SEQUENCE_FAST #include "pycore_format.h" // F_LJUST -#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec #include "pycore_long.h" // _PyLong_FormatWriter() @@ -56,9 +55,8 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_pyhash.h" // _Py_HashSecret_t #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_template.h" // _PyTemplate_Concat() -#include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +#include "pycore_unicodectype.h" // _PyUnicode_IsXidStart #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_unicodeobject_generated.h" // _PyUnicode_InitStaticStrings() @@ -105,9 +103,8 @@ NOTE: In the interpreter's initialization phase, some globals are currently */ -// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). -// The value must be the same in fileutils.c. -#define MAX_UNICODE 0x10ffff +#define MAX_UNICODE _Py_MAX_UNICODE +#define ensure_unicode _PyUnicode_EnsureUnicode #ifdef Py_DEBUG # define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0) @@ -115,14 +112,6 @@ NOTE: In the interpreter's initialization phase, some globals are currently # define _PyUnicode_CHECK(op) PyUnicode_Check(op) #endif -#ifdef Py_GIL_DISABLED -# define LOCK_INTERNED(interp) PyMutex_Lock(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex)) -# define UNLOCK_INTERNED(interp) PyMutex_Unlock(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex)) -#else -# define LOCK_INTERNED(interp) -# define UNLOCK_INTERNED(interp) -#endif - static inline char* _PyUnicode_UTF8(PyObject *op) { return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8); @@ -167,11 +156,7 @@ static inline void PyUnicode_SET_UTF8_LENGTH(PyObject *op, Py_ssize_t length) #define _PyUnicode_HASH(op) \ (_PyASCIIObject_CAST(op)->hash) -static inline Py_hash_t PyUnicode_HASH(PyObject *op) -{ - assert(_PyUnicode_CHECK(op)); - return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(op)->hash); -} +#define PyUnicode_HASH PyUnstable_Unicode_GET_CACHED_HASH static inline void PyUnicode_SET_HASH(PyObject *op, Py_hash_t hash) { @@ -198,45 +183,9 @@ static inline int _PyUnicode_HAS_UTF8_MEMORY(PyObject *op) } -/* Generic helper macro to convert characters of different types. - from_type and to_type have to be valid type names, begin and end - are pointers to the source characters which should be of type - "from_type *". to is a pointer of type "to_type *" and points to the - buffer where the result characters are written to. */ -#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ - do { \ - to_type *_to = (to_type *)(to); \ - const from_type *_iter = (const from_type *)(begin);\ - const from_type *_end = (const from_type *)(end);\ - Py_ssize_t n = (_end) - (_iter); \ - const from_type *_unrolled_end = \ - _iter + _Py_SIZE_ROUND_DOWN(n, 4); \ - while (_iter < (_unrolled_end)) { \ - _to[0] = (to_type) _iter[0]; \ - _to[1] = (to_type) _iter[1]; \ - _to[2] = (to_type) _iter[2]; \ - _to[3] = (to_type) _iter[3]; \ - _iter += 4; _to += 4; \ - } \ - while (_iter < (_end)) \ - *_to++ = (to_type) *_iter++; \ - } while (0) - #define LATIN1 _Py_LATIN1_CHR -#ifdef MS_WINDOWS - /* On Windows, overallocate by 50% is the best factor */ -# define OVERALLOCATE_FACTOR 2 -#else - /* On Linux, overallocate by 25% is the best factor */ -# define OVERALLOCATE_FACTOR 4 -#endif - /* Forward declaration */ -static inline int -_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch); -static inline void -_PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer); static PyObject * unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, const char *errors); @@ -244,11 +193,6 @@ static PyObject * unicode_decode_utf8(const char *s, Py_ssize_t size, _Py_error_handler error_handler, const char *errors, Py_ssize_t *consumed); -static int -unicode_decode_utf8_writer(_PyUnicodeWriter *writer, - const char *s, Py_ssize_t size, - _Py_error_handler error_handler, const char *errors, - Py_ssize_t *consumed); #ifdef Py_DEBUG static inline int unicode_is_finalizing(void); static int unicode_is_singleton(PyObject *unicode); @@ -256,7 +200,8 @@ static int unicode_is_singleton(PyObject *unicode); // Return a reference to the immortal empty string singleton. -static inline PyObject* unicode_get_empty(void) +PyObject* +_PyUnicode_GetEmpty(void) { _Py_DECLARE_STR(empty, ""); return &_Py_STR(empty); @@ -430,42 +375,9 @@ static void clear_global_interned_strings(void) #define _Py_RETURN_UNICODE_EMPTY() \ do { \ - return unicode_get_empty(); \ + return _PyUnicode_GetEmpty();\ } while (0) -static inline void -unicode_fill(int kind, void *data, Py_UCS4 value, - Py_ssize_t start, Py_ssize_t length) -{ - assert(0 <= start); - switch (kind) { - case PyUnicode_1BYTE_KIND: { - assert(value <= 0xff); - Py_UCS1 ch = (unsigned char)value; - Py_UCS1 *to = (Py_UCS1 *)data + start; - memset(to, ch, length); - break; - } - case PyUnicode_2BYTE_KIND: { - assert(value <= 0xffff); - Py_UCS2 ch = (Py_UCS2)value; - Py_UCS2 *to = (Py_UCS2 *)data + start; - const Py_UCS2 *end = to + length; - for (; to < end; ++to) *to = ch; - break; - } - case PyUnicode_4BYTE_KIND: { - assert(value <= MAX_UNICODE); - Py_UCS4 ch = value; - Py_UCS4 * to = (Py_UCS4 *)data + start; - const Py_UCS4 *end = to + length; - for (; to < end; ++to) *to = ch; - break; - } - default: Py_UNREACHABLE(); - } -} - /* Fast detection of the most frequent whitespace characters */ const unsigned char _Py_ascii_whitespace[] = { @@ -500,7 +412,6 @@ const unsigned char _Py_ascii_whitespace[] = { /* forward */ static PyObject* get_latin1_char(unsigned char ch); -static int unicode_modifiable(PyObject *unicode); static PyObject * @@ -636,7 +547,8 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) } /* Disable checks during Python finalization. For example, it allows to - call _PyObject_Dump() during finalization for debugging purpose. */ + * call PyUnstable_Object_Dump() during finalization for debugging purpose. + */ if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; } @@ -796,14 +708,14 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) #undef CHECK } -static PyObject* -unicode_result(PyObject *unicode) +PyObject* +_PyUnicode_Result(PyObject *unicode) { assert(_PyUnicode_CHECK(unicode)); Py_ssize_t length = PyUnicode_GET_LENGTH(unicode); if (length == 0) { - PyObject *empty = unicode_get_empty(); + PyObject *empty = _PyUnicode_GetEmpty(); if (unicode != empty) { Py_DECREF(unicode); } @@ -826,6 +738,7 @@ unicode_result(PyObject *unicode) assert(_PyUnicode_CheckConsistency(unicode, 1)); return unicode; } +#define unicode_result _PyUnicode_Result static PyObject* unicode_result_unchanged(PyObject *unicode) @@ -841,7 +754,7 @@ unicode_result_unchanged(PyObject *unicode) /* Implementation of the "backslashreplace" error handler for 8-bit encodings: ASCII, Latin1, UTF-8, etc. */ static char* -backslashreplace(_PyBytesWriter *writer, char *str, +backslashreplace(PyBytesWriter *writer, char *str, PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) { Py_ssize_t size, i; @@ -874,9 +787,10 @@ backslashreplace(_PyBytesWriter *writer, char *str, size += incr; } - str = _PyBytesWriter_Prepare(writer, str, size); - if (str == NULL) + str = PyBytesWriter_GrowAndUpdatePointer(writer, size, str); + if (str == NULL) { return NULL; + } /* generate replacement */ for (i = collstart; i < collend; ++i) { @@ -907,7 +821,7 @@ backslashreplace(_PyBytesWriter *writer, char *str, /* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings: ASCII, Latin1, UTF-8, etc. */ static char* -xmlcharrefreplace(_PyBytesWriter *writer, char *str, +xmlcharrefreplace(PyBytesWriter *writer, char *str, PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) { Py_ssize_t size, i; @@ -948,9 +862,10 @@ xmlcharrefreplace(_PyBytesWriter *writer, char *str, size += incr; } - str = _PyBytesWriter_Prepare(writer, str, size); - if (str == NULL) + str = PyBytesWriter_GrowAndUpdatePointer(writer, size, str); + if (str == NULL) { return NULL; + } /* generate replacement */ for (i = collstart; i < collend; ++i) { @@ -1029,21 +944,9 @@ make_bloom_mask(int kind, const void* ptr, Py_ssize_t len) #undef BLOOM_UPDATE } -static int -ensure_unicode(PyObject *obj) -{ - if (!PyUnicode_Check(obj)) { - PyErr_Format(PyExc_TypeError, - "must be str, not %.100s", - Py_TYPE(obj)->tp_name); - return -1; - } - return 0; -} - /* Compilation of templated routines */ -#define STRINGLIB_GET_EMPTY() unicode_get_empty() +#define STRINGLIB_GET_EMPTY() _PyUnicode_GetEmpty() #include "stringlib/asciilib.h" #include "stringlib/fastsearch.h" @@ -1155,8 +1058,8 @@ resize_copy(PyObject *unicode, Py_ssize_t length) return copy; } -static PyObject* -resize_compact(PyObject *unicode, Py_ssize_t length) +PyObject* +_PyUnicode_ResizeCompact(PyObject *unicode, Py_ssize_t length) { Py_ssize_t char_size; Py_ssize_t struct_size; @@ -1166,7 +1069,7 @@ resize_compact(PyObject *unicode, Py_ssize_t length) Py_ssize_t old_length = _PyUnicode_LENGTH(unicode); #endif - if (!unicode_modifiable(unicode)) { + if (!_PyUnicode_IsModifiable(unicode)) { PyObject *copy = resize_copy(unicode, length); if (copy == NULL) { return NULL; @@ -1364,7 +1267,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) { /* Optimization for empty strings */ if (size == 0) { - return unicode_get_empty(); + return _PyUnicode_GetEmpty(); } PyObject *obj; @@ -1458,7 +1361,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) static int unicode_check_modifiable(PyObject *unicode) { - if (!unicode_modifiable(unicode)) { + if (!_PyUnicode_IsModifiable(unicode)) { PyErr_SetString(PyExc_SystemError, "Cannot modify a string currently used"); return -1; @@ -1820,8 +1723,8 @@ unicode_is_singleton(PyObject *unicode) } #endif -static int -unicode_modifiable(PyObject *unicode) +int +_PyUnicode_IsModifiable(PyObject *unicode) { assert(_PyUnicode_CHECK(unicode)); if (!_PyObject_IsUniquelyReferenced(unicode)) @@ -1857,12 +1760,12 @@ unicode_resize(PyObject **p_unicode, Py_ssize_t length) return 0; if (length == 0) { - PyObject *empty = unicode_get_empty(); + PyObject *empty = _PyUnicode_GetEmpty(); Py_SETREF(*p_unicode, empty); return 0; } - if (!unicode_modifiable(unicode)) { + if (!_PyUnicode_IsModifiable(unicode)) { PyObject *copy = resize_copy(unicode, length); if (copy == NULL) return -1; @@ -1871,7 +1774,7 @@ unicode_resize(PyObject **p_unicode, Py_ssize_t length) } if (PyUnicode_IS_COMPACT(unicode)) { - PyObject *new_unicode = resize_compact(unicode, length); + PyObject *new_unicode = _PyUnicode_ResizeCompact(unicode, length); if (new_unicode == NULL) return -1; *p_unicode = new_unicode; @@ -1897,58 +1800,6 @@ PyUnicode_Resize(PyObject **p_unicode, Py_ssize_t length) return unicode_resize(p_unicode, length); } -/* Copy an ASCII or latin1 char* string into a Python Unicode string. - - WARNING: The function doesn't copy the terminating null character and - doesn't check the maximum character (may write a latin1 character in an - ASCII string). */ -static void -unicode_write_cstr(PyObject *unicode, Py_ssize_t index, - const char *str, Py_ssize_t len) -{ - int kind = PyUnicode_KIND(unicode); - const void *data = PyUnicode_DATA(unicode); - const char *end = str + len; - - assert(index + len <= PyUnicode_GET_LENGTH(unicode)); - switch (kind) { - case PyUnicode_1BYTE_KIND: { -#ifdef Py_DEBUG - if (PyUnicode_IS_ASCII(unicode)) { - Py_UCS4 maxchar = ucs1lib_find_max_char( - (const Py_UCS1*)str, - (const Py_UCS1*)str + len); - assert(maxchar < 128); - } -#endif - memcpy((char *) data + index, str, len); - break; - } - case PyUnicode_2BYTE_KIND: { - Py_UCS2 *start = (Py_UCS2 *)data + index; - Py_UCS2 *ucs2 = start; - - for (; str < end; ++ucs2, ++str) - *ucs2 = (Py_UCS2)*str; - - assert((ucs2 - start) <= PyUnicode_GET_LENGTH(unicode)); - break; - } - case PyUnicode_4BYTE_KIND: { - Py_UCS4 *start = (Py_UCS4 *)data + index; - Py_UCS4 *ucs4 = start; - - for (; str < end; ++ucs4, ++str) - *ucs4 = (Py_UCS4)*str; - - assert((ucs4 - start) <= PyUnicode_GET_LENGTH(unicode)); - break; - } - default: - Py_UNREACHABLE(); - } -} - static PyObject* get_latin1_char(Py_UCS1 ch) { @@ -2163,7 +2014,7 @@ PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) "NULL string with positive size with NULL passed to PyUnicode_FromStringAndSize"); return NULL; } - return unicode_get_empty(); + return _PyUnicode_GetEmpty(); } PyObject * @@ -2730,8 +2581,8 @@ unicode_fromformat_write_utf8(_PyUnicodeWriter *writer, const char *str, } if (width < 0) { - return unicode_decode_utf8_writer(writer, str, length, - _Py_ERROR_REPLACE, "replace", pconsumed); + return _PyUnicode_DecodeUTF8Writer(writer, str, length, + _Py_ERROR_REPLACE, "replace", pconsumed); } PyObject *unicode = PyUnicode_DecodeUTF8Stateful(str, length, @@ -3276,14 +3127,22 @@ PyUnicode_FromFormat(const char *format, ...) int PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + int res = _PyUnicodeWriter_FormatV(writer, format, vargs); + va_end(vargs); + return res; +} + +int +_PyUnicodeWriter_FormatV(PyUnicodeWriter *writer, const char *format, + va_list vargs) { _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; Py_ssize_t old_pos = _writer->pos; - va_list vargs; - va_start(vargs, format); int res = unicode_from_format(_writer, format, vargs); - va_end(vargs); if (res < 0) { _writer->pos = old_pos; @@ -3591,13 +3450,14 @@ PyUnicode_FromEncodedObject(PyObject *obj, return v; } -/* Normalize an encoding name: similar to encodings.normalize_encoding(), but - also convert to lowercase. Return 1 on success, or 0 on error (encoding is - longer than lower_len-1). */ +/* Normalize an encoding name like encodings.normalize_encoding() + but allow to convert to lowercase if *to_lower* is true. + Return 1 on success, or 0 on error (encoding is longer than lower_len-1). */ int _Py_normalize_encoding(const char *encoding, char *lower, - size_t lower_len) + size_t lower_len, + int to_lower) { const char *e; char *l; @@ -3628,7 +3488,7 @@ _Py_normalize_encoding(const char *encoding, if (l == l_end) { return 0; } - *l++ = Py_TOLOWER(c); + *l++ = to_lower ? Py_TOLOWER(c) : c; } else { punct = 1; @@ -3663,7 +3523,7 @@ PyUnicode_Decode(const char *s, } /* Shortcuts for common default encodings */ - if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) { + if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower), 1)) { char *lower = buflower; /* Fast paths */ @@ -3730,7 +3590,7 @@ PyUnicode_Decode(const char *s, return NULL; } -PyObject * +PyAPI_FUNC(PyObject *) PyUnicode_AsDecodedObject(PyObject *unicode, const char *encoding, const char *errors) @@ -3740,12 +3600,6 @@ PyUnicode_AsDecodedObject(PyObject *unicode, return NULL; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyUnicode_AsDecodedObject() is deprecated " - "and will be removed in 3.15; " - "use PyCodec_Decode() to decode from str", 1) < 0) - return NULL; - if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -3753,7 +3607,7 @@ PyUnicode_AsDecodedObject(PyObject *unicode, return PyCodec_Decode(unicode, encoding, errors); } -PyObject * +PyAPI_FUNC(PyObject *) PyUnicode_AsDecodedUnicode(PyObject *unicode, const char *encoding, const char *errors) @@ -3765,12 +3619,6 @@ PyUnicode_AsDecodedUnicode(PyObject *unicode, goto onError; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyUnicode_AsDecodedUnicode() is deprecated " - "and will be removed in 3.15; " - "use PyCodec_Decode() to decode from str to str", 1) < 0) - return NULL; - if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -3793,7 +3641,7 @@ PyUnicode_AsDecodedUnicode(PyObject *unicode, return NULL; } -PyObject * +PyAPI_FUNC(PyObject *) PyUnicode_AsEncodedObject(PyObject *unicode, const char *encoding, const char *errors) @@ -3805,13 +3653,6 @@ PyUnicode_AsEncodedObject(PyObject *unicode, goto onError; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyUnicode_AsEncodedObject() is deprecated " - "and will be removed in 3.15; " - "use PyUnicode_AsEncodedString() to encode from str to bytes " - "or PyCodec_Encode() for generic encoding", 1) < 0) - return NULL; - if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -3939,7 +3780,7 @@ PyUnicode_AsEncodedString(PyObject *unicode, } /* Shortcuts for common default encodings */ - if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) { + if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower), 1)) { char *lower = buflower; /* Fast paths */ @@ -4017,7 +3858,7 @@ PyUnicode_AsEncodedString(PyObject *unicode, return NULL; } -PyObject * +PyAPI_FUNC(PyObject *) PyUnicode_AsEncodedUnicode(PyObject *unicode, const char *encoding, const char *errors) @@ -4029,12 +3870,6 @@ PyUnicode_AsEncodedUnicode(PyObject *unicode, goto onError; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyUnicode_AsEncodedUnicode() is deprecated " - "and will be removed in 3.15; " - "use PyCodec_Encode() to encode from str to str", 1) < 0) - return NULL; - if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -4698,15 +4533,12 @@ char utf7_category[128] = { /* ENCODE_DIRECT: this character should be encoded as itself. The * answer depends on whether we are encoding set O as itself, and also - * on whether we are encoding whitespace as itself. RFC2152 makes it + * on whether we are encoding whitespace as itself. RFC 2152 makes it * clear that the answers to these questions vary between * applications, so this code needs to be flexible. */ -#define ENCODE_DIRECT(c, directO, directWS) \ - ((c) < 128 && (c) > 0 && \ - ((utf7_category[(c)] == 0) || \ - (directWS && (utf7_category[(c)] == 2)) || \ - (directO && (utf7_category[(c)] == 1)))) +#define ENCODE_DIRECT(c) \ + ((c) < 128 && (c) > 0 && ((utf7_category[(c)] != 3))) PyObject * PyUnicode_DecodeUTF7(const char *s, @@ -4923,41 +4755,33 @@ PyUnicode_DecodeUTF7Stateful(const char *s, PyObject * _PyUnicode_EncodeUTF7(PyObject *str, - int base64SetO, - int base64WhiteSpace, const char *errors) { - int kind; - const void *data; - Py_ssize_t len; - PyObject *v; - int inShift = 0; - Py_ssize_t i; - unsigned int base64bits = 0; - unsigned long base64buffer = 0; - char * out; - const char * start; - - kind = PyUnicode_KIND(str); - data = PyUnicode_DATA(str); - len = PyUnicode_GET_LENGTH(str); - - if (len == 0) - return PyBytes_FromStringAndSize(NULL, 0); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (len == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + int kind = PyUnicode_KIND(str); + const void *data = PyUnicode_DATA(str); /* It might be possible to tighten this worst case */ - if (len > PY_SSIZE_T_MAX / 8) + if (len > PY_SSIZE_T_MAX / 8) { return PyErr_NoMemory(); - v = PyBytes_FromStringAndSize(NULL, len * 8); - if (v == NULL) + } + PyBytesWriter *writer = PyBytesWriter_Create(len * 8); + if (writer == NULL) { return NULL; + } - start = out = PyBytes_AS_STRING(v); - for (i = 0; i < len; ++i) { + int inShift = 0; + unsigned int base64bits = 0; + unsigned long base64buffer = 0; + char *out = PyBytesWriter_GetData(writer); + for (Py_ssize_t i = 0; i < len; ++i) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (inShift) { - if (ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) { + if (ENCODE_DIRECT(ch)) { /* shifting out */ if (base64bits) { /* output remaining bits */ *out++ = TO_BASE64(base64buffer << (6-base64bits)); @@ -4981,7 +4805,7 @@ _PyUnicode_EncodeUTF7(PyObject *str, *out++ = '+'; *out++ = '-'; } - else if (ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) { + else if (ENCODE_DIRECT(ch)) { *out++ = (char) ch; } else { @@ -5016,9 +4840,7 @@ _PyUnicode_EncodeUTF7(PyObject *str, *out++= TO_BASE64(base64buffer << (6-base64bits) ); if (inShift) *out++ = '-'; - if (_PyBytes_Resize(&v, out - start) < 0) - return NULL; - return v; + return PyBytesWriter_FinishWithPointer(writer, out); } #undef IS_BASE64 @@ -5498,7 +5320,6 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, if (maxchr <= 255) { memcpy(PyUnicode_1BYTE_DATA(u), s, pos); s += pos; - size -= pos; writer.pos = pos; } @@ -5513,11 +5334,11 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, // Used by PyUnicodeWriter_WriteUTF8() implementation -static int -unicode_decode_utf8_writer(_PyUnicodeWriter *writer, - const char *s, Py_ssize_t size, - _Py_error_handler error_handler, const char *errors, - Py_ssize_t *consumed) +int +_PyUnicode_DecodeUTF8Writer(_PyUnicodeWriter *writer, + const char *s, Py_ssize_t size, + _Py_error_handler error_handler, const char *errors, + Py_ssize_t *consumed) { if (size == 0) { if (consumed) { @@ -5546,7 +5367,6 @@ unicode_decode_utf8_writer(_PyUnicodeWriter *writer, return 0; } s += decoded; - size -= decoded; } return unicode_decode_utf8_impl(writer, starts, s, end, @@ -5866,7 +5686,7 @@ unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, const void *data = PyUnicode_DATA(unicode); Py_ssize_t size = PyUnicode_GET_LENGTH(unicode); - _PyBytesWriter writer; + PyBytesWriter *writer; char *end; switch (kind) { @@ -5875,21 +5695,24 @@ unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, case PyUnicode_1BYTE_KIND: /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */ assert(!PyUnicode_IS_ASCII(unicode)); - end = ucs1lib_utf8_encoder(&writer, unicode, data, size, error_handler, errors); + writer = ucs1lib_utf8_encoder(unicode, data, size, + error_handler, errors, &end); break; case PyUnicode_2BYTE_KIND: - end = ucs2lib_utf8_encoder(&writer, unicode, data, size, error_handler, errors); + writer = ucs2lib_utf8_encoder(unicode, data, size, + error_handler, errors, &end); break; case PyUnicode_4BYTE_KIND: - end = ucs4lib_utf8_encoder(&writer, unicode, data, size, error_handler, errors); + writer = ucs4lib_utf8_encoder(unicode, data, size, + error_handler, errors, &end); break; } - if (end == NULL) { - _PyBytesWriter_Dealloc(&writer); + if (writer == NULL) { + PyBytesWriter_Discard(writer); return NULL; } - return _PyBytesWriter_Finish(&writer, end); + return PyBytesWriter_FinishWithPointer(writer, end); } static int @@ -5903,37 +5726,35 @@ unicode_fill_utf8(PyObject *unicode) const void *data = PyUnicode_DATA(unicode); Py_ssize_t size = PyUnicode_GET_LENGTH(unicode); - _PyBytesWriter writer; + PyBytesWriter *writer; char *end; switch (kind) { default: Py_UNREACHABLE(); case PyUnicode_1BYTE_KIND: - end = ucs1lib_utf8_encoder(&writer, unicode, data, size, - _Py_ERROR_STRICT, NULL); + writer = ucs1lib_utf8_encoder(unicode, data, size, + _Py_ERROR_STRICT, NULL, &end); break; case PyUnicode_2BYTE_KIND: - end = ucs2lib_utf8_encoder(&writer, unicode, data, size, - _Py_ERROR_STRICT, NULL); + writer = ucs2lib_utf8_encoder(unicode, data, size, + _Py_ERROR_STRICT, NULL, &end); break; case PyUnicode_4BYTE_KIND: - end = ucs4lib_utf8_encoder(&writer, unicode, data, size, - _Py_ERROR_STRICT, NULL); + writer = ucs4lib_utf8_encoder(unicode, data, size, + _Py_ERROR_STRICT, NULL, &end); break; } - if (end == NULL) { - _PyBytesWriter_Dealloc(&writer); + if (writer == NULL) { return -1; } - const char *start = writer.use_small_buffer ? writer.small_buffer : - PyBytes_AS_STRING(writer.buffer); + const char *start = PyBytesWriter_GetData(writer); Py_ssize_t len = end - start; char *cache = PyMem_Malloc(len + 1); if (cache == NULL) { - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); PyErr_NoMemory(); return -1; } @@ -5941,7 +5762,7 @@ unicode_fill_utf8(PyObject *unicode) cache[len] = '\0'; PyUnicode_SET_UTF8_LENGTH(unicode, len); PyUnicode_SET_UTF8(unicode, cache); - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); return 0; } @@ -6119,45 +5940,61 @@ _PyUnicode_EncodeUTF32(PyObject *str, const char *errors, int byteorder) { - int kind; - const void *data; - Py_ssize_t len; - PyObject *v; - uint32_t *out; + if (!PyUnicode_Check(str)) { + PyErr_BadArgument(); + return NULL; + } + int kind = PyUnicode_KIND(str); + const void *data = PyUnicode_DATA(str); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + + if (len > PY_SSIZE_T_MAX / 4 - (byteorder == 0)) + return PyErr_NoMemory(); + Py_ssize_t nsize = len + (byteorder == 0); + #if PY_LITTLE_ENDIAN int native_ordering = byteorder <= 0; #else int native_ordering = byteorder >= 0; #endif - const char *encoding; - Py_ssize_t nsize, pos; - PyObject *errorHandler = NULL; - PyObject *exc = NULL; - PyObject *rep = NULL; - if (!PyUnicode_Check(str)) { - PyErr_BadArgument(); + if (kind == PyUnicode_1BYTE_KIND) { + // gh-139156: Don't use PyBytesWriter API here since it has an overhead + // on short strings + PyObject *v = PyBytes_FromStringAndSize(NULL, nsize * 4); + if (v == NULL) { + return NULL; + } + + /* output buffer is 4-bytes aligned */ + assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 4)); + uint32_t *out = (uint32_t *)PyBytes_AS_STRING(v); + if (byteorder == 0) { + *out++ = 0xFEFF; + } + if (len > 0) { + ucs1lib_utf32_encode((const Py_UCS1 *)data, len, + &out, native_ordering); + } + return v; + } + + PyBytesWriter *writer = PyBytesWriter_Create(nsize * 4); + if (writer == NULL) { return NULL; } - kind = PyUnicode_KIND(str); - data = PyUnicode_DATA(str); - len = PyUnicode_GET_LENGTH(str); - - if (len > PY_SSIZE_T_MAX / 4 - (byteorder == 0)) - return PyErr_NoMemory(); - nsize = len + (byteorder == 0); - v = PyBytes_FromStringAndSize(NULL, nsize * 4); - if (v == NULL) - return NULL; /* output buffer is 4-bytes aligned */ - assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 4)); - out = (uint32_t *)PyBytes_AS_STRING(v); - if (byteorder == 0) + assert(_Py_IS_ALIGNED(PyBytesWriter_GetData(writer), 4)); + uint32_t *out = (uint32_t *)PyBytesWriter_GetData(writer); + if (byteorder == 0) { *out++ = 0xFEFF; - if (len == 0) - goto done; + } + if (len == 0) { + return PyBytesWriter_Finish(writer); + } + const char *encoding; if (byteorder == -1) encoding = "utf-32-le"; else if (byteorder == 1) @@ -6165,15 +6002,11 @@ _PyUnicode_EncodeUTF32(PyObject *str, else encoding = "utf-32"; - if (kind == PyUnicode_1BYTE_KIND) { - ucs1lib_utf32_encode((const Py_UCS1 *)data, len, &out, native_ordering); - goto done; - } - - pos = 0; - while (pos < len) { - Py_ssize_t newpos, repsize, moreunits; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + PyObject *rep = NULL; + for (Py_ssize_t pos = 0; pos < len; ) { if (kind == PyUnicode_2BYTE_KIND) { pos += ucs2lib_utf32_encode((const Py_UCS2 *)data + pos, len - pos, &out, native_ordering); @@ -6186,6 +6019,7 @@ _PyUnicode_EncodeUTF32(PyObject *str, if (pos == len) break; + Py_ssize_t newpos; rep = unicode_encode_call_errorhandler( errors, &errorHandler, encoding, "surrogates not allowed", @@ -6193,6 +6027,7 @@ _PyUnicode_EncodeUTF32(PyObject *str, if (!rep) goto error; + Py_ssize_t repsize, moreunits; if (PyBytes_Check(rep)) { repsize = PyBytes_GET_SIZE(rep); if (repsize & 3) { @@ -6218,21 +6053,18 @@ _PyUnicode_EncodeUTF32(PyObject *str, /* four bytes are reserved for each surrogate */ if (moreunits > 0) { - Py_ssize_t outpos = out - (uint32_t*) PyBytes_AS_STRING(v); - if (moreunits >= (PY_SSIZE_T_MAX - PyBytes_GET_SIZE(v)) / 4) { - /* integer overflow */ - PyErr_NoMemory(); + out = PyBytesWriter_GrowAndUpdatePointer(writer, 4 * moreunits, out); + if (out == NULL) { goto error; } - if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + 4 * moreunits) < 0) - goto error; - out = (uint32_t*) PyBytes_AS_STRING(v) + outpos; } if (PyBytes_Check(rep)) { memcpy(out, PyBytes_AS_STRING(rep), repsize); out += repsize / 4; - } else /* rep is unicode */ { + } + else { + /* rep is unicode */ assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); ucs1lib_utf32_encode(PyUnicode_1BYTE_DATA(rep), repsize, &out, native_ordering); @@ -6241,21 +6073,19 @@ _PyUnicode_EncodeUTF32(PyObject *str, Py_CLEAR(rep); } + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + /* Cut back to size actually needed. This is necessary for, for example, encoding of a string containing isolated surrogates and the 'ignore' handler is used. */ - nsize = (unsigned char*) out - (unsigned char*) PyBytes_AS_STRING(v); - if (nsize != PyBytes_GET_SIZE(v)) - _PyBytes_Resize(&v, nsize); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - done: - return v; + return PyBytesWriter_FinishWithPointer(writer, out); + error: Py_XDECREF(rep); Py_XDECREF(errorHandler); Py_XDECREF(exc); - Py_XDECREF(v); + PyBytesWriter_Discard(writer); return NULL; } @@ -6436,32 +6266,15 @@ _PyUnicode_EncodeUTF16(PyObject *str, const char *errors, int byteorder) { - int kind; - const void *data; - Py_ssize_t len; - PyObject *v; - unsigned short *out; - Py_ssize_t pairs; -#if PY_BIG_ENDIAN - int native_ordering = byteorder >= 0; -#else - int native_ordering = byteorder <= 0; -#endif - const char *encoding; - Py_ssize_t nsize, pos; - PyObject *errorHandler = NULL; - PyObject *exc = NULL; - PyObject *rep = NULL; - if (!PyUnicode_Check(str)) { PyErr_BadArgument(); return NULL; } - kind = PyUnicode_KIND(str); - data = PyUnicode_DATA(str); - len = PyUnicode_GET_LENGTH(str); + int kind = PyUnicode_KIND(str); + const void *data = PyUnicode_DATA(str); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); - pairs = 0; + Py_ssize_t pairs = 0; if (kind == PyUnicode_4BYTE_KIND) { const Py_UCS4 *in = (const Py_UCS4 *)data; const Py_UCS4 *end = in + len; @@ -6474,27 +6287,50 @@ _PyUnicode_EncodeUTF16(PyObject *str, if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) { return PyErr_NoMemory(); } - nsize = len + pairs + (byteorder == 0); - v = PyBytes_FromStringAndSize(NULL, nsize * 2); - if (v == NULL) { + Py_ssize_t nsize = len + pairs + (byteorder == 0); + +#if PY_BIG_ENDIAN + int native_ordering = byteorder >= 0; +#else + int native_ordering = byteorder <= 0; +#endif + + if (kind == PyUnicode_1BYTE_KIND) { + // gh-139156: Don't use PyBytesWriter API here since it has an overhead + // on short strings + PyObject *v = PyBytes_FromStringAndSize(NULL, nsize * 2); + if (v == NULL) { + return NULL; + } + + /* output buffer is 2-bytes aligned */ + assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 2)); + unsigned short *out = (unsigned short *)PyBytes_AS_STRING(v); + if (byteorder == 0) { + *out++ = 0xFEFF; + } + if (len > 0) { + ucs1lib_utf16_encode((const Py_UCS1 *)data, len, &out, native_ordering); + } + return v; + } + + PyBytesWriter *writer = PyBytesWriter_Create(nsize * 2); + if (writer == NULL) { return NULL; } /* output buffer is 2-bytes aligned */ - assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 2)); - out = (unsigned short *)PyBytes_AS_STRING(v); + assert(_Py_IS_ALIGNED(PyBytesWriter_GetData(writer), 2)); + unsigned short *out = PyBytesWriter_GetData(writer); if (byteorder == 0) { *out++ = 0xFEFF; } if (len == 0) { - goto done; - } - - if (kind == PyUnicode_1BYTE_KIND) { - ucs1lib_utf16_encode((const Py_UCS1 *)data, len, &out, native_ordering); - goto done; + return PyBytesWriter_Finish(writer); } + const char *encoding; if (byteorder < 0) { encoding = "utf-16-le"; } @@ -6505,10 +6341,11 @@ _PyUnicode_EncodeUTF16(PyObject *str, encoding = "utf-16"; } - pos = 0; - while (pos < len) { - Py_ssize_t newpos, repsize, moreunits; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + PyObject *rep = NULL; + for (Py_ssize_t pos = 0; pos < len; ) { if (kind == PyUnicode_2BYTE_KIND) { pos += ucs2lib_utf16_encode((const Py_UCS2 *)data + pos, len - pos, &out, native_ordering); @@ -6521,6 +6358,7 @@ _PyUnicode_EncodeUTF16(PyObject *str, if (pos == len) break; + Py_ssize_t newpos; rep = unicode_encode_call_errorhandler( errors, &errorHandler, encoding, "surrogates not allowed", @@ -6528,6 +6366,7 @@ _PyUnicode_EncodeUTF16(PyObject *str, if (!rep) goto error; + Py_ssize_t repsize, moreunits; if (PyBytes_Check(rep)) { repsize = PyBytes_GET_SIZE(rep); if (repsize & 1) { @@ -6553,21 +6392,17 @@ _PyUnicode_EncodeUTF16(PyObject *str, /* two bytes are reserved for each surrogate */ if (moreunits > 0) { - Py_ssize_t outpos = out - (unsigned short*) PyBytes_AS_STRING(v); - if (moreunits >= (PY_SSIZE_T_MAX - PyBytes_GET_SIZE(v)) / 2) { - /* integer overflow */ - PyErr_NoMemory(); + out = PyBytesWriter_GrowAndUpdatePointer(writer, 2 * moreunits, out); + if (out == NULL) { goto error; } - if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + 2 * moreunits) < 0) - goto error; - out = (unsigned short*) PyBytes_AS_STRING(v) + outpos; } if (PyBytes_Check(rep)) { memcpy(out, PyBytes_AS_STRING(rep), repsize); out += repsize / 2; - } else /* rep is unicode */ { + } else { + /* rep is unicode */ assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); ucs1lib_utf16_encode(PyUnicode_1BYTE_DATA(rep), repsize, &out, native_ordering); @@ -6576,23 +6411,20 @@ _PyUnicode_EncodeUTF16(PyObject *str, Py_CLEAR(rep); } + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + /* Cut back to size actually needed. This is necessary for, for example, encoding of a string containing isolated surrogates and the 'ignore' handler is used. */ - nsize = (unsigned char*) out - (unsigned char*) PyBytes_AS_STRING(v); - if (nsize != PyBytes_GET_SIZE(v)) - _PyBytes_Resize(&v, nsize); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - done: - return v; + return PyBytesWriter_FinishWithPointer(writer, out); + error: Py_XDECREF(rep); Py_XDECREF(errorHandler); Py_XDECREF(exc); - Py_XDECREF(v); + PyBytesWriter_Discard(writer); return NULL; -#undef STORECHAR } PyObject * @@ -6621,13 +6453,15 @@ _PyUnicode_GetNameCAPI(void) /* --- Unicode Escape Codec ----------------------------------------------- */ PyObject * -_PyUnicode_DecodeUnicodeEscapeInternal(const char *s, +_PyUnicode_DecodeUnicodeEscapeInternal2(const char *s, Py_ssize_t size, const char *errors, Py_ssize_t *consumed, - const char **first_invalid_escape) + int *first_invalid_escape_char, + const char **first_invalid_escape_ptr) { const char *starts = s; + const char *initial_starts = starts; _PyUnicodeWriter writer; const char *end; PyObject *errorHandler = NULL; @@ -6635,7 +6469,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, _PyUnicode_Name_CAPI *ucnhash_capi; // so we can remember if we've seen an invalid escape char or not - *first_invalid_escape = NULL; + *first_invalid_escape_char = -1; + *first_invalid_escape_ptr = NULL; if (size == 0) { if (consumed) { @@ -6723,9 +6558,12 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, } } if (ch > 0377) { - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-3; /* Back up 3 chars, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = ch; + if (starts == initial_starts) { + /* Back up 3 chars, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 3; + } } } WRITE_CHAR(ch); @@ -6820,9 +6658,12 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, goto error; default: - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-1; /* Back up one char, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = c; + if (starts == initial_starts) { + /* Back up one char, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 1; + } } WRITE_ASCII_CHAR('\\'); WRITE_CHAR(c); @@ -6867,19 +6708,20 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s, const char *errors, Py_ssize_t *consumed) { - const char *first_invalid_escape; - PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal(s, size, errors, + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal2(s, size, errors, consumed, - &first_invalid_escape); + &first_invalid_escape_char, + &first_invalid_escape_ptr); if (result == NULL) return NULL; - if (first_invalid_escape != NULL) { - unsigned char c = *first_invalid_escape; - if ('4' <= c && c <= '7') { + if (first_invalid_escape_char != -1) { + if (first_invalid_escape_char > 0xff) { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "\"\\%.3s\" is an invalid octal escape sequence. " + "\"\\%o\" is an invalid octal escape sequence. " "Such sequences will not work in the future. ", - first_invalid_escape) < 0) + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; @@ -6889,7 +6731,7 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s, if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "\"\\%c\" is an invalid escape sequence. " "Such sequences will not work in the future. ", - c) < 0) + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; @@ -6912,46 +6754,36 @@ PyUnicode_DecodeUnicodeEscape(const char *s, PyObject * PyUnicode_AsUnicodeEscapeString(PyObject *unicode) { - Py_ssize_t i, len; - PyObject *repr; - char *p; - int kind; - const void *data; - Py_ssize_t expandsize; - - /* Initial allocation is based on the longest-possible character - escape. - - For UCS1 strings it's '\xxx', 4 bytes per source character. - For UCS2 strings it's '\uxxxx', 6 bytes per source character. - For UCS4 strings it's '\U00xxxxxx', 10 bytes per source character. - */ - if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - len = PyUnicode_GET_LENGTH(unicode); + Py_ssize_t len = PyUnicode_GET_LENGTH(unicode); if (len == 0) { - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } + int kind = PyUnicode_KIND(unicode); + const void *data = PyUnicode_DATA(unicode); - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 - bytes, and 1 byte characters 4. */ - expandsize = kind * 2 + 2; + /* Initial allocation is based on the longest-possible character + * escape. + * + * For UCS1 strings it's '\xxx', 4 bytes per source character. + * For UCS2 strings it's '\uxxxx', 6 bytes per source character. + * For UCS4 strings it's '\U00xxxxxx', 10 bytes per source character. */ + Py_ssize_t expandsize = kind * 2 + 2; if (len > PY_SSIZE_T_MAX / expandsize) { return PyErr_NoMemory(); } - repr = PyBytes_FromStringAndSize(NULL, expandsize * len); - if (repr == NULL) { + + PyBytesWriter *writer = PyBytesWriter_Create(expandsize * len); + if (writer == NULL) { return NULL; } + char *p = PyBytesWriter_GetData(writer); - p = PyBytes_AS_STRING(repr); - for (i = 0; i < len; i++) { + for (Py_ssize_t i = 0; i < len; i++) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* U+0000-U+00ff range */ @@ -7017,11 +6849,7 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) } } - assert(p - PyBytes_AS_STRING(repr) > 0); - if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) { - return NULL; - } - return repr; + return PyBytesWriter_FinishWithPointer(writer, p); } /* --- Raw Unicode Escape Codec ------------------------------------------- */ @@ -7174,41 +7002,34 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, PyObject * PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) { - PyObject *repr; - char *p; - Py_ssize_t expandsize, pos; - int kind; - const void *data; - Py_ssize_t len; - if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - len = PyUnicode_GET_LENGTH(unicode); + int kind = PyUnicode_KIND(unicode); + const void *data = PyUnicode_DATA(unicode); + Py_ssize_t len = PyUnicode_GET_LENGTH(unicode); + if (len == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } if (kind == PyUnicode_1BYTE_KIND) { return PyBytes_FromStringAndSize(data, len); } /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 bytes, and 1 byte characters 4. */ - expandsize = kind * 2 + 2; - + Py_ssize_t expandsize = kind * 2 + 2; if (len > PY_SSIZE_T_MAX / expandsize) { return PyErr_NoMemory(); } - repr = PyBytes_FromStringAndSize(NULL, expandsize * len); - if (repr == NULL) { + + PyBytesWriter *writer = PyBytesWriter_Create(expandsize * len); + if (writer == NULL) { return NULL; } - if (len == 0) { - return repr; - } + char *p = PyBytesWriter_GetData(writer); - p = PyBytes_AS_STRING(repr); - for (pos = 0; pos < len; pos++) { + for (Py_ssize_t pos = 0; pos < len; pos++) { Py_UCS4 ch = PyUnicode_READ(kind, data, pos); /* U+0000-U+00ff range: Copy 8-bit characters as-is */ @@ -7240,11 +7061,7 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) } } - assert(p > PyBytes_AS_STRING(repr)); - if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) { - return NULL; - } - return repr; + return PyBytesWriter_FinishWithPointer(writer, p); } /* --- Latin-1 Codec ------------------------------------------------------ */ @@ -7367,16 +7184,12 @@ unicode_encode_ucs1(PyObject *unicode, Py_ssize_t pos=0, size; int kind; const void *data; - /* pointer into the output */ - char *str; const char *encoding = (limit == 256) ? "latin-1" : "ascii"; const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)"; PyObject *error_handler_obj = NULL; PyObject *exc = NULL; _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; PyObject *rep = NULL; - /* output object */ - _PyBytesWriter writer; size = PyUnicode_GET_LENGTH(unicode); kind = PyUnicode_KIND(unicode); @@ -7384,12 +7197,15 @@ unicode_encode_ucs1(PyObject *unicode, /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ if (size == 0) - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); - _PyBytesWriter_Init(&writer); - str = _PyBytesWriter_Alloc(&writer, size); - if (str == NULL) + /* output object */ + PyBytesWriter *writer = PyBytesWriter_Create(size); + if (writer == NULL) { return NULL; + } + /* pointer into the output */ + char *str = PyBytesWriter_GetData(writer); while (pos < size) { Py_UCS4 ch = PyUnicode_READ(kind, data, pos); @@ -7411,7 +7227,7 @@ unicode_encode_ucs1(PyObject *unicode, ++collend; /* Only overallocate the buffer if it's not the last write */ - writer.overallocate = (collend < size); + writer->overallocate = (collend < size); /* cache callback name lookup (if not done yet, i.e. it's the first error) */ if (error_handler == _Py_ERROR_UNKNOWN) @@ -7432,8 +7248,8 @@ unicode_encode_ucs1(PyObject *unicode, case _Py_ERROR_BACKSLASHREPLACE: /* subtract preallocated bytes */ - writer.min_size -= (collend - collstart); - str = backslashreplace(&writer, str, + writer->size -= (collend - collstart); + str = backslashreplace(writer, str, unicode, collstart, collend); if (str == NULL) goto onError; @@ -7442,8 +7258,8 @@ unicode_encode_ucs1(PyObject *unicode, case _Py_ERROR_XMLCHARREFREPLACE: /* subtract preallocated bytes */ - writer.min_size -= (collend - collstart); - str = xmlcharrefreplace(&writer, str, + writer->size -= (collend - collstart); + str = xmlcharrefreplace(writer, str, unicode, collstart, collend); if (str == NULL) goto onError; @@ -7474,24 +7290,27 @@ unicode_encode_ucs1(PyObject *unicode, goto onError; if (newpos < collstart) { - writer.overallocate = 1; - str = _PyBytesWriter_Prepare(&writer, str, - collstart - newpos); - if (str == NULL) + writer->overallocate = 1; + str = PyBytesWriter_GrowAndUpdatePointer(writer, + collstart - newpos, + str); + if (str == NULL) { goto onError; + } } else { /* subtract preallocated bytes */ - writer.min_size -= newpos - collstart; + writer->size -= newpos - collstart; /* Only overallocate the buffer if it's not the last write */ - writer.overallocate = (newpos < size); + writer->overallocate = (newpos < size); } + char *rep_str; + Py_ssize_t rep_len; if (PyBytes_Check(rep)) { /* Directly copy bytes result to output. */ - str = _PyBytesWriter_WriteBytes(&writer, str, - PyBytes_AS_STRING(rep), - PyBytes_GET_SIZE(rep)); + rep_str = PyBytes_AS_STRING(rep); + rep_len = PyBytes_GET_SIZE(rep); } else { assert(PyUnicode_Check(rep)); @@ -7506,12 +7325,16 @@ unicode_encode_ucs1(PyObject *unicode, goto onError; } assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); - str = _PyBytesWriter_WriteBytes(&writer, str, - PyUnicode_DATA(rep), - PyUnicode_GET_LENGTH(rep)); + rep_str = PyUnicode_DATA(rep); + rep_len = PyUnicode_GET_LENGTH(rep); } - if (str == NULL) + + str = PyBytesWriter_GrowAndUpdatePointer(writer, rep_len, str); + if (str == NULL) { goto onError; + } + memcpy(str, rep_str, rep_len); + str += rep_len; pos = newpos; Py_CLEAR(rep); @@ -7519,17 +7342,17 @@ unicode_encode_ucs1(PyObject *unicode, /* If overallocation was disabled, ensure that it was the last write. Otherwise, we missed an optimization */ - assert(writer.overallocate || pos == size); + assert(writer->overallocate || pos == size); } } Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - return _PyBytesWriter_Finish(&writer, str); + return PyBytesWriter_FinishWithPointer(writer, str); onError: Py_XDECREF(rep); - _PyBytesWriter_Dealloc(&writer); + PyBytesWriter_Discard(writer); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; @@ -7704,10 +7527,6 @@ code_page_name(UINT code_page, PyObject **obj) *obj = NULL; if (code_page == CP_ACP) return "mbcs"; - if (code_page == CP_UTF7) - return "CP_UTF7"; - if (code_page == CP_UTF8) - return "CP_UTF8"; *obj = PyBytes_FromFormat("cp%u", code_page); if (*obj == NULL) @@ -8027,7 +7846,7 @@ encode_code_page_flags(UINT code_page, const char *errors) * an OSError and returns -1 on other error. */ static int -encode_code_page_strict(UINT code_page, PyObject **outbytes, +encode_code_page_strict(UINT code_page, PyBytesWriter **writer, PyObject *unicode, Py_ssize_t offset, int len, const char* errors) { @@ -8073,25 +7892,21 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes, goto done; } - if (*outbytes == NULL) { + if (*writer == NULL) { /* Create string object */ - *outbytes = PyBytes_FromStringAndSize(NULL, outsize); - if (*outbytes == NULL) { + *writer = PyBytesWriter_Create(outsize); + if (*writer == NULL) { goto done; } - out = PyBytes_AS_STRING(*outbytes); + out = PyBytesWriter_GetData(*writer); } else { /* Extend string object */ - const Py_ssize_t n = PyBytes_Size(*outbytes); - if (outsize > PY_SSIZE_T_MAX - n) { - PyErr_NoMemory(); + Py_ssize_t n = PyBytesWriter_GetSize(*writer); + if (PyBytesWriter_Grow(*writer, outsize) < 0) { goto done; } - if (_PyBytes_Resize(outbytes, n + outsize) < 0) { - goto done; - } - out = PyBytes_AS_STRING(*outbytes) + n; + out = (char*)PyBytesWriter_GetData(*writer) + n; } /* Do the conversion */ @@ -8128,7 +7943,7 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes, * -1 on other error. */ static int -encode_code_page_errors(UINT code_page, PyObject **outbytes, +encode_code_page_errors(UINT code_page, PyBytesWriter **writer, PyObject *unicode, Py_ssize_t unicode_offset, Py_ssize_t insize, const char* errors) { @@ -8147,7 +7962,7 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, PyObject *exc = NULL; PyObject *encoding_obj = NULL; const char *encoding; - Py_ssize_t newpos, newoutsize; + Py_ssize_t newpos; PyObject *rep; int ret = -1; @@ -8180,23 +7995,21 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, } outsize = insize * Py_ARRAY_LENGTH(buffer); - if (*outbytes == NULL) { + if (*writer == NULL) { /* Create string object */ - *outbytes = PyBytes_FromStringAndSize(NULL, outsize); - if (*outbytes == NULL) + *writer = PyBytesWriter_Create(outsize); + if (*writer == NULL) { goto error; - out = PyBytes_AS_STRING(*outbytes); + } + out = PyBytesWriter_GetData(*writer); } else { /* Extend string object */ - Py_ssize_t n = PyBytes_Size(*outbytes); - if (n > PY_SSIZE_T_MAX - outsize) { - PyErr_NoMemory(); + Py_ssize_t n = PyBytesWriter_GetSize(*writer); + if (PyBytesWriter_Grow(*writer, outsize) < 0) { goto error; } - if (_PyBytes_Resize(outbytes, n + outsize) < 0) - goto error; - out = PyBytes_AS_STRING(*outbytes) + n; + out = (char*)PyBytesWriter_GetData(*writer) + n; } /* Encode the string character per character */ @@ -8245,13 +8058,11 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, outsize = PyBytes_GET_SIZE(rep); morebytes += outsize; if (morebytes > 0) { - Py_ssize_t offset = out - PyBytes_AS_STRING(*outbytes); - newoutsize = PyBytes_GET_SIZE(*outbytes) + morebytes; - if (_PyBytes_Resize(outbytes, newoutsize) < 0) { + out = PyBytesWriter_GrowAndUpdatePointer(*writer, morebytes, out); + if (out == NULL) { Py_DECREF(rep); goto error; } - out = PyBytes_AS_STRING(*outbytes) + offset; } memcpy(out, PyBytes_AS_STRING(rep), outsize); out += outsize; @@ -8264,13 +8075,11 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, outsize = PyUnicode_GET_LENGTH(rep); morebytes += outsize; if (morebytes > 0) { - Py_ssize_t offset = out - PyBytes_AS_STRING(*outbytes); - newoutsize = PyBytes_GET_SIZE(*outbytes) + morebytes; - if (_PyBytes_Resize(outbytes, newoutsize) < 0) { + out = PyBytesWriter_GrowAndUpdatePointer(*writer, morebytes, out); + if (out == NULL) { Py_DECREF(rep); goto error; } - out = PyBytes_AS_STRING(*outbytes) + offset; } kind = PyUnicode_KIND(rep); data = PyUnicode_DATA(rep); @@ -8293,10 +8102,11 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, } /* write a NUL byte */ *out = 0; - outsize = out - PyBytes_AS_STRING(*outbytes); - assert(outsize <= PyBytes_GET_SIZE(*outbytes)); - if (_PyBytes_Resize(outbytes, outsize) < 0) + outsize = out - (char*)PyBytesWriter_GetData(*writer); + assert(outsize <= PyBytesWriter_GetSize(*writer)); + if (PyBytesWriter_Resize(*writer, outsize) < 0) { goto error; + } ret = 0; error: @@ -8306,13 +8116,14 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, return ret; } -static PyObject * -encode_code_page(int code_page, - PyObject *unicode, - const char *errors) + +PyObject * +PyUnicode_EncodeCodePage(int code_page, + PyObject *unicode, + const char *errors) { Py_ssize_t len; - PyObject *outbytes = NULL; + PyBytesWriter *writer = NULL; Py_ssize_t offset; int chunk_len, ret, done; @@ -8329,7 +8140,7 @@ encode_code_page(int code_page, } if (len == 0) - return PyBytes_FromStringAndSize(NULL, 0); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); offset = 0; do @@ -8346,15 +8157,15 @@ encode_code_page(int code_page, done = 1; } - ret = encode_code_page_strict(code_page, &outbytes, + ret = encode_code_page_strict(code_page, &writer, unicode, offset, chunk_len, errors); if (ret == -2) - ret = encode_code_page_errors(code_page, &outbytes, + ret = encode_code_page_errors(code_page, &writer, unicode, offset, chunk_len, errors); if (ret < 0) { - Py_XDECREF(outbytes); + PyBytesWriter_Discard(writer); return NULL; } @@ -8362,16 +8173,9 @@ encode_code_page(int code_page, len -= chunk_len; } while (!done); - return outbytes; + return PyBytesWriter_Finish(writer); } -PyObject * -PyUnicode_EncodeCodePage(int code_page, - PyObject *unicode, - const char *errors) -{ - return encode_code_page(code_page, unicode, errors); -} PyObject * PyUnicode_AsMBCSString(PyObject *unicode) @@ -8881,15 +8685,13 @@ charmapencode_lookup(Py_UCS4 c, PyObject *mapping, unsigned char *replace) } static int -charmapencode_resize(PyObject **outobj, Py_ssize_t *outpos, Py_ssize_t requiredsize) +charmapencode_resize(PyBytesWriter *writer, Py_ssize_t *outpos, Py_ssize_t requiredsize) { - Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj); + Py_ssize_t outsize = PyBytesWriter_GetSize(writer); /* exponentially overallocate to minimize reallocations */ - if (requiredsize < 2*outsize) - requiredsize = 2*outsize; - if (_PyBytes_Resize(outobj, requiredsize)) - return -1; - return 0; + if (requiredsize < 2 * outsize) + requiredsize = 2 * outsize; + return PyBytesWriter_Resize(writer, requiredsize); } typedef enum charmapencode_result { @@ -8903,22 +8705,26 @@ typedef enum charmapencode_result { reallocation error occurred. The caller must decref the result */ static charmapencode_result charmapencode_output(Py_UCS4 c, PyObject *mapping, - PyObject **outobj, Py_ssize_t *outpos) + PyBytesWriter *writer, Py_ssize_t *outpos) { PyObject *rep; unsigned char replace; char *outstart; - Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj); + Py_ssize_t outsize = _PyBytesWriter_GetSize(writer); if (Py_IS_TYPE(mapping, &EncodingMapType)) { int res = encoding_map_lookup(c, mapping); Py_ssize_t requiredsize = *outpos+1; - if (res == -1) + if (res == -1) { return enc_FAILED; - if (outsize<requiredsize) - if (charmapencode_resize(outobj, outpos, requiredsize)) + } + + if (outsize<requiredsize) { + if (charmapencode_resize(writer, outpos, requiredsize)) { return enc_EXCEPTION; - outstart = PyBytes_AS_STRING(*outobj); + } + } + outstart = _PyBytesWriter_GetData(writer); outstart[(*outpos)++] = (char)res; return enc_SUCCESS; } @@ -8933,11 +8739,11 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, if (PyLong_Check(rep)) { Py_ssize_t requiredsize = *outpos+1; if (outsize<requiredsize) - if (charmapencode_resize(outobj, outpos, requiredsize)) { + if (charmapencode_resize(writer, outpos, requiredsize)) { Py_DECREF(rep); return enc_EXCEPTION; } - outstart = PyBytes_AS_STRING(*outobj); + outstart = _PyBytesWriter_GetData(writer); outstart[(*outpos)++] = (char)replace; } else { @@ -8945,11 +8751,11 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, Py_ssize_t repsize = PyBytes_GET_SIZE(rep); Py_ssize_t requiredsize = *outpos+repsize; if (outsize<requiredsize) - if (charmapencode_resize(outobj, outpos, requiredsize)) { + if (charmapencode_resize(writer, outpos, requiredsize)) { Py_DECREF(rep); return enc_EXCEPTION; } - outstart = PyBytes_AS_STRING(*outobj); + outstart = _PyBytesWriter_GetData(writer); memcpy(outstart + *outpos, repchars, repsize); *outpos += repsize; } @@ -8958,14 +8764,14 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, return enc_SUCCESS; } -/* handle an error in PyUnicode_EncodeCharmap +/* handle an error in _PyUnicode_EncodeCharmap() Return 0 on success, -1 on error */ static int charmap_encoding_error( PyObject *unicode, Py_ssize_t *inpos, PyObject *mapping, PyObject **exceptionObject, _Py_error_handler *error_handler, PyObject **error_handler_obj, const char *errors, - PyObject **res, Py_ssize_t *respos) + PyBytesWriter *writer, Py_ssize_t *respos) { PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ Py_ssize_t size, repsize; @@ -9020,7 +8826,7 @@ charmap_encoding_error( case _Py_ERROR_REPLACE: for (collpos = collstartpos; collpos<collendpos; ++collpos) { - x = charmapencode_output('?', mapping, res, respos); + x = charmapencode_output('?', mapping, writer, respos); if (x==enc_EXCEPTION) { return -1; } @@ -9041,7 +8847,7 @@ charmap_encoding_error( char *cp; sprintf(buffer, "&#%d;", (int)PyUnicode_READ_CHAR(unicode, collpos)); for (cp = buffer; *cp; ++cp) { - x = charmapencode_output(*cp, mapping, res, respos); + x = charmapencode_output(*cp, mapping, writer, respos); if (x==enc_EXCEPTION) return -1; else if (x==enc_FAILED) { @@ -9061,17 +8867,17 @@ charmap_encoding_error( return -1; if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ - Py_ssize_t outsize = PyBytes_Size(*res); + Py_ssize_t outsize = PyBytesWriter_GetSize(writer); Py_ssize_t requiredsize; repsize = PyBytes_Size(repunicode); requiredsize = *respos + repsize; if (requiredsize > outsize) /* Make room for all additional bytes. */ - if (charmapencode_resize(res, respos, requiredsize)) { + if (charmapencode_resize(writer, respos, requiredsize)) { Py_DECREF(repunicode); return -1; } - memcpy(PyBytes_AsString(*res) + *respos, + memcpy((char*)PyBytesWriter_GetData(writer) + *respos, PyBytes_AsString(repunicode), repsize); *respos += repsize; *inpos = newpos; @@ -9084,7 +8890,7 @@ charmap_encoding_error( kind = PyUnicode_KIND(repunicode); for (index = 0; index < repsize; index++) { Py_UCS4 repch = PyUnicode_READ(kind, data, index); - x = charmapencode_output(repch, mapping, res, respos); + x = charmapencode_output(repch, mapping, writer, respos); if (x==enc_EXCEPTION) { Py_DECREF(repunicode); return -1; @@ -9106,65 +8912,105 @@ _PyUnicode_EncodeCharmap(PyObject *unicode, PyObject *mapping, const char *errors) { - /* output object */ - PyObject *res = NULL; - /* current input position */ - Py_ssize_t inpos = 0; - Py_ssize_t size; - /* current output position */ - Py_ssize_t respos = 0; + /* Default to Latin-1 */ + if (mapping == NULL) { + return unicode_encode_ucs1(unicode, errors, 256); + } + + Py_ssize_t size = PyUnicode_GET_LENGTH(unicode); + if (size == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + const void *data = PyUnicode_DATA(unicode); + int kind = PyUnicode_KIND(unicode); + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; - _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; - const void *data; - int kind; - - size = PyUnicode_GET_LENGTH(unicode); - data = PyUnicode_DATA(unicode); - kind = PyUnicode_KIND(unicode); - - /* Default to Latin-1 */ - if (mapping == NULL) - return unicode_encode_ucs1(unicode, errors, 256); + /* output object */ + PyBytesWriter *writer; /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ - res = PyBytes_FromStringAndSize(NULL, size); - if (res == NULL) + writer = PyBytesWriter_Create(size); + if (writer == NULL) { goto onError; - if (size == 0) - return res; + } - while (inpos<size) { - Py_UCS4 ch = PyUnicode_READ(kind, data, inpos); - /* try to encode it */ - charmapencode_result x = charmapencode_output(ch, mapping, &res, &respos); - if (x==enc_EXCEPTION) /* error */ - goto onError; - if (x==enc_FAILED) { /* unencodable character */ + /* current input position */ + Py_ssize_t inpos = 0; + /* current output position */ + Py_ssize_t respos = 0; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; + + if (Py_IS_TYPE(mapping, &EncodingMapType)) { + char *outstart = _PyBytesWriter_GetData(writer); + Py_ssize_t outsize = _PyBytesWriter_GetSize(writer); + + while (inpos<size) { + Py_UCS4 ch = PyUnicode_READ(kind, data, inpos); + + /* try to encode it */ + int res = encoding_map_lookup(ch, mapping); + Py_ssize_t requiredsize = respos+1; + if (res == -1) { + goto enc_FAILED; + } + + if (outsize<requiredsize) { + if (charmapencode_resize(writer, &respos, requiredsize)) { + goto onError; + } + outstart = _PyBytesWriter_GetData(writer); + outsize = _PyBytesWriter_GetSize(writer); + } + outstart[respos++] = (char)res; + + /* done with this character => adjust input position */ + ++inpos; + continue; + +enc_FAILED: if (charmap_encoding_error(unicode, &inpos, mapping, &exc, &error_handler, &error_handler_obj, errors, - &res, &respos)) { + writer, &respos)) { goto onError; } + outstart = _PyBytesWriter_GetData(writer); + outsize = _PyBytesWriter_GetSize(writer); + } + } + else { + while (inpos<size) { + Py_UCS4 ch = PyUnicode_READ(kind, data, inpos); + /* try to encode it */ + charmapencode_result x = charmapencode_output(ch, mapping, writer, &respos); + if (x==enc_EXCEPTION) { /* error */ + goto onError; + } + if (x==enc_FAILED) { /* unencodable character */ + if (charmap_encoding_error(unicode, &inpos, mapping, + &exc, + &error_handler, &error_handler_obj, errors, + writer, &respos)) { + goto onError; + } + } + else { + /* done with this character => adjust input position */ + ++inpos; + } } - else - /* done with this character => adjust input position */ - ++inpos; } - - /* Resize if we allocated to much */ - if (respos<PyBytes_GET_SIZE(res)) - if (_PyBytes_Resize(&res, respos) < 0) - goto onError; Py_XDECREF(exc); Py_XDECREF(error_handler_obj); - return res; + + /* Resize if we allocated too much */ + return PyBytesWriter_FinishWithSize(writer, respos); onError: - Py_XDECREF(res); + PyBytesWriter_Discard(writer); Py_XDECREF(exc); Py_XDECREF(error_handler_obj); return NULL; @@ -9750,142 +9596,6 @@ any_find_slice(PyObject* s1, PyObject* s2, return result; } -/* _PyUnicode_InsertThousandsGrouping() helper functions */ -#include "stringlib/localeutil.h" - -/** - * InsertThousandsGrouping: - * @writer: Unicode writer. - * @n_buffer: Number of characters in @buffer. - * @digits: Digits we're reading from. If count is non-NULL, this is unused. - * @d_pos: Start of digits string. - * @n_digits: The number of digits in the string, in which we want - * to put the grouping chars. - * @min_width: The minimum width of the digits in the output string. - * Output will be zero-padded on the left to fill. - * @grouping: see definition in localeconv(). - * @thousands_sep: see definition in localeconv(). - * - * There are 2 modes: counting and filling. If @writer is NULL, - * we are in counting mode, else filling mode. - * If counting, the required buffer size is returned. - * If filling, we know the buffer will be large enough, so we don't - * need to pass in the buffer size. - * Inserts thousand grouping characters (as defined by grouping and - * thousands_sep) into @writer. - * - * Return value: -1 on error, number of characters otherwise. - **/ -Py_ssize_t -_PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar, - int forward) -{ - min_width = Py_MAX(0, min_width); - if (writer) { - assert(digits != NULL); - assert(maxchar == NULL); - } - else { - assert(digits == NULL); - assert(maxchar != NULL); - } - assert(0 <= d_pos); - assert(0 <= n_digits); - assert(grouping != NULL); - - Py_ssize_t count = 0; - Py_ssize_t n_zeros; - int loop_broken = 0; - int use_separator = 0; /* First time through, don't append the - separator. They only go between - groups. */ - Py_ssize_t buffer_pos; - Py_ssize_t digits_pos; - Py_ssize_t len; - Py_ssize_t n_chars; - Py_ssize_t remaining = n_digits; /* Number of chars remaining to - be looked at */ - /* A generator that returns all of the grouping widths, until it - returns 0. */ - GroupGenerator groupgen; - GroupGenerator_init(&groupgen, grouping); - const Py_ssize_t thousands_sep_len = PyUnicode_GET_LENGTH(thousands_sep); - - /* if digits are not grouped, thousands separator - should be an empty string */ - assert(!(grouping[0] == CHAR_MAX && thousands_sep_len != 0)); - - digits_pos = d_pos + (forward ? 0 : n_digits); - if (writer) { - buffer_pos = writer->pos + (forward ? 0 : n_buffer); - assert(buffer_pos <= PyUnicode_GET_LENGTH(writer->buffer)); - assert(digits_pos <= PyUnicode_GET_LENGTH(digits)); - } - else { - buffer_pos = forward ? 0 : n_buffer; - } - - if (!writer) { - *maxchar = 127; - } - - while ((len = GroupGenerator_next(&groupgen)) > 0) { - len = Py_MIN(len, Py_MAX(Py_MAX(remaining, min_width), 1)); - n_zeros = Py_MAX(0, len - remaining); - n_chars = Py_MAX(0, Py_MIN(remaining, len)); - - /* Use n_zero zero's and n_chars chars */ - - /* Count only, don't do anything. */ - count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; - - /* Copy into the writer. */ - InsertThousandsGrouping_fill(writer, &buffer_pos, - digits, &digits_pos, - n_chars, n_zeros, - use_separator ? thousands_sep : NULL, - thousands_sep_len, maxchar, forward); - - /* Use a separator next time. */ - use_separator = 1; - - remaining -= n_chars; - min_width -= len; - - if (remaining <= 0 && min_width <= 0) { - loop_broken = 1; - break; - } - min_width -= thousands_sep_len; - } - if (!loop_broken) { - /* We left the loop without using a break statement. */ - - len = Py_MAX(Py_MAX(remaining, min_width), 1); - n_zeros = Py_MAX(0, len - remaining); - n_chars = Py_MAX(0, Py_MIN(remaining, len)); - - /* Use n_zero zero's and n_chars chars */ - count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; - - /* Copy into the writer. */ - InsertThousandsGrouping_fill(writer, &buffer_pos, - digits, &digits_pos, - n_chars, n_zeros, - use_separator ? thousands_sep : NULL, - thousands_sep_len, maxchar, forward); - } - return count; -} Py_ssize_t PyUnicode_Count(PyObject *str, @@ -10438,11 +10148,11 @@ _PyUnicode_FastFill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, { const int kind = PyUnicode_KIND(unicode); void *data = PyUnicode_DATA(unicode); - assert(unicode_modifiable(unicode)); + assert(_PyUnicode_IsModifiable(unicode)); assert(fill_char <= PyUnicode_MAX_CHAR_VALUE(unicode)); assert(start >= 0); assert(start + length <= PyUnicode_GET_LENGTH(unicode)); - unicode_fill(kind, data, fill_char, start, length); + _PyUnicode_Fill(kind, data, fill_char, start, length); } Py_ssize_t @@ -10511,9 +10221,10 @@ pad(PyObject *self, kind = PyUnicode_KIND(u); data = PyUnicode_DATA(u); if (left) - unicode_fill(kind, data, fill, 0, left); + _PyUnicode_Fill(kind, data, fill, 0, left); if (right) - unicode_fill(kind, data, fill, left + _PyUnicode_LENGTH(self), right); + _PyUnicode_Fill(kind, data, fill, + left + _PyUnicode_LENGTH(self), right); _PyUnicode_FastCopyCharacters(u, left, self, 0, _PyUnicode_LENGTH(self)); assert(_PyUnicode_CheckConsistency(u, 1)); return u; @@ -10965,7 +10676,7 @@ replace(PyObject *self, PyObject *str1, } new_size = slen + n * (len2 - len1); if (new_size == 0) { - u = unicode_get_empty(); + u = _PyUnicode_GetEmpty(); goto done; } if (new_size > (PY_SSIZE_T_MAX / rkind)) { @@ -11078,6 +10789,7 @@ replace(PyObject *self, PyObject *str1, /* --- Unicode Object Methods --------------------------------------------- */ /*[clinic input] +@permit_long_docstring_body str.title as unicode_title Return a version of the string where each word is titlecased. @@ -11088,12 +10800,13 @@ cased characters have lower case. static PyObject * unicode_title_impl(PyObject *self) -/*[clinic end generated code: output=c75ae03809574902 input=fa945d669b26e683]*/ +/*[clinic end generated code: output=c75ae03809574902 input=533ce0eb6a7f5d1b]*/ { return case_operation(self, do_title); } /*[clinic input] +@permit_long_docstring_body str.capitalize as unicode_capitalize Return a capitalized version of the string. @@ -11104,7 +10817,7 @@ case. static PyObject * unicode_capitalize_impl(PyObject *self) -/*[clinic end generated code: output=e49a4c333cdb7667 input=f4cbf1016938da6d]*/ +/*[clinic end generated code: output=e49a4c333cdb7667 input=a4a15ade41f6f9e9]*/ { if (PyUnicode_GET_LENGTH(self) == 0) return unicode_result_unchanged(self); @@ -11481,47 +11194,6 @@ _PyUnicode_EqualToASCIIString(PyObject *unicode, const char *str) memcmp(PyUnicode_1BYTE_DATA(unicode), str, len) == 0; } -int -_PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) -{ - PyObject *right_uni; - - assert(_PyUnicode_CHECK(left)); - assert(right->string); -#ifndef NDEBUG - for (const char *p = right->string; *p; p++) { - assert((unsigned char)*p < 128); - } -#endif - - if (!PyUnicode_IS_ASCII(left)) - return 0; - - right_uni = _PyUnicode_FromId(right); /* borrowed */ - if (right_uni == NULL) { - /* memory error or bad data */ - PyErr_Clear(); - return _PyUnicode_EqualToASCIIString(left, right->string); - } - - if (left == right_uni) - return 1; - - assert(PyUnicode_CHECK_INTERNED(right_uni)); - if (PyUnicode_CHECK_INTERNED(left)) { - return 0; - } - - Py_hash_t right_hash = PyUnicode_HASH(right_uni); - assert(right_hash != -1); - Py_hash_t hash = PyUnicode_HASH(left); - if (hash != -1 && hash != right_hash) { - return 0; - } - - return unicode_eq(left, right_uni); -} - PyObject * PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) { @@ -11629,20 +11301,14 @@ PyUnicode_Concat(PyObject *left, PyObject *right) return NULL; if (!PyUnicode_Check(right)) { - if (_PyTemplate_CheckExact(right)) { - // str + tstring is implemented in the tstring type - return _PyTemplate_Concat(left, right); - } - else { - PyErr_Format(PyExc_TypeError, - "can only concatenate str (not \"%.200s\") to str", - Py_TYPE(right)->tp_name); - return NULL; - } + PyErr_Format(PyExc_TypeError, + "can only concatenate str (not \"%.200s\") to str", + Py_TYPE(right)->tp_name); + return NULL; } /* Shortcuts */ - PyObject *empty = unicode_get_empty(); // Borrowed reference + PyObject *empty = _PyUnicode_GetEmpty(); // Borrowed reference if (left == empty) { return PyUnicode_FromObject(right); } @@ -11694,7 +11360,7 @@ PyUnicode_Append(PyObject **p_left, PyObject *right) } /* Shortcuts */ - PyObject *empty = unicode_get_empty(); // Borrowed reference + PyObject *empty = _PyUnicode_GetEmpty(); // Borrowed reference if (left == empty) { Py_DECREF(left); *p_left = Py_NewRef(right); @@ -11713,7 +11379,7 @@ PyUnicode_Append(PyObject **p_left, PyObject *right) } new_len = left_len + right_len; - if (unicode_modifiable(left) + if (_PyUnicode_IsModifiable(left) && PyUnicode_CheckExact(right) && PyUnicode_KIND(right) <= PyUnicode_KIND(left) /* Don't resize for ascii += latin1. Convert ascii to latin1 requires @@ -11758,6 +11424,7 @@ PyUnicode_AppendAndDel(PyObject **pleft, PyObject *right) } /*[clinic input] +@permit_long_summary @text_signature "($self, sub[, start[, end]], /)" str.count as unicode_count -> Py_ssize_t @@ -11775,7 +11442,7 @@ Optional arguments start and end are interpreted as in slice notation. static Py_ssize_t unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=8fcc3aef0b18edbf input=6f168ffd94be8785]*/ +/*[clinic end generated code: output=8fcc3aef0b18edbf input=8590716ee228b935]*/ { assert(PyUnicode_Check(str)); assert(PyUnicode_Check(substr)); @@ -11928,7 +11595,7 @@ unicode_expandtabs_impl(PyObject *self, int tabsize) if (tabsize > 0) { incr = tabsize - (line_pos % tabsize); line_pos += incr; - unicode_fill(kind, dest_data, ' ', j, incr); + _PyUnicode_Fill(kind, dest_data, ' ', j, incr); j += incr; } } @@ -11949,6 +11616,7 @@ unicode_expandtabs_impl(PyObject *self, int tabsize) } /*[clinic input] +@permit_long_summary str.find as unicode_find = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. @@ -11960,7 +11628,7 @@ Return -1 on failure. static Py_ssize_t unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=51dbe6255712e278 input=4a89d2d68ef57256]*/ +/*[clinic end generated code: output=51dbe6255712e278 input=3a9d650fe4c24695]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result < 0) { @@ -12012,6 +11680,7 @@ unicode_hash(PyObject *self) } /*[clinic input] +@permit_long_summary str.index as unicode_index = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. @@ -12023,7 +11692,7 @@ Raises ValueError when the substring is not found. static Py_ssize_t unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=77558288837cdf40 input=d986aeac0be14a1c]*/ +/*[clinic end generated code: output=77558288837cdf40 input=ae5e48f69ed75b06]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result == -1) { @@ -12052,6 +11721,7 @@ unicode_isascii_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.islower as unicode_islower Return True if the string is a lowercase string, False otherwise. @@ -12062,7 +11732,7 @@ there is at least one cased character in the string. static PyObject * unicode_islower_impl(PyObject *self) -/*[clinic end generated code: output=dbd41995bd005b81 input=acec65ac6821ae47]*/ +/*[clinic end generated code: output=dbd41995bd005b81 input=c6fc0295241a1aaa]*/ { Py_ssize_t i, length; int kind; @@ -12095,6 +11765,7 @@ unicode_islower_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.isupper as unicode_isupper Return True if the string is an uppercase string, False otherwise. @@ -12105,7 +11776,7 @@ there is at least one cased character in the string. static PyObject * unicode_isupper_impl(PyObject *self) -/*[clinic end generated code: output=049209c8e7f15f59 input=e9b1feda5d17f2d3]*/ +/*[clinic end generated code: output=049209c8e7f15f59 input=8d5cb33e67efde72]*/ { Py_ssize_t i, length; int kind; @@ -12194,6 +11865,7 @@ unicode_istitle_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.isspace as unicode_isspace Return True if the string is a whitespace string, False otherwise. @@ -12204,7 +11876,7 @@ is at least one character in the string. static PyObject * unicode_isspace_impl(PyObject *self) -/*[clinic end generated code: output=163a63bfa08ac2b9 input=fe462cb74f8437d8]*/ +/*[clinic end generated code: output=163a63bfa08ac2b9 input=44fe05e248c6e159]*/ { Py_ssize_t i, length; int kind; @@ -12232,6 +11904,7 @@ unicode_isspace_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.isalpha as unicode_isalpha Return True if the string is an alphabetic string, False otherwise. @@ -12242,7 +11915,7 @@ is at least one character in the string. static PyObject * unicode_isalpha_impl(PyObject *self) -/*[clinic end generated code: output=cc81b9ac3883ec4f input=d0fd18a96cbca5eb]*/ +/*[clinic end generated code: output=cc81b9ac3883ec4f input=c233000624a56e0d]*/ { Py_ssize_t i, length; int kind; @@ -12269,6 +11942,7 @@ unicode_isalpha_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.isalnum as unicode_isalnum Return True if the string is an alpha-numeric string, False otherwise. @@ -12279,7 +11953,7 @@ there is at least one character in the string. static PyObject * unicode_isalnum_impl(PyObject *self) -/*[clinic end generated code: output=a5a23490ffc3660c input=5c6579bf2e04758c]*/ +/*[clinic end generated code: output=a5a23490ffc3660c input=5d63ba9c9bafdb6b]*/ { int kind; const void *data; @@ -12308,6 +11982,7 @@ unicode_isalnum_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.isdecimal as unicode_isdecimal Return True if the string is a decimal string, False otherwise. @@ -12318,7 +11993,7 @@ there is at least one character in the string. static PyObject * unicode_isdecimal_impl(PyObject *self) -/*[clinic end generated code: output=fb2dcdb62d3fc548 input=336bc97ab4c8268f]*/ +/*[clinic end generated code: output=fb2dcdb62d3fc548 input=8e84a58b414935a3]*/ { Py_ssize_t i, length; int kind; @@ -12345,6 +12020,7 @@ unicode_isdecimal_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.isdigit as unicode_isdigit Return True if the string is a digit string, False otherwise. @@ -12355,7 +12031,7 @@ is at least one character in the string. static PyObject * unicode_isdigit_impl(PyObject *self) -/*[clinic end generated code: output=10a6985311da6858 input=901116c31deeea4c]*/ +/*[clinic end generated code: output=10a6985311da6858 input=99e284affb54d4a0]*/ { Py_ssize_t i, length; int kind; @@ -12383,6 +12059,7 @@ unicode_isdigit_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.isnumeric as unicode_isnumeric Return True if the string is a numeric string, False otherwise. @@ -12393,7 +12070,7 @@ least one character in the string. static PyObject * unicode_isnumeric_impl(PyObject *self) -/*[clinic end generated code: output=9172a32d9013051a input=722507db976f826c]*/ +/*[clinic end generated code: output=9172a32d9013051a input=e9f5b6b8b29b0ee6]*/ { Py_ssize_t i, length; int kind; @@ -12463,6 +12140,7 @@ PyUnicode_IsIdentifier(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.isidentifier as unicode_isidentifier Return True if the string is a valid Python identifier, False otherwise. @@ -12473,12 +12151,13 @@ such as "def" or "class". static PyObject * unicode_isidentifier_impl(PyObject *self) -/*[clinic end generated code: output=fe585a9666572905 input=2d807a104f21c0c5]*/ +/*[clinic end generated code: output=fe585a9666572905 input=86315dd889d7bd04]*/ { return PyBool_FromLong(PyUnicode_IsIdentifier(self)); } /*[clinic input] +@permit_long_summary str.isprintable as unicode_isprintable Return True if all characters in the string are printable, False otherwise. @@ -12488,7 +12167,7 @@ A character is printable if repr() may use it in its output. static PyObject * unicode_isprintable_impl(PyObject *self) -/*[clinic end generated code: output=3ab9626cd32dd1a0 input=4e56bcc6b06ca18c]*/ +/*[clinic end generated code: output=3ab9626cd32dd1a0 input=18345ba847084ec5]*/ { Py_ssize_t i, length; int kind; @@ -12512,6 +12191,7 @@ unicode_isprintable_impl(PyObject *self) } /*[clinic input] +@permit_long_docstring_body str.join as unicode_join iterable: object @@ -12527,7 +12207,7 @@ Example: '.'.join(['ab', 'pq', 'rs']) -> 'ab.pq.rs' static PyObject * unicode_join(PyObject *self, PyObject *iterable) -/*[clinic end generated code: output=6857e7cecfe7bf98 input=2f70422bfb8fa189]*/ +/*[clinic end generated code: output=6857e7cecfe7bf98 input=bac724ed412ef3f8]*/ { return PyUnicode_Join(self, iterable); } @@ -12748,6 +12428,7 @@ do_argstrip(PyObject *self, int striptype, PyObject *sep) /*[clinic input] +@permit_long_summary str.strip as unicode_strip chars: object = None @@ -12760,7 +12441,7 @@ If chars is given and not None, remove characters in chars instead. static PyObject * unicode_strip_impl(PyObject *self, PyObject *chars) -/*[clinic end generated code: output=ca19018454345d57 input=385289c6f423b954]*/ +/*[clinic end generated code: output=ca19018454345d57 input=8bc6353450345fbd]*/ { return do_argstrip(self, BOTHSTRIP, chars); } @@ -12871,6 +12552,7 @@ PyUnicode_Replace(PyObject *str, } /*[clinic input] +@permit_long_docstring_body str.replace as unicode_replace old: unicode @@ -12889,12 +12571,13 @@ replaced. static PyObject * unicode_replace_impl(PyObject *self, PyObject *old, PyObject *new, Py_ssize_t count) -/*[clinic end generated code: output=b63f1a8b5eebf448 input=3345c455d60a5499]*/ +/*[clinic end generated code: output=b63f1a8b5eebf448 input=f27ca92ac46b65a1]*/ { return replace(self, old, new, count); } /*[clinic input] +@permit_long_docstring_body str.removeprefix as unicode_removeprefix prefix: unicode @@ -12908,7 +12591,7 @@ Otherwise, return a copy of the original string. static PyObject * unicode_removeprefix_impl(PyObject *self, PyObject *prefix) -/*[clinic end generated code: output=f1e5945e9763bcb9 input=27ec40b99a37eb88]*/ +/*[clinic end generated code: output=f1e5945e9763bcb9 input=1989a856dbb813f1]*/ { int match = tailmatch(self, prefix, 0, PY_SSIZE_T_MAX, -1); if (match == -1) { @@ -13041,6 +12724,7 @@ unicode_repr(PyObject *unicode) } /*[clinic input] +@permit_long_summary str.rfind as unicode_rfind = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. @@ -13052,7 +12736,7 @@ Return -1 on failure. static Py_ssize_t unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=880b29f01dd014c8 input=898361fb71f59294]*/ +/*[clinic end generated code: output=880b29f01dd014c8 input=7f7e97d5cd3299a2]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result < 0) { @@ -13062,6 +12746,7 @@ unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, } /*[clinic input] +@permit_long_summary str.rindex as unicode_rindex = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. @@ -13073,7 +12758,7 @@ Raises ValueError when the substring is not found. static Py_ssize_t unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=5f3aef124c867fe1 input=35943dead6c1ea9d]*/ +/*[clinic end generated code: output=5f3aef124c867fe1 input=0363a324740b3e62]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result == -1) { @@ -13117,6 +12802,7 @@ PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) } /*[clinic input] +@permit_long_summary str.split as unicode_split sep: object = None @@ -13141,7 +12827,7 @@ the regular expression module. static PyObject * unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=a29bcc0c7a5af0eb]*/ +/*[clinic end generated code: output=3a65b1db356948dc input=2c1fd08a78e038b8]*/ { if (sep == Py_None) return split(self, NULL, maxsplit); @@ -13170,7 +12856,7 @@ PyUnicode_Partition(PyObject *str_obj, PyObject *sep_obj) len1 = PyUnicode_GET_LENGTH(str_obj); len2 = PyUnicode_GET_LENGTH(sep_obj); if (kind1 < kind2 || len1 < len2) { - PyObject *empty = unicode_get_empty(); // Borrowed reference + PyObject *empty = _PyUnicode_GetEmpty(); // Borrowed reference return PyTuple_Pack(3, str_obj, empty, empty); } buf1 = PyUnicode_DATA(str_obj); @@ -13222,7 +12908,7 @@ PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj) len1 = PyUnicode_GET_LENGTH(str_obj); len2 = PyUnicode_GET_LENGTH(sep_obj); if (kind1 < kind2 || len1 < len2) { - PyObject *empty = unicode_get_empty(); // Borrowed reference + PyObject *empty = _PyUnicode_GetEmpty(); // Borrowed reference return PyTuple_Pack(3, empty, empty, str_obj); } buf1 = PyUnicode_DATA(str_obj); @@ -13258,6 +12944,7 @@ PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj) } /*[clinic input] +@permit_long_docstring_body str.partition as unicode_partition sep: object @@ -13275,12 +12962,13 @@ and two empty strings. static PyObject * unicode_partition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=e4ced7bd253ca3c4 input=f29b8d06c63e50be]*/ +/*[clinic end generated code: output=e4ced7bd253ca3c4 input=4d854b520d7b0e97]*/ { return PyUnicode_Partition(self, sep); } /*[clinic input] +@permit_long_docstring_body str.rpartition as unicode_rpartition = str.partition Partition the string into three parts using the given separator. @@ -13295,7 +12983,7 @@ and the original string. static PyObject * unicode_rpartition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=1aa13cf1156572aa input=c4b7db3ef5cf336a]*/ +/*[clinic end generated code: output=1aa13cf1156572aa input=a6adabe91e75b486]*/ { return PyUnicode_RPartition(self, sep); } @@ -13310,6 +12998,7 @@ PyUnicode_RSplit(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) } /*[clinic input] +@permit_long_summary str.rsplit as unicode_rsplit = str.split Return a list of the substrings in the string, using sep as the separator string. @@ -13319,7 +13008,7 @@ Splitting starts at the end of the string and works to the front. static PyObject * unicode_rsplit_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=c2b815c63bcabffc input=ea78406060fce33c]*/ +/*[clinic end generated code: output=c2b815c63bcabffc input=0f762e30d267fa83]*/ { if (sep == Py_None) return rsplit(self, NULL, maxsplit); @@ -13333,6 +13022,7 @@ unicode_rsplit_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) } /*[clinic input] +@permit_long_docstring_body str.splitlines as unicode_splitlines keepends: bool = False @@ -13345,7 +13035,7 @@ true. static PyObject * unicode_splitlines_impl(PyObject *self, int keepends) -/*[clinic end generated code: output=f664dcdad153ec40 input=ba6ad05ee85d2b55]*/ +/*[clinic end generated code: output=f664dcdad153ec40 input=39eeafbfef61c827]*/ { return PyUnicode_Splitlines(self, keepends); } @@ -13357,6 +13047,7 @@ PyObject *unicode_str(PyObject *self) } /*[clinic input] +@permit_long_summary str.swapcase as unicode_swapcase Convert uppercase characters to lowercase and lowercase characters to uppercase. @@ -13364,7 +13055,7 @@ Convert uppercase characters to lowercase and lowercase characters to uppercase. static PyObject * unicode_swapcase_impl(PyObject *self) -/*[clinic end generated code: output=5d28966bf6d7b2af input=3f3ef96d5798a7bb]*/ +/*[clinic end generated code: output=5d28966bf6d7b2af input=85bc39a9b4e8ee91]*/ { return case_operation(self, do_swapcase); } @@ -13500,6 +13191,7 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) } /*[clinic input] +@permit_long_docstring_body str.translate as unicode_translate table: object @@ -13516,7 +13208,7 @@ left untouched. Characters mapped to None are deleted. static PyObject * unicode_translate(PyObject *self, PyObject *table) -/*[clinic end generated code: output=3cb448ff2fd96bf3 input=6d38343db63d8eb0]*/ +/*[clinic end generated code: output=3cb448ff2fd96bf3 input=699e5fa0ebf9f5e9]*/ { return _PyUnicode_TranslateCharmap(self, table, "ignore"); } @@ -13537,6 +13229,7 @@ unicode_upper_impl(PyObject *self) } /*[clinic input] +@permit_long_summary str.zfill as unicode_zfill width: Py_ssize_t @@ -13549,7 +13242,7 @@ The string is never truncated. static PyObject * unicode_zfill_impl(PyObject *self, Py_ssize_t width) -/*[clinic end generated code: output=e13fb6bdf8e3b9df input=c6b2f772c6f27799]*/ +/*[clinic end generated code: output=e13fb6bdf8e3b9df input=25a4ee0ea3e58ce0]*/ { Py_ssize_t fill; PyObject *u; @@ -13582,6 +13275,7 @@ unicode_zfill_impl(PyObject *self, Py_ssize_t width) } /*[clinic input] +@permit_long_summary @text_signature "($self, prefix[, start[, end]], /)" str.startswith as unicode_startswith @@ -13599,7 +13293,7 @@ Return True if the string starts with the specified prefix, False otherwise. static PyObject * unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=4bd7cfd0803051d4 input=5f918b5f5f89d856]*/ +/*[clinic end generated code: output=4bd7cfd0803051d4 input=766bdbd33df251dc]*/ { if (PyTuple_Check(subobj)) { Py_ssize_t i; @@ -13638,6 +13332,7 @@ unicode_startswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, /*[clinic input] +@permit_long_summary @text_signature "($self, suffix[, start[, end]], /)" str.endswith as unicode_endswith @@ -13655,7 +13350,7 @@ Return True if the string ends with the specified suffix, False otherwise. static PyObject * unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=cce6f8ceb0102ca9 input=00fbdc774a7d4d71]*/ +/*[clinic end generated code: output=cce6f8ceb0102ca9 input=b66bf6d5547ba1aa]*/ { if (PyTuple_Check(subobj)) { Py_ssize_t i; @@ -13692,515 +13387,6 @@ unicode_endswith_impl(PyObject *self, PyObject *subobj, Py_ssize_t start, } -static inline void -_PyUnicodeWriter_Update(_PyUnicodeWriter *writer) -{ - writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); - writer->data = PyUnicode_DATA(writer->buffer); - - if (!writer->readonly) { - writer->kind = PyUnicode_KIND(writer->buffer); - writer->size = PyUnicode_GET_LENGTH(writer->buffer); - } - else { - /* use a value smaller than PyUnicode_1BYTE_KIND() so - _PyUnicodeWriter_PrepareKind() will copy the buffer. */ - writer->kind = 0; - assert(writer->kind <= PyUnicode_1BYTE_KIND); - - /* Copy-on-write mode: set buffer size to 0 so - * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on - * next write. */ - writer->size = 0; - } -} - - -void -_PyUnicodeWriter_Init(_PyUnicodeWriter *writer) -{ - memset(writer, 0, sizeof(*writer)); - - /* ASCII is the bare minimum */ - writer->min_char = 127; - - /* use a kind value smaller than PyUnicode_1BYTE_KIND so - _PyUnicodeWriter_PrepareKind() will copy the buffer. */ - assert(writer->kind == 0); - assert(writer->kind < PyUnicode_1BYTE_KIND); -} - - -PyUnicodeWriter* -PyUnicodeWriter_Create(Py_ssize_t length) -{ - if (length < 0) { - PyErr_SetString(PyExc_ValueError, - "length must be positive"); - return NULL; - } - - const size_t size = sizeof(_PyUnicodeWriter); - PyUnicodeWriter *pub_writer; - pub_writer = _Py_FREELIST_POP_MEM(unicode_writers); - if (pub_writer == NULL) { - pub_writer = (PyUnicodeWriter *)PyMem_Malloc(size); - if (pub_writer == NULL) { - return (PyUnicodeWriter *)PyErr_NoMemory(); - } - } - _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; - - _PyUnicodeWriter_Init(writer); - if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { - PyUnicodeWriter_Discard(pub_writer); - return NULL; - } - writer->overallocate = 1; - - return pub_writer; -} - - -void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) -{ - if (writer == NULL) { - return; - } - _PyUnicodeWriter_Dealloc((_PyUnicodeWriter*)writer); - _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); -} - - -// Initialize _PyUnicodeWriter with initial buffer -static inline void -_PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) -{ - memset(writer, 0, sizeof(*writer)); - writer->buffer = buffer; - _PyUnicodeWriter_Update(writer); - writer->min_length = writer->size; -} - - -int -_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar) -{ - Py_ssize_t newlen; - PyObject *newbuffer; - - assert(length >= 0); - assert(maxchar <= MAX_UNICODE); - - /* ensure that the _PyUnicodeWriter_Prepare macro was used */ - assert((maxchar > writer->maxchar && length >= 0) - || length > 0); - - if (length > PY_SSIZE_T_MAX - writer->pos) { - PyErr_NoMemory(); - return -1; - } - newlen = writer->pos + length; - - maxchar = Py_MAX(maxchar, writer->min_char); - - if (writer->buffer == NULL) { - assert(!writer->readonly); - if (writer->overallocate - && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - newlen += newlen / OVERALLOCATE_FACTOR; - } - if (newlen < writer->min_length) - newlen = writer->min_length; - - writer->buffer = PyUnicode_New(newlen, maxchar); - if (writer->buffer == NULL) - return -1; - } - else if (newlen > writer->size) { - if (writer->overallocate - && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - newlen += newlen / OVERALLOCATE_FACTOR; - } - if (newlen < writer->min_length) - newlen = writer->min_length; - - if (maxchar > writer->maxchar || writer->readonly) { - /* resize + widen */ - maxchar = Py_MAX(maxchar, writer->maxchar); - newbuffer = PyUnicode_New(newlen, maxchar); - if (newbuffer == NULL) - return -1; - _PyUnicode_FastCopyCharacters(newbuffer, 0, - writer->buffer, 0, writer->pos); - Py_DECREF(writer->buffer); - writer->readonly = 0; - } - else { - newbuffer = resize_compact(writer->buffer, newlen); - if (newbuffer == NULL) - return -1; - } - writer->buffer = newbuffer; - } - else if (maxchar > writer->maxchar) { - assert(!writer->readonly); - newbuffer = PyUnicode_New(writer->size, maxchar); - if (newbuffer == NULL) - return -1; - _PyUnicode_FastCopyCharacters(newbuffer, 0, - writer->buffer, 0, writer->pos); - Py_SETREF(writer->buffer, newbuffer); - } - _PyUnicodeWriter_Update(writer); - return 0; - -#undef OVERALLOCATE_FACTOR -} - -int -_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, - int kind) -{ - Py_UCS4 maxchar; - - /* ensure that the _PyUnicodeWriter_PrepareKind macro was used */ - assert(writer->kind < kind); - - switch (kind) - { - case PyUnicode_1BYTE_KIND: maxchar = 0xff; break; - case PyUnicode_2BYTE_KIND: maxchar = 0xffff; break; - case PyUnicode_4BYTE_KIND: maxchar = MAX_UNICODE; break; - default: - Py_UNREACHABLE(); - } - - return _PyUnicodeWriter_PrepareInternal(writer, 0, maxchar); -} - -static inline int -_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) -{ - assert(ch <= MAX_UNICODE); - if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) - return -1; - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); - writer->pos++; - return 0; -} - -int -_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, Py_UCS4 ch) -{ - return _PyUnicodeWriter_WriteCharInline(writer, ch); -} - -int -PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) -{ - if (ch > MAX_UNICODE) { - PyErr_SetString(PyExc_ValueError, - "character must be in range(0x110000)"); - return -1; - } - - return _PyUnicodeWriter_WriteChar((_PyUnicodeWriter*)writer, ch); -} - -int -_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) -{ - assert(PyUnicode_Check(str)); - - Py_UCS4 maxchar; - Py_ssize_t len; - - len = PyUnicode_GET_LENGTH(str); - if (len == 0) - return 0; - maxchar = PyUnicode_MAX_CHAR_VALUE(str); - if (maxchar > writer->maxchar || len > writer->size - writer->pos) { - if (writer->buffer == NULL && !writer->overallocate) { - assert(_PyUnicode_CheckConsistency(str, 1)); - writer->readonly = 1; - writer->buffer = Py_NewRef(str); - _PyUnicodeWriter_Update(writer); - writer->pos += len; - return 0; - } - if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1) - return -1; - } - _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, - str, 0, len); - writer->pos += len; - return 0; -} - -int -PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) -{ - if (Py_TYPE(obj) == &PyLong_Type) { - return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); - } - - PyObject *str = PyObject_Str(obj); - if (str == NULL) { - return -1; - } - - int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, str); - Py_DECREF(str); - return res; -} - - -int -PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) -{ - if (Py_TYPE(obj) == &PyLong_Type) { - return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); - } - - PyObject *repr = PyObject_Repr(obj); - if (repr == NULL) { - return -1; - } - - int res = _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, repr); - Py_DECREF(repr); - return res; -} - - -int -_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, - Py_ssize_t start, Py_ssize_t end) -{ - assert(0 <= start); - assert(end <= PyUnicode_GET_LENGTH(str)); - assert(start <= end); - - if (start == 0 && end == PyUnicode_GET_LENGTH(str)) - return _PyUnicodeWriter_WriteStr(writer, str); - - Py_ssize_t len = end - start; - if (len == 0) { - return 0; - } - - Py_UCS4 maxchar; - if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) { - maxchar = _PyUnicode_FindMaxChar(str, start, end); - } - else { - maxchar = writer->maxchar; - } - if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) { - return -1; - } - - _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, - str, start, len); - writer->pos += len; - return 0; -} - - -int -PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, - Py_ssize_t start, Py_ssize_t end) -{ - if (!PyUnicode_Check(str)) { - PyErr_Format(PyExc_TypeError, "expect str, not %T", str); - return -1; - } - if (start < 0 || start > end) { - PyErr_Format(PyExc_ValueError, "invalid start argument"); - return -1; - } - if (end > PyUnicode_GET_LENGTH(str)) { - PyErr_Format(PyExc_ValueError, "invalid end argument"); - return -1; - } - - return _PyUnicodeWriter_WriteSubstring((_PyUnicodeWriter*)writer, str, - start, end); -} - - -int -_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, - const char *ascii, Py_ssize_t len) -{ - if (len == -1) - len = strlen(ascii); - - assert(ucs1lib_find_max_char((const Py_UCS1*)ascii, (const Py_UCS1*)ascii + len) < 128); - - if (writer->buffer == NULL && !writer->overallocate) { - PyObject *str; - - str = _PyUnicode_FromASCII(ascii, len); - if (str == NULL) - return -1; - - writer->readonly = 1; - writer->buffer = str; - _PyUnicodeWriter_Update(writer); - writer->pos += len; - return 0; - } - - if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) - return -1; - - switch (writer->kind) - { - case PyUnicode_1BYTE_KIND: - { - const Py_UCS1 *str = (const Py_UCS1 *)ascii; - Py_UCS1 *data = writer->data; - - memcpy(data + writer->pos, str, len); - break; - } - case PyUnicode_2BYTE_KIND: - { - _PyUnicode_CONVERT_BYTES( - Py_UCS1, Py_UCS2, - ascii, ascii + len, - (Py_UCS2 *)writer->data + writer->pos); - break; - } - case PyUnicode_4BYTE_KIND: - { - _PyUnicode_CONVERT_BYTES( - Py_UCS1, Py_UCS4, - ascii, ascii + len, - (Py_UCS4 *)writer->data + writer->pos); - break; - } - default: - Py_UNREACHABLE(); - } - - writer->pos += len; - return 0; -} - -int -PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, - const char *str, - Py_ssize_t size) -{ - if (size < 0) { - size = strlen(str); - } - - _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; - Py_ssize_t old_pos = _writer->pos; - int res = unicode_decode_utf8_writer(_writer, str, size, - _Py_ERROR_STRICT, NULL, NULL); - if (res < 0) { - _writer->pos = old_pos; - } - return res; -} - - -int -PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, - const char *string, - Py_ssize_t length, - const char *errors, - Py_ssize_t *consumed) -{ - if (length < 0) { - length = strlen(string); - } - - _PyUnicodeWriter *_writer = (_PyUnicodeWriter*)writer; - Py_ssize_t old_pos = _writer->pos; - int res = unicode_decode_utf8_writer(_writer, string, length, - _Py_ERROR_UNKNOWN, errors, consumed); - if (res < 0) { - _writer->pos = old_pos; - if (consumed) { - *consumed = 0; - } - } - return res; -} - - -int -_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, - const char *str, Py_ssize_t len) -{ - Py_UCS4 maxchar; - - maxchar = ucs1lib_find_max_char((const Py_UCS1*)str, (const Py_UCS1*)str + len); - if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1) - return -1; - unicode_write_cstr(writer->buffer, writer->pos, str, len); - writer->pos += len; - return 0; -} - -PyObject * -_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) -{ - PyObject *str; - - if (writer->pos == 0) { - Py_CLEAR(writer->buffer); - _Py_RETURN_UNICODE_EMPTY(); - } - - str = writer->buffer; - writer->buffer = NULL; - - if (writer->readonly) { - assert(PyUnicode_GET_LENGTH(str) == writer->pos); - return str; - } - - if (PyUnicode_GET_LENGTH(str) != writer->pos) { - PyObject *str2; - str2 = resize_compact(str, writer->pos); - if (str2 == NULL) { - Py_DECREF(str); - return NULL; - } - str = str2; - } - - assert(_PyUnicode_CheckConsistency(str, 1)); - return unicode_result(str); -} - - -PyObject* -PyUnicodeWriter_Finish(PyUnicodeWriter *writer) -{ - PyObject *str = _PyUnicodeWriter_Finish((_PyUnicodeWriter*)writer); - assert(((_PyUnicodeWriter*)writer)->buffer == NULL); - _Py_FREELIST_FREE(unicode_writers, writer, PyMem_Free); - return str; -} - - -void -_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) -{ - Py_CLEAR(writer->buffer); -} - #include "stringlib/unicode_format.h" PyDoc_STRVAR(format__doc__, @@ -14602,947 +13788,6 @@ static PyMappingMethods unicode_as_mapping = { }; -/* Helpers for PyUnicode_Format() */ - -struct unicode_formatter_t { - PyObject *args; - int args_owned; - Py_ssize_t arglen, argidx; - PyObject *dict; - - int fmtkind; - Py_ssize_t fmtcnt, fmtpos; - const void *fmtdata; - PyObject *fmtstr; - - _PyUnicodeWriter writer; -}; - -struct unicode_format_arg_t { - Py_UCS4 ch; - int flags; - Py_ssize_t width; - int prec; - int sign; -}; - -static PyObject * -unicode_format_getnextarg(struct unicode_formatter_t *ctx) -{ - Py_ssize_t argidx = ctx->argidx; - - if (argidx < ctx->arglen) { - ctx->argidx++; - if (ctx->arglen < 0) - return ctx->args; - else - return PyTuple_GetItem(ctx->args, argidx); - } - PyErr_SetString(PyExc_TypeError, - "not enough arguments for format string"); - return NULL; -} - -/* Returns a new reference to a PyUnicode object, or NULL on failure. */ - -/* Format a float into the writer if the writer is not NULL, or into *p_output - otherwise. - - Return 0 on success, raise an exception and return -1 on error. */ -static int -formatfloat(PyObject *v, struct unicode_format_arg_t *arg, - PyObject **p_output, - _PyUnicodeWriter *writer) -{ - char *p; - double x; - Py_ssize_t len; - int prec; - int dtoa_flags = 0; - - x = PyFloat_AsDouble(v); - if (x == -1.0 && PyErr_Occurred()) - return -1; - - prec = arg->prec; - if (prec < 0) - prec = 6; - - if (arg->flags & F_ALT) - dtoa_flags |= Py_DTSF_ALT; - p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL); - if (p == NULL) - return -1; - len = strlen(p); - if (writer) { - if (_PyUnicodeWriter_WriteASCIIString(writer, p, len) < 0) { - PyMem_Free(p); - return -1; - } - } - else - *p_output = _PyUnicode_FromASCII(p, len); - PyMem_Free(p); - return 0; -} - -/* formatlong() emulates the format codes d, u, o, x and X, and - * the F_ALT flag, for Python's long (unbounded) ints. It's not used for - * Python's regular ints. - * Return value: a new PyUnicodeObject*, or NULL if error. - * The output string is of the form - * "-"? ("0x" | "0X")? digit+ - * "0x"/"0X" are present only for x and X conversions, with F_ALT - * set in flags. The case of hex digits will be correct, - * There will be at least prec digits, zero-filled on the left if - * necessary to get that many. - * val object to be converted - * flags bitmask of format flags; only F_ALT is looked at - * prec minimum number of digits; 0-fill on left if needed - * type a character in [duoxX]; u acts the same as d - * - * CAUTION: o, x and X conversions on regular ints can never - * produce a '-' sign, but can for Python's unbounded ints. - */ -PyObject * -_PyUnicode_FormatLong(PyObject *val, int alt, int prec, int type) -{ - PyObject *result = NULL; - char *buf; - Py_ssize_t i; - int sign; /* 1 if '-', else 0 */ - int len; /* number of characters */ - Py_ssize_t llen; - int numdigits; /* len == numnondigits + numdigits */ - int numnondigits = 0; - - /* Avoid exceeding SSIZE_T_MAX */ - if (prec > INT_MAX-3) { - PyErr_SetString(PyExc_OverflowError, - "precision too large"); - return NULL; - } - - assert(PyLong_Check(val)); - - switch (type) { - default: - Py_UNREACHABLE(); - case 'd': - case 'i': - case 'u': - /* int and int subclasses should print numerically when a numeric */ - /* format code is used (see issue18780) */ - result = PyNumber_ToBase(val, 10); - break; - case 'o': - numnondigits = 2; - result = PyNumber_ToBase(val, 8); - break; - case 'x': - case 'X': - numnondigits = 2; - result = PyNumber_ToBase(val, 16); - break; - } - if (!result) - return NULL; - - assert(unicode_modifiable(result)); - assert(PyUnicode_IS_ASCII(result)); - - /* To modify the string in-place, there can only be one reference. */ - if (!_PyObject_IsUniquelyReferenced(result)) { - Py_DECREF(result); - PyErr_BadInternalCall(); - return NULL; - } - buf = PyUnicode_DATA(result); - llen = PyUnicode_GET_LENGTH(result); - if (llen > INT_MAX) { - Py_DECREF(result); - PyErr_SetString(PyExc_ValueError, - "string too large in _PyUnicode_FormatLong"); - return NULL; - } - len = (int)llen; - sign = buf[0] == '-'; - numnondigits += sign; - numdigits = len - numnondigits; - assert(numdigits > 0); - - /* Get rid of base marker unless F_ALT */ - if (((alt) == 0 && - (type == 'o' || type == 'x' || type == 'X'))) { - assert(buf[sign] == '0'); - assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || - buf[sign+1] == 'o'); - numnondigits -= 2; - buf += 2; - len -= 2; - if (sign) - buf[0] = '-'; - assert(len == numnondigits + numdigits); - assert(numdigits > 0); - } - - /* Fill with leading zeroes to meet minimum width. */ - if (prec > numdigits) { - PyObject *r1 = PyBytes_FromStringAndSize(NULL, - numnondigits + prec); - char *b1; - if (!r1) { - Py_DECREF(result); - return NULL; - } - b1 = PyBytes_AS_STRING(r1); - for (i = 0; i < numnondigits; ++i) - *b1++ = *buf++; - for (i = 0; i < prec - numdigits; i++) - *b1++ = '0'; - for (i = 0; i < numdigits; i++) - *b1++ = *buf++; - *b1 = '\0'; - Py_SETREF(result, r1); - buf = PyBytes_AS_STRING(result); - len = numnondigits + prec; - } - - /* Fix up case for hex conversions. */ - if (type == 'X') { - /* Need to convert all lower case letters to upper case. - and need to convert 0x to 0X (and -0x to -0X). */ - for (i = 0; i < len; i++) - if (buf[i] >= 'a' && buf[i] <= 'x') - buf[i] -= 'a'-'A'; - } - if (!PyUnicode_Check(result) - || buf != PyUnicode_DATA(result)) { - PyObject *unicode; - unicode = _PyUnicode_FromASCII(buf, len); - Py_SETREF(result, unicode); - } - else if (len != PyUnicode_GET_LENGTH(result)) { - if (PyUnicode_Resize(&result, len) < 0) - Py_CLEAR(result); - } - return result; -} - -/* Format an integer or a float as an integer. - * Return 1 if the number has been formatted into the writer, - * 0 if the number has been formatted into *p_output - * -1 and raise an exception on error */ -static int -mainformatlong(PyObject *v, - struct unicode_format_arg_t *arg, - PyObject **p_output, - _PyUnicodeWriter *writer) -{ - PyObject *iobj, *res; - char type = (char)arg->ch; - - if (!PyNumber_Check(v)) - goto wrongtype; - - /* make sure number is a type of integer for o, x, and X */ - if (!PyLong_Check(v)) { - if (type == 'o' || type == 'x' || type == 'X') { - iobj = _PyNumber_Index(v); - } - else { - iobj = PyNumber_Long(v); - } - if (iobj == NULL ) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - goto wrongtype; - return -1; - } - assert(PyLong_Check(iobj)); - } - else { - iobj = Py_NewRef(v); - } - - if (PyLong_CheckExact(v) - && arg->width == -1 && arg->prec == -1 - && !(arg->flags & (F_SIGN | F_BLANK)) - && type != 'X') - { - /* Fast path */ - int alternate = arg->flags & F_ALT; - int base; - - switch(type) - { - default: - Py_UNREACHABLE(); - case 'd': - case 'i': - case 'u': - base = 10; - break; - case 'o': - base = 8; - break; - case 'x': - case 'X': - base = 16; - break; - } - - if (_PyLong_FormatWriter(writer, v, base, alternate) == -1) { - Py_DECREF(iobj); - return -1; - } - Py_DECREF(iobj); - return 1; - } - - res = _PyUnicode_FormatLong(iobj, arg->flags & F_ALT, arg->prec, type); - Py_DECREF(iobj); - if (res == NULL) - return -1; - *p_output = res; - return 0; - -wrongtype: - switch(type) - { - case 'o': - case 'x': - case 'X': - PyErr_Format(PyExc_TypeError, - "%%%c format: an integer is required, " - "not %.200s", - type, Py_TYPE(v)->tp_name); - break; - default: - PyErr_Format(PyExc_TypeError, - "%%%c format: a real number is required, " - "not %.200s", - type, Py_TYPE(v)->tp_name); - break; - } - return -1; -} - -static Py_UCS4 -formatchar(PyObject *v) -{ - /* presume that the buffer is at least 3 characters long */ - if (PyUnicode_Check(v)) { - if (PyUnicode_GET_LENGTH(v) == 1) { - return PyUnicode_READ_CHAR(v, 0); - } - PyErr_Format(PyExc_TypeError, - "%%c requires an int or a unicode character, " - "not a string of length %zd", - PyUnicode_GET_LENGTH(v)); - return (Py_UCS4) -1; - } - else { - int overflow; - long x = PyLong_AsLongAndOverflow(v, &overflow); - if (x == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Format(PyExc_TypeError, - "%%c requires an int or a unicode character, not %T", - v); - return (Py_UCS4) -1; - } - return (Py_UCS4) -1; - } - - if (x < 0 || x > MAX_UNICODE) { - /* this includes an overflow in converting to C long */ - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x110000)"); - return (Py_UCS4) -1; - } - - return (Py_UCS4) x; - } -} - -/* Parse options of an argument: flags, width, precision. - Handle also "%(name)" syntax. - - Return 0 if the argument has been formatted into arg->str. - Return 1 if the argument has been written into ctx->writer, - Raise an exception and return -1 on error. */ -static int -unicode_format_arg_parse(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg) -{ -#define FORMAT_READ(ctx) \ - PyUnicode_READ((ctx)->fmtkind, (ctx)->fmtdata, (ctx)->fmtpos) - - PyObject *v; - - if (arg->ch == '(') { - /* Get argument value from a dictionary. Example: "%(name)s". */ - Py_ssize_t keystart; - Py_ssize_t keylen; - PyObject *key; - int pcount = 1; - - if (ctx->dict == NULL) { - PyErr_SetString(PyExc_TypeError, - "format requires a mapping"); - return -1; - } - ++ctx->fmtpos; - --ctx->fmtcnt; - keystart = ctx->fmtpos; - /* Skip over balanced parentheses */ - while (pcount > 0 && --ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - if (arg->ch == ')') - --pcount; - else if (arg->ch == '(') - ++pcount; - ctx->fmtpos++; - } - keylen = ctx->fmtpos - keystart - 1; - if (ctx->fmtcnt < 0 || pcount > 0) { - PyErr_SetString(PyExc_ValueError, - "incomplete format key"); - return -1; - } - key = PyUnicode_Substring(ctx->fmtstr, - keystart, keystart + keylen); - if (key == NULL) - return -1; - if (ctx->args_owned) { - ctx->args_owned = 0; - Py_DECREF(ctx->args); - } - ctx->args = PyObject_GetItem(ctx->dict, key); - Py_DECREF(key); - if (ctx->args == NULL) - return -1; - ctx->args_owned = 1; - ctx->arglen = -1; - ctx->argidx = -2; - } - - /* Parse flags. Example: "%+i" => flags=F_SIGN. */ - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - switch (arg->ch) { - case '-': arg->flags |= F_LJUST; continue; - case '+': arg->flags |= F_SIGN; continue; - case ' ': arg->flags |= F_BLANK; continue; - case '#': arg->flags |= F_ALT; continue; - case '0': arg->flags |= F_ZERO; continue; - } - break; - } - - /* Parse width. Example: "%10s" => width=10 */ - if (arg->ch == '*') { - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - if (!PyLong_Check(v)) { - PyErr_SetString(PyExc_TypeError, - "* wants int"); - return -1; - } - arg->width = PyLong_AsSsize_t(v); - if (arg->width == -1 && PyErr_Occurred()) - return -1; - if (arg->width < 0) { - arg->flags |= F_LJUST; - arg->width = -arg->width; - } - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } - } - else if (arg->ch >= '0' && arg->ch <= '9') { - arg->width = arg->ch - '0'; - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - if (arg->ch < '0' || arg->ch > '9') - break; - /* Since arg->ch is unsigned, the RHS would end up as unsigned, - mixing signed and unsigned comparison. Since arg->ch is between - '0' and '9', casting to int is safe. */ - if (arg->width > (PY_SSIZE_T_MAX - ((int)arg->ch - '0')) / 10) { - PyErr_SetString(PyExc_ValueError, - "width too big"); - return -1; - } - arg->width = arg->width*10 + (arg->ch - '0'); - } - } - - /* Parse precision. Example: "%.3f" => prec=3 */ - if (arg->ch == '.') { - arg->prec = 0; - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } - if (arg->ch == '*') { - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - if (!PyLong_Check(v)) { - PyErr_SetString(PyExc_TypeError, - "* wants int"); - return -1; - } - arg->prec = PyLong_AsInt(v); - if (arg->prec == -1 && PyErr_Occurred()) - return -1; - if (arg->prec < 0) - arg->prec = 0; - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } - } - else if (arg->ch >= '0' && arg->ch <= '9') { - arg->prec = arg->ch - '0'; - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - if (arg->ch < '0' || arg->ch > '9') - break; - if (arg->prec > (INT_MAX - ((int)arg->ch - '0')) / 10) { - PyErr_SetString(PyExc_ValueError, - "precision too big"); - return -1; - } - arg->prec = arg->prec*10 + (arg->ch - '0'); - } - } - } - - /* Ignore "h", "l" and "L" format prefix (ex: "%hi" or "%ls") */ - if (ctx->fmtcnt >= 0) { - if (arg->ch == 'h' || arg->ch == 'l' || arg->ch == 'L') { - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } - } - } - if (ctx->fmtcnt < 0) { - PyErr_SetString(PyExc_ValueError, - "incomplete format"); - return -1; - } - return 0; - -#undef FORMAT_READ -} - -/* Format one argument. Supported conversion specifiers: - - - "s", "r", "a": any type - - "i", "d", "u": int or float - - "o", "x", "X": int - - "e", "E", "f", "F", "g", "G": float - - "c": int or str (1 character) - - When possible, the output is written directly into the Unicode writer - (ctx->writer). A string is created when padding is required. - - Return 0 if the argument has been formatted into *p_str, - 1 if the argument has been written into ctx->writer, - -1 on error. */ -static int -unicode_format_arg_format(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg, - PyObject **p_str) -{ - PyObject *v; - _PyUnicodeWriter *writer = &ctx->writer; - - if (ctx->fmtcnt == 0) - ctx->writer.overallocate = 0; - - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - - - switch (arg->ch) { - case 's': - case 'r': - case 'a': - if (PyLong_CheckExact(v) && arg->width == -1 && arg->prec == -1) { - /* Fast path */ - if (_PyLong_FormatWriter(writer, v, 10, arg->flags & F_ALT) == -1) - return -1; - return 1; - } - - if (PyUnicode_CheckExact(v) && arg->ch == 's') { - *p_str = Py_NewRef(v); - } - else { - if (arg->ch == 's') - *p_str = PyObject_Str(v); - else if (arg->ch == 'r') - *p_str = PyObject_Repr(v); - else - *p_str = PyObject_ASCII(v); - } - break; - - case 'i': - case 'd': - case 'u': - case 'o': - case 'x': - case 'X': - { - int ret = mainformatlong(v, arg, p_str, writer); - if (ret != 0) - return ret; - arg->sign = 1; - break; - } - - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - if (arg->width == -1 && arg->prec == -1 - && !(arg->flags & (F_SIGN | F_BLANK))) - { - /* Fast path */ - if (formatfloat(v, arg, NULL, writer) == -1) - return -1; - return 1; - } - - arg->sign = 1; - if (formatfloat(v, arg, p_str, NULL) == -1) - return -1; - break; - - case 'c': - { - Py_UCS4 ch = formatchar(v); - if (ch == (Py_UCS4) -1) - return -1; - if (arg->width == -1 && arg->prec == -1) { - /* Fast path */ - if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) - return -1; - return 1; - } - *p_str = PyUnicode_FromOrdinal(ch); - break; - } - - default: - PyErr_Format(PyExc_ValueError, - "unsupported format character '%c' (0x%x) " - "at index %zd", - (31<=arg->ch && arg->ch<=126) ? (char)arg->ch : '?', - (int)arg->ch, - ctx->fmtpos - 1); - return -1; - } - if (*p_str == NULL) - return -1; - assert (PyUnicode_Check(*p_str)); - return 0; -} - -static int -unicode_format_arg_output(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg, - PyObject *str) -{ - Py_ssize_t len; - int kind; - const void *pbuf; - Py_ssize_t pindex; - Py_UCS4 signchar; - Py_ssize_t buflen; - Py_UCS4 maxchar; - Py_ssize_t sublen; - _PyUnicodeWriter *writer = &ctx->writer; - Py_UCS4 fill; - - fill = ' '; - if (arg->sign && arg->flags & F_ZERO) - fill = '0'; - - len = PyUnicode_GET_LENGTH(str); - if ((arg->width == -1 || arg->width <= len) - && (arg->prec == -1 || arg->prec >= len) - && !(arg->flags & (F_SIGN | F_BLANK))) - { - /* Fast path */ - if (_PyUnicodeWriter_WriteStr(writer, str) == -1) - return -1; - return 0; - } - - /* Truncate the string for "s", "r" and "a" formats - if the precision is set */ - if (arg->ch == 's' || arg->ch == 'r' || arg->ch == 'a') { - if (arg->prec >= 0 && len > arg->prec) - len = arg->prec; - } - - /* Adjust sign and width */ - kind = PyUnicode_KIND(str); - pbuf = PyUnicode_DATA(str); - pindex = 0; - signchar = '\0'; - if (arg->sign) { - Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex); - if (ch == '-' || ch == '+') { - signchar = ch; - len--; - pindex++; - } - else if (arg->flags & F_SIGN) - signchar = '+'; - else if (arg->flags & F_BLANK) - signchar = ' '; - else - arg->sign = 0; - } - if (arg->width < len) - arg->width = len; - - /* Prepare the writer */ - maxchar = writer->maxchar; - if (!(arg->flags & F_LJUST)) { - if (arg->sign) { - if ((arg->width-1) > len) - maxchar = Py_MAX(maxchar, fill); - } - else { - if (arg->width > len) - maxchar = Py_MAX(maxchar, fill); - } - } - if (PyUnicode_MAX_CHAR_VALUE(str) > maxchar) { - Py_UCS4 strmaxchar = _PyUnicode_FindMaxChar(str, 0, pindex+len); - maxchar = Py_MAX(maxchar, strmaxchar); - } - - buflen = arg->width; - if (arg->sign && len == arg->width) - buflen++; - if (_PyUnicodeWriter_Prepare(writer, buflen, maxchar) == -1) - return -1; - - /* Write the sign if needed */ - if (arg->sign) { - if (fill != ' ') { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); - writer->pos += 1; - } - if (arg->width > len) - arg->width--; - } - - /* Write the numeric prefix for "x", "X" and "o" formats - if the alternate form is used. - For example, write "0x" for the "%#x" format. */ - if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { - assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); - assert(PyUnicode_READ(kind, pbuf, pindex + 1) == arg->ch); - if (fill != ' ') { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); - writer->pos += 2; - pindex += 2; - } - arg->width -= 2; - if (arg->width < 0) - arg->width = 0; - len -= 2; - } - - /* Pad left with the fill character if needed */ - if (arg->width > len && !(arg->flags & F_LJUST)) { - sublen = arg->width - len; - unicode_fill(writer->kind, writer->data, fill, writer->pos, sublen); - writer->pos += sublen; - arg->width = len; - } - - /* If padding with spaces: write sign if needed and/or numeric prefix if - the alternate form is used */ - if (fill == ' ') { - if (arg->sign) { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); - writer->pos += 1; - } - if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { - assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); - assert(PyUnicode_READ(kind, pbuf, pindex+1) == arg->ch); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); - writer->pos += 2; - pindex += 2; - } - } - - /* Write characters */ - if (len) { - _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, - str, pindex, len); - writer->pos += len; - } - - /* Pad right with the fill character if needed */ - if (arg->width > len) { - sublen = arg->width - len; - unicode_fill(writer->kind, writer->data, ' ', writer->pos, sublen); - writer->pos += sublen; - } - return 0; -} - -/* Helper of PyUnicode_Format(): format one arg. - Return 0 on success, raise an exception and return -1 on error. */ -static int -unicode_format_arg(struct unicode_formatter_t *ctx) -{ - struct unicode_format_arg_t arg; - PyObject *str; - int ret; - - arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos); - if (arg.ch == '%') { - ctx->fmtpos++; - ctx->fmtcnt--; - if (_PyUnicodeWriter_WriteCharInline(&ctx->writer, '%') < 0) - return -1; - return 0; - } - arg.flags = 0; - arg.width = -1; - arg.prec = -1; - arg.sign = 0; - str = NULL; - - ret = unicode_format_arg_parse(ctx, &arg); - if (ret == -1) - return -1; - - ret = unicode_format_arg_format(ctx, &arg, &str); - if (ret == -1) - return -1; - - if (ret != 1) { - ret = unicode_format_arg_output(ctx, &arg, str); - Py_DECREF(str); - if (ret == -1) - return -1; - } - - if (ctx->dict && (ctx->argidx < ctx->arglen)) { - PyErr_SetString(PyExc_TypeError, - "not all arguments converted during string formatting"); - return -1; - } - return 0; -} - -PyObject * -PyUnicode_Format(PyObject *format, PyObject *args) -{ - struct unicode_formatter_t ctx; - - if (format == NULL || args == NULL) { - PyErr_BadInternalCall(); - return NULL; - } - - if (ensure_unicode(format) < 0) - return NULL; - - ctx.fmtstr = format; - ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr); - ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr); - ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr); - ctx.fmtpos = 0; - - _PyUnicodeWriter_Init(&ctx.writer); - ctx.writer.min_length = ctx.fmtcnt + 100; - ctx.writer.overallocate = 1; - - if (PyTuple_Check(args)) { - ctx.arglen = PyTuple_Size(args); - ctx.argidx = 0; - } - else { - ctx.arglen = -1; - ctx.argidx = -2; - } - ctx.args_owned = 0; - if (PyMapping_Check(args) && !PyTuple_Check(args) && !PyUnicode_Check(args)) - ctx.dict = args; - else - ctx.dict = NULL; - ctx.args = args; - - while (--ctx.fmtcnt >= 0) { - if (PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { - Py_ssize_t nonfmtpos; - - nonfmtpos = ctx.fmtpos++; - while (ctx.fmtcnt >= 0 && - PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { - ctx.fmtpos++; - ctx.fmtcnt--; - } - if (ctx.fmtcnt < 0) { - ctx.fmtpos--; - ctx.writer.overallocate = 0; - } - - if (_PyUnicodeWriter_WriteSubstring(&ctx.writer, ctx.fmtstr, - nonfmtpos, ctx.fmtpos) < 0) - goto onError; - } - else { - ctx.fmtpos++; - if (unicode_format_arg(&ctx) == -1) - goto onError; - } - } - - if (ctx.argidx < ctx.arglen && !ctx.dict) { - PyErr_SetString(PyExc_TypeError, - "not all arguments converted during string formatting"); - goto onError; - } - - if (ctx.args_owned) { - Py_DECREF(ctx.args); - } - return _PyUnicodeWriter_Finish(&ctx.writer); - - onError: - _PyUnicodeWriter_Dealloc(&ctx.writer); - if (ctx.args_owned) { - Py_DECREF(ctx.args); - } - return NULL; -} - static PyObject * unicode_subtype_new(PyTypeObject *type, PyObject *unicode); @@ -15563,7 +13808,7 @@ unicode_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, { PyObject *unicode; if (x == NULL) { - unicode = unicode_get_empty(); + unicode = _PyUnicode_GetEmpty(); } else if (encoding == NULL && errors == NULL) { unicode = PyObject_Str(x); @@ -15599,7 +13844,7 @@ unicode_vectorcall(PyObject *type, PyObject *const *args, Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) { // Fallback to unicode_new() - PyObject *tuple = _PyTuple_FromArray(args, nargs); + PyObject *tuple = PyTuple_FromArray(args, nargs); if (tuple == NULL) { return NULL; } @@ -15617,7 +13862,7 @@ unicode_vectorcall(PyObject *type, PyObject *const *args, return NULL; } if (nargs == 0) { - return unicode_get_empty(); + return _PyUnicode_GetEmpty(); } PyObject *object = args[0]; if (nargs == 1) { @@ -15999,14 +14244,16 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, /* Do a setdefault on the per-interpreter cache. */ PyObject *interned = get_interned_dict(interp); assert(interned != NULL); - - LOCK_INTERNED(interp); +#ifdef Py_GIL_DISABLED +# define INTERN_MUTEX &_Py_INTERP_CACHED_OBJECT(interp, interned_mutex) +#endif + FT_MUTEX_LOCK(INTERN_MUTEX); PyObject *t; { int res = PyDict_SetDefaultRef(interned, s, s, &t); if (res < 0) { PyErr_Clear(); - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return s; } else if (res == 1) { @@ -16016,7 +14263,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, PyUnicode_CHECK_INTERNED(t) == SSTATE_INTERNED_MORTAL) { immortalize_interned(t); } - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return t; } else { @@ -16049,7 +14296,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, immortalize_interned(s); } - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return s; } @@ -16291,7 +14538,7 @@ unicodeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) if (it->it_seq != NULL) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { - PyObject *u = unicode_get_empty(); + PyObject *u = _PyUnicode_GetEmpty(); if (u == NULL) { Py_XDECREF(iter); return NULL; diff --git a/Objects/unicodetype_db.h b/Objects/unicodetype_db.h index 5be810dd674..536f51324ae 100644 --- a/Objects/unicodetype_db.h +++ b/Objects/unicodetype_db.h @@ -508,6 +508,8 @@ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { {-40, 0, -40, 0, 0, 9993}, {0, 39, 0, 0, 0, 10113}, {-39, 0, -39, 0, 0, 9993}, + {0, 27, 0, 0, 0, 10113}, + {-27, 0, -27, 0, 0, 9993}, {0, 34, 0, 0, 0, 10113}, {-34, 0, -34, 0, 0, 9993}, {0, 0, 0, 0, 0, 9344}, @@ -1755,1218 +1757,612 @@ const Py_UCS4 _PyUnicode_ExtendedCase[] = { }; /* type indexes */ -#define SHIFT 6 +#define SHIFT 7 static const unsigned short index1[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 26, 26, 26, 26, 26, 68, 69, - 70, 71, 72, 73, 74, 75, 26, 26, 26, 26, 26, 26, 26, 26, 76, 77, 78, 79, - 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - 98, 99, 100, 101, 102, 103, 104, 105, 12, 106, 106, 107, 106, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 119, 119, 119, 119, - 119, 119, 119, 119, 120, 121, 122, 123, 119, 119, 119, 119, 119, 119, - 119, 119, 119, 124, 125, 119, 119, 119, 119, 119, 119, 119, 119, 119, - 119, 119, 119, 119, 119, 126, 127, 119, 128, 129, 106, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 119, 119, 119, 139, 140, 141, 142, 143, - 144, 26, 145, 146, 147, 148, 149, 119, 119, 119, 119, 119, 150, 26, 151, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 152, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 153, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 119, 154, 155, 156, 157, 153, - 158, 26, 159, 160, 26, 26, 26, 161, 162, 26, 26, 26, 26, 26, 26, 26, 163, - 26, 164, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 165, 26, 26, 26, 26, - 26, 26, 26, 166, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 167, 26, 168, 169, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 170, 26, 171, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 172, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 173, 26, 26, 26, 26, - 26, 26, 26, 160, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 174, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 175, 176, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 177, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 160, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 178, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 179, 26, 158, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 180, 26, 26, 26, 26, 26, 26, 26, 26, 26, 159, 26, 26, 26, 26, 26, 181, - 182, 26, 183, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 184, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 185, 186, 26, 26, 26, 26, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, - 208, 209, 210, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 211, 212, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 26, 214, 215, 216, 26, 217, 26, 218, 219, - 220, 221, 222, 26, 223, 26, 26, 224, 225, 226, 227, 228, 229, 26, 230, - 231, 232, 233, 234, 235, 236, 26, 237, 238, 239, 240, 241, 213, 213, 242, - 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 26, 26, - 26, 26, 256, 257, 258, 213, 259, 260, 261, 262, 263, 213, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 213, 26, 273, 274, 275, 276, 277, 278, 213, - 213, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, - 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, - 213, 213, 306, 307, 308, 309, 310, 311, 312, 313, 213, 213, 314, 213, - 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 213, 213, 326, - 327, 328, 329, 213, 330, 331, 332, 213, 213, 213, 213, 333, 334, 335, - 336, 337, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 218, - 213, 338, 339, 26, 26, 26, 340, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 341, 342, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 343, 344, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 237, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 313, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 345, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, - 26, 26, 26, 26, 26, 26, 324, 346, 347, 348, 349, 350, 351, 213, 213, 213, - 213, 213, 213, 352, 213, 213, 213, 353, 354, 213, 26, 355, 356, 357, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 358, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 359, 273, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 360, 26, 26, 26, 26, 361, 362, 26, 26, 26, 26, 26, - 363, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 364, 365, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 119, 119, 119, 366, 119, 119, 119, 119, 119, 119, 138, 213, - 367, 368, 119, 369, 119, 119, 119, 370, 371, 372, 373, 374, 119, 375, - 213, 376, 119, 377, 213, 213, 378, 379, 380, 381, 382, 383, 384, 385, - 386, 387, 388, 389, 390, 391, 392, 393, 119, 119, 119, 119, 119, 119, - 119, 119, 394, 395, 396, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 397, 213, 213, 213, 398, 399, - 400, 213, 401, 402, 213, 213, 213, 213, 403, 404, 213, 213, 213, 213, - 213, 213, 213, 405, 213, 213, 213, 406, 213, 213, 213, 213, 213, 213, - 213, 407, 26, 26, 26, 408, 409, 410, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 411, 412, 213, 413, 213, 213, 213, 414, 415, 416, - 417, 213, 213, 213, 213, 418, 119, 419, 420, 421, 422, 423, 424, 425, - 426, 213, 213, 119, 119, 119, 427, 119, 119, 119, 119, 119, 119, 119, - 119, 119, 119, 119, 428, 119, 429, 119, 430, 431, 432, 433, 434, 119, - 119, 119, 119, 119, 435, 436, 437, 119, 119, 438, 366, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 439, - 440, 26, 441, 181, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 152, 26, 442, 26, 26, 26, 26, 443, 444, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 445, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 446, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 165, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 177, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 447, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 448, 26, 26, 26, 449, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 450, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 451, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 452, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, 445, 26, - 26, 26, 26, 26, 452, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 453, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 454, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 455, 456, 213, 213, 12, 12, 12, 457, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - 213, 213, 213, 213, 213, 213, 213, 213, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 34, 35, 36, 37, + 38, 39, 34, 34, 34, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 64, 64, 64, 65, 66, 64, + 64, 64, 64, 67, 68, 64, 64, 64, 64, 64, 64, 69, 64, 70, 71, 72, 73, 74, + 75, 64, 76, 77, 78, 79, 80, 81, 82, 64, 64, 83, 84, 34, 34, 34, 34, 34, + 34, 85, 34, 34, 34, 34, 34, 86, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 87, 88, 89, 90, 91, 92, 34, 93, 34, 34, + 34, 94, 95, 34, 34, 34, 34, 34, 96, 34, 34, 34, 97, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 98, 99, 100, 34, 34, 34, 34, 34, 34, 101, 102, 34, + 34, 34, 34, 34, 34, 34, 34, 103, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 104, 34, 34, 34, 92, 34, 34, 34, 34, 34, 34, 34, 34, 105, 34, 34, 34, 34, + 106, 107, 34, 34, 34, 34, 34, 108, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 92, 34, 34, 34, 34, 34, 34, 109, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 110, 111, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 112, 34, 34, 34, 34, 113, 34, 34, 114, 115, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 116, 34, 34, 34, + 34, 34, 34, 34, 34, 117, 34, 34, 118, 119, 120, 121, 122, 123, 124, 125, + 126, 127, 128, 129, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 130, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 132, 133, + 134, 135, 136, 137, 138, 34, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 148, 131, 149, 150, 151, 152, 153, 154, 155, 34, 34, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 131, 184, 185, 186, + 187, 131, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, + 131, 200, 201, 202, 203, 34, 34, 34, 204, 34, 205, 206, 207, 34, 208, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 209, 34, 34, 34, 34, 34, 34, 34, 34, 210, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 146, 34, 34, 34, 34, 211, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 212, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 34, 34, 34, 34, + 213, 214, 215, 216, 131, 131, 217, 131, 218, 219, 220, 221, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 222, 223, 224, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 225, 34, 34, 226, 34, 34, 227, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 228, 229, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 64, + 230, 64, 64, 64, 231, 232, 233, 64, 234, 235, 236, 237, 238, 239, 131, + 240, 241, 242, 243, 244, 245, 246, 247, 64, 64, 64, 64, 248, 249, 131, + 131, 131, 131, 131, 131, 131, 131, 250, 131, 251, 252, 253, 131, 131, + 254, 131, 131, 131, 255, 131, 256, 131, 257, 131, 258, 34, 259, 260, 131, + 131, 131, 131, 131, 261, 262, 263, 131, 264, 265, 131, 131, 266, 267, + 268, 269, 270, 131, 64, 271, 64, 64, 64, 64, 64, 272, 64, 273, 274, 275, + 64, 64, 276, 277, 64, 278, 131, 131, 131, 131, 131, 131, 131, 131, 279, + 280, 281, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 85, + 282, 34, 283, 284, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 285, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 286, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 287, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 108, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 288, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 289, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 290, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 291, 34, 34, 34, 34, 292, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 34, 285, 34, + 34, 293, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 294, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 295, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 296, 131, 297, 298, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, }; static const unsigned short index2[] = { @@ -3005,7 +2401,7 @@ static const unsigned short index2[] = { 19, 77, 81, 19, 82, 83, 84, 85, 19, 86, 87, 85, 88, 89, 19, 19, 87, 19, 90, 91, 19, 19, 92, 19, 19, 19, 19, 19, 19, 19, 93, 19, 19, 94, 19, 95, 94, 19, 19, 19, 96, 94, 97, 98, 98, 99, 19, 19, 19, 19, 19, 100, 19, 55, - 19, 19, 19, 19, 19, 19, 19, 19, 101, 102, 19, 19, 19, 19, 19, 19, 19, 19, + 55, 19, 19, 19, 19, 19, 19, 19, 101, 102, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 103, 103, 103, 103, 103, 103, 103, 103, 103, 104, 104, 104, 104, 104, 104, 104, 103, 103, 5, 5, 5, 5, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 5, 5, 5, 5, 5, 5, @@ -3089,7 +2485,7 @@ static const unsigned short index2[] = { 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 0, 0, 4, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 5, 55, 55, 55, 55, 55, 55, 0, 20, 20, 0, 0, 0, 0, 0, 24, + 55, 55, 55, 55, 5, 55, 55, 55, 55, 55, 55, 55, 20, 20, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 24, 24, 24, 24, @@ -3139,14 +2535,14 @@ static const unsigned short index2[] = { 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 55, 24, 24, 24, 17, 17, 17, 17, 0, 24, 24, 24, 0, 24, 24, 24, 24, 0, 0, 0, 0, - 0, 0, 0, 24, 24, 0, 55, 55, 55, 0, 0, 55, 0, 0, 55, 55, 24, 24, 0, 0, 6, + 0, 0, 0, 24, 24, 0, 55, 55, 55, 0, 55, 55, 0, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 4, 26, 26, 26, 26, 26, 26, 26, 4, 55, 24, 17, 17, 4, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 0, 24, 55, 17, 24, 17, 17, 17, 17, 17, 0, 24, 17, 17, 0, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 17, 17, 0, 0, 0, 0, - 0, 0, 55, 55, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 55, 55, 55, 0, 55, 55, 24, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 55, 55, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, @@ -3202,112 +2598,130 @@ static const unsigned short index2[] = { 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 142, 4, 103, 142, 142, 142, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, - 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, - 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 0, 0, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 143, 144, 145, 146, 147, - 148, 149, 150, 151, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 24, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 143, 144, 145, 146, 147, 148, 149, 150, 151, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, + 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, + 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, + 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, + 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, + 234, 235, 236, 237, 0, 0, 238, 239, 240, 241, 242, 243, 0, 0, 4, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 1, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 152, 153, 154, 155, 156, 157, 158, - 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, - 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, - 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, 231, 232, 233, 234, 235, 236, 237, 0, 0, 238, 239, 240, 241, - 242, 243, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 1, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 0, 0, 0, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 4, 4, 4, 244, 244, 244, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 244, 244, 244, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 24, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 17, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 24, 24, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, - 17, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, - 0, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 24, 24, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 17, 24, 24, 24, 24, 24, - 24, 24, 17, 17, 17, 17, 17, 17, 17, 17, 24, 17, 17, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 4, 4, 4, 104, 4, 4, 4, 4, 55, 24, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, - 24, 20, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, + 17, 17, 17, 17, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 4, 4, 4, 104, 4, 4, 4, 4, 55, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 20, 24, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 245, 245, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 24, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 24, 24, 24, 17, 17, 17, 17, 24, 24, + 17, 17, 17, 0, 0, 0, 0, 17, 17, 24, 17, 17, 17, 17, 17, 17, 24, 24, 24, + 0, 0, 0, 0, 4, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, - 0, 0, 0, 55, 55, 55, 55, 55, 245, 245, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 24, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 143, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 24, 24, 17, 17, 24, 0, 0, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 17, 24, 17, 24, 24, 24, 24, 24, 24, 24, 0, + 24, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 17, 17, 17, + 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, + 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 104, 4, 4, 4, 4, 4, 4, 0, 0, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 24, 24, 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 24, 24, 24, - 17, 17, 17, 17, 24, 24, 17, 17, 17, 0, 0, 0, 0, 17, 17, 24, 17, 17, 17, - 17, 17, 17, 24, 24, 24, 0, 0, 0, 0, 4, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, - 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, + 24, 24, 24, 24, 24, 17, 24, 17, 17, 17, 17, 17, 24, 17, 17, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 17, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, 24, + 24, 24, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 143, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 24, 24, 17, 17, 24, 0, 0, 4, 4, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 24, 17, 24, 24, - 24, 24, 24, 24, 24, 0, 24, 17, 24, 17, 17, 24, 24, 24, 24, 24, 24, 24, - 24, 17, 17, 17, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, - 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 104, 4, 4, 4, - 4, 4, 4, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, - 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 24, 24, - 24, 24, 24, 17, 24, 17, 17, 17, 17, 17, 24, 17, 17, 55, 55, 55, 55, 55, - 55, 55, 55, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 17, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, 24, 24, - 24, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 24, 17, 24, 24, 17, 17, 17, 24, 17, 24, 24, 24, 17, 17, 0, 0, 0, - 0, 0, 0, 0, 0, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 24, 17, 24, 24, 17, 17, 17, 24, 17, 24, 24, 24, 17, 17, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 17, 17, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 0, 0, 0, 4, 4, 4, 4, 4, 6, 7, 8, 9, @@ -3333,44 +2747,55 @@ static const unsigned short index2[] = { 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 258, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 29, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 259, 260, 261, 262, - 263, 264, 19, 19, 265, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, 267, 267, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 259, 260, + 261, 262, 263, 264, 19, 19, 265, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 266, 266, + 266, 266, 266, 266, 266, 266, 267, 267, 267, 267, 267, 267, 267, 267, + 266, 266, 266, 266, 266, 266, 0, 0, 267, 267, 267, 267, 267, 267, 0, 0, + 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, 267, 267, 267, 267, + 267, 267, 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, 267, 267, 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 0, 0, 267, 267, 267, - 267, 267, 267, 0, 0, 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, - 267, 267, 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 266, 266, - 267, 267, 267, 267, 267, 267, 267, 267, 266, 266, 266, 266, 266, 266, 0, - 0, 267, 267, 267, 267, 267, 267, 0, 0, 268, 266, 269, 266, 270, 266, 271, - 266, 0, 267, 0, 267, 0, 267, 0, 267, 266, 266, 266, 266, 266, 266, 266, - 266, 267, 267, 267, 267, 267, 267, 267, 267, 272, 272, 273, 273, 273, - 273, 274, 274, 275, 275, 276, 276, 277, 277, 0, 0, 278, 279, 280, 281, - 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, - 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, - 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, - 324, 325, 266, 266, 326, 327, 328, 0, 329, 330, 267, 267, 331, 331, 332, - 5, 333, 5, 5, 5, 334, 335, 336, 0, 337, 338, 339, 339, 339, 339, 340, 5, - 5, 5, 266, 266, 341, 342, 0, 0, 343, 344, 267, 267, 345, 345, 0, 5, 5, 5, - 266, 266, 346, 347, 348, 128, 349, 350, 267, 267, 351, 351, 132, 5, 5, 5, - 0, 0, 352, 353, 354, 0, 355, 356, 357, 357, 358, 358, 359, 5, 5, 0, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 20, 360, 360, 20, 20, 4, 4, 4, 4, 4, 4, 4, 4, - 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 5, 2, 2, 20, 20, 20, 20, 20, - 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 1, 20, 20, 20, 20, 20, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 361, - 103, 0, 0, 362, 363, 364, 365, 366, 367, 4, 4, 4, 4, 4, 103, 361, 25, 21, - 22, 362, 363, 364, 365, 366, 367, 4, 4, 4, 4, 4, 0, 103, 103, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 4, 4, 4, 4, 4, 4, + 267, 267, 267, 0, 0, 268, 266, 269, 266, 270, 266, 271, 266, 0, 267, 0, + 267, 0, 267, 0, 267, 266, 266, 266, 266, 266, 266, 266, 266, 267, 267, + 267, 267, 267, 267, 267, 267, 272, 272, 273, 273, 273, 273, 274, 274, + 275, 275, 276, 276, 277, 277, 0, 0, 278, 279, 280, 281, 282, 283, 284, + 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, + 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, + 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 266, + 266, 326, 327, 328, 0, 329, 330, 267, 267, 331, 331, 332, 5, 333, 5, 5, + 5, 334, 335, 336, 0, 337, 338, 339, 339, 339, 339, 340, 5, 5, 5, 266, + 266, 341, 342, 0, 0, 343, 344, 267, 267, 345, 345, 0, 5, 5, 5, 266, 266, + 346, 347, 348, 128, 349, 350, 267, 267, 351, 351, 132, 5, 5, 5, 0, 0, + 352, 353, 354, 0, 355, 356, 357, 357, 358, 358, 359, 5, 5, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 20, 360, 360, 20, 20, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 4, 4, 5, 2, 2, 20, 20, 20, 20, 20, 1, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 20, + 20, 20, 20, 20, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 361, 103, 0, + 0, 362, 363, 364, 365, 366, 367, 4, 4, 4, 4, 4, 103, 361, 25, 21, 22, + 362, 363, 364, 365, 366, 367, 4, 4, 4, 4, 4, 0, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 5, 5, 5, 5, 24, 5, 5, 5, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 4, 4, 122, 4, 4, 4, 4, 122, 4, 4, 19, 122, 122, 122, 19, 19, 122, 122, + 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 5, 5, 5, 5, 24, 5, 5, 5, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 122, 4, 4, 4, 4, 122, 4, 4, 19, 122, 122, 122, 19, 19, 122, 122, 122, 19, 4, 122, 4, 4, 368, 122, 122, 122, 122, 122, 4, 4, 4, 4, 4, 4, 122, 4, 369, 4, 122, 4, 370, 371, 122, 122, 368, 19, 122, 122, 372, 122, 19, 55, 55, 55, 55, 19, 4, 4, 19, 19, 122, 122, 4, 4, 4, 4, 4, 122, 19, @@ -3384,178 +2809,328 @@ static const unsigned short index2[] = { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 376, 376, 376, 376, 376, - 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, - 376, 376, 376, 376, 376, 376, 376, 377, 377, 377, 377, 377, 377, 377, - 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, - 377, 377, 377, 377, 377, 361, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 25, - 21, 22, 362, 363, 364, 365, 366, 367, 26, 361, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 25, 21, - 22, 362, 363, 364, 365, 366, 367, 26, 25, 21, 22, 362, 363, 364, 365, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 21, 22, 362, 363, 364, 365, 366, 367, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 25, 21, 22, 362, 363, 364, + 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 25, 21, 22, + 362, 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, 377, + 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, + 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, 361, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, + 361, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 25, + 21, 22, 362, 363, 364, 365, 366, 367, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 138, 138, 138, 138, 138, 138, 138, 138, + 137, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, - 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, 29, 30, 378, - 379, 380, 381, 382, 29, 30, 29, 30, 29, 30, 383, 384, 385, 386, 19, 29, - 30, 19, 29, 30, 19, 19, 19, 19, 19, 103, 103, 387, 387, 29, 30, 29, 30, + 138, 138, 138, 138, 138, 138, 138, 29, 30, 378, 379, 380, 381, 382, 29, + 30, 29, 30, 29, 30, 383, 384, 385, 386, 19, 29, 30, 19, 29, 30, 19, 19, + 19, 19, 19, 103, 103, 387, 387, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 4, 4, 4, 4, - 4, 4, 29, 30, 29, 30, 24, 24, 24, 29, 30, 0, 0, 0, 0, 0, 4, 4, 4, 4, 26, - 4, 4, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 19, 4, 4, 4, 4, 4, 4, 29, 30, 29, 30, 24, 24, 24, 29, 30, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 26, 4, 4, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, - 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 0, 388, 0, 0, 0, - 0, 0, 388, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 0, + 388, 0, 0, 0, 0, 0, 388, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 104, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, - 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 104, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, - 55, 55, 55, 55, 55, 55, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, + 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 389, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 389, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 4, 4, 4, 4, 104, 55, 244, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 24, 24, 24, 24, 17, 17, 4, 104, - 104, 104, 104, 104, 4, 4, 244, 244, 244, 104, 55, 4, 4, 4, 0, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 5, 5, 104, 104, - 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 17, 104, 104, 104, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 4, 4, 26, 26, 26, - 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, - 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 1, 4, 4, 4, 4, 104, 55, 244, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 24, 24, 24, 24, 17, 17, 4, 104, 104, 104, + 104, 104, 4, 4, 244, 244, 244, 104, 55, 4, 4, 4, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 5, 5, 104, 104, 55, 4, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 17, 104, 104, 104, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 4, 4, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, + 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 390, 55, 55, 390, 55, 55, 55, 390, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 390, 55, 390, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 390, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 390, 55, 55, 390, 55, 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, - 55, 55, 55, 55, 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, - 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 390, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 390, 55, 390, 390, 390, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 390, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, + 55, 390, 390, 390, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 390, 390, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 390, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 390, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 390, - 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, - 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 390, 390, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, @@ -3563,41 +3138,27 @@ static const unsigned short index2[] = { 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, - 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 390, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, @@ -3605,373 +3166,418 @@ static const unsigned short index2[] = { 55, 55, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 104, 104, + 104, 104, 104, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, + 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 55, + 24, 5, 5, 5, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 104, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 29, 30, 29, 30, 29, 30, 103, 103, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 104, 104, 104, 104, 104, 104, 4, 4, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 104, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, - 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, - 29, 30, 29, 30, 29, 30, 29, 30, 55, 24, 5, 5, 5, 4, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 4, 104, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 24, 24, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 104, + 104, 104, 104, 104, 104, 104, 104, 104, 5, 5, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 19, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 103, - 103, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 19, 19, 19, 19, 19, 19, 19, 19, 29, 30, 29, 30, 391, 29, 30, 29, 30, 29, + 30, 29, 30, 29, 30, 104, 5, 5, 29, 30, 392, 19, 55, 29, 30, 29, 30, 393, + 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, + 30, 29, 30, 394, 395, 396, 397, 394, 19, 398, 399, 400, 401, 29, 30, 29, + 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 402, 403, 404, 29, + 30, 29, 30, 405, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, + 29, 30, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 103, 103, 103, 103, 29, 30, 55, 103, 103, 19, 55, 55, 55, 55, 55, 55, 55, + 24, 55, 55, 55, 24, 55, 55, 55, 55, 24, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 24, + 24, 17, 4, 4, 4, 4, 24, 0, 0, 0, 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 24, 24, 4, 4, 4, 4, - 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 104, 104, 104, 104, 104, 104, 104, 104, 104, - 5, 5, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 19, 19, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 103, 19, 19, 19, 19, 19, 19, 19, 19, 29, 30, - 29, 30, 391, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 104, 5, 5, 29, 30, - 392, 19, 55, 29, 30, 29, 30, 393, 19, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 394, 395, 396, 397, 394, 19, - 398, 399, 400, 401, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, 30, 29, - 30, 29, 30, 402, 403, 404, 29, 30, 29, 30, 405, 29, 30, 0, 0, 29, 30, 0, - 19, 0, 19, 29, 30, 29, 30, 29, 30, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 103, 103, 29, 30, 55, 103, 103, 19, - 55, 55, 55, 55, 55, 55, 55, 24, 55, 55, 55, 24, 55, 55, 55, 55, 24, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 17, 17, 24, 24, 17, 4, 4, 4, 4, 24, 0, 0, 0, 26, 26, 26, - 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 17, - 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, + 4, 4, 4, 55, 4, 55, 55, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 24, 24, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 55, 55, 55, 55, 55, 55, 4, 4, 4, 55, 4, 55, 55, 24, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, - 24, 24, 24, 24, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 24, 24, 17, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 24, 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 17, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, 17, 17, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 0, 104, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, + 4, 4, 55, 55, 55, 55, 55, 24, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 24, 17, 17, 24, 24, 24, 24, 17, 17, 24, 24, 17, - 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 104, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 55, 55, 55, 55, 55, 24, 104, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, - 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 17, 17, - 24, 24, 17, 17, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 24, 55, - 55, 55, 55, 55, 55, 55, 55, 24, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 0, 0, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 104, 55, 55, 55, 55, 55, 55, 4, 4, 4, 55, 17, 24, 17, 55, 55, + 55, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 17, 17, 24, 24, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 24, 55, 55, 55, 55, 55, 55, 55, 55, 24, 17, 0, + 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 4, 4, 4, 4, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, 55, 55, 55, 55, + 55, 4, 4, 4, 55, 17, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 55, 24, 24, 24, 55, - 55, 24, 24, 55, 55, 55, 55, 55, 24, 24, 55, 24, 55, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 104, 4, 4, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 24, 24, 17, 17, 4, 4, 55, - 104, 104, 17, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, - 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, - 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 55, 55, 55, 24, 55, 24, 24, 24, 55, 55, 24, 24, 55, 55, 55, 55, 55, 24, + 24, 55, 24, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 104, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 17, 24, 24, 17, 17, 4, 4, 55, 104, 104, 17, 24, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 407, 19, 19, 19, 19, 19, 19, 19, 5, 103, 103, 103, 103, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 103, 5, 5, 0, 0, 0, 0, 408, 409, 410, 411, 412, 413, - 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, - 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, - 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, - 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, - 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, - 484, 485, 486, 487, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 19, 19, 19, 19, 19, 19, 19, 19, 407, 19, 19, 19, 19, 19, 19, 19, 5, 103, + 103, 103, 103, 19, 19, 19, 19, 19, 19, 19, 19, 19, 103, 5, 5, 0, 0, 0, 0, + 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, + 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, + 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, + 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, + 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 17, 17, 24, 17, 17, 24, 17, 17, 4, 17, 24, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 24, 17, 17, 24, 17, + 17, 4, 17, 24, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 390, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 390, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 488, 489, 490, 491, 492, 493, 494, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 495, 496, 497, 498, 499, 0, 0, 0, 0, 0, 55, 24, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 0, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 500, 500, 500, 500, 500, 500, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 500, 500, 4, 4, 4, 4, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, - 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 4, 4, 4, 17, 17, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 17, 4, 4, 5, 0, 4, 5, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 0, 0, 0, 0, 500, 55, 500, - 55, 500, 0, 500, 55, 500, 55, 500, 55, 500, 55, 500, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 0, 0, 20, 0, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 5, 4, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 4, 4, 4, 4, 4, 4, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 4, 4, 4, 5, 17, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 501, 501, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, - 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 0, 0, 0, 4, - 4, 4, 5, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 20, 20, 4, 4, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 488, 489, 490, 491, 492, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 495, 496, 497, 498, 499, 0, 0, 0, 0, 0, 55, 24, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 0, 55, 55, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 500, + 500, 500, 500, 500, 500, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 500, 500, + 4, 4, 4, 4, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 17, 17, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 17, + 4, 4, 5, 0, 4, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, + 4, 4, 4, 4, 0, 0, 0, 0, 500, 55, 500, 55, 500, 0, 500, 55, 500, 55, 500, + 55, 500, 55, 500, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 20, 0, 4, 4, 4, 4, 4, 4, 5, 4, 4, 4, 4, 4, 4, 5, 4, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 4, 4, 4, 4, 4, 4, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 4, 4, 4, 5, 17, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 501, 501, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, + 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 0, 0, 0, + 4, 4, 4, 5, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 20, 20, 20, 4, 4, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 4, 4, + 4, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 26, + 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 26, 26, 4, + 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 244, 55, 55, 55, 55, 55, 55, 55, 55, 244, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, + 24, 24, 24, 24, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 4, 244, 244, 244, + 244, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, + 502, 502, 502, 502, 0, 0, 0, 0, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, + 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 0, 504, 504, + 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 0, 504, + 504, 504, 504, 504, 504, 504, 0, 504, 504, 0, 505, 505, 505, 505, 505, + 505, 505, 505, 505, 505, 505, 0, 505, 505, 505, 505, 505, 505, 505, 505, + 505, 505, 505, 505, 505, 505, 505, 0, 505, 505, 505, 505, 505, 505, 505, + 0, 505, 505, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 103, 104, 104, 103, 103, 103, 0, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 0, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, + 55, 55, 0, 0, 0, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 4, 26, 26, 26, 26, 26, + 26, 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 26, 26, 26, 26, 26, 26, 26, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 0, 0, 0, 0, 26, 26, 26, 26, + 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 26, 26, 26, 26, 26, 26, 0, 0, 0, 4, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 26, 26, 55, 55, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 55, 24, 24, 24, 0, 24, 24, 0, 0, 0, 0, 0, + 24, 24, 24, 24, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 24, 24, 24, 0, 0, 0, 0, 24, 25, 21, 22, 362, + 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 4, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 24, 24, 0, 0, 0, 0, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, + 26, 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 0, 0, 0, 0, 0, 4, 4, 4, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 26, 26, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 24, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 0, 0, 0, + 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, + 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 55, 55, 55, 55, 104, 55, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 0, 0, 0, 24, + 24, 24, 24, 24, 4, 104, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 21, 22, 362, 363, 364, + 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 24, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 24, 24, 4, + 0, 0, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 104, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 244, 55, 55, 55, 55, 55, 55, 55, 55, 244, 0, + 55, 55, 55, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 55, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 26, 26, + 26, 26, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 24, 24, 24, 24, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, - 55, 55, 4, 244, 244, 244, 244, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, - 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, - 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, - 0, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, 502, - 502, 502, 502, 502, 502, 502, 502, 502, 0, 0, 0, 0, 503, 503, 503, 503, - 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, - 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, - 503, 503, 503, 503, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 4, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 0, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, 504, - 504, 504, 504, 0, 504, 504, 504, 504, 504, 504, 504, 0, 504, 504, 0, 505, - 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 0, 505, 505, 505, 505, - 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 505, 0, 505, 505, 505, - 505, 505, 505, 505, 0, 505, 505, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 104, 104, 103, 103, 103, 0, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, - 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 0, 0, 55, 0, 0, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 0, 4, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 4, 4, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 0, 55, 55, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, - 26, 26, 26, 26, 26, 0, 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, - 0, 0, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 0, 0, 0, 26, 26, 55, 55, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 55, 24, 24, 24, 0, 24, 24, 0, 0, 0, 0, 0, 24, 24, 24, 24, 55, 55, - 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, - 24, 24, 24, 0, 0, 0, 0, 24, 25, 21, 22, 362, 26, 26, 26, 26, 26, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 4, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, - 55, 55, 55, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 0, 0, 0, - 0, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, - 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, - 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 110, 110, 110, 110, - 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, - 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, - 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, 110, - 110, 110, 110, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 117, 117, - 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, - 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, - 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, - 117, 117, 117, 117, 117, 117, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, - 26, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 55, 55, 55, - 55, 104, 55, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 0, 0, 0, 24, 24, 24, 24, 24, 4, 104, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 21, 22, - 362, 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 17, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 0, 24, 24, 4, 0, 0, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 25, 21, 22, 362, 363, 364, + 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 24, 55, 55, 24, 24, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 24, 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 55, 0, 0, - 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 26, 26, 26, 26, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, + 24, 24, 24, 17, 17, 24, 24, 4, 4, 20, 4, 4, 4, 4, 24, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 20, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, + 24, 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 4, 4, 4, 4, 55, 17, 17, 55, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 4, 4, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 25, 21, 22, 362, - 363, 364, 365, 366, 367, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 55, 55, 24, 24, 55, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 24, 24, 24, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, - 17, 24, 24, 24, 24, 17, 17, 24, 24, 4, 4, 20, 4, 4, 4, 4, 24, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 20, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, - 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, - 24, 24, 24, 24, 17, 24, 24, 24, 24, 24, 24, 24, 24, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 4, 4, 4, 4, 55, 17, 17, 55, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, - 4, 4, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 17, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 17, 17, 55, 55, 55, 55, 4, 4, 4, 4, 24, 24, 24, 24, 4, 17, 24, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 55, 4, 55, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, + 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, + 17, 55, 55, 55, 55, 4, 4, 4, 4, 24, 24, 24, 24, 4, 17, 24, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 55, 4, 55, 4, 4, 4, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, @@ -4041,38 +3647,46 @@ static const unsigned short index2[] = { 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, - 55, 0, 0, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 17, 17, 17, 17, 17, 17, 0, 17, 17, 0, 0, 24, 24, 17, 24, - 55, 17, 55, 17, 24, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, - 24, 24, 24, 24, 0, 0, 24, 24, 17, 17, 17, 17, 24, 55, 4, 55, 17, 0, 0, 0, + 17, 17, 17, 0, 17, 17, 0, 0, 24, 24, 17, 24, 55, 17, 55, 17, 24, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, - 24, 24, 24, 17, 55, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 55, 24, 24, 24, 24, 24, 24, 17, 17, 24, 24, 24, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 24, 24, 24, 24, 0, 0, 24, 24, + 17, 17, 17, 17, 24, 55, 4, 55, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 17, 55, 24, 24, 24, + 24, 4, 4, 4, 4, 4, 4, 4, 4, 24, 0, 0, 0, 0, 0, 0, 0, 0, 55, 24, 24, 24, + 24, 24, 24, 17, 17, 24, 24, 24, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 17, 24, 24, 4, 4, 4, 55, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 4, 4, 4, + 55, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 24, 17, 24, 24, 24, 17, 24, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, @@ -4088,105 +3702,188 @@ static const unsigned short index2[] = { 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 17, 24, 24, 24, 24, 24, 24, 24, 17, 24, 24, 17, - 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 24, 24, 24, 24, 24, 24, 0, 0, 0, 24, 0, 24, 24, 0, 24, 24, - 24, 24, 24, 24, 24, 55, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 17, - 17, 0, 24, 24, 0, 17, 17, 24, 17, 24, 55, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 17, 17, 4, 4, 0, - 0, 0, 0, 0, 0, 0, 24, 24, 55, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 17, 17, 24, 24, 24, 24, 24, 0, 0, 0, 17, 17, 24, 17, 24, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 0, + 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, - 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 0, - 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, + 24, 24, 24, 0, 0, 0, 24, 0, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 55, + 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 24, 55, 55, 55, 55, 55, 55, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 17, 17, 17, 0, 24, 24, 0, 17, 17, + 24, 17, 24, 55, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 17, 17, 17, 24, 24, 24, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 24, 24, - 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 104, - 104, 104, 104, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 0, 26, 26, 26, 26, 26, 26, 26, 0, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 104, 104, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 104, 4, 4, 4, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 24, 55, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 104, 104, 104, 104, 104, 104, 104, - 104, 104, 104, 104, 104, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 55, 55, 0, 0, 0, 0, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 104, 104, 4, 104, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 24, 24, 17, 17, 4, 4, 0, 0, 0, 0, 0, 0, 0, 24, 24, + 55, 17, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 17, 17, 24, 24, + 24, 24, 24, 0, 0, 0, 17, 17, 24, 17, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 390, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 390, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, + 244, 244, 244, 244, 244, 244, 244, 244, 244, 244, 0, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 24, 55, + 55, 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 17, 17, + 17, 24, 24, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 24, 24, 24, 24, 24, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, + 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 104, 104, 104, 104, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 26, + 26, 26, 26, 26, 26, 26, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 104, 104, 104, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 104, 4, 4, 4, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 4, 4, 4, 4, 0, 0, 0, 0, 0, 506, 506, 506, 506, 506, 506, 506, + 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, + 506, 506, 506, 506, 0, 0, 507, 507, 507, 507, 507, 507, 507, 507, 507, + 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, + 507, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 24, 55, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 104, + 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 104, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 104, 4, 104, 24, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 17, 17, 104, 104, 244, 244, 244, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 104, 104, 104, 0, 104, 104, 104, 104, 104, 104, 104, 0, 104, 104, 0, 55, @@ -4199,58 +3896,89 @@ static const unsigned short index2[] = { 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, - 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 0, 0, 4, 24, 24, 4, 20, 20, 20, 20, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 4, 24, 24, 4, 20, 20, 20, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 24, 24, 24, 4, 4, 4, 17, 17, 17, - 17, 17, 17, 20, 20, 20, 20, 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, 24, - 24, 4, 4, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 4, 4, + 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 24, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, - 0, 0, 0, 0, 0, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 17, 17, 24, 24, 24, 4, 4, 4, + 17, 17, 17, 17, 17, 17, 20, 20, 20, 20, 20, 20, 20, 20, 24, 24, 24, 24, + 24, 24, 24, 24, 4, 4, 24, 24, 24, 24, 24, 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 24, + 24, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 24, 24, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 0, 0, 0, 0, 0, 0, 0, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, - 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, + 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, - 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, + 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, - 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 122, 0, 122, 122, 0, 0, 122, 0, 0, 122, 122, 0, 0, 122, 122, 122, + 19, 19, 122, 0, 122, 122, 0, 0, 122, 0, 0, 122, 122, 0, 0, 122, 122, 122, 122, 0, 122, 122, 122, 122, 122, 122, 122, 122, 19, 19, 19, 19, 0, 19, 0, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, @@ -4312,199 +4040,320 @@ static const unsigned short index2[] = { 4, 4, 4, 4, 24, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 55, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 0, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, - 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 0, 24, 24, 24, 24, - 24, 0, 0, 0, 0, 0, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, - 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 55, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 0, 0, + 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 0, 0, 24, 24, 24, 24, 24, 24, 24, 0, 24, 24, + 0, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 24, 24, 24, 24, 24, - 24, 104, 104, 104, 104, 104, 104, 104, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 0, 0, 0, 0, 55, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 104, + 104, 104, 104, 104, 104, 104, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0, 0, 0, 0, 55, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 24, 24, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 104, 24, 24, 24, 24, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, 55, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 24, 24, - 24, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 104, 24, 24, 24, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 24, 24, 55, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, - 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 0, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, - 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, 506, - 506, 506, 506, 506, 506, 506, 506, 506, 506, 507, 507, 507, 507, 507, - 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, - 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, 507, - 507, 24, 24, 24, 24, 24, 24, 24, 104, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 24, 55, 55, 24, 55, 55, 55, 55, 55, 55, 55, 24, 24, 55, 55, 55, 55, 55, + 24, 0, 0, 0, 0, 0, 0, 0, 0, 55, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 4, 26, 26, 26, 4, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 0, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 55, 55, 0, 55, 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 0, 55, 0, 0, 0, 0, 0, 0, 55, 0, - 0, 0, 0, 55, 0, 55, 0, 55, 0, 55, 55, 55, 0, 55, 55, 0, 55, 0, 0, 55, 0, - 55, 0, 55, 0, 55, 0, 55, 0, 55, 55, 0, 55, 0, 0, 55, 55, 55, 55, 0, 55, - 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 0, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 0, 55, - 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 361, 361, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 0, 26, 26, 26, 26, 26, 26, 26, 26, 26, 24, 24, 24, 24, 24, 24, 24, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, - 508, 508, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, - 508, 508, 508, 4, 4, 4, 4, 4, 4, 508, 508, 508, 508, 508, 508, 508, 508, - 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, 508, - 508, 508, 508, 508, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, + 508, 508, 509, 509, 509, 509, 509, 509, 509, 509, 509, 509, 509, 509, + 509, 509, 509, 509, 509, 509, 509, 509, 509, 509, 509, 509, 509, 509, + 509, 509, 509, 509, 509, 509, 509, 509, 24, 24, 24, 24, 24, 24, 24, 104, + 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, - 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 4, 26, 26, 26, 4, 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 4, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 0, + 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, + 55, 0, 55, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 55, 0, 55, 0, 55, 0, 55, 55, + 55, 0, 55, 55, 0, 55, 0, 0, 55, 0, 55, 0, 55, 0, 55, 0, 55, 0, 55, 55, 0, + 55, 0, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, + 0, 0, 0, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, - 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 361, 361, 25, 21, 22, 362, 363, 364, 365, 366, 367, 26, 26, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, - 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 55, 390, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 510, 510, 510, 510, 510, 510, 510, 510, 510, + 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, + 510, 510, 510, 4, 4, 4, 4, 4, 4, 510, 510, 510, 510, 510, 510, 510, 510, + 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, + 510, 510, 510, 510, 4, 4, 4, 4, 4, 4, 510, 510, 510, 510, 510, 510, 510, + 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, 510, + 510, 510, 510, 510, 510, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, + 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 4, 0, 0, 0, 0, 0, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, + 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 390, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 20, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, }; /* Returns the numeric value as double for Unicode characters @@ -4586,6 +4435,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C50: case 0x11D50: case 0x11DA0: + case 0x11DE0: case 0x11F50: case 0x16130: case 0x16A60: @@ -4726,7 +4576,12 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C5A: case 0x11D51: case 0x11DA1: + case 0x11DE1: case 0x11F51: + case 0x12038: + case 0x12039: + case 0x12079: + case 0x1230B: case 0x12415: case 0x1241E: case 0x1242C: @@ -4740,6 +4595,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x16D71: case 0x16E81: case 0x16E94: + case 0x16FF4: case 0x1CCF1: case 0x1D2C1: case 0x1D2E1: @@ -4796,6 +4652,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10F26: case 0x11FD1: case 0x11FD2: + case 0x12226: case 0x12464: case 0x1ECAE: case 0x1ED3C: @@ -5000,7 +4857,6 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1ECA0: case 0x1ECB4: return (double) 100000.0; - case 0x5146: case 0x16B5E: return (double) 1000000.0; case 0x1ECA1: @@ -5013,6 +4869,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) return (double) 1000000000.0; case 0x16B60: return (double) 10000000000.0; + case 0x5146: case 0x16B61: return (double) 1000000000000.0; case 0x4EAC: @@ -5219,7 +5076,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C5B: case 0x11D52: case 0x11DA2: + case 0x11DE2: case 0x11F52: + case 0x1222B: + case 0x12399: case 0x12400: case 0x12416: case 0x1241F: @@ -5237,6 +5097,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x16D72: case 0x16E82: case 0x16E95: + case 0x16FF6: case 0x1CCF2: case 0x1D2C2: case 0x1D2E2: @@ -5457,7 +5318,9 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C5C: case 0x11D53: case 0x11DA3: + case 0x11DE3: case 0x11F53: + case 0x1230D: case 0x12401: case 0x12408: case 0x12417: @@ -5516,6 +5379,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11FCE: return (double) 3.0/16.0; case 0x0F2B: + case 0x16FF5: return (double) 3.0/2.0; case 0x0D5D: case 0x11FCD: @@ -5691,6 +5555,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C5D: case 0x11D54: case 0x11DA4: + case 0x11DE4: case 0x11F54: case 0x12402: case 0x12409: @@ -5901,6 +5766,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C5E: case 0x11D55: case 0x11DA5: + case 0x11DE5: case 0x11F55: case 0x12403: case 0x1240A: @@ -6108,6 +5974,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C5F: case 0x11D56: case 0x11DA6: + case 0x11DE6: case 0x11F56: case 0x12404: case 0x1240B: @@ -6269,6 +6136,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C60: case 0x11D57: case 0x11DA7: + case 0x11DE7: case 0x11F57: case 0x12405: case 0x1240C: @@ -6431,6 +6299,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C61: case 0x11D58: case 0x11DA8: + case 0x11DE8: case 0x11F58: case 0x12406: case 0x1240D: @@ -6589,6 +6458,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x11C62: case 0x11D59: case 0x11DA9: + case 0x11DE9: case 0x11F59: case 0x12407: case 0x1240E: diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 66435924b6c..a47d6193d70 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -4,6 +4,7 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_typing_type_repr #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString #include "pycore_unionobject.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() typedef struct { @@ -21,9 +22,7 @@ unionobject_dealloc(PyObject *self) unionobject *alias = (unionobject *)self; _PyObject_GC_UNTRACK(self); - if (alias->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)alias); - } + FT_CLEAR_WEAKREFS(self, alias->weakreflist); Py_XDECREF(alias->args); Py_XDECREF(alias->hashable_args); @@ -290,7 +289,7 @@ union_repr(PyObject *self) } for (Py_ssize_t i = 0; i < len; i++) { - if (i > 0 && PyUnicodeWriter_WriteUTF8(writer, " | ", 3) < 0) { + if (i > 0 && PyUnicodeWriter_WriteASCII(writer, " | ", 3) < 0) { goto error; } PyObject *p = PyTuple_GET_ITEM(alias->args, i); @@ -300,12 +299,12 @@ union_repr(PyObject *self) } #if 0 - PyUnicodeWriter_WriteUTF8(writer, "|args=", 6); + PyUnicodeWriter_WriteASCII(writer, "|args=", 6); PyUnicodeWriter_WriteRepr(writer, alias->args); - PyUnicodeWriter_WriteUTF8(writer, "|h=", 3); + PyUnicodeWriter_WriteASCII(writer, "|h=", 3); PyUnicodeWriter_WriteRepr(writer, alias->hashable_args); if (alias->unhashable_args) { - PyUnicodeWriter_WriteUTF8(writer, "|u=", 3); + PyUnicodeWriter_WriteASCII(writer, "|u=", 3); PyUnicodeWriter_WriteRepr(writer, alias->unhashable_args); } #endif @@ -394,8 +393,23 @@ static PyGetSetDef union_properties[] = { {0} }; +static PyObject * +union_nb_or(PyObject *a, PyObject *b) +{ + unionbuilder ub; + if (!unionbuilder_init(&ub, true)) { + return NULL; + } + if (!unionbuilder_add_single(&ub, a) || + !unionbuilder_add_single(&ub, b)) { + unionbuilder_finalize(&ub); + return NULL; + } + return make_union(&ub); +} + static PyNumberMethods union_as_number = { - .nb_or = _Py_union_type_or, // Add __or__ function + .nb_or = union_nb_or, // Add __or__ function }; static const char* const cls_attrs[] = { @@ -475,11 +489,13 @@ _Py_union_from_tuple(PyObject *args) } if (PyTuple_CheckExact(args)) { if (!unionbuilder_add_tuple(&ub, args)) { + unionbuilder_finalize(&ub); return NULL; } } else { if (!unionbuilder_add_single(&ub, args)) { + unionbuilder_finalize(&ub); return NULL; } } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index bd4c4ac9b34..61fa3ddad0b 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -964,7 +964,8 @@ PyWeakref_GetRef(PyObject *ref, PyObject **pobj) } -PyObject * +/* removed in 3.15, but kept for stable ABI compatibility */ +PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref) { if (ref == NULL || !PyWeakref_Check(ref)) { diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index b6efb3e4a20..86df2c7183c 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -57,11 +57,11 @@ _query_thread(LPVOID param) IEnumWbemClassObject* enumerator = NULL; HRESULT hr = S_OK; BSTR bstrQuery = NULL; - struct _query_data *data = (struct _query_data*)param; + _query_data data = *(struct _query_data*)param; // gh-125315: Copy the query string first, so that if the main thread gives // up on waiting we aren't left with a dangling pointer (and a likely crash) - bstrQuery = SysAllocString(data->query); + bstrQuery = SysAllocString(data.query); if (!bstrQuery) { hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); } @@ -71,7 +71,7 @@ _query_thread(LPVOID param) } if (FAILED(hr)) { - CloseHandle(data->writePipe); + CloseHandle(data.writePipe); if (bstrQuery) { SysFreeString(bstrQuery); } @@ -96,7 +96,7 @@ _query_thread(LPVOID param) IID_IWbemLocator, (LPVOID *)&locator ); } - if (SUCCEEDED(hr) && !SetEvent(data->initEvent)) { + if (SUCCEEDED(hr) && !SetEvent(data.initEvent)) { hr = HRESULT_FROM_WIN32(GetLastError()); } if (SUCCEEDED(hr)) { @@ -105,7 +105,7 @@ _query_thread(LPVOID param) NULL, NULL, 0, NULL, 0, 0, &services ); } - if (SUCCEEDED(hr) && !SetEvent(data->connectEvent)) { + if (SUCCEEDED(hr) && !SetEvent(data.connectEvent)) { hr = HRESULT_FROM_WIN32(GetLastError()); } if (SUCCEEDED(hr)) { @@ -143,7 +143,7 @@ _query_thread(LPVOID param) if (FAILED(hr) || got != 1 || !value) { continue; } - if (!startOfEnum && !WriteFile(data->writePipe, (LPVOID)L"\0", 2, &written, NULL)) { + if (!startOfEnum && !WriteFile(data.writePipe, (LPVOID)L"\0", 2, &written, NULL)) { hr = HRESULT_FROM_WIN32(GetLastError()); break; } @@ -171,10 +171,10 @@ _query_thread(LPVOID param) DWORD cbStr1, cbStr2; cbStr1 = (DWORD)(wcslen(propName) * sizeof(propName[0])); cbStr2 = (DWORD)(wcslen(propStr) * sizeof(propStr[0])); - if (!WriteFile(data->writePipe, propName, cbStr1, &written, NULL) || - !WriteFile(data->writePipe, (LPVOID)L"=", 2, &written, NULL) || - !WriteFile(data->writePipe, propStr, cbStr2, &written, NULL) || - !WriteFile(data->writePipe, (LPVOID)L"\0", 2, &written, NULL) + if (!WriteFile(data.writePipe, propName, cbStr1, &written, NULL) || + !WriteFile(data.writePipe, (LPVOID)L"=", 2, &written, NULL) || + !WriteFile(data.writePipe, propStr, cbStr2, &written, NULL) || + !WriteFile(data.writePipe, (LPVOID)L"\0", 2, &written, NULL) ) { hr = HRESULT_FROM_WIN32(GetLastError()); } @@ -200,7 +200,7 @@ _query_thread(LPVOID param) locator->Release(); } CoUninitialize(); - CloseHandle(data->writePipe); + CloseHandle(data.writePipe); return (DWORD)hr; } @@ -224,6 +224,7 @@ wait_event(HANDLE event, DWORD timeout) /*[clinic input] +@permit_long_docstring_body _wmi.exec_query query: unicode @@ -236,7 +237,7 @@ by null characters. static PyObject * _wmi_exec_query_impl(PyObject *module, PyObject *query) -/*[clinic end generated code: output=a62303d5bb5e003f input=48d2d0a1e1a7e3c2]*/ +/*[clinic end generated code: output=a62303d5bb5e003f input=621f5c50c56d06d0]*/ /*[clinic end generated code]*/ { diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index a77d0855af2..647aadfa46b 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -690,9 +690,21 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) PyObject *return_value = NULL; unsigned int mode; - mode = (unsigned int)PyLong_AsUnsignedLongMask(arg); - if (mode == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(arg, &mode, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } return_value = msvcrt_SetErrorMode_impl(module, mode); @@ -731,4 +743,4 @@ exit: #ifndef MSVCRT_GETERRORMODE_METHODDEF #define MSVCRT_GETERRORMODE_METHODDEF #endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ -/*[clinic end generated code: output=692c6f52bb9193ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f67eaf745685429d input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 45d54e3c902..d76a8d8aef8 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -1633,6 +1633,70 @@ exit: #if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) +PyDoc_STRVAR(winreg_DeleteTree__doc__, +"DeleteTree($module, key, sub_key=None, /)\n" +"--\n" +"\n" +"Deletes the specified key and all its subkeys and values recursively.\n" +"\n" +" key\n" +" An already open key, or any one of the predefined HKEY_* constants.\n" +" sub_key\n" +" A string that names the subkey to delete. If None, deletes all subkeys\n" +" and values of the specified key.\n" +"\n" +"This function deletes a key and all its descendants. If sub_key is None,\n" +"all subkeys and values of the specified key are deleted."); + +#define WINREG_DELETETREE_METHODDEF \ + {"DeleteTree", _PyCFunction_CAST(winreg_DeleteTree), METH_FASTCALL, winreg_DeleteTree__doc__}, + +static PyObject * +winreg_DeleteTree_impl(PyObject *module, HKEY key, const wchar_t *sub_key); + +static PyObject * +winreg_DeleteTree(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + HKEY key; + const wchar_t *sub_key = NULL; + + if (!_PyArg_CheckPositional("DeleteTree", nargs, 1, 2)) { + goto exit; + } + if (!clinic_HKEY_converter(_PyModule_GetState(module), args[0], &key)) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + if (args[1] == Py_None) { + sub_key = NULL; + } + else if (PyUnicode_Check(args[1])) { + sub_key = PyUnicode_AsWideCharString(args[1], NULL); + if (sub_key == NULL) { + goto exit; + } + } + else { + _PyArg_BadArgument("DeleteTree", "argument 2", "str or None", args[1]); + goto exit; + } +skip_optional: + return_value = winreg_DeleteTree_impl(module, key, sub_key); + +exit: + /* Cleanup for sub_key */ + PyMem_Free((void *)sub_key); + + return return_value; +} + +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_QueryReflectionKey__doc__, "QueryReflectionKey($module, key, /)\n" "--\n" @@ -1771,7 +1835,11 @@ exit: #define WINREG_ENABLEREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_ENABLEREFLECTIONKEY_METHODDEF) */ +#ifndef WINREG_DELETETREE_METHODDEF + #define WINREG_DELETETREE_METHODDEF +#endif /* !defined(WINREG_DELETETREE_METHODDEF) */ + #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=be4b6857b95558b5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ce7e8e38884851fb input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index 6ce2131c7b8..51b46c64d99 100644 --- a/PC/config.c +++ b/PC/config.c @@ -13,6 +13,7 @@ extern PyObject* PyInit_errno(void); extern PyObject* PyInit_faulthandler(void); extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); +extern PyObject* PyInit__math_integer(void); extern PyObject* PyInit_math(void); extern PyObject* PyInit_nt(void); extern PyObject* PyInit__operator(void); @@ -100,6 +101,7 @@ struct _inittab _PyImport_Inittab[] = { {"errno", PyInit_errno}, {"faulthandler", PyInit_faulthandler}, {"gc", PyInit_gc}, + {"_math_integer", PyInit__math_integer}, {"math", PyInit_math}, {"nt", PyInit_nt}, /* Use the NT os functions, not posix */ {"_operator", PyInit__operator}, diff --git a/PC/icons/idlex150.png b/PC/icons/idlex150.png index 806cb0c8aa2..dbdca5b00f8 100644 Binary files a/PC/icons/idlex150.png and b/PC/icons/idlex150.png differ diff --git a/PC/launcher.c b/PC/launcher.c index 47fafbc3bf6..fed5e156b92 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -140,7 +140,7 @@ static wchar_t * get_env(wchar_t * key) return buf; } -#if defined(_DEBUG) +#if defined(Py_DEBUG) /* Do not define EXECUTABLEPATH_VALUE in debug builds as it'll never point to the debug build. */ #if defined(_WINDOWS) @@ -1272,6 +1272,7 @@ static PYC_MAGIC magic_values[] = { { 3500, 3549, L"3.12" }, { 3550, 3599, L"3.13" }, { 3600, 3649, L"3.14" }, + { 3650, 3699, L"3.15" }, { 0 } }; diff --git a/PC/layout/main.py b/PC/layout/main.py index 7324a135133..8543e7c56e1 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -247,9 +247,15 @@ def in_build(f, dest="", new_name=None, no_lib=False): if ns.include_freethreaded: yield from in_build("venvlaunchert.exe", "Lib/venv/scripts/nt/") yield from in_build("venvwlaunchert.exe", "Lib/venv/scripts/nt/") - else: + elif (VER_MAJOR, VER_MINOR) > (3, 12): yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/") yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/") + else: + # Older versions of venv expected the scripts to be named 'python' + # and they were renamed at this stage. We need to replicate that + # when packaging older versions. + yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") + yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") if ns.include_tools: @@ -652,15 +658,6 @@ def main(): ns.doc_build = (Path.cwd() / ns.doc_build).resolve() if ns.include_cat and not ns.include_cat.is_absolute(): ns.include_cat = (Path.cwd() / ns.include_cat).resolve() - if not ns.arch: - # TODO: Calculate arch from files in ns.build instead - if sys.winver.endswith("-arm64"): - ns.arch = "arm64" - elif sys.winver.endswith("-32"): - ns.arch = "win32" - else: - ns.arch = "amd64" - if ns.zip and not ns.zip.is_absolute(): ns.zip = (Path.cwd() / ns.zip).resolve() if ns.catalog and not ns.catalog.is_absolute(): @@ -668,6 +665,17 @@ def main(): configure_logger(ns) + if not ns.arch: + from .support.arch import calculate_from_build_dir + ns.arch = calculate_from_build_dir(ns.build) + + expect = f"{VER_MAJOR}.{VER_MINOR}.{VER_MICRO}{VER_SUFFIX}" + actual = check_patchlevel_version(ns.source) + if actual and actual != expect: + log_error(f"Inferred version {expect} does not match {actual} from patchlevel.h. " + "You should set %PYTHONINCLUDE% or %PYTHON_HEXVERSION% before launching.") + return 5 + log_info( """OPTIONS Source: {ns.source} diff --git a/PC/layout/support/arch.py b/PC/layout/support/arch.py new file mode 100644 index 00000000000..daf4efbc7ab --- /dev/null +++ b/PC/layout/support/arch.py @@ -0,0 +1,34 @@ +from struct import unpack +from .constants import * +from .logging import * + +def calculate_from_build_dir(root): + candidates = [ + root / PYTHON_DLL_NAME, + root / FREETHREADED_PYTHON_DLL_NAME, + *root.glob("*.dll"), + *root.glob("*.pyd"), + # Check EXE last because it's easier to have cross-platform EXE + *root.glob("*.exe"), + ] + + ARCHS = { + b"PE\0\0\x4c\x01": "win32", + b"PE\0\0\x64\x86": "amd64", + b"PE\0\0\x64\xAA": "arm64" + } + + first_exc = None + for pe in candidates: + try: + # Read the PE header to grab the machine type + with open(pe, "rb") as f: + f.seek(0x3C) + offset = int.from_bytes(f.read(4), "little") + f.seek(offset) + arch = ARCHS[f.read(6)] + except (FileNotFoundError, PermissionError, LookupError) as ex: + log_debug("Failed to open {}: {}", pe, ex) + continue + log_info("Inferred architecture {} from {}", arch, pe) + return arch diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index ae22aa16ebf..6b8c915e519 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -6,6 +6,8 @@ __version__ = "3.8" import os +import pathlib +import re import struct import sys @@ -13,9 +15,15 @@ def _unpack_hexversion(): try: hexversion = int(os.getenv("PYTHON_HEXVERSION"), 16) + return struct.pack(">i", hexversion) except (TypeError, ValueError): - hexversion = sys.hexversion - return struct.pack(">i", hexversion) + pass + if os.getenv("PYTHONINCLUDE"): + try: + return _read_patchlevel_version(pathlib.Path(os.getenv("PYTHONINCLUDE"))) + except OSError: + pass + return struct.pack(">i", sys.hexversion) def _get_suffix(field4): @@ -26,6 +34,39 @@ def _get_suffix(field4): return "" +def _read_patchlevel_version(sources): + if not sources.match("Include"): + sources /= "Include" + values = {} + with open(sources / "patchlevel.h", "r", encoding="utf-8") as f: + for line in f: + m = re.match(r'#\s*define\s+(PY_\S+?)\s+(\S+)', line.strip(), re.I) + if m and m.group(2): + v = m.group(2) + if v.startswith('"'): + v = v[1:-1] + else: + v = values.get(v, v) + if isinstance(v, str): + try: + v = int(v, 16 if v.startswith("0x") else 10) + except ValueError: + pass + values[m.group(1)] = v + return ( + values["PY_MAJOR_VERSION"], + values["PY_MINOR_VERSION"], + values["PY_MICRO_VERSION"], + values["PY_RELEASE_LEVEL"] << 4 | values["PY_RELEASE_SERIAL"], + ) + + +def check_patchlevel_version(sources): + got = _read_patchlevel_version(sources) + if got != (VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4): + return f"{got[0]}.{got[1]}.{got[2]}{_get_suffix(got[3])}" + + VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = _unpack_hexversion() VER_SUFFIX = _get_suffix(VER_FIELD4) VER_FIELD3 = VER_MICRO << 8 | VER_FIELD4 diff --git a/PC/layout/support/pymanager.py b/PC/layout/support/pymanager.py index 667c89cdd2c..831d49ea3f9 100644 --- a/PC/layout/support/pymanager.py +++ b/PC/layout/support/pymanager.py @@ -80,7 +80,9 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): # Tag used in runtime ID (for side-by-side install/updates) ID_TAG = XY_ARCH_TAG - # Tag shown in 'py list' output + # Tag shown in 'py list' output. + # gh-139810: We only include '-dev' here for prereleases, even though it + # works for final releases too. DISPLAY_TAG = f"{XY_TAG}-dev{TAG_ARCH}" if VER_SUFFIX else XY_ARCH_TAG # Tag used for PEP 514 registration SYS_WINVER = XY_TAG + (TAG_ARCH if TAG_ARCH != '-64' else '') @@ -102,9 +104,10 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): FULL_ARCH_TAG, XY_ARCH_TAG, X_ARCH_TAG, - # X_TAG and XY_TAG doesn't include VER_SUFFIX, so create -dev versions - f"{XY_TAG}-dev{TAG_ARCH}" if XY_TAG and VER_SUFFIX else "", - f"{X_TAG}-dev{TAG_ARCH}" if X_TAG and VER_SUFFIX else "", + # gh-139810: The -dev tags are always included so that the latest + # release is chosen, no matter whether it's prerelease or final. + f"{XY_TAG}-dev{TAG_ARCH}" if XY_TAG else "", + f"{X_TAG}-dev{TAG_ARCH}" if X_TAG else "", ] # Generate run-for entries for each target. @@ -115,16 +118,15 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): ]: if not base["target"]: continue - STD_RUN_FOR.append({**base, "tag": FULL_ARCH_TAG}) + STD_RUN_FOR.extend([ + {**base, "tag": FULL_ARCH_TAG}, + {**base, "tag": f"{XY_TAG}-dev{TAG_ARCH}" if XY_TAG else ""}, + {**base, "tag": f"{X_TAG}-dev{TAG_ARCH}" if X_TAG else ""}, + ]) if XY_TAG: STD_RUN_FOR.append({**base, "tag": XY_ARCH_TAG}) if X_TAG: STD_RUN_FOR.append({**base, "tag": X_ARCH_TAG}) - if VER_SUFFIX: - STD_RUN_FOR.extend([ - {**base, "tag": f"{XY_TAG}-dev{TAG_ARCH}" if XY_TAG else ""}, - {**base, "tag": f"{X_TAG}-dev{TAG_ARCH}" if X_TAG else ""}, - ]) # Generate alias entries for each target. We need both arch and non-arch # versions as well as windowed/non-windowed versions to make sure that all diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index b170e06b47d..8826e7e85a7 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -218,6 +218,7 @@ msvcrt_get_osfhandle_impl(PyObject *module, int fd) /* Console I/O */ /*[clinic input] +@permit_long_summary msvcrt.kbhit -> long Returns a nonzero value if a keypress is waiting to be read. Otherwise, return 0. @@ -225,7 +226,7 @@ Returns a nonzero value if a keypress is waiting to be read. Otherwise, return 0 static long msvcrt_kbhit_impl(PyObject *module) -/*[clinic end generated code: output=940dfce6587c1890 input=d0f4cb3289ff51e2]*/ +/*[clinic end generated code: output=940dfce6587c1890 input=52c0c44143f3fba5]*/ { return _kbhit(); } diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h similarity index 91% rename from PC/pyconfig.h.in rename to PC/pyconfig.h index 9e70303868e..a126fca6f5a 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h @@ -94,12 +94,22 @@ WIN32 is still required for the locale module. #endif #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ -/* Define to 1 if you want to disable the GIL */ -/* Uncomment the definition for free-threaded builds, or define it manually - * when compiling extension modules. Note that we test with #ifdef, so - * defining as 0 will still disable the GIL. */ -#ifndef Py_GIL_DISABLED -/* #define Py_GIL_DISABLED 1 */ +/* _DEBUG implies Py_DEBUG */ +#ifdef _DEBUG +# define Py_DEBUG 1 +#endif + +/* Define to 1 when compiling for experimental free-threaded builds */ +#ifdef Py_GIL_DISABLED +/* We undefine if it was set to zero because all later checks are #ifdef. + * Note that non-Windows builds do not do this, and so every effort should + * be made to avoid defining the variable at all when not desired. However, + * sysconfig.get_config_var always returns a 1 or a 0, and so it seems likely + * that a build backend will define it with the value. + */ +#if Py_GIL_DISABLED == 0 +#undef Py_GIL_DISABLED +#endif #endif /* Compiler specific defines */ @@ -108,7 +118,7 @@ WIN32 is still required for the locale module. /* Microsoft C defines _MSC_VER, as does clang-cl.exe */ #ifdef _MSC_VER -/* We want COMPILER to expand to a string containing _MSC_VER's *value*. +/* We want _Py_COMPILER to expand to a string containing _MSC_VER's *value*. * This is horridly tricky, because the stringization operator only works * on macro arguments, and doesn't evaluate macros passed *as* arguments. */ @@ -138,7 +148,7 @@ WIN32 is still required for the locale module. #define MS_WIN64 #endif -/* set the COMPILER and support tier +/* set the _Py_COMPILER and support tier * * win_amd64 MSVC (x86_64-pc-windows-msvc): 1 * win32 MSVC (i686-pc-windows-msvc): 1 @@ -148,22 +158,22 @@ WIN32 is still required for the locale module. #ifdef MS_WIN64 #if defined(_M_X64) || defined(_M_AMD64) #if defined(__clang__) -#define COMPILER ("[Clang " __clang_version__ "] 64 bit (AMD64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[Clang " __clang_version__ "] 64 bit (AMD64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #elif defined(__INTEL_COMPILER) -#define COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 64 bit (amd64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 64 bit (amd64) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("64 bit (AMD64)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (AMD64)") #define PY_SUPPORT_TIER 1 #endif /* __clang__ */ #define PYD_PLATFORM_TAG "win_amd64" #elif defined(_M_ARM64) -#define COMPILER _Py_PASTE_VERSION("64 bit (ARM64)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (ARM64)") #define PY_SUPPORT_TIER 3 #define PYD_PLATFORM_TAG "win_arm64" #else -#define COMPILER _Py_PASTE_VERSION("64 bit (Unknown)") +#define _Py_COMPILER _Py_PASTE_VERSION("64 bit (Unknown)") #define PY_SUPPORT_TIER 0 #endif #endif /* MS_WIN64 */ @@ -210,22 +220,22 @@ typedef _W64 int Py_ssize_t; #if defined(MS_WIN32) && !defined(MS_WIN64) #if defined(_M_IX86) #if defined(__clang__) -#define COMPILER ("[Clang " __clang_version__ "] 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[Clang " __clang_version__ "] 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #elif defined(__INTEL_COMPILER) -#define COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") +#define _Py_COMPILER ("[ICC v." _Py_STRINGIZE(__INTEL_COMPILER) " 32 bit (Intel) with MSC v." _Py_STRINGIZE(_MSC_VER) " CRT]") #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("32 bit (Intel)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (Intel)") #define PY_SUPPORT_TIER 1 #endif /* __clang__ */ #define PYD_PLATFORM_TAG "win32" #elif defined(_M_ARM) -#define COMPILER _Py_PASTE_VERSION("32 bit (ARM)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (ARM)") #define PYD_PLATFORM_TAG "win_arm32" #define PY_SUPPORT_TIER 0 #else -#define COMPILER _Py_PASTE_VERSION("32 bit (Unknown)") +#define _Py_COMPILER _Py_PASTE_VERSION("32 bit (Unknown)") #define PY_SUPPORT_TIER 0 #endif #endif /* MS_WIN32 && !MS_WIN64 */ @@ -263,7 +273,7 @@ typedef int pid_t; #warning "Please use an up-to-date version of gcc! (>2.91 recommended)" #endif -#define COMPILER "[gcc]" +#define _Py_COMPILER "[gcc]" #define PY_LONG_LONG long long #define PY_LLONG_MIN LLONG_MIN #define PY_LLONG_MAX LLONG_MAX @@ -276,7 +286,7 @@ typedef int pid_t; /* XXX These defines are likely incomplete, but should be easy to fix. They should be complete enough to build extension modules. */ -#define COMPILER "[lcc-win32]" +#define _Py_COMPILER "[lcc-win32]" typedef int pid_t; /* __declspec() is supported here too - do nothing to get the defaults */ @@ -319,21 +329,21 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ This is relevant when using build-system generator (e.g CMake) where the linking is explicitly handled */ # if defined(Py_GIL_DISABLED) -# if defined(_DEBUG) -# pragma comment(lib,"python314t_d.lib") +# if defined(Py_DEBUG) +# pragma comment(lib,"python315t_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3t.lib") # else -# pragma comment(lib,"python314t.lib") -# endif /* _DEBUG */ +# pragma comment(lib,"python315t.lib") +# endif /* Py_DEBUG */ # else /* Py_GIL_DISABLED */ -# if defined(_DEBUG) -# pragma comment(lib,"python314_d.lib") +# if defined(Py_DEBUG) +# pragma comment(lib,"python315_d.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python314.lib") -# endif /* _DEBUG */ +# pragma comment(lib,"python315.lib") +# endif /* Py_DEBUG */ # endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER && !Py_NO_LINK_LIB */ # endif /* Py_BUILD_CORE */ @@ -376,11 +386,6 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # define ALIGNOF_MAX_ALIGN_T 8 #endif -#ifdef _DEBUG -# define Py_DEBUG -#endif - - #ifdef MS_WIN32 #define SIZEOF_SHORT 2 diff --git a/PC/python3dll.c b/PC/python3dll.c index f0c578e11c6..0d9e7e9a1ba 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -71,6 +71,7 @@ EXPORT_FUNC(Py_IncRef) EXPORT_FUNC(Py_Initialize) EXPORT_FUNC(Py_InitializeEx) EXPORT_FUNC(Py_Is) +EXPORT_FUNC(Py_IS_TYPE) EXPORT_FUNC(Py_IsFalse) EXPORT_FUNC(Py_IsFinalizing) EXPORT_FUNC(Py_IsInitialized) @@ -86,13 +87,16 @@ EXPORT_FUNC(Py_PACK_VERSION) EXPORT_FUNC(Py_REFCNT) EXPORT_FUNC(Py_ReprEnter) EXPORT_FUNC(Py_ReprLeave) +EXPORT_FUNC(Py_SET_SIZE) EXPORT_FUNC(Py_SetPath) EXPORT_FUNC(Py_SetProgramName) EXPORT_FUNC(Py_SetPythonHome) EXPORT_FUNC(Py_SetRecursionLimit) +EXPORT_FUNC(Py_SIZE) EXPORT_FUNC(Py_TYPE) EXPORT_FUNC(Py_VaBuildValue) EXPORT_FUNC(Py_XNewRef) +EXPORT_FUNC(PyABIInfo_Check) EXPORT_FUNC(PyAIter_Check) EXPORT_FUNC(PyArg_Parse) EXPORT_FUNC(PyArg_ParseTuple) @@ -190,6 +194,7 @@ EXPORT_FUNC(PyDict_Merge) EXPORT_FUNC(PyDict_MergeFromSeq2) EXPORT_FUNC(PyDict_New) EXPORT_FUNC(PyDict_Next) +EXPORT_FUNC(PyDict_SetDefaultRef) EXPORT_FUNC(PyDict_SetItem) EXPORT_FUNC(PyDict_SetItemString) EXPORT_FUNC(PyDict_Size) @@ -415,8 +420,10 @@ EXPORT_FUNC(PyModule_AddObjectRef) EXPORT_FUNC(PyModule_AddStringConstant) EXPORT_FUNC(PyModule_AddType) EXPORT_FUNC(PyModule_Create2) +EXPORT_FUNC(PyModule_Exec) EXPORT_FUNC(PyModule_ExecDef) EXPORT_FUNC(PyModule_FromDefAndSpec2) +EXPORT_FUNC(PyModule_FromSlotsAndSpec) EXPORT_FUNC(PyModule_GetDef) EXPORT_FUNC(PyModule_GetDict) EXPORT_FUNC(PyModule_GetFilename) @@ -424,6 +431,8 @@ EXPORT_FUNC(PyModule_GetFilenameObject) EXPORT_FUNC(PyModule_GetName) EXPORT_FUNC(PyModule_GetNameObject) EXPORT_FUNC(PyModule_GetState) +EXPORT_FUNC(PyModule_GetStateSize) +EXPORT_FUNC(PyModule_GetToken) EXPORT_FUNC(PyModule_New) EXPORT_FUNC(PyModule_NewObject) EXPORT_FUNC(PyModule_SetDocString) @@ -595,7 +604,11 @@ EXPORT_FUNC(PySys_Audit) EXPORT_FUNC(PySys_AuditTuple) EXPORT_FUNC(PySys_FormatStderr) EXPORT_FUNC(PySys_FormatStdout) +EXPORT_FUNC(PySys_GetAttr) +EXPORT_FUNC(PySys_GetAttrString) EXPORT_FUNC(PySys_GetObject) +EXPORT_FUNC(PySys_GetOptionalAttr) +EXPORT_FUNC(PySys_GetOptionalAttrString) EXPORT_FUNC(PySys_GetXOptions) EXPORT_FUNC(PySys_HasWarnOptions) EXPORT_FUNC(PySys_ResetWarnOptions) @@ -663,6 +676,7 @@ EXPORT_FUNC(PyType_GetFlags) EXPORT_FUNC(PyType_GetFullyQualifiedName) EXPORT_FUNC(PyType_GetModule) EXPORT_FUNC(PyType_GetModuleByDef) +EXPORT_FUNC(PyType_GetModuleByToken) EXPORT_FUNC(PyType_GetModuleName) EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index b9c408a580c..8cdb8d722cd 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -19,13 +19,13 @@ #include <winrt\Windows.Storage.h> #ifdef PYTHONW -#ifdef _DEBUG +#ifdef Py_DEBUG const wchar_t *PROGNAME = L"pythonw_d.exe"; #else const wchar_t *PROGNAME = L"pythonw.exe"; #endif #else -#ifdef _DEBUG +#ifdef Py_DEBUG const wchar_t *PROGNAME = L"python_d.exe"; #else const wchar_t *PROGNAME = L"python.exe"; diff --git a/PC/python_ver_rc.h b/PC/python_ver_rc.h index ee867fe4122..70c805d3da8 100644 --- a/PC/python_ver_rc.h +++ b/PC/python_ver_rc.h @@ -8,9 +8,8 @@ #define PYTHON_COPYRIGHT "Copyright \xA9 2001 Python Software Foundation. Copyright \xA9 2000 BeOpen.com. Copyright \xA9 1995-2001 CNRI. Copyright \xA9 1991-1995 SMC." #define MS_WINDOWS -#include "modsupport.h" #include "patchlevel.h" -#ifdef _DEBUG +#ifdef Py_DEBUG # define PYTHON_DEBUG_EXT "_d" #else # define PYTHON_DEBUG_EXT diff --git a/PC/winreg.c b/PC/winreg.c index c0de5c1353a..c1be920fc1d 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -49,8 +49,11 @@ PyDoc_STRVAR(module_doc, "ConnectRegistry() - Establishes a connection to a predefined registry handle\n" " on another computer.\n" "CreateKey() - Creates the specified key, or opens it if it already exists.\n" +"CreateKeyEx() - Creates the specified key, or opens it if it already exists.\n" "DeleteKey() - Deletes the specified key.\n" +"DeleteKeyEx() - Deletes the specified key.\n" "DeleteValue() - Removes a named value from the specified registry key.\n" +"DeleteTree() - Deletes the specified key and all its subkeys and values recursively.\n" "EnumKey() - Enumerates subkeys of the specified open registry key.\n" "EnumValue() - Enumerates values of the specified open registry key.\n" "ExpandEnvironmentStrings() - Expand the env strings in a REG_EXPAND_SZ\n" @@ -69,6 +72,9 @@ PyDoc_STRVAR(module_doc, "SaveKey() - Saves the specified key, and all its subkeys a file.\n" "SetValue() - Associates a value with a specified key.\n" "SetValueEx() - Stores data in the value field of an open registry key.\n" +"DisableReflectionKey() - Disables registry reflection for 32bit processes running on a 64bit OS.\n" +"EnableReflectionKey() - Restores registry reflection for a key.\n" +"QueryReflectionKey() - Determines the reflection state for a key.\n" "\n" "Special objects:\n" "\n" @@ -102,7 +108,9 @@ PyDoc_STRVAR(PyHKEY_doc, "Operations:\n" "__bool__ - Handles with an open object return true, otherwise false.\n" "__int__ - Converting a handle to an integer returns the Win32 handle.\n" -"rich comparison - Handle objects are compared using the handle value."); +"__enter__, __exit__ - Context manager support for 'with' statement,\n" +"automatically closes handle.\n" +"__eq__, __ne__ - Equality comparison based on Windows handle value."); @@ -156,13 +164,6 @@ PyHKEY_deallocFunc(PyObject *ob) Py_DECREF(tp); } -static int -PyHKEY_traverseFunc(PyObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - return 0; -} - static int PyHKEY_boolFunc(PyObject *ob) { @@ -183,13 +184,38 @@ PyHKEY_strFunc(PyObject *ob) return PyUnicode_FromFormat("<PyHKEY:%p>", pyhkey->hkey); } -static int -PyHKEY_compareFunc(PyObject *ob1, PyObject *ob2) +static PyObject * +PyHKEY_richcompare(PyObject *ob1, PyObject *ob2, int op) { + /* Both objects must be PyHKEY objects from the same module */ + if (Py_TYPE(ob1) != Py_TYPE(ob2)) { + Py_RETURN_NOTIMPLEMENTED; + } + PyHKEYObject *pyhkey1 = (PyHKEYObject *)ob1; PyHKEYObject *pyhkey2 = (PyHKEYObject *)ob2; - return pyhkey1 == pyhkey2 ? 0 : - (pyhkey1 < pyhkey2 ? -1 : 1); + HKEY hkey1 = pyhkey1->hkey; + HKEY hkey2 = pyhkey2->hkey; + int result; + + switch (op) { + case Py_EQ: + result = (hkey1 == hkey2); + break; + case Py_NE: + result = (hkey1 != hkey2); + break; + default: + /* Only support equality comparisons, not ordering */ + Py_RETURN_NOTIMPLEMENTED; + } + + if (result) { + Py_RETURN_TRUE; + } + else { + Py_RETURN_FALSE; + } } static Py_hash_t @@ -364,9 +390,10 @@ static PyType_Slot pyhkey_type_slots[] = { {Py_tp_members, PyHKEY_memberlist}, {Py_tp_methods, PyHKEY_methods}, {Py_tp_doc, (char *)PyHKEY_doc}, - {Py_tp_traverse, PyHKEY_traverseFunc}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_hash, PyHKEY_hashFunc}, {Py_tp_str, PyHKEY_strFunc}, + {Py_tp_richcompare, PyHKEY_richcompare}, // Number protocol {Py_nb_add, PyHKEY_binaryFailureFunc}, @@ -401,19 +428,6 @@ static PyType_Spec pyhkey_type_spec = { /************************************************************************ The public PyHKEY API (well, not public yet :-) ************************************************************************/ -PyObject * -PyHKEY_New(PyObject *m, HKEY hInit) -{ - winreg_state *st = _PyModule_GetState(m); - PyHKEYObject *key = PyObject_GC_New(PyHKEYObject, st->PyHKEY_Type); - if (key == NULL) { - return NULL; - } - key->hkey = hInit; - PyObject_GC_Track(key); - return (PyObject *)key; -} - BOOL PyHKEY_Close(winreg_state *st, PyObject *ob_handle) { @@ -426,7 +440,9 @@ PyHKEY_Close(winreg_state *st, PyObject *ob_handle) if (PyHKEY_Check(st, ob_handle)) { ((PyHKEYObject*)ob_handle)->hkey = 0; } + Py_BEGIN_ALLOW_THREADS rc = key ? RegCloseKey(key) : ERROR_SUCCESS; + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); return rc == ERROR_SUCCESS; @@ -487,41 +503,6 @@ PyHKEY_FromHKEY(winreg_state *st, HKEY h) } -/************************************************************************ - The module methods -************************************************************************/ -BOOL -PyWinObject_CloseHKEY(winreg_state *st, PyObject *obHandle) -{ - BOOL ok; - if (PyHKEY_Check(st, obHandle)) { - ok = PyHKEY_Close(st, obHandle); - } -#if SIZEOF_LONG >= SIZEOF_HKEY - else if (PyLong_Check(obHandle)) { - long rc = RegCloseKey((HKEY)PyLong_AsLong(obHandle)); - ok = (rc == ERROR_SUCCESS); - if (!ok) - PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); - } -#else - else if (PyLong_Check(obHandle)) { - long rc = RegCloseKey((HKEY)PyLong_AsVoidPtr(obHandle)); - ok = (rc == ERROR_SUCCESS); - if (!ok) - PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); - } -#endif - else { - PyErr_SetString( - PyExc_TypeError, - "A handle must be a HKEY object or an integer"); - return FALSE; - } - return ok; -} - - /* Private Helper functions for the registry interfaces @@ -924,7 +905,9 @@ winreg_CreateKey_impl(PyObject *module, HKEY key, const wchar_t *sub_key) (Py_ssize_t)KEY_WRITE) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegCreateKeyW(key, sub_key, &retKey); + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) { PyErr_SetFromWindowsErrWithFunction(rc, "CreateKey"); return NULL; @@ -973,8 +956,10 @@ winreg_CreateKeyEx_impl(PyObject *module, HKEY key, const wchar_t *sub_key, (Py_ssize_t)access) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegCreateKeyExW(key, sub_key, reserved, NULL, 0, access, NULL, &retKey, NULL); + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) { PyErr_SetFromWindowsErrWithFunction(rc, "CreateKeyEx"); return NULL; @@ -1187,10 +1172,12 @@ winreg_EnumValue_impl(PyObject *module, HKEY key, int index) (Py_ssize_t)key, index) < 0) { return NULL; } - if ((rc = RegQueryInfoKeyW(key, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, - &retValueSize, &retDataSize, NULL, NULL)) - != ERROR_SUCCESS) + + Py_BEGIN_ALLOW_THREADS + rc = RegQueryInfoKeyW(key, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + &retValueSize, &retDataSize, NULL, NULL); + Py_END_ALLOW_THREADS + if (rc != ERROR_SUCCESS) return PyErr_SetFromWindowsErrWithFunction(rc, "RegQueryInfoKey"); ++retValueSize; /* include null terminators */ @@ -1290,7 +1277,7 @@ winreg_ExpandEnvironmentStrings_impl(PyObject *module, const wchar_t *string) return PyErr_SetFromWindowsErrWithFunction(retValueSize, "ExpandEnvironmentStrings"); } - o = PyUnicode_FromWideChar(retValue, wcslen(retValue)); + o = PyUnicode_FromWideChar(retValue, -1); PyMem_Free(retValue); return o; } @@ -1477,9 +1464,11 @@ winreg_QueryInfoKey_impl(PyObject *module, HKEY key) if (PySys_Audit("winreg.QueryInfoKey", "n", (Py_ssize_t)key) < 0) { return NULL; } - if ((rc = RegQueryInfoKeyW(key, NULL, NULL, 0, &nSubKeys, NULL, NULL, - &nValues, NULL, NULL, NULL, &ft)) - != ERROR_SUCCESS) { + Py_BEGIN_ALLOW_THREADS + rc = RegQueryInfoKeyW(key, NULL, NULL, 0, &nSubKeys, NULL, NULL, + &nValues, NULL, NULL, NULL, &ft); + Py_END_ALLOW_THREADS + if (rc != ERROR_SUCCESS) { return PyErr_SetFromWindowsErrWithFunction(rc, "RegQueryInfoKey"); } li.LowPart = ft.dwLowDateTime; @@ -1587,7 +1576,9 @@ winreg_QueryValue_impl(PyObject *module, HKEY key, const wchar_t *sub_key) PyMem_Free(pbuf); } if (childKey != key) { + Py_BEGIN_ALLOW_THREADS RegCloseKey(childKey); + Py_END_ALLOW_THREADS } return result; } @@ -1625,7 +1616,9 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const wchar_t *name) (Py_ssize_t)key, NULL, name) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegQueryValueExW(key, name, NULL, NULL, NULL, &bufSize); + Py_END_ALLOW_THREADS if (rc == ERROR_MORE_DATA) bufSize = 256; else if (rc != ERROR_SUCCESS) @@ -1637,8 +1630,10 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const wchar_t *name) while (1) { retSize = bufSize; + Py_BEGIN_ALLOW_THREADS rc = RegQueryValueExW(key, name, NULL, &typ, (BYTE *)retBuf, &retSize); + Py_END_ALLOW_THREADS if (rc != ERROR_MORE_DATA) break; @@ -1998,6 +1993,45 @@ winreg_EnableReflectionKey_impl(PyObject *module, HKEY key) Py_RETURN_NONE; } +/*[clinic input] +winreg.DeleteTree + + key: HKEY + An already open key, or any one of the predefined HKEY_* constants. + sub_key: Py_UNICODE(accept={str, NoneType}) = None + A string that names the subkey to delete. If None, deletes all subkeys + and values of the specified key. + / + +Deletes the specified key and all its subkeys and values recursively. + +This function deletes a key and all its descendants. If sub_key is None, +all subkeys and values of the specified key are deleted. +[clinic start generated code]*/ + +static PyObject * +winreg_DeleteTree_impl(PyObject *module, HKEY key, const wchar_t *sub_key) +/*[clinic end generated code: output=c34395ee59290501 input=419ef9bb8b06e4bf]*/ +{ + LONG rc; + + if (PySys_Audit("winreg.DeleteTree", "nu", + (Py_ssize_t)key, sub_key) < 0) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rc = RegDeleteTreeW(key, sub_key); + Py_END_ALLOW_THREADS + + if (rc != ERROR_SUCCESS) { + PyErr_SetFromWindowsErrWithFunction(rc, "RegDeleteTreeW"); + return NULL; + } + + Py_RETURN_NONE; +} + /*[clinic input] winreg.QueryReflectionKey @@ -2056,6 +2090,7 @@ static struct PyMethodDef winreg_methods[] = { WINREG_DELETEKEY_METHODDEF WINREG_DELETEKEYEX_METHODDEF WINREG_DELETEVALUE_METHODDEF + WINREG_DELETETREE_METHODDEF WINREG_DISABLEREFLECTIONKEY_METHODDEF WINREG_ENABLEREFLECTIONKEY_METHODDEF WINREG_ENUMKEY_METHODDEF diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index ee7421484b5..3ba49370d58 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -108,7 +108,6 @@ <ClInclude Include="$(mpdecimalDir)\libmpdec\convolute.h" /> <ClInclude Include="$(mpdecimalDir)\libmpdec\crt.h" /> <ClInclude Include="$(mpdecimalDir)\libmpdec\difradix2.h" /> - <ClInclude Include="..\Modules\_decimal\docstrings.h" /> <ClInclude Include="$(mpdecimalDir)\libmpdec\fnt.h" /> <ClInclude Include="$(mpdecimalDir)\libmpdec\fourstep.h" /> <ClInclude Include="..\Modules\_decimal\windows\mpdecimal.h" /> diff --git a/PCbuild/_decimal.vcxproj.filters b/PCbuild/_decimal.vcxproj.filters index e4bdb64ec1f..7cb6b867a47 100644 --- a/PCbuild/_decimal.vcxproj.filters +++ b/PCbuild/_decimal.vcxproj.filters @@ -18,9 +18,6 @@ </Filter> </ItemGroup> <ItemGroup> - <ClInclude Include="..\Modules\_decimal\docstrings.h"> - <Filter>Header Files</Filter> - </ClInclude> <ClInclude Include="$(mpdecimalDir)\libmpdec\basearith.h"> <Filter>Header Files\libmpdec</Filter> </ClInclude> diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 8c161835af9..605861ad3fd 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -106,6 +106,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\atexitmodule.c" /> + <ClCompile Include="..\Modules\_datetimemodule.c" /> <ClCompile Include="..\Modules\faulthandler.c" /> <ClCompile Include="..\Modules\gcmodule.c" /> <ClCompile Include="..\Modules\getbuildinfo.c" /> @@ -163,7 +164,10 @@ <ClCompile Include="..\Objects\tupleobject.c" /> <ClCompile Include="..\Objects\typeobject.c" /> <ClCompile Include="..\Objects\typevarobject.c" /> + <ClCompile Include="..\Objects\unicode_format.c" /> <ClCompile Include="..\Objects\unicodectype.c" /> + <ClCompile Include="..\Objects\unicode_formatter.c" /> + <ClCompile Include="..\Objects\unicode_writer.c" /> <ClCompile Include="..\Objects\unicodeobject.c" /> <ClCompile Include="..\Objects\unionobject.c" /> <ClCompile Include="..\Objects\weakrefobject.c" /> @@ -208,7 +212,6 @@ <ClCompile Include="..\Python\errors.c" /> <ClCompile Include="..\Python\fileutils.c" /> <ClCompile Include="..\Python\flowgraph.c" /> - <ClCompile Include="..\Python\formatter_unicode.c" /> <ClCompile Include="..\Python\frame.c" /> <ClCompile Include="..\Python\future.c" /> <ClCompile Include="..\Python\gc.c" /> @@ -254,6 +257,7 @@ <ClCompile Include="..\Python\pylifecycle.c" /> <ClCompile Include="..\Python\pymath.c" /> <ClCompile Include="..\Python\pystate.c" /> + <ClCompile Include="..\Python\pystats.c" /> <ClCompile Include="..\Python\pystrcmp.c" /> <ClCompile Include="..\Python\pystrhex.c" /> <ClCompile Include="..\Python\pystrtod.c" /> @@ -276,7 +280,7 @@ <ClCompile Include="..\Python\uniqueid.c" /> </ItemGroup> <ItemGroup> - <ClInclude Include="..\PC\pyconfig.h.in" /> + <ClInclude Include="..\PC\pyconfig.h" /> </ItemGroup> <ItemGroup> <!-- BEGIN frozen modules --> @@ -436,31 +440,6 @@ <ImportGroup Label="ExtensionTargets"> </ImportGroup> - <!-- Direct copy from pythoncore.vcxproj, but this one is only used for our - own build. All other extension modules will use the copy that pythoncore - generates. --> - <Target Name="_UpdatePyconfig" BeforeTargets="PrepareForBuild"> - <MakeDir Directories="$(IntDir)" Condition="!Exists($(IntDir))" /> - <ItemGroup> - <PyConfigH Remove="@(PyConfigH)" /> - <PyConfigH Include="@(ClInclude)" Condition="'%(Filename)%(Extension)' == 'pyconfig.h.in'" /> - </ItemGroup> - <Error Text="Did not find pyconfig.h" Condition="@(ClInclude) == ''" /> - <PropertyGroup> - <PyConfigH>@(PyConfigH->'%(FullPath)', ';')</PyConfigH> - <PyConfigHText>$([System.IO.File]::ReadAllText($(PyConfigH)))</PyConfigHText> - <OldPyConfigH Condition="Exists('$(IntDir)pyconfig.h')">$([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h'))</OldPyConfigH> - </PropertyGroup> - <PropertyGroup Condition="$(DisableGil) == 'true'"> - <PyConfigHText>$(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1'))</PyConfigHText> - </PropertyGroup> - <Message Text="Updating pyconfig.h" Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> - <WriteLinesToFile File="$(IntDir)pyconfig.h" - Lines="$(PyConfigHText)" - Overwrite="true" - Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> - </Target> - <Target Name="_RebuildGetPath" AfterTargets="_RebuildFrozen" Condition="$(Configuration) != 'PGUpdate'"> <Exec Command='"$(TargetPath)" "%(GetPath.ModName)" "%(GetPath.FullPath)" "%(GetPath.IntFile)"' /> diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 332d466b1f7..c67fe53363e 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -160,9 +160,6 @@ <ClCompile Include="..\Python\flowgraph.c"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="..\Python\formatter_unicode.c"> - <Filter>Source Files</Filter> - </ClCompile> <ClCompile Include="..\Python\frame.c"> <Filter>Source Files</Filter> </ClCompile> @@ -370,6 +367,9 @@ <ClCompile Include="..\Python\pystate.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\pystats.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Python\pystrcmp.c"> <Filter>Source Files</Filter> </ClCompile> @@ -484,9 +484,18 @@ <ClCompile Include="..\Objects\typeobject.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Objects\unicode_format.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Objects\unicodectype.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Objects\unicode_formatter.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\unicode_writer.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Objects\unicodeobject.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/_remote_debugging.vcxproj b/PCbuild/_remote_debugging.vcxproj index c55f2908e03..c91c9cf3652 100644 --- a/PCbuild/_remote_debugging.vcxproj +++ b/PCbuild/_remote_debugging.vcxproj @@ -92,8 +92,22 @@ <PropertyGroup> <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion> </PropertyGroup> + <ItemDefinitionGroup> + <Link> + <AdditionalDependencies>ntdll.lib;%(AdditionalDependencies)</AdditionalDependencies> + </Link> +</ItemDefinitionGroup> <ItemGroup> - <ClCompile Include="..\Modules\_remote_debugging_module.c" /> + <ClCompile Include="..\Modules\_remote_debugging\module.c" /> + <ClCompile Include="..\Modules\_remote_debugging\object_reading.c" /> + <ClCompile Include="..\Modules\_remote_debugging\code_objects.c" /> + <ClCompile Include="..\Modules\_remote_debugging\frames.c" /> + <ClCompile Include="..\Modules\_remote_debugging\frame_cache.c" /> + <ClCompile Include="..\Modules\_remote_debugging\threads.c" /> + <ClCompile Include="..\Modules\_remote_debugging\asyncio.c" /> + </ItemGroup> + <ItemGroup> + <ClInclude Include="..\Modules\_remote_debugging\_remote_debugging.h" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> diff --git a/PCbuild/_remote_debugging.vcxproj.filters b/PCbuild/_remote_debugging.vcxproj.filters index ce4437f74e0..b37a2c5575c 100644 --- a/PCbuild/_remote_debugging.vcxproj.filters +++ b/PCbuild/_remote_debugging.vcxproj.filters @@ -4,12 +4,40 @@ <Filter Include="Source Files"> <UniqueIdentifier>{6d101329-41df-49a0-8639-f35408ad7c6d}</UniqueIdentifier> </Filter> + <Filter Include="Header Files"> + <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier> + </Filter> <Filter Include="Resource Files"> <UniqueIdentifier>{711941d1-269c-49cb-a733-759b2b91fc61}</UniqueIdentifier> </Filter> </ItemGroup> <ItemGroup> - <ClCompile Include="..\Modules\_remote_debugging_module.c" /> + <ClCompile Include="..\Modules\_remote_debugging\module.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_remote_debugging\object_reading.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_remote_debugging\code_objects.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_remote_debugging\frames.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_remote_debugging\frame_cache.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_remote_debugging\threads.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_remote_debugging\asyncio.c"> + <Filter>Source Files</Filter> + </ClCompile> + </ItemGroup> + <ItemGroup> + <ClInclude Include="..\Modules\_remote_debugging\_remote_debugging.h"> + <Filter>Header Files</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc"> diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index a68f15d25aa..68707a54ff6 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -125,6 +125,8 @@ <ClCompile Include="..\Modules\_testcapi\immortal.c" /> <ClCompile Include="..\Modules\_testcapi\gc.c" /> <ClCompile Include="..\Modules\_testcapi\run.c" /> + <ClCompile Include="..\Modules\_testcapi\modsupport.c" /> + <ClCompile Include="..\Modules\_testcapi\module.c" /> <ClCompile Include="..\Modules\_testcapi\monitoring.c" /> <ClCompile Include="..\Modules\_testcapi\config.c" /> <ClCompile Include="..\Modules\_testcapi\import.c" /> diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 21091e9dc1a..b0e75ce433a 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -108,6 +108,12 @@ <ClCompile Include="..\Modules\_testcapi\run.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\_testcapi\modsupport.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_testcapi\module.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Modules\_testcapi\monitoring.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/_testclinic_limited.vcxproj b/PCbuild/_testclinic_limited.vcxproj index 183a55080e8..95c205309b1 100644 --- a/PCbuild/_testclinic_limited.vcxproj +++ b/PCbuild/_testclinic_limited.vcxproj @@ -70,6 +70,7 @@ <ProjectGuid>{01FDF29A-40A1-46DF-84F5-85EBBD2A2410}</ProjectGuid> <RootNamespace>_testclinic_limited</RootNamespace> <Keyword>Win32Proj</Keyword> + <SupportPGO>false</SupportPGO> </PropertyGroup> <Import Project="python.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> diff --git a/PCbuild/_zstd.vcxproj b/PCbuild/_zstd.vcxproj index b53f93a6af8..6f91b8d05cc 100644 --- a/PCbuild/_zstd.vcxproj +++ b/PCbuild/_zstd.vcxproj @@ -137,6 +137,7 @@ <ItemGroup> <ClInclude Include="..\Modules\_zstd\_zstdmodule.h" /> <ClInclude Include="..\Modules\_zstd\buffer.h" /> + <ClInclude Include="..\Modules\_zstd\zstddict.h" /> <ClInclude Include="$(zstdDir)lib\common\bitstream.h" /> <ClInclude Include="$(zstdDir)lib\common\error_private.h" /> <ClInclude Include="$(zstdDir)lib\common\fse.h" /> diff --git a/PCbuild/_zstd.vcxproj.filters b/PCbuild/_zstd.vcxproj.filters index d4d36063c85..eec666e5eaf 100644 --- a/PCbuild/_zstd.vcxproj.filters +++ b/PCbuild/_zstd.vcxproj.filters @@ -128,6 +128,9 @@ <ClInclude Include="..\Modules\_zstd\buffer.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\Modules\_zstd\zstddict.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="$(zstdDir)lib\zstd.h"> <Filter>Header Files\zstd</Filter> </ClInclude> diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 2f358991e48..e4de9a80d76 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -33,7 +33,7 @@ echo. -k Attempt to kill any running Pythons before building (usually done echo. automatically by the pythoncore project) echo. --pgo Build with Profile-Guided Optimization. This flag echo. overrides -c and -d -echo. --disable-gil Enable experimental support for running without the GIL. +echo. --disable-gil Enable support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. echo. --experimental-jit Enable the experimental just-in-time compiler. @@ -41,6 +41,7 @@ echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enable echo. --experimental-jit-interpreter Enable the experimental Tier 2 interpreter. echo. --pystats Enable PyStats collection. echo. --tail-call-interp Enable tail-calling interpreter (requires LLVM 19 or higher). +echo. --enable-stackref-debug Enable stackref debugging mode. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -98,6 +99,7 @@ if "%~1"=="--experimental-jit-interpreter-off" (set UseTIER2=6) & shift & goto C if "%~1"=="--without-remote-debug" (set DisableRemoteDebug=true) & shift & goto CheckOpts if "%~1"=="--pystats" (set PyStats=1) & shift & goto CheckOpts if "%~1"=="--tail-call-interp" (set UseTailCallInterp=true) & shift & goto CheckOpts +if "%~1"=="--enable-stackref-debug" (set StackRefDebug=true) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -202,6 +204,7 @@ echo on /p:PyStats=%PyStats%^ /p:UseTailCallInterp=%UseTailCallInterp%^ /p:DisableRemoteDebug=%DisableRemoteDebug%^ + /p:StackRefDebug=%StackRefDebug%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/field3.py b/PCbuild/field3.py deleted file mode 100644 index edcbe36ae08..00000000000 --- a/PCbuild/field3.py +++ /dev/null @@ -1,35 +0,0 @@ -# An absurd workaround for the lack of arithmetic in MS's resource compiler. -# After building Python, run this, then paste the output into the appropriate -# part of PC\python_nt.rc. -# Example output: -# -# * For 2.3a0, -# * PY_MICRO_VERSION = 0 -# * PY_RELEASE_LEVEL = 'alpha' = 0xA -# * PY_RELEASE_SERIAL = 1 -# * -# * and 0*1000 + 10*10 + 1 = 101. -# */ -# #define FIELD3 101 - -import sys - -major, minor, micro, level, serial = sys.version_info -levelnum = {'alpha': 0xA, - 'beta': 0xB, - 'candidate': 0xC, - 'final': 0xF, - }[level] -string = sys.version.split()[0] # like '2.3a0' - -print(" * For %s," % string) -print(" * PY_MICRO_VERSION = %d" % micro) -print(" * PY_RELEASE_LEVEL = %r = %s" % (level, hex(levelnum))) -print(" * PY_RELEASE_SERIAL = %d" % serial) -print(" *") - -field3 = micro * 1000 + levelnum * 10 + serial - -print(" * and %d*1000 + %d*10 + %d = %d" % (micro, levelnum, serial, field3)) -print(" */") -print("#define FIELD3", field3) diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d65d080ca71..841d83968c6 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -47,7 +47,7 @@ @rem If py.exe finds a recent enough version, use that one @rem It is fine to add new versions to this list when they have released, @rem but we do not use prerelease builds here. -@for %%p in (3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.14 3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 4ecc8925349..edf14ce578b 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -4,26 +4,63 @@ import os import pathlib import sys +import tarfile import time +import urllib.error +import urllib.request import zipfile -from urllib.request import urlretrieve +def retrieve_with_retries(download_location, output_path, reporthook, + max_retries=7): + """Download a file with exponential backoff retry and save to disk.""" + for attempt in range(max_retries + 1): + try: + resp = urllib.request.urlretrieve( + download_location, + output_path, + reporthook=reporthook, + ) + except (urllib.error.URLError, ConnectionError) as ex: + if attempt == max_retries: + raise OSError(f'Download from {download_location} failed.') from ex + time.sleep(2.25**attempt) + else: + return resp + def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): - repo = f'cpython-{"bin" if binary else "source"}-deps' + repo = 'cpython-bin-deps' if binary else 'cpython-source-deps' url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip' reporthook = None if verbose: reporthook = print zip_dir.mkdir(parents=True, exist_ok=True) - filename, headers = urlretrieve( + filename, _headers = retrieve_with_retries( url, zip_dir / f'{commit_hash}.zip', - reporthook=reporthook, + reporthook ) return filename +def fetch_release(tag, tarball_dir, *, org='python', verbose=False): + url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{tag}.tar.xz' + reporthook = None + if verbose: + reporthook = print + tarball_dir.mkdir(parents=True, exist_ok=True) + output_path = tarball_dir / f'{tag}.tar.xz' + retrieve_with_retries(url, output_path, reporthook) + return output_path + + +def extract_tarball(externals_dir, tarball_path, tag): + output_path = externals_dir / tag + with tarfile.open(tarball_path) as tf: + tf.extractall(os.fspath(externals_dir)) + return output_path + + def extract_zip(externals_dir, zip_path): with zipfile.ZipFile(os.fspath(zip_path)) as zf: zf.extractall(os.fspath(externals_dir)) @@ -35,6 +72,8 @@ def parse_args(): p.add_argument('-v', '--verbose', action='store_true') p.add_argument('-b', '--binary', action='store_true', help='Is the dependency in the binary repo?') + p.add_argument('-r', '--release', action='store_true', + help='Download from GitHub release assets instead of branch') p.add_argument('-O', '--organization', help='Organization owning the deps repos', default='python') p.add_argument('-e', '--externals-dir', type=pathlib.Path, @@ -47,30 +86,53 @@ def parse_args(): def main(): args = parse_args() - zip_path = fetch_zip( - args.tag, - args.externals_dir / 'zips', - org=args.organization, - binary=args.binary, - verbose=args.verbose, - ) final_name = args.externals_dir / args.tag - extracted = extract_zip(args.externals_dir, zip_path) - for wait in [1, 2, 3, 5, 8, 0]: - try: - extracted.replace(final_name) - break - except PermissionError as ex: - retry = f" Retrying in {wait}s..." if wait else "" - print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) - time.sleep(wait) - else: - print( - f"ERROR: Failed to extract {final_name}.", - "You may need to restart your build", - file=sys.stderr, + + # Check if the dependency already exists in externals/ directory + # (either already downloaded/extracted, or checked into the git tree) + if final_name.exists(): + if args.verbose: + print(f'{args.tag} already exists at {final_name}, skipping download.') + return + + # Determine download method: release artifacts for large deps (like LLVM), + # otherwise zip download from GitHub branches + if args.release: + tarball_path = fetch_release( + args.tag, + args.externals_dir / 'tarballs', + org=args.organization, + verbose=args.verbose, ) - sys.exit(1) + extracted = extract_tarball(args.externals_dir, tarball_path, args.tag) + else: + # Use zip download from GitHub branches + # (cpython-bin-deps if --binary, cpython-source-deps otherwise) + zip_path = fetch_zip( + args.tag, + args.externals_dir / 'zips', + org=args.organization, + binary=args.binary, + verbose=args.verbose, + ) + extracted = extract_zip(args.externals_dir, zip_path) + + if extracted != final_name: + for wait in [1, 2, 3, 5, 8, 0]: + try: + extracted.replace(final_name) + break + except PermissionError as ex: + retry = f" Retrying in {wait}s..." if wait else "" + print(f"Encountered permission error '{ex}'.{retry}", file=sys.stderr) + time.sleep(wait) + else: + print( + f"ERROR: Failed to rename {extracted} to {final_name}.", + "You may need to restart your build", + file=sys.stderr, + ) + sys.exit(1) if __name__ == '__main__': diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index e29054f5734..9d02e2121cc 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,12 +54,12 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.16 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.18 set libraries=%libraries% mpdecimal-4.0.0 -set libraries=%libraries% sqlite-3.49.1.0 +set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 -set libraries=%libraries% xz-5.2.5 +set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 @@ -79,10 +79,10 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.16.2 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-19.1.7.0 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -92,7 +92,11 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - %PYTHON% -E "%PCBUILD%\get_external.py" -b -O %ORG% -e "%EXTERNALS_DIR%" %%b + if "%%b"=="llvm-21.1.4.0" ( + %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b + ) else ( + %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b + ) ) ) diff --git a/PCbuild/liblzma.vcxproj b/PCbuild/liblzma.vcxproj index 97938692328..75d4e162346 100644 --- a/PCbuild/liblzma.vcxproj +++ b/PCbuild/liblzma.vcxproj @@ -92,7 +92,7 @@ <ItemDefinitionGroup> <ClCompile> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories>$(lzmaDir)windows/vs2019;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories>$(lzmaDir)windows;$(lzmaDir)src/liblzma/common;$(lzmaDir)src/common;$(lzmaDir)src/liblzma/api;$(lzmaDir)src/liblzma/check;$(lzmaDir)src/liblzma/delta;$(lzmaDir)src/liblzma/lz;$(lzmaDir)src/liblzma/lzma;$(lzmaDir)src/liblzma/rangecoder;$(lzmaDir)src/liblzma/simple;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <DisableSpecificWarnings>4244;4267;4996;%(DisableSpecificWarnings)</DisableSpecificWarnings> <AdditionalOptions Condition="$(PlatformToolset) == 'ClangCL'">%(AdditionalOptions) -Wno-deprecated-declarations</AdditionalOptions> </ClCompile> @@ -102,9 +102,7 @@ <ClCompile Include="$(lzmaDir)src\common\tuklib_physmem.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\check\check.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\check\crc32_fast.c" /> - <ClCompile Include="$(lzmaDir)src\liblzma\check\crc32_table.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\check\crc64_fast.c" /> - <ClCompile Include="$(lzmaDir)src\liblzma\check\crc64_table.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\check\sha256.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\common\alone_decoder.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\common\alone_encoder.c" /> @@ -163,6 +161,7 @@ <ClCompile Include="$(lzmaDir)src\liblzma\lz\lz_encoder_mf.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\rangecoder\price_table.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\simple\arm.c" /> + <ClCompile Include="$(lzmaDir)src\liblzma\simple\arm64.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\simple\armthumb.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\simple\ia64.c" /> <ClCompile Include="$(lzmaDir)src\liblzma\simple\powerpc.c" /> @@ -239,7 +238,7 @@ <ClInclude Include="$(lzmaDir)src\liblzma\simple\simple_decoder.h" /> <ClInclude Include="$(lzmaDir)src\liblzma\simple\simple_encoder.h" /> <ClInclude Include="$(lzmaDir)src\liblzma\simple\simple_private.h" /> - <ClInclude Include="$(lzmaDir)windows\vs2019\config.h" /> + <ClInclude Include="$(lzmaDir)windows\config.h" /> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> diff --git a/PCbuild/liblzma.vcxproj.filters b/PCbuild/liblzma.vcxproj.filters index ebe2a7d5fa9..42feca5a341 100644 --- a/PCbuild/liblzma.vcxproj.filters +++ b/PCbuild/liblzma.vcxproj.filters @@ -18,6 +18,9 @@ <ClCompile Include="$(lzmaDir)src\liblzma\simple\arm.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="$(lzmaDir)src\liblzma\simple\arm64.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="$(lzmaDir)src\liblzma\simple\armthumb.c"> <Filter>Source Files</Filter> </ClCompile> @@ -54,15 +57,9 @@ <ClCompile Include="$(lzmaDir)src\liblzma\check\crc32_fast.c"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="$(lzmaDir)src\liblzma\check\crc32_table.c"> - <Filter>Source Files</Filter> - </ClCompile> <ClCompile Include="$(lzmaDir)src\liblzma\check\crc64_fast.c"> <Filter>Source Files</Filter> </ClCompile> - <ClCompile Include="$(lzmaDir)src\liblzma\check\crc64_table.c"> - <Filter>Source Files</Filter> - </ClCompile> <ClCompile Include="$(lzmaDir)src\liblzma\delta\delta_common.c"> <Filter>Source Files</Filter> </ClCompile> @@ -428,7 +425,7 @@ <ClInclude Include="$(lzmaDir)src\liblzma\simple\simple_private.h"> <Filter>Header Files</Filter> </ClInclude> - <ClInclude Include="$(lzmaDir)windows\vs2019\config.h"> + <ClInclude Include="$(lzmaDir)windows\config.h"> <Filter>Header Files</Filter> </ClInclude> </ItemGroup> diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 4e414dc913b..53bfe5e3ea9 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,10 +10,10 @@ <Py_IntDir Condition="'$(Py_IntDir)' == ''">$(MSBuildThisFileDirectory)obj\</Py_IntDir> <IntDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir> <IntDir>$(IntDir.Replace(`\\`, `\`))</IntDir> - <!-- pyconfig.h is updated by pythoncore.vcxproj, so it's always in pythoncore's IntDir --> - <GeneratedPyConfigDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\</GeneratedPyConfigDir> <GeneratedFrozenModulesDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\</GeneratedFrozenModulesDir> <GeneratedZlibNgDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\</GeneratedZlibNgDir> + <GeneratedJitStencilsDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration)</GeneratedJitStencilsDir> + <GeneratedJitStencilsDir Condition="$(Configuration) == 'PGUpdate'">$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_PGInstrument</GeneratedJitStencilsDir> <TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName> <TargetName>$(TargetName)$(PyDebugExt)</TargetName> <GenerateManifest>false</GenerateManifest> @@ -49,11 +49,12 @@ <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64;</_PlatformPreprocessorDefinition> <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition)</_PlatformPreprocessorDefinition> <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)";</_Py3NamePreprocessorDefinition> + <_FreeThreadedPreprocessorDefinition Condition="$(DisableGil) == 'true'">Py_GIL_DISABLED=1;</_FreeThreadedPreprocessorDefinition> </PropertyGroup> <ItemDefinitionGroup> <ClCompile> - <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <PreprocessorDefinitions>WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions> + <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions>WIN32;$(_Py3NamePreprocessorDefinition)$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)$(_FreeThreadedPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="'$(SupportPGO)' and ($(Configuration) == 'PGInstrument' or $(Configuration) == 'PGUpdate')">_Py_USING_PGO=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <Optimization>MaxSpeed</Optimization> @@ -96,19 +97,16 @@ <TargetMachine Condition="'$(Platform)' == 'x64'">MachineX64</TargetMachine> <TargetMachine Condition="'$(Platform)'=='ARM'">MachineARM</TargetMachine> <TargetMachine Condition="'$(Platform)'=='ARM64'">MachineARM64</TargetMachine> - <ProfileGuidedDatabase Condition="$(SupportPGO)">$(OutDir)$(TargetName).pgd</ProfileGuidedDatabase> - <LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">UseLinkTimeCodeGeneration</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">PGInstrument</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">PGUpdate</LinkTimeCodeGeneration> + <LinkTimeCodeGeneration Condition="$(Configuration) != 'Debug'">UseLinkTimeCodeGeneration</LinkTimeCodeGeneration> <AdditionalDependencies>advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalOptions Condition="$(Configuration) != 'Debug'">/OPT:REF,NOICF %(AdditionalOptions)</AdditionalOptions> <AdditionalOptions Condition="$(MSVCHasBrokenARM64Clamping) == 'true' and $(Platform) == 'ARM64'">-d2:-pattern-opt-disable:-932189325 %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument' and $(PlatformToolset) != 'ClangCL'">/GENPROFILE %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate' and $(PlatformToolset) != 'ClangCL'">/USEPROFILE %(AdditionalOptions)</AdditionalOptions> </Link> <Lib> <LinkTimeCodeGeneration>false</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">true</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">true</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">true</LinkTimeCodeGeneration> + <LinkTimeCodeGeneration Condition="$(Configuration) != 'Debug'">true</LinkTimeCodeGeneration> </Lib> <ResourceCompile> <AdditionalIncludeDirectories>$(PySourcePath)PC;$(PySourcePath)Include;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> diff --git a/PCbuild/python.props b/PCbuild/python.props index ddc7696d276..7840e2a1cfc 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -11,6 +11,7 @@ We set BasePlatformToolset for ICC's benefit, it's otherwise ignored. --> + <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '18.0'">v143</BasePlatformToolset> <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '17.0'">v143</BasePlatformToolset> <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '16.0'">v142</BasePlatformToolset> <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and ('$(MSBuildToolsVersion)' == '15.0' or '$(VisualStudioVersion)' == '15.0')">v141</BasePlatformToolset> @@ -74,15 +75,15 @@ <Import Project="$(ExternalProps)" Condition="$(ExternalProps) != '' and Exists('$(ExternalProps)')" /> <PropertyGroup> - <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.49.1.0\</sqlite3Dir> + <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.50.4.0\</sqlite3Dir> <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir> - <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.2.5\</lzmaDir> + <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.8.1.1\</lzmaDir> <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir> <libffiOutDir Condition="$(libffiOutDir) == ''">$(libffiDir)$(ArchName)\</libffiOutDir> <libffiIncludeDir Condition="$(libffiIncludeDir) == ''">$(libffiOutDir)include</libffiIncludeDir> <mpdecimalDir Condition="$(mpdecimalDir) == ''">$(ExternalsDir)\mpdecimal-4.0.0\</mpdecimalDir> - <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.16\</opensslDir> - <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.16.2\$(ArchName)\</opensslOutDir> + <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.18\</opensslDir> + <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.18\$(ArchName)\</opensslOutDir> <opensslIncludeDir Condition="$(opensslIncludeDir) == ''">$(opensslOutDir)include</opensslIncludeDir> <nasmDir Condition="$(nasmDir) == ''">$(ExternalsDir)\nasm-2.11.06\</nasmDir> <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.3.1\</zlibDir> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 549d6284972..85363949c23 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -102,13 +102,15 @@ <AdditionalOptions>/Zm200 %(AdditionalOptions)</AdditionalOptions> <AdditionalIncludeDirectories>$(PySourcePath)Modules\_hacl;$(PySourcePath)Modules\_hacl\include;$(PySourcePath)Python;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibNgDir);$(GeneratedZlibNgDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(UseJIT)' == 'true'">$(GeneratedJitStencilsDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";ZLIB_COMPAT;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="'$(UseTIER2)' != '' and '$(UseTIER2)' != '0'">_Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions)</PreprocessorDefinitions> - <PreprocessorDefinitions Condition="'$(UseTailCallInterp)' == 'true'">Py_TAIL_CALL_INTERP=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(UseTailCallInterp)' == 'true'">_Py_TAIL_CALL_INTERP=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="'$(WITH_COMPUTED_GOTOS)' != ''">HAVE_COMPUTED_GOTOS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="'$(DisableRemoteDebug)' != 'true'">Py_REMOTE_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(StackRefDebug)' == 'true'">Py_STACKREF_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> <Link> <AdditionalDependencies>version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies)</AdditionalDependencies> @@ -164,12 +166,14 @@ <ClInclude Include="..\Include\cpython\import.h" /> <ClInclude Include="..\Include\cpython\initconfig.h" /> <ClInclude Include="..\Include\cpython\listobject.h" /> - <ClInclude Include="..\Include\cpython\lock.h" /> + <ClInclude Include="..\Include\cpython\pylock.h" /> <ClInclude Include="..\Include\cpython\longintrepr.h" /> <ClInclude Include="..\Include\cpython\longobject.h" /> + <ClInclude Include="..\Include\cpython\marshal.h" /> <ClInclude Include="..\Include\cpython\memoryobject.h" /> <ClInclude Include="..\Include\cpython\methodobject.h" /> <ClInclude Include="..\Include\cpython\modsupport.h" /> + <ClInclude Include="..\Include\cpython\monitoring.h" /> <ClInclude Include="..\Include\cpython\object.h" /> <ClInclude Include="..\Include\cpython\objimpl.h" /> <ClInclude Include="..\Include\cpython\odictobject.h" /> @@ -189,6 +193,8 @@ <ClInclude Include="..\Include\cpython\pythonrun.h" /> <ClInclude Include="..\Include\cpython\pythread.h" /> <ClInclude Include="..\Include\cpython\setobject.h" /> + <ClInclude Include="..\Include\cpython\sliceobject.h" /> + <ClInclude Include="..\Include\cpython\structseq.h" /> <ClInclude Include="..\Include\cpython\traceback.h" /> <ClInclude Include="..\Include\cpython\tracemalloc.h" /> <ClInclude Include="..\Include\cpython\tupleobject.h" /> @@ -323,6 +329,7 @@ <ClInclude Include="..\Include\internal\pycore_typevarobject.h" /> <ClInclude Include="..\Include\internal\pycore_ucnhash.h" /> <ClInclude Include="..\Include\internal\pycore_unionobject.h" /> + <ClInclude Include="..\Include\internal\pycore_unicodectype.h" /> <ClInclude Include="..\Include\internal\pycore_unicodeobject.h" /> <ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h" /> <ClInclude Include="..\Include\internal\pycore_uniqueid.h" /> @@ -331,7 +338,6 @@ <ClInclude Include="..\Include\intrcheck.h" /> <ClInclude Include="..\Include\iterobject.h" /> <ClInclude Include="..\Include\listobject.h" /> - <ClInclude Include="..\Include\lock.h" /> <ClInclude Include="..\Include\longobject.h" /> <ClInclude Include="..\Include\marshal.h" /> <ClInclude Include="..\Include\memoryobject.h" /> @@ -409,7 +415,7 @@ <ClInclude Include="..\Parser\string_parser.h" /> <ClInclude Include="..\Parser\pegen.h" /> <ClInclude Include="..\PC\errmap.h" /> - <ClInclude Include="..\PC\pyconfig.h.in" /> + <ClInclude Include="..\PC\pyconfig.h" /> <ClInclude Include="..\Python\condvar.h" /> <ClInclude Include="..\Python\stdlib_module_names.h" /> <ClInclude Include="..\Python\thread_nt.h" /> @@ -418,8 +424,12 @@ <ClCompile Include="..\Modules\_abc.c" /> <ClCompile Include="..\Modules\_bisectmodule.c" /> <ClCompile Include="..\Modules\blake2module.c"> - <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'">HACL_CAN_COMPILE_SIMD128;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'">HACL_CAN_COMPILE_SIMD256;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'"> + _Py_HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions) + </PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'"> + _Py_HACL_CAN_COMPILE_VEC256;%(PreprocessorDefinitions) + </PreprocessorDefinitions> </ClCompile> <ClCompile Include="..\Modules\_codecsmodule.c" /> <ClCompile Include="..\Modules\_collectionsmodule.c" /> @@ -470,6 +480,7 @@ <ClCompile Include="..\Modules\hmacmodule.c" /> <ClCompile Include="..\Modules\itertoolsmodule.c" /> <ClCompile Include="..\Modules\main.c" /> + <ClCompile Include="..\Modules\mathintegermodule.c" /> <ClCompile Include="..\Modules\mathmodule.c" /> <ClCompile Include="..\Modules\md5module.c" /> <ClCompile Include="..\Modules\mmapmodule.c" /> @@ -550,7 +561,10 @@ <ClCompile Include="..\Objects\tupleobject.c" /> <ClCompile Include="..\Objects\typeobject.c" /> <ClCompile Include="..\Objects\typevarobject.c" /> + <ClCompile Include="..\Objects\unicode_format.c" /> <ClCompile Include="..\Objects\unicodectype.c" /> + <ClCompile Include="..\Objects\unicode_formatter.c" /> + <ClCompile Include="..\Objects\unicode_writer.c" /> <ClCompile Include="..\Objects\unicodeobject.c" /> <ClCompile Include="..\Objects\unionobject.c" /> <ClCompile Include="..\Objects\weakrefobject.c" /> @@ -597,7 +611,6 @@ <ClCompile Include="..\Python\errors.c" /> <ClCompile Include="..\Python\fileutils.c" /> <ClCompile Include="..\Python\flowgraph.c" /> - <ClCompile Include="..\Python\formatter_unicode.c" /> <ClCompile Include="..\Python\frame.c" /> <ClCompile Include="..\Python\frozen.c"> <AdditionalIncludeDirectories>$(GeneratedFrozenModulesDir)Python;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> @@ -646,6 +659,7 @@ <ClCompile Include="..\Python\pymath.c" /> <ClCompile Include="..\Python\pytime.c" /> <ClCompile Include="..\Python\pystate.c" /> + <ClCompile Include="..\Python\pystats.c" /> <ClCompile Include="..\Python\pystrcmp.c" /> <ClCompile Include="..\Python\pystrhex.c" /> <ClCompile Include="..\Python\pystrtod.c" /> @@ -688,34 +702,6 @@ </ImportGroup> <Target Name="_TriggerRegen" BeforeTargets="PrepareForBuild" DependsOnTargets="Regen" /> - <Target Name="_UpdatePyconfig" BeforeTargets="PrepareForBuild"> - <MakeDir Directories="$(IntDir)" Condition="!Exists($(IntDir))" /> - <ItemGroup> - <PyConfigH Remove="@(PyConfigH)" /> - <PyConfigH Include="@(ClInclude)" Condition="'%(Filename)%(Extension)' == 'pyconfig.h.in'" /> - </ItemGroup> - <Error Text="Did not find pyconfig.h" Condition="@(ClInclude) == ''" /> - <PropertyGroup> - <PyConfigH>@(PyConfigH->'%(FullPath)', ';')</PyConfigH> - <PyConfigHText>$([System.IO.File]::ReadAllText($(PyConfigH)))</PyConfigHText> - <OldPyConfigH Condition="Exists('$(IntDir)pyconfig.h')">$([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h'))</OldPyConfigH> - </PropertyGroup> - <PropertyGroup Condition="$(DisableGil) == 'true'"> - <PyConfigHText>$(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1'))</PyConfigHText> - </PropertyGroup> - <Message Text="Updating pyconfig.h" Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> - <WriteLinesToFile File="$(IntDir)pyconfig.h" - Lines="$(PyConfigHText)" - Overwrite="true" - Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> - </Target> - <Target Name="_CopyPyconfig" Inputs="$(IntDir)pyconfig.h" Outputs="$(OutDir)pyconfig.h" AfterTargets="Build" DependsOnTargets="_UpdatePyconfig"> - <Copy SourceFiles="$(IntDir)pyconfig.h" DestinationFolder="$(OutDir)" /> - </Target> - <Target Name="_CleanPyconfig" AfterTargets="Clean"> - <Delete Files="$(IntDir)pyconfig.h;$(OutDir)pyconfig.h" /> - </Target> - <Target Name="_GetBuildInfo" BeforeTargets="PrepareForBuild"> <PropertyGroup> <GIT Condition="$(GIT) == ''">git</GIT> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0e6d42cc959..17999690990 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -120,9 +120,6 @@ <ClInclude Include="..\Include\listobject.h"> <Filter>Include</Filter> </ClInclude> - <ClInclude Include="..\Include\lock.h"> - <Filter>Include</Filter> - </ClInclude> <ClInclude Include="..\Include\longobject.h"> <Filter>Include</Filter> </ClInclude> @@ -414,7 +411,7 @@ <ClInclude Include="..\Include\cpython\listobject.h"> <Filter>Include\cpython</Filter> </ClInclude> - <ClInclude Include="..\Include\cpython\lock.h"> + <ClInclude Include="..\Include\cpython\pylock.h"> <Filter>Include</Filter> </ClInclude> <ClInclude Include="..\Include\cpython\longintrepr.h"> @@ -489,9 +486,15 @@ <ClInclude Include="..\Include\cpython\pylifecycle.h"> <Filter>Include\cpython</Filter> </ClInclude> + <ClInclude Include="..\Include\cpython\structseq.h"> + <Filter>Include\cpython</Filter> + </ClInclude> <ClInclude Include="..\Include\cpython\tupleobject.h"> <Filter>Include\cpython</Filter> </ClInclude> + <ClInclude Include="..\Include\cpython\sliceobject.h"> + <Filter>Include\cpython</Filter> + </ClInclude> <ClInclude Include="..\Include\cpython\traceback.h"> <Filter>Include\cpython</Filter> </ClInclude> @@ -525,6 +528,9 @@ <ClInclude Include="..\Include\cpython\initconfig.h"> <Filter>Include\cpython</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_unicodectype.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_unicodeobject.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -882,6 +888,15 @@ <ClInclude Include="..\Include\internal\pycore_uniqueid.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_uop.h"> + <Filter>Include\internal</Filter> + </ClInclude> + <ClInclude Include="..\Include\internal\pycore_uop_ids.h"> + <Filter>Include\internal</Filter> + </ClInclude> + <ClInclude Include="..\Include\internal\pycore_uop_metadata.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\mimalloc\mimalloc.h"> <Filter>Include\internal\mimalloc</Filter> </ClInclude> @@ -1055,6 +1070,9 @@ <ClCompile Include="..\Modules\main.c"> <Filter>Modules</Filter> </ClCompile> + <ClCompile Include="..\Modules\mathintegermodule.c"> + <Filter>Modules</Filter> + </ClCompile> <ClCompile Include="..\Modules\mathmodule.c"> <Filter>Modules</Filter> </ClCompile> @@ -1259,9 +1277,18 @@ <ClCompile Include="..\Objects\typeobject.c"> <Filter>Objects</Filter> </ClCompile> + <ClCompile Include="..\Objects\unicode_format.c"> + <Filter>Objects</Filter> + </ClCompile> <ClCompile Include="..\Objects\unicodectype.c"> <Filter>Objects</Filter> </ClCompile> + <ClCompile Include="..\Objects\unicode_formatter.c"> + <Filter>Objects</Filter> + </ClCompile> + <ClCompile Include="..\Objects\unicode_writer.c"> + <Filter>Objects</Filter> + </ClCompile> <ClCompile Include="..\Objects\unicodeobject.c"> <Filter>Objects</Filter> </ClCompile> @@ -1370,9 +1397,6 @@ <ClCompile Include="..\Python\flowgraph.c"> <Filter>Python</Filter> </ClCompile> - <ClCompile Include="..\Python\formatter_unicode.c"> - <Filter>Python</Filter> - </ClCompile> <ClCompile Include="..\Python\frozen.c"> <Filter>Python</Filter> </ClCompile> diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 01e19aabdec..27c0d382281 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -173,24 +173,27 @@ library which are implemented in C; each one builds a DLL (renamed to * _asyncio * _ctypes * _ctypes_test - * _zoneinfo * _decimal * _elementtree * _hashlib * _multiprocessing * _overlapped + * _queue + * _remote_debugging * _socket * _testbuffer * _testcapi - * _testlimitedcapi - * _testinternalcapi * _testclinic * _testclinic_limited * _testconsole * _testimportmultiple + * _testinternalcapi + * _testlimitedcapi * _testmultiphase * _testsinglephase - * _tkinter + * _uuid + * _wmi + * _zoneinfo * pyexpat * select * unicodedata @@ -202,18 +205,22 @@ interpreter, but they do implement several major features. See the "Getting External Sources" section below for additional information about getting the source for building these libraries. The sub-projects are: + _bz2 Python wrapper for version 1.0.8 of the libbzip2 compression library Homepage: http://www.bzip.org/ + _lzma - Python wrapper for version 5.2.2 of the liblzma compression library + Python wrapper for version 5.2.2 of the liblzma compression library, + which is itself built by liblzma.vcxproj. Homepage: https://tukaani.org/xz/ + _ssl Python wrapper for version 3.0.15 of the OpenSSL secure sockets - library, which is downloaded from our binaries repository at - https://github.com/python/cpython-bin-deps. + library, which is itself downloaded from our binaries repository at + https://github.com/python/cpython-bin-deps and built by openssl.vcxproj. Homepage: https://www.openssl.org/ @@ -230,9 +237,10 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.49.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.50.4, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ + _tkinter Wraps version 8.6.15 of the Tk windowing system, which is downloaded from our binaries repository at @@ -245,13 +253,20 @@ _tkinter PCbuild\prepare_tcltk.bat. This will retrieve the version of the sources matched to the current commit from the Tcl and Tk branches in our source repository at - https://github.com/python/cpython-source-deps. + https://github.com/python/cpython-source-deps and build them via the + tcl.vcxproj and tk.vcxproj sub-projects. The two projects install their respective components in a directory alongside the source directories called "tcltk" on Win32 and "tcltk64" on x64. They also copy the Tcl and Tk DLLs into the current output directory, which should ensure that Tkinter is able to load Tcl/Tk without having to change your PATH. + +_zstd + Python wrapper for version 1.5.7 of the zstd compression library + Homepage: + https://facebook.github.io/zstd/ + zlib-ng Compiles zlib-ng as a static library, which is later included by pythoncore.vcxproj. This was generated using CMake against zlib-ng diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 3ad17737807..742597f5cb5 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -29,12 +29,12 @@ <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" /> <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: --> - <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(PySourcePath)PC\pyconfig.h;$(PySourcePath)Tools\jit\**"/> <!-- Need to explicitly enumerate these, since globbing doesn't work for missing outputs: --> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils.h"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> @@ -116,7 +116,7 @@ <Target Name="_RegenJIT" Condition="'$(UseJIT)' == 'true'" - DependsOnTargets="_UpdatePyconfig;FindPythonForBuild" + DependsOnTargets="FindPythonForBuild" Inputs="@(_JITSources)" Outputs="@(_JITOutputs)"> <PropertyGroup> @@ -125,8 +125,7 @@ <JITArgs Condition="$(Platform) == 'x64'">x86_64-pc-windows-msvc</JITArgs> <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs> </PropertyGroup> - <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs)' - WorkingDirectory="$(GeneratedPyConfigDir)"/> + <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs) --output-dir "$(GeneratedJitStencilsDir)" --pyconfig-dir "$(PySourcePath)PC"'/> </Target> <Target Name="_CleanJIT" AfterTargets="Clean"> <Delete Files="@(_JITOutputs)"/> diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index c436215780f..f1e06073934 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -42,7 +42,7 @@ if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts if "%~1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%~1"=="-d" (set suffix=_d) & shift & goto CheckOpts rem HACK: Need some way to infer the version number in this script -if "%~1"=="--disable-gil" (set pyname=python3.14t) & shift & goto CheckOpts +if "%~1"=="--disable-gil" (set pyname=python3.15t) & shift & goto CheckOpts if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 3bcc0870882..b7a5b9d5e30 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -965,7 +965,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv) if (conv_token->lineno != conv->lineno || conv_token->end_col_offset != conv->col_offset) { return RAISE_SYNTAX_ERROR_KNOWN_RANGE( conv_token, conv, - "%c-string: conversion type must come right after the exclamanation mark", + "%c-string: conversion type must come right after the exclamation mark", TOK_GET_STRING_PREFIX(p->tok) ); } @@ -1404,7 +1404,15 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) { return NULL; } - PyObject* str = _PyPegen_decode_string(p, 0, bstr, bsize, tok); + + // Check if we're inside a raw f-string for format spec decoding + int is_raw = 0; + if (INSIDE_FSTRING(p->tok)) { + tokenizer_mode *mode = TOK_GET_MODE(p->tok); + is_raw = mode->raw; + } + + PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok); if (str == NULL) { return NULL; } @@ -1604,19 +1612,46 @@ _build_concatenated_bytes(Parser *p, asdl_expr_seq *strings, int lineno, Py_ssize_t len = asdl_seq_LEN(strings); assert(len > 0); - PyObject* res = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); - /* Bytes literals never get a kind, but just for consistency since they are represented as Constant nodes, we'll mirror the same behavior as unicode strings for determining the kind. */ - PyObject* kind = asdl_seq_GET(strings, 0)->v.Constant.kind; + PyObject *kind = asdl_seq_GET(strings, 0)->v.Constant.kind; + + Py_ssize_t total = 0; for (Py_ssize_t i = 0; i < len; i++) { expr_ty elem = asdl_seq_GET(strings, i); - PyBytes_Concat(&res, elem->v.Constant.value); + PyObject *bytes = elem->v.Constant.value; + Py_ssize_t part = PyBytes_GET_SIZE(bytes); + if (part > PY_SSIZE_T_MAX - total) { + PyErr_NoMemory(); + return NULL; + } + total += part; } - if (!res || _PyArena_AddPyObject(arena, res) < 0) { - Py_XDECREF(res); + + PyBytesWriter *writer = PyBytesWriter_Create(total); + if (writer == NULL) { + return NULL; + } + char *out = PyBytesWriter_GetData(writer); + + for (Py_ssize_t i = 0; i < len; i++) { + expr_ty elem = asdl_seq_GET(strings, i); + PyObject *bytes = elem->v.Constant.value; + Py_ssize_t part = PyBytes_GET_SIZE(bytes); + if (part > 0) { + memcpy(out, PyBytes_AS_STRING(bytes), part); + out += part; + } + } + + PyObject *res = PyBytesWriter_Finish(writer); + if (res == NULL) { + return NULL; + } + if (_PyArena_AddPyObject(arena, res) < 0) { + Py_DECREF(res); return NULL; } return _PyAST_Constant(res, kind, lineno, col_offset, end_lineno, end_col_offset, p->arena); @@ -1834,8 +1869,8 @@ _build_concatenated_joined_str(Parser *p, asdl_expr_seq *strings, return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena); } -static expr_ty -_build_concatenated_template_str(Parser *p, asdl_expr_seq *strings, +expr_ty +_PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *strings, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { @@ -1853,7 +1888,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, Py_ssize_t len = asdl_seq_LEN(strings); assert(len > 0); - int t_string_found = 0; int f_string_found = 0; int unicode_string_found = 0; int bytes_found = 0; @@ -1873,7 +1907,8 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, f_string_found = 1; break; case TemplateStr_kind: - t_string_found = 1; + // python.gram handles this; we should never get here + assert(0); break; default: f_string_found = 1; @@ -1882,13 +1917,13 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } // Cannot mix unicode and bytes - if ((unicode_string_found || f_string_found || t_string_found) && bytes_found) { + if ((unicode_string_found || f_string_found) && bytes_found) { RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals"); return NULL; } // If it's only bytes or only unicode string, do a simple concat - if (!f_string_found && !t_string_found) { + if (!f_string_found) { if (len == 1) { return asdl_seq_GET(strings, 0); } @@ -1902,11 +1937,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } } - if (t_string_found) { - return _build_concatenated_template_str(p, strings, lineno, - col_offset, end_lineno, end_col_offset, arena); - } - return _build_concatenated_joined_str(p, strings, lineno, col_offset, end_lineno, end_col_offset, arena); } @@ -1936,6 +1966,9 @@ _PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts) { return stmts; } stmt_ty last_stmt = asdl_seq_GET(stmts, len - 1); + if (p->last_stmt_location.lineno > last_stmt->lineno) { + return stmts; + } p->last_stmt_location.lineno = last_stmt->lineno; p->last_stmt_location.col_offset = last_stmt->col_offset; p->last_stmt_location.end_lineno = last_stmt->end_lineno; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 09e014534fb..3e252cbc488 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1009,7 +1009,7 @@ def visitModule(self, mod): else { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "Field '%U' is missing from %.400s._field_types. " + "Field %R is missing from %.400s._field_types. " "This will become an error in Python 3.15.", name, Py_TYPE(self)->tp_name ) < 0) { @@ -1044,7 +1044,7 @@ def visitModule(self, mod): // simple field (e.g., identifier) if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "%.400s.__init__ missing 1 required positional argument: '%U'. " + "%.400s.__init__ missing 1 required positional argument: %R. " "This will become an error in Python 3.15.", Py_TYPE(self)->tp_name, name ) < 0) { @@ -1244,6 +1244,32 @@ def visitModule(self, mod): Py_DECREF(unused); } } + + // Discard fields from 'expecting' that default to None + PyObject *field_types = NULL; + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), + &_Py_ID(_field_types), + &field_types) < 0) + { + Py_DECREF(expecting); + return -1; + } + if (field_types != NULL) { + Py_ssize_t pos = 0; + PyObject *field_name, *field_type; + while (PyDict_Next(field_types, &pos, &field_name, &field_type)) { + if (_PyUnion_Check(field_type)) { + // optional field + if (PySet_Discard(expecting, field_name) < 0) { + Py_DECREF(expecting); + Py_DECREF(field_types); + return -1; + } + } + } + Py_DECREF(field_types); + } + // Now 'expecting' contains the fields or attributes // that would not be filled inside ast_type_replace(). Py_ssize_t m = PySet_GET_SIZE(expecting); @@ -1486,7 +1512,7 @@ def visitModule(self, mod): for (Py_ssize_t i = 0; i < Py_MIN(length, 2); i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -1510,7 +1536,7 @@ def visitModule(self, mod): } if (i == 0 && length > 2) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ...", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ...", 5) < 0) { goto error; } } @@ -1614,7 +1640,7 @@ def visitModule(self, mod): } if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { Py_DECREF(name); Py_DECREF(value_repr); goto error; diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 4d10bccf0a5..7f25afec302 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -121,38 +121,88 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { } PyObject *res = NULL; - // Check if there is a # character in the expression + // Look for a # character outside of string literals int hash_detected = 0; + int in_string = 0; + char quote_char = 0; + for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { - if (tok_mode->last_expr_buffer[i] == '#') { + char ch = tok_mode->last_expr_buffer[i]; + + // Skip escaped characters + if (ch == '\\') { + i++; + continue; + } + + // Handle quotes + if (ch == '"' || ch == '\'') { + // The following if/else block works becase there is an off number + // of quotes in STRING tokens and the lexer only ever reaches this + // function with valid STRING tokens. + // For example: """hello""" + // First quote: in_string = 1 + // Second quote: in_string = 0 + // Third quote: in_string = 1 + if (!in_string) { + in_string = 1; + quote_char = ch; + } + else if (ch == quote_char) { + in_string = 0; + } + continue; + } + + // Check for # outside strings + if (ch == '#' && !in_string) { hash_detected = 1; break; } } - + // If we found a # character in the expression, we need to handle comments if (hash_detected) { - Py_ssize_t input_length = tok_mode->last_expr_size - tok_mode->last_expr_end; - char *result = (char *)PyMem_Malloc((input_length + 1) * sizeof(char)); + // Allocate buffer for processed result + char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); if (!result) { return -1; } - Py_ssize_t i = 0; - Py_ssize_t j = 0; + Py_ssize_t i = 0; // Input position + Py_ssize_t j = 0; // Output position + in_string = 0; // Whether we're in a string + quote_char = 0; // Current string quote char - for (i = 0, j = 0; i < input_length; i++) { - if (tok_mode->last_expr_buffer[i] == '#') { - // Skip characters until newline or end of string - while (i < input_length && tok_mode->last_expr_buffer[i] != '\0') { - if (tok_mode->last_expr_buffer[i] == '\n') { - result[j++] = tok_mode->last_expr_buffer[i]; - break; - } + // Process each character + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + char ch = tok_mode->last_expr_buffer[i]; + + // Handle string quotes + if (ch == '"' || ch == '\'') { + // See comment above to understand this part + if (!in_string) { + in_string = 1; + quote_char = ch; + } else if (ch == quote_char) { + in_string = 0; + } + result[j++] = ch; + } + // Skip comments + else if (ch == '#' && !in_string) { + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && + tok_mode->last_expr_buffer[i] != '\n') { i++; } - } else { - result[j++] = tok_mode->last_expr_buffer[i]; + if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + result[j++] = '\n'; + } } + // Copy other chars + else { + result[j++] = ch; + } + i++; } result[j] = '\0'; // Null-terminate the result string @@ -164,11 +214,9 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { tok_mode->last_expr_size - tok_mode->last_expr_end, NULL ); - } - - if (!res) { + if (!res) { return -1; } token->metadata = res; @@ -491,6 +539,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } } + else if (c == EOF && PyErr_Occurred()) { + return MAKE_TOKEN(ERRORTOKEN); + } else { break; } @@ -1328,7 +1379,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c)); } - if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) { + if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) { current_tok->in_debug = 1; } @@ -1421,7 +1472,8 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct return MAKE_TOKEN( _PyTokenizer_syntaxerror( tok, - "f-string: newlines are not allowed in format specifiers for single quoted f-strings" + "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings", + TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok) ) ); } diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2de9004fe08..3663dc3eb7f 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -43,6 +43,7 @@ _PyTokenizer_tok_new(void) tok->encoding = NULL; tok->cont_line = 0; tok->filename = NULL; + tok->module = NULL; tok->decoding_readline = NULL; tok->decoding_buffer = NULL; tok->readline = NULL; @@ -91,6 +92,7 @@ _PyTokenizer_Free(struct tok_state *tok) Py_XDECREF(tok->decoding_buffer); Py_XDECREF(tok->readline); Py_XDECREF(tok->filename); + Py_XDECREF(tok->module); if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) { PyMem_Free(tok->buf); } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 5e8cac7249b..9cd196a114c 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -9,6 +9,8 @@ #define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0) #define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0) +#define INSIDE_FSTRING_EXPR_AT_TOP(tok) \ + (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1) enum decoding_state { STATE_INIT, @@ -100,6 +102,7 @@ struct tok_state { int parenlinenostack[MAXLEVEL]; int parencolstack[MAXLEVEL]; PyObject *filename; + PyObject *module; /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ diff --git a/Parser/parser.c b/Parser/parser.c index 509fac7df6e..648b3702d8f 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -1,5 +1,6 @@ // @generated by pegen from python.gram #include "pegen.h" +#include "pycore_ceval.h" #if defined(Py_DEBUG) && defined(Py_BUILD_CORE) # define D(x) if (p->debug) { x; } @@ -14,66 +15,66 @@ # define MAXSTACK 4000 # endif #else -# define MAXSTACK 4000 +# define MAXSTACK 6000 #endif static const int n_keyword_lists = 9; static KeywordToken *reserved_keywords[] = { (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) { - {"if", 682}, - {"as", 680}, - {"in", 695}, - {"or", 588}, - {"is", 596}, + {"if", 691}, + {"as", 689}, + {"in", 704}, + {"or", 589}, + {"is", 597}, {NULL, -1}, }, (KeywordToken[]) { - {"del", 625}, - {"def", 699}, - {"for", 694}, - {"try", 656}, - {"and", 589}, - {"not", 703}, + {"del", 630}, + {"def", 708}, + {"for", 703}, + {"try", 665}, + {"and", 590}, + {"not", 712}, {NULL, -1}, }, (KeywordToken[]) { - {"from", 633}, - {"pass", 526}, - {"with", 647}, - {"elif", 687}, - {"else", 686}, - {"None", 623}, - {"True", 622}, + {"from", 642}, + {"pass", 527}, + {"with", 656}, + {"elif", 696}, + {"else", 695}, + {"None", 624}, + {"True", 623}, {NULL, -1}, }, (KeywordToken[]) { - {"raise", 525}, - {"yield", 587}, - {"break", 527}, - {"async", 698}, - {"class", 701}, - {"while", 689}, - {"False", 624}, - {"await", 597}, + {"raise", 628}, + {"yield", 588}, + {"break", 528}, + {"async", 707}, + {"class", 710}, + {"while", 698}, + {"False", 625}, + {"await", 598}, {NULL, -1}, }, (KeywordToken[]) { {"return", 522}, - {"import", 634}, - {"assert", 532}, - {"global", 529}, - {"except", 677}, - {"lambda", 621}, + {"import", 643}, + {"assert", 634}, + {"global", 530}, + {"except", 686}, + {"lambda", 622}, {NULL, -1}, }, (KeywordToken[]) { - {"finally", 673}, + {"finally", 682}, {NULL, -1}, }, (KeywordToken[]) { - {"continue", 528}, - {"nonlocal", 530}, + {"continue", 529}, + {"nonlocal", 531}, {NULL, -1}, }, }; @@ -298,235 +299,240 @@ static char *soft_keywords[] = { #define invalid_named_expression_type 1211 #define invalid_assignment_type 1212 #define invalid_ann_assign_target_type 1213 -#define invalid_del_stmt_type 1214 -#define invalid_block_type 1215 -#define invalid_comprehension_type 1216 -#define invalid_dict_comprehension_type 1217 -#define invalid_parameters_type 1218 -#define invalid_default_type 1219 -#define invalid_star_etc_type 1220 -#define invalid_kwds_type 1221 -#define invalid_parameters_helper_type 1222 -#define invalid_lambda_parameters_type 1223 -#define invalid_lambda_parameters_helper_type 1224 -#define invalid_lambda_star_etc_type 1225 -#define invalid_lambda_kwds_type 1226 -#define invalid_double_type_comments_type 1227 -#define invalid_with_item_type 1228 -#define invalid_for_if_clause_type 1229 -#define invalid_for_target_type 1230 -#define invalid_group_type 1231 -#define invalid_import_type 1232 -#define invalid_dotted_as_name_type 1233 -#define invalid_import_from_as_name_type 1234 -#define invalid_import_from_targets_type 1235 -#define invalid_with_stmt_type 1236 -#define invalid_with_stmt_indent_type 1237 -#define invalid_try_stmt_type 1238 -#define invalid_except_stmt_type 1239 -#define invalid_except_star_stmt_type 1240 -#define invalid_finally_stmt_type 1241 -#define invalid_except_stmt_indent_type 1242 -#define invalid_except_star_stmt_indent_type 1243 -#define invalid_match_stmt_type 1244 -#define invalid_case_block_type 1245 -#define invalid_as_pattern_type 1246 -#define invalid_class_pattern_type 1247 -#define invalid_class_argument_pattern_type 1248 -#define invalid_if_stmt_type 1249 -#define invalid_elif_stmt_type 1250 -#define invalid_else_stmt_type 1251 -#define invalid_while_stmt_type 1252 -#define invalid_for_stmt_type 1253 -#define invalid_def_raw_type 1254 -#define invalid_class_def_raw_type 1255 -#define invalid_double_starred_kvpairs_type 1256 -#define invalid_kvpair_type 1257 -#define invalid_starred_expression_unpacking_type 1258 -#define invalid_starred_expression_type 1259 -#define invalid_fstring_replacement_field_type 1260 -#define invalid_fstring_conversion_character_type 1261 -#define invalid_tstring_replacement_field_type 1262 -#define invalid_tstring_conversion_character_type 1263 -#define invalid_arithmetic_type 1264 -#define invalid_factor_type 1265 -#define invalid_type_params_type 1266 -#define _loop0_1_type 1267 -#define _loop1_2_type 1268 -#define _loop0_3_type 1269 -#define _gather_4_type 1270 -#define _tmp_5_type 1271 -#define _tmp_6_type 1272 -#define _tmp_7_type 1273 -#define _tmp_8_type 1274 -#define _tmp_9_type 1275 -#define _tmp_10_type 1276 -#define _tmp_11_type 1277 -#define _loop1_12_type 1278 -#define _tmp_13_type 1279 -#define _loop0_14_type 1280 -#define _gather_15_type 1281 -#define _tmp_16_type 1282 -#define _tmp_17_type 1283 -#define _loop0_18_type 1284 -#define _loop1_19_type 1285 -#define _loop0_20_type 1286 -#define _gather_21_type 1287 -#define _tmp_22_type 1288 -#define _loop0_23_type 1289 -#define _gather_24_type 1290 -#define _loop1_25_type 1291 -#define _tmp_26_type 1292 -#define _tmp_27_type 1293 -#define _loop0_28_type 1294 -#define _loop0_29_type 1295 -#define _loop1_30_type 1296 -#define _loop1_31_type 1297 -#define _loop0_32_type 1298 -#define _loop1_33_type 1299 -#define _loop0_34_type 1300 -#define _gather_35_type 1301 -#define _tmp_36_type 1302 -#define _loop1_37_type 1303 -#define _loop1_38_type 1304 -#define _loop1_39_type 1305 -#define _loop0_40_type 1306 -#define _gather_41_type 1307 -#define _tmp_42_type 1308 -#define _tmp_43_type 1309 -#define _tmp_44_type 1310 -#define _loop0_45_type 1311 -#define _gather_46_type 1312 -#define _loop0_47_type 1313 -#define _gather_48_type 1314 -#define _tmp_49_type 1315 -#define _loop0_50_type 1316 -#define _gather_51_type 1317 -#define _loop0_52_type 1318 -#define _gather_53_type 1319 -#define _loop0_54_type 1320 -#define _gather_55_type 1321 -#define _loop1_56_type 1322 -#define _loop1_57_type 1323 -#define _loop0_58_type 1324 -#define _gather_59_type 1325 -#define _loop1_60_type 1326 -#define _loop1_61_type 1327 -#define _loop1_62_type 1328 -#define _tmp_63_type 1329 -#define _loop0_64_type 1330 -#define _gather_65_type 1331 -#define _tmp_66_type 1332 -#define _tmp_67_type 1333 -#define _tmp_68_type 1334 -#define _tmp_69_type 1335 -#define _tmp_70_type 1336 -#define _loop0_71_type 1337 -#define _loop0_72_type 1338 -#define _loop1_73_type 1339 -#define _loop1_74_type 1340 -#define _loop0_75_type 1341 -#define _loop1_76_type 1342 -#define _loop0_77_type 1343 -#define _loop0_78_type 1344 -#define _loop0_79_type 1345 -#define _loop0_80_type 1346 -#define _loop1_81_type 1347 -#define _tmp_82_type 1348 -#define _loop0_83_type 1349 -#define _gather_84_type 1350 -#define _loop1_85_type 1351 -#define _loop0_86_type 1352 -#define _tmp_87_type 1353 -#define _loop0_88_type 1354 -#define _gather_89_type 1355 -#define _tmp_90_type 1356 -#define _loop0_91_type 1357 -#define _gather_92_type 1358 -#define _loop0_93_type 1359 -#define _gather_94_type 1360 -#define _loop0_95_type 1361 -#define _loop0_96_type 1362 -#define _gather_97_type 1363 -#define _loop1_98_type 1364 -#define _tmp_99_type 1365 -#define _loop0_100_type 1366 -#define _gather_101_type 1367 -#define _loop0_102_type 1368 -#define _gather_103_type 1369 -#define _tmp_104_type 1370 -#define _tmp_105_type 1371 -#define _loop0_106_type 1372 -#define _gather_107_type 1373 -#define _tmp_108_type 1374 -#define _tmp_109_type 1375 -#define _tmp_110_type 1376 -#define _tmp_111_type 1377 -#define _tmp_112_type 1378 -#define _loop1_113_type 1379 -#define _tmp_114_type 1380 -#define _tmp_115_type 1381 -#define _tmp_116_type 1382 -#define _tmp_117_type 1383 -#define _tmp_118_type 1384 -#define _loop0_119_type 1385 -#define _loop0_120_type 1386 -#define _tmp_121_type 1387 -#define _tmp_122_type 1388 -#define _tmp_123_type 1389 -#define _tmp_124_type 1390 -#define _tmp_125_type 1391 -#define _tmp_126_type 1392 -#define _tmp_127_type 1393 -#define _tmp_128_type 1394 -#define _tmp_129_type 1395 -#define _loop0_130_type 1396 -#define _gather_131_type 1397 -#define _tmp_132_type 1398 -#define _tmp_133_type 1399 -#define _tmp_134_type 1400 -#define _tmp_135_type 1401 -#define _loop0_136_type 1402 -#define _gather_137_type 1403 -#define _tmp_138_type 1404 -#define _loop0_139_type 1405 -#define _gather_140_type 1406 -#define _loop0_141_type 1407 -#define _gather_142_type 1408 -#define _tmp_143_type 1409 -#define _loop0_144_type 1410 -#define _tmp_145_type 1411 -#define _tmp_146_type 1412 -#define _tmp_147_type 1413 -#define _tmp_148_type 1414 -#define _tmp_149_type 1415 -#define _tmp_150_type 1416 -#define _tmp_151_type 1417 -#define _tmp_152_type 1418 -#define _tmp_153_type 1419 -#define _tmp_154_type 1420 -#define _tmp_155_type 1421 -#define _tmp_156_type 1422 -#define _tmp_157_type 1423 -#define _tmp_158_type 1424 -#define _tmp_159_type 1425 -#define _tmp_160_type 1426 -#define _tmp_161_type 1427 -#define _tmp_162_type 1428 -#define _tmp_163_type 1429 -#define _tmp_164_type 1430 -#define _tmp_165_type 1431 -#define _tmp_166_type 1432 -#define _tmp_167_type 1433 -#define _tmp_168_type 1434 -#define _tmp_169_type 1435 -#define _tmp_170_type 1436 -#define _loop0_171_type 1437 -#define _tmp_172_type 1438 -#define _tmp_173_type 1439 -#define _tmp_174_type 1440 -#define _tmp_175_type 1441 -#define _tmp_176_type 1442 +#define invalid_raise_stmt_type 1214 +#define invalid_del_stmt_type 1215 +#define invalid_assert_stmt_type 1216 +#define invalid_block_type 1217 +#define invalid_comprehension_type 1218 +#define invalid_dict_comprehension_type 1219 +#define invalid_parameters_type 1220 +#define invalid_default_type 1221 +#define invalid_star_etc_type 1222 +#define invalid_kwds_type 1223 +#define invalid_parameters_helper_type 1224 +#define invalid_lambda_parameters_type 1225 +#define invalid_lambda_parameters_helper_type 1226 +#define invalid_lambda_star_etc_type 1227 +#define invalid_lambda_kwds_type 1228 +#define invalid_double_type_comments_type 1229 +#define invalid_with_item_type 1230 +#define invalid_for_if_clause_type 1231 +#define invalid_for_target_type 1232 +#define invalid_group_type 1233 +#define invalid_import_type 1234 +#define invalid_dotted_as_name_type 1235 +#define invalid_import_from_as_name_type 1236 +#define invalid_import_from_targets_type 1237 +#define invalid_with_stmt_type 1238 +#define invalid_with_stmt_indent_type 1239 +#define invalid_try_stmt_type 1240 +#define invalid_except_stmt_type 1241 +#define invalid_except_star_stmt_type 1242 +#define invalid_finally_stmt_type 1243 +#define invalid_except_stmt_indent_type 1244 +#define invalid_except_star_stmt_indent_type 1245 +#define invalid_match_stmt_type 1246 +#define invalid_case_block_type 1247 +#define invalid_as_pattern_type 1248 +#define invalid_class_pattern_type 1249 +#define invalid_mapping_pattern_type 1250 +#define invalid_class_argument_pattern_type 1251 +#define invalid_if_stmt_type 1252 +#define invalid_elif_stmt_type 1253 +#define invalid_else_stmt_type 1254 +#define invalid_while_stmt_type 1255 +#define invalid_for_stmt_type 1256 +#define invalid_def_raw_type 1257 +#define invalid_class_def_raw_type 1258 +#define invalid_double_starred_kvpairs_type 1259 +#define invalid_kvpair_type 1260 +#define invalid_starred_expression_unpacking_type 1261 +#define invalid_starred_expression_type 1262 +#define invalid_fstring_replacement_field_type 1263 +#define invalid_fstring_conversion_character_type 1264 +#define invalid_tstring_replacement_field_type 1265 +#define invalid_tstring_conversion_character_type 1266 +#define invalid_string_tstring_concat_type 1267 +#define invalid_arithmetic_type 1268 +#define invalid_factor_type 1269 +#define invalid_type_params_type 1270 +#define _loop0_1_type 1271 +#define _loop1_2_type 1272 +#define _loop0_3_type 1273 +#define _gather_4_type 1274 +#define _tmp_5_type 1275 +#define _tmp_6_type 1276 +#define _tmp_7_type 1277 +#define _tmp_8_type 1278 +#define _tmp_9_type 1279 +#define _tmp_10_type 1280 +#define _tmp_11_type 1281 +#define _loop1_12_type 1282 +#define _loop0_13_type 1283 +#define _gather_14_type 1284 +#define _tmp_15_type 1285 +#define _tmp_16_type 1286 +#define _loop0_17_type 1287 +#define _loop1_18_type 1288 +#define _loop0_19_type 1289 +#define _gather_20_type 1290 +#define _tmp_21_type 1291 +#define _loop0_22_type 1292 +#define _gather_23_type 1293 +#define _loop1_24_type 1294 +#define _tmp_25_type 1295 +#define _tmp_26_type 1296 +#define _loop0_27_type 1297 +#define _loop0_28_type 1298 +#define _loop1_29_type 1299 +#define _loop1_30_type 1300 +#define _loop0_31_type 1301 +#define _loop1_32_type 1302 +#define _loop0_33_type 1303 +#define _gather_34_type 1304 +#define _tmp_35_type 1305 +#define _loop1_36_type 1306 +#define _loop1_37_type 1307 +#define _loop1_38_type 1308 +#define _loop0_39_type 1309 +#define _gather_40_type 1310 +#define _tmp_41_type 1311 +#define _tmp_42_type 1312 +#define _tmp_43_type 1313 +#define _loop0_44_type 1314 +#define _gather_45_type 1315 +#define _loop0_46_type 1316 +#define _gather_47_type 1317 +#define _tmp_48_type 1318 +#define _loop0_49_type 1319 +#define _gather_50_type 1320 +#define _loop0_51_type 1321 +#define _gather_52_type 1322 +#define _loop0_53_type 1323 +#define _gather_54_type 1324 +#define _loop1_55_type 1325 +#define _loop1_56_type 1326 +#define _loop0_57_type 1327 +#define _gather_58_type 1328 +#define _loop1_59_type 1329 +#define _loop1_60_type 1330 +#define _loop1_61_type 1331 +#define _tmp_62_type 1332 +#define _loop0_63_type 1333 +#define _gather_64_type 1334 +#define _tmp_65_type 1335 +#define _tmp_66_type 1336 +#define _tmp_67_type 1337 +#define _tmp_68_type 1338 +#define _tmp_69_type 1339 +#define _loop0_70_type 1340 +#define _loop0_71_type 1341 +#define _loop1_72_type 1342 +#define _loop1_73_type 1343 +#define _loop0_74_type 1344 +#define _loop1_75_type 1345 +#define _loop0_76_type 1346 +#define _loop0_77_type 1347 +#define _loop0_78_type 1348 +#define _loop0_79_type 1349 +#define _loop1_80_type 1350 +#define _loop1_81_type 1351 +#define _tmp_82_type 1352 +#define _loop0_83_type 1353 +#define _gather_84_type 1354 +#define _loop1_85_type 1355 +#define _loop0_86_type 1356 +#define _tmp_87_type 1357 +#define _loop0_88_type 1358 +#define _gather_89_type 1359 +#define _tmp_90_type 1360 +#define _loop0_91_type 1361 +#define _gather_92_type 1362 +#define _loop0_93_type 1363 +#define _gather_94_type 1364 +#define _loop0_95_type 1365 +#define _loop0_96_type 1366 +#define _gather_97_type 1367 +#define _loop1_98_type 1368 +#define _tmp_99_type 1369 +#define _loop0_100_type 1370 +#define _gather_101_type 1371 +#define _loop0_102_type 1372 +#define _gather_103_type 1373 +#define _tmp_104_type 1374 +#define _tmp_105_type 1375 +#define _loop0_106_type 1376 +#define _gather_107_type 1377 +#define _tmp_108_type 1378 +#define _tmp_109_type 1379 +#define _tmp_110_type 1380 +#define _tmp_111_type 1381 +#define _tmp_112_type 1382 +#define _loop1_113_type 1383 +#define _tmp_114_type 1384 +#define _tmp_115_type 1385 +#define _tmp_116_type 1386 +#define _tmp_117_type 1387 +#define _tmp_118_type 1388 +#define _loop0_119_type 1389 +#define _loop0_120_type 1390 +#define _tmp_121_type 1391 +#define _tmp_122_type 1392 +#define _tmp_123_type 1393 +#define _tmp_124_type 1394 +#define _tmp_125_type 1395 +#define _tmp_126_type 1396 +#define _tmp_127_type 1397 +#define _tmp_128_type 1398 +#define _tmp_129_type 1399 +#define _loop0_130_type 1400 +#define _gather_131_type 1401 +#define _tmp_132_type 1402 +#define _tmp_133_type 1403 +#define _tmp_134_type 1404 +#define _tmp_135_type 1405 +#define _loop0_136_type 1406 +#define _gather_137_type 1407 +#define _tmp_138_type 1408 +#define _loop0_139_type 1409 +#define _gather_140_type 1410 +#define _loop0_141_type 1411 +#define _gather_142_type 1412 +#define _tmp_143_type 1413 +#define _loop0_144_type 1414 +#define _tmp_145_type 1415 +#define _tmp_146_type 1416 +#define _tmp_147_type 1417 +#define _tmp_148_type 1418 +#define _tmp_149_type 1419 +#define _tmp_150_type 1420 +#define _tmp_151_type 1421 +#define _tmp_152_type 1422 +#define _tmp_153_type 1423 +#define _tmp_154_type 1424 +#define _tmp_155_type 1425 +#define _tmp_156_type 1426 +#define _tmp_157_type 1427 +#define _tmp_158_type 1428 +#define _tmp_159_type 1429 +#define _tmp_160_type 1430 +#define _tmp_161_type 1431 +#define _tmp_162_type 1432 +#define _tmp_163_type 1433 +#define _tmp_164_type 1434 +#define _tmp_165_type 1435 +#define _tmp_166_type 1436 +#define _tmp_167_type 1437 +#define _tmp_168_type 1438 +#define _tmp_169_type 1439 +#define _tmp_170_type 1440 +#define _tmp_171_type 1441 +#define _loop0_172_type 1442 +#define _tmp_173_type 1443 +#define _tmp_174_type 1444 +#define _tmp_175_type 1445 +#define _tmp_176_type 1446 +#define _tmp_177_type 1447 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -742,7 +748,9 @@ static void *invalid_expression_rule(Parser *p); static void *invalid_named_expression_rule(Parser *p); static void *invalid_assignment_rule(Parser *p); static expr_ty invalid_ann_assign_target_rule(Parser *p); +static void *invalid_raise_stmt_rule(Parser *p); static void *invalid_del_stmt_rule(Parser *p); +static void *invalid_assert_stmt_rule(Parser *p); static void *invalid_block_rule(Parser *p); static void *invalid_comprehension_rule(Parser *p); static void *invalid_dict_comprehension_rule(Parser *p); @@ -776,6 +784,7 @@ static void *invalid_match_stmt_rule(Parser *p); static void *invalid_case_block_rule(Parser *p); static void *invalid_as_pattern_rule(Parser *p); static void *invalid_class_pattern_rule(Parser *p); +static void *invalid_mapping_pattern_rule(Parser *p); static asdl_pattern_seq* invalid_class_argument_pattern_rule(Parser *p); static void *invalid_if_stmt_rule(Parser *p); static void *invalid_elif_stmt_rule(Parser *p); @@ -792,6 +801,7 @@ static void *invalid_fstring_replacement_field_rule(Parser *p); static void *invalid_fstring_conversion_character_rule(Parser *p); static void *invalid_tstring_replacement_field_rule(Parser *p); static void *invalid_tstring_conversion_character_rule(Parser *p); +static void *invalid_string_tstring_concat_rule(Parser *p); static void *invalid_arithmetic_rule(Parser *p); static void *invalid_factor_rule(Parser *p); static void *invalid_type_params_rule(Parser *p); @@ -807,74 +817,74 @@ static void *_tmp_9_rule(Parser *p); static void *_tmp_10_rule(Parser *p); static void *_tmp_11_rule(Parser *p); static asdl_seq *_loop1_12_rule(Parser *p); -static void *_tmp_13_rule(Parser *p); -static asdl_seq *_loop0_14_rule(Parser *p); -static asdl_seq *_gather_15_rule(Parser *p); +static asdl_seq *_loop0_13_rule(Parser *p); +static asdl_seq *_gather_14_rule(Parser *p); +static void *_tmp_15_rule(Parser *p); static void *_tmp_16_rule(Parser *p); -static void *_tmp_17_rule(Parser *p); -static asdl_seq *_loop0_18_rule(Parser *p); -static asdl_seq *_loop1_19_rule(Parser *p); -static asdl_seq *_loop0_20_rule(Parser *p); -static asdl_seq *_gather_21_rule(Parser *p); -static void *_tmp_22_rule(Parser *p); -static asdl_seq *_loop0_23_rule(Parser *p); -static asdl_seq *_gather_24_rule(Parser *p); -static asdl_seq *_loop1_25_rule(Parser *p); +static asdl_seq *_loop0_17_rule(Parser *p); +static asdl_seq *_loop1_18_rule(Parser *p); +static asdl_seq *_loop0_19_rule(Parser *p); +static asdl_seq *_gather_20_rule(Parser *p); +static void *_tmp_21_rule(Parser *p); +static asdl_seq *_loop0_22_rule(Parser *p); +static asdl_seq *_gather_23_rule(Parser *p); +static asdl_seq *_loop1_24_rule(Parser *p); +static void *_tmp_25_rule(Parser *p); static void *_tmp_26_rule(Parser *p); -static void *_tmp_27_rule(Parser *p); +static asdl_seq *_loop0_27_rule(Parser *p); static asdl_seq *_loop0_28_rule(Parser *p); -static asdl_seq *_loop0_29_rule(Parser *p); +static asdl_seq *_loop1_29_rule(Parser *p); static asdl_seq *_loop1_30_rule(Parser *p); -static asdl_seq *_loop1_31_rule(Parser *p); -static asdl_seq *_loop0_32_rule(Parser *p); -static asdl_seq *_loop1_33_rule(Parser *p); -static asdl_seq *_loop0_34_rule(Parser *p); -static asdl_seq *_gather_35_rule(Parser *p); -static void *_tmp_36_rule(Parser *p); +static asdl_seq *_loop0_31_rule(Parser *p); +static asdl_seq *_loop1_32_rule(Parser *p); +static asdl_seq *_loop0_33_rule(Parser *p); +static asdl_seq *_gather_34_rule(Parser *p); +static void *_tmp_35_rule(Parser *p); +static asdl_seq *_loop1_36_rule(Parser *p); static asdl_seq *_loop1_37_rule(Parser *p); static asdl_seq *_loop1_38_rule(Parser *p); -static asdl_seq *_loop1_39_rule(Parser *p); -static asdl_seq *_loop0_40_rule(Parser *p); -static asdl_seq *_gather_41_rule(Parser *p); +static asdl_seq *_loop0_39_rule(Parser *p); +static asdl_seq *_gather_40_rule(Parser *p); +static void *_tmp_41_rule(Parser *p); static void *_tmp_42_rule(Parser *p); static void *_tmp_43_rule(Parser *p); -static void *_tmp_44_rule(Parser *p); -static asdl_seq *_loop0_45_rule(Parser *p); -static asdl_seq *_gather_46_rule(Parser *p); -static asdl_seq *_loop0_47_rule(Parser *p); -static asdl_seq *_gather_48_rule(Parser *p); -static void *_tmp_49_rule(Parser *p); -static asdl_seq *_loop0_50_rule(Parser *p); -static asdl_seq *_gather_51_rule(Parser *p); -static asdl_seq *_loop0_52_rule(Parser *p); -static asdl_seq *_gather_53_rule(Parser *p); -static asdl_seq *_loop0_54_rule(Parser *p); -static asdl_seq *_gather_55_rule(Parser *p); +static asdl_seq *_loop0_44_rule(Parser *p); +static asdl_seq *_gather_45_rule(Parser *p); +static asdl_seq *_loop0_46_rule(Parser *p); +static asdl_seq *_gather_47_rule(Parser *p); +static void *_tmp_48_rule(Parser *p); +static asdl_seq *_loop0_49_rule(Parser *p); +static asdl_seq *_gather_50_rule(Parser *p); +static asdl_seq *_loop0_51_rule(Parser *p); +static asdl_seq *_gather_52_rule(Parser *p); +static asdl_seq *_loop0_53_rule(Parser *p); +static asdl_seq *_gather_54_rule(Parser *p); +static asdl_seq *_loop1_55_rule(Parser *p); static asdl_seq *_loop1_56_rule(Parser *p); -static asdl_seq *_loop1_57_rule(Parser *p); -static asdl_seq *_loop0_58_rule(Parser *p); -static asdl_seq *_gather_59_rule(Parser *p); +static asdl_seq *_loop0_57_rule(Parser *p); +static asdl_seq *_gather_58_rule(Parser *p); +static asdl_seq *_loop1_59_rule(Parser *p); static asdl_seq *_loop1_60_rule(Parser *p); static asdl_seq *_loop1_61_rule(Parser *p); -static asdl_seq *_loop1_62_rule(Parser *p); -static void *_tmp_63_rule(Parser *p); -static asdl_seq *_loop0_64_rule(Parser *p); -static asdl_seq *_gather_65_rule(Parser *p); +static void *_tmp_62_rule(Parser *p); +static asdl_seq *_loop0_63_rule(Parser *p); +static asdl_seq *_gather_64_rule(Parser *p); +static void *_tmp_65_rule(Parser *p); static void *_tmp_66_rule(Parser *p); static void *_tmp_67_rule(Parser *p); static void *_tmp_68_rule(Parser *p); static void *_tmp_69_rule(Parser *p); -static void *_tmp_70_rule(Parser *p); +static asdl_seq *_loop0_70_rule(Parser *p); static asdl_seq *_loop0_71_rule(Parser *p); -static asdl_seq *_loop0_72_rule(Parser *p); +static asdl_seq *_loop1_72_rule(Parser *p); static asdl_seq *_loop1_73_rule(Parser *p); -static asdl_seq *_loop1_74_rule(Parser *p); -static asdl_seq *_loop0_75_rule(Parser *p); -static asdl_seq *_loop1_76_rule(Parser *p); +static asdl_seq *_loop0_74_rule(Parser *p); +static asdl_seq *_loop1_75_rule(Parser *p); +static asdl_seq *_loop0_76_rule(Parser *p); static asdl_seq *_loop0_77_rule(Parser *p); static asdl_seq *_loop0_78_rule(Parser *p); static asdl_seq *_loop0_79_rule(Parser *p); -static asdl_seq *_loop0_80_rule(Parser *p); +static asdl_seq *_loop1_80_rule(Parser *p); static asdl_seq *_loop1_81_rule(Parser *p); static void *_tmp_82_rule(Parser *p); static asdl_seq *_loop0_83_rule(Parser *p); @@ -965,12 +975,13 @@ static void *_tmp_167_rule(Parser *p); static void *_tmp_168_rule(Parser *p); static void *_tmp_169_rule(Parser *p); static void *_tmp_170_rule(Parser *p); -static asdl_seq *_loop0_171_rule(Parser *p); -static void *_tmp_172_rule(Parser *p); +static void *_tmp_171_rule(Parser *p); +static asdl_seq *_loop0_172_rule(Parser *p); static void *_tmp_173_rule(Parser *p); static void *_tmp_174_rule(Parser *p); static void *_tmp_175_rule(Parser *p); static void *_tmp_176_rule(Parser *p); +static void *_tmp_177_rule(Parser *p); // file: statements? $ @@ -1197,7 +1208,7 @@ statements_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ statements[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "statement+")); - _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_seq_flatten ( p , a ) ); + _res = ( asdl_stmt_seq* ) _PyPegen_seq_flatten ( p , a ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -1240,7 +1251,7 @@ statement_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ statement[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt")); - _res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ); + _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -1677,7 +1688,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('import' | 'from') import_stmt")); stmt_ty import_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_5_rule, p) + _PyPegen_lookahead(1, _tmp_5_rule, p) && (import_stmt_var = import_stmt_rule(p)) // import_stmt ) @@ -1698,7 +1709,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'raise' raise_stmt")); stmt_ty raise_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 525) // token='raise' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 628) // token='raise' && (raise_stmt_var = raise_stmt_rule(p)) // raise_stmt ) @@ -1719,7 +1730,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'pass' pass_stmt")); stmt_ty pass_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 526) // token='pass' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 527) // token='pass' && (pass_stmt_var = pass_stmt_rule(p)) // pass_stmt ) @@ -1740,7 +1751,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'del' del_stmt")); stmt_ty del_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 625) // token='del' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 630) // token='del' && (del_stmt_var = del_stmt_rule(p)) // del_stmt ) @@ -1761,7 +1772,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'yield' yield_stmt")); stmt_ty yield_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 587) // token='yield' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 588) // token='yield' && (yield_stmt_var = yield_stmt_rule(p)) // yield_stmt ) @@ -1782,7 +1793,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'assert' assert_stmt")); stmt_ty assert_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 532) // token='assert' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 634) // token='assert' && (assert_stmt_var = assert_stmt_rule(p)) // assert_stmt ) @@ -1803,7 +1814,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'break' break_stmt")); stmt_ty break_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 527) // token='break' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 528) // token='break' && (break_stmt_var = break_stmt_rule(p)) // break_stmt ) @@ -1824,7 +1835,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'continue' continue_stmt")); stmt_ty continue_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 528) // token='continue' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 529) // token='continue' && (continue_stmt_var = continue_stmt_rule(p)) // continue_stmt ) @@ -1845,7 +1856,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'global' global_stmt")); stmt_ty global_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 529) // token='global' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 530) // token='global' && (global_stmt_var = global_stmt_rule(p)) // global_stmt ) @@ -1866,7 +1877,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'nonlocal' nonlocal_stmt")); stmt_ty nonlocal_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 530) // token='nonlocal' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 531) // token='nonlocal' && (nonlocal_stmt_var = nonlocal_stmt_rule(p)) // nonlocal_stmt ) @@ -1915,7 +1926,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('def' | '@' | 'async') function_def")); stmt_ty function_def_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_6_rule, p) + _PyPegen_lookahead(1, _tmp_6_rule, p) && (function_def_var = function_def_rule(p)) // function_def ) @@ -1936,7 +1947,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'if' if_stmt")); stmt_ty if_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 682) // token='if' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 691) // token='if' && (if_stmt_var = if_stmt_rule(p)) // if_stmt ) @@ -1957,7 +1968,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('class' | '@') class_def")); stmt_ty class_def_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_7_rule, p) + _PyPegen_lookahead(1, _tmp_7_rule, p) && (class_def_var = class_def_rule(p)) // class_def ) @@ -1978,7 +1989,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('with' | 'async') with_stmt")); stmt_ty with_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_8_rule, p) + _PyPegen_lookahead(1, _tmp_8_rule, p) && (with_stmt_var = with_stmt_rule(p)) // with_stmt ) @@ -1999,7 +2010,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('for' | 'async') for_stmt")); stmt_ty for_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_9_rule, p) + _PyPegen_lookahead(1, _tmp_9_rule, p) && (for_stmt_var = for_stmt_rule(p)) // for_stmt ) @@ -2020,7 +2031,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'try' try_stmt")); stmt_ty try_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 656) // token='try' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 665) // token='try' && (try_stmt_var = try_stmt_rule(p)) // try_stmt ) @@ -2041,7 +2052,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'while' while_stmt")); stmt_ty while_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 689) // token='while' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 698) // token='while' && (while_stmt_var = while_stmt_rule(p)) // while_stmt ) @@ -2767,7 +2778,11 @@ return_stmt_rule(Parser *p) return _res; } -// raise_stmt: 'raise' expression ['from' expression] | 'raise' +// raise_stmt: +// | 'raise' expression 'from' expression +// | invalid_raise_stmt +// | 'raise' expression +// | 'raise' static stmt_ty raise_stmt_rule(Parser *p) { @@ -2789,24 +2804,27 @@ raise_stmt_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // 'raise' expression ['from' expression] + { // 'raise' expression 'from' expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' expression ['from' expression]")); + D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from' expression")); Token * _keyword; + Token * _keyword_1; expr_ty a; - void *b; + expr_ty b; if ( - (_keyword = _PyPegen_expect_token(p, 525)) // token='raise' + (_keyword = _PyPegen_expect_token(p, 628)) // token='raise' && (a = expression_rule(p)) // expression && - (b = _tmp_13_rule(p), !p->error_indicator) // ['from' expression] + (_keyword_1 = _PyPegen_expect_token(p, 642)) // token='from' + && + (b = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression ['from' expression]")); + D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from' expression")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -2826,7 +2844,62 @@ raise_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' expression ['from' expression]")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' expression 'from' expression")); + } + if (p->call_invalid_rules) { // invalid_raise_stmt + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_raise_stmt")); + void *invalid_raise_stmt_var; + if ( + (invalid_raise_stmt_var = invalid_raise_stmt_rule(p)) // invalid_raise_stmt + ) + { + D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_raise_stmt")); + _res = invalid_raise_stmt_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_raise_stmt")); + } + { // 'raise' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' expression")); + Token * _keyword; + expr_ty a; + if ( + (_keyword = _PyPegen_expect_token(p, 628)) // token='raise' + && + (a = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_Raise ( a , NULL , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' expression")); } { // 'raise' if (p->error_indicator) { @@ -2836,7 +2909,7 @@ raise_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 525)) // token='raise' + (_keyword = _PyPegen_expect_token(p, 628)) // token='raise' ) { D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise'")); @@ -2897,7 +2970,7 @@ pass_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> pass_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'pass'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 526)) // token='pass' + (_keyword = _PyPegen_expect_token(p, 527)) // token='pass' ) { D(fprintf(stderr, "%*c+ pass_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'pass'")); @@ -2958,7 +3031,7 @@ break_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> break_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'break'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 527)) // token='break' + (_keyword = _PyPegen_expect_token(p, 528)) // token='break' ) { D(fprintf(stderr, "%*c+ break_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'break'")); @@ -3019,7 +3092,7 @@ continue_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> continue_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'continue'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 528)) // token='continue' + (_keyword = _PyPegen_expect_token(p, 529)) // token='continue' ) { D(fprintf(stderr, "%*c+ continue_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'continue'")); @@ -3081,9 +3154,9 @@ global_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 529)) // token='global' + (_keyword = _PyPegen_expect_token(p, 530)) // token='global' && - (a = (asdl_expr_seq*)_gather_15_rule(p)) // ','.NAME+ + (a = (asdl_expr_seq*)_gather_14_rule(p)) // ','.NAME+ ) { D(fprintf(stderr, "%*c+ global_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'global' ','.NAME+")); @@ -3145,9 +3218,9 @@ nonlocal_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 530)) // token='nonlocal' + (_keyword = _PyPegen_expect_token(p, 531)) // token='nonlocal' && - (a = (asdl_expr_seq*)_gather_15_rule(p)) // ','.NAME+ + (a = (asdl_expr_seq*)_gather_14_rule(p)) // ','.NAME+ ) { D(fprintf(stderr, "%*c+ nonlocal_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'nonlocal' ','.NAME+")); @@ -3209,11 +3282,11 @@ del_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 625)) // token='del' + (_keyword = _PyPegen_expect_token(p, 630)) // token='del' && (a = del_targets_rule(p)) // del_targets && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_16_rule, p) + _PyPegen_lookahead(1, _tmp_15_rule, p) ) { D(fprintf(stderr, "%*c+ del_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'del' del_targets &(';' | NEWLINE)")); @@ -3324,7 +3397,7 @@ yield_stmt_rule(Parser *p) return _res; } -// assert_stmt: 'assert' expression [',' expression] +// assert_stmt: invalid_assert_stmt | 'assert' expression [',' expression] static stmt_ty assert_stmt_rule(Parser *p) { @@ -3346,6 +3419,25 @@ assert_stmt_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro + if (p->call_invalid_rules) { // invalid_assert_stmt + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> assert_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_assert_stmt")); + void *invalid_assert_stmt_var; + if ( + (invalid_assert_stmt_var = invalid_assert_stmt_rule(p)) // invalid_assert_stmt + ) + { + D(fprintf(stderr, "%*c+ assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_assert_stmt")); + _res = invalid_assert_stmt_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s assert_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_assert_stmt")); + } { // 'assert' expression [',' expression] if (p->error_indicator) { p->level--; @@ -3356,11 +3448,11 @@ assert_stmt_rule(Parser *p) expr_ty a; void *b; if ( - (_keyword = _PyPegen_expect_token(p, 532)) // token='assert' + (_keyword = _PyPegen_expect_token(p, 634)) // token='assert' && (a = expression_rule(p)) // expression && - (b = _tmp_17_rule(p), !p->error_indicator) // [',' expression] + (b = _tmp_16_rule(p), !p->error_indicator) // [',' expression] ) { D(fprintf(stderr, "%*c+ assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression [',' expression]")); @@ -3498,7 +3590,7 @@ import_name_rule(Parser *p) Token * _keyword; asdl_alias_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword = _PyPegen_expect_token(p, 643)) // token='import' && (a = dotted_as_names_rule(p)) // dotted_as_names ) @@ -3567,13 +3659,13 @@ import_from_rule(Parser *p) expr_ty b; asdl_alias_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 642)) // token='from' && - (a = _loop0_18_rule(p)) // (('.' | '...'))* + (a = _loop0_17_rule(p)) // (('.' | '...'))* && (b = dotted_name_rule(p)) // dotted_name && - (_keyword_1 = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 643)) // token='import' && (c = import_from_targets_rule(p)) // import_from_targets ) @@ -3611,11 +3703,11 @@ import_from_rule(Parser *p) asdl_seq * a; asdl_alias_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 642)) // token='from' && - (a = _loop1_19_rule(p)) // (('.' | '...'))+ + (a = _loop1_18_rule(p)) // (('.' | '...'))+ && - (_keyword_1 = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 643)) // token='import' && (b = import_from_targets_rule(p)) // import_from_targets ) @@ -3808,7 +3900,7 @@ import_from_as_names_rule(Parser *p) D(fprintf(stderr, "%*c> import_from_as_names[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.import_from_as_name+")); asdl_alias_seq* a; if ( - (a = (asdl_alias_seq*)_gather_21_rule(p)) // ','.import_from_as_name+ + (a = (asdl_alias_seq*)_gather_20_rule(p)) // ','.import_from_as_name+ ) { D(fprintf(stderr, "%*c+ import_from_as_names[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.import_from_as_name+")); @@ -3882,7 +3974,7 @@ import_from_as_name_rule(Parser *p) if ( (a = _PyPegen_name_token(p)) // NAME && - (b = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (b = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ['as' NAME]")); @@ -3934,7 +4026,7 @@ dotted_as_names_rule(Parser *p) D(fprintf(stderr, "%*c> dotted_as_names[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.dotted_as_name+")); asdl_alias_seq* a; if ( - (a = (asdl_alias_seq*)_gather_24_rule(p)) // ','.dotted_as_name+ + (a = (asdl_alias_seq*)_gather_23_rule(p)) // ','.dotted_as_name+ ) { D(fprintf(stderr, "%*c+ dotted_as_names[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.dotted_as_name+")); @@ -4008,7 +4100,7 @@ dotted_as_name_rule(Parser *p) if ( (a = dotted_name_rule(p)) // dotted_name && - (b = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (b = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name ['as' NAME]")); @@ -4259,7 +4351,7 @@ decorators_rule(Parser *p) D(fprintf(stderr, "%*c> decorators[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(('@' named_expression NEWLINE))+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_loop1_25_rule(p)) // (('@' named_expression NEWLINE))+ + (a = (asdl_expr_seq*)_loop1_24_rule(p)) // (('@' named_expression NEWLINE))+ ) { D(fprintf(stderr, "%*c+ decorators[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(('@' named_expression NEWLINE))+")); @@ -4402,13 +4494,13 @@ class_def_raw_rule(Parser *p) asdl_stmt_seq* c; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 701)) // token='class' + (_keyword = _PyPegen_expect_token(p, 710)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (b = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (b = _tmp_25_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4569,7 +4661,7 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 699)) // token='def' + (_keyword = _PyPegen_expect_token(p, 708)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -4581,7 +4673,7 @@ function_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] + (a = _tmp_26_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4630,9 +4722,9 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 707)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='def' + (_keyword_1 = _PyPegen_expect_token(p, 708)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -4644,7 +4736,7 @@ function_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] + (a = _tmp_26_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4769,9 +4861,9 @@ parameters_rule(Parser *p) if ( (a = slash_no_default_rule(p)) // slash_no_default && - (b = (asdl_arg_seq*)_loop0_28_rule(p)) // param_no_default* + (b = (asdl_arg_seq*)_loop0_27_rule(p)) // param_no_default* && - (c = _loop0_29_rule(p)) // param_with_default* + (c = _loop0_28_rule(p)) // param_with_default* && (d = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4801,7 +4893,7 @@ parameters_rule(Parser *p) if ( (a = slash_with_default_rule(p)) // slash_with_default && - (b = _loop0_29_rule(p)) // param_with_default* + (b = _loop0_28_rule(p)) // param_with_default* && (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4829,9 +4921,9 @@ parameters_rule(Parser *p) asdl_seq * b; void *c; if ( - (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_29_rule(p)) // param_no_default+ && - (b = _loop0_29_rule(p)) // param_with_default* + (b = _loop0_28_rule(p)) // param_with_default* && (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4858,7 +4950,7 @@ parameters_rule(Parser *p) asdl_seq * a; void *b; if ( - (a = _loop1_31_rule(p)) // param_with_default+ + (a = _loop1_30_rule(p)) // param_with_default+ && (b = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4929,7 +5021,7 @@ slash_no_default_rule(Parser *p) Token * _literal_1; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_29_rule(p)) // param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -4958,7 +5050,7 @@ slash_no_default_rule(Parser *p) Token * _literal; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_29_rule(p)) // param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -5010,9 +5102,9 @@ slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_28_rule(p)) // param_no_default* + (a = _loop0_27_rule(p)) // param_no_default* && - (b = _loop1_31_rule(p)) // param_with_default+ + (b = _loop1_30_rule(p)) // param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -5042,9 +5134,9 @@ slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_28_rule(p)) // param_no_default* + (a = _loop0_27_rule(p)) // param_no_default* && - (b = _loop1_31_rule(p)) // param_with_default+ + (b = _loop1_30_rule(p)) // param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -5122,7 +5214,7 @@ star_etc_rule(Parser *p) && (a = param_no_default_rule(p)) // param_no_default && - (b = _loop0_32_rule(p)) // param_maybe_default* + (b = _loop0_31_rule(p)) // param_maybe_default* && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -5155,7 +5247,7 @@ star_etc_rule(Parser *p) && (a = param_no_default_star_annotation_rule(p)) // param_no_default_star_annotation && - (b = _loop0_32_rule(p)) // param_maybe_default* + (b = _loop0_31_rule(p)) // param_maybe_default* && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -5188,7 +5280,7 @@ star_etc_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _loop1_33_rule(p)) // param_maybe_default+ + (b = _loop1_32_rule(p)) // param_maybe_default+ && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -5970,7 +6062,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6015,7 +6107,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6110,7 +6202,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 687)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 696)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6155,7 +6247,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 687)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 696)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6236,7 +6328,7 @@ else_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword = _PyPegen_expect_token(p, 695)) // token='else' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6315,7 +6407,7 @@ while_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 689)) // token='while' + (_keyword = _PyPegen_expect_token(p, 698)) // token='while' && (a = named_expression_rule(p)) // named_expression && @@ -6415,11 +6507,11 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 703)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 704)) // token='in' && (_cut_var = 1) && @@ -6477,13 +6569,13 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 707)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 703)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 704)) // token='in' && (_cut_var = 1) && @@ -6612,11 +6704,11 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 656)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_34_rule(p)) // ','.with_item+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -6663,9 +6755,9 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 656)) // token='with' && - (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_34_rule(p)) // ','.with_item+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -6712,13 +6804,13 @@ with_stmt_rule(Parser *p) asdl_withitem_seq* a; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 707)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 656)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_34_rule(p)) // ','.with_item+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -6764,11 +6856,11 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 707)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 656)) // token='with' && - (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_34_rule(p)) // ','.with_item+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -6852,11 +6944,11 @@ with_item_rule(Parser *p) if ( (e = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && (t = star_target_rule(p)) // star_target && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_36_rule, p) + _PyPegen_lookahead(1, _tmp_35_rule, p) ) { D(fprintf(stderr, "%*c+ with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' star_target &(',' | ')' | ':')")); @@ -6977,7 +7069,7 @@ try_stmt_rule(Parser *p) asdl_stmt_seq* b; asdl_stmt_seq* f; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 665)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7021,13 +7113,13 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 665)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && (b = block_rule(p)) // block && - (ex = (asdl_excepthandler_seq*)_loop1_37_rule(p)) // except_block+ + (ex = (asdl_excepthandler_seq*)_loop1_36_rule(p)) // except_block+ && (el = else_block_rule(p), !p->error_indicator) // else_block? && @@ -7069,13 +7161,13 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 665)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && (b = block_rule(p)) // block && - (ex = (asdl_excepthandler_seq*)_loop1_38_rule(p)) // except_star_block+ + (ex = (asdl_excepthandler_seq*)_loop1_37_rule(p)) // except_star_block+ && (el = else_block_rule(p), !p->error_indicator) // else_block? && @@ -7168,7 +7260,7 @@ except_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (e = expression_rule(p)) // expression && @@ -7212,11 +7304,11 @@ except_block_rule(Parser *p) expr_ty e; expr_ty t; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (e = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 689)) // token='as' && (t = _PyPegen_name_token(p)) // NAME && @@ -7258,7 +7350,7 @@ except_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (e = expressions_rule(p)) // expressions && @@ -7299,7 +7391,7 @@ except_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7411,7 +7503,7 @@ except_star_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7458,13 +7550,13 @@ except_star_block_rule(Parser *p) expr_ty e; expr_ty t; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (e = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 689)) // token='as' && (t = _PyPegen_name_token(p)) // NAME && @@ -7507,7 +7599,7 @@ except_star_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7607,7 +7699,7 @@ finally_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 673)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 682)) // token='finally' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7681,7 +7773,7 @@ match_stmt_rule(Parser *p) && (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' && - (cases = (asdl_match_case_seq*)_loop1_39_rule(p)) // case_block+ + (cases = (asdl_match_case_seq*)_loop1_38_rule(p)) // case_block+ && (dedent_var = _PyPegen_expect_token(p, DEDENT)) // token='DEDENT' ) @@ -7915,7 +8007,7 @@ guard_rule(Parser *p) Token * _keyword; expr_ty guard; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (guard = named_expression_rule(p)) // named_expression ) @@ -8110,7 +8202,7 @@ as_pattern_rule(Parser *p) if ( (pattern = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && (target = pattern_capture_target_rule(p)) // pattern_capture_target ) @@ -8192,7 +8284,7 @@ or_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> or_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|'.closed_pattern+")); asdl_pattern_seq* patterns; if ( - (patterns = (asdl_pattern_seq*)_gather_41_rule(p)) // '|'.closed_pattern+ + (patterns = (asdl_pattern_seq*)_gather_40_rule(p)) // '|'.closed_pattern+ ) { D(fprintf(stderr, "%*c+ or_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'|'.closed_pattern+")); @@ -8445,7 +8537,7 @@ literal_pattern_rule(Parser *p) if ( (value = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_42_rule, p) + _PyPegen_lookahead(0, _tmp_41_rule, p) ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8544,7 +8636,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -8577,7 +8669,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -8610,7 +8702,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -8679,7 +8771,7 @@ literal_expr_rule(Parser *p) if ( (signed_number_var = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_42_rule, p) + _PyPegen_lookahead(0, _tmp_41_rule, p) ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8717,7 +8809,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START | TSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_43_rule, p) + _PyPegen_lookahead(1, _tmp_42_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -8738,7 +8830,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -8771,7 +8863,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -8804,7 +8896,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -9281,7 +9373,7 @@ pattern_capture_target_rule(Parser *p) && (name = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_44_rule, p) + _PyPegen_lookahead(0, _tmp_43_rule, p) ) { D(fprintf(stderr, "%*c+ pattern_capture_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); @@ -9396,7 +9488,7 @@ value_pattern_rule(Parser *p) if ( (attr = attr_rule(p)) // attr && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_44_rule, p) + _PyPegen_lookahead(0, _tmp_43_rule, p) ) { D(fprintf(stderr, "%*c+ value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr !('.' | '(' | '=')")); @@ -9815,7 +9907,7 @@ maybe_sequence_pattern_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_seq * patterns; if ( - (patterns = _gather_46_rule(p)) // ','.maybe_star_pattern+ + (patterns = _gather_45_rule(p)) // ','.maybe_star_pattern+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -10006,6 +10098,7 @@ star_pattern_rule(Parser *p) // | '{' double_star_pattern ','? '}' // | '{' items_pattern ',' double_star_pattern ','? '}' // | '{' items_pattern ','? '}' +// | invalid_mapping_pattern static pattern_ty mapping_pattern_rule(Parser *p) { @@ -10198,6 +10291,25 @@ mapping_pattern_rule(Parser *p) D(fprintf(stderr, "%*c%s mapping_pattern[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' items_pattern ','? '}'")); } + if (p->call_invalid_rules) { // invalid_mapping_pattern + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> mapping_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_mapping_pattern")); + void *invalid_mapping_pattern_var; + if ( + (invalid_mapping_pattern_var = invalid_mapping_pattern_rule(p)) // invalid_mapping_pattern + ) + { + D(fprintf(stderr, "%*c+ mapping_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_mapping_pattern")); + _res = invalid_mapping_pattern_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s mapping_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_mapping_pattern")); + } _res = NULL; done: p->level--; @@ -10223,13 +10335,13 @@ items_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> items_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+")); - asdl_seq * _gather_48_var; + asdl_seq * _gather_47_var; if ( - (_gather_48_var = _gather_48_rule(p)) // ','.key_value_pattern+ + (_gather_47_var = _gather_47_rule(p)) // ','.key_value_pattern+ ) { D(fprintf(stderr, "%*c+ items_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+")); - _res = _gather_48_var; + _res = _gather_47_var; goto done; } p->mark = _mark; @@ -10265,7 +10377,7 @@ key_value_pattern_rule(Parser *p) void *key; pattern_ty pattern; if ( - (key = _tmp_49_rule(p)) // literal_expr | attr + (key = _tmp_48_rule(p)) // literal_expr | attr && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -10593,7 +10705,7 @@ positional_patterns_rule(Parser *p) D(fprintf(stderr, "%*c> positional_patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.pattern+")); asdl_pattern_seq* args; if ( - (args = (asdl_pattern_seq*)_gather_51_rule(p)) // ','.pattern+ + (args = (asdl_pattern_seq*)_gather_50_rule(p)) // ','.pattern+ ) { D(fprintf(stderr, "%*c+ positional_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.pattern+")); @@ -10634,13 +10746,13 @@ keyword_patterns_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> keyword_patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.keyword_pattern+")); - asdl_seq * _gather_53_var; + asdl_seq * _gather_52_var; if ( - (_gather_53_var = _gather_53_rule(p)) // ','.keyword_pattern+ + (_gather_52_var = _gather_52_rule(p)) // ','.keyword_pattern+ ) { D(fprintf(stderr, "%*c+ keyword_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.keyword_pattern+")); - _res = _gather_53_var; + _res = _gather_52_var; goto done; } p->mark = _mark; @@ -10866,7 +10978,7 @@ type_param_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_type_param_seq* a; if ( - (a = (asdl_type_param_seq*)_gather_55_rule(p)) // ','.type_param+ + (a = (asdl_type_param_seq*)_gather_54_rule(p)) // ','.type_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11236,7 +11348,7 @@ expressions_rule(Parser *p) if ( (a = expression_rule(p)) // expression && - (b = _loop1_56_rule(p)) // ((',' expression))+ + (b = _loop1_55_rule(p)) // ((',' expression))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11407,11 +11519,11 @@ expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='else' && (c = expression_rule(p)) // expression ) @@ -11515,9 +11627,9 @@ yield_expr_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 587)) // token='yield' + (_keyword = _PyPegen_expect_token(p, 588)) // token='yield' && - (_keyword_1 = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword_1 = _PyPegen_expect_token(p, 642)) // token='from' && (a = expression_rule(p)) // expression ) @@ -11553,7 +11665,7 @@ yield_expr_rule(Parser *p) Token * _keyword; void *a; if ( - (_keyword = _PyPegen_expect_token(p, 587)) // token='yield' + (_keyword = _PyPegen_expect_token(p, 588)) // token='yield' && (a = star_expressions_rule(p), !p->error_indicator) // star_expressions? ) @@ -11624,7 +11736,7 @@ star_expressions_rule(Parser *p) if ( (a = star_expression_rule(p)) // star_expression && - (b = _loop1_57_rule(p)) // ((',' star_expression))+ + (b = _loop1_56_rule(p)) // ((',' star_expression))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11823,7 +11935,7 @@ star_named_expressions_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_59_rule(p)) // ','.star_named_expression+ + (a = (asdl_expr_seq*)_gather_58_rule(p)) // ','.star_named_expression+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -12119,7 +12231,7 @@ disjunction_rule(Parser *p) if ( (a = conjunction_rule(p)) // conjunction && - (b = _loop1_60_rule(p)) // (('or' conjunction))+ + (b = _loop1_59_rule(p)) // (('or' conjunction))+ ) { D(fprintf(stderr, "%*c+ disjunction[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "conjunction (('or' conjunction))+")); @@ -12207,7 +12319,7 @@ conjunction_rule(Parser *p) if ( (a = inversion_rule(p)) // inversion && - (b = _loop1_61_rule(p)) // (('and' inversion))+ + (b = _loop1_60_rule(p)) // (('and' inversion))+ ) { D(fprintf(stderr, "%*c+ conjunction[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "inversion (('and' inversion))+")); @@ -12293,7 +12405,7 @@ inversion_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 703)) // token='not' + (_keyword = _PyPegen_expect_token(p, 712)) // token='not' && (a = inversion_rule(p)) // inversion ) @@ -12379,7 +12491,7 @@ comparison_rule(Parser *p) if ( (a = bitwise_or_rule(p)) // bitwise_or && - (b = _loop1_62_rule(p)) // compare_op_bitwise_or_pair+ + (b = _loop1_61_rule(p)) // compare_op_bitwise_or_pair+ ) { D(fprintf(stderr, "%*c+ comparison[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or compare_op_bitwise_or_pair+")); @@ -12713,10 +12825,10 @@ noteq_bitwise_or_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> noteq_bitwise_or[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('!=') bitwise_or")); - void *_tmp_63_var; + void *_tmp_62_var; expr_ty a; if ( - (_tmp_63_var = _tmp_63_rule(p)) // '!=' + (_tmp_62_var = _tmp_62_rule(p)) // '!=' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12947,9 +13059,9 @@ notin_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 703)) // token='not' + (_keyword = _PyPegen_expect_token(p, 712)) // token='not' && - (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 704)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12995,7 +13107,7 @@ in_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword = _PyPegen_expect_token(p, 704)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -13042,9 +13154,9 @@ isnot_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 596)) // token='is' + (_keyword = _PyPegen_expect_token(p, 597)) // token='is' && - (_keyword_1 = _PyPegen_expect_token(p, 703)) // token='not' + (_keyword_1 = _PyPegen_expect_token(p, 712)) // token='not' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -13090,7 +13202,7 @@ is_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 596)) // token='is' + (_keyword = _PyPegen_expect_token(p, 597)) // token='is' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -14406,7 +14518,7 @@ await_primary_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 597)) // token='await' + (_keyword = _PyPegen_expect_token(p, 598)) // token='await' && (a = primary_rule(p)) // primary ) @@ -14764,7 +14876,7 @@ slices_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_65_rule(p)) // ','.(slice | starred_expression)+ + (a = (asdl_expr_seq*)_gather_64_rule(p)) // ','.(slice | starred_expression)+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -14836,7 +14948,7 @@ slice_rule(Parser *p) && (b = expression_rule(p), !p->error_indicator) // expression? && - (c = _tmp_66_rule(p), !p->error_indicator) // [':' expression?] + (c = _tmp_65_rule(p), !p->error_indicator) // [':' expression?] ) { D(fprintf(stderr, "%*c+ slice[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression? ':' expression? [':' expression?]")); @@ -14950,7 +15062,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -14983,7 +15095,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -15016,7 +15128,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -15049,7 +15161,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START | TSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_43_rule, p) + _PyPegen_lookahead(1, _tmp_42_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -15087,15 +15199,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'(' (tuple | group | genexp)")); - void *_tmp_67_var; + void *_tmp_66_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 7) // token='(' && - (_tmp_67_var = _tmp_67_rule(p)) // tuple | group | genexp + (_tmp_66_var = _tmp_66_rule(p)) // tuple | group | genexp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'(' (tuple | group | genexp)")); - _res = _tmp_67_var; + _res = _tmp_66_var; goto done; } p->mark = _mark; @@ -15108,15 +15220,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'[' (list | listcomp)")); - void *_tmp_68_var; + void *_tmp_67_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 9) // token='[' && - (_tmp_68_var = _tmp_68_rule(p)) // list | listcomp + (_tmp_67_var = _tmp_67_rule(p)) // list | listcomp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'[' (list | listcomp)")); - _res = _tmp_68_var; + _res = _tmp_67_var; goto done; } p->mark = _mark; @@ -15129,15 +15241,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'{' (dict | set | dictcomp | setcomp)")); - void *_tmp_69_var; + void *_tmp_68_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 25) // token='{' && - (_tmp_69_var = _tmp_69_rule(p)) // dict | set | dictcomp | setcomp + (_tmp_68_var = _tmp_68_rule(p)) // dict | set | dictcomp | setcomp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'{' (dict | set | dictcomp | setcomp)")); - _res = _tmp_69_var; + _res = _tmp_68_var; goto done; } p->mark = _mark; @@ -15208,7 +15320,7 @@ group_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_70_rule(p)) // yield_expr | named_expression + (a = _tmp_69_rule(p)) // yield_expr | named_expression && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -15284,7 +15396,7 @@ lambdef_rule(Parser *p) void *a; expr_ty b; if ( - (_keyword = _PyPegen_expect_token(p, 621)) // token='lambda' + (_keyword = _PyPegen_expect_token(p, 622)) // token='lambda' && (a = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -15409,9 +15521,9 @@ lambda_parameters_rule(Parser *p) if ( (a = lambda_slash_no_default_rule(p)) // lambda_slash_no_default && - (b = (asdl_arg_seq*)_loop0_71_rule(p)) // lambda_param_no_default* + (b = (asdl_arg_seq*)_loop0_70_rule(p)) // lambda_param_no_default* && - (c = _loop0_72_rule(p)) // lambda_param_with_default* + (c = _loop0_71_rule(p)) // lambda_param_with_default* && (d = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15441,7 +15553,7 @@ lambda_parameters_rule(Parser *p) if ( (a = lambda_slash_with_default_rule(p)) // lambda_slash_with_default && - (b = _loop0_72_rule(p)) // lambda_param_with_default* + (b = _loop0_71_rule(p)) // lambda_param_with_default* && (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15469,9 +15581,9 @@ lambda_parameters_rule(Parser *p) asdl_seq * b; void *c; if ( - (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_72_rule(p)) // lambda_param_no_default+ && - (b = _loop0_72_rule(p)) // lambda_param_with_default* + (b = _loop0_71_rule(p)) // lambda_param_with_default* && (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15498,7 +15610,7 @@ lambda_parameters_rule(Parser *p) asdl_seq * a; void *b; if ( - (a = _loop1_74_rule(p)) // lambda_param_with_default+ + (a = _loop1_73_rule(p)) // lambda_param_with_default+ && (b = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15571,7 +15683,7 @@ lambda_slash_no_default_rule(Parser *p) Token * _literal_1; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_72_rule(p)) // lambda_param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15600,7 +15712,7 @@ lambda_slash_no_default_rule(Parser *p) Token * _literal; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_72_rule(p)) // lambda_param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15652,9 +15764,9 @@ lambda_slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_71_rule(p)) // lambda_param_no_default* + (a = _loop0_70_rule(p)) // lambda_param_no_default* && - (b = _loop1_74_rule(p)) // lambda_param_with_default+ + (b = _loop1_73_rule(p)) // lambda_param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15684,9 +15796,9 @@ lambda_slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_71_rule(p)) // lambda_param_no_default* + (a = _loop0_70_rule(p)) // lambda_param_no_default* && - (b = _loop1_74_rule(p)) // lambda_param_with_default+ + (b = _loop1_73_rule(p)) // lambda_param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15763,7 +15875,7 @@ lambda_star_etc_rule(Parser *p) && (a = lambda_param_no_default_rule(p)) // lambda_param_no_default && - (b = _loop0_75_rule(p)) // lambda_param_maybe_default* + (b = _loop0_74_rule(p)) // lambda_param_maybe_default* && (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) @@ -15796,7 +15908,7 @@ lambda_star_etc_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _loop1_76_rule(p)) // lambda_param_maybe_default+ + (b = _loop1_75_rule(p)) // lambda_param_maybe_default+ && (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) @@ -16436,7 +16548,7 @@ fstring_full_format_spec_rule(Parser *p) if ( (colon = _PyPegen_expect_token(p, 11)) // token=':' && - (spec = _loop0_77_rule(p)) // fstring_format_spec* + (spec = _loop0_76_rule(p)) // fstring_format_spec* ) { D(fprintf(stderr, "%*c+ fstring_full_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' fstring_format_spec*")); @@ -16554,7 +16666,7 @@ fstring_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, FSTRING_START)) // token='FSTRING_START' && - (b = _loop0_78_rule(p)) // fstring_middle* + (b = _loop0_77_rule(p)) // fstring_middle* && (c = _PyPegen_expect_token(p, FSTRING_END)) // token='FSTRING_END' ) @@ -16770,7 +16882,7 @@ tstring_full_format_spec_rule(Parser *p) if ( (colon = _PyPegen_expect_token(p, 11)) // token=':' && - (spec = _loop0_79_rule(p)) // tstring_format_spec* + (spec = _loop0_78_rule(p)) // tstring_format_spec* ) { D(fprintf(stderr, "%*c+ tstring_full_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' tstring_format_spec*")); @@ -16989,7 +17101,7 @@ tstring_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, TSTRING_START)) // token='TSTRING_START' && - (b = _loop0_80_rule(p)) // tstring_middle* + (b = _loop0_79_rule(p)) // tstring_middle* && (c = _PyPegen_expect_token(p, TSTRING_END)) // token='TSTRING_END' ) @@ -17057,7 +17169,7 @@ string_rule(Parser *p) return _res; } -// strings: ((fstring | string | tstring))+ +// strings: invalid_string_tstring_concat | ((fstring | string))+ | tstring+ static expr_ty strings_rule(Parser *p) { @@ -17083,18 +17195,37 @@ strings_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // ((fstring | string | tstring))+ + if (p->call_invalid_rules) { // invalid_string_tstring_concat if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string | tstring))+")); - asdl_expr_seq* a; + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_string_tstring_concat")); + void *invalid_string_tstring_concat_var; if ( - (a = (asdl_expr_seq*)_loop1_81_rule(p)) // ((fstring | string | tstring))+ + (invalid_string_tstring_concat_var = invalid_string_tstring_concat_rule(p)) // invalid_string_tstring_concat ) { - D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string | tstring))+")); + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_string_tstring_concat")); + _res = invalid_string_tstring_concat_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_string_tstring_concat")); + } + { // ((fstring | string))+ + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); + asdl_expr_seq* a; + if ( + (a = (asdl_expr_seq*)_loop1_80_rule(p)) // ((fstring | string))+ + ) + { + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -17114,7 +17245,40 @@ strings_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string | tstring))+")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string))+")); + } + { // tstring+ + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring+")); + asdl_expr_seq* a; + if ( + (a = (asdl_expr_seq*)_loop1_81_rule(p)) // tstring+ + ) + { + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring+")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyPegen_concatenate_tstrings ( p , a , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring+")); } _res = NULL; done: @@ -17651,13 +17815,13 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 707)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 703)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 704)) // token='in' && (_cut_var = 1) && @@ -17696,11 +17860,11 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 703)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 704)) // token='in' && (_cut_var = 1) && @@ -19078,7 +19242,7 @@ target_with_star_atom_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ target_with_star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -19122,7 +19286,7 @@ target_with_star_atom_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ target_with_star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -19469,7 +19633,7 @@ single_subscript_attribute_target_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ single_subscript_attribute_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -19513,7 +19677,7 @@ single_subscript_attribute_target_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ single_subscript_attribute_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -19623,7 +19787,7 @@ t_primary_raw(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME &t_lookahead")); @@ -19667,7 +19831,7 @@ t_primary_raw(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' &t_lookahead")); @@ -19705,7 +19869,7 @@ t_primary_raw(Parser *p) && (b = genexp_rule(p)) // genexp && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary genexp &t_lookahead")); @@ -19749,7 +19913,7 @@ t_primary_raw(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '(' arguments? ')' &t_lookahead")); @@ -19784,7 +19948,7 @@ t_primary_raw(Parser *p) if ( (a = atom_rule(p)) // atom && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "atom &t_lookahead")); @@ -19974,7 +20138,7 @@ del_target_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ del_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -20018,7 +20182,7 @@ del_target_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ del_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -20506,7 +20670,7 @@ func_type_comment_rule(Parser *p) && (t = _PyPegen_expect_token(p, TYPE_COMMENT)) // token='TYPE_COMMENT' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_104_rule, p) + _PyPegen_lookahead(1, _tmp_104_rule, p) ) { D(fprintf(stderr, "%*c+ func_type_comment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE TYPE_COMMENT &(NEWLINE INDENT)")); @@ -20700,7 +20864,7 @@ invalid_arguments_rule(Parser *p) && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_110_rule, p) + _PyPegen_lookahead(1, _tmp_110_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); @@ -20898,7 +21062,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_112_rule, p) + _PyPegen_lookahead(0, _tmp_112_rule, p) && (a = expression_rule(p)) // expression && @@ -21001,11 +21165,11 @@ expression_without_invalid_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='else' && (c = expression_rule(p)) // expression ) @@ -21273,7 +21437,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_114_rule, p) + _PyPegen_lookahead(0, _tmp_114_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -21305,11 +21469,11 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_115_rule, p) + _PyPegen_lookahead(0, _tmp_115_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); @@ -21338,13 +21502,13 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='else' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) expression_rule, p) + _PyPegen_lookahead_for_expr(0, expression_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction 'else' !expression")); @@ -21374,11 +21538,11 @@ invalid_expression_rule(Parser *p) if ( (a = (stmt_ty)_tmp_116_rule(p)) // pass_stmt | break_stmt | continue_stmt && - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='else' && (c = simple_stmt_rule(p)) // simple_stmt ) @@ -21407,7 +21571,7 @@ invalid_expression_rule(Parser *p) Token * a; Token * b; if ( - (a = _PyPegen_expect_token(p, 621)) // token='lambda' + (a = _PyPegen_expect_token(p, 622)) // token='lambda' && (_opt_var = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -21440,7 +21604,7 @@ invalid_expression_rule(Parser *p) Token * a; Token * b; if ( - (a = _PyPegen_expect_token(p, 621)) // token='lambda' + (a = _PyPegen_expect_token(p, 622)) // token='lambda' && (_opt_var = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -21534,7 +21698,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_117_rule, p) + _PyPegen_lookahead(0, _tmp_117_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -21560,7 +21724,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_118_rule, p) + _PyPegen_lookahead(0, _tmp_118_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -21568,7 +21732,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_117_rule, p) + _PyPegen_lookahead(0, _tmp_117_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -21889,6 +22053,82 @@ invalid_ann_assign_target_rule(Parser *p) return _res; } +// invalid_raise_stmt: 'raise' 'from' | 'raise' expression 'from' +static void * +invalid_raise_stmt_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // 'raise' 'from' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' 'from'")); + Token * a; + Token * b; + if ( + (a = _PyPegen_expect_token(p, 628)) // token='raise' + && + (b = _PyPegen_expect_token(p, 642)) // token='from' + ) + { + D(fprintf(stderr, "%*c+ invalid_raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' 'from'")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "did you forget an expression between 'raise' and 'from'?" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' 'from'")); + } + { // 'raise' expression 'from' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from'")); + Token * _keyword; + Token * a; + expr_ty expression_var; + if ( + (_keyword = _PyPegen_expect_token(p, 628)) // token='raise' + && + (expression_var = expression_rule(p)) // expression + && + (a = _PyPegen_expect_token(p, 642)) // token='from' + ) + { + D(fprintf(stderr, "%*c+ invalid_raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from'")); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "did you forget an expression after 'from'?" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' expression 'from'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_del_stmt: 'del' star_expressions static void * invalid_del_stmt_rule(Parser *p) @@ -21911,7 +22151,7 @@ invalid_del_stmt_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 625)) // token='del' + (_keyword = _PyPegen_expect_token(p, 630)) // token='del' && (a = star_expressions_rule(p)) // star_expressions ) @@ -21935,6 +22175,173 @@ invalid_del_stmt_rule(Parser *p) return _res; } +// invalid_assert_stmt: +// | 'assert' expression '=' expression +// | 'assert' expression ',' expression '=' expression +// | 'assert' expression ':=' expression +// | 'assert' expression ',' expression ':=' expression +static void * +invalid_assert_stmt_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // 'assert' expression '=' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_assert_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'assert' expression '=' expression")); + Token * _keyword; + Token * _literal; + expr_ty a; + expr_ty b; + if ( + (_keyword = _PyPegen_expect_token(p, 634)) // token='assert' + && + (a = expression_rule(p)) // expression + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (b = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ invalid_assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression '=' expression")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot assign to %s here. Maybe you meant '==' instead of '='?" , _PyPegen_get_expr_name ( a ) ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_assert_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'assert' expression '=' expression")); + } + { // 'assert' expression ',' expression '=' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_assert_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'assert' expression ',' expression '=' expression")); + Token * _keyword; + Token * _literal; + Token * _literal_1; + expr_ty a; + expr_ty b; + expr_ty expression_var; + if ( + (_keyword = _PyPegen_expect_token(p, 634)) // token='assert' + && + (expression_var = expression_rule(p)) // expression + && + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (a = expression_rule(p)) // expression + && + (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' + && + (b = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ invalid_assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression ',' expression '=' expression")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot assign to %s here. Maybe you meant '==' instead of '='?" , _PyPegen_get_expr_name ( a ) ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_assert_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'assert' expression ',' expression '=' expression")); + } + { // 'assert' expression ':=' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_assert_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'assert' expression ':=' expression")); + Token * _keyword; + Token * _literal; + expr_ty a; + expr_ty b; + if ( + (_keyword = _PyPegen_expect_token(p, 634)) // token='assert' + && + (a = expression_rule(p)) // expression + && + (_literal = _PyPegen_expect_token(p, 53)) // token=':=' + && + (b = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ invalid_assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression ':=' expression")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot use named expression without parentheses here" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_assert_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'assert' expression ':=' expression")); + } + { // 'assert' expression ',' expression ':=' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_assert_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'assert' expression ',' expression ':=' expression")); + Token * _keyword; + Token * _literal; + Token * _literal_1; + expr_ty a; + expr_ty b; + expr_ty expression_var; + if ( + (_keyword = _PyPegen_expect_token(p, 634)) // token='assert' + && + (expression_var = expression_rule(p)) // expression + && + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (a = expression_rule(p)) // expression + && + (_literal_1 = _PyPegen_expect_token(p, 53)) // token=':=' + && + (b = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ invalid_assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression ',' expression ':=' expression")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot use named expression without parentheses here" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_assert_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'assert' expression ',' expression ':=' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_block: NEWLINE !INDENT static void * invalid_block_rule(Parser *p) @@ -22190,7 +22597,7 @@ invalid_parameters_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"/\" ','")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "at least one argument must precede /" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "at least one parameter must precede /" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22208,13 +22615,13 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); - asdl_seq * _loop0_32_var; + asdl_seq * _loop0_31_var; void *_tmp_123_var; Token * a; if ( (_tmp_123_var = _tmp_123_rule(p)) // slash_no_default | slash_with_default && - (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* + (_loop0_31_var = _loop0_31_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22238,7 +22645,7 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); - asdl_seq * _loop0_28_var; + asdl_seq * _loop0_27_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -22246,7 +22653,7 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = slash_no_default_rule(p), !p->error_indicator) // slash_no_default? && - (_loop0_28_var = _loop0_28_rule(p)) // param_no_default* + (_loop0_27_var = _loop0_27_rule(p)) // param_no_default* && (invalid_parameters_helper_var = invalid_parameters_helper_rule(p)) // invalid_parameters_helper && @@ -22272,18 +22679,18 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); - asdl_seq * _loop0_28_var; - asdl_seq * _loop1_30_var; + asdl_seq * _loop0_27_var; + asdl_seq * _loop1_29_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_28_var = _loop0_28_rule(p)) // param_no_default* + (_loop0_27_var = _loop0_27_rule(p)) // param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_loop1_30_var = _loop1_30_rule(p)) // param_no_default+ + (_loop1_29_var = _loop1_29_rule(p)) // param_no_default+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22310,8 +22717,8 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_32_var; - asdl_seq * _loop0_32_var_1; + asdl_seq * _loop0_31_var; + asdl_seq * _loop0_31_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_tmp_124_var; @@ -22319,13 +22726,13 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = _tmp_123_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && - (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* + (_loop0_31_var = _loop0_31_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (_tmp_124_var = _tmp_124_rule(p)) // ',' | param_no_default && - (_loop0_32_var_1 = _loop0_32_rule(p)) // param_maybe_default* + (_loop0_31_var_1 = _loop0_31_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22350,10 +22757,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_33_var; + asdl_seq * _loop1_32_var; Token * a; if ( - (_loop1_33_var = _loop1_33_rule(p)) // param_maybe_default+ + (_loop1_32_var = _loop1_32_rule(p)) // param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -22402,7 +22809,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_125_rule, p) + _PyPegen_lookahead(1, _tmp_125_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -22456,7 +22863,7 @@ invalid_star_etc_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "named arguments must follow bare *" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "named parameters must follow bare *" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22516,7 +22923,7 @@ invalid_star_etc_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' param '='")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-positional argument cannot have default value" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-positional parameter cannot have default value" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22535,7 +22942,7 @@ invalid_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_32_var; + asdl_seq * _loop0_31_var; void *_tmp_127_var; void *_tmp_127_var_1; Token * a; @@ -22544,7 +22951,7 @@ invalid_star_etc_rule(Parser *p) && (_tmp_127_var = _tmp_127_rule(p)) // param_no_default | ',' && - (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* + (_loop0_31_var = _loop0_31_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && @@ -22552,7 +22959,7 @@ invalid_star_etc_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "* argument may appear only once" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "* may appear only once" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22601,7 +23008,7 @@ invalid_kwds_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param '='")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-keyword argument cannot have default value" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-keyword parameter cannot have default value" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22634,7 +23041,7 @@ invalid_kwds_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' param")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "arguments cannot follow var-keyword argument" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "parameters cannot follow var-keyword parameter" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22667,7 +23074,7 @@ invalid_kwds_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "arguments cannot follow var-keyword argument" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "parameters cannot follow var-keyword parameter" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22728,13 +23135,13 @@ invalid_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - asdl_seq * _loop1_31_var; + asdl_seq * _loop1_30_var; if ( - (_loop1_31_var = _loop1_31_rule(p)) // param_with_default+ + (_loop1_30_var = _loop1_30_rule(p)) // param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - _res = _loop1_31_var; + _res = _loop1_30_var; goto done; } p->mark = _mark; @@ -22781,7 +23188,7 @@ invalid_lambda_parameters_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"/\" ','")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "at least one argument must precede /" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "at least one parameter must precede /" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -22799,13 +23206,13 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); - asdl_seq * _loop0_75_var; + asdl_seq * _loop0_74_var; void *_tmp_129_var; Token * a; if ( (_tmp_129_var = _tmp_129_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && - (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* + (_loop0_74_var = _loop0_74_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22829,7 +23236,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); - asdl_seq * _loop0_71_var; + asdl_seq * _loop0_70_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -22837,7 +23244,7 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = lambda_slash_no_default_rule(p), !p->error_indicator) // lambda_slash_no_default? && - (_loop0_71_var = _loop0_71_rule(p)) // lambda_param_no_default* + (_loop0_70_var = _loop0_70_rule(p)) // lambda_param_no_default* && (invalid_lambda_parameters_helper_var = invalid_lambda_parameters_helper_rule(p)) // invalid_lambda_parameters_helper && @@ -22864,13 +23271,13 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); asdl_seq * _gather_131_var; - asdl_seq * _loop0_71_var; + asdl_seq * _loop0_70_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_71_var = _loop0_71_rule(p)) // lambda_param_no_default* + (_loop0_70_var = _loop0_70_rule(p)) // lambda_param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && @@ -22901,8 +23308,8 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_75_var; - asdl_seq * _loop0_75_var_1; + asdl_seq * _loop0_74_var; + asdl_seq * _loop0_74_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_tmp_132_var; @@ -22910,13 +23317,13 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = _tmp_129_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && - (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* + (_loop0_74_var = _loop0_74_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (_tmp_132_var = _tmp_132_rule(p)) // ',' | lambda_param_no_default && - (_loop0_75_var_1 = _loop0_75_rule(p)) // lambda_param_maybe_default* + (_loop0_74_var_1 = _loop0_74_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22941,10 +23348,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_76_var; + asdl_seq * _loop1_75_var; Token * a; if ( - (_loop1_76_var = _loop1_76_rule(p)) // lambda_param_maybe_default+ + (_loop1_75_var = _loop1_75_rule(p)) // lambda_param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -23015,13 +23422,13 @@ invalid_lambda_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - asdl_seq * _loop1_74_var; + asdl_seq * _loop1_73_var; if ( - (_loop1_74_var = _loop1_74_rule(p)) // lambda_param_with_default+ + (_loop1_73_var = _loop1_73_rule(p)) // lambda_param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - _res = _loop1_74_var; + _res = _loop1_73_var; goto done; } p->mark = _mark; @@ -23065,7 +23472,7 @@ invalid_lambda_star_etc_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); - _res = RAISE_SYNTAX_ERROR ( "named arguments must follow bare *" ); + _res = RAISE_SYNTAX_ERROR ( "named parameters must follow bare *" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -23095,7 +23502,7 @@ invalid_lambda_star_etc_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' lambda_param '='")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-positional argument cannot have default value" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-positional parameter cannot have default value" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -23114,7 +23521,7 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_75_var; + asdl_seq * _loop0_74_var; void *_tmp_134_var; void *_tmp_134_var_1; Token * a; @@ -23123,7 +23530,7 @@ invalid_lambda_star_etc_rule(Parser *p) && (_tmp_134_var = _tmp_134_rule(p)) // lambda_param_no_default | ',' && - (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* + (_loop0_74_var = _loop0_74_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && @@ -23131,7 +23538,7 @@ invalid_lambda_star_etc_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "* argument may appear only once" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "* may appear only once" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -23183,7 +23590,7 @@ invalid_lambda_kwds_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param '='")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-keyword argument cannot have default value" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-keyword parameter cannot have default value" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -23216,7 +23623,7 @@ invalid_lambda_kwds_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' lambda_param")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "arguments cannot follow var-keyword argument" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "parameters cannot follow var-keyword parameter" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -23249,7 +23656,7 @@ invalid_lambda_kwds_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); - _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "arguments cannot follow var-keyword argument" ); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "parameters cannot follow var-keyword parameter" ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -23347,11 +23754,11 @@ invalid_with_item_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_36_rule, p) + _PyPegen_lookahead(1, _tmp_35_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -23397,13 +23804,13 @@ invalid_for_if_clause_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_tmp_135_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 703)) // token='for' && (_tmp_135_var = _tmp_135_rule(p)) // bitwise_or ((',' bitwise_or))* ','? && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 695) // token='in' + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 704) // token='in' ) { D(fprintf(stderr, "%*c+ invalid_for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); @@ -23449,9 +23856,9 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 703)) // token='for' && (a = star_expressions_rule(p)) // star_expressions ) @@ -23581,11 +23988,11 @@ invalid_import_rule(Parser *p) Token * a; expr_ty dotted_name_var; if ( - (a = _PyPegen_expect_token(p, 634)) // token='import' + (a = _PyPegen_expect_token(p, 643)) // token='import' && (_gather_137_var = _gather_137_rule(p)) // ','.dotted_name+ && - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 642)) // token='from' && (dotted_name_var = dotted_name_rule(p)) // dotted_name ) @@ -23612,7 +24019,7 @@ invalid_import_rule(Parser *p) Token * _keyword; Token * token; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword = _PyPegen_expect_token(p, 643)) // token='import' && (token = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23636,7 +24043,7 @@ invalid_import_rule(Parser *p) return _res; } -// invalid_dotted_as_name: dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression +// invalid_dotted_as_name: dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression static void * invalid_dotted_as_name_rule(Parser *p) { @@ -23649,26 +24056,26 @@ invalid_dotted_as_name_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression + { // dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_dotted_as_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + D(fprintf(stderr, "%*c> invalid_dotted_as_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); Token * _keyword; expr_ty a; expr_ty dotted_name_var; if ( (dotted_name_var = dotted_name_rule(p)) // dotted_name && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) + _PyPegen_lookahead(0, _tmp_138_rule, p) && (a = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ invalid_dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + D(fprintf(stderr, "%*c+ invalid_dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use %s as import target" , _PyPegen_get_expr_name ( a ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -23679,7 +24086,7 @@ invalid_dotted_as_name_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_dotted_as_name[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); } _res = NULL; done: @@ -23687,7 +24094,7 @@ invalid_dotted_as_name_rule(Parser *p) return _res; } -// invalid_import_from_as_name: NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression +// invalid_import_from_as_name: NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression static void * invalid_import_from_as_name_rule(Parser *p) { @@ -23700,26 +24107,26 @@ invalid_import_from_as_name_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression + { // NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_import_from_as_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + D(fprintf(stderr, "%*c> invalid_import_from_as_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); Token * _keyword; expr_ty a; expr_ty name_var; if ( (name_var = _PyPegen_name_token(p)) // NAME && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) + _PyPegen_lookahead(0, _tmp_138_rule, p) && (a = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ invalid_import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + D(fprintf(stderr, "%*c+ invalid_import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use %s as import target" , _PyPegen_get_expr_name ( a ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -23730,7 +24137,7 @@ invalid_import_from_as_name_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_import_from_as_name[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); } _res = NULL; done: @@ -23838,9 +24245,9 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 656)) // token='with' && (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ && @@ -23876,9 +24283,9 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 656)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -23938,9 +24345,9 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 647)) // token='with' + (a = _PyPegen_expect_token(p, 656)) // token='with' && (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ && @@ -23981,9 +24388,9 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 647)) // token='with' + (a = _PyPegen_expect_token(p, 656)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -24046,7 +24453,7 @@ invalid_try_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 656)) // token='try' + (a = _PyPegen_expect_token(p, 665)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24078,13 +24485,13 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 665)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_143_rule, p) + _PyPegen_lookahead(0, _tmp_143_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -24110,28 +24517,28 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; Token * _literal_1; asdl_seq * _loop0_144_var; - asdl_seq * _loop1_37_var; + asdl_seq * _loop1_36_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 665)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (_loop0_144_var = _loop0_144_rule(p)) // block* && - (_loop1_37_var = _loop1_37_rule(p)) // except_block+ + (_loop1_36_var = _loop1_36_rule(p)) // except_block+ && - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (b = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -24159,20 +24566,20 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; Token * _literal_1; asdl_seq * _loop0_144_var; - asdl_seq * _loop1_38_var; + asdl_seq * _loop1_37_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; if ( - (_keyword = _PyPegen_expect_token(p, 656)) // token='try' + (_keyword = _PyPegen_expect_token(p, 665)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (_loop0_144_var = _loop0_144_rule(p)) // block* && - (_loop1_38_var = _loop1_38_rule(p)) // except_star_block+ + (_loop1_37_var = _loop1_37_rule(p)) // except_star_block+ && - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (_opt_var = _tmp_145_rule(p), !p->error_indicator) // [expression ['as' NAME]] && @@ -24202,7 +24609,7 @@ invalid_try_stmt_rule(Parser *p) // | 'except' expression ',' expressions 'as' NAME ':' // | 'except' expression ['as' NAME] NEWLINE // | 'except' NEWLINE -// | 'except' expression 'as' expression +// | 'except' expression 'as' expression ':' block static void * invalid_except_stmt_rule(Parser *p) { @@ -24229,7 +24636,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty expressions_var; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (a = expression_rule(p)) // expression && @@ -24237,7 +24644,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 689)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24269,11 +24676,11 @@ invalid_except_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24300,7 +24707,7 @@ invalid_except_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24318,27 +24725,33 @@ invalid_except_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_except_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' NEWLINE")); } - { // 'except' expression 'as' expression + { // 'except' expression 'as' expression ':' block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression")); + D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression ':' block")); Token * _keyword; Token * _keyword_1; + Token * _literal; expr_ty a; + asdl_stmt_seq* block_var; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 689)) // token='as' && (a = expression_rule(p)) // expression + && + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + && + (block_var = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression")); + D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression ':' block")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use except statement with %s" , _PyPegen_get_expr_name ( a ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24349,7 +24762,7 @@ invalid_except_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_except_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' expression 'as' expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' expression 'as' expression ':' block")); } _res = NULL; done: @@ -24361,7 +24774,7 @@ invalid_except_stmt_rule(Parser *p) // | 'except' '*' expression ',' expressions 'as' NAME ':' // | 'except' '*' expression ['as' NAME] NEWLINE // | 'except' '*' (NEWLINE | ':') -// | 'except' '*' expression 'as' expression +// | 'except' '*' expression 'as' expression ':' block static void * invalid_except_star_stmt_rule(Parser *p) { @@ -24389,7 +24802,7 @@ invalid_except_star_stmt_rule(Parser *p) expr_ty expressions_var; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24399,7 +24812,7 @@ invalid_except_star_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 689)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24432,13 +24845,13 @@ invalid_except_star_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24466,7 +24879,7 @@ invalid_except_star_stmt_rule(Parser *p) void *_tmp_146_var; Token * a; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24486,30 +24899,36 @@ invalid_except_star_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_except_star_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' (NEWLINE | ':')")); } - { // 'except' '*' expression 'as' expression + { // 'except' '*' expression 'as' expression ':' block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression")); + D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); Token * _keyword; Token * _keyword_1; Token * _literal; + Token * _literal_1; expr_ty a; + asdl_stmt_seq* block_var; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 689)) // token='as' && (a = expression_rule(p)) // expression + && + (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' + && + (block_var = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression")); + D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use except* statement with %s" , _PyPegen_get_expr_name ( a ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24520,7 +24939,7 @@ invalid_except_star_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_except_star_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' expression 'as' expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); } _res = NULL; done: @@ -24551,7 +24970,7 @@ invalid_finally_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 673)) // token='finally' + (a = _PyPegen_expect_token(p, 682)) // token='finally' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24607,11 +25026,11 @@ invalid_except_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24643,7 +25062,7 @@ invalid_except_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24699,13 +25118,13 @@ invalid_except_star_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='except' + (a = _PyPegen_expect_token(p, 686)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24736,6 +25155,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) // invalid_match_stmt: // | "match" subject_expr NEWLINE // | "match" subject_expr ':' NEWLINE !INDENT +// | "case" patterns guard? ':' block static void * invalid_match_stmt_rule(Parser *p) { @@ -24813,6 +25233,43 @@ invalid_match_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_match_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "\"match\" subject_expr ':' NEWLINE !INDENT")); } + { // "case" patterns guard? ':' block + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_match_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "\"case\" patterns guard? ':' block")); + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty a; + Token * b; + asdl_stmt_seq* block_var; + pattern_ty patterns_var; + if ( + (a = _PyPegen_expect_soft_keyword(p, "case")) // soft_keyword='"case"' + && + (patterns_var = patterns_rule(p)) // patterns + && + (_opt_var = guard_rule(p), !p->error_indicator) // guard? + && + (b = _PyPegen_expect_token(p, 11)) // token=':' + && + (block_var = block_rule(p)) // block + ) + { + D(fprintf(stderr, "%*c+ invalid_match_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"case\" patterns guard? ':' block")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "case statement must be inside match statement" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_match_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "\"case\" patterns guard? ':' block")); + } _res = NULL; done: p->level--; @@ -24938,7 +25395,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && (a = _PyPegen_expect_soft_keyword(p, "_")) // soft_keyword='"_"' ) @@ -24968,7 +25425,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && (a = expression_rule(p)) // expression ) @@ -25041,6 +25498,70 @@ invalid_class_pattern_rule(Parser *p) return _res; } +// invalid_mapping_pattern: +// | '{' [(items_pattern ',')] double_star_pattern ',' items_pattern ','? '}' +static void * +invalid_mapping_pattern_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '{' [(items_pattern ',')] double_star_pattern ',' items_pattern ','? '}' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_mapping_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' [(items_pattern ',')] double_star_pattern ',' items_pattern ','? '}'")); + Token * _literal; + Token * _literal_1; + Token * _literal_2; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + void *_opt_var_1; + UNUSED(_opt_var_1); // Silence compiler warnings + asdl_seq* items_pattern_var; + expr_ty rest; + if ( + (_literal = _PyPegen_expect_token(p, 25)) // token='{' + && + (_opt_var = _tmp_147_rule(p), !p->error_indicator) // [(items_pattern ',')] + && + (rest = double_star_pattern_rule(p)) // double_star_pattern + && + (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' + && + (items_pattern_var = items_pattern_rule(p)) // items_pattern + && + (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? + && + (_literal_2 = _PyPegen_expect_token(p, 26)) // token='}' + ) + { + D(fprintf(stderr, "%*c+ invalid_mapping_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' [(items_pattern ',')] double_star_pattern ',' items_pattern ','? '}'")); + _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( rest , "double star pattern must be the last (right-most) subpattern in the mapping pattern" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_mapping_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' [(items_pattern ',')] double_star_pattern ',' items_pattern ','? '}'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_class_argument_pattern: // | [positional_patterns ','] keyword_patterns ',' positional_patterns static asdl_pattern_seq* @@ -25067,7 +25588,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_147_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_148_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -25120,7 +25641,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25151,7 +25672,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty a_1; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 682)) // token='if' + (a = _PyPegen_expect_token(p, 691)) // token='if' && (a_1 = named_expression_rule(p)) // named_expression && @@ -25206,7 +25727,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 687)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 696)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25237,7 +25758,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 687)) // token='elif' + (a = _PyPegen_expect_token(p, 696)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25290,7 +25811,7 @@ invalid_else_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 686)) // token='else' + (a = _PyPegen_expect_token(p, 695)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -25323,13 +25844,13 @@ invalid_else_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword = _PyPegen_expect_token(p, 695)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (block_var = block_rule(p)) // block && - (_keyword_1 = _PyPegen_expect_token(p, 687)) // token='elif' + (_keyword_1 = _PyPegen_expect_token(p, 696)) // token='elif' ) { D(fprintf(stderr, "%*c+ invalid_else_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else' ':' block 'elif'")); @@ -25376,7 +25897,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 689)) // token='while' + (_keyword = _PyPegen_expect_token(p, 698)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25407,7 +25928,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 689)) // token='while' + (a = _PyPegen_expect_token(p, 698)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25466,13 +25987,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 703)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 704)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -25507,13 +26028,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 694)) // token='for' + (a = _PyPegen_expect_token(p, 703)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword = _PyPegen_expect_token(p, 695)) // token='in' + (_keyword = _PyPegen_expect_token(p, 704)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -25579,9 +26100,9 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 699)) // token='def' + (a = _PyPegen_expect_token(p, 708)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -25593,7 +26114,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_3 = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_26_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -25638,9 +26159,9 @@ invalid_def_raw_rule(Parser *p) asdl_stmt_seq* block_var; expr_ty name_var; if ( - (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 707), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 699)) // token='def' + (_keyword = _PyPegen_expect_token(p, 708)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -25652,7 +26173,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_3 = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_26_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -25704,13 +26225,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 701)) // token='class' + (_keyword = _PyPegen_expect_token(p, 710)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_25_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -25743,13 +26264,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 701)) // token='class' + (a = _PyPegen_expect_token(p, 710)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_25_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -25864,7 +26385,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) + _PyPegen_lookahead(1, _tmp_149_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -25974,7 +26495,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) + _PyPegen_lookahead(1, _tmp_149_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -26233,7 +26754,7 @@ invalid_fstring_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) annotated_rhs_rule, p) + _PyPegen_lookahead_for_expr(0, annotated_rhs_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !annotated_rhs")); @@ -26262,7 +26783,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_149_rule, p) + _PyPegen_lookahead(0, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -26294,7 +26815,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_150_rule, p) + _PyPegen_lookahead(0, _tmp_151_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -26358,9 +26879,9 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(0, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -26384,7 +26905,7 @@ invalid_fstring_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_fstring_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_77_var; + asdl_seq * _loop0_76_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; @@ -26397,11 +26918,11 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_77_var = _loop0_77_rule(p)) // fstring_format_spec* + (_loop0_76_var = _loop0_76_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -26438,7 +26959,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -26485,7 +27006,7 @@ invalid_fstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(1, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -26511,7 +27032,7 @@ invalid_fstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) + _PyPegen_lookahead_for_expr(0, _PyPegen_name_token, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' !NAME")); @@ -26675,7 +27196,7 @@ invalid_tstring_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) annotated_rhs_rule, p) + _PyPegen_lookahead_for_expr(0, annotated_rhs_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !annotated_rhs")); @@ -26704,7 +27225,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_149_rule, p) + _PyPegen_lookahead(0, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -26736,7 +27257,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_150_rule, p) + _PyPegen_lookahead(0, _tmp_151_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -26800,9 +27321,9 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(0, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -26826,7 +27347,7 @@ invalid_tstring_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_tstring_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_77_var; + asdl_seq * _loop0_76_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; @@ -26839,11 +27360,11 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_77_var = _loop0_77_rule(p)) // fstring_format_spec* + (_loop0_76_var = _loop0_76_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -26880,7 +27401,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -26927,7 +27448,7 @@ invalid_tstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(1, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -26953,7 +27474,7 @@ invalid_tstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) + _PyPegen_lookahead_for_expr(0, _PyPegen_name_token, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' !NAME")); @@ -26975,6 +27496,81 @@ invalid_tstring_conversion_character_rule(Parser *p) return _res; } +// invalid_string_tstring_concat: +// | ((fstring | string))+ tstring +// | tstring+ (fstring | string) +static void * +invalid_string_tstring_concat_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ((fstring | string))+ tstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_string_tstring_concat[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+ tstring")); + asdl_seq * a; + expr_ty b; + if ( + (a = _loop1_80_rule(p)) // ((fstring | string))+ + && + (b = (expr_ty)tstring_rule(p)) // tstring + ) + { + D(fprintf(stderr, "%*c+ invalid_string_tstring_concat[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+ tstring")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_last_item ( a , expr_ty ) , b , "cannot mix t-string literals with string or bytes literals" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_string_tstring_concat[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string))+ tstring")); + } + { // tstring+ (fstring | string) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_string_tstring_concat[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring+ (fstring | string)")); + asdl_seq * a; + expr_ty b; + if ( + (a = _loop1_81_rule(p)) // tstring+ + && + (b = (expr_ty)_tmp_154_rule(p)) // fstring | string + ) + { + D(fprintf(stderr, "%*c+ invalid_string_tstring_concat[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring+ (fstring | string)")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_last_item ( a , expr_ty ) , b , "cannot mix t-string literals with string or bytes literals" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_string_tstring_concat[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring+ (fstring | string)")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_arithmetic: sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion static void * invalid_arithmetic_rule(Parser *p) @@ -26994,16 +27590,16 @@ invalid_arithmetic_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); - void *_tmp_153_var; + void *_tmp_155_var; Token * a; expr_ty b; expr_ty sum_var; if ( (sum_var = sum_rule(p)) // sum && - (_tmp_153_var = _tmp_153_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + (_tmp_155_var = _tmp_155_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && - (a = _PyPegen_expect_token(p, 703)) // token='not' + (a = _PyPegen_expect_token(p, 712)) // token='not' && (b = inversion_rule(p)) // inversion ) @@ -27046,13 +27642,13 @@ invalid_factor_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); - void *_tmp_154_var; + void *_tmp_156_var; Token * a; expr_ty b; if ( - (_tmp_154_var = _tmp_154_rule(p)) // '+' | '-' | '~' + (_tmp_156_var = _tmp_156_rule(p)) // '+' | '-' | '~' && - (a = _PyPegen_expect_token(p, 703)) // token='not' + (a = _PyPegen_expect_token(p, 712)) // token='not' && (b = factor_rule(p)) // factor ) @@ -27399,7 +27995,7 @@ _tmp_5_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 634)) // token='import' + (_keyword = _PyPegen_expect_token(p, 643)) // token='import' ) { D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import'")); @@ -27418,7 +28014,7 @@ _tmp_5_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' + (_keyword = _PyPegen_expect_token(p, 642)) // token='from' ) { D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from'")); @@ -27456,7 +28052,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 699)) // token='def' + (_keyword = _PyPegen_expect_token(p, 708)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -27494,7 +28090,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 707)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27532,7 +28128,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 701)) // token='class' + (_keyword = _PyPegen_expect_token(p, 710)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -27589,7 +28185,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 647)) // token='with' + (_keyword = _PyPegen_expect_token(p, 656)) // token='with' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with'")); @@ -27608,7 +28204,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 707)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27646,7 +28242,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 694)) // token='for' + (_keyword = _PyPegen_expect_token(p, 703)) // token='for' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); @@ -27665,7 +28261,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='async' + (_keyword = _PyPegen_expect_token(p, 707)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27824,12 +28420,12 @@ _loop1_12_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_12[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_155_var; + void *_tmp_157_var; while ( - (_tmp_155_var = _tmp_155_rule(p)) // star_targets '=' + (_tmp_157_var = _tmp_157_rule(p)) // star_targets '=' ) { - _res = _tmp_155_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27868,55 +28464,9 @@ _loop1_12_rule(Parser *p) return _seq; } -// _tmp_13: 'from' expression -static void * -_tmp_13_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'from' expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_13[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from' expression")); - Token * _keyword; - expr_ty z; - if ( - (_keyword = _PyPegen_expect_token(p, 633)) // token='from' - && - (z = expression_rule(p)) // expression - ) - { - D(fprintf(stderr, "%*c+ _tmp_13[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from' expression")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_13[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'from' expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_14: ',' NAME +// _loop0_13: ',' NAME static asdl_seq * -_loop0_14_rule(Parser *p) +_loop0_13_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -27941,7 +28491,7 @@ _loop0_14_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' NAME")); + D(fprintf(stderr, "%*c> _loop0_13[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' NAME")); Token * _literal; expr_ty elem; while ( @@ -27973,7 +28523,7 @@ _loop0_14_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_14[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_13[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' NAME")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -27990,9 +28540,9 @@ _loop0_14_rule(Parser *p) return _seq; } -// _gather_15: NAME _loop0_14 +// _gather_14: NAME _loop0_13 static asdl_seq * -_gather_15_rule(Parser *p) +_gather_14_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28003,27 +28553,27 @@ _gather_15_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // NAME _loop0_14 + { // NAME _loop0_13 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME _loop0_14")); + D(fprintf(stderr, "%*c> _gather_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME _loop0_13")); expr_ty elem; asdl_seq * seq; if ( (elem = _PyPegen_name_token(p)) // NAME && - (seq = _loop0_14_rule(p)) // _loop0_14 + (seq = _loop0_13_rule(p)) // _loop0_13 ) { - D(fprintf(stderr, "%*c+ _gather_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME _loop0_14")); + D(fprintf(stderr, "%*c+ _gather_14[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME _loop0_13")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_15[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME _loop0_14")); + D(fprintf(stderr, "%*c%s _gather_14[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME _loop0_13")); } _res = NULL; done: @@ -28031,9 +28581,9 @@ _gather_15_rule(Parser *p) return _res; } -// _tmp_16: ';' | NEWLINE +// _tmp_15: ';' | NEWLINE static void * -_tmp_16_rule(Parser *p) +_tmp_15_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28049,18 +28599,18 @@ _tmp_16_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';'")); + D(fprintf(stderr, "%*c> _tmp_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 13)) // token=';' ) { - D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'")); + D(fprintf(stderr, "%*c+ _tmp_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_15[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "';'")); } { // NEWLINE @@ -28068,18 +28618,18 @@ _tmp_16_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_15[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } _res = NULL; @@ -28088,9 +28638,9 @@ _tmp_16_rule(Parser *p) return _res; } -// _tmp_17: ',' expression +// _tmp_16: ',' expression static void * -_tmp_17_rule(Parser *p) +_tmp_16_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28106,7 +28656,7 @@ _tmp_17_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_17[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty z; if ( @@ -28115,7 +28665,7 @@ _tmp_17_rule(Parser *p) (z = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_17[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -28125,7 +28675,7 @@ _tmp_17_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_17[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -28134,9 +28684,9 @@ _tmp_17_rule(Parser *p) return _res; } -// _loop0_18: ('.' | '...') +// _loop0_17: ('.' | '...') static asdl_seq * -_loop0_18_rule(Parser *p) +_loop0_17_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28161,13 +28711,13 @@ _loop0_18_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_18[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_156_var; + D(fprintf(stderr, "%*c> _loop0_17[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); + void *_tmp_158_var; while ( - (_tmp_156_var = _tmp_156_rule(p)) // '.' | '...' + (_tmp_158_var = _tmp_158_rule(p)) // '.' | '...' ) { - _res = _tmp_156_var; + _res = _tmp_158_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28184,7 +28734,7 @@ _loop0_18_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_18[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_17[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28201,9 +28751,9 @@ _loop0_18_rule(Parser *p) return _seq; } -// _loop1_19: ('.' | '...') +// _loop1_18: ('.' | '...') static asdl_seq * -_loop1_19_rule(Parser *p) +_loop1_18_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28228,13 +28778,13 @@ _loop1_19_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_19[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_156_var; + D(fprintf(stderr, "%*c> _loop1_18[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); + void *_tmp_158_var; while ( - (_tmp_156_var = _tmp_156_rule(p)) // '.' | '...' + (_tmp_158_var = _tmp_158_rule(p)) // '.' | '...' ) { - _res = _tmp_156_var; + _res = _tmp_158_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28251,7 +28801,7 @@ _loop1_19_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_19[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_18[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); } if (_n == 0 || p->error_indicator) { @@ -28273,9 +28823,9 @@ _loop1_19_rule(Parser *p) return _seq; } -// _loop0_20: ',' import_from_as_name +// _loop0_19: ',' import_from_as_name static asdl_seq * -_loop0_20_rule(Parser *p) +_loop0_19_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28300,7 +28850,7 @@ _loop0_20_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_20[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' import_from_as_name")); + D(fprintf(stderr, "%*c> _loop0_19[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' import_from_as_name")); Token * _literal; alias_ty elem; while ( @@ -28332,7 +28882,7 @@ _loop0_20_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_20[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_19[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' import_from_as_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28349,9 +28899,9 @@ _loop0_20_rule(Parser *p) return _seq; } -// _gather_21: import_from_as_name _loop0_20 +// _gather_20: import_from_as_name _loop0_19 static asdl_seq * -_gather_21_rule(Parser *p) +_gather_20_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28362,27 +28912,27 @@ _gather_21_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // import_from_as_name _loop0_20 + { // import_from_as_name _loop0_19 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_21[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_20")); + D(fprintf(stderr, "%*c> _gather_20[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_19")); alias_ty elem; asdl_seq * seq; if ( (elem = import_from_as_name_rule(p)) // import_from_as_name && - (seq = _loop0_20_rule(p)) // _loop0_20 + (seq = _loop0_19_rule(p)) // _loop0_19 ) { - D(fprintf(stderr, "%*c+ _gather_21[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_20")); + D(fprintf(stderr, "%*c+ _gather_20[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_19")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_21[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_name _loop0_20")); + D(fprintf(stderr, "%*c%s _gather_20[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_name _loop0_19")); } _res = NULL; done: @@ -28390,9 +28940,9 @@ _gather_21_rule(Parser *p) return _res; } -// _tmp_22: 'as' NAME +// _tmp_21: 'as' NAME static void * -_tmp_22_rule(Parser *p) +_tmp_21_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28408,16 +28958,16 @@ _tmp_22_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_22[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_21[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_22[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_21[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -28427,7 +28977,7 @@ _tmp_22_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_22[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_21[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -28436,9 +28986,9 @@ _tmp_22_rule(Parser *p) return _res; } -// _loop0_23: ',' dotted_as_name +// _loop0_22: ',' dotted_as_name static asdl_seq * -_loop0_23_rule(Parser *p) +_loop0_22_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28463,7 +29013,7 @@ _loop0_23_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_23[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_as_name")); + D(fprintf(stderr, "%*c> _loop0_22[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_as_name")); Token * _literal; alias_ty elem; while ( @@ -28495,7 +29045,7 @@ _loop0_23_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_23[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_22[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_as_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28512,9 +29062,9 @@ _loop0_23_rule(Parser *p) return _seq; } -// _gather_24: dotted_as_name _loop0_23 +// _gather_23: dotted_as_name _loop0_22 static asdl_seq * -_gather_24_rule(Parser *p) +_gather_23_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28525,27 +29075,27 @@ _gather_24_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_as_name _loop0_23 + { // dotted_as_name _loop0_22 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_23")); + D(fprintf(stderr, "%*c> _gather_23[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_22")); alias_ty elem; asdl_seq * seq; if ( (elem = dotted_as_name_rule(p)) // dotted_as_name && - (seq = _loop0_23_rule(p)) // _loop0_23 + (seq = _loop0_22_rule(p)) // _loop0_22 ) { - D(fprintf(stderr, "%*c+ _gather_24[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_23")); + D(fprintf(stderr, "%*c+ _gather_23[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_22")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_24[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_as_name _loop0_23")); + D(fprintf(stderr, "%*c%s _gather_23[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_as_name _loop0_22")); } _res = NULL; done: @@ -28553,9 +29103,9 @@ _gather_24_rule(Parser *p) return _res; } -// _loop1_25: ('@' named_expression NEWLINE) +// _loop1_24: ('@' named_expression NEWLINE) static asdl_seq * -_loop1_25_rule(Parser *p) +_loop1_24_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28580,13 +29130,13 @@ _loop1_25_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_157_var; + D(fprintf(stderr, "%*c> _loop1_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); + void *_tmp_159_var; while ( - (_tmp_157_var = _tmp_157_rule(p)) // '@' named_expression NEWLINE + (_tmp_159_var = _tmp_159_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_157_var; + _res = _tmp_159_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28603,7 +29153,7 @@ _loop1_25_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_25[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_24[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('@' named_expression NEWLINE)")); } if (_n == 0 || p->error_indicator) { @@ -28625,9 +29175,9 @@ _loop1_25_rule(Parser *p) return _seq; } -// _tmp_26: '(' arguments? ')' +// _tmp_25: '(' arguments? ')' static void * -_tmp_26_rule(Parser *p) +_tmp_25_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28643,7 +29193,7 @@ _tmp_26_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *z; @@ -28655,7 +29205,7 @@ _tmp_26_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_26[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_25[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -28665,7 +29215,7 @@ _tmp_26_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_26[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_25[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -28674,9 +29224,9 @@ _tmp_26_rule(Parser *p) return _res; } -// _tmp_27: '->' expression +// _tmp_26: '->' expression static void * -_tmp_27_rule(Parser *p) +_tmp_26_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28692,7 +29242,7 @@ _tmp_27_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_27[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty z; if ( @@ -28701,7 +29251,7 @@ _tmp_27_rule(Parser *p) (z = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_27[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_26[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -28711,7 +29261,7 @@ _tmp_27_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_27[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_26[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -28720,76 +29270,76 @@ _tmp_27_rule(Parser *p) return _res; } -// _loop0_28: param_no_default +// _loop0_27: param_no_default +static asdl_seq * +_loop0_27_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // param_no_default + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_27[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + arg_ty param_no_default_var; + while ( + (param_no_default_var = param_no_default_rule(p)) // param_no_default + ) + { + _res = param_no_default_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_27[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop0_28: param_with_default static asdl_seq * _loop0_28_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_28[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default - ) - { - _res = param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_28[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_29: param_with_default -static asdl_seq * -_loop0_29_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28814,7 +29364,7 @@ _loop0_29_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_29[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop0_28[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -28837,7 +29387,7 @@ _loop0_29_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_29[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_28[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28854,81 +29404,81 @@ _loop0_29_rule(Parser *p) return _seq; } -// _loop1_30: param_no_default +// _loop1_29: param_no_default +static asdl_seq * +_loop1_29_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // param_no_default + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_29[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + arg_ty param_no_default_var; + while ( + (param_no_default_var = param_no_default_rule(p)) // param_no_default + ) + { + _res = param_no_default_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_29[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop1_30: param_with_default static asdl_seq * _loop1_30_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_30[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); - arg_ty param_no_default_var; - while ( - (param_no_default_var = param_no_default_rule(p)) // param_no_default - ) - { - _res = param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_30[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_31: param_with_default -static asdl_seq * -_loop1_31_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28953,7 +29503,7 @@ _loop1_31_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_31[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop1_30[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -28976,7 +29526,7 @@ _loop1_31_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_31[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_30[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -28998,9 +29548,9 @@ _loop1_31_rule(Parser *p) return _seq; } -// _loop0_32: param_maybe_default +// _loop0_31: param_maybe_default static asdl_seq * -_loop0_32_rule(Parser *p) +_loop0_31_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29025,7 +29575,7 @@ _loop0_32_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_31[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -29048,7 +29598,7 @@ _loop0_32_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_32[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_31[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -29065,9 +29615,9 @@ _loop0_32_rule(Parser *p) return _seq; } -// _loop1_33: param_maybe_default +// _loop1_32: param_maybe_default static asdl_seq * -_loop1_33_rule(Parser *p) +_loop1_32_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29092,7 +29642,7 @@ _loop1_33_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -29115,7 +29665,7 @@ _loop1_33_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_33[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_32[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -29137,9 +29687,9 @@ _loop1_33_rule(Parser *p) return _seq; } -// _loop0_34: ',' with_item +// _loop0_33: ',' with_item static asdl_seq * -_loop0_34_rule(Parser *p) +_loop0_33_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29164,7 +29714,7 @@ _loop0_34_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_34[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c> _loop0_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); Token * _literal; withitem_ty elem; while ( @@ -29196,7 +29746,7 @@ _loop0_34_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_34[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_33[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' with_item")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -29213,9 +29763,9 @@ _loop0_34_rule(Parser *p) return _seq; } -// _gather_35: with_item _loop0_34 +// _gather_34: with_item _loop0_33 static asdl_seq * -_gather_35_rule(Parser *p) +_gather_34_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29226,27 +29776,27 @@ _gather_35_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // with_item _loop0_34 + { // with_item _loop0_33 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_34")); + D(fprintf(stderr, "%*c> _gather_34[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_33")); withitem_ty elem; asdl_seq * seq; if ( (elem = with_item_rule(p)) // with_item && - (seq = _loop0_34_rule(p)) // _loop0_34 + (seq = _loop0_33_rule(p)) // _loop0_33 ) { - D(fprintf(stderr, "%*c+ _gather_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_34")); + D(fprintf(stderr, "%*c+ _gather_34[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_33")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_35[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_34")); + D(fprintf(stderr, "%*c%s _gather_34[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_33")); } _res = NULL; done: @@ -29254,9 +29804,9 @@ _gather_35_rule(Parser *p) return _res; } -// _tmp_36: ',' | ')' | ':' +// _tmp_35: ',' | ')' | ':' static void * -_tmp_36_rule(Parser *p) +_tmp_35_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29272,18 +29822,18 @@ _tmp_36_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_35[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -29291,18 +29841,18 @@ _tmp_36_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_35[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ':' @@ -29310,18 +29860,18 @@ _tmp_36_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_35[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -29330,9 +29880,9 @@ _tmp_36_rule(Parser *p) return _res; } -// _loop1_37: except_block +// _loop1_36: except_block static asdl_seq * -_loop1_37_rule(Parser *p) +_loop1_36_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29357,7 +29907,7 @@ _loop1_37_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_37[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + D(fprintf(stderr, "%*c> _loop1_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); excepthandler_ty except_block_var; while ( (except_block_var = except_block_rule(p)) // except_block @@ -29380,7 +29930,7 @@ _loop1_37_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_37[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_36[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); } if (_n == 0 || p->error_indicator) { @@ -29402,9 +29952,9 @@ _loop1_37_rule(Parser *p) return _seq; } -// _loop1_38: except_star_block +// _loop1_37: except_star_block static asdl_seq * -_loop1_38_rule(Parser *p) +_loop1_37_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29429,7 +29979,7 @@ _loop1_38_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_38[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c> _loop1_37[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); excepthandler_ty except_star_block_var; while ( (except_star_block_var = except_star_block_rule(p)) // except_star_block @@ -29452,7 +30002,7 @@ _loop1_38_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_38[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_37[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); } if (_n == 0 || p->error_indicator) { @@ -29474,9 +30024,9 @@ _loop1_38_rule(Parser *p) return _seq; } -// _loop1_39: case_block +// _loop1_38: case_block static asdl_seq * -_loop1_39_rule(Parser *p) +_loop1_38_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29501,7 +30051,7 @@ _loop1_39_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_39[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "case_block")); + D(fprintf(stderr, "%*c> _loop1_38[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "case_block")); match_case_ty case_block_var; while ( (case_block_var = case_block_rule(p)) // case_block @@ -29524,7 +30074,7 @@ _loop1_39_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_39[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_38[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "case_block")); } if (_n == 0 || p->error_indicator) { @@ -29546,9 +30096,9 @@ _loop1_39_rule(Parser *p) return _seq; } -// _loop0_40: '|' closed_pattern +// _loop0_39: '|' closed_pattern static asdl_seq * -_loop0_40_rule(Parser *p) +_loop0_39_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29573,7 +30123,7 @@ _loop0_40_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_40[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|' closed_pattern")); + D(fprintf(stderr, "%*c> _loop0_39[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|' closed_pattern")); Token * _literal; pattern_ty elem; while ( @@ -29605,7 +30155,7 @@ _loop0_40_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_40[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_39[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'|' closed_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -29622,9 +30172,9 @@ _loop0_40_rule(Parser *p) return _seq; } -// _gather_41: closed_pattern _loop0_40 +// _gather_40: closed_pattern _loop0_39 static asdl_seq * -_gather_41_rule(Parser *p) +_gather_40_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29635,27 +30185,27 @@ _gather_41_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // closed_pattern _loop0_40 + { // closed_pattern _loop0_39 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_41[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_40")); + D(fprintf(stderr, "%*c> _gather_40[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_39")); pattern_ty elem; asdl_seq * seq; if ( (elem = closed_pattern_rule(p)) // closed_pattern && - (seq = _loop0_40_rule(p)) // _loop0_40 + (seq = _loop0_39_rule(p)) // _loop0_39 ) { - D(fprintf(stderr, "%*c+ _gather_41[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_40")); + D(fprintf(stderr, "%*c+ _gather_40[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_39")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_41[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "closed_pattern _loop0_40")); + D(fprintf(stderr, "%*c%s _gather_40[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "closed_pattern _loop0_39")); } _res = NULL; done: @@ -29663,9 +30213,9 @@ _gather_41_rule(Parser *p) return _res; } -// _tmp_42: '+' | '-' +// _tmp_41: '+' | '-' static void * -_tmp_42_rule(Parser *p) +_tmp_41_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29681,18 +30231,18 @@ _tmp_42_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_41[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_41[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_41[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -29700,18 +30250,18 @@ _tmp_42_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_41[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_41[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_41[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } _res = NULL; @@ -29720,9 +30270,9 @@ _tmp_42_rule(Parser *p) return _res; } -// _tmp_43: STRING | FSTRING_START | TSTRING_START +// _tmp_42: STRING | FSTRING_START | TSTRING_START static void * -_tmp_43_rule(Parser *p) +_tmp_42_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29738,18 +30288,18 @@ _tmp_43_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "STRING")); + D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "STRING")); expr_ty string_var; if ( (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "STRING")); + D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "STRING")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "STRING")); } { // FSTRING_START @@ -29757,18 +30307,18 @@ _tmp_43_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); + D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); Token * fstring_start_var; if ( (fstring_start_var = _PyPegen_expect_token(p, FSTRING_START)) // token='FSTRING_START' ) { - D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); + D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); _res = fstring_start_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "FSTRING_START")); } { // TSTRING_START @@ -29776,18 +30326,18 @@ _tmp_43_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "TSTRING_START")); + D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "TSTRING_START")); Token * tstring_start_var; if ( (tstring_start_var = _PyPegen_expect_token(p, TSTRING_START)) // token='TSTRING_START' ) { - D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "TSTRING_START")); + D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "TSTRING_START")); _res = tstring_start_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "TSTRING_START")); } _res = NULL; @@ -29796,9 +30346,9 @@ _tmp_43_rule(Parser *p) return _res; } -// _tmp_44: '.' | '(' | '=' +// _tmp_43: '.' | '(' | '=' static void * -_tmp_44_rule(Parser *p) +_tmp_43_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29814,18 +30364,18 @@ _tmp_44_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_44[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_44[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '(' @@ -29833,18 +30383,18 @@ _tmp_44_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_44[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_44[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '=' @@ -29852,18 +30402,18 @@ _tmp_44_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_44[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_44[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } _res = NULL; @@ -29872,9 +30422,9 @@ _tmp_44_rule(Parser *p) return _res; } -// _loop0_45: ',' maybe_star_pattern +// _loop0_44: ',' maybe_star_pattern static asdl_seq * -_loop0_45_rule(Parser *p) +_loop0_44_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29899,7 +30449,7 @@ _loop0_45_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_45[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' maybe_star_pattern")); + D(fprintf(stderr, "%*c> _loop0_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' maybe_star_pattern")); Token * _literal; pattern_ty elem; while ( @@ -29931,7 +30481,7 @@ _loop0_45_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_45[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_44[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' maybe_star_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -29948,9 +30498,9 @@ _loop0_45_rule(Parser *p) return _seq; } -// _gather_46: maybe_star_pattern _loop0_45 +// _gather_45: maybe_star_pattern _loop0_44 static asdl_seq * -_gather_46_rule(Parser *p) +_gather_45_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29961,27 +30511,27 @@ _gather_46_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // maybe_star_pattern _loop0_45 + { // maybe_star_pattern _loop0_44 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_46[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_45")); + D(fprintf(stderr, "%*c> _gather_45[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_44")); pattern_ty elem; asdl_seq * seq; if ( (elem = maybe_star_pattern_rule(p)) // maybe_star_pattern && - (seq = _loop0_45_rule(p)) // _loop0_45 + (seq = _loop0_44_rule(p)) // _loop0_44 ) { - D(fprintf(stderr, "%*c+ _gather_46[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_45")); + D(fprintf(stderr, "%*c+ _gather_45[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_44")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_46[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "maybe_star_pattern _loop0_45")); + D(fprintf(stderr, "%*c%s _gather_45[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "maybe_star_pattern _loop0_44")); } _res = NULL; done: @@ -29989,9 +30539,9 @@ _gather_46_rule(Parser *p) return _res; } -// _loop0_47: ',' key_value_pattern +// _loop0_46: ',' key_value_pattern static asdl_seq * -_loop0_47_rule(Parser *p) +_loop0_46_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30016,7 +30566,7 @@ _loop0_47_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_47[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' key_value_pattern")); + D(fprintf(stderr, "%*c> _loop0_46[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' key_value_pattern")); Token * _literal; KeyPatternPair* elem; while ( @@ -30048,7 +30598,7 @@ _loop0_47_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_47[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_46[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' key_value_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30065,9 +30615,9 @@ _loop0_47_rule(Parser *p) return _seq; } -// _gather_48: key_value_pattern _loop0_47 +// _gather_47: key_value_pattern _loop0_46 static asdl_seq * -_gather_48_rule(Parser *p) +_gather_47_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30078,27 +30628,27 @@ _gather_48_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // key_value_pattern _loop0_47 + { // key_value_pattern _loop0_46 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_47")); + D(fprintf(stderr, "%*c> _gather_47[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_46")); KeyPatternPair* elem; asdl_seq * seq; if ( (elem = key_value_pattern_rule(p)) // key_value_pattern && - (seq = _loop0_47_rule(p)) // _loop0_47 + (seq = _loop0_46_rule(p)) // _loop0_46 ) { - D(fprintf(stderr, "%*c+ _gather_48[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_47")); + D(fprintf(stderr, "%*c+ _gather_47[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_46")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_48[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "key_value_pattern _loop0_47")); + D(fprintf(stderr, "%*c%s _gather_47[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "key_value_pattern _loop0_46")); } _res = NULL; done: @@ -30106,9 +30656,9 @@ _gather_48_rule(Parser *p) return _res; } -// _tmp_49: literal_expr | attr +// _tmp_48: literal_expr | attr static void * -_tmp_49_rule(Parser *p) +_tmp_48_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30124,18 +30674,18 @@ _tmp_49_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_49[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_expr")); + D(fprintf(stderr, "%*c> _tmp_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_expr")); expr_ty literal_expr_var; if ( (literal_expr_var = literal_expr_rule(p)) // literal_expr ) { - D(fprintf(stderr, "%*c+ _tmp_49[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "literal_expr")); + D(fprintf(stderr, "%*c+ _tmp_48[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "literal_expr")); _res = literal_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_49[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_48[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "literal_expr")); } { // attr @@ -30143,18 +30693,18 @@ _tmp_49_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_49[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "attr")); + D(fprintf(stderr, "%*c> _tmp_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "attr")); expr_ty attr_var; if ( (attr_var = attr_rule(p)) // attr ) { - D(fprintf(stderr, "%*c+ _tmp_49[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr")); + D(fprintf(stderr, "%*c+ _tmp_48[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr")); _res = attr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_49[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_48[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "attr")); } _res = NULL; @@ -30163,9 +30713,9 @@ _tmp_49_rule(Parser *p) return _res; } -// _loop0_50: ',' pattern +// _loop0_49: ',' pattern static asdl_seq * -_loop0_50_rule(Parser *p) +_loop0_49_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30190,7 +30740,7 @@ _loop0_50_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_50[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' pattern")); + D(fprintf(stderr, "%*c> _loop0_49[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' pattern")); Token * _literal; pattern_ty elem; while ( @@ -30222,7 +30772,7 @@ _loop0_50_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_50[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_49[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30239,9 +30789,9 @@ _loop0_50_rule(Parser *p) return _seq; } -// _gather_51: pattern _loop0_50 +// _gather_50: pattern _loop0_49 static asdl_seq * -_gather_51_rule(Parser *p) +_gather_50_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30252,27 +30802,27 @@ _gather_51_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // pattern _loop0_50 + { // pattern _loop0_49 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_51[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern _loop0_50")); + D(fprintf(stderr, "%*c> _gather_50[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern _loop0_49")); pattern_ty elem; asdl_seq * seq; if ( (elem = pattern_rule(p)) // pattern && - (seq = _loop0_50_rule(p)) // _loop0_50 + (seq = _loop0_49_rule(p)) // _loop0_49 ) { - D(fprintf(stderr, "%*c+ _gather_51[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pattern _loop0_50")); + D(fprintf(stderr, "%*c+ _gather_50[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pattern _loop0_49")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_51[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pattern _loop0_50")); + D(fprintf(stderr, "%*c%s _gather_50[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pattern _loop0_49")); } _res = NULL; done: @@ -30280,9 +30830,9 @@ _gather_51_rule(Parser *p) return _res; } -// _loop0_52: ',' keyword_pattern +// _loop0_51: ',' keyword_pattern static asdl_seq * -_loop0_52_rule(Parser *p) +_loop0_51_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30307,7 +30857,7 @@ _loop0_52_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_52[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' keyword_pattern")); + D(fprintf(stderr, "%*c> _loop0_51[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' keyword_pattern")); Token * _literal; KeyPatternPair* elem; while ( @@ -30339,7 +30889,7 @@ _loop0_52_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_52[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_51[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' keyword_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30356,9 +30906,9 @@ _loop0_52_rule(Parser *p) return _seq; } -// _gather_53: keyword_pattern _loop0_52 +// _gather_52: keyword_pattern _loop0_51 static asdl_seq * -_gather_53_rule(Parser *p) +_gather_52_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30369,27 +30919,27 @@ _gather_53_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // keyword_pattern _loop0_52 + { // keyword_pattern _loop0_51 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_53[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_52")); + D(fprintf(stderr, "%*c> _gather_52[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_51")); KeyPatternPair* elem; asdl_seq * seq; if ( (elem = keyword_pattern_rule(p)) // keyword_pattern && - (seq = _loop0_52_rule(p)) // _loop0_52 + (seq = _loop0_51_rule(p)) // _loop0_51 ) { - D(fprintf(stderr, "%*c+ _gather_53[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_52")); + D(fprintf(stderr, "%*c+ _gather_52[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_51")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_53[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "keyword_pattern _loop0_52")); + D(fprintf(stderr, "%*c%s _gather_52[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "keyword_pattern _loop0_51")); } _res = NULL; done: @@ -30397,9 +30947,9 @@ _gather_53_rule(Parser *p) return _res; } -// _loop0_54: ',' type_param +// _loop0_53: ',' type_param static asdl_seq * -_loop0_54_rule(Parser *p) +_loop0_53_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30424,7 +30974,7 @@ _loop0_54_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' type_param")); + D(fprintf(stderr, "%*c> _loop0_53[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' type_param")); Token * _literal; type_param_ty elem; while ( @@ -30456,7 +31006,7 @@ _loop0_54_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_54[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_53[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' type_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30473,9 +31023,9 @@ _loop0_54_rule(Parser *p) return _seq; } -// _gather_55: type_param _loop0_54 +// _gather_54: type_param _loop0_53 static asdl_seq * -_gather_55_rule(Parser *p) +_gather_54_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30486,27 +31036,27 @@ _gather_55_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // type_param _loop0_54 + { // type_param _loop0_53 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "type_param _loop0_54")); + D(fprintf(stderr, "%*c> _gather_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "type_param _loop0_53")); type_param_ty elem; asdl_seq * seq; if ( (elem = type_param_rule(p)) // type_param && - (seq = _loop0_54_rule(p)) // _loop0_54 + (seq = _loop0_53_rule(p)) // _loop0_53 ) { - D(fprintf(stderr, "%*c+ _gather_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "type_param _loop0_54")); + D(fprintf(stderr, "%*c+ _gather_54[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "type_param _loop0_53")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_55[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "type_param _loop0_54")); + D(fprintf(stderr, "%*c%s _gather_54[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "type_param _loop0_53")); } _res = NULL; done: @@ -30514,9 +31064,9 @@ _gather_55_rule(Parser *p) return _res; } -// _loop1_56: (',' expression) +// _loop1_55: (',' expression) static asdl_seq * -_loop1_56_rule(Parser *p) +_loop1_55_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30541,13 +31091,13 @@ _loop1_56_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_17_var; + D(fprintf(stderr, "%*c> _loop1_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); + void *_tmp_16_var; while ( - (_tmp_17_var = _tmp_17_rule(p)) // ',' expression + (_tmp_16_var = _tmp_16_rule(p)) // ',' expression ) { - _res = _tmp_17_var; + _res = _tmp_16_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30564,7 +31114,7 @@ _loop1_56_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_56[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_55[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' expression)")); } if (_n == 0 || p->error_indicator) { @@ -30586,9 +31136,9 @@ _loop1_56_rule(Parser *p) return _seq; } -// _loop1_57: (',' star_expression) +// _loop1_56: (',' star_expression) static asdl_seq * -_loop1_57_rule(Parser *p) +_loop1_56_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30613,13 +31163,13 @@ _loop1_57_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_158_var; + D(fprintf(stderr, "%*c> _loop1_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); + void *_tmp_160_var; while ( - (_tmp_158_var = _tmp_158_rule(p)) // ',' star_expression + (_tmp_160_var = _tmp_160_rule(p)) // ',' star_expression ) { - _res = _tmp_158_var; + _res = _tmp_160_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30636,7 +31186,7 @@ _loop1_57_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_57[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_56[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_expression)")); } if (_n == 0 || p->error_indicator) { @@ -30658,9 +31208,9 @@ _loop1_57_rule(Parser *p) return _seq; } -// _loop0_58: ',' star_named_expression +// _loop0_57: ',' star_named_expression static asdl_seq * -_loop0_58_rule(Parser *p) +_loop0_57_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30685,7 +31235,7 @@ _loop0_58_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); + D(fprintf(stderr, "%*c> _loop0_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); Token * _literal; expr_ty elem; while ( @@ -30717,7 +31267,7 @@ _loop0_58_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_58[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_57[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_named_expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30734,9 +31284,9 @@ _loop0_58_rule(Parser *p) return _seq; } -// _gather_59: star_named_expression _loop0_58 +// _gather_58: star_named_expression _loop0_57 static asdl_seq * -_gather_59_rule(Parser *p) +_gather_58_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30747,27 +31297,27 @@ _gather_59_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // star_named_expression _loop0_58 + { // star_named_expression _loop0_57 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_58")); + D(fprintf(stderr, "%*c> _gather_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_57")); expr_ty elem; asdl_seq * seq; if ( (elem = star_named_expression_rule(p)) // star_named_expression && - (seq = _loop0_58_rule(p)) // _loop0_58 + (seq = _loop0_57_rule(p)) // _loop0_57 ) { - D(fprintf(stderr, "%*c+ _gather_59[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_58")); + D(fprintf(stderr, "%*c+ _gather_58[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_57")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_59[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_58")); + D(fprintf(stderr, "%*c%s _gather_58[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_57")); } _res = NULL; done: @@ -30775,9 +31325,9 @@ _gather_59_rule(Parser *p) return _res; } -// _loop1_60: ('or' conjunction) +// _loop1_59: ('or' conjunction) static asdl_seq * -_loop1_60_rule(Parser *p) +_loop1_59_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30802,13 +31352,13 @@ _loop1_60_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_159_var; + D(fprintf(stderr, "%*c> _loop1_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); + void *_tmp_161_var; while ( - (_tmp_159_var = _tmp_159_rule(p)) // 'or' conjunction + (_tmp_161_var = _tmp_161_rule(p)) // 'or' conjunction ) { - _res = _tmp_159_var; + _res = _tmp_161_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30825,7 +31375,7 @@ _loop1_60_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_60[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_59[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('or' conjunction)")); } if (_n == 0 || p->error_indicator) { @@ -30847,9 +31397,9 @@ _loop1_60_rule(Parser *p) return _seq; } -// _loop1_61: ('and' inversion) +// _loop1_60: ('and' inversion) static asdl_seq * -_loop1_61_rule(Parser *p) +_loop1_60_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30874,13 +31424,13 @@ _loop1_61_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_160_var; + D(fprintf(stderr, "%*c> _loop1_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); + void *_tmp_162_var; while ( - (_tmp_160_var = _tmp_160_rule(p)) // 'and' inversion + (_tmp_162_var = _tmp_162_rule(p)) // 'and' inversion ) { - _res = _tmp_160_var; + _res = _tmp_162_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30897,7 +31447,7 @@ _loop1_61_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_61[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_60[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('and' inversion)")); } if (_n == 0 || p->error_indicator) { @@ -30919,9 +31469,9 @@ _loop1_61_rule(Parser *p) return _seq; } -// _loop1_62: compare_op_bitwise_or_pair +// _loop1_61: compare_op_bitwise_or_pair static asdl_seq * -_loop1_62_rule(Parser *p) +_loop1_61_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30946,7 +31496,7 @@ _loop1_62_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_62[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compare_op_bitwise_or_pair")); + D(fprintf(stderr, "%*c> _loop1_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compare_op_bitwise_or_pair")); CmpopExprPair* compare_op_bitwise_or_pair_var; while ( (compare_op_bitwise_or_pair_var = compare_op_bitwise_or_pair_rule(p)) // compare_op_bitwise_or_pair @@ -30969,7 +31519,7 @@ _loop1_62_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_62[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_61[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compare_op_bitwise_or_pair")); } if (_n == 0 || p->error_indicator) { @@ -30991,9 +31541,9 @@ _loop1_62_rule(Parser *p) return _seq; } -// _tmp_63: '!=' +// _tmp_62: '!=' static void * -_tmp_63_rule(Parser *p) +_tmp_62_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31009,13 +31559,13 @@ _tmp_63_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_63[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!='")); + D(fprintf(stderr, "%*c> _tmp_62[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!='")); Token * tok; if ( (tok = _PyPegen_expect_token(p, 28)) // token='!=' ) { - D(fprintf(stderr, "%*c+ _tmp_63[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!='")); + D(fprintf(stderr, "%*c+ _tmp_62[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!='")); _res = _PyPegen_check_barry_as_flufl ( p , tok ) ? NULL : tok; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -31025,7 +31575,7 @@ _tmp_63_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_63[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_62[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!='")); } _res = NULL; @@ -31034,9 +31584,9 @@ _tmp_63_rule(Parser *p) return _res; } -// _loop0_64: ',' (slice | starred_expression) +// _loop0_63: ',' (slice | starred_expression) static asdl_seq * -_loop0_64_rule(Parser *p) +_loop0_63_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31061,13 +31611,13 @@ _loop0_64_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_64[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (slice | starred_expression)")); + D(fprintf(stderr, "%*c> _loop0_63[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (slice | starred_expression)")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_161_rule(p)) // slice | starred_expression + (elem = _tmp_163_rule(p)) // slice | starred_expression ) { _res = elem; @@ -31093,7 +31643,7 @@ _loop0_64_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_64[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_63[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (slice | starred_expression)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -31110,9 +31660,9 @@ _loop0_64_rule(Parser *p) return _seq; } -// _gather_65: (slice | starred_expression) _loop0_64 +// _gather_64: (slice | starred_expression) _loop0_63 static asdl_seq * -_gather_65_rule(Parser *p) +_gather_64_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31123,27 +31673,27 @@ _gather_65_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (slice | starred_expression) _loop0_64 + { // (slice | starred_expression) _loop0_63 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_65[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_64")); + D(fprintf(stderr, "%*c> _gather_64[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_63")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_161_rule(p)) // slice | starred_expression + (elem = _tmp_163_rule(p)) // slice | starred_expression && - (seq = _loop0_64_rule(p)) // _loop0_64 + (seq = _loop0_63_rule(p)) // _loop0_63 ) { - D(fprintf(stderr, "%*c+ _gather_65[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_64")); + D(fprintf(stderr, "%*c+ _gather_64[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_63")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_65[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(slice | starred_expression) _loop0_64")); + D(fprintf(stderr, "%*c%s _gather_64[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(slice | starred_expression) _loop0_63")); } _res = NULL; done: @@ -31151,9 +31701,9 @@ _gather_65_rule(Parser *p) return _res; } -// _tmp_66: ':' expression? +// _tmp_65: ':' expression? static void * -_tmp_66_rule(Parser *p) +_tmp_65_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31169,7 +31719,7 @@ _tmp_66_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' expression?")); + D(fprintf(stderr, "%*c> _tmp_65[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' expression?")); Token * _literal; void *d; if ( @@ -31178,7 +31728,7 @@ _tmp_66_rule(Parser *p) (d = expression_rule(p), !p->error_indicator) // expression? ) { - D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); + D(fprintf(stderr, "%*c+ _tmp_65[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); _res = d; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -31188,7 +31738,7 @@ _tmp_66_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_65[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':' expression?")); } _res = NULL; @@ -31197,9 +31747,9 @@ _tmp_66_rule(Parser *p) return _res; } -// _tmp_67: tuple | group | genexp +// _tmp_66: tuple | group | genexp static void * -_tmp_67_rule(Parser *p) +_tmp_66_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31215,18 +31765,18 @@ _tmp_67_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // group @@ -31234,18 +31784,18 @@ _tmp_67_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "group")); + D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "group")); expr_ty group_var; if ( (group_var = group_rule(p)) // group ) { - D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "group")); + D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "group")); _res = group_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "group")); } { // genexp @@ -31253,18 +31803,18 @@ _tmp_67_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } _res = NULL; @@ -31273,9 +31823,9 @@ _tmp_67_rule(Parser *p) return _res; } -// _tmp_68: list | listcomp +// _tmp_67: list | listcomp static void * -_tmp_68_rule(Parser *p) +_tmp_67_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31291,18 +31841,18 @@ _tmp_68_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // listcomp @@ -31310,18 +31860,18 @@ _tmp_68_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "listcomp")); + D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "listcomp")); expr_ty listcomp_var; if ( (listcomp_var = listcomp_rule(p)) // listcomp ) { - D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "listcomp")); + D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "listcomp")); _res = listcomp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "listcomp")); } _res = NULL; @@ -31330,9 +31880,9 @@ _tmp_68_rule(Parser *p) return _res; } -// _tmp_69: dict | set | dictcomp | setcomp +// _tmp_68: dict | set | dictcomp | setcomp static void * -_tmp_69_rule(Parser *p) +_tmp_68_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31348,18 +31898,18 @@ _tmp_69_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dict")); + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dict")); expr_ty dict_var; if ( (dict_var = dict_rule(p)) // dict ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dict")); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dict")); _res = dict_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dict")); } { // set @@ -31367,18 +31917,18 @@ _tmp_69_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "set")); + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "set")); expr_ty set_var; if ( (set_var = set_rule(p)) // set ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "set")); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "set")); _res = set_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "set")); } { // dictcomp @@ -31386,18 +31936,18 @@ _tmp_69_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dictcomp")); + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dictcomp")); expr_ty dictcomp_var; if ( (dictcomp_var = dictcomp_rule(p)) // dictcomp ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dictcomp")); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dictcomp")); _res = dictcomp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dictcomp")); } { // setcomp @@ -31405,18 +31955,18 @@ _tmp_69_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "setcomp")); + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "setcomp")); expr_ty setcomp_var; if ( (setcomp_var = setcomp_rule(p)) // setcomp ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "setcomp")); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "setcomp")); _res = setcomp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "setcomp")); } _res = NULL; @@ -31425,9 +31975,9 @@ _tmp_69_rule(Parser *p) return _res; } -// _tmp_70: yield_expr | named_expression +// _tmp_69: yield_expr | named_expression static void * -_tmp_70_rule(Parser *p) +_tmp_69_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31443,18 +31993,18 @@ _tmp_70_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // named_expression @@ -31462,18 +32012,18 @@ _tmp_70_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "named_expression")); + D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "named_expression")); expr_ty named_expression_var; if ( (named_expression_var = named_expression_rule(p)) // named_expression ) { - D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "named_expression")); + D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "named_expression")); _res = named_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "named_expression")); } _res = NULL; @@ -31482,76 +32032,76 @@ _tmp_70_rule(Parser *p) return _res; } -// _loop0_71: lambda_param_no_default +// _loop0_70: lambda_param_no_default +static asdl_seq * +_loop0_70_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // lambda_param_no_default + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop0_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + arg_ty lambda_param_no_default_var; + while ( + (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default + ) + { + _res = lambda_param_no_default_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop0_70[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop0_71: lambda_param_with_default static asdl_seq * _loop0_71_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop0_71[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default - ) - { - _res = lambda_param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_71[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop0_72: lambda_param_with_default -static asdl_seq * -_loop0_72_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31576,7 +32126,7 @@ _loop0_72_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_72[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop0_71[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -31599,7 +32149,7 @@ _loop0_72_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_72[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_71[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -31616,81 +32166,81 @@ _loop0_72_rule(Parser *p) return _seq; } -// _loop1_73: lambda_param_no_default +// _loop1_72: lambda_param_no_default +static asdl_seq * +_loop1_72_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // lambda_param_no_default + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_72[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + arg_ty lambda_param_no_default_var; + while ( + (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default + ) + { + _res = lambda_param_no_default_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_72[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop1_73: lambda_param_with_default static asdl_seq * _loop1_73_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void *_res = NULL; - int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; - Py_ssize_t _n = 0; - { // lambda_param_no_default - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _loop1_73[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); - arg_ty lambda_param_no_default_var; - while ( - (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default - ) - { - _res = lambda_param_no_default_var; - if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - _children = _new_children; - } - _children[_n++] = _res; - _mark = p->mark; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_73[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); - } - if (_n == 0 || p->error_indicator) { - PyMem_Free(_children); - p->level--; - return NULL; - } - asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); - if (!_seq) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); - PyMem_Free(_children); - p->level--; - return _seq; -} - -// _loop1_74: lambda_param_with_default -static asdl_seq * -_loop1_74_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31715,7 +32265,7 @@ _loop1_74_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_74[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop1_73[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -31738,7 +32288,7 @@ _loop1_74_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_74[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_73[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -31760,9 +32310,9 @@ _loop1_74_rule(Parser *p) return _seq; } -// _loop0_75: lambda_param_maybe_default +// _loop0_74: lambda_param_maybe_default static asdl_seq * -_loop0_75_rule(Parser *p) +_loop0_74_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31787,7 +32337,7 @@ _loop0_75_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_75[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_74[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -31810,7 +32360,7 @@ _loop0_75_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_75[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_74[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -31827,9 +32377,9 @@ _loop0_75_rule(Parser *p) return _seq; } -// _loop1_76: lambda_param_maybe_default +// _loop1_75: lambda_param_maybe_default static asdl_seq * -_loop1_76_rule(Parser *p) +_loop1_75_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31854,7 +32404,7 @@ _loop1_76_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_76[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_75[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -31877,7 +32427,7 @@ _loop1_76_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_76[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_75[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -31899,9 +32449,9 @@ _loop1_76_rule(Parser *p) return _seq; } -// _loop0_77: fstring_format_spec +// _loop0_76: fstring_format_spec static asdl_seq * -_loop0_77_rule(Parser *p) +_loop0_76_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31926,7 +32476,7 @@ _loop0_77_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_77[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_76[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -31949,7 +32499,7 @@ _loop0_77_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_77[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_76[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -31966,9 +32516,9 @@ _loop0_77_rule(Parser *p) return _seq; } -// _loop0_78: fstring_middle +// _loop0_77: fstring_middle static asdl_seq * -_loop0_78_rule(Parser *p) +_loop0_77_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31993,7 +32543,7 @@ _loop0_78_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_78[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_middle")); + D(fprintf(stderr, "%*c> _loop0_77[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_middle")); expr_ty fstring_middle_var; while ( (fstring_middle_var = fstring_middle_rule(p)) // fstring_middle @@ -32016,7 +32566,7 @@ _loop0_78_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_78[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_77[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_middle")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32033,9 +32583,9 @@ _loop0_78_rule(Parser *p) return _seq; } -// _loop0_79: tstring_format_spec +// _loop0_78: tstring_format_spec static asdl_seq * -_loop0_79_rule(Parser *p) +_loop0_78_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32060,7 +32610,7 @@ _loop0_79_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_79[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_78[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring_format_spec")); expr_ty tstring_format_spec_var; while ( (tstring_format_spec_var = tstring_format_spec_rule(p)) // tstring_format_spec @@ -32083,7 +32633,7 @@ _loop0_79_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_79[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_78[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32100,9 +32650,9 @@ _loop0_79_rule(Parser *p) return _seq; } -// _loop0_80: tstring_middle +// _loop0_79: tstring_middle static asdl_seq * -_loop0_80_rule(Parser *p) +_loop0_79_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32127,7 +32677,7 @@ _loop0_80_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_80[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring_middle")); + D(fprintf(stderr, "%*c> _loop0_79[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring_middle")); expr_ty tstring_middle_var; while ( (tstring_middle_var = tstring_middle_rule(p)) // tstring_middle @@ -32150,7 +32700,7 @@ _loop0_80_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_80[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_79[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring_middle")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32167,7 +32717,79 @@ _loop0_80_rule(Parser *p) return _seq; } -// _loop1_81: (fstring | string | tstring) +// _loop1_80: (fstring | string) +static asdl_seq * +_loop1_80_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // (fstring | string) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_80[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); + void *_tmp_154_var; + while ( + (_tmp_154_var = _tmp_154_rule(p)) // fstring | string + ) + { + _res = _tmp_154_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_80[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string)")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop1_81: tstring static asdl_seq * _loop1_81_rule(Parser *p) { @@ -32189,18 +32811,18 @@ _loop1_81_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // (fstring | string | tstring) + { // tstring if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string | tstring)")); - void *_tmp_162_var; + D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring")); + expr_ty tstring_var; while ( - (_tmp_162_var = _tmp_162_rule(p)) // fstring | string | tstring + (tstring_var = tstring_rule(p)) // tstring ) { - _res = _tmp_162_var; + _res = tstring_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32218,7 +32840,7 @@ _loop1_81_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop1_81[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string | tstring)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -32505,12 +33127,12 @@ _loop0_86_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_163_var; + void *_tmp_164_var; while ( - (_tmp_163_var = _tmp_163_rule(p)) // 'if' disjunction + (_tmp_164_var = _tmp_164_rule(p)) // 'if' disjunction ) { - _res = _tmp_163_var; + _res = _tmp_164_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32636,7 +33258,7 @@ _loop0_88_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_164_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_165_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -32702,7 +33324,7 @@ _gather_89_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_164_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_165_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && (seq = _loop0_88_rule(p)) // _loop0_88 ) @@ -33029,12 +33651,12 @@ _loop0_95_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_95[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_165_var; + void *_tmp_166_var; while ( - (_tmp_165_var = _tmp_165_rule(p)) // ',' star_target + (_tmp_166_var = _tmp_166_rule(p)) // ',' star_target ) { - _res = _tmp_165_var; + _res = _tmp_166_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33213,12 +33835,12 @@ _loop1_98_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_98[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_165_var; + void *_tmp_166_var; while ( - (_tmp_165_var = _tmp_165_rule(p)) // ',' star_target + (_tmp_166_var = _tmp_166_rule(p)) // ',' star_target ) { - _res = _tmp_165_var; + _res = _tmp_166_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33593,13 +34215,13 @@ _tmp_105_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _tmp_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_166_var; + void *_tmp_167_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_167_var = _tmp_167_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { D(fprintf(stderr, "%*c+ _tmp_105[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_166_var; + _res = _tmp_167_var; goto done; } p->mark = _mark; @@ -33664,7 +34286,7 @@ _loop0_106_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_167_rule(p)) // starred_expression !'=' + (elem = _tmp_168_rule(p)) // starred_expression !'=' ) { _res = elem; @@ -33729,7 +34351,7 @@ _gather_107_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_167_rule(p)) // starred_expression !'=' + (elem = _tmp_168_rule(p)) // starred_expression !'=' && (seq = _loop0_106_rule(p)) // _loop0_106 ) @@ -33927,7 +34549,7 @@ _tmp_111_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -33946,7 +34568,7 @@ _tmp_111_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -33965,7 +34587,7 @@ _tmp_111_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -34051,12 +34673,12 @@ _loop1_113_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(!STRING expression_without_invalid)")); - void *_tmp_168_var; + void *_tmp_169_var; while ( - (_tmp_168_var = _tmp_168_rule(p)) // !STRING expression_without_invalid + (_tmp_169_var = _tmp_169_rule(p)) // !STRING expression_without_invalid ) { - _res = _tmp_168_var; + _res = _tmp_169_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34176,7 +34798,7 @@ _tmp_115_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='else' + (_keyword = _PyPegen_expect_token(p, 695)) // token='else' ) { D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); @@ -34423,7 +35045,7 @@ _tmp_118_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -34442,7 +35064,7 @@ _tmp_118_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -34461,7 +35083,7 @@ _tmp_118_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -34573,12 +35195,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_155_var; + void *_tmp_157_var; while ( - (_tmp_155_var = _tmp_155_rule(p)) // star_targets '=' + (_tmp_157_var = _tmp_157_rule(p)) // star_targets '=' ) { - _res = _tmp_155_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34955,15 +35577,15 @@ _tmp_126_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_169_var; + void *_tmp_170_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_169_var = _tmp_169_rule(p)) // ')' | '**' + (_tmp_170_var = _tmp_170_rule(p)) // ')' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_169_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_170_var); goto done; } p->mark = _mark; @@ -35379,15 +36001,15 @@ _tmp_133_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_170_var; + void *_tmp_171_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_170_var = _tmp_170_rule(p)) // ':' | '**' + (_tmp_171_var = _tmp_171_rule(p)) // ':' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_170_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_171_var); goto done; } p->mark = _mark; @@ -35476,20 +36098,20 @@ _tmp_135_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _tmp_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - asdl_seq * _loop0_171_var; + asdl_seq * _loop0_172_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty bitwise_or_var; if ( (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - (_loop0_171_var = _loop0_171_rule(p)) // ((',' bitwise_or))* + (_loop0_172_var = _loop0_172_rule(p)) // ((',' bitwise_or))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ _tmp_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_171_var, _opt_var); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_172_var, _opt_var); goto done; } p->mark = _mark; @@ -35619,7 +36241,7 @@ _gather_137_rule(Parser *p) return _res; } -// _tmp_138: NAME (',' | ')' | NEWLINE) +// _tmp_138: NAME (',' | ')' | ';' | NEWLINE) static void * _tmp_138_rule(Parser *p) { @@ -35632,27 +36254,27 @@ _tmp_138_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // NAME (',' | ')' | NEWLINE) + { // NAME (',' | ')' | ';' | NEWLINE) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); - void *_tmp_172_var; + D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | ';' | NEWLINE)")); + void *_tmp_173_var; expr_ty name_var; if ( (name_var = _PyPegen_name_token(p)) // NAME && - (_tmp_172_var = _tmp_172_rule(p)) // ',' | ')' | NEWLINE + (_tmp_173_var = _tmp_173_rule(p)) // ',' | ')' | ';' | NEWLINE ) { - D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); - _res = _PyPegen_dummy_name(p, name_var, _tmp_172_var); + D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | ';' | NEWLINE)")); + _res = _PyPegen_dummy_name(p, name_var, _tmp_173_var); goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_138[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME (',' | ')' | ';' | NEWLINE)")); } _res = NULL; done: @@ -35693,7 +36315,7 @@ _loop0_139_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_173_rule(p)) // expression ['as' star_target] + (elem = _tmp_174_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -35758,7 +36380,7 @@ _gather_140_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_173_rule(p)) // expression ['as' star_target] + (elem = _tmp_174_rule(p)) // expression ['as' star_target] && (seq = _loop0_139_rule(p)) // _loop0_139 ) @@ -35810,7 +36432,7 @@ _loop0_141_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_174_rule(p)) // expressions ['as' star_target] + (elem = _tmp_175_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -35875,7 +36497,7 @@ _gather_142_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_174_rule(p)) // expressions ['as' star_target] + (elem = _tmp_175_rule(p)) // expressions ['as' star_target] && (seq = _loop0_141_rule(p)) // _loop0_141 ) @@ -35915,7 +36537,7 @@ _tmp_143_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='except' + (_keyword = _PyPegen_expect_token(p, 686)) // token='except' ) { D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); @@ -35934,7 +36556,7 @@ _tmp_143_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 673)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 682)) // token='finally' ) { D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); @@ -36043,7 +36665,7 @@ _tmp_145_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ _tmp_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); @@ -36117,9 +36739,50 @@ _tmp_146_rule(Parser *p) return _res; } -// _tmp_147: positional_patterns ',' +// _tmp_147: items_pattern ',' static void * _tmp_147_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // items_pattern ',' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "items_pattern ','")); + Token * _literal; + asdl_seq* items_pattern_var; + if ( + (items_pattern_var = items_pattern_rule(p)) // items_pattern + && + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + ) + { + D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "items_pattern ','")); + _res = _PyPegen_dummy_name(p, items_pattern_var, _literal); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "items_pattern ','")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_148: positional_patterns ',' +static void * +_tmp_148_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36135,7 +36798,7 @@ _tmp_147_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -36144,12 +36807,12 @@ _tmp_147_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -36158,64 +36821,7 @@ _tmp_147_rule(Parser *p) return _res; } -// _tmp_148: '}' | ',' -static void * -_tmp_148_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '}' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 26)) // token='}' - ) - { - D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); - } - { // ',' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - ) - { - D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_149: '=' | '!' | ':' | '}' +// _tmp_149: '}' | ',' static void * _tmp_149_rule(Parser *p) { @@ -36228,63 +36834,6 @@ _tmp_149_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // '=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 22)) // token='=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); - } - { // '!' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 54)) // token='!' - ) - { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); - } - { // ':' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - ) - { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); - } { // '}' if (p->error_indicator) { p->level--; @@ -36304,13 +36853,32 @@ _tmp_149_rule(Parser *p) D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } + { // ',' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + ) + { + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); + } _res = NULL; done: p->level--; return _res; } -// _tmp_150: '!' | ':' | '}' +// _tmp_150: '=' | '!' | ':' | '}' static void * _tmp_150_rule(Parser *p) { @@ -36323,6 +36891,25 @@ _tmp_150_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; + { // '=' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + ) + { + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); + } { // '!' if (p->error_indicator) { p->level--; @@ -36386,9 +36973,85 @@ _tmp_150_rule(Parser *p) return _res; } -// _tmp_151: '!' NAME +// _tmp_151: '!' | ':' | '}' static void * _tmp_151_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '!' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 54)) // token='!' + ) + { + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); + } + { // ':' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + ) + { + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + } + { // '}' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 26)) // token='}' + ) + { + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_152: '!' NAME +static void * +_tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36404,7 +37067,7 @@ _tmp_151_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -36413,12 +37076,12 @@ _tmp_151_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -36427,9 +37090,9 @@ _tmp_151_rule(Parser *p) return _res; } -// _tmp_152: ':' | '}' +// _tmp_153: ':' | '}' static void * -_tmp_152_rule(Parser *p) +_tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36445,18 +37108,18 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -36464,18 +37127,18 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -36484,584 +37147,9 @@ _tmp_152_rule(Parser *p) return _res; } -// _tmp_153: '+' | '-' | '*' | '/' | '%' | '//' | '@' -static void * -_tmp_153_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '+' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 14)) // token='+' - ) - { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); - } - { // '-' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 15)) // token='-' - ) - { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); - } - { // '*' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 16)) // token='*' - ) - { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); - } - { // '/' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 17)) // token='/' - ) - { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); - } - { // '%' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 24)) // token='%' - ) - { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); - } - { // '//' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 47)) // token='//' - ) - { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); - } - { // '@' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 49)) // token='@' - ) - { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_154: '+' | '-' | '~' +// _tmp_154: fstring | string static void * _tmp_154_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '+' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 14)) // token='+' - ) - { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); - } - { // '-' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 15)) // token='-' - ) - { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); - } - { // '~' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 31)) // token='~' - ) - { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_155: star_targets '=' -static void * -_tmp_155_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // star_targets '=' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); - Token * _literal; - expr_ty z; - if ( - (z = star_targets_rule(p)) // star_targets - && - (_literal = _PyPegen_expect_token(p, 22)) // token='=' - ) - { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_156: '.' | '...' -static void * -_tmp_156_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '.' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 23)) // token='.' - ) - { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); - } - { // '...' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 52)) // token='...' - ) - { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_157: '@' named_expression NEWLINE -static void * -_tmp_157_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // '@' named_expression NEWLINE - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); - Token * _literal; - expr_ty f; - Token * newline_var; - if ( - (_literal = _PyPegen_expect_token(p, 49)) // token='@' - && - (f = named_expression_rule(p)) // named_expression - && - (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' - ) - { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); - _res = f; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_158: ',' star_expression -static void * -_tmp_158_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ',' star_expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); - Token * _literal; - expr_ty c; - if ( - (_literal = _PyPegen_expect_token(p, 12)) // token=',' - && - (c = star_expression_rule(p)) // star_expression - ) - { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); - _res = c; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_159: 'or' conjunction -static void * -_tmp_159_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'or' conjunction - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); - Token * _keyword; - expr_ty c; - if ( - (_keyword = _PyPegen_expect_token(p, 588)) // token='or' - && - (c = conjunction_rule(p)) // conjunction - ) - { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); - _res = c; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_160: 'and' inversion -static void * -_tmp_160_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'and' inversion - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); - Token * _keyword; - expr_ty c; - if ( - (_keyword = _PyPegen_expect_token(p, 589)) // token='and' - && - (c = inversion_rule(p)) // inversion - ) - { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); - _res = c; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_161: slice | starred_expression -static void * -_tmp_161_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // slice - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); - expr_ty slice_var; - if ( - (slice_var = slice_rule(p)) // slice - ) - { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); - _res = slice_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); - } - { // starred_expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); - expr_ty starred_expression_var; - if ( - (starred_expression_var = starred_expression_rule(p)) // starred_expression - ) - { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); - _res = starred_expression_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_162: fstring | string | tstring -static void * -_tmp_162_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37077,18 +37165,18 @@ _tmp_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -37096,48 +37184,604 @@ _tmp_162_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } - { // tstring - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring")); - expr_ty tstring_var; - if ( - (tstring_var = tstring_rule(p)) // tstring - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring")); - _res = tstring_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring")); - } _res = NULL; done: p->level--; return _res; } -// _tmp_163: 'if' disjunction +// _tmp_155: '+' | '-' | '*' | '/' | '%' | '//' | '@' +static void * +_tmp_155_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '+' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + ) + { + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); + } + { // '-' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 15)) // token='-' + ) + { + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); + } + { // '*' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + ) + { + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); + } + { // '/' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 17)) // token='/' + ) + { + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); + } + { // '%' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 24)) // token='%' + ) + { + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); + } + { // '//' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 47)) // token='//' + ) + { + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); + } + { // '@' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 49)) // token='@' + ) + { + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_156: '+' | '-' | '~' +static void * +_tmp_156_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '+' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + ) + { + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); + } + { // '-' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 15)) // token='-' + ) + { + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); + } + { // '~' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 31)) // token='~' + ) + { + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_157: star_targets '=' +static void * +_tmp_157_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // star_targets '=' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + Token * _literal; + expr_ty z; + if ( + (z = star_targets_rule(p)) // star_targets + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + ) + { + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + _res = z; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_158: '.' | '...' +static void * +_tmp_158_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '.' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 23)) // token='.' + ) + { + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); + } + { // '...' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 52)) // token='...' + ) + { + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_159: '@' named_expression NEWLINE +static void * +_tmp_159_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '@' named_expression NEWLINE + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + Token * _literal; + expr_ty f; + Token * newline_var; + if ( + (_literal = _PyPegen_expect_token(p, 49)) // token='@' + && + (f = named_expression_rule(p)) // named_expression + && + (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' + ) + { + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + _res = f; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_160: ',' star_expression +static void * +_tmp_160_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ',' star_expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + Token * _literal; + expr_ty c; + if ( + (_literal = _PyPegen_expect_token(p, 12)) // token=',' + && + (c = star_expression_rule(p)) // star_expression + ) + { + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + _res = c; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_161: 'or' conjunction +static void * +_tmp_161_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // 'or' conjunction + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + Token * _keyword; + expr_ty c; + if ( + (_keyword = _PyPegen_expect_token(p, 589)) // token='or' + && + (c = conjunction_rule(p)) // conjunction + ) + { + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + _res = c; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_162: 'and' inversion +static void * +_tmp_162_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // 'and' inversion + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + Token * _keyword; + expr_ty c; + if ( + (_keyword = _PyPegen_expect_token(p, 590)) // token='and' + && + (c = inversion_rule(p)) // inversion + ) + { + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + _res = c; + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_163: slice | starred_expression static void * _tmp_163_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // slice + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + expr_ty slice_var; + if ( + (slice_var = slice_rule(p)) // slice + ) + { + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + _res = slice_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); + } + { // starred_expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + expr_ty starred_expression_var; + if ( + (starred_expression_var = starred_expression_rule(p)) // starred_expression + ) + { + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + _res = starred_expression_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_164: 'if' disjunction +static void * +_tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37153,16 +37797,16 @@ _tmp_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 682)) // token='if' + (_keyword = _PyPegen_expect_token(p, 691)) // token='if' && (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37172,7 +37816,7 @@ _tmp_163_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -37181,9 +37825,9 @@ _tmp_163_rule(Parser *p) return _res; } -// _tmp_164: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_165: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_164_rule(Parser *p) +_tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37199,18 +37843,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -37218,7 +37862,7 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); void *_tmp_87_var; if ( (_tmp_87_var = _tmp_87_rule(p)) // assignment_expression | expression !':=' @@ -37226,12 +37870,12 @@ _tmp_164_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); _res = _tmp_87_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -37240,9 +37884,9 @@ _tmp_164_rule(Parser *p) return _res; } -// _tmp_165: ',' star_target +// _tmp_166: ',' star_target static void * -_tmp_165_rule(Parser *p) +_tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37258,7 +37902,7 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -37267,7 +37911,7 @@ _tmp_165_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37277,7 +37921,7 @@ _tmp_165_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -37286,10 +37930,10 @@ _tmp_165_rule(Parser *p) return _res; } -// _tmp_166: +// _tmp_167: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_166_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37305,7 +37949,7 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); asdl_seq * _gather_89_var; Token * _literal; asdl_seq* kwargs_var; @@ -37317,12 +37961,12 @@ _tmp_166_rule(Parser *p) (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); _res = _PyPegen_dummy_name(p, _gather_89_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -37331,9 +37975,9 @@ _tmp_166_rule(Parser *p) return _res; } -// _tmp_167: starred_expression !'=' +// _tmp_168: starred_expression !'=' static void * -_tmp_167_rule(Parser *p) +_tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37349,7 +37993,7 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression @@ -37357,12 +38001,12 @@ _tmp_167_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); } _res = NULL; @@ -37371,9 +38015,9 @@ _tmp_167_rule(Parser *p) return _res; } -// _tmp_168: !STRING expression_without_invalid +// _tmp_169: !STRING expression_without_invalid static void * -_tmp_168_rule(Parser *p) +_tmp_169_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37389,20 +38033,20 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); + D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); expr_ty expression_without_invalid_var; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _PyPegen_string_token, p) + _PyPegen_lookahead(0, _PyPegen_string_token, p) && (expression_without_invalid_var = expression_without_invalid_rule(p)) // expression_without_invalid ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); + D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); _res = expression_without_invalid_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!STRING expression_without_invalid")); } _res = NULL; @@ -37411,9 +38055,9 @@ _tmp_168_rule(Parser *p) return _res; } -// _tmp_169: ')' | '**' +// _tmp_170: ')' | '**' static void * -_tmp_169_rule(Parser *p) +_tmp_170_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37429,76 +38073,19 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); - } - { // '**' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 35)) // token='**' - ) - { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); - _res = _literal; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_170: ':' | '**' -static void * -_tmp_170_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // ':' - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); - Token * _literal; - if ( - (_literal = _PyPegen_expect_token(p, 11)) // token=':' - ) - { - D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' if (p->error_indicator) { @@ -37525,9 +38112,66 @@ _tmp_170_rule(Parser *p) return _res; } -// _loop0_171: (',' bitwise_or) +// _tmp_171: ':' | '**' +static void * +_tmp_171_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ':' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + ) + { + D(fprintf(stderr, "%*c+ _tmp_171[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_171[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); + } + { // '**' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 35)) // token='**' + ) + { + D(fprintf(stderr, "%*c+ _tmp_171[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_171[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _loop0_172: (',' bitwise_or) static asdl_seq * -_loop0_171_rule(Parser *p) +_loop0_172_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37552,13 +38196,13 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); - void *_tmp_175_var; + D(fprintf(stderr, "%*c> _loop0_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_176_var; while ( - (_tmp_175_var = _tmp_175_rule(p)) // ',' bitwise_or + (_tmp_176_var = _tmp_176_rule(p)) // ',' bitwise_or ) { - _res = _tmp_175_var; + _res = _tmp_176_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -37575,7 +38219,7 @@ _loop0_171_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37592,9 +38236,9 @@ _loop0_171_rule(Parser *p) return _seq; } -// _tmp_172: ',' | ')' | NEWLINE +// _tmp_173: ',' | ')' | ';' | NEWLINE static void * -_tmp_172_rule(Parser *p) +_tmp_173_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37610,18 +38254,18 @@ _tmp_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -37629,37 +38273,56 @@ _tmp_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } + { // ';' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 13)) // token=';' + ) + { + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "';'")); + } { // NEWLINE if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } _res = NULL; @@ -37668,9 +38331,9 @@ _tmp_172_rule(Parser *p) return _res; } -// _tmp_173: expression ['as' star_target] +// _tmp_174: expression ['as' star_target] static void * -_tmp_173_rule(Parser *p) +_tmp_174_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37686,22 +38349,22 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_176_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_177_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -37710,9 +38373,9 @@ _tmp_173_rule(Parser *p) return _res; } -// _tmp_174: expressions ['as' star_target] +// _tmp_175: expressions ['as' star_target] static void * -_tmp_174_rule(Parser *p) +_tmp_175_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37728,22 +38391,22 @@ _tmp_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_176_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_177_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -37752,9 +38415,9 @@ _tmp_174_rule(Parser *p) return _res; } -// _tmp_175: ',' bitwise_or +// _tmp_176: ',' bitwise_or static void * -_tmp_175_rule(Parser *p) +_tmp_176_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37770,7 +38433,7 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); Token * _literal; expr_ty bitwise_or_var; if ( @@ -37779,12 +38442,12 @@ _tmp_175_rule(Parser *p) (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; @@ -37793,9 +38456,9 @@ _tmp_175_rule(Parser *p) return _res; } -// _tmp_176: 'as' star_target +// _tmp_177: 'as' star_target static void * -_tmp_176_rule(Parser *p) +_tmp_177_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37811,21 +38474,21 @@ _tmp_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 680)) // token='as' + (_keyword = _PyPegen_expect_token(p, 689)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_177[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; diff --git a/Parser/peg_api.c b/Parser/peg_api.c index d4acc3e4935..e30ca0453bd 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -4,13 +4,15 @@ mod_ty _PyParser_ASTFromString(const char *str, PyObject* filename, int mode, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, + PyObject *module) { if (PySys_Audit("compile", "yO", str, filename) < 0) { return NULL; } - mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, arena); + mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, + arena, module); return result; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 3efeba78450..a38e973b3f6 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -5,6 +5,7 @@ #include "pycore_pyerrors.h" // PyExc_IncompleteInputError #include "pycore_runtime.h" // _PyRuntime #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal +#include "pycore_pyatomic_ft_wrappers.h" #include <errcode.h> #include "lexer/lexer.h" @@ -299,22 +300,14 @@ _PyPegen_fill_token(Parser *p) #define NSTATISTICS _PYPEGEN_NSTATISTICS #define memo_statistics _PyRuntime.parser.memo_statistics -#ifdef Py_GIL_DISABLED -#define MUTEX_LOCK() PyMutex_Lock(&_PyRuntime.parser.mutex) -#define MUTEX_UNLOCK() PyMutex_Unlock(&_PyRuntime.parser.mutex) -#else -#define MUTEX_LOCK() -#define MUTEX_UNLOCK() -#endif - void _PyPegen_clear_memo_statistics(void) { - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); for (int i = 0; i < NSTATISTICS; i++) { memo_statistics[i] = 0; } - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); } PyObject * @@ -325,22 +318,22 @@ _PyPegen_get_memo_statistics(void) return NULL; } - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); for (int i = 0; i < NSTATISTICS; i++) { PyObject *value = PyLong_FromLong(memo_statistics[i]); if (value == NULL) { - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); Py_DECREF(ret); return NULL; } // PyList_SetItem borrows a reference to value. if (PyList_SetItem(ret, i, value) < 0) { - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); Py_DECREF(ret); return NULL; } } - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); return ret; } #endif @@ -366,9 +359,9 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) if (count <= 0) { count = 1; } - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); memo_statistics[type] += count; - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); } #endif p->mark = m->mark; @@ -379,44 +372,34 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) return 0; } -int -_PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p) -{ - int mark = p->mark; - void *res = func(p); - p->mark = mark; - return (res != NULL) == positive; -} +#define LOOKAHEAD1(NAME, RES_TYPE) \ + int \ + NAME (int positive, RES_TYPE (func)(Parser *), Parser *p) \ + { \ + int mark = p->mark; \ + void *res = func(p); \ + p->mark = mark; \ + return (res != NULL) == positive; \ + } -int -_PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg) -{ - int mark = p->mark; - void *res = func(p, arg); - p->mark = mark; - return (res != NULL) == positive; -} +LOOKAHEAD1(_PyPegen_lookahead, void *) +LOOKAHEAD1(_PyPegen_lookahead_for_expr, expr_ty) +LOOKAHEAD1(_PyPegen_lookahead_for_stmt, stmt_ty) +#undef LOOKAHEAD1 -int -_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg) -{ - int mark = p->mark; - void *res = func(p, arg); - p->mark = mark; - return (res != NULL) == positive; -} +#define LOOKAHEAD2(NAME, RES_TYPE, T) \ + int \ + NAME (int positive, RES_TYPE (func)(Parser *, T), Parser *p, T arg) \ + { \ + int mark = p->mark; \ + void *res = func(p, arg); \ + p->mark = mark; \ + return (res != NULL) == positive; \ + } -// gh-111178: Use _Py_NO_SANITIZE_UNDEFINED to disable sanitizer checks on -// undefined behavior (UBsan) in this function, rather than changing 'func' -// callback API. -int _Py_NO_SANITIZE_UNDEFINED -_PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p) -{ - int mark = p->mark; - void *res = func(p); - p->mark = mark; - return (res != NULL) == positive; -} +LOOKAHEAD2(_PyPegen_lookahead_with_int, Token *, int) +LOOKAHEAD2(_PyPegen_lookahead_with_string, expr_ty, const char *) +#undef LOOKAHEAD2 Token * _PyPegen_expect_token(Parser *p, int type) @@ -549,6 +532,21 @@ _PyPegen_new_identifier(Parser *p, const char *n) } id = id2; } + static const char * const forbidden[] = { + "None", + "True", + "False", + NULL + }; + for (int i = 0; forbidden[i] != NULL; i++) { + if (_PyUnicode_EqualToASCIIString(id, forbidden[i])) { + PyErr_Format(PyExc_ValueError, + "identifier field can't represent '%s' constant", + forbidden[i]); + Py_DECREF(id); + goto error; + } + } PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUnicode_InternImmortal(interp, &id); if (_PyArena_AddPyObject(p->arena, id) < 0) @@ -605,7 +603,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p) { Py_ssize_t size; PyBytes_AsStringAndSize(t->bytes, &the_token, &size); for (char **keyword = p->soft_keywords; *keyword != NULL; keyword++) { - if (strncmp(*keyword, the_token, (size_t)size) == 0) { + if (strlen(*keyword) == (size_t)size && + strncmp(*keyword, the_token, (size_t)size) == 0) { return _PyPegen_name_from_token(p, t); } } @@ -1011,6 +1010,11 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena // From here on we need to clean up even if there's an error mod_ty result = NULL; + tok->module = PyUnicode_FromString("__main__"); + if (tok->module == NULL) { + goto error; + } + int parser_flags = compute_parser_flags(flags); Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION, errcode, NULL, arena); @@ -1037,7 +1041,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena mod_ty _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, PyObject *module) { int exec_input = start_rule == Py_file_input; @@ -1055,6 +1059,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } // This transfers the ownership to the tokenizer tok->filename = Py_NewRef(filename_ob); + tok->module = Py_XNewRef(module); // We need to clear up from here on mod_ty result = NULL; diff --git a/Parser/pegen.h b/Parser/pegen.h index 1862fd7407e..b8f887608b1 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -145,10 +145,11 @@ int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node); int _PyPegen_update_memo(Parser *p, int mark, int type, void *node); int _PyPegen_is_memoized(Parser *p, int type, void *pres); -int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *); -int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); -int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*); int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *); +int _PyPegen_lookahead_for_expr(int, expr_ty (func)(Parser *), Parser *); +int _PyPegen_lookahead_for_stmt(int, stmt_ty (func)(Parser *), Parser *); +int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); +int _PyPegen_lookahead_with_string(int, expr_ty (func)(Parser *, const char*), Parser *, const char*); Token *_PyPegen_expect_token(Parser *p, int type); void* _PyPegen_expect_forced_result(Parser *p, void* result, const char* expected); @@ -161,7 +162,6 @@ int _PyPegen_fill_token(Parser *p); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); -PyObject *_PyPegen_set_source_in_metadata(Parser *p, Token *t); Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset); Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset); Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset); @@ -350,6 +350,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *, expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok); expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok); expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok); +expr_ty _PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int); expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty); @@ -377,7 +378,7 @@ mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char const char *, const char *, PyCompilerFlags *, int *, PyObject **, PyArena *); void *_PyPegen_run_parser(Parser *); -mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); +mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *); asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); // Generated function in parse.c - function definition in python.gram diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index f62b8695995..0639a4e4243 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -2,6 +2,7 @@ #include <errcode.h> #include "pycore_pyerrors.h" // _PyErr_ProgramDecodedTextObject() +#include "pycore_runtime.h" // _Py_ID() #include "lexer/state.h" #include "lexer/lexer.h" #include "pegen.h" @@ -23,6 +24,13 @@ _PyPegen_raise_tokenizer_init_error(PyObject *filename) PyObject *value; PyObject *tback; PyErr_Fetch(&type, &value, &tback); + if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) { + if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) { + goto error; + } + PyErr_Restore(type, value, tback); + return; + } errstr = PyObject_Str(value); if (!errstr) { goto error; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index d3631b114c5..b164dfbc81a 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -88,7 +88,7 @@ warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_in } if (PyErr_WarnExplicitObject(category, msg, p->tok->filename, - lineno, NULL, NULL) < 0) { + lineno, p->tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the Syntax/DeprecationWarning exception with a SyntaxError to get a more accurate error report */ @@ -196,15 +196,18 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) len = (size_t)(p - buf); s = buf; - const char *first_invalid_escape; - v = _PyUnicode_DecodeUnicodeEscapeInternal(s, (Py_ssize_t)len, NULL, NULL, &first_invalid_escape); + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + v = _PyUnicode_DecodeUnicodeEscapeInternal2(s, (Py_ssize_t)len, NULL, NULL, + &first_invalid_escape_char, + &first_invalid_escape_ptr); // HACK: later we can simply pass the line no, since we don't preserve the tokens // when we are decoding the string but we preserve the line numbers. - if (v != NULL && first_invalid_escape != NULL && t != NULL) { - if (warn_invalid_escape_sequence(parser, s, first_invalid_escape, t) < 0) { - /* We have not decref u before because first_invalid_escape points - inside u. */ + if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL) { + if (warn_invalid_escape_sequence(parser, s, first_invalid_escape_ptr, t) < 0) { + /* We have not decref u before because first_invalid_escape_ptr + points inside u. */ Py_XDECREF(u); Py_DECREF(v); return NULL; @@ -217,14 +220,17 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) static PyObject * decode_bytes_with_escapes(Parser *p, const char *s, Py_ssize_t len, Token *t) { - const char *first_invalid_escape; - PyObject *result = _PyBytes_DecodeEscape(s, len, NULL, &first_invalid_escape); + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + PyObject *result = _PyBytes_DecodeEscape2(s, len, NULL, + &first_invalid_escape_char, + &first_invalid_escape_ptr); if (result == NULL) { return NULL; } - if (first_invalid_escape != NULL) { - if (warn_invalid_escape_sequence(p, s, first_invalid_escape, t) < 0) { + if (first_invalid_escape_ptr != NULL) { + if (warn_invalid_escape_sequence(p, s, first_invalid_escape_ptr, t) < 0) { Py_DECREF(result); return NULL; } diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index 01e473f58a0..8c836a3f725 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -282,10 +282,8 @@ tok_underflow_interactive(struct tok_state *tok) { } static int -tok_underflow_file(struct tok_state *tok) { - if (tok->start == NULL && !INSIDE_FSTRING(tok)) { - tok->cur = tok->inp = tok->buf; - } +tok_underflow_file(struct tok_state *tok) +{ if (tok->decoding_state == STATE_INIT) { /* We have not yet determined the encoding. If an encoding is found, use the file-pointer @@ -296,8 +294,16 @@ tok_underflow_file(struct tok_state *tok) { } assert(tok->decoding_state != STATE_INIT); } + int raw = tok->decoding_readline == NULL; + if (raw && tok->decoding_state != STATE_NORMAL) { + /* Keep the first line in the buffer to validate it later if + * the encoding has not yet been determined. */ + } + else if (tok->start == NULL && !INSIDE_FSTRING(tok)) { + tok->cur = tok->inp = tok->buf; + } /* Read until '\n' or EOF */ - if (tok->decoding_readline != NULL) { + if (!raw) { /* We already have a codec associated with this input. */ if (!tok_readline_recode(tok)) { return 0; @@ -328,20 +334,35 @@ tok_underflow_file(struct tok_state *tok) { ADVANCE_LINENO(); if (tok->decoding_state != STATE_NORMAL) { - if (tok->lineno > 2) { - tok->decoding_state = STATE_NORMAL; - } - else if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur), + if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur), tok, fp_setreadl)) { return 0; } + if (tok->lineno >= 2) { + tok->decoding_state = STATE_NORMAL; + } } - /* The default encoding is UTF-8, so make sure we don't have any - non-UTF-8 sequences in it. */ - if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) { - _PyTokenizer_error_ret(tok); - return 0; + if (raw && tok->decoding_state == STATE_NORMAL) { + const char *line = tok->lineno <= 2 ? tok->buf : tok->cur; + int lineno = tok->lineno <= 2 ? 1 : tok->lineno; + if (!tok->encoding) { + /* The default encoding is UTF-8, so make sure we don't have any + non-UTF-8 sequences in it. */ + if (!_PyTokenizer_ensure_utf8(line, tok, lineno)) { + _PyTokenizer_error_ret(tok); + return 0; + } + } + else { + PyObject *tmp = PyUnicode_Decode(line, strlen(line), + tok->encoding, NULL); + if (tmp == NULL) { + _PyTokenizer_error_ret(tok); + return 0; + } + Py_DECREF(tmp); + } } assert(tok->done == E_OK); return tok->done == E_OK; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 5a416adb875..a03531a7441 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -47,8 +47,10 @@ _syntaxerror_range(struct tok_state *tok, const char *format, goto error; } - args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno, - col_offset, errtext, tok->lineno, end_col_offset); + args = Py_BuildValue("(O(OiiNii))", errmsg, + tok->filename ? tok->filename : Py_None, + tok->lineno, col_offset, errtext, + tok->lineno, end_col_offset); if (args) { PyErr_SetObject(PyExc_SyntaxError, args); Py_DECREF(args); @@ -125,7 +127,7 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval } if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { Py_DECREF(msg); if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { @@ -164,7 +166,7 @@ _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char * } if (PyErr_WarnExplicitObject(category, errmsg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the DeprecationWarning exception with a SyntaxError to get a more accurate error report */ @@ -422,10 +424,13 @@ _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_sta tok->encoding = cs; } else { /* then, compare cs with BOM */ if (strcmp(tok->encoding, cs) != 0) { - _PyTokenizer_error_ret(tok); - PyErr_Format(PyExc_SyntaxError, - "encoding problem: %s with BOM", cs); + tok->line_start = line; + tok->cur = (char *)line; + assert(size <= INT_MAX); + _PyTokenizer_syntaxerror_known_range(tok, 0, (int)size, + "encoding problem: %s with BOM", cs); PyMem_Free(cs); + _PyTokenizer_error_ret(tok); return 0; } PyMem_Free(cs); @@ -496,24 +501,38 @@ valid_utf8(const unsigned char* s) } int -_PyTokenizer_ensure_utf8(char *line, struct tok_state *tok) +_PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) { - int badchar = 0; - unsigned char *c; + const char *badchar = NULL; + const char *c; int length; - for (c = (unsigned char *)line; *c; c += length) { - if (!(length = valid_utf8(c))) { - badchar = *c; + int col_offset = 0; + const char *line_start = line; + for (c = line; *c; c += length) { + if (!(length = valid_utf8((const unsigned char *)c))) { + badchar = c; break; } + col_offset++; + if (*c == '\n') { + lineno++; + col_offset = 0; + line_start = c + 1; + } } if (badchar) { - PyErr_Format(PyExc_SyntaxError, - "Non-UTF-8 code starting with '\\x%.2x' " - "in file %U on line %i, " - "but no encoding declared; " - "see https://peps.python.org/pep-0263/ for details", - badchar, tok->filename, tok->lineno); + tok->lineno = lineno; + tok->line_start = line_start; + tok->cur = (char *)badchar; + _PyTokenizer_syntaxerror_known_range(tok, + col_offset + 1, col_offset + 1, + "Non-UTF-8 code starting with '\\x%.2x'" + "%s%V on line %i, " + "but no encoding declared; " + "see https://peps.python.org/pep-0263/ for details", + (unsigned char)*badchar, + tok->filename ? " in file " : "", tok->filename, "", + lineno); return 0; } return 1; diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 42ea13cd1f8..98f6445d5a3 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -26,7 +26,7 @@ int _PyTokenizer_check_bom(int get_char(struct tok_state *), struct tok_state *tok); int _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, int set_readline(struct tok_state *, const char *)); -int _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok); +int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno); #ifdef Py_DEBUG void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index 22f84c77a12..0f7769aeb8f 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -97,7 +97,7 @@ tok_underflow_readline(struct tok_state* tok) { ADVANCE_LINENO(); /* The default encoding is UTF-8, so make sure we don't have any non-UTF-8 sequences in it. */ - if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) { + if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTokenizer_error_ret(tok); return 0; } diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c index 0c26d5df8d4..7299ecf483c 100644 --- a/Parser/tokenizer/string_tokenizer.c +++ b/Parser/tokenizer/string_tokenizer.c @@ -86,15 +86,18 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr /* need to check line 1 and 2 separately since check_coding_spec assumes a single line as input */ if (newl[0]) { + tok->lineno = 1; if (!_PyTokenizer_check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) { return NULL; } if (tok->enc == NULL && tok->decoding_state != STATE_NORMAL && newl[1]) { + tok->lineno = 2; if (!_PyTokenizer_check_coding_spec(newl[0]+1, newl[1] - newl[0], tok, buf_setreadl)) return NULL; } } + tok->lineno = 0; if (tok->enc != NULL) { assert(utf8 == NULL); utf8 = _PyTokenizer_translate_into_utf8(str, tok->enc); @@ -102,6 +105,9 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr return _PyTokenizer_error_ret(tok); str = PyBytes_AS_STRING(utf8); } + else if (!_PyTokenizer_ensure_utf8(str, tok, 1)) { + return _PyTokenizer_error_ret(tok); + } assert(tok->decoding_buffer == NULL); tok->decoding_buffer = utf8; /* CAUTION */ return str; diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 06d1ee016dc..a5809b37b6b 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -45,27 +45,40 @@ static const char header[] = static void runtime_init(void) { - PyConfig config; - PyConfig_InitIsolatedConfig(&config); + PyInitConfig *config = PyInitConfig_Create(); + if (config == NULL) { + printf("memory allocation failed\n"); + exit(1); + } - config.site_import = 0; - - PyStatus status; - status = PyConfig_SetString(&config, &config.program_name, - L"./_freeze_module"); - if (PyStatus_Exception(status)) { - PyConfig_Clear(&config); - Py_ExitStatusException(status); + if (PyInitConfig_SetInt(config, "site_import", 0) < 0) { + goto error; + } + if (PyInitConfig_SetStr(config, "program_name", "./_freeze_module") < 0) { + goto error; } /* Don't install importlib, since it could execute outdated bytecode. */ - config._install_importlib = 0; - config._init_main = 0; + if (PyInitConfig_SetInt(config, "_install_importlib", 0) < 0) { + goto error; + } + if (PyInitConfig_SetInt(config, "_init_main", 0) < 0) { + goto error; + } - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); + if (Py_InitializeFromInitConfig(config) < 0) { + goto error; + } + PyInitConfig_Free(config); + return; + +error: + { + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("Python init error: %s\n", err_msg); + PyInitConfig_Free(config); + exit(1); } } diff --git a/Programs/_freeze_module.py b/Programs/_freeze_module.py index ba638eef6c4..62274e4aa9c 100644 --- a/Programs/_freeze_module.py +++ b/Programs/_freeze_module.py @@ -23,7 +23,7 @@ def read_text(inpath: str) -> bytes: def compile_and_marshal(name: str, text: bytes) -> bytes: filename = f"<frozen {name}>" # exec == Py_file_input - code = compile(text, filename, "exec", optimize=0, dont_inherit=True) + code = compile(text, filename, "exec", optimize=0, dont_inherit=True, module=name) return marshal.dumps(code) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 8a7412c7019..c6a18249e3c 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -8,6 +8,7 @@ #include <Python.h> #include "pycore_initconfig.h" // _PyConfig_InitCompatConfig() #include "pycore_runtime.h" // _PyRuntime +#include "pycore_lock.h" // PyEvent #include "pycore_pythread.h" // PyThread_start_joinable_thread() #include "pycore_import.h" // _PyImport_FrozenBootstrap #include <inttypes.h> @@ -21,6 +22,10 @@ extern void PySys_AddWarnOption(const wchar_t *s); extern void PySys_AddXOption(const wchar_t *s); extern void Py_SetPath(const wchar_t *path); +// These functions were removed from Python 3.15 API but are still exported +// for the stable ABI. We want to test them in this program. +extern void PySys_ResetWarnOptions(void); + int main_argc; char **main_argv; @@ -80,7 +85,7 @@ static void init_from_config_clear(PyConfig *config) } -static void _testembed_Py_InitializeFromConfig(void) +static void _testembed_initialize(void) { PyConfig config; _PyConfig_InitCompatConfig(&config); @@ -88,16 +93,10 @@ static void _testembed_Py_InitializeFromConfig(void) init_from_config_clear(&config); } -static void _testembed_Py_Initialize(void) -{ - Py_SetProgramName(PROGRAM_NAME); - Py_Initialize(); -} - static int test_import_in_subinterpreters(void) { - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); PyThreadState_Swap(Py_NewInterpreter()); return PyRun_SimpleString("import readline"); // gh-124160 } @@ -131,7 +130,7 @@ static int test_repeated_init_and_subinterpreters(void) for (int i=1; i <= INIT_LOOPS; i++) { printf("--- Pass %d ---\n", i); - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); mainstate = PyThreadState_Get(); PyEval_ReleaseThread(mainstate); @@ -167,6 +166,8 @@ static PyModuleDef embedded_ext = { static PyObject* PyInit_embedded_ext(void) { + // keep this as a single-phase initialization module; + // see test_create_module_from_initfunc return PyModule_Create(&embedded_ext); } @@ -197,7 +198,7 @@ static int test_repeated_init_exec(void) code = main_argv[i+2]; } - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); int err = PyRun_SimpleString(code); Py_Finalize(); if (err) { @@ -217,7 +218,7 @@ static int test_repeated_simple_init(void) fprintf(stderr, "--- Loop #%d ---\n", i); fflush(stderr); - _testembed_Py_Initialize(); + _testembed_initialize(); Py_Finalize(); printf("Finalized\n"); // Give test_embed some output to check } @@ -301,24 +302,8 @@ static int test_pre_initialization_api(void) /* the test doesn't support custom memory allocators */ putenv("PYTHONMALLOC="); - /* Leading "./" ensures getpath.c can still find the standard library */ - _Py_EMBED_PREINIT_CHECK("Checking Py_DecodeLocale\n"); - wchar_t *program = Py_DecodeLocale("./spam", NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode program name\n"); - return 1; - } - _Py_EMBED_PREINIT_CHECK("Checking Py_SetProgramName\n"); - Py_SetProgramName(program); - - _Py_EMBED_PREINIT_CHECK("Checking !Py_IsInitialized pre-initialization\n"); - if (Py_IsInitialized()) { - fprintf(stderr, "Fatal error: initialized before initialization!\n"); - return 1; - } - _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); - Py_Initialize(); + _testembed_initialize(); _Py_EMBED_PREINIT_CHECK("Checking Py_IsInitialized post-initialization\n"); if (!Py_IsInitialized()) { @@ -340,9 +325,6 @@ static int test_pre_initialization_api(void) fprintf(stderr, "Fatal error: still initialized after finalization!\n"); return 1; } - - _Py_EMBED_PREINIT_CHECK("Freeing memory allocated by Py_DecodeLocale\n"); - PyMem_RawFree(program); return 0; } @@ -360,8 +342,18 @@ static int test_pre_initialization_sys_options(void) size_t xoption_len = wcslen(static_xoption); wchar_t *dynamic_once_warnoption = \ (wchar_t *) calloc(warnoption_len+1, sizeof(wchar_t)); + if (dynamic_once_warnoption == NULL) { + error("out of memory allocating warnoption"); + return 1; + } wchar_t *dynamic_xoption = \ (wchar_t *) calloc(xoption_len+1, sizeof(wchar_t)); + if (dynamic_xoption == NULL) { + free(dynamic_once_warnoption); + error("out of memory allocating xoption"); + return 1; + } + wcsncpy(dynamic_once_warnoption, static_warnoption, warnoption_len+1); wcsncpy(dynamic_xoption, static_xoption, xoption_len+1); @@ -384,7 +376,7 @@ static int test_pre_initialization_sys_options(void) dynamic_xoption = NULL; _Py_EMBED_PREINIT_CHECK("Initializing interpreter\n"); - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); _Py_EMBED_PREINIT_CHECK("Check sys module contents\n"); PyRun_SimpleString( "import sys; " @@ -403,9 +395,9 @@ static int test_pre_initialization_sys_options(void) /* bpo-20891: Avoid race condition when initialising the GIL */ -static void bpo20891_thread(void *lockp) +static void bpo20891_thread(void *eventp) { - PyThread_type_lock lock = *((PyThread_type_lock*)lockp); + PyEvent *event = (PyEvent *)eventp; PyGILState_STATE state = PyGILState_Ensure(); if (!PyGILState_Check()) { @@ -414,8 +406,7 @@ static void bpo20891_thread(void *lockp) } PyGILState_Release(state); - - PyThread_release_lock(lock); + _PyEvent_Notify(event); } static int test_bpo20891(void) @@ -425,27 +416,16 @@ static int test_bpo20891(void) /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread must not crash. */ - PyThread_type_lock lock = PyThread_allocate_lock(); - if (!lock) { - error("PyThread_allocate_lock failed!"); - return 1; - } + PyEvent event = {0}; + _testembed_initialize(); - _testembed_Py_InitializeFromConfig(); - - unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &lock); + unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &event); if (thrd == PYTHREAD_INVALID_THREAD_ID) { error("PyThread_start_new_thread failed!"); return 1; } - PyThread_acquire_lock(lock, WAIT_LOCK); - Py_BEGIN_ALLOW_THREADS - /* wait until the thread exit */ - PyThread_acquire_lock(lock, WAIT_LOCK); - Py_END_ALLOW_THREADS - - PyThread_free_lock(lock); + PyEvent_Wait(&event); Py_Finalize(); @@ -454,7 +434,7 @@ static int test_bpo20891(void) static int test_initialize_twice(void) { - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); /* bpo-33932: Calling Py_Initialize() twice should do nothing * (and not crash!). */ @@ -472,7 +452,7 @@ static int test_initialize_pymain(void) L"print(f'Py_Main() after Py_Initialize: " L"sys.argv={sys.argv}')"), L"arg2"}; - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); /* bpo-34008: Calling Py_Main() after Py_Initialize() must not crash */ Py_Main(Py_ARRAY_LENGTH(argv), argv); @@ -495,7 +475,7 @@ dump_config(void) static int test_init_initialize_config(void) { - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); dump_config(); Py_Finalize(); return 0; @@ -569,9 +549,6 @@ static int test_init_global_config(void) putenv("PYTHONUTF8=0"); Py_UTF8Mode = 1; - /* Test initialization from global configuration variables (Py_xxx) */ - Py_SetProgramName(L"./globalvar"); - /* Py_IsolatedFlag is not tested */ Py_NoSiteFlag = 1; Py_BytesWarningFlag = 1; @@ -604,7 +581,7 @@ static int test_init_global_config(void) /* FIXME: test Py_LegacyWindowsFSEncodingFlag */ /* FIXME: test Py_LegacyWindowsStdioFlag */ - Py_Initialize(); + _testembed_initialize(); dump_config(); Py_Finalize(); return 0; @@ -666,7 +643,6 @@ static int test_init_from_config(void) putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix"); config_set_string(&config, &config.pycache_prefix, L"conf_pycache_prefix"); - Py_SetProgramName(L"./globalvar"); config_set_string(&config, &config.program_name, L"./conf_program_name"); wchar_t* argv[] = { @@ -853,7 +829,7 @@ static int test_init_compat_env(void) /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; set_all_env_vars(); - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); dump_config(); Py_Finalize(); return 0; @@ -889,7 +865,7 @@ static int test_init_env_dev_mode(void) /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; set_all_env_vars_dev_mode(); - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); dump_config(); Py_Finalize(); return 0; @@ -906,7 +882,7 @@ static int test_init_env_dev_mode_alloc(void) #else putenv("PYTHONMALLOC=mimalloc"); #endif - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); dump_config(); Py_Finalize(); return 0; @@ -1246,7 +1222,7 @@ static int test_open_code_hook(void) } Py_IgnoreEnvironmentFlag = 0; - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); result = 0; PyObject *r = PyFile_OpenCode("$$test-filename"); @@ -1310,7 +1286,7 @@ static int _test_audit(Py_ssize_t setValue) Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_hook, &sawSet); - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); if (PySys_Audit("_testembed.raise", NULL) == 0) { printf("No error raised"); @@ -1369,7 +1345,7 @@ static int test_audit_tuple(void) // we need at least one hook, otherwise code checking for // PySys_AuditTuple() is skipped. PySys_AddAuditHook(_audit_hook, &sawSet); - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); ASSERT(!PyErr_Occurred(), 0); @@ -1422,11 +1398,14 @@ static int test_audit_subinterpreter(void) { Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_subinterpreter_hook, NULL); - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); - Py_NewInterpreter(); - Py_NewInterpreter(); - Py_NewInterpreter(); + PyThreadState *tstate = PyThreadState_Get(); + for (int i = 0; i < 3; ++i) + { + Py_EndInterpreter(Py_NewInterpreter()); + PyThreadState_Swap(tstate); + } Py_Finalize(); @@ -1529,44 +1508,6 @@ static int test_audit_run_stdin(void) return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test); } -static int test_init_read_set(void) -{ - PyStatus status; - PyConfig config; - PyConfig_InitPythonConfig(&config); - - config_set_string(&config, &config.program_name, L"./init_read_set"); - - status = PyConfig_Read(&config); - if (PyStatus_Exception(status)) { - goto fail; - } - - status = PyWideStringList_Insert(&config.module_search_paths, - 1, L"test_path_insert1"); - if (PyStatus_Exception(status)) { - goto fail; - } - - status = PyWideStringList_Append(&config.module_search_paths, - L"test_path_append"); - if (PyStatus_Exception(status)) { - goto fail; - } - - /* override executable computed by PyConfig_Read() */ - config_set_string(&config, &config.executable, L"my_executable"); - init_from_config_clear(&config); - - dump_config(); - Py_Finalize(); - return 0; - -fail: - PyConfig_Clear(&config); - Py_ExitStatusException(status); -} - static int test_init_sys_add(void) { @@ -1882,9 +1823,9 @@ static int test_initconfig_get_api(void) assert(initconfig_getint(config, "dev_mode") == 1); // test PyInitConfig_GetInt() on a PyPreConfig option - assert(initconfig_getint(config, "utf8_mode") == 0); - assert(PyInitConfig_SetInt(config, "utf8_mode", 1) == 0); assert(initconfig_getint(config, "utf8_mode") == 1); + assert(PyInitConfig_SetInt(config, "utf8_mode", 0) == 0); + assert(initconfig_getint(config, "utf8_mode") == 0); // test PyInitConfig_GetStr() char *str; @@ -1955,8 +1896,16 @@ static int test_initconfig_exit(void) } +int +extension_module_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "exec_slot_ran", "yes"); +} + + static PyModuleDef_Slot extension_slots[] = { {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, extension_module_exec}, {0, NULL} }; @@ -2166,13 +2115,13 @@ static int test_unicode_id_init(void) }; // Initialize Python once without using the identifier - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); Py_Finalize(); // Now initialize Python multiple times and use the identifier. // The first _PyUnicode_FromId() call initializes the identifier index. for (int i=0; i<3; i++) { - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); PyObject *str1, *str2; @@ -2195,7 +2144,7 @@ static int test_unicode_id_init(void) static int test_init_main_interpreter_settings(void) { - _testembed_Py_Initialize(); + _testembed_initialize(); (void) PyRun_SimpleStringFlags( "import _testinternalcapi, json; " "print(json.dumps(_testinternalcapi.get_interp_settings(0)))", @@ -2206,7 +2155,7 @@ static int test_init_main_interpreter_settings(void) static void do_init(void *unused) { - _testembed_Py_Initialize(); + _testembed_initialize(); Py_Finalize(); } @@ -2274,6 +2223,277 @@ static int test_repeated_init_and_inittab(void) return 0; } +static PyObject* +create_module(PyObject* self, PyObject* spec) +{ + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (!name) { + return NULL; + } + if (PyUnicode_EqualToUTF8(name, "my_test_extension")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension); + } + if (PyUnicode_EqualToUTF8(name, "embedded_ext")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); + } + PyErr_Format(PyExc_LookupError, "static module %R not found", name); + Py_DECREF(name); + return NULL; +} + +static PyObject* +exec_module(PyObject* self, PyObject* mod) +{ + if (PyModule_Exec(mod) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef create_static_module_methods[] = { + {"create_module", create_module, METH_O, NULL}, + {"exec_module", exec_module, METH_O, NULL}, + {NULL} +}; + +static struct PyModuleDef create_static_module_def = { + PyModuleDef_HEAD_INIT, + .m_name = "create_static_module", + .m_size = 0, + .m_methods = create_static_module_methods, + .m_slots = extension_slots, +}; + +PyMODINIT_FUNC PyInit_create_static_module(void) { + return PyModuleDef_Init(&create_static_module_def); +} + +static int +test_create_module_from_initfunc(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + // Multi-phase initialization + L"import my_test_extension;" + L"print(my_test_extension);" + L"print(f'{my_test_extension.executed=}');" + L"print(f'{my_test_extension.exec_slot_ran=}');" + // Single-phase initialization + L"import embedded_ext;" + L"print(embedded_ext);" + L"print(f'{embedded_ext.executed=}');" + }; + PyConfig config; + if (PyImport_AppendInittab("create_static_module", + &PyInit_create_static_module) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + int result = PyRun_SimpleString( + "import sys\n" + "from importlib.util import spec_from_loader\n" + "import create_static_module\n" + "class StaticExtensionImporter:\n" + " _ORIGIN = \"static-extension\"\n" + " @classmethod\n" + " def find_spec(cls, fullname, path, target=None):\n" + " if fullname in {'my_test_extension', 'embedded_ext'}:\n" + " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" + " return None\n" + " @staticmethod\n" + " def create_module(spec):\n" + " return create_static_module.create_module(spec)\n" + " @staticmethod\n" + " def exec_module(module):\n" + " create_static_module.exec_module(module)\n" + " module.executed = 'yes'\n" + "sys.meta_path.append(StaticExtensionImporter)\n" + ); + if (result < 0) { + fprintf(stderr, "PyRun_SimpleString() failed\n"); + return 1; + } + return Py_RunMain(); +} + +/// Multi-phase initialization package & submodule /// + +int +mp_pkg_exec(PyObject *mod) +{ + // make this a namespace package + // empty list = namespace package + if (PyModule_Add(mod, "__path__", PyList_New(0)) < 0) { + return -1; + } + if (PyModule_AddStringConstant(mod, "mp_pkg_exec_slot_ran", "yes") < 0) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot mp_pkg_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, mp_pkg_exec}, + {0, NULL} +}; + +static struct PyModuleDef mp_pkg_def = { + PyModuleDef_HEAD_INIT, + .m_name = "mp_pkg", + .m_size = 0, + .m_slots = mp_pkg_slots, +}; + +PyMODINIT_FUNC +PyInit_mp_pkg(void) +{ + return PyModuleDef_Init(&mp_pkg_def); +} + +static PyObject * +submod_greet(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("Hello from sub-module"); +} + +static PyMethodDef submod_methods[] = { + {"greet", submod_greet, METH_NOARGS, NULL}, + {NULL}, +}; + +int +mp_submod_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "mp_submod_exec_slot_ran", "yes"); +} + +static PyModuleDef_Slot mp_submod_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, mp_submod_exec}, + {0, NULL} +}; + +static struct PyModuleDef mp_submod_def = { + PyModuleDef_HEAD_INIT, + .m_name = "mp_pkg.mp_submod", + .m_size = 0, + .m_methods = submod_methods, + .m_slots = mp_submod_slots, +}; + +PyMODINIT_FUNC +PyInit_mp_submod(void) +{ + return PyModuleDef_Init(&mp_submod_def); +} + +static int +test_inittab_submodule_multiphase(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + L"import sys;" + L"import mp_pkg.mp_submod;" + L"print(mp_pkg.mp_submod);" + L"print(sys.modules['mp_pkg.mp_submod']);" + L"print(mp_pkg.mp_submod.greet());" + L"print(f'{mp_pkg.mp_submod.mp_submod_exec_slot_ran=}');" + L"print(f'{mp_pkg.mp_pkg_exec_slot_ran=}');" + }; + PyConfig config; + if (PyImport_AppendInittab("mp_pkg", + &PyInit_mp_pkg) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + if (PyImport_AppendInittab("mp_pkg.mp_submod", + &PyInit_mp_submod) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + return Py_RunMain(); +} + +/// Single-phase initialization package & submodule /// + +static struct PyModuleDef sp_pkg_def = { + PyModuleDef_HEAD_INIT, + .m_name = "sp_pkg", + .m_size = 0, +}; + +PyMODINIT_FUNC +PyInit_sp_pkg(void) +{ + PyObject *mod = PyModule_Create(&sp_pkg_def); + if (mod == NULL) { + return NULL; + } + // make this a namespace package + // empty list = namespace package + if (PyModule_Add(mod, "__path__", PyList_New(0)) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static struct PyModuleDef sp_submod_def = { + PyModuleDef_HEAD_INIT, + .m_name = "sp_pkg.sp_submod", + .m_size = 0, + .m_methods = submod_methods, +}; + +PyMODINIT_FUNC +PyInit_sp_submod(void) +{ + return PyModule_Create(&sp_submod_def); +} + +static int +test_inittab_submodule_singlephase(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + L"import sys;" + L"import sp_pkg.sp_submod;" + L"print(sp_pkg.sp_submod);" + L"print(sys.modules['sp_pkg.sp_submod']);" + L"print(sp_pkg.sp_submod.greet());" + }; + PyConfig config; + if (PyImport_AppendInittab("sp_pkg", + &PyInit_sp_pkg) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + if (PyImport_AppendInittab("sp_pkg.sp_submod", + &PyInit_sp_submod) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + return Py_RunMain(); +} + static void wrap_allocator(PyMemAllocatorEx *allocator); static void unwrap_allocator(PyMemAllocatorEx *allocator); @@ -2331,7 +2551,7 @@ unwrap_allocator(PyMemAllocatorEx *allocator) static int test_get_incomplete_frame(void) { - _testembed_Py_InitializeFromConfig(); + _testembed_initialize(); PyMemAllocatorEx allocator; wrap_allocator(&allocator); // Force an allocation with an incomplete (generator) frame: @@ -2341,6 +2561,32 @@ test_get_incomplete_frame(void) return result; } +static void +do_gilstate_ensure(void *event_ptr) +{ + PyEvent *event = (PyEvent *)event_ptr; + // Signal to the calling thread that we've started + _PyEvent_Notify(event); + PyGILState_Ensure(); // This should hang + assert(NULL); +} + +static int +test_gilstate_after_finalization(void) +{ + _testembed_initialize(); + Py_Finalize(); + PyThread_handle_t handle; + PyThread_ident_t ident; + PyEvent event = {0}; + if (PyThread_start_joinable_thread(&do_gilstate_ensure, &event, &ident, &handle) < 0) { + return -1; + } + PyEvent_Wait(&event); + // We're now pretty confident that the thread went for + // PyGILState_Ensure(), but that means it got hung. + return PyThread_detach_thread(handle); +} /* ********************************************************* * List of test cases and the function that implements it. @@ -2395,7 +2641,6 @@ static struct TestCase TestCases[] = { {"test_preinit_isolated2", test_preinit_isolated2}, {"test_preinit_parse_argv", test_preinit_parse_argv}, {"test_preinit_dont_parse_argv", test_preinit_dont_parse_argv}, - {"test_init_read_set", test_init_read_set}, {"test_init_run_main", test_init_run_main}, {"test_init_sys_add", test_init_sys_add}, {"test_init_setpath", test_init_setpath}, @@ -2431,7 +2676,10 @@ static struct TestCase TestCases[] = { {"test_frozenmain", test_frozenmain}, #endif {"test_get_incomplete_frame", test_get_incomplete_frame}, - + {"test_gilstate_after_finalization", test_gilstate_after_finalization}, + {"test_create_module_from_initfunc", test_create_module_from_initfunc}, + {"test_inittab_submodule_multiphase", test_inittab_submodule_multiphase}, + {"test_inittab_submodule_singlephase", test_inittab_submodule_singlephase}, {NULL, NULL} }; diff --git a/Programs/freeze_test_frozenmain.py b/Programs/freeze_test_frozenmain.py index 848fc31b3d6..1a986bbac2a 100644 --- a/Programs/freeze_test_frozenmain.py +++ b/Programs/freeze_test_frozenmain.py @@ -24,7 +24,7 @@ def dump(fp, filename, name): with tokenize.open(filename) as source_fp: source = source_fp.read() - code = compile(source, code_filename, 'exec') + code = compile(source, code_filename, 'exec', module=name) data = marshal.dumps(code) writecode(fp, name, data) diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index b7d23f57018..dbeedb7ffe0 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,6 +1,6 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { - 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, + 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,243,184,0,0,0,128,0,94,0,82,1, 73,0,116,0,94,0,82,1,73,1,116,1,93,2,33,0, 82,2,52,1,0,0,0,0,0,0,31,0,93,2,33,0, diff --git a/Python/Python-ast.c b/Python/Python-ast.c index df035568f84..aac24ed7d3c 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5293,7 +5293,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) else { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "Field '%U' is missing from %.400s._field_types. " + "Field %R is missing from %.400s._field_types. " "This will become an error in Python 3.15.", name, Py_TYPE(self)->tp_name ) < 0) { @@ -5328,7 +5328,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) // simple field (e.g., identifier) if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "%.400s.__init__ missing 1 required positional argument: '%U'. " + "%.400s.__init__ missing 1 required positional argument: %R. " "This will become an error in Python 3.15.", Py_TYPE(self)->tp_name, name ) < 0) { @@ -5528,6 +5528,32 @@ ast_type_replace_check(PyObject *self, Py_DECREF(unused); } } + + // Discard fields from 'expecting' that default to None + PyObject *field_types = NULL; + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), + &_Py_ID(_field_types), + &field_types) < 0) + { + Py_DECREF(expecting); + return -1; + } + if (field_types != NULL) { + Py_ssize_t pos = 0; + PyObject *field_name, *field_type; + while (PyDict_Next(field_types, &pos, &field_name, &field_type)) { + if (_PyUnion_Check(field_type)) { + // optional field + if (PySet_Discard(expecting, field_name) < 0) { + Py_DECREF(expecting); + Py_DECREF(field_types); + return -1; + } + } + } + Py_DECREF(field_types); + } + // Now 'expecting' contains the fields or attributes // that would not be filled inside ast_type_replace(). Py_ssize_t m = PySet_GET_SIZE(expecting); @@ -5770,7 +5796,7 @@ ast_repr_list(PyObject *list, int depth) for (Py_ssize_t i = 0; i < Py_MIN(length, 2); i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -5794,7 +5820,7 @@ ast_repr_list(PyObject *list, int depth) } if (i == 0 && length > 2) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ...", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ...", 5) < 0) { goto error; } } @@ -5898,7 +5924,7 @@ ast_repr_max_depth(AST_object *self, int depth) } if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { Py_DECREF(name); Py_DECREF(value_repr); goto error; diff --git a/Python/_warnings.c b/Python/_warnings.c index 39bf1b225cc..d44d414bc93 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -6,7 +6,6 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttr() #include "pycore_traceback.h" // _Py_DisplaySourceLine() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() @@ -172,7 +171,7 @@ _PyWarnings_InitState(PyInterpreterState *interp) /*************************************************************************/ static int -check_matched(PyInterpreterState *interp, PyObject *obj, PyObject *arg) +check_matched(PyInterpreterState *interp, PyObject *obj, PyObject *arg, PyObject *arg2) { PyObject *result; int rc; @@ -183,6 +182,9 @@ check_matched(PyInterpreterState *interp, PyObject *obj, PyObject *arg) /* An internal plain text default filter must match exactly */ if (PyUnicode_CheckExact(obj)) { + if (arg == NULL) { + return 0; + } int cmp_result = PyUnicode_Compare(obj, arg); if (cmp_result == -1 && PyErr_Occurred()) { return -1; @@ -191,10 +193,19 @@ check_matched(PyInterpreterState *interp, PyObject *obj, PyObject *arg) } /* Otherwise assume a regex filter and call its match() method */ - result = PyObject_CallMethodOneArg(obj, &_Py_ID(match), arg); + if (arg != NULL) { + result = PyObject_CallMethodOneArg(obj, &_Py_ID(match), arg); + } + else { + PyObject *match = PyImport_ImportModuleAttrString("_py_warnings", "_match_filename"); + if (match == NULL) { + return -1; + } + result = PyObject_CallFunctionObjArgs(match, obj, arg2, NULL); + Py_DECREF(match); + } if (result == NULL) return -1; - rc = PyObject_IsTrue(result); Py_DECREF(result); return rc; @@ -424,7 +435,7 @@ get_default_action(PyInterpreterState *interp) static bool filter_search(PyInterpreterState *interp, PyObject *category, PyObject *text, Py_ssize_t lineno, - PyObject *module, char *list_name, PyObject *filters, + PyObject *module, PyObject *filename, char *list_name, PyObject *filters, PyObject **item, PyObject **matched_action) { bool result = true; *matched_action = NULL; @@ -460,14 +471,14 @@ filter_search(PyInterpreterState *interp, PyObject *category, break; } - good_msg = check_matched(interp, msg, text); + good_msg = check_matched(interp, msg, text, NULL); if (good_msg == -1) { Py_DECREF(tmp_item); result = false; break; } - good_mod = check_matched(interp, mod, module); + good_mod = check_matched(interp, mod, module, filename); if (good_mod == -1) { Py_DECREF(tmp_item); result = false; @@ -505,7 +516,7 @@ filter_search(PyInterpreterState *interp, PyObject *category, static PyObject* get_filter(PyInterpreterState *interp, PyObject *category, PyObject *text, Py_ssize_t lineno, - PyObject *module, PyObject **item) + PyObject *module, PyObject *filename, PyObject **item) { #ifdef Py_DEBUG WarningsState *st = warnings_get_state(interp); @@ -523,7 +534,7 @@ get_filter(PyInterpreterState *interp, PyObject *category, use_global_filters = true; } else { PyObject *context_action = NULL; - if (!filter_search(interp, category, text, lineno, module, "_warnings_context _filters", + if (!filter_search(interp, category, text, lineno, module, filename, "_warnings_context _filters", context_filters, item, &context_action)) { Py_DECREF(context_filters); return NULL; @@ -542,7 +553,7 @@ get_filter(PyInterpreterState *interp, PyObject *category, if (filters == NULL) { return NULL; } - if (!filter_search(interp, category, text, lineno, module, "filters", + if (!filter_search(interp, category, text, lineno, module, filename, "filters", filters, item, &action)) { return NULL; } @@ -613,39 +624,6 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, return 0; } -/* New reference. */ -static PyObject * -normalize_module(PyObject *filename) -{ - PyObject *module; - int kind; - const void *data; - Py_ssize_t len; - - len = PyUnicode_GetLength(filename); - if (len < 0) - return NULL; - - if (len == 0) - return PyUnicode_FromString("<unknown>"); - - kind = PyUnicode_KIND(filename); - data = PyUnicode_DATA(filename); - - /* if filename.endswith(".py"): */ - if (len >= 3 && - PyUnicode_READ(kind, data, len-3) == '.' && - PyUnicode_READ(kind, data, len-2) == 'p' && - PyUnicode_READ(kind, data, len-1) == 'y') - { - module = PyUnicode_Substring(filename, 0, len-3); - } - else { - module = Py_NewRef(filename); - } - return module; -} - static int update_registry(PyInterpreterState *interp, PyObject *registry, PyObject *text, PyObject *category, int add_zero) @@ -678,7 +656,7 @@ show_warning(PyThreadState *tstate, PyObject *filename, int lineno, goto error; } - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &f_stderr) <= 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &f_stderr) <= 0) { fprintf(stderr, "lost sys.stderr\n"); goto error; } @@ -813,22 +791,9 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, return NULL; } - /* Normalize module. */ - if (module == NULL) { - module = normalize_module(filename); - if (module == NULL) - return NULL; - } - else - Py_INCREF(module); - /* Normalize message. */ Py_INCREF(message); /* DECREF'ed in cleanup. */ - rc = PyObject_IsInstance(message, PyExc_Warning); - if (rc == -1) { - goto cleanup; - } - if (rc == 1) { + if (PyObject_TypeCheck(message, (PyTypeObject *)PyExc_Warning)) { text = PyObject_Str(message); if (text == NULL) goto cleanup; @@ -863,7 +828,7 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, /* Else this warning hasn't been generated before. */ } - action = get_filter(interp, category, text, lineno, module, &item); + action = get_filter(interp, category, text, lineno, module, filename, &item); if (action == NULL) goto cleanup; @@ -926,7 +891,6 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, Py_XDECREF(key); Py_XDECREF(text); Py_XDECREF(lineno_obj); - Py_DECREF(module); Py_XDECREF(message); return result; /* Py_None or NULL. */ } @@ -1125,26 +1089,25 @@ setup_context(Py_ssize_t stack_level, static PyObject * get_category(PyObject *message, PyObject *category) { - int rc; - - /* Get category. */ - rc = PyObject_IsInstance(message, PyExc_Warning); - if (rc == -1) - return NULL; - - if (rc == 1) - category = (PyObject*)Py_TYPE(message); - else if (category == NULL || category == Py_None) - category = PyExc_UserWarning; + if (PyObject_TypeCheck(message, (PyTypeObject *)PyExc_Warning)) { + /* Ignore the category argument. */ + return (PyObject*)Py_TYPE(message); + } + if (category == NULL || category == Py_None) { + return PyExc_UserWarning; + } /* Validate category. */ - rc = PyObject_IsSubclass(category, PyExc_Warning); - /* category is not a subclass of PyExc_Warning or - PyObject_IsSubclass raised an error */ - if (rc == -1 || rc == 0) { + if (!PyType_Check(category)) { PyErr_Format(PyExc_TypeError, - "category must be a Warning subclass, not '%s'", - Py_TYPE(category)->tp_name); + "category must be a Warning subclass, not '%T'", + category); + return NULL; + } + if (!PyType_IsSubtype((PyTypeObject *)category, (PyTypeObject *)PyExc_Warning)) { + PyErr_Format(PyExc_TypeError, + "category must be a Warning subclass, not class '%N'", + (PyTypeObject *)category); return NULL; } @@ -1479,28 +1442,6 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message, return 0; } -/* Like PyErr_WarnExplicitObject, but automatically sets up context */ -int -_PyErr_WarnExplicitObjectWithContext(PyObject *category, PyObject *message, - PyObject *filename, int lineno) -{ - PyObject *unused_filename, *module, *registry; - int unused_lineno; - int stack_level = 1; - - if (!setup_context(stack_level, NULL, &unused_filename, &unused_lineno, - &module, &registry)) { - return -1; - } - - int rc = PyErr_WarnExplicitObject(category, message, filename, lineno, - module, registry); - Py_DECREF(unused_filename); - Py_DECREF(registry); - Py_DECREF(module); - return rc; -} - int PyErr_WarnExplicit(PyObject *category, const char *text, const char *filename_str, int lineno, diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 0a3265dfeee..93adae3d990 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,5 +1,9 @@ .text +#if defined(__APPLE__) + .globl __Py_trampoline_func_start +#else .globl _Py_trampoline_func_start +#endif # The following assembly is equivalent to: # PyObject * # trampoline(PyThreadState *ts, _PyInterpreterFrame *f, @@ -7,11 +11,19 @@ # { # return evaluator(ts, f, throwflag); # } +#if defined(__APPLE__) +__Py_trampoline_func_start: +#else _Py_trampoline_func_start: +#endif #ifdef __x86_64__ - sub $8, %rsp - call *%rcx - add $8, %rsp +#if defined(__CET__) && (__CET__ & 1) + endbr64 +#endif + push %rbp + mov %rsp, %rbp + call *%rcx + pop %rbp ret #endif // __x86_64__ #if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) @@ -31,6 +43,30 @@ _Py_trampoline_func_start: addi sp,sp,16 jr ra #endif +#if defined(__APPLE__) + .globl __Py_trampoline_func_end +__Py_trampoline_func_end: +#else .globl _Py_trampoline_func_end _Py_trampoline_func_end: .section .note.GNU-stack,"",@progbits +#endif +# Note for indicating the assembly code supports CET +#if defined(__x86_64__) && defined(__CET__) && (__CET__ & 1) + .section .note.gnu.property,"a" + .align 8 + .long 1f - 0f + .long 4f - 1f + .long 5 +0: + .string "GNU" +1: + .align 8 + .long 0xc0000002 + .long 3f - 2f +2: + .long 0x3 +3: + .align 8 +4: +#endif // __x86_64__ diff --git a/Python/ast_preprocess.c b/Python/ast_preprocess.c index bafd67ed790..d45435257cc 100644 --- a/Python/ast_preprocess.c +++ b/Python/ast_preprocess.c @@ -16,9 +16,11 @@ typedef struct { typedef struct { PyObject *filename; + PyObject *module; int optimize; int ff_features; int syntax_check_only; + int enable_warnings; _Py_c_array_t cf_finally; /* context for PEP 765 check */ int cf_finally_used; @@ -70,7 +72,8 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTPreprocessState } int ret = _PyErr_EmitSyntaxWarning(msg, state->filename, n->lineno, n->col_offset + 1, n->end_lineno, - n->end_col_offset + 1); + n->end_col_offset + 1, + state->module); Py_DECREF(msg); return ret < 0 ? 0 : 1; } @@ -78,7 +81,7 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTPreprocessState static int before_return(_PyASTPreprocessState *state, stmt_ty node_) { - if (state->cf_finally_used > 0) { + if (state->enable_warnings && state->cf_finally_used > 0) { ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); if (ctx->in_finally && ! ctx->in_funcdef) { if (!control_flow_in_finally_warning("return", node_, state)) { @@ -92,7 +95,7 @@ before_return(_PyASTPreprocessState *state, stmt_ty node_) static int before_loop_exit(_PyASTPreprocessState *state, stmt_ty node_, const char *kw) { - if (state->cf_finally_used > 0) { + if (state->enable_warnings && state->cf_finally_used > 0) { ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); if (ctx->in_finally && ! ctx->in_loop) { if (!control_flow_in_finally_warning(kw, node_, state)) { @@ -435,13 +438,38 @@ stmt_seq_remove_item(asdl_stmt_seq *stmts, Py_ssize_t idx) return 1; } +static int +remove_docstring(asdl_stmt_seq *stmts, Py_ssize_t idx, PyArena *ctx_) +{ + assert(_PyAST_GetDocString(stmts) != NULL); + // In case there's just the docstring in the body, replace it with `pass` + // keyword, so body won't be empty. + if (asdl_seq_LEN(stmts) == 1) { + stmt_ty docstring = (stmt_ty)asdl_seq_GET(stmts, 0); + stmt_ty pass = _PyAST_Pass( + docstring->lineno, docstring->col_offset, + // we know that `pass` always takes 4 chars and a single line, + // while docstring can span on multiple lines + docstring->lineno, docstring->col_offset + 4, + ctx_ + ); + if (pass == NULL) { + return 0; + } + asdl_seq_SET(stmts, 0, pass); + return 1; + } + // In case there are more than 1 body items, just remove the docstring. + return stmt_seq_remove_item(stmts, idx); +} + static int astfold_body(asdl_stmt_seq *stmts, PyArena *ctx_, _PyASTPreprocessState *state) { int docstring = _PyAST_GetDocString(stmts) != NULL; if (docstring && (state->optimize >= 2)) { /* remove the docstring */ - if (!stmt_seq_remove_item(stmts, 0)) { + if (!remove_docstring(stmts, 0, ctx_)) { return 0; } docstring = 0; @@ -943,14 +971,17 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTPreprocessState *st int _PyAST_Preprocess(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, - int ff_features, int syntax_check_only) + int ff_features, int syntax_check_only, int enable_warnings, + PyObject *module) { _PyASTPreprocessState state; memset(&state, 0, sizeof(_PyASTPreprocessState)); state.filename = filename; + state.module = module; state.optimize = optimize; state.ff_features = ff_features; state.syntax_check_only = syntax_check_only; + state.enable_warnings = enable_warnings; if (_Py_CArray_Init(&state.cf_finally, sizeof(ControlFlowInFinallyContext), 20) < 0) { return -1; } diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index c121ec096ae..c25699978cf 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -663,27 +663,15 @@ build_ftstring_body(asdl_expr_seq *values, bool is_format_spec) } static int -_write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_t first_idx, - Py_ssize_t last_idx, char prefix, PyArena *arena) +append_templatestr(PyUnicodeWriter *writer, expr_ty e) { int result = -1; - - asdl_expr_seq *new_values = _Py_asdl_expr_seq_new(last_idx - first_idx + 1, arena); - if (!new_values) { - return result; - } - - Py_ssize_t j = 0; - for (Py_ssize_t i = first_idx; i <= last_idx; ++i) { - asdl_seq_SET(new_values, j++, asdl_seq_GET(values, i)); - } - - PyObject *body = build_ftstring_body(new_values, false); + PyObject *body = build_ftstring_body(e->v.TemplateStr.values, false); if (!body) { - return result; + return -1; } - if (-1 != append_char(writer, prefix) && + if (-1 != append_charp(writer, "t") && -1 != append_repr(writer, body)) { result = 0; @@ -692,64 +680,6 @@ _write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_ return result; } -static int -append_templatestr(PyUnicodeWriter *writer, expr_ty e) -{ - PyArena *arena = _PyArena_New(); - if (!arena) { - return -1; - } - - Py_ssize_t last_idx = 0; - Py_ssize_t len = asdl_seq_LEN(e->v.TemplateStr.values); - for (Py_ssize_t i = 0; i < len; i++) { - expr_ty value = asdl_seq_GET(e->v.TemplateStr.values, i); - - // Handle implicit concat of t-strings with f-strings - if (value->kind == FormattedValue_kind) { - if (i > last_idx) { - // Create a new TemplateStr with the values between last_idx and i - // and append it to the writer. - if (_write_values_subarray(writer, e->v.TemplateStr.values, - last_idx, i - 1, 't', arena) == -1) { - goto error; - } - - if (append_charp(writer, " ") == -1) { - goto error; - } - } - - // Append the FormattedValue to the writer. - if (_write_values_subarray(writer, e->v.TemplateStr.values, - i, i, 'f', arena) == -1) { - goto error; - } - - if (i + 1 < len) { - if (append_charp(writer, " ") == -1) { - goto error; - } - } - - last_idx = i + 1; - } - } - - if (last_idx < len) { - if (_write_values_subarray(writer, e->v.TemplateStr.values, - last_idx, len - 1, 't', arena) == -1) { - goto error; - } - } - - return 0; - -error: - _PyArena_Free(arena); - return -1; -} - static int append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec) { @@ -774,30 +704,35 @@ append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec) } static int -append_interpolation_value(PyUnicodeWriter *writer, expr_ty e) +append_interpolation_str(PyUnicodeWriter *writer, PyObject *str) { const char *outer_brace = "{"; + if (PyUnicode_Find(str, _Py_LATIN1_CHR('{'), 0, 1, 1) == 0) { + /* Expression starts with a brace, split it with a space from the outer + one. */ + outer_brace = "{ "; + } + if (-1 == append_charp(writer, outer_brace)) { + return -1; + } + if (-1 == PyUnicodeWriter_WriteStr(writer, str)) { + return -1; + } + return 0; +} + +static int +append_interpolation_value(PyUnicodeWriter *writer, expr_ty e) +{ /* Grammar allows PR_TUPLE, but use >PR_TEST for adding parenthesis around a lambda with ':' */ PyObject *temp_fv_str = expr_as_unicode(e, PR_TEST + 1); if (!temp_fv_str) { return -1; } - if (PyUnicode_Find(temp_fv_str, _Py_LATIN1_CHR('{'), 0, 1, 1) == 0) { - /* Expression starts with a brace, split it with a space from the outer - one. */ - outer_brace = "{ "; - } - if (-1 == append_charp(writer, outer_brace)) { - Py_DECREF(temp_fv_str); - return -1; - } - if (-1 == PyUnicodeWriter_WriteStr(writer, temp_fv_str)) { - Py_DECREF(temp_fv_str); - return -1; - } + int result = append_interpolation_str(writer, temp_fv_str); Py_DECREF(temp_fv_str); - return 0; + return result; } static int @@ -843,7 +778,7 @@ append_interpolation_format_spec(PyUnicodeWriter *writer, expr_ty e) static int append_interpolation(PyUnicodeWriter *writer, expr_ty e) { - if (-1 == append_interpolation_value(writer, e->v.Interpolation.value)) { + if (-1 == append_interpolation_str(writer, e->v.Interpolation.str)) { return -1; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3d0295ee388..c2d780ac9b9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_ast.h" // _PyAST_Validate() #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_ceval.h" // _PyEval_Vector() #include "pycore_compile.h" // _PyAST_Compile() #include "pycore_fileutils.h" // _PyFile_Flush @@ -14,9 +15,7 @@ #include "pycore_pyerrors.h" // _PyErr_NoMemory() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_pythonrun.h" // _Py_SourceAsString() -#include "pycore_sysmodule.h" // _PySys_GetRequiredAttr() -#include "pycore_tuple.h" // _PyTuple_FromArray() -#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_tuple.h" // _PyTuple_Recycle() #include "clinic/bltinmodule.c.h" @@ -124,7 +123,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, "__build_class__: name is not a string"); return NULL; } - orig_bases = _PyTuple_FromArray(args + 2, nargs - 2); + orig_bases = PyTuple_FromArray(args + 2, nargs - 2); if (orig_bases == NULL) return NULL; @@ -252,6 +251,7 @@ PyDoc_STRVAR(build_class_doc, Internal helper function used by the class statement."); /*[clinic input] +@permit_long_docstring_body __import__ as builtin___import__ name: object @@ -280,7 +280,7 @@ is the number of parent directories to search relative to the current module. static PyObject * builtin___import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=4febeda88a0cd245 input=73f4b960ea5b9dd6]*/ +/*[clinic end generated code: output=4febeda88a0cd245 input=01a3283590eae93a]*/ { return PyImport_ImportModuleLevelObject(name, globals, locals, fromlist, level); @@ -290,17 +290,17 @@ builtin___import___impl(PyObject *module, PyObject *name, PyObject *globals, /*[clinic input] abs as builtin_abs - x: object + number: object / Return the absolute value of the argument. [clinic start generated code]*/ static PyObject * -builtin_abs(PyObject *module, PyObject *x) -/*[clinic end generated code: output=b1b433b9e51356f5 input=bed4ca14e29c20d1]*/ +builtin_abs(PyObject *module, PyObject *number) +/*[clinic end generated code: output=861a9db97dee0595 input=a356196903543505]*/ { - return PyNumber_Absolute(x); + return PyNumber_Absolute(number); } /*[clinic input] @@ -426,7 +426,7 @@ builtin_ascii(PyObject *module, PyObject *obj) /*[clinic input] bin as builtin_bin - number: object + integer: object / Return the binary representation of an integer. @@ -436,10 +436,10 @@ Return the binary representation of an integer. [clinic start generated code]*/ static PyObject * -builtin_bin(PyObject *module, PyObject *number) -/*[clinic end generated code: output=b6fc4ad5e649f4f7 input=53f8a0264bacaf90]*/ +builtin_bin(PyObject *module, PyObject *integer) +/*[clinic end generated code: output=533f9388441805cc input=d16518f148341e70]*/ { - return PyNumber_ToBase(number, 2); + return PyNumber_ToBase(integer, 2); } @@ -465,7 +465,7 @@ builtin_callable(PyObject *module, PyObject *obj) static PyObject * builtin_breakpoint(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords) { - PyObject *hook = _PySys_GetRequiredAttrString("breakpointhook"); + PyObject *hook = PySys_GetAttrString("breakpointhook"); if (hook == NULL) { return NULL; } @@ -745,12 +745,13 @@ builtin_chr(PyObject *module, PyObject *i) compile as builtin_compile source: object - filename: object(converter="PyUnicode_FSDecoder") + filename: unicode_fs_decoded mode: str flags: int = 0 dont_inherit: bool = False optimize: int = -1 * + module as modname: object = None _feature_version as feature_version: int = -1 Compile source into a code object that can be executed by exec() or eval(). @@ -770,8 +771,8 @@ in addition to any features explicitly specified. static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=cc78e20e7c7682ba]*/ + int optimize, PyObject *modname, int feature_version) +/*[clinic end generated code: output=9a0dce1945917a86 input=ddeae1e0253459dc]*/ { PyObject *source_copy; const char *str; @@ -800,6 +801,15 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, "compile(): invalid optimize value"); goto error; } + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "compile() argument 'module' must be str or None, not %T", + modname); + goto error; + } if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); @@ -845,8 +855,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } int syntax_check_only = ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, - arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, arena, + syntax_check_only, modname) < 0) + { _PyArena_Free(arena); goto error; } @@ -859,7 +870,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + &cf, optimize, arena, modname); } _PyArena_Free(arena); goto finally; @@ -877,7 +888,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, tstate->suppress_co_const_immortalization++; #endif - result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); + result = _Py_CompileStringObjectWithModule(str, filename, + start[compile_mode], &cf, + optimize, modname); #ifdef Py_GIL_DISABLED tstate->suppress_co_const_immortalization--; @@ -889,7 +902,6 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, error: result = NULL; finally: - Py_DECREF(filename); return result; } @@ -958,6 +970,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) /*[clinic end generated code: output=0a0824aa70093116 input=7c7bce5299a89062]*/ { + PyThreadState *tstate = _PyThreadState_GET(); PyObject *result = NULL, *source_copy; const char *str; @@ -971,35 +984,46 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, : "globals must be a dict"); return NULL; } - if (globals == Py_None) { - globals = PyEval_GetGlobals(); - if (locals == Py_None) { - locals = _PyEval_GetFrameLocals(); - if (locals == NULL) - return NULL; - } - else { - Py_INCREF(locals); - } + + int fromframe = 0; + if (globals != Py_None) { + Py_INCREF(globals); + } + else if (_PyEval_GetFrame() != NULL) { + fromframe = 1; + globals = PyEval_GetGlobals(); + assert(globals != NULL); + Py_INCREF(globals); } - else if (locals == Py_None) - locals = Py_NewRef(globals); else { + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (!_PyErr_Occurred(tstate)) { + PyErr_SetString(PyExc_TypeError, + "eval must be given globals and locals " + "when called without a frame"); + } + return NULL; + } + Py_INCREF(globals); + } + + if (locals != Py_None) { Py_INCREF(locals); } - - if (globals == NULL || locals == NULL) { - PyErr_SetString(PyExc_TypeError, - "eval must be given globals and locals " - "when called without a frame"); - goto error; + else if (fromframe) { + locals = _PyEval_GetFrameLocals(); + if (locals == NULL) { + assert(PyErr_Occurred()); + Py_DECREF(globals); + return NULL; + } + } + else { + locals = Py_NewRef(globals); } - int r = PyDict_Contains(globals, &_Py_ID(__builtins__)); - if (r == 0) { - r = PyDict_SetItem(globals, &_Py_ID(__builtins__), PyEval_GetBuiltins()); - } - if (r < 0) { + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { goto error; } @@ -1040,6 +1064,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, } error: + Py_XDECREF(globals); Py_XDECREF(locals); return result; } @@ -1070,29 +1095,44 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals, PyObject *closure) /*[clinic end generated code: output=7579eb4e7646743d input=25e989b6d87a3a21]*/ { + PyThreadState *tstate = _PyThreadState_GET(); PyObject *v; - if (globals == Py_None) { + int fromframe = 0; + if (globals != Py_None) { + Py_INCREF(globals); + } + else if (_PyEval_GetFrame() != NULL) { + fromframe = 1; globals = PyEval_GetGlobals(); - if (locals == Py_None) { - locals = _PyEval_GetFrameLocals(); - if (locals == NULL) - return NULL; + assert(globals != NULL); + Py_INCREF(globals); + } + else { + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (!_PyErr_Occurred(tstate)) { + PyErr_SetString(PyExc_SystemError, + "globals and locals cannot be NULL"); + } + goto error; } - else { - Py_INCREF(locals); - } - if (!globals || !locals) { - PyErr_SetString(PyExc_SystemError, - "globals and locals cannot be NULL"); + Py_INCREF(globals); + } + + if (locals != Py_None) { + Py_INCREF(locals); + } + else if (fromframe) { + locals = _PyEval_GetFrameLocals(); + if (locals == NULL) { + assert(PyErr_Occurred()); + Py_DECREF(globals); return NULL; } } - else if (locals == Py_None) { - locals = Py_NewRef(globals); - } else { - Py_INCREF(locals); + locals = Py_NewRef(globals); } if (!PyDict_Check(globals)) { @@ -1106,11 +1146,8 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, Py_TYPE(locals)->tp_name); goto error; } - int r = PyDict_Contains(globals, &_Py_ID(__builtins__)); - if (r == 0) { - r = PyDict_SetItem(globals, &_Py_ID(__builtins__), PyEval_GetBuiltins()); - } - if (r < 0) { + + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { goto error; } @@ -1187,11 +1224,13 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } if (v == NULL) goto error; + Py_DECREF(globals); Py_DECREF(locals); Py_DECREF(v); Py_RETURN_NONE; error: + Py_XDECREF(globals); Py_XDECREF(locals); return NULL; } @@ -1241,10 +1280,21 @@ static PyObject * builtin_globals_impl(PyObject *module) /*[clinic end generated code: output=e5dd1527067b94d2 input=9327576f92bb48ba]*/ { - PyObject *d; - - d = PyEval_GetGlobals(); - return Py_XNewRef(d); + PyObject *globals; + if (_PyEval_GetFrame() != NULL) { + globals = PyEval_GetGlobals(); + assert(globals != NULL); + return Py_NewRef(globals); + } + PyThreadState *tstate = _PyThreadState_GET(); + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (_PyErr_Occurred(tstate)) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef(globals); } @@ -1464,34 +1514,27 @@ map_next(PyObject *self) } Py_ssize_t nargs = 0; - for (i=0; i < niters; i++) { + for (i = 0; i < niters; i++) { PyObject *it = PyTuple_GET_ITEM(lz->iters, i); PyObject *val = Py_TYPE(it)->tp_iternext(it); if (val == NULL) { if (lz->strict) { goto check; } - goto exit; + goto exit_no_result; } stack[i] = val; nargs++; } result = _PyObject_VectorcallTstate(tstate, lz->func, stack, nargs, NULL); + goto exit; -exit: - for (i=0; i < nargs; i++) { - Py_DECREF(stack[i]); - } - if (stack != small_stack) { - PyMem_Free(stack); - } - return result; check: if (PyErr_Occurred()) { if (!PyErr_ExceptionMatches(PyExc_StopIteration)) { // next() on argument i raised an exception (not StopIteration) - return NULL; + goto exit_no_result; } PyErr_Clear(); } @@ -1499,9 +1542,10 @@ map_next(PyObject *self) // ValueError: map() argument 2 is shorter than argument 1 // ValueError: map() argument 3 is shorter than arguments 1-2 const char* plural = i == 1 ? " " : "s 1-"; - return PyErr_Format(PyExc_ValueError, - "map() argument %d is shorter than argument%s%d", - i + 1, plural, i); + PyErr_Format(PyExc_ValueError, + "map() argument %d is shorter than argument%s%d", + i + 1, plural, i); + goto exit_no_result; } for (i = 1; i < niters; i++) { PyObject *it = PyTuple_GET_ITEM(lz->iters, i); @@ -1509,21 +1553,33 @@ map_next(PyObject *self) if (val) { Py_DECREF(val); const char* plural = i == 1 ? " " : "s 1-"; - return PyErr_Format(PyExc_ValueError, - "map() argument %d is longer than argument%s%d", - i + 1, plural, i); + PyErr_Format(PyExc_ValueError, + "map() argument %d is longer than argument%s%d", + i + 1, plural, i); + goto exit_no_result; } if (PyErr_Occurred()) { if (!PyErr_ExceptionMatches(PyExc_StopIteration)) { // next() on argument i raised an exception (not StopIteration) - return NULL; + goto exit_no_result; } PyErr_Clear(); } // Argument i is exhausted. So far so good... } // All arguments are exhausted. Success! - goto exit; + +exit_no_result: + assert(result == NULL); + +exit: + for (i = 0; i < nargs; i++) { + Py_DECREF(stack[i]); + } + if (stack != small_stack) { + PyMem_Free(stack); + } + return result; } static PyObject * @@ -1742,7 +1798,7 @@ builtin_hash(PyObject *module, PyObject *obj) /*[clinic input] hex as builtin_hex - number: object + integer: object / Return the hexadecimal representation of an integer. @@ -1752,10 +1808,10 @@ Return the hexadecimal representation of an integer. [clinic start generated code]*/ static PyObject * -builtin_hex(PyObject *module, PyObject *number) -/*[clinic end generated code: output=e46b612169099408 input=e645aff5fc7d540e]*/ +builtin_hex(PyObject *module, PyObject *integer) +/*[clinic end generated code: output=e5de857ba61aae08 input=3bef4746efc62fac]*/ { - return PyNumber_ToBase(number, 16); + return PyNumber_ToBase(integer, 16); } @@ -1809,7 +1865,7 @@ PyObject *PyAnextAwaitable_New(PyObject *, PyObject *); /*[clinic input] anext as builtin_anext - aiterator: object + async_iterator as aiterator: object default: object = NULL / @@ -1822,7 +1878,7 @@ it is returned instead of raising StopAsyncIteration. static PyObject * builtin_anext_impl(PyObject *module, PyObject *aiterator, PyObject *default_value) -/*[clinic end generated code: output=f02c060c163a81fa input=2900e4a370d39550]*/ +/*[clinic end generated code: output=f02c060c163a81fa input=f3dc5a93f073e5ac]*/ { PyTypeObject *t; PyObject *awaitable; @@ -1888,7 +1944,21 @@ static PyObject * builtin_locals_impl(PyObject *module) /*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ { - return _PyEval_GetFrameLocals(); + PyObject *locals; + if (_PyEval_GetFrame() != NULL) { + locals = _PyEval_GetFrameLocals(); + assert(locals != NULL || PyErr_Occurred()); + return locals; + } + PyThreadState *tstate = _PyThreadState_GET(); + locals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (locals == NULL) { + if (_PyErr_Occurred(tstate)) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef(locals); } @@ -2047,7 +2117,7 @@ With two or more positional arguments, return the largest argument."); /*[clinic input] oct as builtin_oct - number: object + integer: object / Return the octal representation of an integer. @@ -2057,25 +2127,31 @@ Return the octal representation of an integer. [clinic start generated code]*/ static PyObject * -builtin_oct(PyObject *module, PyObject *number) -/*[clinic end generated code: output=40a34656b6875352 input=ad6b274af4016c72]*/ +builtin_oct(PyObject *module, PyObject *integer) +/*[clinic end generated code: output=8c15f2145a74c390 input=b97c377b15fedf8d]*/ { - return PyNumber_ToBase(number, 8); + return PyNumber_ToBase(integer, 8); } /*[clinic input] ord as builtin_ord - c: object + character as c: object / -Return the Unicode code point for a one-character string. +Return the ordinal value of a character. + +If the argument is a one-character string, return the Unicode code +point of that character. + +If the argument is a bytes or bytearray object of length 1, return its +single byte value. [clinic start generated code]*/ static PyObject * builtin_ord(PyObject *module, PyObject *c) -/*[clinic end generated code: output=4fa5e87a323bae71 input=3064e5d6203ad012]*/ +/*[clinic end generated code: output=4fa5e87a323bae71 input=98d38480432e1177]*/ { long ord; Py_ssize_t size; @@ -2141,7 +2217,7 @@ builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp, /*[clinic input] print as builtin_print - *args: array + *objects: array sep: object(c_default="Py_None") = ' ' string inserted between values, default a space. end: object(c_default="Py_None") = '\n' @@ -2156,15 +2232,15 @@ Prints the values to a stream, or to sys.stdout by default. [clinic start generated code]*/ static PyObject * -builtin_print_impl(PyObject *module, PyObject * const *args, - Py_ssize_t args_length, PyObject *sep, PyObject *end, +builtin_print_impl(PyObject *module, PyObject * const *objects, + Py_ssize_t objects_length, PyObject *sep, PyObject *end, PyObject *file, int flush) -/*[clinic end generated code: output=3cb7e5b66f1a8547 input=66ea4de1605a2437]*/ +/*[clinic end generated code: output=38d8def56c837bcc input=ff35cb3d59ee8115]*/ { int i, err; if (file == Py_None) { - file = _PySys_GetRequiredAttr(&_Py_ID(stdout)); + file = PySys_GetAttr(&_Py_ID(stdout)); if (file == NULL) { return NULL; } @@ -2200,7 +2276,7 @@ builtin_print_impl(PyObject *module, PyObject * const *args, return NULL; } - for (i = 0; i < args_length; i++) { + for (i = 0; i < objects_length; i++) { if (i > 0) { if (sep == NULL) { err = PyFile_WriteString(" ", file); @@ -2213,7 +2289,7 @@ builtin_print_impl(PyObject *module, PyObject * const *args, return NULL; } } - err = PyFile_WriteObject(args[i], file, Py_PRINT_RAW); + err = PyFile_WriteObject(objects[i], file, Py_PRINT_RAW); if (err) { Py_DECREF(file); return NULL; @@ -2270,7 +2346,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) int tty; /* Check that stdin/out/err are intact */ - fin = _PySys_GetRequiredAttr(&_Py_ID(stdin)); + fin = PySys_GetAttr(&_Py_ID(stdin)); if (fin == NULL) { goto error; } @@ -2278,7 +2354,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyErr_SetString(PyExc_RuntimeError, "lost sys.stdin"); goto error; } - fout = _PySys_GetRequiredAttr(&_Py_ID(stdout)); + fout = PySys_GetAttr(&_Py_ID(stdout)); if (fout == NULL) { goto error; } @@ -2286,7 +2362,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) PyErr_SetString(PyExc_RuntimeError, "lost sys.stdout"); goto error; } - ferr = _PySys_GetRequiredAttr(&_Py_ID(stderr)); + ferr = PySys_GetAttr(&_Py_ID(stderr)); if (ferr == NULL) { goto error; } @@ -2624,7 +2700,22 @@ builtin_vars(PyObject *self, PyObject *args) if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) return NULL; if (v == NULL) { - d = _PyEval_GetFrameLocals(); + if (_PyEval_GetFrame() != NULL) { + d = _PyEval_GetFrameLocals(); + } + else { + PyThreadState *tstate = _PyThreadState_GET(); + d = _PyEval_GetGlobalsFromRunningMain(tstate); + if (d == NULL) { + if (!_PyErr_Occurred(tstate)) { + d = _PyEval_GetFrameLocals(); + assert(_PyErr_Occurred(tstate)); + } + } + else { + Py_INCREF(d); + } + } } else { if (PyObject_GetOptionalAttr(v, &_Py_ID(__dict__), &d) == 0) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6a076662640..4ba255d28bd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -11,7 +11,6 @@ #include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_cell.h" // PyCell_GetRef() -#include "pycore_ceval.h" #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -153,22 +152,19 @@ dummy_func( macro(NOT_TAKEN) = NOP; op(_CHECK_PERIODIC, (--)) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - int err = _Py_HandlePending(tstate); - ERROR_IF(err != 0); - } + int err = check_periodics(tstate); + ERROR_IF(err != 0); + } + + replaced op(_CHECK_PERIODIC_AT_END, (--)) { + int err = check_periodics(tstate); + ERROR_IF(err != 0); } op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - int err = _Py_HandlePending(tstate); - ERROR_IF(err != 0); - } + int err = check_periodics(tstate); + ERROR_IF(err != 0); } } @@ -181,7 +177,15 @@ dummy_func( } tier1 op(_MAYBE_INSTRUMENT, (--)) { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + // For thread-safety, we need to check instrumentation version + // even when tracing. Otherwise, another thread may concurrently + // re-write the bytecode while we are executing this function. + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -295,55 +299,18 @@ dummy_func( value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); } - family(LOAD_CONST, 0) = { - LOAD_CONST_MORTAL, - LOAD_CONST_IMMORTAL, - }; - inst(LOAD_CONST, (-- value)) { - /* We can't do this in the bytecode compiler as - * marshalling can intern strings and make them immortal. */ PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); -#if ENABLE_SPECIALIZATION_FT -#ifdef Py_GIL_DISABLED - uint8_t expected = LOAD_CONST; - if (!_Py_atomic_compare_exchange_uint8( - &this_instr->op.code, &expected, - _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { - // We might lose a race with instrumentation, which we don't care about. - assert(expected >= MIN_INSTRUMENTED_OPCODE); - } -#else - if (this_instr->op.code == LOAD_CONST) { - this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; - } -#endif -#endif - } - - inst(LOAD_CONST_MORTAL, (-- value)) { - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNewMortal(obj); - } - - inst(LOAD_CONST_IMMORTAL, (-- value)) { - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); } replicate(4) inst(LOAD_SMALL_INT, (-- value)) { assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); } replicate(8) inst(STORE_FAST, (value --)) { - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; DEAD(value); @@ -355,10 +322,6 @@ dummy_func( }; inst(STORE_FAST_LOAD_FAST, (value1 -- value2)) { - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value1) - ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); @@ -369,14 +332,6 @@ dummy_func( } inst(STORE_FAST_STORE_FAST, (value2, value1 --)) { - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value1) - ); - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value2) - ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); @@ -390,7 +345,33 @@ dummy_func( } pure inst(POP_TOP, (value --)) { - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); + } + + op(_POP_TOP_NOP, (value --)) { + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + DEAD(value); + } + + op(_POP_TOP_INT, (value --)) { + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + + op(_POP_TOP_FLOAT, (value --)) { + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + + op(_POP_TOP_UNICODE, (value --)) { + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + + tier2 op(_POP_TWO, (nos, tos --)) { + PyStackRef_CLOSE(tos); + PyStackRef_CLOSE(nos); } pure inst(PUSH_NULL, (-- res)) { @@ -406,9 +387,14 @@ dummy_func( PyStackRef_CLOSE(value); } - macro(POP_ITER) = POP_TOP; - no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) { + inst(POP_ITER, (iter, index_or_null -- )) { + (void)index_or_null; + DEAD(index_or_null); + PyStackRef_CLOSE(iter); + } + + no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, index_or_null, value -- receiver, index_or_null)) { /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyStackRef_GenCheck(receiver)) { @@ -420,7 +406,9 @@ dummy_func( PyStackRef_CLOSE(value); } - tier1 inst(INSTRUMENTED_POP_ITER, (iter -- )) { + tier1 inst(INSTRUMENTED_POP_ITER, (iter, index_or_null -- )) { + (void)index_or_null; + DEAD(index_or_null); INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); PyStackRef_CLOSE(iter); } @@ -606,12 +594,24 @@ dummy_func( op(_GUARD_NOS_INT, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - EXIT_IF(!PyLong_CheckExact(left_o)); + EXIT_IF(!_PyLong_CheckExactAndCompact(left_o)); } op(_GUARD_TOS_INT, (value -- value)) { PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - EXIT_IF(!PyLong_CheckExact(value_o)); + EXIT_IF(!_PyLong_CheckExactAndCompact(value_o)); + } + + op(_GUARD_NOS_OVERFLOWED, (left, unused -- left, unused)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + EXIT_IF(!_PyLong_IsCompact((PyLongObject *)left_o)); + } + + op(_GUARD_TOS_OVERFLOWED, (value -- value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + EXIT_IF(!_PyLong_IsCompact((PyLongObject *)value_o)); } pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { @@ -619,14 +619,14 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + EXIT_IF(PyStackRef_IsNull(res)); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_ADD_INT, (left, right -- res)) { @@ -634,14 +634,14 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + EXIT_IF(PyStackRef_IsNull(res)); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); } pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { @@ -649,20 +649,22 @@ dummy_func( PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + EXIT_IF(PyStackRef_IsNull(res)); PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); - ERROR_IF(res_o == NULL); - res = PyStackRef_FromPyObjectSteal(res_o); } macro(BINARY_OP_MULTIPLY_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT; + macro(BINARY_OP_ADD_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT; + macro(BINARY_OP_SUBTRACT_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT; @@ -721,6 +723,52 @@ dummy_func( ERROR_IF(PyStackRef_IsNull(res)); } + + pure op(_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + + pure op(_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, (left, right -- res)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + INPUTS_DEAD(); + ERROR_IF(PyStackRef_IsNull(res)); + } + macro(BINARY_OP_MULTIPLY_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT; macro(BINARY_OP_ADD_FLOAT) = @@ -762,7 +810,7 @@ dummy_func( assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); @@ -806,7 +854,7 @@ dummy_func( DEOPT_IF(!res); } - pure op(_BINARY_OP_EXTEND, (descr/4, left, right -- res)) { + op(_BINARY_OP_EXTEND, (descr/4, left, right -- res)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); @@ -942,7 +990,7 @@ dummy_func( PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(str_st); - res = PyStackRef_FromPyObjectImmortal(res_o); + res = PyStackRef_FromPyObjectBorrow(res_o); } op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { @@ -1022,12 +1070,13 @@ dummy_func( STAT_INC(BINARY_OP, hit); } - op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _PyInterpreterFrame* )) { - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; + op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; INPUTS_DEAD(); frame->return_offset = INSTRUCTION_SIZE; + new_frame = PyStackRef_Wrap(pushed_frame); } macro(BINARY_OP_SUBSCR_GETITEM) = @@ -1169,7 +1218,7 @@ dummy_func( tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); -#if !Py_TAIL_CALL_INTERP +#if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 @@ -1333,20 +1382,21 @@ dummy_func( macro(SEND) = _SPECIALIZE_SEND + _SEND; - op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame: _PyInterpreterFrame *)) { + op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); DEAD(v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); - gen_frame->previous = frame; + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); } macro(SEND_GEN) = @@ -1458,7 +1508,7 @@ dummy_func( tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value_st -- none, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !Py_TAIL_CALL_INTERP + #if !_Py_TAIL_CALL_INTERP assert(throwflag); #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); @@ -1926,14 +1976,8 @@ dummy_func( } inst(BUILD_STRING, (pieces[oparg] -- str)) { - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - DECREF_INPUTS(); + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); + DEAD(pieces); ERROR_IF(str_o == NULL); str = PyStackRef_FromPyObjectSteal(str_o); } @@ -2048,17 +2092,9 @@ dummy_func( } inst(BUILD_MAP, (values[oparg*2] -- map)) { - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - DECREF_INPUTS(); + + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); + DEAD(values); ERROR_IF(map_o == NULL); map = PyStackRef_FromPyObjectStealMortal(map_o); } @@ -2281,19 +2317,18 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { + op(_LOAD_ATTR, (owner -- attr[1], self_or_null[oparg&1])) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + *attr = PyStackRef_NULL; + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr_o != NULL); // No errors on this branch + assert(!PyStackRef_IsNull(*attr)); // No errors on this branch self_or_null[0] = owner; // Transfer ownership DEAD(owner); } @@ -2305,17 +2340,17 @@ dummy_func( meth | NULL | arg1 | ... | argN */ PyStackRef_CLOSE(owner); - ERROR_IF(attr_o == NULL); + ERROR_IF(PyStackRef_IsNull(*attr)); self_or_null[0] = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); PyStackRef_CLOSE(owner); ERROR_IF(attr_o == NULL); + *attr = PyStackRef_FromPyObjectSteal(attr_o); } - attr = PyStackRef_FromPyObjectSteal(attr_o); } macro(LOAD_ATTR) = @@ -2500,7 +2535,7 @@ dummy_func( _LOAD_ATTR_CLASS + _PUSH_NULL_CONDITIONAL; - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _PyInterpreterFrame *)) { + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; @@ -2510,9 +2545,10 @@ dummy_func( DEOPT_IF(code->co_argcount != 1); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; DEAD(owner); + new_frame = PyStackRef_Wrap(pushed_frame); } macro(LOAD_ATTR_PROPERTY) = @@ -2595,12 +2631,6 @@ dummy_func( PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); DEOPT_IF(!LOCK_OBJECT(dict)); - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true); - } - #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || @@ -2722,8 +2752,8 @@ dummy_func( PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left_o)); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right_o)); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -2894,10 +2924,11 @@ dummy_func( JUMP_BACKWARD_JIT, }; - tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { + specializing tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); // Need to re-dispatch so the warmup counter isn't off by one: next_instr = this_instr; DISPATCH_SAME_OPARG(); @@ -2908,23 +2939,21 @@ dummy_func( tier1 op(_JIT, (--)) { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ + _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; - start--; + insert_exec_at--; } - _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - ERROR_IF(optimized < 0); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); } else { - this_instr[1].counter = initial_jump_backoff_counter(); - assert(tstate->current_executor == NULL); - GOTO_TIER_TWO(executor); + this_instr[1].counter = restart_backoff_counter(counter); } } else { @@ -2970,6 +2999,10 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + goto stop_tracing; + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -2988,7 +3021,9 @@ dummy_func( } DISPATCH_GOTO(); } - GOTO_TIER_TWO(executor); + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; + TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ @@ -3029,7 +3064,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + replaced inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3085,15 +3120,24 @@ dummy_func( values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); } - inst(GET_ITER, (iterable -- iter)) { + inst(GET_ITER, (iterable -- iter, index_or_null)) { #ifdef Py_STATS _Py_GatherStats_GetIter(iterable); #endif /* before: [obj]; after [getiter(obj)] */ - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - PyStackRef_CLOSE(iterable); - ERROR_IF(iter_o == NULL); - iter = PyStackRef_FromPyObjectSteal(iter_o); + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + DEAD(iterable); + index_or_null = PyStackRef_TagInt(0); + } + else { + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + PyStackRef_CLOSE(iterable); + ERROR_IF(iter_o == NULL); + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + } } inst(GET_YIELD_FROM_ITER, (iterable -- iter)) { @@ -3140,11 +3184,11 @@ dummy_func( FOR_ITER_GEN, }; - specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { + specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter, null_or_index -- iter, null_or_index)) { #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -3152,111 +3196,71 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - replaced op(_FOR_ITER, (iter -- iter, next)) { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); - } - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + replaced op(_FOR_ITER, (iter, null_or_index -- iter, null_or_index, next)) { + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + ERROR_NO_POP(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Jump forward oparg, then skip following END_FOR */ + // Jump forward by oparg and skip the following END_FOR JUMPBY(oparg + 1); DISPATCH(); } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator + next = item; } - op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); - } - _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); - _PyErr_Clear(tstate); + op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + ERROR_NO_POP(); } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ EXIT_IF(true); } - next = PyStackRef_FromPyObjectSteal(next_o); - // Common case: no jump, leave it to the code generator + next = item; } + macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; - inst(INSTRUMENTED_FOR_ITER, (unused/1, iter -- iter, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - if (_PyErr_Occurred(tstate)) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (!matches) { - ERROR_NO_POP(); - } - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + inst(INSTRUMENTED_FOR_ITER, (unused/1, iter, null_or_index -- iter, null_or_index, next)) { + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + ERROR_NO_POP(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - /* Skip END_FOR */ + // Jump forward by oparg and skip the following END_FOR JUMPBY(oparg + 1); DISPATCH(); } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); } - - op(_ITER_CHECK_LIST, (iter -- iter)) { + op(_ITER_CHECK_LIST, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - EXIT_IF(Py_TYPE(iter_o) != &PyListIter_Type); + EXIT_IF(Py_TYPE(iter_o) != &PyList_Type); + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - EXIT_IF(!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)); + EXIT_IF(!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)); #endif } - replaced op(_ITER_JUMP_LIST, (iter -- iter)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(iter_o) == &PyListIter_Type); -// For free-threaded Python, the loop exit can happen at any point during -// item retrieval, so it doesn't make much sense to check and jump -// separately before item retrieval. Any length check we do here can be -// invalid by the time we actually try to fetch the item. + replaced op(_ITER_JUMP_LIST, (iter, null_or_index -- iter, null_or_index)) { #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - (void)iter_o; + // For free-threaded Python, the loop exit can happen at any point during + // item retrieval, so it doesn't make much sense to check and jump + // separately before item retrieval. Any length check we do here can be + // invalid by the time we actually try to fetch the item. #else - _PyListIterObject *it = (_PyListIterObject *)iter_o; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3265,73 +3269,54 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) { + op(_GUARD_NOT_EXHAUSTED_LIST, (iter, null_or_index -- iter, null_or_index)) { #ifndef Py_GIL_DISABLED - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - EXIT_IF(seq == NULL); - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - EXIT_IF(1); - } + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + EXIT_IF((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)); #endif } - replaced op(_ITER_NEXT_LIST, (iter -- iter, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + replaced op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); // A negative result means we lost a race with another thread // and we need to take the slow path. DEOPT_IF(result < 0); if (result == 0) { - it->it_index = -1; + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } // Only used by Tier 2 - op(_ITER_NEXT_LIST_TIER_TWO, (iter -- iter, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + op(_ITER_NEXT_LIST_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); // A negative result means we lost a race with another thread // and we need to take the slow path. - EXIT_IF(result < 0); - if (result == 0) { - it->it_index = -1; - EXIT_IF(1); - } - it->it_index++; + DEOPT_IF(result <= 0); #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } macro(FOR_ITER_LIST) = @@ -3340,31 +3325,19 @@ dummy_func( _ITER_JUMP_LIST + _ITER_NEXT_LIST; - op(_ITER_CHECK_TUPLE, (iter -- iter)) { + op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - EXIT_IF(Py_TYPE(iter_o) != &PyTupleIter_Type); -#ifdef Py_GIL_DISABLED - EXIT_IF(!_PyObject_IsUniquelyReferenced(iter_o)); -#endif + EXIT_IF(Py_TYPE(iter_o) != &PyTuple_Type); + assert(PyStackRef_IsTaggedInt(null_or_index)); } - replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - (void)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + replaced op(_ITER_JUMP_TUPLE, (iter, null_or_index -- iter, null_or_index)) { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { -#ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - Py_DECREF(seq); - } -#endif + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3372,29 +3345,19 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter -- iter)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - PyTupleObject *seq = it->it_seq; - EXIT_IF(seq == NULL); - EXIT_IF(it->it_index >= PyTuple_GET_SIZE(seq)); + op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter, null_or_index -- iter, null_or_index)) { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + EXIT_IF((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)); } - op(_ITER_NEXT_TUPLE, (iter -- iter, next)) { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; -#ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); -#endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + op(_ITER_NEXT_TUPLE, (iter, null_or_index -- iter, null_or_index, next)) { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } macro(FOR_ITER_TUPLE) = @@ -3403,7 +3366,7 @@ dummy_func( _ITER_JUMP_TUPLE + _ITER_NEXT_TUPLE; - op(_ITER_CHECK_RANGE, (iter -- iter)) { + op(_ITER_CHECK_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(r) != &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3411,7 +3374,7 @@ dummy_func( #endif } - replaced op(_ITER_JUMP_RANGE, (iter -- iter)) { + replaced op(_ITER_JUMP_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3426,13 +3389,13 @@ dummy_func( } // Only used by Tier 2 - op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) { + op(_GUARD_NOT_EXHAUSTED_RANGE, (iter, null_or_index -- iter, null_or_index)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); EXIT_IF(r->len <= 0); } - op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -3453,7 +3416,7 @@ dummy_func( _ITER_JUMP_RANGE + _ITER_NEXT_RANGE; - op(_FOR_ITER_GEN_FRAME, (iter -- iter, gen_frame: _PyInterpreterFrame*)) { + op(_FOR_ITER_GEN_FRAME, (iter, null -- iter, null, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); #ifdef Py_GIL_DISABLED @@ -3465,14 +3428,15 @@ dummy_func( #endif DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; + pushed_frame->previous = frame; // oparg is the return offset from the next instruction. frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); } macro(FOR_ITER_GEN) = @@ -3711,7 +3675,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -3821,10 +3785,10 @@ dummy_func( ERROR_IF(err); } - macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; - macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC_AT_END; + macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC_AT_END; - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); // oparg counts all of the args, but *not* self: @@ -3846,7 +3810,7 @@ dummy_func( if (temp == NULL) { ERROR_NO_POP(); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { @@ -3913,27 +3877,20 @@ dummy_func( #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -3943,7 +3900,7 @@ dummy_func( unused/2 + _CHECK_IS_NOT_PY_CALLABLE + _CALL_NON_PY_GENERAL + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { EXIT_IF(!PyStackRef_IsNull(null)); @@ -3983,27 +3940,26 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } INPUTS_DEAD(); + new_frame = PyStackRef_Wrap(pushed_frame); } - op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- )) { - // Write it out explicitly because it's subtly different. - // Eventually this should be the only occurrence of this code. + op(_PUSH_FRAME, (new_frame -- )) { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); DEAD(new_frame); SYNC_SP(); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -4041,6 +3997,15 @@ dummy_func( DEOPT_IF(!PyStackRef_IsNull(null)); } + op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + EXIT_IF(o == NULL); + } + + op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { + DEOPT_IF(!PyStackRef_IsNull(null)); + } + op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); DEOPT_IF(callable_o != (PyObject *)&PyType_Type); @@ -4092,7 +4057,7 @@ dummy_func( _GUARD_NOS_NULL + _GUARD_CALLABLE_STR_1 + _CALL_STR_1 + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4120,7 +4085,7 @@ dummy_func( _GUARD_NOS_NULL + _GUARD_CALLABLE_TUPLE_1 + _CALL_TUPLE_1 + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4146,7 +4111,7 @@ dummy_func( PyStackRef_CLOSE(temp); } - op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame: _PyInterpreterFrame *)) { + op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); @@ -4163,12 +4128,12 @@ dummy_func( _PyEval_FrameClearAndPop(tstate, shim); ERROR_NO_POP(); } - init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); } macro(CALL_ALLOC_AND_ENTER_INIT) = @@ -4200,14 +4165,13 @@ dummy_func( } DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - DECREF_INPUTS(); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4216,7 +4180,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_CLASS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ @@ -4251,35 +4215,28 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_O + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4288,34 +4245,24 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_FAST + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); - /* res = func(self, arguments, nargs, kwnames) */ - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4324,7 +4271,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_FAST_WITH_KEYWORDS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; macro(CALL_LEN) = unused/1 + @@ -4359,41 +4306,56 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { - /* isinstance(o, o2) */ + op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - DEOPT_IF(total_args != 2); PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable_o != interp->callable_cache.isinstance); + } + + op(_CALL_ISINSTANCE, (callable, null, instance, cls -- res)) { + /* isinstance(o, o2) */ STAT_INC(CALL, hit); - _PyStackRef cls_stackref = arguments[1]; - _PyStackRef inst_stackref = arguments[0]; - int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + int retval = PyObject_IsInstance(inst_o, cls_o); if (retval < 0) { ERROR_NO_POP(); } + (void)null; // Silence compiler warnings about unused variables + PyStackRef_CLOSE(cls); + PyStackRef_CLOSE(instance); + DEAD(null); + PyStackRef_CLOSE(callable); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + } + + macro(CALL_ISINSTANCE) = + unused/1 + + unused/2 + + _GUARD_THIRD_NULL + + _GUARD_CALLABLE_ISINSTANCE + + _CALL_ISINSTANCE; + + macro(CALL_LIST_APPEND) = + unused/1 + + unused/2 + + _GUARD_CALLABLE_LIST_APPEND + + _GUARD_NOS_NOT_NULL + + _GUARD_NOS_LIST + + _CALL_LIST_APPEND; + + op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)){ + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + DEOPT_IF(callable_o != interp->callable_cache.list_append); } // This is secretly a super-instruction - inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- )) { + op(_CALL_LIST_APPEND, (callable, self, arg -- )) { assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable_o != interp->callable_cache.list_append); - DEOPT_IF(self_o == NULL); - DEOPT_IF(!PyList_Check(self_o)); DEOPT_IF(!LOCK_OBJECT(self_o)); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); @@ -4446,7 +4408,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_O + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4467,19 +4429,16 @@ dummy_func( assert(self != NULL); EXIT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); - int nargs = total_args - 1; - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4488,7 +4447,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); @@ -4526,7 +4485,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_NOARGS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4547,18 +4506,16 @@ dummy_func( assert(self != NULL); EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); - int nargs = total_args - 1; - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4567,7 +4524,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_FAST + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; // Cache layout: counter/1, func_version/2 family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { @@ -4678,7 +4635,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _PyInterpreterFrame*)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); // oparg counts all of the args, but *not* self: @@ -4705,7 +4662,7 @@ dummy_func( DEAD(callable); SYNC_SP(); ERROR_IF(temp == NULL); - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { @@ -4719,6 +4676,7 @@ dummy_func( unused/1 + // Skip over the counter _CHECK_PEP_523 + _CHECK_FUNCTION_VERSION_KW + + _CHECK_RECURSION_REMAINING + _PY_FRAME_KW + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4790,30 +4748,21 @@ dummy_func( #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - PyStackRef_CLOSE(kwnames); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); + DEAD(kwnames); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4823,7 +4772,7 @@ dummy_func( unused/2 + _CHECK_IS_NOT_PY_CALLABLE_KW + _CALL_KW_NON_PY + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs -- func, unused, callargs, kwargs)) { PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); @@ -4924,12 +4873,12 @@ dummy_func( macro(CALL_FUNCTION_EX) = _MAKE_CALLARGS_A_TUPLE + _DO_CALL_FUNCTION_EX + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; macro(INSTRUMENTED_CALL_FUNCTION_EX) = _MAKE_CALLARGS_A_TUPLE + _DO_CALL_FUNCTION_EX + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; inst(MAKE_FUNCTION, (codeobj_st -- func)) { PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); @@ -5024,8 +4973,7 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - pure inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { - assert(oparg > 0); + pure replicate(1:4) inst(COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { top = PyStackRef_DUP(bottom); } @@ -5058,12 +5006,11 @@ dummy_func( macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + unused/4 + _BINARY_OP; - pure inst(SWAP, (bottom, unused[oparg-2], top -- + pure replicate(2:4) inst(SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { _PyStackRef temp = bottom; bottom = top; top = temp; - assert(oparg >= 2); } inst(INSTRUMENTED_LINE, ( -- )) { @@ -5194,23 +5141,20 @@ dummy_func( op (_GUARD_IS_TRUE_POP, (flag -- )) { int is_true = PyStackRef_IsTrue(flag); DEAD(flag); - SYNC_SP(); - EXIT_IF(!is_true); + AT_END_EXIT_IF(!is_true); } op (_GUARD_IS_FALSE_POP, (flag -- )) { int is_false = PyStackRef_IsFalse(flag); DEAD(flag); - SYNC_SP(); - EXIT_IF(!is_false); + AT_END_EXIT_IF(!is_false); } op (_GUARD_IS_NONE_POP, (val -- )) { int is_none = PyStackRef_IsNone(val); if (!is_none) { PyStackRef_CLOSE(val); - SYNC_SP(); - EXIT_IF(1); + AT_END_EXIT_IF(1); } DEAD(val); } @@ -5218,8 +5162,7 @@ dummy_func( op (_GUARD_IS_NOT_NONE_POP, (val -- )) { int is_none = PyStackRef_IsNone(val); PyStackRef_CLOSE(val); - SYNC_SP(); - EXIT_IF(is_none); + AT_END_EXIT_IF(is_none); } op(_JUMP_TO_TOP, (--)) { @@ -5247,46 +5190,41 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %lu, temp %d, target %d -> %s]\n", + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + } + #endif + tstate->jit_exit = exit; + TIER2_TO_TIER2(exit->executor); + } + + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); } - #endif - if (exit->executor && !exit->executor->vm_data.valid) { - exit->temperature = initial_temperature_backoff_counter(); - Py_CLEAR(exit->executor); - } - if (exit->executor == NULL) { - _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - int chain_depth = current_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } - exit->temperature = initial_temperature_backoff_counter(); - } - exit->executor = executor; - } - GOTO_TIER_TWO(exit->executor); + #endif + // Disabled for now (gh-139109) as it slows down dynamic code tremendously. + // Compile and jump to the cold dynamic executors in the future. + GOTO_TIER_ONE(frame->instr_ptr); } tier2 op(_CHECK_VALIDITY, (--)) { @@ -5303,39 +5241,93 @@ dummy_func( } tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyStackRef_FromPyObjectImmortal(ptr); + value = PyStackRef_FromPyObjectBorrow(ptr); } - tier2 pure op (_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + tier2 op(_POP_CALL, (callable, null --)) { + (void)null; // Silence compiler warnings about unused variables + DEAD(null); + PyStackRef_CLOSE(callable); + } + + tier2 op(_POP_CALL_ONE, (callable, null, pop --)) { PyStackRef_CLOSE(pop); - value = PyStackRef_FromPyObjectImmortal(ptr); + (void)null; // Silence compiler warnings about unused variables + DEAD(null); + PyStackRef_CLOSE(callable); } - tier2 pure op(_POP_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, pop1, pop2 -- value)) { + tier2 op(_POP_CALL_TWO, (callable, null, pop1, pop2 --)) { PyStackRef_CLOSE(pop2); PyStackRef_CLOSE(pop1); - value = PyStackRef_FromPyObjectImmortal(ptr); + (void)null; // Silence compiler warnings about unused variables + DEAD(null); + PyStackRef_CLOSE(callable); } - tier2 op(_CHECK_FUNCTION, (func_version/2 -- )) { - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - DEOPT_IF(func->func_version != func_version); + tier2 op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { + PyStackRef_CLOSE(pop); + value = PyStackRef_FromPyObjectBorrow(ptr); + } + + tier2 op(_POP_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, pop1, pop2 -- value)) { + PyStackRef_CLOSE(pop2); + PyStackRef_CLOSE(pop1); + value = PyStackRef_FromPyObjectBorrow(ptr); + } + + tier2 op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null -- value)) { + (void)null; // Silence compiler warnings about unused variables + DEAD(null); + PyStackRef_CLOSE(callable); + value = PyStackRef_FromPyObjectBorrow(ptr); + } + + tier2 op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, pop -- value)) { + PyStackRef_CLOSE(pop); + (void)null; // Silence compiler warnings about unused variables + DEAD(null); + PyStackRef_CLOSE(callable); + value = PyStackRef_FromPyObjectBorrow(ptr); + } + + tier2 op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, pop1, pop2 -- value)) { + PyStackRef_CLOSE(pop2); + PyStackRef_CLOSE(pop1); + (void)null; // Silence compiler warnings about unused variables + DEAD(null); + PyStackRef_CLOSE(callable); + value = PyStackRef_FromPyObjectBorrow(ptr); + } + + tier2 op(_LOAD_CONST_UNDER_INLINE, (ptr/4, old -- value, new)) { + new = old; + DEAD(old); + value = PyStackRef_FromPyObjectNew(ptr); + } + + tier2 op(_LOAD_CONST_UNDER_INLINE_BORROW, (ptr/4, old -- value, new)) { + new = old; + DEAD(old); + value = PyStackRef_FromPyObjectBorrow(ptr); } tier2 op(_START_EXECUTOR, (executor/4 --)) { #ifndef _Py_JIT - current_executor = (_PyExecutorObject*)executor; + assert(current_executor == (_PyExecutorObject*)executor); #endif - assert(((_PyExecutorObject *)executor)->vm_data.valid); + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyExecutor_ClearExit(tstate->jit_exit); + DEOPT_IF(true); + } } tier2 op(_MAKE_WARM, (--)) { current_executor->vm_data.warm = true; - // It's okay if this ends up going negative. - if (--tstate->interp->trace_run_counter == 0) { - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); - } } tier2 op(_FATAL_ERROR, (--)) { @@ -5344,7 +5336,13 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + + tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { + int err = _Py_HandlePending(tstate); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } tier2 op(_ERROR_POP_N, (target/2 --)) { @@ -5358,14 +5356,89 @@ dummy_func( * ENTER_EXECUTOR will not re-enter tier 2 with the eval breaker set. */ tier2 op(_TIER2_RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0); + HANDLE_PENDING_AND_DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); + HANDLE_PENDING_AND_DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + tier2 op(_COLD_EXIT, ( -- )) { + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_BackoffCounter temperature = exit->temperature; + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); + } + else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + // For control-flow guards, we don't want to increase the chain depth, as those don't actually + // represent deopts but rather just normal programs! + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. + // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. + // So setting it to anything else is wrong. + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + GOTO_TIER_ONE(target); + } + } + + tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { + // TODO (gh-139109): This should be similar to _COLD_EXIT in the future. + _Py_CODEUNIT *target = frame->instr_ptr; + GOTO_TIER_ONE(target); + } + + tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(_PUSH_FRAME); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(_PUSH_FRAME); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(YIELD_VALUE); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(YIELD_VALUE); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_VALUE); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(RETURN_VALUE); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_GENERATOR); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(RETURN_GENERATOR); + EXIT_IF(true); + } + } + label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); @@ -5450,7 +5523,7 @@ dummy_func( } #endif RELOAD_STACK(); -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP int opcode; #endif DISPATCH(); @@ -5468,7 +5541,7 @@ dummy_func( if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { /* Restore previous frame and exit */ tstate->current_frame = frame->previous; -#if !Py_TAIL_CALL_INTERP +#if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 @@ -5504,12 +5577,71 @@ dummy_func( assert(!_PyErr_Occurred(tstate)); #endif RELOAD_STACK(); -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP int opcode; #endif DISPATCH(); } + inst(TRACE_RECORD, (--)) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); + if (full) { + LEAVE_TRACING(); + int err = stop_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH(); + } + // Super instructions. Instruction deopted. There's a mismatch in what the stack expects + // in the optimizer. So we have to reflect in the trace correctly. + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + } + + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); +#else + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); +#endif + } + + label(stop_tracing) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); + LEAVE_TRACING(); + int err = stop_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index 490b653f132..a1d54bd058b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -8,7 +8,7 @@ #include "pycore_backoff.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_cell.h" // PyCell_GetRef() -#include "pycore_ceval.h" +#include "pycore_ceval.h" // SPECIAL___ENTER__ #include "pycore_code.h" #include "pycore_dict.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS @@ -139,6 +139,19 @@ #endif +static void +check_invalid_reentrancy(void) +{ +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) + // In the free-threaded build, the interpreter must not be re-entered if + // the world-is-stopped. If so, that's a bug somewhere (quite likely in + // the painfully complex typeobject code). + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!interp->stoptheworld.world_stopped); +#endif +} + + #ifdef Py_DEBUG static void dump_item(_PyStackRef item) @@ -147,6 +160,10 @@ dump_item(_PyStackRef item) printf("<NULL>"); return; } + if (PyStackRef_IsMalformed(item)) { + printf("<INVALID>"); + return; + } if (PyStackRef_IsTaggedInt(item)) { printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item)); return; @@ -262,7 +279,8 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) } int r = PyDict_Contains(globals, &_Py_ID(__lltrace__)); if (r < 0) { - return -1; + PyErr_Clear(); + return 0; } int lltrace = r * 5; // Levels 1-4 only trace uops if (!lltrace) { @@ -333,13 +351,23 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if (here_addr > _tstate->c_stack_soft_limit + margin_count * PYOS_STACK_MARGIN_BYTES) { +#if _Py_STACK_GROWS_DOWN + if (here_addr > _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES) { +#else + if (here_addr <= _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES) { +#endif return 0; } if (_tstate->c_stack_hard_limit == 0) { _Py_InitializeRecursionLimits(tstate); } - return here_addr <= _tstate->c_stack_soft_limit + margin_count * PYOS_STACK_MARGIN_BYTES; +#if _Py_STACK_GROWS_DOWN + return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; +#else + return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; +#endif } void @@ -347,7 +375,11 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN if (here_addr < _tstate->c_stack_hard_limit) { +#else + if (here_addr > _tstate->c_stack_hard_limit) { +#endif Py_FatalError("Unchecked stack overflow."); } } @@ -360,9 +392,6 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) # define Py_C_STACK_SIZE 1200000 #elif defined(__sparc__) # define Py_C_STACK_SIZE 1600000 -#elif defined(__wasi__) - /* Web assembly has two stacks, so this isn't really the stack depth */ -# define Py_C_STACK_SIZE 131072 // wasi-libc DEFAULT_STACK_SIZE #elif defined(__hppa__) || defined(__powerpc64__) # define Py_C_STACK_SIZE 2000000 #else @@ -427,22 +456,28 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif - -void -_Py_InitializeRecursionLimits(PyThreadState *tstate) +static void +hardware_stack_limits(uintptr_t *base, uintptr_t *top, uintptr_t sp) { - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; #ifdef WIN32 ULONG_PTR low, high; GetCurrentThreadStackLimits(&low, &high); - _tstate->c_stack_top = (uintptr_t)high; + *top = (uintptr_t)high; ULONG guarantee = 0; SetThreadStackGuarantee(&guarantee); - _tstate->c_stack_hard_limit = ((uintptr_t)low) + guarantee + PYOS_STACK_MARGIN_BYTES; - _tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + PYOS_STACK_MARGIN_BYTES; + *base = (uintptr_t)low + guarantee; +#elif defined(__APPLE__) + pthread_t this_thread = pthread_self(); + void *stack_addr = pthread_get_stackaddr_np(this_thread); // top of the stack + size_t stack_size = pthread_get_stacksize_np(this_thread); + *top = (uintptr_t)stack_addr; + *base = ((uintptr_t)stack_addr) - stack_size; #else - uintptr_t here_addr = _Py_get_machine_stack_pointer(); -# if defined(HAVE_PTHREAD_GETATTR_NP) && !defined(_AIX) && !defined(__NetBSD__) + /// XXX musl supports HAVE_PTHRED_GETATTR_NP, but the resulting stack size + /// (on alpine at least) is much smaller than expected and imposes undue limits + /// compared to the old stack size estimation. (We assume musl is not glibc.) +# if defined(HAVE_PTHREAD_GETATTR_NP) && !defined(_AIX) && \ + !defined(__NetBSD__) && (defined(__GLIBC__) || !defined(__linux__)) size_t stack_size, guard_size; void *stack_addr; pthread_attr_t attr; @@ -453,28 +488,118 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) err |= pthread_attr_destroy(&attr); } if (err == 0) { - uintptr_t base = ((uintptr_t)stack_addr) + guard_size; - _tstate->c_stack_top = base + stack_size; -#ifdef _Py_THREAD_SANITIZER - // Thread sanitizer crashes if we use a bit more than half the stack. - _tstate->c_stack_soft_limit = base + (stack_size / 2); -#else - _tstate->c_stack_soft_limit = base + PYOS_STACK_MARGIN_BYTES * 2; -#endif - _tstate->c_stack_hard_limit = base + PYOS_STACK_MARGIN_BYTES; - assert(_tstate->c_stack_soft_limit < here_addr); - assert(here_addr < _tstate->c_stack_top); + *base = ((uintptr_t)stack_addr) + guard_size; + *top = (uintptr_t)stack_addr + stack_size; return; } # endif - _tstate->c_stack_top = _Py_SIZE_ROUND_UP(here_addr, 4096); - _tstate->c_stack_soft_limit = _tstate->c_stack_top - Py_C_STACK_SIZE; - _tstate->c_stack_hard_limit = _tstate->c_stack_top - (Py_C_STACK_SIZE + PYOS_STACK_MARGIN_BYTES); + // Add some space for caller function then round to minimum page size + // This is a guess at the top of the stack, but should be a reasonably + // good guess if called from _PyThreadState_Attach when creating a thread. + // If the thread is attached deep in a call stack, then the guess will be poor. +#if _Py_STACK_GROWS_DOWN + uintptr_t top_addr = _Py_SIZE_ROUND_UP(sp + 8*sizeof(void*), SYSTEM_PAGE_SIZE); + *top = top_addr; + *base = top_addr - Py_C_STACK_SIZE; +# else + uintptr_t base_addr = _Py_SIZE_ROUND_DOWN(sp - 8*sizeof(void*), SYSTEM_PAGE_SIZE); + *base = base_addr; + *top = base_addr + Py_C_STACK_SIZE; +#endif #endif } +static void +tstate_set_stack(PyThreadState *tstate, + uintptr_t base, uintptr_t top) +{ + assert(base < top); + assert((top - base) >= _PyOS_MIN_STACK_SIZE); + +#ifdef _Py_THREAD_SANITIZER + // Thread sanitizer crashes if we use more than half the stack. + uintptr_t stacksize = top - base; +# if _Py_STACK_GROWS_DOWN + base += stacksize/2; +# else + top -= stacksize/2; +# endif +#endif + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN + _tstate->c_stack_top = top; + _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; +# ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +# endif +#else + _tstate->c_stack_top = base; + _tstate->c_stack_hard_limit = top - _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = top - _PyOS_STACK_MARGIN_BYTES * 2; +# ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit >= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit > ts->c_stack_top); +# endif +#endif +} + + +void +_Py_InitializeRecursionLimits(PyThreadState *tstate) +{ + uintptr_t base, top; + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + hardware_stack_limits(&base, &top, here_addr); + assert(top != 0); + + tstate_set_stack(tstate, base, top); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + ts->c_stack_init_base = base; + ts->c_stack_init_top = top; +} + + +int +PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, + void *stack_start_addr, size_t stack_size) +{ + if (stack_size < _PyOS_MIN_STACK_SIZE) { + PyErr_Format(PyExc_ValueError, + "stack_size must be at least %zu bytes", + _PyOS_MIN_STACK_SIZE); + return -1; + } + + uintptr_t base = (uintptr_t)stack_start_addr; + uintptr_t top = base + stack_size; + tstate_set_stack(tstate, base, top); + return 0; +} + + +void +PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) +{ + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + if (ts->c_stack_init_top != 0) { + tstate_set_stack(tstate, + ts->c_stack_init_base, + ts->c_stack_init_top); + return; + } + + _Py_InitializeRecursionLimits(tstate); +} + + /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() - if the recursion_depth reaches recursion_limit. */ + if the stack pointer is between the stack base and c_stack_hard_limit. */ int _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { @@ -482,9 +607,17 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) uintptr_t here_addr = _Py_get_machine_stack_pointer(); assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN + assert(here_addr >= _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES); if (here_addr < _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + assert(here_addr <= _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES); + if (here_addr > _tstate->c_stack_hard_limit) { + /* Overflowing while handling an overflow. Give up. */ + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif char buffer[80]; snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where); Py_FatalError(buffer); @@ -493,7 +626,11 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) return 0; } else { +#if _Py_STACK_GROWS_DOWN int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif tstate->recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, "Stack overflow (used %d kB)%s", @@ -617,12 +754,14 @@ _PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys) PyObject *seen = NULL; PyObject *dummy = NULL; PyObject *values = NULL; - PyObject *get = NULL; // We use the two argument form of map.get(key, default) for two reasons: // - Atomically check for a key and get its value without error handling. // - Don't cause key creation or resizing in dict subclasses like // collections.defaultdict that define __missing__ (or similar). - int meth_found = _PyObject_GetMethod(map, &_Py_ID(get), &get); + _PyCStackRef cref; + _PyThreadState_PushCStackRef(tstate, &cref); + int meth_found = _PyObject_GetMethodStackRef(tstate, map, &_Py_ID(get), &cref.ref); + PyObject *get = PyStackRef_AsPyObjectBorrow(cref.ref); if (get == NULL) { goto fail; } @@ -672,12 +811,12 @@ _PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys) } // Success: done: - Py_DECREF(get); + _PyThreadState_PopCStackRef(tstate, &cref); Py_DECREF(seen); Py_DECREF(dummy); return values; fail: - Py_XDECREF(get); + _PyThreadState_PopCStackRef(tstate, &cref); Py_XDECREF(seen); Py_XDECREF(dummy); Py_XDECREF(values); @@ -878,6 +1017,283 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #include "ceval_macros.h" + +/* Helper functions to keep the size of the largest uops down */ + +PyObject * +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args; + if (kwnames_o != NULL) { + positional_args -= (int)PyTuple_GET_SIZE(kwnames_o); + } + res = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + PyStackRef_XCLOSE(kwnames); + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuiltinCallFast_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + res = _PyCFunctionFast_CAST(cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args + ); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuiltinCallFastWithKeywords_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + res = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_PyCallMethodDescriptorFast_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth); + assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); + + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); + res = cfunc(self, (args_o + 1), total_args - 1); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth); + assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); + + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); + res = cfunc(self, (args_o + 1), total_args-1, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_CallBuiltinClass_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(callable); + res = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + res = _PyUnicode_JoinArray(&_Py_STR(empty), args_o, total_args); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + return res; +} + +PyObject * +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, half_args*2, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + res = _PyDict_FromItems( + args_o, 2, + args_o+1, 2, + half_args + ); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = half_args*2-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + return res; +} + +#ifdef Py_DEBUG +void +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno +) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + return; + } + int level = (int)(stack_pointer - _PyFrame_Stackbase(frame)); + if (level < 0) { + printf("Stack underflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } + int size = _PyFrame_GetCode(frame)->co_stacksize; + if (level > size) { + printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } +} +#endif + int _Py_CheckRecursiveCallPy( PyThreadState *tstate) { @@ -909,6 +1325,8 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; +const _Py_CODEUNIT *_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR = (_Py_CODEUNIT*)&_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS; + #ifdef Py_DEBUG extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif @@ -937,11 +1355,12 @@ _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject if (result == NULL) { return NULL; } - result++; } else { result = scratch; } + result++; + result[0] = NULL; /* Keep GCC happy */ for (int i = 0; i < nargs; i++) { result[i] = PyStackRef_AsPyObjectBorrow(input[i]); } @@ -956,6 +1375,49 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + +#if _Py_TIER2 +// 0 for success, -1 for error. +static int +stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + int err = 0; + if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { + err = _PyOptimizer_Optimize(frame, tstate); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Deal with backoffs + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; + if (exit == NULL) { + // We hold a strong reference to the code object, so the instruction won't be freed. + if (err <= 0) { + _Py_BackoffCounter counter = _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter; + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); + } + else { + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); + } + } + else { + // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed + // to be valid to access. + if (err <= 0) { + exit->temperature = restart_backoff_counter(exit->temperature); + } + else { + exit->temperature = initial_temperature_backoff_counter(); + } + } + _PyJit_FinalizeTracing(tstate); + return err; +} +#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ @@ -970,7 +1432,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) /* This setting is reversed below following _PyEval_EvalFrameDefault */ #endif -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP #include "opcode_targets.h" #include "generated_cases.c.h" #endif @@ -999,20 +1461,26 @@ PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { _Py_EnsureTstateNotNULL(tstate); + check_invalid_reentrancy(); CALL_STAT_INC(pyeval_calls); -#if USE_COMPUTED_GOTOS && !Py_TAIL_CALL_INTERP +#if USE_COMPUTED_GOTOS && !_Py_TAIL_CALL_INTERP /* Import the static jump table */ #include "opcode_targets.h" + void **opcode_targets = opcode_targets_table; #endif #ifdef Py_STATS int lastopcode = 0; #endif -#if !Py_TAIL_CALL_INTERP +#if !_Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); +#if !USE_COMPUTED_GOTOS + uint8_t tracing_mode = 0; + uint8_t dispatch_code; +#endif #endif _PyEntryFrame entry; @@ -1081,27 +1549,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_instr = frame->instr_ptr; monitor_throw(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, 0, lastopcode); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, 0); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0); # endif #else goto error; #endif } -#if defined(_Py_TIER2) && !defined(_Py_JIT) - /* Tier 2 interpreter state */ - _PyExecutorObject *current_executor = NULL; - const _PyUOpInstruction *next_uop = NULL; -#endif -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0, lastopcode); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0); # endif #else goto start_frame; @@ -1109,14 +1572,43 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif +early_exit: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + return NULL; +} #ifdef _Py_TIER2 - -// Tier 2 is also here! -enter_tier_two: - #ifdef _Py_JIT - assert(0); +_PyJitEntryFuncPtr _Py_jit_entry = _Py_LazyJitTrampoline; #else +_PyJitEntryFuncPtr _Py_jit_entry = _PyTier2Interpreter; +#endif +#endif + +#if defined(_Py_TIER2) && !defined(_Py_JIT) + +_Py_CODEUNIT * +_PyTier2Interpreter( + _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +) { + const _PyUOpInstruction *next_uop; + int oparg; +tier2_start: + + next_uop = current_executor->trace; + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT); #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1134,14 +1626,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #undef ENABLE_SPECIALIZATION_FT #define ENABLE_SPECIALIZATION_FT 0 - ; // dummy statement after a label, before a declaration uint16_t uopcode; #ifdef Py_STATS int lastuop = 0; uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; @@ -1194,6 +1687,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int printf(" @ %d -> %s]\n", (int)(next_uop - current_executor->trace - 1), _PyOpcode_OpName[frame->instr_ptr->op.code]); + fflush(stdout); } #endif assert(next_uop[-1].format == UOP_FORMAT_JUMP); @@ -1207,24 +1701,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_uop = current_executor->trace + target; goto tier2_dispatch; -#endif // _Py_JIT - +} #endif // _Py_TIER2 -early_exit: - assert(_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - frame->return_offset = 0; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - /* Restore previous frame and exit */ - tstate->current_frame = frame->previous; - return NULL; -} #ifdef DO_NOT_OPTIMIZE_INTERP_LOOP # pragma optimize("", on) @@ -1809,6 +2288,16 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) { + // Update last_profiled_frame for remote profiler frame caching. + // By this point, tstate->current_frame is already set to the parent frame. + // Only update if we're popping the exact frame that was last profiled. + // This avoids corrupting the cache when transient frames (called and returned + // between profiler samples) update last_profiled_frame to addresses the + // profiler never saw. + if (tstate->last_profiled_frame != NULL && tstate->last_profiled_frame == frame) { + tstate->last_profiled_frame = tstate->current_frame; + } + if (frame->owner == FRAME_OWNED_BY_THREAD) { clear_thread_frame(tstate, frame); } @@ -1865,7 +2354,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, PyObject *kwnames = NULL; _PyStackRef *newargs; PyObject *const *object_array = NULL; - _PyStackRef stack_array[8]; + _PyStackRef stack_array[8] = {0}; if (has_dict) { object_array = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); if (object_array == NULL) { @@ -1928,7 +2417,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, if (kwnames) { total_args += PyTuple_GET_SIZE(kwnames); } - _PyStackRef stack_array[8]; + _PyStackRef stack_array[8] = {0}; _PyStackRef *arguments; if (total_args <= 8) { arguments = stack_array; @@ -1974,7 +2463,7 @@ PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals, { PyThreadState *tstate = _PyThreadState_GET(); PyObject *res = NULL; - PyObject *defaults = _PyTuple_FromArray(defs, defcount); + PyObject *defaults = PyTuple_FromArray(defs, defcount); if (defaults == NULL) { return NULL; } @@ -2107,6 +2596,7 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) "calling %R should have returned an instance of " "BaseException, not %R", cause, Py_TYPE(fixed_cause)); + Py_DECREF(fixed_cause); goto raise_error; } Py_DECREF(cause); @@ -2425,6 +2915,10 @@ monitor_unwind(PyThreadState *tstate, do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); } +bool +_PyEval_NoToolsForUnwind(PyThreadState *tstate) { + return no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND); +} static int monitor_handled(PyThreadState *tstate, @@ -2492,21 +2986,10 @@ PyEval_SetProfile(Py_tracefunc func, PyObject *arg) void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) { - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; - - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - - while (ts) { - if (_PyEval_SetProfile(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetProfileAllThreads(interp, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); } } @@ -2523,21 +3006,10 @@ PyEval_SetTrace(Py_tracefunc func, PyObject *arg) void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg) { - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; - - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - - while (ts) { - if (_PyEval_SetTrace(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetTraceAllThreads(interp, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); } } @@ -2650,12 +3122,6 @@ _PyEval_GetBuiltin(PyObject *name) return attr; } -PyObject * -_PyEval_GetBuiltinId(_Py_Identifier *name) -{ - return _PyEval_GetBuiltin(_PyUnicode_FromId(name)); -} - PyObject * PyEval_GetLocals(void) { @@ -2735,10 +3201,9 @@ _PyEval_GetFrameLocals(void) return locals; } -PyObject * -PyEval_GetGlobals(void) +static PyObject * +_PyEval_GetGlobals(PyThreadState *tstate) { - PyThreadState *tstate = _PyThreadState_GET(); _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { return NULL; @@ -2746,6 +3211,120 @@ PyEval_GetGlobals(void) return current_frame->f_globals; } +PyObject * +PyEval_GetGlobals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_GetGlobals(tstate); +} + +PyObject * +_PyEval_GetGlobalsFromRunningMain(PyThreadState *tstate) +{ + if (!_PyInterpreterState_IsRunningMain(tstate->interp)) { + return NULL; + } + PyObject *mod = _Py_GetMainModule(tstate); + if (_Py_CheckMainModule(mod) < 0) { + Py_XDECREF(mod); + return NULL; + } + PyObject *globals = PyModule_GetDict(mod); // borrowed + Py_DECREF(mod); + return globals; +} + +static PyObject * +get_globals_builtins(PyObject *globals) +{ + PyObject *builtins = NULL; + if (PyDict_Check(globals)) { + if (PyDict_GetItemRef(globals, &_Py_ID(__builtins__), &builtins) < 0) { + return NULL; + } + } + else { + if (PyMapping_GetOptionalItem( + globals, &_Py_ID(__builtins__), &builtins) < 0) + { + return NULL; + } + } + return builtins; +} + +static int +set_globals_builtins(PyObject *globals, PyObject *builtins) +{ + if (PyDict_Check(globals)) { + if (PyDict_SetItem(globals, &_Py_ID(__builtins__), builtins) < 0) { + return -1; + } + } + else { + if (PyObject_SetItem(globals, &_Py_ID(__builtins__), builtins) < 0) { + return -1; + } + } + return 0; +} + +int +_PyEval_EnsureBuiltins(PyThreadState *tstate, PyObject *globals, + PyObject **p_builtins) +{ + PyObject *builtins = get_globals_builtins(globals); + if (builtins == NULL) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + builtins = PyEval_GetBuiltins(); // borrowed + if (builtins == NULL) { + assert(_PyErr_Occurred(tstate)); + return -1; + } + Py_INCREF(builtins); + if (set_globals_builtins(globals, builtins) < 0) { + Py_DECREF(builtins); + return -1; + } + } + if (p_builtins != NULL) { + *p_builtins = builtins; + } + else { + Py_DECREF(builtins); + } + return 0; +} + +int +_PyEval_EnsureBuiltinsWithModule(PyThreadState *tstate, PyObject *globals, + PyObject **p_builtins) +{ + PyObject *builtins = get_globals_builtins(globals); + if (builtins == NULL) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + builtins = PyImport_ImportModuleLevel("builtins", NULL, NULL, NULL, 0); + if (builtins == NULL) { + return -1; + } + if (set_globals_builtins(globals, builtins) < 0) { + Py_DECREF(builtins); + return -1; + } + } + if (p_builtins != NULL) { + *p_builtins = builtins; + } + else { + Py_DECREF(builtins); + } + return 0; +} + PyObject* PyEval_GetFrameLocals(void) { @@ -2773,6 +3352,9 @@ PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { PyThreadState *tstate = _PyThreadState_GET(); _PyInterpreterFrame *current_frame = tstate->current_frame; + if (current_frame == tstate->base_frame) { + current_frame = NULL; + } int result = cf->cf_flags != 0; if (current_frame != NULL) { @@ -2968,7 +3550,7 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) int is_possibly_shadowing_stdlib = 0; if (is_possibly_shadowing) { PyObject *stdlib_modules; - if (_PySys_GetOptionalAttrString("stdlib_module_names", &stdlib_modules) < 0) { + if (PySys_GetOptionalAttrString("stdlib_module_names", &stdlib_modules) < 0) { goto done; } if (stdlib_modules && PyAnySet_Check(stdlib_modules)) { @@ -3140,17 +3722,9 @@ int _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args) { if (Py_TYPE(args)->tp_iter == NULL && !PySequence_Check(args)) { - /* _Py_Check_ArgsIterable() may be called with a live exception: - * clear it to prevent calling _PyObject_FunctionStr() with an - * exception set. */ - _PyErr_Clear(tstate); - PyObject *funcstr = _PyObject_FunctionStr(func); - if (funcstr != NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "%U argument after * must be an iterable, not %.200s", - funcstr, Py_TYPE(args)->tp_name); - Py_DECREF(funcstr); - } + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(args)->tp_name); return -1; } return 0; @@ -3166,20 +3740,15 @@ _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwarg * is not a mapping. */ if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Clear(tstate); - PyObject *funcstr = _PyObject_FunctionStr(func); - if (funcstr != NULL) { - _PyErr_Format( - tstate, PyExc_TypeError, - "%U argument after ** must be a mapping, not %.200s", - funcstr, Py_TYPE(kwargs)->tp_name); - Py_DECREF(funcstr); - } + _PyErr_Format( + tstate, PyExc_TypeError, + "Value after ** must be a mapping, not %.200s", + Py_TYPE(kwargs)->tp_name); } else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { PyObject *exc = _PyErr_GetRaisedException(tstate); PyObject *args = PyException_GetArgs(exc); - if (exc && PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1) { + if (PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1) { _PyErr_Clear(tstate); PyObject *funcstr = _PyObject_FunctionStr(func); if (funcstr != NULL) { @@ -3428,6 +3997,50 @@ _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *na return value; } +static _PyStackRef +foriter_next(PyObject *seq, _PyStackRef index) +{ + assert(PyStackRef_IsTaggedInt(index)); + assert(PyTuple_CheckExact(seq) || PyList_CheckExact(seq)); + intptr_t i = PyStackRef_UntagInt(index); + if (PyTuple_CheckExact(seq)) { + size_t size = PyTuple_GET_SIZE(seq); + if ((size_t)i >= size) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, i)); + } + PyObject *item = _PyList_GetItemRef((PyListObject *)seq, i); + if (item == NULL) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectSteal(item); +} + +_PyStackRef _PyForIter_VirtualIteratorNext(PyThreadState* tstate, _PyInterpreterFrame* frame, _PyStackRef iter, _PyStackRef* index_ptr) +{ + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + _PyStackRef index = *index_ptr; + if (PyStackRef_IsTaggedInt(index)) { + *index_ptr = PyStackRef_IncrementTaggedIntNoOverflow(index); + return foriter_next(iter_o, index); + } + PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + if (next_o == NULL) { + if (_PyErr_Occurred(tstate)) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); + _PyErr_Clear(tstate); + } + else { + return PyStackRef_ERROR; + } + } + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectSteal(next_o); +} + /* Check if a 'cls' provides the given special method. */ static inline int type_has_special_method(PyTypeObject *cls, PyObject *name) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 5b5018a6373..f6ada3892f8 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -207,6 +207,7 @@ drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) _Py_atomic_store_int_relaxed(&gil->locked, 0); if (tstate != NULL) { tstate->holds_gil = 0; + tstate->gil_requested = 0; } COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); @@ -320,6 +321,8 @@ take_gil(PyThreadState *tstate) MUTEX_LOCK(gil->mutex); + tstate->gil_requested = 1; + int drop_requested = 0; while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; @@ -407,6 +410,7 @@ take_gil(PyThreadState *tstate) } assert(_PyThreadState_CheckConsistency(tstate)); + tstate->gil_requested = 0; tstate->holds_gil = 1; _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); update_eval_breaker_for_thread(interp, tstate); @@ -907,13 +911,9 @@ unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) static void clear_pending_handling_thread(struct _pending_calls *pending) { -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&pending->mutex); + FT_MUTEX_LOCK(&pending->mutex); pending->handling_thread = NULL; - PyMutex_Unlock(&pending->mutex); -#else - pending->handling_thread = NULL; -#endif + FT_MUTEX_UNLOCK(&pending->mutex); } static int @@ -1218,30 +1218,30 @@ static inline int run_remote_debugger_source(PyObject *source) // Note that this function is inline to avoid creating a PLT entry // that would be an easy target for a ROP gadget. -static inline void run_remote_debugger_script(const char *path) +static inline void run_remote_debugger_script(PyObject *path) { - if (0 != PySys_Audit("remote_debugger_script", "s", path)) { + if (0 != PySys_Audit("cpython.remote_debugger_script", "O", path)) { PyErr_FormatUnraisable( - "Audit hook failed for remote debugger script %s", path); + "Audit hook failed for remote debugger script %U", path); return; } // Open the debugger script with the open code hook, and reopen the // resulting file object to get a C FILE* object. - PyObject* fileobj = PyFile_OpenCode(path); + PyObject* fileobj = PyFile_OpenCodeObject(path); if (!fileobj) { - PyErr_FormatUnraisable("Can't open debugger script %s", path); + PyErr_FormatUnraisable("Can't open debugger script %U", path); return; } PyObject* source = PyObject_CallMethodNoArgs(fileobj, &_Py_ID(read)); if (!source) { - PyErr_FormatUnraisable("Error reading debugger script %s", path); + PyErr_FormatUnraisable("Error reading debugger script %U", path); } PyObject* res = PyObject_CallMethodNoArgs(fileobj, &_Py_ID(close)); if (!res) { - PyErr_FormatUnraisable("Error closing debugger script %s", path); + PyErr_FormatUnraisable("Error closing debugger script %U", path); } else { Py_DECREF(res); } @@ -1249,7 +1249,7 @@ static inline void run_remote_debugger_script(const char *path) if (source) { if (0 != run_remote_debugger_source(source)) { - PyErr_FormatUnraisable("Error executing debugger script %s", path); + PyErr_FormatUnraisable("Error executing debugger script %U", path); } Py_DECREF(source); } @@ -1278,7 +1278,14 @@ int _PyRunRemoteDebugger(PyThreadState *tstate) pathsz); path[pathsz - 1] = '\0'; if (*path) { - run_remote_debugger_script(path); + PyObject *path_obj = PyUnicode_DecodeFSDefault(path); + if (path_obj == NULL) { + PyErr_FormatUnraisable("Can't decode debugger script"); + } + else { + run_remote_debugger_script(path_obj); + Py_DECREF(path_obj); + } } PyMem_Free(path); } @@ -1380,6 +1387,10 @@ _Py_HandlePending(PyThreadState *tstate) _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT); _Py_brc_merge_refcounts(tstate); } + /* Process deferred memory frees held by QSBR */ + if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) { + _PyMem_ProcessDelayed(tstate); + } #endif /* GC scheduled to run */ @@ -1391,7 +1402,7 @@ _Py_HandlePending(PyThreadState *tstate) if ((breaker & _PY_EVAL_JIT_INVALIDATE_COLD_BIT) != 0) { _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); _Py_Executors_InvalidateCold(tstate->interp); - tstate->interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + tstate->interp->executor_creation_counter = JIT_CLEANUP_THRESHOLD; } /* GIL drop request */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 187ec8fdd26..edf8fc9a57d 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -62,8 +62,9 @@ #ifdef Py_STATS #define INSTRUCTION_STATS(op) \ do { \ + PyStats *s = _PyStats_GET(); \ OPCODE_EXE_INC(op); \ - if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ + if (s) s->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else @@ -71,23 +72,39 @@ #endif #ifdef Py_STATS -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg, int lastopcode -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg, lastopcode +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode #else -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg #endif -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP +# if defined(__clang__) || defined(__GNUC__) +# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) +# error "This compiler does not have support for efficient tail calling." +# endif +# elif defined(_MSC_VER) && (_MSC_VER < 1950) +# error "You need at least VS 2026 / PlatformToolset v145 for tail calling." +# endif + // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) + # define DISPATCH_GOTO() \ do { \ - Py_MUSTTAIL return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ } while (0) # define JUMP_TO_LABEL(name) \ do { \ @@ -96,29 +113,46 @@ # ifdef Py_STATS # define JUMP_TO_PREDICTED(name) \ do { \ - Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, oparg, lastopcode); \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode); \ } while (0) # else # define JUMP_TO_PREDICTED(name) \ do { \ - Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, oparg); \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg); \ } while (0) # endif # define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: -# define DISPATCH_GOTO() goto dispatch_opcode +# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: #endif +#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +# define ENTER_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; +#else +# define IS_JIT_TRACING() (tracing_mode != 0) +# define ENTER_TRACING() tracing_mode = 255 +# define LEAVE_TRACING() tracing_mode = 0 +#endif + /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ @@ -133,9 +167,6 @@ do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (lltrace < 0) { \ - JUMP_TO_LABEL(exit_unwind); \ - } \ frame->lltrace = lltrace; \ } while (0) #else @@ -158,11 +189,19 @@ do { \ DISPATCH_GOTO(); \ } +#define DISPATCH_NON_TRACING() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ } #define DISPATCH_INLINED(NEW_FRAME) \ @@ -274,6 +313,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ +// Force re-specialization when tracing a side exit to get good side exits. #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) @@ -354,52 +394,52 @@ _PyFrame_SetStackPointer(frame, stack_pointer) /* Tier-switching macros. */ -#ifdef _Py_JIT -#define GOTO_TIER_TWO(EXECUTOR) \ +#define TIER1_TO_TIER2(EXECUTOR) \ do { \ OPT_STAT_INC(traces_executed); \ - _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ - jit_func jitted = _executor->jit_code; \ - /* Keep the shim frame alive via the executor: */ \ - Py_INCREF(_executor); \ - next_instr = jitted(frame, stack_pointer, tstate); \ - Py_DECREF(_executor); \ + next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ if (next_instr == NULL) { \ - next_instr = frame->instr_ptr; \ + /* gh-140104: The exception handler expects frame->instr_ptr + to after this_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ + if (keep_tracing_bit) { \ + assert(((_PyThreadStateImpl *)tstate)->jit_tracer_state.prev_state.code_curr_size == 2); \ + ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ + } \ DISPATCH(); \ } while (0) -#else -#define GOTO_TIER_TWO(EXECUTOR) \ -do { \ - OPT_STAT_INC(traces_executed); \ - _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ - next_uop = _executor->trace; \ - assert(next_uop->opcode == _START_EXECUTOR); \ - goto enter_tier_two; \ -} while (0) -#endif -#define GOTO_TIER_ONE(TARGET) \ - do \ - { \ - tstate->current_executor = NULL; \ - next_instr = (TARGET); \ - assert(tstate->current_executor == NULL); \ - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (next_instr == NULL) \ - { \ - next_instr = frame->instr_ptr; \ - goto error; \ - } \ - DISPATCH(); \ +#define TIER2_TO_TIER2(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + current_executor = (EXECUTOR); \ + goto tier2_start; \ +} while (0) + +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); + +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(TARGET); \ + } while (0) + +#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) @@ -418,10 +458,21 @@ do { \ #define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ /* +1 because vectorcall might use -1 to write self */ \ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ - PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp); #define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ /* +1 because we +1 previously */ \ _PyObjectArray_Free(NAME - 1, NAME##_temp); #define CONVERSION_FAILED(NAME) ((NAME) == NULL) + +static inline int +check_periodics(PyThreadState *tstate) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + return _Py_HandlePending(tstate); + } + return 0; +} + diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index c826a5724f7..f08e5847abe 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -114,7 +114,7 @@ exit: } PyDoc_STRVAR(builtin_abs__doc__, -"abs($module, x, /)\n" +"abs($module, number, /)\n" "--\n" "\n" "Return the absolute value of the argument."); @@ -159,7 +159,7 @@ PyDoc_STRVAR(builtin_ascii__doc__, {"ascii", (PyCFunction)builtin_ascii, METH_O, builtin_ascii__doc__}, PyDoc_STRVAR(builtin_bin__doc__, -"bin($module, number, /)\n" +"bin($module, integer, /)\n" "--\n" "\n" "Return the binary representation of an integer.\n" @@ -238,7 +238,8 @@ PyDoc_STRVAR(builtin_chr__doc__, PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" -" dont_inherit=False, optimize=-1, *, _feature_version=-1)\n" +" dont_inherit=False, optimize=-1, *, module=None,\n" +" _feature_version=-1)\n" "--\n" "\n" "Compile source into a code object that can be executed by exec() or eval().\n" @@ -260,7 +261,7 @@ PyDoc_STRVAR(builtin_compile__doc__, static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version); + int optimize, PyObject *modname, int feature_version); static PyObject * builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -268,7 +269,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 7 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -277,7 +278,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(_feature_version), }, + .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(module), &_Py_ID(_feature_version), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -286,21 +287,22 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "_feature_version", NULL}; + static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "module", "_feature_version", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compile", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[7]; + PyObject *argsbuf[8]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; - PyObject *filename; + PyObject *filename = NULL; const char *mode; int flags = 0; int dont_inherit = 0; int optimize = -1; + PyObject *modname = Py_None; int feature_version = -1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, @@ -359,14 +361,23 @@ skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - feature_version = PyLong_AsInt(args[6]); + if (args[6]) { + modname = args[6]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + feature_version = PyLong_AsInt(args[7]); if (feature_version == -1 && PyErr_Occurred()) { goto exit; } skip_optional_kwonly: - return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, feature_version); + return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, modname, feature_version); exit: + /* Cleanup for filename */ + Py_XDECREF(filename); + return return_value; } @@ -729,7 +740,7 @@ PyDoc_STRVAR(builtin_hash__doc__, {"hash", (PyCFunction)builtin_hash, METH_O, builtin_hash__doc__}, PyDoc_STRVAR(builtin_hex__doc__, -"hex($module, number, /)\n" +"hex($module, integer, /)\n" "--\n" "\n" "Return the hexadecimal representation of an integer.\n" @@ -750,7 +761,7 @@ PyDoc_STRVAR(builtin_aiter__doc__, {"aiter", (PyCFunction)builtin_aiter, METH_O, builtin_aiter__doc__}, PyDoc_STRVAR(builtin_anext__doc__, -"anext($module, aiterator, default=<unrepresentable>, /)\n" +"anext($module, async_iterator, default=<unrepresentable>, /)\n" "--\n" "\n" "Return the next item from the async iterator.\n" @@ -819,7 +830,7 @@ builtin_locals(PyObject *module, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(builtin_oct__doc__, -"oct($module, number, /)\n" +"oct($module, integer, /)\n" "--\n" "\n" "Return the octal representation of an integer.\n" @@ -831,10 +842,16 @@ PyDoc_STRVAR(builtin_oct__doc__, {"oct", (PyCFunction)builtin_oct, METH_O, builtin_oct__doc__}, PyDoc_STRVAR(builtin_ord__doc__, -"ord($module, c, /)\n" +"ord($module, character, /)\n" "--\n" "\n" -"Return the Unicode code point for a one-character string."); +"Return the ordinal value of a character.\n" +"\n" +"If the argument is a one-character string, return the Unicode code\n" +"point of that character.\n" +"\n" +"If the argument is a bytes or bytearray object of length 1, return its\n" +"single byte value."); #define BUILTIN_ORD_METHODDEF \ {"ord", (PyCFunction)builtin_ord, METH_O, builtin_ord__doc__}, @@ -911,7 +928,7 @@ exit: } PyDoc_STRVAR(builtin_print__doc__, -"print($module, /, *args, sep=\' \', end=\'\\n\', file=None, flush=False)\n" +"print($module, /, *objects, sep=\' \', end=\'\\n\', file=None, flush=False)\n" "--\n" "\n" "Prints the values to a stream, or to sys.stdout by default.\n" @@ -929,8 +946,8 @@ PyDoc_STRVAR(builtin_print__doc__, {"print", _PyCFunction_CAST(builtin_print), METH_FASTCALL|METH_KEYWORDS, builtin_print__doc__}, static PyObject * -builtin_print_impl(PyObject *module, PyObject * const *args, - Py_ssize_t args_length, PyObject *sep, PyObject *end, +builtin_print_impl(PyObject *module, PyObject * const *objects, + Py_ssize_t objects_length, PyObject *sep, PyObject *end, PyObject *file, int flush); static PyObject * @@ -967,8 +984,8 @@ builtin_print(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec PyObject *argsbuf[4]; PyObject * const *fastargs; Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject * const *__clinic_args; - Py_ssize_t args_length; + PyObject * const *objects; + Py_ssize_t objects_length; PyObject *sep = Py_None; PyObject *end = Py_None; PyObject *file = Py_None; @@ -1005,9 +1022,9 @@ builtin_print(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec goto exit; } skip_optional_kwonly: - __clinic_args = args; - args_length = nargs; - return_value = builtin_print_impl(module, __clinic_args, args_length, sep, end, file, flush); + objects = args; + objects_length = nargs; + return_value = builtin_print_impl(module, objects, objects_length, sep, end, file, flush); exit: return return_value; @@ -1268,4 +1285,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e7a5d0851d7f2cfb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06500bcc9a341e68 input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index a47e4d11b54..4c4a86de2f9 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -7,7 +7,6 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -#include "pycore_tuple.h" // _PyTuple_FromArray() PyDoc_STRVAR(sys_addaudithook__doc__, "addaudithook($module, /, hook)\n" @@ -102,7 +101,7 @@ sys_audit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - __clinic_args = _PyTuple_FromArray(args + 1, nargs - 1); + __clinic_args = PyTuple_FromArray(args + 1, nargs - 1); if (__clinic_args == NULL) { goto exit; } @@ -1948,4 +1947,4 @@ exit: #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=449d16326e69dcf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5f7d84c5bf00d557 input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index caf8d9d5f3c..0bde56c0ac6 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -16,7 +16,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_runtime.h" // _Py_ID() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // _PyUnicode_InternMortal() - +#include "pycore_pyatomic_ft_wrappers.h" static const char *codecs_builtin_error_handlers[] = { "strict", "ignore", "replace", @@ -40,13 +40,10 @@ int PyCodec_Register(PyObject *search_function) PyErr_SetString(PyExc_TypeError, "argument must be callable"); goto onError; } -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_LOCK(&interp->codecs.search_path_mutex); int ret = PyList_Append(interp->codecs.search_path, search_function); -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_UNLOCK(&interp->codecs.search_path_mutex); + return ret; onError: @@ -66,9 +63,7 @@ PyCodec_Unregister(PyObject *search_function) PyObject *codec_search_path = interp->codecs.search_path; assert(PyList_CheckExact(codec_search_path)); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(codec_search_path); i++) { -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_LOCK(&interp->codecs.search_path_mutex); PyObject *item = PyList_GetItemRef(codec_search_path, i); int ret = 1; if (item == search_function) { @@ -76,9 +71,7 @@ PyCodec_Unregister(PyObject *search_function) // while we hold search_path_mutex. ret = PyList_SetSlice(codec_search_path, i, i+1, NULL); } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_UNLOCK(&interp->codecs.search_path_mutex); Py_DECREF(item); if (ret != 1) { assert(interp->codecs.search_cache != NULL); @@ -90,16 +83,15 @@ PyCodec_Unregister(PyObject *search_function) return 0; } -extern int _Py_normalize_encoding(const char *, char *, size_t); +/* Convert a string to a normalized Python string: all ASCII letters are + converted to lower case, spaces are replaced with hyphens. */ -/* Convert a string to a normalized Python string(decoded from UTF-8): all characters are - converted to lower case, spaces and hyphens are replaced with underscores. */ - -static -PyObject *normalizestring(const char *string) +static PyObject* +normalizestring(const char *string) { + size_t i; size_t len = strlen(string); - char *encoding; + char *p; PyObject *v; if (len > PY_SSIZE_T_MAX) { @@ -107,28 +99,30 @@ PyObject *normalizestring(const char *string) return NULL; } - encoding = PyMem_Malloc(len + 1); - if (encoding == NULL) + p = PyMem_Malloc(len + 1); + if (p == NULL) return PyErr_NoMemory(); - - if (!_Py_normalize_encoding(string, encoding, len + 1)) - { - PyErr_SetString(PyExc_RuntimeError, "_Py_normalize_encoding() failed"); - PyMem_Free(encoding); - return NULL; + for (i = 0; i < len; i++) { + char ch = string[i]; + if (ch == ' ') + ch = '-'; + else + ch = Py_TOLOWER(Py_CHARMASK(ch)); + p[i] = ch; } - - v = PyUnicode_FromString(encoding); - PyMem_Free(encoding); + p[i] = '\0'; + v = PyUnicode_FromString(p); + PyMem_Free(p); return v; } /* Lookup the given encoding and return a tuple providing the codec facilities. - The encoding string is looked up converted to all lower-case - characters. This makes encodings looked up through this mechanism - effectively case-insensitive. + ASCII letters in the encoding string is looked up converted to all + lower case. This makes encodings looked up through this mechanism + effectively case-insensitive. Spaces are replaced with hyphens for + names like "US ASCII" and "ISO 8859-1". If no codec is found, a LookupError is set and NULL returned. @@ -149,8 +143,8 @@ PyObject *_PyCodec_Lookup(const char *encoding) assert(interp->codecs.initialized); /* Convert the encoding to a normalized Python string: all - characters are converted to lower case, spaces and hyphens are - replaced with underscores. */ + ASCII letters are converted to lower case, spaces are + replaced with hyphens. */ PyObject *v = normalizestring(encoding); if (v == NULL) { return NULL; @@ -1204,7 +1198,7 @@ get_standard_encoding_impl(const char *encoding, int *bytelength) } } } - else if (strcmp(encoding, "CP_UTF8") == 0) { + else if (strcmp(encoding, "cp65001") == 0) { *bytelength = 3; return ENC_UTF8; } diff --git a/Python/codegen.c b/Python/codegen.c index 683601103ec..c4109fcaa48 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -28,6 +28,8 @@ #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_symtable.h" // PySTEntryObject #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString +#include "pycore_ceval.h" // SPECIAL___ENTER__ +#include "pycore_template.h" // _PyTemplate_Type #define NEED_OPCODE_METADATA #include "pycore_opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed @@ -525,6 +527,15 @@ codegen_unwind_fblock(compiler *c, location *ploc, return SUCCESS; case COMPILE_FBLOCK_FOR_LOOP: + /* Pop the iterator */ + if (preserve_tos) { + ADDOP_I(c, *ploc, SWAP, 3); + } + ADDOP(c, *ploc, POP_TOP); + ADDOP(c, *ploc, POP_TOP); + return SUCCESS; + + case COMPILE_FBLOCK_ASYNC_FOR_LOOP: /* Pop the iterator */ if (preserve_tos) { ADDOP_I(c, *ploc, SWAP, 2); @@ -629,7 +640,8 @@ codegen_unwind_fblock_stack(compiler *c, location *ploc, c, *ploc, "'break', 'continue' and 'return' cannot appear in an except* block"); } if (loop != NULL && (top->fb_type == COMPILE_FBLOCK_WHILE_LOOP || - top->fb_type == COMPILE_FBLOCK_FOR_LOOP)) { + top->fb_type == COMPILE_FBLOCK_FOR_LOOP || + top->fb_type == COMPILE_FBLOCK_ASYNC_FOR_LOOP)) { *loop = top; return SUCCESS; } @@ -2125,7 +2137,7 @@ codegen_async_for(compiler *c, stmt_ty s) ADDOP(c, LOC(s->v.AsyncFor.iter), GET_AITER); USE_LABEL(c, start); - RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_FOR_LOOP, start, end, NULL)); + RETURN_IF_ERROR(_PyCompile_PushFBlock(c, loc, COMPILE_FBLOCK_ASYNC_FOR_LOOP, start, end, NULL)); /* SETUP_FINALLY to guard the __anext__ call */ ADDOP_JUMP(c, loc, SETUP_FINALLY, except); @@ -2142,7 +2154,7 @@ codegen_async_for(compiler *c, stmt_ty s) /* Mark jump as artificial */ ADDOP_JUMP(c, NO_LOCATION, JUMP, start); - _PyCompile_PopFBlock(c, COMPILE_FBLOCK_FOR_LOOP, start); + _PyCompile_PopFBlock(c, COMPILE_FBLOCK_ASYNC_FOR_LOOP, start); /* Except block for __anext__ */ USE_LABEL(c, except); @@ -3606,10 +3618,11 @@ infer_type(expr_ty e) return &PyGen_Type; case Lambda_kind: return &PyFunction_Type; - case JoinedStr_kind: case TemplateStr_kind: - case FormattedValue_kind: case Interpolation_kind: + return &_PyTemplate_Type; + case JoinedStr_kind: + case FormattedValue_kind: return &PyUnicode_Type; case Constant_kind: return Py_TYPE(e->v.Constant.value); @@ -3663,6 +3676,8 @@ check_subscripter(compiler *c, expr_ty e) case Set_kind: case SetComp_kind: case GeneratorExp_kind: + case TemplateStr_kind: + case Interpolation_kind: case Lambda_kind: { location loc = LOC(e); return _PyCompile_Warn(c, loc, "'%.200s' object is not subscriptable; " @@ -3697,9 +3712,7 @@ check_index(compiler *c, expr_ty e, expr_ty s) case List_kind: case ListComp_kind: case JoinedStr_kind: - case TemplateStr_kind: - case FormattedValue_kind: - case Interpolation_kind: { + case FormattedValue_kind: { location loc = LOC(e); return _PyCompile_Warn(c, loc, "%.200s indices must be integers " "or slices, not %.200s; " @@ -3895,10 +3908,11 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) NEW_JUMP_TARGET_LABEL(c, loop); NEW_JUMP_TARGET_LABEL(c, cleanup); + ADDOP(c, loc, PUSH_NULL); // Push NULL index for loop USE_LABEL(c, loop); ADDOP_JUMP(c, loc, FOR_ITER, cleanup); if (const_oparg == CONSTANT_BUILTIN_TUPLE) { - ADDOP_I(c, loc, LIST_APPEND, 2); + ADDOP_I(c, loc, LIST_APPEND, 3); ADDOP_JUMP(c, loc, JUMP, loop); } else { @@ -4069,16 +4083,6 @@ codegen_template_str(compiler *c, expr_ty e) } else { VISIT(c, expr, value); - Py_ssize_t j; - for (j = i + 1; j < value_count; j++) { - value = asdl_seq_GET(e->v.TemplateStr.values, j); - if (value->kind == Interpolation_kind) { - break; - } - VISIT(c, expr, value); - ADDOP_INPLACE(c, loc, Add); - } - i = j - 1; stringslen++; last_was_interpolation = 0; } @@ -4442,13 +4446,12 @@ codegen_sync_comprehension_generator(compiler *c, location loc, } if (IS_JUMP_TARGET_LABEL(start)) { VISIT(c, expr, gen->iter); - ADDOP(c, LOC(gen->iter), GET_ITER); } } } if (IS_JUMP_TARGET_LABEL(start)) { - depth++; + depth += 2; ADDOP(c, LOC(gen->iter), GET_ITER); USE_LABEL(c, start); ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor); @@ -4543,9 +4546,9 @@ codegen_async_comprehension_generator(compiler *c, location loc, else { /* Sub-iter - calculate on the fly */ VISIT(c, expr, gen->iter); - ADDOP(c, LOC(gen->iter), GET_AITER); } } + ADDOP(c, LOC(gen->iter), GET_AITER); USE_LABEL(c, start); /* Runtime will push a block here, so we need to account for that */ @@ -4757,19 +4760,6 @@ pop_inlined_comprehension_state(compiler *c, location loc, return SUCCESS; } -static inline int -codegen_comprehension_iter(compiler *c, comprehension_ty comp) -{ - VISIT(c, expr, comp->iter); - if (comp->is_async) { - ADDOP(c, LOC(comp->iter), GET_AITER); - } - else { - ADDOP(c, LOC(comp->iter), GET_ITER); - } - return SUCCESS; -} - static int codegen_comprehension(compiler *c, expr_ty e, int type, identifier name, asdl_comprehension_seq *generators, expr_ty elt, @@ -4789,9 +4779,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, outermost = (comprehension_ty) asdl_seq_GET(generators, 0); if (is_inlined) { - if (codegen_comprehension_iter(c, outermost)) { - goto error; - } + VISIT(c, expr, outermost->iter); if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { goto error; } @@ -5426,23 +5414,6 @@ codegen_check_ann_expr(compiler *c, expr_ty e) return SUCCESS; } -static int -codegen_check_annotation(compiler *c, stmt_ty s) -{ - /* Annotations of complex targets does not produce anything - under annotations future */ - if (FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) { - return SUCCESS; - } - - /* Annotations are only evaluated in a module or class. */ - if (SCOPE_TYPE(c) == COMPILE_SCOPE_MODULE || - SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS) { - return codegen_check_ann_expr(c, s->v.AnnAssign.annotation); - } - return SUCCESS; -} - static int codegen_check_ann_subscr(compiler *c, expr_ty e) { @@ -5506,10 +5477,12 @@ codegen_annassign(compiler *c, stmt_ty s) RETURN_IF_ERROR(_PyCompile_AddDeferredAnnotation( c, s, &conditional_annotation_index)); if (conditional_annotation_index != NULL) { - ADDOP_NAME( - c, loc, - SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS ? LOAD_DEREF : LOAD_NAME, - &_Py_ID(__conditional_annotations__), cellvars); + if (SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS) { + ADDOP_NAME(c, loc, LOAD_DEREF, &_Py_ID(__conditional_annotations__), cellvars); + } + else { + ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__conditional_annotations__), names); + } ADDOP_LOAD_CONST_NEW(c, loc, conditional_annotation_index); ADDOP_I(c, loc, SET_ADD, 1); ADDOP(c, loc, POP_TOP); @@ -5536,10 +5509,6 @@ codegen_annassign(compiler *c, stmt_ty s) targ->kind); return ERROR; } - /* Annotation is evaluated last. */ - if (future_annotations && !s->v.AnnAssign.simple && codegen_check_annotation(c, s) < 0) { - return ERROR; - } return SUCCESS; } diff --git a/Python/compile.c b/Python/compile.c index c04391e682f..6951c98500d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -103,11 +103,14 @@ typedef struct _PyCompiler { bool c_save_nested_seqs; /* if true, construct recursive instruction sequences * (including instructions for nested code objects) */ + int c_disable_warning; + PyObject *c_module; } compiler; static int compiler_setup(compiler *c, mod_ty mod, PyObject *filename, - PyCompilerFlags *flags, int optimize, PyArena *arena) + PyCompilerFlags *flags, int optimize, PyArena *arena, + PyObject *module) { PyCompilerFlags local_flags = _PyCompilerFlags_INIT; @@ -125,6 +128,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return ERROR; } + c->c_module = Py_XNewRef(module); if (!flags) { flags = &local_flags; } @@ -135,7 +139,9 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, 0)) { + if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, + 0, 1, module)) + { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -155,6 +161,7 @@ compiler_free(compiler *c) _PySymtable_Free(c->c_st); } Py_XDECREF(c->c_filename); + Py_XDECREF(c->c_module); Py_XDECREF(c->c_const_cache); Py_XDECREF(c->c_stack); PyMem_Free(c); @@ -162,13 +169,13 @@ compiler_free(compiler *c) static compiler* new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { compiler *c = PyMem_Calloc(1, sizeof(compiler)); if (c == NULL) { return NULL; } - if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { + if (compiler_setup(c, mod, filename, pflags, optimize, arena, module) < 0) { compiler_free(c); return NULL; } @@ -765,6 +772,9 @@ _PyCompile_PushFBlock(compiler *c, location loc, f->fb_loc = loc; f->fb_exit = exit; f->fb_datum = datum; + if (t == COMPILE_FBLOCK_FINALLY_END) { + c->c_disable_warning++; + } return SUCCESS; } @@ -776,6 +786,9 @@ _PyCompile_PopFBlock(compiler *c, fblocktype t, jump_target_label block_label) u->u_nfblocks--; assert(u->u_fblock[u->u_nfblocks].fb_type == t); assert(SAME_JUMP_TARGET_LABEL(u->u_fblock[u->u_nfblocks].fb_block, block_label)); + if (t == COMPILE_FBLOCK_FINALLY_END) { + c->c_disable_warning--; + } } fblockinfo * @@ -1203,6 +1216,9 @@ _PyCompile_Error(compiler *c, location loc, const char *format, ...) int _PyCompile_Warn(compiler *c, location loc, const char *format, ...) { + if (c->c_disable_warning) { + return 0; + } va_list vargs; va_start(vargs, format); PyObject *msg = PyUnicode_FromFormatV(format, vargs); @@ -1211,7 +1227,8 @@ _PyCompile_Warn(compiler *c, location loc, const char *format, ...) return ERROR; } int ret = _PyErr_EmitSyntaxWarning(msg, c->c_filename, loc.lineno, loc.col_offset + 1, - loc.end_lineno, loc.end_col_offset + 1); + loc.end_lineno, loc.end_col_offset + 1, + c->c_module); Py_DECREF(msg); return ret; } @@ -1466,10 +1483,10 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) PyCodeObject * _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { assert(!PyErr_Occurred()); - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, module); if (c == NULL) { return NULL; } @@ -1482,7 +1499,8 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena, int no_const_folding) + int optimize, PyArena *arena, int no_const_folding, + PyObject *module) { _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { @@ -1492,7 +1510,9 @@ _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, no_const_folding)) { + if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, + no_const_folding, 0, module)) + { return -1; } return 0; @@ -1617,7 +1637,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, NULL); if (c == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Python/context.c b/Python/context.c index dceaae9b429..2f978b1c0ab 100644 --- a/Python/context.c +++ b/Python/context.c @@ -979,7 +979,7 @@ contextvar_tp_repr(PyObject *op) return NULL; } - if (PyUnicodeWriter_WriteUTF8(writer, "<ContextVar name=", 17) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "<ContextVar name=", 17) < 0) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, self->var_name) < 0) { @@ -987,7 +987,7 @@ contextvar_tp_repr(PyObject *op) } if (self->var_default != NULL) { - if (PyUnicodeWriter_WriteUTF8(writer, " default=", 9) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, " default=", 9) < 0) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, self->var_default) < 0) { @@ -1007,6 +1007,7 @@ contextvar_tp_repr(PyObject *op) /*[clinic input] +@permit_long_docstring_body _contextvars.ContextVar.get default: object = NULL / @@ -1022,7 +1023,7 @@ If there is no value for the variable in the current context, the method will: static PyObject * _contextvars_ContextVar_get_impl(PyContextVar *self, PyObject *default_value) -/*[clinic end generated code: output=0746bd0aa2ced7bf input=30aa2ab9e433e401]*/ +/*[clinic end generated code: output=0746bd0aa2ced7bf input=da66664d5d0af4ad]*/ { if (!PyContextVar_CheckExact(self)) { PyErr_SetString( @@ -1044,6 +1045,7 @@ _contextvars_ContextVar_get_impl(PyContextVar *self, PyObject *default_value) } /*[clinic input] +@permit_long_docstring_body _contextvars.ContextVar.set value: object / @@ -1058,12 +1060,13 @@ value via the `ContextVar.reset()` method. static PyObject * _contextvars_ContextVar_set_impl(PyContextVar *self, PyObject *value) -/*[clinic end generated code: output=1b562d35cc79c806 input=c0a6887154227453]*/ +/*[clinic end generated code: output=1b562d35cc79c806 input=73ebbbfc7c98f6cd]*/ { return PyContextVar_Set((PyObject *)self, value); } /*[clinic input] +@permit_long_docstring_body _contextvars.ContextVar.reset token: object / @@ -1076,7 +1079,7 @@ created the token was used. static PyObject * _contextvars_ContextVar_reset_impl(PyContextVar *self, PyObject *token) -/*[clinic end generated code: output=3205d2bdff568521 input=ebe2881e5af4ffda]*/ +/*[clinic end generated code: output=3205d2bdff568521 input=b8bc514a9245242a]*/ { if (!PyContextToken_CheckExact(token)) { PyErr_Format(PyExc_TypeError, @@ -1182,15 +1185,15 @@ token_tp_repr(PyObject *op) if (writer == NULL) { return NULL; } - if (PyUnicodeWriter_WriteUTF8(writer, "<Token", 6) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "<Token", 6) < 0) { goto error; } if (self->tok_used) { - if (PyUnicodeWriter_WriteUTF8(writer, " used", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, " used", 5) < 0) { goto error; } } - if (PyUnicodeWriter_WriteUTF8(writer, " var=", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, " var=", 5) < 0) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, (PyObject *)self->tok_var) < 0) { @@ -1357,11 +1360,8 @@ get_token_missing(void) PyStatus _PyContext_Init(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return _PyStatus_OK(); - } - PyObject *missing = get_token_missing(); + assert(PyUnstable_IsImmortal(missing)); if (PyDict_SetItemString( _PyType_GetDict(&PyContextToken_Type), "MISSING", missing)) { diff --git a/Python/critical_section.c b/Python/critical_section.c index 73857b85496..2c2152f5de4 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -17,18 +17,30 @@ untag_critical_section(uintptr_t tag) #endif void -_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) +_PyCriticalSection_BeginSlow(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m) { #ifdef Py_GIL_DISABLED - PyThreadState *tstate = _PyThreadState_GET(); // As an optimisation for locking the same object recursively, skip // locking if the mutex is currently locked by the top-most critical // section. - if (tstate->critical_section && - untag_critical_section(tstate->critical_section)->_cs_mutex == m) { - c->_cs_mutex = NULL; - c->_cs_prev = 0; - return; + // If the top-most critical section is a two-mutex critical section, + // then locking is skipped if either mutex is m. + if (tstate->critical_section) { + PyCriticalSection *prev = untag_critical_section(tstate->critical_section); + if (prev->_cs_mutex == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + if (tstate->critical_section & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *prev2 = (PyCriticalSection2 *) + untag_critical_section(tstate->critical_section); + if (prev2->_cs_mutex2 == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + } } c->_cs_mutex = NULL; c->_cs_prev = (uintptr_t)tstate->critical_section; @@ -40,11 +52,10 @@ _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) } void -_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, +_PyCriticalSection2_BeginSlow(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, int is_m1_locked) { #ifdef Py_GIL_DISABLED - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_base._cs_mutex = NULL; c->_cs_mutex2 = NULL; c->_cs_base._cs_prev = tstate->critical_section; @@ -126,7 +137,16 @@ void PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) { #ifdef Py_GIL_DISABLED - _PyCriticalSection_Begin(c, op); + _PyCriticalSection_Begin(_PyThreadState_GET(), c, op); +#endif +} + +#undef PyCriticalSection_BeginMutex +void +PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection_BeginMutex(_PyThreadState_GET(), c, m); #endif } @@ -135,7 +155,7 @@ void PyCriticalSection_End(PyCriticalSection *c) { #ifdef Py_GIL_DISABLED - _PyCriticalSection_End(c); + _PyCriticalSection_End(_PyThreadState_GET(), c); #endif } @@ -144,7 +164,16 @@ void PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) { #ifdef Py_GIL_DISABLED - _PyCriticalSection2_Begin(c, a, b); + _PyCriticalSection2_Begin(_PyThreadState_GET(), c, a, b); +#endif +} + +#undef PyCriticalSection2_BeginMutex +void +PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection2_BeginMutex(_PyThreadState_GET(), c, m1, m2); #endif } @@ -153,6 +182,6 @@ void PyCriticalSection2_End(PyCriticalSection2 *c) { #ifdef Py_GIL_DISABLED - _PyCriticalSection2_End(c); + _PyCriticalSection2_End(_PyThreadState_GET(), c); #endif } diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 74ce02f1a26..542253c14de 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -6,8 +6,14 @@ #include "osdefs.h" // MAXPATHLEN #include "pycore_ceval.h" // _Py_simple_func #include "pycore_crossinterp.h" // _PyXIData_t +#include "pycore_function.h" // _PyFunction_VerifyStateless() +#include "pycore_global_strings.h" // _Py_ID() +#include "pycore_import.h" // _PyImport_SetModule() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" // _PyNamespace_New() +#include "pycore_pythonrun.h" // _Py_SourceAsString() +#include "pycore_runtime.h" // _PyRuntime +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_typeobject.h" // _PyStaticType_InitBuiltin() @@ -19,6 +25,7 @@ _Py_GetMainfile(char *buffer, size_t maxlen) PyThreadState *tstate = _PyThreadState_GET(); PyObject *module = _Py_GetMainModule(tstate); if (_Py_CheckMainModule(module) < 0) { + Py_XDECREF(module); return -1; } Py_ssize_t size = _PyModule_GetFilenameUTF8(module, buffer, maxlen); @@ -27,27 +34,6 @@ _Py_GetMainfile(char *buffer, size_t maxlen) } -static PyObject * -import_get_module(PyThreadState *tstate, const char *modname) -{ - PyObject *module = NULL; - if (strcmp(modname, "__main__") == 0) { - module = _Py_GetMainModule(tstate); - if (_Py_CheckMainModule(module) < 0) { - assert(_PyErr_Occurred(tstate)); - return NULL; - } - } - else { - module = PyImport_ImportModule(modname); - if (module == NULL) { - return NULL; - } - } - return module; -} - - static PyObject * runpy_run_path(const char *filename, const char *modname) { @@ -67,97 +53,192 @@ runpy_run_path(const char *filename, const char *modname) } -static PyObject * -pyerr_get_message(PyObject *exc) +static void +set_exc_with_cause(PyObject *exctype, const char *msg) { - assert(!PyErr_Occurred()); - PyObject *args = PyException_GetArgs(exc); - if (args == NULL || args == Py_None || PyObject_Size(args) < 1) { - return NULL; - } - if (PyUnicode_Check(args)) { - return args; - } - PyObject *msg = PySequence_GetItem(args, 0); - Py_DECREF(args); - if (msg == NULL) { - PyErr_Clear(); - return NULL; - } - if (!PyUnicode_Check(msg)) { - Py_DECREF(msg); - return NULL; - } - return msg; + PyObject *cause = PyErr_GetRaisedException(); + PyErr_SetString(exctype, msg); + PyObject *exc = PyErr_GetRaisedException(); + PyException_SetCause(exc, cause); + PyErr_SetRaisedException(exc); } -#define MAX_MODNAME (255) -#define MAX_ATTRNAME (255) -struct attributeerror_info { - char modname[MAX_MODNAME+1]; - char attrname[MAX_ATTRNAME+1]; +/****************************/ +/* module duplication utils */ +/****************************/ + +struct sync_module_result { + PyObject *module; + PyObject *loaded; + PyObject *failed; }; -static int -_parse_attributeerror(PyObject *exc, struct attributeerror_info *info) +struct sync_module { + const char *filename; + char _filename[MAXPATHLEN+1]; + struct sync_module_result cached; +}; + +static void +sync_module_clear(struct sync_module *data) { - assert(exc != NULL); - assert(PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)); - int res = -1; - - PyObject *msgobj = pyerr_get_message(exc); - if (msgobj == NULL) { - return -1; - } - const char *err = PyUnicode_AsUTF8(msgobj); - - if (strncmp(err, "module '", 8) != 0) { - goto finally; - } - err += 8; - - const char *matched = strchr(err, '\''); - if (matched == NULL) { - goto finally; - } - Py_ssize_t len = matched - err; - if (len > MAX_MODNAME) { - goto finally; - } - (void)strncpy(info->modname, err, len); - info->modname[len] = '\0'; - err = matched; - - if (strncmp(err, "' has no attribute '", 20) != 0) { - goto finally; - } - err += 20; - - matched = strchr(err, '\''); - if (matched == NULL) { - goto finally; - } - len = matched - err; - if (len > MAX_ATTRNAME) { - goto finally; - } - (void)strncpy(info->attrname, err, len); - info->attrname[len] = '\0'; - err = matched + 1; - - if (strlen(err) > 0) { - goto finally; - } - res = 0; - -finally: - Py_DECREF(msgobj); - return res; + data->filename = NULL; + Py_CLEAR(data->cached.module); + Py_CLEAR(data->cached.loaded); + Py_CLEAR(data->cached.failed); } -#undef MAX_MODNAME -#undef MAX_ATTRNAME +static void +sync_module_capture_exc(PyThreadState *tstate, struct sync_module *data) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *context = data->cached.failed; + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc)); + if (context != NULL) { + PyException_SetContext(exc, context); + } + data->cached.failed = exc; +} + + +static int +ensure_isolated_main(PyThreadState *tstate, struct sync_module *main) +{ + // Load the module from the original file (or from a cache). + + // First try the local cache. + if (main->cached.failed != NULL) { + // We'll deal with this in apply_isolated_main(). + assert(main->cached.module == NULL); + assert(main->cached.loaded == NULL); + return 0; + } + else if (main->cached.loaded != NULL) { + assert(main->cached.module != NULL); + return 0; + } + assert(main->cached.module == NULL); + + if (main->filename == NULL) { + _PyErr_SetString(tstate, PyExc_NotImplementedError, ""); + return -1; + } + + // It wasn't in the local cache so we'll need to populate it. + PyObject *mod = _Py_GetMainModule(tstate); + if (_Py_CheckMainModule(mod) < 0) { + // This is probably unrecoverable, so don't bother caching the error. + assert(_PyErr_Occurred(tstate)); + Py_XDECREF(mod); + return -1; + } + PyObject *loaded = NULL; + + // Try the per-interpreter cache for the loaded module. + // XXX Store it in sys.modules? + PyObject *interpns = PyInterpreterState_GetDict(tstate->interp); + assert(interpns != NULL); + PyObject *key = PyUnicode_FromString("CACHED_MODULE_NS___main__"); + if (key == NULL) { + // It's probably unrecoverable, so don't bother caching the error. + Py_DECREF(mod); + return -1; + } + else if (PyDict_GetItemRef(interpns, key, &loaded) < 0) { + // It's probably unrecoverable, so don't bother caching the error. + Py_DECREF(mod); + Py_DECREF(key); + return -1; + } + else if (loaded == NULL) { + // It wasn't already loaded from file. + loaded = PyModule_NewObject(&_Py_ID(__main__)); + if (loaded == NULL) { + goto error; + } + PyObject *ns = _PyModule_GetDict(loaded); + + // We don't want to trigger "if __name__ == '__main__':", + // so we use a bogus module name. + PyObject *loaded_ns = + runpy_run_path(main->filename, "<fake __main__>"); + if (loaded_ns == NULL) { + goto error; + } + int res = PyDict_Update(ns, loaded_ns); + Py_DECREF(loaded_ns); + if (res < 0) { + goto error; + } + + // Set the per-interpreter cache entry. + if (PyDict_SetItem(interpns, key, loaded) < 0) { + goto error; + } + } + + Py_DECREF(key); + main->cached = (struct sync_module_result){ + .module = mod, + .loaded = loaded, + }; + return 0; + +error: + sync_module_capture_exc(tstate, main); + Py_XDECREF(loaded); + Py_DECREF(mod); + Py_XDECREF(key); + return -1; +} + +#ifndef NDEBUG +static int +main_mod_matches(PyObject *expected) +{ + PyObject *mod = PyImport_GetModule(&_Py_ID(__main__)); + Py_XDECREF(mod); + return mod == expected; +} +#endif + +static int +apply_isolated_main(PyThreadState *tstate, struct sync_module *main) +{ + assert((main->cached.loaded == NULL) == (main->cached.loaded == NULL)); + if (main->cached.failed != NULL) { + // It must have failed previously. + assert(main->cached.loaded == NULL); + _PyErr_SetRaisedException(tstate, main->cached.failed); + return -1; + } + assert(main->cached.loaded != NULL); + + assert(main_mod_matches(main->cached.module)); + if (_PyImport_SetModule(&_Py_ID(__main__), main->cached.loaded) < 0) { + sync_module_capture_exc(tstate, main); + return -1; + } + return 0; +} + +static void +restore_main(PyThreadState *tstate, struct sync_module *main) +{ + assert(main->cached.failed == NULL); + assert(main->cached.module != NULL); + assert(main->cached.loaded != NULL); + PyObject *exc = _PyErr_GetRaisedException(tstate); + assert(main_mod_matches(main->cached.loaded)); + int res = _PyImport_SetModule(&_Py_ID(__main__), main->cached.module); + assert(res == 0); + if (res < 0) { + PyErr_FormatUnraisable("Exception ignored while restoring __main__"); + } + _PyErr_SetRaisedException(tstate, exc); +} /**************/ @@ -207,16 +288,16 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, /* cross-interpreter data */ /**************************/ -/* registry of {type -> xidatafunc} */ +/* registry of {type -> _PyXIData_getdata_t} */ -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - xidatafunc. It would be simpler and more efficient. */ +/* For now we use a global registry of shareable classes. + An alternative would be to add a tp_* slot for a class's + _PyXIData_getdata_t. It would be simpler and more efficient. */ static void xid_lookup_init(_PyXIData_lookup_t *); static void xid_lookup_fini(_PyXIData_lookup_t *); struct _dlcontext; -static xidatafunc lookup_getdata(struct _dlcontext *, PyObject *); +static _PyXIData_getdata_t lookup_getdata(struct _dlcontext *, PyObject *); #include "crossinterp_data_lookup.h" @@ -340,7 +421,7 @@ _set_xid_lookup_failure(PyThreadState *tstate, PyObject *obj, const char *msg, set_notshareableerror(tstate, cause, 0, msg); } else { - msg = "%S does not support cross-interpreter data"; + msg = "%R does not support cross-interpreter data"; format_notshareableerror(tstate, cause, 0, msg, obj); } } @@ -353,8 +434,8 @@ _PyObject_CheckXIData(PyThreadState *tstate, PyObject *obj) if (get_lookup_context(tstate, &ctx) < 0) { return -1; } - xidatafunc getdata = lookup_getdata(&ctx, obj); - if (getdata == NULL) { + _PyXIData_getdata_t getdata = lookup_getdata(&ctx, obj); + if (getdata.basic == NULL && getdata.fallback == NULL) { if (!_PyErr_Occurred(tstate)) { _set_xid_lookup_failure(tstate, obj, NULL, NULL); } @@ -385,9 +466,9 @@ _check_xidata(PyThreadState *tstate, _PyXIData_t *xidata) return 0; } -int -_PyObject_GetXIData(PyThreadState *tstate, - PyObject *obj, _PyXIData_t *xidata) +static int +_get_xidata(PyThreadState *tstate, + PyObject *obj, xidata_fallback_t fallback, _PyXIData_t *xidata) { PyInterpreterState *interp = tstate->interp; @@ -395,6 +476,7 @@ _PyObject_GetXIData(PyThreadState *tstate, assert(xidata->obj == NULL); if (xidata->data != NULL || xidata->obj != NULL) { _PyErr_SetString(tstate, PyExc_ValueError, "xidata not cleared"); + return -1; } // Call the "getdata" func for the object. @@ -403,8 +485,8 @@ _PyObject_GetXIData(PyThreadState *tstate, return -1; } Py_INCREF(obj); - xidatafunc getdata = lookup_getdata(&ctx, obj); - if (getdata == NULL) { + _PyXIData_getdata_t getdata = lookup_getdata(&ctx, obj); + if (getdata.basic == NULL && getdata.fallback == NULL) { if (PyErr_Occurred()) { Py_DECREF(obj); return -1; @@ -416,7 +498,9 @@ _PyObject_GetXIData(PyThreadState *tstate, } return -1; } - int res = getdata(tstate, obj, xidata); + int res = getdata.basic != NULL + ? getdata.basic(tstate, obj, xidata) + : getdata.fallback(tstate, obj, fallback, xidata); Py_DECREF(obj); if (res != 0) { PyObject *cause = _PyErr_GetRaisedException(tstate); @@ -436,6 +520,51 @@ _PyObject_GetXIData(PyThreadState *tstate, return 0; } +int +_PyObject_GetXIDataNoFallback(PyThreadState *tstate, + PyObject *obj, _PyXIData_t *xidata) +{ + return _get_xidata(tstate, obj, _PyXIDATA_XIDATA_ONLY, xidata); +} + +int +_PyObject_GetXIData(PyThreadState *tstate, + PyObject *obj, xidata_fallback_t fallback, + _PyXIData_t *xidata) +{ + switch (fallback) { + case _PyXIDATA_XIDATA_ONLY: + return _get_xidata(tstate, obj, fallback, xidata); + case _PyXIDATA_FULL_FALLBACK: + if (_get_xidata(tstate, obj, fallback, xidata) == 0) { + return 0; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (PyFunction_Check(obj)) { + if (_PyFunction_GetXIData(tstate, obj, xidata) == 0) { + Py_DECREF(exc); + return 0; + } + _PyErr_Clear(tstate); + } + // We could try _PyMarshal_GetXIData() but we won't for now. + if (_PyPickle_GetXIData(tstate, obj, xidata) == 0) { + Py_DECREF(exc); + return 0; + } + // Raise the original exception. + _PyErr_SetRaisedException(tstate, exc); + return -1; + default: +#ifdef Py_DEBUG + Py_FatalError("unsupported xidata fallback option"); +#endif + _PyErr_SetString(tstate, PyExc_SystemError, + "unsupported xidata fallback option"); + return -1; + } +} + /* pickle C-API */ @@ -456,28 +585,6 @@ _PyPickle_Dumps(struct _pickle_context *ctx, PyObject *obj) } -struct sync_module_result { - PyObject *module; - PyObject *loaded; - PyObject *failed; -}; - -struct sync_module { - const char *filename; - char _filename[MAXPATHLEN+1]; - struct sync_module_result cached; -}; - -static void -sync_module_clear(struct sync_module *data) -{ - data->filename = NULL; - Py_CLEAR(data->cached.module); - Py_CLEAR(data->cached.loaded); - Py_CLEAR(data->cached.failed); -} - - struct _unpickle_context { PyThreadState *tstate; // We only special-case the __main__ module, @@ -491,142 +598,88 @@ _unpickle_context_clear(struct _unpickle_context *ctx) sync_module_clear(&ctx->main); } -static struct sync_module_result -_unpickle_context_get_module(struct _unpickle_context *ctx, - const char *modname) -{ - if (strcmp(modname, "__main__") == 0) { - return ctx->main.cached; - } - else { - return (struct sync_module_result){ - .failed = PyExc_NotImplementedError, - }; - } -} - -static struct sync_module_result -_unpickle_context_set_module(struct _unpickle_context *ctx, - const char *modname) -{ - struct sync_module_result res = {0}; - struct sync_module_result *cached = NULL; - const char *filename = NULL; - const char *run_modname = modname; - if (strcmp(modname, "__main__") == 0) { - cached = &ctx->main.cached; - filename = ctx->main.filename; - // We don't want to trigger "if __name__ == '__main__':". - run_modname = "<fake __main__>"; - } - else { - res.failed = PyExc_NotImplementedError; - goto finally; - } - - res.module = import_get_module(ctx->tstate, modname); - if (res.module == NULL) { - res.failed = _PyErr_GetRaisedException(ctx->tstate); - assert(res.failed != NULL); - goto finally; - } - - if (filename == NULL) { - Py_CLEAR(res.module); - res.failed = PyExc_NotImplementedError; - goto finally; - } - res.loaded = runpy_run_path(filename, run_modname); - if (res.loaded == NULL) { - Py_CLEAR(res.module); - res.failed = _PyErr_GetRaisedException(ctx->tstate); - assert(res.failed != NULL); - goto finally; - } - -finally: - if (cached != NULL) { - assert(cached->module == NULL); - assert(cached->loaded == NULL); - assert(cached->failed == NULL); - *cached = res; - } - return res; -} - - static int -_handle_unpickle_missing_attr(struct _unpickle_context *ctx, PyObject *exc) +check_missing___main___attr(PyObject *exc) { - // The caller must check if an exception is set or not when -1 is returned. - assert(!_PyErr_Occurred(ctx->tstate)); - assert(PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)); - struct attributeerror_info info; - if (_parse_attributeerror(exc, &info) < 0) { - return -1; + assert(!PyErr_Occurred()); + if (!PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)) { + return 0; } - // Get the module. - struct sync_module_result mod = _unpickle_context_get_module(ctx, info.modname); - if (mod.failed != NULL) { - // It must have failed previously. - return -1; + // Get the error message. + PyObject *args = PyException_GetArgs(exc); + if (args == NULL || args == Py_None || PyObject_Size(args) < 1) { + assert(!PyErr_Occurred()); + return 0; } - if (mod.module == NULL) { - mod = _unpickle_context_set_module(ctx, info.modname); - if (mod.failed != NULL) { - return -1; + PyObject *msgobj = args; + if (!PyUnicode_Check(msgobj)) { + msgobj = PySequence_GetItem(args, 0); + Py_DECREF(args); + if (msgobj == NULL) { + PyErr_Clear(); + return 0; } - assert(mod.module != NULL); } + const char *err = PyUnicode_AsUTF8(msgobj); - // Bail out if it is unexpectedly set already. - if (PyObject_HasAttrString(mod.module, info.attrname)) { - return -1; - } - - // Try setting the attribute. - PyObject *value = NULL; - if (PyDict_GetItemStringRef(mod.loaded, info.attrname, &value) <= 0) { - return -1; - } - assert(value != NULL); - int res = PyObject_SetAttrString(mod.module, info.attrname, value); - Py_DECREF(value); - if (res < 0) { - return -1; - } - - return 0; + // Check if it's a missing __main__ attr. + int cmp = strncmp(err, "module '__main__' has no attribute '", 36); + Py_DECREF(msgobj); + return cmp == 0; } static PyObject * _PyPickle_Loads(struct _unpickle_context *ctx, PyObject *pickled) { + PyThreadState *tstate = ctx->tstate; + + PyObject *exc = NULL; PyObject *loads = PyImport_ImportModuleAttrString("pickle", "loads"); if (loads == NULL) { return NULL; } + + // Make an initial attempt to unpickle. PyObject *obj = PyObject_CallOneArg(loads, pickled); - if (ctx != NULL) { - while (obj == NULL) { - assert(_PyErr_Occurred(ctx->tstate)); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - // We leave other failures unhandled. - break; - } - // Try setting the attr if not set. - PyObject *exc = _PyErr_GetRaisedException(ctx->tstate); - if (_handle_unpickle_missing_attr(ctx, exc) < 0) { - // Any resulting exceptions are ignored - // in favor of the original. - _PyErr_SetRaisedException(ctx->tstate, exc); - break; - } - Py_CLEAR(exc); - // Retry with the attribute set. - obj = PyObject_CallOneArg(loads, pickled); + if (obj != NULL) { + goto finally; + } + assert(_PyErr_Occurred(tstate)); + if (ctx == NULL) { + goto finally; + } + exc = _PyErr_GetRaisedException(tstate); + if (!check_missing___main___attr(exc)) { + goto finally; + } + + // Temporarily swap in a fake __main__ loaded from the original + // file and cached. Note that functions will use the cached ns + // for __globals__, // not the actual module. + if (ensure_isolated_main(tstate, &ctx->main) < 0) { + goto finally; + } + if (apply_isolated_main(tstate, &ctx->main) < 0) { + goto finally; + } + + // Try to unpickle once more. + obj = PyObject_CallOneArg(loads, pickled); + restore_main(tstate, &ctx->main); + if (obj == NULL) { + goto finally; + } + Py_CLEAR(exc); + +finally: + if (exc != NULL) { + if (_PyErr_Occurred(tstate)) { + sync_module_capture_exc(tstate, &ctx->main); } + // We restore the original exception. + // It might make sense to chain it (__context__). + _PyErr_SetRaisedException(tstate, exc); } Py_DECREF(loads); return obj; @@ -784,6 +837,138 @@ _PyMarshal_GetXIData(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) } +/* script wrapper */ + +static int +verify_script(PyThreadState *tstate, PyCodeObject *co, int checked, int pure) +{ + // Make sure it isn't a closure and (optionally) doesn't use globals. + PyObject *builtins = NULL; + if (pure) { + builtins = _PyEval_GetBuiltins(tstate); + assert(builtins != NULL); + } + if (checked) { + assert(_PyCode_VerifyStateless(tstate, co, NULL, NULL, builtins) == 0); + } + else if (_PyCode_VerifyStateless(tstate, co, NULL, NULL, builtins) < 0) { + return -1; + } + // Make sure it doesn't have args. + if (co->co_argcount > 0 + || co->co_posonlyargcount > 0 + || co->co_kwonlyargcount > 0 + || co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) + { + _PyErr_SetString(tstate, PyExc_ValueError, + "code with args not supported"); + return -1; + } + // Make sure it doesn't return anything. + if (!_PyCode_ReturnsOnlyNone(co)) { + _PyErr_SetString(tstate, PyExc_ValueError, + "code that returns a value is not a script"); + return -1; + } + return 0; +} + +static int +get_script_xidata(PyThreadState *tstate, PyObject *obj, int pure, + _PyXIData_t *xidata) +{ + // Get the corresponding code object. + PyObject *code = NULL; + int checked = 0; + if (PyCode_Check(obj)) { + code = obj; + Py_INCREF(code); + } + else if (PyFunction_Check(obj)) { + code = PyFunction_GET_CODE(obj); + assert(code != NULL); + Py_INCREF(code); + if (pure) { + if (_PyFunction_VerifyStateless(tstate, obj) < 0) { + goto error; + } + checked = 1; + } + } + else { + const char *filename = "<script>"; + int optimize = 0; + PyCompilerFlags cf = _PyCompilerFlags_INIT; + cf.cf_flags = PyCF_SOURCE_IS_UTF8; + PyObject *ref = NULL; + const char *script = _Py_SourceAsString(obj, "???", "???", &cf, &ref); + if (script == NULL) { + if (!_PyObject_SupportedAsScript(obj)) { + // We discard the raised exception. + _PyErr_Format(tstate, PyExc_TypeError, + "unsupported script %R", obj); + } + goto error; + } +#ifdef Py_GIL_DISABLED + // Don't immortalize code constants to avoid memory leaks. + ((_PyThreadStateImpl *)tstate)->suppress_co_const_immortalization++; +#endif + code = Py_CompileStringExFlags( + script, filename, Py_file_input, &cf, optimize); +#ifdef Py_GIL_DISABLED + ((_PyThreadStateImpl *)tstate)->suppress_co_const_immortalization--; +#endif + Py_XDECREF(ref); + if (code == NULL) { + goto error; + } + // Compiled text can't have args or any return statements, + // nor be a closure. It can use globals though. + if (!pure) { + // We don't need to check for globals either. + checked = 1; + } + } + + // Make sure it's actually a script. + if (verify_script(tstate, (PyCodeObject *)code, checked, pure) < 0) { + goto error; + } + + // Convert the code object. + int res = _PyCode_GetXIData(tstate, code, xidata); + Py_DECREF(code); + if (res < 0) { + return -1; + } + return 0; + +error: + Py_XDECREF(code); + PyObject *cause = _PyErr_GetRaisedException(tstate); + assert(cause != NULL); + _set_xid_lookup_failure( + tstate, NULL, "object not a valid script", cause); + Py_DECREF(cause); + return -1; +} + +int +_PyCode_GetScriptXIData(PyThreadState *tstate, + PyObject *obj, _PyXIData_t *xidata) +{ + return get_script_xidata(tstate, obj, 0, xidata); +} + +int +_PyCode_GetPureScriptXIData(PyThreadState *tstate, + PyObject *obj, _PyXIData_t *xidata) +{ + return get_script_xidata(tstate, obj, 1, xidata); +} + + /* using cross-interpreter data */ PyObject * @@ -968,8 +1153,8 @@ _release_xid_data(_PyXIData_t *xidata, int rawfree) { PyObject *exc = PyErr_GetRaisedException(); int res = rawfree - ? _PyXIData_Release(xidata) - : _PyXIData_ReleaseAndRawFree(xidata); + ? _PyXIData_ReleaseAndRawFree(xidata) + : _PyXIData_Release(xidata); if (res < 0) { /* The owning interpreter is already destroyed. */ _PyXIData_Clear(NULL, xidata); @@ -1130,8 +1315,14 @@ _excinfo_normalize_type(struct _excinfo_type *info, *p_module = module; } +static int +excinfo_is_set(_PyXI_excinfo *info) +{ + return info->type.name != NULL || info->msg != NULL; +} + static void -_PyXI_excinfo_Clear(_PyXI_excinfo *info) +_PyXI_excinfo_clear(_PyXI_excinfo *info) { _excinfo_clear_type(&info->type); if (info->msg != NULL) { @@ -1181,7 +1372,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) assert(exc != NULL); if (PyErr_GivenExceptionMatches(exc, PyExc_MemoryError)) { - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return NULL; } const char *failure = NULL; @@ -1227,7 +1418,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) error: assert(failure != NULL); - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return failure; } @@ -1278,7 +1469,7 @@ _PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj) error: assert(failure != NULL); - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return failure; } @@ -1291,6 +1482,11 @@ _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) if (tbexc == NULL) { PyErr_Clear(); } + else { + PyErr_SetObject(exctype, tbexc); + Py_DECREF(tbexc); + return; + } } PyObject *formatted = _PyXI_excinfo_format(info); @@ -1436,13 +1632,17 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info) } -int -_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) +_PyXI_excinfo * +_PyXI_NewExcInfo(PyObject *exc) { assert(!PyErr_Occurred()); if (exc == NULL || exc == Py_None) { PyErr_SetString(PyExc_ValueError, "missing exc"); - return -1; + return NULL; + } + _PyXI_excinfo *info = PyMem_RawCalloc(1, sizeof(_PyXI_excinfo)); + if (info == NULL) { + return NULL; } const char *failure; if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) { @@ -1452,10 +1652,18 @@ _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) failure = _PyXI_excinfo_InitFromObject(info, exc); } if (failure != NULL) { - PyErr_SetString(PyExc_Exception, failure); - return -1; + PyMem_RawFree(info); + set_exc_with_cause(PyExc_Exception, failure); + return NULL; } - return 0; + return info; +} + +void +_PyXI_FreeExcInfo(_PyXI_excinfo *info) +{ + _PyXI_excinfo_clear(info); + PyMem_RawFree(info); } PyObject * @@ -1470,12 +1678,6 @@ _PyXI_ExcInfoAsObject(_PyXI_excinfo *info) return _PyXI_excinfo_AsObject(info); } -void -_PyXI_ClearExcInfo(_PyXI_excinfo *info) -{ - _PyXI_excinfo_Clear(info); -} - /***************************/ /* short-term data sharing */ @@ -1489,14 +1691,9 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) PyThreadState *tstate = _PyThreadState_GET(); assert(!PyErr_Occurred()); + assert(code != _PyXI_ERR_NO_ERROR); + assert(code != _PyXI_ERR_UNCAUGHT_EXCEPTION); switch (code) { - case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH; - case _PyXI_ERR_UNCAUGHT_EXCEPTION: - // There is nothing to apply. -#ifdef Py_DEBUG - Py_UNREACHABLE(); -#endif - return 0; case _PyXI_ERR_OTHER: // XXX msg? PyErr_SetNone(PyExc_InterpreterError); @@ -1516,12 +1713,20 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) PyErr_SetString(PyExc_InterpreterError, "failed to apply namespace to __main__"); break; + case _PyXI_ERR_PRESERVE_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to preserve objects across session"); + break; + case _PyXI_ERR_EXC_PROPAGATION_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to transfer exception between interpreters"); + break; case _PyXI_ERR_NOT_SHAREABLE: _set_xid_lookup_failure(tstate, NULL, NULL, NULL); break; default: #ifdef Py_DEBUG - Py_UNREACHABLE(); + Py_FatalError("unsupported error code"); #else PyErr_Format(PyExc_RuntimeError, "unsupported error code %d", code); #endif @@ -1530,70 +1735,276 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) return -1; } +/* basic failure info */ + +struct xi_failure { + // The kind of error to propagate. + _PyXI_errcode code; + // The propagated message. + const char *msg; + int msg_owned; +}; // _PyXI_failure + +#define XI_FAILURE_INIT (_PyXI_failure){ .code = _PyXI_ERR_NO_ERROR } + +static void +clear_xi_failure(_PyXI_failure *failure) +{ + if (failure->msg != NULL && failure->msg_owned) { + PyMem_RawFree((void*)failure->msg); + } + *failure = XI_FAILURE_INIT; +} + +static void +copy_xi_failure(_PyXI_failure *dest, _PyXI_failure *src) +{ + *dest = *src; + dest->msg_owned = 0; +} + +_PyXI_failure * +_PyXI_NewFailure(void) +{ + _PyXI_failure *failure = PyMem_RawMalloc(sizeof(_PyXI_failure)); + if (failure == NULL) { + PyErr_NoMemory(); + return NULL; + } + *failure = XI_FAILURE_INIT; + return failure; +} + +void +_PyXI_FreeFailure(_PyXI_failure *failure) +{ + clear_xi_failure(failure); + PyMem_RawFree(failure); +} + +_PyXI_errcode +_PyXI_GetFailureCode(_PyXI_failure *failure) +{ + if (failure == NULL) { + return _PyXI_ERR_NO_ERROR; + } + return failure->code; +} + +void +_PyXI_InitFailureUTF8(_PyXI_failure *failure, + _PyXI_errcode code, const char *msg) +{ + *failure = (_PyXI_failure){ + .code = code, + .msg = msg, + .msg_owned = 0, + }; +} + +int +_PyXI_InitFailure(_PyXI_failure *failure, _PyXI_errcode code, PyObject *obj) +{ + *failure = (_PyXI_failure){ + .code = code, + .msg = NULL, + .msg_owned = 0, + }; + if (obj == NULL) { + return 0; + } + + PyObject *msgobj = PyObject_Str(obj); + if (msgobj == NULL) { + return -1; + } + // This will leak if not paired with clear_xi_failure(). + // That happens automatically in _capture_current_exception(). + const char *msg = _copy_string_obj_raw(msgobj, NULL); + Py_DECREF(msgobj); + if (msg == NULL) { + return -1; + } + *failure = (_PyXI_failure){ + .code = code, + .msg = msg, + .msg_owned = 1, + }; + return 0; +} + /* shared exceptions */ -static const char * -_PyXI_InitError(_PyXI_error *error, PyObject *excobj, _PyXI_errcode code) -{ - if (error->interp == NULL) { - error->interp = PyInterpreterState_Get(); - } +typedef struct { + // The originating interpreter. + PyInterpreterState *interp; + // The error to propagate, if different from the uncaught exception. + _PyXI_failure *override; + _PyXI_failure _override; + // The exception information to propagate, if applicable. + // This is populated only for some error codes, + // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. + _PyXI_excinfo uncaught; +} _PyXI_error; - const char *failure = NULL; - if (code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // There is an unhandled exception we need to propagate. - failure = _PyXI_excinfo_InitFromException(&error->uncaught, excobj); - if (failure != NULL) { - // We failed to initialize error->uncaught. - // XXX Print the excobj/traceback? Emit a warning? - // XXX Print the current exception/traceback? - if (PyErr_ExceptionMatches(PyExc_MemoryError)) { - error->code = _PyXI_ERR_NO_MEMORY; - } - else { - error->code = _PyXI_ERR_OTHER; - } - PyErr_Clear(); +static void +xi_error_clear(_PyXI_error *err) +{ + err->interp = NULL; + if (err->override != NULL) { + clear_xi_failure(err->override); + } + _PyXI_excinfo_clear(&err->uncaught); +} + +static int +xi_error_is_set(_PyXI_error *error) +{ + if (error->override != NULL) { + assert(error->override->code != _PyXI_ERR_NO_ERROR); + assert(error->override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION + || excinfo_is_set(&error->uncaught)); + return 1; + } + return excinfo_is_set(&error->uncaught); +} + +static int +xi_error_has_override(_PyXI_error *err) +{ + if (err->override == NULL) { + return 0; + } + return (err->override->code != _PyXI_ERR_NO_ERROR + && err->override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); +} + +static PyObject * +xi_error_resolve_current_exc(PyThreadState *tstate, + _PyXI_failure *override) +{ + assert(override == NULL || override->code != _PyXI_ERR_NO_ERROR); + + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (exc == NULL) { + assert(override == NULL + || override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); + } + else if (override == NULL) { + // This is equivalent to _PyXI_ERR_UNCAUGHT_EXCEPTION. + } + else if (override->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + // We want to actually capture the current exception. + } + else if (exc != NULL) { + // It might make sense to do similarly for other codes. + if (override->code == _PyXI_ERR_ALREADY_RUNNING) { + // We don't need the exception info. + Py_CLEAR(exc); + } + // ...else we want to actually capture the current exception. + } + return exc; +} + +static void +xi_error_set_override(PyThreadState *tstate, _PyXI_error *err, + _PyXI_failure *override) +{ + assert(err->override == NULL); + assert(override != NULL); + assert(override->code != _PyXI_ERR_NO_ERROR); + // Use xi_error_set_exc() instead of setting _PyXI_ERR_UNCAUGHT_EXCEPTION.. + assert(override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); + err->override = &err->_override; + // The caller still owns override->msg. + copy_xi_failure(&err->_override, override); + err->interp = tstate->interp; +} + +static void +xi_error_set_override_code(PyThreadState *tstate, _PyXI_error *err, + _PyXI_errcode code) +{ + _PyXI_failure override = XI_FAILURE_INIT; + override.code = code; + xi_error_set_override(tstate, err, &override); +} + +static const char * +xi_error_set_exc(PyThreadState *tstate, _PyXI_error *err, PyObject *exc) +{ + assert(!_PyErr_Occurred(tstate)); + assert(!xi_error_is_set(err)); + assert(err->override == NULL); + assert(err->interp == NULL); + assert(exc != NULL); + const char *failure = + _PyXI_excinfo_InitFromException(&err->uncaught, exc); + if (failure != NULL) { + // We failed to initialize err->uncaught. + // XXX Print the excobj/traceback? Emit a warning? + // XXX Print the current exception/traceback? + if (_PyErr_ExceptionMatches(tstate, PyExc_MemoryError)) { + xi_error_set_override_code(tstate, err, _PyXI_ERR_NO_MEMORY); } else { - error->code = code; + xi_error_set_override_code(tstate, err, _PyXI_ERR_OTHER); } - assert(error->code != _PyXI_ERR_NO_ERROR); - } - else { - // There is an error code we need to propagate. - assert(excobj == NULL); - assert(code != _PyXI_ERR_NO_ERROR); - error->code = code; - _PyXI_excinfo_Clear(&error->uncaught); + PyErr_Clear(); } return failure; } -PyObject * -_PyXI_ApplyError(_PyXI_error *error) +static PyObject * +_PyXI_ApplyError(_PyXI_error *error, const char *failure) { PyThreadState *tstate = PyThreadState_Get(); - if (error->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // Raise an exception that proxies the propagated exception. + + if (failure != NULL) { + xi_error_clear(error); + return NULL; + } + + _PyXI_errcode code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + if (error->override != NULL) { + code = error->override->code; + assert(code != _PyXI_ERR_NO_ERROR); + } + + if (code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + // We will raise an exception that proxies the propagated exception. return _PyXI_excinfo_AsObject(&error->uncaught); } - else if (error->code == _PyXI_ERR_NOT_SHAREABLE) { + else if (code == _PyXI_ERR_NOT_SHAREABLE) { // Propagate the exception directly. assert(!_PyErr_Occurred(tstate)); - _set_xid_lookup_failure(tstate, NULL, error->uncaught.msg, NULL); + PyObject *cause = NULL; + if (excinfo_is_set(&error->uncaught)) { + // Maybe instead set a PyExc_ExceptionSnapshot as __cause__? + // That type doesn't exist currently + // but would look like interpreters.ExecutionFailed. + _PyXI_excinfo_Apply(&error->uncaught, PyExc_Exception); + cause = _PyErr_GetRaisedException(tstate); + } + const char *msg = error->override != NULL + ? error->override->msg + : error->uncaught.msg; + _set_xid_lookup_failure(tstate, NULL, msg, cause); + Py_XDECREF(cause); } else { // Raise an exception corresponding to the code. - assert(error->code != _PyXI_ERR_NO_ERROR); - (void)_PyXI_ApplyErrorCode(error->code, error->interp); - if (error->uncaught.type.name != NULL || error->uncaught.msg != NULL) { + (void)_PyXI_ApplyErrorCode(code, error->interp); + assert(error->override == NULL || error->override->msg == NULL); + if (excinfo_is_set(&error->uncaught)) { // __context__ will be set to a proxy of the propagated exception. - PyObject *exc = PyErr_GetRaisedException(); + // (or use PyExc_ExceptionSnapshot like _PyXI_ERR_NOT_SHAREABLE?) + PyObject *exc = _PyErr_GetRaisedException(tstate); _PyXI_excinfo_Apply(&error->uncaught, PyExc_InterpreterError); - PyObject *exc2 = PyErr_GetRaisedException(); + PyObject *exc2 = _PyErr_GetRaisedException(tstate); PyException_SetContext(exc, exc2); - PyErr_SetRaisedException(exc); + _PyErr_SetRaisedException(tstate, exc); } } assert(PyErr_Occurred()); @@ -1624,6 +2035,7 @@ typedef struct _sharednsitem { // in a different interpreter to release the XI data. } _PyXI_namespace_item; +#ifndef NDEBUG static int _sharednsitem_is_initialized(_PyXI_namespace_item *item) { @@ -1632,6 +2044,7 @@ _sharednsitem_is_initialized(_PyXI_namespace_item *item) } return 0; } +#endif static int _sharednsitem_init(_PyXI_namespace_item *item, PyObject *key) @@ -1659,7 +2072,8 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid) } static int -_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) +_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value, + xidata_fallback_t fallback) { assert(_sharednsitem_is_initialized(item)); assert(item->xidata == NULL); @@ -1668,7 +2082,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) return -1; } PyThreadState *tstate = PyThreadState_Get(); - if (_PyObject_GetXIData(tstate, value, item->xidata) != 0) { + if (_PyObject_GetXIData(tstate, value, fallback, item->xidata) < 0) { PyMem_RawFree(item->xidata); item->xidata = NULL; // The caller may want to propagate PyExc_NotShareableError @@ -1700,7 +2114,8 @@ _sharednsitem_clear(_PyXI_namespace_item *item) } static int -_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) +_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns, + xidata_fallback_t fallback) { assert(item->name != NULL); assert(item->xidata == NULL); @@ -1712,7 +2127,7 @@ _sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) // When applied, this item will be set to the default (or fail). return 0; } - if (_sharednsitem_set_value(item, value) < 0) { + if (_sharednsitem_set_value(item, value, fallback) < 0) { return -1; } return 0; @@ -1742,156 +2157,212 @@ _sharednsitem_apply(_PyXI_namespace_item *item, PyObject *ns, PyObject *dflt) return res; } -struct _sharedns { - Py_ssize_t len; - _PyXI_namespace_item *items; -}; + +typedef struct { + Py_ssize_t maxitems; + Py_ssize_t numnames; + Py_ssize_t numvalues; + _PyXI_namespace_item items[1]; +} _PyXI_namespace; + +#ifndef NDEBUG +static int +_sharedns_check_counts(_PyXI_namespace *ns) +{ + if (ns->maxitems <= 0) { + return 0; + } + if (ns->numnames < 0) { + return 0; + } + if (ns->numnames > ns->maxitems) { + return 0; + } + if (ns->numvalues < 0) { + return 0; + } + if (ns->numvalues > ns->numnames) { + return 0; + } + return 1; +} + +static int +_sharedns_check_consistency(_PyXI_namespace *ns) +{ + if (!_sharedns_check_counts(ns)) { + return 0; + } + + Py_ssize_t i = 0; + _PyXI_namespace_item *item; + if (ns->numvalues > 0) { + item = &ns->items[0]; + if (!_sharednsitem_is_initialized(item)) { + return 0; + } + int64_t interpid0 = -1; + if (!_sharednsitem_has_value(item, &interpid0)) { + return 0; + } + i += 1; + for (; i < ns->numvalues; i++) { + item = &ns->items[i]; + if (!_sharednsitem_is_initialized(item)) { + return 0; + } + int64_t interpid = -1; + if (!_sharednsitem_has_value(item, &interpid)) { + return 0; + } + if (interpid != interpid0) { + return 0; + } + } + } + for (; i < ns->numnames; i++) { + item = &ns->items[i]; + if (!_sharednsitem_is_initialized(item)) { + return 0; + } + if (_sharednsitem_has_value(item, NULL)) { + return 0; + } + } + for (; i < ns->maxitems; i++) { + item = &ns->items[i]; + if (_sharednsitem_is_initialized(item)) { + return 0; + } + if (_sharednsitem_has_value(item, NULL)) { + return 0; + } + } + return 1; +} +#endif static _PyXI_namespace * -_sharedns_new(void) +_sharedns_alloc(Py_ssize_t maxitems) { - _PyXI_namespace *ns = PyMem_RawCalloc(sizeof(_PyXI_namespace), 1); + if (maxitems < 0) { + if (!PyErr_Occurred()) { + PyErr_BadInternalCall(); + } + return NULL; + } + else if (maxitems == 0) { + PyErr_SetString(PyExc_ValueError, "empty namespaces not allowed"); + return NULL; + } + + // Check for overflow. + size_t fixedsize = sizeof(_PyXI_namespace) - sizeof(_PyXI_namespace_item); + if ((size_t)maxitems > + ((size_t)PY_SSIZE_T_MAX - fixedsize) / sizeof(_PyXI_namespace_item)) + { + PyErr_NoMemory(); + return NULL; + } + + // Allocate the value, including items. + size_t size = fixedsize + sizeof(_PyXI_namespace_item) * maxitems; + + _PyXI_namespace *ns = PyMem_RawCalloc(size, 1); if (ns == NULL) { PyErr_NoMemory(); return NULL; } - *ns = (_PyXI_namespace){ 0 }; + ns->maxitems = maxitems; + assert(_sharedns_check_consistency(ns)); return ns; } -static int -_sharedns_is_initialized(_PyXI_namespace *ns) -{ - if (ns->len == 0) { - assert(ns->items == NULL); - return 0; - } - - assert(ns->len > 0); - assert(ns->items != NULL); - assert(_sharednsitem_is_initialized(&ns->items[0])); - assert(ns->len == 1 - || _sharednsitem_is_initialized(&ns->items[ns->len - 1])); - return 1; -} - -#define HAS_COMPLETE_DATA 1 -#define HAS_PARTIAL_DATA 2 - -static int -_sharedns_has_xidata(_PyXI_namespace *ns, int64_t *p_interpid) -{ - // We expect _PyXI_namespace to always be initialized. - assert(_sharedns_is_initialized(ns)); - int res = 0; - _PyXI_namespace_item *item0 = &ns->items[0]; - if (!_sharednsitem_is_initialized(item0)) { - return 0; - } - int64_t interpid0 = -1; - if (!_sharednsitem_has_value(item0, &interpid0)) { - return 0; - } - if (ns->len > 1) { - // At this point we know it is has at least partial data. - _PyXI_namespace_item *itemN = &ns->items[ns->len-1]; - if (!_sharednsitem_is_initialized(itemN)) { - res = HAS_PARTIAL_DATA; - goto finally; - } - int64_t interpidN = -1; - if (!_sharednsitem_has_value(itemN, &interpidN)) { - res = HAS_PARTIAL_DATA; - goto finally; - } - assert(interpidN == interpid0); - } - res = HAS_COMPLETE_DATA; - *p_interpid = interpid0; - -finally: - return res; -} - -static void -_sharedns_clear(_PyXI_namespace *ns) -{ - if (!_sharedns_is_initialized(ns)) { - return; - } - - // If the cross-interpreter data were allocated as part of - // _PyXI_namespace_item (instead of dynamically), this is where - // we would need verify that we are clearing the items in the - // correct interpreter, to avoid a race with releasing the XI data - // via a pending call. See _sharedns_has_xidata(). - for (Py_ssize_t i=0; i < ns->len; i++) { - _sharednsitem_clear(&ns->items[i]); - } - PyMem_RawFree(ns->items); - ns->items = NULL; - ns->len = 0; -} - static void _sharedns_free(_PyXI_namespace *ns) { - _sharedns_clear(ns); + // If we weren't always dynamically allocating the cross-interpreter + // data in each item then we would need to use a pending call + // to call _sharedns_free(), to avoid the race between freeing + // the shared namespace and releasing the XI data. + assert(_sharedns_check_counts(ns)); + Py_ssize_t i = 0; + _PyXI_namespace_item *item; + if (ns->numvalues > 0) { + // One or more items may have interpreter-specific data. +#ifndef NDEBUG + int64_t interpid = PyInterpreterState_GetID(PyInterpreterState_Get()); + int64_t interpid_i; +#endif + for (; i < ns->numvalues; i++) { + item = &ns->items[i]; + assert(_sharednsitem_is_initialized(item)); + // While we do want to ensure consistency across items, + // technically they don't need to match the current + // interpreter. However, we keep the constraint for + // simplicity, by giving _PyXI_FreeNamespace() the exclusive + // responsibility of dealing with the owning interpreter. + assert(_sharednsitem_has_value(item, &interpid_i)); + assert(interpid_i == interpid); + _sharednsitem_clear(item); + } + } + for (; i < ns->numnames; i++) { + item = &ns->items[i]; + assert(_sharednsitem_is_initialized(item)); + assert(!_sharednsitem_has_value(item, NULL)); + _sharednsitem_clear(item); + } +#ifndef NDEBUG + for (; i < ns->maxitems; i++) { + item = &ns->items[i]; + assert(!_sharednsitem_is_initialized(item)); + assert(!_sharednsitem_has_value(item, NULL)); + } +#endif + PyMem_RawFree(ns); } -static int -_sharedns_init(_PyXI_namespace *ns, PyObject *names) +static _PyXI_namespace * +_create_sharedns(PyObject *names) { - assert(!_sharedns_is_initialized(ns)); assert(names != NULL); - Py_ssize_t len = PyDict_CheckExact(names) + Py_ssize_t numnames = PyDict_CheckExact(names) ? PyDict_Size(names) : PySequence_Size(names); - if (len < 0) { - return -1; - } - if (len == 0) { - PyErr_SetString(PyExc_ValueError, "empty namespaces not allowed"); - return -1; - } - assert(len > 0); - // Allocate the items. - _PyXI_namespace_item *items = - PyMem_RawCalloc(sizeof(struct _sharednsitem), len); - if (items == NULL) { - PyErr_NoMemory(); - return -1; + _PyXI_namespace *ns = _sharedns_alloc(numnames); + if (ns == NULL) { + return NULL; } + _PyXI_namespace_item *items = ns->items; // Fill in the names. - Py_ssize_t i = -1; if (PyDict_CheckExact(names)) { + Py_ssize_t i = 0; Py_ssize_t pos = 0; - for (i=0; i < len; i++) { - PyObject *key; - if (!PyDict_Next(names, &pos, &key, NULL)) { - // This should not be possible. - assert(0); - goto error; - } - if (_sharednsitem_init(&items[i], key) < 0) { + PyObject *name; + while(PyDict_Next(names, &pos, &name, NULL)) { + if (_sharednsitem_init(&items[i], name) < 0) { goto error; } + ns->numnames += 1; + i += 1; } } else if (PySequence_Check(names)) { - for (i=0; i < len; i++) { - PyObject *key = PySequence_GetItem(names, i); - if (key == NULL) { + for (Py_ssize_t i = 0; i < numnames; i++) { + PyObject *name = PySequence_GetItem(names, i); + if (name == NULL) { goto error; } - int res = _sharednsitem_init(&items[i], key); - Py_DECREF(key); + int res = _sharednsitem_init(&items[i], name); + Py_DECREF(name); if (res < 0) { goto error; } + ns->numnames += 1; } } else { @@ -1899,140 +2370,84 @@ _sharedns_init(_PyXI_namespace *ns, PyObject *names) "non-sequence namespace not supported"); goto error; } - - ns->items = items; - ns->len = len; - assert(_sharedns_is_initialized(ns)); - return 0; - -error: - for (Py_ssize_t j=0; j < i; j++) { - _sharednsitem_clear(&items[j]); - } - PyMem_RawFree(items); - assert(!_sharedns_is_initialized(ns)); - return -1; -} - -void -_PyXI_FreeNamespace(_PyXI_namespace *ns) -{ - if (!_sharedns_is_initialized(ns)) { - return; - } - - int64_t interpid = -1; - if (!_sharedns_has_xidata(ns, &interpid)) { - _sharedns_free(ns); - return; - } - - if (interpid == PyInterpreterState_GetID(PyInterpreterState_Get())) { - _sharedns_free(ns); - } - else { - // If we weren't always dynamically allocating the cross-interpreter - // data in each item then we would need to using a pending call - // to call _sharedns_free(), to avoid the race between freeing - // the shared namespace and releasing the XI data. - _sharedns_free(ns); - } -} - -_PyXI_namespace * -_PyXI_NamespaceFromNames(PyObject *names) -{ - if (names == NULL || names == Py_None) { - return NULL; - } - - _PyXI_namespace *ns = _sharedns_new(); - if (ns == NULL) { - return NULL; - } - - if (_sharedns_init(ns, names) < 0) { - PyMem_RawFree(ns); - if (PySequence_Size(names) == 0) { - PyErr_Clear(); - } - return NULL; - } - - return ns; -} - -#ifndef NDEBUG -static int _session_is_active(_PyXI_session *); -#endif -static void _propagate_not_shareable_error(_PyXI_session *); - -int -_PyXI_FillNamespaceFromDict(_PyXI_namespace *ns, PyObject *nsobj, - _PyXI_session *session) -{ - // session must be entered already, if provided. - assert(session == NULL || _session_is_active(session)); - assert(_sharedns_is_initialized(ns)); - for (Py_ssize_t i=0; i < ns->len; i++) { - _PyXI_namespace_item *item = &ns->items[i]; - if (_sharednsitem_copy_from_ns(item, nsobj) < 0) { - _propagate_not_shareable_error(session); - // Clear out the ones we set so far. - for (Py_ssize_t j=0; j < i; j++) { - _sharednsitem_clear_value(&ns->items[j]); - } - return -1; - } - } - return 0; -} - -// All items are expected to be shareable. -static _PyXI_namespace * -_PyXI_NamespaceFromDict(PyObject *nsobj, _PyXI_session *session) -{ - // session must be entered already, if provided. - assert(session == NULL || _session_is_active(session)); - if (nsobj == NULL || nsobj == Py_None) { - return NULL; - } - if (!PyDict_CheckExact(nsobj)) { - PyErr_SetString(PyExc_TypeError, "expected a dict"); - return NULL; - } - - _PyXI_namespace *ns = _sharedns_new(); - if (ns == NULL) { - return NULL; - } - - if (_sharedns_init(ns, nsobj) < 0) { - if (PyDict_Size(nsobj) == 0) { - PyMem_RawFree(ns); - PyErr_Clear(); - return NULL; - } - goto error; - } - - if (_PyXI_FillNamespaceFromDict(ns, nsobj, session) < 0) { - goto error; - } - + assert(ns->numnames == ns->maxitems); return ns; error: - assert(PyErr_Occurred() - || (session != NULL && session->error_override != NULL)); _sharedns_free(ns); return NULL; } -int -_PyXI_ApplyNamespace(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) +static void _propagate_not_shareable_error(PyThreadState *, + _PyXI_failure *); + +static int +_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, + xidata_fallback_t fallback, _PyXI_failure *p_err) { - for (Py_ssize_t i=0; i < ns->len; i++) { + // All items are expected to be shareable. + assert(_sharedns_check_counts(ns)); + assert(ns->numnames == ns->maxitems); + assert(ns->numvalues == 0); + PyThreadState *tstate = PyThreadState_Get(); + for (Py_ssize_t i=0; i < ns->maxitems; i++) { + if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj, fallback) < 0) { + if (p_err != NULL) { + _propagate_not_shareable_error(tstate, p_err); + } + // Clear out the ones we set so far. + for (Py_ssize_t j=0; j < i; j++) { + _sharednsitem_clear_value(&ns->items[j]); + ns->numvalues -= 1; + } + return -1; + } + ns->numvalues += 1; + } + return 0; +} + +static int +_sharedns_free_pending(void *data) +{ + _sharedns_free((_PyXI_namespace *)data); + return 0; +} + +static void +_destroy_sharedns(_PyXI_namespace *ns) +{ + assert(_sharedns_check_counts(ns)); + assert(ns->numnames == ns->maxitems); + if (ns->numvalues == 0) { + _sharedns_free(ns); + return; + } + + int64_t interpid0; + if (!_sharednsitem_has_value(&ns->items[0], &interpid0)) { + // This shouldn't have been possible. + // We can deal with it in _sharedns_free(). + _sharedns_free(ns); + return; + } + PyInterpreterState *interp = _PyInterpreterState_LookUpID(interpid0); + if (interp == PyInterpreterState_Get()) { + _sharedns_free(ns); + return; + } + + // One or more items may have interpreter-specific data. + // Currently the xidata for each value is dynamically allocated, + // so technically we don't need to worry about that. + // However, explicitly adding a pending call here is simpler. + (void)_Py_CallInInterpreter(interp, _sharedns_free_pending, ns); +} + +static int +_apply_sharedns(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) +{ + for (Py_ssize_t i=0; i < ns->maxitems; i++) { if (_sharednsitem_apply(&ns->items[i], nsobj, dflt) != 0) { return -1; } @@ -2041,9 +2456,69 @@ _PyXI_ApplyNamespace(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) } -/**********************/ -/* high-level helpers */ -/**********************/ +/*********************************/ +/* switched-interpreter sessions */ +/*********************************/ + +struct xi_session { +#define SESSION_UNUSED 0 +#define SESSION_ACTIVE 1 + int status; + int switched; + + // Once a session has been entered, this is the tstate that was + // current before the session. If it is different from cur_tstate + // then we must have switched interpreters. Either way, this will + // be the current tstate once we exit the session. + PyThreadState *prev_tstate; + // Once a session has been entered, this is the current tstate. + // It must be current when the session exits. + PyThreadState *init_tstate; + // This is true if init_tstate needs cleanup during exit. + int own_init_tstate; + + // This is true if, while entering the session, init_thread took + // "ownership" of the interpreter's __main__ module. This means + // it is the only thread that is allowed to run code there. + // (Caveat: for now, users may still run exec() against the + // __main__ module's dict, though that isn't advisable.) + int running; + // This is a cached reference to the __dict__ of the entered + // interpreter's __main__ module. It is looked up when at the + // beginning of the session as a convenience. + PyObject *main_ns; + + // This is a dict of objects that will be available (via sharing) + // once the session exits. Do not access this directly; use + // _PyXI_Preserve() and _PyXI_GetPreserved() instead; + PyObject *_preserved; +}; + +_PyXI_session * +_PyXI_NewSession(void) +{ + _PyXI_session *session = PyMem_RawCalloc(1, sizeof(_PyXI_session)); + if (session == NULL) { + PyErr_NoMemory(); + return NULL; + } + return session; +} + +void +_PyXI_FreeSession(_PyXI_session *session) +{ + assert(session->status == SESSION_UNUSED); + PyMem_RawFree(session); +} + + +static inline int +_session_is_active(_PyXI_session *session) +{ + return session->status == SESSION_ACTIVE; +} + /* enter/exit a cross-interpreter session */ @@ -2051,29 +2526,33 @@ static void _enter_session(_PyXI_session *session, PyInterpreterState *interp) { // Set here and cleared in _exit_session(). + assert(session->status == SESSION_UNUSED); assert(!session->own_init_tstate); assert(session->init_tstate == NULL); assert(session->prev_tstate == NULL); // Set elsewhere and cleared in _exit_session(). assert(!session->running); assert(session->main_ns == NULL); - // Set elsewhere and cleared in _capture_current_exception(). - assert(session->error_override == NULL); - // Set elsewhere and cleared in _PyXI_ApplyCapturedException(). - assert(session->error == NULL); // Switch to interpreter. PyThreadState *tstate = PyThreadState_Get(); PyThreadState *prev = tstate; - if (interp != tstate->interp) { + int same_interp = (interp == tstate->interp); + if (!same_interp) { tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); // XXX Possible GILState issues? - session->prev_tstate = PyThreadState_Swap(tstate); - assert(session->prev_tstate == prev); - session->own_init_tstate = 1; + PyThreadState *swapped = PyThreadState_Swap(tstate); + assert(swapped == prev); + (void)swapped; } - session->init_tstate = tstate; - session->prev_tstate = prev; + + *session = (_PyXI_session){ + .status = SESSION_ACTIVE, + .switched = !same_interp, + .init_tstate = tstate, + .prev_tstate = prev, + .own_init_tstate = !same_interp, + }; } static void @@ -2082,16 +2561,16 @@ _exit_session(_PyXI_session *session) PyThreadState *tstate = session->init_tstate; assert(tstate != NULL); assert(PyThreadState_Get() == tstate); + assert(!_PyErr_Occurred(tstate)); // Release any of the entered interpreters resources. - if (session->main_ns != NULL) { - Py_CLEAR(session->main_ns); - } + Py_CLEAR(session->main_ns); + Py_CLEAR(session->_preserved); // Ensure this thread no longer owns __main__. if (session->running) { _PyInterpreterState_SetNotRunningMain(tstate->interp); - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); session->running = 0; } @@ -2107,25 +2586,15 @@ _exit_session(_PyXI_session *session) else { assert(!session->own_init_tstate); } - session->prev_tstate = NULL; - session->init_tstate = NULL; -} -#ifndef NDEBUG -static int -_session_is_active(_PyXI_session *session) -{ - return (session->init_tstate != NULL); + *session = (_PyXI_session){0}; } -#endif static void -_propagate_not_shareable_error(_PyXI_session *session) +_propagate_not_shareable_error(PyThreadState *tstate, + _PyXI_failure *override) { - if (session == NULL) { - return; - } - PyThreadState *tstate = PyThreadState_Get(); + assert(override != NULL); PyObject *exctype = get_notshareableerror_type(tstate); if (exctype == NULL) { PyErr_FormatUnraisable( @@ -2134,166 +2603,463 @@ _propagate_not_shareable_error(_PyXI_session *session) } if (PyErr_ExceptionMatches(exctype)) { // We want to propagate the exception directly. - session->_error_override = _PyXI_ERR_NOT_SHAREABLE; - session->error_override = &session->_error_override; + *override = (_PyXI_failure){ + .code = _PyXI_ERR_NOT_SHAREABLE, + }; } } -static void -_capture_current_exception(_PyXI_session *session) -{ - assert(session->error == NULL); - if (!PyErr_Occurred()) { - assert(session->error_override == NULL); - return; - } - // Handle the exception override. - _PyXI_errcode *override = session->error_override; - session->error_override = NULL; - _PyXI_errcode errcode = override != NULL - ? *override - : _PyXI_ERR_UNCAUGHT_EXCEPTION; - - // Pop the exception object. - PyObject *excval = NULL; - if (errcode == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // We want to actually capture the current exception. - excval = PyErr_GetRaisedException(); - } - else if (errcode == _PyXI_ERR_ALREADY_RUNNING) { - // We don't need the exception info. - PyErr_Clear(); - } - else { - // We could do a variety of things here, depending on errcode. - // However, for now we simply capture the exception and save - // the errcode. - excval = PyErr_GetRaisedException(); - } - - // Capture the exception. - _PyXI_error *err = &session->_error; - *err = (_PyXI_error){ - .interp = session->init_tstate->interp, - }; - const char *failure; - if (excval == NULL) { - failure = _PyXI_InitError(err, NULL, errcode); - } - else { - failure = _PyXI_InitError(err, excval, _PyXI_ERR_UNCAUGHT_EXCEPTION); - Py_DECREF(excval); - if (failure == NULL && override != NULL) { - err->code = errcode; - } - } - - // Handle capture failure. - if (failure != NULL) { - // XXX Make this error message more generic. - fprintf(stderr, - "RunFailedError: script raised an uncaught exception (%s)", - failure); - err = NULL; - } - - // Finished! - assert(!PyErr_Occurred()); - session->error = err; -} - -PyObject * -_PyXI_ApplyCapturedException(_PyXI_session *session) -{ - assert(!PyErr_Occurred()); - assert(session->error != NULL); - PyObject *res = _PyXI_ApplyError(session->error); - assert((res == NULL) != (PyErr_Occurred() == NULL)); - session->error = NULL; - return res; -} - -int -_PyXI_HasCapturedException(_PyXI_session *session) -{ - return session->error != NULL; -} +static int _ensure_main_ns(_PyXI_session *, _PyXI_failure *); +static const char * capture_session_error(_PyXI_session *, _PyXI_error *, + _PyXI_failure *); int _PyXI_Enter(_PyXI_session *session, - PyInterpreterState *interp, PyObject *nsupdates) + PyInterpreterState *interp, PyObject *nsupdates, + _PyXI_session_result *result) { +#ifndef NDEBUG + PyThreadState *tstate = _PyThreadState_GET(); // Only used for asserts +#endif + // Convert the attrs for cross-interpreter use. _PyXI_namespace *sharedns = NULL; if (nsupdates != NULL) { - sharedns = _PyXI_NamespaceFromDict(nsupdates, NULL); - if (sharedns == NULL && PyErr_Occurred()) { - assert(session->error == NULL); + assert(PyDict_Check(nsupdates)); + Py_ssize_t len = PyDict_Size(nsupdates); + if (len < 0) { + if (result != NULL) { + result->errcode = _PyXI_ERR_APPLY_NS_FAILURE; + } return -1; } + if (len > 0) { + sharedns = _create_sharedns(nsupdates); + if (sharedns == NULL) { + if (result != NULL) { + result->errcode = _PyXI_ERR_APPLY_NS_FAILURE; + } + return -1; + } + // For now we limit it to shareable objects. + xidata_fallback_t fallback = _PyXIDATA_XIDATA_ONLY; + _PyXI_failure _err = XI_FAILURE_INIT; + if (_fill_sharedns(sharedns, nsupdates, fallback, &_err) < 0) { + assert(_PyErr_Occurred(tstate)); + if (_err.code == _PyXI_ERR_NO_ERROR) { + _err.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } + _destroy_sharedns(sharedns); + if (result != NULL) { + assert(_err.msg == NULL); + result->errcode = _err.code; + } + return -1; + } + } } // Switch to the requested interpreter (if necessary). _enter_session(session, interp); - PyThreadState *session_tstate = session->init_tstate; - _PyXI_errcode errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + _PyXI_failure override = XI_FAILURE_INIT; + override.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; +#ifndef NDEBUG + tstate = _PyThreadState_GET(); +#endif // Ensure this thread owns __main__. if (_PyInterpreterState_SetRunningMain(interp) < 0) { // In the case where we didn't switch interpreters, it would // be more efficient to leave the exception in place and return // immediately. However, life is simpler if we don't. - errcode = _PyXI_ERR_ALREADY_RUNNING; + override.code = _PyXI_ERR_ALREADY_RUNNING; goto error; } session->running = 1; + // Apply the cross-interpreter data. + if (sharedns != NULL) { + if (_ensure_main_ns(session, &override) < 0) { + goto error; + } + if (_apply_sharedns(sharedns, session->main_ns, NULL) < 0) { + override.code = _PyXI_ERR_APPLY_NS_FAILURE; + goto error; + } + _destroy_sharedns(sharedns); + } + + override.code = _PyXI_ERR_NO_ERROR; + assert(!_PyErr_Occurred(tstate)); + return 0; + +error: + // We want to propagate all exceptions here directly (best effort). + assert(override.code != _PyXI_ERR_NO_ERROR); + _PyXI_error err = {0}; + const char *failure = capture_session_error(session, &err, &override); + + // Exit the session. + _exit_session(session); +#ifndef NDEBUG + tstate = _PyThreadState_GET(); +#endif + + if (sharedns != NULL) { + _destroy_sharedns(sharedns); + } + + // Apply the error from the other interpreter. + PyObject *excinfo = _PyXI_ApplyError(&err, failure); + xi_error_clear(&err); + if (excinfo != NULL) { + if (result != NULL) { + result->excinfo = excinfo; + } + else { +#ifdef Py_DEBUG + fprintf(stderr, "_PyXI_Enter(): uncaught exception discarded"); +#endif + Py_DECREF(excinfo); + } + } + assert(_PyErr_Occurred(tstate)); + + return -1; +} + +static int _pop_preserved(_PyXI_session *, _PyXI_namespace **, PyObject **, + _PyXI_failure *); +static int _finish_preserved(_PyXI_namespace *, PyObject **); + +int +_PyXI_Exit(_PyXI_session *session, _PyXI_failure *override, + _PyXI_session_result *result) +{ + PyThreadState *tstate = _PyThreadState_GET(); + int res = 0; + + // Capture the raised exception, if any. + _PyXI_error err = {0}; + const char *failure = NULL; + if (override != NULL && override->code == _PyXI_ERR_NO_ERROR) { + assert(override->msg == NULL); + override = NULL; + } + if (_PyErr_Occurred(tstate)) { + failure = capture_session_error(session, &err, override); + } + else { + assert(override == NULL); + } + + // Capture the preserved namespace. + _PyXI_namespace *preserved = NULL; + PyObject *preservedobj = NULL; + if (result != NULL) { + assert(!_PyErr_Occurred(tstate)); + _PyXI_failure _override = XI_FAILURE_INIT; + if (_pop_preserved( + session, &preserved, &preservedobj, &_override) < 0) + { + assert(preserved == NULL); + assert(preservedobj == NULL); + if (xi_error_is_set(&err)) { + // XXX Chain the exception (i.e. set __context__)? + PyErr_FormatUnraisable( + "Exception ignored while capturing preserved objects"); + clear_xi_failure(&_override); + } + else { + if (_override.code == _PyXI_ERR_NO_ERROR) { + _override.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } + failure = capture_session_error(session, &err, &_override); + } + } + } + + // Exit the session. + assert(!_PyErr_Occurred(tstate)); + _exit_session(session); + tstate = _PyThreadState_GET(); + + // Restore the preserved namespace. + assert(preserved == NULL || preservedobj == NULL); + if (_finish_preserved(preserved, &preservedobj) < 0) { + assert(preservedobj == NULL); + if (xi_error_is_set(&err)) { + // XXX Chain the exception (i.e. set __context__)? + PyErr_FormatUnraisable( + "Exception ignored while capturing preserved objects"); + } + else { + xi_error_set_override_code( + tstate, &err, _PyXI_ERR_PRESERVE_FAILURE); + _propagate_not_shareable_error(tstate, err.override); + } + } + if (result != NULL) { + result->preserved = preservedobj; + result->errcode = err.override != NULL + ? err.override->code + : _PyXI_ERR_NO_ERROR; + } + + // Apply the error from the other interpreter, if any. + if (xi_error_is_set(&err)) { + res = -1; + assert(!_PyErr_Occurred(tstate)); + PyObject *excinfo = _PyXI_ApplyError(&err, failure); + if (excinfo == NULL) { + assert(_PyErr_Occurred(tstate)); + if (result != NULL && !xi_error_has_override(&err)) { + _PyXI_ClearResult(result); + *result = (_PyXI_session_result){ + .errcode = _PyXI_ERR_EXC_PROPAGATION_FAILURE, + }; + } + } + else if (result != NULL) { + result->excinfo = excinfo; + } + else { +#ifdef Py_DEBUG + fprintf(stderr, "_PyXI_Exit(): uncaught exception discarded"); +#endif + Py_DECREF(excinfo); + } + xi_error_clear(&err); + } + return res; +} + + +/* in an active cross-interpreter session */ + +static const char * +capture_session_error(_PyXI_session *session, _PyXI_error *err, + _PyXI_failure *override) +{ + assert(_session_is_active(session)); + assert(!xi_error_is_set(err)); + PyThreadState *tstate = session->init_tstate; + + // Normalize the exception override. + if (override != NULL) { + if (override->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + assert(override->msg == NULL); + override = NULL; + } + else { + assert(override->code != _PyXI_ERR_NO_ERROR); + } + } + + // Handle the exception, if any. + const char *failure = NULL; + PyObject *exc = xi_error_resolve_current_exc(tstate, override); + if (exc != NULL) { + // There is an unhandled exception we need to preserve. + failure = xi_error_set_exc(tstate, err, exc); + Py_DECREF(exc); + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable(failure); + } + } + + // Handle the override. + if (override != NULL && failure == NULL) { + xi_error_set_override(tstate, err, override); + } + + // Finished! + assert(!_PyErr_Occurred(tstate)); + return failure; +} + +static int +_ensure_main_ns(_PyXI_session *session, _PyXI_failure *failure) +{ + assert(_session_is_active(session)); + PyThreadState *tstate = session->init_tstate; + if (session->main_ns != NULL) { + return 0; + } // Cache __main__.__dict__. - PyObject *main_mod = _Py_GetMainModule(session_tstate); + PyObject *main_mod = _Py_GetMainModule(tstate); if (_Py_CheckMainModule(main_mod) < 0) { - errcode = _PyXI_ERR_MAIN_NS_FAILURE; - goto error; + Py_XDECREF(main_mod); + if (failure != NULL) { + *failure = (_PyXI_failure){ + .code = _PyXI_ERR_MAIN_NS_FAILURE, + }; + } + return -1; } PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) { - errcode = _PyXI_ERR_MAIN_NS_FAILURE; - goto error; + if (failure != NULL) { + *failure = (_PyXI_failure){ + .code = _PyXI_ERR_MAIN_NS_FAILURE, + }; + } + return -1; } session->main_ns = Py_NewRef(ns); + return 0; +} - // Apply the cross-interpreter data. - if (sharedns != NULL) { - if (_PyXI_ApplyNamespace(sharedns, ns, NULL) < 0) { - errcode = _PyXI_ERR_APPLY_NS_FAILURE; +PyObject * +_PyXI_GetMainNamespace(_PyXI_session *session, _PyXI_failure *failure) +{ + if (!_session_is_active(session)) { + PyErr_SetString(PyExc_RuntimeError, "session not active"); + return NULL; + } + if (_ensure_main_ns(session, failure) < 0) { + return NULL; + } + return session->main_ns; +} + + +static int +_pop_preserved(_PyXI_session *session, + _PyXI_namespace **p_xidata, PyObject **p_obj, + _PyXI_failure *p_failure) +{ + _PyXI_failure failure = XI_FAILURE_INIT; + _PyXI_namespace *xidata = NULL; + assert(_PyThreadState_GET() == session->init_tstate); // active session + + if (session->_preserved == NULL) { + *p_xidata = NULL; + *p_obj = NULL; + return 0; + } + if (session->init_tstate == session->prev_tstate) { + // We did not switch interpreters. + *p_xidata = NULL; + *p_obj = session->_preserved; + session->_preserved = NULL; + return 0; + } + *p_obj = NULL; + + // We did switch interpreters. + Py_ssize_t len = PyDict_Size(session->_preserved); + if (len < 0) { + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; + } + else if (len == 0) { + *p_xidata = NULL; + } + else { + _PyXI_namespace *xidata = _create_sharedns(session->_preserved); + if (xidata == NULL) { + failure.code = _PyXI_ERR_PRESERVE_FAILURE; goto error; } - _PyXI_FreeNamespace(sharedns); + if (_fill_sharedns(xidata, session->_preserved, + _PyXIDATA_FULL_FALLBACK, &failure) < 0) + { + if (failure.code != _PyXI_ERR_NOT_SHAREABLE) { + assert(failure.msg != NULL); + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + } + goto error; + } + *p_xidata = xidata; } - - errcode = _PyXI_ERR_NO_ERROR; - assert(!PyErr_Occurred()); + Py_CLEAR(session->_preserved); return 0; error: - assert(PyErr_Occurred()); - // We want to propagate all exceptions here directly (best effort). - assert(errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION); - session->error_override = &errcode; - _capture_current_exception(session); - _exit_session(session); - if (sharedns != NULL) { - _PyXI_FreeNamespace(sharedns); + if (p_failure != NULL) { + *p_failure = failure; + } + if (xidata != NULL) { + _destroy_sharedns(xidata); } return -1; } -void -_PyXI_Exit(_PyXI_session *session) +static int +_finish_preserved(_PyXI_namespace *xidata, PyObject **p_preserved) { - _capture_current_exception(session); - _exit_session(session); + if (xidata == NULL) { + return 0; + } + int res = -1; + if (p_preserved != NULL) { + PyObject *ns = PyDict_New(); + if (ns == NULL) { + goto finally; + } + if (_apply_sharedns(xidata, ns, NULL) < 0) { + Py_CLEAR(ns); + goto finally; + } + *p_preserved = ns; + } + res = 0; + +finally: + _destroy_sharedns(xidata); + return res; +} + +int +_PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value, + _PyXI_failure *p_failure) +{ + _PyXI_failure failure = XI_FAILURE_INIT; + if (!_session_is_active(session)) { + PyErr_SetString(PyExc_RuntimeError, "session not active"); + return -1; + } + if (session->_preserved == NULL) { + session->_preserved = PyDict_New(); + if (session->_preserved == NULL) { + set_exc_with_cause(PyExc_RuntimeError, + "failed to initialize preserved objects"); + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; + } + } + if (PyDict_SetItemString(session->_preserved, name, value) < 0) { + set_exc_with_cause(PyExc_RuntimeError, "failed to preserve object"); + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; + } + return 0; + +error: + if (p_failure != NULL) { + *p_failure = failure; + } + return -1; +} + +PyObject * +_PyXI_GetPreserved(_PyXI_session_result *result, const char *name) +{ + PyObject *value = NULL; + if (result->preserved != NULL) { + (void)PyDict_GetItemStringRef(result->preserved, name, &value); + } + return value; +} + +void +_PyXI_ClearResult(_PyXI_session_result *result) +{ + Py_CLEAR(result->preserved); + Py_CLEAR(result->excinfo); } diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index 231537c66d7..c3c76ae8d9a 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -12,7 +12,8 @@ typedef _PyXIData_regitem_t dlregitem_t; // forward static void _xidregistry_init(dlregistry_t *); static void _xidregistry_fini(dlregistry_t *); -static xidatafunc _lookup_getdata_from_registry(dlcontext_t *, PyObject *); +static _PyXIData_getdata_t _lookup_getdata_from_registry( + dlcontext_t *, PyObject *); /* used in crossinterp.c */ @@ -49,7 +50,7 @@ get_lookup_context(PyThreadState *tstate, dlcontext_t *res) return 0; } -static xidatafunc +static _PyXIData_getdata_t lookup_getdata(dlcontext_t *ctx, PyObject *obj) { /* Cross-interpreter objects are looked up by exact match on the class. @@ -87,25 +88,52 @@ _PyXIData_FormatNotShareableError(PyThreadState *tstate, va_end(vargs); } +int +_PyXI_UnwrapNotShareableError(PyThreadState * tstate, _PyXI_failure *failure) +{ + PyObject *exctype = get_notshareableerror_type(tstate); + assert(exctype != NULL); + if (!_PyErr_ExceptionMatches(tstate, exctype)) { + return -1; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (failure != NULL) { + _PyXI_errcode code = _PyXI_ERR_NOT_SHAREABLE; + if (_PyXI_InitFailure(failure, code, exc) < 0) { + return -1; + } + } + PyObject *cause = PyException_GetCause(exc); + if (cause != NULL) { + Py_DECREF(exc); + exc = cause; + } + else { + assert(PyException_GetContext(exc) == NULL); + } + _PyErr_SetRaisedException(tstate, exc); + return 0; +} -xidatafunc + +_PyXIData_getdata_t _PyXIData_Lookup(PyThreadState *tstate, PyObject *obj) { dlcontext_t ctx; if (get_lookup_context(tstate, &ctx) < 0) { - return NULL; + return (_PyXIData_getdata_t){0}; } return lookup_getdata(&ctx, obj); } /***********************************************/ -/* a registry of {type -> xidatafunc} */ +/* a registry of {type -> _PyXIData_getdata_t} */ /***********************************************/ -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - xidatafunc. It would be simpler and more efficient. */ +/* For now we use a global registry of shareable classes. + An alternative would be to add a tp_* slot for a class's + _PyXIData_getdata_t. It would be simpler and more efficient. */ /* registry lifecycle */ @@ -200,7 +228,7 @@ _xidregistry_find_type(dlregistry_t *xidregistry, PyTypeObject *cls) return NULL; } -static xidatafunc +static _PyXIData_getdata_t _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) { PyTypeObject *cls = Py_TYPE(obj); @@ -209,10 +237,12 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) _xidregistry_lock(xidregistry); dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); - xidatafunc func = matched != NULL ? matched->getdata : NULL; + _PyXIData_getdata_t getdata = matched != NULL + ? matched->getdata + : (_PyXIData_getdata_t){0}; _xidregistry_unlock(xidregistry); - return func; + return getdata; } @@ -220,12 +250,13 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) static int _xidregistry_add_type(dlregistry_t *xidregistry, - PyTypeObject *cls, xidatafunc getdata) + PyTypeObject *cls, _PyXIData_getdata_t getdata) { dlregitem_t *newhead = PyMem_RawMalloc(sizeof(dlregitem_t)); if (newhead == NULL) { return -1; } + assert((getdata.basic == NULL) != (getdata.fallback == NULL)); *newhead = (dlregitem_t){ // We do not keep a reference, to avoid keeping the class alive. .cls = cls, @@ -283,13 +314,13 @@ _xidregistry_clear(dlregistry_t *xidregistry) int _PyXIData_RegisterClass(PyThreadState *tstate, - PyTypeObject *cls, xidatafunc getdata) + PyTypeObject *cls, _PyXIData_getdata_t getdata) { if (!PyType_Check(cls)) { PyErr_Format(PyExc_ValueError, "only classes may be registered"); return -1; } - if (getdata == NULL) { + if (getdata.basic == NULL && getdata.fallback == NULL) { PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); return -1; } @@ -304,7 +335,8 @@ _PyXIData_RegisterClass(PyThreadState *tstate, dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); if (matched != NULL) { - assert(matched->getdata == getdata); + assert(matched->getdata.basic == getdata.basic); + assert(matched->getdata.fallback == getdata.fallback); matched->refcount += 1; goto finally; } @@ -608,7 +640,8 @@ _tuple_shared_free(void* data) } static int -_tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) +_tuple_shared(PyThreadState *tstate, PyObject *obj, xidata_fallback_t fallback, + _PyXIData_t *xidata) { Py_ssize_t len = PyTuple_GET_SIZE(obj); if (len < 0) { @@ -636,7 +669,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) int res = -1; if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { - res = _PyObject_GetXIData(tstate, item, xidata_i); + res = _PyObject_GetXIData(tstate, item, fallback, xidata_i); _Py_LeaveRecursiveCallTstate(tstate); } if (res < 0) { @@ -677,44 +710,126 @@ _PyCode_GetXIData(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) return 0; } +// function + +PyObject * +_PyFunction_FromXIData(_PyXIData_t *xidata) +{ + // For now "stateless" functions are the only ones we must accommodate. + + PyObject *code = _PyMarshal_ReadObjectFromXIData(xidata); + if (code == NULL) { + return NULL; + } + // Create a new function. + // For stateless functions (no globals) we use __main__ as __globals__, + // just like we do for builtins like exec(). + assert(PyCode_Check(code)); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *globals = _PyEval_GetGlobalsFromRunningMain(tstate); // borrowed + if (globals == NULL) { + if (_PyErr_Occurred(tstate)) { + Py_DECREF(code); + return NULL; + } + globals = PyDict_New(); + if (globals == NULL) { + Py_DECREF(code); + return NULL; + } + } + else { + Py_INCREF(globals); + } + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { + Py_DECREF(code); + Py_DECREF(globals); + return NULL; + } + PyObject *func = PyFunction_New(code, globals); + Py_DECREF(code); + Py_DECREF(globals); + return func; +} + +int +_PyFunction_GetXIData(PyThreadState *tstate, PyObject *func, + _PyXIData_t *xidata) +{ + if (!PyFunction_Check(func)) { + const char *msg = "expected a function, got %R"; + format_notshareableerror(tstate, NULL, 0, msg, func); + return -1; + } + if (_PyFunction_VerifyStateless(tstate, func) < 0) { + PyObject *cause = _PyErr_GetRaisedException(tstate); + assert(cause != NULL); + const char *msg = "only stateless functions are shareable"; + set_notshareableerror(tstate, cause, 0, msg); + Py_DECREF(cause); + return -1; + } + PyObject *code = PyFunction_GET_CODE(func); + + // Ideally code objects would be immortal and directly shareable. + // In the meantime, we use marshal. + if (_PyMarshal_GetXIData(tstate, code, xidata) < 0) { + return -1; + } + // Replace _PyMarshal_ReadObjectFromXIData. + // (_PyFunction_FromXIData() will call it.) + _PyXIData_SET_NEW_OBJECT(xidata, _PyFunction_FromXIData); + return 0; +} + // registration static void _register_builtins_for_crossinterpreter_data(dlregistry_t *xidregistry) { +#define REGISTER(TYPE, GETDATA) \ + _xidregistry_add_type(xidregistry, (PyTypeObject *)TYPE, \ + ((_PyXIData_getdata_t){.basic=(GETDATA)})) +#define REGISTER_FALLBACK(TYPE, GETDATA) \ + _xidregistry_add_type(xidregistry, (PyTypeObject *)TYPE, \ + ((_PyXIData_getdata_t){.fallback=(GETDATA)})) // None - if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + if (REGISTER(Py_TYPE(Py_None), _none_shared) != 0) { Py_FatalError("could not register None for cross-interpreter sharing"); } // int - if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { + if (REGISTER(&PyLong_Type, _long_shared) != 0) { Py_FatalError("could not register int for cross-interpreter sharing"); } // bytes - if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _PyBytes_GetXIData) != 0) { + if (REGISTER(&PyBytes_Type, _PyBytes_GetXIData) != 0) { Py_FatalError("could not register bytes for cross-interpreter sharing"); } // str - if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + if (REGISTER(&PyUnicode_Type, _str_shared) != 0) { Py_FatalError("could not register str for cross-interpreter sharing"); } // bool - if (_xidregistry_add_type(xidregistry, &PyBool_Type, _bool_shared) != 0) { + if (REGISTER(&PyBool_Type, _bool_shared) != 0) { Py_FatalError("could not register bool for cross-interpreter sharing"); } // float - if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) { + if (REGISTER(&PyFloat_Type, _float_shared) != 0) { Py_FatalError("could not register float for cross-interpreter sharing"); } // tuple - if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) { + if (REGISTER_FALLBACK(&PyTuple_Type, _tuple_shared) != 0) { Py_FatalError("could not register tuple for cross-interpreter sharing"); } + + // For now, we do not register PyCode_Type or PyFunction_Type. +#undef REGISTER +#undef REGISTER_FALLBACK } diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index ca4ca1cf123..98411adc5eb 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -7,13 +7,6 @@ _ensure_current_cause(PyThreadState *tstate, PyObject *cause) } PyObject *exc = _PyErr_GetRaisedException(tstate); assert(exc != NULL); - PyObject *ctx = PyException_GetContext(exc); - if (ctx == NULL) { - PyException_SetContext(exc, Py_NewRef(cause)); - } - else { - Py_DECREF(ctx); - } assert(PyException_GetCause(exc) == NULL); PyException_SetCause(exc, Py_NewRef(cause)); _PyErr_SetRaisedException(tstate, exc); @@ -24,7 +17,7 @@ _ensure_current_cause(PyThreadState *tstate, PyObject *cause) static PyTypeObject _PyExc_InterpreterError = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "interpreters.InterpreterError", + .tp_name = "concurrent.interpreters.InterpreterError", .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, @@ -37,7 +30,7 @@ PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; static PyTypeObject _PyExc_InterpreterNotFoundError = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "interpreters.InterpreterNotFoundError", + .tp_name = "concurrent.interpreters.InterpreterNotFoundError", .tp_doc = PyDoc_STR("An interpreter was not found"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, @@ -51,7 +44,7 @@ PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFou static int _init_notshareableerror(exceptions_t *state) { - const char *name = "interpreters.NotShareableError"; + const char *name = "concurrent.interpreters.NotShareableError"; PyObject *base = PyExc_TypeError; PyObject *ns = NULL; PyObject *exctype = PyErr_NewException(name, base, ns); diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 6324063401e..de9b0a77817 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -108,7 +108,7 @@ static char *GetPythonImport (HINSTANCE hModule) char *pch; /* Don't claim that python3.dll is a Python DLL. */ -#ifdef _DEBUG +#ifdef Py_DEBUG if (strcmp(import_name, "python3_d.dll") == 0) { #else if (strcmp(import_name, "python3.dll") == 0) { @@ -120,7 +120,7 @@ static char *GetPythonImport (HINSTANCE hModule) /* Ensure python prefix is followed only by numbers to the end of the basename */ pch = import_name + 6; -#ifdef _DEBUG +#ifdef Py_DEBUG while (*pch && pch[0] != '_' && pch[1] != 'd' && pch[2] != '.') { #else while (*pch && *pch != '.') { @@ -300,7 +300,7 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, char buffer[256]; PyOS_snprintf(buffer, sizeof(buffer), -#ifdef _DEBUG +#ifdef Py_DEBUG "python%d%d_d.dll", #else "python%d%d.dll", diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c new file mode 100644 index 00000000000..98ee44276e5 --- /dev/null +++ b/Python/emscripten_syscalls.c @@ -0,0 +1,323 @@ +#include "emscripten.h" +#include "stdio.h" + +// All system calls: return nonnegative number on success, return -errno on +// failure. Negative results get stored back into errno here: +// https://github.com/emscripten-core/emscripten/blob/main/system/lib/libc/musl/src/internal/syscall_ret.c#L7 + +// If we're running in node, report the UID of the user in the native system as +// the UID of the user. Since the nodefs will report the uid correctly, if we +// don't make getuid report it correctly too we'll see some permission errors. +// Normally __syscall_getuid32 is a stub that always returns 0 but it is +// defined with weak linkage so we can override it. +EM_JS(int, __syscall_getuid32_js, (void), { + // If we're in node and we can, report the native uid + if (ENVIRONMENT_IS_NODE) { + return process.getuid(); + } + // Fall back to the stub case of returning 0. + return 0; +}) + +int __syscall_getuid32(void) { + return __syscall_getuid32_js(); +} + +EM_JS(int, __syscall_umask_js, (int mask), { + // If we're in node and we can, call native process.umask() + if (ENVIRONMENT_IS_NODE) { + try { + return process.umask(mask); + } catch(e) { + // oops... + // NodeJS docs: "In Worker threads, process.umask(mask) will throw an exception." + // umask docs: "This system call always succeeds" + return 0; + } + } + // Fall back to the stub case of returning 0. + return 0; +}) + +int __syscall_umask(int mask) { + return __syscall_umask_js(mask); +} + +#include <wasi/api.h> +#include <errno.h> +#include <fcntl.h> + +// Variant of EM_JS that does C preprocessor substitution on the body +#define EM_JS_MACROS(ret, func_name, args, body...) \ + EM_JS(ret, func_name, args, body) + +EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { + // Define FS.createAsyncInputDevice(), This is quite similar to + // FS.createDevice() defined here: + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 + // but instead of returning one byte at a time, the input() function should + // return a Uint8Array. This makes the handler code simpler, the + // `createAsyncInputDevice` simpler, and everything faster. + FS.createAsyncInputDevice = function(parent, name, input) { + parent = typeof parent == 'string' ? parent : FS.getPath(parent); + var path = PATH.join2(parent, name); + var mode = FS_getMode(true, false); + FS.createDevice.major ||= 64; + var dev = FS.makedev(FS.createDevice.major++, 0); + async function getDataBuf() { + var buf; + try { + buf = await input(); + } catch (e) { + throw new FS.ErrnoError(EIO); + } + if (!buf?.byteLength) { + throw new FS.ErrnoError(EAGAIN); + } + ops._dataBuf = buf; + } + + var ops = { + _dataBuf: new Uint8Array(0), + open(stream) { + stream.seekable = false; + }, + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + buffer = buffer.subarray(offset, offset + length); + if (!ops._dataBuf.byteLength) { + await getDataBuf(); + } + var toRead = Math.min(ops._dataBuf.byteLength, buffer.byteLength); + buffer.subarray(0, toRead).set(ops._dataBuf); + buffer = buffer.subarray(toRead); + ops._dataBuf = ops._dataBuf.subarray(toRead); + if (toRead) { + stream.node.atime = Date.now(); + } + return toRead; + }, + }; + FS.registerDevice(dev, ops); + return FS.mkdev(path, mode, dev); + }; + if (!WebAssembly.promising) { + // No stack switching support =( + return; + } + const origResolveGlobalSymbol = resolveGlobalSymbol; + if (ENVIRONMENT_IS_NODE && !Module.onExit) { + Module.onExit = (code) => process.exit(code); + } + // * wrap the main symbol with WebAssembly.promising, + // * call exit_with_live_runtime() to prevent emscripten from shutting down + // the runtime before the promise resolves, + // * call onExit / process.exit ourselves, since exit_with_live_runtime() + // prevented Emscripten from calling it normally. + resolveGlobalSymbol = function (name, direct = false) { + const orig = origResolveGlobalSymbol(name, direct); + if (name === "main") { + const main = WebAssembly.promising(orig.sym); + orig.sym = (...args) => { + (async () => { + const ret = await main(...args); + Module.onExit?.(ret); + })(); + _emscripten_exit_with_live_runtime(); + }; + } + return orig; + }; +}) + +__attribute__((constructor)) void _emscripten_promising_main(void) { + _emscripten_promising_main_js(); +} + + +#define IOVEC_T_BUF_OFFSET 0 +#define IOVEC_T_BUF_LEN_OFFSET 4 +#define IOVEC_T_SIZE 8 +_Static_assert(offsetof(__wasi_iovec_t, buf) == IOVEC_T_BUF_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(offsetof(__wasi_iovec_t, buf_len) == IOVEC_T_BUF_LEN_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(sizeof(__wasi_iovec_t) == IOVEC_T_SIZE, + "Unexpected __wasi_iovec_t layout"); + +// If the stream has a readAsync handler, read to buffer defined in iovs, write +// number of bytes read to *nread, and return a promise that resolves to the +// errno. Otherwise, return null. +EM_JS_MACROS(__externref_t, __maybe_fd_read_async, ( + __wasi_fd_t fd, + const __wasi_iovec_t *iovs, + size_t iovcnt, + __wasi_size_t *nread +), { + if (!WebAssembly.promising) { + return null; + } + var stream; + try { + stream = SYSCALLS.getStreamFromFD(fd); + } catch (e) { + // If the fd was already closed or never existed, getStreamFromFD() + // raises. We'll let fd_read_orig() handle setting errno. + return null; + } + if (!stream.stream_ops.readAsync) { + // Not an async device. Fall back to __wasi_fd_read_orig(). + return null; + } + return (async () => { + // This is the same as libwasi.js fd_read() and doReadv() except we use + // readAsync and we await it. + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L331 + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L197 + try { + var ret = 0; + for (var i = 0; i < iovcnt; i++) { + var ptr = HEAP32[(iovs + IOVEC_T_BUF_OFFSET)/4]; + var len = HEAP32[(iovs + IOVEC_T_BUF_LEN_OFFSET)/4]; + iovs += IOVEC_T_SIZE; + var curr = await stream.stream_ops.readAsync(stream, HEAP8, ptr, len); + if (curr < 0) return -1; + ret += curr; + if (curr < len) break; // nothing more to read + } + HEAP32[nread/4] = ret; + return 0; + } catch (e) { + if (e.name !== 'ErrnoError') { + throw e; + } + return e["errno"]; + } + })(); +}; +); + +// Bind original fd_read syscall to __wasi_fd_read_orig(). +__wasi_errno_t __wasi_fd_read_orig(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("fd_read"), __warn_unused_result__)); + +// Take a promise that resolves to __wasi_errno_t and suspend until it resolves, +// get the output. +EM_JS(int, __block_for_int, (__externref_t p), { + return p; +} +if (WebAssembly.Suspending) { + __block_for_int = new WebAssembly.Suspending(__block_for_int); +} +) + +// Replacement for fd_read syscall. Call __maybe_fd_read_async. If it returned +// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_int +// to get the result. +__wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) { + __externref_t p = __maybe_fd_read_async(fd, iovs, iovs_len, nread); + if (__builtin_wasm_ref_is_null_extern(p)) { + return __wasi_fd_read_orig(fd, iovs, iovs_len, nread); + } + return __block_for_int(p); +} + +#include <poll.h> +#define POLLFD_FD 0 +#define POLLFD_EVENTS 4 +#define POLLFD_REVENTS 6 +#define POLLFD_SIZE 8 +_Static_assert(offsetof(struct pollfd, fd) == 0, "Unepxected pollfd struct layout"); +_Static_assert(offsetof(struct pollfd, events) == 4, "Unepxected pollfd struct layout"); +_Static_assert(offsetof(struct pollfd, revents) == 6, "Unepxected pollfd struct layout"); +_Static_assert(sizeof(struct pollfd) == 8, "Unepxected pollfd struct layout"); + +EM_JS_MACROS(__externref_t, __maybe_poll_async, (intptr_t fds, int nfds, int timeout), { + if (!WebAssembly.promising) { + return null; + } + return (async function() { + try { + var nonzero = 0; + var promises = []; + for (var i = 0; i < nfds; i++) { + var pollfd = fds + POLLFD_SIZE * i; + var fd = HEAP32[(pollfd + POLLFD_FD)/4]; + var events = HEAP16[(pollfd + POLLFD_EVENTS)/2]; + var mask = POLLNVAL; + var stream = FS.getStream(fd); + if (stream) { + mask = POLLIN | POLLOUT; + if (stream.stream_ops.pollAsync) { + promises.push(stream.stream_ops.pollAsync(stream, timeout).then((mask) => { + mask &= events | POLLERR | POLLHUP; + HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask; + if (mask) { + nonzero ++; + } + })); + } else if (stream.stream_ops.poll) { + var mask = stream.stream_ops.poll(stream, timeout); + mask &= events | POLLERR | POLLHUP; + HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask; + if (mask) { + nonzero ++; + } + } + } + } + await Promise.all(promises); + return nonzero; + } catch(e) { + if (e?.name !== "ErrnoError") throw e; + return -e["errno"]; + } + })(); +}); + +// Bind original poll syscall to syscall_poll_orig(). +int syscall_poll_orig(intptr_t fds, int nfds, int timeout) + __attribute__((__import_module__("env"), + __import_name__("__syscall_poll"), __warn_unused_result__)); + +int __syscall_poll(intptr_t fds, int nfds, int timeout) { + __externref_t p = __maybe_poll_async(fds, nfds, timeout); + if (__builtin_wasm_ref_is_null_extern(p)) { + return syscall_poll_orig(fds, nfds, timeout); + } + return __block_for_int(p); +} + +#include <sys/ioctl.h> + +int syscall_ioctl_orig(int fd, int request, void* varargs) + __attribute__((__import_module__("env"), + __import_name__("__syscall_ioctl"), __warn_unused_result__)); + +int __syscall_ioctl(int fd, int request, void* varargs) { + if (request == FIOCLEX || request == FIONCLEX) { + return 0; + } + if (request == FIONBIO) { + // Implement FIONBIO via fcntl. + // TODO: Upstream this. + int flags = fcntl(fd, F_GETFL, 0); + int nonblock = **((int**)varargs); + if (flags < 0) { + return -errno; + } + if (nonblock) { + flags |= O_NONBLOCK; + } else { + flags &= (~O_NONBLOCK); + } + int res = fcntl(fd, F_SETFL, flags); + if (res < 0) { + return -errno; + } + return res; + } + return syscall_ioctl_orig(fd, request, varargs); +} diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index a7bb685bf3d..d61146504d0 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -2,200 +2,45 @@ #include <emscripten.h> // EM_JS, EM_JS_DEPS #include <Python.h> -#include "pycore_runtime.h" // _PyRuntime -typedef int (*CountArgsFunc)(PyCFunctionWithKeywords func); - -// Offset of emscripten_count_args_function in _PyRuntimeState. There's a couple -// of alternatives: -// 1. Just make emscripten_count_args_function a real C global variable instead -// of a field of _PyRuntimeState. This would violate our rule against mutable -// globals. -// 2. #define a preprocessor constant equal to a hard coded number and make a -// _Static_assert(offsetof(_PyRuntimeState, emscripten_count_args_function) -// == OURCONSTANT) This has the disadvantage that we have to update the hard -// coded constant when _PyRuntimeState changes -// -// So putting the mutable constant in _PyRuntime and using a immutable global to -// record the offset so we can access it from JS is probably the best way. -EMSCRIPTEN_KEEPALIVE const int _PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET = offsetof(_PyRuntimeState, emscripten_count_args_function); - -EM_JS(CountArgsFunc, _PyEM_GetCountArgsPtr, (), { - return Module._PyEM_CountArgsPtr; // initialized below -} -// Binary module for the checks. It has to be done in web assembly because -// clang/llvm have no support yet for the reference types yet. In fact, the wasm -// binary toolkit doesn't yet support the ref.test instruction either. To -// convert the following textual wasm to a binary, you can build wabt from this -// branch: https://github.com/WebAssembly/wabt/pull/2529 and then use that -// wat2wasm binary. -// -// (module -// (type $type0 (func (param) (result i32))) -// (type $type1 (func (param i32) (result i32))) -// (type $type2 (func (param i32 i32) (result i32))) -// (type $type3 (func (param i32 i32 i32) (result i32))) -// (type $blocktype (func (param i32) (result))) -// (table $funcs (import "e" "t") 0 funcref) -// (export "f" (func $f)) -// (func $f (param $fptr i32) (result i32) -// (local $fref funcref) -// local.get $fptr -// table.get $funcs -// local.tee $fref -// ref.test $type3 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b -// i32.const 3 -// return -// ) -// local.get $fref -// ref.test $type2 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b -// i32.const 2 -// return -// ) -// local.get $fref -// ref.test $type1 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b -// i32.const 1 -// return -// ) -// local.get $fref -// ref.test $type0 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b -// i32.const 0 -// return -// ) -// i32.const -1 -// ) -// ) - -function getPyEMCountArgsPtr() { - let isIOS = globalThis.navigator && /iPad|iPhone|iPod/.test(navigator.platform); - if (isIOS) { - return 0; - } - - // Try to initialize countArgsFunc - const code = new Uint8Array([ - 0x00, 0x61, 0x73, 0x6d, // \0asm magic number - 0x01, 0x00, 0x00, 0x00, // version 1 - 0x01, 0x1b, // Type section, body is 0x1b bytes - 0x05, // 6 entries - 0x60, 0x00, 0x01, 0x7f, // (type $type0 (func (param) (result i32))) - 0x60, 0x01, 0x7f, 0x01, 0x7f, // (type $type1 (func (param i32) (result i32))) - 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, // (type $type2 (func (param i32 i32) (result i32))) - 0x60, 0x03, 0x7f, 0x7f, 0x7f, 0x01, 0x7f, // (type $type3 (func (param i32 i32 i32) (result i32))) - 0x60, 0x01, 0x7f, 0x00, // (type $blocktype (func (param i32) (result))) - 0x02, 0x09, // Import section, 0x9 byte body - 0x01, // 1 import (table $funcs (import "e" "t") 0 funcref) - 0x01, 0x65, // "e" - 0x01, 0x74, // "t" - 0x01, // importing a table - 0x70, // of entry type funcref - 0x00, 0x00, // table limits: no max, min of 0 - 0x03, 0x02, // Function section - 0x01, 0x01, // We're going to define one function of type 1 (func (param i32) (result i32)) - 0x07, 0x05, // export section - 0x01, // 1 export - 0x01, 0x66, // called "f" - 0x00, // a function - 0x00, // at index 0 - - 0x0a, 0x44, // Code section, - 0x01, 0x42, // one entry of length 50 - 0x01, 0x01, 0x70, // one local of type funcref - // Body of the function - 0x20, 0x00, // local.get $fptr - 0x25, 0x00, // table.get $funcs - 0x22, 0x01, // local.tee $fref - 0xfb, 0x14, 0x03, // ref.test $type3 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b - 0x41, 0x03, // i32.const 3 - 0x0f, // return - 0x0b, // end block - - 0x20, 0x01, // local.get $fref - 0xfb, 0x14, 0x02, // ref.test $type2 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b - 0x41, 0x02, // i32.const 2 - 0x0f, // return - 0x0b, // end block - - 0x20, 0x01, // local.get $fref - 0xfb, 0x14, 0x01, // ref.test $type1 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b - 0x41, 0x01, // i32.const 1 - 0x0f, // return - 0x0b, // end block - - 0x20, 0x01, // local.get $fref - 0xfb, 0x14, 0x00, // ref.test $type0 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b - 0x41, 0x00, // i32.const 0 - 0x0f, // return - 0x0b, // end block - - 0x41, 0x7f, // i32.const -1 - 0x0b // end function - ]); - try { - const mod = new WebAssembly.Module(code); - const inst = new WebAssembly.Instance(mod, { e: { t: wasmTable } }); - return addFunction(inst.exports.f); - } catch (e) { - // If something goes wrong, we'll null out _PyEM_CountFuncParams and fall - // back to the JS trampoline. - return 0; - } -} - -addOnPreRun(() => { - const ptr = getPyEMCountArgsPtr(); - Module._PyEM_CountArgsPtr = ptr; - const offset = HEAP32[__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET / 4]; - HEAP32[(__PyRuntime + offset) / 4] = ptr; -}); -); - -void -_Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime) -{ - runtime->emscripten_count_args_function = _PyEM_GetCountArgsPtr(); -} - -// We have to be careful to work correctly with memory snapshots. Even if we are -// loading a memory snapshot, we need to perform the JS initialization work. -// That means we can't call the initialization code from C. Instead, we export -// this function pointer to JS and then fill it in a preRun function which runs -// unconditionally. -/** - * Backwards compatible trampoline works with all JS runtimes - */ -EM_JS(PyObject*, _PyEM_TrampolineCall_JS, (PyCFunctionWithKeywords func, PyObject *arg1, PyObject *arg2, PyObject *arg3), { +EM_JS( +PyObject*, +_PyEM_TrampolineCall_inner, (int* success, + PyCFunctionWithKeywords func, + PyObject *arg1, + PyObject *arg2, + PyObject *arg3), { + // JavaScript fallback trampoline return wasmTable.get(func)(arg1, arg2, arg3); -}); - -typedef PyObject* (*zero_arg)(void); -typedef PyObject* (*one_arg)(PyObject*); -typedef PyObject* (*two_arg)(PyObject*, PyObject*); -typedef PyObject* (*three_arg)(PyObject*, PyObject*, PyObject*); +} +// Try to replace the JS definition of _PyEM_TrampolineCall_inner with a wasm +// version. +(function () { + // Starting with iOS 18.3.1, WebKit on iOS has an issue with the garbage + // collector that breaks the call trampoline. See #130418 and + // https://bugs.webkit.org/show_bug.cgi?id=293113 for details. + let isIOS = globalThis.navigator && ( + /iPad|iPhone|iPod/.test(navigator.userAgent) || + // Starting with iPadOS 13, iPads might send a platform string that looks like a desktop Mac. + // To differentiate, we check if the platform is 'MacIntel' (common for Macs and newer iPads) + // AND if the device has multi-touch capabilities (navigator.maxTouchPoints > 1) + (navigator.platform === 'MacIntel' && typeof navigator.maxTouchPoints !== 'undefined' && navigator.maxTouchPoints > 1) + ); + if (isIOS) { + return; + } + try { + const trampolineModule = getWasmTrampolineModule(); + const trampolineInstance = new WebAssembly.Instance(trampolineModule, { + env: { __indirect_function_table: wasmTable, memory: wasmMemory }, + }); + _PyEM_TrampolineCall_inner = trampolineInstance.exports.trampoline_call; + } catch (e) { + // Compilation error due to missing wasm-gc support, fall back to JS + // trampoline + } +})(); +); PyObject* _PyEM_TrampolineCall(PyCFunctionWithKeywords func, @@ -203,23 +48,12 @@ _PyEM_TrampolineCall(PyCFunctionWithKeywords func, PyObject* args, PyObject* kw) { - CountArgsFunc count_args = _PyRuntime.emscripten_count_args_function; - if (count_args == 0) { - return _PyEM_TrampolineCall_JS(func, self, args, kw); - } - switch (count_args(func)) { - case 0: - return ((zero_arg)func)(); - case 1: - return ((one_arg)func)(self); - case 2: - return ((two_arg)func)(self, args); - case 3: - return ((three_arg)func)(self, args, kw); - default: - PyErr_SetString(PyExc_SystemError, "Handler takes too many arguments"); - return NULL; + int success = 1; + PyObject *result = _PyEM_TrampolineCall_inner(&success, func, self, args, kw); + if (!success) { + PyErr_SetString(PyExc_SystemError, "Handler takes too many arguments"); } + return result; } #endif diff --git a/Python/emscripten_trampoline_inner.c b/Python/emscripten_trampoline_inner.c new file mode 100644 index 00000000000..a2bad4857ed --- /dev/null +++ b/Python/emscripten_trampoline_inner.c @@ -0,0 +1,38 @@ +// This file must be compiled with -mgc to enable the extra wasm-gc +// instructions. It has to be compiled separately because not enough JS runtimes +// support wasm-gc yet. If the JS runtime does not support wasm-gc (or has buggy +// support like iOS), we will use the JS trampoline fallback. + +// We can't import Python.h here because it is compiled/linked with -nostdlib. +// We don't need to know what's inside PyObject* anyways. We could just call it +// void* everywhere. There are two reasons to do this: +// 1. to improve readability +// 2. eventually when we are comfortable requiring wasm-gc, we can merge this +// into emscripten_trampoline.c without worrying about it. +typedef void PyObject; + +typedef PyObject* (*three_arg)(PyObject*, PyObject*, PyObject*); +typedef PyObject* (*two_arg)(PyObject*, PyObject*); +typedef PyObject* (*one_arg)(PyObject*); +typedef PyObject* (*zero_arg)(void); + +#define TRY_RETURN_CALL(ty, args...) \ + if (__builtin_wasm_test_function_pointer_signature((ty)func)) { \ + return ((ty)func)(args); \ + } + +__attribute__((export_name("trampoline_call"))) PyObject* +trampoline_call(int* success, + void* func, + PyObject* self, + PyObject* args, + PyObject* kw) +{ + *success = 1; + TRY_RETURN_CALL(three_arg, self, args, kw); + TRY_RETURN_CALL(two_arg, self, args); + TRY_RETURN_CALL(one_arg, self); + TRY_RETURN_CALL(zero_arg); + *success = 0; + return 0; +} diff --git a/Python/errors.c b/Python/errors.c index 81f267b043a..5c6ac48371a 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -10,7 +10,6 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttr() #include "pycore_traceback.h" // _PyTraceBack_FromFrame() #include "pycore_unicodeobject.h" // _PyUnicode_Equal() @@ -1445,12 +1444,16 @@ make_unraisable_hook_args(PyThreadState *tstate, PyObject *exc_type, It can be called to log the exception of a custom sys.unraisablehook. - Do nothing if sys.stderr attribute doesn't exist or is set to None. */ + This assumes 'file' is neither NULL nor None. + */ static int write_unraisable_exc_file(PyThreadState *tstate, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb, PyObject *err_msg, PyObject *obj, PyObject *file) { + assert(file != NULL); + assert(!Py_IsNone(file)); + if (obj != NULL && obj != Py_None) { if (err_msg != NULL && err_msg != Py_None) { if (PyFile_WriteObject(err_msg, file, Py_PRINT_RAW) < 0) { @@ -1485,6 +1488,27 @@ write_unraisable_exc_file(PyThreadState *tstate, PyObject *exc_type, } } + // Try printing the exception using the stdlib module. + // If this fails, then we have to use the C implementation. + PyObject *print_exception_fn = PyImport_ImportModuleAttrString("traceback", + "_print_exception_bltin"); + if (print_exception_fn != NULL && PyCallable_Check(print_exception_fn)) { + PyObject *args[2] = {exc_value, file}; + PyObject *result = PyObject_Vectorcall(print_exception_fn, args, 2, NULL); + int ok = (result != NULL); + Py_DECREF(print_exception_fn); + Py_XDECREF(result); + if (ok) { + // Nothing else to do + return 0; + } + } + else { + Py_XDECREF(print_exception_fn); + } + // traceback module failed, fall back to pure C + _PyErr_Clear(tstate); + if (exc_tb != NULL && exc_tb != Py_None) { if (PyTraceBack_Print(exc_tb, file) < 0) { /* continue even if writing the traceback failed */ @@ -1570,7 +1594,7 @@ write_unraisable_exc(PyThreadState *tstate, PyObject *exc_type, PyObject *obj) { PyObject *file; - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { return -1; } if (file == NULL || file == Py_None) { @@ -1677,7 +1701,7 @@ format_unraisable_v(const char *format, va_list va, PyObject *obj) } PyObject *hook; - if (_PySys_GetOptionalAttr(&_Py_ID(unraisablehook), &hook) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(unraisablehook), &hook) < 0) { Py_DECREF(hook_args); err_msg_str = NULL; obj = NULL; @@ -1936,10 +1960,11 @@ _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_o */ int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset) + int end_lineno, int end_col_offset, + PyObject *module) { - if (_PyErr_WarnExplicitObjectWithContext(PyExc_SyntaxWarning, msg, - filename, lineno) < 0) + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, filename, lineno, + module, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { /* Replace the SyntaxWarning exception with a SyntaxError diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3e51ac41fa3..7273a87681b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -13,31 +13,25 @@ } case _CHECK_PERIODIC: { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_ERROR(); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_ERROR(); } break; } + /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 because it is replaced */ + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { oparg = CURRENT_OPARG(); if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_ERROR(); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_ERROR(); } } break; @@ -90,7 +84,7 @@ value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -102,7 +96,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -114,7 +108,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -126,7 +120,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -138,7 +132,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -150,7 +144,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -162,7 +156,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -174,7 +168,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -186,7 +180,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -197,7 +191,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -209,7 +203,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -221,7 +215,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -233,7 +227,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -245,7 +239,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -257,7 +251,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -269,7 +263,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -281,7 +275,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -293,7 +287,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -304,7 +298,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -315,32 +309,18 @@ GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } - /* _LOAD_CONST is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - - case _LOAD_CONST_MORTAL: { + case _LOAD_CONST: { _PyStackRef value; oparg = CURRENT_OPARG(); PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNewMortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_CONST_IMMORTAL: { - _PyStackRef value; - oparg = CURRENT_OPARG(); - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -350,10 +330,10 @@ assert(oparg == CURRENT_OPARG()); assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -363,10 +343,10 @@ assert(oparg == CURRENT_OPARG()); assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -376,10 +356,10 @@ assert(oparg == CURRENT_OPARG()); assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -389,10 +369,10 @@ assert(oparg == CURRENT_OPARG()); assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -401,10 +381,10 @@ oparg = CURRENT_OPARG(); assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -413,14 +393,10 @@ oparg = 0; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -432,14 +408,10 @@ oparg = 1; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -451,14 +423,10 @@ oparg = 2; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -470,14 +438,10 @@ oparg = 3; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -489,14 +453,10 @@ oparg = 4; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -508,14 +468,10 @@ oparg = 5; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -527,14 +483,10 @@ oparg = 6; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -546,14 +498,10 @@ oparg = 7; assert(oparg == CURRENT_OPARG()); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -564,14 +512,10 @@ _PyStackRef value; oparg = CURRENT_OPARG(); value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -582,9 +526,67 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } + + case _POP_TOP_NOP: { + _PyStackRef value; + value = stack_pointer[-1]; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TOP_INT: { + _PyStackRef value; + value = stack_pointer[-1]; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TOP_FLOAT: { + _PyStackRef value; + value = stack_pointer[-1]; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TOP_UNICODE: { + _PyStackRef value; + value = stack_pointer[-1]; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TWO: { + _PyStackRef tos; + _PyStackRef nos; + tos = stack_pointer[-1]; + nos = stack_pointer[-2]; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(tos); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(nos); stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -594,7 +596,7 @@ res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -602,13 +604,27 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); break; } + case _POP_ITER: { + _PyStackRef index_or_null; + _PyStackRef iter; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } + case _END_SEND: { _PyStackRef value; _PyStackRef receiver; @@ -618,7 +634,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -633,7 +649,7 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -643,7 +659,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -666,7 +682,7 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -676,7 +692,7 @@ res = err ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -707,7 +723,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -817,7 +833,7 @@ else { assert(Py_SIZE(value_o)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -833,14 +849,14 @@ _PyStackRef res; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -852,7 +868,7 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -862,7 +878,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -870,7 +886,7 @@ _PyStackRef left; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!_PyLong_CheckExactAndCompact(left_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -881,7 +897,31 @@ _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_NOS_OVERFLOWED: { + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_OVERFLOWED: { + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -898,21 +938,18 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -926,21 +963,18 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -954,21 +988,18 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1012,12 +1043,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1039,12 +1070,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1066,12 +1097,93 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_ERROR(); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1091,13 +1203,13 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1114,7 +1226,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); @@ -1128,7 +1240,7 @@ PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyObject *right_o = PyStackRef_AsPyObjectSteal(right); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1192,11 +1304,11 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1218,7 +1330,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); @@ -1226,7 +1338,7 @@ stack_pointer += 2; } stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1236,7 +1348,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1259,7 +1371,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); @@ -1277,7 +1389,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1331,7 +1443,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1360,14 +1472,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1399,14 +1511,14 @@ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectImmortal(res_o); + res = PyStackRef_FromPyObjectBorrow(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1457,7 +1569,7 @@ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); res = PyStackRef_FromPyObjectNew(res_o); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = tuple_st; tuple_st = res; @@ -1519,14 +1631,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (rc <= 0) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1561,7 +1673,7 @@ STAT_INC(BINARY_OP, hit); stack_pointer[0] = getitem; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1569,17 +1681,18 @@ _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; getitem = stack_pointer[-1]; sub = stack_pointer[-2]; container = stack_pointer[-3]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = 6 ; - stack_pointer[-3].bits = (uintptr_t)new_frame; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-3] = new_frame; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1593,11 +1706,11 @@ PyStackRef_AsPyObjectSteal(v)); if (err < 0) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1613,11 +1726,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1644,7 +1757,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1686,7 +1799,7 @@ UNLOCK_OBJECT(list); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); Py_DECREF(old_value); @@ -1710,7 +1823,7 @@ PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1738,7 +1851,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1755,7 +1868,7 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1765,7 +1878,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1791,14 +1904,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1809,7 +1922,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -1822,7 +1935,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1845,7 +1958,7 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1855,7 +1968,7 @@ iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1876,7 +1989,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1893,7 +2006,7 @@ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1906,7 +2019,7 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1916,7 +2029,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1925,7 +2038,7 @@ case _SEND_GEN_FRAME: { _PyStackRef v; _PyStackRef receiver; - _PyInterpreterFrame *gen_frame; + _PyStackRef gen_frame; oparg = CURRENT_OPARG(); v = stack_pointer[-1]; receiver = stack_pointer[-2]; @@ -1939,15 +2052,16 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - gen_frame->previous = frame; - stack_pointer[-1].bits = (uintptr_t)gen_frame; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-1] = gen_frame; break; } @@ -1964,7 +2078,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1987,7 +2101,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2001,7 +2115,7 @@ ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2012,7 +2126,7 @@ value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2035,7 +2149,7 @@ bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2052,7 +2166,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2069,7 +2183,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2113,7 +2227,7 @@ top = &stack_pointer[-1 + oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); Py_DECREF(seq_o); @@ -2122,7 +2236,7 @@ JUMP_TO_ERROR(); } stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2145,7 +2259,7 @@ stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2170,7 +2284,7 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2203,7 +2317,7 @@ } UNLOCK_OBJECT(seq_o); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2218,7 +2332,7 @@ top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq_o); @@ -2227,7 +2341,7 @@ JUMP_TO_ERROR(); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2251,7 +2365,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -2267,7 +2381,7 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2286,7 +2400,7 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2328,7 +2442,7 @@ locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2347,7 +2461,7 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2363,7 +2477,7 @@ JUMP_TO_ERROR(); } stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2375,7 +2489,7 @@ null[0] = PyStackRef_NULL; } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2429,7 +2543,7 @@ STAT_INC(LOAD_GLOBAL, hit); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2466,7 +2580,7 @@ STAT_INC(LOAD_GLOBAL, hit); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2549,14 +2663,14 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2570,7 +2684,7 @@ if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2578,7 +2692,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2591,7 +2705,7 @@ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2615,39 +2729,18 @@ _PyStackRef str; oparg = CURRENT_OPARG(); pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2675,7 +2768,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(format[0]); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2684,12 +2777,12 @@ stack_pointer += -(oparg & 1); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2699,7 +2792,7 @@ interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); stack_pointer[0] = interpolation; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2715,12 +2808,12 @@ PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(interpolations); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(strings); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2730,7 +2823,7 @@ template = PyStackRef_FromPyObjectSteal(template_o); stack_pointer[0] = template; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2746,7 +2839,7 @@ tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2764,7 +2857,7 @@ list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2794,7 +2887,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2802,7 +2895,7 @@ } assert(Py_IsNone(none_val)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2820,7 +2913,7 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2848,7 +2941,7 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } int err = 0; @@ -2868,7 +2961,7 @@ } if (err) { stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2877,7 +2970,7 @@ set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2886,44 +2979,18 @@ _PyStackRef map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2989,14 +3056,14 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3022,14 +3089,14 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3055,11 +3122,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3102,14 +3169,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { JUMP_TO_ERROR(); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[0] = attr_st; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3150,7 +3217,7 @@ self_or_null = self_st; } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3158,7 +3225,7 @@ stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; @@ -3170,40 +3237,40 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; stack_pointer[1] = self_or_null; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_ATTR: { _PyStackRef owner; - _PyStackRef attr; + _PyStackRef *attr; _PyStackRef *self_or_null; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; + attr = &stack_pointer[-1]; self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { - attr_o = NULL; + *attr = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); stack_pointer = _PyFrame_GetStackPointer(frame); if (is_meth) { - assert(attr_o != NULL); + assert(!PyStackRef_IsNull(*attr)); self_or_null[0] = owner; } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { + if (PyStackRef_IsNull(*attr)) { JUMP_TO_ERROR(); } self_or_null[0] = PyStackRef_NULL; @@ -3212,22 +3279,21 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { JUMP_TO_ERROR(); } + *attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer += 1; } - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-1] = attr; stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3489,7 +3555,7 @@ case _LOAD_ATTR_PROPERTY_FRAME: { _PyStackRef owner; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *fget = (PyObject *)CURRENT_OPERAND0(); @@ -3514,9 +3580,10 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; - stack_pointer[-1].bits = (uintptr_t)new_frame; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-1] = new_frame; break; } @@ -3558,7 +3625,7 @@ } UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3584,15 +3651,6 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || @@ -3626,7 +3684,7 @@ UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3651,7 +3709,7 @@ FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3681,7 +3739,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } @@ -3700,7 +3758,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3722,7 +3780,7 @@ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3735,14 +3793,8 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (!_PyLong_IsCompact((PyLongObject *)right_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -3754,7 +3806,7 @@ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3778,7 +3830,7 @@ res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3801,11 +3853,11 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3830,14 +3882,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3875,14 +3927,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3909,14 +3961,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3944,7 +3996,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } PyObject *match_o = NULL; @@ -3962,7 +4014,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } @@ -3980,7 +4032,7 @@ stack_pointer[0] = rest; stack_pointer[1] = match; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4003,14 +4055,14 @@ int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4036,14 +4088,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4062,7 +4114,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4090,6 +4142,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 because it is replaced */ + case _GET_LEN: { _PyStackRef obj; _PyStackRef len; @@ -4107,7 +4161,7 @@ len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4140,7 +4194,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -4153,7 +4207,7 @@ } stack_pointer[0] = attrs; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4165,7 +4219,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4177,7 +4231,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4197,34 +4251,46 @@ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GET_ITER: { _PyStackRef iterable; _PyStackRef iter; + _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_ERROR(); + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4269,56 +4335,46 @@ /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ case _FOR_ITER_TIER_TWO: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_ERROR(); } if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } } - next = PyStackRef_FromPyObjectSteal(next_o); + next = item; + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ case _ITER_CHECK_LIST: { + _PyStackRef null_or_index; _PyStackRef iter; - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyListIter_Type) { + if (Py_TYPE(iter_o) != &PyList_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4329,24 +4385,17 @@ /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_LIST: { + _PyStackRef null_or_index; _PyStackRef iter; - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; #ifndef Py_GIL_DISABLED - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - if (seq == NULL) { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } #endif break; } @@ -4354,78 +4403,60 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ case _ITER_NEXT_LIST_TIER_TWO: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); - if (result < 0) { + if (result <= 0) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - if (result == 0) { - it->it_index = -1; - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _ITER_CHECK_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTupleIter_Type) { + if (Py_TYPE(iter_o) != &PyTuple_Type) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - #endif + assert(PyStackRef_IsTaggedInt(null_or_index)); break; } /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ case _GUARD_NOT_EXHAUSTED_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; - iter = stack_pointer[-1]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - PyTupleObject *seq = it->it_seq; - if (seq == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (it->it_index >= PyTuple_GET_SIZE(seq)) { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4433,28 +4464,27 @@ } case _ITER_NEXT_TUPLE: { + _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _ITER_CHECK_RANGE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); @@ -4473,7 +4503,7 @@ case _GUARD_NOT_EXHAUSTED_RANGE: { _PyStackRef iter; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { @@ -4486,7 +4516,7 @@ case _ITER_NEXT_RANGE: { _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -4503,15 +4533,15 @@ next = PyStackRef_FromPyObjectSteal(res); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _FOR_ITER_GEN_FRAME: { _PyStackRef iter; - _PyInterpreterFrame *gen_frame; + _PyStackRef gen_frame; oparg = CURRENT_OPARG(); - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); @@ -4529,16 +4559,17 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - frame->return_offset = (uint16_t)( 2 + oparg); - stack_pointer[0].bits = (uintptr_t)gen_frame; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[0] = gen_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4550,7 +4581,7 @@ method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4615,7 +4646,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4637,7 +4668,7 @@ stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4684,7 +4715,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4705,7 +4736,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4719,14 +4750,14 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4741,14 +4772,14 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4781,7 +4812,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4817,7 +4848,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -4838,14 +4869,14 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_ERROR(); } - new_frame = temp; - stack_pointer[0].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(temp); + stack_pointer[0] = new_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4956,68 +4987,28 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5109,7 +5100,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 0; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5117,15 +5108,16 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5133,7 +5125,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 1; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5141,15 +5133,16 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5157,7 +5150,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 2; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5165,15 +5158,16 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5181,7 +5175,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 3; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5189,15 +5183,16 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5205,7 +5200,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = 4; assert(oparg == CURRENT_OPARG()); args = &stack_pointer[-oparg]; @@ -5213,15 +5208,16 @@ callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5229,34 +5225,35 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } - stack_pointer[-2 - oparg].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(pushed_frame); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _PUSH_FRAME: { - _PyInterpreterFrame *new_frame; - new_frame = (_PyInterpreterFrame *)stack_pointer[-1].bits; + _PyStackRef new_frame; + new_frame = stack_pointer[-1]; assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -5276,6 +5273,27 @@ break; } + case _GUARD_NOS_NOT_NULL: { + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_THIRD_NULL: { + _PyStackRef null; + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _GUARD_CALLABLE_TYPE_1: { _PyStackRef callable; callable = stack_pointer[-3]; @@ -5304,7 +5322,7 @@ res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5340,7 +5358,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5350,7 +5368,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5383,7 +5401,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5393,7 +5411,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5450,7 +5468,7 @@ _PyStackRef *args; _PyStackRef self; _PyStackRef init; - _PyInterpreterFrame *init_frame; + _PyStackRef init_frame; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; @@ -5467,19 +5485,19 @@ tstate, init, NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } - init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; tstate->py_recursion_remaining--; - stack_pointer[0].bits = (uintptr_t)init_frame; + init_frame = PyStackRef_Wrap(temp); + stack_pointer[0] = init_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5495,7 +5513,7 @@ JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5525,57 +5543,21 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5622,7 +5604,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5632,7 +5614,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5645,13 +5627,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5661,62 +5643,22 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5729,13 +5671,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5746,61 +5688,17 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5839,75 +5737,86 @@ JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } - case _CALL_ISINSTANCE: { - _PyStackRef *args; - _PyStackRef self_or_null; + case _GUARD_CALLABLE_ISINSTANCE: { _PyStackRef callable; - _PyStackRef res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-4]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args != 2) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + break; + } + + case _CALL_ISINSTANCE: { + _PyStackRef cls; + _PyStackRef instance; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef res; + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + null = stack_pointer[-3]; + callable = stack_pointer[-4]; STAT_INC(CALL, hit); - _PyStackRef cls_stackref = arguments[1]; - _PyStackRef inst_stackref = arguments[0]; + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); + int retval = PyObject_IsInstance(inst_o, cls_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { JUMP_TO_ERROR(); } + (void)null; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(cls); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(instance); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = callable; - callable = res; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _GUARD_CALLABLE_LIST_APPEND: { + _PyStackRef callable; + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); break; } @@ -5920,21 +5829,7 @@ self = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (self_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (!PyList_Check(self_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5943,12 +5838,12 @@ int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6030,14 +5925,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6079,61 +5974,24 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6188,7 +6046,7 @@ PyStackRef_CLOSE(self_stackref); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6198,7 +6056,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6239,60 +6097,24 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6329,7 +6151,7 @@ _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; oparg = CURRENT_OPARG(); kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; @@ -6354,19 +6176,19 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_ERROR(); } - new_frame = temp; - stack_pointer[0].bits = (uintptr_t)new_frame; + new_frame = PyStackRef_Wrap(temp); + stack_pointer[0] = new_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6467,78 +6289,28 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6584,7 +6356,7 @@ PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6596,7 +6368,7 @@ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[0] = func; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6618,7 +6390,7 @@ *ptr = attr; stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6650,7 +6422,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6672,14 +6444,14 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { JUMP_TO_ERROR(); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6695,7 +6467,7 @@ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6705,7 +6477,7 @@ result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6719,7 +6491,7 @@ PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6734,7 +6506,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6756,14 +6528,47 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _COPY_1: { + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _COPY_2: { + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-2]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _COPY_3: { + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-3]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6772,11 +6577,10 @@ _PyStackRef top; oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6808,7 +6612,33 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _SWAP_2: { + _PyStackRef top; + _PyStackRef bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-2] = bottom; + stack_pointer[-1] = top; + break; + } + + case _SWAP_3: { + _PyStackRef top; + _PyStackRef bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-3]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-3] = bottom; + stack_pointer[-1] = top; break; } @@ -6821,7 +6651,6 @@ _PyStackRef temp = bottom; bottom = top; top = temp; - assert(oparg >= 2); stack_pointer[-2 - (oparg-2)] = bottom; stack_pointer[-1] = top; break; @@ -6850,7 +6679,7 @@ flag = stack_pointer[-1]; int is_true = PyStackRef_IsTrue(flag); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!is_true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -6863,7 +6692,7 @@ flag = stack_pointer[-1]; int is_false = PyStackRef_IsFalse(flag); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!is_false) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -6877,7 +6706,7 @@ int is_none = PyStackRef_IsNone(val); if (!is_none) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(val); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6887,7 +6716,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6896,7 +6725,7 @@ val = stack_pointer[-1]; int is_none = PyStackRef_IsNone(val); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(val); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6946,52 +6775,46 @@ case _EXIT_TRACE: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %lu, temp %d, target %d -> %s]\n", + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code], exit->is_control_flow); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + tstate->jit_exit = exit; + TIER2_TO_TIER2(exit->executor); + break; + } + + case _DYNAMIC_EXIT: { + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - if (exit->executor && !exit->executor->vm_data.valid) { - exit->temperature = initial_temperature_backoff_counter(); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(exit->executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (exit->executor == NULL) { - _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - int chain_depth = current_executor->vm_data.chain_depth + 1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } - exit->temperature = initial_temperature_backoff_counter(); - } - exit->executor = executor; - } - GOTO_TIER_TWO(exit->executor); + + GOTO_TIER_ONE(frame->instr_ptr); break; } @@ -7009,7 +6832,7 @@ value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7019,24 +6842,87 @@ pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_CONST_INLINE_BORROW: { _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - value = PyStackRef_FromPyObjectImmortal(ptr); + value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_CALL: { + _PyStackRef null; + _PyStackRef callable; + null = stack_pointer[-1]; + callable = stack_pointer[-2]; + (void)null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } + + case _POP_CALL_ONE: { + _PyStackRef pop; + _PyStackRef null; + _PyStackRef callable; + pop = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + break; + } + + case _POP_CALL_TWO: { + _PyStackRef pop2; + _PyStackRef pop1; + _PyStackRef null; + _PyStackRef callable; + pop2 = stack_pointer[-1]; + pop1 = stack_pointer[-2]; + null = stack_pointer[-3]; + callable = stack_pointer[-4]; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop2); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop1); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); break; } @@ -7046,14 +6932,14 @@ pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectImmortal(ptr); + value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7065,47 +6951,156 @@ pop1 = stack_pointer[-2]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectImmortal(ptr); + value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } - case _CHECK_FUNCTION: { - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - if (func->func_version != func_version) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _POP_CALL_LOAD_CONST_INLINE_BORROW: { + _PyStackRef null; + _PyStackRef callable; + _PyStackRef value; + null = stack_pointer[-1]; + callable = stack_pointer[-2]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + (void)null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { + _PyStackRef pop; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef value; + pop = stack_pointer[-1]; + null = stack_pointer[-2]; + callable = stack_pointer[-3]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { + _PyStackRef pop2; + _PyStackRef pop1; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef value; + pop2 = stack_pointer[-1]; + pop1 = stack_pointer[-2]; + null = stack_pointer[-3]; + callable = stack_pointer[-4]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop2); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop1); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _LOAD_CONST_UNDER_INLINE: { + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + old = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectNew(ptr); + stack_pointer[-1] = value; + stack_pointer[0] = new; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _LOAD_CONST_UNDER_INLINE_BORROW: { + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + old = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[-1] = value; + stack_pointer[0] = new; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND0(); #ifndef _Py_JIT - current_executor = (_PyExecutorObject*)executor; + assert(current_executor == (_PyExecutorObject*)executor); #endif - assert(((_PyExecutorObject *)executor)->vm_data.valid); + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyExecutor_ClearExit(tstate->jit_exit); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } break; } case _MAKE_WARM: { current_executor->vm_data.warm = true; - if (--tstate->interp->trace_run_counter == 0) { - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); - } break; } @@ -7116,7 +7111,16 @@ } case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + break; + } + + case _HANDLE_PENDING_AND_DEOPT: { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); break; } @@ -7146,4 +7150,106 @@ break; } + case _COLD_EXIT: { + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_BackoffCounter temperature = exit->temperature; + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); + } + else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + GOTO_TIER_ONE(target); + } + break; + } + + case _COLD_DYNAMIC_EXIT: { + _Py_CODEUNIT *target = frame->instr_ptr; + GOTO_TIER_ONE(target); + break; + } + + case _GUARD_IP__PUSH_FRAME: { + #define OFFSET_OF__PUSH_FRAME ((0)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF__PUSH_FRAME; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF__PUSH_FRAME + break; + } + + case _GUARD_IP_YIELD_VALUE: { + #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_YIELD_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_YIELD_VALUE + break; + } + + case _GUARD_IP_RETURN_VALUE: { + #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_RETURN_VALUE + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_RETURN_GENERATOR + break; + } + + /* _TRACE_RECORD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + #undef TIER_TWO diff --git a/Python/fileutils.c b/Python/fileutils.c index 78603d40704..0c1766b8804 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2,6 +2,7 @@ #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime #include "pycore_pystate.h" // _Py_AssertHoldsTstate() +#include "pycore_unicodeobject.h" // _Py_MAX_UNICODE #include "osdefs.h" // SEP #include <stdlib.h> // mbstowcs() @@ -50,9 +51,6 @@ extern int winerror_to_errno(int); int _Py_open_cloexec_works = -1; #endif -// The value must be the same in unicodeobject.c. -#define MAX_UNICODE 0x10ffff - // mbstowcs() and mbrtowc() errors static const size_t DECODE_ERROR = ((size_t)-1); #ifdef HAVE_MBRTOWC @@ -123,7 +121,7 @@ is_valid_wide_char(wchar_t ch) { #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION /* Oracle Solaris doesn't use Unicode code points as wchar_t encoding - for non-Unicode locales, which makes values higher than MAX_UNICODE + for non-Unicode locales, which makes values higher than _Py_MAX_UNICODE possibly valid. */ return 1; #endif @@ -132,7 +130,7 @@ is_valid_wide_char(wchar_t ch) return 0; } #if SIZEOF_WCHAR_T > 2 - if (ch > MAX_UNICODE) { + if (ch > _Py_MAX_UNICODE) { // bpo-35883: Reject characters outside [U+0000; U+10ffff] range. // The glibc mbstowcs() UTF-8 decoder does not respect the RFC 3629, // it creates characters outside the [U+0000; U+10ffff] range: @@ -180,7 +178,7 @@ _Py_mbrtowc(wchar_t *pwc, const char *str, size_t len, mbstate_t *pmbs) #define USE_FORCE_ASCII -extern int _Py_normalize_encoding(const char *, char *, size_t); +extern int _Py_normalize_encoding(const char *, char *, size_t, int); /* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale and POSIX locale. nl_langinfo(CODESET) announces an alias of the @@ -231,7 +229,7 @@ check_force_ascii(void) } char encoding[20]; /* longest name: "iso_646.irv_1991\0" */ - if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) { + if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding), 1)) { goto error; } @@ -2120,7 +2118,6 @@ _Py_wrealpath(const wchar_t *path, wchar_t *resolved_path, size_t resolved_path_len) { char *cpath; - char cresolved_path[MAXPATHLEN]; wchar_t *wresolved_path; char *res; size_t r; @@ -2129,12 +2126,14 @@ _Py_wrealpath(const wchar_t *path, errno = EINVAL; return NULL; } - res = realpath(cpath, cresolved_path); + res = realpath(cpath, NULL); PyMem_RawFree(cpath); if (res == NULL) return NULL; - wresolved_path = Py_DecodeLocale(cresolved_path, &r); + wresolved_path = Py_DecodeLocale(res, &r); + free(res); + if (wresolved_path == NULL) { errno = EINVAL; return NULL; @@ -2784,6 +2783,43 @@ _Py_set_blocking(int fd, int blocking) return -1; } #else /* MS_WINDOWS */ + +// The Windows Games API family doesn't expose GetNamedPipeHandleStateW so attempt +// to load it directly from the Kernel32.dll +#if !defined(MS_WINDOWS_APP) && !defined(MS_WINDOWS_SYSTEM) +BOOL +GetNamedPipeHandleStateW(HANDLE hNamedPipe, LPDWORD lpState, LPDWORD lpCurInstances, LPDWORD lpMaxCollectionCount, + LPDWORD lpCollectDataTimeout, LPWSTR lpUserName, DWORD nMaxUserNameSize) +{ + static int initialized = 0; + typedef BOOL(__stdcall* PGetNamedPipeHandleStateW) ( + HANDLE hNamedPipe, LPDWORD lpState, LPDWORD lpCurInstances, LPDWORD lpMaxCollectionCount, + LPDWORD lpCollectDataTimeout, LPWSTR lpUserName, DWORD nMaxUserNameSize); + static PGetNamedPipeHandleStateW _GetNamedPipeHandleStateW; + + if (initialized == 0) { + HMODULE api = LoadLibraryExW(L"Kernel32.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + if (api) { + _GetNamedPipeHandleStateW = (PGetNamedPipeHandleStateW)GetProcAddress( + api, "GetNamedPipeHandleStateW"); + } + else { + _GetNamedPipeHandleStateW = NULL; + } + initialized = 1; + } + + if (!_GetNamedPipeHandleStateW) { + SetLastError(E_NOINTERFACE); + return FALSE; + } + + return _GetNamedPipeHandleStateW( + hNamedPipe, lpState, lpCurInstances, lpMaxCollectionCount, lpCollectDataTimeout, lpUserName, nMaxUserNameSize + ); +} +#endif /* !MS_WINDOWS_APP && !MS_WINDOWS_SYSTEM */ + int _Py_get_blocking(int fd) { diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 78ef02a911a..a43f8b45b87 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -199,8 +199,10 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) cfg_instr *i = &b->b_instr[off]; i->i_opcode = opcode; i->i_oparg = oparg; - i->i_target = NULL; i->i_loc = loc; + // memory is already zero initialized + assert(i->i_target == NULL); + assert(i->i_except == NULL); return SUCCESS; } @@ -299,26 +301,34 @@ basicblock_returns(const basicblock *b) { } static void -dump_basicblock(const basicblock *b) +dump_basicblock(const basicblock *b, bool highlight) { const char *b_return = basicblock_returns(b) ? "return " : ""; + if (highlight) { + fprintf(stderr, ">>> "); + } fprintf(stderr, "%d: [EH=%d CLD=%d WRM=%d NO_FT=%d %p] used: %d, depth: %d, preds: %d %s\n", b->b_label.id, b->b_except_handler, b->b_cold, b->b_warm, BB_NO_FALLTHROUGH(b), b, b->b_iused, b->b_startdepth, b->b_predecessors, b_return); + int depth = b->b_startdepth; if (b->b_instr) { int i; for (i = 0; i < b->b_iused; i++) { - fprintf(stderr, " [%02d] ", i); + fprintf(stderr, " [%02d] depth: %d ", i, depth); dump_instr(b->b_instr + i); + + int popped = _PyOpcode_num_popped(b->b_instr[i].i_opcode, b->b_instr[i].i_oparg); + int pushed = _PyOpcode_num_pushed(b->b_instr[i].i_opcode, b->b_instr[i].i_oparg); + depth += (pushed - popped); } } } void -_PyCfgBuilder_DumpGraph(const basicblock *entryblock) +_PyCfgBuilder_DumpGraph(const basicblock *entryblock, const basicblock *mark) { for (const basicblock *b = entryblock; b != NULL; b = b->b_next) { - dump_basicblock(b); + dump_basicblock(b, b == mark); } } @@ -1096,6 +1106,7 @@ basicblock_remove_redundant_nops(basicblock *bb) { assert(dest <= bb->b_iused); int num_removed = bb->b_iused - dest; bb->b_iused = dest; + memset(&bb->b_instr[dest], 0, sizeof(cfg_instr) * num_removed); return num_removed; } @@ -1884,6 +1895,10 @@ eval_const_unaryop(PyObject *operand, int opcode, int oparg) result = PyNumber_Negative(operand); break; case UNARY_INVERT: + // XXX: This should be removed once the ~bool depreciation expires. + if (PyBool_Check(operand)) { + return NULL; + } result = PyNumber_Invert(operand); break; case UNARY_NOT: { @@ -2862,8 +2877,11 @@ optimize_load_fast(cfg_builder *g) // how many inputs should be left on the stack. // Opcodes that consume no inputs + case FORMAT_SIMPLE: case GET_ANEXT: + case GET_ITER: case GET_LEN: + case GET_YIELD_FROM_ITER: case IMPORT_FROM: case MATCH_KEYS: case MATCH_MAPPING: @@ -2873,7 +2891,7 @@ optimize_load_fast(cfg_builder *g) int num_pushed = _PyOpcode_num_pushed(opcode, oparg); int net_pushed = num_pushed - num_popped; assert(net_pushed >= 0); - for (int i = 0; i < net_pushed; i++) { + for (int j = 0; j < net_pushed; j++) { PUSH_REF(i, NOT_LOCAL); } break; @@ -2898,6 +2916,16 @@ optimize_load_fast(cfg_builder *g) break; } + case END_SEND: + case SET_FUNCTION_ATTRIBUTE: { + assert(_PyOpcode_num_popped(opcode, oparg) == 2); + assert(_PyOpcode_num_pushed(opcode, oparg) == 1); + ref tos = ref_stack_pop(&refs); + ref_stack_pop(&refs); + PUSH_REF(tos.instr, tos.local); + break; + } + // Opcodes that consume some inputs and push new values case CHECK_EXC_MATCH: { ref_stack_pop(&refs); @@ -2927,6 +2955,14 @@ optimize_load_fast(cfg_builder *g) break; } + case LOAD_SPECIAL: + case PUSH_EXC_INFO: { + ref tos = ref_stack_pop(&refs); + PUSH_REF(i, NOT_LOCAL); + PUSH_REF(tos.instr, tos.local); + break; + } + case SEND: { load_fast_push_block(&sp, instr->i_target, refs.size); ref_stack_pop(&refs); @@ -2957,11 +2993,8 @@ optimize_load_fast(cfg_builder *g) } // Push fallthrough block - cfg_instr *term = basicblock_last_instr(block); - if (term != NULL && block->b_next != NULL && - !(IS_UNCONDITIONAL_JUMP_OPCODE(term->i_opcode) || - IS_SCOPE_EXIT_OPCODE(term->i_opcode))) { - assert(BB_HAS_FALLTHROUGH(block)); + if (BB_HAS_FALLTHROUGH(block)) { + assert(block->b_next != NULL); load_fast_push_block(&sp, block->b_next, refs.size); } @@ -3439,11 +3472,13 @@ convert_pseudo_conditional_jumps(cfg_builder *g) instr->i_opcode = instr->i_opcode == JUMP_IF_FALSE ? POP_JUMP_IF_FALSE : POP_JUMP_IF_TRUE; location loc = instr->i_loc; + basicblock *except = instr->i_except; cfg_instr copy = { .i_opcode = COPY, .i_oparg = 1, .i_loc = loc, .i_target = NULL, + .i_except = except, }; RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &copy)); cfg_instr to_bool = { @@ -3451,6 +3486,7 @@ convert_pseudo_conditional_jumps(cfg_builder *g) .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = except, }; RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &to_bool)); } @@ -3555,8 +3591,19 @@ duplicate_exits_without_lineno(cfg_builder *g) * Also reduces the size of the line number table, * but has no impact on the generated line number events. */ + +static inline void +maybe_propagate_location(basicblock *b, int i, location loc) +{ + assert(b->b_iused > i); + if (b->b_instr[i].i_loc.lineno == NO_LOCATION.lineno) { + b->b_instr[i].i_loc = loc; + } +} + static void -propagate_line_numbers(basicblock *entryblock) { +propagate_line_numbers(basicblock *entryblock) +{ for (basicblock *b = entryblock; b != NULL; b = b->b_next) { cfg_instr *last = basicblock_last_instr(b); if (last == NULL) { @@ -3565,26 +3612,21 @@ propagate_line_numbers(basicblock *entryblock) { location prev_location = NO_LOCATION; for (int i = 0; i < b->b_iused; i++) { - if (b->b_instr[i].i_loc.lineno == NO_LOCATION.lineno) { - b->b_instr[i].i_loc = prev_location; - } - else { - prev_location = b->b_instr[i].i_loc; - } + maybe_propagate_location(b, i, prev_location); + prev_location = b->b_instr[i].i_loc; } if (BB_HAS_FALLTHROUGH(b) && b->b_next->b_predecessors == 1) { if (b->b_next->b_iused > 0) { - if (b->b_next->b_instr[0].i_loc.lineno == NO_LOCATION.lineno) { - b->b_next->b_instr[0].i_loc = prev_location; - } + maybe_propagate_location(b->b_next, 0, prev_location); } } if (is_jump(last)) { basicblock *target = last->i_target; + while (target->b_iused == 0 && target->b_predecessors == 1) { + target = target->b_next; + } if (target->b_predecessors == 1) { - if (target->b_instr[0].i_loc.lineno == NO_LOCATION.lineno) { - target->b_instr[0].i_loc = prev_location; - } + maybe_propagate_location(target, 0, prev_location); } } } @@ -3693,6 +3735,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &make_gen)); cfg_instr pop_top = { @@ -3700,6 +3743,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 1, &pop_top)); } @@ -3730,6 +3774,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = oldindex, .i_loc = NO_LOCATION, .i_target = NULL, + .i_except = NULL, }; if (basicblock_insert_instruction(entryblock, ncellsused, &make_cell) < 0) { PyMem_RawFree(sorted); @@ -3746,6 +3791,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = nfreevars, .i_loc = NO_LOCATION, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &copy_frees)); } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index ec4566bd4f8..3de587c0423 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -1,8 +1,7 @@ /* Python interpreter main program for frozen scripts */ #include "Python.h" -#include "pycore_pystate.h" // _Py_GetConfig() -#include "pycore_runtime.h" // _PyRuntime_Initialize() +#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #ifdef HAVE_UNISTD_H # include <unistd.h> // isatty() @@ -20,11 +19,6 @@ extern int PyInitFrozenExtensions(void); int Py_FrozenMain(int argc, char **argv) { - PyStatus status = _PyRuntime_Initialize(); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - PyConfig config; PyConfig_InitPythonConfig(&config); // Suppress errors from getpath.c @@ -32,7 +26,7 @@ Py_FrozenMain(int argc, char **argv) // Don't parse command line options like -E config.parse_argv = 0; - status = PyConfig_SetBytesArgv(&config, argc, argv); + PyStatus status = PyConfig_SetBytesArgv(&config, argc, argv); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); Py_ExitStatusException(status); @@ -64,7 +58,12 @@ Py_FrozenMain(int argc, char **argv) PyWinFreeze_ExeInit(); #endif - if (_Py_GetConfig()->verbose) { + int verbose; + if (PyConfig_GetInt("verbose", &verbose) < 0) { + verbose = 0; + PyErr_Clear(); + } + if (verbose) { fprintf(stderr, "Python %s\n%s\n", Py_GetVersion(), Py_GetCopyright()); } diff --git a/Python/gc.c b/Python/gc.c index 7b0e6d6e803..d067a6144b0 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1,6 +1,6 @@ // This implements the reference cycle garbage collector. // The Python module interface to the collector is in gcmodule.c. -// See https://devguide.python.org/internals/garbage-collector/ +// See InternalDocs/garbage_collector.md for more infromation. #include "Python.h" #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() @@ -483,11 +483,12 @@ validate_consistent_old_space(PyGC_Head *head) /* Set all gc_refs = ob_refcnt. After this, gc_refs is > 0 and * PREV_MASK_COLLECTING bit is set for all objects in containers. */ -static void +static Py_ssize_t update_refs(PyGC_Head *containers) { PyGC_Head *next; PyGC_Head *gc = GC_NEXT(containers); + Py_ssize_t candidates = 0; while (gc != containers) { next = GC_NEXT(gc); @@ -519,7 +520,9 @@ update_refs(PyGC_Head *containers) */ _PyObject_ASSERT(op, gc_get_refs(gc) != 0); gc = next; + candidates++; } + return candidates; } /* A traversal callback for subtract_refs. */ @@ -570,8 +573,7 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) } /* Subtract internal references from gc_refs. After this, gc_refs is >= 0 - * for all objects in containers, and is GC_REACHABLE for all tracked gc - * objects not in containers. The ones with gc_refs > 0 are directly + * for all objects in containers. The ones with gc_refs > 0 are directly * reachable from outside containers, and so can't be collected. */ static void @@ -858,58 +860,65 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) } } -/* Clear all weakrefs to unreachable objects, and if such a weakref has a - * callback, invoke it if necessary. Note that it's possible for such - * weakrefs to be outside the unreachable set -- indeed, those are precisely - * the weakrefs whose callbacks must be invoked. See gc_weakref.txt for - * overview & some details. Some weakrefs with callbacks may be reclaimed - * directly by this routine; the number reclaimed is the return value. Other - * weakrefs with callbacks may be moved into the `old` generation. Objects - * moved into `old` have gc_refs set to GC_REACHABLE; the objects remaining in - * unreachable are left at GC_TENTATIVELY_UNREACHABLE. When this returns, - * no object in `unreachable` is weakly referenced anymore. +/* Handle weakref callbacks. Note that it's possible for such weakrefs to be + * outside the unreachable set -- indeed, those are precisely the weakrefs + * whose callbacks must be invoked. See gc_weakref.txt for overview & some + * details. + * + * The clearing of weakrefs is suble and must be done carefully, as there was + * previous bugs related to this. First, weakrefs to the unreachable set of + * objects must be cleared before we start calling `tp_clear`. If we don't, + * those weakrefs can reveal unreachable objects to Python-level code and that + * is not safe. Many objects are not usable after `tp_clear` has been called + * and could even cause crashes if accessed (see bpo-38006 and gh-91636 as + * example bugs). + * + * Weakrefs with callbacks always need to be cleared before executing the + * callback. That's because sometimes a callback will call the ref object, + * to check if the reference is actually dead (KeyedRef does this, for + * example). We want to indicate that it is dead, even though it is possible + * a finalizer might resurrect it. Clearing also prevents the callback from + * executing more than once. + * + * Since Python 2.3, all weakrefs to cyclic garbage have been cleared *before* + * calling finalizers. However, since tp_subclasses started being necessary + * to invalidate caches (e.g. by PyType_Modified), that clearing has created + * a bug. If the weakref to the subclass is cleared before a finalizer is + * called, the cache may not be correctly invalidated. That can lead to + * segfaults since the caches can refer to deallocated objects (GH-135552 + * is an example). Now, we delay the clear of weakrefs without callbacks + * until *after* finalizers have been executed. That means weakrefs without + * callbacks are still usable while finalizers are being executed. + * + * The weakrefs that are inside the unreachable set must also be cleared. + * The reason this is required is not immediately obvious. If the weakref + * refers to an object outside of the unreachable set, that object might go + * away when we start clearing objects. Normally, the object should also be + * part of the unreachable set but that's not true in the case of incomplete + * or missing `tp_traverse` methods. When that object goes away, the callback + * for weakref can be executed and that could reveal unreachable objects to + * Python-level code. See bpo-38006 as an example bug. */ static int -handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old) { PyGC_Head *gc; - PyObject *op; /* generally FROM_GC(gc) */ - PyWeakReference *wr; /* generally a cast of op */ PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */ PyGC_Head *next; int num_freed = 0; gc_list_init(&wrcb_to_call); - /* Clear all weakrefs to the objects in unreachable. If such a weakref - * also has a callback, move it into `wrcb_to_call` if the callback - * needs to be invoked. Note that we cannot invoke any callbacks until - * all weakrefs to unreachable objects are cleared, lest the callback - * resurrect an unreachable object via a still-active weakref. We - * make another pass over wrcb_to_call, invoking callbacks, after this - * pass completes. + /* Find all weakrefs with callbacks and move into `wrcb_to_call` if the + * callback needs to be invoked. We make another pass over wrcb_to_call, + * invoking callbacks, after this pass completes. */ for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { PyWeakReference **wrlist; - op = FROM_GC(gc); + PyObject *op = FROM_GC(gc); next = GC_NEXT(gc); - if (PyWeakref_Check(op)) { - /* A weakref inside the unreachable set must be cleared. If we - * allow its callback to execute inside delete_garbage(), it - * could expose objects that have tp_clear already called on - * them. Or, it could resurrect unreachable objects. One way - * this can happen is if some container objects do not implement - * tp_traverse. Then, wr_object can be outside the unreachable - * set but can be deallocated as a result of breaking the - * reference cycle. If we don't clear the weakref, the callback - * will run and potentially cause a crash. See bpo-38006 for - * one example. - */ - _PyWeakref_ClearRef((PyWeakReference *)op); - } - if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { continue; } @@ -921,25 +930,29 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) */ wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); - /* `op` may have some weakrefs. March over the list, clear - * all the weakrefs, and move the weakrefs with callbacks - * that must be called into wrcb_to_call. + /* `op` may have some weakrefs. March over the list and move the + * weakrefs with callbacks that must be called into wrcb_to_call. */ - for (wr = *wrlist; wr != NULL; wr = *wrlist) { - PyGC_Head *wrasgc; /* AS_GC(wr) */ + PyWeakReference *next_wr; + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = next_wr) { + // Get the next list element to get iterator progress if we omit + // clearing of the weakref (because _PyWeakref_ClearRef changes + // next pointer in the wrlist). + next_wr = wr->wr_next; - /* _PyWeakref_ClearRef clears the weakref but leaves - * the callback pointer intact. Obscure: it also - * changes *wrlist. - */ - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); - _PyWeakref_ClearRef(wr); - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); if (wr->wr_callback == NULL) { /* no callback */ continue; } + // Clear the weakref. See the comments above this function for + // reasons why we need to clear weakrefs that have callbacks. + // Note that _PyWeakref_ClearRef clears the weakref but leaves the + // callback pointer intact. Obscure: it also changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + /* Headache time. `op` is going away, and is weakly referenced by * `wr`, which has a callback. Should the callback be invoked? If wr * is also trash, no: @@ -955,10 +968,10 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) * outside the current generation, CT may be reachable from the * callback. Then the callback could resurrect insane objects. * - * Since the callback is never needed and may be unsafe in this case, - * wr is simply left in the unreachable set. Note that because we - * already called _PyWeakref_ClearRef(wr), its callback will never - * trigger. + * Since the callback is never needed and may be unsafe in this + * case, wr is simply left in the unreachable set. Note that + * clear_weakrefs() will ensure its callback will not trigger + * inside delete_garbage(). * * OTOH, if wr isn't part of CT, we should invoke the callback: the * weakref outlived the trash. Note that since wr isn't CT in this @@ -969,8 +982,6 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) * is moved to wrcb_to_call in this case. */ if (gc_is_collecting(AS_GC((PyObject *)wr))) { - /* it should already have been cleared above */ - _PyObject_ASSERT((PyObject*)wr, wr->wr_object == Py_None); continue; } @@ -980,7 +991,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) Py_INCREF(wr); /* Move wr to wrcb_to_call, for the next pass. */ - wrasgc = AS_GC((PyObject *)wr); + PyGC_Head *wrasgc = AS_GC((PyObject *)wr); // wrasgc is reachable, but next isn't, so they can't be the same _PyObject_ASSERT((PyObject *)wr, wrasgc != next); gc_list_move(wrasgc, &wrcb_to_call); @@ -996,9 +1007,9 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) PyObject *callback; gc = (PyGC_Head*)wrcb_to_call._gc_next; - op = FROM_GC(gc); + PyObject *op = FROM_GC(gc); _PyObject_ASSERT(op, PyWeakref_Check(op)); - wr = (PyWeakReference *)op; + PyWeakReference *wr = (PyWeakReference *)op; callback = wr->wr_callback; _PyObject_ASSERT(op, callback != NULL); @@ -1037,6 +1048,56 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) return num_freed; } +/* Clear all weakrefs to unreachable objects. When this returns, no object in + * `unreachable` is weakly referenced anymore. See the comments above + * handle_weakref_callbacks() for why these weakrefs need to be cleared. + */ +static void +clear_weakrefs(PyGC_Head *unreachable) +{ + PyGC_Head *gc; + PyGC_Head *next; + + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + PyWeakReference **wrlist; + + PyObject *op = FROM_GC(gc); + next = GC_NEXT(gc); + + if (PyWeakref_Check(op)) { + /* A weakref inside the unreachable set is always cleared. See + * the comments above handle_weakref_callbacks() for why these + * must be cleared. + */ + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + /* It supports weakrefs. Does it have any? + * + * This is never triggered for static types so we can avoid the + * (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + */ + wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + /* `op` may have some weakrefs. March over the list, clear + * all the weakrefs. + */ + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + /* _PyWeakref_ClearRef clears the weakref but leaves + * the callback pointer intact. Obscure: it also + * changes *wrlist. + */ + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + } + } +} + static void debug_cycle(const char *msg, PyObject *op) { @@ -1182,7 +1243,7 @@ flag set but it does not clear it to skip unnecessary iteration. Before the flag is cleared (for example, by using 'clear_unreachable_mask' function or by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal list and we can not use most gc_list_* functions for it. */ -static inline void +static inline Py_ssize_t deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { validate_list(base, collecting_clear_unreachable_clear); /* Using ob_refcnt and gc_refs, calculate which objects in the @@ -1190,7 +1251,7 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { * refcount greater than 0 when all the references within the * set are taken into account). */ - update_refs(base); // gc_prev is used for gc_refs + Py_ssize_t candidates = update_refs(base); // gc_prev is used for gc_refs subtract_refs(base); /* Leave everything reachable from outside base in base, and move @@ -1231,6 +1292,7 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { move_unreachable(base, unreachable); // gc_prev is pointer again validate_list(base, collecting_clear_unreachable_clear); validate_list(unreachable, collecting_set_unreachable_set); + return candidates; } /* Handle objects that may have resurrected after a call to 'finalize_garbage', moving @@ -1305,8 +1367,10 @@ gc_list_set_space(PyGC_Head *list, int space) static void add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats) { + gcstate->generation_stats[gen].duration += stats->duration; gcstate->generation_stats[gen].collected += stats->collected; gcstate->generation_stats[gen].uncollectable += stats->uncollectable; + gcstate->generation_stats[gen].candidates += stats->candidates; gcstate->generation_stats[gen].collections += 1; } @@ -1320,15 +1384,6 @@ gc_collect_young(PyThreadState *tstate, PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; untrack_tuples(young); GC_STAT_ADD(0, collections, 1); -#ifdef Py_STATS - { - Py_ssize_t count = 0; - PyGC_Head *gc; - for (gc = GC_NEXT(young); gc != young; gc = GC_NEXT(gc)) { - count++; - } - } -#endif PyGC_Head survivors; gc_list_init(&survivors); @@ -1338,7 +1393,6 @@ gc_collect_young(PyThreadState *tstate, validate_spaces(gcstate); gcstate->young.count = 0; gcstate->old[gcstate->visited_space].count++; - add_stats(gcstate, 0, stats); validate_spaces(gcstate); } @@ -1394,7 +1448,7 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat assert(_PyObject_GC_IS_TRACKED(op)); if (_Py_IsImmortal(op)) { PyGC_Head *next = GC_NEXT(gc); - gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc_list_move(gc, &gcstate->permanent_generation.head); gc = next; continue; } @@ -1590,7 +1644,7 @@ assess_work_to_do(GCState *gcstate) scale_factor = 2; } intptr_t new_objects = gcstate->young.count; - intptr_t max_heap_fraction = new_objects*3/2; + intptr_t max_heap_fraction = new_objects*2; intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; if (heap_fraction > max_heap_fraction) { heap_fraction = max_heap_fraction; @@ -1605,11 +1659,15 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) GC_STAT_ADD(1, collections, 1); GCState *gcstate = &tstate->interp->gc; gcstate->work_to_do += assess_work_to_do(gcstate); + if (gcstate->work_to_do < 0) { + return; + } untrack_tuples(&gcstate->young.head); if (gcstate->phase == GC_PHASE_MARK) { Py_ssize_t objects_marked = mark_at_start(tstate); GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); gcstate->work_to_do -= objects_marked; + stats->candidates += objects_marked; validate_spaces(gcstate); return; } @@ -1647,10 +1705,8 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) gc_collect_region(tstate, &increment, &survivors, stats); gc_list_merge(&survivors, visited); assert(gc_list_is_empty(&increment)); - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; gcstate->work_to_do -= increment_size; - add_stats(gcstate, 1, stats); if (gc_list_is_empty(not_visited)) { completed_scavenge(gcstate); } @@ -1685,7 +1741,6 @@ gc_collect_full(PyThreadState *tstate, completed_scavenge(gcstate); _PyGC_ClearAllFreeLists(tstate->interp); validate_spaces(gcstate); - add_stats(gcstate, 2, stats); } /* This is the main function. Read this to understand how the @@ -1705,15 +1760,16 @@ gc_collect_region(PyThreadState *tstate, assert(!_PyErr_Occurred(tstate)); gc_list_init(&unreachable); - deduce_unreachable(from, &unreachable); + stats->candidates = deduce_unreachable(from, &unreachable); validate_consistent_old_space(from); untrack_tuples(from); + + /* Move reachable objects to next generation. */ validate_consistent_old_space(to); if (from != to) { gc_list_merge(from, to); } validate_consistent_old_space(to); - /* Move reachable objects to next generation. */ /* All objects in unreachable are trash, but objects reachable from * legacy finalizers (e.g. tp_del) can't safely be deleted. @@ -1736,8 +1792,8 @@ gc_collect_region(PyThreadState *tstate, } } - /* Clear weakrefs and invoke callbacks as necessary. */ - stats->collected += handle_weakrefs(&unreachable, to); + /* Invoke weakref callbacks as necessary. */ + stats->collected += handle_weakref_callbacks(&unreachable, to); gc_list_validate_space(to, gcstate->visited_space); validate_list(to, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); @@ -1751,6 +1807,11 @@ gc_collect_region(PyThreadState *tstate, gc_list_init(&final_unreachable); handle_resurrected_objects(&unreachable, &final_unreachable, to); + /* Clear weakrefs to objects in the unreachable set. See the comments + * above handle_weakref_callbacks() for details. + */ + clear_weakrefs(&final_unreachable); + /* Call tp_clear on objects in the final_unreachable set. This will cause * the reference cycles to be broken. It may also cause some objects * in finalizers to be freed. @@ -1763,7 +1824,7 @@ gc_collect_region(PyThreadState *tstate, Py_ssize_t n = 0; for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { n++; - if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) debug_cycle("uncollectable", FROM_GC(gc)); } stats->uncollectable = n; @@ -1789,10 +1850,12 @@ do_gc_callback(GCState *gcstate, const char *phase, assert(PyList_CheckExact(gcstate->callbacks)); PyObject *info = NULL; if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", + info = Py_BuildValue("{sisnsnsnsd}", "generation", generation, "collected", stats->collected, - "uncollectable", stats->uncollectable); + "uncollectable", stats->uncollectable, + "candidates", stats->candidates, + "duration", stats->duration); if (info == NULL) { PyErr_FormatUnraisable("Exception ignored while invoking gc callbacks"); return; @@ -2017,6 +2080,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) // Don't start a garbage collection if one is already in progress. return 0; } + gcstate->frame = tstate->current_frame; struct gc_collection_stats stats = { 0 }; if (reason != _Py_GC_REASON_SHUTDOWN) { @@ -2029,6 +2093,8 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (PyDTrace_GC_START_ENABLED()) { PyDTrace_GC_START(generation); } + PyTime_t start, stop; + (void)PyTime_PerfCounterRaw(&start); PyObject *exc = _PyErr_GetRaisedException(tstate); switch(generation) { case 0: @@ -2043,6 +2109,9 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) default: Py_UNREACHABLE(); } + (void)PyTime_PerfCounterRaw(&stop); + stats.duration = PyTime_AsSecondsDouble(stop - start); + add_stats(gcstate, generation, &stats); if (PyDTrace_GC_DONE_ENABLED()) { PyDTrace_GC_DONE(stats.uncollectable + stats.collected); } @@ -2052,14 +2121,24 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) _PyErr_SetRaisedException(tstate, exc); GC_STAT_ADD(generation, objects_collected, stats.collected); #ifdef Py_STATS - if (_Py_stats) { + PyStats *s = _PyStats_GET(); + if (s) { GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; + s->object_stats.object_visits); + s->object_stats.object_visits = 0; } #endif validate_spaces(gcstate); + gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr( + "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", + stats.collected + stats.uncollectable, stats.uncollectable, stats.duration + ); + } + return stats.uncollectable + stats.collected; } @@ -2164,7 +2243,7 @@ _PyGC_Fini(PyInterpreterState *interp) void _PyGC_Dump(PyGC_Head *g) { - _PyObject_Dump(FROM_GC(g)); + PyUnstable_Object_Dump(FROM_GC(g)); } @@ -2231,21 +2310,11 @@ _Py_ScheduleGC(PyThreadState *tstate) } void -_PyObject_GC_Link(PyObject *op) +_Py_TriggerGC(struct _gc_runtime_state *gcstate) { - PyGC_Head *gc = AS_GC(op); - // gc must be correctly aligned - _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - gc->_gc_next = 0; - gc->_gc_prev = 0; - gcstate->young.count++; /* number of allocated GC objects */ - gcstate->heap_size++; - if (gcstate->young.count > gcstate->young.threshold && - gcstate->enabled && - gcstate->young.threshold && + if (gcstate->enabled && + gcstate->young.threshold != 0 && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { @@ -2253,6 +2322,17 @@ _PyObject_GC_Link(PyObject *op) } } +void +_PyObject_GC_Link(PyObject *op) +{ + PyGC_Head *gc = AS_GC(op); + // gc must be correctly aligned + _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); + gc->_gc_next = 0; + gc->_gc_prev = 0; + +} + void _Py_RunGC(PyThreadState *tstate) { @@ -2359,6 +2439,11 @@ PyObject_GC_Del(void *op) PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); + GCState *gcstate = get_gc_state(); + if (gcstate->young.count > 0) { + gcstate->young.count--; + } + gcstate->heap_size--; #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -2372,11 +2457,6 @@ PyObject_GC_Del(void *op) PyErr_SetRaisedException(exc); #endif } - GCState *gcstate = get_gc_state(); - if (gcstate->young.count > 0) { - gcstate->young.count--; - } - gcstate->heap_size--; PyObject_Free(((char *)op)-presize); } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 757e9cb3227..e672e870db2 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -100,6 +100,7 @@ struct collection_state { int skip_deferred_objects; Py_ssize_t collected; Py_ssize_t uncollectable; + Py_ssize_t candidates; Py_ssize_t long_lived_total; struct worklist unreachable; struct worklist legacy_finalizers; @@ -675,10 +676,11 @@ gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, PyObject **end) else { ss->capacity *= 2; } - ss->stack = (gc_span_t *)PyMem_Realloc(ss->stack, ss->capacity * sizeof(gc_span_t)); - if (ss->stack == NULL) { + gc_span_t *new_stack = (gc_span_t *)PyMem_Realloc(ss->stack, ss->capacity * sizeof(gc_span_t)); + if (new_stack == NULL) { return -1; } + ss->stack = new_stack; } assert(end > start); ss->stack[ss->size].start = start; @@ -974,15 +976,12 @@ static bool update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) { + struct collection_state *state = (struct collection_state *)args; PyObject *op = op_from_block(block, args, false); if (op == NULL) { return true; } - if (gc_is_alive(op)) { - return true; - } - // Exclude immortal objects from garbage collection if (_Py_IsImmortal(op)) { op->ob_tid = 0; @@ -990,6 +989,11 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area, gc_clear_unreachable(op); return true; } + // Marked objects count as candidates, immortals don't: + state->candidates++; + if (gc_is_alive(op)) { + return true; + } Py_ssize_t refcount = Py_REFCNT(op); if (_PyObject_HasDeferredRefcount(op)) { @@ -1073,6 +1077,14 @@ validate_refcounts(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } + // This assert mirrors the one in Python/gc.c:update_refs(). There must be + // no tracked objects with a reference count of 0 when the cyclic + // collector starts. If there is, then the collector will double dealloc + // the object. The likely cause for hitting this is a faulty .tp_dealloc. + // Also see the comment in `update_refs()`. + _PyObject_ASSERT_WITH_MSG(op, Py_REFCNT(op) > 0, + "tracked objects must have a reference count > 0"); + _PyObject_ASSERT_WITH_MSG(op, !gc_is_unreachable(op), "object should not be marked as unreachable yet"); @@ -1422,13 +1434,6 @@ static int deduce_unreachable_heap(PyInterpreterState *interp, struct collection_state *state) { - -#ifdef GC_DEBUG - // Check that all objects are marked as unreachable and that the computed - // reference count difference (stored in `ob_tid`) is non-negative. - gc_visit_heaps(interp, &validate_refcounts, &state->base); -#endif - // Identify objects that are directly reachable from outside the GC heap // by computing the difference between the refcount and the number of // incoming references. @@ -1491,22 +1496,49 @@ move_legacy_finalizer_reachable(struct collection_state *state) return 0; } -// Clear all weakrefs to unreachable objects. Weakrefs with callbacks are -// enqueued in `wrcb_to_call`, but not invoked yet. +// Weakrefs with callbacks are enqueued in `wrcb_to_call`, but not invoked +// yet. Note that it's possible for such weakrefs to be outside the +// unreachable set -- indeed, those are precisely the weakrefs whose callbacks +// must be invoked. See gc_weakref.txt for overview & some details. +// +// The clearing of weakrefs is suble and must be done carefully, as there was +// previous bugs related to this. First, weakrefs to the unreachable set of +// objects must be cleared before we start calling `tp_clear`. If we don't, +// those weakrefs can reveal unreachable objects to Python-level code and that +// is not safe. Many objects are not usable after `tp_clear` has been called +// and could even cause crashes if accessed (see bpo-38006 and gh-91636 as +// example bugs). +// +// Weakrefs with callbacks always need to be cleared before executing the +// callback. That's because sometimes a callback will call the ref object, +// to check if the reference is actually dead (KeyedRef does this, for +// example). We want to indicate that it is dead, even though it is possible +// a finalizer might resurrect it. Clearing also prevents the callback from +// executing more than once. +// +// Since Python 2.3, all weakrefs to cyclic garbage have been cleared *before* +// calling finalizers. However, since tp_subclasses started being necessary +// to invalidate caches (e.g. by PyType_Modified), that clearing has created +// a bug. If the weakref to the subclass is cleared before a finalizer is +// called, the cache may not be correctly invalidated. That can lead to +// segfaults since the caches can refer to deallocated objects (GH-91636 +// is an example). Now, we delay the clear of weakrefs without callbacks +// until *after* finalizers have been executed. That means weakrefs without +// callbacks are still usable while finalizers are being executed. +// +// The weakrefs that are inside the unreachable set must also be cleared. +// The reason this is required is not immediately obvious. If the weakref +// refers to an object outside of the unreachable set, that object might go +// away when we start clearing objects. Normally, the object should also be +// part of the unreachable set but that's not true in the case of incomplete +// or missing `tp_traverse` methods. When that object goes away, the callback +// for weakref can be executed and that could reveal unreachable objects to +// Python-level code. See bpo-38006 as an example bug. static void -clear_weakrefs(struct collection_state *state) +find_weakref_callbacks(struct collection_state *state) { PyObject *op; WORKSTACK_FOR_EACH(&state->unreachable, op) { - if (PyWeakref_Check(op)) { - // Clear weakrefs that are themselves unreachable to ensure their - // callbacks will not be executed later from a `tp_clear()` - // inside delete_garbage(). That would be unsafe: it could - // resurrect a dead object or access a an already cleared object. - // See bpo-38006 for one example. - _PyWeakref_ClearRef((PyWeakReference *)op); - } - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { continue; } @@ -1518,13 +1550,22 @@ clear_weakrefs(struct collection_state *state) // `op` may have some weakrefs. March over the list, clear // all the weakrefs, and enqueue the weakrefs with callbacks // that must be called into wrcb_to_call. - for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { - // _PyWeakref_ClearRef clears the weakref but leaves - // the callback pointer intact. Obscure: it also - // changes *wrlist. - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); - _PyWeakref_ClearRef(wr); - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + PyWeakReference *next_wr; + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = next_wr) { + // Get the next list element to get iterator progress if we omit + // clearing of the weakref (because _PyWeakref_ClearRef changes + // next pointer in the wrlist). + next_wr = wr->wr_next; + + // Weakrefs with callbacks always need to be cleared before + // executing the callback. + if (wr->wr_callback != NULL) { + // _PyWeakref_ClearRef clears the weakref but leaves the + // callback pointer intact. Obscure: it also changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + } // We do not invoke callbacks for weakrefs that are themselves // unreachable. This is partly for historical reasons: weakrefs @@ -1545,6 +1586,39 @@ clear_weakrefs(struct collection_state *state) } } +// Clear weakrefs to objects in the unreachable set. See comments +// above find_weakref_callbacks() for why this clearing is required. +static void +clear_weakrefs(struct collection_state *state) +{ + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (PyWeakref_Check(op)) { + // Clear weakrefs that are themselves unreachable. + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + // NOTE: This is never triggered for static types so we can avoid the + // (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + PyWeakReference **wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + // `op` may have some weakrefs. March over the list, clear + // all the weakrefs. + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + // _PyWeakref_ClearRef clears the weakref but leaves + // the callback pointer intact. Obscure: it also + // changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + } + } +} + static void call_weakref_callbacks(struct collection_state *state) { @@ -1840,7 +1914,8 @@ handle_resurrected_objects(struct collection_state *state) static void invoke_gc_callback(PyThreadState *tstate, const char *phase, int generation, Py_ssize_t collected, - Py_ssize_t uncollectable) + Py_ssize_t uncollectable, Py_ssize_t candidates, + double duration) { assert(!_PyErr_Occurred(tstate)); @@ -1854,10 +1929,12 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, assert(PyList_CheckExact(gcstate->callbacks)); PyObject *info = NULL; if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", + info = Py_BuildValue("{sisnsnsnsd}", "generation", generation, "collected", collected, - "uncollectable", uncollectable); + "uncollectable", uncollectable, + "candidates", candidates, + "duration", duration); if (info == NULL) { PyErr_FormatUnraisable("Exception ignored while " "invoking gc callbacks"); @@ -1927,8 +2004,7 @@ get_process_mem_usage(void) } #elif __linux__ - // Linux, use smaps_rollup (Kernel >= 4.4) for RSS + Swap - FILE* fp = fopen("/proc/self/smaps_rollup", "r"); + FILE* fp = fopen("/proc/self/status", "r"); if (fp == NULL) { return -1; } @@ -1938,11 +2014,11 @@ get_process_mem_usage(void) long long swap_kb = -1; while (fgets(line_buffer, sizeof(line_buffer), fp) != NULL) { - if (rss_kb == -1 && strncmp(line_buffer, "Rss:", 4) == 0) { - sscanf(line_buffer + 4, "%lld", &rss_kb); + if (rss_kb == -1 && strncmp(line_buffer, "VmRSS:", 6) == 0) { + sscanf(line_buffer + 6, "%lld", &rss_kb); } - else if (swap_kb == -1 && strncmp(line_buffer, "Swap:", 5) == 0) { - sscanf(line_buffer + 5, "%lld", &swap_kb); + else if (swap_kb == -1 && strncmp(line_buffer, "VmSwap:", 7) == 0) { + sscanf(line_buffer + 7, "%lld", &swap_kb); } if (rss_kb != -1 && swap_kb != -1) { break; // Found both @@ -2063,7 +2139,7 @@ gc_should_collect_mem_usage(GCState *gcstate) // 70,000 new container objects. return true; } - Py_ssize_t last_mem = gcstate->last_mem; + Py_ssize_t last_mem = _Py_atomic_load_ssize_relaxed(&gcstate->last_mem); Py_ssize_t mem_threshold = Py_MAX(last_mem / 10, 128); if ((mem - last_mem) > mem_threshold) { // The process memory usage has increased too much, do a collection. @@ -2134,7 +2210,19 @@ record_deallocation(PyThreadState *tstate) gc->alloc_count--; if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); + int new_count; + do { + if (count == 0) { + break; + } + new_count = count + (int)gc->alloc_count; + if (new_count < 0) { + new_count = 0; + } + } while (!_Py_atomic_compare_exchange_int(&gcstate->young.count, + &count, + new_count)); gc->alloc_count = 0; } } @@ -2155,6 +2243,13 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, state->gcstate->old[i-1].count = 0; } +#ifdef GC_DEBUG + // Before we start, check that the heap is in a good condition. There must + // be no objects with a zero reference count. And `ob_tid` must only have a + // thread if the refcount is unmerged. + gc_visit_heaps(interp, &validate_refcounts, &state->base); +#endif + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; @@ -2211,8 +2306,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Record the number of live GC objects interp->gc.long_lived_total = state->long_lived_total; - // Clear weakrefs and enqueue callbacks (but do not call them). - clear_weakrefs(state); + // Find weakref callbacks we will honor (but do not call them). + find_weakref_callbacks(state); _PyEval_StartTheWorld(interp); // Deallocate any object from the refcount merge step @@ -2223,11 +2318,14 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, call_weakref_callbacks(state); finalize_garbage(state); - // Handle any objects that may have resurrected after the finalization. _PyEval_StopTheWorld(interp); + // Handle any objects that may have resurrected after the finalization. err = handle_resurrected_objects(state); // Clear free lists in all threads _PyGC_ClearAllFreeLists(interp); + if (err == 0) { + clear_weakrefs(state); + } _PyEval_StartTheWorld(interp); if (err < 0) { @@ -2246,7 +2344,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Store the current memory usage, can be smaller now if breaking cycles // freed some memory. - state->gcstate->last_mem = get_process_mem_usage(); + Py_ssize_t last_mem = get_process_mem_usage(); + _Py_atomic_store_ssize_relaxed(&state->gcstate->last_mem, last_mem); // Append objects with legacy finalizers to the "gc.garbage" list. handle_legacy_finalizers(state); @@ -2259,7 +2358,6 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) { Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ - PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ GCState *gcstate = &tstate->interp->gc; // gc_collect_main() must not be called before _PyGC_Init @@ -2278,30 +2376,32 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) _Py_atomic_store_int(&gcstate->collecting, 0); return 0; } + gcstate->frame = tstate->current_frame; assert(generation >= 0 && generation < NUM_GENERATIONS); #ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; + PyStats *s = _PyStats_GET(); + if (s) { + s->object_stats.object_visits = 0; } #endif GC_STAT_ADD(generation, collections, 1); if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(tstate, "start", generation, 0, 0); + invoke_gc_callback(tstate, "start", generation, 0, 0, 0, 0.0); } if (gcstate->debug & _PyGC_DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - // ignore error: don't interrupt the GC if reading the clock fails - (void)PyTime_PerfCounterRaw(&t1); } if (PyDTrace_GC_START_ENABLED()) { PyDTrace_GC_START(generation); } + PyTime_t start, stop; + (void)PyTime_PerfCounterRaw(&start); PyInterpreterState *interp = tstate->interp; @@ -2316,13 +2416,13 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) m = state.collected; n = state.uncollectable; + (void)PyTime_PerfCounterRaw(&stop); + double duration = PyTime_AsSecondsDouble(stop - start); + if (gcstate->debug & _PyGC_DEBUG_STATS) { - PyTime_t t2; - (void)PyTime_PerfCounterRaw(&t2); - double d = PyTime_AsSecondsDouble(t2 - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", - n+m, n, d); + n+m, n, duration); } // Clear the current thread's free-list again. @@ -2343,13 +2443,18 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) stats->collections++; stats->collected += m; stats->uncollectable += n; + stats->duration += duration; + stats->candidates += state.candidates; GC_STAT_ADD(generation, objects_collected, m); #ifdef Py_STATS - if (_Py_stats) { - GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; + { + PyStats *s = _PyStats_GET(); + if (s) { + GC_STAT_ADD(generation, object_visits, + s->object_stats.object_visits); + s->object_stats.object_visits = 0; + } } #endif @@ -2358,10 +2463,11 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(tstate, "stop", generation, m, n); + invoke_gc_callback(tstate, "stop", generation, m, n, state.candidates, duration); } assert(!_PyErr_Occurred(tstate)); + gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); return n + m; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7e8b05b747e..68d73cccec4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,18 +8,18 @@ #endif #define TIER_ONE 1 -#if !Py_TAIL_CALL_INTERP +#if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif { -#endif /* Py_TAIL_CALL_INTERP */ +#endif /* _Py_TAIL_CALL_INTERP */ /* BEGIN INSTRUCTIONS */ TARGET(BINARY_OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; (void)(opcode); #endif @@ -76,13 +76,13 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); #endif @@ -135,12 +135,12 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_INT; (void)(opcode); #endif @@ -158,7 +158,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -168,7 +168,7 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -182,25 +182,25 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); #endif @@ -255,12 +255,12 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_EXTEND) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_EXTEND; (void)(opcode); #endif @@ -314,17 +314,17 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); #endif @@ -371,7 +371,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); @@ -386,7 +386,7 @@ PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyObject *right_o = PyStackRef_AsPyObjectSteal(right); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -407,7 +407,7 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); #endif @@ -460,12 +460,12 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); #endif @@ -483,7 +483,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -493,7 +493,7 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -507,25 +507,25 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBSCR_DICT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); #endif @@ -578,7 +578,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (rc <= 0) { JUMP_TO_LABEL(error); } @@ -586,12 +586,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBSCR_GETITEM) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); #endif @@ -604,7 +604,7 @@ _PyStackRef container; _PyStackRef getitem; _PyStackRef sub; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 5 cache entries */ // _CHECK_PEP_523 { @@ -650,19 +650,20 @@ // _BINARY_OP_SUBSCR_INIT_CALL { sub = stack_pointer[-1]; - new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - new_frame->localsplus[0] = container; - new_frame->localsplus[1] = sub; - frame->return_offset = 6 ; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -674,7 +675,7 @@ } TARGET(BINARY_OP_SUBSCR_LIST_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); #endif @@ -693,7 +694,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -758,13 +759,13 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } DISPATCH(); } TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_LIST_SLICE; (void)(opcode); #endif @@ -823,7 +824,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -831,12 +832,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBSCR_STR_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); #endif @@ -855,7 +856,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -901,20 +902,20 @@ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectImmortal(res_o); + res = PyStackRef_FromPyObjectBorrow(res_o); } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); #endif @@ -933,7 +934,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -975,7 +976,7 @@ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); res = PyStackRef_FromPyObjectNew(res_o); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = tuple_st; tuple_st = res; @@ -987,7 +988,7 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); #endif @@ -1040,12 +1041,12 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); #endif @@ -1063,7 +1064,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1073,7 +1074,7 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1087,25 +1088,25 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_SLICE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BINARY_SLICE; (void)(opcode); #endif @@ -1137,7 +1138,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); @@ -1145,7 +1146,7 @@ stack_pointer += 2; } stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1156,12 +1157,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_INTERPOLATION) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BUILD_INTERPOLATION; (void)(opcode); #endif @@ -1190,7 +1191,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(format[0]); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1199,12 +1200,12 @@ stack_pointer += -(oparg & 1); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1214,12 +1215,12 @@ interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); stack_pointer[0] = interpolation; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_LIST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BUILD_LIST; (void)(opcode); #endif @@ -1238,12 +1239,12 @@ list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_MAP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BUILD_MAP; (void)(opcode); #endif @@ -1253,49 +1254,23 @@ _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_SET) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BUILD_SET; (void)(opcode); #endif @@ -1318,7 +1293,7 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } int err = 0; @@ -1338,7 +1313,7 @@ } if (err) { stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1347,12 +1322,12 @@ set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_SLICE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BUILD_SLICE; (void)(opcode); #endif @@ -1375,19 +1350,19 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_STRING) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BUILD_STRING; (void)(opcode); #endif @@ -1397,44 +1372,23 @@ _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_TEMPLATE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BUILD_TEMPLATE; (void)(opcode); #endif @@ -1452,12 +1406,12 @@ PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(interpolations); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(strings); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1467,12 +1421,12 @@ template = PyStackRef_FromPyObjectSteal(template_o); stack_pointer[0] = template; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_TUPLE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = BUILD_TUPLE; (void)(opcode); #endif @@ -1489,12 +1443,12 @@ tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CACHE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CACHE; (void)(opcode); #endif @@ -1507,7 +1461,7 @@ } TARGET(CALL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL; (void)(opcode); #endif @@ -1532,7 +1486,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -1582,11 +1536,11 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } - frame->return_offset = 4 ; + frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -1610,7 +1564,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer[-2 - oparg] = callable; @@ -1663,37 +1617,29 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); #endif @@ -1708,8 +1654,8 @@ _PyStackRef init; _PyStackRef self; _PyStackRef *args; - _PyInterpreterFrame *init_frame; - _PyInterpreterFrame *new_frame; + _PyStackRef init_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -1785,24 +1731,24 @@ tstate, init, NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } - init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); } // _PUSH_FRAME { new_frame = init_frame; assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -1814,7 +1760,7 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); #endif @@ -1828,7 +1774,7 @@ _PyStackRef null; _PyStackRef self_or_null; _PyStackRef *args; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -1921,12 +1867,13 @@ args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } + new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { @@ -1940,11 +1887,11 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -1956,7 +1903,7 @@ } TARGET(CALL_BOUND_METHOD_GENERAL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); #endif @@ -1970,7 +1917,7 @@ _PyStackRef null; _PyStackRef self_or_null; _PyStackRef *args; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -2052,11 +1999,11 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { @@ -2070,9 +2017,9 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -2084,7 +2031,7 @@ } TARGET(CALL_BUILTIN_CLASS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_CLASS; (void)(opcode); #endif @@ -2124,80 +2071,36 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_BUILTIN_FAST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_FAST; (void)(opcode); #endif @@ -2218,13 +2121,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2236,85 +2139,37 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); #endif @@ -2335,13 +2190,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2354,84 +2209,32 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_BUILTIN_O) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_O; (void)(opcode); #endif @@ -2490,7 +2293,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2499,31 +2302,23 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_FUNCTION_EX) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_FUNCTION_EX; (void)(opcode); #endif @@ -2630,18 +2425,18 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } - assert( 1 == 1); + assert( 1u == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -2655,17 +2450,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2674,31 +2469,23 @@ } result = PyStackRef_FromPyObjectSteal(result_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_INTRINSIC_1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_INTRINSIC_1; (void)(opcode); #endif @@ -2713,7 +2500,7 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2723,12 +2510,12 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CALL_INTRINSIC_2) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_INTRINSIC_2; (void)(opcode); #endif @@ -2755,19 +2542,19 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CALL_ISINSTANCE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_ISINSTANCE; (void)(opcode); #endif @@ -2777,66 +2564,73 @@ next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; + _PyStackRef instance; + _PyStackRef cls; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; + // _GUARD_THIRD_NULL + { + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _GUARD_CALLABLE_ISINSTANCE + { + callable = stack_pointer[-4]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _CALL_ISINSTANCE + { + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + STAT_INC(CALL, hit); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(inst_o, cls_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + JUMP_TO_LABEL(error); + } + (void)null; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(cls); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(instance); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); } - STAT_INC(CALL, hit); - _PyStackRef cls_stackref = arguments[1]; - _PyStackRef inst_stackref = arguments[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(PyStackRef_AsPyObjectBorrow(inst_stackref), PyStackRef_AsPyObjectBorrow(cls_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - JUMP_TO_LABEL(error); - } - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = callable; - callable = res; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CALL_KW) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW; (void)(opcode); #endif @@ -2915,15 +2709,15 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2950,7 +2744,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer[-3 - oparg] = callable; @@ -3005,7 +2799,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3013,12 +2807,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CALL_KW_BOUND_METHOD) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); #endif @@ -3033,7 +2827,7 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -3111,16 +2905,16 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { @@ -3134,9 +2928,9 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -3148,7 +2942,7 @@ } TARGET(CALL_KW_NON_PY) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_NON_PY; (void)(opcode); #endif @@ -3189,101 +2983,43 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_KW_PY) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_PY; (void)(opcode); #endif @@ -3297,7 +3033,7 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -3324,6 +3060,14 @@ JUMP_TO_PREDICTED(CALL_KW); } } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } // _PY_FRAME_KW { kwnames = stack_pointer[-1]; @@ -3348,16 +3092,16 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { @@ -3371,9 +3115,9 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -3385,7 +3129,7 @@ } TARGET(CALL_LEN) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_LEN; (void)(opcode); #endif @@ -3439,12 +3183,12 @@ JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3452,12 +3196,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CALL_LIST_APPEND) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_LIST_APPEND; (void)(opcode); #endif @@ -3468,63 +3212,79 @@ INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; + _PyStackRef nos; _PyStackRef self; _PyStackRef arg; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _GUARD_CALLABLE_LIST_APPEND + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - if (self_o == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - if (!PyList_Check(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + // _GUARD_NOS_LIST + { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); - } - #if TIER_ONE + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + #if TIER_ONE - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + } DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); #endif @@ -3577,83 +3337,39 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); #endif @@ -3707,84 +3423,39 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); #endif @@ -3852,7 +3523,7 @@ PyStackRef_CLOSE(self_stackref); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3861,31 +3532,23 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_O) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); #endif @@ -3969,37 +3632,29 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_NON_PY_GENERAL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_NON_PY_GENERAL; (void)(opcode); #endif @@ -4038,91 +3693,43 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_PY_EXACT_ARGS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); #endif @@ -4135,7 +3742,7 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -4199,12 +3806,13 @@ args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; - new_frame->localsplus[0] = self_or_null; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } + new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { @@ -4218,11 +3826,11 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -4234,7 +3842,7 @@ } TARGET(CALL_PY_GENERAL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_PY_GENERAL; (void)(opcode); #endif @@ -4247,7 +3855,7 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -4302,11 +3910,11 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } - new_frame = temp; + new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { @@ -4320,9 +3928,9 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -4334,7 +3942,7 @@ } TARGET(CALL_STR_1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_STR_1; (void)(opcode); #endif @@ -4381,7 +3989,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4390,31 +3998,23 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_TUPLE_1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_TUPLE_1; (void)(opcode); #endif @@ -4461,7 +4061,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4470,31 +4070,23 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(CALL_TYPE_1) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CALL_TYPE_1; (void)(opcode); #endif @@ -4540,7 +4132,7 @@ res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4549,7 +4141,7 @@ } TARGET(CHECK_EG_MATCH) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CHECK_EG_MATCH; (void)(opcode); #endif @@ -4579,7 +4171,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } PyObject *match_o = NULL; @@ -4597,7 +4189,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -4615,12 +4207,12 @@ stack_pointer[0] = rest; stack_pointer[1] = match; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CHECK_EXC_MATCH) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CHECK_EXC_MATCH; (void)(opcode); #endif @@ -4645,19 +4237,19 @@ int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CLEANUP_THROW) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CLEANUP_THROW; (void)(opcode); #endif @@ -4675,7 +4267,7 @@ last_sent_val = stack_pointer[-2]; sub_iter = stack_pointer[-3]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !Py_TAIL_CALL_INTERP + #if !_Py_TAIL_CALL_INTERP assert(throwflag); #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); @@ -4699,7 +4291,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); none = PyStackRef_None; } else { @@ -4711,12 +4303,12 @@ stack_pointer[0] = none; stack_pointer[1] = value; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COMPARE_OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP; (void)(opcode); #endif @@ -4764,7 +4356,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4784,12 +4376,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COMPARE_OP_FLOAT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_FLOAT; (void)(opcode); #endif @@ -4839,12 +4431,12 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COMPARE_OP_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_INT; (void)(opcode); #endif @@ -4862,7 +4454,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -4872,7 +4464,7 @@ { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyLong_CheckExact(left_o)) { + if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); @@ -4884,16 +4476,8 @@ right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); - } - if (!_PyLong_IsCompact((PyLongObject *)right_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); - } + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); @@ -4906,12 +4490,12 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COMPARE_OP_STR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_STR; (void)(opcode); #endif @@ -4965,12 +4549,12 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CONTAINS_OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP; (void)(opcode); #endif @@ -5017,7 +4601,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5025,12 +4609,12 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CONTAINS_OP_DICT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP_DICT; (void)(opcode); #endif @@ -5075,7 +4659,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5083,12 +4667,12 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CONTAINS_OP_SET) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP_SET; (void)(opcode); #endif @@ -5133,7 +4717,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5141,12 +4725,12 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CONVERT_VALUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = CONVERT_VALUE; (void)(opcode); #endif @@ -5163,7 +4747,7 @@ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5173,12 +4757,12 @@ result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COPY) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = COPY; (void)(opcode); #endif @@ -5188,16 +4772,15 @@ _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; - assert(oparg > 0); top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COPY_FREE_VARS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = COPY_FREE_VARS; (void)(opcode); #endif @@ -5218,7 +4801,7 @@ } TARGET(DELETE_ATTR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = DELETE_ATTR; (void)(opcode); #endif @@ -5232,7 +4815,7 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5243,7 +4826,7 @@ } TARGET(DELETE_DEREF) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = DELETE_DEREF; (void)(opcode); #endif @@ -5265,7 +4848,7 @@ } TARGET(DELETE_FAST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = DELETE_FAST; (void)(opcode); #endif @@ -5291,7 +4874,7 @@ } TARGET(DELETE_GLOBAL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = DELETE_GLOBAL; (void)(opcode); #endif @@ -5316,7 +4899,7 @@ } TARGET(DELETE_NAME) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = DELETE_NAME; (void)(opcode); #endif @@ -5348,7 +4931,7 @@ } TARGET(DELETE_SUBSCR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = DELETE_SUBSCR; (void)(opcode); #endif @@ -5372,7 +4955,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -5380,7 +4963,7 @@ } TARGET(DICT_MERGE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = DICT_MERGE; (void)(opcode); #endif @@ -5404,14 +4987,14 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5419,7 +5002,7 @@ } TARGET(DICT_UPDATE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = DICT_UPDATE; (void)(opcode); #endif @@ -5447,14 +5030,14 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5462,7 +5045,7 @@ } TARGET(END_ASYNC_FOR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = END_ASYNC_FOR; (void)(opcode); #endif @@ -5494,7 +5077,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); @@ -5507,7 +5090,7 @@ } TARGET(END_FOR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = END_FOR; (void)(opcode); #endif @@ -5516,7 +5099,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5524,7 +5107,7 @@ } TARGET(END_SEND) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = END_SEND; (void)(opcode); #endif @@ -5539,7 +5122,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5547,7 +5130,7 @@ } TARGET(ENTER_EXECUTOR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = ENTER_EXECUTOR; (void)(opcode); #endif @@ -5558,6 +5141,10 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + JUMP_TO_LABEL(stop_tracing); + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -5573,7 +5160,9 @@ } DISPATCH_GOTO(); } - GOTO_TIER_TWO(executor); + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; + TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ @@ -5581,7 +5170,7 @@ } TARGET(EXIT_INIT_CHECK) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = EXIT_INIT_CHECK; (void)(opcode); #endif @@ -5599,12 +5188,12 @@ JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(EXTENDED_ARG) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = EXTENDED_ARG; (void)(opcode); #endif @@ -5620,7 +5209,7 @@ } TARGET(FORMAT_SIMPLE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = FORMAT_SIMPLE; (void)(opcode); #endif @@ -5636,7 +5225,7 @@ PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5651,12 +5240,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FORMAT_WITH_SPEC) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = FORMAT_WITH_SPEC; (void)(opcode); #endif @@ -5680,19 +5269,19 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER; (void)(opcode); #endif @@ -5703,17 +5292,19 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; // _SPECIALIZE_FOR_ITER { - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -5723,38 +5314,28 @@ } // _FOR_ITER { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o == NULL) { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } - next = PyStackRef_FromPyObjectSteal(next_o); + next = item; } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER_GEN) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_GEN; (void)(opcode); #endif @@ -5765,8 +5346,8 @@ INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; + _PyStackRef gen_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -5778,7 +5359,7 @@ } // _FOR_ITER_GEN_FRAME { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5798,21 +5379,22 @@ JUMP_TO_PREDICTED(FOR_ITER); } STAT_INC(FOR_ITER, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_None); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - gen_frame->previous = frame; - frame->return_offset = (uint16_t)( 2 + oparg); + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { new_frame = gen_frame; assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -5824,7 +5406,7 @@ } TARGET(FOR_ITER_LIST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_LIST; (void)(opcode); #endif @@ -5835,26 +5417,22 @@ INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST { - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyListIter_Type) { + if (Py_TYPE(iter_o) != &PyList_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } + assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) || - !_PyObject_GC_IS_SHARED(it->it_seq)) { + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); @@ -5863,42 +5441,30 @@ } // _ITER_JUMP_LIST { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(iter_o) == &PyListIter_Type); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - (void)iter_o; + #else - _PyListIterObject *it = (_PyListIterObject *)iter_o; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); STAT_INC(FOR_ITER, hit); - PyListObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { - it->it_index = -1; - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } #endif } // _ITER_NEXT_LIST { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyListIterObject *it = (_PyListIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyListIter_Type); - PyListObject *seq = it->it_seq; - assert(seq); + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - assert(_Py_IsOwnedByCurrentThread((PyObject *)seq) || - _PyObject_GC_IS_SHARED(seq)); + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock(seq, it->it_index, &next); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); stack_pointer = _PyFrame_GetStackPointer(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); @@ -5906,24 +5472,25 @@ JUMP_TO_PREDICTED(FOR_ITER); } if (result == 0) { - it->it_index = -1; + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } - it->it_index++; #else - assert(it->it_index < PyList_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(seq, it->it_index++)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER_RANGE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_RANGE; (void)(opcode); #endif @@ -5938,7 +5505,7 @@ /* Skip 1 cache entry */ // _ITER_CHECK_RANGE { - iter = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); @@ -5985,12 +5552,12 @@ } stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER_TUPLE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_TUPLE; (void)(opcode); #endif @@ -6001,71 +5568,52 @@ INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE { - iter = stack_pointer[-1]; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTupleIter_Type) { + if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - #endif + assert(PyStackRef_IsTaggedInt(null_or_index)); } // _ITER_JUMP_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - (void)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); STAT_INC(FOR_ITER, hit); - PyTupleObject *seq = it->it_seq; - if (seq == NULL || (size_t)it->it_index >= (size_t)PyTuple_GET_SIZE(seq)) { - #ifndef Py_GIL_DISABLED - if (seq != NULL) { - it->it_seq = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif - + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } } // _ITER_NEXT_TUPLE { - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - _PyTupleIterObject *it = (_PyTupleIterObject *)iter_o; - assert(Py_TYPE(iter_o) == &PyTupleIter_Type); - PyTupleObject *seq = it->it_seq; - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced(iter_o)); - #endif - assert(seq); - assert(it->it_index < PyTuple_GET_SIZE(seq)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq, it->it_index++)); + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_AITER) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = GET_AITER; (void)(opcode); #endif @@ -6090,7 +5638,7 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6100,7 +5648,7 @@ iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6121,12 +5669,12 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_ANEXT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = GET_ANEXT; (void)(opcode); #endif @@ -6145,12 +5693,12 @@ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_AWAITABLE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = GET_AWAITABLE; (void)(opcode); #endif @@ -6164,7 +5712,7 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6174,12 +5722,12 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_ITER) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = GET_ITER; (void)(opcode); #endif @@ -6188,32 +5736,44 @@ INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef index_or_null; iterable = stack_pointer[-1]; #ifdef Py_STATS _PyFrame_SetStackPointer(frame, stack_pointer); _Py_GatherStats_GetIter(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_LEN) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = GET_LEN; (void)(opcode); #endif @@ -6236,12 +5796,12 @@ len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_YIELD_FROM_ITER) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = GET_YIELD_FROM_ITER; (void)(opcode); #endif @@ -6286,7 +5846,7 @@ } TARGET(IMPORT_FROM) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_FROM; (void)(opcode); #endif @@ -6306,12 +5866,12 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(IMPORT_NAME) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_NAME; (void)(opcode); #endif @@ -6338,19 +5898,19 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_CALL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL; (void)(opcode); #endif @@ -6439,11 +5999,11 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } - frame->return_offset = 4 ; + frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -6465,7 +6025,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6516,37 +6076,29 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); #endif @@ -6653,18 +6205,18 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } - assert( 1 == 1); + assert( 1u == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -6678,17 +6230,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6697,31 +6249,23 @@ } result = PyStackRef_FromPyObjectSteal(result_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } TARGET(INSTRUMENTED_CALL_KW) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); #endif @@ -6808,15 +6352,15 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -6841,7 +6385,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6894,7 +6438,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6902,12 +6446,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_END_ASYNC_FOR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_ASYNC_FOR; (void)(opcode); #endif @@ -6946,7 +6490,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); @@ -6960,7 +6504,7 @@ } TARGET(INSTRUMENTED_END_FOR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_FOR; (void)(opcode); #endif @@ -6971,7 +6515,7 @@ _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -6981,7 +6525,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6989,7 +6533,7 @@ } TARGET(INSTRUMENTED_END_SEND) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_SEND; (void)(opcode); #endif @@ -7015,7 +6559,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7023,7 +6567,7 @@ } TARGET(INSTRUMENTED_FOR_ITER) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); #endif @@ -7033,43 +6577,33 @@ next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _PyStackRef iter; + _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ - iter = stack_pointer[-1]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_o != NULL) { - next = PyStackRef_FromPyObjectSteal(next_o); - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - } - else { - if (_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!matches) { - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - _PyErr_Clear(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); } - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; DISPATCH(); } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); #endif @@ -7096,7 +6630,7 @@ } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); #endif @@ -7108,15 +6642,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _MONITOR_JUMP_BACKWARD @@ -7127,7 +6657,7 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); #endif @@ -7141,7 +6671,7 @@ } TARGET(INSTRUMENTED_LINE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_LINE; (void)(opcode); #endif @@ -7181,7 +6711,7 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); #endif @@ -7228,7 +6758,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } @@ -7273,7 +6803,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { JUMP_TO_LABEL(error); } @@ -7296,12 +6826,12 @@ } stack_pointer[0] = attr; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_NOT_TAKEN) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); #endif @@ -7317,7 +6847,7 @@ } TARGET(INSTRUMENTED_POP_ITER) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); #endif @@ -7328,10 +6858,13 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; - iter = stack_pointer[-1]; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7339,7 +6872,7 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); #endif @@ -7358,12 +6891,12 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); #endif @@ -7382,19 +6915,19 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif @@ -7410,7 +6943,7 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7423,7 +6956,7 @@ } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); #endif @@ -7442,12 +6975,12 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_RESUME) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_RESUME; (void)(opcode); #endif @@ -7478,7 +7011,13 @@ } // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -7496,15 +7035,11 @@ // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } } @@ -7525,7 +7060,7 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); #endif @@ -7555,7 +7090,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -7569,12 +7104,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_YIELD_VALUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); #endif @@ -7613,7 +7148,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -7637,12 +7172,12 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INTERPRETER_EXIT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = INTERPRETER_EXIT; (void)(opcode); #endif @@ -7656,7 +7191,7 @@ tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); - #if !Py_TAIL_CALL_INTERP + #if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 @@ -7665,7 +7200,7 @@ if (!PyStackRef_IsNull(executor)) { tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(executor); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7677,7 +7212,7 @@ } TARGET(IS_OP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = IS_OP; (void)(opcode); #endif @@ -7701,16 +7236,16 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(JUMP_BACKWARD) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD; (void)(opcode); #endif @@ -7725,7 +7260,8 @@ { #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; DISPATCH_SAME_OPARG(); } @@ -7733,15 +7269,11 @@ } // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7753,7 +7285,7 @@ } TARGET(JUMP_BACKWARD_JIT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_JIT; (void)(opcode); #endif @@ -7766,15 +7298,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7786,28 +7314,20 @@ { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; - start--; + insert_exec_at--; } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) { - JUMP_TO_LABEL(error); - } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == NULL); - GOTO_TIER_TWO(executor); + this_instr[1].counter = restart_backoff_counter(counter); } } else { @@ -7819,7 +7339,7 @@ } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); #endif @@ -7832,7 +7352,7 @@ } TARGET(JUMP_BACKWARD_NO_JIT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); #endif @@ -7843,15 +7363,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7863,7 +7379,7 @@ } TARGET(JUMP_FORWARD) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = JUMP_FORWARD; (void)(opcode); #endif @@ -7875,7 +7391,7 @@ } TARGET(LIST_APPEND) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LIST_APPEND; (void)(opcode); #endif @@ -7892,12 +7408,12 @@ JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LIST_EXTEND) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LIST_EXTEND; (void)(opcode); #endif @@ -7928,7 +7444,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7936,7 +7452,7 @@ } assert(Py_IsNone(none_val)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7944,7 +7460,7 @@ } TARGET(LOAD_ATTR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR; (void)(opcode); #endif @@ -7955,7 +7471,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; - _PyStackRef attr; + _PyStackRef *attr; _PyStackRef *self_or_null; // _SPECIALIZE_LOAD_ATTR { @@ -7978,25 +7494,25 @@ /* Skip 8 cache entries */ // _LOAD_ATTR { + attr = &stack_pointer[-1]; self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { - attr_o = NULL; + *attr = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); stack_pointer = _PyFrame_GetStackPointer(frame); if (is_meth) { - assert(attr_o != NULL); + assert(!PyStackRef_IsNull(*attr)); self_or_null[0] = owner; } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { + if (PyStackRef_IsNull(*attr)) { JUMP_TO_LABEL(error); } self_or_null[0] = PyStackRef_NULL; @@ -8005,28 +7521,27 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } + *attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer += 1; } - attr = PyStackRef_FromPyObjectSteal(attr_o); } - stack_pointer[-1] = attr; stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_CLASS; (void)(opcode); #endif @@ -8079,12 +7594,12 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); #endif @@ -8147,12 +7662,12 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); #endif @@ -8203,14 +7718,14 @@ tstate, PyStackRef_FromPyObjectNew(f), 2, frame); new_frame->localsplus[0] = owner; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10 ; + frame->return_offset = 10u ; DISPATCH_INLINED(new_frame); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); #endif @@ -8285,12 +7800,12 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); #endif @@ -8341,12 +7856,12 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); #endif @@ -8387,12 +7902,12 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); #endif @@ -8454,12 +7969,12 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_MODULE; (void)(opcode); #endif @@ -8528,12 +8043,12 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); #endif @@ -8567,7 +8082,7 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8575,12 +8090,12 @@ } stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); #endif @@ -8635,7 +8150,7 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8643,12 +8158,12 @@ } stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); #endif @@ -8659,7 +8174,7 @@ INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; - _PyInterpreterFrame *new_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -8710,8 +8225,9 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); - new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - new_frame->localsplus[0] = owner; + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { @@ -8725,11 +8241,11 @@ // _PUSH_FRAME { assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -8741,7 +8257,7 @@ } TARGET(LOAD_ATTR_SLOT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_SLOT; (void)(opcode); #endif @@ -8805,12 +8321,12 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); #endif @@ -8915,12 +8431,12 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_BUILD_CLASS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_BUILD_CLASS; (void)(opcode); #endif @@ -8945,12 +8461,12 @@ bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_COMMON_CONSTANT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); #endif @@ -8962,83 +8478,29 @@ value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_CONST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_CONST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PREDICTED_LOAD_CONST:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; _PyStackRef value; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNew(obj); - #if ENABLE_SPECIALIZATION_FT - #ifdef Py_GIL_DISABLED - uint8_t expected = LOAD_CONST; - if (!_Py_atomic_compare_exchange_uint8( - &this_instr->op.code, &expected, - _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) { - assert(expected >= MIN_INSTRUMENTED_OPCODE); - } - #else - if (this_instr->op.code == LOAD_CONST) { - this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL; - } - #endif - #endif + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_CONST_IMMORTAL) { - #if Py_TAIL_CALL_INTERP - int opcode = LOAD_CONST_IMMORTAL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - assert(_Py_IsImmortal(obj)); - value = PyStackRef_FromPyObjectImmortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); - } - - TARGET(LOAD_CONST_MORTAL) { - #if Py_TAIL_CALL_INTERP - int opcode = LOAD_CONST_MORTAL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST_MORTAL); - static_assert(0 == 0, "incorrect cache size"); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectNewMortal(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_DEREF) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_DEREF; (void)(opcode); #endif @@ -9053,7 +8515,7 @@ if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9061,12 +8523,12 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST; (void)(opcode); #endif @@ -9078,12 +8540,12 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_AND_CLEAR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); #endif @@ -9095,12 +8557,12 @@ GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_BORROW) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_BORROW; (void)(opcode); #endif @@ -9112,12 +8574,12 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; (void)(opcode); #endif @@ -9133,12 +8595,12 @@ stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_CHECK) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_CHECK; (void)(opcode); #endif @@ -9159,12 +8621,12 @@ value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_LOAD_FAST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); #endif @@ -9180,12 +8642,12 @@ stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FROM_DICT_OR_DEREF) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); #endif @@ -9218,19 +8680,19 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); #endif @@ -9246,7 +8708,7 @@ int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(mod_or_class_dict); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9300,12 +8762,12 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_GLOBAL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL; (void)(opcode); #endif @@ -9356,12 +8818,12 @@ } } stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); #endif @@ -9436,12 +8898,12 @@ } stack_pointer[0] = res; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); #endif @@ -9503,12 +8965,12 @@ } stack_pointer[0] = res; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_LOCALS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_LOCALS; (void)(opcode); #endif @@ -9527,12 +8989,12 @@ locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_NAME) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_NAME; (void)(opcode); #endif @@ -9550,12 +9012,12 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_SMALL_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SMALL_INT; (void)(opcode); #endif @@ -9565,15 +9027,15 @@ _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectImmortal(obj); + value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_SPECIAL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SPECIAL; (void)(opcode); #endif @@ -9594,7 +9056,7 @@ method_and_self = &stack_pointer[-1]; PyObject *name = _Py_SpecialMethods[oparg].name; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9619,7 +9081,7 @@ } TARGET(LOAD_SUPER_ATTR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR; (void)(opcode); #endif @@ -9683,7 +9145,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } @@ -9728,7 +9190,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { JUMP_TO_LABEL(error); } @@ -9751,12 +9213,12 @@ } stack_pointer[0] = attr; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_SUPER_ATTR_ATTR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); #endif @@ -9806,19 +9268,19 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { JUMP_TO_LABEL(error); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[0] = attr_st; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_SUPER_ATTR_METHOD) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); #endif @@ -9866,7 +9328,7 @@ self_or_null = self_st; } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9874,7 +9336,7 @@ stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; @@ -9886,17 +9348,17 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; stack_pointer[1] = self_or_null; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MAKE_CELL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = MAKE_CELL; (void)(opcode); #endif @@ -9917,7 +9379,7 @@ } TARGET(MAKE_FUNCTION) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = MAKE_FUNCTION; (void)(opcode); #endif @@ -9933,7 +9395,7 @@ PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9945,12 +9407,12 @@ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[0] = func; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MAP_ADD) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = MAP_ADD; (void)(opcode); #endif @@ -9976,12 +9438,12 @@ JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MATCH_CLASS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = MATCH_CLASS; (void)(opcode); #endif @@ -10015,7 +9477,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -10028,12 +9490,12 @@ } stack_pointer[0] = attrs; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MATCH_KEYS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = MATCH_KEYS; (void)(opcode); #endif @@ -10055,12 +9517,12 @@ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MATCH_MAPPING) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = MATCH_MAPPING; (void)(opcode); #endif @@ -10074,12 +9536,12 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MATCH_SEQUENCE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = MATCH_SEQUENCE; (void)(opcode); #endif @@ -10093,12 +9555,12 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(NOP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = NOP; (void)(opcode); #endif @@ -10109,7 +9571,7 @@ } TARGET(NOT_TAKEN) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = NOT_TAKEN; (void)(opcode); #endif @@ -10120,7 +9582,7 @@ } TARGET(POP_EXCEPT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = POP_EXCEPT; (void)(opcode); #endif @@ -10136,30 +9598,33 @@ ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_ITER) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = POP_ITER; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } TARGET(POP_JUMP_IF_FALSE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_FALSE; (void)(opcode); #endif @@ -10176,12 +9641,12 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_NONE; (void)(opcode); #endif @@ -10219,12 +9684,12 @@ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif @@ -10262,12 +9727,12 @@ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_TRUE; (void)(opcode); #endif @@ -10284,12 +9749,12 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_TOP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = POP_TOP; (void)(opcode); #endif @@ -10299,15 +9764,15 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } TARGET(PUSH_EXC_INFO) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = PUSH_EXC_INFO; (void)(opcode); #endif @@ -10331,12 +9796,12 @@ stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(PUSH_NULL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = PUSH_NULL; (void)(opcode); #endif @@ -10347,12 +9812,12 @@ res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(RAISE_VARARGS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = RAISE_VARARGS; (void)(opcode); #endif @@ -10367,7 +9832,7 @@ PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = do_raise(tstate, exc, cause); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10381,7 +9846,7 @@ } TARGET(RERAISE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = RERAISE; (void)(opcode); #endif @@ -10401,7 +9866,7 @@ } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); @@ -10409,7 +9874,7 @@ } TARGET(RESERVED) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = RESERVED; (void)(opcode); #endif @@ -10422,7 +9887,7 @@ } TARGET(RESUME) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = RESUME; (void)(opcode); #endif @@ -10454,7 +9919,13 @@ } // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -10480,15 +9951,11 @@ // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } } @@ -10496,7 +9963,7 @@ } TARGET(RESUME_CHECK) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = RESUME_CHECK; (void)(opcode); #endif @@ -10534,7 +10001,7 @@ } TARGET(RETURN_GENERATOR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = RETURN_GENERATOR; (void)(opcode); #endif @@ -10568,12 +10035,12 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(RETURN_VALUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = RETURN_VALUE; (void)(opcode); #endif @@ -10586,7 +10053,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -10599,12 +10066,12 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(SEND) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = SEND; (void)(opcode); #endif @@ -10648,12 +10115,12 @@ _PyInterpreterFrame *gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); assert(gen_frame->previous == NULL); gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); @@ -10688,7 +10155,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10696,7 +10163,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10704,12 +10171,12 @@ } stack_pointer[0] = retval; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(SEND_GEN) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = SEND_GEN; (void)(opcode); #endif @@ -10721,8 +10188,8 @@ static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; - _PyInterpreterFrame *gen_frame; - _PyInterpreterFrame *new_frame; + _PyStackRef gen_frame; + _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -10748,24 +10215,25 @@ JUMP_TO_PREDICTED(SEND); } STAT_INC(SEND, hit); - gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - gen_frame->previous = frame; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { new_frame = gen_frame; assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = new_frame; + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - assert(new_frame->previous == frame || new_frame->previous->previous == frame); + assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; @@ -10777,7 +10245,7 @@ } TARGET(SETUP_ANNOTATIONS) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = SETUP_ANNOTATIONS; (void)(opcode); #endif @@ -10823,7 +10291,7 @@ } TARGET(SET_ADD) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = SET_ADD; (void)(opcode); #endif @@ -10842,12 +10310,12 @@ JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(SET_FUNCTION_ATTRIBUTE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); #endif @@ -10870,12 +10338,12 @@ *ptr = attr; stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(SET_UPDATE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = SET_UPDATE; (void)(opcode); #endif @@ -10891,7 +10359,7 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10902,7 +10370,7 @@ } TARGET(STORE_ATTR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR; (void)(opcode); #endif @@ -10950,7 +10418,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -10959,7 +10427,7 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); #endif @@ -11025,7 +10493,7 @@ } UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -11035,7 +10503,7 @@ } TARGET(STORE_ATTR_SLOT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_SLOT; (void)(opcode); #endif @@ -11076,7 +10544,7 @@ FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -11086,7 +10554,7 @@ } TARGET(STORE_ATTR_WITH_HINT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); #endif @@ -11128,16 +10596,6 @@ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); - } - } - #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || @@ -11174,7 +10632,7 @@ UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -11184,7 +10642,7 @@ } TARGET(STORE_DEREF) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_DEREF; (void)(opcode); #endif @@ -11198,12 +10656,12 @@ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(STORE_FAST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST; (void)(opcode); #endif @@ -11212,14 +10670,10 @@ INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; value = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value) - ); _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11227,7 +10681,7 @@ } TARGET(STORE_FAST_LOAD_FAST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); #endif @@ -11237,10 +10691,6 @@ _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value1) - ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); @@ -11254,7 +10704,7 @@ } TARGET(STORE_FAST_STORE_FAST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST_STORE_FAST; (void)(opcode); #endif @@ -11265,27 +10715,19 @@ _PyStackRef value1; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value1) - ); - assert( - ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) || - PyStackRef_IsHeapSafe(value2) - ); uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); GETLOCAL(oparg1) = value1; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); tmp = GETLOCAL(oparg2); GETLOCAL(oparg2) = value2; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11293,7 +10735,7 @@ } TARGET(STORE_GLOBAL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_GLOBAL; (void)(opcode); #endif @@ -11307,7 +10749,7 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11318,7 +10760,7 @@ } TARGET(STORE_NAME) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_NAME; (void)(opcode); #endif @@ -11336,7 +10778,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11353,7 +10795,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11364,7 +10806,7 @@ } TARGET(STORE_SLICE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_SLICE; (void)(opcode); #endif @@ -11397,7 +10839,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); @@ -11415,7 +10857,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -11424,7 +10866,7 @@ } TARGET(STORE_SUBSCR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR; (void)(opcode); #endif @@ -11474,7 +10916,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -11483,7 +10925,7 @@ } TARGET(STORE_SUBSCR_DICT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR_DICT; (void)(opcode); #endif @@ -11522,7 +10964,7 @@ PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11534,7 +10976,7 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); #endif @@ -11552,7 +10994,7 @@ { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { + if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); @@ -11605,7 +11047,7 @@ UNLOCK_OBJECT(list); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); Py_DECREF(old_value); @@ -11615,7 +11057,7 @@ } TARGET(SWAP) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = SWAP; (void)(opcode); #endif @@ -11629,14 +11071,13 @@ _PyStackRef temp = bottom; bottom = top; top = temp; - assert(oparg >= 2); stack_pointer[-2 - (oparg-2)] = bottom; stack_pointer[-1] = top; DISPATCH(); } TARGET(TO_BOOL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; (void)(opcode); #endif @@ -11672,7 +11113,7 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11683,12 +11124,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(TO_BOOL_ALWAYS_TRUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); #endif @@ -11718,7 +11159,7 @@ { value = owner; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11726,12 +11167,12 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(TO_BOOL_BOOL) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_BOOL; (void)(opcode); #endif @@ -11755,7 +11196,7 @@ } TARGET(TO_BOOL_INT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_INT; (void)(opcode); #endif @@ -11783,7 +11224,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11795,7 +11236,7 @@ } TARGET(TO_BOOL_LIST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_LIST; (void)(opcode); #endif @@ -11838,7 +11279,7 @@ } TARGET(TO_BOOL_NONE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_NONE; (void)(opcode); #endif @@ -11865,7 +11306,7 @@ } TARGET(TO_BOOL_STR) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_STR; (void)(opcode); #endif @@ -11900,7 +11341,7 @@ else { assert(Py_SIZE(value_o)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11912,8 +11353,70 @@ DISPATCH(); } + TARGET(TRACE_RECORD) { + #if _Py_TAIL_CALL_INTERP + int opcode = TRACE_RECORD; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TRACE_RECORD); + opcode = TRACE_RECORD; + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); + #endif + } + TARGET(UNARY_INVERT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = UNARY_INVERT; (void)(opcode); #endif @@ -11927,7 +11430,7 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11937,12 +11440,12 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(UNARY_NEGATIVE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = UNARY_NEGATIVE; (void)(opcode); #endif @@ -11956,7 +11459,7 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11966,12 +11469,12 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(UNARY_NOT) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = UNARY_NOT; (void)(opcode); #endif @@ -11989,7 +11492,7 @@ } TARGET(UNPACK_EX) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_EX; (void)(opcode); #endif @@ -12002,7 +11505,7 @@ top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq_o); @@ -12011,12 +11514,12 @@ JUMP_TO_LABEL(error); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(UNPACK_SEQUENCE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE; (void)(opcode); #endif @@ -12052,7 +11555,7 @@ top = &stack_pointer[-1 + oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); Py_DECREF(seq_o); @@ -12062,12 +11565,12 @@ } } stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(UNPACK_SEQUENCE_LIST) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); #endif @@ -12117,7 +11620,7 @@ } UNLOCK_OBJECT(seq_o); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12126,7 +11629,7 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); #endif @@ -12167,7 +11670,7 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12176,7 +11679,7 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); #endif @@ -12218,7 +11721,7 @@ stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12227,7 +11730,7 @@ } TARGET(WITH_EXCEPT_START) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = WITH_EXCEPT_START; (void)(opcode); #endif @@ -12267,12 +11770,12 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(YIELD_VALUE) { - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode = YIELD_VALUE; (void)(opcode); #endif @@ -12290,7 +11793,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -12313,12 +11816,12 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } /* END INSTRUCTIONS */ -#if !Py_TAIL_CALL_INTERP +#if !_Py_TAIL_CALL_INTERP #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -12340,7 +11843,7 @@ JUMP_TO_LABEL(error); /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); -#endif /* Py_TAIL_CALL_INTERP */ +#endif /* _Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ LABEL(pop_2_error) @@ -12426,7 +11929,7 @@ JUMP_TO_LABEL(error); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode; #endif DISPATCH(); @@ -12443,7 +11946,7 @@ JUMP_TO_LABEL(error); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { tstate->current_frame = frame->previous; - #if !Py_TAIL_CALL_INTERP + #if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 @@ -12477,11 +11980,32 @@ JUMP_TO_LABEL(error); assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode; #endif DISPATCH(); } + LABEL(stop_tracing) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + /* END LABELS */ #undef TIER_ONE diff --git a/Python/getargs.c b/Python/getargs.c index 0cf596285cc..c119ca5c353 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1,8 +1,6 @@ /* New getargs implementation */ -#include <stdbool.h> - #define PY_CXX_CONST const #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() @@ -468,12 +466,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; int i; Py_ssize_t len; - bool nullable = false; int istuple = PyTuple_Check(arg); int mustbetuple = istuple; - assert(*format == '('); - format++; for (;;) { int c = *format++; if (c == '(') { @@ -482,12 +477,8 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, level++; } else if (c == ')') { - if (level == 0) { - if (*format == '?') { - nullable = true; - } + if (level == 0) break; - } level--; } else if (c == ':' || c == ';' || c == '\0') @@ -524,13 +515,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - if (arg == Py_None && nullable) { - const char *msg = skipitem(p_format, p_va, flags); - if (msg != NULL) { - levels[0] = 0; - } - return msg; - } if (istuple) { /* fallthrough */ } @@ -539,10 +523,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, - "must be %d-item tuple%s, not %.50s", - n, - nullable ? " or None" : "", - arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); + "must be %d-item tuple, not %.50s", + n, + arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); return msgbuf; } else { @@ -579,7 +562,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, return msgbuf; } - format = *p_format + 1; + format = *p_format; for (i = 0; i < n; i++) { const char *msg; PyObject *item = PyTuple_GET_ITEM(arg, i); @@ -594,10 +577,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - format++; - if (*format == '?') { - format++; - } *p_format = format; if (!istuple) { Py_DECREF(arg); @@ -616,8 +595,11 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; if (*format == '(' /* ')' */) { + format++; msg = converttuple(arg, &format, p_va, flags, levels, msgbuf, bufsize, freelist); + if (msg == NULL) + format++; } else { msg = convertsimple(arg, &format, p_va, flags, @@ -647,7 +629,7 @@ _PyArg_BadArgument(const char *fname, const char *displayname, } static const char * -converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) +converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) { assert(expected != NULL); assert(arg != NULL); @@ -657,23 +639,20 @@ converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, siz } else { PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s", expected, - nullable ? " or None" : "", + "must be %.50s, not %.50s", expected, arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); } return msgbuf; } static const char * -convertcharerr(bool nullable, const char *expected, const char *what, Py_ssize_t size, +convertcharerr(const char *expected, const char *what, Py_ssize_t size, char *msgbuf, size_t bufsize) { assert(expected != NULL); PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s of length %zd", - expected, - nullable ? " or None" : "", - what, size); + "must be %.50s, not %.50s of length %zd", + expected, what, size); return msgbuf; } @@ -693,26 +672,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *msgbuf, size_t bufsize, freelist_t *freelist) { #define RETURN_ERR_OCCURRED return msgbuf -#define HANDLE_NULLABLE \ - if (*format == '?') { \ - format++; \ - if (arg == Py_None) { \ - break; \ - } \ - nullable = true; \ - } - const char *format = *p_format; char c = *format++; const char *sarg; - bool nullable = false; switch (c) { case 'b': { /* unsigned byte -- very short int */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -726,6 +694,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, "unsigned byte integer is greater than maximum"); RETURN_ERR_OCCURRED; } + else *p = (unsigned char) ival; break; } @@ -733,18 +702,25 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'B': {/* byte sized bitfield - both signed and unsigned values allowed */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; - unsigned long ival = PyLong_AsUnsignedLongMask(arg); - if (ival == (unsigned long)-1 && PyErr_Occurred()) + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned char), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; - else - *p = (unsigned char) ival; + } + if ((size_t)bytes > sizeof(unsigned char)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } case 'h': {/* signed short int */ short *p = va_arg(*p_va, short *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -766,18 +742,25 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'H': { /* short int sized bitfield, both signed and unsigned allowed */ unsigned short *p = va_arg(*p_va, unsigned short *); - HANDLE_NULLABLE; - unsigned long ival = PyLong_AsUnsignedLongMask(arg); - if (ival == (unsigned long)-1 && PyErr_Occurred()) + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned short), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; - else - *p = (unsigned short) ival; + } + if ((size_t)bytes > sizeof(unsigned short)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } case 'i': {/* signed int */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -799,12 +782,20 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'I': { /* int sized bitfield, both signed and unsigned allowed */ unsigned int *p = va_arg(*p_va, unsigned int *); - HANDLE_NULLABLE; - unsigned long ival = PyLong_AsUnsignedLongMask(arg); - if (ival == (unsigned long)-1 && PyErr_Occurred()) + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; - else - *p = (unsigned int) ival; + } + if ((size_t)bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } @@ -812,7 +803,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { PyObject *iobj; Py_ssize_t *p = va_arg(*p_va, Py_ssize_t *); - HANDLE_NULLABLE; Py_ssize_t ival = -1; iobj = _PyNumber_Index(arg); if (iobj != NULL) { @@ -826,7 +816,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } case 'l': {/* long int */ long *p = va_arg(*p_va, long *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -837,22 +826,28 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'k': { /* long sized bitfield */ unsigned long *p = va_arg(*p_va, unsigned long *); - HANDLE_NULLABLE; - unsigned long ival; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } - ival = PyLong_AsUnsignedLongMask(arg); - if (ival == (unsigned long)(long)-1 && PyErr_Occurred()) { + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; } - *p = ival; + if ((size_t)bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } case 'L': {/* long long */ long long *p = va_arg( *p_va, long long * ); - HANDLE_NULLABLE; long long ival = PyLong_AsLongLong(arg); if (ival == (long long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -863,22 +858,28 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'K': { /* long long sized bitfield */ unsigned long long *p = va_arg(*p_va, unsigned long long *); - HANDLE_NULLABLE; - unsigned long long ival; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } - ival = PyLong_AsUnsignedLongLongMask(arg); - if (ival == (unsigned long long)(long long)-1 && PyErr_Occurred()) { + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned long long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; } - *p = ival; + if ((size_t)bytes > sizeof(unsigned long long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } case 'f': {/* float */ float *p = va_arg(*p_va, float *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -889,7 +890,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'd': {/* double */ double *p = va_arg(*p_va, double *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -900,7 +900,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'D': {/* complex double */ Py_complex *p = va_arg(*p_va, Py_complex *); - HANDLE_NULLABLE; Py_complex cval; cval = PyComplex_AsCComplex(arg); if (PyErr_Occurred()) @@ -912,10 +911,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'c': {/* char */ char *p = va_arg(*p_va, char *); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) { if (PyBytes_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytes object", PyBytes_GET_SIZE(arg), msgbuf, bufsize); } @@ -923,28 +921,27 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (PyByteArray_Check(arg)) { if (PyByteArray_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytearray object", PyByteArray_GET_SIZE(arg), msgbuf, bufsize); } *p = PyByteArray_AS_STRING(arg)[0]; } else - return converterr(nullable, "a byte string of length 1", arg, msgbuf, bufsize); + return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; } case 'C': {/* unicode char */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int kind; const void *data; if (!PyUnicode_Check(arg)) - return converterr(nullable, "a unicode character", arg, msgbuf, bufsize); + return converterr("a unicode character", arg, msgbuf, bufsize); if (PyUnicode_GET_LENGTH(arg) != 1) { - return convertcharerr(nullable, "a unicode character", + return convertcharerr("a unicode character", "a string", PyUnicode_GET_LENGTH(arg), msgbuf, bufsize); } @@ -957,7 +954,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'p': {/* boolean *p*redicate */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int val = PyObject_IsTrue(arg); if (val > 0) *p = 1; @@ -976,31 +972,24 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count; if (*format == '*') { - format++; - HANDLE_NULLABLE; if (getbuffer(arg, (Py_buffer*)p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); + format++; if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } - else if (*format == '#') { + count = convertbuffer(arg, (const void **)p, &buf); + if (count < 0) + return converterr(buf, arg, msgbuf, bufsize); + if (*format == '#') { Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); *psize = count; - } - else { - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + format++; + } else { if (strlen(*p) != (size_t)count) { PyErr_SetString(PyExc_ValueError, "embedded null byte"); RETURN_ERR_OCCURRED; @@ -1016,35 +1005,32 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* "s*" or "z*" */ Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); else if (PyUnicode_Check(arg)) { Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); PyBuffer_FillInfo(p, arg, (void *)sarg, len, 1, 0); } else { /* any bytes-like object */ const char *buf; if (getbuffer(arg, p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); } if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } + format++; } else if (*format == '#') { /* a string or read-only bytes-like object */ /* "s#" or "z#" */ const void **p = (const void **)va_arg(*p_va, const char **); Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) { *p = NULL; *psize = 0; @@ -1053,7 +1039,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); *p = sarg; *psize = len; @@ -1063,22 +1049,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count = convertbuffer(arg, p, &buf); if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); *psize = count; } + format++; } else { /* "s" or "z" */ const char **p = va_arg(*p_va, const char **); Py_ssize_t len; sarg = NULL; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) *p = NULL; else if (PyUnicode_Check(arg)) { sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); if (strlen(sarg) != (size_t)len) { PyErr_SetString(PyExc_ValueError, "embedded null character"); @@ -1087,7 +1073,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *p = sarg; } else - return converterr(c == 'z' || nullable, "str", + return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); } break; @@ -1116,14 +1102,48 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, recode_strings = 0; else return converterr( - nullable, "(unknown parser marker combination)", + "(unknown parser marker combination)", arg, msgbuf, bufsize); buffer = (char **)va_arg(*p_va, char **); format++; if (buffer == NULL) - return converterr(nullable, "(buffer is NULL)", + return converterr("(buffer is NULL)", arg, msgbuf, bufsize); - Py_ssize_t *psize = NULL; + + /* Encode object */ + if (!recode_strings && + (PyBytes_Check(arg) || PyByteArray_Check(arg))) { + s = Py_NewRef(arg); + if (PyBytes_Check(arg)) { + size = PyBytes_GET_SIZE(s); + ptr = PyBytes_AS_STRING(s); + } + else { + size = PyByteArray_GET_SIZE(s); + ptr = PyByteArray_AS_STRING(s); + } + } + else if (PyUnicode_Check(arg)) { + /* Encode object; use default error handling */ + s = PyUnicode_AsEncodedString(arg, + encoding, + NULL); + if (s == NULL) + return converterr("(encoding failed)", + arg, msgbuf, bufsize); + assert(PyBytes_Check(s)); + size = PyBytes_GET_SIZE(s); + ptr = PyBytes_AS_STRING(s); + if (ptr == NULL) + ptr = ""; + } + else { + return converterr( + recode_strings ? "str" : "str, bytes or bytearray", + arg, msgbuf, bufsize); + } + + /* Write output; output is guaranteed to be 0-terminated */ if (*format == '#') { /* Using buffer length parameter '#': @@ -1146,55 +1166,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, trailing 0-byte */ - psize = va_arg(*p_va, Py_ssize_t*); + Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); format++; if (psize == NULL) { + Py_DECREF(s); return converterr( - nullable, "(buffer_len is NULL)", + "(buffer_len is NULL)", arg, msgbuf, bufsize); } - } - HANDLE_NULLABLE; - - /* Encode object */ - if (!recode_strings && - (PyBytes_Check(arg) || PyByteArray_Check(arg))) { - s = Py_NewRef(arg); - if (PyBytes_Check(arg)) { - size = PyBytes_GET_SIZE(s); - ptr = PyBytes_AS_STRING(s); - } - else { - size = PyByteArray_GET_SIZE(s); - ptr = PyByteArray_AS_STRING(s); - } - } - else if (PyUnicode_Check(arg)) { - /* Encode object; use default error handling */ - s = PyUnicode_AsEncodedString(arg, - encoding, - NULL); - if (s == NULL) - return converterr(nullable, "(encoding failed)", - arg, msgbuf, bufsize); - assert(PyBytes_Check(s)); - size = PyBytes_GET_SIZE(s); - ptr = PyBytes_AS_STRING(s); - if (ptr == NULL) - ptr = ""; - } - else { - return converterr( - nullable, - recode_strings ? "str" - : nullable ? "str, bytes, bytearray" - : "str, bytes or bytearray", - arg, msgbuf, bufsize); - } - - /* Write output; output is guaranteed to be 0-terminated */ - if (psize != NULL) { if (*buffer == NULL) { *buffer = PyMem_NEW(char, size + 1); if (*buffer == NULL) { @@ -1205,7 +1185,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } } else { @@ -1239,7 +1219,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if ((Py_ssize_t)strlen(ptr) != size) { Py_DECREF(s); return converterr( - nullable, "encoded string without null bytes", + "encoded string without null bytes", arg, msgbuf, bufsize); } *buffer = PyMem_NEW(char, size + 1); @@ -1250,7 +1230,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } memcpy(*buffer, ptr, size+1); @@ -1261,32 +1241,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'S': { /* PyBytes object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) *p = arg; else - return converterr(nullable, "bytes", arg, msgbuf, bufsize); + return converterr("bytes", arg, msgbuf, bufsize); break; } case 'Y': { /* PyByteArray object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyByteArray_Check(arg)) *p = arg; else - return converterr(nullable, "bytearray", arg, msgbuf, bufsize); + return converterr("bytearray", arg, msgbuf, bufsize); break; } case 'U': { /* PyUnicode object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyUnicode_Check(arg)) { *p = arg; } else - return converterr(nullable, "str", arg, msgbuf, bufsize); + return converterr("str", arg, msgbuf, bufsize); break; } @@ -1297,11 +1274,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, type = va_arg(*p_va, PyTypeObject*); p = va_arg(*p_va, PyObject **); format++; - HANDLE_NULLABLE; if (PyType_IsSubtype(Py_TYPE(arg), type)) *p = arg; else - return converterr(nullable, type->tp_name, arg, msgbuf, bufsize); + return converterr(type->tp_name, arg, msgbuf, bufsize); } else if (*format == '&') { @@ -1310,18 +1286,16 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, void *addr = va_arg(*p_va, void *); int res; format++; - HANDLE_NULLABLE; if (! (res = (*convert)(arg, addr))) - return converterr(nullable, "(unspecified)", + return converterr("(unspecified)", arg, msgbuf, bufsize); if (res == Py_CLEANUP_SUPPORTED && addcleanup(addr, freelist, convert) == -1) - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } else { p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; *p = arg; } break; @@ -1333,30 +1307,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (*format != '*') return converterr( - nullable, "(invalid use of 'w' format character)", + "(invalid use of 'w' format character)", arg, msgbuf, bufsize); format++; - HANDLE_NULLABLE; /* Caller is interested in Py_buffer, and the object supports it directly. The request implicitly asks for PyBUF_SIMPLE, so the result is C-contiguous with format 'B'. */ if (PyObject_GetBuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { PyErr_Clear(); - return converterr(nullable, "read-write bytes-like object", + return converterr("read-write bytes-like object", arg, msgbuf, bufsize); } assert(PyBuffer_IsContiguous((Py_buffer *)p, 'C')); if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } default: - return converterr(nullable, "(impossible<bad format char>)", arg, msgbuf, bufsize); + return converterr("(impossible<bad format char>)", arg, msgbuf, bufsize); } @@ -2751,9 +2724,6 @@ skipitem(const char **p_format, va_list *p_va, int flags) return "impossible<bad format char>"; } - if (*format == '?') { - format++; - } *p_format = format; return NULL; diff --git a/Python/getcompiler.c b/Python/getcompiler.c index a5d26239e87..cc56ad8c895 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -3,6 +3,10 @@ #include "Python.h" +#ifdef _Py_COMPILER +# define COMPILER _Py_COMPILER +#endif + #ifndef COMPILER // Note the __clang__ conditional has to come before the __GNUC__ one because diff --git a/Python/getversion.c b/Python/getversion.c index 226b2f999a6..8d8bc6ea700 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -15,7 +15,7 @@ void _Py_InitVersion(void) } initialized = 1; #ifdef Py_GIL_DISABLED - const char *buildinfo_format = "%.80s experimental free-threading build (%.80s) %.80s"; + const char *buildinfo_format = "%.80s free-threading build (%.80s) %.80s"; #else const char *buildinfo_format = "%.80s (%.80s) %.80s"; #endif diff --git a/Python/hamt.c b/Python/hamt.c index f9bbf63961d..e372b1a1b4c 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -256,9 +256,9 @@ Debug ===== The HAMT datatype is accessible for testing purposes under the -`_testcapi` module: +`_testinternalcapi` module: - >>> from _testcapi import hamt + >>> from _testinternalcapi import hamt >>> h = hamt() >>> h2 = h.set('a', 2) >>> h3 = h2.set('b', 3) @@ -1176,7 +1176,7 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } if (key_or_null == NULL) { - if (PyUnicodeWriter_WriteUTF8(writer, "NULL:\n", -1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "NULL:\n", 6) < 0) { goto error; } @@ -1194,7 +1194,7 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } } - if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "\n", 1) < 0) { goto error; } } @@ -1915,7 +1915,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "\n", 1) < 0) { goto error; } } diff --git a/Python/import.c b/Python/import.c index afdc28eda31..e91c95b40d9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -309,13 +310,8 @@ PyImport_GetModule(PyObject *name) if not, create a new one and insert it in the modules dictionary. */ static PyObject * -import_add_module(PyThreadState *tstate, PyObject *name) +import_add_module_lock_held(PyObject *modules, PyObject *name) { - PyObject *modules = get_modules_dict(tstate, false); - if (modules == NULL) { - return NULL; - } - PyObject *m; if (PyMapping_GetOptionalItem(modules, name, &m) < 0) { return NULL; @@ -335,6 +331,21 @@ import_add_module(PyThreadState *tstate, PyObject *name) return m; } +static PyObject * +import_add_module(PyThreadState *tstate, PyObject *name) +{ + PyObject *modules = get_modules_dict(tstate, false); + if (modules == NULL) { + return NULL; + } + + PyObject *m; + Py_BEGIN_CRITICAL_SECTION(modules); + m = import_add_module_lock_held(modules, name); + Py_END_CRITICAL_SECTION(); + return m; +} + PyObject * PyImport_AddModuleRef(const char *name) { @@ -672,8 +683,8 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (6). first time (not found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() - C. _PyImport_GetModInitFunc(): load <module init func> + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load <module init func> D. _imp_create_dynamic_impl() -> import_run_extension() E. import_run_extension() -> _PyImport_RunModInitFunc() F. _PyImport_RunModInitFunc(): call <module init func> @@ -743,16 +754,19 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) A. noop - ...for multi-phase init modules: + ...for multi-phase init modules from PyModInit_* (PyModuleDef): (6). every time: A. _imp_create_dynamic_impl() -> import_find_extension() (not found) - B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc() - C. _PyImport_GetModInitFunc(): load <module init func> + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load <module init func> D. _imp_create_dynamic_impl() -> import_run_extension() E. import_run_extension() -> _PyImport_RunModInitFunc() F. _PyImport_RunModInitFunc(): call <module init func> G. import_run_extension() -> PyModule_FromDefAndSpec() + + PyModule_FromDefAndSpec(): + H. PyModule_FromDefAndSpec(): gather/check moduledef slots I. if there's a Py_mod_create slot: 1. PyModule_FromDefAndSpec(): call its function @@ -765,10 +779,29 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (10). every time: A. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() B. if mod->md_state == NULL (including if m_size == 0): - 1. exec_builtin_or_dynamic() -> PyModule_ExecDef() - 2. PyModule_ExecDef(): allocate mod->md_state + 1. exec_builtin_or_dynamic() -> PyModule_Exec() + 2. PyModule_Exec(): allocate mod->md_state 3. if there's a Py_mod_exec slot: - 1. PyModule_ExecDef(): call its function + 1. PyModule_Exec(): call its function + + + ...for multi-phase init modules from PyModExport_* (slots array): + + (6). every time: + + A. _imp_create_dynamic_impl() -> import_find_extension() (not found) + B. _imp_create_dynamic_impl() -> _PyImport_GetModuleExportHooks() + C. _PyImport_GetModuleExportHooks(): load <module export func> + D. _imp_create_dynamic_impl() -> import_run_modexport() + E. import_run_modexport(): call <module init func> + F. import_run_modexport() -> PyModule_FromSlotsAndSpec() + G. PyModule_FromSlotsAndSpec(): create temporary PyModuleDef-like + H. PyModule_FromSlotsAndSpec() -> PyModule_FromDefAndSpec() + + (PyModule_FromDefAndSpec behaves as for PyModInit_*, above) + + (10). every time: as for PyModInit_*, above + */ @@ -782,18 +815,13 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) substitute this (if the name actually matches). */ -#ifdef HAVE_THREAD_LOCAL -_Py_thread_local const char *pkgcontext = NULL; +static _Py_thread_local const char *pkgcontext = NULL; # undef PKGCONTEXT # define PKGCONTEXT pkgcontext -#endif const char * _PyImport_ResolveNameWithPackageContext(const char *name) { -#ifndef HAVE_THREAD_LOCAL - PyMutex_Lock(&EXTENSIONS.mutex); -#endif if (PKGCONTEXT != NULL) { const char *p = strrchr(PKGCONTEXT, '.'); if (p != NULL && strcmp(name, p+1) == 0) { @@ -801,23 +829,14 @@ _PyImport_ResolveNameWithPackageContext(const char *name) PKGCONTEXT = NULL; } } -#ifndef HAVE_THREAD_LOCAL - PyMutex_Unlock(&EXTENSIONS.mutex); -#endif return name; } const char * _PyImport_SwapPackageContext(const char *newcontext) { -#ifndef HAVE_THREAD_LOCAL - PyMutex_Lock(&EXTENSIONS.mutex); -#endif const char *oldcontext = PKGCONTEXT; PKGCONTEXT = newcontext; -#ifndef HAVE_THREAD_LOCAL - PyMutex_Unlock(&EXTENSIONS.mutex); -#endif return oldcontext; } @@ -839,25 +858,19 @@ _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val) /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ static int exec_builtin_or_dynamic(PyObject *mod) { - PyModuleDef *def; void *state; if (!PyModule_Check(mod)) { return 0; } - def = PyModule_GetDef(mod); - if (def == NULL) { - return 0; - } - state = PyModule_GetState(mod); if (state) { /* Already initialized; skip reload */ return 0; } - return PyModule_ExecDef(mod, def); + return PyModule_Exec(mod); } @@ -1015,9 +1028,10 @@ struct extensions_cache_value { _Py_ext_module_origin origin; #ifdef Py_GIL_DISABLED - /* The module's md_gil slot, for legacy modules that are reinitialized from - m_dict rather than calling their initialization function again. */ - void *md_gil; + /* The module's md_requires_gil member, for legacy modules that are + * reinitialized from m_dict rather than calling their initialization + * function again. */ + bool md_requires_gil; #endif }; @@ -1348,7 +1362,7 @@ static struct extensions_cache_value * _extensions_cache_set(PyObject *path, PyObject *name, PyModuleDef *def, PyModInitFunction m_init, Py_ssize_t m_index, PyObject *m_dict, - _Py_ext_module_origin origin, void *md_gil) + _Py_ext_module_origin origin, bool requires_gil) { struct extensions_cache_value *value = NULL; void *key = NULL; @@ -1403,11 +1417,11 @@ _extensions_cache_set(PyObject *path, PyObject *name, /* m_dict is set by set_cached_m_dict(). */ .origin=origin, #ifdef Py_GIL_DISABLED - .md_gil=md_gil, + .md_requires_gil=requires_gil, #endif }; #ifndef Py_GIL_DISABLED - (void)md_gil; + (void)requires_gil; #endif if (init_cached_m_dict(newvalue, m_dict) < 0) { goto finally; @@ -1545,26 +1559,13 @@ _PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) } if (!PyModule_Check(module) || - ((PyModuleObject *)module)->md_gil == Py_MOD_GIL_USED) { - if (_PyEval_EnableGILPermanent(tstate)) { - int warn_result = PyErr_WarnFormat( - PyExc_RuntimeWarning, - 1, - "The global interpreter lock (GIL) has been enabled to load " - "module '%U', which has not declared that it can run safely " - "without the GIL. To override this behavior and keep the GIL " - "disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.", - module_name - ); - if (warn_result < 0) { - return warn_result; - } + ((PyModuleObject *)module)->md_requires_gil) + { + if (PyModule_Check(module)) { + assert(((PyModuleObject *)module)->md_token_is_def); } - - const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); - if (config->enable_gil == _PyConfig_GIL_DEFAULT && config->verbose) { - PySys_FormatStderr("# loading module '%U', which requires the GIL\n", - module_name); + if (_PyImport_EnableGILAndWarn(tstate, module_name) < 0) { + return -1; } } else { @@ -1573,6 +1574,28 @@ _PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) return 0; } + +int +_PyImport_EnableGILAndWarn(PyThreadState *tstate, PyObject *module_name) +{ + if (_PyEval_EnableGILPermanent(tstate)) { + return PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "The global interpreter lock (GIL) has been enabled to load " + "module '%U', which has not declared that it can run safely " + "without the GIL. To override this behavior and keep the GIL " + "disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.", + module_name + ); + } + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil == _PyConfig_GIL_DEFAULT && config->verbose) { + PySys_FormatStderr("# loading module '%U', which requires the GIL\n", + module_name); + } + return 0; +} #endif static PyThreadState * @@ -1723,7 +1746,7 @@ struct singlephase_global_update { Py_ssize_t m_index; PyObject *m_dict; _Py_ext_module_origin origin; - void *md_gil; + bool md_requires_gil; }; static struct extensions_cache_value * @@ -1782,7 +1805,7 @@ update_global_state_for_extension(PyThreadState *tstate, #endif cached = _extensions_cache_set( path, name, def, m_init, singlephase->m_index, m_dict, - singlephase->origin, singlephase->md_gil); + singlephase->origin, singlephase->md_requires_gil); if (cached == NULL) { // XXX Ignore this error? Doing so would effectively // mark the module as not loadable. @@ -1801,7 +1824,7 @@ finish_singlephase_extension(PyThreadState *tstate, PyObject *mod, PyObject *name, PyObject *modules) { assert(mod != NULL && PyModule_Check(mod)); - assert(cached->def == _PyModule_GetDef(mod)); + assert(cached->def == _PyModule_GetDefOrNull(mod)); Py_ssize_t index = _get_cached_module_index(cached); if (_modules_by_index_set(tstate->interp, index, mod) < 0) { @@ -1871,7 +1894,7 @@ reload_singlephase_extension(PyThreadState *tstate, if (def->m_base.m_copy != NULL) { // For non-core modules, fetch the GIL slot that was stored by // import_run_extension(). - ((PyModuleObject *)mod)->md_gil = cached->md_gil; + ((PyModuleObject *)mod)->md_requires_gil = cached->md_requires_gil; } #endif /* We can't set mod->md_def if it's missing, @@ -1879,8 +1902,8 @@ reload_singlephase_extension(PyThreadState *tstate, * due to violating interpreter isolation. * See the note in set_cached_m_dict(). * Until that is solved, we leave md_def set to NULL. */ - assert(_PyModule_GetDef(mod) == NULL - || _PyModule_GetDef(mod) == def); + assert(_PyModule_GetDefOrNull(mod) == NULL + || _PyModule_GetDefOrNull(mod) == def); } else { assert(cached->m_dict == NULL); @@ -1967,6 +1990,43 @@ import_find_extension(PyThreadState *tstate, return mod; } +static PyObject * +import_run_modexport(PyThreadState *tstate, PyModExportFunction ex0, + struct _Py_ext_module_loader_info *info, + PyObject *spec) +{ + /* This is like import_run_extension, but avoids interpreter switching + * and code for for single-phase modules. + */ + PyModuleDef_Slot *slots = ex0(); + if (!slots) { + if (!PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "slot export function for module %s failed without setting an exception", + info->name); + } + return NULL; + } + if (PyErr_Occurred()) { + PyErr_Format( + PyExc_SystemError, + "slot export function for module %s raised unreported exception", + info->name); + } + PyObject *result = PyModule_FromSlotsAndSpec(slots, spec); + if (!result) { + return NULL; + } + if (PyModule_Check(result)) { + PyModuleObject *mod = (PyModuleObject *)result; + if (mod && !mod->md_token) { + mod->md_token = slots; + } + } + return result; +} + static PyObject * import_run_extension(PyThreadState *tstate, PyModInitFunction p0, struct _Py_ext_module_loader_info *info, @@ -2089,7 +2149,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, .m_index=def->m_base.m_index, .origin=info->origin, #ifdef Py_GIL_DISABLED - .md_gil=((PyModuleObject *)mod)->md_gil, + .md_requires_gil=((PyModuleObject *)mod)->md_requires_gil, #endif }; // gh-88216: Extensions and def->m_base.m_copy can be updated @@ -2139,7 +2199,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, assert_multiphase_def(def); assert(mod == NULL); /* Note that we cheat a little by not repeating the calls - * to _PyImport_GetModInitFunc() and _PyImport_RunModInitFunc(). */ + * to _PyImport_GetModuleExportHooks() and _PyImport_RunModInitFunc(). */ mod = PyModule_FromDefAndSpec(def, spec); if (mod == NULL) { goto error; @@ -2253,8 +2313,9 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, return -1; } - PyModuleDef *def = PyModule_GetDef(mod); + PyModuleDef *def = _PyModule_GetDefOrNull(mod); if (def == NULL) { + assert(!PyErr_Occurred()); PyErr_BadInternalCall(); goto finally; } @@ -2283,7 +2344,7 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, .origin=_Py_ext_module_origin_CORE, #ifdef Py_GIL_DISABLED /* Unused when m_dict == NULL. */ - .md_gil=NULL, + .md_requires_gil=false, #endif }; cached = update_global_state_for_extension( @@ -2322,8 +2383,23 @@ is_builtin(PyObject *name) return 0; } +static PyModInitFunction +lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info) +{ + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(info->name, p->name)) { + return (PyModInitFunction)p->initfunc; + } + } + // not found + return NULL; +} + static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +create_builtin( + PyThreadState *tstate, PyObject *name, + PyObject *spec, + PyModInitFunction initfunc) { struct _Py_ext_module_loader_info info; if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) { @@ -2336,8 +2412,8 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) assert(!_PyErr_Occurred(tstate)); assert(cached != NULL); /* The module might not have md_def set in certain reload cases. */ - assert(_PyModule_GetDef(mod) == NULL - || cached->def == _PyModule_GetDef(mod)); + assert(_PyModule_GetDefOrNull(mod) == NULL + || cached->def == _PyModule_GetDefOrNull(mod)); assert_singlephase(cached); goto finally; } @@ -2354,24 +2430,15 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) _extensions_cache_delete(info.path, info.name); } - struct _inittab *found = NULL; - for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { - found = p; - } - } - if (found == NULL) { - // not found - mod = Py_NewRef(Py_None); - goto finally; - } - - PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + PyModInitFunction p0 = initfunc; if (p0 == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - assert(is_core_module(tstate->interp, info.name, info.path)); - mod = import_add_module(tstate, info.name); - goto finally; + p0 = lookup_inittab_initfunc(&info); + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; + } } #ifdef Py_GIL_DISABLED @@ -2397,6 +2464,33 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) return mod; } +PyObject* +PyImport_CreateModuleFromInitfunc( + PyObject *spec, PyObject *(*initfunc)(void)) +{ + if (initfunc == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name)); + if (name == NULL) { + return NULL; + } + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "spec name must be string, not %T", name); + Py_DECREF(name); + return NULL; + } + + PyObject *mod = create_builtin(tstate, name, spec, initfunc); + Py_DECREF(name); + return mod; +} /*****************************/ /* the builtin modules table */ @@ -3166,7 +3260,7 @@ bootstrap_imp(PyThreadState *tstate) } // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_CLEAR(name); Py_DECREF(spec); if (mod == NULL) { @@ -3369,11 +3463,11 @@ PyObject * PyImport_GetImporter(PyObject *path) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *path_importer_cache = _PySys_GetRequiredAttrString("path_importer_cache"); + PyObject *path_importer_cache = PySys_GetAttrString("path_importer_cache"); if (path_importer_cache == NULL) { return NULL; } - PyObject *path_hooks = _PySys_GetRequiredAttrString("path_hooks"); + PyObject *path_hooks = PySys_GetAttrString("path_hooks"); if (path_hooks == NULL) { Py_DECREF(path_importer_cache); return NULL; @@ -3434,8 +3528,10 @@ PyImport_ImportModule(const char *name) * ImportError instead of blocking. * * Returns the module object with incremented ref count. + * + * Removed in 3.15, but kept for stable ABI compatibility. */ -PyObject * +PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock(const char *name) { if (PyErr_WarnEx(PyExc_DeprecationWarning, @@ -3679,33 +3775,6 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyTime_t t1 = 0, accumulated_copy = accumulated; - PyObject *sys_path, *sys_meta_path, *sys_path_hooks; - if (_PySys_GetOptionalAttrString("path", &sys_path) < 0) { - return NULL; - } - if (_PySys_GetOptionalAttrString("meta_path", &sys_meta_path) < 0) { - Py_XDECREF(sys_path); - return NULL; - } - if (_PySys_GetOptionalAttrString("path_hooks", &sys_path_hooks) < 0) { - Py_XDECREF(sys_meta_path); - Py_XDECREF(sys_path); - return NULL; - } - if (_PySys_Audit(tstate, "import", "OOOOO", - abs_name, Py_None, sys_path ? sys_path : Py_None, - sys_meta_path ? sys_meta_path : Py_None, - sys_path_hooks ? sys_path_hooks : Py_None) < 0) { - Py_XDECREF(sys_path_hooks); - Py_XDECREF(sys_meta_path); - Py_XDECREF(sys_path); - return NULL; - } - Py_XDECREF(sys_path_hooks); - Py_XDECREF(sys_meta_path); - Py_XDECREF(sys_path); - - /* XOptions is initialized after first some imports. * So we can't have negative cache before completed initialization. * Anyway, importlib._find_and_load is much slower than @@ -3852,15 +3921,17 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } final_mod = import_get_module(tstate, to_return); - Py_DECREF(to_return); if (final_mod == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_KeyError, "%R not in sys.modules as expected", to_return); } + Py_DECREF(to_return); goto error; } + + Py_DECREF(to_return); } } else { @@ -3956,23 +4027,28 @@ PyImport_Import(PyObject *module_name) } /* Get the builtins from current globals */ - globals = PyEval_GetGlobals(); + globals = PyEval_GetGlobals(); // borrowed if (globals != NULL) { Py_INCREF(globals); + // XXX Use _PyEval_EnsureBuiltins()? builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); - if (builtins == NULL) + if (builtins == NULL) { + // XXX Fall back to interp->builtins or sys.modules['builtins']? goto err; + } + } + else if (_PyErr_Occurred(tstate)) { + goto err; } else { /* No globals -- use standard builtins, and fake globals */ - builtins = PyImport_ImportModuleLevel("builtins", - NULL, NULL, NULL, 0); - if (builtins == NULL) { + globals = PyDict_New(); + if (globals == NULL) { goto err; } - globals = Py_BuildValue("{OO}", &_Py_ID(__builtins__), builtins); - if (globals == NULL) + if (_PyEval_EnsureBuiltinsWithModule(tstate, globals, &builtins) < 0) { goto err; + } } /* Get the __import__ function from the builtins */ @@ -4123,7 +4199,7 @@ _PyImport_FiniCore(PyInterpreterState *interp) static int init_zipimport(PyThreadState *tstate, int verbose) { - PyObject *path_hooks = _PySys_GetRequiredAttrString("path_hooks"); + PyObject *path_hooks = PySys_GetAttrString("path_hooks"); if (path_hooks == NULL) { return -1; } @@ -4250,6 +4326,7 @@ _imp_lock_held_impl(PyObject *module) } /*[clinic input] +@permit_long_docstring_body _imp.acquire_lock Acquires the interpreter's import lock for the current thread. @@ -4260,7 +4337,7 @@ modules. On platforms without threads, this function does nothing. static PyObject * _imp_acquire_lock_impl(PyObject *module) -/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ +/*[clinic end generated code: output=1aff58cb0ee1b026 input=e1a4ef049d34e7dd]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyImport_AcquireLock(interp); @@ -4343,7 +4420,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_DECREF(name); return mod; } @@ -4683,8 +4760,8 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) assert(!_PyErr_Occurred(tstate)); assert(cached != NULL); /* The module might not have md_def set in certain reload cases. */ - assert(_PyModule_GetDef(mod) == NULL - || cached->def == _PyModule_GetDef(mod)); + assert(_PyModule_GetDefOrNull(mod) == NULL + || cached->def == _PyModule_GetDefOrNull(mod)); assert_singlephase(cached); goto finally; } @@ -4709,7 +4786,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } /* We would move this (and the fclose() below) into - * _PyImport_GetModInitFunc(), but it isn't clear if the intervening + * _PyImport_GetModuleExportHooks(), but it isn't clear if the intervening * code relies on fp still being open. */ FILE *fp; if (file != NULL) { @@ -4722,7 +4799,15 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) fp = NULL; } - PyModInitFunction p0 = _PyImport_GetModInitFunc(&info, fp); + PyModInitFunction p0 = NULL; + PyModExportFunction ex0 = NULL; + _PyImport_GetModuleExportHooks(&info, fp, &p0, &ex0); + if (ex0) { + mod = import_run_modexport(tstate, ex0, &info, spec); + // Modules created from slots handle GIL enablement (Py_mod_gil slot) + // when they're created. + goto cleanup; + } if (p0 == NULL) { goto finally; } @@ -4744,6 +4829,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } #endif +cleanup: // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? if (fp) { fclose(fp); diff --git a/Python/importdl.c b/Python/importdl.c index 802843fe7b9..537e8d869dc 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -5,38 +5,19 @@ #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_import.h" // _PyImport_SwapPackageContext() #include "pycore_importdl.h" -#include "pycore_moduleobject.h" // _PyModule_GetDef() +#include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_runtime.h" // _Py_ID() -/* ./configure sets HAVE_DYNAMIC_LOADING if dynamic loading of modules is - supported on this platform. configure will then compile and link in one - of the dynload_*.c files, as appropriate. We will call a function in - those modules to get a function pointer to the module's init function. -*/ -#ifdef HAVE_DYNAMIC_LOADING - -#ifdef MS_WINDOWS -extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, - const char *shortname, - PyObject *pathname, - FILE *fp); -#else -extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, - const char *shortname, - const char *pathname, FILE *fp); -#endif - -#endif /* HAVE_DYNAMIC_LOADING */ - - /***********************************/ /* module info to use when loading */ /***********************************/ -static const char * const ascii_only_prefix = "PyInit"; -static const char * const nonascii_prefix = "PyInitU"; +static const struct hook_prefixes ascii_only_prefixes = { + "PyInit", "PyModExport"}; +static const struct hook_prefixes nonascii_prefixes = { + "PyInitU", "PyModExportU"}; /* Get the variable part of a module's export symbol name. * Returns a bytes instance. For non-ASCII-named modules, the name is @@ -45,7 +26,7 @@ static const char * const nonascii_prefix = "PyInitU"; * nonascii_prefix, as appropriate. */ static PyObject * -get_encoded_name(PyObject *name, const char **hook_prefix) { +get_encoded_name(PyObject *name, const struct hook_prefixes **hook_prefixes) { PyObject *tmp; PyObject *encoded = NULL; PyObject *modname = NULL; @@ -72,7 +53,7 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { /* Encode to ASCII or Punycode, as needed */ encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); if (encoded != NULL) { - *hook_prefix = ascii_only_prefix; + *hook_prefixes = &ascii_only_prefixes; } else { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) { PyErr_Clear(); @@ -80,7 +61,7 @@ get_encoded_name(PyObject *name, const char **hook_prefix) { if (encoded == NULL) { goto error; } - *hook_prefix = nonascii_prefix; + *hook_prefixes = &nonascii_prefixes; } else { goto error; } @@ -130,7 +111,7 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, assert(PyUnicode_GetLength(name) > 0); info.name = Py_NewRef(name); - info.name_encoded = get_encoded_name(info.name, &info.hook_prefix); + info.name_encoded = get_encoded_name(info.name, &info.hook_prefixes); if (info.name_encoded == NULL) { _Py_ext_module_loader_info_clear(&info); return -1; @@ -175,7 +156,6 @@ _Py_ext_module_loader_info_init_for_builtin( PyObject *name) { assert(PyUnicode_Check(name)); - assert(PyUnicode_FindChar(name, '.', 0, PyUnicode_GetLength(name), -1) == -1); assert(PyUnicode_GetLength(name) > 0); PyObject *name_encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); @@ -189,7 +169,7 @@ _Py_ext_module_loader_info_init_for_builtin( /* We won't need filename. */ .path=name, .origin=_Py_ext_module_origin_BUILTIN, - .hook_prefix=ascii_only_prefix, + .hook_prefixes=&ascii_only_prefixes, .newcontext=NULL, }; return 0; @@ -377,39 +357,66 @@ _Py_ext_module_loader_result_apply_error( /********************************************/ #ifdef HAVE_DYNAMIC_LOADING -PyModInitFunction -_PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, - FILE *fp) +static dl_funcptr +findfuncptr(const char *prefix, const char *name_buf, + struct _Py_ext_module_loader_info *info, + FILE *fp) { +#ifdef MS_WINDOWS + return _PyImport_FindSharedFuncptrWindows( + prefix, name_buf, info->filename, fp); +#else + const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); + return _PyImport_FindSharedFuncptr( + prefix, name_buf, path_buf, fp); +#endif +} + +int +_PyImport_GetModuleExportHooks( + struct _Py_ext_module_loader_info *info, + FILE *fp, + PyModInitFunction *modinit, + PyModExportFunction *modexport) +{ + *modinit = NULL; + *modexport = NULL; + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); dl_funcptr exportfunc; -#ifdef MS_WINDOWS - exportfunc = _PyImport_FindSharedFuncptrWindows( - info->hook_prefix, name_buf, info->filename, fp); -#else - { - const char *path_buf = PyBytes_AS_STRING(info->filename_encoded); - exportfunc = _PyImport_FindSharedFuncptr( - info->hook_prefix, name_buf, path_buf, fp); - } -#endif - if (exportfunc == NULL) { - if (!PyErr_Occurred()) { - PyObject *msg; - msg = PyUnicode_FromFormat( - "dynamic module does not define " - "module export function (%s_%s)", - info->hook_prefix, name_buf); - if (msg != NULL) { - PyErr_SetImportError(msg, info->name, info->filename); - Py_DECREF(msg); - } + exportfunc = findfuncptr( + info->hook_prefixes->export_prefix, + name_buf, info, fp); + if (exportfunc) { + *modexport = (PyModExportFunction)exportfunc; + return 2; + } + if (PyErr_Occurred()) { + return -1; + } + + exportfunc = findfuncptr( + info->hook_prefixes->init_prefix, + name_buf, info, fp); + if (exportfunc) { + *modinit = (PyModInitFunction)exportfunc; + return 1; + } + + if (!PyErr_Occurred()) { + PyObject *msg; + msg = PyUnicode_FromFormat( + "dynamic module does not define " + "module export function (%s_%s or %s_%s)", + info->hook_prefixes->export_prefix, name_buf, + info->hook_prefixes->init_prefix, name_buf); + if (msg != NULL) { + PyErr_SetImportError(msg, info->name, info->filename); + Py_DECREF(msg); } - return NULL; } - - return (PyModInitFunction)exportfunc; + return -1; } #endif /* HAVE_DYNAMIC_LOADING */ @@ -477,7 +484,7 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, res.def = (PyModuleDef *)m; /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } - else if (info->hook_prefix == nonascii_prefix) { + else if (info->hook_prefixes == &nonascii_prefixes) { /* Non-ASCII is only supported for multi-phase init. */ res.kind = _Py_ext_module_kind_MULTIPHASE; /* Don't allow legacy init for non-ASCII module names. */ @@ -496,7 +503,7 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, goto error; } - res.def = _PyModule_GetDef(m); + res.def = _PyModule_GetDefOrNull(m); if (res.def == NULL) { PyErr_Clear(); _Py_ext_module_loader_result_set_error( diff --git a/Python/index_pool.c b/Python/index_pool.c index 007c81a0fc1..520a65938ec 100644 --- a/Python/index_pool.c +++ b/Python/index_pool.c @@ -172,6 +172,9 @@ _PyIndexPool_AllocIndex(_PyIndexPool *pool) else { index = heap_pop(free_indices); } + + pool->tlbc_generation++; + UNLOCK_POOL(pool); return index; } @@ -180,6 +183,7 @@ void _PyIndexPool_FreeIndex(_PyIndexPool *pool, int32_t index) { LOCK_POOL(pool); + pool->tlbc_generation++; heap_add(&pool->free_indices, index); UNLOCK_POOL(pool); } diff --git a/Python/initconfig.c b/Python/initconfig.c index e8270911721..7176670c110 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -239,9 +239,11 @@ static const PyConfigSpec PYPRECONFIG_SPEC[] = { // Forward declarations -static PyObject* -config_get(const PyConfig *config, const PyConfigSpec *spec, - int use_sys); +static PyObject* config_get(const PyConfig *config, const PyConfigSpec *spec, + int use_sys); +static void initconfig_free_wstr(wchar_t *member); +static void initconfig_free_wstr_list(PyWideStringList *list); +static void initconfig_free_config(const PyConfig *config); /* --- Command line options --------------------------------------- */ @@ -256,6 +258,7 @@ static const char usage_help[] = "\ Options (and corresponding environment variables):\n\ -b : issue warnings about converting bytes/bytearray to str and comparing\n\ bytes/bytearray with str or bytes with int. (-bb: issue errors)\n\ + deprecated since 3.15 and will become no-op in 3.17.\n\ -B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ -c cmd : program passed in as string (terminates option list)\n\ -d : turn on parser debugging output (for experts only, only works on\n\ @@ -312,7 +315,7 @@ The following implementation-specific options are available:\n\ "-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" #endif "\ --X importtime[=2]: show how long each import takes; use -X importtime=2 to\ +-X importtime[=2]: show how long each import takes; use -X importtime=2 to\n\ log imports of already-loaded modules; also PYTHONPROFILEIMPORTTIME\n\ -X int_max_str_digits=N: limit the size of int<->str conversions;\n\ 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ @@ -459,7 +462,7 @@ static const char usage_envvars[] = /* --- Global configuration variables ----------------------------- */ -/* UTF-8 mode (PEP 540): if equals to 1, use the UTF-8 encoding, and change +/* UTF-8 mode (PEP 540): if equal to 1, use the UTF-8 encoding, and change stdin and stdout error handler to "surrogateescape". */ int Py_UTF8Mode = 0; int Py_DebugFlag = 0; /* Needed by parser.c */ @@ -2807,12 +2810,6 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime) return _PyStatus_NO_MEMORY(); } -#ifdef Py_STATS - if (config->_pystats) { - _Py_StatsOn(); - } -#endif - return _PyStatus_OK(); } @@ -2962,8 +2959,6 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, /* option handled by _PyPreCmdline_Read() */ break; - /* case 'J': reserved for Jython */ - case 'O': config->optimization_level++; break; @@ -3649,7 +3644,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) #define DUMP_SYS(NAME) \ do { \ PySys_FormatStderr(" sys.%s = ", #NAME); \ - if (_PySys_GetOptionalAttrString(#NAME, &obj) < 0) { \ + if (PySys_GetOptionalAttrString(#NAME, &obj) < 0) { \ PyErr_Clear(); \ } \ if (obj != NULL) { \ @@ -3673,7 +3668,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) #undef DUMP_SYS PyObject *sys_path; - (void) _PySys_GetOptionalAttrString("path", &sys_path); + (void) PySys_GetOptionalAttrString("path", &sys_path); if (sys_path != NULL && PyList_Check(sys_path)) { PySys_WriteStderr(" sys.path = [\n"); Py_ssize_t len = PyList_GET_SIZE(sys_path); @@ -3727,6 +3722,9 @@ PyInitConfig_Free(PyInitConfig *config) if (config == NULL) { return; } + + initconfig_free_config(&config->config); + PyMem_RawFree(config->inittab); free(config->err_msg); free(config); } @@ -4095,13 +4093,51 @@ PyInitConfig_SetStr(PyInitConfig *config, const char *name, const char* value) } +static void +initconfig_free_wstr(wchar_t *member) +{ + if (member) { + free(member); + } +} + + +static void +initconfig_free_wstr_list(PyWideStringList *list) +{ + for (Py_ssize_t i = 0; i < list->length; i++) { + free(list->items[i]); + } + free(list->items); +} + + +static void +initconfig_free_config(const PyConfig *config) +{ + const PyConfigSpec *spec = PYCONFIG_SPEC; + for (; spec->name != NULL; spec++) { + void *member = config_get_spec_member(config, spec); + if (spec->type == PyConfig_MEMBER_WSTR + || spec->type == PyConfig_MEMBER_WSTR_OPT) + { + wchar_t *wstr = *(wchar_t **)member; + initconfig_free_wstr(wstr); + } + else if (spec->type == PyConfig_MEMBER_WSTR_LIST) { + initconfig_free_wstr_list(member); + } + } +} + + static int -_PyWideStringList_FromUTF8(PyInitConfig *config, PyWideStringList *list, - Py_ssize_t length, char * const *items) +initconfig_set_str_list(PyInitConfig *config, PyWideStringList *list, + Py_ssize_t length, char * const *items) { PyWideStringList wlist = _PyWideStringList_INIT; size_t size = sizeof(wchar_t*) * length; - wlist.items = (wchar_t **)PyMem_RawMalloc(size); + wlist.items = (wchar_t **)malloc(size); if (wlist.items == NULL) { config->status = _PyStatus_NO_MEMORY(); return -1; @@ -4110,14 +4146,14 @@ _PyWideStringList_FromUTF8(PyInitConfig *config, PyWideStringList *list, for (Py_ssize_t i = 0; i < length; i++) { wchar_t *arg = utf8_to_wstr(config, items[i]); if (arg == NULL) { - _PyWideStringList_Clear(&wlist); + initconfig_free_wstr_list(&wlist); return -1; } wlist.items[i] = arg; wlist.length++; } - _PyWideStringList_Clear(list); + initconfig_free_wstr_list(list); *list = wlist; return 0; } @@ -4138,7 +4174,7 @@ PyInitConfig_SetStrList(PyInitConfig *config, const char *name, return -1; } PyWideStringList *list = raw_member; - if (_PyWideStringList_FromUTF8(config, list, length, items) < 0) { + if (initconfig_set_str_list(config, list, length, items) < 0) { return -1; } @@ -4296,7 +4332,7 @@ _PyConfig_CreateXOptionsDict(const PyConfig *config) static int config_get_sys_write_bytecode(const PyConfig *config, int *value) { - PyObject *attr = _PySys_GetRequiredAttrString("dont_write_bytecode"); + PyObject *attr = PySys_GetAttrString("dont_write_bytecode"); if (attr == NULL) { return -1; } @@ -4317,7 +4353,7 @@ config_get(const PyConfig *config, const PyConfigSpec *spec, { if (use_sys) { if (spec->sys.attr != NULL) { - return _PySys_GetRequiredAttrString(spec->sys.attr); + return PySys_GetAttrString(spec->sys.attr); } if (strcmp(spec->name, "write_bytecode") == 0) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 13bdd041bec..9e750433cff 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -18,6 +18,7 @@ #include "pycore_tuple.h" // _PyTuple_FromArraySteal() #include "opcode_ids.h" +#include "pycore_optimizer.h" /* Uncomment this to dump debugging output when assertions fail */ @@ -190,7 +191,7 @@ is_instrumented(int opcode) { assert(opcode != 0); assert(opcode != RESERVED); - return opcode != ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; + return opcode < ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; } #ifndef NDEBUG @@ -525,7 +526,7 @@ valid_opcode(int opcode) if (IS_VALID_OPCODE(opcode) && opcode != CACHE && opcode != RESERVED && - opcode < 255) + opcode < 254) { return true; } @@ -1040,6 +1041,8 @@ set_version_raw(uintptr_t *ptr, uint32_t version) static void set_global_version(PyThreadState *tstate, uint32_t version) { + ASSERT_WORLD_STOPPED(); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); PyInterpreterState *interp = tstate->interp; set_version_raw(&interp->ceval.instrumentation_version, version); @@ -1190,9 +1193,10 @@ call_instrumentation_vector( break; } else { - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_tools(code, offset, event, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } } @@ -1381,9 +1385,10 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } else { /* DISABLE */ - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_line_tools(code, i, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } while (tools); Py_DECREF(line_obj); @@ -1438,9 +1443,10 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* } else { /* DISABLE */ - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_per_instruction_tools(code, offset, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } Py_DECREF(offset_obj); @@ -1530,7 +1536,7 @@ initialize_lines(PyCodeObject *code, int bytes_per_entry) case END_FOR: case END_SEND: case RESUME: - case POP_ITER: + case POP_ITER: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * END_SEND cannot start a line, as it is skipped by SEND * RESUME and POP_ITER must not be instrumented with INSTRUMENTED_LINE */ @@ -1780,6 +1786,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), code); #endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation @@ -1936,28 +1943,26 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) static int -instrument_all_executing_code_objects(PyInterpreterState *interp) { +instrument_all_executing_code_objects(PyInterpreterState *interp) +{ ASSERT_WORLD_STOPPED(); - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - while (ts) { + int err = 0; + _Py_FOR_EACH_TSTATE_BEGIN(interp, ts) { _PyInterpreterFrame *frame = ts->current_frame; while (frame) { if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { - if (instrument_lock_held(_PyFrame_GetCode(frame), interp)) { - return -1; + err = instrument_lock_held(_PyFrame_GetCode(frame), interp); + if (err) { + goto done; } } frame = frame->previous; } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); } - return 0; +done: + _Py_FOR_EACH_TSTATE_END(interp); + return err; } static void @@ -2003,6 +2008,7 @@ check_tool(PyInterpreterState *interp, int tool_id) int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { + ASSERT_WORLD_STOPPED(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; @@ -2011,33 +2017,28 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) return -1; } - int res; - _PyEval_StopTheWorld(interp); uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { - res = 0; - goto done; + return 0; } - set_events(&interp->monitors, tool_id, events); uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (new_version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); - res = -1; - goto done; + return -1; } + set_events(&interp->monitors, tool_id, events); set_global_version(tstate, new_version); #ifdef _Py_TIER2 _Py_Executors_InvalidateAll(interp, 1); #endif - res = instrument_all_executing_code_objects(interp); -done: - _PyEval_StartTheWorld(interp); - return res; + return instrument_all_executing_code_objects(interp); } int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events) { + ASSERT_WORLD_STOPPED(); + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS)); @@ -2049,11 +2050,8 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent return -1; } - int res; - _PyEval_StopTheWorld(interp); if (allocate_instrumentation_data(code)) { - res = -1; - goto done; + return -1; } code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id]; @@ -2061,16 +2059,11 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { - res = 0; - goto done; + return 0; } set_local_events(local, tool_id, events); - res = force_instrument_lock_held(code, interp); - -done: - _PyEval_StartTheWorld(interp); - return res; + return force_instrument_lock_held(code, interp); } int @@ -2102,11 +2095,12 @@ int _PyMonitoring_ClearToolId(int tool_id) } } + _PyEval_StopTheWorld(interp); if (_PyMonitoring_SetEvents(tool_id, 0) < 0) { + _PyEval_StartTheWorld(interp); return -1; } - _PyEval_StopTheWorld(interp); uint32_t version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); @@ -2343,7 +2337,11 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); } - if (_PyMonitoring_SetEvents(tool_id, event_set)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int err = _PyMonitoring_SetEvents(tool_id, event_set); + _PyEval_StartTheWorld(interp); + if (err) { return NULL; } Py_RETURN_NONE; @@ -2424,7 +2422,11 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, return NULL; } - if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int err = _PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set); + _PyEval_StartTheWorld(interp); + if (err) { return NULL; } Py_RETURN_NONE; @@ -2558,18 +2560,22 @@ PyObject *_Py_CreateMonitoringObject(void) err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero()); if (err) goto error; PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); + assert(val != NULL); /* Can't return NULL because the int is small. */ err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "COVERAGE_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_PROFILER_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "PROFILER_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val); Py_DECREF(val); if (err) goto error; @@ -2991,9 +2997,10 @@ branch_handler_vectorcall( // Orphaned NOT_TAKEN -- Jump removed by the compiler return res; } - LOCK_CODE(code); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); remove_tools(code, offset, other_event, 1 << self->tool_id); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } return res; } diff --git a/Python/intrinsics.c b/Python/intrinsics.c index ff44ba0ee64..9cfc285c6a5 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -9,8 +9,6 @@ #include "pycore_intrinsics.h" // INTRINSIC_PRINT #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_sysmodule.h" // _PySys_GetRequiredAttr() -#include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_typevarobject.h" // _Py_make_typevar() #include "pycore_unicodeobject.h" // _PyUnicode_FromASCII() @@ -27,7 +25,7 @@ no_intrinsic1(PyThreadState* tstate, PyObject *unused) static PyObject * print_expr(PyThreadState* Py_UNUSED(ignored), PyObject *value) { - PyObject *hook = _PySys_GetRequiredAttr(&_Py_ID(displayhook)); + PyObject *hook = PySys_GetAttr(&_Py_ID(displayhook)); if (hook == NULL) { return NULL; } @@ -193,7 +191,7 @@ static PyObject * list_to_tuple(PyThreadState* unused, PyObject *v) { assert(PyList_Check(v)); - return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); + return PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); } static PyObject * diff --git a/Python/jit.c b/Python/jit.c index e232cc1f7d9..47d3d7a5d27 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -60,6 +60,10 @@ jit_error(const char *message) static unsigned char * jit_alloc(size_t size) { + if (size > PY_MAX_JIT_CODE_SIZE) { + jit_error("code too big; refactor bytecodes.c to keep uop size down, or reduce maximum trace length."); + return NULL; + } assert(size); assert(size % get_page_size() == 0); #ifdef MS_WINDOWS @@ -69,10 +73,6 @@ jit_alloc(size_t size) #else int flags = MAP_ANONYMOUS | MAP_PRIVATE; int prot = PROT_READ | PROT_WRITE; -# ifdef MAP_JIT - flags |= MAP_JIT; - prot |= PROT_EXEC; -# endif unsigned char *memory = mmap(NULL, size, prot, flags, -1, 0); int failed = memory == MAP_FAILED; #endif @@ -118,11 +118,8 @@ mark_executable(unsigned char *memory, size_t size) int old; int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); #else - int failed = 0; __builtin___clear_cache((char *)memory, (char *)memory + size); -#ifndef MAP_JIT - failed = mprotect(memory, size, PROT_EXEC | PROT_READ); -#endif + int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); #endif if (failed) { jit_error("unable to protect executable memory"); @@ -164,21 +161,29 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, uint8_t width) { assert(loc_start + width <= 32); + uint32_t temp_val; + // Use memcpy to safely read the value, avoiding potential alignment + // issues and strict aliasing violations. + memcpy(&temp_val, loc, sizeof(temp_val)); // Clear the bits we're about to patch: - *loc &= ~(((1ULL << width) - 1) << loc_start); - assert(get_bits(*loc, loc_start, width) == 0); + temp_val &= ~(((1ULL << width) - 1) << loc_start); + assert(get_bits(temp_val, loc_start, width) == 0); // Patch the bits: - *loc |= get_bits(value, value_start, width) << loc_start; - assert(get_bits(*loc, loc_start, width) == get_bits(value, value_start, width)); + temp_val |= get_bits(value, value_start, width) << loc_start; + assert(get_bits(temp_val, loc_start, width) == get_bits(value, value_start, width)); + // Safely write the modified value back to memory. + memcpy(loc, &temp_val, sizeof(temp_val)); } // See https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions // for instruction encodings: -#define IS_AARCH64_ADD_OR_SUB(I) (((I) & 0x11C00000) == 0x11000000) -#define IS_AARCH64_ADRP(I) (((I) & 0x9F000000) == 0x90000000) -#define IS_AARCH64_BRANCH(I) (((I) & 0x7C000000) == 0x14000000) -#define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) -#define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) +#define IS_AARCH64_ADD_OR_SUB(I) (((I) & 0x11C00000) == 0x11000000) +#define IS_AARCH64_ADRP(I) (((I) & 0x9F000000) == 0x90000000) +#define IS_AARCH64_BRANCH(I) (((I) & 0x7C000000) == 0x14000000) +#define IS_AARCH64_BRANCH_COND(I) (((I) & 0x7C000000) == 0x54000000) +#define IS_AARCH64_TEST_AND_BRANCH(I) (((I) & 0x7E000000) == 0x36000000) +#define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) +#define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) // LLD is a great reference for performing relocations... just keep in // mind that Tools/jit/build.py does filtering and preprocessing for us! @@ -209,30 +214,29 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, void patch_32(unsigned char *location, uint64_t value) { - uint32_t *loc32 = (uint32_t *)location; // Check that we're not out of range of 32 unsigned bits: assert(value < (1ULL << 32)); - *loc32 = (uint32_t)value; + uint32_t final_value = (uint32_t)value; + memcpy(location, &final_value, sizeof(final_value)); } // 32-bit relative address. void patch_32r(unsigned char *location, uint64_t value) { - uint32_t *loc32 = (uint32_t *)location; value -= (uintptr_t)location; // Check that we're not out of range of 32 signed bits: assert((int64_t)value >= -(1LL << 31)); assert((int64_t)value < (1LL << 31)); - *loc32 = (uint32_t)value; + uint32_t final_value = (uint32_t)value; + memcpy(location, &final_value, sizeof(final_value)); } // 64-bit absolute address. void patch_64(unsigned char *location, uint64_t value) { - uint64_t *loc64 = (uint64_t *)location; - *loc64 = value; + memcpy(location, &value, sizeof(value)); } // 12-bit low part of an absolute address. Pairs nicely with patch_aarch64_21r @@ -339,6 +343,21 @@ patch_aarch64_21rx(unsigned char *location, uint64_t value) patch_aarch64_21r(location, value); } +// 21-bit relative branch. +void +patch_aarch64_19r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_BRANCH_COND(*loc32)); + value -= (uintptr_t)location; + // Check that we're not out of range of 21 signed bits: + assert((int64_t)value >= -(1 << 20)); + assert((int64_t)value < (1 << 20)); + // Since instructions are 4-byte aligned, only use 19 bits: + assert(get_bits(value, 0, 2) == 0); + set_bits(loc32, 5, value, 2, 19); +} + // 28-bit relative branch. void patch_aarch64_26r(unsigned char *location, uint64_t value) @@ -400,7 +419,10 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) { uint8_t *loc8 = (uint8_t *)location; // Try to relax the GOT load into an immediate value: - uint64_t relaxed = *(uint64_t *)(value + 4) - 4; + uint64_t relaxed; + memcpy(&relaxed, (void *)(value + 4), sizeof(relaxed)); + relaxed -= 4; + if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) { @@ -426,15 +448,42 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) } void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); +void patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state); #include "jit_stencils.h" #if defined(__aarch64__) || defined(_M_ARM64) #define TRAMPOLINE_SIZE 16 + #define DATA_ALIGN 8 +#elif defined(__x86_64__) && defined(__APPLE__) + // LLVM 20 on macOS x86_64 debug builds: GOT entries may exceed ±2GB PC-relative + // range. + #define TRAMPOLINE_SIZE 16 // 14 bytes + 2 bytes padding for alignment + #define DATA_ALIGN 8 #else #define TRAMPOLINE_SIZE 0 + #define DATA_ALIGN 1 #endif +// Get the trampoline memory location for a given symbol ordinal. +static unsigned char * +get_trampoline_slot(int ordinal, jit_state *state) +{ + const uint32_t symbol_mask = 1 << (ordinal % 32); + const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; + assert(symbol_mask & trampoline_mask); + + // Count the number of set bits in the trampoline mask lower than ordinal + int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); + for (int i = 0; i < ordinal / 32; i++) { + index += _Py_popcount32(state->trampolines.mask[i]); + } + + unsigned char *trampoline = state->trampolines.mem + index * TRAMPOLINE_SIZE; + assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + return trampoline; +} + // Generate and patch AArch64 trampolines. The symbols to jump to are stored // in the jit_stencils.h in the symbols_map. void @@ -451,20 +500,8 @@ patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) return; } - // Masking is done modulo 32 as the mask is stored as an array of uint32_t - const uint32_t symbol_mask = 1 << (ordinal % 32); - const uint32_t trampoline_mask = state->trampolines.mask[ordinal / 32]; - assert(symbol_mask & trampoline_mask); - - // Count the number of set bits in the trampoline mask lower than ordinal, - // this gives the index into the array of trampolines. - int index = _Py_popcount32(trampoline_mask & (symbol_mask - 1)); - for (int i = 0; i < ordinal / 32; i++) { - index += _Py_popcount32(state->trampolines.mask[i]); - } - - uint32_t *p = (uint32_t*)(state->trampolines.mem + index * TRAMPOLINE_SIZE); - assert((size_t)(index + 1) * TRAMPOLINE_SIZE <= state->trampolines.size); + // Out of range - need a trampoline + uint32_t *p = (uint32_t *)get_trampoline_slot(ordinal, state); /* Generate the trampoline @@ -481,6 +518,37 @@ patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) patch_aarch64_26r(location, (uintptr_t)p); } +// Generate and patch x86_64 trampolines. +void +patch_x86_64_trampoline(unsigned char *location, int ordinal, jit_state *state) +{ + uint64_t value = (uintptr_t)symbols_map[ordinal]; + int64_t range = (int64_t)value - 4 - (int64_t)location; + + // If we are in range of 32 signed bits, we can patch directly + if (range >= -(1LL << 31) && range < (1LL << 31)) { + patch_32r(location, value - 4); + return; + } + + // Out of range - need a trampoline + unsigned char *trampoline = get_trampoline_slot(ordinal, state); + + /* Generate the trampoline (14 bytes, padded to 16): + 0: ff 25 00 00 00 00 jmp *(%rip) + 6: XX XX XX XX XX XX XX XX (64-bit target address) + + Reference: https://wiki.osdev.org/X86-64_Instruction_Encoding#FF (JMP r/m64) + */ + trampoline[0] = 0xFF; + trampoline[1] = 0x25; + memset(trampoline + 2, 0, 4); + memcpy(trampoline + 6, &value, 8); + + // Patch the call site to call the trampoline instead + patch_32r(location, (uintptr_t)trampoline - 4); +} + static void combine_symbol_mask(const symbol_mask src, symbol_mask dest) { @@ -499,10 +567,6 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz size_t code_size = 0; size_t data_size = 0; jit_state state = {0}; - group = &shim; - code_size += group->code_size; - data_size += group->data_size; - combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; @@ -522,15 +586,13 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz // Round up to the nearest page: size_t page_size = get_page_size(); assert((page_size & (page_size - 1)) == 0); - size_t padding = page_size - ((code_size + state.trampolines.size + data_size) & (page_size - 1)); - size_t total_size = code_size + state.trampolines.size + data_size + padding; + size_t code_padding = DATA_ALIGN - ((code_size + state.trampolines.size) & (DATA_ALIGN - 1)); + size_t padding = page_size - ((code_size + state.trampolines.size + code_padding + data_size) & (page_size - 1)); + size_t total_size = code_size + state.trampolines.size + code_padding + data_size + padding; unsigned char *memory = jit_alloc(total_size); if (memory == NULL) { return -1; } -#ifdef MAP_JIT - pthread_jit_write_protect_np(0); -#endif // Collect memory stats OPT_STAT_ADD(jit_total_memory_size, total_size); OPT_STAT_ADD(jit_code_size, code_size); @@ -545,15 +607,8 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz // Loop again to emit the code: unsigned char *code = memory; state.trampolines.mem = memory + code_size; - unsigned char *data = memory + code_size + state.trampolines.size; - // Compile the shim, which handles converting between the native - // calling convention and the calling convention used by jitted code - // (which may be different for efficiency reasons). - group = &shim; - group->emit(code, data, executor, NULL, &state); - code += group->code_size; - data += group->data_size; - assert(trace[0].opcode == _START_EXECUTOR); + unsigned char *data = memory + code_size + state.trampolines.size + code_padding; + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT || trace[0].opcode == _COLD_DYNAMIC_EXIT); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; @@ -567,20 +622,81 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz code += group->code_size; data += group->data_size; assert(code == memory + code_size); - assert(data == memory + code_size + state.trampolines.size + data_size); -#ifdef MAP_JIT - pthread_jit_write_protect_np(1); -#endif + assert(data == memory + code_size + state.trampolines.size + code_padding + data_size); if (mark_executable(memory, total_size)) { jit_free(memory, total_size); return -1; } executor->jit_code = memory; - executor->jit_side_entry = memory + shim.code_size; executor->jit_size = total_size; return 0; } +/* One-off compilation of the jit entry trampoline + * We compile this once only as it effectively a normal + * function, but we need to use the JIT because it needs + * to understand the jit-specific calling convention. + */ +static _PyJitEntryFuncPtr +compile_trampoline(void) +{ + _PyExecutorObject dummy; + const StencilGroup *group; + size_t code_size = 0; + size_t data_size = 0; + jit_state state = {0}; + group = &trampoline; + code_size += group->code_size; + data_size += group->data_size; + combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); + // Round up to the nearest page: + size_t page_size = get_page_size(); + assert((page_size & (page_size - 1)) == 0); + size_t code_padding = DATA_ALIGN - ((code_size + state.trampolines.size) & (DATA_ALIGN - 1)); + size_t padding = page_size - ((code_size + state.trampolines.size + code_padding + data_size) & (page_size - 1)); + size_t total_size = code_size + state.trampolines.size + code_padding + data_size + padding; + unsigned char *memory = jit_alloc(total_size); + if (memory == NULL) { + return NULL; + } + unsigned char *code = memory; + state.trampolines.mem = memory + code_size; + unsigned char *data = memory + code_size + state.trampolines.size + code_padding; + // Compile the shim, which handles converting between the native + // calling convention and the calling convention used by jitted code + // (which may be different for efficiency reasons). + group = &trampoline; + group->emit(code, data, &dummy, NULL, &state); + code += group->code_size; + data += group->data_size; + assert(code == memory + code_size); + assert(data == memory + code_size + state.trampolines.size + code_padding + data_size); + if (mark_executable(memory, total_size)) { + jit_free(memory, total_size); + return NULL; + } + return (_PyJitEntryFuncPtr)memory; +} + +static PyMutex lazy_jit_mutex = { 0 }; + +_Py_CODEUNIT * +_Py_LazyJitTrampoline( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate +) { + PyMutex_Lock(&lazy_jit_mutex); + if (_Py_jit_entry == _Py_LazyJitTrampoline) { + _PyJitEntryFuncPtr trampoline = compile_trampoline(); + if (trampoline == NULL) { + PyMutex_Unlock(&lazy_jit_mutex); + Py_FatalError("Cannot allocate core JIT code"); + } + _Py_jit_entry = trampoline; + } + PyMutex_Unlock(&lazy_jit_mutex); + return _Py_jit_entry(executor, frame, stack_pointer, tstate); +} + void _PyJIT_Free(_PyExecutorObject *executor) { @@ -588,7 +704,6 @@ _PyJIT_Free(_PyExecutorObject *executor) size_t size = executor->jit_size; if (memory) { executor->jit_code = NULL; - executor->jit_side_entry = NULL; executor->jit_size = 0; if (jit_free(memory, size)) { PyErr_FormatUnraisable("Exception ignored while " diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index dbd19d7755c..594d5c5ead5 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -20,13 +20,6 @@ typedef struct _PyLegacyEventHandler { #define _PyLegacyEventHandler_CAST(op) ((_PyLegacyEventHandler *)(op)) -#ifdef Py_GIL_DISABLED -#define LOCK_SETUP() PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#define UNLOCK_SETUP() PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#else -#define LOCK_SETUP() -#define UNLOCK_SETUP() -#endif /* The Py_tracefunc function expects the following arguments: * obj: the trace object (PyObject *) * frame: the current frame (PyFrameObject *) @@ -133,16 +126,10 @@ sys_profile_call_or_return( Py_RETURN_NONE; } -int -_PyEval_SetOpcodeTrace( - PyFrameObject *frame, - bool enable -) { - assert(frame != NULL); - - PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); +static int +set_opcode_trace_world_stopped(PyCodeObject *code, bool enable) +{ _PyMonitoringEventSet events = 0; - if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { return -1; } @@ -161,6 +148,32 @@ _PyEval_SetOpcodeTrace( return _PyMonitoring_SetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, events); } +int +_PyEval_SetOpcodeTrace(PyFrameObject *frame, bool enable) +{ + assert(frame != NULL); + + PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); + +#ifdef Py_GIL_DISABLED + // First check if a change is necessary outside of the stop-the-world pause + _PyMonitoringEventSet events = 0; + if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { + return -1; + } + int is_enabled = (events & (1 << PY_MONITORING_EVENT_INSTRUCTION)) != 0; + if (is_enabled == enable) { + return 0; // No change needed + } +#endif + + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int res = set_opcode_trace_world_stopped(code, enable); + _PyEval_StartTheWorld(interp); + return res; +} + static PyObject * call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) { @@ -438,59 +451,74 @@ is_tstate_valid(PyThreadState *tstate) } #endif -static Py_ssize_t -setup_profile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_profileobj) +static int +setup_profile_callbacks(void *Py_UNUSED(arg)) { - *old_profileobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ - if (!tstate->interp->sys_profile_initialized) { - tstate->interp->sys_profile_initialized = true; - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_start, PyTrace_CALL, - PY_MONITORING_EVENT_PY_START, - PY_MONITORING_EVENT_PY_RESUME)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_throw, PyTrace_CALL, - PY_MONITORING_EVENT_PY_THROW, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_return, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_RETURN, - PY_MONITORING_EVENT_PY_YIELD)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_unwind, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_UNWIND, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_CALL, - PY_MONITORING_EVENT_CALL, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_RETURN, - PY_MONITORING_EVENT_C_RETURN, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_EXCEPTION, - PY_MONITORING_EVENT_C_RAISE, -1)) { - return -1; - } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_start, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, + PY_MONITORING_EVENT_PY_RESUME)) { + return -1; } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_throw, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, + PY_MONITORING_EVENT_PY_YIELD)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_unwind, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_CALL, + PY_MONITORING_EVENT_CALL, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_RETURN, + PY_MONITORING_EVENT_C_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_EXCEPTION, + PY_MONITORING_EVENT_C_RAISE, -1)) { + return -1; + } + return 0; +} +static PyObject * +swap_profile_func_arg(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ int delta = (func != NULL) - (tstate->c_profilefunc != NULL); tstate->c_profilefunc = func; - *old_profileobj = tstate->c_profileobj; + PyObject *old_profileobj = tstate->c_profileobj; tstate->c_profileobj = Py_XNewRef(arg); tstate->interp->sys_profiling_threads += delta; assert(tstate->interp->sys_profiling_threads >= 0); - return tstate->interp->sys_profiling_threads; + return old_profileobj; +} + +static int +set_monitoring_profile_events(PyInterpreterState *interp) +{ + uint32_t events = 0; + if (interp->sys_profiling_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } int @@ -507,87 +535,155 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } - // needs to be decref'd outside of the lock - PyObject *old_profileobj; - LOCK_SETUP(); - Py_ssize_t profiling_threads = setup_profile(tstate, func, arg, &old_profileobj); - UNLOCK_SETUP(); - Py_XDECREF(old_profileobj); - - uint32_t events = 0; - if (profiling_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW); + PyInterpreterState *interp = tstate->interp; + if (_PyOnceFlag_CallOnce(&interp->sys_profile_once_flag, + setup_profile_callbacks, NULL) < 0) { + return -1; } - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); + + _PyEval_StopTheWorld(interp); + PyObject *old_profileobj = swap_profile_func_arg(tstate, func, arg); + int ret = set_monitoring_profile_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_profileobj); // needs to be decref'd outside of stop-the-world + return ret; } -static Py_ssize_t -setup_tracing(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_traceobj) +int +_PyEval_SetProfileAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg) { - *old_traceobj = NULL; - /* Setup PEP 669 monitoring callbacks and events. */ - if (!tstate->interp->sys_trace_initialized) { - tstate->interp->sys_trace_initialized = true; - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_start, PyTrace_CALL, - PY_MONITORING_EVENT_PY_START, - PY_MONITORING_EVENT_PY_RESUME)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_throw, PyTrace_CALL, - PY_MONITORING_EVENT_PY_THROW, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_return, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_RETURN, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_yield, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_YIELD, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_exception_func, PyTrace_EXCEPTION, - PY_MONITORING_EVENT_RAISE, - PY_MONITORING_EVENT_STOP_ITERATION)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_line_func, PyTrace_LINE, - PY_MONITORING_EVENT_LINE, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_unwind, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_UNWIND, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_jump_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_instruction_func, PyTrace_OPCODE, - PY_MONITORING_EVENT_INSTRUCTION, -1)) { - return -1; - } + PyThreadState *current_tstate = _PyThreadState_GET(); + assert(is_tstate_valid(current_tstate)); + assert(current_tstate->interp == interp); + + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { + return -1; } + if (_PyOnceFlag_CallOnce(&interp->sys_profile_once_flag, + setup_profile_callbacks, NULL) < 0) { + return -1; + } + + PyObject *old_profileobjs = NULL; + _PyEval_StopTheWorld(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t num_thread_states = 0; + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p) { + num_thread_states++; + } + old_profileobjs = PyTuple_New(num_thread_states); + if (old_profileobjs == NULL) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + return -1; + } + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + PyObject *old = swap_profile_func_arg(tstate, func, arg); + PyTuple_SET_ITEM(old_profileobjs, --num_thread_states, old); + } + HEAD_UNLOCK(&_PyRuntime); + int ret = set_monitoring_profile_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_profileobjs); // needs to be decref'd outside of stop-the-world + return ret; +} + +static int +setup_trace_callbacks(void *Py_UNUSED(arg)) +{ + /* Setup PEP 669 monitoring callbacks and events. */ + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_start, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, + PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_throw, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_yield, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_YIELD, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_exception_func, PyTrace_EXCEPTION, + PY_MONITORING_EVENT_RAISE, + PY_MONITORING_EVENT_STOP_ITERATION)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_line_func, PyTrace_LINE, + PY_MONITORING_EVENT_LINE, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_unwind, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_jump_func, PyTrace_LINE, + PY_MONITORING_EVENT_JUMP, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_instruction_func, PyTrace_OPCODE, + PY_MONITORING_EVENT_INSTRUCTION, -1)) { + return -1; + } + return 0; +} + +static PyObject * +swap_trace_func_arg(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = func; - *old_traceobj = tstate->c_traceobj; + PyObject *old_traceobj = tstate->c_traceobj; tstate->c_traceobj = Py_XNewRef(arg); tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); - return tstate->interp->sys_tracing_threads; + return old_traceobj; +} + +static int +set_monitoring_trace_events(PyInterpreterState *interp) +{ + uint32_t events = 0; + if (interp->sys_tracing_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | + (1 << PY_MONITORING_EVENT_JUMP) | + (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | + (1 << PY_MONITORING_EVENT_STOP_ITERATION); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); +} + +// Enable opcode tracing for the thread's current frame if needed. +static int +maybe_set_opcode_trace(PyThreadState *tstate) +{ + _PyInterpreterFrame *iframe = tstate->current_frame; + if (iframe == NULL) { + return 0; + } + PyFrameObject *frame = iframe->frame_obj; + if (frame == NULL || !frame->f_trace_opcodes) { + return 0; + } + return set_opcode_trace_world_stopped(_PyFrame_GetCode(iframe), true); } int @@ -603,35 +699,76 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { return -1; } - // needs to be decref'd outside of the lock - PyObject *old_traceobj; - LOCK_SETUP(); - assert(tstate->interp->sys_tracing_threads >= 0); - Py_ssize_t tracing_threads = setup_tracing(tstate, func, arg, &old_traceobj); - UNLOCK_SETUP(); - Py_XDECREF(old_traceobj); - if (tracing_threads < 0) { + + PyInterpreterState *interp = tstate->interp; + if (_PyOnceFlag_CallOnce(&interp->sys_trace_once_flag, + setup_trace_callbacks, NULL) < 0) { return -1; } - uint32_t events = 0; - if (tracing_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP) | - (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | - (1 << PY_MONITORING_EVENT_STOP_ITERATION); + int err = 0; + _PyEval_StopTheWorld(interp); + PyObject *old_traceobj = swap_trace_func_arg(tstate, func, arg); + err = set_monitoring_trace_events(interp); + if (err != 0) { + goto done; + } + if (interp->sys_tracing_threads) { + err = maybe_set_opcode_trace(tstate); + } +done: + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_traceobj); // needs to be decref'd outside stop-the-world + return err; +} - PyFrameObject* frame = PyEval_GetFrame(); - if (frame && frame->f_trace_opcodes) { - int ret = _PyEval_SetOpcodeTrace(frame, true); - if (ret != 0) { - return ret; +int +_PyEval_SetTraceAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg) +{ + PyThreadState *current_tstate = _PyThreadState_GET(); + assert(is_tstate_valid(current_tstate)); + assert(current_tstate->interp == interp); + + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + + if (_PyOnceFlag_CallOnce(&interp->sys_trace_once_flag, + setup_trace_callbacks, NULL) < 0) { + return -1; + } + + PyObject *old_trace_objs = NULL; + _PyEval_StopTheWorld(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t num_thread_states = 0; + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p) { + num_thread_states++; + } + old_trace_objs = PyTuple_New(num_thread_states); + if (old_trace_objs == NULL) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + return -1; + } + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + PyObject *old = swap_trace_func_arg(tstate, func, arg); + PyTuple_SET_ITEM(old_trace_objs, --num_thread_states, old); + } + if (interp->sys_tracing_threads) { + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + int err = maybe_set_opcode_trace(tstate); + if (err != 0) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_trace_objs); + return -1; } } } - - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); + HEAD_UNLOCK(&_PyRuntime); + int err = set_monitoring_trace_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_trace_objs); // needs to be decref'd outside of stop-the-world + return err; } diff --git a/Python/lock.c b/Python/lock.c index 28a12ad1835..789065d8162 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -6,6 +6,7 @@ #include "pycore_parking_lot.h" #include "pycore_semaphore.h" #include "pycore_time.h" // _PyTime_Add() +#include "pycore_stats.h" // FT_STAT_MUTEX_SLEEP_INC() #ifdef MS_WINDOWS # ifndef WIN32_LEAN_AND_MEAN @@ -58,10 +59,12 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_ACQUIRED; } } - else if (timeout == 0) { + if (timeout == 0) { return PY_LOCK_FAILURE; } + FT_STAT_MUTEX_SLEEP_INC(); + PyTime_t now; // silently ignore error: cannot report error to the caller (void)PyTime_MonotonicRaw(&now); @@ -95,6 +98,18 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) if (timeout == 0) { return PY_LOCK_FAILURE; } + if ((flags & _PY_LOCK_PYTHONLOCK) && Py_IsFinalizing()) { + // At this phase of runtime shutdown, only the finalization thread + // can have attached thread state; others hang if they try + // attaching. And since operations on this lock requires attached + // thread state (_PY_LOCK_PYTHONLOCK), the finalization thread is + // running this code, and no other thread can unlock. + // Raise rather than hang. (_PY_LOCK_PYTHONLOCK allows raising + // exceptons.) + PyErr_SetString(PyExc_PythonFinalizationError, + "cannot acquire lock at interpreter finalization"); + return PY_LOCK_FAILURE; + } uint8_t newv = v; if (!(v & _Py_HAS_PARKED)) { @@ -119,6 +134,9 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_INTR; } } + else if (ret == Py_PARK_INTR && (flags & _PY_FAIL_IF_INTERRUPTED)) { + return PY_LOCK_INTR; + } else if (ret == Py_PARK_TIMEOUT) { assert(timeout >= 0); return PY_LOCK_FAILURE; @@ -212,7 +230,7 @@ _PyRawMutex_LockSlow(_PyRawMutex *m) // Wait for us to be woken up. Note that we still have to lock the // mutex ourselves: it is NOT handed off to us. - _PySemaphore_Wait(&waiter.sema, -1, /*detach=*/0); + _PySemaphore_Wait(&waiter.sema, -1); } _PySemaphore_Destroy(&waiter.sema); @@ -619,3 +637,11 @@ PyMutex_Unlock(PyMutex *m) Py_FatalError("unlocking mutex that is not locked"); } } + + +#undef PyMutex_IsLocked +int +PyMutex_IsLocked(PyMutex *m) +{ + return _PyMutex_IsLocked(m); +} diff --git a/Python/marshal.c b/Python/marshal.c index b39c1a5b1ad..69d6dd7cf0f 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -11,6 +11,7 @@ #include "pycore_code.h" // _PyCode_New() #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_long.h" // _PyLong_IsZero() +#include "pycore_object.h" // _PyObject_IsUniquelyReferenced #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntryRef() #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() @@ -38,7 +39,7 @@ module marshal * On Windows PGO builds, the r_object function overallocates its stack and * can cause a stack overflow. We reduce the maximum depth for all Windows * releases to protect against this. - * #if defined(MS_WINDOWS) && defined(_DEBUG) + * #if defined(MS_WINDOWS) && defined(Py_DEBUG) */ #if defined(MS_WINDOWS) # define MAX_MARSHAL_STACK_DEPTH 1000 @@ -309,7 +310,7 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p) } if (!long_export.digits) { int8_t sign = long_export.value < 0 ? -1 : 1; - uint64_t abs_value = Py_ABS(long_export.value); + uint64_t abs_value = _Py_ABS_CAST(uint64_t, long_export.value); uint64_t d = abs_value; long l = 0; @@ -388,7 +389,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) * But we use TYPE_REF always for interned string, to PYC file stable * as possible. */ - if (Py_REFCNT(v) == 1 && + if (_PyObject_IsUniquelyReferenced(v) && !(PyUnicode_CheckExact(v) && PyUnicode_CHECK_INTERNED(v))) { return 0; } @@ -1656,6 +1657,9 @@ r_object(RFILE *p) case TYPE_SLICE: { Py_ssize_t idx = r_ref_reserve(flag, p); + if (idx < 0) { + break; + } PyObject *stop = NULL; PyObject *step = NULL; PyObject *start = r_object(p); @@ -1994,6 +1998,8 @@ marshal_load_impl(PyObject *module, PyObject *file, int allow_code) } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body marshal.dumps value: object @@ -2014,7 +2020,7 @@ unsupported type. static PyObject * marshal_dumps_impl(PyObject *module, PyObject *value, int version, int allow_code) -/*[clinic end generated code: output=115f90da518d1d49 input=167eaecceb63f0a8]*/ +/*[clinic end generated code: output=115f90da518d1d49 input=80cd3f30c1637ade]*/ { return _PyMarshal_WriteObjectToString(value, version, allow_code); } diff --git a/Python/modsupport.c b/Python/modsupport.c index 2caf595949d..239c6c6a1b3 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_object.h" // _PyType_IsReady() +#include "pycore_unicodeobject.h" // _PyUnicodeWriter_FormatV() typedef double va_double; @@ -33,6 +34,19 @@ _Py_convert_optional_to_ssize_t(PyObject *obj, void *result) return 1; } +int +_Py_convert_optional_to_non_negative_ssize_t(PyObject *obj, void *result) +{ + if (!_Py_convert_optional_to_ssize_t(obj, result)) { + return 0; + } + if (obj != Py_None && *((Py_ssize_t *)result) < 0) { + PyErr_SetString(PyExc_ValueError, "argument cannot be negative"); + return 0; + } + return 1; +} + /* Helper for mkvalue() to scan the length of a format */ @@ -655,6 +669,116 @@ PyModule_AddType(PyObject *module, PyTypeObject *type) return PyModule_AddObjectRef(module, name, (PyObject *)type); } +static int _abiinfo_raise(const char *module_name, const char *format, ...) +{ + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (!writer) { + return -1; + } + if (module_name) { + if (PyUnicodeWriter_WriteUTF8(writer, module_name, -1) < 0) { + PyUnicodeWriter_Discard(writer); + return -1; + } + if (PyUnicodeWriter_WriteASCII(writer, ": ", 2) < 0) { + PyUnicodeWriter_Discard(writer); + return -1; + } + } + va_list vargs; + va_start(vargs, format); + if (_PyUnicodeWriter_FormatV(writer, format, vargs) < 0) { + PyUnicodeWriter_Discard(writer); + return -1; + } + PyObject *message = PyUnicodeWriter_Finish(writer); + if (!message) { + return -1; + } + PyErr_SetObject(PyExc_ImportError, message); + Py_DECREF(message); + return -1; +} + +int PyABIInfo_Check(PyABIInfo *info, const char *module_name) +{ + if (!info) { + return _abiinfo_raise(module_name, "NULL PyABIInfo"); + } + + /* abiinfo_major_version */ + if (info->abiinfo_major_version == 0) { + return 0; + } + if (info->abiinfo_major_version > 1) { + return _abiinfo_raise(module_name, "PyABIInfo version too high"); + } + + /* Internal ABI */ + if (info->flags & PyABIInfo_INTERNAL) { + if (info->abi_version && (info->abi_version != PY_VERSION_HEX)) { + return _abiinfo_raise( + module_name, + "incompatible internal ABI (0x%x != 0x%x)", + info->abi_version, PY_VERSION_HEX); + } + } + +#define XY_MASK 0xffff0000 + if (info->flags & PyABIInfo_STABLE) { + /* Greater-than major.minor version check */ + if (info->abi_version) { + if ((info->abi_version & XY_MASK) > (PY_VERSION_HEX & XY_MASK)) { + return _abiinfo_raise( + module_name, + "incompatible future stable ABI version (%d.%d)", + ((info->abi_version) >> 24) % 0xff, + ((info->abi_version) >> 16) % 0xff); + } + if (info->abi_version < Py_PACK_VERSION(3, 2)) { + return _abiinfo_raise( + module_name, + "invalid stable ABI version (%d.%d)", + ((info->abi_version) >> 24) % 0xff, + ((info->abi_version) >> 16) % 0xff); + } + } + if (info->flags & PyABIInfo_INTERNAL) { + return _abiinfo_raise(module_name, + "cannot use both internal and stable ABI"); + } + } + else { + /* Exact major.minor version check */ + if (info->abi_version) { + if ((info->abi_version & XY_MASK) != (PY_VERSION_HEX & XY_MASK)) { + return _abiinfo_raise( + module_name, + "incompatible ABI version (%d.%d)", + ((info->abi_version) >> 24) % 0xff, + ((info->abi_version) >> 16) % 0xff); + } + } + } +#undef XY_MASK + + /* Free-threading/GIL */ + uint16_t gilflags = info->flags & (PyABIInfo_GIL | PyABIInfo_FREETHREADED); +#if Py_GIL_DISABLED + if (gilflags == PyABIInfo_GIL) { + return _abiinfo_raise(module_name, + "incompatible with free-threaded CPython"); + } +#else + if (gilflags == PyABIInfo_FREETHREADED) { + return _abiinfo_raise(module_name, + "only compatible with free-threaded CPython"); + } +#endif + + return 0; +} + /* Exported functions for version helper macros */ @@ -669,5 +793,5 @@ Py_PACK_FULL_VERSION(int x, int y, int z, int level, int serial) uint32_t Py_PACK_VERSION(int x, int y) { - return Py_PACK_FULL_VERSION(x, y, 0, 0, 0); + return _Py_PACK_VERSION(x, y); } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 8af445d7d6a..b2fa7d01e8f 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -1,5 +1,5 @@ -#if !Py_TAIL_CALL_INTERP -static void *opcode_targets[256] = { +#if !_Py_TAIL_CALL_INTERP +static void *opcode_targets_table[256] = { &&TARGET_CACHE, &&TARGET_BINARY_SLICE, &&TARGET_BUILD_TEMPLATE, @@ -190,8 +190,6 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_CONST_IMMORTAL, - &&TARGET_LOAD_CONST_MORTAL, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_SUPER_ATTR_ATTR, @@ -234,6 +232,7 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -256,9 +255,272 @@ static void *opcode_targets[256] = { &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, + &&TARGET_TRACE_RECORD, }; -#else /* Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr INSTRUCTION_TABLE[256]; +#if _Py_TIER2 +static void *opcode_tracing_targets_table[256] = { + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, +}; +#endif +#else /* _Py_TAIL_CALL_INTERP */ +static py_tail_call_funcptr instruction_funcptr_handler_table[256]; + +static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); @@ -266,6 +528,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); @@ -410,8 +673,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PA Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_IMMORTAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST_MORTAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); @@ -484,6 +745,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); @@ -505,7 +767,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { +static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -649,8 +911,6 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, - [LOAD_CONST_IMMORTAL] = _TAIL_CALL_LOAD_CONST_IMMORTAL, - [LOAD_CONST_MORTAL] = _TAIL_CALL_LOAD_CONST_MORTAL, [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, @@ -723,6 +983,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, @@ -740,6 +1001,8 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, [212] = _TAIL_CALL_UNKNOWN_OPCODE, [213] = _TAIL_CALL_UNKNOWN_OPCODE, [214] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -761,6 +1024,263 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; -#endif /* Py_TAIL_CALL_INTERP */ +static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACE_RECORD, + [BUILD_LIST] = _TAIL_CALL_TRACE_RECORD, + [BUILD_MAP] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SET] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_STRING] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [CACHE] = _TAIL_CALL_TRACE_RECORD, + [CALL] = _TAIL_CALL_TRACE_RECORD, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CLEANUP_THROW] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_INT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_STR] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACE_RECORD, + [CONVERT_VALUE] = _TAIL_CALL_TRACE_RECORD, + [COPY] = _TAIL_CALL_TRACE_RECORD, + [COPY_FREE_VARS] = _TAIL_CALL_TRACE_RECORD, + [DELETE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [DELETE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [DELETE_FAST] = _TAIL_CALL_TRACE_RECORD, + [DELETE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [DELETE_NAME] = _TAIL_CALL_TRACE_RECORD, + [DELETE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, + [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_SEND] = _TAIL_CALL_TRACE_RECORD, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_GEN] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [GET_AITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ANEXT] = _TAIL_CALL_TRACE_RECORD, + [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER] = _TAIL_CALL_TRACE_RECORD, + [GET_LEN] = _TAIL_CALL_TRACE_RECORD, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACE_RECORD, + [IS_OP] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_CONST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_LOCALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_NAME] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SPECIAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACE_RECORD, + [MAKE_CELL] = _TAIL_CALL_TRACE_RECORD, + [MAKE_FUNCTION] = _TAIL_CALL_TRACE_RECORD, + [MAP_ADD] = _TAIL_CALL_TRACE_RECORD, + [MATCH_CLASS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_KEYS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_MAPPING] = _TAIL_CALL_TRACE_RECORD, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [NOP] = _TAIL_CALL_TRACE_RECORD, + [NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [POP_EXCEPT] = _TAIL_CALL_TRACE_RECORD, + [POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [POP_TOP] = _TAIL_CALL_TRACE_RECORD, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, + [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, + [RERAISE] = _TAIL_CALL_TRACE_RECORD, + [RESERVED] = _TAIL_CALL_TRACE_RECORD, + [RESUME] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD, + [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, + [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, + [SET_ADD] = _TAIL_CALL_TRACE_RECORD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, + [SET_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [STORE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [STORE_NAME] = _TAIL_CALL_TRACE_RECORD, + [STORE_SLICE] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [SWAP] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_INT] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_LIST] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_NONE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_STR] = _TAIL_CALL_TRACE_RECORD, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NOT] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_EX] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, + [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, + [212] = _TAIL_CALL_UNKNOWN_OPCODE, + [213] = _TAIL_CALL_UNKNOWN_OPCODE, + [214] = _TAIL_CALL_UNKNOWN_OPCODE, + [215] = _TAIL_CALL_UNKNOWN_OPCODE, + [216] = _TAIL_CALL_UNKNOWN_OPCODE, + [217] = _TAIL_CALL_UNKNOWN_OPCODE, + [218] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, +}; +#endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer.c b/Python/optimizer.c index dde3dd8ebe7..9db894f0bf0 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -6,6 +6,7 @@ #include "pycore_interp.h" #include "pycore_backoff.h" #include "pycore_bitutils.h" // _Py_popcount32() +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit #include "pycore_code.h" // _Py_GetBaseCodeUnit #include "pycore_function.h" // _PyFunction_LookupByVersion() #include "pycore_interpframe.h" @@ -14,7 +15,7 @@ #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include "pycore_optimizer.h" // _Py_uop_analyze_and_optimize() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_tuple.h" // _PyTuple_FromArraySteal +#include "pycore_tuple.h" // _PyTuple_FromArraySteal #include "pycore_unicodeobject.h" // _PyUnicode_FromASCII #include "pycore_uop_ids.h" #include "pycore_jit.h" @@ -28,11 +29,24 @@ #define MAX_EXECUTORS_SIZE 256 +// Trace too short, no progress: +// _START_EXECUTOR +// _MAKE_WARM +// _CHECK_VALIDITY +// _SET_IP +// is 4-5 instructions. +#define CODE_SIZE_NO_PROGRESS 5 +// We start with _START_EXECUTOR, _MAKE_WARM +#define CODE_SIZE_EMPTY 2 + #define _PyExecutorObject_CAST(op) ((_PyExecutorObject *)(op)) static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { + if (code == (PyCodeObject *)&_Py_InitCleanup) { + return false; + } if (instr->op.code == ENTER_EXECUTOR) { return true; } @@ -99,39 +113,60 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth); static int -uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, int curr_stackentries, +uop_optimize(_PyInterpreterFrame *frame, PyThreadState *tstate, + _PyExecutorObject **exec_ptr, bool progress_needed); /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. * If optimized, *executor_ptr contains a new reference to the executor */ -int +// gh-137573: inlining this function causes stack overflows +Py_NO_INLINE int _PyOptimizer_Optimize( - _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - _PyExecutorObject **executor_ptr, int chain_depth) + _PyInterpreterFrame *frame, PyThreadState *tstate) { - _PyStackRef *stack_pointer = frame->stackpointer; - assert(_PyInterpreterState_GET()->jit); + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + int chain_depth = _tstate->jit_tracer_state.initial_state.chain_depth; + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!interp->jit) { + // gh-140936: It is possible that interp->jit will become false during + // interpreter finalization. However, the specialized JUMP_BACKWARD_JIT + // instruction may still be present. In this case, we should + // return immediately without optimization. + return 0; + } + assert(!interp->compiling); + assert(_tstate->jit_tracer_state.initial_state.stack_depth >= 0); +#ifndef Py_GIL_DISABLED + assert(_tstate->jit_tracer_state.initial_state.func != NULL); + interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long // side-exit chains. We can only insert the executor into the bytecode if // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = _PyFrame_GetCode(frame); - assert(PyCode_Check(code)); + PyCodeObject *code = (PyCodeObject *)_tstate->jit_tracer_state.initial_state.code; + _Py_CODEUNIT *start = _tstate->jit_tracer_state.initial_state.start_instr; if (progress_needed && !has_space_for_executor(code, start)) { + interp->compiling = false; return 0; } - int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed); + // One of our dependencies while tracing was invalidated. Not worth compiling. + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { + interp->compiling = false; + return 0; + } + _PyExecutorObject *executor; + int err = uop_optimize(frame, tstate, &executor, progress_needed); if (err <= 0) { + interp->compiling = false; return err; } - assert(*executor_ptr != NULL); + assert(executor != NULL); if (progress_needed) { int index = get_index_for_executor(code, start); if (index < 0) { @@ -141,17 +176,26 @@ _PyOptimizer_Optimize( * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(*executor_ptr); + Py_DECREF(executor); + interp->compiling = false; return 0; } - insert_executor(code, start, index, *executor_ptr); + insert_executor(code, start, index, executor); } else { - (*executor_ptr)->vm_data.code = NULL; + executor->vm_data.code = NULL; } - (*executor_ptr)->vm_data.chain_depth = chain_depth; - assert((*executor_ptr)->vm_data.valid); + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; + if (exit != NULL) { + exit->executor = executor; + } + executor->vm_data.chain_depth = chain_depth; + assert(executor->vm_data.valid); + interp->compiling = false; return 1; +#else + return 0; +#endif } static _PyExecutorObject * @@ -205,8 +249,8 @@ static int executor_clear(PyObject *executor); static void unlink_executor(_PyExecutorObject *executor); -static void -free_executor(_PyExecutorObject *self) +void +_PyExecutor_Free(_PyExecutorObject *self) { #ifdef _Py_JIT _PyJIT_Free(self); @@ -242,7 +286,7 @@ _Py_ClearExecutorDeletionList(PyInterpreterState *interp) } else { *prev_to_next_ptr = exec->vm_data.links.next; - free_executor(exec); + _PyExecutor_Free(exec); } exec = *prev_to_next_ptr; } @@ -350,7 +394,7 @@ uop_item(PyObject *op, Py_ssize_t index) return NULL; } PyObject *target = PyLong_FromUnsignedLong(self->trace[index].target); - if (oparg == NULL) { + if (target == NULL) { Py_DECREF(oparg); Py_DECREF(oname); return NULL; @@ -432,6 +476,7 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, [_FOR_ITER] = _FOR_ITER_TIER_TWO, [_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO, + [_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK, }; static const uint8_t @@ -454,6 +499,14 @@ BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, }; +static const uint16_t +guard_ip_uop[MAX_UOP_ID + 1] = { + [_PUSH_FRAME] = _GUARD_IP__PUSH_FRAME, + [_RETURN_GENERATOR] = _GUARD_IP_RETURN_GENERATOR, + [_RETURN_VALUE] = _GUARD_IP_RETURN_VALUE, + [_YIELD_VALUE] = _GUARD_IP_YIELD_VALUE, +}; + #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -510,64 +563,19 @@ add_to_trace( DPRINTF(2, "No room for %s (need %d, got %d)\n", \ (opname), (n), max_length - trace_length); \ OPT_STAT_INC(trace_too_long); \ - goto done; \ + goto full; \ } -// Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE -#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE) -#define TRACE_STACK_PUSH() \ - if (trace_stack_depth >= TRACE_STACK_SIZE) { \ - DPRINTF(2, "Trace stack overflow\n"); \ - OPT_STAT_INC(trace_stack_overflow); \ - return 0; \ - } \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - trace_stack[trace_stack_depth].func = func; \ - trace_stack[trace_stack_depth].code = code; \ - trace_stack[trace_stack_depth].instr = instr; \ - trace_stack_depth++; -#define TRACE_STACK_POP() \ - if (trace_stack_depth <= 0) { \ - Py_FatalError("Trace stack underflow\n"); \ - } \ - trace_stack_depth--; \ - func = trace_stack[trace_stack_depth].func; \ - code = trace_stack[trace_stack_depth].code; \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - instr = trace_stack[trace_stack_depth].instr; - -/* Returns the length of the trace on success, - * 0 if it failed to produce a worthwhile trace, - * and -1 on an error. +/* Returns 1 on success (added to trace), 0 on trace end. */ -static int -translate_bytecode_to_trace( +int +_PyJit_translate_single_bytecode_to_trace( + PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - _PyUOpInstruction *trace, - int buffer_size, - _PyBloomFilter *dependencies, bool progress_needed) + _Py_CODEUNIT *next_instr, + int stop_tracing_opcode) { - bool first = true; - PyCodeObject *code = _PyFrame_GetCode(frame); - PyFunctionObject *func = _PyFrame_GetFunction(frame); - assert(PyFunction_Check(func)); - PyCodeObject *initial_code = code; - _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *initial_instr = instr; - int trace_length = 0; - // Leave space for possible trailing _EXIT_TRACE - int max_length = buffer_size-2; - struct { - PyFunctionObject *func; - PyCodeObject *code; - _Py_CODEUNIT *instr; - } trace_stack[TRACE_STACK_SIZE]; - int trace_stack_depth = 0; - int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions - bool jump_seen = false; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -576,408 +584,468 @@ translate_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + PyCodeObject *old_code = _tstate->jit_tracer_state.prev_state.instr_code; + bool progress_needed = (_tstate->jit_tracer_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; + int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; + int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; - DPRINTF(2, - "Optimizing %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); - ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); + _Py_CODEUNIT *this_instr = _tstate->jit_tracer_state.prev_state.instr; + _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; - for (;;) { - target = INSTR_IP(instr, code); - // Need space for _DEOPT - max_length--; + target = Py_IsNone((PyObject *)old_code) + ? (int)(target_instr - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR) + : INSTR_IP(target_instr, old_code); - uint32_t opcode = instr->op.code; - uint32_t oparg = instr->op.arg; + // Rewind EXTENDED_ARG so that we see the whole thing. + // We must point to the first EXTENDED_ARG when deopting. + int oparg = _tstate->jit_tracer_state.prev_state.instr_oparg; + int opcode = this_instr->op.code; + int rewind_oparg = oparg; + while (rewind_oparg > 255) { + rewind_oparg >>= 8; + target--; + } - if (!first && instr == initial_instr) { - // We have looped around to the start: - RESERVE(1); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + int old_stack_level = _tstate->jit_tracer_state.prev_state.instr_stacklevel; + + // Strange control-flow + bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && + (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size < CODE_SIZE_NO_PROGRESS) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } + + bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); + if (has_dynamic_jump_taken && !needs_guard_ip) { + DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; + } + + int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (is_sys_tracing) { + goto full; + } + + if (stop_tracing_opcode != 0) { + ADD_TO_TRACE(stop_tracing_opcode, 0, 0, target); + goto done; + } + + DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); + +#ifdef Py_DEBUG + if (oparg > 255) { + assert(_Py_GetBaseCodeUnit(old_code, target).op.code == EXTENDED_ARG); + } +#endif + + // Skip over super instructions. + if (_tstate->jit_tracer_state.prev_state.instr_is_super) { + _tstate->jit_tracer_state.prev_state.instr_is_super = false; + return 1; + } + + if (opcode == ENTER_EXECUTOR) { + goto full; + } + + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { + goto done; + } + + // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls + // If we haven't guarded the IP, then it's untraceable. + if (frame != _tstate->jit_tracer_state.prev_state.instr_frame && !needs_guard_ip) { + DPRINTF(2, "Unsupported: unguardable jump taken\n"); + goto unsupported; + } + + if (oparg > 0xFFFF) { + DPRINTF(2, "Unsupported: oparg too large\n"); + goto unsupported; + } + + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. + if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { + unsupported: + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 2) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 2); + if (curr->opcode == _SET_IP) { + int32_t old_target = (int32_t)uop_get_target(curr); + curr++; + trace_length++; + curr->opcode = _EXIT_TRACE; + curr->format = UOP_FORMAT_TARGET; + curr->target = old_target; + } goto done; } + } - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + if (opcode == NOP) { + return 1; + } - if (opcode == EXTENDED_ARG) { - instr++; - opcode = instr->op.code; - oparg = (oparg << 8) | instr->op.arg; - if (opcode == EXTENDED_ARG) { - instr--; + if (opcode == JUMP_FORWARD) { + return 1; + } + + if (opcode == EXTENDED_ARG) { + return 1; + } + + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length -= 2; + + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + assert(!_PyErr_Occurred(tstate)); + + + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for side exit and final _EXIT_TRACE: + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error stub and final _EXIT_TRACE: + max_length--; + } + + // _GUARD_IP leads to an exit. + max_length -= needs_guard_ip; + + RESERVE_RAW(expansion->nuops + needs_guard_ip + 2 + (!OPCODE_HAS_NO_SAVE_IP(opcode)), "uop and various checks"); + + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } + + // Can be NULL for the entry frame. + if (old_code != NULL) { + _Py_BloomFilter_Add(dependencies, old_code); + } + + switch (opcode) { + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + { + _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); + _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; + assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); + int jump_happened = computed_jump_instr == next_instr; + assert(jump_happened == (target_instr[1].cache & 1)); + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + break; + } + case JUMP_BACKWARD_JIT: + // This is possible as the JIT might have re-activated after it was disabled + case JUMP_BACKWARD_NO_JIT: + case JUMP_BACKWARD: + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); + _Py_FALLTHROUGH; + case JUMP_BACKWARD_NO_INTERRUPT: + { + if ((next_instr != _tstate->jit_tracer_state.initial_state.close_loop_instr) && + (next_instr != _tstate->jit_tracer_state.initial_state.start_instr) && + _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS && + // For side exits, we don't want to terminate them early. + _tstate->jit_tracer_state.initial_state.exit == NULL && + // These are coroutines, and we want to unroll those usually. + opcode != JUMP_BACKWARD_NO_INTERRUPT) { + // We encountered a JUMP_BACKWARD but not to the top of our own loop. + // We don't want to continue tracing as we might get stuck in the + // inner loop. Instead, end the trace where the executor of the + // inner loop might start and let the traces rejoin. + OPT_STAT_INC(inner_loop); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow + DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, + _tstate->jit_tracer_state.initial_state.close_loop_instr, _tstate->jit_tracer_state.initial_state.start_instr); goto done; } - } - if (opcode == ENTER_EXECUTOR) { - // We have a couple of options here. We *could* peek "underneath" - // this executor and continue tracing, which could give us a longer, - // more optimizeable trace (at the expense of lots of duplicated - // tier two code). Instead, we choose to just end here and stitch to - // the other trace, which allows a side-exit traces to rejoin the - // "main" trace periodically (and also helps protect us against - // pathological behavior where the amount of tier two code explodes - // for a medium-length, branchy code path). This seems to work - // better in practice, but in the future we could be smarter about - // what we do here: - goto done; - } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - RESERVE_RAW(2, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)instr, target); + break; } - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (first && progress_needed) { - assert(first); - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; + case RESUME: + case RESUME_CHECK: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + default: + { + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) + int nuops = expansion->nuops; + if (nuops == 0) { + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } + assert(nuops > 0); + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + uint32_t orig_target = target; + for (int i = 0; i < nuops; i++) { + oparg = orig_oparg; + target = orig_target; + uint32_t uop = expansion->uops[i].uop; + uint64_t operand = 0; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; + switch (expansion->uops[i].size) { + case OPARG_SIMPLE: + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + break; + case OPARG_CACHE_1: + operand = read_u16(&this_instr[offset].cache); + break; + case OPARG_CACHE_2: + operand = read_u32(&this_instr[offset].cache); + break; + case OPARG_CACHE_4: + operand = read_u64(&this_instr[offset].cache); + break; + case OPARG_TOP: // First half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg >> 4; + break; + case OPARG_BOTTOM: // Second half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg & 0xF; + break; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); - if (OPCODE_HAS_EXIT(opcode)) { - // Make space for side exit and final _EXIT_TRACE: - RESERVE_RAW(2, "_EXIT_TRACE"); - max_length--; - } - if (OPCODE_HAS_ERROR(opcode)) { - // Make space for error stub and final _EXIT_TRACE: - RESERVE_RAW(2, "_ERROR_POP_N"); - max_length--; - } - switch (opcode) { - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - { - RESERVE(1); - int counter = instr[1].cache; - int bitcount = _Py_popcount32(counter); - int jump_likely = bitcount > 8; - /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. - For values in between, adjust proportionally. */ - if (jump_likely) { - confidence = confidence * bitcount / 16; + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } + else { + int extended_arg = orig_oparg > 255; + uint32_t jump_target = next_inst + orig_oparg + extended_arg; + assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); + if (is_for_iter_test[uop]) { + target = jump_target + 1; + } + } + break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + default: + fprintf(stderr, + "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", + opcode, oparg, nuops, i, + expansion->uops[i].size, + expansion->uops[i].offset); + Py_FatalError("garbled expansion"); } - else { - confidence = confidence * (16 - bitcount) / 16; - } - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - target, _PyOpcode_OpName[opcode], oparg, - counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); - if (confidence < CONFIDENCE_CUTOFF) { - DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); - OPT_STAT_INC(low_confidence); - goto done; - } - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *target_instr = next_instr + oparg; - if (jump_likely) { - DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", - instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); - instr = target_instr; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); - goto top; - } - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); - break; - } + if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - case JUMP_BACKWARD: - case JUMP_BACKWARD_JIT: - ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; - case JUMP_BACKWARD_NO_INTERRUPT: - { - instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg; - if (jump_seen) { - OPT_STAT_INC(inner_loop); - DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); - goto done; - } - jump_seen = true; - goto top; - } - - case JUMP_FORWARD: - { - RESERVE(0); - // This will emit two _SET_IP instructions; leave it to the optimizer - instr += oparg; - break; - } - - case RESUME: - /* Use a special tier 2 version of RESUME_CHECK to allow traces to - * start with RESUME_CHECK */ - ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); - break; - - default: - { - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops > 0) { - // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) - int nuops = expansion->nuops; - RESERVE(nuops + 1); /* One extra for exit */ - int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { - // Check for trace stack underflow now: - // We can't bail e.g. in the middle of - // LOAD_CONST + _RETURN_VALUE. - if (trace_stack_depth == 0) { - DPRINTF(2, "Trace stack underflow\n"); - OPT_STAT_INC(trace_stack_underflow); - return 0; + operand = 0; + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + // Don't add nested code objects to the dependency. + // It causes endless re-traces. + if (new_func != NULL && !Py_IsNone((PyObject*)new_func) && !(new_code->co_flags & CO_NESTED)) { + operand = (uintptr_t)new_func; + DPRINTF(2, "Adding %p func to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_func); + } + else if (new_code != NULL && !Py_IsNone((PyObject*)new_code)) { + operand = (uintptr_t)new_code | 1; + DPRINTF(2, "Adding %p code to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_code); } } - uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM - for (int i = 0; i < nuops; i++) { - oparg = orig_oparg; - uint32_t uop = expansion->uops[i].uop; - uint64_t operand = 0; - // Add one to account for the actual opcode/oparg pair: - int offset = expansion->uops[i].offset + 1; - switch (expansion->uops[i].size) { - case OPARG_SIMPLE: - assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); - break; - case OPARG_CACHE_1: - operand = read_u16(&instr[offset].cache); - break; - case OPARG_CACHE_2: - operand = read_u32(&instr[offset].cache); - break; - case OPARG_CACHE_4: - operand = read_u64(&instr[offset].cache); - break; - case OPARG_TOP: // First half of super-instr - oparg = orig_oparg >> 4; - break; - case OPARG_BOTTOM: // Second half of super-instr - oparg = orig_oparg & 0xF; - break; - case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset - oparg = offset; - assert(uop == _SAVE_RETURN_OFFSET); - break; - case OPARG_REPLACED: - uop = _PyUOp_Replacements[uop]; - assert(uop != 0); -#ifdef Py_DEBUG - { - uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255); - uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); - } -#endif - break; - case OPERAND1_1: - assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_2: - assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_4: - assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - default: - fprintf(stderr, - "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", - opcode, oparg, nuops, i, - expansion->uops[i].size, - expansion->uops[i].offset); - Py_FatalError("garbled expansion"); - } - - if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { - TRACE_STACK_POP(); - /* Set the operand to the function or code object returned to, - * to assist optimization passes. (See _PUSH_FRAME below.) - */ - if (func != NULL) { - operand = (uintptr_t)func; - } - else if (code != NULL) { - operand = (uintptr_t)code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - DPRINTF(2, - "Returning to %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - - if (uop == _PUSH_FRAME) { - assert(i + 1 == nuops); - if (opcode == FOR_ITER_GEN || - opcode == LOAD_ATTR_PROPERTY || - opcode == BINARY_OP_SUBSCR_GETITEM || - opcode == SEND_GEN) - { - DPRINTF(2, "Bailing due to dynamic target\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); - int func_version_offset = - offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) - // Add one to account for the actual opcode/oparg pair: - + 1; - uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyCodeObject *new_code = NULL; - PyFunctionObject *new_func = - _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); - DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", - (int)func_version, new_func, new_code); - if (new_code != NULL) { - if (new_code == code) { - // Recursive call, bail (we could be here forever). - DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", - PyUnicode_AsUTF8(new_code->co_qualname), - PyUnicode_AsUTF8(new_code->co_filename), - new_code->co_firstlineno); - OPT_STAT_INC(recursive_call); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - if (new_code->co_version != func_version) { - // func.__code__ was updated. - // Perhaps it may happen again, so don't bother tracing. - // TODO: Reason about this -- is it better to bail or not? - DPRINTF(2, "Bailing because co_version != func_version\n"); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - // Increment IP to the return address - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; - TRACE_STACK_PUSH(); - _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's function or code object, - * to assist optimization passes. - * We prefer setting it to the function (for remove_globals()) - * but if that's not available but the code is available, - * use the code, setting the low bit so the optimizer knows. - */ - if (new_func != NULL) { - operand = (uintptr_t)new_func; - } - else if (new_code != NULL) { - operand = (uintptr_t)new_code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - code = new_code; - func = new_func; - instr = _PyCode_CODE(code); - DPRINTF(2, - "Continuing in %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - DPRINTF(2, "Bail, new_code == NULL\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - - if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { - assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; - // Skip the STORE_FAST: - instr++; - } - - // All other instructions - ADD_TO_TRACE(uop, oparg, operand, target); - } + ADD_TO_TRACE(uop, oparg, operand, target); + trace[trace_length - 1].operand1 = PyStackRef_IsNone(frame->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); break; } - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - OPT_UNSUPPORTED_OPCODE(opcode); - goto done; // Break out of loop - } // End default + if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { + assert(i + 1 == nuops); + _Py_CODEUNIT *next = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next->op.code == STORE_FAST); + operand = next->op.arg; + } + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); + } + break; + } // End default - } // End switch (opcode) + } // End switch (opcode) - instr++; - // Add cache size for opcode - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - - if (opcode == CALL_LIST_APPEND) { - assert(instr->op.code == POP_TOP); - instr++; + if (needs_guard_ip) { + uint16_t guard_ip = guard_ip_uop[trace[trace_length-1].opcode]; + if (guard_ip == 0) { + DPRINTF(1, "Unknown uop needing guard ip %s\n", _PyOpcode_uop_name[trace[trace_length-1].opcode]); + Py_UNREACHABLE(); } - top: - // Jump here after _PUSH_FRAME or likely branches. - first = false; - } // End for (;;) - -done: - while (trace_stack_depth > 0) { - TRACE_STACK_POP(); + ADD_TO_TRACE(guard_ip, 0, (uintptr_t)next_instr, 0); } - assert(code == initial_code); - // Skip short traces where we can't even translate a single instruction: - if (first) { - OPT_STAT_INC(trace_too_short); - DPRINTF(2, - "No trace for %s (%s:%d) at byte offset %d (no progress)\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); + // Loop back to the start + int is_first_instr = _tstate->jit_tracer_state.initial_state.close_loop_instr == next_instr || + _tstate->jit_tracer_state.initial_state.start_instr == next_instr; + if (is_first_instr && _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS) { + if (needs_guard_ip) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); + } + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } + DPRINTF(2, "Trace continuing\n"); + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 1; +done: + DPRINTF(2, "Trace done\n"); + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 0; +full: + DPRINTF(2, "Trace full\n"); + if (!is_terminator(&_tstate->jit_tracer_state.code_buffer[trace_length-1])) { + // Undo the last few instructions. + trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + max_length = _tstate->jit_tracer_state.prev_state.code_max_size; + // We previously reversed one. + max_length += 1; + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow + } + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 0; +} + +// Returns 0 for do not enter tracing, 1 on enter tracing. +int +_PyJit_TryInitializeTracing( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, + _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, + _PyExitData *exit, int oparg) +{ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // A recursive trace. + // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. + if (_tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_EMPTY) { return 0; } - if (!is_terminator(&trace[trace_length-1])) { - /* Allow space for _EXIT_TRACE */ - max_length += 2; - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + if (oparg > 0xFFFF) { + return 0; } - DPRINTF(1, - "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code), - trace_length); - OPT_HIST(trace_length, trace_length_hist); - return trace_length; + if (_tstate->jit_tracer_state.code_buffer == NULL) { + _tstate->jit_tracer_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (_tstate->jit_tracer_state.code_buffer == NULL) { + // Don't error, just go to next instruction. + return 0; + } + } + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func == NULL) { + return 0; + } + PyCodeObject *code = _PyFrame_GetCode(frame); +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + DPRINTF(2, + "Tracing %s (%s:%d) at byte offset %d at chain depth %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(close_loop_instr, code), + chain_depth); +#endif + + add_to_trace(_tstate->jit_tracer_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); + add_to_trace(_tstate->jit_tracer_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; + + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; + _tstate->jit_tracer_state.initial_state.start_instr = start_instr; + _tstate->jit_tracer_state.initial_state.close_loop_instr = close_loop_instr; + _tstate->jit_tracer_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); + _tstate->jit_tracer_state.initial_state.func = (PyFunctionObject *)Py_NewRef(func); + _tstate->jit_tracer_state.initial_state.exit = exit; + _tstate->jit_tracer_state.initial_state.stack_depth = curr_stackdepth; + _tstate->jit_tracer_state.initial_state.chain_depth = chain_depth; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = true; + _tstate->jit_tracer_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _tstate->jit_tracer_state.prev_state.instr = curr_instr; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = curr_stackdepth; + _tstate->jit_tracer_state.prev_state.instr_is_super = false; + assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); + _tstate->jit_tracer_state.initial_state.jump_backward_instr = curr_instr; + + if (_PyOpcode_Caches[_PyOpcode_Deopt[close_loop_instr->op.code]]) { + close_loop_instr[1].counter = trigger_backoff_counter(); + } + _Py_BloomFilter_Init(&_tstate->jit_tracer_state.prev_state.dependencies); + return 1; } +void +_PyJit_FinalizeTracing(PyThreadState *tstate) +{ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + Py_CLEAR(_tstate->jit_tracer_state.initial_state.code); + Py_CLEAR(_tstate->jit_tracer_state.initial_state.func); + Py_CLEAR(_tstate->jit_tracer_state.prev_state.instr_code); + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; +} + + #undef RESERVE #undef RESERVE_RAW #undef INSTR_IP @@ -996,20 +1064,21 @@ count_exits(_PyUOpInstruction *buffer, int length) int exit_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } } return exit_count; } -static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +static void make_exit(_PyUOpInstruction *inst, int opcode, int target, bool is_control_flow) { inst->opcode = opcode; inst->oparg = 0; inst->operand0 = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; + inst->operand1 = is_control_flow; #ifdef Py_STATS inst->execution_count = 0; #endif @@ -1043,19 +1112,27 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; int32_t target = (int32_t)uop_get_target(inst); - if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { - uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? - _EXIT_TRACE : _DEOPT; - int32_t jump_target = target; - if (is_for_iter_test[opcode]) { - /* Target the POP_TOP immediately after the END_FOR, - * leaving only the iterator on the stack. */ - int extended_arg = inst->oparg > 255; - int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; - jump_target = next_inst + inst->oparg + 1; + uint16_t exit_flags = _PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG); + if (exit_flags) { + uint16_t exit_op = _EXIT_TRACE; + if (exit_flags & HAS_DEOPT_FLAG) { + exit_op = _DEOPT; } + else if (exit_flags & HAS_PERIODIC_FLAG) { + exit_op = _HANDLE_PENDING_AND_DEOPT; + } + int32_t jump_target = target; + if ( + opcode == _GUARD_IP__PUSH_FRAME || + opcode == _GUARD_IP_RETURN_VALUE || + opcode == _GUARD_IP_YIELD_VALUE || + opcode == _GUARD_IP_RETURN_GENERATOR + ) { + exit_op = _DYNAMIC_EXIT; + } + bool is_control_flow = (opcode == _GUARD_IS_FALSE_POP || opcode == _GUARD_IS_TRUE_POP || is_for_iter_test[opcode]); if (jump_target != current_jump_target || current_exit_op != exit_op) { - make_exit(&buffer[next_spare], exit_op, jump_target); + make_exit(&buffer[next_spare], exit_op, jump_target, is_control_flow); current_exit_op = exit_op; current_jump_target = jump_target; current_jump = next_spare; @@ -1071,7 +1148,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_popped = popped; current_error = next_spare; current_error_target = target; - make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + make_exit(&buffer[next_spare], _ERROR_POP_N, 0, false); buffer[next_spare].operand0 = target; next_spare++; } @@ -1129,7 +1206,9 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR); + CHECK(executor->trace[0].opcode == _START_EXECUTOR || + executor->trace[0].opcode == _COLD_EXIT || + executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1159,8 +1238,10 @@ sanity_check(_PyExecutorObject *executor) uint16_t opcode = inst->opcode; CHECK( opcode == _DEOPT || + opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || - opcode == _ERROR_POP_N); + opcode == _ERROR_POP_N || + opcode == _DYNAMIC_EXIT); } } @@ -1173,7 +1254,7 @@ sanity_check(_PyExecutorObject *executor) * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth) { int exit_count = count_exits(buffer, length); _PyExecutorObject *executor = allocate_executor(exit_count, length); @@ -1182,8 +1263,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } /* Initialize exits */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); + cold->vm_data.chain_depth = chain_depth; for (int i = 0; i < exit_count; i++) { - executor->exits[i].executor = NULL; + executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); } int next_exit = exit_count-1; @@ -1194,11 +1278,13 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int opcode = buffer[i].opcode; dest--; *dest = buffer[i]; - assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; + exit->executor = opcode == _EXIT_TRACE ? cold : cold_dynamic; + exit->is_dynamic = (char)(opcode == _DYNAMIC_EXIT); + exit->is_control_flow = (char)buffer[i].operand1; next_exit--; } } @@ -1224,7 +1310,6 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil #endif #ifdef _Py_JIT executor->jit_code = NULL; - executor->jit_side_entry = NULL; executor->jit_size = 0; // This is initialized to true so we can prevent the executor // from being immediately detected as cold and invalidated. @@ -1261,27 +1346,32 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) static int uop_optimize( _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, + PyThreadState *tstate, _PyExecutorObject **exec_ptr, - int curr_stackentries, bool progress_needed) { - _PyBloomFilter dependencies; - _Py_BloomFilter_Init(&dependencies); - _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; + _PyUOpInstruction *buffer = _tstate->jit_tracer_state.code_buffer; OPT_STAT_INC(attempts); - int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); - if (length <= 0) { - // Error or nothing translated - return length; + char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); + bool is_noopt = true; + if (env_var == NULL || *env_var == '\0' || *env_var > '0') { + is_noopt = false; } + int curr_stackentries = _tstate->jit_tracer_state.initial_state.stack_depth; + int length = _tstate->jit_tracer_state.prev_state.code_curr_size; + if (length <= CODE_SIZE_NO_PROGRESS) { + return 0; + } + assert(length > 0); assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); - char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); - if (env_var == NULL || *env_var == '\0' || *env_var > '0') { - length = _Py_uop_analyze_and_optimize(frame, buffer, - length, - curr_stackentries, &dependencies); + if (!is_noopt) { + length = _Py_uop_analyze_and_optimize( + _tstate->jit_tracer_state.initial_state.func, + buffer,length, + curr_stackentries, dependencies); if (length <= 0) { return length; } @@ -1292,8 +1382,8 @@ uop_optimize( for (int pc = 0; pc < length; pc++) { int opcode = buffer[pc].opcode; int oparg = buffer[pc].oparg; - if (oparg < _PyUop_Replication[opcode]) { - buffer[pc].opcode = opcode + oparg + 1; + if (oparg < _PyUop_Replication[opcode].stop && oparg >= _PyUop_Replication[opcode].start) { + buffer[pc].opcode = opcode + oparg + 1 - _PyUop_Replication[opcode].start; assert(strncmp(_PyOpcode_uop_name[buffer[pc].opcode], _PyOpcode_uop_name[opcode], strlen(_PyOpcode_uop_name[opcode])) == 0); } else if (is_terminator(&buffer[pc])) { @@ -1304,11 +1394,19 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); + _PyExecutorObject *executor = make_executor_from_uops( + buffer, length, dependencies, _tstate->jit_tracer_state.initial_state.chain_depth); if (executor == NULL) { return -1; } assert(length <= UOP_MAX_TRACE_LENGTH); + + // Check executor coldness + // It's okay if this ends up going negative. + if (--tstate->interp->executor_creation_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + *exec_ptr = executor; return 1; } @@ -1462,6 +1560,79 @@ _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_s link_executor(executor); } +_PyExecutorObject * +_PyExecutor_GetColdExecutor(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->cold_executor != NULL) { + return interp->cold_executor; + } + _PyExecutorObject *cold = allocate_executor(0, 1); + if (cold == NULL) { + Py_FatalError("Cannot allocate core JIT code"); + } + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_EXIT; +#ifdef _Py_JIT + cold->jit_code = NULL; + cold->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + cold->vm_data.warm = true; + if (_PyJIT_Compile(cold, cold->trace, 1)) { + Py_DECREF(cold); + Py_FatalError("Cannot allocate core JIT code"); + } +#endif + _Py_SetImmortal((PyObject *)cold); + interp->cold_executor = cold; + return cold; +} + +_PyExecutorObject * +_PyExecutor_GetColdDynamicExecutor(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->cold_dynamic_executor != NULL) { + assert(interp->cold_dynamic_executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); + return interp->cold_dynamic_executor; + } + _PyExecutorObject *cold = allocate_executor(0, 1); + if (cold == NULL) { + Py_FatalError("Cannot allocate core JIT code"); + } + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_DYNAMIC_EXIT; +#ifdef _Py_JIT + cold->jit_code = NULL; + cold->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + cold->vm_data.warm = true; + if (_PyJIT_Compile(cold, cold->trace, 1)) { + Py_DECREF(cold); + Py_FatalError("Cannot allocate core JIT code"); + } +#endif + _Py_SetImmortal((PyObject *)cold); + interp->cold_dynamic_executor = cold; + return cold; +} + +void +_PyExecutor_ClearExit(_PyExitData *exit) +{ + if (exit == NULL) { + return; + } + _PyExecutorObject *old = exit->executor; + if (exit->is_dynamic) { + exit->executor = _PyExecutor_GetColdDynamicExecutor(); + } + else { + exit->executor = _PyExecutor_GetColdExecutor(); + } + Py_DECREF(old); +} + /* Detaches the executor from the code object (if any) that * holds a reference to it */ void @@ -1492,14 +1663,18 @@ executor_clear(PyObject *op) assert(executor->vm_data.valid == 1); unlink_executor(executor); executor->vm_data.valid = 0; + /* It is possible for an executor to form a reference * cycle with itself, so decref'ing a side exit could * free the executor unless we hold a strong reference to it */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); Py_INCREF(executor); for (uint32_t i = 0; i < executor->exit_count; i++) { executor->exits[i].temperature = initial_unreachable_backoff_counter(); - Py_CLEAR(executor->exits[i].executor); + _PyExecutorObject *e = executor->exits[i].executor; + executor->exits[i].executor = cold; + Py_DECREF(e); } _Py_ExecutorDetach(executor); Py_DECREF(executor); @@ -1556,6 +1731,18 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_Executors_InvalidateAll(interp, is_invalidation); } +void +_PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) +{ + _PyBloomFilter obj_filter; + _Py_BloomFilter_Init(&obj_filter); + _Py_BloomFilter_Add(&obj_filter, obj); + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (bloom_filter_may_contain(&_tstate->jit_tracer_state.prev_state.dependencies, &obj_filter)) + { + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = false; + } +} /* Invalidate all executors */ void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) @@ -1685,7 +1872,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) #ifdef Py_STATS fprintf(out, " <tr><td port=\"i%d\" border=\"1\" >%s -- %" PRIu64 "</td></tr>\n", i, opname, inst->execution_count); #else - fprintf(out, " <tr><td port=\"i%d\" border=\"1\" >%s</td></tr>\n", i, opname); + fprintf(out, " <tr><td port=\"i%d\" border=\"1\" >%s op0=%" PRIu64 "</td></tr>\n", i, opname, inst->operand0); #endif if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { break; @@ -1695,6 +1882,8 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) fprintf(out, "]\n\n"); /* Write all the outgoing edges */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; uint16_t flags = _PyUop_Flags[inst->opcode]; @@ -1705,10 +1894,10 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) else if (flags & HAS_EXIT_FLAG) { assert(inst->format == UOP_FORMAT_JUMP); _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; - assert(exit_inst->opcode == _EXIT_TRACE); + assert(exit_inst->opcode == _EXIT_TRACE || exit_inst->opcode == _DYNAMIC_EXIT); exit = (_PyExitData *)exit_inst->operand0; } - if (exit != NULL && exit->executor != NULL) { + if (exit != NULL && exit->executor != cold && exit->executor != cold_dynamic) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { @@ -1741,4 +1930,11 @@ _PyDumpExecutors(FILE *out) return -1; } +void +_PyExecutor_Free(struct _PyExecutorObject *self) +{ + /* This should never be called */ + Py_UNREACHABLE(); +} + #endif /* _Py_TIER2 */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 8b0bd1e9518..51722556554 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -26,6 +26,9 @@ #include "pycore_function.h" #include "pycore_uop_ids.h" #include "pycore_range.h" +#include "pycore_unicodeobject.h" +#include "pycore_ceval.h" +#include "pycore_floatobject.h" #include <stdarg.h> #include <stdbool.h> @@ -103,6 +106,10 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop) if ((int)index >= dict->ma_keys->dk_nentries) { return NULL; } + PyDictKeysObject *keys = dict->ma_keys; + if (keys->dk_version != inst->operand0) { + return NULL; + } PyObject *res = entries[index].me_value; if (res == NULL) { return NULL; @@ -121,203 +128,36 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop) return res; } -static int -incorrect_keys(_PyUOpInstruction *inst, PyObject *obj) +static bool +incorrect_keys(PyObject *obj, uint32_t version) { if (!PyDict_CheckExact(obj)) { - return 1; + return true; } PyDictObject *dict = (PyDictObject *)obj; - if (dict->ma_keys->dk_version != inst->operand0) { - return 1; - } - return 0; + return dict->ma_keys->dk_version != version; } -/* Returns 1 if successfully optimized - * 0 if the trace is not suitable for optimization (yet) - * -1 if there was an error. */ -static int -remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, - int buffer_size, _PyBloomFilter *dependencies) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *builtins = frame->f_builtins; - if (builtins != interp->builtins) { - OPT_STAT_INC(remove_globals_builtins_changed); - return 1; - } - PyObject *globals = frame->f_globals; - PyFunctionObject *function = _PyFrame_GetFunction(frame); - assert(PyFunction_Check(function)); - assert(function->func_builtins == builtins); - assert(function->func_globals == globals); - uint32_t function_version = _PyFunction_GetVersionForCurrentState(function); - /* In order to treat globals as constants, we need to - * know that the globals dict is the one we expected, and - * that it hasn't changed - * In order to treat builtins as constants, we need to - * know that the builtins dict is the one we expected, and - * that it hasn't changed and that the global dictionary's - * keys have not changed */ - - /* These values represent stacks of booleans (one bool per bit). - * Pushing a frame shifts left, popping a frame shifts right. */ - uint32_t function_checked = 0; - uint32_t builtins_watched = 0; - uint32_t globals_watched = 0; - uint32_t prechecked_function_version = 0; - if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { - interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; - } - if (interp->type_watchers[TYPE_WATCHER_ID] == NULL) { - interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; - } - for (int pc = 0; pc < buffer_size; pc++) { - _PyUOpInstruction *inst = &buffer[pc]; - int opcode = inst->opcode; - switch(opcode) { - case _GUARD_GLOBALS_VERSION: - if (incorrect_keys(inst, globals)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - return 0; - } - if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - continue; - } - if ((globals_watched & 1) == 0) { - PyDict_Watch(GLOBALS_WATCHER_ID, globals); - _Py_BloomFilter_Add(dependencies, globals); - globals_watched |= 1; - } - if (function_checked & 1) { - buffer[pc].opcode = NOP; - } - else { - buffer[pc].opcode = _CHECK_FUNCTION; - buffer[pc].operand0 = function_version; - function_checked |= 1; - } - break; - case _LOAD_GLOBAL_BUILTINS: - if (incorrect_keys(inst, builtins)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - return 0; - } - if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { - continue; - } - if ((builtins_watched & 1) == 0) { - PyDict_Watch(BUILTINS_WATCHER_ID, builtins); - builtins_watched |= 1; - } - if (function_checked & globals_watched & 1) { - convert_global_to_const(inst, builtins, false); - } - break; - case _LOAD_GLOBAL_MODULE: - if (incorrect_keys(inst, globals)) { - OPT_STAT_INC(remove_globals_incorrect_keys); - return 0; - } - if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { - continue; - } - if ((globals_watched & 1) == 0) { - PyDict_Watch(GLOBALS_WATCHER_ID, globals); - _Py_BloomFilter_Add(dependencies, globals); - globals_watched |= 1; - } - if ((function_checked & 1) == 0 && buffer[pc-1].opcode == _NOP) { - buffer[pc-1].opcode = _CHECK_FUNCTION; - buffer[pc-1].operand0 = function_version; - function_checked |= 1; - } - if (function_checked & 1) { - convert_global_to_const(inst, globals, false); - } - break; - case _PUSH_FRAME: - { - builtins_watched <<= 1; - globals_watched <<= 1; - function_checked <<= 1; - uint64_t operand = buffer[pc].operand0; - if (operand == 0 || (operand & 1)) { - // It's either a code object or NULL, so bail - return 1; - } - PyFunctionObject *func = (PyFunctionObject *)operand; - if (func == NULL) { - return 1; - } - assert(PyFunction_Check(func)); - function_version = func->func_version; - if (prechecked_function_version == function_version) { - function_checked |= 1; - } - prechecked_function_version = 0; - globals = func->func_globals; - builtins = func->func_builtins; - if (builtins != interp->builtins) { - OPT_STAT_INC(remove_globals_builtins_changed); - return 1; - } - break; - } - case _RETURN_VALUE: - { - builtins_watched >>= 1; - globals_watched >>= 1; - function_checked >>= 1; - uint64_t operand = buffer[pc].operand0; - if (operand == 0 || (operand & 1)) { - // It's either a code object or NULL, so bail - return 1; - } - PyFunctionObject *func = (PyFunctionObject *)operand; - if (func == NULL) { - return 1; - } - assert(PyFunction_Check(func)); - function_version = func->func_version; - globals = func->func_globals; - builtins = func->func_builtins; - break; - } - case _CHECK_FUNCTION_EXACT_ARGS: - prechecked_function_version = (uint32_t)buffer[pc].operand0; - break; - default: - if (is_terminator(inst)) { - return 1; - } - break; - } - } - return 0; -} - - #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) -#define WITHIN_STACK_BOUNDS() \ - (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()) - +#define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) #define REPLACE_OP(INST, OP, ARG, OPERAND) \ - INST->opcode = OP; \ - INST->oparg = ARG; \ - INST->operand0 = OPERAND; + (INST)->opcode = OP; \ + (INST)->oparg = ARG; \ + (INST)->operand0 = OPERAND; /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null #define sym_is_const _Py_uop_sym_is_const +#define sym_is_safe_const _Py_uop_sym_is_safe_const #define sym_get_const _Py_uop_sym_get_const +#define sym_new_const_steal _Py_uop_sym_new_const_steal +#define sym_get_const_as_stackref _Py_uop_sym_get_const_as_stackref #define sym_new_unknown _Py_uop_sym_new_unknown #define sym_new_not_null _Py_uop_sym_new_not_null #define sym_new_type _Py_uop_sym_new_type @@ -333,6 +173,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM) #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new @@ -340,15 +181,40 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_new_tuple _Py_uop_sym_new_tuple #define sym_tuple_getitem _Py_uop_sym_tuple_getitem #define sym_tuple_length _Py_uop_sym_tuple_length -#define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_is_immortal _Py_uop_symbol_is_immortal +#define sym_is_compact_int _Py_uop_sym_is_compact_int +#define sym_new_compact_int _Py_uop_sym_new_compact_int #define sym_new_truthiness _Py_uop_sym_new_truthiness +#define JUMP_TO_LABEL(label) goto label; + +static int +check_stack_bounds(JitOptContext *ctx, JitOptRef *stack_pointer, int offset, int opcode) +{ + int stack_level = (int)(stack_pointer + (offset) - ctx->frame->stack); + int should_check = !CURRENT_FRAME_IS_INIT_SHIM() || + (opcode == _RETURN_VALUE) || + (opcode == _RETURN_GENERATOR) || + (opcode == _YIELD_VALUE); + if (should_check && (stack_level < 0 || stack_level > STACK_SIZE())) { + ctx->contradiction = true; + ctx->done = true; + return 1; + } + return 0; +} + +#define CHECK_STACK_BOUNDS(offset) \ + if (check_stack_bounds(ctx, stack_pointer, offset, opcode)) { \ + break; \ + } \ + static int optimize_to_bool( _PyUOpInstruction *this_instr, JitOptContext *ctx, - JitOptSymbol *value, - JitOptSymbol **result_ptr) + JitOptRef value, + JitOptRef *result_ptr) { if (sym_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -375,28 +241,21 @@ eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) } } -/* _PUSH_FRAME/_RETURN_VALUE's operand can be 0, a PyFunctionObject *, or a - * PyCodeObject *. Retrieve the code object if possible. - */ -static PyCodeObject * -get_code(_PyUOpInstruction *op) +static JitOptRef +lookup_attr(JitOptContext *ctx, _PyUOpInstruction *this_instr, + PyTypeObject *type, PyObject *name, uint16_t immortal, + uint16_t mortal) { - assert(op->opcode == _PUSH_FRAME || op->opcode == _RETURN_VALUE || op->opcode == _RETURN_GENERATOR); - PyCodeObject *co = NULL; - uint64_t operand = op->operand0; - if (operand == 0) { - return NULL; + // The cached value may be dead, so we need to do the lookup again... :( + if (type && PyType_Check(type)) { + PyObject *lookup = _PyType_Lookup(type, name); + if (lookup) { + int opcode = _Py_IsImmortal(lookup) ? immortal : mortal; + REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup); + return sym_new_const(ctx, lookup); + } } - if (operand & 1) { - co = (PyCodeObject *)(operand & ~1); - } - else { - PyFunctionObject *func = (PyFunctionObject *)operand; - assert(PyFunction_Check(func)); - co = (PyCodeObject *)func->func_code; - } - assert(PyCode_Check(co)); - return co; + return sym_new_not_null(ctx); } static PyCodeObject * @@ -423,44 +282,101 @@ get_code_with_logging(_PyUOpInstruction *op) return co; } -/* 1 for success, 0 for not ready, cannot error at the moment. */ +static +PyCodeObject * +get_current_code_object(JitOptContext *ctx) +{ + return (PyCodeObject *)ctx->frame->code; +} + +static PyObject * +get_co_name(JitOptContext *ctx, int index) +{ + return PyTuple_GET_ITEM(get_current_code_object(ctx)->co_names, index); +} + +#ifdef Py_DEBUG +void +_Py_opt_assert_within_stack_bounds( + _Py_UOpsAbstractFrame *frame, JitOptRef *stack_pointer, + const char *filename, int lineno +) { + if (frame->code == ((PyCodeObject *)&_Py_InitCleanup)) { + return; + } + int level = (int)(stack_pointer - frame->stack); + if (level < 0) { + printf("Stack underflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } + int size = (int)(frame->stack_len); + if (level > size) { + printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } +} +#endif + +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_opt_assert_within_stack_bounds(ctx->frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + +// TODO (gh-134584) generate most of this table automatically +const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { + [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_FLOAT] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, +}; + +/* >0 (length) for success, 0 for not ready, clears all possible errors. */ static int optimize_uops( - PyCodeObject *co, + PyFunctionObject *func, _PyUOpInstruction *trace, int trace_len, int curr_stacklen, _PyBloomFilter *dependencies ) { + assert(!PyErr_Occurred()); JitOptContext context; JitOptContext *ctx = &context; uint32_t opcode = UINT16_MAX; - int curr_space = 0; - int max_space = 0; - _PyUOpInstruction *first_valid_check_stack = NULL; - _PyUOpInstruction *corresponding_check_stack = NULL; + + // Make sure that watchers are set up + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { + interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + } _Py_uop_abstractcontext_init(ctx); - _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, (PyCodeObject *)func->func_code, curr_stacklen, NULL, 0); if (frame == NULL) { - return -1; + return 0; } + frame->func = func; ctx->curr_frame_depth++; ctx->frame = frame; - ctx->done = false; - ctx->out_of_space = false; - ctx->contradiction = false; _PyUOpInstruction *this_instr = NULL; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; + for (int i = 0; !ctx->done; i++) { assert(i < trace_len); this_instr = &trace[i]; int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptSymbol **stack_pointer = ctx->frame->stack_pointer; + + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + stack_pointer = ctx->frame->stack_pointer; + } #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -479,9 +395,11 @@ optimize_uops( Py_UNREACHABLE(); } assert(ctx->frame != NULL); - DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); - ctx->frame->stack_pointer = stack_pointer; - assert(STACK_LEVEL() >= 0); + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + } } if (ctx->out_of_space) { DPRINTF(3, "\n"); @@ -489,27 +407,21 @@ optimize_uops( } if (ctx->contradiction) { // Attempted to push a "bottom" (contradiction) symbol onto the stack. - // This means that the abstract interpreter has hit unreachable code. + // This means that the abstract interpreter has optimized to trace + // to an unreachable estate. // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting - // bottom indicates type instability, so we are probably better off + // bottom usually indicates an optimizer bug, so we are probably better off // retrying later. DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); + OPT_STAT_INC(optimizer_contradiction); return 0; } /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - if (first_valid_check_stack != NULL) { - assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); - assert(max_space > 0); - assert(max_space <= INT_MAX); - assert(max_space <= INT32_MAX); - first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; - first_valid_check_stack->operand0 = max_space; - } return trace_len; error: @@ -519,10 +431,53 @@ optimize_uops( OPT_ERROR_IN_OPCODE(opcode); } _Py_uop_abstractcontext_fini(ctx); - return -1; + + assert(PyErr_Occurred()); + PyErr_Clear(); + + return 0; } +const uint16_t op_without_push[MAX_UOP_ID + 1] = { + [_COPY] = _NOP, + [_LOAD_CONST_INLINE] = _NOP, + [_LOAD_CONST_INLINE_BORROW] = _NOP, + [_LOAD_CONST_UNDER_INLINE] = _POP_TOP_LOAD_CONST_INLINE, + [_LOAD_CONST_UNDER_INLINE_BORROW] = _POP_TOP_LOAD_CONST_INLINE_BORROW, + [_LOAD_FAST] = _NOP, + [_LOAD_FAST_BORROW] = _NOP, + [_LOAD_SMALL_INT] = _NOP, + [_POP_TOP_LOAD_CONST_INLINE] = _POP_TOP, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = _POP_TOP, + [_POP_TWO_LOAD_CONST_INLINE_BORROW] = _POP_TWO, + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = _POP_CALL_TWO, +}; + +const bool op_skip[MAX_UOP_ID + 1] = { + [_NOP] = true, + [_CHECK_VALIDITY] = true, + [_CHECK_PERIODIC] = true, + [_SET_IP] = true, +}; + +const uint16_t op_without_pop[MAX_UOP_ID + 1] = { + [_POP_TOP] = _NOP, + [_POP_TOP_LOAD_CONST_INLINE] = _LOAD_CONST_INLINE, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = _LOAD_CONST_INLINE_BORROW, + [_POP_TWO] = _POP_TOP, + [_POP_TWO_LOAD_CONST_INLINE_BORROW] = _POP_TOP_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = _POP_CALL_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_TWO] = _POP_CALL_ONE, + [_POP_CALL_ONE] = _POP_CALL, +}; + +const uint16_t op_without_pop_null[MAX_UOP_ID + 1] = { + [_POP_CALL] = _POP_TOP, + [_POP_CALL_LOAD_CONST_INLINE_BORROW] = _POP_TOP_LOAD_CONST_INLINE_BORROW, +}; + static int remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) @@ -551,53 +506,41 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _NOP; } break; - case _POP_TOP: - case _POP_TOP_LOAD_CONST_INLINE: - case _POP_TOP_LOAD_CONST_INLINE_BORROW: - case _POP_TWO_LOAD_CONST_INLINE_BORROW: - optimize_pop_top_again: - { - _PyUOpInstruction *last = &buffer[pc-1]; - while (last->opcode == _NOP) { - last--; - } - switch (last->opcode) { - case _POP_TWO_LOAD_CONST_INLINE_BORROW: - last->opcode = _POP_TOP; - break; - case _POP_TOP_LOAD_CONST_INLINE: - case _POP_TOP_LOAD_CONST_INLINE_BORROW: - last->opcode = _NOP; - goto optimize_pop_top_again; - case _COPY: - case _LOAD_CONST_INLINE: - case _LOAD_CONST_INLINE_BORROW: - case _LOAD_FAST: - case _LOAD_FAST_BORROW: - case _LOAD_SMALL_INT: - last->opcode = _NOP; - if (opcode == _POP_TOP) { - opcode = buffer[pc].opcode = _NOP; - } - else if (opcode == _POP_TOP_LOAD_CONST_INLINE) { - opcode = buffer[pc].opcode = _LOAD_CONST_INLINE; - } - else if (opcode == _POP_TOP_LOAD_CONST_INLINE_BORROW) { - opcode = buffer[pc].opcode = _LOAD_CONST_INLINE_BORROW; - } - else { - assert(opcode == _POP_TWO_LOAD_CONST_INLINE_BORROW); - opcode = buffer[pc].opcode = _POP_TOP_LOAD_CONST_INLINE_BORROW; - goto optimize_pop_top_again; - } - } - _Py_FALLTHROUGH; - } + case _EXIT_TRACE: default: { + // Cancel out pushes and pops, repeatedly. So: + // _LOAD_FAST + _POP_TWO_LOAD_CONST_INLINE_BORROW + _POP_TOP + // ...becomes: + // _NOP + _POP_TOP + _NOP + while (op_without_pop[opcode] || op_without_pop_null[opcode]) { + _PyUOpInstruction *last = &buffer[pc - 1]; + while (op_skip[last->opcode]) { + last--; + } + if (op_without_push[last->opcode] && op_without_pop[opcode]) { + last->opcode = op_without_push[last->opcode]; + opcode = buffer[pc].opcode = op_without_pop[opcode]; + if (op_without_pop[last->opcode]) { + opcode = last->opcode; + pc = (int)(last - buffer); + } + } + else if (last->opcode == _PUSH_NULL) { + // Handle _POP_CALL and _POP_CALL_LOAD_CONST_INLINE_BORROW separately. + // This looks for a preceding _PUSH_NULL instruction and + // simplifies to _POP_TOP(_LOAD_CONST_INLINE_BORROW). + last->opcode = _NOP; + opcode = buffer[pc].opcode = op_without_pop_null[opcode]; + assert(opcode); + } + else { + break; + } + } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ - bool needs_ip = opcode == _PUSH_FRAME; + bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT || opcode == _EXIT_TRACE); if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; @@ -607,10 +550,14 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[last_set_ip].opcode = _SET_IP; last_set_ip = -1; } + if (opcode == _EXIT_TRACE) { + return pc + 1; + } break; } case _JUMP_TO_TOP: - case _EXIT_TRACE: + case _DYNAMIC_EXIT: + case _DEOPT: return pc + 1; } } @@ -622,7 +569,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) // > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( - _PyInterpreterFrame *frame, + PyFunctionObject *func, _PyUOpInstruction *buffer, int length, int curr_stacklen, @@ -631,19 +578,16 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - int err = remove_globals(frame, buffer, length, dependencies); - if (err <= 0) { - return err; - } - length = optimize_uops( - _PyFrame_GetCode(frame), buffer, - length, curr_stacklen, dependencies); + func, buffer, + length, curr_stacklen, dependencies); - if (length <= 0) { + if (length == 0) { return length; } + assert(length > 0); + length = remove_unneeded_uops(buffer, length); assert(length > 0); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e99421a3aff..06fa8a4522a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -27,13 +27,16 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_set_compact_int(SYM) _Py_uop_sym_set_compact_int(ctx, SYM) #define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop #define sym_new_tuple _Py_uop_sym_new_tuple #define sym_tuple_getitem _Py_uop_sym_tuple_getitem #define sym_tuple_length _Py_uop_sym_tuple_length -#define sym_is_immortal _Py_uop_sym_is_immortal +#define sym_is_immortal _Py_uop_symbol_is_immortal +#define sym_new_compact_int _Py_uop_sym_new_compact_int +#define sym_is_compact_int _Py_uop_sym_is_compact_int #define sym_new_truthiness _Py_uop_sym_new_truthiness extern int @@ -87,12 +90,12 @@ dummy_func(void) { } op(_LOAD_FAST_BORROW, (-- value)) { - value = GETLOCAL(oparg); + value = PyJitRef_Borrow(GETLOCAL(oparg)); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } @@ -104,18 +107,40 @@ dummy_func(void) { res = sym_new_null(ctx); } - op(_GUARD_TOS_INT, (tos -- tos)) { - if (sym_matches_type(tos, &PyLong_Type)) { + op(_GUARD_TOS_INT, (value -- value)) { + if (sym_is_compact_int(value)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyLong_Type); + else { + if (sym_get_type(value) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(value); + } } - op(_GUARD_NOS_INT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyLong_Type)) { + op(_GUARD_NOS_INT, (left, unused -- left, unused)) { + if (sym_is_compact_int(left)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyLong_Type); + else { + if (sym_get_type(left) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(left); + } + } + + op(_CHECK_ATTR_CLASS, (type_version/2, owner -- owner)) { + PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); + if (type) { + if (type == sym_get_const(ctx, owner)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + sym_set_const(owner, type); + } + } } op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { @@ -141,25 +166,26 @@ dummy_func(void) { } } - op(_GUARD_TOS_FLOAT, (tos -- tos)) { - if (sym_matches_type(tos, &PyFloat_Type)) { + op(_GUARD_TOS_FLOAT, (value -- value)) { + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); } - op(_GUARD_NOS_FLOAT, (nos, unused -- nos, unused)) { - if (sym_matches_type(nos, &PyFloat_Type)) { + op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); } - op(_BINARY_OP, (left, right -- res)) { - bool lhs_int = sym_matches_type(left, &PyLong_Type); - bool rhs_int = sym_matches_type(right, &PyLong_Type); - bool lhs_float = sym_matches_type(left, &PyFloat_Type); - bool rhs_float = sym_matches_type(right, &PyFloat_Type); + op(_BINARY_OP, (lhs, rhs -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(lhs, rhs); + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { // There's something other than an int or float involved: res = sym_new_unknown(ctx); @@ -185,11 +211,11 @@ dummy_func(void) { // Case C: res = sym_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, right)) { + else if (!sym_is_const(ctx, rhs)) { // Case A or B... can't know without the sign of the RHS: res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, right))) { + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { // Case B: res = sym_new_type(ctx, &PyFloat_Type); } @@ -210,140 +236,54 @@ dummy_func(void) { } op(_BINARY_OP_ADD_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and add tests! - } - else { - res = sym_new_type(ctx, &PyLong_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_compact_int(ctx); } op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyFloat_Type); + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyFloat_Type); + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - // TODO gh-115506: - // replace opcode with constant propagated one and update tests! - } - else { - res = sym_new_type(ctx, &PyFloat_Type); + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyFloat_Type); + // TODO (gh-134584): Refactor this to use another uop + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } } op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; - } - res = sym_new_const(ctx, temp); - Py_DECREF(temp); - } - else { - res = sym_new_type(ctx, &PyUnicode_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyUnicode_Type); } op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right -- )) { - JitOptSymbol *res; + JitOptRef res; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); @@ -361,12 +301,12 @@ dummy_func(void) { GETLOCAL(this_instr->operand0) = res; } - op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) { - new_frame = NULL; + op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame)) { + new_frame = PyJitRef_NULL; ctx->done = true; } - op(_BINARY_OP_SUBSCR_STR_INT, (left, right -- res)) { + op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) { res = sym_new_type(ctx, &PyUnicode_Type); } @@ -376,7 +316,7 @@ dummy_func(void) { assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = sym_tuple_length(tuple_st); + Py_ssize_t tuple_length = sym_tuple_length(tuple_st); if (tuple_length == -1) { // Unknown length res = sym_new_not_null(ctx); @@ -398,11 +338,10 @@ dummy_func(void) { } } - op(_TO_BOOL_BOOL, (value -- res)) { - int already_bool = optimize_to_bool(this_instr, ctx, value, &res); + op(_TO_BOOL_BOOL, (value -- value)) { + int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, true); } } @@ -451,11 +390,44 @@ dummy_func(void) { } op(_UNARY_NOT, (value -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); sym_set_type(value, &PyBool_Type); res = sym_new_truthiness(ctx, value, false); } + op(_UNARY_NEGATIVE, (value -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); + if (sym_is_compact_int(value)) { + res = sym_new_compact_int(ctx); + } + else { + PyTypeObject *type = sym_get_type(value); + if (type == &PyLong_Type || type == &PyFloat_Type) { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } + } + } + + op(_UNARY_INVERT, (value -- res)) { + // Required to avoid a warning due to the deprecation of bitwise inversion of bools + if (!sym_matches_type(value, &PyBool_Type)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); + } + if (sym_matches_type(value, &PyLong_Type)) { + res = sym_new_type(ctx, &PyLong_Type); + } + else { + res = sym_new_not_null(ctx); + } + } + op(_COMPARE_OP, (left, right -- res)) { + // Comparison between bytes and str or int is not impacted by this optimization as bytes + // is not a safe type (due to its ability to raise a warning during comparisons). + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } @@ -465,73 +437,50 @@ dummy_func(void) { } op(_COMPARE_OP_INT, (left, right -- res)) { - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), - sym_get_const(ctx, right), - oparg >> 5); - if (tmp == NULL) { - goto error; - } - assert(PyBool_Check(tmp)); - assert(_Py_IsImmortal(tmp)); - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = sym_new_const(ctx, tmp); - Py_DECREF(tmp); - } - else { - res = sym_new_type(ctx, &PyBool_Type); - } + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + res = sym_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_FLOAT, (left, right -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); res = sym_new_type(ctx, &PyBool_Type); } op(_COMPARE_OP_STR, (left, right -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); res = sym_new_type(ctx, &PyBool_Type); } - op(_IS_OP, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_IS_OP, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP, (left, right -- b)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP_SET, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP_SET, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } - op(_CONTAINS_OP_DICT, (left, right -- res)) { - res = sym_new_type(ctx, &PyBool_Type); + op(_CONTAINS_OP_DICT, (left, right -- b)) { + b = sym_new_type(ctx, &PyBool_Type); } op(_LOAD_CONST, (-- value)) { - PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); - int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } - - op(_LOAD_CONST_MORTAL, (-- value)) { - PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); - int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } - - op(_LOAD_CONST_IMMORTAL, (-- value)) { - PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + PyCodeObject *co = get_current_code_object(ctx); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { - PyObject *val = PyLong_FromLong(this_instr->oparg); - value = sym_new_const(ctx, val); + PyObject *val = PyLong_FromLong(oparg); + assert(val); + assert(_Py_IsImmortal(val)); + REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { @@ -539,7 +488,7 @@ dummy_func(void) { } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } op(_POP_TOP_LOAD_CONST_INLINE, (ptr/4, pop -- value)) { @@ -547,7 +496,37 @@ dummy_func(void) { } op(_POP_TOP_LOAD_CONST_INLINE_BORROW, (ptr/4, pop -- value)) { - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_CALL_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused -- value)) { + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused -- value)) { + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, (ptr/4, unused, unused, unused, unused -- value)) { + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + } + + op(_POP_TOP, (value -- )) { + PyTypeObject *typ = sym_get_type(value); + if (PyJitRef_IsBorrowed(value) || + sym_is_immortal(PyJitRef_Unwrap(value)) || + sym_is_null(value)) { + REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + } + else if (typ == &PyLong_Type) { + REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + } + else if (typ == &PyFloat_Type) { + REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + } + else if (typ == &PyUnicode_Type) { + REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + } } op(_COPY, (bottom, unused[oparg-1] -- bottom, unused[oparg-1], top)) { @@ -556,7 +535,7 @@ dummy_func(void) { } op(_SWAP, (bottom, unused[oparg-2], top -- bottom, unused[oparg-2], top)) { - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -570,7 +549,7 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (dict_version/2, index/1, owner -- attr)) { (void)dict_version; (void)index; - attr = NULL; + attr = PyJitRef_NULL; if (sym_is_const(ctx, owner)) { PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { @@ -580,11 +559,17 @@ dummy_func(void) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + if (res == NULL) { + attr = sym_new_not_null(ctx); + } + else { + attr = sym_new_const(ctx, res); + } + } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -600,10 +585,10 @@ dummy_func(void) { } } - op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { + op(_LOAD_ATTR, (owner -- attr[1], self_or_null[oparg&1])) { (void)owner; - attr = sym_new_not_null(ctx); - if (oparg &1) { + *attr = sym_new_not_null(ctx); + if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } } @@ -619,31 +604,65 @@ dummy_func(void) { } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { - attr = sym_new_not_null(ctx); (void)descr; + PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); + } + + op(_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (descr/4, owner -- attr)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); + } + + op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); } op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; } op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; } op(_LOAD_ATTR_METHOD_LAZY_DICT, (descr/4, owner -- attr, self)) { (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; } - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; } @@ -661,6 +680,16 @@ dummy_func(void) { sym_set_type(callable, &PyFunction_Type); } + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { + PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); + assert(PyMethod_Check(method)); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)method->im_func; + } + sym_set_type(callable, &PyMethod_Type); + } + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(ctx, callable)) { @@ -679,19 +708,17 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { int argcount = oparg; - PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; } - - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -700,69 +727,88 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); - + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- func, maybe_self, args[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)args; - func = sym_new_not_null(ctx); - maybe_self = sym_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { - PyCodeObject *co = NULL; + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; } - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { - new_frame = NULL; - ctx->done = true; + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } - op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) { + op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { (void)type_version; (void)args; - self = sym_new_not_null(ctx); - init = sym_new_not_null(ctx); + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { - init_frame = NULL; - ctx->done = true; + op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + /* Push self onto stack of shim */ + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); } op(_RETURN_VALUE, (retval -- res)) { - JitOptSymbol *temp = retval; + // Mimics PyStackRef_MakeHeapSafe in the interpreter. + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); DEAD(retval); SAVE_STACK(); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - - co = get_code(this_instr); - if (co == NULL) { - // might be impossible, but bailing is still safe - ctx->done = true; - } RELOAD_STACK(); res = temp; } @@ -770,42 +816,65 @@ dummy_func(void) { op(_RETURN_GENERATOR, ( -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); + } - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - - co = get_code(this_instr); - if (co == NULL) { - // might be impossible, but bailing is still safe + op(_YIELD_VALUE, (retval -- value)) { + // Mimics PyStackRef_MakeHeapSafe in the interpreter. + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + DEAD(retval); + SAVE_STACK(); + ctx->frame->stack_pointer = stack_pointer; + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + RELOAD_STACK(); + value = temp; + } + + op(_GET_ITER, (iterable -- iter, index_or_null)) { + if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { + iter = iterable; + index_or_null = sym_new_not_null(ctx); + } + else { + iter = sym_new_not_null(ctx); + index_or_null = sym_new_unknown(ctx); } } - op(_YIELD_VALUE, (unused -- res)) { - res = sym_new_unknown(ctx); - } - - op(_FOR_ITER_GEN_FRAME, ( -- )) { + op(_FOR_ITER_GEN_FRAME, (unused, unused -- unused, unused, gen_frame)) { + gen_frame = PyJitRef_NULL; /* We are about to hit the end of the trace */ ctx->done = true; } - op(_SEND_GEN_FRAME, ( -- )) { + op(_SEND_GEN_FRAME, (unused, unused -- unused, gen_frame)) { + gen_frame = PyJitRef_NULL; // We are about to hit the end of the trace: ctx->done = true; } - op(_CHECK_STACK_SPACE, ( --)) { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; + op(_CHECK_STACK_SPACE, (unused, unused, unused[oparg] -- unused, unused, unused[oparg])) { } op (_CHECK_STACK_SPACE_OPERAND, (framesize/2 -- )) { @@ -815,47 +884,43 @@ dummy_func(void) { Py_UNREACHABLE(); } - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- )) { + op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; - co = get_code(this_instr); - if (co == NULL) { - // should be about to _EXIT_TRACE anyway + stack_pointer = ctx->frame->stack_pointer; + uint64_t operand = this_instr->operand0; + if (operand == 0) { ctx->done = true; break; } - - /* Stack space handling */ - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - // won't fit in signed 32-bit int - ctx->done = true; - break; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + // No need to re-add to dependencies here. Already + // handled by the tracer. + ctx->frame->func = func; } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; + // Fixed calls don't need IP guards. + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); } - else if (corresponding_check_stack) { - // delete all but the first valid _CHECK_STACK_SPACE - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; } - op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { + op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { + (void)top; /* This has to be done manually */ for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } } - op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8])) { + op(_UNPACK_EX, (seq -- values[oparg & 0xFF], unused, unused[oparg >> 8], top[0])) { + (void)top; /* This has to be done manually */ int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { @@ -863,13 +928,23 @@ dummy_func(void) { } } - op(_ITER_NEXT_RANGE, (iter -- iter, next)) { + op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { + if (sym_matches_type(iter, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(iter, &PyTuple_Type); + } + + op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { next = sym_new_type(ctx, &PyLong_Type); } op(_CALL_TYPE_1, (unused, unused, arg -- res)) { - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + PyObject* type = (PyObject *)sym_get_type(arg); + if (type) { + res = sym_new_const(ctx, type); + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); @@ -879,13 +954,35 @@ dummy_func(void) { op(_CALL_STR_1, (unused, unused, arg -- res)) { if (sym_matches_type(arg, &PyUnicode_Type)) { // e.g. str('foo') or str(foo) where foo is known to be a string - res = arg; + // Note: we must strip the reference information because it goes + // through str() which strips the reference information from it. + res = PyJitRef_StripReferenceInfo(arg); } else { res = sym_new_type(ctx, &PyUnicode_Type); } } + op(_CALL_ISINSTANCE, (unused, unused, instance, cls -- res)) { + // the result is always a bool, but sometimes we can + // narrow it down to True or False + res = sym_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = sym_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); + if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { + // isinstance(inst, cls) where both inst and cls have + // known types, meaning we can deduce either True or False + + // The below check is equivalent to PyObject_TypeCheck(inst, cls) + PyObject *out = Py_False; + if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { + out = Py_True; + } + sym_set_const(res, out); + REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + } + } + op(_GUARD_IS_TRUE_POP, (flag -- )) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -904,27 +1001,27 @@ dummy_func(void) { sym_set_const(flag, Py_False); } - op(_GUARD_IS_NONE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + op(_GUARD_IS_NONE_POP, (val -- )) { + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(flag, Py_None); + sym_set_const(val, Py_None); } - op(_GUARD_IS_NOT_NONE_POP, (flag -- )) { - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + op(_GUARD_IS_NOT_NONE_POP, (val -- )) { + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } } @@ -956,6 +1053,10 @@ dummy_func(void) { ctx->done = true; } + op(_DEOPT, (--)) { + ctx->done = true; + } + op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); @@ -969,7 +1070,7 @@ dummy_func(void) { list = sym_new_type(ctx, &PyList_Type); } - op(_BUILD_SLICE, (values[oparg] -- slice)) { + op(_BUILD_SLICE, (args[oparg] -- slice)) { slice = sym_new_type(ctx, &PySlice_Type); } @@ -977,7 +1078,7 @@ dummy_func(void) { map = sym_new_type(ctx, &PyDict_Type); } - op(_BUILD_STRING, (values[oparg] -- str)) { + op(_BUILD_STRING, (pieces[oparg] -- str)) { str = sym_new_type(ctx, &PyUnicode_Type); } @@ -999,7 +1100,9 @@ dummy_func(void) { op(_CALL_TUPLE_1, (callable, null, arg -- res)) { if (sym_matches_type(arg, &PyTuple_Type)) { // e.g. tuple((1, 2)) or tuple(foo) where foo is known to be a tuple - res = arg; + // Note: we must strip the reference information because it goes + // through tuple() which strips the reference information from it. + res = PyJitRef_StripReferenceInfo(arg); } else { res = sym_new_type(ctx, &PyTuple_Type); @@ -1063,6 +1166,20 @@ dummy_func(void) { sym_set_null(null); } + op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { + if (sym_is_not_null(nos)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_non_null(nos); + } + + op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { + if (sym_is_null(null)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_null(null); + } + op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused, unused)) { if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1084,8 +1201,40 @@ dummy_func(void) { sym_set_const(callable, (PyObject *)&PyUnicode_Type); } - op(_CALL_LEN, (unused, unused, unused -- res)) { + op(_CALL_LEN, (callable, null, arg -- res)) { res = sym_new_type(ctx, &PyLong_Type); + Py_ssize_t tuple_length = sym_tuple_length(arg); + if (tuple_length >= 0) { + PyObject *temp = PyLong_FromSsize_t(tuple_length); + if (temp == NULL) { + goto error; + } + if (_Py_IsImmortal(temp)) { + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, + 0, (uintptr_t)temp); + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + } + + op(_GET_LEN, (obj -- obj, len)) { + Py_ssize_t tuple_length = sym_tuple_length(obj); + if (tuple_length == -1) { + len = sym_new_type(ctx, &PyLong_Type); + } + else { + assert(tuple_length >= 0); + PyObject *temp = PyLong_FromSsize_t(tuple_length); + if (temp == NULL) { + goto error; + } + if (_Py_IsImmortal(temp)) { + REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + } + len = sym_new_const(ctx, temp); + Py_DECREF(temp); + } } op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)) { @@ -1096,6 +1245,126 @@ dummy_func(void) { sym_set_const(callable, len); } + op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused -- callable, unused, unused, unused)) { + PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; + if (sym_get_const(ctx, callable) == isinstance) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_const(callable, isinstance); + } + + op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable, unused, unused)) { + PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; + if (sym_get_const(ctx, callable) == list_append) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_const(callable, list_append); + } + + op(_BINARY_SLICE, (container, start, stop -- res)) { + // Slicing a string/list/tuple always returns the same type. + PyTypeObject *type = sym_get_type(container); + if (type == &PyUnicode_Type || + type == &PyList_Type || + type == &PyTuple_Type) + { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } + } + + op(_GUARD_GLOBALS_VERSION, (version/1 --)) { + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + /* Do nothing */ + } + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; + } + if (ctx->frame->globals_checked_version == version) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + } + } + ctx->frame->globals_checked_version = version; + } + + op(_LOAD_GLOBAL_BUILTINS, (version/1, index/1 -- res)) { + (void)version; + (void)index; + PyObject *cnst = NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = interp->builtins; + if (incorrect_keys(builtins, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + /* Do nothing */ + } + else { + if (!ctx->builtins_watched) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + ctx->builtins_watched = true; + } + if (ctx->frame->globals_checked_version != 0 && ctx->frame->globals_watched) { + cnst = convert_global_to_const(this_instr, builtins, false); + } + } + if (cnst == NULL) { + res = sym_new_not_null(ctx); + } + else { + res = sym_new_const(ctx, cnst); + } + } + + op(_LOAD_GLOBAL_MODULE, (version/1, unused/1, index/1 -- res)) { + (void)index; + PyObject *cnst = NULL; + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + /* Do nothing */ + } + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; + } + if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { + REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + ctx->frame->globals_checked_version = version; + } + if (ctx->frame->globals_checked_version == version) { + cnst = convert_global_to_const(this_instr, globals, false); + } + } + } + if (cnst == NULL) { + res = sym_new_not_null(ctx); + } + else { + res = sym_new_const(ctx, cnst); + } + } + + // END BYTECODES // } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 56b4b9983fb..85bebed5867 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -11,6 +11,8 @@ break; } + /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 */ + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { break; } @@ -26,131 +28,251 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); if (sym_is_null(value)) { ctx->done = true; } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_FAST: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW: { - JitOptSymbol *value; - value = GETLOCAL(oparg); + JitOptRef value; + value = PyJitRef_Borrow(GETLOCAL(oparg)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_FAST_AND_CLEAR: { - JitOptSymbol *value; + JitOptRef value; value = GETLOCAL(oparg); - JitOptSymbol *temp = sym_new_null(ctx); + JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } - /* _LOAD_CONST is not a viable micro-op for tier 2 */ - - case _LOAD_CONST_MORTAL: { - JitOptSymbol *value; - PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); - int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; - REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _LOAD_CONST_IMMORTAL: { - JitOptSymbol *value; - PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); + case _LOAD_CONST: { + JitOptRef value; + PyCodeObject *co = get_current_code_object(ctx); + PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT: { - JitOptSymbol *value; - PyObject *val = PyLong_FromLong(this_instr->oparg); - value = sym_new_const(ctx, val); + JitOptRef value; + PyObject *val = PyLong_FromLong(oparg); + assert(val); + assert(_Py_IsImmortal(val)); + REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_FAST: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP: { + JitOptRef value; + value = stack_pointer[-1]; + PyTypeObject *typ = sym_get_type(value); + if (PyJitRef_IsBorrowed(value) || + sym_is_immortal(PyJitRef_Unwrap(value)) || + sym_is_null(value)) { + REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0); + } + else if (typ == &PyLong_Type) { + REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0); + } + else if (typ == &PyFloat_Type) { + REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0); + } + else if (typ == &PyUnicode_Type) { + REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); + } + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TOP_NOP: { + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TOP_INT: { + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TOP_FLOAT: { + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TOP_UNICODE: { + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_TWO: { + CHECK_STACK_BOUNDS(-2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _PUSH_NULL: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _END_FOR: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_ITER: { + CHECK_STACK_BOUNDS(-2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _END_SEND: { - JitOptSymbol *val; + JitOptRef val; val = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _UNARY_NEGATIVE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef value; + JitOptRef res; + value = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + goto error; + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } + if (sym_is_compact_int(value)) { + res = sym_new_compact_int(ctx); + } + else { + PyTypeObject *type = sym_get_type(value); + if (type == &PyLong_Type || type == &PyFloat_Type) { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } + } stack_pointer[-1] = res; break; } case _UNARY_NOT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + assert(PyStackRef_BoolCheck(value)); + res_stackref = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } sym_set_type(value, &PyBool_Type); res = sym_new_truthiness(ctx, value, false); stack_pointer[-1] = res; @@ -158,8 +280,8 @@ } case _TO_BOOL: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -170,21 +292,19 @@ } case _TO_BOOL_BOOL: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; value = stack_pointer[-1]; - int already_bool = optimize_to_bool(this_instr, ctx, value, &res); + int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - res = sym_new_truthiness(ctx, value, true); } - stack_pointer[-1] = res; + stack_pointer[-1] = value; break; } case _TO_BOOL_INT: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -196,7 +316,7 @@ } case _GUARD_NOS_LIST: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -206,7 +326,7 @@ } case _GUARD_TOS_LIST: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyList_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -220,8 +340,8 @@ } case _TO_BOOL_LIST: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -232,8 +352,8 @@ } case _TO_BOOL_NONE: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -245,7 +365,7 @@ } case _GUARD_NOS_UNICODE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -255,7 +375,7 @@ } case _GUARD_TOS_UNICODE: { - JitOptSymbol *value; + JitOptRef value; value = stack_pointer[-1]; if (sym_matches_type(value, &PyUnicode_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -265,8 +385,8 @@ } case _TO_BOOL_STR: { - JitOptSymbol *value; - JitOptSymbol *res; + JitOptRef value; + JitOptRef res; value = stack_pointer[-1]; int already_bool = optimize_to_bool(this_instr, ctx, value, &res); if (!already_bool) { @@ -277,7 +397,7 @@ } case _REPLACE_WITH_TRUE: { - JitOptSymbol *res; + JitOptRef res; REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); stack_pointer[-1] = res; @@ -285,256 +405,509 @@ } case _UNARY_INVERT: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef value; + JitOptRef res; + value = stack_pointer[-1]; + if (!sym_matches_type(value, &PyBool_Type)) { + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + goto error; + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } + } + if (sym_matches_type(value, &PyLong_Type)) { + res = sym_new_type(ctx, &PyLong_Type); + } + else { + res = sym_new_not_null(ctx); + } stack_pointer[-1] = res; break; } case _GUARD_NOS_INT: { - JitOptSymbol *nos; - nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyLong_Type)) { + JitOptRef left; + left = stack_pointer[-2]; + if (sym_is_compact_int(left)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyLong_Type); + else { + if (sym_get_type(left) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(left); + } break; } case _GUARD_TOS_INT: { - JitOptSymbol *tos; - tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyLong_Type)) { + JitOptRef value; + value = stack_pointer[-1]; + if (sym_is_compact_int(value)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyLong_Type); + else { + if (sym_get_type(value) == &PyLong_Type) { + REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0); + } + sym_set_compact_int(value); + } + break; + } + + case _GUARD_NOS_OVERFLOWED: { + break; + } + + case _GUARD_TOS_OVERFLOWED: { break; } case _BINARY_OP_MULTIPLY_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res_stackref = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res_stackref )) { + ctx->done = true; + break; } - res = sym_new_const(ctx, temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; } - else { - res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; - } - stack_pointer[-1] = res; + res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res_stackref = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res_stackref )) { + ctx->done = true; + break; } - res = sym_new_const(ctx, temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; } - else { - res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; - } - stack_pointer[-1] = res; + res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left), - (PyLongObject *)sym_get_const(ctx, right)); - if (temp == NULL) { - goto error; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res_stackref = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res_stackref )) { + ctx->done = true; + break; } - res = sym_new_const(ctx, temp); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; } - else { - res = sym_new_type(ctx, &PyLong_Type); - stack_pointer += -1; - } - stack_pointer[-1] = res; + res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_NOS_FLOAT: { - JitOptSymbol *nos; - nos = stack_pointer[-2]; - if (sym_matches_type(nos, &PyFloat_Type)) { + JitOptRef left; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(nos, &PyFloat_Type); + sym_set_type(left, &PyFloat_Type); break; } case _GUARD_TOS_FLOAT: { - JitOptSymbol *tos; - tos = stack_pointer[-1]; - if (sym_matches_type(tos, &PyFloat_Type)) { + JitOptRef value; + value = stack_pointer[-1]; + if (sym_matches_type(value, &PyFloat_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); } - sym_set_type(tos, &PyFloat_Type); + sym_set_type(value, &PyFloat_Type); break; } case _BINARY_OP_MULTIPLY_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) * - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res_stackref = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res_stackref )) { goto error; } - res = sym_new_const(ctx, temp); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; } - else { - res = sym_new_type(ctx, &PyFloat_Type); - stack_pointer += -1; + res = sym_new_type(ctx, &PyFloat_Type); + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } - stack_pointer[-1] = res; + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) + - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res_stackref = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res_stackref )) { goto error; } - res = sym_new_const(ctx, temp); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; } - else { - res = sym_new_type(ctx, &PyFloat_Type); - stack_pointer += -1; + res = sym_new_type(ctx, &PyFloat_Type); + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } - stack_pointer[-1] = res; + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyFloat_CheckExact(sym_get_const(ctx, left))); - assert(PyFloat_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyFloat_FromDouble( - PyFloat_AS_DOUBLE(sym_get_const(ctx, left)) - - PyFloat_AS_DOUBLE(sym_get_const(ctx, right))); - if (temp == NULL) { + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res_stackref = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res_stackref )) { goto error; } - res = sym_new_const(ctx, temp); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; } - else { - res = sym_new_type(ctx, &PyFloat_Type); - stack_pointer += -1; + res = sym_new_type(ctx, &PyFloat_Type); + if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { + REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } - stack_pointer[-1] = res; + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { + JitOptRef res; + res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); - assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); - PyObject *temp = PyUnicode_Concat(sym_get_const(ctx, left), sym_get_const(ctx, right)); - if (temp == NULL) { + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) { goto error; } - res = sym_new_const(ctx, temp); + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(temp); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; } - else { - res = sym_new_type(ctx, &PyUnicode_Type); - stack_pointer += -1; - } - stack_pointer[-1] = res; + res = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_INPLACE_ADD_UNICODE: { - JitOptSymbol *right; - JitOptSymbol *left; + JitOptRef right; + JitOptRef left; right = stack_pointer[-1]; left = stack_pointer[-2]; - JitOptSymbol *res; + JitOptRef res; if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { assert(PyUnicode_CheckExact(sym_get_const(ctx, left))); assert(PyUnicode_CheckExact(sym_get_const(ctx, right))); @@ -549,8 +922,9 @@ res = sym_new_type(ctx, &PyUnicode_Type); } GETLOCAL(this_instr->operand0) = res; + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -559,58 +933,75 @@ } case _BINARY_OP_EXTEND: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_SLICE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef container; + JitOptRef res; + container = stack_pointer[-3]; + PyTypeObject *type = sym_get_type(container); + if (type == &PyUnicode_Type || + type == &PyList_Type || + type == &PyTuple_Type) + { + res = sym_new_type(ctx, type); + } + else { + res = sym_new_not_null(ctx); + } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SLICE: { + CHECK_STACK_BOUNDS(-4); stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_LIST_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_LIST_SLICE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_STR_INT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_NOS_TUPLE: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -620,7 +1011,7 @@ } case _GUARD_TOS_TUPLE: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyTuple_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -630,9 +1021,9 @@ } case _BINARY_OP_SUBSCR_TUPLE_INT: { - JitOptSymbol *sub_st; - JitOptSymbol *tuple_st; - JitOptSymbol *res; + JitOptRef sub_st; + JitOptRef tuple_st; + JitOptRef res; sub_st = stack_pointer[-1]; tuple_st = stack_pointer[-2]; assert(sym_matches_type(tuple_st, &PyTuple_Type)); @@ -640,7 +1031,7 @@ assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); assert(index >= 0); - int tuple_length = sym_tuple_length(tuple_st); + Py_ssize_t tuple_length = sym_tuple_length(tuple_st); if (tuple_length == -1) { res = sym_new_not_null(ctx); } @@ -652,14 +1043,15 @@ else { res = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT: { - JitOptSymbol *nos; + JitOptRef nos; nos = stack_pointer[-2]; if (sym_matches_type(nos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -669,7 +1061,7 @@ } case _GUARD_TOS_DICT: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PyDict_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -679,130 +1071,148 @@ } case _BINARY_OP_SUBSCR_DICT: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_CHECK_FUNC: { - JitOptSymbol *getitem; + JitOptRef getitem; getitem = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = getitem; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_INIT_CALL: { - _Py_UOpsAbstractFrame *new_frame; - new_frame = NULL; + JitOptRef new_frame; + new_frame = PyJitRef_NULL; ctx->done = true; - stack_pointer[-3] = (JitOptSymbol *)new_frame; + CHECK_STACK_BOUNDS(-2); + stack_pointer[-3] = new_frame; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LIST_APPEND: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_ADD: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_LIST_INT: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_DICT: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_SUBSCR: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_INTRINSIC_1: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _RETURN_VALUE: { - JitOptSymbol *retval; - JitOptSymbol *res; + JitOptRef retval; + JitOptRef res; retval = stack_pointer[-1]; - JitOptSymbol *temp = retval; + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); - stack_pointer = ctx->frame->stack_pointer; - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - co = get_code(this_instr); - if (co == NULL) { + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { ctx->done = true; + break; } + int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; res = temp; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GET_AITER: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - JitOptSymbol *awaitable; + JitOptRef awaitable; awaitable = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GET_AWAITABLE: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -811,44 +1221,72 @@ /* _SEND is not a viable micro-op for tier 2 */ case _SEND_GEN_FRAME: { + JitOptRef gen_frame; + gen_frame = PyJitRef_NULL; ctx->done = true; + stack_pointer[-1] = gen_frame; break; } case _YIELD_VALUE: { - JitOptSymbol *res; - res = sym_new_unknown(ctx); - stack_pointer[-1] = res; + JitOptRef retval; + JitOptRef value; + retval = stack_pointer[-1]; + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + CHECK_STACK_BOUNDS(-1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ctx->frame->stack_pointer = stack_pointer; + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + value = temp; + CHECK_STACK_BOUNDS(1); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_EXCEPT: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_COMMON_CONSTANT: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_BUILD_CLASS: { - JitOptSymbol *bc; + JitOptRef bc; bc = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_NAME: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -857,81 +1295,95 @@ } case _UNPACK_SEQUENCE: { - JitOptSymbol **values; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; + top = &stack_pointer[-1 + oparg]; + (void)top; for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_TWO_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol *val1; - JitOptSymbol *val0; + JitOptRef seq; + JitOptRef val1; + JitOptRef val0; seq = stack_pointer[-1]; val0 = sym_tuple_getitem(ctx, seq, 0); val1 = sym_tuple_getitem(ctx, seq, 1); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_TUPLE: { - JitOptSymbol *seq; - JitOptSymbol **values; + JitOptRef seq; + JitOptRef *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_LIST: { - JitOptSymbol **values; + JitOptRef *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _UNPACK_EX: { - JitOptSymbol **values; + JitOptRef *values; + JitOptRef *top; values = &stack_pointer[-1]; + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + (void)top; int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; for (int i = 0; i < totalargs; i++) { values[i] = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS((oparg & 0xFF) + (oparg >> 8)); stack_pointer += (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_ATTR: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_GLOBAL: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -940,36 +1392,39 @@ } case _LOAD_LOCALS: { - JitOptSymbol *locals; + JitOptRef locals; locals = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - JitOptSymbol *v; + JitOptRef v; v = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_GLOBAL: { - JitOptSymbol **res; + JitOptRef *res; res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _PUSH_NULL_CONDITIONAL: { - JitOptSymbol **null; + JitOptRef *null; null = &stack_pointer[0]; if (oparg & 1) { REPLACE_OP(this_instr, _PUSH_NULL, 0, 0); @@ -978,30 +1433,113 @@ else { REPLACE_OP(this_instr, _NOP, 0, 0); } + CHECK_STACK_BOUNDS((oparg & 1)); stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_GLOBALS_VERSION: { + uint16_t version = (uint16_t)this_instr->operand0; + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + } + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; + } + if (ctx->frame->globals_checked_version == version) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + } + } + ctx->frame->globals_checked_version = version; break; } case _LOAD_GLOBAL_MODULE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + uint16_t version = (uint16_t)this_instr->operand0; + uint16_t index = (uint16_t)this_instr->operand0; + (void)index; + PyObject *cnst = NULL; + if (ctx->frame->func != NULL) { + PyObject *globals = ctx->frame->func->func_globals; + if (incorrect_keys(globals, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + } + else { + if (!ctx->frame->globals_watched) { + PyDict_Watch(GLOBALS_WATCHER_ID, globals); + _Py_BloomFilter_Add(dependencies, globals); + ctx->frame->globals_watched = true; + } + if (ctx->frame->globals_checked_version != version && this_instr[-1].opcode == _NOP) { + REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0, version); + ctx->frame->globals_checked_version = version; + } + if (ctx->frame->globals_checked_version == version) { + cnst = convert_global_to_const(this_instr, globals, false); + } + } + } + if (cnst == NULL) { + res = sym_new_not_null(ctx); + } + else { + res = sym_new_const(ctx, cnst); + } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_GLOBAL_BUILTINS: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); + JitOptRef res; + uint16_t version = (uint16_t)this_instr->operand0; + uint16_t index = (uint16_t)this_instr->operand0; + (void)version; + (void)index; + PyObject *cnst = NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *builtins = interp->builtins; + if (incorrect_keys(builtins, version)) { + OPT_STAT_INC(remove_globals_incorrect_keys); + ctx->done = true; + } + else if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { + } + else { + if (!ctx->builtins_watched) { + PyDict_Watch(BUILTINS_WATCHER_ID, builtins); + ctx->builtins_watched = true; + } + if (ctx->frame->globals_checked_version != 0 && ctx->frame->globals_watched) { + cnst = convert_global_to_const(this_instr, builtins, false); + } + } + if (cnst == NULL) { + res = sym_new_not_null(ctx); + } + else { + res = sym_new_const(ctx, cnst); + } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1018,24 +1556,26 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_DEREF: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1044,79 +1584,88 @@ } case _BUILD_STRING: { - JitOptSymbol *str; + JitOptRef str; str = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BUILD_INTERPOLATION: { - JitOptSymbol *interpolation; + JitOptRef interpolation; interpolation = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - (oparg & 1)); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BUILD_TEMPLATE: { - JitOptSymbol *template; + JitOptRef template; template = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = template; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BUILD_TUPLE: { - JitOptSymbol **values; - JitOptSymbol *tup; + JitOptRef *values; + JitOptRef tup; values = &stack_pointer[-oparg]; tup = sym_new_tuple(ctx, oparg, values); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BUILD_LIST: { - JitOptSymbol *list; + JitOptRef list; list = sym_new_type(ctx, &PyList_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LIST_EXTEND: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_UPDATE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BUILD_SET: { - JitOptSymbol *set; + JitOptRef set; set = sym_new_type(ctx, &PySet_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BUILD_MAP: { - JitOptSymbol *map; + JitOptRef map; map = sym_new_type(ctx, &PyDict_Type); + CHECK_STACK_BOUNDS(1 - oparg*2); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1125,63 +1674,69 @@ } case _DICT_UPDATE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DICT_MERGE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MAP_ADD: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_SUPER_ATTR_ATTR: { - JitOptSymbol *attr_st; + JitOptRef attr_st; attr_st = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = attr_st; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_SUPER_ATTR_METHOD: { - JitOptSymbol *attr; - JitOptSymbol *self_or_null; + JitOptRef attr; + JitOptRef self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_ATTR: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol **self_or_null; + JitOptRef owner; + JitOptRef *attr; + JitOptRef *self_or_null; owner = stack_pointer[-1]; + attr = &stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; - attr = sym_new_not_null(ctx); - if (oparg &1) { + *attr = sym_new_not_null(ctx); + if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } - stack_pointer[-1] = attr; + CHECK_STACK_BOUNDS((oparg&1)); stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION: { - JitOptSymbol *owner; + JitOptRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand0; assert(type_version); @@ -1208,7 +1763,7 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t offset = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)offset; @@ -1217,14 +1772,14 @@ } case _LOAD_ATTR_MODULE: { - JitOptSymbol *owner; - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand0; uint16_t index = (uint16_t)this_instr->operand0; (void)dict_version; (void)index; - attr = NULL; + attr = PyJitRef_NULL; if (sym_is_const(ctx, owner)) { PyModuleObject *mod = (PyModuleObject *)sym_get_const(ctx, owner); if (PyModule_CheckExact(mod)) { @@ -1235,11 +1790,16 @@ PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + if (res == NULL) { + attr = sym_new_not_null(ctx); + } + else { + attr = sym_new_const(ctx, res); + } } } } - if (attr == NULL) { + if (PyJitRef_IsNull(attr)) { attr = sym_new_not_null(ctx); } stack_pointer[-1] = attr; @@ -1247,7 +1807,7 @@ } case _LOAD_ATTR_WITH_HINT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)hint; @@ -1256,7 +1816,7 @@ } case _LOAD_ATTR_SLOT: { - JitOptSymbol *attr; + JitOptRef attr; uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)index; @@ -1265,25 +1825,43 @@ } case _CHECK_ATTR_CLASS: { + JitOptRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)this_instr->operand0; + PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version); + if (type) { + if (type == sym_get_const(ctx, owner)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + sym_set_const(owner, type); + } + } break; } case _LOAD_ATTR_CLASS: { - JitOptSymbol *attr; + JitOptRef owner; + JitOptRef attr; + owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; - attr = sym_new_not_null(ctx); (void)descr; + PyTypeObject *type = (PyTypeObject *)sym_get_const(ctx, owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_PROPERTY_FRAME: { - _Py_UOpsAbstractFrame *new_frame; + JitOptRef new_frame; PyObject *fget = (PyObject *)this_instr->operand0; (void)fget; - new_frame = NULL; + new_frame = PyJitRef_NULL; ctx->done = true; - stack_pointer[-1] = (JitOptSymbol *)new_frame; + stack_pointer[-1] = new_frame; break; } @@ -1294,107 +1872,296 @@ } case _STORE_ATTR_INSTANCE_VALUE: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_WITH_HINT: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_SLOT: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _COMPARE_OP: { - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + if (res_o == NULL) { + goto error; + } + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + if (res_bool < 0) { + goto error; + } + res_stackref = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } else { res = _Py_uop_sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _COMPARE_OP_FLOAT: { - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res_stackref = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _COMPARE_OP_INT: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (sym_is_const(ctx, left) && sym_is_const(ctx, right)) { - assert(PyLong_CheckExact(sym_get_const(ctx, left))); - assert(PyLong_CheckExact(sym_get_const(ctx, right))); - PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left), - sym_get_const(ctx, right), - oparg >> 5); - if (tmp == NULL) { - goto error; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res_stackref = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } } - assert(PyBool_Check(tmp)); - assert(_Py_IsImmortal(tmp)); - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp); - res = sym_new_const(ctx, tmp); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - Py_DECREF(tmp); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; } - else { - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer += -1; - } - stack_pointer[-1] = res; + res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _COMPARE_OP_STR: { - JitOptSymbol *res; + JitOptRef right; + JitOptRef left; + JitOptRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res_stackref = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _IS_OP: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + JitOptRef b; + b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CONTAINS_OP: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + JitOptRef right; + JitOptRef left; + JitOptRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef b_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int res = PySequence_Contains(right_o, left_o); + if (res < 0) { + goto error; + } + b_stackref = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + /* End of uop copied from bytecodes for constant evaluation */ + b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); + if (sym_is_const(ctx, b)) { + PyObject *result = sym_get_const(ctx, b); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = b; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_SET: { - JitOptSymbol *tos; + JitOptRef tos; tos = stack_pointer[-1]; if (sym_matches_type(tos, &PySet_Type) || sym_matches_type(tos, &PyFrozenSet_Type)) @@ -1405,26 +2172,28 @@ } case _CONTAINS_OP_SET: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + JitOptRef b; + b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CONTAINS_OP_DICT: { - JitOptSymbol *res; - res = sym_new_type(ctx, &PyBool_Type); - stack_pointer[-2] = res; + JitOptRef b; + b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CHECK_EG_MATCH: { - JitOptSymbol *rest; - JitOptSymbol *match; + JitOptRef rest; + JitOptRef match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1433,27 +2202,29 @@ } case _CHECK_EXC_MATCH: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _IMPORT_FROM: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1462,66 +2233,109 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - JitOptSymbol *b; + JitOptRef b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 */ + case _GET_LEN: { - JitOptSymbol *len; - len = sym_new_not_null(ctx); + JitOptRef obj; + JitOptRef len; + obj = stack_pointer[-1]; + Py_ssize_t tuple_length = sym_tuple_length(obj); + if (tuple_length == -1) { + len = sym_new_type(ctx, &PyLong_Type); + } + else { + assert(tuple_length >= 0); + PyObject *temp = PyLong_FromSsize_t(tuple_length); + if (temp == NULL) { + goto error; + } + if (_Py_IsImmortal(temp)) { + REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); + } + len = sym_new_const(ctx, temp); + CHECK_STACK_BOUNDS(1); + stack_pointer[0] = len; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + Py_DECREF(temp); + stack_pointer += -1; + } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MATCH_CLASS: { - JitOptSymbol *attrs; + JitOptRef attrs; attrs = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = attrs; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MATCH_MAPPING: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MATCH_SEQUENCE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MATCH_KEYS: { - JitOptSymbol *values_or_none; + JitOptRef values_or_none; values_or_none = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GET_ITER: { - JitOptSymbol *iter; - iter = sym_new_not_null(ctx); + JitOptRef iterable; + JitOptRef iter; + JitOptRef index_or_null; + iterable = stack_pointer[-1]; + if (sym_matches_type(iterable, &PyTuple_Type) || sym_matches_type(iterable, &PyList_Type)) { + iter = iterable; + index_or_null = sym_new_not_null(ctx); + } + else { + iter = sym_new_not_null(ctx); + index_or_null = sym_new_unknown(ctx); + } + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GET_YIELD_FROM_ITER: { - JitOptSymbol *iter; + JitOptRef iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1530,11 +2344,12 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1553,15 +2368,22 @@ /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ case _ITER_NEXT_LIST_TIER_TWO: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _ITER_CHECK_TUPLE: { + JitOptRef iter; + iter = stack_pointer[-2]; + if (sym_matches_type(iter, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(iter, &PyTuple_Type); break; } @@ -1572,11 +2394,12 @@ } case _ITER_NEXT_TUPLE: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1591,33 +2414,41 @@ } case _ITER_NEXT_RANGE: { - JitOptSymbol *next; + JitOptRef next; next = sym_new_type(ctx, &PyLong_Type); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _FOR_ITER_GEN_FRAME: { + JitOptRef gen_frame; + gen_frame = PyJitRef_NULL; ctx->done = true; + CHECK_STACK_BOUNDS(1); + stack_pointer[0] = gen_frame; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _INSERT_NULL: { - JitOptSymbol *self; - JitOptSymbol **method_and_self; + JitOptRef self; + JitOptRef *method_and_self; self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; + CHECK_STACK_BOUNDS(1); stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_SPECIAL: { - JitOptSymbol **method_and_self; + JitOptRef *method_and_self; method_and_self = &stack_pointer[-2]; method_and_self[0] = sym_new_not_null(ctx); method_and_self[1] = sym_new_unknown(ctx); @@ -1625,23 +2456,25 @@ } case _WITH_EXCEPT_START: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _PUSH_EXC_INFO: { - JitOptSymbol *prev_exc; - JitOptSymbol *new_exc; + JitOptRef prev_exc; + JitOptRef new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1654,47 +2487,73 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_NO_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - JitOptSymbol *attr; - attr = sym_new_not_null(ctx); + JitOptRef owner; + JitOptRef attr; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand0; + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - JitOptSymbol *attr; - attr = sym_new_not_null(ctx); + JitOptRef owner; + JitOptRef attr; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand0; + (void)descr; + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _POP_TOP_LOAD_CONST_INLINE_BORROW, + _POP_TOP_LOAD_CONST_INLINE); stack_pointer[-1] = attr; break; } @@ -1704,32 +2563,38 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - JitOptSymbol *owner; - JitOptSymbol *attr; - JitOptSymbol *self; + JitOptRef owner; + JitOptRef attr; + JitOptRef self; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand0; (void)descr; - attr = sym_new_not_null(ctx); + PyTypeObject *type = sym_get_type(owner); + PyObject *name = get_co_name(ctx, oparg >> 1); + attr = lookup_attr(ctx, this_instr, type, name, + _LOAD_CONST_UNDER_INLINE_BORROW, + _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MAYBE_EXPAND_METHOD: { - JitOptSymbol **args; - JitOptSymbol *func; - JitOptSymbol *maybe_self; - args = &stack_pointer[-oparg]; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; (void)args; - func = sym_new_not_null(ctx); - maybe_self = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; break; } @@ -1738,23 +2603,23 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _Py_UOpsAbstractFrame *new_frame; - PyCodeObject *co = NULL; + JitOptRef new_frame; assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; } - new_frame = frame_new(ctx, co, 0, NULL, 0); - stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + CHECK_STACK_BOUNDS(-1 - oparg); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyFunction_Type)) { @@ -1771,6 +2636,16 @@ } case _CHECK_METHOD_VERSION: { + JitOptRef callable; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)this_instr->operand0; + if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { + PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); + assert(PyMethod_Check(method)); + REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version); + this_instr->operand1 = (uintptr_t)method->im_func; + } + sym_set_type(callable, &PyMethod_Type); break; } @@ -1783,17 +2658,18 @@ } case _CALL_NON_PY_GENERAL: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *null; - JitOptSymbol *callable; + JitOptRef null; + JitOptRef callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_null(null); @@ -1802,8 +2678,8 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; callable = sym_new_not_null(ctx); @@ -1821,8 +2697,8 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - JitOptSymbol *self_or_null; - JitOptSymbol *callable; + JitOptRef self_or_null; + JitOptRef callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(sym_matches_type(callable, &PyFunction_Type)); @@ -1839,8 +2715,6 @@ } case _CHECK_STACK_SPACE: { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; break; } @@ -1849,70 +2723,67 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - JitOptSymbol **args; - JitOptSymbol *self_or_null; - _Py_UOpsAbstractFrame *new_frame; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int argcount = oparg; - PyCodeObject *co = NULL; assert((this_instr + 2)->opcode == _PUSH_FRAME); - co = get_code_with_logging((this_instr + 2)); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); if (co == NULL) { ctx->done = true; break; } - assert(self_or_null != NULL); + assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { args--; argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args, argcount)); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } - stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame; + CHECK_STACK_BOUNDS(-1 - oparg); + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _PUSH_FRAME: { - _Py_UOpsAbstractFrame *new_frame; - new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + JitOptRef new_frame; + new_frame = stack_pointer[-1]; + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } + ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; - co = get_code(this_instr); - if (co == NULL) { + stack_pointer = ctx->frame->stack_pointer; + uint64_t operand = this_instr->operand0; + if (operand == 0) { ctx->done = true; break; } - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - ctx->done = true; - break; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + ctx->frame->func = func; } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); } - else if (corresponding_check_stack) { - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; break; } case _GUARD_NOS_NULL: { - JitOptSymbol *null; + JitOptRef null; null = stack_pointer[-2]; if (sym_is_null(null)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1921,8 +2792,28 @@ break; } + case _GUARD_NOS_NOT_NULL: { + JitOptRef nos; + nos = stack_pointer[-2]; + if (sym_is_not_null(nos)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_non_null(nos); + break; + } + + case _GUARD_THIRD_NULL: { + JitOptRef null; + null = stack_pointer[-3]; + if (sym_is_null(null)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_null(null); + break; + } + case _GUARD_CALLABLE_TYPE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1932,23 +2823,27 @@ } case _CALL_TYPE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; - if (sym_has_type(arg)) { - res = sym_new_const(ctx, (PyObject *)sym_get_type(arg)); + PyObject* type = (PyObject *)sym_get_type(arg); + if (type) { + res = sym_new_const(ctx, type); + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, 0, + (uintptr_t)type); } else { res = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_STR_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1958,23 +2853,24 @@ } case _CALL_STR_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyUnicode_Type)) { - res = arg; + res = PyJitRef_StripReferenceInfo(arg); } else { res = sym_new_type(ctx, &PyUnicode_Type); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TUPLE_1: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -1984,91 +2880,114 @@ } case _CALL_TUPLE_1: { - JitOptSymbol *arg; - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyTuple_Type)) { - res = arg; + res = PyJitRef_StripReferenceInfo(arg); } else { res = sym_new_type(ctx, &PyTuple_Type); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CHECK_AND_ALLOCATE_OBJECT: { - JitOptSymbol **args; - JitOptSymbol *self; - JitOptSymbol *init; - args = &stack_pointer[-oparg]; + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t type_version = (uint32_t)this_instr->operand0; (void)type_version; (void)args; - self = sym_new_not_null(ctx); - init = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = self; - stack_pointer[-1 - oparg] = init; + callable = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; break; } case _CREATE_INIT_FRAME: { - _Py_UOpsAbstractFrame *init_frame; - init_frame = NULL; - ctx->done = true; - stack_pointer[-2 - oparg] = (JitOptSymbol *)init_frame; + JitOptRef *args; + JitOptRef self; + JitOptRef init_frame; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); + CHECK_STACK_BOUNDS(-1 - oparg); + stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _EXIT_INIT_CHECK: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_BUILTIN_CLASS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_BUILTIN_O: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_BUILTIN_FAST: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LEN: { - JitOptSymbol *callable; + JitOptRef callable; callable = stack_pointer[-3]; PyObject *len = _PyInterpreterState_GET()->callable_cache.len; if (sym_get_const(ctx, callable) == len) { @@ -2079,62 +2998,125 @@ } case _CALL_LEN: { - JitOptSymbol *res; + JitOptRef arg; + JitOptRef res; + arg = stack_pointer[-1]; res = sym_new_type(ctx, &PyLong_Type); + Py_ssize_t tuple_length = sym_tuple_length(arg); + if (tuple_length >= 0) { + PyObject *temp = PyLong_FromSsize_t(tuple_length); + if (temp == NULL) { + goto error; + } + if (_Py_IsImmortal(temp)) { + REPLACE_OP(this_instr, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, + 0, (uintptr_t)temp); + } + res = sym_new_const(ctx, temp); + CHECK_STACK_BOUNDS(-2); + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + Py_DECREF(temp); + stack_pointer += 2; + } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _GUARD_CALLABLE_ISINSTANCE: { + JitOptRef callable; + callable = stack_pointer[-4]; + PyObject *isinstance = _PyInterpreterState_GET()->callable_cache.isinstance; + if (sym_get_const(ctx, callable) == isinstance) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_const(callable, isinstance); break; } case _CALL_ISINSTANCE: { - JitOptSymbol *res; - res = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + JitOptRef cls; + JitOptRef instance; + JitOptRef res; + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + res = sym_new_type(ctx, &PyBool_Type); + PyTypeObject *inst_type = sym_get_type(instance); + PyTypeObject *cls_o = (PyTypeObject *)sym_get_const(ctx, cls); + if (inst_type && cls_o && sym_matches_type(cls, &PyType_Type)) { + PyObject *out = Py_False; + if (inst_type == cls_o || PyType_IsSubtype(inst_type, cls_o)) { + out = Py_True; + } + sym_set_const(res, out); + REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); + } + CHECK_STACK_BOUNDS(-3); + stack_pointer[-4] = res; + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _GUARD_CALLABLE_LIST_APPEND: { + JitOptRef callable; + callable = stack_pointer[-3]; + PyObject *list_append = _PyInterpreterState_GET()->callable_cache.list_append; + if (sym_get_const(ctx, callable) == list_append) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_const(callable, list_append); break; } case _CALL_LIST_APPEND: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_O: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_FAST: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2147,12 +3129,18 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _Py_UOpsAbstractFrame *new_frame; - new_frame = NULL; - ctx->done = true; - stack_pointer[-3 - oparg] = (JitOptSymbol *)new_frame; + JitOptRef new_frame; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + CHECK_STACK_BOUNDS(-2 - oparg); + stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2173,11 +3161,12 @@ } case _CALL_KW_NON_PY: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2 - oparg); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2188,97 +3177,134 @@ /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - JitOptSymbol *func; + JitOptRef func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - JitOptSymbol *func_out; + JitOptRef func_out; func_out = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _RETURN_GENERATOR: { - JitOptSymbol *res; + JitOptRef res; ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - co = get_code(this_instr); - if (co == NULL) { - ctx->done = true; - } + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BUILD_SLICE: { - JitOptSymbol *slice; + JitOptRef slice; slice = sym_new_type(ctx, &PySlice_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CONVERT_VALUE: { - JitOptSymbol *result; + JitOptRef result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - JitOptSymbol *res; + JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _COPY: { - JitOptSymbol *bottom; - JitOptSymbol *top; + JitOptRef bottom; + JitOptRef top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _BINARY_OP: { - JitOptSymbol *right; - JitOptSymbol *left; - JitOptSymbol *res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - bool lhs_int = sym_matches_type(left, &PyLong_Type); - bool rhs_int = sym_matches_type(right, &PyLong_Type); - bool lhs_float = sym_matches_type(left, &PyFloat_Type); - bool rhs_float = sym_matches_type(right, &PyFloat_Type); + JitOptRef rhs; + JitOptRef lhs; + JitOptRef res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + if ( + sym_is_safe_const(ctx, lhs) && + sym_is_safe_const(ctx, rhs) + ) { + JitOptRef lhs_sym = lhs; + JitOptRef rhs_sym = rhs; + _PyStackRef lhs = sym_get_const_as_stackref(ctx, lhs_sym); + _PyStackRef rhs = sym_get_const_as_stackref(ctx, rhs_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + bool lhs_int = sym_matches_type(lhs, &PyLong_Type); + bool rhs_int = sym_matches_type(rhs, &PyLong_Type); + bool lhs_float = sym_matches_type(lhs, &PyFloat_Type); + bool rhs_float = sym_matches_type(rhs, &PyFloat_Type); if (!((lhs_int || lhs_float) && (rhs_int || rhs_float))) { res = sym_new_unknown(ctx); } @@ -2289,10 +3315,10 @@ else if (lhs_float) { res = sym_new_type(ctx, &PyFloat_Type); } - else if (!sym_is_const(ctx, right)) { + else if (!sym_is_const(ctx, rhs)) { res = sym_new_unknown(ctx); } - else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, right))) { + else if (_PyLong_IsNegative((PyLongObject *)sym_get_const(ctx, rhs))) { res = sym_new_type(ctx, &PyFloat_Type); } else { @@ -2308,18 +3334,19 @@ else { res = sym_new_type(ctx, &PyFloat_Type); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SWAP: { - JitOptSymbol *top; - JitOptSymbol *bottom; + JitOptRef top; + JitOptRef bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; - JitOptSymbol *temp = bottom; + JitOptRef temp = bottom; bottom = top; top = temp; assert(oparg >= 2); @@ -2347,7 +3374,7 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -2355,13 +3382,14 @@ eliminate_pop_guard(this_instr, value != Py_True); } sym_set_const(flag, Py_True); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_IS_FALSE_POP: { - JitOptSymbol *flag; + JitOptRef flag; flag = stack_pointer[-1]; if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); @@ -2369,43 +3397,46 @@ eliminate_pop_guard(this_instr, value != Py_False); } sym_set_const(flag, Py_False); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_IS_NONE_POP: { - JitOptSymbol *flag; - flag = stack_pointer[-1]; - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + JitOptRef val; + val = stack_pointer[-1]; + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, !Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, true); } - sym_set_const(flag, Py_None); + sym_set_const(val, Py_None); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_IS_NOT_NONE_POP: { - JitOptSymbol *flag; - flag = stack_pointer[-1]; - if (sym_is_const(ctx, flag)) { - PyObject *value = sym_get_const(ctx, flag); + JitOptRef val; + val = stack_pointer[-1]; + if (sym_is_const(ctx, val)) { + PyObject *value = sym_get_const(ctx, val); assert(value != NULL); eliminate_pop_guard(this_instr, Py_IsNone(value)); } - else if (sym_has_type(flag)) { - assert(!sym_matches_type(flag, &_PyNone_Type)); + else if (sym_has_type(val)) { + assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2436,22 +3467,27 @@ break; } + case _DYNAMIC_EXIT: { + break; + } + case _CHECK_VALIDITY: { break; } case _LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_LOAD_CONST_INLINE: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); stack_pointer[-1] = value; @@ -2459,33 +3495,111 @@ } case _LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_CALL: { + CHECK_STACK_BOUNDS(-2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_CALL_ONE: { + CHECK_STACK_BOUNDS(-3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_CALL_TWO: { + CHECK_STACK_BOUNDS(-4); + stack_pointer += -4; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = sym_new_const(ctx, ptr); + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-1] = value; break; } case _POP_TWO_LOAD_CONST_INLINE_BORROW: { - JitOptSymbol *value; + JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } - case _CHECK_FUNCTION: { + case _POP_CALL_LOAD_CONST_INLINE_BORROW: { + JitOptRef value; + PyObject *ptr = (PyObject *)this_instr->operand0; + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = value; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { + JitOptRef value; + PyObject *ptr = (PyObject *)this_instr->operand0; + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-2); + stack_pointer[-3] = value; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { + JitOptRef value; + PyObject *ptr = (PyObject *)this_instr->operand0; + value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-3); + stack_pointer[-4] = value; + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _LOAD_CONST_UNDER_INLINE: { + JitOptRef value; + JitOptRef new; + value = sym_new_not_null(ctx); + new = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-1] = value; + stack_pointer[0] = new; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _LOAD_CONST_UNDER_INLINE_BORROW: { + JitOptRef value; + JitOptRef new; + value = sym_new_not_null(ctx); + new = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-1] = value; + stack_pointer[0] = new; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2502,6 +3616,11 @@ } case _DEOPT: { + ctx->done = true; + break; + } + + case _HANDLE_PENDING_AND_DEOPT: { break; } @@ -2513,3 +3632,29 @@ break; } + case _COLD_EXIT: { + break; + } + + case _COLD_DYNAMIC_EXIT: { + break; + } + + case _GUARD_IP__PUSH_FRAME: { + break; + } + + case _GUARD_IP_YIELD_VALUE: { + break; + } + + case _GUARD_IP_RETURN_VALUE: { + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + break; + } + + /* _TRACE_RECORD is not a viable micro-op for tier 2 */ + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index e8a4f87031b..8a71eff465e 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -7,28 +7,55 @@ #include "pycore_long.h" #include "pycore_optimizer.h" #include "pycore_stats.h" -#include "pycore_tuple.h" // _PyTuple_FromArray() #include <stdbool.h> #include <stdint.h> #include <stddef.h> -/* Symbols - ======= +/* - See the diagram at - https://github.com/faster-cpython/ideas/blob/main/3.13/redundancy_eliminator.md +Symbols +======= - We represent the nodes in the diagram as follows - (the flag bits are only defined in optimizer_symbols.c): - - Top: no flag bits, typ and const_val are NULL. - - NULL: IS_NULL flag set, type and const_val NULL. - - Not NULL: NOT_NULL flag set, type and const_val NULL. - - None/not None: not used. (None could be represented as any other constant.) - - Known type: NOT_NULL flag set and typ set; const_val is NULL. - - Known constant: NOT_NULL flag set, type set, const_val set. - - Bottom: IS_NULL and NOT_NULL flags set, type and const_val NULL. - */ +https://github.com/faster-cpython/ideas/blob/main/3.13/redundancy_eliminator.md + +Logically, all symbols begin as UNKNOWN, and can transition downwards along the +edges of the lattice, but *never* upwards (see the diagram below). The UNKNOWN +state represents no information, and the BOTTOM state represents contradictory +information. Though symbols logically progress through all intermediate nodes, +we often skip in-between states for convenience: + + UNKNOWN + | | +NULL | +| | <- Anything below this level is an object. +| NON_NULL-+ +| | | <- Anything below this level has a known type version. +| TYPE_VERSION | +| | | <- Anything below this level has a known type. +| KNOWN_CLASS | +| | | | | | +| | | INT* | | +| | | | | | <- Anything below this level has a known truthiness. +| | | | | TRUTHINESS +| | | | | | +| TUPLE | | | | +| | | | | | <- Anything below this level is a known constant. +| KNOWN_VALUE--+ +| | <- Anything below this level is unreachable. +BOTTOM + +For example, after guarding that the type of an UNKNOWN local is int, we can +narrow the symbol to KNOWN_CLASS (logically progressing though NON_NULL and +TYPE_VERSION to get there). Later, we may learn that it is falsey based on the +result of a truth test, which would allow us to narrow the symbol to KNOWN_VALUE +(with a value of integer zero). If at any point we encounter a float guard on +the same symbol, that would be a contradiction, and the symbol would be set to +BOTTOM (indicating that the code is unreachable). + +INT* is a limited range int, currently a "compact" int. + +*/ #ifdef Py_DEBUG static inline int get_lltrace(void) { @@ -64,6 +91,12 @@ out_of_space(JitOptContext *ctx) return &NO_SPACE_SYMBOL; } +JitOptRef +out_of_space_ref(JitOptContext *ctx) +{ + return PyJitRef_Wrap(out_of_space(ctx)); +} + static JitOptSymbol * sym_new(JitOptContext *ctx) { @@ -74,7 +107,7 @@ sym_new(JitOptContext *ctx) return NULL; } ctx->t_arena.ty_curr_number++; - self->tag = JIT_SYM_UNKNOWN_TAG; + self->tag = JIT_SYM_UNKNOWN_TAG;; return self; } @@ -93,25 +126,28 @@ sym_set_bottom(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_bottom(JitOptSymbol *sym) +_Py_uop_sym_is_bottom(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_not_null(JitOptSymbol *sym) { +_Py_uop_sym_is_not_null(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); return sym->tag == JIT_SYM_NON_NULL_TAG || sym->tag > JIT_SYM_BOTTOM_TAG; } bool -_Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return true; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return false; } @@ -122,21 +158,22 @@ _Py_uop_sym_is_const(JitOptContext *ctx, JitOptSymbol *sym) } bool -_Py_uop_sym_is_null(JitOptSymbol *sym) +_Py_uop_sym_is_null(JitOptRef ref) { - return sym->tag == JIT_SYM_NULL_TAG; + return PyJitRef_Unwrap(ref)->tag == JIT_SYM_NULL_TAG; } PyObject * -_Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_get_const(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return sym->value.value; } if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, PyJitRef_Wrap(value)); if (truthiness < 0) { return NULL; } @@ -147,9 +184,41 @@ _Py_uop_sym_get_const(JitOptContext *ctx, JitOptSymbol *sym) return NULL; } -void -_Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) +_PyStackRef +_Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptRef sym) { + PyObject *const_val = _Py_uop_sym_get_const(ctx, sym); + if (const_val == NULL) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectBorrow(const_val); +} + +/* + Indicates whether the constant is safe to constant evaluate + (without side effects). + */ +bool +_Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym) +{ + PyObject *const_val = _Py_uop_sym_get_const(ctx, sym); + if (const_val == NULL) { + return false; + } + if (_PyLong_CheckExactAndCompact(const_val)) { + return true; + } + PyTypeObject *typ = Py_TYPE(const_val); + return (typ == &PyUnicode_Type) || + (typ == &PyFloat_Type) || + (typ == &_PyNone_Type) || + (typ == &PyBool_Type); +} + +void +_Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef ref, PyTypeObject *typ) +{ + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -194,12 +263,22 @@ _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ) sym_set_bottom(ctx, sym); } return; + case JIT_SYM_COMPACT_INT: + if (typ != &PyLong_Type) { + sym_set_bottom(ctx, sym); + } + return; } } bool -_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version) +_Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptRef ref, unsigned int version) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); + PyTypeObject *type = _PyType_LookupByVersion(version); + if (type) { + _Py_uop_sym_set_type(ctx, ref, type); + } JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -215,18 +294,24 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int return true; } case JIT_SYM_KNOWN_VALUE_TAG: - Py_CLEAR(sym->value.value); - sym_set_bottom(ctx, sym); - return false; + if (Py_TYPE(sym->value.value)->tp_version_tag != version) { + Py_CLEAR(sym->value.value); + sym_set_bottom(ctx, sym); + return false; + }; + return true; case JIT_SYM_TUPLE_TAG: - sym_set_bottom(ctx, sym); - return false; + if (PyTuple_Type.tp_version_tag != version) { + sym_set_bottom(ctx, sym); + return false; + }; + return true; case JIT_SYM_TYPE_VERSION_TAG: - if (sym->version.version == version) { - return true; + if (sym->version.version != version) { + sym_set_bottom(ctx, sym); + return false; } - sym_set_bottom(ctx, sym); - return false; + return true; case JIT_SYM_BOTTOM_TAG: return false; case JIT_SYM_NON_NULL_TAG: @@ -240,13 +325,20 @@ _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int return false; } return true; + case JIT_SYM_COMPACT_INT: + if (version != PyLong_Type.tp_version_tag) { + sym_set_bottom(ctx, sym); + return false; + } + return true; } Py_UNREACHABLE(); } void -_Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val) +_Py_uop_sym_set_const(JitOptContext *ctx, JitOptRef ref, PyObject *const_val) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -266,6 +358,18 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val } return; case JIT_SYM_TUPLE_TAG: + if (PyTuple_CheckExact(const_val)) { + Py_ssize_t len = _Py_uop_sym_tuple_length(ref); + if (len == PyTuple_GET_SIZE(const_val)) { + for (Py_ssize_t i = 0; i < len; i++) { + JitOptRef sym_item = _Py_uop_sym_tuple_getitem(ctx, ref, i); + PyObject *item = PyTuple_GET_ITEM(const_val, i); + _Py_uop_sym_set_const(ctx, sym_item, item); + } + make_const(sym, const_val); + return; + } + } sym_set_bottom(ctx, sym); return; case JIT_SYM_TYPE_VERSION_TAG: @@ -283,13 +387,14 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val return; case JIT_SYM_TRUTHINESS_TAG: if (!PyBool_Check(const_val) || - (_Py_uop_sym_is_const(ctx, sym) && - _Py_uop_sym_get_const(ctx, sym) != const_val)) + (_Py_uop_sym_is_const(ctx, ref) && + _Py_uop_sym_get_const(ctx, ref) != const_val)) { sym_set_bottom(ctx, sym); return; } - JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; + JitOptRef value = PyJitRef_Wrap( + allocation_base(ctx) + sym->truthiness.value); PyTypeObject *type = _Py_uop_sym_get_type(value); if (const_val == (sym->truthiness.invert ? Py_False : Py_True)) { // value is truthy. This is only useful for bool: @@ -310,12 +415,21 @@ _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val // TODO: More types (GH-130415)! make_const(sym, const_val); return; + case JIT_SYM_COMPACT_INT: + if (_PyLong_CheckExactAndCompact(const_val)) { + make_const(sym, const_val); + } + else { + sym_set_bottom(ctx, sym); + } + return; } } void -_Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NULL_TAG; } @@ -325,8 +439,9 @@ _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym) } void -_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_UNKNOWN_TAG) { sym->tag = JIT_SYM_NON_NULL_TAG; } @@ -335,70 +450,82 @@ _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym) } } - -JitOptSymbol * +JitOptRef _Py_uop_sym_new_unknown(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_not_null(JitOptContext *ctx) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } res->tag = JIT_SYM_NON_NULL_TAG; - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_type(JitOptContext *ctx, PyTypeObject *typ) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_type(ctx, res, typ); - return res; + JitOptRef ref = PyJitRef_Wrap(res); + _Py_uop_sym_set_type(ctx, ref, typ); + return ref; } // Adds a new reference to const_val, owned by the symbol. -JitOptSymbol * +JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val) { assert(const_val != NULL); JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_const(ctx, res, const_val); + JitOptRef ref = PyJitRef_Wrap(res); + _Py_uop_sym_set_const(ctx, ref, const_val); + return ref; +} + +JitOptRef +_Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val) +{ + assert(const_val != NULL); + JitOptRef res = _Py_uop_sym_new_const(ctx, const_val); + // Decref once because sym_new_const increfs it. + Py_DECREF(const_val); return res; } -JitOptSymbol * +JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx) { JitOptSymbol *null_sym = sym_new(ctx); if (null_sym == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - _Py_uop_sym_set_null(ctx, null_sym); - return null_sym; + JitOptRef ref = PyJitRef_Wrap(null_sym); + _Py_uop_sym_set_null(ctx, ref); + return ref; } PyTypeObject * -_Py_uop_sym_get_type(JitOptSymbol *sym) +_Py_uop_sym_get_type(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: - case JIT_SYM_TYPE_VERSION_TAG: case JIT_SYM_BOTTOM_TAG: case JIT_SYM_NON_NULL_TAG: case JIT_SYM_UNKNOWN_TAG: @@ -407,17 +534,23 @@ _Py_uop_sym_get_type(JitOptSymbol *sym) return sym->cls.type; case JIT_SYM_KNOWN_VALUE_TAG: return Py_TYPE(sym->value.value); + case JIT_SYM_TYPE_VERSION_TAG: + return _PyType_LookupByVersion(sym->version.version); case JIT_SYM_TUPLE_TAG: return &PyTuple_Type; case JIT_SYM_TRUTHINESS_TAG: return &PyBool_Type; + case JIT_SYM_COMPACT_INT: + return &PyLong_Type; + } Py_UNREACHABLE(); } unsigned int -_Py_uop_sym_get_type_version(JitOptSymbol *sym) +_Py_uop_sym_get_type_version(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); JitSymType tag = sym->tag; switch(tag) { case JIT_SYM_NULL_TAG: @@ -435,52 +568,42 @@ _Py_uop_sym_get_type_version(JitOptSymbol *sym) return PyTuple_Type.tp_version_tag; case JIT_SYM_TRUTHINESS_TAG: return PyBool_Type.tp_version_tag; + case JIT_SYM_COMPACT_INT: + return PyLong_Type.tp_version_tag; } Py_UNREACHABLE(); } bool -_Py_uop_sym_has_type(JitOptSymbol *sym) +_Py_uop_sym_has_type(JitOptRef sym) { - JitSymType tag = sym->tag; - switch(tag) { - case JIT_SYM_NULL_TAG: - case JIT_SYM_TYPE_VERSION_TAG: - case JIT_SYM_BOTTOM_TAG: - case JIT_SYM_NON_NULL_TAG: - case JIT_SYM_UNKNOWN_TAG: - return false; - case JIT_SYM_KNOWN_CLASS_TAG: - case JIT_SYM_KNOWN_VALUE_TAG: - case JIT_SYM_TUPLE_TAG: - case JIT_SYM_TRUTHINESS_TAG: - return true; - } - Py_UNREACHABLE(); + return _Py_uop_sym_get_type(sym) != NULL; } bool -_Py_uop_sym_matches_type(JitOptSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); return _Py_uop_sym_get_type(sym) == typ; } bool -_Py_uop_sym_matches_type_version(JitOptSymbol *sym, unsigned int version) +_Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version) { return _Py_uop_sym_get_type_version(sym) == version; } int -_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) +_Py_uop_sym_truthiness(JitOptContext *ctx, JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); switch(sym->tag) { case JIT_SYM_NULL_TAG: case JIT_SYM_TYPE_VERSION_TAG: case JIT_SYM_BOTTOM_TAG: case JIT_SYM_NON_NULL_TAG: case JIT_SYM_UNKNOWN_TAG: + case JIT_SYM_COMPACT_INT: return -1; case JIT_SYM_KNOWN_CLASS_TAG: /* TODO : @@ -494,7 +617,8 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) case JIT_SYM_TRUTHINESS_TAG: ; JitOptSymbol *value = allocation_base(ctx) + sym->truthiness.value; - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, + PyJitRef_Wrap(value)); if (truthiness < 0) { return truthiness; } @@ -520,12 +644,12 @@ _Py_uop_sym_truthiness(JitOptContext *ctx, JitOptSymbol *sym) return -1; } -JitOptSymbol * -_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) +JitOptRef +_Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptRef *args) { JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } if (size > MAX_SYMBOLIC_TUPLE_SIZE) { res->tag = JIT_SYM_KNOWN_CLASS_TAG; @@ -535,15 +659,16 @@ _Py_uop_sym_new_tuple(JitOptContext *ctx, int size, JitOptSymbol **args) res->tag = JIT_SYM_TUPLE_TAG; res->tuple.length = size; for (int i = 0; i < size; i++) { - res->tuple.items[i] = (uint16_t)(args[i] - allocation_base(ctx)); + res->tuple.items[i] = (uint16_t)(PyJitRef_Unwrap(args[i]) - allocation_base(ctx)); } } - return res; + return PyJitRef_Wrap(res); } -JitOptSymbol * -_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) +JitOptRef +_Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptRef ref, Py_ssize_t item) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); assert(item >= 0); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; @@ -552,14 +677,15 @@ _Py_uop_sym_tuple_getitem(JitOptContext *ctx, JitOptSymbol *sym, int item) } } else if (sym->tag == JIT_SYM_TUPLE_TAG && item < sym->tuple.length) { - return allocation_base(ctx) + sym->tuple.items[item]; + return PyJitRef_Wrap(allocation_base(ctx) + sym->tuple.items[item]); } - return _Py_uop_sym_new_unknown(ctx); + return _Py_uop_sym_new_not_null(ctx); } -int -_Py_uop_sym_tuple_length(JitOptSymbol *sym) +Py_ssize_t +_Py_uop_sym_tuple_length(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { PyObject *tuple = sym->value.value; if (PyTuple_CheckExact(tuple)) { @@ -574,7 +700,7 @@ _Py_uop_sym_tuple_length(JitOptSymbol *sym) // Return true if known to be immortal. bool -_Py_uop_sym_is_immortal(JitOptSymbol *sym) +_Py_uop_symbol_is_immortal(JitOptSymbol *sym) { if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { return _Py_IsImmortal(sym->value.value); @@ -582,25 +708,84 @@ _Py_uop_sym_is_immortal(JitOptSymbol *sym) if (sym->tag == JIT_SYM_KNOWN_CLASS_TAG) { return sym->cls.type == &PyBool_Type; } - if (sym->tag == JIT_SYM_TRUTHINESS_TAG) { - return true; - } return false; } -JitOptSymbol * -_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) +bool +_Py_uop_sym_is_compact_int(JitOptRef ref) { + JitOptSymbol *sym = PyJitRef_Unwrap(ref); + if (sym->tag == JIT_SYM_KNOWN_VALUE_TAG) { + return (bool)_PyLong_CheckExactAndCompact(sym->value.value); + } + return sym->tag == JIT_SYM_COMPACT_INT; +} + +bool +_Py_uop_sym_is_immortal(JitOptRef ref) +{ + JitOptSymbol *sym = PyJitRef_Unwrap(ref); + return _Py_uop_symbol_is_immortal(sym); +} + +void +_Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef ref) +{ + JitOptSymbol *sym = PyJitRef_Unwrap(ref); + JitSymType tag = sym->tag; + switch(tag) { + case JIT_SYM_NULL_TAG: + sym_set_bottom(ctx, sym); + return; + case JIT_SYM_KNOWN_CLASS_TAG: + if (sym->cls.type == &PyLong_Type) { + sym->tag = JIT_SYM_COMPACT_INT; + } else { + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_TYPE_VERSION_TAG: + if (sym->version.version == PyLong_Type.tp_version_tag) { + sym->tag = JIT_SYM_COMPACT_INT; + } + else { + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_KNOWN_VALUE_TAG: + if (!_PyLong_CheckExactAndCompact(sym->value.value)) { + Py_CLEAR(sym->value.value); + sym_set_bottom(ctx, sym); + } + return; + case JIT_SYM_TUPLE_TAG: + case JIT_SYM_TRUTHINESS_TAG: + sym_set_bottom(ctx, sym); + return; + case JIT_SYM_BOTTOM_TAG: + case JIT_SYM_COMPACT_INT: + return; + case JIT_SYM_NON_NULL_TAG: + case JIT_SYM_UNKNOWN_TAG: + sym->tag = JIT_SYM_COMPACT_INT; + return; + } +} + +JitOptRef +_Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef ref, bool truthy) +{ + JitOptSymbol *value = PyJitRef_Unwrap(ref); // It's clearer to invert this in the signature: bool invert = !truthy; if (value->tag == JIT_SYM_TRUTHINESS_TAG && value->truthiness.invert == invert) { - return value; + return ref; } JitOptSymbol *res = sym_new(ctx); if (res == NULL) { - return out_of_space(ctx); + return out_of_space_ref(ctx); } - int truthiness = _Py_uop_sym_truthiness(ctx, value); + int truthiness = _Py_uop_sym_truthiness(ctx, ref); if (truthiness < 0) { res->tag = JIT_SYM_TRUTHINESS_TAG; res->truthiness.invert = invert; @@ -609,7 +794,18 @@ _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptSymbol *value, bool truthy) else { make_const(res, (truthiness ^ invert) ? Py_True : Py_False); } - return res; + return PyJitRef_Wrap(res); +} + +JitOptRef +_Py_uop_sym_new_compact_int(JitOptContext *ctx) +{ + JitOptSymbol *sym = sym_new(ctx); + if (sym == NULL) { + return out_of_space_ref(ctx); + } + sym->tag = JIT_SYM_COMPACT_INT; + return PyJitRef_Wrap(sym); } // 0 on success, -1 on error. @@ -618,18 +814,26 @@ _Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, int curr_stackentries, - JitOptSymbol **args, + JitOptRef *args, int arg_len) { - assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + if (ctx->curr_frame_depth >= MAX_ABSTRACT_FRAME_DEPTH) { + ctx->done = true; + ctx->out_of_space = true; + OPT_STAT_INC(optimizer_frame_overflow); + return NULL; + } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; - + frame->code = co; frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; frame->locals = ctx->n_consumed; frame->stack = frame->locals + co->co_nlocalsplus; frame->stack_pointer = frame->stack + curr_stackentries; + frame->globals_checked_version = 0; + frame->globals_watched = false; + frame->func = NULL; ctx->n_consumed = ctx->n_consumed + (co->co_nlocalsplus + co->co_stacksize); if (ctx->n_consumed >= ctx->limit) { ctx->done = true; @@ -643,14 +847,14 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptSymbol *local = _Py_uop_sym_new_unknown(ctx); + JitOptRef local = _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - JitOptSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); + JitOptRef stackvar = _Py_uop_sym_new_unknown(ctx); frame->stack[i] = stackvar; } @@ -676,12 +880,12 @@ _Py_uop_abstractcontext_fini(JitOptContext *ctx) void _Py_uop_abstractcontext_init(JitOptContext *ctx) { - static_assert(sizeof(JitOptSymbol) <= 2 * sizeof(uint64_t), "JitOptSymbol has grown"); + static_assert(sizeof(JitOptSymbol) <= 3 * sizeof(uint64_t), "JitOptSymbol has grown"); ctx->limit = ctx->locals_and_stack + MAX_ABSTRACT_INTERP_SIZE; ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; + ctx->locals_and_stack[i] = PyJitRef_NULL; } #endif @@ -691,16 +895,53 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) // Frame setup ctx->curr_frame_depth = 0; + + // Ctx signals. + // Note: this must happen before frame_new, as it might override + // the result should frame_new set things to bottom. + ctx->done = false; + ctx->out_of_space = false; + ctx->contradiction = false; + ctx->builtins_watched = false; } int -_Py_uop_frame_pop(JitOptContext *ctx) +_Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries) { _Py_UOpsAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; - assert(ctx->curr_frame_depth >= 1); - ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + if (ctx->curr_frame_depth >= 1) { + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + // We returned to the correct code. Nothing to do here. + if (co == ctx->frame->code) { + return 0; + } + // Else: the code we recorded doesn't match the code we *think* we're + // returning to. We could trace anything, we can't just return to the + // old frame. We have to restore what the tracer recorded + // as the traced next frame. + // Remove the current frame, and later swap it out with the right one. + else { + ctx->curr_frame_depth--; + } + } + // Else: trace stack underflow. + + // This handles swapping out frames. + assert(curr_stackentries >= 1); + // -1 to stackentries as we push to the stack our return value after this. + _Py_UOpsAbstractFrame *new_frame = _Py_uop_frame_new(ctx, co, curr_stackentries - 1, NULL, 0); + if (new_frame == NULL) { + ctx->done = true; + return 1; + } + + ctx->curr_frame_depth++; + ctx->frame = new_frame; return 0; } @@ -731,47 +972,48 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_uop_abstractcontext_init(ctx); PyObject *val_42 = NULL; PyObject *val_43 = NULL; + PyObject *val_big = NULL; PyObject *tuple = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - JitOptSymbol *sym = _Py_uop_sym_new_unknown(ctx); - if (sym == NULL) { + JitOptRef ref = _Py_uop_sym_new_unknown(ctx); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "top is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "top matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "top is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "top as constant is not NULL"); - TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "top is NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "top is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "top matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "top is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "top as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_bottom(ref), "top is bottom"); - sym = make_bottom(ctx); - if (sym == NULL) { + ref = PyJitRef_Wrap(make_bottom(ctx)); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_is_not_null(sym), "bottom is not NULL is not false"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyLong_Type), "bottom matches a type"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "bottom is a constant is not false"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "bottom as constant is not NULL"); - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "bottom is NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_is_not_null(ref), "bottom is not NULL is not false"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyLong_Type), "bottom matches a type"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "bottom is a constant is not false"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "bottom as constant is not NULL"); + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "bottom isn't bottom"); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "int isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "int isn't int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "int matches float"); - TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, sym), "int is a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "int as constant is not NULL"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "int is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "int isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "int isn't int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "int matches float"); + TEST_PREDICATE(!_Py_uop_sym_is_const(ctx, ref), "int is a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "int as constant is not NULL"); - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(int and int) isn't int"); + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(int and int) isn't int"); - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(int and float) isn't bottom"); + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(int and float) isn't bottom"); val_42 = PyLong_FromLong(42); assert(val_42 != NULL); @@ -781,84 +1023,118 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(val_43 != NULL); assert(_Py_IsImmortal(val_43)); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 1, "bool(42) is not True"); - TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); - TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); - TEST_PREDICATE(!_Py_uop_sym_matches_type(sym, &PyFloat_Type), "42 matches float"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym), "42 is not a constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) != NULL, "42 as constant is NULL"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "42 as constant isn't 42"); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "42 is not immortal"); + _Py_uop_sym_set_const(ctx, ref, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 1, "bool(42) is not True"); + TEST_PREDICATE(!_Py_uop_sym_is_null(ref), "42 is NULL"); + TEST_PREDICATE(_Py_uop_sym_is_not_null(ref), "42 isn't not NULL"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "42 isn't an int"); + TEST_PREDICATE(!_Py_uop_sym_matches_type(ref, &PyFloat_Type), "42 matches float"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref), "42 is not a constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) != NULL, "42 as constant is NULL"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "42 as constant isn't 42"); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "42 is not immortal"); - _Py_uop_sym_set_type(ctx, sym, &PyLong_Type); // Should be a no-op - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "(42 and 42) isn't an int"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == val_42, "(42 and 42) as constant isn't 42"); + _Py_uop_sym_set_type(ctx, ref, &PyLong_Type); // Should be a no-op + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyLong_Type), "(42 and 42) isn't an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == val_42, "(42 and 42) as constant isn't 42"); - _Py_uop_sym_set_type(ctx, sym, &PyFloat_Type); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); + _Py_uop_sym_set_type(ctx, ref, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and float) isn't bottom"); - sym = _Py_uop_sym_new_type(ctx, &PyBool_Type); - TEST_PREDICATE(_Py_uop_sym_is_immortal(sym), "a bool is not immortal"); + ref = _Py_uop_sym_new_type(ctx, &PyBool_Type); + TEST_PREDICATE(_Py_uop_sym_is_immortal(ref), "a bool is not immortal"); - sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + ref = _Py_uop_sym_new_type(ctx, &PyLong_Type); + if (PyJitRef_IsNull(ref)) { goto fail; } - _Py_uop_sym_set_const(ctx, sym, val_42); - _Py_uop_sym_set_const(ctx, sym, val_43); // Should make it bottom - TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + _Py_uop_sym_set_const(ctx, ref, val_42); + _Py_uop_sym_set_const(ctx, ref, val_43); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref), "(42 and 43) isn't bottom"); - sym = _Py_uop_sym_new_const(ctx, Py_None); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(None) is not False"); - sym = _Py_uop_sym_new_const(ctx, Py_False); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(False) is not False"); - sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "bool(0) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(None) is not False"); + ref = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(False) is not False"); + ref = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "bool(0) is not False"); - JitOptSymbol *i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); - JitOptSymbol *i2 = _Py_uop_sym_new_const(ctx, val_43); - JitOptSymbol *array[2] = { i1, i2 }; - sym = _Py_uop_sym_new_tuple(ctx, 2, array); + JitOptRef i1 = _Py_uop_sym_new_type(ctx, &PyFloat_Type); + JitOptRef i2 = _Py_uop_sym_new_const(ctx, val_43); + JitOptRef array[2] = { i1, i2 }; + ref = _Py_uop_sym_new_tuple(ctx, 2, array); TEST_PREDICATE( - _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, sym, 0), &PyFloat_Type), + _Py_uop_sym_matches_type(_Py_uop_sym_tuple_getitem(ctx, ref, 0), &PyFloat_Type), "tuple item does not match value used to create tuple" ); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); PyObject *pair[2] = { val_42, val_43 }; - tuple = _PyTuple_FromArray(pair, 2); - sym = _Py_uop_sym_new_const(ctx, tuple); + tuple = PyTuple_FromArray(pair, 2); + ref = _Py_uop_sym_new_const(ctx, tuple); TEST_PREDICATE( - _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, sym, 1)) == val_43, + _Py_uop_sym_get_const(ctx, _Py_uop_sym_tuple_getitem(ctx, ref, 1)) == val_43, "tuple item does not match value used to create tuple" ); - JitOptSymbol *value = _Py_uop_sym_new_type(ctx, &PyBool_Type); - sym = _Py_uop_sym_new_truthiness(ctx, value, false); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == -1, "truthiness is not unknown"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == false, "truthiness is constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == NULL, "truthiness is not NULL"); + ref = _Py_uop_sym_new_type(ctx, &PyTuple_Type); + TEST_PREDICATE( + _Py_uop_sym_is_not_null(_Py_uop_sym_tuple_getitem(ctx, ref, 42)), + "Unknown tuple item is not narrowed to non-NULL" + ); + JitOptRef value = _Py_uop_sym_new_type(ctx, &PyBool_Type); + ref = _Py_uop_sym_new_truthiness(ctx, value, false); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == -1, "truthiness is not unknown"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == false, "truthiness is constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == NULL, "truthiness is not NULL"); TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == false, "value is constant"); TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == NULL, "value is not NULL"); - _Py_uop_sym_set_const(ctx, sym, Py_False); - TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyBool_Type), "truthiness is not boolean"); - TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, sym) == 0, "truthiness is not True"); - TEST_PREDICATE(_Py_uop_sym_is_const(ctx, sym) == true, "truthiness is not constant"); - TEST_PREDICATE(_Py_uop_sym_get_const(ctx, sym) == Py_False, "truthiness is not False"); + _Py_uop_sym_set_const(ctx, ref, Py_False); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref, &PyBool_Type), "truthiness is not boolean"); + TEST_PREDICATE(_Py_uop_sym_truthiness(ctx, ref) == 0, "truthiness is not True"); + TEST_PREDICATE(_Py_uop_sym_is_const(ctx, ref) == true, "truthiness is not constant"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref) == Py_False, "truthiness is not False"); TEST_PREDICATE(_Py_uop_sym_is_const(ctx, value) == true, "value is not constant"); TEST_PREDICATE(_Py_uop_sym_get_const(ctx, value) == Py_True, "value is not True"); + + + val_big = PyNumber_Lshift(_PyLong_GetOne(), PyLong_FromLong(66)); + if (val_big == NULL) { + goto fail; + } + + JitOptRef ref_42 = _Py_uop_sym_new_const(ctx, val_42); + JitOptRef ref_big = _Py_uop_sym_new_const(ctx, val_big); + JitOptRef ref_int = _Py_uop_sym_new_compact_int(ctx); + TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_42), "42 is not a compact int"); + TEST_PREDICATE(!_Py_uop_sym_is_compact_int(ref_big), "(1 << 66) is a compact int"); + TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "compact int is not a compact int"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "compact int is not an int"); + + _Py_uop_sym_set_type(ctx, ref_int, &PyLong_Type); // Should have no effect + TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "compact int is not a compact int after cast"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "compact int is not an int after cast"); + + _Py_uop_sym_set_type(ctx, ref_int, &PyFloat_Type); // Should make it bottom + TEST_PREDICATE(_Py_uop_sym_is_bottom(ref_int), "compact int cast to float isn't bottom"); + + ref_int = _Py_uop_sym_new_compact_int(ctx); + _Py_uop_sym_set_const(ctx, ref_int, val_43); + TEST_PREDICATE(_Py_uop_sym_is_compact_int(ref_int), "43 is not a compact int"); + TEST_PREDICATE(_Py_uop_sym_matches_type(ref_int, &PyLong_Type), "43 is not an int"); + TEST_PREDICATE(_Py_uop_sym_get_const(ctx, ref_int) == val_43, "43 isn't 43"); + _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); + Py_DECREF(val_big); Py_DECREF(tuple); Py_RETURN_NONE; @@ -866,6 +1142,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_uop_abstractcontext_fini(ctx); Py_XDECREF(val_42); Py_XDECREF(val_43); + Py_XDECREF(val_big); Py_DECREF(tuple); return NULL; } diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 8edf4323594..99c1ad848be 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -91,8 +91,8 @@ _PySemaphore_Destroy(_PySemaphore *sema) #endif } -static int -_PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) +int +_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout) { int res; #if defined(MS_WINDOWS) @@ -112,17 +112,27 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) } } - // NOTE: we wait on the sigint event even in non-main threads to match the - // behavior of the other platforms. Non-main threads will ignore the - // Py_PARK_INTR result. - HANDLE sigint_event = _PyOS_SigintEvent(); - HANDLE handles[2] = { sema->platform_sem, sigint_event }; - DWORD count = sigint_event != NULL ? 2 : 1; + HANDLE handles[2] = { sema->platform_sem, NULL }; + HANDLE sigint_event = NULL; + DWORD count = 1; + if (_Py_IsMainThread()) { + // gh-135099: Wait on the SIGINT event only in the main thread. Other + // threads would ignore the result anyways, and accessing + // `_PyOS_SigintEvent()` from non-main threads may race with + // interpreter shutdown, which closes the event handle. Note that + // non-main interpreters will ignore the result. + sigint_event = _PyOS_SigintEvent(); + if (sigint_event != NULL) { + handles[1] = sigint_event; + count = 2; + } + } wait = WaitForMultipleObjects(count, handles, FALSE, millis); if (wait == WAIT_OBJECT_0) { res = Py_PARK_OK; } else if (wait == WAIT_OBJECT_0 + 1) { + assert(sigint_event != NULL); ResetEvent(sigint_event); res = Py_PARK_INTR; } @@ -215,27 +225,6 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) return res; } -int -_PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout, int detach) -{ - PyThreadState *tstate = NULL; - if (detach) { - tstate = _PyThreadState_GET(); - if (tstate && _PyThreadState_IsAttached(tstate)) { - // Only detach if we are attached - PyEval_ReleaseThread(tstate); - } - else { - tstate = NULL; - } - } - int res = _PySemaphore_PlatformWait(sema, timeout); - if (tstate) { - PyEval_AcquireThread(tstate); - } - return res; -} - void _PySemaphore_Wakeup(_PySemaphore *sema) { @@ -332,7 +321,19 @@ _PyParkingLot_Park(const void *addr, const void *expected, size_t size, enqueue(bucket, addr, &wait); _PyRawMutex_Unlock(&bucket->mutex); - int res = _PySemaphore_Wait(&wait.sema, timeout_ns, detach); + PyThreadState *tstate = NULL; + if (detach) { + tstate = _PyThreadState_GET(); + if (tstate && _PyThreadState_IsAttached(tstate)) { + // Only detach if we are attached + PyEval_ReleaseThread(tstate); + } + else { + tstate = NULL; + } + } + + int res = _PySemaphore_Wait(&wait.sema, timeout_ns); if (res == Py_PARK_OK) { goto done; } @@ -344,7 +345,7 @@ _PyParkingLot_Park(const void *addr, const void *expected, size_t size, // Another thread has started to unpark us. Wait until we process the // wakeup signal. do { - res = _PySemaphore_Wait(&wait.sema, -1, detach); + res = _PySemaphore_Wait(&wait.sema, -1); } while (res != Py_PARK_OK); goto done; } @@ -356,6 +357,9 @@ _PyParkingLot_Park(const void *addr, const void *expected, size_t size, done: _PySemaphore_Destroy(&wait.sema); + if (tstate) { + PyEval_AcquireThread(tstate); + } return res; } diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 92360c1bb02..d034562c43f 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -272,7 +272,8 @@ Py_SetProgramName(const wchar_t *program_name) } -wchar_t * +/* removed in 3.15, but kept for stable ABI compatibility */ +PyAPI_FUNC(wchar_t *) Py_GetPath(void) { /* If the user has provided a path, return that */ @@ -284,7 +285,7 @@ Py_GetPath(void) } -wchar_t * +PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void) { wchar_t *stdlib_dir = _Py_path_config.stdlib_dir; @@ -295,35 +296,40 @@ _Py_GetStdlibDir(void) } -wchar_t * +/* removed in 3.15, but kept for stable ABI compatibility */ +PyAPI_FUNC(wchar_t *) Py_GetPrefix(void) { return _Py_path_config.prefix; } -wchar_t * +/* removed in 3.15, but kept for stable ABI compatibility */ +PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void) { return _Py_path_config.exec_prefix; } -wchar_t * +/* removed in 3.15, but kept for stable ABI compatibility */ +PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void) { return _Py_path_config.program_full_path; } -wchar_t* +/* removed in 3.15, but kept for stable ABI compatibility */ +PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void) { return _Py_path_config.home; } -wchar_t * +/* removed in 3.15, but kept for stable ABI compatibility */ +PyAPI_FUNC(wchar_t *) Py_GetProgramName(void) { return _Py_path_config.program_name; diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 1211e0e9f11..8732be97361 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -1,241 +1,377 @@ +/* + * Python Perf Trampoline Support - JIT Dump Implementation + * + * This file implements the perf jitdump API for Python's performance profiling + * integration. It allows perf (Linux performance analysis tool) to understand + * and profile dynamically generated Python bytecode by creating JIT dump files + * that perf can inject into its analysis. + * + * + * IMPORTANT: This file exports specific callback functions that are part of + * Python's internal API. Do not modify the function signatures or behavior + * of exported functions without coordinating with the Python core team. + * + * Usually the binary and libraries are mapped in separate region like below: + * + * address -> + * --+---------------------+--//--+---------------------+-- + * | .text | .data | ... | | .text | .data | ... | + * --+---------------------+--//--+---------------------+-- + * myprog libc.so + * + * So it'd be easy and straight-forward to find a mapped binary or library from an + * address. + * + * But for JIT code, the code arena only cares about the code section. But the + * resulting DSOs (which is generated by perf inject -j) contain ELF headers and + * unwind info too. Then it'd generate following address space with synthesized + * MMAP events. Let's say it has a sample between address B and C. + * + * sample + * | + * address -> A B v C + * --------------------------------------------------------------------------------------------------- + * /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + * /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + * /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + * ... + * --------------------------------------------------------------------------------------------------- + * + * If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see + * the unwind info. If it maps both .text section and unwind sections, the sample + * could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing + * which one is right. So to make perf happy we have non-overlapping ranges for each + * DSO: + * + * address -> + * ------------------------------------------------------------------------------------------------------- + * /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + * /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + * /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + * ... + * ------------------------------------------------------------------------------------------------------- + * + * As the trampolines are constant, we add a constant padding but in general the padding needs to have the + * size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + */ + + + #include "Python.h" #include "pycore_ceval.h" // _PyPerf_Callbacks #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_runtime.h" // _PyRuntime - #ifdef PY_HAVE_PERF_TRAMPOLINE -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <sys/mman.h> // mmap() -#include <sys/types.h> -#include <unistd.h> // sysconf() -#include <sys/time.h> // gettimeofday() -#include <sys/syscall.h> +/* Standard library includes for perf jitdump implementation */ +#if defined(__linux__) +# include <elf.h> // ELF architecture constants +#endif +#include <fcntl.h> // File control operations +#include <stdio.h> // Standard I/O operations +#include <stdlib.h> // Standard library functions +#include <sys/mman.h> // Memory mapping functions (mmap) +#include <sys/types.h> // System data types +#include <unistd.h> // System calls (sysconf, getpid) +#include <sys/time.h> // Time functions (gettimeofday) +#if defined(__linux__) +# include <sys/syscall.h> // System call interface +#endif -// ---------------------------------- -// Perf jitdump API -// ---------------------------------- - -typedef struct { - FILE* perf_map; - PyThread_type_lock map_lock; - void* mapped_buffer; - size_t mapped_size; - int code_id; -} PerfMapJitState; - -static PerfMapJitState perf_jit_map_state; +// ============================================================================= +// CONSTANTS AND CONFIGURATION +// ============================================================================= /* -Usually the binary and libraries are mapped in separate region like below: - - address -> - --+---------------------+--//--+---------------------+-- - | .text | .data | ... | | .text | .data | ... | - --+---------------------+--//--+---------------------+-- - myprog libc.so - -So it'd be easy and straight-forward to find a mapped binary or library from an -address. - -But for JIT code, the code arena only cares about the code section. But the -resulting DSOs (which is generated by perf inject -j) contain ELF headers and -unwind info too. Then it'd generate following address space with synthesized -MMAP events. Let's say it has a sample between address B and C. - - sample - | - address -> A B v C - --------------------------------------------------------------------------------------------------- - /tmp/jitted-PID-0.so | (headers) | .text | unwind info | - /tmp/jitted-PID-1.so | (headers) | .text | unwind info | - /tmp/jitted-PID-2.so | (headers) | .text | unwind info | - ... - --------------------------------------------------------------------------------------------------- - -If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see -the unwind info. If it maps both .text section and unwind sections, the sample -could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing -which one is right. So to make perf happy we have non-overlapping ranges for each -DSO: - - address -> - ------------------------------------------------------------------------------------------------------- - /tmp/jitted-PID-0.so | (headers) | .text | unwind info | - /tmp/jitted-PID-1.so | (headers) | .text | unwind info | - /tmp/jitted-PID-2.so | (headers) | .text | unwind info | - ... - ------------------------------------------------------------------------------------------------------- - -As the trampolines are constant, we add a constant padding but in general the padding needs to have the -size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + * Memory layout considerations for perf jitdump: + * + * Perf expects non-overlapping memory regions for each JIT-compiled function. + * When perf processes the jitdump file, it creates synthetic DSO (Dynamic + * Shared Object) files that contain: + * - ELF headers + * - .text section (actual machine code) + * - Unwind information (for stack traces) + * + * To ensure proper address space layout, we add padding between code regions. + * This prevents address conflicts when perf maps the synthesized DSOs. + * + * Memory layout example: + * /tmp/jitted-PID-0.so: [headers][.text][unwind_info][padding] + * /tmp/jitted-PID-1.so: [headers][.text][unwind_info][padding] + * + * The padding size is now calculated automatically during initialization + * based on the actual unwind information requirements. */ -#define PERF_JIT_CODE_PADDING 0x100 + +/* These constants are defined inside <elf.h>, which we can't use outside of linux. */ +#if !defined(__linux__) +# if defined(__i386__) || defined(_M_IX86) +# define EM_386 3 +# elif defined(__arm__) || defined(_M_ARM) +# define EM_ARM 40 +# elif defined(__x86_64__) || defined(_M_X64) +# define EM_X86_64 62 +# elif defined(__aarch64__) +# define EM_AARCH64 183 +# elif defined(__riscv) +# define EM_RISCV 243 +# endif +#endif + +/* Convenient access to the global trampoline API state */ #define trampoline_api _PyRuntime.ceval.perf.trampoline_api -typedef uint64_t uword; -typedef const char* CodeComments; +/* Type aliases for clarity and portability */ +typedef uint64_t uword; // Word-sized unsigned integer +typedef const char* CodeComments; // Code comment strings -#define Pd "d" -#define MB (1024 * 1024) +/* Memory size constants */ +#define MB (1024 * 1024) // 1 Megabyte for buffer sizing -#define EM_386 3 -#define EM_X86_64 62 -#define EM_ARM 40 -#define EM_AARCH64 183 -#define EM_RISCV 243 +// ============================================================================= +// ARCHITECTURE-SPECIFIC DEFINITIONS +// ============================================================================= -#define TARGET_ARCH_IA32 0 -#define TARGET_ARCH_X64 0 -#define TARGET_ARCH_ARM 0 -#define TARGET_ARCH_ARM64 0 -#define TARGET_ARCH_RISCV32 0 -#define TARGET_ARCH_RISCV64 0 - -#define FLAG_generate_perf_jitdump 0 -#define FLAG_write_protect_code 0 -#define FLAG_write_protect_vm_isolate 0 -#define FLAG_code_comments 0 - -#define UNREACHABLE() - -static uword GetElfMachineArchitecture(void) { -#if TARGET_ARCH_IA32 - return EM_386; -#elif TARGET_ARCH_X64 +/* + * Returns the ELF machine architecture constant for the current platform. + * This is required for the jitdump header to correctly identify the target + * architecture for perf processing. + * + */ +static uint64_t GetElfMachineArchitecture(void) { +#if defined(__x86_64__) || defined(_M_X64) return EM_X86_64; -#elif TARGET_ARCH_ARM - return EM_ARM; -#elif TARGET_ARCH_ARM64 +#elif defined(__i386__) || defined(_M_IX86) + return EM_386; +#elif defined(__aarch64__) return EM_AARCH64; -#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 +#elif defined(__arm__) || defined(_M_ARM) + return EM_ARM; +#elif defined(__riscv) return EM_RISCV; #else - UNREACHABLE(); + Py_UNREACHABLE(); // Unsupported architecture - should never reach here return 0; #endif } +// ============================================================================= +// PERF JITDUMP DATA STRUCTURES +// ============================================================================= + +/* + * Perf jitdump file format structures + * + * These structures define the binary format that perf expects for JIT dump files. + * The format is documented in the Linux perf tools source code and must match + * exactly for proper perf integration. + */ + +/* + * Jitdump file header - written once at the beginning of each jitdump file + * Contains metadata about the process and jitdump format version + */ typedef struct { - uint32_t magic; - uint32_t version; - uint32_t size; - uint32_t elf_mach_target; - uint32_t reserved; - uint32_t process_id; - uint64_t time_stamp; - uint64_t flags; + uint32_t magic; // Magic number (0x4A695444 = "JiTD") + uint32_t version; // Jitdump format version (currently 1) + uint32_t size; // Size of this header structure + uint32_t elf_mach_target; // Target architecture (from GetElfMachineArchitecture) + uint32_t reserved; // Reserved field (must be 0) + uint32_t process_id; // Process ID of the JIT compiler + uint64_t time_stamp; // Timestamp when jitdump was created + uint64_t flags; // Feature flags (currently unused) } Header; - enum PerfEvent { - PerfLoad = 0, - PerfMove = 1, - PerfDebugInfo = 2, - PerfClose = 3, - PerfUnwindingInfo = 4 +/* + * Perf event types supported by the jitdump format + * Each event type has a corresponding structure format + */ +enum PerfEvent { + PerfLoad = 0, // Code load event (new JIT function) + PerfMove = 1, // Code move event (function relocated) + PerfDebugInfo = 2, // Debug information event + PerfClose = 3, // JIT session close event + PerfUnwindingInfo = 4 // Stack unwinding information event }; +/* + * Base event structure - common header for all perf events + * Every event in the jitdump file starts with this structure + */ struct BaseEvent { - uint32_t event; - uint32_t size; - uint64_t time_stamp; - }; + uint32_t event; // Event type (from PerfEvent enum) + uint32_t size; // Total size of this event including payload + uint64_t time_stamp; // Timestamp when event occurred +}; +/* + * Code load event - indicates a new JIT-compiled function is available + * This is the most important event type for Python profiling + */ typedef struct { - struct BaseEvent base; - uint32_t process_id; - uint32_t thread_id; - uint64_t vma; - uint64_t code_address; - uint64_t code_size; - uint64_t code_id; + struct BaseEvent base; // Common event header + uint32_t process_id; // Process ID where code was generated +#if defined(__APPLE__) + uint64_t thread_id; // Thread ID where code was generated +#else + uint32_t thread_id; // Thread ID where code was generated +#endif + uint64_t vma; // Virtual memory address where code is loaded + uint64_t code_address; // Address of the actual machine code + uint64_t code_size; // Size of the machine code in bytes + uint64_t code_id; // Unique identifier for this code region + /* Followed by: + * - null-terminated function name string + * - raw machine code bytes + */ } CodeLoadEvent; +/* + * Code unwinding information event - provides DWARF data for stack traces + * Essential for proper stack unwinding during profiling + */ typedef struct { - struct BaseEvent base; - uint64_t unwind_data_size; - uint64_t eh_frame_hdr_size; - uint64_t mapped_size; + struct BaseEvent base; // Common event header + uint64_t unwind_data_size; // Size of the unwinding data + uint64_t eh_frame_hdr_size; // Size of the EH frame header + uint64_t mapped_size; // Total mapped size (with padding) + /* Followed by: + * - EH frame header + * - DWARF unwinding information + * - Padding to alignment boundary + */ } CodeUnwindingInfoEvent; +// ============================================================================= +// GLOBAL STATE MANAGEMENT +// ============================================================================= + +/* + * Global state for the perf jitdump implementation + * + * This structure maintains all the state needed for generating jitdump files. + * It's designed as a singleton since there's typically only one jitdump file + * per Python process. + */ +typedef struct { + FILE* perf_map; // File handle for the jitdump file + PyThread_type_lock map_lock; // Thread synchronization lock + void* mapped_buffer; // Memory-mapped region (signals perf we're active) + size_t mapped_size; // Size of the mapped region + int code_id; // Counter for unique code region identifiers +} PerfMapJitState; + +/* Global singleton instance */ +static PerfMapJitState perf_jit_map_state; + +// ============================================================================= +// TIME UTILITIES +// ============================================================================= + +/* Time conversion constant */ static const intptr_t nanoseconds_per_second = 1000000000; -// Dwarf encoding constants - -static const uint8_t DwarfUData4 = 0x03; -static const uint8_t DwarfSData4 = 0x0b; -static const uint8_t DwarfPcRel = 0x10; -static const uint8_t DwarfDataRel = 0x30; -// static uint8_t DwarfOmit = 0xff; -typedef struct { - unsigned char version; - unsigned char eh_frame_ptr_enc; - unsigned char fde_count_enc; - unsigned char table_enc; - int32_t eh_frame_ptr; - int32_t eh_fde_count; - int32_t from; - int32_t to; -} EhFrameHeader; - +/* + * Get current monotonic time in nanoseconds + * + * Monotonic time is preferred for event timestamps because it's not affected + * by system clock adjustments. This ensures consistent timing relationships + * between events even if the system clock is changed. + * + * Returns: Current monotonic time in nanoseconds since an arbitrary epoch + */ static int64_t get_current_monotonic_ticks(void) { struct timespec ts; if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { - UNREACHABLE(); + Py_UNREACHABLE(); // Should never fail on supported systems return 0; } - // Convert to nanoseconds. + + /* Convert to nanoseconds for maximum precision */ int64_t result = ts.tv_sec; result *= nanoseconds_per_second; result += ts.tv_nsec; return result; } +/* + * Get current wall clock time in microseconds + * + * Used for the jitdump file header timestamp. Unlike monotonic time, + * this represents actual wall clock time that can be correlated with + * other system events. + * + * Returns: Current time in microseconds since Unix epoch + */ static int64_t get_current_time_microseconds(void) { - // gettimeofday has microsecond resolution. - struct timeval tv; - if (gettimeofday(&tv, NULL) < 0) { - UNREACHABLE(); - return 0; - } - return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; + struct timeval tv; + if (gettimeofday(&tv, NULL) < 0) { + Py_UNREACHABLE(); // Should never fail on supported systems + return 0; + } + return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; } +// ============================================================================= +// UTILITY FUNCTIONS +// ============================================================================= +/* + * Round up a value to the next multiple of a given number + * + * This is essential for maintaining proper alignment requirements in the + * jitdump format. Many structures need to be aligned to specific boundaries + * (typically 8 or 16 bytes) for efficient processing by perf. + * + * Args: + * value: The value to round up + * multiple: The multiple to round up to + * + * Returns: The smallest value >= input that is a multiple of 'multiple' + */ static size_t round_up(int64_t value, int64_t multiple) { if (multiple == 0) { - // Avoid division by zero - return value; + return value; // Avoid division by zero } int64_t remainder = value % multiple; if (remainder == 0) { - // Value is already a multiple of 'multiple' - return value; + return value; // Already aligned } - // Calculate the difference to the next multiple + /* Calculate how much to add to reach the next multiple */ int64_t difference = multiple - remainder; - - // Add the difference to the value int64_t rounded_up_value = value + difference; return rounded_up_value; } +// ============================================================================= +// FILE I/O UTILITIES +// ============================================================================= +/* + * Write data to the jitdump file with error handling + * + * This function ensures that all data is written to the file, handling + * partial writes that can occur with large buffers or when the system + * is under load. + * + * Args: + * buffer: Pointer to data to write + * size: Number of bytes to write + */ static void perf_map_jit_write_fully(const void* buffer, size_t size) { FILE* out_file = perf_jit_map_state.perf_map; const char* ptr = (const char*)(buffer); + while (size > 0) { const size_t written = fwrite(ptr, 1, size, out_file); if (written == 0) { - UNREACHABLE(); + Py_UNREACHABLE(); // Write failure - should be very rare break; } size -= written; @@ -243,284 +379,826 @@ static void perf_map_jit_write_fully(const void* buffer, size_t size) { } } +/* + * Write the jitdump file header + * + * The header must be written exactly once at the beginning of each jitdump + * file. It provides metadata that perf uses to parse the rest of the file. + * + * Args: + * pid: Process ID to include in the header + * out_file: File handle to write to (currently unused, uses global state) + */ static void perf_map_jit_write_header(int pid, FILE* out_file) { Header header; - header.magic = 0x4A695444; - header.version = 1; - header.size = sizeof(Header); - header.elf_mach_target = GetElfMachineArchitecture(); - header.process_id = pid; - header.time_stamp = get_current_time_microseconds(); - header.flags = 0; + + /* Initialize header with required values */ + header.magic = 0x4A695444; // "JiTD" magic number + header.version = 1; // Current jitdump version + header.size = sizeof(Header); // Header size for validation + header.elf_mach_target = GetElfMachineArchitecture(); // Target architecture + header.process_id = pid; // Process identifier + header.time_stamp = get_current_time_microseconds(); // Creation time + header.flags = 0; // No special flags currently used + perf_map_jit_write_fully(&header, sizeof(header)); } -static void* perf_map_jit_init(void) { - char filename[100]; - int pid = getpid(); - snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); - const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); - if (fd == -1) { - return NULL; - } +// ============================================================================= +// DWARF CONSTANTS AND UTILITIES +// ============================================================================= - const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) - if (page_size == -1) { - close(fd); - return NULL; - } - - // The perf jit interface forces us to map the first page of the file - // to signal that we are using the interface. - perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); - if (perf_jit_map_state.mapped_buffer == NULL) { - close(fd); - return NULL; - } - perf_jit_map_state.mapped_size = page_size; - perf_jit_map_state.perf_map = fdopen(fd, "w+"); - if (perf_jit_map_state.perf_map == NULL) { - close(fd); - return NULL; - } - setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); - perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); - - perf_jit_map_state.map_lock = PyThread_allocate_lock(); - if (perf_jit_map_state.map_lock == NULL) { - fclose(perf_jit_map_state.perf_map); - return NULL; - } - perf_jit_map_state.code_id = 0; - - trampoline_api.code_padding = PERF_JIT_CODE_PADDING; - return &perf_jit_map_state; -} - -/* DWARF definitions. */ +/* + * DWARF (Debug With Arbitrary Record Formats) constants + * + * DWARF is a debugging data format used to provide stack unwinding information. + * These constants define the various encoding types and opcodes used in + * DWARF Call Frame Information (CFI) records. + */ +/* DWARF Call Frame Information version */ #define DWRF_CIE_VERSION 1 +/* DWARF CFA (Call Frame Address) opcodes */ enum { - DWRF_CFA_nop = 0x0, - DWRF_CFA_offset_extended = 0x5, - DWRF_CFA_def_cfa = 0xc, - DWRF_CFA_def_cfa_offset = 0xe, - DWRF_CFA_offset_extended_sf = 0x11, - DWRF_CFA_advance_loc = 0x40, - DWRF_CFA_offset = 0x80 + DWRF_CFA_nop = 0x0, // No operation + DWRF_CFA_offset_extended = 0x5, // Extended offset instruction + DWRF_CFA_def_cfa = 0xc, // Define CFA rule + DWRF_CFA_def_cfa_register = 0xd, // Define CFA register + DWRF_CFA_def_cfa_offset = 0xe, // Define CFA offset + DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset + DWRF_CFA_advance_loc = 0x40, // Advance location counter + DWRF_CFA_offset = 0x80, // Simple offset instruction + DWRF_CFA_restore = 0xc0 // Restore register }; -enum - { - DWRF_EH_PE_absptr = 0x00, - DWRF_EH_PE_omit = 0xff, +/* DWARF Exception Handling pointer encodings */ +enum { + DWRF_EH_PE_absptr = 0x00, // Absolute pointer + DWRF_EH_PE_omit = 0xff, // Omitted value - /* FDE data encoding. */ - DWRF_EH_PE_uleb128 = 0x01, - DWRF_EH_PE_udata2 = 0x02, - DWRF_EH_PE_udata4 = 0x03, - DWRF_EH_PE_udata8 = 0x04, - DWRF_EH_PE_sleb128 = 0x09, - DWRF_EH_PE_sdata2 = 0x0a, - DWRF_EH_PE_sdata4 = 0x0b, - DWRF_EH_PE_sdata8 = 0x0c, - DWRF_EH_PE_signed = 0x08, + /* Data type encodings */ + DWRF_EH_PE_uleb128 = 0x01, // Unsigned LEB128 + DWRF_EH_PE_udata2 = 0x02, // Unsigned 2-byte + DWRF_EH_PE_udata4 = 0x03, // Unsigned 4-byte + DWRF_EH_PE_udata8 = 0x04, // Unsigned 8-byte + DWRF_EH_PE_sleb128 = 0x09, // Signed LEB128 + DWRF_EH_PE_sdata2 = 0x0a, // Signed 2-byte + DWRF_EH_PE_sdata4 = 0x0b, // Signed 4-byte + DWRF_EH_PE_sdata8 = 0x0c, // Signed 8-byte + DWRF_EH_PE_signed = 0x08, // Signed flag - /* FDE flags. */ - DWRF_EH_PE_pcrel = 0x10, - DWRF_EH_PE_textrel = 0x20, - DWRF_EH_PE_datarel = 0x30, - DWRF_EH_PE_funcrel = 0x40, - DWRF_EH_PE_aligned = 0x50, - - DWRF_EH_PE_indirect = 0x80 - }; + /* Reference type encodings */ + DWRF_EH_PE_pcrel = 0x10, // PC-relative + DWRF_EH_PE_textrel = 0x20, // Text-relative + DWRF_EH_PE_datarel = 0x30, // Data-relative + DWRF_EH_PE_funcrel = 0x40, // Function-relative + DWRF_EH_PE_aligned = 0x50, // Aligned + DWRF_EH_PE_indirect = 0x80 // Indirect +}; +/* Additional DWARF constants for debug information */ enum { DWRF_TAG_compile_unit = 0x11 }; - enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; +enum { + DWRF_AT_name = 0x03, // Name attribute + DWRF_AT_stmt_list = 0x10, // Statement list + DWRF_AT_low_pc = 0x11, // Low PC address + DWRF_AT_high_pc = 0x12 // High PC address +}; +enum { + DWRF_FORM_addr = 0x01, // Address form + DWRF_FORM_data4 = 0x06, // 4-byte data + DWRF_FORM_string = 0x08 // String form +}; -enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; +/* Line number program opcodes */ +enum { + DWRF_LNS_extended_op = 0, // Extended opcode + DWRF_LNS_copy = 1, // Copy operation + DWRF_LNS_advance_pc = 2, // Advance program counter + DWRF_LNS_advance_line = 3 // Advance line number +}; -enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; - -enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; - -enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; +/* Line number extended opcodes */ +enum { + DWRF_LNE_end_sequence = 1, // End of sequence + DWRF_LNE_set_address = 2 // Set address +}; +/* + * Architecture-specific DWARF register numbers + * + * These constants define the register numbering scheme used by DWARF + * for each supported architecture. The numbers must match the ABI + * specification for proper stack unwinding. + */ enum { #ifdef __x86_64__ - /* Yes, the order is strange, but correct. */ - DWRF_REG_AX, - DWRF_REG_DX, - DWRF_REG_CX, - DWRF_REG_BX, - DWRF_REG_SI, - DWRF_REG_DI, - DWRF_REG_BP, - DWRF_REG_SP, - DWRF_REG_8, - DWRF_REG_9, - DWRF_REG_10, - DWRF_REG_11, - DWRF_REG_12, - DWRF_REG_13, - DWRF_REG_14, - DWRF_REG_15, - DWRF_REG_RA, + /* x86_64 register numbering (note: order is defined by x86_64 ABI) */ + DWRF_REG_AX, // RAX + DWRF_REG_DX, // RDX + DWRF_REG_CX, // RCX + DWRF_REG_BX, // RBX + DWRF_REG_SI, // RSI + DWRF_REG_DI, // RDI + DWRF_REG_BP, // RBP + DWRF_REG_SP, // RSP + DWRF_REG_8, // R8 + DWRF_REG_9, // R9 + DWRF_REG_10, // R10 + DWRF_REG_11, // R11 + DWRF_REG_12, // R12 + DWRF_REG_13, // R13 + DWRF_REG_14, // R14 + DWRF_REG_15, // R15 + DWRF_REG_RA, // Return address (RIP) #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_REG_SP = 31, - DWRF_REG_RA = 30, + /* AArch64 register numbering */ + DWRF_REG_FP = 29, // Frame Pointer + DWRF_REG_RA = 30, // Link register (return address) + DWRF_REG_SP = 31, // Stack pointer #else # error "Unsupported target architecture" #endif }; -typedef struct ELFObjectContext -{ - uint8_t* p; /* Pointer to next address in obj.space. */ - uint8_t* startp; /* Pointer to start address in obj.space. */ - uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ - uint32_t code_size; /* Size of machine code. */ +/* DWARF encoding constants used in EH frame headers */ +static const uint8_t DwarfUData4 = 0x03; // Unsigned 4-byte data +static const uint8_t DwarfSData4 = 0x0b; // Signed 4-byte data +static const uint8_t DwarfPcRel = 0x10; // PC-relative encoding +static const uint8_t DwarfDataRel = 0x30; // Data-relative encoding + +// ============================================================================= +// ELF OBJECT CONTEXT +// ============================================================================= + +/* + * Context for building ELF/DWARF structures + * + * This structure maintains state while constructing DWARF unwind information. + * It acts as a simple buffer manager with pointers to track current position + * and important landmarks within the buffer. + */ +typedef struct ELFObjectContext { + uint8_t* p; // Current write position in buffer + uint8_t* startp; // Start of buffer (for offset calculations) + uint8_t* eh_frame_p; // Start of EH frame data (for relative offsets) + uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) + uint32_t code_size; // Size of the code being described } ELFObjectContext; -/* Append a null-terminated string. */ -static uint32_t -elfctx_append_string(ELFObjectContext* ctx, const char* str) -{ +/* + * EH Frame Header structure for DWARF unwinding + * + * This structure provides metadata about the DWARF unwinding information + * that follows. It's required by the perf jitdump format to enable proper + * stack unwinding during profiling. + */ +typedef struct { + unsigned char version; // EH frame version (always 1) + unsigned char eh_frame_ptr_enc; // Encoding of EH frame pointer + unsigned char fde_count_enc; // Encoding of FDE count + unsigned char table_enc; // Encoding of table entries + int32_t eh_frame_ptr; // Pointer to EH frame data + int32_t eh_fde_count; // Number of FDEs (Frame Description Entries) + int32_t from; // Start address of code range + int32_t to; // End address of code range +} EhFrameHeader; + +// ============================================================================= +// DWARF GENERATION UTILITIES +// ============================================================================= + +/* + * Append a null-terminated string to the ELF context buffer + * + * Args: + * ctx: ELF object context + * str: String to append (must be null-terminated) + * + * Returns: Offset from start of buffer where string was written + */ +static uint32_t elfctx_append_string(ELFObjectContext* ctx, const char* str) { uint8_t* p = ctx->p; uint32_t ofs = (uint32_t)(p - ctx->startp); + + /* Copy string including null terminator */ do { *p++ = (uint8_t)*str; } while (*str++); + ctx->p = p; return ofs; } -/* Append a SLEB128 value. */ -static void -elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) -{ +/* + * Append a SLEB128 (Signed Little Endian Base 128) value + * + * SLEB128 is a variable-length encoding used extensively in DWARF. + * It efficiently encodes small numbers in fewer bytes. + * + * Args: + * ctx: ELF object context + * v: Signed value to encode + */ +static void elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) { uint8_t* p = ctx->p; + + /* Encode 7 bits at a time, with continuation bit in MSB */ for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { - *p++ = (uint8_t)((v & 0x7f) | 0x80); + *p++ = (uint8_t)((v & 0x7f) | 0x80); // Set continuation bit } - *p++ = (uint8_t)(v & 0x7f); + *p++ = (uint8_t)(v & 0x7f); // Final byte without continuation bit + ctx->p = p; } -/* Append a ULEB128 to buffer. */ -static void -elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) -{ +/* + * Append a ULEB128 (Unsigned Little Endian Base 128) value + * + * Similar to SLEB128 but for unsigned values. + * + * Args: + * ctx: ELF object context + * v: Unsigned value to encode + */ +static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { uint8_t* p = ctx->p; + + /* Encode 7 bits at a time, with continuation bit in MSB */ for (; v >= 0x80; v >>= 7) { - *p++ = (char)((v & 0x7f) | 0x80); + *p++ = (char)((v & 0x7f) | 0x80); // Set continuation bit } - *p++ = (char)v; + *p++ = (char)v; // Final byte without continuation bit + ctx->p = p; } -/* Shortcuts to generate DWARF structures. */ -#define DWRF_U8(x) (*p++ = (x)) -#define DWRF_I8(x) (*(int8_t*)p = (x), p++) -#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) -#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) -#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) -#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) -#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) -#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) -#define DWRF_ALIGNNOP(s) \ - while ((uintptr_t)p & ((s)-1)) { \ - *p++ = DWRF_CFA_nop; \ - } -#define DWRF_SECTION(name, stmt) \ - { \ - uint32_t* szp_##name = (uint32_t*)p; \ - p += 4; \ - stmt; \ - *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ +/* + * Macros for generating DWARF structures + * + * These macros provide a convenient way to write various data types + * to the DWARF buffer while automatically advancing the pointer. + */ +#define DWRF_U8(x) (*p++ = (x)) // Write unsigned 8-bit +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) // Write signed 8-bit +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) // Write unsigned 16-bit +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) // Write unsigned 32-bit +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) // Write address +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) // Write ULEB128 +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) // Write SLEB128 +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) // Write string + +/* Align to specified boundary with NOP instructions */ +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ } -/* Initialize .eh_frame section. */ -static void -elf_init_ehframe(ELFObjectContext* ctx) -{ +/* Write a DWARF section with automatic size calculation */ +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + } + +// ============================================================================= +// DWARF EH FRAME GENERATION +// ============================================================================= + +static void elf_init_ehframe(ELFObjectContext* ctx); + +/* + * Initialize DWARF .eh_frame section for a code region + * + * The .eh_frame section contains Call Frame Information (CFI) that describes + * how to unwind the stack at any point in the code. This is essential for + * proper profiling as it allows perf to generate accurate call graphs. + * + * The function generates two main components: + * 1. CIE (Common Information Entry) - describes calling conventions + * 2. FDE (Frame Description Entry) - describes specific function unwinding + * + * Args: + * ctx: ELF object context containing code size and buffer pointers + */ +static size_t calculate_eh_frame_size(void) { + /* Calculate the EH frame size for the trampoline function */ + extern void *_Py_trampoline_func_start; + extern void *_Py_trampoline_func_end; + + size_t code_size = (char*)&_Py_trampoline_func_end - (char*)&_Py_trampoline_func_start; + + ELFObjectContext ctx; + char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + ctx.fde_p = NULL; + + elf_init_ehframe(&ctx); + return ctx.p - ctx.startp; +} + +static void elf_init_ehframe(ELFObjectContext* ctx) { uint8_t* p = ctx->p; - uint8_t* framep = p; + uint8_t* framep = p; // Remember start of frame data - /* Emit DWARF EH CIE. */ - DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ - DWRF_U8(DWRF_CIE_VERSION); - DWRF_STR("zR"); /* Augmentation. */ - DWRF_UV(1); /* Code alignment factor. */ - DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ - DWRF_U8(DWRF_REG_RA); /* Return address register. */ - DWRF_UV(1); - DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ - DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); - DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); - DWRF_ALIGNNOP(sizeof(uintptr_t)); + /* + * DWARF Unwind Table for Trampoline Function + * + * This section defines DWARF Call Frame Information (CFI) using encoded macros + * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function + * preserves and restores registers. This is used by profiling tools (e.g., `perf`) + * and debuggers for stack unwinding in JIT-compiled code. + * + * ------------------------------------------------- + * TO REGENERATE THIS TABLE FROM GCC OBJECTS: + * ------------------------------------------------- + * + * 1. Create a trampoline source file (e.g., `trampoline.c`): + * + * #include <Python.h> + * typedef PyObject* (*py_evaluator)(void*, void*, int); + * PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) { + * return evaluator(ts, f, throwflag); + * } + * + * 2. Compile to an object file with frame pointer preservation: + * + * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c + * + * 3. Extract DWARF unwind info from the object file: + * + * readelf -w trampoline.o + * + * Example output from `.eh_frame`: + * + * 00000000 CIE + * Version: 1 + * Augmentation: "zR" + * Code alignment factor: 4 + * Data alignment factor: -8 + * Return address column: 30 + * DW_CFA_def_cfa: r31 (sp) ofs 0 + * + * 00000014 FDE cie=00000000 pc=0..14 + * DW_CFA_advance_loc: 4 + * DW_CFA_def_cfa_offset: 16 + * DW_CFA_offset: r29 at cfa-16 + * DW_CFA_offset: r30 at cfa-8 + * DW_CFA_advance_loc: 12 + * DW_CFA_restore: r30 + * DW_CFA_restore: r29 + * DW_CFA_def_cfa_offset: 0 + * + * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`. + * + * ---------------------------------- + * HOW TO TRANSLATE TO DWRF_* MACROS: + * ---------------------------------- + * + * After compiling your trampoline with: + * + * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c + * + * run: + * + * readelf -w trampoline.o + * + * to inspect the generated `.eh_frame` data. You will see two main components: + * + * 1. A CIE (Common Information Entry): shared configuration used by all FDEs. + * 2. An FDE (Frame Description Entry): function-specific unwind instructions. + * + * --------------------- + * Translating the CIE: + * --------------------- + * From `readelf -w`, you might see: + * + * 00000000 0000000000000010 00000000 CIE + * Version: 1 + * Augmentation: "zR" + * Code alignment factor: 4 + * Data alignment factor: -8 + * Return address column: 30 + * Augmentation data: 1b + * DW_CFA_def_cfa: r31 (sp) ofs 0 + * + * Map this to: + * + * DWRF_SECTION(CIE, + * DWRF_U32(0); // CIE ID (always 0 for CIEs) + * DWRF_U8(DWRF_CIE_VERSION); // Version: 1 + * DWRF_STR("zR"); // Augmentation string "zR" + * DWRF_UV(4); // Code alignment factor = 4 + * DWRF_SV(-8); // Data alignment factor = -8 + * DWRF_U8(DWRF_REG_RA); // Return address register (e.g., x30 = 30) + * DWRF_UV(1); // Augmentation data length = 1 + * DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers + * + * DWRF_U8(DWRF_CFA_def_cfa); // DW_CFA_def_cfa + * DWRF_UV(DWRF_REG_SP); // Register: SP (r31) + * DWRF_UV(0); // Offset = 0 + * + * DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer size boundary + * ) + * + * Notes: + * - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128. + * - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants. + * + * --------------------- + * Translating the FDE: + * --------------------- + * From `readelf -w`: + * + * 00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014 + * DW_CFA_advance_loc: 4 + * DW_CFA_def_cfa_offset: 16 + * DW_CFA_offset: r29 at cfa-16 + * DW_CFA_offset: r30 at cfa-8 + * DW_CFA_advance_loc: 12 + * DW_CFA_restore: r30 + * DW_CFA_restore: r29 + * DW_CFA_def_cfa_offset: 0 + * + * Map the FDE header and instructions to: + * + * DWRF_SECTION(FDE, + * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) + * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) + * DWRF_U32(ctx->code_size); // Code range covered by this FDE + * DWRF_U8(0); // Augmentation data length (none) + * + * DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 unit (1 * 4 = 4 bytes) + * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 + * DWRF_UV(16); + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer) + * DWRF_UV(2); // At offset 2 * 8 = 16 bytes + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address) + * DWRF_UV(1); // At offset 1 * 8 = 8 bytes + * + * DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance location by 3 units (3 * 4 = 12 bytes) + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30 + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29 + * + * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + * DWRF_UV(0); + * ) + * + * To regenerate: + * 1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE. + * 2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as + * the code is in a different address space every time. + * 3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro: + * - `DW_CFA_def_cfa_offset` → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value) + * - `DW_CFA_offset: rX` → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset) + * - `DW_CFA_restore: rX` → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset + * - `DW_CFA_advance_loc: N` → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor)) + * 4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers. + * 5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment. + */ + + /* + * Emit DWARF EH CIE (Common Information Entry) + * + * The CIE describes the calling conventions and basic unwinding rules + * that apply to all functions in this compilation unit. + */ + DWRF_SECTION(CIE, + DWRF_U32(0); // CIE ID (0 indicates this is a CIE) + DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) + DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) +#ifdef __x86_64__ + DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) +#endif + DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) + DWRF_U8(DWRF_REG_RA); // Return address register number + DWRF_UV(1); // Augmentation data length + DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // FDE pointer encoding + + /* Initial CFI instructions - describe default calling convention */ +#ifdef __x86_64__ + /* x86_64 initial CFI state */ + DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) + DWRF_UV(DWRF_REG_SP); // CFA = SP register + DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved + DWRF_UV(1); // At offset 1 from CFA +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + /* AArch64 initial CFI state */ + DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) + DWRF_UV(DWRF_REG_SP); // CFA = SP register + DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) + // No initial register saves in AArch64 CIE +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary ) - ctx->eh_frame_p = p; + ctx->eh_frame_p = p; // Remember start of FDE data - /* Emit DWARF EH FDE. */ - DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ - DWRF_U32(-0x30); /* Machine code offset relative to .text. */ - DWRF_U32(ctx->code_size); /* Machine code length. */ - DWRF_U8(0); /* Augmentation data. */ - /* Registers saved in CFRAME. */ + /* + * Emit DWARF EH FDE (Frame Description Entry) + * + * The FDE describes unwinding information specific to this function. + * It references the CIE and provides function-specific CFI instructions. + * + * The PC-relative offset is calculated after the entire EH frame is built + * to ensure accurate positioning relative to the synthesized DSO layout. + */ + DWRF_SECTION(FDE, + DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) + ctx->fde_p = p; // Remember where PC offset field is located for later calculation + DWRF_U32(0); // Placeholder for PC-relative offset (calculated at end of elf_init_ehframe) + DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) + DWRF_U8(0); // Augmentation data length (none) + + /* + * Architecture-specific CFI instructions + * + * These instructions describe how registers are saved and restored + * during function calls. Each architecture has different calling + * conventions and register usage patterns. + */ #ifdef __x86_64__ - DWRF_U8(DWRF_CFA_advance_loc | 4); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); - DWRF_U8(DWRF_CFA_advance_loc | 6); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); - /* Extra registers saved for JIT-compiled code. */ + /* x86_64 calling convention unwinding rules with frame pointer */ +# if defined(__CET__) && (__CET__ & 1) + DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) +# endif + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) + DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 + DWRF_UV(16); // New offset: SP + 16 + DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 + DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) + DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 + DWRF_UV(DWRF_REG_BP); // Use base pointer register + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 + DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 + DWRF_UV(DWRF_REG_SP); // Use stack pointer register + DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_U8(DWRF_CFA_advance_loc | 1); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); - DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); - DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); - DWRF_U8(DWRF_CFA_advance_loc | 3); - DWRF_U8(DWRF_CFA_offset | -(64 - 29)); - DWRF_U8(DWRF_CFA_offset | -(64 - 30)); - DWRF_U8(DWRF_CFA_def_cfa_offset); - DWRF_UV(0); + /* AArch64 calling convention unwinding rules */ + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 + DWRF_UV(16); // Stack pointer moved by 16 bytes + DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved + DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) + DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved + DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) + DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 0 (stack restored) + DWRF_UV(0); // Back to original stack position #else # error "Unsupported target architecture" #endif - DWRF_ALIGNNOP(sizeof(uintptr_t));) - ctx->p = p; + DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary + ) + + ctx->p = p; // Update context pointer to end of generated data + + /* Calculate and update the PC-relative offset in the FDE + * + * When perf processes the jitdump, it creates a synthesized DSO with this layout: + * + * Synthesized DSO Memory Layout: + * ┌─────────────────────────────────────────────────────────────┐ < code_start + * │ Code Section │ + * │ (round_up(code_size, 8) bytes) │ + * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data + * │ EH Frame Data │ + * │ ┌─────────────────────────────────────────────────────┐ │ + * │ │ CIE data │ │ + * │ └─────────────────────────────────────────────────────┘ │ + * │ ┌─────────────────────────────────────────────────────┐ │ + * │ │ FDE Header: │ │ + * │ │ - CIE offset (4 bytes) │ │ + * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start + * │ │ - address range (4 bytes) │ │ (this specific field) + * │ │ CFI Instructions... │ │ + * │ └─────────────────────────────────────────────────────┘ │ + * ├─────────────────────────────────────────────────────────────┤ < reference_point + * │ EhFrameHeader │ + * │ (navigation metadata) │ + * └─────────────────────────────────────────────────────────────┘ + * + * The PC offset field in the FDE must contain the distance from itself to code_start: + * + * distance = code_start - fde_pc_field + * + * Where: + * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame + * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) + * + * Therefore: + * distance = code_start_location - fde_pc_field_location + * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) + * = -rounded_code_size - fde_offset_in_frame + * = -(round_up(code_size, 8) + fde_offset_in_frame) + * + * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field, + * + */ + if (ctx->fde_p != NULL) { + int32_t fde_offset_in_frame = (ctx->fde_p - ctx->startp); + int32_t rounded_code_size = round_up(ctx->code_size, 8); + int32_t pc_relative_offset = -(rounded_code_size + fde_offset_in_frame); + + + // Update the PC-relative offset in the FDE + *(int32_t*)ctx->fde_p = pc_relative_offset; + } } -static void perf_map_jit_write_entry(void *state, const void *code_addr, - unsigned int code_size, PyCodeObject *co) -{ +// ============================================================================= +// JITDUMP INITIALIZATION +// ============================================================================= +/* + * Initialize the perf jitdump interface + * + * This function sets up everything needed to generate jitdump files: + * 1. Creates the jitdump file with a unique name + * 2. Maps the first page to signal perf that we're using the interface + * 3. Writes the jitdump header + * 4. Initializes synchronization primitives + * + * The memory mapping is crucial - perf detects jitdump files by scanning + * for processes that have mapped files matching the pattern /tmp/jit-*.dump + * + * Returns: Pointer to initialized state, or NULL on failure + */ +static void* perf_map_jit_init(void) { + char filename[100]; + int pid = getpid(); + + /* Create unique filename based on process ID */ + snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); + + /* Create/open the jitdump file with appropriate permissions */ + const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (fd == -1) { + return NULL; // Failed to create file + } + + /* Get system page size for memory mapping */ + const long page_size = sysconf(_SC_PAGESIZE); + if (page_size == -1) { + close(fd); + return NULL; // Failed to get page size + } + +#if defined(__APPLE__) + // On macOS, samply uses a preload to find jitdumps and this mmap can be slow. + perf_jit_map_state.mapped_buffer = NULL; +#else + /* + * Map the first page of the jitdump file + * + * This memory mapping serves as a signal to perf that this process + * is generating JIT code. Perf scans /proc/.../maps looking for mapped + * files that match the jitdump naming pattern. + * + * The mapping must be PROT_READ | PROT_EXEC to be detected by perf. + */ + perf_jit_map_state.mapped_buffer = mmap( + NULL, // Let kernel choose address + page_size, // Map one page + PROT_READ | PROT_EXEC, // Read and execute permissions (required by perf) + MAP_PRIVATE, // Private mapping + fd, // File descriptor + 0 // Offset 0 (first page) + ); + + if (perf_jit_map_state.mapped_buffer == NULL) { + close(fd); + return NULL; // Memory mapping failed + } +#endif + + perf_jit_map_state.mapped_size = page_size; + + /* Convert file descriptor to FILE* for easier I/O operations */ + perf_jit_map_state.perf_map = fdopen(fd, "w+"); + if (perf_jit_map_state.perf_map == NULL) { + close(fd); + return NULL; // Failed to create FILE* + } + + /* + * Set up file buffering for better performance + * + * We use a large buffer (2MB) because jitdump files can be written + * frequently during program execution. Buffering reduces system call + * overhead and improves overall performance. + */ + setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); + + /* Write the jitdump file header */ + perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); + + /* + * Initialize thread synchronization lock + * + * Multiple threads may attempt to write to the jitdump file + * simultaneously. This lock ensures thread-safe access to the + * global jitdump state. + */ + perf_jit_map_state.map_lock = PyThread_allocate_lock(); + if (perf_jit_map_state.map_lock == NULL) { + fclose(perf_jit_map_state.perf_map); + return NULL; // Failed to create lock + } + + /* Initialize code ID counter */ + perf_jit_map_state.code_id = 0; + + /* Calculate padding size based on actual unwind info requirements */ + size_t eh_frame_size = calculate_eh_frame_size(); + size_t unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + trampoline_api.code_padding = round_up(unwind_data_size, 16); + trampoline_api.code_alignment = 32; + + return &perf_jit_map_state; +} + +// ============================================================================= +// MAIN JITDUMP ENTRY WRITING +// ============================================================================= + +/* + * Write a complete jitdump entry for a Python function + * + * This is the main function called by Python's trampoline system whenever + * a new piece of JIT-compiled code needs to be recorded. It writes both + * the unwinding information and the code load event to the jitdump file. + * + * The function performs these steps: + * 1. Initialize jitdump system if not already done + * 2. Extract function name and filename from Python code object + * 3. Generate DWARF unwinding information + * 4. Write unwinding info event to jitdump file + * 5. Write code load event to jitdump file + * + * Args: + * state: Jitdump state (currently unused, uses global state) + * code_addr: Address where the compiled code resides + * code_size: Size of the compiled code in bytes + * co: Python code object containing metadata + * + * IMPORTANT: This function signature is part of Python's internal API + * and must not be changed without coordinating with core Python development. + */ +static void perf_map_jit_write_entry(void *state, const void *code_addr, + unsigned int code_size, PyCodeObject *co) +{ + /* Initialize jitdump system on first use */ if (perf_jit_map_state.perf_map == NULL) { void* ret = perf_map_jit_init(); if(ret == NULL){ - return; + return; // Initialization failed, silently abort } } + /* + * Extract function information from Python code object + * + * We create a human-readable function name by combining the qualified + * name (includes class/module context) with the filename. This helps + * developers identify functions in perf reports. + */ const char *entry = ""; if (co->co_qualname != NULL) { entry = PyUnicode_AsUTF8(co->co_qualname); } + const char *filename = ""; if (co->co_filename != NULL) { filename = PyUnicode_AsUTF8(co->co_filename); } - + /* + * Create formatted function name for perf display + * + * Format: "py::<function_name>:<filename>" + * The "py::" prefix helps identify Python functions in mixed-language + * profiles (e.g., when profiling C extensions alongside Python code). + */ size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); if (perf_map_entry == NULL) { - return; + return; // Memory allocation failed } snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); @@ -528,90 +1206,190 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, uword base = (uword)code_addr; uword size = code_size; - // Write the code unwinding info event. - - // Create unwinding information (eh frame) + /* + * Generate DWARF unwinding information + * + * DWARF data is essential for proper stack unwinding during profiling. + * Without it, perf cannot generate accurate call graphs, especially + * in optimized code where frame pointers may be omitted. + */ ELFObjectContext ctx; - char buffer[1024]; + char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) ctx.code_size = code_size; ctx.startp = ctx.p = (uint8_t*)buffer; + ctx.fde_p = NULL; // Initialize to NULL, will be set when FDE is written + + /* Generate EH frame (Exception Handling frame) data */ elf_init_ehframe(&ctx); int eh_frame_size = ctx.p - ctx.startp; - // Populate the unwind info event for perf + /* + * Write Code Unwinding Information Event + * + * This event must be written before the code load event to ensure + * perf has the unwinding information available when it processes + * the code region. + */ CodeUnwindingInfoEvent ev2; ev2.base.event = PerfUnwindingInfo; ev2.base.time_stamp = get_current_monotonic_ticks(); ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; - // Ensure we have enough space between DSOs when perf maps them - assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + + /* Verify we don't exceed our padding budget */ + assert(ev2.unwind_data_size <= (uint64_t)trampoline_api.code_padding); + ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); - ev2.mapped_size = round_up(ev2.unwind_data_size, 16); + ev2.mapped_size = round_up(ev2.unwind_data_size, 16); // 16-byte alignment + + /* Calculate total event size with padding */ int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size; - int padding_size = round_up(content_size, 8) - content_size; + int padding_size = round_up(content_size, 8) - content_size; // 8-byte align ev2.base.size = content_size + padding_size; + + /* Write the unwinding info event header */ perf_map_jit_write_fully(&ev2, sizeof(ev2)); - - // Populate the eh Frame header + /* + * Write EH Frame Header + * + * The EH frame header provides metadata about the DWARF unwinding + * information that follows. It includes pointers and counts that + * help perf navigate the unwinding data efficiently. + */ EhFrameHeader f; f.version = 1; - f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; - f.fde_count_enc = DwarfUData4; - f.table_enc = DwarfSData4 | DwarfDataRel; + f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; // PC-relative signed 4-byte + f.fde_count_enc = DwarfUData4; // Unsigned 4-byte count + f.table_enc = DwarfSData4 | DwarfDataRel; // Data-relative signed 4-byte + + /* Calculate relative offsets for EH frame navigation */ f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); - f.eh_fde_count = 1; + f.eh_fde_count = 1; // We generate exactly one FDE per function f.from = -(round_up(code_size, 8) + eh_frame_size); + int cie_size = ctx.eh_frame_p - ctx.startp; f.to = -(eh_frame_size - cie_size); + /* Write EH frame data and header */ perf_map_jit_write_fully(ctx.startp, eh_frame_size); perf_map_jit_write_fully(&f, sizeof(f)); + /* Write padding to maintain alignment */ char padding_bytes[] = "\0\0\0\0\0\0\0\0"; perf_map_jit_write_fully(&padding_bytes, padding_size); - // Write the code load event. + /* + * Write Code Load Event + * + * This event tells perf about the new code region. It includes: + * - Memory addresses and sizes + * - Process and thread identification + * - Function name for symbol resolution + * - The actual machine code bytes + */ CodeLoadEvent ev; ev.base.event = PerfLoad; ev.base.size = sizeof(ev) + (name_length+1) + size; ev.base.time_stamp = get_current_monotonic_ticks(); ev.process_id = getpid(); - ev.thread_id = syscall(SYS_gettid); - ev.vma = base; - ev.code_address = base; +#if defined(__APPLE__) + pthread_threadid_np(NULL, &ev.thread_id); +#else + ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call +#endif + ev.vma = base; // Virtual memory address + ev.code_address = base; // Same as VMA for our use case ev.code_size = size; + + /* Assign unique code ID and increment counter */ perf_jit_map_state.code_id += 1; ev.code_id = perf_jit_map_state.code_id; + /* Write code load event and associated data */ perf_map_jit_write_fully(&ev, sizeof(ev)); - perf_map_jit_write_fully(perf_map_entry, name_length+1); - perf_map_jit_write_fully((void*)(base), size); - return; + perf_map_jit_write_fully(perf_map_entry, name_length+1); // Include null terminator + perf_map_jit_write_fully((void*)(base), size); // Copy actual machine code + + /* Clean up allocated memory */ + PyMem_RawFree(perf_map_entry); } +// ============================================================================= +// CLEANUP AND FINALIZATION +// ============================================================================= + +/* + * Finalize and cleanup the perf jitdump system + * + * This function is called when Python is shutting down or when the + * perf trampoline system is being disabled. It ensures all resources + * are properly released and all buffered data is flushed to disk. + * + * Args: + * state: Jitdump state (currently unused, uses global state) + * + * Returns: 0 on success + * + * IMPORTANT: This function signature is part of Python's internal API + * and must not be changed without coordinating with core Python development. + */ static int perf_map_jit_fini(void* state) { + /* + * Close jitdump file with proper synchronization + * + * We need to acquire the lock to ensure no other threads are + * writing to the file when we close it. This prevents corruption + * and ensures all data is properly flushed. + */ if (perf_jit_map_state.perf_map != NULL) { - // close the file PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); - fclose(perf_jit_map_state.perf_map); + fclose(perf_jit_map_state.perf_map); // This also flushes buffers PyThread_release_lock(perf_jit_map_state.map_lock); - // clean up the lock and state + /* Clean up synchronization primitive */ PyThread_free_lock(perf_jit_map_state.map_lock); perf_jit_map_state.perf_map = NULL; } + + /* + * Unmap the memory region + * + * This removes the signal to perf that we were generating JIT code. + * After this point, perf will no longer detect this process as + * having JIT capabilities. + */ if (perf_jit_map_state.mapped_buffer != NULL) { munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); + perf_jit_map_state.mapped_buffer = NULL; } + + /* Clear global state reference */ trampoline_api.state = NULL; - return 0; + + return 0; // Success } +// ============================================================================= +// PUBLIC API EXPORT +// ============================================================================= + +/* + * Python Perf Callbacks Structure + * + * This structure defines the callback interface that Python's trampoline + * system uses to integrate with perf profiling. It contains function + * pointers for initialization, event writing, and cleanup. + * + * CRITICAL: This structure and its contents are part of Python's internal + * API. The function signatures and behavior must remain stable to maintain + * compatibility with the Python interpreter's perf integration system. + * + * Used by: Python's _PyPerf_Callbacks system in pycore_ceval.h + */ _PyPerf_Callbacks _Py_perfmap_jit_callbacks = { - &perf_map_jit_init, - &perf_map_jit_write_entry, - &perf_map_jit_fini, + &perf_map_jit_init, // Initialization function + &perf_map_jit_write_entry, // Event writing function + &perf_map_jit_fini, // Cleanup function }; -#endif +#endif /* PY_HAVE_PERF_TRAMPOLINE */ diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 996e54b82b6..987e8d2a11a 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -202,6 +202,7 @@ enum perf_trampoline_type { #define perf_map_file _PyRuntime.ceval.perf.map_file #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork #define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type +#define prev_eval_frame _PyRuntime.ceval.perf.prev_eval_frame static void perf_map_write_entry(void *state, const void *code_addr, @@ -230,6 +231,7 @@ perf_map_init_state(void) { PyUnstable_PerfMapState_Init(); trampoline_api.code_padding = 0; + trampoline_api.code_alignment = 32; perf_trampoline_type = PERF_TRAMPOLINE_TYPE_MAP; return NULL; } @@ -291,7 +293,9 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; - size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); + size_t unaligned_size = code_size + trampoline_api.code_padding; + size_t chunk_size = round_up(unaligned_size, trampoline_api.code_alignment); + assert(chunk_size % trampoline_api.code_alignment == 0); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -356,7 +360,9 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, + trampoline_api.code_alignment); + assert(total_code_size % trampoline_api.code_alignment == 0); code_arena->size_left -= total_code_size; code_arena->current_addr += total_code_size; return trampoline; @@ -402,9 +408,12 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, f = new_trampoline; } assert(f != NULL); - return f(ts, frame, throw, _PyEval_EvalFrameDefault); + return f(ts, frame, throw, prev_eval_frame != NULL ? prev_eval_frame : _PyEval_EvalFrameDefault); default_eval: // Something failed, fall back to the default evaluator. + if (prev_eval_frame) { + return prev_eval_frame(ts, frame, throw); + } return _PyEval_EvalFrameDefault(ts, frame, throw); } #endif // PY_HAVE_PERF_TRAMPOLINE @@ -476,22 +485,13 @@ _PyPerfTrampoline_Init(int activate) { #ifdef PY_HAVE_PERF_TRAMPOLINE PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->interp->eval_frame && - tstate->interp->eval_frame != py_trampoline_evaluator) { - PyErr_SetString(PyExc_RuntimeError, - "Trampoline cannot be initialized as a custom eval " - "frame is already present"); - return -1; - } if (!activate) { - _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, prev_eval_frame); perf_status = PERF_STATUS_NO_INIT; } - else { + else if (tstate->interp->eval_frame != py_trampoline_evaluator) { + prev_eval_frame = _PyInterpreterState_GetEvalFrameFunc(tstate->interp); _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); - if (new_code_arena() < 0) { - return -1; - } extra_code_index = _PyEval_RequestCodeExtraIndex(NULL); if (extra_code_index == -1) { return -1; @@ -499,6 +499,9 @@ _PyPerfTrampoline_Init(int activate) if (trampoline_api.state == NULL && trampoline_api.init_state != NULL) { trampoline_api.state = trampoline_api.init_state(); } + if (new_code_arena() < 0) { + return -1; + } perf_status = PERF_STATUS_OK; } #endif diff --git a/Python/preconfig.c b/Python/preconfig.c index 5b26c75de8b..e4cd10d9e3d 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -291,12 +291,12 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config) config->use_environment = -1; config->configure_locale = 1; - /* bpo-36443: C locale coercion (PEP 538) and UTF-8 Mode (PEP 540) - are disabled by default using the Compat configuration. + /* gh-80624: C locale coercion (PEP 538) is disabled by default using + the Compat configuration. - Py_UTF8Mode=1 enables the UTF-8 mode. PYTHONUTF8 environment variable + Py_UTF8Mode=0 disables the UTF-8 mode. PYTHONUTF8 environment variable is ignored (even if use_environment=1). */ - config->utf8_mode = 0; + config->utf8_mode = 1; config->coerce_c_locale = 0; config->coerce_c_locale_warn = 0; @@ -317,8 +317,8 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config) config->isolated = 0; config->parse_argv = 1; config->use_environment = 1; - /* Set to -1 to enable C locale coercion (PEP 538) and UTF-8 Mode (PEP 540) - depending on the LC_CTYPE locale, PYTHONUTF8 and PYTHONCOERCECLOCALE + /* Set to -1 to enable C locale coercion (PEP 538) depending on + the LC_CTYPE locale, PYTHONUTF8 and PYTHONCOERCECLOCALE environment variables. */ config->coerce_c_locale = -1; config->coerce_c_locale_warn = -1; @@ -338,7 +338,7 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config) config->configure_locale = 0; config->isolated = 1; config->use_environment = 0; - config->utf8_mode = 0; + config->utf8_mode = 1; config->dev_mode = 0; #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; @@ -649,23 +649,7 @@ preconfig_init_utf8_mode(PyPreConfig *config, const _PyPreCmdline *cmdline) return _PyStatus_OK(); } - -#ifndef MS_WINDOWS - if (config->utf8_mode < 0) { - /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ - const char *ctype_loc = setlocale(LC_CTYPE, NULL); - if (ctype_loc != NULL - && (strcmp(ctype_loc, "C") == 0 - || strcmp(ctype_loc, "POSIX") == 0)) - { - config->utf8_mode = 1; - } - } -#endif - - if (config->utf8_mode < 0) { - config->utf8_mode = 0; - } + config->utf8_mode = 1; return _PyStatus_OK(); } @@ -700,7 +684,7 @@ preconfig_init_coerce_c_locale(PyPreConfig *config) /* Test if coerce_c_locale equals to -1 or equals to 1: PYTHONCOERCECLOCALE=1 doesn't imply that the C locale is always coerced. - It is only coerced if if the LC_CTYPE locale is "C". */ + It is only coerced if the LC_CTYPE locale is "C". */ if (config->coerce_c_locale < 0 || config->coerce_c_locale == 1) { /* The C locale enables the C locale coercion (PEP 538) */ if (_Py_LegacyLocaleDetected(0)) { diff --git a/Python/pyhash.c b/Python/pyhash.c index 216f437dd9a..157312a936b 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -29,7 +29,7 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; #endif /* For numeric types, the hash of a number x is based on the reduction - of x modulo the prime P = 2**_PyHASH_BITS - 1. It's designed so that + of x modulo the prime P = 2**PyHASH_BITS - 1. It's designed so that hash(x) == hash(y) whenever x and y are numerically equal, even if x and y have different types. @@ -52,8 +52,8 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; If the result of the reduction is infinity (this is impossible for integers, floats and Decimals) then use the predefined hash value - _PyHASH_INF for x >= 0, or -_PyHASH_INF for x < 0, instead. - _PyHASH_INF and -_PyHASH_INF are also used for the + PyHASH_INF for x >= 0, or -PyHASH_INF for x < 0, instead. + PyHASH_INF and -PyHASH_INF are also used for the hashes of float and Decimal infinities. NaNs hash with a pointer hash. Having distinct hash values prevents @@ -65,16 +65,16 @@ static Py_ssize_t hashstats[Py_HASH_STATS_MAX + 1] = {0}; efficiently, even if the exponent of the binary or decimal number is large. The key point is that - reduce(x * y) == reduce(x) * reduce(y) (modulo _PyHASH_MODULUS) + reduce(x * y) == reduce(x) * reduce(y) (modulo PyHASH_MODULUS) provided that {reduce(x), reduce(y)} != {0, infinity}. The reduction of a binary or decimal float is never infinity, since the denominator is a power of 2 (for binary) or a divisor of a power of 10 (for decimal). So we have, for nonnegative x, - reduce(x * 2**e) == reduce(x) * reduce(2**e) % _PyHASH_MODULUS + reduce(x * 2**e) == reduce(x) * reduce(2**e) % PyHASH_MODULUS - reduce(x * 10**e) == reduce(x) * reduce(10**e) % _PyHASH_MODULUS + reduce(x * 10**e) == reduce(x) * reduce(10**e) % PyHASH_MODULUS and reduce(10**e) can be computed efficiently by the usual modular exponentiation algorithm. For reduce(2**e) it's even better: since @@ -92,7 +92,7 @@ _Py_HashDouble(PyObject *inst, double v) if (!isfinite(v)) { if (isinf(v)) - return v > 0 ? _PyHASH_INF : -_PyHASH_INF; + return v > 0 ? PyHASH_INF : -PyHASH_INF; else return PyObject_GenericHash(inst); } @@ -109,19 +109,19 @@ _Py_HashDouble(PyObject *inst, double v) and hexadecimal floating point. */ x = 0; while (m) { - x = ((x << 28) & _PyHASH_MODULUS) | x >> (_PyHASH_BITS - 28); + x = ((x << 28) & PyHASH_MODULUS) | x >> (PyHASH_BITS - 28); m *= 268435456.0; /* 2**28 */ e -= 28; y = (Py_uhash_t)m; /* pull out integer part */ m -= y; x += y; - if (x >= _PyHASH_MODULUS) - x -= _PyHASH_MODULUS; + if (x >= PyHASH_MODULUS) + x -= PyHASH_MODULUS; } - /* adjust for the exponent; first reduce it modulo _PyHASH_BITS */ - e = e >= 0 ? e % _PyHASH_BITS : _PyHASH_BITS-1-((-1-e) % _PyHASH_BITS); - x = ((x << e) & _PyHASH_MODULUS) | x >> (_PyHASH_BITS - e); + /* adjust for the exponent; first reduce it modulo PyHASH_BITS */ + e = e >= 0 ? e % PyHASH_BITS : PyHASH_BITS-1-((-1-e) % PyHASH_BITS); + x = ((x << e) & PyHASH_MODULUS) | x >> (PyHASH_BITS - e); x = x * sign; if (x == (Py_uhash_t)-1) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c4c1d9fd9e1..2527dca71d7 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -26,6 +26,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_stats.h" // _PyStats_InterpInit() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() #include "pycore_typeobject.h" // _PyTypes_InitTypes() @@ -209,7 +210,10 @@ _Py_LegacyLocaleDetected(int warn) * we may also want to check for that explicitly. */ const char *ctype_loc = setlocale(LC_CTYPE, NULL); - return ctype_loc != NULL && strcmp(ctype_loc, "C") == 0; + if (ctype_loc == NULL) { + return 0; + } + return (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0); #else /* Windows uses code pages instead of locales, so no locale is legacy */ return 0; @@ -503,6 +507,7 @@ pycore_init_runtime(_PyRuntimeState *runtime, _PyRuntimeState_SetFinalizing(runtime, NULL); _Py_InitVersion(); + _Py_DumpTraceback_Init(); status = _Py_HashRandomization_Init(config); if (_PyStatus_EXCEPTION(status)) { @@ -652,6 +657,14 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } +#ifdef Py_STATS + // initialize pystats. This must be done after the settings are loaded. + status = _PyStats_InterpInit(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + // initialize the interp->obmalloc state. This must be done after // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. @@ -760,6 +773,11 @@ pycore_init_types(PyInterpreterState *interp) return status; } + status = _PyDateTime_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } @@ -828,6 +846,10 @@ pycore_init_builtins(PyThreadState *tstate) } interp->callable_cache.object__getattribute__ = object__getattribute__; + if (_PyType_InitSlotDefs(interp) < 0) { + return _PyStatus_ERR("failed to init slotdefs"); + } + if (_PyBuiltins_AddExceptions(bimod) < 0) { return _PyStatus_ERR("failed to add exceptions to builtins"); } @@ -1283,7 +1305,7 @@ init_interp_main(PyThreadState *tstate) if (is_main_interp) { /* Initialize warnings. */ PyObject *warnoptions; - if (_PySys_GetOptionalAttrString("warnoptions", &warnoptions) < 0) { + if (PySys_GetOptionalAttrString("warnoptions", &warnoptions) < 0) { return _PyStatus_ERR("can't initialize warnings"); } if (warnoptions != NULL && PyList_Check(warnoptions) && @@ -1697,13 +1719,16 @@ finalize_modules(PyThreadState *tstate) // Invalidate all executors and turn off JIT: interp->jit = false; + interp->compiling = false; #ifdef _Py_TIER2 _Py_Executors_InvalidateAll(interp, 0); #endif // Stop watching __builtin__ modifications - PyDict_Unwatch(0, interp->builtins); - + if (PyDict_Unwatch(0, interp->builtins) < 0) { + // might happen if interp is cleared before watching the __builtin__ + PyErr_Clear(); + } PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done @@ -1806,7 +1831,7 @@ flush_std_files(void) PyObject *file; int status = 0; - if (_PySys_GetOptionalAttr(&_Py_ID(stdout), &file) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stdout), &file) < 0) { status = -1; } else if (file != NULL && file != Py_None && !file_is_closed(file)) { @@ -1819,7 +1844,7 @@ flush_std_files(void) } Py_XDECREF(file); - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { PyErr_Clear(); status = -1; } @@ -1899,6 +1924,7 @@ finalize_interp_clear(PyThreadState *tstate) _PyXI_Fini(tstate->interp); _PyExc_ClearExceptionGroupType(tstate->interp); _Py_clear_generic_types(tstate->interp); + _PyTypes_FiniCachedDescriptors(tstate->interp); /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); @@ -1992,6 +2018,7 @@ resolve_final_tstate(_PyRuntimeState *runtime) } else { /* Fall back to the current tstate. It's better than nothing. */ + // XXX No it's not main_tstate = tstate; } } @@ -2003,6 +2030,133 @@ resolve_final_tstate(_PyRuntimeState *runtime) return main_tstate; } +#ifdef Py_GIL_DISABLED +#define ASSERT_WORLD_STOPPED(interp) assert(interp->runtime->stoptheworld.world_stopped) +#else +#define ASSERT_WORLD_STOPPED(interp) +#endif + +static int +interp_has_threads(PyInterpreterState *interp) +{ + /* This needs to check for non-daemon threads only, otherwise we get stuck + * in an infinite loop. */ + assert(interp != NULL); + ASSERT_WORLD_STOPPED(interp); + assert(interp->threads.head != NULL); + if (interp->threads.head->next == NULL) { + // No other threads active, easy way out. + return 0; + } + + // We don't have to worry about locking this because the + // world is stopped. + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + if (tstate->_whence == _PyThreadState_WHENCE_THREADING) { + return 1; + } + } + + return 0; +} + +static int +interp_has_pending_calls(PyInterpreterState *interp) +{ + assert(interp != NULL); + ASSERT_WORLD_STOPPED(interp); + return interp->ceval.pending.npending != 0; +} + +static int +interp_has_atexit_callbacks(PyInterpreterState *interp) +{ + assert(interp != NULL); + assert(interp->atexit.callbacks != NULL); + ASSERT_WORLD_STOPPED(interp); + assert(PyList_CheckExact(interp->atexit.callbacks)); + return PyList_GET_SIZE(interp->atexit.callbacks) != 0; +} + +static int +runtime_has_subinterpreters(_PyRuntimeState *runtime) +{ + assert(runtime != NULL); + HEAD_LOCK(runtime); + PyInterpreterState *interp = runtime->interpreters.head; + HEAD_UNLOCK(runtime); + return interp->next != NULL; +} + +static void +make_pre_finalization_calls(PyThreadState *tstate, int subinterpreters) +{ + assert(tstate != NULL); + PyInterpreterState *interp = tstate->interp; + /* Each of these functions can start one another, e.g. a pending call + * could start a thread or vice versa. To ensure that we properly clean + * call everything, we run these in a loop until none of them run anything. */ + for (;;) { + assert(!interp->runtime->stoptheworld.world_stopped); + + // Wrap up existing "threading"-module-created, non-daemon threads. + wait_for_thread_shutdown(tstate); + + // Make any remaining pending calls. + _Py_FinishPendingCalls(tstate); + + /* The interpreter is still entirely intact at this point, and the + * exit funcs may be relying on that. In particular, if some thread + * or exit func is still waiting to do an import, the import machinery + * expects Py_IsInitialized() to return true. So don't say the + * runtime is uninitialized until after the exit funcs have run. + * Note that Threading.py uses an exit func to do a join on all the + * threads created thru it, so this also protects pending imports in + * the threads created via Threading. + */ + + _PyAtExit_Call(tstate->interp); + + if (subinterpreters) { + /* Clean up any lingering subinterpreters. + + Two preconditions need to be met here: + + - This has to happen before _PyRuntimeState_SetFinalizing is + called, or else threads might get prematurely blocked. + - The world must not be stopped, as finalizers can run. + */ + finalize_subinterpreters(); + } + + + /* Stop the world to prevent other threads from creating threads or + * atexit callbacks. On the default build, this is simply locked by + * the GIL. For pending calls, we acquire the dedicated mutex, because + * Py_AddPendingCall() can be called without an attached thread state. + */ + + PyMutex_Lock(&interp->ceval.pending.mutex); + // XXX Why does _PyThreadState_DeleteList() rely on all interpreters + // being stopped? + _PyEval_StopTheWorldAll(interp->runtime); + int has_subinterpreters = subinterpreters + ? runtime_has_subinterpreters(interp->runtime) + : 0; + int should_continue = (interp_has_threads(interp) + || interp_has_atexit_callbacks(interp) + || interp_has_pending_calls(interp) + || has_subinterpreters); + if (!should_continue) { + break; + } + _PyEval_StartTheWorldAll(interp->runtime); + PyMutex_Unlock(&interp->ceval.pending.mutex); + } + assert(PyMutex_IsLocked(&interp->ceval.pending.mutex)); + ASSERT_WORLD_STOPPED(interp); +} + static int _Py_Finalize(_PyRuntimeState *runtime) { @@ -2019,23 +2173,8 @@ _Py_Finalize(_PyRuntimeState *runtime) // Block some operations. tstate->interp->finalizing = 1; - // Wrap up existing "threading"-module-created, non-daemon threads. - wait_for_thread_shutdown(tstate); - - // Make any remaining pending calls. - _Py_FinishPendingCalls(tstate); - - /* The interpreter is still entirely intact at this point, and the - * exit funcs may be relying on that. In particular, if some thread - * or exit func is still waiting to do an import, the import machinery - * expects Py_IsInitialized() to return true. So don't say the - * runtime is uninitialized until after the exit funcs have run. - * Note that Threading.py uses an exit func to do a join on all the - * threads created thru it, so this also protects pending imports in - * the threads created via Threading. - */ - - _PyAtExit_Call(tstate->interp); + // This call stops the world and takes the pending calls lock. + make_pre_finalization_calls(tstate, /*subinterpreters=*/1); assert(_PyThreadState_GET() == tstate); @@ -2053,7 +2192,7 @@ _Py_Finalize(_PyRuntimeState *runtime) #endif /* Ensure that remaining threads are detached */ - _PyEval_StopTheWorldAll(runtime); + ASSERT_WORLD_STOPPED(tstate->interp); /* Remaining daemon threads will be trapped in PyThread_hang_thread when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ @@ -2074,6 +2213,7 @@ _Py_Finalize(_PyRuntimeState *runtime) _PyThreadState_SetShuttingDown(p); } _PyEval_StartTheWorldAll(runtime); + PyMutex_Unlock(&tstate->interp->ceval.pending.mutex); /* Clear frames of other threads to call objects destructors. Destructors will be called in the current Python thread. Since @@ -2124,9 +2264,6 @@ _Py_Finalize(_PyRuntimeState *runtime) _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); - /* Clean up any lingering subinterpreters. */ - finalize_subinterpreters(); - /* Print debug stats if any */ _PyEval_Fini(); @@ -2291,17 +2428,16 @@ new_interpreter(PyThreadState **tstate_p, interpreters: disable PyGILState_Check(). */ runtime->gilstate.check_enabled = 0; - PyInterpreterState *interp = PyInterpreterState_New(); - if (interp == NULL) { - *tstate_p = NULL; - return _PyStatus_OK(); - } - _PyInterpreterState_SetWhence(interp, whence); - interp->_ready = 1; - // XXX Might new_interpreter() have been called without the GIL held? PyThreadState *save_tstate = _PyThreadState_GET(); PyThreadState *tstate = NULL; + PyInterpreterState *interp; + status = _PyInterpreterState_New(save_tstate, &interp); + if (interp == NULL) { + goto error; + } + _PyInterpreterState_SetWhence(interp, whence); + interp->_ready = 1; /* From this point until the init_interp_create_gil() call, we must not do anything that requires that the GIL be held @@ -2342,6 +2478,14 @@ new_interpreter(PyThreadState **tstate_p, return status; } +#ifdef Py_STATS + // initialize pystats. This must be done after the settings are loaded. + status = _PyStats_InterpInit(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + // initialize the interp->obmalloc state. This must be done after // the settings are loaded (so that feature_flags are set) but before // any calls are made to obmalloc functions. @@ -2377,15 +2521,13 @@ new_interpreter(PyThreadState **tstate_p, error: *tstate_p = NULL; if (tstate != NULL) { - PyThreadState_Clear(tstate); - _PyThreadState_Detach(tstate); - PyThreadState_Delete(tstate); + Py_EndInterpreter(tstate); + } else if (interp != NULL) { + PyInterpreterState_Delete(interp); } if (save_tstate != NULL) { _PyThreadState_Attach(save_tstate); } - PyInterpreterState_Delete(interp); - return status; } @@ -2410,9 +2552,8 @@ Py_NewInterpreter(void) return tstate; } -/* Delete an interpreter and its last thread. This requires that the - given thread state is current, that the thread has no remaining - frames, and that it is its interpreter's only remaining thread. +/* Delete an interpreter. This requires that the given thread state + is current, and that the thread has no remaining frames. It is a fatal error to violate these constraints. (Py_FinalizeEx() doesn't have these constraints -- it zaps @@ -2430,27 +2571,28 @@ Py_EndInterpreter(PyThreadState *tstate) if (tstate != _PyThreadState_GET()) { Py_FatalError("thread is not current"); } - if (tstate->current_frame != NULL) { + if (tstate->current_frame != tstate->base_frame) { Py_FatalError("thread still has a frame"); } interp->finalizing = 1; - // Wrap up existing "threading"-module-created, non-daemon threads. - wait_for_thread_shutdown(tstate); - - // Make any remaining pending calls. - _Py_FinishPendingCalls(tstate); - - _PyAtExit_Call(tstate->interp); - - if (tstate != interp->threads.head || tstate->next != NULL) { - Py_FatalError("not the last thread"); - } + // This call stops the world and takes the pending calls lock. + make_pre_finalization_calls(tstate, /*subinterpreters=*/0); + ASSERT_WORLD_STOPPED(interp); /* Remaining daemon threads will automatically exit when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ _PyInterpreterState_SetFinalizing(interp, tstate); + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + for (PyThreadState *p = list; p != NULL; p = p->next) { + _PyThreadState_SetShuttingDown(p); + } + + _PyEval_StartTheWorldAll(interp->runtime); + PyMutex_Unlock(&interp->ceval.pending.mutex); + _PyThreadState_DeleteList(list, /*is_after_fork=*/0); + // XXX Call something like _PyImport_Disable() here? _PyImport_FiniExternal(tstate->interp); @@ -2480,6 +2622,8 @@ finalize_subinterpreters(void) PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(final_tstate->interp == main_interp); _PyRuntimeState *runtime = main_interp->runtime; + assert(!runtime->stoptheworld.world_stopped); + assert(_PyRuntimeState_GetFinalizing(runtime) == NULL); struct pyinterpreters *interpreters = &runtime->interpreters; /* Get the first interpreter in the list. */ @@ -2499,7 +2643,7 @@ finalize_subinterpreters(void) (void)PyErr_WarnEx( PyExc_RuntimeWarning, "remaining subinterpreters; " - "destroy them with _interpreters.destroy()", + "close them with Interpreter.close()", 0); /* Swap out the current tstate, which we know must belong @@ -2508,27 +2652,17 @@ finalize_subinterpreters(void) /* Clean up all remaining subinterpreters. */ while (interp != NULL) { - assert(!_PyInterpreterState_IsRunningMain(interp)); - - /* Find the tstate to use for fini. We assume the interpreter - will have at most one tstate at this point. */ - PyThreadState *tstate = interp->threads.head; - if (tstate != NULL) { - /* Ideally we would be able to use tstate as-is, and rely - on it being in a ready state: no exception set, not - running anything (tstate->current_frame), matching the - current thread ID (tstate->thread_id). To play it safe, - we always delete it and use a fresh tstate instead. */ - assert(tstate != final_tstate); - _PyThreadState_Attach(tstate); - PyThreadState_Clear(tstate); - _PyThreadState_Detach(tstate); - PyThreadState_Delete(tstate); + /* Make a tstate for finalization. */ + PyThreadState *tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); + if (tstate == NULL) { + // XXX Some graceful way to always get a thread state? + Py_FatalError("thread state allocation failed"); } - tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); + + /* Enter the subinterpreter. */ + _PyThreadState_Attach(tstate); /* Destroy the subinterpreter. */ - _PyThreadState_Attach(tstate); Py_EndInterpreter(tstate); assert(_PyThreadState_GET() == NULL); @@ -3046,7 +3180,7 @@ _Py_FatalError_PrintExc(PyThreadState *tstate) } PyObject *ferr; - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &ferr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &ferr) < 0) { _PyErr_Clear(tstate); } if (ferr == NULL || ferr == Py_None) { @@ -3144,7 +3278,7 @@ static inline void _Py_NO_RETURN fatal_error_exit(int status) { if (status < 0) { -#if defined(MS_WINDOWS) && defined(_DEBUG) +#if defined(MS_WINDOWS) && defined(Py_DEBUG) DebugBreak(); #endif abort(); @@ -3438,6 +3572,27 @@ Py_ExitStatusException(PyStatus status) } +static void +handle_thread_shutdown_exception(PyThreadState *tstate) +{ + assert(tstate != NULL); + assert(_PyErr_Occurred(tstate)); + PyInterpreterState *interp = tstate->interp; + assert(interp->threads.head != NULL); + _PyEval_StopTheWorld(interp); + + // We don't have to worry about locking this because the + // world is stopped. + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + if (tstate->_whence == _PyThreadState_WHENCE_THREADING) { + tstate->_whence = _PyThreadState_WHENCE_THREADING_DAEMON; + } + } + + _PyEval_StartTheWorld(interp); + PyErr_FormatUnraisable("Exception ignored on threading shutdown"); +} + /* Wait until threading._shutdown completes, provided the threading module was imported in the first place. The shutdown routine will wait until all non-daemon @@ -3449,14 +3604,14 @@ wait_for_thread_shutdown(PyThreadState *tstate) PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); if (threading == NULL) { if (_PyErr_Occurred(tstate)) { - PyErr_FormatUnraisable("Exception ignored on threading shutdown"); + handle_thread_shutdown_exception(tstate); } /* else: threading not imported */ return; } result = PyObject_CallMethodNoArgs(threading, &_Py_ID(_shutdown)); if (result == NULL) { - PyErr_FormatUnraisable("Exception ignored on threading shutdown"); + handle_thread_shutdown_exception(tstate); } else { Py_DECREF(result); @@ -3589,7 +3744,7 @@ PyOS_getsig(int sig) /* * All of the code in this function must only use async-signal-safe functions, - * listed at `man 7 signal` or + * listed at `man 7 signal-safety` or * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. */ PyOS_sighandler_t diff --git a/Python/pystate.c b/Python/pystate.c index 1ac13440085..2956e785405 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -8,7 +8,6 @@ #include "pycore_codecs.h" // _PyCodec_Fini() #include "pycore_critical_section.h" // _PyCriticalSection_Resume() #include "pycore_dtoa.h" // _dtoa_state_INIT() -#include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init() #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interpframe.h" // _PyThreadState_HasStackSpace() @@ -22,7 +21,9 @@ #include "pycore_runtime.h" // _PyRuntime #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_stackref.h" // Py_STACKREF_DEBUG +#include "pycore_stats.h" // FT_STAT_WORLD_STOP_INC() #include "pycore_time.h" // _PyTime_Init() +#include "pycore_uop.h" // UOP_BUFFER_SIZE #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() @@ -68,42 +69,37 @@ to avoid the expense of doing their own locking). */ -#ifdef HAVE_THREAD_LOCAL +/* The attached thread state for the current thread. */ _Py_thread_local PyThreadState *_Py_tss_tstate = NULL; -#endif + +/* The "bound" thread state used by PyGILState_Ensure(), + also known as a "gilstate." */ +_Py_thread_local PyThreadState *_Py_tss_gilstate = NULL; + +/* The interpreter of the attached thread state, + and is same as tstate->interp. */ +_Py_thread_local PyInterpreterState *_Py_tss_interp = NULL; static inline PyThreadState * current_fast_get(void) { -#ifdef HAVE_THREAD_LOCAL return _Py_tss_tstate; -#else - // XXX Fall back to the PyThread_tss_*() API. -# error "no supported thread-local variable storage classifier" -#endif } static inline void current_fast_set(_PyRuntimeState *Py_UNUSED(runtime), PyThreadState *tstate) { assert(tstate != NULL); -#ifdef HAVE_THREAD_LOCAL _Py_tss_tstate = tstate; -#else - // XXX Fall back to the PyThread_tss_*() API. -# error "no supported thread-local variable storage classifier" -#endif + assert(tstate->interp != NULL); + _Py_tss_interp = tstate->interp; } static inline void current_fast_clear(_PyRuntimeState *Py_UNUSED(runtime)) { -#ifdef HAVE_THREAD_LOCAL _Py_tss_tstate = NULL; -#else - // XXX Fall back to the PyThread_tss_*() API. -# error "no supported thread-local variable storage classifier" -#endif + _Py_tss_interp = NULL; } #define tstate_verify_not_active(tstate) \ @@ -118,79 +114,9 @@ _PyThreadState_GetCurrent(void) } -//------------------------------------------------ -// the thread state bound to the current OS thread -//------------------------------------------------ - -static inline int -tstate_tss_initialized(Py_tss_t *key) -{ - return PyThread_tss_is_created(key); -} - -static inline int -tstate_tss_init(Py_tss_t *key) -{ - assert(!tstate_tss_initialized(key)); - return PyThread_tss_create(key); -} - -static inline void -tstate_tss_fini(Py_tss_t *key) -{ - assert(tstate_tss_initialized(key)); - PyThread_tss_delete(key); -} - -static inline PyThreadState * -tstate_tss_get(Py_tss_t *key) -{ - assert(tstate_tss_initialized(key)); - return (PyThreadState *)PyThread_tss_get(key); -} - -static inline int -tstate_tss_set(Py_tss_t *key, PyThreadState *tstate) -{ - assert(tstate != NULL); - assert(tstate_tss_initialized(key)); - return PyThread_tss_set(key, (void *)tstate); -} - -static inline int -tstate_tss_clear(Py_tss_t *key) -{ - assert(tstate_tss_initialized(key)); - return PyThread_tss_set(key, (void *)NULL); -} - -#ifdef HAVE_FORK -/* Reset the TSS key - called by PyOS_AfterFork_Child(). - * This should not be necessary, but some - buggy - pthread implementations - * don't reset TSS upon fork(), see issue #10517. - */ -static PyStatus -tstate_tss_reinit(Py_tss_t *key) -{ - if (!tstate_tss_initialized(key)) { - return _PyStatus_OK(); - } - PyThreadState *tstate = tstate_tss_get(key); - - tstate_tss_fini(key); - if (tstate_tss_init(key) != 0) { - return _PyStatus_NO_MEMORY(); - } - - /* If the thread had an associated auto thread state, reassociate it with - * the new key. */ - if (tstate && tstate_tss_set(key, tstate) != 0) { - return _PyStatus_ERR("failed to re-set autoTSSkey"); - } - return _PyStatus_OK(); -} -#endif - +//--------------------------------------------- +// The thread state used by PyGILState_Ensure() +//--------------------------------------------- /* The stored thread state is set by bind_tstate() (AKA PyThreadState_Bind(). @@ -198,36 +124,23 @@ tstate_tss_reinit(Py_tss_t *key) The GIL does no need to be held for these. */ -#define gilstate_tss_initialized(runtime) \ - tstate_tss_initialized(&(runtime)->autoTSSkey) -#define gilstate_tss_init(runtime) \ - tstate_tss_init(&(runtime)->autoTSSkey) -#define gilstate_tss_fini(runtime) \ - tstate_tss_fini(&(runtime)->autoTSSkey) -#define gilstate_tss_get(runtime) \ - tstate_tss_get(&(runtime)->autoTSSkey) -#define _gilstate_tss_set(runtime, tstate) \ - tstate_tss_set(&(runtime)->autoTSSkey, tstate) -#define _gilstate_tss_clear(runtime) \ - tstate_tss_clear(&(runtime)->autoTSSkey) -#define gilstate_tss_reinit(runtime) \ - tstate_tss_reinit(&(runtime)->autoTSSkey) - -static inline void -gilstate_tss_set(_PyRuntimeState *runtime, PyThreadState *tstate) +static inline PyThreadState * +gilstate_get(void) { - assert(tstate != NULL && tstate->interp->runtime == runtime); - if (_gilstate_tss_set(runtime, tstate) != 0) { - Py_FatalError("failed to set current tstate (TSS)"); - } + return _Py_tss_gilstate; } static inline void -gilstate_tss_clear(_PyRuntimeState *runtime) +gilstate_set(PyThreadState *tstate) { - if (_gilstate_tss_clear(runtime) != 0) { - Py_FatalError("failed to clear current tstate (TSS)"); - } + assert(tstate != NULL); + _Py_tss_gilstate = tstate; +} + +static inline void +gilstate_clear(void) +{ + _Py_tss_gilstate = NULL; } @@ -253,7 +166,7 @@ bind_tstate(PyThreadState *tstate) assert(tstate_is_alive(tstate) && !tstate->_status.bound); assert(!tstate->_status.unbound); // just in case assert(!tstate->_status.bound_gilstate); - assert(tstate != gilstate_tss_get(tstate->interp->runtime)); + assert(tstate != gilstate_get()); assert(!tstate->_status.active); assert(tstate->thread_id == 0); assert(tstate->native_thread_id == 0); @@ -328,14 +241,13 @@ bind_gilstate_tstate(PyThreadState *tstate) // XXX assert(!tstate->_status.active); assert(!tstate->_status.bound_gilstate); - _PyRuntimeState *runtime = tstate->interp->runtime; - PyThreadState *tcur = gilstate_tss_get(runtime); + PyThreadState *tcur = gilstate_get(); assert(tstate != tcur); if (tcur != NULL) { tcur->_status.bound_gilstate = 0; } - gilstate_tss_set(runtime, tstate); + gilstate_set(tstate); tstate->_status.bound_gilstate = 1; } @@ -347,9 +259,8 @@ unbind_gilstate_tstate(PyThreadState *tstate) assert(tstate_is_bound(tstate)); // XXX assert(!tstate->_status.active); assert(tstate->_status.bound_gilstate); - assert(tstate == gilstate_tss_get(tstate->interp->runtime)); - - gilstate_tss_clear(tstate->interp->runtime); + assert(tstate == gilstate_get()); + gilstate_clear(); tstate->_status.bound_gilstate = 0; } @@ -373,7 +284,7 @@ holds_gil(PyThreadState *tstate) // (and tstate->interp->runtime->ceval.gil.locked). assert(tstate != NULL); /* Must be the tstate for this thread */ - assert(tstate == gilstate_tss_get(tstate->interp->runtime)); + assert(tstate == gilstate_get()); return tstate == current_fast_get(); } @@ -404,7 +315,6 @@ _Py_COMP_DIAG_POP &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ &(runtime)->ceval.pending_mainthread.mutex, \ - &(runtime)->ceval.sys_trace_profile_mutex, \ &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ @@ -434,11 +344,6 @@ init_runtime(_PyRuntimeState *runtime, runtime->main_thread = PyThread_get_thread_ident(); runtime->unicode_state.ids.next_index = unicode_next_index; - -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) - _Py_EmscriptenTrampoline_Init(runtime); -#endif - runtime->_initialized = 1; } @@ -469,16 +374,6 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) return status; } - if (gilstate_tss_init(runtime) != 0) { - _PyRuntimeState_Fini(runtime); - return _PyStatus_NO_MEMORY(); - } - - if (PyThread_tss_create(&runtime->trashTSSkey) != 0) { - _PyRuntimeState_Fini(runtime); - return _PyStatus_NO_MEMORY(); - } - init_runtime(runtime, open_code_hook, open_code_userdata, audit_hook_head, unicode_next_index); @@ -492,14 +387,7 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) /* The count is cleared by _Py_FinalizeRefTotal(). */ assert(runtime->object_state.interpreter_leaks == 0); #endif - - if (gilstate_tss_initialized(runtime)) { - gilstate_tss_fini(runtime); - } - - if (PyThread_tss_is_created(&runtime->trashTSSkey)) { - PyThread_tss_delete(&runtime->trashTSSkey); - } + gilstate_clear(); } #ifdef HAVE_FORK @@ -532,18 +420,6 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) _PyTypes_AfterFork(); - PyStatus status = gilstate_tss_reinit(runtime); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - if (PyThread_tss_is_created(&runtime->trashTSSkey)) { - PyThread_tss_delete(&runtime->trashTSSkey); - } - if (PyThread_tss_create(&runtime->trashTSSkey) != 0) { - return _PyStatus_NO_MEMORY(); - } - _PyThread_AfterFork(&runtime->threads); return _PyStatus_OK(); @@ -572,21 +448,30 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) static PyInterpreterState * alloc_interpreter(void) { + // Aligned allocation for PyInterpreterState. + // the first word of the memory block is used to store + // the original pointer to be used later to free the memory. size_t alignment = _Alignof(PyInterpreterState); - size_t allocsize = sizeof(PyInterpreterState) + alignment - 1; + size_t allocsize = sizeof(PyInterpreterState) + sizeof(void *) + alignment - 1; void *mem = PyMem_RawCalloc(1, allocsize); if (mem == NULL) { return NULL; } - PyInterpreterState *interp = _Py_ALIGN_UP(mem, alignment); - assert(_Py_IS_ALIGNED(interp, alignment)); - interp->_malloced = mem; - return interp; + void *ptr = _Py_ALIGN_UP((char *)mem + sizeof(void *), alignment); + ((void **)ptr)[-1] = mem; + assert(_Py_IS_ALIGNED(ptr, alignment)); + return ptr; } static void free_interpreter(PyInterpreterState *interp) { +#ifdef Py_STATS + if (interp->pystats_struct) { + PyMem_RawFree(interp->pystats_struct); + interp->pystats_struct = NULL; + } +#endif // The main interpreter is statically allocated so // should not be freed. if (interp != &_PyRuntime._main_interpreter) { @@ -596,7 +481,7 @@ free_interpreter(PyInterpreterState *interp) interp->obmalloc = NULL; } assert(_Py_IS_ALIGNED(interp, _Alignof(PyInterpreterState))); - PyMem_RawFree(interp->_malloced); + PyMem_RawFree(((void **)interp)[-1]); } } @@ -604,6 +489,11 @@ free_interpreter(PyInterpreterState *interp) static inline int check_interpreter_whence(long); #endif +extern _Py_CODEUNIT * +_Py_LazyJitTrampoline( + struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate +); + /* Get the interpreter state to a minimal consistent state. Further init happens in pylifecycle.c before it can be used. All fields not initialized here are expected to be zeroed out, @@ -661,6 +551,7 @@ init_interpreter(PyInterpreterState *interp, #ifdef Py_GIL_DISABLED _Py_brc_init_state(interp); #endif + llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); interp->asyncio_tasks_lock = (PyMutex){0}; @@ -674,13 +565,13 @@ init_interpreter(PyInterpreterState *interp, } interp->monitoring_tool_versions[t] = 0; } - interp->sys_profile_initialized = false; - interp->sys_trace_initialized = false; + interp->_code_object_generation = 0; interp->jit = false; + interp->compiling = false; interp->executor_list_head = NULL; interp->executor_deletion_list_head = NULL; interp->executor_deletion_list_remaining_capacity = 0; - interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + interp->executor_creation_counter = JIT_CLEANUP_THRESHOLD; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); @@ -868,10 +759,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - // At this time, all the threads should be cleared so we don't need atomic - // operations for instrumentation_version or eval_breaker. + // gh-140257: Threads have already been cleared, but daemon threads may + // still access eval_breaker atomically via take_gil() right before they + // hang. Use an atomic store to prevent data races during finalization. interp->ceval.instrumentation_version = 0; - tstate->eval_breaker = 0; + _Py_atomic_store_uintptr(&tstate->eval_breaker, 0); for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; @@ -881,11 +773,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->monitoring_callables[t][e]); } } - interp->sys_profile_initialized = false; - interp->sys_trace_initialized = false; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->monitoring_tool_names[t]); } + interp->_code_object_generation = 0; +#ifdef Py_GIL_DISABLED + interp->tlbc_indices.tlbc_generation = 0; +#endif PyConfig_Clear(&interp->config); _PyCodec_Fini(interp); @@ -909,7 +803,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) _Py_ClearExecutorDeletionList(interp); #endif _PyAST_Fini(interp); - _PyWarnings_Fini(interp); _PyAtExit_Fini(interp); // All Python types must be destroyed before the last GC collection. Python @@ -920,6 +813,24 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) _PyGC_CollectNoFail(tstate); _PyGC_Fini(interp); + // Finalize warnings after last gc so that any finalizers can + // access warnings state + _PyWarnings_Fini(interp); + struct _PyExecutorObject *cold = interp->cold_executor; + if (cold != NULL) { + interp->cold_executor = NULL; + assert(cold->vm_data.valid); + assert(cold->vm_data.warm); + _PyExecutor_Free(cold); + } + + struct _PyExecutorObject *cold_dynamic = interp->cold_dynamic_executor; + if (cold_dynamic != NULL) { + interp->cold_dynamic_executor = NULL; + assert(cold_dynamic->vm_data.valid); + assert(cold_dynamic->vm_data.warm); + _PyExecutor_Free(cold_dynamic); + } /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code which requires sysdict and builtins. */ @@ -1049,6 +960,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) _PyObject_FiniState(interp); + PyConfig_Clear(&interp->config); + free_interpreter(interp); } @@ -1246,6 +1159,7 @@ _Py_CheckMainModule(PyObject *module) PyObject *msg = PyUnicode_FromString("invalid __main__ module"); if (msg != NULL) { (void)PyErr_SetImportError(msg, &_Py_ID(__main__), NULL); + Py_DECREF(msg); } return -1; } @@ -1378,9 +1292,8 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) PyInterpreterState* PyInterpreterState_Get(void) { - PyThreadState *tstate = current_fast_get(); - _Py_EnsureTstateNotNULL(tstate); - PyInterpreterState *interp = tstate->interp; + _Py_AssertHoldsTstate(); + PyInterpreterState *interp = _Py_tss_interp; if (interp == NULL) { Py_FatalError("no current interpreter"); } @@ -1393,10 +1306,8 @@ interp_look_up_id(_PyRuntimeState *runtime, int64_t requested_id) { PyInterpreterState *interp = runtime->interpreters.head; while (interp != NULL) { - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } + int64_t id = interp->id; + assert(id >= 0); if (requested_id == id) { return interp; } @@ -1457,9 +1368,6 @@ tstate_is_alive(PyThreadState *tstate) // lifecycle //---------- -/* Minimum size of data stack chunk */ -#define DATA_STACK_CHUNK_SIZE (16*1024) - static _PyStackChunk* allocate_chunk(int size_in_bytes, _PyStackChunk* previous) { @@ -1506,6 +1414,9 @@ static void free_threadstate(_PyThreadStateImpl *tstate) { PyInterpreterState *interp = tstate->base.interp; +#ifdef Py_STATS + _PyStats_ThreadFini(tstate); +#endif // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. if (tstate == &interp->_initial_thread) { @@ -1555,7 +1466,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, assert(tstate->prev == NULL); assert(tstate->_whence == _PyThreadState_WHENCE_NOTSET); - assert(whence >= 0 && whence <= _PyThreadState_WHENCE_EXEC); + assert(whence >= 0 && whence <= _PyThreadState_WHENCE_THREADING_DAEMON); tstate->_whence = whence; assert(id > 0); @@ -1571,21 +1482,52 @@ init_threadstate(_PyThreadStateImpl *_tstate, // This is cleared when PyGILState_Ensure() creates the thread state. tstate->gilstate_counter = 1; - tstate->current_frame = NULL; + // Initialize the embedded base frame - sentinel at the bottom of the frame stack + _tstate->base_frame.previous = NULL; + _tstate->base_frame.f_executable = PyStackRef_None; + _tstate->base_frame.f_funcobj = PyStackRef_NULL; + _tstate->base_frame.f_globals = NULL; + _tstate->base_frame.f_builtins = NULL; + _tstate->base_frame.f_locals = NULL; + _tstate->base_frame.frame_obj = NULL; + _tstate->base_frame.instr_ptr = NULL; + _tstate->base_frame.stackpointer = _tstate->base_frame.localsplus; + _tstate->base_frame.return_offset = 0; + _tstate->base_frame.owner = FRAME_OWNED_BY_INTERPRETER; + _tstate->base_frame.visited = 0; +#ifdef Py_DEBUG + _tstate->base_frame.lltrace = 0; +#endif +#ifdef Py_GIL_DISABLED + _tstate->base_frame.tlbc_index = 0; +#endif + _tstate->base_frame.localsplus[0] = PyStackRef_NULL; + + // current_frame starts pointing to the base frame + tstate->current_frame = &_tstate->base_frame; + // base_frame pointer for profilers to validate stack unwinding + tstate->base_frame = &_tstate->base_frame; tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; tstate->what_event = -1; tstate->current_executor = NULL; + tstate->jit_exit = NULL; tstate->dict_global_version = 0; _tstate->c_stack_soft_limit = UINTPTR_MAX; _tstate->c_stack_top = 0; _tstate->c_stack_hard_limit = 0; + _tstate->c_stack_init_base = 0; + _tstate->c_stack_init_top = 0; + _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; +#ifdef _Py_TIER2 + _tstate->jit_tracer_state.code_buffer = NULL; +#endif tstate->delete_later = NULL; llist_init(&_tstate->mem_free_queue); @@ -1633,6 +1575,13 @@ new_threadstate(PyInterpreterState *interp, int whence) return NULL; } #endif +#ifdef Py_STATS + // The PyStats structure is quite large and is allocated separated from tstate. + if (!_PyStats_ThreadInit(interp, tstate)) { + free_threadstate(tstate); + return NULL; + } +#endif /* We serialize concurrent creation to protect global state. */ HEAD_LOCK(interp->runtime); @@ -1671,7 +1620,7 @@ _PyThreadState_NewBound(PyInterpreterState *interp, int whence) bind_tstate(tstate); // This makes sure there's a gilstate tstate bound // as soon as possible. - if (gilstate_tss_get(tstate->interp->runtime) == NULL) { + if (gilstate_get() == NULL) { bind_gilstate_tstate(tstate); } } @@ -1718,7 +1667,11 @@ PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); assert(current_fast_get()->interp == tstate->interp); - assert(!_PyThreadState_IsRunningMain(tstate)); + // GH-126016: In the _interpreters module, KeyboardInterrupt exceptions + // during PyEval_EvalCode() are sent to finalization, which doesn't let us + // mark threads as "not running main". So, for now this assertion is + // disabled. + // XXX assert(!_PyThreadState_IsRunningMain(tstate)); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1731,7 +1684,7 @@ PyThreadState_Clear(PyThreadState *tstate) int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; - if (verbose && tstate->current_frame != NULL) { + if (verbose && tstate->current_frame != tstate->base_frame) { /* bpo-20526: After the main thread calls _PyInterpreterState_SetFinalizing() in Py_FinalizeEx() (or in Py_EndInterpreter() for subinterpreters), @@ -1791,13 +1744,14 @@ PyThreadState_Clear(PyThreadState *tstate) } if (tstate->c_profilefunc != NULL) { - tstate->interp->sys_profiling_threads--; + FT_ATOMIC_ADD_SSIZE(tstate->interp->sys_profiling_threads, -1); tstate->c_profilefunc = NULL; } if (tstate->c_tracefunc != NULL) { - tstate->interp->sys_tracing_threads--; + FT_ATOMIC_ADD_SSIZE(tstate->interp->sys_tracing_threads, -1); tstate->c_tracefunc = NULL; } + Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); @@ -1880,6 +1834,14 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) assert(tstate_impl->refcounts.values == NULL); #endif +#if _Py_TIER2 + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (_tstate->jit_tracer_state.code_buffer != NULL) { + _PyObject_VirtualFree(_tstate->jit_tracer_state.code_buffer, UOP_BUFFER_SIZE); + _tstate->jit_tracer_state.code_buffer = NULL; + } +#endif + HEAD_UNLOCK(runtime); // XXX Unbind in PyThreadState_Clear(), or earlier @@ -1908,9 +1870,14 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) static void zapthreads(PyInterpreterState *interp) { + PyThreadState *tstate; /* No need to lock the mutex here because this should only happen - when the threads are all really dead (XXX famous last words). */ - _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + when the threads are all really dead (XXX famous last words). + + Cannot use _Py_FOR_EACH_TSTATE_UNLOCKED because we are freeing + the thread states here. + */ + while ((tstate = interp->threads.head) != NULL) { tstate_verify_not_active(tstate); tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); @@ -1934,6 +1901,9 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) _Py_EnsureTstateNotNULL(tstate); #ifdef Py_GIL_DISABLED _Py_qsbr_detach(((_PyThreadStateImpl *)tstate)->qsbr); +#endif +#ifdef Py_STATS + _PyStats_Detach((_PyThreadStateImpl *)tstate); #endif current_fast_clear(tstate->interp->runtime); tstate_delete_common(tstate, 1); // release GIL as part of call @@ -2092,7 +2062,7 @@ tstate_activate(PyThreadState *tstate) assert(!tstate->_status.active); assert(!tstate->_status.bound_gilstate || - tstate == gilstate_tss_get((tstate->interp->runtime))); + tstate == gilstate_get()); if (!tstate->_status.bound_gilstate) { bind_gilstate_tstate(tstate); } @@ -2108,6 +2078,10 @@ tstate_deactivate(PyThreadState *tstate) assert(tstate_is_bound(tstate)); assert(tstate->_status.active); +#if Py_STATS + _PyStats_Detach((_PyThreadStateImpl *)tstate); +#endif + tstate->_status.active = 0; // We do not unbind the gilstate tstate here. @@ -2211,6 +2185,10 @@ _PyThreadState_Attach(PyThreadState *tstate) _PyCriticalSection_Resume(tstate); } +#ifdef Py_STATS + _PyStats_Attach((_PyThreadStateImpl *)tstate); +#endif + #if defined(Py_DEBUG) errno = err; #endif @@ -2345,19 +2323,22 @@ stop_the_world(struct _stoptheworld_state *stw) { _PyRuntimeState *runtime = &_PyRuntime; - PyMutex_Lock(&stw->mutex); + // gh-137433: Acquire the rwmutex first to avoid deadlocks with daemon + // threads that may hang when blocked on lock acquisition. if (stw->is_global) { _PyRWMutex_Lock(&runtime->stoptheworld_mutex); } else { _PyRWMutex_RLock(&runtime->stoptheworld_mutex); } + PyMutex_Lock(&stw->mutex); HEAD_LOCK(runtime); stw->requested = 1; stw->thread_countdown = 0; stw->stop_event = (PyEvent){0}; // zero-initialize (unset) stw->requester = _PyThreadState_GET(); // may be NULL + FT_STAT_WORLD_STOP_INC(); _Py_FOR_EACH_STW_INTERP(stw, i) { _Py_FOR_EACH_TSTATE_UNLOCKED(i, t) { @@ -2417,13 +2398,13 @@ start_the_world(struct _stoptheworld_state *stw) } stw->requester = NULL; HEAD_UNLOCK(runtime); + PyMutex_Unlock(&stw->mutex); if (stw->is_global) { _PyRWMutex_Unlock(&runtime->stoptheworld_mutex); } else { _PyRWMutex_RUnlock(&runtime->stoptheworld_mutex); } - PyMutex_Unlock(&stw->mutex); } #endif // Py_GIL_DISABLED @@ -2560,7 +2541,7 @@ _PyThreadState_Bind(PyThreadState *tstate) bind_tstate(tstate); // This makes sure there's a gilstate tstate bound // as soon as possible. - if (gilstate_tss_get(tstate->interp->runtime) == NULL) { + if (gilstate_get() == NULL) { bind_gilstate_tstate(tstate); } } @@ -2569,14 +2550,10 @@ _PyThreadState_Bind(PyThreadState *tstate) uintptr_t _Py_GetThreadLocal_Addr(void) { -#ifdef HAVE_THREAD_LOCAL // gh-112535: Use the address of the thread-local PyThreadState variable as // a unique identifier for the current thread. Each thread has a unique // _Py_tss_tstate variable with a unique address. return (uintptr_t)&_Py_tss_tstate; -#else -# error "no supported thread-local variable storage classifier" -#endif } #endif @@ -2762,7 +2739,7 @@ _PyGILState_Init(PyInterpreterState *interp) return _PyStatus_OK(); } _PyRuntimeState *runtime = interp->runtime; - assert(gilstate_tss_get(runtime) == NULL); + assert(gilstate_get() == NULL); assert(runtime->gilstate.autoInterpreterState == NULL); runtime->gilstate.autoInterpreterState = interp; return _PyStatus_OK(); @@ -2798,7 +2775,7 @@ _PyGILState_SetTstate(PyThreadState *tstate) _PyRuntimeState *runtime = tstate->interp->runtime; assert(runtime->gilstate.autoInterpreterState == tstate->interp); - assert(gilstate_tss_get(runtime) == tstate); + assert(gilstate_get() == tstate); assert(tstate->gilstate_counter == 1); #endif } @@ -2814,11 +2791,7 @@ _PyGILState_GetInterpreterStateUnsafe(void) PyThreadState * PyGILState_GetThisThreadState(void) { - _PyRuntimeState *runtime = &_PyRuntime; - if (!gilstate_tss_initialized(runtime)) { - return NULL; - } - return gilstate_tss_get(runtime); + return gilstate_get(); } int @@ -2829,16 +2802,12 @@ PyGILState_Check(void) return 1; } - if (!gilstate_tss_initialized(runtime)) { - return 1; - } - PyThreadState *tstate = current_fast_get(); if (tstate == NULL) { return 0; } - PyThreadState *tcur = gilstate_tss_get(runtime); + PyThreadState *tcur = gilstate_get(); return (tstate == tcur); } @@ -2853,12 +2822,17 @@ PyGILState_Ensure(void) called Py_Initialize(). */ /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been - called by Py_Initialize() */ - assert(_PyEval_ThreadsInitialized()); - assert(gilstate_tss_initialized(runtime)); - assert(runtime->gilstate.autoInterpreterState != NULL); + called by Py_Initialize() - PyThreadState *tcur = gilstate_tss_get(runtime); + TODO: This isn't thread-safe. There's no protection here against + concurrent finalization of the interpreter; it's simply a guard + for *after* the interpreter has finalized. + */ + if (!_PyEval_ThreadsInitialized() || runtime->gilstate.autoInterpreterState == NULL) { + PyThread_hang_thread(); + } + + PyThreadState *tcur = gilstate_get(); int has_gil; if (tcur == NULL) { /* Create a new Python thread state for this thread */ @@ -2898,8 +2872,7 @@ PyGILState_Ensure(void) void PyGILState_Release(PyGILState_STATE oldstate) { - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = gilstate_tss_get(runtime); + PyThreadState *tstate = gilstate_get(); if (tstate == NULL) { Py_FatalError("auto-releasing thread-state, " "but no thread-state for this thread"); @@ -3007,7 +2980,7 @@ _PyInterpreterState_HasFeature(PyInterpreterState *interp, unsigned long feature static PyObject ** push_chunk(PyThreadState *tstate, int size) { - int allocate_size = DATA_STACK_CHUNK_SIZE; + int allocate_size = _PY_DATA_STACK_CHUNK_SIZE; while (allocate_size < (int)sizeof(PyObject*)*(size + MINIMUM_OVERHEAD)) { allocate_size *= 2; } diff --git a/Python/pystats.c b/Python/pystats.c new file mode 100644 index 00000000000..2e377b8e6da --- /dev/null +++ b/Python/pystats.c @@ -0,0 +1,819 @@ +#include "Python.h" + +#include "pycore_opcode_metadata.h" // _PyOpcode_Caches +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() +#include "pycore_tstate.h" +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_uop_metadata.h" // _PyOpcode_uop_name +#include "pycore_uop_ids.h" // MAX_UOP_ID +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime.h" // NUM_GENERATIONS + +#include <stdlib.h> // rand() + +#ifdef Py_STATS + +PyStats * +_PyStats_GetLocal(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate) { + return tstate->pystats; + } + return NULL; +} + +#ifdef Py_GIL_DISABLED +#define STATS_LOCK(interp) PyMutex_Lock(&interp->pystats_mutex) +#define STATS_UNLOCK(interp) PyMutex_Unlock(&interp->pystats_mutex) +#else +#define STATS_LOCK(interp) +#define STATS_UNLOCK(interp) +#endif + + +#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID +#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" +#endif + +#define ADD_STAT_TO_DICT(res, field) \ + do { \ + PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ + if (val == NULL) { \ + Py_DECREF(res); \ + return NULL; \ + } \ + if (PyDict_SetItemString(res, #field, val) == -1) { \ + Py_DECREF(res); \ + Py_DECREF(val); \ + return NULL; \ + } \ + Py_DECREF(val); \ + } while(0); + +static PyObject* +stats_to_dict(SpecializationStats *stats) +{ + PyObject *res = PyDict_New(); + if (res == NULL) { + return NULL; + } + ADD_STAT_TO_DICT(res, success); + ADD_STAT_TO_DICT(res, failure); + ADD_STAT_TO_DICT(res, hit); + ADD_STAT_TO_DICT(res, deferred); + ADD_STAT_TO_DICT(res, miss); + ADD_STAT_TO_DICT(res, deopt); + PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS); + if (failure_kinds == NULL) { + Py_DECREF(res); + return NULL; + } + for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { + PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]); + if (stat == NULL) { + Py_DECREF(res); + Py_DECREF(failure_kinds); + return NULL; + } + PyTuple_SET_ITEM(failure_kinds, i, stat); + } + if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) { + Py_DECREF(res); + Py_DECREF(failure_kinds); + return NULL; + } + Py_DECREF(failure_kinds); + return res; +} +#undef ADD_STAT_TO_DICT + +static int +add_stat_dict( + PyStats *src, + PyObject *res, + int opcode, + const char *name) { + + SpecializationStats *stats = &src->opcode_stats[opcode].specialization; + PyObject *d = stats_to_dict(stats); + if (d == NULL) { + return -1; + } + int err = PyDict_SetItemString(res, name, d); + Py_DECREF(d); + return err; +} + +PyObject* +_Py_GetSpecializationStats(void) { + PyThreadState *tstate = _PyThreadState_GET(); + PyStats *src = FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct); + if (src == NULL) { + Py_RETURN_NONE; + } + PyObject *stats = PyDict_New(); + if (stats == NULL) { + return NULL; + } + int err = 0; + err += add_stat_dict(src, stats, CONTAINS_OP, "contains_op"); + err += add_stat_dict(src, stats, LOAD_SUPER_ATTR, "load_super_attr"); + err += add_stat_dict(src, stats, LOAD_ATTR, "load_attr"); + err += add_stat_dict(src, stats, LOAD_GLOBAL, "load_global"); + err += add_stat_dict(src, stats, STORE_SUBSCR, "store_subscr"); + err += add_stat_dict(src, stats, STORE_ATTR, "store_attr"); + err += add_stat_dict(src, stats, JUMP_BACKWARD, "jump_backward"); + err += add_stat_dict(src, stats, CALL, "call"); + err += add_stat_dict(src, stats, CALL_KW, "call_kw"); + err += add_stat_dict(src, stats, BINARY_OP, "binary_op"); + err += add_stat_dict(src, stats, COMPARE_OP, "compare_op"); + err += add_stat_dict(src, stats, UNPACK_SEQUENCE, "unpack_sequence"); + err += add_stat_dict(src, stats, FOR_ITER, "for_iter"); + err += add_stat_dict(src, stats, TO_BOOL, "to_bool"); + err += add_stat_dict(src, stats, SEND, "send"); + if (err < 0) { + Py_DECREF(stats); + return NULL; + } + return stats; +} + + +#define PRINT_STAT(i, field) \ + if (stats[i].field) { \ + fprintf(out, " opcode[%s]." #field " : %" PRIu64 "\n", _PyOpcode_OpName[i], stats[i].field); \ + } + +static void +print_spec_stats(FILE *out, OpcodeStats *stats) +{ + /* Mark some opcodes as specializable for stats, + * even though we don't specialize them yet. */ + fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); + fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); + fprintf(out, "opcode[GET_ITER].specializable : 1\n"); + for (int i = 0; i < 256; i++) { + if (_PyOpcode_Caches[i]) { + /* Ignore jumps as they cannot be specialized */ + switch (i) { + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case JUMP_BACKWARD: + break; + default: + fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); + } + } + PRINT_STAT(i, specialization.success); + PRINT_STAT(i, specialization.failure); + PRINT_STAT(i, specialization.hit); + PRINT_STAT(i, specialization.deferred); + PRINT_STAT(i, specialization.miss); + PRINT_STAT(i, specialization.deopt); + PRINT_STAT(i, execution_count); + for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) { + uint64_t val = stats[i].specialization.failure_kinds[j]; + if (val) { + fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %" + PRIu64 "\n", _PyOpcode_OpName[i], j, val); + } + } + for (int j = 0; j < 256; j++) { + if (stats[i].pair_count[j]) { + fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n", + _PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]); + } + } + } +} +#undef PRINT_STAT + + +static void +print_call_stats(FILE *out, CallStats *stats) +{ + fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls); + fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); + fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); + fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); + for (int i = 0; i < EVAL_CALL_KINDS; i++) { + fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); + } +} + +static void +print_object_stats(FILE *out, ObjectStats *stats) +{ + fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist); + fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist); + fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); + fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512); + fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); + fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); + fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); + fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); + fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); + fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); + fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); + fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); + fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); + fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); + fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); + fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); + fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); + fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); + fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); + fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); + fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); + fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); + fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); + fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); + fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); +} + +static void +print_gc_stats(FILE *out, GCStats *stats) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); + fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); + fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); + fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); + } +} + +#ifdef _Py_TIER2 +static void +print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) +{ + for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { + fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]); + } +} + +extern const char *_PyUOpName(int index); + +static void +print_optimization_stats(FILE *out, OptimizationStats *stats) +{ + fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts); + fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created); + fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed); + fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed); + fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); + fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); + fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); + fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); + fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); + fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); + fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); + fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee); + fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); + + print_histogram(out, "Trace length", stats->trace_length_hist); + print_histogram(out, "Trace run length", stats->trace_run_length_hist); + print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); + + fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); + fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); + fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", + stats->optimizer_failure_reason_no_memory); + fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); + fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); + for (int i = 0; i <= MAX_UOP_ID; i++) { + if (stats->opcode[i].execution_count) { + fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); + } + if (stats->opcode[i].miss) { + fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); + } + } + for (int i = 0; i < 256; i++) { + if (stats->unsupported_opcode[i]) { + fprintf( + out, + "unsupported_opcode[%s].count : %" PRIu64 "\n", + _PyOpcode_OpName[i], + stats->unsupported_opcode[i] + ); + } + } + + for (int i = 1; i <= MAX_UOP_ID; i++){ + for (int j = 1; j <= MAX_UOP_ID; j++) { + if (stats->opcode[i].pair_count[j]) { + fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", + _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); + } + } + } + for (int i = 0; i < MAX_UOP_ID; i++) { + if (stats->error_in_opcode[i]) { + fprintf( + out, + "error_in_opcode[%s].count : %" PRIu64 "\n", + _PyUOpName(i), + stats->error_in_opcode[i] + ); + } + } + fprintf(out, "JIT total memory size: %" PRIu64 "\n", stats->jit_total_memory_size); + fprintf(out, "JIT code size: %" PRIu64 "\n", stats->jit_code_size); + fprintf(out, "JIT trampoline size: %" PRIu64 "\n", stats->jit_trampoline_size); + fprintf(out, "JIT data size: %" PRIu64 "\n", stats->jit_data_size); + fprintf(out, "JIT padding size: %" PRIu64 "\n", stats->jit_padding_size); + fprintf(out, "JIT freed memory size: %" PRIu64 "\n", stats->jit_freed_memory_size); + + print_histogram(out, "Trace total memory size", stats->trace_total_memory_hist); +} +#endif + +#ifdef Py_GIL_DISABLED +static void +print_ft_stats(FILE *out, FTStats *stats) +{ + fprintf(out, "Mutex sleeps (mutex_sleeps): %" PRIu64 "\n", stats->mutex_sleeps); + fprintf(out, "QSBR polls (qsbr_polls): %" PRIu64 "\n", stats->qsbr_polls); + fprintf(out, "World stops (world_stops): %" PRIu64 "\n", stats->world_stops); +} +#endif + +static void +print_rare_event_stats(FILE *out, RareEventStats *stats) +{ + fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); + fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); + fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); + fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); + fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); + fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); + fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); +} + +static void +print_stats(FILE *out, PyStats *stats) +{ + print_spec_stats(out, stats->opcode_stats); + print_call_stats(out, &stats->call_stats); + print_object_stats(out, &stats->object_stats); + print_gc_stats(out, stats->gc_stats); +#ifdef _Py_TIER2 + print_optimization_stats(out, &stats->optimization_stats); +#endif +#ifdef Py_GIL_DISABLED + print_ft_stats(out, &stats->ft_stats); +#endif + print_rare_event_stats(out, &stats->rare_event_stats); +} + +#ifdef Py_GIL_DISABLED + +static void +merge_specialization_stats(SpecializationStats *dest, const SpecializationStats *src) +{ + dest->success += src->success; + dest->failure += src->failure; + dest->hit += src->hit; + dest->deferred += src->deferred; + dest->miss += src->miss; + dest->deopt += src->deopt; + for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { + dest->failure_kinds[i] += src->failure_kinds[i]; + } +} + +static void +merge_opcode_stats_array(OpcodeStats *dest, const OpcodeStats *src) +{ + for (int i = 0; i < 256; i++) { + merge_specialization_stats(&dest[i].specialization, &src[i].specialization); + dest[i].execution_count += src[i].execution_count; + for (int j = 0; j < 256; j++) { + dest[i].pair_count[j] += src[i].pair_count[j]; + } + } +} + +static void +merge_call_stats(CallStats *dest, const CallStats *src) +{ + dest->inlined_py_calls += src->inlined_py_calls; + dest->pyeval_calls += src->pyeval_calls; + dest->frames_pushed += src->frames_pushed; + dest->frame_objects_created += src->frame_objects_created; + for (int i = 0; i < EVAL_CALL_KINDS; i++) { + dest->eval_calls[i] += src->eval_calls[i]; + } +} + +static void +merge_object_stats(ObjectStats *dest, const ObjectStats *src) +{ + dest->increfs += src->increfs; + dest->decrefs += src->decrefs; + dest->interpreter_increfs += src->interpreter_increfs; + dest->interpreter_decrefs += src->interpreter_decrefs; + dest->immortal_increfs += src->immortal_increfs; + dest->immortal_decrefs += src->immortal_decrefs; + dest->interpreter_immortal_increfs += src->interpreter_immortal_increfs; + dest->interpreter_immortal_decrefs += src->interpreter_immortal_decrefs; + dest->allocations += src->allocations; + dest->allocations512 += src->allocations512; + dest->allocations4k += src->allocations4k; + dest->allocations_big += src->allocations_big; + dest->frees += src->frees; + dest->to_freelist += src->to_freelist; + dest->from_freelist += src->from_freelist; + dest->inline_values += src->inline_values; + dest->dict_materialized_on_request += src->dict_materialized_on_request; + dest->dict_materialized_new_key += src->dict_materialized_new_key; + dest->dict_materialized_too_big += src->dict_materialized_too_big; + dest->dict_materialized_str_subclass += src->dict_materialized_str_subclass; + dest->type_cache_hits += src->type_cache_hits; + dest->type_cache_misses += src->type_cache_misses; + dest->type_cache_dunder_hits += src->type_cache_dunder_hits; + dest->type_cache_dunder_misses += src->type_cache_dunder_misses; + dest->type_cache_collisions += src->type_cache_collisions; + dest->object_visits += src->object_visits; +} + +static void +merge_uop_stats_array(UOpStats *dest, const UOpStats *src) +{ + for (int i = 0; i <= PYSTATS_MAX_UOP_ID; i++) { + dest[i].execution_count += src[i].execution_count; + dest[i].miss += src[i].miss; + for (int j = 0; j <= PYSTATS_MAX_UOP_ID; j++) { + dest[i].pair_count[j] += src[i].pair_count[j]; + } + } +} + +static void +merge_optimization_stats(OptimizationStats *dest, const OptimizationStats *src) +{ + dest->attempts += src->attempts; + dest->traces_created += src->traces_created; + dest->traces_executed += src->traces_executed; + dest->uops_executed += src->uops_executed; + dest->trace_stack_overflow += src->trace_stack_overflow; + dest->trace_stack_underflow += src->trace_stack_underflow; + dest->trace_too_long += src->trace_too_long; + dest->trace_too_short += src->trace_too_short; + dest->inner_loop += src->inner_loop; + dest->recursive_call += src->recursive_call; + dest->low_confidence += src->low_confidence; + dest->unknown_callee += src->unknown_callee; + dest->executors_invalidated += src->executors_invalidated; + dest->optimizer_attempts += src->optimizer_attempts; + dest->optimizer_successes += src->optimizer_successes; + dest->optimizer_failure_reason_no_memory += src->optimizer_failure_reason_no_memory; + dest->remove_globals_builtins_changed += src->remove_globals_builtins_changed; + dest->remove_globals_incorrect_keys += src->remove_globals_incorrect_keys; + dest->jit_total_memory_size += src->jit_total_memory_size; + dest->jit_code_size += src->jit_code_size; + dest->jit_trampoline_size += src->jit_trampoline_size; + dest->jit_data_size += src->jit_data_size; + dest->jit_padding_size += src->jit_padding_size; + dest->jit_freed_memory_size += src->jit_freed_memory_size; + + merge_uop_stats_array(dest->opcode, src->opcode); + + for (int i = 0; i < 256; i++) { + dest->unsupported_opcode[i] += src->unsupported_opcode[i]; + } + for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { + dest->trace_length_hist[i] += src->trace_length_hist[i]; + dest->trace_run_length_hist[i] += src->trace_run_length_hist[i]; + dest->optimized_trace_length_hist[i] += src->optimized_trace_length_hist[i]; + dest->trace_total_memory_hist[i] += src->trace_total_memory_hist[i]; + } + for (int i = 0; i <= PYSTATS_MAX_UOP_ID; i++) { + dest->error_in_opcode[i] += src->error_in_opcode[i]; + } +} + +static void +merge_ft_stats(FTStats *dest, const FTStats *src) +{ + dest->mutex_sleeps = src->mutex_sleeps; + dest->qsbr_polls = src->qsbr_polls; + dest->world_stops = src->world_stops; +} + +static void +merge_rare_event_stats(RareEventStats *dest, const RareEventStats *src) +{ + dest->set_class += src->set_class; + dest->set_bases += src->set_bases; + dest->set_eval_frame_func += src->set_eval_frame_func; + dest->builtin_dict += src->builtin_dict; + dest->func_modification += src->func_modification; + dest->watched_dict_modification += src->watched_dict_modification; + dest->watched_globals_modification += src->watched_globals_modification; +} + +static void +merge_gc_stats_array(GCStats *dest, const GCStats *src) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + dest[i].collections += src[i].collections; + dest[i].object_visits += src[i].object_visits; + dest[i].objects_collected += src[i].objects_collected; + dest[i].objects_transitively_reachable += src[i].objects_transitively_reachable; + dest[i].objects_not_transitively_reachable += src[i].objects_not_transitively_reachable; + } +} + +void +stats_zero_thread(_PyThreadStateImpl *tstate) +{ + // Zero the thread local stat counters + if (tstate->pystats_struct) { + memset(tstate->pystats_struct, 0, sizeof(PyStats)); + } +} + +// merge stats for a single thread into the global structure +void +stats_merge_thread(_PyThreadStateImpl *tstate) +{ + PyStats *src = tstate->pystats_struct; + PyStats *dest = ((PyThreadState *)tstate)->interp->pystats_struct; + + if (src == NULL || dest == NULL) { + return; + } + + // Merge each category of stats using the helper functions. + merge_opcode_stats_array(dest->opcode_stats, src->opcode_stats); + merge_call_stats(&dest->call_stats, &src->call_stats); + merge_object_stats(&dest->object_stats, &src->object_stats); + merge_optimization_stats(&dest->optimization_stats, &src->optimization_stats); + merge_ft_stats(&dest->ft_stats, &src->ft_stats); + merge_rare_event_stats(&dest->rare_event_stats, &src->rare_event_stats); + merge_gc_stats_array(dest->gc_stats, src->gc_stats); +} +#endif // Py_GIL_DISABLED + +// toggle stats collection on or off for all threads +static int +stats_toggle_on_off(PyThreadState *tstate, int on) +{ + bool changed = false; + PyInterpreterState *interp = tstate->interp; + STATS_LOCK(interp); + if (on && interp->pystats_struct == NULL) { + PyStats *s = PyMem_RawCalloc(1, sizeof(PyStats)); + if (s == NULL) { + STATS_UNLOCK(interp); + return -1; + } + FT_ATOMIC_STORE_PTR_RELAXED(interp->pystats_struct, s); + } + if (tstate->interp->pystats_enabled != on) { + FT_ATOMIC_STORE_INT_RELAXED(interp->pystats_enabled, on); + changed = true; + } + STATS_UNLOCK(interp); + if (!changed) { + return 0; + } + _PyEval_StopTheWorld(interp); + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + PyStats *s = NULL; + if (interp->pystats_enabled) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *ts_impl = (_PyThreadStateImpl *)ts; + if (ts_impl->pystats_struct == NULL) { + // first activation for this thread, allocate structure + ts_impl->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + } + s = ts_impl->pystats_struct; +#else + s = ts->interp->pystats_struct; +#endif + } + ts->pystats = s; + } + _PyEval_StartTheWorld(interp); + return 0; +} + +// zero stats for all threads and for the interpreter +static void +stats_zero_all(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + return; + } + if (FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct) == NULL) { + return; + } + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); +#ifdef Py_GIL_DISABLED + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + stats_zero_thread((_PyThreadStateImpl *)ts); + } +#endif + if (interp->pystats_struct) { + memset(interp->pystats_struct, 0, sizeof(PyStats)); + } + _PyEval_StartTheWorld(interp); +} + +// merge stats for all threads into the per-interpreter structure +static void +stats_merge_all(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + return; + } + if (FT_ATOMIC_LOAD_PTR_RELAXED(tstate->interp->pystats_struct) == NULL) { + return; + } + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); +#ifdef Py_GIL_DISABLED + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, ts) { + stats_merge_thread((_PyThreadStateImpl *)ts); + stats_zero_thread((_PyThreadStateImpl *)ts); + } +#endif + _PyEval_StartTheWorld(interp); +} + +int +_Py_StatsOn(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return stats_toggle_on_off(tstate, 1); +} + +void +_Py_StatsOff(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + stats_toggle_on_off(tstate, 0); +} + +void +_Py_StatsClear(void) +{ + stats_zero_all(); +} + +static int +mem_is_zero(unsigned char *ptr, size_t size) +{ + for (size_t i=0; i < size; i++) { + if (*ptr != 0) { + return 0; + } + ptr++; + } + return 1; +} + +int +_Py_PrintSpecializationStats(int to_file) +{ + assert(to_file); + stats_merge_all(); + PyThreadState *tstate = _PyThreadState_GET(); + STATS_LOCK(tstate->interp); + PyStats *stats = tstate->interp->pystats_struct; + if (stats == NULL) { + STATS_UNLOCK(tstate->interp); + return 0; + } +#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA))) + int is_zero = ( + MEM_IS_ZERO(stats->gc_stats) // is a pointer + && MEM_IS_ZERO(&stats->opcode_stats) + && MEM_IS_ZERO(&stats->call_stats) + && MEM_IS_ZERO(&stats->object_stats) + ); +#undef MEM_IS_ZERO + STATS_UNLOCK(tstate->interp); + if (is_zero) { + // gh-108753: -X pystats command line was used, but then _stats_off() + // and _stats_clear() have been called: in this case, avoid printing + // useless "all zeros" statistics. + return 0; + } + + FILE *out = stderr; + if (to_file) { + /* Write to a file instead of stderr. */ +# ifdef MS_WINDOWS + const char *dirname = "c:\\temp\\py_stats\\"; +# else + const char *dirname = "/tmp/py_stats/"; +# endif + /* Use random 160 bit number as file name, + * to avoid both accidental collisions and + * symlink attacks. */ + unsigned char rand[20]; + char hex_name[41]; + _PyOS_URandomNonblock(rand, 20); + for (int i = 0; i < 20; i++) { + hex_name[2*i] = Py_hexdigits[rand[i]&15]; + hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15]; + } + hex_name[40] = '\0'; + char buf[64]; + assert(strlen(dirname) + 40 + strlen(".txt") < 64); + sprintf(buf, "%s%s.txt", dirname, hex_name); + FILE *fout = fopen(buf, "w"); + if (fout) { + out = fout; + } + } + else { + fprintf(out, "Specialization stats:\n"); + } + STATS_LOCK(tstate->interp); + print_stats(out, stats); + STATS_UNLOCK(tstate->interp); + if (out != stderr) { + fclose(out); + } + return 1; +} + +PyStatus +_PyStats_InterpInit(PyInterpreterState *interp) +{ + if (interp->config._pystats) { + // start with pystats enabled, can be disabled via sys._stats_off() + // this needs to be set before the first tstate is created + interp->pystats_enabled = 1; + interp->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + if (interp->pystats_struct == NULL) { + return _PyStatus_ERR("out-of-memory while initializing interpreter"); + } + } + return _PyStatus_OK(); +} + +bool +_PyStats_ThreadInit(PyInterpreterState *interp, _PyThreadStateImpl *tstate) +{ +#ifdef Py_GIL_DISABLED + if (FT_ATOMIC_LOAD_INT_RELAXED(interp->pystats_enabled)) { + assert(interp->pystats_struct != NULL); + tstate->pystats_struct = PyMem_RawCalloc(1, sizeof(PyStats)); + if (tstate->pystats_struct == NULL) { + return false; + } + } +#endif + return true; +} + +void +_PyStats_ThreadFini(_PyThreadStateImpl *tstate) +{ +#ifdef Py_GIL_DISABLED + STATS_LOCK(((PyThreadState *)tstate)->interp); + stats_merge_thread(tstate); + STATS_UNLOCK(((PyThreadState *)tstate)->interp); + PyMem_RawFree(tstate->pystats_struct); +#endif +} + +void +_PyStats_Attach(_PyThreadStateImpl *tstate_impl) +{ + PyStats *s; + PyThreadState *tstate = (PyThreadState *)tstate_impl; + PyInterpreterState *interp = tstate->interp; + if (FT_ATOMIC_LOAD_INT_RELAXED(interp->pystats_enabled)) { +#ifdef Py_GIL_DISABLED + s = ((_PyThreadStateImpl *)tstate)->pystats_struct; +#else + s = tstate->interp->pystats_struct; +#endif + } + else { + s = NULL; + } + tstate->pystats = s; +} + +void +_PyStats_Detach(_PyThreadStateImpl *tstate_impl) +{ + ((PyThreadState *)tstate_impl)->pystats = NULL; +} + +#endif // Py_STATS diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 38484f5a7d4..af2f5c5dce5 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -42,8 +42,7 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, else { bytes_per_sep_group = 0; } - - unsigned int abs_bytes_per_sep = Py_ABS(bytes_per_sep_group); + unsigned int abs_bytes_per_sep = _Py_ABS_CAST(unsigned int, bytes_per_sep_group); Py_ssize_t resultlen = 0; if (bytes_per_sep_group && arglen > 0) { /* How many sep characters we'll be inserting. */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 7b74f613ed5..e8aca939d1f 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -43,7 +43,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) s += 3; if (case_insensitive_match(s, "inity")) s += 5; - retval = negate ? -Py_INFINITY : Py_INFINITY; + retval = negate ? -INFINITY : INFINITY; } else if (case_insensitive_match(s, "nan")) { s += 3; @@ -286,7 +286,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) string, -1.0 is returned and again ValueError is raised. On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), - if overflow_exception is NULL then +-Py_INFINITY is returned, and no Python + if overflow_exception is NULL then +-INFINITY is returned, and no Python exception is raised. Otherwise, overflow_exception should point to a Python exception, this exception will be raised, -1.0 will be returned, and *endptr will point just past the end of the converted value. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 4ee287af72f..272be504a68 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -114,7 +114,7 @@ _PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flag } PyObject *v; - if (_PySys_GetOptionalAttr(&_Py_ID(ps1), &v) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(ps1), &v) < 0) { PyErr_Print(); return -1; } @@ -128,7 +128,7 @@ _PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flag } } Py_XDECREF(v); - if (_PySys_GetOptionalAttr(&_Py_ID(ps2), &v) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(ps2), &v) < 0) { PyErr_Print(); return -1; } @@ -206,7 +206,7 @@ pyrun_one_parse_ast(FILE *fp, PyObject *filename, PyObject *encoding_obj = NULL; const char *encoding = NULL; if (fp == stdin) { - if (_PySys_GetOptionalAttr(&_Py_ID(stdin), &attr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stdin), &attr) < 0) { PyErr_Clear(); } else if (attr != NULL && attr != Py_None) { @@ -226,7 +226,7 @@ pyrun_one_parse_ast(FILE *fp, PyObject *filename, // Get sys.ps1 (as UTF-8) PyObject *ps1_obj = NULL; const char *ps1 = ""; - if (_PySys_GetOptionalAttr(&_Py_ID(ps1), &attr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(ps1), &attr) < 0) { PyErr_Clear(); } else if (attr != NULL) { @@ -247,7 +247,7 @@ pyrun_one_parse_ast(FILE *fp, PyObject *filename, // Get sys.ps2 (as UTF-8) PyObject *ps2_obj = NULL; const char *ps2 = ""; - if (_PySys_GetOptionalAttr(&_Py_ID(ps2), &attr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(ps2), &attr) < 0) { PyErr_Clear(); } else if (attr != NULL) { @@ -658,7 +658,7 @@ _Py_HandleSystemExitAndKeyboardInterrupt(int *exitcode_p) } PyObject *sys_stderr; - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &sys_stderr) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &sys_stderr) < 0) { PyErr_Clear(); } else if (sys_stderr != NULL && sys_stderr != Py_None) { @@ -722,7 +722,7 @@ _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars) _PyErr_Clear(tstate); } } - if (_PySys_GetOptionalAttr(&_Py_ID(excepthook), &hook) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(excepthook), &hook) < 0) { PyErr_Clear(); } if (_PySys_Audit(tstate, "sys.excepthook", "OOOO", hook ? hook : Py_None, @@ -1181,7 +1181,7 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) } if (print_exception_recursive(&ctx, value) < 0) { PyErr_Clear(); - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); } Py_XDECREF(ctx.seen); @@ -1197,16 +1197,16 @@ void PyErr_Display(PyObject *unused, PyObject *value, PyObject *tb) { PyObject *file; - if (_PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { PyObject *exc = PyErr_GetRaisedException(); - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); - _PyObject_Dump(exc); + PyUnstable_Object_Dump(exc); Py_DECREF(exc); return; } if (file == NULL) { - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); return; } @@ -1252,12 +1252,19 @@ _PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, } else { name = &_Py_STR(anon_string); } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + goto done; + } - mod = _PyParser_ASTFromString(str, name, start, flags, arena); + mod = _PyParser_ASTFromString(str, name, start, flags, arena, module); + Py_XDECREF(module); - if (mod != NULL) { + if (mod != NULL) { ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source); } + +done: Py_XDECREF(source); _PyArena_Free(arena); return ret; @@ -1321,7 +1328,7 @@ static void flush_io_stream(PyThreadState *tstate, PyObject *name) { PyObject *f; - if (_PySys_GetOptionalAttr(name, &f) < 0) { + if (PySys_GetOptionalAttr(name, &f) < 0) { PyErr_Clear(); } if (f != NULL) { @@ -1365,6 +1372,29 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py return PyEval_EvalCode((PyObject*)co, globals, locals); } +static PyObject * +get_interactive_filename(PyObject *filename, Py_ssize_t count) +{ + PyObject *result; + Py_ssize_t len = PyUnicode_GET_LENGTH(filename); + + if (len >= 2 + && PyUnicode_ReadChar(filename, 0) == '<' + && PyUnicode_ReadChar(filename, len - 1) == '>') { + PyObject *middle = PyUnicode_Substring(filename, 1, len-1); + if (middle == NULL) { + return NULL; + } + result = PyUnicode_FromFormat("<%U-%d>", middle, count); + Py_DECREF(middle); + } else { + result = PyUnicode_FromFormat( + "%U-%d", filename, count); + } + return result; + +} + static PyObject * run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src, @@ -1375,8 +1405,8 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, if (interactive_src) { PyInterpreterState *interp = tstate->interp; if (generate_new_source) { - interactive_filename = PyUnicode_FromFormat( - "%U-%d", filename, interp->_interactive_src_count++); + interactive_filename = get_interactive_filename( + filename, interp->_interactive_src_count++); } else { Py_INCREF(interactive_filename); } @@ -1384,8 +1414,17 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, return NULL; } } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + if (interactive_src) { + Py_DECREF(interactive_filename); + } + return NULL; + } - PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); + PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, + arena, module); + Py_XDECREF(module); if (co == NULL) { if (interactive_src) { Py_DECREF(interactive_filename); @@ -1484,6 +1523,14 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) +{ + return _Py_CompileStringObjectWithModule(str, filename, start, + flags, optimize, NULL); +} + +PyObject * +_Py_CompileStringObjectWithModule(const char *str, PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, PyObject *module) { PyCodeObject *co; mod_ty mod; @@ -1491,14 +1538,16 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, + syntax_check_only, module) < 0) + { _PyArena_Free(arena); return NULL; } @@ -1506,7 +1555,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, _PyArena_Free(arena); return result; } - co = _PyAST_Compile(mod, filename, flags, optimize, arena); + co = _PyAST_Compile(mod, filename, flags, optimize, arena, module); _PyArena_Free(arena); return (PyObject *)co; } @@ -1524,6 +1573,26 @@ Py_CompileStringExFlags(const char *str, const char *filename_str, int start, return co; } +int +_PyObject_SupportedAsScript(PyObject *cmd) +{ + if (PyUnicode_Check(cmd)) { + return 1; + } + else if (PyBytes_Check(cmd)) { + return 1; + } + else if (PyByteArray_Check(cmd)) { + return 1; + } + else if (PyObject_CheckBuffer(cmd)) { + return 1; + } + else { + return 0; + } +} + const char * _Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy) { diff --git a/Python/pytime.c b/Python/pytime.c index 67cf6437264..2f3d854428b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -2,7 +2,7 @@ #include "pycore_initconfig.h" // _PyStatus_ERR #include "pycore_pystate.h" // _Py_AssertHoldsTstate() #include "pycore_runtime.h" // _PyRuntime -#include "pycore_time.h" // PyTime_t +#include "pycore_time.h" // export _PyLong_FromTime_t() #include <time.h> // gmtime_r() #ifdef HAVE_SYS_TIME_H @@ -368,8 +368,20 @@ pytime_object_to_denominator(PyObject *obj, time_t *sec, long *numerator, { assert(denominator >= 1); - if (PyFloat_Check(obj)) { + if (PyIndex_Check(obj)) { + *sec = _PyLong_AsTime_t(obj); + *numerator = 0; + if (*sec == (time_t)-1 && PyErr_Occurred()) { + return -1; + } + return 0; + } + else { double d = PyFloat_AsDouble(obj); + if (d == -1 && PyErr_Occurred()) { + *numerator = 0; + return -1; + } if (isnan(d)) { *numerator = 0; PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); @@ -378,30 +390,28 @@ pytime_object_to_denominator(PyObject *obj, time_t *sec, long *numerator, return pytime_double_to_denominator(d, sec, numerator, denominator, round); } - else { - *sec = _PyLong_AsTime_t(obj); - *numerator = 0; - if (*sec == (time_t)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Format(PyExc_TypeError, - "argument must be int or float, not %T", obj); - } - return -1; - } - return 0; - } } int _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) { - if (PyFloat_Check(obj)) { + if (PyIndex_Check(obj)) { + *sec = _PyLong_AsTime_t(obj); + if (*sec == (time_t)-1 && PyErr_Occurred()) { + return -1; + } + return 0; + } + else { double intpart; /* volatile avoids optimization changing how numbers are rounded */ volatile double d; d = PyFloat_AsDouble(obj); + if (d == -1 && PyErr_Occurred()) { + return -1; + } if (isnan(d)) { PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; @@ -418,13 +428,6 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) *sec = (time_t)intpart; return 0; } - else { - *sec = _PyLong_AsTime_t(obj); - if (*sec == (time_t)-1 && PyErr_Occurred()) { - return -1; - } - return 0; - } } @@ -469,31 +472,6 @@ _PyTime_FromMicrosecondsClamp(PyTime_t us) } -int -_PyTime_FromLong(PyTime_t *tp, PyObject *obj) -{ - if (!PyLong_Check(obj)) { - PyErr_Format(PyExc_TypeError, "expect int, got %s", - Py_TYPE(obj)->tp_name); - return -1; - } - - static_assert(sizeof(long long) == sizeof(PyTime_t), - "PyTime_t is not long long"); - long long nsec = PyLong_AsLongLong(obj); - if (nsec == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - pytime_overflow(); - } - return -1; - } - - PyTime_t t = (PyTime_t)nsec; - *tp = t; - return 0; -} - - #ifdef HAVE_CLOCK_GETTIME static int pytime_fromtimespec(PyTime_t *tp, const struct timespec *ts, int raise_exc) @@ -586,39 +564,38 @@ static int pytime_from_object(PyTime_t *tp, PyObject *obj, _PyTime_round_t round, long unit_to_ns) { - if (PyFloat_Check(obj)) { + if (PyIndex_Check(obj)) { + long long sec = PyLong_AsLongLong(obj); + if (sec == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + pytime_overflow(); + } + return -1; + } + + static_assert(sizeof(long long) <= sizeof(PyTime_t), + "PyTime_t is smaller than long long"); + PyTime_t ns = (PyTime_t)sec; + if (pytime_mul(&ns, unit_to_ns) < 0) { + pytime_overflow(); + return -1; + } + + *tp = ns; + return 0; + } + else { double d; d = PyFloat_AsDouble(obj); + if (d == -1 && PyErr_Occurred()) { + return -1; + } if (isnan(d)) { PyErr_SetString(PyExc_ValueError, "Invalid value NaN (not a number)"); return -1; } return pytime_from_double(tp, d, round, unit_to_ns); } - - long long sec = PyLong_AsLongLong(obj); - if (sec == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - pytime_overflow(); - } - else if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_Format(PyExc_TypeError, - "'%T' object cannot be interpreted as an integer or float", - obj); - } - return -1; - } - - static_assert(sizeof(long long) <= sizeof(PyTime_t), - "PyTime_t is smaller than long long"); - PyTime_t ns = (PyTime_t)sec; - if (pytime_mul(&ns, unit_to_ns) < 0) { - pytime_overflow(); - return -1; - } - - *tp = ns; - return 0; } @@ -656,14 +633,6 @@ PyTime_AsSecondsDouble(PyTime_t ns) } -PyObject * -_PyTime_AsLong(PyTime_t ns) -{ - static_assert(sizeof(long long) >= sizeof(PyTime_t), - "PyTime_t is larger than long long"); - return PyLong_FromLongLong((long long)ns); -} - int _PyTime_FromSecondsDouble(double seconds, _PyTime_round_t round, PyTime_t *result) { diff --git a/Python/qsbr.c b/Python/qsbr.c index bf34fb2523d..b2153bf9d67 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -1,6 +1,6 @@ /* * Implementation of safe memory reclamation scheme using - * quiescent states. + * quiescent states. See InternalDocs/qsbr.md. * * This is derived from the "GUS" safe memory reclamation technique * in FreeBSD written by Jeffrey Roberson. It is heavily modified. Any bugs @@ -36,15 +36,12 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_qsbr.h" #include "pycore_tstate.h" // _PyThreadStateImpl +#include "pycore_stats.h" // FT_STAT_QSBR_POLL_INC() // Starting size of the array of qsbr thread states #define MIN_ARRAY_SIZE 8 -// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing -// the write sequence. -#define QSBR_DEFERRED_LIMIT 10 - // Allocate a QSBR thread state from the freelist static struct _qsbr_thread_state * qsbr_allocate(struct _qsbr_shared *shared) @@ -117,13 +114,9 @@ _Py_qsbr_advance(struct _qsbr_shared *shared) } uint64_t -_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) +_Py_qsbr_shared_next(struct _qsbr_shared *shared) { - if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { - return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; - } - qsbr->deferrals = 0; - return _Py_qsbr_advance(qsbr->shared); + return _Py_qsbr_shared_current(shared) + QSBR_INCR; } static uint64_t @@ -166,7 +159,7 @@ _Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) if (_Py_qbsr_goal_reached(qsbr, goal)) { return true; } - + FT_STAT_QSBR_POLL_INC(); uint64_t rd_seq = qsbr_poll_scan(qsbr->shared); return QSBR_LEQ(goal, rd_seq); } diff --git a/Python/remote_debug.h b/Python/remote_debug.h index edc77c30291..1c02870d3af 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -13,6 +13,16 @@ If you need to add a new function ensure that is declared 'static'. extern "C" { #endif +#ifdef __clang__ + #define UNUSED __attribute__((unused)) +#elif defined(__GNUC__) + #define UNUSED __attribute__((unused)) +#elif defined(_MSC_VER) + #define UNUSED __pragma(warning(suppress: 4505)) +#else + #define UNUSED +#endif + #if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_MODULE define" #endif @@ -35,14 +45,16 @@ extern "C" { # include <sys/mman.h> #endif -#if defined(__APPLE__) && TARGET_OS_OSX +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX # include <libproc.h> # include <mach-o/fat.h> # include <mach-o/loader.h> # include <mach-o/nlist.h> +# include <mach/error.h> # include <mach/mach.h> # include <mach/mach_vm.h> # include <mach/machine.h> +# include <mach/task_info.h> # include <sys/mman.h> # include <sys/proc.h> # include <sys/sysctl.h> @@ -73,43 +85,127 @@ extern "C" { # define HAVE_PROCESS_VM_READV 0 #endif +#define _set_debug_exception_cause(exception, format, ...) \ + do { \ + if (!PyErr_ExceptionMatches(PyExc_PermissionError)) { \ + PyThreadState *tstate = _PyThreadState_GET(); \ + if (!_PyErr_Occurred(tstate)) { \ + _PyErr_Format(tstate, exception, format, ##__VA_ARGS__); \ + } else { \ + _PyErr_FormatFromCause(exception, format, ##__VA_ARGS__); \ + } \ + } \ + } while (0) + +static inline size_t +get_page_size(void) { + size_t page_size = 0; + if (page_size == 0) { +#ifdef MS_WINDOWS + SYSTEM_INFO si; + GetSystemInfo(&si); + page_size = si.dwPageSize; +#else + page_size = (size_t)getpagesize(); +#endif + } + return page_size; +} + +typedef struct page_cache_entry { + uintptr_t page_addr; // page-aligned base address + char *data; + int valid; + struct page_cache_entry *next; +} page_cache_entry_t; + +#define MAX_PAGES 1024 + // Define a platform-independent process handle structure typedef struct { pid_t pid; -#ifdef MS_WINDOWS +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + mach_port_t task; +#elif defined(MS_WINDOWS) HANDLE hProcess; +#elif defined(__linux__) + int memfd; #endif + page_cache_entry_t pages[MAX_PAGES]; + Py_ssize_t page_size; } proc_handle_t; +static void +_Py_RemoteDebug_FreePageCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + PyMem_RawFree(handle->pages[i].data); + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } +} + +UNUSED static void +_Py_RemoteDebug_ClearCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].valid = 0; + } +} + +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX +static mach_port_t pid_to_task(pid_t pid); +#endif + // Initialize the process handle -static int +UNUSED static int _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->pid = pid; -#ifdef MS_WINDOWS +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + handle->task = pid_to_task(handle->pid); + if (handle->task == 0) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize macOS process handle"); + return -1; + } +#elif defined(MS_WINDOWS) handle->hProcess = OpenProcess( PROCESS_VM_READ | PROCESS_VM_WRITE | PROCESS_VM_OPERATION | PROCESS_QUERY_INFORMATION, FALSE, pid); if (handle->hProcess == NULL) { PyErr_SetFromWindowsErr(0); + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize Windows process handle"); return -1; } +#elif defined(__linux__) + handle->memfd = -1; #endif + handle->page_size = get_page_size(); + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } return 0; } // Clean up the process handle -static void +UNUSED static void _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { #ifdef MS_WINDOWS if (handle->hProcess != NULL) { CloseHandle(handle->hProcess); handle->hProcess = NULL; } +#elif defined(__linux__) + if (handle->memfd != -1) { + close(handle->memfd); + handle->memfd = -1; + } #endif handle->pid = 0; + _Py_RemoteDebug_FreePageCache(handle); } -#if defined(__APPLE__) && TARGET_OS_OSX +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX static uintptr_t return_section_address64( @@ -148,8 +244,10 @@ return_section_address64( &object_name ); if (ret != KERN_SUCCESS) { - PyErr_SetString( - PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + PyErr_Format(PyExc_RuntimeError, + "mach_vm_region failed while parsing 64-bit Mach-O binary " + "at base address 0x%lx (kern_return_t: %d)", + base, ret); return 0; } } @@ -169,9 +267,6 @@ return_section_address64( cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); } - // We should not be here, but if we are there, we should say about this - PyErr_SetString( - PyExc_RuntimeError, "Cannot find section address.\n"); return 0; } @@ -212,8 +307,10 @@ return_section_address32( &object_name ); if (ret != KERN_SUCCESS) { - PyErr_SetString( - PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + PyErr_Format(PyExc_RuntimeError, + "mach_vm_region failed while parsing 32-bit Mach-O binary " + "at base address 0x%lx (kern_return_t: %d)", + base, ret); return 0; } } @@ -233,9 +330,6 @@ return_section_address32( cmd = (struct segment_command*)((void*)cmd + cmd->cmdsize); } - // We should not be here, but if we are there, we should say about this - PyErr_SetString( - PyExc_RuntimeError, "Cannot find section address.\n"); return 0; } @@ -253,8 +347,20 @@ return_section_address_fat( int is_abi64; size_t cpu_size = sizeof(cpu), abi64_size = sizeof(is_abi64); - sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0); - sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0); + if (sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0) != 0) { + PyErr_Format(PyExc_OSError, + "Failed to determine CPU type via sysctlbyname " + "for fat binary analysis at 0x%lx: %s", + base, strerror(errno)); + return 0; + } + if (sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0) != 0) { + PyErr_Format(PyExc_OSError, + "Failed to determine CPU ABI capability via sysctlbyname " + "for fat binary analysis at 0x%lx: %s", + base, strerror(errno)); + return 0; + } cpu |= is_abi64 * CPU_ARCH_ABI64; @@ -285,13 +391,18 @@ return_section_address_fat( return return_section_address64(section, proc_ref, base, (void*)hdr); default: - PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic in fat binary.\n"); + PyErr_Format(PyExc_RuntimeError, + "Unknown Mach-O magic number 0x%x in fat binary architecture %u at base 0x%lx", + hdr->magic, i, base); return 0; } } } - PyErr_SetString(PyExc_RuntimeError, "No matching architecture found in fat binary.\n"); + PyErr_Format(PyExc_RuntimeError, + "No matching architecture found for CPU type 0x%x " + "in fat binary at base 0x%lx (%u architectures examined)", + cpu, base, nfat_arch); return 0; } @@ -300,20 +411,26 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ { int fd = open(path, O_RDONLY); if (fd == -1) { - PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); + PyErr_Format(PyExc_OSError, + "Cannot open binary file '%s' for section '%s' search: %s", + path, secname, strerror(errno)); return 0; } struct stat fs; if (fstat(fd, &fs) == -1) { - PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + PyErr_Format(PyExc_OSError, + "Cannot get file size for binary '%s' during section '%s' search: %s", + path, secname, strerror(errno)); close(fd); return 0; } void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { - PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); + PyErr_Format(PyExc_OSError, + "Cannot memory map binary file '%s' (size: %lld bytes) for section '%s' search: %s", + path, (long long)fs.st_size, secname, strerror(errno)); close(fd); return 0; } @@ -335,13 +452,22 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ result = return_section_address_fat(secname, proc_ref, base, map); break; default: - PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); + PyErr_Format(PyExc_RuntimeError, + "Unrecognized Mach-O magic number 0x%x in binary file '%s' for section '%s' search", + magic, path, secname); break; } - munmap(map, fs.st_size); + if (munmap(map, fs.st_size) != 0) { + PyErr_Format(PyExc_OSError, + "Failed to unmap binary file '%s' (size: %lld bytes): %s", + path, (long long)fs.st_size, strerror(errno)); + result = 0; + } if (close(fd) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Failed to close binary file '%s': %s", + path, strerror(errno)); result = 0; } return result; @@ -356,7 +482,10 @@ pid_to_task(pid_t pid) result = task_for_pid(mach_task_self(), pid, &task); if (result != KERN_SUCCESS) { - PyErr_Format(PyExc_PermissionError, "Cannot get task for PID %d", pid); + PyErr_Format(PyExc_PermissionError, + "Cannot get task port for PID %d (kern_return_t: %d). " + "This typically requires running as root or having the 'com.apple.system-task-ports' entitlement.", + pid, result); return 0; } return task; @@ -373,13 +502,15 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s mach_port_t proc_ref = pid_to_task(handle->pid); if (proc_ref == 0) { if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); + PyErr_Format(PyExc_PermissionError, + "Cannot get task port for PID %d during section search", + handle->pid); } return 0; } - int match_found = 0; char map_filename[MAXPATHLEN + 1]; + while (mach_vm_region( proc_ref, &address, @@ -389,6 +520,7 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s &count, &object_name) == KERN_SUCCESS) { + if ((region_info.protection & VM_PROT_READ) == 0 || (region_info.protection & VM_PROT_EXECUTE) == 0) { address += size; @@ -409,21 +541,21 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s filename = map_filename; // No path, use the whole string } - if (!match_found && strncmp(filename, substr, strlen(substr)) == 0) { - match_found = 1; - return search_section_in_file( + if (strncmp(filename, substr, strlen(substr)) == 0) { + uintptr_t result = search_section_in_file( secname, map_filename, address, size, proc_ref); + if (result != 0) { + return result; + } } address += size; } - PyErr_SetString(PyExc_RuntimeError, - "mach_vm_region failed to find the section"); return 0; } -#endif // (__APPLE__ && TARGET_OS_OSX) +#endif // (__APPLE__ && defined(TARGET_OS_OSX) && TARGET_OS_OSX) #if defined(__linux__) && HAVE_PROCESS_VM_READV static uintptr_t @@ -442,24 +574,38 @@ search_elf_file_for_section( int fd = open(elf_file, O_RDONLY); if (fd < 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Cannot open ELF file '%s' for section '%s' search: %s", + elf_file, secname, strerror(errno)); goto exit; } struct stat file_stats; if (fstat(fd, &file_stats) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Cannot get file size for ELF file '%s' during section '%s' search: %s", + elf_file, secname, strerror(errno)); goto exit; } file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (file_memory == MAP_FAILED) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Cannot memory map ELF file '%s' (size: %lld bytes) for section '%s' search: %s", + elf_file, (long long)file_stats.st_size, secname, strerror(errno)); goto exit; } Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; + // Validate ELF header + if (elf_header->e_shstrndx >= elf_header->e_shnum) { + PyErr_Format(PyExc_RuntimeError, + "Invalid ELF file '%s': string table index %u >= section count %u", + elf_file, elf_header->e_shstrndx, elf_header->e_shnum); + goto exit; + } + Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); Elf_Shdr* shstrtab_section = &section_header_table[elf_header->e_shstrndx]; @@ -476,6 +622,10 @@ search_elf_file_for_section( } } + if (section == NULL) { + goto exit; + } + Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); // Find the first PT_LOAD segment Elf_Phdr* first_load_segment = NULL; @@ -486,18 +636,25 @@ search_elf_file_for_section( } } - if (section != NULL && first_load_segment != NULL) { - uintptr_t elf_load_addr = first_load_segment->p_vaddr - - (first_load_segment->p_vaddr % first_load_segment->p_align); - result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; + if (first_load_segment == NULL) { + PyErr_Format(PyExc_RuntimeError, + "No PT_LOAD segment found in ELF file '%s' (%u program headers examined)", + elf_file, elf_header->e_phnum); + goto exit; } + uintptr_t elf_load_addr = first_load_segment->p_vaddr + - (first_load_segment->p_vaddr % first_load_segment->p_align); + result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; + exit: if (file_memory != NULL) { munmap(file_memory, file_stats.st_size); } if (fd >= 0 && close(fd) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Failed to close ELF file '%s': %s", + elf_file, strerror(errno)); result = 0; } return result; @@ -511,7 +668,9 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c FILE* maps_file = fopen(maps_file_path, "r"); if (maps_file == NULL) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Cannot open process memory map file '%s' for PID %d section search: %s", + maps_file_path, handle->pid, strerror(errno)); return 0; } @@ -520,11 +679,14 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c char *line = PyMem_Malloc(linesz); if (!line) { fclose(maps_file); - PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate memory for reading process map file '%s'", + maps_file_path); return 0; } uintptr_t retval = 0; + while (fgets(line + linelen, linesz - linelen, maps_file) != NULL) { linelen = strlen(line); if (line[linelen - 1] != '\n') { @@ -535,7 +697,9 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c if (!biggerline) { PyMem_Free(line); fclose(maps_file); - PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot reallocate memory while reading process map file '%s' (attempted size: %zu)", + maps_file_path, linesz); return 0; } line = biggerline; @@ -575,7 +739,9 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c PyMem_Free(line); if (fclose(maps_file) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Failed to close process map file '%s': %s", + maps_file_path, strerror(errno)); retval = 0; } @@ -587,15 +753,32 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c #ifdef MS_WINDOWS +static int is_process_alive(HANDLE hProcess) { + DWORD exitCode; + if (GetExitCodeProcess(hProcess, &exitCode)) { + return exitCode == STILL_ACTIVE; + } + return 0; +} + static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* secname) { HANDLE hFile = CreateFileW(mod_path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) { PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_Format(PyExc_OSError, + "Cannot open PE file for section '%s' analysis (error %lu)", + secname, error); return NULL; } + HANDLE hMap = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, 0); if (!hMap) { PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_Format(PyExc_OSError, + "Cannot create file mapping for PE file section '%s' analysis (error %lu)", + secname, error); CloseHandle(hFile); return NULL; } @@ -603,6 +786,10 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* BYTE* mapView = (BYTE*)MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0); if (!mapView) { PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_Format(PyExc_OSError, + "Cannot map view of PE file for section '%s' analysis (error %lu)", + secname, error); CloseHandle(hMap); CloseHandle(hFile); return NULL; @@ -610,7 +797,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* IMAGE_DOS_HEADER* pDOSHeader = (IMAGE_DOS_HEADER*)mapView; if (pDOSHeader->e_magic != IMAGE_DOS_SIGNATURE) { - PyErr_SetString(PyExc_RuntimeError, "Invalid DOS signature."); + PyErr_Format(PyExc_RuntimeError, + "Invalid DOS signature (0x%x) in PE file for section '%s' analysis (expected 0x%x)", + pDOSHeader->e_magic, secname, IMAGE_DOS_SIGNATURE); UnmapViewOfFile(mapView); CloseHandle(hMap); CloseHandle(hFile); @@ -619,7 +808,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* IMAGE_NT_HEADERS* pNTHeaders = (IMAGE_NT_HEADERS*)(mapView + pDOSHeader->e_lfanew); if (pNTHeaders->Signature != IMAGE_NT_SIGNATURE) { - PyErr_SetString(PyExc_RuntimeError, "Invalid NT signature."); + PyErr_Format(PyExc_RuntimeError, + "Invalid NT signature (0x%lx) in PE file for section '%s' analysis (expected 0x%lx)", + pNTHeaders->Signature, secname, IMAGE_NT_SIGNATURE); UnmapViewOfFile(mapView); CloseHandle(hMap); CloseHandle(hFile); @@ -653,7 +844,12 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const } while (hProcSnap == INVALID_HANDLE_VALUE && GetLastError() == ERROR_BAD_LENGTH); if (hProcSnap == INVALID_HANDLE_VALUE) { - PyErr_SetString(PyExc_PermissionError, "Unable to create module snapshot. Check permissions or PID."); + PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_Format(PyExc_PermissionError, + "Unable to create module snapshot for PID %d section '%s' " + "search (error %lu). Check permissions or PID validity", + handle->pid, secname, error); return 0; } @@ -672,13 +868,14 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const } CloseHandle(hProcSnap); + return (uintptr_t)runtime_addr; } #endif // MS_WINDOWS // Get the PyRuntime section address for any platform -static uintptr_t +UNUSED static uintptr_t _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) { uintptr_t address; @@ -689,33 +886,104 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the PyRuntime section in the process."); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d on Windows platform", + handle->pid); _PyErr_ChainExceptions1(exc); } -#elif defined(__linux__) +#elif defined(__linux__) && HAVE_PROCESS_VM_READV // On Linux, search for 'python' in executable or DLL address = search_linux_map_for_section(handle, "PyRuntime", "python"); if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the PyRuntime section in the process."); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d on Linux platform", + handle->pid); _PyErr_ChainExceptions1(exc); } -#elif defined(__APPLE__) && TARGET_OS_OSX +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "PyRuntime", "libpython"); - if (address == 0) { - // TODO: Differentiate between not found and error + const char* candidates[] = {"libpython", "python", "Python", NULL}; + for (const char** candidate = candidates; *candidate; candidate++) { PyErr_Clear(); - address = search_map_for_section(handle, "PyRuntime", "python"); + address = search_map_for_section(handle, "PyRuntime", *candidate); + if (address != 0) { + break; + } + } + if (address == 0) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d " + "on macOS platform (tried both libpython and python)", + handle->pid); + _PyErr_ChainExceptions1(exc); } #else - Py_UNREACHABLE(); + _set_debug_exception_cause(PyExc_RuntimeError, + "Reading the PyRuntime section is not supported on this platform"); + return 0; #endif return address; } +#if defined(__linux__) && HAVE_PROCESS_VM_READV + +static int +open_proc_mem_fd(proc_handle_t *handle) +{ + char mem_file_path[64]; + sprintf(mem_file_path, "/proc/%d/mem", handle->pid); + + handle->memfd = open(mem_file_path, O_RDWR); + if (handle->memfd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "failed to open file %s: %s", mem_file_path, strerror(errno)); + return -1; + } + return 0; +} + +// Why is pwritev not guarded? Except on Android API level 23 (no longer +// supported), HAVE_PROCESS_VM_READV is sufficient. +static int +read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t read_bytes = 0; + + do { + local[0].iov_base = (char*)dst + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + read_bytes = preadv(handle->memfd, local, 1, offset); + if (read_bytes < 0) { + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "preadv failed for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes): %s", + handle->pid, remote_address + result, len - result, result, strerror(errno)); + return -1; + } + + result += read_bytes; + } while ((size_t)read_bytes != local[0].iov_len); + return 0; +} + +#endif // __linux__ + // Platform-independent memory read function static int _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) @@ -725,13 +993,28 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address SIZE_T result = 0; do { if (!ReadProcessMemory(handle->hProcess, (LPCVOID)(remote_address + result), (char*)dst + result, len - result, &read_bytes)) { + // Check if the process is still alive: we need to be able to tell our caller + // that the process is dead and not just that the read failed. + if (!is_process_alive(handle->hProcess)) { + _set_errno(ESRCH); + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + _set_debug_exception_cause(PyExc_OSError, + "ReadProcessMemory failed for PID %d at address 0x%lx " + "(size %zu, partial read %zu bytes): Windows error %lu", + handle->pid, remote_address + result, len - result, result, error); return -1; } result += read_bytes; } while (result < len); return 0; #elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return read_remote_memory_fallback(handle, remote_address, len, dst); + } struct iovec local[1]; struct iovec remote[1]; Py_ssize_t result = 0; @@ -745,32 +1028,71 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address read_bytes = process_vm_readv(handle->pid, local, 1, remote, 1, 0); if (read_bytes < 0) { + if (errno == ENOSYS) { + return read_remote_memory_fallback(handle, remote_address, len, dst); + } PyErr_SetFromErrno(PyExc_OSError); + if (errno == ESRCH) { + return -1; + } + _set_debug_exception_cause(PyExc_OSError, + "process_vm_readv failed for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes): %s", + handle->pid, remote_address + result, len - result, result, strerror(errno)); return -1; } result += read_bytes; } while ((size_t)read_bytes != local[0].iov_len); return 0; -#elif defined(__APPLE__) && TARGET_OS_OSX +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX Py_ssize_t result = -1; kern_return_t kr = mach_vm_read_overwrite( - pid_to_task(handle->pid), + handle->task, (mach_vm_address_t)remote_address, len, (mach_vm_address_t)dst, (mach_vm_size_t*)&result); if (kr != KERN_SUCCESS) { - switch (kr) { + switch (err_get_code(kr)) { case KERN_PROTECTION_FAILURE: - PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + PyErr_Format(PyExc_PermissionError, + "Memory protection failure reading from PID %d at address " + "0x%lx (size %zu): insufficient permissions", + handle->pid, remote_address, len); break; - case KERN_INVALID_ARGUMENT: - PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + case KERN_INVALID_ARGUMENT: { + // Perform a task_info check to see if the invalid argument is due + // to the process being terminated + task_basic_info_data_t task_basic_info; + mach_msg_type_number_t task_info_count = TASK_BASIC_INFO_COUNT; + kern_return_t task_valid_check = task_info(handle->task, TASK_BASIC_INFO, + (task_info_t)&task_basic_info, + &task_info_count); + if (task_valid_check == KERN_INVALID_ARGUMENT) { + PyErr_Format(PyExc_ProcessLookupError, + "Process %d is no longer accessible (process terminated)", + handle->pid); + } else { + PyErr_Format(PyExc_ValueError, + "Invalid argument to mach_vm_read_overwrite for PID %d at " + "address 0x%lx (size %zu) - check memory permissions", + handle->pid, remote_address, len); + } + break; + } + case KERN_NO_SPACE: + case KERN_MEMORY_ERROR: + PyErr_Format(PyExc_ProcessLookupError, + "Process %d memory space no longer available (process terminated)", + handle->pid); break; default: - PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + PyErr_Format(PyExc_RuntimeError, + "mach_vm_read_overwrite failed for PID %d at address 0x%lx " + "(size %zu): kern_return_t %d", + handle->pid, remote_address, len, kr); } return -1; } @@ -780,7 +1102,172 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address #endif } +#if defined(__linux__) && HAVE_PROCESS_VM_READV +// Fallback write using /proc/pid/mem static int +_Py_RemoteDebug_WriteRemoteMemoryFallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t written = 0; + + do { + local[0].iov_base = (char*)src + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + written = pwritev(handle->memfd, local, 1, offset); + if (written < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += written; + } while ((size_t)written != local[0].iov_len); + return 0; +} +#endif // __linux__ + +// Platform-independent memory write function +UNUSED static int +_Py_RemoteDebug_WriteRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) +{ +#ifdef MS_WINDOWS + SIZE_T written = 0; + SIZE_T result = 0; + do { + if (!WriteProcessMemory(handle->hProcess, (LPVOID)(remote_address + result), (const char*)src + result, len - result, &written)) { + PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + _set_debug_exception_cause(PyExc_OSError, + "WriteProcessMemory failed for PID %d at address 0x%lx " + "(size %zu, partial write %zu bytes): Windows error %lu", + handle->pid, remote_address + result, len - result, result, error); + return -1; + } + result += written; + } while (result < len); + return 0; +#elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return _Py_RemoteDebug_WriteRemoteMemoryFallback(handle, remote_address, len, src); + } + struct iovec local[1]; + struct iovec remote[1]; + Py_ssize_t result = 0; + Py_ssize_t written = 0; + + do { + local[0].iov_base = (void*)((char*)src + result); + local[0].iov_len = len - result; + remote[0].iov_base = (void*)((char*)remote_address + result); + remote[0].iov_len = len - result; + + written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); + if (written < 0) { + if (errno == ENOSYS) { + return _Py_RemoteDebug_WriteRemoteMemoryFallback(handle, remote_address, len, src); + } + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "process_vm_writev failed for PID %d at address 0x%lx " + "(size %zu, partial write %zd bytes): %s", + handle->pid, remote_address + result, len - result, result, strerror(errno)); + return -1; + } + + result += written; + } while ((size_t)written != local[0].iov_len); + return 0; +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + kern_return_t kr = mach_vm_write( + handle->task, + (mach_vm_address_t)remote_address, + (vm_offset_t)src, + (mach_msg_type_number_t)len); + + if (kr != KERN_SUCCESS) { + switch (kr) { + case KERN_PROTECTION_FAILURE: + PyErr_SetString(PyExc_PermissionError, "Not enough permissions to write memory"); + break; + case KERN_INVALID_ARGUMENT: + PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_write"); + break; + default: + PyErr_Format(PyExc_RuntimeError, "Unknown error writing memory: %d", (int)kr); + } + return -1; + } + return 0; +#else + Py_UNREACHABLE(); +#endif +} + +UNUSED static int +_Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, + uintptr_t addr, + size_t size, + void *out) +{ + size_t page_size = handle->page_size; + uintptr_t page_base = addr & ~(page_size - 1); + size_t offset_in_page = addr - page_base; + + if (offset_in_page + size > page_size) { + return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); + } + + // Search for valid cached page + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (entry->valid && entry->page_addr == page_base) { + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + + // Find reusable slot + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (!entry->valid) { + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); + if (entry->data == NULL) { + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate %zu bytes for page cache entry " + "during read from PID %d at address 0x%lx", + page_size, handle->pid, addr); + return -1; + } + } + + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact ammount as a fallback + PyErr_Clear(); + goto fallback; + } + + entry->page_addr = page_base; + entry->valid = 1; + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + +fallback: + // Cache full — fallback to uncached read + return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); +} + +UNUSED static int _Py_RemoteDebug_ReadDebugOffsets( proc_handle_t *handle, uintptr_t *runtime_start_address, @@ -789,13 +1276,16 @@ _Py_RemoteDebug_ReadDebugOffsets( *runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle); if (!*runtime_start_address) { if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get PyRuntime address"); + PyErr_Format(PyExc_RuntimeError, + "Failed to locate PyRuntime address for PID %d", + handle->pid); } + _set_debug_exception_cause(PyExc_RuntimeError, "PyRuntime address lookup failed during debug offsets initialization"); return -1; } size_t size = sizeof(struct _Py_DebugOffsets); if (0 != _Py_RemoteDebug_ReadRemoteMemory(handle, *runtime_start_address, size, debug_offsets)) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to read debug offsets structure from remote process"); return -1; } return 0; diff --git a/Python/remote_debugging.c b/Python/remote_debugging.c index dd55b7812d4..5b50b95db94 100644 --- a/Python/remote_debugging.c +++ b/Python/remote_debugging.c @@ -19,70 +19,16 @@ cleanup_proc_handle(proc_handle_t *handle) { } static int -read_memory(proc_handle_t *handle, uint64_t remote_address, size_t len, void* dst) +read_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) { return _Py_RemoteDebug_ReadRemoteMemory(handle, remote_address, len, dst); } +// Use the shared write function from remote_debug.h static int write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { -#ifdef MS_WINDOWS - SIZE_T written = 0; - SIZE_T result = 0; - do { - if (!WriteProcessMemory(handle->hProcess, (LPVOID)(remote_address + result), (const char*)src + result, len - result, &written)) { - PyErr_SetFromWindowsErr(0); - return -1; - } - result += written; - } while (result < len); - return 0; -#elif defined(__linux__) && HAVE_PROCESS_VM_READV - struct iovec local[1]; - struct iovec remote[1]; - Py_ssize_t result = 0; - Py_ssize_t written = 0; - - do { - local[0].iov_base = (void*)((char*)src + result); - local[0].iov_len = len - result; - remote[0].iov_base = (void*)((char*)remote_address + result); - remote[0].iov_len = len - result; - - written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); - if (written < 0) { - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - - result += written; - } while ((size_t)written != local[0].iov_len); - return 0; -#elif defined(__APPLE__) && TARGET_OS_OSX - kern_return_t kr = mach_vm_write( - pid_to_task(handle->pid), - (mach_vm_address_t)remote_address, - (vm_offset_t)src, - (mach_msg_type_number_t)len); - - if (kr != KERN_SUCCESS) { - switch (kr) { - case KERN_PROTECTION_FAILURE: - PyErr_SetString(PyExc_PermissionError, "Not enough permissions to write memory"); - break; - case KERN_INVALID_ARGUMENT: - PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_write"); - break; - default: - PyErr_Format(PyExc_RuntimeError, "Unknown error writing memory: %d", (int)kr); - } - return -1; - } - return 0; -#else - Py_UNREACHABLE(); -#endif + return _Py_RemoteDebug_WriteRemoteMemory(handle, remote_address, len, src); } static int @@ -196,7 +142,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc int is_remote_debugging_enabled = 0; if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.debugger_support.remote_debugging_enabled, + interpreter_state_addr + (uintptr_t)debug_offsets.debugger_support.remote_debugging_enabled, sizeof(int), &is_remote_debugging_enabled)) { @@ -216,7 +162,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc if (tid != 0) { if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.interpreter_state.threads_head, + interpreter_state_addr + (uintptr_t)debug_offsets.interpreter_state.threads_head, sizeof(void*), &thread_state_addr)) { @@ -225,7 +171,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc while (thread_state_addr != 0) { if (0 != read_memory( handle, - thread_state_addr + debug_offsets.thread_state.native_thread_id, + thread_state_addr + (uintptr_t)debug_offsets.thread_state.native_thread_id, sizeof(this_tid), &this_tid)) { @@ -238,7 +184,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc if (0 != read_memory( handle, - thread_state_addr + debug_offsets.thread_state.next, + thread_state_addr + (uintptr_t)debug_offsets.thread_state.next, sizeof(void*), &thread_state_addr)) { @@ -255,7 +201,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc } else { if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.interpreter_state.threads_main, + interpreter_state_addr + (uintptr_t)debug_offsets.interpreter_state.threads_main, sizeof(void*), &thread_state_addr)) { @@ -307,7 +253,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc uintptr_t eval_breaker; if (0 != read_memory( handle, - thread_state_addr + debug_offsets.debugger_support.eval_breaker, + thread_state_addr + (uintptr_t)debug_offsets.debugger_support.eval_breaker, sizeof(uintptr_t), &eval_breaker)) { diff --git a/Python/specialize.c b/Python/specialize.c index bbe725c8ec8..19433bc7a74 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -22,436 +22,23 @@ #include <stdlib.h> // rand() -extern const char *_PyUOpName(int index); - /* For guidance on adding or extending families of instructions see * InternalDocs/interpreter.md `Specialization` section. */ -#ifdef Py_STATS -GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; -static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; -PyStats *_Py_stats = NULL; - -#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID -#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" -#endif - -#define ADD_STAT_TO_DICT(res, field) \ - do { \ - PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \ - if (val == NULL) { \ - Py_DECREF(res); \ - return NULL; \ - } \ - if (PyDict_SetItemString(res, #field, val) == -1) { \ - Py_DECREF(res); \ - Py_DECREF(val); \ - return NULL; \ - } \ - Py_DECREF(val); \ - } while(0); - -static PyObject* -stats_to_dict(SpecializationStats *stats) -{ - PyObject *res = PyDict_New(); - if (res == NULL) { - return NULL; - } - ADD_STAT_TO_DICT(res, success); - ADD_STAT_TO_DICT(res, failure); - ADD_STAT_TO_DICT(res, hit); - ADD_STAT_TO_DICT(res, deferred); - ADD_STAT_TO_DICT(res, miss); - ADD_STAT_TO_DICT(res, deopt); - PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS); - if (failure_kinds == NULL) { - Py_DECREF(res); - return NULL; - } - for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) { - PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]); - if (stat == NULL) { - Py_DECREF(res); - Py_DECREF(failure_kinds); - return NULL; - } - PyTuple_SET_ITEM(failure_kinds, i, stat); - } - if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) { - Py_DECREF(res); - Py_DECREF(failure_kinds); - return NULL; - } - Py_DECREF(failure_kinds); - return res; -} -#undef ADD_STAT_TO_DICT - -static int -add_stat_dict( - PyObject *res, - int opcode, - const char *name) { - - SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization; - PyObject *d = stats_to_dict(stats); - if (d == NULL) { - return -1; - } - int err = PyDict_SetItemString(res, name, d); - Py_DECREF(d); - return err; -} - -PyObject* -_Py_GetSpecializationStats(void) { - PyObject *stats = PyDict_New(); - if (stats == NULL) { - return NULL; - } - int err = 0; - err += add_stat_dict(stats, CONTAINS_OP, "contains_op"); - err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr"); - err += add_stat_dict(stats, LOAD_ATTR, "load_attr"); - err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); - err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); - err += add_stat_dict(stats, STORE_ATTR, "store_attr"); - err += add_stat_dict(stats, CALL, "call"); - err += add_stat_dict(stats, CALL_KW, "call_kw"); - err += add_stat_dict(stats, BINARY_OP, "binary_op"); - err += add_stat_dict(stats, COMPARE_OP, "compare_op"); - err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence"); - err += add_stat_dict(stats, FOR_ITER, "for_iter"); - err += add_stat_dict(stats, TO_BOOL, "to_bool"); - err += add_stat_dict(stats, SEND, "send"); - if (err < 0) { - Py_DECREF(stats); - return NULL; - } - return stats; -} - - -#define PRINT_STAT(i, field) \ - if (stats[i].field) { \ - fprintf(out, " opcode[%s]." #field " : %" PRIu64 "\n", _PyOpcode_OpName[i], stats[i].field); \ - } - -static void -print_spec_stats(FILE *out, OpcodeStats *stats) -{ - /* Mark some opcodes as specializable for stats, - * even though we don't specialize them yet. */ - fprintf(out, "opcode[BINARY_SLICE].specializable : 1\n"); - fprintf(out, "opcode[STORE_SLICE].specializable : 1\n"); - fprintf(out, "opcode[GET_ITER].specializable : 1\n"); - for (int i = 0; i < 256; i++) { - if (_PyOpcode_Caches[i]) { - /* Ignore jumps as they cannot be specialized */ - switch (i) { - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case JUMP_BACKWARD: - break; - default: - fprintf(out, "opcode[%s].specializable : 1\n", _PyOpcode_OpName[i]); - } - } - PRINT_STAT(i, specialization.success); - PRINT_STAT(i, specialization.failure); - PRINT_STAT(i, specialization.hit); - PRINT_STAT(i, specialization.deferred); - PRINT_STAT(i, specialization.miss); - PRINT_STAT(i, specialization.deopt); - PRINT_STAT(i, execution_count); - for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) { - uint64_t val = stats[i].specialization.failure_kinds[j]; - if (val) { - fprintf(out, " opcode[%s].specialization.failure_kinds[%d] : %" - PRIu64 "\n", _PyOpcode_OpName[i], j, val); - } - } - for (int j = 0; j < 256; j++) { - if (stats[i].pair_count[j]) { - fprintf(out, "opcode[%s].pair_count[%s] : %" PRIu64 "\n", - _PyOpcode_OpName[i], _PyOpcode_OpName[j], stats[i].pair_count[j]); - } - } - } -} -#undef PRINT_STAT - - -static void -print_call_stats(FILE *out, CallStats *stats) -{ - fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls); - fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls); - fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed); - fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created); - for (int i = 0; i < EVAL_CALL_KINDS; i++) { - fprintf(out, "Calls via PyEval_EvalFrame[%d] : %" PRIu64 "\n", i, stats->eval_calls[i]); - } -} - -static void -print_object_stats(FILE *out, ObjectStats *stats) -{ - fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist); - fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist); - fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); - fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512); - fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k); - fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big); - fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); - fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values); - fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs); - fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs); - fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs); - fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs); - fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs); - fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs); - fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs); - fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs); - fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); - fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); - fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); - fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); - fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); - fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); - fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); - fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); - fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); -} - -static void -print_gc_stats(FILE *out, GCStats *stats) -{ - for (int i = 0; i < NUM_GENERATIONS; i++) { - fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); - fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); - fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); - fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); - fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); - } -} - -#ifdef _Py_TIER2 -static void -print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) -{ - for (int i = 0; i < _Py_UOP_HIST_SIZE; i++) { - fprintf(out, "%s[%" PRIu64"]: %" PRIu64 "\n", name, (uint64_t)1 << i, hist[i]); - } -} - -static void -print_optimization_stats(FILE *out, OptimizationStats *stats) -{ - fprintf(out, "Optimization attempts: %" PRIu64 "\n", stats->attempts); - fprintf(out, "Optimization traces created: %" PRIu64 "\n", stats->traces_created); - fprintf(out, "Optimization traces executed: %" PRIu64 "\n", stats->traces_executed); - fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->uops_executed); - fprintf(out, "Optimization trace stack overflow: %" PRIu64 "\n", stats->trace_stack_overflow); - fprintf(out, "Optimization trace stack underflow: %" PRIu64 "\n", stats->trace_stack_underflow); - fprintf(out, "Optimization trace too long: %" PRIu64 "\n", stats->trace_too_long); - fprintf(out, "Optimization trace too short: %" PRIu64 "\n", stats->trace_too_short); - fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop); - fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call); - fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); - fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee); - fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); - - print_histogram(out, "Trace length", stats->trace_length_hist); - print_histogram(out, "Trace run length", stats->trace_run_length_hist); - print_histogram(out, "Optimized trace length", stats->optimized_trace_length_hist); - - fprintf(out, "Optimization optimizer attempts: %" PRIu64 "\n", stats->optimizer_attempts); - fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); - fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", - stats->optimizer_failure_reason_no_memory); - fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); - fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); - for (int i = 0; i <= MAX_UOP_ID; i++) { - if (stats->opcode[i].execution_count) { - fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); - } - if (stats->opcode[i].miss) { - fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); - } - } - for (int i = 0; i < 256; i++) { - if (stats->unsupported_opcode[i]) { - fprintf( - out, - "unsupported_opcode[%s].count : %" PRIu64 "\n", - _PyOpcode_OpName[i], - stats->unsupported_opcode[i] - ); - } - } - - for (int i = 1; i <= MAX_UOP_ID; i++){ - for (int j = 1; j <= MAX_UOP_ID; j++) { - if (stats->opcode[i].pair_count[j]) { - fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", - _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); - } - } - } - for (int i = 0; i < MAX_UOP_ID; i++) { - if (stats->error_in_opcode[i]) { - fprintf( - out, - "error_in_opcode[%s].count : %" PRIu64 "\n", - _PyUOpName(i), - stats->error_in_opcode[i] - ); - } - } - fprintf(out, "JIT total memory size: %" PRIu64 "\n", stats->jit_total_memory_size); - fprintf(out, "JIT code size: %" PRIu64 "\n", stats->jit_code_size); - fprintf(out, "JIT trampoline size: %" PRIu64 "\n", stats->jit_trampoline_size); - fprintf(out, "JIT data size: %" PRIu64 "\n", stats->jit_data_size); - fprintf(out, "JIT padding size: %" PRIu64 "\n", stats->jit_padding_size); - fprintf(out, "JIT freed memory size: %" PRIu64 "\n", stats->jit_freed_memory_size); - - print_histogram(out, "Trace total memory size", stats->trace_total_memory_hist); -} -#endif - -static void -print_rare_event_stats(FILE *out, RareEventStats *stats) -{ - fprintf(out, "Rare event (set_class): %" PRIu64 "\n", stats->set_class); - fprintf(out, "Rare event (set_bases): %" PRIu64 "\n", stats->set_bases); - fprintf(out, "Rare event (set_eval_frame_func): %" PRIu64 "\n", stats->set_eval_frame_func); - fprintf(out, "Rare event (builtin_dict): %" PRIu64 "\n", stats->builtin_dict); - fprintf(out, "Rare event (func_modification): %" PRIu64 "\n", stats->func_modification); - fprintf(out, "Rare event (watched_dict_modification): %" PRIu64 "\n", stats->watched_dict_modification); - fprintf(out, "Rare event (watched_globals_modification): %" PRIu64 "\n", stats->watched_globals_modification); -} - -static void -print_stats(FILE *out, PyStats *stats) -{ - print_spec_stats(out, stats->opcode_stats); - print_call_stats(out, &stats->call_stats); - print_object_stats(out, &stats->object_stats); - print_gc_stats(out, stats->gc_stats); -#ifdef _Py_TIER2 - print_optimization_stats(out, &stats->optimization_stats); -#endif - print_rare_event_stats(out, &stats->rare_event_stats); -} - -void -_Py_StatsOn(void) -{ - _Py_stats = &_Py_stats_struct; -} - -void -_Py_StatsOff(void) -{ - _Py_stats = NULL; -} - -void -_Py_StatsClear(void) -{ - memset(&_py_gc_stats, 0, sizeof(_py_gc_stats)); - memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct)); - _Py_stats_struct.gc_stats = _py_gc_stats; -} - -static int -mem_is_zero(unsigned char *ptr, size_t size) -{ - for (size_t i=0; i < size; i++) { - if (*ptr != 0) { - return 0; - } - ptr++; - } - return 1; -} - -int -_Py_PrintSpecializationStats(int to_file) -{ - PyStats *stats = &_Py_stats_struct; -#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA))) - int is_zero = ( - MEM_IS_ZERO(stats->gc_stats) // is a pointer - && MEM_IS_ZERO(&stats->opcode_stats) - && MEM_IS_ZERO(&stats->call_stats) - && MEM_IS_ZERO(&stats->object_stats) - ); -#undef MEM_IS_ZERO - if (is_zero) { - // gh-108753: -X pystats command line was used, but then _stats_off() - // and _stats_clear() have been called: in this case, avoid printing - // useless "all zeros" statistics. - return 0; - } - - FILE *out = stderr; - if (to_file) { - /* Write to a file instead of stderr. */ -# ifdef MS_WINDOWS - const char *dirname = "c:\\temp\\py_stats\\"; -# else - const char *dirname = "/tmp/py_stats/"; -# endif - /* Use random 160 bit number as file name, - * to avoid both accidental collisions and - * symlink attacks. */ - unsigned char rand[20]; - char hex_name[41]; - _PyOS_URandomNonblock(rand, 20); - for (int i = 0; i < 20; i++) { - hex_name[2*i] = Py_hexdigits[rand[i]&15]; - hex_name[2*i+1] = Py_hexdigits[(rand[i]>>4)&15]; - } - hex_name[40] = '\0'; - char buf[64]; - assert(strlen(dirname) + 40 + strlen(".txt") < 64); - sprintf(buf, "%s%s.txt", dirname, hex_name); - FILE *fout = fopen(buf, "w"); - if (fout) { - out = fout; - } - } - else { - fprintf(out, "Specialization stats:\n"); - } - print_stats(out, stats); - if (out != stderr) { - fclose(out); - } - return 1; -} - +#if Py_STATS #define SPECIALIZATION_FAIL(opcode, kind) \ do { \ - if (_Py_stats) { \ + PyStats *s = _PyStats_GET(); \ + if (s) { \ int _kind = (kind); \ assert(_kind < SPECIALIZATION_FAILURE_KINDS); \ - _Py_stats->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \ + s->opcode_stats[opcode].specialization.failure_kinds[_kind]++; \ } \ } while (0) - -#endif // Py_STATS - - -#ifndef SPECIALIZATION_FAIL +#else # define SPECIALIZATION_FAIL(opcode, kind) ((void)0) -#endif +#endif // Py_STATS // Initialize warmup counters and optimize instructions. This cannot fail. void @@ -629,6 +216,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters #define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21 #define SPEC_FAIL_CALL_PEP_523 22 #define SPEC_FAIL_CALL_BOUND_METHOD 23 +#define SPEC_FAIL_CALL_VECTORCALL 24 #define SPEC_FAIL_CALL_CLASS_MUTABLE 26 #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 @@ -934,8 +522,7 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, un PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)); has_getattr = getattr != NULL; if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && + if (!has_getattr && Py_IS_TYPE(getattribute, &PyFunction_Type)) { *descr = getattribute; *tp_version = ga_version; @@ -1258,12 +845,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { - #ifndef Py_GIL_DISABLED - // In free-threaded builds it's possible for tp_getattro to change - // after the call to analyze_descriptor. That is fine: the version - // guard will fail. - assert(type->tp_getattro == _Py_slot_tp_getattro); - #endif assert(Py_IS_TYPE(descr, &PyFunction_Type)); _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); if (!function_check_args(descr, 2, LOAD_ATTR)) { @@ -2021,8 +1602,8 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } static int -specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, - int nargs) +specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, + _Py_CODEUNIT *instr, int nargs) { switch (descr->d_method->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | @@ -2046,8 +1627,11 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - specialize(instr, CALL_LIST_APPEND); - return 0; + assert(self_or_null != NULL); + if (PyList_CheckExact(self_or_null)) { + specialize(instr, CALL_LIST_APPEND); + return 0; + } } specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; @@ -2077,6 +1661,10 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } + if (func->vectorcall != _PyFunction_Vectorcall) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL); + return -1; + } int argcount = -1; if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); @@ -2116,6 +1704,10 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } + if (func->vectorcall != _PyFunction_Vectorcall) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL); + return -1; + } if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); return -1; @@ -2158,7 +1750,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) if (nargs == 2) { /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); - if (callable == interp->callable_cache.isinstance) { + if (callable == interp->callable_cache.isinstance && instr->op.arg == 2) { specialize(instr, CALL_ISINSTANCE); return 0; } @@ -2177,7 +1769,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } Py_NO_INLINE void -_Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +_Py_Specialize_Call(_PyStackRef callable_st, _PyStackRef self_or_null_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); @@ -2195,7 +1787,9 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) fail = specialize_class_call(callable, instr, nargs); } else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { - fail = specialize_method_descriptor((PyMethodDescrObject *)callable, instr, nargs); + PyObject *self_or_null = PyStackRef_AsPyObjectBorrow(self_or_null_st); + fail = specialize_method_descriptor((PyMethodDescrObject *)callable, + self_or_null, instr, nargs); } else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; @@ -2601,7 +2195,7 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in specialize(instr, BINARY_OP_ADD_UNICODE); return; } - if (PyLong_CheckExact(lhs)) { + if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { specialize(instr, BINARY_OP_ADD_INT); return; } @@ -2615,7 +2209,7 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { break; } - if (PyLong_CheckExact(lhs)) { + if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { specialize(instr, BINARY_OP_MULTIPLY_INT); return; } @@ -2629,7 +2223,7 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { break; } - if (PyLong_CheckExact(lhs)) { + if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { specialize(instr, BINARY_OP_SUBTRACT_INT); return; } @@ -2904,53 +2498,57 @@ int #endif // Py_STATS Py_NO_INLINE void -_Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) +_Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg) { assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); PyTypeObject *tp = Py_TYPE(iter_o); + + if (PyStackRef_IsNull(null_or_index)) { #ifdef Py_GIL_DISABLED - // Only specialize for uniquely referenced iterators, so that we know - // they're only referenced by this one thread. This is more limiting - // than we need (even `it = iter(mylist); for item in it:` won't get - // specialized) but we don't have a way to check whether we're the only - // _thread_ who has access to the object. - if (!_PyObject_IsUniquelyReferenced(iter_o)) - goto failure; -#endif - if (tp == &PyListIter_Type) { -#ifdef Py_GIL_DISABLED - _PyListIterObject *it = (_PyListIterObject *)iter_o; - if (!_Py_IsOwnedByCurrentThread((PyObject *)it->it_seq) && - !_PyObject_GC_IS_SHARED(it->it_seq)) { - // Maybe this should just set GC_IS_SHARED in a critical - // section, instead of leaving it to the first iteration? + // Only specialize for uniquely referenced iterators, so that we know + // they're only referenced by this one thread. This is more limiting + // than we need (even `it = iter(mylist); for item in it:` won't get + // specialized) but we don't have a way to check whether we're the only + // _thread_ who has access to the object. + if (!_PyObject_IsUniquelyReferenced(iter_o)) { goto failure; } #endif - specialize(instr, FOR_ITER_LIST); - return; + if (tp == &PyRangeIter_Type) { + specialize(instr, FOR_ITER_RANGE); + return; + } + else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { + // Generators are very much not thread-safe, so don't worry about + // the specialization not being thread-safe. + assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || + instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR + ); + /* Don't specialize if PEP 523 is active */ + if (_PyInterpreterState_GET()->eval_frame) { + goto failure; + } + specialize(instr, FOR_ITER_GEN); + return; + } } - else if (tp == &PyTupleIter_Type) { - specialize(instr, FOR_ITER_TUPLE); - return; - } - else if (tp == &PyRangeIter_Type) { - specialize(instr, FOR_ITER_RANGE); - return; - } - else if (tp == &PyGen_Type && oparg <= SHRT_MAX) { - // Generators are very much not thread-safe, so don't worry about - // the specialization not being thread-safe. - assert(instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == END_FOR || - instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code == INSTRUMENTED_END_FOR - ); - /* Don't specialize if PEP 523 is active */ - if (_PyInterpreterState_GET()->eval_frame) - goto failure; - specialize(instr, FOR_ITER_GEN); - return; + else { + if (tp == &PyList_Type) { +#ifdef Py_GIL_DISABLED + // Only specialize for lists owned by this thread or shared + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + goto failure; + } +#endif + specialize(instr, FOR_ITER_LIST); + return; + } + else if (tp == &PyTuple_Type) { + specialize(instr, FOR_ITER_TUPLE); + return; + } } failure: SPECIALIZATION_FAIL(FOR_ITER, diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 979a6b1c628..0c13cc65510 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -1,4 +1,3 @@ - #include "Python.h" #include "pycore_object.h" @@ -20,6 +19,8 @@ typedef struct _table_entry { int linenumber; const char *filename_borrow; int linenumber_borrow; + int borrows; + _PyStackRef borrowed_from; } TableEntry; TableEntry * @@ -34,23 +35,29 @@ make_table_entry(PyObject *obj, const char *filename, int linenumber) result->filename = filename; result->linenumber = linenumber; result->filename_borrow = NULL; + result->linenumber_borrow = 0; + result->borrows = 0; + result->borrowed_from = PyStackRef_NULL; return result; } PyObject * _Py_stackref_get_object(_PyStackRef ref) { + assert(!PyStackRef_IsError(ref)); if (ref.index == 0) { return NULL; } PyInterpreterState *interp = PyInterpreterState_Get(); assert(interp != NULL); if (ref.index >= interp->next_stackref) { - _Py_FatalErrorFormat(__func__, "Garbled stack ref with ID %" PRIu64 "\n", ref.index); + _Py_FatalErrorFormat(__func__, + "Garbled stack ref with ID %" PRIu64 "\n", ref.index); } TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); if (entry == NULL) { - _Py_FatalErrorFormat(__func__, "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); + _Py_FatalErrorFormat(__func__, + "Accessing closed stack ref with ID %" PRIu64 "\n", ref.index); } return entry->obj; } @@ -64,15 +71,19 @@ PyStackRef_Is(_PyStackRef a, _PyStackRef b) PyObject * _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); PyInterpreterState *interp = PyInterpreterState_Get(); if (ref.index >= interp->next_stackref) { - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber); - + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); } PyObject *obj; if (ref.index < INITIAL_STACKREF_INDEX) { if (ref.index == 0) { - _Py_FatalErrorFormat(__func__, "Passing NULL to PyStackRef_CLOSE at %s:%d\n", filename, linenumber); + _Py_FatalErrorFormat(__func__, + "Passing NULL to _Py_stackref_close at %s:%d\n", + filename, linenumber); } // Pre-allocated reference to None, False or True -- Do not clear TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); @@ -86,10 +97,27 @@ _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) if (entry != NULL) { _Py_FatalErrorFormat(__func__, "Double close of ref ID %" PRIu64 " at %s:%d. Referred to instance of %s at %p. Closed at %s:%d\n", - (void *)ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); + ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); } #endif - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 "\n", (void *)ref.index); + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + if (!PyStackRef_IsNull(entry->borrowed_from)) { + _PyStackRef borrowed_from = entry->borrowed_from; + TableEntry *entry_borrowed = _Py_hashtable_get(interp->open_stackrefs_table, (void *)borrowed_from.index); + if (entry_borrowed == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid borrowed StackRef with ID %" PRIu64 " at %s:%d\n", + borrowed_from.index, filename, linenumber); + } + entry_borrowed->borrows--; + } + if (entry->borrows > 0) { + _Py_FatalErrorFormat(__func__, + "StackRef with ID %" PRIu64 " closed with %d borrowed refs at %s:%d. Opened at %s:%d\n", + ref.index, entry->borrows, filename, linenumber, entry->filename, entry->linenumber); } obj = entry->obj; free(entry); @@ -107,18 +135,19 @@ _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) } _PyStackRef -_Py_stackref_create(PyObject *obj, const char *filename, int linenumber) +_Py_stackref_create(PyObject *obj, uint16_t flags, const char *filename, int linenumber) { if (obj == NULL) { Py_FatalError("Cannot create a stackref for NULL"); } PyInterpreterState *interp = PyInterpreterState_Get(); uint64_t new_id = interp->next_stackref; - interp->next_stackref = new_id + 2; + interp->next_stackref = new_id + (1 << Py_TAGGED_SHIFT); TableEntry *entry = make_table_entry(obj, filename, linenumber); if (entry == NULL) { Py_FatalError("No memory left for stackref debug table"); } + new_id |= flags; if (_Py_hashtable_set(interp->open_stackrefs_table, (void *)new_id, entry) < 0) { Py_FatalError("No memory left for stackref debug table"); } @@ -128,6 +157,7 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber) void _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber) { + assert(!PyStackRef_IsError(ref)); if (ref.index < INITIAL_STACKREF_INDEX) { return; } @@ -139,19 +169,67 @@ _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber if (entry != NULL) { _Py_FatalErrorFormat(__func__, "Borrow of closed ref ID %" PRIu64 " at %s:%d. Referred to instance of %s at %p. Closed at %s:%d\n", - (void *)ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); + ref.index, filename, linenumber, entry->classname, entry->obj, entry->filename, entry->linenumber); } #endif - _Py_FatalErrorFormat(__func__, "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", (void *)ref.index, filename, linenumber); + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); } entry->filename_borrow = filename; entry->linenumber_borrow = linenumber; } +_PyStackRef +_Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + PyInterpreterState *interp = PyInterpreterState_Get(); + + TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + + return entry->borrowed_from; +} + +// This function should be used no more than once per ref. +void +_Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + PyInterpreterState *interp = PyInterpreterState_Get(); + + TableEntry *entry = _Py_hashtable_get(interp->open_stackrefs_table, (void *)ref.index); + if (entry == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef (ref) with ID %" PRIu64 " at %s:%d\n", + ref.index, filename, linenumber); + } + + assert(PyStackRef_IsNull(entry->borrowed_from)); + if (PyStackRef_IsNull(borrowed_from)) { + return; + } + + TableEntry *entry_borrowed = _Py_hashtable_get(interp->open_stackrefs_table, (void *)borrowed_from.index); + if (entry_borrowed == NULL) { + _Py_FatalErrorFormat(__func__, + "Invalid StackRef (borrowed_from) with ID %" PRIu64 " at %s:%d\n", + borrowed_from.index, filename, linenumber); + } + + entry->borrowed_from = borrowed_from; + entry_borrowed->borrows++; +} void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref) { + assert(!PyStackRef_IsError(ref)); assert(ref.index < INITIAL_STACKREF_INDEX); TableEntry *entry = make_table_entry(obj, "builtin-object", 0); if (entry == NULL) { @@ -190,16 +268,10 @@ _Py_stackref_report_leaks(PyInterpreterState *interp) } } -void -_PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber) -{ - PyObject *obj = _Py_stackref_close(ref, filename, linenumber); - _Py_DECREF_SPECIALIZED(obj, destruct); -} - _PyStackRef PyStackRef_TagInt(intptr_t i) { - return (_PyStackRef){ .index = (i << 1) + 1 }; + assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << Py_TAGGED_SHIFT), Py_TAGGED_SHIFT) == i); + return (_PyStackRef){ .index = (i << Py_TAGGED_SHIFT) | Py_INT_TAG }; } intptr_t @@ -207,7 +279,7 @@ PyStackRef_UntagInt(_PyStackRef i) { assert(PyStackRef_IsTaggedInt(i)); intptr_t val = (intptr_t)i.index; - return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 1); + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, Py_TAGGED_SHIFT); } bool @@ -216,4 +288,13 @@ PyStackRef_IsNullOrInt(_PyStackRef ref) return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); } +_PyStackRef +PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) +{ + assert(PyStackRef_IsTaggedInt(ref)); + assert((ref.index & (~Py_TAG_BITS)) != (INTPTR_MAX & (~Py_TAG_BITS))); // Isn't about to overflow + return (_PyStackRef){ .index = ref.index + (1 << Py_TAGGED_SHIFT) }; +} + + #endif diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 56e349a544c..8937e666bbb 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -51,6 +51,7 @@ static const char* _Py_stdlib_module_names[] = { "_lsprof", "_lzma", "_markupbase", +"_math_integer", "_md5", "_multibytecodec", "_multiprocessing", @@ -215,6 +216,7 @@ static const char* _Py_stdlib_module_names[] = { "posixpath", "pprint", "profile", +"profiling", "pstats", "pty", "pwd", @@ -245,9 +247,6 @@ static const char* _Py_stdlib_module_names[] = { "socket", "socketserver", "sqlite3", -"sre_compile", -"sre_constants", -"sre_parse", "ssl", "stat", "statistics", diff --git a/Python/symtable.c b/Python/symtable.c index f633e281019..29cf9190a4e 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -380,8 +380,8 @@ static void dump_symtable(PySTEntryObject* ste) } #endif -#define DUPLICATE_ARGUMENT \ -"duplicate argument '%U' in function definition" +#define DUPLICATE_PARAMETER \ +"duplicate parameter '%U' in function definition" static struct symtable * symtable_new(void) @@ -1494,7 +1494,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s } if ((flag & DEF_PARAM) && (val & DEF_PARAM)) { /* Is it better to use 'mangled' or 'name' here? */ - PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name); + PyErr_Format(PyExc_SyntaxError, DUPLICATE_PARAMETER, name); SET_ERROR_LOCATION(st->st_filename, loc); goto error; } @@ -2413,7 +2413,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); if (e->v.Lambda.args->kw_defaults) VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults); - if (!symtable_enter_block(st, &_Py_ID(lambda), + if (!symtable_enter_block(st, &_Py_STR(anon_lambda), FunctionBlock, (void *)e, LOCATION(e))) { return 0; } @@ -3055,7 +3055,7 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, static int symtable_visit_genexp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, &_Py_ID(genexpr), + return symtable_handle_comprehension(st, e, &_Py_STR(anon_genexpr), e->v.GeneratorExp.generators, e->v.GeneratorExp.elt, NULL); } @@ -3063,7 +3063,7 @@ symtable_visit_genexp(struct symtable *st, expr_ty e) static int symtable_visit_listcomp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, &_Py_ID(listcomp), + return symtable_handle_comprehension(st, e, &_Py_STR(anon_listcomp), e->v.ListComp.generators, e->v.ListComp.elt, NULL); } @@ -3071,7 +3071,7 @@ symtable_visit_listcomp(struct symtable *st, expr_ty e) static int symtable_visit_setcomp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, &_Py_ID(setcomp), + return symtable_handle_comprehension(st, e, &_Py_STR(anon_setcomp), e->v.SetComp.generators, e->v.SetComp.elt, NULL); } @@ -3079,7 +3079,7 @@ symtable_visit_setcomp(struct symtable *st, expr_ty e) static int symtable_visit_dictcomp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, &_Py_ID(dictcomp), + return symtable_handle_comprehension(st, e, &_Py_STR(anon_dictcomp), e->v.DictComp.generators, e->v.DictComp.key, e->v.DictComp.value); @@ -3137,7 +3137,7 @@ symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_Source struct symtable * _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, - int start, PyCompilerFlags *flags) + int start, PyCompilerFlags *flags, PyObject *module) { struct symtable *st; mod_ty mod; @@ -3147,7 +3147,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 00dce4527fb..b4b441bf4d9 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -76,12 +76,12 @@ module sys PyObject * -_PySys_GetRequiredAttr(PyObject *name) +PySys_GetAttr(PyObject *name) { if (!PyUnicode_Check(name)) { PyErr_Format(PyExc_TypeError, - "attribute name must be string, not '%.200s'", - Py_TYPE(name)->tp_name); + "attribute name must be string, not '%T'", + name); return NULL; } PyThreadState *tstate = _PyThreadState_GET(); @@ -98,7 +98,7 @@ _PySys_GetRequiredAttr(PyObject *name) } PyObject * -_PySys_GetRequiredAttrString(const char *name) +PySys_GetAttrString(const char *name) { PyThreadState *tstate = _PyThreadState_GET(); PyObject *sysdict = tstate->interp->sysdict; @@ -114,12 +114,12 @@ _PySys_GetRequiredAttrString(const char *name) } int -_PySys_GetOptionalAttr(PyObject *name, PyObject **value) +PySys_GetOptionalAttr(PyObject *name, PyObject **value) { if (!PyUnicode_Check(name)) { PyErr_Format(PyExc_TypeError, - "attribute name must be string, not '%.200s'", - Py_TYPE(name)->tp_name); + "attribute name must be string, not '%T'", + name); *value = NULL; return -1; } @@ -133,7 +133,7 @@ _PySys_GetOptionalAttr(PyObject *name, PyObject **value) } int -_PySys_GetOptionalAttrString(const char *name, PyObject **value) +PySys_GetOptionalAttrString(const char *name, PyObject **value) { PyThreadState *tstate = _PyThreadState_GET(); PyObject *sysdict = tstate->interp->sysdict; @@ -610,7 +610,7 @@ sys_breakpointhook(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb if (last_dot == NULL) { /* The breakpoint is a built-in, e.g. PYTHONBREAKPOINT=int */ - modulepath = PyUnicode_FromString("builtins"); + modulepath = &_Py_ID(builtins); attrname = envar; } else if (last_dot != envar) { @@ -773,7 +773,7 @@ sys_displayhook(PyObject *module, PyObject *o) } if (PyObject_SetAttr(builtins, _Py_LATIN1_CHR('_'), Py_None) != 0) return NULL; - outf = _PySys_GetRequiredAttr(&_Py_ID(stdout)); + outf = PySys_GetAttr(&_Py_ID(stdout)); if (outf == NULL) { return NULL; } @@ -1160,6 +1160,7 @@ sys_settrace(PyObject *module, PyObject *function) } /*[clinic input] +@permit_long_summary sys._settraceallthreads function as arg: object @@ -1173,7 +1174,7 @@ in the library manual. static PyObject * sys__settraceallthreads(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=161cca30207bf3ca input=d4bde1f810d73675]*/ +/*[clinic end generated code: output=161cca30207bf3ca input=e5750f5dc01142eb]*/ { PyObject* argument = NULL; Py_tracefunc func = NULL; @@ -1183,9 +1184,10 @@ sys__settraceallthreads(PyObject *module, PyObject *arg) argument = arg; } - - PyEval_SetTraceAllThreads(func, argument); - + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetTraceAllThreads(interp, func, argument) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -1240,6 +1242,7 @@ sys_setprofile(PyObject *module, PyObject *function) } /*[clinic input] +@permit_long_summary sys._setprofileallthreads function as arg: object @@ -1253,7 +1256,7 @@ chapter in the library manual. static PyObject * sys__setprofileallthreads(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=2d61319e27b309fe input=a10589439ba20cee]*/ +/*[clinic end generated code: output=2d61319e27b309fe input=9a3dc3352c63b471]*/ { PyObject* argument = NULL; Py_tracefunc func = NULL; @@ -1263,8 +1266,10 @@ sys__setprofileallthreads(PyObject *module, PyObject *arg) argument = arg; } - PyEval_SetProfileAllThreads(func, argument); - + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetProfileAllThreads(interp, func, argument) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -1582,10 +1587,10 @@ get_hash_info(PyThreadState *tstate) } while(0) SET_HASH_INFO_ITEM(PyLong_FromLong(8 * sizeof(Py_hash_t))); - SET_HASH_INFO_ITEM(PyLong_FromSsize_t(_PyHASH_MODULUS)); - SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_INF)); + SET_HASH_INFO_ITEM(PyLong_FromSsize_t(PyHASH_MODULUS)); + SET_HASH_INFO_ITEM(PyLong_FromLong(PyHASH_INF)); SET_HASH_INFO_ITEM(PyLong_FromLong(0)); // This is no longer used - SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_IMAG)); + SET_HASH_INFO_ITEM(PyLong_FromLong(PyHASH_IMAG)); SET_HASH_INFO_ITEM(PyUnicode_FromString(hashfunc->name)); SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->hash_bits)); SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->seed_bits)); @@ -1643,6 +1648,7 @@ static PyObject * _sys_getwindowsversion_from_kernel32(void) { #ifndef MS_WINDOWS_DESKTOP + PyErr_SetString(PyExc_OSError, "cannot read version info on this platform"); return NULL; #else HANDLE hKernel32; @@ -1670,6 +1676,9 @@ _sys_getwindowsversion_from_kernel32(void) !GetFileVersionInfoW(kernel32_path, 0, verblock_size, verblock) || !VerQueryValueW(verblock, L"", (LPVOID)&ffi, &ffi_len)) { PyErr_SetFromWindowsErr(0); + if (verblock) { + PyMem_RawFree(verblock); + } return NULL; } @@ -2134,6 +2143,7 @@ sys__current_frames_impl(PyObject *module) } /*[clinic input] +@permit_long_summary sys._current_exceptions Return a dict mapping each thread's identifier to its current raised exception. @@ -2143,7 +2153,7 @@ This function should be used for specialized purposes only. static PyObject * sys__current_exceptions_impl(PyObject *module) -/*[clinic end generated code: output=2ccfd838c746f0ba input=0e91818fbf2edc1f]*/ +/*[clinic end generated code: output=2ccfd838c746f0ba input=4ba429b6cfcd736d]*/ { return _PyThread_CurrentExceptions(); } @@ -2271,7 +2281,9 @@ static PyObject * sys__stats_on_impl(PyObject *module) /*[clinic end generated code: output=aca53eafcbb4d9fe input=43b5bfe145299e55]*/ { - _Py_StatsOn(); + if (_Py_StatsOn() < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -2304,6 +2316,7 @@ sys__stats_clear_impl(PyObject *module) } /*[clinic input] +@permit_long_docstring_body sys._stats_dump -> bool Dump stats to file, and clears the stats. @@ -2313,7 +2326,7 @@ Return False if no statistics were not dumped because stats gathering was off. static int sys__stats_dump_impl(PyObject *module) -/*[clinic end generated code: output=6e346b4ba0de4489 input=31a489e39418b2a5]*/ +/*[clinic end generated code: output=6e346b4ba0de4489 input=5a3ab40d2fb5af47]*/ { int res = _Py_PrintSpecializationStats(1); _Py_StatsClear(); @@ -2367,14 +2380,14 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } - else if (strcmp(backend, "perf_jit") == 0) { - _PyPerf_Callbacks cur_cb; - _PyPerfTrampoline_GetCallbacks(&cur_cb); - if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { - PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); - return NULL; - } + } + else if (strcmp(backend, "perf_jit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; } } } @@ -2448,60 +2461,6 @@ sys_is_remote_debug_enabled_impl(PyObject *module) #endif } -static PyObject * -sys_remote_exec_unicode_path(PyObject *module, int pid, PyObject *script) -{ - const char *debugger_script_path = PyUnicode_AsUTF8(script); - if (debugger_script_path == NULL) { - return NULL; - } - -#ifdef MS_WINDOWS - // Use UTF-16 (wide char) version of the path for permission checks - wchar_t *debugger_script_path_w = PyUnicode_AsWideCharString(script, NULL); - if (debugger_script_path_w == NULL) { - return NULL; - } - - // Check file attributes using wide character version (W) instead of ANSI (A) - DWORD attr = GetFileAttributesW(debugger_script_path_w); - PyMem_Free(debugger_script_path_w); - if (attr == INVALID_FILE_ATTRIBUTES) { - DWORD err = GetLastError(); - if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) { - PyErr_SetString(PyExc_FileNotFoundError, "Script file does not exist"); - } - else if (err == ERROR_ACCESS_DENIED) { - PyErr_SetString(PyExc_PermissionError, "Script file cannot be read"); - } - else { - PyErr_SetFromWindowsErr(0); - } - return NULL; - } -#else - if (access(debugger_script_path, F_OK | R_OK) != 0) { - switch (errno) { - case ENOENT: - PyErr_SetString(PyExc_FileNotFoundError, "Script file does not exist"); - break; - case EACCES: - PyErr_SetString(PyExc_PermissionError, "Script file cannot be read"); - break; - default: - PyErr_SetFromErrno(PyExc_OSError); - } - return NULL; - } -#endif - - if (_PySysRemoteDebug_SendExec(pid, 0, debugger_script_path) < 0) { - return NULL; - } - - Py_RETURN_NONE; -} - /*[clinic input] sys.remote_exec @@ -2532,13 +2491,70 @@ static PyObject * sys_remote_exec_impl(PyObject *module, int pid, PyObject *script) /*[clinic end generated code: output=7d94c56afe4a52c0 input=39908ca2c5fe1eb0]*/ { - PyObject *ret = NULL; PyObject *path; - if (PyUnicode_FSDecoder(script, &path)) { - ret = sys_remote_exec_unicode_path(module, pid, path); - Py_DECREF(path); + const char *debugger_script_path; + + if (PyUnicode_FSConverter(script, &path) == 0) { + return NULL; } - return ret; + + if (PySys_Audit("sys.remote_exec", "iO", pid, script) < 0) { + return NULL; + } + + debugger_script_path = PyBytes_AS_STRING(path); +#ifdef MS_WINDOWS + PyObject *unicode_path; + if (PyUnicode_FSDecoder(path, &unicode_path) < 0) { + goto error; + } + // Use UTF-16 (wide char) version of the path for permission checks + wchar_t *debugger_script_path_w = PyUnicode_AsWideCharString(unicode_path, NULL); + Py_DECREF(unicode_path); + if (debugger_script_path_w == NULL) { + goto error; + } + DWORD attr = GetFileAttributesW(debugger_script_path_w); + if (attr == INVALID_FILE_ATTRIBUTES) { + DWORD err = GetLastError(); + PyMem_Free(debugger_script_path_w); + if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) { + PyErr_SetString(PyExc_FileNotFoundError, "Script file does not exist"); + } + else if (err == ERROR_ACCESS_DENIED) { + PyErr_SetString(PyExc_PermissionError, "Script file cannot be read"); + } + else { + PyErr_SetFromWindowsErr(err); + } + goto error; + } + PyMem_Free(debugger_script_path_w); +#else // MS_WINDOWS + if (access(debugger_script_path, F_OK | R_OK) != 0) { + switch (errno) { + case ENOENT: + PyErr_SetString(PyExc_FileNotFoundError, "Script file does not exist"); + break; + case EACCES: + PyErr_SetString(PyExc_PermissionError, "Script file cannot be read"); + break; + default: + PyErr_SetFromErrno(PyExc_OSError); + } + goto error; + } +#endif // MS_WINDOWS + if (_PySysRemoteDebug_SendExec(pid, 0, debugger_script_path) < 0) { + goto error; + } + + Py_DECREF(path); + Py_RETURN_NONE; + +error: + Py_DECREF(path); + return NULL; } @@ -2634,6 +2650,7 @@ sys__baserepl_impl(PyObject *module) Py_RETURN_NONE; } + /*[clinic input] sys._is_gil_enabled -> bool @@ -3003,7 +3020,7 @@ static PyObject * get_warnoptions(PyThreadState *tstate) { PyObject *warnoptions; - if (_PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { return NULL; } if (warnoptions == NULL || !PyList_Check(warnoptions)) { @@ -3030,7 +3047,7 @@ get_warnoptions(PyThreadState *tstate) return warnoptions; } -void +PyAPI_FUNC(void) PySys_ResetWarnOptions(void) { PyThreadState *tstate = _PyThreadState_GET(); @@ -3040,7 +3057,7 @@ PySys_ResetWarnOptions(void) } PyObject *warnoptions; - if (_PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { PyErr_Clear(); return; } @@ -3104,7 +3121,7 @@ PyAPI_FUNC(int) PySys_HasWarnOptions(void) { PyObject *warnoptions; - if (_PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(warnoptions), &warnoptions) < 0) { PyErr_Clear(); return 0; } @@ -3118,7 +3135,7 @@ static PyObject * get_xoptions(PyThreadState *tstate) { PyObject *xoptions; - if (_PySys_GetOptionalAttr(&_Py_ID(_xoptions), &xoptions) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(_xoptions), &xoptions) < 0) { return NULL; } if (xoptions == NULL || !PyDict_Check(xoptions)) { @@ -3253,6 +3270,7 @@ PyDoc_STR( "\n\ Static objects:\n\ \n\ +abi_info -- Python ABI information.\n\ builtin_module_names -- tuple of module names built into this interpreter\n\ copyright -- copyright notice pertaining to this interpreter\n\ exec_prefix -- prefix used to find the machine-specific Python library\n\ @@ -3371,7 +3389,7 @@ sys_set_flag(PyObject *flags, Py_ssize_t pos, PyObject *value) int _PySys_SetFlagObj(Py_ssize_t pos, PyObject *value) { - PyObject *flags = _PySys_GetRequiredAttrString("flags"); + PyObject *flags = PySys_GetAttrString("flags"); if (flags == NULL) { return -1; } @@ -3600,6 +3618,18 @@ make_impl_info(PyObject *version_info) goto error; #endif + // PEP-734 +#if defined(__wasi__) || defined(__EMSCRIPTEN__) + // It is not enabled on WASM builds just yet + value = Py_False; +#else + value = Py_True; +#endif + res = PyDict_SetItemString(impl_info, "supports_isolated_interpreters", value); + if (res < 0) { + goto error; + } + /* dict ready */ ns = _PyNamespace_New(impl_info); @@ -3611,6 +3641,66 @@ make_impl_info(PyObject *version_info) return NULL; } + +static PyObject * +make_abi_info(void) +{ + // New entries should be added when needed for a supported platform, + // or by core dev consensus for enabling an unsupported one. + + PyObject *value; + PyObject *abi_info = PyDict_New(); + if (abi_info == NULL) { + return NULL; + } + + value = PyLong_FromLong(sizeof(void *) * 8); + if (value == NULL) { + goto error; + } + if (PyDict_SetItem(abi_info, &_Py_ID(pointer_bits), value) < 0) { + goto error; + } + Py_DECREF(value); + +#ifdef Py_GIL_DISABLED + value = Py_True; +#else + value = Py_False; +#endif + if (PyDict_SetItem(abi_info, &_Py_ID(free_threaded), value) < 0) { + goto error; + } + +#ifdef Py_DEBUG + value = Py_True; +#else + value = Py_False; +#endif + if (PyDict_SetItem(abi_info, &_Py_ID(debug), value) < 0) { + goto error; + } + +#if PY_BIG_ENDIAN + value = &_Py_ID(big); +#else + value = &_Py_ID(little); +#endif + if (PyDict_SetItem(abi_info, &_Py_ID(byteorder), value) < 0) { + goto error; + } + + PyObject *ns = _PyNamespace_New(abi_info); + Py_DECREF(abi_info); + return ns; + +error: + Py_DECREF(abi_info); + Py_XDECREF(value); + return NULL; +} + + #ifdef __EMSCRIPTEN__ PyDoc_STRVAR(emscripten_info__doc__, @@ -3785,9 +3875,9 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("builtin_module_names", list_builtin_module_names()); SET_SYS("stdlib_module_names", list_stdlib_module_names()); #if PY_BIG_ENDIAN - SET_SYS_FROM_STRING("byteorder", "big"); + SET_SYS("byteorder", &_Py_ID(big)); #else - SET_SYS_FROM_STRING("byteorder", "little"); + SET_SYS("byteorder", &_Py_ID(little)); #endif #ifdef MS_COREDLL @@ -3829,13 +3919,15 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) /* float repr style: 0.03 (short) vs 0.029999999999999999 (legacy) */ #if _PY_SHORT_FLOAT_REPR == 1 - SET_SYS_FROM_STRING("float_repr_style", "short"); + SET_SYS("float_repr_style", &_Py_ID(short)); #else - SET_SYS_FROM_STRING("float_repr_style", "legacy"); + SET_SYS("float_repr_style", &_Py_ID(legacy)); #endif SET_SYS("thread_info", PyThread_GetInfo()); + SET_SYS("abi_info", make_abi_info()); + /* initialize asyncgen_hooks */ if (_PyStructSequence_InitBuiltin(interp, &AsyncGenHooksType, &asyncgen_hooks_desc) < 0) @@ -3933,7 +4025,7 @@ _PySys_UpdateConfig(PyThreadState *tstate) #undef COPY_WSTR // sys.flags - PyObject *flags = _PySys_GetRequiredAttrString("flags"); + PyObject *flags = PySys_GetAttrString("flags"); if (flags == NULL) { return -1; } @@ -3994,13 +4086,14 @@ module _jit PyDoc_STRVAR(_jit_doc, "Utilities for observing just-in-time compilation."); /*[clinic input] +@permit_long_summary _jit.is_available -> bool Return True if the current Python executable supports JIT compilation, and False otherwise. [clinic start generated code]*/ static int _jit_is_available_impl(PyObject *module) -/*[clinic end generated code: output=6849a9cd2ff4aac9 input=03add84aa8347cf1]*/ +/*[clinic end generated code: output=6849a9cd2ff4aac9 input=d009d751ae64c9ef]*/ { (void)module; #ifdef _Py_TIER2 @@ -4011,26 +4104,28 @@ _jit_is_available_impl(PyObject *module) } /*[clinic input] +@permit_long_summary _jit.is_enabled -> bool Return True if JIT compilation is enabled for the current Python process (implies sys._jit.is_available()), and False otherwise. [clinic start generated code]*/ static int _jit_is_enabled_impl(PyObject *module) -/*[clinic end generated code: output=55865f8de993fe42 input=02439394da8e873f]*/ +/*[clinic end generated code: output=55865f8de993fe42 input=0524151e857f4f3a]*/ { (void)module; return _PyInterpreterState_GET()->jit; } /*[clinic input] +@permit_long_summary _jit.is_active -> bool Return True if the topmost Python frame is currently executing JIT code (implies sys._jit.is_enabled()), and False otherwise. [clinic start generated code]*/ static int _jit_is_active_impl(PyObject *module) -/*[clinic end generated code: output=7facca06b10064d4 input=be2fcd8a269d9b72]*/ +/*[clinic end generated code: output=7facca06b10064d4 input=081ee32563dc2086]*/ { (void)module; return _PyThreadState_GET()->current_executor != NULL; @@ -4249,7 +4344,7 @@ PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) } PyObject *sys_path; - if (_PySys_GetOptionalAttr(&_Py_ID(path), &sys_path) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(path), &sys_path) < 0) { Py_FatalError("can't get sys.path"); } else if (sys_path != NULL) { @@ -4345,7 +4440,7 @@ sys_write(PyObject *key, FILE *fp, const char *format, va_list va) PyObject *exc = _PyErr_GetRaisedException(tstate); written = PyOS_vsnprintf(buffer, sizeof(buffer), format, va); - file = _PySys_GetRequiredAttr(key); + file = PySys_GetAttr(key); if (sys_pyfile_write(buffer, file) != 0) { _PyErr_Clear(tstate); fputs(buffer, fp); @@ -4389,7 +4484,7 @@ sys_format(PyObject *key, FILE *fp, const char *format, va_list va) PyObject *exc = _PyErr_GetRaisedException(tstate); message = PyUnicode_FromFormatV(format, va); if (message != NULL) { - file = _PySys_GetRequiredAttr(key); + file = PySys_GetAttr(key); if (sys_pyfile_write_unicode(message, file) != 0) { _PyErr_Clear(tstate); utf8 = PyUnicode_AsUTF8(message); diff --git a/Python/thread.c b/Python/thread.c index 4ff5f11a348..0365f977d82 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -39,7 +39,8 @@ const long long PY_TIMEOUT_MAX = PY_TIMEOUT_MAX_VALUE; -static void PyThread__init_thread(void); /* Forward */ +/* Forward declaration */ +static void PyThread__init_thread(void); #define initialized _PyRuntime.threads.initialized @@ -71,6 +72,79 @@ PyThread_init_thread(void) #endif +/* + * Lock support. + */ + +PyThread_type_lock +PyThread_allocate_lock(void) +{ + if (!initialized) { + PyThread_init_thread(); + } + + PyMutex *lock = (PyMutex *)PyMem_RawMalloc(sizeof(PyMutex)); + if (lock) { + *lock = (PyMutex){0}; + } + + return (PyThread_type_lock)lock; +} + +void +PyThread_free_lock(PyThread_type_lock lock) +{ + PyMem_RawFree(lock); +} + +PyLockStatus +PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, + int intr_flag) +{ + PyTime_t timeout; // relative timeout + if (microseconds >= 0) { + // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout + // overflow to the caller, so clamp the timeout to + // [PyTime_MIN, PyTime_MAX]. + // + // PyTime_MAX nanoseconds is around 292.3 years. + // + // _thread.Lock.acquire() and _thread.RLock.acquire() raise an + // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. + timeout = _PyTime_FromMicrosecondsClamp(microseconds); + } + else { + timeout = -1; + } + + _PyLockFlags flags = _Py_LOCK_DONT_DETACH; + if (intr_flag) { + flags |= _PY_FAIL_IF_INTERRUPTED; + } + + return _PyMutex_LockTimed((PyMutex *)lock, timeout, flags); +} + +void +PyThread_release_lock(PyThread_type_lock lock) +{ + PyMutex_Unlock((PyMutex *)lock); +} + +int +_PyThread_at_fork_reinit(PyThread_type_lock *lock) +{ + _PyMutex_at_fork_reinit((PyMutex *)lock); + return 0; +} + +int +PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) +{ + return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0, /*intr_flag=*/0); +} + + /* return the current thread stack size */ size_t PyThread_get_stacksize(void) @@ -260,18 +334,12 @@ PyThread_GetInfo(void) #ifdef HAVE_PTHREAD_STUBS value = Py_NewRef(Py_None); -#elif defined(_POSIX_THREADS) -#ifdef USE_SEMAPHORES - value = PyUnicode_FromString("semaphore"); #else - value = PyUnicode_FromString("mutex+cond"); -#endif + value = PyUnicode_FromString("pymutex"); if (value == NULL) { Py_DECREF(threadinfo); return NULL; } -#else - value = Py_NewRef(Py_None); #endif PyStructSequence_SET_ITEM(threadinfo, pos++, value); diff --git a/Python/thread_nt.h b/Python/thread_nt.h index e078b98be3c..9a29d14ef67 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -300,98 +300,6 @@ PyThread_hang_thread(void) } } -/* - * Lock support. It has to be implemented as semaphores. - * I [Dag] tried to implement it with mutex but I could find a way to - * tell whether a thread already own the lock or not. - */ -PyThread_type_lock -PyThread_allocate_lock(void) -{ - PNRMUTEX mutex; - - if (!initialized) - PyThread_init_thread(); - - mutex = AllocNonRecursiveMutex() ; - - PyThread_type_lock aLock = (PyThread_type_lock) mutex; - assert(aLock); - - return aLock; -} - -void -PyThread_free_lock(PyThread_type_lock aLock) -{ - FreeNonRecursiveMutex(aLock) ; -} - -// WaitForSingleObject() accepts timeout in milliseconds in the range -// [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no -// timeout. 0xFFFFFFFE milliseconds is around 49.7 days. -const DWORD TIMEOUT_MS_MAX = 0xFFFFFFFE; - -/* - * Return 1 on success if the lock was acquired - * - * and 0 if the lock was not acquired. This means a 0 is returned - * if the lock has already been acquired by this thread! - */ -PyLockStatus -PyThread_acquire_lock_timed(PyThread_type_lock aLock, - PY_TIMEOUT_T microseconds, int intr_flag) -{ - assert(aLock); - - /* Fow now, intr_flag does nothing on Windows, and lock acquires are - * uninterruptible. */ - PyLockStatus success; - PY_TIMEOUT_T milliseconds; - - if (microseconds >= 0) { - milliseconds = microseconds / 1000; - // Round milliseconds away from zero - if (microseconds % 1000 > 0) { - milliseconds++; - } - if (milliseconds > (PY_TIMEOUT_T)TIMEOUT_MS_MAX) { - // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout - // overflow to the caller, so clamp the timeout to - // [0, TIMEOUT_MS_MAX] milliseconds. - // - // _thread.Lock.acquire() and _thread.RLock.acquire() raise an - // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. - milliseconds = TIMEOUT_MS_MAX; - } - assert(milliseconds != INFINITE); - } - else { - milliseconds = INFINITE; - } - - if (EnterNonRecursiveMutex((PNRMUTEX)aLock, - (DWORD)milliseconds) == WAIT_OBJECT_0) { - success = PY_LOCK_ACQUIRED; - } - else { - success = PY_LOCK_FAILURE; - } - - return success; -} -int -PyThread_acquire_lock(PyThread_type_lock aLock, int waitflag) -{ - return PyThread_acquire_lock_timed(aLock, waitflag ? -1 : 0, 0); -} - -void -PyThread_release_lock(PyThread_type_lock aLock) -{ - assert(aLock); - (void)LeaveNonRecursiveMutex((PNRMUTEX) aLock); -} /* minimum/maximum thread stack sizes supported */ #define THREAD_MIN_STACKSIZE 0x8000 /* 32 KiB */ diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index da405824244..8496f91db2e 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -30,6 +30,8 @@ # include <lwp.h> /* _lwp_self() */ #elif defined(__DragonFly__) # include <sys/lwp.h> /* lwp_gettid() */ +#elif defined(__sun__) && SIZEOF_LONG >= 8 +# include <thread.h> #endif /* The POSIX spec requires that use of pthread_attr_setstacksize @@ -99,16 +101,6 @@ #undef HAVE_SEM_CLOCKWAIT #endif -/* Whether or not to use semaphores directly rather than emulating them with - * mutexes and condition variables: - */ -#if (defined(_POSIX_SEMAPHORES) && !defined(HAVE_BROKEN_POSIX_SEMAPHORES) && \ - (defined(HAVE_SEM_TIMEDWAIT) || defined(HAVE_SEM_CLOCKWAIT))) -# define USE_SEMAPHORES -#else -# undef USE_SEMAPHORES -#endif - /* On platforms that don't use standard POSIX threads pthread_sigmask() * isn't present. DEC threads uses sigprocmask() instead as do most @@ -409,6 +401,8 @@ PyThread_get_thread_native_id(void) #elif defined(__DragonFly__) lwpid_t native_id; native_id = lwp_gettid(); +#elif defined(__sun__) && SIZEOF_LONG >= 8 + unsigned long native_id = (unsigned long)getpid() << 32 | thr_self(); #endif return (unsigned long) native_id; } @@ -442,388 +436,6 @@ PyThread_hang_thread(void) } } -#ifdef USE_SEMAPHORES - -/* - * Lock support. - */ - -PyThread_type_lock -PyThread_allocate_lock(void) -{ - sem_t *lock; - int status, error = 0; - - if (!initialized) - PyThread_init_thread(); - - lock = (sem_t *)PyMem_RawMalloc(sizeof(sem_t)); - - if (lock) { - status = sem_init(lock,0,1); - CHECK_STATUS("sem_init"); - - if (error) { - PyMem_RawFree((void *)lock); - lock = NULL; - } - } - - return (PyThread_type_lock)lock; -} - -void -PyThread_free_lock(PyThread_type_lock lock) -{ - sem_t *thelock = (sem_t *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - if (!thelock) - return; - - status = sem_destroy(thelock); - CHECK_STATUS("sem_destroy"); - - PyMem_RawFree((void *)thelock); -} - -/* - * As of February 2002, Cygwin thread implementations mistakenly report error - * codes in the return value of the sem_ calls (like the pthread_ functions). - * Correct implementations return -1 and put the code in errno. This supports - * either. - */ -static int -fix_status(int status) -{ - return (status == -1) ? errno : status; -} - -PyLockStatus -PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, - int intr_flag) -{ - PyLockStatus success; - sem_t *thelock = (sem_t *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - PyTime_t timeout; // relative timeout - if (microseconds >= 0) { - // bpo-41710: PyThread_acquire_lock_timed() cannot report timeout - // overflow to the caller, so clamp the timeout to - // [PyTime_MIN, PyTime_MAX]. - // - // PyTime_MAX nanoseconds is around 292.3 years. - // - // _thread.Lock.acquire() and _thread.RLock.acquire() raise an - // OverflowError if microseconds is greater than PY_TIMEOUT_MAX. - timeout = _PyTime_FromMicrosecondsClamp(microseconds); - } - else { - timeout = -1; - } - -#ifdef HAVE_SEM_CLOCKWAIT - struct timespec abs_timeout; - // Local scope for deadline - { - PyTime_t now; - // silently ignore error: cannot report error to the caller - (void)PyTime_MonotonicRaw(&now); - PyTime_t deadline = _PyTime_Add(now, timeout); - _PyTime_AsTimespec_clamp(deadline, &abs_timeout); - } -#else - PyTime_t deadline = 0; - if (timeout > 0 && !intr_flag) { - deadline = _PyDeadline_Init(timeout); - } -#endif - - while (1) { - if (timeout > 0) { -#ifdef HAVE_SEM_CLOCKWAIT - status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, - &abs_timeout)); -#else - PyTime_t now; - // silently ignore error: cannot report error to the caller - (void)PyTime_TimeRaw(&now); - PyTime_t abs_time = _PyTime_Add(now, timeout); - - struct timespec ts; - _PyTime_AsTimespec_clamp(abs_time, &ts); - status = fix_status(sem_timedwait(thelock, &ts)); -#endif - } - else if (timeout == 0) { - status = fix_status(sem_trywait(thelock)); - } - else { - status = fix_status(sem_wait(thelock)); - } - - /* Retry if interrupted by a signal, unless the caller wants to be - notified. */ - if (intr_flag || status != EINTR) { - break; - } - - // sem_clockwait() uses an absolute timeout, there is no need - // to recompute the relative timeout. -#ifndef HAVE_SEM_CLOCKWAIT - if (timeout > 0) { - /* wait interrupted by a signal (EINTR): recompute the timeout */ - timeout = _PyDeadline_Get(deadline); - if (timeout < 0) { - status = ETIMEDOUT; - break; - } - } -#endif - } - - /* Don't check the status if we're stopping because of an interrupt. */ - if (!(intr_flag && status == EINTR)) { - if (timeout > 0) { - if (status != ETIMEDOUT) { -#ifdef HAVE_SEM_CLOCKWAIT - CHECK_STATUS("sem_clockwait"); -#else - CHECK_STATUS("sem_timedwait"); -#endif - } - } - else if (timeout == 0) { - if (status != EAGAIN) { - CHECK_STATUS("sem_trywait"); - } - } - else { - CHECK_STATUS("sem_wait"); - } - } - - if (status == 0) { - success = PY_LOCK_ACQUIRED; - } else if (intr_flag && status == EINTR) { - success = PY_LOCK_INTR; - } else { - success = PY_LOCK_FAILURE; - } - - return success; -} - -void -PyThread_release_lock(PyThread_type_lock lock) -{ - sem_t *thelock = (sem_t *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - status = sem_post(thelock); - CHECK_STATUS("sem_post"); -} - -#else /* USE_SEMAPHORES */ - -/* - * Lock support. - */ -PyThread_type_lock -PyThread_allocate_lock(void) -{ - pthread_lock *lock; - int status, error = 0; - - if (!initialized) - PyThread_init_thread(); - - lock = (pthread_lock *) PyMem_RawCalloc(1, sizeof(pthread_lock)); - if (lock) { - lock->locked = 0; - - status = pthread_mutex_init(&lock->mut, NULL); - CHECK_STATUS_PTHREAD("pthread_mutex_init"); - /* Mark the pthread mutex underlying a Python mutex as - pure happens-before. We can't simply mark the - Python-level mutex as a mutex because it can be - acquired and released in different threads, which - will cause errors. */ - _Py_ANNOTATE_PURE_HAPPENS_BEFORE_MUTEX(&lock->mut); - - status = _PyThread_cond_init(&lock->lock_released); - CHECK_STATUS_PTHREAD("pthread_cond_init"); - - if (error) { - PyMem_RawFree((void *)lock); - lock = 0; - } - } - - return (PyThread_type_lock) lock; -} - -void -PyThread_free_lock(PyThread_type_lock lock) -{ - pthread_lock *thelock = (pthread_lock *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - /* some pthread-like implementations tie the mutex to the cond - * and must have the cond destroyed first. - */ - status = pthread_cond_destroy( &thelock->lock_released ); - CHECK_STATUS_PTHREAD("pthread_cond_destroy"); - - status = pthread_mutex_destroy( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_destroy"); - - PyMem_RawFree((void *)thelock); -} - -PyLockStatus -PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, - int intr_flag) -{ - PyLockStatus success = PY_LOCK_FAILURE; - pthread_lock *thelock = (pthread_lock *)lock; - int status, error = 0; - - if (microseconds == 0) { - status = pthread_mutex_trylock( &thelock->mut ); - if (status != EBUSY) { - CHECK_STATUS_PTHREAD("pthread_mutex_trylock[1]"); - } - } - else { - status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_lock[1]"); - } - if (status != 0) { - goto done; - } - - if (thelock->locked == 0) { - success = PY_LOCK_ACQUIRED; - goto unlock; - } - if (microseconds == 0) { - goto unlock; - } - - struct timespec abs_timeout; - if (microseconds > 0) { - _PyThread_cond_after(microseconds, &abs_timeout); - } - // Continue trying until we get the lock - - // mut must be locked by me -- part of the condition protocol - while (1) { - if (microseconds > 0) { - status = pthread_cond_timedwait(&thelock->lock_released, - &thelock->mut, &abs_timeout); - if (status == 1) { - break; - } - if (status == ETIMEDOUT) { - break; - } - CHECK_STATUS_PTHREAD("pthread_cond_timedwait"); - } - else { - status = pthread_cond_wait( - &thelock->lock_released, - &thelock->mut); - CHECK_STATUS_PTHREAD("pthread_cond_wait"); - } - - if (intr_flag && status == 0 && thelock->locked) { - // We were woken up, but didn't get the lock. We probably received - // a signal. Return PY_LOCK_INTR to allow the caller to handle - // it and retry. - success = PY_LOCK_INTR; - break; - } - - if (status == 0 && !thelock->locked) { - success = PY_LOCK_ACQUIRED; - break; - } - - // Wait got interrupted by a signal: retry - } - -unlock: - if (success == PY_LOCK_ACQUIRED) { - thelock->locked = 1; - } - status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_unlock[1]"); - -done: - if (error) { - success = PY_LOCK_FAILURE; - } - return success; -} - -void -PyThread_release_lock(PyThread_type_lock lock) -{ - pthread_lock *thelock = (pthread_lock *)lock; - int status, error = 0; - - (void) error; /* silence unused-but-set-variable warning */ - - status = pthread_mutex_lock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_lock[3]"); - - thelock->locked = 0; - - /* wake up someone (anyone, if any) waiting on the lock */ - status = pthread_cond_signal( &thelock->lock_released ); - CHECK_STATUS_PTHREAD("pthread_cond_signal"); - - status = pthread_mutex_unlock( &thelock->mut ); - CHECK_STATUS_PTHREAD("pthread_mutex_unlock[3]"); -} - -#endif /* USE_SEMAPHORES */ - -int -_PyThread_at_fork_reinit(PyThread_type_lock *lock) -{ - PyThread_type_lock new_lock = PyThread_allocate_lock(); - if (new_lock == NULL) { - return -1; - } - - /* bpo-6721, bpo-40089: The old lock can be in an inconsistent state. - fork() can be called in the middle of an operation on the lock done by - another thread. So don't call PyThread_free_lock(*lock). - - Leak memory on purpose. Don't release the memory either since the - address of a mutex is relevant. Putting two mutexes at the same address - can lead to problems. */ - - *lock = new_lock; - return 0; -} - -int -PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) -{ - return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0, /*intr_flag=*/0); -} /* set the thread stack size. * Return 0 if size is valid, -1 if size is invalid, diff --git a/Python/traceback.c b/Python/traceback.c index c06cb1a5908..8af63c22a9f 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -9,7 +9,6 @@ #include "pycore_interpframe.h" // _PyFrame_GetCode() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_sysmodule.h" // _PySys_GetOptionalAttr() #include "pycore_traceback.h" // EXCEPTION_TB_HEADER #include "frameobject.h" // PyFrame_New() @@ -70,6 +69,13 @@ class traceback "PyTracebackObject *" "&PyTraceback_Type" #include "clinic/traceback.c.h" + +#ifdef MS_WINDOWS +typedef HRESULT (WINAPI *PF_GET_THREAD_DESCRIPTION)(HANDLE, PCWSTR*); +static PF_GET_THREAD_DESCRIPTION pGetThreadDescription = NULL; +#endif + + static PyObject * tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti, int lineno) @@ -399,7 +405,7 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * taillen = strlen(tail); PyThreadState *tstate = _PyThreadState_GET(); - if (_PySys_GetOptionalAttr(&_Py_ID(path), &syspath) < 0) { + if (PySys_GetOptionalAttr(&_Py_ID(path), &syspath) < 0) { PyErr_Clear(); goto error; } @@ -777,7 +783,7 @@ _PyTraceBack_Print(PyObject *v, const char *header, PyObject *f) PyErr_BadInternalCall(); return -1; } - if (_PySys_GetOptionalAttrString("tracebacklimit", &limitv) < 0) { + if (PySys_GetOptionalAttrString("tracebacklimit", &limitv) < 0) { return -1; } else if (limitv != NULL && PyLong_Check(limitv)) { @@ -974,16 +980,72 @@ _Py_DumpASCII(int fd, PyObject *text) } } + +#ifdef MS_WINDOWS +static void +_Py_DumpWideString(int fd, wchar_t *str) +{ + Py_ssize_t size = wcslen(str); + int truncated; + if (MAX_STRING_LENGTH < size) { + size = MAX_STRING_LENGTH; + truncated = 1; + } + else { + truncated = 0; + } + + for (Py_ssize_t i=0; i < size; i++) { + Py_UCS4 ch = str[i]; + if (' ' <= ch && ch <= 126) { + /* printable ASCII character */ + dump_char(fd, (char)ch); + } + else if (ch <= 0xff) { + PUTS(fd, "\\x"); + _Py_DumpHexadecimal(fd, ch, 2); + } + else if (Py_UNICODE_IS_HIGH_SURROGATE(ch) + && Py_UNICODE_IS_LOW_SURROGATE(str[i+1])) { + ch = Py_UNICODE_JOIN_SURROGATES(ch, str[i+1]); + i++; // Skip the low surrogate character + PUTS(fd, "\\U"); + _Py_DumpHexadecimal(fd, ch, 8); + } + else { + Py_BUILD_ASSERT(sizeof(wchar_t) == 2); + PUTS(fd, "\\u"); + _Py_DumpHexadecimal(fd, ch, 4); + } + } + + if (truncated) { + PUTS(fd, "..."); + } +} +#endif + + /* Write a frame into the file fd: "File "xxx", line xxx in xxx". - This function is signal safe. */ + This function is signal safe. -static void + Return 0 on success. Return -1 if the frame is invalid. */ + +static int dump_frame(int fd, _PyInterpreterFrame *frame) { - assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Ignore trampoline frames and base frame sentinel */ + return 0; + } - PyCodeObject *code =_PyFrame_GetCode(frame); + PyCodeObject *code = _PyFrame_SafeGetCode(frame); + if (code == NULL) { + return -1; + } + + int res = 0; PUTS(fd, " File "); if (code->co_filename != NULL && PyUnicode_Check(code->co_filename)) @@ -991,29 +1053,36 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "\""); _Py_DumpASCII(fd, code->co_filename); PUTS(fd, "\""); - } else { + } + else { PUTS(fd, "???"); + res = -1; } - int lineno = PyUnstable_InterpreterFrame_GetLine(frame); PUTS(fd, ", line "); + int lasti = _PyFrame_SafeGetLasti(frame); + int lineno = -1; + if (lasti >= 0) { + lineno = _PyCode_SafeAddr2Line(code, lasti); + } if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); } else { PUTS(fd, "???"); + res = -1; } - PUTS(fd, " in "); - if (code->co_name != NULL - && PyUnicode_Check(code->co_name)) { + PUTS(fd, " in "); + if (code->co_name != NULL && PyUnicode_Check(code->co_name)) { _Py_DumpASCII(fd, code->co_name); } else { PUTS(fd, "???"); + res = -1; } - PUTS(fd, "\n"); + return res; } static int @@ -1025,6 +1094,9 @@ tstate_is_freed(PyThreadState *tstate) if (_PyMem_IsPtrFreed(tstate->interp)) { return 1; } + if (_PyMem_IsULongFreed(tstate->thread_id)) { + return 1; + } return 0; } @@ -1044,7 +1116,7 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) } if (tstate_is_freed(tstate)) { - PUTS(fd, " <tstate is freed>\n"); + PUTS(fd, " <freed thread state>\n"); return; } @@ -1056,17 +1128,6 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - /* Trampoline frame */ - frame = frame->previous; - if (frame == NULL) { - break; - } - - /* Can't have more than one shim frame in a row */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - } - if (MAX_FRAME_DEPTH <= depth) { if (MAX_FRAME_DEPTH < depth) { PUTS(fd, "plus "); @@ -1076,8 +1137,20 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) break; } - dump_frame(fd, frame); - frame = frame->previous; + if (_PyMem_IsPtrFreed(frame)) { + PUTS(fd, " <freed frame>\n"); + break; + } + // Read frame->previous early since memory can be freed during + // dump_frame() + _PyInterpreterFrame *previous = frame->previous; + + if (dump_frame(fd, frame) < 0) { + PUTS(fd, " <invalid frame>\n"); + break; + } + + frame = previous; if (frame == NULL) { break; } @@ -1108,23 +1181,12 @@ _Py_DumpTraceback(int fd, PyThreadState *tstate) # endif #endif -/* Write the thread identifier into the file 'fd': "Current thread 0xHHHH:\" if - is_current is true, "Thread 0xHHHH:\n" otherwise. - - This function is signal safe. */ +// Write the thread name static void -write_thread_id(int fd, PyThreadState *tstate, int is_current) +write_thread_name(int fd, PyThreadState *tstate) { - if (is_current) - PUTS(fd, "Current thread 0x"); - else - PUTS(fd, "Thread 0x"); - _Py_DumpHexadecimal(fd, - tstate->thread_id, - sizeof(unsigned long) * 2); - - // Write the thread name +#ifndef MS_WINDOWS #if defined(HAVE_PTHREAD_GETNAME_NP) || defined(HAVE_PTHREAD_GET_NAME_NP) char name[100]; pthread_t thread = (pthread_t)tstate->thread_id; @@ -1143,6 +1205,51 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) } } #endif +#else + // Windows implementation + if (pGetThreadDescription == NULL) { + return; + } + + HANDLE thread = OpenThread(THREAD_QUERY_LIMITED_INFORMATION, FALSE, tstate->thread_id); + if (thread == NULL) { + return; + } + + wchar_t *name; + HRESULT hr = pGetThreadDescription(thread, &name); + if (!FAILED(hr)) { + if (name[0] != 0) { + PUTS(fd, " ["); + _Py_DumpWideString(fd, name); + PUTS(fd, "]"); + } + LocalFree(name); + } + CloseHandle(thread); +#endif +} + + +/* Write the thread identifier into the file 'fd': "Current thread 0xHHHH:\" if + is_current is true, "Thread 0xHHHH:\n" otherwise. + + This function is signal safe (except on Windows). */ + +static void +write_thread_id(int fd, PyThreadState *tstate, int is_current) +{ + if (is_current) + PUTS(fd, "Current thread 0x"); + else + PUTS(fd, "Thread 0x"); + _Py_DumpHexadecimal(fd, + tstate->thread_id, + sizeof(unsigned long) * 2); + + if (!_PyMem_IsULongFreed(tstate->thread_id)) { + write_thread_name(fd, tstate); + } PUTS(fd, " (most recent call first):\n"); } @@ -1200,7 +1307,6 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, return "unable to get the thread head state"; /* Dump the traceback of each thread */ - tstate = PyInterpreterState_ThreadHead(interp); unsigned int nthreads = 0; _Py_BEGIN_SUPPRESS_IPH do @@ -1211,11 +1317,18 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, PUTS(fd, "...\n"); break; } + + if (tstate_is_freed(tstate)) { + PUTS(fd, "<freed thread state>\n"); + break; + } + write_thread_id(fd, tstate, tstate == current_tstate); if (tstate == current_tstate && tstate->interp->gc.collecting) { PUTS(fd, " Garbage-collecting\n"); } dump_traceback(fd, tstate, 0); + tstate = PyThreadState_Next(tstate); nthreads++; } while (tstate != NULL); @@ -1327,3 +1440,30 @@ _Py_DumpStack(int fd) PUTS(fd, " <cannot get C stack on this system>\n"); } #endif + +void +_Py_InitDumpStack(void) +{ +#ifdef CAN_C_BACKTRACE + // gh-137185: Call backtrace() once to force libgcc to be loaded early. + void *callstack[1]; + (void)backtrace(callstack, 1); +#endif +} + + +void +_Py_DumpTraceback_Init(void) +{ +#ifdef MS_WINDOWS + if (pGetThreadDescription != NULL) { + return; + } + + HMODULE kernelbase = GetModuleHandleW(L"kernelbase.dll"); + if (kernelbase != NULL) { + pGetThreadDescription = (PF_GET_THREAD_DESCRIPTION)GetProcAddress( + kernelbase, "GetThreadDescription"); + } +#endif +} diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 7066a214f10..20351618721 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -36,7 +36,7 @@ static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, the GIL held from PyMem_RawFree(). It cannot acquire the lock because it would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ #define tables_lock _PyRuntime.tracemalloc.tables_lock -#define TABLES_LOCK() PyMutex_LockFlags(&tables_lock, _Py_LOCK_DONT_DETACH) +#define TABLES_LOCK() PyMutex_Lock(&tables_lock) #define TABLES_UNLOCK() PyMutex_Unlock(&tables_lock) @@ -46,7 +46,7 @@ typedef struct tracemalloc_frame frame_t; typedef struct tracemalloc_traceback traceback_t; #define TRACEBACK_SIZE(NFRAME) \ - (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1)) + (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME)) static const int MAX_NFRAME = UINT16_MAX; @@ -230,7 +230,7 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) } frame->lineno = (unsigned int)lineno; - PyObject *filename = filename = _PyFrame_GetCode(pyframe)->co_filename; + PyObject *filename = _PyFrame_GetCode(pyframe)->co_filename; if (filename == NULL) { #ifdef TRACE_DEBUG tracemalloc_error("failed to get the filename of the code object"); @@ -329,8 +329,9 @@ traceback_new(void) traceback->nframe = 0; traceback->total_nframe = 0; traceback_get_frames(traceback); - if (traceback->nframe == 0) - return &tracemalloc_empty_traceback; + if (traceback->nframe == 0) { + return tracemalloc_empty_traceback; + } traceback->hash = traceback_hash(traceback); /* intern the traceback */ @@ -754,12 +755,18 @@ _PyTraceMalloc_Init(void) return _PyStatus_NO_MEMORY(); } - tracemalloc_empty_traceback.nframe = 1; - tracemalloc_empty_traceback.total_nframe = 1; + assert(tracemalloc_empty_traceback == NULL); + tracemalloc_empty_traceback = raw_malloc(TRACEBACK_SIZE(1)); + if (tracemalloc_empty_traceback == NULL) { + return _PyStatus_NO_MEMORY(); + } + + tracemalloc_empty_traceback->nframe = 1; + tracemalloc_empty_traceback->total_nframe = 1; /* borrowed reference */ - tracemalloc_empty_traceback.frames[0].filename = &_Py_STR(anon_unknown); - tracemalloc_empty_traceback.frames[0].lineno = 0; - tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback); + tracemalloc_empty_traceback->frames[0].filename = &_Py_STR(anon_unknown); + tracemalloc_empty_traceback->frames[0].lineno = 0; + tracemalloc_empty_traceback->hash = traceback_hash(tracemalloc_empty_traceback); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; return _PyStatus_OK(); @@ -782,6 +789,9 @@ tracemalloc_deinit(void) _Py_hashtable_destroy(tracemalloc_filenames); PyThread_tss_delete(&tracemalloc_reentrant_key); + + raw_free(tracemalloc_empty_traceback); + tracemalloc_empty_traceback = NULL; } diff --git a/README.rst b/README.rst index 897d8995b63..bc1c1df2069 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.14.0 beta 1 -==================================== +This is Python version 3.15.0 alpha 2 +===================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push :alt: CPython build status on GitHub Actions @@ -135,8 +135,8 @@ libraries for additional performance gains. What's New ---------- -We have a comprehensive overview of the changes in the `What's New in Python -3.14 <https://docs.python.org/3.14/whatsnew/3.14.html>`_ document. For a more +We have a comprehensive overview of the changes in the `What's new in Python +3.15 <https://docs.python.org/3.15/whatsnew/3.15.html>`_ document. For a more detailed change log, read `Misc/NEWS <https://github.com/python/cpython/tree/main/Misc/NEWS.d>`_, but a full accounting of changes can only be gleaned from the `commit history @@ -149,11 +149,11 @@ entitled "Installing multiple versions". Documentation ------------- -`Documentation for Python 3.14 <https://docs.python.org/3.14/>`_ is online, +`Documentation for Python 3.15 <https://docs.python.org/3.15/>`_ is online, updated daily. It can also be downloaded in many formats for faster access. The documentation -is downloadable in HTML, PDF, and reStructuredText formats; the latter version +is downloadable in HTML, EPUB, and reStructuredText formats; the latter version is primarily for documentation authors, translators, and people with special formatting requirements. @@ -200,15 +200,15 @@ intend to install multiple versions using the same prefix you must decide which version (if any) is your "primary" version. Install that version using ``make install``. Install all other versions using ``make altinstall``. -For example, if you want to install Python 2.7, 3.6, and 3.14 with 3.14 being the -primary version, you would execute ``make install`` in your 3.14 build directory +For example, if you want to install Python 2.7, 3.6, and 3.15 with 3.15 being the +primary version, you would execute ``make install`` in your 3.15 build directory and ``make altinstall`` in the others. Release Schedule ---------------- -See `PEP 745 <https://peps.python.org/pep-0745/>`__ for Python 3.14 release details. +See `PEP 790 <https://peps.python.org/pep-0790/>`__ for Python 3.15 release details. Copyright and License Information @@ -232,4 +232,4 @@ This Python distribution contains *no* GNU General Public License (GPL) code, so it may be used in proprietary projects. There are interfaces to some GNU code but these are entirely optional. -All trademarks referenced herein are property of their respective holders. \ No newline at end of file +All trademarks referenced herein are property of their respective holders. diff --git a/Tools/README b/Tools/README index 09bd6fb4798..22d76dfdbcf 100644 --- a/Tools/README +++ b/Tools/README @@ -16,6 +16,8 @@ clinic A preprocessor for CPython C files in order to automate freeze Create a stand-alone executable from a Python program. +ftscalingbench Benchmarks for free-threading and finding bottlenecks. + gdb Python code to be run inside gdb, to make it easier to debug Python itself (by David Malcolm). @@ -26,6 +28,12 @@ i18n Tools for internationalization. pygettext.py importbench A set of micro-benchmarks for various import scenarios. +inspection Tooling for PEP-678 "Safe external debugger interface for CPython". + +jit Tooling for building the JIT. + +lockbench Benchmarks for PyMutex and critical sections. + msi Support for packaging Python as an MSI package on Windows. nuget Files for the NuGet package manager for .NET. @@ -41,7 +49,7 @@ scripts A number of useful single-file programs, e.g. run_tests.py ssl Scripts to generate ssl_data.h from OpenSSL sources, and run tests against multiple installations of OpenSSL and LibreSSL. -tz A script to dump timezone from /usr/share/zoneinfo. +tsan Utilities for building CPython with thread-sanitizer. unicode Tools for generating unicodedata and codecs from unicode.org and other mapping files (by Fredrik Lundh, Marc-Andre Lemburg diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index fa7689d45db..996f725fdcb 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -1,7 +1,7 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings [per-file-target-version] -"deepfreeze.py" = "py310" +"deepfreeze.py" = "py311" # requires `code.co_exceptiontable` "stable_abi.py" = "py311" # requires 'tomllib' [format] @@ -29,7 +29,6 @@ ignore = [ "F541", # f-string without any placeholders "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` "PYI025", # Use `from collections.abc import Set as AbstractSet` - "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` ] [lint.per-file-ignores] diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 9815bcfe27d..f23c1d5286f 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -17,25 +17,32 @@ See --help for more information """ + +from __future__ import annotations + import _imp import argparse -import collections import enum +import json import logging import os import pathlib +import pprint import re import sys import sysconfig import warnings from collections.abc import Iterable -from importlib._bootstrap import _load as bootstrap_load +from importlib._bootstrap import ( # type: ignore[attr-defined] + _load as bootstrap_load, +) from importlib.machinery import ( BuiltinImporter, ExtensionFileLoader, ModuleSpec, ) from importlib.util import spec_from_file_location, spec_from_loader +from typing import NamedTuple SRC_DIR = pathlib.Path(__file__).parent.parent.parent @@ -111,7 +118,20 @@ help="Print a list of module names to stdout and exit", ) +parser.add_argument( + "--generate-missing-stdlib-info", + action="store_true", + help="Generate file with stdlib module info", +) +parser.add_argument( + "--with-missing-stdlib-config", + metavar="CONFIG_FILE", + help="Path to JSON config file with custom missing module messages", +) + + +@enum.unique class ModuleState(enum.Enum): # Makefile state "yes" BUILTIN = "builtin" @@ -123,11 +143,13 @@ class ModuleState(enum.Enum): # disabled by Setup / makesetup rule DISABLED_SETUP = "disabled_setup" - def __bool__(self): + def __bool__(self) -> bool: return self.value in {"builtin", "shared"} -ModuleInfo = collections.namedtuple("ModuleInfo", "name state") +class ModuleInfo(NamedTuple): + name: str + state: ModuleState class ModuleChecker: @@ -135,9 +157,9 @@ class ModuleChecker: setup_files = ( # see end of configure.ac - "Modules/Setup.local", - "Modules/Setup.stdlib", - "Modules/Setup.bootstrap", + pathlib.Path("Modules/Setup.local"), + pathlib.Path("Modules/Setup.stdlib"), + pathlib.Path("Modules/Setup.bootstrap"), SRC_DIR / "Modules/Setup", ) @@ -149,15 +171,15 @@ def __init__(self, cross_compiling: bool = False, strict: bool = False): self.builddir = self.get_builddir() self.modules = self.get_modules() - self.builtin_ok = [] - self.shared_ok = [] - self.failed_on_import = [] - self.missing = [] - self.disabled_configure = [] - self.disabled_setup = [] - self.notavailable = [] + self.builtin_ok: list[ModuleInfo] = [] + self.shared_ok: list[ModuleInfo] = [] + self.failed_on_import: list[ModuleInfo] = [] + self.missing: list[ModuleInfo] = [] + self.disabled_configure: list[ModuleInfo] = [] + self.disabled_setup: list[ModuleInfo] = [] + self.notavailable: list[ModuleInfo] = [] - def check(self): + def check(self) -> None: if not hasattr(_imp, 'create_dynamic'): logger.warning( ('Dynamic extensions not supported ' @@ -189,10 +211,10 @@ def check(self): assert modinfo.state == ModuleState.SHARED self.shared_ok.append(modinfo) - def summary(self, *, verbose: bool = False): + def summary(self, *, verbose: bool = False) -> None: longest = max([len(e.name) for e in self.modules], default=0) - def print_three_column(modinfos: list[ModuleInfo]): + def print_three_column(modinfos: list[ModuleInfo]) -> None: names = [modinfo.name for modinfo in modinfos] names.sort(key=str.lower) # guarantee zip() doesn't drop anything @@ -262,17 +284,50 @@ def print_three_column(modinfos: list[ModuleInfo]): f"{len(self.failed_on_import)} failed on import)" ) - def check_strict_build(self): + def check_strict_build(self) -> None: """Fail if modules are missing and it's a strict build""" if self.strict_extensions_build and (self.failed_on_import or self.missing): raise RuntimeError("Failed to build some stdlib modules") - def list_module_names(self, *, all: bool = False) -> set: + def list_module_names(self, *, all: bool = False) -> set[str]: names = {modinfo.name for modinfo in self.modules} if all: names.update(WINDOWS_MODULES) return names + def generate_missing_stdlib_info(self, config_path: str | None = None) -> None: + config_messages = {} + if config_path: + try: + with open(config_path, encoding='utf-8') as f: + config_messages = json.load(f) + except (FileNotFoundError, json.JSONDecodeError) as e: + raise RuntimeError(f"Failed to load missing stdlib config {config_path!r}") from e + + messages = {} + for name in WINDOWS_MODULES: + messages[name] = f"Unsupported platform for Windows-only standard library module {name!r}" + + for modinfo in self.modules: + if modinfo.state in (ModuleState.DISABLED, ModuleState.DISABLED_SETUP): + messages[modinfo.name] = f"Standard library module disabled during build {modinfo.name!r} was not found" + elif modinfo.state == ModuleState.NA: + messages[modinfo.name] = f"Unsupported platform for standard library module {modinfo.name!r}" + + messages.update(config_messages) + + content = f'''\ +# Standard library information used by the traceback module for more informative +# ModuleNotFound error messages. +# Generated by check_extension_modules.py + +_MISSING_STDLIB_MODULE_MESSAGES = {pprint.pformat(messages)} +''' + + output_path = self.builddir / "_missing_stdlib_info.py" + with open(output_path, "w", encoding="utf-8") as f: + f.write(content) + def get_builddir(self) -> pathlib.Path: try: with open(self.pybuilddir_txt, encoding="utf-8") as f: @@ -280,9 +335,9 @@ def get_builddir(self) -> pathlib.Path: except FileNotFoundError: logger.error("%s must be run from the top build directory", __file__) raise - builddir = pathlib.Path(builddir) - logger.debug("%s: %s", self.pybuilddir_txt, builddir) - return builddir + builddir_path = pathlib.Path(builddir) + logger.debug("%s: %s", self.pybuilddir_txt, builddir_path) + return builddir_path def get_modules(self) -> list[ModuleInfo]: """Get module info from sysconfig and Modules/Setup* files""" @@ -367,7 +422,7 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: case ["*disabled*"]: state = ModuleState.DISABLED case ["*noconfig*"]: - state = None + continue case [*items]: if state == ModuleState.DISABLED: # *disabled* can disable multiple modules per line @@ -384,26 +439,33 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec: """Get ModuleSpec for builtin or extension module""" if modinfo.state == ModuleState.SHARED: - location = os.fspath(self.get_location(modinfo)) + mod_location = self.get_location(modinfo) + assert mod_location is not None + location = os.fspath(mod_location) loader = ExtensionFileLoader(modinfo.name, location) - return spec_from_file_location(modinfo.name, location, loader=loader) + spec = spec_from_file_location(modinfo.name, location, loader=loader) + assert spec is not None + return spec elif modinfo.state == ModuleState.BUILTIN: - return spec_from_loader(modinfo.name, loader=BuiltinImporter) + spec = spec_from_loader(modinfo.name, loader=BuiltinImporter) + assert spec is not None + return spec else: raise ValueError(modinfo) - def get_location(self, modinfo: ModuleInfo) -> pathlib.Path: + def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: """Get shared library location in build directory""" if modinfo.state == ModuleState.SHARED: return self.builddir / f"{modinfo.name}{self.ext_suffix}" else: return None - def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): + def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" - if spec.loader is BuiltinImporter: + if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] return try: + assert spec.origin is not None st = os.stat(spec.origin) except FileNotFoundError: logger.error("%s (%s) is missing", modinfo.name, spec.origin) @@ -411,7 +473,7 @@ def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): if not st.st_size: raise ImportError(f"{spec.origin} is an empty file") - def check_module_import(self, modinfo: ModuleInfo): + def check_module_import(self, modinfo: ModuleInfo) -> None: """Attempt to import module and report errors""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) @@ -430,7 +492,7 @@ def check_module_import(self, modinfo: ModuleInfo): logger.exception("Importing extension '%s' failed!", modinfo.name) raise - def check_module_cross(self, modinfo: ModuleInfo): + def check_module_cross(self, modinfo: ModuleInfo) -> None: """Sanity check for cross compiling""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) @@ -443,6 +505,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None: failed_name = f"{modinfo.name}_failed{self.ext_suffix}" builddir_path = self.get_location(modinfo) + assert builddir_path is not None if builddir_path.is_symlink(): symlink = builddir_path module_path = builddir_path.resolve().relative_to(os.getcwd()) @@ -466,7 +529,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None: logger.debug("Rename '%s' -> '%s'", module_path, failed_path) -def main(): +def main() -> None: args = parser.parse_args() if args.debug: args.verbose = True @@ -483,6 +546,9 @@ def main(): names = checker.list_module_names(all=True) for name in sorted(names): print(name) + elif args.generate_missing_stdlib_info: + checker.check() + checker.generate_missing_stdlib_info(args.with_missing_stdlib_config) else: checker.check() checker.summary(verbose=args.verbose) diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 3f49d8e7f2e..44ccf9708ad 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -8,21 +8,29 @@ import sys from collections import defaultdict from pathlib import Path -from typing import NamedTuple +from typing import NamedTuple, TypedDict class IgnoreRule(NamedTuple): file_path: str - count: int + count: int # type: ignore[assignment] ignore_all: bool = False is_directory: bool = False +class CompileWarning(TypedDict): + file: str + line: str + column: str + message: str + option: str + + def parse_warning_ignore_file(file_path: str) -> set[IgnoreRule]: """ Parses the warning ignore file and returns a set of IgnoreRules """ - files_with_expected_warnings = set() + files_with_expected_warnings: set[IgnoreRule] = set() with Path(file_path).open(encoding="UTF-8") as ignore_rules_file: files_with_expected_warnings = set() for i, line in enumerate(ignore_rules_file): @@ -46,7 +54,7 @@ def parse_warning_ignore_file(file_path: str) -> set[IgnoreRule]: ) sys.exit(1) if ignore_all: - count = 0 + count = "0" files_with_expected_warnings.add( IgnoreRule( @@ -61,7 +69,7 @@ def extract_warnings_from_compiler_output( compiler_output: str, compiler_output_type: str, path_prefix: str = "", -) -> list[dict]: +) -> list[CompileWarning]: """ Extracts warnings from the compiler output based on compiler output type. Removes path prefix from file paths if provided. @@ -78,8 +86,12 @@ def extract_warnings_from_compiler_output( r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: " r"(?P<message>.*) (?P<option>\[-[^\]]+\])$" ) + else: + raise RuntimeError( + f"Unsupported compiler output type: {compiler_output_type}", + ) compiled_regex = re.compile(regex_pattern) - compiler_warnings = [] + compiler_warnings: list[CompileWarning] = [] for i, line in enumerate(compiler_output.splitlines(), start=1): if match := compiled_regex.match(line): try: @@ -100,7 +112,9 @@ def extract_warnings_from_compiler_output( return compiler_warnings -def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: +def get_warnings_by_file( + warnings: list[CompileWarning], +) -> dict[str, list[CompileWarning]]: """ Returns a dictionary where the key is the file and the data is the warnings in that file. Does not include duplicate warnings for a @@ -138,7 +152,7 @@ def is_file_ignored( def get_unexpected_warnings( ignore_rules: set[IgnoreRule], - files_with_warnings: set[IgnoreRule], + files_with_warnings: dict[str, list[CompileWarning]], ) -> int: """ Returns failure status if warnings discovered in list of warnings @@ -180,7 +194,7 @@ def get_unexpected_warnings( def get_unexpected_improvements( ignore_rules: set[IgnoreRule], - files_with_warnings: set[IgnoreRule], + files_with_warnings: dict[str, list[CompileWarning]], ) -> int: """ Returns failure status if the number of warnings for a file is greater diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index cfdd55fd192..b5993d29b92 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -57,7 +57,7 @@ class Outputs: def compute_changes() -> None: target_branch, head_ref = git_refs() - if target_branch and head_ref: + if os.environ.get("GITHUB_EVENT_NAME", "") == "pull_request": # Getting changed files only makes sense on a pull request files = get_changed_files(target_branch, head_ref) outputs = process_changed_files(files) diff --git a/Tools/build/consts_getter.py b/Tools/build/consts_getter.py new file mode 100644 index 00000000000..4cd454448ba --- /dev/null +++ b/Tools/build/consts_getter.py @@ -0,0 +1,20 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +INTERNAL = ROOT / "Include" / "internal" + +def get_nsmallnegints_and_nsmallposints() -> tuple[int, int]: + nsmallposints = None + nsmallnegints = None + with open(INTERNAL / "pycore_runtime_structs.h") as infile: + for line in infile: + if line.startswith("#define _PY_NSMALLPOSINTS"): + nsmallposints = int(line.split()[-1]) + elif line.startswith("#define _PY_NSMALLNEGINTS"): + nsmallnegints = int(line.split()[-1]) + break + else: + raise NotImplementedError + assert nsmallposints + assert nsmallnegints + return nsmallnegints, nsmallposints diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 23f58447937..477c3d0f5b3 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -2,9 +2,12 @@ The script may be executed by _bootstrap_python interpreter. Shared library extension modules are not available in that case. -On Windows, and in cross-compilation cases, it is executed -by Python 3.10, and 3.11 features are not available. +Requires 3.11+ to be executed, +because relies on `code.co_qualname` and `code.co_exceptiontable`. """ + +from __future__ import annotations + import argparse import builtins import collections @@ -13,10 +16,15 @@ import re import time import types -from typing import TextIO +import consts_getter import umarshal +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any, TextIO + ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) verbose = False @@ -45,8 +53,8 @@ def make_string_literal(b: bytes) -> str: next_code_version = 1 -def get_localsplus(code: types.CodeType): - a = collections.defaultdict(int) +def get_localsplus(code: types.CodeType) -> tuple[tuple[str, ...], bytes]: + a: collections.defaultdict[str, int] = collections.defaultdict(int) for name in code.co_varnames: a[name] |= CO_FAST_LOCAL for name in code.co_cellvars: @@ -136,7 +144,7 @@ def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]: return identifiers, strings @contextlib.contextmanager - def indent(self) -> None: + def indent(self) -> Iterator[None]: save_level = self.level try: self.level += 1 @@ -148,7 +156,7 @@ def write(self, arg: str) -> None: self.file.writelines((" "*self.level, arg, "\n")) @contextlib.contextmanager - def block(self, prefix: str, suffix: str = "") -> None: + def block(self, prefix: str, suffix: str = "") -> Iterator[None]: self.write(prefix + " {") with self.indent(): yield @@ -250,9 +258,17 @@ def generate_code(self, name: str, code: types.CodeType) -> str: co_names = self.generate(name + "_names", code.co_names) co_filename = self.generate(name + "_filename", code.co_filename) co_name = self.generate(name + "_name", code.co_name) - co_qualname = self.generate(name + "_qualname", code.co_qualname) co_linetable = self.generate(name + "_linetable", code.co_linetable) - co_exceptiontable = self.generate(name + "_exceptiontable", code.co_exceptiontable) + # We use 3.10 for type checking, but this module requires 3.11 + # TODO: bump python version for this script. + co_qualname = self.generate( + name + "_qualname", + code.co_qualname, # type: ignore[attr-defined] + ) + co_exceptiontable = self.generate( + name + "_exceptiontable", + code.co_exceptiontable, # type: ignore[attr-defined] + ) # These fields are not directly accessible localsplusnames, localspluskinds = get_localsplus(code) co_localsplusnames = self.generate(name + "_localsplusnames", localsplusnames) @@ -347,7 +363,9 @@ def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: self.write(f".ob_digit = {{ {ds} }},") def generate_int(self, name: str, i: int) -> str: - if -5 <= i <= 256: + nsmallnegints, nsmallposints = consts_getter.get_nsmallnegints_and_nsmallposints() + + if -nsmallnegints <= i <= nsmallposints: return f"(PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + {i}]" if i >= 0: name = f"const_int_{i}" @@ -379,13 +397,13 @@ def generate_complex(self, name: str, z: complex) -> str: self.write(f".cval = {{ {z.real}, {z.imag} }},") return f"&{name}.ob_base" - def generate_frozenset(self, name: str, fs: frozenset[object]) -> str: + def generate_frozenset(self, name: str, fs: frozenset[Any]) -> str: try: - fs = sorted(fs) + fs_sorted = sorted(fs) except TypeError: # frozen set with incompatible types, fallback to repr() - fs = sorted(fs, key=repr) - ret = self.generate_tuple(name, tuple(fs)) + fs_sorted = sorted(fs, key=repr) + ret = self.generate_tuple(name, tuple(fs_sorted)) self.write("// TODO: The above tuple should be a frozenset") return ret @@ -402,7 +420,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, (types.CodeType, umarshal.Code)) : + if isinstance(obj, types.CodeType) : val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) @@ -458,7 +476,7 @@ def decode_frozen_data(source: str) -> types.CodeType: if re.match(FROZEN_DATA_LINE, line): values.extend([int(x) for x in line.split(",") if x.strip()]) data = bytes(values) - return umarshal.loads(data) + return umarshal.loads(data) # type: ignore[no-any-return] def generate(args: list[str], output: TextIO) -> None: @@ -494,12 +512,12 @@ def generate(args: list[str], output: TextIO) -> None: help="Input file and module name (required) in file:modname format") @contextlib.contextmanager -def report_time(label: str): - t0 = time.time() +def report_time(label: str) -> Iterator[None]: + t0 = time.perf_counter() try: yield finally: - t1 = time.time() + t1 = time.perf_counter() if verbose: print(f"{label}: {t1-t0:.3f} sec") diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 0da6c2948d6..ed9ab2844d2 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -3,6 +3,8 @@ # Script initially imported from: # https://github.com/FFY00/python-instrospection/blob/main/python_introspection/scripts/generate-build-details.py +from __future__ import annotations + import argparse import collections import importlib.machinery @@ -11,19 +13,23 @@ import sys import sysconfig +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Any -def version_info_to_dict(obj): # (object) -> dict[str, Any] + +def version_info_to_dict(obj: sys._version_info) -> dict[str, Any]: field_names = ('major', 'minor', 'micro', 'releaselevel', 'serial') return {field: getattr(obj, field) for field in field_names} -def get_dict_key(container, key): # (dict[str, Any], str) -> dict[str, Any] +def get_dict_key(container: dict[str, Any], key: str) -> dict[str, Any]: for part in key.split('.'): container = container[part] return container -def generate_data(schema_version): +def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: """Generate the build-details.json data (PEP 739). :param schema_version: The schema version of the data we want to generate. @@ -32,7 +38,9 @@ def generate_data(schema_version): if schema_version != '1.0': raise ValueError(f'Unsupported schema_version: {schema_version}') - data = collections.defaultdict(lambda: collections.defaultdict(dict)) + data: collections.defaultdict[str, Any] = collections.defaultdict( + lambda: collections.defaultdict(dict), + ) data['schema_version'] = schema_version @@ -47,7 +55,7 @@ def generate_data(schema_version): data['language']['version'] = sysconfig.get_python_version() data['language']['version_info'] = version_info_to_dict(sys.version_info) - data['implementation'] = vars(sys.implementation) + data['implementation'] = vars(sys.implementation).copy() data['implementation']['version'] = version_info_to_dict(sys.implementation.version) # Fix cross-compilation if '_multiarch' in data['implementation']: @@ -67,7 +75,7 @@ def generate_data(schema_version): PY3LIBRARY = sysconfig.get_config_var('PY3LIBRARY') LIBPYTHON = sysconfig.get_config_var('LIBPYTHON') LIBPC = sysconfig.get_config_var('LIBPC') - INCLUDEDIR = sysconfig.get_config_var('INCLUDEDIR') + INCLUDEPY = sysconfig.get_config_var('INCLUDEPY') if os.name == 'posix': # On POSIX, LIBRARY is always the static library, while LDLIBRARY is the @@ -96,7 +104,7 @@ def generate_data(schema_version): data['abi']['extension_suffix'] = sysconfig.get_config_var('EXT_SUFFIX') # EXTENSION_SUFFIXES has been constant for a long time, and currently we - # don't have a better information source to find the stable ABI suffix. + # don't have a better information source to find the stable ABI suffix. for suffix in importlib.machinery.EXTENSION_SUFFIXES: if suffix.startswith('.abi'): data['abi']['stable_abi_suffix'] = suffix @@ -115,44 +123,62 @@ def generate_data(schema_version): if has_static_library: data['libpython']['static'] = os.path.join(LIBDIR, LIBRARY) - data['c_api']['include'] = INCLUDEDIR + data['c_api']['headers'] = INCLUDEPY if LIBPC: data['c_api']['pkgconfig_path'] = LIBPC return data -def make_paths_relative(data, config_path=None): # (dict[str, Any], str | None) -> None +def make_paths_relative(data: dict[str, Any], config_path: str | None = None) -> None: # Make base_prefix relative to the config_path directory if config_path: - data['base_prefix'] = os.path.relpath(data['base_prefix'], os.path.dirname(config_path)) + data['base_prefix'] = relative_path(data['base_prefix'], + os.path.dirname(config_path)) + base_prefix = data['base_prefix'] + # Update path values to make them relative to base_prefix - PATH_KEYS = [ + PATH_KEYS = ( 'base_interpreter', 'libpython.dynamic', 'libpython.dynamic_stableabi', 'libpython.static', 'c_api.headers', 'c_api.pkgconfig_path', - ] + ) for entry in PATH_KEYS: - parent, _, child = entry.rpartition('.') + *parents, child = entry.split('.') # Get the key container object try: container = data - for part in parent.split('.'): + for part in parents: container = container[part] + if child not in container: + raise KeyError(child) current_path = container[child] except KeyError: continue # Get the relative path - new_path = os.path.relpath(current_path, data['base_prefix']) + new_path = relative_path(current_path, base_prefix) # Join '.' so that the path is formated as './path' instead of 'path' new_path = os.path.join('.', new_path) container[child] = new_path -def main(): # () -> None +def relative_path(path: str, base: str) -> str: + if os.name != 'nt': + return os.path.relpath(path, base) + + # There are no relative paths between drives on Windows. + path_drv, _ = os.path.splitdrive(path) + base_drv, _ = os.path.splitdrive(base) + if path_drv.lower() == base_drv.lower(): + return os.path.relpath(path, base) + + return path + + +def main() -> None: parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('location') parser.add_argument( @@ -178,8 +204,9 @@ def main(): # () -> None make_paths_relative(data, args.config_file_path) json_output = json.dumps(data, indent=2) - with open(args.location, 'w') as f: - print(json_output, file=f) + with open(args.location, 'w', encoding='utf-8') as f: + f.write(json_output) + f.write('\n') if __name__ == '__main__': diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 94905b3756d..5b188e338ba 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -3,6 +3,8 @@ import os.path import re +import consts_getter + SCRIPT_NAME = 'Tools/build/generate_global_objects.py' __file__ = os.path.abspath(__file__) ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) @@ -274,20 +276,7 @@ def generate_global_strings(identifiers, strings): def generate_runtime_init(identifiers, strings): - # First get some info from the declarations. - nsmallposints = None - nsmallnegints = None - with open(os.path.join(INTERNAL, 'pycore_runtime_structs.h')) as infile: - for line in infile: - if line.startswith('#define _PY_NSMALLPOSINTS'): - nsmallposints = int(line.split()[-1]) - elif line.startswith('#define _PY_NSMALLNEGINTS'): - nsmallnegints = int(line.split()[-1]) - break - else: - raise NotImplementedError - assert nsmallposints - assert nsmallnegints + nsmallnegints, nsmallposints = consts_getter.get_nsmallnegints_and_nsmallposints() # Then target the runtime initializer. filename = os.path.join(INTERNAL, 'pycore_runtime_init_generated.h') diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index db01426e972..f5a2f26cbb0 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -4,10 +4,13 @@ import hashlib import json import os +import random import re import subprocess import sys +import time import typing +import urllib.error import urllib.request from pathlib import Path, PurePosixPath, PureWindowsPath @@ -161,6 +164,23 @@ def get_externals() -> list[str]: return externals +def download_with_retries(download_location: str, + max_retries: int = 7, + base_delay: float = 2.25, + max_jitter: float = 1.0) -> typing.Any: + """Download a file with exponential backoff retry.""" + for attempt in range(max_retries + 1): + try: + resp = urllib.request.urlopen(download_location) + except (urllib.error.URLError, ConnectionError) as ex: + if attempt == max_retries: + msg = f"Download from {download_location} failed." + raise OSError(msg) from ex + time.sleep(base_delay**attempt + random.uniform(0, max_jitter)) + else: + return resp + + def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: """Make a bunch of assertions about the SBOM package data to ensure it's consistent.""" @@ -175,7 +195,7 @@ def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: # and that the download URL is valid. if "checksums" not in package or "CI" in os.environ: download_location = package["downloadLocation"] - resp = urllib.request.urlopen(download_location) + resp = download_with_retries(download_location) error_if(resp.status != 200, f"Couldn't access URL: {download_location}'") package["checksums"] = [{ @@ -222,14 +242,14 @@ def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: ) # libexpat specifies its expected rev in a refresh script. - if package["name"] == "libexpat": + if package["name"] == "expat": libexpat_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/expat/refresh.sh").read_text() libexpat_expected_version_match = re.search( r"expected_libexpat_version=\"([0-9]+\.[0-9]+\.[0-9]+)\"", libexpat_refresh_sh ) libexpat_expected_sha256_match = re.search( - r"expected_libexpat_sha256=\"[a-f0-9]{40}\"", + r"expected_libexpat_sha256=\"([a-f0-9]{64})\"", libexpat_refresh_sh ) libexpat_expected_version = libexpat_expected_version_match and libexpat_expected_version_match.group(1) diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 88414cdbb37..bda72539640 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -1,9 +1,12 @@ # This script lists the names of standard library modules # to update Python/stdlib_module_names.h +from __future__ import annotations + import _imp import os.path import sys import sysconfig +from typing import TextIO from check_extension_modules import ModuleChecker @@ -48,12 +51,12 @@ } # Built-in modules -def list_builtin_modules(names): +def list_builtin_modules(names: set[str]) -> None: names |= set(sys.builtin_module_names) # Pure Python modules (Lib/*.py) -def list_python_modules(names): +def list_python_modules(names: set[str]) -> None: for filename in os.listdir(STDLIB_PATH): if not filename.endswith(".py"): continue @@ -62,7 +65,7 @@ def list_python_modules(names): # Packages in Lib/ -def list_packages(names): +def list_packages(names: set[str]) -> None: for name in os.listdir(STDLIB_PATH): if name in IGNORE: continue @@ -76,16 +79,16 @@ def list_packages(names): # Built-in and extension modules built by Modules/Setup* # includes Windows and macOS extensions. -def list_modules_setup_extensions(names): +def list_modules_setup_extensions(names: set[str]) -> None: checker = ModuleChecker() names.update(checker.list_module_names(all=True)) # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). # Use the "./Programs/_testembed list_frozen" command. -def list_frozen(names): +def list_frozen(names: set[str]) -> None: submodules = set() - for name in _imp._frozen_module_names(): + for name in _imp._frozen_module_names(): # type: ignore[attr-defined] # To skip __hello__, __hello_alias__ and etc. if name.startswith('__'): continue @@ -101,8 +104,8 @@ def list_frozen(names): raise Exception(f'unexpected frozen submodules: {sorted(submodules)}') -def list_modules(): - names = set() +def list_modules() -> set[str]: + names: set[str] = set() list_builtin_modules(names) list_modules_setup_extensions(names) @@ -127,7 +130,7 @@ def list_modules(): return names -def write_modules(fp, names): +def write_modules(fp: TextIO, names: set[str]) -> None: print(f"// Auto-generated by {SCRIPT_NAME}.", file=fp) print("// List used to create sys.stdlib_module_names.", file=fp) @@ -138,7 +141,7 @@ def write_modules(fp, names): print("};", file=fp) -def main(): +def main() -> None: if not sysconfig.is_python_build(): print(f"ERROR: {sys.executable} is not a Python build", file=sys.stderr) diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 06224163884..7d341afd1cd 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -1,7 +1,20 @@ [mypy] + +# Please, when adding new files here, also add them to: +# .github/workflows/mypy.yml files = + Tools/build/check_extension_modules.py, + Tools/build/check_warnings.py, Tools/build/compute-changes.py, - Tools/build/generate_sbom.py + Tools/build/consts_getter.py, + Tools/build/deepfreeze.py, + Tools/build/generate-build-details.py, + Tools/build/generate_sbom.py, + Tools/build/generate_stdlib_module_names.py, + Tools/build/verify_ensurepip_wheels.py, + Tools/build/update_file.py, + Tools/build/umarshal.py + pretty = True # Make sure Python can still be built @@ -10,6 +23,8 @@ python_version = 3.10 # ...And be strict: strict = True +strict_bytes = True +local_partial_types = True extra_checks = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 9a360412a73..424fa6ad4a1 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -21,8 +21,6 @@ }) IGNORED_EXTENSION = "_ctypes_test" -# Ignore constructor and destructor functions -IGNORED_SYMBOLS = {'_init', '_fini'} def is_local_symbol_type(symtype): @@ -34,19 +32,12 @@ def is_local_symbol_type(symtype): if symtype.islower() and symtype not in "uvw": return True - # Ignore the initialized data section (d and D) and the BSS data - # section. For example, ignore "__bss_start (type: B)" - # and "_edata (type: D)". - if symtype in "bBdD": - return True - return False def get_exported_symbols(library, dynamic=False): print(f"Check that {library} only exports symbols starting with Py or _Py") - # Only look at dynamic symbols args = ['nm', '--no-sort'] if dynamic: args.append('--dynamic') @@ -89,8 +80,6 @@ def get_smelly_symbols(stdout, dynamic=False): if is_local_symbol_type(symtype): local_symbols.append(result) - elif symbol in IGNORED_SYMBOLS: - local_symbols.append(result) else: smelly_symbols.append(result) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 1ddd76cdd9b..39115b331ba 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -232,7 +232,7 @@ def sort_key(item): 'data': 'data', 'struct': 'type', 'macro': 'macro', - # 'const': 'const', # all undocumented + 'const': 'macro', 'typedef': 'type', } diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index 679fa7caf9f..865cffc2440 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -145,12 +145,12 @@ def r_PyLong(self) -> int: def r_float_bin(self) -> float: buf = self.r_string(8) import struct # Lazy import to avoid breaking UNIX build - return struct.unpack("d", buf)[0] + return struct.unpack("d", buf)[0] # type: ignore[no-any-return] def r_float_str(self) -> float: n = self.r_byte() buf = self.r_string(n) - return ast.literal_eval(buf.decode("ascii")) + return ast.literal_eval(buf.decode("ascii")) # type: ignore[no-any-return] def r_ref_reserve(self, flag: int) -> int: if flag: @@ -306,7 +306,7 @@ def loads(data: bytes) -> Any: return r.r_object() -def main(): +def main() -> None: # Test import marshal import pprint @@ -314,8 +314,9 @@ def main(): data = marshal.dumps(sample) retval = loads(data) assert retval == sample, retval - sample = main.__code__ - data = marshal.dumps(sample) + + sample2 = main.__code__ + data = marshal.dumps(sample2) retval = loads(data) assert isinstance(retval, Code), retval pprint.pprint(retval.__dict__) diff --git a/Tools/build/update_file.py b/Tools/build/update_file.py index b4182c1d0cb..b4a5fb6e778 100644 --- a/Tools/build/update_file.py +++ b/Tools/build/update_file.py @@ -6,14 +6,27 @@ actually change the in-tree generated code. """ +from __future__ import annotations + import contextlib import os import os.path import sys +TYPE_CHECKING = False +if TYPE_CHECKING: + import typing + from collections.abc import Iterator + from io import TextIOWrapper + + _Outcome: typing.TypeAlias = typing.Literal['created', 'updated', 'same'] + @contextlib.contextmanager -def updating_file_with_tmpfile(filename, tmpfile=None): +def updating_file_with_tmpfile( + filename: str, + tmpfile: str | None = None, +) -> Iterator[tuple[TextIOWrapper, TextIOWrapper]]: """A context manager for updating a file via a temp file. The context manager provides two open files: the source file open @@ -46,13 +59,18 @@ def updating_file_with_tmpfile(filename, tmpfile=None): update_file_with_tmpfile(filename, tmpfile) -def update_file_with_tmpfile(filename, tmpfile, *, create=False): +def update_file_with_tmpfile( + filename: str, + tmpfile: str, + *, + create: bool = False, +) -> _Outcome: try: targetfile = open(filename, 'rb') except FileNotFoundError: if not create: raise # re-raise - outcome = 'created' + outcome: _Outcome = 'created' os.replace(tmpfile, filename) else: with targetfile: diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py index a37da2f7075..46c42916d93 100755 --- a/Tools/build/verify_ensurepip_wheels.py +++ b/Tools/build/verify_ensurepip_wheels.py @@ -20,13 +20,13 @@ GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true" -def print_notice(file_path: str, message: str) -> None: +def print_notice(file_path: str | Path, message: str) -> None: if GITHUB_ACTIONS: message = f"::notice file={file_path}::{message}" print(message, end="\n\n") -def print_error(file_path: str, message: str) -> None: +def print_error(file_path: str | Path, message: str) -> None: if GITHUB_ACTIONS: message = f"::error file={file_path}::{message}" print(message, end="\n\n") @@ -67,6 +67,7 @@ def verify_wheel(package_name: str) -> bool: return False release_files = json.loads(raw_text)["releases"][package_version] + expected_digest = "" for release_info in release_files: if package_path.name != release_info["filename"]: continue @@ -95,6 +96,7 @@ def verify_wheel(package_name: str) -> bool: return True + if __name__ == "__main__": exit_status = int(not verify_wheel("pip")) raise SystemExit(exit_status) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index ff4f303c4a2..f3f09107aef 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -116,6 +116,8 @@ * alt impl using a state machine (& tokenizer or split on delimiters) """ +import textwrap + from ..info import ParsedItem from ._info import SourceInfo @@ -208,7 +210,27 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): return # At this point either the file ended prematurely # or there's "too much" text. - filename, lno, text = srcinfo.filename, srcinfo._start, srcinfo.text + filename, lno_from, lno_to = srcinfo.filename, srcinfo.start, srcinfo.end + text = srcinfo.text if len(text) > 500: text = text[:500] + '...' - raise Exception(f'unmatched text ({filename} starting at line {lno}):\n{text}') + + if srcinfo.too_much_text(maxtext): + msg = f''' + too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py + {filename} starting at line {lno_from} to {lno_to} + has code with length {len(text)} greater than {maxtext}: + {text} + ''' + raise RuntimeError(textwrap.dedent(msg)) + + if srcinfo.too_many_lines(maxlines): + msg = f''' + too many lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py + {filename} starting at line {lno_from} to {lno_to} + has code with number of lines {lno_to - lno_from} greater than {maxlines}: + {text} + ''' + raise RuntimeError(textwrap.dedent(msg)) + + raise RuntimeError(f'unmatched text ({filename} starting at line {lno_from}):\n{text}') diff --git a/Tools/c-analyzer/c_parser/parser/_info.py b/Tools/c-analyzer/c_parser/parser/_info.py index 340223db933..13b192e2ce9 100644 --- a/Tools/c-analyzer/c_parser/parser/_info.py +++ b/Tools/c-analyzer/c_parser/parser/_info.py @@ -123,10 +123,16 @@ def resolve(self, kind, data, name, parent=None): def done(self): self._set_ready() + def too_much_text(self, maxtext): + return maxtext and len(self.text) > maxtext + + def too_many_lines(self, maxlines): + return maxlines and self.end - self.start > maxlines + def too_much(self, maxtext, maxlines): - if maxtext and len(self.text) > maxtext: + if self.too_much_text(maxtext): pass - elif maxlines and self.end - self.start > maxlines: + elif self.too_many_lines(maxlines): pass else: return False diff --git a/Tools/c-analyzer/c_parser/preprocessor/__init__.py b/Tools/c-analyzer/c_parser/preprocessor/__init__.py index 30a86cbd7dc..f8d2f805cb1 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/__init__.py +++ b/Tools/c-analyzer/c_parser/preprocessor/__init__.py @@ -16,6 +16,7 @@ from . import ( pure as _pure, gcc as _gcc, + clang as _clang, ) @@ -234,7 +235,7 @@ def handling_errors(ignore_exc=None, *, log_err=None): 'bcpp': None, # aliases/extras: 'gcc': _gcc.preprocess, - 'clang': None, + 'clang': _clang.preprocess, } diff --git a/Tools/c-analyzer/c_parser/preprocessor/clang.py b/Tools/c-analyzer/c_parser/preprocessor/clang.py new file mode 100644 index 00000000000..574a23f8f6d --- /dev/null +++ b/Tools/c-analyzer/c_parser/preprocessor/clang.py @@ -0,0 +1,104 @@ +import os.path +import re, sys + +from . import common as _common +from . import gcc as _gcc + +_normpath = _gcc._normpath + +TOOL = 'clang' + +META_FILES = { + '<built-in>', + '<command line>', +} + + +def preprocess(filename, + incldirs=None, + includes=None, + macros=None, + samefiles=None, + cwd=None, + ): + if not cwd or not os.path.isabs(cwd): + cwd = os.path.abspath(cwd or '.') + filename = _normpath(filename, cwd) + + postargs = _gcc.POST_ARGS + basename = os.path.basename(filename) + dirname = os.path.basename(os.path.dirname(filename)) + if (basename not in _gcc.FILES_WITHOUT_INTERNAL_CAPI + and dirname not in _gcc.DIRS_WITHOUT_INTERNAL_CAPI): + postargs += ('-DPy_BUILD_CORE=1',) + + text = _common.preprocess( + TOOL, + filename, + incldirs=incldirs, + includes=includes, + macros=macros, + #preargs=PRE_ARGS, + postargs=postargs, + executable=['clang'], + compiler='unix', + cwd=cwd, + ) + return _iter_lines(text, filename, samefiles, cwd) + + +EXIT_MARKERS = {'# 2 "<built-in>" 2', '# 3 "<built-in>" 2', '# 4 "<built-in>" 2'} + + +def _iter_lines(text, reqfile, samefiles, cwd, raw=False): + lines = iter(text.splitlines()) + + # The first line is special. + # The subsequent lines are consistent. + firstlines = [ + f'# 1 "{reqfile}"', + '# 1 "<built-in>" 1', + '# 1 "<built-in>" 3', + '# 370 "<built-in>" 3', + '# 1 "<command line>" 1', + '# 1 "<built-in>" 2', + ] + for expected in firstlines: + line = next(lines) + if line != expected: + raise NotImplementedError((line, expected)) + + # Do all the CLI-provided includes. + filter_reqfile = (lambda f: _gcc._filter_reqfile(f, reqfile, samefiles)) + make_info = (lambda lno: _common.FileInfo(reqfile, lno)) + last = None + for line in lines: + assert last != reqfile, (last,) + # NOTE:condition is clang specific + if not line: + continue + lno, included, flags = _gcc._parse_marker_line(line, reqfile) + if not included: + raise NotImplementedError((line,)) + if included == reqfile: + # This will be the last one. + assert 2 in flags, (line, flags) + else: + # NOTE:first condition is specific to clang + if _normpath(included, cwd) == reqfile: + assert 1 in flags or 2 in flags, (line, flags, included, reqfile) + else: + assert 1 in flags, (line, flags, included, reqfile) + yield from _gcc._iter_top_include_lines( + lines, + _normpath(included, cwd), + cwd, + filter_reqfile, + make_info, + raw, + EXIT_MARKERS + ) + last = included + # The last one is always the requested file. + # NOTE:_normpath is clang specific + assert _normpath(included, cwd) == reqfile, (line,) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index d20cd19f6e6..4a55a1a24ee 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -65,6 +65,8 @@ '-E', ) +EXIT_MARKERS = {'# 0 "<command-line>" 2', '# 1 "<command-line>" 2'} + def preprocess(filename, incldirs=None, @@ -138,6 +140,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): filter_reqfile, make_info, raw, + EXIT_MARKERS ) last = included # The last one is always the requested file. @@ -146,7 +149,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): def _iter_top_include_lines(lines, topfile, cwd, filter_reqfile, make_info, - raw): + raw, exit_markers): partial = 0 # depth files = [topfile] # We start at 1 in case there are source lines (including blank ones) @@ -154,12 +157,20 @@ def _iter_top_include_lines(lines, topfile, cwd, # _parse_marker_line() that the preprocessor reported lno as 1. lno = 1 for line in lines: - if line == '# 0 "<command-line>" 2' or line == '# 1 "<command-line>" 2': + if line in exit_markers: # We're done with this top-level include. return _lno, included, flags = _parse_marker_line(line) if included: + # HACK: + # Mixes curses.h and ncurses.h marker lines + # gcc silently passes this, while clang fails + # See: /Include/py_curses.h #if-elif directives + # And compare with preprocessor output + if os.path.basename(included) == 'curses.h': + included = os.path.join(os.path.dirname(included), 'ncurses.h') + lno = _lno included = _normpath(included, cwd) # We hit a marker line. diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py index 6204353e9bd..6f0f4648928 100644 --- a/Tools/c-analyzer/cpython/_analyzer.py +++ b/Tools/c-analyzer/cpython/_analyzer.py @@ -67,6 +67,7 @@ 'PyMethodDef', 'PyMethodDef[]', 'PyMemberDef[]', + 'PyGetSetDef', 'PyGetSetDef[]', 'PyNumberMethods', 'PySequenceMethods', diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 037fe11ea22..6332dec3aef 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -1,5 +1,4 @@ import os.path -import re from c_parser.preprocessor import ( get_preprocessor as _get_preprocessor, @@ -18,16 +17,16 @@ def _abs(relfile): return os.path.join(REPO_ROOT, relfile) -def clean_lines(text): - """Clear out comments, blank lines, and leading/trailing whitespace.""" - lines = (line.strip() for line in text.splitlines()) - lines = (line.partition('#')[0].rstrip() - for line in lines - if line and not line.startswith('#')) - glob_all = f'{GLOB_ALL} ' - lines = (re.sub(r'^[*] ', glob_all, line) for line in lines) - lines = (_abs(line) for line in lines) - return list(lines) +def format_conf_lines(lines): + """Format conf entries for use in a TSV table.""" + return [_abs(entry) for entry in lines] + + +def format_tsv_lines(lines): + """Format entries for use in a TSV table.""" + lines = ((_abs(first), *rest) for first, *rest in lines) + lines = map('\t'.join, lines) + return list(map(str.rstrip, lines)) ''' @@ -47,252 +46,246 @@ def clean_lines(text): ''' # XXX Handle these. -# Tab separated: -EXCLUDED = clean_lines(''' -# @begin=conf@ +EXCLUDED = format_conf_lines([ + # macOS + 'Modules/_scproxy.c', # SystemConfiguration/SystemConfiguration.h -# OSX -Modules/_scproxy.c # SystemConfiguration/SystemConfiguration.h + # Windows + 'Modules/_winapi.c', # windows.h + 'Modules/expat/winconfig.h', + 'Modules/overlapped.c', # winsock.h + 'Python/dynload_win.c', # windows.h + 'Python/thread_nt.h', -# Windows -Modules/_winapi.c # windows.h -Modules/expat/winconfig.h -Modules/overlapped.c # winsock.h -Python/dynload_win.c # windows.h -Python/thread_nt.h + # other OS-dependent + 'Python/dynload_aix.c', # sys/ldr.h + 'Python/dynload_dl.c', # dl.h + 'Python/dynload_hpux.c', # dl.h + 'Python/emscripten_signal.c', + 'Python/emscripten_syscalls.c', + 'Python/emscripten_trampoline_inner.c', + 'Python/thread_pthread.h', + 'Python/thread_pthread_stubs.h', -# other OS-dependent -Python/dynload_aix.c # sys/ldr.h -Python/dynload_dl.c # dl.h -Python/dynload_hpux.c # dl.h -Python/emscripten_signal.c -Python/thread_pthread.h -Python/thread_pthread_stubs.h + # only huge constants (safe but parsing is slow) + 'Modules/_ssl_data_*.h', + 'Modules/cjkcodecs/mappings_*.h', + 'Modules/unicodedata_db.h', + 'Modules/unicodename_db.h', + 'Objects/unicodetype_db.h', -# only huge constants (safe but parsing is slow) -Modules/_ssl_data_*.h -Modules/cjkcodecs/mappings_*.h -Modules/unicodedata_db.h -Modules/unicodename_db.h -Objects/unicodetype_db.h + # generated + 'Python/deepfreeze/*.c', + 'Python/frozen_modules/*.h', + 'Python/generated_cases.c.h', + 'Python/executor_cases.c.h', + 'Python/optimizer_cases.c.h', + 'Python/opcode_targets.h', + # XXX: Throws errors if PY_VERSION_HEX is not mocked out + 'Modules/clinic/_testclinic_depr.c.h', -# generated -Python/deepfreeze/*.c -Python/frozen_modules/*.h -Python/generated_cases.c.h -Python/executor_cases.c.h -Python/optimizer_cases.c.h + # not actually source + 'Python/bytecodes.c', + 'Python/optimizer_bytecodes.c', -# not actually source -Python/bytecodes.c -Python/optimizer_bytecodes.c - -# mimalloc -Objects/mimalloc/*.c -Include/internal/mimalloc/*.h -Include/internal/mimalloc/mimalloc/*.h - -# @end=conf@ -''') + # mimalloc + 'Objects/mimalloc/*.c', + 'Include/internal/mimalloc/*.h', + 'Include/internal/mimalloc/mimalloc/*.h', +]) # XXX Fix the parser. -EXCLUDED += clean_lines(''' -# The tool should be able to parse these... +EXCLUDED += format_conf_lines([ + # The tool should be able to parse these... -# The problem with xmlparse.c is that something -# has gone wrong where # we handle "maybe inline actual" -# in Tools/c-analyzer/c_parser/parser/_global.py. -Modules/expat/internal.h -Modules/expat/xmlparse.c -''') + # The problem with xmlparse.c is that something + # has gone wrong where # we handle "maybe inline actual" + # in Tools/c-analyzer/c_parser/parser/_global.py. + 'Modules/expat/internal.h', + 'Modules/expat/xmlparse.c', +]) -INCL_DIRS = clean_lines(''' -# @begin=tsv@ +INCL_DIRS = format_tsv_lines([ + # (glob, dirname) -glob dirname -* . -* ./Include -* ./Include/internal -* ./Include/internal/mimalloc + ('*', '.'), + ('*', './Include'), + ('*', './Include/internal'), + ('*', './Include/internal/mimalloc'), -Modules/_decimal/**/*.c Modules/_decimal/libmpdec -Modules/_elementtree.c Modules/expat -Modules/_hacl/*.c Modules/_hacl/include -Modules/_hacl/*.c Modules/_hacl/ -Modules/_hacl/*.h Modules/_hacl/include -Modules/_hacl/*.h Modules/_hacl/ -Modules/md5module.c Modules/_hacl/include -Modules/sha1module.c Modules/_hacl/include -Modules/sha2module.c Modules/_hacl/include -Modules/sha3module.c Modules/_hacl/include -Modules/blake2module.c Modules/_hacl/include -Modules/hmacmodule.c Modules/_hacl/include -Objects/stringlib/*.h Objects + ('Modules/_decimal/**/*.c', 'Modules/_decimal/libmpdec'), -# possible system-installed headers, just in case -Modules/_tkinter.c /usr/include/tcl8.6 -Modules/_uuidmodule.c /usr/include/uuid -Modules/tkappinit.c /usr/include/tcl + ('Modules/_elementtree.c', 'Modules/expat'), + ('Modules/pyexpat.c', 'Modules/expat'), -# @end=tsv@ -''')[1:] + ('Modules/_hacl/*.c', 'Modules/_hacl/include'), + ('Modules/_hacl/*.c', 'Modules/_hacl/'), + ('Modules/_hacl/*.h', 'Modules/_hacl/include'), + ('Modules/_hacl/*.h', 'Modules/_hacl/'), + ('Modules/md5module.c', 'Modules/_hacl/include'), + ('Modules/sha1module.c', 'Modules/_hacl/include'), + ('Modules/sha2module.c', 'Modules/_hacl/include'), + ('Modules/sha3module.c', 'Modules/_hacl/include'), + ('Modules/blake2module.c', 'Modules/_hacl/include'), + ('Modules/hmacmodule.c', 'Modules/_hacl/include'), -INCLUDES = clean_lines(''' -# @begin=tsv@ + ('Objects/stringlib/*.h', 'Objects'), -glob include + # possible system-installed headers, just in case + ('Modules/_tkinter.c', '/usr/include/tcl8.6'), + ('Modules/_uuidmodule.c', '/usr/include/uuid'), + ('Modules/tkappinit.c', '/usr/include/tcl'), -**/*.h Python.h -Include/**/*.h object.h +]) -# for Py_HAVE_CONDVAR -Include/internal/pycore_gil.h pycore_condvar.h -Python/thread_pthread.h pycore_condvar.h +INCLUDES = format_tsv_lines([ + # (glob, include) -# other + ('**/*.h', 'Python.h'), + ('Include/**/*.h', 'object.h'), -Objects/stringlib/join.h stringlib/stringdefs.h -Objects/stringlib/ctype.h stringlib/stringdefs.h -Objects/stringlib/transmogrify.h stringlib/stringdefs.h -#Objects/stringlib/fastsearch.h stringlib/stringdefs.h -#Objects/stringlib/count.h stringlib/stringdefs.h -#Objects/stringlib/find.h stringlib/stringdefs.h -#Objects/stringlib/partition.h stringlib/stringdefs.h -#Objects/stringlib/split.h stringlib/stringdefs.h -Objects/stringlib/fastsearch.h stringlib/ucs1lib.h -Objects/stringlib/count.h stringlib/ucs1lib.h -Objects/stringlib/find.h stringlib/ucs1lib.h -Objects/stringlib/partition.h stringlib/ucs1lib.h -Objects/stringlib/split.h stringlib/ucs1lib.h -Objects/stringlib/find_max_char.h Objects/stringlib/ucs1lib.h -Objects/stringlib/count.h Objects/stringlib/fastsearch.h -Objects/stringlib/find.h Objects/stringlib/fastsearch.h -Objects/stringlib/partition.h Objects/stringlib/fastsearch.h -Objects/stringlib/replace.h Objects/stringlib/fastsearch.h -Objects/stringlib/repr.h Objects/stringlib/fastsearch.h -Objects/stringlib/split.h Objects/stringlib/fastsearch.h + # for Py_HAVE_CONDVAR + ('Include/internal/pycore_gil.h', 'pycore_condvar.h'), + ('Python/thread_pthread.h', 'pycore_condvar.h'), -# @end=tsv@ -''')[1:] + # other -MACROS = clean_lines(''' -# @begin=tsv@ + ('Objects/stringlib/join.h', 'stringlib/stringdefs.h'), + ('Objects/stringlib/ctype.h', 'stringlib/stringdefs.h'), + ('Objects/stringlib/transmogrify.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/fastsearch.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/count.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/find.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/partition.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/split.h', 'stringlib/stringdefs.h'), + ('Objects/stringlib/fastsearch.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/count.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/find.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/partition.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/split.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/find_max_char.h', 'Objects/stringlib/ucs1lib.h'), + ('Objects/stringlib/count.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/find.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/partition.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/replace.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/repr.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/split.h', 'Objects/stringlib/fastsearch.h'), +]) -glob name value +MACROS = format_tsv_lines([ + # (glob, name, value) -Include/internal/*.h Py_BUILD_CORE 1 -Python/**/*.c Py_BUILD_CORE 1 -Python/**/*.h Py_BUILD_CORE 1 -Parser/**/*.c Py_BUILD_CORE 1 -Parser/**/*.h Py_BUILD_CORE 1 -Objects/**/*.c Py_BUILD_CORE 1 -Objects/**/*.h Py_BUILD_CORE 1 + ('Include/internal/*.h', 'Py_BUILD_CORE', '1'), + ('Python/**/*.c', 'Py_BUILD_CORE', '1'), + ('Python/**/*.h', 'Py_BUILD_CORE', '1'), + ('Parser/**/*.c', 'Py_BUILD_CORE', '1'), + ('Parser/**/*.h', 'Py_BUILD_CORE', '1'), + ('Objects/**/*.c', 'Py_BUILD_CORE', '1'), + ('Objects/**/*.h', 'Py_BUILD_CORE', '1'), -Modules/_asynciomodule.c Py_BUILD_CORE 1 -Modules/_codecsmodule.c Py_BUILD_CORE 1 -Modules/_collectionsmodule.c Py_BUILD_CORE 1 -Modules/_ctypes/_ctypes.c Py_BUILD_CORE 1 -Modules/_ctypes/cfield.c Py_BUILD_CORE 1 -Modules/_cursesmodule.c Py_BUILD_CORE 1 -Modules/_datetimemodule.c Py_BUILD_CORE 1 -Modules/_functoolsmodule.c Py_BUILD_CORE 1 -Modules/_heapqmodule.c Py_BUILD_CORE 1 -Modules/_io/*.c Py_BUILD_CORE 1 -Modules/_io/*.h Py_BUILD_CORE 1 -Modules/_localemodule.c Py_BUILD_CORE 1 -Modules/_operator.c Py_BUILD_CORE 1 -Modules/_posixsubprocess.c Py_BUILD_CORE 1 -Modules/_sre/sre.c Py_BUILD_CORE 1 -Modules/_threadmodule.c Py_BUILD_CORE 1 -Modules/_tracemalloc.c Py_BUILD_CORE 1 -Modules/_weakref.c Py_BUILD_CORE 1 -Modules/_zoneinfo.c Py_BUILD_CORE 1 -Modules/atexitmodule.c Py_BUILD_CORE 1 -Modules/cmathmodule.c Py_BUILD_CORE 1 -Modules/faulthandler.c Py_BUILD_CORE 1 -Modules/gcmodule.c Py_BUILD_CORE 1 -Modules/getpath.c Py_BUILD_CORE 1 -Modules/getpath_noop.c Py_BUILD_CORE 1 -Modules/itertoolsmodule.c Py_BUILD_CORE 1 -Modules/main.c Py_BUILD_CORE 1 -Modules/mathmodule.c Py_BUILD_CORE 1 -Modules/posixmodule.c Py_BUILD_CORE 1 -Modules/sha256module.c Py_BUILD_CORE 1 -Modules/sha512module.c Py_BUILD_CORE 1 -Modules/signalmodule.c Py_BUILD_CORE 1 -Modules/symtablemodule.c Py_BUILD_CORE 1 -Modules/timemodule.c Py_BUILD_CORE 1 -Modules/unicodedata.c Py_BUILD_CORE 1 + ('Modules/_asynciomodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_codecsmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_collectionsmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_ctypes/_ctypes.c', 'Py_BUILD_CORE', '1'), + ('Modules/_ctypes/cfield.c', 'Py_BUILD_CORE', '1'), + ('Modules/_cursesmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_datetimemodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_functoolsmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_heapqmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_io/*.c', 'Py_BUILD_CORE', '1'), + ('Modules/_io/*.h', 'Py_BUILD_CORE', '1'), + ('Modules/_localemodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_operator.c', 'Py_BUILD_CORE', '1'), + ('Modules/_posixsubprocess.c', 'Py_BUILD_CORE', '1'), + ('Modules/_sre/sre.c', 'Py_BUILD_CORE', '1'), + ('Modules/_threadmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_tracemalloc.c', 'Py_BUILD_CORE', '1'), + ('Modules/_weakref.c', 'Py_BUILD_CORE', '1'), + ('Modules/_zoneinfo.c', 'Py_BUILD_CORE', '1'), + ('Modules/atexitmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/cmathmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/faulthandler.c', 'Py_BUILD_CORE', '1'), + ('Modules/gcmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/getpath.c', 'Py_BUILD_CORE', '1'), + ('Modules/getpath_noop.c', 'Py_BUILD_CORE', '1'), + ('Modules/itertoolsmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/main.c', 'Py_BUILD_CORE', '1'), + ('Modules/mathmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/posixmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/sha256module.c', 'Py_BUILD_CORE', '1'), + ('Modules/sha512module.c', 'Py_BUILD_CORE', '1'), + ('Modules/signalmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/symtablemodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/timemodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/unicodedata.c', 'Py_BUILD_CORE', '1'), -Modules/_json.c Py_BUILD_CORE_BUILTIN 1 -Modules/_pickle.c Py_BUILD_CORE_BUILTIN 1 -Modules/_testinternalcapi.c Py_BUILD_CORE_BUILTIN 1 + ('Modules/_json.c', 'Py_BUILD_CORE_BUILTIN', '1'), + ('Modules/_pickle.c', 'Py_BUILD_CORE_BUILTIN', '1'), + ('Modules/_testinternalcapi.c', 'Py_BUILD_CORE_BUILTIN', '1'), -Include/cpython/abstract.h Py_CPYTHON_ABSTRACTOBJECT_H 1 -Include/cpython/bytearrayobject.h Py_CPYTHON_BYTEARRAYOBJECT_H 1 -Include/cpython/bytesobject.h Py_CPYTHON_BYTESOBJECT_H 1 -Include/cpython/ceval.h Py_CPYTHON_CEVAL_H 1 -Include/cpython/code.h Py_CPYTHON_CODE_H 1 -Include/cpython/dictobject.h Py_CPYTHON_DICTOBJECT_H 1 -Include/cpython/fileobject.h Py_CPYTHON_FILEOBJECT_H 1 -Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 -Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 -Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 -Include/cpython/interpreteridobject.h Py_CPYTHON_INTERPRETERIDOBJECT_H 1 -Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 -Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 -Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 -Include/cpython/objimpl.h Py_CPYTHON_OBJIMPL_H 1 -Include/cpython/pyerrors.h Py_CPYTHON_ERRORS_H 1 -Include/cpython/pylifecycle.h Py_CPYTHON_PYLIFECYCLE_H 1 -Include/cpython/pymem.h Py_CPYTHON_PYMEM_H 1 -Include/cpython/pystate.h Py_CPYTHON_PYSTATE_H 1 -Include/cpython/sysmodule.h Py_CPYTHON_SYSMODULE_H 1 -Include/cpython/traceback.h Py_CPYTHON_TRACEBACK_H 1 -Include/cpython/tupleobject.h Py_CPYTHON_TUPLEOBJECT_H 1 -Include/cpython/unicodeobject.h Py_CPYTHON_UNICODEOBJECT_H 1 + ('Include/cpython/abstract.h', 'Py_CPYTHON_ABSTRACTOBJECT_H', '1'), + ('Include/cpython/bytearrayobject.h', 'Py_CPYTHON_BYTEARRAYOBJECT_H', '1'), + ('Include/cpython/bytesobject.h', 'Py_CPYTHON_BYTESOBJECT_H', '1'), + ('Include/cpython/ceval.h', 'Py_CPYTHON_CEVAL_H', '1'), + ('Include/cpython/code.h', 'Py_CPYTHON_CODE_H', '1'), + ('Include/cpython/dictobject.h', 'Py_CPYTHON_DICTOBJECT_H', '1'), + ('Include/cpython/fileobject.h', 'Py_CPYTHON_FILEOBJECT_H', '1'), + ('Include/cpython/fileutils.h', 'Py_CPYTHON_FILEUTILS_H', '1'), + ('Include/cpython/frameobject.h', 'Py_CPYTHON_FRAMEOBJECT_H', '1'), + ('Include/cpython/import.h', 'Py_CPYTHON_IMPORT_H', '1'), + ('Include/cpython/interpreteridobject.h', 'Py_CPYTHON_INTERPRETERIDOBJECT_H', '1'), + ('Include/cpython/listobject.h', 'Py_CPYTHON_LISTOBJECT_H', '1'), + ('Include/cpython/methodobject.h', 'Py_CPYTHON_METHODOBJECT_H', '1'), + ('Include/cpython/object.h', 'Py_CPYTHON_OBJECT_H', '1'), + ('Include/cpython/objimpl.h', 'Py_CPYTHON_OBJIMPL_H', '1'), + ('Include/cpython/pyerrors.h', 'Py_CPYTHON_ERRORS_H', '1'), + ('Include/cpython/pylifecycle.h', 'Py_CPYTHON_PYLIFECYCLE_H', '1'), + ('Include/cpython/pymem.h', 'Py_CPYTHON_PYMEM_H', '1'), + ('Include/cpython/pystate.h', 'Py_CPYTHON_PYSTATE_H', '1'), + ('Include/cpython/sysmodule.h', 'Py_CPYTHON_SYSMODULE_H', '1'), + ('Include/cpython/traceback.h', 'Py_CPYTHON_TRACEBACK_H', '1'), + ('Include/cpython/tupleobject.h', 'Py_CPYTHON_TUPLEOBJECT_H', '1'), + ('Include/cpython/unicodeobject.h', 'Py_CPYTHON_UNICODEOBJECT_H', '1'), -# implied include of <unistd.h> -Include/**/*.h _POSIX_THREADS 1 -Include/**/*.h HAVE_PTHREAD_H 1 + # implied include of <unistd.h> + ('Include/**/*.h', '_POSIX_THREADS', '1'), + ('Include/**/*.h', 'HAVE_PTHREAD_H', '1'), -# from pyconfig.h -Include/cpython/pthread_stubs.h HAVE_PTHREAD_STUBS 1 -Python/thread_pthread_stubs.h HAVE_PTHREAD_STUBS 1 + # from pyconfig.h + ('Include/cpython/pthread_stubs.h', 'HAVE_PTHREAD_STUBS', '1'), + ('Python/thread_pthread_stubs.h', 'HAVE_PTHREAD_STUBS', '1'), -# from Objects/bytesobject.c -Objects/stringlib/partition.h STRINGLIB_GET_EMPTY() bytes_get_empty() -Objects/stringlib/join.h STRINGLIB_MUTABLE 0 -Objects/stringlib/partition.h STRINGLIB_MUTABLE 0 -Objects/stringlib/split.h STRINGLIB_MUTABLE 0 -Objects/stringlib/transmogrify.h STRINGLIB_MUTABLE 0 + # from Objects/bytesobject.c + ('Objects/stringlib/partition.h', 'STRINGLIB_GET_EMPTY()', 'bytes_get_empty()'), + ('Objects/stringlib/join.h', 'STRINGLIB_MUTABLE', '0'), + ('Objects/stringlib/partition.h', 'STRINGLIB_MUTABLE', '0'), + ('Objects/stringlib/split.h', 'STRINGLIB_MUTABLE', '0'), + ('Objects/stringlib/transmogrify.h', 'STRINGLIB_MUTABLE', '0'), -# from Makefile -Modules/getpath.c PYTHONPATH 1 -Modules/getpath.c PREFIX ... -Modules/getpath.c EXEC_PREFIX ... -Modules/getpath.c VERSION ... -Modules/getpath.c VPATH ... -Modules/getpath.c PLATLIBDIR ... -#Modules/_dbmmodule.c USE_GDBM_COMPAT 1 -Modules/_dbmmodule.c USE_NDBM 1 -#Modules/_dbmmodule.c USE_BERKDB 1 + # from Makefile + ('Modules/getpath.c', 'PYTHONPATH', '1'), + ('Modules/getpath.c', 'PREFIX', '...'), + ('Modules/getpath.c', 'EXEC_PREFIX', '...'), + ('Modules/getpath.c', 'VERSION', '...'), + ('Modules/getpath.c', 'VPATH', '...'), + ('Modules/getpath.c', 'PLATLIBDIR', '...'), + # ('Modules/_dbmmodule.c', 'USE_GDBM_COMPAT', '1'), + ('Modules/_dbmmodule.c', 'USE_NDBM', '1'), + # ('Modules/_dbmmodule.c', 'USE_BERKDB', '1'), -# See: setup.py -Modules/_decimal/**/*.c CONFIG_64 1 -Modules/_decimal/**/*.c ASM 1 -Modules/expat/xmlparse.c HAVE_EXPAT_CONFIG_H 1 -Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 -Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 + # See: setup.py + ('Modules/_decimal/**/*.c', 'CONFIG_64', '1'), + ('Modules/_decimal/**/*.c', 'ASM', '1'), + ('Modules/expat/xmlparse.c', 'HAVE_EXPAT_CONFIG_H', '1'), + ('Modules/expat/xmlparse.c', 'XML_POOR_ENTROPY', '1'), + ('Modules/_dbmmodule.c', 'HAVE_GDBM_DASH_NDBM_H', '1'), -# others -Modules/_sre/sre_lib.h LOCAL(type) static inline type -Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F -Objects/stringlib/codecs.h STRINGLIB_IS_UNICODE 1 -Include/internal/pycore_crossinterp_data_registry.h Py_CORE_CROSSINTERP_DATA_REGISTRY_H 1 - -# @end=tsv@ -''')[1:] + # others + ('Modules/_sre/sre_lib.h', 'LOCAL(type)', 'static inline type'), + ('Modules/_sre/sre_lib.h', 'SRE(F)', 'sre_ucs2_##F'), + ('Objects/stringlib/codecs.h', 'STRINGLIB_IS_UNICODE', '1'), + ('Include/internal/pycore_crossinterp_data_registry.h', 'Py_CORE_CROSSINTERP_DATA_REGISTRY_H', '1'), +]) # -pthread # -Wno-unused-result @@ -323,12 +316,12 @@ def clean_lines(text): _abs('Modules/_testcapimodule.c'): (20_000, 400), _abs('Modules/expat/expat.h'): (10_000, 400), _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), - _abs('Objects/typeobject.c'): (35_000, 200), + _abs('Objects/typeobject.c'): (380_000, 13_000), _abs('Python/compile.c'): (20_000, 500), _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), - _abs('Python/pylifecycle.c'): (500_000, 5000), - _abs('Python/pystate.c'): (500_000, 5000), + _abs('Python/pylifecycle.c'): (750_000, 5000), + _abs('Python/pystate.c'): (750_000, 5000), _abs('Python/initconfig.c'): (50_000, 500), # Generated files: @@ -351,6 +344,10 @@ def clean_lines(text): # Catch-alls: _abs('Include/**/*.h'): (5_000, 500), + + # Specific to clang + _abs('Modules/selectmodule.c'): (40_000, 3000), + _abs('Modules/_testcapi/pyatomic.c'): (30_000, 1000), } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 3c3cb2f9c86..301784f773d 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -400,6 +400,7 @@ Modules/_tkinter.c - tcl_lock - Modules/_tkinter.c - excInCmd - Modules/_tkinter.c - valInCmd - Modules/_tkinter.c - trbInCmd - +Modules/socketmodule.c - netdb_lock - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index b128abca39f..adb183000de 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -16,6 +16,7 @@ filename funcname name reason ## indicators for resource availability/capability # (set during first init) Python/bootstrap_hash.c py_getrandom getrandom_works - +Python/bootstrap_hash.c py_getentropy getentropy_works - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c set_inheritable ioctl_works - # (set lazily, *after* first init) @@ -24,6 +25,7 @@ Modules/posixmodule.c os_dup2_impl dup3_works - ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized - +Modules/_struct.c - endian_tables_init_once - ##----------------------- ## other values (not Python-specific) @@ -56,7 +58,7 @@ Python/pyhash.c - _Py_HashSecret - Python/parking_lot.c - buckets - ## data needed for introspecting asyncio state from debuggers and profilers -Modules/_asynciomodule.c - _AsyncioDebug - +Modules/_asynciomodule.c - _Py_AsyncioDebug - ################################## @@ -167,6 +169,8 @@ Python/sysmodule.c - _preinit_xoptions - # XXX need race protection? Modules/faulthandler.c faulthandler_dump_traceback reentrant - Modules/faulthandler.c faulthandler_dump_c_stack reentrant - +Modules/grpmodule.c - group_db_mutex - +Modules/pwdmodule.c - pwd_db_mutex - Python/pylifecycle.c _Py_FatalErrorFormat reentrant - Python/pylifecycle.c fatal_error reentrant - @@ -191,6 +195,8 @@ Python/pyfpe.c - PyFPE_counter - Python/import.c - pkgcontext - Python/pystate.c - _Py_tss_tstate - +Python/pystate.c - _Py_tss_gilstate - +Python/pystate.c - _Py_tss_interp - ##----------------------- ## should be const @@ -225,6 +231,7 @@ Modules/_datetimemodule.c datetime_isoformat specs - Modules/_datetimemodule.c parse_hh_mm_ss_ff correction - Modules/_datetimemodule.c time_isoformat specs - Modules/_datetimemodule.c - capi_types - +Modules/_datetimemodule.c normalize_century cache - Modules/_decimal/_decimal.c - cond_map_template - Modules/_decimal/_decimal.c - dec_signal_string - Modules/_decimal/_decimal.c - dflt_ctx - @@ -340,6 +347,8 @@ Objects/obmalloc.c - obmalloc_state_main - Objects/obmalloc.c - obmalloc_state_initialized - Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - +# It initialized only once when main interpeter starts +Objects/typeobject.c - slotdefs_name_counts - Objects/unicodeobject.c - stripfuncnames - Objects/unicodeobject.c - utf7_category - Objects/unicodeobject.c unicode_decode_call_errorhandler_wchar argparse - @@ -351,6 +360,7 @@ Parser/parser.c - soft_keywords - Parser/lexer/lexer.c - type_comment_prefix - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - +Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR - Python/codecs.c - Py_hexdigits - Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - diff --git a/Tools/c-analyzer/distutils/util.py b/Tools/c-analyzer/distutils/util.py index 89ca094336f..f687a28ec2f 100644 --- a/Tools/c-analyzer/distutils/util.py +++ b/Tools/c-analyzer/distutils/util.py @@ -19,8 +19,8 @@ def get_host_platform(): particularly important. Examples of returned values: - linux-i586 - linux-alpha (?) + linux-x86_64 + linux-aarch64 solaris-2.6-sun4u Windows will return one of: diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 2b3a90c7db9..93aa4899fe6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -5,7 +5,7 @@ import re from typing import Optional, Callable -from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt +from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt, ForStmt, MacroIfStmt @dataclass class EscapingCall: @@ -20,6 +20,7 @@ class Properties: error_with_pop: bool error_without_pop: bool deopts: bool + deopts_periodic: bool oparg: bool jumps: bool eval_breaker: bool @@ -33,6 +34,8 @@ class Properties: side_exit: bool pure: bool uses_opcode: bool + needs_guard_ip: bool + unpredictable_jump: bool tier: int | None = None const_oparg: int = -1 needs_prev: bool = False @@ -58,6 +61,7 @@ def from_list(properties: list["Properties"]) -> "Properties": error_with_pop=any(p.error_with_pop for p in properties), error_without_pop=any(p.error_without_pop for p in properties), deopts=any(p.deopts for p in properties), + deopts_periodic=any(p.deopts_periodic for p in properties), oparg=any(p.oparg for p in properties), jumps=any(p.jumps for p in properties), eval_breaker=any(p.eval_breaker for p in properties), @@ -73,6 +77,8 @@ def from_list(properties: list["Properties"]) -> "Properties": pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), no_save_ip=all(p.no_save_ip for p in properties), + needs_guard_ip=any(p.needs_guard_ip for p in properties), + unpredictable_jump=any(p.unpredictable_jump for p in properties), ) @property @@ -85,6 +91,7 @@ def infallible(self) -> bool: error_with_pop=False, error_without_pop=False, deopts=False, + deopts_periodic=False, oparg=False, jumps=False, eval_breaker=False, @@ -99,6 +106,8 @@ def infallible(self) -> bool: side_exit=False, pure=True, no_save_ip=False, + needs_guard_ip=False, + unpredictable_jump=False, ) @@ -135,15 +144,13 @@ def size(self) -> int: @dataclass class StackItem: name: str - type: str | None size: str peek: bool = False used: bool = False def __str__(self) -> str: size = f"[{self.size}]" if self.size else "" - type = "" if self.type is None else f"{self.type} " - return f"{type}{self.name}{size} {self.peek}" + return f"{self.name}{size} {self.peek}" def is_array(self) -> bool: return self.size != "" @@ -182,7 +189,7 @@ class Uop: properties: Properties _size: int = -1 implicitly_created: bool = False - replicated = 0 + replicated = range(0) replicates: "Uop | None" = None # Size of the instruction(s), only set for uops containing the INSTRUCTION_SIZE macro instruction_size: int | None = None @@ -345,7 +352,7 @@ def override_error( def convert_stack_item( item: parser.StackEffect, replace_op_arg_1: str | None ) -> StackItem: - return StackItem(item.name, item.type, item.size) + return StackItem(item.name, item.size) def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]) -> None: "Unused items cannot be on the stack above used, non-peek items" @@ -585,7 +592,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_CLOSE_SPECIALIZED", "PyStackRef_DUP", "PyStackRef_False", - "PyStackRef_FromPyObjectImmortal", + "PyStackRef_FromPyObjectBorrow", "PyStackRef_FromPyObjectNew", "PyStackRef_FromPyObjectSteal", "PyStackRef_IsExactly", @@ -598,6 +605,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_IsNull", "PyStackRef_MakeHeapSafe", "PyStackRef_None", + "PyStackRef_RefcountOnObject", "PyStackRef_TYPE", "PyStackRef_True", "PyTuple_GET_ITEM", @@ -637,6 +645,10 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyLong_IsNegative", "_PyLong_IsNonNegativeCompact", "_PyLong_IsZero", + "_PyLong_BothAreCompact", + "_PyCompactLong_Add", + "_PyCompactLong_Multiply", + "_PyCompactLong_Subtract", "_PyManagedDictPointer_IsValues", "_PyObject_GC_IS_SHARED", "_PyObject_GC_IS_TRACKED", @@ -679,8 +691,21 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_IsTaggedInt", "PyStackRef_TagInt", "PyStackRef_UntagInt", + "PyStackRef_IncrementTaggedIntNoOverflow", + "PyStackRef_IsNullOrInt", + "PyStackRef_IsError", + "PyStackRef_IsValid", + "PyStackRef_Wrap", + "PyStackRef_Unwrap", + "_PyLong_CheckExactAndCompact", + "_PyExecutor_FromExit", + "_PyJit_TryInitializeTracing", + "_Py_unset_eval_breaker_bit", + "_Py_set_eval_breaker_bit", + "trigger_backoff_counter", ) + def check_escaping_calls(instr: parser.CodeDef, escapes: dict[SimpleStmt, EscapingCall]) -> None: error: lexer.Token | None = None calls = {e.call for e in escapes.values()} @@ -695,7 +720,7 @@ def visit(stmt: Stmt) -> None: in_if = 0 tkn_iter = iter(stmt.contents) for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF"): + if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF", "HANDLE_PENDING_AND_DEOPT_IF", "AT_END_EXIT_IF"): in_if = 1 next(tkn_iter) elif tkn.kind == "LPAREN": @@ -712,53 +737,57 @@ def visit(stmt: Stmt) -> None: if error is not None: raise analysis_error(f"Escaping call '{error.text} in condition", error) +def escaping_call_in_simple_stmt(stmt: SimpleStmt, result: dict[SimpleStmt, EscapingCall]) -> None: + tokens = stmt.contents + for idx, tkn in enumerate(tokens): + try: + next_tkn = tokens[idx+1] + except IndexError: + break + if next_tkn.kind != lexer.LPAREN: + continue + if tkn.kind == lexer.IDENTIFIER: + if tkn.text.upper() == tkn.text: + # simple macro + continue + #if not tkn.text.startswith(("Py", "_Py", "monitor")): + # continue + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): + # Optimize functions + continue + if tkn.text.endswith("Check"): + continue + if tkn.text.startswith("Py_Is"): + continue + if tkn.text.endswith("CheckExact"): + continue + if tkn.text in NON_ESCAPING_FUNCTIONS: + continue + elif tkn.kind == "RPAREN": + prev = tokens[idx-1] + if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": + #cast + continue + elif tkn.kind != "RBRACKET": + continue + if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): + if len(tokens) <= idx+2: + raise analysis_error("Unexpected end of file", next_tkn) + kills = tokens[idx+2] + if kills.kind != "IDENTIFIER": + raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) + else: + kills = None + result[stmt] = EscapingCall(stmt, tkn, kills) + + def find_escaping_api_calls(instr: parser.CodeDef) -> dict[SimpleStmt, EscapingCall]: result: dict[SimpleStmt, EscapingCall] = {} def visit(stmt: Stmt) -> None: if not isinstance(stmt, SimpleStmt): return - tokens = stmt.contents - for idx, tkn in enumerate(tokens): - try: - next_tkn = tokens[idx+1] - except IndexError: - break - if next_tkn.kind != lexer.LPAREN: - continue - if tkn.kind == lexer.IDENTIFIER: - if tkn.text.upper() == tkn.text: - # simple macro - continue - #if not tkn.text.startswith(("Py", "_Py", "monitor")): - # continue - if tkn.text.startswith(("sym_", "optimize_")): - # Optimize functions - continue - if tkn.text.endswith("Check"): - continue - if tkn.text.startswith("Py_Is"): - continue - if tkn.text.endswith("CheckExact"): - continue - if tkn.text in NON_ESCAPING_FUNCTIONS: - continue - elif tkn.kind == "RPAREN": - prev = tokens[idx-1] - if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": - #cast - continue - elif tkn.kind != "RBRACKET": - continue - if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): - if len(tokens) <= idx+2: - raise analysis_error("Unexpected end of file", next_tkn) - kills = tokens[idx+2] - if kills.kind != "IDENTIFIER": - raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) - else: - kills = None - result[stmt] = EscapingCall(stmt, tkn, kills) + escaping_call_in_simple_stmt(stmt, result) instr.block.accept(visit) check_escaping_calls(instr, result) @@ -806,11 +835,105 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: if len(stack_inputs) == 0: return False return all( - (s.name == other.name and s.type == other.type and s.size == other.size) + (s.name == other.name and s.size == other.size) for s, other in zip(stack_inputs, instr.outputs) ) +def stmt_is_simple_exit(stmt: Stmt) -> bool: + if not isinstance(stmt, SimpleStmt): + return False + tokens = stmt.contents + if len(tokens) < 4: + return False + return ( + tokens[0].text in ("ERROR_IF", "DEOPT_IF", "EXIT_IF", "AT_END_EXIT_IF") + and + tokens[1].text == "(" + and + tokens[2].text in ("true", "1") + and + tokens[3].text == ")" + ) + + +def stmt_list_escapes(stmts: list[Stmt]) -> bool: + if not stmts: + return False + if stmt_is_simple_exit(stmts[-1]): + return False + for stmt in stmts: + if stmt_escapes(stmt): + return True + return False + + +def stmt_escapes(stmt: Stmt) -> bool: + if isinstance(stmt, BlockStmt): + return stmt_list_escapes(stmt.body) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "DECREF_INPUTS": + return True + d: dict[SimpleStmt, EscapingCall] = {} + escaping_call_in_simple_stmt(stmt, d) + return bool(d) + elif isinstance(stmt, IfStmt): + if stmt.else_body and stmt_escapes(stmt.else_body): + return True + return stmt_escapes(stmt.body) + elif isinstance(stmt, MacroIfStmt): + if stmt.else_body and stmt_list_escapes(stmt.else_body): + return True + return stmt_list_escapes(stmt.body) + elif isinstance(stmt, ForStmt): + return stmt_escapes(stmt.body) + elif isinstance(stmt, WhileStmt): + return stmt_escapes(stmt.body) + else: + assert False, "Unexpected statement type" + +def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> tuple[bool, int]: + if not stmts: + return False, branches_seen + predict = False + seen = 0 + for st in stmts: + predict_body, seen_body = stmt_has_jump_on_unpredictable_path(st, branches_seen) + predict = predict or predict_body + seen += seen_body + return predict, seen + +def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> tuple[bool, int]: + if isinstance(stmt, BlockStmt): + return stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "JUMPBY": + return True, branches_seen + return False, branches_seen + elif isinstance(stmt, IfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + 1 + return predict, seen + 1 + elif isinstance(stmt, MacroIfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path_body(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + return predict, seen + elif isinstance(stmt, ForStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + elif isinstance(stmt, WhileStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + else: + assert False, f"Unexpected statement type {stmt}" + + def compute_properties(op: parser.CodeDef) -> Properties: escaping_calls = find_escaping_api_calls(op) has_free = ( @@ -820,11 +943,13 @@ def compute_properties(op: parser.CodeDef) -> Properties: or variable_used(op, "PyCell_SwapTakeRef") ) deopts_if = variable_used(op, "DEOPT_IF") - exits_if = variable_used(op, "EXIT_IF") - if deopts_if and exits_if: + exits_if = variable_used(op, "EXIT_IF") or variable_used(op, "AT_END_EXIT_IF") + deopts_periodic = variable_used(op, "HANDLE_PENDING_AND_DEOPT_IF") + exits_and_deopts = sum((deopts_if, exits_if, deopts_periodic)) + if exits_and_deopts > 1: tkn = op.tokens[0] raise lexer.make_syntax_error( - "Op cannot contain both EXIT_IF and DEOPT_IF", + "Op cannot contain more than one of EXIT_IF, DEOPT_IF and HANDLE_PENDING_AND_DEOPT_IF", tkn.filename, tkn.line, tkn.column, @@ -832,15 +957,18 @@ def compute_properties(op: parser.CodeDef) -> Properties: ) error_with_pop = has_error_with_pop(op) error_without_pop = has_error_without_pop(op) - escapes = bool(escaping_calls) + escapes = stmt_escapes(op.block) pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(op.block, 0) + unpredictable_jump = False if isinstance(op, parser.LabelDef) else (unpredictable and branches_seen > 0) return Properties( escaping_calls=escaping_calls, escapes=escapes, error_with_pop=error_with_pop, error_without_pop=error_without_pop, deopts=deopts_if, + deopts_periodic=deopts_periodic, side_exit=exits_if, oparg=oparg_used(op), jumps=variable_used(op, "JUMPBY"), @@ -857,8 +985,35 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), + needs_guard_ip=(isinstance(op, parser.InstDef) + and (unpredictable_jump and "replaced" not in op.annotations)) + or variable_used(op, "LOAD_IP") + or variable_used(op, "DISPATCH_INLINED"), + unpredictable_jump=unpredictable_jump, ) +def expand(items: list[StackItem], oparg: int) -> list[StackItem]: + # Only replace array item with scalar if no more than one item is an array + index = -1 + for i, item in enumerate(items): + if "oparg" in item.size: + if index >= 0: + return items + index = i + if index < 0: + return items + try: + count = int(eval(items[index].size.replace("oparg", str(oparg)))) + except ValueError: + return items + return items[:index] + [ + StackItem(items[index].name + f"_{i}", "", items[index].peek, items[index].used) for i in range(count) + ] + items[index+1:] + +def scalarize_stack(stack: StackEffect, oparg: int) -> StackEffect: + stack.inputs = expand(stack.inputs, oparg) + stack.outputs = expand(stack.outputs, oparg) + return stack def make_uop( name: str, @@ -878,20 +1033,26 @@ def make_uop( ) for anno in op.annotations: if anno.startswith("replicate"): - result.replicated = int(anno[10:-1]) + text = anno[10:-1] + start, stop = text.split(":") + result.replicated = range(int(start), int(stop)) break else: return result - for oparg in range(result.replicated): + for oparg in result.replicated: name_x = name + "_" + str(oparg) properties = compute_properties(op) properties.oparg = False - properties.const_oparg = oparg + stack = analyze_stack(op) + if not variable_used(op, "oparg"): + stack = scalarize_stack(stack, oparg) + else: + properties.const_oparg = oparg rep = Uop( name=name_x, context=op.context, annotations=op.annotations, - stack=analyze_stack(op), + stack=stack, caches=analyze_caches(inputs), local_stores=find_variable_stores(op), body=op.block, @@ -1034,8 +1195,9 @@ def assign_opcodes( # This is an historical oddity. instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 - instmap["INSTRUMENTED_LINE"] = 254 - instmap["ENTER_EXECUTOR"] = 255 + instmap["INSTRUMENTED_LINE"] = 253 + instmap["ENTER_EXECUTOR"] = 254 + instmap["TRACE_RECORD"] = 255 instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] @@ -1060,7 +1222,7 @@ def assign_opcodes( # Specialized ops appear in their own section # Instrumented opcodes are at the end of the valid range min_internal = instmap["RESUME"] + 1 - min_instrumented = 254 - (len(instrumented) - 1) + min_instrumented = 254 - len(instrumented) assert min_internal + len(specialized) < min_instrumented next_opcode = 1 diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 9e60d219a71..0b5f764ec52 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -7,6 +7,7 @@ analysis_error, Label, CodeSection, + Uop, ) from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable @@ -56,9 +57,7 @@ def root_relative_path(filename: str) -> str: def type_and_null(var: StackItem) -> tuple[str, str]: - if var.type: - return var.type, "NULL" - elif var.is_array(): + if var.is_array(): return "_PyStackRef *", "NULL" else: return "_PyStackRef", "PyStackRef_NULL" @@ -108,11 +107,15 @@ class Emitter: out: CWriter labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] + cannot_escape: bool + jump_prefix: str - def __init__(self, out: CWriter, labels: dict[str, Label]): + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False, jump_prefix: str = ""): self._replacers = { "EXIT_IF": self.exit_if, + "AT_END_EXIT_IF": self.exit_if_after, "DEOPT_IF": self.deopt_if, + "HANDLE_PENDING_AND_DEOPT_IF": self.periodic_if, "ERROR_IF": self.error_if, "ERROR_NO_POP": self.error_no_pop, "DECREF_INPUTS": self.decref_inputs, @@ -129,6 +132,8 @@ def __init__(self, out: CWriter, labels: dict[str, Label]): } self.out = out self.labels = labels + self.cannot_escape = cannot_escape + self.jump_prefix = jump_prefix def dispatch( self, @@ -140,6 +145,7 @@ def dispatch( ) -> bool: if storage.spilled: raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) self.emit(tkn) return False @@ -164,18 +170,41 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"JUMP_TO_PREDICTED({family_name});\n") + self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn) exit_if = deopt_if + def periodic_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + raise NotImplementedError("HANDLE_PENDING_AND_DEOPT_IF not support in tier 1") + + def exit_if_after( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + storage.clear_inputs("in AT_END_EXIT_IF") + storage.flush(self.out) + storage.stack.clear(self.out) + return self.exit_if(tkn, tkn_iter, uop, storage, inst) + def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: - return f"JUMP_TO_LABEL(pop_{offset}_error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(pop_{offset}_error);" if offset < 0: storage.copy().flush(self.out) - return f"JUMP_TO_LABEL(error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(error);" def error_if( self, @@ -239,7 +268,8 @@ def decref_inputs( next(tkn_iter) self._print_storage("DECREF_INPUTS", storage) try: - storage.close_inputs(self.out) + if not self.cannot_escape: + storage.close_inputs(self.out) except StackError as ex: raise analysis_error(ex.args[0], tkn) except Exception as ex: @@ -394,7 +424,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.start_line() - self.out.emit("JUMP_TO_LABEL(") + self.out.emit(f"{self.jump_prefix}JUMP_TO_LABEL(") self.out.emit(label) self.out.emit(")") @@ -443,7 +473,7 @@ def instruction_size(self, """Replace the INSTRUCTION_SIZE macro with the size of the current instruction.""" if uop.instruction_size is None: raise analysis_error("The INSTRUCTION_SIZE macro requires uop.instruction_size to be set", tkn) - self.out.emit(f" {uop.instruction_size} ") + self.out.emit(f" {uop.instruction_size}u ") return True def _print_storage(self, reason:str, storage: Storage) -> None: @@ -477,7 +507,7 @@ def emit_SimpleStmt( reachable = True tkn = stmt.contents[-1] try: - if stmt in uop.properties.escaping_calls: + if stmt in uop.properties.escaping_calls and not self.cannot_escape: escape = uop.properties.escaping_calls[stmt] if escape.kills is not None: self.stackref_kill(escape.kills, storage, True) @@ -514,7 +544,7 @@ def emit_SimpleStmt( self.out.emit(tkn) else: self.out.emit(tkn) - if stmt in uop.properties.escaping_calls: + if stmt in uop.properties.escaping_calls and not self.cannot_escape: self.emit_reload(storage) return reachable, None, storage except StackError as ex: @@ -690,6 +720,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_EVAL_BREAK_FLAG") if p.deopts: flags.append("HAS_DEOPT_FLAG") + if p.deopts_periodic: + flags.append("HAS_PERIODIC_FLAG") if p.side_exit: flags.append("HAS_EXIT_FLAG") if not p.infallible: @@ -702,6 +734,10 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.no_save_ip: flags.append("HAS_NO_SAVE_IP_FLAG") + if p.unpredictable_jump: + flags.append("HAS_UNPREDICTABLE_JUMP_FLAG") + if p.needs_guard_ip: + flags.append("HAS_NEEDS_GUARD_IP_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 1ee4306f3ea..29e4e74da72 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -81,7 +81,7 @@ ### Syntax (definition | family | pseudo)+ definition: - "inst" "(" NAME ["," stack_effect] ")" "{" C-code "}" + "inst" "(" NAME "," stack_effect ")" "{" C-code "}" | "op" "(" NAME "," stack_effect ")" "{" C-code "}" | @@ -174,7 +174,13 @@ ### Special instruction annotations * `override`. For external use by other interpreter definitions to override the current instruction definition. * `pure`. This instruction has no side effects. -* 'tierN'. This instruction is only used by the tier N interpreter. +* `tierN`. This instruction is only used by the tier N interpreter. +* `specializing`. A prefix for an instructions related to adaptive interpreter. +* `replaced`. This instruction will be replaced in the final bytecode by its directed + version (either forward or backward). +* `register`. Currently does nothing. +* `replicate(N)`. Replicate the instruction N times to store the oparg "inside" the instruction. +* `no_save_ip`. This instruction does not affect the instruction pointer. ### Special functions/macros diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 620e4b6f1f4..21ae785a0ec 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -53,10 +53,11 @@ "ESCAPES", "EXIT", "PURE", - "PASSTHROUGH", - "OPARG_AND_1", "ERROR_NO_POP", "NO_SAVE_IP", + "PERIODIC", + "UNPREDICTABLE_JUMP", + "NEEDS_GUARD_IP", ] @@ -157,6 +158,13 @@ def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: if inst.family is not None: deopt = inst.family.name deopts.append((inst.name, deopt)) + defined = set(analysis.opmap.values()) + for i in range(256): + if i not in defined: + deopts.append((f'{i}', f'{i}')) + + assert len(deopts) == 256 + assert len(set(x[0] for x in deopts)) == 256 for name, deopt in sorted(deopts): out.emit(f"[{name}] = {deopt},\n") out.emit("};\n\n") @@ -195,7 +203,7 @@ def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: out.emit("struct opcode_metadata {\n") out.emit("uint8_t valid_entry;\n") out.emit("uint8_t instr_format;\n") - out.emit("uint16_t flags;\n") + out.emit("uint32_t flags;\n") out.emit("};\n\n") out.emit( f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" @@ -235,14 +243,10 @@ def generate_expansion_table(analysis: Analysis, out: CWriter) -> None: assert name2 in analysis.instructions, f"{name2} doesn't match any instr" instr1 = analysis.instructions[name1] instr2 = analysis.instructions[name2] - assert ( - len(instr1.parts) == 1 - ), f"{name1} is not a good superinstruction part" - assert ( - len(instr2.parts) == 1 - ), f"{name2} is not a good superinstruction part" - expansions.append((instr1.parts[0].name, "OPARG_TOP", 0)) - expansions.append((instr2.parts[0].name, "OPARG_BOTTOM", 0)) + for part in instr1.parts: + expansions.append((part.name, "OPARG_TOP", 0)) + for part in instr2.parts: + expansions.append((part.name, "OPARG_BOTTOM", 0)) elif not is_viable_expansion(inst): continue else: diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 7a32275347e..ab0a90e234b 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -12,6 +12,8 @@ analyze_files, StackItem, analysis_error, + CodeSection, + Label, ) from generators_common import ( DEFAULT_INPUT, @@ -19,6 +21,7 @@ write_header, Emitter, TokenIterator, + always_true, ) from cwriter import CWriter from typing import TextIO @@ -30,17 +33,54 @@ def validate_uop(override: Uop, uop: Uop) -> None: - # To do - pass + """ + Check that the overridden uop (defined in 'optimizer_bytecodes.c') + has the same stack effects as the original uop (defined in 'bytecodes.c'). + + Ensure that: + - The number of inputs and outputs is the same. + - The names of the inputs and outputs are the same + (except for 'unused' which is ignored). + - The sizes of the inputs and outputs are the same. + """ + for stack_effect in ('inputs', 'outputs'): + orig_effects = getattr(uop.stack, stack_effect) + new_effects = getattr(override.stack, stack_effect) + + if len(orig_effects) != len(new_effects): + msg = ( + f"{uop.name}: Must have the same number of {stack_effect} " + "in bytecodes.c and optimizer_bytecodes.c " + f"({len(orig_effects)} != {len(new_effects)})" + ) + raise analysis_error(msg, override.body.open) + + for orig, new in zip(orig_effects, new_effects, strict=True): + if orig.name != new.name and orig.name != "unused" and new.name != "unused": + msg = ( + f"{uop.name}: {stack_effect.capitalize()} must have " + "equal names in bytecodes.c and optimizer_bytecodes.c " + f"({orig.name} != {new.name})" + ) + raise analysis_error(msg, override.body.open) + + if orig.size != new.size: + msg = ( + f"{uop.name}: {stack_effect.capitalize()} must have " + "equal sizes in bytecodes.c and optimizer_bytecodes.c " + f"({orig.size!r} != {new.size!r})" + ) + raise analysis_error(msg, override.body.open) def type_name(var: StackItem) -> str: if var.is_array(): - return f"JitOptSymbol **" - if var.type: - return var.type - return f"JitOptSymbol *" + return "JitOptRef *" + return "JitOptRef " +def stackref_type_name(var: StackItem) -> str: + assert not var.is_array(), "Unsafe to convert a symbol to an array-like StackRef." + return "_PyStackRef " def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: variables = {"unused"} @@ -101,6 +141,12 @@ def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: class OptimizerEmitter(Emitter): + def __init__(self, out: CWriter, labels: dict[str, Label], original_uop: Uop, stack: Stack): + super().__init__(out, labels) + self._replacers["REPLACE_OPCODE_IF_EVALUATES_PURE"] = self.replace_opcode_if_evaluates_pure + self.original_uop = original_uop + self.stack = stack + def emit_save(self, storage: Storage) -> None: storage.flush(self.out) @@ -111,6 +157,210 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: self.out.emit(goto) self.out.emit(label) + def replace_opcode_if_evaluates_pure( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + assert isinstance(uop, Uop) + input_identifiers = [] + for token in tkn_iter: + if token.kind == "IDENTIFIER": + input_identifiers.append(token) + if token.kind == "SEMI": + break + + if len(input_identifiers) == 0: + raise analysis_error( + "To evaluate an operation as pure, it must have at least 1 input", + tkn + ) + # Check that the input identifiers belong to the uop's + # input stack effect + uop_stack_effect_input_identifers = {inp.name for inp in uop.stack.inputs} + for input_tkn in input_identifiers: + if input_tkn.text not in uop_stack_effect_input_identifers: + raise analysis_error(f"{input_tkn.text} referenced in " + f"REPLACE_OPCODE_IF_EVALUATES_PURE but does not " + f"exist in the base uop's input stack effects", + input_tkn) + input_identifiers_as_str = {tkn.text for tkn in input_identifiers} + used_stack_inputs = [inp for inp in uop.stack.inputs if inp.name in input_identifiers_as_str] + assert len(used_stack_inputs) > 0 + self.out.start_line() + emitter = OptimizerConstantEmitter(self.out, {}, self.original_uop, self.stack.copy()) + emitter.emit("if (\n") + for inp in used_stack_inputs[:-1]: + emitter.emit(f"sym_is_safe_const(ctx, {inp.name}) &&\n") + emitter.emit(f"sym_is_safe_const(ctx, {used_stack_inputs[-1].name})\n") + emitter.emit(') {\n') + # Declare variables, before they are shadowed. + for inp in used_stack_inputs: + if inp.used: + emitter.emit(f"{type_name(inp)}{inp.name}_sym = {inp.name};\n") + # Shadow the symbolic variables with stackrefs. + for inp in used_stack_inputs: + if inp.is_array(): + raise analysis_error("Pure evaluation cannot take array-like inputs.", tkn) + if inp.used: + emitter.emit(f"{stackref_type_name(inp)}{inp.name} = sym_get_const_as_stackref(ctx, {inp.name}_sym);\n") + # Rename all output variables to stackref variant. + for outp in self.original_uop.stack.outputs: + if outp.is_array(): + raise analysis_error( + "Array output StackRefs not supported for evaluating pure ops.", + self.original_uop.body.open + ) + emitter.emit(f"_PyStackRef {outp.name}_stackref;\n") + + + storage = Storage.for_uop(self.stack, self.original_uop, CWriter.null(), check_liveness=False) + # No reference management of outputs needed. + for var in storage.outputs: + var.in_local = True + emitter.emit("/* Start of uop copied from bytecodes for constant evaluation */\n") + emitter.emit_tokens(self.original_uop, storage, inst=None, emit_braces=False) + self.out.start_line() + emitter.emit("/* End of uop copied from bytecodes for constant evaluation */\n") + # Finally, assign back the output stackrefs to symbolics. + for outp in self.original_uop.stack.outputs: + # All new stackrefs are created from new references. + # That's how the stackref contract works. + if not outp.peek: + emitter.emit(f"{outp.name} = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal({outp.name}_stackref));\n") + else: + emitter.emit(f"{outp.name} = sym_new_const(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") + if len(self.original_uop.stack.outputs) == 1: + outp = self.original_uop.stack.outputs[0] + if not outp.peek: + if self.original_uop.name.startswith('_'): + # Map input count to the appropriate constant-loading uop + input_count_to_uop = { + 1: "_POP_TOP_LOAD_CONST_INLINE_BORROW", + 2: "_POP_TWO_LOAD_CONST_INLINE_BORROW" + } + + input_count = len(used_stack_inputs) + if input_count in input_count_to_uop: + replacement_uop = input_count_to_uop[input_count] + input_desc = "one input" if input_count == 1 else "two inputs" + + emitter.emit(f"if (sym_is_const(ctx, {outp.name})) {{\n") + emitter.emit(f"PyObject *result = sym_get_const(ctx, {outp.name});\n") + emitter.emit(f"if (_Py_IsImmortal(result)) {{\n") + emitter.emit(f"// Replace with {replacement_uop} since we have {input_desc} and an immortal result\n") + emitter.emit(f"REPLACE_OP(this_instr, {replacement_uop}, 0, (uintptr_t)result);\n") + emitter.emit("}\n") + emitter.emit("}\n") + + storage.flush(self.out) + emitter.emit("break;\n") + emitter.emit("}\n") + return True + +class OptimizerConstantEmitter(OptimizerEmitter): + def __init__(self, out: CWriter, labels: dict[str, Label], original_uop: Uop, stack: Stack): + super().__init__(out, labels, original_uop, stack) + # Replace all outputs to point to their stackref versions. + overrides = { + outp.name: self.emit_stackref_override for outp in self.original_uop.stack.outputs + } + self._replacers = {**self._replacers, **overrides} + self.cannot_escape = True + + def emit_to_with_replacement( + self, + out: CWriter, + tkn_iter: TokenIterator, + end: str, + uop: CodeSection, + storage: Storage, + inst: Instruction | None + ) -> Token: + parens = 0 + for tkn in tkn_iter: + if tkn.kind == end and parens == 0: + return tkn + if tkn.kind == "LPAREN": + parens += 1 + if tkn.kind == "RPAREN": + parens -= 1 + if tkn.text in self._replacers: + self._replacers[tkn.text](tkn, tkn_iter, uop, storage, inst) + else: + out.emit(tkn) + raise analysis_error(f"Expecting {end}. Reached end of file", tkn) + + def emit_stackref_override( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit(tkn) + self.out.emit("_stackref ") + return True + + def deopt_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.start_line() + self.out.emit("if (") + lparen = next(tkn_iter) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() + self.emit_to_with_replacement(self.out, tkn_iter, "RPAREN", uop, storage, inst) + self.emit(") {\n") + next(tkn_iter) # Semi colon + # We guarantee this will deopt in real-world code + # via constants analysis. So just bail. + self.emit("ctx->done = true;\n") + self.emit("break;\n") + self.emit("}\n") + return not always_true(first_tkn) + + exit_if = deopt_if + + def error_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + lparen = next(tkn_iter) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() + unconditional = always_true(first_tkn) + if unconditional: + next(tkn_iter) + next(tkn_iter) # RPAREN + self.out.start_line() + else: + self.out.emit_at("if ", tkn) + self.emit(lparen) + self.emit_to_with_replacement(self.out, tkn_iter, "RPAREN", uop, storage, inst) + self.out.emit(") {\n") + next(tkn_iter) # Semi colon + storage.clear_inputs("at ERROR_IF") + + self.out.emit("goto error;\n") + if not unconditional: + self.out.emit("}\n") + return not unconditional + + def write_uop( override: Uop | None, uop: Uop, @@ -141,13 +391,14 @@ def write_uop( cast = f"uint{cache.size*16}_t" out.emit(f"{type}{cache.name} = ({cast})this_instr->operand0;\n") if override: - emitter = OptimizerEmitter(out, {}) + emitter = OptimizerEmitter(out, {}, uop, stack.copy()) # No reference management of inputs needed. for var in storage.inputs: # type: ignore[possibly-undefined] var.in_local = False _, storage = emitter.emit_tokens(override, storage, None, False) out.start_line() storage.flush(out) + out.start_line() else: emit_default(out, uop, stack) out.start_line() @@ -171,9 +422,9 @@ def generate_abstract_interpreter( out.emit("\n") base_uop_names = set([uop.name for uop in base.uops.values()]) for abstract_uop_name in abstract.uops: - assert ( - abstract_uop_name in base_uop_names - ), f"All abstract uops should override base uops, but {abstract_uop_name} is not." + if abstract_uop_name not in base_uop_names: + raise ValueError(f"All abstract uops should override base uops, " + "but {abstract_uop_name} is not.") for uop in base.uops.values(): override: Uop | None = None @@ -194,7 +445,7 @@ def generate_abstract_interpreter( declare_variables(override, out, skip_inputs=False) else: declare_variables(uop, out, skip_inputs=True) - stack = Stack(extract_bits=False, cast_type="JitOptSymbol *") + stack = Stack(check_stack_bounds=True) write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) out.start_line() out.emit("break;\n") diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 9c9b0053a59..c7fe0d162ac 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -247,12 +247,11 @@ def accept(self, visitor: Visitor) -> None: @dataclass class StackEffect(Node): name: str = field(compare=False) # __eq__ only uses type, cond, size - type: str = "" # Optional `:type` size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond def __repr__(self) -> str: - items = [self.name, self.type, self.size] + items = [self.name, self.size] while items and items[-1] == "": del items[-1] return f"StackEffect({', '.join(repr(item) for item in items)})" @@ -380,9 +379,13 @@ def inst_header(self) -> InstHeader | None: while anno := self.expect(lx.ANNOTATION): if anno.text == "replicate": self.require(lx.LPAREN) - times = self.require(lx.NUMBER) + stop = self.require(lx.NUMBER) + start_text = "0" + if self.expect(lx.COLON): + start_text = stop.text + stop = self.require(lx.NUMBER) self.require(lx.RPAREN) - annotations.append(f"replicate({times.text})") + annotations.append(f"replicate({start_text}:{stop.text})") else: annotations.append(anno.text) tkn = self.expect(lx.INST) @@ -463,20 +466,13 @@ def stack_effect(self) -> StackEffect | None: # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): - type_text = "" - if self.expect(lx.COLON): - type_text = self.require(lx.IDENTIFIER).text.strip() - if self.expect(lx.TIMES): - type_text += " *" size_text = "" if self.expect(lx.LBRACKET): - if type_text: - raise self.make_syntax_error("Unexpected [") if not (size := self.expression()): raise self.make_syntax_error("Expected expression") self.require(lx.RBRACKET) size_text = size.text.strip() - return StackEffect(tkn.text, type_text, size_text) + return StackEffect(tkn.text, size_text) return None @contextual diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 6b681775f48..53499558aed 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -168,7 +168,7 @@ def from_memory(defn: StackItem, offset: PointerOffset) -> "Local": @staticmethod def register(name: str) -> "Local": - item = StackItem(name, None, "", False, True) + item = StackItem(name, "", False, True) return Local(item, None, True) def kill(self) -> None: @@ -216,13 +216,12 @@ def array_or_scalar(var: StackItem | Local) -> str: return "array" if var.is_array() else "scalar" class Stack: - def __init__(self, extract_bits: bool=True, cast_type: str = "uintptr_t") -> None: + def __init__(self, check_stack_bounds: bool = False) -> None: self.base_offset = PointerOffset.zero() self.physical_sp = PointerOffset.zero() self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] - self.extract_bits = extract_bits - self.cast_type = cast_type + self.check_stack_bounds = check_stack_bounds def drop(self, var: StackItem, check_liveness: bool) -> None: self.logical_sp = self.logical_sp.pop(var) @@ -268,10 +267,8 @@ def pop(self, var: StackItem, out: CWriter) -> Local: self.base_offset = self.logical_sp if var.name in UNUSED or not var.used: return Local.unused(var, self.base_offset) - cast = f"({var.type})" if (not indirect and var.type) else "" - bits = ".bits" if cast and self.extract_bits else "" c_offset = (self.base_offset - self.physical_sp).to_c() - assign = f"{var.name} = {cast}{indirect}stack_pointer[{c_offset}]{bits};\n" + assign = f"{var.name} = {indirect}stack_pointer[{c_offset}];\n" out.emit(assign) self._print(out) return Local.from_memory(var, self.base_offset) @@ -292,19 +289,15 @@ def _do_emit( out: CWriter, var: StackItem, stack_offset: PointerOffset, - cast_type: str, - extract_bits: bool, ) -> None: - cast = f"({cast_type})" if var.type else "" - bits = ".bits" if cast and extract_bits else "" - out.emit(f"stack_pointer[{stack_offset.to_c()}]{bits} = {cast}{var.name};\n") + out.emit(f"stack_pointer[{stack_offset.to_c()}] = {var.name};\n") def _save_physical_sp(self, out: CWriter) -> None: if self.physical_sp != self.logical_sp: diff = self.logical_sp - self.physical_sp out.start_line() out.emit(f"stack_pointer += {diff.to_c()};\n") - out.emit(f"assert(WITHIN_STACK_BOUNDS());\n") + out.emit(f"ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);\n") self.physical_sp = self.logical_sp self._print(out) @@ -320,12 +313,21 @@ def save_variables(self, out: CWriter) -> None: self._print(out) var.memory_offset = var_offset stack_offset = var_offset - self.physical_sp - Stack._do_emit(out, var.item, stack_offset, self.cast_type, self.extract_bits) + Stack._do_emit(out, var.item, stack_offset) self._print(out) var_offset = var_offset.push(var.item) + def stack_bound_check(self, out: CWriter) -> None: + if not self.check_stack_bounds: + return + if self.physical_sp != self.logical_sp: + diff = self.logical_sp - self.physical_sp + out.start_line() + out.emit(f"CHECK_STACK_BOUNDS({diff});\n") + def flush(self, out: CWriter) -> None: self._print(out) + self.stack_bound_check(out) self.save_variables(out) self._save_physical_sp(out) out.start_line() @@ -350,11 +352,12 @@ def _print(self, out: CWriter) -> None: out.emit(self.as_comment() + "\n") def copy(self) -> "Stack": - other = Stack(self.extract_bits, self.cast_type) + other = Stack() other.base_offset = self.base_offset other.physical_sp = self.physical_sp other.logical_sp = self.logical_sp other.variables = [var.copy() for var in self.variables] + other.check_stack_bounds = self.check_stack_bounds return other def __eq__(self, other: object) -> bool: @@ -496,7 +499,7 @@ def _push_defined_outputs(self) -> None: f"Expected '{undefined}' to be defined before '{out.name}'" else: undefined = out.name - while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[0]): + while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[self.peeks]): out = self.outputs.pop(self.peeks) self.stack.push(out) diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index ca151ff640a..f633f704485 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -26,19 +26,31 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for name, op in analysis.opmap.items(): if op < 256: targets[op] = f"&&TARGET_{name},\n" - out.emit("#if !Py_TAIL_CALL_INTERP\n") - out.emit("static void *opcode_targets[256] = {\n") + out.emit("#if !_Py_TAIL_CALL_INTERP\n") + out.emit("static void *opcode_targets_table[256] = {\n") for target in targets: out.emit(target) out.emit("};\n") - out.emit("#else /* Py_TAIL_CALL_INTERP */\n") + targets = ["&&_unknown_opcode,\n"] * 256 + for name, op in analysis.opmap.items(): + if op < 256: + targets[op] = f"&&TARGET_TRACE_RECORD,\n" + out.emit("#if _Py_TIER2\n") + out.emit("static void *opcode_tracing_targets_table[256] = {\n") + for target in targets: + out.emit(target) + out.emit("};\n") + out.emit(f"#endif\n") + out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: return f"Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_{name}(TAIL_CALL_PARAMS)" def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256];\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") + out.emit("\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") # Emit function prototypes for labels. @@ -60,7 +72,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -68,7 +80,17 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: if rest not in named_values: out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") - outfile.write("#endif /* Py_TAIL_CALL_INTERP */\n") + + # Emit the tracing dispatch table. + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_TRACE_RECORD,\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") + outfile.write("#endif /* _Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( description="Generate the file with dispatch targets.", diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 32dc346d5e8..c7ff5de681e 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -132,7 +132,7 @@ def uses_this(inst: Instruction) -> bool: continue for tkn in uop.body.tokens(): if (tkn.kind == "IDENTIFIER" - and (tkn.text in {"DEOPT_IF", "EXIT_IF"})): + and (tkn.text in {"DEOPT_IF", "EXIT_IF", "AT_END_EXIT_IF"})): return True return False @@ -157,20 +157,20 @@ def generate_tier1( #define TIER_ONE 1 """) outfile.write(f""" -#if !Py_TAIL_CALL_INTERP +#if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif {{ -#endif /* Py_TAIL_CALL_INTERP */ +#endif /* _Py_TAIL_CALL_INTERP */ {INSTRUCTION_START_MARKER} """ ) generate_tier1_cases(analysis, outfile, lines) outfile.write(f""" {INSTRUCTION_END_MARKER} -#if !Py_TAIL_CALL_INTERP +#if !_Py_TAIL_CALL_INTERP #if USE_COMPUTED_GOTOS _unknown_opcode: #else @@ -186,7 +186,7 @@ def generate_tier1( /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); -#endif /* Py_TAIL_CALL_INTERP */ +#endif /* _Py_TAIL_CALL_INTERP */ {LABEL_START_MARKER} """) out = CWriter(outfile, 2, lines) @@ -226,7 +226,7 @@ def generate_tier1_cases( popped = get_popped(inst, analysis) # We need to ifdef it because this breaks platforms # without computed gotos/tail calling. - out.emit(f"#if Py_TAIL_CALL_INTERP\n") + out.emit(f"#if _Py_TAIL_CALL_INTERP\n") out.emit(f"int opcode = {name};\n") out.emit(f"(void)(opcode);\n") out.emit(f"#endif\n") diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 276f306dfff..ac3e6b94afe 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,6 +63,7 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self._replacers["IP_OFFSET_OF"] = self.ip_offset_of def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -91,7 +92,7 @@ def deopt_if( self.emit("}\n") return not always_true(first_tkn) - def exit_if( # type: ignore[override] + def exit_if( self, tkn: Token, tkn_iter: TokenIterator, @@ -111,6 +112,8 @@ def exit_if( # type: ignore[override] self.emit("}\n") return not always_true(first_tkn) + periodic_if = deopt_if + def oparg( self, tkn: Token, @@ -132,10 +135,30 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True + def ip_offset_of( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + assert uop.name.startswith("_GUARD_IP") + # LPAREN + next(tkn_iter) + tok = next(tkn_iter) + self.emit(f" OFFSET_OF_{tok.text};\n") + # RPAREN + next(tkn_iter) + # SEMI + next(tkn_iter) + return True -def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: +def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, tuple[str, str]]) -> Stack: locals: dict[str, Local] = {} try: + if name_offset_pair := offset_strs.get(uop.name): + emitter.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") emitter.out.start_line() if uop.properties.oparg: emitter.emit("oparg = CURRENT_OPARG();\n") @@ -156,6 +179,8 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: idx += 1 _, storage = emitter.emit_tokens(uop, storage, None, False) storage.flush(emitter.out) + if name_offset_pair: + emitter.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") except StackError as ex: raise analysis_error(ex.args[0], uop.body.open) from None return storage.stack @@ -163,6 +188,29 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: SKIPS = ("_EXTENDED_ARG",) +def populate_offset_strs(analysis: Analysis) -> dict[str, tuple[str, str]]: + offset_strs: dict[str, tuple[str, str]] = {} + for name, uop in analysis.uops.items(): + if not f"_GUARD_IP_{name}" in analysis.uops: + continue + tkn_iter = uop.body.tokens() + found = False + offset_str = "" + for token in tkn_iter: + if token.kind == "IDENTIFIER" and token.text == "LOAD_IP": + if found: + raise analysis_error("Cannot have two LOAD_IP in a guarded single uop.", uop.body.open) + offset = [] + while token.kind != "SEMI": + offset.append(token.text) + token = next(tkn_iter) + # 1: to remove the LOAD_IP text + offset_str = "".join(offset[1:]) + found = True + assert offset_str + offset_strs[f"_GUARD_IP_{name}"] = (name, offset_str) + return offset_strs + def generate_tier2( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -177,7 +225,9 @@ def generate_tier2( ) out = CWriter(outfile, 2, lines) emitter = Tier2Emitter(out, analysis.labels) + offset_strs = populate_offset_strs(analysis) out.emit("\n") + for name, uop in analysis.uops.items(): if uop.properties.tier == 1: continue @@ -192,13 +242,15 @@ def generate_tier2( out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() - stack = write_uop(uop, emitter, stack) + stack = write_uop(uop, emitter, stack, offset_strs) out.start_line() if not uop.properties.always_exits: out.emit("break;\n") out.start_line() out.emit("}") out.emit("\n\n") + + out.emit("\n") outfile.write("#undef TIER_TWO\n") diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 6f995e5c46b..0e0396e5143 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -23,21 +23,23 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") - out.emit("extern const uint8_t _PyUop_Replication[MAX_UOP_ID+1];\n") + out.emit("extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange;\n") + out.emit("extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") - out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") + out.emit("const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): if uop.is_viable() and uop.properties.tier != 1: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") out.emit("};\n\n") - out.emit("const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = {\n") + out.emit("const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): if uop.replicated: - out.emit(f"[{uop.name}] = {uop.replicated},\n") + assert(uop.replicated.step == 1) + out.emit(f"[{uop.name}] = {{ {uop.replicated.start}, {uop.replicated.stop} }},\n") out.emit("};\n\n") out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt new file mode 100644 index 00000000000..e81ffd51e19 --- /dev/null +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -0,0 +1,93 @@ +# pydtrace_probes.h +PyDTrace_AUDIT +PyDTrace_FUNCTION_ENTRY +PyDTrace_FUNCTION_RETURN +PyDTrace_GC_DONE +PyDTrace_GC_START +PyDTrace_IMPORT_FIND_LOAD_DONE +PyDTrace_IMPORT_FIND_LOAD_START +PyDTrace_INSTANCE_DELETE_DONE +PyDTrace_INSTANCE_DELETE_START +PyDTrace_INSTANCE_NEW_DONE +PyDTrace_INSTANCE_NEW_START +PyDTrace_LINE +# fileobject.h +Py_FileSystemDefaultEncodeErrors +Py_FileSystemDefaultEncoding +Py_HasFileSystemDefaultEncoding +Py_UTF8Mode +# pyhash.h +Py_HASH_EXTERNAL +# exports.h +PyAPI_DATA +Py_EXPORTED_SYMBOL +Py_IMPORTED_SYMBOL +Py_LOCAL_SYMBOL +# modsupport.h +PyABIInfo_FREETHREADING_AGNOSTIC +# moduleobject.h +PyModuleDef_Type +# object.h +Py_INVALID_SIZE +Py_TPFLAGS_HAVE_VERSION_TAG +Py_TPFLAGS_INLINE_VALUES +Py_TPFLAGS_IS_ABSTRACT +# pyexpat.h +PyExpat_CAPI_MAGIC +PyExpat_CAPSULE_NAME +# pyport.h +Py_ALIGNED +Py_ARITHMETIC_RIGHT_SHIFT +Py_CAN_START_THREADS +Py_FORCE_EXPANSION +Py_GCC_ATTRIBUTE +Py_LL +Py_SAFE_DOWNCAST +Py_ULL +Py_VA_COPY +# unicodeobject.h +Py_UNICODE_SIZE +# cpython/methodobject.h +PyCFunction_GET_CLASS +# cpython/compile.h +PyCF_ALLOW_INCOMPLETE_INPUT +PyCF_COMPILE_MASK +PyCF_DONT_IMPLY_DEDENT +PyCF_IGNORE_COOKIE +PyCF_MASK +PyCF_MASK_OBSOLETE +PyCF_SOURCE_IS_UTF8 +# cpython/descrobject.h +PyDescr_COMMON +PyDescr_NAME +PyDescr_TYPE +PyWrapperFlag_KEYWORDS +# cpython/fileobject.h +PyFile_NewStdPrinter +PyStdPrinter_Type +Py_UniversalNewlineFgets +# cpython/setobject.h +PySet_MINSIZE +# cpython/ceval.h +PyUnstable_CopyPerfMapFile +PyUnstable_PerfTrampoline_CompileCode +PyUnstable_PerfTrampoline_SetPersistAfterFork +# cpython/genobject.h +PyAsyncGenASend_CheckExact +# cpython/longintrepr.h +PyLong_BASE +PyLong_MASK +PyLong_SHIFT +# cpython/pyerrors.h +PyException_HEAD +# cpython/pyframe.h +PyUnstable_EXECUTABLE_KINDS +PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION +PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR +PyUnstable_EXECUTABLE_KIND_PY_FUNCTION +PyUnstable_EXECUTABLE_KIND_SKIP +# cpython/pylifecycle.h +Py_FrozenMain +# cpython/unicodeobject.h +PyUnicode_IS_COMPACT +PyUnicode_IS_COMPACT_ASCII diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py new file mode 100644 index 00000000000..6bdf80a9ae8 --- /dev/null +++ b/Tools/check-c-api-docs/main.py @@ -0,0 +1,193 @@ +import re +from pathlib import Path +import sys +import _colorize +import textwrap + +SIMPLE_FUNCTION_REGEX = re.compile(r"PyAPI_FUNC(.+) (\w+)\(") +SIMPLE_MACRO_REGEX = re.compile(r"# *define *(\w+)(\(.+\))? ") +SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)") +SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") + +CPYTHON = Path(__file__).parent.parent.parent +INCLUDE = CPYTHON / "Include" +C_API_DOCS = CPYTHON / "Doc" / "c-api" +IGNORED = ( + (CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt") + .read_text() + .split("\n") +) + +for index, line in enumerate(IGNORED): + if line.startswith("#"): + IGNORED.pop(index) + +MISTAKE = """ +If this is a mistake and this script should not be failing, create an +issue and tag Peter (@ZeroIntensity) on it.\ +""" + + +def found_undocumented(singular: bool) -> str: + some = "an" if singular else "some" + s = "" if singular else "s" + these = "this" if singular else "these" + them = "it" if singular else "them" + were = "was" if singular else "were" + + return ( + textwrap.dedent( + f""" + Found {some} undocumented C API{s}! + + Python requires documentation on all public C API symbols, macros, and types. + If {these} API{s} {were} not meant to be public, prefix {them} with a + leading underscore (_PySomething_API) or move {them} to the internal C API + (pycore_*.h files). + + In exceptional cases, certain APIs can be ignored by adding them to + Tools/check-c-api-docs/ignored_c_api.txt + """ + ) + + MISTAKE + ) + + +def found_ignored_documented(singular: bool) -> str: + some = "a" if singular else "some" + s = "" if singular else "s" + them = "it" if singular else "them" + were = "was" if singular else "were" + they = "it" if singular else "they" + + return ( + textwrap.dedent( + f""" + Found {some} C API{s} listed in Tools/c-api-docs-check/ignored_c_api.txt, but + {they} {were} found in the documentation. To fix this, remove {them} from + ignored_c_api.txt. + """ + ) + + MISTAKE + ) + + +def is_documented(name: str) -> bool: + """ + Is a name present in the C API documentation? + """ + for path in C_API_DOCS.iterdir(): + if path.is_dir(): + continue + if path.suffix != ".rst": + continue + + text = path.read_text(encoding="utf-8") + if name in text: + return True + + return False + + +def scan_file_for_docs(filename: str, text: str) -> tuple[list[str], list[str]]: + """ + Scan a header file for C API functions. + """ + undocumented: list[str] = [] + documented_ignored: list[str] = [] + colors = _colorize.get_colors() + + def check_for_name(name: str) -> None: + documented = is_documented(name) + if documented and (name in IGNORED): + documented_ignored.append(name) + elif not documented and (name not in IGNORED): + undocumented.append(name) + + for function in SIMPLE_FUNCTION_REGEX.finditer(text): + name = function.group(2) + if not name.startswith("Py"): + continue + + check_for_name(name) + + for macro in SIMPLE_MACRO_REGEX.finditer(text): + name = macro.group(1) + if not name.startswith("Py"): + continue + + if "(" in name: + name = name[: name.index("(")] + + check_for_name(name) + + for inline in SIMPLE_INLINE_REGEX.finditer(text): + name = inline.group(2) + if not name.startswith("Py"): + continue + + check_for_name(name) + + for data in SIMPLE_DATA_REGEX.finditer(text): + name = data.group(1) + if not name.startswith("Py"): + continue + + check_for_name(name) + + # Remove duplicates and sort alphabetically to keep the output deterministic + undocumented = list(set(undocumented)) + undocumented.sort() + + if undocumented or documented_ignored: + print(f"{filename} {colors.RED}BAD{colors.RESET}") + for name in undocumented: + print(f"{colors.BOLD_RED}UNDOCUMENTED:{colors.RESET} {name}") + for name in documented_ignored: + print(f"{colors.BOLD_YELLOW}DOCUMENTED BUT IGNORED:{colors.RESET} {name}") + else: + print(f"{filename} {colors.GREEN}OK{colors.RESET}") + + return undocumented, documented_ignored + + +def main() -> None: + print("Scanning for undocumented C API functions...") + files = [*INCLUDE.iterdir(), *(INCLUDE / "cpython").iterdir()] + all_missing: list[str] = [] + all_found_ignored: list[str] = [] + + for file in files: + if file.is_dir(): + continue + assert file.exists() + text = file.read_text(encoding="utf-8") + missing, ignored = scan_file_for_docs(str(file.relative_to(INCLUDE)), text) + all_found_ignored += ignored + all_missing += missing + + fail = False + to_check = [ + (all_missing, "missing", found_undocumented(len(all_missing) == 1)), + ( + all_found_ignored, + "documented but ignored", + found_ignored_documented(len(all_found_ignored) == 1), + ), + ] + for name_list, what, message in to_check: + if not name_list: + continue + + s = "s" if len(name_list) != 1 else "" + print(f"-- {len(name_list)} {what} C API{s} --") + for name in name_list: + print(f" - {name}") + print(message) + fail = True + + sys.exit(1 if fail else 0) + + +if __name__ == "__main__": + main() diff --git a/Tools/clinic/.ruff.toml b/Tools/clinic/.ruff.toml index 5033887df0c..944d17ee3e9 100644 --- a/Tools/clinic/.ruff.toml +++ b/Tools/clinic/.ruff.toml @@ -17,9 +17,6 @@ ignore = [ # Use f-strings instead of format specifiers. # Doesn't always make code more readable. "UP032", - # Use PEP-604 unions rather than tuples for isinstance() checks. - # Makes code slower and more verbose. https://github.com/astral-sh/ruff/issues/7871. - "UP038", ] unfixable = [ # The autofixes sometimes do the wrong things for these; diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 7c5cede2396..9e9bdeadcc0 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -84,6 +84,7 @@ "argsbuf", "fastargs", "kwargs", + "kwds", "kwnames", "nargs", "noptargs", diff --git a/Tools/clinic/libclinic/app.py b/Tools/clinic/libclinic/app.py index 632bed3ce53..9e8cec5320f 100644 --- a/Tools/clinic/libclinic/app.py +++ b/Tools/clinic/libclinic/app.py @@ -255,14 +255,14 @@ def _module_and_class( cls: Class | None = None for idx, field in enumerate(fields): + fullname = ".".join(fields[:idx + 1]) if not isinstance(parent, Class): - if field in parent.modules: - parent = module = parent.modules[field] + if fullname in parent.modules: + parent = module = parent.modules[fullname] continue if field in parent.classes: parent = cls = parent.classes[field] else: - fullname = ".".join(fields[idx:]) fail(f"Parent class or module {fullname!r} does not exist.") return module, cls diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py index 2c93dda3541..ac66e79f93b 100644 --- a/Tools/clinic/libclinic/converter.py +++ b/Tools/clinic/libclinic/converter.py @@ -274,7 +274,7 @@ def _render_non_self( data.modifications.append('/* modifications for ' + name + ' */\n' + modifications.rstrip()) # keywords - if parameter.is_vararg(): + if parameter.is_variable_length(): pass elif parameter.is_positional_only(): data.keywords.append('') diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 633fb5f56a6..bc21ae84e1c 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -17,6 +17,87 @@ TypeSet = set[bltns.type[object]] +class BaseUnsignedIntConverter(CConverter): + bitwise = False + + def use_converter(self) -> None: + if self.converter: + self.add_include('pycore_long.h', + f'{self.converter}()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.bitwise: + result = self.format_code(""" + {{{{ + Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) {{{{ + goto exit; + }}}} + if ((size_t)_bytes > sizeof({type})) {{{{ + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + {{{{ + goto exit; + }}}} + }}}} + }}}} + """, + argname=argname, + type=self.type, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi)) + if self.format_unit in ('k', 'K'): + result = self.format_code(""" + if (!PyIndex_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}}""", + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi)) + result + return result + + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + return self.format_code(""" + {{{{ + Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_REJECT_NEGATIVE | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) {{{{ + goto exit; + }}}} + if ((size_t)_bytes > sizeof({type})) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C {type}"); + goto exit; + }}}} + }}}} + """, + argname=argname, + type=self.type) + + +class uint8_converter(BaseUnsignedIntConverter): + type = "uint8_t" + converter = '_PyLong_UInt8_Converter' + +class uint16_converter(BaseUnsignedIntConverter): + type = "uint16_t" + converter = '_PyLong_UInt16_Converter' + +class uint32_converter(BaseUnsignedIntConverter): + type = "uint32_t" + converter = '_PyLong_UInt32_Converter' + +class uint64_converter(BaseUnsignedIntConverter): + type = "uint64_t" + converter = '_PyLong_UInt64_Converter' + + class bool_converter(CConverter): type = 'int' default_type = bool @@ -124,13 +205,14 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st @add_legacy_c_converter('B', bitwise=True) -class unsigned_char_converter(CConverter): +class unsigned_char_converter(BaseUnsignedIntConverter): type = 'unsigned char' default_type = int format_unit = 'b' c_ignored_default = "'\0'" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'B' @@ -158,19 +240,6 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - elif self.format_unit == 'B': - return self.format_code(""" - {{{{ - unsigned long ival = PyLong_AsUnsignedLongMask({argname}); - if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else {{{{ - {paramname} = (unsigned char) ival; - }}}} - }}}} - """, - argname=argname) return super().parse_arg(argname, displayname, limited_capi=limited_capi) @@ -211,57 +280,18 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -def format_inline_unsigned_int_converter(self: CConverter, argname: str) -> str: - return self.format_code(""" - {{{{ - Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}), - Py_ASNATIVEBYTES_NATIVE_ENDIAN | - Py_ASNATIVEBYTES_ALLOW_INDEX | - Py_ASNATIVEBYTES_REJECT_NEGATIVE | - Py_ASNATIVEBYTES_UNSIGNED_BUFFER); - if (_bytes < 0) {{{{ - goto exit; - }}}} - if ((size_t)_bytes > sizeof({type})) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C {type}"); - goto exit; - }}}} - }}}} - """, - argname=argname, - type=self.type) - - -class unsigned_short_converter(CConverter): +class unsigned_short_converter(BaseUnsignedIntConverter): type = 'unsigned short' default_type = int c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'H' else: self.converter = '_PyLong_UnsignedShort_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedShort_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedShort_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'H': - return self.format_code(""" - {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) - @add_legacy_c_converter('C', accept={str}) class int_converter(CConverter): @@ -311,35 +341,18 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_int_converter(CConverter): +class unsigned_int_converter(BaseUnsignedIntConverter): type = 'unsigned int' default_type = int c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'I' else: self.converter = '_PyLong_UnsignedInt_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedInt_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedInt_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'I': - return self.format_code(""" - {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) - class long_converter(CConverter): type = 'long' @@ -359,38 +372,18 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_long_converter(CConverter): +class unsigned_long_converter(BaseUnsignedIntConverter): type = 'unsigned long' default_type = int c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'k' else: self.converter = '_PyLong_UnsignedLong_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedLong_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedLong_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'k': - return self.format_code(""" - if (!PyIndex_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) - class long_long_converter(CConverter): type = 'long long' @@ -410,58 +403,56 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_long_long_converter(CConverter): +class unsigned_long_long_converter(BaseUnsignedIntConverter): type = 'unsigned long long' default_type = int c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'K' else: self.converter = '_PyLong_UnsignedLongLong_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedLongLong_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedLongLong_Converter()') - - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'K': - return self.format_code(""" - if (!PyIndex_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) - class Py_ssize_t_converter(CConverter): type = 'Py_ssize_t' c_ignored_default = "0" - def converter_init(self, *, accept: TypeSet = {int}) -> None: + def converter_init(self, *, accept: TypeSet = {int}, + allow_negative: bool = True) -> None: + self.allow_negative = allow_negative if accept == {int}: self.format_unit = 'n' self.default_type = int elif accept == {int, NoneType}: - self.converter = '_Py_convert_optional_to_ssize_t' + if self.allow_negative: + self.converter = '_Py_convert_optional_to_ssize_t' + else: + self.converter = '_Py_convert_optional_to_non_negative_ssize_t' else: fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") def use_converter(self) -> None: - if self.converter == '_Py_convert_optional_to_ssize_t': - self.add_include('pycore_abstract.h', - '_Py_convert_optional_to_ssize_t()') + if self.converter in { + '_Py_convert_optional_to_ssize_t', + '_Py_convert_optional_to_non_negative_ssize_t', + }: + self.add_include('pycore_abstract.h', f'{self.converter}()') def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.allow_negative: + non_negative_check = '' + else: + non_negative_check = self.format_code(""" + if ({paramname} < 0) {{{{ + PyErr_SetString(PyExc_ValueError, + "{paramname} cannot be negative"); + goto exit; + }}}}""", + argname=argname, + ) if self.format_unit == 'n': if limited_capi: PyNumber_Index = 'PyNumber_Index' @@ -479,11 +470,13 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st if (ival == -1 && PyErr_Occurred()) {{{{ goto exit; }}}} - {paramname} = ival; + {paramname} = ival;{non_negative_check} }}}} """, argname=argname, - PyNumber_Index=PyNumber_Index) + PyNumber_Index=PyNumber_Index, + non_negative_check=non_negative_check, + ) if not limited_capi: return super().parse_arg(argname, displayname, limited_capi=limited_capi) return self.format_code(""" @@ -492,7 +485,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st {paramname} = PyNumber_AsSsize_t({argname}, PyExc_OverflowError); if ({paramname} == -1 && PyErr_Occurred()) {{{{ goto exit; - }}}} + }}}}{non_negative_check} }}}} else {{{{ {bad_argument} @@ -502,6 +495,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st """, argname=argname, bad_argument=self.bad_argument(displayname, 'integer or None', limited_capi=limited_capi), + non_negative_check=non_negative_check, ) @@ -557,15 +551,11 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st argname=argname) -class size_t_converter(CConverter): +class size_t_converter(BaseUnsignedIntConverter): type = 'size_t' converter = '_PyLong_Size_t_Converter' c_ignored_default = "0" - def use_converter(self) -> None: - self.add_include('pycore_long.h', - '_PyLong_Size_t_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'n': return self.format_code(""" @@ -575,9 +565,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class fildes_converter(CConverter): @@ -928,6 +916,26 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) +class _unicode_fs_converter_base(CConverter): + type = 'PyObject *' + + def converter_init(self) -> None: + if self.default is not unspecified: + fail(f"{self.__class__.__name__} does not support default values") + self.c_default = 'NULL' + + def cleanup(self) -> str: + return f"Py_XDECREF({self.parser_name});" + + +class unicode_fs_encoded_converter(_unicode_fs_converter_base): + converter = 'PyUnicode_FSConverter' + + +class unicode_fs_decoded_converter(_unicode_fs_converter_base): + converter = 'PyUnicode_FSDecoder' + + @add_legacy_c_converter('u') @add_legacy_c_converter('u#', zeroes=True) @add_legacy_c_converter('Z', accept={str, NoneType}) @@ -1258,13 +1266,12 @@ def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, }}}} """ else: - self.add_include('pycore_tuple.h', '_PyTuple_FromArray()') start = f'args + {max_pos}' if max_pos else 'args' size = f'nargs - {max_pos}' if max_pos else 'nargs' if min(pos_only, min_pos) < max_pos: return f""" {paramname} = nargs > {max_pos} - ? _PyTuple_FromArray({start}, {size}) + ? PyTuple_FromArray({start}, {size}) : PyTuple_New(0); if ({paramname} == NULL) {{{{ goto exit; @@ -1272,7 +1279,7 @@ def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, """ else: return f""" - {paramname} = _PyTuple_FromArray({start}, {size}); + {paramname} = PyTuple_FromArray({start}, {size}); if ({paramname} == NULL) {{{{ goto exit; }}}} @@ -1312,3 +1319,37 @@ def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, {paramname} = {start}; {self.length_name} = {size}; """ + + +# Converters for var-keyword parameters. + +class VarKeywordCConverter(CConverter): + format_unit = '' + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + raise AssertionError('should never be called') + + def parse_var_keyword(self) -> str: + raise NotImplementedError + + +class var_keyword_dict_converter(VarKeywordCConverter): + type = 'PyObject *' + c_default = 'NULL' + + def cleanup(self) -> str: + return f'Py_XDECREF({self.parser_name});\n' + + def parse_var_keyword(self) -> str: + param_name = self.parser_name + return f""" + if (kwargs == NULL) {{{{ + {param_name} = PyDict_New(); + if ({param_name} == NULL) {{{{ + goto exit; + }}}} + }}}} + else {{{{ + {param_name} = Py_NewRef(kwargs); + }}}} + """ diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index 282ff64cd33..0d83baeba9e 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -246,6 +246,7 @@ def dedent(self, line: str) -> str: class DSLParser: function: Function | None state: StateKeeper + expecting_parameters: bool keyword_only: bool positional_only: bool deprecated_positional: VersionTuple | None @@ -262,6 +263,8 @@ class DSLParser: target_critical_section: list[str] disable_fastcall: bool from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') + permit_long_summary = False + permit_long_docstring_body = False def __init__(self, clinic: Clinic) -> None: self.clinic = clinic @@ -283,6 +286,7 @@ def __init__(self, clinic: Clinic) -> None: def reset(self) -> None: self.function = None self.state = self.state_dsl_start + self.expecting_parameters = True self.keyword_only = False self.positional_only = False self.deprecated_positional = None @@ -298,6 +302,8 @@ def reset(self) -> None: self.critical_section = False self.target_critical_section = [] self.disable_fastcall = False + self.permit_long_summary = False + self.permit_long_docstring_body = False def directive_module(self, name: str) -> None: fields = name.split('.')[:-1] @@ -470,6 +476,16 @@ def at_text_signature(self, text_signature: str) -> None: fail("Called @text_signature twice!") self.forced_text_signature = text_signature + def at_permit_long_summary(self) -> None: + if self.permit_long_summary: + fail("Called @permit_long_summary twice!") + self.permit_long_summary = True + + def at_permit_long_docstring_body(self) -> None: + if self.permit_long_docstring_body: + fail("Called @permit_long_docstring_body twice!") + self.permit_long_docstring_body = True + def parse(self, block: Block) -> None: self.reset() self.block = block @@ -862,6 +878,10 @@ def state_parameter(self, line: str) -> None: def parse_parameter(self, line: str) -> None: assert self.function is not None + if not self.expecting_parameters: + fail('Encountered parameter line when not expecting ' + f'parameters: {line}') + match self.parameter_state: case ParamState.START | ParamState.REQUIRED: self.to_required() @@ -877,43 +897,16 @@ def parse_parameter(self, line: str) -> None: # handle "as" for parameters too c_name = None - name, have_as_token, trailing = line.partition(' as ') - if have_as_token: - name = name.strip() - if ' ' not in name: - fields = trailing.strip().split(' ') - if not fields: - fail("Invalid 'as' clause!") - c_name = fields[0] - if c_name.endswith(':'): - name += ':' - c_name = c_name[:-1] - fields[0] = name - line = ' '.join(fields) + m = re.match(r'(?:\* *)?\w+( +as +(\w+))', line) + if m: + c_name = m[2] + line = line[:m.start(1)] + line[m.end(1):] - default: str | None - base, equals, default = line.rpartition('=') - if not equals: - base = default - default = None - - module = None try: - ast_input = f"def x({base}): pass" + ast_input = f"def x({line}\n): pass" module = ast.parse(ast_input) except SyntaxError: - try: - # the last = was probably inside a function call, like - # c: int(accept={str}) - # so assume there was no actual default value. - default = None - ast_input = f"def x({line}): pass" - module = ast.parse(ast_input) - except SyntaxError: - pass - if not module: - fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", - repr(line)) + fail(f"Function {self.function.name!r} has an invalid parameter declaration: {line!r}") function = module.body[0] assert isinstance(function, ast.FunctionDef) @@ -922,30 +915,40 @@ def parse_parameter(self, line: str) -> None: if len(function_args.args) > 1: fail(f"Function {self.function.name!r} has an " f"invalid parameter declaration (comma?): {line!r}") - if function_args.defaults or function_args.kw_defaults: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (default value?): {line!r}") - if function_args.kwarg: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (**kwargs?): {line!r}") + is_vararg = is_var_keyword = False if function_args.vararg: self.check_previous_star() self.check_remaining_star() is_vararg = True parameter = function_args.vararg + elif function_args.kwarg: + # If the existing parameters are all positional only or ``*args`` + # (var-positional), then we allow ``**kwds`` (var-keyword). + # Currently, pos-or-keyword or keyword-only arguments are not + # allowed with the ``**kwds`` converter. + has_non_positional_param = any( + p.is_positional_or_keyword() or p.is_keyword_only() + for p in self.function.parameters.values() + ) + if has_non_positional_param: + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (**kwargs?): {line!r}") + is_var_keyword = True + parameter = function_args.kwarg else: - is_vararg = False parameter = function_args.args[0] parameter_name = parameter.arg name, legacy, kwargs = self.parse_converter(parameter.annotation) if is_vararg: - name = 'varpos_' + name + name = f'varpos_{name}' + elif is_var_keyword: + name = f'var_keyword_{name}' value: object - if not default: - if is_vararg: + if not function_args.defaults: + if is_vararg or is_var_keyword: value = NULL else: if self.parameter_state is ParamState.OPTIONAL: @@ -955,17 +958,13 @@ def parse_parameter(self, line: str) -> None: if 'py_default' in kwargs: fail("You can't specify py_default without specifying a default value!") else: - if is_vararg: - fail("Vararg can't take a default value!") + expr = function_args.defaults[0] + default = ast_input[expr.col_offset: expr.end_col_offset].strip() if self.parameter_state is ParamState.REQUIRED: self.parameter_state = ParamState.OPTIONAL - default = default.strip() bad = False - ast_input = f"x = {default}" try: - module = ast.parse(ast_input) - if 'c_default' not in kwargs: # we can only represent very simple data values in C. # detect whether default is okay, via a denylist @@ -992,13 +991,14 @@ def bad_node(self, node: ast.AST) -> None: visit_Starred = bad_node denylist = DetectBadNodes() - denylist.visit(module) + denylist.visit(expr) bad = denylist.bad else: # if they specify a c_default, we can be more lenient about the default value. # but at least make an attempt at ensuring it's a valid expression. + code = compile(ast.Expression(expr), '<expr>', 'eval') try: - value = eval(default) + value = eval(code) except NameError: pass # probably a named constant except Exception as e: @@ -1010,9 +1010,6 @@ def bad_node(self, node: ast.AST) -> None: if bad: fail(f"Unsupported expression as default value: {default!r}") - assignment = module.body[0] - assert isinstance(assignment, ast.Assign) - expr = assignment.value # mild hack: explicitly support NULL as a default value c_default: str | None if isinstance(expr, ast.Name) and expr.id == 'NULL': @@ -1064,8 +1061,6 @@ def bad_node(self, node: ast.AST) -> None: else: c_default = py_default - except SyntaxError as e: - fail(f"Syntax error: {e.text!r}") except (ValueError, AttributeError): value = unknown c_default = kwargs.get("c_default") @@ -1089,6 +1084,8 @@ def bad_node(self, node: ast.AST) -> None: kind: inspect._ParameterKind if is_vararg: kind = inspect.Parameter.VAR_POSITIONAL + elif is_var_keyword: + kind = inspect.Parameter.VAR_KEYWORD elif self.keyword_only: kind = inspect.Parameter.KEYWORD_ONLY else: @@ -1142,6 +1139,8 @@ def bad_node(self, node: ast.AST) -> None: if is_vararg: self.keyword_only = True + if is_var_keyword: + self.expecting_parameters = False @staticmethod def parse_converter( @@ -1183,6 +1182,9 @@ def parse_star(self, function: Function, version: VersionTuple | None) -> None: The 'version' parameter signifies the future version from which the marker will take effect (None means it is already in effect). """ + if not self.expecting_parameters: + fail("Encountered '*' when not expecting parameters") + if version is None: self.check_previous_star() self.check_remaining_star() @@ -1238,6 +1240,9 @@ def parse_slash(self, function: Function, version: VersionTuple | None) -> None: The 'version' parameter signifies the future version from which the marker will take effect (None means it is already in effect). """ + if not self.expecting_parameters: + fail("Encountered '/' when not expecting parameters") + if version is None: if self.deprecated_keyword: fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") @@ -1474,11 +1479,13 @@ def add_parameter(text: str) -> None: if p.is_vararg(): p_lines.append("*") added_star = True + if p.is_var_keyword(): + p_lines.append("**") name = p.converter.signature_name or p.name p_lines.append(name) - if not p.is_vararg() and p.converter.is_optional(): + if not p.is_variable_length() and p.converter.is_optional(): p_lines.append('=') value = p.converter.py_default if not value: @@ -1553,6 +1560,30 @@ def format_docstring(self) -> str: # between it and the {parameters} we're about to add. lines.append('') + # Fail if the summary line is too long. + # Warn if any of the body lines are too long. + # Existing violations are recorded in OVERLONG_{SUMMARY,BODY}. + max_width = f.docstring_line_width + summary_len = len(lines[0]) + max_body = max(map(len, lines[1:])) + if summary_len > max_width: + if not self.permit_long_summary: + fail(f"Summary line for {f.full_name!r} is too long!\n" + f"The summary line must be no longer than {max_width} characters.") + else: + if self.permit_long_summary: + warn("Remove the @permit_long_summary decorator from " + f"{f.full_name!r}!\n") + + if max_body > max_width: + if not self.permit_long_docstring_body: + warn(f"Docstring lines for {f.full_name!r} are too long!\n" + f"Lines should be no longer than {max_width} characters.") + else: + if self.permit_long_docstring_body: + warn("Remove the @permit_long_docstring_body decorator from " + f"{f.full_name!r}!\n") + parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 if parameters_marker_count > 1: fail('You may not specify {parameters} more than once in a docstring!') @@ -1583,8 +1614,11 @@ def check_remaining_star(self, lineno: int | None = None) -> None: for p in reversed(self.function.parameters.values()): if self.keyword_only: - if (p.kind == inspect.Parameter.KEYWORD_ONLY or - p.kind == inspect.Parameter.VAR_POSITIONAL): + if p.kind in { + inspect.Parameter.KEYWORD_ONLY, + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD + }: return elif self.deprecated_positional: if p.deprecated_positional == self.deprecated_positional: diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index e80e2f5f13f..f981f0bcaf8 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -167,6 +167,19 @@ def methoddef_flags(self) -> str | None: flags.append('METH_COEXIST') return '|'.join(flags) + @property + def docstring_line_width(self) -> int: + """Return the maximum line width for docstring lines. + + Pydoc adds indentation when displaying functions and methods. + To keep the total width of within 80 characters, we use a + maximum of 76 characters for global functions and classes, + and 72 characters for methods. + """ + if self.cls is not None and not self.kind.new_or_init: + return 72 + return 76 + def __repr__(self) -> str: return f'<clinic.Function {self.name!r}>' @@ -207,9 +220,18 @@ def is_keyword_only(self) -> bool: def is_positional_only(self) -> bool: return self.kind == inspect.Parameter.POSITIONAL_ONLY + def is_positional_or_keyword(self) -> bool: + return self.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + def is_vararg(self) -> bool: return self.kind == inspect.Parameter.VAR_POSITIONAL + def is_var_keyword(self) -> bool: + return self.kind == inspect.Parameter.VAR_KEYWORD + + def is_variable_length(self) -> bool: + return self.is_vararg() or self.is_var_keyword() + def is_optional(self) -> bool: return not self.is_vararg() and (self.default is not unspecified) diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index 0e15d2f163b..bca87ecd751 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -36,7 +36,7 @@ def declare_parser( num_keywords = len([ p for p in f.parameters.values() - if not p.is_positional_only() and not p.is_vararg() + if p.is_positional_or_keyword() or p.is_keyword_only() ]) condition = '#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)' @@ -220,6 +220,7 @@ class ParseArgsCodeGen: max_pos: int = 0 min_kw_only: int = 0 varpos: Parameter | None = None + var_keyword: Parameter | None = None docstring_prototype: str docstring_definition: str @@ -255,13 +256,24 @@ def __init__(self, func: Function, codegen: CodeGen) -> None: del self.parameters[i] break + for i, p in enumerate(self.parameters): + if p.is_var_keyword(): + self.var_keyword = p + del self.parameters[i] + break + self.converters = [p.converter for p in self.parameters] if self.func.critical_section: self.codegen.add_include('pycore_critical_section.h', 'Py_BEGIN_CRITICAL_SECTION()') + + # Use fastcall if not disabled, except if in a __new__ or + # __init__ method, or if there is a **kwargs parameter. if self.func.disable_fastcall: self.fastcall = False + elif self.var_keyword is not None: + self.fastcall = False else: self.fastcall = not self.is_new_or_init() @@ -469,6 +481,12 @@ def _parse_vararg(self) -> str: fastcall=self.fastcall, limited_capi=self.limited_capi) + def _parse_kwarg(self) -> str: + assert self.var_keyword is not None + c = self.var_keyword.converter + assert isinstance(c, libclinic.converters.VarKeywordCConverter) + return c.parse_var_keyword() + def parse_pos_only(self) -> None: if self.fastcall: # positional-only, but no option groups @@ -564,6 +582,8 @@ def parse_pos_only(self) -> None: parser_code.append("skip_optional:") if self.varpos: parser_code.append(libclinic.normalize_snippet(self._parse_vararg(), indent=4)) + elif self.var_keyword: + parser_code.append(libclinic.normalize_snippet(self._parse_kwarg(), indent=4)) else: for parameter in self.parameters: parameter.converter.use_converter() @@ -590,6 +610,45 @@ def parse_pos_only(self) -> None: """, indent=4)] self.parser_body(*parser_code) + def parse_var_keyword(self) -> None: + self.flags = "METH_VARARGS|METH_KEYWORDS" + self.parser_prototype = PARSER_PROTOTYPE_KEYWORD + nargs = 'PyTuple_GET_SIZE(args)' + + parser_code = [] + max_args = NO_VARARG if self.varpos else self.max_pos + if self.varpos is None and self.min_pos == self.max_pos == 0: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_NoPositional()') + parser_code.append(libclinic.normalize_snippet(""" + if (!_PyArg_NoPositional("{name}", args)) {{ + goto exit; + }} + """, indent=4)) + elif self.min_pos or max_args != NO_VARARG: + self.codegen.add_include('pycore_modsupport.h', + '_PyArg_CheckPositional()') + parser_code.append(libclinic.normalize_snippet(f""" + if (!_PyArg_CheckPositional("{{name}}", {nargs}, {self.min_pos}, {max_args})) {{{{ + goto exit; + }}}} + """, indent=4)) + + for i, p in enumerate(self.parameters): + parse_arg = p.converter.parse_arg( + f'PyTuple_GET_ITEM(args, {i})', + p.get_displayname(i+1), + limited_capi=self.limited_capi, + ) + assert parse_arg is not None + parser_code.append(libclinic.normalize_snippet(parse_arg, indent=4)) + + if self.varpos: + parser_code.append(libclinic.normalize_snippet(self._parse_vararg(), indent=4)) + if self.var_keyword: + parser_code.append(libclinic.normalize_snippet(self._parse_kwarg(), indent=4)) + self.parser_body(*parser_code) + def parse_general(self, clang: CLanguage) -> None: parsearg: str | None deprecated_positionals: dict[int, Parameter] = {} @@ -921,12 +980,14 @@ def parse_args(self, clang: CLanguage) -> dict[str, str]: # previous call to parser_body. this is used for an awful hack. self.parser_body_fields: tuple[str, ...] = () - if not self.parameters and not self.varpos: + if not self.parameters and not self.varpos and not self.var_keyword: self.parse_no_args() elif self.use_meth_o(): self.parse_one_arg() elif self.has_option_groups(): self.parse_option_groups() + elif self.var_keyword is not None: + self.parse_var_keyword() elif (not self.requires_defining_class and self.pos_only == len(self.parameters)): self.parse_pos_only() diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 926bc66b944..097a065f368 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -27,6 +27,8 @@ import sys import threading import time +from dataclasses import dataclass +from operator import methodcaller # The iterations in individual benchmarks are scaled by this factor. WORK_SCALE = 100 @@ -188,6 +190,29 @@ def thread_local_read(): _ = tmp.x _ = tmp.x +class MyClass: + __slots__ = () + + def func(self): + pass + +@register_benchmark +def method_caller(): + mc = methodcaller("func") + obj = MyClass() + for i in range(1000 * WORK_SCALE): + mc(obj) + +@dataclass +class MyDataClass: + x: int + y: int + z: int + +@register_benchmark +def instantiate_dataclass(): + for _ in range(1000 * WORK_SCALE): + obj = MyDataClass(x=1, y=2, z=3) def bench_one_thread(func): t0 = time.perf_counter_ns() diff --git a/Tools/i18n/.ruff.toml b/Tools/i18n/.ruff.toml new file mode 100644 index 00000000000..a8f4f2f5a96 --- /dev/null +++ b/Tools/i18n/.ruff.toml @@ -0,0 +1,10 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +target-version = "py313" + +[lint] +select = [ + "F", # pyflakes + "I", # isort + "UP", # pyupgrade +] diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py index b407a8a643b..f825862ffdf 100755 --- a/Tools/i18n/makelocalealias.py +++ b/Tools/i18n/makelocalealias.py @@ -8,6 +8,7 @@ """ import locale import sys + _locale = locale # Location of the X11 alias file. @@ -44,6 +45,13 @@ def parse(filename): # Ignore one letter locale mappings (except for 'c') if len(locale) == 1 and locale != 'c': continue + if '@' in locale and '@' not in alias: + # Do not simply remove the "@euro" modifier. + # Glibc generates separate locales with the "@euro" modifier, and + # not always generates a locale without it with the same encoding. + # It can also affect collation. + if locale.endswith('@euro') and not locale.endswith('.utf-8@euro'): + alias += '@euro' # Normalize encoding, if given if '.' in locale: lang, encoding = locale.split('.')[:2] @@ -51,6 +59,10 @@ def parse(filename): encoding = encoding.replace('_', '') locale = lang + '.' + encoding data[locale] = alias + # Conflict with glibc. + data.pop('el_gr@euro', None) + data.pop('uz_uz@cyrillic', None) + data.pop('uz_uz.utf8@cyrillic', None) return data def parse_glibc_supported(filename): @@ -81,7 +93,7 @@ def parse_glibc_supported(filename): # Add an encoding to alias alias, _, modifier = alias.partition('@') alias = _locale._replace_encoding(alias, alias_encoding) - if modifier and not (modifier == 'euro' and alias_encoding == 'ISO-8859-15'): + if modifier: alias += '@' + modifier data[locale] = alias return data @@ -89,16 +101,15 @@ def parse_glibc_supported(filename): def pprint(data): items = sorted(data.items()) for k, v in items: - print(' %-40s%a,' % ('%a:' % k, v)) + print(f" {k!a:<40}{v!a},") def print_differences(data, olddata): items = sorted(olddata.items()) for k, v in items: if k not in data: - print('# removed %a' % k) + print(f'# removed {k!a}') elif olddata[k] != data[k]: - print('# updated %a -> %a to %a' % \ - (k, olddata[k], data[k])) + print(f'# updated {k!a} -> {olddata[k]!a} to {data[k]!a}') # Additions are not mentioned def optimize(data): @@ -121,7 +132,7 @@ def check(data): errors = 0 for k, v in data.items(): if locale.normalize(k) != v: - print('ERROR: %a -> %a != %a' % (k, locale.normalize(k), v), + print(f'ERROR: {k!a} -> {locale.normalize(k)!a} != {v!a}', file=sys.stderr) errors += 1 return errors @@ -131,15 +142,18 @@ def check(data): parser = argparse.ArgumentParser() parser.add_argument('--locale-alias', default=LOCALE_ALIAS, help='location of the X11 alias file ' - '(default: %a)' % LOCALE_ALIAS) + f'(default: {LOCALE_ALIAS})') parser.add_argument('--glibc-supported', default=SUPPORTED, help='location of the glibc SUPPORTED locales file ' - '(default: %a)' % SUPPORTED) + f'(default: {SUPPORTED})') args = parser.parse_args() data = locale.locale_alias.copy() data.update(parse_glibc_supported(args.glibc_supported)) data.update(parse(args.locale_alias)) + # Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist + # on all platforms. + data['c.utf8'] = 'C.UTF-8' while True: # Repeat optimization while the size is decreased. n = len(data) diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index cd5f1ed9f3e..3351511bbc8 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -5,8 +5,7 @@ This program converts a textual Uniforum-style message catalog (.po file) into a binary GNU catalog (.mo file). This is essentially the same function as the -GNU msgfmt program, however, it is a simpler implementation. Currently it -does not handle plural forms but it does handle message contexts. +GNU msgfmt program, however, it is a simpler implementation. Usage: msgfmt.py [OPTIONS] filename.po @@ -25,14 +24,14 @@ Display version information and exit. """ -import os -import sys -import ast -import getopt -import struct import array -from email.parser import HeaderParser +import ast import codecs +import getopt +import os +import struct +import sys +from email.parser import HeaderParser __version__ = "1.2" @@ -114,7 +113,7 @@ def make(filename, outfile): try: with open(infile, 'rb') as f: lines = f.readlines() - except IOError as msg: + except OSError as msg: print(msg, file=sys.stderr) sys.exit(1) @@ -127,6 +126,7 @@ def make(filename, outfile): sys.exit(1) section = msgctxt = None + msgid = msgstr = b'' fuzzy = 0 # Start off assuming Latin-1, so everything decodes without failure, @@ -178,7 +178,7 @@ def make(filename, outfile): # This is a message with plural forms elif l.startswith('msgid_plural'): if section != ID: - print('msgid_plural not preceded by msgid on %s:%d' % (infile, lno), + print(f'msgid_plural not preceded by msgid on {infile}:{lno}', file=sys.stderr) sys.exit(1) l = l[12:] @@ -189,7 +189,7 @@ def make(filename, outfile): section = STR if l.startswith('msgstr['): if not is_plural: - print('plural without msgid_plural on %s:%d' % (infile, lno), + print(f'plural without msgid_plural on {infile}:{lno}', file=sys.stderr) sys.exit(1) l = l.split(']', 1)[1] @@ -197,7 +197,7 @@ def make(filename, outfile): msgstr += b'\0' # Separator of the various plural forms else: if is_plural: - print('indexed msgstr required for plural on %s:%d' % (infile, lno), + print(f'indexed msgstr required for plural on {infile}:{lno}', file=sys.stderr) sys.exit(1) l = l[6:] @@ -213,8 +213,7 @@ def make(filename, outfile): elif section == STR: msgstr += l.encode(encoding) else: - print('Syntax error on %s:%d' % (infile, lno), \ - 'before:', file=sys.stderr) + print(f'Syntax error on {infile}:{lno} before:', file=sys.stderr) print(l, file=sys.stderr) sys.exit(1) # Add last entry @@ -227,7 +226,7 @@ def make(filename, outfile): try: with open(outfile,"wb") as f: f.write(output) - except IOError as msg: + except OSError as msg: print(msg, file=sys.stderr) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index f46b05067d7..ddf4474d2bc 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -193,7 +193,7 @@ def make_escapes(pass_nonascii): escape = escape_ascii else: escape = escape_nonascii - escapes = [r"\%03o" % i for i in range(256)] + escapes = [fr"\{i:03o}" for i in range(256)] for i in range(32, 127): escapes[i] = chr(i) escapes[ord('\\')] = r'\\' @@ -796,7 +796,7 @@ class Options: try: with open(options.excludefilename) as fp: options.toexclude = fp.readlines() - except IOError: + except OSError: print(f"Can't read --exclude-file: {options.excludefilename}", file=sys.stderr) sys.exit(1) diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py new file mode 100644 index 00000000000..9c40c2f4492 --- /dev/null +++ b/Tools/inspection/benchmark_external_inspection.py @@ -0,0 +1,473 @@ +import _remote_debugging +import time +import subprocess +import sys +import contextlib +import tempfile +import os +import argparse +from _colorize import get_colors, can_colorize + +CODE = '''\ +import time +import os +import sys +import math + +def slow_fibonacci(n): + """Intentionally slow recursive fibonacci - should show up prominently in profiler""" + if n <= 1: + return n + return slow_fibonacci(n-1) + slow_fibonacci(n-2) + +def medium_computation(): + """Medium complexity function""" + result = 0 + for i in range(1000): + result += math.sqrt(i) * math.sin(i) + return result + +def fast_loop(): + """Fast simple loop""" + total = 0 + for i in range(100): + total += i + return total + +def string_operations(): + """String manipulation that should be visible in profiler""" + text = "hello world " * 100 + words = text.split() + return " ".join(reversed(words)) + +def nested_calls(): + """Nested function calls to test call stack depth""" + def level1(): + def level2(): + def level3(): + return medium_computation() + return level3() + return level2() + return level1() + +def main_loop(): + """Main computation loop with different execution paths""" + iteration = 0 + + while True: + iteration += 1 + + # Different execution paths with different frequencies + if iteration % 50 == 0: + # Expensive operation - should show high per-call time + result = slow_fibonacci(20) + + elif iteration % 10 == 0: + # Medium operation + result = nested_calls() + + elif iteration % 5 == 0: + # String operations + result = string_operations() + + else: + # Fast operation - most common + result = fast_loop() + + # Small delay to make sampling more interesting + time.sleep(0.001) + +if __name__ == "__main__": + main_loop() +''' + +DEEP_STATIC_CODE = """\ +import time +def factorial(n): + if n <= 1: + time.sleep(10000) + return 1 + return n * factorial(n-1) + +factorial(900) +""" + +CODE_WITH_TONS_OF_THREADS = '''\ +import time +import threading +import random +import math + +def cpu_intensive_work(): + """Do some CPU intensive calculations""" + result = 0 + for _ in range(10000): + result += math.sin(random.random()) * math.cos(random.random()) + return result + +def io_intensive_work(): + """Simulate IO intensive work with sleeps""" + time.sleep(0.1) + +def mixed_workload(): + """Mix of CPU and IO work""" + while True: + if random.random() < 0.3: + cpu_intensive_work() + else: + io_intensive_work() + +def create_threads(n): + """Create n threads doing mixed workloads""" + threads = [] + for _ in range(n): + t = threading.Thread(target=mixed_workload, daemon=True) + t.start() + threads.append(t) + return threads + +# Start with 5 threads +active_threads = create_threads(5) +thread_count = 5 + +# Main thread manages threads and does work +while True: + # Randomly add or remove threads + if random.random() < 0.1: # 10% chance each iteration + if random.random() < 0.5 and thread_count < 100: + # Add 1-5 new threads + new_count = random.randint(1, 5) + new_threads = create_threads(new_count) + active_threads.extend(new_threads) + thread_count += new_count + elif thread_count > 10: + # Remove 1-3 threads + remove_count = random.randint(1, 5) + # The threads will terminate naturally since they're daemons + active_threads = active_threads[remove_count:] + thread_count -= remove_count + + cpu_intensive_work() + time.sleep(0.05) +''' + +CODE_EXAMPLES = { + "basic": { + "code": CODE, + "description": "Mixed workload with fibonacci, computations, and string operations", + }, + "deep_static": { + "code": DEEP_STATIC_CODE, + "description": "Deep recursive call stack with 900+ frames (factorial)", + }, + "threads": { + "code": CODE_WITH_TONS_OF_THREADS, + "description": "Tons of threads doing mixed CPU/IO work", + }, +} + + +def benchmark(unwinder, duration_seconds=10): + """Benchmark mode - measure raw sampling speed for specified duration""" + sample_count = 0 + fail_count = 0 + total_work_time = 0.0 + start_time = time.perf_counter() + end_time = start_time + duration_seconds + total_attempts = 0 + + colors = get_colors(can_colorize()) + + print( + f"{colors.BOLD_BLUE}Benchmarking sampling speed for {duration_seconds} seconds...{colors.RESET}" + ) + + try: + while time.perf_counter() < end_time: + total_attempts += 1 + work_start = time.perf_counter() + try: + stack_trace = unwinder.get_stack_trace() + if stack_trace: + sample_count += 1 + except (OSError, RuntimeError, UnicodeDecodeError) as e: + fail_count += 1 + + work_end = time.perf_counter() + total_work_time += work_end - work_start + + if total_attempts % 10000 == 0: + avg_work_time_us = (total_work_time / total_attempts) * 1e6 + work_rate = ( + total_attempts / total_work_time if total_work_time > 0 else 0 + ) + success_rate = (sample_count / total_attempts) * 100 + + # Color code the success rate + if success_rate >= 95: + success_color = colors.GREEN + elif success_rate >= 80: + success_color = colors.YELLOW + else: + success_color = colors.RED + + print( + f"{colors.CYAN}Attempts:{colors.RESET} {total_attempts} | " + f"{colors.CYAN}Success:{colors.RESET} {success_color}{success_rate:.1f}%{colors.RESET} | " + f"{colors.CYAN}Rate:{colors.RESET} {colors.MAGENTA}{work_rate:.1f}Hz{colors.RESET} | " + f"{colors.CYAN}Avg:{colors.RESET} {colors.YELLOW}{avg_work_time_us:.2f}µs{colors.RESET}" + ) + except KeyboardInterrupt: + print(f"\n{colors.YELLOW}Benchmark interrupted by user{colors.RESET}") + + actual_end_time = time.perf_counter() + wall_time = actual_end_time - start_time + + # Return final statistics + return { + "wall_time": wall_time, + "total_attempts": total_attempts, + "sample_count": sample_count, + "fail_count": fail_count, + "success_rate": ( + (sample_count / total_attempts) * 100 if total_attempts > 0 else 0 + ), + "total_work_time": total_work_time, + "avg_work_time_us": ( + (total_work_time / total_attempts) * 1e6 if total_attempts > 0 else 0 + ), + "work_rate_hz": total_attempts / total_work_time if total_work_time > 0 else 0, + "samples_per_sec": sample_count / wall_time if wall_time > 0 else 0, + } + + +def print_benchmark_results(results): + """Print comprehensive benchmark results""" + colors = get_colors(can_colorize()) + + print(f"\n{colors.BOLD_GREEN}{'='*60}{colors.RESET}") + print(f"{colors.BOLD_GREEN}get_stack_trace() Benchmark Results{colors.RESET}") + print(f"{colors.BOLD_GREEN}{'='*60}{colors.RESET}") + + # Basic statistics + print(f"\n{colors.BOLD_CYAN}Basic Statistics:{colors.RESET}") + print( + f" {colors.CYAN}Wall time:{colors.RESET} {colors.YELLOW}{results['wall_time']:.3f}{colors.RESET} seconds" + ) + print( + f" {colors.CYAN}Total attempts:{colors.RESET} {colors.MAGENTA}{results['total_attempts']:,}{colors.RESET}" + ) + print( + f" {colors.CYAN}Successful samples:{colors.RESET} {colors.GREEN}{results['sample_count']:,}{colors.RESET}" + ) + print( + f" {colors.CYAN}Failed samples:{colors.RESET} {colors.RED}{results['fail_count']:,}{colors.RESET}" + ) + + # Color code the success rate + success_rate = results["success_rate"] + if success_rate >= 95: + success_color = colors.BOLD_GREEN + elif success_rate >= 80: + success_color = colors.BOLD_YELLOW + else: + success_color = colors.BOLD_RED + + print( + f" {colors.CYAN}Success rate:{colors.RESET} {success_color}{success_rate:.2f}%{colors.RESET}" + ) + + # Performance metrics + print(f"\n{colors.BOLD_CYAN}Performance Metrics:{colors.RESET}") + print( + f" {colors.CYAN}Average call time:{colors.RESET} {colors.YELLOW}{results['avg_work_time_us']:.2f}{colors.RESET} µs" + ) + print( + f" {colors.CYAN}Work rate:{colors.RESET} {colors.MAGENTA}{results['work_rate_hz']:.1f}{colors.RESET} calls/sec" + ) + print( + f" {colors.CYAN}Sample rate:{colors.RESET} {colors.MAGENTA}{results['samples_per_sec']:.1f}{colors.RESET} samples/sec" + ) + print( + f" {colors.CYAN}Total work time:{colors.RESET} {colors.YELLOW}{results['total_work_time']:.3f}{colors.RESET} seconds" + ) + + # Color code work efficiency + efficiency = (results["total_work_time"] / results["wall_time"]) * 100 + if efficiency >= 80: + efficiency_color = colors.GREEN + elif efficiency >= 50: + efficiency_color = colors.YELLOW + else: + efficiency_color = colors.RED + + print( + f" {colors.CYAN}Work efficiency:{colors.RESET} {efficiency_color}{efficiency:.1f}%{colors.RESET}" + ) + + +def parse_arguments(): + """Parse command line arguments""" + # Build the code examples description + examples_desc = "\n".join( + [f" {name}: {info['description']}" for name, info in CODE_EXAMPLES.items()] + ) + + parser = argparse.ArgumentParser( + description="Benchmark get_stack_trace() performance", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=f""" +Examples: + %(prog)s # Run basic benchmark for 10 seconds (default) + %(prog)s --duration 30 # Run basic benchmark for 30 seconds + %(prog)s -d 60 # Run basic benchmark for 60 seconds + %(prog)s --code deep_static # Run deep static call stack benchmark + %(prog)s --code deep_static -d 30 # Run deep static benchmark for 30 seconds + +Available code examples: +{examples_desc} + """, + color=True, + ) + + parser.add_argument( + "--duration", + "-d", + type=int, + default=10, + help="Benchmark duration in seconds (default: 10)", + ) + + parser.add_argument( + "--code", + "-c", + choices=list(CODE_EXAMPLES.keys()), + default="basic", + help="Code example to benchmark (default: basic)", + ) + + parser.add_argument( + "--threads", + choices=["all", "main", "only_active"], + default="all", + help="Which threads to include in the benchmark (default: all)", + ) + + return parser.parse_args() + + +def create_target_process(temp_file, code_example="basic"): + """Create and start the target process for benchmarking""" + example_info = CODE_EXAMPLES.get(code_example, {"code": CODE}) + selected_code = example_info["code"] + temp_file.write(selected_code) + temp_file.flush() + + process = subprocess.Popen( + [sys.executable, temp_file.name], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + # Give it time to start + time.sleep(1.0) + + # Check if it's still running + if process.poll() is not None: + stdout, stderr = process.communicate() + raise RuntimeError( + f"Target process exited unexpectedly:\nSTDOUT: {stdout.decode()}\nSTDERR: {stderr.decode()}" + ) + + return process, temp_file.name + + +def cleanup_process(process, temp_file_path): + """Clean up the target process and temporary file""" + with contextlib.suppress(Exception): + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5.0) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def main(): + """Main benchmark function""" + colors = get_colors(can_colorize()) + args = parse_arguments() + + print(f"{colors.BOLD_MAGENTA}External Inspection Benchmark Tool{colors.RESET}") + print(f"{colors.BOLD_MAGENTA}{'=' * 34}{colors.RESET}") + + example_info = CODE_EXAMPLES.get(args.code, {"description": "Unknown"}) + print( + f"\n{colors.CYAN}Code Example:{colors.RESET} {colors.GREEN}{args.code}{colors.RESET}" + ) + print(f"{colors.CYAN}Description:{colors.RESET} {example_info['description']}") + print( + f"{colors.CYAN}Benchmark Duration:{colors.RESET} {colors.YELLOW}{args.duration}{colors.RESET} seconds" + ) + + process = None + temp_file_path = None + + try: + # Create target process + print(f"\n{colors.BLUE}Creating and starting target process...{colors.RESET}") + with tempfile.NamedTemporaryFile(mode="w", suffix=".py") as temp_file: + process, temp_file_path = create_target_process(temp_file, args.code) + print( + f"{colors.GREEN}Target process started with PID: {colors.BOLD_WHITE}{process.pid}{colors.RESET}" + ) + + # Run benchmark with specified duration + with process: + # Create unwinder and run benchmark + print(f"{colors.BLUE}Initializing unwinder...{colors.RESET}") + try: + kwargs = {} + if args.threads == "all": + kwargs["all_threads"] = True + elif args.threads == "main": + kwargs["all_threads"] = False + elif args.threads == "only_active": + kwargs["only_active_thread"] = True + unwinder = _remote_debugging.RemoteUnwinder( + process.pid, cache_frames=True, **kwargs + ) + results = benchmark(unwinder, duration_seconds=args.duration) + finally: + cleanup_process(process, temp_file_path) + + # Print results + print_benchmark_results(results) + + except PermissionError as e: + print( + f"{colors.BOLD_RED}Error: Insufficient permissions to read stack trace: {e}{colors.RESET}" + ) + print( + f"{colors.YELLOW}Try running with appropriate privileges (e.g., sudo){colors.RESET}" + ) + return 1 + except Exception as e: + print(f"{colors.BOLD_RED}Error during benchmarking: {e}{colors.RESET}") + if process: + with contextlib.suppress(Exception): + stdout, stderr = process.communicate(timeout=1) + if stdout: + print( + f"{colors.CYAN}Process STDOUT:{colors.RESET} {stdout.decode()}" + ) + if stderr: + print( + f"{colors.RED}Process STDERR:{colors.RESET} {stderr.decode()}" + ) + raise + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 8e817574b4d..c70c0c47d94 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ ## Installing LLVM The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 19 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 19 on Ubuntu/Debian: +Install LLVM 21 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 19 +sudo ./llvm.sh 21 ``` -Install LLVM 19 on Fedora Linux 40 or newer: +Install LLVM 21 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 19' 'llvm(major) = 19' +sudo dnf install 'clang(major) = 21' 'llvm(major) = 21' ``` ### macOS -Install LLVM 19 with [Homebrew](https://brew.sh): +Install LLVM 21 with [Homebrew](https://brew.sh): ```sh -brew install llvm@19 +brew install llvm@21 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,14 +43,18 @@ ### Windows LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=19.1.0 +choco install llvm --version=21.1.0 ``` +### Dev Containers + +If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no +need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building @@ -62,6 +66,9 @@ ## Building The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit). +## Miscellaneous +If you're looking for information on how to update the JIT build dependencies, see [JIT Build Infrastructure](jit_infra.md). + [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f09a8404871..0b9cb5192f1 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -10,9 +10,9 @@ import _targets -_LLVM_VERSION = 19 -_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") -_EXTERNALS_LLVM_TAG = "llvm-19.1.7.0" + +_LLVM_VERSION = "21" +_EXTERNALS_LLVM_TAG = "llvm-21.1.4.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") @@ -56,53 +56,71 @@ async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str @_async_cache -async def _check_tool_version(name: str, *, echo: bool = False) -> bool: +async def _check_tool_version( + name: str, llvm_version: str, *, echo: bool = False +) -> bool: output = await _run(name, ["--version"], echo=echo) - return bool(output and _LLVM_VERSION_PATTERN.search(output)) + _llvm_version_pattern = re.compile(rf"version\s+{llvm_version}\.\d+\.\d+\S*\s+") + return bool(output and _llvm_version_pattern.search(output)) @_async_cache -async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None: - output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo) +async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str | None: + output = await _run("brew", ["--prefix", f"llvm@{llvm_version}"], echo=echo) return output and output.removesuffix("\n") @_async_cache -async def _find_tool(tool: str, *, echo: bool = False) -> str | None: +async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: # Unversioned executables: path = tool - if await _check_tool_version(path, echo=echo): + if await _check_tool_version(path, llvm_version, echo=echo): return path # Versioned executables: - path = f"{tool}-{_LLVM_VERSION}" - if await _check_tool_version(path, echo=echo): + path = f"{tool}-{llvm_version}" + if await _check_tool_version(path, llvm_version, echo=echo): return path # PCbuild externals: externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS) path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool) - if await _check_tool_version(path, echo=echo): + # On Windows, executables need .exe extension + if os.name == "nt" and not path.endswith(".exe"): + path_with_exe = path + ".exe" + if os.path.exists(path_with_exe): + path = path_with_exe + if await _check_tool_version(path, llvm_version, echo=echo): return path # Homebrew-installed executables: - prefix = await _get_brew_llvm_prefix(echo=echo) + prefix = await _get_brew_llvm_prefix(llvm_version, echo=echo) if prefix is not None: path = os.path.join(prefix, "bin", tool) - if await _check_tool_version(path, echo=echo): + if await _check_tool_version(path, llvm_version, echo=echo): return path # Nothing found: return None async def maybe_run( - tool: str, args: typing.Iterable[str], echo: bool = False + tool: str, + args: typing.Iterable[str], + echo: bool = False, + llvm_version: str = _LLVM_VERSION, ) -> str | None: """Run an LLVM tool if it can be found. Otherwise, return None.""" - path = await _find_tool(tool, echo=echo) + + path = await _find_tool(tool, llvm_version, echo=echo) return path and await _run(path, args, echo=echo) -async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str: +async def run( + tool: str, + args: typing.Iterable[str], + echo: bool = False, + llvm_version: str = _LLVM_VERSION, +) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" - output = await maybe_run(tool, args, echo=echo) + + output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version) if output is None: - raise RuntimeError(f"Can't find {tool}-{_LLVM_VERSION}!") + raise RuntimeError(f"Can't find {tool}-{llvm_version}!") return output diff --git a/Tools/jit/_optimizers.py b/Tools/jit/_optimizers.py new file mode 100644 index 00000000000..0adc550ba5e --- /dev/null +++ b/Tools/jit/_optimizers.py @@ -0,0 +1,375 @@ +"""Low-level optimization of textual assembly.""" + +import dataclasses +import pathlib +import re +import typing + +# Same as saying "not string.startswith('')": +_RE_NEVER_MATCH = re.compile(r"(?!)") +# Dictionary mapping branch instructions to their inverted branch instructions. +# If a branch cannot be inverted, the value is None: +_X86_BRANCH_NAMES = { + # https://www.felixcloutier.com/x86/jcc + "ja": "jna", + "jae": "jnae", + "jb": "jnb", + "jbe": "jnbe", + "jc": "jnc", + "jcxz": None, + "je": "jne", + "jecxz": None, + "jg": "jng", + "jge": "jnge", + "jl": "jnl", + "jle": "jnle", + "jo": "jno", + "jp": "jnp", + "jpe": "jpo", + "jrcxz": None, + "js": "jns", + "jz": "jnz", + # https://www.felixcloutier.com/x86/loop:loopcc + "loop": None, + "loope": None, + "loopne": None, + "loopnz": None, + "loopz": None, +} +# Update with all of the inverted branches, too: +_X86_BRANCH_NAMES |= {v: k for k, v in _X86_BRANCH_NAMES.items() if v} +# No custom relocations needed +_X86_BRANCHES: dict[str, tuple[str | None, str | None]] = { + k: (v, None) for k, v in _X86_BRANCH_NAMES.items() +} + +_AARCH64_COND_CODES = { + # https://developer.arm.com/documentation/dui0801/b/CJAJIHAD?lang=en + "eq": "ne", + "ne": "eq", + "lt": "ge", + "ge": "lt", + "gt": "le", + "le": "gt", + "vs": "vc", + "vc": "vs", + "mi": "pl", + "pl": "mi", + "cs": "cc", + "cc": "cs", + "hs": "lo", + "lo": "hs", + "hi": "ls", + "ls": "hi", +} +# MyPy doesn't understand that a invariant variable can be initialized by a covariant value +CUSTOM_AARCH64_BRANCH19: str | None = "CUSTOM_AARCH64_BRANCH19" + +# Branches are either b.{cond} or bc.{cond} +_AARCH64_BRANCHES: dict[str, tuple[str | None, str | None]] = { + "b." + cond: (("b." + inverse if inverse else None), CUSTOM_AARCH64_BRANCH19) + for (cond, inverse) in _AARCH64_COND_CODES.items() +} | { + "bc." + cond: (("bc." + inverse if inverse else None), CUSTOM_AARCH64_BRANCH19) + for (cond, inverse) in _AARCH64_COND_CODES.items() +} + + +@dataclasses.dataclass +class _Block: + label: str | None = None + # Non-instruction lines like labels, directives, and comments: + noninstructions: list[str] = dataclasses.field(default_factory=list) + # Instruction lines: + instructions: list[str] = dataclasses.field(default_factory=list) + # If this block ends in a jump, where to? + target: typing.Self | None = None + # The next block in the linked list: + link: typing.Self | None = None + # Whether control flow can fall through to the linked block above: + fallthrough: bool = True + # Whether this block can eventually reach the next uop (_JIT_CONTINUE): + hot: bool = False + + def resolve(self) -> typing.Self: + """Find the first non-empty block reachable from this one.""" + block = self + while block.link and not block.instructions: + block = block.link + return block + + +@dataclasses.dataclass +class Optimizer: + """Several passes of analysis and optimization for textual assembly.""" + + path: pathlib.Path + _: dataclasses.KW_ONLY + # Prefixes used to mangle local labels and symbols: + label_prefix: str + symbol_prefix: str + # The first block in the linked list: + _root: _Block = dataclasses.field(init=False, default_factory=_Block) + _labels: dict[str, _Block] = dataclasses.field(init=False, default_factory=dict) + # No groups: + _re_noninstructions: typing.ClassVar[re.Pattern[str]] = re.compile( + r"\s*(?:\.|#|//|;|$)" + ) + # One group (label): + _re_label: typing.ClassVar[re.Pattern[str]] = re.compile( + r'\s*(?P<label>[\w."$?@]+):' + ) + # Override everything that follows in subclasses: + _supports_external_relocations = True + _branches: typing.ClassVar[dict[str, tuple[str | None, str | None]]] = {} + # Two groups (instruction and target): + _re_branch: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH + # One group (target): + _re_jump: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH + # No groups: + _re_return: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH + + def __post_init__(self) -> None: + # Split the code into a linked list of basic blocks. A basic block is an + # optional label, followed by zero or more non-instruction lines, + # followed by zero or more instruction lines (only the last of which may + # be a branch, jump, or return): + text = self._preprocess(self.path.read_text()) + block = self._root + for line in text.splitlines(): + # See if we need to start a new block: + if match := self._re_label.match(line): + # Label. New block: + block.link = block = self._lookup_label(match["label"]) + block.noninstructions.append(line) + continue + if self._re_noninstructions.match(line): + if block.instructions: + # Non-instruction lines. New block: + block.link = block = _Block() + block.noninstructions.append(line) + continue + if block.target or not block.fallthrough: + # Current block ends with a branch, jump, or return. New block: + block.link = block = _Block() + block.instructions.append(line) + if match := self._re_branch.match(line): + # A block ending in a branch has a target and fallthrough: + block.target = self._lookup_label(match["target"]) + assert block.fallthrough + elif match := self._re_jump.match(line): + # A block ending in a jump has a target and no fallthrough: + block.target = self._lookup_label(match["target"]) + block.fallthrough = False + elif self._re_return.match(line): + # A block ending in a return has no target and fallthrough: + assert not block.target + block.fallthrough = False + + def _preprocess(self, text: str) -> str: + # Override this method to do preprocessing of the textual assembly. + # In all cases, replace references to the _JIT_CONTINUE symbol with + # references to a local _JIT_CONTINUE label (which we will add later): + continue_symbol = rf"\b{re.escape(self.symbol_prefix)}_JIT_CONTINUE\b" + continue_label = f"{self.label_prefix}_JIT_CONTINUE" + return re.sub(continue_symbol, continue_label, text) + + @classmethod + def _invert_branch(cls, line: str, target: str) -> str | None: + match = cls._re_branch.match(line) + assert match + inverted_reloc = cls._branches.get(match["instruction"]) + if inverted_reloc is None: + return None + inverted = inverted_reloc[0] + if not inverted: + return None + (a, b), (c, d) = match.span("instruction"), match.span("target") + # Before: + # je FOO + # After: + # jne BAR + return "".join([line[:a], inverted, line[b:c], target, line[d:]]) + + @classmethod + def _update_jump(cls, line: str, target: str) -> str: + match = cls._re_jump.match(line) + assert match + a, b = match.span("target") + # Before: + # jmp FOO + # After: + # jmp BAR + return "".join([line[:a], target, line[b:]]) + + def _lookup_label(self, label: str) -> _Block: + if label not in self._labels: + self._labels[label] = _Block(label) + return self._labels[label] + + def _blocks(self) -> typing.Generator[_Block, None, None]: + block: _Block | None = self._root + while block: + yield block + block = block.link + + def _body(self) -> str: + lines = [] + hot = True + for block in self._blocks(): + if hot != block.hot: + hot = block.hot + # Make it easy to tell at a glance where cold code is: + lines.append(f"# JIT: {'HOT' if hot else 'COLD'} ".ljust(80, "#")) + lines.extend(block.noninstructions) + lines.extend(block.instructions) + return "\n".join(lines) + + def _predecessors(self, block: _Block) -> typing.Generator[_Block, None, None]: + # This is inefficient, but it's never wrong: + for pre in self._blocks(): + if pre.target is block or pre.fallthrough and pre.link is block: + yield pre + + def _insert_continue_label(self) -> None: + # Find the block with the last instruction: + for end in reversed(list(self._blocks())): + if end.instructions: + break + # Before: + # jmp FOO + # After: + # jmp FOO + # _JIT_CONTINUE: + # This lets the assembler encode _JIT_CONTINUE jumps at build time! + continuation = self._lookup_label(f"{self.label_prefix}_JIT_CONTINUE") + assert continuation.label + continuation.noninstructions.append(f"{continuation.label}:") + end.link, continuation.link = continuation, end.link + + def _mark_hot_blocks(self) -> None: + # Start with the last block, and perform a DFS to find all blocks that + # can eventually reach it: + todo = list(self._blocks())[-1:] + while todo: + block = todo.pop() + block.hot = True + todo.extend(pre for pre in self._predecessors(block) if not pre.hot) + + def _invert_hot_branches(self) -> None: + for branch in self._blocks(): + link = branch.link + if link is None: + continue + jump = link.resolve() + # Before: + # je HOT + # jmp COLD + # After: + # jne COLD + # jmp HOT + if ( + # block ends with a branch to hot code... + branch.target + and branch.fallthrough + and branch.target.hot + # ...followed by a jump to cold code with no other predecessors: + and jump.target + and not jump.fallthrough + and not jump.target.hot + and len(jump.instructions) == 1 + and list(self._predecessors(jump)) == [branch] + ): + assert jump.target.label + assert branch.target.label + inverted = self._invert_branch( + branch.instructions[-1], jump.target.label + ) + # Check to see if the branch can even be inverted: + if inverted is None: + continue + branch.instructions[-1] = inverted + jump.instructions[-1] = self._update_jump( + jump.instructions[-1], branch.target.label + ) + branch.target, jump.target = jump.target, branch.target + jump.hot = True + + def _remove_redundant_jumps(self) -> None: + # Zero-length jumps can be introduced by _insert_continue_label and + # _invert_hot_branches: + for block in self._blocks(): + # Before: + # jmp FOO + # FOO: + # After: + # FOO: + if ( + block.target + and block.link + and block.target.resolve() is block.link.resolve() + ): + block.target = None + block.fallthrough = True + block.instructions.pop() + + def _fixup_external_labels(self) -> None: + if self._supports_external_relocations: + # Nothing to fix up + return + for block in self._blocks(): + if block.target and block.fallthrough: + branch = block.instructions[-1] + match = self._re_branch.match(branch) + assert match is not None + target = match["target"] + reloc = self._branches[match["instruction"]][1] + if reloc is not None and not target.startswith(self.label_prefix): + name = target[len(self.symbol_prefix) :] + block.instructions[-1] = ( + f"// target='{target}' prefix='{self.label_prefix}'" + ) + block.instructions.append( + f"{self.symbol_prefix}{reloc}_JIT_RELOCATION_{name}:" + ) + a, b = match.span("target") + branch = "".join([branch[:a], "0", branch[b:]]) + block.instructions.append(branch) + + def run(self) -> None: + """Run this optimizer.""" + self._insert_continue_label() + self._mark_hot_blocks() + self._invert_hot_branches() + self._remove_redundant_jumps() + self._fixup_external_labels() + self.path.write_text(self._body()) + + +class OptimizerAArch64(Optimizer): # pylint: disable = too-few-public-methods + """aarch64-pc-windows-msvc/aarch64-apple-darwin/aarch64-unknown-linux-gnu""" + + _branches = _AARCH64_BRANCHES + # Mach-O does not support the 19 bit branch locations needed for branch reordering + _supports_external_relocations = False + _re_branch = re.compile( + rf"\s*(?P<instruction>{'|'.join(_AARCH64_BRANCHES)})\s+(.+,\s+)*(?P<target>[\w.]+)" + ) + + # https://developer.arm.com/documentation/ddi0602/2025-03/Base-Instructions/B--Branch- + _re_jump = re.compile(r"\s*b\s+(?P<target>[\w.]+)") + # https://developer.arm.com/documentation/ddi0602/2025-09/Base-Instructions/RET--Return-from-subroutine- + _re_return = re.compile(r"\s*ret\b") + + +class OptimizerX86(Optimizer): # pylint: disable = too-few-public-methods + """i686-pc-windows-msvc/x86_64-apple-darwin/x86_64-unknown-linux-gnu""" + + _branches = _X86_BRANCHES + _re_branch = re.compile( + rf"\s*(?P<instruction>{'|'.join(_X86_BRANCHES)})\s+(?P<target>[\w.]+)" + ) + # https://www.felixcloutier.com/x86/jmp + _re_jump = re.compile(r"\s*jmp\s+(?P<target>[\w.]+)") + # https://www.felixcloutier.com/x86/ret + _re_return = re.compile(r"\s*ret\b") diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 228fc389584..4e86abe6049 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -10,6 +10,7 @@ "ARM64_RELOC_PAGEOFF12", "ARM64_RELOC_UNSIGNED", "IMAGE_REL_AMD64_REL32", + "IMAGE_REL_ARM64_BRANCH19", "IMAGE_REL_ARM64_BRANCH26", "IMAGE_REL_ARM64_PAGEBASE_REL21", "IMAGE_REL_ARM64_PAGEOFFSET_12A", @@ -20,6 +21,7 @@ "R_AARCH64_ADR_GOT_PAGE", "R_AARCH64_ADR_PREL_PG_HI21", "R_AARCH64_CALL26", + "R_AARCH64_CONDBR19", "R_AARCH64_JUMP26", "R_AARCH64_ADD_ABS_LO12_NC", "R_AARCH64_LD64_GOT_LO12_NC", @@ -87,6 +89,7 @@ class COFFSection(typing.TypedDict): Characteristics: dict[ typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] ] + Name: dict[typing.Literal["Value"], str] Number: int RawDataSize: int Relocations: list[dict[typing.Literal["Relocation"], COFFRelocation]] diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 03b0ba647b0..e717365b6b9 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -17,8 +17,6 @@ class HoleValue(enum.Enum): # The base address of the machine code for the current uop (exposed as _JIT_ENTRY): CODE = enum.auto() - # The base address of the machine code for the next uop (exposed as _JIT_CONTINUE): - CONTINUE = enum.auto() # The base address of the read-only data for this uop: DATA = enum.auto() # The address of the current executor (exposed as _JIT_EXECUTOR): @@ -60,9 +58,11 @@ class HoleValue(enum.Enum): "ARM64_RELOC_PAGE21": "patch_aarch64_21r", "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12", "ARM64_RELOC_UNSIGNED": "patch_64", + "CUSTOM_AARCH64_BRANCH19": "patch_aarch64_19r", # x86_64-pc-windows-msvc: "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx", # aarch64-pc-windows-msvc: + "IMAGE_REL_ARM64_BRANCH19": "patch_aarch64_19r", "IMAGE_REL_ARM64_BRANCH26": "patch_aarch64_26r", "IMAGE_REL_ARM64_PAGEBASE_REL21": "patch_aarch64_21rx", "IMAGE_REL_ARM64_PAGEOFFSET_12A": "patch_aarch64_12", @@ -76,6 +76,7 @@ class HoleValue(enum.Enum): "R_AARCH64_ADR_GOT_PAGE": "patch_aarch64_21rx", "R_AARCH64_ADR_PREL_PG_HI21": "patch_aarch64_21r", "R_AARCH64_CALL26": "patch_aarch64_26r", + "R_AARCH64_CONDBR19": "patch_aarch64_19r", "R_AARCH64_JUMP26": "patch_aarch64_26r", "R_AARCH64_LD64_GOT_LO12_NC": "patch_aarch64_12x", "R_AARCH64_MOVW_UABS_G0_NC": "patch_aarch64_16a", @@ -97,7 +98,6 @@ class HoleValue(enum.Enum): # Translate HoleValues to C expressions: _HOLE_EXPRS = { HoleValue.CODE: "(uintptr_t)code", - HoleValue.CONTINUE: "(uintptr_t)code + sizeof(code_body)", HoleValue.DATA: "(uintptr_t)data", HoleValue.EXECUTOR: "(uintptr_t)executor", # These should all have been turned into DATA values by process_relocations: @@ -140,11 +140,7 @@ class Hole: def __post_init__(self) -> None: self.func = _PATCH_FUNCS[self.kind] - def fold( - self, - other: typing.Self, - body: bytes | bytearray, - ) -> typing.Self | None: + def fold(self, other: typing.Self, body: bytearray) -> typing.Self | None: """Combine two holes into a single hole, if possible.""" instruction_a = int.from_bytes( body[self.offset : self.offset + 4], byteorder=sys.byteorder @@ -209,64 +205,6 @@ def pad(self, alignment: int) -> None: self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") self.body.extend([0] * padding) - def add_nops(self, nop: bytes, alignment: int) -> None: - """Add NOPs until there is alignment. Fail if it is not possible.""" - offset = len(self.body) - nop_size = len(nop) - - # Calculate the gap to the next multiple of alignment. - gap = -offset % alignment - if gap: - if gap % nop_size == 0: - count = gap // nop_size - self.body.extend(nop * count) - else: - raise ValueError( - f"Cannot add nops of size '{nop_size}' to a body with " - f"offset '{offset}' to align with '{alignment}'" - ) - - def remove_jump(self) -> None: - """Remove a zero-length continuation jump, if it exists.""" - hole = max(self.holes, key=lambda hole: hole.offset) - match hole: - case Hole( - offset=offset, - kind="IMAGE_REL_AMD64_REL32", - value=HoleValue.GOT, - symbol="_JIT_CONTINUE", - addend=-4, - ) as hole: - # jmp qword ptr [rip] - jump = b"\x48\xff\x25\x00\x00\x00\x00" - offset -= 3 - case Hole( - offset=offset, - kind="IMAGE_REL_I386_REL32" | "R_X86_64_PLT32" | "X86_64_RELOC_BRANCH", - value=HoleValue.CONTINUE, - symbol=None, - addend=addend, - ) as hole if ( - _signed(addend) == -4 - ): - # jmp 5 - jump = b"\xe9\x00\x00\x00\x00" - offset -= 1 - case Hole( - offset=offset, - kind="R_AARCH64_JUMP26", - value=HoleValue.CONTINUE, - symbol=None, - addend=0, - ) as hole: - # b #4 - jump = b"\x00\x00\x00\x14" - case _: - return - if self.body[offset:] == jump: - self.body = self.body[:offset] - self.holes.remove(hole) - @dataclasses.dataclass class StencilGroup: @@ -284,9 +222,18 @@ class StencilGroup: _got: dict[str, int] = dataclasses.field(default_factory=dict, init=False) _trampolines: set[int] = dataclasses.field(default_factory=set, init=False) - def process_relocations( - self, known_symbols: dict[str, int], *, alignment: int = 1, nop: bytes = b"" - ) -> None: + def convert_labels_to_relocations(self) -> None: + for name, hole_plus in self.symbols.items(): + if isinstance(name, str) and "_JIT_RELOCATION_" in name: + _, offset = hole_plus + reloc, target = name.split("_JIT_RELOCATION_") + value, symbol = symbol_to_value(target) + hole = Hole( + int(offset), typing.cast(_schema.HoleKind, reloc), value, symbol, 0 + ) + self.code.holes.append(hole) + + def process_relocations(self, known_symbols: dict[str, int]) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( @@ -306,8 +253,23 @@ def process_relocations( self._trampolines.add(ordinal) hole.addend = ordinal hole.symbol = None - self.code.remove_jump() - self.code.add_nops(nop=nop, alignment=alignment) + # x86_64 Darwin trampolines for external symbols + elif ( + hole.kind == "X86_64_RELOC_BRANCH" + and hole.value is HoleValue.ZERO + and hole.symbol not in self.symbols + ): + hole.func = "patch_x86_64_trampoline" + hole.need_state = True + assert hole.symbol is not None + if hole.symbol in known_symbols: + ordinal = known_symbols[hole.symbol] + else: + ordinal = len(known_symbols) + known_symbols[hole.symbol] = ordinal + self._trampolines.add(ordinal) + hole.addend = ordinal + hole.symbol = None self.data.pad(8) for stencil in [self.code, self.data]: for hole in stencil.holes: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 6ceb4404e74..4c188d74a68 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -10,8 +10,10 @@ import sys import tempfile import typing +import shlex import _llvm +import _optimizers import _schema import _stencils import _writer @@ -40,13 +42,17 @@ class _Target(typing.Generic[_S, _R]): triple: str condition: str _: dataclasses.KW_ONLY - alignment: int = 1 args: typing.Sequence[str] = () - prefix: str = "" + optimizer: type[_optimizers.Optimizer] = _optimizers.Optimizer + label_prefix: typing.ClassVar[str] + symbol_prefix: typing.ClassVar[str] stable: bool = False debug: bool = False verbose: bool = False + cflags: str = "" + llvm_version: str = _llvm._LLVM_VERSION known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) + pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() def _get_nop(self) -> bytes: if re.fullmatch(r"aarch64-.*", self.triple): @@ -57,14 +63,18 @@ def _get_nop(self) -> bytes: raise ValueError(f"NOP not defined for {self.triple}") return nop - def _compute_digest(self, out: pathlib.Path) -> str: + def _compute_digest(self) -> str: hasher = hashlib.sha256() hasher.update(self.triple.encode()) hasher.update(self.debug.to_bytes()) + hasher.update(self.cflags.encode()) # These dependencies are also reflected in _JITSources in regen.targets: hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) - hasher.update((out / "pyconfig.h").read_bytes()) + hasher.update((self.pyconfig_dir / "pyconfig.h").read_bytes()) for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)): + # Exclude cache files from digest computation to ensure reproducible builds. + if dirpath.endswith("__pycache__"): + continue for filename in filenames: hasher.update(pathlib.Path(dirpath, filename).read_bytes()) return hasher.hexdigest() @@ -72,7 +82,9 @@ def _compute_digest(self, out: pathlib.Path) -> str: async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group = _stencils.StencilGroup() args = ["--disassemble", "--reloc", f"{path}"] - output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose) + output = await _llvm.maybe_run( + "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version + ) if output is not None: # Make sure that full paths don't leak out (for reproducibility): long, short = str(path), str(path.name) @@ -90,7 +102,9 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: "--sections", f"{path}", ] - output = await _llvm.run("llvm-readobj", args, echo=self.verbose) + output = await _llvm.run( + "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version + ) # --elf-output-style=JSON is only *slightly* broken on Mach-O... output = output.replace("PrivateExtern\n", "\n") output = output.replace("Extern\n", "\n") @@ -110,29 +124,38 @@ def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: raise NotImplementedError(type(self)) def _handle_relocation( - self, base: int, relocation: _R, raw: bytes | bytearray + self, base: int, relocation: _R, raw: bytearray ) -> _stencils.Hole: raise NotImplementedError(type(self)) async def _compile( self, opname: str, c: pathlib.Path, tempdir: pathlib.Path ) -> _stencils.StencilGroup: + s = tempdir / f"{opname}.s" o = tempdir / f"{opname}.o" - args = [ + args_s = [ f"--target={self.triple}", "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", f"-D_JIT_OPCODE={opname}", "-D_PyJIT_ACTIVE", "-D_Py_JIT", - "-I.", + f"-I{self.pyconfig_dir}", f"-I{CPYTHON / 'Include'}", f"-I{CPYTHON / 'Include' / 'internal'}", f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", f"-I{CPYTHON / 'Python'}", f"-I{CPYTHON / 'Tools' / 'jit'}", - "-O3", - "-c", + # -O2 and -O3 include some optimizations that make sense for + # standalone functions, but not for snippets of code that are going + # to be laid out end-to-end (like ours)... common examples include + # passes like tail-duplication, or aligning jump targets with nops. + # -Os is equivalent to -O2 with many of these problematic passes + # disabled. Based on manual review, for *our* purposes it usually + # generates better code than -O2 (and -O2 usually generates better + # code than -O3). As a nice benefit, it uses less memory too: + "-Os", + "-S", # Shorten full absolute file paths in the generated code (like the # __FILE__ macro and assert failure messages) for reproducibility: f"-ffile-prefix-map={CPYTHON}=.", @@ -143,19 +166,26 @@ async def _compile( "-fno-asynchronous-unwind-tables", # Don't call built-in functions that we can't find or patch: "-fno-builtin", - # Emit relaxable 64-bit calls/jumps, so we don't have to worry about - # about emitting in-range trampolines for out-of-range targets. - # We can probably remove this and emit trampolines in the future: - "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-std=c11", "-o", - f"{o}", + f"{s}", f"{c}", *self.args, + # Allow user-provided CFLAGS to override any defaults + *shlex.split(self.cflags), ] - await _llvm.run("clang", args, echo=self.verbose) + await _llvm.run( + "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version + ) + self.optimizer( + s, label_prefix=self.label_prefix, symbol_prefix=self.symbol_prefix + ).run() + args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] + await _llvm.run( + "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version + ) return await self._parse(o) async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: @@ -169,8 +199,8 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: with tempfile.TemporaryDirectory() as tempdir: work = pathlib.Path(tempdir).resolve() async with asyncio.TaskGroup() as group: - coro = self._compile("shim", TOOLS_JIT / "shim.c", work) - tasks.append(group.create_task(coro, name="shim")) + coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) + tasks.append(group.create_task(coro, name="trampoline")) template = TOOLS_JIT_TEMPLATE_C.read_text() for case, opname in cases_and_opnames: # Write out a copy of the template with *only* this case @@ -184,29 +214,27 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: tasks.append(group.create_task(coro, name=opname)) stencil_groups = {task.get_name(): task.result() for task in tasks} for stencil_group in stencil_groups.values(): - stencil_group.process_relocations( - known_symbols=self.known_symbols, - alignment=self.alignment, - nop=self._get_nop(), - ) + stencil_group.convert_labels_to_relocations() + stencil_group.process_relocations(self.known_symbols) return stencil_groups def build( self, - out: pathlib.Path, *, comment: str = "", force: bool = False, - stencils_h: str = "jit_stencils.h", + jit_stencils: pathlib.Path, ) -> None: """Build jit_stencils.h in the given directory.""" + jit_stencils.parent.mkdir(parents=True, exist_ok=True) if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" request = "Please report any issues you encounter.".center(len(warning)) + if self.llvm_version != _llvm._LLVM_VERSION: + request = f"Warning! Building with an LLVM version other than {_llvm._LLVM_VERSION} is not supported." outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) - digest = f"// {self._compute_digest(out)}\n" - jit_stencils = out / stencils_h + digest = f"// {self._compute_digest()}\n" if ( not force and jit_stencils.exists() @@ -214,7 +242,7 @@ def build( ): return stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) - jit_stencils_new = out / "jit_stencils.h.new" + jit_stencils_new = jit_stencils.parent / "jit_stencils.h.new" try: with jit_stencils_new.open("w") as file: file.write(digest) @@ -239,6 +267,10 @@ class _COFF( def _handle_section( self, section: _schema.COFFSection, group: _stencils.StencilGroup ) -> None: + name = section["Name"]["Value"] + if name == ".debug$S": + # skip debug sections + return flags = {flag["Name"] for flag in section["Characteristics"]["Flags"]} if "SectionData" in section: section_data_bytes = section["SectionData"]["Bytes"] @@ -260,7 +292,7 @@ def _handle_section( symbol = wrapped_symbol["Symbol"] offset = base + symbol["Value"] name = symbol["Name"] - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) if name not in group.symbols: group.symbols[name] = value, offset for wrapped_relocation in section["Relocations"]: @@ -271,16 +303,13 @@ def _handle_section( def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]: if name.startswith("__imp_"): name = name.removeprefix("__imp_") - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) return _stencils.HoleValue.GOT, name - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) return _stencils.symbol_to_value(name) def _handle_relocation( - self, - base: int, - relocation: _schema.COFFRelocation, - raw: bytes | bytearray, + self, base: int, relocation: _schema.COFFRelocation, raw: bytearray ) -> _stencils.Hole: match relocation: case { @@ -307,7 +336,8 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Name": "IMAGE_REL_ARM64_BRANCH26" + "Name": "IMAGE_REL_ARM64_BRANCH19" + | "IMAGE_REL_ARM64_BRANCH26" | "IMAGE_REL_ARM64_PAGEBASE_REL21" | "IMAGE_REL_ARM64_PAGEOFFSET_12A" | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind @@ -321,9 +351,24 @@ def _handle_relocation( return _stencils.Hole(offset, kind, value, symbol, addend) +class _COFF32(_COFF): + # These mangle like Mach-O and other "older" formats: + label_prefix = "L" + symbol_prefix = "_" + + +class _COFF64(_COFF): + # These mangle like ELF and other "newer" formats: + label_prefix = ".L" + symbol_prefix = "" + + class _ELF( _Target[_schema.ELFSection, _schema.ELFRelocation] ): # pylint: disable = too-few-public-methods + label_prefix = ".L" + symbol_prefix = "" + def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: @@ -360,7 +405,7 @@ def _handle_section( symbol = wrapped_symbol["Symbol"] offset = len(stencil.body) + symbol["Value"] name = symbol["Name"]["Name"] - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) group.symbols[name] = value, offset stencil.body.extend(section["SectionData"]["Bytes"]) assert not section["Relocations"] @@ -375,10 +420,7 @@ def _handle_section( }, section_type def _handle_relocation( - self, - base: int, - relocation: _schema.ELFRelocation, - raw: bytes | bytearray, + self, base: int, relocation: _schema.ELFRelocation, raw: bytearray ) -> _stencils.Hole: symbol: str | None match relocation: @@ -395,7 +437,7 @@ def _handle_relocation( }, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.HoleValue.GOT, s case { "Addend": addend, @@ -404,7 +446,7 @@ def _handle_relocation( "Type": {"Name": kind}, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.symbol_to_value(s) case _: raise NotImplementedError(relocation) @@ -414,6 +456,9 @@ def _handle_relocation( class _MachO( _Target[_schema.MachOSection, _schema.MachORelocation] ): # pylint: disable = too-few-public-methods + label_prefix = "L" + symbol_prefix = "_" + def _handle_section( self, section: _schema.MachOSection, group: _stencils.StencilGroup ) -> None: @@ -421,10 +466,10 @@ def _handle_section( assert "SectionData" in section flags = {flag["Name"] for flag in section["Attributes"]["Flags"]} name = section["Name"]["Value"] - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) if "Debug" in flags: return - if "SomeInstructions" in flags: + if "PureInstructions" in flags: value = _stencils.HoleValue.CODE stencil = group.code start_address = 0 @@ -445,7 +490,7 @@ def _handle_section( symbol = wrapped_symbol["Symbol"] offset = symbol["Value"] - start_address name = symbol["Name"]["Name"] - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) group.symbols[name] = value, offset assert "Relocations" in section for wrapped_relocation in section["Relocations"]: @@ -454,10 +499,7 @@ def _handle_section( stencil.holes.append(hole) def _handle_relocation( - self, - base: int, - relocation: _schema.MachORelocation, - raw: bytes | bytearray, + self, base: int, relocation: _schema.MachORelocation, raw: bytearray ) -> _stencils.Hole: symbol: str | None match relocation: @@ -470,7 +512,7 @@ def _handle_relocation( }, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.HoleValue.GOT, s addend = 0 case { @@ -479,7 +521,7 @@ def _handle_relocation( "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.HoleValue.GOT, s addend = ( int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 @@ -494,7 +536,7 @@ def _handle_relocation( "Type": {"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind}, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.symbol_to_value(s) addend = ( int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 @@ -509,7 +551,7 @@ def _handle_relocation( "Type": {"Name": kind}, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.symbol_to_value(s) addend = 0 case _: @@ -517,44 +559,52 @@ def _handle_relocation( return _stencils.Hole(offset, kind, value, symbol, addend) -def get_target(host: str) -> _COFF | _ELF | _MachO: +def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" - target: _COFF | _ELF | _MachO + optimizer: type[_optimizers.Optimizer] + target: _COFF32 | _COFF64 | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): + host = "aarch64-apple-darwin" condition = "defined(__aarch64__) && defined(__APPLE__)" - target = _MachO(host, condition, alignment=8, prefix="_") + optimizer = _optimizers.OptimizerAArch64 + target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"aarch64-pc-windows-msvc", host): - args = ["-fms-runtime-lib=dll", "-fplt"] + host = "aarch64-pc-windows-msvc" condition = "defined(_M_ARM64)" - target = _COFF(host, condition, alignment=8, args=args) - elif re.fullmatch(r"aarch64-.*-linux-gnu", host): - args = [ - "-fpic", - # On aarch64 Linux, intrinsics were being emitted and this flag - # was required to disable them. - "-mno-outline-atomics", - ] - condition = "defined(__aarch64__) && defined(__linux__)" - target = _ELF(host, condition, alignment=8, args=args) - elif re.fullmatch(r"i686-pc-windows-msvc", host): - args = [ - "-DPy_NO_ENABLE_SHARED", - # __attribute__((preserve_none)) is not supported - "-Wno-ignored-attributes", - ] - condition = "defined(_M_IX86)" - target = _COFF(host, condition, args=args, prefix="_") - elif re.fullmatch(r"x86_64-apple-darwin.*", host): - condition = "defined(__x86_64__) && defined(__APPLE__)" - target = _MachO(host, condition, prefix="_") - elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] + optimizer = _optimizers.OptimizerAArch64 + target = _COFF64(host, condition, args=args, optimizer=optimizer) + elif re.fullmatch(r"aarch64-.*-linux-gnu", host): + host = "aarch64-unknown-linux-gnu" + condition = "defined(__aarch64__) && defined(__linux__)" + # -mno-outline-atomics: Keep intrinsics from being emitted. + args = ["-fpic", "-mno-outline-atomics", "-fno-plt"] + optimizer = _optimizers.OptimizerAArch64 + target = _ELF(host, condition, args=args, optimizer=optimizer) + elif re.fullmatch(r"i686-pc-windows-msvc", host): + host = "i686-pc-windows-msvc" + condition = "defined(_M_IX86)" + # -Wno-ignored-attributes: __attribute__((preserve_none)) is not supported here. + args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes"] + optimizer = _optimizers.OptimizerX86 + target = _COFF32(host, condition, args=args, optimizer=optimizer) + elif re.fullmatch(r"x86_64-apple-darwin.*", host): + host = "x86_64-apple-darwin" + condition = "defined(__x86_64__) && defined(__APPLE__)" + optimizer = _optimizers.OptimizerX86 + target = _MachO(host, condition, optimizer=optimizer) + elif re.fullmatch(r"x86_64-pc-windows-msvc", host): + host = "x86_64-pc-windows-msvc" condition = "defined(_M_X64)" - target = _COFF(host, condition, args=args) + args = ["-fms-runtime-lib=dll"] + optimizer = _optimizers.OptimizerX86 + target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): - args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0"] + host = "x86_64-unknown-linux-gnu" condition = "defined(__x86_64__) && defined(__linux__)" - target = _ELF(host, condition, args=args) + args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0", "-fno-plt"] + optimizer = _optimizers.OptimizerX86 + target = _ELF(host, condition, args=args, optimizer=optimizer) else: raise ValueError(host) return target diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 090b52660f0..4f373011ebf 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -22,11 +22,11 @@ def _dump_footer( yield " symbol_mask trampoline_mask;" yield "} StencilGroup;" yield "" - yield f"static const StencilGroup shim = {groups['shim'].as_c('shim')};" + yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};" yield "" yield "static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {" for opname, group in sorted(groups.items()): - if opname == "shim": + if opname == "trampoline": continue yield f" [{opname}] = {group.as_c(opname)}," yield "};" diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 49b08f477db..127d93b317f 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -8,7 +8,6 @@ import _targets if __name__ == "__main__": - out = pathlib.Path.cwd().resolve() comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -23,21 +22,42 @@ parser.add_argument( "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" ) + parser.add_argument( + "-o", + "--output-dir", + help="where to output generated files", + required=True, + type=lambda p: pathlib.Path(p).resolve(), + ) + parser.add_argument( + "-p", + "--pyconfig-dir", + help="where to find pyconfig.h", + required=True, + type=lambda p: pathlib.Path(p).resolve(), + ) parser.add_argument( "-v", "--verbose", action="store_true", help="echo commands as they are run" ) + parser.add_argument( + "--cflags", help="additional flags to pass to the compiler", default="" + ) + parser.add_argument("--llvm-version", help="LLVM version to use") args = parser.parse_args() for target in args.target: target.debug = args.debug target.force = args.force target.verbose = args.verbose + target.cflags = args.cflags + target.pyconfig_dir = args.pyconfig_dir + if args.llvm_version: + target.llvm_version = args.llvm_version target.build( - out, comment=comment, - stencils_h=f"jit_stencils-{target.triple}.h", force=args.force, + jit_stencils=args.output_dir / f"jit_stencils-{target.triple}.h", ) - jit_stencils_h = out / "jit_stencils.h" + jit_stencils_h = args.output_dir / "jit_stencils.h" lines = [f"// {comment}\n"] guard = "#if" for target in args.target: diff --git a/Tools/jit/jit.h b/Tools/jit/jit.h index f767ef68127..10829654eab 100644 --- a/Tools/jit/jit.h +++ b/Tools/jit/jit.h @@ -6,3 +6,7 @@ typedef jit_func __attribute__((preserve_none)) jit_func_preserve_none; #define PATCH_VALUE(TYPE, NAME, ALIAS) \ PyAPI_DATA(void) ALIAS; \ TYPE NAME = (TYPE)(uintptr_t)&ALIAS; + +#define DECLARE_TARGET(NAME) \ + _Py_CODEUNIT *__attribute__((preserve_none, visibility("hidden"))) \ + NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); diff --git a/Tools/jit/jit_infra.md b/Tools/jit/jit_infra.md new file mode 100644 index 00000000000..1a954755611 --- /dev/null +++ b/Tools/jit/jit_infra.md @@ -0,0 +1,28 @@ +# JIT Build Infrastructure + +This document includes details about the intricacies of the JIT build infrastructure. + +## Updating LLVM + +When we update LLVM, we need to also update the LLVM release artifact for Windows builds. This is because Windows builds automatically pull prebuilt LLVM binaries in our pipelines (e.g. notice that `.github/workflows/jit.yml` does not explicitly download LLVM or build it from source). + +To update the LLVM release artifact for Windows builds, follow these steps: +1. Go to the [LLVM releases page](https://github.com/llvm/llvm-project/releases). +1. Download x86_64 Windows artifact for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz`). +1. Extract and repackage the tarball with the correct directory structure. For example: + ```bash + tar -xf clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz + mv clang+llvm-21.1.4-x86_64-pc-windows-msvc llvm-21.1.4.0 + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0.tar.xz + ``` + The tarball must contain a top-level directory named `llvm-{version}.0/`. +1. Go to [cpython-bin-deps](https://github.com/python/cpython-bin-deps). +1. Create a new release with the updated LLVM artifact. + - Create a new tag to match the LLVM version (e.g. `llvm-21.1.4.0`). + - Specify the release title (e.g. `LLVM 21.1.4 for x86_64 Windows`). + - Upload the asset (you can leave all other fields the same). + +### Other notes +- You must make sure that the name of the artifact matches exactly what is expected in `Tools/jit/_llvm.py` and `PCbuild/get_externals.py`. +- We don't need multiple release artifacts for each architecture because LLVM can cross-compile for different architectures on Windows; x86_64 is sufficient. +- You must have permissions to create releases in the `cpython-bin-deps` repository. If you don't have permissions, you should contact one of the organization admins. \ No newline at end of file diff --git a/Tools/jit/shim.c b/Tools/jit/shim.c deleted file mode 100644 index ebd4e9bc858..00000000000 --- a/Tools/jit/shim.c +++ /dev/null @@ -1,15 +0,0 @@ -#include "Python.h" - -#include "pycore_ceval.h" -#include "pycore_frame.h" -#include "pycore_jit.h" - -#include "jit.h" - -_Py_CODEUNIT * -_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) -{ - // Note that this is *not* a tail call: - PATCH_VALUE(jit_func_preserve_none, call, _JIT_CONTINUE); - return call(frame, stack_pointer, tstate); -} diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 5ee26f93f1e..0167f1b0ae5 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -46,23 +46,19 @@ #undef CURRENT_TARGET #define CURRENT_TARGET() (_target) -#undef GOTO_TIER_TWO -#define GOTO_TIER_TWO(EXECUTOR) \ +#undef TIER2_TO_TIER2 +#define TIER2_TO_TIER2(EXECUTOR) \ do { \ OPT_STAT_INC(traces_executed); \ _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ - jit_func_preserve_none jitted = _executor->jit_side_entry; \ + jit_func_preserve_none jitted = _executor->jit_code; \ __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) -#undef GOTO_TIER_ONE -#define GOTO_TIER_ONE(TARGET) \ -do { \ - tstate->current_executor = NULL; \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ -} while (0) +#undef GOTO_TIER_ONE_SETUP +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + _PyFrame_SetStackPointer(frame, stack_pointer); #undef LOAD_IP #define LOAD_IP(UNUSED) \ @@ -70,14 +66,16 @@ do { \ } while (0) #undef LLTRACE_RESUME_FRAME -#define LLTRACE_RESUME_FRAME() \ - do { \ - } while (0) +#ifdef Py_DEBUG +#define LLTRACE_RESUME_FRAME() (frame->lltrace = 0) +#else +#define LLTRACE_RESUME_FRAME() do {} while (0) +#endif -#define PATCH_JUMP(ALIAS) \ -do { \ - PATCH_VALUE(jit_func_preserve_none, jump, ALIAS); \ - __attribute__((musttail)) return jump(frame, stack_pointer, tstate); \ +#define PATCH_JUMP(ALIAS) \ +do { \ + DECLARE_TARGET(ALIAS); \ + __attribute__((musttail)) return ALIAS(frame, stack_pointer, tstate); \ } while (0) #undef JUMP_TO_JUMP_TARGET @@ -88,6 +86,12 @@ do { \ #define TIER_TWO 2 +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + __attribute__((preserve_none)) _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c new file mode 100644 index 00000000000..fdc63b7fc40 --- /dev/null +++ b/Tools/jit/trampoline.c @@ -0,0 +1,16 @@ +#include "Python.h" + +#include "pycore_ceval.h" +#include "pycore_frame.h" +#include "pycore_jit.h" + +#include "jit.h" + +_Py_CODEUNIT * +_JIT_ENTRY( + _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate +) { + // Note that this is *not* a tail call + jit_func_preserve_none jitted = (jit_func_preserve_none)exec->jit_code; + return jitted(frame, stack_pointer, tstate); +} diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 0bd3644b58b..335c1d922d9 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -92,7 +92,7 @@ Select Customize to review current options.</String> <String Id="PrecompileLabel">&amp;Precompile standard library</String> <String Id="Include_symbolsLabel">Download debugging &amp;symbols</String> <String Id="Include_debugLabel">Download debu&amp;g binaries (requires VS 2017 or later)</String> - <String Id="Include_freethreadedLabel">Download &amp;free-threaded binaries (experimental)</String> + <String Id="Include_freethreadedLabel">Download &amp;free-threaded binaries</String> <String Id="ProgressHeader">[ActionLikeInstallation] Progress</String> <String Id="ProgressLabel">[ActionLikeInstalling]:</String> diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs index abfeb887848..3fcb00553f5 100644 --- a/Tools/msi/bundle/bundle.wxs +++ b/Tools/msi/bundle/bundle.wxs @@ -106,11 +106,11 @@ <Variable Name="SimpleInstallDescription" Value="" bal:Overridable="yes" /> <Chain ParallelCache="yes"> + <PackageGroupRef Id="core" /> + <PackageGroupRef Id="exe" /> <?if $(var.Platform)!="ARM64" ?> <PackageGroupRef Id="crt" /> <?endif ?> - <PackageGroupRef Id="core" /> - <PackageGroupRef Id="exe" /> <PackageGroupRef Id="dev" /> <PackageGroupRef Id="lib" /> <?if $(var.IncludeFreethreaded)~="true" ?> diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 4357dc86d9d..21f9c848cc6 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -3,7 +3,7 @@ <Fragment> <ComponentGroup Id="dev_pyconfig"> <Component Id="include_pyconfig.h" Directory="include" Guid="*"> - <File Id="include_pyconfig.h" Name="pyconfig.h" Source="pyconfig.h" KeyPath="yes" /> + <File Id="include_pyconfig.h" Name="pyconfig.h" Source="!(bindpath.src)PC\pyconfig.h" KeyPath="yes" /> </Component> </ComponentGroup> </Fragment> diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index 86d9a8b83f6..0707e77b5e9 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -103,7 +103,7 @@ </ComponentGroup> </Fragment> - <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_uuid;_wmi;_zoneinfo;_zstd;_testcapi;_ctypes_test;_testbuffer;_testimportmultiple;_testmultiphase;_testsinglephase;_testconsole;_testinternalcapi;_testclinic;_testclinic_limited;_tkinter ?> + <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_remote_debugging;_uuid;_wmi;_zoneinfo;_zstd;_testcapi;_ctypes_test;_testbuffer;_testimportmultiple;_testmultiphase;_testsinglephase;_testconsole;_testinternalcapi;_testclinic;_testclinic_limited;_tkinter ?> <Fragment> <DirectoryRef Id="Lib_venv_scripts_nt__freethreaded" /> diff --git a/Tools/msi/lib/lib.wixproj b/Tools/msi/lib/lib.wixproj index 02078e503d7..3ea46dd40ea 100644 --- a/Tools/msi/lib/lib.wixproj +++ b/Tools/msi/lib/lib.wixproj @@ -15,12 +15,11 @@ <EmbeddedResource Include="*.wxl" /> </ItemGroup> <ItemGroup> - <ExcludeFolders Include="Lib\test;Lib\tests;Lib\tkinter;Lib\idlelib;Lib\turtledemo" /> + <ExcludeFolders Include="Lib\site-packages;Lib\test;Lib\tests;Lib\tkinter;Lib\idlelib;Lib\turtledemo" /> <InstallFiles Include="$(PySourcePath)Lib\**\*" Exclude="$(PySourcePath)Lib\**\*.pyc; $(PySourcePath)Lib\**\*.pyo; $(PySourcePath)Lib\turtle.py; - $(PySourcePath)Lib\site-packages\README; @(ExcludeFolders->'$(PySourcePath)%(Identity)\*'); @(ExcludeFolders->'$(PySourcePath)%(Identity)\**\*')"> <SourceBase>$(PySourcePath)Lib</SourceBase> diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index 8439518bcbd..4d44299f783 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"> - <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_uuid;_wmi;_zoneinfo;_zstd ?> + <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_remote_debugging;_uuid;_wmi;_zoneinfo;_zstd ?> <Fragment> <DirectoryRef Id="Lib_venv_scripts_nt" /> diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index 0dcf6ef844a..1a5797f7422 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -53,19 +53,43 @@ def get_git_branch(): def get_git_upstream_remote(): - """Get the remote name to use for upstream branches - - Uses "upstream" if it exists, "origin" otherwise """ - cmd = "git remote get-url upstream".split() - try: - subprocess.check_output(cmd, - stderr=subprocess.DEVNULL, - cwd=SRCDIR, - encoding='UTF-8') - except subprocess.CalledProcessError: - return "origin" - return "upstream" + Get the remote name to use for upstream branches + + Check for presence of "https://github.com/python/cpython" remote URL. + If only one is found, return that remote name. If multiple are found, + check for and return "upstream", "origin", or "python", in that + order. Raise an error if no valid matches are found. + """ + cmd = "git remote -v".split() + output = subprocess.check_output( + cmd, + stderr=subprocess.DEVNULL, + cwd=SRCDIR, + encoding="UTF-8" + ) + # Filter to desired remotes, accounting for potential uppercasing + filtered_remotes = { + remote.split("\t")[0].lower() for remote in output.split('\n') + if "python/cpython" in remote.lower() and remote.endswith("(fetch)") + } + if len(filtered_remotes) == 1: + [remote] = filtered_remotes + return remote + for remote_name in ["upstream", "origin", "python"]: + if remote_name in filtered_remotes: + return remote_name + remotes_found = "\n".join( + {remote for remote in output.split('\n') if remote.endswith("(fetch)")} + ) + raise ValueError( + f"Patchcheck was unable to find an unambiguous upstream remote, " + f"with URL matching 'https://github.com/python/cpython'. " + f"For help creating an upstream remote, see Dev Guide: " + f"https://devguide.python.org/getting-started/" + f"git-boot-camp/#cloning-a-forked-cpython-repository " + f"\nRemotes found: \n{remotes_found}" + ) def get_git_remote_default_branch(remote_name): @@ -152,12 +176,6 @@ def docs_modified(file_paths): return bool(file_paths) -@status("Misc/ACKS updated", modal=True) -def credit_given(file_paths): - """Check if Misc/ACKS has been changed.""" - return os.path.join('Misc', 'ACKS') in file_paths - - @status("Misc/NEWS.d updated with `blurb`", modal=True) def reported_news(file_paths): """Check if Misc/NEWS.d has been changed.""" @@ -191,8 +209,6 @@ def main(): misc_files = {p for p in file_paths if p.startswith('Misc')} # Docs updated. docs_modified(has_doc_files) - # Misc/ACKS changed. - credit_given(misc_files) # Misc/NEWS changed. reported_news(misc_files) # Regenerated configure, if necessary. diff --git a/Tools/peg_generator/.ruff.toml b/Tools/peg_generator/.ruff.toml new file mode 100644 index 00000000000..bcf57248713 --- /dev/null +++ b/Tools/peg_generator/.ruff.toml @@ -0,0 +1,22 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +extend-exclude = [ + # Generated files: + "Tools/peg_generator/pegen/grammar_parser.py", +] + +[lint] +select = [ + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "RUF100", # Ban unused `# noqa` comments + "PGH004", # Ban blanket `# noqa` comments (only ignore specific error codes) +] +unfixable = [ + # The autofixes sometimes do the wrong things for these; + # it's better to have to manually look at the code and see how it needs fixing + "F841", # Detects unused variables + "F601", # Detects dictionaries that have duplicate keys + "F602", # Also detects dictionaries that have duplicate keys +] diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index 1587d53d594..2fec5b05129 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -8,7 +8,7 @@ _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *ar PyObject *result = NULL; if (mode == 2) { - result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena); + result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena, NULL); } else if (mode == 1) { result = PyAST_mod2obj(module); } else { @@ -93,7 +93,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) PyCompilerFlags flags = _PyCompilerFlags_INIT; mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob, - &flags, arena); + &flags, arena, NULL); if (res == NULL) { goto error; } diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 0b0b4b291c2..7a82ad861b4 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -10,7 +10,6 @@ import time import token import traceback -from typing import Tuple from pegen.grammar import Grammar from pegen.parser import Parser @@ -21,7 +20,7 @@ def generate_c_code( args: argparse.Namespace, -) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: +) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]: from pegen.build import build_c_parser_and_generator verbose = args.verbose @@ -50,7 +49,7 @@ def generate_c_code( def generate_python_code( args: argparse.Namespace, -) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: +) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]: from pegen.build import build_python_parser_and_generator verbose = args.verbose @@ -188,7 +187,7 @@ def main() -> None: if __name__ == "__main__": - if sys.version_info < (3, 8): - print("ERROR: using pegen requires at least Python 3.8!", file=sys.stderr) + if sys.version_info < (3, 10): # noqa: UP036 + print("ERROR: using pegen requires at least Python 3.10!", file=sys.stderr) sys.exit(1) main() diff --git a/Tools/peg_generator/pegen/ast_dump.py b/Tools/peg_generator/pegen/ast_dump.py index 07f8799c114..60dc1b04672 100644 --- a/Tools/peg_generator/pegen/ast_dump.py +++ b/Tools/peg_generator/pegen/ast_dump.py @@ -6,7 +6,7 @@ TODO: Remove the above-described hack. """ -from typing import Any, Optional, Tuple +from typing import Any def ast_dump( @@ -14,9 +14,9 @@ def ast_dump( annotate_fields: bool = True, include_attributes: bool = False, *, - indent: Optional[str] = None, + indent: str | None = None, ) -> str: - def _format(node: Any, level: int = 0) -> Tuple[str, bool]: + def _format(node: Any, level: int = 0) -> tuple[str, bool]: if indent is not None: level += 1 prefix = "\n" + indent * level @@ -41,7 +41,7 @@ def _format(node: Any, level: int = 0) -> Tuple[str, bool]: value, simple = _format(value, level) allsimple = allsimple and simple if keywords: - args.append("%s=%s" % (name, value)) + args.append(f"{name}={value}") else: args.append(value) if include_attributes and node._attributes: @@ -54,16 +54,16 @@ def _format(node: Any, level: int = 0) -> Tuple[str, bool]: continue value, simple = _format(value, level) allsimple = allsimple and simple - args.append("%s=%s" % (name, value)) + args.append(f"{name}={value}") if allsimple and len(args) <= 3: - return "%s(%s)" % (node.__class__.__name__, ", ".join(args)), not args - return "%s(%s%s)" % (node.__class__.__name__, prefix, sep.join(args)), False + return "{}({})".format(node.__class__.__name__, ", ".join(args)), not args + return f"{node.__class__.__name__}({prefix}{sep.join(args)})", False elif isinstance(node, list): if not node: return "[]", True - return "[%s%s]" % (prefix, sep.join(_format(x, level)[0] for x in node)), False + return f"[{prefix}{sep.join(_format(x, level)[0] for x in node)}]", False return repr(node), True if all(cls.__name__ != "AST" for cls in node.__class__.__mro__): - raise TypeError("expected AST, got %r" % node.__class__.__name__) + raise TypeError(f"expected AST, got {node.__class__.__name__!r}") return _format(node)[0] diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 41338c29bdd..44a58581686 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -6,7 +6,7 @@ import sysconfig import tempfile import tokenize -from typing import IO, Any, Dict, List, Optional, Set, Tuple +from typing import IO, Any from pegen.c_generator import CParserGenerator from pegen.grammar import Grammar @@ -18,11 +18,11 @@ MOD_DIR = pathlib.Path(__file__).resolve().parent -TokenDefinitions = Tuple[Dict[int, str], Dict[str, int], Set[str]] +TokenDefinitions = tuple[dict[int, str], dict[str, int], set[str]] Incomplete = Any # TODO: install `types-setuptools` and remove this alias -def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[str]: +def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> list[str]: flags = sysconfig.get_config_var(compiler_flags) py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist) if flags is None or py_flags_nodist is None: @@ -71,11 +71,11 @@ def fixup_build_ext(cmd: Incomplete) -> None: def compile_c_extension( generated_source_path: str, - build_dir: Optional[str] = None, + build_dir: str | None = None, verbose: bool = False, keep_asserts: bool = True, disable_optimization: bool = False, - library_dir: Optional[str] = None, + library_dir: str | None = None, ) -> pathlib.Path: """Compile the generated source for a parser generator into an extension module. @@ -93,11 +93,10 @@ def compile_c_extension( """ import setuptools.command.build_ext import setuptools.logging - - from setuptools import Extension, Distribution - from setuptools.modified import newer_group + from setuptools import Distribution, Extension from setuptools._distutils.ccompiler import new_compiler from setuptools._distutils.sysconfig import customize_compiler + from setuptools.modified import newer_group if verbose: setuptools.logging.set_threshold(logging.DEBUG) @@ -108,6 +107,8 @@ def compile_c_extension( extra_compile_args.append("-DPy_BUILD_CORE_MODULE") # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c extra_compile_args.append("-D_Py_TEST_PEGEN") + if sys.platform == "win32" and sysconfig.get_config_var("Py_GIL_DISABLED"): + extra_compile_args.append("-DPy_GIL_DISABLED") extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST") if keep_asserts: extra_compile_args.append("-UNDEBUG") @@ -239,7 +240,7 @@ def compile_c_extension( def build_parser( grammar_file: str, verbose_tokenizer: bool = False, verbose_parser: bool = False -) -> Tuple[Grammar, Parser, Tokenizer]: +) -> tuple[Grammar, Parser, Tokenizer]: with open(grammar_file) as file: tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer) parser = GrammarParser(tokenizer, verbose=verbose_parser) @@ -290,7 +291,7 @@ def build_c_generator( keep_asserts_in_extension: bool = True, skip_actions: bool = False, ) -> ParserGenerator: - with open(tokens_file, "r") as tok_file: + with open(tokens_file) as tok_file: all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file) with open(output_file, "w") as file: gen: ParserGenerator = CParserGenerator( @@ -331,7 +332,7 @@ def build_c_parser_and_generator( verbose_c_extension: bool = False, keep_asserts_in_extension: bool = True, skip_actions: bool = False, -) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: +) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]: """Generate rules, C parser, tokenizer, parser generator for a given grammar Args: @@ -371,7 +372,7 @@ def build_python_parser_and_generator( verbose_tokenizer: bool = False, verbose_parser: bool = False, skip_actions: bool = False, -) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: +) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]: """Generate rules, python parser, tokenizer, parser generator for a given grammar Args: diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 2be85a163b4..a4e111972bd 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -1,9 +1,10 @@ import ast import os.path import re +from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum -from typing import IO, Any, Callable, Dict, List, Optional, Set, Text, Tuple +from typing import IO, Any from pegen import grammar from pegen.grammar import ( @@ -30,6 +31,7 @@ EXTENSION_PREFIX = """\ #include "pegen.h" +#include "pycore_ceval.h" #if defined(Py_DEBUG) && defined(Py_BUILD_CORE) # define D(x) if (p->debug) { x; } @@ -44,7 +46,7 @@ # define MAXSTACK 4000 # endif #else -# define MAXSTACK 4000 +# define MAXSTACK 6000 #endif """ @@ -86,13 +88,13 @@ class NodeTypes(Enum): @dataclass class FunctionCall: function: str - arguments: List[Any] = field(default_factory=list) - assigned_variable: Optional[str] = None - assigned_variable_type: Optional[str] = None - return_type: Optional[str] = None - nodetype: Optional[NodeTypes] = None + arguments: list[Any] = field(default_factory=list) + assigned_variable: str | None = None + assigned_variable_type: str | None = None + return_type: str | None = None + nodetype: NodeTypes | None = None force_true: bool = False - comment: Optional[str] = None + comment: str | None = None def __str__(self) -> str: parts = [] @@ -124,14 +126,14 @@ class CCallMakerVisitor(GrammarVisitor): def __init__( self, parser_generator: ParserGenerator, - exact_tokens: Dict[str, int], - non_exact_tokens: Set[str], + exact_tokens: dict[str, int], + non_exact_tokens: set[str], ): self.gen = parser_generator self.exact_tokens = exact_tokens self.non_exact_tokens = non_exact_tokens - self.cache: Dict[str, str] = {} - self.cleanup_statements: List[str] = [] + self.cache: dict[str, str] = {} + self.cleanup_statements: list[str] = [] def keyword_helper(self, keyword: str) -> FunctionCall: return FunctionCall( @@ -167,7 +169,7 @@ def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall: ) return FunctionCall( assigned_variable=f"{name.lower()}_var", - function=f"_PyPegen_expect_token", + function="_PyPegen_expect_token", arguments=["p", name], nodetype=NodeTypes.GENERIC_TOKEN, return_type="Token *", @@ -199,7 +201,7 @@ def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall: type = self.exact_tokens[val] return FunctionCall( assigned_variable="_literal", - function=f"_PyPegen_expect_token", + function="_PyPegen_expect_token", arguments=["p", type], nodetype=NodeTypes.GENERIC_TOKEN, return_type="Token *", @@ -214,33 +216,47 @@ def visit_NamedItem(self, node: NamedItem) -> FunctionCall: call.assigned_variable_type = node.type return call + def assert_no_undefined_behavior( + self, call: FunctionCall, wrapper: str, expected_rtype: str | None, + ) -> None: + if call.return_type != expected_rtype: + raise RuntimeError( + f"{call.function} return type is incompatible with {wrapper}: " + f"expect: {expected_rtype}, actual: {call.return_type}" + ) + def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: call = self.generate_call(node.node) - if call.nodetype == NodeTypes.NAME_TOKEN: - return FunctionCall( - function=f"_PyPegen_lookahead_with_name", - arguments=[positive, call.function, *call.arguments], - return_type="int", - ) + comment = None + if call.nodetype is NodeTypes.NAME_TOKEN: + function = "_PyPegen_lookahead_for_expr" + self.assert_no_undefined_behavior(call, function, "expr_ty") + elif call.nodetype is NodeTypes.STRING_TOKEN: + # _PyPegen_string_token() returns 'void *' instead of 'Token *'; + # in addition, the overall function call would return 'expr_ty'. + assert call.function == "_PyPegen_string_token" + function = "_PyPegen_lookahead" + self.assert_no_undefined_behavior(call, function, "expr_ty") elif call.nodetype == NodeTypes.SOFT_KEYWORD: - return FunctionCall( - function=f"_PyPegen_lookahead_with_string", - arguments=[positive, call.function, *call.arguments], - return_type="int", - ) + function = "_PyPegen_lookahead_with_string" + self.assert_no_undefined_behavior(call, function, "expr_ty") elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}: - return FunctionCall( - function=f"_PyPegen_lookahead_with_int", - arguments=[positive, call.function, *call.arguments], - return_type="int", - comment=f"token={node.node}", - ) + function = "_PyPegen_lookahead_with_int" + self.assert_no_undefined_behavior(call, function, "Token *") + comment = f"token={node.node}" + elif call.return_type == "expr_ty": + function = "_PyPegen_lookahead_for_expr" + elif call.return_type == "stmt_ty": + function = "_PyPegen_lookahead_for_stmt" else: - return FunctionCall( - function=f"_PyPegen_lookahead", - arguments=[positive, f"(void *(*)(Parser *)) {call.function}", *call.arguments], - return_type="int", - ) + function = "_PyPegen_lookahead" + self.assert_no_undefined_behavior(call, function, None) + return FunctionCall( + function=function, + arguments=[positive, call.function, *call.arguments], + return_type="int", + comment=comment, + ) def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall: return self.lookahead_call_helper(node, 1) @@ -257,7 +273,7 @@ def visit_Forced(self, node: Forced) -> FunctionCall: type = self.exact_tokens[val] return FunctionCall( assigned_variable="_literal", - function=f"_PyPegen_expect_forced_token", + function="_PyPegen_expect_forced_token", arguments=["p", type, f'"{val}"'], nodetype=NodeTypes.GENERIC_TOKEN, return_type="Token *", @@ -269,7 +285,7 @@ def visit_Forced(self, node: Forced) -> FunctionCall: call.comment = None return FunctionCall( assigned_variable="_literal", - function=f"_PyPegen_expect_forced_result", + function="_PyPegen_expect_forced_result", arguments=["p", str(call), f'"{node.node.rhs!s}"'], return_type="void *", comment=f"forced_token=({node.node.rhs!s})", @@ -292,7 +308,7 @@ def _generate_artificial_rule_call( node: Any, prefix: str, rule_generation_func: Callable[[], str], - return_type: Optional[str] = None, + return_type: str | None = None, ) -> FunctionCall: node_str = f"{node}" key = f"{prefix}_{node_str}" @@ -363,10 +379,10 @@ class CParserGenerator(ParserGenerator, GrammarVisitor): def __init__( self, grammar: grammar.Grammar, - tokens: Dict[int, str], - exact_tokens: Dict[str, int], - non_exact_tokens: Set[str], - file: Optional[IO[Text]], + tokens: dict[int, str], + exact_tokens: dict[str, int], + non_exact_tokens: set[str], + file: IO[str] | None, debug: bool = False, skip_actions: bool = False, ): @@ -377,7 +393,7 @@ def __init__( self._varname_counter = 0 self.debug = debug self.skip_actions = skip_actions - self.cleanup_statements: List[str] = [] + self.cleanup_statements: list[str] = [] def add_level(self) -> None: self.print("if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) {") @@ -413,12 +429,12 @@ def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None: self.print(f"if ({error_var}) {{") with self.indent(): self.print(f"goto {goto_target};") - self.print(f"}}") + self.print("}") def out_of_memory_return( self, expr: str, - cleanup_code: Optional[str] = None, + cleanup_code: str | None = None, ) -> None: self.print(f"if ({expr}) {{") with self.indent(): @@ -427,14 +443,14 @@ def out_of_memory_return( self.print("p->error_indicator = 1;") self.print("PyErr_NoMemory();") self.add_return("NULL") - self.print(f"}}") + self.print("}") def out_of_memory_goto(self, expr: str, goto_target: str) -> None: self.print(f"if ({expr}) {{") with self.indent(): self.print("PyErr_NoMemory();") self.print(f"goto {goto_target};") - self.print(f"}}") + self.print("}") def generate(self, filename: str) -> None: self.collect_rules() @@ -477,8 +493,8 @@ def generate(self, filename: str) -> None: if trailer: self.print(trailer.rstrip("\n") % dict(mode=mode, modulename=modulename)) - def _group_keywords_by_length(self) -> Dict[int, List[Tuple[str, int]]]: - groups: Dict[int, List[Tuple[str, int]]] = {} + def _group_keywords_by_length(self) -> dict[int, list[tuple[str, int]]]: + groups: dict[int, list[tuple[str, int]]] = {} for keyword_str, keyword_type in self.keywords.items(): length = len(keyword_str) if length in groups: @@ -570,17 +586,17 @@ def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None: self.print("if (_raw == NULL || p->mark <= _resmark)") with self.indent(): self.print("break;") - self.print(f"_resmark = p->mark;") + self.print("_resmark = p->mark;") self.print("_res = _raw;") self.print("}") - self.print(f"p->mark = _resmark;") + self.print("p->mark = _resmark;") self.add_return("_res") self.print("}") self.print(f"static {result_type}") self.print(f"{node.name}_raw(Parser *p)") def _should_memoize(self, node: Rule) -> bool: - return node.memo and not node.left_recursive + return "memo" in node.flags and not node.left_recursive def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None: memoize = self._should_memoize(node) @@ -629,7 +645,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: if memoize: self.print("int _start_mark = p->mark;") self.print("void **_children = PyMem_Malloc(sizeof(void *));") - self.out_of_memory_return(f"!_children") + self.out_of_memory_return("!_children") self.print("Py_ssize_t _children_capacity = 1;") self.print("Py_ssize_t _n = 0;") if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): @@ -647,7 +663,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.add_return("NULL") self.print("}") self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);") - self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") + self.out_of_memory_return("!_seq", cleanup_code="PyMem_Free(_children);") self.print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") self.print("PyMem_Free(_children);") if memoize and node.name: @@ -701,7 +717,7 @@ def visit_NamedItem(self, node: NamedItem) -> None: self.print(call) def visit_Rhs( - self, node: Rhs, is_loop: bool, is_gather: bool, rulename: Optional[str] + self, node: Rhs, is_loop: bool, is_gather: bool, rulename: str | None ) -> None: if is_loop: assert len(node.alts) == 1 @@ -720,7 +736,7 @@ def join_conditions(self, keyword: str, node: Any) -> None: self.visit(item) self.print(")") - def emit_action(self, node: Alt, cleanup_code: Optional[str] = None) -> None: + def emit_action(self, node: Alt, cleanup_code: str | None = None) -> None: self.print(f"_res = {node.action};") self.print("if (_res == NULL && PyErr_Occurred()) {") @@ -762,7 +778,7 @@ def emit_default_action(self, is_gather: bool, node: Alt) -> None: def emit_dummy_action(self) -> None: self.print("_res = _PyPegen_dummy_name(p);") - def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: Optional[str]) -> None: + def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: str | None) -> None: self.join_conditions(keyword="if", node=node) self.print("{") # We have parsed successfully all the conditions for the option. @@ -782,10 +798,10 @@ def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: Optional[str]) self.emit_default_action(is_gather, node) # As the current option has parsed correctly, do not continue with the rest. - self.print(f"goto done;") + self.print("goto done;") self.print("}") - def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) -> None: + def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: str | None) -> None: # Condition of the main body of the alternative self.join_conditions(keyword="while", node=node) self.print("{") @@ -809,7 +825,7 @@ def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) - self.print( "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" ) - self.out_of_memory_return(f"!_new_children", cleanup_code="PyMem_Free(_children);") + self.out_of_memory_return("!_new_children", cleanup_code="PyMem_Free(_children);") self.print("_children = _new_children;") self.print("}") self.print("_children[_n++] = _res;") @@ -817,7 +833,7 @@ def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) - self.print("}") def visit_Alt( - self, node: Alt, is_loop: bool, is_gather: bool, rulename: Optional[str] + self, node: Alt, is_loop: bool, is_gather: bool, rulename: str | None ) -> None: if len(node.items) == 1 and str(node.items[0]).startswith("invalid_"): self.print(f"if (p->call_invalid_rules) {{ // {node}") @@ -861,7 +877,7 @@ def visit_Alt( self.print("}") self.print("}") - def collect_vars(self, node: Alt) -> Dict[Optional[str], Optional[str]]: + def collect_vars(self, node: Alt) -> dict[str | None, str | None]: types = {} with self.local_variable_context(): for item in node.items: @@ -869,7 +885,7 @@ def collect_vars(self, node: Alt) -> Dict[Optional[str], Optional[str]]: types[name] = type return types - def add_var(self, node: NamedItem) -> Tuple[Optional[str], Optional[str]]: + def add_var(self, node: NamedItem) -> tuple[str | None, str | None]: call = self.callmakervisitor.generate_call(node.item) name = node.name if node.name else call.assigned_variable if name is not None: diff --git a/Tools/peg_generator/pegen/first_sets.py b/Tools/peg_generator/pegen/first_sets.py index 6d794ffa4bf..f6b83c0ca31 100755 --- a/Tools/peg_generator/pegen/first_sets.py +++ b/Tools/peg_generator/pegen/first_sets.py @@ -3,7 +3,6 @@ import argparse import pprint import sys -from typing import Dict, Set from pegen.build import build_parser from pegen.grammar import ( @@ -33,20 +32,20 @@ class FirstSetCalculator(GrammarVisitor): - def __init__(self, rules: Dict[str, Rule]) -> None: + def __init__(self, rules: dict[str, Rule]) -> None: self.rules = rules self.nullables = compute_nullables(rules) - self.first_sets: Dict[str, Set[str]] = dict() - self.in_process: Set[str] = set() + self.first_sets: dict[str, set[str]] = dict() + self.in_process: set[str] = set() - def calculate(self) -> Dict[str, Set[str]]: + def calculate(self) -> dict[str, set[str]]: for name, rule in self.rules.items(): self.visit(rule) return self.first_sets - def visit_Alt(self, item: Alt) -> Set[str]: - result: Set[str] = set() - to_remove: Set[str] = set() + def visit_Alt(self, item: Alt) -> set[str]: + result: set[str] = set() + to_remove: set[str] = set() for other in item.items: new_terminals = self.visit(other) if isinstance(other.item, NegativeLookahead): @@ -71,34 +70,34 @@ def visit_Alt(self, item: Alt) -> Set[str]: return result - def visit_Cut(self, item: Cut) -> Set[str]: + def visit_Cut(self, item: Cut) -> set[str]: return set() - def visit_Group(self, item: Group) -> Set[str]: + def visit_Group(self, item: Group) -> set[str]: return self.visit(item.rhs) - def visit_PositiveLookahead(self, item: Lookahead) -> Set[str]: + def visit_PositiveLookahead(self, item: Lookahead) -> set[str]: return self.visit(item.node) - def visit_NegativeLookahead(self, item: NegativeLookahead) -> Set[str]: + def visit_NegativeLookahead(self, item: NegativeLookahead) -> set[str]: return self.visit(item.node) - def visit_NamedItem(self, item: NamedItem) -> Set[str]: + def visit_NamedItem(self, item: NamedItem) -> set[str]: return self.visit(item.item) - def visit_Opt(self, item: Opt) -> Set[str]: + def visit_Opt(self, item: Opt) -> set[str]: return self.visit(item.node) - def visit_Gather(self, item: Gather) -> Set[str]: + def visit_Gather(self, item: Gather) -> set[str]: return self.visit(item.node) - def visit_Repeat0(self, item: Repeat0) -> Set[str]: + def visit_Repeat0(self, item: Repeat0) -> set[str]: return self.visit(item.node) - def visit_Repeat1(self, item: Repeat1) -> Set[str]: + def visit_Repeat1(self, item: Repeat1) -> set[str]: return self.visit(item.node) - def visit_NameLeaf(self, item: NameLeaf) -> Set[str]: + def visit_NameLeaf(self, item: NameLeaf) -> set[str]: if item.value not in self.rules: return {item.value} @@ -110,16 +109,16 @@ def visit_NameLeaf(self, item: NameLeaf) -> Set[str]: return self.first_sets[item.value] - def visit_StringLeaf(self, item: StringLeaf) -> Set[str]: + def visit_StringLeaf(self, item: StringLeaf) -> set[str]: return {item.value} - def visit_Rhs(self, item: Rhs) -> Set[str]: - result: Set[str] = set() + def visit_Rhs(self, item: Rhs) -> set[str]: + result: set[str] = set() for alt in item.alts: result |= self.visit(alt) return result - def visit_Rule(self, item: Rule) -> Set[str]: + def visit_Rule(self, item: Rule) -> set[str]: if item.name in self.in_process: return set() elif item.name not in self.first_sets: @@ -138,7 +137,7 @@ def main() -> None: try: grammar, parser, tokenizer = build_parser(args.grammar_file) except Exception as err: - print("ERROR: Failed to parse grammar file", file=sys.stderr) + print("ERROR: Failed to parse grammar file", err, file=sys.stderr) sys.exit(1) firs_sets = FirstSetCalculator(grammar.rules).calculate() diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py index 1ee9d100b61..d3c2eca6615 100644 --- a/Tools/peg_generator/pegen/grammar.py +++ b/Tools/peg_generator/pegen/grammar.py @@ -1,15 +1,7 @@ from __future__ import annotations -from typing import ( - AbstractSet, - Any, - Iterable, - Iterator, - List, - Optional, - Tuple, - Union, -) +from collections.abc import Iterable, Iterator, Set +from typing import Any class GrammarError(Exception): @@ -34,7 +26,7 @@ def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> Any: class Grammar: - def __init__(self, rules: Iterable[Rule], metas: Iterable[Tuple[str, Optional[str]]]): + def __init__(self, rules: Iterable[Rule], metas: Iterable[tuple[str, str | None]]): # Check if there are repeated rules in "rules" all_rules = {} for rule in rules: @@ -66,11 +58,11 @@ def __iter__(self) -> Iterator[Rule]: class Rule: - def __init__(self, name: str, type: Optional[str], rhs: Rhs, memo: Optional[object] = None): + def __init__(self, name: str, type: str | None, rhs: Rhs, flags: frozenset[str] | None = None): self.name = name self.type = type self.rhs = rhs - self.memo = bool(memo) + self.flags = flags or frozenset() self.left_recursive = False self.leader = False @@ -141,9 +133,8 @@ def __repr__(self) -> str: class Rhs: - def __init__(self, alts: List[Alt]): + def __init__(self, alts: list[Alt]): self.alts = alts - self.memo: Optional[Tuple[Optional[str], str]] = None def __str__(self) -> str: return " | ".join(str(alt) for alt in self.alts) @@ -151,7 +142,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"Rhs({self.alts!r})" - def __iter__(self) -> Iterator[List[Alt]]: + def __iter__(self) -> Iterator[list[Alt]]: yield self.alts @property @@ -165,7 +156,7 @@ def can_be_inlined(self) -> bool: class Alt: - def __init__(self, items: List[NamedItem], *, icut: int = -1, action: Optional[str] = None): + def __init__(self, items: list[NamedItem], *, icut: int = -1, action: str | None = None): self.items = items self.icut = icut self.action = action @@ -185,12 +176,12 @@ def __repr__(self) -> str: args.append(f"action={self.action!r}") return f"Alt({', '.join(args)})" - def __iter__(self) -> Iterator[List[NamedItem]]: + def __iter__(self) -> Iterator[list[NamedItem]]: yield self.items class NamedItem: - def __init__(self, name: Optional[str], item: Item, type: Optional[str] = None): + def __init__(self, name: str | None, item: Item, type: str | None = None): self.name = name self.item = item self.type = type @@ -271,7 +262,6 @@ class Repeat: def __init__(self, node: Plain): self.node = node - self.memo: Optional[Tuple[Optional[str], str]] = None def __iter__(self) -> Iterator[Plain]: yield self.node @@ -334,12 +324,12 @@ def __init__(self) -> None: pass def __repr__(self) -> str: - return f"Cut()" + return "Cut()" def __str__(self) -> str: - return f"~" + return "~" - def __iter__(self) -> Iterator[Tuple[str, str]]: + def __iter__(self) -> Iterator[tuple[str, str]]: yield from () def __eq__(self, other: object) -> bool: @@ -347,15 +337,15 @@ def __eq__(self, other: object) -> bool: return NotImplemented return True - def initial_names(self) -> AbstractSet[str]: + def initial_names(self) -> Set[str]: return set() -Plain = Union[Leaf, Group] -Item = Union[Plain, Opt, Repeat, Forced, Lookahead, Rhs, Cut] -RuleName = Tuple[str, Optional[str]] -MetaTuple = Tuple[str, Optional[str]] -MetaList = List[MetaTuple] -RuleList = List[Rule] -NamedItemList = List[NamedItem] -LookaheadOrCut = Union[Lookahead, Cut] +Plain = Leaf | Group +Item = Plain | Opt | Repeat | Forced | Lookahead | Rhs | Cut +RuleName = tuple[str, str | None] +MetaTuple = tuple[str, str | None] +MetaList = list[MetaTuple] +RuleList = list[Rule] +NamedItemList = list[NamedItem] +LookaheadOrCut = Lookahead | Cut diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py index 2e3a607f720..4fa27392707 100644 --- a/Tools/peg_generator/pegen/grammar_parser.py +++ b/Tools/peg_generator/pegen/grammar_parser.py @@ -147,12 +147,12 @@ class GeneratedParser(Parser): @memoize def rule(self) -> Optional[Rule]: - # rule: rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT | rulename memoflag? ":" alts NEWLINE + # rule: rulename flags? ":" alts NEWLINE INDENT more_alts DEDENT | rulename flags? ":" NEWLINE INDENT more_alts DEDENT | rulename flags? ":" alts NEWLINE mark = self._mark() if ( (rulename := self.rulename()) and - (opt := self.memoflag(),) + (flags := self.flags(),) and (literal := self.expect(":")) and @@ -166,12 +166,12 @@ class GeneratedParser(Parser): and (_dedent := self.expect('DEDENT')) ): - return Rule ( rulename [0] , rulename [1] , Rhs ( alts . alts + more_alts . alts ) , memo = opt ) + return Rule ( rulename [0] , rulename [1] , Rhs ( alts . alts + more_alts . alts ) , flags = flags ) self._reset(mark) if ( (rulename := self.rulename()) and - (opt := self.memoflag(),) + (flags := self.flags(),) and (literal := self.expect(":")) and @@ -183,12 +183,12 @@ class GeneratedParser(Parser): and (_dedent := self.expect('DEDENT')) ): - return Rule ( rulename [0] , rulename [1] , more_alts , memo = opt ) + return Rule ( rulename [0] , rulename [1] , more_alts , flags = flags ) self._reset(mark) if ( (rulename := self.rulename()) and - (opt := self.memoflag(),) + (flags := self.flags(),) and (literal := self.expect(":")) and @@ -196,7 +196,7 @@ class GeneratedParser(Parser): and (_newline := self.expect('NEWLINE')) ): - return Rule ( rulename [0] , rulename [1] , alts , memo = opt ) + return Rule ( rulename [0] , rulename [1] , alts , flags = flags ) self._reset(mark) return None @@ -219,17 +219,28 @@ class GeneratedParser(Parser): return None @memoize - def memoflag(self) -> Optional[str]: - # memoflag: '(' "memo" ')' + def flags(self) -> Optional[frozenset [str]]: + # flags: '(' ','.flag+ ')' mark = self._mark() if ( (literal := self.expect('(')) and - (literal_1 := self.expect("memo")) + (a := self._gather_2()) and - (literal_2 := self.expect(')')) + (literal_1 := self.expect(')')) ): - return "memo" + return frozenset ( a ) + self._reset(mark) + return None + + @memoize + def flag(self) -> Optional[str]: + # flag: NAME + mark = self._mark() + if ( + (name := self.name()) + ): + return name . string self._reset(mark) return None @@ -661,8 +672,38 @@ class GeneratedParser(Parser): self._reset(mark) return None + @memoize + def _loop0_1(self) -> Optional[Any]: + # _loop0_1: ',' flag + mark = self._mark() + children = [] + while ( + (literal := self.expect(',')) + and + (elem := self.flag()) + ): + children.append(elem) + mark = self._mark() + self._reset(mark) + return children + + @memoize + def _gather_2(self) -> Optional[Any]: + # _gather_2: flag _loop0_1 + mark = self._mark() + if ( + (elem := self.flag()) + is not None + and + (seq := self._loop0_1()) + is not None + ): + return [elem] + seq + self._reset(mark) + return None + KEYWORDS = () - SOFT_KEYWORDS = ('memo',) + SOFT_KEYWORDS = () if __name__ == '__main__': diff --git a/Tools/peg_generator/pegen/grammar_visualizer.py b/Tools/peg_generator/pegen/grammar_visualizer.py index 11f784f45b6..eadb6987389 100644 --- a/Tools/peg_generator/pegen/grammar_visualizer.py +++ b/Tools/peg_generator/pegen/grammar_visualizer.py @@ -1,6 +1,7 @@ import argparse import sys -from typing import Any, Callable, Iterator +from collections.abc import Callable, Iterator +from typing import Any from pegen.build import build_parser from pegen.grammar import Grammar, Rule @@ -52,7 +53,7 @@ def main() -> None: try: grammar, parser, tokenizer = build_parser(args.filename) except Exception as err: - print("ERROR: Failed to parse grammar file", file=sys.stderr) + print("ERROR: Failed to parse grammar file", err, file=sys.stderr) sys.exit(1) visitor = ASTGrammarPrinter() diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram index f484c478182..cae91ab9c41 100644 --- a/Tools/peg_generator/pegen/metagrammar.gram +++ b/Tools/peg_generator/pegen/metagrammar.gram @@ -50,19 +50,21 @@ rules[RuleList]: | rule { [rule] } rule[Rule]: - | rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT { - Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), memo=opt) } - | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT { - Rule(rulename[0], rulename[1], more_alts, memo=opt) } - | rulename memoflag? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, memo=opt) } + | rulename flags=flags? ":" alts NEWLINE INDENT more_alts DEDENT { + Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), flags=flags) } + | rulename flags=flags? ":" NEWLINE INDENT more_alts DEDENT { + Rule(rulename[0], rulename[1], more_alts, flags=flags) } + | rulename flags=flags? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, flags=flags) } rulename[RuleName]: | NAME annotation { (name.string, annotation) } | NAME { (name.string, None) } -# In the future this may return something more complicated -memoflag[str]: - | '(' "memo" ')' { "memo" } +flags[frozenset[str]]: + | '(' a=','.flag+ ')' { frozenset(a) } + +flag[str]: + | NAME { name.string } alts[Rhs]: | alt "|" alts { Rhs([alt] + alts.alts)} diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index a987d30a9d6..806003c8350 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -5,7 +5,8 @@ import tokenize import traceback from abc import abstractmethod -from typing import Any, Callable, ClassVar, Dict, Optional, Tuple, Type, TypeVar, cast +from collections.abc import Callable +from typing import Any, ClassVar, TypeVar, cast from pegen.tokenizer import Mark, Tokenizer, exact_token_types @@ -74,12 +75,12 @@ def memoize_wrapper(self: "Parser", *args: object) -> Any: def memoize_left_rec( - method: Callable[["Parser"], Optional[T]] -) -> Callable[["Parser"], Optional[T]]: + method: Callable[["Parser"], T | None] +) -> Callable[["Parser"], T | None]: """Memoize a left-recursive symbol method.""" method_name = method.__name__ - def memoize_left_rec_wrapper(self: "Parser") -> Optional[T]: + def memoize_left_rec_wrapper(self: "Parser") -> T | None: mark = self._mark() key = mark, method_name, () # Fast path: cache hit, and not verbose. @@ -160,15 +161,15 @@ def memoize_left_rec_wrapper(self: "Parser") -> Optional[T]: class Parser: """Parsing base class.""" - KEYWORDS: ClassVar[Tuple[str, ...]] + KEYWORDS: ClassVar[tuple[str, ...]] - SOFT_KEYWORDS: ClassVar[Tuple[str, ...]] + SOFT_KEYWORDS: ClassVar[tuple[str, ...]] def __init__(self, tokenizer: Tokenizer, *, verbose: bool = False): self._tokenizer = tokenizer self._verbose = verbose self._level = 0 - self._cache: Dict[Tuple[Mark, str, Tuple[Any, ...]], Tuple[Any, Mark]] = {} + self._cache: dict[tuple[Mark, str, tuple[Any, ...]], tuple[Any, Mark]] = {} # Integer tracking whether we are in a left recursive rule or not. Can be useful # for error reporting. self.in_recursive_rule = 0 @@ -185,28 +186,28 @@ def showpeek(self) -> str: return f"{tok.start[0]}.{tok.start[1]}: {token.tok_name[tok.type]}:{tok.string!r}" @memoize - def name(self) -> Optional[tokenize.TokenInfo]: + def name(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.NAME and tok.string not in self.KEYWORDS: return self._tokenizer.getnext() return None @memoize - def number(self) -> Optional[tokenize.TokenInfo]: + def number(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.NUMBER: return self._tokenizer.getnext() return None @memoize - def string(self) -> Optional[tokenize.TokenInfo]: + def string(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.STRING: return self._tokenizer.getnext() return None @memoize - def fstring_start(self) -> Optional[tokenize.TokenInfo]: + def fstring_start(self) -> tokenize.TokenInfo | None: FSTRING_START = getattr(token, "FSTRING_START", None) if not FSTRING_START: return None @@ -216,7 +217,7 @@ def fstring_start(self) -> Optional[tokenize.TokenInfo]: return None @memoize - def fstring_middle(self) -> Optional[tokenize.TokenInfo]: + def fstring_middle(self) -> tokenize.TokenInfo | None: FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None) if not FSTRING_MIDDLE: return None @@ -226,7 +227,7 @@ def fstring_middle(self) -> Optional[tokenize.TokenInfo]: return None @memoize - def fstring_end(self) -> Optional[tokenize.TokenInfo]: + def fstring_end(self) -> tokenize.TokenInfo | None: FSTRING_END = getattr(token, "FSTRING_END", None) if not FSTRING_END: return None @@ -236,28 +237,28 @@ def fstring_end(self) -> Optional[tokenize.TokenInfo]: return None @memoize - def op(self) -> Optional[tokenize.TokenInfo]: + def op(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.OP: return self._tokenizer.getnext() return None @memoize - def type_comment(self) -> Optional[tokenize.TokenInfo]: + def type_comment(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.TYPE_COMMENT: return self._tokenizer.getnext() return None @memoize - def soft_keyword(self) -> Optional[tokenize.TokenInfo]: + def soft_keyword(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.NAME and tok.string in self.SOFT_KEYWORDS: return self._tokenizer.getnext() return None @memoize - def expect(self, type: str) -> Optional[tokenize.TokenInfo]: + def expect(self, type: str) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.string == type: return self._tokenizer.getnext() @@ -271,7 +272,7 @@ def expect(self, type: str) -> Optional[tokenize.TokenInfo]: return self._tokenizer.getnext() return None - def expect_forced(self, res: Any, expectation: str) -> Optional[tokenize.TokenInfo]: + def expect_forced(self, res: Any, expectation: str) -> tokenize.TokenInfo | None: if res is None: raise self.make_syntax_error(f"expected {expectation}") return res @@ -293,7 +294,7 @@ def make_syntax_error(self, message: str, filename: str = "<unknown>") -> Syntax return SyntaxError(message, (filename, tok.start[0], 1 + tok.start[1], tok.line)) -def simple_parser_main(parser_class: Type[Parser]) -> None: +def simple_parser_main(parser_class: type[Parser]) -> None: argparser = argparse.ArgumentParser() argparser.add_argument( "-v", @@ -330,7 +331,7 @@ def simple_parser_main(parser_class: Type[Parser]) -> None: endpos = 0 else: endpos = file.tell() - except IOError: + except OSError: endpos = 0 finally: if file is not sys.stdin: diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 6ce0649aefe..81314b0cc07 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -1,22 +1,10 @@ -import sys import ast import contextlib import re +import sys from abc import abstractmethod -from typing import ( - IO, - AbstractSet, - Any, - Dict, - Iterable, - Iterator, - List, - Optional, - Set, - Text, - Tuple, - Union, -) +from collections.abc import Iterable, Iterator, Set +from typing import IO, Any from pegen import sccutils from pegen.grammar import ( @@ -44,8 +32,7 @@ class RuleCollectorVisitor(GrammarVisitor): """Visitor that invokes a provided callmaker visitor with just the NamedItem nodes""" - def __init__(self, rules: Dict[str, Rule], callmakervisitor: GrammarVisitor) -> None: - self.rulses = rules + def __init__(self, callmakervisitor: GrammarVisitor) -> None: self.callmaker = callmakervisitor def visit_Rule(self, rule: Rule) -> None: @@ -56,9 +43,9 @@ def visit_NamedItem(self, item: NamedItem) -> None: class KeywordCollectorVisitor(GrammarVisitor): - """Visitor that collects all the keywods and soft keywords in the Grammar""" + """Visitor that collects all the keywords and soft keywords in the Grammar""" - def __init__(self, gen: "ParserGenerator", keywords: Dict[str, int], soft_keywords: Set[str]): + def __init__(self, gen: "ParserGenerator", keywords: dict[str, int], soft_keywords: set[str]): self.generator = gen self.keywords = keywords self.soft_keywords = soft_keywords @@ -73,7 +60,7 @@ def visit_StringLeaf(self, node: StringLeaf) -> None: class RuleCheckingVisitor(GrammarVisitor): - def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): + def __init__(self, rules: dict[str, Rule], tokens: set[str]): self.rules = rules self.tokens = tokens # If python < 3.12 add the virtual fstring tokens @@ -81,6 +68,11 @@ def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.tokens.add("FSTRING_START") self.tokens.add("FSTRING_END") self.tokens.add("FSTRING_MIDDLE") + # If python < 3.14 add the virtual tstring tokens + if sys.version_info < (3, 14, 0, 'beta', 1): + self.tokens.add("TSTRING_START") + self.tokens.add("TSTRING_END") + self.tokens.add("TSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: @@ -95,11 +87,11 @@ def visit_NamedItem(self, node: NamedItem) -> None: class ParserGenerator: callmakervisitor: GrammarVisitor - def __init__(self, grammar: Grammar, tokens: Set[str], file: Optional[IO[Text]]): + def __init__(self, grammar: Grammar, tokens: set[str], file: IO[str] | None): self.grammar = grammar self.tokens = tokens - self.keywords: Dict[str, int] = {} - self.soft_keywords: Set[str] = set() + self.keywords: dict[str, int] = {} + self.soft_keywords: set[str] = set() self.rules = grammar.rules self.validate_rule_names() if "trailer" not in grammar.metas and "start" not in self.rules: @@ -112,8 +104,8 @@ def __init__(self, grammar: Grammar, tokens: Set[str], file: Optional[IO[Text]]) self.first_graph, self.first_sccs = compute_left_recursives(self.rules) self.counter = 0 # For name_rule()/name_loop() self.keyword_counter = 499 # For keyword_type() - self.all_rules: Dict[str, Rule] = self.rules.copy() # Rules + temporal rules - self._local_variable_stack: List[List[str]] = [] + self.all_rules: dict[str, Rule] = self.rules.copy() # Rules + temporal rules + self._local_variable_stack: list[list[str]] = [] def validate_rule_names(self) -> None: for rule in self.rules: @@ -127,7 +119,7 @@ def local_variable_context(self) -> Iterator[None]: self._local_variable_stack.pop() @property - def local_variable_names(self) -> List[str]: + def local_variable_names(self) -> list[str]: return self._local_variable_stack[-1] @abstractmethod @@ -158,8 +150,8 @@ def collect_rules(self) -> None: for rule in self.all_rules.values(): keyword_collector.visit(rule) - rule_collector = RuleCollectorVisitor(self.rules, self.callmakervisitor) - done: Set[str] = set() + rule_collector = RuleCollectorVisitor(self.callmakervisitor) + done: set[str] = set() while True: computed_rules = list(self.all_rules) todo = [i for i in computed_rules if i not in done] @@ -224,10 +216,10 @@ def dedupe(self, name: str) -> str: class NullableVisitor(GrammarVisitor): - def __init__(self, rules: Dict[str, Rule]) -> None: + def __init__(self, rules: dict[str, Rule]) -> None: self.rules = rules - self.visited: Set[Any] = set() - self.nullables: Set[Union[Rule, NamedItem]] = set() + self.visited: set[Any] = set() + self.nullables: set[Rule | NamedItem] = set() def visit_Rule(self, rule: Rule) -> bool: if rule in self.visited: @@ -289,7 +281,7 @@ def visit_StringLeaf(self, node: StringLeaf) -> bool: return not node.value -def compute_nullables(rules: Dict[str, Rule]) -> Set[Any]: +def compute_nullables(rules: dict[str, Rule]) -> set[Any]: """Compute which rules in a grammar are nullable. Thanks to TatSu (tatsu/leftrec.py) for inspiration. @@ -301,12 +293,12 @@ def compute_nullables(rules: Dict[str, Rule]) -> Set[Any]: class InitialNamesVisitor(GrammarVisitor): - def __init__(self, rules: Dict[str, Rule]) -> None: + def __init__(self, rules: dict[str, Rule]) -> None: self.rules = rules self.nullables = compute_nullables(rules) - def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> Set[Any]: - names: Set[str] = set() + def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> set[Any]: + names: set[str] = set() for value in node: if isinstance(value, list): for item in value: @@ -315,33 +307,33 @@ def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> Set[A names |= self.visit(value, *args, **kwargs) return names - def visit_Alt(self, alt: Alt) -> Set[Any]: - names: Set[str] = set() + def visit_Alt(self, alt: Alt) -> set[Any]: + names: set[str] = set() for item in alt.items: names |= self.visit(item) if item not in self.nullables: break return names - def visit_Forced(self, force: Forced) -> Set[Any]: + def visit_Forced(self, force: Forced) -> set[Any]: return set() - def visit_LookAhead(self, lookahead: Lookahead) -> Set[Any]: + def visit_LookAhead(self, lookahead: Lookahead) -> set[Any]: return set() - def visit_Cut(self, cut: Cut) -> Set[Any]: + def visit_Cut(self, cut: Cut) -> set[Any]: return set() - def visit_NameLeaf(self, node: NameLeaf) -> Set[Any]: + def visit_NameLeaf(self, node: NameLeaf) -> set[Any]: return {node.value} - def visit_StringLeaf(self, node: StringLeaf) -> Set[Any]: + def visit_StringLeaf(self, node: StringLeaf) -> set[Any]: return set() def compute_left_recursives( - rules: Dict[str, Rule] -) -> Tuple[Dict[str, AbstractSet[str]], List[AbstractSet[str]]]: + rules: dict[str, Rule] +) -> tuple[dict[str, Set[str]], list[Set[str]]]: graph = make_first_graph(rules) sccs = list(sccutils.strongly_connected_components(graph.keys(), graph)) for scc in sccs: @@ -369,7 +361,7 @@ def compute_left_recursives( return graph, sccs -def make_first_graph(rules: Dict[str, Rule]) -> Dict[str, AbstractSet[str]]: +def make_first_graph(rules: dict[str, Rule]) -> dict[str, Set[str]]: """Compute the graph of left-invocations. There's an edge from A to B if A may invoke B at its initial @@ -379,7 +371,7 @@ def make_first_graph(rules: Dict[str, Rule]) -> Dict[str, AbstractSet[str]]: """ initial_name_visitor = InitialNamesVisitor(rules) graph = {} - vertices: Set[str] = set() + vertices: set[str] = set() for rulename, rhs in rules.items(): graph[rulename] = names = initial_name_visitor.visit(rhs) vertices |= names diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 4bb26480ebc..e69ea4b5f4e 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -1,6 +1,7 @@ import os.path import token -from typing import IO, Any, Callable, Dict, Optional, Sequence, Set, Text, Tuple +from collections.abc import Callable, Sequence +from typing import IO, Any from pegen import grammar from pegen.grammar import ( @@ -74,10 +75,10 @@ def visit_NegativeLookahead(self, node: NegativeLookahead) -> bool: def visit_Opt(self, node: Opt) -> bool: return self.visit(node.node) - def visit_Repeat(self, node: Repeat0) -> Tuple[str, str]: + def visit_Repeat(self, node: Repeat0) -> tuple[str, str]: return self.visit(node.node) - def visit_Gather(self, node: Gather) -> Tuple[str, str]: + def visit_Gather(self, node: Gather) -> tuple[str, str]: return self.visit(node.node) def visit_Group(self, node: Group) -> bool: @@ -93,9 +94,9 @@ def visit_Forced(self, node: Forced) -> bool: class PythonCallMakerVisitor(GrammarVisitor): def __init__(self, parser_generator: ParserGenerator): self.gen = parser_generator - self.cache: Dict[str, Tuple[str, str]] = {} + self.cache: dict[str, tuple[str, str]] = {} - def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: + def visit_NameLeaf(self, node: NameLeaf) -> tuple[str | None, str]: name = node.value if name == "SOFT_KEYWORD": return "soft_keyword", "self.soft_keyword()" @@ -108,31 +109,31 @@ def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: return "_" + name.lower(), f"self.expect({name!r})" return name, f"self.{name}()" - def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: + def visit_StringLeaf(self, node: StringLeaf) -> tuple[str, str]: return "literal", f"self.expect({node.value})" - def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str]: + def visit_NamedItem(self, node: NamedItem) -> tuple[str | None, str]: name, call = self.visit(node.item) if node.name: name = node.name return name, call - def lookahead_call_helper(self, node: Lookahead) -> Tuple[str, str]: + def lookahead_call_helper(self, node: Lookahead) -> tuple[str, str]: name, call = self.visit(node.node) head, tail = call.split("(", 1) assert tail[-1] == ")" tail = tail[:-1] return head, tail - def visit_PositiveLookahead(self, node: PositiveLookahead) -> Tuple[None, str]: + def visit_PositiveLookahead(self, node: PositiveLookahead) -> tuple[None, str]: head, tail = self.lookahead_call_helper(node) return None, f"self.positive_lookahead({head}, {tail})" - def visit_NegativeLookahead(self, node: NegativeLookahead) -> Tuple[None, str]: + def visit_NegativeLookahead(self, node: NegativeLookahead) -> tuple[None, str]: head, tail = self.lookahead_call_helper(node) return None, f"self.negative_lookahead({head}, {tail})" - def visit_Opt(self, node: Opt) -> Tuple[str, str]: + def visit_Opt(self, node: Opt) -> tuple[str, str]: name, call = self.visit(node.node) # Note trailing comma (the call may already have one comma # at the end, for example when rules have both repeat0 and optional @@ -148,7 +149,7 @@ def _generate_artificial_rule_call( prefix: str, call_by_name_func: Callable[[str], str], rule_generation_func: Callable[[], str], - ) -> Tuple[str, str]: + ) -> tuple[str, str]: node_str = f"{node}" key = f"{prefix}_{node_str}" if key in self.cache: @@ -159,7 +160,7 @@ def _generate_artificial_rule_call( self.cache[key] = name, call return self.cache[key] - def visit_Rhs(self, node: Rhs) -> Tuple[str, str]: + def visit_Rhs(self, node: Rhs) -> tuple[str, str]: if len(node.alts) == 1 and len(node.alts[0].items) == 1: return self.visit(node.alts[0].items[0]) @@ -170,7 +171,7 @@ def visit_Rhs(self, node: Rhs) -> Tuple[str, str]: lambda: self.gen.artificial_rule_from_rhs(node), ) - def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: + def visit_Repeat0(self, node: Repeat0) -> tuple[str, str]: return self._generate_artificial_rule_call( node, "repeat0", @@ -178,7 +179,7 @@ def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), ) - def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: + def visit_Repeat1(self, node: Repeat1) -> tuple[str, str]: return self._generate_artificial_rule_call( node, "repeat1", @@ -186,7 +187,7 @@ def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), ) - def visit_Gather(self, node: Gather) -> Tuple[str, str]: + def visit_Gather(self, node: Gather) -> tuple[str, str]: return self._generate_artificial_rule_call( node, "gather", @@ -194,13 +195,13 @@ def visit_Gather(self, node: Gather) -> Tuple[str, str]: lambda: self.gen.artificial_rule_from_gather(node), ) - def visit_Group(self, node: Group) -> Tuple[Optional[str], str]: + def visit_Group(self, node: Group) -> tuple[str | None, str]: return self.visit(node.rhs) - def visit_Cut(self, node: Cut) -> Tuple[str, str]: + def visit_Cut(self, node: Cut) -> tuple[str, str]: return "cut", "True" - def visit_Forced(self, node: Forced) -> Tuple[str, str]: + def visit_Forced(self, node: Forced) -> tuple[str, str]: if isinstance(node.node, Group): _, val = self.visit(node.node.rhs) return "forced", f"self.expect_forced({val}, '''({node.node.rhs!s})''')" @@ -215,10 +216,10 @@ class PythonParserGenerator(ParserGenerator, GrammarVisitor): def __init__( self, grammar: grammar.Grammar, - file: Optional[IO[Text]], - tokens: Set[str] = set(token.tok_name.values()), - location_formatting: Optional[str] = None, - unreachable_formatting: Optional[str] = None, + file: IO[str] | None, + tokens: set[str] = set(token.tok_name.values()), + location_formatting: str | None = None, + unreachable_formatting: str | None = None, ): tokens.add("SOFT_KEYWORD") super().__init__(grammar, tokens, file) @@ -355,7 +356,7 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: if is_loop: self.print(f"children.append({action})") - self.print(f"mark = self._mark()") + self.print("mark = self._mark()") else: if "UNREACHABLE" in action: action = action.replace("UNREACHABLE", self.unreachable_formatting) diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py index da4c9331625..db30fc28346 100644 --- a/Tools/peg_generator/pegen/sccutils.py +++ b/Tools/peg_generator/pegen/sccutils.py @@ -1,11 +1,11 @@ # Adapted from mypy (mypy/build.py) under the MIT license. -from typing import * +from collections.abc import Iterable, Iterator, Set def strongly_connected_components( - vertices: AbstractSet[str], edges: Dict[str, AbstractSet[str]] -) -> Iterator[AbstractSet[str]]: + vertices: Set[str], edges: dict[str, Set[str]] +) -> Iterator[Set[str]]: """Compute Strongly Connected Components of a directed graph. Args: @@ -20,12 +20,12 @@ def strongly_connected_components( From https://code.activestate.com/recipes/578507-strongly-connected-components-of-a-directed-graph/. """ - identified: Set[str] = set() - stack: List[str] = [] - index: Dict[str, int] = {} - boundaries: List[int] = [] + identified: set[str] = set() + stack: list[str] = [] + index: dict[str, int] = {} + boundaries: list[int] = [] - def dfs(v: str) -> Iterator[Set[str]]: + def dfs(v: str) -> Iterator[set[str]]: index[v] = len(stack) stack.append(v) boundaries.append(index[v]) @@ -49,57 +49,9 @@ def dfs(v: str) -> Iterator[Set[str]]: yield from dfs(v) -def topsort( - data: Dict[AbstractSet[str], Set[AbstractSet[str]]] -) -> Iterable[AbstractSet[AbstractSet[str]]]: - """Topological sort. - - Args: - data: A map from SCCs (represented as frozen sets of strings) to - sets of SCCs, its dependencies. NOTE: This data structure - is modified in place -- for normalization purposes, - self-dependencies are removed and entries representing - orphans are added. - - Returns: - An iterator yielding sets of SCCs that have an equivalent - ordering. NOTE: The algorithm doesn't care about the internal - structure of SCCs. - - Example: - Suppose the input has the following structure: - - {A: {B, C}, B: {D}, C: {D}} - - This is normalized to: - - {A: {B, C}, B: {D}, C: {D}, D: {}} - - The algorithm will yield the following values: - - {D} - {B, C} - {A} - - From https://code.activestate.com/recipes/577413-topological-sort/history/1/. - """ - # TODO: Use a faster algorithm? - for k, v in data.items(): - v.discard(k) # Ignore self dependencies. - for item in set.union(*data.values()) - set(data.keys()): - data[item] = set() - while True: - ready = {item for item, dep in data.items() if not dep} - if not ready: - break - yield ready - data = {item: (dep - ready) for item, dep in data.items() if item not in ready} - assert not data, "A cyclic dependency exists amongst %r" % data - - def find_cycles_in_scc( - graph: Dict[str, AbstractSet[str]], scc: AbstractSet[str], start: str -) -> Iterable[List[str]]: + graph: dict[str, Set[str]], scc: Set[str], start: str +) -> Iterable[list[str]]: """Find cycles in SCC emanating from start. Yields lists of the form ['A', 'B', 'C', 'A'], which means there's @@ -117,7 +69,7 @@ def find_cycles_in_scc( assert start in graph # Recursive helper that yields cycles. - def dfs(node: str, path: List[str]) -> Iterator[List[str]]: + def dfs(node: str, path: list[str]) -> Iterator[list[str]]: if node in path: yield path + [node] return diff --git a/Tools/peg_generator/pegen/testutil.py b/Tools/peg_generator/pegen/testutil.py index 0e85b844ef1..46f6c7fec1b 100644 --- a/Tools/peg_generator/pegen/testutil.py +++ b/Tools/peg_generator/pegen/testutil.py @@ -6,7 +6,7 @@ import textwrap import token import tokenize -from typing import IO, Any, Dict, Final, Optional, Type, cast +from typing import IO, Any, Final, cast from pegen.build import compile_c_extension from pegen.c_generator import CParserGenerator @@ -23,19 +23,19 @@ } -def generate_parser(grammar: Grammar) -> Type[Parser]: +def generate_parser(grammar: Grammar) -> type[Parser]: # Generate a parser. out = io.StringIO() genr = PythonParserGenerator(grammar, out) genr.generate("<string>") # Load the generated parser class. - ns: Dict[str, Any] = {} + ns: dict[str, Any] = {} exec(out.getvalue(), ns) return ns["GeneratedParser"] -def run_parser(file: IO[bytes], parser_class: Type[Parser], *, verbose: bool = False) -> Any: +def run_parser(file: IO[bytes], parser_class: type[Parser], *, verbose: bool = False) -> Any: # Run a parser on a file (stream). tokenizer = Tokenizer(tokenize.generate_tokens(file.readline)) # type: ignore[arg-type] # typeshed issue #3515 parser = parser_class(tokenizer, verbose=verbose) @@ -46,7 +46,7 @@ def run_parser(file: IO[bytes], parser_class: Type[Parser], *, verbose: bool = F def parse_string( - source: str, parser_class: Type[Parser], *, dedent: bool = True, verbose: bool = False + source: str, parser_class: type[Parser], *, dedent: bool = True, verbose: bool = False ) -> Any: # Run the parser on a string. if dedent: @@ -55,7 +55,7 @@ def parse_string( return run_parser(file, parser_class, verbose=verbose) # type: ignore[arg-type] # typeshed issue #3515 -def make_parser(source: str) -> Type[Parser]: +def make_parser(source: str) -> type[Parser]: # Combine parse_string() and generate_parser(). grammar = parse_string(source, GrammarParser) return generate_parser(grammar) @@ -86,7 +86,7 @@ def generate_parser_c_extension( grammar: Grammar, path: pathlib.PurePath, debug: bool = False, - library_dir: Optional[str] = None, + library_dir: str | None = None, ) -> Any: """Generate a parser c extension for the given grammar in the given path diff --git a/Tools/peg_generator/pegen/tokenizer.py b/Tools/peg_generator/pegen/tokenizer.py index 7ee49e1432b..fba5bac9517 100644 --- a/Tools/peg_generator/pegen/tokenizer.py +++ b/Tools/peg_generator/pegen/tokenizer.py @@ -1,6 +1,6 @@ import token import tokenize -from typing import Dict, Iterator, List +from collections.abc import Iterator Mark = int # NewType('Mark', int) @@ -8,7 +8,11 @@ def shorttok(tok: tokenize.TokenInfo) -> str: - return "%-25.25s" % f"{tok.start[0]}.{tok.start[1]}: {token.tok_name[tok.type]}:{tok.string!r}" + formatted = ( + f"{tok.start[0]}.{tok.start[1]}: " + f"{token.tok_name[tok.type]}:{tok.string!r}" + ) + return f"{formatted:<25.25}" class Tokenizer: @@ -17,7 +21,7 @@ class Tokenizer: This is pretty tied to Python's syntax. """ - _tokens: List[tokenize.TokenInfo] + _tokens: list[tokenize.TokenInfo] def __init__( self, tokengen: Iterator[tokenize.TokenInfo], *, path: str = "", verbose: bool = False @@ -26,7 +30,7 @@ def __init__( self._tokens = [] self._index = 0 self._verbose = verbose - self._lines: Dict[int, str] = {} + self._lines: dict[int, str] = {} self._path = path if verbose: self.report(False, False) @@ -72,7 +76,7 @@ def get_last_non_whitespace_token(self) -> tokenize.TokenInfo: break return tok - def get_lines(self, line_numbers: List[int]) -> List[str]: + def get_lines(self, line_numbers: list[int]) -> list[str]: """Retrieve source lines corresponding to line numbers.""" if self._lines: lines = self._lines diff --git a/Tools/peg_generator/pegen/validator.py b/Tools/peg_generator/pegen/validator.py index 4699d5712d9..635eb398b41 100644 --- a/Tools/peg_generator/pegen/validator.py +++ b/Tools/peg_generator/pegen/validator.py @@ -1,5 +1,3 @@ -from typing import Optional - from pegen import grammar from pegen.grammar import Alt, GrammarVisitor, Rhs, Rule @@ -11,7 +9,7 @@ class ValidationError(Exception): class GrammarValidator(GrammarVisitor): def __init__(self, grammar: grammar.Grammar) -> None: self.grammar = grammar - self.rulename: Optional[str] = None + self.rulename: str | None = None def validate_rule(self, rulename: str, node: Rule) -> None: self.rulename = rulename diff --git a/Tools/picklebench/README.md b/Tools/picklebench/README.md new file mode 100644 index 00000000000..7d52485c386 --- /dev/null +++ b/Tools/picklebench/README.md @@ -0,0 +1,232 @@ +# Pickle Chunked Reading Benchmark + +This benchmark measures the performance impact of the chunked reading optimization in GH PR #119204 for the pickle module. + +## What This Tests + +The PR adds chunked reading (1MB chunks) to prevent memory exhaustion when unpickling large objects: +- **BINBYTES8** - Large bytes objects (protocol 4+) +- **BINUNICODE8** - Large strings (protocol 4+) +- **BYTEARRAY8** - Large bytearrays (protocol 5) +- **FRAME** - Large frames +- **LONG4** - Large integers +- An antagonistic mode that tests using memory denial of service inducing malicious pickles. + +## Quick Start + +```bash +# Run full benchmark suite (1MiB → 200MiB, takes several minutes) +build/python Tools/picklebench/memory_dos_impact.py + +# Test just a few sizes (quick test: 1, 10, 50 MiB) +build/python Tools/picklebench/memory_dos_impact.py --sizes 1 10 50 + +# Test smaller range for faster results +build/python Tools/picklebench/memory_dos_impact.py --sizes 1 5 10 + +# Output as markdown for reports +build/python Tools/picklebench/memory_dos_impact.py --format markdown > results.md + +# Test with protocol 4 instead of 5 +build/python Tools/picklebench/memory_dos_impact.py --protocol 4 +``` + +**Note:** Sizes are specified in MiB. Use `--sizes 1 2 5` for 1MiB, 2MiB, 5MiB objects. + +## Antagonistic Mode (DoS Protection Test) + +The `--antagonistic` flag tests **malicious pickles** that demonstrate the memory DoS protection: + +```bash +# Quick DoS protection test (claims 10, 50, 100 MB but provides 1KB) +build/python Tools/picklebench/memory_dos_impact.py --antagonistic --sizes 10 50 100 + +# Full DoS test (default: 10, 50, 100, 500, 1000, 5000 MB claimed) +build/python Tools/picklebench/memory_dos_impact.py --antagonistic +``` + +### What This Tests + +Unlike normal benchmarks that test **legitimate pickles**, antagonistic mode tests: +- **Truncated BINBYTES8**: Claims 100MB but provides only 1KB (will fail to unpickle) +- **Truncated BINUNICODE8**: Same for strings +- **Truncated BYTEARRAY8**: Same for bytearrays +- **Sparse memo attacks**: PUT at index 1 billion (would allocate huge array before PR) + +**Key difference:** +- **Normal mode**: Tests real data, shows ~5% time overhead +- **Antagonistic mode**: Tests malicious data, shows ~99% memory savings + +### Expected Results + +``` +100MB Claimed (actual: 1KB) + binbytes8_100MB_claim + Peak memory: 1.00 MB (claimed: 100 MB, saved: 99.00 MB, 99.0%) + Error: UnpicklingError ← Expected! + +Summary: + Average claimed: 126.2 MB + Average peak: 0.54 MB + Average saved: 125.7 MB (99.6% reduction) +Protection Status: ✓ Memory DoS attacks mitigated by chunked reading +``` + +**Before PR**: Would allocate full claimed size (100MB+), potentially crash +**After PR**: Allocates 1MB chunks, fails fast with minimal memory + +This demonstrates the **security improvement** - protection against memory exhaustion attacks. + +## Before/After Comparison + +The benchmark includes an automatic comparison feature that runs the same tests on both a baseline and current Python build. + +### Option 1: Automatic Comparison (Recommended) + +Build both versions, then use `--baseline` to automatically compare: + +```bash +# Build the baseline (main branch without PR) +git checkout main +mkdir -p build-main +cd build-main && ../configure && make -j $(nproc) && cd .. + +# Build the current version (with PR) +git checkout unpickle-overallocate +mkdir -p build +cd build && ../configure && make -j $(nproc) && cd .. + +# Run automatic comparison (quick test with a few sizes) +build/python Tools/picklebench/memory_dos_impact.py \ + --baseline build-main/python \ + --sizes 1 10 50 + +# Full comparison (all default sizes) +build/python Tools/picklebench/memory_dos_impact.py \ + --baseline build-main/python +``` + +The comparison output shows: +- Side-by-side metrics (Current vs Baseline) +- Percentage change for time and memory +- Overall summary statistics + +### Interpreting Comparison Results + +- **Time change**: Small positive % is expected (chunking adds overhead, typically 5-10%) +- **Memory change**: Negative % is good (chunking saves memory, especially for large objects) +- **Trade-off**: Slightly slower but much safer against memory exhaustion attacks + +### Option 2: Manual Comparison + +Save results separately and compare manually: + +```bash +# Baseline results +build-main/python Tools/picklebench/memory_dos_impact.py --format json > baseline.json + +# Current results +build/python Tools/picklebench/memory_dos_impact.py --format json > current.json + +# Manual comparison +diff -y <(jq '.' baseline.json) <(jq '.' current.json) +``` + +## Understanding the Results + +### Critical Sizes + +The default test suite includes: +- **< 1MiB (999,000 bytes)**: No chunking, allocates full size upfront +- **= 1MiB (1,048,576 bytes)**: Threshold, chunking just starts +- **> 1MiB (1,048,577 bytes)**: Chunked reading engaged +- **1, 2, 5, 10MiB**: Show scaling behavior with chunking +- **20, 50, 100, 200MiB**: Stress test large object handling + +**Note:** The full suite may require more than 16GiB of RAM. + +### Key Metrics + +- **Time (mean)**: Average unpickling time - should be similar before/after +- **Time (stdev)**: Consistency - lower is better +- **Peak Memory**: Maximum memory during unpickling - **expected to be LOWER after PR** +- **Pickle Size**: Size of the serialized data on disk + +### Test Types + +| Test | What It Stresses | +|------|------------------| +| `bytes_*` | BINBYTES8 opcode, raw binary data | +| `string_ascii_*` | BINUNICODE8 with simple ASCII | +| `string_utf8_*` | BINUNICODE8 with multibyte UTF-8 (€ chars) | +| `bytearray_*` | BYTEARRAY8 opcode (protocol 5) | +| `list_large_items_*` | Multiple chunked reads in sequence | +| `dict_large_values_*` | Chunking in dict deserialization | +| `nested_*` | Realistic mixed data structures | +| `tuple_*` | Immutable structures | + +## Expected Results + +### Before PR (main branch) +- Single large allocation per object +- Risk of memory exhaustion with malicious pickles + +### After PR (unpickle-overallocate branch) +- Chunked allocation (1MB at a time) +- **Slightly higher CPU time** (multiple allocations + resizing) +- **Significantly lower peak memory** (no large pre-allocation) +- Protection against DoS via memory exhaustion + +## Advanced Usage + +### Test Specific Sizes + +```bash +# Test only 5MiB and 10MiB objects +build/python Tools/picklebench/memory_dos_impact.py --sizes 5 10 + +# Test large objects: 50, 100, 200 MiB +build/python Tools/picklebench/memory_dos_impact.py --sizes 50 100 200 +``` + +### More Iterations for Stable Timing + +```bash +# Run 10 iterations per test for better statistics +build/python Tools/picklebench/memory_dos_impact.py --iterations 10 --sizes 1 10 +``` + +### JSON Output for Analysis + +```bash +# Generate JSON for programmatic analysis +build/python Tools/picklebench/memory_dos_impact.py --format json | python -m json.tool +``` + +## Interpreting Memory Results + +The **peak memory** metric shows the maximum memory allocated during unpickling: + +- **Without chunking**: Allocates full size immediately + - 10MB object → 10MB allocation upfront + +- **With chunking**: Allocates in 1MB chunks, grows geometrically + - 10MB object → starts with 1MB, grows: 2MB, 4MB, 8MB (final: ~10MB total) + - Peak is lower because allocation is incremental + +## Typical Results + +On a system with the PR applied, you should see: + +``` +1.00MiB Test Results + bytes_1.00MiB: ~0.3ms, 1.00MiB peak (just at threshold) + +2.00MiB Test Results + bytes_2.00MiB: ~0.8ms, 2.00MiB peak (chunked: 1MiB → 2MiB) + +10.00MiB Test Results + bytes_10.00MiB: ~3-5ms, 10.00MiB peak (chunked: 1→2→4→8→10 MiB) +``` + +Time overhead is minimal (~10-20% for very large objects), but memory safety is significantly improved. diff --git a/Tools/picklebench/memory_dos_impact.py b/Tools/picklebench/memory_dos_impact.py new file mode 100755 index 00000000000..3bad6586c46 --- /dev/null +++ b/Tools/picklebench/memory_dos_impact.py @@ -0,0 +1,1069 @@ +#!/usr/bin/env python3 +# +# Author: Claude Sonnet 4.5 as driven by gpshead +# +""" +Microbenchmark for pickle module chunked reading performance (GH PR #119204). + +This script generates Python data structures that act as antagonistic load +tests for the chunked reading code introduced to prevent memory exhaustion when +unpickling large objects. + +The PR adds chunked reading (1MB chunks) for: +- BINBYTES8 (large bytes) +- BINUNICODE8 (large strings) +- BYTEARRAY8 (large bytearrays) +- FRAME (large frames) +- LONG4 (large integers) + +Including an antagonistic mode that exercies memory denial of service pickles. + +Usage: + python memory_dos_impact.py --help +""" + +import argparse +import gc +import io +import json +import os +import pickle +import statistics +import struct +import subprocess +import sys +import tempfile +import tracemalloc +from pathlib import Path +from time import perf_counter +from typing import Any, Dict, List, Tuple, Optional + + +# Configuration +MIN_READ_BUF_SIZE = 1 << 20 # 1MB - matches pickle.py _MIN_READ_BUF_SIZE + +# Test sizes in MiB +DEFAULT_SIZES_MIB = [1, 2, 5, 10, 20, 50, 100, 200] + +# Convert to bytes, plus threshold boundary tests +DEFAULT_SIZES = ( + [999_000] # Below 1MiB (no chunking) + + [size * (1 << 20) for size in DEFAULT_SIZES_MIB] # MiB to bytes + + [1_048_577] # Just above 1MiB (minimal chunking overhead) +) +DEFAULT_SIZES.sort() + +# Baseline benchmark configuration +BASELINE_BENCHMARK_TIMEOUT_SECONDS = 600 # 10 minutes + +# Sparse memo attack test configuration +# Format: test_name -> (memo_index, baseline_memory_note) +SPARSE_MEMO_TESTS = { + "sparse_memo_1M": (1_000_000, "~8 MB array"), + "sparse_memo_100M": (100_000_000, "~800 MB array"), + "sparse_memo_1B": (1_000_000_000, "~8 GB array"), +} + + +# Utility functions + +def _extract_size_mb(size_key: str) -> float: + """Extract numeric MiB value from size_key like '10.00MB' or '1.00MiB'. + + Returns 0.0 for non-numeric keys (they'll be sorted last). + """ + try: + return float(size_key.replace('MB', '').replace('MiB', '')) + except ValueError: + return 999999.0 # Put non-numeric keys last + + +def _format_output(results: Dict[str, Dict[str, Any]], format_type: str, is_antagonistic: bool) -> str: + """Format benchmark results according to requested format. + + Args: + results: Benchmark results dictionary + format_type: Output format ('text', 'markdown', or 'json') + is_antagonistic: Whether these are antagonistic (DoS) test results + + Returns: + Formatted output string + """ + if format_type == 'json': + return Reporter.format_json(results) + elif is_antagonistic: + # Antagonistic mode uses specialized formatter for text/markdown + return Reporter.format_antagonistic(results) + elif format_type == 'text': + return Reporter.format_text(results) + elif format_type == 'markdown': + return Reporter.format_markdown(results) + else: + # Default to text format + return Reporter.format_text(results) + + +class AntagonisticGenerator: + """Generate malicious/truncated pickles for DoS protection testing. + + These pickles claim large sizes but provide minimal data, causing them to fail + during unpickling. They demonstrate the memory protection of chunked reading. + """ + + @staticmethod + def truncated_binbytes8(claimed_size: int, actual_size: int = 1024) -> bytes: + """BINBYTES8 claiming `claimed_size` but providing only `actual_size` bytes. + + This will fail with UnpicklingError but demonstrates peak memory usage. + Before PR: Allocates full claimed_size + After PR: Allocates in 1MB chunks, fails fast + """ + return b'\x8e' + struct.pack('<Q', claimed_size) + b'x' * actual_size + + @staticmethod + def truncated_binunicode8(claimed_size: int, actual_size: int = 1024) -> bytes: + """BINUNICODE8 claiming `claimed_size` but providing only `actual_size` bytes.""" + return b'\x8d' + struct.pack('<Q', claimed_size) + b'x' * actual_size + + @staticmethod + def truncated_bytearray8(claimed_size: int, actual_size: int = 1024) -> bytes: + """BYTEARRAY8 claiming `claimed_size` but providing only `actual_size` bytes.""" + return b'\x96' + struct.pack('<Q', claimed_size) + b'x' * actual_size + + @staticmethod + def truncated_frame(claimed_size: int) -> bytes: + """FRAME claiming `claimed_size` but providing minimal data.""" + return b'\x95' + struct.pack('<Q', claimed_size) + b'N.' + + @staticmethod + def sparse_memo_attack(index: int) -> bytes: + """LONG_BINPUT with huge sparse index. + + Before PR: Tries to allocate array with `index` slots (OOM) + After PR: Uses dict-based memo for sparse indices + """ + return (b'(]r' + struct.pack('<I', index & 0xFFFFFFFF) + + b'j' + struct.pack('<I', index & 0xFFFFFFFF) + b't.') + + @staticmethod + def multi_claim_attack(count: int, size_each: int) -> bytes: + """Multiple BINBYTES8 claims in sequence. + + Tests that multiple large claims don't accumulate memory. + """ + data = b'(' # MARK + for _ in range(count): + data += b'\x8e' + struct.pack('<Q', size_each) + b'x' * 1024 + data += b't.' # TUPLE + STOP + return data + + +class DataGenerator: + """Generate various types of large data structures for pickle testing.""" + + @staticmethod + def large_bytes(size: int) -> bytes: + """Generate random bytes of specified size.""" + return os.urandom(size) + + @staticmethod + def large_string_ascii(size: int) -> str: + """Generate ASCII string of specified size.""" + return 'x' * size + + @staticmethod + def large_string_multibyte(size: int) -> str: + """Generate multibyte UTF-8 string (3 bytes per char for €).""" + # Each € is 3 bytes in UTF-8 + return '€' * (size // 3) + + @staticmethod + def large_bytearray(size: int) -> bytearray: + """Generate bytearray of specified size.""" + return bytearray(os.urandom(size)) + + @staticmethod + def list_of_large_bytes(item_size: int, count: int) -> List[bytes]: + """Generate list containing multiple large bytes objects.""" + return [os.urandom(item_size) for _ in range(count)] + + @staticmethod + def dict_with_large_values(value_size: int, count: int) -> Dict[str, bytes]: + """Generate dict with large bytes values.""" + return { + f'key_{i}': os.urandom(value_size) + for i in range(count) + } + + @staticmethod + def nested_structure(size: int) -> Dict[str, Any]: + """Generate nested structure with various large objects.""" + chunk_size = size // 4 + return { + 'name': 'test_object', + 'data': { + 'bytes': os.urandom(chunk_size), + 'string': 's' * chunk_size, + 'bytearray': bytearray(b'b' * chunk_size), + }, + 'items': [os.urandom(chunk_size // 4) for _ in range(4)], + 'metadata': { + 'size': size, + 'type': 'nested', + }, + } + + @staticmethod + def tuple_of_large_objects(size: int) -> Tuple[bytes, str, bytearray]: + """Generate tuple with large objects (immutable, different pickle path).""" + chunk_size = size // 3 + return ( + os.urandom(chunk_size), + 'x' * chunk_size, + bytearray(b'y' * chunk_size), + ) + + +class PickleBenchmark: + """Benchmark pickle unpickling performance and memory usage.""" + + def __init__(self, obj: Any, protocol: int = 5, iterations: int = 3): + self.obj = obj + self.protocol = protocol + self.iterations = iterations + self.pickle_data = pickle.dumps(obj, protocol=protocol) + self.pickle_size = len(self.pickle_data) + + def benchmark_time(self) -> Dict[str, float]: + """Measure unpickling time over multiple iterations.""" + times = [] + + for _ in range(self.iterations): + start = perf_counter() + result = pickle.loads(self.pickle_data) + elapsed = perf_counter() - start + times.append(elapsed) + + # Verify correctness (first iteration only) + if len(times) == 1: + if result != self.obj: + raise ValueError("Unpickled object doesn't match original!") + + return { + 'mean': statistics.mean(times), + 'median': statistics.median(times), + 'stdev': statistics.stdev(times) if len(times) > 1 else 0.0, + 'min': min(times), + 'max': max(times), + } + + def benchmark_memory(self) -> int: + """Measure peak memory usage during unpickling.""" + tracemalloc.start() + + # Warmup + pickle.loads(self.pickle_data) + + # Actual measurement + gc.collect() + tracemalloc.reset_peak() + result = pickle.loads(self.pickle_data) + current, peak = tracemalloc.get_traced_memory() + + tracemalloc.stop() + + # Verify correctness + if result != self.obj: + raise ValueError("Unpickled object doesn't match original!") + + return peak + + def run_all(self) -> Dict[str, Any]: + """Run all benchmarks and return comprehensive results.""" + time_stats = self.benchmark_time() + peak_memory = self.benchmark_memory() + + return { + 'pickle_size_bytes': self.pickle_size, + 'pickle_size_mb': self.pickle_size / (1 << 20), + 'protocol': self.protocol, + 'time': time_stats, + 'memory_peak_bytes': peak_memory, + 'memory_peak_mb': peak_memory / (1 << 20), + 'iterations': self.iterations, + } + + +class AntagonisticBenchmark: + """Benchmark antagonistic/malicious pickles that demonstrate DoS protection. + + These pickles are designed to FAIL unpickling, but we measure peak memory + usage before the failure to demonstrate the memory protection. + """ + + def __init__(self, pickle_data: bytes, name: str): + self.pickle_data = pickle_data + self.name = name + + def measure_peak_memory(self, expect_success: bool = False) -> Dict[str, Any]: + """Measure peak memory when attempting to unpickle antagonistic data. + + Args: + expect_success: If True, test expects successful unpickling (e.g., sparse memo). + If False, test expects failure (e.g., truncated data). + """ + tracemalloc.start() + gc.collect() + tracemalloc.reset_peak() + + error_type = None + error_msg = None + succeeded = False + + try: + result = pickle.loads(self.pickle_data) + succeeded = True + if expect_success: + error_type = "Success (expected)" + else: + error_type = "WARNING: Expected failure but succeeded" + except (pickle.UnpicklingError, EOFError, ValueError, OverflowError) as e: + if expect_success: + error_type = f"UNEXPECTED FAILURE: {type(e).__name__}" + error_msg = str(e)[:100] + else: + # Expected failure for truncated data tests + error_type = type(e).__name__ + error_msg = str(e)[:100] + + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + return { + 'test_name': self.name, + 'peak_memory_bytes': peak, + 'peak_memory_mb': peak / (1 << 20), + 'error_type': error_type, + 'error_msg': error_msg, + 'pickle_size_bytes': len(self.pickle_data), + 'expected_outcome': 'success' if expect_success else 'failure', + 'succeeded': succeeded, + } + + +class AntagonisticTestSuite: + """Manage a suite of antagonistic (DoS protection) tests.""" + + # Default sizes in MB to claim (will provide only 1KB actual data) + DEFAULT_ANTAGONISTIC_SIZES_MB = [10, 50, 100, 500, 1000, 5000] + + def __init__(self, claimed_sizes_mb: List[int]): + self.claimed_sizes_mb = claimed_sizes_mb + + def _run_truncated_test( + self, + test_type: str, + generator_func, + claimed_bytes: int, + claimed_mb: int, + size_key: str, + all_results: Dict[str, Dict[str, Any]] + ) -> None: + """Run a single truncated data test and store results. + + Args: + test_type: Type identifier (e.g., 'binbytes8', 'binunicode8') + generator_func: Function to generate malicious pickle data + claimed_bytes: Size claimed in the pickle (bytes) + claimed_mb: Size claimed in the pickle (MB) + size_key: Result key for this size (e.g., '10MB') + all_results: Dictionary to store results in + """ + test_name = f"{test_type}_{size_key}_claim" + data = generator_func(claimed_bytes) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=False) + result['claimed_mb'] = claimed_mb + all_results[size_key][test_name] = result + + def run_all_tests(self) -> Dict[str, Dict[str, Any]]: + """Run comprehensive antagonistic test suite.""" + all_results = {} + + for claimed_mb in self.claimed_sizes_mb: + claimed_bytes = claimed_mb << 20 + size_key = f"{claimed_mb}MB" + all_results[size_key] = {} + + # Run truncated data tests (expect failure) + self._run_truncated_test('binbytes8', AntagonisticGenerator.truncated_binbytes8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('binunicode8', AntagonisticGenerator.truncated_binunicode8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('bytearray8', AntagonisticGenerator.truncated_bytearray8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('frame', AntagonisticGenerator.truncated_frame, + claimed_bytes, claimed_mb, size_key, all_results) + + # Test 5: Sparse memo (expect success - dict-based memo works!) + all_results["Sparse Memo (Success Expected)"] = {} + for test_name, (index, baseline_note) in SPARSE_MEMO_TESTS.items(): + data = AntagonisticGenerator.sparse_memo_attack(index) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=True) + result['claimed_mb'] = "N/A" + result['baseline_note'] = f"Without PR: {baseline_note}" + all_results["Sparse Memo (Success Expected)"][test_name] = result + + # Test 6: Multi-claim attack (expect failure) + test_name = "multi_claim_10x100MB" + data = AntagonisticGenerator.multi_claim_attack(10, 100 << 20) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=False) + result['claimed_mb'] = 1000 # 10 * 100MB + all_results["Multi-Claim (Failure Expected)"] = {test_name: result} + + return all_results + + +class TestSuite: + """Manage a suite of benchmark tests.""" + + def __init__(self, sizes: List[int], protocol: int = 5, iterations: int = 3): + self.sizes = sizes + self.protocol = protocol + self.iterations = iterations + self.results = {} + + def run_test(self, name: str, obj: Any) -> Dict[str, Any]: + """Run benchmark for a single test object.""" + bench = PickleBenchmark(obj, self.protocol, self.iterations) + results = bench.run_all() + results['test_name'] = name + results['object_type'] = type(obj).__name__ + return results + + def run_all_tests(self) -> Dict[str, Dict[str, Any]]: + """Run comprehensive test suite across all sizes and types.""" + all_results = {} + + for size in self.sizes: + size_key = f"{size / (1 << 20):.2f}MB" + all_results[size_key] = {} + + # Test 1: Large bytes object (BINBYTES8) + test_name = f"bytes_{size_key}" + obj = DataGenerator.large_bytes(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 2: Large ASCII string (BINUNICODE8) + test_name = f"string_ascii_{size_key}" + obj = DataGenerator.large_string_ascii(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 3: Large multibyte UTF-8 string + if size >= 3: + test_name = f"string_utf8_{size_key}" + obj = DataGenerator.large_string_multibyte(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 4: Large bytearray (BYTEARRAY8, protocol 5) + if self.protocol >= 5: + test_name = f"bytearray_{size_key}" + obj = DataGenerator.large_bytearray(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 5: List of large objects (repeated chunking) + if size >= MIN_READ_BUF_SIZE * 2: + test_name = f"list_large_items_{size_key}" + item_size = size // 5 + obj = DataGenerator.list_of_large_bytes(item_size, 5) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 6: Dict with large values + if size >= MIN_READ_BUF_SIZE * 2: + test_name = f"dict_large_values_{size_key}" + value_size = size // 3 + obj = DataGenerator.dict_with_large_values(value_size, 3) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 7: Nested structure + if size >= MIN_READ_BUF_SIZE: + test_name = f"nested_{size_key}" + obj = DataGenerator.nested_structure(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 8: Tuple (immutable) + if size >= 3: + test_name = f"tuple_{size_key}" + obj = DataGenerator.tuple_of_large_objects(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + return all_results + + +class Comparator: + """Compare benchmark results between current and baseline interpreters.""" + + @staticmethod + def _extract_json_from_output(output: str) -> Dict[str, Dict[str, Any]]: + """Extract JSON data from subprocess output. + + Skips any print statements before the JSON output and parses the JSON. + + Args: + output: Raw stdout from subprocess + + Returns: + Parsed JSON as dictionary + + Raises: + SystemExit: If JSON cannot be found or parsed + """ + output_lines = output.strip().split('\n') + json_start = -1 + for i, line in enumerate(output_lines): + if line.strip().startswith('{'): + json_start = i + break + + if json_start == -1: + print("Error: Could not find JSON output from baseline", file=sys.stderr) + sys.exit(1) + + json_output = '\n'.join(output_lines[json_start:]) + try: + return json.loads(json_output) + except json.JSONDecodeError as e: + print(f"Error: Could not parse baseline JSON output: {e}", file=sys.stderr) + sys.exit(1) + + @staticmethod + def run_baseline_benchmark(baseline_python: str, args: argparse.Namespace) -> Dict[str, Dict[str, Any]]: + """Run the benchmark using the baseline Python interpreter.""" + # Build command to run this script with baseline Python + cmd = [ + baseline_python, + __file__, + '--format', 'json', + '--protocol', str(args.protocol), + '--iterations', str(args.iterations), + ] + + if args.sizes is not None: + cmd.extend(['--sizes'] + [str(s) for s in args.sizes]) + + if args.antagonistic: + cmd.append('--antagonistic') + + print(f"\nRunning baseline benchmark with: {baseline_python}") + print(f"Command: {' '.join(cmd)}\n") + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=BASELINE_BENCHMARK_TIMEOUT_SECONDS, + ) + + if result.returncode != 0: + print(f"Error running baseline benchmark:", file=sys.stderr) + print(result.stderr, file=sys.stderr) + sys.exit(1) + + # Extract and parse JSON from output + return Comparator._extract_json_from_output(result.stdout) + + except subprocess.TimeoutExpired: + print("Error: Baseline benchmark timed out", file=sys.stderr) + sys.exit(1) + + @staticmethod + def calculate_change(baseline_value: float, current_value: float) -> float: + """Calculate percentage change from baseline to current.""" + if baseline_value == 0: + return 0.0 + return ((current_value - baseline_value) / baseline_value) * 100 + + @staticmethod + def format_comparison( + current_results: Dict[str, Dict[str, Any]], + baseline_results: Dict[str, Dict[str, Any]] + ) -> str: + """Format comparison results as readable text.""" + lines = [] + lines.append("=" * 100) + lines.append("Pickle Unpickling Benchmark Comparison") + lines.append("=" * 100) + lines.append("") + lines.append("Legend: Current vs Baseline | % Change (+ is slower/more memory, - is faster/less memory)") + lines.append("") + + # Sort size keys numerically + for size_key in sorted(current_results.keys(), key=_extract_size_mb): + if size_key not in baseline_results: + continue + + lines.append(f"\n{size_key} Comparison") + lines.append("-" * 100) + + current_tests = current_results[size_key] + baseline_tests = baseline_results[size_key] + + for test_name in sorted(current_tests.keys()): + if test_name not in baseline_tests: + continue + + curr = current_tests[test_name] + base = baseline_tests[test_name] + + time_change = Comparator.calculate_change( + base['time']['mean'], curr['time']['mean'] + ) + mem_change = Comparator.calculate_change( + base['memory_peak_mb'], curr['memory_peak_mb'] + ) + + lines.append(f"\n {curr['test_name']}") + lines.append(f" Time: {curr['time']['mean']*1000:6.2f}ms vs {base['time']['mean']*1000:6.2f}ms | " + f"{time_change:+6.1f}%") + lines.append(f" Memory: {curr['memory_peak_mb']:6.2f}MB vs {base['memory_peak_mb']:6.2f}MB | " + f"{mem_change:+6.1f}%") + + lines.append("\n" + "=" * 100) + lines.append("\nSummary:") + + # Calculate overall statistics + time_changes = [] + mem_changes = [] + + for size_key in current_results.keys(): + if size_key not in baseline_results: + continue + for test_name in current_results[size_key].keys(): + if test_name not in baseline_results[size_key]: + continue + curr = current_results[size_key][test_name] + base = baseline_results[size_key][test_name] + + time_changes.append(Comparator.calculate_change( + base['time']['mean'], curr['time']['mean'] + )) + mem_changes.append(Comparator.calculate_change( + base['memory_peak_mb'], curr['memory_peak_mb'] + )) + + if time_changes: + lines.append(f" Time change: mean={statistics.mean(time_changes):+.1f}%, " + f"median={statistics.median(time_changes):+.1f}%") + if mem_changes: + lines.append(f" Memory change: mean={statistics.mean(mem_changes):+.1f}%, " + f"median={statistics.median(mem_changes):+.1f}%") + + lines.append("=" * 100) + return "\n".join(lines) + + @staticmethod + def format_antagonistic_comparison( + current_results: Dict[str, Dict[str, Any]], + baseline_results: Dict[str, Dict[str, Any]] + ) -> str: + """Format antagonistic benchmark comparison results.""" + lines = [] + lines.append("=" * 100) + lines.append("Antagonistic Pickle Benchmark Comparison (Memory DoS Protection)") + lines.append("=" * 100) + lines.append("") + lines.append("Legend: Current vs Baseline | Memory Change (- is better, shows memory saved)") + lines.append("") + lines.append("This compares TWO types of DoS protection:") + lines.append(" 1. Truncated data → Baseline allocates full claimed size, Current uses chunked reading") + lines.append(" 2. Sparse memo → Baseline uses huge arrays, Current uses dict-based memo") + lines.append("") + + # Track statistics + truncated_memory_changes = [] + sparse_memory_changes = [] + + # Sort size keys numerically + for size_key in sorted(current_results.keys(), key=_extract_size_mb): + if size_key not in baseline_results: + continue + + lines.append(f"\n{size_key} Comparison") + lines.append("-" * 100) + + current_tests = current_results[size_key] + baseline_tests = baseline_results[size_key] + + for test_name in sorted(current_tests.keys()): + if test_name not in baseline_tests: + continue + + curr = current_tests[test_name] + base = baseline_tests[test_name] + + curr_peak_mb = curr['peak_memory_mb'] + base_peak_mb = base['peak_memory_mb'] + expected_outcome = curr.get('expected_outcome', 'failure') + + mem_change = Comparator.calculate_change(base_peak_mb, curr_peak_mb) + mem_saved_mb = base_peak_mb - curr_peak_mb + + lines.append(f"\n {curr['test_name']}") + lines.append(f" Memory: {curr_peak_mb:6.2f}MB vs {base_peak_mb:6.2f}MB | " + f"{mem_change:+6.1f}% ({mem_saved_mb:+.2f}MB saved)") + + # Track based on test type + if expected_outcome == 'success': + sparse_memory_changes.append(mem_change) + if curr.get('baseline_note'): + lines.append(f" Note: {curr['baseline_note']}") + else: + truncated_memory_changes.append(mem_change) + claimed_mb = curr.get('claimed_mb', 'N/A') + if claimed_mb != 'N/A': + lines.append(f" Claimed: {claimed_mb:,}MB") + + # Show status + curr_status = curr.get('error_type', 'Unknown') + base_status = base.get('error_type', 'Unknown') + if curr_status != base_status: + lines.append(f" Status: {curr_status} (baseline: {base_status})") + else: + lines.append(f" Status: {curr_status}") + + lines.append("\n" + "=" * 100) + lines.append("\nSummary:") + lines.append("") + + if truncated_memory_changes: + lines.append(" Truncated Data Protection (chunked reading):") + lines.append(f" Mean memory change: {statistics.mean(truncated_memory_changes):+.1f}%") + lines.append(f" Median memory change: {statistics.median(truncated_memory_changes):+.1f}%") + avg_change = statistics.mean(truncated_memory_changes) + if avg_change < -50: + lines.append(f" Result: ✓ Dramatic memory reduction ({avg_change:.1f}%) - DoS protection working!") + elif avg_change < 0: + lines.append(f" Result: ✓ Memory reduced ({avg_change:.1f}%)") + else: + lines.append(f" Result: ⚠ Memory increased ({avg_change:.1f}%) - unexpected!") + lines.append("") + + if sparse_memory_changes: + lines.append(" Sparse Memo Protection (dict-based memo):") + lines.append(f" Mean memory change: {statistics.mean(sparse_memory_changes):+.1f}%") + lines.append(f" Median memory change: {statistics.median(sparse_memory_changes):+.1f}%") + avg_change = statistics.mean(sparse_memory_changes) + if avg_change < -50: + lines.append(f" Result: ✓ Dramatic memory reduction ({avg_change:.1f}%) - Dict optimization working!") + elif avg_change < 0: + lines.append(f" Result: ✓ Memory reduced ({avg_change:.1f}%)") + else: + lines.append(f" Result: ⚠ Memory increased ({avg_change:.1f}%) - unexpected!") + + lines.append("") + lines.append("=" * 100) + return "\n".join(lines) + + +class Reporter: + """Format and display benchmark results.""" + + @staticmethod + def format_text(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as readable text.""" + lines = [] + lines.append("=" * 80) + lines.append("Pickle Unpickling Benchmark Results") + lines.append("=" * 80) + lines.append("") + + for size_key, tests in results.items(): + lines.append(f"\n{size_key} Test Results") + lines.append("-" * 80) + + for test_name, data in tests.items(): + lines.append(f"\n Test: {data['test_name']}") + lines.append(f" Type: {data['object_type']}") + lines.append(f" Pickle size: {data['pickle_size_mb']:.2f} MB") + lines.append(f" Time (mean): {data['time']['mean']*1000:.2f} ms") + lines.append(f" Time (stdev): {data['time']['stdev']*1000:.2f} ms") + lines.append(f" Peak memory: {data['memory_peak_mb']:.2f} MB") + lines.append(f" Protocol: {data['protocol']}") + + lines.append("\n" + "=" * 80) + return "\n".join(lines) + + @staticmethod + def format_markdown(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as markdown table.""" + lines = [] + lines.append("# Pickle Unpickling Benchmark Results\n") + + for size_key, tests in results.items(): + lines.append(f"## {size_key}\n") + lines.append("| Test | Type | Pickle Size (MB) | Time (ms) | Stdev (ms) | Peak Memory (MB) |") + lines.append("|------|------|------------------|-----------|------------|------------------|") + + for test_name, data in tests.items(): + lines.append( + f"| {data['test_name']} | " + f"{data['object_type']} | " + f"{data['pickle_size_mb']:.2f} | " + f"{data['time']['mean']*1000:.2f} | " + f"{data['time']['stdev']*1000:.2f} | " + f"{data['memory_peak_mb']:.2f} |" + ) + lines.append("") + + return "\n".join(lines) + + @staticmethod + def format_json(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as JSON.""" + import json + return json.dumps(results, indent=2) + + @staticmethod + def format_antagonistic(results: Dict[str, Dict[str, Any]]) -> str: + """Format antagonistic benchmark results.""" + lines = [] + lines.append("=" * 100) + lines.append("Antagonistic Pickle Benchmark (Memory DoS Protection Test)") + lines.append("=" * 100) + lines.append("") + lines.append("This benchmark tests TWO types of DoS protection:") + lines.append(" 1. Truncated data attacks → Expect FAILURE with minimal memory before failure") + lines.append(" 2. Sparse memo attacks → Expect SUCCESS with dict-based memo (vs huge array)") + lines.append("") + + # Sort size keys numerically + for size_key in sorted(results.keys(), key=_extract_size_mb): + tests = results[size_key] + + # Determine test type from first test + if tests: + first_test = next(iter(tests.values())) + expected_outcome = first_test.get('expected_outcome', 'failure') + claimed_mb = first_test.get('claimed_mb', 'N/A') + + # Header varies by test type + if "Sparse Memo" in size_key: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + elif "Multi-Claim" in size_key: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + elif claimed_mb != 'N/A': + lines.append(f"\n{size_key} Claimed (actual: 1KB) - Expect Failure") + lines.append("-" * 100) + else: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + + for test_name, data in tests.items(): + peak_mb = data['peak_memory_mb'] + claimed = data.get('claimed_mb', 'N/A') + expected_outcome = data.get('expected_outcome', 'failure') + succeeded = data.get('succeeded', False) + baseline_note = data.get('baseline_note', '') + + lines.append(f" {data['test_name']}") + + # Format output based on test type + if expected_outcome == 'success': + # Sparse memo test - show success with dict + status_icon = "✓" if succeeded else "✗" + lines.append(f" Peak memory: {peak_mb:8.2f} MB {status_icon}") + lines.append(f" Status: {data['error_type']}") + if baseline_note: + lines.append(f" {baseline_note}") + else: + # Truncated data test - show savings before failure + if claimed != 'N/A': + saved_mb = claimed - peak_mb + savings_pct = (saved_mb / claimed * 100) if claimed > 0 else 0 + lines.append(f" Peak memory: {peak_mb:8.2f} MB (claimed: {claimed:,} MB, saved: {saved_mb:.2f} MB, {savings_pct:.1f}%)") + else: + lines.append(f" Peak memory: {peak_mb:8.2f} MB") + lines.append(f" Status: {data['error_type']}") + + lines.append("\n" + "=" * 100) + + # Calculate statistics by test type + truncated_claimed = 0 + truncated_peak = 0 + truncated_count = 0 + + sparse_peak_total = 0 + sparse_count = 0 + + for size_key, tests in results.items(): + for test_name, data in tests.items(): + expected_outcome = data.get('expected_outcome', 'failure') + + if expected_outcome == 'failure': + # Truncated data test + claimed = data.get('claimed_mb', 0) + if claimed != 'N/A' and claimed > 0: + truncated_claimed += claimed + truncated_peak += data['peak_memory_mb'] + truncated_count += 1 + else: + # Sparse memo test + sparse_peak_total += data['peak_memory_mb'] + sparse_count += 1 + + lines.append("\nSummary:") + lines.append("") + + if truncated_count > 0: + avg_claimed = truncated_claimed / truncated_count + avg_peak = truncated_peak / truncated_count + avg_saved = avg_claimed - avg_peak + avg_savings_pct = (avg_saved / avg_claimed * 100) if avg_claimed > 0 else 0 + + lines.append(" Truncated Data Protection (chunked reading):") + lines.append(f" Average claimed: {avg_claimed:,.1f} MB") + lines.append(f" Average peak: {avg_peak:,.2f} MB") + lines.append(f" Average saved: {avg_saved:,.2f} MB ({avg_savings_pct:.1f}% reduction)") + lines.append(f" Status: ✓ Fails fast with minimal memory") + lines.append("") + + if sparse_count > 0: + avg_sparse_peak = sparse_peak_total / sparse_count + lines.append(" Sparse Memo Protection (dict-based memo):") + lines.append(f" Average peak: {avg_sparse_peak:,.2f} MB") + lines.append(f" Status: ✓ Succeeds with dict (vs GB-sized arrays without PR)") + lines.append(f" Note: Compare with --baseline to see actual memory savings") + + lines.append("") + lines.append("=" * 100) + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark pickle unpickling performance for large objects" + ) + parser.add_argument( + '--sizes', + type=int, + nargs='+', + default=None, + metavar='MiB', + help=f'Object sizes to test in MiB (default: {DEFAULT_SIZES_MIB})' + ) + parser.add_argument( + '--protocol', + type=int, + default=5, + choices=[0, 1, 2, 3, 4, 5], + help='Pickle protocol version (default: 5)' + ) + parser.add_argument( + '--iterations', + type=int, + default=3, + help='Number of benchmark iterations (default: 3)' + ) + parser.add_argument( + '--format', + choices=['text', 'markdown', 'json'], + default='text', + help='Output format (default: text)' + ) + parser.add_argument( + '--baseline', + type=str, + metavar='PYTHON', + help='Path to baseline Python interpreter for comparison (e.g., ../main-build/python)' + ) + parser.add_argument( + '--antagonistic', + action='store_true', + help='Run antagonistic/malicious pickle tests (DoS protection benchmark)' + ) + + args = parser.parse_args() + + # Handle antagonistic mode + if args.antagonistic: + # Antagonistic mode uses claimed sizes in MB, not actual data sizes + if args.sizes is None: + claimed_sizes_mb = AntagonisticTestSuite.DEFAULT_ANTAGONISTIC_SIZES_MB + else: + claimed_sizes_mb = args.sizes + + print(f"Running ANTAGONISTIC pickle benchmark (DoS protection test)...") + print(f"Claimed sizes: {claimed_sizes_mb} MiB (actual data: 1KB each)") + print(f"NOTE: These pickles will FAIL to unpickle (expected)") + print() + + # Run antagonistic benchmark suite + suite = AntagonisticTestSuite(claimed_sizes_mb) + results = suite.run_all_tests() + + # Format and display results + if args.baseline: + # Verify baseline Python exists + baseline_path = Path(args.baseline) + if not baseline_path.exists(): + print(f"Error: Baseline Python not found: {args.baseline}", file=sys.stderr) + return 1 + + # Run baseline benchmark + baseline_results = Comparator.run_baseline_benchmark(args.baseline, args) + + # Show comparison + comparison_output = Comparator.format_antagonistic_comparison(results, baseline_results) + print(comparison_output) + else: + # Format and display results + output = _format_output(results, args.format, is_antagonistic=True) + print(output) + + else: + # Normal mode: legitimate pickle benchmarks + # Convert sizes from MiB to bytes + if args.sizes is None: + sizes_bytes = DEFAULT_SIZES + else: + sizes_bytes = [size * (1 << 20) for size in args.sizes] + + print(f"Running pickle benchmark with protocol {args.protocol}...") + print(f"Test sizes: {[f'{s/(1<<20):.2f}MiB' for s in sizes_bytes]}") + print(f"Iterations per test: {args.iterations}") + print() + + # Run benchmark suite + suite = TestSuite(sizes_bytes, args.protocol, args.iterations) + results = suite.run_all_tests() + + # If baseline comparison requested, run baseline and compare + if args.baseline: + # Verify baseline Python exists + baseline_path = Path(args.baseline) + if not baseline_path.exists(): + print(f"Error: Baseline Python not found: {args.baseline}", file=sys.stderr) + return 1 + + # Run baseline benchmark + baseline_results = Comparator.run_baseline_benchmark(args.baseline, args) + + # Show comparison + comparison_output = Comparator.format_comparison(results, baseline_results) + print(comparison_output) + + else: + # Format and display results + output = _format_output(results, args.format, is_antagonistic=False) + print(output) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 5bf180bb30a..73236767374 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.15 +mypy==1.17.1 # needed for peg_generator: -types-psutil==6.0.0.20240901 -types-setuptools==74.0.0.20240831 +types-psutil==7.0.0.20250801 +types-setuptools==80.9.0.20250801 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 66898885c0a..e5deac497fb 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.111.2 +hypothesis==6.135.26 diff --git a/Tools/scripts/README b/Tools/scripts/README index a078bfbf662..4e52cda38e8 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -1,8 +1,6 @@ This directory contains a collection of executable Python scripts that are useful while building, extending or managing Python. -checkpip.py Checks the version of the projects bundled in ensurepip - are the latest available combinerefs.py A helper for analyzing PYTHONDUMPREFS output divmod_threshold.py Determine threshold for switching from longobject.c divmod to _pylong.int_divmod() diff --git a/Tools/scripts/checkpip.py b/Tools/scripts/checkpip.py deleted file mode 100755 index a4a9ddfa6f3..00000000000 --- a/Tools/scripts/checkpip.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 -""" -Checks that the version of the projects bundled in ensurepip are the latest -versions available. -""" -import ensurepip -import json -import urllib.request -import sys - - -def main(): - outofdate = False - - for project, version in ensurepip._PROJECTS: - data = json.loads(urllib.request.urlopen( - "https://pypi.org/pypi/{}/json".format(project), - cadefault=True, - ).read().decode("utf8")) - upstream_version = data["info"]["version"] - - if version != upstream_version: - outofdate = True - print("The latest version of {} on PyPI is {}, but ensurepip " - "has {}".format(project, upstream_version, version)) - - if outofdate: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 68cfad3f92c..905af9dcfd8 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -492,7 +492,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ): (trace_too_long, attempts), Doc( "Trace too short", - "A potential trace is abandoned because it it too short.", + "A potential trace is abandoned because it is too short.", ): (trace_too_short, attempts), Doc( "Inner loop found", "A trace is truncated because it has an inner loop" diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index b1a5df91901..56976de4998 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -1,12 +1,12 @@ #!./python -"""Run Python tests against multiple installations of OpenSSL and LibreSSL +"""Run Python tests against multiple installations of cryptography libraries The script - (1) downloads OpenSSL / LibreSSL tar bundle + (1) downloads the tar bundle (2) extracts it to ./src - (3) compiles OpenSSL / LibreSSL - (4) installs OpenSSL / LibreSSL into ../multissl/$LIB/$VERSION/ + (3) compiles the relevant library + (4) installs that library into ../multissl/$LIB/$VERSION/ (5) forces a recompilation of Python modules using the header and library files from ../multissl/$LIB/$VERSION/ (6) runs Python's test suite @@ -44,14 +44,15 @@ OPENSSL_OLD_VERSIONS = [ "1.1.1w", + "3.1.8", ] OPENSSL_RECENT_VERSIONS = [ - "3.0.16", - "3.1.8", - "3.2.4", - "3.3.3", - "3.4.1", + "3.0.18", + "3.2.6", + "3.3.5", + "3.4.3", + "3.5.4", # See make_ssl_data.py for notes on adding a new version. ] @@ -61,6 +62,10 @@ LIBRESSL_RECENT_VERSIONS = [ ] +AWSLC_RECENT_VERSIONS = [ + "1.55.0", +] + # store files in ../multissl HERE = os.path.dirname(os.path.abspath(__file__)) PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..')) @@ -70,9 +75,8 @@ parser = argparse.ArgumentParser( prog='multissl', description=( - "Run CPython tests with multiple OpenSSL and LibreSSL " - "versions." - ) + "Run CPython tests with multiple cryptography libraries/versions." + ), ) parser.add_argument( '--debug', @@ -102,6 +106,14 @@ "OpenSSL and LibreSSL versions are given." ).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS) ) +parser.add_argument( + '--awslc', + nargs='+', + default=(), + help=( + "AWS-LC versions, defaults to '{}' if no crypto library versions are given." + ).format(AWSLC_RECENT_VERSIONS) +) parser.add_argument( '--tests', nargs='*', @@ -111,7 +123,7 @@ parser.add_argument( '--base-directory', default=MULTISSL_DIR, - help="Base directory for OpenSSL / LibreSSL sources and builds." + help="Base directory for crypto library sources and builds." ) parser.add_argument( '--no-network', @@ -124,8 +136,8 @@ choices=['library', 'modules', 'tests'], default='tests', help=( - "Which steps to perform. 'library' downloads and compiles OpenSSL " - "or LibreSSL. 'module' also compiles Python modules. 'tests' builds " + "Which steps to perform. 'library' downloads and compiles a crypto" + "library. 'module' also compiles Python modules. 'tests' builds " "all and runs the test suite." ) ) @@ -294,7 +306,7 @@ def _unpack_src(self): raise ValueError(member.name, base) member.name = member.name[len(base):].lstrip('/') log.info("Unpacking files to {}".format(self.build_dir)) - tf.extractall(self.build_dir, members) + tf.extractall(self.build_dir, members, filter='data') def _build_src(self, config_args=()): """Now build openssl""" @@ -453,6 +465,34 @@ class BuildLibreSSL(AbstractBuilder): build_template = "libressl-{}" +class BuildAWSLC(AbstractBuilder): + library = "AWS-LC" + url_templates = ( + "https://github.com/aws/aws-lc/archive/refs/tags/v{v}.tar.gz", + ) + src_template = "aws-lc-{}.tar.gz" + build_template = "aws-lc-{}" + + def _build_src(self, config_args=()): + cwd = self.build_dir + log.info("Running build in {}".format(cwd)) + env = os.environ.copy() + env["LD_RUN_PATH"] = self.lib_dir # set rpath + if self.system: + env['SYSTEM'] = self.system + cmd = [ + "cmake", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_PREFIX_PATH={}".format(self.install_dir), + "-DCMAKE_INSTALL_PREFIX={}".format(self.install_dir), + "-DBUILD_SHARED_LIBS=ON", + "-DBUILD_TESTING=OFF", + "-DFIPS=OFF", + ] + self._subprocess_call(cmd, cwd=cwd, env=env) + self._subprocess_call(["make", "-j{}".format(self.jobs)], cwd=cwd, env=env) + + def configure_make(): if not os.path.isfile('Makefile'): log.info('Running ./configure') @@ -467,9 +507,10 @@ def configure_make(): def main(): args = parser.parse_args() - if not args.openssl and not args.libressl: + if not args.openssl and not args.libressl and not args.awslc: args.openssl = list(OPENSSL_RECENT_VERSIONS) args.libressl = list(LIBRESSL_RECENT_VERSIONS) + args.awslc = list(AWSLC_RECENT_VERSIONS) if not args.disable_ancient: args.openssl.extend(OPENSSL_OLD_VERSIONS) args.libressl.extend(LIBRESSL_OLD_VERSIONS) @@ -496,22 +537,15 @@ def main(): # download and register builder builds = [] - - for version in args.openssl: - build = BuildOpenSSL( - version, - args - ) - build.install() - builds.append(build) - - for version in args.libressl: - build = BuildLibreSSL( - version, - args - ) - build.install() - builds.append(build) + for build_class, versions in [ + (BuildOpenSSL, args.openssl), + (BuildLibreSSL, args.libressl), + (BuildAWSLC, args.awslc), + ]: + for version in versions: + build = build_class(version, args) + build.install() + builds.append(build) if args.steps in {'modules', 'tests'}: for build in builds: @@ -539,7 +573,7 @@ def main(): else: print('Executed all SSL tests.') - print('OpenSSL / LibreSSL versions:') + print('OpenSSL / LibreSSL / AWS-LC versions:') for build in builds: print(" * {0.library} {0.version}".format(build)) diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 21224e490b8..404c3015736 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -12,15 +12,12 @@ # These warnings trigger directly in a CPython function. -race_top:assign_version_tag -race_top:_Py_slot_tp_getattr_hook race_top:dump_traceback race_top:fatal_error race_top:_PyFrame_GetCode race_top:_PyFrame_Initialize race_top:_PyObject_TryGetInstanceAttribute race_top:PyUnstable_InterpreterFrame_GetLine -race_top:type_modified_unlocked race_top:write_thread_id # gh-129068: race on shared range iterators (test_free_threading.test_zip.ZipThreading.test_threading) @@ -29,9 +26,6 @@ race_top:rangeiter_next # gh-129748: test.test_free_threading.test_slots.TestSlots.test_object race_top:mi_block_set_nextx -# gh-127266: type slot updates are not thread-safe (test_opcache.test_load_attr_method_lazy_dict) -race_top:update_one_slot - # https://gist.github.com/mpage/6962e8870606cfc960e159b407a0cb40 thread:pthread_create @@ -46,4 +40,4 @@ race:list_inplace_repeat_lock_held # PyObject_Realloc internally does memcpy which isn't atomic so can race # with non-locking reads. See #132070 -race:PyObject_Realloc \ No newline at end of file +race:PyObject_Realloc diff --git a/Tools/tz/zdump.py b/Tools/tz/zdump.py deleted file mode 100644 index 39de0a416d0..00000000000 --- a/Tools/tz/zdump.py +++ /dev/null @@ -1,81 +0,0 @@ -import sys -import os -import struct -from array import array -from collections import namedtuple -from datetime import datetime - -ttinfo = namedtuple('ttinfo', ['tt_gmtoff', 'tt_isdst', 'tt_abbrind']) - -class TZInfo: - def __init__(self, transitions, type_indices, ttis, abbrs): - self.transitions = transitions - self.type_indices = type_indices - self.ttis = ttis - self.abbrs = abbrs - - @classmethod - def fromfile(cls, fileobj): - if fileobj.read(4).decode() != "TZif": - raise ValueError("not a zoneinfo file") - fileobj.seek(20) - header = fileobj.read(24) - tzh = (tzh_ttisgmtcnt, tzh_ttisstdcnt, tzh_leapcnt, - tzh_timecnt, tzh_typecnt, tzh_charcnt) = struct.unpack(">6l", header) - transitions = array('i') - transitions.fromfile(fileobj, tzh_timecnt) - if sys.byteorder != 'big': - transitions.byteswap() - - type_indices = array('B') - type_indices.fromfile(fileobj, tzh_timecnt) - - ttis = [] - for i in range(tzh_typecnt): - ttis.append(ttinfo._make(struct.unpack(">lbb", fileobj.read(6)))) - - abbrs = fileobj.read(tzh_charcnt) - - self = cls(transitions, type_indices, ttis, abbrs) - self.tzh = tzh - - return self - - def dump(self, stream, start=None, end=None): - for j, (trans, i) in enumerate(zip(self.transitions, self.type_indices)): - utc = datetime.utcfromtimestamp(trans) - tti = self.ttis[i] - lmt = datetime.utcfromtimestamp(trans + tti.tt_gmtoff) - abbrind = tti.tt_abbrind - abbr = self.abbrs[abbrind:self.abbrs.find(0, abbrind)].decode() - if j > 0: - prev_tti = self.ttis[self.type_indices[j - 1]] - shift = " %+g" % ((tti.tt_gmtoff - prev_tti.tt_gmtoff) / 3600) - else: - shift = '' - print("%s UTC = %s %-5s isdst=%d" % (utc, lmt, abbr, tti[1]) + shift, file=stream) - - @classmethod - def zonelist(cls, zonedir='/usr/share/zoneinfo'): - zones = [] - for root, _, files in os.walk(zonedir): - for f in files: - p = os.path.join(root, f) - with open(p, 'rb') as o: - magic = o.read(4) - if magic == b'TZif': - zones.append(p[len(zonedir) + 1:]) - return zones - -if __name__ == '__main__': - if len(sys.argv) < 2: - zones = TZInfo.zonelist() - for z in zones: - print(z) - sys.exit() - filepath = sys.argv[1] - if not filepath.startswith('/'): - filepath = os.path.join('/usr/share/zoneinfo', filepath) - with open(filepath, 'rb') as fileobj: - tzi = TZInfo.fromfile(fileobj) - tzi.dump(sys.stdout) diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index 889ae8fc869..ddd564deffd 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -43,8 +43,8 @@ # When changing UCD version please update # * Doc/library/stdtypes.rst, and # * Doc/library/unicodedata.rst -# * Doc/reference/lexical_analysis.rst (two occurrences) -UNIDATA_VERSION = "16.0.0" +# * Doc/reference/lexical_analysis.rst (three occurrences) +UNIDATA_VERSION = "17.0.0" UNICODE_DATA = "UnicodeData%s.txt" COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt" EASTASIAN_WIDTH = "EastAsianWidth%s.txt" @@ -104,13 +104,14 @@ ('3400', '4DBF'), # CJK Ideograph Extension A CJK ('4E00', '9FFF'), # CJK Ideograph ('20000', '2A6DF'), # CJK Ideograph Extension B - ('2A700', '2B739'), # CJK Ideograph Extension C + ('2A700', '2B73F'), # CJK Ideograph Extension C ('2B740', '2B81D'), # CJK Ideograph Extension D - ('2B820', '2CEA1'), # CJK Ideograph Extension E + ('2B820', '2CEAD'), # CJK Ideograph Extension E ('2CEB0', '2EBE0'), # CJK Ideograph Extension F ('2EBF0', '2EE5D'), # CJK Ideograph Extension I ('30000', '3134A'), # CJK Ideograph Extension G ('31350', '323AF'), # CJK Ideograph Extension H + ('323B0', '33479'), # CJK Ideograph Extension J ] diff --git a/Tools/wasm/.ruff.toml b/Tools/wasm/.ruff.toml new file mode 100644 index 00000000000..3d8e59fa3f2 --- /dev/null +++ b/Tools/wasm/.ruff.toml @@ -0,0 +1,25 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + "E501", # Line too long +] diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 232321c5157..6615f1e2664 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -9,68 +9,14 @@ # Python WebAssembly (WASM) build run in modern browsers and JavaScript runtimes like *Node.js*. WASI builds use WASM runtimes such as *wasmtime*. -Users and developers are encouraged to use the script -`Tools/wasm/wasm_build.py`. The tool automates the build process and provides -assistance with installation of SDKs, running tests, etc. +**NOTE**: If you are looking for general information about WebAssembly that is +not directly related to CPython, please see https://github.com/psf/webassembly. -**NOTE**: If you are looking for information that is not directly related to -building CPython for WebAssembly (or the resulting build), please see -https://github.com/psf/webassembly for more information. - -## wasm32-emscripten +## Emscripten (wasm32-emscripten) ### Build -To cross compile to the ``wasm32-emscripten`` platform you need -[the Emscripten compiler toolchain](https://emscripten.org/), -a Python interpreter, and an installation of Node version 18 or newer. -Emscripten version 4.0.2 is recommended; newer versions may also work, but all -official testing is performed with that version. All commands below are relative -to a checkout of the Python repository. - -#### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html) - -You can install the Emscripten toolchain as follows: -```shell -git clone https://github.com/emscripten-core/emsdk.git --depth 1 -./emsdk/emsdk install latest -./emsdk/emsdk activate latest -``` -To add the Emscripten compiler to your path: -```shell -source ./emsdk/emsdk_env.sh -``` -This adds `emcc` and `emconfigure` to your path. - -##### Optionally: enable ccache for EMSDK - -The ``EM_COMPILER_WRAPPER`` must be set after the EMSDK environment is -sourced. Otherwise the source script removes the environment variable. - -```shell -export EM_COMPILER_WRAPPER=ccache -``` - -#### Compile and build Python interpreter - -You can use `python Tools/wasm/emscripten` to compile and build targeting -Emscripten. You can do everything at once with: -```shell -python Tools/wasm/emscripten build -``` -or you can break it out into four separate steps: -```shell -python Tools/wasm/emscripten configure-build-python -python Tools/wasm/emscripten make-build-python -python Tools/wasm/emscripten make-libffi -python Tools/wasm/emscripten configure-host -python Tools/wasm/emscripten make-host -``` -Extra arguments to the configure steps are passed along to configure. For -instance, to do a debug build, you can use: -```shell -python Tools/wasm/emscripten build --with-py-debug -``` +See [the devguide instructions for building for Emscripten](https://devguide.python.org/getting-started/setup-building/#emscripten). ### Running from node @@ -84,13 +30,24 @@ ### Running from node CLI you will need to write your own alternative to `node_entry.mjs`. +### Running tests + +After building, you can run the full test suite with: +```shell +./cross-build/wasm32-emscripten/build/python/python.sh -m test -uall +``` +You can run the browser smoke test with: +```shell +./Tools/wasm/emscripten/browser_test/run_test.sh +``` + ### The Web Example -When building for Emscripten, the web example will be built automatically. It is +When building for Emscripten, a small web example will be built automatically in the ``web_example`` directory. To run the web example, ``cd`` into the ``web_example`` directory, then run ``python server.py``. This will start a web -server; you can then visit ``http://localhost:8000/python.html`` in a browser to -see a simple REPL example. +server; you can then visit ``http://localhost:8000/`` in a browser to see a +simple REPL example. The web example relies on a bug fix in Emscripten version 3.1.73 so if you build with earlier versions of Emscripten it may not work. The web example uses diff --git a/Tools/wasm/.editorconfig b/Tools/wasm/emscripten/.editorconfig similarity index 100% rename from Tools/wasm/.editorconfig rename to Tools/wasm/emscripten/.editorconfig diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 849bd5de44e..c88e9edba6d 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -3,15 +3,16 @@ import argparse import contextlib import functools +import hashlib import os import shutil import subprocess import sys import sysconfig import tempfile -from urllib.request import urlopen from pathlib import Path from textwrap import dedent +from urllib.request import urlopen try: from os import process_cpu_count as cpu_count @@ -32,7 +33,7 @@ PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") +LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/emscripten.py\n" def updated_env(updates={}): @@ -44,7 +45,9 @@ def updated_env(updates={}): # https://reproducible-builds.org/docs/source-date-epoch/ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() env_defaults["SOURCE_DATE_EPOCH"] = epoch except subprocess.CalledProcessError: pass # Might be building from a tarball. @@ -78,7 +81,11 @@ def wrapper(context): terminal_width = 80 print("⎯" * terminal_width) print("📁", working_dir) - if clean_ok and getattr(context, "clean", False) and working_dir.exists(): + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): print("🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) @@ -127,7 +134,9 @@ def build_python_path(): if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}" + ) return binary @@ -157,26 +166,75 @@ def make_build_python(context, working_dir): cmd = [ binary, "-c", - "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", ] version = subprocess.check_output(cmd, encoding="utf-8").strip() print(f"🎉 {binary} {version}") +def check_shasum(file: str, expected_shasum: str): + with open(file, "rb") as f: + digest = hashlib.file_digest(f, "sha256") + if digest.hexdigest() != expected_shasum: + raise RuntimeError(f"Unexpected shasum for {file}") + + +def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): + with tempfile.NamedTemporaryFile( + suffix=".tar.gz", delete_on_close=False + ) as tmp_file: + with urlopen(url) as response: + shutil.copyfileobj(response, tmp_file) + tmp_file.close() + check_shasum(tmp_file.name, expected_shasum) + shutil.unpack_archive(tmp_file.name, working_dir) + + @subdir(HOST_BUILD_DIR, clean_ok=True) def make_emscripten_libffi(context, working_dir): - shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True) - with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file: - with urlopen( - "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz" - ) as response: - shutil.copyfileobj(response, tmp_file) - shutil.unpack_archive(tmp_file.name, working_dir) + ver = "3.4.6" + libffi_dir = working_dir / f"libffi-{ver}" + shutil.rmtree(libffi_dir, ignore_errors=True) + download_and_unpack( + working_dir, + f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", + "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e", + ) call( [EMSCRIPTEN_DIR / "make_libffi.sh"], env=updated_env({"PREFIX": PREFIX_DIR}), - cwd=working_dir / "libffi-3.4.6", + cwd=libffi_dir, + quiet=context.quiet, + ) + + +@subdir(HOST_BUILD_DIR, clean_ok=True) +def make_mpdec(context, working_dir): + ver = "4.0.1" + mpdec_dir = working_dir / f"mpdecimal-{ver}" + shutil.rmtree(mpdec_dir, ignore_errors=True) + download_and_unpack( + working_dir, + f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", + "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", + ) + call( + [ + "emconfigure", + mpdec_dir / "configure", + "CFLAGS=-fPIC", + "--prefix", + PREFIX_DIR, + "--disable-shared", + ], + cwd=mpdec_dir, + quiet=context.quiet, + ) + call( + ["make", "install"], + cwd=mpdec_dir, quiet=context.quiet, ) @@ -184,17 +242,15 @@ def make_emscripten_libffi(context, working_dir): @subdir(HOST_DIR, clean_ok=True) def configure_emscripten_python(context, working_dir): """Configure the emscripten/host build.""" - config_site = os.fsdecode( - CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-emscripten" - ) + config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten") emscripten_build_dir = working_dir.relative_to(CHECKOUT) python_build_dir = NATIVE_BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) - assert ( - len(lib_dirs) == 1 - ), f"Expected a single lib.* directory in {python_build_dir}" + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) lib_dir = os.fsdecode(lib_dirs[0]) pydebug = lib_dir.endswith("-pydebug") python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] @@ -205,6 +261,17 @@ def configure_emscripten_python(context, working_dir): sysconfig_data += "-pydebug" host_runner = context.host_runner + if node_version := os.environ.get("PYTHON_NODE_VERSION", None): + res = subprocess.run( + [ + "bash", + "-c", + f"source ~/.nvm/nvm.sh && nvm which {node_version}", + ], + text=True, + capture_output=True, + ) + host_runner = res.stdout.strip() pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() env_additions = { "CONFIG_SITE": config_site, @@ -237,7 +304,9 @@ def configure_emscripten_python(context, working_dir): quiet=context.quiet, ) - shutil.copy(EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs") + shutil.copy( + EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs" + ) node_entry = working_dir / "node_entry.mjs" exec_script = working_dir / "python.sh" @@ -262,10 +331,20 @@ def configure_emscripten_python(context, working_dir): REALPATH=abs_path fi + # Before node 24, --experimental-wasm-jspi uses different API, + # After node 24 JSPI is on by default. + ARGS=$({host_runner} -e "$(cat <<"EOF" + const major_version = Number(process.version.split(".")[0].slice(1)); + if (major_version === 24) {{ + process.stdout.write("--experimental-wasm-jspi"); + }} + EOF + )") + # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" + exec {host_runner} $NODEFLAGS $ARGS {node_entry} --this-program="$($REALPATH "$0")" "$@" """ ) ) @@ -293,6 +372,7 @@ def build_all(context): configure_build_python, make_build_python, make_emscripten_libffi, + make_mpdec, configure_emscripten_python, make_emscripten_python, ] @@ -319,10 +399,15 @@ def main(): subcommands = parser.add_subparsers(dest="subcommand") build = subcommands.add_parser("build", help="Build everything") configure_build = subcommands.add_parser( - "configure-build-python", help="Run `configure` for the " "build Python" + "configure-build-python", help="Run `configure` for the build Python" + ) + make_mpdec_cmd = subcommands.add_parser( + "make-mpdec", + help="Clone mpdec repo, configure and build it for emscripten", ) make_libffi_cmd = subcommands.add_parser( - "make-libffi", help="Clone libffi repo, configure and build it for emscripten" + "make-libffi", + help="Clone libffi repo, configure and build it for emscripten", ) make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" @@ -341,9 +426,11 @@ def main(): build, configure_build, make_libffi_cmd, + make_mpdec_cmd, make_build, configure_host, make_host, + clean, ): subcommand.add_argument( "--quiet", @@ -378,6 +465,7 @@ def main(): dispatch = { "make-libffi": make_emscripten_libffi, + "make-mpdec": make_mpdec, "configure-build-python": configure_build_python, "make-build-python": make_build_python, "configure-host": configure_emscripten_python, @@ -388,7 +476,11 @@ def main(): if not context.subcommand: # No command provided, display help and exit - print("Expected one of", ", ".join(sorted(dispatch.keys())), file=sys.stderr) + print( + "Expected one of", + ", ".join(sorted(dispatch.keys())), + file=sys.stderr, + ) parser.print_help(sys.stderr) sys.exit(1) dispatch[context.subcommand](context) diff --git a/Tools/wasm/emscripten/browser_test/.gitignore b/Tools/wasm/emscripten/browser_test/.gitignore new file mode 100644 index 00000000000..3d54eab82a0 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/.gitignore @@ -0,0 +1,4 @@ +node_modules +test-results +playwright-report +test_log.txt diff --git a/Tools/wasm/emscripten/browser_test/index.spec.ts b/Tools/wasm/emscripten/browser_test/index.spec.ts new file mode 100644 index 00000000000..2594b73022b --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/index.spec.ts @@ -0,0 +1,15 @@ +import { test, expect } from '@playwright/test'; + +test('has title', async ({ page }) => { + await page.goto('/'); + + await expect(page).toHaveTitle("Emscripten PyRepl Example"); + const xterm = await page.locator('css=#terminal'); + await expect(xterm).toHaveText(/Python.*on emscripten.*Type.*for more information/); + const xtermInput = await page.getByRole('textbox'); + await xtermInput.pressSequentially(`def f():\nprint("hello", "emscripten repl!")\n\n`); + await xtermInput.pressSequentially(`f()\n`); + await expect(xterm).toHaveText(/hello emscripten repl!/); + +}); + diff --git a/Tools/wasm/emscripten/browser_test/package-lock.json b/Tools/wasm/emscripten/browser_test/package-lock.json new file mode 100644 index 00000000000..044e3c19ce1 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/package-lock.json @@ -0,0 +1,672 @@ +{ + "name": "browser_test", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "browser_test", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@playwright/test": "^1.54.1", + "@types/node": "^24.1.0", + "http-server": "^14.1.1", + "playwright": "^1.54.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.54.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.54.1.tgz", + "integrity": "sha512-FS8hQ12acieG2dYSksmLOF7BNxnVf2afRJdCuM1eMSxj6QTSE6G4InGF7oApGgDb65MX7AwMVlIkpru0yZA4Xw==", + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.54.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "24.1.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.1.0.tgz", + "integrity": "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.8.0" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/corser": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", + "integrity": "sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-server": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", + "integrity": "sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==", + "license": "MIT", + "dependencies": { + "basic-auth": "^2.0.1", + "chalk": "^4.1.2", + "corser": "^2.0.1", + "he": "^1.2.0", + "html-encoding-sniffer": "^3.0.0", + "http-proxy": "^1.18.1", + "mime": "^1.6.0", + "minimist": "^1.2.6", + "opener": "^1.5.1", + "portfinder": "^1.0.28", + "secure-compare": "3.0.1", + "union": "~0.5.0", + "url-join": "^4.0.1" + }, + "bin": { + "http-server": "bin/http-server" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/opener": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", + "license": "(WTFPL OR MIT)", + "bin": { + "opener": "bin/opener-bin.js" + } + }, + "node_modules/playwright": { + "version": "1.54.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.54.1.tgz", + "integrity": "sha512-peWpSwIBmSLi6aW2auvrUtf2DqY16YYcCMO8rTVx486jKmDTJg7UAhyrraP98GB8BoPURZP8+nxO7TSd4cPr5g==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.54.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.54.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.54.1.tgz", + "integrity": "sha512-Nbjs2zjj0htNhzgiy5wu+3w09YetDx5pkrpI/kZotDlDUaYk0HVA5xrBVPdow4SAUIlhgKcJeJg4GRKW6xHusA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/portfinder": { + "version": "1.0.37", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.37.tgz", + "integrity": "sha512-yuGIEjDAYnnOex9ddMnKZEMFE0CcGo6zbfzDklkmT1m5z734ss6JMzN9rNB3+RR7iS+F10D4/BVIaXOyh8PQKw==", + "license": "MIT", + "dependencies": { + "async": "^3.2.6", + "debug": "^4.3.6" + }, + "engines": { + "node": ">= 10.12" + } + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/secure-compare": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", + "integrity": "sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==", + "license": "MIT" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/undici-types": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz", + "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==", + "license": "MIT" + }, + "node_modules/union": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", + "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", + "dependencies": { + "qs": "^6.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", + "license": "MIT" + }, + "node_modules/whatwg-encoding": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=12" + } + } + } +} diff --git a/Tools/wasm/emscripten/browser_test/package.json b/Tools/wasm/emscripten/browser_test/package.json new file mode 100644 index 00000000000..3320d4cccd0 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/package.json @@ -0,0 +1,18 @@ +{ + "name": "browser_test", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "@playwright/test": "^1.54.1", + "@types/node": "^24.1.0", + "http-server": "^14.1.1", + "playwright": "^1.54.1" + } +} diff --git a/Tools/wasm/emscripten/browser_test/playwright.config.ts b/Tools/wasm/emscripten/browser_test/playwright.config.ts new file mode 100644 index 00000000000..81d53ce11cb --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/playwright.config.ts @@ -0,0 +1,22 @@ +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: '.', + forbidOnly: true, + retries: 2, + reporter: process.env.CI ? 'dot' : 'html', + use: { + baseURL: 'http://localhost:8787', + trace: 'on-first-retry', + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], + webServer: { + command: 'npx http-server ../../../../cross-build/wasm32-emscripten/build/python/web_example_pyrepl_jspi/ -p 8787', + url: 'http://localhost:8787', + }, +}); diff --git a/Tools/wasm/emscripten/browser_test/run_test.sh b/Tools/wasm/emscripten/browser_test/run_test.sh new file mode 100755 index 00000000000..9166e0d7405 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/run_test.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -euo pipefail +cd "$(dirname "$0")" +rm -f test_log.txt +echo "Installing node packages" | tee test_log.txt +npm ci >> test_log.txt 2>&1 +echo "Installing playwright browsers" | tee test_log.txt +npx playwright install 2>> test_log.txt +echo "Running tests" | tee test_log.txt +CI=1 npx playwright test | tee test_log.txt diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Tools/wasm/emscripten/config.site-wasm32-emscripten similarity index 97% rename from Tools/wasm/config.site-wasm32-emscripten rename to Tools/wasm/emscripten/config.site-wasm32-emscripten index 1471546a5ee..9f98e3f3c3b 100644 --- a/Tools/wasm/config.site-wasm32-emscripten +++ b/Tools/wasm/emscripten/config.site-wasm32-emscripten @@ -1,6 +1,6 @@ # config.site override for cross compiling to wasm32-emscripten platform # -# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \ +# CONFIG_SITE=Tools/wasm/emscripten/config.site-wasm32-emscripten \ # emconfigure ./configure --host=wasm32-unknown-emscripten --build=... # # Written by Christian Heimes <christian@python.org> @@ -69,7 +69,6 @@ ac_cv_func_posix_fallocate=no # Syscalls that resulted in a segfault ac_cv_func_utimensat=no -ac_cv_header_sys_ioctl_h=no # sockets are supported, but only AF_INET / AF_INET6 in non-blocking mode. # Disable AF_UNIX and AF_PACKET support, see socketmodule.h. diff --git a/Tools/wasm/emscripten/prepare_external_wasm.py b/Tools/wasm/emscripten/prepare_external_wasm.py new file mode 100644 index 00000000000..1b0a9de4b1f --- /dev/null +++ b/Tools/wasm/emscripten/prepare_external_wasm.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +JS_TEMPLATE = """ +#include "emscripten.h" + +EM_JS(void, {function_name}, (void), {{ + return new WebAssembly.Module(hexStringToUTF8Array("{hex_string}")); +}} +function hexStringToUTF8Array(hex) {{ + const bytes = []; + for (let i = 0; i < hex.length; i += 2) {{ + bytes.push(parseInt(hex.substr(i, 2), 16)); + }} + return new Uint8Array(bytes); +}}); +""" + + +def prepare_wasm(input_file, output_file, function_name): + # Read the compiled WASM as binary and convert to hex + wasm_bytes = Path(input_file).read_bytes() + + hex_string = "".join(f"{byte:02x}" for byte in wasm_bytes) + + # Generate JavaScript module + js_content = JS_TEMPLATE.format( + function_name=function_name, hex_string=hex_string + ) + Path(output_file).write_text(js_content) + + print(f"Successfully compiled {input_file} and generated {output_file}") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Compile WebAssembly text files using wasm-as" + ) + parser.add_argument("input_file", help="Input .wat file to compile") + parser.add_argument("output_file", help="Output file name") + parser.add_argument("function_name", help="Name of the export function") + + args = parser.parse_args() + + return prepare_wasm(args.input_file, args.output_file, args.function_name) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Tools/wasm/emscripten/web_example/wasm_assets.py b/Tools/wasm/emscripten/wasm_assets.py similarity index 95% rename from Tools/wasm/emscripten/web_example/wasm_assets.py rename to Tools/wasm/emscripten/wasm_assets.py index deeb9229a44..38479087235 100755 --- a/Tools/wasm/emscripten/web_example/wasm_assets.py +++ b/Tools/wasm/emscripten/wasm_assets.py @@ -15,10 +15,9 @@ import sys import sysconfig import zipfile -from typing import Dict # source directory -SRCDIR = pathlib.Path(__file__).parents[4].absolute() +SRCDIR = pathlib.Path(__file__).parents[3].absolute() SRCDIR_LIB = SRCDIR / "Lib" @@ -27,7 +26,9 @@ WASM_STDLIB_ZIP = ( WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" ) -WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +WASM_STDLIB = ( + WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +) WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" @@ -58,7 +59,6 @@ # socket.create_connection() raises an exception: # "BlockingIOError: [Errno 26] Operation in progress". OMIT_NETWORKING_FILES = ( - "email/", "ftplib.py", "http/", "imaplib.py", @@ -84,7 +84,6 @@ "_json": ["json/"], "_multiprocessing": ["concurrent/futures/process.py", "multiprocessing/"], "pyexpat": ["xml/", "xmlrpc/"], - "readline": ["rlcompleter.py"], "_sqlite3": ["sqlite3/"], "_ssl": ["ssl.py"], "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"], @@ -134,7 +133,7 @@ def filterfunc(filename: str) -> bool: pzf.writepy(entry, filterfunc=filterfunc) -def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: +def detect_extension_modules(args: argparse.Namespace) -> dict[str, bool]: modules = {} # disabled by Modules/Setup.local ? @@ -149,7 +148,7 @@ def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: # disabled by configure? with open(args.sysconfig_data) as f: data = f.read() - loc: Dict[str, Dict[str, str]] = {} + loc: dict[str, dict[str, str]] = {} exec(data, globals(), loc) for key, value in loc["build_time_vars"].items(): diff --git a/Tools/wasm/emscripten/web_example/python.html b/Tools/wasm/emscripten/web_example/index.html similarity index 52% rename from Tools/wasm/emscripten/web_example/python.html rename to Tools/wasm/emscripten/web_example/index.html index 078f86eb764..9c89c9c0ed3 100644 --- a/Tools/wasm/emscripten/web_example/python.html +++ b/Tools/wasm/emscripten/web_example/index.html @@ -1,31 +1,39 @@ <!doctype html> <html lang="en"> <head> - <meta charset="UTF-8" /> - <meta http-equiv="X-UA-Compatible" content="IE=edge" /> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="author" content="Katie Bell" /> - <meta name="description" content="Simple REPL for Python WASM" /> + <meta charset="UTF-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta name="author" content="Katie Bell, Adam Hartz"> + <meta name="description" content="Simple REPL for Python WASM"> <title>wasm-python terminal</title> <link rel="stylesheet" href="https://unpkg.com/xterm@4.18.0/css/xterm.css" crossorigin integrity="sha384-4eEEn/eZgVHkElpKAzzPx/Kow/dTSgFk1BNe+uHdjHa+NkZJDh5Vqkq31+y7Eycd" - /> + > <style> body { font-family: arial; max-width: 800px; margin: 0 auto; } - #code { + #editor { + padding: 5px; + border: 1px solid black; width: 100%; - height: 180px; + height: 300px; } #info { padding-top: 20px; } + .error { + border: 1px solid red; + background-color: #ffd9d9; + padding: 5px; + margin-top: 20px; + } .button-container { display: flex; justify-content: end; @@ -41,8 +49,14 @@ src="https://unpkg.com/xterm@4.18.0/lib/xterm.js" crossorigin integrity="sha384-yYdNmem1ioP5Onm7RpXutin5A8TimLheLNQ6tnMi01/ZpxXdAwIm2t4fJMx1Djs+" - /> + ></script> + <script + src="https://cdnjs.cloudflare.com/ajax/libs/ace/1.43.1/ace.js" + crossorigin + integrity="sha512-kmA5vhcxOkZI0ReiKJMGNb8/KKbgbExIlnt6aXuPtl86AgHBEi6OHHOz2wsTazBDGZKxe7fmiE+pIuZJQks4+A==" + ></script> <script type="module"> + const _magic_ctrlc_string = "__WASM_REPL_CTRLC_" + (Date.now()) + "__"; class WorkerManager { constructor( workerURL, @@ -132,11 +146,14 @@ class WasmTerminal { constructor() { - this.inputBuffer = new BufferQueue(); - this.input = ""; - this.resolveInput = null; - this.activeInput = false; - this.inputStartCursor = null; + try { + this.history = JSON.parse(sessionStorage.getItem('__python_wasm_repl.history')); + this.historyBuffer = this.history.slice(); + } catch(e) { + this.history = []; + this.historyBuffer = []; + } + this.reset(); this.xterm = new Terminal({ scrollback: 10000, @@ -155,6 +172,18 @@ this.xterm.onData(this.handleTermData); } + reset() { + this.inputBuffer = new BufferQueue(); + this.input = ""; + this.resolveInput = null; + this.activeInput = false; + this.inputStartCursor = null; + + this.cursorPosition = 0; + this.historyIndex = -1; + this.beforeHistoryNav = ""; + } + open(container) { this.xterm.open(container); } @@ -186,9 +215,34 @@ if (!(ord === 0x1b || ord == 0x7f || ord < 32)) { this.inputBuffer.addData(data); } - // TODO: Handle ANSI escape sequences + // TODO: Handle more escape sequences? } else if (ord === 0x1b) { // Handle special characters + switch (data.slice(1)) { + case "[A": // up + this.historyBack(); + break; + case "[B": // down + this.historyForward(); + break; + case "[C": // right + this.cursorRight(); + break; + case "[D": // left + this.cursorLeft(); + break; + case "[H": // home key + this.cursorHome(true); + break; + case "[F": // end key + this.cursorEnd(true); + break; + case "[3~": // delete key + this.deleteAtCursor(); + break; + default: + break; + } } else if (ord < 32 || ord === 0x7f) { switch (data) { case "\x0c": // CTRL+L @@ -201,8 +255,18 @@ this.input + this.writeLine("\n"), ); this.input = ""; + this.cursorPosition = 0; this.activeInput = false; break; + case "\x03": // CTRL+C + this.input = ""; + this.cursorPosition = 0; + this.historyIndex = -1; + this.resolveInput(_magic_ctrlc_string + "\n"); + break; + case "\x09": // TAB + this.handleTab(); + break; case "\x7F": // BACKSPACE case "\x08": // CTRL+H this.handleCursorErase(true); @@ -211,14 +275,20 @@ // Send empty input if (this.input === "") { this.resolveInput(""); + this.cursorPosition = 0; this.activeInput = false; } } } else { this.handleCursorInsert(data); + this.updateHistory(); } }; + clearLine() { + this.xterm.write("\x1b[K"); + } + writeLine(line) { this.xterm.write(line.slice(0, -1)); this.xterm.write("\r\n"); @@ -226,8 +296,36 @@ } handleCursorInsert(data) { - this.input += data; + const trailing = this.input.slice(this.cursorPosition); + this.input = + this.input.slice(0, this.cursorPosition) + + data + + trailing; + this.cursorPosition += data.length; this.xterm.write(data); + if (trailing.length !== 0) { + this.xterm.write(trailing); + this.xterm.write("\x1b[" + trailing.length + "D"); + } + this.updateHistory(); + } + + handleTab() { + // handle tabs: from the current position, add spaces until + // this.cursorPosition is a multiple of 4. + const prefix = this.input.slice(0, this.cursorPosition); + const suffix = this.input.slice(this.cursorPosition); + const count = 4 - (this.cursorPosition % 4); + const toAdd = " ".repeat(count); + this.input = prefix + toAdd + suffix; + this.cursorHome(false); + this.clearLine(); + this.xterm.write(this.input); + if (suffix) { + this.xterm.write("\x1b[" + suffix.length + "D"); + } + this.cursorPosition += count; + this.updateHistory(); } handleCursorErase() { @@ -238,9 +336,113 @@ ) { return; } - this.input = this.input.slice(0, -1); - this.xterm.write("\x1B[D"); - this.xterm.write("\x1B[P"); + const trailing = this.input.slice(this.cursorPosition); + this.input = + this.input.slice(0, this.cursorPosition - 1) + trailing; + this.cursorLeft(); + this.clearLine(); + if (trailing.length !== 0) { + this.xterm.write(trailing); + this.xterm.write("\x1b[" + trailing.length + "D"); + } + this.updateHistory(); + } + + deleteAtCursor() { + if (this.cursorPosition < this.input.length) { + const trailing = this.input.slice( + this.cursorPosition + 1, + ); + this.input = + this.input.slice(0, this.cursorPosition) + trailing; + this.clearLine(); + if (trailing.length !== 0) { + this.xterm.write(trailing); + this.xterm.write("\x1b[" + trailing.length + "D"); + } + this.updateHistory(); + } + } + + cursorRight() { + if (this.cursorPosition < this.input.length) { + this.cursorPosition += 1; + this.xterm.write("\x1b[C"); + } + } + + cursorLeft() { + if (this.cursorPosition > 0) { + this.cursorPosition -= 1; + this.xterm.write("\x1b[D"); + } + } + + cursorHome(updatePosition) { + if (this.cursorPosition > 0) { + this.xterm.write("\x1b[" + this.cursorPosition + "D"); + if (updatePosition) { + this.cursorPosition = 0; + } + } + } + + cursorEnd() { + if (this.cursorPosition < this.input.length) { + this.xterm.write( + "\x1b[" + + (this.input.length - this.cursorPosition) + + "C", + ); + this.cursorPosition = this.input.length; + } + } + + updateHistory() { + if (this.historyIndex !== -1) { + this.historyBuffer[this.historyIndex] = this.input; + } else { + this.beforeHistoryNav = this.input; + } + } + + historyBack() { + if (this.history.length === 0) { + return; + } else if (this.historyIndex === -1) { + // we're not currently navigating the history; store + // the current command and then look at the end of our + // history buffer + this.beforeHistoryNav = this.input; + this.historyIndex = this.history.length - 1; + } else if (this.historyIndex > 0) { + this.historyIndex -= 1; + } + this.input = this.historyBuffer[this.historyIndex]; + this.cursorHome(false); + this.clearLine(); + this.xterm.write(this.input); + this.cursorPosition = this.input.length; + } + + historyForward() { + if (this.history.length === 0 || this.historyIndex === -1) { + // we're not currently navigating the history; NOP. + return; + } else if (this.historyIndex < this.history.length - 1) { + this.historyIndex += 1; + this.input = this.historyBuffer[this.historyIndex]; + } else if (this.historyIndex == this.history.length - 1) { + // we're coming back from the last history value; reset + // the input to whatever it was when we started going + // through the history + this.input = this.beforeHistoryNav; + this.historyIndex = -1; + } + this.cursorHome(false); + this.clearLine(); + this.xterm.write(this.input); + this.cursorPosition = this.input.length; } prompt = async () => { @@ -263,12 +465,29 @@ // Hack to ensure cursor input start doesn't end up after user input setTimeout(() => { this.handleCursorInsert( - this.inputBuffer.nextLine(), + this.inputBuffer.nextLine() ); }, 1); } return new Promise((resolve, reject) => { this.resolveInput = (value) => { + if ( + value.replace(/\s/g, "").length != 0 && + value != _magic_ctrlc_string + "\n" + ) { + if (this.historyIndex !== -1) { + this.historyBuffer[this.historyIndex] = + this.history[this.historyIndex]; + } + this.history.push(value.slice(0, -1)); + this.historyBuffer.push(value.slice(0, -1)); + this.historyIndex = -1; + this.cursorPosition = 0; + try { + sessionStorage.setItem('__python_wasm_repl.history', JSON.stringify(this.history)); + } catch(e) { + } + } resolve(value); }; }); @@ -327,8 +546,6 @@ const stopButton = document.getElementById("stop"); const clearButton = document.getElementById("clear"); - const codeBox = document.getElementById("codebox"); - window.onload = () => { const terminal = new WasmTerminal(); terminal.open(document.getElementById("terminal")); @@ -362,8 +579,9 @@ runButton.addEventListener("click", (e) => { terminal.clear(); + terminal.reset(); // reset the history programRunning(true); - const code = codeBox.value; + const code = editor.getValue(); pythonWorkerManager.run({ args: ["main.py"], files: { "main.py": code }, @@ -372,10 +590,28 @@ replButton.addEventListener("click", (e) => { terminal.clear(); + terminal.reset(); // reset the history + const REPL = ` +class WASMREPLKeyboardInterrupt(KeyboardInterrupt): + pass + +import sys +import code +import builtins + +def _interrupt_aware_input(prompt=''): + line = builtins.input(prompt) + if line.strip() == "${_magic_ctrlc_string}": + raise KeyboardInterrupt() + return line + +cprt = 'Type "help", "copyright", "credits" or "license" for more information.' +banner = f'Python {sys.version} on {sys.platform}\\n{cprt}' + +code.interact(banner=banner, readfunc=_interrupt_aware_input, exitmsg='') +`; programRunning(true); - // Need to use "-i -" to force interactive mode. - // Looks like isatty always returns false in emscripten - pythonWorkerManager.run({ args: ["-i", "-"], files: {} }); + pythonWorkerManager.run({ args: ["-c", REPL], files: {} }); }); stopButton.addEventListener("click", (e) => { @@ -395,6 +631,7 @@ const finishedCallback = () => { programRunning(false); + pythonWorkerManager.reset(); }; const pythonWorkerManager = new WorkerManager( @@ -404,30 +641,63 @@ finishedCallback, ); }; + var editor; + document.addEventListener("DOMContentLoaded", () => { + editor = ace.edit("editor"); + editor.session.setMode("ace/mode/python"); + }); </script> </head> <body> - <h1>Simple REPL for Python WASM</h1> - <textarea id="codebox" cols="108" rows="16"> -print('Welcome to WASM!') -</textarea - > - <div class="button-container"> - <button id="run" disabled>Run</button> - <button id="repl" disabled>Start REPL</button> - <button id="stop" disabled>Stop</button> - <button id="clear" disabled>Clear</button> + <div id="repldemo"> + <h1>Simple REPL for Python WASM</h1> + <div id="editor">print('Welcome to WASM!')</div> + <div class="button-container"> + <button id="run" disabled>Run code</button> + <button id="repl" disabled>Start REPL</button> + <button id="stop" disabled>Stop</button> + <button id="clear" disabled>Clear</button> + </div> + <div id="terminal"></div> + <div id="info"> + The simple REPL provides a limited Python experience in the + browser. + <a + href="https://github.com/python/cpython/blob/main/Tools/wasm/README.md" + > + Tools/wasm/README.md + </a> + contains a list of known limitations and issues. Networking, + subprocesses, and threading are not available. + </div> </div> - <div id="terminal"></div> - <div id="info"> - The simple REPL provides a limited Python experience in the browser. - <a - href="https://github.com/python/cpython/blob/main/Tools/wasm/README.md" - > - Tools/wasm/README.md - </a> - contains a list of known limitations and issues. Networking, - subprocesses, and threading are not available. + <div id="buffererror" class="error" style="display: none"> + <p> + <code>SharedArrayBuffer</code>, which is required for this demo, + is not available in your browser environment. One common cause + of this failure is loading <code>index.html</code> directly in + your browser instead of using <code>server.py</code> as + described in + <a + href="https://github.com/python/cpython/blob/main/Tools/wasm/README.md#the-web-example" + > + Tools/wasm/README.md + </a>. + </p> + <p> + For more details about security requirements for + <code>SharedArrayBuffer</code>, see + <a + href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements" + >this MDN page</a + >. + </p> + <script> + if (typeof SharedArrayBuffer === 'undefined') { + document.getElementById('repldemo').style.display = 'none'; + document.getElementById('buffererror').style.display = 'block'; + } + </script> </div> </body> </html> diff --git a/Tools/wasm/emscripten/web_example/server.py b/Tools/wasm/emscripten/web_example/server.py index 768e6f84e07..f2e6ed56c6b 100755 --- a/Tools/wasm/emscripten/web_example/server.py +++ b/Tools/wasm/emscripten/web_example/server.py @@ -6,10 +6,16 @@ description="Start a local webserver with a Python terminal." ) parser.add_argument( - "--port", type=int, default=8000, help="port for the http server to listen on" + "--port", + type=int, + default=8000, + help="port for the http server to listen on", ) parser.add_argument( - "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" + "--bind", + type=str, + default="127.0.0.1", + help="Bind address (empty for all)", ) diff --git a/Tools/wasm/emscripten/web_example_pyrepl_jspi/index.html b/Tools/wasm/emscripten/web_example_pyrepl_jspi/index.html new file mode 100644 index 00000000000..cc7413b5caa --- /dev/null +++ b/Tools/wasm/emscripten/web_example_pyrepl_jspi/index.html @@ -0,0 +1,35 @@ +<!doctype html> +<html> + <head> + <title>Emscripten PyRepl Example</title> + <link + rel="stylesheet" + href="https://unpkg.com/xterm@4.18.0/css/xterm.css" + /> + <style> + body { + background-color: #300a24; + } + + .xterm-dom-renderer-owner-1 .xterm-fg-3 { + color: #c4a000 !important; + } + + .xterm-dom-renderer-owner-1 .xterm-fg-6 { + color: #2aa1b3 !important; + } + + .xterm-dom-renderer-owner-1 .xterm-fg-12 { + color: #1054a6 !important; + } + + .xterm-dom-renderer-owner-1 .xterm-fg-13 { + color: #a347ba !important; + } + </style> + </head> + <body> + <div id="terminal"></div> + <script type="module" src="src.mjs"></script> + </body> +</html> diff --git a/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs b/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs new file mode 100644 index 00000000000..5642372c9d2 --- /dev/null +++ b/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs @@ -0,0 +1,194 @@ +// Much of this is adapted from here: +// https://github.com/mame/xterm-pty/blob/main/emscripten-pty.js +// Thanks to xterm-pty for making this possible! + +import createEmscriptenModule from "./python.mjs"; +import { openpty } from "https://unpkg.com/xterm-pty/index.mjs"; +import "https://unpkg.com/@xterm/xterm/lib/xterm.js"; + +var term = new Terminal(); +term.open(document.getElementById("terminal")); +const { master, slave: PTY } = openpty(); +term.loadAddon(master); +globalThis.PTY = PTY; + +async function setupStdlib(Module) { + const versionInt = Module.HEAPU32[Module._Py_Version >>> 2]; + const major = (versionInt >>> 24) & 0xff; + const minor = (versionInt >>> 16) & 0xff; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile( + `/lib/python${major}${minor}.zip`, + new Uint8Array(stdlibBuffer), + { canOwn: true }, + ); +} + +const tty_ops = { + ioctl_tcgets: () => { + const termios = PTY.ioctl("TCGETS"); + const data = { + c_iflag: termios.iflag, + c_oflag: termios.oflag, + c_cflag: termios.cflag, + c_lflag: termios.lflag, + c_cc: termios.cc, + }; + return data; + }, + + ioctl_tcsets: (_tty, _optional_actions, data) => { + PTY.ioctl("TCSETS", { + iflag: data.c_iflag, + oflag: data.c_oflag, + cflag: data.c_cflag, + lflag: data.c_lflag, + cc: data.c_cc, + }); + return 0; + }, + + ioctl_tiocgwinsz: () => PTY.ioctl("TIOCGWINSZ").reverse(), + + get_char: () => { + throw new Error("Should not happen"); + }, + put_char: () => { + throw new Error("Should not happen"); + }, + + fsync: () => {}, +}; + +const POLLIN = 1; +const POLLOUT = 4; + +const waitResult = { + READY: 0, + SIGNAL: 1, + TIMEOUT: 2, +}; + +function onReadable() { + var handle; + var promise = new Promise((resolve) => { + handle = PTY.onReadable(() => resolve(waitResult.READY)); + }); + return [promise, handle]; +} + +function onSignal() { + // TODO: signal handling + var handle = { dispose() {} }; + var promise = new Promise((resolve) => {}); + return [promise, handle]; +} + +function onTimeout(timeout) { + var id; + var promise = new Promise((resolve) => { + if (timeout > 0) { + id = setTimeout(resolve, timeout, waitResult.TIMEOUT); + } + }); + var handle = { + dispose() { + if (id) { + clearTimeout(id); + } + }, + }; + return [promise, handle]; +} + +async function waitForReadable(timeout) { + let p1, p2, p3; + let h1, h2, h3; + try { + [p1, h1] = onReadable(); + [p2, h2] = onTimeout(timeout); + [p3, h3] = onSignal(); + return await Promise.race([p1, p2, p3]); + } finally { + h1.dispose(); + h2.dispose(); + h3.dispose(); + } +} + +const FIONREAD = 0x541b; + +const tty_stream_ops = { + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + let readBytes = PTY.read(length); + if (length && !readBytes.length) { + const status = await waitForReadable(-1); + if (status === waitResult.READY) { + readBytes = PTY.read(length); + } else { + throw new Error("Not implemented"); + } + } + buffer.set(readBytes, offset); + return readBytes.length; + }, + + write: (stream, buffer, offset, length) => { + // Note: default `buffer` is for some reason `HEAP8` (signed), while we want unsigned `HEAPU8`. + buffer = new Uint8Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength, + ); + const toWrite = Array.from(buffer.subarray(offset, offset + length)); + PTY.write(toWrite); + return length; + }, + + async pollAsync(stream, timeout) { + if (!PTY.readable && timeout) { + await waitForReadable(timeout); + } + return (PTY.readable ? POLLIN : 0) | (PTY.writable ? POLLOUT : 0); + }, + ioctl(stream, request, varargs) { + if (request === FIONREAD) { + const res = PTY.fromLdiscToUpperBuffer.length; + Module.HEAPU32[varargs / 4] = res; + return 0; + } + throw new Error("Unimplemented ioctl request"); + }, +}; + +async function setupStdio(Module) { + Object.assign(Module.TTY.default_tty_ops, tty_ops); + Object.assign(Module.TTY.stream_ops, tty_stream_ops); +} + +const emscriptenSettings = { + async preRun(Module) { + Module.addRunDependency("pre-run"); + Module.ENV.TERM = "xterm-256color"; + // Uncomment next line to turn on tracing (messages go to browser console). + // Module.ENV.PYREPL_TRACE = "1"; + + // Leak module so we can try to show traceback if we crash on startup + globalThis.Module = Module; + await Promise.all([setupStdlib(Module), setupStdio(Module)]); + Module.removeRunDependency("pre-run"); + }, +}; + +try { + await createEmscriptenModule(emscriptenSettings); +} catch (e) { + // Show JavaScript exception and traceback + console.warn(e); + // Show Python exception and traceback + Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + process.exit(1); +} diff --git a/Tools/wasm/mypy.ini b/Tools/wasm/mypy.ini deleted file mode 100644 index 4de0a30c260..00000000000 --- a/Tools/wasm/mypy.ini +++ /dev/null @@ -1,11 +0,0 @@ -[mypy] -files = Tools/wasm/wasm_*.py -pretty = True -show_traceback = True - -# Make sure the wasm can be run using Python 3.8: -python_version = 3.8 - -# Be strict... -strict = True -enable_error_code = truthy-bool,ignore-without-code diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 4c5078a1f67..08d4f499baa 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -1,7 +1,8 @@ #!/bin/sh set -e -# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py . +# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py ; +# expected in Python 3.18 as 3.13 is when `wasi.py` was introduced. # function usage() { diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index a742043e4be..af55e03d10f 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -1,367 +1,12 @@ -#!/usr/bin/env python3 +if __name__ == "__main__": + import pathlib + import runpy + import sys -import argparse -import contextlib -import functools -import os -try: - from os import process_cpu_count as cpu_count -except ImportError: - from os import cpu_count -import pathlib -import shutil -import subprocess -import sys -import sysconfig -import tempfile - - -CHECKOUT = pathlib.Path(__file__).parent.parent.parent - -CROSS_BUILD_DIR = CHECKOUT / "cross-build" -BUILD_DIR = CROSS_BUILD_DIR / "build" - -LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") - -WASMTIME_VAR_NAME = "WASMTIME" -WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - print("🌎 Environment changes:") - for key in sorted(env_diff.keys()): - print(f" {key}={env_diff[key]}") - - return environment - - -def subdir(working_dir, *, clean_ok=False): - """Decorator to change to a working directory.""" - def decorator(func): - @functools.wraps(func) - def wrapper(context): - nonlocal working_dir - - if callable(working_dir): - working_dir = working_dir(context) - try: - tput_output = subprocess.check_output(["tput", "cols"], - encoding="utf-8") - except subprocess.CalledProcessError: - terminal_width = 80 - else: - terminal_width = int(tput_output.strip()) - print("⎯" * terminal_width) - print("📁", working_dir) - if (clean_ok and getattr(context, "clean", False) and - working_dir.exists()): - print(f"🚮 Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, quiet, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - print("❯", " ".join(map(str, command))) - if not quiet: - stdout = None - stderr = None - else: - stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", - delete=False, - prefix="cpython-wasi-", - suffix=".log") - stderr = subprocess.STDOUT - print(f"📝 Logging output to {stdout.name} (--quiet)...") - - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - - -def build_platform(): - """The name of the build/host platform.""" - # Can also be found via `config.guess`.` - return sysconfig.get_config_var("BUILD_GNU_TYPE") - - -def build_python_path(): - """The path to the build Python binary.""" - binary = BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{BUILD_DIR}") - - return binary - - -@subdir(BUILD_DIR, clean_ok=True) -def configure_build_python(context, working_dir): - """Configure the build/host Python.""" - if LOCAL_SETUP.exists(): - print(f"👍 {LOCAL_SETUP} exists ...") - else: - print(f"📝 Touching {LOCAL_SETUP} ...") - LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] - if context.args: - configure.extend(context.args) - - call(configure, quiet=context.quiet) - - -@subdir(BUILD_DIR) -def make_build_python(context, working_dir): - """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], - quiet=context.quiet) - - binary = build_python_path() - cmd = [binary, "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] - version = subprocess.check_output(cmd, encoding="utf-8").strip() - - print(f"🎉 {binary} {version}") - - -def find_wasi_sdk(): - """Find the path to wasi-sdk.""" - if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): - return pathlib.Path(wasi_sdk_path) - elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists(): - return default_path - - -def wasi_sdk_env(context): - """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path - sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", - "AR": "llvm-ar", "RANLIB": "ranlib"} - - for env_var, binary_name in list(env.items()): - env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) - - if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): - for compiler in ["CC", "CPP", "CXX"]: - env[compiler] += f" --sysroot={sysroot}" - - env["PKG_CONFIG_PATH"] = "" - env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map(os.fsdecode, - [sysroot / "lib" / "pkgconfig", - sysroot / "share" / "pkgconfig"])) - env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) - - env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) - env["WASI_SYSROOT"] = os.fsdecode(sysroot) - - env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"]]) - - return env - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) -def configure_wasi_python(context, working_dir): - """Configure the WASI/host build.""" - if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk") - - config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") - - wasi_build_dir = working_dir.relative_to(CHECKOUT) - - python_build_dir = BUILD_DIR / "build" - lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" - lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] - sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" - if pydebug: - sysconfig_data += "-pydebug" - - # Use PYTHONPATH to include sysconfig data which must be anchored to the - # WASI guest's `/` directory. - args = {"GUEST_DIR": "/", - "HOST_DIR": CHECKOUT, - "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data}", - "PYTHON_WASM": working_dir / "python.wasm"} - # Check dynamically for wasmtime in case it was specified manually via - # `--host-runner`. - if WASMTIME_HOST_RUNNER_VAR in context.host_runner: - if wasmtime := shutil.which("wasmtime"): - args[WASMTIME_VAR_NAME] = wasmtime - else: - raise FileNotFoundError("wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime") - host_runner = context.host_runner.format_map(args) - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} - build_python = os.fsdecode(build_python_path()) - # The path to `configure` MUST be relative, else `python.wasm` is unable - # to find the stdlib due to Python not recognizing that it's being - # executed from within a checkout. - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={context.host_triple}", - f"--build={build_platform()}", - f"--with-build-python={build_python}"] - if pydebug: - configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) - call(configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - quiet=context.quiet) - - python_wasm = working_dir / "python.wasm" - exec_script = working_dir / "python.sh" - with exec_script.open("w", encoding="utf-8") as file: - file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') - exec_script.chmod(0o755) - print(f"🏃‍♀️ Created {exec_script} ... ") - sys.stdout.flush() - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) -def make_wasi_python(context, working_dir): - """Run `make` for the WASI/host build.""" - call(["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - quiet=context.quiet) - - exec_script = working_dir / "python.sh" - subprocess.check_call([exec_script, "--version"]) print( - f"🎉 Use '{exec_script.relative_to(context.init_dir)}' " - "to run CPython in wasm runtime" + "⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; " + "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n", + file=sys.stderr, ) - -def build_all(context): - """Build everything.""" - steps = [configure_build_python, make_build_python, configure_wasi_python, - make_wasi_python] - for step in steps: - step(context) - -def clean_contents(context): - """Delete all files created by this script.""" - if CROSS_BUILD_DIR.exists(): - print(f"🧹 Deleting {CROSS_BUILD_DIR} ...") - shutil.rmtree(CROSS_BUILD_DIR) - - if LOCAL_SETUP.exists(): - with LOCAL_SETUP.open("rb") as file: - if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: - print(f"🧹 Deleting generated {LOCAL_SETUP} ...") - - -def main(): - default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " - # Make sure the stack size will work for a pydebug - # build. - # Use 16 MiB stack. - "--wasm max-wasm-stack=16777216 " - # Enable thread support; causes use of preview1. - #"--wasm threads=y --wasi threads=y " - # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") - - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build-python", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build-python", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for the " - "host/WASI (pydebug builds " - "are inferred from the build " - "Python)") - make_host = subcommands.add_parser("make-host", - help="Run `make` for the host/WASI") - clean = subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") - for subcommand in build, configure_build, make_build, configure_host, make_host: - subcommand.add_argument("--quiet", action="store_true", default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file") - for subcommand in configure_build, configure_host: - subcommand.add_argument("--clean", action="store_true", default=False, - dest="clean", - help="Delete any relevant directories before building") - for subcommand in build, configure_build, configure_host: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") - for subcommand in build, configure_host: - subcommand.add_argument("--wasi-sdk", type=pathlib.Path, - dest="wasi_sdk_path", - default=find_wasi_sdk(), - help="Path to wasi-sdk; defaults to " - "$WASI_SDK_PATH or /opt/wasi-sdk") - subcommand.add_argument("--host-runner", action="store", - default=default_host_runner, dest="host_runner", - help="Command template for running the WASI host " - "(default designed for wasmtime 14 or newer: " - f"`{default_host_runner}`)") - for subcommand in build, configure_host, make_host: - subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1", - help="The target triple for the WASI host build") - - context = parser.parse_args() - context.init_dir = pathlib.Path().absolute() - - dispatch = {"configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build": build_all, - "clean": clean_contents} - dispatch[context.subcommand](context) - - -if __name__ == "__main__": - main() + runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__") diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py new file mode 100644 index 00000000000..3d9a2472d4d --- /dev/null +++ b/Tools/wasm/wasi/__main__.py @@ -0,0 +1,554 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os + +import tomllib + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count +import pathlib +import shutil +import subprocess +import sys +import sysconfig +import tempfile + +HERE = pathlib.Path(__file__).parent + +# Path is: cpython/Tools/wasm/wasi +CHECKOUT = HERE.parent.parent.parent +assert (CHECKOUT / "configure").is_file(), ( + "Please update the location of the file" +) + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +# Build platform can also be found via `config.guess`. +BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = ( + b"# Generated by Tools/wasm/wasi .\n" + b"# Required to statically build extension modules." +) + +WASI_SDK_VERSION = 29 + +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + + +def separator(): + """Print a separator line across the terminal width.""" + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + except subprocess.CalledProcessError: + terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) + print("⎯" * terminal_width) + + +def log(emoji, message, *, spacing=None): + """Print a notification with an emoji. + + If 'spacing' is None, calculate the spacing based on the number of code points + in the emoji as terminals "eat" a space when the emoji has multiple code points. + """ + if spacing is None: + spacing = " " if len(emoji) == 1 else " " + print("".join([emoji, spacing, message])) + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + env_vars = ( + f"\n {key}={item}" for key, item in sorted(env_diff.items()) + ) + log("🌎", f"Environment changes:{''.join(env_vars)}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + nonlocal working_dir + + if callable(working_dir): + working_dir = working_dir(context) + separator() + log("📁", os.fsdecode(working_dir)) + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): + log("🚮", "Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, context=None, quiet=False, logdir=None, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + if context is not None: + quiet = context.quiet + logdir = context.logdir + elif quiet and logdir is None: + raise ValueError("When quiet is True, logdir must be specified") + + log("❯", " ".join(map(str, command)), spacing=" ") + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=logdir, + prefix="cpython-wasi-", + suffix=".log", + ) + stderr = subprocess.STDOUT + log("📝", f"Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) + + return binary + + +def build_python_is_pydebug(): + """Find out if the build Python is a pydebug build.""" + test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" + result = subprocess.run( + [build_python_path(), "-c", test], + capture_output=True, + ) + return bool(result.returncode) + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + log("👍", f"{LOCAL_SETUP} exists ...") + else: + log("⚠️", f"{LOCAL_SETUP} exists, but has unexpected contents") + else: + log("📝", f"Creating {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, context=context) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], context=context) + + binary = build_python_path() + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + log("🎉", f"{binary} {version}") + + +def find_wasi_sdk(config): + """Find the path to the WASI SDK.""" + wasi_sdk_path = None + wasi_sdk_version = config["targets"]["wasi-sdk"] + + if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): + wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) + else: + opt_path = pathlib.Path("/opt") + # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team + # has said they don't plan to ever do a point release and all of their Git tags + # lack the ``.0`` suffix. + # Starting with WASI SDK 23, the tarballs went from containing a directory named + # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. + # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. + potential_sdks = [ + path + for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") + if path.is_dir() + ] + if len(potential_sdks) == 1: + wasi_sdk_path = potential_sdks[0] + elif (default_path := opt_path / "wasi-sdk").is_dir(): + wasi_sdk_path = default_path + + # Starting with WASI SDK 25, a VERSION file is included in the root + # of the SDK directory that we can read to warn folks when they are using + # an unsupported version. + if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file(): + version_details = version_file.read_text(encoding="utf-8") + found_version = version_details.splitlines()[0] + # Make sure there's a trailing dot to avoid false positives if somehow the + # supported version is a prefix of the found version (e.g. `25` and `2567`). + if not found_version.startswith(f"{wasi_sdk_version}."): + major_version = found_version.partition(".")[0] + log( + "⚠️", + f" Found WASI SDK {major_version}, " + f"but WASI SDK {wasi_sdk_version} is the supported version", + ) + + return wasi_sdk_path + + +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = context.wasi_sdk_path + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if not wasi_sdk_path.name.startswith("wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) + + return env + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError( + "WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) + + config_site = os.fsdecode(HERE / "config.site-wasm32-wasi") + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) + lib_dir = os.fsdecode(lib_dirs[0]) + python_version = lib_dir.rpartition("-")[-1] + sysconfig_data_dir = ( + f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + ) + + # Use PYTHONPATH to include sysconfig data which must be anchored to the + # WASI guest's `/` directory. + args = { + "PYTHONPATH": f"/{sysconfig_data_dir}", + "PYTHON_WASM": working_dir / "python.wasm", + } + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if WASMTIME_HOST_RUNNER_VAR in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args[WASMTIME_VAR_NAME] = wasmtime + else: + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime" + ) + host_runner = context.host_runner.format_map(args) + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={BUILD_DIR.name}", + f"--with-build-python={build_python}", + ] + if build_python_is_pydebug(): + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + context=context, + ) + + python_wasm = working_dir / "python.wasm" + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') + exec_script.chmod(0o755) + log("🏃", f"Created {exec_script} (--host-runner)... ") + sys.stdout.flush() + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + context=context, + ) + + exec_script = working_dir / "python.sh" + call([exec_script, "--version"], quiet=False) + log( + "🎉", + f"Use `{exec_script.relative_to(context.init_dir)}` " + "to run CPython w/ the WASI host specified by --host-runner", + ) + + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + log("🧹", f"Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + log("🧹", f"Deleting generated {LOCAL_SETUP} ...") + + +def build_steps(*steps): + """Construct a command from other steps.""" + + def builder(context): + for step in steps: + step(context) + + return builder + + +def main(): + with (HERE / "config.toml").open("rb") as file: + config = tomllib.load(file) + default_wasi_sdk = find_wasi_sdk(config) + default_host_triple = config["targets"]["host-triple"] + default_host_runner = ( + f"{WASMTIME_HOST_RUNNER_VAR} run " + # For setting PYTHONPATH to the sysconfig data directory. + "--env PYTHONPATH={PYTHONPATH} " + # Map the checkout to / to load the stdlib from /Lib. + f"--dir {os.fsdecode(CHECKOUT)}::/ " + # Flags involving --optimize, --codegen, --debug, --wasm, and --wasi can be kept + # in a config file. + # We are using such a file to act as defaults in case a user wants to override + # only some of the settings themselves, make it easy to modify settings + # post-build so that they immediately apply to the Makefile instead of having to + # regenerate it, and allow for easy copying of the settings for anyone else who + # may want to use them. + f"--config {os.fsdecode(HERE / 'wasmtime.toml')}" + ) + default_logdir = pathlib.Path(tempfile.gettempdir()) + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + build_python = subcommands.add_parser( + "build-python", help="Build the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/WASI" + ) + build_host = subcommands.add_parser( + "build-host", help="Build the host/WASI Python" + ) + subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_build, + build_python, + configure_host, + make_host, + build_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + subcommand.add_argument( + "--logdir", + type=pathlib.Path, + default=default_logdir, + help=f"Directory to store log files; defaults to {default_logdir}", + ) + for subcommand in ( + configure_build, + configure_host, + build_python, + build_host, + ): + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) + for subcommand in ( + build, + configure_build, + configure_host, + build_python, + build_host, + ): + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + for subcommand in build, configure_host, build_host: + subcommand.add_argument( + "--wasi-sdk", + type=pathlib.Path, + dest="wasi_sdk_path", + default=default_wasi_sdk, + help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", + ) + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the WASI host; defaults to " + f"`{default_host_runner}`", + ) + for subcommand in build, configure_host, make_host, build_host: + subcommand.add_argument( + "--host-triple", + action="store", + default=default_host_triple, + help="The target triple for the WASI host build; " + f"defaults to {default_host_triple}", + ) + + context = parser.parse_args() + context.init_dir = pathlib.Path().absolute() + + build_build_python = build_steps(configure_build_python, make_build_python) + build_wasi_python = build_steps(configure_wasi_python, make_wasi_python) + + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "build-python": build_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build-host": build_wasi_python, + "build": build_steps(build_build_python, build_wasi_python), + "clean": clean_contents, + } + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/wasi/config.site-wasm32-wasi similarity index 100% rename from Tools/wasm/config.site-wasm32-wasi rename to Tools/wasm/wasi/config.site-wasm32-wasi diff --git a/Tools/wasm/wasi/config.toml b/Tools/wasm/wasi/config.toml new file mode 100644 index 00000000000..7ca2f76f56d --- /dev/null +++ b/Tools/wasm/wasi/config.toml @@ -0,0 +1,6 @@ +# Any data that can vary between Python versions is to be kept in this file. +# This allows for blanket copying of the WASI build code between supported +# Python versions. +[targets] +wasi-sdk = 29 +host-triple = "wasm32-wasip1" diff --git a/Tools/wasm/wasi/wasmtime.toml b/Tools/wasm/wasi/wasmtime.toml new file mode 100644 index 00000000000..5a45e8c3db9 --- /dev/null +++ b/Tools/wasm/wasi/wasmtime.toml @@ -0,0 +1,5 @@ +# https://docs.wasmtime.dev/cli-options.html#cli-options-using-toml-file + +[wasm] +# 32 MiB; big enough for the test suite to pass under a debug build. +max-wasm-stack = 33_554_432 diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py deleted file mode 100755 index bcb80212362..00000000000 --- a/Tools/wasm/wasm_build.py +++ /dev/null @@ -1,932 +0,0 @@ -#!/usr/bin/env python3 -"""Build script for Python on WebAssembly platforms. - - $ ./Tools/wasm/wasm_builder.py emscripten-browser build repl - $ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test - $ ./Tools/wasm/wasm_builder.py wasi build test - -Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking), -"emscripten-browser", and "wasi". - -Emscripten builds require a recent Emscripten SDK. The tools looks for an -activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages -(Debian, Homebrew) are not supported. - -WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH' -and falls back to /opt/wasi-sdk. - -The 'build' Python interpreter must be rebuilt every time Python's byte code -changes. - - ./Tools/wasm/wasm_builder.py --clean build build - -""" -import argparse -import enum -import dataclasses -import logging -import os -import pathlib -import re -import shlex -import shutil -import socket -import subprocess -import sys -import sysconfig -import tempfile -import time -import warnings -import webbrowser - -# for Python 3.8 -from typing import ( - cast, - Any, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, -) - -logger = logging.getLogger("wasm_build") - -SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() -WASMTOOLS = SRCDIR / "Tools" / "wasm" -BUILDDIR = SRCDIR / "builddir" -CONFIGURE = SRCDIR / "configure" -SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local" - -HAS_CCACHE = shutil.which("ccache") is not None - -# path to WASI-SDK root -WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk")) - -# path to Emscripten SDK config file. -# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh". -EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) -EMSDK_MIN_VERSION = (3, 1, 19) -EMSDK_BROKEN_VERSION = { - (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338", - (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393", - (3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720", -} -_MISSING = pathlib.Path("MISSING") - -WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py" - -CLEAN_SRCDIR = f""" -Builds require a clean source directory. Please use a clean checkout or -run "make clean -C '{SRCDIR}'". -""" - -INSTALL_NATIVE = """ -Builds require a C compiler (gcc, clang), make, pkg-config, and development -headers for dependencies like zlib. - -Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev -Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel -""" - -INSTALL_EMSDK = """ -wasm32-emscripten builds need Emscripten SDK. Please follow instructions at -https://emscripten.org/docs/getting_started/downloads.html how to install -Emscripten and how to activate the SDK with "emsdk_env.sh". - - git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk - cd /path/to/emsdk - ./emsdk install latest - ./emsdk activate latest - source /path/to/emsdk_env.sh -""" - -INSTALL_WASI_SDK = """ -wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from -https://github.com/WebAssembly/wasi-sdk/releases and install it to -"/opt/wasi-sdk". Alternatively you can install the SDK in a different location -and point the environment variable WASI_SDK_PATH to the root directory -of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW. -""" - -INSTALL_WASMTIME = """ -wasm32-wasi tests require wasmtime on PATH. Please follow instructions at -https://wasmtime.dev/ to install wasmtime. -""" - - -def parse_emconfig( - emconfig: pathlib.Path = EM_CONFIG, -) -> Tuple[pathlib.Path, pathlib.Path]: - """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS. - - The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" - environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten" - subdirectory with tools like "emconfigure". - """ - if not emconfig.exists(): - return _MISSING, _MISSING - with open(emconfig, encoding="utf-8") as f: - code = f.read() - # EM_CONFIG file is a Python snippet - local: Dict[str, Any] = {} - exec(code, globals(), local) - emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"]) - node_js = pathlib.Path(local["NODE_JS"]) - return emscripten_root, node_js - - -EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig() - - -def read_python_version(configure: pathlib.Path = CONFIGURE) -> str: - """Read PACKAGE_VERSION from configure script - - configure and configure.ac are the canonical source for major and - minor version number. - """ - version_re = re.compile(r"^PACKAGE_VERSION='(\d\.\d+)'") - with configure.open(encoding="utf-8") as f: - for line in f: - mo = version_re.match(line) - if mo: - return mo.group(1) - raise ValueError(f"PACKAGE_VERSION not found in {configure}") - - -PYTHON_VERSION = read_python_version() - - -class ConditionError(ValueError): - def __init__(self, info: str, text: str) -> None: - self.info = info - self.text = text - - def __str__(self) -> str: - return f"{type(self).__name__}: '{self.info}'\n{self.text}" - - -class MissingDependency(ConditionError): - pass - - -class DirtySourceDirectory(ConditionError): - pass - - -@dataclasses.dataclass -class Platform: - """Platform-specific settings - - - CONFIG_SITE override - - configure wrapper (e.g. emconfigure) - - make wrapper (e.g. emmake) - - additional environment variables - - check function to verify SDK - """ - - name: str - pythonexe: str - config_site: Optional[pathlib.PurePath] - configure_wrapper: Optional[pathlib.Path] - make_wrapper: Optional[pathlib.PurePath] - environ: Dict[str, Any] - check: Callable[[], None] - # Used for build_emports(). - ports: Optional[pathlib.PurePath] - cc: Optional[pathlib.PurePath] - - def getenv(self, profile: "BuildProfile") -> Dict[str, Any]: - return self.environ.copy() - - -def _check_clean_src() -> None: - candidates = [ - SRCDIR / "Programs" / "python.o", - SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h", - ] - for candidate in candidates: - if candidate.exists(): - raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR) - - -def _check_native() -> None: - if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]): - raise MissingDependency("cc", INSTALL_NATIVE) - if not shutil.which("make"): - raise MissingDependency("make", INSTALL_NATIVE) - if sys.platform == "linux": - # skip pkg-config check on macOS - if not shutil.which("pkg-config"): - raise MissingDependency("pkg-config", INSTALL_NATIVE) - # zlib is needed to create zip files - for devel in ["zlib"]: - try: - subprocess.check_call(["pkg-config", "--exists", devel]) - except subprocess.CalledProcessError: - raise MissingDependency(devel, INSTALL_NATIVE) from None - _check_clean_src() - - -NATIVE = Platform( - "native", - # macOS has python.exe - pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python", - config_site=None, - configure_wrapper=None, - ports=None, - cc=None, - make_wrapper=None, - environ={}, - check=_check_native, -) - - -def _check_emscripten() -> None: - if EMSCRIPTEN_ROOT is _MISSING: - raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK) - # sanity check - emconfigure = EMSCRIPTEN.configure_wrapper - if emconfigure is not None and not emconfigure.exists(): - raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK) - # version check - version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt" - if not version_txt.exists(): - raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK) - with open(version_txt) as f: - version = f.read().strip().strip('"') - if version.endswith("-git"): - # git / upstream / tot-upstream installation - version = version[:-4] - version_tuple = cast( - Tuple[int, int, int], - tuple(int(v) for v in version.split(".")) - ) - if version_tuple < EMSDK_MIN_VERSION: - raise ConditionError( - os.fspath(version_txt), - f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than " - "minimum required version " - f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.", - ) - broken = EMSDK_BROKEN_VERSION.get(version_tuple) - if broken is not None: - raise ConditionError( - os.fspath(version_txt), - ( - f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known " - f"bugs, see {broken}." - ), - ) - if os.environ.get("PKG_CONFIG_PATH"): - warnings.warn( - "PKG_CONFIG_PATH is set and not empty. emconfigure overrides " - "this environment variable. Use EM_PKG_CONFIG_PATH instead." - ) - _check_clean_src() - - -EMSCRIPTEN = Platform( - "emscripten", - pythonexe="python.js", - config_site=WASMTOOLS / "config.site-wasm32-emscripten", - configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure", - ports=EMSCRIPTEN_ROOT / "embuilder", - cc=EMSCRIPTEN_ROOT / "emcc", - make_wrapper=EMSCRIPTEN_ROOT / "emmake", - environ={ - # workaround for https://github.com/emscripten-core/emscripten/issues/17635 - "TZ": "UTC", - "EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None, - "PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]], - }, - check=_check_emscripten, -) - - -def _check_wasi() -> None: - wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld" - if not wasm_ld.exists(): - raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK) - wasmtime = shutil.which("wasmtime") - if wasmtime is None: - raise MissingDependency("wasmtime", INSTALL_WASMTIME) - _check_clean_src() - - -WASI = Platform( - "wasi", - pythonexe="python.wasm", - config_site=WASMTOOLS / "config.site-wasm32-wasi", - configure_wrapper=WASMTOOLS / "wasi-env", - ports=None, - cc=WASI_SDK_PATH / "bin" / "clang", - make_wrapper=None, - environ={ - "WASI_SDK_PATH": WASI_SDK_PATH, - # workaround for https://github.com/python/cpython/issues/95952 - "HOSTRUNNER": ( - "wasmtime run " - "--wasm max-wasm-stack=16777216 " - "--wasi preview2 " - "--dir {srcdir}::/ " - "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib" - ), - "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], - }, - check=_check_wasi, -) - - -class Host(enum.Enum): - """Target host triplet""" - - wasm32_emscripten = "wasm32-unknown-emscripten" - wasm64_emscripten = "wasm64-unknown-emscripten" - wasm32_wasi = "wasm32-unknown-wasi" - wasm64_wasi = "wasm64-unknown-wasi" - # current platform - build = sysconfig.get_config_var("BUILD_GNU_TYPE") - - @property - def platform(self) -> Platform: - if self.is_emscripten: - return EMSCRIPTEN - elif self.is_wasi: - return WASI - else: - return NATIVE - - @property - def is_emscripten(self) -> bool: - cls = type(self) - return self in {cls.wasm32_emscripten, cls.wasm64_emscripten} - - @property - def is_wasi(self) -> bool: - cls = type(self) - return self in {cls.wasm32_wasi, cls.wasm64_wasi} - - def get_extra_paths(self) -> Iterable[pathlib.PurePath]: - """Host-specific os.environ["PATH"] entries. - - Emscripten's Node version 14.x works well for wasm32-emscripten. - wasm64-emscripten requires more recent v8 version, e.g. node 16.x. - Attempt to use system's node command. - """ - cls = type(self) - if self == cls.wasm32_emscripten: - return [NODE_JS.parent] - elif self == cls.wasm64_emscripten: - # TODO: look for recent node - return [] - else: - return [] - - @property - def emport_args(self) -> List[str]: - """Host-specific port args (Emscripten).""" - cls = type(self) - if self is cls.wasm64_emscripten: - return ["-sMEMORY64=1"] - elif self is cls.wasm32_emscripten: - return ["-sMEMORY64=0"] - else: - return [] - - @property - def embuilder_args(self) -> List[str]: - """Host-specific embuilder args (Emscripten).""" - cls = type(self) - if self is cls.wasm64_emscripten: - return ["--wasm64"] - else: - return [] - - -class EmscriptenTarget(enum.Enum): - """Emscripten-specific targets (--with-emscripten-target)""" - - browser = "browser" - browser_debug = "browser-debug" - node = "node" - node_debug = "node-debug" - - @property - def is_browser(self) -> bool: - cls = type(self) - return self in {cls.browser, cls.browser_debug} - - @property - def emport_args(self) -> List[str]: - """Target-specific port args.""" - cls = type(self) - if self in {cls.browser_debug, cls.node_debug}: - # some libs come in debug and non-debug builds - return ["-O0"] - else: - return ["-O2"] - - -class SupportLevel(enum.Enum): - supported = "tier 3, supported" - working = "working, unsupported" - experimental = "experimental, may be broken" - broken = "broken / unavailable" - - def __bool__(self) -> bool: - cls = type(self) - return self in {cls.supported, cls.working} - - -@dataclasses.dataclass -class BuildProfile: - name: str - support_level: SupportLevel - host: Host - target: Union[EmscriptenTarget, None] = None - dynamic_linking: Union[bool, None] = None - pthreads: Union[bool, None] = None - default_testopts: str = "-j2" - - @property - def is_browser(self) -> bool: - """Is this a browser build?""" - return self.target is not None and self.target.is_browser - - @property - def builddir(self) -> pathlib.Path: - """Path to build directory""" - return BUILDDIR / self.name - - @property - def python_cmd(self) -> pathlib.Path: - """Path to python executable""" - return self.builddir / self.host.platform.pythonexe - - @property - def makefile(self) -> pathlib.Path: - """Path to Makefile""" - return self.builddir / "Makefile" - - @property - def configure_cmd(self) -> List[str]: - """Generate configure command""" - # use relative path, so WASI tests can find lib prefix. - # pathlib.Path.relative_to() does not work here. - configure = os.path.relpath(CONFIGURE, self.builddir) - cmd = [configure, "-C"] - platform = self.host.platform - if platform.configure_wrapper: - cmd.insert(0, os.fspath(platform.configure_wrapper)) - - cmd.append(f"--host={self.host.value}") - cmd.append(f"--build={Host.build.value}") - - if self.target is not None: - assert self.host.is_emscripten - cmd.append(f"--with-emscripten-target={self.target.value}") - - if self.dynamic_linking is not None: - assert self.host.is_emscripten - opt = "enable" if self.dynamic_linking else "disable" - cmd.append(f"--{opt}-wasm-dynamic-linking") - - if self.pthreads is not None: - opt = "enable" if self.pthreads else "disable" - cmd.append(f"--{opt}-wasm-pthreads") - - if self.host != Host.build: - cmd.append(f"--with-build-python={BUILD.python_cmd}") - - if platform.config_site is not None: - cmd.append(f"CONFIG_SITE={platform.config_site}") - - return cmd - - @property - def make_cmd(self) -> List[str]: - """Generate make command""" - cmd = ["make"] - platform = self.host.platform - if platform.make_wrapper: - cmd.insert(0, os.fspath(platform.make_wrapper)) - return cmd - - def getenv(self) -> Dict[str, Any]: - """Generate environ dict for platform""" - env = os.environ.copy() - if hasattr(os, 'process_cpu_count'): - cpu_count = os.process_cpu_count() - else: - cpu_count = os.cpu_count() - env.setdefault("MAKEFLAGS", f"-j{cpu_count}") - platenv = self.host.platform.getenv(self) - for key, value in platenv.items(): - if value is None: - env.pop(key, None) - elif key == "PATH": - # list of path items, prefix with extra paths - new_path: List[pathlib.PurePath] = [] - new_path.extend(self.host.get_extra_paths()) - new_path.extend(value) - env[key] = os.pathsep.join(os.fspath(p) for p in new_path) - elif isinstance(value, str): - env[key] = value.format( - relbuilddir=self.builddir.relative_to(SRCDIR), - srcdir=SRCDIR, - version=PYTHON_VERSION, - ) - else: - env[key] = value - return env - - def _run_cmd( - self, - cmd: Iterable[str], - args: Iterable[str] = (), - cwd: Optional[pathlib.Path] = None, - ) -> int: - cmd = list(cmd) - cmd.extend(args) - if cwd is None: - cwd = self.builddir - logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd) - return subprocess.check_call( - cmd, - cwd=os.fspath(cwd), - env=self.getenv(), - ) - - def _check_execute(self) -> None: - if self.is_browser: - raise ValueError(f"Cannot execute on {self.target}") - - def run_build(self, *args: str) -> None: - """Run configure (if necessary) and make""" - if not self.makefile.exists(): - logger.info("Makefile not found, running configure") - self.run_configure(*args) - self.run_make("all", *args) - - def run_configure(self, *args: str) -> int: - """Run configure script to generate Makefile""" - os.makedirs(self.builddir, exist_ok=True) - return self._run_cmd(self.configure_cmd, args) - - def run_make(self, *args: str) -> int: - """Run make (defaults to build all)""" - return self._run_cmd(self.make_cmd, args) - - def run_pythoninfo(self, *args: str) -> int: - """Run 'make pythoninfo'""" - self._check_execute() - return self.run_make("pythoninfo", *args) - - def run_test(self, target: str, testopts: Optional[str] = None) -> int: - """Run buildbottests""" - self._check_execute() - if testopts is None: - testopts = self.default_testopts - return self.run_make(target, f"TESTOPTS={testopts}") - - def run_py(self, *args: str) -> int: - """Run Python with hostrunner""" - self._check_execute() - return self.run_make( - "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run" - ) - - def run_browser(self, bind: str = "127.0.0.1", port: int = 8000) -> None: - """Run WASM webserver and open build in browser""" - relbuilddir = self.builddir.relative_to(SRCDIR) - url = f"http://{bind}:{port}/{relbuilddir}/python.html" - args = [ - sys.executable, - os.fspath(WASM_WEBSERVER), - "--bind", - bind, - "--port", - str(port), - ] - srv = subprocess.Popen(args, cwd=SRCDIR) - # wait for server - end = time.monotonic() + 3.0 - while time.monotonic() < end and srv.returncode is None: - try: - with socket.create_connection((bind, port), timeout=0.1) as _: - pass - except OSError: - time.sleep(0.01) - else: - break - - webbrowser.open(url) - - try: - srv.wait() - except KeyboardInterrupt: - pass - - def clean(self, all: bool = False) -> None: - """Clean build directory""" - if all: - if self.builddir.exists(): - shutil.rmtree(self.builddir) - elif self.makefile.exists(): - self.run_make("clean") - - def build_emports(self, force: bool = False) -> None: - """Pre-build emscripten ports.""" - platform = self.host.platform - if platform.ports is None or platform.cc is None: - raise ValueError("Need ports and CC command") - - embuilder_cmd = [os.fspath(platform.ports)] - embuilder_cmd.extend(self.host.embuilder_args) - if force: - embuilder_cmd.append("--force") - - ports_cmd = [os.fspath(platform.cc)] - ports_cmd.extend(self.host.emport_args) - if self.target: - ports_cmd.extend(self.target.emport_args) - - if self.dynamic_linking: - # Trigger PIC build. - ports_cmd.append("-sMAIN_MODULE") - embuilder_cmd.append("--pic") - - if self.pthreads: - # Trigger multi-threaded build. - ports_cmd.append("-sUSE_PTHREADS") - - # Pre-build libbz2, libsqlite3, libz, and some system libs. - ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) - # Multi-threaded sqlite3 has different suffix - embuilder_cmd.extend( - ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"] - ) - - self._run_cmd(embuilder_cmd, cwd=SRCDIR) - - with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir: - tmppath = pathlib.Path(tmpdir) - main_c = tmppath / "main.c" - main_js = tmppath / "main.js" - with main_c.open("w") as f: - f.write("int main(void) { return 0; }\n") - args = [ - os.fspath(main_c), - "-o", - os.fspath(main_js), - ] - self._run_cmd(ports_cmd, args, cwd=tmppath) - - -# native build (build Python) -BUILD = BuildProfile( - "build", - support_level=SupportLevel.working, - host=Host.build, -) - -_profiles = [ - BUILD, - # wasm32-emscripten - BuildProfile( - "emscripten-browser", - support_level=SupportLevel.supported, - host=Host.wasm32_emscripten, - target=EmscriptenTarget.browser, - dynamic_linking=True, - ), - BuildProfile( - "emscripten-browser-debug", - support_level=SupportLevel.working, - host=Host.wasm32_emscripten, - target=EmscriptenTarget.browser_debug, - dynamic_linking=True, - ), - BuildProfile( - "emscripten-node-dl", - support_level=SupportLevel.supported, - host=Host.wasm32_emscripten, - target=EmscriptenTarget.node, - dynamic_linking=True, - ), - BuildProfile( - "emscripten-node-dl-debug", - support_level=SupportLevel.working, - host=Host.wasm32_emscripten, - target=EmscriptenTarget.node_debug, - dynamic_linking=True, - ), - BuildProfile( - "emscripten-node-pthreads", - support_level=SupportLevel.supported, - host=Host.wasm32_emscripten, - target=EmscriptenTarget.node, - pthreads=True, - ), - BuildProfile( - "emscripten-node-pthreads-debug", - support_level=SupportLevel.working, - host=Host.wasm32_emscripten, - target=EmscriptenTarget.node_debug, - pthreads=True, - ), - # Emscripten build with both pthreads and dynamic linking is crashing. - BuildProfile( - "emscripten-node-dl-pthreads-debug", - support_level=SupportLevel.broken, - host=Host.wasm32_emscripten, - target=EmscriptenTarget.node_debug, - dynamic_linking=True, - pthreads=True, - ), - # wasm64-emscripten (requires Emscripten >= 3.1.21) - BuildProfile( - "wasm64-emscripten-node-debug", - support_level=SupportLevel.experimental, - host=Host.wasm64_emscripten, - target=EmscriptenTarget.node_debug, - # MEMORY64 is not compatible with dynamic linking - dynamic_linking=False, - pthreads=False, - ), - # wasm32-wasi - BuildProfile( - "wasi", - support_level=SupportLevel.supported, - host=Host.wasm32_wasi, - ), - # wasm32-wasi-threads - BuildProfile( - "wasi-threads", - support_level=SupportLevel.experimental, - host=Host.wasm32_wasi, - pthreads=True, - ), - # no SDK available yet - # BuildProfile( - # "wasm64-wasi", - # support_level=SupportLevel.broken, - # host=Host.wasm64_wasi, - # ), -] - -PROFILES = {p.name: p for p in _profiles} - -parser = argparse.ArgumentParser( - "wasm_build.py", - description=__doc__, - formatter_class=argparse.RawTextHelpFormatter, -) - -parser.add_argument( - "--clean", - "-c", - help="Clean build directories first", - action="store_true", -) - -parser.add_argument( - "--verbose", - "-v", - help="Verbose logging", - action="store_true", -) - -parser.add_argument( - "--silent", - help="Run configure and make in silent mode", - action="store_true", -) - -parser.add_argument( - "--testopts", - help=( - "Additional test options for 'test' and 'hostrunnertest', e.g. " - "--testopts='-v test_os'." - ), - default=None, -) - -# Don't list broken and experimental variants in help -platforms_choices = list(p.name for p in _profiles) + ["cleanall"] -platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"] -parser.add_argument( - "platform", - metavar="PLATFORM", - help=f"Build platform: {', '.join(platforms_help)}", - choices=platforms_choices, -) - -ops = dict( - build="auto build (build 'build' Python, emports, configure, compile)", - configure="run ./configure", - compile="run 'make all'", - pythoninfo="run 'make pythoninfo'", - test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)", - hostrunnertest="run 'make hostrunnertest TESTOPTS=...'", - repl="start interactive REPL / webserver + browser session", - clean="run 'make clean'", - cleanall="remove all build directories", - emports="build Emscripten port with embuilder (only Emscripten)", -) -ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items()) -parser.add_argument( - "ops", - metavar="OP", - help=f"operation (default: build)\n\n{ops_help}", - choices=tuple(ops), - default="build", - nargs="*", -) - - -def main() -> None: - args = parser.parse_args() - logging.basicConfig( - level=logging.INFO if args.verbose else logging.ERROR, - format="%(message)s", - ) - - if args.platform == "cleanall": - for builder in PROFILES.values(): - builder.clean(all=True) - parser.exit(0) - - # additional configure and make args - cm_args = ("--silent",) if args.silent else () - - # nargs=* with default quirk - if args.ops == "build": - args.ops = ["build"] - - builder = PROFILES[args.platform] - try: - builder.host.platform.check() - except ConditionError as e: - parser.error(str(e)) - - if args.clean: - builder.clean(all=False) - - # hack for WASI - if builder.host.is_wasi and not SETUP_LOCAL.exists(): - SETUP_LOCAL.touch() - - # auto-build - if "build" in args.ops: - # check and create build Python - if builder is not BUILD: - logger.info("Auto-building 'build' Python.") - try: - BUILD.host.platform.check() - except ConditionError as e: - parser.error(str(e)) - if args.clean: - BUILD.clean(all=False) - BUILD.run_build(*cm_args) - # build Emscripten ports with embuilder - if builder.host.is_emscripten and "emports" not in args.ops: - builder.build_emports() - - for op in args.ops: - logger.info("\n*** %s %s", args.platform, op) - if op == "build": - builder.run_build(*cm_args) - elif op == "configure": - builder.run_configure(*cm_args) - elif op == "compile": - builder.run_make("all", *cm_args) - elif op == "pythoninfo": - builder.run_pythoninfo(*cm_args) - elif op == "repl": - if builder.is_browser: - builder.run_browser() - else: - builder.run_py() - elif op == "test": - builder.run_test("buildbottest", testopts=args.testopts) - elif op == "hostrunnertest": - builder.run_test("hostrunnertest", testopts=args.testopts) - elif op == "clean": - builder.clean(all=False) - elif op == "cleanall": - builder.clean(all=True) - elif op == "emports": - builder.build_emports(force=args.clean) - else: - raise ValueError(op) - - print(builder.builddir) - parser.exit(0) - - -if __name__ == "__main__": - main() diff --git a/configure b/configure index c51192f12c8..7561fb9c7ad 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.72 for python 3.14. +# Generated by GNU Autoconf 2.72 for python 3.15. # # Report bugs to <https://github.com/python/cpython/issues/>. # @@ -604,8 +604,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='python' PACKAGE_TARNAME='python' -PACKAGE_VERSION='3.14' -PACKAGE_STRING='python 3.14' +PACKAGE_VERSION='3.15' +PACKAGE_STRING='python 3.15' PACKAGE_BUGREPORT='https://github.com/python/cpython/issues/' PACKAGE_URL='' @@ -644,6 +644,7 @@ ac_includes_default="\ ac_header_c_list= ac_subst_vars='LTLIBOBJS MODULE_BLOCK +JIT_STENCILS_H MODULE_XXLIMITED_35_FALSE MODULE_XXLIMITED_35_TRUE MODULE_XXLIMITED_FALSE @@ -813,6 +814,8 @@ MODULE__BISECT_FALSE MODULE__BISECT_TRUE MODULE__ASYNCIO_FALSE MODULE__ASYNCIO_TRUE +MODULE__MATH_INTEGER_FALSE +MODULE__MATH_INTEGER_TRUE MODULE_ARRAY_FALSE MODULE_ARRAY_TRUE MODULE_TIME_FALSE @@ -904,7 +907,6 @@ LDSHARED SHLIB_SUFFIX DSYMUTIL_PATH DSYMUTIL -JIT_STENCILS_H REGEN_JIT_COMMAND UNIVERSAL_ARCH_FLAGS WASM_STDLIB @@ -965,6 +967,7 @@ LDLIBRARY LIBRARY BUILDEXEEXT NO_AS_NEEDED +_Py_STACK_GROWS_DOWN MULTIARCH_CPPFLAGS PLATFORM_TRIPLET MULTIARCH @@ -1009,6 +1012,7 @@ UNIVERSALSDK host_exec_prefix host_prefix MACHDEP +MISSING_STDLIB_CONFIG PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG @@ -1080,6 +1084,7 @@ ac_user_opts=' enable_option_checking with_build_python with_pkg_config +with_missing_stdlib_config enable_universalsdk with_universal_archs with_framework_name @@ -1734,7 +1739,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -'configure' configures python 3.14 to adapt to many kinds of systems. +'configure' configures python 3.15 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1800,7 +1805,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.14:";; + short | recursive ) echo "Configuration of python 3.15:";; esac cat <<\_ACEOF @@ -1819,15 +1824,16 @@ Optional Features: no) --enable-wasm-dynamic-linking Enable dynamic linking support for WebAssembly - (default is no) + (default is no); WASI requires an external dynamic + loader to handle imports --enable-wasm-pthreads Enable pthread emulation for WebAssembly (default is no) --enable-shared enable building a shared Python library (default is no) --enable-profiling enable C-level code profiling with gprof (default is no) - --disable-gil enable experimental support for running without the - GIL (default is no) + --disable-gil enable support for running without the GIL (default + is no) --enable-pystats enable internal statistics gathering (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) @@ -1853,12 +1859,15 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.14 + --with-build-python=python3.15 path to build python binary for cross compiling - (default: _bootstrap_python or python3.14) + (default: _bootstrap_python or python3.15) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) + --with-missing-stdlib-config=FILE + File with custom module error messages for missing + stdlib modules --with-universal-archs=ARCH specify the kind of macOS universal binary that should be created. This option is only valid when @@ -2103,7 +2112,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.14 +python configure 3.15 generated by GNU Autoconf 2.72 Copyright (C) 2023 Free Software Foundation, Inc. @@ -2781,7 +2790,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.14, which was +It was created by python $as_me 3.15, which was generated by GNU Autoconf 2.72. Invocation command line was $ $0$ac_configure_args_raw @@ -3816,7 +3825,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -3892,7 +3901,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.14 +VERSION=3.15 # Version number of Python's own shared library file. @@ -4092,6 +4101,19 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then as_fn_error $? "pkg-config is required" "$LINENO" 5] fi + +# Check whether --with-missing-stdlib-config was given. +if test ${with_missing_stdlib_config+y} +then : + withval=$with_missing_stdlib_config; MISSING_STDLIB_CONFIG="$withval" +else case e in #( + e) MISSING_STDLIB_CONFIG="" + ;; +esac +fi + + + # Set name for machine-dependent library files { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 @@ -4359,7 +4381,7 @@ then : yes) case $ac_sys_system in Darwin) enableval=/Library/Frameworks ;; - iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + iOS) enableval=Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 esac esac @@ -4470,9 +4492,9 @@ then : prefix=$PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" - RESSRCDIR=iOS/Resources + RESSRCDIR=Apple/iOS/Resources - ac_config_files="$ac_config_files iOS/Resources/Info.plist" + ac_config_files="$ac_config_files Apple/iOS/Resources/Info.plist" ;; *) @@ -7211,6 +7233,18 @@ if test x$MULTIARCH != x; then fi +# Guess C stack direction +case $host in #( + hppa*) : + _Py_STACK_GROWS_DOWN=0 ;; #( + *) : + _Py_STACK_GROWS_DOWN=1 ;; +esac + +printf "%s\n" "#define _Py_STACK_GROWS_DOWN $_Py_STACK_GROWS_DOWN" >>confdefs.h + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PEP 11 support tier" >&5 printf %s "checking for PEP 11 support tier... " >&6; } case $host/$ac_cv_cc_name in #( @@ -7254,6 +7288,8 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( x86_64-*-linux-android/clang) : PY_SUPPORT_TIER=3 ;; #( + wasm32-*-emscripten/emcc) : + PY_SUPPORT_TIER=3 ;; #( *) : PY_SUPPORT_TIER=0 @@ -7380,7 +7416,7 @@ then : Emscripten) : ;; #( WASI) : - as_fn_error $? "WASI dynamic linking is not implemented yet." "$LINENO" 5 ;; #( + ;; #( *) : as_fn_error $? "--enable-wasm-dynamic-linking only applies to Emscripten and WASI" "$LINENO" 5 ;; @@ -7773,10 +7809,6 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 printf "%s\n" "$LDLIBRARY" >&6; } -if test "$cross_compiling" = yes; then - RUNSHARED= -fi - # HOSTRUNNER - Program to run CPython for the host platform { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 printf %s "checking HOSTRUNNER... " >&6; } @@ -7901,8 +7933,10 @@ then : as_fn_append HOSTRUNNER " --experimental-wasm-memory64" fi ;; #( - WASI) : - HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( + WASI) : + + as_fn_error $? "HOSTRUNNER must be set when cross-compiling to WASI" "$LINENO" 5 + ;; #( *) : HOSTRUNNER='' ;; @@ -8405,6 +8439,13 @@ fi if test "$Py_OPT" = 'true' ; then + # Check for conflicting CFLAGS=-O0 and --enable-optimizations + case "$CFLAGS" in + *-O0*) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: CFLAGS contains -O0 which may conflict with --enable-optimizations. Consider removing -O0 from CFLAGS for optimal performance." >&5 +printf "%s\n" "$as_me: WARNING: CFLAGS contains -O0 which may conflict with --enable-optimizations. Consider removing -O0 from CFLAGS for optimal performance." >&2;} + ;; + esac # Intentionally not forcing Py_LTO='true' here. Too many toolchains do not # compile working code using it and both test_distutils and test_gdb are # broken when you do manage to get a toolchain that works with it. People @@ -9385,7 +9426,7 @@ fi printf %s "checking BOLT_COMMON_FLAGS... " >&6; } if test -z "${BOLT_COMMON_FLAGS}" then - BOLT_COMMON_FLAGS=" -update-debug-sections -skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1 " + BOLT_COMMON_FLAGS=" -update-debug-sections -skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1,sre_ucs1_match.lto_priv.0/1,sre_ucs2_match.lto_priv.0/1,sre_ucs4_match.lto_priv.0/1 " fi @@ -9601,9 +9642,10 @@ fi as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" - as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" + as_fn_append LINKFORSHARED " -sTEXTDECODER=2" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -10863,8 +10905,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}" - JIT_STENCILS_H="jit_stencils.h" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\"" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" @@ -10872,14 +10913,14 @@ fi ;; esac fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 printf "%s\n" "$tier2_flags $jit_flags" >&6; } if test "$disable_gil" = "yes" -a "$enable_experimental_jit" != "no"; then # GH-133171: This configuration builds the JIT but never actually uses it, # which is surprising (and strictly worse than not building it at all): - as_fn_error $? "--enable-experimental-jit cannot be used with --disable-gil." "$LINENO" 5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: --enable-experimental-jit does not work correctly with --disable-gil." >&5 +printf "%s\n" "$as_me: WARNING: --enable-experimental-jit does not work correctly with --disable-gil." >&2;} fi case "$ac_cv_cc_name" in @@ -11992,7 +12033,7 @@ then : fi -# On Linux, can.h, can/bcm.h, can/j1939.h, can/raw.h require sys/socket.h +# On Linux, can.h, can/bcm.h, can/isotp.h, can/j1939.h, can/raw.h require sys/socket.h # On NetBSD, netcan/can.h requires sys/socket.h ac_fn_c_check_header_compile "$LINENO" "linux/can.h" "ac_cv_header_linux_can_h" " #ifdef HAVE_SYS_SOCKET_H @@ -12015,6 +12056,17 @@ if test "x$ac_cv_header_linux_can_bcm_h" = xyes then : printf "%s\n" "#define HAVE_LINUX_CAN_BCM_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "linux/can/isotp.h" "ac_cv_header_linux_can_isotp_h" " +#ifdef HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif + +" +if test "x$ac_cv_header_linux_can_isotp_h" = xyes +then : + printf "%s\n" "#define HAVE_LINUX_CAN_ISOTP_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "linux/can/j1939.h" "ac_cv_header_linux_can_j1939_h" " #ifdef HAVE_SYS_SOCKET_H @@ -12979,13 +13031,6 @@ if test "$ac_cv_sizeof_off_t" -gt "$ac_cv_sizeof_long" -a \ else have_largefile_support="no" fi -case $ac_sys_system in #( - Emscripten) : - have_largefile_support="no" - ;; #( - *) : - ;; -esac if test "x$have_largefile_support" = xyes then : @@ -13820,6 +13865,8 @@ case $PLATFORM_TRIPLET in #( perf_trampoline=yes ;; #( aarch64-linux-gnu) : perf_trampoline=yes ;; #( + darwin) : + perf_trampoline=yes ;; #( *) : perf_trampoline=no ;; @@ -14052,6 +14099,7 @@ fi + have_uuid=missing for ac_header in uuid.h @@ -14061,6 +14109,7 @@ if test "x$ac_cv_header_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h + for ac_func in uuid_create uuid_enc_be do : as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` @@ -14070,7 +14119,9 @@ then : cat >>confdefs.h <<_ACEOF #define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF - have_uuid=yes + + have_uuid=yes + ac_cv_have_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} @@ -14160,6 +14211,7 @@ if test "x$ac_cv_header_uuid_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + ac_cv_have_uuid_uuid_h=yes py_check_lib_save_LIBS=$LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uuid_generate_time in -luuid" >&5 printf %s "checking for uuid_generate_time in -luuid... " >&6; } @@ -14257,8 +14309,9 @@ fi printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } if test "x$ac_cv_lib_uuid_uuid_generate_time_safe" = xyes then : - have_uuid=yes - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes fi @@ -14302,6 +14355,7 @@ if test "x$ac_cv_header_uuid_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + ac_cv_have_uuid_uuid_h=yes py_check_lib_save_LIBS=$LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uuid_generate_time in -luuid" >&5 printf %s "checking for uuid_generate_time in -luuid... " >&6; } @@ -14399,8 +14453,9 @@ fi printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } if test "x$ac_cv_lib_uuid_uuid_generate_time_safe" = xyes then : - have_uuid=yes - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes fi @@ -14431,10 +14486,16 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } have_uuid=yes - printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h - - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h - + ac_cv_have_uuid_generate_time_safe=yes + # The uuid.h file to include may be <uuid.h> *or* <uuid/uuid.h>. + # Since pkg-config --cflags uuid may return -I/usr/include/uuid, + # it's possible to write '#include <uuid.h>' in _uuidmodule.c, + # assuming that the compiler flags are properly updated. + # + # Ideally, we should have defined HAVE_UUID_H if and only if + # #include <uuid.h> can be written, *without* assuming extra + # include path. + ac_cv_have_uuid_h=yes fi @@ -14455,6 +14516,7 @@ if test "x$ac_cv_func_uuid_generate_time" = xyes then : have_uuid=yes + ac_cv_have_uuid_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} @@ -14465,6 +14527,24 @@ fi done +fi + +if test "x$ac_cv_have_uuid_h" = xyes +then : + printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h + +fi +if test "x$ac_cv_have_uuid_uuid_h" = xyes +then : + printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + +fi +if test "x$ac_cv_have_uuid_generate_time_safe" = xyes +then : + + printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + fi # gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. @@ -14480,6 +14560,164 @@ then : have_uuid=no fi +# gh-132710: The UUID node is fetched by using libuuid when possible +# and cached. While the node is constant within the same process, +# different interpreters may have different values as libuuid may +# randomize the node value if the latter cannot be deduced. +# +# Consumers may define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC +# to indicate that libuuid is unstable and should not be relied +# upon to deduce the MAC address. + + +if test "$have_uuid" = "yes" -a "$HAVE_UUID_GENERATE_TIME_SAFE" = "1" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if uuid_generate_time_safe() node value is stable" >&5 +printf %s "checking if uuid_generate_time_safe() node value is stable... " >&6; } + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though <uuid.h> is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + if test "$cross_compiling" = yes +then : + + +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include <inttypes.h> // PRIu64 + #include <stdint.h> // uint64_t + #include <stdio.h> // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include <uuid.h> + #else + #include <uuid/uuid.h> + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + } +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + + py_cv_uuid_node1=`cat conftest.out` + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though <uuid.h> is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + if test "$cross_compiling" = yes +then : + + +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include <inttypes.h> // PRIu64 + #include <stdint.h> // uint64_t + #include <stdio.h> // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include <uuid.h> + #else + #include <uuid/uuid.h> + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + } +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + + py_cv_uuid_node2=`cat conftest.out` + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + if test -n "$py_cv_uuid_node1" -a "$py_cv_uuid_node1" = "$py_cv_uuid_node2" + then + printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC 1" >>confdefs.h + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: stable" >&5 +printf "%s\n" "stable" >&6; } + else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unstable" >&5 +printf "%s\n" "unstable" >&6; } + fi +fi + # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux @@ -15558,7 +15796,7 @@ fi printf "%s\n" "$ac_cv_ffi_complex_double_supported" >&6; } if test "$ac_cv_ffi_complex_double_supported" = "yes"; then -printf "%s\n" "#define Py_FFI_SUPPORT_C_COMPLEX 1" >>confdefs.h +printf "%s\n" "#define _Py_FFI_SUPPORT_C_COMPLEX 1" >>confdefs.h fi @@ -15566,10 +15804,18 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-system-libmpdec" >&5 printf %s "checking for --with-system-libmpdec... " >&6; } + # Check whether --with-system_libmpdec was given. if test ${with_system_libmpdec+y} then : - withval=$with_system_libmpdec; + withval=$with_system_libmpdec; if test "x$with_system_libmpdec" = xno +then : + LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no +fi else case e in #( e) with_system_libmpdec="yes" ;; esac @@ -15578,8 +15824,6 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 printf "%s\n" "$with_system_libmpdec" >&6; } - - if test "x$with_system_libmpdec" = xyes then : @@ -15657,13 +15901,6 @@ else printf "%s\n" "yes" >&6; } fi -else case e in #( - e) LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - have_mpdec=yes - with_system_libmpdec=no ;; -esac fi if test "x$with_system_libmpdec" = xyes @@ -15710,21 +15947,6 @@ LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS -else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 -printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} ;; -esac -fi - -if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&5 -printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&2;} - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - have_mpdec=yes - with_system_libmpdec=no fi # Disable forced inlining in debug builds, see GH-94847 @@ -18864,15 +19086,27 @@ printf "%s\n" "#define WITH_DTRACE 1" >>confdefs.h # linked into the binary. Correspondingly, dtrace(1) is missing the ELF # generation flag '-G'. We check for presence of this flag, rather than # hardcoding support by OS, in the interest of robustness. + # + # NetBSD DTrace requires the -x nolibs flag to avoid system library conflicts + # and uses header generation for testing instead of object generation. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether DTrace probes require linking" >&5 printf %s "checking whether DTrace probes require linking... " >&6; } if test ${ac_cv_dtrace_link+y} then : printf %s "(cached) " >&6 else case e in #( - e) ac_cv_dtrace_link=no + e) + ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d - "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + case $host in + *netbsd*) + DTRACE_TEST_FLAGS="-x nolibs -h" + ;; + *) + DTRACE_TEST_FLAGS="-G" + ;; + esac + "$DTRACE" $DFLAGS $DTRACE_TEST_FLAGS -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes ;; esac @@ -18882,6 +19116,12 @@ printf "%s\n" "$ac_cv_dtrace_link" >&6; } if test "$ac_cv_dtrace_link" = "yes"; then DTRACE_OBJS="Python/pydtrace.o" fi + # Set NetBSD-specific DTrace flags in DFLAGS + case $host in + *netbsd*) + DFLAGS="$DFLAGS -x nolibs" + ;; + esac fi PLATFORM_HEADERS= @@ -18890,7 +19130,7 @@ PLATFORM_OBJS= case $ac_sys_system in #( Emscripten) : - as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o' + as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_trampoline_wasm.o Python/emscripten_syscalls.o' as_fn_append PLATFORM_HEADERS ' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h' ;; #( *) : @@ -19015,6 +19255,12 @@ if test "x$ac_cv_func_chown" = xyes then : printf "%s\n" "#define HAVE_CHOWN 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "clearenv" "ac_cv_func_clearenv" +if test "x$ac_cv_func_clearenv" = xyes +then : + printf "%s\n" "#define HAVE_CLEARENV 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "clock" "ac_cv_func_clock" if test "x$ac_cv_func_clock" = xyes @@ -19267,6 +19513,12 @@ if test "x$ac_cv_func_getlogin" = xyes then : printf "%s\n" "#define HAVE_GETLOGIN 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "getlogin_r" "ac_cv_func_getlogin_r" +if test "x$ac_cv_func_getlogin_r" = xyes +then : + printf "%s\n" "#define HAVE_GETLOGIN_R 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getpeername" "ac_cv_func_getpeername" if test "x$ac_cv_func_getpeername" = xyes @@ -20170,6 +20422,22 @@ then : fi +# os.statx uses Linux's statx function. AIX also has a function named statx, +# but it's unrelated. Check only on Linux (including Android). +case $ac_sys_system in #( + Linux*) : + ac_fn_c_check_func "$LINENO" "statx" "ac_cv_func_statx" +if test "x$ac_cv_func_statx" = xyes +then : + printf "%s\n" "#define HAVE_STATX 1" >>confdefs.h + +fi + + ;; #( + *) : + ;; +esac + # Force lchmod off for Linux. Linux disallows changing the mode of symbolic # links. Some libc implementations have a stub lchmod implementation that always # returns an error. @@ -21514,19 +21782,19 @@ fi pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for zlib >= 1.2.0" >&5 -printf %s "checking for zlib >= 1.2.0... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for zlib >= 1.2.2.1" >&5 +printf %s "checking for zlib >= 1.2.2.1... " >&6; } if test -n "$ZLIB_CFLAGS"; then pkg_cv_ZLIB_CFLAGS="$ZLIB_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zlib >= 1.2.0\""; } >&5 - ($PKG_CONFIG --exists --print-errors "zlib >= 1.2.0") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zlib >= 1.2.2.1\""; } >&5 + ($PKG_CONFIG --exists --print-errors "zlib >= 1.2.2.1") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_ZLIB_CFLAGS=`$PKG_CONFIG --cflags "zlib >= 1.2.0" 2>/dev/null` + pkg_cv_ZLIB_CFLAGS=`$PKG_CONFIG --cflags "zlib >= 1.2.2.1" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -21538,12 +21806,12 @@ if test -n "$ZLIB_LIBS"; then pkg_cv_ZLIB_LIBS="$ZLIB_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zlib >= 1.2.0\""; } >&5 - ($PKG_CONFIG --exists --print-errors "zlib >= 1.2.0") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"zlib >= 1.2.2.1\""; } >&5 + ($PKG_CONFIG --exists --print-errors "zlib >= 1.2.2.1") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_ZLIB_LIBS=`$PKG_CONFIG --libs "zlib >= 1.2.0" 2>/dev/null` + pkg_cv_ZLIB_LIBS=`$PKG_CONFIG --libs "zlib >= 1.2.2.1" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -21564,9 +21832,9 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - ZLIB_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` + ZLIB_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "zlib >= 1.2.2.1" 2>&1` else - ZLIB_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zlib >= 1.2.0" 2>&1` + ZLIB_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "zlib >= 1.2.2.1" 2>&1` fi # Put the nasty error message in config.log where it belongs echo "$ZLIB_PKG_ERRORS" >&5 @@ -22451,11 +22719,301 @@ fi # Put the nasty error message in config.log where it belongs echo "$LIBZSTD_PKG_ERRORS" >&5 - have_libzstd=no -elif test $pkg_failed = untried; then + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + CFLAGS="$CFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing ZDICT_finalizeDictionary" >&5 +printf %s "checking for library containing ZDICT_finalizeDictionary... " >&6; } +if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ZDICT_finalizeDictionary (void); +int +main (void) +{ +return ZDICT_finalizeDictionary (); + ; + return 0; +} +_ACEOF +for ac_lib in '' zstd +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" +then : + ac_cv_search_ZDICT_finalizeDictionary=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + break +fi +done +if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + +else case e in #( + e) ac_cv_search_ZDICT_finalizeDictionary=no ;; +esac +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ZDICT_finalizeDictionary" >&5 +printf "%s\n" "$ac_cv_search_ZDICT_finalizeDictionary" >&6; } +ac_res=$ac_cv_search_ZDICT_finalizeDictionary +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ZSTD_VERSION_NUMBER >= 1.4.5" >&5 +printf %s "checking ZSTD_VERSION_NUMBER >= 1.4.5... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include "zstd.h" +int +main (void) +{ + + #if ZSTD_VERSION_NUMBER < 10405 + # error "zstd version is too old" + #endif + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + for ac_header in zstd.h zdict.h +do : + as_ac_Header=`printf "%s\n" "ac_cv_header_$ac_header" | sed "$as_sed_sh"` +ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" +if eval test \"x\$"$as_ac_Header"\" = x"yes" +then : + cat >>confdefs.h <<_ACEOF +#define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 +_ACEOF + have_libzstd=yes +else case e in #( + e) have_libzstd=no ;; +esac +fi + +done + +else case e in #( + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } have_libzstd=no + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +else case e in #( + e) have_libzstd=no ;; +esac +fi + + if test "x$have_libzstd" = xyes +then : + + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + CFLAGS="$CFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing ZDICT_finalizeDictionary" >&5 +printf %s "checking for library containing ZDICT_finalizeDictionary... " >&6; } +if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ZDICT_finalizeDictionary (void); +int +main (void) +{ +return ZDICT_finalizeDictionary (); + ; + return 0; +} +_ACEOF +for ac_lib in '' zstd +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" +then : + ac_cv_search_ZDICT_finalizeDictionary=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + break +fi +done +if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + +else case e in #( + e) ac_cv_search_ZDICT_finalizeDictionary=no ;; +esac +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ZDICT_finalizeDictionary" >&5 +printf "%s\n" "$ac_cv_search_ZDICT_finalizeDictionary" >&6; } +ac_res=$ac_cv_search_ZDICT_finalizeDictionary +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ZSTD_VERSION_NUMBER >= 1.4.5" >&5 +printf %s "checking ZSTD_VERSION_NUMBER >= 1.4.5... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include "zstd.h" +int +main (void) +{ + + #if ZSTD_VERSION_NUMBER < 10405 + # error "zstd version is too old" + #endif + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + for ac_header in zstd.h zdict.h +do : + as_ac_Header=`printf "%s\n" "ac_cv_header_$ac_header" | sed "$as_sed_sh"` +ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" +if eval test \"x\$"$as_ac_Header"\" = x"yes" +then : + cat >>confdefs.h <<_ACEOF +#define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 +_ACEOF + have_libzstd=yes +else case e in #( + e) have_libzstd=no ;; +esac +fi + +done + +else case e in #( + e) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_libzstd=no + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +else case e in #( + e) have_libzstd=no ;; +esac +fi + + if test "x$have_libzstd" = xyes +then : + + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + else LIBZSTD_CFLAGS=$pkg_cv_LIBZSTD_CFLAGS LIBZSTD_LIBS=$pkg_cv_LIBZSTD_LIBS @@ -23362,6 +23920,33 @@ fi +ac_fn_check_decl "$LINENO" "MAXLOGNAME" "ac_cv_have_decl_MAXLOGNAME" "#include <sys/param.h> +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_MAXLOGNAME" = xyes +then : + +printf "%s\n" "#define HAVE_MAXLOGNAME 1" >>confdefs.h + +fi + +ac_fn_check_decl "$LINENO" "UT_NAMESIZE" "ac_cv_have_decl_UT_NAMESIZE" "#include <utmp.h> +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_UT_NAMESIZE" = xyes +then : + ac_have_decl=1 +else case e in #( + e) ac_have_decl=0 ;; +esac +fi +printf "%s\n" "#define HAVE_DECL_UT_NAMESIZE $ac_have_decl" >>confdefs.h +if test $ac_have_decl = 1 +then : + +printf "%s\n" "#define HAVE_UT_NAMESIZE 1" >>confdefs.h + +fi + + # check for openpty, login_tty, and forkpty @@ -24588,6 +25173,72 @@ printf "%s\n" "#define HAVE_SIGINFO_T_SI_BAND 1" >>confdefs.h fi +if test "$ac_cv_func_statx" = yes; then + # Some systems have the definitions of the mask bits without having the + # corresponding members in struct statx. Check for members added after Linux + # 4.11 (when statx itself was added). + ac_fn_c_check_member "$LINENO" "struct statx" "stx_mnt_id" "ac_cv_member_struct_statx_stx_mnt_id" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_mnt_id" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_MNT_ID 1" >>confdefs.h + + +fi + + ac_fn_c_check_member "$LINENO" "struct statx" "stx_dio_mem_align" "ac_cv_member_struct_statx_stx_dio_mem_align" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_dio_mem_align" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN 1" >>confdefs.h + + +fi + + # stx_dio_offset_align was added together with stx_dio_mem_align + ac_fn_c_check_member "$LINENO" "struct statx" "stx_subvol" "ac_cv_member_struct_statx_stx_subvol" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_subvol" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_SUBVOL 1" >>confdefs.h + + +fi + + ac_fn_c_check_member "$LINENO" "struct statx" "stx_atomic_write_unit_min" "ac_cv_member_struct_statx_stx_atomic_write_unit_min" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_atomic_write_unit_min" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN 1" >>confdefs.h + + +fi + + # stx_atomic_write_unit_max and stx_atomic_write_segments_max were added + # together with stx_atomic_write_unit_min + ac_fn_c_check_member "$LINENO" "struct statx" "stx_dio_read_offset_align" "ac_cv_member_struct_statx_stx_dio_read_offset_align" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_dio_read_offset_align" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN 1" >>confdefs.h + + +fi + + # stx_atomic_write_unit_max_opt was added in Linux 6.16, but is controlled by + # the STATX_WRITE_ATOMIC mask bit added in Linux 6.11, so having the mask bit + # doesn't imply having the member. + ac_fn_c_check_member "$LINENO" "struct statx" "stx_atomic_write_unit_max_opt" "ac_cv_member_struct_statx_stx_atomic_write_unit_max_opt" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_atomic_write_unit_max_opt" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MAX_OPT 1" >>confdefs.h + + +fi + +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for time.h that defines altzone" >&5 printf %s "checking for time.h that defines altzone... " >&6; } if test ${ac_cv_header_time_altzone+y} @@ -27676,110 +28327,6 @@ printf "%s\n" "#define HAVE_STAT_TV_NSEC2 1" >>confdefs.h fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether year with century should be normalized for strftime" >&5 -printf %s "checking whether year with century should be normalized for strftime... " >&6; } -if test ${ac_cv_normalize_century+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) -if test "$cross_compiling" = yes -then : - ac_cv_normalize_century=yes -else case e in #( - e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include <time.h> -#include <string.h> - -int main(void) -{ - char year[5]; - struct tm date = { - .tm_year = -1801, - .tm_mon = 0, - .tm_mday = 1 - }; - if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { - return 1; - } - return 0; -} - -_ACEOF -if ac_fn_c_try_run "$LINENO" -then : - ac_cv_normalize_century=yes -else case e in #( - e) ac_cv_normalize_century=no ;; -esac -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext ;; -esac -fi - ;; -esac -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_normalize_century" >&5 -printf "%s\n" "$ac_cv_normalize_century" >&6; } -if test "$ac_cv_normalize_century" = yes -then - -printf "%s\n" "#define Py_NORMALIZE_CENTURY 1" >>confdefs.h - -fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C99-compatible strftime specifiers are supported" >&5 -printf %s "checking whether C99-compatible strftime specifiers are supported... " >&6; } -if test ${ac_cv_strftime_c99_support+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) -if test "$cross_compiling" = yes -then : - ac_cv_strftime_c99_support= -else case e in #( - e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include <time.h> -#include <string.h> - -int main(void) -{ - char full_date[11]; - struct tm date = { - .tm_year = 0, - .tm_mon = 0, - .tm_mday = 1 - }; - if (strftime(full_date, sizeof(full_date), "%F", &date) && !strcmp(full_date, "1900-01-01")) { - return 0; - } - return 1; -} - -_ACEOF -if ac_fn_c_try_run "$LINENO" -then : - ac_cv_strftime_c99_support=yes -else case e in #( - e) as_fn_error $? "Python requires C99-compatible strftime specifiers" "$LINENO" 5 ;; -esac -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext ;; -esac -fi - ;; -esac -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_strftime_c99_support" >&5 -printf "%s\n" "$ac_cv_strftime_c99_support" >&6; } - have_curses=no have_panel=no @@ -29384,7 +29931,7 @@ then : if test "$withval" = yes then -printf "%s\n" "#define Py_TAIL_CALL_INTERP 1" >>confdefs.h +printf "%s\n" "#define _Py_TAIL_CALL_INTERP 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } @@ -29392,7 +29939,7 @@ fi if test "$withval" = no then -printf "%s\n" "#define Py_TAIL_CALL_INTERP 0" >>confdefs.h +printf "%s\n" "#define _Py_TAIL_CALL_INTERP 0" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } @@ -29426,9 +29973,6 @@ printf "%s\n" "#define Py_REMOTE_DEBUG 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } else - -printf "%s\n" "#define Py_REMOTE_DEBUG 0" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } fi @@ -29458,6 +30002,7 @@ SRCDIRS="\ Modules/_hacl \ Modules/_io \ Modules/_multiprocessing \ + Modules/_remote_debugging \ Modules/_sqlite \ Modules/_sre \ Modules/_testcapi \ @@ -30358,8 +30903,8 @@ main (void) OBJ_nid2sn(NID_md5); OBJ_nid2sn(NID_sha1); + OBJ_nid2sn(NID_sha512); OBJ_nid2sn(NID_sha3_512); - OBJ_nid2sn(NID_blake2b512); EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); ; @@ -30689,9 +31234,7 @@ case $ac_sys_system in #( - py_cv_module_fcntl=n/a py_cv_module_readline=n/a - py_cv_module_termios=n/a py_cv_module_=n/a ;; #( @@ -30822,6 +31365,28 @@ then : +fi + + + if test "$py_cv_module__math_integer" != "n/a" +then : + py_cv_module__math_integer=yes +fi + if test "$py_cv_module__math_integer" = yes; then + MODULE__MATH_INTEGER_TRUE= + MODULE__MATH_INTEGER_FALSE='#' +else + MODULE__MATH_INTEGER_TRUE='#' + MODULE__MATH_INTEGER_FALSE= +fi + + as_fn_append MODULE_BLOCK "MODULE__MATH_INTEGER_STATE=$py_cv_module__math_integer$as_nl" + if test "x$py_cv_module__math_integer" = xyes +then : + + + + fi @@ -32076,6 +32641,14 @@ LIBHACL_CFLAGS="${LIBHACL_FLAG_I} ${LIBHACL_FLAG_D} \$(PY_STDMODULE_CFLAGS) \$(C LIBHACL_LDFLAGS= # for now, no specific linker flags are needed +if test "$UNIVERSAL_ARCHS" = "universal2" -o \ + \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \) +then + use_hacl_universal2_impl=yes +else + use_hacl_universal2_impl=no +fi + # The SIMD files use aligned_alloc, which is not available on older versions of # Android. # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. @@ -32121,7 +32694,7 @@ then : LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2" -printf "%s\n" "#define HACL_CAN_COMPILE_SIMD128 1" >>confdefs.h +printf "%s\n" "#define _Py_HACL_CAN_COMPILE_VEC128 1" >>confdefs.h # macOS universal2 builds *support* the -msse etc flags because they're @@ -32129,7 +32702,7 @@ printf "%s\n" "#define HACL_CAN_COMPILE_SIMD128 1" >>confdefs.h # isn't great, so it's disabled on ARM64. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD128 implementation" >&5 printf %s "checking for HACL* SIMD128 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } @@ -32197,7 +32770,7 @@ then : LIBHACL_SIMD256_FLAGS="-mavx2" -printf "%s\n" "#define HACL_CAN_COMPILE_SIMD256 1" >>confdefs.h +printf "%s\n" "#define _Py_HACL_CAN_COMPILE_VEC256 1" >>confdefs.h # macOS universal2 builds *support* the -mavx2 compiler flag because it's @@ -32206,7 +32779,7 @@ printf "%s\n" "#define HACL_CAN_COMPILE_SIMD256 1" >>confdefs.h # wrapped implementation if we're building for universal2. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD256 implementation" >&5 printf %s "checking for HACL* SIMD256 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } @@ -32682,6 +33255,18 @@ fi printf "%s\n" "$py_cv_module__decimal" >&6; } +if test "x$with_system_libmpdec" = xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library." >&2;} +fi +if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdec found; falling back to pure-Python version for the decimal module" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdec found; falling back to pure-Python version for the decimal module" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _dbm" >&5 printf %s "checking for stdlib extension module _dbm... " >&6; } if test "$py_cv_module__dbm" != "n/a" @@ -33748,6 +34333,40 @@ fi printf "%s\n" "$py_cv_module_xxlimited_35" >&6; } +# Determine JIT stencils header files based on target platform +JIT_STENCILS_H="" +if test "x$enable_experimental_jit" = xno +then : + +else case e in #( + e) case "$host" in + aarch64-apple-darwin*) + JIT_STENCILS_H="jit_stencils-aarch64-apple-darwin.h" + ;; + x86_64-apple-darwin*) + JIT_STENCILS_H="jit_stencils-x86_64-apple-darwin.h" + ;; + aarch64-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-aarch64-pc-windows-msvc.h" + ;; + i686-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-i686-pc-windows-msvc.h" + ;; + x86_64-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" + ;; + aarch64-*-linux-gnu) + JIT_STENCILS_H="jit_stencils-aarch64-unknown-linux-gnu.h" + ;; + x86_64-*-linux-gnu) + JIT_STENCILS_H="jit_stencils-x86_64-unknown-linux-gnu.h" + ;; + esac ;; +esac +fi + + + # substitute multiline block, must come after last PY_STDLIB_MOD() @@ -33881,6 +34500,10 @@ if test -z "${MODULE_ARRAY_TRUE}" && test -z "${MODULE_ARRAY_FALSE}"; then as_fn_error $? "conditional \"MODULE_ARRAY\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE__MATH_INTEGER_TRUE}" && test -z "${MODULE__MATH_INTEGER_FALSE}"; then + as_fn_error $? "conditional \"MODULE__MATH_INTEGER\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${MODULE__ASYNCIO_TRUE}" && test -z "${MODULE__ASYNCIO_FALSE}"; then as_fn_error $? "conditional \"MODULE__ASYNCIO\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -34598,7 +35221,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.14, which was +This file was extended by python $as_me 3.15, which was generated by GNU Autoconf 2.72. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -34662,7 +35285,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -python config.status 3.14 +python config.status 3.15 configured by $0, generated by GNU Autoconf 2.72, with options \\"\$ac_cs_config\\" @@ -34790,7 +35413,7 @@ do "Mac/PythonLauncher/Makefile") CONFIG_FILES="$CONFIG_FILES Mac/PythonLauncher/Makefile" ;; "Mac/Resources/framework/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/framework/Info.plist" ;; "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;; - "iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES iOS/Resources/Info.plist" ;; + "Apple/iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES Apple/iOS/Resources/Info.plist" ;; "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;; "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; diff --git a/configure.ac b/configure.ac index a7b2f62579b..fa24bc78a26 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ dnl to regenerate the configure script. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define([PYTHON_VERSION], [3.14]) +m4_define([PYTHON_VERSION], [3.15]) AC_PREREQ([2.72]) @@ -205,7 +205,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python], + [python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST([PYTHON_FOR_REGEN]) @@ -307,6 +307,15 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then AC_MSG_ERROR([pkg-config is required])] fi +dnl Allow distributors to provide custom missing stdlib module error messages +AC_ARG_WITH([missing-stdlib-config], + [AS_HELP_STRING([--with-missing-stdlib-config=FILE], + [File with custom module error messages for missing stdlib modules])], + [MISSING_STDLIB_CONFIG="$withval"], + [MISSING_STDLIB_CONFIG=""] +) +AC_SUBST([MISSING_STDLIB_CONFIG]) + # Set name for machine-dependent library files AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) AC_MSG_CHECKING([MACHDEP]) @@ -559,7 +568,7 @@ AC_ARG_ENABLE([framework], yes) case $ac_sys_system in Darwin) enableval=/Library/Frameworks ;; - iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + iOS) enableval=Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; *) AC_MSG_ERROR([Unknown platform for framework build]) esac esac @@ -666,9 +675,9 @@ AC_ARG_ENABLE([framework], prefix=$PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" - RESSRCDIR=iOS/Resources + RESSRCDIR=Apple/iOS/Resources - AC_CONFIG_FILES([iOS/Resources/Info.plist]) + AC_CONFIG_FILES([Apple/iOS/Resources/Info.plist]) ;; *) AC_MSG_ERROR([Unknown platform for framework build]) @@ -1202,33 +1211,42 @@ if test x$MULTIARCH != x; then fi AC_SUBST([MULTIARCH_CPPFLAGS]) +# Guess C stack direction +AS_CASE([$host], + [hppa*], [_Py_STACK_GROWS_DOWN=0], + [_Py_STACK_GROWS_DOWN=1]) +AC_DEFINE_UNQUOTED([_Py_STACK_GROWS_DOWN], [$_Py_STACK_GROWS_DOWN], + [Define to 1 if the machine stack grows down (default); 0 if it grows up.]) +AC_SUBST([_Py_STACK_GROWS_DOWN]) + dnl Support tiers according to https://peps.python.org/pep-0011/ dnl dnl NOTE: Windows support tiers are defined in PC/pyconfig.h. dnl AC_MSG_CHECKING([for PEP 11 support tier]) AS_CASE([$host/$ac_cv_cc_name], - [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc - [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version - [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version - [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC - [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC + [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc + [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version + [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version + [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC + [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC - [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang - [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], - [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc - [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang - [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang + [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang + [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], + [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc + [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang + [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang - [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC - [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc - [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang - [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc - [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 - [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 - [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 - [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 - [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC + [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc + [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang + [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc + [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 + [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 + [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [wasm32-*-emscripten/emcc], [PY_SUPPORT_TIER=3], dnl Emscripten [PY_SUPPORT_TIER=0] ) @@ -1305,11 +1323,11 @@ dnl See https://emscripten.org/docs/compiling/Dynamic-Linking.html AC_MSG_CHECKING([for --enable-wasm-dynamic-linking]) AC_ARG_ENABLE([wasm-dynamic-linking], [AS_HELP_STRING([--enable-wasm-dynamic-linking], - [Enable dynamic linking support for WebAssembly (default is no)])], + [Enable dynamic linking support for WebAssembly (default is no); WASI requires an external dynamic loader to handle imports])], [ AS_CASE([$ac_sys_system], [Emscripten], [], - [WASI], [AC_MSG_ERROR([WASI dynamic linking is not implemented yet.])], + [WASI], [], [AC_MSG_ERROR([--enable-wasm-dynamic-linking only applies to Emscripten and WASI])] ) ], [ @@ -1621,10 +1639,6 @@ else # shared is disabled fi AC_MSG_RESULT([$LDLIBRARY]) -if test "$cross_compiling" = yes; then - RUNSHARED= -fi - # HOSTRUNNER - Program to run CPython for the host platform AC_MSG_CHECKING([HOSTRUNNER]) if test -z "$HOSTRUNNER" @@ -1635,10 +1649,9 @@ then HOSTRUNNER="$NODE" AS_VAR_IF([host_cpu], [wasm64], [AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-memory64"])]) ], - dnl TODO: support other WASI runtimes - dnl wasmtime starts the process with "/" as CWD. For OOT builds add the - dnl directory containing _sysconfigdata to PYTHONPATH. - [WASI], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=16777216 --wasi preview2=n --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], + [WASI], [ + AC_MSG_ERROR([HOSTRUNNER must be set when cross-compiling to WASI]) + ], [HOSTRUNNER=''] ) fi @@ -1716,7 +1729,7 @@ ABI_THREAD="" # --disable-gil AC_MSG_CHECKING([for --disable-gil]) AC_ARG_ENABLE([gil], - [AS_HELP_STRING([--disable-gil], [enable experimental support for running without the GIL (default is no)])], + [AS_HELP_STRING([--disable-gil], [enable support for running without the GIL (default is no)])], [AS_VAR_IF([enable_gil], [yes], [disable_gil=no], [disable_gil=yes])], [disable_gil=no] ) AC_MSG_RESULT([$disable_gil]) @@ -1822,6 +1835,14 @@ fi], [AC_MSG_RESULT([no])]) if test "$Py_OPT" = 'true' ; then + # Check for conflicting CFLAGS=-O0 and --enable-optimizations + case "$CFLAGS" in + *-O0*) + AC_MSG_WARN([m4_normalize([ + CFLAGS contains -O0 which may conflict with --enable-optimizations. + Consider removing -O0 from CFLAGS for optimal performance.])]) + ;; + esac # Intentionally not forcing Py_LTO='true' here. Too many toolchains do not # compile working code using it and both test_distutils and test_gdb are # broken when you do manage to get a toolchain that works with it. People @@ -2154,7 +2175,8 @@ then dnl At least LLVM 19.x doesn't support computed gotos in PIC compiled code. dnl Exclude functions containing computed gotos. dnl TODO this may be fixed in LLVM 20.x via https://github.com/llvm/llvm-project/pull/120267. - [-skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1] + dnl GCC's LTO creates .lto_priv.0 clones of these functions. + [-skip-funcs=_PyEval_EvalFrameDefault,sre_ucs1_match/1,sre_ucs2_match/1,sre_ucs4_match/1,sre_ucs1_match.lto_priv.0/1,sre_ucs2_match.lto_priv.0/1,sre_ucs4_match.lto_priv.0/1] ")] ) fi @@ -2334,9 +2356,11 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) + dnl Avoid bugs in JS fallback string decoding path + AS_VAR_APPEND([LINKFORSHARED], [" -sTEXTDECODER=2"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) @@ -2776,20 +2800,18 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}"]) - AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\""]) AS_VAR_IF([Py_DEBUG], [true], [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], [])]) AC_SUBST([REGEN_JIT_COMMAND]) -AC_SUBST([JIT_STENCILS_H]) AC_MSG_RESULT([$tier2_flags $jit_flags]) if test "$disable_gil" = "yes" -a "$enable_experimental_jit" != "no"; then # GH-133171: This configuration builds the JIT but never actually uses it, # which is surprising (and strictly worse than not building it at all): - AC_MSG_ERROR([--enable-experimental-jit cannot be used with --disable-gil.]) + AC_MSG_WARN([--enable-experimental-jit does not work correctly with --disable-gil.]) fi case "$ac_cv_cc_name" in @@ -3043,10 +3065,10 @@ AC_CHECK_HEADERS([linux/vm_sockets.h], [], [], [ #endif ]) -# On Linux, can.h, can/bcm.h, can/j1939.h, can/raw.h require sys/socket.h +# On Linux, can.h, can/bcm.h, can/isotp.h, can/j1939.h, can/raw.h require sys/socket.h # On NetBSD, netcan/can.h requires sys/socket.h AC_CHECK_HEADERS( -[linux/can.h linux/can/bcm.h linux/can/j1939.h linux/can/raw.h netcan/can.h], +[linux/can.h linux/can/bcm.h linux/can/isotp.h linux/can/j1939.h linux/can/raw.h netcan/can.h], [], [], [ #ifdef HAVE_SYS_SOCKET_H #include <sys/socket.h> @@ -3172,10 +3194,6 @@ if test "$ac_cv_sizeof_off_t" -gt "$ac_cv_sizeof_long" -a \ else have_largefile_support="no" fi -dnl LFS does not work with Emscripten 3.1 -AS_CASE([$ac_sys_system], - [Emscripten], [have_largefile_support="no"] -) AS_VAR_IF([have_largefile_support], [yes], [ AC_DEFINE([HAVE_LARGEFILE_SUPPORT], [1], [Defined to enable large file support when an off_t is bigger than a long @@ -3693,12 +3711,13 @@ case "$ac_sys_system" in esac AC_MSG_RESULT([$SHLIBS]) -dnl perf trampoline is Linux specific and requires an arch-specific +dnl perf trampoline is Linux and macOS specific and requires an arch-specific dnl trampoline in assembly. AC_MSG_CHECKING([perf trampoline]) AS_CASE([$PLATFORM_TRIPLET], [x86_64-linux-gnu], [perf_trampoline=yes], [aarch64-linux-gnu], [perf_trampoline=yes], + [darwin], [perf_trampoline=yes], [perf_trampoline=no] ) AC_MSG_RESULT([$perf_trampoline]) @@ -3740,15 +3759,17 @@ dnl check for uuid dependencies AH_TEMPLATE([HAVE_UUID_H], [Define to 1 if you have the <uuid.h> header file.]) AH_TEMPLATE([HAVE_UUID_UUID_H], [Define to 1 if you have the <uuid/uuid.h> header file.]) AH_TEMPLATE([HAVE_UUID_GENERATE_TIME_SAFE], [Define if uuid_generate_time_safe() exists.]) +AH_TEMPLATE([HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC], [Define if uuid_generate_time_safe() is able to deduce a MAC address.]) have_uuid=missing dnl AIX provides support for RFC4122 (uuid) in libc.a starting with AIX 6.1 dnl (anno 2007). FreeBSD and OpenBSD provides support in libc as well. dnl Little-endian FreeBSD, OpenBSD and NetBSD needs encoding into an octet dnl stream in big-endian byte-order -AC_CHECK_HEADERS([uuid.h], - [AC_CHECK_FUNCS([uuid_create uuid_enc_be], - [have_uuid=yes +AC_CHECK_HEADERS([uuid.h], [ + AC_CHECK_FUNCS([uuid_create uuid_enc_be], [ + have_uuid=yes + ac_cv_have_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} ]) @@ -3758,19 +3779,29 @@ AS_VAR_IF([have_uuid], [missing], [ PKG_CHECK_MODULES( [LIBUUID], [uuid >= 2.20], [dnl linux-util's libuuid has uuid_generate_time_safe() since v2.20 (2011) - dnl and provides <uuid.h>. + dnl and provides <uuid.h> assuming specific include paths are given have_uuid=yes - AC_DEFINE([HAVE_UUID_H], [1]) - AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) + ac_cv_have_uuid_generate_time_safe=yes + # The uuid.h file to include may be <uuid.h> *or* <uuid/uuid.h>. + # Since pkg-config --cflags uuid may return -I/usr/include/uuid, + # it's possible to write '#include <uuid.h>' in _uuidmodule.c, + # assuming that the compiler flags are properly updated. + # + # Ideally, we should have defined HAVE_UUID_H if and only if + # #include <uuid.h> can be written, *without* assuming extra + # include path. + ac_cv_have_uuid_h=yes ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" LIBS="$LIBS $LIBUUID_LIBS" AC_CHECK_HEADERS([uuid/uuid.h], [ + ac_cv_have_uuid_uuid_h=yes PY_CHECK_LIB([uuid], [uuid_generate_time], [have_uuid=yes]) - PY_CHECK_LIB([uuid], [uuid_generate_time_safe], - [have_uuid=yes - AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) ]) ]) + PY_CHECK_LIB([uuid], [uuid_generate_time_safe], [ + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes + ])]) AS_VAR_IF([have_uuid], [yes], [ LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-"-luuid"} @@ -3785,12 +3816,19 @@ AS_VAR_IF([have_uuid], [missing], [ AC_CHECK_HEADERS([uuid/uuid.h], [ AC_CHECK_FUNC([uuid_generate_time], [ have_uuid=yes + ac_cv_have_uuid_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} ]) ]) ]) +AS_VAR_IF([ac_cv_have_uuid_h], [yes], [AC_DEFINE([HAVE_UUID_H], [1])]) +AS_VAR_IF([ac_cv_have_uuid_uuid_h], [yes], [AC_DEFINE([HAVE_UUID_UUID_H], [1])]) +AS_VAR_IF([ac_cv_have_uuid_generate_time_safe], [yes], [ + AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) +]) + # gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. # This restriction inhibits the proper generation of time-based UUIDs. if test "$ac_sys_system" = "NetBSD"; then @@ -3800,6 +3838,68 @@ fi AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) +# gh-132710: The UUID node is fetched by using libuuid when possible +# and cached. While the node is constant within the same process, +# different interpreters may have different values as libuuid may +# randomize the node value if the latter cannot be deduced. +# +# Consumers may define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC +# to indicate that libuuid is unstable and should not be relied +# upon to deduce the MAC address. +AC_DEFUN([PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC], [WITH_SAVE_ENV([ + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though <uuid.h> is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + AC_RUN_IFELSE([AC_LANG_SOURCE([[ + #include <inttypes.h> // PRIu64 + #include <stdint.h> // uint64_t + #include <stdio.h> // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include <uuid.h> + #else + #include <uuid/uuid.h> + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + }]])], [ + AS_VAR_SET([$1], [`cat conftest.out`]) + ], [], [] + )])]) + +if test "$have_uuid" = "yes" -a "$HAVE_UUID_GENERATE_TIME_SAFE" = "1" +then + AC_MSG_CHECKING([if uuid_generate_time_safe() node value is stable]) + PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC([py_cv_uuid_node1]) + PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC([py_cv_uuid_node2]) + if test -n "$py_cv_uuid_node1" -a "$py_cv_uuid_node1" = "$py_cv_uuid_node2" + then + AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC], [1]) + AC_MSG_RESULT([stable]) + else + AC_MSG_RESULT([unstable]) + fi +fi + # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux @@ -4086,28 +4186,28 @@ int main(void) [ac_cv_ffi_complex_double_supported=no]) ])]) if test "$ac_cv_ffi_complex_double_supported" = "yes"; then - AC_DEFINE([Py_FFI_SUPPORT_C_COMPLEX], [1], + AC_DEFINE([_Py_FFI_SUPPORT_C_COMPLEX], [1], [Defined if _Complex C type can be used with libffi.]) fi # Check for use of the system libmpdec library AC_MSG_CHECKING([for --with-system-libmpdec]) -AC_ARG_WITH( - [system_libmpdec], - [AS_HELP_STRING( - [--with-system-libmpdec], - [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] - )], - [], - [with_system_libmpdec="yes"]) -AC_MSG_RESULT([$with_system_libmpdec]) - AC_DEFUN([USE_BUNDLED_LIBMPDEC], [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" have_mpdec=yes with_system_libmpdec=no]) +AC_ARG_WITH( + [system_libmpdec], + [AS_HELP_STRING( + [--with-system-libmpdec], + [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] + )], + [AS_IF([test "x$with_system_libmpdec" = xno], + [USE_BUNDLED_LIBMPDEC()])], + [with_system_libmpdec="yes"]) +AC_MSG_RESULT([$with_system_libmpdec]) AS_VAR_IF( [with_system_libmpdec], [yes], @@ -4115,8 +4215,7 @@ AS_VAR_IF( [LIBMPDEC], [libmpdec >= 2.5.0], [], [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} - LIBMPDEC_INTERNAL=])], - [USE_BUNDLED_LIBMPDEC()]) + LIBMPDEC_INTERNAL=])]) AS_VAR_IF([with_system_libmpdec], [yes], [WITH_SAVE_ENV([ @@ -4132,16 +4231,7 @@ AS_VAR_IF([with_system_libmpdec], [yes], ], [const char *x = mpd_version();])], [have_mpdec=yes], [have_mpdec=no]) - ])], - [AC_MSG_WARN([m4_normalize([ - the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; - consider using a system installed mpdecimal library.])])]) - -AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], - [AC_MSG_WARN([m4_normalize([ - no system libmpdecimal found; falling back to bundled libmpdecimal - (deprecated and scheduled for removal in Python 3.15)])]) - USE_BUNDLED_LIBMPDEC()]) + ])]) # Disable forced inlining in debug builds, see GH-94847 AS_VAR_IF( @@ -5033,16 +5123,33 @@ then # linked into the binary. Correspondingly, dtrace(1) is missing the ELF # generation flag '-G'. We check for presence of this flag, rather than # hardcoding support by OS, in the interest of robustness. + # + # NetBSD DTrace requires the -x nolibs flag to avoid system library conflicts + # and uses header generation for testing instead of object generation. AC_CACHE_CHECK([whether DTrace probes require linking], - [ac_cv_dtrace_link], [dnl + [ac_cv_dtrace_link], [ ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d - "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + case $host in + *netbsd*) + DTRACE_TEST_FLAGS="-x nolibs -h" + ;; + *) + DTRACE_TEST_FLAGS="-G" + ;; + esac + "$DTRACE" $DFLAGS $DTRACE_TEST_FLAGS -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes ]) if test "$ac_cv_dtrace_link" = "yes"; then DTRACE_OBJS="Python/pydtrace.o" fi + # Set NetBSD-specific DTrace flags in DFLAGS + case $host in + *netbsd*) + DFLAGS="$DFLAGS -x nolibs" + ;; + esac fi dnl Platform-specific C and header files. @@ -5051,7 +5158,7 @@ PLATFORM_OBJS= AS_CASE([$ac_sys_system], [Emscripten], [ - AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o']) + AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_trampoline_wasm.o Python/emscripten_syscalls.o']) AS_VAR_APPEND([PLATFORM_HEADERS], [' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h']) ], ) @@ -5133,12 +5240,13 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ - accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ + accept4 alarm bind_textdomain_codeset chmod chown clearenv \ + clock closefrom close_range confstr \ copy_file_range ctermid dladdr dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ - getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin \ + getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin getlogin_r \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ @@ -5148,7 +5256,7 @@ AC_CHECK_FUNCS([ \ posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 process_vm_readv \ pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ - pthread_kill pthread_get_name_np pthread_getname_np pthread_set_name_np + pthread_kill pthread_get_name_np pthread_getname_np pthread_set_name_np \ pthread_setname_np pthread_getattr_np \ ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ @@ -5163,6 +5271,12 @@ AC_CHECK_FUNCS([ \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) +# os.statx uses Linux's statx function. AIX also has a function named statx, +# but it's unrelated. Check only on Linux (including Android). +AS_CASE([$ac_sys_system], + [Linux*], [AC_CHECK_FUNCS([statx])] +) + # Force lchmod off for Linux. Linux disallows changing the mode of symbolic # links. Some libc implementations have a stub lchmod implementation that always # returns an error. @@ -5330,7 +5444,7 @@ AH_TEMPLATE([HAVE_ZLIB_COPY], [Define if the zlib library has inflateCopy]) dnl detect zlib from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([ZLIB], [-sUSE_ZLIB]) -PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0], [ +PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.2.1], [ have_zlib=yes dnl zlib 1.2.0 (2003) added inflateCopy AC_DEFINE([HAVE_ZLIB_COPY], [1]) @@ -5387,7 +5501,33 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ ]) dnl zstd 1.4.5 stabilised ZDICT_finalizeDictionary -PKG_CHECK_MODULES([LIBZSTD], [libzstd >= 1.4.5], [have_libzstd=yes], [have_libzstd=no]) +PKG_CHECK_MODULES([LIBZSTD], [libzstd >= 1.4.5], [have_libzstd=yes], [ + WITH_SAVE_ENV([ + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + CFLAGS="$CFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + AC_SEARCH_LIBS([ZDICT_finalizeDictionary], [zstd], [ + AC_MSG_CHECKING([ZSTD_VERSION_NUMBER >= 1.4.5]) + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([@%:@include "zstd.h"], [ + #if ZSTD_VERSION_NUMBER < 10405 + # error "zstd version is too old" + #endif + ]) + ], [ + AC_MSG_RESULT([yes]) + AC_CHECK_HEADERS([zstd.h zdict.h], [have_libzstd=yes], [have_libzstd=no]) + ], [ + AC_MSG_RESULT([no]) + have_libzstd=no + ]) + ], [have_libzstd=no]) + AS_VAR_IF([have_libzstd], [yes], [ + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + ]) + ]) +]) dnl PY_CHECK_NETDB_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [@%:@include <netdb.h>])]) @@ -5431,6 +5571,18 @@ PY_CHECK_FUNC([setgroups], [ #endif ]) +AC_CHECK_DECL([MAXLOGNAME], + [AC_DEFINE([HAVE_MAXLOGNAME], [1], + [Define if you have the 'MAXLOGNAME' constant.])], + [], + [@%:@include <sys/param.h>]) + +AC_CHECK_DECLS([UT_NAMESIZE], + [AC_DEFINE([HAVE_UT_NAMESIZE], [1], + [Define if you have the 'HAVE_UT_NAMESIZE' constant.])], + [], + [@%:@include <utmp.h>]) + # check for openpty, login_tty, and forkpty AC_CHECK_FUNCS([openpty], [], @@ -5687,6 +5839,24 @@ AC_CHECK_MEMBERS([struct passwd.pw_gecos, struct passwd.pw_passwd], [], [], [[ # Issue #21085: In Cygwin, siginfo_t does not have si_band field. AC_CHECK_MEMBERS([siginfo_t.si_band], [], [], [[@%:@include <signal.h>]]) +if test "$ac_cv_func_statx" = yes; then + # Some systems have the definitions of the mask bits without having the + # corresponding members in struct statx. Check for members added after Linux + # 4.11 (when statx itself was added). + AC_CHECK_MEMBERS([struct statx.stx_mnt_id]) + AC_CHECK_MEMBERS([struct statx.stx_dio_mem_align]) + # stx_dio_offset_align was added together with stx_dio_mem_align + AC_CHECK_MEMBERS([struct statx.stx_subvol]) + AC_CHECK_MEMBERS([struct statx.stx_atomic_write_unit_min]) + # stx_atomic_write_unit_max and stx_atomic_write_segments_max were added + # together with stx_atomic_write_unit_min + AC_CHECK_MEMBERS([struct statx.stx_dio_read_offset_align]) + # stx_atomic_write_unit_max_opt was added in Linux 6.16, but is controlled by + # the STATX_WRITE_ATOMIC mask bit added in Linux 6.11, so having the mask bit + # doesn't imply having the member. + AC_CHECK_MEMBERS([struct statx.stx_atomic_write_unit_max_opt]) +fi + AC_CACHE_CHECK([for time.h that defines altzone], [ac_cv_header_time_altzone], [ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <time.h>]], [[return altzone;]])], [ac_cv_header_time_altzone=yes], @@ -6674,57 +6844,6 @@ then [Define if you have struct stat.st_mtimensec]) fi -AC_CACHE_CHECK([whether year with century should be normalized for strftime], [ac_cv_normalize_century], [ -AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include <time.h> -#include <string.h> - -int main(void) -{ - char year[5]; - struct tm date = { - .tm_year = -1801, - .tm_mon = 0, - .tm_mday = 1 - }; - if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { - return 1; - } - return 0; -} -]])], -[ac_cv_normalize_century=yes], -[ac_cv_normalize_century=no], -[ac_cv_normalize_century=yes])]) -if test "$ac_cv_normalize_century" = yes -then - AC_DEFINE([Py_NORMALIZE_CENTURY], [1], - [Define if year with century should be normalized for strftime.]) -fi - -AC_CACHE_CHECK([whether C99-compatible strftime specifiers are supported], [ac_cv_strftime_c99_support], [ -AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include <time.h> -#include <string.h> - -int main(void) -{ - char full_date[11]; - struct tm date = { - .tm_year = 0, - .tm_mon = 0, - .tm_mday = 1 - }; - if (strftime(full_date, sizeof(full_date), "%F", &date) && !strcmp(full_date, "1900-01-01")) { - return 0; - } - return 1; -} -]])], -[ac_cv_strftime_c99_support=yes], -[AC_MSG_ERROR([Python requires C99-compatible strftime specifiers])], -[ac_cv_strftime_c99_support=])]) - dnl check for ncursesw/ncurses and panelw/panel dnl NOTE: old curses is not detected. dnl have_curses=[no, yes] @@ -7025,13 +7144,13 @@ AC_ARG_WITH( [ if test "$withval" = yes then - AC_DEFINE([Py_TAIL_CALL_INTERP], [1], + AC_DEFINE([_Py_TAIL_CALL_INTERP], [1], [Define if you want to use tail-calling interpreters in CPython.]) AC_MSG_RESULT([yes]) fi if test "$withval" = no then - AC_DEFINE([Py_TAIL_CALL_INTERP], [0], + AC_DEFINE([_Py_TAIL_CALL_INTERP], [0], [Define if you want to use tail-calling interpreters in CPython.]) AC_MSG_RESULT([no]) fi @@ -7053,8 +7172,6 @@ if test "$with_remote_debug" = yes; then [Define if you want to enable remote debugging support.]) AC_MSG_RESULT([yes]) else - AC_DEFINE([Py_REMOTE_DEBUG], [0], - [Define if you want to enable remote debugging support.]) AC_MSG_RESULT([no]) fi @@ -7082,6 +7199,7 @@ SRCDIRS="\ Modules/_hacl \ Modules/_io \ Modules/_multiprocessing \ + Modules/_remote_debugging \ Modules/_sqlite \ Modules/_sre \ Modules/_testcapi \ @@ -7425,8 +7543,8 @@ WITH_SAVE_ENV([ ], [ OBJ_nid2sn(NID_md5); OBJ_nid2sn(NID_sha1); + OBJ_nid2sn(NID_sha512); OBJ_nid2sn(NID_sha3_512); - OBJ_nid2sn(NID_blake2b512); EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); ])], [ac_cv_working_openssl_hashlib=yes], [ac_cv_working_openssl_hashlib=no]) ]) @@ -7662,9 +7780,7 @@ AS_CASE([$ac_sys_system], ) dnl fcntl, readline, and termios are not particularly useful in browsers. PY_STDLIB_MOD_SET_NA( - [fcntl], [readline], - [termios], ) ], [WASI], [ @@ -7779,6 +7895,7 @@ PY_STDLIB_MOD_SIMPLE([time], [], [$TIMEMODULE_LIB]) dnl always enabled extension modules PY_STDLIB_MOD_SIMPLE([array]) +PY_STDLIB_MOD_SIMPLE([_math_integer]) PY_STDLIB_MOD_SIMPLE([_asyncio]) PY_STDLIB_MOD_SIMPLE([_bisect]) PY_STDLIB_MOD_SIMPLE([_csv]) @@ -7894,6 +8011,15 @@ AC_SUBST([LIBHACL_CFLAGS]) LIBHACL_LDFLAGS= # for now, no specific linker flags are needed AC_SUBST([LIBHACL_LDFLAGS]) +dnl Check if universal2 HACL* implementation should be used. +if test "$UNIVERSAL_ARCHS" = "universal2" -o \ + \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \) +then + use_hacl_universal2_impl=yes +else + use_hacl_universal2_impl=no +fi + # The SIMD files use aligned_alloc, which is not available on older versions of # Android. # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. @@ -7904,13 +8030,14 @@ then AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"] - AC_DEFINE([HACL_CAN_COMPILE_SIMD128], [1], [HACL* library can compile SIMD128 implementations]) + AC_DEFINE([_Py_HACL_CAN_COMPILE_VEC128], [1], [ + HACL* library can compile SIMD128 implementations]) # macOS universal2 builds *support* the -msse etc flags because they're # available on x86_64. However, performance of the HACL SIMD128 implementation # isn't great, so it's disabled on ARM64. AC_MSG_CHECKING([for HACL* SIMD128 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then [LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"] AC_MSG_RESULT([universal2]) else @@ -7935,14 +8062,15 @@ if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || \ then AX_CHECK_COMPILE_FLAG([-mavx2],[ [LIBHACL_SIMD256_FLAGS="-mavx2"] - AC_DEFINE([HACL_CAN_COMPILE_SIMD256], [1], [HACL* library can compile SIMD256 implementations]) + AC_DEFINE([_Py_HACL_CAN_COMPILE_VEC256], [1], [ + HACL* library can compile SIMD256 implementations]) # macOS universal2 builds *support* the -mavx2 compiler flag because it's # available on x86_64; but the HACL SIMD256 build then fails because the # implementation requires symbols that aren't available on ARM64. Use a # wrapped implementation if we're building for universal2. AC_MSG_CHECKING([for HACL* SIMD256 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then [LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"] AC_MSG_RESULT([universal2]) else @@ -8012,6 +8140,16 @@ PY_STDLIB_MOD([_curses_panel], PY_STDLIB_MOD([_decimal], [], [test "$have_mpdec" = "yes"], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) + +AS_VAR_IF([with_system_libmpdec], [no], + [AC_MSG_WARN([m4_normalize([ + the bundled copy of libmpdec is scheduled for removal in Python 3.16; + consider using a system installed mpdecimal library.])])]) +AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], + [AC_MSG_WARN([m4_normalize([ + no system libmpdec found; falling back to pure-Python version + for the decimal module])])]) + PY_STDLIB_MOD([_dbm], [test -n "$with_dbmliborder"], [test "$have_dbm" != "no"], [$DBM_CFLAGS], [$DBM_LIBS]) @@ -8074,6 +8212,36 @@ dnl Emscripten does not support shared libraries yet. PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +# Determine JIT stencils header files based on target platform +JIT_STENCILS_H="" +AS_VAR_IF([enable_experimental_jit], [no], + [], + [case "$host" in + aarch64-apple-darwin*) + JIT_STENCILS_H="jit_stencils-aarch64-apple-darwin.h" + ;; + x86_64-apple-darwin*) + JIT_STENCILS_H="jit_stencils-x86_64-apple-darwin.h" + ;; + aarch64-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-aarch64-pc-windows-msvc.h" + ;; + i686-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-i686-pc-windows-msvc.h" + ;; + x86_64-pc-windows-msvc) + JIT_STENCILS_H="jit_stencils-x86_64-pc-windows-msvc.h" + ;; + aarch64-*-linux-gnu) + JIT_STENCILS_H="jit_stencils-aarch64-unknown-linux-gnu.h" + ;; + x86_64-*-linux-gnu) + JIT_STENCILS_H="jit_stencils-x86_64-unknown-linux-gnu.h" + ;; + esac]) + +AC_SUBST([JIT_STENCILS_H]) + # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) diff --git a/iOS/README.rst b/iOS/README.rst deleted file mode 100644 index 13b88514493..00000000000 --- a/iOS/README.rst +++ /dev/null @@ -1,375 +0,0 @@ -==================== -Python on iOS README -==================== - -:Authors: - Russell Keith-Magee (2023-11) - -This document provides a quick overview of some iOS specific features in the -Python distribution. - -These instructions are only needed if you're planning to compile Python for iOS -yourself. Most users should *not* need to do this. If you're looking to -experiment with writing an iOS app in Python, tools such as `BeeWare's Briefcase -<https://briefcase.readthedocs.io>`__ and `Kivy's Buildozer -<https://buildozer.readthedocs.io>`__ will provide a much more approachable -user experience. - -Compilers for building on iOS -============================= - -Building for iOS requires the use of Apple's Xcode tooling. It is strongly -recommended that you use the most recent stable release of Xcode. This will -require the use of the most (or second-most) recently released macOS version, -as Apple does not maintain Xcode for older macOS versions. The Xcode Command -Line Tools are not sufficient for iOS development; you need a *full* Xcode -install. - -If you want to run your code on the iOS simulator, you'll also need to install -an iOS Simulator Platform. You should be prompted to select an iOS Simulator -Platform when you first run Xcode. Alternatively, you can add an iOS Simulator -Platform by selecting an open the Platforms tab of the Xcode Settings panel. - -iOS specific arguments to configure -=================================== - -* ``--enable-framework[=DIR]`` - - This argument specifies the location where the Python.framework will be - installed. If ``DIR`` is not specified, the framework will be installed into - a subdirectory of the ``iOS/Frameworks`` folder. - - This argument *must* be provided when configuring iOS builds. iOS does not - support non-framework builds. - -* ``--with-framework-name=NAME`` - - Specify the name for the Python framework; defaults to ``Python``. - - .. admonition:: Use this option with care! - - Unless you know what you're doing, changing the name of the Python - framework on iOS is not advised. If you use this option, you won't be able - to run the ``make testios`` target without making significant manual - alterations, and you won't be able to use any binary packages unless you - compile them yourself using your own framework name. - -Building Python on iOS -====================== - -ABIs and Architectures ----------------------- - -iOS apps can be deployed on physical devices, and on the iOS simulator. Although -the API used on these devices is identical, the ABI is different - you need to -link against different libraries for an iOS device build (``iphoneos``) or an -iOS simulator build (``iphonesimulator``). - -Apple uses the ``XCframework`` format to allow specifying a single dependency -that supports multiple ABIs. An ``XCframework`` is a wrapper around multiple -ABI-specific frameworks that share a common API. - -iOS can also support different CPU architectures within each ABI. At present, -there is only a single supported architecture on physical devices - ARM64. -However, the *simulator* supports 2 architectures - ARM64 (for running on Apple -Silicon machines), and x86_64 (for running on older Intel-based machines). - -To support multiple CPU architectures on a single platform, Apple uses a "fat -binary" format - a single physical file that contains support for multiple -architectures. It is possible to compile and use a "thin" single architecture -version of a binary for testing purposes; however, the "thin" binary will not be -portable to machines using other architectures. - -Building a single-architecture framework ----------------------------------------- - -The Python build system will create a ``Python.framework`` that supports a -*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a -framework to contain non-library content, so the iOS build will produce a -``bin`` and ``lib`` folder in the same output folder as ``Python.framework``. -The ``lib`` folder will be needed at runtime to support the Python library. - -If you want to use Python in a real iOS project, you need to produce multiple -``Python.framework`` builds, one for each ABI and architecture. iOS builds of -Python *must* be constructed as framework builds. To support this, you must -provide the ``--enable-framework`` flag when configuring the build. The build -also requires the use of cross-compilation. The minimal commands for building -Python for the ARM64 iOS simulator will look something like:: - - $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" - $ ./configure \ - --enable-framework \ - --host=arm64-apple-ios-simulator \ - --build=arm64-apple-darwin \ - --with-build-python=/path/to/python.exe - $ make - $ make install - -In this invocation: - -* ``iOS/Resources/bin`` has been added to the path, providing some shims for the - compilers and linkers needed by the build. Xcode requires the use of ``xcrun`` - to invoke compiler tooling. However, if ``xcrun`` is pre-evaluated and the - result passed to ``configure``, these results can embed user- and - version-specific paths into the sysconfig data, which limits the portability - of the compiled Python. Alternatively, if ``xcrun`` is used *as* the compiler, - it requires that compiler variables like ``CC`` include spaces, which can - cause significant problems with many C configuration systems which assume that - ``CC`` will be a single executable. - - To work around this problem, the ``iOS/Resources/bin`` folder contains some - wrapper scripts that present as simple compilers and linkers, but wrap - underlying calls to ``xcrun``. This allows configure to use a ``CC`` - definition without spaces, and without user- or version-specific paths, while - retaining the ability to adapt to the local Xcode install. These scripts are - included in the ``bin`` directory of an iOS install. - - These scripts will, by default, use the currently active Xcode installation. - If you want to use a different Xcode installation, you can use - ``xcode-select`` to set a new default Xcode globally, or you can use the - ``DEVELOPER_DIR`` environment variable to specify an Xcode install. The - scripts will use the default ``iphoneos``/``iphonesimulator`` SDK version for - the select Xcode install; if you want to use a different SDK, you can set the - ``IOS_SDK_VERSION`` environment variable. (e.g, setting - ``IOS_SDK_VERSION=17.1`` would cause the scripts to use the ``iphoneos17.1`` - and ``iphonesimulator17.1`` SDKs, regardless of the Xcode default.) - - The path has also been cleared of any user customizations. A common source of - bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS - build. Resetting the path to a known "bare bones" value is the easiest way to - avoid these problems. - -* ``--host`` is the architecture and ABI that you want to build, in GNU compiler - triple format. This will be one of: - - - ``arm64-apple-ios`` for ARM64 iOS devices. - - ``arm64-apple-ios-simulator`` for the iOS simulator running on Apple - Silicon devices. - - ``x86_64-apple-ios-simulator`` for the iOS simulator running on Intel - devices. - -* ``--build`` is the GNU compiler triple for the machine that will be running - the compiler. This is one of: - - - ``arm64-apple-darwin`` for Apple Silicon devices. - - ``x86_64-apple-darwin`` for Intel devices. - -* ``/path/to/python.exe`` is the path to a Python binary on the machine that - will be running the compiler. This is needed because the Python compilation - process involves running some Python code. On a normal desktop build of - Python, you can compile a python interpreter and then use that interpreter to - run Python code. However, the binaries produced for iOS won't run on macOS, so - you need to provide an external Python interpreter. This interpreter must be - the same version as the Python that is being compiled. To be completely safe, - this should be the *exact* same commit hash. However, the longer a Python - release has been stable, the more likely it is that this constraint can be - relaxed - the same micro version will often be sufficient. - -* The ``install`` target for iOS builds is slightly different to other - platforms. On most platforms, ``make install`` will install the build into - the final runtime location. This won't be the case for iOS, as the final - runtime location will be on a physical device. - - However, you still need to run the ``install`` target for iOS builds, as it - performs some final framework assembly steps. The location specified with - ``--enable-framework`` will be the location where ``make install`` will - assemble the complete iOS framework. This completed framework can then - be copied and relocated as required. - -For a full CPython build, you also need to specify the paths to iOS builds of -the binary libraries that CPython depends on (XZ, BZip2, LibFFI and OpenSSL). -This can be done by defining the ``LIBLZMA_CFLAGS``, ``LIBLZMA_LIBS``, -``BZIP2_CFLAGS``, ``BZIP2_LIBS``, ``LIBFFI_CFLAGS``, and ``LIBFFI_LIBS`` -environment variables, and the ``--with-openssl`` configure option. Versions of -these libraries pre-compiled for iOS can be found in `this repository -<https://github.com/beeware/cpython-apple-source-deps/releases>`__. LibFFI is -especially important, as many parts of the standard library (including the -``platform``, ``sysconfig`` and ``webbrowser`` modules) require the use of the -``ctypes`` module at runtime. - -By default, Python will be compiled with an iOS deployment target (i.e., the -minimum supported iOS version) of 13.0. To specify a different deployment -target, provide the version number as part of the ``--host`` argument - for -example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 -simulator build with a deployment target of 15.4. - -Merge thin frameworks into fat frameworks ------------------------------------------ - -Once you've built a ``Python.framework`` for each ABI and and architecture, you -must produce a "fat" framework for each ABI that contains all the architectures -for that ABI. - -The ``iphoneos`` build only needs to support a single architecture, so it can be -used without modification. - -If you only want to support a single simulator architecture, (e.g., only support -ARM64 simulators), you can use a single architecture ``Python.framework`` build. -However, if you want to create ``Python.xcframework`` that supports *all* -architectures, you'll need to merge the ``iphonesimulator`` builds for ARM64 and -x86_64 into a single "fat" framework. - -The "fat" framework can be constructed by performing a directory merge of the -content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and -``lib`` folders for each thin framework. When performing this merge: - -* The pure Python standard library content is identical for each architecture, - except for a handful of platform-specific files (such as the ``sysconfig`` - module). Ensure that the "fat" framework has the union of all standard library - files. - -* Any binary files in the standard library, plus the main - ``libPython3.X.dylib``, can be merged using the ``lipo`` tool, provide by - Xcode:: - - $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib - -* The header files will be identical on both architectures, except for - ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename - ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the - other architecture into the merged header folder as ``pyconfig-x86_64.h``. - Then copy the ``iOS/Resources/pyconfig.h`` file from the CPython sources into - the merged headers folder. This will allow the two Python architectures to - share a common ``pyconfig.h`` header file. - -At this point, you should have 2 Python.framework folders - one for ``iphoneos``, -and one for ``iphonesimulator`` that is a merge of x86+64 and ARM64 content. - -Merge frameworks into an XCframework ------------------------------------- - -Now that we have 2 (potentially fat) ABI-specific frameworks, we can merge those -frameworks into a single ``XCframework``. - -The initial skeleton of an ``XCframework`` is built using:: - - xcodebuild -create-xcframework -output Python.xcframework -framework path/to/iphoneos/Python.framework -framework path/to/iphonesimulator/Python.framework - -Then, copy the ``bin`` and ``lib`` folders into the architecture-specific slices of -the XCframework:: - - cp path/to/iphoneos/bin Python.xcframework/ios-arm64 - cp path/to/iphoneos/lib Python.xcframework/ios-arm64 - - cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86_64-simulator - cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86_64-simulator - -Note that the name of the architecture-specific slice for the simulator will -depend on the CPU architecture(s) that you build. - -You now have a Python.xcframework that can be used in a project. - -Testing Python on iOS -===================== - -The ``iOS/testbed`` folder that contains an Xcode project that is able to run -the iOS test suite. This project converts the Python test suite into a single -test case in Xcode's XCTest framework. The single XCTest passes if the test -suite passes. - -To run the test suite, configure a Python build for an iOS simulator (i.e., -``--host=arm64-apple-ios-simulator`` or ``--host=x86_64-apple-ios-simulator`` -), specifying a framework build (i.e. ``--enable-framework``). Ensure that your -``PATH`` has been configured to include the ``iOS/Resources/bin`` folder and -exclude any non-iOS tools, then run:: - - $ make all - $ make install - $ make testios - -This will: - -* Build an iOS framework for your chosen architecture; -* Finalize the single-platform framework; -* Make a clean copy of the testbed project; -* Install the Python iOS framework into the copy of the testbed project; and -* Run the test suite on an "iPhone SE (3rd generation)" simulator. - -On success, the test suite will exit and report successful completion of the -test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 -minutes to run; a couple of extra minutes is required to compile the testbed -project, and then boot and prepare the iOS simulator. - -Debugging test failures ------------------------ - -Running ``make test`` generates a standalone version of the ``iOS/testbed`` -project, and runs the full test suite. It does this using ``iOS/testbed`` -itself - the folder is an executable module that can be used to create and run -a clone of the testbed project. - -You can generate your own standalone testbed instance by running:: - - $ python iOS/testbed clone --framework iOS/Frameworks/arm64-iphonesimulator my-testbed - -This invocation assumes that ``iOS/Frameworks/arm64-iphonesimulator`` is the -path to the iOS simulator framework for your platform (ARM64 in this case); -``my-testbed`` is the name of the folder for the new testbed clone. - -You can then use the ``my-testbed`` folder to run the Python test suite, -passing in any command line arguments you may require. For example, if you're -trying to diagnose a failure in the ``os`` module, you might run:: - - $ python my-testbed run -- test -W test_os - -This is the equivalent of running ``python -m test -W test_os`` on a desktop -Python build. Any arguments after the ``--`` will be passed to testbed as if -they were arguments to ``python -m`` on a desktop machine. - -You can also open the testbed project in Xcode by running:: - - $ open my-testbed/iOSTestbed.xcodeproj - -This will allow you to use the full Xcode suite of tools for debugging. - -Testing on an iOS device -^^^^^^^^^^^^^^^^^^^^^^^^ - -To test on an iOS device, the app needs to be signed with known developer -credentials. To obtain these credentials, you must have an iOS Developer -account, and your Xcode install will need to be logged into your account (see -the Accounts tab of the Preferences dialog). - -Once the project is open, and you're signed into your Apple Developer account, -select the root node of the project tree (labeled "iOSTestbed"), then the -"Signing & Capabilities" tab in the details page. Select a development team -(this will likely be your own name), and plug in a physical device to your -macOS machine with a USB cable. You should then be able to select your physical -device from the list of targets in the pulldown in the Xcode titlebar. - -Running specific tests -^^^^^^^^^^^^^^^^^^^^^^ - -As the test suite is being executed on an iOS simulator, it is not possible to -pass in command line arguments to configure test suite operation. To work -around this limitation, the arguments that would normally be passed as command -line arguments are configured as part of the ``iOSTestbed-Info.plist`` file -that is used to configure the iOS testbed app. In this file, the ``TestArgs`` -key is an array containing the arguments that would be passed to ``python -m`` -on the command line (including ``test`` in position 0, the name of the test -module to be executed). - -Disabling automated breakpoints -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, Xcode will inserts an automatic breakpoint whenever a signal is -raised. The Python test suite raises many of these signals as part of normal -operation; unless you are trying to diagnose an issue with signals, the -automatic breakpoints can be inconvenient. However, they can be disabled by -creating a symbolic breakpoint that is triggered at the start of the test run. - -Select "Debug > Breakpoints > Create Symbolic Breakpoint" from the Xcode menu, and -populate the new brewpoint with the following details: - -* **Name**: IgnoreSignals -* **Symbol**: UIApplicationMain -* **Action**: Add debugger commands for: - - ``process handle SIGINT -n true -p true -s false`` - - ``process handle SIGUSR1 -n true -p true -s false`` - - ``process handle SIGUSR2 -n true -p true -s false`` - - ``process handle SIGXFSZ -n true -p true -s false`` -* Check the "Automatically continue after evaluating" box. - -All other details can be left blank. When the process executes the -``UIApplicationMain`` entry point, the breakpoint will trigger, run the debugger -commands to disable the automatic breakpoints, and automatically resume. diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-strip b/iOS/Resources/bin/arm64-apple-ios-simulator-strip new file mode 100755 index 00000000000..fd59d309b73 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-strip b/iOS/Resources/bin/arm64-apple-ios-strip new file mode 100755 index 00000000000..75e823a3d02 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-strip b/iOS/Resources/bin/x86_64-apple-ios-simulator-strip new file mode 100755 index 00000000000..c5cfb289291 --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch x86_64 "$@" diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py deleted file mode 100644 index c05497ede3a..00000000000 --- a/iOS/testbed/__main__.py +++ /dev/null @@ -1,548 +0,0 @@ -import argparse -import asyncio -import fcntl -import json -import os -import plistlib -import re -import shutil -import subprocess -import sys -import tempfile -from contextlib import asynccontextmanager -from datetime import datetime -from pathlib import Path - - -DECODE_ARGS = ("UTF-8", "backslashreplace") - -# The system log prefixes each line: -# 2025-01-17 16:14:29.090 Df iOSTestbed[23987:1fd393b4] (Python) ... -# 2025-01-17 16:14:29.090 E iOSTestbed[23987:1fd393b4] (Python) ... - -LOG_PREFIX_REGEX = re.compile( - r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD - r"\s+\d+:\d{2}:\d{2}\.\d+" # HH:MM:SS.sss - r"\s+\w+" # Df/E - r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID - r"\s+\(Python\)\s" # Logger name -) - - -# Work around a bug involving sys.exit and TaskGroups -# (https://github.com/python/cpython/issues/101515). -def exit(*args): - raise MySystemExit(*args) - - -class MySystemExit(Exception): - pass - - -class SimulatorLock: - # An fcntl-based filesystem lock that can be used to ensure that - def __init__(self, timeout): - self.filename = Path(tempfile.gettempdir()) / "python-ios-testbed" - self.timeout = timeout - - self.fd = None - - async def acquire(self): - # Ensure the lockfile exists - self.filename.touch(exist_ok=True) - - # Try `timeout` times to acquire the lock file, with a 1 second pause - # between each attempt. Report status every 10 seconds. - for i in range(0, self.timeout): - try: - fd = os.open(self.filename, os.O_RDWR | os.O_TRUNC, 0o644) - fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - os.close(fd) - if i % 10 == 0: - print("... waiting", flush=True) - await asyncio.sleep(1) - else: - self.fd = fd - return - - # If we reach the end of the loop, we've exceeded the allowed number of - # attempts. - raise ValueError("Unable to obtain lock on iOS simulator creation") - - def release(self): - # If a lock is held, release it. - if self.fd is not None: - # Release the lock. - fcntl.flock(self.fd, fcntl.LOCK_UN) - os.close(self.fd) - self.fd = None - - -# All subprocesses are executed through this context manager so that no matter -# what happens, they can always be cancelled from another task, and they will -# always be cleaned up on exit. -@asynccontextmanager -async def async_process(*args, **kwargs): - process = await asyncio.create_subprocess_exec(*args, **kwargs) - try: - yield process - finally: - if process.returncode is None: - # Allow a reasonably long time for Xcode to clean itself up, - # because we don't want stale emulators left behind. - timeout = 10 - process.terminate() - try: - await asyncio.wait_for(process.wait(), timeout) - except TimeoutError: - print( - f"Command {args} did not terminate after {timeout} seconds " - f" - sending SIGKILL" - ) - process.kill() - - # Even after killing the process we must still wait for it, - # otherwise we'll get the warning "Exception ignored in __del__". - await asyncio.wait_for(process.wait(), timeout=1) - - -async def async_check_output(*args, **kwargs): - async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) as process: - stdout, stderr = await process.communicate() - if process.returncode == 0: - return stdout.decode(*DECODE_ARGS) - else: - raise subprocess.CalledProcessError( - process.returncode, - args, - stdout.decode(*DECODE_ARGS), - stderr.decode(*DECODE_ARGS), - ) - - -# Select a simulator device to use. -async def select_simulator_device(): - # List the testing simulators, in JSON format - raw_json = await async_check_output( - "xcrun", "simctl", "--set", "testing", "list", "-j" - ) - json_data = json.loads(raw_json) - - # Any device will do; we'll look for "SE" devices - but the name isn't - # consistent over time. Older Xcode versions will use "iPhone SE (Nth - # generation)"; As of 2025, they've started using "iPhone 16e". - # - # When Xcode is updated after a new release, new devices will be available - # and old ones will be dropped from the set available on the latest iOS - # version. Select the one with the highest minimum runtime version - this - # is an indicator of the "newest" released device, which should always be - # supported on the "most recent" iOS version. - se_simulators = sorted( - (devicetype["minRuntimeVersion"], devicetype["name"]) - for devicetype in json_data["devicetypes"] - if devicetype["productFamily"] == "iPhone" - and ( - ("iPhone " in devicetype["name"] and devicetype["name"].endswith("e")) - or "iPhone SE " in devicetype["name"] - ) - ) - - return se_simulators[-1][1] - - -# Return a list of UDIDs associated with booted simulators -async def list_devices(): - try: - # List the testing simulators, in JSON format - raw_json = await async_check_output( - "xcrun", "simctl", "--set", "testing", "list", "-j" - ) - json_data = json.loads(raw_json) - - # Filter out the booted iOS simulators - return [ - simulator["udid"] - for runtime, simulators in json_data["devices"].items() - for simulator in simulators - if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted" - ] - except subprocess.CalledProcessError as e: - # If there's no ~/Library/Developer/XCTestDevices folder (which is the - # case on fresh installs, and in some CI environments), `simctl list` - # returns error code 1, rather than an empty list. Handle that case, - # but raise all other errors. - if e.returncode == 1: - return [] - else: - raise - - -async def find_device(initial_devices, lock): - while True: - new_devices = set(await list_devices()).difference(initial_devices) - if len(new_devices) == 0: - await asyncio.sleep(1) - elif len(new_devices) == 1: - udid = new_devices.pop() - print(f"{datetime.now():%Y-%m-%d %H:%M:%S}: New test simulator detected") - print(f"UDID: {udid}", flush=True) - lock.release() - return udid - else: - exit(f"Found more than one new device: {new_devices}") - - -async def log_stream_task(initial_devices, lock): - # Wait up to 5 minutes for the build to complete and the simulator to boot. - udid = await asyncio.wait_for(find_device(initial_devices, lock), 5 * 60) - - # Stream the iOS device's logs, filtering out messages that come from the - # XCTest test suite (catching NSLog messages from the test method), or - # Python itself (catching stdout/stderr content routed to the system log - # with config->use_system_logger). - args = [ - "xcrun", - "simctl", - "--set", - "testing", - "spawn", - udid, - "log", - "stream", - "--style", - "compact", - "--predicate", - ( - 'senderImagePath ENDSWITH "/iOSTestbedTests.xctest/iOSTestbedTests"' - ' OR senderImagePath ENDSWITH "/Python.framework/Python"' - ), - ] - - async with async_process( - *args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) as process: - suppress_dupes = False - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - # Strip the prefix from each log line - line = LOG_PREFIX_REGEX.sub("", line) - # The iOS log streamer can sometimes lag; when it does, it outputs - # a warning about messages being dropped... often multiple times. - # Only print the first of these duplicated warnings. - if line.startswith("=== Messages dropped "): - if not suppress_dupes: - suppress_dupes = True - sys.stdout.write(line) - else: - suppress_dupes = False - sys.stdout.write(line) - sys.stdout.flush() - - -async def xcode_test(location, simulator, verbose): - # Run the test suite on the named simulator - print("Starting xcodebuild...", flush=True) - args = [ - "xcodebuild", - "test", - "-project", - str(location / "iOSTestbed.xcodeproj"), - "-scheme", - "iOSTestbed", - "-destination", - f"platform=iOS Simulator,name={simulator}", - "-resultBundlePath", - str(location / f"{datetime.now():%Y%m%d-%H%M%S}.xcresult"), - "-derivedDataPath", - str(location / "DerivedData"), - ] - if not verbose: - args += ["-quiet"] - - async with async_process( - *args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - sys.stdout.write(line) - sys.stdout.flush() - - status = await asyncio.wait_for(process.wait(), timeout=1) - exit(status) - - -def clone_testbed( - source: Path, - target: Path, - framework: Path, - apps: list[Path], -) -> None: - if target.exists(): - print(f"{target} already exists; aborting without creating project.") - sys.exit(10) - - if framework is None: - if not ( - source / "Python.xcframework/ios-arm64_x86_64-simulator/bin" - ).is_dir(): - print( - f"The testbed being cloned ({source}) does not contain " - f"a simulator framework. Re-run with --framework" - ) - sys.exit(11) - else: - if not framework.is_dir(): - print(f"{framework} does not exist.") - sys.exit(12) - elif not ( - framework.suffix == ".xcframework" - or (framework / "Python.framework").is_dir() - ): - print( - f"{framework} is not an XCframework, " - f"or a simulator slice of a framework build." - ) - sys.exit(13) - - print("Cloning testbed project:") - print(f" Cloning {source}...", end="", flush=True) - shutil.copytree(source, target, symlinks=True) - print(" done") - - xc_framework_path = target / "Python.xcframework" - sim_framework_path = xc_framework_path / "ios-arm64_x86_64-simulator" - if framework is not None: - if framework.suffix == ".xcframework": - print(" Installing XCFramework...", end="", flush=True) - if xc_framework_path.is_dir(): - shutil.rmtree(xc_framework_path) - else: - xc_framework_path.unlink(missing_ok=True) - xc_framework_path.symlink_to( - framework.relative_to(xc_framework_path.parent, walk_up=True) - ) - print(" done") - else: - print(" Installing simulator framework...", end="", flush=True) - if sim_framework_path.is_dir(): - shutil.rmtree(sim_framework_path) - else: - sim_framework_path.unlink(missing_ok=True) - sim_framework_path.symlink_to( - framework.relative_to(sim_framework_path.parent, walk_up=True) - ) - print(" done") - else: - if ( - xc_framework_path.is_symlink() - and not xc_framework_path.readlink().is_absolute() - ): - # XCFramework is a relative symlink. Rewrite the symlink relative - # to the new location. - print(" Rewriting symlink to XCframework...", end="", flush=True) - orig_xc_framework_path = ( - source - / xc_framework_path.readlink() - ).resolve() - xc_framework_path.unlink() - xc_framework_path.symlink_to( - orig_xc_framework_path.relative_to( - xc_framework_path.parent, walk_up=True - ) - ) - print(" done") - elif ( - sim_framework_path.is_symlink() - and not sim_framework_path.readlink().is_absolute() - ): - print(" Rewriting symlink to simulator framework...", end="", flush=True) - # Simulator framework is a relative symlink. Rewrite the symlink - # relative to the new location. - orig_sim_framework_path = ( - source - / "Python.XCframework" - / sim_framework_path.readlink() - ).resolve() - sim_framework_path.unlink() - sim_framework_path.symlink_to( - orig_sim_framework_path.relative_to( - sim_framework_path.parent, walk_up=True - ) - ) - print(" done") - else: - print(" Using pre-existing iOS framework.") - - for app_src in apps: - print(f" Installing app {app_src.name!r}...", end="", flush=True) - app_target = target / f"iOSTestbed/app/{app_src.name}" - if app_target.is_dir(): - shutil.rmtree(app_target) - shutil.copytree(app_src, app_target) - print(" done") - - print(f"Successfully cloned testbed: {target.resolve()}") - - -def update_plist(testbed_path, args): - # Add the test runner arguments to the testbed's Info.plist file. - info_plist = testbed_path / "iOSTestbed" / "iOSTestbed-Info.plist" - with info_plist.open("rb") as f: - info = plistlib.load(f) - - info["TestArgs"] = args - - with info_plist.open("wb") as f: - plistlib.dump(info, f) - - -async def run_testbed(simulator: str | None, args: list[str], verbose: bool=False): - location = Path(__file__).parent - print("Updating plist...", end="", flush=True) - update_plist(location, args) - print(" done.", flush=True) - - if simulator is None: - simulator = await select_simulator_device() - print(f"Running test on {simulator}", flush=True) - - # We need to get an exclusive lock on simulator creation, to avoid issues - # with multiple simulators starting and being unable to tell which - # simulator is due to which testbed instance. See - # https://github.com/python/cpython/issues/130294 for details. Wait up to - # 10 minutes for a simulator to boot. - print("Obtaining lock on simulator creation...", flush=True) - simulator_lock = SimulatorLock(timeout=10*60) - await simulator_lock.acquire() - print("Simulator lock acquired.", flush=True) - - # Get the list of devices that are booted at the start of the test run. - # The simulator started by the test suite will be detected as the new - # entry that appears on the device list. - initial_devices = await list_devices() - - try: - async with asyncio.TaskGroup() as tg: - tg.create_task(log_stream_task(initial_devices, simulator_lock)) - tg.create_task(xcode_test(location, simulator=simulator, verbose=verbose)) - except* MySystemExit as e: - raise SystemExit(*e.exceptions[0].args) from None - except* subprocess.CalledProcessError as e: - # Extract it from the ExceptionGroup so it can be handled by `main`. - raise e.exceptions[0] - finally: - simulator_lock.release() - - -def main(): - parser = argparse.ArgumentParser( - description=( - "Manages the process of testing a Python project in the iOS simulator." - ), - ) - - subcommands = parser.add_subparsers(dest="subcommand") - - clone = subcommands.add_parser( - "clone", - description=( - "Clone the testbed project, copying in an iOS Python framework and" - "any specified application code." - ), - help="Clone a testbed project to a new location.", - ) - clone.add_argument( - "--framework", - help=( - "The location of the XCFramework (or simulator-only slice of an " - "XCFramework) to use when running the testbed" - ), - ) - clone.add_argument( - "--app", - dest="apps", - action="append", - default=[], - help="The location of any code to include in the testbed project", - ) - clone.add_argument( - "location", - help="The path where the testbed will be cloned.", - ) - - run = subcommands.add_parser( - "run", - usage="%(prog)s [-h] [--simulator SIMULATOR] -- <test arg> [<test arg> ...]", - description=( - "Run a testbed project. The arguments provided after `--` will be " - "passed to the running iOS process as if they were arguments to " - "`python -m`." - ), - help="Run a testbed project", - ) - run.add_argument( - "--simulator", - help=( - "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to ", - "the most recently released 'entry level' iPhone device." - ) - ) - run.add_argument( - "-v", "--verbose", - action="store_true", - help="Enable verbose output", - ) - - try: - pos = sys.argv.index("--") - testbed_args = sys.argv[1:pos] - test_args = sys.argv[pos + 1 :] - except ValueError: - testbed_args = sys.argv[1:] - test_args = [] - - context = parser.parse_args(testbed_args) - - if context.subcommand == "clone": - clone_testbed( - source=Path(__file__).parent.resolve(), - target=Path(context.location).resolve(), - framework=Path(context.framework).resolve() if context.framework else None, - apps=[Path(app) for app in context.apps], - ) - elif context.subcommand == "run": - if test_args: - if not ( - Path(__file__).parent / "Python.xcframework/ios-arm64_x86_64-simulator/bin" - ).is_dir(): - print( - f"Testbed does not contain a compiled iOS framework. Use " - f"`python {sys.argv[0]} clone ...` to create a runnable " - f"clone of this testbed." - ) - sys.exit(20) - - asyncio.run( - run_testbed( - simulator=context.simulator, - verbose=context.verbose, - args=test_args, - ) - ) - else: - print(f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)") - print() - parser.print_help(sys.stderr) - sys.exit(21) - else: - parser.print_help(sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/iOS/testbed/iOSTestbed/dylib-Info-template.plist b/iOS/testbed/iOSTestbed/dylib-Info-template.plist deleted file mode 100644 index f652e272f71..00000000000 --- a/iOS/testbed/iOSTestbed/dylib-Info-template.plist +++ /dev/null @@ -1,26 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>CFBundleDevelopmentRegion</key> - <string>en</string> - <key>CFBundleExecutable</key> - <string></string> - <key>CFBundleIdentifier</key> - <string></string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundlePackageType</key> - <string>APPL</string> - <key>CFBundleShortVersionString</key> - <string>1.0</string> - <key>CFBundleSupportedPlatforms</key> - <array> - <string>iPhoneOS</string> - </array> - <key>MinimumOSVersion</key> - <string>12.0</string> - <key>CFBundleVersion</key> - <string>1</string> -</dict> -</plist> diff --git a/pyconfig.h.in b/pyconfig.h.in index 7586ad3f266..8a9f5ca8ec8 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -50,12 +50,6 @@ /* Define if getpgrp() must be called as getpgrp(0). */ #undef GETPGRP_HAVE_ARG -/* HACL* library can compile SIMD128 implementations */ -#undef HACL_CAN_COMPILE_SIMD128 - -/* HACL* library can compile SIMD256 implementations */ -#undef HACL_CAN_COMPILE_SIMD256 - /* Define if you have the 'accept' function. */ #undef HAVE_ACCEPT @@ -147,6 +141,9 @@ /* Define if you have the 'chroot' function. */ #undef HAVE_CHROOT +/* Define to 1 if you have the 'clearenv' function. */ +#undef HAVE_CLEARENV + /* Define to 1 if you have the 'clock' function. */ #undef HAVE_CLOCK @@ -267,6 +264,10 @@ */ #undef HAVE_DECL_TZNAME +/* Define to 1 if you have the declaration of 'UT_NAMESIZE', and to 0 if you + don't. */ +#undef HAVE_DECL_UT_NAMESIZE + /* Define to 1 if you have the device macros. */ #undef HAVE_DEVICE_MACROS @@ -539,6 +540,9 @@ /* Define to 1 if you have the 'getlogin' function. */ #undef HAVE_GETLOGIN +/* Define to 1 if you have the 'getlogin_r' function. */ +#undef HAVE_GETLOGIN_R + /* Define to 1 if you have the 'getnameinfo' function. */ #undef HAVE_GETNAMEINFO @@ -726,6 +730,9 @@ /* Define to 1 if you have the <linux/can.h> header file. */ #undef HAVE_LINUX_CAN_H +/* Define to 1 if you have the <linux/can/isotp.h> header file. */ +#undef HAVE_LINUX_CAN_ISOTP_H + /* Define to 1 if you have the <linux/can/j1939.h> header file. */ #undef HAVE_LINUX_CAN_J1939_H @@ -807,6 +814,9 @@ /* Define this if you have the makedev macro. */ #undef HAVE_MAKEDEV +/* Define if you have the 'MAXLOGNAME' constant. */ +#undef HAVE_MAXLOGNAME + /* Define to 1 if you have the 'mbrtowc' function. */ #undef HAVE_MBRTOWC @@ -1275,6 +1285,9 @@ /* Define to 1 if you have the 'statvfs' function. */ #undef HAVE_STATVFS +/* Define to 1 if you have the 'statx' function. */ +#undef HAVE_STATX + /* Define if you have struct stat.st_mtim.tv_nsec */ #undef HAVE_STAT_TV_NSEC @@ -1317,6 +1330,27 @@ /* Define to 1 if 'pw_passwd' is a member of 'struct passwd'. */ #undef HAVE_STRUCT_PASSWD_PW_PASSWD +/* Define to 1 if 'stx_atomic_write_unit_max_opt' is a member of 'struct + statx'. */ +#undef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MAX_OPT + +/* Define to 1 if 'stx_atomic_write_unit_min' is a member of 'struct statx'. + */ +#undef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN + +/* Define to 1 if 'stx_dio_mem_align' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN + +/* Define to 1 if 'stx_dio_read_offset_align' is a member of 'struct statx'. + */ +#undef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN + +/* Define to 1 if 'stx_mnt_id' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_MNT_ID + +/* Define to 1 if 'stx_subvol' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_SUBVOL + /* Define to 1 if 'st_birthtime' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BIRTHTIME @@ -1575,6 +1609,9 @@ /* Define to 1 if you have the <utmp.h> header file. */ #undef HAVE_UTMP_H +/* Define if you have the 'HAVE_UT_NAMESIZE' constant. */ +#undef HAVE_UT_NAMESIZE + /* Define to 1 if you have the 'uuid_create' function. */ #undef HAVE_UUID_CREATE @@ -1584,6 +1621,9 @@ /* Define if uuid_generate_time_safe() exists. */ #undef HAVE_UUID_GENERATE_TIME_SAFE +/* Define if uuid_generate_time_safe() is able to deduce a MAC address. */ +#undef HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC + /* Define to 1 if you have the <uuid.h> header file. */ #undef HAVE_UUID_H @@ -1630,12 +1670,18 @@ /* Define to 1 if you have the 'writev' function. */ #undef HAVE_WRITEV +/* Define to 1 if you have the <zdict.h> header file. */ +#undef HAVE_ZDICT_H + /* Define if the zlib library has inflateCopy */ #undef HAVE_ZLIB_COPY /* Define to 1 if you have the <zlib.h> header file. */ #undef HAVE_ZLIB_H +/* Define to 1 if you have the <zstd.h> header file. */ +#undef HAVE_ZSTD_H + /* Define to 1 if you have the '_getpty' function. */ #undef HAVE__GETPTY @@ -1714,9 +1760,6 @@ /* Defined if Python is built as a shared library. */ #undef Py_ENABLE_SHARED -/* Defined if _Complex C type can be used with libffi. */ -#undef Py_FFI_SUPPORT_C_COMPLEX - /* Define if you want to disable the GIL */ #undef Py_GIL_DISABLED @@ -1724,9 +1767,6 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM -/* Define if year with century should be normalized for strftime. */ -#undef Py_NORMALIZE_CENTURY - /* Define if you want to enable remote debugging support. */ #undef Py_REMOTE_DEBUG @@ -1739,9 +1779,6 @@ /* The version of SunOS/Solaris as reported by `uname -r' without the dot. */ #undef Py_SUNOS_VERSION -/* Define if you want to use tail-calling interpreters in CPython. */ -#undef Py_TAIL_CALL_INTERP - /* Define if you want to enable tracing references for debugging purpose */ #undef Py_TRACE_REFS @@ -2004,6 +2041,21 @@ /* Maximum length in bytes of a thread name */ #undef _PYTHREAD_NAME_MAXLEN +/* Defined if _Complex C type can be used with libffi. */ +#undef _Py_FFI_SUPPORT_C_COMPLEX + +/* HACL* library can compile SIMD128 implementations */ +#undef _Py_HACL_CAN_COMPILE_VEC128 + +/* HACL* library can compile SIMD256 implementations */ +#undef _Py_HACL_CAN_COMPILE_VEC256 + +/* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ +#undef _Py_STACK_GROWS_DOWN + +/* Define if you want to use tail-calling interpreters in CPython. */ +#undef _Py_TAIL_CALL_INTERP + /* Define to force use of thread-safe errno, h_errno, and other functions */ #undef _REENTRANT